From 03359e9864bfb925f577fa5b16c3ef22884127aa Mon Sep 17 00:00:00 2001
From: rupertbaxter2 <rupertbaxter2@yahoo.com>
Date: Sun, 3 Aug 2014 07:34:04 -0700
Subject: [PATCH 0001/6219] Added --sleep-interval option

---
 youtube_dl/__init__.py          | 8 ++++++++
 youtube_dl/downloader/common.py | 3 +++
 youtube_dl/utils.py             | 3 +++
 3 files changed, 14 insertions(+)

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 429630ce5..2bd5ec33b 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -351,6 +351,8 @@ def parseOpts(overrideArguments=None):
 
     downloader.add_option('-r', '--rate-limit',
             dest='ratelimit', metavar='LIMIT', help='maximum download rate in bytes per second (e.g. 50K or 4.2M)')
+    downloader.add_option('--sleep-interval',
+            dest='sleepinterval', metavar='SLEEPINTERVAL', help='number of seconds to sleep between downloads (default is %default)', default="0")
     downloader.add_option('-R', '--retries',
             dest='retries', metavar='RETRIES', help='number of retries (default is %default)', default=10)
     downloader.add_option('--buffer-size',
@@ -671,6 +673,11 @@ def _real_main(argv=None):
         if numeric_limit is None:
             parser.error(u'invalid rate limit specified')
         opts.ratelimit = numeric_limit
+    if opts.sleepinterval is not None:
+        try:
+            opts.sleepinterval = abs(int(opts.sleepinterval))
+        except ValueError:
+            parser.error(u'invalid sleep interval specified')
     if opts.min_filesize is not None:
         numeric_limit = FileDownloader.parse_bytes(opts.min_filesize)
         if numeric_limit is None:
@@ -767,6 +774,7 @@ def _real_main(argv=None):
         'restrictfilenames': opts.restrictfilenames,
         'ignoreerrors': opts.ignoreerrors,
         'ratelimit': opts.ratelimit,
+        'sleepinterval': opts.sleepinterval,
         'nooverwrites': opts.nooverwrites,
         'retries': opts.retries,
         'buffersize': opts.buffersize,
diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index 917f3450e..8e0e386bf 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -278,6 +278,9 @@ class FileDownloader(object):
         """Download to a filename using the info from info_dict
         Return True on success and False otherwise
         """
+        sleep_interval = self.params.get('sleepinterval', 0)
+        self.to_screen(u'[download] Sleeping %d seconds...' %sleep_interval)
+        time.sleep(sleep_interval)
         # Check file already present
         if self.params.get('continuedl', False) and os.path.isfile(encodeFilename(filename)) and not self.params.get('nopart', False):
             self.report_file_already_downloaded(filename)
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index e40b367c2..d199d26d2 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -6,6 +6,7 @@ import codecs
 import contextlib
 import ctypes
 import datetime
+import time
 import email.utils
 import errno
 import getpass
@@ -747,6 +748,8 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
                 del req.headers['User-agent']
             req.headers['User-agent'] = req.headers['Youtubedl-user-agent']
             del req.headers['Youtubedl-user-agent']
+        #print("sleeping\n")
+        #time.sleep(1)
         return req
 
     def http_response(self, req, resp):

From 2f61fe4cccc1ef4186943f4eed2e89f8fe2e2c23 Mon Sep 17 00:00:00 2001
From: rupertbaxter2 <rupertbaxter2@yahoo.com>
Date: Sun, 3 Aug 2014 07:38:04 -0700
Subject: [PATCH 0002/6219] Removed unneccesary changes to utils.py

---
 youtube_dl/utils.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index d199d26d2..e40b367c2 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -6,7 +6,6 @@ import codecs
 import contextlib
 import ctypes
 import datetime
-import time
 import email.utils
 import errno
 import getpass
@@ -748,8 +747,6 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
                 del req.headers['User-agent']
             req.headers['User-agent'] = req.headers['Youtubedl-user-agent']
             del req.headers['Youtubedl-user-agent']
-        #print("sleeping\n")
-        #time.sleep(1)
         return req
 
     def http_response(self, req, resp):

From a42c9215983c4d62d1c000c9dede6e0850dbb5e4 Mon Sep 17 00:00:00 2001
From: rupertbaxter2 <rupertbaxter2@yahoo.com>
Date: Wed, 13 Aug 2014 04:38:40 -0700
Subject: [PATCH 0003/6219] Removed sleep and sleep output when interval is
 zero

---
 youtube_dl/downloader/common.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index 48e829deb..c1da065b5 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -279,8 +279,9 @@ class FileDownloader(object):
         Return True on success and False otherwise
         """
         sleep_interval = self.params.get('sleepinterval', 0)
-        self.to_screen(u'[download] Sleeping %d seconds...' %sleep_interval)
-        time.sleep(sleep_interval)
+        if sleep_interval > 0:
+            self.to_screen(u'[download] Sleeping %d seconds...' %sleep_interval)
+            time.sleep(sleep_interval)
         # Check file already present
         if self.params.get('continuedl', False) and os.path.isfile(encodeFilename(filename)) and not self.params.get('nopart', False):
             self.report_file_already_downloaded(filename)

From 4231235cdaa18d050f3119a80ac409138fb8e8bd Mon Sep 17 00:00:00 2001
From: h-collector <githcoll@gmail.com>
Date: Sun, 6 Jul 2014 01:42:41 +0200
Subject: [PATCH 0004/6219] Fix issues with fc2

Fix issues #2912 and #3171
---
 youtube_dl/extractor/fc2.py | 49 ++++++++++++++++++++++++++++++++++---
 1 file changed, 46 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/fc2.py b/youtube_dl/extractor/fc2.py
index c663a0f81..ecfb233f9 100644
--- a/youtube_dl/extractor/fc2.py
+++ b/youtube_dl/extractor/fc2.py
@@ -7,14 +7,16 @@ import hashlib
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
+    compat_urllib_parse,
     compat_urllib_request,
     compat_urlparse,
 )
 
 
 class FC2IE(InfoExtractor):
-    _VALID_URL = r'^http://video\.fc2\.com/((?P<lang>[^/]+)/)?content/(?P<id>[^/]+)'
+    _VALID_URL = r'^http://video\.fc2\.com/((?P<lang>[^/]+)/)?(a/)?content/(?P<id>[^/]+)'
     IE_NAME = 'fc2'
+    _NETRC_MACHINE = 'fc2'
     _TEST = {
         'url': 'http://video.fc2.com/en/content/20121103kUan1KHs',
         'md5': 'a6ebe8ebe0396518689d963774a54eb7',
@@ -25,17 +27,53 @@ class FC2IE(InfoExtractor):
         },
     }
 
+    #def _real_initialize(self):
+    #    self._login()
+
+    def _login(self):
+        (username, password) = self._get_login_info()
+        if (username is None) or (password is None):
+           self._downloader.report_warning('unable to log in: will be downloading in non authorized mode')
+           return False
+
+        # Log in
+        login_form_strs = {
+            'email':    username,
+            'password': password,
+            'done':     'video',
+            'Submit':   ' Login ',
+        }
+
+        # Convert to UTF-8 *before* urlencode because Python 2.x's urlencode
+        # chokes on unicode
+        login_form = dict((k.encode('utf-8'), v.encode('utf-8')) for k, v in login_form_strs.items())
+        login_data = compat_urllib_parse.urlencode(login_form).encode('utf-8')
+        request    = compat_urllib_request.Request(
+            'https://secure.id.fc2.com/index.php?mode=login&switch_language=en', login_data)
+
+        login_results = self._download_webpage(request, None, note='Logging in', errnote='Unable to log in')
+        if 'mode=redirect&login=done' not in login_results:
+            self._downloader.report_warning('unable to log in: bad username or password')
+            return False
+        
+        # this is also needed
+        login_redir = compat_urllib_request.Request('http://id.fc2.com/?mode=redirect&login=done')
+        redir_res   = self._download_webpage(login_redir, None, note='Login redirect', errnote='Something is not right')
+
+        return True
+
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
 
         webpage = self._download_webpage(url, video_id)
         self._downloader.cookiejar.clear_session_cookies()  # must clear
+        self._login()
 
         title = self._og_search_title(webpage)
         thumbnail = self._og_search_thumbnail(webpage)
-        refer = url.replace('/content/', '/a/content/')
 
+        refer = (url if '/a/content/' in url else url.replace('/content/', '/a/content/'));
         mimi = hashlib.md5((video_id + '_gGddgPfeaf_gzyr').encode('utf-8')).hexdigest()
 
         info_url = (
@@ -47,7 +85,12 @@ class FC2IE(InfoExtractor):
         info = compat_urlparse.parse_qs(info_webpage)
 
         if 'err_code' in info:
-            raise ExtractorError('Error code: %s' % info['err_code'][0])
+            #raise ExtractorError('Error code: %s' % info['err_code'][0])
+            # most of the time we can still download wideo even if err_code is 403 or 602
+            print 'Error code was: %s... but still trying' % info['err_code'][0]
+            
+        if 'filepath' not in info:
+            raise ExtractorError('No file path for download. Maybe not logged?')
 
         video_url = info['filepath'][0] + '?mid=' + info['mid'][0]
         title_info = info.get('title')

From 40b1cbafacea338e51d43fa78438eaf21a1bcbcd Mon Sep 17 00:00:00 2001
From: h-collector <githcoll@gmail.com>
Date: Sun, 6 Jul 2014 01:48:07 +0200
Subject: [PATCH 0005/6219] Update fc2.py

---
 youtube_dl/extractor/fc2.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/fc2.py b/youtube_dl/extractor/fc2.py
index ecfb233f9..0933485e0 100644
--- a/youtube_dl/extractor/fc2.py
+++ b/youtube_dl/extractor/fc2.py
@@ -87,10 +87,10 @@ class FC2IE(InfoExtractor):
         if 'err_code' in info:
             #raise ExtractorError('Error code: %s' % info['err_code'][0])
             # most of the time we can still download wideo even if err_code is 403 or 602
-            print 'Error code was: %s... but still trying' % info['err_code'][0]
-            
+            self._downloader.report_warning('Error code was: %s... but still trying' % info['err_code'][0])
+
         if 'filepath' not in info:
-            raise ExtractorError('No file path for download. Maybe not logged?')
+            raise ExtractorError('Cannot download file. Are you logged?')
 
         video_url = info['filepath'][0] + '?mid=' + info['mid'][0]
         title_info = info.get('title')

From 5a000b45b339f6516f2a5a3bdfd2869713e8438a Mon Sep 17 00:00:00 2001
From: h-collector <githcoll@gmail.com>
Date: Mon, 7 Jul 2014 22:47:28 +0200
Subject: [PATCH 0006/6219] Don't use report_warning for reporting warnings

In tests warning is converted to error
---
 youtube_dl/extractor/fc2.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/fc2.py b/youtube_dl/extractor/fc2.py
index 0933485e0..3073ab0d4 100644
--- a/youtube_dl/extractor/fc2.py
+++ b/youtube_dl/extractor/fc2.py
@@ -33,7 +33,7 @@ class FC2IE(InfoExtractor):
     def _login(self):
         (username, password) = self._get_login_info()
         if (username is None) or (password is None):
-           self._downloader.report_warning('unable to log in: will be downloading in non authorized mode')
+           self.to_screen('unable to log in: will be downloading in non authorized mode') # report_warning
            return False
 
         # Log in
@@ -53,7 +53,7 @@ class FC2IE(InfoExtractor):
 
         login_results = self._download_webpage(request, None, note='Logging in', errnote='Unable to log in')
         if 'mode=redirect&login=done' not in login_results:
-            self._downloader.report_warning('unable to log in: bad username or password')
+            self.to_screen('unable to log in: bad username or password') # report_warning
             return False
         
         # this is also needed
@@ -87,7 +87,7 @@ class FC2IE(InfoExtractor):
         if 'err_code' in info:
             #raise ExtractorError('Error code: %s' % info['err_code'][0])
             # most of the time we can still download wideo even if err_code is 403 or 602
-            self._downloader.report_warning('Error code was: %s... but still trying' % info['err_code'][0])
+            self.to_screen('Error code was: %s... but still trying' % info['err_code'][0]) # report_warning
 
         if 'filepath' not in info:
             raise ExtractorError('Cannot download file. Are you logged?')

From 3d5f7a3947a8d304bc7ad46217f171996e95c475 Mon Sep 17 00:00:00 2001
From: Johannes Knoedtel <johannes@dev-baron.de>
Date: Mon, 12 Jan 2015 22:26:20 +0100
Subject: [PATCH 0007/6219] [utils] Prevent override of custom headers.

The dict of headers of request objects in urllib has its keys always
capitalized.

This causes the lookup to fail and overwrite the header. If for example
a Extractor tries to add a "User-Agent" header the internal
representation in the request object is "User-agent". The header is
therefore clobbered by the "User-Agent" in std_headers, because the
strings are not equal.
---
 youtube_dl/utils.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 98732e8e9..daf94abd1 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -611,7 +611,9 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
 
     def http_request(self, req):
         for h, v in std_headers.items():
-            if h not in req.headers:
+            # Capitalize is needed because of Python bug 2275: http://bugs.python.org/issue2275
+            # The dict keys are capitalized because of this bug by urllib
+            if h.capitalize() not in req.headers:
                 req.add_header(h, v)
         if 'Youtubedl-no-compression' in req.headers:
             if 'Accept-encoding' in req.headers:

From ba319696a99f34342a321d23bb41d3d442a773cc Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 17 Jan 2015 23:56:34 +0100
Subject: [PATCH 0008/6219] [options] Clarify that --password can be left out
 (#4723)

---
 youtube_dl/options.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index a30974efd..f25c12e52 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -264,7 +264,7 @@ def parseOpts(overrideArguments=None):
     authentication.add_option(
         '-p', '--password',
         dest='password', metavar='PASSWORD',
-        help='account password')
+        help='account password. If this option is left out, youtube-dl will ask interactively.')
     authentication.add_option(
         '-2', '--twofactor',
         dest='twofactor', metavar='TWOFACTOR',

From fdb2ed7455d509a5f0b17ad5c1d721d5484bde8f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 18 Jan 2015 08:09:18 +0600
Subject: [PATCH 0009/6219] [abc7news] Add extractor (Closes #4734)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/abc7news.py | 68 ++++++++++++++++++++++++++++++++
 2 files changed, 69 insertions(+)
 create mode 100644 youtube_dl/extractor/abc7news.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 0902eb437..2b9d4455d 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -1,6 +1,7 @@
 from __future__ import unicode_literals
 
 from .abc import ABCIE
+from .abc7news import Abc7NewsIE
 from .academicearth import AcademicEarthCourseIE
 from .addanime import AddAnimeIE
 from .adobetv import AdobeTVIE
diff --git a/youtube_dl/extractor/abc7news.py b/youtube_dl/extractor/abc7news.py
new file mode 100644
index 000000000..c04949c21
--- /dev/null
+++ b/youtube_dl/extractor/abc7news.py
@@ -0,0 +1,68 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import parse_iso8601
+
+
+class Abc7NewsIE(InfoExtractor):
+    _VALID_URL = r'https?://abc7news\.com(?:/[^/]+/(?P<display_id>[^/]+))?/(?P<id>\d+)'
+    _TESTS = [
+        {
+            'url': 'http://abc7news.com/entertainment/east-bay-museum-celebrates-vintage-synthesizers/472581/',
+            'info_dict': {
+                'id': '472581',
+                'display_id': 'east-bay-museum-celebrates-vintage-synthesizers',
+                'ext': 'mp4',
+                'title': 'East Bay museum celebrates history of synthesized music',
+                'description': 'md5:a4f10fb2f2a02565c1749d4adbab4b10',
+                'thumbnail': 're:^https?://.*\.jpg$',
+                'timestamp': 1421123075,
+                'upload_date': '20150113',
+                'uploader': 'Jonathan Bloom',
+            },
+            'params': {
+                # m3u8 download
+                'skip_download': True,
+            },
+        },
+        {
+            'url': 'http://abc7news.com/472581',
+            'only_matching': True,
+        },
+    ]
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        display_id = mobj.group('display_id') or video_id
+
+        webpage = self._download_webpage(url, display_id)
+
+        m3u8 = self._html_search_meta(
+            'contentURL', webpage, 'm3u8 url', fatal=True)
+
+        formats = self._extract_m3u8_formats(m3u8, display_id, 'mp4')
+        self._sort_formats(formats)
+
+        title = self._og_search_title(webpage).strip()
+        description = self._og_search_description(webpage).strip()
+        thumbnail = self._og_search_thumbnail(webpage)
+        timestamp = parse_iso8601(self._search_regex(
+            r'<div class="meta">\s*<time class="timeago" datetime="([^"]+)">',
+            webpage, 'upload date', fatal=False))
+        uploader = self._search_regex(
+            r'rel="author">([^<]+)</a>',
+            webpage, 'uploader', default=None)
+
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'timestamp': timestamp,
+            'uploader': uploader,
+            'formats': formats,
+        }

From bd3cbe0716c3cfc4c6ce7a5a338e881a06b19ef8 Mon Sep 17 00:00:00 2001
From: David Powell <bitlooter@users.noreply.github.com>
Date: Sun, 18 Jan 2015 15:01:05 -0800
Subject: [PATCH 0010/6219] Embed description and URL as MP4 tags

---
 youtube_dl/postprocessor/ffmpeg.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 5b0ff32b1..faccdc43d 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -509,6 +509,10 @@ class FFmpegMetadataPP(FFmpegPostProcessor):
             metadata['artist'] = info['uploader']
         elif info.get('uploader_id') is not None:
             metadata['artist'] = info['uploader_id']
+        if info.get('description') is not None:
+            metadata['description'] = info['description']
+        if info.get('webpage_url') is not None:
+            metadata['comment'] = info['webpage_url']
 
         if not metadata:
             self._downloader.to_screen('[ffmpeg] There isn\'t any metadata to add')

From c511f13f223d843dd4affc30c76bf333a6863227 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 19 Jan 2015 10:10:05 +0100
Subject: [PATCH 0011/6219] [ndtv] Modernize

---
 youtube_dl/extractor/ndtv.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/ndtv.py b/youtube_dl/extractor/ndtv.py
index 95e7d63aa..2a1ca80df 100644
--- a/youtube_dl/extractor/ndtv.py
+++ b/youtube_dl/extractor/ndtv.py
@@ -27,9 +27,7 @@ class NDTVIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
         filename = self._search_regex(

From 030aa5d9e7f482ab7baba18a9ff45e41021d60e3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 19 Jan 2015 23:00:22 +0600
Subject: [PATCH 0012/6219] [tvp] Fix extraction

---
 youtube_dl/extractor/tvp.py | 37 +++++++++++++++++--------------------
 1 file changed, 17 insertions(+), 20 deletions(-)

diff --git a/youtube_dl/extractor/tvp.py b/youtube_dl/extractor/tvp.py
index cc26f417a..f57d609d4 100644
--- a/youtube_dl/extractor/tvp.py
+++ b/youtube_dl/extractor/tvp.py
@@ -12,61 +12,59 @@ class TvpIE(InfoExtractor):
 
     _TESTS = [{
         'url': 'http://vod.tvp.pl/filmy-fabularne/filmy-za-darmo/ogniem-i-mieczem/wideo/odc-2/4278035',
+        'md5': 'cdd98303338b8a7f7abab5cd14092bf2',
         'info_dict': {
             'id': '4278035',
             'ext': 'wmv',
             'title': 'Ogniem i mieczem, odc. 2',
-            'description': 'Bohun dowiaduje się o złamaniu przez kniahinię danego mu słowa i wyrusza do Rozłogów. Helenie w ostatniej chwili udaje się uciec dzięki pomocy Zagłoby.',
         },
     }, {
         'url': 'http://vod.tvp.pl/seriale/obyczajowe/czas-honoru/sezon-1-1-13/i-seria-odc-13/194536',
+        'md5': '8aa518c15e5cc32dfe8db400dc921fbb',
         'info_dict': {
             'id': '194536',
             'ext': 'mp4',
             'title': 'Czas honoru, I seria – odc. 13',
-            #  'description': 'WŁADEK\nCzesław prosi Marię o dostarczenie Władkowi zarazki tyfusu. Jeśli zachoruje zostanie przewieziony do szpitala skąd łatwiej będzie go odbić. Czy matka zdecyduje się zarazić syna? Karol odwiedza Wandę przyznaje się, że ją oszukiwał, ale ostrzega też, że grozi jej aresztowanie i nalega, żeby wyjechała z Warszawy. Czy dziewczyna zdecyduje się znów oddalić od ukochanego? Rozpoczyna się akcja odbicia Władka.',
         },
     }, {
         'url': 'http://www.tvp.pl/there-can-be-anything-so-i-shortened-it/17916176',
+        'md5': 'c3b15ed1af288131115ff17a17c19dda',
         'info_dict': {
             'id': '17916176',
             'ext': 'mp4',
             'title': 'TVP Gorzów pokaże filmy studentów z podroży dookoła świata',
         },
-        'params': {
-            # m3u8 download
-            'skip_download': 'true',
-        },
     }, {
         'url': 'http://vod.tvp.pl/seriale/obyczajowe/na-sygnale/sezon-2-27-/odc-39/17834272',
+        'md5': 'c3b15ed1af288131115ff17a17c19dda',
         'info_dict': {
             'id': '17834272',
             'ext': 'mp4',
             'title': 'Na sygnale, odc. 39',
-            'description': 'Ekipa Wiktora ratuje młodą matkę, która spadła ze schodów trzymając na rękach noworodka. Okazuje się, że dziewczyna jest surogatką, a biologiczni rodzice dziecka próbują zmusić ją do oddania synka…',
-        },
-        'params': {
-            # m3u8 download
-            'skip_download': 'true',
         },
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
+
         webpage = self._download_webpage(
             'http://www.tvp.pl/sess/tvplayer.php?object_id=%s' % video_id, video_id)
 
-        title = self._og_search_title(webpage)
-        series = self._search_regex(
-            r'{name:\s*([\'"])SeriesTitle\1,\s*value:\s*\1(?P<series>.*?)\1},',
+        title = self._search_regex(
+            r'name\s*:\s*([\'"])Title\1\s*,\s*value\s*:\s*\1(?P<title>.+?)\1',
+            webpage, 'title', group='title')
+        series_title = self._search_regex(
+            r'name\s*:\s*([\'"])SeriesTitle\1\s*,\s*value\s*:\s*\1(?P<series>.+?)\1',
             webpage, 'series', group='series', default=None)
-        if series is not None and series not in title:
-            title = '%s, %s' % (series, title)
-        description = self._og_search_description(webpage, default=None)
+        if series_title:
+            title = '%s, %s' % (series_title, title)
+
+        thumbnail = self._search_regex(
+            r"poster\s*:\s*'([^']+)'", webpage, 'thumbnail', default=None)
 
         video_url = self._search_regex(
             r'0:{src:([\'"])(?P<url>.*?)\1', webpage, 'formats', group='url', default=None)
-        if video_url is None:
+        if not video_url:
             video_url = self._download_json(
                 'http://www.tvp.pl/pub/stat/videofileinfo?video_id=%s' % video_id,
                 video_id)['video_url']
@@ -89,8 +87,7 @@ class TvpIE(InfoExtractor):
         return {
             'id': video_id,
             'title': title,
-            'thumbnail': self._og_search_thumbnail(webpage),
-            'description': description,
+            'thumbnail': thumbnail,
             'formats': formats,
         }
 

From 2df54b4ba81a2615d8f5a9016061eed389ba5b6e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 20 Jan 2015 21:23:51 +0600
Subject: [PATCH 0013/6219] [nbcnews] Ignore HTTP errors while coping with
 playlists (Closes #4749)

---
 youtube_dl/extractor/nbc.py | 24 ++++++++++++++++++++++--
 1 file changed, 22 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index 690c46b6a..7ae66616a 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -6,6 +6,7 @@ import json
 from .common import InfoExtractor
 from ..compat import (
     compat_str,
+    compat_HTTPError,
 )
 from ..utils import (
     ExtractorError,
@@ -78,6 +79,16 @@ class NBCNewsIE(InfoExtractor):
             },
             'add_ie': ['ThePlatform'],
         },
+        {
+            'url': 'http://www.nbcnews.com/feature/dateline-full-episodes/full-episode-family-business-n285156',
+            'md5': 'fdbf39ab73a72df5896b6234ff98518a',
+            'info_dict': {
+                'id': 'Wjf9EDR3A_60',
+                'ext': 'mp4',
+                'title': 'FULL EPISODE: Family Business',
+                'description': 'md5:757988edbaae9d7be1d585eb5d55cc04',
+            },
+        },
     ]
 
     def _real_extract(self, url):
@@ -115,10 +126,19 @@ class NBCNewsIE(InfoExtractor):
                 if not base_url:
                     continue
                 playlist_url = base_url + '?form=MPXNBCNewsAPI'
-                all_videos = self._download_json(playlist_url, title)['videos']
 
                 try:
-                    info = next(v for v in all_videos if v['mpxId'] == mpxid)
+                    all_videos = self._download_json(playlist_url, title)
+                except ExtractorError as ee:
+                    if isinstance(ee.cause, compat_HTTPError):
+                        continue
+                    raise
+
+                if not all_videos or not 'videos' in all_videos:
+                    continue
+
+                try:
+                    info = next(v for v in all_videos['videos'] if v['mpxId'] == mpxid)
                     break
                 except StopIteration:
                     continue

From 564bb5e964fc263012784baa579a10eee822a436 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 21 Jan 2015 02:15:28 +0600
Subject: [PATCH 0014/6219] [tinypic] Tweak VALID_URL regex (Closes #4754)

---
 youtube_dl/extractor/tinypic.py | 24 +++++++++++++++---------
 1 file changed, 15 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/tinypic.py b/youtube_dl/extractor/tinypic.py
index 4fe89dbe5..e036b8cdf 100644
--- a/youtube_dl/extractor/tinypic.py
+++ b/youtube_dl/extractor/tinypic.py
@@ -9,17 +9,23 @@ from ..utils import ExtractorError
 class TinyPicIE(InfoExtractor):
     IE_NAME = 'tinypic'
     IE_DESC = 'tinypic.com videos'
-    _VALID_URL = r'http://tinypic\.com/player\.php\?v=(?P<id>[^&]+)&s=\d+'
+    _VALID_URL = r'http://(?:.+?\.)?tinypic\.com/player\.php\?v=(?P<id>[^&]+)&s=\d+'
 
-    _TEST = {
-        'url': 'http://tinypic.com/player.php?v=6xw7tc%3E&s=5#.UtqZmbRFCM8',
-        'md5': '609b74432465364e72727ebc6203f044',
-        'info_dict': {
-            'id': '6xw7tc',
-            'ext': 'flv',
-            'title': 'shadow phenomenon weird',
+    _TESTS = [
+        {
+            'url': 'http://tinypic.com/player.php?v=6xw7tc%3E&s=5#.UtqZmbRFCM8',
+            'md5': '609b74432465364e72727ebc6203f044',
+            'info_dict': {
+                'id': '6xw7tc',
+                'ext': 'flv',
+                'title': 'shadow phenomenon weird',
+            },
+        },
+        {
+            'url': 'http://de.tinypic.com/player.php?v=dy90yh&s=8',
+            'only_matching': True,
         }
-    }
+    ]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)

From 5fe511258998666e7617c22a5db27506c16eb955 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 21 Jan 2015 10:27:18 +0100
Subject: [PATCH 0015/6219] [CNNArticle] Update test

---
 youtube_dl/extractor/cnn.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/cnn.py b/youtube_dl/extractor/cnn.py
index 93e8d0de3..4bb58f25e 100644
--- a/youtube_dl/extractor/cnn.py
+++ b/youtube_dl/extractor/cnn.py
@@ -143,13 +143,13 @@ class CNNArticleIE(InfoExtractor):
     _VALID_URL = r'https?://(?:(?:edition|www)\.)?cnn\.com/(?!video/)'
     _TEST = {
         'url': 'http://www.cnn.com/2014/12/21/politics/obama-north-koreas-hack-not-war-but-cyber-vandalism/',
-        'md5': '275b326f85d80dff7592a9820f5dc887',
+        'md5': '689034c2a3d9c6dc4aa72d65a81efd01',
         'info_dict': {
-            'id': 'bestoftv/2014/12/21/sotu-crowley-president-obama-north-korea-not-going-to-be-intimidated.cnn',
+            'id': 'bestoftv/2014/12/21/ip-north-korea-obama.cnn',
             'ext': 'mp4',
-            'title': 'Obama: We\'re not going to be intimidated',
-            'description': 'md5:e735586f3dc936075fa654a4d91b21f9',
-            'upload_date': '20141220',
+            'title': 'Obama: Cyberattack not an act of war',
+            'description': 'md5:51ce6750450603795cad0cdfbd7d05c5',
+            'upload_date': '20141221',
         },
         'add_ie': ['CNN'],
     }

From efcddaebe9302eaaea1cbb5570c5c2bd25abf1fe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 21 Jan 2015 10:31:57 +0100
Subject: [PATCH 0016/6219] [cnn] Use edition.cnn.com for getting the
 information (fixes #4757)

Some videos (like http://edition.cnn.com/videos/us/2015/01/20/orig-yellowstone-oil-spill.cnn) will fail if we use cnn.com.
---
 youtube_dl/extractor/cnn.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/cnn.py b/youtube_dl/extractor/cnn.py
index 4bb58f25e..90ea07438 100644
--- a/youtube_dl/extractor/cnn.py
+++ b/youtube_dl/extractor/cnn.py
@@ -51,7 +51,7 @@ class CNNIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         path = mobj.group('path')
         page_title = mobj.group('title')
-        info_url = 'http://cnn.com/video/data/3.0/%s/index.xml' % path
+        info_url = 'http://edition.cnn.com/video/data/3.0/%s/index.xml' % path
         info = self._download_xml(info_url, page_title)
 
         formats = []

From 47e0e1e0e2306dd7119844bbbc3ef39ccb175ed1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 21 Jan 2015 10:36:15 +0100
Subject: [PATCH 0017/6219] [nbc] Fix pep8 issue

---
 youtube_dl/extractor/nbc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index 7ae66616a..f840f6532 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -134,7 +134,7 @@ class NBCNewsIE(InfoExtractor):
                         continue
                     raise
 
-                if not all_videos or not 'videos' in all_videos:
+                if not all_videos or 'videos' not in all_videos:
                     continue
 
                 try:

From c8dc41a6e7eea2a1235238cfeadb4caf5619d0ae Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 21 Jan 2015 23:05:47 +0800
Subject: [PATCH 0018/6219] [StreetVoice] Add new extractor

---
 youtube_dl/extractor/__init__.py    |  1 +
 youtube_dl/extractor/streetvoice.py | 36 +++++++++++++++++++++++++++++
 2 files changed, 37 insertions(+)
 create mode 100644 youtube_dl/extractor/streetvoice.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 2b9d4455d..9e942e6f9 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -409,6 +409,7 @@ from .stanfordoc import StanfordOpenClassroomIE
 from .steam import SteamIE
 from .streamcloud import StreamcloudIE
 from .streamcz import StreamCZIE
+from .streetvoice import StreetVoiceIE
 from .sunporno import SunPornoIE
 from .swrmediathek import SWRMediathekIE
 from .syfy import SyfyIE
diff --git a/youtube_dl/extractor/streetvoice.py b/youtube_dl/extractor/streetvoice.py
new file mode 100644
index 000000000..55ceea75b
--- /dev/null
+++ b/youtube_dl/extractor/streetvoice.py
@@ -0,0 +1,36 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class StreetVoiceIE(InfoExtractor):
+    _VALID_URL = r'http://tw.streetvoice.com/[^/]+/songs/(?P<id>[0-9]+)/'
+    _TESTS = [
+        {
+            'url': 'http://tw.streetvoice.com/skippylu/songs/94440/',
+            'md5': '15974627fc01a29e492c98593c2fd472',
+            'info_dict': {
+                'id': '94440',
+                'ext': 'mp3',
+                'title': '輸',
+                'description': '輸 - Crispy脆樂團'
+            }
+        }
+    ]
+
+    def _real_extract(self, url):
+        song_id = self._match_id(url)
+
+        api_url = 'http://tw.streetvoice.com/music/api/song/%s/' % song_id
+        info_dict = self._download_json(api_url, song_id)
+
+        author = info_dict['musician']['name']
+        title = info_dict['name']
+        return {
+            'id': song_id,
+            'ext': 'mp3',
+            'title': title,
+            'url': info_dict['file'],
+            'description': '%s - %s' % (title, author)
+        }

From c5db6bb32ba708bc5c92a68487b07be2374f416d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 21 Jan 2015 22:27:21 +0600
Subject: [PATCH 0019/6219] [twitch] Refactor and add support for past
 broadcasts

---
 youtube_dl/extractor/__init__.py |   8 +-
 youtube_dl/extractor/twitch.py   | 345 ++++++++++++++++---------------
 2 files changed, 184 insertions(+), 169 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 2b9d4455d..0b0f3be19 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -458,7 +458,13 @@ from .tvigle import TvigleIE
 from .tvp import TvpIE, TvpSeriesIE
 from .tvplay import TVPlayIE
 from .twentyfourvideo import TwentyFourVideoIE
-from .twitch import TwitchIE
+from .twitch import (
+    TwitchVideoIE,
+    TwitchChapterIE,
+    TwitchVodIE,
+    TwitchProfileIE,
+    TwitchPastBroadcastsIE,
+)
 from .ubu import UbuIE
 from .udemy import (
     UdemyIE,
diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index b11a1d561..f70002f68 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -15,44 +15,11 @@ from ..utils import (
 )
 
 
-class TwitchIE(InfoExtractor):
-    # TODO: One broadcast may be split into multiple videos. The key
-    # 'broadcast_id' is the same for all parts, and 'broadcast_part'
-    # starts at 1 and increases. Can we treat all parts as one video?
-    _VALID_URL = r"""(?x)^(?:http://)?(?:www\.)?twitch\.tv/
-        (?:
-            (?P<channelid>[^/]+)|
-            (?:(?:[^/]+)/v/(?P<vodid>[^/]+))|
-            (?:(?:[^/]+)/b/(?P<videoid>[^/]+))|
-            (?:(?:[^/]+)/c/(?P<chapterid>[^/]+))
-        )
-        /?(?:\#.*)?$
-        """
-    _PAGE_LIMIT = 100
+class TwitchBaseIE(InfoExtractor):
+    _VALID_URL_BASE = r'http://(?:www\.)?twitch\.tv'
+
     _API_BASE = 'https://api.twitch.tv'
     _LOGIN_URL = 'https://secure.twitch.tv/user/login'
-    _TESTS = [{
-        'url': 'http://www.twitch.tv/riotgames/b/577357806',
-        'info_dict': {
-            'id': 'a577357806',
-            'title': 'Worlds Semifinals - Star Horn Royal Club vs. OMG',
-        },
-        'playlist_mincount': 12,
-    }, {
-        'url': 'http://www.twitch.tv/acracingleague/c/5285812',
-        'info_dict': {
-            'id': 'c5285812',
-            'title': 'ACRL Off Season - Sports Cars @ Nordschleife',
-        },
-        'playlist_mincount': 3,
-    }, {
-        'url': 'http://www.twitch.tv/vanillatv',
-        'info_dict': {
-            'id': 'vanillatv',
-            'title': 'VanillaTV',
-        },
-        'playlist_mincount': 412,
-    }]
 
     def _handle_error(self, response):
         if not isinstance(response, dict):
@@ -64,71 +31,10 @@ class TwitchIE(InfoExtractor):
                 expected=True)
 
     def _download_json(self, url, video_id, note='Downloading JSON metadata'):
-        response = super(TwitchIE, self)._download_json(url, video_id, note)
+        response = super(TwitchBaseIE, self)._download_json(url, video_id, note)
         self._handle_error(response)
         return response
 
-    def _extract_media(self, item, item_id):
-        ITEMS = {
-            'a': 'video',
-            'v': 'vod',
-            'c': 'chapter',
-        }
-        info = self._extract_info(self._download_json(
-            '%s/kraken/videos/%s%s' % (self._API_BASE, item, item_id), item_id,
-            'Downloading %s info JSON' % ITEMS[item]))
-
-        if item == 'v':
-            access_token = self._download_json(
-                '%s/api/vods/%s/access_token' % (self._API_BASE, item_id), item_id,
-                'Downloading %s access token' % ITEMS[item])
-            formats = self._extract_m3u8_formats(
-                'http://usher.twitch.tv/vod/%s?nauth=%s&nauthsig=%s'
-                % (item_id, access_token['token'], access_token['sig']),
-                item_id, 'mp4')
-            info['formats'] = formats
-            return info
-
-        response = self._download_json(
-            '%s/api/videos/%s%s' % (self._API_BASE, item, item_id), item_id,
-            'Downloading %s playlist JSON' % ITEMS[item])
-        entries = []
-        chunks = response['chunks']
-        qualities = list(chunks.keys())
-        for num, fragment in enumerate(zip(*chunks.values()), start=1):
-            formats = []
-            for fmt_num, fragment_fmt in enumerate(fragment):
-                format_id = qualities[fmt_num]
-                fmt = {
-                    'url': fragment_fmt['url'],
-                    'format_id': format_id,
-                    'quality': 1 if format_id == 'live' else 0,
-                }
-                m = re.search(r'^(?P<height>\d+)[Pp]', format_id)
-                if m:
-                    fmt['height'] = int(m.group('height'))
-                formats.append(fmt)
-            self._sort_formats(formats)
-            entry = dict(info)
-            entry['id'] = '%s_%d' % (entry['id'], num)
-            entry['title'] = '%s part %d' % (entry['title'], num)
-            entry['formats'] = formats
-            entries.append(entry)
-        return self.playlist_result(entries, info['id'], info['title'])
-
-    def _extract_info(self, info):
-        return {
-            'id': info['_id'],
-            'title': info['title'],
-            'description': info['description'],
-            'duration': info['length'],
-            'thumbnail': info['preview'],
-            'uploader': info['channel']['display_name'],
-            'uploader_id': info['channel']['name'],
-            'timestamp': parse_iso8601(info['recorded_at']),
-            'view_count': info['views'],
-        }
-
     def _real_initialize(self):
         self._login()
 
@@ -167,81 +73,184 @@ class TwitchIE(InfoExtractor):
             raise ExtractorError(
                 'Unable to login: %s' % m.group('msg').strip(), expected=True)
 
+
+class TwitchItemBaseIE(TwitchBaseIE):
+    def _download_info(self, item, item_id):
+        return self._extract_info(self._download_json(
+            '%s/kraken/videos/%s%s' % (self._API_BASE, item, item_id), item_id,
+            'Downloading %s info JSON' % self._ITEM_TYPE))
+
+    def _extract_media(self, item_id):
+        info = self._download_info(self._ITEM_SHORTCUT, item_id)
+        response = self._download_json(
+            '%s/api/videos/%s%s' % (self._API_BASE, self._ITEM_SHORTCUT, item_id), item_id,
+            'Downloading %s playlist JSON' % self._ITEM_TYPE)
+        entries = []
+        chunks = response['chunks']
+        qualities = list(chunks.keys())
+        for num, fragment in enumerate(zip(*chunks.values()), start=1):
+            formats = []
+            for fmt_num, fragment_fmt in enumerate(fragment):
+                format_id = qualities[fmt_num]
+                fmt = {
+                    'url': fragment_fmt['url'],
+                    'format_id': format_id,
+                    'quality': 1 if format_id == 'live' else 0,
+                }
+                m = re.search(r'^(?P<height>\d+)[Pp]', format_id)
+                if m:
+                    fmt['height'] = int(m.group('height'))
+                formats.append(fmt)
+            self._sort_formats(formats)
+            entry = dict(info)
+            entry['id'] = '%s_%d' % (entry['id'], num)
+            entry['title'] = '%s part %d' % (entry['title'], num)
+            entry['formats'] = formats
+            entries.append(entry)
+        return self.playlist_result(entries, info['id'], info['title'])
+
+    def _extract_info(self, info):
+        return {
+            'id': info['_id'],
+            'title': info['title'],
+            'description': info['description'],
+            'duration': info['length'],
+            'thumbnail': info['preview'],
+            'uploader': info['channel']['display_name'],
+            'uploader_id': info['channel']['name'],
+            'timestamp': parse_iso8601(info['recorded_at']),
+            'view_count': info['views'],
+        }
+
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        if mobj.group('chapterid'):
-            return self._extract_media('c', mobj.group('chapterid'))
+        return self._extract_media(self._match_id(url))
 
-            """
-            webpage = self._download_webpage(url, chapter_id)
-            m = re.search(r'PP\.archive_id = "([0-9]+)";', webpage)
-            if not m:
-                raise ExtractorError('Cannot find archive of a chapter')
-            archive_id = m.group(1)
 
-            api = api_base + '/broadcast/by_chapter/%s.xml' % chapter_id
-            doc = self._download_xml(
-                api, chapter_id,
-                note='Downloading chapter information',
-                errnote='Chapter information download failed')
-            for a in doc.findall('.//archive'):
-                if archive_id == a.find('./id').text:
-                    break
-            else:
-                raise ExtractorError('Could not find chapter in chapter information')
+class TwitchVideoIE(TwitchItemBaseIE):
+    IE_NAME = 'twitch:video'
+    _VALID_URL = r'%s/[^/]+/b/(?P<id>[^/]+)' % TwitchBaseIE._VALID_URL_BASE
+    _ITEM_TYPE = 'video'
+    _ITEM_SHORTCUT = 'a'
 
-            video_url = a.find('./video_file_url').text
-            video_ext = video_url.rpartition('.')[2] or 'flv'
+    _TEST = {
+        'url': 'http://www.twitch.tv/riotgames/b/577357806',
+        'info_dict': {
+            'id': 'a577357806',
+            'title': 'Worlds Semifinals - Star Horn Royal Club vs. OMG',
+        },
+        'playlist_mincount': 12,
+    }
 
-            chapter_api_url = 'https://api.twitch.tv/kraken/videos/c' + chapter_id
-            chapter_info = self._download_json(
-                chapter_api_url, 'c' + chapter_id,
-                note='Downloading chapter metadata',
-                errnote='Download of chapter metadata failed')
 
-            bracket_start = int(doc.find('.//bracket_start').text)
-            bracket_end = int(doc.find('.//bracket_end').text)
+class TwitchChapterIE(TwitchItemBaseIE):
+    IE_NAME = 'twitch:chapter'
+    _VALID_URL = r'%s/[^/]+/c/(?P<id>[^/]+)' % TwitchBaseIE._VALID_URL_BASE
+    _ITEM_TYPE = 'chapter'
+    _ITEM_SHORTCUT = 'c'
 
-            # TODO determine start (and probably fix up file)
-            #  youtube-dl -v http://www.twitch.tv/firmbelief/c/1757457
-            #video_url += '?start=' + TODO:start_timestamp
-            # bracket_start is 13290, but we want 51670615
-            self._downloader.report_warning('Chapter detected, but we can just download the whole file. '
-                                            'Chapter starts at %s and ends at %s' % (formatSeconds(bracket_start), formatSeconds(bracket_end)))
+    _TEST = {
+        'url': 'http://www.twitch.tv/acracingleague/c/5285812',
+        'info_dict': {
+            'id': 'c5285812',
+            'title': 'ACRL Off Season - Sports Cars @ Nordschleife',
+        },
+        'playlist_mincount': 3,
+    }
 
-            info = {
-                'id': 'c' + chapter_id,
-                'url': video_url,
-                'ext': video_ext,
-                'title': chapter_info['title'],
-                'thumbnail': chapter_info['preview'],
-                'description': chapter_info['description'],
-                'uploader': chapter_info['channel']['display_name'],
-                'uploader_id': chapter_info['channel']['name'],
-            }
-            return info
-            """
-        elif mobj.group('videoid'):
-            return self._extract_media('a', mobj.group('videoid'))
-        elif mobj.group('vodid'):
-            return self._extract_media('v', mobj.group('vodid'))
-        elif mobj.group('channelid'):
-            channel_id = mobj.group('channelid')
-            info = self._download_json(
-                '%s/kraken/channels/%s' % (self._API_BASE, channel_id),
-                channel_id, 'Downloading channel info JSON')
-            channel_name = info.get('display_name') or info.get('name')
-            entries = []
-            offset = 0
-            limit = self._PAGE_LIMIT
-            for counter in itertools.count(1):
-                response = self._download_json(
-                    '%s/kraken/channels/%s/videos/?offset=%d&limit=%d'
-                    % (self._API_BASE, channel_id, offset, limit),
-                    channel_id, 'Downloading channel videos JSON page %d' % counter)
-                videos = response['videos']
-                if not videos:
-                    break
-                entries.extend([self.url_result(video['url'], 'Twitch') for video in videos])
-                offset += limit
-            return self.playlist_result(entries, channel_id, channel_name)
+
+class TwitchVodIE(TwitchItemBaseIE):
+    IE_NAME = 'twitch:vod'
+    _VALID_URL = r'%s/[^/]+/v/(?P<id>[^/]+)' % TwitchBaseIE._VALID_URL_BASE
+    _ITEM_TYPE = 'vod'
+    _ITEM_SHORTCUT = 'v'
+
+    _TEST = {
+        'url': 'http://www.twitch.tv/ksptv/v/3622000',
+        'info_dict': {
+            'id': 'v3622000',
+            'ext': 'mp4',
+            'title': '''KSPTV: Squadcast: "Everyone's on vacation so here's Dahud" Edition!''',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 6951,
+            'timestamp': 1419028564,
+            'upload_date': '20141219',
+            'uploader': 'KSPTV',
+            'uploader_id': 'ksptv',
+            'view_count': int,
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }
+
+    def _real_extract(self, url):
+        item_id = self._match_id(url)
+        info = self._download_info(self._ITEM_SHORTCUT, item_id)
+        access_token = self._download_json(
+            '%s/api/vods/%s/access_token' % (self._API_BASE, item_id), item_id,
+            'Downloading %s access token' % self._ITEM_TYPE)
+        formats = self._extract_m3u8_formats(
+            'http://usher.twitch.tv/vod/%s?nauth=%s&nauthsig=%s'
+            % (item_id, access_token['token'], access_token['sig']),
+            item_id, 'mp4')
+        info['formats'] = formats
+        return info
+
+
+class TwitchPlaylistBaseIE(TwitchBaseIE):
+    _PLAYLIST_URL = '%s/kraken/channels/%%s/videos/?offset=%%d&limit=%%d' % TwitchBaseIE._API_BASE
+    _PAGE_LIMIT = 100
+
+    def _extract_playlist(self, channel_id):
+        info = self._download_json(
+            '%s/kraken/channels/%s' % (self._API_BASE, channel_id),
+            channel_id, 'Downloading channel info JSON')
+        channel_name = info.get('display_name') or info.get('name')
+        entries = []
+        offset = 0
+        limit = self._PAGE_LIMIT
+        for counter in itertools.count(1):
+            response = self._download_json(
+                self._PLAYLIST_URL % (channel_id, offset, limit),
+                channel_id, 'Downloading %s videos JSON page %d' % (self._PLAYLIST_TYPE, counter))
+            videos = response['videos']
+            if not videos:
+                break
+            entries.extend([self.url_result(video['url']) for video in videos])
+            offset += limit
+        return self.playlist_result(entries, channel_id, channel_name)
+
+    def _real_extract(self, url):
+        return self._extract_playlist(self._match_id(url))
+
+
+class TwitchProfileIE(TwitchPlaylistBaseIE):
+    IE_NAME = 'twitch:profile'
+    _VALID_URL = r'%s/(?P<id>[^/]+)/profile/?(?:\#.*)?$' % TwitchBaseIE._VALID_URL_BASE
+    _PLAYLIST_TYPE = 'profile'
+
+    _TEST = {
+        'url': 'http://www.twitch.tv/vanillatv/profile',
+        'info_dict': {
+            'id': 'vanillatv',
+            'title': 'VanillaTV',
+        },
+        'playlist_mincount': 412,
+    }
+
+
+class TwitchPastBroadcastsIE(TwitchPlaylistBaseIE):
+    IE_NAME = 'twitch:profile'
+    _VALID_URL = r'%s/(?P<id>[^/]+)/profile/past_broadcasts/?(?:\#.*)?$' % TwitchBaseIE._VALID_URL_BASE
+    _PLAYLIST_URL = TwitchPlaylistBaseIE._PLAYLIST_URL + '&broadcasts=true'
+    _PLAYLIST_TYPE = 'past broadcasts'
+
+    _TEST = {
+        'url': 'http://www.twitch.tv/spamfish/profile/past_broadcasts',
+        'info_dict': {
+            'id': 'spamfish',
+            'title': 'Spamfish',
+        },
+        'playlist_mincount': 54,
+    }

From bb0aa4cb3ca1b21084e6e6229e4a69014e61ccff Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 21 Jan 2015 22:53:51 +0600
Subject: [PATCH 0020/6219] [streetvoice] Improve

---
 youtube_dl/extractor/streetvoice.py | 53 ++++++++++++++++++-----------
 1 file changed, 34 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/extractor/streetvoice.py b/youtube_dl/extractor/streetvoice.py
index 55ceea75b..6a57fa60a 100644
--- a/youtube_dl/extractor/streetvoice.py
+++ b/youtube_dl/extractor/streetvoice.py
@@ -2,35 +2,50 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..compat import compat_str
+from ..utils import unified_strdate
 
 
 class StreetVoiceIE(InfoExtractor):
-    _VALID_URL = r'http://tw.streetvoice.com/[^/]+/songs/(?P<id>[0-9]+)/'
-    _TESTS = [
-        {
-            'url': 'http://tw.streetvoice.com/skippylu/songs/94440/',
-            'md5': '15974627fc01a29e492c98593c2fd472',
-            'info_dict': {
-                'id': '94440',
-                'ext': 'mp3',
-                'title': '輸',
-                'description': '輸 - Crispy脆樂團'
-            }
+    _VALID_URL = r'https?://(?:.+?\.)?streetvoice\.com/[^/]+/songs/(?P<id>[0-9]+)'
+    _TESTS = [{
+        'url': 'http://streetvoice.com/skippylu/songs/94440/',
+        'md5': '15974627fc01a29e492c98593c2fd472',
+        'info_dict': {
+            'id': '94440',
+            'ext': 'mp3',
+            'filesize': 4167053,
+            'title': '輸',
+            'description': 'Crispy脆樂團 - 輸',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 260,
+            'upload_date': '20091018',
+            'uploader': 'Crispy脆樂團',
+            'uploader_id': '627810',
         }
-    ]
+    }, {
+        'url': 'http://tw.streetvoice.com/skippylu/songs/94440/',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         song_id = self._match_id(url)
 
-        api_url = 'http://tw.streetvoice.com/music/api/song/%s/' % song_id
-        info_dict = self._download_json(api_url, song_id)
+        song = self._download_json(
+            'http://streetvoice.com/music/api/song/%s' % song_id, song_id)
+
+        title = song['name']
+        author = song['musician']['name']
 
-        author = info_dict['musician']['name']
-        title = info_dict['name']
         return {
             'id': song_id,
-            'ext': 'mp3',
+            'url': song['file'],
+            'filesize': song.get('size'),
             'title': title,
-            'url': info_dict['file'],
-            'description': '%s - %s' % (title, author)
+            'description': '%s - %s' % (author, title),
+            'thumbnail': self._proto_relative_url(song.get('image'), 'http:'),
+            'duration': song.get('length'),
+            'upload_date': unified_strdate(song.get('created_at')),
+            'uploader': author,
+            'uploader_id': compat_str(song['musician']['id']),
         }

From c62b44976545401af60fd35f0188fd09a93e449b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 21 Jan 2015 22:56:28 +0600
Subject: [PATCH 0021/6219] Credit @yan12125 for streetvoice (#4758)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 35e256d49..b8bf3cb6f 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -103,3 +103,4 @@ Christopher Krooss
 Ondřej Caletka
 Dinesh S
 Johan K. Jensen
+Yen Chi Hsuan

From 8bb1bdfae91246e1306ee31674c06a2ef7e42a08 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 21 Jan 2015 23:06:16 +0600
Subject: [PATCH 0022/6219] [twitch:past_broadcasts] Fix IE_NAME

---
 youtube_dl/extractor/twitch.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index f70002f68..8866e5182 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -241,7 +241,7 @@ class TwitchProfileIE(TwitchPlaylistBaseIE):
 
 
 class TwitchPastBroadcastsIE(TwitchPlaylistBaseIE):
-    IE_NAME = 'twitch:profile'
+    IE_NAME = 'twitch:past_broadcasts'
     _VALID_URL = r'%s/(?P<id>[^/]+)/profile/past_broadcasts/?(?:\#.*)?$' % TwitchBaseIE._VALID_URL_BASE
     _PLAYLIST_URL = TwitchPlaylistBaseIE._PLAYLIST_URL + '&broadcasts=true'
     _PLAYLIST_TYPE = 'past broadcasts'

From e5763a7a7e630cfd94b8993a9592cb3243890e0c Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Wed, 21 Jan 2015 21:47:55 +0200
Subject: [PATCH 0023/6219] [hearthisat] Add new extractor (Closes #4743)

---
 youtube_dl/extractor/__init__.py   |  1 +
 youtube_dl/extractor/hearthisat.py | 92 ++++++++++++++++++++++++++++++
 2 files changed, 93 insertions(+)
 create mode 100644 youtube_dl/extractor/hearthisat.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index c41f2de0f..be3fecd0d 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -176,6 +176,7 @@ from .goshgay import GoshgayIE
 from .grooveshark import GroovesharkIE
 from .groupon import GrouponIE
 from .hark import HarkIE
+from .hearthisat import HearThisAtIE
 from .heise import HeiseIE
 from .hellporno import HellPornoIE
 from .helsinki import HelsinkiIE
diff --git a/youtube_dl/extractor/hearthisat.py b/youtube_dl/extractor/hearthisat.py
new file mode 100644
index 000000000..c1177a9c5
--- /dev/null
+++ b/youtube_dl/extractor/hearthisat.py
@@ -0,0 +1,92 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..compat import compat_urllib_request
+from ..utils import (
+    str_to_int,
+    urlencode_postdata,
+)
+
+
+class HearThisAtIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?hearthis\.at/(?P<artist>[^/]+)/(?P<title>[A-Za-z0-9\-]+)/?$'
+    _PLAYLIST_URL = 'https://hearthis.at/playlist.php'
+    _TEST = {
+        'url': 'https://hearthis.at/moofi/dr-kreep',
+        'md5': 'd594c573227a89f4256f0b03e68c80cc',
+        'info_dict': {
+            'id': '150939',
+            'ext': 'mp3',
+            'title': 'Moofi - Dr. Kreep',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'timestamp': 1421564134,
+            'description': 'Creepy Patch. Mutable Instruments Braids Vowel + Formant Mode.',
+            'upload_date': '20150118',
+            'comment_count': int,
+            'view_count': int,
+            'like_count': int,
+            'duration': 71,
+            'categories': ['Experimental'],
+        }
+    }
+
+    def _real_extract(self, url):
+        m = re.match(self._VALID_URL, url)
+        display_id = '{artist:s} - {title:s}'.format(**m.groupdict())
+
+        webpage = self._download_webpage(url, display_id)
+        track_id = self._search_regex(
+            r'intTrackId\s*=\s*(\d+)', webpage, 'track ID')
+
+        payload = urlencode_postdata({'tracks[]': track_id})
+        req = compat_urllib_request.Request(self._PLAYLIST_URL, payload)
+        req.add_header('Content-type', 'application/x-www-form-urlencoded')
+
+        track = self._download_json(req, track_id, 'Downloading playlist')[0]
+        title = '{artist:s} - {title:s}'.format(**track)
+
+        categories = None
+        if track.get('category'):
+            categories = [track['category']]
+
+        description = self._og_search_description(webpage)
+        thumbnail = self._og_search_thumbnail(webpage)
+
+        meta_span = r'<span[^>]+class="%s".*?</i>([^<]+)</span>'
+        view_count = str_to_int(self._search_regex(
+            meta_span % 'plays_count', webpage, 'view count', fatal=False))
+        like_count = str_to_int(self._search_regex(
+            meta_span % 'likes_count', webpage, 'like count', fatal=False))
+        comment_count = str_to_int(self._search_regex(
+            meta_span % 'comment_count', webpage, 'comment count', fatal=False))
+        duration = str_to_int(self._search_regex(
+            r'data-length="(\d+)', webpage, 'duration', fatal=False))
+        timestamp = str_to_int(self._search_regex(
+            r'<span[^>]+class="calctime"[^>]+data-time="(\d+)', webpage, 'timestamp', fatal=False))
+
+        track_url = self._search_regex(
+            r'<a[^>]+data-mp3="([^"]+)"', webpage, 'track URL')
+
+        formats = [{
+            'format_id': 'mp3',
+            'url': track_url,
+            'vcodec': 'none',
+        }]
+
+        return {
+            'id': track_id,
+            'display-id': display_id,
+            'title': title,
+            'formats': formats,
+            'thumbnail': thumbnail,
+            'description': description,
+            'duration': duration,
+            'timestamp': timestamp,
+            'view_count': view_count,
+            'comment_count': comment_count,
+            'like_count': like_count,
+            'categories': categories,
+        }

From b55ee18ff3a9642fe25a977e1152472877294493 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 22 Jan 2015 12:04:07 +0100
Subject: [PATCH 0024/6219] [hearthisat] Add support for more high-quality
 download links

---
 youtube_dl/extractor/hearthisat.py | 49 ++++++++++++++++++++++--------
 youtube_dl/utils.py                |  8 +++++
 2 files changed, 45 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/hearthisat.py b/youtube_dl/extractor/hearthisat.py
index c1177a9c5..d83072c1a 100644
--- a/youtube_dl/extractor/hearthisat.py
+++ b/youtube_dl/extractor/hearthisat.py
@@ -4,10 +4,15 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_request
+from ..compat import (
+    compat_urllib_request,
+    compat_urlparse,
+)
 from ..utils import (
+    HEADRequest,
     str_to_int,
     urlencode_postdata,
+    urlhandle_detect_ext,
 )
 
 
@@ -16,10 +21,10 @@ class HearThisAtIE(InfoExtractor):
     _PLAYLIST_URL = 'https://hearthis.at/playlist.php'
     _TEST = {
         'url': 'https://hearthis.at/moofi/dr-kreep',
-        'md5': 'd594c573227a89f4256f0b03e68c80cc',
+        'md5': 'ab6ec33c8fed6556029337c7885eb4e0',
         'info_dict': {
             'id': '150939',
-            'ext': 'mp3',
+            'ext': 'wav',
             'title': 'Moofi - Dr. Kreep',
             'thumbnail': 're:^https?://.*\.jpg$',
             'timestamp': 1421564134,
@@ -67,18 +72,38 @@ class HearThisAtIE(InfoExtractor):
         timestamp = str_to_int(self._search_regex(
             r'<span[^>]+class="calctime"[^>]+data-time="(\d+)', webpage, 'timestamp', fatal=False))
 
-        track_url = self._search_regex(
-            r'<a[^>]+data-mp3="([^"]+)"', webpage, 'track URL')
-
-        formats = [{
-            'format_id': 'mp3',
-            'url': track_url,
-            'vcodec': 'none',
-        }]
+        formats = []
+        mp3_url = self._search_regex(
+            r'(?s)<a class="player-link"\s+(?:[a-zA-Z0-9_:-]+="[^"]+"\s+)*?data-mp3="([^"]+)"',
+            webpage, 'title', fatal=False)
+        if mp3_url:
+            formats.append({
+                'format_id': 'mp3',
+                'vcodec': 'none',
+                'acodec': 'mp3',
+                'url': mp3_url,
+            })
+        download_path = self._search_regex(
+            r'<a class="[^"]*download_fct[^"]*"\s+href="([^"]+)"',
+            webpage, 'download URL', default=None)
+        if download_path:
+            download_url = compat_urlparse.urljoin(url, download_path)
+            ext_req = HEADRequest(download_url)
+            ext_handle = self._request_webpage(
+                ext_req, display_id, note='Determining extension')
+            ext = urlhandle_detect_ext(ext_handle)
+            formats.append({
+                'format_id': 'download',
+                'vcodec': 'none',
+                'ext': ext,
+                'url': download_url,
+                'preference': 2,  # Usually better quality
+            })
+        self._sort_formats(formats)
 
         return {
             'id': track_id,
-            'display-id': display_id,
+            'display_id': display_id,
             'title': title,
             'formats': formats,
             'thumbnail': thumbnail,
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 7832ed87f..764474c33 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1612,6 +1612,14 @@ def urlhandle_detect_ext(url_handle):
     except AttributeError:  # Python < 3
         getheader = url_handle.info().getheader
 
+    cd = getheader('Content-Disposition')
+    if cd:
+        m = re.match(r'attachment;\s*filename="(?P<filename>[^"]+)"', cd)
+        if m:
+            e = determine_ext(m.group('filename'), default_ext=None)
+            if e:
+                return e
+
     return getheader('Content-Type').split("/")[1]
 
 

From 898c23c03f634d14fa414f0e0693d5c4033b2671 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 22 Jan 2015 12:04:26 +0100
Subject: [PATCH 0025/6219] release 2015.01.22

---
 README.md             | 3 ++-
 youtube_dl/version.py | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 078e9df82..60bd7e2b6 100644
--- a/README.md
+++ b/README.md
@@ -304,7 +304,8 @@ which means you can modify it, redistribute it or use it however you like.
 
 ## Authentication Options:
     -u, --username USERNAME          login with this account ID
-    -p, --password PASSWORD          account password
+    -p, --password PASSWORD          account password. If this option is left
+                                     out, youtube-dl will ask interactively.
     -2, --twofactor TWOFACTOR        two-factor auth code
     -n, --netrc                      use .netrc authentication data
     --video-password PASSWORD        video password (vimeo, smotri)
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 63a79a7ee..f5df7be31 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.01.16'
+__version__ = '2015.01.22'

From bb6e38787dcc3593a4d9cb66bcf9f736570e944a Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Thu, 22 Jan 2015 18:35:53 +0200
Subject: [PATCH 0026/6219] [videomega] Fix extraction (Closes #4763)

---
 youtube_dl/extractor/videomega.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/videomega.py b/youtube_dl/extractor/videomega.py
index 9fc64d172..fc6e05fe0 100644
--- a/youtube_dl/extractor/videomega.py
+++ b/youtube_dl/extractor/videomega.py
@@ -1,12 +1,15 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse,
     compat_urllib_request,
 )
 from ..utils import (
+    ExtractorError,
     remove_start,
 )
 
@@ -35,8 +38,11 @@ class VideoMegaIE(InfoExtractor):
         req.add_header('Referer', url)
         webpage = self._download_webpage(req, video_id)
 
-        escaped_data = self._search_regex(
-            r'unescape\("([^"]+)"\)', webpage, 'escaped data')
+        try:
+            escaped_data = re.findall(r'unescape\("([^"]+)"\)', webpage)[-1]
+        except IndexError:
+            raise ExtractorError('Unable to extract escaped data')
+
         playlist = compat_urllib_parse.unquote(escaped_data)
 
         thumbnail = self._search_regex(

From 240b9b7a5cd5f19f50ae28fb3ef991fede216ef6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 22 Jan 2015 23:11:22 +0600
Subject: [PATCH 0027/6219] [twitch] Add support for streams (Closes #893,
 closes #3693, closes #1884)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/twitch.py   | 97 +++++++++++++++++++++++++++++++-
 2 files changed, 95 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index be3fecd0d..a983fff76 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -466,6 +466,7 @@ from .twitch import (
     TwitchVodIE,
     TwitchProfileIE,
     TwitchPastBroadcastsIE,
+    TwitchStreamIE,
 )
 from .ubu import UbuIE
 from .udemy import (
diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 8866e5182..2891c4680 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -6,6 +6,7 @@ import re
 
 from .common import InfoExtractor
 from ..compat import (
+    compat_str,
     compat_urllib_parse,
     compat_urllib_request,
 )
@@ -16,9 +17,10 @@ from ..utils import (
 
 
 class TwitchBaseIE(InfoExtractor):
-    _VALID_URL_BASE = r'http://(?:www\.)?twitch\.tv'
+    _VALID_URL_BASE = r'https?://(?:www\.)?twitch\.tv'
 
     _API_BASE = 'https://api.twitch.tv'
+    _USHER_BASE = 'http://usher.twitch.tv'
     _LOGIN_URL = 'https://secure.twitch.tv/user/login'
 
     def _handle_error(self, response):
@@ -191,8 +193,8 @@ class TwitchVodIE(TwitchItemBaseIE):
             '%s/api/vods/%s/access_token' % (self._API_BASE, item_id), item_id,
             'Downloading %s access token' % self._ITEM_TYPE)
         formats = self._extract_m3u8_formats(
-            'http://usher.twitch.tv/vod/%s?nauth=%s&nauthsig=%s'
-            % (item_id, access_token['token'], access_token['sig']),
+            '%s/vod/%s?nauth=%s&nauthsig=%s'
+            % (self._USHER_BASE, item_id, access_token['token'], access_token['sig']),
             item_id, 'mp4')
         info['formats'] = formats
         return info
@@ -254,3 +256,92 @@ class TwitchPastBroadcastsIE(TwitchPlaylistBaseIE):
         },
         'playlist_mincount': 54,
     }
+
+
+class TwitchStreamIE(TwitchBaseIE):
+    IE_NAME = 'twitch:stream'
+    _VALID_URL = r'%s/(?P<id>[^/]+)/?(?:\#.*)?$' % TwitchBaseIE._VALID_URL_BASE
+
+    _TEST = {
+        'url': 'http://www.twitch.tv/shroomztv',
+        'info_dict': {
+            'id': '12772022048',
+            'display_id': 'shroomztv',
+            'ext': 'mp4',
+            'title': 're:^ShroomzTV [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
+            'description': 'H1Z1 - lonewolfing with ShroomzTV | A3 Battle Royale later - @ShroomzTV',
+            'is_live': True,
+            'timestamp': 1421928037,
+            'upload_date': '20150122',
+            'uploader': 'ShroomzTV',
+            'uploader_id': 'shroomztv',
+            'view_count': int,
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }
+
+    def _real_extract(self, url):
+        channel_id = self._match_id(url)
+
+        stream = self._download_json(
+            '%s/kraken/streams/%s' % (self._API_BASE, channel_id), channel_id,
+            'Downloading stream JSON').get('stream')
+
+        # Fallback on profile extraction if stream is offline
+        if not stream:
+            return self.url_result(
+                'http://www.twitch.tv/%s/profile' % channel_id,
+                'TwitchProfile', channel_id)
+
+        access_token = self._download_json(
+            '%s/api/channels/%s/access_token' % (self._API_BASE, channel_id), channel_id,
+            'Downloading channel access token')
+
+        query = {
+            'allow_source': 'true',
+            'p': '9386337',
+            'player': 'twitchweb',
+            'segment_preference': '4',
+            'sig': access_token['sig'],
+            'token': access_token['token'],
+        }
+
+        formats = self._extract_m3u8_formats(
+            '%s/api/channel/hls/%s.m3u8?%s'
+            % (self._USHER_BASE, channel_id, compat_urllib_parse.urlencode(query).encode('utf-8')),
+            channel_id, 'mp4')
+
+        view_count = stream.get('viewers')
+        timestamp = parse_iso8601(stream.get('created_at'))
+
+        channel = stream['channel']
+        title = self._live_title(channel.get('display_name') or channel.get('name'))
+        description = channel.get('status')
+
+        thumbnails = []
+        for thumbnail_key, thumbnail_url in stream['preview'].items():
+            m = re.search(r'(?P<width>\d+)x(?P<height>\d+)\.jpg$', thumbnail_key)
+            if not m:
+                continue
+            thumbnails.append({
+                'url': thumbnail_url,
+                'width': int(m.group('width')),
+                'height': int(m.group('height')),
+            })
+
+        return {
+            'id': compat_str(stream['_id']),
+            'display_id': channel_id,
+            'title': title,
+            'description': description,
+            'thumbnails': thumbnails,
+            'uploader': channel.get('display_name'),
+            'uploader_id': channel.get('name'),
+            'timestamp': timestamp,
+            'view_count': view_count,
+            'formats': formats,
+            'is_live': True,
+        }
\ No newline at end of file

From 7932de6352f1fcb0418e78dc049fe60b92ca6c68 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 22 Jan 2015 18:15:04 +0100
Subject: [PATCH 0028/6219] [hearthisat] Correct error message

---
 youtube_dl/extractor/hearthisat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/hearthisat.py b/youtube_dl/extractor/hearthisat.py
index d83072c1a..a19b31ac0 100644
--- a/youtube_dl/extractor/hearthisat.py
+++ b/youtube_dl/extractor/hearthisat.py
@@ -75,7 +75,7 @@ class HearThisAtIE(InfoExtractor):
         formats = []
         mp3_url = self._search_regex(
             r'(?s)<a class="player-link"\s+(?:[a-zA-Z0-9_:-]+="[^"]+"\s+)*?data-mp3="([^"]+)"',
-            webpage, 'title', fatal=False)
+            webpage, 'mp3 URL', fatal=False)
         if mp3_url:
             formats.append({
                 'format_id': 'mp3',

From 650ab5beebfa555383365602c914729249fad4c6 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 22 Jan 2015 18:15:58 +0100
Subject: [PATCH 0029/6219] [comedycentral:shows] Remove references to colbert
 report

---
 test/test_all_urls.py                 | 2 --
 youtube_dl/extractor/comedycentral.py | 4 ++--
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/test/test_all_urls.py b/test/test_all_urls.py
index bd4fe17bf..88fda76a4 100644
--- a/test/test_all_urls.py
+++ b/test/test_all_urls.py
@@ -115,8 +115,6 @@ class TestAllURLsMatching(unittest.TestCase):
         self.assertMatch(':ythistory', ['youtube:history'])
         self.assertMatch(':thedailyshow', ['ComedyCentralShows'])
         self.assertMatch(':tds', ['ComedyCentralShows'])
-        self.assertMatch(':colbertreport', ['ComedyCentralShows'])
-        self.assertMatch(':cr', ['ComedyCentralShows'])
 
     def test_vimeo_matching(self):
         self.assertMatch('http://vimeo.com/channels/tributes', ['vimeo:channel'])
diff --git a/youtube_dl/extractor/comedycentral.py b/youtube_dl/extractor/comedycentral.py
index 8d27af5e5..15ca361f0 100644
--- a/youtube_dl/extractor/comedycentral.py
+++ b/youtube_dl/extractor/comedycentral.py
@@ -34,12 +34,12 @@ class ComedyCentralIE(MTVServicesInfoExtractor):
 
 class ComedyCentralShowsIE(MTVServicesInfoExtractor):
     IE_DESC = 'The Daily Show / The Colbert Report'
-    # urls can be abbreviations like :thedailyshow or :colbert
+    # urls can be abbreviations like :thedailyshow
     # urls for episodes like:
     # or urls for clips like: http://www.thedailyshow.com/watch/mon-december-10-2012/any-given-gun-day
     #                     or: http://www.colbertnation.com/the-colbert-report-videos/421667/november-29-2012/moon-shattering-news
     #                     or: http://www.colbertnation.com/the-colbert-report-collections/422008/festival-of-lights/79524
-    _VALID_URL = r'''(?x)^(:(?P<shortname>tds|thedailyshow|cr|colbert|colbertnation|colbertreport)
+    _VALID_URL = r'''(?x)^(:(?P<shortname>tds|thedailyshow)
                       |https?://(:www\.)?
                           (?P<showname>thedailyshow|thecolbertreport)\.(?:cc\.)?com/
                          ((?:full-)?episodes/(?:[0-9a-z]{6}/)?(?P<episode>.*)|

From 78111136db788918452a80233e74fd4c506eb6f8 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 22 Jan 2015 18:18:21 +0100
Subject: [PATCH 0030/6219] [twitch] Move URL matching tests into extractor

---
 test/test_all_urls.py          | 13 -------------
 youtube_dl/extractor/twitch.py |  7 +++++--
 2 files changed, 5 insertions(+), 15 deletions(-)

diff --git a/test/test_all_urls.py b/test/test_all_urls.py
index 88fda76a4..e66264b4b 100644
--- a/test/test_all_urls.py
+++ b/test/test_all_urls.py
@@ -14,7 +14,6 @@ from test.helper import gettestcases
 from youtube_dl.extractor import (
     FacebookIE,
     gen_extractors,
-    TwitchIE,
     YoutubeIE,
 )
 
@@ -72,18 +71,6 @@ class TestAllURLsMatching(unittest.TestCase):
         self.assertMatch('http://www.youtube.com/results?search_query=making+mustard', ['youtube:search_url'])
         self.assertMatch('https://www.youtube.com/results?baz=bar&search_query=youtube-dl+test+video&filters=video&lclk=video', ['youtube:search_url'])
 
-    def test_twitch_channelid_matching(self):
-        self.assertTrue(TwitchIE.suitable('twitch.tv/vanillatv'))
-        self.assertTrue(TwitchIE.suitable('www.twitch.tv/vanillatv'))
-        self.assertTrue(TwitchIE.suitable('http://www.twitch.tv/vanillatv'))
-        self.assertTrue(TwitchIE.suitable('http://www.twitch.tv/vanillatv/'))
-
-    def test_twitch_videoid_matching(self):
-        self.assertTrue(TwitchIE.suitable('http://www.twitch.tv/vanillatv/b/328087483'))
-
-    def test_twitch_chapterid_matching(self):
-        self.assertTrue(TwitchIE.suitable('http://www.twitch.tv/tsm_theoddone/c/2349361'))
-
     def test_youtube_extract(self):
         assertExtractId = lambda url, id: self.assertEqual(YoutubeIE.extract_id(url), id)
         assertExtractId('http://www.youtube.com/watch?&v=BaW_jenozKc', 'BaW_jenozKc')
diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 8866e5182..017eff742 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -148,14 +148,17 @@ class TwitchChapterIE(TwitchItemBaseIE):
     _ITEM_TYPE = 'chapter'
     _ITEM_SHORTCUT = 'c'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.twitch.tv/acracingleague/c/5285812',
         'info_dict': {
             'id': 'c5285812',
             'title': 'ACRL Off Season - Sports Cars @ Nordschleife',
         },
         'playlist_mincount': 3,
-    }
+    }, {
+        'url': 'http://www.twitch.tv/tsm_theoddone/c/2349361',
+        'only_matching': True,
+    }]
 
 
 class TwitchVodIE(TwitchItemBaseIE):

From b4a64c592b5f0f547c11c72e6c840612b336de5d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 22 Jan 2015 18:21:17 +0100
Subject: [PATCH 0031/6219] [README] Add an FAQ entry about destination folder

---
 README.md | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/README.md b/README.md
index 60bd7e2b6..efa48f280 100644
--- a/README.md
+++ b/README.md
@@ -488,6 +488,10 @@ To make a different directory work - either for ffmpeg, or for youtube-dl, or fo
 
 From then on, after restarting your shell, you will be able to access both youtube-dl and ffmpeg (and youtube-dl will be able to find ffmpeg) by simply typing `youtube-dl` or `ffmpeg`, no matter what directory you're in.
 
+### How do I put downloads into a specific folder?
+
+Use the `-o` to specify an [output template](#output-template), for example `-o "/home/user/videos/%(title)s-%(id)s.%(ext)s"`. If you want this for all of your downloads, put the option into your [configuration file](#configuration).
+
 ### How can I detect whether a given URL is supported by youtube-dl?
 
 For one, have a look at the [list of supported sites](docs/supportedsites). Note that it can sometimes happen that the site changes its URL scheme (say, from http://example.com/v/1234567 to http://example.com/v/1234567 ) and youtube-dl reports an URL of a service in that list as unsupported. In that case, simply report a bug.

From f353cbdb2f91564d4637462875c1a12bfbcd2ada Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 22 Jan 2015 23:34:40 +0600
Subject: [PATCH 0032/6219] [twitch:stream] Randomize query

---
 youtube_dl/extractor/twitch.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 14314916a..87e3c453d 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -3,6 +3,7 @@ from __future__ import unicode_literals
 
 import itertools
 import re
+import random
 
 from .common import InfoExtractor
 from ..compat import (
@@ -305,7 +306,7 @@ class TwitchStreamIE(TwitchBaseIE):
 
         query = {
             'allow_source': 'true',
-            'p': '9386337',
+            'p': random.randint(1000000, 10000000),
             'player': 'twitchweb',
             'segment_preference': '4',
             'sig': access_token['sig'],

From dc1b027cd4672c6b4772c12daf2a6a2a93469270 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 22 Jan 2015 23:06:03 +0100
Subject: [PATCH 0033/6219] [twitch] PEP8

---
 youtube_dl/extractor/twitch.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 14314916a..6831bb596 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -347,4 +347,4 @@ class TwitchStreamIE(TwitchBaseIE):
             'view_count': view_count,
             'formats': formats,
             'is_live': True,
-        }
\ No newline at end of file
+        }

From 50789175edbe3aaad77f45e6fe883ba09580cc6f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 22 Jan 2015 23:48:58 +0100
Subject: [PATCH 0034/6219] [pornhub] Detect private videos and emit an error
 message (Closes #4764)

---
 youtube_dl/extractor/pornhub.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index 634142d0d..fb2032832 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -10,6 +10,7 @@ from ..compat import (
     compat_urllib_request,
 )
 from ..utils import (
+    ExtractorError,
     str_to_int,
 )
 from ..aes import (
@@ -44,6 +45,15 @@ class PornHubIE(InfoExtractor):
         req.add_header('Cookie', 'age_verified=1')
         webpage = self._download_webpage(req, video_id)
 
+        error_msg = self._html_search_regex(
+            r'(?s)<div class="userMessageSection[^"]*".*?>(.*?)</div>',
+            webpage, 'error message', default=None)
+        if error_msg:
+            error_msg = re.sub(r'\s+', ' ', error_msg)
+            raise ExtractorError(
+                'PornHub said: %s' % error_msg,
+                expected=True, video_id=video_id)
+
         video_title = self._html_search_regex(r'<h1 [^>]+>([^<]+)', webpage, 'title')
         video_uploader = self._html_search_regex(
             r'(?s)From:&nbsp;.+?<(?:a href="/users/|a href="/channels/|<span class="username)[^>]+>(.+?)<',

From 083c9df93b08a24e967b68fbdd2f4a71ae74c8c8 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 23 Jan 2015 00:04:05 +0100
Subject: [PATCH 0035/6219] [YoutubeDL] Allow filtering by properties (Fixes
 #4584)

---
 test/test_YoutubeDL.py  | 55 +++++++++++++++++++++++++++++++++++++++++
 youtube_dl/YoutubeDL.py | 54 ++++++++++++++++++++++++++++++++++++++++
 youtube_dl/options.py   | 11 +++++++++
 3 files changed, 120 insertions(+)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 85d87f2c3..678b9f7d1 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -281,6 +281,61 @@ class TestFormatSelection(unittest.TestCase):
             downloaded = ydl.downloaded_info_dicts[0]
             self.assertEqual(downloaded['format_id'], f1id)
 
+    def test_format_filtering(self):
+        formats = [
+            {'format_id': 'A', 'filesize': 500, 'width': 1000},
+            {'format_id': 'B', 'filesize': 1000, 'width': 500},
+            {'format_id': 'C', 'filesize': 1000, 'width': 400},
+            {'format_id': 'D', 'filesize': 2000, 'width': 600},
+            {'format_id': 'E', 'filesize': 3000},
+            {'format_id': 'F'},
+            {'format_id': 'G', 'filesize': 1000000},
+        ]
+        for f in formats:
+            f['url'] = 'http://_/'
+            f['ext'] = 'unknown'
+        info_dict = _make_result(formats)
+
+        ydl = YDL({'format': 'best[filesize<3000]'})
+        ydl.process_ie_result(info_dict)
+        downloaded = ydl.downloaded_info_dicts[0]
+        self.assertEqual(downloaded['format_id'], 'D')
+
+        ydl = YDL({'format': 'best[filesize<=3000]'})
+        ydl.process_ie_result(info_dict)
+        downloaded = ydl.downloaded_info_dicts[0]
+        self.assertEqual(downloaded['format_id'], 'E')
+
+        ydl = YDL({'format': 'best[filesize <= ? 3000]'})
+        ydl.process_ie_result(info_dict)
+        downloaded = ydl.downloaded_info_dicts[0]
+        self.assertEqual(downloaded['format_id'], 'F')
+
+        ydl = YDL({'format': 'best [filesize = 1000] [width>450]'})
+        ydl.process_ie_result(info_dict)
+        downloaded = ydl.downloaded_info_dicts[0]
+        self.assertEqual(downloaded['format_id'], 'B')
+
+        ydl = YDL({'format': 'best [filesize = 1000] [width!=450]'})
+        ydl.process_ie_result(info_dict)
+        downloaded = ydl.downloaded_info_dicts[0]
+        self.assertEqual(downloaded['format_id'], 'C')
+
+        ydl = YDL({'format': '[filesize>?1]'})
+        ydl.process_ie_result(info_dict)
+        downloaded = ydl.downloaded_info_dicts[0]
+        self.assertEqual(downloaded['format_id'], 'G')
+
+        ydl = YDL({'format': '[filesize<1M]'})
+        ydl.process_ie_result(info_dict)
+        downloaded = ydl.downloaded_info_dicts[0]
+        self.assertEqual(downloaded['format_id'], 'E')
+
+        ydl = YDL({'format': '[filesize<1MiB]'})
+        ydl.process_ie_result(info_dict)
+        downloaded = ydl.downloaded_info_dicts[0]
+        self.assertEqual(downloaded['format_id'], 'G')
+
     def test_add_extra_info(self):
         test_dict = {
             'extractor': 'Foo',
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 772fddd45..8ef74e414 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -10,6 +10,7 @@ import io
 import itertools
 import json
 import locale
+import operator
 import os
 import platform
 import re
@@ -49,6 +50,7 @@ from .utils import (
     make_HTTPS_handler,
     MaxDownloadsReached,
     PagedList,
+    parse_filesize,
     PostProcessingError,
     platform_name,
     preferredencoding,
@@ -768,7 +770,59 @@ class YoutubeDL(object):
         else:
             raise Exception('Invalid result type: %s' % result_type)
 
+    def _apply_format_filter(self, format_spec, available_formats):
+        " Returns a tuple of the remaining format_spec and filtered formats "
+
+        OPERATORS = {
+            '<': operator.lt,
+            '<=': operator.le,
+            '>': operator.gt,
+            '>=': operator.ge,
+            '=': operator.eq,
+            '!=': operator.ne,
+        }
+        operator_rex = re.compile(r'''(?x)\s*\[
+            (?P<key>width|height|tbr|abr|vbr|filesize)
+            \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
+            (?P<value>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)
+            \]$
+            ''' % '|'.join(map(re.escape, OPERATORS.keys())))
+        m = operator_rex.search(format_spec)
+        if not m:
+            raise ValueError('Invalid format specification %r' % format_spec)
+
+        try:
+            comparison_value = int(m.group('value'))
+        except ValueError:
+            comparison_value = parse_filesize(m.group('value'))
+            if comparison_value is None:
+                comparison_value = parse_filesize(m.group('value') + 'B')
+            if comparison_value is None:
+                raise ValueError(
+                    'Invalid value %r in format specification %r' % (
+                        m.group('value'), format_spec))
+        op = OPERATORS[m.group('op')]
+
+        def _filter(f):
+            actual_value = f.get(m.group('key'))
+            if actual_value is None:
+                return m.group('none_inclusive')
+            return op(actual_value, comparison_value)
+        new_formats = [f for f in available_formats if _filter(f)]
+
+        new_format_spec = format_spec[:-len(m.group(0))]
+        if not new_format_spec:
+            new_format_spec = 'best'
+
+        return (new_format_spec, new_formats)
+
     def select_format(self, format_spec, available_formats):
+        while format_spec.endswith(']'):
+            format_spec, available_formats = self._apply_format_filter(
+                format_spec, available_formats)
+        if not available_formats:
+            return None
+
         if format_spec == 'best' or format_spec is None:
             return available_formats[-1]
         elif format_spec == 'worst':
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index f25c12e52..fd7b400b2 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -289,6 +289,17 @@ def parseOpts(overrideArguments=None):
             'extensions aac, m4a, mp3, mp4, ogg, wav, webm. '
             'You can also use the special names "best",'
             ' "bestvideo", "bestaudio", "worst". '
+            ' You can filter the video results by putting a condition in'
+            ' brackets, as in -f "best[height=720]"'
+            ' (or -f "[filesize>10M]"). '
+            ' This works for filesize, height, width, tbr, abr, and vbr'
+            ' and the comparisons <, <=, >, >=, =, != .'
+            ' Formats for which the value is not known are excluded unless you'
+            ' put a question mark (?) after the operator.'
+            ' You can combine format filters, so  '
+            '-f "[height <=? 720][tbr>500]" '
+            'selects up to 720p videos (or videos where the height is not '
+            'known) with a bitrate of at least 500 KBit/s.'
             ' By default, youtube-dl will pick the best quality.'
             ' Use commas to download multiple audio formats, such as'
             ' -f  136/137/mp4/bestvideo,140/m4a/bestaudio.'

From c994e6bd63feb95e1e385af2333ae2f615314ac5 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 23 Jan 2015 00:06:11 +0100
Subject: [PATCH 0036/6219] release 2015.01.23

---
 README.md             | 20 ++++++++++++++++----
 youtube_dl/version.py |  2 +-
 2 files changed, 17 insertions(+), 5 deletions(-)

diff --git a/README.md b/README.md
index efa48f280..1408ebba0 100644
--- a/README.md
+++ b/README.md
@@ -267,10 +267,22 @@ which means you can modify it, redistribute it or use it however you like.
                                      by extension for the extensions aac, m4a,
                                      mp3, mp4, ogg, wav, webm. You can also use
                                      the special names "best", "bestvideo",
-                                     "bestaudio", "worst".  By default, youtube-
-                                     dl will pick the best quality. Use commas
-                                     to download multiple audio formats, such as
-                                     -f
+                                     "bestaudio", "worst".  You can filter the
+                                     video results by putting a condition in
+                                     brackets, as in -f "best[height=720]" (or
+                                     -f "[filesize>10M]").  This works for
+                                     filesize, height, width, tbr, abr, and vbr
+                                     and the comparisons <, <=, >, >=, =, != .
+                                     Formats for which the value is not known
+                                     are excluded unless you put a question mark
+                                     (?) after the operator. You can combine
+                                     format filters, so  -f "[height <=?
+                                     720][tbr>500]" selects up to 720p videos
+                                     (or videos where the height is not known)
+                                     with a bitrate of at least 500 KBit/s. By
+                                     default, youtube-dl will pick the best
+                                     quality. Use commas to download multiple
+                                     audio formats, such as -f
                                      136/137/mp4/bestvideo,140/m4a/bestaudio.
                                      You can merge the video and audio of two
                                      formats into a single file using -f <video-
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index f5df7be31..6c552064b 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.01.22'
+__version__ = '2015.01.23'

From c2a30b250cdea5a8c73225e6a14707a669640de0 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 23 Jan 2015 00:32:35 +0100
Subject: [PATCH 0037/6219] [testtube] Add new extractor (Fixes #4759)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/testtube.py | 60 ++++++++++++++++++++++++++++++++
 2 files changed, 61 insertions(+)
 create mode 100644 youtube_dl/extractor/testtube.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index a983fff76..9ab90ac62 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -432,6 +432,7 @@ from .telemb import TeleMBIE
 from .teletask import TeleTaskIE
 from .tenplay import TenPlayIE
 from .testurl import TestURLIE
+from .testtube import TestTubeIE
 from .tf1 import TF1IE
 from .theonion import TheOnionIE
 from .theplatform import ThePlatformIE
diff --git a/youtube_dl/extractor/testtube.py b/youtube_dl/extractor/testtube.py
new file mode 100644
index 000000000..fd47e71a2
--- /dev/null
+++ b/youtube_dl/extractor/testtube.py
@@ -0,0 +1,60 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import int_or_none
+
+
+class TestTubeIE(InfoExtractor):
+    _VALID_URL = r'https?://testtube\.com/[^/?#]+/(?P<id>[^/?#]+)'
+    _TESTS = [{
+        'url': 'https://testtube.com/dnews/5-weird-ways-plants-can-eat-animals?utm_source=FB&utm_medium=DNews&utm_campaign=DNewsSocial',
+        'info_dict': {
+            'id': '60163',
+            'display_id': '5-weird-ways-plants-can-eat-animals',
+            'duration': 275,
+            'ext': 'mp4',
+            'title': '5 Weird Ways Plants Can Eat Animals',
+            'description': 'Why have some plants evolved to eat meat?',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'uploader': 'DNews',
+            'uploader_id': 'dnews',
+        },
+    }]
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, display_id)
+        video_id = self._search_regex(
+            r"player\.loadRevision3Item\('video_id',\s*([0-9]+)\);",
+            webpage, 'video ID')
+
+        all_info = self._download_json(
+            'https://testtube.com/api/getPlaylist.json?api_key=ba9c741bce1b9d8e3defcc22193f3651b8867e62&codecs=h264,vp8,theora&video_id=%s' % video_id,
+            video_id)
+        info = all_info['items'][0]
+
+        formats = []
+        for vcodec, fdatas in info['media'].items():
+            for name, fdata in fdatas.items():
+                formats.append({
+                    'format_id': '%s-%s' % (vcodec, name),
+                    'url': fdata['url'],
+                    'vcodec': vcodec,
+                    'tbr': fdata.get('bitrate'),
+                })
+        self._sort_formats(formats)
+
+        duration = int_or_none(info.get('duration'))
+
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            'title': info['title'],
+            'description': info.get('summary'),
+            'thumbnail': info.get('images', {}).get('large'),
+            'uploader': info.get('show', {}).get('name'),
+            'uploader_id': info.get('show', {}).get('slug'),
+            'duration': duration,
+            'formats': formats,
+        }

From 317239b097a913824e20d436cdb8c74161564268 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 23 Jan 2015 00:33:14 +0100
Subject: [PATCH 0038/6219] release 2015.01.23.1

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 6c552064b..28458fd69 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.01.23'
+__version__ = '2015.01.23.1'

From 61ca9a80b34b14fa0e5e19e35ee76e0086d49edf Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 23 Jan 2015 01:21:30 +0100
Subject: [PATCH 0039/6219] [generic] Add support for BOMs (Fixes #4753)

---
 test/test_utils.py              | 17 +++++++++++++++++
 youtube_dl/extractor/generic.py |  3 ++-
 youtube_dl/utils.py             | 20 ++++++++++++++++++++
 3 files changed, 39 insertions(+), 1 deletion(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 206760d99..bdd7f268a 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -28,6 +28,7 @@ from youtube_dl.utils import (
     fix_xml_ampersands,
     InAdvancePagedList,
     intlist_to_bytes,
+    is_html,
     js_to_json,
     limit_length,
     OnDemandPagedList,
@@ -417,5 +418,21 @@ ffmpeg version 2.4.4 Copyright (c) 2000-2014 the FFmpeg ...'''), '2.4.4')
         self.assertTrue(age_restricted(18, 14))
         self.assertFalse(age_restricted(18, 18))
 
+    def test_is_html(self):
+        self.assertFalse(is_html(b'\x49\x44\x43<html'))
+        self.assertTrue(is_html(b'<!DOCTYPE foo>\xaaa'))
+        self.assertTrue(is_html(  # UTF-8 with BOM
+            b'\xef\xbb\xbf<!DOCTYPE foo>\xaaa'))
+        self.assertTrue(is_html(  # UTF-16-LE
+            b'\xff\xfe<\x00h\x00t\x00m\x00l\x00>\x00\xe4\x00'
+        ))
+        self.assertTrue(is_html(  # UTF-16-BE
+            b'\xfe\xff\x00<\x00h\x00t\x00m\x00l\x00>\x00\xe4'
+        ))
+        self.assertTrue(is_html(  # UTF-32-BE
+            b'\x00\x00\xFE\xFF\x00\x00\x00<\x00\x00\x00h\x00\x00\x00t\x00\x00\x00m\x00\x00\x00l\x00\x00\x00>\x00\x00\x00\xe4'))
+        self.assertTrue(is_html(  # UTF-32-LE
+            b'\xFF\xFE\x00\x00<\x00\x00\x00h\x00\x00\x00t\x00\x00\x00m\x00\x00\x00l\x00\x00\x00>\x00\x00\x00\xe4\x00\x00\x00'))
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 7a5bf9392..b893d8149 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -17,6 +17,7 @@ from ..utils import (
     ExtractorError,
     float_or_none,
     HEADRequest,
+    is_html,
     orderedSet,
     parse_xml,
     smuggle_url,
@@ -647,7 +648,7 @@ class GenericIE(InfoExtractor):
         # Maybe it's a direct link to a video?
         # Be careful not to download the whole thing!
         first_bytes = full_response.read(512)
-        if not re.match(r'^\s*<', first_bytes.decode('utf-8', 'replace')):
+        if not is_html(first_bytes):
             self._downloader.report_warning(
                 'URL could be a direct video link, returning it as such.')
             upload_date = unified_strdate(
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 764474c33..b433b591b 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1631,3 +1631,23 @@ def age_restricted(content_limit, age_limit):
     if content_limit is None:
         return False  # Content available for everyone
     return age_limit < content_limit
+
+
+def is_html(first_bytes):
+    """ Detect whether a file contains HTML by examining its first bytes. """
+
+    BOMS = [
+        (b'\xef\xbb\xbf', 'utf-8'),
+        (b'\x00\x00\xfe\xff', 'utf-32-be'),
+        (b'\xff\xfe\x00\x00', 'utf-32-le'),
+        (b'\xff\xfe', 'utf-16-le'),
+        (b'\xfe\xff', 'utf-16-be'),
+    ]
+    for bom, enc in BOMS:
+        if first_bytes.startswith(bom):
+            s = first_bytes[len(bom):].decode(enc, 'replace')
+            break
+    else:
+        s = first_bytes.decode('utf-8', 'replace')
+
+    return re.match(r'^\s*<', s)

From 08ff6ab07e9d08853ca19cf3b8745fac11abf0f6 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 23 Jan 2015 01:34:24 +0100
Subject: [PATCH 0040/6219] [gamestar] Modernize slightly

---
 youtube_dl/extractor/gamestar.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/gamestar.py b/youtube_dl/extractor/gamestar.py
index 50f8fc7e7..7591a151e 100644
--- a/youtube_dl/extractor/gamestar.py
+++ b/youtube_dl/extractor/gamestar.py
@@ -1,8 +1,6 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..utils import (
     int_or_none,
@@ -29,9 +27,7 @@ class GameStarIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
         og_title = self._og_search_title(webpage)

From d79323136fabc2cd72afc7c124e17797e32df514 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 23 Jan 2015 11:15:18 +0100
Subject: [PATCH 0041/6219] [utils] Simplify HTTPS socket creation

We were duplicating (bad) code and doing crazy things with SSL.
Just use TLSv1 across the board, and do with one implementation of HTTPSConnection.connect.
Fixes #4696.
---
 youtube_dl/utils.py | 24 +++++-------------------
 1 file changed, 5 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 3536a5bd6..4be323926 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -411,25 +411,9 @@ def make_HTTPS_handler(params, **kwargs):
             pass
 
     if sys.version_info < (3, 2):
-        import httplib
-
-        class HTTPSConnectionV3(httplib.HTTPSConnection):
-            def __init__(self, *args, **kwargs):
-                httplib.HTTPSConnection.__init__(self, *args, **kwargs)
-
-            def connect(self):
-                sock = socket.create_connection((self.host, self.port), self.timeout)
-                if getattr(self, '_tunnel_host', False):
-                    self.sock = sock
-                    self._tunnel()
-                try:
-                    self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file, ssl_version=ssl.PROTOCOL_TLSv1)
-                except ssl.SSLError:
-                    self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file, ssl_version=ssl.PROTOCOL_SSLv23)
-
-        return YoutubeDLHTTPSHandler(params, https_conn_class=HTTPSConnectionV3, **kwargs)
+        return YoutubeDLHTTPSHandler(params, **kwargs)
     else:  # Python < 3.4
-        context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
+        context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
         context.verify_mode = (ssl.CERT_NONE
                                if opts_no_check_certificate
                                else ssl.CERT_REQUIRED)
@@ -560,7 +544,9 @@ def _create_http_connection(ydl_handler, http_class, is_https, *args, **kwargs):
                 sock = compat_socket_create_connection(
                     (self.host, self.port), self.timeout, sa)
                 if is_https:
-                    self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file)
+                    self.sock = ssl.wrap_socket(
+                        sock, self.key_file, self.cert_file,
+                        ssl_version=ssl.PROTOCOL_TLSv1)
                 else:
                     self.sock = sock
             hc.connect = functools.partial(_hc_connect, hc)

From ba551681573aafedf9a0a234a9522cc06b12a4b7 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 23 Jan 2015 11:20:24 +0100
Subject: [PATCH 0042/6219] release 2015.01.23.2

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 28458fd69..674c2f279 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.01.23.1'
+__version__ = '2015.01.23.2'

From 501f13fbf3d1f7225f91e3e0ad008df2cd3219f1 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 23 Jan 2015 12:00:25 +0100
Subject: [PATCH 0043/6219] [generic] Add support for Cinerama player (Fixes
 #4752)

---
 youtube_dl/extractor/generic.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index b893d8149..a028c4ed4 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -489,6 +489,16 @@ class GenericIE(InfoExtractor):
                 'title': 'Jack Tips: 5 Steps to Permanent Gut Healing',
             }
         },
+        # Cinerama player
+        {
+            'url': 'http://www.abc.net.au/7.30/content/2015/s4164797.htm',
+            'info_dict': {
+                'id': '730m_DandD_1901_512k',
+                'ext': 'mp4',
+                'uploader': 'www.abc.net.au',
+                'title': 'Game of Thrones with dice - Dungeons and Dragons fantasy role-playing game gets new life - 19/01/2015',
+            }
+        }
     ]
 
     def report_following_redirect(self, new_url):
@@ -1046,6 +1056,10 @@ class GenericIE(InfoExtractor):
                     \s*{[^}]+? ["']?clip["']?\s*:\s*\{\s*
                         ["']?url["']?\s*:\s*["']([^"']+)["']
             ''', webpage))
+        if not found:
+            # Cinerama player
+            found = re.findall(
+                r"cinerama\.embedPlayer\(\s*\'[^']+\',\s*'([^']+)'", webpage)
         if not found:
             # Try to find twitter cards info
             found = filter_video(re.findall(

From 649f7966f73b215f02b9c4db5a07d8c4b7e17acb Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 23 Jan 2015 12:07:13 +0100
Subject: [PATCH 0044/6219] Fix --sleep-interval (#3426)

---
 youtube_dl/__init__.py | 6 ------
 youtube_dl/options.py  | 2 +-
 2 files changed, 1 insertion(+), 7 deletions(-)

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index ea1660452..7bd7295e2 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -132,11 +132,6 @@ def _real_main(argv=None):
         if numeric_limit is None:
             parser.error('invalid rate limit specified')
         opts.ratelimit = numeric_limit
-    if opts.sleepinterval is not None:
-        try:
-            opts.sleepinterval = abs(int(opts.sleepinterval))
-        except ValueError:
-            parser.error(u'invalid sleep interval specified')
     if opts.min_filesize is not None:
         numeric_limit = FileDownloader.parse_bytes(opts.min_filesize)
         if numeric_limit is None:
@@ -272,7 +267,6 @@ def _real_main(argv=None):
         'restrictfilenames': opts.restrictfilenames,
         'ignoreerrors': opts.ignoreerrors,
         'ratelimit': opts.ratelimit,
-        'sleepinterval': opts.sleepinterval,
         'nooverwrites': opts.nooverwrites,
         'retries': opts.retries,
         'buffersize': opts.buffersize,
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 12c9826f8..262c60013 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -423,7 +423,7 @@ def parseOpts(overrideArguments=None):
         help='Work around terminals that lack bidirectional text support. Requires bidiv or fribidi executable in PATH')
     workarounds.add_option(
         '--sleep-interval', metavar='SECONDS',
-        dest='sleep_interval',
+        dest='sleep_interval', type=float,
         help='Number of seconds to sleep before each download.')
 
     verbosity = optparse.OptionGroup(parser, 'Verbosity / Simulation Options')

From aa42e87340e491d0b151e9dad368711fc275c7b9 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 23 Jan 2015 12:17:12 +0100
Subject: [PATCH 0045/6219] [utils] Catch strange Windows errors (Closes #4733)

---
 youtube_dl/utils.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 4be323926..463cc20ff 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -863,6 +863,9 @@ def _windows_write_string(s, out):
     except AttributeError:
         # If the output stream doesn't have a fileno, it's virtual
         return False
+    except io.UnsupportedOperation:
+        # Some strange Windows pseudo files?
+        return False
     if fileno not in WIN_OUTPUT_IDS:
         return False
 

From 6f58db89820fe03effb2014ee70e1ffd0b1f3017 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 23 Jan 2015 12:17:19 +0100
Subject: [PATCH 0046/6219] release 2015.01.23.3

---
 README.md             | 2 ++
 youtube_dl/version.py | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 1408ebba0..869523720 100644
--- a/README.md
+++ b/README.md
@@ -259,6 +259,8 @@ which means you can modify it, redistribute it or use it however you like.
     --bidi-workaround                Work around terminals that lack
                                      bidirectional text support. Requires bidiv
                                      or fribidi executable in PATH
+    --sleep-interval SECONDS         Number of seconds to sleep before each
+                                     download.
 
 ## Video Format Options:
     -f, --format FORMAT              video format code, specify the order of
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 674c2f279..f39b94361 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.01.23.2'
+__version__ = '2015.01.23.3'

From bf7fa94ec7202bde963a75ab903996ac575910db Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 23 Jan 2015 16:31:52 +0100
Subject: [PATCH 0047/6219] [downloader/f4m] build_fragments_list: Support
 videos with more than 1 segment

---
 youtube_dl/downloader/f4m.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index c460c167a..c68b2c303 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -177,13 +177,12 @@ def build_fragments_list(boot_info):
     """ Return a list of (segment, fragment) for each fragment in the video """
     res = []
     segment_run_table = boot_info['segments'][0]
-    # I've only found videos with one segment
-    segment_run_entry = segment_run_table['segment_run'][0]
-    n_frags = segment_run_entry[1]
     fragment_run_entry_table = boot_info['fragments'][0]['fragments']
     first_frag_number = fragment_run_entry_table[0]['first']
-    for (i, frag_number) in zip(range(1, n_frags + 1), itertools.count(first_frag_number)):
-        res.append((1, frag_number))
+    fragments_counter = itertools.count(first_frag_number)
+    for segment, fragments_count in segment_run_table['segment_run']:
+        for _ in range(fragments_count):
+            res.append((segment, next(fragments_counter)))
     return res
 
 

From 0920e5830f890580ec16cdd10bfe8def73a1a09f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 23 Jan 2015 16:39:23 +0100
Subject: [PATCH 0048/6219] [atresplayer] Don't include f4m formats if they are
 protected by DRM (fixes #4705)

---
 youtube_dl/extractor/atresplayer.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/atresplayer.py b/youtube_dl/extractor/atresplayer.py
index 5db1941b3..37321ef1d 100644
--- a/youtube_dl/extractor/atresplayer.py
+++ b/youtube_dl/extractor/atresplayer.py
@@ -107,7 +107,14 @@ class AtresPlayerIE(InfoExtractor):
 
             for _, video_url in fmt_json['resultObject'].items():
                 if video_url.endswith('/Manifest'):
-                    formats.extend(self._extract_f4m_formats(video_url[:-9] + '/manifest.f4m', video_id))
+                    if 'geodeswowsmpra3player' in video_url:
+                        f4m_path = video_url.split('smil:', 1)[-1].split('free_', 1)[0]
+                        f4m_url = 'http://drg.antena3.com/{0}hds/es/sd.f4m'.format(f4m_path)
+                        # this videos are protected by DRM, the f4m downloader doesn't support them
+                        continue
+                    else:
+                        f4m_url = video_url[:-9] + '/manifest.f4m'
+                    formats.extend(self._extract_f4m_formats(f4m_url, video_id))
                 else:
                     formats.append({
                         'url': video_url,

From c2e64f71d075bbc916f343916ff1e679f642a821 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 23 Jan 2015 21:58:40 +0600
Subject: [PATCH 0049/6219] [twitch] Add support for bookmarks

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/twitch.py   | 41 ++++++++++++++++++++++++++++----
 2 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 9ab90ac62..9e1ce5db3 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -467,6 +467,7 @@ from .twitch import (
     TwitchVodIE,
     TwitchProfileIE,
     TwitchPastBroadcastsIE,
+    TwitchBookmarksIE,
     TwitchStreamIE,
 )
 from .ubu import UbuIE
diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 340cadcf5..741df7cbc 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -220,12 +220,18 @@ class TwitchPlaylistBaseIE(TwitchBaseIE):
             response = self._download_json(
                 self._PLAYLIST_URL % (channel_id, offset, limit),
                 channel_id, 'Downloading %s videos JSON page %d' % (self._PLAYLIST_TYPE, counter))
-            videos = response['videos']
-            if not videos:
+            page_entries = self._extract_playlist_page(response)
+            if not page_entries:
                 break
-            entries.extend([self.url_result(video['url']) for video in videos])
+            entries.extend(page_entries)
             offset += limit
-        return self.playlist_result(entries, channel_id, channel_name)
+        return self.playlist_result(
+            [self.url_result(entry) for entry in set(entries)],
+            channel_id, channel_name)
+
+    def _extract_playlist_page(self, response):
+        videos = response.get('videos')
+        return [video['url'] for video in videos] if videos else []
 
     def _real_extract(self, url):
         return self._extract_playlist(self._match_id(url))
@@ -262,6 +268,31 @@ class TwitchPastBroadcastsIE(TwitchPlaylistBaseIE):
     }
 
 
+class TwitchBookmarksIE(TwitchPlaylistBaseIE):
+    IE_NAME = 'twitch:bookmarks'
+    _VALID_URL = r'%s/(?P<id>[^/]+)/profile/bookmarks/?(?:\#.*)?$' % TwitchBaseIE._VALID_URL_BASE
+    _PLAYLIST_URL = '%s/api/bookmark/?user=%%s&offset=%%d&limit=%%d' % TwitchBaseIE._API_BASE
+    _PLAYLIST_TYPE = 'bookmarks'
+
+    _TEST = {
+        'url': 'http://www.twitch.tv/ognos/profile/bookmarks',
+        'info_dict': {
+            'id': 'ognos',
+            'title': 'Ognos',
+        },
+        'playlist_mincount': 3,
+    }
+
+    def _extract_playlist_page(self, response):
+        entries = []
+        for bookmark in response.get('bookmarks', []):
+            video = bookmark.get('video')
+            if not video:
+                continue
+            entries.append(video['url'])
+        return entries
+
+
 class TwitchStreamIE(TwitchBaseIE):
     IE_NAME = 'twitch:stream'
     _VALID_URL = r'%s/(?P<id>[^/]+)/?(?:\#.*)?$' % TwitchBaseIE._VALID_URL_BASE
@@ -348,4 +379,4 @@ class TwitchStreamIE(TwitchBaseIE):
             'view_count': view_count,
             'formats': formats,
             'is_live': True,
-        }
+        }
\ No newline at end of file

From e793f7671c3e303e4fbe2c4cd18e80ad4b5e5417 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 23 Jan 2015 17:09:26 +0100
Subject: [PATCH 0050/6219] [liveleak] Modernize

---
 youtube_dl/extractor/liveleak.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/liveleak.py b/youtube_dl/extractor/liveleak.py
index b04be1e8c..e56a4d6b1 100644
--- a/youtube_dl/extractor/liveleak.py
+++ b/youtube_dl/extractor/liveleak.py
@@ -8,7 +8,7 @@ from ..utils import int_or_none
 
 
 class LiveLeakIE(InfoExtractor):
-    _VALID_URL = r'^(?:http://)?(?:\w+\.)?liveleak\.com/view\?(?:.*?)i=(?P<video_id>[\w_]+)(?:.*)'
+    _VALID_URL = r'https?://(?:\w+\.)?liveleak\.com/view\?(?:.*?)i=(?P<id>[\w_]+)(?:.*)'
     _TESTS = [{
         'url': 'http://www.liveleak.com/view?i=757_1364311680',
         'md5': '0813c2430bea7a46bf13acf3406992f4',
@@ -43,8 +43,7 @@ class LiveLeakIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('video_id')
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
         video_title = self._og_search_title(webpage).replace('LiveLeak.com -', '').strip()

From ebd46aed5119899826629cf751ba5abe7a65d50b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 23 Jan 2015 22:21:55 +0600
Subject: [PATCH 0051/6219] [atresplayer] Filter URLs and clarify android
 format ids

---
 youtube_dl/extractor/atresplayer.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/atresplayer.py b/youtube_dl/extractor/atresplayer.py
index 37321ef1d..8fd69b971 100644
--- a/youtube_dl/extractor/atresplayer.py
+++ b/youtube_dl/extractor/atresplayer.py
@@ -105,7 +105,9 @@ class AtresPlayerIE(InfoExtractor):
                 raise ExtractorError(
                     '%s returned error: %s' % (self.IE_NAME, result), expected=True)
 
-            for _, video_url in fmt_json['resultObject'].items():
+            for format_id, video_url in fmt_json['resultObject'].items():
+                if format_id == 'token' or not video_url.startswith('http'):
+                    continue
                 if video_url.endswith('/Manifest'):
                     if 'geodeswowsmpra3player' in video_url:
                         f4m_path = video_url.split('smil:', 1)[-1].split('free_', 1)[0]
@@ -118,7 +120,7 @@ class AtresPlayerIE(InfoExtractor):
                 else:
                     formats.append({
                         'url': video_url,
-                        'format_id': 'android',
+                        'format_id': 'android-%s' % format_id,
                         'preference': 1,
                     })
         self._sort_formats(formats)

From 26e274666d285026dfb6033cdad74cc2a108e2bc Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 23 Jan 2015 17:22:14 +0100
Subject: [PATCH 0052/6219] [liveleak] Add original videos (Fixes #4768)

---
 youtube_dl/extractor/liveleak.py | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/liveleak.py b/youtube_dl/extractor/liveleak.py
index e56a4d6b1..35822067f 100644
--- a/youtube_dl/extractor/liveleak.py
+++ b/youtube_dl/extractor/liveleak.py
@@ -11,17 +11,17 @@ class LiveLeakIE(InfoExtractor):
     _VALID_URL = r'https?://(?:\w+\.)?liveleak\.com/view\?(?:.*?)i=(?P<id>[\w_]+)(?:.*)'
     _TESTS = [{
         'url': 'http://www.liveleak.com/view?i=757_1364311680',
-        'md5': '0813c2430bea7a46bf13acf3406992f4',
+        'md5': '50f79e05ba149149c1b4ea961223d5b3',
         'info_dict': {
             'id': '757_1364311680',
-            'ext': 'mp4',
+            'ext': 'flv',
             'description': 'extremely bad day for this guy..!',
             'uploader': 'ljfriel2',
             'title': 'Most unlucky car accident'
         }
     }, {
         'url': 'http://www.liveleak.com/view?i=f93_1390833151',
-        'md5': 'd3f1367d14cc3c15bf24fbfbe04b9abf',
+        'md5': 'b13a29626183c9d33944e6a04f41aafc',
         'info_dict': {
             'id': 'f93_1390833151',
             'ext': 'mp4',
@@ -80,9 +80,19 @@ class LiveLeakIE(InfoExtractor):
         sources = json.loads(sources_json)
 
         formats = [{
+            'format_id': '%s' % i,
             'format_note': s.get('label'),
             'url': s['file'],
-        } for s in sources]
+        } for i, s in enumerate(sources)]
+        for i, s in enumerate(sources):
+            orig_url = s['file'].replace('.h264_base.mp4', '')
+            if s['file'] != orig_url:
+                formats.append({
+                    'format_id': 'original-%s' % i,
+                    'format_note': s.get('label'),
+                    'url': orig_url,
+                    'preference': 1,
+                })
         self._sort_formats(formats)
 
         return {

From 0c17278843a30d4aea1f8334985e6fbae17000d9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 23 Jan 2015 22:54:29 +0600
Subject: [PATCH 0053/6219] [atresplayer] Extract subtitles

---
 youtube_dl/extractor/atresplayer.py | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/atresplayer.py b/youtube_dl/extractor/atresplayer.py
index 8fd69b971..f42862be3 100644
--- a/youtube_dl/extractor/atresplayer.py
+++ b/youtube_dl/extractor/atresplayer.py
@@ -3,7 +3,7 @@ from __future__ import unicode_literals
 import time
 import hmac
 
-from .common import InfoExtractor
+from .subtitles import SubtitlesInfoExtractor
 from ..compat import (
     compat_str,
     compat_urllib_parse,
@@ -17,7 +17,7 @@ from ..utils import (
 )
 
 
-class AtresPlayerIE(InfoExtractor):
+class AtresPlayerIE(SubtitlesInfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?atresplayer\.com/television/[^/]+/[^/]+/[^/]+/(?P<id>.+?)_\d+\.html'
     _TESTS = [
         {
@@ -143,6 +143,15 @@ class AtresPlayerIE(InfoExtractor):
         description = xpath_text(art, './description', 'description')
         thumbnail = xpath_text(episode, './media/asset/files/background', 'thumbnail')
 
+        subtitles = {}
+        subtitle = xpath_text(episode, './media/asset/files/subtitle', 'subtitle')
+        if subtitle:
+            subtitles['es'] = subtitle
+
+        if self._downloader.params.get('listsubtitles', False):
+            self._list_available_subtitles(video_id, subtitles)
+            return
+
         return {
             'id': video_id,
             'title': title,
@@ -150,4 +159,5 @@ class AtresPlayerIE(InfoExtractor):
             'thumbnail': thumbnail,
             'duration': duration,
             'formats': formats,
+            'subtitles': self.extract_subtitles(video_id, subtitles),
         }

From 62cd676c7474f696804eda653558ada94c5953a0 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 23 Jan 2015 18:39:12 +0100
Subject: [PATCH 0054/6219] [youtube] Fixup DASH m4a headers

This fixes #2288, #2506, #2607, #3681, #4741, #4767.
---
 youtube_dl/YoutubeDL.py              | 28 +++++++++++++++++++++++-----
 youtube_dl/extractor/youtube.py      |  6 +++---
 youtube_dl/postprocessor/__init__.py |  2 ++
 youtube_dl/postprocessor/ffmpeg.py   | 20 +++++++++++++++++++-
 4 files changed, 47 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 8f34b17b4..521e4055e 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -73,6 +73,7 @@ from .extractor import get_info_extractor, gen_extractors
 from .downloader import get_suitable_downloader
 from .downloader.rtmp import rtmpdump_version
 from .postprocessor import (
+    FFmpegFixupM4aPP,
     FFmpegFixupStretchedPP,
     FFmpegMergerPP,
     FFmpegPostProcessor,
@@ -213,7 +214,7 @@ class YoutubeDL(object):
                        - "never": do nothing
                        - "warn": only emit a warning
                        - "detect_or_warn": check whether we can do anything
-                                           about it, warn otherwise
+                                           about it, warn otherwise (default)
     source_address:    (Experimental) Client-side IP address to bind to.
     call_home:         Boolean, true iff we are allowed to contact the
                        youtube-dl servers for debugging.
@@ -1219,11 +1220,12 @@ class YoutubeDL(object):
 
             if success:
                 # Fixup content
+                fixup_policy = self.params.get('fixup')
+                if fixup_policy is None:
+                    fixup_policy = 'detect_or_warn'
+
                 stretched_ratio = info_dict.get('stretched_ratio')
                 if stretched_ratio is not None and stretched_ratio != 1:
-                    fixup_policy = self.params.get('fixup')
-                    if fixup_policy is None:
-                        fixup_policy = 'detect_or_warn'
                     if fixup_policy == 'warn':
                         self.report_warning('%s: Non-uniform pixel ratio (%s)' % (
                             info_dict['id'], stretched_ratio))
@@ -1237,7 +1239,23 @@ class YoutubeDL(object):
                                 '%s: Non-uniform pixel ratio (%s). Install ffmpeg or avconv to fix this automatically.' % (
                                     info_dict['id'], stretched_ratio))
                     else:
-                        assert fixup_policy == 'ignore'
+                        assert fixup_policy in ('ignore', 'never')
+
+                if info_dict.get('requested_formats') is None and info_dict.get('container') == 'm4a_dash':
+                    if fixup_policy == 'warn':
+                        self.report_warning('%s: writing DASH m4a. Only some players support this container.' % (
+                            info_dict['id']))
+                    elif fixup_policy == 'detect_or_warn':
+                        fixup_pp = FFmpegFixupM4aPP(self)
+                        if fixup_pp.available:
+                            info_dict.setdefault('__postprocessors', [])
+                            info_dict['__postprocessors'].append(fixup_pp)
+                        else:
+                            self.report_warning(
+                                '%s: writing DASH m4a. Only some players support this container. Install ffmpeg or avconv to fix this automatically.' % (
+                                    info_dict['id']))
+                    else:
+                        assert fixup_policy in ('ignore', 'never')
 
                 try:
                     self.post_process(filename, info_dict)
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 2a1f8be0a..eb55d24ce 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -264,9 +264,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
         '266': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'vcodec': 'h264'},
 
         # Dash mp4 audio
-        '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'vcodec': 'none', 'abr': 48, 'preference': -50},
-        '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'vcodec': 'none', 'abr': 128, 'preference': -50},
-        '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'vcodec': 'none', 'abr': 256, 'preference': -50},
+        '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'vcodec': 'none', 'abr': 48, 'preference': -50, 'container': 'm4a_dash'},
+        '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'vcodec': 'none', 'abr': 128, 'preference': -50, 'container': 'm4a_dash'},
+        '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'vcodec': 'none', 'abr': 256, 'preference': -50, 'container': 'm4a_dash'},
 
         # Dash webm
         '167': {'ext': 'webm', 'height': 360, 'width': 640, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'VP8', 'preference': -40},
diff --git a/youtube_dl/postprocessor/__init__.py b/youtube_dl/postprocessor/__init__.py
index f8507951c..0ffbca258 100644
--- a/youtube_dl/postprocessor/__init__.py
+++ b/youtube_dl/postprocessor/__init__.py
@@ -7,6 +7,7 @@ from .ffmpeg import (
     FFmpegEmbedSubtitlePP,
     FFmpegExtractAudioPP,
     FFmpegFixupStretchedPP,
+    FFmpegFixupM4aPP,
     FFmpegMergerPP,
     FFmpegMetadataPP,
     FFmpegVideoConvertorPP,
@@ -25,6 +26,7 @@ __all__ = [
     'FFmpegAudioFixPP',
     'FFmpegEmbedSubtitlePP',
     'FFmpegExtractAudioPP',
+    'FFmpegFixupM4aPP',
     'FFmpegFixupStretchedPP',
     'FFmpegMergerPP',
     'FFmpegMetadataPP',
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index faccdc43d..855d1e6db 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -564,7 +564,7 @@ class FFmpegFixupStretchedPP(FFmpegPostProcessor):
     def run(self, info):
         stretched_ratio = info.get('stretched_ratio')
         if stretched_ratio is None or stretched_ratio == 1:
-            return
+            return True, info
 
         filename = info['filepath']
         temp_filename = prepend_extension(filename, 'temp')
@@ -577,3 +577,21 @@ class FFmpegFixupStretchedPP(FFmpegPostProcessor):
         os.rename(encodeFilename(temp_filename), encodeFilename(filename))
 
         return True, info
+
+
+class FFmpegFixupM4aPP(FFmpegPostProcessor):
+    def run(self, info):
+        if info.get('container') != 'm4a_dash':
+            return True, info
+
+        filename = info['filepath']
+        temp_filename = prepend_extension(filename, 'temp')
+
+        options = ['-c', 'copy', '-f', 'mp4']
+        self._downloader.to_screen('[ffmpeg] Correcting container in "%s"' % filename)
+        self.run_ffmpeg(filename, temp_filename, options)
+
+        os.remove(encodeFilename(filename))
+        os.rename(encodeFilename(temp_filename), encodeFilename(filename))
+
+        return True, info

From 6896a52721ce3a0dc4f33fa5dda508c30c56ce22 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 23 Jan 2015 18:58:32 +0100
Subject: [PATCH 0055/6219] release 2015.01.23.4

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index f39b94361..35f3e1b6b 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.01.23.3'
+__version__ = '2015.01.23.4'

From 3e055aa5c3c069476b1ac9fc103dc1386cad763c Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Fri, 23 Jan 2015 21:22:49 +0200
Subject: [PATCH 0056/6219] [cliphunter] Fix extraction and update test (Fixes
 #4362)

---
 youtube_dl/extractor/cliphunter.py | 56 ++++++++++++------------------
 1 file changed, 23 insertions(+), 33 deletions(-)

diff --git a/youtube_dl/extractor/cliphunter.py b/youtube_dl/extractor/cliphunter.py
index 2edab90a3..d46592cc5 100644
--- a/youtube_dl/extractor/cliphunter.py
+++ b/youtube_dl/extractor/cliphunter.py
@@ -1,9 +1,7 @@
 from __future__ import unicode_literals
 
-import json
-import re
-
 from .common import InfoExtractor
+from ..utils import determine_ext
 
 
 _translation_table = {
@@ -27,10 +25,10 @@ class CliphunterIE(InfoExtractor):
     '''
     _TEST = {
         'url': 'http://www.cliphunter.com/w/1012420/Fun_Jynx_Maze_solo',
-        'md5': 'a2ba71eebf523859fe527a61018f723e',
+        'md5': 'b7c9bbd4eb3a226ab91093714dcaa480',
         'info_dict': {
             'id': '1012420',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': 'Fun Jynx Maze solo',
             'thumbnail': 're:^https?://.*\.jpg$',
             'age_limit': 18,
@@ -44,39 +42,31 @@ class CliphunterIE(InfoExtractor):
         video_title = self._search_regex(
             r'mediaTitle = "([^"]+)"', webpage, 'title')
 
-        pl_fiji = self._search_regex(
-            r'pl_fiji = \'([^\']+)\'', webpage, 'video data')
-        pl_c_qual = self._search_regex(
-            r'pl_c_qual = "(.)"', webpage, 'video quality')
-        video_url = _decode(pl_fiji)
-        formats = [{
-            'url': video_url,
-            'format_id': 'default-%s' % pl_c_qual,
-        }]
+        fmts = {}
+        for fmt in ('mp4', 'flv'):
+            fmt_list = self._parse_json(self._search_regex(
+                r'var %sjson\s*=\s*(\[.*?\]);' % fmt, webpage, '%s formats' % fmt), video_id)
+            for f in fmt_list:
+                fmts[f['fname']] = _decode(f['sUrl'])
 
-        qualities_json = self._search_regex(
-            r'var pl_qualities\s*=\s*(.*?);\n', webpage, 'quality info')
-        qualities_data = json.loads(qualities_json)
+        qualities = self._parse_json(self._search_regex(
+            r'var player_btns\s*=\s*(.*?);\n', webpage, 'quality info'), video_id)
 
-        for i, t in enumerate(
-                re.findall(r"pl_fiji_([a-z0-9]+)\s*=\s*'([^']+')", webpage)):
-            quality_id, crypted_url = t
-            video_url = _decode(crypted_url)
+        formats = []
+        for fname, url in fmts.items():
             f = {
-                'format_id': quality_id,
-                'url': video_url,
-                'quality': i,
+                'url': url,
             }
-            if quality_id in qualities_data:
-                qd = qualities_data[quality_id]
-                m = re.match(
-                    r'''(?x)<b>(?P<width>[0-9]+)x(?P<height>[0-9]+)<\\/b>
-                        \s*\(\s*(?P<tbr>[0-9]+)\s*kb\\/s''', qd)
-                if m:
-                    f['width'] = int(m.group('width'))
-                    f['height'] = int(m.group('height'))
-                    f['tbr'] = int(m.group('tbr'))
+            if fname in qualities:
+                qual = qualities[fname]
+                f.update({
+                    'format_id': '%s_%sp' % (determine_ext(url), qual['h']),
+                    'width': qual['w'],
+                    'height': qual['h'],
+                    'tbr': qual['br'],
+                })
             formats.append(f)
+
         self._sort_formats(formats)
 
         thumbnail = self._search_regex(

From 48f00d15b1bdbe05be397849f6659fe5e0daa9c1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 23 Jan 2015 21:03:00 +0100
Subject: [PATCH 0057/6219] [auengine] Remove extractor

The test is probably infringing copyright and nobody has provided a new test (see #4643).
---
 youtube_dl/extractor/__init__.py |  1 -
 youtube_dl/extractor/auengine.py | 50 --------------------------------
 2 files changed, 51 deletions(-)
 delete mode 100644 youtube_dl/extractor/auengine.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 9e1ce5db3..3f7ca6f7d 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -29,7 +29,6 @@ from .arte import (
 from .atresplayer import AtresPlayerIE
 from .atttechchannel import ATTTechChannelIE
 from .audiomack import AudiomackIE, AudiomackAlbumIE
-from .auengine import AUEngineIE
 from .azubu import AzubuIE
 from .bambuser import BambuserIE, BambuserChannelIE
 from .bandcamp import BandcampIE, BandcampAlbumIE
diff --git a/youtube_dl/extractor/auengine.py b/youtube_dl/extractor/auengine.py
deleted file mode 100644
index a1b666be0..000000000
--- a/youtube_dl/extractor/auengine.py
+++ /dev/null
@@ -1,50 +0,0 @@
-from __future__ import unicode_literals
-
-import re
-
-from .common import InfoExtractor
-from ..compat import compat_urllib_parse
-from ..utils import (
-    determine_ext,
-    ExtractorError,
-    remove_end,
-)
-
-
-class AUEngineIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?auengine\.com/embed\.php\?.*?file=(?P<id>[^&]+).*?'
-
-    _TEST = {
-        'url': 'http://auengine.com/embed.php?file=lfvlytY6&w=650&h=370',
-        'md5': '48972bdbcf1a3a2f5533e62425b41d4f',
-        'info_dict': {
-            'id': 'lfvlytY6',
-            'ext': 'mp4',
-            'title': '[Commie]The Legend of the Legendary Heroes - 03 - Replication Eye (Alpha Stigma)[F9410F5A]'
-        }
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        webpage = self._download_webpage(url, video_id)
-        title = self._html_search_regex(
-            r'<title>\s*(?P<title>.+?)\s*</title>', webpage, 'title')
-        video_urls = re.findall(r'http://\w+.auengine.com/vod/.*[^\W]', webpage)
-        video_url = compat_urllib_parse.unquote(video_urls[0])
-        thumbnails = re.findall(r'http://\w+.auengine.com/thumb/.*[^\W]', webpage)
-        thumbnail = compat_urllib_parse.unquote(thumbnails[0])
-
-        if not video_url:
-            raise ExtractorError('Could not find video URL')
-
-        ext = '.' + determine_ext(video_url)
-        title = remove_end(title, ext)
-
-        return {
-            'id': video_id,
-            'url': video_url,
-            'title': title,
-            'thumbnail': thumbnail,
-            'http_referer': 'http://www.auengine.com/flowplayer/flowplayer.commercial-3.2.14.swf',
-        }

From 12d1fb5aa9aabfe48676d6c60bbd114cc53a513f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 23 Jan 2015 21:05:07 +0100
Subject: [PATCH 0058/6219] [twitch] PEP8

---
 youtube_dl/extractor/twitch.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 741df7cbc..87290d002 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -379,4 +379,4 @@ class TwitchStreamIE(TwitchBaseIE):
             'view_count': view_count,
             'formats': formats,
             'is_live': True,
-        }
\ No newline at end of file
+        }

From fdaaaaa878c42975936bf7f6ecf39a97436fefe2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 23 Jan 2015 21:10:10 +0100
Subject: [PATCH 0059/6219] README: Recommend using flake8 instead of pyflake
 and pep8 separately

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 869523720..36b87444e 100644
--- a/README.md
+++ b/README.md
@@ -586,7 +586,7 @@ If you want to add support for a new site, you can follow this quick list (assum
 5. Add an import in [`youtube_dl/extractor/__init__.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/__init__.py).
 6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will be then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc.
 7. Have a look at [`youtube_dl/common/extractor/common.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should return](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L38). Add tests and code for as many as you want.
-8. If you can, check the code with [pyflakes](https://pypi.python.org/pypi/pyflakes) (a good idea) and [pep8](https://pypi.python.org/pypi/pep8) (optional, ignore E501).
+8. If you can, check the code with [flake8](https://pypi.python.org/pypi/flake8).
 9. When the tests pass, [add](http://git-scm.com/docs/git-add) the new files and [commit](http://git-scm.com/docs/git-commit) them and [push](http://git-scm.com/docs/git-push) the result, like this:
 
         $ git add youtube_dl/extractor/__init__.py

From a055469fafe088b6aa0e569d989cbf7f70535951 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 23 Jan 2015 23:50:31 +0100
Subject: [PATCH 0060/6219] [downloader] Improve downloader selection

---
 youtube_dl/YoutubeDL.py           |  2 +-
 youtube_dl/downloader/__init__.py | 31 +++++++++++++++----------------
 youtube_dl/utils.py               | 22 ++++++++++++++++++++++
 3 files changed, 38 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 521e4055e..e61e6c2a7 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1179,7 +1179,7 @@ class YoutubeDL(object):
         if not self.params.get('skip_download', False):
             try:
                 def dl(name, info):
-                    fd = get_suitable_downloader(info)(self, self.params)
+                    fd = get_suitable_downloader(info, self.params)(self, self.params)
                     for ph in self._progress_hooks:
                         fd.add_progress_hook(ph)
                     if self.params.get('verbose'):
diff --git a/youtube_dl/downloader/__init__.py b/youtube_dl/downloader/__init__.py
index 31e28df58..2aca3cab5 100644
--- a/youtube_dl/downloader/__init__.py
+++ b/youtube_dl/downloader/__init__.py
@@ -9,27 +9,26 @@ from .rtmp import RtmpFD
 from .f4m import F4mFD
 
 from ..utils import (
-    determine_ext,
+    determine_protocol,
 )
 
+PROTOCOL_MAP = {
+    'rtmp': RtmpFD,
+    'm3u8_native': NativeHlsFD,
+    'm3u8': HlsFD,
+    'mms': MplayerFD,
+    'rtsp': MplayerFD,
+    'f4m': F4mFD,
+}
 
-def get_suitable_downloader(info_dict):
+
+def get_suitable_downloader(info_dict, params={}):
     """Get the downloader class that can handle the info dict."""
-    url = info_dict['url']
-    protocol = info_dict.get('protocol')
+    protocol = determine_protocol(info_dict)
+    info_dict['protocol'] = protocol
+
+    return PROTOCOL_MAP.get(protocol, HttpFD)
 
-    if url.startswith('rtmp'):
-        return RtmpFD
-    if protocol == 'm3u8_native':
-        return NativeHlsFD
-    if (protocol == 'm3u8') or (protocol is None and determine_ext(url) == 'm3u8'):
-        return HlsFD
-    if url.startswith('mms') or url.startswith('rtsp'):
-        return MplayerFD
-    if determine_ext(url) == 'f4m':
-        return F4mFD
-    else:
-        return HttpFD
 
 __all__ = [
     'get_suitable_downloader',
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 463cc20ff..2970d02a1 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1642,3 +1642,25 @@ def is_html(first_bytes):
         s = first_bytes.decode('utf-8', 'replace')
 
     return re.match(r'^\s*<', s)
+
+
+def determine_protocol(info_dict):
+    protocol = info_dict.get('protocol')
+    if protocol is not None:
+        return protocol
+
+    url = info_dict['url']
+    if url.startswith('rtmp'):
+        return 'rtmp'
+    elif url.startswith('mms'):
+        return 'mms'
+    elif url.startswith('rtsp'):
+        return 'rtsp'
+
+    ext = determine_ext(url)
+    if ext == 'm3u8':
+        return 'm3u8'
+    elif ext == 'f4m':
+        return 'f4m'
+
+    return compat_urllib_parse_urlparse(url).scheme

From 222516d97d5ff9e62f3a9860fe2e65aa99c001b3 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 24 Jan 2015 01:38:48 +0100
Subject: [PATCH 0061/6219] [downloader] Lay groundwork for external
 downloaders.

This comes with a very simply implementation for wget; the real work is in setting up the infrastructure.
---
 youtube_dl/YoutubeDL.py           |   1 +
 youtube_dl/__init__.py            |   1 +
 youtube_dl/downloader/__init__.py |   9 +-
 youtube_dl/downloader/common.py   |  21 +++++
 youtube_dl/downloader/external.py | 131 ++++++++++++++++++++++++++++++
 youtube_dl/downloader/rtmp.py     |  14 +---
 youtube_dl/options.py             |   6 ++
 7 files changed, 169 insertions(+), 14 deletions(-)
 create mode 100644 youtube_dl/downloader/external.py

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index e61e6c2a7..54e732943 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -219,6 +219,7 @@ class YoutubeDL(object):
     call_home:         Boolean, true iff we are allowed to contact the
                        youtube-dl servers for debugging.
     sleep_interval:    Number of seconds to sleep before each download.
+    external_downloader:  Executable of the external downloader to call.
 
 
     The following parameters are not used by YoutubeDL itself, they are used by
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 7bd7295e2..3fc7dc5c2 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -330,6 +330,7 @@ def _real_main(argv=None):
         'source_address': opts.source_address,
         'call_home': opts.call_home,
         'sleep_interval': opts.sleep_interval,
+        'external_downloader': opts.external_downloader,
     }
 
     with YoutubeDL(ydl_opts) as ydl:
diff --git a/youtube_dl/downloader/__init__.py b/youtube_dl/downloader/__init__.py
index 2aca3cab5..eff1122c5 100644
--- a/youtube_dl/downloader/__init__.py
+++ b/youtube_dl/downloader/__init__.py
@@ -1,12 +1,13 @@
 from __future__ import unicode_literals
 
 from .common import FileDownloader
+from .external import get_external_downloader
+from .f4m import F4mFD
 from .hls import HlsFD
 from .hls import NativeHlsFD
 from .http import HttpFD
 from .mplayer import MplayerFD
 from .rtmp import RtmpFD
-from .f4m import F4mFD
 
 from ..utils import (
     determine_protocol,
@@ -27,6 +28,12 @@ def get_suitable_downloader(info_dict, params={}):
     protocol = determine_protocol(info_dict)
     info_dict['protocol'] = protocol
 
+    external_downloader = params.get('external_downloader')
+    if external_downloader is not None:
+        ed = get_external_downloader(external_downloader)
+        if ed.supports(info_dict):
+            return ed
+
     return PROTOCOL_MAP.get(protocol, HttpFD)
 
 
diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index 82c917d92..c35c42c1d 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -325,3 +325,24 @@ class FileDownloader(object):
         # See YoutubeDl.py (search for progress_hooks) for a description of
         # this interface
         self._progress_hooks.append(ph)
+
+    def _debug_cmd(self, args, subprocess_encoding, exe=None):
+        if not self.params.get('verbose', False):
+            return
+
+        if exe is None:
+            exe = os.path.basename(args[0])
+
+        if subprocess_encoding:
+            str_args = [
+                a.decode(subprocess_encoding) if isinstance(a, bytes) else a
+                for a in args]
+        else:
+            str_args = args
+        try:
+            import pipes
+            shell_quote = lambda args: ' '.join(map(pipes.quote, str_args))
+        except ImportError:
+            shell_quote = repr
+        self.to_screen('[debug] %s command line: %s' % (
+            exe, shell_quote(str_args)))
diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
new file mode 100644
index 000000000..c05596255
--- /dev/null
+++ b/youtube_dl/downloader/external.py
@@ -0,0 +1,131 @@
+from __future__ import unicode_literals
+
+import os.path
+import subprocess
+import sys
+
+from .common import FileDownloader
+from ..utils import (
+    encodeFilename,
+    std_headers,
+)
+
+
+class ExternalFD(FileDownloader):
+    def real_download(self, filename, info_dict):
+        self.report_destination(filename)
+        tmpfilename = self.temp_name(filename)
+
+        retval = self._call_downloader(tmpfilename, info_dict)
+        if retval == 0:
+            fsize = os.path.getsize(encodeFilename(tmpfilename))
+            self.to_screen('\r[%s] Downloaded %s bytes' % (self.get_basename(), fsize))
+            self.try_rename(tmpfilename, filename)
+            self._hook_progress({
+                'downloaded_bytes': fsize,
+                'total_bytes': fsize,
+                'filename': filename,
+                'status': 'finished',
+            })
+            return True
+        else:
+            self.to_stderr('\n')
+            self.report_error('%s exited with code %d' % (
+                self.get_basename(), retval))
+            return False
+
+    @classmethod
+    def get_basename(cls):
+        return cls.__name__[:-2].lower()
+
+    @property
+    def exe(self):
+        return self.params.get('external_downloader')
+
+    @classmethod
+    def supports(cls, info_dict):
+        return info_dict['protocol'] in ('http', 'https', 'ftp', 'ftps')
+
+    def _calc_headers(self, info_dict):
+        res = std_headers.copy()
+
+        ua = info_dict.get('user_agent')
+        if ua is not None:
+            res['User-Agent'] = ua
+
+        cookies = self._calc_cookies(info_dict)
+        if cookies:
+            res['Cookie'] = cookies
+
+        return res
+
+    def _calc_cookies(self, info_dict):
+        class _PseudoRequest(object):
+            def __init__(self, url):
+                self.url = url
+                self.headers = {}
+                self.unverifiable = False
+
+            def add_unredirected_header(self, k, v):
+                self.headers[k] = v
+
+            def get_full_url(self):
+                return self.url
+
+            def is_unverifiable(self):
+                return self.unverifiable
+
+            def has_header(self, h):
+                return h in self.headers
+
+        pr = _PseudoRequest(info_dict['url'])
+        self.ydl.cookiejar.add_cookie_header(pr)
+        return pr.headers.get('Cookie')
+
+    def _call_downloader(self, tmpfilename, info_dict):
+        """ Either overwrite this or implement _make_cmd """
+        cmd = self._make_cmd(tmpfilename, info_dict)
+
+        if sys.platform == 'win32' and sys.version_info < (3, 0):
+            # Windows subprocess module does not actually support Unicode
+            # on Python 2.x
+            # See http://stackoverflow.com/a/9951851/35070
+            subprocess_encoding = sys.getfilesystemencoding()
+            cmd = [a.encode(subprocess_encoding, 'ignore') for a in cmd]
+        else:
+            subprocess_encoding = None
+        self._debug_cmd(cmd, subprocess_encoding)
+
+        p = subprocess.Popen(
+            cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        stdout, stderr = p.communicate()
+        if p.returncode != 0:
+            self.to_stderr(stderr)
+        return p.returncode
+
+
+class WgetFD(ExternalFD):
+    def _make_cmd(self, tmpfilename, info_dict):
+        cmd = [self.exe, '-O', tmpfilename, '-nv', '--no-cookies']
+        for key, val in self._calc_headers(info_dict).items():
+            cmd += ['--header', '%s: %s' % (key, val)]
+        cmd += ['--', info_dict['url']]
+        return cmd
+
+
+_BY_NAME = dict(
+    (klass.get_basename(), klass)
+    for name, klass in globals().items()
+    if name.endswith('FD') and name != 'ExternalFD'
+)
+
+
+def list_external_downloaders():
+    return sorted(_BY_NAME.keys())
+
+
+def get_external_downloader(external_downloader):
+    """ Given the name of the executable, see whether we support the given
+        downloader . """
+    bn = os.path.basename(external_downloader)
+    return _BY_NAME[bn]
diff --git a/youtube_dl/downloader/rtmp.py b/youtube_dl/downloader/rtmp.py
index 5346cb9a0..6dbbc053c 100644
--- a/youtube_dl/downloader/rtmp.py
+++ b/youtube_dl/downloader/rtmp.py
@@ -152,19 +152,7 @@ class RtmpFD(FileDownloader):
         else:
             subprocess_encoding = None
 
-        if self.params.get('verbose', False):
-            if subprocess_encoding:
-                str_args = [
-                    a.decode(subprocess_encoding) if isinstance(a, bytes) else a
-                    for a in args]
-            else:
-                str_args = args
-            try:
-                import pipes
-                shell_quote = lambda args: ' '.join(map(pipes.quote, str_args))
-            except ImportError:
-                shell_quote = repr
-            self.to_screen('[debug] rtmpdump command line: ' + shell_quote(str_args))
+        self._debug_cmd(args, subprocess_encoding, exe='rtmpdump')
 
         RD_SUCCESS = 0
         RD_FAILED = 1
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 262c60013..b38b8349f 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -5,6 +5,7 @@ import optparse
 import shlex
 import sys
 
+from .downloader.external import list_external_downloaders
 from .compat import (
     compat_expanduser,
     compat_getenv,
@@ -389,6 +390,11 @@ def parseOpts(overrideArguments=None):
         '--playlist-reverse',
         action='store_true',
         help='Download playlist videos in reverse order')
+    downloader.add_option(
+        '--external-downloader',
+        dest='external_downloader', metavar='COMMAND',
+        help='(experimental) Use the specified external downloader. '
+             'Currently supports %s' % ','.join(list_external_downloaders()))
 
     workarounds = optparse.OptionGroup(parser, 'Workarounds')
     workarounds.add_option(

From 6aa4f54d6660bfb8fefe225e47cba7ade62fe564 Mon Sep 17 00:00:00 2001
From: Irfan Charania <irfan.charania@gmail.com>
Date: Fri, 23 Jan 2015 17:41:07 -0800
Subject: [PATCH 0062/6219] [videott] improve extraction

---
 youtube_dl/extractor/videott.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/videott.py b/youtube_dl/extractor/videott.py
index 1f938838c..6f8084140 100644
--- a/youtube_dl/extractor/videott.py
+++ b/youtube_dl/extractor/videott.py
@@ -13,7 +13,7 @@ from ..utils import (
 class VideoTtIE(InfoExtractor):
     ID_NAME = 'video.tt'
     IE_DESC = 'video.tt - Your True Tube'
-    _VALID_URL = r'http://(?:www\.)?video\.tt/(?:video/|watch_video\.php\?v=)(?P<id>[\da-zA-Z]{9})'
+    _VALID_URL = r'http://(?:www\.)?video\.tt/(?:video\/|embed\/|watch_video\.php\?v=)(?P<id>[\da-zA-Z]{9})'
 
     _TEST = {
         'url': 'http://www.video.tt/watch_video.php?v=amd5YujV8',

From fc2d6abfe7e7a216825296001b889e5455e0412f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 24 Jan 2015 16:11:40 +0600
Subject: [PATCH 0063/6219] [videott] Improve _VALID_URL and add test

---
 youtube_dl/extractor/videott.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/videott.py b/youtube_dl/extractor/videott.py
index 6f8084140..ececc7ee0 100644
--- a/youtube_dl/extractor/videott.py
+++ b/youtube_dl/extractor/videott.py
@@ -13,9 +13,9 @@ from ..utils import (
 class VideoTtIE(InfoExtractor):
     ID_NAME = 'video.tt'
     IE_DESC = 'video.tt - Your True Tube'
-    _VALID_URL = r'http://(?:www\.)?video\.tt/(?:video\/|embed\/|watch_video\.php\?v=)(?P<id>[\da-zA-Z]{9})'
+    _VALID_URL = r'http://(?:www\.)?video\.tt/(?:(?:video|embed)/|watch_video\.php\?v=)(?P<id>[\da-zA-Z]{9})'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.video.tt/watch_video.php?v=amd5YujV8',
         'md5': 'b13aa9e2f267effb5d1094443dff65ba',
         'info_dict': {
@@ -26,7 +26,10 @@ class VideoTtIE(InfoExtractor):
             'upload_date': '20130827',
             'uploader': 'joseph313',
         }
-    }
+    }, {
+        'url': 'http://video.tt/embed/amd5YujV8',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)

From b95aab8482881e8b1f7fba856da816a2dbc50d0f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 24 Jan 2015 11:42:20 +0100
Subject: [PATCH 0064/6219] [youtube:truncated_url] Add x-yt-cl URLs (#4773)

---
 youtube_dl/extractor/youtube.py | 22 +++++++++++++++++-----
 1 file changed, 17 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index eb55d24ce..b7b91f354 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1682,11 +1682,17 @@ class YoutubeTruncatedURLIE(InfoExtractor):
     IE_NAME = 'youtube:truncated_url'
     IE_DESC = False  # Do not list
     _VALID_URL = r'''(?x)
-        (?:https?://)?[^/]+/watch\?(?:
+        (?:https?://)?
+        (?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie)?\.com/
+        (?:watch\?(?:
             feature=[a-z_]+|
-            annotation_id=annotation_[^&]+
-        )?$|
-        (?:https?://)?(?:www\.)?youtube\.com/attribution_link\?a=[^&]+$
+            annotation_id=annotation_[^&]+|
+            x-yt-cl=[0-9]+|
+        )?
+        |
+            attribution_link\?a=[^&]+
+        )
+        $
     '''
 
     _TESTS = [{
@@ -1695,6 +1701,12 @@ class YoutubeTruncatedURLIE(InfoExtractor):
     }, {
         'url': 'http://www.youtube.com/watch?',
         'only_matching': True,
+    }, {
+        'url': 'https://www.youtube.com/watch?x-yt-cl=84503534',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.youtube.com/watch?feature=foo',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
@@ -1710,7 +1722,7 @@ class YoutubeTruncatedURLIE(InfoExtractor):
 class YoutubeTruncatedIDIE(InfoExtractor):
     IE_NAME = 'youtube:truncated_id'
     IE_DESC = False  # Do not list
-    _VALID_URL = r'https?://(?:www\.)youtube\.com/watch\?v=(?P<id>[0-9A-Za-z_-]{1,10})$'
+    _VALID_URL = r'https?://(?:www\.)?youtube\.com/watch\?v=(?P<id>[0-9A-Za-z_-]{1,10})$'
 
     _TESTS = [{
         'url': 'https://www.youtube.com/watch?v=N_708QY7Ob',

From 384b62028a4c3c35727d714ccfc9944a36934069 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 24 Jan 2015 13:33:45 +0100
Subject: [PATCH 0065/6219] [downloader/external] Add curl and aria2c (Closes
 #182)

---
 youtube_dl/downloader/external.py | 27 +++++++++++++++++++++++++--
 1 file changed, 25 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index c05596255..7ebe40096 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -97,13 +97,22 @@ class ExternalFD(FileDownloader):
         self._debug_cmd(cmd, subprocess_encoding)
 
         p = subprocess.Popen(
-            cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        stdout, stderr = p.communicate()
+            cmd, stderr=subprocess.PIPE)
+        _, stderr = p.communicate()
         if p.returncode != 0:
             self.to_stderr(stderr)
         return p.returncode
 
 
+class CurlFD(ExternalFD):
+    def _make_cmd(self, tmpfilename, info_dict):
+        cmd = [self.exe, '-o', tmpfilename]
+        for key, val in self._calc_headers(info_dict).items():
+            cmd += ['--header', '%s: %s' % (key, val)]
+        cmd += ['--', info_dict['url']]
+        return cmd
+
+
 class WgetFD(ExternalFD):
     def _make_cmd(self, tmpfilename, info_dict):
         cmd = [self.exe, '-O', tmpfilename, '-nv', '--no-cookies']
@@ -113,6 +122,20 @@ class WgetFD(ExternalFD):
         return cmd
 
 
+class Aria2cFD(ExternalFD):
+    def _make_cmd(self, tmpfilename, info_dict):
+        cmd = [
+            self.exe, '-c',
+            '--min-split-size', '1M', '--max-connection-per-server', '4']
+        dn = os.path.dirname(tmpfilename)
+        if dn:
+            cmd += ['--dir', dn]
+        cmd += ['--out', os.path.basename(tmpfilename)]
+        for key, val in self._calc_headers(info_dict).items():
+            cmd += ['--header', '%s: %s' % (key, val)]
+        cmd += ['--', info_dict['url']]
+        return cmd
+
 _BY_NAME = dict(
     (klass.get_basename(), klass)
     for name, klass in globals().items()

From 3fcfb8e9faf3cf1dcadedd6fecc5158a86d07065 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 24 Jan 2015 18:07:21 +0100
Subject: [PATCH 0066/6219] [utils] YoutubeDLHandler: don't use
 'Youtubedl-user-agent' for overriding the default user agent

Setting the 'User-Agent' header is enough
---
 youtube_dl/downloader/http.py       | 2 +-
 youtube_dl/extractor/atresplayer.py | 2 +-
 youtube_dl/extractor/bliptv.py      | 2 +-
 youtube_dl/extractor/mtv.py         | 2 +-
 youtube_dl/utils.py                 | 5 -----
 5 files changed, 4 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index e68f20c9f..90a2e4c53 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -25,7 +25,7 @@ class HttpFD(FileDownloader):
         # Do not include the Accept-Encoding header
         headers = {'Youtubedl-no-compression': 'True'}
         if 'user_agent' in info_dict:
-            headers['Youtubedl-user-agent'] = info_dict['user_agent']
+            headers['User-agent'] = info_dict['user_agent']
         if 'http_referer' in info_dict:
             headers['Referer'] = info_dict['http_referer']
         add_headers = info_dict.get('http_headers')
diff --git a/youtube_dl/extractor/atresplayer.py b/youtube_dl/extractor/atresplayer.py
index f42862be3..f016368fa 100644
--- a/youtube_dl/extractor/atresplayer.py
+++ b/youtube_dl/extractor/atresplayer.py
@@ -95,7 +95,7 @@ class AtresPlayerIE(SubtitlesInfoExtractor):
         for fmt in ['windows', 'android_tablet']:
             request = compat_urllib_request.Request(
                 self._URL_VIDEO_TEMPLATE.format(fmt, episode_id, timestamp_shifted, token))
-            request.add_header('Youtubedl-user-agent', self._USER_AGENT)
+            request.add_header('User-Agent', self._USER_AGENT)
 
             fmt_json = self._download_json(
                 request, video_id, 'Downloading %s video JSON' % fmt)
diff --git a/youtube_dl/extractor/bliptv.py b/youtube_dl/extractor/bliptv.py
index 14b814120..436cc5155 100644
--- a/youtube_dl/extractor/bliptv.py
+++ b/youtube_dl/extractor/bliptv.py
@@ -199,7 +199,7 @@ class BlipTVIE(SubtitlesInfoExtractor):
         # For some weird reason, blip.tv serves a video instead of subtitles
         # when we request with a common UA
         req = compat_urllib_request.Request(url)
-        req.add_header('Youtubedl-user-agent', 'youtube-dl')
+        req.add_header('User-Agent', 'youtube-dl')
         return self._download_webpage(req, None, note=False)
 
 
diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index 5ebc78033..22a726327 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -53,7 +53,7 @@ class MTVServicesInfoExtractor(InfoExtractor):
         webpage_url = self._MOBILE_TEMPLATE % mtvn_id
         req = compat_urllib_request.Request(webpage_url)
         # Otherwise we get a webpage that would execute some javascript
-        req.add_header('Youtubedl-user-agent', 'curl/7')
+        req.add_header('User-Agent', 'curl/7')
         webpage = self._download_webpage(req, mtvn_id,
                                          'Downloading mobile page')
         metrics_url = unescapeHTML(self._search_regex(r'<a href="(http://metrics.+?)"', webpage, 'url'))
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 2970d02a1..d22b03134 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -606,11 +606,6 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
             if 'Accept-encoding' in req.headers:
                 del req.headers['Accept-encoding']
             del req.headers['Youtubedl-no-compression']
-        if 'Youtubedl-user-agent' in req.headers:
-            if 'User-agent' in req.headers:
-                del req.headers['User-agent']
-            req.headers['User-agent'] = req.headers['Youtubedl-user-agent']
-            del req.headers['Youtubedl-user-agent']
 
         if sys.version_info < (2, 7) and '#' in req.get_full_url():
             # Python 2.6 is brain-dead when it comes to fragments

From e1554a407d8953d0ec4fabec7f5a3b3906d1029b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 24 Jan 2015 18:19:58 +0100
Subject: [PATCH 0067/6219] [extractors] Use http_headers for setting the
 User-Agent and the Referer

---
 youtube_dl/downloader/http.py         | 4 ----
 youtube_dl/extractor/appletrailers.py | 4 +++-
 youtube_dl/extractor/common.py        | 1 -
 youtube_dl/extractor/videomega.py     | 4 +++-
 youtube_dl/extractor/wdr.py           | 4 +++-
 5 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index 90a2e4c53..4db50ee90 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -24,10 +24,6 @@ class HttpFD(FileDownloader):
 
         # Do not include the Accept-Encoding header
         headers = {'Youtubedl-no-compression': 'True'}
-        if 'user_agent' in info_dict:
-            headers['User-agent'] = info_dict['user_agent']
-        if 'http_referer' in info_dict:
-            headers['Referer'] = info_dict['http_referer']
         add_headers = info_dict.get('http_headers')
         if add_headers:
             headers.update(add_headers)
diff --git a/youtube_dl/extractor/appletrailers.py b/youtube_dl/extractor/appletrailers.py
index 7cd0482c7..70621946d 100644
--- a/youtube_dl/extractor/appletrailers.py
+++ b/youtube_dl/extractor/appletrailers.py
@@ -129,7 +129,9 @@ class AppleTrailersIE(InfoExtractor):
                 'thumbnail': thumbnail,
                 'upload_date': upload_date,
                 'uploader_id': uploader_id,
-                'user_agent': 'QuickTime compatible (youtube-dl)',
+                'http_headers': {
+                    'User-Agent': 'QuickTime compatible (youtube-dl)',
+                },
             })
 
         return {
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 03f3f18c8..523400062 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -108,7 +108,6 @@ class InfoExtractor(object):
                                   (quality takes higher priority)
                                  -1 for default (order by other properties),
                                  -2 or smaller for less than default.
-                    * http_referer  HTTP Referer header value to set.
                     * http_method  HTTP method to use for the download.
                     * http_headers  A dictionary of additional HTTP headers
                                  to add to the request.
diff --git a/youtube_dl/extractor/videomega.py b/youtube_dl/extractor/videomega.py
index fc6e05fe0..273030316 100644
--- a/youtube_dl/extractor/videomega.py
+++ b/youtube_dl/extractor/videomega.py
@@ -62,5 +62,7 @@ class VideoMegaIE(InfoExtractor):
             'title': title,
             'formats': formats,
             'thumbnail': thumbnail,
-            'http_referer': iframe_url,
+            'http_headers': {
+                'Referer': iframe_url,
+            },
         }
diff --git a/youtube_dl/extractor/wdr.py b/youtube_dl/extractor/wdr.py
index 45466e31b..313b9c15d 100644
--- a/youtube_dl/extractor/wdr.py
+++ b/youtube_dl/extractor/wdr.py
@@ -169,7 +169,9 @@ class WDRMobileIE(InfoExtractor):
             'title': mobj.group('title'),
             'age_limit': int(mobj.group('age_limit')),
             'url': url,
-            'user_agent': 'mobile',
+            'http_headers': {
+                'User-Agent': 'mobile',
+            },
         }
 
 

From 587a9c27496979d983296944fee8d1fa589e1b85 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 24 Jan 2015 18:25:09 +0100
Subject: [PATCH 0068/6219] [downloader/external] Use the 'http_headers' field

---
 youtube_dl/downloader/external.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index 7ebe40096..5bf24ccbb 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -49,9 +49,9 @@ class ExternalFD(FileDownloader):
     def _calc_headers(self, info_dict):
         res = std_headers.copy()
 
-        ua = info_dict.get('user_agent')
-        if ua is not None:
-            res['User-Agent'] = ua
+        add_headers = info_dict.get('http_headers')
+        if add_headers:
+            res.update(add_headers)
 
         cookies = self._calc_cookies(info_dict)
         if cookies:

From 8011fba3ae4301995f77c64ac63a4a467ef73b7e Mon Sep 17 00:00:00 2001
From: David-Development <david-dev@live.de>
Date: Sat, 24 Jan 2015 18:28:16 +0100
Subject: [PATCH 0069/6219] [rtl2] Add new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/rtl2.py     | 98 ++++++++++++++++++++++++++++++++
 2 files changed, 99 insertions(+)
 create mode 100644 youtube_dl/extractor/rtl2.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 3f7ca6f7d..03c56156a 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -349,6 +349,7 @@ from .rtbf import RTBFIE
 from .rte import RteIE
 from .rtlnl import RtlXlIE
 from .rtlnow import RTLnowIE
+from .rtl2 import RTL2IE
 from .rtp import RTPIE
 from .rts import RTSIE
 from .rtve import RTVEALaCartaIE, RTVELiveIE
diff --git a/youtube_dl/extractor/rtl2.py b/youtube_dl/extractor/rtl2.py
new file mode 100644
index 000000000..2bdbf2ec2
--- /dev/null
+++ b/youtube_dl/extractor/rtl2.py
@@ -0,0 +1,98 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+import re
+import json
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    clean_html,
+    unified_strdate,
+    int_or_none,
+)
+
+
+class RTL2IE(InfoExtractor):
+    """Information Extractor for RTL NOW, RTL2 NOW, RTL NITRO, SUPER RTL NOW, VOX NOW and n-tv NOW"""
+    _VALID_URL = r'http?://(?P<url>(?P<domain>(www\.)?rtl2\.de)/.*/(?P<video_id>.*))'
+    _TEST = {
+        'url': 'http://www.rtl2.de/sendung/grip-das-motormagazin/folge/folge-203-0',
+        'md5': 'dsadasdada',
+        'info_dict': {
+            'id': 'folge-203-0',
+            'ext': 'f4v',
+            'title': 'GRIP sucht den Sommerk\xf6nig',
+	    'description' : 'Matthias, Det und Helge treten gegeneinander an.'
+            # TODO more properties, either as:
+            # * A value
+            # * MD5 checksum; start the string with md5:
+            # * A regular expression; start the string with re:
+            # * Any Python type (for example int or float)
+        },
+	#'params': {
+                # rtmp download
+        #	'skip_download': True,
+	#},
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_page_url = 'http://%s/' % mobj.group('domain')
+        video_id = mobj.group('video_id')
+	
+        webpage = self._download_webpage('http://' + mobj.group('url'), video_id)
+
+	vico_id = self._html_search_regex(r'vico_id: ([0-9]+)', webpage, '%s');
+	vivi_id = self._html_search_regex(r'vivi_id: ([0-9]+)', webpage, '%s');
+
+	info_url = 'http://www.rtl2.de/video/php/get_video.php?vico_id=' + vico_id + '&vivi_id=' + vivi_id
+	webpage = self._download_webpage(info_url, '')
+
+	video_info = json.loads(webpage.decode("latin1"))
+	print video_info
+
+
+	#self._download_webpage('http://cp108781.edgefcs.net/crossdomain.xml', '')
+
+	download_url = video_info["video"]["streamurl"] # self._html_search_regex(r'streamurl\":\"(.*?)\"', webpage, '%s');
+	title = video_info["video"]["titel"] # self._html_search_regex(r'titel\":\"(.*?)\"', webpage, '%s');
+	description = video_info["video"]["beschreibung"] # self._html_search_regex(r'beschreibung\":\"(.*?)\"', webpage, '%s');
+	#ext = self._html_search_regex(r'streamurl\":\".*?(\..{2,4})\"', webpage, '%s');
+
+	thumbnail = video_info["video"]["image"]
+
+	download_url = download_url.replace("\\", "")
+
+	stream_url = 'mp4:' + self._html_search_regex(r'ondemand/(.*)', download_url, '%s');
+
+	#upload_date = self._html_search_regex(r'property=\"dc:date\".*?datatype=\"xsd:dateTime\".*?content=\"(.*?)\"', webpage, 'title')
+	#download_url += " -y " + stream_url
+	
+	#print stream_url
+	#print download_url
+	#print description
+	#print title
+	#print ext
+
+	formats = []
+
+	fmt = {
+	    'url' : download_url,
+            #'app': 'ondemand?_fcs_vhost=cp108781.edgefcs.net',
+            'play_path': stream_url,
+            #'player_url': 'http://www.cbsnews.com/[[IMPORT]]/vidtech.cbsinteractive.com/player/3_3_0/CBSI_PLAYER_HD.swf',
+            #'page_url': 'http://www.cbsnews.com',
+            #'ext': ext,	
+        }
+
+	formats.append(fmt)
+
+
+        return {
+	    'id': video_id,
+            'title': title,
+	    'thumbnail' : thumbnail,
+	    'description' : description,
+            'formats': formats,
+        }

From e5660ee6aebacee4b149648bee1a7ce1ec72c1e4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 24 Jan 2015 18:52:26 +0100
Subject: [PATCH 0070/6219] [YoutubeDL] Fill the info dict 'http_headers' field
 with all the headers available

Useful for external tools using the json output.

The methods '_calc_headers' and '_calc_cookies' have been copied from the downloader/external, now they just use "info_dict['http_headers']".
---
 youtube_dl/YoutubeDL.py           | 36 ++++++++++++++++++++++++++
 youtube_dl/downloader/external.py | 43 +++----------------------------
 2 files changed, 39 insertions(+), 40 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 54e732943..4c3d45f48 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -56,6 +56,7 @@ from .utils import (
     preferredencoding,
     SameFileError,
     sanitize_filename,
+    std_headers,
     subtitles_filename,
     takewhile_inclusive,
     UnavailableVideoError,
@@ -865,6 +866,36 @@ class YoutubeDL(object):
                 return matches[-1]
         return None
 
+    def _calc_headers(self, info_dict):
+        res = std_headers.copy()
+
+        add_headers = info_dict.get('http_headers')
+        if add_headers:
+            res.update(add_headers)
+
+        cookies = self._calc_cookies(info_dict)
+        if cookies:
+            res['Cookie'] = cookies
+
+        return res
+
+    def _calc_cookies(self, info_dict):
+        class _PseudoRequest(object):
+            def __init__(self, url):
+                self.url = url
+                self.headers = {}
+                self.unverifiable = False
+
+            def add_unredirected_header(self, k, v):
+                self.headers[k] = v
+
+            def get_full_url(self):
+                return self.url
+
+        pr = _PseudoRequest(info_dict['url'])
+        self.cookiejar.add_cookie_header(pr)
+        return pr.headers.get('Cookie')
+
     def process_video_result(self, info_dict, download=True):
         assert info_dict.get('_type', 'video') == 'video'
 
@@ -933,6 +964,11 @@ class YoutubeDL(object):
             # Automatically determine file extension if missing
             if 'ext' not in format:
                 format['ext'] = determine_ext(format['url']).lower()
+            # Add HTTP headers, so that external programs can use them from the
+            # json output
+            full_format_info = info_dict.copy()
+            full_format_info.update(format)
+            format['http_headers'] = self._calc_headers(full_format_info)
 
         format_limit = self.params.get('format_limit', None)
         if format_limit:
diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index 5bf24ccbb..af9fdba75 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -7,7 +7,6 @@ import sys
 from .common import FileDownloader
 from ..utils import (
     encodeFilename,
-    std_headers,
 )
 
 
@@ -46,42 +45,6 @@ class ExternalFD(FileDownloader):
     def supports(cls, info_dict):
         return info_dict['protocol'] in ('http', 'https', 'ftp', 'ftps')
 
-    def _calc_headers(self, info_dict):
-        res = std_headers.copy()
-
-        add_headers = info_dict.get('http_headers')
-        if add_headers:
-            res.update(add_headers)
-
-        cookies = self._calc_cookies(info_dict)
-        if cookies:
-            res['Cookie'] = cookies
-
-        return res
-
-    def _calc_cookies(self, info_dict):
-        class _PseudoRequest(object):
-            def __init__(self, url):
-                self.url = url
-                self.headers = {}
-                self.unverifiable = False
-
-            def add_unredirected_header(self, k, v):
-                self.headers[k] = v
-
-            def get_full_url(self):
-                return self.url
-
-            def is_unverifiable(self):
-                return self.unverifiable
-
-            def has_header(self, h):
-                return h in self.headers
-
-        pr = _PseudoRequest(info_dict['url'])
-        self.ydl.cookiejar.add_cookie_header(pr)
-        return pr.headers.get('Cookie')
-
     def _call_downloader(self, tmpfilename, info_dict):
         """ Either overwrite this or implement _make_cmd """
         cmd = self._make_cmd(tmpfilename, info_dict)
@@ -107,7 +70,7 @@ class ExternalFD(FileDownloader):
 class CurlFD(ExternalFD):
     def _make_cmd(self, tmpfilename, info_dict):
         cmd = [self.exe, '-o', tmpfilename]
-        for key, val in self._calc_headers(info_dict).items():
+        for key, val in info_dict['http_headers'].items():
             cmd += ['--header', '%s: %s' % (key, val)]
         cmd += ['--', info_dict['url']]
         return cmd
@@ -116,7 +79,7 @@ class CurlFD(ExternalFD):
 class WgetFD(ExternalFD):
     def _make_cmd(self, tmpfilename, info_dict):
         cmd = [self.exe, '-O', tmpfilename, '-nv', '--no-cookies']
-        for key, val in self._calc_headers(info_dict).items():
+        for key, val in info_dict['http_headers'].items():
             cmd += ['--header', '%s: %s' % (key, val)]
         cmd += ['--', info_dict['url']]
         return cmd
@@ -131,7 +94,7 @@ class Aria2cFD(ExternalFD):
         if dn:
             cmd += ['--dir', dn]
         cmd += ['--out', os.path.basename(tmpfilename)]
-        for key, val in self._calc_headers(info_dict).items():
+        for key, val in info_dict['http_headers'].items():
             cmd += ['--header', '%s: %s' % (key, val)]
         cmd += ['--', info_dict['url']]
         return cmd

From 4b405cfc6e4f9184567b94ee2f8a4a9851a912b5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 24 Jan 2015 20:05:35 +0100
Subject: [PATCH 0071/6219] [YoutubeDL._calc_cookies] Restore the 'has_header'
 method

I didn't copied it from downloader/external
---
 youtube_dl/YoutubeDL.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 4c3d45f48..458fd15ea 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -892,6 +892,9 @@ class YoutubeDL(object):
             def get_full_url(self):
                 return self.url
 
+            def has_header(self, h):
+                return h in self.headers
+
         pr = _PseudoRequest(info_dict['url'])
         self.cookiejar.add_cookie_header(pr)
         return pr.headers.get('Cookie')

From 1070711d6003e6750b8cf803c3926b2e273a9e85 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 24 Jan 2015 20:12:47 +0100
Subject: [PATCH 0072/6219] [YoutubeDL._calc_cookies] Restore the
 'is_unverifiable'

I should have check everything was copied before commiting 4b405cfc6e4f9184567b94ee2f8a4a9851a912b5.
---
 youtube_dl/YoutubeDL.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 458fd15ea..d6728b2dd 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -892,6 +892,9 @@ class YoutubeDL(object):
             def get_full_url(self):
                 return self.url
 
+            def is_unverifiable(self):
+                return self.unverifiable
+
             def has_header(self, h):
                 return h in self.headers
 

From 7906d199a11c0f8deccc0f87d7ee850106c361dc Mon Sep 17 00:00:00 2001
From: David-Development <david-dev@live.de>
Date: Sat, 24 Jan 2015 18:28:16 +0100
Subject: [PATCH 0073/6219] [rtl2] Add new extractor

---
 youtube_dl/downloader/rtmp.py    |   8 ++-
 youtube_dl/extractor/__init__.py |   1 +
 youtube_dl/extractor/rtl2.py     | 100 +++++++++++++++++++++++++++++++
 3 files changed, 107 insertions(+), 2 deletions(-)
 create mode 100644 youtube_dl/extractor/rtl2.py

diff --git a/youtube_dl/downloader/rtmp.py b/youtube_dl/downloader/rtmp.py
index 6dbbc053c..6f7ad588e 100644
--- a/youtube_dl/downloader/rtmp.py
+++ b/youtube_dl/downloader/rtmp.py
@@ -104,7 +104,8 @@ class RtmpFD(FileDownloader):
         live = info_dict.get('rtmp_live', False)
         conn = info_dict.get('rtmp_conn', None)
         protocol = info_dict.get('rtmp_protocol', None)
-
+        no_resume = info_dict.get('no_resume', False)
+        
         self.report_destination(filename)
         tmpfilename = self.temp_name(filename)
         test = self.params.get('test', False)
@@ -141,7 +142,10 @@ class RtmpFD(FileDownloader):
             basic_args += ['--conn', conn]
         if protocol is not None:
             basic_args += ['--protocol', protocol]
-        args = basic_args + [[], ['--resume', '--skip', '1']][not live and self.params.get('continuedl', False)]
+        if not no_resume:
+            basic_args += ['--resume']
+        
+        args = basic_args + [[], ['--skip', '1']][not live and self.params.get('continuedl', False)]
 
         if sys.platform == 'win32' and sys.version_info < (3, 0):
             # Windows subprocess module does not actually support Unicode
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 3f7ca6f7d..03c56156a 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -349,6 +349,7 @@ from .rtbf import RTBFIE
 from .rte import RteIE
 from .rtlnl import RtlXlIE
 from .rtlnow import RTLnowIE
+from .rtl2 import RTL2IE
 from .rtp import RTPIE
 from .rts import RTSIE
 from .rtve import RTVEALaCartaIE, RTVELiveIE
diff --git a/youtube_dl/extractor/rtl2.py b/youtube_dl/extractor/rtl2.py
new file mode 100644
index 000000000..14b45e86e
--- /dev/null
+++ b/youtube_dl/extractor/rtl2.py
@@ -0,0 +1,100 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+import re
+import json
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    clean_html,
+    unified_strdate,
+    int_or_none,
+)
+
+
+class RTL2IE(InfoExtractor):
+    """Information Extractor for RTL NOW, RTL2 NOW, RTL NITRO, SUPER RTL NOW, VOX NOW and n-tv NOW"""
+    _VALID_URL = r'http?://(?P<url>(?P<domain>(www\.)?rtl2\.de)/.*/(?P<video_id>.*))'
+    _TESTS = [{
+            'url': 'http://www.rtl2.de/sendung/grip-das-motormagazin/folge/folge-203-0',
+            'info_dict': {
+                'id': 'folge-203-0',
+                'ext': 'f4v',
+                'title': 'GRIP sucht den Sommerk\xf6nig',
+                'description' : 'Matthias, Det und Helge treten gegeneinander an.'
+            },
+            'params': {
+                # rtmp download
+                #'skip_download': True,
+            },
+        },
+        {
+            'url': 'http://www.rtl2.de/sendung/koeln-50667/video/5512-anna/21040-anna-erwischt-alex/',
+            'info_dict': {
+                'id': '21040-anna-erwischt-alex',
+                'ext': 'f4v',
+                'title': 'GRIP sucht den Sommerk\xf6nig',
+                'description' : 'Matthias, Det und Helge treten gegeneinander an.'
+            },
+            'params': {
+                # rtmp download
+                #'skip_download': True,
+            },
+        },
+    ]
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_page_url = 'http://%s/' % mobj.group('domain')
+        video_id = mobj.group('video_id')
+    
+        webpage = self._download_webpage('http://' + mobj.group('url'), video_id)
+
+        vico_id = self._html_search_regex(r'vico_id\s*:\s*([0-9]+)', webpage, '%s');
+        vivi_id = self._html_search_regex(r'vivi_id\s*:\s*([0-9]+)', webpage, '%s');
+
+        info_url = 'http://www.rtl2.de/video/php/get_video.php?vico_id=' + vico_id + '&vivi_id=' + vivi_id
+        webpage = self._download_webpage(info_url, '')
+
+        video_info = json.loads(webpage)
+
+        download_url = video_info["video"]["streamurl"] # self._html_search_regex(r'streamurl\":\"(.*?)\"', webpage, '%s');
+        title = video_info["video"]["titel"] # self._html_search_regex(r'titel\":\"(.*?)\"', webpage, '%s');
+        description = video_info["video"]["beschreibung"] # self._html_search_regex(r'beschreibung\":\"(.*?)\"', webpage, '%s');
+        #ext = self._html_search_regex(r'streamurl\":\".*?(\..{2,4})\"', webpage, '%s');
+
+        thumbnail = video_info["video"]["image"]
+
+        download_url = download_url.replace("\\", "")
+
+        stream_url = 'mp4:' + self._html_search_regex(r'ondemand/(.*)', download_url, '%s')
+
+        #print(download_url)
+        #print(stream_url)
+        #print(title)
+        #print(description)
+        #print(video_id)
+        
+        formats = []
+
+        fmt = {
+            'url' : download_url,
+                #'app': 'ondemand?_fcs_vhost=cp108781.edgefcs.net',
+                'play_path': stream_url,
+                'player_url': 'http://www.rtl2.de/flashplayer/vipo_player.swf',
+                'page_url': url,
+                'flash_version' : "LNX 11,2,202,429",
+                'rtmp_conn' : ["S:connect", "O:1", "NS:pageUrl:" + url, "NB:fpad:0", "NN:videoFunction:1", "O:0"],
+                'no_resume' : 1,
+        }
+
+        formats.append(fmt)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'thumbnail' : thumbnail,
+            'description' : description,
+            'formats': formats,
+        }

From 1e108029907ca28b75f37d2cf0bf25bcabbfbdac Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 25 Jan 2015 05:21:39 +0600
Subject: [PATCH 0074/6219] [krasview] Fix extraction

---
 youtube_dl/extractor/krasview.py | 22 +++++++++-------------
 1 file changed, 9 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/krasview.py b/youtube_dl/extractor/krasview.py
index 6f3d2345b..e46954b47 100644
--- a/youtube_dl/extractor/krasview.py
+++ b/youtube_dl/extractor/krasview.py
@@ -2,18 +2,17 @@
 from __future__ import unicode_literals
 
 import json
-import re
 
 from .common import InfoExtractor
 from ..utils import (
     int_or_none,
-    unescapeHTML,
+    js_to_json,
 )
 
 
 class KrasViewIE(InfoExtractor):
     IE_DESC = 'Красвью'
-    _VALID_URL = r'https?://krasview\.ru/video/(?P<id>\d+)'
+    _VALID_URL = r'https?://krasview\.ru/(?:video|embed)/(?P<id>\d+)'
 
     _TEST = {
         'url': 'http://krasview.ru/video/512228',
@@ -29,20 +28,18 @@ class KrasViewIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id)
 
-        flashvars = json.loads(self._search_regex(
-            r'flashvars\s*:\s*({.+?})\s*}\);', webpage, 'flashvars'))
+        flashvars = json.loads(js_to_json(self._search_regex(
+            r'video_Init\(({.+?})', webpage, 'flashvars')))
 
         video_url = flashvars['url']
-        title = unescapeHTML(flashvars['title'])
-        description = unescapeHTML(flashvars.get('subtitle') or self._og_search_description(webpage, default=None))
-        thumbnail = flashvars['image']
-        duration = int(flashvars['duration'])
-        filesize = int(flashvars['size'])
+        title = self._og_search_title(webpage)
+        description = self._og_search_description(webpage, default=None)
+        thumbnail = flashvars.get('image') or self._og_search_thumbnail(webpage)
+        duration = int_or_none(flashvars.get('duration'))
         width = int_or_none(self._og_search_property('video:width', webpage, 'video width'))
         height = int_or_none(self._og_search_property('video:height', webpage, 'video height'))
 
@@ -53,7 +50,6 @@ class KrasViewIE(InfoExtractor):
             'description': description,
             'thumbnail': thumbnail,
             'duration': duration,
-            'filesize': filesize,
             'width': width,
             'height': height,
         }

From cfb56d1af38f3e1e0251dbd8a20e3ed8884976ff Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 25 Jan 2015 02:38:47 +0100
Subject: [PATCH 0075/6219] Add --list-thumbnails

---
 test/test_utils.py               | 11 ++++++++++
 youtube_dl/YoutubeDL.py          | 37 ++++++++++++++++++++++++++++----
 youtube_dl/__init__.py           |  1 +
 youtube_dl/extractor/common.py   |  2 ++
 youtube_dl/extractor/testtube.py | 16 ++++++++++++--
 youtube_dl/options.py            | 15 +++++++++----
 youtube_dl/utils.py              |  8 +++++++
 7 files changed, 80 insertions(+), 10 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index bdd7f268a..ebec7986f 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -52,6 +52,7 @@ from youtube_dl.utils import (
     urlencode_postdata,
     version_tuple,
     xpath_with_ns,
+    render_table,
 )
 
 
@@ -434,5 +435,15 @@ ffmpeg version 2.4.4 Copyright (c) 2000-2014 the FFmpeg ...'''), '2.4.4')
         self.assertTrue(is_html(  # UTF-32-LE
             b'\xFF\xFE\x00\x00<\x00\x00\x00h\x00\x00\x00t\x00\x00\x00m\x00\x00\x00l\x00\x00\x00>\x00\x00\x00\xe4\x00\x00\x00'))
 
+    def test_render_table(self):
+        self.assertEqual(
+            render_table(
+                ['a', 'bcd'],
+                [[123, 4], [9999, 51]]),
+            'a    bcd\n'
+            '123  4\n'
+            '9999 51')
+
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index d6728b2dd..e0f5a0d74 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -54,6 +54,7 @@ from .utils import (
     PostProcessingError,
     platform_name,
     preferredencoding,
+    render_table,
     SameFileError,
     sanitize_filename,
     std_headers,
@@ -221,6 +222,8 @@ class YoutubeDL(object):
                        youtube-dl servers for debugging.
     sleep_interval:    Number of seconds to sleep before each download.
     external_downloader:  Executable of the external downloader to call.
+    listformats:       Print an overview of available video formats and exit.
+    list_thumbnails:   Print a table of all thumbnails and exit.
 
 
     The following parameters are not used by YoutubeDL itself, they are used by
@@ -916,9 +919,14 @@ class YoutubeDL(object):
             info_dict['playlist_index'] = None
 
         thumbnails = info_dict.get('thumbnails')
+        if thumbnails is None:
+            thumbnail = info_dict.get('thumbnail')
+            if thumbnail:
+                thumbnails = [{'url': thumbnail}]
         if thumbnails:
             thumbnails.sort(key=lambda t: (
-                t.get('width'), t.get('height'), t.get('url')))
+                t.get('preference'), t.get('width'), t.get('height'),
+                t.get('id'), t.get('url')))
             for t in thumbnails:
                 if 'width' in t and 'height' in t:
                     t['resolution'] = '%dx%d' % (t['width'], t['height'])
@@ -990,9 +998,12 @@ class YoutubeDL(object):
             # element in the 'formats' field in info_dict is info_dict itself,
             # wich can't be exported to json
             info_dict['formats'] = formats
-        if self.params.get('listformats', None):
+        if self.params.get('listformats'):
             self.list_formats(info_dict)
             return
+        if self.params.get('list_thumbnails'):
+            self.list_thumbnails(info_dict)
+            return
 
         req_format = self.params.get('format')
         if req_format is None:
@@ -1500,8 +1511,26 @@ class YoutubeDL(object):
         header_line = line({
             'format_id': 'format code', 'ext': 'extension',
             'resolution': 'resolution', 'format_note': 'note'}, idlen=idlen)
-        self.to_screen('[info] Available formats for %s:\n%s\n%s' %
-                       (info_dict['id'], header_line, '\n'.join(formats_s)))
+        self.to_screen(
+            '[info] Available formats for %s:\n%s\n%s' %
+            (info_dict['id'], header_line, '\n'.join(formats_s)))
+
+    def list_thumbnails(self, info_dict):
+        thumbnails = info_dict.get('thumbnails')
+        if not thumbnails:
+            tn_url = info_dict.get('thumbnail')
+            if tn_url:
+                thumbnails = [{'id': '0', 'url': tn_url}]
+            else:
+                self.to_screen(
+                    '[info] No thumbnails present for %s' % info_dict['id'])
+                return
+
+        self.to_screen(
+            '[info] Thumbnails for %s:' % info_dict['id'])
+        self.to_screen(render_table(
+            ['ID', 'width', 'height', 'URL'],
+            [[t['id'], t.get('width', 'unknown'), t.get('height', 'unknown'), t['url']] for t in thumbnails]))
 
     def urlopen(self, req):
         """ Start an HTTP download """
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 3fc7dc5c2..a3f82612c 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -331,6 +331,7 @@ def _real_main(argv=None):
         'call_home': opts.call_home,
         'sleep_interval': opts.sleep_interval,
         'external_downloader': opts.external_downloader,
+        'list_thumbnails': opts.list_thumbnails,
     }
 
     with YoutubeDL(ydl_opts) as ydl:
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 523400062..7b7a832dc 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -129,7 +129,9 @@ class InfoExtractor(object):
                     something like "4234987", title "Dancing naked mole rats",
                     and display_id "dancing-naked-mole-rats"
     thumbnails:     A list of dictionaries, with the following entries:
+                        * "id" (optional, string) - Thumbnail format ID
                         * "url"
+                        * "preference" (optional, int) - quality of the image
                         * "width" (optional, int)
                         * "height" (optional, int)
                         * "resolution" (optional, string "{width}x{height"},
diff --git a/youtube_dl/extractor/testtube.py b/youtube_dl/extractor/testtube.py
index fd47e71a2..6a7b5e49d 100644
--- a/youtube_dl/extractor/testtube.py
+++ b/youtube_dl/extractor/testtube.py
@@ -1,7 +1,10 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..utils import int_or_none
+from ..utils import (
+    int_or_none,
+    qualities,
+)
 
 
 class TestTubeIE(InfoExtractor):
@@ -46,13 +49,22 @@ class TestTubeIE(InfoExtractor):
         self._sort_formats(formats)
 
         duration = int_or_none(info.get('duration'))
+        images = info.get('images')
+        thumbnails = None
+        preference = qualities(['mini', 'small', 'medium', 'large'])
+        if images:
+            thumbnails = [{
+                'id': thumbnail_id,
+                'url': img_url,
+                'preference': preference(thumbnail_id)
+            } for thumbnail_id, img_url in images.items()]
 
         return {
             'id': video_id,
             'display_id': display_id,
             'title': info['title'],
             'description': info.get('summary'),
-            'thumbnail': info.get('images', {}).get('large'),
+            'thumbnails': thumbnails,
             'uploader': info.get('show', {}).get('name'),
             'uploader_id': info.get('show', {}).get('slug'),
             'duration': duration,
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index b38b8349f..e3b4b8a8a 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -614,10 +614,6 @@ def parseOpts(overrideArguments=None):
         '--write-annotations',
         action='store_true', dest='writeannotations', default=False,
         help='write video annotations to a .annotation file')
-    filesystem.add_option(
-        '--write-thumbnail',
-        action='store_true', dest='writethumbnail', default=False,
-        help='write thumbnail image to disk')
     filesystem.add_option(
         '--load-info',
         dest='load_info_filename', metavar='FILE',
@@ -637,6 +633,16 @@ def parseOpts(overrideArguments=None):
         action='store_true', dest='rm_cachedir',
         help='Delete all filesystem cache files')
 
+    thumbnail = optparse.OptionGroup(parser, 'Thumbnail images')
+    thumbnail.add_option(
+        '--write-thumbnail',
+        action='store_true', dest='writethumbnail', default=False,
+        help='write thumbnail image to disk')
+    thumbnail.add_option(
+        '--list-thumbnails',
+        action='store_true', dest='list_thumbnails', default=False,
+        help='Simulate and list all available thumbnail formats')
+
     postproc = optparse.OptionGroup(parser, 'Post-processing Options')
     postproc.add_option(
         '-x', '--extract-audio',
@@ -702,6 +708,7 @@ def parseOpts(overrideArguments=None):
     parser.add_option_group(selection)
     parser.add_option_group(downloader)
     parser.add_option_group(filesystem)
+    parser.add_option_group(thumbnail)
     parser.add_option_group(verbosity)
     parser.add_option_group(workarounds)
     parser.add_option_group(video_format)
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index d22b03134..b8c52af74 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1659,3 +1659,11 @@ def determine_protocol(info_dict):
         return 'f4m'
 
     return compat_urllib_parse_urlparse(url).scheme
+
+
+def render_table(header_row, data):
+    """ Render a list of rows, each as a list of values """
+    table = [header_row] + data
+    max_lens = [max(len(compat_str(v)) for v in col) for col in zip(*table)]
+    format_str = ' '.join('%-' + compat_str(ml + 1) + 's' for ml in max_lens[:-1]) + '%s'
+    return '\n'.join(format_str % tuple(row) for row in table)

From ec82d85acdc497436e7e2e767088d3ecb5947f68 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 25 Jan 2015 03:11:12 +0100
Subject: [PATCH 0076/6219] [YoutubeDL] Implement --write-all-thumbnails
 (Closes #2269)

---
 youtube_dl/YoutubeDL.py | 57 +++++++++++++++++++++++++++--------------
 youtube_dl/__init__.py  |  1 +
 youtube_dl/options.py   |  4 +++
 3 files changed, 43 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index e0f5a0d74..0e73dc8ff 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -146,6 +146,7 @@ class YoutubeDL(object):
     writeinfojson:     Write the video description to a .info.json file
     writeannotations:  Write the video annotations to a .annotations.xml file
     writethumbnail:    Write the thumbnail image to a file
+    write_all_thumbnails:  Write all thumbnail formats to files
     writesubtitles:    Write the video subtitles to a file
     writeautomaticsub: Write the automatic subtitles to a file
     allsubtitles:      Downloads all the subtitles of the video
@@ -1210,25 +1211,7 @@ class YoutubeDL(object):
                     self.report_error('Cannot write metadata to JSON file ' + infofn)
                     return
 
-        if self.params.get('writethumbnail', False):
-            if info_dict.get('thumbnail') is not None:
-                thumb_format = determine_ext(info_dict['thumbnail'], 'jpg')
-                thumb_filename = os.path.splitext(filename)[0] + '.' + thumb_format
-                if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(thumb_filename)):
-                    self.to_screen('[%s] %s: Thumbnail is already present' %
-                                   (info_dict['extractor'], info_dict['id']))
-                else:
-                    self.to_screen('[%s] %s: Downloading thumbnail ...' %
-                                   (info_dict['extractor'], info_dict['id']))
-                    try:
-                        uf = self.urlopen(info_dict['thumbnail'])
-                        with open(thumb_filename, 'wb') as thumbf:
-                            shutil.copyfileobj(uf, thumbf)
-                        self.to_screen('[%s] %s: Writing thumbnail to: %s' %
-                                       (info_dict['extractor'], info_dict['id'], thumb_filename))
-                    except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
-                        self.report_warning('Unable to download thumbnail "%s": %s' %
-                                            (info_dict['thumbnail'], compat_str(err)))
+        self._write_thumbnails(info_dict, filename)
 
         if not self.params.get('skip_download', False):
             try:
@@ -1676,3 +1659,39 @@ class YoutubeDL(object):
         if encoding is None:
             encoding = preferredencoding()
         return encoding
+
+    def _write_thumbnails(self, info_dict, filename):
+        if self.params.get('writethumbnail', False):
+            thumbnails = info_dict.get('thumbnails')
+            if thumbnails:
+                thumbnails = [thumbnails[-1]]
+        elif self.params.get('write_all_thumbnails', False):
+            thumbnails = info_dict.get('thumbnails')
+        else:
+            return
+
+        if not thumbnails:
+            # No thumbnails present, so return immediately
+            return
+
+        for t in thumbnails:
+            thumb_ext = determine_ext(t['url'], 'jpg')
+            suffix = '_%s' % t['id'] if len(thumbnails) > 1 else ''
+            thumb_display_id = '%s ' % t['id'] if len(thumbnails) > 1 else ''
+            thumb_filename = os.path.splitext(filename)[0] + suffix + '.' + thumb_ext
+
+            if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(thumb_filename)):
+                self.to_screen('[%s] %s: Thumbnail %sis already present' %
+                               (info_dict['extractor'], info_dict['id'], thumb_display_id))
+            else:
+                self.to_screen('[%s] %s: Downloading thumbnail %s...' %
+                               (info_dict['extractor'], info_dict['id'], thumb_display_id))
+                try:
+                    uf = self.urlopen(t['url'])
+                    with open(thumb_filename, 'wb') as thumbf:
+                        shutil.copyfileobj(uf, thumbf)
+                    self.to_screen('[%s] %s: Writing thumbnail %sto: %s' %
+                                   (info_dict['extractor'], info_dict['id'], thumb_display_id, thumb_filename))
+                except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
+                    self.report_warning('Unable to download thumbnail "%s": %s' %
+                                        (t['url'], compat_str(err)))
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index a3f82612c..04f668334 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -286,6 +286,7 @@ def _real_main(argv=None):
         'writeannotations': opts.writeannotations,
         'writeinfojson': opts.writeinfojson,
         'writethumbnail': opts.writethumbnail,
+        'write_all_thumbnails': opts.write_all_thumbnails,
         'writesubtitles': opts.writesubtitles,
         'writeautomaticsub': opts.writeautomaticsub,
         'allsubtitles': opts.allsubtitles,
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index e3b4b8a8a..a3b012ddb 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -638,6 +638,10 @@ def parseOpts(overrideArguments=None):
         '--write-thumbnail',
         action='store_true', dest='writethumbnail', default=False,
         help='write thumbnail image to disk')
+    thumbnail.add_option(
+        '--write-all-thumbnails',
+        action='store_true', dest='write_all_thumbnails', default=False,
+        help='write all thumbnail image formats to disk')
     thumbnail.add_option(
         '--list-thumbnails',
         action='store_true', dest='list_thumbnails', default=False,

From c14e88f0f561c5ac0a1cb9e6764fe4702bd9f7ca Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 25 Jan 2015 04:24:55 +0100
Subject: [PATCH 0077/6219] [YoutubeDL] Add --playlist-items option (Fixes
 #2662)

---
 AUTHORS                 |  1 +
 youtube_dl/YoutubeDL.py | 38 +++++++++++++++++++++++++++++++++-----
 youtube_dl/__init__.py  |  1 +
 youtube_dl/options.py   |  4 ++++
 4 files changed, 39 insertions(+), 5 deletions(-)

diff --git a/AUTHORS b/AUTHORS
index b8bf3cb6f..8362b6d8a 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -104,3 +104,4 @@ Ondřej Caletka
 Dinesh S
 Johan K. Jensen
 Yen Chi Hsuan
+Enam Mijbah Noor
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 0e73dc8ff..b7e93b8dd 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -137,6 +137,7 @@ class YoutubeDL(object):
     nooverwrites:      Prevent overwriting files.
     playliststart:     Playlist item to start at.
     playlistend:       Playlist item to end at.
+    playlist_items:    Specific indices of playlist to download.
     playlistreverse:   Download playlist items in reverse order.
     matchtitle:        Download only matching titles.
     rejecttitle:       Reject downloads for matching titles.
@@ -703,24 +704,51 @@ class YoutubeDL(object):
             if playlistend == -1:
                 playlistend = None
 
+            playlistitems_str = self.params.get('playlist_items', None)
+            playlistitems = None
+            if playlistitems_str is not None:
+                def iter_playlistitems(format):
+                    for string_segment in format.split(','):
+                        if '-' in string_segment:
+                            start, end = string_segment.split('-')
+                            for item in range(int(start), int(end) + 1):
+                                yield int(item)
+                        else:
+                            yield int(string_segment)
+                playlistitems = iter_playlistitems(playlistitems_str)
+
             ie_entries = ie_result['entries']
             if isinstance(ie_entries, list):
                 n_all_entries = len(ie_entries)
-                entries = ie_entries[playliststart:playlistend]
+                if playlistitems:
+                    entries = [ie_entries[i - 1] for i in playlistitems]
+                else:
+                    entries = ie_entries[playliststart:playlistend]
                 n_entries = len(entries)
                 self.to_screen(
                     "[%s] playlist %s: Collected %d video ids (downloading %d of them)" %
                     (ie_result['extractor'], playlist, n_all_entries, n_entries))
             elif isinstance(ie_entries, PagedList):
-                entries = ie_entries.getslice(
-                    playliststart, playlistend)
+                if playlistitems:
+                    entries = []
+                    for item in playlistitems:
+                        entries.extend(ie_entries.getslice(
+                            item - 1, item
+                        ))
+                else:
+                    entries = ie_entries.getslice(
+                        playliststart, playlistend)
                 n_entries = len(entries)
                 self.to_screen(
                     "[%s] playlist %s: Downloading %d videos" %
                     (ie_result['extractor'], playlist, n_entries))
             else:  # iterable
-                entries = list(itertools.islice(
-                    ie_entries, playliststart, playlistend))
+                if playlistitems:
+                    entry_list = list(ie_entries)
+                    entries = [entry_list[i - 1] for i in playlistitems]
+                else:
+                    entries = list(itertools.islice(
+                        ie_entries, playliststart, playlistend))
                 n_entries = len(entries)
                 self.to_screen(
                     "[%s] playlist %s: Downloading %d videos" %
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 04f668334..0bd7b68c3 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -333,6 +333,7 @@ def _real_main(argv=None):
         'sleep_interval': opts.sleep_interval,
         'external_downloader': opts.external_downloader,
         'list_thumbnails': opts.list_thumbnails,
+        'playlist_items': opts.playlist_items,
     }
 
     with YoutubeDL(ydl_opts) as ydl:
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index a3b012ddb..872835295 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -200,6 +200,10 @@ def parseOpts(overrideArguments=None):
         '--playlist-end',
         dest='playlistend', metavar='NUMBER', default=None, type=int,
         help='playlist video to end at (default is last)')
+    selection.add_option(
+        '--playlist-items',
+        dest='playlist_items', metavar='ITEM_SPEC', default=None,
+        help='playlist video items to download. Specify indices of the videos in the playlist seperated by commas like: "--playlist-items 1,2,5,8" if you want to download videos indexed 1, 2, 5, 8 in the playlist. You can specify range: "--playlist-items 1-3,7,10-13", it will download the videos at index 1, 2, 3, 7, 10, 11, 12 and 13.')
     selection.add_option(
         '--match-title',
         dest='matchtitle', metavar='REGEX',

From baeaeffce550b23848983cd281f173a7906cd7f9 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 25 Jan 2015 04:34:38 +0100
Subject: [PATCH 0078/6219] [options] Add support for infinite retries (Fixes
 #507)

---
 youtube_dl/__init__.py | 13 ++++++++-----
 youtube_dl/options.py  |  2 +-
 2 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 0bd7b68c3..09da8802d 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -143,10 +143,13 @@ def _real_main(argv=None):
             parser.error('invalid max_filesize specified')
         opts.max_filesize = numeric_limit
     if opts.retries is not None:
-        try:
-            opts.retries = int(opts.retries)
-        except (TypeError, ValueError):
-            parser.error('invalid retry count specified')
+        if opts.retries in ('inf', 'infinite'):
+            opts_retries = float('inf')
+        else:
+            try:
+                opts_retries = int(opts.retries)
+            except (TypeError, ValueError):
+                parser.error('invalid retry count specified')
     if opts.buffersize is not None:
         numeric_buffersize = FileDownloader.parse_bytes(opts.buffersize)
         if numeric_buffersize is None:
@@ -268,7 +271,7 @@ def _real_main(argv=None):
         'ignoreerrors': opts.ignoreerrors,
         'ratelimit': opts.ratelimit,
         'nooverwrites': opts.nooverwrites,
-        'retries': opts.retries,
+        'retries': opts_retries,
         'buffersize': opts.buffersize,
         'noresizebuffer': opts.noresizebuffer,
         'continuedl': opts.continue_dl,
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 872835295..1ddbdbc78 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -377,7 +377,7 @@ def parseOpts(overrideArguments=None):
     downloader.add_option(
         '-R', '--retries',
         dest='retries', metavar='RETRIES', default=10,
-        help='number of retries (default is %default)')
+        help='number of retries (default is %default), or "infinite".')
     downloader.add_option(
         '--buffer-size',
         dest='buffersize', metavar='SIZE', default='1024',

From 881e6a1f5c47a65348879f817ad833081e8c5ada Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 25 Jan 2015 04:49:44 +0100
Subject: [PATCH 0079/6219] Add --xattr-set-filesize option (Fixes #1348)

---
 youtube_dl/YoutubeDL.py         |  3 ++-
 youtube_dl/__init__.py          |  6 ++++++
 youtube_dl/downloader/common.py | 32 +++++++++++++++++---------------
 youtube_dl/downloader/http.py   |  8 ++++++++
 youtube_dl/options.py           |  4 ++++
 5 files changed, 37 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index b7e93b8dd..0241f7e3c 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -231,7 +231,8 @@ class YoutubeDL(object):
     The following parameters are not used by YoutubeDL itself, they are used by
     the FileDownloader:
     nopart, updatetime, buffersize, ratelimit, min_filesize, max_filesize, test,
-    noresizebuffer, retries, continuedl, noprogress, consoletitle
+    noresizebuffer, retries, continuedl, noprogress, consoletitle,
+    xattr_set_filesize.
 
     The following options are used by the post processors:
     prefer_ffmpeg:     If True, use ffmpeg instead of avconv if both are available,
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 09da8802d..112a8ba60 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -241,6 +241,11 @@ def _real_main(argv=None):
             'verboseOutput': opts.verbose,
             'exec_cmd': opts.exec_cmd,
         })
+    if opts.xattr_set_filesize:
+        try:
+            import xattr
+        except ImportError:
+            parser.error('setting filesize xattr requested but python-xattr is not available')
 
     ydl_opts = {
         'usenetrc': opts.usenetrc,
@@ -337,6 +342,7 @@ def _real_main(argv=None):
         'external_downloader': opts.external_downloader,
         'list_thumbnails': opts.list_thumbnails,
         'playlist_items': opts.playlist_items,
+        'xattr_set_filesize': opts.xattr_set_filesize,
     }
 
     with YoutubeDL(ydl_opts) as ydl:
diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index c35c42c1d..7bb3a948d 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -25,21 +25,23 @@ class FileDownloader(object):
 
     Available options:
 
-    verbose:           Print additional info to stdout.
-    quiet:             Do not print messages to stdout.
-    ratelimit:         Download speed limit, in bytes/sec.
-    retries:           Number of times to retry for HTTP error 5xx
-    buffersize:        Size of download buffer in bytes.
-    noresizebuffer:    Do not automatically resize the download buffer.
-    continuedl:        Try to continue downloads if possible.
-    noprogress:        Do not print the progress bar.
-    logtostderr:       Log messages to stderr instead of stdout.
-    consoletitle:      Display progress in console window's titlebar.
-    nopart:            Do not use temporary .part files.
-    updatetime:        Use the Last-modified header to set output file timestamps.
-    test:              Download only first bytes to test the downloader.
-    min_filesize:      Skip files smaller than this size
-    max_filesize:      Skip files larger than this size
+    verbose:            Print additional info to stdout.
+    quiet:              Do not print messages to stdout.
+    ratelimit:          Download speed limit, in bytes/sec.
+    retries:            Number of times to retry for HTTP error 5xx
+    buffersize:         Size of download buffer in bytes.
+    noresizebuffer:     Do not automatically resize the download buffer.
+    continuedl:         Try to continue downloads if possible.
+    noprogress:         Do not print the progress bar.
+    logtostderr:        Log messages to stderr instead of stdout.
+    consoletitle:       Display progress in console window's titlebar.
+    nopart:             Do not use temporary .part files.
+    updatetime:         Use the Last-modified header to set output file timestamps.
+    test:               Download only first bytes to test the downloader.
+    min_filesize:       Skip files smaller than this size
+    max_filesize:       Skip files larger than this size
+    xattr_set_filesize: Set ytdl.filesize user xattribute with expected size.
+                        (experimenatal)
 
     Subclasses of this one must re-define the real_download method.
     """
diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index 4db50ee90..8a1d578d5 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -157,6 +157,14 @@ class HttpFD(FileDownloader):
                 except (OSError, IOError) as err:
                     self.report_error('unable to open for writing: %s' % str(err))
                     return False
+
+                if self.params.get('xattr_set_filesize', False) and data_len is not None:
+                    try:
+                        import xattr
+                        xattr.setxattr(tmpfilename, 'user.ytdl.filesize', str(data_len))
+                    except(OSError, IOError, ImportError) as err:
+                        self.report_error('unable to set filesize xattr: %s' % str(err))
+
             try:
                 stream.write(data_block)
             except (IOError, OSError) as err:
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 1ddbdbc78..dbc6f5528 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -394,6 +394,10 @@ def parseOpts(overrideArguments=None):
         '--playlist-reverse',
         action='store_true',
         help='Download playlist videos in reverse order')
+    downloader.add_option(
+        '--xattr-set-filesize',
+        dest='xattr_set_filesize', action='store_true',
+        help='(experimental) set file xattribute ytdl.filesize with expected filesize')
     downloader.add_option(
         '--external-downloader',
         dest='external_downloader', metavar='COMMAND',

From e1ccc04e9f68988df0520b7502edc6479b62378f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 25 Jan 2015 04:56:32 +0100
Subject: [PATCH 0080/6219] Test rtmpdump on travis (Fixes #1601)

---
 .travis.yml                         | 3 +++
 youtube_dl/extractor/folketinget.py | 4 +---
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index f14014414..fb34299fc 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -4,6 +4,9 @@ python:
   - "2.7"
   - "3.3"
   - "3.4"
+before_install:
+  - sudo apt-get update -qq
+  - sudo apt-get install -yqq rtmpdump
 script: nosetests test --verbose
 notifications:
   email:
diff --git a/youtube_dl/extractor/folketinget.py b/youtube_dl/extractor/folketinget.py
index 68e2db943..0fb29de75 100644
--- a/youtube_dl/extractor/folketinget.py
+++ b/youtube_dl/extractor/folketinget.py
@@ -16,6 +16,7 @@ class FolketingetIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?ft\.dk/webtv/video/[^?#]*?\.(?P<id>[0-9]+)\.aspx'
     _TEST = {
         'url': 'http://www.ft.dk/webtv/video/20141/eru/td.1165642.aspx?as=1#player',
+        'md5': '6269e8626fa1a891bf5369b386ae996a',
         'info_dict': {
             'id': '1165642',
             'ext': 'mp4',
@@ -29,9 +30,6 @@ class FolketingetIE(InfoExtractor):
             'upload_date': '20141120',
             'duration': 3960,
         },
-        'params': {
-            'skip_download': 'rtmpdump required',
-        }
     }
 
     def _real_extract(self, url):

From 7d346331b53498b5966790677e0b41631edd8a4e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 25 Jan 2015 05:15:47 +0100
Subject: [PATCH 0081/6219] [audiomack:album] Update testcase

---
 test/helper.py                    |  2 +-
 youtube_dl/extractor/audiomack.py | 13 +++++++++----
 2 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/test/helper.py b/test/helper.py
index c416f388c..27a68091f 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -140,7 +140,7 @@ def expect_info_dict(self, got_dict, expected_dict):
     # Are checkable fields missing from the test case definition?
     test_info_dict = dict((key, value if not isinstance(value, compat_str) or len(value) < 250 else 'md5:' + md5(value))
                           for key, value in got_dict.items()
-                          if value and key in ('title', 'description', 'uploader', 'upload_date', 'timestamp', 'uploader_id', 'location'))
+                          if value and key in ('id', 'title', 'description', 'uploader', 'upload_date', 'timestamp', 'uploader_id', 'location'))
     missing_keys = set(test_info_dict.keys()) - set(expected_dict.keys())
     if missing_keys:
         def _repr(v):
diff --git a/youtube_dl/extractor/audiomack.py b/youtube_dl/extractor/audiomack.py
index 8bfe50214..693ba22c6 100644
--- a/youtube_dl/extractor/audiomack.py
+++ b/youtube_dl/extractor/audiomack.py
@@ -88,16 +88,21 @@ class AudiomackAlbumIE(InfoExtractor):
         # Album playlist ripped from fakeshoredrive with no metadata
         {
             'url': 'http://www.audiomack.com/album/fakeshoredrive/ppp-pistol-p-project',
+            'info_dict': {
+                'title': 'PPP (Pistol P Project)',
+                'id': '837572',
+            },
             'playlist': [{
                 'info_dict': {
-                    'title': '9.-heaven-or-hell-chimaca-ft-zuse-prod-by-dj-fu',
-                    'id': '9.-heaven-or-hell-chimaca-ft-zuse-prod-by-dj-fu',
+                    'title': 'PPP (Pistol P Project) - 9. Heaven or Hell (CHIMACA) ft Zuse (prod by DJ FU)',
+                    'id': '837577',
                     'ext': 'mp3',
+                    'uploader': 'Lil Herb a.k.a. G Herbo',
                 }
             }],
             'params': {
-                'playliststart': 8,
-                'playlistend': 8,
+                'playliststart': 9,
+                'playlistend': 9,
             }
         }
     ]

From 37f4ce538ab572812465fa6d0e429bf767a030fd Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 25 Jan 2015 05:17:15 +0100
Subject: [PATCH 0082/6219] [smotri] Fix test case

---
 youtube_dl/extractor/smotri.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/smotri.py b/youtube_dl/extractor/smotri.py
index 26f361c93..e94f41362 100644
--- a/youtube_dl/extractor/smotri.py
+++ b/youtube_dl/extractor/smotri.py
@@ -102,6 +102,7 @@ class SmotriIE(InfoExtractor):
                 'uploader_id': 'mopeder',
                 'duration': 71,
                 'thumbnail': 'http://frame9.loadup.ru/d7/32/2888853.2.3.jpg',
+                'upload_date': '20150114',
             },
         },
         # swf player

From cc1237f4845e108017687d6727c3dfe6efc3cb4d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 25 Jan 2015 05:17:38 +0100
Subject: [PATCH 0083/6219] [__init__] Work around flake8 false positive

---
 youtube_dl/__init__.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 112a8ba60..71d2c6f35 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -244,6 +244,7 @@ def _real_main(argv=None):
     if opts.xattr_set_filesize:
         try:
             import xattr
+            xattr  # Confuse flake8
         except ImportError:
             parser.error('setting filesize xattr requested but python-xattr is not available')
 

From 8604e882a8f99fcd632efbb94e449477d860218d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 25 Jan 2015 05:23:21 +0100
Subject: [PATCH 0084/6219] [ubu] Fix test and modernize

---
 youtube_dl/extractor/ubu.py | 27 ++++++++++++++-------------
 1 file changed, 14 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/ubu.py b/youtube_dl/extractor/ubu.py
index 0182d67ec..d50237758 100644
--- a/youtube_dl/extractor/ubu.py
+++ b/youtube_dl/extractor/ubu.py
@@ -3,50 +3,51 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import int_or_none
+from ..utils import (
+    int_or_none,
+    qualities,
+)
 
 
 class UbuIE(InfoExtractor):
     _VALID_URL = r'http://(?:www\.)?ubu\.com/film/(?P<id>[\da-z_-]+)\.html'
     _TEST = {
         'url': 'http://ubu.com/film/her_noise.html',
-        'md5': '8edd46ee8aa6b265fb5ed6cf05c36bc9',
+        'md5': '138d5652618bf0f03878978db9bef1ee',
         'info_dict': {
             'id': 'her_noise',
-            'ext': 'mp4',
+            'ext': 'm4v',
             'title': 'Her Noise - The Making Of (2007)',
             'duration': 3600,
         },
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
         title = self._html_search_regex(
             r'<title>.+?Film &amp; Video: ([^<]+)</title>', webpage, 'title')
 
         duration = int_or_none(self._html_search_regex(
-            r'Duration: (\d+) minutes', webpage, 'duration', fatal=False, default=None))
-        if duration:
-            duration *= 60
+            r'Duration: (\d+) minutes', webpage, 'duration', fatal=False),
+            invscale=60)
 
         formats = []
-
         FORMAT_REGEXES = [
-            ['sq', r"'flashvars'\s*,\s*'file=([^']+)'"],
-            ['hq', r'href="(http://ubumexico\.centro\.org\.mx/video/[^"]+)"']
+            ('sq', r"'flashvars'\s*,\s*'file=([^']+)'"),
+            ('hq', r'href="(http://ubumexico\.centro\.org\.mx/video/[^"]+)"'),
         ]
-
+        preference = qualities([fid for fid, _ in FORMAT_REGEXES])
         for format_id, format_regex in FORMAT_REGEXES:
             m = re.search(format_regex, webpage)
             if m:
                 formats.append({
                     'url': m.group(1),
                     'format_id': format_id,
+                    'preference': preference(format_id),
                 })
+        self._sort_formats(formats)
 
         return {
             'id': video_id,

From ee69b99af6d00202a5ad8510604c807876d1b7b1 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 25 Jan 2015 06:15:51 +0100
Subject: [PATCH 0085/6219] [YoutubeDL] clarify hook documentation

---
 youtube_dl/YoutubeDL.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 0241f7e3c..b772f87f1 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -198,11 +198,12 @@ class YoutubeDL(object):
                        postprocessor.
     progress_hooks:    A list of functions that get called on download
                        progress, with a dictionary with the entries
-                       * filename: The final filename
-                       * status: One of "downloading" and "finished"
-
-                       The dict may also have some of the following entries:
+                       * status: One of "downloading" and "finished".
+                                 Check this first and ignore unknown values.
 
+                       If status is one of "downloading" or "finished", the
+                       following properties may also be present:
+                       * filename: The final filename (always present)
                        * downloaded_bytes: Bytes on disk
                        * total_bytes: Size of the whole file, None if unknown
                        * tmpfilename: The filename we're currently writing to
@@ -1251,6 +1252,7 @@ class YoutubeDL(object):
                     if self.params.get('verbose'):
                         self.to_stdout('[debug] Invoking downloader on %r' % info.get('url'))
                     return fd.download(name, info)
+
                 if info_dict.get('requested_formats') is not None:
                     downloaded = []
                     success = True

From fe41ddbb285abccc3c4d7a3ebc1238c13ec72577 Mon Sep 17 00:00:00 2001
From: David Development <david-dev@live.de>
Date: Sun, 25 Jan 2015 11:53:53 +0100
Subject: [PATCH 0086/6219] refactoring - bug fixes

---
 youtube_dl/extractor/rtl2.py | 66 ++++++++++++++++++------------------
 1 file changed, 33 insertions(+), 33 deletions(-)

diff --git a/youtube_dl/extractor/rtl2.py b/youtube_dl/extractor/rtl2.py
index 14b45e86e..7086d698f 100644
--- a/youtube_dl/extractor/rtl2.py
+++ b/youtube_dl/extractor/rtl2.py
@@ -2,7 +2,6 @@
 from __future__ import unicode_literals
 
 import re
-import json
 
 from .common import InfoExtractor
 from ..utils import (
@@ -14,82 +13,83 @@ from ..utils import (
 
 
 class RTL2IE(InfoExtractor):
-    """Information Extractor for RTL NOW, RTL2 NOW, RTL NITRO, SUPER RTL NOW, VOX NOW and n-tv NOW"""
-    _VALID_URL = r'http?://(?P<url>(?P<domain>(www\.)?rtl2\.de)/.*/(?P<video_id>.*))'
+    """Information Extractor for RTL2"""
+    _VALID_URL = r'http?://(?P<url>(?P<domain>(www\.)?rtl2\.de)/.*/(?P<video_id>.*))/'
     _TESTS = [{
             'url': 'http://www.rtl2.de/sendung/grip-das-motormagazin/folge/folge-203-0',
             'info_dict': {
                 'id': 'folge-203-0',
                 'ext': 'f4v',
-                'title': 'GRIP sucht den Sommerk\xf6nig',
+                'title': 'GRIP sucht den Sommerkönig',
                 'description' : 'Matthias, Det und Helge treten gegeneinander an.'
             },
             'params': {
                 # rtmp download
-                #'skip_download': True,
+                'skip_download': True,
             },
         },
         {
             'url': 'http://www.rtl2.de/sendung/koeln-50667/video/5512-anna/21040-anna-erwischt-alex/',
             'info_dict': {
                 'id': '21040-anna-erwischt-alex',
-                'ext': 'f4v',
-                'title': 'GRIP sucht den Sommerk\xf6nig',
-                'description' : 'Matthias, Det und Helge treten gegeneinander an.'
+                'ext': 'mp4',
+                'title': 'Anna erwischt Alex!',
+                'description' : 'Anna ist Alex\' Tochter bei Köln 50667.'
             },
             'params': {
                 # rtmp download
-                #'skip_download': True,
+                'skip_download': True,
             },
         },
     ]
 
     def _real_extract(self, url):
+        
+        #Some rtl2 urls have no slash at the end, so append it.
+        if not url.endswith("/"):
+            url += '/'
+        
         mobj = re.match(self._VALID_URL, url)
-        video_page_url = 'http://%s/' % mobj.group('domain')
         video_id = mobj.group('video_id')
-    
-        webpage = self._download_webpage('http://' + mobj.group('url'), video_id)
 
-        vico_id = self._html_search_regex(r'vico_id\s*:\s*([0-9]+)', webpage, '%s');
-        vivi_id = self._html_search_regex(r'vivi_id\s*:\s*([0-9]+)', webpage, '%s');
+        webpage = self._download_webpage(url, video_id)
+
+        vico_id = self._html_search_regex(r'vico_id\s*:\s*([0-9]+)', webpage, 'vico_id not found');
+        vivi_id = self._html_search_regex(r'vivi_id\s*:\s*([0-9]+)', webpage, 'vivi_id not found');
 
         info_url = 'http://www.rtl2.de/video/php/get_video.php?vico_id=' + vico_id + '&vivi_id=' + vivi_id
         webpage = self._download_webpage(info_url, '')
 
-        video_info = json.loads(webpage)
+        video_info = self._download_json(info_url, video_id)
 
-        download_url = video_info["video"]["streamurl"] # self._html_search_regex(r'streamurl\":\"(.*?)\"', webpage, '%s');
-        title = video_info["video"]["titel"] # self._html_search_regex(r'titel\":\"(.*?)\"', webpage, '%s');
-        description = video_info["video"]["beschreibung"] # self._html_search_regex(r'beschreibung\":\"(.*?)\"', webpage, '%s');
-        #ext = self._html_search_regex(r'streamurl\":\".*?(\..{2,4})\"', webpage, '%s');
+        download_url = video_info["video"]["streamurl"]
+        title = video_info["video"]["titel"]
+        description = video_info["video"]["beschreibung"]
 
         thumbnail = video_info["video"]["image"]
 
         download_url = download_url.replace("\\", "")
 
         stream_url = 'mp4:' + self._html_search_regex(r'ondemand/(.*)', download_url, '%s')
-
-        #print(download_url)
-        #print(stream_url)
-        #print(title)
-        #print(description)
-        #print(video_id)
         
-        formats = []
-
-        fmt = {
-            'url' : download_url,
+        #Debug output
+        #print('URL: '        + url)
+        #print('DL URL: '     + download_url)
+        #print('Stream URL: ' + stream_url)
+        #print('Title: '      + title)
+        #print('Description: '+ description)
+        #print('Video ID: '   + video_id)
+                
+        formats = [{
+                'url' : download_url,
                 #'app': 'ondemand?_fcs_vhost=cp108781.edgefcs.net',
                 'play_path': stream_url,
                 'player_url': 'http://www.rtl2.de/flashplayer/vipo_player.swf',
                 'page_url': url,
                 'flash_version' : "LNX 11,2,202,429",
                 'rtmp_conn' : ["S:connect", "O:1", "NS:pageUrl:" + url, "NB:fpad:0", "NN:videoFunction:1", "O:0"],
-                'no_resume' : 1,
-        }
-
-        formats.append(fmt)
+                'no_resume' : True,
+            }]
 
         return {
             'id': video_id,

From c9326b38b824eb8babbf2306b3bfe4bd5e683cfe Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 25 Jan 2015 18:09:09 +0100
Subject: [PATCH 0087/6219] flake8: Ignore .git

---
 setup.cfg | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.cfg b/setup.cfg
index 13dcd8af6..02aa56257 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -2,5 +2,5 @@
 universal = True
 
 [flake8]
-exclude = youtube_dl/extractor/__init__.py,devscripts/buildserver.py,setup.py,build
+exclude = youtube_dl/extractor/__init__.py,devscripts/buildserver.py,setup.py,build,.git
 ignore = E501

From 3dee7826e7133fb4a73aa0aabaee78499e49264f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 25 Jan 2015 18:09:48 +0100
Subject: [PATCH 0088/6219] [rtl2] PEP8, simplify, make rtmp tests run (#470)

---
 youtube_dl/downloader/rtmp.py  |  13 ++--
 youtube_dl/extractor/common.py |   7 +-
 youtube_dl/extractor/rtl2.py   | 116 +++++++++++++--------------------
 3 files changed, 57 insertions(+), 79 deletions(-)

diff --git a/youtube_dl/downloader/rtmp.py b/youtube_dl/downloader/rtmp.py
index 6f7ad588e..e06ebe826 100644
--- a/youtube_dl/downloader/rtmp.py
+++ b/youtube_dl/downloader/rtmp.py
@@ -105,7 +105,8 @@ class RtmpFD(FileDownloader):
         conn = info_dict.get('rtmp_conn', None)
         protocol = info_dict.get('rtmp_protocol', None)
         no_resume = info_dict.get('no_resume', False)
-        
+        continue_dl = info_dict.get('continuedl', False)
+
         self.report_destination(filename)
         tmpfilename = self.temp_name(filename)
         test = self.params.get('test', False)
@@ -142,10 +143,12 @@ class RtmpFD(FileDownloader):
             basic_args += ['--conn', conn]
         if protocol is not None:
             basic_args += ['--protocol', protocol]
-        if not no_resume:
-            basic_args += ['--resume']
-        
-        args = basic_args + [[], ['--skip', '1']][not live and self.params.get('continuedl', False)]
+
+        args = basic_args
+        if not no_resume and continue_dl and not live:
+            args += ['--resume']
+        if not live and continue_dl:
+            args += ['--skip', '1']
 
         if sys.platform == 'win32' and sys.version_info < (3, 0):
             # Windows subprocess module does not actually support Unicode
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 7b7a832dc..388c55e99 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -114,8 +114,11 @@ class InfoExtractor(object):
                     * http_post_data  Additional data to send with a POST
                                  request.
                     * stretched_ratio  If given and not 1, indicates that the
-                                       video's pixels are not square.
-                                       width : height ratio as float.
+                                 video's pixels are not square.
+                                 width : height ratio as float.
+                    * no_resume  The server does not support resuming the
+                                 (HTTP or RTMP) download. Boolean.
+
     url:            Final video URL.
     ext:            Video filename extension.
     format:         The video format, defaults to ext (used for --get-format)
diff --git a/youtube_dl/extractor/rtl2.py b/youtube_dl/extractor/rtl2.py
index 7086d698f..72cd80498 100644
--- a/youtube_dl/extractor/rtl2.py
+++ b/youtube_dl/extractor/rtl2.py
@@ -1,100 +1,72 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
-    clean_html,
-    unified_strdate,
-    int_or_none,
-)
 
 
 class RTL2IE(InfoExtractor):
-    """Information Extractor for RTL2"""
-    _VALID_URL = r'http?://(?P<url>(?P<domain>(www\.)?rtl2\.de)/.*/(?P<video_id>.*))/'
+    _VALID_URL = r'http?://(?:www\.)?rtl2\.de/[^?#]*?/(?P<id>[^?#/]*?)(?:$|/(?:$|[?#]))'
     _TESTS = [{
-            'url': 'http://www.rtl2.de/sendung/grip-das-motormagazin/folge/folge-203-0',
-            'info_dict': {
-                'id': 'folge-203-0',
-                'ext': 'f4v',
-                'title': 'GRIP sucht den Sommerkönig',
-                'description' : 'Matthias, Det und Helge treten gegeneinander an.'
-            },
-            'params': {
-                # rtmp download
-                'skip_download': True,
-            },
+        'url': 'http://www.rtl2.de/sendung/grip-das-motormagazin/folge/folge-203-0',
+        'md5': 'bfcc179030535b08dc2b36b469b5adc7',
+        'info_dict': {
+            'id': 'folge-203-0',
+            'ext': 'f4v',
+            'title': 'GRIP sucht den Sommerkönig',
+            'description': 'Matthias, Det und Helge treten gegeneinander an.'
         },
-        {
-            'url': 'http://www.rtl2.de/sendung/koeln-50667/video/5512-anna/21040-anna-erwischt-alex/',
-            'info_dict': {
-                'id': '21040-anna-erwischt-alex',
-                'ext': 'mp4',
-                'title': 'Anna erwischt Alex!',
-                'description' : 'Anna ist Alex\' Tochter bei Köln 50667.'
-            },
-            'params': {
-                # rtmp download
-                'skip_download': True,
-            },
+    }, {
+        'url': 'http://www.rtl2.de/sendung/koeln-50667/video/5512-anna/21040-anna-erwischt-alex/',
+        'md5': 'ffcd517d2805b57ce11a58a2980c2b02',
+        'info_dict': {
+            'id': '21040-anna-erwischt-alex',
+            'ext': 'mp4',
+            'title': 'Anna erwischt Alex!',
+            'description': 'Anna ist Alex\' Tochter bei Köln 50667.'
         },
-    ]
+    }]
 
     def _real_extract(self, url):
-        
-        #Some rtl2 urls have no slash at the end, so append it.
-        if not url.endswith("/"):
+        # Some rtl2 urls have no slash at the end, so append it.
+        if not url.endswith('/'):
             url += '/'
-        
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('video_id')
 
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        vico_id = self._html_search_regex(r'vico_id\s*:\s*([0-9]+)', webpage, 'vico_id not found');
-        vivi_id = self._html_search_regex(r'vivi_id\s*:\s*([0-9]+)', webpage, 'vivi_id not found');
-
+        vico_id = self._html_search_regex(
+            r'vico_id\s*:\s*([0-9]+)', webpage, 'vico_id')
+        vivi_id = self._html_search_regex(
+            r'vivi_id\s*:\s*([0-9]+)', webpage, 'vivi_id')
         info_url = 'http://www.rtl2.de/video/php/get_video.php?vico_id=' + vico_id + '&vivi_id=' + vivi_id
         webpage = self._download_webpage(info_url, '')
 
-        video_info = self._download_json(info_url, video_id)
+        info = self._download_json(info_url, video_id)
+        video_info = info['video']
+        title = video_info['titel']
+        description = video_info.get('beschreibung')
+        thumbnail = video_info.get('image')
 
-        download_url = video_info["video"]["streamurl"]
-        title = video_info["video"]["titel"]
-        description = video_info["video"]["beschreibung"]
+        download_url = video_info['streamurl']
+        download_url = download_url.replace('\\', '')
+        stream_url = 'mp4:' + self._html_search_regex(r'ondemand/(.*)', download_url, 'stream URL')
+        rtmp_conn = ["S:connect", "O:1", "NS:pageUrl:" + url, "NB:fpad:0", "NN:videoFunction:1", "O:0"]
 
-        thumbnail = video_info["video"]["image"]
-
-        download_url = download_url.replace("\\", "")
-
-        stream_url = 'mp4:' + self._html_search_regex(r'ondemand/(.*)', download_url, '%s')
-        
-        #Debug output
-        #print('URL: '        + url)
-        #print('DL URL: '     + download_url)
-        #print('Stream URL: ' + stream_url)
-        #print('Title: '      + title)
-        #print('Description: '+ description)
-        #print('Video ID: '   + video_id)
-                
         formats = [{
-                'url' : download_url,
-                #'app': 'ondemand?_fcs_vhost=cp108781.edgefcs.net',
-                'play_path': stream_url,
-                'player_url': 'http://www.rtl2.de/flashplayer/vipo_player.swf',
-                'page_url': url,
-                'flash_version' : "LNX 11,2,202,429",
-                'rtmp_conn' : ["S:connect", "O:1", "NS:pageUrl:" + url, "NB:fpad:0", "NN:videoFunction:1", "O:0"],
-                'no_resume' : True,
-            }]
+            'url': download_url,
+            'play_path': stream_url,
+            'player_url': 'http://www.rtl2.de/flashplayer/vipo_player.swf',
+            'page_url': url,
+            'flash_version': 'LNX 11,2,202,429',
+            'rtmp_conn': rtmp_conn,
+            'no_resume': True,
+        }]
+        self._sort_formats(formats)
 
         return {
             'id': video_id,
             'title': title,
-            'thumbnail' : thumbnail,
-            'description' : description,
+            'thumbnail': thumbnail,
+            'description': description,
             'formats': formats,
         }

From b1b0b1ca3030a2b04dd94ec28d5ece3fda88a282 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 25 Jan 2015 18:14:59 +0100
Subject: [PATCH 0089/6219] [generic] Improve description testcase in rss test

---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index a028c4ed4..ad16b8330 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -362,7 +362,7 @@ class GenericIE(InfoExtractor):
             'info_dict': {
                 'id': 'http://phihag.de/2014/youtube-dl/rss2.xml',
                 'title': 'Zero Punctuation',
-                'description': 're:'
+                'description': 're:.*groundbreaking video review series.*'
             },
             'playlist_mincount': 11,
         },

From 6d2749aac407df1e039f5b61a294991c1e810cff Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 25 Jan 2015 18:56:04 +0100
Subject: [PATCH 0090/6219] [drtv] Prefer the version without spoken subtitles
 (fixes #4779)

For example for http://www.dr.dk/tv/se/moderne-klassikere/moderne-klassikere-one-republic-apologize#!/, there's a version where everytime someone speaks in English a computer voice translates it.
---
 youtube_dl/extractor/drtv.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/drtv.py b/youtube_dl/extractor/drtv.py
index c44adb109..510ef04b0 100644
--- a/youtube_dl/extractor/drtv.py
+++ b/youtube_dl/extractor/drtv.py
@@ -48,14 +48,20 @@ class DRTVIE(SubtitlesInfoExtractor):
             elif asset['Kind'] == 'VideoResource':
                 duration = asset['DurationInMilliseconds'] / 1000.0
                 restricted_to_denmark = asset['RestrictedToDenmark']
+                spoken_subtitles = asset['Target'] == 'SpokenSubtitles'
                 for link in asset['Links']:
                     target = link['Target']
                     uri = link['Uri']
+                    format_id = target
+                    preference = -1 if target == 'HDS' else -2
+                    if spoken_subtitles:
+                        preference -= 2
+                        format_id += '-spoken-subtitles'
                     formats.append({
                         'url': uri + '?hdcore=3.3.0&plugin=aasp-3.3.0.99.43' if target == 'HDS' else uri,
-                        'format_id': target,
+                        'format_id': format_id,
                         'ext': link['FileFormat'],
-                        'preference': -1 if target == 'HDS' else -2,
+                        'preference': preference,
                     })
                 subtitles_list = asset.get('SubtitlesList')
                 if isinstance(subtitles_list, list):

From 96a53167fa64293506f446d0c2bf3e0db6c8df31 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 26 Jan 2015 00:32:31 +0600
Subject: [PATCH 0091/6219] [common] Generalize URLs' HTTP errors pre-testing

---
 youtube_dl/extractor/common.py | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 388c55e99..478232682 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -14,6 +14,7 @@ import xml.etree.ElementTree
 
 from ..compat import (
     compat_cookiejar,
+    compat_HTTPError,
     compat_http_client,
     compat_urllib_error,
     compat_urllib_parse_urlparse,
@@ -26,6 +27,7 @@ from ..utils import (
     compiled_regex_type,
     ExtractorError,
     float_or_none,
+    HEADRequest,
     int_or_none,
     RegexNotFoundError,
     sanitize_filename,
@@ -716,6 +718,27 @@ class InfoExtractor(object):
             )
         formats.sort(key=_formats_key)
 
+    def _check_formats(self, formats, video_id):
+        if formats:
+            formats[:] = filter(
+                lambda f: self._is_valid_url(
+                    f['url'], video_id,
+                    item='%s video format' % f.get('format_id') if f.get('format_id') else 'video'),
+                formats)
+
+    def _is_valid_url(self, url, video_id, item='video'):
+        try:
+            self._request_webpage(
+                HEADRequest(url), video_id,
+                'Checking %s URL' % item)
+            return True
+        except ExtractorError as e:
+            if isinstance(e.cause, compat_HTTPError):
+                self.report_warning(
+                    '%s URL is invalid, skipping' % item, video_id)
+                return False
+            raise
+
     def http_scheme(self):
         """ Either "http:" or "https:", depending on the user's preferences """
         return (

From a57e8ce6580202c179c38a15abc31f84ca471521 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 26 Jan 2015 00:33:42 +0600
Subject: [PATCH 0092/6219] [lynda] Pre-test video URLs for HTTP errors (Closes
 #2185, closes #4782)

---
 youtube_dl/extractor/lynda.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index 26e84970d..762cefa34 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -85,6 +85,7 @@ class LyndaIE(SubtitlesInfoExtractor):
                 } for format_id, video_url in prioritized_streams['0'].items()
             ])
 
+        self._check_formats(formats, video_id)
         self._sort_formats(formats)
 
         if self._downloader.params.get('listsubtitles', False):

From d862a4f94fac46e5c6de790a9b66d78d463d666d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 26 Jan 2015 00:34:31 +0600
Subject: [PATCH 0093/6219] [spiegel] Use generalized formats pre-testing

---
 youtube_dl/extractor/spiegel.py | 20 ++------------------
 1 file changed, 2 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/spiegel.py b/youtube_dl/extractor/spiegel.py
index f345883c7..b868241d5 100644
--- a/youtube_dl/extractor/spiegel.py
+++ b/youtube_dl/extractor/spiegel.py
@@ -4,14 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urlparse,
-    compat_HTTPError,
-)
-from ..utils import (
-    HEADRequest,
-    ExtractorError,
-)
+from ..compat import compat_urlparse
 from .spiegeltv import SpiegeltvIE
 
 
@@ -72,16 +65,6 @@ class SpiegelIE(InfoExtractor):
             if n.tag.startswith('type') and n.tag != 'type6':
                 format_id = n.tag.rpartition('type')[2]
                 video_url = base_url + n.find('./filename').text
-                # Test video URLs beforehand as some of them are invalid
-                try:
-                    self._request_webpage(
-                        HEADRequest(video_url), video_id,
-                        'Checking %s video URL' % format_id)
-                except ExtractorError as e:
-                    if isinstance(e.cause, compat_HTTPError) and e.cause.code == 404:
-                        self.report_warning(
-                            '%s video URL is invalid, skipping' % format_id, video_id)
-                        continue
                 formats.append({
                     'format_id': format_id,
                     'url': video_url,
@@ -94,6 +77,7 @@ class SpiegelIE(InfoExtractor):
                 })
         duration = float(idoc[0].findall('./duration')[0].text)
 
+        self._check_formats(formats, video_id)
         self._sort_formats(formats)
 
         return {

From 80a49d3d7bcd235ba15bd491cc62a0345c9abce1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 26 Jan 2015 02:08:29 +0600
Subject: [PATCH 0094/6219] Credit @David-Development for rtl2 (#4780)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 8362b6d8a..1596a7548 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -105,3 +105,4 @@ Dinesh S
 Johan K. Jensen
 Yen Chi Hsuan
 Enam Mijbah Noor
+David Luhmer

From 2b1bd292ae72fdea4ab731f93777a99b3729b31d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 25 Jan 2015 21:40:43 +0100
Subject: [PATCH 0095/6219] release 2015.01.25

---
 README.md             | 23 +++++++++++++++++++++--
 youtube_dl/version.py |  2 +-
 2 files changed, 22 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index 36b87444e..0580ab2a3 100644
--- a/README.md
+++ b/README.md
@@ -93,6 +93,14 @@ which means you can modify it, redistribute it or use it however you like.
 ## Video Selection:
     --playlist-start NUMBER          playlist video to start at (default is 1)
     --playlist-end NUMBER            playlist video to end at (default is last)
+    --playlist-items ITEM_SPEC       playlist video items to download. Specify
+                                     indices of the videos in the playlist
+                                     seperated by commas like: "--playlist-items
+                                     1,2,5,8" if you want to download videos
+                                     indexed 1, 2, 5, 8 in the playlist. You can
+                                     specify range: "--playlist-items
+                                     1-3,7,10-13", it will download the videos
+                                     at index 1, 2, 3, 7, 10, 11, 12 and 13.
     --match-title REGEX              download only matching titles (regex or
                                      caseless sub-string)
     --reject-title REGEX             skip download for matching titles (regex or
@@ -124,7 +132,8 @@ which means you can modify it, redistribute it or use it however you like.
 ## Download Options:
     -r, --rate-limit LIMIT           maximum download rate in bytes per second
                                      (e.g. 50K or 4.2M)
-    -R, --retries RETRIES            number of retries (default is 10)
+    -R, --retries RETRIES            number of retries (default is 10), or
+                                     "infinite".
     --buffer-size SIZE               size of download buffer (e.g. 1024 or 16K)
                                      (default is 1024)
     --no-resize-buffer               do not automatically adjust the buffer
@@ -132,6 +141,11 @@ which means you can modify it, redistribute it or use it however you like.
                                      automatically resized from an initial value
                                      of SIZE.
     --playlist-reverse               Download playlist videos in reverse order
+    --xattr-set-filesize             (experimental) set file xattribute
+                                     ytdl.filesize with expected filesize
+    --external-downloader COMMAND    (experimental) Use the specified external
+                                     downloader. Currently supports
+                                     aria2c,curl,wget
 
 ## Filesystem Options:
     -a, --batch-file FILE            file containing URLs to download ('-' for
@@ -191,7 +205,6 @@ which means you can modify it, redistribute it or use it however you like.
     --write-info-json                write video metadata to a .info.json file
     --write-annotations              write video annotations to a .annotation
                                      file
-    --write-thumbnail                write thumbnail image to disk
     --load-info FILE                 json file containing the video information
                                      (created with the "--write-json" option)
     --cookies FILE                   file to read cookies from and dump cookie
@@ -206,6 +219,12 @@ which means you can modify it, redistribute it or use it however you like.
     --no-cache-dir                   Disable filesystem caching
     --rm-cache-dir                   Delete all filesystem cache files
 
+## Thumbnail images:
+    --write-thumbnail                write thumbnail image to disk
+    --write-all-thumbnails           write all thumbnail image formats to disk
+    --list-thumbnails                Simulate and list all available thumbnail
+                                     formats
+
 ## Verbosity / Simulation Options:
     -q, --quiet                      activates quiet mode
     --no-warnings                    Ignore warnings
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 35f3e1b6b..0d59cddd7 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.01.23.4'
+__version__ = '2015.01.25'

From e72c7e41238681338c92427f6731c74947645308 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 26 Jan 2015 12:01:43 +0100
Subject: [PATCH 0096/6219] [YoutubeDL] Always set the '_filename' field in the
 info_dict (reported in #4053)

It's also useful when you use the '--write-info-json' option.
---
 youtube_dl/YoutubeDL.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index b772f87f1..ae62432c6 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1130,7 +1130,7 @@ class YoutubeDL(object):
 
         self._num_downloads += 1
 
-        filename = self.prepare_filename(info_dict)
+        info_dict['_filename'] = filename = self.prepare_filename(info_dict)
 
         # Forced printings
         if self.params.get('forcetitle', False):
@@ -1155,10 +1155,7 @@ class YoutubeDL(object):
         if self.params.get('forceformat', False):
             self.to_stdout(info_dict['format'])
         if self.params.get('forcejson', False):
-            info_dict['_filename'] = filename
             self.to_stdout(json.dumps(info_dict))
-        if self.params.get('dump_single_json', False):
-            info_dict['_filename'] = filename
 
         # Do nothing else if in simulate mode
         if self.params.get('simulate', False):

From 9f0df77ab13ca19bc074b501296bb6a494193cd1 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 26 Jan 2015 14:36:22 +0100
Subject: [PATCH 0097/6219] [YoutubeDL] Allow format filtering by fps

---
 youtube_dl/YoutubeDL.py | 2 +-
 youtube_dl/options.py   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index ae62432c6..b5dd77e3f 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -820,7 +820,7 @@ class YoutubeDL(object):
             '!=': operator.ne,
         }
         operator_rex = re.compile(r'''(?x)\s*\[
-            (?P<key>width|height|tbr|abr|vbr|filesize)
+            (?P<key>width|height|tbr|abr|vbr|filesize|fps)
             \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
             (?P<value>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)
             \]$
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index dbc6f5528..5e4a1ad9b 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -297,7 +297,7 @@ def parseOpts(overrideArguments=None):
             ' You can filter the video results by putting a condition in'
             ' brackets, as in -f "best[height=720]"'
             ' (or -f "[filesize>10M]"). '
-            ' This works for filesize, height, width, tbr, abr, and vbr'
+            ' This works for filesize, height, width, tbr, abr, vbr, and fps'
             ' and the comparisons <, <=, >, >=, =, != .'
             ' Formats for which the value is not known are excluded unless you'
             ' put a question mark (?) after the operator.'

From 6ca85be6f8697dc8cb0378854c2c6cdc154593d4 Mon Sep 17 00:00:00 2001
From: rzhxeo <rzhxeot7z81b4700@mailcatch.com>
Date: Wed, 28 May 2014 18:19:23 +0200
Subject: [PATCH 0098/6219] Filter DRM protected media in f4m downloader

---
 youtube_dl/downloader/f4m.py | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index c68b2c303..29de7630d 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -230,6 +230,23 @@ class F4mFD(FileDownloader):
     A downloader for f4m manifests or AdobeHDS.
     """
 
+    def _get_unencrypted_media(self, doc):
+        media=doc.findall(_add_ns('media'))
+        if not media:
+            self.report_error('No media found')
+        for e in (doc.findall(_add_ns('drmAdditionalHeader')) +
+                  doc.findall(_add_ns('drmAdditionalHeaderSet'))):
+            # If id attribute is missing it's valid for all media nodes
+            # without drmAdditionalHeaderId or drmAdditionalHeaderSetId attribute
+            if not 'id' in e.attrib:
+                self.report_error('Media is DRM protected')
+        media = list(filter(lambda e: 'drmAdditionalHeaderId' not in e.attrib and
+                                      'drmAdditionalHeaderSetId' not in e.attrib,
+                            media))
+        if not media:
+            self.report_error('Media is DRM protected')
+        return media
+
     def real_download(self, filename, info_dict):
         man_url = info_dict['url']
         requested_bitrate = info_dict.get('tbr')
@@ -248,7 +265,8 @@ class F4mFD(FileDownloader):
         )
 
         doc = etree.fromstring(manifest)
-        formats = [(int(f.attrib.get('bitrate', -1)), f) for f in doc.findall(_add_ns('media'))]
+        formats = [(int(f.attrib.get('bitrate', -1)), f)
+                   for f in self._get_unencrypted_media(doc)]
         if requested_bitrate is None:
             # get the best format
             formats = sorted(formats, key=lambda f: f[0])

From 2103d038b3dac6f489d13282dcd2a6f4b74975c3 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Tue, 27 Jan 2015 15:38:54 +0200
Subject: [PATCH 0099/6219] [lnkgo] Adapt to website changes

---
 youtube_dl/extractor/lnkgo.py | 61 ++++++++++++++---------------------
 1 file changed, 25 insertions(+), 36 deletions(-)

diff --git a/youtube_dl/extractor/lnkgo.py b/youtube_dl/extractor/lnkgo.py
index a8e357859..fd23b0b43 100644
--- a/youtube_dl/extractor/lnkgo.py
+++ b/youtube_dl/extractor/lnkgo.py
@@ -6,13 +6,12 @@ import re
 from .common import InfoExtractor
 from ..utils import (
     int_or_none,
-    js_to_json,
     unified_strdate,
 )
 
 
 class LnkGoIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?lnkgo\.alfa\.lt/visi\-video/(?P<show>[^/]+)/ziurek\-(?P<display_id>[A-Za-z0-9\-]+)'
+    _VALID_URL = r'https?://(?:www\.)?lnkgo\.alfa\.lt/visi-video/(?P<show>[^/]+)/ziurek-(?P<id>[A-Za-z0-9-]+)'
     _TESTS = [{
         'url': 'http://lnkgo.alfa.lt/visi-video/yra-kaip-yra/ziurek-yra-kaip-yra-162',
         'info_dict': {
@@ -51,8 +50,7 @@ class LnkGoIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        display_id = mobj.group('display_id')
+        display_id = self._match_id(url)
 
         webpage = self._download_webpage(
             url, display_id, 'Downloading player webpage')
@@ -61,6 +59,8 @@ class LnkGoIE(InfoExtractor):
             r'data-ep="([^"]+)"', webpage, 'video ID')
         title = self._og_search_title(webpage)
         description = self._og_search_description(webpage)
+        upload_date = unified_strdate(self._search_regex(
+            r'class="[^"]*meta-item[^"]*air-time[^"]*">.*?<strong>([^<]+)</strong>', webpage, 'upload date', fatal=False))
 
         thumbnail_w = int_or_none(
             self._og_search_property('image:width', webpage, 'thumbnail width', fatal=False))
@@ -75,39 +75,28 @@ class LnkGoIE(InfoExtractor):
                 'height': thumbnail_h,
             })
 
-        upload_date = unified_strdate(self._search_regex(
-            r'class="meta-item\sair-time">.*?<strong>([^<]+)</strong>', webpage, 'upload date', fatal=False))
-        duration = int_or_none(self._search_regex(
-            r'VideoDuration = "([^"]+)"', webpage, 'duration', fatal=False))
+        config = self._parse_json(self._search_regex(
+            r'episodePlayer\((\{.*?\}),\s*\{', webpage, 'sources'), video_id)
 
-        pg_rating = self._search_regex(
-            r'pgrating="([^"]+)"', webpage, 'PG rating', fatal=False, default='')
-        age_limit = self._AGE_LIMITS.get(pg_rating.upper(), 0)
+        if config.get('pGeo'):
+            self.report_warning(
+                'This content might not be available in your country due to copyright reasons')
 
-        sources_js = self._search_regex(
-            r'(?s)sources:\s(\[.*?\]),', webpage, 'sources')
-        sources = self._parse_json(
-            sources_js, video_id, transform_source=js_to_json)
+        formats = [{
+            'format_id': 'hls',
+            'ext': 'mp4',
+            'url': config['EpisodeVideoLink_HLS'],
+        }]
 
-        formats = []
-        for source in sources:
-            if source.get('provider') == 'rtmp':
-                m = re.search(r'^(?P<url>rtmp://[^/]+/(?P<app>[^/]+))/(?P<play_path>.+)$', source['file'])
-                if not m:
-                    continue
-                formats.append({
-                    'format_id': 'rtmp',
-                    'ext': 'flv',
-                    'url': m.group('url'),
-                    'play_path': m.group('play_path'),
-                    'page_url': url,
-                })
-            elif source.get('file').endswith('.m3u8'):
-                formats.append({
-                    'format_id': 'hls',
-                    'ext': source.get('type', 'mp4'),
-                    'url': source['file'],
-                })
+        m = re.search(r'^(?P<url>rtmp://[^/]+/(?P<app>[^/]+))/(?P<play_path>.+)$', config['EpisodeVideoLink'])
+        if m:
+            formats.append({
+                'format_id': 'rtmp',
+                'ext': 'flv',
+                'url': m.group('url'),
+                'play_path': m.group('play_path'),
+                'page_url': url,
+            })
 
         self._sort_formats(formats)
 
@@ -117,8 +106,8 @@ class LnkGoIE(InfoExtractor):
             'title': title,
             'formats': formats,
             'thumbnails': [thumbnail],
-            'duration': duration,
+            'duration': int_or_none(config.get('VideoTime')),
             'description': description,
-            'age_limit': age_limit,
+            'age_limit': self._AGE_LIMITS.get(config.get('PGRating'), 0),
             'upload_date': upload_date,
         }

From fe7710cbccdade0b66ef48a7e2eedad71a0702cc Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 27 Jan 2015 23:55:22 +0800
Subject: [PATCH 0100/6219] [xuite] Add new extractor

---
 youtube_dl/extractor/__init__.py |   1 +
 youtube_dl/extractor/xuite.py    | 147 +++++++++++++++++++++++++++++++
 2 files changed, 148 insertions(+)
 create mode 100644 youtube_dl/extractor/xuite.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 03c56156a..09070daa4 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -546,6 +546,7 @@ from .xminus import XMinusIE
 from .xnxx import XNXXIE
 from .xvideos import XVideosIE
 from .xtube import XTubeUserIE, XTubeIE
+from .xuite import XuiteIE
 from .xxxymovies import XXXYMoviesIE
 from .yahoo import (
     YahooIE,
diff --git a/youtube_dl/extractor/xuite.py b/youtube_dl/extractor/xuite.py
new file mode 100644
index 000000000..943757c27
--- /dev/null
+++ b/youtube_dl/extractor/xuite.py
@@ -0,0 +1,147 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+import base64
+from .common import InfoExtractor
+from ..compat import (
+    compat_urlparse,
+    compat_urllib_parse_unquote,
+    compat_parse_qs
+)
+from ..utils import (
+    ExtractorError,
+    parse_iso8601,
+    parse_duration
+)
+
+# ref: http://stackoverflow.com/questions/475074/regex-to-parse-or-validate-base64-data
+REGEX_BASE64 = r'(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?'
+
+CST_ZONE = +8  # China Standard Time
+
+
+class XuiteIE(InfoExtractor):
+    _VALID_URL = r'http://vlog.xuite.net/play/(?P<id>%s)(/.*)?' % REGEX_BASE64
+    _TESTS = [{
+        # Audio
+        'url': 'http://vlog.xuite.net/play/RGkzc1ZULTM4NjA5MTQuZmx2',
+        'md5': '63a42c705772aa53fd4c1a0027f86adf',
+        'info_dict': {
+            'id': 'RGkzc1ZULTM4NjA5MTQuZmx2',
+            'ext': 'mp3',
+            'upload_date': '20110902',
+            'uploader_id': '15973816',
+            'uploader': '阿能',
+            'timestamp': 1314932940,
+            'title': '孤單南半球-歐德陽'
+        }
+    }, {
+        # Audio with alternative form of url
+        'url': 'http://vlog.xuite.net/play/S1dDUjdyLTMyOTc3NjcuZmx2/%E5%AD%AB%E7%87%95%E5%A7%BF-%E7%9C%BC%E6%B7%9A%E6%88%90%E8%A9%A9',
+        'md5': 'c91645f324de53d82ebb80930e1a73d2',
+        'info_dict': {
+            'id': 'S1dDUjdyLTMyOTc3NjcuZmx2',
+            'ext': 'mp3',
+            'upload_date': '20101226',
+            'uploader_id': '10102699',
+            'uploader': '蠍',
+            'timestamp': 1293367080,
+            'title': '孫燕姿-眼淚成詩',
+        }
+    }, {
+        # Video with only one format
+        'url': 'http://vlog.xuite.net/play/TkRZNjhULTM0NDE2MjkuZmx2',
+        'md5': 'c45737fc8ac5dc8ac2f92ecbcecf505e',
+        'info_dict': {
+            'id': 'TkRZNjhULTM0NDE2MjkuZmx2',
+            'ext': 'mp4',
+            'upload_date': '20110306',
+            'uploader_id': '10400126',
+            'uploader': 'Valen',
+            'timestamp': 1299383640,
+            'title': '孫燕姿 - 眼淚成詩',
+        }
+    }, {
+        # Video with two formats
+        'url': 'http://vlog.xuite.net/play/bWo1N1pLLTIxMzAxMTcwLmZsdg==',
+        'md5': '1166e0f461efe55b62e26a2d2a68e6de',
+        'info_dict': {
+            'id': 'bWo1N1pLLTIxMzAxMTcwLmZsdg==',
+            'ext': 'mp4',
+            'upload_date': '20150117',
+            'uploader_id': '242127761',
+            'uploader': '我只是想認真點',
+            'timestamp': 1421481240,
+            'title': '暗殺教室 02',
+        }
+    }]
+
+    def _flv_config(self, media_id):
+        base64_media_id = base64.b64encode(media_id.encode('utf-8')).decode('utf-8')
+        flv_config_url = 'http://vlog.xuite.net/flash/player?media=' + base64_media_id
+        flv_config = self._download_xml(flv_config_url, 'flv config')
+
+        prop_dict = {}
+        for prop in flv_config.findall('./property'):
+            prop_id = base64.b64decode(prop.attrib['id']).decode('utf-8')
+
+            if not prop.text:
+                continue  # CDATA may be empty in flv config
+
+            encoded_content = base64.b64decode(prop.text).decode('utf-8')
+            prop_dict[prop_id] = compat_urllib_parse_unquote(encoded_content)
+
+        return prop_dict
+
+    def _type_string(self, media_url):
+        query_string = compat_urlparse.urlparse(media_url).query
+        type_string = compat_parse_qs(query_string)['q'][0]
+        return type_string
+
+    def _guess_ext(self, media_url):
+        type_string = self._type_string(media_url)
+        if type_string == 'mp3':
+            return 'mp3'
+        elif type_string == '360' or type_string == '720':
+            return 'mp4'
+        else:
+            raise ExtractorError('Unknown type string %s' % type_string)
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        page = self._download_webpage(url, video_id)
+        media_id = self._html_search_regex(r'data-mediaid="(\d+)"', page, 'media id')
+        flv_config = self._flv_config(media_id)
+
+        timestamp_local = parse_iso8601(flv_config['publish_datetime'], ' ')
+        timestamp_gmt = timestamp_local - CST_ZONE * 3600
+
+        ret_attrs = {
+            'id': video_id,
+            'title': flv_config['title'],
+            'thumbnail': flv_config['thumb'],
+            'uploader': flv_config['author_name'],
+            'timestamp': timestamp_gmt,
+            'uploader_id': flv_config['author_id'],
+            'duration': parse_duration(flv_config['duration']),
+            'categories': [flv_config['category']]
+        }
+
+        if 'hq_src' in flv_config:
+            src = flv_config['src']
+            src_hq = flv_config['hq_src']
+            ret_attrs['formats'] = [{
+                'url': src,
+                'ext': self._guess_ext(src),
+                'format_id': self._type_string(src)
+            }, {
+                'url': src_hq,
+                'ext': self._guess_ext(src_hq),
+                'format_id': self._type_string(src_hq)
+            }]
+        else:
+            ret_attrs['url'] = flv_config['src']
+            ret_attrs['ext'] = self._guess_ext(flv_config['src'])
+
+        return ret_attrs

From 6348ad12a057cc1c454488ce7c37070cf39a8f06 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 28 Jan 2015 00:13:40 +0800
Subject: [PATCH 0101/6219] [xuite] Add height information for the two formats

---
 youtube_dl/extractor/xuite.py | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/xuite.py b/youtube_dl/extractor/xuite.py
index 943757c27..aef9a14a3 100644
--- a/youtube_dl/extractor/xuite.py
+++ b/youtube_dl/extractor/xuite.py
@@ -129,17 +129,17 @@ class XuiteIE(InfoExtractor):
         }
 
         if 'hq_src' in flv_config:
-            src = flv_config['src']
-            src_hq = flv_config['hq_src']
-            ret_attrs['formats'] = [{
-                'url': src,
-                'ext': self._guess_ext(src),
-                'format_id': self._type_string(src)
-            }, {
-                'url': src_hq,
-                'ext': self._guess_ext(src_hq),
-                'format_id': self._type_string(src_hq)
-            }]
+            urls = [flv_config['src'], flv_config['hq_src']]
+
+            ret_attrs['formats'] = []
+
+            for url in urls:
+                ret_attrs['formats'].append({
+                    'url': url,
+                    'ext': self._guess_ext(url),
+                    'format_id': self._type_string(url),
+                    'height': int(self._type_string(url))
+                })
         else:
             ret_attrs['url'] = flv_config['src']
             ret_attrs['ext'] = self._guess_ext(flv_config['src'])

From 3a0d2f520a0f95c2f87b1c95049135b10206f97f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 27 Jan 2015 22:38:28 +0600
Subject: [PATCH 0102/6219] [YoutubeDL] Temporary fix for subprocess encoding
 issues on python2 @ Windows (Closes #4787)

For now filenames will be encoded with preferrefencoding before written to disk
---
 youtube_dl/YoutubeDL.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index b5dd77e3f..7f054cdff 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -543,6 +543,11 @@ class YoutubeDL(object):
             outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
             tmpl = compat_expanduser(outtmpl)
             filename = tmpl % template_dict
+            # Temporary fix for #4787
+            # 'Treat' all problem characters by passing filename through preferredencoding
+            # to workaround encoding issues with subprocess on python2 @ Windows
+            if sys.version_info < (3, 0) and sys.platform == 'win32':
+                filename = encodeFilename(filename, True).decode(preferredencoding())
             return filename
         except ValueError as err:
             self.report_error('Error in output template: ' + str(err) + ' (encoding: ' + repr(preferredencoding()) + ')')

From a28383834b07787b22a837bd6028bf24bea7ec94 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 28 Jan 2015 01:30:14 +0800
Subject: [PATCH 0103/6219] [xuite] Update tests

---
 youtube_dl/extractor/xuite.py | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/xuite.py b/youtube_dl/extractor/xuite.py
index aef9a14a3..8466d4bc5 100644
--- a/youtube_dl/extractor/xuite.py
+++ b/youtube_dl/extractor/xuite.py
@@ -33,7 +33,10 @@ class XuiteIE(InfoExtractor):
             'uploader_id': '15973816',
             'uploader': '阿能',
             'timestamp': 1314932940,
-            'title': '孤單南半球-歐德陽'
+            'title': '孤單南半球-歐德陽',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'categories': ['個人短片'],
+            'duration': 247.246
         }
     }, {
         # Audio with alternative form of url
@@ -47,6 +50,9 @@ class XuiteIE(InfoExtractor):
             'uploader': '蠍',
             'timestamp': 1293367080,
             'title': '孫燕姿-眼淚成詩',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'categories': ['個人短片'],
+            'duration': 223.19
         }
     }, {
         # Video with only one format
@@ -60,6 +66,9 @@ class XuiteIE(InfoExtractor):
             'uploader': 'Valen',
             'timestamp': 1299383640,
             'title': '孫燕姿 - 眼淚成詩',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'categories': ['影視娛樂'],
+            'duration': 217.399
         }
     }, {
         # Video with two formats
@@ -73,6 +82,9 @@ class XuiteIE(InfoExtractor):
             'uploader': '我只是想認真點',
             'timestamp': 1421481240,
             'title': '暗殺教室 02',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'categories': ['電玩動漫'],
+            'duration': 1384.907
         }
     }]
 

From 796df3c631dcf28f7ebdf256e57fcfa3b6463abe Mon Sep 17 00:00:00 2001
From: Shaya G <hyshai29@gmail.com>
Date: Wed, 28 Jan 2015 00:08:19 -0500
Subject: [PATCH 0104/6219] fixed viddler support - needed a Referer header;
 also added a viddler generic extractor

---
 youtube_dl/extractor/generic.py | 20 ++++++++++++++++++
 youtube_dl/extractor/viddler.py | 36 +++++++++++++++++++++++++++++----
 2 files changed, 52 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index ad16b8330..a937a1e09 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -498,6 +498,19 @@ class GenericIE(InfoExtractor):
                 'uploader': 'www.abc.net.au',
                 'title': 'Game of Thrones with dice - Dungeons and Dragons fantasy role-playing game gets new life - 19/01/2015',
             }
+        },
+        # embedded viddler video
+        {
+            'url': 'http://deadspin.com/i-cant-stop-watching-john-wall-chop-the-nuggets-with-th-1681801597',
+            'info_dict': {
+                'id': '4d03aad9',
+                'ext': 'mp4',
+                'uploader': 'deadspin',
+                'title': 'WALL-TO-GORTAT',
+                'timestamp': 1422285291,
+                'upload_date': '20150126',
+            },
+            'add_ie': ['Viddler'],
         }
     ]
 
@@ -860,6 +873,13 @@ class GenericIE(InfoExtractor):
         if mobj is not None:
             return self.url_result(mobj.group('url'))
 
+        # Look for embedded Viddler player
+        mobj = (re.search(r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?viddler\.com/embed/.+?)\1', webpage) or
+                re.search(r'<param[^>]+?value=(["\'])(?P<url>(?:https?:)?//(?:www\.)?viddler\.com/player/.+?)\1', webpage))
+
+        if mobj is not None:
+            return self.url_result(mobj.group('url'))
+
         # Look for Ooyala videos
         mobj = (re.search(r'player.ooyala.com/[^"?]+\?[^"]*?(?:embedCode|ec)=(?P<ec>[^"&]+)', webpage) or
                 re.search(r'OO.Player.create\([\'"].*?[\'"],\s*[\'"](?P<ec>.{32})[\'"]', webpage))
diff --git a/youtube_dl/extractor/viddler.py b/youtube_dl/extractor/viddler.py
index 0faa729c6..bacbad50a 100644
--- a/youtube_dl/extractor/viddler.py
+++ b/youtube_dl/extractor/viddler.py
@@ -5,11 +5,14 @@ from ..utils import (
     float_or_none,
     int_or_none,
 )
+from ..compat import (
+   compat_urllib_request
+)
 
 
 class ViddlerIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?viddler\.com/(?:v|embed|player)/(?P<id>[a-z0-9]+)'
-    _TEST = {
+    _TESTS = [{
         "url": "http://www.viddler.com/v/43903784",
         'md5': 'ae43ad7cb59431ce043f0ff7fa13cbf4',
         'info_dict': {
@@ -25,7 +28,30 @@ class ViddlerIE(InfoExtractor):
             'view_count': int,
             'categories': ['video content', 'high quality video', 'video made easy', 'how to produce video with limited resources', 'viddler'],
         }
-    }
+    }, {
+        "url": "http://www.viddler.com/v/4d03aad9/",
+        "file": "4d03aad9.mp4",
+        "md5": "faa71fbf70c0bee7ab93076fd007f4b0",
+        "info_dict": {
+            'upload_date': '20150126',
+            'uploader': 'deadspin',
+            'id': '4d03aad9',
+            'timestamp': 1422285291,
+            'title': 'WALL-TO-GORTAT',
+        }
+    }, {
+        "url": "http://www.viddler.com/player/221ebbbd/0/",
+        "file": "221ebbbd.mp4",
+        "md5": "0defa2bd0ea613d14a6e9bd1db6be326",
+        "info_dict": {
+            'upload_date': '20140929',
+            'uploader': 'BCLETeens',
+            'id': '221ebbbd',
+            'timestamp': 1411997190,
+            'title': 'LETeens-Grammar-snack-third-conditional',
+            'description': ' '
+        }
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -33,7 +59,9 @@ class ViddlerIE(InfoExtractor):
         json_url = (
             'http://api.viddler.com/api/v2/viddler.videos.getPlaybackDetails.json?video_id=%s&key=v0vhrt7bg2xq1vyxhkct' %
             video_id)
-        data = self._download_json(json_url, video_id)['video']
+        headers = {'Referer': 'http://static.cdn-ec.viddler.com/js/arpeggio/v2/embed.html'}
+        request = compat_urllib_request.Request(json_url, None, headers)
+        data = self._download_json(request, video_id)['video']
 
         formats = []
         for filed in data['files']:
@@ -53,7 +81,7 @@ class ViddlerIE(InfoExtractor):
 
             if filed.get('cdn_url'):
                 f = f.copy()
-                f['url'] = self._proto_relative_url(filed['cdn_url'])
+                f['url'] = self._proto_relative_url(filed['cdn_url'], 'http:')
                 f['format_id'] = filed['profile_id'] + '-cdn'
                 f['source_preference'] = 1
                 formats.append(f)

From 0865f397ae6c875ec4194093af5c8f53e75f2285 Mon Sep 17 00:00:00 2001
From: Paul Hartmann <phaaurlt@gmail.com>
Date: Wed, 28 Jan 2015 08:21:04 +0100
Subject: [PATCH 0105/6219] added extractor for dctp.tv

---
 youtube_dl/downloader/rtmp.py    |  3 +++
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/dctp.py     | 41 ++++++++++++++++++++++++++++++++
 3 files changed, 45 insertions(+)
 create mode 100644 youtube_dl/extractor/dctp.py

diff --git a/youtube_dl/downloader/rtmp.py b/youtube_dl/downloader/rtmp.py
index e06ebe826..6cb1bfc50 100644
--- a/youtube_dl/downloader/rtmp.py
+++ b/youtube_dl/downloader/rtmp.py
@@ -106,6 +106,7 @@ class RtmpFD(FileDownloader):
         protocol = info_dict.get('rtmp_protocol', None)
         no_resume = info_dict.get('no_resume', False)
         continue_dl = info_dict.get('continuedl', False)
+        real_time = info_dict.get('real_time', False)
 
         self.report_destination(filename)
         tmpfilename = self.temp_name(filename)
@@ -143,6 +144,8 @@ class RtmpFD(FileDownloader):
             basic_args += ['--conn', conn]
         if protocol is not None:
             basic_args += ['--protocol', protocol]
+        if real_time:
+            basic_args += ['--realtime']
 
         args = basic_args
         if not no_resume and continue_dl and not live:
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 03c56156a..873ae69d3 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -89,6 +89,7 @@ from .dailymotion import (
 )
 from .daum import DaumIE
 from .dbtv import DBTVIE
+from .dctp import DctpTvIE
 from .deezer import DeezerPlaylistIE
 from .dfb import DFBIE
 from .dotsub import DotsubIE
diff --git a/youtube_dl/extractor/dctp.py b/youtube_dl/extractor/dctp.py
new file mode 100644
index 000000000..5382255f0
--- /dev/null
+++ b/youtube_dl/extractor/dctp.py
@@ -0,0 +1,41 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+class DctpTvIE(InfoExtractor):
+    _VALID_URL = r'^http://www.dctp.tv/(#/)?filme/(?P<id>.+?)/$'
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        base_url = 'http://dctp-ivms2-restapi.s3.amazonaws.com/'
+        version_json = self._download_json(base_url + 'version.json', video_id)
+        version = version_json['version_name']
+        info_json = self._download_json(
+            '{}{}/restapi/slugs/{}.json'.format(base_url, version, video_id), video_id)
+        object_id = info_json['object_id']
+        meta_json = self._download_json(
+            '{}{}/restapi/media/{}.json'.format(base_url, version, object_id), video_id)
+        uuid = meta_json['uuid']
+        title = meta_json['title']
+        wide = meta_json['is_wide']
+        if wide:
+            ratio = '16x9'
+        else:
+            ratio = '4x3'
+        play_path = 'mp4:{}_dctp_0500_{}.m4v'.format(uuid, ratio)
+
+        servers_json = self._download_json('http://www.dctp.tv/streaming_servers/', video_id)
+        url = servers_json[0]['endpoint']
+
+        return {
+            'id': video_id,
+            'title': title,
+            'format': 'rtmp',
+            'url': url,
+            'play_path': play_path,
+            'real_time': True,
+            'ext': 'flv'
+        }
+
+

From 48a1e5141ad9f6b5c4ce8a355dcd7bf99c80e333 Mon Sep 17 00:00:00 2001
From: Paul Hartmann <phaaurlt@gmail.com>
Date: Wed, 28 Jan 2015 08:59:58 +0100
Subject: [PATCH 0106/6219] added test for dctp

---
 youtube_dl/extractor/dctp.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/dctp.py b/youtube_dl/extractor/dctp.py
index 5382255f0..9b687ef43 100644
--- a/youtube_dl/extractor/dctp.py
+++ b/youtube_dl/extractor/dctp.py
@@ -3,8 +3,16 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 
+
 class DctpTvIE(InfoExtractor):
     _VALID_URL = r'^http://www.dctp.tv/(#/)?filme/(?P<id>.+?)/$'
+    _TEST = {
+        'url': 'http://www.dctp.tv/filme/videoinstallation-fuer-eine-kaufhausfassade/',
+        'info_dict': {
+            'id': 'videoinstallation-fuer-eine-kaufhausfassade',
+            'ext': 'flv',
+            'title': 'Videoinstallation für eine Kaufhausfassade'}
+        }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -38,4 +46,3 @@ class DctpTvIE(InfoExtractor):
             'ext': 'flv'
         }
 
-

From b04fbd789c1efd5f918f81b5a5b9b6dafa806900 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 28 Jan 2015 22:49:42 +0600
Subject: [PATCH 0107/6219] [viddler] Modernize

---
 youtube_dl/extractor/viddler.py | 46 ++++++++++++++++-----------------
 1 file changed, 22 insertions(+), 24 deletions(-)

diff --git a/youtube_dl/extractor/viddler.py b/youtube_dl/extractor/viddler.py
index bacbad50a..9caee94e7 100644
--- a/youtube_dl/extractor/viddler.py
+++ b/youtube_dl/extractor/viddler.py
@@ -13,43 +13,43 @@ from ..compat import (
 class ViddlerIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?viddler\.com/(?:v|embed|player)/(?P<id>[a-z0-9]+)'
     _TESTS = [{
-        "url": "http://www.viddler.com/v/43903784",
+        'url': 'http://www.viddler.com/v/43903784',
         'md5': 'ae43ad7cb59431ce043f0ff7fa13cbf4',
         'info_dict': {
             'id': '43903784',
             'ext': 'mp4',
-            "title": "Video Made Easy",
-            'description': 'You don\'t need to be a professional to make high-quality video content. Viddler provides some quick and easy tips on how to produce great video content with limited resources. ',
-            "uploader": "viddler",
+            'title': 'Video Made Easy',
+            'description': 'md5:6a697ebd844ff3093bd2e82c37b409cd',
+            'uploader': 'viddler',
             'timestamp': 1335371429,
             'upload_date': '20120425',
-            "duration": 100.89,
+            'duration': 100.89,
             'thumbnail': 're:^https?://.*\.jpg$',
             'view_count': int,
             'categories': ['video content', 'high quality video', 'video made easy', 'how to produce video with limited resources', 'viddler'],
         }
     }, {
-        "url": "http://www.viddler.com/v/4d03aad9/",
-        "file": "4d03aad9.mp4",
-        "md5": "faa71fbf70c0bee7ab93076fd007f4b0",
-        "info_dict": {
+        'url': 'http://www.viddler.com/v/4d03aad9/',
+        'md5': 'faa71fbf70c0bee7ab93076fd007f4b0',
+        'info_dict': {
+            'id': '4d03aad9',
+            'ext': 'mp4',
+            'title': 'WALL-TO-GORTAT',
             'upload_date': '20150126',
             'uploader': 'deadspin',
-            'id': '4d03aad9',
             'timestamp': 1422285291,
-            'title': 'WALL-TO-GORTAT',
         }
     }, {
-        "url": "http://www.viddler.com/player/221ebbbd/0/",
-        "file": "221ebbbd.mp4",
-        "md5": "0defa2bd0ea613d14a6e9bd1db6be326",
-        "info_dict": {
+        'url': 'http://www.viddler.com/player/221ebbbd/0/',
+        'md5': '0defa2bd0ea613d14a6e9bd1db6be326',
+        'info_dict': {
+            'id': '221ebbbd',
+            'ext': 'mp4',
+            'title': 'LETeens-Grammar-snack-third-conditional',
+            'description': ' ',
             'upload_date': '20140929',
             'uploader': 'BCLETeens',
-            'id': '221ebbbd',
             'timestamp': 1411997190,
-            'title': 'LETeens-Grammar-snack-third-conditional',
-            'description': ' '
         }
     }]
 
@@ -68,7 +68,7 @@ class ViddlerIE(InfoExtractor):
             if filed.get('status', 'ready') != 'ready':
                 continue
             f = {
-                'format_id': filed['profile_id'],
+                'format_id': filed['profile_id'] or filed['profile_name'],
                 'format_note': filed['profile_name'],
                 'url': self._proto_relative_url(filed['url']),
                 'width': int_or_none(filed.get('width')),
@@ -82,15 +82,14 @@ class ViddlerIE(InfoExtractor):
             if filed.get('cdn_url'):
                 f = f.copy()
                 f['url'] = self._proto_relative_url(filed['cdn_url'], 'http:')
-                f['format_id'] = filed['profile_id'] + '-cdn'
+                f['format_id'] = (filed['profile_id'] or filed['profile_name']) + '-cdn'
                 f['source_preference'] = 1
                 formats.append(f)
 
             if filed.get('html5_video_source'):
                 f = f.copy()
-                f['url'] = self._proto_relative_url(
-                    filed['html5_video_source'])
-                f['format_id'] = filed['profile_id'] + '-html5'
+                f['url'] = self._proto_relative_url(filed['html5_video_source'])
+                f['format_id'] = (filed['profile_id'] or filed['profile_name']) + '-html5'
                 f['source_preference'] = 0
                 formats.append(f)
         self._sort_formats(formats)
@@ -99,7 +98,6 @@ class ViddlerIE(InfoExtractor):
             t.get('text') for t in data.get('tags', []) if 'text' in t]
 
         return {
-            '_type': 'video',
             'id': video_id,
             'title': data['title'],
             'formats': formats,

From e0d9f85aee92ebcb40ce688ed43172c718751e27 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 28 Jan 2015 22:56:06 +0600
Subject: [PATCH 0108/6219] Credit @HyShai for viddler fixes (#4794)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 1596a7548..20e620c35 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -106,3 +106,4 @@ Johan K. Jensen
 Yen Chi Hsuan
 Enam Mijbah Noor
 David Luhmer
+Shaya Goldberg

From cb454b333d91718a0c2b36c34c8b0d6858ff9505 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 28 Jan 2015 23:07:37 +0600
Subject: [PATCH 0109/6219] [generic] Improve some regexes

---
 youtube_dl/extractor/generic.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index a937a1e09..41884ed7a 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -874,15 +874,15 @@ class GenericIE(InfoExtractor):
             return self.url_result(mobj.group('url'))
 
         # Look for embedded Viddler player
-        mobj = (re.search(r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?viddler\.com/embed/.+?)\1', webpage) or
-                re.search(r'<param[^>]+?value=(["\'])(?P<url>(?:https?:)?//(?:www\.)?viddler\.com/player/.+?)\1', webpage))
-
+        mobj = re.search(
+            r'<(?:iframe[^>]+?src|param[^>]+?value)=(["\'])(?P<url>(?:https?:)?//(?:www\.)?viddler\.com/(?:embed|player)/.+?)\1',
+            webpage)
         if mobj is not None:
             return self.url_result(mobj.group('url'))
 
         # Look for Ooyala videos
-        mobj = (re.search(r'player.ooyala.com/[^"?]+\?[^"]*?(?:embedCode|ec)=(?P<ec>[^"&]+)', webpage) or
-                re.search(r'OO.Player.create\([\'"].*?[\'"],\s*[\'"](?P<ec>.{32})[\'"]', webpage))
+        mobj = (re.search(r'player\.ooyala\.com/[^"?]+\?[^"]*?(?:embedCode|ec)=(?P<ec>[^"&]+)', webpage) or
+                re.search(r'OO\.Player\.create\([\'"].*?[\'"],\s*[\'"](?P<ec>.{32})[\'"]', webpage))
         if mobj is not None:
             return OoyalaIE._build_url_result(mobj.group('ec'))
 

From 18b4e9e79d6cc1ed21df7479273915ca3a763b80 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 28 Jan 2015 23:21:17 +0600
Subject: [PATCH 0110/6219] [viddler] Extract comment count

---
 youtube_dl/extractor/viddler.py | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/viddler.py b/youtube_dl/extractor/viddler.py
index 9caee94e7..ef104dc29 100644
--- a/youtube_dl/extractor/viddler.py
+++ b/youtube_dl/extractor/viddler.py
@@ -26,6 +26,7 @@ class ViddlerIE(InfoExtractor):
             'duration': 100.89,
             'thumbnail': 're:^https?://.*\.jpg$',
             'view_count': int,
+            'comment_count': int,
             'categories': ['video content', 'high quality video', 'video made easy', 'how to produce video with limited resources', 'viddler'],
         }
     }, {
@@ -38,6 +39,8 @@ class ViddlerIE(InfoExtractor):
             'upload_date': '20150126',
             'uploader': 'deadspin',
             'timestamp': 1422285291,
+            'view_count': int,
+            'comment_count': int,
         }
     }, {
         'url': 'http://www.viddler.com/player/221ebbbd/0/',
@@ -50,6 +53,8 @@ class ViddlerIE(InfoExtractor):
             'upload_date': '20140929',
             'uploader': 'BCLETeens',
             'timestamp': 1411997190,
+            'view_count': int,
+            'comment_count': int,
         }
     }]
 
@@ -67,8 +72,9 @@ class ViddlerIE(InfoExtractor):
         for filed in data['files']:
             if filed.get('status', 'ready') != 'ready':
                 continue
+            format_id = filed.get('profile_id') or filed['profile_name']
             f = {
-                'format_id': filed['profile_id'] or filed['profile_name'],
+                'format_id': format_id,
                 'format_note': filed['profile_name'],
                 'url': self._proto_relative_url(filed['url']),
                 'width': int_or_none(filed.get('width')),
@@ -82,14 +88,14 @@ class ViddlerIE(InfoExtractor):
             if filed.get('cdn_url'):
                 f = f.copy()
                 f['url'] = self._proto_relative_url(filed['cdn_url'], 'http:')
-                f['format_id'] = (filed['profile_id'] or filed['profile_name']) + '-cdn'
+                f['format_id'] = format_id + '-cdn'
                 f['source_preference'] = 1
                 formats.append(f)
 
             if filed.get('html5_video_source'):
                 f = f.copy()
                 f['url'] = self._proto_relative_url(filed['html5_video_source'])
-                f['format_id'] = (filed['profile_id'] or filed['profile_name']) + '-html5'
+                f['format_id'] = format_id + '-html5'
                 f['source_preference'] = 0
                 formats.append(f)
         self._sort_formats(formats)
@@ -107,5 +113,6 @@ class ViddlerIE(InfoExtractor):
             'uploader': data.get('author'),
             'duration': float_or_none(data.get('length')),
             'view_count': int_or_none(data.get('view_count')),
+            'comment_count': int_or_none(data.get('comment_count')),
             'categories': categories,
         }

From 63be3b89894f26caf69d9667f8e7db5ecbcb03ab Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 28 Jan 2015 23:58:14 +0600
Subject: [PATCH 0111/6219] [ivi] Modernize

---
 youtube_dl/extractor/ivi.py | 27 ++++++++++++++++-----------
 1 file changed, 16 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/ivi.py b/youtube_dl/extractor/ivi.py
index 7a400323d..e82594444 100644
--- a/youtube_dl/extractor/ivi.py
+++ b/youtube_dl/extractor/ivi.py
@@ -16,7 +16,7 @@ from ..utils import (
 class IviIE(InfoExtractor):
     IE_DESC = 'ivi.ru'
     IE_NAME = 'ivi'
-    _VALID_URL = r'https?://(?:www\.)?ivi\.ru/(?:watch/(?:[^/]+/)?|video/player\?.*?videoId=)(?P<videoid>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?ivi\.ru/(?:watch/(?:[^/]+/)?|video/player\?.*?videoId=)(?P<id>\d+)'
 
     _TESTS = [
         # Single movie
@@ -63,29 +63,34 @@ class IviIE(InfoExtractor):
         return int(m.group('commentcount')) if m is not None else 0
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('videoid')
+        video_id = self._match_id(url)
 
         api_url = 'http://api.digitalaccess.ru/api/json/'
 
-        data = {'method': 'da.content.get',
-                'params': [video_id, {'site': 's183',
-                                      'referrer': 'http://www.ivi.ru/watch/%s' % video_id,
-                                      'contentid': video_id
-                                      }
-                           ]
+        data = {
+            'method': 'da.content.get',
+            'params': [
+                video_id, {
+                    'site': 's183',
+                    'referrer': 'http://www.ivi.ru/watch/%s' % video_id,
+                    'contentid': video_id
                 }
+            ]
+        }
 
         request = compat_urllib_request.Request(api_url, json.dumps(data))
 
-        video_json_page = self._download_webpage(request, video_id, 'Downloading video JSON')
+        video_json_page = self._download_webpage(
+            request, video_id, 'Downloading video JSON')
         video_json = json.loads(video_json_page)
 
         if 'error' in video_json:
             error = video_json['error']
             if error['origin'] == 'NoRedisValidData':
                 raise ExtractorError('Video %s does not exist' % video_id, expected=True)
-            raise ExtractorError('Unable to download video %s: %s' % (video_id, error['message']), expected=True)
+            raise ExtractorError(
+                'Unable to download video %s: %s' % (video_id, error['message']),
+                expected=True)
 
         result = video_json['result']
 

From dcf53d440801505a27ee5615e3fb58b6a794bc73 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 29 Jan 2015 00:44:40 +0600
Subject: [PATCH 0112/6219] [YoutubeDL] Set format_id for video+audio (Closes
 #3634)

---
 youtube_dl/YoutubeDL.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 7f054cdff..4c238c555 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1074,6 +1074,7 @@ class YoutubeDL(object):
                             selected_format = {
                                 'requested_formats': formats_info,
                                 'format': rf,
+                                'format_id': rf,
                                 'ext': formats_info[0]['ext'],
                                 'width': formats_info[0].get('width'),
                                 'height': formats_info[0].get('height'),

From 206dba27a4d134cf9a65b93f7a12bee033a80b18 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 29 Jan 2015 03:18:53 +0800
Subject: [PATCH 0113/6219] [NextMedia] Add new extractor

---
 youtube_dl/extractor/__init__.py  |   6 ++
 youtube_dl/extractor/nextmedia.py | 163 ++++++++++++++++++++++++++++++
 2 files changed, 169 insertions(+)
 create mode 100644 youtube_dl/extractor/nextmedia.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 03c56156a..444af8b6e 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -284,6 +284,12 @@ from .netzkino import NetzkinoIE
 from .nerdcubed import NerdCubedFeedIE
 from .newgrounds import NewgroundsIE
 from .newstube import NewstubeIE
+from .nextmedia import (
+    NextMediaIE,
+    NextMediaActionNewsIE,
+    AppleDailyRealtimeNewsIE,
+    AppleDailyAnimationNewsIE
+)
 from .nfb import NFBIE
 from .nfl import NFLIE
 from .nhl import NHLIE, NHLVideocenterIE
diff --git a/youtube_dl/extractor/nextmedia.py b/youtube_dl/extractor/nextmedia.py
new file mode 100644
index 000000000..02dba4ef6
--- /dev/null
+++ b/youtube_dl/extractor/nextmedia.py
@@ -0,0 +1,163 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import parse_iso8601
+
+
+class NextMediaIE(InfoExtractor):
+    _VALID_URL = r'http://hk.apple.nextmedia.com/[^/]+/[^/]+/(?P<date>\d+)/(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'http://hk.apple.nextmedia.com/realtime/news/20141108/53109199',
+        'md5': 'dff9fad7009311c421176d1ac90bfe4f',
+        'info_dict': {
+            'id': '53109199',
+            'ext': 'mp4',
+            'title': '【佔領金鐘】50外國領事議員撐場 讚學生勇敢香港有希望',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'description': 'md5:28222b9912b6665a21011b034c70fcc7',
+            'timestamp': 1415456273,
+            'upload_date': '20141108',
+        }
+    }]
+
+    _URL_PATTERN = r'\{ url: \'(.+)\' \}'
+
+    def _real_extract(self, url):
+        news_id = self._match_id(url)
+        page = self._download_webpage(url, news_id)
+        return self._extract_from_nextmedia_page(news_id, url, page)
+
+    def _extract_from_nextmedia_page(self, news_id, url, page):
+        title = self._fetch_title(page)
+        video_url = self._search_regex(self._URL_PATTERN, page, 'video url')
+
+        attrs = {
+            'id': news_id,
+            'title': title,
+            'url': video_url,  # ext can be inferred from url
+            'thumbnail': self._fetch_thumbnail(page),
+            'description': self._fetch_description(page),
+        }
+
+        timestamp = self._fetch_timestamp(page)
+        if timestamp:
+            attrs['timestamp'] = timestamp
+        else:
+            attrs['upload_date'] = self._fetch_upload_date(url)
+
+        return attrs
+
+    def _fetch_title(self, page):
+        return self._og_search_title(page)
+
+    def _fetch_thumbnail(self, page):
+        return self._og_search_thumbnail(page)
+
+    def _fetch_timestamp(self, page):
+        dateCreated = self._search_regex('"dateCreated":"([^"]+)"', page, 'created time')
+        return parse_iso8601(dateCreated)
+
+    def _fetch_upload_date(self, url):
+        return self._search_regex(self._VALID_URL, url, 'upload date', group='date')
+
+    def _fetch_description(self, page):
+        return self._og_search_property('description', page)
+
+
+class NextMediaActionNewsIE(NextMediaIE):
+    _VALID_URL = r'http://hk.dv.nextmedia.com/actionnews/[^/]+/(?P<date>\d+)/(?P<id>\d+)/\d+'
+    _TESTS = [{
+        'url': 'http://hk.dv.nextmedia.com/actionnews/hit/20150121/19009428/20061460',
+        'md5': '05fce8ffeed7a5e00665d4b7cf0f9201',
+        'info_dict': {
+            'id': '19009428',
+            'ext': 'mp4',
+            'title': '【壹週刊】細10年男友偷食　50歲邵美琪再失戀',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'description': 'md5:cd802fad1f40fd9ea178c1e2af02d659',
+            'timestamp': 1421791200,
+            'upload_date': '20150120',
+        }
+    }]
+
+    def _real_extract(self, url):
+        news_id = self._match_id(url)
+        actionnews_page = self._download_webpage(url, news_id)
+        article_url = self._og_search_url(actionnews_page)
+        article_page = self._download_webpage(article_url, news_id)
+        return self._extract_from_nextmedia_page(news_id, url, article_page)
+
+
+class AppleDailyRealtimeNewsIE(NextMediaIE):
+    _VALID_URL = r'http://(www|ent).appledaily.com.tw/(realtimenews|enews)/[^/]+/[^/]+/(?P<date>\d+)/(?P<id>\d+)(/.*)?'
+    _TESTS = [{
+        'url': 'http://ent.appledaily.com.tw/enews/article/entertainment/20150128/36354694',
+        'md5': 'a843ab23d150977cc55ef94f1e2c1e4d',
+        'info_dict': {
+            'id': '36354694',
+            'ext': 'mp4',
+            'title': '周亭羽走過摩鐵陰霾2男陪吃 九把刀孤寒看醫生',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'description': 'md5:b23787119933404ce515c6356a8c355c',
+            'upload_date': '20150128',
+        }
+    }, {
+        'url': 'http://www.appledaily.com.tw/realtimenews/article/strange/20150128/550549/%E4%B8%8D%E6%BB%BF%E8%A2%AB%E8%B8%A9%E8%85%B3%E3%80%80%E5%B1%B1%E6%9D%B1%E5%85%A9%E5%A4%A7%E5%AA%BD%E4%B8%80%E8%B7%AF%E6%89%93%E4%B8%8B%E8%BB%8A',
+        'md5': '86b4e9132d158279c7883822d94ccc49',
+        'info_dict': {
+            'id': '550549',
+            'ext': 'mp4',
+            'title': '不滿被踩腳　山東兩大媽一路打下車',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'description': 'md5:2648aaf6fc4f401f6de35a91d111aa1d',
+            'upload_date': '20150128',
+        }
+    }]
+
+    _URL_PATTERN = r'\{url: \'(.+)\'\}'
+
+    def _fetch_title(self, page):
+        return self._html_search_regex(r'<h1 id="h1">([^<>]+)</h1>', page, 'news title')
+
+    def _fetch_thumbnail(self, page):
+        return self._html_search_regex(r"setInitialImage\(\'([^']+)'\)", page, 'video thumbnail', fatal=False)
+
+    def _fetch_timestamp(self, page):
+        return None
+
+
+class AppleDailyAnimationNewsIE(AppleDailyRealtimeNewsIE):
+    _VALID_URL = 'http://www.appledaily.com.tw/animation/[^/]+/[^/]+/(?P<date>\d+)/(?P<id>\d+)(/.*)?'
+    _TESTS = [{
+        'url': 'http://www.appledaily.com.tw/animation/realtimenews/new/20150128/5003671',
+        'md5': '03df296d95dedc2d5886debbb80cb43f',
+        'info_dict': {
+            'id': '5003671',
+            'ext': 'mp4',
+            'title': '20正妹熱舞　《刀龍傳說Online》火辣上市',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'description': 'md5:23c0aac567dc08c9c16a3161a2c2e3cd',
+            'upload_date': '20150128',
+        }
+    }, {
+        # No thumbnail
+        'url': 'http://www.appledaily.com.tw/animation/realtimenews/new/20150128/5003673/',
+        'md5': 'b06182cd386ea7bc6115ec7ff0f72aeb',
+        'info_dict': {
+            'id': '5003673',
+            'ext': 'mp4',
+            'title': '半夜尿尿　好像會看到___',
+            'description': 'md5:61d2da7fe117fede148706cdb85ac066',
+            'upload_date': '20150128',
+        },
+        'expected_warnings': [
+            'video thumbnail',
+        ]
+    }]
+
+    def _fetch_title(self, page):
+        return self._html_search_meta('description', page, 'news title')
+
+    def _fetch_description(self, page):
+        return self._html_search_meta('description', page, 'news description')

From 367cc95aa76731aa951e949b325ac9c909af639d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 29 Jan 2015 03:49:56 +0800
Subject: [PATCH 0114/6219] [CtsNews] Add new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/ctsnews.py  | 51 ++++++++++++++++++++++++++++++++
 2 files changed, 52 insertions(+)
 create mode 100644 youtube_dl/extractor/ctsnews.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 03c56156a..72c5e4973 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -82,6 +82,7 @@ from .crunchyroll import (
     CrunchyrollShowPlaylistIE
 )
 from .cspan import CSpanIE
+from .ctsnews import CtsNewsIE
 from .dailymotion import (
     DailymotionIE,
     DailymotionPlaylistIE,
diff --git a/youtube_dl/extractor/ctsnews.py b/youtube_dl/extractor/ctsnews.py
new file mode 100644
index 000000000..e1d8c814e
--- /dev/null
+++ b/youtube_dl/extractor/ctsnews.py
@@ -0,0 +1,51 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import parse_iso8601, ExtractorError
+
+
+class CtsNewsIE(InfoExtractor):
+    _VALID_URL = r'http://news\.cts\.com\.tw/[a-z]+/[a-z]+/\d+/(?P<id>\d+)\.html'
+    _TESTS = [{
+        'url': 'http://news.cts.com.tw/cts/international/201309/201309031304098.html',
+        'md5': '3aee7e0df7cdff94e43581f54c22619e',
+        'info_dict': {
+            'id': '201309031304098',
+            'ext': 'mp4',
+            'title': '韓國31歲童顏男 貌如十多歲小孩',
+            'description': 'md5:f183feeba3752b683827aab71adad584',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'timestamp': 1378205880,
+            'upload_date': '20130903',
+        }
+    }]
+
+    def _real_extract(self, url):
+        news_id = self._match_id(url)
+        page = self._download_webpage(url, news_id)
+
+        if not self._search_regex(r'(CTSPlayer2)', page, 'CTSPlayer2 identifier', fatal=False):
+            raise ExtractorError('The news includes no videos!')
+
+        feed_pattern = r'(http://news.cts.com.tw/action/mp4feed.php\?news_id=\d+)'
+        feed_url = self._html_search_regex(feed_pattern, page, 'feed url')
+        feed_page = self._download_webpage(feed_url, news_id)
+
+        description = self._html_search_meta('description', page)
+        title = self._html_search_meta('title', page)
+        thumbnail = self._html_search_meta('image', page)
+
+        datetime_pattern = r'(\d{4}/\d{2}/\d{2} \d{2}:\d{2})'
+        datetime_str = self._html_search_regex(datetime_pattern, page, 'date and time')
+        time = (datetime_str + ':00+08:00').replace('/', '-')
+        timestamp = parse_iso8601(time, delimiter=' ')
+
+        return {
+            'id': news_id,
+            'title': title,
+            'description': description,
+            'url': feed_page,
+            'thumbnail': thumbnail,
+            'timestamp': timestamp
+        }

From d20547610314e0821542fecd1872b5851bd8d5d7 Mon Sep 17 00:00:00 2001
From: Paul Hartmann <phaaurlt@gmail.com>
Date: Thu, 29 Jan 2015 01:36:15 +0100
Subject: [PATCH 0115/6219] fix srmediathek description

---
 youtube_dl/extractor/srmediathek.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/srmediathek.py b/youtube_dl/extractor/srmediathek.py
index 666a7dcc8..5d583c720 100644
--- a/youtube_dl/extractor/srmediathek.py
+++ b/youtube_dl/extractor/srmediathek.py
@@ -8,7 +8,7 @@ from ..utils import js_to_json
 
 
 class SRMediathekIE(InfoExtractor):
-    IE_DESC = 'Süddeutscher Rundfunk'
+    IE_DESC = 'Saarländischer Rundfunk'
     _VALID_URL = r'https?://sr-mediathek\.sr-online\.de/index\.php\?.*?&id=(?P<id>[0-9]+)'
 
     _TEST = {

From 4f264c02c73c8f7551bcdbc960c9635860003196 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 29 Jan 2015 13:37:17 +0100
Subject: [PATCH 0116/6219] [utils] YoutubeDLHTTPSHandler.https_open: pass all
 required arguments to do_open

With this change the '--no-check-certificate' works again (#4807).
---
 youtube_dl/utils.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index b8c52af74..a4c9813ec 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -654,9 +654,14 @@ class YoutubeDLHTTPSHandler(compat_urllib_request.HTTPSHandler):
         self._params = params
 
     def https_open(self, req):
+        kwargs = {}
+        if hasattr(self, '_context'):  # python > 2.6
+            kwargs['context'] = self._context
+        if hasattr(self, '_check_hostname'):  # python 3.x
+            kwargs['check_hostname'] = self._check_hostname
         return self.do_open(functools.partial(
             _create_http_connection, self, self._https_conn_class, True),
-            req)
+            req, **kwargs)
 
 
 def parse_iso8601(date_str, delimiter='T'):

From affd04a45dcc46f21b55e4d9419812c33b9aa9ce Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 29 Jan 2015 22:09:59 +0600
Subject: [PATCH 0117/6219] [xuite] Simplify and improve

---
 youtube_dl/extractor/xuite.py | 181 +++++++++++++++-------------------
 1 file changed, 82 insertions(+), 99 deletions(-)

diff --git a/youtube_dl/extractor/xuite.py b/youtube_dl/extractor/xuite.py
index 8466d4bc5..a9dbf8c2d 100644
--- a/youtube_dl/extractor/xuite.py
+++ b/youtube_dl/extractor/xuite.py
@@ -2,158 +2,141 @@
 from __future__ import unicode_literals
 
 import base64
+
 from .common import InfoExtractor
-from ..compat import (
-    compat_urlparse,
-    compat_urllib_parse_unquote,
-    compat_parse_qs
-)
+from ..compat import compat_urllib_parse_unquote
 from ..utils import (
     ExtractorError,
     parse_iso8601,
-    parse_duration
+    parse_duration,
 )
 
-# ref: http://stackoverflow.com/questions/475074/regex-to-parse-or-validate-base64-data
-REGEX_BASE64 = r'(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?'
-
-CST_ZONE = +8  # China Standard Time
-
 
 class XuiteIE(InfoExtractor):
-    _VALID_URL = r'http://vlog.xuite.net/play/(?P<id>%s)(/.*)?' % REGEX_BASE64
+    _REGEX_BASE64 = r'(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?'
+    _VALID_URL = r'http://vlog.xuite.net/(?:play|embed)/(?P<id>%s)' % _REGEX_BASE64
     _TESTS = [{
         # Audio
         'url': 'http://vlog.xuite.net/play/RGkzc1ZULTM4NjA5MTQuZmx2',
         'md5': '63a42c705772aa53fd4c1a0027f86adf',
         'info_dict': {
-            'id': 'RGkzc1ZULTM4NjA5MTQuZmx2',
+            'id': '3860914',
             'ext': 'mp3',
-            'upload_date': '20110902',
-            'uploader_id': '15973816',
-            'uploader': '阿能',
-            'timestamp': 1314932940,
             'title': '孤單南半球-歐德陽',
             'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 247.246,
+            'timestamp': 1314932940,
+            'upload_date': '20110902',
+            'uploader': '阿能',
+            'uploader_id': '15973816',
             'categories': ['個人短片'],
-            'duration': 247.246
-        }
-    }, {
-        # Audio with alternative form of url
-        'url': 'http://vlog.xuite.net/play/S1dDUjdyLTMyOTc3NjcuZmx2/%E5%AD%AB%E7%87%95%E5%A7%BF-%E7%9C%BC%E6%B7%9A%E6%88%90%E8%A9%A9',
-        'md5': 'c91645f324de53d82ebb80930e1a73d2',
-        'info_dict': {
-            'id': 'S1dDUjdyLTMyOTc3NjcuZmx2',
-            'ext': 'mp3',
-            'upload_date': '20101226',
-            'uploader_id': '10102699',
-            'uploader': '蠍',
-            'timestamp': 1293367080,
-            'title': '孫燕姿-眼淚成詩',
-            'thumbnail': 're:^https?://.*\.jpg$',
-            'categories': ['個人短片'],
-            'duration': 223.19
-        }
+        },
     }, {
         # Video with only one format
         'url': 'http://vlog.xuite.net/play/TkRZNjhULTM0NDE2MjkuZmx2',
         'md5': 'c45737fc8ac5dc8ac2f92ecbcecf505e',
         'info_dict': {
-            'id': 'TkRZNjhULTM0NDE2MjkuZmx2',
+            'id': '3441629',
             'ext': 'mp4',
-            'upload_date': '20110306',
-            'uploader_id': '10400126',
-            'uploader': 'Valen',
-            'timestamp': 1299383640,
             'title': '孫燕姿 - 眼淚成詩',
             'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 217.399,
+            'timestamp': 1299383640,
+            'upload_date': '20110306',
+            'uploader': 'Valen',
+            'uploader_id': '10400126',
             'categories': ['影視娛樂'],
-            'duration': 217.399
-        }
+        },
     }, {
         # Video with two formats
         'url': 'http://vlog.xuite.net/play/bWo1N1pLLTIxMzAxMTcwLmZsdg==',
         'md5': '1166e0f461efe55b62e26a2d2a68e6de',
         'info_dict': {
-            'id': 'bWo1N1pLLTIxMzAxMTcwLmZsdg==',
+            'id': '21301170',
             'ext': 'mp4',
-            'upload_date': '20150117',
-            'uploader_id': '242127761',
-            'uploader': '我只是想認真點',
-            'timestamp': 1421481240,
             'title': '暗殺教室 02',
+            'description': '字幕:【極影字幕社】',
             'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 1384.907,
+            'timestamp': 1421481240,
+            'upload_date': '20150117',
+            'uploader': '我只是想認真點',
+            'uploader_id': '242127761',
             'categories': ['電玩動漫'],
-            'duration': 1384.907
-        }
+        },
+    }, {
+        'url': 'http://vlog.xuite.net/play/S1dDUjdyLTMyOTc3NjcuZmx2/%E5%AD%AB%E7%87%95%E5%A7%BF-%E7%9C%BC%E6%B7%9A%E6%88%90%E8%A9%A9',
+        'only_matching': True,
     }]
 
-    def _flv_config(self, media_id):
+    def _extract_flv_config(self, media_id):
         base64_media_id = base64.b64encode(media_id.encode('utf-8')).decode('utf-8')
-        flv_config_url = 'http://vlog.xuite.net/flash/player?media=' + base64_media_id
-        flv_config = self._download_xml(flv_config_url, 'flv config')
-
+        flv_config = self._download_xml(
+            'http://vlog.xuite.net/flash/player?media=%s' % base64_media_id,
+            'flv config')
         prop_dict = {}
         for prop in flv_config.findall('./property'):
             prop_id = base64.b64decode(prop.attrib['id']).decode('utf-8')
-
+            # CDATA may be empty in flv config
             if not prop.text:
-                continue  # CDATA may be empty in flv config
-
+                continue
             encoded_content = base64.b64decode(prop.text).decode('utf-8')
             prop_dict[prop_id] = compat_urllib_parse_unquote(encoded_content)
-
         return prop_dict
 
-    def _type_string(self, media_url):
-        query_string = compat_urlparse.urlparse(media_url).query
-        type_string = compat_parse_qs(query_string)['q'][0]
-        return type_string
-
-    def _guess_ext(self, media_url):
-        type_string = self._type_string(media_url)
-        if type_string == 'mp3':
-            return 'mp3'
-        elif type_string == '360' or type_string == '720':
-            return 'mp4'
-        else:
-            raise ExtractorError('Unknown type string %s' % type_string)
-
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        page = self._download_webpage(url, video_id)
-        media_id = self._html_search_regex(r'data-mediaid="(\d+)"', page, 'media id')
-        flv_config = self._flv_config(media_id)
+        webpage = self._download_webpage(url, video_id)
 
-        timestamp_local = parse_iso8601(flv_config['publish_datetime'], ' ')
-        timestamp_gmt = timestamp_local - CST_ZONE * 3600
+        error_msg = self._search_regex(
+            r'<div id="error-message-content">([^<]+)',
+            webpage, 'error message', default=None)
+        if error_msg:
+            raise ExtractorError(
+                '%s returned error: %s' % (self.IE_NAME, error_msg),
+                expected=True)
 
-        ret_attrs = {
-            'id': video_id,
-            'title': flv_config['title'],
-            'thumbnail': flv_config['thumb'],
-            'uploader': flv_config['author_name'],
-            'timestamp': timestamp_gmt,
-            'uploader_id': flv_config['author_id'],
-            'duration': parse_duration(flv_config['duration']),
-            'categories': [flv_config['category']]
+        video_id = self._html_search_regex(
+            r'data-mediaid="(\d+)"', webpage, 'media id')
+        flv_config = self._extract_flv_config(video_id)
+
+        FORMATS = {
+            'audio': 'mp3',
+            'video': 'mp4',
         }
 
-        if 'hq_src' in flv_config:
-            urls = [flv_config['src'], flv_config['hq_src']]
+        formats = []
+        for format_tag in ('src', 'hq_src'):
+            video_url = flv_config.get(format_tag)
+            if not video_url:
+                continue
+            format_id = self._search_regex(
+                r'\bq=(.+?)\b', video_url, 'format id', default=format_tag)
+            formats.append({
+                'url': video_url,
+                'ext': FORMATS.get(flv_config['type'], 'mp4'),
+                'format_id': format_id,
+                'height': int(format_id) if format_id.isnumeric() else None,
+            })
+        self._sort_formats(formats)
 
-            ret_attrs['formats'] = []
+        timestamp = flv_config.get('publish_datetime')
+        if timestamp:
+            timestamp = parse_iso8601(timestamp + ' +0800', ' ')
 
-            for url in urls:
-                ret_attrs['formats'].append({
-                    'url': url,
-                    'ext': self._guess_ext(url),
-                    'format_id': self._type_string(url),
-                    'height': int(self._type_string(url))
-                })
-        else:
-            ret_attrs['url'] = flv_config['src']
-            ret_attrs['ext'] = self._guess_ext(flv_config['src'])
+        category = flv_config.get('category')
+        categories = [category] if category else []
 
-        return ret_attrs
+        return {
+            'id': video_id,
+            'title': flv_config['title'],
+            'description': flv_config.get('description'),
+            'thumbnail': flv_config.get('thumb'),
+            'timestamp': timestamp,
+            'uploader': flv_config.get('author_name'),
+            'uploader_id': flv_config.get('author_id'),
+            'duration': parse_duration(flv_config.get('duration')),
+            'categories': categories,
+            'formats': formats,
+        }

From 219337990b8008eab81e1f9adf9eba20757f3cad Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 29 Jan 2015 23:11:13 +0600
Subject: [PATCH 0118/6219] [xuite] Fix _VALID_URL

---
 youtube_dl/extractor/xuite.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/xuite.py b/youtube_dl/extractor/xuite.py
index a9dbf8c2d..4971965f9 100644
--- a/youtube_dl/extractor/xuite.py
+++ b/youtube_dl/extractor/xuite.py
@@ -14,7 +14,7 @@ from ..utils import (
 
 class XuiteIE(InfoExtractor):
     _REGEX_BASE64 = r'(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?'
-    _VALID_URL = r'http://vlog.xuite.net/(?:play|embed)/(?P<id>%s)' % _REGEX_BASE64
+    _VALID_URL = r'https?://vlog\.xuite\.net/(?:play|embed)/(?P<id>%s)' % _REGEX_BASE64
     _TESTS = [{
         # Audio
         'url': 'http://vlog.xuite.net/play/RGkzc1ZULTM4NjA5MTQuZmx2',

From a7a14d958604f5334413e2fc1872a8317d5e4884 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 29 Jan 2015 20:15:38 +0100
Subject: [PATCH 0119/6219] [YoutubeDL] set the 'thumbnails' field if the
 info_dict has the 'thumbnails' field

Since the '--write-thumbnail' uses the 'thumbnails' field and we didn't updated the info_dict, it wouldn't detect the thumbnail. (fixes #4812)
---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 4c238c555..14e92ddcf 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -958,7 +958,7 @@ class YoutubeDL(object):
         if thumbnails is None:
             thumbnail = info_dict.get('thumbnail')
             if thumbnail:
-                thumbnails = [{'url': thumbnail}]
+                info_dict['thumbnails'] = thumbnails = [{'url': thumbnail}]
         if thumbnails:
             thumbnails.sort(key=lambda t: (
                 t.get('preference'), t.get('width'), t.get('height'),

From e683a48d0e8650e1d9e8c323cacb468b104ea7bc Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 30 Jan 2015 05:38:05 +0800
Subject: [PATCH 0120/6219] [ctsnews] Detect youtube embedde videos

---
 youtube_dl/extractor/ctsnews.py | 75 +++++++++++++++++++++++++++------
 1 file changed, 63 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/ctsnews.py b/youtube_dl/extractor/ctsnews.py
index e1d8c814e..35f3756f5 100644
--- a/youtube_dl/extractor/ctsnews.py
+++ b/youtube_dl/extractor/ctsnews.py
@@ -1,13 +1,27 @@
-# coding: utf-8
+# -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..compat import compat_urllib_request
 from ..utils import parse_iso8601, ExtractorError
 
 
 class CtsNewsIE(InfoExtractor):
+    # https connection failed (Connection reset)
     _VALID_URL = r'http://news\.cts\.com\.tw/[a-z]+/[a-z]+/\d+/(?P<id>\d+)\.html'
     _TESTS = [{
+        'url': 'http://news.cts.com.tw/cts/international/201501/201501291578109.html',
+        'md5': 'a9875cb790252b08431186d741beaabe',
+        'info_dict': {
+            'id': '201501291578109',
+            'ext': 'mp4',
+            'title': '以色列.真主黨交火 3人死亡',
+            'description': 'md5:95e9b295c898b7ff294f09d450178d7d',
+            'timestamp': 1422528540,
+            'upload_date': '20150129',
+        }
+    }, {
+        # News count not appear on page but still available in database
         'url': 'http://news.cts.com.tw/cts/international/201309/201309031304098.html',
         'md5': '3aee7e0df7cdff94e43581f54c22619e',
         'info_dict': {
@@ -19,33 +33,70 @@ class CtsNewsIE(InfoExtractor):
             'timestamp': 1378205880,
             'upload_date': '20130903',
         }
+    }, {
+        # With Youtube embedded video
+        'url': 'http://news.cts.com.tw/cts/money/201501/201501291578003.html',
+        'md5': '1d842c771dc94c8c3bca5af2cc1db9c5',
+        'add_ie': ['Youtube'],
+        'info_dict': {
+            'id': 'OVbfO7d0_hQ',
+            'ext': 'mp4',
+            'title': 'iPhone6熱銷 蘋果財報亮眼',
+            'description': 'md5:f395d4f485487bb0f992ed2c4b07aa7d',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'upload_date': '20150128',
+            'uploader_id': 'TBSCTS',
+            'uploader': '中華電視公司',
+        }
     }]
 
     def _real_extract(self, url):
         news_id = self._match_id(url)
         page = self._download_webpage(url, news_id)
 
-        if not self._search_regex(r'(CTSPlayer2)', page, 'CTSPlayer2 identifier', fatal=False):
-            raise ExtractorError('The news includes no videos!')
+        if self._search_regex(r'(CTSPlayer2)', page, 'CTSPlayer2 identifier', default=None):
+            feed_url = self._html_search_regex(
+                r'(http://news\.cts\.com\.tw/action/mp4feed\.php\?news_id=\d+)',
+                page, 'feed url')
+            video_url = self._download_webpage(feed_url, news_id)
+        else:
+            self.to_screen('Not CTSPlayer video, trying Youtube...')
+            youtube_url = self._search_regex(
+                r'src="(//www\.youtube\.com/embed/[^"]+)"', page, 'youtube url',
+                default=None)
+            if not youtube_url:
+                raise ExtractorError('The news includes no videos!', expected=True)
 
-        feed_pattern = r'(http://news.cts.com.tw/action/mp4feed.php\?news_id=\d+)'
-        feed_url = self._html_search_regex(feed_pattern, page, 'feed url')
-        feed_page = self._download_webpage(feed_url, news_id)
+            return {
+                '_type': 'url',
+                'url': youtube_url,
+                'ie_key': 'Youtube',
+            }
 
         description = self._html_search_meta('description', page)
         title = self._html_search_meta('title', page)
         thumbnail = self._html_search_meta('image', page)
 
-        datetime_pattern = r'(\d{4}/\d{2}/\d{2} \d{2}:\d{2})'
-        datetime_str = self._html_search_regex(datetime_pattern, page, 'date and time')
-        time = (datetime_str + ':00+08:00').replace('/', '-')
-        timestamp = parse_iso8601(time, delimiter=' ')
+        datetime_str = self._html_search_regex(
+            r'(\d{4}/\d{2}/\d{2} \d{2}:\d{2})', page, 'date and time')
+        # Transform into ISO 8601 format with timezone info
+        datetime_str = datetime_str.replace('/', '-') + ':00+0800'
+        timestamp = parse_iso8601(datetime_str, delimiter=' ')
+
+        # Note: the news count may decrease as time goes by
+        # It should be a bug in CTS website
+        req = compat_urllib_request.Request(
+            'http://news.cts.com.tw/action/news_count.php?callback=cb&news_id=' + news_id)
+        req.add_header('Referer', url)
+        newscount_page = self._download_webpage(req, news_id)
+        news_count = self._search_regex(r'cb\((\d+)\)', newscount_page, 'news count')
 
         return {
             'id': news_id,
+            'url': video_url,
             'title': title,
             'description': description,
-            'url': feed_page,
             'thumbnail': thumbnail,
-            'timestamp': timestamp
+            'timestamp': timestamp,
+            'view_count': news_count,
         }

From f345fe9db77a900a603d753c73c5566d62c1cff9 Mon Sep 17 00:00:00 2001
From: Paul Hartmann <phaaurlt@gmail.com>
Date: Thu, 29 Jan 2015 23:32:23 +0100
Subject: [PATCH 0121/6219] [dctp] fix python 2.6 compatibility

---
 youtube_dl/extractor/dctp.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/dctp.py b/youtube_dl/extractor/dctp.py
index 9b687ef43..31bcd35c3 100644
--- a/youtube_dl/extractor/dctp.py
+++ b/youtube_dl/extractor/dctp.py
@@ -20,10 +20,10 @@ class DctpTvIE(InfoExtractor):
         version_json = self._download_json(base_url + 'version.json', video_id)
         version = version_json['version_name']
         info_json = self._download_json(
-            '{}{}/restapi/slugs/{}.json'.format(base_url, version, video_id), video_id)
+            '{0}{1}/restapi/slugs/{2}.json'.format(base_url, version, video_id), video_id)
         object_id = info_json['object_id']
         meta_json = self._download_json(
-            '{}{}/restapi/media/{}.json'.format(base_url, version, object_id), video_id)
+            '{0}{1}/restapi/media/{2}.json'.format(base_url, version, object_id), video_id)
         uuid = meta_json['uuid']
         title = meta_json['title']
         wide = meta_json['is_wide']
@@ -31,7 +31,7 @@ class DctpTvIE(InfoExtractor):
             ratio = '16x9'
         else:
             ratio = '4x3'
-        play_path = 'mp4:{}_dctp_0500_{}.m4v'.format(uuid, ratio)
+        play_path = 'mp4:{0}_dctp_0500_{1}.m4v'.format(uuid, ratio)
 
         servers_json = self._download_json('http://www.dctp.tv/streaming_servers/', video_id)
         url = servers_json[0]['endpoint']

From 87673cd4389f6798bb438bb46a306dc08c906cf5 Mon Sep 17 00:00:00 2001
From: Paul Hartmann <phaaurlt@gmail.com>
Date: Thu, 29 Jan 2015 23:34:56 +0100
Subject: [PATCH 0122/6219] [dctp] follow id conventions

---
 youtube_dl/extractor/dctp.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/dctp.py b/youtube_dl/extractor/dctp.py
index 31bcd35c3..752cff978 100644
--- a/youtube_dl/extractor/dctp.py
+++ b/youtube_dl/extractor/dctp.py
@@ -37,12 +37,13 @@ class DctpTvIE(InfoExtractor):
         url = servers_json[0]['endpoint']
 
         return {
-            'id': video_id,
+            'id': object_id,
             'title': title,
             'format': 'rtmp',
             'url': url,
             'play_path': play_path,
             'real_time': True,
-            'ext': 'flv'
+            'ext': 'flv',
+            'display_id': video_id
         }
 

From 75a4fc5b7233d4cb4376b205c585eb4d3d486241 Mon Sep 17 00:00:00 2001
From: Paul Hartmann <phaaurlt@gmail.com>
Date: Fri, 30 Jan 2015 00:35:53 +0100
Subject: [PATCH 0123/6219] [dctp] fix test

---
 youtube_dl/extractor/dctp.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/dctp.py b/youtube_dl/extractor/dctp.py
index 752cff978..c80d28865 100644
--- a/youtube_dl/extractor/dctp.py
+++ b/youtube_dl/extractor/dctp.py
@@ -9,7 +9,8 @@ class DctpTvIE(InfoExtractor):
     _TEST = {
         'url': 'http://www.dctp.tv/filme/videoinstallation-fuer-eine-kaufhausfassade/',
         'info_dict': {
-            'id': 'videoinstallation-fuer-eine-kaufhausfassade',
+            'id': '1324',
+            'display_id': 'videoinstallation-fuer-eine-kaufhausfassade',
             'ext': 'flv',
             'title': 'Videoinstallation für eine Kaufhausfassade'}
         }
@@ -21,7 +22,7 @@ class DctpTvIE(InfoExtractor):
         version = version_json['version_name']
         info_json = self._download_json(
             '{0}{1}/restapi/slugs/{2}.json'.format(base_url, version, video_id), video_id)
-        object_id = info_json['object_id']
+        object_id = str(info_json['object_id'])
         meta_json = self._download_json(
             '{0}{1}/restapi/media/{2}.json'.format(base_url, version, object_id), video_id)
         uuid = meta_json['uuid']

From 7bb3ceb4c731c9783d2c565204cbc1071c3545ed Mon Sep 17 00:00:00 2001
From: Paul Hartmann <phaaurlt@gmail.com>
Date: Fri, 30 Jan 2015 00:47:43 +0100
Subject: [PATCH 0124/6219] [dctp] prefix real_time parameter with rtmp_

---
 youtube_dl/downloader/rtmp.py | 2 +-
 youtube_dl/extractor/dctp.py  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/downloader/rtmp.py b/youtube_dl/downloader/rtmp.py
index 6cb1bfc50..f7eeb6f43 100644
--- a/youtube_dl/downloader/rtmp.py
+++ b/youtube_dl/downloader/rtmp.py
@@ -104,9 +104,9 @@ class RtmpFD(FileDownloader):
         live = info_dict.get('rtmp_live', False)
         conn = info_dict.get('rtmp_conn', None)
         protocol = info_dict.get('rtmp_protocol', None)
+        real_time = info_dict.get('rtmp_real_time', False)
         no_resume = info_dict.get('no_resume', False)
         continue_dl = info_dict.get('continuedl', False)
-        real_time = info_dict.get('real_time', False)
 
         self.report_destination(filename)
         tmpfilename = self.temp_name(filename)
diff --git a/youtube_dl/extractor/dctp.py b/youtube_dl/extractor/dctp.py
index c80d28865..8a77f2b66 100644
--- a/youtube_dl/extractor/dctp.py
+++ b/youtube_dl/extractor/dctp.py
@@ -43,7 +43,7 @@ class DctpTvIE(InfoExtractor):
             'format': 'rtmp',
             'url': url,
             'play_path': play_path,
-            'real_time': True,
+            'rtmp_real_time': True,
             'ext': 'flv',
             'display_id': video_id
         }

From adc0ae3ceb9371eb23a9a4d81d0e134ac861723e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 30 Jan 2015 01:56:15 +0100
Subject: [PATCH 0125/6219] [__init__] Provide a better error messages if URLs
 are missing (Closes #4813)

---
 youtube_dl/__init__.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 71d2c6f35..e90679ff9 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -361,7 +361,9 @@ def _real_main(argv=None):
                 sys.exit()
 
             ydl.warn_if_short_id(sys.argv[1:] if argv is None else argv)
-            parser.error('you must provide at least one URL')
+            parser.error(
+                'You must provide at least one URL.\n'
+                'Type youtube-dl --help to see a list of all options.')
 
         try:
             if opts.load_info_filename is not None:

From 4fe8495a23271e164a1ce618682d50f927adc075 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 30 Jan 2015 02:13:37 +0100
Subject: [PATCH 0126/6219] [viddler] PEP8

---
 youtube_dl/extractor/viddler.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/viddler.py b/youtube_dl/extractor/viddler.py
index ef104dc29..8516a2940 100644
--- a/youtube_dl/extractor/viddler.py
+++ b/youtube_dl/extractor/viddler.py
@@ -6,7 +6,7 @@ from ..utils import (
     int_or_none,
 )
 from ..compat import (
-   compat_urllib_request
+    compat_urllib_request
 )
 
 

From 83fda3c000a680317cfc9bb6fec899beb8bca773 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 30 Jan 2015 02:57:37 +0100
Subject: [PATCH 0127/6219] Add a test for --no-check-certificate

---
 test/test_http.py    | 71 ++++++++++++++++++++++++++++++++++++++++++++
 test/testcert.pem    | 52 ++++++++++++++++++++++++++++++++
 youtube_dl/compat.py |  6 ++++
 3 files changed, 129 insertions(+)
 create mode 100644 test/test_http.py
 create mode 100644 test/testcert.pem

diff --git a/test/test_http.py b/test/test_http.py
new file mode 100644
index 000000000..5cce5b3ae
--- /dev/null
+++ b/test/test_http.py
@@ -0,0 +1,71 @@
+#!/usr/bin/env python
+from __future__ import unicode_literals
+
+# Allow direct execution
+import os
+import sys
+import unittest
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from youtube_dl import YoutubeDL
+from youtube_dl.compat import compat_http_server
+import ssl
+import threading
+
+TEST_DIR = os.path.dirname(os.path.abspath(__file__))
+
+class HTTPTestRequestHandler(compat_http_server.BaseHTTPRequestHandler):
+    def log_message(self, format, *args):
+        pass
+
+    def do_GET(self):
+        if self.path == '/video.html':
+            self.send_response(200)
+            self.send_header('Content-Type', 'text/html; charset=utf-8')
+            self.end_headers()
+            self.wfile.write(b'<html><video src="/vid.mp4" /></html>')
+        elif self.path == '/vid.mp4':
+            self.send_response(200)
+            self.send_header('Content-Type', 'video/mp4')
+            self.end_headers()
+            self.wfile.write(b'\x00\x00\x00\x00\x20\x66\x74[video]')
+        else:
+            assert False
+
+
+class FakeLogger(object):
+    def debug(self, msg):
+        pass
+
+    def warning(self, msg):
+        pass
+
+    def error(self, msg):
+        pass
+
+
+class TestHTTP(unittest.TestCase):
+    def setUp(self):
+        certfn = os.path.join(TEST_DIR, 'testcert.pem')
+        self.httpd = compat_http_server.HTTPServer(
+            ('localhost', 0), HTTPTestRequestHandler)
+        self.httpd.socket = ssl.wrap_socket(
+            self.httpd.socket, certfile=certfn, server_side=True)
+        self.port = self.httpd.socket.getsockname()[1]
+        self.server_thread = threading.Thread(target=self.httpd.serve_forever)
+        self.server_thread.daemon = True
+        self.server_thread.start()
+
+    def test_nocheckcertificate(self):
+        if sys.version_info >= (2, 7, 9):  # No certificate checking anyways
+            ydl = YoutubeDL({'logger': FakeLogger()})
+            self.assertRaises(
+                Exception,
+                ydl.extract_info, 'https://localhost:%d/video.html' % self.port)
+
+        ydl = YoutubeDL({'logger': FakeLogger(), 'nocheckcertificate': True})
+        r = ydl.extract_info('https://localhost:%d/video.html' % self.port)
+        self.assertEqual(r['url'], 'https://localhost:%d/vid.mp4' % self.port)
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/test/testcert.pem b/test/testcert.pem
new file mode 100644
index 000000000..b3e0f00c7
--- /dev/null
+++ b/test/testcert.pem
@@ -0,0 +1,52 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDMF0bAzaHAdIyB
+HRmnIp4vv40lGqEePmWqicCl0QZ0wsb5dNysSxSa7330M2QeQopGfdaUYF1uTcNp
+Qx6ECgBSfg+RrOBI7r/u4F+sKX8MUXVaf/5QoBUrGNGSn/pp7HMGOuQqO6BVg4+h
+A1ySSwUG8mZItLRry1ISyErmW8b9xlqfd97uLME/5tX+sMelRFjUbAx8A4CK58Ev
+mMguHVTlXzx5RMdYcf1VScYcjlV/qA45uzP8zwI5aigfcmUD+tbGuQRhKxUhmw0J
+aobtOR6+JSOAULW5gYa/egE4dWLwbyM6b6eFbdnjlQzEA1EW7ChMPAW/Mo83KyiP
+tKMCSQulAgMBAAECggEALCfBDAexPjU5DNoh6bIorUXxIJzxTNzNHCdvgbCGiA54
+BBKPh8s6qwazpnjT6WQWDIg/O5zZufqjE4wM9x4+0Zoqfib742ucJO9wY4way6x4
+Clt0xzbLPabB+MoZ4H7ip+9n2+dImhe7pGdYyOHoNYeOL57BBi1YFW42Hj6u/8pd
+63YCXisto3Rz1YvRQVjwsrS+cRKZlzAFQRviL30jav7Wh1aWEfcXxjj4zhm8pJdk
+ITGtq6howz57M0NtX6hZnfe8ywzTnDFIGKIMA2cYHuYJcBh9bc4tCGubTvTKK9UE
+8fM+f6UbfGqfpKCq1mcgs0XMoFDSzKS9+mSJn0+5JQKBgQD+OCKaeH3Yzw5zGnlw
+XuQfMJGNcgNr+ImjmvzUAC2fAZUJLAcQueE5kzMv5Fmd+EFE2CEX1Vit3tg0SXvA
+G+bq609doILHMA03JHnV1npO/YNIhG3AAtJlKYGxQNfWH9mflYj9mEui8ZFxG52o
+zWhHYuifOjjZszUR+/eio6NPzwKBgQDNhUBTrT8LIX4SE/EFUiTlYmWIvOMgXYvN
+8Cm3IRNQ/yyphZaXEU0eJzfX5uCDfSVOgd6YM/2pRah+t+1Hvey4H8e0GVTu5wMP
+gkkqwKPGIR1YOmlw6ippqwvoJD7LuYrm6Q4D6e1PvkjwCq6lEndrOPmPrrXNd0JJ
+XO60y3U2SwKBgQDLkyZarryQXxcCI6Q10Tc6pskYDMIit095PUbTeiUOXNT9GE28
+Hi32ziLCakk9kCysNasii81MxtQ54tJ/f5iGbNMMddnkKl2a19Hc5LjjAm4cJzg/
+98KGEhvyVqvAo5bBDZ06/rcrD+lZOzUglQS5jcIcqCIYa0LHWQ/wJLxFzwKBgFcZ
+1SRhdSmDfUmuF+S4ZpistflYjC3IV5rk4NkS9HvMWaJS0nqdw4A3AMzItXgkjq4S
+DkOVLTkTI5Do5HAWRv/VwC5M2hkR4NMu1VGAKSisGiKtRsirBWSZMEenLNHshbjN
+Jrpz5rZ4H7NT46ZkCCZyFBpX4gb9NyOedjA7Via3AoGARF8RxbYjnEGGFuhnbrJB
+FTPR0vaL4faY3lOgRZ8jOG9V2c9Hzi/y8a8TU4C11jnJSDqYCXBTd5XN28npYxtD
+pjRsCwy6ze+yvYXPO7C978eMG3YRyj366NXUxnXN59ibwe/lxi2OD9z8J1LEdF6z
+VJua1Wn8HKxnXMI61DhTCSo=
+-----END PRIVATE KEY-----
+-----BEGIN CERTIFICATE-----
+MIIEEzCCAvugAwIBAgIJAK1haYi6gmSKMA0GCSqGSIb3DQEBCwUAMIGeMQswCQYD
+VQQGEwJERTEMMAoGA1UECAwDTlJXMRQwEgYDVQQHDAtEdWVzc2VsZG9yZjEbMBkG
+A1UECgwSeW91dHViZS1kbCBwcm9qZWN0MRkwFwYDVQQLDBB5b3V0dWJlLWRsIHRl
+c3RzMRIwEAYDVQQDDAlsb2NhbGhvc3QxHzAdBgkqhkiG9w0BCQEWEHBoaWhhZ0Bw
+aGloYWcuZGUwIBcNMTUwMTMwMDExNTA4WhgPMjExNTAxMDYwMTE1MDhaMIGeMQsw
+CQYDVQQGEwJERTEMMAoGA1UECAwDTlJXMRQwEgYDVQQHDAtEdWVzc2VsZG9yZjEb
+MBkGA1UECgwSeW91dHViZS1kbCBwcm9qZWN0MRkwFwYDVQQLDBB5b3V0dWJlLWRs
+IHRlc3RzMRIwEAYDVQQDDAlsb2NhbGhvc3QxHzAdBgkqhkiG9w0BCQEWEHBoaWhh
+Z0BwaGloYWcuZGUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDMF0bA
+zaHAdIyBHRmnIp4vv40lGqEePmWqicCl0QZ0wsb5dNysSxSa7330M2QeQopGfdaU
+YF1uTcNpQx6ECgBSfg+RrOBI7r/u4F+sKX8MUXVaf/5QoBUrGNGSn/pp7HMGOuQq
+O6BVg4+hA1ySSwUG8mZItLRry1ISyErmW8b9xlqfd97uLME/5tX+sMelRFjUbAx8
+A4CK58EvmMguHVTlXzx5RMdYcf1VScYcjlV/qA45uzP8zwI5aigfcmUD+tbGuQRh
+KxUhmw0JaobtOR6+JSOAULW5gYa/egE4dWLwbyM6b6eFbdnjlQzEA1EW7ChMPAW/
+Mo83KyiPtKMCSQulAgMBAAGjUDBOMB0GA1UdDgQWBBTBUZoqhQkzHQ6xNgZfFxOd
+ZEVt8TAfBgNVHSMEGDAWgBTBUZoqhQkzHQ6xNgZfFxOdZEVt8TAMBgNVHRMEBTAD
+AQH/MA0GCSqGSIb3DQEBCwUAA4IBAQCUOCl3T/J9B08Z+ijfOJAtkbUaEHuVZb4x
+5EpZSy2ZbkLvtsftMFieHVNXn9dDswQc5qjYStCC4o60LKw4M6Y63FRsAZ/DNaqb
+PY3jyCyuugZ8/sNf50vHYkAcF7SQYqOQFQX4TQsNUk2xMJIt7H0ErQFmkf/u3dg6
+cy89zkT462IwxzSG7NNhIlRkL9o5qg+Y1mF9eZA1B0rcL6hO24PPTHOd90HDChBu
+SZ6XMi/LzYQSTf0Vg2R+uMIVlzSlkdcZ6sqVnnqeLL8dFyIa4e9sj/D4ZCYP8Mqe
+Z73H5/NNhmwCHRqVUTgm307xblQaWGhwAiDkaRvRW2aJQ0qGEdZK
+-----END CERTIFICATE-----
diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 4453b34fc..497ca52de 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -71,6 +71,11 @@ try:
 except ImportError:
     compat_subprocess_get_DEVNULL = lambda: open(os.path.devnull, 'w')
 
+try:
+    import http.server as compat_http_server
+except ImportError:
+    import BaseHTTPServer as compat_http_server
+
 try:
     from urllib.parse import unquote as compat_urllib_parse_unquote
 except ImportError:
@@ -365,6 +370,7 @@ __all__ = [
     'compat_html_entities',
     'compat_html_parser',
     'compat_http_client',
+    'compat_http_server',
     'compat_kwargs',
     'compat_ord',
     'compat_parse_qs',

From 03d8d4df385bbf1e4cf1f63609d06dc4ad11313a Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 30 Jan 2015 03:06:40 +0100
Subject: [PATCH 0128/6219] [test_http] PEP8

---
 test/test_http.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/test/test_http.py b/test/test_http.py
index 5cce5b3ae..bd4d46fef 100644
--- a/test/test_http.py
+++ b/test/test_http.py
@@ -14,6 +14,7 @@ import threading
 
 TEST_DIR = os.path.dirname(os.path.abspath(__file__))
 
+
 class HTTPTestRequestHandler(compat_http_server.BaseHTTPRequestHandler):
     def log_message(self, format, *args):
         pass

From ec4161a57d66c92ea7460ac44dda13aadfb85f21 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 30 Jan 2015 03:06:51 +0100
Subject: [PATCH 0129/6219] [ctsnews] Remove news count check (#4802)

That's the number of articles (or at least some constant value); view count is the number of views of the video!
Also add a better description so that we can distinguish the two downloads.
---
 youtube_dl/extractor/ctsnews.py | 12 ++----------
 1 file changed, 2 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/ctsnews.py b/youtube_dl/extractor/ctsnews.py
index 35f3756f5..f769dc031 100644
--- a/youtube_dl/extractor/ctsnews.py
+++ b/youtube_dl/extractor/ctsnews.py
@@ -58,7 +58,8 @@ class CtsNewsIE(InfoExtractor):
             feed_url = self._html_search_regex(
                 r'(http://news\.cts\.com\.tw/action/mp4feed\.php\?news_id=\d+)',
                 page, 'feed url')
-            video_url = self._download_webpage(feed_url, news_id)
+            video_url = self._download_webpage(
+                feed_url, news_id, note='Fetching feed')
         else:
             self.to_screen('Not CTSPlayer video, trying Youtube...')
             youtube_url = self._search_regex(
@@ -83,14 +84,6 @@ class CtsNewsIE(InfoExtractor):
         datetime_str = datetime_str.replace('/', '-') + ':00+0800'
         timestamp = parse_iso8601(datetime_str, delimiter=' ')
 
-        # Note: the news count may decrease as time goes by
-        # It should be a bug in CTS website
-        req = compat_urllib_request.Request(
-            'http://news.cts.com.tw/action/news_count.php?callback=cb&news_id=' + news_id)
-        req.add_header('Referer', url)
-        newscount_page = self._download_webpage(req, news_id)
-        news_count = self._search_regex(r'cb\((\d+)\)', newscount_page, 'news count')
-
         return {
             'id': news_id,
             'url': video_url,
@@ -98,5 +91,4 @@ class CtsNewsIE(InfoExtractor):
             'description': description,
             'thumbnail': thumbnail,
             'timestamp': timestamp,
-            'view_count': news_count,
         }

From 3198291f26e215a7b26906a630d42a30615869b4 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 30 Jan 2015 03:09:51 +0100
Subject: [PATCH 0130/6219] [ctsnews] Remove unused import

---
 youtube_dl/extractor/ctsnews.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/ctsnews.py b/youtube_dl/extractor/ctsnews.py
index f769dc031..0226f8036 100644
--- a/youtube_dl/extractor/ctsnews.py
+++ b/youtube_dl/extractor/ctsnews.py
@@ -2,7 +2,6 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_request
 from ..utils import parse_iso8601, ExtractorError
 
 

From f452f72c6ba0cc3fe560e7449fcb95e804da5b69 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 30 Jan 2015 03:12:12 +0100
Subject: [PATCH 0131/6219] Credit @yan12125 for ctsnews and nextmedia

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 20e620c35..2203c4b63 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -107,3 +107,4 @@ Yen Chi Hsuan
 Enam Mijbah Noor
 David Luhmer
 Shaya Goldberg
+Yen Chi Hsuan

From 6ebb0dca9f40ecf9fb82882c097f42f497c70cda Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 30 Jan 2015 03:15:34 +0100
Subject: [PATCH 0132/6219] [dctp] Improve output (#4796)

---
 youtube_dl/extractor/dctp.py | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/dctp.py b/youtube_dl/extractor/dctp.py
index 8a77f2b66..dbedafa7d 100644
--- a/youtube_dl/extractor/dctp.py
+++ b/youtube_dl/extractor/dctp.py
@@ -2,10 +2,11 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..compat import compat_str
 
 
 class DctpTvIE(InfoExtractor):
-    _VALID_URL = r'^http://www.dctp.tv/(#/)?filme/(?P<id>.+?)/$'
+    _VALID_URL = r'http://www.dctp.tv/(#/)?filme/(?P<id>.+?)/$'
     _TEST = {
         'url': 'http://www.dctp.tv/filme/videoinstallation-fuer-eine-kaufhausfassade/',
         'info_dict': {
@@ -18,13 +19,17 @@ class DctpTvIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
         base_url = 'http://dctp-ivms2-restapi.s3.amazonaws.com/'
-        version_json = self._download_json(base_url + 'version.json', video_id)
+        version_json = self._download_json(
+            base_url + 'version.json',
+            video_id, note='Determining file version')
         version = version_json['version_name']
         info_json = self._download_json(
-            '{0}{1}/restapi/slugs/{2}.json'.format(base_url, version, video_id), video_id)
-        object_id = str(info_json['object_id'])
+            '{0}{1}/restapi/slugs/{2}.json'.format(base_url, version, video_id),
+            video_id, note='Fetching object ID')
+        object_id = compat_str(info_json['object_id'])
         meta_json = self._download_json(
-            '{0}{1}/restapi/media/{2}.json'.format(base_url, version, object_id), video_id)
+            '{0}{1}/restapi/media/{2}.json'.format(base_url, version, object_id),
+            video_id, note='Downloading metadata')
         uuid = meta_json['uuid']
         title = meta_json['title']
         wide = meta_json['is_wide']
@@ -34,7 +39,9 @@ class DctpTvIE(InfoExtractor):
             ratio = '4x3'
         play_path = 'mp4:{0}_dctp_0500_{1}.m4v'.format(uuid, ratio)
 
-        servers_json = self._download_json('http://www.dctp.tv/streaming_servers/', video_id)
+        servers_json = self._download_json(
+            'http://www.dctp.tv/streaming_servers/',
+            video_id, note='Downloading server list')
         url = servers_json[0]['endpoint']
 
         return {

From 1798791df11920928818161c48fc2b12334e1e52 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 30 Jan 2015 03:16:36 +0100
Subject: [PATCH 0133/6219] Credit @bastik for dctp.tv (#4796)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 2203c4b63..4a70e7cca 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -108,3 +108,4 @@ Enam Mijbah Noor
 David Luhmer
 Shaya Goldberg
 Yen Chi Hsuan
+Paul Hartmann

From 83fddfd493ffcbb7256f0f1a467cc48ca2191090 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 30 Jan 2015 03:45:06 +0100
Subject: [PATCH 0134/6219] [dctp] PEP8

---
 youtube_dl/extractor/dctp.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/dctp.py b/youtube_dl/extractor/dctp.py
index dbedafa7d..6ed354379 100644
--- a/youtube_dl/extractor/dctp.py
+++ b/youtube_dl/extractor/dctp.py
@@ -13,8 +13,9 @@ class DctpTvIE(InfoExtractor):
             'id': '1324',
             'display_id': 'videoinstallation-fuer-eine-kaufhausfassade',
             'ext': 'flv',
-            'title': 'Videoinstallation für eine Kaufhausfassade'}
+            'title': 'Videoinstallation für eine Kaufhausfassade'
         }
+    }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -54,4 +55,3 @@ class DctpTvIE(InfoExtractor):
             'ext': 'flv',
             'display_id': video_id
         }
-

From c1708b89c09d7288213110fba859f4c5e0339c31 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 30 Jan 2015 03:45:29 +0100
Subject: [PATCH 0135/6219] [youtube:truncated_url] Add hl= forms (#4799)

---
 youtube_dl/extractor/youtube.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index b7b91f354..bc5fa0b39 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1688,6 +1688,7 @@ class YoutubeTruncatedURLIE(InfoExtractor):
             feature=[a-z_]+|
             annotation_id=annotation_[^&]+|
             x-yt-cl=[0-9]+|
+            hl=[^&]*|
         )?
         |
             attribution_link\?a=[^&]+
@@ -1707,6 +1708,9 @@ class YoutubeTruncatedURLIE(InfoExtractor):
     }, {
         'url': 'https://www.youtube.com/watch?feature=foo',
         'only_matching': True,
+    }, {
+        'url': 'https://www.youtube.com/watch?hl=en-GB',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 5a8a29cfea8da1afaa94bd2d5be5c1bf72b2d17c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 30 Jan 2015 03:46:57 +0100
Subject: [PATCH 0136/6219] release 2015.01.30

---
 README.md             | 6 +++---
 youtube_dl/version.py | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index 0580ab2a3..314f9a8b2 100644
--- a/README.md
+++ b/README.md
@@ -292,9 +292,9 @@ which means you can modify it, redistribute it or use it however you like.
                                      video results by putting a condition in
                                      brackets, as in -f "best[height=720]" (or
                                      -f "[filesize>10M]").  This works for
-                                     filesize, height, width, tbr, abr, and vbr
-                                     and the comparisons <, <=, >, >=, =, != .
-                                     Formats for which the value is not known
+                                     filesize, height, width, tbr, abr, vbr, and
+                                     fps and the comparisons <, <=, >, >=, =, !=
+                                     . Formats for which the value is not known
                                      are excluded unless you put a question mark
                                      (?) after the operator. You can combine
                                      format filters, so  -f "[height <=?
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 0d59cddd7..4c75db08f 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.01.25'
+__version__ = '2015.01.30'

From 5d6677ca28147495ffbeaf5bd49647e611cf50fc Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 30 Jan 2015 03:50:40 +0100
Subject: [PATCH 0137/6219] release 2015.01.30.1

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 4c75db08f..a5d558520 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.01.30'
+__version__ = '2015.01.30.1'

From e2a08185c6b9b3f1f4b7e78138b2b6735d6b0184 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 30 Jan 2015 04:17:44 +0100
Subject: [PATCH 0138/6219] [README] Add an FAQ for YouTube IDs starting with a
 dash (Closes #4800)

---
 README.md | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/README.md b/README.md
index 314f9a8b2..09da0129e 100644
--- a/README.md
+++ b/README.md
@@ -525,6 +525,13 @@ From then on, after restarting your shell, you will be able to access both youtu
 
 Use the `-o` to specify an [output template](#output-template), for example `-o "/home/user/videos/%(title)s-%(id)s.%(ext)s"`. If you want this for all of your downloads, put the option into your [configuration file](#configuration).
 
+### How do I download a video starting with a `-` ?
+
+Either prepend `http://www.youtube.com/watch?v=` or separate the ID from the options with `--`:
+
+    youtube-dl -- -wNyEUrxzFU
+    youtube-dl "http://www.youtube.com/watch?v=-wNyEUrxzFU"
+
 ### How can I detect whether a given URL is supported by youtube-dl?
 
 For one, have a look at the [list of supported sites](docs/supportedsites). Note that it can sometimes happen that the site changes its URL scheme (say, from http://example.com/v/1234567 to http://example.com/v/1234567 ) and youtube-dl reports an URL of a service in that list as unsupported. In that case, simply report a bug.

From 6449cd807e07e04eb88b1a34063e00677de0713e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 30 Jan 2015 04:43:50 +0100
Subject: [PATCH 0139/6219] [youtube] Fall back to embed webpage when content
 is blocked on main (Fixes #4717)

---
 youtube_dl/extractor/youtube.py | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index bc5fa0b39..a9e444351 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -28,6 +28,7 @@ from ..utils import (
     get_element_by_attribute,
     get_element_by_id,
     int_or_none,
+    js_to_json,
     OnDemandPagedList,
     orderedSet,
     unescapeHTML,
@@ -809,6 +810,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
             player_url = None
 
         # Get video info
+        embed_webpage = None
         if re.search(r'player-age-gate-content">', video_webpage) is not None:
             age_gate = True
             # We simulate the access to the video from www.youtube.com/v/{video_id}
@@ -1016,10 +1018,21 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
                     url += '&signature=' + url_data['sig'][0]
                 elif 's' in url_data:
                     encrypted_sig = url_data['s'][0]
+                    ASSETS_RE = r'"assets":.+?"js":\s*("[^"]+")'
 
                     jsplayer_url_json = self._search_regex(
-                        r'"assets":.+?"js":\s*("[^"]+")',
-                        embed_webpage if age_gate else video_webpage, 'JS player URL')
+                        ASSETS_RE,
+                        embed_webpage if age_gate else video_webpage,
+                        'JS player URL (1)', default=None)
+                    if not jsplayer_url_json and not age_gate:
+                        # We need the embed website after all
+                        if embed_webpage is None:
+                            embed_url = proto + '://www.youtube.com/embed/%s' % video_id
+                            embed_webpage = self._download_webpage(
+                                embed_url, video_id, 'Downloading embed webpage')
+                        jsplayer_url_json = self._search_regex(
+                            ASSETS_RE, embed_webpage, 'JS player URL')
+
                     player_url = json.loads(jsplayer_url_json)
                     if player_url is None:
                         player_url_json = self._search_regex(

From c03844a4ec0bffef4803394545c7a5cc16dc70e1 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 30 Jan 2015 04:44:00 +0100
Subject: [PATCH 0140/6219] release 2015.01.30.2

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index a5d558520..5e6288e8e 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.01.30.1'
+__version__ = '2015.01.30.2'

From 52585fd6dcd5020f2800d133588d7232419dddc0 Mon Sep 17 00:00:00 2001
From: codesparkle <adam.thalhammer@outlook.com>
Date: Fri, 30 Jan 2015 18:41:40 +1100
Subject: [PATCH 0141/6219] The opening curly brace `{` is a regex reserved
 [control character](http://stackoverflow.com/a/400316/1106367), so it needs
 to be escaped.

---
 youtube_dl/extractor/generic.py     | 2 +-
 youtube_dl/extractor/grooveshark.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 41884ed7a..a6f102a4e 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1073,7 +1073,7 @@ class GenericIE(InfoExtractor):
             found = filter_video(re.findall(r'''(?xs)
                 flowplayer\("[^"]+",\s*
                     \{[^}]+?\}\s*,
-                    \s*{[^}]+? ["']?clip["']?\s*:\s*\{\s*
+                    \s*\{[^}]+? ["']?clip["']?\s*:\s*\{\s*
                         ["']?url["']?\s*:\s*["']([^"']+)["']
             ''', webpage))
         if not found:
diff --git a/youtube_dl/extractor/grooveshark.py b/youtube_dl/extractor/grooveshark.py
index fff74a70a..848d17beb 100644
--- a/youtube_dl/extractor/grooveshark.py
+++ b/youtube_dl/extractor/grooveshark.py
@@ -83,7 +83,7 @@ class GroovesharkIE(InfoExtractor):
         return compat_urlparse.urlunparse((uri.scheme, uri.netloc, obj['attrs']['data'], None, None, None))
 
     def _transform_bootstrap(self, js):
-        return re.split('(?m)^\s*try\s*{', js)[0] \
+        return re.split('(?m)^\s*try\s*\{', js)[0] \
                  .split(' = ', 1)[1].strip().rstrip(';')
 
     def _transform_meta(self, js):

From 1d2daaea6372f5119be8fc21637a275fe93a52ff Mon Sep 17 00:00:00 2001
From: codesparkle <adam.thalhammer@outlook.com>
Date: Fri, 30 Jan 2015 20:10:12 +1100
Subject: [PATCH 0142/6219] Simplify list creation

---
 youtube_dl/extractor/cinchcast.py | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/cinchcast.py b/youtube_dl/extractor/cinchcast.py
index 0c9a24bef..e23c515d9 100644
--- a/youtube_dl/extractor/cinchcast.py
+++ b/youtube_dl/extractor/cinchcast.py
@@ -28,12 +28,7 @@ class CinchcastIE(InfoExtractor):
             item, './{http://developer.longtailvideo.com/trac/}date')
         upload_date = unified_strdate(date_str, day_first=False)
         # duration is present but wrong
-        formats = []
-        formats.append({
-            'format_id': 'main',
-            'url': item.find(
-                './{http://search.yahoo.com/mrss/}content').attrib['url'],
-        })
+        formats = [{'format_id': 'main', 'url': item.find('./{http://search.yahoo.com/mrss/}content').attrib['url']}]
         backup_url = xpath_text(
             item, './{http://developer.longtailvideo.com/trac/}backupContent')
         if backup_url:

From 5468ff4d917eeb292ce0bfe7cf13eef75cccae19 Mon Sep 17 00:00:00 2001
From: codesparkle <adam.thalhammer@outlook.com>
Date: Fri, 30 Jan 2015 20:11:51 +1100
Subject: [PATCH 0143/6219] Remove duplicate dictionary keys

---
 youtube_dl/YoutubeDL.py               | 1 -
 youtube_dl/extractor/appletrailers.py | 1 -
 2 files changed, 2 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 14e92ddcf..ce3abaee3 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1075,7 +1075,6 @@ class YoutubeDL(object):
                                 'requested_formats': formats_info,
                                 'format': rf,
                                 'format_id': rf,
-                                'ext': formats_info[0]['ext'],
                                 'width': formats_info[0].get('width'),
                                 'height': formats_info[0].get('height'),
                                 'resolution': formats_info[0].get('resolution'),
diff --git a/youtube_dl/extractor/appletrailers.py b/youtube_dl/extractor/appletrailers.py
index 70621946d..287f71e07 100644
--- a/youtube_dl/extractor/appletrailers.py
+++ b/youtube_dl/extractor/appletrailers.py
@@ -122,7 +122,6 @@ class AppleTrailersIE(InfoExtractor):
             playlist.append({
                 '_type': 'video',
                 'id': video_id,
-                'title': title,
                 'formats': formats,
                 'title': title,
                 'duration': duration,

From 749f2ca044dc32c6a3ae9e9c55e4a22b091e2635 Mon Sep 17 00:00:00 2001
From: codesparkle <adam.thalhammer@outlook.com>
Date: Fri, 30 Jan 2015 20:35:20 +1100
Subject: [PATCH 0144/6219] Smotri info extractor: removed unreachable code and
 updated old md5 for test video

---
 youtube_dl/extractor/smotri.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/smotri.py b/youtube_dl/extractor/smotri.py
index e94f41362..24746a09a 100644
--- a/youtube_dl/extractor/smotri.py
+++ b/youtube_dl/extractor/smotri.py
@@ -108,7 +108,7 @@ class SmotriIE(InfoExtractor):
         # swf player
         {
             'url': 'http://pics.smotri.com/scrubber_custom8.swf?file=v9188090500',
-            'md5': '4d47034979d9390d14acdf59c4935bc2',
+            'md5': '31099eeb4bc906712c5f40092045108d',
             'info_dict': {
                 'id': 'v9188090500',
                 'ext': 'mp4',
@@ -139,9 +139,6 @@ class SmotriIE(InfoExtractor):
     def _search_meta(self, name, html, display_name=None):
         if display_name is None:
             display_name = name
-        return self._html_search_regex(
-            r'<meta itemprop="%s" content="([^"]+)" />' % re.escape(name),
-            html, display_name, fatal=False)
         return self._html_search_meta(name, html, display_name)
 
     def _real_extract(self, url):

From 83e7a314b4974d6d5674d88b72223d6785f29af7 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 30 Jan 2015 10:48:39 +0100
Subject: [PATCH 0145/6219] dedup AUTHORS

---
 AUTHORS | 1 -
 1 file changed, 1 deletion(-)

diff --git a/AUTHORS b/AUTHORS
index 4a70e7cca..83d8161a9 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -107,5 +107,4 @@ Yen Chi Hsuan
 Enam Mijbah Noor
 David Luhmer
 Shaya Goldberg
-Yen Chi Hsuan
 Paul Hartmann

From a52c6335361d0c86ddc8798d291139faa15de54c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 30 Jan 2015 10:59:07 +0100
Subject: [PATCH 0146/6219] [cinchcast] Wrap overly long lines (#4820)

---
 youtube_dl/extractor/cinchcast.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/cinchcast.py b/youtube_dl/extractor/cinchcast.py
index e23c515d9..562c9bbbb 100644
--- a/youtube_dl/extractor/cinchcast.py
+++ b/youtube_dl/extractor/cinchcast.py
@@ -28,7 +28,10 @@ class CinchcastIE(InfoExtractor):
             item, './{http://developer.longtailvideo.com/trac/}date')
         upload_date = unified_strdate(date_str, day_first=False)
         # duration is present but wrong
-        formats = [{'format_id': 'main', 'url': item.find('./{http://search.yahoo.com/mrss/}content').attrib['url']}]
+        formats = [{
+            'format_id': 'main',
+            'url': item.find('./{http://search.yahoo.com/mrss/}content').attrib['url'],
+        }]
         backup_url = xpath_text(
             item, './{http://developer.longtailvideo.com/trac/}backupContent')
         if backup_url:

From 70fca8d694e6f611384fdcabad748fb7a65235e4 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 30 Jan 2015 10:59:19 +0100
Subject: [PATCH 0147/6219] [youtube] Remove unused import

---
 youtube_dl/extractor/youtube.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index a9e444351..3b3678c6e 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -28,7 +28,6 @@ from ..utils import (
     get_element_by_attribute,
     get_element_by_id,
     int_or_none,
-    js_to_json,
     OnDemandPagedList,
     orderedSet,
     unescapeHTML,

From dc35bfd2d50728d04c1b8213524f6edd2a06513c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 30 Jan 2015 15:52:31 +0100
Subject: [PATCH 0148/6219] [test/helper] Clarify which keys have to be added

---
 test/helper.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/test/helper.py b/test/helper.py
index 27a68091f..805467c82 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -148,9 +148,15 @@ def expect_info_dict(self, got_dict, expected_dict):
                 return "'%s'" % v.replace('\\', '\\\\').replace("'", "\\'").replace('\n', '\\n')
             else:
                 return repr(v)
-        info_dict_str = ''.join(
-            '    %s: %s,\n' % (_repr(k), _repr(v))
-            for k, v in test_info_dict.items())
+        info_dict_str = ''
+        if len(missing_keys) != len(expected_dict):
+            info_dict_str += ''.join(
+                '    %s: %s,\n' % (_repr(k), _repr(v))
+                for k, v in test_info_dict.items() if k not in missing_keys)
+            info_dict_str += '\n'
+        info_dict_str += ''.join(
+            '    %s: %s,\n' % (_repr(k), _repr(test_info_dict[k]))
+            for k in missing_keys)
         write_string(
             '\n\'info_dict\': {\n' + info_dict_str + '}\n', out=sys.stderr)
         self.assertFalse(

From b04b8852710834bfbfdd38722e2cc212eab2a9ec Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 30 Jan 2015 15:53:16 +0100
Subject: [PATCH 0149/6219] [extractor/common] Document all protocol values

---
 youtube_dl/extractor/common.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 478232682..8b4ef3f09 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -89,7 +89,8 @@ class InfoExtractor(object):
                     * player_url SWF Player URL (used for rtmpdump).
                     * protocol   The protocol that will be used for the actual
                                  download, lower-case.
-                                 "http", "https", "rtsp", "rtmp", "m3u8" or so.
+                                 "http", "https", "rtsp", "rtmp", "rtmpe",
+                                 "m3u8", or "m3u8_native".
                     * preference Order number of this format. If this field is
                                  present and not None, the formats get sorted
                                  by this field, regardless of all other values.

From 01c46659c4aa312a6d7c935d3a92d9fa07d66c00 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 30 Jan 2015 15:53:58 +0100
Subject: [PATCH 0150/6219] [washingtonpost] Catch more UUIDs

---
 youtube_dl/extractor/washingtonpost.py | 34 +++++++++++++++++++++++---
 1 file changed, 30 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/washingtonpost.py b/youtube_dl/extractor/washingtonpost.py
index c17bebd6e..72eb010f8 100644
--- a/youtube_dl/extractor/washingtonpost.py
+++ b/youtube_dl/extractor/washingtonpost.py
@@ -1,3 +1,4 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
 import re
@@ -11,9 +12,10 @@ from ..utils import (
 
 class WashingtonPostIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?washingtonpost\.com/.*?/(?P<id>[^/]+)/(?:$|[?#])'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.washingtonpost.com/sf/national/2014/03/22/sinkhole-of-bureaucracy/',
         'info_dict': {
+            'id': 'sinkhole-of-bureaucracy',
             'title': 'Sinkhole of bureaucracy',
         },
         'playlist': [{
@@ -40,15 +42,38 @@ class WashingtonPostIE(InfoExtractor):
                 'upload_date': '20140322',
                 'uploader': 'The Washington Post',
             },
+        }],
+    }, {
+        'url': 'http://www.washingtonpost.com/blogs/wonkblog/wp/2014/12/31/one-airline-figured-out-how-to-make-sure-its-airplanes-never-disappear/',
+        'info_dict': {
+            'id': 'one-airline-figured-out-how-to-make-sure-its-airplanes-never-disappear',
+            'title': 'One airline figured out how to make sure its airplanes never disappear',
+        },
+        'playlist': [{
+            'md5': 'a7c1b5634ba5e57a6a82cdffa5b1e0d0',
+            'info_dict': {
+                'id': '0e4bb54c-9065-11e4-a66f-0ca5037a597d',
+                'ext': 'mp4',
+                'description': 'Washington Post transportation reporter Ashley Halsey III explains why a plane\'s black box needs to be recovered from a crash site instead of having its information streamed in real time throughout the flight.',
+                'upload_date': '20141230',
+                'uploader': 'The Washington Post',
+                'timestamp': 1419974765,
+                'title': 'Why black boxes don’t transmit data in real time',
+            }
         }]
-    }
+    }]
 
     def _real_extract(self, url):
         page_id = self._match_id(url)
         webpage = self._download_webpage(url, page_id)
 
         title = self._og_search_title(webpage)
-        uuids = re.findall(r'data-video-uuid="([^"]+)"', webpage)
+
+        uuids = re.findall(r'''(?x)
+            (?:
+                <div\s+class="posttv-video-embed[^>]*?data-uuid=|
+                data-video-uuid=
+            )"([^"]+)"''', webpage)
         entries = []
         for i, uuid in enumerate(uuids, start=1):
             vinfo_all = self._download_json(
@@ -75,10 +100,11 @@ class WashingtonPostIE(InfoExtractor):
                 'filesize': s.get('fileSize'),
                 'url': s.get('url'),
                 'ext': 'mp4',
+                'preference': -100 if s.get('type') == 'smil' else None,
                 'protocol': {
                     'MP4': 'http',
                     'F4F': 'f4m',
-                }.get(s.get('type'))
+                }.get(s.get('type')),
             } for s in vinfo.get('streams', [])]
             source_media_url = vinfo.get('sourceMediaURL')
             if source_media_url:

From d543bdc351c073621e2f8647fad0f9c8da1cf5b6 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 30 Jan 2015 16:06:55 +0100
Subject: [PATCH 0151/6219] [downloader/f4m] Clarify that we should eventually
 just implement the DRM scheme (#3000)

---
 youtube_dl/downloader/f4m.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index 29de7630d..13055a67e 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -238,13 +238,13 @@ class F4mFD(FileDownloader):
                   doc.findall(_add_ns('drmAdditionalHeaderSet'))):
             # If id attribute is missing it's valid for all media nodes
             # without drmAdditionalHeaderId or drmAdditionalHeaderSetId attribute
-            if not 'id' in e.attrib:
-                self.report_error('Media is DRM protected')
+            if 'id' not in e.attrib:
+                self.report_error('Missing ID in f4m DRM')
         media = list(filter(lambda e: 'drmAdditionalHeaderId' not in e.attrib and
                                       'drmAdditionalHeaderSetId' not in e.attrib,
                             media))
         if not media:
-            self.report_error('Media is DRM protected')
+            self.report_error('Unsupported DRM')
         return media
 
     def real_download(self, filename, info_dict):

From dee3f73787226e3faaf409f8c355b5f8d5dc5632 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 30 Jan 2015 21:54:14 +0600
Subject: [PATCH 0152/6219] [spike] Modernize

---
 youtube_dl/extractor/spike.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/spike.py b/youtube_dl/extractor/spike.py
index a3adf54e3..abba66aee 100644
--- a/youtube_dl/extractor/spike.py
+++ b/youtube_dl/extractor/spike.py
@@ -25,8 +25,7 @@ class SpikeIE(MTVServicesInfoExtractor):
     _MOBILE_TEMPLATE = 'http://m.spike.com/videos/video.rbml?id=%s'
 
     def _real_extract(self, url):
-        mobj = re.search(self._VALID_URL, url)
-        mobile_id = mobj.group('mobile_id')
-        if mobile_id is not None:
+        mobile_id = self._match_id(url)
+        if mobile_id:
             url = 'http://www.spike.com/video-clips/%s' % mobile_id
         return super(SpikeIE, self)._real_extract(url)

From 20b4492c718a5c3cfb2aa493134b9ba8acb0f750 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 30 Jan 2015 21:54:48 +0600
Subject: [PATCH 0153/6219] [spike] Improve _VALID_URL

---
 youtube_dl/extractor/spike.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/spike.py b/youtube_dl/extractor/spike.py
index abba66aee..1af2e9725 100644
--- a/youtube_dl/extractor/spike.py
+++ b/youtube_dl/extractor/spike.py
@@ -7,8 +7,8 @@ from .mtv import MTVServicesInfoExtractor
 
 class SpikeIE(MTVServicesInfoExtractor):
     _VALID_URL = r'''(?x)https?://
-        (www\.spike\.com/(video-clips|episodes)/.+|
-         m\.spike\.com/videos/video.rbml\?id=(?P<mobile_id>[^&]+))
+        (?:www\.spike\.com/(?:video-clips|(?:full-)?episodes)/.+|
+         m\.spike\.com/videos/video\.rbml\?id=(?P<id>[^&]+))
         '''
     _TEST = {
         'url': 'http://www.spike.com/video-clips/lhtu8m/auction-hunters-can-allen-ride-a-hundred-year-old-motorcycle',

From e525d9a3dfb03152e133b8c0ccc8a104289cf5cf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 30 Jan 2015 21:57:59 +0600
Subject: [PATCH 0154/6219] [mtv] Extract subtitles (Closes #4811)

---
 youtube_dl/extractor/mtv.py | 34 +++++++++++++++++++++++++++++++---
 1 file changed, 31 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index 22a726327..7a3b62ebe 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -2,10 +2,11 @@ from __future__ import unicode_literals
 
 import re
 
-from .common import InfoExtractor
+from .subtitles import SubtitlesInfoExtractor
 from ..compat import (
     compat_urllib_parse,
     compat_urllib_request,
+    compat_str,
 )
 from ..utils import (
     ExtractorError,
@@ -22,7 +23,7 @@ def _media_xml_tag(tag):
     return '{http://search.yahoo.com/mrss/}%s' % tag
 
 
-class MTVServicesInfoExtractor(InfoExtractor):
+class MTVServicesInfoExtractor(SubtitlesInfoExtractor):
     _MOBILE_TEMPLATE = None
 
     @staticmethod
@@ -89,6 +90,28 @@ class MTVServicesInfoExtractor(InfoExtractor):
         self._sort_formats(formats)
         return formats
 
+    def _extract_subtitles(self, mdoc, mtvn_id):
+        subtitles = {}
+        FORMATS = {
+            'scc': 'cea-608',
+            'eia-608': 'cea-608',
+            'xml': 'ttml',
+        }
+        subtitles_format = FORMATS.get(
+            self._downloader.params.get('subtitlesformat'), 'ttml')
+        for transcript in mdoc.findall('.//transcript'):
+            if transcript.get('kind') != 'captions':
+                continue
+            lang = transcript.get('srclang')
+            for typographic in transcript.findall('./typographic'):
+                captions_format = typographic.get('format')
+                if captions_format == subtitles_format:
+                    subtitles[lang] = compat_str(typographic.get('src'))
+                    break
+        if self._downloader.params.get('listsubtitles', False):
+            self._list_available_subtitles(mtvn_id, subtitles)
+        return self.extract_subtitles(mtvn_id, subtitles)
+
     def _get_video_info(self, itemdoc):
         uri = itemdoc.find('guid').text
         video_id = self._id_from_uri(uri)
@@ -135,6 +158,7 @@ class MTVServicesInfoExtractor(InfoExtractor):
         return {
             'title': title,
             'formats': self._extract_video_formats(mediagen_doc, mtvn_id),
+            'subtitles': self._extract_subtitles(mediagen_doc, mtvn_id),
             'id': video_id,
             'thumbnail': self._get_thumbnail_url(uri, itemdoc),
             'description': description,
@@ -167,7 +191,11 @@ class MTVServicesInfoExtractor(InfoExtractor):
             mgid = self._search_regex(
                 [r'data-mgid="(.*?)"', r'swfobject.embedSWF\(".*?(mgid:.*?)"'],
                 webpage, 'mgid')
-        return self._get_videos_info(mgid)
+
+        videos_info = self._get_videos_info(mgid)
+        if self._downloader.params.get('listsubtitles', False):
+            return
+        return videos_info
 
 
 class MTVServicesEmbeddedIE(MTVServicesInfoExtractor):

From ef1269fb07833a06a15d0601e54d58e1678bc06a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 30 Jan 2015 22:42:11 +0600
Subject: [PATCH 0155/6219] [drtv] Improve _VALID_URL (#4814)

---
 youtube_dl/extractor/drtv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/drtv.py b/youtube_dl/extractor/drtv.py
index 510ef04b0..8d113a5e8 100644
--- a/youtube_dl/extractor/drtv.py
+++ b/youtube_dl/extractor/drtv.py
@@ -6,7 +6,7 @@ from ..utils import parse_iso8601
 
 
 class DRTVIE(SubtitlesInfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?dr\.dk/tv/se/(?:[^/]+/)+(?P<id>[\da-z-]+)(?:[/#?]|$)'
+    _VALID_URL = r'https?://(?:www\.)?dr\.dk/tv/se/(?:[^/]+/)*(?P<id>[\da-z-]+)(?:[/#?]|$)'
 
     _TEST = {
         'url': 'http://www.dr.dk/tv/se/partiets-mand/partiets-mand-7-8',

From 2c58674e0ede5d656cb559a18a3b86ffbb21b3a0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 30 Jan 2015 22:46:53 +0600
Subject: [PATCH 0156/6219] [nhl] Improve _VALID_URL (#4805)

---
 youtube_dl/extractor/nhl.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nhl.py b/youtube_dl/extractor/nhl.py
index d3a4fc513..cb279074e 100644
--- a/youtube_dl/extractor/nhl.py
+++ b/youtube_dl/extractor/nhl.py
@@ -54,7 +54,7 @@ class NHLBaseInfoExtractor(InfoExtractor):
 
 class NHLIE(NHLBaseInfoExtractor):
     IE_NAME = 'nhl.com'
-    _VALID_URL = r'https?://video(?P<team>\.[^.]*)?\.nhl\.com/videocenter/console(?:\?(?:.*?[?&])?)id=(?P<id>[-0-9a-zA-Z]+)'
+    _VALID_URL = r'https?://video(?P<team>\.[^.]*)?\.nhl\.com/videocenter/(?:console)?(?:\?(?:.*?[?&])?)id=(?P<id>[-0-9a-zA-Z]+)'
 
     _TESTS = [{
         'url': 'http://video.canucks.nhl.com/videocenter/console?catid=6?id=453614',
@@ -92,6 +92,9 @@ class NHLIE(NHLBaseInfoExtractor):
     }, {
         'url': 'http://video.flames.nhl.com/videocenter/console?id=630616',
         'only_matching': True,
+    }, {
+        'url': 'http://video.nhl.com/videocenter/?id=736722',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From e4c17d7274235ec880857d7691bc7c818e2c7a01 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 30 Jan 2015 23:12:27 +0600
Subject: [PATCH 0157/6219] [nhl:news] Add extractor (Closes #4805)

---
 youtube_dl/extractor/__init__.py |  6 ++++-
 youtube_dl/extractor/nhl.py      | 41 +++++++++++++++++++++++++++-----
 2 files changed, 40 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index d3f51d8c4..9c14b096f 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -294,7 +294,11 @@ from .nextmedia import (
 )
 from .nfb import NFBIE
 from .nfl import NFLIE
-from .nhl import NHLIE, NHLVideocenterIE
+from .nhl import (
+    NHLIE,
+    NHLNewsIE,
+    NHLVideocenterIE,
+)
 from .niconico import NiconicoIE, NiconicoPlaylistIE
 from .ninegag import NineGagIE
 from .noco import NocoIE
diff --git a/youtube_dl/extractor/nhl.py b/youtube_dl/extractor/nhl.py
index cb279074e..407465998 100644
--- a/youtube_dl/extractor/nhl.py
+++ b/youtube_dl/extractor/nhl.py
@@ -20,6 +20,12 @@ class NHLBaseInfoExtractor(InfoExtractor):
     def _fix_json(json_string):
         return json_string.replace('\\\'', '\'')
 
+    def _real_extract_video(self, video_id):
+        json_url = 'http://video.nhl.com/videocenter/servlets/playlist?ids=%s&format=json' % video_id
+        data = self._download_json(
+            json_url, video_id, transform_source=self._fix_json)
+        return self._extract_video(data[0])
+
     def _extract_video(self, info):
         video_id = info['id']
         self.report_extraction(video_id)
@@ -98,12 +104,35 @@ class NHLIE(NHLBaseInfoExtractor):
     }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        json_url = 'http://video.nhl.com/videocenter/servlets/playlist?ids=%s&format=json' % video_id
-        data = self._download_json(
-            json_url, video_id, transform_source=self._fix_json)
-        return self._extract_video(data[0])
+        video_id = self._match_id(url)
+        return self._real_extract_video(video_id)
+
+
+class NHLNewsIE(NHLBaseInfoExtractor):
+    IE_NAME = 'nhl.com:news'
+    IE_DESC = 'NHL news'
+    _VALID_URL = r'https?://(?:www\.)?nhl\.com/ice/news\.html?(?:\?(?:.*?[?&])?)id=(?P<id>[-0-9a-zA-Z]+)'
+
+    _TEST = {
+        'url': 'http://www.nhl.com/ice/news.htm?id=750727',
+        'md5': '4b3d1262e177687a3009937bd9ec0be8',
+        'info_dict': {
+            'id': '736722',
+            'ext': 'mp4',
+            'title': 'Cal Clutterbuck has been fined $2,000',
+            'description': 'md5:45fe547d30edab88b23e0dd0ab1ed9e6',
+            'duration': 37,
+            'upload_date': '20150128',
+        },
+    }
+
+    def _real_extract(self, url):
+        news_id = self._match_id(url)
+        webpage = self._download_webpage(url, news_id)
+        video_id = self._search_regex(
+            [r'pVid(\d+)', r"nlid\s*:\s*'(\d+)'"],
+            webpage, 'video id')
+        return self._real_extract_video(video_id)
 
 
 class NHLVideocenterIE(NHLBaseInfoExtractor):

From b80505a40938444c6f7b0b80c445cc0726c39bb7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 30 Jan 2015 23:21:44 +0600
Subject: [PATCH 0158/6219] [mixcloud] Fix extraction (Closes #4784)

---
 youtube_dl/extractor/mixcloud.py | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dl/extractor/mixcloud.py
index 07d194562..162046fe6 100644
--- a/youtube_dl/extractor/mixcloud.py
+++ b/youtube_dl/extractor/mixcloud.py
@@ -10,6 +10,7 @@ from ..utils import (
     ExtractorError,
     HEADRequest,
     int_or_none,
+    str_to_int,
     parse_iso8601,
 )
 
@@ -85,15 +86,17 @@ class MixcloudIE(InfoExtractor):
         uploader_id = self._search_regex(
             r'\s+"profile": "([^"]+)",', webpage, 'uploader id', fatal=False)
         description = self._og_search_description(webpage)
-        like_count = int_or_none(self._search_regex(
-            r'<meta itemprop="interactionCount" content="UserLikes:([0-9]+)"',
+        like_count = str_to_int(self._search_regex(
+            [r'<meta itemprop="interactionCount" content="UserLikes:([0-9]+)"',
+             r'/favorites/?">([0-9]+)<'],
             webpage, 'like count', fatal=False))
-        view_count = int_or_none(self._search_regex(
-            r'<meta itemprop="interactionCount" content="UserPlays:([0-9]+)"',
+        view_count = str_to_int(self._search_regex(
+            [r'<meta itemprop="interactionCount" content="UserPlays:([0-9]+)"',
+             r'/listeners/?">([0-9,.]+)</a>'],
             webpage, 'play count', fatal=False))
         timestamp = parse_iso8601(self._search_regex(
             r'<time itemprop="dateCreated" datetime="([^"]+)">',
-            webpage, 'upload date'))
+            webpage, 'upload date', default=None))
 
         return {
             'id': track_id,

From 439d9be27dc3cf811f3337e40c3222bc8d2ed339 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 30 Jan 2015 23:21:58 +0600
Subject: [PATCH 0159/6219] [mixcloud] Remove unused import

---
 youtube_dl/extractor/mixcloud.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dl/extractor/mixcloud.py
index 162046fe6..141bb463e 100644
--- a/youtube_dl/extractor/mixcloud.py
+++ b/youtube_dl/extractor/mixcloud.py
@@ -9,7 +9,6 @@ from ..compat import (
 from ..utils import (
     ExtractorError,
     HEADRequest,
-    int_or_none,
     str_to_int,
     parse_iso8601,
 )

From 0692ef86ef85da69074b00b5525c13bfde9159de Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 30 Jan 2015 23:47:09 +0600
Subject: [PATCH 0160/6219] [bbccouk] Improve _VALID_URL

---
 youtube_dl/extractor/bbccouk.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbccouk.py b/youtube_dl/extractor/bbccouk.py
index 1cf48fe0d..126c8824c 100644
--- a/youtube_dl/extractor/bbccouk.py
+++ b/youtube_dl/extractor/bbccouk.py
@@ -10,7 +10,7 @@ from ..compat import compat_HTTPError
 class BBCCoUkIE(SubtitlesInfoExtractor):
     IE_NAME = 'bbc.co.uk'
     IE_DESC = 'BBC iPlayer'
-    _VALID_URL = r'https?://(?:www\.)?bbc\.co\.uk/(?:(?:(?:programmes|iplayer/(?:episode|playlist))/)|music/clips[/#])(?P<id>[\da-z]{8})'
+    _VALID_URL = r'https?://(?:www\.)?bbc\.co\.uk/(?:(?:(?:programmes|iplayer(?:/[^/]+)?/(?:episode|playlist))/)|music/clips[/#])(?P<id>[\da-z]{8})'
 
     _TESTS = [
         {
@@ -118,6 +118,9 @@ class BBCCoUkIE(SubtitlesInfoExtractor):
         }, {
             'url': 'http://www.bbc.co.uk/music/clips#p02frcc3',
             'only_matching': True,
+        }, {
+            'url': 'http://www.bbc.co.uk/iplayer/cbeebies/episode/b0480276/bing-14-atchoo',
+            'only_matching': True,
         }
     ]
 

From 91755ee384ddee314ba40748a44ad1289df8b86c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 30 Jan 2015 19:43:46 +0100
Subject: [PATCH 0161/6219] [comedycentral:shows] Generate better IDs and add a
 test for that

---
 youtube_dl/extractor/comedycentral.py | 34 ++++++++++++++++++++++++++-
 1 file changed, 33 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/comedycentral.py b/youtube_dl/extractor/comedycentral.py
index 15ca361f0..f9256dcf4 100644
--- a/youtube_dl/extractor/comedycentral.py
+++ b/youtube_dl/extractor/comedycentral.py
@@ -49,7 +49,9 @@ class ComedyCentralShowsIE(MTVServicesInfoExtractor):
                               |(watch/(?P<date>[^/]*)/(?P<tdstitle>.*))
                           )|
                           (?P<interview>
-                              extended-interviews/(?P<interID>[0-9a-z]+)/(?:playlist_tds_extended_)?(?P<interview_title>.*?)(/.*?)?)))
+                              extended-interviews/(?P<interID>[0-9a-z]+)/
+                              (?:playlist_tds_extended_)?(?P<interview_title>[^/?#]*?)
+                              (?:/[^/?#]?|[?#]|$))))
                      '''
     _TESTS = [{
         'url': 'http://thedailyshow.cc.com/watch/thu-december-13-2012/kristen-stewart',
@@ -62,6 +64,35 @@ class ComedyCentralShowsIE(MTVServicesInfoExtractor):
             'uploader': 'thedailyshow',
             'title': 'thedailyshow kristen-stewart part 1',
         }
+    }, {
+        'url': 'http://thedailyshow.cc.com/extended-interviews/b6364d/sarah-chayes-extended-interview',
+        'info_dict': {
+            'id': 'sarah-chayes-extended-interview',
+            'description': 'Carnegie Endowment Senior Associate Sarah Chayes discusses how corrupt institutions function throughout the world in her book "Thieves of State: Why Corruption Threatens Global Security."',
+            'title': 'thedailyshow Sarah Chayes Extended Interview',
+        },
+        'playlist': [{
+            'info_dict': {
+                'id': '0baad492-cbec-4ec1-9e50-ad91c291127f',
+                'ext': 'mp4',
+                'upload_date': '20150129',
+                'description': 'Carnegie Endowment Senior Associate Sarah Chayes discusses how corrupt institutions function throughout the world in her book "Thieves of State: Why Corruption Threatens Global Security."',
+                'uploader': 'thedailyshow',
+                'title': 'thedailyshow sarah-chayes-extended-interview part 1',
+            }
+         }, {
+            'info_dict': {
+                'id': '1e4fb91b-8ce7-4277-bd7c-98c9f1bbd283',
+                'ext': 'mp4',
+                'upload_date': '20150129',
+                'description': 'Carnegie Endowment Senior Associate Sarah Chayes discusses how corrupt institutions function throughout the world in her book "Thieves of State: Why Corruption Threatens Global Security."',
+                'uploader': 'thedailyshow',
+                'title': 'thedailyshow sarah-chayes-extended-interview part 2',
+            }
+        }],
+        'params': {
+            'skip_download': True,
+        },
     }, {
         'url': 'http://thedailyshow.cc.com/extended-interviews/xm3fnq/andrew-napolitano-extended-interview',
         'only_matching': True,
@@ -230,6 +261,7 @@ class ComedyCentralShowsIE(MTVServicesInfoExtractor):
 
         return {
             '_type': 'playlist',
+            'id': epTitle,
             'entries': entries,
             'title': show_name + ' ' + title,
             'description': description,

From 6d593c3276da6a0a1ad738c779b87a1998c5b06b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 31 Jan 2015 03:50:11 +0600
Subject: [PATCH 0162/6219] [YoutubeDL] Fix video+audio format_id (Closes
 #4824)

---
 youtube_dl/YoutubeDL.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index ce3abaee3..e5a96cad5 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1074,7 +1074,8 @@ class YoutubeDL(object):
                             selected_format = {
                                 'requested_formats': formats_info,
                                 'format': rf,
-                                'format_id': rf,
+                                'format_id': '%s+%s' % (formats_info[0].get('format_id'),
+                                                        formats_info[1].get('format_id')),
                                 'width': formats_info[0].get('width'),
                                 'height': formats_info[0].get('height'),
                                 'resolution': formats_info[0].get('resolution'),

From cc57bd33a803a061ac0bbe971088fb1ef78be82b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 30 Jan 2015 23:05:06 +0100
Subject: [PATCH 0163/6219] [rtve] Fix extraction

Skip live stream test, we can't use the f4m manifest yet
---
 youtube_dl/extractor/rtve.py | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/rtve.py b/youtube_dl/extractor/rtve.py
index 0ce22d60c..9e8688fbf 100644
--- a/youtube_dl/extractor/rtve.py
+++ b/youtube_dl/extractor/rtve.py
@@ -74,7 +74,8 @@ class RTVEALaCartaIE(InfoExtractor):
             'id': '1694255',
             'ext': 'flv',
             'title': 'TODO',
-        }
+        },
+        'skip': 'The f4m manifest can\'t be used yet',
     }]
 
     def _real_extract(self, url):
@@ -86,6 +87,18 @@ class RTVEALaCartaIE(InfoExtractor):
         png_url = 'http://www.rtve.es/ztnr/movil/thumbnail/default/videos/%s.png' % video_id
         png = self._download_webpage(png_url, video_id, 'Downloading url information')
         video_url = _decrypt_url(png)
+        if not video_url.endswith('.f4m'):
+            auth_url = video_url.replace(
+                'resources/', 'auth/resources/'
+            ).replace('.net.rtve', '.multimedia.cdn.rtve')
+            video_path = self._download_webpage(
+                auth_url, video_id, 'Getting video url')
+            # Use mvod.akcdn instead of flash.akamaihd.multimedia.cdn to get
+            # the right Content-Length header and the mp4 format
+            video_url = (
+                'http://mvod.akcdn.rtve.es/{0}&v=2.6.8'
+                '&fp=MAC%2016,0,0,296&r=MRUGG&g=OEOJWFXNFGCP'.format(video_path)
+            )
 
         return {
             'id': video_id,

From cd596028d6c873ac7e9a61902bbe8f86d517b5b9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 30 Jan 2015 23:12:53 +0100
Subject: [PATCH 0164/6219] [rtve] Recognize mobile urls (fixes #4823)

---
 youtube_dl/extractor/rtve.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/rtve.py b/youtube_dl/extractor/rtve.py
index 9e8688fbf..3469d9578 100644
--- a/youtube_dl/extractor/rtve.py
+++ b/youtube_dl/extractor/rtve.py
@@ -57,7 +57,7 @@ def _decrypt_url(png):
 class RTVEALaCartaIE(InfoExtractor):
     IE_NAME = 'rtve.es:alacarta'
     IE_DESC = 'RTVE a la carta'
-    _VALID_URL = r'http://www\.rtve\.es/alacarta/videos/[^/]+/[^/]+/(?P<id>\d+)'
+    _VALID_URL = r'http://www\.rtve\.es/(m/)?alacarta/videos/[^/]+/[^/]+/(?P<id>\d+)'
 
     _TESTS = [{
         'url': 'http://www.rtve.es/alacarta/videos/balonmano/o-swiss-cup-masculina-final-espana-suecia/2491869/',
@@ -76,6 +76,9 @@ class RTVEALaCartaIE(InfoExtractor):
             'title': 'TODO',
         },
         'skip': 'The f4m manifest can\'t be used yet',
+    }, {
+        'url': 'http://www.rtve.es/m/alacarta/videos/cuentame-como-paso/cuentame-como-paso-t16-ultimo-minuto-nuestra-vida-capitulo-276/2969138/?media=tve',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From e41b1f73854d8a422235a3b92ca4255affbcd39e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 31 Jan 2015 10:51:39 +0100
Subject: [PATCH 0165/6219] Fix flake8 errors

---
 youtube_dl/downloader/f4m.py          |  2 +-
 youtube_dl/extractor/comedycentral.py | 41 ++++++++++++++-------------
 youtube_dl/extractor/spike.py         |  2 --
 3 files changed, 23 insertions(+), 22 deletions(-)

diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index 13055a67e..0e7a1c200 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -231,7 +231,7 @@ class F4mFD(FileDownloader):
     """
 
     def _get_unencrypted_media(self, doc):
-        media=doc.findall(_add_ns('media'))
+        media = doc.findall(_add_ns('media'))
         if not media:
             self.report_error('No media found')
         for e in (doc.findall(_add_ns('drmAdditionalHeader')) +
diff --git a/youtube_dl/extractor/comedycentral.py b/youtube_dl/extractor/comedycentral.py
index f9256dcf4..b24538981 100644
--- a/youtube_dl/extractor/comedycentral.py
+++ b/youtube_dl/extractor/comedycentral.py
@@ -71,25 +71,28 @@ class ComedyCentralShowsIE(MTVServicesInfoExtractor):
             'description': 'Carnegie Endowment Senior Associate Sarah Chayes discusses how corrupt institutions function throughout the world in her book "Thieves of State: Why Corruption Threatens Global Security."',
             'title': 'thedailyshow Sarah Chayes Extended Interview',
         },
-        'playlist': [{
-            'info_dict': {
-                'id': '0baad492-cbec-4ec1-9e50-ad91c291127f',
-                'ext': 'mp4',
-                'upload_date': '20150129',
-                'description': 'Carnegie Endowment Senior Associate Sarah Chayes discusses how corrupt institutions function throughout the world in her book "Thieves of State: Why Corruption Threatens Global Security."',
-                'uploader': 'thedailyshow',
-                'title': 'thedailyshow sarah-chayes-extended-interview part 1',
-            }
-         }, {
-            'info_dict': {
-                'id': '1e4fb91b-8ce7-4277-bd7c-98c9f1bbd283',
-                'ext': 'mp4',
-                'upload_date': '20150129',
-                'description': 'Carnegie Endowment Senior Associate Sarah Chayes discusses how corrupt institutions function throughout the world in her book "Thieves of State: Why Corruption Threatens Global Security."',
-                'uploader': 'thedailyshow',
-                'title': 'thedailyshow sarah-chayes-extended-interview part 2',
-            }
-        }],
+        'playlist': [
+            {
+                'info_dict': {
+                    'id': '0baad492-cbec-4ec1-9e50-ad91c291127f',
+                    'ext': 'mp4',
+                    'upload_date': '20150129',
+                    'description': 'Carnegie Endowment Senior Associate Sarah Chayes discusses how corrupt institutions function throughout the world in her book "Thieves of State: Why Corruption Threatens Global Security."',
+                    'uploader': 'thedailyshow',
+                    'title': 'thedailyshow sarah-chayes-extended-interview part 1',
+                },
+            },
+            {
+                'info_dict': {
+                    'id': '1e4fb91b-8ce7-4277-bd7c-98c9f1bbd283',
+                    'ext': 'mp4',
+                    'upload_date': '20150129',
+                    'description': 'Carnegie Endowment Senior Associate Sarah Chayes discusses how corrupt institutions function throughout the world in her book "Thieves of State: Why Corruption Threatens Global Security."',
+                    'uploader': 'thedailyshow',
+                    'title': 'thedailyshow sarah-chayes-extended-interview part 2',
+                },
+            },
+        ],
         'params': {
             'skip_download': True,
         },
diff --git a/youtube_dl/extractor/spike.py b/youtube_dl/extractor/spike.py
index 1af2e9725..e529bb55c 100644
--- a/youtube_dl/extractor/spike.py
+++ b/youtube_dl/extractor/spike.py
@@ -1,7 +1,5 @@
 from __future__ import unicode_literals
 
-import re
-
 from .mtv import MTVServicesInfoExtractor
 
 

From e77d2975afda3750104c90b7c0012c02a2a00b10 Mon Sep 17 00:00:00 2001
From: light94 <mishra.rahul1712@gmail.com>
Date: Sun, 1 Feb 2015 00:10:58 +0530
Subject: [PATCH 0166/6219] Handling Connection Reset by Peer Error

---
 youtube_dl/downloader/http.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index 8a1d578d5..9b023c5c1 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -3,6 +3,9 @@ from __future__ import unicode_literals
 import os
 import time
 
+from socket import error as SocketError
+import errno
+
 from .common import FileDownloader
 from ..compat import (
     compat_urllib_request,
@@ -99,6 +102,11 @@ class HttpFD(FileDownloader):
                             resume_len = 0
                             open_mode = 'wb'
                             break
+
+            except SocketError as e:
+                if e.errno != errno.ECONNRESET:
+                    raise # Not error we are looking for
+                pass
             # Retry
             count += 1
             if count <= retries:

From 1ee94db2d03770c0280047c974f0e6886d2a753f Mon Sep 17 00:00:00 2001
From: kinetoskombi <filipeamiranda@gmail.com>
Date: Sat, 31 Jan 2015 20:07:43 -0200
Subject: [PATCH 0167/6219] [globo] Fix error on some globo videos

---
 youtube_dl/extractor/globo.py | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/globo.py b/youtube_dl/extractor/globo.py
index 6949a57c7..d936be384 100644
--- a/youtube_dl/extractor/globo.py
+++ b/youtube_dl/extractor/globo.py
@@ -70,6 +70,19 @@ class GloboIE(InfoExtractor):
                 'like_count': int,
             }
         },
+        {
+            'url': 'http://globotv.globo.com/canal-brasil/sangue-latino/t/todos-os-videos/v/ator-e-diretor-argentino-ricado-darin-fala-sobre-utopias-e-suas-perdas/3928201/',
+            'md5': 'c1defca721ce25b2354e927d3e4b3dec',
+            'info_dict': {
+                'id': '3928201',
+                'ext': 'mp4',
+                'title': 'Ator e diretor argentino, Ricado Darín fala sobre utopias e suas perdas',
+                'duration': 1472.906,
+                'uploader': 'Canal Brasil',
+                'uploader_id': 705,
+                'like_count': int,
+            }
+        },
     ]
 
     class MD5():
@@ -355,7 +368,9 @@ class GloboIE(InfoExtractor):
 
         for resource in video['resources']:
             resource_id = resource.get('_id')
-            if not resource_id:
+            resource_height = resource.get('height')
+
+            if not (resource_id or resource_height):
                 continue
 
             security = self._download_json(
@@ -384,7 +399,7 @@ class GloboIE(InfoExtractor):
             formats.append({
                 'url': '%s?h=%s&k=%s' % (resource['url'], signed_hash, 'flash'),
                 'format_id': resource_id,
-                'height': resource['height']
+                'height': resource_height
             })
 
         self._sort_formats(formats)

From 8c72beb25eaa1f1960a3b50295bcdcfd50c450af Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 1 Feb 2015 04:36:24 +0600
Subject: [PATCH 0168/6219] [globo] Properly extract m3u8 formats (#4346 #4832)

---
 youtube_dl/extractor/globo.py | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/globo.py b/youtube_dl/extractor/globo.py
index d936be384..29638a194 100644
--- a/youtube_dl/extractor/globo.py
+++ b/youtube_dl/extractor/globo.py
@@ -368,9 +368,7 @@ class GloboIE(InfoExtractor):
 
         for resource in video['resources']:
             resource_id = resource.get('_id')
-            resource_height = resource.get('height')
-
-            if not (resource_id or resource_height):
+            if not resource_id:
                 continue
 
             security = self._download_json(
@@ -396,11 +394,16 @@ class GloboIE(InfoExtractor):
             signed_md5 = self.MD5.b64_md5(received_md5 + compat_str(sign_time) + padding)
             signed_hash = hash_code + compat_str(received_time) + received_random + compat_str(sign_time) + padding + signed_md5
 
-            formats.append({
-                'url': '%s?h=%s&k=%s' % (resource['url'], signed_hash, 'flash'),
-                'format_id': resource_id,
-                'height': resource_height
-            })
+            resource_url = resource['url']
+            signed_url = '%s?h=%s&k=%s' % (resource_url, signed_hash, 'flash')
+            if resource_id.endswith('m3u8') or resource_url.endswith('.m3u8'):
+                formats.extend(self._extract_m3u8_formats(signed_url, resource_id, 'mp4'))
+            else:
+                formats.append({
+                    'url': signed_url,
+                    'format_id': resource_id,
+                    'height': resource.get('height'),
+                })
 
         self._sort_formats(formats)
 

From 8f9312c38779f4e0746c01ef9a8a93ab4767699b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 11:30:56 +0100
Subject: [PATCH 0169/6219] Appease pyflakes8-3

---
 youtube_dl/YoutubeDL.py |  3 ++-
 youtube_dl/compat.py    | 37 +++++++++++++++++++++----------------
 youtube_dl/utils.py     |  7 ++++---
 3 files changed, 27 insertions(+), 20 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index e5a96cad5..c18ce9660 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -25,6 +25,7 @@ if os.name == 'nt':
     import ctypes
 
 from .compat import (
+    compat_basestring,
     compat_cookiejar,
     compat_expanduser,
     compat_http_client,
@@ -1558,7 +1559,7 @@ class YoutubeDL(object):
         # urllib chokes on URLs with non-ASCII characters (see http://bugs.python.org/issue3991)
         # To work around aforementioned issue we will replace request's original URL with
         # percent-encoded one
-        req_is_string = isinstance(req, basestring if sys.version_info < (3, 0) else compat_str)
+        req_is_string = isinstance(req, compat_basestring)
         url = req if req_is_string else req.get_full_url()
         url_escaped = escape_url(url)
 
diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 497ca52de..ec685d50e 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -114,6 +114,26 @@ except ImportError:
             string += pct_sequence.decode(encoding, errors)
         return string
 
+try:
+    compat_str = unicode  # Python 2
+except NameError:
+    compat_str = str
+
+try:
+    compat_basestr = basestring  # Python 2
+except NameError:
+    compat_basestr = str
+
+try:
+    compat_chr = unichr  # Python 2
+except NameError:
+    compat_chr = chr
+
+try:
+    from xml.etree.ElementTree import ParseError as compat_xml_parse_error
+except ImportError:  # Python 2.6
+    from xml.parsers.expat import ExpatError as compat_xml_parse_error
+
 
 try:
     from urllib.parse import parse_qs as compat_parse_qs
@@ -123,7 +143,7 @@ except ImportError:  # Python 2
 
     def _parse_qsl(qs, keep_blank_values=False, strict_parsing=False,
                    encoding='utf-8', errors='replace'):
-        qs, _coerce_result = qs, unicode
+        qs, _coerce_result = qs, compat_str
         pairs = [s2 for s1 in qs.split('&') for s2 in s1.split(';')]
         r = []
         for name_value in pairs:
@@ -162,21 +182,6 @@ except ImportError:  # Python 2
                 parsed_result[name] = [value]
         return parsed_result
 
-try:
-    compat_str = unicode  # Python 2
-except NameError:
-    compat_str = str
-
-try:
-    compat_chr = unichr  # Python 2
-except NameError:
-    compat_chr = chr
-
-try:
-    from xml.etree.ElementTree import ParseError as compat_xml_parse_error
-except ImportError:  # Python 2.6
-    from xml.parsers.expat import ExpatError as compat_xml_parse_error
-
 try:
     from shlex import quote as shlex_quote
 except ImportError:  # Python < 3.3
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index a4c9813ec..4ade0554e 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -32,6 +32,7 @@ import xml.etree.ElementTree
 import zlib
 
 from .compat import (
+    compat_basestring,
     compat_chr,
     compat_getenv,
     compat_html_entities,
@@ -140,7 +141,7 @@ else:
     def find_xpath_attr(node, xpath, key, val):
         # Here comes the crazy part: In 2.6, if the xpath is a unicode,
         # .//node does not match if a node is a direct child of . !
-        if isinstance(xpath, unicode):
+        if isinstance(xpath, compat_str):
             xpath = xpath.encode('ascii')
 
         for f in node.findall(xpath):
@@ -1262,7 +1263,7 @@ def float_or_none(v, scale=1, invscale=1, default=None):
 
 
 def parse_duration(s):
-    if not isinstance(s, basestring if sys.version_info < (3, 0) else compat_str):
+    if not isinstance(s, compat_basestring):
         return None
 
     s = s.strip()
@@ -1426,7 +1427,7 @@ def uppercase_escape(s):
 
 def escape_rfc3986(s):
     """Escape non-ASCII characters as suggested by RFC 3986"""
-    if sys.version_info < (3, 0) and isinstance(s, unicode):
+    if sys.version_info < (3, 0) and isinstance(s, compat_str):
         s = s.encode('utf-8')
     return compat_urllib_parse.quote(s, b"%/;:@&=+$,!~*'()?#[]")
 

From 0196149c5bf6b72c1d268a4beb2c50e4ecb063d7 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 11:36:59 +0100
Subject: [PATCH 0170/6219] [compat] Correct compat_basestring definition

---
 youtube_dl/compat.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index ec685d50e..e989cdbbd 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -120,9 +120,9 @@ except NameError:
     compat_str = str
 
 try:
-    compat_basestr = basestring  # Python 2
+    compat_basestring = basestring  # Python 2
 except NameError:
-    compat_basestr = str
+    compat_basestring = str
 
 try:
     compat_chr = unichr  # Python 2
@@ -367,6 +367,7 @@ def workaround_optparse_bug9161():
 
 __all__ = [
     'compat_HTTPError',
+    'compat_basestring',
     'compat_chr',
     'compat_cookiejar',
     'compat_expanduser',

From 9158b2b301f83a1cec13a6340f354646434c69c1 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 11:58:37 +0100
Subject: [PATCH 0171/6219] [mpora] Modernize

---
 youtube_dl/extractor/mpora.py | 16 ++++++----------
 1 file changed, 6 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/mpora.py b/youtube_dl/extractor/mpora.py
index 88c9501cd..6db3c67a5 100644
--- a/youtube_dl/extractor/mpora.py
+++ b/youtube_dl/extractor/mpora.py
@@ -1,21 +1,19 @@
 from __future__ import unicode_literals
 
-import json
-import re
-
 from .common import InfoExtractor
 from ..utils import int_or_none
 
 
 class MporaIE(InfoExtractor):
-    _VALID_URL = r'^https?://(www\.)?mpora\.(?:com|de)/videos/(?P<id>[^?#/]+)'
+    _VALID_URL = r'https?://(www\.)?mpora\.(?:com|de)/videos/(?P<id>[^?#/]+)'
     IE_NAME = 'MPORA'
 
     _TEST = {
         'url': 'http://mpora.de/videos/AAdo8okx4wiz/embed?locale=de',
-        'file': 'AAdo8okx4wiz.mp4',
         'md5': 'a7a228473eedd3be741397cf452932eb',
         'info_dict': {
+            'id': 'AAdo8okx4wiz',
+            'ext': 'mp4',
             'title': 'Katy Curd -  Winter in the Forest',
             'duration': 416,
             'uploader': 'Peter Newman Media',
@@ -23,14 +21,12 @@ class MporaIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        m = re.match(self._VALID_URL, url)
-        video_id = m.group('id')
-
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
+
         data_json = self._search_regex(
             r"new FM\.Player\('[^']+',\s*(\{.*?)\).player;", webpage, 'json')
-
-        data = json.loads(data_json)
+        data = self._parse_json(data_json, video_id)
 
         uploader = data['info_overlay'].get('username')
         duration = data['video']['duration'] // 1000

From 29aef5a33c46ceaa5884a3f074fbb5af4fa44f32 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 12:00:47 +0100
Subject: [PATCH 0172/6219] [ard] Remove deleted video test case

---
 youtube_dl/extractor/ard.py | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/ard.py b/youtube_dl/extractor/ard.py
index 967bd865c..783b53e23 100644
--- a/youtube_dl/extractor/ard.py
+++ b/youtube_dl/extractor/ard.py
@@ -23,13 +23,7 @@ class ARDMediathekIE(InfoExtractor):
 
     _TESTS = [{
         'url': 'http://mediathek.daserste.de/sendungen_a-z/328454_anne-will/22429276_vertrauen-ist-gut-spionieren-ist-besser-geht',
-        'file': '22429276.mp4',
-        'md5': '469751912f1de0816a9fc9df8336476c',
-        'info_dict': {
-            'title': 'Vertrauen ist gut, Spionieren ist besser - Geht so deutsch-amerikanische Freundschaft?',
-            'description': 'Das Erste Mediathek [ARD]: Vertrauen ist gut, Spionieren ist besser - Geht so deutsch-amerikanische Freundschaft?, Anne Will, Über die Spionage-Affäre diskutieren Clemens Binninger, Katrin Göring-Eckardt, Georg Mascolo, Andrew B. Denison und Constanze Kurz.. Das Video zur Sendung Anne Will am Mittwoch, 16.07.2014',
-        },
-        'skip': 'Blocked outside of Germany',
+        'only_matching': True,
     }, {
         'url': 'http://www.ardmediathek.de/tv/Tatort/Das-Wunder-von-Wolbeck-Video-tgl-ab-20/Das-Erste/Video?documentId=22490580&bcastId=602916',
         'info_dict': {

From ca0f500ecfde0a0ada7002348e8b04ac5c79d4b4 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 12:08:21 +0100
Subject: [PATCH 0173/6219] [mtv] Modernize and clean up test

---
 youtube_dl/extractor/mtv.py | 34 +++++++++++++---------------------
 1 file changed, 13 insertions(+), 21 deletions(-)

diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index 7a3b62ebe..bc7f49ebb 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -79,12 +79,15 @@ class MTVServicesInfoExtractor(SubtitlesInfoExtractor):
             try:
                 _, _, ext = rendition.attrib['type'].partition('/')
                 rtmp_video_url = rendition.find('./src').text
-                formats.append({'ext': ext,
-                                'url': self._transform_rtmp_url(rtmp_video_url),
-                                'format_id': rendition.get('bitrate'),
-                                'width': int(rendition.get('width')),
-                                'height': int(rendition.get('height')),
-                                })
+                if rtmp_video_url.endswith('siteunavail.png'):
+                    continue
+                formats.append({
+                    'ext': ext,
+                    'url': self._transform_rtmp_url(rtmp_video_url),
+                    'format_id': rendition.get('bitrate'),
+                    'width': int(rendition.get('width')),
+                    'height': int(rendition.get('height')),
+                })
             except (KeyError, TypeError):
                 raise ExtractorError('Invalid rendition field.')
         self._sort_formats(formats)
@@ -240,25 +243,14 @@ class MTVIE(MTVServicesInfoExtractor):
     _TESTS = [
         {
             'url': 'http://www.mtv.com/videos/misc/853555/ours-vh1-storytellers.jhtml',
-            'file': '853555.mp4',
             'md5': '850f3f143316b1e71fa56a4edfd6e0f8',
             'info_dict': {
+                'id': '853555',
+                'ext': 'mp4',
                 'title': 'Taylor Swift - "Ours (VH1 Storytellers)"',
                 'description': 'Album: Taylor Swift performs "Ours" for VH1 Storytellers at Harvey Mudd College.',
             },
         },
-        {
-            'add_ie': ['Vevo'],
-            'url': 'http://www.mtv.com/videos/taylor-swift/916187/everything-has-changed-ft-ed-sheeran.jhtml',
-            'file': 'USCJY1331283.mp4',
-            'md5': '73b4e7fcadd88929292fe52c3ced8caf',
-            'info_dict': {
-                'title': 'Everything Has Changed',
-                'upload_date': '20130606',
-                'uploader': 'Taylor Swift',
-            },
-            'skip': 'VEVO is only available in some countries',
-        },
     ]
 
     def _get_thumbnail_url(self, uri, itemdoc):
@@ -272,8 +264,8 @@ class MTVIE(MTVServicesInfoExtractor):
             webpage = self._download_webpage(url, video_id)
 
             # Some videos come from Vevo.com
-            m_vevo = re.search(r'isVevoVideo = true;.*?vevoVideoId = "(.*?)";',
-                               webpage, re.DOTALL)
+            m_vevo = re.search(
+                r'(?s)isVevoVideo = true;.*?vevoVideoId = "(.*?)";', webpage)
             if m_vevo:
                 vevo_id = m_vevo.group(1)
                 self.to_screen('Vevo video detected: %s' % vevo_id)

From 2886be15aa00e064c494eb0b6977246cdb905b1f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 12:10:15 +0100
Subject: [PATCH 0174/6219] [defense] Modernize

---
 youtube_dl/extractor/defense.py | 29 +++++++++++++++--------------
 1 file changed, 15 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/defense.py b/youtube_dl/extractor/defense.py
index 5e50c63d9..1637fa4a7 100644
--- a/youtube_dl/extractor/defense.py
+++ b/youtube_dl/extractor/defense.py
@@ -8,33 +8,34 @@ from .common import InfoExtractor
 
 class DefenseGouvFrIE(InfoExtractor):
     IE_NAME = 'defense.gouv.fr'
-    _VALID_URL = (r'http://.*?\.defense\.gouv\.fr/layout/set/'
-                  r'ligthboxvideo/base-de-medias/webtv/(.*)')
+    _VALID_URL = r'http://.*?\.defense\.gouv\.fr/layout/set/ligthboxvideo/base-de-medias/webtv/(?P<id>[^/?#]*)'
 
     _TEST = {
         'url': 'http://www.defense.gouv.fr/layout/set/ligthboxvideo/base-de-medias/webtv/attaque-chimique-syrienne-du-21-aout-2013-1',
-        'file': '11213.mp4',
         'md5': '75bba6124da7e63d2d60b5244ec9430c',
-        "info_dict": {
-            "title": "attaque-chimique-syrienne-du-21-aout-2013-1"
+        'info_dict': {
+            'id': '11213',
+            'ext': 'mp4',
+            'title': 'attaque-chimique-syrienne-du-21-aout-2013-1'
         }
     }
 
     def _real_extract(self, url):
-        title = re.match(self._VALID_URL, url).group(1)
+        title = self._match_id(url)
         webpage = self._download_webpage(url, title)
+
         video_id = self._search_regex(
             r"flashvars.pvg_id=\"(\d+)\";",
             webpage, 'ID')
 
         json_url = ('http://static.videos.gouv.fr/brightcovehub/export/json/'
                     + video_id)
-        info = self._download_webpage(json_url, title,
-                                      'Downloading JSON config')
-        video_url = json.loads(info)['renditions'][0]['url']
+        info = self._download_json(json_url, title, 'Downloading JSON config')
+        video_url = info['renditions'][0]['url']
 
-        return {'id': video_id,
-                'ext': 'mp4',
-                'url': video_url,
-                'title': title,
-                }
+        return {
+            'id': video_id,
+            'ext': 'mp4',
+            'url': video_url,
+            'title': title,
+        }

From 96aded8d3d48c5987a687b9233351ccbfc9c9237 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 12:11:14 +0100
Subject: [PATCH 0175/6219] [rottentomatoes] Modernize test definition

---
 youtube_dl/extractor/rottentomatoes.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/rottentomatoes.py b/youtube_dl/extractor/rottentomatoes.py
index c1500b82f..e8bb20a08 100644
--- a/youtube_dl/extractor/rottentomatoes.py
+++ b/youtube_dl/extractor/rottentomatoes.py
@@ -10,8 +10,9 @@ class RottenTomatoesIE(VideoDetectiveIE):
 
     _TEST = {
         'url': 'http://www.rottentomatoes.com/m/toy_story_3/trailers/11028566/',
-        'file': '613340.mp4',
         'info_dict': {
+            'id': '613340',
+            'ext': 'mp4',
             'title': 'TOY STORY 3',
             'description': 'From the creators of the beloved TOY STORY films, comes a story that will reunite the gang in a whole new way.',
         },

From 146303136fa30ef0edf2dfbac5acd69fb2f0dbcc Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 12:11:20 +0100
Subject: [PATCH 0176/6219] [nerdcubed] Modernize test definition

---
 youtube_dl/extractor/nerdcubed.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/nerdcubed.py b/youtube_dl/extractor/nerdcubed.py
index efc903afa..dff78e486 100644
--- a/youtube_dl/extractor/nerdcubed.py
+++ b/youtube_dl/extractor/nerdcubed.py
@@ -11,6 +11,7 @@ class NerdCubedFeedIE(InfoExtractor):
     _TEST = {
         'url': 'http://www.nerdcubed.co.uk/feed.json',
         'info_dict': {
+            'id': 'nerdcubed-feed',
             'title': 'nerdcubed.co.uk feed',
         },
         'playlist_mincount': 1300,

From fc09240e24c2455fa9373f77a13bf4649b90ca87 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 12:12:27 +0100
Subject: [PATCH 0177/6219] [vimeo] Modernize test definition

---
 youtube_dl/extractor/vimeo.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 06b0bed41..1bb473514 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -501,9 +501,10 @@ class VimeoReviewIE(InfoExtractor):
     _VALID_URL = r'https?://vimeo\.com/[^/]+/review/(?P<id>[^/]+)'
     _TESTS = [{
         'url': 'https://vimeo.com/user21297594/review/75524534/3c257a1b5d',
-        'file': '75524534.mp4',
         'md5': 'c507a72f780cacc12b2248bb4006d253',
         'info_dict': {
+            'id': '75524534',
+            'ext': 'mp4',
             'title': "DICK HARDWICK 'Comedian'",
             'uploader': 'Richard Hardwick',
         }

From 17d2712d9c58af788f234a2682d486f1d0a3ab78 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 15:00:54 +0100
Subject: [PATCH 0178/6219] [teamcoco] Modernize and fix extraction

---
 youtube_dl/extractor/teamcoco.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/teamcoco.py b/youtube_dl/extractor/teamcoco.py
index 5fa67eb8d..18a823719 100644
--- a/youtube_dl/extractor/teamcoco.py
+++ b/youtube_dl/extractor/teamcoco.py
@@ -10,17 +10,19 @@ class TeamcocoIE(InfoExtractor):
     _TESTS = [
         {
             'url': 'http://teamcoco.com/video/80187/conan-becomes-a-mary-kay-beauty-consultant',
-            'file': '80187.mp4',
             'md5': '3f7746aa0dc86de18df7539903d399ea',
             'info_dict': {
+                'id': '80187',
+                'ext': 'mp4',
                 'title': 'Conan Becomes A Mary Kay Beauty Consultant',
                 'description': 'Mary Kay is perhaps the most trusted name in female beauty, so of course Conan is a natural choice to sell their products.'
             }
         }, {
             'url': 'http://teamcoco.com/video/louis-ck-interview-george-w-bush',
-            'file': '19705.mp4',
             'md5': 'cde9ba0fa3506f5f017ce11ead928f9a',
             'info_dict': {
+                'id': '19705',
+                'ext': 'mp4',
                 "description": "Louis C.K. got starstruck by George W. Bush, so what? Part one.",
                 "title": "Louis C.K. Interview Pt. 1 11/3/11"
             }
@@ -36,7 +38,7 @@ class TeamcocoIE(InfoExtractor):
         video_id = mobj.group("video_id")
         if not video_id:
             video_id = self._html_search_regex(
-                r'data-node-id="(\d+?)"',
+                r'<div\s+class="player".*?data-id="(\d+?)"',
                 webpage, 'video id')
 
         data_url = 'http://teamcoco.com/cvp/2.0/%s.xml' % video_id

From 057c0609fc6620017ff4a5b30847e0dc2644df9f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 15:01:33 +0100
Subject: [PATCH 0179/6219] [toutv] Modernize test definition

---
 youtube_dl/extractor/toutv.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/toutv.py b/youtube_dl/extractor/toutv.py
index 2837f9c8e..4797d1310 100644
--- a/youtube_dl/extractor/toutv.py
+++ b/youtube_dl/extractor/toutv.py
@@ -16,8 +16,9 @@ class TouTvIE(InfoExtractor):
 
     _TEST = {
         'url': 'http://www.tou.tv/30-vies/S04E41',
-        'file': '30-vies_S04E41.mp4',
         'info_dict': {
+            'id': '30-vies_S04E41',
+            'ext': 'mp4',
             'title': '30 vies Saison 4 / Épisode 41',
             'description': 'md5:da363002db82ccbe4dafeb9cab039b09',
             'age_limit': 8,

From 228d30ed0616f6c1b5cff335b8b56f4fde48b76d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 15:03:03 +0100
Subject: [PATCH 0180/6219] [la7] Modernize

---
 youtube_dl/extractor/la7.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/la7.py b/youtube_dl/extractor/la7.py
index db2028e9f..60625c42d 100644
--- a/youtube_dl/extractor/la7.py
+++ b/youtube_dl/extractor/la7.py
@@ -20,9 +20,10 @@ class LA7IE(InfoExtractor):
 
     _TEST = {
         'url': 'http://www.la7.tv/richplayer/?assetid=50355319',
-        'file': '50355319.mp4',
         'md5': 'ec7d1f0224d20ba293ab56cf2259651f',
         'info_dict': {
+            'id': '50355319',
+            'ext': 'mp4',
             'title': 'IL DIVO',
             'description': 'Un film di Paolo Sorrentino con Toni Servillo, Anna Bonaiuto, Giulio Bosetti  e Flavio Bucci',
             'duration': 6254,
@@ -31,9 +32,7 @@ class LA7IE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
+        video_id = self._match_id(url)
         xml_url = 'http://www.la7.tv/repliche/content/index.php?contentId=%s' % video_id
         doc = self._download_xml(xml_url, video_id)
 

From 9a3e5e69554c06fbf8bb7fd248c2e921593f8046 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 15:03:55 +0100
Subject: [PATCH 0181/6219] [kankan] Modernize

---
 youtube_dl/extractor/kankan.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/kankan.py b/youtube_dl/extractor/kankan.py
index dbfe4cc03..364dc878e 100644
--- a/youtube_dl/extractor/kankan.py
+++ b/youtube_dl/extractor/kankan.py
@@ -13,17 +13,17 @@ class KankanIE(InfoExtractor):
 
     _TEST = {
         'url': 'http://yinyue.kankan.com/vod/48/48863.shtml',
-        'file': '48863.flv',
         'md5': '29aca1e47ae68fc28804aca89f29507e',
         'info_dict': {
+            'id': '48863',
+            'ext': 'flv',
             'title': 'Ready To Go',
         },
         'skip': 'Only available from China',
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
         title = self._search_regex(r'(?:G_TITLE=|G_MOVIE_TITLE = )[\'"](.+?)[\'"]', webpage, 'video title')

From c4e817ce4aeeb4012b0e128afe6c138d3d905a24 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 15:06:55 +0100
Subject: [PATCH 0182/6219] [france2.fr:generation-quoi] Modernize

---
 youtube_dl/extractor/francetv.py | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index bbc760a49..170d68075 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -230,12 +230,13 @@ class FranceTVIE(FranceTVBaseInfoExtractor):
 
 class GenerationQuoiIE(InfoExtractor):
     IE_NAME = 'france2.fr:generation-quoi'
-    _VALID_URL = r'https?://generation-quoi\.france2\.fr/portrait/(?P<name>.*)(\?|$)'
+    _VALID_URL = r'https?://generation-quoi\.france2\.fr/portrait/(?P<id>[^/?#]+)'
 
     _TEST = {
         'url': 'http://generation-quoi.france2.fr/portrait/garde-a-vous',
-        'file': 'k7FJX8VBcvvLmX4wA5Q.mp4',
         'info_dict': {
+            'id': 'k7FJX8VBcvvLmX4wA5Q',
+            'ext': 'mp4',
             'title': 'Génération Quoi - Garde à Vous',
             'uploader': 'Génération Quoi',
         },
@@ -243,14 +244,12 @@ class GenerationQuoiIE(InfoExtractor):
             # It uses Dailymotion
             'skip_download': True,
         },
-        'skip': 'Only available from France',
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        name = mobj.group('name')
-        info_url = compat_urlparse.urljoin(url, '/medias/video/%s.json' % name)
-        info_json = self._download_webpage(info_url, name)
+        display_id = self._match_id(url)
+        info_url = compat_urlparse.urljoin(url, '/medias/video/%s.json' % display_id)
+        info_json = self._download_webpage(info_url, display_id)
         info = json.loads(info_json)
         return self.url_result('http://www.dailymotion.com/video/%s' % info['id'],
                                ie='Dailymotion')

From c1f06d6307109acfa4434a38131c7d9a07d58ab6 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 15:08:33 +0100
Subject: [PATCH 0183/6219] [macgamestore] Modernize

---
 youtube_dl/extractor/macgamestore.py | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/macgamestore.py b/youtube_dl/extractor/macgamestore.py
index b818cf50c..3cd4a3a19 100644
--- a/youtube_dl/extractor/macgamestore.py
+++ b/youtube_dl/extractor/macgamestore.py
@@ -1,7 +1,5 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..utils import ExtractorError
 
@@ -13,21 +11,22 @@ class MacGameStoreIE(InfoExtractor):
 
     _TEST = {
         'url': 'http://www.macgamestore.com/mediaviewer.php?trailer=2450',
-        'file': '2450.m4v',
         'md5': '8649b8ea684b6666b4c5be736ecddc61',
         'info_dict': {
+            'id': '2450',
+            'ext': 'm4v',
             'title': 'Crow',
         }
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(
+            url, video_id, 'Downloading trailer page')
 
-        webpage = self._download_webpage(url, video_id, 'Downloading trailer page')
-
-        if re.search(r'>Missing Media<', webpage) is not None:
-            raise ExtractorError('Trailer %s does not exist' % video_id, expected=True)
+        if '>Missing Media<' in webpage:
+            raise ExtractorError(
+                'Trailer %s does not exist' % video_id, expected=True)
 
         video_title = self._html_search_regex(
             r'<title>MacGameStore: (.*?) Trailer</title>', webpage, 'title')

From 748a0fab8a35fdfde03407d49eb4b41e4be3a1fc Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 15:08:50 +0100
Subject: [PATCH 0184/6219] Remove unused imports

---
 youtube_dl/extractor/defense.py | 3 ---
 youtube_dl/extractor/la7.py     | 2 --
 2 files changed, 5 deletions(-)

diff --git a/youtube_dl/extractor/defense.py b/youtube_dl/extractor/defense.py
index 1637fa4a7..2b90bf4fc 100644
--- a/youtube_dl/extractor/defense.py
+++ b/youtube_dl/extractor/defense.py
@@ -1,8 +1,5 @@
 from __future__ import unicode_literals
 
-import re
-import json
-
 from .common import InfoExtractor
 
 
diff --git a/youtube_dl/extractor/la7.py b/youtube_dl/extractor/la7.py
index 60625c42d..b08f6e3c9 100644
--- a/youtube_dl/extractor/la7.py
+++ b/youtube_dl/extractor/la7.py
@@ -1,7 +1,5 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..utils import (
     parse_duration,

From 9f281cacd2bb1de0dc11dac35685f04dcb5f447c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 15:13:41 +0100
Subject: [PATCH 0185/6219] [keezmovies] Fix extraction and modernize test

---
 youtube_dl/extractor/keezmovies.py | 16 ++++++----------
 1 file changed, 6 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/keezmovies.py b/youtube_dl/extractor/keezmovies.py
index 97dcb518a..82eddec51 100644
--- a/youtube_dl/extractor/keezmovies.py
+++ b/youtube_dl/extractor/keezmovies.py
@@ -7,10 +7,6 @@ from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse_urlparse,
     compat_urllib_request,
-    compat_urllib_parse,
-)
-from ..aes import (
-    aes_decrypt_text
 )
 
 
@@ -18,9 +14,10 @@ class KeezMoviesIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?keezmovies\.com/video/.+?(?P<id>[0-9]+)(?:[/?&]|$)'
     _TEST = {
         'url': 'http://www.keezmovies.com/video/petite-asian-lady-mai-playing-in-bathtub-1214711',
-        'file': '1214711.mp4',
         'md5': '6e297b7e789329923fcf83abb67c9289',
         'info_dict': {
+            'id': '1214711',
+            'ext': 'mp4',
             'title': 'Petite Asian Lady Mai Playing In Bathtub',
             'age_limit': 18,
         }
@@ -39,11 +36,10 @@ class KeezMoviesIE(InfoExtractor):
             embedded_url = mobj.group(1)
             return self.url_result(embedded_url)
 
-        video_title = self._html_search_regex(r'<h1 [^>]*>([^<]+)', webpage, 'title')
-        video_url = compat_urllib_parse.unquote(self._html_search_regex(r'video_url=(.+?)&amp;', webpage, 'video_url'))
-        if 'encrypted=true' in webpage:
-            password = self._html_search_regex(r'video_title=(.+?)&amp;', webpage, 'password')
-            video_url = aes_decrypt_text(video_url, password, 32).decode('utf-8')
+        video_title = self._html_search_regex(
+            r'<h1 [^>]*>([^<]+)', webpage, 'title')
+        video_url = self._html_search_regex(
+            r'(?s)html5VideoPlayer = .*?src="([^"]+)"', webpage, 'video URL')
         path = compat_urllib_parse_urlparse(video_url).path
         extension = os.path.splitext(path)[1][1:]
         format = path.split('/')[4].split('_')[:2]

From 437cac8cc18e541dd1c4c55ef1b19de7d6529ae9 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 15:16:35 +0100
Subject: [PATCH 0186/6219] [sina] Modernize and simplify

---
 youtube_dl/extractor/sina.py | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/sina.py b/youtube_dl/extractor/sina.py
index a63d126d4..0891a441f 100644
--- a/youtube_dl/extractor/sina.py
+++ b/youtube_dl/extractor/sina.py
@@ -11,7 +11,7 @@ from ..compat import (
 
 
 class SinaIE(InfoExtractor):
-    _VALID_URL = r'''https?://(.*?\.)?video\.sina\.com\.cn/
+    _VALID_URL = r'''(?x)https?://(.*?\.)?video\.sina\.com\.cn/
                         (
                             (.+?/(((?P<pseudo_id>\d+).html)|(.*?(\#|(vid=)|b/)(?P<id>\d+?)($|&|\-))))
                             |
@@ -23,9 +23,10 @@ class SinaIE(InfoExtractor):
     _TESTS = [
         {
             'url': 'http://video.sina.com.cn/news/vlist/zt/chczlj2013/?opsubject_id=top12#110028898',
-            'file': '110028898.flv',
             'md5': 'd65dd22ddcf44e38ce2bf58a10c3e71f',
             'info_dict': {
+                'id': '110028898',
+                'ext': 'flv',
                 'title': '《中国新闻》 朝鲜要求巴拿马立即释放被扣船员',
             }
         },
@@ -39,10 +40,6 @@ class SinaIE(InfoExtractor):
         },
     ]
 
-    @classmethod
-    def suitable(cls, url):
-        return re.match(cls._VALID_URL, url, flags=re.VERBOSE) is not None
-
     def _extract_video(self, video_id):
         data = compat_urllib_parse.urlencode({'vid': video_id})
         url_doc = self._download_xml('http://v.iask.com/v_play.php?%s' % data,
@@ -59,7 +56,7 @@ class SinaIE(InfoExtractor):
                 }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url, flags=re.VERBOSE)
+        mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
         if mobj.group('token') is not None:
             # The video id is in the redirected url

From e332772531d15bf15ccfc807085c6a1bdae163f3 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 15:18:52 +0100
Subject: [PATCH 0187/6219] [servingsys] Modernize

---
 youtube_dl/extractor/servingsys.py | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/servingsys.py b/youtube_dl/extractor/servingsys.py
index 16dc3736b..c013d678f 100644
--- a/youtube_dl/extractor/servingsys.py
+++ b/youtube_dl/extractor/servingsys.py
@@ -1,7 +1,5 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..utils import (
     int_or_none,
@@ -13,10 +11,15 @@ class ServingSysIE(InfoExtractor):
 
     _TEST = {
         'url': 'http://bs.serving-sys.com/BurstingPipe/adServer.bs?cn=is&c=23&pl=VAST&pli=5349193&PluID=0&pos=7135&ord=[timestamp]&cim=1?',
+        'info_dict': {
+            'id': '5349193',
+            'title': 'AdAPPter_Hyundai_demo',
+        },
         'playlist': [{
-            'file': '29955898.flv',
             'md5': 'baed851342df6846eb8677a60a011a0f',
             'info_dict': {
+                'id': '29955898',
+                'ext': 'flv',
                 'title': 'AdAPPter_Hyundai_demo (1)',
                 'duration': 74,
                 'tbr': 1378,
@@ -24,9 +27,10 @@ class ServingSysIE(InfoExtractor):
                 'height': 400,
             },
         }, {
-            'file': '29907998.flv',
             'md5': '979b4da2655c4bc2d81aeb915a8c5014',
             'info_dict': {
+                'id': '29907998',
+                'ext': 'flv',
                 'title': 'AdAPPter_Hyundai_demo (2)',
                 'duration': 34,
                 'width': 854,
@@ -37,14 +41,13 @@ class ServingSysIE(InfoExtractor):
         'params': {
             'playlistend': 2,
         },
-        'skip': 'Blocked in the US [sic]',
+        '_skip': 'Blocked in the US [sic]',
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        pl_id = mobj.group('id')
-
+        pl_id = self._match_id(url)
         vast_doc = self._download_xml(url, pl_id)
+
         title = vast_doc.find('.//AdTitle').text
         media = vast_doc.find('.//MediaFile').text
         info_url = self._search_regex(r'&adData=([^&]+)&', media, 'info URL')

From c172440ac50b75eaee660a5c00f44c9c8a2f37c0 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 15:21:02 +0100
Subject: [PATCH 0188/6219] [ringtv] Modernize test definition

---
 youtube_dl/extractor/ringtv.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/ringtv.py b/youtube_dl/extractor/ringtv.py
index 59dc137cc..efa4afeb6 100644
--- a/youtube_dl/extractor/ringtv.py
+++ b/youtube_dl/extractor/ringtv.py
@@ -6,12 +6,13 @@ from .common import InfoExtractor
 
 
 class RingTVIE(InfoExtractor):
-    _VALID_URL = r'(?:http://)?(?:www\.)?ringtv\.craveonline\.com/(?P<type>news|videos/video)/(?P<id>[^/?#]+)'
+    _VALID_URL = r'http://(?:www\.)?ringtv\.craveonline\.com/(?P<type>news|videos/video)/(?P<id>[^/?#]+)'
     _TEST = {
         "url": "http://ringtv.craveonline.com/news/310833-luis-collazo-says-victor-ortiz-better-not-quit-on-jan-30",
-        "file": "857645.mp4",
         "md5": "d25945f5df41cdca2d2587165ac28720",
         "info_dict": {
+            'id': '857645',
+            'ext': 'mp4',
             "title": 'Video: Luis Collazo says Victor Ortiz "better not quit on Jan. 30" - Ring TV',
             "description": 'Luis Collazo is excited about his Jan. 30 showdown with fellow former welterweight titleholder Victor Ortiz at Barclays Center in his hometown of Brooklyn. The SuperBowl week fight headlines a Golden Boy Live! card on Fox Sports 1.',
         }

From 4e980275b517b101d5cd3646fee6ced5f627da8a Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 15:21:18 +0100
Subject: [PATCH 0189/6219] [test_download] Remove references to "file" in test
 definitions

---
 test/test_download.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/test_download.py b/test/test_download.py
index 412f3dbce..6a149ae4f 100644
--- a/test/test_download.py
+++ b/test/test_download.py
@@ -89,7 +89,7 @@ def generator(test_case):
 
         for tc in test_cases:
             info_dict = tc.get('info_dict', {})
-            if not tc.get('file') and not (info_dict.get('id') and info_dict.get('ext')):
+            if not (info_dict.get('id') and info_dict.get('ext')):
                 raise Exception('Test definition incorrect. The output file cannot be known. Are both \'id\' and \'ext\' keys present?')
 
         if 'skip' in test_case:
@@ -116,7 +116,7 @@ def generator(test_case):
         expect_warnings(ydl, test_case.get('expected_warnings', []))
 
         def get_tc_filename(tc):
-            return tc.get('file') or ydl.prepare_filename(tc.get('info_dict', {}))
+            return ydl.prepare_filename(tc.get('info_dict', {}))
 
         res_dict = None
 

From a9551e90207a2bb2ef8f35279fc525846e21cbe4 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 15:24:38 +0100
Subject: [PATCH 0190/6219] [soundcloud:set] Fix test definition

---
 youtube_dl/extractor/soundcloud.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 5d60c4939..c5284fa67 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -246,6 +246,7 @@ class SoundcloudSetIE(SoundcloudIE):
     _TESTS = [{
         'url': 'https://soundcloud.com/the-concept-band/sets/the-royal-concept-ep',
         'info_dict': {
+            'id': '2284613',
             'title': 'The Royal Concept EP',
         },
         'playlist_mincount': 6,
@@ -279,7 +280,7 @@ class SoundcloudSetIE(SoundcloudIE):
         return {
             '_type': 'playlist',
             'entries': [self._extract_info_dict(track, secret_token=token) for track in info['tracks']],
-            'id': info['id'],
+            'id': '%s' % info['id'],
             'title': info['title'],
         }
 

From 654bd52f5847dc3f1604611f7f1ca3beafee7046 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 15:25:33 +0100
Subject: [PATCH 0191/6219] [teletask] Fix test definition

---
 youtube_dl/extractor/teletask.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/teletask.py b/youtube_dl/extractor/teletask.py
index e54145105..b9e2ef8ca 100644
--- a/youtube_dl/extractor/teletask.py
+++ b/youtube_dl/extractor/teletask.py
@@ -11,6 +11,7 @@ class TeleTaskIE(InfoExtractor):
     _TEST = {
         'url': 'http://www.tele-task.de/archive/video/html5/26168/',
         'info_dict': {
+            'id': '26168',
             'title': 'Duplicate Detection',
         },
         'playlist': [{
@@ -34,7 +35,6 @@ class TeleTaskIE(InfoExtractor):
 
     def _real_extract(self, url):
         lecture_id = self._match_id(url)
-
         webpage = self._download_webpage(url, lecture_id)
 
         title = self._html_search_regex(

From dd8982f19c97a539fc11ee0ee74bb6c6fb6beb57 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 15:27:16 +0100
Subject: [PATCH 0192/6219] [wdr] Complete test definition

---
 youtube_dl/extractor/wdr.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/wdr.py b/youtube_dl/extractor/wdr.py
index 313b9c15d..c90488500 100644
--- a/youtube_dl/extractor/wdr.py
+++ b/youtube_dl/extractor/wdr.py
@@ -71,6 +71,9 @@ class WDRIE(InfoExtractor):
         {
             'url': 'http://www1.wdr.de/mediathek/video/sendungen/quarks_und_co/filterseite-quarks-und-co100.html',
             'playlist_mincount': 146,
+            'info_dict': {
+                'id': 'mediathek/video/sendungen/quarks_und_co/filterseite-quarks-und-co100',
+            }
         }
     ]
 

From acf757f42eea16266a539fa0eb871dae422a1d22 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 15:33:32 +0100
Subject: [PATCH 0193/6219] [youtube] Complete test definition

---
 youtube_dl/extractor/youtube.py | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 3b3678c6e..e4b26b84f 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1160,6 +1160,7 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
     }, {
         'url': 'https://www.youtube.com/playlist?list=PLtPgu7CB4gbZDA7i_euNxn75ISqxwZPYx',
         'info_dict': {
+            'id': 'PLtPgu7CB4gbZDA7i_euNxn75ISqxwZPYx',
             'title': 'YDL_Empty_List',
         },
         'playlist_count': 0,
@@ -1168,6 +1169,7 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
         'url': 'https://www.youtube.com/playlist?list=PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
         'info_dict': {
             'title': '29C3: Not my department',
+            'id': 'PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
         },
         'playlist_count': 95,
     }, {
@@ -1175,6 +1177,7 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
         'url': 'PLBB231211A4F62143',
         'info_dict': {
             'title': '[OLD]Team Fortress 2 (Class-based LP)',
+            'id': 'PLBB231211A4F62143',
         },
         'playlist_mincount': 26,
     }, {
@@ -1182,12 +1185,14 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
         'url': 'https://www.youtube.com/playlist?list=UUBABnxM4Ar9ten8Mdjj1j0Q',
         'info_dict': {
             'title': 'Uploads from Cauchemar',
+            'id': 'UUBABnxM4Ar9ten8Mdjj1j0Q',
         },
         'playlist_mincount': 799,
     }, {
         'url': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
         'info_dict': {
             'title': 'YDL_safe_search',
+            'id': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
         },
         'playlist_count': 2,
     }, {
@@ -1196,6 +1201,7 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
         'playlist_count': 4,
         'info_dict': {
             'title': 'JODA15',
+            'id': 'PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
         }
     }, {
         'note': 'Embedded SWF player',
@@ -1203,12 +1209,14 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
         'playlist_count': 4,
         'info_dict': {
             'title': 'JODA7',
+            'id': 'YN5VISEtHet5D4NEvfTd0zcgFk84NqFZ',
         }
     }, {
         'note': 'Buggy playlist: the webpage has a "Load more" button but it doesn\'t have more videos',
         'url': 'https://www.youtube.com/playlist?list=UUXw-G3eDE9trcvY2sBMM_aA',
         'info_dict': {
-                'title': 'Uploads from Interstellar Movie',
+            'title': 'Uploads from Interstellar Movie',
+            'id': 'UUXw-G3eDE9trcvY2sBMM_aA',
         },
         'playlist_mincout': 21,
     }]
@@ -1314,6 +1322,9 @@ class YoutubeChannelIE(InfoExtractor):
         'note': 'paginated channel',
         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
         'playlist_mincount': 91,
+        'info_dict': {
+            'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
+        }
     }]
 
     def extract_videos_from_page(self, page):

From c6df692466aebed1b86c02233b579c03f0888704 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 1 Feb 2015 23:48:23 +0600
Subject: [PATCH 0194/6219] [rutv] Extract all m3u8 formats

---
 youtube_dl/extractor/rutv.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/rutv.py b/youtube_dl/extractor/rutv.py
index a73e6f331..ef766237b 100644
--- a/youtube_dl/extractor/rutv.py
+++ b/youtube_dl/extractor/rutv.py
@@ -162,10 +162,8 @@ class RUTVIE(InfoExtractor):
                         'vbr': int(quality),
                     }
                 elif transport == 'm3u8':
-                    fmt = {
-                        'url': url,
-                        'ext': 'mp4',
-                    }
+                    formats.extend(self._extract_m3u8_formats(url, video_id, 'mp4'))
+                    continue
                 else:
                     fmt = {
                         'url': url

From 027008b14ebe3d41ac7b9940ab73eaed120dab5c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 1 Feb 2015 23:49:23 +0600
Subject: [PATCH 0195/6219] [hls] Fix encode issues on python2 @ Windows

---
 youtube_dl/downloader/hls.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index aa58b52ab..b642786cc 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -11,6 +11,7 @@ from ..compat import (
     compat_urllib_request,
 )
 from ..utils import (
+    encodeArgument,
     encodeFilename,
 )
 
@@ -21,18 +22,17 @@ class HlsFD(FileDownloader):
         self.report_destination(filename)
         tmpfilename = self.temp_name(filename)
 
-        args = [
-            '-y', '-i', url, '-f', 'mp4', '-c', 'copy',
-            '-bsf:a', 'aac_adtstoasc',
-            encodeFilename(tmpfilename, for_subprocess=True)]
-
         ffpp = FFmpegPostProcessor(downloader=self)
         program = ffpp._executable
         if program is None:
             self.report_error('m3u8 download detected but ffmpeg or avconv could not be found. Please install one.')
             return False
         ffpp.check_version()
-        cmd = [program] + args
+
+        args = [encodeArgument(opt) for opt in ('-y', '-i', url, '-f', 'mp4', '-c', 'copy', '-bsf:a', 'aac_adtstoasc')]
+        args.append(encodeFilename(tmpfilename, True))
+
+        cmd = [encodeArgument(program)] + args
 
         retval = subprocess.call(cmd)
         if retval == 0:

From 4a3da4ebdb1de5e35b47176822ab9ed3b2762c3f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 1 Feb 2015 23:54:38 +0600
Subject: [PATCH 0196/6219] [hls] Simplify

---
 youtube_dl/downloader/hls.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index b642786cc..e527ee425 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -29,15 +29,15 @@ class HlsFD(FileDownloader):
             return False
         ffpp.check_version()
 
-        args = [encodeArgument(opt) for opt in ('-y', '-i', url, '-f', 'mp4', '-c', 'copy', '-bsf:a', 'aac_adtstoasc')]
+        args = [
+            encodeArgument(opt)
+            for opt in (program, '-y', '-i', url, '-f', 'mp4', '-c', 'copy', '-bsf:a', 'aac_adtstoasc')]
         args.append(encodeFilename(tmpfilename, True))
 
-        cmd = [encodeArgument(program)] + args
-
-        retval = subprocess.call(cmd)
+        retval = subprocess.call(args)
         if retval == 0:
             fsize = os.path.getsize(encodeFilename(tmpfilename))
-            self.to_screen('\r[%s] %s bytes' % (cmd[0], fsize))
+            self.to_screen('\r[%s] %s bytes' % (args[0], fsize))
             self.try_rename(tmpfilename, filename)
             self._hook_progress({
                 'downloaded_bytes': fsize,

From 9e3f19919aa3e152760f4cd0a7cad348428f3b35 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 22:38:26 +0100
Subject: [PATCH 0197/6219] [jsinterp] Beef up and add tests

In preparation for #4822, extend jsinterp by a lot. (We may even have to/want to write a proper interpreter with actual parsing)
---
 test/test_jsinterp.py  |  95 +++++++++++++++++++++++
 youtube_dl/jsinterp.py | 172 ++++++++++++++++++++++++++++++-----------
 2 files changed, 221 insertions(+), 46 deletions(-)
 create mode 100644 test/test_jsinterp.py

diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py
new file mode 100644
index 000000000..b91b8c492
--- /dev/null
+++ b/test/test_jsinterp.py
@@ -0,0 +1,95 @@
+#!/usr/bin/env python
+
+from __future__ import unicode_literals
+
+# Allow direct execution
+import os
+import sys
+import unittest
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from youtube_dl.jsinterp import JSInterpreter
+
+
+class TestJSInterpreter(unittest.TestCase):
+    def test_basic(self):
+        jsi = JSInterpreter('function x(){;}')
+        self.assertEqual(jsi.call_function('x'), None)
+
+        jsi = JSInterpreter('function x3(){return 42;}')
+        self.assertEqual(jsi.call_function('x3'), 42)
+
+    def test_calc(self):
+        jsi = JSInterpreter('function x4(a){return 2*a+1;}')
+        self.assertEqual(jsi.call_function('x4', 3), 7)
+
+    def test_empty_return(self):
+        jsi = JSInterpreter('function f(){return; y()}')
+        self.assertEqual(jsi.call_function('f'), None)
+
+    def test_morespace(self):
+        jsi = JSInterpreter('function x (a) { return 2 * a + 1 ; }')
+        self.assertEqual(jsi.call_function('x', 3), 7)
+
+        jsi = JSInterpreter('function f () { x =  2  ; return x; }')
+        self.assertEqual(jsi.call_function('f'), 2)
+
+    def test_strange_chars(self):
+        jsi = JSInterpreter('function $_xY1 ($_axY1) { var $_axY2 = $_axY1 + 1; return $_axY2; }')
+        self.assertEqual(jsi.call_function('$_xY1', 20), 21)
+
+    def test_operators(self):
+        jsi = JSInterpreter('function f(){return 1 << 5;}')
+        self.assertEqual(jsi.call_function('f'), 32)
+
+        jsi = JSInterpreter('function f(){return 19 & 21;}')
+        self.assertEqual(jsi.call_function('f'), 17)
+
+        jsi = JSInterpreter('function f(){return 11 >> 2;}')
+        self.assertEqual(jsi.call_function('f'), 2)
+
+    def test_array_access(self):
+        jsi = JSInterpreter('function f(){var x = [1,2,3]; x[0] = 4; x[0] = 5; x[2] = 7; return x;}')
+        self.assertEqual(jsi.call_function('f'), [5, 2, 7])
+
+    def test_parens(self):
+        jsi = JSInterpreter('function f(){return (1) + (2) * ((( (( (((((3)))))) )) ));}')
+        self.assertEqual(jsi.call_function('f'), 7)
+
+        jsi = JSInterpreter('function f(){return (1 + 2) * 3;}')
+        self.assertEqual(jsi.call_function('f'), 9)
+
+    def test_assignments(self):
+        jsi = JSInterpreter('function f(){var x = 20; x = 30 + 1; return x;}')
+        self.assertEqual(jsi.call_function('f'), 31)
+
+        jsi = JSInterpreter('function f(){var x = 20; x += 30 + 1; return x;}')
+        self.assertEqual(jsi.call_function('f'), 51)
+
+        jsi = JSInterpreter('function f(){var x = 20; x -= 30 + 1; return x;}')
+        self.assertEqual(jsi.call_function('f'), -11)
+
+    def test_comments(self):
+        jsi = JSInterpreter('''
+        function x() {
+            var x = /* 1 + */ 2;
+            var y = /* 30
+            * 40 */ 50;
+            return x + y;
+        }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 52)
+
+    def test_precedence(self):
+        jsi = JSInterpreter('''
+        function x() {
+            var a = [10, 20, 30, 40, 50];
+            var b = 6;
+            a[0]=a[b%a.length];
+            return a;
+        }''')
+        self.assertEqual(jsi.call_function('x'), [20, 20, 30, 40, 50])
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index b4617fbad..49364786b 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -1,59 +1,122 @@
 from __future__ import unicode_literals
 
 import json
+import operator
 import re
 
 from .utils import (
     ExtractorError,
 )
 
+_OPERATORS = [
+    ('|', operator.or_),
+    ('^', operator.xor),
+    ('&', operator.and_),
+    ('>>', operator.rshift),
+    ('<<', operator.lshift),
+    ('-', operator.sub),
+    ('+', operator.add),
+    ('%', operator.mod),
+    ('/', operator.div),
+    ('*', operator.mul),
+]
+_ASSIGN_OPERATORS = [(op + '=', opfunc) for op, opfunc in _OPERATORS]
+_ASSIGN_OPERATORS.append(('=', lambda cur, right: right))
+
+_NAME_RE = r'[a-zA-Z_$][a-zA-Z_$0-9]*'
+
 
 class JSInterpreter(object):
-    def __init__(self, code):
-        self.code = code
+    def __init__(self, code, objects=None):
+        if objects is None:
+            objects = {}
+        self.code = self._remove_comments(code)
         self._functions = {}
-        self._objects = {}
+        self._objects = objects
 
-    def interpret_statement(self, stmt, local_vars, allow_recursion=20):
+    def _remove_comments(self, code):
+        return re.sub(r'(?s)/\*.*?\*/', '', code)
+
+    def interpret_statement(self, stmt, local_vars, allow_recursion=100):
         if allow_recursion < 0:
             raise ExtractorError('Recursion limit reached')
 
-        if stmt.startswith('var '):
-            stmt = stmt[len('var '):]
-        ass_m = re.match(r'^(?P<out>[a-z]+)(?:\[(?P<index>[^\]]+)\])?' +
-                         r'=(?P<expr>.*)$', stmt)
-        if ass_m:
-            if ass_m.groupdict().get('index'):
-                def assign(val):
-                    lvar = local_vars[ass_m.group('out')]
-                    idx = self.interpret_expression(
-                        ass_m.group('index'), local_vars, allow_recursion)
-                    assert isinstance(idx, int)
-                    lvar[idx] = val
-                    return val
-                expr = ass_m.group('expr')
-            else:
-                def assign(val):
-                    local_vars[ass_m.group('out')] = val
-                    return val
-                expr = ass_m.group('expr')
-        elif stmt.startswith('return '):
-            assign = lambda v: v
-            expr = stmt[len('return '):]
+        should_abort = False
+        stmt = stmt.lstrip()
+        stmt_m = re.match(r'var\s', stmt)
+        if stmt_m:
+            expr = stmt[len(stmt_m.group(0)):]
         else:
-            # Try interpreting it as an expression
-            expr = stmt
-            assign = lambda v: v
+            return_m = re.match(r'return(?:\s+|$)', stmt)
+            if return_m:
+                expr = stmt[len(return_m.group(0)):]
+                should_abort = True
+            else:
+                # Try interpreting it as an expression
+                expr = stmt
 
         v = self.interpret_expression(expr, local_vars, allow_recursion)
-        return assign(v)
+        return v, should_abort
 
     def interpret_expression(self, expr, local_vars, allow_recursion):
+        expr = expr.strip()
+
+        if expr == '':  # Empty expression
+            return None
+
+        if expr.startswith('('):
+            parens_count = 0
+            for m in re.finditer(r'[()]', expr):
+                if m.group(0) == '(':
+                    parens_count += 1
+                else:
+                    parens_count -= 1
+                    if parens_count == 0:
+                        sub_expr = expr[1:m.start()]
+                        sub_result = self.interpret_expression(
+                            sub_expr, local_vars, allow_recursion)
+                        remaining_expr = expr[m.end():].strip()
+                        if not remaining_expr:
+                            return sub_result
+                        else:
+                            expr = json.dumps(sub_result) + remaining_expr
+                        break
+            else:
+                raise ExtractorError('Premature end of parens in %r' % expr)
+
+        for op, opfunc in _ASSIGN_OPERATORS:
+            m = re.match(r'''(?x)
+                (?P<out>%s)(?:\[(?P<index>[^\]]+?)\])?
+                \s*%s
+                (?P<expr>.*)$''' % (_NAME_RE, re.escape(op)), expr)
+            if not m:
+                continue
+            right_val = self.interpret_expression(
+                m.group('expr'), local_vars, allow_recursion - 1)
+
+            if m.groupdict().get('index'):
+                lvar = local_vars[m.group('out')]
+                idx = self.interpret_expression(
+                    m.group('index'), local_vars, allow_recursion)
+                assert isinstance(idx, int)
+                cur = lvar[idx]
+                val = opfunc(cur, right_val)
+                lvar[idx] = val
+                return val
+            else:
+                cur = local_vars.get(m.group('out'))
+                val = opfunc(cur, right_val)
+                local_vars[m.group('out')] = val
+                return val
+
         if expr.isdigit():
             return int(expr)
 
-        if expr.isalpha():
-            return local_vars[expr]
+        var_m = re.match(
+            r'(?!if|return|true|false)(?P<name>%s)$' % _NAME_RE,
+            expr)
+        if var_m:
+            return local_vars[var_m.group('name')]
 
         try:
             return json.loads(expr)
@@ -61,7 +124,7 @@ class JSInterpreter(object):
             pass
 
         m = re.match(
-            r'^(?P<var>[$a-zA-Z0-9_]+)\.(?P<member>[^(]+)(?:\(+(?P<args>[^()]*)\))?$',
+            r'(?P<var>%s)\.(?P<member>[^(]+)(?:\(+(?P<args>[^()]*)\))?$' % _NAME_RE,
             expr)
         if m:
             variable = m.group('var')
@@ -114,23 +177,31 @@ class JSInterpreter(object):
             return obj[member](argvals)
 
         m = re.match(
-            r'^(?P<in>[a-z]+)\[(?P<idx>.+)\]$', expr)
+            r'(?P<in>%s)\[(?P<idx>.+)\]$' % _NAME_RE, expr)
         if m:
             val = local_vars[m.group('in')]
             idx = self.interpret_expression(
                 m.group('idx'), local_vars, allow_recursion - 1)
             return val[idx]
 
-        m = re.match(r'^(?P<a>.+?)(?P<op>[%])(?P<b>.+?)$', expr)
-        if m:
-            a = self.interpret_expression(
-                m.group('a'), local_vars, allow_recursion)
-            b = self.interpret_expression(
-                m.group('b'), local_vars, allow_recursion)
-            return a % b
+        for op, opfunc in _OPERATORS:
+            m = re.match(r'(?P<x>.+?)%s(?P<y>.+)' % re.escape(op), expr)
+            if not m:
+                continue
+            x, abort = self.interpret_statement(
+                m.group('x'), local_vars, allow_recursion - 1)
+            if abort:
+                raise ExtractorError(
+                    'Premature left-side return of %s in %r' % (op, expr))
+            y, abort = self.interpret_statement(
+                m.group('y'), local_vars, allow_recursion - 1)
+            if abort:
+                raise ExtractorError(
+                    'Premature right-side return of %s in %r' % (op, expr))
+            return opfunc(x, y)
 
         m = re.match(
-            r'^(?P<func>[a-zA-Z$]+)\((?P<args>[a-z0-9,]+)\)$', expr)
+            r'^(?P<func>%s)\((?P<args>[a-zA-Z0-9_$,]+)\)$' % _NAME_RE, expr)
         if m:
             fname = m.group('func')
             argvals = tuple([
@@ -139,6 +210,7 @@ class JSInterpreter(object):
             if fname not in self._functions:
                 self._functions[fname] = self.extract_function(fname)
             return self._functions[fname](argvals)
+
         raise ExtractorError('Unsupported JS expression %r' % expr)
 
     def extract_object(self, objname):
@@ -162,9 +234,11 @@ class JSInterpreter(object):
 
     def extract_function(self, funcname):
         func_m = re.search(
-            (r'(?:function %s|[{;]%s\s*=\s*function)' % (
-                re.escape(funcname), re.escape(funcname))) +
-            r'\((?P<args>[a-z,]+)\){(?P<code>[^}]+)}',
+            r'''(?x)
+                (?:function\s+%s|[{;]%s\s*=\s*function)\s*
+                \((?P<args>[^)]*)\)\s*
+                \{(?P<code>[^}]+)\}''' % (
+                re.escape(funcname), re.escape(funcname)),
             self.code)
         if func_m is None:
             raise ExtractorError('Could not find JS function %r' % funcname)
@@ -172,10 +246,16 @@ class JSInterpreter(object):
 
         return self.build_function(argnames, func_m.group('code'))
 
+    def call_function(self, funcname, *args):
+        f = self.extract_function(funcname)
+        return f(args)
+
     def build_function(self, argnames, code):
         def resf(args):
             local_vars = dict(zip(argnames, args))
             for stmt in code.split(';'):
-                res = self.interpret_statement(stmt, local_vars)
+                res, abort = self.interpret_statement(stmt, local_vars)
+                if abort:
+                    break
             return res
         return resf

From 0a7055c90dac10e7f4c3393922bccf546081262b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 22:42:19 +0100
Subject: [PATCH 0198/6219] release 2015.02.01

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 5e6288e8e..e46ca56a9 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.01.30.2'
+__version__ = '2015.02.01'

From 044131ba213be1bf277561432bcb98a07bbdb265 Mon Sep 17 00:00:00 2001
From: Sahebjot singh <sahebjot94@gmail.com>
Date: Mon, 2 Feb 2015 03:49:28 +0530
Subject: [PATCH 0199/6219] supported sites was not opening

required a .md at the end.
---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 09da0129e..68b41970a 100644
--- a/README.md
+++ b/README.md
@@ -534,7 +534,7 @@ Either prepend `http://www.youtube.com/watch?v=` or separate the ID from the opt
 
 ### How can I detect whether a given URL is supported by youtube-dl?
 
-For one, have a look at the [list of supported sites](docs/supportedsites). Note that it can sometimes happen that the site changes its URL scheme (say, from http://example.com/v/1234567 to http://example.com/v/1234567 ) and youtube-dl reports an URL of a service in that list as unsupported. In that case, simply report a bug.
+For one, have a look at the [list of supported sites](docs/supportedsites.md). Note that it can sometimes happen that the site changes its URL scheme (say, from http://example.com/v/1234567 to http://example.com/v/1234567 ) and youtube-dl reports an URL of a service in that list as unsupported. In that case, simply report a bug.
 
 It is *not* possible to detect whether a URL is supported or not. That's because youtube-dl contains a generic extractor which matches **all** URLs. You may be tempted to disable, exclude, or remove the generic extractor, but the generic extractor not only allows users to extract videos from lots of websites that embed a video from another service, but may also be used to extract video from a service that it's hosting itself. Therefore, we neither recommend nor support disabling, excluding, or removing the generic extractor.
 

From 8cfb6efe6f4a007a4faab132d08717cea0cdfbaf Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 2 Feb 2015 01:49:32 +0100
Subject: [PATCH 0200/6219] [jsinterp] Correct div command

---
 youtube_dl/jsinterp.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index 49364786b..453e2732c 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -17,7 +17,7 @@ _OPERATORS = [
     ('-', operator.sub),
     ('+', operator.add),
     ('%', operator.mod),
-    ('/', operator.div),
+    ('/', operator.truediv),
     ('*', operator.mul),
 ]
 _ASSIGN_OPERATORS = [(op + '=', opfunc) for op, opfunc in _OPERATORS]

From acf2a6e97b53b191edf3f92af1eae300633ea2d9 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 2 Feb 2015 01:49:40 +0100
Subject: [PATCH 0201/6219] release 2015.02.02

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index e46ca56a9..1e469d93b 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.01'
+__version__ = '2015.02.02'

From 12832049170e8b3bd457706f49e8e09de12a368c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 2 Feb 2015 12:05:39 +0100
Subject: [PATCH 0202/6219] [http] PEP8 (#4831)

---
 youtube_dl/downloader/http.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index 9b023c5c1..49170cf9d 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -102,11 +102,11 @@ class HttpFD(FileDownloader):
                             resume_len = 0
                             open_mode = 'wb'
                             break
-
             except SocketError as e:
                 if e.errno != errno.ECONNRESET:
-                    raise # Not error we are looking for
-                pass
+                    # Connection reset is no problem, just retry
+                    raise
+
             # Retry
             count += 1
             if count <= retries:

From 92bf0bcdf847682cee9f66a5cef4c96c19a100d9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 2 Feb 2015 20:52:37 +0600
Subject: [PATCH 0203/6219] [historicfilms] Add extractor (Closes #4825)

---
 youtube_dl/extractor/__init__.py      |  1 +
 youtube_dl/extractor/historicfilms.py | 46 +++++++++++++++++++++++++++
 2 files changed, 47 insertions(+)
 create mode 100644 youtube_dl/extractor/historicfilms.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 9c14b096f..dabf86ec8 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -182,6 +182,7 @@ from .heise import HeiseIE
 from .hellporno import HellPornoIE
 from .helsinki import HelsinkiIE
 from .hentaistigma import HentaiStigmaIE
+from .historicfilms import HistoricFilmsIE
 from .hitbox import HitboxIE, HitboxLiveIE
 from .hornbunny import HornBunnyIE
 from .hostingbulk import HostingBulkIE
diff --git a/youtube_dl/extractor/historicfilms.py b/youtube_dl/extractor/historicfilms.py
new file mode 100644
index 000000000..40afbe537
--- /dev/null
+++ b/youtube_dl/extractor/historicfilms.py
@@ -0,0 +1,46 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import parse_duration
+
+
+class HistoricFilmsIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?historicfilms\.com/(?:tapes/|play)(?P<id>\d+)'
+    _TEST = {
+        'url': 'http://www.historicfilms.com/tapes/4728',
+        'md5': 'd4a437aec45d8d796a38a215db064e9a',
+        'info_dict': {
+            'id': '4728',
+            'ext': 'mov',
+            'title': 'Historic Films: GP-7',
+            'description': 'md5:1a86a0f3ac54024e419aba97210d959a',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 2096,
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        tape_id = self._search_regex(
+            r'class="tapeId">([^<]+)<', webpage, 'tape id')
+
+        title = self._og_search_title(webpage)
+        description = self._og_search_description(webpage)
+        thumbnail = self._html_search_meta(
+            'thumbnailUrl', webpage, 'thumbnails') or self._og_search_thumbnail(webpage)
+        duration = parse_duration(self._html_search_meta(
+            'duration', webpage, 'duration'))
+
+        video_url = 'http://www.historicfilms.com/video/%s_%s_web.mov' % (tape_id, video_id)
+
+        return {
+            'id': video_id,
+            'url': video_url,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'duration': duration,
+        }

From 78271e33197e56452dc60f075bd326cb908e8e7f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 2 Feb 2015 21:11:25 +0600
Subject: [PATCH 0204/6219] [drtv] Extract material id (Closes #4814)

---
 youtube_dl/extractor/drtv.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/drtv.py b/youtube_dl/extractor/drtv.py
index 8d113a5e8..d5df18d7c 100644
--- a/youtube_dl/extractor/drtv.py
+++ b/youtube_dl/extractor/drtv.py
@@ -25,9 +25,15 @@ class DRTVIE(SubtitlesInfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        programcard = self._download_json(
-            'http://www.dr.dk/mu/programcard/expanded/%s' % video_id, video_id, 'Downloading video JSON')
+        webpage = self._download_webpage(url, video_id)
 
+        video_id = self._search_regex(
+            r'data-(?:material-identifier|episode-slug)="([^"]+)"',
+            webpage, 'video id')
+
+        programcard = self._download_json(
+            'http://www.dr.dk/mu/programcard/expanded/%s' % video_id,
+            video_id, 'Downloading video JSON')
         data = programcard['Data'][0]
 
         title = data['Title']

From 6a66904f8ed469a547d8781fd61b26a97a6ae7aa Mon Sep 17 00:00:00 2001
From: vijayanand nandam <vijay@cybrilla.com>
Date: Mon, 2 Feb 2015 20:51:40 +0530
Subject: [PATCH 0205/6219] passing source address option to external
 downloaders

---
 youtube_dl/downloader/external.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index af9fdba75..3d6554fe4 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -45,6 +45,12 @@ class ExternalFD(FileDownloader):
     def supports(cls, info_dict):
         return info_dict['protocol'] in ('http', 'https', 'ftp', 'ftps')
 
+    def _source_address(self, command_option):
+        command_part = []
+        if self.ydl.params['source_address'] is not None:
+            command_part = [command_option, self.ydl.params['source_address']]
+        return command_part
+
     def _call_downloader(self, tmpfilename, info_dict):
         """ Either overwrite this or implement _make_cmd """
         cmd = self._make_cmd(tmpfilename, info_dict)
@@ -72,6 +78,7 @@ class CurlFD(ExternalFD):
         cmd = [self.exe, '-o', tmpfilename]
         for key, val in info_dict['http_headers'].items():
             cmd += ['--header', '%s: %s' % (key, val)]
+        cmd += self._source_address('--interface')
         cmd += ['--', info_dict['url']]
         return cmd
 
@@ -81,6 +88,7 @@ class WgetFD(ExternalFD):
         cmd = [self.exe, '-O', tmpfilename, '-nv', '--no-cookies']
         for key, val in info_dict['http_headers'].items():
             cmd += ['--header', '%s: %s' % (key, val)]
+        cmd += self._source_address('--bind-address')
         cmd += ['--', info_dict['url']]
         return cmd
 
@@ -96,6 +104,7 @@ class Aria2cFD(ExternalFD):
         cmd += ['--out', os.path.basename(tmpfilename)]
         for key, val in info_dict['http_headers'].items():
             cmd += ['--header', '%s: %s' % (key, val)]
+        cmd += self._source_address('--interface')
         cmd += ['--', info_dict['url']]
         return cmd
 

From 469a64cebf534b702e50ad9585e63ae3f8ed5d16 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 2 Feb 2015 21:40:52 +0600
Subject: [PATCH 0206/6219] [downloader/external] Simplify

---
 youtube_dl/downloader/external.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index 3d6554fe4..012057ecb 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -47,8 +47,9 @@ class ExternalFD(FileDownloader):
 
     def _source_address(self, command_option):
         command_part = []
-        if self.ydl.params['source_address'] is not None:
-            command_part = [command_option, self.ydl.params['source_address']]
+        source_address = self.ydl.params.get('source_address')
+        if source_address:
+            command_part = [command_option, source_address]
         return command_part
 
     def _call_downloader(self, tmpfilename, info_dict):

From ae7773942ecf816e9592e55bdef7234f41c89c85 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 2 Feb 2015 21:51:38 +0600
Subject: [PATCH 0207/6219] [downloader/external] Simplify

---
 youtube_dl/downloader/external.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index 012057ecb..faac11805 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -47,7 +47,7 @@ class ExternalFD(FileDownloader):
 
     def _source_address(self, command_option):
         command_part = []
-        source_address = self.ydl.params.get('source_address')
+        source_address = self.params.get('source_address')
         if source_address:
             command_part = [command_option, source_address]
         return command_part

From 9fffd0469f1353c5f5e96b91a87e34d02ae7491a Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 2 Feb 2015 20:28:18 +0100
Subject: [PATCH 0208/6219] [options] Mark --fixup as non-experimental and
 correct its help

---
 youtube_dl/downloader/external.py | 2 +-
 youtube_dl/options.py             | 5 ++---
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index 012057ecb..e47e3bf4c 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -48,7 +48,7 @@ class ExternalFD(FileDownloader):
     def _source_address(self, command_option):
         command_part = []
         source_address = self.ydl.params.get('source_address')
-        if source_address:
+        if source_address is None:
             command_part = [command_option, source_address]
         return command_part
 
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 5e4a1ad9b..d5e575964 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -698,10 +698,9 @@ def parseOpts(overrideArguments=None):
     postproc.add_option(
         '--fixup',
         metavar='POLICY', dest='fixup', default='detect_or_warn',
-        help='(experimental) Automatically correct known faults of the file. '
+        help='Automatically correct known faults of the file. '
              'One of never (do nothing), warn (only emit a warning), '
-             'detect_or_warn(check whether we can do anything about it, warn '
-             'otherwise')
+             'detect_or_warn(the default; fix file if we can, warn otherwise)')
     postproc.add_option(
         '--prefer-avconv',
         action='store_false', dest='prefer_ffmpeg',

From cc9ca3ba6e0ba24b318ef191fdf84927e773008d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 2 Feb 2015 20:33:23 +0100
Subject: [PATCH 0209/6219] [downloader/external] Simplify source_address

'' might actually be passed in, so check for None.
---
 youtube_dl/downloader/external.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index faac11805..ff031d2e0 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -46,11 +46,10 @@ class ExternalFD(FileDownloader):
         return info_dict['protocol'] in ('http', 'https', 'ftp', 'ftps')
 
     def _source_address(self, command_option):
-        command_part = []
         source_address = self.params.get('source_address')
-        if source_address:
-            command_part = [command_option, source_address]
-        return command_part
+        if source_address is None:
+            return []
+        return [command_option, source_address]
 
     def _call_downloader(self, tmpfilename, info_dict):
         """ Either overwrite this or implement _make_cmd """

From 869b4aeff4239b02ff8f699741902fe7f91b999a Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 2 Feb 2015 20:35:04 +0100
Subject: [PATCH 0210/6219] release 2015.02.02.1

---
 README.md             | 10 +++++-----
 youtube_dl/version.py |  2 +-
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/README.md b/README.md
index 68b41970a..d432904fa 100644
--- a/README.md
+++ b/README.md
@@ -368,11 +368,11 @@ which means you can modify it, redistribute it or use it however you like.
     --add-metadata                   write metadata to the video file
     --xattrs                         write metadata to the video file's xattrs
                                      (using dublin core and xdg standards)
-    --fixup POLICY                   (experimental) Automatically correct known
-                                     faults of the file. One of never (do
-                                     nothing), warn (only emit a warning),
-                                     detect_or_warn(check whether we can do
-                                     anything about it, warn otherwise
+    --fixup POLICY                   Automatically correct known faults of the
+                                     file. One of never (do nothing), warn (only
+                                     emit a warning), detect_or_warn(the
+                                     default; fix file if we can, warn
+                                     otherwise)
     --prefer-avconv                  Prefer avconv over ffmpeg for running the
                                      postprocessors (default)
     --prefer-ffmpeg                  Prefer ffmpeg over avconv for running the
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 1e469d93b..94c181a46 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.02'
+__version__ = '2015.02.02.1'

From 3fd45e03bf86c2ad2ef17362452a9a745dc10755 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 2 Feb 2015 20:43:02 +0100
Subject: [PATCH 0211/6219] [ntvru] Rename from NTV to clarify the difference
 between n-tv.de and ntv.ru

---
 youtube_dl/extractor/__init__.py          | 2 +-
 youtube_dl/extractor/{ntv.py => ntvru.py} | 6 ++----
 2 files changed, 3 insertions(+), 5 deletions(-)
 rename youtube_dl/extractor/{ntv.py => ntvru.py} (97%)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index dabf86ec8..bd707e028 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -317,7 +317,7 @@ from .nrk import (
     NRKIE,
     NRKTVIE,
 )
-from .ntv import NTVIE
+from .ntvru import NTVRuIE
 from .nytimes import NYTimesIE
 from .nuvid import NuvidIE
 from .oktoberfesttv import OktoberfestTVIE
diff --git a/youtube_dl/extractor/ntv.py b/youtube_dl/extractor/ntvru.py
similarity index 97%
rename from youtube_dl/extractor/ntv.py
rename to youtube_dl/extractor/ntvru.py
index ee740cd9c..8b9166d48 100644
--- a/youtube_dl/extractor/ntv.py
+++ b/youtube_dl/extractor/ntvru.py
@@ -9,7 +9,7 @@ from ..utils import (
 )
 
 
-class NTVIE(InfoExtractor):
+class NTVRuIE(InfoExtractor):
     _VALID_URL = r'http://(?:www\.)?ntv\.ru/(?P<id>.+)'
 
     _TESTS = [
@@ -92,9 +92,7 @@ class NTVIE(InfoExtractor):
     ]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
+        video_id = self._match_id(url)
         page = self._download_webpage(url, video_id)
 
         video_id = self._html_search_regex(self._VIDEO_ID_REGEXES, page, 'video id')

From 8f4b58d70e29df13d6002579b5c9448acc89d6c1 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 2 Feb 2015 21:48:54 +0100
Subject: [PATCH 0212/6219] [ntvde] Add new extractor (Fixes #4850)

---
 test/test_utils.py               | 12 ++++++
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/ntvde.py    | 65 ++++++++++++++++++++++++++++++++
 youtube_dl/extractor/ntvru.py    |  1 +
 youtube_dl/utils.py              |  9 ++++-
 5 files changed, 86 insertions(+), 2 deletions(-)
 create mode 100644 youtube_dl/extractor/ntvde.py

diff --git a/test/test_utils.py b/test/test_utils.py
index ebec7986f..0ffccd35f 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -238,6 +238,8 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(parse_duration('5 s'), 5)
         self.assertEqual(parse_duration('3 min'), 180)
         self.assertEqual(parse_duration('2.5 hours'), 9000)
+        self.assertEqual(parse_duration('02:03:04'), 7384)
+        self.assertEqual(parse_duration('01:02:03:04'), 93784)
 
     def test_fix_xml_ampersands(self):
         self.assertEqual(
@@ -371,6 +373,16 @@ class TestUtil(unittest.TestCase):
         on = js_to_json('{"abc": true}')
         self.assertEqual(json.loads(on), {'abc': True})
 
+        # Ignore JavaScript code as well
+        on = js_to_json('''{
+            "x": 1,
+            y: "a",
+            z: some.code
+        }''')
+        d = json.loads(on)
+        self.assertEqual(d['x'], 1)
+        self.assertEqual(d['y'], 'a')
+
     def test_clean_html(self):
         self.assertEqual(clean_html('a:\nb'), 'a: b')
         self.assertEqual(clean_html('a:\n   "b"'), 'a:    "b"')
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index bd707e028..205bd4338 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -317,6 +317,7 @@ from .nrk import (
     NRKIE,
     NRKTVIE,
 )
+from .ntvde import NTVDeIE
 from .ntvru import NTVRuIE
 from .nytimes import NYTimesIE
 from .nuvid import NuvidIE
diff --git a/youtube_dl/extractor/ntvde.py b/youtube_dl/extractor/ntvde.py
new file mode 100644
index 000000000..a4f628fcd
--- /dev/null
+++ b/youtube_dl/extractor/ntvde.py
@@ -0,0 +1,65 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    js_to_json,
+    parse_duration,
+)
+
+
+class NTVDeIE(InfoExtractor):
+    IE_NAME = 'n-tv.de'
+    _VALID_URL = r'https?://(?:www\.)?n-tv\.de/mediathek/videos/[^/?#]+/[^/?#]+-article(?P<id>.+)\.html'
+
+    _TESTS = [{
+        'url': 'http://www.n-tv.de/mediathek/videos/panorama/Schnee-und-Glaette-fuehren-zu-zahlreichen-Unfaellen-und-Staus-article14438086.html',
+        'md5': 'd37b7df1eea32265c51a062499ca488f',
+        'info_dict': {
+            'id': '14438086',
+            'ext': 'mp4',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'title': 'Schnee und Glätte führen zu zahlreichen Unfällen und Staus',
+            'alt_title': 'Winterchaos auf deutschen Straßen',
+            'description': 'Schnee und Glätte sorgen deutschlandweit für einen chaotischen Start in die Woche: Auf den Straßen kommt es zu kilometerlangen Staus und Dutzenden Glätteunfällen. In Düsseldorf und München wirbelt der Schnee zudem den Flugplan durcheinander. Dutzende Flüge landen zu spät, einige fallen ganz aus.',
+            'duration': 4020,
+            'timestamp': 1422892797,
+            'upload_date': '20150202',
+        },
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        info = self._parse_json(self._search_regex(
+            r'(?s)ntv.pageInfo.article =\s(\{.*?\});', webpage, 'info'),
+            video_id, transform_source=js_to_json)
+        timestamp = int_or_none(info.get('publishedDateAsUnixTimeStamp'))
+        vdata = self._parse_json(self._search_regex(
+            r'(?s)\$\(\s*"\#player"\s*\)\s*\.data\(\s*"player",\s*(\{.*?\})\);',
+            webpage, 'player data'),
+            video_id, transform_source=js_to_json)
+        duration = parse_duration(vdata.get('duration'))
+        formats = [{
+            'format_id': 'flash',
+            'url': 'rtmp://fms.n-tv.de/' + vdata['video'],
+        }, {
+            'format_id': 'mp4',
+            'url': 'http://video.n-tv.de' + vdata['videoMp4'],
+        }]
+        m3u8_url = 'http://video.n-tv.de' + vdata['videoM3u8']
+        formats.extend(self._extract_m3u8_formats(m3u8_url, video_id))
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': info['headline'],
+            'description': info.get('intro'),
+            'alt_title': info.get('kicker'),
+            'timestamp': timestamp,
+            'thumbnail': vdata.get('html5VideoPoster'),
+            'duration': duration,
+            'formats': formats,
+        }
diff --git a/youtube_dl/extractor/ntvru.py b/youtube_dl/extractor/ntvru.py
index 8b9166d48..963bb715a 100644
--- a/youtube_dl/extractor/ntvru.py
+++ b/youtube_dl/extractor/ntvru.py
@@ -10,6 +10,7 @@ from ..utils import (
 
 
 class NTVRuIE(InfoExtractor):
+    IE_NAME = 'ntv.ru'
     _VALID_URL = r'http://(?:www\.)?ntv\.ru/(?P<id>.+)'
 
     _TESTS = [
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 4ade0554e..251074bf5 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1275,7 +1275,10 @@ def parse_duration(s):
             (?P<only_hours>[0-9.]+)\s*(?:hours?)|
 
             (?:
-                (?:(?P<hours>[0-9]+)\s*(?:[:h]|hours?)\s*)?
+                (?:
+                    (?:(?P<days>[0-9]+)\s*(?:[:d]|days?)\s*)?
+                    (?P<hours>[0-9]+)\s*(?:[:h]|hours?)\s*
+                )?
                 (?P<mins>[0-9]+)\s*(?:[:m]|mins?|minutes?)\s*
             )?
             (?P<secs>[0-9]+)(?P<ms>\.[0-9]+)?\s*(?:s|secs?|seconds?)?
@@ -1293,6 +1296,8 @@ def parse_duration(s):
         res += int(m.group('mins')) * 60
     if m.group('hours'):
         res += int(m.group('hours')) * 60 * 60
+    if m.group('days'):
+        res += int(m.group('days')) * 24 * 60 * 60
     if m.group('ms'):
         res += float(m.group('ms'))
     return res
@@ -1543,7 +1548,7 @@ def js_to_json(code):
     res = re.sub(r'''(?x)
         "(?:[^"\\]*(?:\\\\|\\")?)*"|
         '(?:[^'\\]*(?:\\\\|\\')?)*'|
-        [a-zA-Z_][a-zA-Z_0-9]*
+        [a-zA-Z_][.a-zA-Z_0-9]*
         ''', fix_kv, code)
     res = re.sub(r',(\s*\])', lambda m: m.group(1), res)
     return res

From 7a7bd19c4537a4abd0ccea166610c9f2353046a4 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 2 Feb 2015 21:57:48 +0100
Subject: [PATCH 0213/6219] [n-tv.de] Use native m3u8 as best format

---
 youtube_dl/extractor/ntvde.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/ntvde.py b/youtube_dl/extractor/ntvde.py
index a4f628fcd..d2cfe0961 100644
--- a/youtube_dl/extractor/ntvde.py
+++ b/youtube_dl/extractor/ntvde.py
@@ -15,7 +15,7 @@ class NTVDeIE(InfoExtractor):
 
     _TESTS = [{
         'url': 'http://www.n-tv.de/mediathek/videos/panorama/Schnee-und-Glaette-fuehren-zu-zahlreichen-Unfaellen-und-Staus-article14438086.html',
-        'md5': 'd37b7df1eea32265c51a062499ca488f',
+        'md5': '6ef2514d4b1e8e03ca24b49e2f167153',
         'info_dict': {
             'id': '14438086',
             'ext': 'mp4',
@@ -46,11 +46,14 @@ class NTVDeIE(InfoExtractor):
             'format_id': 'flash',
             'url': 'rtmp://fms.n-tv.de/' + vdata['video'],
         }, {
-            'format_id': 'mp4',
+            'format_id': 'mobile',
             'url': 'http://video.n-tv.de' + vdata['videoMp4'],
+            'tbr': 400,  # estimation
         }]
         m3u8_url = 'http://video.n-tv.de' + vdata['videoM3u8']
-        formats.extend(self._extract_m3u8_formats(m3u8_url, video_id))
+        formats.extend(self._extract_m3u8_formats(
+            m3u8_url, video_id, ext='mp4',
+            entry_protocol='m3u8_native', preference=0))
         self._sort_formats(formats)
 
         return {

From f920ce295e65dc1446120c9375ef11bfd6cede26 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 2 Feb 2015 21:58:17 +0100
Subject: [PATCH 0214/6219] [ntvru] Remove unused import

---
 youtube_dl/extractor/ntvru.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/youtube_dl/extractor/ntvru.py b/youtube_dl/extractor/ntvru.py
index 963bb715a..0ab8d5100 100644
--- a/youtube_dl/extractor/ntvru.py
+++ b/youtube_dl/extractor/ntvru.py
@@ -1,8 +1,6 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..utils import (
     unescapeHTML

From c21b1fbeeb0305fbf7804fcad9d4aa2dc6288de6 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 2 Feb 2015 21:58:58 +0100
Subject: [PATCH 0215/6219] release 2015.02.02.2

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 94c181a46..c1d17844e 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.02.1'
+__version__ = '2015.02.02.2'

From 7d3d06a16cb585be35c2152f4bd4cba9c32081a3 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 2 Feb 2015 22:48:12 +0100
Subject: [PATCH 0216/6219] [vevo] Restore SMIL support (#3656)

---
 youtube_dl/extractor/vevo.py | 25 +++++++++++++++++++++++--
 1 file changed, 23 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py
index 43f6b029d..c17094f81 100644
--- a/youtube_dl/extractor/vevo.py
+++ b/youtube_dl/extractor/vevo.py
@@ -9,6 +9,7 @@ from ..compat import (
 )
 from ..utils import (
     ExtractorError,
+    int_or_none,
 )
 
 
@@ -192,9 +193,29 @@ class VevoIE(InfoExtractor):
         # Download via HLS API
         formats.extend(self._download_api_formats(video_id))
 
+        # Download SMIL
+        smil_blocks = sorted((
+            f for f in video_info['videoVersions']
+            if f['sourceType'] == 13),
+            key=lambda f: f['version'])
+        smil_url = '%s/Video/V2/VFILE/%s/%sr.smil' % (
+            self._SMIL_BASE_URL, video_id, video_id.lower())
+        if smil_blocks:
+            smil_url_m = self._search_regex(
+                r'url="([^"]+)"', smil_blocks[-1]['data'], 'SMIL URL',
+                default=None)
+            if smil_url_m is not None:
+                smil_url = smil_url_m
+        if smil_url:
+            smil_xml = self._download_webpage(
+                smil_url, video_id, 'Downloading SMIL info', fatal=False)
+            if smil_xml:
+                formats.extend(self._formats_from_smil(smil_xml))
+
         self._sort_formats(formats)
-        timestamp_ms = int(self._search_regex(
-            r'/Date\((\d+)\)/', video_info['launchDate'], 'launch date'))
+        timestamp_ms = int_or_none(self._search_regex(
+            r'/Date\((\d+)\)/',
+            video_info['launchDate'], 'launch date', fatal=False))
 
         return {
             'id': video_id,

From 3a4cca687f674fd5b93fafbe80453b23ef7ea748 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 2 Feb 2015 22:56:15 +0100
Subject: [PATCH 0217/6219] release 2015.02.02.3

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index c1d17844e..dadec36d3 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.02.2'
+__version__ = '2015.02.02.3'

From 531572578e542fcee26f1a30fd9f87ad158e6594 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 2 Feb 2015 23:04:39 +0100
Subject: [PATCH 0218/6219] [normalboots] Modernize

---
 youtube_dl/extractor/normalboots.py | 25 ++++++++++++++-----------
 1 file changed, 14 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/normalboots.py b/youtube_dl/extractor/normalboots.py
index c13ff0d65..78e75266f 100644
--- a/youtube_dl/extractor/normalboots.py
+++ b/youtube_dl/extractor/normalboots.py
@@ -11,7 +11,7 @@ from ..utils import (
 
 
 class NormalbootsIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?normalboots\.com/video/(?P<videoid>[0-9a-z-]*)/?$'
+    _VALID_URL = r'http://(?:www\.)?normalboots\.com/video/(?P<id>[0-9a-z-]*)/?$'
     _TEST = {
         'url': 'http://normalboots.com/video/home-alone-games-jontron/',
         'md5': '8bf6de238915dd501105b44ef5f1e0f6',
@@ -30,19 +30,22 @@ class NormalbootsIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('videoid')
-
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
-        video_uploader = self._html_search_regex(r'Posted\sby\s<a\shref="[A-Za-z0-9/]*">(?P<uploader>[A-Za-z]*)\s</a>',
-                                                 webpage, 'uploader')
-        raw_upload_date = self._html_search_regex('<span style="text-transform:uppercase; font-size:inherit;">[A-Za-z]+, (?P<date>.*)</span>',
-                                                  webpage, 'date')
-        video_upload_date = unified_strdate(raw_upload_date)
 
-        player_url = self._html_search_regex(r'<iframe\swidth="[0-9]+"\sheight="[0-9]+"\ssrc="(?P<url>[\S]+)"', webpage, 'url')
+        video_uploader = self._html_search_regex(
+            r'Posted\sby\s<a\shref="[A-Za-z0-9/]*">(?P<uploader>[A-Za-z]*)\s</a>',
+            webpage, 'uploader', fatal=False)
+        video_upload_date = unified_strdate(self._html_search_regex(
+            r'<span style="text-transform:uppercase; font-size:inherit;">[A-Za-z]+, (?P<date>.*)</span>',
+            webpage, 'date', fatal=False))
+
+        player_url = self._html_search_regex(
+            r'<iframe\swidth="[0-9]+"\sheight="[0-9]+"\ssrc="(?P<url>[\S]+)"',
+            webpage, 'player url')
         player_page = self._download_webpage(player_url, video_id)
-        video_url = self._html_search_regex(r"file:\s'(?P<file>[^']+\.mp4)'", player_page, 'file')
+        video_url = self._html_search_regex(
+            r"file:\s'(?P<file>[^']+\.mp4)'", player_page, 'file')
 
         return {
             'id': video_id,

From a57b562cff57a5cbd06cb22b37962861a423e428 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 2 Feb 2015 23:13:28 +0100
Subject: [PATCH 0219/6219] [nfl] Add support for articles pages (fixes #4848)

---
 youtube_dl/extractor/nfl.py | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/nfl.py b/youtube_dl/extractor/nfl.py
index 606e2294e..2684dd250 100644
--- a/youtube_dl/extractor/nfl.py
+++ b/youtube_dl/extractor/nfl.py
@@ -46,7 +46,18 @@ class NFLIE(InfoExtractor):
                 'timestamp': 1388354455,
                 'thumbnail': 're:^https?://.*\.jpg$',
             }
-        }
+        },
+        {
+            'url': 'http://www.nfl.com/news/story/0ap3000000467586/article/patriots-seahawks-involved-in-lategame-skirmish',
+            'info_dict': {
+                'id': '0ap3000000467607',
+                'ext': 'mp4',
+                'title': 'Frustrations flare on the field',
+                'description': 'Emotions ran high at the end of the Super Bowl on both sides of the ball after a dramatic finish.',
+                'timestamp': 1422850320,
+                'upload_date': '20150202',
+            },
+        },
     ]
 
     @staticmethod
@@ -80,7 +91,11 @@ class NFLIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         config_url = NFLIE.prepend_host(host, self._search_regex(
-            r'(?:config|configURL)\s*:\s*"([^"]+)"', webpage, 'config URL'))
+            r'(?:config|configURL)\s*:\s*"([^"]+)"', webpage, 'config URL',
+            default='static/content/static/config/video/config.json'))
+        # For articles, the id in the url is not the video id
+        video_id = self._search_regex(
+            r'contentId\s*:\s*"([^"]+)"', webpage, 'video id', default=video_id)
         config = self._download_json(config_url, video_id,
                                      note='Downloading player config')
         url_template = NFLIE.prepend_host(

From 995029a142cd0046ee43b583f2d09fc3d5fa321a Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 2 Feb 2015 23:38:35 +0100
Subject: [PATCH 0220/6219] [nerdist] Add new extractor (Fixes #4851)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/common.py   |  7 ++-
 youtube_dl/extractor/nerdist.py  | 80 ++++++++++++++++++++++++++++++++
 3 files changed, 86 insertions(+), 2 deletions(-)
 create mode 100644 youtube_dl/extractor/nerdist.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 205bd4338..5dcb14feb 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -285,6 +285,7 @@ from .ndr import NDRIE
 from .ndtv import NDTVIE
 from .netzkino import NetzkinoIE
 from .nerdcubed import NerdCubedFeedIE
+from .nerdist import NerdistIE
 from .newgrounds import NewgroundsIE
 from .newstube import NewstubeIE
 from .nextmedia import (
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 8b4ef3f09..9e517e3ac 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -860,10 +860,13 @@ class InfoExtractor(object):
         return formats
 
     # TODO: improve extraction
-    def _extract_smil_formats(self, smil_url, video_id):
+    def _extract_smil_formats(self, smil_url, video_id, fatal=True):
         smil = self._download_xml(
             smil_url, video_id, 'Downloading SMIL file',
-            'Unable to download SMIL file')
+            'Unable to download SMIL file', fatal=fatal)
+        if smil is False:
+            assert not fatal
+            return []
 
         base = smil.find('./head/meta').get('base')
 
diff --git a/youtube_dl/extractor/nerdist.py b/youtube_dl/extractor/nerdist.py
new file mode 100644
index 000000000..c6dc34be4
--- /dev/null
+++ b/youtube_dl/extractor/nerdist.py
@@ -0,0 +1,80 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+from ..utils import (
+    determine_ext,
+    parse_iso8601,
+    xpath_text,
+)
+
+
+class NerdistIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?nerdist\.com/vepisode/(?P<id>[^/?#]+)'
+    _TEST = {
+        'url': 'http://www.nerdist.com/vepisode/exclusive-which-dc-characters-w',
+        'md5': '3698ed582931b90d9e81e02e26e89f23',
+        'info_dict': {
+            'display_id': 'exclusive-which-dc-characters-w',
+            'id': 'RPHpvJyr',
+            'ext': 'mp4',
+            'title': 'Your TEEN TITANS Revealed! Who\'s on the show?',
+            'thumbnail': 're:^https?://.*/thumbs/.*\.jpg$',
+            'description': 'Exclusive: Find out which DC Comics superheroes will star in TEEN TITANS Live-Action TV Show on Nerdist News with Jessica Chobot!',
+            'uploader': 'Eric Diaz',
+            'upload_date': '20150202',
+            'timestamp': 1422892808,
+        }
+    }
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+
+        video_id = self._search_regex(
+            r'''(?x)<script\s+(?:type="text/javascript"\s+)?
+                src="https?://content\.nerdist\.com/players/([a-zA-Z0-9_]+)-''',
+            webpage, 'video ID')
+        timestamp = parse_iso8601(self._html_search_meta(
+            'shareaholic:article_published_time', webpage, 'upload date'))
+        uploader = self._html_search_meta(
+            'shareaholic:article_author_name', webpage, 'article author')
+
+        doc = self._download_xml(
+            'http://content.nerdist.com/jw6/%s.xml' % video_id, video_id)
+        video_info = doc.find('.//item')
+        title = xpath_text(video_info, './title', fatal=True)
+        description = xpath_text(video_info, './description')
+        thumbnail = xpath_text(
+            video_info, './{http://rss.jwpcdn.com/}image', 'thumbnail')
+
+        formats = []
+        for source in video_info.findall('./{http://rss.jwpcdn.com/}source'):
+            vurl = source.attrib['file']
+            ext = determine_ext(vurl)
+            if ext == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    vurl, video_id, entry_protocol='m3u8_native', ext='mp4',
+                    preference=0))
+            elif ext == 'smil':
+                formats.extend(self._extract_smil_formats(
+                    vurl, video_id, fatal=False
+                ))
+            else:
+                formats.append({
+                    'format_id': ext,
+                    'url': vurl,
+                })
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'timestamp': timestamp,
+            'formats': formats,
+            'uploader': uploader,
+        }

From 5c4862bad4ba38f4ddd86d4fb1305aa5b82b2bd6 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 2 Feb 2015 23:38:45 +0100
Subject: [PATCH 0221/6219] [normalboots] Remove unused import

---
 youtube_dl/extractor/normalboots.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/youtube_dl/extractor/normalboots.py b/youtube_dl/extractor/normalboots.py
index 78e75266f..5952d136f 100644
--- a/youtube_dl/extractor/normalboots.py
+++ b/youtube_dl/extractor/normalboots.py
@@ -1,8 +1,6 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 
 from ..utils import (

From 55898ad2cf4a5cf7628cf921fb5785ed0c7dceff Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 2 Feb 2015 23:39:03 +0100
Subject: [PATCH 0222/6219] release 2015.02.02.4

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index dadec36d3..22464a358 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.02.3'
+__version__ = '2015.02.02.4'

From c1147c05e172f8d8c2b2b8d752eda3f385f1bed1 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 2 Feb 2015 23:47:14 +0100
Subject: [PATCH 0223/6219] [brightcove] Fix up more generically invalid XML
 (Fixes #4849)

---
 youtube_dl/extractor/brightcove.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 003152c4e..ea0969d4d 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -108,7 +108,7 @@ class BrightcoveIE(InfoExtractor):
         """
 
         # Fix up some stupid HTML, see https://github.com/rg3/youtube-dl/issues/1553
-        object_str = re.sub(r'(<param name="[^"]+" value="[^"]+")>',
+        object_str = re.sub(r'(<param(?:\s+[a-zA-Z0-9_]+="[^"]*")*)>',
                             lambda m: m.group(1) + '/>', object_str)
         # Fix up some stupid XML, see https://github.com/rg3/youtube-dl/issues/1608
         object_str = object_str.replace('<--', '<!--')

From d0eca21021b6321197caccab110de13843be6e04 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 2 Feb 2015 23:47:19 +0100
Subject: [PATCH 0224/6219] release 2015.02.02.5

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 22464a358..46af42171 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.02.4'
+__version__ = '2015.02.02.5'

From d200b11c7ea34138d9b8a778fd2dd9226308b48f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 3 Feb 2015 00:14:42 +0100
Subject: [PATCH 0225/6219] [Makefile] Simplify clean/cleanall

---
 Makefile              | 5 +----
 devscripts/release.sh | 2 +-
 2 files changed, 2 insertions(+), 5 deletions(-)

diff --git a/Makefile b/Makefile
index 578079879..0636fc4cb 100644
--- a/Makefile
+++ b/Makefile
@@ -1,10 +1,7 @@
 all: youtube-dl README.md CONTRIBUTING.md README.txt youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtube-dl.fish supportedsites
 
 clean:
-	rm -rf youtube-dl.1.temp.md youtube-dl.1 youtube-dl.bash-completion README.txt MANIFEST build/ dist/ .coverage cover/ youtube-dl.tar.gz youtube-dl.zsh youtube-dl.fish *.dump *.part *.info.json CONTRIBUTING.md.tmp
-
-cleanall: clean
-	rm -f youtube-dl youtube-dl.exe
+	rm -rf youtube-dl.1.temp.md youtube-dl.1 youtube-dl.bash-completion README.txt MANIFEST build/ dist/ .coverage cover/ youtube-dl.tar.gz youtube-dl.zsh youtube-dl.fish *.dump *.part *.info.json *.mp4 *.flv *.mp3 CONTRIBUTING.md.tmp youtube-dl youtube-dl.exe
 
 PREFIX ?= /usr/local
 BINDIR ?= $(PREFIX)/bin
diff --git a/devscripts/release.sh b/devscripts/release.sh
index 691517ceb..47f54c414 100755
--- a/devscripts/release.sh
+++ b/devscripts/release.sh
@@ -35,7 +35,7 @@ if [ ! -z "$useless_files" ]; then echo "ERROR: Non-.py files in youtube_dl: $us
 if [ ! -f "updates_key.pem" ]; then echo 'ERROR: updates_key.pem missing'; exit 1; fi
 
 /bin/echo -e "\n### First of all, testing..."
-make cleanall
+make clean
 if $skip_tests ; then
     echo 'SKIPPING TESTS'
 else

From 2cf0ecac7b11e68a94990ce7c0e4a661cc2d8c74 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 3 Feb 2015 00:16:45 +0100
Subject: [PATCH 0226/6219] [ffmpeg] --add-metadata: Set comment and purl
 fields (Fixes #4847)

---
 youtube_dl/postprocessor/ffmpeg.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 855d1e6db..4a4422c5a 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -511,8 +511,9 @@ class FFmpegMetadataPP(FFmpegPostProcessor):
             metadata['artist'] = info['uploader_id']
         if info.get('description') is not None:
             metadata['description'] = info['description']
+            metadata['comment'] = info['description']
         if info.get('webpage_url') is not None:
-            metadata['comment'] = info['webpage_url']
+            metadata['purl'] = info['webpage_url']
 
         if not metadata:
             self._downloader.to_screen('[ffmpeg] There isn\'t any metadata to add')

From 330537d08aa9875c9d66354f2c9bed7aeaf86297 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 3 Feb 2015 00:20:57 +0100
Subject: [PATCH 0227/6219] [README] typo

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index d432904fa..c1221417e 100644
--- a/README.md
+++ b/README.md
@@ -728,7 +728,7 @@ In particular, every site support request issue should only pertain to services
 
 ###  Is anyone going to need the feature?
 
-Only post features that you (or an incapicated friend you can personally talk to) require. Do not post features because they seem like a good idea. If they are really useful, they will be requested by someone who requires them.
+Only post features that you (or an incapacitated friend you can personally talk to) require. Do not post features because they seem like a good idea. If they are really useful, they will be requested by someone who requires them.
 
 ###  Is your question about youtube-dl?
 

From ee623d9247752e11e98829834ba0e3308a21df52 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 3 Feb 2015 00:22:17 +0100
Subject: [PATCH 0228/6219] [descripts/release] Regenerate auxiliary
 documentation on build as well

---
 devscripts/release.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/devscripts/release.sh b/devscripts/release.sh
index 47f54c414..61806961c 100755
--- a/devscripts/release.sh
+++ b/devscripts/release.sh
@@ -45,9 +45,9 @@ fi
 /bin/echo -e "\n### Changing version in version.py..."
 sed -i "s/__version__ = '.*'/__version__ = '$version'/" youtube_dl/version.py
 
-/bin/echo -e "\n### Committing README.md and youtube_dl/version.py..."
-make README.md
-git add README.md youtube_dl/version.py
+/bin/echo -e "\n### Committing documentation and youtube_dl/version.py..."
+make README.md CONTRIBUTING.md supportedsites
+git add README.md CONTRIBUTING.md docs/supportedsites.md youtube_dl/version.py
 git commit -m "release $version"
 
 /bin/echo -e "\n### Now tagging, signing and pushing..."

From 4c1a017e6961c18fdef13d49bcdd0daa08ab0575 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 3 Feb 2015 00:22:52 +0100
Subject: [PATCH 0229/6219] release 2015.02.03

---
 CONTRIBUTING.md        |  6 +++--
 docs/supportedsites.md | 59 +++++++++++++++++++++++++++++++++++++-----
 youtube_dl/version.py  |  2 +-
 3 files changed, 58 insertions(+), 9 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 7917abfc6..351229f21 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -1,4 +1,6 @@
-Please include the full output of the command when run with `--verbose`. The output (including the first lines) contain important debugging information. Issues without the full output are often not reproducible and therefore do not get solved in short order, if ever.
+**Please include the full output of youtube-dl when run with `-v`**.
+
+The output (including the first lines) contain important debugging information. Issues without the full output are often not reproducible and therefore do not get solved in short order, if ever.
 
 Please re-read your issue once again to avoid a couple of common mistakes (you can and should use this as a checklist):
 
@@ -122,7 +124,7 @@ If you want to add support for a new site, you can follow this quick list (assum
 5. Add an import in [`youtube_dl/extractor/__init__.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/__init__.py).
 6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will be then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc.
 7. Have a look at [`youtube_dl/common/extractor/common.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should return](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L38). Add tests and code for as many as you want.
-8. If you can, check the code with [pyflakes](https://pypi.python.org/pypi/pyflakes) (a good idea) and [pep8](https://pypi.python.org/pypi/pep8) (optional, ignore E501).
+8. If you can, check the code with [flake8](https://pypi.python.org/pypi/flake8).
 9. When the tests pass, [add](http://git-scm.com/docs/git-add) the new files and [commit](http://git-scm.com/docs/git-commit) them and [push](http://git-scm.com/docs/git-push) the result, like this:
 
         $ git add youtube_dl/extractor/__init__.py
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index dbbf4a797..0cebe3981 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -9,6 +9,7 @@
  - **8tracks**
  - **9gag**
  - **abc.net.au**
+ - **Abc7News**
  - **AcademicEarth:Course**
  - **AddAnime**
  - **AdobeTV**
@@ -16,9 +17,12 @@
  - **Aftonbladet**
  - **AlJazeera**
  - **Allocine**
+ - **AlphaPorno**
  - **anitube.se**
  - **AnySex**
  - **Aparat**
+ - **AppleDailyAnimationNews**
+ - **AppleDailyRealtimeNews**
  - **AppleTrailers**
  - **archive.org**: archive.org videos
  - **ARD**
@@ -30,8 +34,10 @@
  - **arte.tv:ddc**
  - **arte.tv:embed**
  - **arte.tv:future**
+ - **AtresPlayer**
+ - **ATTTechChannel**
  - **audiomack**
- - **AUEngine**
+ - **audiomack:album**
  - **Azubu**
  - **bambuser**
  - **bambuser:channel**
@@ -71,8 +77,10 @@
  - **cmt.com**
  - **CNET**
  - **CNN**
+ - **CNNArticle**
  - **CNNBlogs**
  - **CollegeHumor**
+ - **CollegeRama**
  - **ComCarCoff**
  - **ComedyCentral**
  - **ComedyCentralShows**: The Daily Show / The Colbert Report
@@ -82,23 +90,27 @@
  - **Crunchyroll**
  - **crunchyroll:playlist**
  - **CSpan**: C-SPAN
+ - **CtsNews**
  - **culturebox.francetvinfo.fr**
  - **dailymotion**
  - **dailymotion:playlist**
  - **dailymotion:user**
  - **daum.net**
  - **DBTV**
+ - **DctpTv**
  - **DeezerPlaylist**
  - **defense.gouv.fr**
  - **Discovery**
  - **divxstage**: DivxStage
  - **Dotsub**
+ - **DRBonanza**
  - **Dropbox**
  - **DrTuber**
  - **DRTV**
  - **Dump**
  - **dvtv**: http://video.aktualne.cz/
  - **EbaumsWorld**
+ - **EchoMsk**
  - **eHow**
  - **Einthusan**
  - **eitb.tv**
@@ -108,6 +120,7 @@
  - **EMPFlix**
  - **Engadget**
  - **Eporner**
+ - **EroProfile**
  - **Escapist**
  - **EveryonesMixtape**
  - **exfm**: ex.fm
@@ -143,6 +156,7 @@
  - **GDCVault**
  - **generic**: Generic downloader that works on some sites
  - **GiantBomb**
+ - **Giga**
  - **Glide**: Glide mobile video messages (glide.me)
  - **Globo**
  - **GodTube**
@@ -153,9 +167,14 @@
  - **Grooveshark**
  - **Groupon**
  - **Hark**
+ - **HearThisAt**
  - **Heise**
+ - **HellPorno**
  - **Helsinki**: helsinki.fi
  - **HentaiStigma**
+ - **HistoricFilms**
+ - **hitbox**
+ - **hitbox:live**
  - **HornBunny**
  - **HostingBulk**
  - **HotNewHipHop**
@@ -182,6 +201,7 @@
  - **jpopsuki.tv**
  - **Jukebox**
  - **Kankan**
+ - **Karaoketv**
  - **keek**
  - **KeezMovies**
  - **KhanAcademy**
@@ -195,6 +215,7 @@
  - **LiveLeak**
  - **livestream**
  - **livestream:original**
+ - **LnkGo**
  - **lrt.lt**
  - **lynda**: lynda.com videos
  - **lynda:course**: lynda.com online courses
@@ -235,6 +256,7 @@
  - **MySpass**
  - **myvideo**
  - **MyVidster**
+ - **n-tv.de**
  - **Naver**
  - **NBA**
  - **NBC**
@@ -242,11 +264,16 @@
  - **ndr**: NDR.de - Mediathek
  - **NDTV**
  - **NerdCubedFeed**
+ - **Nerdist**
+ - **Netzkino**
  - **Newgrounds**
  - **Newstube**
+ - **NextMedia**
+ - **NextMediaActionNews**
  - **nfb**: National Film Board of Canada
  - **nfl.com**
  - **nhl.com**
+ - **nhl.com:news**: NHL news
  - **nhl.com:videocenter**: NHL videocenter category
  - **niconico**: ニコニコ動画
  - **NiconicoPlaylist**
@@ -257,18 +284,20 @@
  - **Nowness**
  - **nowvideo**: NowVideo
  - **npo.nl**
+ - **npo.nl:live**
  - **NRK**
  - **NRKTV**
- - **NTV**
+ - **ntv.ru**
  - **Nuvid**
  - **NYTimes**
  - **ocw.mit.edu**
  - **OktoberfestTV**
  - **on.aol.com**
  - **Ooyala**
+ - **OpenFilm**
+ - **orf:fm4**: radio FM4
  - **orf:oe1**: Radio Österreich 1
  - **orf:tvthek**: ORF TVthek
- - **ORFFM4**: radio FM4
  - **parliamentlive.tv**: UK parliament videos
  - **Patreon**
  - **PBS**
@@ -290,6 +319,7 @@
  - **Pyvideo**
  - **QuickVid**
  - **radio.de**
+ - **radiobremen**
  - **radiofrance**
  - **Rai**
  - **RBMARadio**
@@ -300,6 +330,8 @@
  - **RottenTomatoes**
  - **Roxwel**
  - **RTBF**
+ - **Rte**
+ - **RTL2**
  - **RTLnow**
  - **rtlxl.nl**
  - **RTP**
@@ -309,6 +341,7 @@
  - **RUHD**
  - **rutube**: Rutube videos
  - **rutube:channel**: Rutube channels
+ - **rutube:embed**: Rutube embedded videos
  - **rutube:movie**: Rutube movies
  - **rutube:person**: Rutube person videos
  - **RUTV**: RUTV.RU
@@ -351,11 +384,12 @@
  - **Sport5**
  - **SportBox**
  - **SportDeutschland**
- - **SRMediathek**: Süddeutscher Rundfunk
+ - **SRMediathek**: Saarländischer Rundfunk
  - **stanfordoc**: Stanford Open ClassRoom
  - **Steam**
  - **streamcloud.eu**
  - **StreamCZ**
+ - **StreetVoice**
  - **SunPorno**
  - **SWRMediathek**
  - **Syfy**
@@ -375,7 +409,9 @@
  - **TeleBruxelles**
  - **telecinco.es**
  - **TeleMB**
+ - **TeleTask**
  - **TenPlay**
+ - **TestTube**
  - **TF1**
  - **TheOnion**
  - **ThePlatform**
@@ -403,8 +439,15 @@
  - **tv.dfb.de**
  - **tvigle**: Интернет-телевидение Tvigle.ru
  - **tvp.pl**
+ - **tvp.pl:Series**
  - **TVPlay**: TV3Play and related services
- - **Twitch**
+ - **twitch:bookmarks**
+ - **twitch:chapter**
+ - **twitch:past_broadcasts**
+ - **twitch:profile**
+ - **twitch:stream**
+ - **twitch:video**
+ - **twitch:vod**
  - **Ubu**
  - **udemy**
  - **udemy:course**
@@ -433,6 +476,8 @@
  - **videoweed**: VideoWeed
  - **Vidme**
  - **Vidzi**
+ - **vier**
+ - **vier:videos**
  - **viki**
  - **vimeo**
  - **vimeo:album**
@@ -460,6 +505,7 @@
  - **WDR**
  - **wdr:mobile**
  - **WDRMaus**: Sendung mit der Maus
+ - **WebOfStories**
  - **Weibo**
  - **Wimp**
  - **Wistia**
@@ -472,7 +518,9 @@
  - **XNXX**
  - **XTube**
  - **XTubeUser**: XTube user profile
+ - **Xuite**
  - **XVideos**
+ - **XXXYMovies**
  - **Yahoo**: Yahoo screen and movies
  - **YesJapan**
  - **Ynet**
@@ -491,7 +539,6 @@
  - **youtube:search_url**: YouTube.com search URLs
  - **youtube:show**: YouTube.com (multi-season) shows
  - **youtube:subscriptions**: YouTube.com subscriptions feed, "ytsubs" keyword (requires authentication)
- - **youtube:toplist**: YouTube.com top lists, "yttoplist:{channel}:{list title}" (Example: "yttoplist:music:Top Tracks")
  - **youtube:user**: YouTube.com user videos (URL or "ytuser" keyword)
  - **youtube:watch_later**: Youtube watch later list, ":ytwatchlater" for short (requires authentication)
  - **ZDF**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 46af42171..a96e74629 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.02.5'
+__version__ = '2015.02.03'

From 23d9ded655f45c5a21c8d93ad167a072322ae88f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 3 Feb 2015 10:15:05 +0100
Subject: [PATCH 0230/6219] [franceculture] Rewrite for new HTML scheme (Fixes
 #4853)

---
 test/helper.py                        | 10 ++++
 youtube_dl/extractor/franceculture.py | 72 ++++++++++++---------------
 2 files changed, 42 insertions(+), 40 deletions(-)

diff --git a/test/helper.py b/test/helper.py
index 805467c82..651ef99b9 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -103,6 +103,16 @@ def expect_info_dict(self, got_dict, expected_dict):
             self.assertTrue(
                 match_rex.match(got),
                 'field %s (value: %r) should match %r' % (info_field, got, match_str))
+        elif isinstance(expected, compat_str) and expected.startswith('startswith:'):
+            got = got_dict.get(info_field)
+            start_str = expected[len('startswith:'):]
+            self.assertTrue(
+                isinstance(got, compat_str),
+                'Expected a %s object, but got %s for field %s' % (
+                    compat_str.__name__, type(got).__name__, info_field))
+            self.assertTrue(
+                got.startswith(start_str),
+                'field %s (value: %r) should start with %r' % (info_field, got, start_str))
         elif isinstance(expected, type):
             got = got_dict.get(info_field)
             self.assertTrue(isinstance(got, expected),
diff --git a/youtube_dl/extractor/franceculture.py b/youtube_dl/extractor/franceculture.py
index 0c2972162..1e83a4e7e 100644
--- a/youtube_dl/extractor/franceculture.py
+++ b/youtube_dl/extractor/franceculture.py
@@ -1,77 +1,69 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import json
-import re
-
 from .common import InfoExtractor
 from ..compat import (
-    compat_parse_qs,
     compat_urlparse,
 )
+from ..utils import (
+    determine_ext,
+    int_or_none,
+)
 
 
 class FranceCultureIE(InfoExtractor):
-    _VALID_URL = r'(?P<baseurl>http://(?:www\.)?franceculture\.fr/)player/reecouter\?play=(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?franceculture\.fr/player/reecouter\?play=(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://www.franceculture.fr/player/reecouter?play=4795174',
         'info_dict': {
             'id': '4795174',
             'ext': 'mp3',
             'title': 'Rendez-vous au pays des geeks',
+            'alt_title': 'Carnet nomade | 13-14',
             'vcodec': 'none',
-            'uploader': 'Colette Fellous',
             'upload_date': '20140301',
-            'duration': 3601,
             'thumbnail': r're:^http://www\.franceculture\.fr/.*/images/player/Carnet-nomade\.jpg$',
-            'description': 'Avec :Jean-Baptiste Péretié pour son documentaire sur Arte "La revanche des « geeks », une enquête menée aux Etats-Unis dans la S ...',
+            'description': 'startswith:Avec :Jean-Baptiste Péretié pour son documentaire sur Arte "La revanche des « geeks », une enquête menée aux Etats',
+            'timestamp': 1393700400,
         }
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        baseurl = mobj.group('baseurl')
-
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
-        params_code = self._search_regex(
-            r"<param name='movie' value='/sites/all/modules/rf/rf_player/swf/loader.swf\?([^']+)' />",
-            webpage, 'parameter code')
-        params = compat_parse_qs(params_code)
-        video_url = compat_urlparse.urljoin(baseurl, params['urlAOD'][0])
+
+        video_path = self._search_regex(
+            r'<a id="player".*?href="([^"]+)"', webpage, 'video path')
+        video_url = compat_urlparse.urljoin(url, video_path)
+        timestamp = int_or_none(self._search_regex(
+            r'<a id="player".*?data-date="([0-9]+)"',
+            webpage, 'upload date', fatal=False))
+        thumbnail = self._search_regex(
+            r'<a id="player".*?>\s+<img src="([^"]+)"',
+            webpage, 'thumbnail', fatal=False)
 
         title = self._html_search_regex(
-            r'<h1 class="title[^"]+">(.+?)</h1>', webpage, 'title')
+            r'<span class="title-diffusion">(.*?)</span>', webpage, 'title')
+        alt_title = self._html_search_regex(
+            r'<span class="title">(.*?)</span>',
+            webpage, 'alt_title', fatal=False)
+        description = self._html_search_regex(
+            r'<span class="description">(.*?)</span>',
+            webpage, 'description', fatal=False)
+
         uploader = self._html_search_regex(
             r'(?s)<div id="emission".*?<span class="author">(.*?)</span>',
-            webpage, 'uploader', fatal=False)
-        thumbnail_part = self._html_search_regex(
-            r'(?s)<div id="emission".*?<img src="([^"]+)"', webpage,
-            'thumbnail', fatal=False)
-        if thumbnail_part is None:
-            thumbnail = None
-        else:
-            thumbnail = compat_urlparse.urljoin(baseurl, thumbnail_part)
-        description = self._html_search_regex(
-            r'(?s)<p class="desc">(.*?)</p>', webpage, 'description')
-
-        info = json.loads(params['infoData'][0])[0]
-        duration = info.get('media_length')
-        upload_date_candidate = info.get('media_section5')
-        upload_date = (
-            upload_date_candidate
-            if (upload_date_candidate is not None and
-                re.match(r'[0-9]{8}$', upload_date_candidate))
-            else None)
+            webpage, 'uploader', default=None)
+        vcodec = 'none' if determine_ext(video_url.lower()) == 'mp3' else None
 
         return {
             'id': video_id,
             'url': video_url,
-            'vcodec': 'none' if video_url.lower().endswith('.mp3') else None,
-            'duration': duration,
+            'vcodec': vcodec,
             'uploader': uploader,
-            'upload_date': upload_date,
+            'timestamp': timestamp,
             'title': title,
+            'alt_title': alt_title,
             'thumbnail': thumbnail,
             'description': description,
         }

From d86204230157fc7a8a6800b7b814e8e3bbfbdf20 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 3 Feb 2015 10:18:32 +0100
Subject: [PATCH 0231/6219] [aftonbladet] Modernize

---
 youtube_dl/extractor/aftonbladet.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/aftonbladet.py b/youtube_dl/extractor/aftonbladet.py
index cfc7370ae..8442019ea 100644
--- a/youtube_dl/extractor/aftonbladet.py
+++ b/youtube_dl/extractor/aftonbladet.py
@@ -1,8 +1,6 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 
 
@@ -21,9 +19,7 @@ class AftonbladetIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.search(self._VALID_URL, url)
-
-        video_id = mobj.group('video_id')
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
         # find internal video meta data

From f6c24009bea95104e186afd77a867abbc61b7f81 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 3 Feb 2015 10:52:22 +0100
Subject: [PATCH 0232/6219] [YoutubeDL] Calculate thumbnail IDs automatically

---
 youtube_dl/YoutubeDL.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index c18ce9660..1730df4cd 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -964,9 +964,11 @@ class YoutubeDL(object):
             thumbnails.sort(key=lambda t: (
                 t.get('preference'), t.get('width'), t.get('height'),
                 t.get('id'), t.get('url')))
-            for t in thumbnails:
+            for i, t in enumerate(thumbnails):
                 if 'width' in t and 'height' in t:
                     t['resolution'] = '%dx%d' % (t['width'], t['height'])
+                if t.get('id') is None:
+                    t['id'] = '%d' % i
 
         if thumbnails and 'thumbnail' not in info_dict:
             info_dict['thumbnail'] = thumbnails[-1]['url']

From 1a6373ef3960add0117f797e0afe3322352c1c52 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 3 Feb 2015 10:53:05 +0100
Subject: [PATCH 0233/6219] [sort_formats] Prefer bitrate over video size

720p @ 1000KB/s looks way better than 1080p @ 500KB/s
---
 youtube_dl/extractor/common.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 9e517e3ac..653d793fc 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -704,11 +704,11 @@ class InfoExtractor(object):
                 preference,
                 f.get('language_preference') if f.get('language_preference') is not None else -1,
                 f.get('quality') if f.get('quality') is not None else -1,
-                f.get('height') if f.get('height') is not None else -1,
-                f.get('width') if f.get('width') is not None else -1,
-                ext_preference,
                 f.get('tbr') if f.get('tbr') is not None else -1,
                 f.get('vbr') if f.get('vbr') is not None else -1,
+                ext_preference,
+                f.get('height') if f.get('height') is not None else -1,
+                f.get('width') if f.get('width') is not None else -1,
                 f.get('abr') if f.get('abr') is not None else -1,
                 audio_ext_preference,
                 f.get('fps') if f.get('fps') is not None else -1,

From 9bb8e0a3f9276f65de38cda431bf72f7bd266693 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 3 Feb 2015 10:58:28 +0100
Subject: [PATCH 0234/6219] [wsj] Add new extractor (Fixes #4854)

---
 test/test_utils.py               |  3 ++
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/common.py   |  1 +
 youtube_dl/extractor/wsj.py      | 89 ++++++++++++++++++++++++++++++++
 youtube_dl/utils.py              |  2 +-
 5 files changed, 95 insertions(+), 1 deletion(-)
 create mode 100644 youtube_dl/extractor/wsj.py

diff --git a/test/test_utils.py b/test/test_utils.py
index 0ffccd35f..80c765bc4 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -156,6 +156,9 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(
             unified_strdate('11/26/2014 11:30:00 AM PST', day_first=False),
             '20141126')
+        self.assertEqual(
+            unified_strdate('2/2/2015 6:47:40 PM', day_first=False),
+            '20150202')
 
     def test_find_xpath_attr(self):
         testxml = '''<root>
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 5dcb14feb..5866a7617 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -554,6 +554,7 @@ from .wimp import WimpIE
 from .wistia import WistiaIE
 from .worldstarhiphop import WorldStarHipHopIE
 from .wrzuta import WrzutaIE
+from .wsj import WSJIE
 from .xbef import XBefIE
 from .xboxclips import XboxClipsIE
 from .xhamster import XHamsterIE
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 653d793fc..602601b24 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -145,6 +145,7 @@ class InfoExtractor(object):
     thumbnail:      Full URL to a video thumbnail image.
     description:    Full video description.
     uploader:       Full name of the video uploader.
+    creator:        The main artist who created the video.
     timestamp:      UNIX timestamp of the moment the video became available.
     upload_date:    Video upload date (YYYYMMDD).
                     If not explicitly set, calculated from timestamp.
diff --git a/youtube_dl/extractor/wsj.py b/youtube_dl/extractor/wsj.py
new file mode 100644
index 000000000..cbe3dc7be
--- /dev/null
+++ b/youtube_dl/extractor/wsj.py
@@ -0,0 +1,89 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    unified_strdate,
+)
+
+
+class WSJIE(InfoExtractor):
+    _VALID_URL = r'https?://video-api\.wsj\.com/api-video/player/iframe\.html\?guid=(?P<id>[a-zA-Z0-9-]+)'
+    IE_DESC = 'Wall Street Journal'
+    _TEST = {
+        'url': 'http://video-api.wsj.com/api-video/player/iframe.html?guid=1BD01A4C-BFE8-40A5-A42F-8A8AF9898B1A',
+        'md5': '9747d7a6ebc2f4df64b981e1dde9efa9',
+        'info_dict': {
+            'id': '1BD01A4C-BFE8-40A5-A42F-8A8AF9898B1A',
+            'ext': 'mp4',
+            'upload_date': '20150202',
+            'uploader_id': 'bbright',
+            'creator': 'bbright',
+            'categories': list,  # a long list
+            'duration': 90,
+            'title': 'Bills Coach Rex Ryan Updates His Old Jets Tattoo',
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        bitrates = [128, 174, 264, 320, 464, 664, 1264]
+        api_url = (
+            'http://video-api.wsj.com/api-video/find_all_videos.asp?'
+            'type=guid&count=1&query=%s&'
+            'fields=hls,adZone,thumbnailList,guid,state,secondsUntilStartTime,'
+            'author,description,name,linkURL,videoStillURL,duration,videoURL,'
+            'adCategory,catastrophic,linkShortURL,doctypeID,youtubeID,'
+            'titletag,rssURL,wsj-section,wsj-subsection,allthingsd-section,'
+            'allthingsd-subsection,sm-section,sm-subsection,provider,'
+            'formattedCreationDate,keywords,keywordsOmniture,column,editor,'
+            'emailURL,emailPartnerID,showName,omnitureProgramName,'
+            'omnitureVideoFormat,linkRelativeURL,touchCastID,'
+            'omniturePublishDate,%s') % (
+                video_id, ','.join('video%dkMP4Url' % br for br in bitrates))
+        info = self._download_json(api_url, video_id)['items'][0]
+
+        # Thumbnails are conveniently in the correct format already
+        thumbnails = info.get('thumbnailList')
+        creator = info.get('author')
+        uploader_id = info.get('editor')
+        categories = info.get('keywords')
+        duration = int_or_none(info.get('duration'))
+        upload_date = unified_strdate(
+            info.get('formattedCreationDate'), day_first=False)
+        title = info.get('name', info.get('titletag'))
+
+        formats = [{
+            'format_id': 'f4m',
+            'format_note': 'f4m (meta URL)',
+            'url': info['videoURL'],
+        }]
+        if info.get('hls'):
+            formats.extend(self._extract_m3u8_formats(
+                info['hls'], video_id, ext='mp4',
+                preference=0, entry_protocol='m3u8_native'))
+        for br in bitrates:
+            field = 'video%dkMP4Url' % br
+            if info.get(field):
+                formats.append({
+                    'format_id': 'mp4-%d' % br,
+                    'container': 'mp4',
+                    'tbr': br,
+                    'url': info[field],
+                })
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'formats': formats,
+            'thumbnails': thumbnails,
+            'creator': creator,
+            'uploader_id': uploader_id,
+            'duration': duration,
+            'upload_date': upload_date,
+            'title': title,
+            'formats': formats,
+            'categories': categories,
+        }
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 251074bf5..8f5463f1c 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -701,7 +701,7 @@ def unified_strdate(date_str, day_first=True):
     # %z (UTC offset) is only supported in python>=3.2
     date_str = re.sub(r' ?(\+|-)[0-9]{2}:?[0-9]{2}$', '', date_str)
     # Remove AM/PM + timezone
-    date_str = re.sub(r'(?i)\s*(?:AM|PM)\s+[A-Z]+', '', date_str)
+    date_str = re.sub(r'(?i)\s*(?:AM|PM)(?:\s+[A-Z]+)?', '', date_str)
 
     format_expressions = [
         '%d %B %Y',

From cd7342755fb6928ca9f089bbd4bfdd17904f91b3 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 3 Feb 2015 10:59:27 +0100
Subject: [PATCH 0235/6219] release 2015.02.03.1

---
 docs/supportedsites.md | 1 +
 youtube_dl/version.py  | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 0cebe3981..b188be636 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -511,6 +511,7 @@
  - **Wistia**
  - **WorldStarHipHop**
  - **wrzuta.pl**
+ - **WSJ**: Wall Street Journal
  - **XBef**
  - **XboxClips**
  - **XHamster**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index a96e74629..4975ff83e 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.03'
+__version__ = '2015.02.03.1'

From 12f0454cd60ffa67b57a822b2e30461cb761040d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 3 Feb 2015 14:18:12 +0100
Subject: [PATCH 0236/6219] [README] Add an FAQ entry about anime sites

---
 README.md | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/README.md b/README.md
index c1221417e..06dea400d 100644
--- a/README.md
+++ b/README.md
@@ -532,6 +532,14 @@ Either prepend `http://www.youtube.com/watch?v=` or separate the ID from the opt
     youtube-dl -- -wNyEUrxzFU
     youtube-dl "http://www.youtube.com/watch?v=-wNyEUrxzFU"
 
+### Can you add support for this anime video site, or site which shows current movies for free?
+
+As a matter of policy (as well as legality), youtube-dl does not include support for services that specialize in infringing copyright. As a rule of thumb, if you cannot easily find a video that the service is quite obviously allowed to distribute (i.e. that has been uploaded by the creator, the creator's distributor, or is published under a free license), the service is probably unfit for inclusion to youtube-dl.
+
+A note on the service that they don't host the infringing content, but just link to those who do, is evidence that the service should **not** be included into youtube-dl. The same goes for any DMCA note when the whole front page of the service is filled with videos they are not allowed to distribute. A "fair use" note is equally unconvincing if the service shows copyright-protected videos in full without authorization.
+
+Support requests for services that **do** purchase the rights to distribute their content are perfectly fine though. If in doubt, you can simply include a source that mentions the legitimate purchase of content.
+
 ### How can I detect whether a given URL is supported by youtube-dl?
 
 For one, have a look at the [list of supported sites](docs/supportedsites.md). Note that it can sometimes happen that the site changes its URL scheme (say, from http://example.com/v/1234567 to http://example.com/v/1234567 ) and youtube-dl reports an URL of a service in that list as unsupported. In that case, simply report a bug.

From f076b63821100af17379ed8b470e508846bb8432 Mon Sep 17 00:00:00 2001
From: naglis <njonaitis@gmail.com>
Date: Wed, 4 Feb 2015 15:33:37 +0200
Subject: [PATCH 0237/6219] [generic/ooyala] Add support for Ooyala embeds on
 SBN network websites (Fixes #4859)

---
 youtube_dl/extractor/generic.py | 24 +++++++++++++++++++++++-
 1 file changed, 23 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index a6f102a4e..fbbc79a57 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -140,6 +140,19 @@ class GenericIE(InfoExtractor):
             },
             'add_ie': ['Ooyala'],
         },
+        # multiple ooyala embeds on SBN network websites
+        {
+            'url': 'http://www.sbnation.com/college-football-recruiting/2015/2/3/7970291/national-signing-day-rationalizations-itll-be-ok-itll-be-ok',
+            'info_dict': {
+                'id': 'national-signing-day-rationalizations-itll-be-ok-itll-be-ok',
+                'title': '25 lies you will tell yourself on National Signing Day - SBNation.com',
+            },
+            'playlist_mincount': 3,
+            'params': {
+                'skip_download': True,
+            },
+            'add_ie': ['Ooyala'],
+        },
         # google redirect
         {
             'url': 'http://www.google.com/url?sa=t&rct=j&q=&esrc=s&source=web&cd=1&cad=rja&ved=0CCUQtwIwAA&url=http%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DcmQHVoWB5FY&ei=F-sNU-LLCaXk4QT52ICQBQ&usg=AFQjCNEw4hL29zgOohLXvpJ-Bdh2bils1Q&bvm=bv.61965928,d.bGE',
@@ -882,10 +895,19 @@ class GenericIE(InfoExtractor):
 
         # Look for Ooyala videos
         mobj = (re.search(r'player\.ooyala\.com/[^"?]+\?[^"]*?(?:embedCode|ec)=(?P<ec>[^"&]+)', webpage) or
-                re.search(r'OO\.Player\.create\([\'"].*?[\'"],\s*[\'"](?P<ec>.{32})[\'"]', webpage))
+                re.search(r'OO\.Player\.create\([\'"].*?[\'"],\s*[\'"](?P<ec>.{32})[\'"]', webpage) or
+                re.search(r'SBN\.VideoLinkset\.ooyala\([\'"](?P<ec>.{32})[\'"]\)', webpage))
         if mobj is not None:
             return OoyalaIE._build_url_result(mobj.group('ec'))
 
+        # Look for multiple Ooyala embeds on SBN network websites
+        mobj = re.search(r'SBN\.VideoLinkset\.entryGroup\((\[.*?\])', webpage)
+        if mobj is not None:
+            embeds = self._parse_json(mobj.group(1), video_id, fatal=False)
+            if embeds:
+                return _playlist_from_matches(
+                    embeds, getter=lambda v: OoyalaIE._url_for_embed_code(v['provider_video_id']), ie='Ooyala')
+
         # Look for Aparat videos
         mobj = re.search(r'<iframe .*?src="(http://www\.aparat\.com/video/[^"]+)"', webpage)
         if mobj is not None:

From 58ba6c016086f1f9f47778f7a64aca72beb55c23 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 4 Feb 2015 19:47:55 +0600
Subject: [PATCH 0238/6219] [mixcloud] Fix extraction (Closes #4862)

---
 youtube_dl/extractor/mixcloud.py | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dl/extractor/mixcloud.py
index 141bb463e..1831c6749 100644
--- a/youtube_dl/extractor/mixcloud.py
+++ b/youtube_dl/extractor/mixcloud.py
@@ -18,7 +18,7 @@ class MixcloudIE(InfoExtractor):
     _VALID_URL = r'^(?:https?://)?(?:www\.)?mixcloud\.com/([^/]+)/([^/]+)'
     IE_NAME = 'mixcloud'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.mixcloud.com/dholbach/cryptkeeper/',
         'info_dict': {
             'id': 'dholbach-cryptkeeper',
@@ -33,7 +33,20 @@ class MixcloudIE(InfoExtractor):
             'view_count': int,
             'like_count': int,
         },
-    }
+    }, {
+        'url': 'http://www.mixcloud.com/gillespeterson/caribou-7-inch-vinyl-mix-chat/',
+        'info_dict': {
+            'id': 'gillespeterson-caribou-7-inch-vinyl-mix-chat',
+            'ext': 'm4a',
+            'title': 'Electric Relaxation vol. 3',
+            'description': 'md5:2b8aec6adce69f9d41724647c65875e8',
+            'uploader': 'Daniel Drumz',
+            'uploader_id': 'gillespeterson',
+            'thumbnail': 're:https?://.*\.jpg',
+            'view_count': int,
+            'like_count': int,
+        },
+    }]
 
     def _get_url(self, track_id, template_url):
         server_count = 30
@@ -60,7 +73,7 @@ class MixcloudIE(InfoExtractor):
         webpage = self._download_webpage(url, track_id)
 
         preview_url = self._search_regex(
-            r'\s(?:data-preview-url|m-preview)="(.+?)"', webpage, 'preview url')
+            r'\s(?:data-preview-url|m-preview)="([^"]+)"', webpage, 'preview url')
         song_url = preview_url.replace('/previews/', '/c/originals/')
         template_url = re.sub(r'(stream\d*)', 'stream%d', song_url)
         final_song_url = self._get_url(track_id, template_url)

From 7dfc3566250894147910bd6b79530f97770a47f3 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 4 Feb 2015 16:09:35 +0100
Subject: [PATCH 0239/6219] release 2015.02.04

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 4975ff83e..36591a43d 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.03.1'
+__version__ = '2015.02.04'

From 25e5ebf382b2fca9d4b6909ac1ac9b5af89bcdd9 Mon Sep 17 00:00:00 2001
From: Frans de Jonge <fransdejonge@gmail.com>
Date: Thu, 5 Feb 2015 12:51:33 +0100
Subject: [PATCH 0240/6219] Add NPO.nl subtitles

Implements #3638
---
 youtube_dl/extractor/npo.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index 175b14583..0580461bb 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -1,6 +1,6 @@
 from __future__ import unicode_literals
 
-from .common import InfoExtractor
+from .subtitles import SubtitlesInfoExtractor
 from ..utils import (
     fix_xml_ampersands,
     parse_duration,
@@ -11,7 +11,7 @@ from ..utils import (
 )
 
 
-class NPOBaseIE(InfoExtractor):
+class NPOBaseIE(SubtitlesInfoExtractor):
     def _get_token(self, video_id):
         token_page = self._download_webpage(
             'http://ida.omroep.nl/npoplayer/i.js',
@@ -159,6 +159,13 @@ class NPOIE(NPOBaseIE):
                     'quality': stream.get('kwaliteit'),
                 })
 
+        subtitles = {}
+
+        tt888 = metadata.get('tt888')
+        if self._have_to_download_any_subtitles and tt888 == 'ja':
+            subtitles['nl'] = 'http://e.omroep.nl/tt888/%s' % video_id
+            subtitles = self.extract_subtitles(video_id, subtitles)
+
         self._sort_formats(formats)
 
         return {
@@ -169,6 +176,7 @@ class NPOIE(NPOBaseIE):
             'upload_date': unified_strdate(metadata.get('gidsdatum')),
             'duration': parse_duration(metadata.get('tijdsduur')),
             'formats': formats,
+            'subtitles': subtitles,
         }
 
 

From 9e91449c8d1dae53eda6fb250384f0afa5122ce6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 5 Feb 2015 20:13:28 +0600
Subject: [PATCH 0241/6219] [npo] Fix subtitles (Closes #3638)

---
 youtube_dl/extractor/npo.py | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index 0580461bb..54be06a4e 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -159,15 +159,18 @@ class NPOIE(NPOBaseIE):
                     'quality': stream.get('kwaliteit'),
                 })
 
-        subtitles = {}
-
-        tt888 = metadata.get('tt888')
-        if self._have_to_download_any_subtitles and tt888 == 'ja':
-            subtitles['nl'] = 'http://e.omroep.nl/tt888/%s' % video_id
-            subtitles = self.extract_subtitles(video_id, subtitles)
-
         self._sort_formats(formats)
 
+        subtitles = {}
+        if metadata.get('tt888') == 'ja':
+            subtitles['nl'] = 'http://e.omroep.nl/tt888/%s' % video_id
+
+        if self._downloader.params.get('listsubtitles', False):
+            self._list_available_subtitles(video_id, subtitles)
+            return
+
+        subtitles = self.extract_subtitles(video_id, subtitles)
+
         return {
             'id': video_id,
             'title': metadata['titel'],

From 6a348cf7d507f0373e9a6f67358202b66a858cd1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 5 Feb 2015 20:14:56 +0600
Subject: [PATCH 0242/6219] Credit @Frenzie for npo subtitles (#4878)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 83d8161a9..85ca81a27 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -108,3 +108,4 @@ Enam Mijbah Noor
 David Luhmer
 Shaya Goldberg
 Paul Hartmann
+Frans de Jonge

From 60ca389c646ae03e90377261857bfe45333a8ff2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 5 Feb 2015 22:16:27 +0600
Subject: [PATCH 0243/6219] [extractor/common] Prefix f4m/m3u8 entries with
 identifier

---
 youtube_dl/extractor/common.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 602601b24..8ec7cee06 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -765,7 +765,7 @@ class InfoExtractor(object):
         self.to_screen(msg)
         time.sleep(timeout)
 
-    def _extract_f4m_formats(self, manifest_url, video_id):
+    def _extract_f4m_formats(self, manifest_url, video_id, preference=None, f4m_id=None):
         manifest = self._download_xml(
             manifest_url, video_id, 'Downloading f4m manifest',
             'Unable to download f4m manifest')
@@ -780,24 +780,25 @@ class InfoExtractor(object):
             if manifest_version == '2.0':
                 manifest_url = '/'.join(manifest_url.split('/')[:-1]) + '/' + media_el.attrib.get('href')
             tbr = int_or_none(media_el.attrib.get('bitrate'))
-            format_id = 'f4m-%d' % (i if tbr is None else tbr)
             formats.append({
-                'format_id': format_id,
+                'format_id': '-'.join(filter(None, [f4m_id, 'f4m-%d' % (i if tbr is None else tbr)])),
                 'url': manifest_url,
                 'ext': 'flv',
                 'tbr': tbr,
                 'width': int_or_none(media_el.attrib.get('width')),
                 'height': int_or_none(media_el.attrib.get('height')),
+                'preference': preference,
             })
         self._sort_formats(formats)
 
         return formats
 
     def _extract_m3u8_formats(self, m3u8_url, video_id, ext=None,
-                              entry_protocol='m3u8', preference=None):
+                              entry_protocol='m3u8', preference=None,
+                              m3u8_id=None):
 
         formats = [{
-            'format_id': 'm3u8-meta',
+            'format_id': '-'.join(filter(None, [m3u8_id, 'm3u8-meta'])),
             'url': m3u8_url,
             'ext': ext,
             'protocol': 'm3u8',
@@ -833,9 +834,8 @@ class InfoExtractor(object):
                     formats.append({'url': format_url(line)})
                     continue
                 tbr = int_or_none(last_info.get('BANDWIDTH'), scale=1000)
-
                 f = {
-                    'format_id': 'm3u8-%d' % (tbr if tbr else len(formats)),
+                    'format_id': '-'.join(filter(None, [m3u8_id, 'm3u8-%d' % (tbr if tbr else len(formats))])),
                     'url': format_url(line.strip()),
                     'tbr': tbr,
                     'ext': ext,

From f13b88c61696eaeb96878b5c68b7ab949a3f4ca6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 5 Feb 2015 22:17:50 +0600
Subject: [PATCH 0244/6219] [rts] Fix f4m and m3u8 extraction (Closes #4873)

---
 youtube_dl/extractor/rts.py | 28 +++++++++++++++++++++++-----
 1 file changed, 23 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/rts.py b/youtube_dl/extractor/rts.py
index 5e84c1098..d0981115d 100644
--- a/youtube_dl/extractor/rts.py
+++ b/youtube_dl/extractor/rts.py
@@ -6,12 +6,14 @@ import re
 from .common import InfoExtractor
 from ..compat import (
     compat_str,
+    compat_urllib_parse_urlparse,
 )
 from ..utils import (
     int_or_none,
     parse_duration,
     parse_iso8601,
     unescapeHTML,
+    xpath_text,
 )
 
 
@@ -159,11 +161,27 @@ class RTSIE(InfoExtractor):
             return int_or_none(self._search_regex(
                 r'-([0-9]+)k\.', url, 'bitrate', default=None))
 
-        formats = [{
-            'format_id': fid,
-            'url': furl,
-            'tbr': extract_bitrate(furl),
-        } for fid, furl in info['streams'].items()]
+        formats = []
+        for format_id, format_url in info['streams'].items():
+            if format_url.endswith('.f4m'):
+                token = self._download_xml(
+                    'http://tp.srgssr.ch/token/akahd.xml?stream=%s/*' % compat_urllib_parse_urlparse(format_url).path,
+                    video_id, 'Downloading %s token' % format_id)
+                auth_params = xpath_text(token, './/authparams', 'auth params')
+                if not auth_params:
+                    continue
+                formats.extend(self._extract_f4m_formats(
+                    '%s?%s&hdcore=3.4.0&plugin=aasp-3.4.0.132.66' % (format_url, auth_params),
+                    video_id, f4m_id=format_id))
+            elif format_url.endswith('.m3u8'):
+                formats.extend(self._extract_m3u8_formats(
+                    format_url, video_id, 'mp4', m3u8_id=format_id))
+            else:
+                formats.append({
+                    'format_id': format_id,
+                    'url': format_url,
+                    'tbr': extract_bitrate(format_url),
+                })
 
         if 'media' in info:
             formats.extend([{

From 300753a0695c17979d6ac42147cb162ff0eedea5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 6 Feb 2015 00:51:16 +0600
Subject: [PATCH 0245/6219] [YoutubeDL] Fix video+audio format field (Closes
 #4880)

---
 youtube_dl/YoutubeDL.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 1730df4cd..9605f8f99 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1076,7 +1076,8 @@ class YoutubeDL(object):
                                 else self.params['merge_output_format'])
                             selected_format = {
                                 'requested_formats': formats_info,
-                                'format': rf,
+                                'format': '%s+%s' % (formats_info[0].get('format'),
+                                                     formats_info[1].get('format')),
                                 'format_id': '%s+%s' % (formats_info[0].get('format_id'),
                                                         formats_info[1].get('format_id')),
                                 'width': formats_info[0].get('width'),

From e3aaace4005e7b767d28152e60d1cf6272c398ec Mon Sep 17 00:00:00 2001
From: robin <rderooij685@gmail.com>
Date: Thu, 5 Feb 2015 19:55:41 +0100
Subject: [PATCH 0246/6219] [tweakers] Add new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/tweakers.py | 41 ++++++++++++++++++++++++++++++++
 2 files changed, 42 insertions(+)
 create mode 100644 youtube_dl/extractor/tweakers.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 5866a7617..047f7002a 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -475,6 +475,7 @@ from .tutv import TutvIE
 from .tvigle import TvigleIE
 from .tvp import TvpIE, TvpSeriesIE
 from .tvplay import TVPlayIE
+from .tweakers import TweakersIE
 from .twentyfourvideo import TwentyFourVideoIE
 from .twitch import (
     TwitchVideoIE,
diff --git a/youtube_dl/extractor/tweakers.py b/youtube_dl/extractor/tweakers.py
new file mode 100644
index 000000000..3080f9e46
--- /dev/null
+++ b/youtube_dl/extractor/tweakers.py
@@ -0,0 +1,41 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+
+class TweakersIE(InfoExtractor):
+    _VALID_URL = r'https?://tweakers\.net/video/(?P<id>[0-9]+).*'
+    _TEST = {
+        'url': 'https://tweakers.net/video/9926/new-nintendo-3ds-xl-op-alle-fronten-beter.html',
+        'md5': 'f7f7f3027166a7f32f024b4ae6571ced',
+        'info_dict': {
+            'id': '9926',
+            'ext': 'mp4',
+            'title': 'New-Nintendo-3Ds-Xl-Op-Alle-Fronten-Beter',
+            # TODO more properties, either as:
+            # * A value
+            # * MD5 checksum; start the string with md5:
+            # * A regular expression; start the string with re:
+            # * Any Python type (for example int or float)
+        }
+    }
+
+    def _real_extract(self, url):
+        splitted_url = re.split('.html|/', url)
+        del splitted_url[-1] # To remove extra '/' at the end
+        video_id = splitted_url[4]
+        title = splitted_url[5].title() # Retrieve title for URL and capitalize
+        splitted_url[3] = splitted_url[3] + '/player' # Add /player to get the player page
+        player_url = '/'.join(splitted_url) + '.html'
+        player_page = self._download_webpage(player_url, video_id)
+
+        return {
+            'id': video_id,
+            'ext': 'mp4',
+            'title': title,
+            'url':  re.findall('http.*mp4', player_page)[0],
+            'player_url': player_url
+        }

From 3ee6e02564f7f6c54480fd4eeb94ae08a747b9b6 Mon Sep 17 00:00:00 2001
From: robin <rderooij685@gmail.com>
Date: Thu, 5 Feb 2015 19:59:36 +0100
Subject: [PATCH 0247/6219] Edit Tweakers extractor

Fixed code conventions (mainly adding two or more spaces before making
an inline comment)
---
 youtube_dl/extractor/tweakers.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/tweakers.py b/youtube_dl/extractor/tweakers.py
index 3080f9e46..29cd04f00 100644
--- a/youtube_dl/extractor/tweakers.py
+++ b/youtube_dl/extractor/tweakers.py
@@ -25,10 +25,10 @@ class TweakersIE(InfoExtractor):
 
     def _real_extract(self, url):
         splitted_url = re.split('.html|/', url)
-        del splitted_url[-1] # To remove extra '/' at the end
+        del splitted_url[-1]  # To remove extra '/' at the end
         video_id = splitted_url[4]
-        title = splitted_url[5].title() # Retrieve title for URL and capitalize
-        splitted_url[3] = splitted_url[3] + '/player' # Add /player to get the player page
+        title = splitted_url[5].title()  # Retrieve title for URL and capitalize
+        splitted_url[3] = splitted_url[3] + '/player'  # Add /player to get the player page
         player_url = '/'.join(splitted_url) + '.html'
         player_page = self._download_webpage(player_url, video_id)
 
@@ -36,6 +36,6 @@ class TweakersIE(InfoExtractor):
             'id': video_id,
             'ext': 'mp4',
             'title': title,
-            'url':  re.findall('http.*mp4', player_page)[0],
+            'url': re.findall('http.*mp4', player_page)[0],
             'player_url': player_url
         }

From c893d70805be9b8f8a10c7d7dbe39216f8976ea5 Mon Sep 17 00:00:00 2001
From: robin <rderooij685@gmail.com>
Date: Thu, 5 Feb 2015 21:38:35 +0100
Subject: [PATCH 0248/6219] Remove player-url in tweakers.py

Player-url only needed for rmftp, not for regular URLs
---
 youtube_dl/extractor/tweakers.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/tweakers.py b/youtube_dl/extractor/tweakers.py
index 29cd04f00..14145e617 100644
--- a/youtube_dl/extractor/tweakers.py
+++ b/youtube_dl/extractor/tweakers.py
@@ -37,5 +37,4 @@ class TweakersIE(InfoExtractor):
             'ext': 'mp4',
             'title': title,
             'url': re.findall('http.*mp4', player_page)[0],
-            'player_url': player_url
         }

From bc63d56ccac8805ca8394e0b41890027e6d677d2 Mon Sep 17 00:00:00 2001
From: robin <rderooij685@gmail.com>
Date: Thu, 5 Feb 2015 21:40:18 +0100
Subject: [PATCH 0249/6219] Remove unnecessary TODO-comments

---
 youtube_dl/extractor/tweakers.py | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/youtube_dl/extractor/tweakers.py b/youtube_dl/extractor/tweakers.py
index 14145e617..a4e749850 100644
--- a/youtube_dl/extractor/tweakers.py
+++ b/youtube_dl/extractor/tweakers.py
@@ -15,11 +15,6 @@ class TweakersIE(InfoExtractor):
             'id': '9926',
             'ext': 'mp4',
             'title': 'New-Nintendo-3Ds-Xl-Op-Alle-Fronten-Beter',
-            # TODO more properties, either as:
-            # * A value
-            # * MD5 checksum; start the string with md5:
-            # * A regular expression; start the string with re:
-            # * Any Python type (for example int or float)
         }
     }
 

From 67ce4f8820f94961c9b094f9c652f421c9d9f6fb Mon Sep 17 00:00:00 2001
From: robin <rderooij685@gmail.com>
Date: Thu, 5 Feb 2015 21:49:13 +0100
Subject: [PATCH 0250/6219] Use match_id method instead of splitted URL

---
 youtube_dl/extractor/tweakers.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tweakers.py b/youtube_dl/extractor/tweakers.py
index a4e749850..e332d4694 100644
--- a/youtube_dl/extractor/tweakers.py
+++ b/youtube_dl/extractor/tweakers.py
@@ -21,7 +21,7 @@ class TweakersIE(InfoExtractor):
     def _real_extract(self, url):
         splitted_url = re.split('.html|/', url)
         del splitted_url[-1]  # To remove extra '/' at the end
-        video_id = splitted_url[4]
+        video_id = self._match_id(url)
         title = splitted_url[5].title()  # Retrieve title for URL and capitalize
         splitted_url[3] = splitted_url[3] + '/player'  # Add /player to get the player page
         player_url = '/'.join(splitted_url) + '.html'

From 9c91a8fa70e3c2f29dd23809e90b3357b6c1cdc4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 6 Feb 2015 03:23:42 +0600
Subject: [PATCH 0251/6219] [tweakers] Switch extraction to xspf playlist,
 extract all formats and meta (#4881)

---
 youtube_dl/extractor/tweakers.py | 58 ++++++++++++++++++++++++--------
 1 file changed, 44 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/tweakers.py b/youtube_dl/extractor/tweakers.py
index e332d4694..c80ec15cf 100644
--- a/youtube_dl/extractor/tweakers.py
+++ b/youtube_dl/extractor/tweakers.py
@@ -1,35 +1,65 @@
-# coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
+from ..utils import (
+    xpath_text,
+    xpath_with_ns,
+    int_or_none,
+    float_or_none,
+)
 
 
 class TweakersIE(InfoExtractor):
-    _VALID_URL = r'https?://tweakers\.net/video/(?P<id>[0-9]+).*'
+    _VALID_URL = r'https?://tweakers\.net/video/(?P<id>\d+)'
     _TEST = {
         'url': 'https://tweakers.net/video/9926/new-nintendo-3ds-xl-op-alle-fronten-beter.html',
-        'md5': 'f7f7f3027166a7f32f024b4ae6571ced',
+        'md5': '1b5afa817403bb5baa08359dca31e6df',
         'info_dict': {
             'id': '9926',
             'ext': 'mp4',
-            'title': 'New-Nintendo-3Ds-Xl-Op-Alle-Fronten-Beter',
+            'title': 'New Nintendo 3DS XL - Op alle fronten beter',
+            'description': 'md5:f97324cc71e86e11c853f0763820e3ba',
+            'thumbnail': 're:^https?://.*\.jpe?g$',
+            'duration': 386,
         }
     }
 
     def _real_extract(self, url):
-        splitted_url = re.split('.html|/', url)
-        del splitted_url[-1]  # To remove extra '/' at the end
         video_id = self._match_id(url)
-        title = splitted_url[5].title()  # Retrieve title for URL and capitalize
-        splitted_url[3] = splitted_url[3] + '/player'  # Add /player to get the player page
-        player_url = '/'.join(splitted_url) + '.html'
-        player_page = self._download_webpage(player_url, video_id)
+
+        playlist = self._download_xml(
+            'https://tweakers.net/video/s1playlist/%s/playlist.xspf' % video_id,
+            video_id)
+
+        NS_MAP = {
+            'xspf': 'http://xspf.org/ns/0/',
+            's1': 'http://static.streamone.nl/player/ns/0',
+        }
+
+        track = playlist.find(xpath_with_ns('./xspf:trackList/xspf:track', NS_MAP))
+
+        title = xpath_text(
+            track, xpath_with_ns('./xspf:title', NS_MAP), 'title')
+        description = xpath_text(
+            track, xpath_with_ns('./xspf:annotation', NS_MAP), 'description')
+        thumbnail = xpath_text(
+            track, xpath_with_ns('./xspf:image', NS_MAP), 'thumbnail')
+        duration = float_or_none(
+            xpath_text(track, xpath_with_ns('./xspf:duration', NS_MAP), 'duration'),
+            1000)
+
+        formats = [{
+            'url': location.text,
+            'format_id': location.get(xpath_with_ns('s1:label', NS_MAP)),
+            'width': int_or_none(location.get(xpath_with_ns('s1:width', NS_MAP))),
+            'height': int_or_none(location.get(xpath_with_ns('s1:height', NS_MAP))),
+        } for location in track.findall(xpath_with_ns('./xspf:location', NS_MAP))]
 
         return {
             'id': video_id,
-            'ext': 'mp4',
             'title': title,
-            'url': re.findall('http.*mp4', player_page)[0],
+            'description': description,
+            'thumbnail': thumbnail,
+            'duration': duration,
+            'formats': formats,
         }

From 3900eec27c0a385d60f2b1dca50faa6f5e25b971 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 6 Feb 2015 04:29:29 +0600
Subject: [PATCH 0252/6219] [extractor/common] Fix 2.0 manifest extraction
 (Closes #4830)

---
 youtube_dl/extractor/common.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 8ec7cee06..df1a4417b 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -778,7 +778,8 @@ class InfoExtractor(object):
             media_nodes = manifest.findall('{http://ns.adobe.com/f4m/2.0}media')
         for i, media_el in enumerate(media_nodes):
             if manifest_version == '2.0':
-                manifest_url = '/'.join(manifest_url.split('/')[:-1]) + '/' + media_el.attrib.get('href')
+                manifest_url = ('/'.join(manifest_url.split('/')[:-1]) + '/'
+                                + (media_el.attrib.get('href') or media_el.attrib.get('url')))
             tbr = int_or_none(media_el.attrib.get('bitrate'))
             formats.append({
                 'format_id': '-'.join(filter(None, [f4m_id, 'f4m-%d' % (i if tbr is None else tbr)])),

From 1a2548d9e91b24f60d6948e658cfc0b68797b254 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Fri, 6 Feb 2015 13:41:58 +0200
Subject: [PATCH 0253/6219] [rtp] Pass --realtime to rtmpdump (Fixes #4882)

A workaround for video jumping back in time.
---
 youtube_dl/extractor/rtp.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/rtp.py b/youtube_dl/extractor/rtp.py
index 7736cabba..4511cba49 100644
--- a/youtube_dl/extractor/rtp.py
+++ b/youtube_dl/extractor/rtp.py
@@ -49,6 +49,7 @@ class RTPIE(InfoExtractor):
             'ext': ext,
             'vcodec': config.get('type') == 'audio' and 'none' or None,
             'player_url': 'http://programas.rtp.pt/play/player.swf?v3',
+            'rtmp_real_time': True,
         }]
 
         return {

From c8319733667ff5bc0e0cbe0a9c253c8cfc7d479e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 6 Feb 2015 14:38:30 +0100
Subject: [PATCH 0254/6219] release 2015.02.06

---
 docs/supportedsites.md | 1 +
 youtube_dl/version.py  | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index b188be636..2d8f9c316 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -441,6 +441,7 @@
  - **tvp.pl**
  - **tvp.pl:Series**
  - **TVPlay**: TV3Play and related services
+ - **Tweakers**
  - **twitch:bookmarks**
  - **twitch:chapter**
  - **twitch:past_broadcasts**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 36591a43d..1091ae61b 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.04'
+__version__ = '2015.02.06'

From 1e1896f2de300bf1905a63adffe17e7ab68cc252 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 6 Feb 2015 15:16:43 +0100
Subject: [PATCH 0255/6219] [extractor/common] Correct sort order.

We should look at height and width before ext_preference.
---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index df1a4417b..980b63680 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -707,9 +707,9 @@ class InfoExtractor(object):
                 f.get('quality') if f.get('quality') is not None else -1,
                 f.get('tbr') if f.get('tbr') is not None else -1,
                 f.get('vbr') if f.get('vbr') is not None else -1,
-                ext_preference,
                 f.get('height') if f.get('height') is not None else -1,
                 f.get('width') if f.get('width') is not None else -1,
+                ext_preference,
                 f.get('abr') if f.get('abr') is not None else -1,
                 audio_ext_preference,
                 f.get('fps') if f.get('fps') is not None else -1,

From 7f2a9f1b4993693659820693c1b6432f3b01d21b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 6 Feb 2015 21:15:01 +0600
Subject: [PATCH 0256/6219] [tvigle] Add support for cloud URLs (Closes #4887)

---
 youtube_dl/extractor/tvigle.py | 22 +++++++++++++++-------
 1 file changed, 15 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/tvigle.py b/youtube_dl/extractor/tvigle.py
index ba65996dc..102362b29 100644
--- a/youtube_dl/extractor/tvigle.py
+++ b/youtube_dl/extractor/tvigle.py
@@ -1,6 +1,8 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..utils import (
     float_or_none,
@@ -11,7 +13,7 @@ from ..utils import (
 class TvigleIE(InfoExtractor):
     IE_NAME = 'tvigle'
     IE_DESC = 'Интернет-телевидение Tvigle.ru'
-    _VALID_URL = r'http://(?:www\.)?tvigle\.ru/(?:[^/]+/)+(?P<id>[^/]+)/$'
+    _VALID_URL = r'https?://(?:www\.)?(?:tvigle\.ru/(?:[^/]+/)+(?P<display_id>[^/]+)/$|cloud\.tvigle\.ru/video/(?P<id>\d+))'
 
     _TESTS = [
         {
@@ -38,16 +40,22 @@ class TvigleIE(InfoExtractor):
                 'duration': 186.080,
                 'age_limit': 0,
             },
-        },
+        }, {
+            'url': 'https://cloud.tvigle.ru/video/5267604/',
+            'only_matching': True,
+        }
     ]
 
     def _real_extract(self, url):
-        display_id = self._match_id(url)
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        display_id = mobj.group('display_id')
 
-        webpage = self._download_webpage(url, display_id)
-
-        video_id = self._html_search_regex(
-            r'<li class="video-preview current_playing" id="(\d+)">', webpage, 'video id')
+        if not video_id:
+            webpage = self._download_webpage(url, display_id)
+            video_id = self._html_search_regex(
+                r'<li class="video-preview current_playing" id="(\d+)">',
+                webpage, 'video id')
 
         video_data = self._download_json(
             'http://cloud.tvigle.ru/api/play/video/%s/' % video_id, display_id)

From d6eb66ed3c0dc1362c1fec46b2d9a60076812f50 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 7 Feb 2015 01:46:54 +0600
Subject: [PATCH 0257/6219] [aftenposten] Add extractor (Closes #4863)

---
 youtube_dl/extractor/__init__.py    |   1 +
 youtube_dl/extractor/aftenposten.py | 103 ++++++++++++++++++++++++++++
 2 files changed, 104 insertions(+)
 create mode 100644 youtube_dl/extractor/aftenposten.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 047f7002a..0d7a120bc 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -6,6 +6,7 @@ from .academicearth import AcademicEarthCourseIE
 from .addanime import AddAnimeIE
 from .adobetv import AdobeTVIE
 from .adultswim import AdultSwimIE
+from .aftenposten import AftenpostenIE
 from .aftonbladet import AftonbladetIE
 from .aljazeera import AlJazeeraIE
 from .alphaporno import AlphaPornoIE
diff --git a/youtube_dl/extractor/aftenposten.py b/youtube_dl/extractor/aftenposten.py
new file mode 100644
index 000000000..2b257ede7
--- /dev/null
+++ b/youtube_dl/extractor/aftenposten.py
@@ -0,0 +1,103 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    parse_iso8601,
+    xpath_with_ns,
+    xpath_text,
+    find_xpath_attr,
+)
+
+
+class AftenpostenIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?aftenposten\.no/webtv/([^/]+/)*(?P<id>[^/]+)-\d+\.html'
+
+    _TEST = {
+        'url': 'http://www.aftenposten.no/webtv/serier-og-programmer/sweatshopenglish/TRAILER-SWEATSHOP---I-cant-take-any-more-7800835.html?paging=&section=webtv_serierogprogrammer_sweatshop_sweatshopenglish',
+        'md5': 'fd828cd29774a729bf4d4425fe192972',
+        'info_dict': {
+            'id': '21039',
+            'ext': 'mov',
+            'title': 'TRAILER: "Sweatshop" - I can´t take any more',
+            'description': 'md5:21891f2b0dd7ec2f78d84a50e54f8238',
+            'timestamp': 1416927969,
+            'upload_date': '20141125',
+        }
+    }
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, display_id)
+
+        video_id = self._html_search_regex(
+            r'data-xs-id="(\d+)"', webpage, 'video id')
+
+        data = self._download_xml(
+            'http://frontend.xstream.dk/ap/feed/video/?platform=web&id=%s' % video_id, video_id)
+
+        NS_MAP = {
+            'atom': 'http://www.w3.org/2005/Atom',
+            'xt': 'http://xstream.dk/',
+            'media': 'http://search.yahoo.com/mrss/',
+        }
+
+        entry = data.find(xpath_with_ns('./atom:entry', NS_MAP))
+
+        title = xpath_text(
+            entry, xpath_with_ns('./atom:title', NS_MAP), 'title')
+        description = xpath_text(
+            entry, xpath_with_ns('./atom:summary', NS_MAP), 'description')
+        timestamp = parse_iso8601(xpath_text(
+            entry, xpath_with_ns('./atom:published', NS_MAP), 'upload date'))
+
+        formats = []
+        media_group = entry.find(xpath_with_ns('./media:group', NS_MAP))
+        for media_content in media_group.findall(xpath_with_ns('./media:content', NS_MAP)):
+            media_url = media_content.get('url')
+            if not media_url:
+                continue
+            tbr = int_or_none(media_content.get('bitrate'))
+            mobj = re.search(r'^(?P<url>rtmp://[^/]+/(?P<app>[^/]+))/(?P<playpath>.+)$', media_url)
+            if mobj:
+                formats.append({
+                    'url': mobj.group('url'),
+                    'play_path': 'mp4:%s' % mobj.group('playpath'),
+                    'app': mobj.group('app'),
+                    'ext': 'flv',
+                    'tbr': tbr,
+                    'format_id': 'rtmp-%d' % tbr,
+                })
+            else:
+                formats.append({
+                    'url': media_url,
+                    'tbr': tbr,
+                })
+        self._sort_formats(formats)
+
+        link = find_xpath_attr(
+            entry, xpath_with_ns('./atom:link', NS_MAP), 'rel', 'original')
+        if link is not None:
+            formats.append({
+                'url': link.get('href'),
+                'format_id': link.get('rel'),
+            })
+
+        thumbnails = [{
+            'url': splash.get('url'),
+            'width': int_or_none(splash.get('width')),
+            'height': int_or_none(splash.get('height')),
+        } for splash in media_group.findall(xpath_with_ns('./xt:splash', NS_MAP))]
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'timestamp': timestamp,
+            'formats': formats,
+            'thumbnails': thumbnails,
+        }

From ad5747bad146c76f545870911d5f2fb586c0f7b1 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Fri, 6 Feb 2015 22:59:17 +0200
Subject: [PATCH 0258/6219] [rtp] Construct regular HTTP download URLs (#4882)

---
 youtube_dl/extractor/rtp.py | 42 +++++++++++++++++++++++++++++--------
 1 file changed, 33 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/rtp.py b/youtube_dl/extractor/rtp.py
index 4511cba49..ecf4939cd 100644
--- a/youtube_dl/extractor/rtp.py
+++ b/youtube_dl/extractor/rtp.py
@@ -1,16 +1,16 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import json
+import re
 
 from .common import InfoExtractor
-from ..utils import js_to_json
 
 
 class RTPIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?rtp\.pt/play/p(?P<program_id>[0-9]+)/(?P<id>[^/?#]+)/?'
     _TESTS = [{
         'url': 'http://www.rtp.pt/play/p405/e174042/paixoes-cruzadas',
+        'md5': 'e736ce0c665e459ddb818546220b4ef8',
         'info_dict': {
             'id': 'e174042',
             'ext': 'mp3',
@@ -18,9 +18,6 @@ class RTPIE(InfoExtractor):
             'description': 'As paixões musicais de António Cartaxo e António Macedo',
             'thumbnail': 're:^https?://.*\.jpg',
         },
-        'params': {
-            'skip_download': True,  # RTMP download
-        },
     }, {
         'url': 'http://www.rtp.pt/play/p831/a-quimica-das-coisas',
         'only_matching': True,
@@ -37,21 +34,48 @@ class RTPIE(InfoExtractor):
 
         player_config = self._search_regex(
             r'(?s)RTPPLAY\.player\.newPlayer\(\s*(\{.*?\})\s*\)', webpage, 'player config')
-        config = json.loads(js_to_json(player_config))
+        config = self._parse_json(player_config, video_id)
 
         path, ext = config.get('file').rsplit('.', 1)
         formats = [{
+            'format_id': 'rtmp',
+            'ext': ext,
+            'vcodec': config.get('type') == 'audio' and 'none' or None,
+            'preference': -2,
+            'url': 'rtmp://{streamer:s}/{application:s}'.format(**config),
             'app': config.get('application'),
             'play_path': '{ext:s}:{path:s}'.format(ext=ext, path=path),
             'page_url': url,
-            'url': 'rtmp://{streamer:s}/{application:s}'.format(**config),
             'rtmp_live': config.get('live', False),
-            'ext': ext,
-            'vcodec': config.get('type') == 'audio' and 'none' or None,
             'player_url': 'http://programas.rtp.pt/play/player.swf?v3',
             'rtmp_real_time': True,
         }]
 
+        # Construct regular HTTP download URLs
+        replacements = {
+            'audio': {
+                'format_id': 'mp3',
+                'pattern': r'^nas2\.share/wavrss/',
+                'repl': 'http://rsspod.rtp.pt/podcasts/',
+                'vcodec': 'none',
+            },
+            'video': {
+                'format_id': 'mp4_h264',
+                'pattern': r'^nas2\.share/h264/',
+                'repl': 'http://rsspod.rtp.pt/videocasts/',
+                'vcodec': 'h264',
+            },
+        }
+        r = replacements[config['type']]
+        if re.match(r['pattern'], config['file']) is not None:
+            formats.append({
+                'format_id': r['format_id'],
+                'url': re.sub(r['pattern'], r['repl'], config['file']),
+                'vcodec': r['vcodec'],
+            })
+
+        self._sort_formats(formats)
+
         return {
             'id': video_id,
             'title': title,

From 467d3c9a0c364fe4e24c6efd1d20317a80eee547 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 6 Feb 2015 22:05:11 +0100
Subject: [PATCH 0259/6219] [ffmpeg] --extrac-audio: Use the same options for
 avconv and ffmpeg

They have been available in ffmpeg since version 0.9, and we require 1.0 or higher.
---
 youtube_dl/postprocessor/ffmpeg.py | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 4a4422c5a..01d25f760 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -166,14 +166,13 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
         if filecodec is None:
             raise PostProcessingError('WARNING: unable to obtain file audio codec with ffprobe')
 
-        uses_avconv = self._uses_avconv()
         more_opts = []
         if self._preferredcodec == 'best' or self._preferredcodec == filecodec or (self._preferredcodec == 'm4a' and filecodec == 'aac'):
             if filecodec == 'aac' and self._preferredcodec in ['m4a', 'best']:
                 # Lossless, but in another container
                 acodec = 'copy'
                 extension = 'm4a'
-                more_opts = ['-bsf:a' if uses_avconv else '-absf', 'aac_adtstoasc']
+                more_opts = ['-bsf:a', 'aac_adtstoasc']
             elif filecodec in ['aac', 'mp3', 'vorbis', 'opus']:
                 # Lossless if possible
                 acodec = 'copy'
@@ -189,9 +188,9 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
                 more_opts = []
                 if self._preferredquality is not None:
                     if int(self._preferredquality) < 10:
-                        more_opts += ['-q:a' if uses_avconv else '-aq', self._preferredquality]
+                        more_opts += ['-q:a', self._preferredquality]
                     else:
-                        more_opts += ['-b:a' if uses_avconv else '-ab', self._preferredquality + 'k']
+                        more_opts += ['-b:a', self._preferredquality + 'k']
         else:
             # We convert the audio (lossy)
             acodec = {'mp3': 'libmp3lame', 'aac': 'aac', 'm4a': 'aac', 'opus': 'opus', 'vorbis': 'libvorbis', 'wav': None}[self._preferredcodec]
@@ -200,13 +199,13 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
             if self._preferredquality is not None:
                 # The opus codec doesn't support the -aq option
                 if int(self._preferredquality) < 10 and extension != 'opus':
-                    more_opts += ['-q:a' if uses_avconv else '-aq', self._preferredquality]
+                    more_opts += ['-q:a', self._preferredquality]
                 else:
-                    more_opts += ['-b:a' if uses_avconv else '-ab', self._preferredquality + 'k']
+                    more_opts += ['-b:a', self._preferredquality + 'k']
             if self._preferredcodec == 'aac':
                 more_opts += ['-f', 'adts']
             if self._preferredcodec == 'm4a':
-                more_opts += ['-bsf:a' if uses_avconv else '-absf', 'aac_adtstoasc']
+                more_opts += ['-bsf:a', 'aac_adtstoasc']
             if self._preferredcodec == 'vorbis':
                 extension = 'ogg'
             if self._preferredcodec == 'wav':

From b3a286d69d2bf28ba32a77b0fe66932eb6d1bc65 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 6 Feb 2015 22:23:06 +0100
Subject: [PATCH 0260/6219] [YoutubeDL] _calc_cookies: add get_header method to
 _PseudoRequest (#4861)

---
 youtube_dl/YoutubeDL.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 9605f8f99..633e3d8a1 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -938,6 +938,9 @@ class YoutubeDL(object):
             def has_header(self, h):
                 return h in self.headers
 
+            def get_header(self, h, default=None):
+                return self.headers.get(h, default)
+
         pr = _PseudoRequest(info_dict['url'])
         self.cookiejar.add_cookie_header(pr)
         return pr.headers.get('Cookie')

From 2b4ecde2c8e7d405758a3a80190c646940fe7e9a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 6 Feb 2015 23:54:25 +0100
Subject: [PATCH 0261/6219] [test/YoutubeDL] Add a simple test for
 postprocesors

Just checks that the 'keepvideo' option works as intended.
---
 test/test_YoutubeDL.py | 32 ++++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 678b9f7d1..47ab102a9 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -13,6 +13,7 @@ import copy
 from test.helper import FakeYDL, assertRegexpMatches
 from youtube_dl import YoutubeDL
 from youtube_dl.extractor import YoutubeIE
+from youtube_dl.postprocessor.common import PostProcessor
 
 
 class YDL(FakeYDL):
@@ -370,5 +371,36 @@ class TestFormatSelection(unittest.TestCase):
             'vbr': 10,
         }), '^\s*10k$')
 
+    def test_postprocessors(self):
+        filename = 'post-processor-testfile.mp4'
+        audiofile = filename + '.mp3'
+
+        class SimplePP(PostProcessor):
+            def run(self, info):
+                print(audiofile)
+                with open(audiofile, 'wt') as f:
+                    f.write('EXAMPLE')
+                info['filepath']
+                return False, info
+
+        def run_pp(params):
+            with open(filename, 'wt') as f:
+                f.write('EXAMPLE')
+            ydl = YoutubeDL(params)
+            ydl.add_post_processor(SimplePP())
+            ydl.post_process(filename, {'filepath': filename})
+
+        run_pp({'keepvideo': True})
+        self.assertTrue(os.path.exists(filename), '%s doesn\'t exist' % filename)
+        self.assertTrue(os.path.exists(audiofile), '%s doesn\'t exist' % audiofile)
+        os.unlink(filename)
+        os.unlink(audiofile)
+
+        run_pp({'keepvideo': False})
+        self.assertFalse(os.path.exists(filename), '%s exists' % filename)
+        self.assertTrue(os.path.exists(audiofile), '%s doesn\'t exist' % audiofile)
+        os.unlink(audiofile)
+
+
 if __name__ == '__main__':
     unittest.main()

From 0161353d7d2d0f0fce1ecaa2f77471d787ee9bdd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 6 Feb 2015 23:58:01 +0100
Subject: [PATCH 0262/6219] [test/test_YoutubeDL] Remove debug print call

---
 test/test_YoutubeDL.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 47ab102a9..b1cd6a69f 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -377,7 +377,6 @@ class TestFormatSelection(unittest.TestCase):
 
         class SimplePP(PostProcessor):
             def run(self, info):
-                print(audiofile)
                 with open(audiofile, 'wt') as f:
                     f.write('EXAMPLE')
                 info['filepath']

From c0e46412e99ef87f5730326ab8fa4a56038c43d1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 8 Feb 2015 17:30:29 +0600
Subject: [PATCH 0263/6219] [aparat] Fix extraction (Closes #4897)

---
 youtube_dl/extractor/aparat.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/aparat.py b/youtube_dl/extractor/aparat.py
index 15006336f..5d70758a3 100644
--- a/youtube_dl/extractor/aparat.py
+++ b/youtube_dl/extractor/aparat.py
@@ -34,7 +34,8 @@ class AparatIE(InfoExtractor):
                      video_id + '/vt/frame')
         webpage = self._download_webpage(embed_url, video_id)
 
-        video_urls = re.findall(r'fileList\[[0-9]+\]\s*=\s*"([^"]+)"', webpage)
+        video_urls = [video_url.replace('\\/', '/') for video_url in re.findall(
+            r'(?:fileList\[[0-9]+\]\s*=|"file"\s*:)\s*"([^"]+)"', webpage)]
         for i, video_url in enumerate(video_urls):
             req = HEADRequest(video_url)
             res = self._request_webpage(
@@ -46,7 +47,7 @@ class AparatIE(InfoExtractor):
 
         title = self._search_regex(r'\s+title:\s*"([^"]+)"', webpage, 'title')
         thumbnail = self._search_regex(
-            r'\s+image:\s*"([^"]+)"', webpage, 'thumbnail', fatal=False)
+            r'image:\s*"([^"]+)"', webpage, 'thumbnail', fatal=False)
 
         return {
             'id': video_id,

From 77d2b106ccf81e50cf41a7aa0bb320433a84e110 Mon Sep 17 00:00:00 2001
From: robin <rderooij685@gmail.com>
Date: Sun, 8 Feb 2015 13:42:41 +0100
Subject: [PATCH 0264/6219] [Gamekings] Fix 404 when large isn't available

When trying to download some GameKings videos, not all worked. This was
because not all videos had a "/large"-URL available. The extractor
checks now if the /large URL is available, if it isn't, it tries to get
the normal URL.
---
 youtube_dl/extractor/gamekings.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/gamekings.py b/youtube_dl/extractor/gamekings.py
index cf8e90d7d..bac325bd8 100644
--- a/youtube_dl/extractor/gamekings.py
+++ b/youtube_dl/extractor/gamekings.py
@@ -31,6 +31,8 @@ class GamekingsIE(InfoExtractor):
 
         # Todo: add medium format
         video_url = video_url.replace(video_id, 'large/' + video_id)
+        if not (self._is_valid_url(video_url, video_id)):
+            video_url = video_url.replace(video_id + '/large', video_id)
 
         return {
             'id': video_id,

From f23a3ca69975346d5fc30ee09b0cdceb4a384879 Mon Sep 17 00:00:00 2001
From: robin <rderooij685@gmail.com>
Date: Sun, 8 Feb 2015 13:47:27 +0100
Subject: [PATCH 0265/6219] [Gamekings] Fixed typo in URL replacement

---
 youtube_dl/extractor/gamekings.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/gamekings.py b/youtube_dl/extractor/gamekings.py
index bac325bd8..1697f5e44 100644
--- a/youtube_dl/extractor/gamekings.py
+++ b/youtube_dl/extractor/gamekings.py
@@ -32,7 +32,7 @@ class GamekingsIE(InfoExtractor):
         # Todo: add medium format
         video_url = video_url.replace(video_id, 'large/' + video_id)
         if not (self._is_valid_url(video_url, video_id)):
-            video_url = video_url.replace(video_id + '/large', video_id)
+            video_url = video_url.replace(video_id + 'large/', video_id)
 
         return {
             'id': video_id,

From 2f38289b79ed6c265f2e7cc91e417d0f2178371a Mon Sep 17 00:00:00 2001
From: robin <rderooij685@gmail.com>
Date: Sun, 8 Feb 2015 13:49:32 +0100
Subject: [PATCH 0266/6219] [Gamekings] Fix order of replacement string

Oops.
---
 youtube_dl/extractor/gamekings.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/gamekings.py b/youtube_dl/extractor/gamekings.py
index 1697f5e44..a5d854ade 100644
--- a/youtube_dl/extractor/gamekings.py
+++ b/youtube_dl/extractor/gamekings.py
@@ -32,7 +32,7 @@ class GamekingsIE(InfoExtractor):
         # Todo: add medium format
         video_url = video_url.replace(video_id, 'large/' + video_id)
         if not (self._is_valid_url(video_url, video_id)):
-            video_url = video_url.replace(video_id + 'large/', video_id)
+            video_url = video_url.replace('large/' + video_id, video_id)
 
         return {
             'id': video_id,

From ba322d82090bd1126774e772b699283121ffa4b8 Mon Sep 17 00:00:00 2001
From: robin <rderooij685@gmail.com>
Date: Sun, 8 Feb 2015 14:23:37 +0100
Subject: [PATCH 0267/6219] [Gamekings] Added test and replaced video_url

Quick and dirty fix for the Gamekings extractor. It gives an error about
the video_url, but it downloads it now instead of giving a 404 error on
newer Gamekings videos
---
 youtube_dl/extractor/gamekings.py | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/gamekings.py b/youtube_dl/extractor/gamekings.py
index a5d854ade..aa589390a 100644
--- a/youtube_dl/extractor/gamekings.py
+++ b/youtube_dl/extractor/gamekings.py
@@ -7,7 +7,8 @@ from .common import InfoExtractor
 
 class GamekingsIE(InfoExtractor):
     _VALID_URL = r'http://www\.gamekings\.tv/videos/(?P<name>[0-9a-z\-]+)'
-    _TEST = {
+    _TESTS = [
+        {
         'url': 'http://www.gamekings.tv/videos/phoenix-wright-ace-attorney-dual-destinies-review/',
         # MD5 is flaky, seems to change regularly
         # 'md5': '2f32b1f7b80fdc5cb616efb4f387f8a3',
@@ -16,8 +17,17 @@ class GamekingsIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Phoenix Wright: Ace Attorney \u2013 Dual Destinies Review',
             'description': 'md5:36fd701e57e8c15ac8682a2374c99731',
+            }
+        },
+        {
+        'url': 'http://www.gamekings.tv/videos/the-legend-of-zelda-majoras-mask/',
+        'info_dict': {
+            'id': '118933752',
+            'ext': 'mp4',
+            'title': 'The Legend of Zelda: Majora’s Mask'
+            }
         }
-    }
+    ]
 
     def _real_extract(self, url):
 
@@ -33,6 +43,7 @@ class GamekingsIE(InfoExtractor):
         video_url = video_url.replace(video_id, 'large/' + video_id)
         if not (self._is_valid_url(video_url, video_id)):
             video_url = video_url.replace('large/' + video_id, video_id)
+            video_url = video_url.replace('http://stream.gamekings.tv/', '')
 
         return {
             'id': video_id,

From 8ca8cbe2bd262676cbe5a63aaee6559dfa795634 Mon Sep 17 00:00:00 2001
From: robin <rderooij685@gmail.com>
Date: Sun, 8 Feb 2015 14:41:14 +0100
Subject: [PATCH 0268/6219] [Gamekings] Check string for vimeo, fix test

The test now doesn't fail anymore. It just checks the string for having
"vimeo" in it, instead of using the method for URL-checking, since it's
returns an error.

The tests don't fail, and the extractor works fine now.
---
 youtube_dl/extractor/gamekings.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/gamekings.py b/youtube_dl/extractor/gamekings.py
index aa589390a..ba4d2e082 100644
--- a/youtube_dl/extractor/gamekings.py
+++ b/youtube_dl/extractor/gamekings.py
@@ -24,7 +24,8 @@ class GamekingsIE(InfoExtractor):
         'info_dict': {
             'id': '118933752',
             'ext': 'mp4',
-            'title': 'The Legend of Zelda: Majora’s Mask'
+            'title': 'The Legend of Zelda: Majora’s Mask',
+            'description': 'md5:9917825fe0e9f4057601fe1e38860de3'
             }
         }
     ]
@@ -41,7 +42,7 @@ class GamekingsIE(InfoExtractor):
 
         # Todo: add medium format
         video_url = video_url.replace(video_id, 'large/' + video_id)
-        if not (self._is_valid_url(video_url, video_id)):
+        if "vimeo" in video_url:
             video_url = video_url.replace('large/' + video_id, video_id)
             video_url = video_url.replace('http://stream.gamekings.tv/', '')
 

From 5d678df64a7a4a695da769f7691c86f39fca26bb Mon Sep 17 00:00:00 2001
From: robin <rderooij685@gmail.com>
Date: Sun, 8 Feb 2015 15:34:37 +0100
Subject: [PATCH 0269/6219] [Gamekings] Download playlist

Todo: URL and Thumbnail should be extracted with XPath
---
 youtube_dl/extractor/gamekings.py | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/gamekings.py b/youtube_dl/extractor/gamekings.py
index ba4d2e082..f2760d444 100644
--- a/youtube_dl/extractor/gamekings.py
+++ b/youtube_dl/extractor/gamekings.py
@@ -3,7 +3,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-
+from ..utils import xpath_text
 
 class GamekingsIE(InfoExtractor):
     _VALID_URL = r'http://www\.gamekings\.tv/videos/(?P<name>[0-9a-z\-]+)'
@@ -35,6 +35,17 @@ class GamekingsIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         name = mobj.group('name')
         webpage = self._download_webpage(url, name)
+
+        playlist_id = re.search(r'(?:gogoVideo)\(\d+,"?(?P<playlist_id>.*)"', webpage, re.MULTILINE).group('playlist_id')
+        playlist_url = 'http://www.gamekings.tv/wp-content/themes/gk2010/rss_playlist.php?id=' + playlist_id
+        playlist_rss = self._download_xml(playlist_url, playlist_id)
+
+        NS_MAP {
+            'rss': 'http://rss.jwpcdn.com/'
+         }
+
+        # Todo: Implement Xpath for searching the video link
+        
         video_url = self._og_search_video_url(webpage)
 
         video = re.search(r'[0-9]+', video_url)

From a14292e8480f339bdbf9e52e93460aec5c31ef13 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Sun, 8 Feb 2015 16:57:07 +0200
Subject: [PATCH 0270/6219] [soulanime] Remove extractor (#4554)

Was supposed to be deleted by 67c2bcd
---
 youtube_dl/extractor/soulanime.py | 80 -------------------------------
 1 file changed, 80 deletions(-)
 delete mode 100644 youtube_dl/extractor/soulanime.py

diff --git a/youtube_dl/extractor/soulanime.py b/youtube_dl/extractor/soulanime.py
deleted file mode 100644
index feef33e27..000000000
--- a/youtube_dl/extractor/soulanime.py
+++ /dev/null
@@ -1,80 +0,0 @@
-from __future__ import unicode_literals
-
-import re
-
-from .common import InfoExtractor
-from ..utils import (
-    HEADRequest,
-    urlhandle_detect_ext,
-)
-
-
-class SoulAnimeWatchingIE(InfoExtractor):
-    IE_NAME = "soulanime:watching"
-    IE_DESC = "SoulAnime video"
-    _TEST = {
-        'url': 'http://www.soul-anime.net/watching/seirei-tsukai-no-blade-dance-episode-9/',
-        'md5': '05fae04abf72298098b528e98abf4298',
-        'info_dict': {
-            'id': 'seirei-tsukai-no-blade-dance-episode-9',
-            'ext': 'mp4',
-            'title': 'seirei-tsukai-no-blade-dance-episode-9',
-            'description': 'seirei-tsukai-no-blade-dance-episode-9'
-        }
-    }
-    _VALID_URL = r'http://[w.]*soul-anime\.(?P<domain>[^/]+)/watch[^/]*/(?P<id>[^/]+)'
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        domain = mobj.group('domain')
-
-        page = self._download_webpage(url, video_id)
-
-        video_url_encoded = self._html_search_regex(
-            r'<div id="download">[^<]*<a href="(?P<url>[^"]+)"', page, 'url')
-        video_url = "http://www.soul-anime." + domain + video_url_encoded
-
-        ext_req = HEADRequest(video_url)
-        ext_handle = self._request_webpage(
-            ext_req, video_id, note='Determining extension')
-        ext = urlhandle_detect_ext(ext_handle)
-
-        return {
-            'id': video_id,
-            'url': video_url,
-            'ext': ext,
-            'title': video_id,
-            'description': video_id
-        }
-
-
-class SoulAnimeSeriesIE(InfoExtractor):
-    IE_NAME = "soulanime:series"
-    IE_DESC = "SoulAnime Series"
-
-    _VALID_URL = r'http://[w.]*soul-anime\.(?P<domain>[^/]+)/anime./(?P<id>[^/]+)'
-
-    _EPISODE_REGEX = r'<option value="(/watch[^/]*/[^"]+)">[^<]*</option>'
-
-    _TEST = {
-        'url': 'http://www.soul-anime.net/anime1/black-rock-shooter-tv/',
-        'info_dict': {
-            'id': 'black-rock-shooter-tv'
-        },
-        'playlist_count': 8
-    }
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        series_id = mobj.group('id')
-        domain = mobj.group('domain')
-
-        pattern = re.compile(self._EPISODE_REGEX)
-
-        page = self._download_webpage(url, series_id, "Downloading series page")
-        mobj = pattern.findall(page)
-
-        entries = [self.url_result("http://www.soul-anime." + domain + obj) for obj in mobj]
-
-        return self.playlist_result(entries, series_id)

From 69319969de40f62ccf1c471427d309b6555483f9 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Sun, 8 Feb 2015 17:39:00 +0200
Subject: [PATCH 0271/6219] [extractor/common] Add new helper method
 _family_friendly_search

---
 youtube_dl/extractor/common.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 980b63680..2f5ba7aee 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -656,6 +656,21 @@ class InfoExtractor(object):
         }
         return RATING_TABLE.get(rating.lower(), None)
 
+    def _family_friendly_search(self, html):
+        # See http://schema.org/VideoObj
+        family_friendly = self._html_search_meta('isFamilyFriendly', html)
+
+        if not family_friendly:
+            return None
+
+        RATING_TABLE = {
+            '1': 0,
+            'true': 0,
+            '0': 18,
+            'false': 18,
+        }
+        return RATING_TABLE.get(family_friendly.lower(), None)
+
     def _twitter_search_player(self, html):
         return self._html_search_meta('twitter:player', html,
                                       'twitter card player')

From 955c5505e7df737b452d54d869958a8f5425edf5 Mon Sep 17 00:00:00 2001
From: robin <rderooij685@gmail.com>
Date: Sun, 8 Feb 2015 16:44:25 +0100
Subject: [PATCH 0272/6219] [Gamekings] Use xpath

XPath is used for extracting the video url and the thumbnail
---
 youtube_dl/extractor/gamekings.py | 27 +++++++++++++++------------
 1 file changed, 15 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/gamekings.py b/youtube_dl/extractor/gamekings.py
index f2760d444..3c1f7f195 100644
--- a/youtube_dl/extractor/gamekings.py
+++ b/youtube_dl/extractor/gamekings.py
@@ -3,7 +3,11 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import xpath_text
+from ..utils import (
+    xpath_text,
+    xpath_with_ns
+ )
+
 
 class GamekingsIE(InfoExtractor):
     _VALID_URL = r'http://www\.gamekings\.tv/videos/(?P<name>[0-9a-z\-]+)'
@@ -39,23 +43,22 @@ class GamekingsIE(InfoExtractor):
         playlist_id = re.search(r'(?:gogoVideo)\(\d+,"?(?P<playlist_id>.*)"', webpage, re.MULTILINE).group('playlist_id')
         playlist_url = 'http://www.gamekings.tv/wp-content/themes/gk2010/rss_playlist.php?id=' + playlist_id
         playlist_rss = self._download_xml(playlist_url, playlist_id)
+        
 
-        NS_MAP {
-            'rss': 'http://rss.jwpcdn.com/'
+        NS_MAP = {
+            'jwplayer': 'http://rss.jwpcdn.com/'
          }
 
-        # Todo: Implement Xpath for searching the video link
+        item = playlist_rss.find('./channel/item')
         
-        video_url = self._og_search_video_url(webpage)
-
+        image = xpath_text(item, xpath_with_ns('./jwplayer:image', NS_MAP), 'image')
+        file_node = item.find(xpath_with_ns('./jwplayer:source', NS_MAP))
+        
+        video_url = file_node.get('file')
         video = re.search(r'[0-9]+', video_url)
         video_id = video.group(0)
-
-        # Todo: add medium format
-        video_url = video_url.replace(video_id, 'large/' + video_id)
-        if "vimeo" in video_url:
-            video_url = video_url.replace('large/' + video_id, video_id)
-            video_url = video_url.replace('http://stream.gamekings.tv/', '')
+        
+        # Todo: Add medium format
 
         return {
             'id': video_id,

From 641eb10d34629c177445dd38e9d3f1f73c4cb92f Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Sun, 8 Feb 2015 17:45:38 +0200
Subject: [PATCH 0273/6219] Use _family_friendly_search for determining
 age_limit

---
 youtube_dl/extractor/aparat.py   | 2 ++
 youtube_dl/extractor/goshgay.py  | 4 +---
 youtube_dl/extractor/izlesene.py | 5 +----
 youtube_dl/extractor/teamcoco.py | 7 +++++--
 4 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/aparat.py b/youtube_dl/extractor/aparat.py
index 5d70758a3..63429780e 100644
--- a/youtube_dl/extractor/aparat.py
+++ b/youtube_dl/extractor/aparat.py
@@ -20,6 +20,7 @@ class AparatIE(InfoExtractor):
             'id': 'wP8On',
             'ext': 'mp4',
             'title': 'تیم گلکسی 11 - زومیت',
+            'age_limit': 0,
         },
         # 'skip': 'Extremely unreliable',
     }
@@ -55,4 +56,5 @@ class AparatIE(InfoExtractor):
             'url': video_url,
             'ext': 'mp4',
             'thumbnail': thumbnail,
+            'age_limit': self._family_friendly_search(webpage),
         }
diff --git a/youtube_dl/extractor/goshgay.py b/youtube_dl/extractor/goshgay.py
index b116d251d..1d9166455 100644
--- a/youtube_dl/extractor/goshgay.py
+++ b/youtube_dl/extractor/goshgay.py
@@ -34,8 +34,6 @@ class GoshgayIE(InfoExtractor):
         duration = parse_duration(self._html_search_regex(
             r'<span class="duration">\s*-?\s*(.*?)</span>',
             webpage, 'duration', fatal=False))
-        family_friendly = self._html_search_meta(
-            'isFamilyFriendly', webpage, default='false')
 
         flashvars = compat_parse_qs(self._html_search_regex(
             r'<embed.+?id="flash-player-embed".+?flashvars="([^"]+)"',
@@ -49,5 +47,5 @@ class GoshgayIE(InfoExtractor):
             'title': title,
             'thumbnail': thumbnail,
             'duration': duration,
-            'age_limit': 0 if family_friendly == 'true' else 18,
+            'age_limit': self._family_friendly_search(webpage),
         }
diff --git a/youtube_dl/extractor/izlesene.py b/youtube_dl/extractor/izlesene.py
index d16d483ee..99a1361f8 100644
--- a/youtube_dl/extractor/izlesene.py
+++ b/youtube_dl/extractor/izlesene.py
@@ -80,9 +80,6 @@ class IzleseneIE(InfoExtractor):
             r'comment_count\s*=\s*\'([^\']+)\';',
             webpage, 'comment_count', fatal=False)
 
-        family_friendly = self._html_search_meta(
-            'isFamilyFriendly', webpage, 'age limit', fatal=False)
-
         content_url = self._html_search_meta(
             'contentURL', webpage, 'content URL', fatal=False)
         ext = determine_ext(content_url, 'mp4')
@@ -120,6 +117,6 @@ class IzleseneIE(InfoExtractor):
             'duration': duration,
             'view_count': int_or_none(view_count),
             'comment_count': int_or_none(comment_count),
-            'age_limit': 18 if family_friendly == 'False' else 0,
+            'age_limit': self._family_friendly_search(webpage),
             'formats': formats,
         }
diff --git a/youtube_dl/extractor/teamcoco.py b/youtube_dl/extractor/teamcoco.py
index 18a823719..e85d452a3 100644
--- a/youtube_dl/extractor/teamcoco.py
+++ b/youtube_dl/extractor/teamcoco.py
@@ -15,7 +15,8 @@ class TeamcocoIE(InfoExtractor):
                 'id': '80187',
                 'ext': 'mp4',
                 'title': 'Conan Becomes A Mary Kay Beauty Consultant',
-                'description': 'Mary Kay is perhaps the most trusted name in female beauty, so of course Conan is a natural choice to sell their products.'
+                'description': 'Mary Kay is perhaps the most trusted name in female beauty, so of course Conan is a natural choice to sell their products.',
+                'age_limit': 0,
             }
         }, {
             'url': 'http://teamcoco.com/video/louis-ck-interview-george-w-bush',
@@ -24,7 +25,8 @@ class TeamcocoIE(InfoExtractor):
                 'id': '19705',
                 'ext': 'mp4',
                 "description": "Louis C.K. got starstruck by George W. Bush, so what? Part one.",
-                "title": "Louis C.K. Interview Pt. 1 11/3/11"
+                "title": "Louis C.K. Interview Pt. 1 11/3/11",
+                'age_limit': 0,
             }
         }
     ]
@@ -83,4 +85,5 @@ class TeamcocoIE(InfoExtractor):
             'title': self._og_search_title(webpage),
             'thumbnail': self._og_search_thumbnail(webpage),
             'description': self._og_search_description(webpage),
+            'age_limit': self._family_friendly_search(webpage),
         }

From c36b09a5026172c1ca452038fffccd68b14c528c Mon Sep 17 00:00:00 2001
From: robin <rderooij685@gmail.com>
Date: Sun, 8 Feb 2015 16:46:13 +0100
Subject: [PATCH 0274/6219] [Gamekings] Use thumbnail in return statement

---
 youtube_dl/extractor/gamekings.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/gamekings.py b/youtube_dl/extractor/gamekings.py
index 3c1f7f195..1821f26d8 100644
--- a/youtube_dl/extractor/gamekings.py
+++ b/youtube_dl/extractor/gamekings.py
@@ -66,4 +66,5 @@ class GamekingsIE(InfoExtractor):
             'url': video_url,
             'title': self._og_search_title(webpage),
             'description': self._og_search_description(webpage),
+            'thumbnail': image
         }

From d87ec897e99d503e30fb677b0b4acf7256fa6143 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 8 Feb 2015 23:03:12 +0600
Subject: [PATCH 0275/6219] [gamekings] Improve extraction

---
 youtube_dl/extractor/gamekings.py | 64 ++++++++++++++-----------------
 1 file changed, 29 insertions(+), 35 deletions(-)

diff --git a/youtube_dl/extractor/gamekings.py b/youtube_dl/extractor/gamekings.py
index 1821f26d8..929779f60 100644
--- a/youtube_dl/extractor/gamekings.py
+++ b/youtube_dl/extractor/gamekings.py
@@ -1,70 +1,64 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..utils import (
     xpath_text,
-    xpath_with_ns
- )
+    xpath_with_ns,
+)
 
 
 class GamekingsIE(InfoExtractor):
-    _VALID_URL = r'http://www\.gamekings\.tv/videos/(?P<name>[0-9a-z\-]+)'
-    _TESTS = [
-        {
+    _VALID_URL = r'http://www\.gamekings\.tv/videos/(?P<id>[^/]+)'
+    _TESTS = [{
         'url': 'http://www.gamekings.tv/videos/phoenix-wright-ace-attorney-dual-destinies-review/',
         # MD5 is flaky, seems to change regularly
         # 'md5': '2f32b1f7b80fdc5cb616efb4f387f8a3',
         'info_dict': {
-            'id': '20130811',
+            'id': 'phoenix-wright-ace-attorney-dual-destinies-review',
             'ext': 'mp4',
             'title': 'Phoenix Wright: Ace Attorney \u2013 Dual Destinies Review',
             'description': 'md5:36fd701e57e8c15ac8682a2374c99731',
-            }
+            'thumbnail': 're:^https?://.*\.jpg$',
         },
-        {
+    }, {
+        # vimeo video
         'url': 'http://www.gamekings.tv/videos/the-legend-of-zelda-majoras-mask/',
+        'md5': '12bf04dfd238e70058046937657ea68d',
         'info_dict': {
-            'id': '118933752',
+            'id': 'the-legend-of-zelda-majoras-mask',
             'ext': 'mp4',
             'title': 'The Legend of Zelda: Majora’s Mask',
-            'description': 'md5:9917825fe0e9f4057601fe1e38860de3'
-            }
-        }
-    ]
+            'description': 'md5:9917825fe0e9f4057601fe1e38860de3',
+            'thumbnail': 're:^https?://.*\.jpg$',
+        },
+    }]
 
     def _real_extract(self, url):
+        video_id = self._match_id(url)
 
-        mobj = re.match(self._VALID_URL, url)
-        name = mobj.group('name')
-        webpage = self._download_webpage(url, name)
+        webpage = self._download_webpage(url, video_id)
 
-        playlist_id = re.search(r'(?:gogoVideo)\(\d+,"?(?P<playlist_id>.*)"', webpage, re.MULTILINE).group('playlist_id')
-        playlist_url = 'http://www.gamekings.tv/wp-content/themes/gk2010/rss_playlist.php?id=' + playlist_id
-        playlist_rss = self._download_xml(playlist_url, playlist_id)
-        
+        playlist_id = self._search_regex(
+            r'gogoVideo\(\s*\d+\s*,\s*"([^"]+)', webpage, 'playlist id')
+
+        playlist = self._download_xml(
+            'http://www.gamekings.tv/wp-content/themes/gk2010/rss_playlist.php?id=%s' % playlist_id,
+            video_id)
 
         NS_MAP = {
             'jwplayer': 'http://rss.jwpcdn.com/'
-         }
+        }
 
-        item = playlist_rss.find('./channel/item')
-        
-        image = xpath_text(item, xpath_with_ns('./jwplayer:image', NS_MAP), 'image')
-        file_node = item.find(xpath_with_ns('./jwplayer:source', NS_MAP))
-        
-        video_url = file_node.get('file')
-        video = re.search(r'[0-9]+', video_url)
-        video_id = video.group(0)
-        
-        # Todo: Add medium format
+        item = playlist.find('./channel/item')
+
+        thumbnail = xpath_text(item, xpath_with_ns('./jwplayer:image', NS_MAP), 'thumbnail')
+        video_url = item.find(xpath_with_ns('./jwplayer:source', NS_MAP)).get('file')
 
         return {
             'id': video_id,
-            'ext': 'mp4',
             'url': video_url,
             'title': self._og_search_title(webpage),
             'description': self._og_search_description(webpage),
-            'thumbnail': image
+            'thumbnail': thumbnail,
         }

From 41c23b0da58ba36805a67f4736890475ab7abedf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 8 Feb 2015 23:12:59 +0600
Subject: [PATCH 0276/6219] [gamekings] Support videos from news pages

---
 youtube_dl/extractor/gamekings.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/gamekings.py b/youtube_dl/extractor/gamekings.py
index 929779f60..027f55eb2 100644
--- a/youtube_dl/extractor/gamekings.py
+++ b/youtube_dl/extractor/gamekings.py
@@ -9,7 +9,7 @@ from ..utils import (
 
 
 class GamekingsIE(InfoExtractor):
-    _VALID_URL = r'http://www\.gamekings\.tv/videos/(?P<id>[^/]+)'
+    _VALID_URL = r'http://www\.gamekings\.tv/(?:videos|nieuws)/(?P<id>[^/]+)'
     _TESTS = [{
         'url': 'http://www.gamekings.tv/videos/phoenix-wright-ace-attorney-dual-destinies-review/',
         # MD5 is flaky, seems to change regularly
@@ -32,6 +32,9 @@ class GamekingsIE(InfoExtractor):
             'description': 'md5:9917825fe0e9f4057601fe1e38860de3',
             'thumbnail': 're:^https?://.*\.jpg$',
         },
+    }, {
+        'url': 'http://www.gamekings.tv/nieuws/gamekings-extra-shelly-en-david-bereiden-zich-voor-op-de-livestream/',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 12931e1c6eb85adeffa19dcbd3b911335a5ab032 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 8 Feb 2015 23:33:29 +0600
Subject: [PATCH 0277/6219] Credit @robin007bond for tweakers (#4881) and
 gamekings fixes (#4901)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 85ca81a27..3d6985ab6 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -109,3 +109,4 @@ David Luhmer
 Shaya Goldberg
 Paul Hartmann
 Frans de Jonge
+Robin de Rooij

From 9ddb6925bfc53e0f31f7d114490259e293e91edd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 9 Feb 2015 01:07:43 +0600
Subject: [PATCH 0278/6219] [YoutubeDL] Allow filtering by string properties
 (#4906)

---
 youtube_dl/YoutubeDL.py | 41 +++++++++++++++++++++++++++++------------
 1 file changed, 29 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 633e3d8a1..0934911ec 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -832,21 +832,38 @@ class YoutubeDL(object):
             \]$
             ''' % '|'.join(map(re.escape, OPERATORS.keys())))
         m = operator_rex.search(format_spec)
+        if m:
+            try:
+                comparison_value = int(m.group('value'))
+            except ValueError:
+                comparison_value = parse_filesize(m.group('value'))
+                if comparison_value is None:
+                    comparison_value = parse_filesize(m.group('value') + 'B')
+                if comparison_value is None:
+                    raise ValueError(
+                        'Invalid value %r in format specification %r' % (
+                            m.group('value'), format_spec))
+            op = OPERATORS[m.group('op')]
+
+        if not m:
+            STR_OPERATORS = {
+                '=': operator.eq,
+                '!=': operator.ne,
+            }
+            str_operator_rex = re.compile(r'''(?x)\s*\[
+                \s*(?P<key>ext|acodec|vcodec|container|protocol)
+                \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?
+                \s*(?P<value>[a-zA-Z0-9_-]+)
+                \s*\]$
+                ''' % '|'.join(map(re.escape, STR_OPERATORS.keys())))
+            m = str_operator_rex.search(format_spec)
+            if m:
+                comparison_value = m.group('value')
+                op = STR_OPERATORS[m.group('op')]
+
         if not m:
             raise ValueError('Invalid format specification %r' % format_spec)
 
-        try:
-            comparison_value = int(m.group('value'))
-        except ValueError:
-            comparison_value = parse_filesize(m.group('value'))
-            if comparison_value is None:
-                comparison_value = parse_filesize(m.group('value') + 'B')
-            if comparison_value is None:
-                raise ValueError(
-                    'Invalid value %r in format specification %r' % (
-                        m.group('value'), format_spec))
-        op = OPERATORS[m.group('op')]
-
         def _filter(f):
             actual_value = f.get(m.group('key'))
             if actual_value is None:

From 2ec19e955870e82bd91949b0996dfd9acdde0232 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 9 Feb 2015 01:09:45 +0600
Subject: [PATCH 0279/6219] [YoutubeDL] Allow filtering by audio sampling rate

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 0934911ec..c7108dbc1 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -826,7 +826,7 @@ class YoutubeDL(object):
             '!=': operator.ne,
         }
         operator_rex = re.compile(r'''(?x)\s*\[
-            (?P<key>width|height|tbr|abr|vbr|filesize|fps)
+            (?P<key>width|height|tbr|abr|vbr|asr|filesize|fps)
             \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
             (?P<value>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)
             \]$

From 6128bf07a91b941b9241c5ea0d611df9e69d48b7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 9 Feb 2015 01:27:27 +0600
Subject: [PATCH 0280/6219] [options] Update help on string comparisons

---
 youtube_dl/options.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index d5e575964..6a795e750 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -298,7 +298,9 @@ def parseOpts(overrideArguments=None):
             ' brackets, as in -f "best[height=720]"'
             ' (or -f "[filesize>10M]"). '
             ' This works for filesize, height, width, tbr, abr, vbr, and fps'
-            ' and the comparisons <, <=, >, >=, =, != .'
+            ' and the comparisons <, <=, >, >=, =, !='
+            ' and for ext, acodec, vcodec, container and protocol'
+            ' and the comparisons =, != .'
             ' Formats for which the value is not known are excluded unless you'
             ' put a question mark (?) after the operator.'
             ' You can combine format filters, so  '

From b29231c0401b7b70337bb52b0c20bc92aa3b704e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 8 Feb 2015 20:28:38 +0100
Subject: [PATCH 0281/6219] release 2015.02.08

---
 README.md              | 4 +++-
 docs/supportedsites.md | 1 +
 youtube_dl/version.py  | 2 +-
 3 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 06dea400d..3eafad7c7 100644
--- a/README.md
+++ b/README.md
@@ -294,7 +294,9 @@ which means you can modify it, redistribute it or use it however you like.
                                      -f "[filesize>10M]").  This works for
                                      filesize, height, width, tbr, abr, vbr, and
                                      fps and the comparisons <, <=, >, >=, =, !=
-                                     . Formats for which the value is not known
+                                     and for ext, acodec, vcodec, container and
+                                     protocol and the comparisons =, != .
+                                     Formats for which the value is not known
                                      are excluded unless you put a question mark
                                      (?) after the operator. You can combine
                                      format filters, so  -f "[height <=?
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 2d8f9c316..84a132684 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -14,6 +14,7 @@
  - **AddAnime**
  - **AdobeTV**
  - **AdultSwim**
+ - **Aftenposten**
  - **Aftonbladet**
  - **AlJazeera**
  - **Allocine**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 1091ae61b..b0ddff6e2 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.06'
+__version__ = '2015.02.08'

From 64f9baa084c60b8bfd8e33179fd6caa50084a642 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 9 Feb 2015 01:35:16 +0600
Subject: [PATCH 0282/6219] [options] Mention asr as possible filter

---
 youtube_dl/options.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 6a795e750..048d9992c 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -297,9 +297,9 @@ def parseOpts(overrideArguments=None):
             ' You can filter the video results by putting a condition in'
             ' brackets, as in -f "best[height=720]"'
             ' (or -f "[filesize>10M]"). '
-            ' This works for filesize, height, width, tbr, abr, vbr, and fps'
+            ' This works for filesize, height, width, tbr, abr, vbr, asr, and fps'
             ' and the comparisons <, <=, >, >=, =, !='
-            ' and for ext, acodec, vcodec, container and protocol'
+            ' and for ext, acodec, vcodec, container, and protocol'
             ' and the comparisons =, != .'
             ' Formats for which the value is not known are excluded unless you'
             ' put a question mark (?) after the operator.'

From bdb186f3b074f1f7ac0f154be419ca04241add6f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Markus=20M=C3=BCller?= <mmu@grummel.net>
Date: Sun, 8 Feb 2015 21:55:39 +0100
Subject: [PATCH 0283/6219] fix rtlnow for newer series like "Der Bachelor"
 season 5

---
 youtube_dl/extractor/rtlnow.py | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/rtlnow.py b/youtube_dl/extractor/rtlnow.py
index 285c3c4be..e235f3c57 100644
--- a/youtube_dl/extractor/rtlnow.py
+++ b/youtube_dl/extractor/rtlnow.py
@@ -134,9 +134,18 @@ class RTLnowIE(InfoExtractor):
                     'player_url': video_page_url + 'includes/vodplayer.swf',
                 }
             else:
-                fmt = {
-                    'url': filename.text,
-                }
+                mobj = re.search(r'.*/(?P<hoster>[^/]+)/videos/(?P<play_path>.+)\.f4m', filename.text)
+                if mobj:
+                    fmt = {
+                        'url': 'rtmpe://fmspay-fra2.rtl.de/' + mobj.group('hoster'),
+                        'play_path': 'mp4:' + mobj.group('play_path'),
+                        'page_url': url,
+                        'player_url': video_page_url + 'includes/vodplayer.swf',
+                    }
+                else:
+                    fmt = {
+                        'url': filename.text,
+                    }
             fmt.update({
                 'width': int_or_none(filename.get('width')),
                 'height': int_or_none(filename.get('height')),

From dd8930684ea4069a1c4e28b3298b0d328145d00f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 9 Feb 2015 10:28:16 +0100
Subject: [PATCH 0284/6219] release 2015.02.09

---
 README.md             | 28 ++++++++++++++--------------
 youtube_dl/version.py |  2 +-
 2 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/README.md b/README.md
index 3eafad7c7..23295df4b 100644
--- a/README.md
+++ b/README.md
@@ -292,20 +292,20 @@ which means you can modify it, redistribute it or use it however you like.
                                      video results by putting a condition in
                                      brackets, as in -f "best[height=720]" (or
                                      -f "[filesize>10M]").  This works for
-                                     filesize, height, width, tbr, abr, vbr, and
-                                     fps and the comparisons <, <=, >, >=, =, !=
-                                     and for ext, acodec, vcodec, container and
-                                     protocol and the comparisons =, != .
-                                     Formats for which the value is not known
-                                     are excluded unless you put a question mark
-                                     (?) after the operator. You can combine
-                                     format filters, so  -f "[height <=?
-                                     720][tbr>500]" selects up to 720p videos
-                                     (or videos where the height is not known)
-                                     with a bitrate of at least 500 KBit/s. By
-                                     default, youtube-dl will pick the best
-                                     quality. Use commas to download multiple
-                                     audio formats, such as -f
+                                     filesize, height, width, tbr, abr, vbr,
+                                     asr, and fps and the comparisons <, <=, >,
+                                     >=, =, != and for ext, acodec, vcodec,
+                                     container, and protocol and the comparisons
+                                     =, != . Formats for which the value is not
+                                     known are excluded unless you put a
+                                     question mark (?) after the operator. You
+                                     can combine format filters, so  -f "[height
+                                     <=? 720][tbr>500]" selects up to 720p
+                                     videos (or videos where the height is not
+                                     known) with a bitrate of at least 500
+                                     KBit/s. By default, youtube-dl will pick
+                                     the best quality. Use commas to download
+                                     multiple audio formats, such as -f
                                      136/137/mp4/bestvideo,140/m4a/bestaudio.
                                      You can merge the video and audio of two
                                      formats into a single file using -f <video-
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index b0ddff6e2..b62fcd63a 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.08'
+__version__ = '2015.02.09'

From a0f719854463c6f4226e4042dfa80c1b17154e1d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 9 Feb 2015 10:42:25 +0100
Subject: [PATCH 0285/6219] [generic] Add support for jwPlayer YouTube videos

This makes nationalarchives.gov.uk work (Fixes #4907, fixes #4876)
---
 youtube_dl/extractor/generic.py | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index fbbc79a57..8a0f10803 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -524,6 +524,19 @@ class GenericIE(InfoExtractor):
                 'upload_date': '20150126',
             },
             'add_ie': ['Viddler'],
+        },
+        # jwplayer YouTube
+        {
+            'url': 'http://media.nationalarchives.gov.uk/index.php/webinar-using-discovery-national-archives-online-catalogue/',
+            'info_dict': {
+                'id': 'Mrj4DVp2zeA',
+                'ext': 'mp4',
+                'upload_date': '20150204',
+                'uploader': 'The National Archives UK',
+                'description': 'md5:a236581cd2449dd2df4f93412f3f01c6',
+                'uploader_id': 'NationalArchives08',
+                'title': 'Webinar: Using Discovery, The National Archives’ online catalogue',
+            },
         }
     ]
 
@@ -1065,6 +1078,8 @@ class GenericIE(InfoExtractor):
             return self.url_result(mobj.group('url'), 'Livestream')
 
         def check_video(vurl):
+            if YoutubeIE.suitable(vurl):
+                return True
             vpath = compat_urlparse.urlparse(vurl).path
             vext = determine_ext(vpath)
             return '.' in vpath and vext not in ('swf', 'png', 'jpg', 'srt', 'sbv', 'sub', 'vtt', 'ttml')
@@ -1082,7 +1097,8 @@ class GenericIE(InfoExtractor):
                     JWPlayerOptions|
                     jwplayer\s*\(\s*["'][^'"]+["']\s*\)\s*\.setup
                 )
-                .*?file\s*:\s*["\'](.*?)["\']''', webpage))
+                .*?
+                ['"]?file['"]?\s*:\s*["\'](.*?)["\']''', webpage))
         if not found:
             # Broaden the search a little bit
             found = filter_video(re.findall(r'[^A-Za-z0-9]?(?:file|source)=(http[^\'"&]*)', webpage))

From 5269028951d5abd6f20d46c267b0ef610b2becef Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 9 Feb 2015 10:47:19 +0100
Subject: [PATCH 0286/6219] [rtlnow] Add test for @mmue's extension (#4908)

---
 youtube_dl/extractor/rtlnow.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/youtube_dl/extractor/rtlnow.py b/youtube_dl/extractor/rtlnow.py
index e235f3c57..fd93cc66f 100644
--- a/youtube_dl/extractor/rtlnow.py
+++ b/youtube_dl/extractor/rtlnow.py
@@ -91,6 +91,15 @@ class RTLnowIE(InfoExtractor):
             },
         },
         {
+            'url': 'http://rtl-now.rtl.de/der-bachelor/folge-4.php?film_id=188729&player=1&season=5',
+            'info_dict': {
+                'id': '188729',
+                'ext': 'flv',
+                'upload_date': '20150204',
+                'description': 'md5:5e1ce23095e61a79c166d134b683cecc',
+                'title': 'Der Bachelor - Folge 4',
+            }
+        }, {
             'url': 'http://www.n-tvnow.de/deluxe-alles-was-spass-macht/thema-ua-luxushotel-fuer-vierbeiner.php?container_id=153819&player=1&season=0',
             'only_matching': True,
         },

From 196121c51baf2e7b7bb9c657f2f832b2840432de Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 9 Feb 2015 10:49:10 +0100
Subject: [PATCH 0287/6219] release 2015.02.09.1

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index b62fcd63a..e3f6d1e90 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.09'
+__version__ = '2015.02.09.1'

From e98b8e79ead03f33705356b32a08664ef2df2628 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 9 Feb 2015 14:46:10 +0100
Subject: [PATCH 0288/6219] [generic] Improve SBS detection (Fixes #4899)

---
 youtube_dl/extractor/generic.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 8a0f10803..f4500e931 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1047,7 +1047,12 @@ class GenericIE(InfoExtractor):
 
         # Look for embedded sbs.com.au player
         mobj = re.search(
-            r'<iframe[^>]+?src=(["\'])(?P<url>https?://(?:www\.)sbs\.com\.au/ondemand/video/single/.+?)\1',
+            r'''(?x)
+            (?:
+                <meta\s+property="og:video"\s+content=|
+                <iframe[^>]+?src=
+            )
+            (["\'])(?P<url>https?://(?:www\.)?sbs\.com\.au/ondemand/video/.+?)\1''',
             webpage)
         if mobj is not None:
             return self.url_result(mobj.group('url'), 'SBS')

From 3255fe7141a1e11d65997da3de436ba3007a8fc3 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 9 Feb 2015 14:46:30 +0100
Subject: [PATCH 0289/6219] release 2015.02.09.2

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index e3f6d1e90..d8911cae6 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.09.1'
+__version__ = '2015.02.09.2'

From ba617964585a118e30932461f24a6c3627047738 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 9 Feb 2015 15:04:22 +0100
Subject: [PATCH 0290/6219] [youtube] Don't override format info from the dash
 manifest (fixes #4911)

---
 youtube_dl/extractor/youtube.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index e4b26b84f..8b91fff18 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -780,8 +780,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
                     fo for fo in formats
                     if fo['format_id'] == format_id)
             except StopIteration:
-                f.update(self._formats.get(format_id, {}).items())
-                formats.append(f)
+                full_info = self._formats.get(format_id, {}).copy()
+                full_info.update(f)
+                formats.append(full_info)
             else:
                 existing_format.update(f)
         return formats

From 1309b396d0ff87f0e762f8893b6d0bd2aa96432a Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 9 Feb 2015 15:56:46 +0100
Subject: [PATCH 0291/6219] [svtplay] Add new extractor (Fixes #4914)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/svtplay.py  | 55 ++++++++++++++++++++++++++++++++
 2 files changed, 56 insertions(+)
 create mode 100644 youtube_dl/extractor/svtplay.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 0d7a120bc..55ca0d6e4 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -428,6 +428,7 @@ from .streamcloud import StreamcloudIE
 from .streamcz import StreamCZIE
 from .streetvoice import StreetVoiceIE
 from .sunporno import SunPornoIE
+from .svtplay import SVTPlayIE
 from .swrmediathek import SWRMediathekIE
 from .syfy import SyfyIE
 from .sztvhu import SztvHuIE
diff --git a/youtube_dl/extractor/svtplay.py b/youtube_dl/extractor/svtplay.py
new file mode 100644
index 000000000..727b3d1fc
--- /dev/null
+++ b/youtube_dl/extractor/svtplay.py
@@ -0,0 +1,55 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    determine_ext,
+)
+
+
+class SVTPlayIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?svtplay\.se/video/(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'http://www.svtplay.se/video/2609989/sm-veckan/sm-veckan-rally-final-sasong-1-sm-veckan-rally-final',
+        'md5': '2521cd644e862936cf2e698206e47385',
+        'info_dict': {
+            'id': '3966754',
+            'ext': 'mp4',
+            'title': 'FIFA 14 - E3 2013 Trailer',
+            'duration': 4500,
+            'thumbnail': 're:^https?://.*\.jpg$',
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        info = self._download_json(
+            'http://www.svtplay.se/video/%s?output=json' % video_id, video_id)
+
+        title = info['context']['title']
+        thumbnail = info['context'].get('thumbnailImage')
+
+        video_info = info['video']
+        formats = []
+        for vr in video_info['videoReferences']:
+            vurl = vr['url']
+            if determine_ext(vurl) == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    vurl, video_id,
+                    ext='mp4', entry_protocol='m3u8_native',
+                    m3u8_id=vr.get('playerType')))
+            else:
+                formats.append({
+                    'format_id': vr.get('playerType'),
+                    'url': vurl,
+                })
+        self._sort_formats(formats)
+
+        duration = video_info.get('materialLength')
+
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': formats,
+            'thumbnail': thumbnail,
+            'duration': duration,
+        }

From 43f244b6d51aefac4683d130d70adf94641bf6d8 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 9 Feb 2015 15:57:40 +0100
Subject: [PATCH 0292/6219] [YoutubeDL] Do not show worst in --list-formats
 output

Nobody wants to know what the worst possible format is. And if they do, they can still provide -f worst.
---
 youtube_dl/YoutubeDL.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index c7108dbc1..3975ae0bc 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1546,7 +1546,6 @@ class YoutubeDL(object):
             line(f, idlen) for f in formats
             if f.get('preference') is None or f['preference'] >= -1000]
         if len(formats) > 1:
-            formats_s[0] += (' ' if self._format_note(formats[0]) else '') + '(worst)'
             formats_s[-1] += (' ' if self._format_note(formats[-1]) else '') + '(best)'
 
         header_line = line({

From 2caf182f37562a8dc3857f88a19f6902f6d829e7 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Mon, 9 Feb 2015 16:55:31 +0200
Subject: [PATCH 0293/6219] [trilulilu] Add support for videos without category
 in the URL (Closes #4067)

Also, update the testcase, detect private/country restricted videos and modernize a bit.
---
 youtube_dl/extractor/trilulilu.py | 51 ++++++++++++++++++++-----------
 1 file changed, 33 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/trilulilu.py b/youtube_dl/extractor/trilulilu.py
index 220a05b7b..185accc4b 100644
--- a/youtube_dl/extractor/trilulilu.py
+++ b/youtube_dl/extractor/trilulilu.py
@@ -1,40 +1,55 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
-import json
+import re
 
 from .common import InfoExtractor
+from ..utils import ExtractorError
 
 
 class TriluliluIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?trilulilu\.ro/video-[^/]+/(?P<id>[^/]+)'
+    _VALID_URL = r'https?://(?:www\.)?trilulilu\.ro/(?:video-[^/]+/)?(?P<id>[^/#\?]+)'
     _TEST = {
         'url': 'http://www.trilulilu.ro/video-animatie/big-buck-bunny-1',
+        'md5': 'c1450a00da251e2769b74b9005601cac',
         'info_dict': {
-            'id': 'big-buck-bunny-1',
+            'id': 'ae2899e124140b',
             'ext': 'mp4',
             'title': 'Big Buck Bunny',
             'description': ':) pentru copilul din noi',
         },
-        # Server ignores Range headers (--test)
-        'params': {
-            'skip_download': True
-        }
     }
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
 
+        if re.search(r'Fişierul nu este disponibil pentru vizionare în ţara dumneavoastră', webpage):
+            raise ExtractorError(
+                'This video is not available in your country.', expected=True)
+        elif re.search('Fişierul poate fi accesat doar de către prietenii lui', webpage):
+            raise ExtractorError('This video is private.', expected=True)
+
+        flashvars_str = self._search_regex(
+            r'block_flash_vars\s*=\s*(\{[^\}]+\})', webpage, 'flashvars', fatal=False, default=None)
+
+        if flashvars_str:
+            flashvars = self._parse_json(flashvars_str, display_id)
+        else:
+            raise ExtractorError(
+                'This page does not contain videos', expected=True)
+
+        if flashvars['isMP3'] == 'true':
+            raise ExtractorError(
+                'Audio downloads are currently not supported', expected=True)
+
+        video_id = flashvars['hash']
         title = self._og_search_title(webpage)
         thumbnail = self._og_search_thumbnail(webpage)
-        description = self._og_search_description(webpage)
-
-        log_str = self._search_regex(
-            r'block_flash_vars[ ]=[ ]({[^}]+})', webpage, 'log info')
-        log = json.loads(log_str)
+        description = self._og_search_description(webpage, default=None)
 
         format_url = ('http://fs%(server)s.trilulilu.ro/%(hash)s/'
-                      'video-formats2' % log)
+                      'video-formats2' % flashvars)
         format_doc = self._download_xml(
             format_url, video_id,
             note='Downloading formats',
@@ -44,10 +59,10 @@ class TriluliluIE(InfoExtractor):
             'http://fs%(server)s.trilulilu.ro/stream.php?type=video'
             '&source=site&hash=%(hash)s&username=%(userid)s&'
             'key=ministhebest&format=%%s&sig=&exp=' %
-            log)
+            flashvars)
         formats = [
             {
-                'format': fnode.text,
+                'format_id': fnode.text.partition('-')[2],
                 'url': video_url_template % fnode.text,
                 'ext': fnode.text.partition('-')[0]
             }
@@ -56,8 +71,8 @@ class TriluliluIE(InfoExtractor):
         ]
 
         return {
-            '_type': 'video',
             'id': video_id,
+            'display_id': display_id,
             'formats': formats,
             'title': title,
             'description': description,

From f18e3a2fc0b73011c98c39d54576fb835cee4561 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 9 Feb 2015 15:59:19 +0100
Subject: [PATCH 0294/6219] release 2015.02.09.3

---
 docs/supportedsites.md | 1 +
 youtube_dl/version.py  | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 84a132684..93572f6a8 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -392,6 +392,7 @@
  - **StreamCZ**
  - **StreetVoice**
  - **SunPorno**
+ - **SVTPlay**
  - **SWRMediathek**
  - **Syfy**
  - **SztvHu**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index d8911cae6..30447a368 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.09.2'
+__version__ = '2015.02.09.3'

From 28f1272870c43700a90af4671b6eae46d0fab6c6 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 9 Feb 2015 16:05:01 +0100
Subject: [PATCH 0295/6219] [svtplay] Correct test case

---
 youtube_dl/extractor/svtplay.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/svtplay.py b/youtube_dl/extractor/svtplay.py
index 727b3d1fc..eadb9ccb4 100644
--- a/youtube_dl/extractor/svtplay.py
+++ b/youtube_dl/extractor/svtplay.py
@@ -1,3 +1,4 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
@@ -10,13 +11,13 @@ class SVTPlayIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?svtplay\.se/video/(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://www.svtplay.se/video/2609989/sm-veckan/sm-veckan-rally-final-sasong-1-sm-veckan-rally-final',
-        'md5': '2521cd644e862936cf2e698206e47385',
+        'md5': 'f4a184968bc9c802a9b41316657aaa80',
         'info_dict': {
-            'id': '3966754',
+            'id': '2609989',
             'ext': 'mp4',
-            'title': 'FIFA 14 - E3 2013 Trailer',
+            'title': 'SM veckan vinter, Örebro - Rally, final',
             'duration': 4500,
-            'thumbnail': 're:^https?://.*\.jpg$',
+            'thumbnail': 're:^https?://.*[\.-]jpg$',
         },
     }
 

From 99c2398bc641c478dbea85c013081d883479827e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 9 Feb 2015 19:08:51 +0100
Subject: [PATCH 0296/6219] [bandcamp] Use our API to get more stable error
 messages (#1194)

---
 youtube_dl/extractor/bandcamp.py | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/bandcamp.py b/youtube_dl/extractor/bandcamp.py
index aea0263d6..27d8652f6 100644
--- a/youtube_dl/extractor/bandcamp.py
+++ b/youtube_dl/extractor/bandcamp.py
@@ -72,26 +72,29 @@ class BandcampIE(InfoExtractor):
 
         download_link = m_download.group(1)
         video_id = self._search_regex(
-            r'var TralbumData = {.*?id: (?P<id>\d+),?$',
-            webpage, 'video id', flags=re.MULTILINE | re.DOTALL)
+            r'(?ms)var TralbumData = {.*?id: (?P<id>\d+),?$',
+            webpage, 'video id')
 
         download_webpage = self._download_webpage(download_link, video_id, 'Downloading free downloads page')
         # We get the dictionary of the track from some javascript code
-        info = re.search(r'items: (.*?),$', download_webpage, re.MULTILINE).group(1)
-        info = json.loads(info)[0]
+        all_info = self._parse_json(self._search_regex(
+            r'(?sm)items: (.*?),$', download_webpage, 'items'), video_id)
+        info = All_info[0]
         # We pick mp3-320 for now, until format selection can be easily implemented.
         mp3_info = info['downloads']['mp3-320']
         # If we try to use this url it says the link has expired
         initial_url = mp3_info['url']
-        re_url = r'(?P<server>http://(.*?)\.bandcamp\.com)/download/track\?enc=mp3-320&fsig=(?P<fsig>.*?)&id=(?P<id>.*?)&ts=(?P<ts>.*)$'
-        m_url = re.match(re_url, initial_url)
+        m_url = re.match(
+            r'(?P<server>http://(.*?)\.bandcamp\.com)/download/track\?enc=mp3-320&fsig=(?P<fsig>.*?)&id=(?P<id>.*?)&ts=(?P<ts>.*)$',
+            initial_url)
         # We build the url we will use to get the final track url
         # This url is build in Bandcamp in the script download_bunde_*.js
         request_url = '%s/statdownload/track?enc=mp3-320&fsig=%s&id=%s&ts=%s&.rand=665028774616&.vrs=1' % (m_url.group('server'), m_url.group('fsig'), video_id, m_url.group('ts'))
         final_url_webpage = self._download_webpage(request_url, video_id, 'Requesting download url')
         # If we could correctly generate the .rand field the url would be
         # in the "download_url" key
-        final_url = re.search(r'"retry_url":"(.*?)"', final_url_webpage).group(1)
+        final_url = self._search_regex(
+            r'"retry_url":"(.*?)"', final_url_webpage, 'final video URL')
 
         return {
             'id': video_id,

From 3a5bcd03263059bd7b905d4586c5f23f7f120974 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 10 Feb 2015 01:13:57 +0100
Subject: [PATCH 0297/6219] [extractor/common] Wrap extractor errors (Fixes
 #1194)

For now, we just wrap some common errors. More may follow. We do not want to catch actual programming errors in the extractors, such as 1 // 0.
---
 youtube_dl/extractor/common.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 2f5ba7aee..eee936a6f 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -264,8 +264,15 @@ class InfoExtractor(object):
 
     def extract(self, url):
         """Extracts URL information and returns it in list of dicts."""
-        self.initialize()
-        return self._real_extract(url)
+        try:
+            self.initialize()
+            return self._real_extract(url)
+        except ExtractorError:
+            raise
+        except compat_http_client.IncompleteRead as e:
+            raise ExtractorError('A network error has occured.', cause=e, expected=True)
+        except (KeyError,) as e:
+            raise ExtractorError('An extractor error has occured.', cause=e)
 
     def set_downloader(self, downloader):
         """Sets the downloader for this IE."""

From 34814eb66e948b5d73420e435f3aea7112b243d5 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 10 Feb 2015 01:19:52 +0100
Subject: [PATCH 0298/6219] release 2015.02.10

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 30447a368..2414627da 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.09.3'
+__version__ = '2015.02.10'

From 753fad4adc32b57a0d18518fddd06d2411d09635 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 10 Feb 2015 01:34:01 +0100
Subject: [PATCH 0299/6219] [commonmistakes] Correct logic error

---
 youtube_dl/extractor/commonmistakes.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/commonmistakes.py b/youtube_dl/extractor/commonmistakes.py
index 75c06903f..dbbf27a74 100644
--- a/youtube_dl/extractor/commonmistakes.py
+++ b/youtube_dl/extractor/commonmistakes.py
@@ -24,6 +24,6 @@ class CommonMistakesIE(InfoExtractor):
             'That doesn\'t make any sense. '
             'Simply remove the parameter in your command or configuration.'
         ) % url
-        if self._downloader.params.get('verbose'):
+        if not self._downloader.params.get('verbose'):
             msg += ' Add -v to the command line to see what arguments and configuration youtube-dl got.'
         raise ExtractorError(msg, expected=True)

From c06a9fa34f41edf090f4beb385212e44a6e1bb08 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 10 Feb 2015 01:36:38 +0100
Subject: [PATCH 0300/6219] Use snake_case instead of camelCase

---
 youtube_dl/options.py | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 048d9992c..9d9195b60 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -734,22 +734,22 @@ def parseOpts(overrideArguments=None):
         if opts.verbose:
             write_string('[debug] Override config: ' + repr(overrideArguments) + '\n')
     else:
-        commandLineConf = sys.argv[1:]
-        if '--ignore-config' in commandLineConf:
-            systemConf = []
-            userConf = []
+        command_line_conf = sys.argv[1:]
+        if '--ignore-config' in command_line_conf:
+            system_conf = []
+            user_conf = []
         else:
-            systemConf = _readOptions('/etc/youtube-dl.conf')
-            if '--ignore-config' in systemConf:
-                userConf = []
+            system_conf = _readOptions('/etc/youtube-dl.conf')
+            if '--ignore-config' in system_conf:
+                user_conf = []
             else:
-                userConf = _readUserConf()
-        argv = systemConf + userConf + commandLineConf
+                user_conf = _readUserConf()
+        argv = system_conf + user_conf + command_line_conf
 
         opts, args = parser.parse_args(argv)
         if opts.verbose:
-            write_string('[debug] System config: ' + repr(_hide_login_info(systemConf)) + '\n')
-            write_string('[debug] User config: ' + repr(_hide_login_info(userConf)) + '\n')
-            write_string('[debug] Command-line args: ' + repr(_hide_login_info(commandLineConf)) + '\n')
+            write_string('[debug] System config: ' + repr(_hide_login_info(system_conf)) + '\n')
+            write_string('[debug] User config: ' + repr(_hide_login_info(user_conf)) + '\n')
+            write_string('[debug] Command-line args: ' + repr(_hide_login_info(command_line_conf)) + '\n')
 
     return parser, opts, args

From 834bf069d20745263586284c98196b5b0605b916 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 10 Feb 2015 01:37:14 +0100
Subject: [PATCH 0301/6219] [bandcamp] Correct variable name

---
 youtube_dl/extractor/bandcamp.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bandcamp.py b/youtube_dl/extractor/bandcamp.py
index 27d8652f6..490cc961a 100644
--- a/youtube_dl/extractor/bandcamp.py
+++ b/youtube_dl/extractor/bandcamp.py
@@ -79,7 +79,7 @@ class BandcampIE(InfoExtractor):
         # We get the dictionary of the track from some javascript code
         all_info = self._parse_json(self._search_regex(
             r'(?sm)items: (.*?),$', download_webpage, 'items'), video_id)
-        info = All_info[0]
+        info = all_info[0]
         # We pick mp3-320 for now, until format selection can be easily implemented.
         mp3_info = info['downloads']['mp3-320']
         # If we try to use this url it says the link has expired

From c73fae1e2e4421df664aefd1d14a72596caf9e2f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 10 Feb 2015 01:39:43 +0100
Subject: [PATCH 0302/6219] [commonmistakes] Detect BOMs at the beginning of
 URLs

Reported at https://bugzilla.redhat.com/show_bug.cgi?id=1093517 .
---
 youtube_dl/extractor/__init__.py       |  2 +-
 youtube_dl/extractor/commonmistakes.py | 17 +++++++++++++++++
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 55ca0d6e4..fb1e7f325 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -74,7 +74,7 @@ from .collegehumor import CollegeHumorIE
 from .collegerama import CollegeRamaIE
 from .comedycentral import ComedyCentralIE, ComedyCentralShowsIE
 from .comcarcoff import ComCarCoffIE
-from .commonmistakes import CommonMistakesIE
+from .commonmistakes import CommonMistakesIE, UnicodeBOMIE
 from .condenast import CondeNastIE
 from .cracked import CrackedIE
 from .criterion import CriterionIE
diff --git a/youtube_dl/extractor/commonmistakes.py b/youtube_dl/extractor/commonmistakes.py
index dbbf27a74..2f86e2381 100644
--- a/youtube_dl/extractor/commonmistakes.py
+++ b/youtube_dl/extractor/commonmistakes.py
@@ -27,3 +27,20 @@ class CommonMistakesIE(InfoExtractor):
         if not self._downloader.params.get('verbose'):
             msg += ' Add -v to the command line to see what arguments and configuration youtube-dl got.'
         raise ExtractorError(msg, expected=True)
+
+
+class UnicodeBOMIE(InfoExtractor):
+        IE_DESC = False
+        _VALID_URL = r'(?P<bom>\ufeff)(?P<id>.*)$'
+
+        _TESTS = [{
+            'url': '\ufeffhttp://www.youtube.com/watch?v=BaW_jenozKc',
+            'only_matching': True,
+        }]
+
+        def _real_extract(self, url):
+            real_url = self._match_id(url)
+            self.report_warning(
+                'Your URL starts with a Byte Order Mark (BOM). '
+                'Removing the BOM and looking for "%s" ...' % real_url)
+            return self.url_result(real_url)

From 88296505131f4b91ff91eaa0af34318664d892c9 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 10 Feb 2015 01:46:09 +0100
Subject: [PATCH 0303/6219] release 2015.02.10.1

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 2414627da..7d144dadc 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.10'
+__version__ = '2015.02.10.1'

From 347de4931cb3e496fb7b1dfb0314c213f44cce6b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 10 Feb 2015 03:32:21 +0100
Subject: [PATCH 0304/6219] [YoutubeDL] Add generic video filtering (Fixes
 #4916)

This functionality is intended to eventually encompass the current format filtering.
---
 test/test_utils.py      | 32 +++++++++++++++++
 youtube_dl/YoutubeDL.py | 14 +++++++-
 youtube_dl/__init__.py  |  7 +++-
 youtube_dl/options.py   | 19 ++++++++++
 youtube_dl/utils.py     | 77 +++++++++++++++++++++++++++++++++++++++++
 5 files changed, 147 insertions(+), 2 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 80c765bc4..1c29d0889 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -53,6 +53,7 @@ from youtube_dl.utils import (
     version_tuple,
     xpath_with_ns,
     render_table,
+    match_str,
 )
 
 
@@ -459,6 +460,37 @@ ffmpeg version 2.4.4 Copyright (c) 2000-2014 the FFmpeg ...'''), '2.4.4')
             '123  4\n'
             '9999 51')
 
+    def test_match_str(self):
+        self.assertRaises(ValueError, match_str, 'xy>foobar', {})
+        self.assertFalse(match_str('xy', {'x': 1200}))
+        self.assertTrue(match_str('!xy', {'x': 1200}))
+        self.assertTrue(match_str('x', {'x': 1200}))
+        self.assertFalse(match_str('!x', {'x': 1200}))
+        self.assertTrue(match_str('x', {'x': 0}))
+        self.assertFalse(match_str('x>0', {'x': 0}))
+        self.assertFalse(match_str('x>0', {}))
+        self.assertTrue(match_str('x>?0', {}))
+        self.assertTrue(match_str('x>1K', {'x': 1200}))
+        self.assertFalse(match_str('x>2K', {'x': 1200}))
+        self.assertTrue(match_str('x>=1200 & x < 1300', {'x': 1200}))
+        self.assertFalse(match_str('x>=1100 & x < 1200', {'x': 1200}))
+        self.assertFalse(match_str('y=a212', {'y': 'foobar42'}))
+        self.assertTrue(match_str('y=foobar42', {'y': 'foobar42'}))
+        self.assertFalse(match_str('y!=foobar42', {'y': 'foobar42'}))
+        self.assertTrue(match_str('y!=foobar2', {'y': 'foobar42'}))
+        self.assertFalse(match_str(
+            'like_count > 100 & dislike_count <? 50 & description',
+            {'like_count': 90, 'description': 'foo'}))
+        self.assertTrue(match_str(
+            'like_count > 100 & dislike_count <? 50 & description',
+            {'like_count': 190, 'description': 'foo'}))
+        self.assertFalse(match_str(
+            'like_count > 100 & dislike_count <? 50 & description',
+            {'like_count': 190, 'dislike_count': 60, 'description': 'foo'}))
+        self.assertFalse(match_str(
+            'like_count > 100 & dislike_count <? 50 & description',
+            {'like_count': 190, 'dislike_count': 10}))
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 3975ae0bc..dda222fee 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -228,6 +228,11 @@ class YoutubeDL(object):
     external_downloader:  Executable of the external downloader to call.
     listformats:       Print an overview of available video formats and exit.
     list_thumbnails:   Print a table of all thumbnails and exit.
+    match_filter:      A function that gets called with the info_dict of
+                       every video.
+                       If it returns a message, the video is ignored.
+                       If it returns None, the video is downloaded.
+                       match_filter_func in utils.py is one example for this.
 
 
     The following parameters are not used by YoutubeDL itself, they are used by
@@ -583,9 +588,16 @@ class YoutubeDL(object):
             if max_views is not None and view_count > max_views:
                 return 'Skipping %s, because it has exceeded the maximum view count (%d/%d)' % (video_title, view_count, max_views)
         if age_restricted(info_dict.get('age_limit'), self.params.get('age_limit')):
-            return 'Skipping "%s" because it is age restricted' % title
+            return 'Skipping "%s" because it is age restricted' % video_title
         if self.in_download_archive(info_dict):
             return '%s has already been recorded in archive' % video_title
+
+        match_filter = self.params.get('match_filter')
+        if match_filter is not None:
+            ret = match_filter(info_dict)
+            if ret is not None:
+                return ret
+
         return None
 
     @staticmethod
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index e90679ff9..eefca0fe4 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -23,9 +23,10 @@ from .compat import (
 )
 from .utils import (
     DateRange,
-    DEFAULT_OUTTMPL,
     decodeOption,
+    DEFAULT_OUTTMPL,
     DownloadError,
+    match_filter_func,
     MaxDownloadsReached,
     preferredencoding,
     read_batch_urls,
@@ -247,6 +248,9 @@ def _real_main(argv=None):
             xattr  # Confuse flake8
         except ImportError:
             parser.error('setting filesize xattr requested but python-xattr is not available')
+    match_filter = (
+        None if opts.match_filter is None
+        else match_filter_func(opts.match_filter))
 
     ydl_opts = {
         'usenetrc': opts.usenetrc,
@@ -344,6 +348,7 @@ def _real_main(argv=None):
         'list_thumbnails': opts.list_thumbnails,
         'playlist_items': opts.playlist_items,
         'xattr_set_filesize': opts.xattr_set_filesize,
+        'match_filter': match_filter,
     }
 
     with YoutubeDL(ydl_opts) as ydl:
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 9d9195b60..f64aa5b85 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -244,6 +244,25 @@ def parseOpts(overrideArguments=None):
         '--max-views',
         metavar='COUNT', dest='max_views', default=None, type=int,
         help='Do not download any videos with more than COUNT views')
+    selection.add_option(
+        '--match-filter',
+        metavar='FILTER', dest='match_filter', default=None,
+        help=(
+            '(Experimental) Generic video filter. '
+            'Specify any key (see help for -o for a list of available keys) to'
+            ' match if the key is present, '
+            '!key to check if the key is not present,'
+            'key > NUMBER (like "comment_count > 12", also works with '
+            '>=, <, <=, !=, =) to compare against a number, and '
+            '& to require multiple matches. '
+            'Values which are not known are excluded unless you'
+            ' put a question mark (?) after the operator.'
+            'For example, to only match videos that have been liked more than '
+            '100 times and disliked less than 50 times (or the dislike '
+            'functionality is not available at the given service), but who '
+            'also have a description, use  --match-filter '
+            '"like_count > 100 & dislike_count <? 50 & description" .'
+        ))
     selection.add_option(
         '--no-playlist',
         action='store_true', dest='noplaylist', default=False,
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 8f5463f1c..03566d223 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -17,6 +17,7 @@ import io
 import json
 import locale
 import math
+import operator
 import os
 import pipes
 import platform
@@ -1678,3 +1679,79 @@ def render_table(header_row, data):
     max_lens = [max(len(compat_str(v)) for v in col) for col in zip(*table)]
     format_str = ' '.join('%-' + compat_str(ml + 1) + 's' for ml in max_lens[:-1]) + '%s'
     return '\n'.join(format_str % tuple(row) for row in table)
+
+
+def _match_one(filter_part, dct):
+    COMPARISON_OPERATORS = {
+        '<': operator.lt,
+        '<=': operator.le,
+        '>': operator.gt,
+        '>=': operator.ge,
+        '=': operator.eq,
+        '!=': operator.ne,
+    }
+    operator_rex = re.compile(r'''(?x)\s*
+        (?P<key>[a-z_]+)
+        \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
+        (?:
+            (?P<intval>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)|
+            (?P<strval>(?![0-9.])[a-z0-9A-Z]*)
+        )
+        \s*$
+        ''' % '|'.join(map(re.escape, COMPARISON_OPERATORS.keys())))
+    m = operator_rex.search(filter_part)
+    if m:
+        op = COMPARISON_OPERATORS[m.group('op')]
+        if m.group('strval') is not None:
+            if m.group('op') not in ('=', '!='):
+                raise ValueError(
+                    'Operator %s does not support string values!' % m.group('op'))
+            comparison_value = m.group('strval')
+        else:
+            try:
+                comparison_value = int(m.group('intval'))
+            except ValueError:
+                comparison_value = parse_filesize(m.group('intval'))
+                if comparison_value is None:
+                    comparison_value = parse_filesize(m.group('intval') + 'B')
+                if comparison_value is None:
+                    raise ValueError(
+                        'Invalid integer value %r in filter part %r' % (
+                            m.group('intval'), filter_part))
+        actual_value = dct.get(m.group('key'))
+        if actual_value is None:
+            return m.group('none_inclusive')
+        return op(actual_value, comparison_value)
+
+    UNARY_OPERATORS = {
+        '': lambda v: v is not None,
+        '!': lambda v: v is None,
+    }
+    operator_rex = re.compile(r'''(?x)\s*
+        (?P<op>%s)\s*(?P<key>[a-z_]+)
+        \s*$
+        ''' % '|'.join(map(re.escape, UNARY_OPERATORS.keys())))
+    m = operator_rex.search(filter_part)
+    if m:
+        op = UNARY_OPERATORS[m.group('op')]
+        actual_value = dct.get(m.group('key'))
+        return op(actual_value)
+
+    raise ValueError('Invalid filter part %r' % filter_part)
+
+
+def match_str(filter_str, dct):
+    """ Filter a dictionary with a simple string syntax. Returns True (=passes filter) or false """
+
+    return all(
+        _match_one(filter_part, dct) for filter_part in filter_str.split('&'))
+
+
+def match_filter_func(filter_str):
+    def _match_func(info_dict):
+        if match_str(filter_str, info_dict):
+            return None
+        else:
+            video_title = info_dict.get('title', info_dict.get('id', 'video'))
+            return '%s does not pass filter %s, skipping ..' % (video_title, filter_str)
+    return _match_func

From 845734773d334b1a338129dad7c656c12c8fbf6f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 10 Feb 2015 03:32:55 +0100
Subject: [PATCH 0305/6219] release 2015.02.10.2

---
 README.md             | 17 +++++++++++++++++
 youtube_dl/version.py |  2 +-
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 23295df4b..7aad3007f 100644
--- a/README.md
+++ b/README.md
@@ -119,6 +119,23 @@ which means you can modify it, redistribute it or use it however you like.
                                      COUNT views
     --max-views COUNT                Do not download any videos with more than
                                      COUNT views
+    --match-filter FILTER            (Experimental) Generic video filter.
+                                     Specify any key (see help for -o for a list
+                                     of available keys) to match if the key is
+                                     present, !key to check if the key is not
+                                     present,key > NUMBER (like "comment_count >
+                                     12", also works with >=, <, <=, !=, =) to
+                                     compare against a number, and & to require
+                                     multiple matches. Values which are not
+                                     known are excluded unless you put a
+                                     question mark (?) after the operator.For
+                                     example, to only match videos that have
+                                     been liked more than 100 times and disliked
+                                     less than 50 times (or the dislike
+                                     functionality is not available at the given
+                                     service), but who also have a description,
+                                     use  --match-filter "like_count > 100 &
+                                     dislike_count <? 50 & description" .
     --no-playlist                    If the URL refers to a video and a
                                      playlist, download only the video.
     --age-limit YEARS                download only videos suitable for the given
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 7d144dadc..9ab5b49b8 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.10.1'
+__version__ = '2015.02.10.2'

From f7a211dcc8d86fe5a4d25f4d209ffb3efc05bdfc Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 10 Feb 2015 03:41:31 +0100
Subject: [PATCH 0306/6219] [pornhd] Fix extraction (fixes #4915)

---
 youtube_dl/extractor/pornhd.py | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/pornhd.py b/youtube_dl/extractor/pornhd.py
index 954dfccb7..dbb2c3bd9 100644
--- a/youtube_dl/extractor/pornhd.py
+++ b/youtube_dl/extractor/pornhd.py
@@ -46,16 +46,17 @@ class PornHdIE(InfoExtractor):
 
         quality = qualities(['sd', 'hd'])
         sources = json.loads(js_to_json(self._search_regex(
-            r"(?s)'sources'\s*:\s*(\{.+?\})\s*\}\);", webpage, 'sources')))
+            r"(?s)'sources'\s*:\s*(\{.+?\})\s*\}[;,)]",
+            webpage, 'sources')))
         formats = []
-        for container, s in sources.items():
-            for qname, video_url in s.items():
-                formats.append({
-                    'url': video_url,
-                    'container': container,
-                    'format_id': '%s-%s' % (container, qname),
-                    'quality': quality(qname),
-                })
+        for qname, video_url in sources.items():
+            if not video_url:
+                continue
+            formats.append({
+                'url': video_url,
+                'format_id': qname,
+                'quality': quality(qname),
+            })
         self._sort_formats(formats)
 
         return {

From 7e5db8c930e5f154287f98c6c0da90fffd2ede4f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 10 Feb 2015 04:22:10 +0100
Subject: [PATCH 0307/6219] [options] Add --no-color

---
 youtube_dl/YoutubeDL.py        | 5 +++--
 youtube_dl/__init__.py         | 1 +
 youtube_dl/extractor/common.py | 2 +-
 youtube_dl/options.py          | 5 +++++
 4 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index dda222fee..e2ec55c90 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -233,6 +233,7 @@ class YoutubeDL(object):
                        If it returns a message, the video is ignored.
                        If it returns None, the video is downloaded.
                        match_filter_func in utils.py is one example for this.
+    no_color:          Do not emit color codes in output.
 
 
     The following parameters are not used by YoutubeDL itself, they are used by
@@ -490,7 +491,7 @@ class YoutubeDL(object):
         else:
             if self.params.get('no_warnings'):
                 return
-            if self._err_file.isatty() and os.name != 'nt':
+            if not self.params.get('no_color') and self._err_file.isatty() and os.name != 'nt':
                 _msg_header = '\033[0;33mWARNING:\033[0m'
             else:
                 _msg_header = 'WARNING:'
@@ -502,7 +503,7 @@ class YoutubeDL(object):
         Do the same as trouble, but prefixes the message with 'ERROR:', colored
         in red if stderr is a tty file.
         '''
-        if self._err_file.isatty() and os.name != 'nt':
+        if not self.params.get('no_color') and self._err_file.isatty() and os.name != 'nt':
             _msg_header = '\033[0;31mERROR:\033[0m'
         else:
             _msg_header = 'ERROR:'
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index eefca0fe4..ed22f169f 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -349,6 +349,7 @@ def _real_main(argv=None):
         'playlist_items': opts.playlist_items,
         'xattr_set_filesize': opts.xattr_set_filesize,
         'match_filter': match_filter,
+        'no_color': opts.no_color,
     }
 
     with YoutubeDL(ydl_opts) as ydl:
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index eee936a6f..382e846c5 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -514,7 +514,7 @@ class InfoExtractor(object):
                 if mobj:
                     break
 
-        if os.name != 'nt' and sys.stderr.isatty():
+        if not self._downloader.params.get('no_color') and os.name != 'nt' and sys.stderr.isatty():
             _name = '\033[0;34m%s\033[0m' % name
         else:
             _name = name
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index f64aa5b85..0219e6ba4 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -165,6 +165,11 @@ def parseOpts(overrideArguments=None):
         action='store_const', dest='extract_flat', const='in_playlist',
         default=False,
         help='Do not extract the videos of a playlist, only list them.')
+    general.add_option(
+        '--no-color', '--no-colors',
+        action='store_true', dest='no_color',
+        default=False,
+        help='Do not emit color codes in output.')
 
     network = optparse.OptionGroup(parser, 'Network Options')
     network.add_option(

From 2f543a2142e05bac988ccd6566e0c9fc5cadca90 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 10 Feb 2015 04:52:33 +0100
Subject: [PATCH 0308/6219] [options] Add alias --dump-header for
 --print-traffic

---
 youtube_dl/options.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 0219e6ba4..873432bee 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -557,7 +557,7 @@ def parseOpts(overrideArguments=None):
         action='store_true', dest='youtube_print_sig_code', default=False,
         help=optparse.SUPPRESS_HELP)
     verbosity.add_option(
-        '--print-traffic',
+        '--print-traffic', '--dump-headers',
         dest='debug_printtraffic', action='store_true', default=False,
         help='Display sent and read HTTP traffic')
     verbosity.add_option(

From bf8f082a90f4396a0a70464b55bf5c11fdb36c47 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 10 Feb 2015 04:53:21 +0100
Subject: [PATCH 0309/6219] [vimeo:album] Add support for album passwords
 (Fixes #4917)

---
 youtube_dl/extractor/vimeo.py | 59 ++++++++++++++++++++++++++++++-----
 1 file changed, 52 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 1bb473514..303e81447 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -188,9 +188,9 @@ class VimeoIE(VimeoBaseInfoExtractor, SubtitlesInfoExtractor):
         password_request = compat_urllib_request.Request(pass_url + '/password', data)
         password_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
         password_request.add_header('Cookie', 'xsrft=%s' % token)
-        self._download_webpage(password_request, video_id,
-                               'Verifying the password',
-                               'Wrong password')
+        return self._download_webpage(
+            password_request, video_id,
+            'Verifying the password', 'Wrong password')
 
     def _verify_player_video_password(self, url, video_id):
         password = self._downloader.params.get('videopassword', None)
@@ -266,7 +266,7 @@ class VimeoIE(VimeoBaseInfoExtractor, SubtitlesInfoExtractor):
             if re.search('The creator of this video has not given you permission to embed it on this domain.', webpage):
                 raise ExtractorError('The author has restricted the access to this video, try with the "--referer" option')
 
-            if re.search('<form[^>]+?id="pw_form"', webpage) is not None:
+            if re.search(r'<form[^>]+?id="pw_form"', webpage) is not None:
                 self._verify_video_password(url, video_id, webpage)
                 return self._real_extract(url)
             else:
@@ -412,12 +412,47 @@ class VimeoChannelIE(InfoExtractor):
     def _extract_list_title(self, webpage):
         return self._html_search_regex(self._TITLE_RE, webpage, 'list title')
 
+    def _login_list_password(self, page_url, list_id, webpage):
+        login_form = self._search_regex(
+            r'(?s)<form[^>]+?id="pw_form"(.*?)</form>',
+            webpage, 'login form', default=None)
+        if not login_form:
+            return webpage
+
+        password = self._downloader.params.get('videopassword', None)
+        if password is None:
+            raise ExtractorError('This album is protected by a password, use the --video-password option', expected=True)
+        fields = dict(re.findall(r'''(?x)<input\s+
+            type="hidden"\s+
+            name="([^"]+)"\s+
+            value="([^"]*)"
+            ''', login_form))
+        token = self._search_regex(r'xsrft: \'(.*?)\'', webpage, 'login token')
+        fields['token'] = token
+        fields['password'] = password
+        post = compat_urllib_parse.urlencode(fields)
+        password_path = self._search_regex(
+            r'action="([^"]+)"', login_form, 'password URL')
+        password_url = compat_urlparse.urljoin(page_url, password_path)
+        password_request = compat_urllib_request.Request(password_url, post)
+        password_request.add_header('Content-type', 'application/x-www-form-urlencoded')
+        self._set_cookie('vimeo.com', 'xsrft', token)
+
+        return self._download_webpage(
+            password_request, list_id,
+            'Verifying the password', 'Wrong password')
+
     def _extract_videos(self, list_id, base_url):
         video_ids = []
         for pagenum in itertools.count(1):
+            page_url = self._page_url(base_url, pagenum)
             webpage = self._download_webpage(
-                self._page_url(base_url, pagenum), list_id,
+                page_url, list_id,
                 'Downloading page %s' % pagenum)
+
+            if pagenum == 1:
+                webpage = self._login_list_password(page_url, list_id, webpage)
+
             video_ids.extend(re.findall(r'id="clip_(\d+?)"', webpage))
             if re.search(self._MORE_PAGES_INDICATOR, webpage, re.DOTALL) is None:
                 break
@@ -464,14 +499,24 @@ class VimeoAlbumIE(VimeoChannelIE):
             'title': 'Staff Favorites: November 2013',
         },
         'playlist_mincount': 13,
+    }, {
+        'note': 'Password-protected album',
+        'url': 'https://vimeo.com/album/3253534',
+        'info_dict': {
+            'title': 'test',
+            'id': '3253534',
+        },
+        'playlist_count': 1,
+        'params': {
+            'videopassword': 'youtube-dl',
+        }
     }]
 
     def _page_url(self, base_url, pagenum):
         return '%s/page:%d/' % (base_url, pagenum)
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        album_id = mobj.group('id')
+        album_id = self._match_id(url)
         return self._extract_videos(album_id, 'http://vimeo.com/album/%s' % album_id)
 
 

From c0e1a415fdcd8d36d98a9fb6ff5bf9146004ddd3 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 10 Feb 2015 05:28:48 +0100
Subject: [PATCH 0310/6219] [firstpost] Modernize

---
 youtube_dl/extractor/firstpost.py | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/firstpost.py b/youtube_dl/extractor/firstpost.py
index 0993af1c9..298227d57 100644
--- a/youtube_dl/extractor/firstpost.py
+++ b/youtube_dl/extractor/firstpost.py
@@ -1,7 +1,5 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 
 
@@ -20,11 +18,10 @@ class FirstpostIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
+        video_id = self._match_id(url)
         page = self._download_webpage(url, video_id)
-        title = self._html_search_meta('twitter:title', page, 'title')
+
+        title = self._html_search_meta('twitter:title', page, 'title', fatal=True)
         description = self._html_search_meta('twitter:description', page, 'title')
 
         data = self._download_xml(
@@ -42,6 +39,7 @@ class FirstpostIE(InfoExtractor):
                 'height': int(details.find('./height').text.strip()),
             } for details in item.findall('./source/file_details') if details.find('./file').text
         ]
+        self._sort_formats(formats)
 
         return {
             'id': video_id,

From 6a78740211b37cab5d20e707db9f73dca6c60049 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 10 Feb 2015 05:28:59 +0100
Subject: [PATCH 0311/6219] [test/test_youtube_signature] Use fake YDL

---
 test/test_youtube_signature.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py
index 13d228cd8..09696e19a 100644
--- a/test/test_youtube_signature.py
+++ b/test/test_youtube_signature.py
@@ -8,11 +8,11 @@ import sys
 import unittest
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-
 import io
 import re
 import string
 
+from test.helper import FakeYDL
 from youtube_dl.extractor import YoutubeIE
 from youtube_dl.compat import compat_str, compat_urlretrieve
 
@@ -88,7 +88,8 @@ def make_tfunc(url, stype, sig_input, expected_sig):
         if not os.path.exists(fn):
             compat_urlretrieve(url, fn)
 
-        ie = YoutubeIE()
+        ydl = FakeYDL()
+        ie = YoutubeIE(ydl)
         if stype == 'js':
             with io.open(fn, encoding='utf-8') as testf:
                 jscode = testf.read()

From 8f84f5718319293485eef7174702340f109c7227 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 10 Feb 2015 05:42:41 +0100
Subject: [PATCH 0312/6219] [ccc] Add new extractor (Fixes #4890)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/ccc.py      | 99 ++++++++++++++++++++++++++++++++
 2 files changed, 100 insertions(+)
 create mode 100644 youtube_dl/extractor/ccc.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index fb1e7f325..da3b2b2d0 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -54,6 +54,7 @@ from .canalplus import CanalplusIE
 from .canalc2 import Canalc2IE
 from .cbs import CBSIE
 from .cbsnews import CBSNewsIE
+from .ccc import CCCIE
 from .ceskatelevize import CeskaTelevizeIE
 from .channel9 import Channel9IE
 from .chilloutzone import ChilloutzoneIE
diff --git a/youtube_dl/extractor/ccc.py b/youtube_dl/extractor/ccc.py
new file mode 100644
index 000000000..2a5d4be18
--- /dev/null
+++ b/youtube_dl/extractor/ccc.py
@@ -0,0 +1,99 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    qualities,
+    unified_strdate,
+)
+
+
+class CCCIE(InfoExtractor):
+    IE_NAME = 'media.ccc.de'
+    _VALID_URL = r'https?://(?:www\.)?media\.ccc\.de/[^?#]+/[^?#/]*?_(?P<id>[0-9]{8,})._[^?#/]*\.html'
+
+    _TEST = {
+        'url': 'http://media.ccc.de/browse/congress/2013/30C3_-_5443_-_en_-_saal_g_-_201312281830_-_introduction_to_processor_design_-_byterazor.html#video',
+        'md5': '205a365d0d57c0b1e43a12c9ffe8f9be',
+        'info_dict': {
+            'id': '20131228183',
+            'ext': 'mp4',
+            'title': 'Introduction to Processor Design',
+            'description': 'md5:5ddbf8c734800267f2cee4eab187bc1b',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'view_count': int,
+            'upload_date': '20131229',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        if self._downloader.params.get('prefer_free_formats'):
+            preference = qualities(['mp3', 'opus', 'mp4-lq', 'webm-lq', 'h264-sd', 'mp4-sd', 'webm-sd', 'mp4', 'webm', 'mp4-hd', 'h264-hd', 'webm-hd'])
+        else:
+            preference = qualities(['opus', 'mp3', 'webm-lq', 'mp4-lq', 'webm-sd', 'h264-sd', 'mp4-sd', 'webm', 'mp4', 'webm-hd', 'mp4-hd', 'h264-hd'])
+
+        title = self._html_search_regex(
+            r'(?s)<h1>(.*?)</h1>', webpage, 'title')
+        description = self._html_search_regex(
+            r"(?s)<p class='description'>(.*?)</p>",
+            webpage, 'description', fatal=False)
+        upload_date = unified_strdate(self._html_search_regex(
+            r"(?s)<span class='[^']*fa-calendar-o'></span>(.*?)</li>",
+            webpage, 'upload date', fatal=False))
+        view_count = int_or_none(self._html_search_regex(
+            r"(?s)<span class='[^']*fa-eye'></span>(.*?)</li>",
+            webpage, 'view count', fatal=False))
+
+        matches = re.finditer(r'''(?xs)
+            <(?:span|div)\s+class='label\s+filetype'>(?P<format>.*?)</(?:span|div)>\s*
+            <a\s+href='(?P<http_url>[^']+)'>\s*
+            (?:
+                .*?
+                <a\s+href='(?P<torrent_url>[^']+\.torrent)'
+            )?''', webpage)
+        formats = []
+        for m in matches:
+            format = m.group('format')
+            format_id = self._search_regex(
+                r'.*/([a-z0-9_-]+)/[^/]*$',
+                m.group('http_url'), 'format id', default=None)
+            vcodec = 'h264' if 'h264' in format_id else (
+                'none' if format_id in ('mp3', 'opus') else None
+            )
+            formats.append({
+                'format_id': format_id,
+                'format': format,
+                'url': m.group('http_url'),
+                'vcodec': vcodec,
+                'preference': preference(format_id),
+            })
+
+            if m.group('torrent_url'):
+                formats.append({
+                    'format_id': 'torrent-%s' % (format if format_id is None else format_id),
+                    'format': '%s (torrent)' % format,
+                    'proto': 'torrent',
+                    'format_note': '(unsupported; will just download the .torrent file)',
+                    'vcodec': vcodec,
+                    'preference': -100 + preference(format_id),
+                    'url': m.group('torrent_url'),
+                })
+        self._sort_formats(formats)
+
+        thumbnail = self._html_search_regex(
+            r"<video.*?poster='([^']+)'", webpage, 'thumbnail', fatal=False)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'view_count': view_count,
+            'upload_date': upload_date,
+            'formats': formats,
+        }

From 04bbe41330a4c8619e6fa3228d47c4e7b0396899 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 10 Feb 2015 05:42:47 +0100
Subject: [PATCH 0313/6219] release 2015.02.10.3

---
 README.md              | 1 +
 docs/supportedsites.md | 1 +
 youtube_dl/version.py  | 2 +-
 3 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 7aad3007f..731cea1e1 100644
--- a/README.md
+++ b/README.md
@@ -77,6 +77,7 @@ which means you can modify it, redistribute it or use it however you like.
                                      on Windows)
     --flat-playlist                  Do not extract the videos of a playlist,
                                      only list them.
+    --no-color                       Do not emit color codes in output.
 
 ## Network Options:
     --proxy URL                      Use the specified HTTP/HTTPS proxy. Pass in
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 93572f6a8..8bce8fede 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -225,6 +225,7 @@
  - **mailru**: Видео@Mail.Ru
  - **Malemotion**
  - **MDR**
+ - **media.ccc.de**
  - **metacafe**
  - **Metacritic**
  - **Mgoon**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 9ab5b49b8..13c436219 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.10.2'
+__version__ = '2015.02.10.3'

From 442c37b7a9a912cf3b0eb972335b7d24407f4537 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 10 Feb 2015 11:28:28 +0100
Subject: [PATCH 0314/6219] [YoutubeDL] Do not perform filter matching on
 partial results (Fixes #4921)

---
 youtube_dl/YoutubeDL.py | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index e2ec55c90..13d18e25e 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -560,7 +560,7 @@ class YoutubeDL(object):
             self.report_error('Error in output template: ' + str(err) + ' (encoding: ' + repr(preferredencoding()) + ')')
             return None
 
-    def _match_entry(self, info_dict):
+    def _match_entry(self, info_dict, incomplete):
         """ Returns None iff the file should be downloaded """
 
         video_title = info_dict.get('title', info_dict.get('id', 'video'))
@@ -593,11 +593,12 @@ class YoutubeDL(object):
         if self.in_download_archive(info_dict):
             return '%s has already been recorded in archive' % video_title
 
-        match_filter = self.params.get('match_filter')
-        if match_filter is not None:
-            ret = match_filter(info_dict)
-            if ret is not None:
-                return ret
+        if not incomplete:
+            match_filter = self.params.get('match_filter')
+            if match_filter is not None:
+                ret = match_filter(info_dict)
+                if ret is not None:
+                    return ret
 
         return None
 
@@ -792,7 +793,7 @@ class YoutubeDL(object):
                     'extractor_key': ie_result['extractor_key'],
                 }
 
-                reason = self._match_entry(entry)
+                reason = self._match_entry(entry, incomplete=True)
                 if reason is not None:
                     self.to_screen('[download] ' + reason)
                     continue
@@ -1166,7 +1167,7 @@ class YoutubeDL(object):
         if 'format' not in info_dict:
             info_dict['format'] = info_dict['ext']
 
-        reason = self._match_entry(info_dict)
+        reason = self._match_entry(info_dict, incomplete=False)
         if reason is not None:
             self.to_screen('[download] ' + reason)
             return

From 612ee37365cf0e0e5c6a59fcce2986fa0754605c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 10 Feb 2015 11:28:34 +0100
Subject: [PATCH 0315/6219] release 2015.02.10.4

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 13c436219..895d2da0d 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.10.3'
+__version__ = '2015.02.10.4'

From 596ac6e31f6baaadf8280e2c54ca90a7cf2fbc2a Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 10 Feb 2015 15:45:36 +0100
Subject: [PATCH 0316/6219] [escapist] Modernize

---
 youtube_dl/extractor/escapist.py | 51 ++++++++++++++++----------------
 1 file changed, 25 insertions(+), 26 deletions(-)

diff --git a/youtube_dl/extractor/escapist.py b/youtube_dl/extractor/escapist.py
index e240cb859..98b698277 100644
--- a/youtube_dl/extractor/escapist.py
+++ b/youtube_dl/extractor/escapist.py
@@ -1,18 +1,17 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse,
 )
 from ..utils import (
     ExtractorError,
+    js_to_json,
 )
 
 
 class EscapistIE(InfoExtractor):
-    _VALID_URL = r'^https?://?(www\.)?escapistmagazine\.com/videos/view/(?P<showname>[^/]+)/(?P<id>[0-9]+)-'
+    _VALID_URL = r'https?://?(www\.)?escapistmagazine\.com/videos/view/[^/?#]+/(?P<id>[0-9]+)-[^/?#]*(?:$|[?#])'
     _TEST = {
         'url': 'http://www.escapistmagazine.com/videos/view/the-escapist-presents/6618-Breaking-Down-Baldurs-Gate',
         'md5': 'ab3a706c681efca53f0a35f1415cf0d1',
@@ -20,31 +19,30 @@ class EscapistIE(InfoExtractor):
             'id': '6618',
             'ext': 'mp4',
             'description': "Baldur's Gate: Original, Modded or Enhanced Edition? I'll break down what you can expect from the new Baldur's Gate: Enhanced Edition.",
-            'uploader': 'the-escapist-presents',
+            'uploader_id': 'the-escapist-presents',
+            'uploader': 'The Escapist Presents',
             'title': "Breaking Down Baldur's Gate",
         }
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        showName = mobj.group('showname')
-        video_id = mobj.group('id')
-
-        self.report_extraction(video_id)
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        videoDesc = self._html_search_regex(
-            r'<meta name="description" content="([^"]*)"',
-            webpage, 'description', fatal=False)
+        uploader_id = self._html_search_regex(
+            r"<h1 class='headline'><a href='/videos/view/(.*?)'",
+            webpage, 'uploader ID', fatal=False)
+        uploader = self._html_search_regex(
+            r"<h1 class='headline'>(.*?)</a>",
+            webpage, 'uploader', fatal=False)
+        description = self._html_search_meta('description', webpage)
 
-        playerUrl = self._og_search_video_url(webpage, name='player URL')
+        raw_title = self._html_search_meta('title', webpage, fatal=True)
+        title = raw_title.partition(' : ')[2]
 
-        title = self._html_search_regex(
-            r'<meta name="title" content="([^"]*)"',
-            webpage, 'title').split(' : ')[-1]
-
-        configUrl = self._search_regex('config=(.*)$', playerUrl, 'config URL')
-        configUrl = compat_urllib_parse.unquote(configUrl)
+        player_url = self._og_search_video_url(webpage, name='player URL')
+        config_url = compat_urllib_parse.unquote(self._search_regex(
+            r'config=(.*)$', player_url, 'config URL'))
 
         formats = []
 
@@ -53,7 +51,7 @@ class EscapistIE(InfoExtractor):
                 cfgurl, video_id,
                 'Downloading ' + name + ' configuration',
                 'Unable to download ' + name + ' configuration',
-                transform_source=lambda s: s.replace("'", '"'))
+                transform_source=js_to_json)
 
             playlist = config['playlist']
             formats.append({
@@ -62,9 +60,9 @@ class EscapistIE(InfoExtractor):
                 'quality': quality,
             })
 
-        _add_format('normal', configUrl, quality=0)
-        hq_url = (configUrl +
-                  ('&hq=1' if '?' in configUrl else configUrl + '?hq=1'))
+        _add_format('normal', config_url, quality=0)
+        hq_url = (config_url +
+                  ('&hq=1' if '?' in config_url else config_url + '?hq=1'))
         try:
             _add_format('hq', hq_url, quality=1)
         except ExtractorError:
@@ -75,9 +73,10 @@ class EscapistIE(InfoExtractor):
         return {
             'id': video_id,
             'formats': formats,
-            'uploader': showName,
+            'uploader': uploader,
+            'uploader_id': uploader_id,
             'title': title,
             'thumbnail': self._og_search_thumbnail(webpage),
-            'description': videoDesc,
-            'player_url': playerUrl,
+            'description': description,
+            'player_url': player_url,
         }

From 9650885be954b3cc626205aa67f24c31f21da7b6 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 10 Feb 2015 15:55:51 +0100
Subject: [PATCH 0317/6219] [escapist] Filter video differently (Fixes #4919)

---
 youtube_dl/extractor/common.py   | 2 +-
 youtube_dl/extractor/escapist.py | 5 ++++-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 382e846c5..8d71494b4 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -271,7 +271,7 @@ class InfoExtractor(object):
             raise
         except compat_http_client.IncompleteRead as e:
             raise ExtractorError('A network error has occured.', cause=e, expected=True)
-        except (KeyError,) as e:
+        except (KeyError, StopIteration) as e:
             raise ExtractorError('An extractor error has occured.', cause=e)
 
     def set_downloader(self, downloader):
diff --git a/youtube_dl/extractor/escapist.py b/youtube_dl/extractor/escapist.py
index 98b698277..4303feccd 100644
--- a/youtube_dl/extractor/escapist.py
+++ b/youtube_dl/extractor/escapist.py
@@ -54,8 +54,11 @@ class EscapistIE(InfoExtractor):
                 transform_source=js_to_json)
 
             playlist = config['playlist']
+            video_url = next(
+                p['url'] for p in playlist
+                if p.get('eventCategory') == 'Video')
             formats.append({
-                'url': playlist[1]['url'],
+                'url': video_url,
                 'format_id': name,
                 'quality': quality,
             })

From af0d11f2441728ce12ba8a424de29aa873b12d86 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 10 Feb 2015 15:56:04 +0100
Subject: [PATCH 0318/6219] release 2015.02.10.5

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 895d2da0d..225b30f8c 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.10.4'
+__version__ = '2015.02.10.5'

From 054fe3cc40cc0feaa69e21b31946c568a749eb76 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 10 Feb 2015 21:35:34 +0600
Subject: [PATCH 0319/6219] [ntvru] Adapt to new direct delivery and modernize
 (Closes #4918)

---
 youtube_dl/extractor/ntvru.py | 97 +++++++++++++++--------------------
 1 file changed, 40 insertions(+), 57 deletions(-)

diff --git a/youtube_dl/extractor/ntvru.py b/youtube_dl/extractor/ntvru.py
index 0ab8d5100..2cd924d05 100644
--- a/youtube_dl/extractor/ntvru.py
+++ b/youtube_dl/extractor/ntvru.py
@@ -3,7 +3,9 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..utils import (
-    unescapeHTML
+    clean_html,
+    xpath_text,
+    int_or_none,
 )
 
 
@@ -14,73 +16,63 @@ class NTVRuIE(InfoExtractor):
     _TESTS = [
         {
             'url': 'http://www.ntv.ru/novosti/863142/',
+            'md5': 'ba7ea172a91cb83eb734cad18c10e723',
             'info_dict': {
                 'id': '746000',
-                'ext': 'flv',
+                'ext': 'mp4',
                 'title': 'Командующий Черноморским флотом провел переговоры в штабе ВМС Украины',
                 'description': 'Командующий Черноморским флотом провел переговоры в штабе ВМС Украины',
+                'thumbnail': 're:^http://.*\.jpg',
                 'duration': 136,
             },
-            'params': {
-                # rtmp download
-                'skip_download': True,
-            },
         },
         {
             'url': 'http://www.ntv.ru/video/novosti/750370/',
+            'md5': 'adecff79691b4d71e25220a191477124',
             'info_dict': {
                 'id': '750370',
-                'ext': 'flv',
+                'ext': 'mp4',
                 'title': 'Родные пассажиров пропавшего Boeing не верят в трагический исход',
                 'description': 'Родные пассажиров пропавшего Boeing не верят в трагический исход',
+                'thumbnail': 're:^http://.*\.jpg',
                 'duration': 172,
             },
-            'params': {
-                # rtmp download
-                'skip_download': True,
-            },
         },
         {
             'url': 'http://www.ntv.ru/peredacha/segodnya/m23700/o232416',
+            'md5': '82dbd49b38e3af1d00df16acbeab260c',
             'info_dict': {
                 'id': '747480',
-                'ext': 'flv',
-                'title': '«Сегодня». 21 марта 2014 года. 16:00 ',
-                'description': '«Сегодня». 21 марта 2014 года. 16:00 ',
+                'ext': 'mp4',
+                'title': '«Сегодня». 21 марта 2014 года. 16:00',
+                'description': '«Сегодня». 21 марта 2014 года. 16:00',
+                'thumbnail': 're:^http://.*\.jpg',
                 'duration': 1496,
             },
-            'params': {
-                # rtmp download
-                'skip_download': True,
-            },
         },
         {
             'url': 'http://www.ntv.ru/kino/Koma_film',
+            'md5': 'f825770930937aa7e5aca0dc0d29319a',
             'info_dict': {
-                'id': '758100',
-                'ext': 'flv',
+                'id': '1007609',
+                'ext': 'mp4',
                 'title': 'Остросюжетный фильм «Кома»',
                 'description': 'Остросюжетный фильм «Кома»',
+                'thumbnail': 're:^http://.*\.jpg',
                 'duration': 5592,
             },
-            'params': {
-                # rtmp download
-                'skip_download': True,
-            },
         },
         {
             'url': 'http://www.ntv.ru/serial/Delo_vrachey/m31760/o233916/',
+            'md5': '9320cd0e23f3ea59c330dc744e06ff3b',
             'info_dict': {
                 'id': '751482',
-                'ext': 'flv',
+                'ext': 'mp4',
                 'title': '«Дело врачей»: «Деревце жизни»',
                 'description': '«Дело врачей»: «Деревце жизни»',
+                'thumbnail': 're:^http://.*\.jpg',
                 'duration': 2590,
             },
-            'params': {
-                # rtmp download
-                'skip_download': True,
-            },
         },
     ]
 
@@ -92,45 +84,36 @@ class NTVRuIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        page = self._download_webpage(url, video_id)
 
-        video_id = self._html_search_regex(self._VIDEO_ID_REGEXES, page, 'video id')
+        webpage = self._download_webpage(url, video_id)
 
-        player = self._download_xml('http://www.ntv.ru/vi%s/' % video_id, video_id, 'Downloading video XML')
-        title = unescapeHTML(player.find('./data/title').text)
-        description = unescapeHTML(player.find('./data/description').text)
+        video_id = self._html_search_regex(self._VIDEO_ID_REGEXES, webpage, 'video id')
+
+        player = self._download_xml(
+            'http://www.ntv.ru/vi%s/' % video_id,
+            video_id, 'Downloading video XML')
+        title = clean_html(xpath_text(player, './data/title', 'title', fatal=True))
+        description = clean_html(xpath_text(player, './data/description', 'description'))
 
         video = player.find('./data/video')
-        video_id = video.find('./id').text
-        thumbnail = video.find('./splash').text
-        duration = int(video.find('./totaltime').text)
-        view_count = int(video.find('./views').text)
-        puid22 = video.find('./puid22').text
+        video_id = xpath_text(video, './id', 'video id')
+        thumbnail = xpath_text(video, './splash', 'thumbnail')
+        duration = int_or_none(xpath_text(video, './totaltime', 'duration'))
+        view_count = int_or_none(xpath_text(video, './views', 'view count'))
 
-        apps = {
-            '4': 'video1',
-            '7': 'video2',
-        }
-
-        app = apps.get(puid22, apps['4'])
+        token = self._download_webpage(
+            'http://stat.ntv.ru/services/access/token',
+            video_id, 'Downloading access token')
 
         formats = []
         for format_id in ['', 'hi', 'webm']:
-            file = video.find('./%sfile' % format_id)
-            if file is None:
+            file_ = video.find('./%sfile' % format_id)
+            if file_ is None:
                 continue
             size = video.find('./%ssize' % format_id)
             formats.append({
-                'url': 'rtmp://media.ntv.ru/%s' % app,
-                'app': app,
-                'play_path': file.text,
-                'rtmp_conn': 'B:1',
-                'player_url': 'http://www.ntv.ru/swf/vps1.swf?update=20131128',
-                'page_url': 'http://www.ntv.ru',
-                'flash_version': 'LNX 11,2,202,341',
-                'rtmp_live': True,
-                'ext': 'flv',
-                'filesize': int(size.text),
+                'url': 'http://media2.ntv.ru/vod/%s&tok=%s' % (file_.text, token),
+                'filesize': int_or_none(size.text if size is not None else None),
             })
         self._sort_formats(formats)
 

From 8708d764252a83e650f36fe47f89ace4f2a91a5b Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 11 Feb 2015 16:39:15 +0800
Subject: [PATCH 0320/6219] [camdemy] Add new extractor

Single file download done, while folder extaction in plan
---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/camdemy.py  | 78 ++++++++++++++++++++++++++++++++
 2 files changed, 79 insertions(+)
 create mode 100644 youtube_dl/extractor/camdemy.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index da3b2b2d0..496e4d0b7 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -49,6 +49,7 @@ from .brightcove import BrightcoveIE
 from .buzzfeed import BuzzFeedIE
 from .byutv import BYUtvIE
 from .c56 import C56IE
+from .camdemy import CamdemyIE
 from .canal13cl import Canal13clIE
 from .canalplus import CanalplusIE
 from .canalc2 import Canalc2IE
diff --git a/youtube_dl/extractor/camdemy.py b/youtube_dl/extractor/camdemy.py
new file mode 100644
index 000000000..54959f2cf
--- /dev/null
+++ b/youtube_dl/extractor/camdemy.py
@@ -0,0 +1,78 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..compat import compat_urlparse
+from ..utils import parse_iso8601
+
+
+class CamdemyIE(InfoExtractor):
+    _VALID_URL = r'http://www.camdemy.com/media/(?P<id>\d+).*'
+    _TESTS = [{
+        # single file
+        'url': 'http://www.camdemy.com/media/5181/',
+        'md5': '5a5562b6a98b37873119102e052e311b',
+        'info_dict': {
+            'id': '5181',
+            'ext': 'mp4',
+            'title': 'Ch1-1 Introduction, Signals (02-23-2012)',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'description': '',
+            'creator': 'ss11spring',
+            'upload_date': '20130114',
+            'timestamp': 1358154556,
+        }
+    }, {
+        # With non-empty description
+        'url': 'http://www.camdemy.com/media/13885',
+        'md5': '4576a3bb2581f86c61044822adbd1249',
+        'info_dict': {
+            'id': '13885',
+            'ext': 'mp4',
+            'title': 'EverCam + Camdemy QuickStart',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'description': 'md5:050b62f71ed62928f8a35f1a41e186c9',
+            'creator': 'evercam',
+            'upload_date': '20140620',
+            'timestamp': 1403271569,
+        }
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        page = self._download_webpage(url, video_id)
+
+        oembed_obj = self._download_json(
+            'http://www.camdemy.com/oembed/?format=json&url=' + url, video_id)
+
+        thumb_url = oembed_obj['thumbnail_url']
+        video_folder = compat_urlparse.urljoin(thumb_url, 'video/')
+        fileListXML = self._download_xml(
+            compat_urlparse.urljoin(video_folder, 'fileList.xml'),
+            video_id, 'Filelist XML')
+        fileName = fileListXML.find('./video/item/fileName').text
+
+        creation_time = self._html_search_regex(
+            r"<div class='title'>Posted :</div>.*<div class='value'>([0-9:\- ]+)<",
+            page, 'creation time', flags=re.MULTILINE | re.DOTALL) + '+08:00'
+        creation_timestamp = parse_iso8601(creation_time, delimiter=' ')
+
+        view_count_str = self._html_search_regex(
+            r"<div class='title'>Views :</div>.*<div class='value'>([0-9,]+)<",
+            page, 'view count', flags=re.MULTILINE | re.DOTALL)
+        views = int(view_count_str.replace(',', ''))
+
+        return {
+            'id': video_id,
+            'url': compat_urlparse.urljoin(video_folder, fileName),
+            'title': oembed_obj['title'],
+            'thumbnail': thumb_url,
+            'description': self._html_search_meta('description', page),
+            'creator': oembed_obj['author_name'],
+            'duration': oembed_obj['duration'],
+            'timestamp': creation_timestamp,
+            'view_count': views,
+        }

From 3811c567e734f38f830e8c2e0a45afa7c48cbf12 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Wed, 11 Feb 2015 15:47:19 +0200
Subject: [PATCH 0321/6219] [teamcoco] Fix video id extraction

---
 youtube_dl/extractor/teamcoco.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/teamcoco.py b/youtube_dl/extractor/teamcoco.py
index e85d452a3..a73da1c9c 100644
--- a/youtube_dl/extractor/teamcoco.py
+++ b/youtube_dl/extractor/teamcoco.py
@@ -30,6 +30,11 @@ class TeamcocoIE(InfoExtractor):
             }
         }
     ]
+    _VIDEO_ID_REGEXES = (
+        r'"eVar42"\s*:\s*(\d+)',
+        r'Ginger\.TeamCoco\.openInApp\("video",\s*"([^"]+)"',
+        r'"id_not"\s*:\s*(\d+)'
+    )
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
@@ -40,8 +45,7 @@ class TeamcocoIE(InfoExtractor):
         video_id = mobj.group("video_id")
         if not video_id:
             video_id = self._html_search_regex(
-                r'<div\s+class="player".*?data-id="(\d+?)"',
-                webpage, 'video id')
+                self._VIDEO_ID_REGEXES, webpage, 'video id')
 
         data_url = 'http://teamcoco.com/cvp/2.0/%s.xml' % video_id
         data = self._download_xml(

From ea5db8469ee3e5f1bb4ba3deb88f28b4d9838e09 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Wed, 11 Feb 2015 16:21:52 +0200
Subject: [PATCH 0322/6219] [canalplus] Add support for itele.fr URLs (Closes
 #4931)

---
 youtube_dl/extractor/canalplus.py | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/canalplus.py b/youtube_dl/extractor/canalplus.py
index 11d18d74a..1b14471e5 100644
--- a/youtube_dl/extractor/canalplus.py
+++ b/youtube_dl/extractor/canalplus.py
@@ -15,12 +15,13 @@ from ..utils import (
 
 class CanalplusIE(InfoExtractor):
     IE_DESC = 'canalplus.fr, piwiplus.fr and d8.tv'
-    _VALID_URL = r'https?://(?:www\.(?P<site>canalplus\.fr|piwiplus\.fr|d8\.tv)/.*?/(?P<path>.*)|player\.canalplus\.fr/#/(?P<id>[0-9]+))'
+    _VALID_URL = r'https?://(?:www\.(?P<site>canalplus\.fr|piwiplus\.fr|d8\.tv|itele\.fr)/.*?/(?P<path>.*)|player\.canalplus\.fr/#/(?P<id>[0-9]+))'
     _VIDEO_INFO_TEMPLATE = 'http://service.canal-plus.com/video/rest/getVideosLiees/%s/%s'
     _SITE_ID_MAP = {
         'canalplus.fr': 'cplus',
         'piwiplus.fr': 'teletoon',
         'd8.tv': 'd8',
+        'itele.fr': 'itele',
     }
 
     _TESTS = [{
@@ -53,6 +54,16 @@ class CanalplusIE(InfoExtractor):
             'upload_date': '20131108',
         },
         'skip': 'videos get deleted after a while',
+    }, {
+        'url': 'http://www.itele.fr/france/video/aubervilliers-un-lycee-en-colere-111559',
+        'md5': '65aa83ad62fe107ce29e564bb8712580',
+        'info_dict': {
+            'id': '1213714',
+            'ext': 'flv',
+            'title': 'Aubervilliers : un lycée en colère - Le 11/02/2015 à 06h45',
+            'description': 'md5:8216206ec53426ea6321321f3b3c16db',
+            'upload_date': '20150211',
+        },
     }]
 
     def _real_extract(self, url):

From c56d7d899d5587cd73cb017b776ceb302494b6fd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 11 Feb 2015 22:10:33 +0600
Subject: [PATCH 0323/6219] [dctptv] Skip rtmp download

---
 youtube_dl/extractor/dctp.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/dctp.py b/youtube_dl/extractor/dctp.py
index 6ed354379..aa2c09eb6 100644
--- a/youtube_dl/extractor/dctp.py
+++ b/youtube_dl/extractor/dctp.py
@@ -14,6 +14,10 @@ class DctpTvIE(InfoExtractor):
             'display_id': 'videoinstallation-fuer-eine-kaufhausfassade',
             'ext': 'flv',
             'title': 'Videoinstallation für eine Kaufhausfassade'
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
         }
     }
 

From 8367d3f3cbad7ddae5116c3cd5d39ebfed9711c5 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 12 Feb 2015 00:11:33 +0800
Subject: [PATCH 0324/6219] [camdemy] Detection of external sources

---
 youtube_dl/extractor/camdemy.py | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/youtube_dl/extractor/camdemy.py b/youtube_dl/extractor/camdemy.py
index 54959f2cf..9a6341f0f 100644
--- a/youtube_dl/extractor/camdemy.py
+++ b/youtube_dl/extractor/camdemy.py
@@ -38,6 +38,19 @@ class CamdemyIE(InfoExtractor):
             'upload_date': '20140620',
             'timestamp': 1403271569,
         }
+    }, {
+        # External source
+        'url': 'http://www.camdemy.com/media/14842',
+        'md5': '50e1c3c3aa233d3d7b7daa2fa10b1cf7',
+        'info_dict': {
+            'id': '2vsYQzNIsJo',
+            'ext': 'mp4',
+            'upload_date': '20130211',
+            'uploader': 'Hun Kim',
+            'description': 'Excel 2013 Tutorial for Beginners - How to add Password Protection',
+            'uploader_id': 'hunkimtutorials',
+            'title': 'Excel 2013 Tutorial - How to add Password Protection',
+        }
     }]
 
     def _real_extract(self, url):
@@ -45,6 +58,13 @@ class CamdemyIE(InfoExtractor):
 
         page = self._download_webpage(url, video_id)
 
+        srcFrom = self._html_search_regex(
+            r"<div class='srcFrom'>Source: <a title='([^']+)'", page,
+            'external source', default=None)
+
+        if srcFrom:
+            return self.url_result(srcFrom)
+
         oembed_obj = self._download_json(
             'http://www.camdemy.com/oembed/?format=json&url=' + url, video_id)
 

From d304209a850793cd121820307be9bee7e9b9ae1e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 11 Feb 2015 17:25:04 +0100
Subject: [PATCH 0325/6219] [test/parameters.json] Set 'fixup' to 'never'

The fixed audio files for Youtube have a size lower that the minimum required.
---
 test/parameters.json | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/test/parameters.json b/test/parameters.json
index 098cd0cd0..af77b89b4 100644
--- a/test/parameters.json
+++ b/test/parameters.json
@@ -39,5 +39,6 @@
     "writesubtitles": false,
     "allsubtitles": false,
     "listssubtitles": false,
-    "socket_timeout": 20
+    "socket_timeout": 20,
+    "fixup": "never"
 }

From 124f3bc67d61adeb4dc257eb8a678df821c27314 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 11 Feb 2015 22:33:03 +0600
Subject: [PATCH 0326/6219] [dotsub] Fix extraction and modernize

---
 youtube_dl/extractor/dotsub.py | 44 +++++++++++++++++++++-------------
 1 file changed, 27 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/dotsub.py b/youtube_dl/extractor/dotsub.py
index 638bb33cd..f51d88a98 100644
--- a/youtube_dl/extractor/dotsub.py
+++ b/youtube_dl/extractor/dotsub.py
@@ -1,13 +1,14 @@
 from __future__ import unicode_literals
 
-import re
-import time
-
 from .common import InfoExtractor
+from ..utils import (
+    float_or_none,
+    int_or_none,
+)
 
 
 class DotsubIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?dotsub\.com/view/(?P<id>[^/]+)'
+    _VALID_URL = r'https?://(?:www\.)?dotsub\.com/view/(?P<id>[^/]+)'
     _TEST = {
         'url': 'http://dotsub.com/view/aed3b8b2-1889-4df5-ae63-ad85f5572f27',
         'md5': '0914d4d69605090f623b7ac329fea66e',
@@ -15,28 +16,37 @@ class DotsubIE(InfoExtractor):
             'id': 'aed3b8b2-1889-4df5-ae63-ad85f5572f27',
             'ext': 'flv',
             'title': 'Pyramids of Waste (2010), AKA The Lightbulb Conspiracy - Planned obsolescence documentary',
+            'description': 'md5:699a0f7f50aeec6042cb3b1db2d0d074',
+            'thumbnail': 're:^https?://dotsub.com/media/aed3b8b2-1889-4df5-ae63-ad85f5572f27/p',
+            'duration': 3169,
             'uploader': '4v4l0n42',
-            'description': 'Pyramids of Waste (2010) also known as "The lightbulb conspiracy" is a documentary about how our economic system based on consumerism  and planned obsolescence is breaking our planet down.\r\n\r\nSolutions to this can be found at:\r\nhttp://robotswillstealyourjob.com\r\nhttp://www.federicopistono.org\r\n\r\nhttp://opensourceecology.org\r\nhttp://thezeitgeistmovement.com',
-            'thumbnail': 'http://dotsub.com/media/aed3b8b2-1889-4df5-ae63-ad85f5572f27/p',
+            'timestamp': 1292248482.625,
             'upload_date': '20101213',
+            'view_count': int,
         }
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        info_url = "https://dotsub.com/api/media/%s/metadata" % video_id
-        info = self._download_json(info_url, video_id)
-        date = time.gmtime(info['dateCreated'] / 1000)  # The timestamp is in miliseconds
+        video_id = self._match_id(url)
+
+        info = self._download_json(
+            'https://dotsub.com/api/media/%s/metadata' % video_id, video_id)
+        video_url = info.get('mediaURI')
+
+        if not video_url:
+            webpage = self._download_webpage(url, video_id)
+            video_url = self._search_regex(
+                r'"file"\s*:\s*\'([^\']+)', webpage, 'video url')
 
         return {
             'id': video_id,
-            'url': info['mediaURI'],
+            'url': video_url,
             'ext': 'flv',
             'title': info['title'],
-            'thumbnail': info['screenshotURI'],
-            'description': info['description'],
-            'uploader': info['user'],
-            'view_count': info['numberOfViews'],
-            'upload_date': '%04i%02i%02i' % (date.tm_year, date.tm_mon, date.tm_mday),
+            'description': info.get('description'),
+            'thumbnail': info.get('screenshotURI'),
+            'duration': int_or_none(info.get('duration'), 1000),
+            'uploader': info.get('user'),
+            'timestamp': float_or_none(info.get('dateCreated'), 1000),
+            'view_count': int_or_none(info.get('numberOfViews')),
         }

From f36f92f4da3664f7286b8b4327ac5791fb98e5b2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 11 Feb 2015 18:07:12 +0100
Subject: [PATCH 0327/6219] [aes] style: Put __all__ variable at the end of the
 file

---
 youtube_dl/aes.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/aes.py b/youtube_dl/aes.py
index 5efd0f836..07224d508 100644
--- a/youtube_dl/aes.py
+++ b/youtube_dl/aes.py
@@ -1,7 +1,5 @@
 from __future__ import unicode_literals
 
-__all__ = ['aes_encrypt', 'key_expansion', 'aes_ctr_decrypt', 'aes_cbc_decrypt', 'aes_decrypt_text']
-
 import base64
 from math import ceil
 
@@ -329,3 +327,5 @@ def inc(data):
             data[i] = data[i] + 1
             break
     return data
+
+__all__ = ['aes_encrypt', 'key_expansion', 'aes_ctr_decrypt', 'aes_cbc_decrypt', 'aes_decrypt_text']

From 050fa4356122926d673635f7ba343293126e6d5c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 11 Feb 2015 18:08:42 +0100
Subject: [PATCH 0328/6219] flake8: Ignore some error added in pep8 1.6

* E402: we exectute code between imports, like modifying 'sys.path' in the tests
* E731: we assign to lambdas in a lot of places, we may want to consider defining functions in a single line instead (what pep8 recommends)
---
 setup.cfg | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.cfg b/setup.cfg
index 02aa56257..26857750c 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -3,4 +3,4 @@ universal = True
 
 [flake8]
 exclude = youtube_dl/extractor/__init__.py,devscripts/buildserver.py,setup.py,build,.git
-ignore = E501
+ignore = E402,E501,E731

From 2d30521ab9f535559dd208293982138516b88e8e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 11 Feb 2015 18:39:31 +0100
Subject: [PATCH 0329/6219] [youtube] Extract average rating (closes #2362)

---
 youtube_dl/extractor/common.py  | 1 +
 youtube_dl/extractor/youtube.py | 2 ++
 2 files changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 8d71494b4..48742189a 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -157,6 +157,7 @@ class InfoExtractor(object):
     view_count:     How many users have watched the video on the platform.
     like_count:     Number of positive ratings of the video
     dislike_count:  Number of negative ratings of the video
+    average_rating: Average rating give by users, the scale used depends on the webpage
     comment_count:  Number of comments on the video
     comments:       A list of comments, each with one or more of the following
                     properties (all but one of text or html optional):
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 8b91fff18..35ef4c303 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -25,6 +25,7 @@ from ..compat import (
 from ..utils import (
     clean_html,
     ExtractorError,
+    float_or_none,
     get_element_by_attribute,
     get_element_by_id,
     int_or_none,
@@ -1124,6 +1125,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
             'view_count': view_count,
             'like_count': like_count,
             'dislike_count': dislike_count,
+            'average_rating': float_or_none(video_info.get('avg_rating', [None])[0]),
             'formats': formats,
         }
 

From 5277f09dfce6ba1a6f191871df1515ddd68ddd42 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 11 Feb 2015 19:02:39 +0100
Subject: [PATCH 0330/6219] release 2015.02.11

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 225b30f8c..492ddf2ea 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.10.5'
+__version__ = '2015.02.11'

From c40feaba77d8717243467a65fcce4dcab6dfe7bf Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 12 Feb 2015 14:13:19 +0800
Subject: [PATCH 0331/6219] [camdemy] Add support for folders

---
 youtube_dl/extractor/__init__.py |  5 ++-
 youtube_dl/extractor/camdemy.py  | 68 +++++++++++++++++++++++++++-----
 2 files changed, 63 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 496e4d0b7..095feabf9 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -49,7 +49,10 @@ from .brightcove import BrightcoveIE
 from .buzzfeed import BuzzFeedIE
 from .byutv import BYUtvIE
 from .c56 import C56IE
-from .camdemy import CamdemyIE
+from .camdemy import (
+    CamdemyIE,
+    CamdemyFolderIE
+)
 from .canal13cl import Canal13clIE
 from .canalplus import CanalplusIE
 from .canalc2 import Canalc2IE
diff --git a/youtube_dl/extractor/camdemy.py b/youtube_dl/extractor/camdemy.py
index 9a6341f0f..0d64b5a16 100644
--- a/youtube_dl/extractor/camdemy.py
+++ b/youtube_dl/extractor/camdemy.py
@@ -4,12 +4,12 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urlparse
+from ..compat import compat_urllib_parse
 from ..utils import parse_iso8601
 
 
 class CamdemyIE(InfoExtractor):
-    _VALID_URL = r'http://www.camdemy.com/media/(?P<id>\d+).*'
+    _VALID_URL = r'http://www.camdemy.com/media/(?P<id>\d+)'
     _TESTS = [{
         # single file
         'url': 'http://www.camdemy.com/media/5181/',
@@ -69,25 +69,25 @@ class CamdemyIE(InfoExtractor):
             'http://www.camdemy.com/oembed/?format=json&url=' + url, video_id)
 
         thumb_url = oembed_obj['thumbnail_url']
-        video_folder = compat_urlparse.urljoin(thumb_url, 'video/')
+        video_folder = compat_urllib_parse.urljoin(thumb_url, 'video/')
         fileListXML = self._download_xml(
-            compat_urlparse.urljoin(video_folder, 'fileList.xml'),
+            compat_urllib_parse.urljoin(video_folder, 'fileList.xml'),
             video_id, 'Filelist XML')
         fileName = fileListXML.find('./video/item/fileName').text
 
         creation_time = self._html_search_regex(
-            r"<div class='title'>Posted :</div>.*<div class='value'>([0-9:\- ]+)<",
-            page, 'creation time', flags=re.MULTILINE | re.DOTALL) + '+08:00'
+            r"<div class='title'>Posted :</div>[\r\n ]*<div class='value'>([^<>]+)<",
+            page, 'creation time', flags=re.MULTILINE) + '+08:00'
         creation_timestamp = parse_iso8601(creation_time, delimiter=' ')
 
         view_count_str = self._html_search_regex(
-            r"<div class='title'>Views :</div>.*<div class='value'>([0-9,]+)<",
-            page, 'view count', flags=re.MULTILINE | re.DOTALL)
+            r"<div class='title'>Views :</div>[\r\n ]*<div class='value'>([^<>]+)<",
+            page, 'view count', flags=re.MULTILINE)
         views = int(view_count_str.replace(',', ''))
 
         return {
             'id': video_id,
-            'url': compat_urlparse.urljoin(video_folder, fileName),
+            'url': compat_urllib_parse.urljoin(video_folder, fileName),
             'title': oembed_obj['title'],
             'thumbnail': thumb_url,
             'description': self._html_search_meta('description', page),
@@ -96,3 +96,53 @@ class CamdemyIE(InfoExtractor):
             'timestamp': creation_timestamp,
             'view_count': views,
         }
+
+
+class CamdemyFolderIE(InfoExtractor):
+    _VALID_URL = r'http://www.camdemy.com/folder/(?P<id>\d+)'
+    _TESTS = [{
+        # links with trailing slash
+        'url': 'http://www.camdemy.com/folder/450',
+        'info_dict': {
+            'id': '450',
+            'title': '信號與系統 2012 & 2011 (Signals and Systems)',
+        },
+        'playlist_mincount': 145
+    }, {
+        # links without trailing slash
+        # and multi-page
+        'url': 'http://www.camdemy.com/folder/853',
+        'info_dict': {
+            'id': '853',
+            'title': '科學計算 - 使用 Matlab'
+        },
+        'playlist_mincount': 20
+    }, {
+        # with displayMode parameter. For testing the codes to add parameters
+        'url': 'http://www.camdemy.com/folder/853/?displayMode=defaultOrderByOrg',
+        'info_dict': {
+            'id': '853',
+            'title': '科學計算 - 使用 Matlab'
+        },
+        'playlist_mincount': 20
+    }]
+
+    def _real_extract(self, url):
+        folder_id = self._match_id(url)
+
+        # Add displayMode=list so that all links are displayed in a single page
+        parsed_url = list(compat_urllib_parse.urlparse(url))
+        query = dict(compat_urllib_parse.parse_qsl(parsed_url[4]))
+        query.update({'displayMode': 'list'})
+        parsed_url[4] = compat_urllib_parse.urlencode(query)
+        final_url = compat_urllib_parse.urlunparse(parsed_url)
+
+        page = self._download_webpage(final_url, folder_id)
+        matches = re.findall(r"href='(/media/\d+/?)'", page)
+
+        entries = [self.url_result('http://www.camdemy.com' + media_path)
+                   for media_path in matches]
+
+        folder_title = self._html_search_meta('keywords', page)
+
+        return self.playlist_result(entries, folder_id, folder_title)

From 7e6011101f00e4f7216467e80091cc89c2846f01 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 12 Feb 2015 14:23:25 +0800
Subject: [PATCH 0332/6219] [camdemy] Python2 compatibility

---
 youtube_dl/extractor/camdemy.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/camdemy.py b/youtube_dl/extractor/camdemy.py
index 0d64b5a16..1bc602c31 100644
--- a/youtube_dl/extractor/camdemy.py
+++ b/youtube_dl/extractor/camdemy.py
@@ -4,7 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse
+from ..compat import (compat_urllib_parse, compat_urlparse)
 from ..utils import parse_iso8601
 
 
@@ -69,9 +69,9 @@ class CamdemyIE(InfoExtractor):
             'http://www.camdemy.com/oembed/?format=json&url=' + url, video_id)
 
         thumb_url = oembed_obj['thumbnail_url']
-        video_folder = compat_urllib_parse.urljoin(thumb_url, 'video/')
+        video_folder = compat_urlparse.urljoin(thumb_url, 'video/')
         fileListXML = self._download_xml(
-            compat_urllib_parse.urljoin(video_folder, 'fileList.xml'),
+            compat_urlparse.urljoin(video_folder, 'fileList.xml'),
             video_id, 'Filelist XML')
         fileName = fileListXML.find('./video/item/fileName').text
 
@@ -87,7 +87,7 @@ class CamdemyIE(InfoExtractor):
 
         return {
             'id': video_id,
-            'url': compat_urllib_parse.urljoin(video_folder, fileName),
+            'url': compat_urlparse.urljoin(video_folder, fileName),
             'title': oembed_obj['title'],
             'thumbnail': thumb_url,
             'description': self._html_search_meta('description', page),
@@ -131,11 +131,11 @@ class CamdemyFolderIE(InfoExtractor):
         folder_id = self._match_id(url)
 
         # Add displayMode=list so that all links are displayed in a single page
-        parsed_url = list(compat_urllib_parse.urlparse(url))
-        query = dict(compat_urllib_parse.parse_qsl(parsed_url[4]))
+        parsed_url = list(compat_urlparse.urlparse(url))
+        query = dict(compat_urlparse.parse_qsl(parsed_url[4]))
         query.update({'displayMode': 'list'})
         parsed_url[4] = compat_urllib_parse.urlencode(query)
-        final_url = compat_urllib_parse.urlunparse(parsed_url)
+        final_url = compat_urlparse.urlunparse(parsed_url)
 
         page = self._download_webpage(final_url, folder_id)
         matches = re.findall(r"href='(/media/\d+/?)'", page)

From 08b38d54015706d90d05371455ba86b7887cabcc Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 12 Feb 2015 08:55:06 +0100
Subject: [PATCH 0333/6219] [camdemy] Simplify and make more robust (#4938)

Do not throw errors if view count or upload date extraction fails.
Dispose of re.MULTILINE, which had absolutely no effect without any ^ or $ in sight.
Follow PEP8 naming conventions.
---
 youtube_dl/extractor/camdemy.py | 47 ++++++++++++++++++---------------
 youtube_dl/utils.py             | 27 ++++++++++---------
 2 files changed, 40 insertions(+), 34 deletions(-)

diff --git a/youtube_dl/extractor/camdemy.py b/youtube_dl/extractor/camdemy.py
index 1bc602c31..5de5879b4 100644
--- a/youtube_dl/extractor/camdemy.py
+++ b/youtube_dl/extractor/camdemy.py
@@ -1,11 +1,18 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import datetime
 import re
 
 from .common import InfoExtractor
-from ..compat import (compat_urllib_parse, compat_urlparse)
-from ..utils import parse_iso8601
+from ..compat import (
+    compat_urllib_parse,
+    compat_urlparse,
+)
+from ..utils import (
+    parse_iso8601,
+    str_to_int,
+)
 
 
 class CamdemyIE(InfoExtractor):
@@ -23,6 +30,7 @@ class CamdemyIE(InfoExtractor):
             'creator': 'ss11spring',
             'upload_date': '20130114',
             'timestamp': 1358154556,
+            'view_count': int,
         }
     }, {
         # With non-empty description
@@ -55,46 +63,43 @@ class CamdemyIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-
         page = self._download_webpage(url, video_id)
 
-        srcFrom = self._html_search_regex(
+        src_from = self._html_search_regex(
             r"<div class='srcFrom'>Source: <a title='([^']+)'", page,
             'external source', default=None)
-
-        if srcFrom:
-            return self.url_result(srcFrom)
+        if src_from:
+            return self.url_result(src_from)
 
         oembed_obj = self._download_json(
             'http://www.camdemy.com/oembed/?format=json&url=' + url, video_id)
 
         thumb_url = oembed_obj['thumbnail_url']
         video_folder = compat_urlparse.urljoin(thumb_url, 'video/')
-        fileListXML = self._download_xml(
+        file_list_doc = self._download_xml(
             compat_urlparse.urljoin(video_folder, 'fileList.xml'),
             video_id, 'Filelist XML')
-        fileName = fileListXML.find('./video/item/fileName').text
+        file_name = file_list_doc.find('./video/item/fileName').text
+        video_url = compat_urlparse.urljoin(video_folder, file_name)
 
-        creation_time = self._html_search_regex(
-            r"<div class='title'>Posted :</div>[\r\n ]*<div class='value'>([^<>]+)<",
-            page, 'creation time', flags=re.MULTILINE) + '+08:00'
-        creation_timestamp = parse_iso8601(creation_time, delimiter=' ')
-
-        view_count_str = self._html_search_regex(
-            r"<div class='title'>Views :</div>[\r\n ]*<div class='value'>([^<>]+)<",
-            page, 'view count', flags=re.MULTILINE)
-        views = int(view_count_str.replace(',', ''))
+        timestamp = parse_iso8601(self._html_search_regex(
+            r"<div class='title'>Posted\s*:</div>\s*<div class='value'>([^<>]+)<",
+            page, 'creation time', fatal=False),
+            delimiter=' ', timezone=datetime.timedelta(hours=8))
+        view_count = str_to_int(self._html_search_regex(
+            r"<div class='title'>Views\s*:</div>\s*<div class='value'>([^<>]+)<",
+            page, 'view count', fatal=False))
 
         return {
             'id': video_id,
-            'url': compat_urlparse.urljoin(video_folder, fileName),
+            'url': video_url,
             'title': oembed_obj['title'],
             'thumbnail': thumb_url,
             'description': self._html_search_meta('description', page),
             'creator': oembed_obj['author_name'],
             'duration': oembed_obj['duration'],
-            'timestamp': creation_timestamp,
-            'view_count': views,
+            'timestamp': timestamp,
+            'view_count': view_count,
         }
 
 
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 03566d223..54fa17c38 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -666,26 +666,27 @@ class YoutubeDLHTTPSHandler(compat_urllib_request.HTTPSHandler):
             req, **kwargs)
 
 
-def parse_iso8601(date_str, delimiter='T'):
+def parse_iso8601(date_str, delimiter='T', timezone=None):
     """ Return a UNIX timestamp from the given date """
 
     if date_str is None:
         return None
 
-    m = re.search(
-        r'(\.[0-9]+)?(?:Z$| ?(?P<sign>\+|-)(?P<hours>[0-9]{2}):?(?P<minutes>[0-9]{2})$)',
-        date_str)
-    if not m:
-        timezone = datetime.timedelta()
-    else:
-        date_str = date_str[:-len(m.group(0))]
-        if not m.group('sign'):
+    if timezone is None:
+        m = re.search(
+            r'(\.[0-9]+)?(?:Z$| ?(?P<sign>\+|-)(?P<hours>[0-9]{2}):?(?P<minutes>[0-9]{2})$)',
+            date_str)
+        if not m:
             timezone = datetime.timedelta()
         else:
-            sign = 1 if m.group('sign') == '+' else -1
-            timezone = datetime.timedelta(
-                hours=sign * int(m.group('hours')),
-                minutes=sign * int(m.group('minutes')))
+            date_str = date_str[:-len(m.group(0))]
+            if not m.group('sign'):
+                timezone = datetime.timedelta()
+            else:
+                sign = 1 if m.group('sign') == '+' else -1
+                timezone = datetime.timedelta(
+                    hours=sign * int(m.group('hours')),
+                    minutes=sign * int(m.group('minutes')))
     date_format = '%Y-%m-%d{0}%H:%M:%S'.format(delimiter)
     dt = datetime.datetime.strptime(date_str, date_format) - timezone
     return calendar.timegm(dt.timetuple())

From 7c24ce225d1ddb350f91b0fd78fefa40f6aa4481 Mon Sep 17 00:00:00 2001
From: robin <rderooij685@gmail.com>
Date: Thu, 12 Feb 2015 19:19:55 +0100
Subject: [PATCH 0334/6219] [NPORadio] Added extractor for live radio

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/nporadio.py | 40 ++++++++++++++++++++++++++++++++
 2 files changed, 41 insertions(+)
 create mode 100644 youtube_dl/extractor/nporadio.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 095feabf9..b118c3d1d 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -320,6 +320,7 @@ from .npo import (
     NPOLiveIE,
     TegenlichtVproIE,
 )
+from .nporadio import NPORadioIE
 from .nrk import (
     NRKIE,
     NRKTVIE,
diff --git a/youtube_dl/extractor/nporadio.py b/youtube_dl/extractor/nporadio.py
new file mode 100644
index 000000000..dd8d97cb7
--- /dev/null
+++ b/youtube_dl/extractor/nporadio.py
@@ -0,0 +1,40 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import json
+
+from .common import InfoExtractor
+
+
+class NPORadioIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?npo\.nl/radio/(?P<id>.*)'
+    _TEST = {
+        'url': 'http://www.npo.nl/radio/radio-1',
+        'info_dict': {
+            'id': 'radio-1',
+            'ext': 'mp3',
+            'title': 'NPO Radio 1',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        title = self._html_search_regex(
+                self._html_get_attribute_regex('data-channel'), webpage, 'title')
+       
+        json_data = json.loads(
+                     self._html_search_regex(
+                     self._html_get_attribute_regex('data-streams'), webpage, 'data-streams'))
+        
+        return {
+            'id': video_id,
+            'title': title,
+            'ext': json_data['codec'],
+            'url': json_data['url']
+        }
+
+    def _html_get_attribute_regex(self, attribute):
+        return r'{0}\s*=\s*\'([^\']+)\''.format(attribute)
+

From c3d64fc1b30b46cc4f58aaa55c02918248bb1bee Mon Sep 17 00:00:00 2001
From: robin <rderooij685@gmail.com>
Date: Thu, 12 Feb 2015 19:27:59 +0100
Subject: [PATCH 0335/6219] [nporadio] Edit to confirm to flake8 standards

---
 youtube_dl/extractor/nporadio.py | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/nporadio.py b/youtube_dl/extractor/nporadio.py
index dd8d97cb7..d1c4ec167 100644
--- a/youtube_dl/extractor/nporadio.py
+++ b/youtube_dl/extractor/nporadio.py
@@ -22,12 +22,12 @@ class NPORadioIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         title = self._html_search_regex(
-                self._html_get_attribute_regex('data-channel'), webpage, 'title')
-       
+            self._html_get_attribute_regex('data-channel'), webpage, 'title')
+
         json_data = json.loads(
-                     self._html_search_regex(
-                     self._html_get_attribute_regex('data-streams'), webpage, 'data-streams'))
-        
+            self._html_search_regex(
+                self._html_get_attribute_regex('data-streams'), webpage, 'data-streams'))
+
         return {
             'id': video_id,
             'title': title,
@@ -37,4 +37,3 @@ class NPORadioIE(InfoExtractor):
 
     def _html_get_attribute_regex(self, attribute):
         return r'{0}\s*=\s*\'([^\']+)\''.format(attribute)
-

From 171ca612af948b2966e56054ce3a25a9f5ab2a76 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 13 Feb 2015 01:36:54 +0600
Subject: [PATCH 0336/6219] [npo:radio] Move to extractor to common npo place
 and add extractor for fragments

---
 youtube_dl/extractor/__init__.py |  3 +-
 youtube_dl/extractor/npo.py      | 83 +++++++++++++++++++++++++++++++-
 youtube_dl/extractor/nporadio.py | 39 ---------------
 3 files changed, 83 insertions(+), 42 deletions(-)
 delete mode 100644 youtube_dl/extractor/nporadio.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index b118c3d1d..a4fab540b 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -318,9 +318,10 @@ from .nowvideo import NowVideoIE
 from .npo import (
     NPOIE,
     NPOLiveIE,
+    NPORadioIE,
+    NPORadioFragmentIE,
     TegenlichtVproIE,
 )
-from .nporadio import NPORadioIE
 from .nrk import (
     NRKIE,
     NRKTVIE,
diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index 54be06a4e..c075618e8 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -1,6 +1,7 @@
 from __future__ import unicode_literals
 
 from .subtitles import SubtitlesInfoExtractor
+from .common import InfoExtractor
 from ..utils import (
     fix_xml_ampersands,
     parse_duration,
@@ -22,7 +23,7 @@ class NPOBaseIE(SubtitlesInfoExtractor):
 
 class NPOIE(NPOBaseIE):
     IE_NAME = 'npo.nl'
-    _VALID_URL = r'https?://www\.npo\.nl/[^/]+/[^/]+/(?P<id>[^/?]+)'
+    _VALID_URL = r'https?://(?:www\.)?npo\.nl/(?!live|radio)[^/]+/[^/]+/(?P<id>[^/?]+)'
 
     _TESTS = [
         {
@@ -185,7 +186,7 @@ class NPOIE(NPOBaseIE):
 
 class NPOLiveIE(NPOBaseIE):
     IE_NAME = 'npo.nl:live'
-    _VALID_URL = r'https?://www\.npo\.nl/live/(?P<id>.+)'
+    _VALID_URL = r'https?://(?:www\.)?npo\.nl/live/(?P<id>.+)'
 
     _TEST = {
         'url': 'http://www.npo.nl/live/npo-1',
@@ -260,6 +261,84 @@ class NPOLiveIE(NPOBaseIE):
         }
 
 
+class NPORadioIE(InfoExtractor):
+    IE_NAME = 'npo.nl:radio'
+    _VALID_URL = r'https?://(?:www\.)?npo\.nl/radio/(?P<id>[^/]+)/?$'
+
+    _TEST = {
+        'url': 'http://www.npo.nl/radio/radio-1',
+        'info_dict': {
+            'id': 'radio-1',
+            'ext': 'mp3',
+            'title': 're:^NPO Radio 1 [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
+            'is_live': True,
+        },
+        'params': {
+            'skip_download': True,
+        }
+    }
+
+    @staticmethod
+    def _html_get_attribute_regex(attribute):
+        return r'{0}\s*=\s*\'([^\']+)\''.format(attribute)
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        title = self._html_search_regex(
+            self._html_get_attribute_regex('data-channel'), webpage, 'title')
+
+        stream = self._parse_json(
+            self._html_search_regex(self._html_get_attribute_regex('data-streams'), webpage, 'data-streams'),
+            video_id)
+
+        codec = stream.get('codec')
+
+        return {
+            'id': video_id,
+            'url': stream['url'],
+            'title': self._live_title(title),
+            'acodec': codec,
+            'ext': codec,
+            'is_live': True,
+        }
+
+
+class NPORadioFragmentIE(InfoExtractor):
+    IE_NAME = 'npo.nl:radio:fragment'
+    _VALID_URL = r'https?://(?:www\.)?npo\.nl/radio/[^/]+/fragment/(?P<id>\d+)'
+
+    _TEST = {
+        'url': 'http://www.npo.nl/radio/radio-5/fragment/174356',
+        'md5': 'dd8cc470dad764d0fdc70a9a1e2d18c2',
+        'info_dict': {
+            'id': '174356',
+            'ext': 'mp3',
+            'title': 'Jubileumconcert Willeke Alberti',
+        },
+    }
+
+    def _real_extract(self, url):
+        audio_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, audio_id)
+
+        title = self._html_search_regex(
+            r'href="/radio/[^/]+/fragment/%s" title="([^"]+)"' % audio_id,
+            webpage, 'title')
+
+        audio_url = self._search_regex(
+            r"data-streams='([^']+)'", webpage, 'audio url')
+
+        return {
+            'id': audio_id,
+            'url': audio_url,
+            'title': title,
+        }
+
+
 class TegenlichtVproIE(NPOIE):
     IE_NAME = 'tegenlicht.vpro.nl'
     _VALID_URL = r'https?://tegenlicht\.vpro\.nl/afleveringen/.*?'
diff --git a/youtube_dl/extractor/nporadio.py b/youtube_dl/extractor/nporadio.py
deleted file mode 100644
index d1c4ec167..000000000
--- a/youtube_dl/extractor/nporadio.py
+++ /dev/null
@@ -1,39 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-import json
-
-from .common import InfoExtractor
-
-
-class NPORadioIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?npo\.nl/radio/(?P<id>.*)'
-    _TEST = {
-        'url': 'http://www.npo.nl/radio/radio-1',
-        'info_dict': {
-            'id': 'radio-1',
-            'ext': 'mp3',
-            'title': 'NPO Radio 1',
-        }
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-
-        title = self._html_search_regex(
-            self._html_get_attribute_regex('data-channel'), webpage, 'title')
-
-        json_data = json.loads(
-            self._html_search_regex(
-                self._html_get_attribute_regex('data-streams'), webpage, 'data-streams'))
-
-        return {
-            'id': video_id,
-            'title': title,
-            'ext': json_data['codec'],
-            'url': json_data['url']
-        }
-
-    def _html_get_attribute_regex(self, attribute):
-        return r'{0}\s*=\s*\'([^\']+)\''.format(attribute)

From 7105440cecf82aff295df4f32575f6c8b64b3c2d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 13 Feb 2015 15:14:23 +0800
Subject: [PATCH 0337/6219] [Yam] Add new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/yam.py      | 70 ++++++++++++++++++++++++++++++++
 youtube_dl/utils.py              | 20 +++++++--
 3 files changed, 87 insertions(+), 4 deletions(-)
 create mode 100644 youtube_dl/extractor/yam.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index a4fab540b..f749ec333 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -578,6 +578,7 @@ from .yahoo import (
     YahooIE,
     YahooSearchIE,
 )
+from .yam import YamIE
 from .yesjapan import YesJapanIE
 from .ynet import YnetIE
 from .youjizz import YouJizzIE
diff --git a/youtube_dl/extractor/yam.py b/youtube_dl/extractor/yam.py
new file mode 100644
index 000000000..511c8b67c
--- /dev/null
+++ b/youtube_dl/extractor/yam.py
@@ -0,0 +1,70 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..compat import compat_urlparse
+from ..utils import month_by_abbreviation
+import re
+
+
+class YamIE(InfoExtractor):
+    _VALID_URL = r'http://mymedia.yam.com/m/(?P<id>\d+)'
+
+    _TESTS = [{
+        # An audio hosted on Yam
+        'url': 'http://mymedia.yam.com/m/2283921',
+        'md5': 'c011b8e262a52d5473d9c2e3c9963b9c',
+        'info_dict': {
+            'id': '2283921',
+            'ext': 'mp3',
+            'title': '發現 - 趙薇 京華煙雲主題曲',
+            'uploader_id': 'princekt',
+            'upload_date': '20080807',
+            'duration': 313.0,
+        }
+    }, {
+        # An external video hosted on YouTube
+        'url': 'http://mymedia.yam.com/m/3598173',
+        'md5': '0238ceec479c654e8c2f1223755bf3e9',
+        'info_dict': {
+            'id': 'pJ2Deys283c',
+            'ext': 'mp4',
+            'upload_date': '20150202',
+            'uploader': '新莊社大瑜伽社',
+            'description': 'md5:f5cc72f0baf259a70fb731654b0d2eff',
+            'uploader_id': '2323agoy',
+            'title': '外婆的澎湖灣KTV-潘安邦',
+        }
+    }]
+
+    def _real_extract(self, url):
+        media_id = self._match_id(url)
+        page = self._download_webpage(url, media_id)
+
+        # Is it hosted externally on YouTube?
+        youtube_url = self._html_search_regex(
+            r'<embed src="(http://www.youtube.com/[^"]+)"',
+            page, 'YouTube url', default=None)
+        if youtube_url:
+            return self.url_result(youtube_url, 'Youtube')
+
+        api_page = self._download_webpage(
+            'http://mymedia.yam.com/api/a/?pID=' + media_id, media_id)
+        api_result_obj = compat_urlparse.parse_qs(api_page)
+
+        author = self._html_search_regex(
+            r'<!-- 發表作者 -->：[\n ]+<a href="/([a-z]+)"', page, 'author')
+        mobj = re.search(r'<!-- 發表於 -->(?P<mon>[A-Z][a-z]{2})  ' +
+                         r'(?P<day>\d{1,2}), (?P<year>\d{4})', page)
+        upload_date = '%s%02d%02d' % (mobj.group('year'),
+                                      month_by_abbreviation(mobj.group('mon')),
+                                      int(mobj.group('day')))
+
+        return {
+            'id': media_id,
+            'url': api_result_obj['mp3file'][0],
+            'title': self._html_search_meta('description', page),
+            'duration': float(api_result_obj['totaltime'][0]) / 1000.0,
+            'uploader_id': author,
+            'upload_date': upload_date,
+        }
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 54fa17c38..3eb6bc6d4 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -62,6 +62,11 @@ std_headers = {
 }
 
 
+ENGLISH_MONTH_NAMES = [
+    'January', 'February', 'March', 'April', 'May', 'June',
+    'July', 'August', 'September', 'October', 'November', 'December']
+
+
 def preferredencoding():
     """Get preferred encoding.
 
@@ -1185,11 +1190,18 @@ def get_term_width():
 def month_by_name(name):
     """ Return the number of a month by (locale-independently) English name """
 
-    ENGLISH_NAMES = [
-        'January', 'February', 'March', 'April', 'May', 'June',
-        'July', 'August', 'September', 'October', 'November', 'December']
     try:
-        return ENGLISH_NAMES.index(name) + 1
+        return ENGLISH_MONTH_NAMES.index(name) + 1
+    except ValueError:
+        return None
+
+
+def month_by_abbreviation(abbrev):
+    """ Return the number of a month by (locale-independently) English
+        abbreviations """
+
+    try:
+        return [s[:3] for s in ENGLISH_MONTH_NAMES].index(abbrev) + 1
     except ValueError:
         return None
 

From ae6423d7042de863c303fcbdc1646ea370233741 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 13 Feb 2015 11:36:33 +0100
Subject: [PATCH 0338/6219] [bambuser] Fix 'uploader_id' extraction (fixes
 #4944)

---
 youtube_dl/extractor/bambuser.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bambuser.py b/youtube_dl/extractor/bambuser.py
index 98e1443ab..c193e66ca 100644
--- a/youtube_dl/extractor/bambuser.py
+++ b/youtube_dl/extractor/bambuser.py
@@ -50,7 +50,7 @@ class BambuserIE(InfoExtractor):
             'duration': int(info['length']),
             'view_count': int(info['views_total']),
             'uploader': info['username'],
-            'uploader_id': info['uid'],
+            'uploader_id': info['owner']['uid'],
         }
 
 

From 36e7a4ca2e85be8dc16955e28f9c563e01aa9eca Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 13 Feb 2015 14:43:50 +0100
Subject: [PATCH 0339/6219] [test/subtitles] Update checksums

---
 test/test_subtitles.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/test_subtitles.py b/test/test_subtitles.py
index 6336dd317..3e329438f 100644
--- a/test/test_subtitles.py
+++ b/test/test_subtitles.py
@@ -247,7 +247,7 @@ class TestVimeoSubtitles(BaseTestSubtitles):
     def test_subtitles(self):
         self.DL.params['writesubtitles'] = True
         subtitles = self.getSubtitles()
-        self.assertEqual(md5(subtitles['en']), '26399116d23ae3cf2c087cea94bc43b4')
+        self.assertEqual(md5(subtitles['en']), '8062383cf4dec168fc40a088aa6d5888')
 
     def test_subtitles_lang(self):
         self.DL.params['writesubtitles'] = True
@@ -334,7 +334,7 @@ class TestCeskaTelevizeSubtitles(BaseTestSubtitles):
         self.DL.params['allsubtitles'] = True
         subtitles = self.getSubtitles()
         self.assertEqual(set(subtitles.keys()), set(['cs']))
-        self.assertEqual(md5(subtitles['cs']), '9bf52d9549533c32c427e264bf0847d4')
+        self.assertEqual(md5(subtitles['cs']), 'cc3957b2c6dff1db71e5f2e83d467480')
 
     def test_nosubtitles(self):
         self.DL.expect_warning('video doesn\'t have subtitles')

From 037e9437e435f32d287354c77f6586d7dc34544b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 13 Feb 2015 20:10:42 +0600
Subject: [PATCH 0340/6219] [camdemy] Fix _VALID_URL

---
 youtube_dl/extractor/camdemy.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/camdemy.py b/youtube_dl/extractor/camdemy.py
index 5de5879b4..897f3a104 100644
--- a/youtube_dl/extractor/camdemy.py
+++ b/youtube_dl/extractor/camdemy.py
@@ -16,7 +16,7 @@ from ..utils import (
 
 
 class CamdemyIE(InfoExtractor):
-    _VALID_URL = r'http://www.camdemy.com/media/(?P<id>\d+)'
+    _VALID_URL = r'http://(?:www\.)?camdemy\.com/media/(?P<id>\d+)'
     _TESTS = [{
         # single file
         'url': 'http://www.camdemy.com/media/5181/',

From 9347fddbfc81114f85fecc72e8452813ef130fdb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Feb 2015 02:04:28 +0600
Subject: [PATCH 0341/6219] [1tv] Cover arbitraty URLs

---
 youtube_dl/extractor/firsttv.py | 57 ++++++++++++++++++++++-----------
 1 file changed, 39 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/firsttv.py b/youtube_dl/extractor/firsttv.py
index 08ceee4ed..6e015ca16 100644
--- a/youtube_dl/extractor/firsttv.py
+++ b/youtube_dl/extractor/firsttv.py
@@ -8,45 +8,66 @@ from ..utils import int_or_none
 
 
 class FirstTVIE(InfoExtractor):
-    IE_NAME = 'firsttv'
-    IE_DESC = 'Видеоархив - Первый канал'
-    _VALID_URL = r'http://(?:www\.)?1tv\.ru/videoarchive/(?P<id>\d+)'
+    IE_NAME = '1tv'
+    IE_DESC = 'Первый канал'
+    _VALID_URL = r'http://(?:www\.)?1tv\.ru/(?:[^/]+/)+(?P<id>.+)'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.1tv.ru/videoarchive/73390',
-        'md5': '3de6390cf0cca4a5eae1d1d83895e5ad',
+        'md5': '777f525feeec4806130f4f764bc18a4f',
         'info_dict': {
             'id': '73390',
             'ext': 'mp4',
             'title': 'Олимпийские канатные дороги',
-            'description': 'md5:cc730d2bf4215463e37fff6a1e277b13',
-            'thumbnail': 'http://img1.1tv.ru/imgsize640x360/PR20140210114657.JPG',
+            'description': 'md5:d41d8cd98f00b204e9800998ecf8427e',
+            'thumbnail': 're:^https?://.*\.(?:jpg|JPG)$',
             'duration': 149,
+            'like_count': int,
+            'dislike_count': int,
         },
         'skip': 'Only works from Russia',
-    }
+    }, {
+        'url': 'http://www.1tv.ru/prj/inprivate/vypusk/35930',
+        'md5': 'a1b6b60d530ebcf8daacf4565762bbaf',
+        'info_dict': {
+            'id': '35930',
+            'ext': 'mp4',
+            'title': 'Наедине со всеми. Людмила Сенчина',
+            'description': 'md5:89553aed1d641416001fe8d450f06cb9',
+            'thumbnail': 're:^https?://.*\.(?:jpg|JPG)$',
+            'duration': 2694,
+        },
+        'skip': 'Only works from Russia',
+    }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id, 'Downloading page')
 
         video_url = self._html_search_regex(
-            r'''(?s)jwplayer\('flashvideoportal_1'\)\.setup\({.*?'file': '([^']+)'.*?}\);''', webpage, 'video URL')
+            r'''(?s)(?:jwplayer\('flashvideoportal_1'\)\.setup\({|var\s+playlistObj\s*=).*?'file'\s*:\s*'([^']+)'.*?}\);''',
+            webpage, 'video URL')
 
         title = self._html_search_regex(
-            r'<div class="tv_translation">\s*<h1><a href="[^"]+">([^<]*)</a>', webpage, 'title')
+            [r'<div class="tv_translation">\s*<h1><a href="[^"]+">([^<]*)</a>',
+             r"'title'\s*:\s*'([^']+)'"], webpage, 'title')
         description = self._html_search_regex(
-            r'<div class="descr">\s*<div>&nbsp;</div>\s*<p>([^<]*)</p></div>', webpage, 'description', fatal=False)
+            r'<div class="descr">\s*<div>&nbsp;</div>\s*<p>([^<]*)</p></div>',
+            webpage, 'description', default=None) or self._html_search_meta(
+                'description', webpage, 'description')
 
         thumbnail = self._og_search_thumbnail(webpage)
-        duration = self._og_search_property('video:duration', webpage, 'video duration', fatal=False)
+        duration = self._og_search_property(
+            'video:duration', webpage,
+            'video duration', fatal=False)
 
-        like_count = self._html_search_regex(r'title="Понравилось".*?/></label> \[(\d+)\]',
-                                             webpage, 'like count', fatal=False)
-        dislike_count = self._html_search_regex(r'title="Не понравилось".*?/></label> \[(\d+)\]',
-                                                webpage, 'dislike count', fatal=False)
+        like_count = self._html_search_regex(
+            r'title="Понравилось".*?/></label> \[(\d+)\]',
+            webpage, 'like count', default=None)
+        dislike_count = self._html_search_regex(
+            r'title="Не понравилось".*?/></label> \[(\d+)\]',
+            webpage, 'dislike count', default=None)
 
         return {
             'id': video_id,

From cffcbc02de504d84e1c2677bb525c00b03e04f53 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 13 Feb 2015 22:25:34 +0100
Subject: [PATCH 0342/6219] [postprocessor/ffmpeg] Don't let ffmpeg read from
 stdin (fixes #4945)

If you run 'while read aurl ; do youtube-dl --extract-audio "${aurl}"; done < path_to_batch_file'  (batch_file contains one url per line) each call to youtube-dl consumed some characters and 'read' would assing to 'aurl' a non valid url, something like 'tube.com/watch?v=<id>'.
---
 youtube_dl/postprocessor/ffmpeg.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 01d25f760..5238ce534 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -95,7 +95,7 @@ class FFmpegPostProcessor(PostProcessor):
 
         if self._downloader.params.get('verbose', False):
             self._downloader.to_screen('[debug] ffmpeg command line: %s' % shell_quote(cmd))
-        p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
         stdout, stderr = p.communicate()
         if p.returncode != 0:
             stderr = stderr.decode('utf-8', 'replace')
@@ -134,7 +134,7 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
                 encodeFilename(self._probe_executable, True),
                 encodeArgument('-show_streams'),
                 encodeFilename(self._ffmpeg_filename_argument(path), True)]
-            handle = subprocess.Popen(cmd, stderr=compat_subprocess_get_DEVNULL(), stdout=subprocess.PIPE)
+            handle = subprocess.Popen(cmd, stderr=compat_subprocess_get_DEVNULL(), stdout=subprocess.PIPE, stdin=subprocess.PIPE)
             output = handle.communicate()[0]
             if handle.wait() != 0:
                 return None

From d9aa2b784d914ae99c7d9bbaf83d06f1b9dbd04e Mon Sep 17 00:00:00 2001
From: Ryan Schmidt <github@ryandesign.com>
Date: Sat, 14 Feb 2015 04:10:23 -0600
Subject: [PATCH 0343/6219] Support NBC Nightly News broadcasts

---
 youtube_dl/extractor/nbc.py | 27 ++++++++++++++++++++++-----
 1 file changed, 22 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index f840f6532..3e3de9e2d 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -54,7 +54,7 @@ class NBCIE(InfoExtractor):
 class NBCNewsIE(InfoExtractor):
     _VALID_URL = r'''(?x)https?://www\.nbcnews\.com/
         ((video/.+?/(?P<id>\d+))|
-        (feature/[^/]+/(?P<title>.+)))
+        ((?P<program>feature|nightly-news)/[^/]+/(?P<title>.+)))
         '''
 
     _TESTS = [
@@ -89,6 +89,16 @@ class NBCNewsIE(InfoExtractor):
                 'description': 'md5:757988edbaae9d7be1d585eb5d55cc04',
             },
         },
+        {
+            'url': 'http://www.nbcnews.com/nightly-news/video/nightly-news-with-brian-williams-full-broadcast-february-4-394064451844',
+            'md5': 'b5dda8cddd8650baa0dcb616dd2cf60d',
+            'info_dict': {
+                'id': 'sekXqyTVnmN3',
+                'ext': 'mp4',
+                'title': 'Nightly News with Brian Williams Full Broadcast (February 4)',
+                'description': 'md5:1c10c1eccbe84a26e5debb4381e2d3c5',
+            },
+        },
     ]
 
     def _real_extract(self, url):
@@ -107,12 +117,19 @@ class NBCNewsIE(InfoExtractor):
                 'thumbnail': find_xpath_attr(info, 'media', 'type', 'thumbnail').text,
             }
         else:
-            # "feature" pages use theplatform.com
+            # "feature" and "nightly-news" pages use theplatform.com
             title = mobj.group('title')
             webpage = self._download_webpage(url, title)
-            bootstrap_json = self._search_regex(
-                r'var bootstrapJson = ({.+})\s*$', webpage, 'bootstrap json',
-                flags=re.MULTILINE)
+            program = mobj.group('program')
+            if program == 'feature':
+                bootstrap_json = self._search_regex(
+                    r'var bootstrapJson = ({.+})\s*$', webpage, 'bootstrap json',
+                    flags=re.MULTILINE)
+            else:
+                # nightly-news
+                bootstrap_json = self._search_regex(
+                    r'var playlistData = ({.+});\s*$', webpage, 'playlist data',
+                    flags=re.MULTILINE)
             bootstrap = json.loads(bootstrap_json)
             info = bootstrap['results'][0]['video']
             mpxid = info['mpxId']

From a4f3d779db13ec3c8bda67e897de8cd849a7f811 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Feb 2015 17:42:12 +0600
Subject: [PATCH 0344/6219] [nbcnews] Simplify

---
 youtube_dl/extractor/nbc.py | 21 +++++++--------------
 1 file changed, 7 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index 3e3de9e2d..52e8595ea 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -52,9 +52,9 @@ class NBCIE(InfoExtractor):
 
 
 class NBCNewsIE(InfoExtractor):
-    _VALID_URL = r'''(?x)https?://www\.nbcnews\.com/
-        ((video/.+?/(?P<id>\d+))|
-        ((?P<program>feature|nightly-news)/[^/]+/(?P<title>.+)))
+    _VALID_URL = r'''(?x)https?://(?:www\.)?nbcnews\.com/
+        (?:video/.+?/(?P<id>\d+)|
+        (?:feature|nightly-news)/[^/]+/(?P<title>.+))
         '''
 
     _TESTS = [
@@ -120,17 +120,10 @@ class NBCNewsIE(InfoExtractor):
             # "feature" and "nightly-news" pages use theplatform.com
             title = mobj.group('title')
             webpage = self._download_webpage(url, title)
-            program = mobj.group('program')
-            if program == 'feature':
-                bootstrap_json = self._search_regex(
-                    r'var bootstrapJson = ({.+})\s*$', webpage, 'bootstrap json',
-                    flags=re.MULTILINE)
-            else:
-                # nightly-news
-                bootstrap_json = self._search_regex(
-                    r'var playlistData = ({.+});\s*$', webpage, 'playlist data',
-                    flags=re.MULTILINE)
-            bootstrap = json.loads(bootstrap_json)
+            bootstrap_json = self._search_regex(
+                r'var\s+(?:bootstrapJson|playlistData)\s*=\s*({.+});?\s*$',
+                webpage, 'bootstrap json', flags=re.MULTILINE)
+            bootstrap = self._parse_json(bootstrap_json, video_id)
             info = bootstrap['results'][0]['video']
             mpxid = info['mpxId']
 

From 3215c50f25bcf01a037e066747d7d60bd274ae71 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Feb 2015 17:44:24 +0600
Subject: [PATCH 0345/6219] Credit @ryandesign for nbcnews nightly news (#4948)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 3d6985ab6..47f12a9ee 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -110,3 +110,4 @@ Shaya Goldberg
 Paul Hartmann
 Frans de Jonge
 Robin de Rooij
+Ryan Schmidt

From ae1580d790ecca2b6ad132da26bc834e4f9873f3 Mon Sep 17 00:00:00 2001
From: peugeot <peugeot.com>
Date: Sat, 14 Feb 2015 13:29:44 +0100
Subject: [PATCH 0346/6219] [sunporno] fix extraction

---
 youtube_dl/extractor/sunporno.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/sunporno.py b/youtube_dl/extractor/sunporno.py
index 8a333f1d2..f2a1d6550 100644
--- a/youtube_dl/extractor/sunporno.py
+++ b/youtube_dl/extractor/sunporno.py
@@ -52,7 +52,7 @@ class SunPornoIE(InfoExtractor):
 
         formats = []
         quality = qualities(['mp4', 'flv'])
-        for video_url in re.findall(r'<source src="([^"]+)"', webpage):
+        for video_url in re.findall(r'<video src="([^"]+)"', webpage):
             video_ext = determine_ext(video_url)
             formats.append({
                 'url': video_url,

From 68f2d273bfeb7aa10eba68b6c62cc0502a948c72 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Feb 2015 18:33:52 +0600
Subject: [PATCH 0347/6219] [sunporno] Keep old video regex just in case

---
 youtube_dl/extractor/sunporno.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/sunporno.py b/youtube_dl/extractor/sunporno.py
index f2a1d6550..854d01bee 100644
--- a/youtube_dl/extractor/sunporno.py
+++ b/youtube_dl/extractor/sunporno.py
@@ -52,7 +52,7 @@ class SunPornoIE(InfoExtractor):
 
         formats = []
         quality = qualities(['mp4', 'flv'])
-        for video_url in re.findall(r'<video src="([^"]+)"', webpage):
+        for video_url in re.findall(r'<(?:source|video) src="([^"]+)"', webpage):
             video_ext = determine_ext(video_url)
             formats.append({
                 'url': video_url,

From 10e3c4c2215a7d65391d23efd19aab96197cdcf8 Mon Sep 17 00:00:00 2001
From: peugeot <peugeot.com>
Date: Sat, 14 Feb 2015 13:40:35 +0100
Subject: [PATCH 0348/6219] [drtuber] fix extraction

---
 youtube_dl/extractor/drtuber.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/drtuber.py b/youtube_dl/extractor/drtuber.py
index ca274dff6..2f06e64cc 100644
--- a/youtube_dl/extractor/drtuber.py
+++ b/youtube_dl/extractor/drtuber.py
@@ -36,7 +36,7 @@ class DrTuberIE(InfoExtractor):
             r'<source src="([^"]+)"', webpage, 'video URL')
 
         title = self._html_search_regex(
-            r'<title>([^<]+)\s*-\s*Free', webpage, 'title')
+            r'<title>([^<]+) - \d+', webpage, 'title')
 
         thumbnail = self._html_search_regex(
             r'poster="([^"]+)"',

From 52e1d0ccc415377bd9f7d862f645c87909705ace Mon Sep 17 00:00:00 2001
From: peugeot <peugeot.com>
Date: Sat, 14 Feb 2015 13:42:42 +0100
Subject: [PATCH 0349/6219] [beeg] fix test

---
 youtube_dl/extractor/beeg.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/beeg.py b/youtube_dl/extractor/beeg.py
index 4e79fea8f..b38057f2f 100644
--- a/youtube_dl/extractor/beeg.py
+++ b/youtube_dl/extractor/beeg.py
@@ -9,7 +9,7 @@ class BeegIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?beeg\.com/(?P<id>\d+)'
     _TEST = {
         'url': 'http://beeg.com/5416503',
-        'md5': '634526ae978711f6b748fe0dd6c11f57',
+        'md5': '1bff67111adb785c51d1b42959ec10e5',
         'info_dict': {
             'id': '5416503',
             'ext': 'mp4',

From 76d1466b08d4e3cebc40a7fb7dae957c3b34b1eb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Feb 2015 18:50:13 +0600
Subject: [PATCH 0350/6219] [drtuber] Add one more title regex

---
 youtube_dl/extractor/drtuber.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/drtuber.py b/youtube_dl/extractor/drtuber.py
index 2f06e64cc..37c5c181f 100644
--- a/youtube_dl/extractor/drtuber.py
+++ b/youtube_dl/extractor/drtuber.py
@@ -15,7 +15,7 @@ class DrTuberIE(InfoExtractor):
             'id': '1740434',
             'display_id': 'hot-perky-blonde-naked-golf',
             'ext': 'mp4',
-            'title': 'Hot Perky Blonde Naked Golf',
+            'title': 'hot perky blonde naked golf',
             'like_count': int,
             'dislike_count': int,
             'comment_count': int,
@@ -36,7 +36,8 @@ class DrTuberIE(InfoExtractor):
             r'<source src="([^"]+)"', webpage, 'video URL')
 
         title = self._html_search_regex(
-            r'<title>([^<]+) - \d+', webpage, 'title')
+            [r'class="hd_title" style="[^"]+">([^<]+)</h1>', r'<title>([^<]+) - \d+'],
+            webpage, 'title')
 
         thumbnail = self._html_search_regex(
             r'poster="([^"]+)"',

From a294bce82f84482091fa6f44ce913a7a86fe8be8 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Sat, 14 Feb 2015 17:48:04 +0200
Subject: [PATCH 0351/6219] [streamcz] Fix extraction (Closes #4940)

---
 youtube_dl/extractor/streamcz.py | 23 +++++++++++++++++++++--
 1 file changed, 21 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/streamcz.py b/youtube_dl/extractor/streamcz.py
index c3ceb5f76..e92b93285 100644
--- a/youtube_dl/extractor/streamcz.py
+++ b/youtube_dl/extractor/streamcz.py
@@ -1,14 +1,30 @@
 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
+import hashlib
+import time
+
 from .common import InfoExtractor
+from ..compat import (
+    compat_urllib_request,
+)
 from ..utils import (
     int_or_none,
 )
 
 
+def _get_api_key(api_path):
+    if api_path.endswith('?'):
+        api_path = api_path[:-1]
+
+    api_key = 'fb5f58a820353bd7095de526253c14fd'
+    a = '{0:}{1:}{2:}'.format(api_key, api_path, int(round(time.time() / 24 / 3600)))
+    return hashlib.md5(a.encode('ascii')).hexdigest()
+
+
 class StreamCZIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?stream\.cz/.+/(?P<id>[0-9]+)'
+    _API_URL = 'http://www.stream.cz/API'
 
     _TESTS = [{
         'url': 'http://www.stream.cz/peklonataliri/765767-ecka-pro-deti',
@@ -36,8 +52,11 @@ class StreamCZIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        data = self._download_json(
-            'http://www.stream.cz/API/episode/%s' % video_id, video_id)
+        api_path = '/episode/%s' % video_id
+
+        req = compat_urllib_request.Request(self._API_URL + api_path)
+        req.add_header('Api-Password', _get_api_key(api_path))
+        data = self._download_json(req, video_id)
 
         formats = []
         for quality, video in enumerate(data['video_qualities']):

From b0ab0fac49057ea0df3db75fb407af92deaaa453 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 14 Feb 2015 22:18:09 +0100
Subject: [PATCH 0352/6219] Remove unused imports

---
 youtube_dl/extractor/firsttv.py | 2 --
 youtube_dl/extractor/nbc.py     | 1 -
 2 files changed, 3 deletions(-)

diff --git a/youtube_dl/extractor/firsttv.py b/youtube_dl/extractor/firsttv.py
index 6e015ca16..510d4b108 100644
--- a/youtube_dl/extractor/firsttv.py
+++ b/youtube_dl/extractor/firsttv.py
@@ -1,8 +1,6 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..utils import int_or_none
 
diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index 52e8595ea..89a2845fe 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -1,7 +1,6 @@
 from __future__ import unicode_literals
 
 import re
-import json
 
 from .common import InfoExtractor
 from ..compat import (

From 6ca7732d5e792f02be030d4a9c9d101fff2f4079 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 14 Feb 2015 22:20:24 +0100
Subject: [PATCH 0353/6219] [extractor/common] Fix link to external
 documentation

---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 48742189a..c784eedb9 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -665,7 +665,7 @@ class InfoExtractor(object):
         return RATING_TABLE.get(rating.lower(), None)
 
     def _family_friendly_search(self, html):
-        # See http://schema.org/VideoObj
+        # See http://schema.org/VideoObject
         family_friendly = self._html_search_meta('isFamilyFriendly', html)
 
         if not family_friendly:

From 9fb2f1cd6d14dafca7a2d6cd74b0394b0f24afae Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 15 Feb 2015 04:56:12 +0600
Subject: [PATCH 0354/6219] [theplatform] Add URL sign capability

---
 youtube_dl/extractor/theplatform.py | 42 ++++++++++++++++++++++++++---
 1 file changed, 38 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index 110ed976d..1579822f2 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -2,6 +2,11 @@ from __future__ import unicode_literals
 
 import re
 import json
+import time
+import hmac
+import binascii
+import hashlib
+
 
 from .subtitles import SubtitlesInfoExtractor
 from ..compat import (
@@ -11,6 +16,7 @@ from ..utils import (
     determine_ext,
     ExtractorError,
     xpath_with_ns,
+    unsmuggle_url,
 )
 
 _x = lambda p: xpath_with_ns(p, {'smil': 'http://www.w3.org/2005/SMIL21/Language'})
@@ -18,7 +24,7 @@ _x = lambda p: xpath_with_ns(p, {'smil': 'http://www.w3.org/2005/SMIL21/Language
 
 class ThePlatformIE(SubtitlesInfoExtractor):
     _VALID_URL = r'''(?x)
-        (?:https?://(?:link|player)\.theplatform\.com/[sp]/[^/]+/
+        (?:https?://(?:link|player)\.theplatform\.com/[sp]/(?P<provider_id>[^/]+)/
            (?P<config>(?:[^/\?]+/(?:swf|config)|onsite)/select/)?
          |theplatform:)(?P<id>[^/\?&]+)'''
 
@@ -38,9 +44,33 @@ class ThePlatformIE(SubtitlesInfoExtractor):
         },
     }
 
+    @staticmethod
+    def _sign_url(url, sig_key, sig_secret, life=600, include_qs=False):
+        flags = '10' if include_qs else '00'
+        expiration_date = '%x' % (int(time.time()) + life)
+
+        def str_to_hex(str):
+            return binascii.b2a_hex(str.encode('ascii')).decode('ascii')
+
+        def hex_to_str(hex):
+            return binascii.a2b_hex(hex)
+
+        relative_path = url.split('http://link.theplatform.com/s/')[1].split('?')[0]
+        clear_text = hex_to_str(flags + expiration_date + str_to_hex(relative_path))
+        checksum = hmac.new(sig_key.encode('ascii'), clear_text, hashlib.sha1).hexdigest()
+        sig = flags + expiration_date + checksum + str_to_hex(sig_secret)
+        return '%s&sig=%s' % (url, sig)
+
     def _real_extract(self, url):
+        url, smuggled_data = unsmuggle_url(url, {})
+
         mobj = re.match(self._VALID_URL, url)
+        provider_id = mobj.group('provider_id')
         video_id = mobj.group('id')
+
+        if not provider_id:
+            provider_id = 'dJ5BDC'
+
         if mobj.group('config'):
             config_url = url + '&form=json'
             config_url = config_url.replace('swf/', 'config/')
@@ -48,8 +78,12 @@ class ThePlatformIE(SubtitlesInfoExtractor):
             config = self._download_json(config_url, video_id, 'Downloading config')
             smil_url = config['releaseUrl'] + '&format=SMIL&formats=MPEG4&manifest=f4m'
         else:
-            smil_url = ('http://link.theplatform.com/s/dJ5BDC/{0}/meta.smil?'
-                        'format=smil&mbr=true'.format(video_id))
+            smil_url = ('http://link.theplatform.com/s/{0}/{1}/meta.smil?'
+                        'format=smil&mbr=true'.format(provider_id, video_id))
+
+        sig = smuggled_data.get('sig')
+        if sig:
+            smil_url = self._sign_url(smil_url, sig['key'], sig['secret'])
 
         meta = self._download_xml(smil_url, video_id)
         try:
@@ -62,7 +96,7 @@ class ThePlatformIE(SubtitlesInfoExtractor):
         else:
             raise ExtractorError(error_msg, expected=True)
 
-        info_url = 'http://link.theplatform.com/s/dJ5BDC/{0}?format=preview'.format(video_id)
+        info_url = 'http://link.theplatform.com/s/{0}/{1}?format=preview'.format(provider_id, video_id)
         info_json = self._download_webpage(info_url, video_id)
         info = json.loads(info_json)
 

From b9c7a97318f4ca68cacf0ff5395c0d4a6f74b525 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 15 Feb 2015 04:57:52 +0600
Subject: [PATCH 0355/6219] [history] Add extractor (Closes #4934)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/history.py  | 31 +++++++++++++++++++++++++++++++
 2 files changed, 32 insertions(+)
 create mode 100644 youtube_dl/extractor/history.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index a4fab540b..13292073c 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -189,6 +189,7 @@ from .hellporno import HellPornoIE
 from .helsinki import HelsinkiIE
 from .hentaistigma import HentaiStigmaIE
 from .historicfilms import HistoricFilmsIE
+from .history import HistoryIE
 from .hitbox import HitboxIE, HitboxLiveIE
 from .hornbunny import HornBunnyIE
 from .hostingbulk import HostingBulkIE
diff --git a/youtube_dl/extractor/history.py b/youtube_dl/extractor/history.py
new file mode 100644
index 000000000..f86164afe
--- /dev/null
+++ b/youtube_dl/extractor/history.py
@@ -0,0 +1,31 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import smuggle_url
+
+
+class HistoryIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?history\.com/(?:[^/]+/)+(?P<id>[^/]+?)(?:$|[?#])'
+
+    _TESTS = [{
+        'url': 'http://www.history.com/topics/valentines-day/history-of-valentines-day/videos/bet-you-didnt-know-valentines-day?m=528e394da93ae&s=undefined&f=1&free=false',
+        'md5': '6fe632d033c92aa10b8d4a9be047a7c5',
+        'info_dict': {
+            'id': 'bLx5Dv5Aka1G',
+            'ext': 'mp4',
+            'title': "Bet You Didn't Know: Valentine's Day",
+            'description': 'md5:7b57ea4829b391995b405fa60bd7b5f7',
+        },
+        'add_ie': ['ThePlatform'],
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        video_url = self._search_regex(
+            r'data-href="[^"]*/%s"[^>]+data-release-url="([^"]+)"' % video_id,
+            webpage, 'video url')
+
+        return self.url_result(smuggle_url(video_url, {'sig': {'key': 'crazyjava', 'secret': 's3cr3t'}}))

From f813928e4b669627007b772c9b150eed135b18be Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 15 Feb 2015 16:32:38 +0600
Subject: [PATCH 0356/6219] [bbccouk] Fix fallback to legacy playlist

---
 youtube_dl/extractor/bbccouk.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbccouk.py b/youtube_dl/extractor/bbccouk.py
index 126c8824c..f23e39545 100644
--- a/youtube_dl/extractor/bbccouk.py
+++ b/youtube_dl/extractor/bbccouk.py
@@ -273,7 +273,7 @@ class BBCCoUkIE(SubtitlesInfoExtractor):
                     formats, subtitles = self._download_media_selector(programme_id)
                 return programme_id, title, description, duration, formats, subtitles
         except ExtractorError as ee:
-            if not isinstance(ee.cause, compat_HTTPError) and ee.cause.code == 404:
+            if not (isinstance(ee.cause, compat_HTTPError) and ee.cause.code == 404):
                 raise
 
         # fallback to legacy playlist

From 8fb474fb17a64ff2aa9f6315ebbc99ae7938c4e1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 15 Feb 2015 14:59:00 +0100
Subject: [PATCH 0357/6219] [test/subtitles] Fix some tests

The checksym for the CeskaTelevize subtitles has changed again, so we just test that it has a reasonable length.
---
 test/test_subtitles.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/test_subtitles.py b/test/test_subtitles.py
index 3e329438f..bcc69a778 100644
--- a/test/test_subtitles.py
+++ b/test/test_subtitles.py
@@ -138,7 +138,7 @@ class TestDailymotionSubtitles(BaseTestSubtitles):
         self.DL.params['writesubtitles'] = True
         self.DL.params['allsubtitles'] = True
         subtitles = self.getSubtitles()
-        self.assertEqual(len(subtitles.keys()), 5)
+        self.assertTrue(len(subtitles.keys()) >= 6)
 
     def test_list_subtitles(self):
         self.DL.expect_warning('Automatic Captions not supported by this server')
@@ -334,7 +334,7 @@ class TestCeskaTelevizeSubtitles(BaseTestSubtitles):
         self.DL.params['allsubtitles'] = True
         subtitles = self.getSubtitles()
         self.assertEqual(set(subtitles.keys()), set(['cs']))
-        self.assertEqual(md5(subtitles['cs']), 'cc3957b2c6dff1db71e5f2e83d467480')
+        self.assertTrue(len(subtitles['cs']) > 20000)
 
     def test_nosubtitles(self):
         self.DL.expect_warning('video doesn\'t have subtitles')

From 73fac4e9119bc53d49e646c38529f1b6d8de734d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 13 Feb 2015 11:14:01 +0100
Subject: [PATCH 0358/6219] [ffmpeg] Add --ffmpeg-location

---
 youtube_dl/YoutubeDL.py            |  4 +-
 youtube_dl/__init__.py             |  1 +
 youtube_dl/downloader/hls.py       |  7 +--
 youtube_dl/options.py              |  4 ++
 youtube_dl/postprocessor/ffmpeg.py | 95 ++++++++++++++++++++++--------
 5 files changed, 80 insertions(+), 31 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 13d18e25e..dbb26272d 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1298,7 +1298,7 @@ class YoutubeDL(object):
                     downloaded = []
                     success = True
                     merger = FFmpegMergerPP(self, not self.params.get('keepvideo'))
-                    if not merger._executable:
+                    if not merger.available():
                         postprocessors = []
                         self.report_warning('You have requested multiple '
                                             'formats but ffmpeg or avconv are not installed.'
@@ -1647,7 +1647,7 @@ class YoutubeDL(object):
         self._write_string('[debug] Python version %s - %s\n' % (
             platform.python_version(), platform_name()))
 
-        exe_versions = FFmpegPostProcessor.get_versions()
+        exe_versions = FFmpegPostProcessor.get_versions(self)
         exe_versions['rtmpdump'] = rtmpdump_version()
         exe_str = ', '.join(
             '%s %s' % (exe, v)
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index ed22f169f..108fb3c7a 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -350,6 +350,7 @@ def _real_main(argv=None):
         'xattr_set_filesize': opts.xattr_set_filesize,
         'match_filter': match_filter,
         'no_color': opts.no_color,
+        'ffmpeg_location': opts.ffmpeg_location,
     }
 
     with YoutubeDL(ydl_opts) as ydl:
diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index e527ee425..8be4f4249 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -23,15 +23,14 @@ class HlsFD(FileDownloader):
         tmpfilename = self.temp_name(filename)
 
         ffpp = FFmpegPostProcessor(downloader=self)
-        program = ffpp._executable
-        if program is None:
+        if not ffpp.available:
             self.report_error('m3u8 download detected but ffmpeg or avconv could not be found. Please install one.')
             return False
         ffpp.check_version()
 
         args = [
             encodeArgument(opt)
-            for opt in (program, '-y', '-i', url, '-f', 'mp4', '-c', 'copy', '-bsf:a', 'aac_adtstoasc')]
+            for opt in (ffpp.executable, '-y', '-i', url, '-f', 'mp4', '-c', 'copy', '-bsf:a', 'aac_adtstoasc')]
         args.append(encodeFilename(tmpfilename, True))
 
         retval = subprocess.call(args)
@@ -48,7 +47,7 @@ class HlsFD(FileDownloader):
             return True
         else:
             self.to_stderr('\n')
-            self.report_error('%s exited with code %d' % (program, retval))
+            self.report_error('%s exited with code %d' % (ffpp.basename, retval))
             return False
 
 
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 873432bee..ba35399cf 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -735,6 +735,10 @@ def parseOpts(overrideArguments=None):
         '--prefer-ffmpeg',
         action='store_true', dest='prefer_ffmpeg',
         help='Prefer ffmpeg over avconv for running the postprocessors')
+    postproc.add_option(
+        '--ffmpeg-location', '--avconv-location', metavar='PATH',
+        dest='ffmpeg_location',
+        help='Location of the ffmpeg/avconv binary; either the path to the binary or its containing directory.')
     postproc.add_option(
         '--exec',
         metavar='CMD', dest='exec_cmd',
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 01d25f760..504a71193 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -30,54 +30,97 @@ class FFmpegPostProcessorError(PostProcessingError):
 class FFmpegPostProcessor(PostProcessor):
     def __init__(self, downloader=None, deletetempfiles=False):
         PostProcessor.__init__(self, downloader)
-        self._versions = self.get_versions()
         self._deletetempfiles = deletetempfiles
+        self._determine_executables()
 
     def check_version(self):
-        if not self._executable:
+        if not self.available():
             raise FFmpegPostProcessorError('ffmpeg or avconv not found. Please install one.')
 
         required_version = '10-0' if self._uses_avconv() else '1.0'
         if is_outdated_version(
-                self._versions[self._executable], required_version):
+                self._versions[self.basename], required_version):
             warning = 'Your copy of %s is outdated, update %s to version %s or newer if you encounter any errors.' % (
-                self._executable, self._executable, required_version)
+                self.basename, self.basename, required_version)
             if self._downloader:
                 self._downloader.report_warning(warning)
 
     @staticmethod
-    def get_versions():
+    def get_versions(downloader=None):
+        return FFmpegPostProcessor(downloader)._versions
+
+    def _determine_executables(self):
         programs = ['avprobe', 'avconv', 'ffmpeg', 'ffprobe']
-        return dict((p, get_exe_version(p, args=['-version'])) for p in programs)
+        prefer_ffmpeg = self._downloader.params.get('prefer_ffmpeg', False)
 
-    @property
-    def available(self):
-        return self._executable is not None
+        self.basename = None
+        self.probe_basename = None
 
-    @property
-    def _executable(self):
-        if self._downloader.params.get('prefer_ffmpeg', False):
+        self._paths = None
+        self._versions = None
+        if self._downloader:
+            location = self._downloader.params.get('ffmpeg_location')
+            if location is not None:
+                if not os.path.exists(location):
+                    self._downloader.report_warning(
+                        'ffmpeg-location %s does not exist! '
+                        'Continuing without avconv/ffmpeg.' % (location))
+                    self._versions = {}
+                    return
+                elif not os.path.isdir(location):
+                    basename = os.path.splitext(os.path.basename(location))[0]
+                    if basename not in programs:
+                        self._downloader.report_warning(
+                            'Cannot identify executable %s, its basename should be one of %s. '
+                            'Continuing without avconv/ffmpeg.' %
+                            (location, ', '.join(programs)))
+                        self._versions = {}
+                        return None
+                    location = os.path.dirname(os.path.abspath(location))
+                    if basename in ('ffmpeg', 'ffprobe'):
+                        prefer_ffmpeg = True
+
+                self._paths = dict(
+                    (p, os.path.join(location, p)) for p in programs)
+                self._versions = dict(
+                    (p, get_exe_version(self._paths[p], args=['-version']))
+                    for p in programs)
+        if self._versions is None:
+            self._versions = dict(
+                (p, get_exe_version(p, args=['-version'])) for p in programs)
+            self._paths = dict((p, p) for p in programs)
+
+        if prefer_ffmpeg:
             prefs = ('ffmpeg', 'avconv')
         else:
             prefs = ('avconv', 'ffmpeg')
         for p in prefs:
             if self._versions[p]:
-                return p
-        return None
+                self.basename = p
+                break
 
-    @property
-    def _probe_executable(self):
-        if self._downloader.params.get('prefer_ffmpeg', False):
+        if prefer_ffmpeg:
             prefs = ('ffprobe', 'avprobe')
         else:
             prefs = ('avprobe', 'ffprobe')
         for p in prefs:
             if self._versions[p]:
-                return p
-        return None
+                self.probe_basename = p
+                break
+
+    def available(self):
+        return self.basename is not None
 
     def _uses_avconv(self):
-        return self._executable == 'avconv'
+        return self.basename == 'avconv'
+
+    @property
+    def executable(self):
+        return self._paths[self.basename]
+
+    @property
+    def probe_executable(self):
+        return self._paths[self.probe_basename]
 
     def run_ffmpeg_multiple_files(self, input_paths, out_path, opts):
         self.check_version()
@@ -88,7 +131,7 @@ class FFmpegPostProcessor(PostProcessor):
         files_cmd = []
         for path in input_paths:
             files_cmd.extend([encodeArgument('-i'), encodeFilename(path, True)])
-        cmd = ([encodeFilename(self._executable, True), encodeArgument('-y')] +
+        cmd = ([encodeFilename(self.executable, True), encodeArgument('-y')] +
                files_cmd +
                [encodeArgument(o) for o in opts] +
                [encodeFilename(self._ffmpeg_filename_argument(out_path), True)])
@@ -127,13 +170,15 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
 
     def get_audio_codec(self, path):
 
-        if not self._probe_executable:
+        if not self.probe_executable:
             raise PostProcessingError('ffprobe or avprobe not found. Please install one.')
         try:
             cmd = [
-                encodeFilename(self._probe_executable, True),
+                encodeFilename(self.probe_executable, True),
                 encodeArgument('-show_streams'),
                 encodeFilename(self._ffmpeg_filename_argument(path), True)]
+            if self._downloader.params.get('verbose', False):
+                self._downloader.to_screen('[debug] ffprobe command line: %s' % shell_quote(cmd))
             handle = subprocess.Popen(cmd, stderr=compat_subprocess_get_DEVNULL(), stdout=subprocess.PIPE)
             output = handle.communicate()[0]
             if handle.wait() != 0:
@@ -223,14 +268,14 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
             if self._nopostoverwrites and os.path.exists(encodeFilename(new_path)):
                 self._downloader.to_screen('[youtube] Post-process file %s exists, skipping' % new_path)
             else:
-                self._downloader.to_screen('[' + self._executable + '] Destination: ' + new_path)
+                self._downloader.to_screen('[' + self.basename + '] Destination: ' + new_path)
                 self.run_ffmpeg(path, new_path, acodec, more_opts)
         except:
             etype, e, tb = sys.exc_info()
             if isinstance(e, AudioConversionError):
                 msg = 'audio conversion failed: ' + e.msg
             else:
-                msg = 'error running ' + self._executable
+                msg = 'error running ' + self.basename
             raise PostProcessingError(msg)
 
         # Try to update the date time for extracted audio file.

From 03cd72b0070c430e95b7e9d612afcad026690153 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 16 Feb 2015 04:37:55 +0100
Subject: [PATCH 0359/6219] [extractor/common] Move up filesize

filesize and tbr should correlate, so it doesn't make sense to treat them differently.
---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index c784eedb9..e74b7bf25 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -729,6 +729,7 @@ class InfoExtractor(object):
                 f.get('language_preference') if f.get('language_preference') is not None else -1,
                 f.get('quality') if f.get('quality') is not None else -1,
                 f.get('tbr') if f.get('tbr') is not None else -1,
+                f.get('filesize') if f.get('filesize') is not None else -1,
                 f.get('vbr') if f.get('vbr') is not None else -1,
                 f.get('height') if f.get('height') is not None else -1,
                 f.get('width') if f.get('width') is not None else -1,
@@ -736,7 +737,6 @@ class InfoExtractor(object):
                 f.get('abr') if f.get('abr') is not None else -1,
                 audio_ext_preference,
                 f.get('fps') if f.get('fps') is not None else -1,
-                f.get('filesize') if f.get('filesize') is not None else -1,
                 f.get('filesize_approx') if f.get('filesize_approx') is not None else -1,
                 f.get('source_preference') if f.get('source_preference') is not None else -1,
                 f.get('format_id'),

From 85e80f71cddafe956886968d0eb60eff0aa7432c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 16 Feb 2015 04:50:57 +0100
Subject: [PATCH 0360/6219] [yam] Allow faults in optional fields (#4943)

---
 youtube_dl/extractor/yam.py | 37 ++++++++++++++++++++++++-------------
 1 file changed, 24 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/yam.py b/youtube_dl/extractor/yam.py
index 511c8b67c..b294767c5 100644
--- a/youtube_dl/extractor/yam.py
+++ b/youtube_dl/extractor/yam.py
@@ -1,10 +1,14 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..compat import compat_urlparse
-from ..utils import month_by_abbreviation
-import re
+from ..utils import (
+    float_or_none,
+    month_by_abbreviation,
+)
 
 
 class YamIE(InfoExtractor):
@@ -38,8 +42,8 @@ class YamIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        media_id = self._match_id(url)
-        page = self._download_webpage(url, media_id)
+        video_id = self._match_id(url)
+        page = self._download_webpage(url, video_id)
 
         # Is it hosted externally on YouTube?
         youtube_url = self._html_search_regex(
@@ -49,22 +53,29 @@ class YamIE(InfoExtractor):
             return self.url_result(youtube_url, 'Youtube')
 
         api_page = self._download_webpage(
-            'http://mymedia.yam.com/api/a/?pID=' + media_id, media_id)
+            'http://mymedia.yam.com/api/a/?pID=' + video_id, video_id,
+            note='Downloading API page')
         api_result_obj = compat_urlparse.parse_qs(api_page)
 
-        author = self._html_search_regex(
-            r'<!-- 發表作者 -->：[\n ]+<a href="/([a-z]+)"', page, 'author')
+        uploader_id = self._html_search_regex(
+            r'<!-- 發表作者 -->：[\n ]+<a href="/([a-z]+)"',
+            page, 'uploader id', fatal=False)
         mobj = re.search(r'<!-- 發表於 -->(?P<mon>[A-Z][a-z]{2})  ' +
                          r'(?P<day>\d{1,2}), (?P<year>\d{4})', page)
-        upload_date = '%s%02d%02d' % (mobj.group('year'),
-                                      month_by_abbreviation(mobj.group('mon')),
-                                      int(mobj.group('day')))
+        if mobj:
+            upload_date = '%s%02d%02d' % (
+                mobj.group('year'),
+                month_by_abbreviation(mobj.group('mon')),
+                int(mobj.group('day')))
+        else:
+            upload_date = None
+        duration = float_or_none(api_result_obj['totaltime'][0], scale=1000)
 
         return {
-            'id': media_id,
+            'id': video_id,
             'url': api_result_obj['mp3file'][0],
             'title': self._html_search_meta('description', page),
-            'duration': float(api_result_obj['totaltime'][0]) / 1000.0,
-            'uploader_id': author,
+            'duration': duration,
+            'uploader_id': uploader_id,
             'upload_date': upload_date,
         }

From a5681804415a9b8d011cd238b7d61e6eb4ec7c4b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 16 Feb 2015 04:51:20 +0100
Subject: [PATCH 0361/6219] release 2015.02.16

---
 README.md              | 3 +++
 docs/supportedsites.md | 8 +++++++-
 youtube_dl/version.py  | 2 +-
 3 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 731cea1e1..d68896202 100644
--- a/README.md
+++ b/README.md
@@ -397,6 +397,9 @@ which means you can modify it, redistribute it or use it however you like.
                                      postprocessors (default)
     --prefer-ffmpeg                  Prefer ffmpeg over avconv for running the
                                      postprocessors
+    --ffmpeg-location PATH           Location of the ffmpeg/avconv binary;
+                                     either the path to the binary or its
+                                     containing directory.
     --exec CMD                       Execute a command on the file after
                                      downloading, similar to find's -exec
                                      syntax. Example: --exec 'adb push {}
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 8bce8fede..894fa4323 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -1,4 +1,5 @@
 # Supported sites
+ - **1tv**: Первый канал
  - **1up.com**
  - **220.ro**
  - **24video**
@@ -60,6 +61,8 @@
  - **Brightcove**
  - **BuzzFeed**
  - **BYUtv**
+ - **Camdemy**
+ - **CamdemyFolder**
  - **Canal13cl**
  - **canalc2.tv**
  - **Canalplus**: canalplus.fr, piwiplus.fr and d8.tv
@@ -134,7 +137,6 @@
  - **fernsehkritik.tv:postecke**
  - **Firedrive**
  - **Firstpost**
- - **firsttv**: Видеоархив - Первый канал
  - **Flickr**
  - **Folketinget**: Folketinget (ft.dk; Danish parliament)
  - **Foxgay**
@@ -174,6 +176,7 @@
  - **Helsinki**: helsinki.fi
  - **HentaiStigma**
  - **HistoricFilms**
+ - **History**
  - **hitbox**
  - **hitbox:live**
  - **HornBunny**
@@ -287,6 +290,8 @@
  - **nowvideo**: NowVideo
  - **npo.nl**
  - **npo.nl:live**
+ - **npo.nl:radio**
+ - **npo.nl:radio:fragment**
  - **NRK**
  - **NRKTV**
  - **ntv.ru**
@@ -527,6 +532,7 @@
  - **XVideos**
  - **XXXYMovies**
  - **Yahoo**: Yahoo screen and movies
+ - **Yam**
  - **YesJapan**
  - **Ynet**
  - **YouJizz**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 492ddf2ea..e58c59969 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.11'
+__version__ = '2015.02.16'

From 59b8ab583491410f4f365b1cb268a4360d7be877 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 16 Feb 2015 15:45:01 +0100
Subject: [PATCH 0362/6219] [rtlnl|generic] Add support for rtl.nl embeds
 (Fixes #4959)

---
 youtube_dl/extractor/__init__.py |  2 +-
 youtube_dl/extractor/generic.py  | 17 ++++++++-
 youtube_dl/extractor/rtlnl.py    | 63 ++++++++++++++++++++++++--------
 3 files changed, 65 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 5b776a7a1..7d0909b5d 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -371,7 +371,7 @@ from .rottentomatoes import RottenTomatoesIE
 from .roxwel import RoxwelIE
 from .rtbf import RTBFIE
 from .rte import RteIE
-from .rtlnl import RtlXlIE
+from .rtlnl import RtlNlIE
 from .rtlnow import RTLnowIE
 from .rtl2 import RTL2IE
 from .rtp import RTPIE
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index f4500e931..c4ba25a96 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -537,6 +537,15 @@ class GenericIE(InfoExtractor):
                 'uploader_id': 'NationalArchives08',
                 'title': 'Webinar: Using Discovery, The National Archives’ online catalogue',
             },
+        },
+        # rtl.nl embed
+        {
+            'url': 'http://www.rtlnieuws.nl/nieuws/buitenland/aanslagen-kopenhagen',
+            'playlist_mincount': 5,
+            'info_dict': {
+                'id': 'aanslagen-kopenhagen',
+                'title': 'Aanslagen Kopenhagen | RTL Nieuws',
+            }
         }
     ]
 
@@ -782,6 +791,13 @@ class GenericIE(InfoExtractor):
                 'entries': entries,
             }
 
+        # Look for embedded rtl.nl player
+        matches = re.findall(
+            r'<iframe\s+(?:[a-zA-Z-]+="[^"]+"\s+)*?src="((?:https?:)?//(?:www\.)?rtl\.nl/system/videoplayer/[^"]+video_embed[^"]+)"',
+            webpage)
+        if matches:
+            return _playlist_from_matches(matches, ie='RtlNl')
+
         # Look for embedded (iframe) Vimeo player
         mobj = re.search(
             r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//player\.vimeo\.com/video/.+?)\1', webpage)
@@ -789,7 +805,6 @@ class GenericIE(InfoExtractor):
             player_url = unescapeHTML(mobj.group('url'))
             surl = smuggle_url(player_url, {'Referer': url})
             return self.url_result(surl)
-
         # Look for embedded (swf embed) Vimeo player
         mobj = re.search(
             r'<embed[^>]+?src="((?:https?:)?//(?:www\.)?vimeo\.com/moogaloop\.swf.+?)"', webpage)
diff --git a/youtube_dl/extractor/rtlnl.py b/youtube_dl/extractor/rtlnl.py
index a3ca79f2c..cfce4550a 100644
--- a/youtube_dl/extractor/rtlnl.py
+++ b/youtube_dl/extractor/rtlnl.py
@@ -1,16 +1,25 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
-from ..utils import parse_duration
+from ..utils import (
+    int_or_none,
+    parse_duration,
+)
 
 
-class RtlXlIE(InfoExtractor):
-    IE_NAME = 'rtlxl.nl'
-    _VALID_URL = r'https?://(www\.)?rtlxl\.nl/#!/[^/]+/(?P<uuid>[^/?]+)'
+class RtlNlIE(InfoExtractor):
+    IE_NAME = 'rtl.nl'
+    IE_DESC = 'rtl.nl and rtlxl.nl'
+    _VALID_URL = r'''(?x)
+        https?://(www\.)?
+        (?:
+            rtlxl\.nl/\#!/[^/]+/|
+            rtl\.nl/system/videoplayer/[^?#]+?/video_embed\.html\#uuid=
+        )
+        (?P<id>[0-9a-f-]+)'''
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.rtlxl.nl/#!/rtl-nieuws-132237/6e4203a6-0a5e-3596-8424-c599a59e0677',
         'md5': 'cc16baa36a6c169391f0764fa6b16654',
         'info_dict': {
@@ -22,21 +31,30 @@ class RtlXlIE(InfoExtractor):
             'upload_date': '20140814',
             'duration': 576.880,
         },
-    }
+    }, {
+        'url': 'http://www.rtl.nl/system/videoplayer/derden/rtlnieuws/video_embed.html#uuid=84ae5571-ac25-4225-ae0c-ef8d9efb2aed/autoplay=false',
+        'md5': 'dea7474214af1271d91ef332fb8be7ea',
+        'info_dict': {
+            'id': '84ae5571-ac25-4225-ae0c-ef8d9efb2aed',
+            'ext': 'mp4',
+            'timestamp': 1424039400,
+            'title': 'RTL Nieuws - Nieuwe beelden Kopenhagen: chaos direct na aanslag',
+            'thumbnail': 're:^https?://screenshots\.rtl\.nl/system/thumb/sz=[0-9]+x[0-9]+/uuid=84ae5571-ac25-4225-ae0c-ef8d9efb2aed$',
+            'upload_date': '20150215',
+            'description': 'Er zijn nieuwe beelden vrijgegeven die vlak na de aanslag in Kopenhagen zijn gemaakt. Op de video is goed te zien hoe omstanders zich bekommeren om één van de slachtoffers, terwijl de eerste agenten ter plaatse komen.',
+        }
+    }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        uuid = mobj.group('uuid')
-
+        uuid = self._match_id(url)
         info = self._download_json(
             'http://www.rtl.nl/system/s4m/vfd/version=2/uuid=%s/fmt=flash/' % uuid,
             uuid)
 
         material = info['material'][0]
-        episode_info = info['episodes'][0]
-
         progname = info['abstracts'][0]['name']
         subtitle = material['title'] or info['episodes'][0]['name']
+        description = material.get('synopsis') or info['episodes'][0]['synopsis']
 
         # Use unencrypted m3u8 streams (See https://github.com/rg3/youtube-dl/issues/4118)
         videopath = material['videopath'].replace('.f4m', '.m3u8')
@@ -58,14 +76,29 @@ class RtlXlIE(InfoExtractor):
                 'quality': 0,
             }
         ])
-
         self._sort_formats(formats)
 
+        thumbnails = []
+        meta = info.get('meta', {})
+        for p in ('poster_base_url', '"thumb_base_url"'):
+            if not meta.get(p):
+                continue
+
+            thumbnails.append({
+                'url': self._proto_relative_url(meta[p] + uuid),
+                'width': int_or_none(self._search_regex(
+                    r'/sz=([0-9]+)', meta[p], 'thumbnail width', fatal=False)),
+                'height': int_or_none(self._search_regex(
+                    r'/sz=[0-9]+x([0-9]+)',
+                    meta[p], 'thumbnail height', fatal=False))
+            })
+
         return {
             'id': uuid,
             'title': '%s - %s' % (progname, subtitle),
             'formats': formats,
             'timestamp': material['original_date'],
-            'description': episode_info['synopsis'],
+            'description': description,
             'duration': parse_duration(material.get('duration')),
+            'thumbnails': thumbnails,
         }

From 1db5fbcfe3ef95654aa642ebe483fb070852cb49 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 16 Feb 2015 15:47:13 +0100
Subject: [PATCH 0363/6219] release 2015.02.16.1

---
 docs/supportedsites.md | 2 +-
 youtube_dl/version.py  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 894fa4323..4bb68fdc5 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -338,9 +338,9 @@
  - **Roxwel**
  - **RTBF**
  - **Rte**
+ - **rtl.nl**: rtl.nl and rtlxl.nl
  - **RTL2**
  - **RTLnow**
- - **rtlxl.nl**
  - **RTP**
  - **RTS**: RTS.ch
  - **rtve.es:alacarta**: RTVE a la carta
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index e58c59969..bbe0e53b5 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.16'
+__version__ = '2015.02.16.1'

From a504ced097e703a9bc6c18b6e31bcafb4783ed80 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 15 Feb 2015 18:03:41 +0100
Subject: [PATCH 0364/6219] Improve subtitles support

For each language the extractor builds a list with the available formats sorted (like for video formats), then YoutubeDL selects one of them using the '--sub-format' option which now allows giving the format preferences (for example 'ass/srt/best').
For each format the 'url' field can be set so that we only download the contents if needed, or if the contents needs to be processed (like in crunchyroll) the 'data' field can be used.

The reasons for this change are:
* We weren't checking that the format given with '--sub-format' was available, checking it in each extractor would be repetitive.
* It allows to easily support giving a format preference.
* The subtitles were automatically downloaded in the extractor, but I think that if you use for example the '--dump-json' option you want to finish as fast as possible.

Currently only the ted extractor has been updated, but the old system still works.
---
 test/test_subtitles.py             | 24 ++++-----
 youtube_dl/YoutubeDL.py            | 85 +++++++++++++++++++++++++++---
 youtube_dl/__init__.py             |  1 -
 youtube_dl/extractor/common.py     | 20 ++++++-
 youtube_dl/extractor/ted.py        | 18 ++++---
 youtube_dl/options.py              |  4 +-
 youtube_dl/postprocessor/ffmpeg.py |  6 +--
 7 files changed, 121 insertions(+), 37 deletions(-)

diff --git a/test/test_subtitles.py b/test/test_subtitles.py
index bcc69a778..fbc9eaf4d 100644
--- a/test/test_subtitles.py
+++ b/test/test_subtitles.py
@@ -27,15 +27,23 @@ class BaseTestSubtitles(unittest.TestCase):
 
     def setUp(self):
         self.DL = FakeYDL()
-        self.ie = self.IE(self.DL)
+        self.ie = self.IE()
+        self.DL.add_info_extractor(self.ie)
 
     def getInfoDict(self):
-        info_dict = self.ie.extract(self.url)
+        info_dict = self.DL.extract_info(self.url, download=False)
         return info_dict
 
     def getSubtitles(self):
         info_dict = self.getInfoDict()
-        return info_dict['subtitles']
+        subtitles = info_dict['subtitles']
+        if not subtitles:
+            return subtitles
+        for sub_info in subtitles.values():
+            if sub_info.get('data') is None:
+                uf = self.DL.urlopen(sub_info['url'])
+                sub_info['data'] = uf.read().decode('utf-8')
+        return dict((l, sub_info['data']) for l, sub_info in subtitles.items())
 
 
 class TestYoutubeSubtitles(BaseTestSubtitles):
@@ -176,7 +184,7 @@ class TestTedSubtitles(BaseTestSubtitles):
 
     def test_no_writesubtitles(self):
         subtitles = self.getSubtitles()
-        self.assertEqual(subtitles, None)
+        self.assertFalse(subtitles)
 
     def test_subtitles(self):
         self.DL.params['writesubtitles'] = True
@@ -196,18 +204,10 @@ class TestTedSubtitles(BaseTestSubtitles):
         self.assertTrue(len(subtitles.keys()) >= 28)
 
     def test_list_subtitles(self):
-        self.DL.expect_warning('Automatic Captions not supported by this server')
         self.DL.params['listsubtitles'] = True
         info_dict = self.getInfoDict()
         self.assertEqual(info_dict, None)
 
-    def test_automatic_captions(self):
-        self.DL.expect_warning('Automatic Captions not supported by this server')
-        self.DL.params['writeautomaticsub'] = True
-        self.DL.params['subtitleslang'] = ['en']
-        subtitles = self.getSubtitles()
-        self.assertTrue(len(subtitles.keys()) == 0)
-
     def test_multiple_langs(self):
         self.DL.params['writesubtitles'] = True
         langs = ['es', 'fr', 'de']
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 13d18e25e..e665e3d53 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -154,7 +154,7 @@ class YoutubeDL(object):
     allsubtitles:      Downloads all the subtitles of the video
                        (requires writesubtitles or writeautomaticsub)
     listsubtitles:     Lists all available subtitles for the video
-    subtitlesformat:   Subtitle format [srt/sbv/vtt] (default=srt)
+    subtitlesformat:   The format code for subtitles
     subtitleslangs:    List of languages of the subtitles to download
     keepvideo:         Keep the video file after post-processing
     daterange:         A DateRange object, download only if the upload_date is in the range.
@@ -1019,6 +1019,11 @@ class YoutubeDL(object):
                 info_dict['timestamp'])
             info_dict['upload_date'] = upload_date.strftime('%Y%m%d')
 
+        if self.params.get('listsubtitles', False):
+            self.list_subtitles(info_dict['id'], info_dict.get('subtitles'))
+            return
+        info_dict['subtitles'] = self.process_subtitles(info_dict['id'], info_dict.get('subtitles'))
+
         # This extractors handle format selection themselves
         if info_dict['extractor'] in ['Youku']:
             if download:
@@ -1147,6 +1152,53 @@ class YoutubeDL(object):
         info_dict.update(formats_to_download[-1])
         return info_dict
 
+    def process_subtitles(self, video_id, available_subs):
+        """Select the requested subtitles and their format"""
+        if not available_subs:
+            return available_subs
+
+        if self.params.get('allsubtitles', False):
+            requested_langs = available_subs.keys()
+        else:
+            if self.params.get('subtitleslangs', False):
+                requested_langs = self.params.get('subtitleslangs')
+            elif 'en' in available_subs:
+                requested_langs = ['en']
+            else:
+                requested_langs = [list(available_subs.keys())[0]]
+
+        formats_query = self.params.get('subtitlesformat', 'best')
+        formats_preference = formats_query.split('/') if formats_query else []
+        subs = {}
+        for lang in requested_langs:
+            formats = available_subs.get(lang)
+            if formats is None:
+                self.report_warning('%s subtitles not available for %s' % (lang, video_id))
+                continue
+            if isinstance(formats, compat_str):
+                # TODO: convert all IE with subtitles support to the new format
+                # and remove this
+                subs[lang] = {
+                    'ext': formats_preference[0],
+                    'data': formats,
+                }
+                continue
+            for ext in formats_preference:
+                if ext == 'best':
+                    f = formats[-1]
+                    break
+                matches = list(filter(lambda f: f['ext'] == ext, formats))
+                if matches:
+                    f = matches[-1]
+                    break
+            else:
+                f = formats[-1]
+                self.report_warning(
+                    'No subtitle format found matching "%s" for language %s, '
+                    'using %s' % (formats_query, lang, f['ext']))
+            subs[lang] = f
+        return subs
+
     def process_info(self, info_dict):
         """Process a single resolved IE result."""
 
@@ -1253,11 +1305,18 @@ class YoutubeDL(object):
             # subtitles download errors are already managed as troubles in relevant IE
             # that way it will silently go on when used with unsupporting IE
             subtitles = info_dict['subtitles']
-            sub_format = self.params.get('subtitlesformat', 'srt')
-            for sub_lang in subtitles.keys():
-                sub = subtitles[sub_lang]
-                if sub is None:
-                    continue
+            for sub_lang, sub_info in subtitles.items():
+                sub_format = sub_info['ext']
+                if sub_info.get('data') is not None:
+                    sub_data = sub_info['data']
+                else:
+                    try:
+                        uf = self.urlopen(sub_info['url'])
+                        sub_data = uf.read().decode('utf-8')
+                    except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
+                        self.report_warning('Unable to download subtitle for "%s": %s' %
+                                            (sub_lang, compat_str(err)))
+                        continue
                 try:
                     sub_filename = subtitles_filename(filename, sub_lang, sub_format)
                     if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(sub_filename)):
@@ -1265,7 +1324,7 @@ class YoutubeDL(object):
                     else:
                         self.to_screen('[info] Writing video subtitles to: ' + sub_filename)
                         with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile:
-                            subfile.write(sub)
+                            subfile.write(sub_data)
                 except (OSError, IOError):
                     self.report_error('Cannot write subtitles file ' + sub_filename)
                     return
@@ -1586,6 +1645,18 @@ class YoutubeDL(object):
             ['ID', 'width', 'height', 'URL'],
             [[t['id'], t.get('width', 'unknown'), t.get('height', 'unknown'), t['url']] for t in thumbnails]))
 
+    def list_subtitles(self, video_id, subtitles):
+        if not subtitles:
+            self.to_screen('%s has no subtitles' % video_id)
+            return
+        header_line = 'Language    formats'
+        sub_lines = [
+            '%-12s%s' % (lang, ', '.join(f['ext'] for f in reversed(formats)))
+            for lang, formats in subtitles.items()]
+        self.to_screen(
+            'Available subtitles for %s:\n%s\n%s' %
+            (video_id, header_line, '\n'.join(sub_lines)))
+
     def urlopen(self, req):
         """ Start an HTTP download """
 
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index ed22f169f..5f2585003 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -226,7 +226,6 @@ def _real_main(argv=None):
     if opts.embedsubtitles:
         postprocessors.append({
             'key': 'FFmpegEmbedSubtitle',
-            'subtitlesformat': opts.subtitlesformat,
         })
     if opts.xattrs:
         postprocessors.append({'key': 'XAttrMetadata'})
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index c784eedb9..161c623eb 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -151,8 +151,14 @@ class InfoExtractor(object):
                     If not explicitly set, calculated from timestamp.
     uploader_id:    Nickname or id of the video uploader.
     location:       Physical location where the video was filmed.
-    subtitles:      The subtitle file contents as a dictionary in the format
-                    {language: subtitles}.
+    subtitles:      The available subtitles as a dictionary in the format
+                    {language: subformats}. "subformats" is a list sorted from
+                    lower to higher preference, each element is a dictionary
+                    with the "ext" entry and one of:
+                        * "data": The subtitles file contents
+                        * "url": A url pointing to the subtitles file
+                    Note: YoutubeDL.extract_info will get the requested
+                    format and replace the "subformats" list with it.
     duration:       Length of the video in seconds, as an integer.
     view_count:     How many users have watched the video on the platform.
     like_count:     Number of positive ratings of the video
@@ -993,6 +999,16 @@ class InfoExtractor(object):
             any_restricted = any_restricted or is_restricted
         return not any_restricted
 
+    def extract_subtitles(self, *args, **kwargs):
+        subtitles = {}
+        list_subtitles = self._downloader.params.get('listsubtitles')
+        if self._downloader.params.get('writesubtitles', False) or list_subtitles:
+            subtitles.update(self._get_subtitles(*args, **kwargs))
+        return subtitles
+
+    def _get_subtitles(self, *args, **kwargs):
+        raise NotImplementedError("This method must be implemented by subclasses")
+
 
 class SearchInfoExtractor(InfoExtractor):
     """
diff --git a/youtube_dl/extractor/ted.py b/youtube_dl/extractor/ted.py
index 10b3b706a..1809eaae4 100644
--- a/youtube_dl/extractor/ted.py
+++ b/youtube_dl/extractor/ted.py
@@ -3,14 +3,14 @@ from __future__ import unicode_literals
 import json
 import re
 
-from .subtitles import SubtitlesInfoExtractor
+from .common import InfoExtractor
 
 from ..compat import (
     compat_str,
 )
 
 
-class TEDIE(SubtitlesInfoExtractor):
+class TEDIE(InfoExtractor):
     _VALID_URL = r'''(?x)
         (?P<proto>https?://)
         (?P<type>www|embed(?:-ssl)?)(?P<urlmain>\.ted\.com/
@@ -165,9 +165,6 @@ class TEDIE(SubtitlesInfoExtractor):
         video_id = compat_str(talk_info['id'])
         # subtitles
         video_subtitles = self.extract_subtitles(video_id, talk_info)
-        if self._downloader.params.get('listsubtitles', False):
-            self._list_available_subtitles(video_id, talk_info)
-            return
 
         thumbnail = talk_info['thumb']
         if not thumbnail.startswith('http'):
@@ -183,13 +180,18 @@ class TEDIE(SubtitlesInfoExtractor):
             'duration': talk_info.get('duration'),
         }
 
-    def _get_available_subtitles(self, video_id, talk_info):
+    def _get_subtitles(self, video_id, talk_info):
         languages = [lang['languageCode'] for lang in talk_info.get('languages', [])]
         if languages:
             sub_lang_list = {}
             for l in languages:
-                url = 'http://www.ted.com/talks/subtitles/id/%s/lang/%s/format/srt' % (video_id, l)
-                sub_lang_list[l] = url
+                sub_lang_list[l] = [
+                    {
+                        'url': 'http://www.ted.com/talks/subtitles/id/%s/lang/%s/format/%s' % (video_id, l, ext),
+                        'ext': ext,
+                    }
+                    for ext in ['ted', 'srt']
+                ]
             return sub_lang_list
         else:
             self._downloader.report_warning('video doesn\'t have subtitles')
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 873432bee..4fcf8c83d 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -387,8 +387,8 @@ def parseOpts(overrideArguments=None):
         help='lists all available subtitles for the video')
     subtitles.add_option(
         '--sub-format',
-        action='store', dest='subtitlesformat', metavar='FORMAT', default='srt',
-        help='subtitle format (default=srt) ([sbv/vtt] youtube only)')
+        action='store', dest='subtitlesformat', metavar='FORMAT', default='best',
+        help='subtitle format, accepts formats preference, for example: "ass/srt/best"')
     subtitles.add_option(
         '--sub-lang', '--sub-langs', '--srt-lang',
         action='callback', dest='subtitleslangs', metavar='LANGS', type='str',
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 5238ce534..d1bbfbfe3 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -453,10 +453,6 @@ class FFmpegEmbedSubtitlePP(FFmpegPostProcessor):
         'zu': 'zul',
     }
 
-    def __init__(self, downloader=None, subtitlesformat='srt'):
-        super(FFmpegEmbedSubtitlePP, self).__init__(downloader)
-        self._subformat = subtitlesformat
-
     @classmethod
     def _conver_lang_code(cls, code):
         """Convert language code from ISO 639-1 to ISO 639-2/T"""
@@ -472,7 +468,7 @@ class FFmpegEmbedSubtitlePP(FFmpegPostProcessor):
 
         sub_langs = [key for key in information['subtitles']]
         filename = information['filepath']
-        input_files = [filename] + [subtitles_filename(filename, lang, self._subformat) for lang in sub_langs]
+        input_files = [filename] + [subtitles_filename(filename, lang, sub_info['ext']) for lang, sub_info in information['subtitles'].items()]
 
         opts = [
             '-map', '0',

From b5857f62e2c2ca70316e041212aa9e89d54cc253 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 15 Feb 2015 18:21:42 +0100
Subject: [PATCH 0365/6219] [crunchyroll] Convert to new subtitles system

---
 youtube_dl/extractor/crunchyroll.py | 66 +++++++++++++++--------------
 1 file changed, 35 insertions(+), 31 deletions(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 1680f532f..f1da7d09b 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -9,7 +9,7 @@ import xml.etree.ElementTree
 
 from hashlib import sha1
 from math import pow, sqrt, floor
-from .subtitles import SubtitlesInfoExtractor
+from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse,
     compat_urllib_request,
@@ -25,10 +25,9 @@ from ..aes import (
     aes_cbc_decrypt,
     inc,
 )
-from .common import InfoExtractor
 
 
-class CrunchyrollIE(SubtitlesInfoExtractor):
+class CrunchyrollIE(InfoExtractor):
     _VALID_URL = r'https?://(?:(?P<prefix>www|m)\.)?(?P<url>crunchyroll\.(?:com|fr)/(?:[^/]*/[^/?&]*?|media/\?id=)(?P<video_id>[0-9]+))(?:[/?&]|$)'
     _TESTS = [{
         'url': 'http://www.crunchyroll.com/wanna-be-the-strongest-in-the-world/episode-1-an-idol-wrestler-is-born-645513',
@@ -187,6 +186,38 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
 
         return output
 
+    def _get_subtitles(self, video_id, webpage):
+        subtitles = {}
+        for sub_id, sub_name in re.findall(r'\?ssid=([0-9]+)" title="([^"]+)', webpage):
+            sub_page = self._download_webpage(
+                'http://www.crunchyroll.com/xml/?req=RpcApiSubtitle_GetXml&subtitle_script_id=' + sub_id,
+                video_id, note='Downloading subtitles for ' + sub_name)
+            id = self._search_regex(r'id=\'([0-9]+)', sub_page, 'subtitle_id', fatal=False)
+            iv = self._search_regex(r'<iv>([^<]+)', sub_page, 'subtitle_iv', fatal=False)
+            data = self._search_regex(r'<data>([^<]+)', sub_page, 'subtitle_data', fatal=False)
+            if not id or not iv or not data:
+                continue
+            id = int(id)
+            iv = base64.b64decode(iv)
+            data = base64.b64decode(data)
+
+            subtitle = self._decrypt_subtitles(data, iv, id).decode('utf-8')
+            lang_code = self._search_regex(r'lang_code=["\']([^"\']+)', subtitle, 'subtitle_lang_code', fatal=False)
+            if not lang_code:
+                continue
+            sub_root = xml.etree.ElementTree.fromstring(subtitle)
+            subtitles[lang_code] = [
+                {
+                    'ext': 'srt',
+                    'data': self._convert_subtitles_to_srt(sub_root),
+                },
+                {
+                    'ext': 'ass',
+                    'data': self._convert_subtitles_to_ass(sub_root),
+                },
+            ]
+        return subtitles
+
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('video_id')
@@ -249,34 +280,7 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
                 'format_id': video_format,
             })
 
-        subtitles = {}
-        sub_format = self._downloader.params.get('subtitlesformat', 'srt')
-        for sub_id, sub_name in re.findall(r'\?ssid=([0-9]+)" title="([^"]+)', webpage):
-            sub_page = self._download_webpage(
-                'http://www.crunchyroll.com/xml/?req=RpcApiSubtitle_GetXml&subtitle_script_id=' + sub_id,
-                video_id, note='Downloading subtitles for ' + sub_name)
-            id = self._search_regex(r'id=\'([0-9]+)', sub_page, 'subtitle_id', fatal=False)
-            iv = self._search_regex(r'<iv>([^<]+)', sub_page, 'subtitle_iv', fatal=False)
-            data = self._search_regex(r'<data>([^<]+)', sub_page, 'subtitle_data', fatal=False)
-            if not id or not iv or not data:
-                continue
-            id = int(id)
-            iv = base64.b64decode(iv)
-            data = base64.b64decode(data)
-
-            subtitle = self._decrypt_subtitles(data, iv, id).decode('utf-8')
-            lang_code = self._search_regex(r'lang_code=["\']([^"\']+)', subtitle, 'subtitle_lang_code', fatal=False)
-            if not lang_code:
-                continue
-            sub_root = xml.etree.ElementTree.fromstring(subtitle)
-            if sub_format == 'ass':
-                subtitles[lang_code] = self._convert_subtitles_to_ass(sub_root)
-            else:
-                subtitles[lang_code] = self._convert_subtitles_to_srt(sub_root)
-
-        if self._downloader.params.get('listsubtitles', False):
-            self._list_available_subtitles(video_id, subtitles)
-            return
+        subtitles = self.extract_subtitles(video_id, webpage)
 
         return {
             'id': video_id,

From 6b597516c12c7fd81e832f3ec05dd0dca6089823 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 15 Feb 2015 18:32:40 +0100
Subject: [PATCH 0366/6219] [atresplayer] Convert to new subtitles system

---
 youtube_dl/extractor/atresplayer.py | 19 +++++++++----------
 1 file changed, 9 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/atresplayer.py b/youtube_dl/extractor/atresplayer.py
index f016368fa..7669e0e3d 100644
--- a/youtube_dl/extractor/atresplayer.py
+++ b/youtube_dl/extractor/atresplayer.py
@@ -3,7 +3,7 @@ from __future__ import unicode_literals
 import time
 import hmac
 
-from .subtitles import SubtitlesInfoExtractor
+from .common import InfoExtractor
 from ..compat import (
     compat_str,
     compat_urllib_parse,
@@ -17,7 +17,7 @@ from ..utils import (
 )
 
 
-class AtresPlayerIE(SubtitlesInfoExtractor):
+class AtresPlayerIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?atresplayer\.com/television/[^/]+/[^/]+/[^/]+/(?P<id>.+?)_\d+\.html'
     _TESTS = [
         {
@@ -144,13 +144,12 @@ class AtresPlayerIE(SubtitlesInfoExtractor):
         thumbnail = xpath_text(episode, './media/asset/files/background', 'thumbnail')
 
         subtitles = {}
-        subtitle = xpath_text(episode, './media/asset/files/subtitle', 'subtitle')
-        if subtitle:
-            subtitles['es'] = subtitle
-
-        if self._downloader.params.get('listsubtitles', False):
-            self._list_available_subtitles(video_id, subtitles)
-            return
+        subtitle_url = xpath_text(episode, './media/asset/files/subtitle', 'subtitle')
+        if subtitle_url:
+            subtitles['es'] = [{
+                'ext': 'srt',
+                'url': subtitle_url,
+            }]
 
         return {
             'id': video_id,
@@ -159,5 +158,5 @@ class AtresPlayerIE(SubtitlesInfoExtractor):
             'thumbnail': thumbnail,
             'duration': duration,
             'formats': formats,
-            'subtitles': self.extract_subtitles(video_id, subtitles),
+            'subtitles': subtitles,
         }

From 65469a7f8b0ba50bd3c8918707e35125962aa2cd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 15 Feb 2015 18:52:07 +0100
Subject: [PATCH 0367/6219] [vimeo] Convert to new subtitles system

Removed some tests, the behaviour should be checked in a test for the YoutubeDL class
---
 test/parameters.json          |  2 +-
 test/test_subtitles.py        | 17 -----------------
 youtube_dl/extractor/vimeo.py | 15 ++++++---------
 3 files changed, 7 insertions(+), 27 deletions(-)

diff --git a/test/parameters.json b/test/parameters.json
index af77b89b4..cbff9bd16 100644
--- a/test/parameters.json
+++ b/test/parameters.json
@@ -28,7 +28,7 @@
     "retries": 10, 
     "simulate": false, 
     "subtitleslang": null, 
-    "subtitlesformat": "srt",
+    "subtitlesformat": "best",
     "test": true, 
     "updatetime": true, 
     "usenetrc": false, 
diff --git a/test/test_subtitles.py b/test/test_subtitles.py
index fbc9eaf4d..3f2d61d36 100644
--- a/test/test_subtitles.py
+++ b/test/test_subtitles.py
@@ -240,10 +240,6 @@ class TestVimeoSubtitles(BaseTestSubtitles):
     url = 'http://vimeo.com/76979871'
     IE = VimeoIE
 
-    def test_no_writesubtitles(self):
-        subtitles = self.getSubtitles()
-        self.assertEqual(subtitles, None)
-
     def test_subtitles(self):
         self.DL.params['writesubtitles'] = True
         subtitles = self.getSubtitles()
@@ -261,19 +257,6 @@ class TestVimeoSubtitles(BaseTestSubtitles):
         subtitles = self.getSubtitles()
         self.assertEqual(set(subtitles.keys()), set(['de', 'en', 'es', 'fr']))
 
-    def test_list_subtitles(self):
-        self.DL.expect_warning('Automatic Captions not supported by this server')
-        self.DL.params['listsubtitles'] = True
-        info_dict = self.getInfoDict()
-        self.assertEqual(info_dict, None)
-
-    def test_automatic_captions(self):
-        self.DL.expect_warning('Automatic Captions not supported by this server')
-        self.DL.params['writeautomaticsub'] = True
-        self.DL.params['subtitleslang'] = ['en']
-        subtitles = self.getSubtitles()
-        self.assertTrue(len(subtitles.keys()) == 0)
-
     def test_nosubtitles(self):
         self.DL.expect_warning('video doesn\'t have subtitles')
         self.url = 'http://vimeo.com/56015672'
diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 303e81447..5930d5984 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -6,7 +6,6 @@ import re
 import itertools
 
 from .common import InfoExtractor
-from .subtitles import SubtitlesInfoExtractor
 from ..compat import (
     compat_HTTPError,
     compat_urllib_parse,
@@ -51,7 +50,7 @@ class VimeoBaseInfoExtractor(InfoExtractor):
         self._download_webpage(login_request, None, False, 'Wrong login info')
 
 
-class VimeoIE(VimeoBaseInfoExtractor, SubtitlesInfoExtractor):
+class VimeoIE(VimeoBaseInfoExtractor):
     """Information extractor for vimeo.com."""
 
     # _VALID_URL matches Vimeo URLs
@@ -368,12 +367,10 @@ class VimeoIE(VimeoBaseInfoExtractor, SubtitlesInfoExtractor):
         text_tracks = config['request'].get('text_tracks')
         if text_tracks:
             for tt in text_tracks:
-                subtitles[tt['lang']] = 'http://vimeo.com' + tt['url']
-
-        video_subtitles = self.extract_subtitles(video_id, subtitles)
-        if self._downloader.params.get('listsubtitles', False):
-            self._list_available_subtitles(video_id, subtitles)
-            return
+                subtitles[tt['lang']] = [{
+                    'ext': 'vtt',
+                    'url': 'http://vimeo.com' + tt['url'],
+                }]
 
         return {
             'id': video_id,
@@ -389,7 +386,7 @@ class VimeoIE(VimeoBaseInfoExtractor, SubtitlesInfoExtractor):
             'view_count': view_count,
             'like_count': like_count,
             'comment_count': comment_count,
-            'subtitles': video_subtitles,
+            'subtitles': subtitles,
         }
 
 

From c84dd8a90dcc75547b343449b921b644a2119c4f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 16 Feb 2015 21:12:31 +0100
Subject: [PATCH 0368/6219] [YoutubeDL] store the subtitles to download in the
 'requested_subtitles' field

We need to keep the orginal subtitles information, so that the '--load-info' option can be used to list or select the subtitles again.
We'll also be able to have a separate field for storing the automatic captions info.
---
 test/test_subtitles.py             | 2 +-
 youtube_dl/YoutubeDL.py            | 6 +++---
 youtube_dl/extractor/common.py     | 2 --
 youtube_dl/postprocessor/ffmpeg.py | 7 ++++---
 4 files changed, 8 insertions(+), 9 deletions(-)

diff --git a/test/test_subtitles.py b/test/test_subtitles.py
index 3f2d61d36..b3c615c4f 100644
--- a/test/test_subtitles.py
+++ b/test/test_subtitles.py
@@ -36,7 +36,7 @@ class BaseTestSubtitles(unittest.TestCase):
 
     def getSubtitles(self):
         info_dict = self.getInfoDict()
-        subtitles = info_dict['subtitles']
+        subtitles = info_dict['requested_subtitles']
         if not subtitles:
             return subtitles
         for sub_info in subtitles.values():
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index e665e3d53..8545dc9e9 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1022,7 +1022,7 @@ class YoutubeDL(object):
         if self.params.get('listsubtitles', False):
             self.list_subtitles(info_dict['id'], info_dict.get('subtitles'))
             return
-        info_dict['subtitles'] = self.process_subtitles(info_dict['id'], info_dict.get('subtitles'))
+        info_dict['requested_subtitles'] = self.process_subtitles(info_dict['id'], info_dict.get('subtitles'))
 
         # This extractors handle format selection themselves
         if info_dict['extractor'] in ['Youku']:
@@ -1301,10 +1301,10 @@ class YoutubeDL(object):
         subtitles_are_requested = any([self.params.get('writesubtitles', False),
                                        self.params.get('writeautomaticsub')])
 
-        if subtitles_are_requested and 'subtitles' in info_dict and info_dict['subtitles']:
+        if subtitles_are_requested and info_dict.get('requested_subtitles'):
             # subtitles download errors are already managed as troubles in relevant IE
             # that way it will silently go on when used with unsupporting IE
-            subtitles = info_dict['subtitles']
+            subtitles = info_dict['requested_subtitles']
             for sub_lang, sub_info in subtitles.items():
                 sub_format = sub_info['ext']
                 if sub_info.get('data') is not None:
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 161c623eb..d149e0f92 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -157,8 +157,6 @@ class InfoExtractor(object):
                     with the "ext" entry and one of:
                         * "data": The subtitles file contents
                         * "url": A url pointing to the subtitles file
-                    Note: YoutubeDL.extract_info will get the requested
-                    format and replace the "subformats" list with it.
     duration:       Length of the video in seconds, as an integer.
     view_count:     How many users have watched the video on the platform.
     like_count:     Number of positive ratings of the video
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index d1bbfbfe3..e42298f0e 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -462,13 +462,14 @@ class FFmpegEmbedSubtitlePP(FFmpegPostProcessor):
         if information['ext'] != 'mp4':
             self._downloader.to_screen('[ffmpeg] Subtitles can only be embedded in mp4 files')
             return True, information
-        if not information.get('subtitles'):
+        subtitles = information.get('requested_subtitles')
+        if not subtitles:
             self._downloader.to_screen('[ffmpeg] There aren\'t any subtitles to embed')
             return True, information
 
-        sub_langs = [key for key in information['subtitles']]
+        sub_langs = list(subtitles.keys())
         filename = information['filepath']
-        input_files = [filename] + [subtitles_filename(filename, lang, sub_info['ext']) for lang, sub_info in information['subtitles'].items()]
+        input_files = [filename] + [subtitles_filename(filename, lang, sub_info['ext']) for lang, sub_info in subtitles.items()]
 
         opts = [
             '-map', '0',

From a1f2a06b34807a2e1b5eb5176fa418da2405392d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 16 Feb 2015 21:28:06 +0100
Subject: [PATCH 0369/6219] [dailymotion] Convert to new subtitles system

---
 test/test_subtitles.py              | 17 -----------------
 youtube_dl/extractor/dailymotion.py | 10 +++-------
 2 files changed, 3 insertions(+), 24 deletions(-)

diff --git a/test/test_subtitles.py b/test/test_subtitles.py
index b3c615c4f..84ae0e714 100644
--- a/test/test_subtitles.py
+++ b/test/test_subtitles.py
@@ -127,10 +127,6 @@ class TestDailymotionSubtitles(BaseTestSubtitles):
     url = 'http://www.dailymotion.com/video/xczg00'
     IE = DailymotionIE
 
-    def test_no_writesubtitles(self):
-        subtitles = self.getSubtitles()
-        self.assertEqual(subtitles, None)
-
     def test_subtitles(self):
         self.DL.params['writesubtitles'] = True
         subtitles = self.getSubtitles()
@@ -148,19 +144,6 @@ class TestDailymotionSubtitles(BaseTestSubtitles):
         subtitles = self.getSubtitles()
         self.assertTrue(len(subtitles.keys()) >= 6)
 
-    def test_list_subtitles(self):
-        self.DL.expect_warning('Automatic Captions not supported by this server')
-        self.DL.params['listsubtitles'] = True
-        info_dict = self.getInfoDict()
-        self.assertEqual(info_dict, None)
-
-    def test_automatic_captions(self):
-        self.DL.expect_warning('Automatic Captions not supported by this server')
-        self.DL.params['writeautomaticsub'] = True
-        self.DL.params['subtitleslang'] = ['en']
-        subtitles = self.getSubtitles()
-        self.assertTrue(len(subtitles.keys()) == 0)
-
     def test_nosubtitles(self):
         self.DL.expect_warning('video doesn\'t have subtitles')
         self.url = 'http://www.dailymotion.com/video/x12u166_le-zapping-tele-star-du-08-aout-2013_tv'
diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index cf5841a7c..4ca892926 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -6,7 +6,6 @@ import json
 import itertools
 
 from .common import InfoExtractor
-from .subtitles import SubtitlesInfoExtractor
 
 from ..compat import (
     compat_str,
@@ -31,7 +30,7 @@ class DailymotionBaseInfoExtractor(InfoExtractor):
         return request
 
 
-class DailymotionIE(DailymotionBaseInfoExtractor, SubtitlesInfoExtractor):
+class DailymotionIE(DailymotionBaseInfoExtractor):
     """Information Extractor for Dailymotion"""
 
     _VALID_URL = r'(?i)(?:https?://)?(?:(www|touch)\.)?dailymotion\.[a-z]{2,3}/(?:(embed|#)/)?video/(?P<id>[^/?_]+)'
@@ -143,9 +142,6 @@ class DailymotionIE(DailymotionBaseInfoExtractor, SubtitlesInfoExtractor):
 
         # subtitles
         video_subtitles = self.extract_subtitles(video_id, webpage)
-        if self._downloader.params.get('listsubtitles', False):
-            self._list_available_subtitles(video_id, webpage)
-            return
 
         view_count = str_to_int(self._search_regex(
             r'video_views_count[^>]+>\s+([\d\.,]+)',
@@ -169,7 +165,7 @@ class DailymotionIE(DailymotionBaseInfoExtractor, SubtitlesInfoExtractor):
             'view_count': view_count,
         }
 
-    def _get_available_subtitles(self, video_id, webpage):
+    def _get_subtitles(self, video_id, webpage):
         try:
             sub_list = self._download_webpage(
                 'https://api.dailymotion.com/video/%s/subtitles?fields=id,language,url' % video_id,
@@ -179,7 +175,7 @@ class DailymotionIE(DailymotionBaseInfoExtractor, SubtitlesInfoExtractor):
             return {}
         info = json.loads(sub_list)
         if (info['total'] > 0):
-            sub_lang_list = dict((l['language'], l['url']) for l in info['list'])
+            sub_lang_list = dict((l['language'], [{'url': l['url'], 'ext': 'srt'}]) for l in info['list'])
             return sub_lang_list
         self._downloader.report_warning('video doesn\'t have subtitles')
         return {}

From 360e1ca5ccabcb5d48228d9472b09f1bce68bbc4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 16 Feb 2015 21:44:17 +0100
Subject: [PATCH 0370/6219] [youtube] Convert to new subtitles system

The automatic captions are stored in the 'automactic_captions' field, which is used if no normal subtitles are found for an specific language.
---
 test/test_subtitles.py          |  5 ---
 youtube_dl/YoutubeDL.py         | 24 ++++++++++----
 youtube_dl/extractor/common.py  | 12 +++++++
 youtube_dl/extractor/youtube.py | 57 ++++++++++++++++++---------------
 4 files changed, 61 insertions(+), 37 deletions(-)

diff --git a/test/test_subtitles.py b/test/test_subtitles.py
index 84ae0e714..91cebce28 100644
--- a/test/test_subtitles.py
+++ b/test/test_subtitles.py
@@ -50,11 +50,6 @@ class TestYoutubeSubtitles(BaseTestSubtitles):
     url = 'QRS8MkLhQmM'
     IE = YoutubeIE
 
-    def test_youtube_no_writesubtitles(self):
-        self.DL.params['writesubtitles'] = False
-        subtitles = self.getSubtitles()
-        self.assertEqual(subtitles, None)
-
     def test_youtube_subtitles(self):
         self.DL.params['writesubtitles'] = True
         subtitles = self.getSubtitles()
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 8545dc9e9..a47f8f5de 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1020,9 +1020,13 @@ class YoutubeDL(object):
             info_dict['upload_date'] = upload_date.strftime('%Y%m%d')
 
         if self.params.get('listsubtitles', False):
-            self.list_subtitles(info_dict['id'], info_dict.get('subtitles'))
+            if 'automatic_captions' in info_dict:
+                self.list_subtitles(info_dict['id'], info_dict.get('automatic_captions'), 'automatic captions')
+            self.list_subtitles(info_dict['id'], info_dict.get('subtitles'), 'subtitles')
             return
-        info_dict['requested_subtitles'] = self.process_subtitles(info_dict['id'], info_dict.get('subtitles'))
+        info_dict['requested_subtitles'] = self.process_subtitles(
+            info_dict['id'], info_dict.get('subtitles'),
+            info_dict.get('automatic_captions'))
 
         # This extractors handle format selection themselves
         if info_dict['extractor'] in ['Youku']:
@@ -1152,8 +1156,14 @@ class YoutubeDL(object):
         info_dict.update(formats_to_download[-1])
         return info_dict
 
-    def process_subtitles(self, video_id, available_subs):
+    def process_subtitles(self, video_id, available_subs, available_autocaps):
         """Select the requested subtitles and their format"""
+        if available_autocaps and self.params.get('writeautomaticsub'):
+            available_subs = available_subs.copy()
+            for lang, cap_info in available_autocaps.items():
+                if lang not in available_subs:
+                    available_subs[lang] = cap_info
+
         if not available_subs:
             return available_subs
 
@@ -1645,17 +1655,17 @@ class YoutubeDL(object):
             ['ID', 'width', 'height', 'URL'],
             [[t['id'], t.get('width', 'unknown'), t.get('height', 'unknown'), t['url']] for t in thumbnails]))
 
-    def list_subtitles(self, video_id, subtitles):
+    def list_subtitles(self, video_id, subtitles, name='subtitles'):
         if not subtitles:
-            self.to_screen('%s has no subtitles' % video_id)
+            self.to_screen('%s has no %s' % (video_id, name))
             return
         header_line = 'Language    formats'
         sub_lines = [
             '%-12s%s' % (lang, ', '.join(f['ext'] for f in reversed(formats)))
             for lang, formats in subtitles.items()]
         self.to_screen(
-            'Available subtitles for %s:\n%s\n%s' %
-            (video_id, header_line, '\n'.join(sub_lines)))
+            'Available %s for %s:\n%s\n%s' %
+            (name, video_id, header_line, '\n'.join(sub_lines)))
 
     def urlopen(self, req):
         """ Start an HTTP download """
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index d149e0f92..fe7d8dbc9 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -157,6 +157,8 @@ class InfoExtractor(object):
                     with the "ext" entry and one of:
                         * "data": The subtitles file contents
                         * "url": A url pointing to the subtitles file
+    automatic_captions: Like 'subtitles', used by the YoutubeIE for
+                    automatically generated captions
     duration:       Length of the video in seconds, as an integer.
     view_count:     How many users have watched the video on the platform.
     like_count:     Number of positive ratings of the video
@@ -1007,6 +1009,16 @@ class InfoExtractor(object):
     def _get_subtitles(self, *args, **kwargs):
         raise NotImplementedError("This method must be implemented by subclasses")
 
+    def extract_automatic_captions(self, *args, **kwargs):
+        automatic_captions = {}
+        list_subtitles = self._downloader.params.get('listsubtitles')
+        if self._downloader.params.get('writeautomaticsub', False) or list_subtitles:
+            automatic_captions.update(self._get_automatic_captions(*args, **kwargs))
+        return automatic_captions
+
+    def _get_automatic_captions(self, *args, **kwargs):
+        raise NotImplementedError("This method must be implemented by subclasses")
+
 
 class SearchInfoExtractor(InfoExtractor):
     """
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 35ef4c303..1b2dbf276 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -11,7 +11,6 @@ import time
 import traceback
 
 from .common import InfoExtractor, SearchInfoExtractor
-from .subtitles import SubtitlesInfoExtractor
 from ..jsinterp import JSInterpreter
 from ..swfinterp import SWFInterpreter
 from ..compat import (
@@ -185,7 +184,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
             return
 
 
-class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
+class YoutubeIE(YoutubeBaseInfoExtractor):
     IE_DESC = 'YouTube.com'
     _VALID_URL = r"""(?x)^
                      (
@@ -644,7 +643,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
             raise ExtractorError(
                 'Signature extraction failed: ' + tb, cause=e)
 
-    def _get_available_subtitles(self, video_id, webpage):
+    def _get_subtitles(self, video_id, webpage):
         try:
             subs_doc = self._download_xml(
                 'https://video.google.com/timedtext?hl=en&type=list&v=%s' % video_id,
@@ -658,23 +657,27 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
             lang = track.attrib['lang_code']
             if lang in sub_lang_list:
                 continue
-            params = compat_urllib_parse.urlencode({
-                'lang': lang,
-                'v': video_id,
-                'fmt': self._downloader.params.get('subtitlesformat', 'srt'),
-                'name': track.attrib['name'].encode('utf-8'),
-            })
-            url = 'https://www.youtube.com/api/timedtext?' + params
-            sub_lang_list[lang] = url
+            sub_formats = []
+            for ext in ['sbv', 'vtt', 'srt']:
+                params = compat_urllib_parse.urlencode({
+                    'lang': lang,
+                    'v': video_id,
+                    'fmt': ext,
+                    'name': track.attrib['name'].encode('utf-8'),
+                })
+                sub_formats.append({
+                    'url': 'https://www.youtube.com/api/timedtext?' + params,
+                    'ext': ext,
+                })
+            sub_lang_list[lang] = sub_formats
         if not sub_lang_list:
             self._downloader.report_warning('video doesn\'t have subtitles')
             return {}
         return sub_lang_list
 
-    def _get_available_automatic_caption(self, video_id, webpage):
+    def _get_automatic_captions(self, video_id, webpage):
         """We need the webpage for getting the captions url, pass it as an
            argument to speed up the process."""
-        sub_format = self._downloader.params.get('subtitlesformat', 'srt')
         self.to_screen('%s: Looking for automatic captions' % video_id)
         mobj = re.search(r';ytplayer.config = ({.*?});', webpage)
         err_msg = 'Couldn\'t find automatic captions for %s' % video_id
@@ -704,14 +707,20 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
             sub_lang_list = {}
             for lang_node in caption_list.findall('target'):
                 sub_lang = lang_node.attrib['lang_code']
-                params = compat_urllib_parse.urlencode({
-                    'lang': original_lang,
-                    'tlang': sub_lang,
-                    'fmt': sub_format,
-                    'ts': timestamp,
-                    'kind': caption_kind,
-                })
-                sub_lang_list[sub_lang] = caption_url + '&' + params
+                sub_formats = []
+                for ext in ['sbv', 'vtt', 'srt']:
+                    params = compat_urllib_parse.urlencode({
+                        'lang': original_lang,
+                        'tlang': sub_lang,
+                        'fmt': ext,
+                        'ts': timestamp,
+                        'kind': caption_kind,
+                    })
+                    sub_formats.append({
+                        'url': caption_url + '&' + params,
+                        'ext': ext,
+                    })
+                sub_lang_list[sub_lang] = sub_formats
             return sub_lang_list
         # An extractor error can be raise by the download process if there are
         # no automatic captions but there are subtitles
@@ -966,10 +975,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
 
         # subtitles
         video_subtitles = self.extract_subtitles(video_id, video_webpage)
-
-        if self._downloader.params.get('listsubtitles', False):
-            self._list_available_subtitles(video_id, video_webpage)
-            return
+        automatic_captions = self.extract_automatic_captions(video_id, video_webpage)
 
         if 'length_seconds' not in video_info:
             self._downloader.report_warning('unable to extract video duration')
@@ -1118,6 +1124,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
             'description': video_description,
             'categories': video_categories,
             'subtitles': video_subtitles,
+            'automatic_captions': automatic_captions,
             'duration': video_duration,
             'age_limit': 18 if age_gate else 0,
             'annotations': video_annotations,

From 85729c51afad484ef784faf5d82bad8acab77d5e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 17 Feb 2015 12:09:12 +0100
Subject: [PATCH 0371/6219] [downloader] Add --hls-prefer-native to use the
 native HLS downloader (#4966)

---
 youtube_dl/YoutubeDL.py           | 5 ++++-
 youtube_dl/__init__.py            | 1 +
 youtube_dl/downloader/__init__.py | 3 +++
 youtube_dl/options.py             | 4 ++++
 4 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index dbb26272d..ea2435e0a 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -225,7 +225,6 @@ class YoutubeDL(object):
     call_home:         Boolean, true iff we are allowed to contact the
                        youtube-dl servers for debugging.
     sleep_interval:    Number of seconds to sleep before each download.
-    external_downloader:  Executable of the external downloader to call.
     listformats:       Print an overview of available video formats and exit.
     list_thumbnails:   Print a table of all thumbnails and exit.
     match_filter:      A function that gets called with the info_dict of
@@ -235,6 +234,10 @@ class YoutubeDL(object):
                        match_filter_func in utils.py is one example for this.
     no_color:          Do not emit color codes in output.
 
+    The following options determine which downloader is picked:
+    external_downloader: Executable of the external downloader to call.
+                       None or unset for standard (built-in) downloader.
+    hls_prefer_native: Use the native HLS downloader instead of ffmpeg/avconv.
 
     The following parameters are not used by YoutubeDL itself, they are used by
     the FileDownloader:
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 108fb3c7a..eac2a26ec 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -351,6 +351,7 @@ def _real_main(argv=None):
         'match_filter': match_filter,
         'no_color': opts.no_color,
         'ffmpeg_location': opts.ffmpeg_location,
+        'hls_prefer_native': opts.hls_prefer_native,
     }
 
     with YoutubeDL(ydl_opts) as ydl:
diff --git a/youtube_dl/downloader/__init__.py b/youtube_dl/downloader/__init__.py
index eff1122c5..9fb66e2f7 100644
--- a/youtube_dl/downloader/__init__.py
+++ b/youtube_dl/downloader/__init__.py
@@ -34,6 +34,9 @@ def get_suitable_downloader(info_dict, params={}):
         if ed.supports(info_dict):
             return ed
 
+    if protocol == 'm3u8' and params.get('hls_prefer_native'):
+        return NativeHlsFD
+
     return PROTOCOL_MAP.get(protocol, HttpFD)
 
 
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index ba35399cf..5f678f76b 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -424,6 +424,10 @@ def parseOpts(overrideArguments=None):
         '--xattr-set-filesize',
         dest='xattr_set_filesize', action='store_true',
         help='(experimental) set file xattribute ytdl.filesize with expected filesize')
+    downloader.add_option(
+        '--hls-prefer-native',
+        dest='hls_prefer_native', action='store_true',
+        help='(experimental) Use the native HLS downloader instead of ffmpeg.')
     downloader.add_option(
         '--external-downloader',
         dest='external_downloader', metavar='COMMAND',

From 163d966707a7e49bcdad4ebd9189922b58223395 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 17 Feb 2015 16:21:02 +0100
Subject: [PATCH 0372/6219] [downloader/external] curl: Add the '--location'
 flag

curl doesn't follow redirections by default
---
 youtube_dl/downloader/external.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index ff031d2e0..51c41c704 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -75,7 +75,7 @@ class ExternalFD(FileDownloader):
 
 class CurlFD(ExternalFD):
     def _make_cmd(self, tmpfilename, info_dict):
-        cmd = [self.exe, '-o', tmpfilename]
+        cmd = [self.exe, '--location', '-o', tmpfilename]
         for key, val in info_dict['http_headers'].items():
             cmd += ['--header', '%s: %s' % (key, val)]
         cmd += self._source_address('--interface')

From 662435f7281da418612131512725fa5e367605be Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 17 Feb 2015 16:29:24 +0100
Subject: [PATCH 0373/6219] [YoutubeDL] Use a Request object for getting the
 cookies (fixes #4970)

So that we don't have to implement all the methods used by the cookiejar.
---
 youtube_dl/YoutubeDL.py | 25 ++-----------------------
 1 file changed, 2 insertions(+), 23 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index ea2435e0a..0a41152d6 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -954,30 +954,9 @@ class YoutubeDL(object):
         return res
 
     def _calc_cookies(self, info_dict):
-        class _PseudoRequest(object):
-            def __init__(self, url):
-                self.url = url
-                self.headers = {}
-                self.unverifiable = False
-
-            def add_unredirected_header(self, k, v):
-                self.headers[k] = v
-
-            def get_full_url(self):
-                return self.url
-
-            def is_unverifiable(self):
-                return self.unverifiable
-
-            def has_header(self, h):
-                return h in self.headers
-
-            def get_header(self, h, default=None):
-                return self.headers.get(h, default)
-
-        pr = _PseudoRequest(info_dict['url'])
+        pr = compat_urllib_request.Request(info_dict['url'])
         self.cookiejar.add_cookie_header(pr)
-        return pr.headers.get('Cookie')
+        return pr.get_header('Cookie')
 
     def process_video_result(self, info_dict, download=True):
         assert info_dict.get('_type', 'video') == 'video'

From fbc503d69698fca33525707b859b41d491d681ff Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 17 Feb 2015 16:40:42 +0100
Subject: [PATCH 0374/6219] [downloader/hls] Fix detection of ffmpeg/avconv
 (reported in #4966)

---
 youtube_dl/downloader/hls.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index 8be4f4249..3650d715e 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -23,7 +23,7 @@ class HlsFD(FileDownloader):
         tmpfilename = self.temp_name(filename)
 
         ffpp = FFmpegPostProcessor(downloader=self)
-        if not ffpp.available:
+        if not ffpp.available():
             self.report_error('m3u8 download detected but ffmpeg or avconv could not be found. Please install one.')
             return False
         ffpp.check_version()

From f740fae2a418ce6c1d229890b48730d257a80dee Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 17 Feb 2015 17:26:41 +0100
Subject: [PATCH 0375/6219] [ffmpeg] Make available a property

---
 youtube_dl/YoutubeDL.py            | 2 +-
 youtube_dl/postprocessor/ffmpeg.py | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index ea2435e0a..a74e73b0c 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1301,7 +1301,7 @@ class YoutubeDL(object):
                     downloaded = []
                     success = True
                     merger = FFmpegMergerPP(self, not self.params.get('keepvideo'))
-                    if not merger.available():
+                    if not merger.available:
                         postprocessors = []
                         self.report_warning('You have requested multiple '
                                             'formats but ffmpeg or avconv are not installed.'
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 16babf6a5..e98fe9834 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -34,7 +34,7 @@ class FFmpegPostProcessor(PostProcessor):
         self._determine_executables()
 
     def check_version(self):
-        if not self.available():
+        if not self.available:
             raise FFmpegPostProcessorError('ffmpeg or avconv not found. Please install one.')
 
         required_version = '10-0' if self._uses_avconv() else '1.0'
@@ -108,6 +108,7 @@ class FFmpegPostProcessor(PostProcessor):
                 self.probe_basename = p
                 break
 
+    @property
     def available(self):
         return self.basename is not None
 

From 65bf37ef83b67bd0e2c594283852d120734b2a3a Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 17 Feb 2015 17:27:29 +0100
Subject: [PATCH 0376/6219] [ffmpeg] Remove trivial helper method

---
 youtube_dl/postprocessor/ffmpeg.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index e98fe9834..3f2e6cf1d 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -37,7 +37,7 @@ class FFmpegPostProcessor(PostProcessor):
         if not self.available:
             raise FFmpegPostProcessorError('ffmpeg or avconv not found. Please install one.')
 
-        required_version = '10-0' if self._uses_avconv() else '1.0'
+        required_version = '10-0' if self.basename == 'avconv' else '1.0'
         if is_outdated_version(
                 self._versions[self.basename], required_version):
             warning = 'Your copy of %s is outdated, update %s to version %s or newer if you encounter any errors.' % (
@@ -112,9 +112,6 @@ class FFmpegPostProcessor(PostProcessor):
     def available(self):
         return self.basename is not None
 
-    def _uses_avconv(self):
-        return self.basename == 'avconv'
-
     @property
     def executable(self):
         return self._paths[self.basename]

From 46312e0b46147ff26008eee62b2f9ba47b30fe73 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 17 Feb 2015 17:29:32 +0100
Subject: [PATCH 0377/6219] release 2015.02.17

---
 README.md             | 2 ++
 youtube_dl/version.py | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index d68896202..acd0ef41b 100644
--- a/README.md
+++ b/README.md
@@ -161,6 +161,8 @@ which means you can modify it, redistribute it or use it however you like.
     --playlist-reverse               Download playlist videos in reverse order
     --xattr-set-filesize             (experimental) set file xattribute
                                      ytdl.filesize with expected filesize
+    --hls-prefer-native              (experimental) Use the native HLS
+                                     downloader instead of ffmpeg.
     --external-downloader COMMAND    (experimental) Use the specified external
                                      downloader. Currently supports
                                      aria2c,curl,wget
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index bbe0e53b5..88834e058 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.16.1'
+__version__ = '2015.02.17'

From 8ac27a68e618ecc789bbef4dd0eb408324498fdd Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 17 Feb 2015 17:35:03 +0100
Subject: [PATCH 0378/6219] [hls] Switch to available as a property

---
 youtube_dl/downloader/hls.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index 3650d715e..8be4f4249 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -23,7 +23,7 @@ class HlsFD(FileDownloader):
         tmpfilename = self.temp_name(filename)
 
         ffpp = FFmpegPostProcessor(downloader=self)
-        if not ffpp.available():
+        if not ffpp.available:
             self.report_error('m3u8 download detected but ffmpeg or avconv could not be found. Please install one.')
             return False
         ffpp.check_version()

From 7010577720bbea309566c38adbd912c15400b9fc Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 17 Feb 2015 17:35:08 +0100
Subject: [PATCH 0379/6219] release 2015.02.17.1

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 88834e058..4ffbd3ae4 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.17'
+__version__ = '2015.02.17.1'

From 4069766c527d10b8e25b9262a3882101367deb3e Mon Sep 17 00:00:00 2001
From: Sergey M? <dstftw@gmail.com>
Date: Tue, 17 Feb 2015 22:31:35 +0600
Subject: [PATCH 0380/6219] [extractor/common] Test URLs with GET

---
 youtube_dl/extractor/common.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index e74b7bf25..4b0ee882c 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -753,9 +753,7 @@ class InfoExtractor(object):
 
     def _is_valid_url(self, url, video_id, item='video'):
         try:
-            self._request_webpage(
-                HEADRequest(url), video_id,
-                'Checking %s URL' % item)
+            self._request_webpage(url, video_id, 'Checking %s URL' % item)
             return True
         except ExtractorError as e:
             if isinstance(e.cause, compat_HTTPError):

From 0d93378887bd527b1df04e6138b4bc41382dd48f Mon Sep 17 00:00:00 2001
From: Sergey M? <dstftw@gmail.com>
Date: Tue, 17 Feb 2015 22:34:29 +0600
Subject: [PATCH 0381/6219] [videolecturesnet] Check http format URLs (Closes
 #4968)

---
 youtube_dl/extractor/videolecturesnet.py | 33 +++++++++++++++++-------
 1 file changed, 24 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/videolecturesnet.py b/youtube_dl/extractor/videolecturesnet.py
index ebd2a3dca..1ec5298e9 100644
--- a/youtube_dl/extractor/videolecturesnet.py
+++ b/youtube_dl/extractor/videolecturesnet.py
@@ -49,15 +49,30 @@ class VideoLecturesNetIE(InfoExtractor):
         thumbnail = (
             None if thumbnail_el is None else thumbnail_el.attrib.get('src'))
 
-        formats = [{
-            'url': v.attrib['src'],
-            'width': int_or_none(v.attrib.get('width')),
-            'height': int_or_none(v.attrib.get('height')),
-            'filesize': int_or_none(v.attrib.get('size')),
-            'tbr': int_or_none(v.attrib.get('systemBitrate')) / 1000.0,
-            'ext': v.attrib.get('ext'),
-        } for v in switch.findall('./video')
-            if v.attrib.get('proto') == 'http']
+        formats = []
+        for v in switch.findall('./video'):
+            proto = v.attrib.get('proto')
+            if not proto in ['http', 'rtmp']:
+                continue
+            f = {
+                'width': int_or_none(v.attrib.get('width')),
+                'height': int_or_none(v.attrib.get('height')),
+                'filesize': int_or_none(v.attrib.get('size')),
+                'tbr': int_or_none(v.attrib.get('systemBitrate')) / 1000.0,
+                'ext': v.attrib.get('ext'),
+            }
+            src = v.attrib['src']
+            if proto == 'http':
+                if self._is_valid_url(src, video_id):
+                    f['url'] = src
+                    formats.append(f)
+            elif proto == 'rtmp':
+                f.update({
+                    'url': v.attrib['streamer'],
+                    'play_path': src,
+                })
+                formats.append(f)
+        self._sort_formats(formats)
 
         return {
             'id': video_id,

From be24c8697f51fbc68e2cd99668e675a07a3735de Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 17 Feb 2015 17:38:31 +0100
Subject: [PATCH 0382/6219] release 2015.02.17.2

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 4ffbd3ae4..34a8d1184 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.17.1'
+__version__ = '2015.02.17.2'

From 4cd95bcbc34d74e506e6158251d50d83697da5fe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 17 Feb 2015 18:55:53 +0100
Subject: [PATCH 0383/6219] [twitch:stream] Prefer the 'source' format (fixes
 #4972)

---
 youtube_dl/extractor/common.py | 11 +++++++++++
 youtube_dl/extractor/twitch.py |  6 ++++++
 2 files changed, 17 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 4b0ee882c..31599bf52 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -839,6 +839,7 @@ class InfoExtractor(object):
             note='Downloading m3u8 information',
             errnote='Failed to download m3u8 information')
         last_info = None
+        last_media= None
         kv_rex = re.compile(
             r'(?P<key>[a-zA-Z_-]+)=(?P<val>"[^"]+"|[^",]+)(?:,|$)')
         for line in m3u8_doc.splitlines():
@@ -849,6 +850,13 @@ class InfoExtractor(object):
                     if v.startswith('"'):
                         v = v[1:-1]
                     last_info[m.group('key')] = v
+            elif line.startswith('#EXT-X-MEDIA:'):
+                last_media = {}
+                for m in kv_rex.finditer(line):
+                    v = m.group('val')
+                    if v.startswith('"'):
+                        v = v[1:-1]
+                    last_media[m.group('key')] = v
             elif line.startswith('#') or not line.strip():
                 continue
             else:
@@ -877,6 +885,9 @@ class InfoExtractor(object):
                     width_str, height_str = resolution.split('x')
                     f['width'] = int(width_str)
                     f['height'] = int(height_str)
+                if last_media is not None:
+                    f['m3u8_media'] = last_media
+                    last_media = None
                 formats.append(f)
                 last_info = {}
         self._sort_formats(formats)
diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 87290d002..8e296698e 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -348,6 +348,12 @@ class TwitchStreamIE(TwitchBaseIE):
             '%s/api/channel/hls/%s.m3u8?%s'
             % (self._USHER_BASE, channel_id, compat_urllib_parse.urlencode(query).encode('utf-8')),
             channel_id, 'mp4')
+        # prefer the 'source' stream, the others are limited to 30 fps
+        def _sort_source(f):
+            if f.get('m3u8_media') is not None and f['m3u8_media'].get('NAME') == 'Source':
+                return 1
+            return 0
+        formats = sorted(formats, key=_sort_source)
 
         view_count = stream.get('viewers')
         timestamp = parse_iso8601(stream.get('created_at'))

From 4a8d4a53b107dde6e78f37fd63dee22b3d29a877 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 18 Feb 2015 01:16:49 +0600
Subject: [PATCH 0384/6219] [videolecturesnet] Fix rtmp stream glitches (Closes
 #4968)

---
 youtube_dl/extractor/videolecturesnet.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/videolecturesnet.py b/youtube_dl/extractor/videolecturesnet.py
index 1ec5298e9..d46234856 100644
--- a/youtube_dl/extractor/videolecturesnet.py
+++ b/youtube_dl/extractor/videolecturesnet.py
@@ -70,6 +70,7 @@ class VideoLecturesNetIE(InfoExtractor):
                 f.update({
                     'url': v.attrib['streamer'],
                     'play_path': src,
+                    'rtmp_real_time': True,
                 })
                 formats.append(f)
         self._sort_formats(formats)

From 48246541da66a12486505804f9519391a298ff54 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 17 Feb 2015 21:17:47 +0100
Subject: [PATCH 0385/6219] [ceskatelevize] Convert to new subtitles system

---
 youtube_dl/extractor/ceskatelevize.py | 30 +++++++++++++--------------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/ceskatelevize.py b/youtube_dl/extractor/ceskatelevize.py
index f70e090bb..65f6be623 100644
--- a/youtube_dl/extractor/ceskatelevize.py
+++ b/youtube_dl/extractor/ceskatelevize.py
@@ -3,7 +3,7 @@ from __future__ import unicode_literals
 
 import re
 
-from .subtitles import SubtitlesInfoExtractor
+from .common import InfoExtractor
 from ..compat import (
     compat_urllib_request,
     compat_urllib_parse,
@@ -15,7 +15,7 @@ from ..utils import (
 )
 
 
-class CeskaTelevizeIE(SubtitlesInfoExtractor):
+class CeskaTelevizeIE(InfoExtractor):
     _VALID_URL = r'https?://www\.ceskatelevize\.cz/(porady|ivysilani)/(.+/)?(?P<id>[^?#]+)'
 
     _TESTS = [
@@ -107,13 +107,7 @@ class CeskaTelevizeIE(SubtitlesInfoExtractor):
         subtitles = {}
         subs = item.get('subtitles')
         if subs:
-            subtitles['cs'] = subs[0]['url']
-
-        if self._downloader.params.get('listsubtitles', False):
-            self._list_available_subtitles(video_id, subtitles)
-            return
-
-        subtitles = self._fix_subtitles(self.extract_subtitles(video_id, subtitles))
+            subtitles = self.extract_subtitles(episode_id, subs)
 
         return {
             'id': episode_id,
@@ -125,11 +119,20 @@ class CeskaTelevizeIE(SubtitlesInfoExtractor):
             'subtitles': subtitles,
         }
 
+    def _get_subtitles(self, episode_id, subs):
+        original_subtitles = self._download_webpage(
+            subs[0]['url'], episode_id, 'Downloading subtitles')
+        srt_subs = self._fix_subtitles(original_subtitles)
+        return {
+            'cs': [{
+                'ext': 'srt',
+                'data': srt_subs,
+            }]
+        }
+
     @staticmethod
     def _fix_subtitles(subtitles):
         """ Convert millisecond-based subtitles to SRT """
-        if subtitles is None:
-            return subtitles  # subtitles not requested
 
         def _msectotimecode(msec):
             """ Helper utility to convert milliseconds to timecode """
@@ -149,7 +152,4 @@ class CeskaTelevizeIE(SubtitlesInfoExtractor):
                 else:
                     yield line
 
-        fixed_subtitles = {}
-        for k, v in subtitles.items():
-            fixed_subtitles[k] = "\r\n".join(_fix_subtitle(v))
-        return fixed_subtitles
+        return "\r\n".join(_fix_subtitle(subtitles))

From bd7fe0cf6668c9ea9272dbe25774072b383e67d4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 17 Feb 2015 21:23:09 +0100
Subject: [PATCH 0386/6219] [walla] Convert to new subtitles system

---
 youtube_dl/extractor/walla.py | 15 ++++++---------
 1 file changed, 6 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/walla.py b/youtube_dl/extractor/walla.py
index 672bda7a7..24efbd6e6 100644
--- a/youtube_dl/extractor/walla.py
+++ b/youtube_dl/extractor/walla.py
@@ -3,14 +3,14 @@ from __future__ import unicode_literals
 
 import re
 
-from .subtitles import SubtitlesInfoExtractor
+from .common import InfoExtractor
 from ..utils import (
     xpath_text,
     int_or_none,
 )
 
 
-class WallaIE(SubtitlesInfoExtractor):
+class WallaIE(InfoExtractor):
     _VALID_URL = r'http://vod\.walla\.co\.il/[^/]+/(?P<id>\d+)/(?P<display_id>.+)'
     _TEST = {
         'url': 'http://vod.walla.co.il/movie/2642630/one-direction-all-for-one',
@@ -52,13 +52,10 @@ class WallaIE(SubtitlesInfoExtractor):
         subtitles = {}
         for subtitle in item.findall('./subtitles/subtitle'):
             lang = xpath_text(subtitle, './title')
-            subtitles[self._SUBTITLE_LANGS.get(lang, lang)] = xpath_text(subtitle, './src')
-
-        if self._downloader.params.get('listsubtitles', False):
-            self._list_available_subtitles(video_id, subtitles)
-            return
-
-        subtitles = self.extract_subtitles(video_id, subtitles)
+            subtitles[self._SUBTITLE_LANGS.get(lang, lang)] = [{
+                'ext': 'srt',
+                'url': xpath_text(subtitle, './src'),
+            }]
 
         formats = []
         for quality in item.findall('./qualities/quality'):

From 98f000409f072e544d6c3b07809022f703b8b23d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 17 Feb 2015 19:54:04 +0100
Subject: [PATCH 0387/6219] [radio.de] Fix extraction

---
 youtube_dl/extractor/radiode.py | 15 ++++++---------
 1 file changed, 6 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/radiode.py b/youtube_dl/extractor/radiode.py
index f95bc9454..aa5f6f8ad 100644
--- a/youtube_dl/extractor/radiode.py
+++ b/youtube_dl/extractor/radiode.py
@@ -1,7 +1,5 @@
 from __future__ import unicode_literals
 
-import json
-
 from .common import InfoExtractor
 
 
@@ -10,13 +8,13 @@ class RadioDeIE(InfoExtractor):
     _VALID_URL = r'https?://(?P<id>.+?)\.(?:radio\.(?:de|at|fr|pt|es|pl|it)|rad\.io)'
     _TEST = {
         'url': 'http://ndr2.radio.de/',
-        'md5': '3b4cdd011bc59174596b6145cda474a4',
         'info_dict': {
             'id': 'ndr2',
             'ext': 'mp3',
             'title': 're:^NDR 2 [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
             'description': 'md5:591c49c702db1a33751625ebfb67f273',
             'thumbnail': 're:^https?://.*\.png',
+            'is_live': True,
         },
         'params': {
             'skip_download': True,
@@ -25,16 +23,15 @@ class RadioDeIE(InfoExtractor):
 
     def _real_extract(self, url):
         radio_id = self._match_id(url)
-
         webpage = self._download_webpage(url, radio_id)
+        jscode = self._search_regex(
+            r"'components/station/stationService':\s*\{\s*'?station'?:\s*(\{.*?\s*\}),\n",
+            webpage, 'broadcast')
 
-        broadcast = json.loads(self._search_regex(
-            r'_getBroadcast\s*=\s*function\(\s*\)\s*{\s*return\s+({.+?})\s*;\s*}',
-            webpage, 'broadcast'))
-
+        broadcast = self._parse_json(jscode, radio_id)
         title = self._live_title(broadcast['name'])
         description = broadcast.get('description') or broadcast.get('shortDescription')
-        thumbnail = broadcast.get('picture4Url') or broadcast.get('picture4TransUrl')
+        thumbnail = broadcast.get('picture4Url') or broadcast.get('picture4TransUrl') or broadcast.get('logo100x100')
 
         formats = [{
             'url': stream['streamUrl'],

From 5cda4eda7253d766611363a880af46895c11ad17 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 17 Feb 2015 21:37:48 +0100
Subject: [PATCH 0388/6219] [YoutubeDL] Use a progress hook for progress
 reporting

Instead of every downloader calling two helper functions, let our progress report be an ordinary progress hook like everyone else's.
Closes #4875.
---
 youtube_dl/YoutubeDL.py         | 13 +++--
 youtube_dl/downloader/common.py | 87 +++++++++++++++++++++------------
 youtube_dl/downloader/f4m.py    | 70 ++++++++++++++++----------
 youtube_dl/downloader/http.py   | 13 +++--
 youtube_dl/downloader/rtmp.py   | 18 +++----
 5 files changed, 128 insertions(+), 73 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 3658332ec..b087d356f 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -199,18 +199,25 @@ class YoutubeDL(object):
                        postprocessor.
     progress_hooks:    A list of functions that get called on download
                        progress, with a dictionary with the entries
-                       * status: One of "downloading" and "finished".
+                       * status: One of "downloading", "error", or "finished".
                                  Check this first and ignore unknown values.
 
-                       If status is one of "downloading" or "finished", the
+                       If status is one of "downloading", or "finished", the
                        following properties may also be present:
                        * filename: The final filename (always present)
+                       * tmpfilename: The filename we're currently writing to
                        * downloaded_bytes: Bytes on disk
                        * total_bytes: Size of the whole file, None if unknown
-                       * tmpfilename: The filename we're currently writing to
+                       * total_bytes_estimate: Guess of the eventual file size,
+                                               None if unavailable.
+                       * elapsed: The number of seconds since download started.
                        * eta: The estimated time in seconds, None if unknown
                        * speed: The download speed in bytes/second, None if
                                 unknown
+                       * fragment_index: The counter of the currently
+                                         downloaded video fragment.
+                       * fragment_count: The number of fragments (= individual
+                                         files that will be merged)
 
                        Progress hooks are guaranteed to be called at least once
                        (with status "finished") if the download is successful.
diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index 7bb3a948d..45e55b99c 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -1,4 +1,4 @@
-from __future__ import unicode_literals
+from __future__ import division, unicode_literals
 
 import os
 import re
@@ -54,6 +54,7 @@ class FileDownloader(object):
         self.ydl = ydl
         self._progress_hooks = []
         self.params = params
+        self.add_progress_hook(self.report_progress)
 
     @staticmethod
     def format_seconds(seconds):
@@ -226,42 +227,64 @@ class FileDownloader(object):
             self.to_screen(clear_line + fullmsg, skip_eol=not is_last_line)
         self.to_console_title('youtube-dl ' + msg)
 
-    def report_progress(self, percent, data_len_str, speed, eta):
-        """Report download progress."""
-        if self.params.get('noprogress', False):
+    def report_progress(self, s):
+        if s['status'] == 'finished':
+            if self.params.get('noprogress', False):
+                self.to_screen('[download] Download completed')
+            else:
+                s['_total_bytes_str'] = format_bytes(s['total_bytes'])
+                if s.get('elapsed') is not None:
+                    s['_elapsed_str'] = self.format_seconds(s['elapsed'])
+                    msg_template = '100%% of %(_total_bytes_str)s in %(_elapsed_str)s'
+                else:
+                    msg_template = '100%% of %(_total_bytes_str)s'
+                self._report_progress_status(
+                    msg_template % s, is_last_line=True)
+
+        if self.params.get('noprogress'):
             return
-        if eta is not None:
-            eta_str = self.format_eta(eta)
-        else:
-            eta_str = 'Unknown ETA'
-        if percent is not None:
-            percent_str = self.format_percent(percent)
-        else:
-            percent_str = 'Unknown %'
-        speed_str = self.format_speed(speed)
 
-        msg = ('%s of %s at %s ETA %s' %
-               (percent_str, data_len_str, speed_str, eta_str))
-        self._report_progress_status(msg)
-
-    def report_progress_live_stream(self, downloaded_data_len, speed, elapsed):
-        if self.params.get('noprogress', False):
+        if s['status'] != 'downloading':
             return
-        downloaded_str = format_bytes(downloaded_data_len)
-        speed_str = self.format_speed(speed)
-        elapsed_str = FileDownloader.format_seconds(elapsed)
-        msg = '%s at %s (%s)' % (downloaded_str, speed_str, elapsed_str)
-        self._report_progress_status(msg)
 
-    def report_finish(self, data_len_str, tot_time):
-        """Report download finished."""
-        if self.params.get('noprogress', False):
-            self.to_screen('[download] Download completed')
+        if s.get('eta') is not None:
+            s['_eta_str'] = self.format_eta(s['eta'])
         else:
-            self._report_progress_status(
-                ('100%% of %s in %s' %
-                 (data_len_str, self.format_seconds(tot_time))),
-                is_last_line=True)
+            s['_eta_str'] = 'Unknown ETA'
+
+        if s.get('total_bytes') and s.get('downloaded_bytes') is not None:
+            s['_percent_str'] = self.format_percent(100 * s['downloaded_bytes'] / s['total_bytes'])
+        elif s.get('total_bytes_estimate') and s.get('downloaded_bytes') is not None:
+            s['_percent_str'] = self.format_percent(100 * s['downloaded_bytes'] / s['total_bytes_estimate'])
+        else:
+            if s.get('downloaded_bytes') == 0:
+                s['_percent_str'] = self.format_percent(0)
+            else:
+                s['_percent_str'] = 'Unknown %'
+
+        if s.get('speed') is not None:
+            s['_speed_str'] = self.format_speed(s['speed'])
+        else:
+            s['_speed_str'] = 'Unknown speed'
+
+        if s.get('total_bytes') is not None:
+            s['_total_bytes_str'] = format_bytes(s['total_bytes'])
+            msg_template = '%(_percent_str)s of %(_total_bytes_str)s at %(_speed_str)s ETA %(_eta_str)s'
+        elif s.get('total_bytes_estimate') is not None:
+            s['_total_bytes_estimate_str'] = format_bytes(s['total_bytes_estimate'])
+            msg_template = '%(_percent_str)s of ~%(_total_bytes_estimate_str)s at %(_speed_str)s ETA %(_eta_str)s'
+        else:
+            if s.get('downloaded_bytes') is not None:
+                s['_downloaded_bytes_str'] = format_bytes(s['downloaded_bytes'])
+                if s.get('elapsed'):
+                    s['_elapsed_str'] = self.format_seconds(s['elapsed'])
+                    msg_template = '%(_downloaded_bytes_str)s at %(_speed_str)s (%(_elapsed_str)s)'
+                else:
+                    msg_template = '%(_downloaded_bytes_str)s at %(_speed_str)s'
+            else:
+                msg_template = '%(_percent_str)s % at %(_speed_str)s ETA %(_eta_str)s'
+
+        self._report_progress_status(msg_template % s)
 
     def report_resuming_byte(self, resume_len):
         """Report attempt to resume at given byte."""
diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index 0e7a1c200..5a1f8e680 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -1,4 +1,4 @@
-from __future__ import unicode_literals
+from __future__ import division, unicode_literals
 
 import base64
 import io
@@ -252,17 +252,6 @@ class F4mFD(FileDownloader):
         requested_bitrate = info_dict.get('tbr')
         self.to_screen('[download] Downloading f4m manifest')
         manifest = self.ydl.urlopen(man_url).read()
-        self.report_destination(filename)
-        http_dl = HttpQuietDownloader(
-            self.ydl,
-            {
-                'continuedl': True,
-                'quiet': True,
-                'noprogress': True,
-                'ratelimit': self.params.get('ratelimit', None),
-                'test': self.params.get('test', False),
-            }
-        )
 
         doc = etree.fromstring(manifest)
         formats = [(int(f.attrib.get('bitrate', -1)), f)
@@ -298,39 +287,67 @@ class F4mFD(FileDownloader):
         # For some akamai manifests we'll need to add a query to the fragment url
         akamai_pv = xpath_text(doc, _add_ns('pv-2.0'))
 
+        self.report_destination(filename)
+        http_dl = HttpQuietDownloader(
+            self.ydl,
+            {
+                'continuedl': True,
+                'quiet': True,
+                'noprogress': True,
+                'ratelimit': self.params.get('ratelimit', None),
+                'test': self.params.get('test', False),
+            }
+        )
         tmpfilename = self.temp_name(filename)
         (dest_stream, tmpfilename) = sanitize_open(tmpfilename, 'wb')
+
         write_flv_header(dest_stream)
         write_metadata_tag(dest_stream, metadata)
 
         # This dict stores the download progress, it's updated by the progress
         # hook
         state = {
+            'status': 'downloading',
             'downloaded_bytes': 0,
-            'frag_counter': 0,
+            'frag_index': 0,
+            'frag_count': total_frags,
+            'filename': filename,
+            'tmpfilename': tmpfilename,
         }
         start = time.time()
 
-        def frag_progress_hook(status):
-            frag_total_bytes = status.get('total_bytes', 0)
-            estimated_size = (state['downloaded_bytes'] +
-                              (total_frags - state['frag_counter']) * frag_total_bytes)
-            if status['status'] == 'finished':
+        def frag_progress_hook(s):
+            if s['status'] not in ('downloading', 'finished'):
+                return
+
+            frag_total_bytes = s.get('total_bytes', 0)
+            if s['status'] == 'finished':
                 state['downloaded_bytes'] += frag_total_bytes
-                state['frag_counter'] += 1
-                progress = self.calc_percent(state['frag_counter'], total_frags)
+                state['frag_index'] += 1
+
+            estimated_size = (
+                (state['downloaded_bytes'] + frag_total_bytes)
+                / (state['frag_index'] + 1) * total_frags)
+            time_now = time.time()
+            state['total_bytes_estimate'] = estimated_size
+            state['elapsed'] = time_now - start
+
+            if s['status'] == 'finished':
+                progress = self.calc_percent(state['frag_index'], total_frags)
                 byte_counter = state['downloaded_bytes']
             else:
-                frag_downloaded_bytes = status['downloaded_bytes']
+                frag_downloaded_bytes = s['downloaded_bytes']
                 byte_counter = state['downloaded_bytes'] + frag_downloaded_bytes
                 frag_progress = self.calc_percent(frag_downloaded_bytes,
                                                   frag_total_bytes)
-                progress = self.calc_percent(state['frag_counter'], total_frags)
+                progress = self.calc_percent(state['frag_index'], total_frags)
                 progress += frag_progress / float(total_frags)
 
-            eta = self.calc_eta(start, time.time(), estimated_size, byte_counter)
-            self.report_progress(progress, format_bytes(estimated_size),
-                                 status.get('speed'), eta)
+                state['eta'] = self.calc_eta(
+                    start, time_now, estimated_size, state['downloaded_bytes'] + frag_downloaded_bytes)
+                state['speed'] = s.get('speed')
+            self._hook_progress(state)
+
         http_dl.add_progress_hook(frag_progress_hook)
 
         frags_filenames = []
@@ -354,8 +371,8 @@ class F4mFD(FileDownloader):
             frags_filenames.append(frag_filename)
 
         dest_stream.close()
-        self.report_finish(format_bytes(state['downloaded_bytes']), time.time() - start)
 
+        elapsed = time.time() - start
         self.try_rename(tmpfilename, filename)
         for frag_file in frags_filenames:
             os.remove(frag_file)
@@ -366,6 +383,7 @@ class F4mFD(FileDownloader):
             'total_bytes': fsize,
             'filename': filename,
             'status': 'finished',
+            'elapsed': elapsed,
         })
 
         return True
diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index 49170cf9d..d37522aea 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -200,16 +200,16 @@ class HttpFD(FileDownloader):
             else:
                 percent = self.calc_percent(byte_counter, data_len)
                 eta = self.calc_eta(start, time.time(), data_len - resume_len, byte_counter - resume_len)
-            self.report_progress(percent, data_len_str, speed, eta)
 
             self._hook_progress({
+                'status': 'downloading',
                 'downloaded_bytes': byte_counter,
                 'total_bytes': data_len,
                 'tmpfilename': tmpfilename,
                 'filename': filename,
-                'status': 'downloading',
                 'eta': eta,
                 'speed': speed,
+                'elapsed': now - start,
             })
 
             if is_test and byte_counter == data_len:
@@ -221,7 +221,13 @@ class HttpFD(FileDownloader):
             return False
         if tmpfilename != '-':
             stream.close()
-        self.report_finish(data_len_str, (time.time() - start))
+
+        self._hook_progress({
+            'downloaded_bytes': byte_counter,
+            'total_bytes': data_len,
+            'tmpfilename': tmpfilename,
+            'status': 'error',
+        })
         if data_len is not None and byte_counter != data_len:
             raise ContentTooShortError(byte_counter, int(data_len))
         self.try_rename(tmpfilename, filename)
@@ -235,6 +241,7 @@ class HttpFD(FileDownloader):
             'total_bytes': byte_counter,
             'filename': filename,
             'status': 'finished',
+            'elapsed': time.time() - start,
         })
 
         return True
diff --git a/youtube_dl/downloader/rtmp.py b/youtube_dl/downloader/rtmp.py
index f7eeb6f43..79836fe99 100644
--- a/youtube_dl/downloader/rtmp.py
+++ b/youtube_dl/downloader/rtmp.py
@@ -51,23 +51,23 @@ class RtmpFD(FileDownloader):
                     if not resume_percent:
                         resume_percent = percent
                         resume_downloaded_data_len = downloaded_data_len
-                    eta = self.calc_eta(start, time.time(), 100 - resume_percent, percent - resume_percent)
-                    speed = self.calc_speed(start, time.time(), downloaded_data_len - resume_downloaded_data_len)
+                    time_now = time.time()
+                    eta = self.calc_eta(start, time_now, 100 - resume_percent, percent - resume_percent)
+                    speed = self.calc_speed(start, time_now, downloaded_data_len - resume_downloaded_data_len)
                     data_len = None
                     if percent > 0:
                         data_len = int(downloaded_data_len * 100 / percent)
-                    data_len_str = '~' + format_bytes(data_len)
-                    self.report_progress(percent, data_len_str, speed, eta)
-                    cursor_in_new_line = False
                     self._hook_progress({
+                        'status': 'downloading',
                         'downloaded_bytes': downloaded_data_len,
-                        'total_bytes': data_len,
+                        'total_bytes_estimate': data_len,
                         'tmpfilename': tmpfilename,
                         'filename': filename,
-                        'status': 'downloading',
                         'eta': eta,
+                        'elapsed': time_now - start,
                         'speed': speed,
                     })
+                    cursor_in_new_line = False
                 else:
                     # no percent for live streams
                     mobj = re.search(r'([0-9]+\.[0-9]{3}) kB / [0-9]+\.[0-9]{2} sec', line)
@@ -75,15 +75,15 @@ class RtmpFD(FileDownloader):
                         downloaded_data_len = int(float(mobj.group(1)) * 1024)
                         time_now = time.time()
                         speed = self.calc_speed(start, time_now, downloaded_data_len)
-                        self.report_progress_live_stream(downloaded_data_len, speed, time_now - start)
-                        cursor_in_new_line = False
                         self._hook_progress({
                             'downloaded_bytes': downloaded_data_len,
                             'tmpfilename': tmpfilename,
                             'filename': filename,
                             'status': 'downloading',
+                            'elapsed': time_now - start,
                             'speed': speed,
                         })
+                        cursor_in_new_line = False
                     elif self.params.get('verbose', False):
                         if not cursor_in_new_line:
                             self.to_screen('')

From 16e7711e22648027739096560914a976b8eea786 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 17 Feb 2015 21:42:31 +0100
Subject: [PATCH 0389/6219] [downloader/http] Remove gruesome import

---
 youtube_dl/downloader/http.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index 49170cf9d..25032ad4e 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -1,10 +1,9 @@
 from __future__ import unicode_literals
 
-import os
-import time
-
-from socket import error as SocketError
 import errno
+import os
+import socket
+import time
 
 from .common import FileDownloader
 from ..compat import (
@@ -102,7 +101,7 @@ class HttpFD(FileDownloader):
                             resume_len = 0
                             open_mode = 'wb'
                             break
-            except SocketError as e:
+            except socket.error as e:
                 if e.errno != errno.ECONNRESET:
                     # Connection reset is no problem, just retry
                     raise

From a91a2c1a83fdd195e850d4ad9c298c01a145ebf0 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 17 Feb 2015 21:44:41 +0100
Subject: [PATCH 0390/6219] [downloader] Remove various unneeded assignments
 and imports

---
 youtube_dl/downloader/f4m.py  | 3 ---
 youtube_dl/downloader/http.py | 5 +----
 youtube_dl/downloader/rtmp.py | 1 -
 3 files changed, 1 insertion(+), 8 deletions(-)

diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index 5a1f8e680..b40ebfa50 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -15,7 +15,6 @@ from ..compat import (
 from ..utils import (
     struct_pack,
     struct_unpack,
-    format_bytes,
     encodeFilename,
     sanitize_open,
     xpath_text,
@@ -334,10 +333,8 @@ class F4mFD(FileDownloader):
 
             if s['status'] == 'finished':
                 progress = self.calc_percent(state['frag_index'], total_frags)
-                byte_counter = state['downloaded_bytes']
             else:
                 frag_downloaded_bytes = s['downloaded_bytes']
-                byte_counter = state['downloaded_bytes'] + frag_downloaded_bytes
                 frag_progress = self.calc_percent(frag_downloaded_bytes,
                                                   frag_total_bytes)
                 progress = self.calc_percent(state['frag_index'], total_frags)
diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index d37522aea..6dec528f2 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -15,7 +15,6 @@ from ..utils import (
     ContentTooShortError,
     encodeFilename,
     sanitize_open,
-    format_bytes,
 )
 
 
@@ -137,7 +136,6 @@ class HttpFD(FileDownloader):
                 self.to_screen('\r[download] File is larger than max-filesize (%s bytes > %s bytes). Aborting.' % (data_len, max_data_len))
                 return False
 
-        data_len_str = format_bytes(data_len)
         byte_counter = 0 + resume_len
         block_size = self.params.get('buffersize', 1024)
         start = time.time()
@@ -196,9 +194,8 @@ class HttpFD(FileDownloader):
             # Progress message
             speed = self.calc_speed(start, now, byte_counter - resume_len)
             if data_len is None:
-                eta = percent = None
+                eta = None
             else:
-                percent = self.calc_percent(byte_counter, data_len)
                 eta = self.calc_eta(start, time.time(), data_len - resume_len, byte_counter - resume_len)
 
             self._hook_progress({
diff --git a/youtube_dl/downloader/rtmp.py b/youtube_dl/downloader/rtmp.py
index 79836fe99..0a52c34c7 100644
--- a/youtube_dl/downloader/rtmp.py
+++ b/youtube_dl/downloader/rtmp.py
@@ -11,7 +11,6 @@ from ..compat import compat_str
 from ..utils import (
     check_executable,
     encodeFilename,
-    format_bytes,
     get_exe_version,
 )
 

From fa156077737d5162795a221fe2995a276d31c6d3 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 17 Feb 2015 21:46:20 +0100
Subject: [PATCH 0391/6219] PEP8 fixes

---
 youtube_dl/extractor/common.py           | 3 +--
 youtube_dl/extractor/twitch.py           | 1 +
 youtube_dl/extractor/videolecturesnet.py | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 31599bf52..08b8ad37c 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -27,7 +27,6 @@ from ..utils import (
     compiled_regex_type,
     ExtractorError,
     float_or_none,
-    HEADRequest,
     int_or_none,
     RegexNotFoundError,
     sanitize_filename,
@@ -839,7 +838,7 @@ class InfoExtractor(object):
             note='Downloading m3u8 information',
             errnote='Failed to download m3u8 information')
         last_info = None
-        last_media= None
+        last_media = None
         kv_rex = re.compile(
             r'(?P<key>[a-zA-Z_-]+)=(?P<val>"[^"]+"|[^",]+)(?:,|$)')
         for line in m3u8_doc.splitlines():
diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 8e296698e..4b0d8988d 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -348,6 +348,7 @@ class TwitchStreamIE(TwitchBaseIE):
             '%s/api/channel/hls/%s.m3u8?%s'
             % (self._USHER_BASE, channel_id, compat_urllib_parse.urlencode(query).encode('utf-8')),
             channel_id, 'mp4')
+
         # prefer the 'source' stream, the others are limited to 30 fps
         def _sort_source(f):
             if f.get('m3u8_media') is not None and f['m3u8_media'].get('NAME') == 'Source':
diff --git a/youtube_dl/extractor/videolecturesnet.py b/youtube_dl/extractor/videolecturesnet.py
index d46234856..d6a7eb203 100644
--- a/youtube_dl/extractor/videolecturesnet.py
+++ b/youtube_dl/extractor/videolecturesnet.py
@@ -52,7 +52,7 @@ class VideoLecturesNetIE(InfoExtractor):
         formats = []
         for v in switch.findall('./video'):
             proto = v.attrib.get('proto')
-            if not proto in ['http', 'rtmp']:
+            if proto not in ['http', 'rtmp']:
                 continue
             f = {
                 'width': int_or_none(v.attrib.get('width')),

From 85920dd01d98cf74ea7d3ab7834a3b50cd6f1fde Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 17 Feb 2015 21:56:25 +0100
Subject: [PATCH 0392/6219] [bliptv] Convert to new subtitles system

---
 test/test_subtitles.py         |  2 --
 youtube_dl/extractor/bliptv.py | 34 ++++++++++++++++++----------------
 2 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/test/test_subtitles.py b/test/test_subtitles.py
index 91cebce28..0ca510310 100644
--- a/test/test_subtitles.py
+++ b/test/test_subtitles.py
@@ -200,13 +200,11 @@ class TestBlipTVSubtitles(BaseTestSubtitles):
     IE = BlipTVIE
 
     def test_list_subtitles(self):
-        self.DL.expect_warning('Automatic Captions not supported by this server')
         self.DL.params['listsubtitles'] = True
         info_dict = self.getInfoDict()
         self.assertEqual(info_dict, None)
 
     def test_allsubtitles(self):
-        self.DL.expect_warning('Automatic Captions not supported by this server')
         self.DL.params['writesubtitles'] = True
         self.DL.params['allsubtitles'] = True
         subtitles = self.getSubtitles()
diff --git a/youtube_dl/extractor/bliptv.py b/youtube_dl/extractor/bliptv.py
index 436cc5155..8c7ba4b91 100644
--- a/youtube_dl/extractor/bliptv.py
+++ b/youtube_dl/extractor/bliptv.py
@@ -3,7 +3,6 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from .subtitles import SubtitlesInfoExtractor
 
 from ..compat import (
     compat_str,
@@ -18,7 +17,7 @@ from ..utils import (
 )
 
 
-class BlipTVIE(SubtitlesInfoExtractor):
+class BlipTVIE(InfoExtractor):
     _VALID_URL = r'https?://(?:\w+\.)?blip\.tv/(?:(?:.+-|rss/flash/)(?P<id>\d+)|((?:play/|api\.swf#)(?P<lookup_id>[\da-zA-Z+_]+)))'
 
     _TESTS = [
@@ -143,7 +142,7 @@ class BlipTVIE(SubtitlesInfoExtractor):
         categories = [category.text for category in item.findall('category')]
 
         formats = []
-        subtitles = {}
+        subtitles_urls = {}
 
         media_group = item.find(media('group'))
         for media_content in media_group.findall(media('content')):
@@ -161,7 +160,7 @@ class BlipTVIE(SubtitlesInfoExtractor):
                 }
                 lang = role.rpartition('-')[-1].strip().lower()
                 langcode = LANGS.get(lang, lang)
-                subtitles[langcode] = url
+                subtitles_urls[langcode] = url
             elif media_type.startswith('video/'):
                 formats.append({
                     'url': real_url,
@@ -175,11 +174,7 @@ class BlipTVIE(SubtitlesInfoExtractor):
                 })
         self._sort_formats(formats)
 
-        # subtitles
-        video_subtitles = self.extract_subtitles(video_id, subtitles)
-        if self._downloader.params.get('listsubtitles', False):
-            self._list_available_subtitles(video_id, subtitles)
-            return
+        subtitles = self.extract_subtitles(video_id, subtitles_urls)
 
         return {
             'id': video_id,
@@ -192,15 +187,22 @@ class BlipTVIE(SubtitlesInfoExtractor):
             'thumbnail': thumbnail,
             'categories': categories,
             'formats': formats,
-            'subtitles': video_subtitles,
+            'subtitles': subtitles,
         }
 
-    def _download_subtitle_url(self, sub_lang, url):
-        # For some weird reason, blip.tv serves a video instead of subtitles
-        # when we request with a common UA
-        req = compat_urllib_request.Request(url)
-        req.add_header('User-Agent', 'youtube-dl')
-        return self._download_webpage(req, None, note=False)
+    def _get_subtitles(self, video_id, subtitles_urls):
+        subtitles = {}
+        for lang, url in subtitles_urls.items():
+            # For some weird reason, blip.tv serves a video instead of subtitles
+            # when we request with a common UA
+            req = compat_urllib_request.Request(url)
+            req.add_header('User-Agent', 'youtube-dl')
+            subtitles[lang] = [{
+                # The extension is 'srt' but it's actually an 'ass' file
+                'ext': 'ass',
+                'data': self._download_webpage(req, None, note=False),
+            }]
+        return subtitles
 
 
 class BlipTVUserIE(InfoExtractor):

From 9868ea493626a3a81d30d084fd00d22982a0f86a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 17 Feb 2015 22:16:29 +0100
Subject: [PATCH 0393/6219] [extractor/common] Simplify subtitles handling
 methods

Initially I was going to use a single method for handling both subtitles and automatic captions, that's why I used the 'list_subtitles' and the 'subtitles' variables.
---
 youtube_dl/extractor/common.py | 18 ++++++++----------
 1 file changed, 8 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index fe7d8dbc9..7d8ce1808 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1000,21 +1000,19 @@ class InfoExtractor(object):
         return not any_restricted
 
     def extract_subtitles(self, *args, **kwargs):
-        subtitles = {}
-        list_subtitles = self._downloader.params.get('listsubtitles')
-        if self._downloader.params.get('writesubtitles', False) or list_subtitles:
-            subtitles.update(self._get_subtitles(*args, **kwargs))
-        return subtitles
+        if (self._downloader.params.get('writesubtitles', False) or
+                self._downloader.params.get('listsubtitles')):
+            return self._get_subtitles(*args, **kwargs)
+        return {}
 
     def _get_subtitles(self, *args, **kwargs):
         raise NotImplementedError("This method must be implemented by subclasses")
 
     def extract_automatic_captions(self, *args, **kwargs):
-        automatic_captions = {}
-        list_subtitles = self._downloader.params.get('listsubtitles')
-        if self._downloader.params.get('writeautomaticsub', False) or list_subtitles:
-            automatic_captions.update(self._get_automatic_captions(*args, **kwargs))
-        return automatic_captions
+        if (self._downloader.params.get('writeautomaticsub', False) or
+                self._downloader.params.get('listsubtitles')):
+            return self._get_automatic_captions(*args, **kwargs)
+        return {}
 
     def _get_automatic_captions(self, *args, **kwargs):
         raise NotImplementedError("This method must be implemented by subclasses")

From edab9dbf4d00a7f76fbfd2df9ef4b205c88e47a8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 17 Feb 2015 22:59:19 +0100
Subject: [PATCH 0394/6219] [YoutubeDL] use the 'render_table' function for
 listing the subtitles

---
 youtube_dl/YoutubeDL.py | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index a47f8f5de..f8b8fb0c1 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1659,13 +1659,12 @@ class YoutubeDL(object):
         if not subtitles:
             self.to_screen('%s has no %s' % (video_id, name))
             return
-        header_line = 'Language    formats'
-        sub_lines = [
-            '%-12s%s' % (lang, ', '.join(f['ext'] for f in reversed(formats)))
-            for lang, formats in subtitles.items()]
         self.to_screen(
-            'Available %s for %s:\n%s\n%s' %
-            (name, video_id, header_line, '\n'.join(sub_lines)))
+            'Available %s for %s:' % (name, video_id))
+        self.to_screen(render_table(
+            ['Language', 'formats'],
+            [[lang, ', '.join(f['ext'] for f in reversed(formats))]
+                for lang, formats in subtitles.items()]))
 
     def urlopen(self, req):
         """ Start an HTTP download """

From 30965ac66a4e33b8e70d6c0cf5da1a746054d6cd Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 18 Feb 2015 00:27:57 +0100
Subject: [PATCH 0395/6219] [vimeo] Prevent infinite loops if video password
 verification fails
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

We're seeing this in the tests¹ right now, which do not terminate.

¹  https://travis-ci.org/jaimeMF/youtube-dl/jobs/51135858
---
 youtube_dl/extractor/vimeo.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 303e81447..1959d5e36 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -18,6 +18,7 @@ from ..utils import (
     InAdvancePagedList,
     int_or_none,
     RegexNotFoundError,
+    smuggle_url,
     std_headers,
     unsmuggle_url,
     urlencode_postdata,
@@ -267,8 +268,11 @@ class VimeoIE(VimeoBaseInfoExtractor, SubtitlesInfoExtractor):
                 raise ExtractorError('The author has restricted the access to this video, try with the "--referer" option')
 
             if re.search(r'<form[^>]+?id="pw_form"', webpage) is not None:
+                if data and '_video_password_verified' in data:
+                    raise ExtractorError('video password verification failed!')
                 self._verify_video_password(url, video_id, webpage)
-                return self._real_extract(url)
+                return self._real_extract(
+                    smuggle_url(url, {'_video_password_verified': 'verified'}))
             else:
                 raise ExtractorError('Unable to extract info section',
                                      cause=e)

From a3fa5da4967bf2699bfd982cf56a8f627a13bda4 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 18 Feb 2015 00:33:31 +0100
Subject: [PATCH 0396/6219] [vimeo] Amend playlist tests

---
 youtube_dl/extractor/vimeo.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 1959d5e36..e7284049d 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -405,6 +405,7 @@ class VimeoChannelIE(InfoExtractor):
     _TESTS = [{
         'url': 'http://vimeo.com/channels/tributes',
         'info_dict': {
+            'id': 'tributes',
             'title': 'Vimeo Tributes',
         },
         'playlist_mincount': 25,
@@ -483,6 +484,7 @@ class VimeoUserIE(VimeoChannelIE):
         'url': 'http://vimeo.com/nkistudio/videos',
         'info_dict': {
             'title': 'Nki',
+            'id': 'nkistudio',
         },
         'playlist_mincount': 66,
     }]
@@ -500,6 +502,7 @@ class VimeoAlbumIE(VimeoChannelIE):
     _TESTS = [{
         'url': 'http://vimeo.com/album/2632481',
         'info_dict': {
+            'id': '2632481',
             'title': 'Staff Favorites: November 2013',
         },
         'playlist_mincount': 13,
@@ -530,6 +533,7 @@ class VimeoGroupsIE(VimeoAlbumIE):
     _TESTS = [{
         'url': 'http://vimeo.com/groups/rolexawards',
         'info_dict': {
+            'id': 'rolexawards',
             'title': 'Rolex Awards for Enterprise',
         },
         'playlist_mincount': 73,
@@ -612,6 +616,7 @@ class VimeoLikesIE(InfoExtractor):
         'url': 'https://vimeo.com/user755559/likes/',
         'playlist_mincount': 293,
         "info_dict": {
+            'id': 'user755559_likes',
             "description": "See all the videos urza likes",
             "title": 'Videos urza likes',
         },

From 15ec669374727c9bdc3aad0df141fdf305cb80be Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 18 Feb 2015 00:33:41 +0100
Subject: [PATCH 0397/6219] [vk] Amend playlist test

---
 youtube_dl/extractor/vk.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 81e02a624..7dea8c59d 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -217,6 +217,9 @@ class VKUserVideosIE(InfoExtractor):
     _TEMPLATE_URL = 'https://vk.com/videos'
     _TEST = {
         'url': 'http://vk.com/videos205387401',
+        'info_dict': {
+            'id': '205387401',
+        },
         'playlist_mincount': 4,
     }
 

From 1def5f359ef6e197e4cc5d610ed86e109e240080 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 18 Feb 2015 00:34:45 +0100
Subject: [PATCH 0398/6219] [livestream] Correct playlist ID and add a test for
 it

---
 youtube_dl/extractor/livestream.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/livestream.py b/youtube_dl/extractor/livestream.py
index 5247c6f58..3642089f7 100644
--- a/youtube_dl/extractor/livestream.py
+++ b/youtube_dl/extractor/livestream.py
@@ -37,6 +37,7 @@ class LivestreamIE(InfoExtractor):
         'url': 'http://new.livestream.com/tedx/cityenglish',
         'info_dict': {
             'title': 'TEDCity2.0 (English)',
+            'id': '2245590',
         },
         'playlist_mincount': 4,
     }, {
@@ -148,7 +149,8 @@ class LivestreamIE(InfoExtractor):
                   if is_relevant(video_data, video_id)]
         if video_id is None:
             # This is an event page:
-            return self.playlist_result(videos, info['id'], info['full_name'])
+            return self.playlist_result(
+                videos, '%s' % info['id'], info['full_name'])
         else:
             if not videos:
                 raise ExtractorError('Cannot find video %s' % video_id)

From 6f53c63df66d95ea684f84ef2f7f0e697acc1d20 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 18 Feb 2015 00:37:48 +0100
Subject: [PATCH 0399/6219] [test/helper] Only output a newline for forgotten
 keys if keys are really missing

---
 test/helper.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/test/helper.py b/test/helper.py
index 651ef99b9..fef2db284 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -163,7 +163,9 @@ def expect_info_dict(self, got_dict, expected_dict):
             info_dict_str += ''.join(
                 '    %s: %s,\n' % (_repr(k), _repr(v))
                 for k, v in test_info_dict.items() if k not in missing_keys)
-            info_dict_str += '\n'
+
+            if info_dict_str:
+                info_dict_str += '\n'
         info_dict_str += ''.join(
             '    %s: %s,\n' % (_repr(k), _repr(test_info_dict[k]))
             for k in missing_keys)

From f21e915fb9ad53a62df65e5c75c984243aaf0aea Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 18 Feb 2015 00:38:42 +0100
Subject: [PATCH 0400/6219] [test/helper] Render info_dict with a final comma

---
 test/helper.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/helper.py b/test/helper.py
index fef2db284..570c91440 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -170,7 +170,7 @@ def expect_info_dict(self, got_dict, expected_dict):
             '    %s: %s,\n' % (_repr(k), _repr(test_info_dict[k]))
             for k in missing_keys)
         write_string(
-            '\n\'info_dict\': {\n' + info_dict_str + '}\n', out=sys.stderr)
+            '\n\'info_dict\': {\n' + info_dict_str + '},\n', out=sys.stderr)
         self.assertFalse(
             missing_keys,
             'Missing keys in test definition: %s' % (

From 4d278fde645a5ab70b046293a26c12bba146acb0 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 18 Feb 2015 00:38:55 +0100
Subject: [PATCH 0401/6219] [ign] Amend playlist test

---
 youtube_dl/extractor/ign.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/ign.py b/youtube_dl/extractor/ign.py
index 3db668cd0..3aade9e74 100644
--- a/youtube_dl/extractor/ign.py
+++ b/youtube_dl/extractor/ign.py
@@ -34,6 +34,9 @@ class IGNIE(InfoExtractor):
         },
         {
             'url': 'http://me.ign.com/en/feature/15775/100-little-things-in-gta-5-that-will-blow-your-mind',
+            'info_dict': {
+                'id': '100-little-things-in-gta-5-that-will-blow-your-mind',
+            },
             'playlist': [
                 {
                     'info_dict': {

From 6e99868e4c73468734f3552d430ba6375c4e0880 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 18 Feb 2015 00:41:45 +0100
Subject: [PATCH 0402/6219] [buzzfeed] Fix playlist test case

---
 youtube_dl/extractor/buzzfeed.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/buzzfeed.py b/youtube_dl/extractor/buzzfeed.py
index a5d2af174..df503ecc0 100644
--- a/youtube_dl/extractor/buzzfeed.py
+++ b/youtube_dl/extractor/buzzfeed.py
@@ -33,6 +33,7 @@ class BuzzFeedIE(InfoExtractor):
             'skip_download': True,  # Got enough YouTube download tests
         },
         'info_dict': {
+            'id': 'look-at-this-cute-dog-omg',
             'description': 're:Munchkin the Teddy Bear is back ?!',
             'title': 'You Need To Stop What You\'re Doing And Watching This Dog Walk On A Treadmill',
         },
@@ -42,8 +43,8 @@ class BuzzFeedIE(InfoExtractor):
                 'ext': 'mp4',
                 'upload_date': '20141124',
                 'uploader_id': 'CindysMunchkin',
-                'description': 're:© 2014 Munchkin the Shih Tzu',
-                'uploader': 'Munchkin the Shih Tzu',
+                'description': 're:© 2014 Munchkin the',
+                'uploader': 're:^Munchkin the',
                 'title': 're:Munchkin the Teddy Bear gets her exercise',
             },
         }]

From 72c1f8de06f11aea670039a17ca63146a1b85af2 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 18 Feb 2015 00:48:52 +0100
Subject: [PATCH 0403/6219] [bandcamp:album] Fix extractor results and
 associated test

---
 youtube_dl/extractor/bandcamp.py | 20 +++++++++++++-------
 1 file changed, 13 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/bandcamp.py b/youtube_dl/extractor/bandcamp.py
index 490cc961a..869294967 100644
--- a/youtube_dl/extractor/bandcamp.py
+++ b/youtube_dl/extractor/bandcamp.py
@@ -109,7 +109,7 @@ class BandcampIE(InfoExtractor):
 
 class BandcampAlbumIE(InfoExtractor):
     IE_NAME = 'Bandcamp:album'
-    _VALID_URL = r'https?://(?:(?P<subdomain>[^.]+)\.)?bandcamp\.com(?:/album/(?P<title>[^?#]+)|/?(?:$|[?#]))'
+    _VALID_URL = r'https?://(?:(?P<subdomain>[^.]+)\.)?bandcamp\.com(?:/album/(?P<album_id>[^?#]+)|/?(?:$|[?#]))'
 
     _TESTS = [{
         'url': 'http://blazo.bandcamp.com/album/jazz-format-mixtape-vol-1',
@@ -133,31 +133,37 @@ class BandcampAlbumIE(InfoExtractor):
         ],
         'info_dict': {
             'title': 'Jazz Format Mixtape vol.1',
+            'id': 'jazz-format-mixtape-vol-1',
+            'uploader_id': 'blazo',
         },
         'params': {
             'playlistend': 2
         },
-        'skip': 'Bandcamp imposes download limits. See test_playlists:test_bandcamp_album for the playlist test'
+        'skip': 'Bandcamp imposes download limits.'
     }, {
         'url': 'http://nightbringer.bandcamp.com/album/hierophany-of-the-open-grave',
         'info_dict': {
             'title': 'Hierophany of the Open Grave',
+            'uploader_id': 'nightbringer',
+            'id': 'hierophany-of-the-open-grave',
         },
         'playlist_mincount': 9,
     }, {
         'url': 'http://dotscale.bandcamp.com',
         'info_dict': {
             'title': 'Loom',
+            'id': 'dotscale',
+            'uploader_id': 'dotscale',
         },
         'playlist_mincount': 7,
     }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        playlist_id = mobj.group('subdomain')
-        title = mobj.group('title')
-        display_id = title or playlist_id
-        webpage = self._download_webpage(url, display_id)
+        uploader_id = mobj.group('subdomain')
+        album_id = mobj.group('album_id')
+        playlist_id = album_id or uploader_id
+        webpage = self._download_webpage(url, playlist_id)
         tracks_paths = re.findall(r'<a href="(.*?)" itemprop="url">', webpage)
         if not tracks_paths:
             raise ExtractorError('The page doesn\'t contain any tracks')
@@ -168,8 +174,8 @@ class BandcampAlbumIE(InfoExtractor):
             r'album_title\s*:\s*"(.*?)"', webpage, 'title', fatal=False)
         return {
             '_type': 'playlist',
+            'uploader_id': uploader_id,
             'id': playlist_id,
-            'display_id': display_id,
             'title': title,
             'entries': entries,
         }

From 11e611a7fa809a447939e8a98c8549f2827c862f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 18 Feb 2015 00:49:10 +0100
Subject: [PATCH 0404/6219] Extend various playlist tests

---
 youtube_dl/extractor/adultswim.py     | 2 ++
 youtube_dl/extractor/appletrailers.py | 3 +++
 youtube_dl/extractor/dailymotion.py   | 1 +
 youtube_dl/extractor/generic.py       | 1 +
 4 files changed, 7 insertions(+)

diff --git a/youtube_dl/extractor/adultswim.py b/youtube_dl/extractor/adultswim.py
index 502a9c25a..34b8b0115 100644
--- a/youtube_dl/extractor/adultswim.py
+++ b/youtube_dl/extractor/adultswim.py
@@ -38,6 +38,7 @@ class AdultSwimIE(InfoExtractor):
             },
         ],
         'info_dict': {
+            'id': 'rQxZvXQ4ROaSOqq-or2Mow',
             'title': 'Rick and Morty - Pilot',
             'description': "Rick moves in with his daughter's family and establishes himself as a bad influence on his grandson, Morty. "
         }
@@ -55,6 +56,7 @@ class AdultSwimIE(InfoExtractor):
             }
         ],
         'info_dict': {
+            'id': '-t8CamQlQ2aYZ49ItZCFog',
             'title': 'American Dad - Putting Francine Out of Business',
             'description': 'Stan hatches a plan to get Francine out of the real estate business.Watch more American Dad on [adult swim].'
         },
diff --git a/youtube_dl/extractor/appletrailers.py b/youtube_dl/extractor/appletrailers.py
index 287f71e07..43e82847f 100644
--- a/youtube_dl/extractor/appletrailers.py
+++ b/youtube_dl/extractor/appletrailers.py
@@ -14,6 +14,9 @@ class AppleTrailersIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?trailers\.apple\.com/trailers/(?P<company>[^/]+)/(?P<movie>[^/]+)'
     _TEST = {
         "url": "http://trailers.apple.com/trailers/wb/manofsteel/",
+        'info_dict': {
+            'id': 'manofsteel',
+        },
         "playlist": [
             {
                 "md5": "d97a8e575432dbcb81b7c3acb741f8a8",
diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index cf5841a7c..b2dbf4a92 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -194,6 +194,7 @@ class DailymotionPlaylistIE(DailymotionBaseInfoExtractor):
         'url': 'http://www.dailymotion.com/playlist/xv4bw_nqtv_sport/1#video=xl8v3q',
         'info_dict': {
             'title': 'SPORT',
+            'id': 'xv4bw_nqtv_sport',
         },
         'playlist_mincount': 20,
     }]
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index c4ba25a96..36a1f65bf 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -473,6 +473,7 @@ class GenericIE(InfoExtractor):
         {
             'url': 'http://discourse.ubuntu.com/t/unity-8-desktop-mode-windows-on-mir/1986',
             'info_dict': {
+                'id': '1986',
                 'title': 'Unity 8 desktop-mode windows on Mir! - Ubuntu Discourse',
             },
             'playlist_mincount': 2,

From 9cad27008b88b5ba3dfb1d747d7476b05582f385 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 18 Feb 2015 00:49:34 +0100
Subject: [PATCH 0405/6219] release 2015.02.18

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 34a8d1184..635b6bdb8 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.17.2'
+__version__ = '2015.02.18'

From 6d1a55a521b524dd0c562e3855f81434afb6f807 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 18 Feb 2015 10:39:14 +0100
Subject: [PATCH 0406/6219] [youtube] Show entire player URL when -v is given

---
 youtube_dl/extractor/youtube.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 35ef4c303..2c02da2b7 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -541,16 +541,21 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
         if cache_spec is not None:
             return lambda s: ''.join(s[i] for i in cache_spec)
 
+        download_note = (
+            'Downloading player %s' % player_url
+            if self._downloader.params.get('verbose') else
+            'Downloading %s player %s' % (player_type, player_id)
+        )
         if player_type == 'js':
             code = self._download_webpage(
                 player_url, video_id,
-                note='Downloading %s player %s' % (player_type, player_id),
+                note=download_note,
                 errnote='Download of %s failed' % player_url)
             res = self._parse_sig_js(code)
         elif player_type == 'swf':
             urlh = self._request_webpage(
                 player_url, video_id,
-                note='Downloading %s player %s' % (player_type, player_id),
+                note=download_note,
                 errnote='Download of %s failed' % player_url)
             code = urlh.read()
             res = self._parse_sig_swf(code)

From 785521bf4fbd99b2916bdab5d847d84424196c1d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 18 Feb 2015 10:42:23 +0100
Subject: [PATCH 0407/6219] [youtube] Remove useless if

---
 youtube_dl/extractor/youtube.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 2c02da2b7..3d3d43491 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -562,10 +562,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
         else:
             assert False, 'Invalid player type %r' % player_type
 
-        if cache_spec is None:
-            test_string = ''.join(map(compat_chr, range(len(example_sig))))
-            cache_res = res(test_string)
-            cache_spec = [ord(c) for c in cache_res]
+        test_string = ''.join(map(compat_chr, range(len(example_sig))))
+        cache_res = res(test_string)
+        cache_spec = [ord(c) for c in cache_res]
 
         self._downloader.cache.store('youtube-sigfuncs', func_id, cache_spec)
         return res

From 3eff81fbf72939abf885afcca2f1155cc976999b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 18 Feb 2015 10:47:40 +0100
Subject: [PATCH 0408/6219] [jsinterp] Disable comment support

We need a proper lexer to be able to understand YouTube's code, which contains /* inside of strings.
For now it's sufficient to just disable comment support altogether.

Fixes #4976, fixes #4979, fixes #4980, fixes #4981, fixes #4982.
Closes #4977.
---
 test/test_jsinterp.py          | 11 +++++++++++
 test/test_youtube_signature.py |  6 ++++++
 youtube_dl/jsinterp.py         |  5 +----
 3 files changed, 18 insertions(+), 4 deletions(-)

diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py
index b91b8c492..fc73e5dc2 100644
--- a/test/test_jsinterp.py
+++ b/test/test_jsinterp.py
@@ -70,6 +70,8 @@ class TestJSInterpreter(unittest.TestCase):
         self.assertEqual(jsi.call_function('f'), -11)
 
     def test_comments(self):
+        'Skipping: Not yet fully implemented'
+        return
         jsi = JSInterpreter('''
         function x() {
             var x = /* 1 + */ 2;
@@ -80,6 +82,15 @@ class TestJSInterpreter(unittest.TestCase):
         ''')
         self.assertEqual(jsi.call_function('x'), 52)
 
+        jsi = JSInterpreter('''
+        function f() {
+            var x = "/*";
+            var y = 1 /* comment */ + 2;
+            return y;
+        }
+        ''')
+        self.assertEqual(jsi.call_function('f'), 3)
+
     def test_precedence(self):
         jsi = JSInterpreter('''
         function x() {
diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py
index 09696e19a..060864434 100644
--- a/test/test_youtube_signature.py
+++ b/test/test_youtube_signature.py
@@ -64,6 +64,12 @@ _TESTS = [
         'js',
         '4646B5181C6C3020DF1D9C7FCFEA.AD80ABF70C39BD369CCCAE780AFBB98FA6B6CB42766249D9488C288',
         '82C8849D94266724DC6B6AF89BBFA087EACCD963.B93C07FBA084ACAEFCF7C9D1FD0203C6C1815B6B'
+    ),
+    (
+        'https://s.ytimg.com/yts/jsbin/html5player-en_US-vflKjOTVq/html5player.js',
+        'js',
+        '312AA52209E3623129A412D56A40F11CB0AF14AE.3EE09501CB14E3BCDC3B2AE808BF3F1D14E7FBF12',
+        '112AA5220913623229A412D56A40F11CB0AF14AE.3EE0950FCB14EEBCDC3B2AE808BF331D14E7FBF3',
     )
 ]
 
diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index 453e2732c..0e0c7d90d 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -30,13 +30,10 @@ class JSInterpreter(object):
     def __init__(self, code, objects=None):
         if objects is None:
             objects = {}
-        self.code = self._remove_comments(code)
+        self.code = code
         self._functions = {}
         self._objects = objects
 
-    def _remove_comments(self, code):
-        return re.sub(r'(?s)/\*.*?\*/', '', code)
-
     def interpret_statement(self, stmt, local_vars, allow_recursion=100):
         if allow_recursion < 0:
             raise ExtractorError('Recursion limit reached')

From b8b928d5cb5d21390690cc21c29a3d679605015f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 18 Feb 2015 10:54:45 +0100
Subject: [PATCH 0409/6219] [README] Add an FAQ entry for the player change in
 anticipation of many more bug reports

---
 README.md | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index acd0ef41b..a2c148311 100644
--- a/README.md
+++ b/README.md
@@ -515,11 +515,15 @@ If you want to play the video on a machine that is not running youtube-dl, you c
 
 ### ERROR: no fmt_url_map or conn information found in video info
 
-youtube has switched to a new video info format in July 2011 which is not supported by old versions of youtube-dl. You can update youtube-dl with `sudo youtube-dl --update`.
+YouTube has switched to a new video info format in July 2011 which is not supported by old versions of youtube-dl. See [above](#how-do-i-update-youtube-dl) for how to update youtube-dl.
 
 ### ERROR: unable to download video ###
 
-youtube requires an additional signature since September 2012 which is not supported by old versions of youtube-dl. You can update youtube-dl with `sudo youtube-dl --update`.
+YouTube requires an additional signature since September 2012 which is not supported by old versions of youtube-dl. See [above](#how-do-i-update-youtube-dl) for how to update youtube-dl.
+
+### ExtractorError: Could not find JS function u'OF'
+
+In February 2015, the new YouTube player contained a character sequence in a string that was misinterpreted by old versions of youtube-dl. See [above](#how-do-i-update-youtube-dl) for how to update youtube-dl.
 
 ### SyntaxError: Non-ASCII character ###
 

From 81975f4693c15c1f97ad92e7fafb1463ed2aa4c1 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 18 Feb 2015 10:54:56 +0100
Subject: [PATCH 0410/6219] release 2015.02.18.1

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 635b6bdb8..1df05a6df 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.18'
+__version__ = '2015.02.18.1'

From 37dd5d4629ae955940265f245316c43cd0373a3c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 18 Feb 2015 16:54:36 +0100
Subject: [PATCH 0411/6219] [mit] Don't set the subtitles field

YouTube already provides them in more formats
---
 youtube_dl/extractor/mit.py | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/youtube_dl/extractor/mit.py b/youtube_dl/extractor/mit.py
index 3c61a850f..d7ab6a9ae 100644
--- a/youtube_dl/extractor/mit.py
+++ b/youtube_dl/extractor/mit.py
@@ -5,9 +5,6 @@ import json
 
 from .common import InfoExtractor
 from .youtube import YoutubeIE
-from ..compat import (
-    compat_urlparse,
-)
 from ..utils import (
     clean_html,
     ExtractorError,
@@ -108,7 +105,6 @@ class OCWMITIE(InfoExtractor):
                 'upload_date': '20121109',
                 'uploader_id': 'MIT',
                 'uploader': 'MIT OpenCourseWare',
-                # 'subtitles': 'http://ocw.mit.edu/courses/electrical-engineering-and-computer-science/6-041-probabilistic-systems-analysis-and-applied-probability-fall-2010/video-lectures/lecture-7-multiple-variables-expectations-independence/MIT6_041F11_lec07_300k.mp4.srt'
             }
         },
         {
@@ -121,7 +117,6 @@ class OCWMITIE(InfoExtractor):
                 'uploader_id': 'MIT',
                 'uploader': 'MIT OpenCourseWare',
                 'description': 'This section contains lecture video excerpts, lecture notes, an interactive mathlet with supporting documents, and problem solving videos.',
-                # 'subtitles': 'http://ocw.mit.edu//courses/mathematics/18-01sc-single-variable-calculus-fall-2010/ocw-18.01-f07-lec01_300k.SRT'
             }
         }
     ]
@@ -140,7 +135,6 @@ class OCWMITIE(InfoExtractor):
             metadata = re.sub(r'[\'"]', '', embed_chapter_media.group(1))
             metadata = re.split(r', ?', metadata)
             yt = metadata[1]
-            subs = compat_urlparse.urljoin(self._BASE_URL, metadata[7])
         else:
             # search for call to ocw_embed_chapter_media(container_id, media_url, provider, page_url, image_url, captions_file)
             embed_media = re.search(r'ocw_embed_media\((.+?)\)', webpage)
@@ -148,7 +142,6 @@ class OCWMITIE(InfoExtractor):
                 metadata = re.sub(r'[\'"]', '', embed_media.group(1))
                 metadata = re.split(r', ?', metadata)
                 yt = metadata[1]
-                subs = compat_urlparse.urljoin(self._BASE_URL, metadata[5])
             else:
                 raise ExtractorError('Unable to find embedded YouTube video.')
         video_id = YoutubeIE.extract_id(yt)
@@ -159,7 +152,5 @@ class OCWMITIE(InfoExtractor):
             'title': title,
             'description': description,
             'url': yt,
-            'url_transparent'
-            'subtitles': subs,
             'ie_key': 'Youtube',
         }

From 18c1c424057dd06f85f4420b14089e032fcb0000 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 18 Feb 2015 17:20:22 +0100
Subject: [PATCH 0412/6219] [drtv] Convert to new subtitles system

---
 youtube_dl/extractor/drtv.py | 13 ++++---------
 1 file changed, 4 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/drtv.py b/youtube_dl/extractor/drtv.py
index d5df18d7c..8257e35a4 100644
--- a/youtube_dl/extractor/drtv.py
+++ b/youtube_dl/extractor/drtv.py
@@ -1,11 +1,10 @@
 from __future__ import unicode_literals
 
-from .subtitles import SubtitlesInfoExtractor
-from .common import ExtractorError
+from .common import InfoExtractor, ExtractorError
 from ..utils import parse_iso8601
 
 
-class DRTVIE(SubtitlesInfoExtractor):
+class DRTVIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?dr\.dk/tv/se/(?:[^/]+/)*(?P<id>[\da-z-]+)(?:[/#?]|$)'
 
     _TEST = {
@@ -76,7 +75,7 @@ class DRTVIE(SubtitlesInfoExtractor):
                     }
                     for subs in subtitles_list:
                         lang = subs['Language']
-                        subtitles[LANGS.get(lang, lang)] = subs['Uri']
+                        subtitles[LANGS.get(lang, lang)] = [{'url': subs['Uri'], 'ext': 'vtt'}]
 
         if not formats and restricted_to_denmark:
             raise ExtractorError(
@@ -84,10 +83,6 @@ class DRTVIE(SubtitlesInfoExtractor):
 
         self._sort_formats(formats)
 
-        if self._downloader.params.get('listsubtitles', False):
-            self._list_available_subtitles(video_id, subtitles)
-            return
-
         return {
             'id': video_id,
             'title': title,
@@ -96,5 +91,5 @@ class DRTVIE(SubtitlesInfoExtractor):
             'timestamp': timestamp,
             'duration': duration,
             'formats': formats,
-            'subtitles': self.extract_subtitles(video_id, subtitles),
+            'subtitles': subtitles,
         }

From 311c39383827e42649a287633a67ef021476d23a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 18 Feb 2015 17:46:33 +0100
Subject: [PATCH 0413/6219] [lynda] Convert to new subtitles system

---
 test/test_subtitles.py        | 13 ++++++++
 youtube_dl/extractor/lynda.py | 60 ++++++++++++++---------------------
 2 files changed, 37 insertions(+), 36 deletions(-)

diff --git a/test/test_subtitles.py b/test/test_subtitles.py
index 0ca510310..ee170879f 100644
--- a/test/test_subtitles.py
+++ b/test/test_subtitles.py
@@ -18,6 +18,7 @@ from youtube_dl.extractor import (
     VimeoIE,
     WallaIE,
     CeskaTelevizeIE,
+    LyndaIE,
 )
 
 
@@ -304,5 +305,17 @@ class TestCeskaTelevizeSubtitles(BaseTestSubtitles):
         self.assertEqual(len(subtitles), 0)
 
 
+class TestLyndaSubtitles(BaseTestSubtitles):
+    url = 'http://www.lynda.com/Bootstrap-tutorials/Using-exercise-files/110885/114408-4.html'
+    IE = LyndaIE
+
+    def test_allsubtitles(self):
+        self.DL.params['writesubtitles'] = True
+        self.DL.params['allsubtitles'] = True
+        subtitles = self.getSubtitles()
+        self.assertEqual(set(subtitles.keys()), set(['en']))
+        self.assertEqual(md5(subtitles['en']), '09bbe67222259bed60deaa26997d73a7')
+
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index 762cefa34..109055e72 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -3,7 +3,6 @@ from __future__ import unicode_literals
 import re
 import json
 
-from .subtitles import SubtitlesInfoExtractor
 from .common import InfoExtractor
 from ..compat import (
     compat_str,
@@ -16,7 +15,7 @@ from ..utils import (
 )
 
 
-class LyndaIE(SubtitlesInfoExtractor):
+class LyndaIE(InfoExtractor):
     IE_NAME = 'lynda'
     IE_DESC = 'lynda.com videos'
     _VALID_URL = r'https?://www\.lynda\.com/[^/]+/[^/]+/\d+/(\d+)-\d\.html'
@@ -88,11 +87,7 @@ class LyndaIE(SubtitlesInfoExtractor):
         self._check_formats(formats, video_id)
         self._sort_formats(formats)
 
-        if self._downloader.params.get('listsubtitles', False):
-            self._list_available_subtitles(video_id, page)
-            return
-
-        subtitles = self._fix_subtitles(self.extract_subtitles(video_id, page))
+        subtitles = self.extract_subtitles(video_id, page)
 
         return {
             'id': video_id,
@@ -144,38 +139,31 @@ class LyndaIE(SubtitlesInfoExtractor):
         if re.search(self._SUCCESSFUL_LOGIN_REGEX, login_page) is None:
             raise ExtractorError('Unable to log in')
 
-    def _fix_subtitles(self, subtitles):
-        if subtitles is None:
-            return subtitles  # subtitles not requested
-
-        fixed_subtitles = {}
-        for k, v in subtitles.items():
-            subs = json.loads(v)
-            if len(subs) == 0:
+    def _fix_subtitles(self, subs):
+        srt = ''
+        for pos in range(0, len(subs) - 1):
+            seq_current = subs[pos]
+            m_current = re.match(self._TIMECODE_REGEX, seq_current['Timecode'])
+            if m_current is None:
                 continue
-            srt = ''
-            for pos in range(0, len(subs) - 1):
-                seq_current = subs[pos]
-                m_current = re.match(self._TIMECODE_REGEX, seq_current['Timecode'])
-                if m_current is None:
-                    continue
-                seq_next = subs[pos + 1]
-                m_next = re.match(self._TIMECODE_REGEX, seq_next['Timecode'])
-                if m_next is None:
-                    continue
-                appear_time = m_current.group('timecode')
-                disappear_time = m_next.group('timecode')
-                text = seq_current['Caption']
-                srt += '%s\r\n%s --> %s\r\n%s' % (str(pos), appear_time, disappear_time, text)
-            if srt:
-                fixed_subtitles[k] = srt
-        return fixed_subtitles
+            seq_next = subs[pos + 1]
+            m_next = re.match(self._TIMECODE_REGEX, seq_next['Timecode'])
+            if m_next is None:
+                continue
+            appear_time = m_current.group('timecode')
+            disappear_time = m_next.group('timecode')
+            text = seq_current['Caption']
+            srt += '%s\r\n%s --> %s\r\n%s' % (str(pos), appear_time, disappear_time, text)
+        if srt:
+            return srt
 
-    def _get_available_subtitles(self, video_id, webpage):
+    def _get_subtitles(self, video_id, webpage):
         url = 'http://www.lynda.com/ajax/player?videoId=%s&type=transcript' % video_id
-        sub = self._download_webpage(url, None, False)
-        sub_json = json.loads(sub)
-        return {'en': url} if len(sub_json) > 0 else {}
+        subs = self._download_json(url, None, False)
+        if subs:
+            return {'en': [{'ext': 'srt', 'data': self._fix_subtitles(subs)}]}
+        else:
+            return {}
 
 
 class LyndaCourseIE(InfoExtractor):

From b9b42f2ea0c564f3e75a8f052bfe0dfe21cf320f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 18 Feb 2015 17:57:10 +0100
Subject: [PATCH 0414/6219] [npo] Convert to new subtitles system

---
 test/test_subtitles.py      | 13 +++++++++++++
 youtube_dl/extractor/npo.py | 14 +++++---------
 2 files changed, 18 insertions(+), 9 deletions(-)

diff --git a/test/test_subtitles.py b/test/test_subtitles.py
index ee170879f..b2195cac4 100644
--- a/test/test_subtitles.py
+++ b/test/test_subtitles.py
@@ -19,6 +19,7 @@ from youtube_dl.extractor import (
     WallaIE,
     CeskaTelevizeIE,
     LyndaIE,
+    NPOIE,
 )
 
 
@@ -317,5 +318,17 @@ class TestLyndaSubtitles(BaseTestSubtitles):
         self.assertEqual(md5(subtitles['en']), '09bbe67222259bed60deaa26997d73a7')
 
 
+class TestNPOSubtitles(BaseTestSubtitles):
+    url = 'http://www.npo.nl/nos-journaal/28-08-2014/POW_00722860'
+    IE = NPOIE
+
+    def test_allsubtitles(self):
+        self.DL.params['writesubtitles'] = True
+        self.DL.params['allsubtitles'] = True
+        subtitles = self.getSubtitles()
+        self.assertEqual(set(subtitles.keys()), set(['nl']))
+        self.assertEqual(md5(subtitles['nl']), 'fc6435027572b63fb4ab143abd5ad3f4')
+
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index c075618e8..9c01eb0af 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -1,6 +1,5 @@
 from __future__ import unicode_literals
 
-from .subtitles import SubtitlesInfoExtractor
 from .common import InfoExtractor
 from ..utils import (
     fix_xml_ampersands,
@@ -12,7 +11,7 @@ from ..utils import (
 )
 
 
-class NPOBaseIE(SubtitlesInfoExtractor):
+class NPOBaseIE(InfoExtractor):
     def _get_token(self, video_id):
         token_page = self._download_webpage(
             'http://ida.omroep.nl/npoplayer/i.js',
@@ -164,13 +163,10 @@ class NPOIE(NPOBaseIE):
 
         subtitles = {}
         if metadata.get('tt888') == 'ja':
-            subtitles['nl'] = 'http://e.omroep.nl/tt888/%s' % video_id
-
-        if self._downloader.params.get('listsubtitles', False):
-            self._list_available_subtitles(video_id, subtitles)
-            return
-
-        subtitles = self.extract_subtitles(video_id, subtitles)
+            subtitles['nl'] = [{
+                'ext': 'vtt',
+                'url': 'http://e.omroep.nl/tt888/%s' % video_id,
+            }]
 
         return {
             'id': video_id,

From 0af25f784bc5bff7cbce2d4af725b4bf2d2262c5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 18 Feb 2015 18:27:45 +0100
Subject: [PATCH 0415/6219] [mtv] Convert to new subtitles system

---
 test/test_subtitles.py      | 16 ++++++++++++++++
 youtube_dl/extractor/mtv.py | 26 +++++++-------------------
 2 files changed, 23 insertions(+), 19 deletions(-)

diff --git a/test/test_subtitles.py b/test/test_subtitles.py
index b2195cac4..c018d9b49 100644
--- a/test/test_subtitles.py
+++ b/test/test_subtitles.py
@@ -20,6 +20,7 @@ from youtube_dl.extractor import (
     CeskaTelevizeIE,
     LyndaIE,
     NPOIE,
+    ComedyCentralIE,
 )
 
 
@@ -330,5 +331,20 @@ class TestNPOSubtitles(BaseTestSubtitles):
         self.assertEqual(md5(subtitles['nl']), 'fc6435027572b63fb4ab143abd5ad3f4')
 
 
+class TestMTVSubtitles(BaseTestSubtitles):
+    url = 'http://www.cc.com/video-clips/kllhuv/stand-up-greg-fitzsimmons--uncensored---too-good-of-a-mother'
+    IE = ComedyCentralIE
+
+    def getInfoDict(self):
+        return super(TestMTVSubtitles, self).getInfoDict()['entries'][0]
+
+    def test_allsubtitles(self):
+        self.DL.params['writesubtitles'] = True
+        self.DL.params['allsubtitles'] = True
+        subtitles = self.getSubtitles()
+        self.assertEqual(set(subtitles.keys()), set(['en']))
+        self.assertEqual(md5(subtitles['en']), 'b9f6ca22a6acf597ec76f61749765e65')
+
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index bc7f49ebb..c11de1cb6 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -2,7 +2,7 @@ from __future__ import unicode_literals
 
 import re
 
-from .subtitles import SubtitlesInfoExtractor
+from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse,
     compat_urllib_request,
@@ -23,7 +23,7 @@ def _media_xml_tag(tag):
     return '{http://search.yahoo.com/mrss/}%s' % tag
 
 
-class MTVServicesInfoExtractor(SubtitlesInfoExtractor):
+class MTVServicesInfoExtractor(InfoExtractor):
     _MOBILE_TEMPLATE = None
 
     @staticmethod
@@ -95,25 +95,15 @@ class MTVServicesInfoExtractor(SubtitlesInfoExtractor):
 
     def _extract_subtitles(self, mdoc, mtvn_id):
         subtitles = {}
-        FORMATS = {
-            'scc': 'cea-608',
-            'eia-608': 'cea-608',
-            'xml': 'ttml',
-        }
-        subtitles_format = FORMATS.get(
-            self._downloader.params.get('subtitlesformat'), 'ttml')
         for transcript in mdoc.findall('.//transcript'):
             if transcript.get('kind') != 'captions':
                 continue
             lang = transcript.get('srclang')
-            for typographic in transcript.findall('./typographic'):
-                captions_format = typographic.get('format')
-                if captions_format == subtitles_format:
-                    subtitles[lang] = compat_str(typographic.get('src'))
-                    break
-        if self._downloader.params.get('listsubtitles', False):
-            self._list_available_subtitles(mtvn_id, subtitles)
-        return self.extract_subtitles(mtvn_id, subtitles)
+            subtitles[lang] = [{
+                'url': compat_str(typographic.get('src')),
+                'ext': typographic.get('format')
+            } for typographic in transcript.findall('./typographic')]
+        return subtitles
 
     def _get_video_info(self, itemdoc):
         uri = itemdoc.find('guid').text
@@ -196,8 +186,6 @@ class MTVServicesInfoExtractor(SubtitlesInfoExtractor):
                 webpage, 'mgid')
 
         videos_info = self._get_videos_info(mgid)
-        if self._downloader.params.get('listsubtitles', False):
-            return
         return videos_info
 
 

From 01561da142485a581e67ce98ef009ebe0ed7b4c1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 18 Feb 2015 18:57:01 +0100
Subject: [PATCH 0416/6219] [nrk] Convert to new subtitles system

---
 test/test_subtitles.py      | 13 +++++++++++++
 youtube_dl/extractor/nrk.py | 15 +++++++--------
 2 files changed, 20 insertions(+), 8 deletions(-)

diff --git a/test/test_subtitles.py b/test/test_subtitles.py
index c018d9b49..1e2324232 100644
--- a/test/test_subtitles.py
+++ b/test/test_subtitles.py
@@ -21,6 +21,7 @@ from youtube_dl.extractor import (
     LyndaIE,
     NPOIE,
     ComedyCentralIE,
+    NRKTVIE,
 )
 
 
@@ -346,5 +347,17 @@ class TestMTVSubtitles(BaseTestSubtitles):
         self.assertEqual(md5(subtitles['en']), 'b9f6ca22a6acf597ec76f61749765e65')
 
 
+class TestNRKSubtitles(BaseTestSubtitles):
+    url = 'http://tv.nrk.no/serie/ikke-gjoer-dette-hjemme/DMPV73000411/sesong-2/episode-1'
+    IE = NRKTVIE
+
+    def test_allsubtitles(self):
+        self.DL.params['writesubtitles'] = True
+        self.DL.params['allsubtitles'] = True
+        subtitles = self.getSubtitles()
+        self.assertEqual(set(subtitles.keys()), set(['no']))
+        self.assertEqual(md5(subtitles['no']), '1d221e6458c95c5494dcd38e6a1f129a')
+
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index f6de26022..46f493cfc 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -10,7 +10,6 @@ from ..utils import (
     parse_duration,
     unified_strdate,
 )
-from .subtitles import SubtitlesInfoExtractor
 
 
 class NRKIE(InfoExtractor):
@@ -73,7 +72,7 @@ class NRKIE(InfoExtractor):
         }
 
 
-class NRKTVIE(SubtitlesInfoExtractor):
+class NRKTVIE(InfoExtractor):
     _VALID_URL = r'(?P<baseurl>http://tv\.nrk(?:super)?\.no/)(?:serie/[^/]+|program)/(?P<id>[a-zA-Z]{4}\d{8})(?:/\d{2}-\d{2}-\d{4})?(?:#del=(?P<part_id>\d+))?'
 
     _TESTS = [
@@ -156,7 +155,7 @@ class NRKTVIE(SubtitlesInfoExtractor):
         if self._downloader.params.get('verbose', False):
             self.to_screen('[debug] %s' % txt)
 
-    def _extract_captions(self, subtitlesurl, video_id, baseurl):
+    def _get_subtitles(self, subtitlesurl, video_id, baseurl):
         url = "%s%s" % (baseurl, subtitlesurl)
         self._debug_print('%s: Subtitle url: %s' % (video_id, url))
         captions = self._download_xml(url, video_id, 'Downloading subtitles')
@@ -170,7 +169,10 @@ class NRKTVIE(SubtitlesInfoExtractor):
             endtime = self._seconds2str(begin + duration)
             text = '\n'.join(p.itertext())
             srt += '%s\r\n%s --> %s\r\n%s\r\n\r\n' % (str(pos), starttime, endtime, text)
-        return {lang: srt}
+        return {lang: [
+            {'ext': 'ttml', 'url': url},
+            {'ext': 'srt', 'data': srt},
+        ]}
 
     def _extract_f4m(self, manifest_url, video_id):
         return self._extract_f4m_formats(manifest_url + '?hdcore=3.1.1&plugin=aasp-3.1.1.69.124', video_id)
@@ -243,10 +245,7 @@ class NRKTVIE(SubtitlesInfoExtractor):
             webpage, 'subtitle URL', default=None)
         subtitles = None
         if subtitles_url:
-            subtitles = self._extract_captions(subtitles_url, video_id, baseurl)
-        if self._downloader.params.get('listsubtitles', False):
-            self._list_available_subtitles(video_id, subtitles)
-            return
+            subtitles = self.extract_subtitles(subtitles_url, video_id, baseurl)
 
         return {
             'id': video_id,

From d47c26e16803abc1d15a677d88bbee78f7680db6 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 18 Feb 2015 19:56:10 +0100
Subject: [PATCH 0417/6219] [brightcove] Correct keys in playlists

---
 youtube_dl/extractor/brightcove.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index ea0969d4d..0733bece7 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -95,6 +95,7 @@ class BrightcoveIE(InfoExtractor):
             'url': 'http://c.brightcove.com/services/viewer/htmlFederated?playerID=3550052898001&playerKey=AQ%7E%7E%2CAAABmA9XpXk%7E%2C-Kp7jNgisre1fG5OdqpAFUTcs0lP_ZoL',
             'info_dict': {
                 'title': 'Sealife',
+                'id': '3550319591001',
             },
             'playlist_mincount': 7,
         },
@@ -247,7 +248,7 @@ class BrightcoveIE(InfoExtractor):
         playlist_info = json_data['videoList']
         videos = [self._extract_video_info(video_info) for video_info in playlist_info['mediaCollectionDTO']['videoDTOs']]
 
-        return self.playlist_result(videos, playlist_id=playlist_info['id'],
+        return self.playlist_result(videos, playlist_id='%s' % playlist_info['id'],
                                     playlist_title=playlist_info['mediaCollectionDTO']['displayName'])
 
     def _extract_video_info(self, video_info):

From d8443cd3f7e50fafb918cff5e1676b8d6fef1a64 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 18 Feb 2015 19:56:24 +0100
Subject: [PATCH 0418/6219] [wsj] Correct test case

---
 youtube_dl/extractor/wsj.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/wsj.py b/youtube_dl/extractor/wsj.py
index cbe3dc7be..2ddf29a69 100644
--- a/youtube_dl/extractor/wsj.py
+++ b/youtube_dl/extractor/wsj.py
@@ -18,8 +18,8 @@ class WSJIE(InfoExtractor):
             'id': '1BD01A4C-BFE8-40A5-A42F-8A8AF9898B1A',
             'ext': 'mp4',
             'upload_date': '20150202',
-            'uploader_id': 'bbright',
-            'creator': 'bbright',
+            'uploader_id': 'jdesai',
+            'creator': 'jdesai',
             'categories': list,  # a long list
             'duration': 90,
             'title': 'Bills Coach Rex Ryan Updates His Old Jets Tattoo',

From 9789d7535d6091f64dd688b0bb1f9869b6c6ca49 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 18 Feb 2015 19:58:41 +0100
Subject: [PATCH 0419/6219] [xtube] Fix test case

---
 test/helper.py                | 10 ++++++++++
 youtube_dl/extractor/xtube.py |  2 +-
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/test/helper.py b/test/helper.py
index 570c91440..12afdf184 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -113,6 +113,16 @@ def expect_info_dict(self, got_dict, expected_dict):
             self.assertTrue(
                 got.startswith(start_str),
                 'field %s (value: %r) should start with %r' % (info_field, got, start_str))
+        elif isinstance(expected, compat_str) and expected.startswith('contains:'):
+            got = got_dict.get(info_field)
+            contains_str = expected[len('contains:'):]
+            self.assertTrue(
+                isinstance(got, compat_str),
+                'Expected a %s object, but got %s for field %s' % (
+                    compat_str.__name__, type(got).__name__, info_field))
+            self.assertTrue(
+                contains_str in got,
+                'field %s (value: %r) should contain %r' % (info_field, got, contains_str))
         elif isinstance(expected, type):
             got = got_dict.get(info_field)
             self.assertTrue(isinstance(got, expected),
diff --git a/youtube_dl/extractor/xtube.py b/youtube_dl/extractor/xtube.py
index e8490b028..1644f53c8 100644
--- a/youtube_dl/extractor/xtube.py
+++ b/youtube_dl/extractor/xtube.py
@@ -22,7 +22,7 @@ class XTubeIE(InfoExtractor):
             'id': 'kVTUy_G222_',
             'ext': 'mp4',
             'title': 'strange erotica',
-            'description': 'http://www.xtube.com an ET kind of thing',
+            'description': 'contains:an ET kind of thing',
             'uploader': 'greenshowers',
             'duration': 450,
             'age_limit': 18,

From 85d586617750d38d742a24f141b099f6b898d269 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 18 Feb 2015 20:03:00 +0100
Subject: [PATCH 0420/6219] [yahoo] Remove md5sum from test case

The md5 sum has changed repeatedly, and we check whether it looks like a video anyways nowadays.
---
 youtube_dl/extractor/yahoo.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/yahoo.py b/youtube_dl/extractor/yahoo.py
index f8e7041a0..97dbac4cc 100644
--- a/youtube_dl/extractor/yahoo.py
+++ b/youtube_dl/extractor/yahoo.py
@@ -24,7 +24,6 @@ class YahooIE(InfoExtractor):
     _TESTS = [
         {
             'url': 'http://screen.yahoo.com/julian-smith-travis-legg-watch-214727115.html',
-            'md5': '4962b075c08be8690a922ee026d05e69',
             'info_dict': {
                 'id': '2d25e626-2378-391f-ada0-ddaf1417e588',
                 'ext': 'mp4',

From afbdd3acc36130d1a717b3cacab69c0dfc716622 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 18 Feb 2015 20:14:42 +0100
Subject: [PATCH 0421/6219] [rai] Convert to new subtitles system

---
 test/test_subtitles.py      | 13 +++++++++++++
 youtube_dl/extractor/rai.py | 22 +++++++++-------------
 2 files changed, 22 insertions(+), 13 deletions(-)

diff --git a/test/test_subtitles.py b/test/test_subtitles.py
index 1e2324232..4dbb50515 100644
--- a/test/test_subtitles.py
+++ b/test/test_subtitles.py
@@ -22,6 +22,7 @@ from youtube_dl.extractor import (
     NPOIE,
     ComedyCentralIE,
     NRKTVIE,
+    RaiIE,
 )
 
 
@@ -359,5 +360,17 @@ class TestNRKSubtitles(BaseTestSubtitles):
         self.assertEqual(md5(subtitles['no']), '1d221e6458c95c5494dcd38e6a1f129a')
 
 
+class TestRaiSubtitles(BaseTestSubtitles):
+    url = 'http://www.rai.tv/dl/RaiTV/programmi/media/ContentItem-cb27157f-9dd0-4aee-b788-b1f67643a391.html'
+    IE = RaiIE
+
+    def test_allsubtitles(self):
+        self.DL.params['writesubtitles'] = True
+        self.DL.params['allsubtitles'] = True
+        subtitles = self.getSubtitles()
+        self.assertEqual(set(subtitles.keys()), set(['it']))
+        self.assertEqual(md5(subtitles['it']), 'b1d90a98755126b61e667567a1f6680a')
+
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/extractor/rai.py b/youtube_dl/extractor/rai.py
index aa26b7e0b..144e33982 100644
--- a/youtube_dl/extractor/rai.py
+++ b/youtube_dl/extractor/rai.py
@@ -2,7 +2,7 @@ from __future__ import unicode_literals
 
 import re
 
-from .subtitles import SubtitlesInfoExtractor
+from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse,
 )
@@ -12,7 +12,7 @@ from ..utils import (
 )
 
 
-class RaiIE(SubtitlesInfoExtractor):
+class RaiIE(InfoExtractor):
     _VALID_URL = r'(?P<url>http://(?:.+?\.)?(?:rai\.it|rai\.tv|rainews\.it)/dl/.+?-(?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})(?:-.+?)?\.html)'
     _TESTS = [
         {
@@ -89,15 +89,7 @@ class RaiIE(SubtitlesInfoExtractor):
                 'ext': 'mp4',
             })
 
-        if self._downloader.params.get('listsubtitles', False):
-            page = self._download_webpage(url, video_id)
-            self._list_available_subtitles(video_id, page)
-            return
-
-        subtitles = {}
-        if self._have_to_download_any_subtitles:
-            page = self._download_webpage(url, video_id)
-            subtitles = self.extract_subtitles(video_id, page)
+        subtitles = self.extract_subtitles(video_id, url)
 
         return {
             'id': video_id,
@@ -111,7 +103,8 @@ class RaiIE(SubtitlesInfoExtractor):
             'subtitles': subtitles,
         }
 
-    def _get_available_subtitles(self, video_id, webpage):
+    def _get_subtitles(self, video_id, url):
+        webpage = self._download_webpage(url, video_id)
         subtitles = {}
         m = re.search(r'<meta name="closedcaption" content="(?P<captions>[^"]+)"', webpage)
         if m:
@@ -120,5 +113,8 @@ class RaiIE(SubtitlesInfoExtractor):
             SRT_EXT = '.srt'
             if captions.endswith(STL_EXT):
                 captions = captions[:-len(STL_EXT)] + SRT_EXT
-            subtitles['it'] = 'http://www.rai.tv%s' % compat_urllib_parse.quote(captions)
+            subtitles['it'] = [{
+                'ext': 'srt',
+                'url': 'http://www.rai.tv%s' % compat_urllib_parse.quote(captions),
+            }]
         return subtitles

From 4f7cea6c5327fdda79877e34e630c478f0e5d4dd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 18 Feb 2015 20:37:16 +0100
Subject: [PATCH 0422/6219] [viki] Convert to new subtitles system

---
 test/test_subtitles.py       | 13 +++++++++++++
 youtube_dl/extractor/viki.py | 17 +++++++++--------
 2 files changed, 22 insertions(+), 8 deletions(-)

diff --git a/test/test_subtitles.py b/test/test_subtitles.py
index 4dbb50515..98d1afff4 100644
--- a/test/test_subtitles.py
+++ b/test/test_subtitles.py
@@ -23,6 +23,7 @@ from youtube_dl.extractor import (
     ComedyCentralIE,
     NRKTVIE,
     RaiIE,
+    VikiIE,
 )
 
 
@@ -372,5 +373,17 @@ class TestRaiSubtitles(BaseTestSubtitles):
         self.assertEqual(md5(subtitles['it']), 'b1d90a98755126b61e667567a1f6680a')
 
 
+class TestVikiSubtitles(BaseTestSubtitles):
+    url = 'http://www.viki.com/videos/1060846v-punch-episode-18'
+    IE = VikiIE
+
+    def test_allsubtitles(self):
+        self.DL.params['writesubtitles'] = True
+        self.DL.params['allsubtitles'] = True
+        subtitles = self.getSubtitles()
+        self.assertEqual(set(subtitles.keys()), set(['en']))
+        self.assertEqual(md5(subtitles['en']), 'b0b781eeb45efd3f6398a925b259150b')
+
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index 944901e14..6816dacb6 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -2,16 +2,17 @@ from __future__ import unicode_literals
 
 import re
 
+from ..compat import compat_urlparse
 from ..utils import (
     ExtractorError,
     unescapeHTML,
     unified_strdate,
     US_RATINGS,
 )
-from .subtitles import SubtitlesInfoExtractor
+from .common import InfoExtractor
 
 
-class VikiIE(SubtitlesInfoExtractor):
+class VikiIE(InfoExtractor):
     IE_NAME = 'viki'
 
     _VALID_URL = r'^https?://(?:www\.)?viki\.com/videos/(?P<id>[0-9]+v)'
@@ -69,9 +70,6 @@ class VikiIE(SubtitlesInfoExtractor):
 
         # subtitles
         video_subtitles = self.extract_subtitles(video_id, info_webpage)
-        if self._downloader.params.get('listsubtitles', False):
-            self._list_available_subtitles(video_id, info_webpage)
-            return
 
         return {
             'id': video_id,
@@ -85,12 +83,15 @@ class VikiIE(SubtitlesInfoExtractor):
             'upload_date': upload_date,
         }
 
-    def _get_available_subtitles(self, video_id, info_webpage):
+    def _get_subtitles(self, video_id, info_webpage):
         res = {}
-        for sturl_html in re.findall(r'<track src="([^"]+)"/>', info_webpage):
+        for sturl_html in re.findall(r'<track src="([^"]+)"', info_webpage):
             sturl = unescapeHTML(sturl_html)
             m = re.search(r'/(?P<lang>[a-z]+)\.vtt', sturl)
             if not m:
                 continue
-            res[m.group('lang')] = sturl
+            res[m.group('lang')] = [{
+                'url': compat_urlparse.urljoin('http://www.viki.com', sturl),
+                'ext': 'vtt',
+            }]
         return res

From 93a16ba238c25392ed66aa0f1d7cbd3980c45f2f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 18 Feb 2015 22:00:12 +0100
Subject: [PATCH 0423/6219] [vimeo] Raise the ExtractorError with expected=True
 when no video password is given

---
 youtube_dl/extractor/vimeo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index e7284049d..78d287e0e 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -175,7 +175,7 @@ class VimeoIE(VimeoBaseInfoExtractor, SubtitlesInfoExtractor):
     def _verify_video_password(self, url, video_id, webpage):
         password = self._downloader.params.get('videopassword', None)
         if password is None:
-            raise ExtractorError('This video is protected by a password, use the --video-password option')
+            raise ExtractorError('This video is protected by a password, use the --video-password option', expected=True)
         token = self._search_regex(r'xsrft: \'(.*?)\'', webpage, 'login token')
         data = compat_urllib_parse.urlencode({
             'password': password,

From d305dd73a3d6927f0a2c63d08662a183fa173833 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 18 Feb 2015 23:59:50 +0100
Subject: [PATCH 0424/6219] [utils] Fix js_to_json

Previously, the runtime could be atrocious for longer inputs.
---
 test/test_utils.py  | 4 ++++
 youtube_dl/utils.py | 4 ++--
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 1c29d0889..c7373af1e 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -370,6 +370,10 @@ class TestUtil(unittest.TestCase):
             "playlist":[{"controls":{"all":null}}]
         }''')
 
+        inp = '"SAND Number: SAND 2013-7800P\\nPresenter: Tom Russo\\nHabanero Software Training - Xyce Software\\nXyce, Sandia\\u0027s"'
+        json_code = js_to_json(inp)
+        self.assertEqual(json.loads(json_code), json.loads(inp))
+
     def test_js_to_json_edgecases(self):
         on = js_to_json("{abc_def:'1\\'\\\\2\\\\\\'3\"4'}")
         self.assertEqual(json.loads(on), {"abc_def": "1'\\2\\'3\"4"})
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 3eb6bc6d4..4358137a0 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1560,8 +1560,8 @@ def js_to_json(code):
         return '"%s"' % v
 
     res = re.sub(r'''(?x)
-        "(?:[^"\\]*(?:\\\\|\\")?)*"|
-        '(?:[^'\\]*(?:\\\\|\\')?)*'|
+        "(?:[^"\\]*(?:\\\\|\\['"nu]))*[^"\\]*"|
+        '(?:[^'\\]*(?:\\\\|\\['"nu]))*[^'\\]*'|
         [a-zA-Z_][.a-zA-Z_0-9]*
         ''', fix_kv, code)
     res = re.sub(r',(\s*\])', lambda m: m.group(1), res)

From b81a359eb67aea83799f1306a441fd6163bf1840 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 19 Feb 2015 00:28:58 +0100
Subject: [PATCH 0425/6219] [YoutubeDL] Use render_table for format listing

---
 youtube_dl/YoutubeDL.py | 25 +++++++------------------
 1 file changed, 7 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index b087d356f..88809783b 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1534,29 +1534,18 @@ class YoutubeDL(object):
         return res
 
     def list_formats(self, info_dict):
-        def line(format, idlen=20):
-            return (('%-' + compat_str(idlen + 1) + 's%-10s%-12s%s') % (
-                format['format_id'],
-                format['ext'],
-                self.format_resolution(format),
-                self._format_note(format),
-            ))
-
         formats = info_dict.get('formats', [info_dict])
-        idlen = max(len('format code'),
-                    max(len(f['format_id']) for f in formats))
-        formats_s = [
-            line(f, idlen) for f in formats
+        table = [
+            [f['format_id'], f['ext'], self.format_resolution(f), self._format_note(f)]
+            for f in formats
             if f.get('preference') is None or f['preference'] >= -1000]
         if len(formats) > 1:
-            formats_s[-1] += (' ' if self._format_note(formats[-1]) else '') + '(best)'
+            table[-1][-1] += (' ' if table[-1][-1] else '') + '(best)'
 
-        header_line = line({
-            'format_id': 'format code', 'ext': 'extension',
-            'resolution': 'resolution', 'format_note': 'note'}, idlen=idlen)
+        header_line = ['format code', 'extension', 'resolution', 'note']
         self.to_screen(
-            '[info] Available formats for %s:\n%s\n%s' %
-            (info_dict['id'], header_line, '\n'.join(formats_s)))
+            '[info] Available formats for %s:\n%s' %
+            (info_dict['id'], render_table(header_line, table)))
 
     def list_thumbnails(self, info_dict):
         thumbnails = info_dict.get('thumbnails')

From c460bdd56b0420bb5175344c0f6523675fdae415 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 19 Feb 2015 00:31:01 +0100
Subject: [PATCH 0426/6219] [sandia] Add new extractor (#4974)

---
 youtube_dl/extractor/__init__.py |   1 +
 youtube_dl/extractor/sandia.py   | 117 +++++++++++++++++++++++++++++++
 youtube_dl/utils.py              |  11 ++-
 3 files changed, 128 insertions(+), 1 deletion(-)
 create mode 100644 youtube_dl/extractor/sandia.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 7d0909b5d..c712b907f 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -386,6 +386,7 @@ from .rutube import (
     RutubePersonIE,
 )
 from .rutv import RUTVIE
+from .sandia import SandiaIE
 from .sapo import SapoIE
 from .savefrom import SaveFromIE
 from .sbs import SBSIE
diff --git a/youtube_dl/extractor/sandia.py b/youtube_dl/extractor/sandia.py
new file mode 100644
index 000000000..9c88167f0
--- /dev/null
+++ b/youtube_dl/extractor/sandia.py
@@ -0,0 +1,117 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import itertools
+import json
+import re
+
+from .common import InfoExtractor
+from ..compat import (
+    compat_urllib_request,
+    compat_urlparse,
+)
+from ..utils import (
+    int_or_none,
+    js_to_json,
+    mimetype2ext,
+    unified_strdate,
+)
+
+
+class SandiaIE(InfoExtractor):
+    IE_DESC = 'Sandia National Laboratories'
+    _VALID_URL = r'https?://digitalops\.sandia\.gov/Mediasite/Play/(?P<id>[0-9a-f]+)'
+    _TEST = {
+        'url': 'http://digitalops.sandia.gov/Mediasite/Play/24aace4429fc450fb5b38cdbf424a66e1d',
+        'md5': '9422edc9b9a60151727e4b6d8bef393d',
+        'info_dict': {
+            'id': '24aace4429fc450fb5b38cdbf424a66e1d',
+            'ext': 'mp4',
+            'title': 'Xyce Software Training - Section 1',
+            'description': 're:(?s)SAND Number: SAND 2013-7800.{200,}',
+            'upload_date': '20120904',
+            'duration': 7794,
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        req = compat_urllib_request.Request(url)
+        req.add_header('Cookie', 'MediasitePlayerCaps=ClientPlugins=4')
+        webpage = self._download_webpage(req, video_id)
+
+        js_path = self._search_regex(
+            r'<script type="text/javascript" src="(/Mediasite/FileServer/Presentation/[^"]+)"',
+            webpage, 'JS code URL')
+        js_url = compat_urlparse.urljoin(url, js_path)
+
+        js_code = self._download_webpage(
+            js_url, video_id, note='Downloading player')
+
+        def extract_str(key, **args):
+            return self._search_regex(
+                r'Mediasite\.PlaybackManifest\.%s\s*=\s*(.+);\s*?\n' % re.escape(key),
+                js_code, key, **args)
+
+        def extract_data(key, **args):
+            data_json = extract_str(key, **args)
+            if data_json is None:
+                return data_json
+            return self._parse_json(
+                data_json, video_id, transform_source=js_to_json)
+
+        formats = []
+        for i in itertools.count():
+            fd = extract_data('VideoUrls[%d]' % i, default=None)
+            if fd is None:
+                break
+            formats.append({
+                'format_id': '%s' % i,
+                'format_note': fd['MimeType'].partition('/')[2],
+                'ext': mimetype2ext(fd['MimeType']),
+                'url': fd['Location'],
+                'protocol': 'f4m' if fd['MimeType'] == 'video/x-mp4-fragmented' else None,
+            })
+        self._sort_formats(formats)
+
+        slide_baseurl = compat_urlparse.urljoin(
+            url, extract_data('SlideBaseUrl'))
+        slide_template = slide_baseurl + re.sub(
+            r'\{0:D?([0-9+])\}', r'%0\1d', extract_data('SlideImageFileNameTemplate'))
+        slides = []
+        last_slide_time = 0
+        for i in itertools.count(1):
+            sd = extract_str('Slides[%d]' % i, default=None)
+            if sd is None:
+                break
+            timestamp = int_or_none(self._search_regex(
+                r'^Mediasite\.PlaybackManifest\.CreateSlide\("[^"]*"\s*,\s*([0-9]+),',
+                sd, 'slide %s timestamp' % i, fatal=False))
+            slides.append({
+                'url': slide_template % i,
+                'duration': timestamp - last_slide_time,
+            })
+            last_slide_time = timestamp
+        formats.append({
+            'format_id': 'slides',
+            'protocol': 'slideshow',
+            'url': json.dumps(slides),
+            'preference': -10000,  # Downloader not yet written
+        })
+        self._sort_formats(formats)
+
+        title = extract_data('Title')
+        description = extract_data('Description', fatal=False)
+        duration = int_or_none(extract_data(
+            'Duration', fatal=False), scale=1000)
+        upload_date = unified_strdate(extract_data('AirDate', fatal=False))
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'formats': formats,
+            'upload_date': upload_date,
+            'duration': duration,
+        }
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 4358137a0..238b6556b 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1616,6 +1616,15 @@ def args_to_str(args):
     return ' '.join(shlex_quote(a) for a in args)
 
 
+def mimetype2ext(mt):
+    _, _, res = mt.rpartition('/')
+
+    return {
+        'x-ms-wmv': 'wmv',
+        'x-mp4-fragmented': 'mp4',
+    }.get(res, res)
+
+
 def urlhandle_detect_ext(url_handle):
     try:
         url_handle.headers
@@ -1631,7 +1640,7 @@ def urlhandle_detect_ext(url_handle):
             if e:
                 return e
 
-    return getheader('Content-Type').split("/")[1]
+    return mimetype2ext(getheader('Content-Type'))
 
 
 def age_restricted(content_limit, age_limit):

From a025d3c5a529a3343d40886e9783fde0bf997b3f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 19 Feb 2015 00:31:23 +0100
Subject: [PATCH 0427/6219] release 2015.02.19

---
 docs/supportedsites.md | 1 +
 youtube_dl/version.py  | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 4bb68fdc5..e28d7d47c 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -352,6 +352,7 @@
  - **rutube:movie**: Rutube movies
  - **rutube:person**: Rutube person videos
  - **RUTV**: RUTV.RU
+ - **Sandia**: Sandia National Laboratories
  - **Sapo**: SAPO Vídeos
  - **savefrom.net**
  - **SBS**: sbs.com.au
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 1df05a6df..8e6ff49d0 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.18.1'
+__version__ = '2015.02.19'

From 770700404365be7b16de89c75825f04e2cf15658 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 19 Feb 2015 00:38:05 +0100
Subject: [PATCH 0428/6219] [patreon] Modernize

---
 youtube_dl/extractor/patreon.py | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/patreon.py b/youtube_dl/extractor/patreon.py
index 5429592a7..e5c9822c5 100644
--- a/youtube_dl/extractor/patreon.py
+++ b/youtube_dl/extractor/patreon.py
@@ -11,7 +11,7 @@ from ..utils import (
 
 
 class PatreonIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?patreon\.com/creation\?hid=(.+)'
+    _VALID_URL = r'https?://(?:www\.)?patreon\.com/creation\?hid=(?P<id>[^&#]+)'
     _TESTS = [
         {
             'url': 'http://www.patreon.com/creation?hid=743933',
@@ -65,9 +65,7 @@ class PatreonIE(InfoExtractor):
     '''
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group(1)
-
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
         title = self._og_search_title(webpage).strip()
 
@@ -80,11 +78,10 @@ class PatreonIE(InfoExtractor):
             uploader = self._html_search_regex(
                 r'<strong>(.*?)</strong> is creating', webpage, 'uploader')
         else:
-            playlist_js = self._search_regex(
+            playlist = self._parse_json(self._search_regex(
                 r'(?s)new\s+jPlayerPlaylist\(\s*\{\s*[^}]*},\s*(\[.*?,?\s*\])',
-                webpage, 'playlist JSON')
-            playlist_json = js_to_json(playlist_js)
-            playlist = json.loads(playlist_json)
+                webpage, 'playlist JSON'),
+                video_id, transform_source=js_to_json)
             data = playlist[0]
             video_url = self._proto_relative_url(data['mp3'])
             thumbnail = self._proto_relative_url(data.get('cover'))

From 6b961a85fdef8219913b7b97cb33ef8d59d15535 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 19 Feb 2015 01:04:19 +0100
Subject: [PATCH 0429/6219] [patreon] Add support for embedlies (fixes #4969)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/embedly.py  | 16 ++++++++++++++++
 youtube_dl/extractor/patreon.py  | 26 +++++++++++++++++++++++---
 3 files changed, 40 insertions(+), 3 deletions(-)
 create mode 100644 youtube_dl/extractor/embedly.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index c712b907f..84a7edffc 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -121,6 +121,7 @@ from .ellentv import (
     EllenTVClipsIE,
 )
 from .elpais import ElPaisIE
+from .embedly import EmbedlyIE
 from .empflix import EMPFlixIE
 from .engadget import EngadgetIE
 from .eporner import EpornerIE
diff --git a/youtube_dl/extractor/embedly.py b/youtube_dl/extractor/embedly.py
new file mode 100644
index 000000000..1cdb11e34
--- /dev/null
+++ b/youtube_dl/extractor/embedly.py
@@ -0,0 +1,16 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..compat import compat_urllib_parse_unquote
+
+
+class EmbedlyIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www|cdn\.)?embedly\.com/widgets/media\.html\?(?:[^#]*?&)?url=(?P<id>[^#&]+)'
+    _TESTS = [{
+        'url': 'https://cdn.embedly.com/widgets/media.html?src=http%3A%2F%2Fwww.youtube.com%2Fembed%2Fvideoseries%3Flist%3DUUGLim4T2loE5rwCMdpCIPVg&url=https%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DSU4fj_aEMVw%26list%3DUUGLim4T2loE5rwCMdpCIPVg&image=http%3A%2F%2Fi.ytimg.com%2Fvi%2FSU4fj_aEMVw%2Fhqdefault.jpg&key=8ee8a2e6a8cc47aab1a5ee67f9a178e0&type=text%2Fhtml&schema=youtube&autoplay=1',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        return self.url_result(compat_urllib_parse_unquote(self._match_id(url)))
diff --git a/youtube_dl/extractor/patreon.py b/youtube_dl/extractor/patreon.py
index e5c9822c5..f179ea200 100644
--- a/youtube_dl/extractor/patreon.py
+++ b/youtube_dl/extractor/patreon.py
@@ -1,9 +1,6 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
-import json
-import re
-
 from .common import InfoExtractor
 from ..utils import (
     js_to_json,
@@ -35,6 +32,23 @@ class PatreonIE(InfoExtractor):
                 'thumbnail': 're:^https?://.*$',
             },
         },
+        {
+            'url': 'https://www.patreon.com/creation?hid=1682498',
+            'info_dict': {
+                'id': 'SU4fj_aEMVw',
+                'ext': 'mp4',
+                'title': 'I\'m on Patreon!',
+                'uploader': 'TraciJHines',
+                'thumbnail': 're:^https?://.*$',
+                'upload_date': '20150211',
+                'description': 'md5:c5a706b1f687817a3de09db1eb93acd4',
+                'uploader_id': 'TraciJHines',
+            },
+            'params': {
+                'noplaylist': True,
+                'skip_download': True,
+            }
+        }
     ]
 
     # Currently Patreon exposes download URL via hidden CSS, so login is not
@@ -72,11 +86,17 @@ class PatreonIE(InfoExtractor):
         attach_fn = self._html_search_regex(
             r'<div class="attach"><a target="_blank" href="([^"]+)">',
             webpage, 'attachment URL', default=None)
+        embed = self._html_search_regex(
+            r'<div id="watchCreation">\s*<iframe class="embedly-embed" src="([^"]+)"',
+            webpage, 'embedded URL', default=None)
+
         if attach_fn is not None:
             video_url = 'http://www.patreon.com' + attach_fn
             thumbnail = self._og_search_thumbnail(webpage)
             uploader = self._html_search_regex(
                 r'<strong>(.*?)</strong> is creating', webpage, 'uploader')
+        elif embed is not None:
+            return self.url_result(embed)
         else:
             playlist = self._parse_json(self._search_regex(
                 r'(?s)new\s+jPlayerPlaylist\(\s*\{\s*[^}]*},\s*(\[.*?,?\s*\])',

From 7bd15b1a0378cff6960cd55e40819ef3967f747c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 19 Feb 2015 01:04:24 +0100
Subject: [PATCH 0430/6219] release 2015.02.19.1

---
 docs/supportedsites.md | 1 +
 youtube_dl/version.py  | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index e28d7d47c..1fcec086a 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -121,6 +121,7 @@
  - **EllenTV**
  - **EllenTV:clips**
  - **ElPais**: El País
+ - **Embedly**
  - **EMPFlix**
  - **Engadget**
  - **Eporner**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 8e6ff49d0..db0ed70da 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.19'
+__version__ = '2015.02.19.1'

From 1c2528c8a3c599740558d5d752dfb368c8714a89 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 19 Feb 2015 01:22:50 +0100
Subject: [PATCH 0431/6219] [cbs] Modernize

---
 youtube_dl/extractor/cbs.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/cbs.py b/youtube_dl/extractor/cbs.py
index e43756ec6..1ceb9d8d9 100644
--- a/youtube_dl/extractor/cbs.py
+++ b/youtube_dl/extractor/cbs.py
@@ -1,7 +1,5 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 
 
@@ -39,8 +37,7 @@ class CBSIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
         real_id = self._search_regex(
             r"video\.settings\.pid\s*=\s*'([^']+)';",

From 3bf5705316adb4e486ec76a9308198b499787947 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 19 Feb 2015 01:43:20 +0100
Subject: [PATCH 0432/6219] [imgur] Add new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/imgur.py    | 84 ++++++++++++++++++++++++++++++++
 2 files changed, 85 insertions(+)
 create mode 100644 youtube_dl/extractor/imgur.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 84a7edffc..1d1f07ff5 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -205,6 +205,7 @@ from .imdb import (
     ImdbIE,
     ImdbListIE
 )
+from .imgur import ImgurIE
 from .ina import InaIE
 from .infoq import InfoQIE
 from .instagram import InstagramIE, InstagramUserIE
diff --git a/youtube_dl/extractor/imgur.py b/youtube_dl/extractor/imgur.py
new file mode 100644
index 000000000..16488e0c4
--- /dev/null
+++ b/youtube_dl/extractor/imgur.py
@@ -0,0 +1,84 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    js_to_json,
+    mimetype2ext,
+)
+
+
+class ImgurIE(InfoExtractor):
+    _VALID_URL = r'https?://i\.imgur\.com/(?P<id>[a-zA-Z0-9]+)\.(?:mp4|gifv)'
+
+    _TESTS = [{
+        'url': 'https://i.imgur.com/A61SaA1.gifv',
+        'info_dict': {
+            'id': 'A61SaA1',
+            'ext': 'mp4',
+            'title': 'MRW gifv is up and running without any bugs',
+            'description': 'The Internet\'s visual storytelling community. Explore, share, and discuss the best visual stories the Internet has to offer.',
+        },
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        width = int_or_none(self._search_regex(
+            r'<param name="width" value="([0-9]+)"',
+            webpage, 'width', fatal=False))
+        height = int_or_none(self._search_regex(
+            r'<param name="height" value="([0-9]+)"',
+            webpage, 'height', fatal=False))
+
+        formats = []
+        video_elements = self._search_regex(
+            r'(?s)<div class="video-elements">(.*?)</div>',
+            webpage, 'video elements')
+        formats = []
+        for m in re.finditer(r'<source\s+src="(?P<src>[^"]+)"\s+type="(?P<type>[^"]+)"', video_elements):
+            formats.append({
+                'format_id': m.group('type').partition('/')[2],
+                'url': self._proto_relative_url(m.group('src')),
+                'ext': mimetype2ext(m.group('type')),
+                'acodec': 'none',
+                'width': width,
+                'height': height,
+                'http_headers': {
+                    'User-Agent': 'youtube-dl (like wget)',
+                },
+            })
+
+        gif_json = self._search_regex(
+            r'(?s)var\s+videoItem\s*=\s*(\{.*?\})',
+            webpage, 'GIF code', fatal=False)
+        if gif_json:
+            gifd = self._parse_json(
+                gif_json, video_id, transform_source=js_to_json)
+            formats.append({
+                'format_id': 'gif',
+                'preference': -10,
+                'width': width,
+                'height': height,
+                'ext': 'gif',
+                'acodec': 'none',
+                'vcodec': 'gif',
+                'container': 'gif',
+                'url': self._proto_relative_url(gifd['gifUrl']),
+                'filesize': gifd.get('size'),
+                'http_headers': {
+                    'User-Agent': 'youtube-dl (like wget)',
+                },
+            })
+
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'formats': formats,
+            'description': self._og_search_description(webpage),
+            'title': self._og_search_title(webpage),
+        }

From 1ac1af9b4702fa0b921ceb45f6e7b13b35e0a2de Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 19 Feb 2015 01:43:28 +0100
Subject: [PATCH 0433/6219] release 2015.02.19.2

---
 docs/supportedsites.md | 1 +
 youtube_dl/version.py  | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 1fcec086a..607e958f6 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -191,6 +191,7 @@
  - **ign.com**
  - **imdb**: Internet Movie Database trailers
  - **imdb:list**: Internet Movie Database lists
+ - **Imgur**
  - **Ina**
  - **InfoQ**
  - **Instagram**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index db0ed70da..131dd571b 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.19.1'
+__version__ = '2015.02.19.2'

From bd03ffc16e0e09c1ec6fb5c3f2c3083238af5349 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 19 Feb 2015 01:58:52 +0100
Subject: [PATCH 0434/6219] [netzkino] Skip download in test case

Works fine from Germany, but fails from everywhere else
---
 youtube_dl/extractor/netzkino.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/netzkino.py b/youtube_dl/extractor/netzkino.py
index 93567d1e3..bc17e20aa 100644
--- a/youtube_dl/extractor/netzkino.py
+++ b/youtube_dl/extractor/netzkino.py
@@ -29,6 +29,9 @@ class NetzkinoIE(InfoExtractor):
             'timestamp': 1344858571,
             'age_limit': 12,
         },
+        'params': {
+            'skip_download': 'Download only works from Germany',
+        }
     }
 
     def _real_extract(self, url):

From f37e3f99f01df51af649a3544570b27facc5d92c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 19 Feb 2015 02:00:50 +0100
Subject: [PATCH 0435/6219] [generic] Correct test case

Video has been reuploaded / edited
---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 36a1f65bf..8dce96a64 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -532,7 +532,7 @@ class GenericIE(InfoExtractor):
             'info_dict': {
                 'id': 'Mrj4DVp2zeA',
                 'ext': 'mp4',
-                'upload_date': '20150204',
+                'upload_date': '20150212',
                 'uploader': 'The National Archives UK',
                 'description': 'md5:a236581cd2449dd2df4f93412f3f01c6',
                 'uploader_id': 'NationalArchives08',

From 71d53ace2f5c4637f5d3390298f475bbeb2452f3 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 19 Feb 2015 02:04:28 +0100
Subject: [PATCH 0436/6219] [sockshare] Do not require thumbnail anymore

Thumbnail is not present on the website anymore.
---
 youtube_dl/extractor/sockshare.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/sockshare.py b/youtube_dl/extractor/sockshare.py
index 7d3c0e937..b5fa6f1da 100644
--- a/youtube_dl/extractor/sockshare.py
+++ b/youtube_dl/extractor/sockshare.py
@@ -25,7 +25,6 @@ class SockshareIE(InfoExtractor):
             'id': '437BE28B89D799D7',
             'title': 'big_buck_bunny_720p_surround.avi',
             'ext': 'avi',
-            'thumbnail': 're:^http://.*\.jpg$',
         }
     }
 
@@ -45,7 +44,7 @@ class SockshareIE(InfoExtractor):
             ''', webpage, 'hash')
 
         fields = {
-            "hash": confirm_hash,
+            "hash": confirm_hash.encode('utf-8'),
             "confirm": "Continue as Free User"
         }
 
@@ -68,7 +67,7 @@ class SockshareIE(InfoExtractor):
             webpage, 'title', default=None)
         thumbnail = self._html_search_regex(
             r'<img\s+src="([^"]*)".+?name="bg"',
-            webpage, 'thumbnail')
+            webpage, 'thumbnail', default=None)
 
         formats = [{
             'format_id': 'sd',

From 3d54788495aae464a4ce63b5f093d4409f600dd8 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 19 Feb 2015 02:12:08 +0100
Subject: [PATCH 0437/6219] [webofstories] Fix extraction

---
 youtube_dl/extractor/webofstories.py | 22 ++++++++++------------
 1 file changed, 10 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/webofstories.py b/youtube_dl/extractor/webofstories.py
index 396cf4e83..73077a312 100644
--- a/youtube_dl/extractor/webofstories.py
+++ b/youtube_dl/extractor/webofstories.py
@@ -45,19 +45,17 @@ class WebOfStoriesIE(InfoExtractor):
         description = self._html_search_meta('description', webpage)
         thumbnail = self._og_search_thumbnail(webpage)
 
-        story_filename = self._search_regex(
-            r'\.storyFileName\("([^"]+)"\)', webpage, 'story filename')
-        speaker_id = self._search_regex(
-            r'\.speakerId\("([^"]+)"\)', webpage, 'speaker ID')
-        story_id = self._search_regex(
-            r'\.storyId\((\d+)\)', webpage, 'story ID')
-        speaker_type = self._search_regex(
-            r'\.speakerType\("([^"]+)"\)', webpage, 'speaker type')
-        great_life = self._search_regex(
-            r'isGreatLifeStory\s*=\s*(true|false)', webpage, 'great life story')
+        embed_params = [s.strip(" \r\n\t'") for s in self._search_regex(
+            r'(?s)\$\("#embedCode"\).html\(getEmbedCode\((.*?)\)',
+            webpage, 'embed params').split(',')]
+
+        (
+            _, speaker_id, story_id, story_duration,
+            speaker_type, great_life, _thumbnail, _has_subtitles,
+            story_filename, _story_order) = embed_params
+
         is_great_life_series = great_life == 'true'
-        duration = int_or_none(self._search_regex(
-            r'\.duration\((\d+)\)', webpage, 'duration', fatal=False))
+        duration = int_or_none(story_duration)
 
         # URL building, see: http://www.webofstories.com/scripts/player.js
         ms_prefix = ''

From 1a13940c8dada638f8298b6c1406f38d4a3bf270 Mon Sep 17 00:00:00 2001
From: John Boehr <jbboehr@gmail.com>
Date: Wed, 18 Feb 2015 18:12:48 -0800
Subject: [PATCH 0438/6219] [imgur] support regular URL

---
 youtube_dl/extractor/imgur.py | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/imgur.py b/youtube_dl/extractor/imgur.py
index 16488e0c4..8449c45f4 100644
--- a/youtube_dl/extractor/imgur.py
+++ b/youtube_dl/extractor/imgur.py
@@ -7,11 +7,11 @@ from ..utils import (
     int_or_none,
     js_to_json,
     mimetype2ext,
+    ExtractorError,
 )
 
-
 class ImgurIE(InfoExtractor):
-    _VALID_URL = r'https?://i\.imgur\.com/(?P<id>[a-zA-Z0-9]+)\.(?:mp4|gifv)'
+    _VALID_URL = r'https?://(?:i\.)?imgur\.com/(?P<id>[a-zA-Z0-9]+)(?:\.)?(?:mp4|gifv)?'
 
     _TESTS = [{
         'url': 'https://i.imgur.com/A61SaA1.gifv',
@@ -21,12 +21,25 @@ class ImgurIE(InfoExtractor):
             'title': 'MRW gifv is up and running without any bugs',
             'description': 'The Internet\'s visual storytelling community. Explore, share, and discuss the best visual stories the Internet has to offer.',
         },
+    }, {
+        'url': 'https://imgur.com/A61SaA1',
+        'info_dict': {
+            'id': 'A61SaA1',
+            'ext': 'mp4',
+            'title': 'MRW gifv is up and running without any bugs',
+            'description': 'The Internet\'s visual storytelling community. Explore, share, and discuss the best visual stories the Internet has to offer.',
+        },
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
+        sources = re.findall(r'<source src="([^"]+)" type="([^"]+)"', webpage)
+        if not sources:
+            raise ExtractorError(
+                'No sources found for video %s' % video_id, expected=True)
+
         width = int_or_none(self._search_regex(
             r'<param name="width" value="([0-9]+)"',
             webpage, 'width', fatal=False))

From 383456aa29c9c47066d422d3fdfdce29da2eee07 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 19 Feb 2015 04:09:52 +0100
Subject: [PATCH 0439/6219] [Makefile] Also delete *.avi files in clean

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 0636fc4cb..573c82685 100644
--- a/Makefile
+++ b/Makefile
@@ -1,7 +1,7 @@
 all: youtube-dl README.md CONTRIBUTING.md README.txt youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtube-dl.fish supportedsites
 
 clean:
-	rm -rf youtube-dl.1.temp.md youtube-dl.1 youtube-dl.bash-completion README.txt MANIFEST build/ dist/ .coverage cover/ youtube-dl.tar.gz youtube-dl.zsh youtube-dl.fish *.dump *.part *.info.json *.mp4 *.flv *.mp3 CONTRIBUTING.md.tmp youtube-dl youtube-dl.exe
+	rm -rf youtube-dl.1.temp.md youtube-dl.1 youtube-dl.bash-completion README.txt MANIFEST build/ dist/ .coverage cover/ youtube-dl.tar.gz youtube-dl.zsh youtube-dl.fish *.dump *.part *.info.json *.mp4 *.flv *.mp3 *.avi CONTRIBUTING.md.tmp youtube-dl youtube-dl.exe
 
 PREFIX ?= /usr/local
 BINDIR ?= $(PREFIX)/bin

From e880c66bd800e4769ffdff41fb9dc88cf3206c12 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 19 Feb 2015 04:12:40 +0100
Subject: [PATCH 0440/6219] [theonion] Modernize

---
 youtube_dl/extractor/theonion.py | 17 +++++------------
 1 file changed, 5 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/theonion.py b/youtube_dl/extractor/theonion.py
index b65d8e03f..10239c906 100644
--- a/youtube_dl/extractor/theonion.py
+++ b/youtube_dl/extractor/theonion.py
@@ -4,11 +4,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import ExtractorError
 
 
 class TheOnionIE(InfoExtractor):
-    _VALID_URL = r'(?x)https?://(?:www\.)?theonion\.com/video/[^,]+,(?P<article_id>[0-9]+)/?'
+    _VALID_URL = r'https?://(?:www\.)?theonion\.com/video/[^,]+,(?P<id>[0-9]+)/?'
     _TEST = {
         'url': 'http://www.theonion.com/video/man-wearing-mm-jacket-gods-image,36918/',
         'md5': '19eaa9a39cf9b9804d982e654dc791ee',
@@ -22,10 +21,8 @@ class TheOnionIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        article_id = mobj.group('article_id')
-
-        webpage = self._download_webpage(url, article_id)
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
 
         video_id = self._search_regex(
             r'"videoId":\s(\d+),', webpage, 'video ID')
@@ -34,10 +31,6 @@ class TheOnionIE(InfoExtractor):
         thumbnail = self._og_search_thumbnail(webpage)
 
         sources = re.findall(r'<source src="([^"]+)" type="([^"]+)"', webpage)
-        if not sources:
-            raise ExtractorError(
-                'No sources found for video %s' % video_id, expected=True)
-
         formats = []
         for src, type_ in sources:
             if type_ == 'video/mp4':
@@ -54,15 +47,15 @@ class TheOnionIE(InfoExtractor):
                 })
             elif type_ == 'application/x-mpegURL':
                 formats.extend(
-                    self._extract_m3u8_formats(src, video_id, preference=-1))
+                    self._extract_m3u8_formats(src, display_id, preference=-1))
             else:
                 self.report_warning(
                     'Encountered unexpected format: %s' % type_)
-
         self._sort_formats(formats)
 
         return {
             'id': video_id,
+            'display_id': display_id,
             'title': title,
             'formats': formats,
             'thumbnail': thumbnail,

From d236b37ac94cd36657c881e18b8d9187483afa80 Mon Sep 17 00:00:00 2001
From: John Boehr <jbboehr@gmail.com>
Date: Wed, 18 Feb 2015 19:28:19 -0800
Subject: [PATCH 0441/6219] [imgur] improve regex #4998

---
 youtube_dl/extractor/imgur.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/imgur.py b/youtube_dl/extractor/imgur.py
index 8449c45f4..38c961773 100644
--- a/youtube_dl/extractor/imgur.py
+++ b/youtube_dl/extractor/imgur.py
@@ -11,7 +11,7 @@ from ..utils import (
 )
 
 class ImgurIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:i\.)?imgur\.com/(?P<id>[a-zA-Z0-9]+)(?:\.)?(?:mp4|gifv)?'
+    _VALID_URL = r'https?://(?:i\.)?imgur\.com/(?P<id>[a-zA-Z0-9]+)(?:\.mp4|\.gifv)?'
 
     _TESTS = [{
         'url': 'https://i.imgur.com/A61SaA1.gifv',

From 9e2d7dca87a15cf455fa6c4843a0241ba0b7ad77 Mon Sep 17 00:00:00 2001
From: John Boehr <jbboehr@gmail.com>
Date: Wed, 18 Feb 2015 19:47:54 -0800
Subject: [PATCH 0442/6219] [imgur] improve error check for non-video URLs

---
 youtube_dl/extractor/imgur.py | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/imgur.py b/youtube_dl/extractor/imgur.py
index 38c961773..7937a5c81 100644
--- a/youtube_dl/extractor/imgur.py
+++ b/youtube_dl/extractor/imgur.py
@@ -5,6 +5,7 @@ import re
 from .common import InfoExtractor
 from ..utils import (
     int_or_none,
+    str_or_none,
     js_to_json,
     mimetype2ext,
     ExtractorError,
@@ -35,11 +36,6 @@ class ImgurIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        sources = re.findall(r'<source src="([^"]+)" type="([^"]+)"', webpage)
-        if not sources:
-            raise ExtractorError(
-                'No sources found for video %s' % video_id, expected=True)
-
         width = int_or_none(self._search_regex(
             r'<param name="width" value="([0-9]+)"',
             webpage, 'width', fatal=False))
@@ -47,10 +43,13 @@ class ImgurIE(InfoExtractor):
             r'<param name="height" value="([0-9]+)"',
             webpage, 'height', fatal=False))
 
-        formats = []
-        video_elements = self._search_regex(
+        video_elements = str_or_none(self._search_regex(
             r'(?s)<div class="video-elements">(.*?)</div>',
-            webpage, 'video elements')
+            webpage, 'video elements', fatal=False))
+        if not video_elements:
+            raise ExtractorError(
+                'No sources found for video %s' % video_id, expected=True)
+
         formats = []
         for m in re.finditer(r'<source\s+src="(?P<src>[^"]+)"\s+type="(?P<type>[^"]+)"', video_elements):
             formats.append({

From b88ba05356bdae07245dd5240b36415002e25d0b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 19 Feb 2015 05:53:09 +0100
Subject: [PATCH 0443/6219] [imgur] Simplify

---
 youtube_dl/extractor/imgur.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/imgur.py b/youtube_dl/extractor/imgur.py
index 7937a5c81..b16c7aed0 100644
--- a/youtube_dl/extractor/imgur.py
+++ b/youtube_dl/extractor/imgur.py
@@ -5,12 +5,12 @@ import re
 from .common import InfoExtractor
 from ..utils import (
     int_or_none,
-    str_or_none,
     js_to_json,
     mimetype2ext,
     ExtractorError,
 )
 
+
 class ImgurIE(InfoExtractor):
     _VALID_URL = r'https?://(?:i\.)?imgur\.com/(?P<id>[a-zA-Z0-9]+)(?:\.mp4|\.gifv)?'
 
@@ -43,12 +43,13 @@ class ImgurIE(InfoExtractor):
             r'<param name="height" value="([0-9]+)"',
             webpage, 'height', fatal=False))
 
-        video_elements = str_or_none(self._search_regex(
+        video_elements = self._search_regex(
             r'(?s)<div class="video-elements">(.*?)</div>',
-            webpage, 'video elements', fatal=False))
+            webpage, 'video elements', default=None)
         if not video_elements:
             raise ExtractorError(
-                'No sources found for video %s' % video_id, expected=True)
+                'No sources found for video %s. Maybe an image?' % video_id,
+                expected=True)
 
         formats = []
         for m in re.finditer(r'<source\s+src="(?P<src>[^"]+)"\s+type="(?P<type>[^"]+)"', video_elements):

From a3b9157f499e0ba0ef9cfd44b70b0d43795f60f1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 19 Feb 2015 13:06:53 +0100
Subject: [PATCH 0444/6219] [cbssports] Add extractor (closes #4996)

---
 youtube_dl/extractor/__init__.py  |  1 +
 youtube_dl/extractor/cbssports.py | 30 ++++++++++++++++++++++++++++++
 2 files changed, 31 insertions(+)
 create mode 100644 youtube_dl/extractor/cbssports.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 1d1f07ff5..c2424ed48 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -58,6 +58,7 @@ from .canalplus import CanalplusIE
 from .canalc2 import Canalc2IE
 from .cbs import CBSIE
 from .cbsnews import CBSNewsIE
+from .cbssports import CBSSportsIE
 from .ccc import CCCIE
 from .ceskatelevize import CeskaTelevizeIE
 from .channel9 import Channel9IE
diff --git a/youtube_dl/extractor/cbssports.py b/youtube_dl/extractor/cbssports.py
new file mode 100644
index 000000000..ae47e74cc
--- /dev/null
+++ b/youtube_dl/extractor/cbssports.py
@@ -0,0 +1,30 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+
+class CBSSportsIE(InfoExtractor):
+    _VALID_URL = r'http://www\.cbssports\.com/video/player/(?P<section>[^/]+)/(?P<id>[^/]+)'
+
+    _TEST = {
+        'url': 'http://www.cbssports.com/video/player/tennis/318462531970/0/us-open-flashbacks-1990s',
+        'info_dict': {
+            'id': '_d5_GbO8p1sT',
+            'ext': 'flv',
+            'title': 'US Open flashbacks: 1990s',
+            'description': 'Bill Macatee relives the best moments in US Open history from the 1990s.',
+        },
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        section = mobj.group('section')
+        video_id = mobj.group('id')
+        all_videos = self._download_json(
+            'http://www.cbssports.com/data/video/player/getVideos/%s?as=json' % section,
+            video_id)
+        # The json file contains the info of all the videos in the section
+        video_info = next(v for v in all_videos if v['pcid'] == video_id)
+        return self.url_result('theplatform:%s' % video_info['pid'], 'ThePlatform')

From 8807f1277f8c69488046fc7215cc79165e976ff3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 19 Feb 2015 14:54:50 +0100
Subject: [PATCH 0445/6219] [theplatform] Convert to new subtitles system

---
 test/test_subtitles.py              | 15 +++++++++++++++
 youtube_dl/extractor/theplatform.py | 18 +++++++-----------
 2 files changed, 22 insertions(+), 11 deletions(-)

diff --git a/test/test_subtitles.py b/test/test_subtitles.py
index 98d1afff4..c04fe6f22 100644
--- a/test/test_subtitles.py
+++ b/test/test_subtitles.py
@@ -24,6 +24,7 @@ from youtube_dl.extractor import (
     NRKTVIE,
     RaiIE,
     VikiIE,
+    ThePlatformIE,
 )
 
 
@@ -385,5 +386,19 @@ class TestVikiSubtitles(BaseTestSubtitles):
         self.assertEqual(md5(subtitles['en']), 'b0b781eeb45efd3f6398a925b259150b')
 
 
+class TestThePlatformSubtitles(BaseTestSubtitles):
+    # from http://www.3playmedia.com/services-features/tools/integrations/theplatform/
+    # (see http://theplatform.com/about/partners/type/subtitles-closed-captioning/)
+    url = 'theplatform:JFUjUE1_ehvq'
+    IE = ThePlatformIE
+
+    def test_allsubtitles(self):
+        self.DL.params['writesubtitles'] = True
+        self.DL.params['allsubtitles'] = True
+        subtitles = self.getSubtitles()
+        self.assertEqual(set(subtitles.keys()), set(['en']))
+        self.assertEqual(md5(subtitles['en']), '97e7670cbae3c4d26ae8bcc7fdd78d4b')
+
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index 1579822f2..5f24189cc 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -8,7 +8,7 @@ import binascii
 import hashlib
 
 
-from .subtitles import SubtitlesInfoExtractor
+from .common import InfoExtractor
 from ..compat import (
     compat_str,
 )
@@ -22,7 +22,7 @@ from ..utils import (
 _x = lambda p: xpath_with_ns(p, {'smil': 'http://www.w3.org/2005/SMIL21/Language'})
 
 
-class ThePlatformIE(SubtitlesInfoExtractor):
+class ThePlatformIE(InfoExtractor):
     _VALID_URL = r'''(?x)
         (?:https?://(?:link|player)\.theplatform\.com/[sp]/(?P<provider_id>[^/]+)/
            (?P<config>(?:[^/\?]+/(?:swf|config)|onsite)/select/)?
@@ -104,15 +104,11 @@ class ThePlatformIE(SubtitlesInfoExtractor):
         captions = info.get('captions')
         if isinstance(captions, list):
             for caption in captions:
-                lang, src = caption.get('lang'), caption.get('src')
-                if lang and src:
-                    subtitles[lang] = src
-
-        if self._downloader.params.get('listsubtitles', False):
-            self._list_available_subtitles(video_id, subtitles)
-            return
-
-        subtitles = self.extract_subtitles(video_id, subtitles)
+                lang, src, mime = caption.get('lang', 'en'), caption.get('src'), caption.get('type')
+                subtitles[lang] = [{
+                    'ext': 'srt' if mime == 'text/srt' else 'ttml',
+                    'url': src,
+                }]
 
         head = meta.find(_x('smil:head'))
         body = meta.find(_x('smil:body'))

From 5c8a3f862a325987a9ddbdc7f08ea62aad80278e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 19 Feb 2015 15:00:39 +0100
Subject: [PATCH 0446/6219] [nbc] Use a test video that works outside the US

---
 youtube_dl/extractor/nbc.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index 89a2845fe..3645d3033 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -18,13 +18,13 @@ class NBCIE(InfoExtractor):
 
     _TESTS = [
         {
-            'url': 'http://www.nbc.com/chicago-fire/video/i-am-a-firefighter/2734188',
+            'url': 'http://www.nbc.com/the-tonight-show/segments/112966',
             # md5 checksum is not stable
             'info_dict': {
-                'id': 'bTmnLCvIbaaH',
+                'id': 'c9xnCo0YPOPH',
                 'ext': 'flv',
-                'title': 'I Am a Firefighter',
-                'description': 'An emergency puts Dawson\'sf irefighter skills to the ultimate test in this four-part digital series.',
+                'title': 'Jimmy Fallon Surprises Fans at Ben & Jerry\'s',
+                'description': 'Jimmy gives out free scoops of his new "Tonight Dough" ice cream flavor by surprising customers at the Ben & Jerry\'s scoop shop.',
             },
         },
         {

From f13b1e7d7fd4a63c9ca4a0aa9930c540033cc408 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 19 Feb 2015 16:46:41 +0100
Subject: [PATCH 0447/6219] [bbccouk] Convert to new subtitles system

I haven't found any video available outside the UK, so I haven't added tests.

I have updated how the srt file is build, because (at least for www.bbc.co.uk/programmes/p02j9b69) the subtitles is inside 'span' elements.
---
 youtube_dl/extractor/bbccouk.py | 33 ++++++++++++++++++++++-----------
 1 file changed, 22 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/bbccouk.py b/youtube_dl/extractor/bbccouk.py
index f23e39545..abc34a576 100644
--- a/youtube_dl/extractor/bbccouk.py
+++ b/youtube_dl/extractor/bbccouk.py
@@ -2,12 +2,12 @@ from __future__ import unicode_literals
 
 import xml.etree.ElementTree
 
-from .subtitles import SubtitlesInfoExtractor
+from .common import InfoExtractor
 from ..utils import ExtractorError
 from ..compat import compat_HTTPError
 
 
-class BBCCoUkIE(SubtitlesInfoExtractor):
+class BBCCoUkIE(InfoExtractor):
     IE_NAME = 'bbc.co.uk'
     IE_DESC = 'BBC iPlayer'
     _VALID_URL = r'https?://(?:www\.)?bbc\.co\.uk/(?:(?:(?:programmes|iplayer(?:/[^/]+)?/(?:episode|playlist))/)|music/clips[/#])(?P<id>[\da-z]{8})'
@@ -215,17 +215,32 @@ class BBCCoUkIE(SubtitlesInfoExtractor):
             formats.extend(conn_formats)
         return formats
 
-    def _extract_captions(self, media, programme_id):
+    def _get_subtitles(self, media, programme_id):
         subtitles = {}
         for connection in self._extract_connections(media):
             captions = self._download_xml(connection.get('href'), programme_id, 'Downloading captions')
             lang = captions.get('{http://www.w3.org/XML/1998/namespace}lang', 'en')
             ps = captions.findall('./{0}body/{0}div/{0}p'.format('{http://www.w3.org/2006/10/ttaf1}'))
             srt = ''
+
+            def _extract_text(p):
+                if p.text is not None:
+                    stripped_text = p.text.strip()
+                    if stripped_text:
+                        return stripped_text
+                return ' '.join(span.text.strip() for span in p.findall('{http://www.w3.org/2006/10/ttaf1}span'))
             for pos, p in enumerate(ps):
-                srt += '%s\r\n%s --> %s\r\n%s\r\n\r\n' % (str(pos), p.get('begin'), p.get('end'),
-                                                          p.text.strip() if p.text is not None else '')
-            subtitles[lang] = srt
+                srt += '%s\r\n%s --> %s\r\n%s\r\n\r\n' % (str(pos), p.get('begin'), p.get('end'), _extract_text(p))
+            subtitles[lang] = [
+                {
+                    'url': connection.get('href'),
+                    'ext': 'ttml',
+                },
+                {
+                    'data': srt,
+                    'ext': 'srt',
+                },
+            ]
         return subtitles
 
     def _download_media_selector(self, programme_id):
@@ -249,7 +264,7 @@ class BBCCoUkIE(SubtitlesInfoExtractor):
             elif kind == 'video':
                 formats.extend(self._extract_video(media, programme_id))
             elif kind == 'captions':
-                subtitles = self._extract_captions(media, programme_id)
+                subtitles = self.extract_subtitles(media, programme_id)
 
         return formats, subtitles
 
@@ -324,10 +339,6 @@ class BBCCoUkIE(SubtitlesInfoExtractor):
         else:
             programme_id, title, description, duration, formats, subtitles = self._download_playlist(group_id)
 
-        if self._downloader.params.get('listsubtitles', False):
-            self._list_available_subtitles(programme_id, subtitles)
-            return
-
         self._sort_formats(formats)
 
         return {

From d5c69f1da41f3e370c500482303196ae845472ed Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 19 Feb 2015 21:47:11 +0600
Subject: [PATCH 0448/6219] [5min] Cover joystiq.com URLs (Closes #4962)

---
 youtube_dl/extractor/fivemin.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/fivemin.py b/youtube_dl/extractor/fivemin.py
index 5b24b921c..157094e8c 100644
--- a/youtube_dl/extractor/fivemin.py
+++ b/youtube_dl/extractor/fivemin.py
@@ -14,6 +14,7 @@ class FiveMinIE(InfoExtractor):
     IE_NAME = '5min'
     _VALID_URL = r'''(?x)
         (?:https?://[^/]*?5min\.com/Scripts/PlayerSeed\.js\?(?:.*?&)?playList=|
+            https?://(?:(?:massively|www)\.)?joystiq\.com/video/|
             5min:)
         (?P<id>\d+)
         '''

From e66e1a0046ca804461c4c9c3e890165ec9d865e1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 19 Feb 2015 22:15:19 +0600
Subject: [PATCH 0449/6219] [pornhub] Add support for playlists (Closes #4995)

---
 youtube_dl/extractor/__init__.py |  5 ++++-
 youtube_dl/extractor/pornhub.py  | 30 ++++++++++++++++++++++++++++++
 2 files changed, 34 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index c2424ed48..9b5855768 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -353,7 +353,10 @@ from .playfm import PlayFMIE
 from .playvid import PlayvidIE
 from .podomatic import PodomaticIE
 from .pornhd import PornHdIE
-from .pornhub import PornHubIE
+from .pornhub import (
+    PornHubIE,
+    PornHubPlaylistIE,
+)
 from .pornotube import PornotubeIE
 from .pornoxo import PornoXOIE
 from .promptfile import PromptFileIE
diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index fb2032832..913995044 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -110,3 +110,33 @@ class PornHubIE(InfoExtractor):
             'formats': formats,
             'age_limit': 18,
         }
+
+
+class PornHubPlaylistIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?pornhub\.com/playlist/(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'http://www.pornhub.com/playlist/6201671',
+        'info_dict': {
+            'id': '6201671',
+            'title': 'P0p4',
+        },
+        'playlist_mincount': 35,
+    }]
+
+    def _real_extract(self, url):
+        playlist_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, playlist_id)
+
+        entries = [
+            self.url_result('http://www.pornhub.com/%s' % video_url, 'PornHub')
+            for video_url in set(re.findall('href="/?(view_video\.php\?viewkey=\d+[^"]*)"', webpage))
+        ]
+
+        playlist = self._parse_json(
+            self._search_regex(
+                r'playlistObject\s*=\s*({.+?});', webpage, 'playlist'),
+            playlist_id)
+
+        return self.playlist_result(
+            entries, playlist_id, playlist.get('title'), playlist.get('description'))

From 8fc642eb5b69f25d0488509c4c4f9d8bfe2b7225 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 19 Feb 2015 22:15:49 +0600
Subject: [PATCH 0450/6219] [pornhub] Fix uploader regex

---
 youtube_dl/extractor/pornhub.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index 913995044..3a27e3789 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -56,7 +56,7 @@ class PornHubIE(InfoExtractor):
 
         video_title = self._html_search_regex(r'<h1 [^>]+>([^<]+)', webpage, 'title')
         video_uploader = self._html_search_regex(
-            r'(?s)From:&nbsp;.+?<(?:a href="/users/|a href="/channels/|<span class="username)[^>]+>(.+?)<',
+            r'(?s)From:&nbsp;.+?<(?:a href="/users/|a href="/channels/|span class="username)[^>]+>(.+?)<',
             webpage, 'uploader', fatal=False)
         thumbnail = self._html_search_regex(r'"image_url":"([^"]+)', webpage, 'thumbnail', fatal=False)
         if thumbnail:

From 6140baf4e1aa7015ac6b43e93236ac6a5813569e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 19 Feb 2015 18:17:31 +0100
Subject: [PATCH 0451/6219] [nationalgeographic] Add extractor (closes #4960)

---
 youtube_dl/extractor/__init__.py           |  1 +
 youtube_dl/extractor/nationalgeographic.py | 38 ++++++++++++++++++++++
 youtube_dl/extractor/theplatform.py        |  4 ++-
 3 files changed, 42 insertions(+), 1 deletion(-)
 create mode 100644 youtube_dl/extractor/nationalgeographic.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 9b5855768..f225ac654 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -285,6 +285,7 @@ from .myspace import MySpaceIE, MySpaceAlbumIE
 from .myspass import MySpassIE
 from .myvideo import MyVideoIE
 from .myvidster import MyVidsterIE
+from .nationalgeographic import NationalGeographicIE
 from .naver import NaverIE
 from .nba import NBAIE
 from .nbc import (
diff --git a/youtube_dl/extractor/nationalgeographic.py b/youtube_dl/extractor/nationalgeographic.py
new file mode 100644
index 000000000..c18640c5a
--- /dev/null
+++ b/youtube_dl/extractor/nationalgeographic.py
@@ -0,0 +1,38 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    smuggle_url,
+    url_basename,
+)
+
+
+class NationalGeographicIE(InfoExtractor):
+    _VALID_URL = r'http://video\.nationalgeographic\.com/video/.*?'
+
+    _TEST = {
+        'url': 'http://video.nationalgeographic.com/video/news/150210-news-crab-mating-vin?source=featuredvideo',
+        'info_dict': {
+            'id': '4DmDACA6Qtk_',
+            'ext': 'flv',
+            'title': 'Mating Crabs Busted by Sharks',
+            'description': 'md5:16f25aeffdeba55aaa8ec37e093ad8b3',
+        },
+        'add_ie': ['ThePlatform'],
+    }
+
+    def _real_extract(self, url):
+        name = url_basename(url)
+
+        webpage = self._download_webpage(url, name)
+        feed_url = self._search_regex(r'data-feed-url="([^"]+)"', webpage, 'feed url')
+        guid = self._search_regex(r'data-video-guid="([^"]+)"', webpage, 'guid')
+
+        feed = self._download_xml('%s?byGuid=%s' % (feed_url, guid), name)
+        content = feed.find('.//{http://search.yahoo.com/mrss/}content')
+        theplatform_id = url_basename(content.attrib.get('url'))
+
+        return self.url_result(smuggle_url(
+            'http://link.theplatform.com/s/ngs/%s?format=SMIL&formats=MPEG4&manifest=f4m' % theplatform_id,
+            # For some reason, the normal links don't work and we must force the use of f4m
+            {'force_smil_url': True}))
diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index 1579822f2..f7b34bd26 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -71,7 +71,9 @@ class ThePlatformIE(SubtitlesInfoExtractor):
         if not provider_id:
             provider_id = 'dJ5BDC'
 
-        if mobj.group('config'):
+        if smuggled_data.get('force_smil_url', False):
+            smil_url = url
+        elif mobj.group('config'):
             config_url = url + '&form=json'
             config_url = config_url.replace('swf/', 'config/')
             config_url = config_url.replace('onsite/', 'onsite/config/')

From a21420389edf665bfeb9610888d2d39dae374945 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 19 Feb 2015 19:28:17 +0100
Subject: [PATCH 0452/6219] release 2015.02.19.3

---
 docs/supportedsites.md | 3 +++
 youtube_dl/version.py  | 2 +-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 607e958f6..f6ba28e7a 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -68,6 +68,7 @@
  - **Canalplus**: canalplus.fr, piwiplus.fr and d8.tv
  - **CBS**
  - **CBSNews**: CBS News
+ - **CBSSports**
  - **CeskaTelevize**
  - **channel9**: Channel 9
  - **Chilloutzone**
@@ -264,6 +265,7 @@
  - **myvideo**
  - **MyVidster**
  - **n-tv.de**
+ - **NationalGeographic**
  - **Naver**
  - **NBA**
  - **NBC**
@@ -321,6 +323,7 @@
  - **podomatic**
  - **PornHd**
  - **PornHub**
+ - **PornHubPlaylist**
  - **Pornotube**
  - **PornoXO**
  - **PromptFile**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 131dd571b..537e8cf60 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.19.2'
+__version__ = '2015.02.19.3'

From dd0a58f5f028980ba76450998b218ea7a4920420 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 20 Feb 2015 01:19:38 +0600
Subject: [PATCH 0453/6219] [blinkx] Fix extraction

---
 youtube_dl/extractor/blinkx.py | 49 ++++++++++++++++------------------
 1 file changed, 23 insertions(+), 26 deletions(-)

diff --git a/youtube_dl/extractor/blinkx.py b/youtube_dl/extractor/blinkx.py
index 3e461e715..3b8eabe8f 100644
--- a/youtube_dl/extractor/blinkx.py
+++ b/youtube_dl/extractor/blinkx.py
@@ -1,40 +1,35 @@
 from __future__ import unicode_literals
 
 import json
-import re
 
 from .common import InfoExtractor
-from ..utils import remove_start
+from ..utils import (
+    remove_start,
+    int_or_none,
+)
 
 
 class BlinkxIE(InfoExtractor):
-    _VALID_URL = r'^(?:https?://(?:www\.)blinkx\.com/#?ce/|blinkx:)(?P<id>[^?]+)'
+    _VALID_URL = r'(?:https?://(?:www\.)blinkx\.com/#?ce/|blinkx:)(?P<id>[^?]+)'
     IE_NAME = 'blinkx'
 
     _TEST = {
-        'url': 'http://www.blinkx.com/ce/8aQUy7GVFYgFzpKhT0oqsilwOGFRVXk3R1ZGWWdGenBLaFQwb3FzaWx3OGFRVXk3R1ZGWWdGenB',
-        'md5': '2e9a07364af40163a908edbf10bb2492',
+        'url': 'http://www.blinkx.com/ce/Da0Gw3xc5ucpNduzLuDDlv4WC9PuI4fDi1-t6Y3LyfdY2SZS5Urbvn-UPJvrvbo8LTKTc67Wu2rPKSQDJyZeeORCR8bYkhs8lI7eqddznH2ofh5WEEdjYXnoRtj7ByQwt7atMErmXIeYKPsSDuMAAqJDlQZ-3Ff4HJVeH_s3Gh8oQ',
+        'md5': '337cf7a344663ec79bf93a526a2e06c7',
         'info_dict': {
-            'id': '8aQUy7GV',
+            'id': 'Da0Gw3xc',
             'ext': 'mp4',
-            'title': 'Police Car Rolls Away',
-            'uploader': 'stupidvideos.com',
-            'upload_date': '20131215',
-            'timestamp': 1387068000,
-            'description': 'A police car gently rolls away from a fight. Maybe it felt weird being around a confrontation and just had to get out of there!',
-            'duration': 14.886,
-            'thumbnails': [{
-                'width': 100,
-                'height': 76,
-                'resolution': '100x76',
-                'url': 'http://cdn.blinkx.com/stream/b/41/StupidVideos/20131215/1873969261/1873969261_tn_0.jpg',
-            }],
+            'title': 'No Daily Show for John Oliver; HBO Show Renewed - IGN News',
+            'uploader': 'IGN News',
+            'upload_date': '20150217',
+            'timestamp': 1424215740,
+            'description': 'HBO has renewed Last Week Tonight With John Oliver for two more seasons.',
+            'duration': 47.743333,
         },
     }
 
-    def _real_extract(self, rl):
-        m = re.match(self._VALID_URL, rl)
-        video_id = m.group('id')
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
         display_id = video_id[:8]
 
         api_url = ('https://apib4.blinkx.com/api.php?action=play_video&' +
@@ -60,18 +55,20 @@ class BlinkxIE(InfoExtractor):
             elif m['type'] in ('flv', 'mp4'):
                 vcodec = remove_start(m['vcodec'], 'ff')
                 acodec = remove_start(m['acodec'], 'ff')
-                tbr = (int(m['vbr']) + int(m['abr'])) // 1000
+                vbr = int_or_none(m.get('vbr') or m.get('vbitrate'), 1000)
+                abr = int_or_none(m.get('abr') or m.get('abitrate'), 1000)
+                tbr = vbr + abr if vbr and abr else None
                 format_id = '%s-%sk-%s' % (vcodec, tbr, m['w'])
                 formats.append({
                     'format_id': format_id,
                     'url': m['link'],
                     'vcodec': vcodec,
                     'acodec': acodec,
-                    'abr': int(m['abr']) // 1000,
-                    'vbr': int(m['vbr']) // 1000,
+                    'abr': abr,
+                    'vbr': vbr,
                     'tbr': tbr,
-                    'width': int(m['w']),
-                    'height': int(m['h']),
+                    'width': int_or_none(m.get('w')),
+                    'height': int_or_none(m.get('h')),
                 })
 
         self._sort_formats(formats)

From fb7cb6823e5ace9437bc79f2e1928a30f317856b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 19 Feb 2015 23:24:24 +0100
Subject: [PATCH 0454/6219] Remove the SubtitlesInfoExtractor class

No longer needed
---
 youtube_dl/extractor/subtitles.py | 99 -------------------------------
 1 file changed, 99 deletions(-)
 delete mode 100644 youtube_dl/extractor/subtitles.py

diff --git a/youtube_dl/extractor/subtitles.py b/youtube_dl/extractor/subtitles.py
deleted file mode 100644
index 59a51268d..000000000
--- a/youtube_dl/extractor/subtitles.py
+++ /dev/null
@@ -1,99 +0,0 @@
-from __future__ import unicode_literals
-from .common import InfoExtractor
-
-from ..compat import compat_str
-from ..utils import (
-    ExtractorError,
-)
-
-
-class SubtitlesInfoExtractor(InfoExtractor):
-    @property
-    def _have_to_download_any_subtitles(self):
-        return any([self._downloader.params.get('writesubtitles', False),
-                    self._downloader.params.get('writeautomaticsub')])
-
-    def _list_available_subtitles(self, video_id, webpage):
-        """ outputs the available subtitles for the video """
-        sub_lang_list = self._get_available_subtitles(video_id, webpage)
-        auto_captions_list = self._get_available_automatic_caption(video_id, webpage)
-        sub_lang = ",".join(list(sub_lang_list.keys()))
-        self.to_screen('%s: Available subtitles for video: %s' %
-                       (video_id, sub_lang))
-        auto_lang = ",".join(auto_captions_list.keys())
-        self.to_screen('%s: Available automatic captions for video: %s' %
-                       (video_id, auto_lang))
-
-    def extract_subtitles(self, video_id, webpage):
-        """
-        returns {sub_lang: sub} ,{} if subtitles not found or None if the
-        subtitles aren't requested.
-        """
-        if not self._have_to_download_any_subtitles:
-            return None
-        available_subs_list = {}
-        if self._downloader.params.get('writeautomaticsub', False):
-            available_subs_list.update(self._get_available_automatic_caption(video_id, webpage))
-        if self._downloader.params.get('writesubtitles', False):
-            available_subs_list.update(self._get_available_subtitles(video_id, webpage))
-
-        if not available_subs_list:  # error, it didn't get the available subtitles
-            return {}
-        if self._downloader.params.get('allsubtitles', False):
-            sub_lang_list = available_subs_list
-        else:
-            if self._downloader.params.get('subtitleslangs', False):
-                requested_langs = self._downloader.params.get('subtitleslangs')
-            elif 'en' in available_subs_list:
-                requested_langs = ['en']
-            else:
-                requested_langs = [list(available_subs_list.keys())[0]]
-
-            sub_lang_list = {}
-            for sub_lang in requested_langs:
-                if sub_lang not in available_subs_list:
-                    self._downloader.report_warning('no closed captions found in the specified language "%s"' % sub_lang)
-                    continue
-                sub_lang_list[sub_lang] = available_subs_list[sub_lang]
-
-        subtitles = {}
-        for sub_lang, url in sub_lang_list.items():
-            subtitle = self._request_subtitle_url(sub_lang, url)
-            if subtitle:
-                subtitles[sub_lang] = subtitle
-        return subtitles
-
-    def _download_subtitle_url(self, sub_lang, url):
-        return self._download_webpage(url, None, note=False)
-
-    def _request_subtitle_url(self, sub_lang, url):
-        """ makes the http request for the subtitle """
-        try:
-            sub = self._download_subtitle_url(sub_lang, url)
-        except ExtractorError as err:
-            self._downloader.report_warning('unable to download video subtitles for %s: %s' % (sub_lang, compat_str(err)))
-            return
-        if not sub:
-            self._downloader.report_warning('Did not fetch video subtitles')
-            return
-        return sub
-
-    def _get_available_subtitles(self, video_id, webpage):
-        """
-        returns {sub_lang: url} or {} if not available
-        Must be redefined by the subclasses
-        """
-
-        # By default, allow implementations to simply pass in the result
-        assert isinstance(webpage, dict), \
-            '_get_available_subtitles not implemented'
-        return webpage
-
-    def _get_available_automatic_caption(self, video_id, webpage):
-        """
-        returns {sub_lang: url} or {} if not available
-        Must be redefined by the subclasses that support automatic captions,
-        otherwise it will return {}
-        """
-        self._downloader.report_warning('Automatic Captions not supported by this server')
-        return {}

From 5e9a033e6e8054605bb87c8448f95a5bb86a71bb Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 20 Feb 2015 02:52:03 +0100
Subject: [PATCH 0455/6219] [imgur] Allow alternative values

Every now and then, imgur.com goes crazy and gives us a generic title and description (otherwise it looks all fine though).
Simply update the test case to allow for that craziness.
---
 youtube_dl/extractor/imgur.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/imgur.py b/youtube_dl/extractor/imgur.py
index b16c7aed0..fe5d95e2c 100644
--- a/youtube_dl/extractor/imgur.py
+++ b/youtube_dl/extractor/imgur.py
@@ -19,16 +19,16 @@ class ImgurIE(InfoExtractor):
         'info_dict': {
             'id': 'A61SaA1',
             'ext': 'mp4',
-            'title': 'MRW gifv is up and running without any bugs',
-            'description': 'The Internet\'s visual storytelling community. Explore, share, and discuss the best visual stories the Internet has to offer.',
+            'title': 're:Imgur GIF$|MRW gifv is up and running without any bugs$',
+            'description': 're:The origin of the Internet\'s most viral images$|The Internet\'s visual storytelling community\. Explore, share, and discuss the best visual stories the Internet has to offer\.$',
         },
     }, {
         'url': 'https://imgur.com/A61SaA1',
         'info_dict': {
             'id': 'A61SaA1',
             'ext': 'mp4',
-            'title': 'MRW gifv is up and running without any bugs',
-            'description': 'The Internet\'s visual storytelling community. Explore, share, and discuss the best visual stories the Internet has to offer.',
+            'title': 're:Imgur GIF$|MRW gifv is up and running without any bugs$',
+            'description': 're:The origin of the Internet\'s most viral images$|The Internet\'s visual storytelling community\. Explore, share, and discuss the best visual stories the Internet has to offer\.$',
         },
     }]
 

From 5da6bd00837236cf8a5dc5aeeadae5cfed7f2021 Mon Sep 17 00:00:00 2001
From: "Leslie P. Polzer" <polzer@gnu.org>
Date: Fri, 20 Feb 2015 10:49:45 +0100
Subject: [PATCH 0456/6219] [chirbit] Add new extractor.

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/chirbit.py  | 34 ++++++++++++++++++++++++++++++++
 2 files changed, 35 insertions(+)
 create mode 100644 youtube_dl/extractor/chirbit.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index f225ac654..de08e69bc 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -63,6 +63,7 @@ from .ccc import CCCIE
 from .ceskatelevize import CeskaTelevizeIE
 from .channel9 import Channel9IE
 from .chilloutzone import ChilloutzoneIE
+from .chirbit import ChirbitIE
 from .cinchcast import CinchcastIE
 from .clipfish import ClipfishIE
 from .cliphunter import CliphunterIE
diff --git a/youtube_dl/extractor/chirbit.py b/youtube_dl/extractor/chirbit.py
new file mode 100644
index 000000000..06a3e1a7a
--- /dev/null
+++ b/youtube_dl/extractor/chirbit.py
@@ -0,0 +1,34 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class ChirbitIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?chirb\.it/(?P<id>[^/]+)'
+    _TEST = {
+        'url': 'http://chirb.it/PrIPv5',
+        'md5': '9847b0dad6ac3e074568bf2cfb197de8',
+        'info_dict': {
+            'id': 'PrIPv5',
+            'display_id': 'kukushtv_1423231243',
+            'ext': 'mp3',
+            'title': 'Фасадстрой',
+            'url': 'http://audio.chirbit.com/kukushtv_1423231243.mp3'
+        }
+    }
+
+    def _real_extract(self, url):
+        audio_linkid = self._match_id(url)
+        webpage = self._download_webpage(url, audio_linkid)
+
+        audio_title = self._html_search_regex(r'<h2\s+itemprop="name">(.*?)</h2>', webpage, 'title')
+        audio_id = self._html_search_regex(r'\("setFile",\s+"http://audio.chirbit.com/(.*?).mp3"\)', webpage, 'audio ID')
+        audio_url = 'http://audio.chirbit.com/' + audio_id + '.mp3';
+
+        return {
+            'id': audio_linkid,
+            'display_id': audio_id,
+            'title': audio_title,
+            'url': audio_url
+        }

From 50efb383f01071fe5ae6ec6da2557031da54abde Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Fri, 20 Feb 2015 14:13:41 +0200
Subject: [PATCH 0457/6219] [tv4] Add new extractor (Closes #4839) (Closes
 #2103)

---
 youtube_dl/extractor/__init__.py |   1 +
 youtube_dl/extractor/tv4.py      | 100 +++++++++++++++++++++++++++++++
 2 files changed, 101 insertions(+)
 create mode 100644 youtube_dl/extractor/tv4.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index f225ac654..7d0c6b5ac 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -490,6 +490,7 @@ from .tumblr import TumblrIE
 from .tunein import TuneInIE
 from .turbo import TurboIE
 from .tutv import TutvIE
+from .tv4 import TV4IE
 from .tvigle import TvigleIE
 from .tvp import TvpIE, TvpSeriesIE
 from .tvplay import TVPlayIE
diff --git a/youtube_dl/extractor/tv4.py b/youtube_dl/extractor/tv4.py
new file mode 100644
index 000000000..1c4b6d635
--- /dev/null
+++ b/youtube_dl/extractor/tv4.py
@@ -0,0 +1,100 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    parse_iso8601,
+)
+
+
+class TV4IE(InfoExtractor):
+    IE_DESC = 'tv4.se and tv4play.se'
+    _VALID_URL = r'''(?x)https?://(?:www\.)?
+        (?:
+            tv4\.se/(?:[^/]+)/klipp/(?:.*)-|
+            tv4play\.se/
+            (?:
+                (?:program|barn)/(?:[^\?]+)\?video_id=|
+                iframe/video/|
+                film/|
+                sport/|
+            )
+        )(?P<id>[0-9]+)'''
+    _TESTS = [
+        {
+            'url': 'http://www.tv4.se/kalla-fakta/klipp/kalla-fakta-5-english-subtitles-2491650',
+            'md5': '909d6454b87b10a25aa04c4bdd416a9b',
+            'info_dict': {
+                'id': '2491650',
+                'ext': 'mp4',
+                'title': 'Kalla Fakta 5 (english subtitles)',
+                'thumbnail': 're:^https?://.*\.jpg$',
+                'timestamp': int,
+                'upload_date': '20131125',
+            },
+        },
+        {
+            'url': 'http://www.tv4play.se/iframe/video/3054113',
+            'md5': '77f851c55139ffe0ebd41b6a5552489b',
+            'info_dict': {
+                'id': '3054113',
+                'ext': 'mp4',
+                'title': 'Så här jobbar ficktjuvarna - se avslöjande bilder',
+                'thumbnail': 're:^https?://.*\.jpg$',
+                'description': 'Unika bilder avslöjar hur turisternas fickor vittjas mitt på Stockholms central. Två experter på ficktjuvarna avslöjar knepen du ska se upp för.',
+                'timestamp': int,
+                'upload_date': '20150130',
+            },
+        },
+        {
+            'url': 'http://www.tv4play.se/sport/3060959',
+            'only_matching': True,
+        },
+        {
+            'url': 'http://www.tv4play.se/film/2378136',
+            'only_matching': True,
+        },
+        {
+            'url': 'http://www.tv4play.se/barn/looney-tunes?video_id=3062412',
+            'only_matching': True,
+        },
+    ]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        info = self._download_json(
+            'http://www.tv4play.se/player/assets/%s.json' % video_id, video_id, 'Downloading video info JSON')
+
+        # If is_geo_restricted is true, it doesn't neceserally mean we can't download it
+        if info['is_geo_restricted']:
+            self.report_warning('This content might not be available in your country due to licensing restrictions.')
+        if info['requires_subscription']:
+            raise ExtractorError('This content requires subscription.', expected=True)
+
+        sources_data = self._download_json(
+            'https://prima.tv4play.se/api/web/asset/%s/play.json?protocol=http&videoFormat=MP4' % video_id, video_id, 'Downloading sources JSON')
+        sources = sources_data['playback']
+
+        formats = []
+        for item in sources.get('items', {}).get('item', []):
+            ext, bitrate = item['mediaFormat'], item['bitrate']
+            formats.append({
+                'format_id': '%s_%s' % (ext, bitrate),
+                'tbr': bitrate,
+                'ext': ext,
+                'url': item['url'],
+            })
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': info['title'],
+            'formats': formats,
+            'description': info.get('description'),
+            'timestamp': parse_iso8601(info.get('broadcast_date_time')),
+            'duration': info.get('duration'),
+            'thumbnail': info.get('image'),
+            'is_live': sources.get('live'),
+        }

From 365577f5676d63089cb834855dd4cdce7d0dc8aa Mon Sep 17 00:00:00 2001
From: "Leslie P. Polzer" <polzer@gnu.org>
Date: Fri, 20 Feb 2015 14:48:12 +0100
Subject: [PATCH 0458/6219] [chirbit] add profile extractor.

---
 youtube_dl/extractor/__init__.py |  2 +-
 youtube_dl/extractor/chirbit.py  | 63 ++++++++++++++++++++++++++++++++
 2 files changed, 64 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index de08e69bc..94e150826 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -63,7 +63,7 @@ from .ccc import CCCIE
 from .ceskatelevize import CeskaTelevizeIE
 from .channel9 import Channel9IE
 from .chilloutzone import ChilloutzoneIE
-from .chirbit import ChirbitIE
+from .chirbit import ChirbitIE, ChirbitProfileIE
 from .cinchcast import CinchcastIE
 from .clipfish import ClipfishIE
 from .cliphunter import CliphunterIE
diff --git a/youtube_dl/extractor/chirbit.py b/youtube_dl/extractor/chirbit.py
index 06a3e1a7a..47ce94aa0 100644
--- a/youtube_dl/extractor/chirbit.py
+++ b/youtube_dl/extractor/chirbit.py
@@ -1,7 +1,10 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
+from ..utils import clean_html
 
 
 class ChirbitIE(InfoExtractor):
@@ -32,3 +35,63 @@ class ChirbitIE(InfoExtractor):
             'title': audio_title,
             'url': audio_url
         }
+
+class ChirbitProfileIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?chirbit.com/(?P<id>[^/]+)'
+    _TEST = {
+        'url': 'http://chirbit.com/ScarletBeauty',
+        'playlist_count': 3,
+        'info_dict': {
+            '_type': 'playlist',
+            'title': 'ScarletBeauty',
+            'id': 'ScarletBeauty'
+        }
+    }
+
+    def _real_extract(self, url):
+        profile_id = self._match_id(url)
+
+        # Chirbit has a pretty weird "Last Page" navigation behavior.
+        # We grab the profile's oldest entry to determine when to
+        # stop fetching entries.
+        oldestpage = self._download_webpage(url + '/24599', profile_id)
+        oldest_page_entries = re.findall(
+            r'''soundFile:\s*"http://audio.chirbit.com/(.*?).mp3"''',
+            oldestpage);
+        oldestentry = clean_html(oldest_page_entries[-1]);
+
+        ids = []
+        titles = []
+        n = 0
+        while True:
+            page = self._download_webpage(url + '/' + str(n), profile_id)
+            page_ids = re.findall(
+                r'''soundFile:\s*"http://audio.chirbit.com/(.*?).mp3"''',
+                page);
+            page_titles = re.findall(
+                r'''<div\s+class="chirbit_title"\s*>(.*?)</div>''',
+                page);
+            ids += page_ids
+            titles += page_titles
+            if oldestentry in page_ids:
+                break
+            n += 1
+
+        entries = []
+        i = 0
+        for id in ids:
+            entries.append({
+                'id': id,
+                'title': titles[i],
+                'url': 'http://audio.chirbit.com/' + id + '.mp3'
+            });
+            i += 1
+
+        info_dict = {
+            '_type': 'playlist',
+            'id': profile_id,
+            'title': profile_id,
+            'entries': entries
+        }
+
+        return info_dict;

From 1bd838608fda2da789c8991ec7fc194ed5ec0a1f Mon Sep 17 00:00:00 2001
From: minusf <minusf@gmail.com>
Date: Fri, 20 Feb 2015 18:24:20 +0100
Subject: [PATCH 0459/6219] prefer 'code' to 'uri' if present

---
 youtube_dl/extractor/ted.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ted.py b/youtube_dl/extractor/ted.py
index 10b3b706a..5e5322974 100644
--- a/youtube_dl/extractor/ted.py
+++ b/youtube_dl/extractor/ted.py
@@ -134,9 +134,13 @@ class TEDIE(SubtitlesInfoExtractor):
 
         if talk_info.get('external') is not None:
             self.to_screen('Found video from %s' % talk_info['external']['service'])
+            if 'code' in talk_info['external']:
+                ext_url = talk_info['external']['code']
+            else:
+                ext_url = talk_info['external']['uri']
             return {
                 '_type': 'url',
-                'url': talk_info['external']['uri'],
+                'url': ext_url,
             }
 
         formats = [{

From a461a11989e147c30e3611005861cc63dcd16af3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Feb 2015 00:14:38 +0600
Subject: [PATCH 0460/6219] [ted] Improve external video handling and add test

---
 youtube_dl/extractor/ted.py | 31 ++++++++++++++++++++++++-------
 1 file changed, 24 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/ted.py b/youtube_dl/extractor/ted.py
index 5e5322974..59678399d 100644
--- a/youtube_dl/extractor/ted.py
+++ b/youtube_dl/extractor/ted.py
@@ -83,6 +83,22 @@ class TEDIE(SubtitlesInfoExtractor):
         'params': {
             'skip_download': True,
         },
+    }, {
+        # YouTube video
+        'url': 'http://www.ted.com/talks/jeffrey_kluger_the_sibling_bond',
+        'add_ie': ['Youtube'],
+        'info_dict': {
+            'id': 'aFBIPO-P7LM',
+            'ext': 'mp4',
+            'title': 'The hidden power of siblings: Jeff Kluger at TEDxAsheville',
+            'description': 'md5:3d7a4f50d95ca5dd67104e2a20f43fe1',
+            'uploader': 'TEDx Talks',
+            'uploader_id': 'TEDxTalks',
+            'upload_date': '20111216',
+        },
+        'params': {
+            'skip_download': True,
+        },
     }]
 
     _NATIVE_FORMATS = {
@@ -132,15 +148,16 @@ class TEDIE(SubtitlesInfoExtractor):
 
         talk_info = self._extract_info(webpage)['talks'][0]
 
-        if talk_info.get('external') is not None:
-            self.to_screen('Found video from %s' % talk_info['external']['service'])
-            if 'code' in talk_info['external']:
-                ext_url = talk_info['external']['code']
-            else:
-                ext_url = talk_info['external']['uri']
+        external = talk_info.get('external')
+        if external:
+            service = external['service']
+            self.to_screen('Found video from %s' % service)
+            ext_url = None
+            if service.lower() == 'youtube':
+                ext_url = external.get('code')
             return {
                 '_type': 'url',
-                'url': ext_url,
+                'url': ext_url or external['uri'],
             }
 
         formats = [{

From 3da0db62e6c4122b151349e7a05f59803da4fbbc Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 20 Feb 2015 23:22:47 +0100
Subject: [PATCH 0461/6219] [escapist] Fix extraction (fixes #5017)

---
 youtube_dl/extractor/escapist.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/escapist.py b/youtube_dl/extractor/escapist.py
index 4303feccd..6b693b3b6 100644
--- a/youtube_dl/extractor/escapist.py
+++ b/youtube_dl/extractor/escapist.py
@@ -22,6 +22,7 @@ class EscapistIE(InfoExtractor):
             'uploader_id': 'the-escapist-presents',
             'uploader': 'The Escapist Presents',
             'title': "Breaking Down Baldur's Gate",
+            'thumbnail': 're:^https?://.*\.jpg$',
         }
     }
 
@@ -40,9 +41,8 @@ class EscapistIE(InfoExtractor):
         raw_title = self._html_search_meta('title', webpage, fatal=True)
         title = raw_title.partition(' : ')[2]
 
-        player_url = self._og_search_video_url(webpage, name='player URL')
-        config_url = compat_urllib_parse.unquote(self._search_regex(
-            r'config=(.*)$', player_url, 'config URL'))
+        config_url = compat_urllib_parse.unquote(self._html_search_regex(
+            r'<param name="flashvars" value="config=([^"&]+)', webpage, 'config URL'))
 
         formats = []
 
@@ -81,5 +81,4 @@ class EscapistIE(InfoExtractor):
             'title': title,
             'thumbnail': self._og_search_thumbnail(webpage),
             'description': description,
-            'player_url': player_url,
         }

From 1e9a9e167d5e518f44402394bb4cc23712ff32fc Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 20 Feb 2015 23:23:12 +0100
Subject: [PATCH 0462/6219] release 2015.02.20

---
 docs/supportedsites.md | 1 +
 youtube_dl/version.py  | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index f6ba28e7a..dbff5e270 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -451,6 +451,7 @@
  - **Turbo**
  - **Tutv**
  - **tv.dfb.de**
+ - **TV4**: tv4.se and tv4play.se
  - **tvigle**: Интернет-телевидение Tvigle.ru
  - **tvp.pl**
  - **tvp.pl:Series**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 537e8cf60..9fd0ee963 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.19.3'
+__version__ = '2015.02.20'

From a00a8bcc8a131e0bea605ab3cfa0f09bedde9e0c Mon Sep 17 00:00:00 2001
From: CyberJacob <cyberjacob@gmail.com>
Date: Fri, 20 Feb 2015 22:43:51 +0000
Subject: [PATCH 0463/6219] Change example URLs in readme (fixes #5018y)

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index a2c148311..8ea31d605 100644
--- a/README.md
+++ b/README.md
@@ -571,7 +571,7 @@ Support requests for services that **do** purchase the rights to distribute thei
 
 ### How can I detect whether a given URL is supported by youtube-dl?
 
-For one, have a look at the [list of supported sites](docs/supportedsites.md). Note that it can sometimes happen that the site changes its URL scheme (say, from http://example.com/v/1234567 to http://example.com/v/1234567 ) and youtube-dl reports an URL of a service in that list as unsupported. In that case, simply report a bug.
+For one, have a look at the [list of supported sites](docs/supportedsites.md). Note that it can sometimes happen that the site changes its URL scheme (say, from http://example.com/video/1234567 to http://example.com/v/1234567 ) and youtube-dl reports an URL of a service in that list as unsupported. In that case, simply report a bug.
 
 It is *not* possible to detect whether a URL is supported or not. That's because youtube-dl contains a generic extractor which matches **all** URLs. You may be tempted to disable, exclude, or remove the generic extractor, but the generic extractor not only allows users to extract videos from lots of websites that embed a video from another service, but may also be used to extract video from a service that it's hosting itself. Therefore, we neither recommend nor support disabling, excluding, or removing the generic extractor.
 

From fad6768bd1a67dccbb153ac371d3e82575321ea9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Feb 2015 18:00:25 +0600
Subject: [PATCH 0464/6219] [vimeo] Fix password protected videos (Closes
 #5001)

---
 youtube_dl/extractor/vimeo.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 78d287e0e..5f8649e35 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 import json
 import re
 import itertools
+import hashlib
 
 from .common import InfoExtractor
 from .subtitles import SubtitlesInfoExtractor
@@ -225,6 +226,10 @@ class VimeoIE(VimeoBaseInfoExtractor, SubtitlesInfoExtractor):
         if mobj.group('pro') or mobj.group('player'):
             url = 'http://player.vimeo.com/video/' + video_id
 
+        password = self._downloader.params.get('videopassword', None)
+        if password:
+            headers['Cookie'] = '%s_password=%s' % (video_id, hashlib.md5(password).hexdigest())
+
         # Retrieve video webpage to extract further information
         request = compat_urllib_request.Request(url, None, headers)
         try:

From 62b013df0dcb5f902d745b77b3de62b64b828863 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Feb 2015 18:31:10 +0600
Subject: [PATCH 0465/6219] [vimeo] Encode password before hash calculation

---
 youtube_dl/extractor/vimeo.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 5f8649e35..4cd2f73d9 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -228,7 +228,8 @@ class VimeoIE(VimeoBaseInfoExtractor, SubtitlesInfoExtractor):
 
         password = self._downloader.params.get('videopassword', None)
         if password:
-            headers['Cookie'] = '%s_password=%s' % (video_id, hashlib.md5(password).hexdigest())
+            headers['Cookie'] = '%s_password=%s' % (
+                video_id, hashlib.md5(password.encode('utf-8')).hexdigest())
 
         # Retrieve video webpage to extract further information
         request = compat_urllib_request.Request(url, None, headers)

From 77b2986b5b0246234b72ae9dd78fb40f9d37374f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 21 Feb 2015 14:51:07 +0100
Subject: [PATCH 0466/6219] [extractor/common] Recognize Indian censorship
 (#5021)

---
 youtube_dl/extractor/common.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 08b8ad37c..ee64ad329 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -391,6 +391,16 @@ class InfoExtractor(object):
             if blocked_iframe:
                 msg += ' Visit %s for more details' % blocked_iframe
             raise ExtractorError(msg, expected=True)
+        if '<title>The URL you requested has been blocked</title>' in content[:512]:
+            msg = (
+                'Access to this webpage has been blocked by Indian censorship. '
+                'Use a VPN or proxy server (with --proxy) to route around it.')
+            block_msg = self._html_search_regex(
+                r'</h1><p>(.*?)</p>',
+                content, 'block message', default=None)
+            if block_msg:
+                msg += ' (Message: "%s")' % block_msg.replace('\n', ' ')
+            raise ExtractorError(msg, expected=True)
 
         return content
 

From 8fb3ac3649ca7df6f328971f58afa84dd9d05cc6 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 21 Feb 2015 14:55:13 +0100
Subject: [PATCH 0467/6219] PEP8: W503

---
 devscripts/check-porn.py        |  8 ++++----
 test/test_swfinterp.py          |  4 ++--
 youtube_dl/YoutubeDL.py         |  8 ++++----
 youtube_dl/__init__.py          | 16 ++++++++--------
 youtube_dl/downloader/common.py | 10 +++++-----
 youtube_dl/downloader/f4m.py    |  4 ++--
 youtube_dl/extractor/adobetv.py |  7 ++++---
 youtube_dl/extractor/common.py  |  4 ++--
 youtube_dl/extractor/defense.py |  5 +++--
 youtube_dl/utils.py             |  4 ++--
 10 files changed, 36 insertions(+), 34 deletions(-)

diff --git a/devscripts/check-porn.py b/devscripts/check-porn.py
index 216282712..6a5bd9eda 100644
--- a/devscripts/check-porn.py
+++ b/devscripts/check-porn.py
@@ -45,12 +45,12 @@ for test in get_testcases():
 
         RESULT = ('.' + domain + '\n' in LIST or '\n' + domain + '\n' in LIST)
 
-    if RESULT and ('info_dict' not in test or 'age_limit' not in test['info_dict']
-                   or test['info_dict']['age_limit'] != 18):
+    if RESULT and ('info_dict' not in test or 'age_limit' not in test['info_dict'] or
+                   test['info_dict']['age_limit'] != 18):
         print('\nPotential missing age_limit check: {0}'.format(test['name']))
 
-    elif not RESULT and ('info_dict' in test and 'age_limit' in test['info_dict']
-                         and test['info_dict']['age_limit'] == 18):
+    elif not RESULT and ('info_dict' in test and 'age_limit' in test['info_dict'] and
+                         test['info_dict']['age_limit'] == 18):
         print('\nPotential false negative: {0}'.format(test['name']))
 
     else:
diff --git a/test/test_swfinterp.py b/test/test_swfinterp.py
index 9f18055e6..f1e899819 100644
--- a/test/test_swfinterp.py
+++ b/test/test_swfinterp.py
@@ -34,8 +34,8 @@ def _make_testfunc(testfile):
     def test_func(self):
         as_file = os.path.join(TEST_DIR, testfile)
         swf_file = os.path.join(TEST_DIR, test_id + '.swf')
-        if ((not os.path.exists(swf_file))
-                or os.path.getmtime(swf_file) < os.path.getmtime(as_file)):
+        if ((not os.path.exists(swf_file)) or
+                os.path.getmtime(swf_file) < os.path.getmtime(as_file)):
             # Recompile
             try:
                 subprocess.check_call([
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 88809783b..ca7c3f5c6 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -308,8 +308,8 @@ class YoutubeDL(object):
                     raise
 
         if (sys.version_info >= (3,) and sys.platform != 'win32' and
-                sys.getfilesystemencoding() in ['ascii', 'ANSI_X3.4-1968']
-                and not params.get('restrictfilenames', False)):
+                sys.getfilesystemencoding() in ['ascii', 'ANSI_X3.4-1968'] and
+                not params.get('restrictfilenames', False)):
             # On Python 3, the Unicode filesystem API will throw errors (#1474)
             self.report_warning(
                 'Assuming --restrict-filenames since file system encoding '
@@ -1366,8 +1366,8 @@ class YoutubeDL(object):
         """Download a given list of URLs."""
         outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
         if (len(url_list) > 1 and
-                '%' not in outtmpl
-                and self.params.get('max_downloads') != 1):
+                '%' not in outtmpl and
+                self.params.get('max_downloads') != 1):
             raise SameFileError(outtmpl)
 
         for url in url_list:
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index eac2a26ec..25ab3fdfe 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -189,14 +189,14 @@ def _real_main(argv=None):
         # In Python 2, sys.argv is a bytestring (also note http://bugs.python.org/issue2128 for Windows systems)
         if opts.outtmpl is not None:
             opts.outtmpl = opts.outtmpl.decode(preferredencoding())
-    outtmpl = ((opts.outtmpl is not None and opts.outtmpl)
-               or (opts.format == '-1' and opts.usetitle and '%(title)s-%(id)s-%(format)s.%(ext)s')
-               or (opts.format == '-1' and '%(id)s-%(format)s.%(ext)s')
-               or (opts.usetitle and opts.autonumber and '%(autonumber)s-%(title)s-%(id)s.%(ext)s')
-               or (opts.usetitle and '%(title)s-%(id)s.%(ext)s')
-               or (opts.useid and '%(id)s.%(ext)s')
-               or (opts.autonumber and '%(autonumber)s-%(id)s.%(ext)s')
-               or DEFAULT_OUTTMPL)
+    outtmpl = ((opts.outtmpl is not None and opts.outtmpl) or
+               (opts.format == '-1' and opts.usetitle and '%(title)s-%(id)s-%(format)s.%(ext)s') or
+               (opts.format == '-1' and '%(id)s-%(format)s.%(ext)s') or
+               (opts.usetitle and opts.autonumber and '%(autonumber)s-%(title)s-%(id)s.%(ext)s') or
+               (opts.usetitle and '%(title)s-%(id)s.%(ext)s') or
+               (opts.useid and '%(id)s.%(ext)s') or
+               (opts.autonumber and '%(autonumber)s-%(id)s.%(ext)s') or
+               DEFAULT_OUTTMPL)
     if not os.path.splitext(outtmpl)[1] and opts.extractaudio:
         parser.error('Cannot download a video and extract audio into the same'
                      ' file! Use "{0}.%(ext)s" instead of "{0}" as the output'
diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index 45e55b99c..3ae90021a 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -311,14 +311,14 @@ class FileDownloader(object):
         """
 
         nooverwrites_and_exists = (
-            self.params.get('nooverwrites', False)
-            and os.path.exists(encodeFilename(filename))
+            self.params.get('nooverwrites', False) and
+            os.path.exists(encodeFilename(filename))
         )
 
         continuedl_and_exists = (
-            self.params.get('continuedl', False)
-            and os.path.isfile(encodeFilename(filename))
-            and not self.params.get('nopart', False)
+            self.params.get('continuedl', False) and
+            os.path.isfile(encodeFilename(filename)) and
+            not self.params.get('nopart', False)
         )
 
         # Check file already present
diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index b40ebfa50..7b8fe8cf5 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -325,8 +325,8 @@ class F4mFD(FileDownloader):
                 state['frag_index'] += 1
 
             estimated_size = (
-                (state['downloaded_bytes'] + frag_total_bytes)
-                / (state['frag_index'] + 1) * total_frags)
+                (state['downloaded_bytes'] + frag_total_bytes) /
+                (state['frag_index'] + 1) * total_frags)
             time_now = time.time()
             state['total_bytes_estimate'] = estimated_size
             state['elapsed'] = time_now - start
diff --git a/youtube_dl/extractor/adobetv.py b/youtube_dl/extractor/adobetv.py
index 28e07f8b0..97d128560 100644
--- a/youtube_dl/extractor/adobetv.py
+++ b/youtube_dl/extractor/adobetv.py
@@ -28,7 +28,6 @@ class AdobeTVIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-
         webpage = self._download_webpage(url, video_id)
 
         player = self._parse_json(
@@ -44,8 +43,10 @@ class AdobeTVIE(InfoExtractor):
             self._html_search_meta('datepublished', webpage, 'upload date'))
 
         duration = parse_duration(
-            self._html_search_meta('duration', webpage, 'duration')
-            or self._search_regex(r'Runtime:\s*(\d{2}:\d{2}:\d{2})', webpage, 'duration'))
+            self._html_search_meta('duration', webpage, 'duration') or
+            self._search_regex(
+                r'Runtime:\s*(\d{2}:\d{2}:\d{2})',
+                webpage, 'duration', fatal=False))
 
         view_count = str_to_int(self._search_regex(
             r'<div class="views">\s*Views?:\s*([\d,.]+)\s*</div>',
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index ee64ad329..d3f86cf4a 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -808,8 +808,8 @@ class InfoExtractor(object):
             media_nodes = manifest.findall('{http://ns.adobe.com/f4m/2.0}media')
         for i, media_el in enumerate(media_nodes):
             if manifest_version == '2.0':
-                manifest_url = ('/'.join(manifest_url.split('/')[:-1]) + '/'
-                                + (media_el.attrib.get('href') or media_el.attrib.get('url')))
+                manifest_url = ('/'.join(manifest_url.split('/')[:-1]) + '/' +
+                                (media_el.attrib.get('href') or media_el.attrib.get('url')))
             tbr = int_or_none(media_el.attrib.get('bitrate'))
             formats.append({
                 'format_id': '-'.join(filter(None, [f4m_id, 'f4m-%d' % (i if tbr is None else tbr)])),
diff --git a/youtube_dl/extractor/defense.py b/youtube_dl/extractor/defense.py
index 2b90bf4fc..98e3aedfd 100644
--- a/youtube_dl/extractor/defense.py
+++ b/youtube_dl/extractor/defense.py
@@ -25,8 +25,9 @@ class DefenseGouvFrIE(InfoExtractor):
             r"flashvars.pvg_id=\"(\d+)\";",
             webpage, 'ID')
 
-        json_url = ('http://static.videos.gouv.fr/brightcovehub/export/json/'
-                    + video_id)
+        json_url = (
+            'http://static.videos.gouv.fr/brightcovehub/export/json/%s' %
+            video_id)
         info = self._download_json(json_url, title, 'Downloading JSON config')
         video_url = info['renditions'][0]['url']
 
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 238b6556b..475fad3c9 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -900,8 +900,8 @@ def _windows_write_string(s, out):
     def not_a_console(handle):
         if handle == INVALID_HANDLE_VALUE or handle is None:
             return True
-        return ((GetFileType(handle) & ~FILE_TYPE_REMOTE) != FILE_TYPE_CHAR
-                or GetConsoleMode(handle, ctypes.byref(ctypes.wintypes.DWORD())) == 0)
+        return ((GetFileType(handle) & ~FILE_TYPE_REMOTE) != FILE_TYPE_CHAR or
+                GetConsoleMode(handle, ctypes.byref(ctypes.wintypes.DWORD())) == 0)
 
     if not_a_console(h):
         return False

From 93540ee10e4143f8de7885af2d68c213aab7d8cb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 21 Feb 2015 19:31:39 +0100
Subject: [PATCH 0468/6219] [rtve] Fix the video url

Changing mvod to mvod1 fixes the url, we don't need to add the query.
---
 youtube_dl/extractor/rtve.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/rtve.py b/youtube_dl/extractor/rtve.py
index 3469d9578..e60f85b5b 100644
--- a/youtube_dl/extractor/rtve.py
+++ b/youtube_dl/extractor/rtve.py
@@ -6,6 +6,7 @@ import re
 import time
 
 from .common import InfoExtractor
+from ..compat import compat_urlparse
 from ..utils import (
     struct_unpack,
     remove_end,
@@ -96,12 +97,10 @@ class RTVEALaCartaIE(InfoExtractor):
             ).replace('.net.rtve', '.multimedia.cdn.rtve')
             video_path = self._download_webpage(
                 auth_url, video_id, 'Getting video url')
-            # Use mvod.akcdn instead of flash.akamaihd.multimedia.cdn to get
+            # Use mvod1.akcdn instead of flash.akamaihd.multimedia.cdn to get
             # the right Content-Length header and the mp4 format
-            video_url = (
-                'http://mvod.akcdn.rtve.es/{0}&v=2.6.8'
-                '&fp=MAC%2016,0,0,296&r=MRUGG&g=OEOJWFXNFGCP'.format(video_path)
-            )
+            video_url = compat_urlparse.urljoin(
+                'http://mvod1.akcdn.rtve.es/', video_path)
 
         return {
             'id': video_id,

From 4aeccadf4ef8528c252c917d071e98a091f7766c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Feb 2015 01:38:57 +0600
Subject: [PATCH 0469/6219] [zapiks] Add extractor (#5014)

---
 youtube_dl/extractor/__init__.py |   1 +
 youtube_dl/extractor/zapiks.py   | 102 +++++++++++++++++++++++++++++++
 2 files changed, 103 insertions(+)
 create mode 100644 youtube_dl/extractor/zapiks.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 7d0c6b5ac..ef0adfd87 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -612,6 +612,7 @@ from .youtube import (
     YoutubeUserIE,
     YoutubeWatchLaterIE,
 )
+from .zapiks import ZapiksIE
 from .zdf import ZDFIE, ZDFChannelIE
 from .zingmp3 import (
     ZingMp3SongIE,
diff --git a/youtube_dl/extractor/zapiks.py b/youtube_dl/extractor/zapiks.py
new file mode 100644
index 000000000..12810637e
--- /dev/null
+++ b/youtube_dl/extractor/zapiks.py
@@ -0,0 +1,102 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    parse_duration,
+    parse_iso8601,
+    xpath_with_ns,
+    xpath_text,
+    int_or_none,
+)
+
+
+class ZapiksIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?zapiks\.fr/(?:(?P<display_id>.+?)\.html|index\.php\?.*\bmedia_id=(?P<id>\d+))'
+    _TESTS = [
+        {
+            'url': 'http://www.zapiks.fr/ep2s3-bon-appetit-eh-be-viva.html',
+            'md5': 'aeb3c473b2d564b2d46d664d28d5f050',
+            'info_dict': {
+                'id': '80798',
+                'ext': 'mp4',
+                'title': 'EP2S3 - Bon Appétit - Eh bé viva les pyrénées con!',
+                'description': 'md5:7054d6f6f620c6519be1fe710d4da847',
+                'thumbnail': 're:^https?://.*\.jpg$',
+                'duration': 528,
+                'timestamp': 1359044972,
+                'upload_date': '20130124',
+                'view_count': int,
+                'comment_count': int,
+            },
+        },
+        {
+            'url': 'http://www.zapiks.fr/index.php?action=playerIframe&amp;media_id=118046&amp;width=640&amp;height=360&amp;autoStart=false&amp;language=fr',
+            'only_matching': True,
+        },
+    ]
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        display_id = mobj.group('display_id') or video_id
+
+        webpage = self._download_webpage(url, display_id)
+
+        if not video_id:
+            video_id = self._search_regex(
+                r'data-media-id="(\d+)"', webpage, 'video id')
+
+        playlist = self._download_xml(
+            'http://www.zapiks.fr/view/index.php?action=playlist&media_id=%s&lang=en' % video_id,
+            display_id)
+
+        NS_MAP = {
+            'jwplayer': 'http://rss.jwpcdn.com/'
+        }
+
+        def ns(path):
+            return xpath_with_ns(path, NS_MAP)
+
+        item = playlist.find('./channel/item')
+
+        title = xpath_text(item, 'title', 'title') or self._og_search_title(webpage)
+        description = self._og_search_description(webpage, default=None)
+        thumbnail = xpath_text(
+            item, ns('./jwplayer:image'), 'thumbnail') or self._og_search_thumbnail(webpage, default=None)
+        duration = parse_duration(self._html_search_meta(
+            'duration', webpage, 'duration', default=None))
+        timestamp = parse_iso8601(self._html_search_meta(
+            'uploadDate', webpage, 'upload date', default=None), ' ')
+
+        view_count = int_or_none(self._search_regex(
+            r'UserPlays:(\d+)', webpage, 'view count', default=None))
+        comment_count = int_or_none(self._search_regex(
+            r'UserComments:(\d+)', webpage, 'comment count', default=None))
+
+        formats = []
+        for source in item.findall(ns('./jwplayer:source')):
+            format_id = source.attrib['label']
+            f = {
+                'url': source.attrib['file'],
+                'format_id': format_id,
+            }
+            m = re.search(r'^(?P<height>\d+)[pP]', format_id)
+            if m:
+                f['height'] = int(m.group('height'))
+            formats.append(f)
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'duration': duration,
+            'timestamp': timestamp,
+            'view_count': view_count,
+            'comment_count': comment_count,
+            'formats': formats,
+        }

From 255fca5eea70a171530a5a0f2af143362f0211cf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Feb 2015 01:39:26 +0600
Subject: [PATCH 0470/6219] [generic] Add support for Zapiks embeds (#5014)

---
 youtube_dl/extractor/generic.py | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 8dce96a64..875e1bf05 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -547,7 +547,16 @@ class GenericIE(InfoExtractor):
                 'id': 'aanslagen-kopenhagen',
                 'title': 'Aanslagen Kopenhagen | RTL Nieuws',
             }
-        }
+        },
+        # Zapiks embed
+        {
+            'url': 'http://www.skipass.com/news/116090-bon-appetit-s5ep3-baqueira-mi-cor.html',
+            'info_dict': {
+                'id': '118046',
+                'ext': 'mp4',
+                'title': 'EP3S5 - Bon Appétit - Baqueira Mi Corazon !',
+            }
+        },
     ]
 
     def report_following_redirect(self, new_url):
@@ -1098,6 +1107,12 @@ class GenericIE(InfoExtractor):
         if mobj is not None:
             return self.url_result(mobj.group('url'), 'Livestream')
 
+        # Look for Zapiks embed
+        mobj = re.search(
+            r'<iframe[^>]+src="(?P<url>https?://(?:www\.)?zapiks\.fr/index\.php\?.+?)"', webpage)
+        if mobj is not None:
+            return self.url_result(mobj.group('url'), 'Zapiks')
+
         def check_video(vurl):
             if YoutubeIE.suitable(vurl):
                 return True

From ea5152cae110d55b82c755c23926f077b90c071c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Feb 2015 01:42:47 +0600
Subject: [PATCH 0471/6219] [zapiks] Extend _VALID_URL (#5014)

---
 youtube_dl/extractor/zapiks.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/zapiks.py b/youtube_dl/extractor/zapiks.py
index 12810637e..22a9a57e8 100644
--- a/youtube_dl/extractor/zapiks.py
+++ b/youtube_dl/extractor/zapiks.py
@@ -14,7 +14,7 @@ from ..utils import (
 
 
 class ZapiksIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?zapiks\.fr/(?:(?P<display_id>.+?)\.html|index\.php\?.*\bmedia_id=(?P<id>\d+))'
+    _VALID_URL = r'https?://(?:www\.)?zapiks\.(?:fr|com)/(?:(?:[a-z]{2}/)?(?P<display_id>.+?)\.html|index\.php\?.*\bmedia_id=(?P<id>\d+))'
     _TESTS = [
         {
             'url': 'http://www.zapiks.fr/ep2s3-bon-appetit-eh-be-viva.html',
@@ -32,6 +32,14 @@ class ZapiksIE(InfoExtractor):
                 'comment_count': int,
             },
         },
+        {
+            'url': 'http://www.zapiks.com/ep3s5-bon-appetit-baqueira-m-1.html',
+            'only_matching': True,
+        },
+        {
+            'url': 'http://www.zapiks.com/nl/ep3s5-bon-appetit-baqueira-m-1.html',
+            'only_matching': True,
+        },
         {
             'url': 'http://www.zapiks.fr/index.php?action=playerIframe&amp;media_id=118046&amp;width=640&amp;height=360&amp;autoStart=false&amp;language=fr',
             'only_matching': True,

From c5181ab4101323de94bdb20850c64711c625c3ad Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Feb 2015 02:10:26 +0600
Subject: [PATCH 0472/6219] [gdcvault] Fix rtmp streams (Closes #5024)

---
 youtube_dl/extractor/gdcvault.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/gdcvault.py b/youtube_dl/extractor/gdcvault.py
index fed968f51..05f58f1af 100644
--- a/youtube_dl/extractor/gdcvault.py
+++ b/youtube_dl/extractor/gdcvault.py
@@ -7,6 +7,7 @@ from ..compat import (
     compat_urllib_parse,
     compat_urllib_request,
 )
+from ..utils import remove_end
 
 
 class GDCVaultIE(InfoExtractor):
@@ -68,7 +69,9 @@ class GDCVaultIE(InfoExtractor):
         akami_url = xml_description.find('./metadata/akamaiHost').text
         slide_video_path = xml_description.find('./metadata/slideVideo').text
         video_formats.append({
-            'url': 'rtmp://' + akami_url + '/' + slide_video_path,
+            'url': 'rtmp://%s/ondemand?ovpfv=1.1' % 'fms.digitallyspeaking.com/cfx/st',
+            'play_path': remove_end(slide_video_path, '.flv'),
+            'ext': 'flv',
             'format_note': 'slide deck video',
             'quality': -2,
             'preference': -2,
@@ -76,7 +79,9 @@ class GDCVaultIE(InfoExtractor):
         })
         speaker_video_path = xml_description.find('./metadata/speakerVideo').text
         video_formats.append({
-            'url': 'rtmp://' + akami_url + '/' + speaker_video_path,
+            'url': 'rtmp://%s/ondemand?ovpfv=1.1' % 'fms.digitallyspeaking.com/cfx/st',
+            'play_path': remove_end(speaker_video_path, '.flv'),
+            'ext': 'flv',
             'format_note': 'speaker video',
             'quality': -1,
             'preference': -1,

From 314368c822428437e60bbc24af65d5415717632c Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Sat, 21 Feb 2015 22:19:39 +0200
Subject: [PATCH 0473/6219] [teamcoco] Fix extraction

Also, use a single style of quotes
---
 youtube_dl/extractor/teamcoco.py | 49 ++++++++++++++++----------------
 1 file changed, 24 insertions(+), 25 deletions(-)

diff --git a/youtube_dl/extractor/teamcoco.py b/youtube_dl/extractor/teamcoco.py
index a73da1c9c..5793dbc10 100644
--- a/youtube_dl/extractor/teamcoco.py
+++ b/youtube_dl/extractor/teamcoco.py
@@ -1,8 +1,10 @@
 from __future__ import unicode_literals
 
+import base64
 import re
 
 from .common import InfoExtractor
+from ..utils import qualities
 
 
 class TeamcocoIE(InfoExtractor):
@@ -24,8 +26,8 @@ class TeamcocoIE(InfoExtractor):
             'info_dict': {
                 'id': '19705',
                 'ext': 'mp4',
-                "description": "Louis C.K. got starstruck by George W. Bush, so what? Part one.",
-                "title": "Louis C.K. Interview Pt. 1 11/3/11",
+                'description': 'Louis C.K. got starstruck by George W. Bush, so what? Part one.',
+                'title': 'Louis C.K. Interview Pt. 1 11/3/11',
                 'age_limit': 0,
             }
         }
@@ -42,42 +44,39 @@ class TeamcocoIE(InfoExtractor):
         display_id = mobj.group('display_id')
         webpage = self._download_webpage(url, display_id)
 
-        video_id = mobj.group("video_id")
+        video_id = mobj.group('video_id')
         if not video_id:
             video_id = self._html_search_regex(
                 self._VIDEO_ID_REGEXES, webpage, 'video id')
 
-        data_url = 'http://teamcoco.com/cvp/2.0/%s.xml' % video_id
-        data = self._download_xml(
-            data_url, display_id, 'Downloading data webpage')
+        embed_url = 'http://teamcoco.com/embed/v/%s' % video_id
+        embed = self._download_webpage(
+            embed_url, video_id, 'Downloading embed page')
+
+        encoded_data = self._search_regex(
+            r'"preload"\s*:\s*"([^"]+)"', embed, 'encoded data')
+        data = self._parse_json(
+            base64.b64decode(encoded_data.encode('ascii')).decode('utf-8'), video_id)
 
-        qualities = ['500k', '480p', '1000k', '720p', '1080p']
         formats = []
-        for filed in data.findall('files/file'):
-            if filed.attrib.get('playmode') == 'all':
-                # it just duplicates one of the entries
-                break
-            file_url = filed.text
-            m_format = re.search(r'(\d+(k|p))\.mp4', file_url)
+        get_quality = qualities(['500k', '480p', '1000k', '720p', '1080p'])
+        for filed in data['files']:
+            m_format = re.search(r'(\d+(k|p))\.mp4', filed['url'])
             if m_format is not None:
                 format_id = m_format.group(1)
             else:
-                format_id = filed.attrib['bitrate']
+                format_id = filed['bitrate']
             tbr = (
-                int(filed.attrib['bitrate'])
-                if filed.attrib['bitrate'].isdigit()
+                int(filed['bitrate'])
+                if filed['bitrate'].isdigit()
                 else None)
 
-            try:
-                quality = qualities.index(format_id)
-            except ValueError:
-                quality = -1
             formats.append({
-                'url': file_url,
+                'url': filed['url'],
                 'ext': 'mp4',
                 'tbr': tbr,
                 'format_id': format_id,
-                'quality': quality,
+                'quality': get_quality(format_id),
             })
 
         self._sort_formats(formats)
@@ -86,8 +85,8 @@ class TeamcocoIE(InfoExtractor):
             'id': video_id,
             'display_id': display_id,
             'formats': formats,
-            'title': self._og_search_title(webpage),
-            'thumbnail': self._og_search_thumbnail(webpage),
-            'description': self._og_search_description(webpage),
+            'title': data['title'],
+            'thumbnail': data.get('thumb', {}).get('href'),
+            'description': data.get('teaser'),
             'age_limit': self._family_friendly_search(webpage),
         }

From e086e0eb6cef80db2d4ab44572a1a5d6b6f1dee0 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 21 Feb 2015 21:25:29 +0100
Subject: [PATCH 0474/6219] release 2015.02.21

---
 docs/supportedsites.md | 1 +
 youtube_dl/version.py  | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index dbff5e270..5fe3e47cd 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -559,6 +559,7 @@
  - **youtube:subscriptions**: YouTube.com subscriptions feed, "ytsubs" keyword (requires authentication)
  - **youtube:user**: YouTube.com user videos (URL or "ytuser" keyword)
  - **youtube:watch_later**: Youtube watch later list, ":ytwatchlater" for short (requires authentication)
+ - **Zapiks**
  - **ZDF**
  - **ZDFChannel**
  - **zingmp3:album**: mp3.zing.vn albums
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 9fd0ee963..7c8b29c3b 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.20'
+__version__ = '2015.02.21'

From 4d1718481755dde078678b6e55d457fc6351fcdd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 21 Feb 2015 22:31:53 +0100
Subject: [PATCH 0475/6219] [YoutubeDL] don't set the 'requested_subtitles'
 without writesubtitles or writeautomaticsub

---
 test/test_subtitles.py  | 10 +++++-----
 youtube_dl/YoutubeDL.py |  6 ++++--
 2 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/test/test_subtitles.py b/test/test_subtitles.py
index c04fe6f22..457f268fa 100644
--- a/test/test_subtitles.py
+++ b/test/test_subtitles.py
@@ -113,7 +113,7 @@ class TestYoutubeSubtitles(BaseTestSubtitles):
         self.DL.params['writesubtitles'] = True
         self.DL.params['allsubtitles'] = True
         subtitles = self.getSubtitles()
-        self.assertEqual(len(subtitles), 0)
+        self.assertFalse(subtitles)
 
     def test_youtube_multiple_langs(self):
         self.url = 'QRS8MkLhQmM'
@@ -152,7 +152,7 @@ class TestDailymotionSubtitles(BaseTestSubtitles):
         self.DL.params['writesubtitles'] = True
         self.DL.params['allsubtitles'] = True
         subtitles = self.getSubtitles()
-        self.assertEqual(len(subtitles), 0)
+        self.assertFalse(subtitles)
 
     def test_multiple_langs(self):
         self.DL.params['writesubtitles'] = True
@@ -246,7 +246,7 @@ class TestVimeoSubtitles(BaseTestSubtitles):
         self.DL.params['writesubtitles'] = True
         self.DL.params['allsubtitles'] = True
         subtitles = self.getSubtitles()
-        self.assertEqual(len(subtitles), 0)
+        self.assertFalse(subtitles)
 
     def test_multiple_langs(self):
         self.DL.params['writesubtitles'] = True
@@ -281,7 +281,7 @@ class TestWallaSubtitles(BaseTestSubtitles):
         self.DL.params['writesubtitles'] = True
         self.DL.params['allsubtitles'] = True
         subtitles = self.getSubtitles()
-        self.assertEqual(len(subtitles), 0)
+        self.assertFalse(subtitles)
 
 
 class TestCeskaTelevizeSubtitles(BaseTestSubtitles):
@@ -308,7 +308,7 @@ class TestCeskaTelevizeSubtitles(BaseTestSubtitles):
         self.DL.params['writesubtitles'] = True
         self.DL.params['allsubtitles'] = True
         subtitles = self.getSubtitles()
-        self.assertEqual(len(subtitles), 0)
+        self.assertFalse(subtitles)
 
 
 class TestLyndaSubtitles(BaseTestSubtitles):
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index f8b8fb0c1..088b111eb 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1164,8 +1164,10 @@ class YoutubeDL(object):
                 if lang not in available_subs:
                     available_subs[lang] = cap_info
 
-        if not available_subs:
-            return available_subs
+        if (not self.params.get('writesubtitles') and not
+                self.params.get('writeautomaticsub') or not
+                available_subs):
+            return None
 
         if self.params.get('allsubtitles', False):
             requested_langs = available_subs.keys()

From 03091e372f7033fa52c7961b1a99cd3790c0f60f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 21 Feb 2015 22:33:11 +0100
Subject: [PATCH 0476/6219] [ted] Always extract the subtitles

The required info is already in the webpage
---
 youtube_dl/extractor/ted.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/ted.py b/youtube_dl/extractor/ted.py
index 1809eaae4..0c38c8f89 100644
--- a/youtube_dl/extractor/ted.py
+++ b/youtube_dl/extractor/ted.py
@@ -163,8 +163,6 @@ class TEDIE(InfoExtractor):
         self._sort_formats(formats)
 
         video_id = compat_str(talk_info['id'])
-        # subtitles
-        video_subtitles = self.extract_subtitles(video_id, talk_info)
 
         thumbnail = talk_info['thumb']
         if not thumbnail.startswith('http'):
@@ -175,7 +173,7 @@ class TEDIE(InfoExtractor):
             'uploader': talk_info['speaker'],
             'thumbnail': thumbnail,
             'description': self._og_search_description(webpage),
-            'subtitles': video_subtitles,
+            'subtitles': self._get_subtitles(video_id, talk_info),
             'formats': formats,
             'duration': talk_info.get('duration'),
         }
@@ -194,7 +192,6 @@ class TEDIE(InfoExtractor):
                 ]
             return sub_lang_list
         else:
-            self._downloader.report_warning('video doesn\'t have subtitles')
             return {}
 
     def _watch_info(self, url, name):

From ab84349b16b3c94775543a04855fc77005f8237e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 22 Feb 2015 11:26:27 +0100
Subject: [PATCH 0477/6219] [test/YoutubeDL] Add test for subtitles

Updated the offlinetest make target to not skip it
---
 Makefile               |  2 +-
 test/test_YoutubeDL.py | 52 ++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 53 insertions(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 0636fc4cb..07c90c225 100644
--- a/Makefile
+++ b/Makefile
@@ -43,7 +43,7 @@ test:
 ot: offlinetest
 
 offlinetest: codetest
-	nosetests --verbose test --exclude test_download --exclude test_age_restriction --exclude test_subtitles --exclude test_write_annotations --exclude test_youtube_lists
+	nosetests --verbose test --exclude test_download.py --exclude test_age_restriction.py --exclude test_subtitles.py --exclude test_write_annotations.py --exclude test_youtube_lists.py
 
 tar: youtube-dl.tar.gz
 
diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index b1cd6a69f..e11292211 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -337,6 +337,58 @@ class TestFormatSelection(unittest.TestCase):
         downloaded = ydl.downloaded_info_dicts[0]
         self.assertEqual(downloaded['format_id'], 'G')
 
+    def test_subtitles(self):
+        def s_formats(lang, autocaption=False):
+            return [{
+                'ext': ext,
+                'url': 'http://localhost/video.%s.%s' % (lang, ext),
+                '_auto': autocaption,
+            } for ext in ['vtt', 'srt', 'ass']]
+        subtitles = dict((l, s_formats(l)) for l in ['en', 'fr', 'es'])
+        auto_captions = dict((l, s_formats(l, True)) for l in ['it', 'pt', 'es'])
+        info_dict = {
+            'id': 'test',
+            'title': 'Test',
+            'url': 'http://localhost/video.mp4',
+            'subtitles': subtitles,
+            'automatic_captions': auto_captions,
+            'extractor': 'TEST',
+        }
+
+        def get_info(params={}):
+            params.setdefault('simulate', True)
+            ydl = YDL(params)
+            ydl.report_warning = lambda *args, **kargs: None
+            return ydl.process_video_result(info_dict, download=False)
+
+        result = get_info()
+        self.assertFalse(result.get('requested_subtitles'))
+        self.assertEqual(result['subtitles'], subtitles)
+        self.assertEqual(result['automatic_captions'], auto_captions)
+
+        result = get_info({'writesubtitles': True})
+        subs = result['requested_subtitles']
+        self.assertTrue(subs)
+        self.assertEqual(set(subs.keys()), set(['en']))
+        self.assertTrue(subs['en'].get('data') is None)
+        self.assertEqual(subs['en']['ext'], 'ass')
+
+        result = get_info({'writesubtitles': True, 'subtitlesformat': 'foo/srt'})
+        subs = result['requested_subtitles']
+        self.assertEqual(subs['en']['ext'], 'srt')
+
+        result = get_info({'writesubtitles': True, 'subtitleslangs': ['es', 'fr', 'it']})
+        subs = result['requested_subtitles']
+        self.assertTrue(subs)
+        self.assertEqual(set(subs.keys()), set(['es', 'fr']))
+
+        result = get_info({'writesubtitles': True, 'writeautomaticsub': True, 'subtitleslangs': ['es', 'pt']})
+        subs = result['requested_subtitles']
+        self.assertTrue(subs)
+        self.assertEqual(set(subs.keys()), set(['es', 'pt']))
+        self.assertFalse(subs['es']['_auto'])
+        self.assertTrue(subs['pt']['_auto'])
+
     def test_add_extra_info(self):
         test_dict = {
             'extractor': 'Foo',

From 98c70d6fc7006c8cbbd76fb1b8661d758fc4f5d8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 22 Feb 2015 11:37:27 +0100
Subject: [PATCH 0478/6219] [YoutubeDL] only add normal subtitles to the
 'requested_subtitles' field if 'writesubtitles' is True

---
 test/test_YoutubeDL.py  |  7 +++++++
 youtube_dl/YoutubeDL.py | 10 ++++++----
 2 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index e11292211..055e42555 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -389,6 +389,13 @@ class TestFormatSelection(unittest.TestCase):
         self.assertFalse(subs['es']['_auto'])
         self.assertTrue(subs['pt']['_auto'])
 
+        result = get_info({'writeautomaticsub': True, 'subtitleslangs': ['es', 'pt']})
+        subs = result['requested_subtitles']
+        self.assertTrue(subs)
+        self.assertEqual(set(subs.keys()), set(['es', 'pt']))
+        self.assertTrue(subs['es']['_auto'])
+        self.assertTrue(subs['pt']['_auto'])
+
     def test_add_extra_info(self):
         test_dict = {
             'extractor': 'Foo',
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 088b111eb..7319323e5 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1156,11 +1156,13 @@ class YoutubeDL(object):
         info_dict.update(formats_to_download[-1])
         return info_dict
 
-    def process_subtitles(self, video_id, available_subs, available_autocaps):
+    def process_subtitles(self, video_id, normal_subtitles, automatic_captions):
         """Select the requested subtitles and their format"""
-        if available_autocaps and self.params.get('writeautomaticsub'):
-            available_subs = available_subs.copy()
-            for lang, cap_info in available_autocaps.items():
+        available_subs = {}
+        if normal_subtitles and self.params.get('writesubtitles'):
+            available_subs.update(normal_subtitles)
+        if automatic_captions and self.params.get('writeautomaticsub'):
+            for lang, cap_info in automatic_captions.items():
                 if lang not in available_subs:
                     available_subs[lang] = cap_info
 

From b7bb76df05f53d4fc0570d07be5abcee238745e6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 22 Feb 2015 11:49:27 +0100
Subject: [PATCH 0479/6219] [test/subtitles] Remove some tests

Test only with 'allsubtitles', the language selection is already tested in test_YoutubeDL.py
---
 test/test_subtitles.py | 121 +++++------------------------------------
 1 file changed, 13 insertions(+), 108 deletions(-)

diff --git a/test/test_subtitles.py b/test/test_subtitles.py
index 457f268fa..aa4e2bec4 100644
--- a/test/test_subtitles.py
+++ b/test/test_subtitles.py
@@ -57,22 +57,15 @@ class TestYoutubeSubtitles(BaseTestSubtitles):
     url = 'QRS8MkLhQmM'
     IE = YoutubeIE
 
-    def test_youtube_subtitles(self):
-        self.DL.params['writesubtitles'] = True
-        subtitles = self.getSubtitles()
-        self.assertEqual(md5(subtitles['en']), '4cd9278a35ba2305f47354ee13472260')
-
-    def test_youtube_subtitles_lang(self):
-        self.DL.params['writesubtitles'] = True
-        self.DL.params['subtitleslangs'] = ['it']
-        subtitles = self.getSubtitles()
-        self.assertEqual(md5(subtitles['it']), '164a51f16f260476a05b50fe4c2f161d')
-
     def test_youtube_allsubtitles(self):
         self.DL.params['writesubtitles'] = True
         self.DL.params['allsubtitles'] = True
         subtitles = self.getSubtitles()
         self.assertEqual(len(subtitles.keys()), 13)
+        self.assertEqual(md5(subtitles['en']), '4cd9278a35ba2305f47354ee13472260')
+        self.assertEqual(md5(subtitles['it']), '164a51f16f260476a05b50fe4c2f161d')
+        for lang in ['it', 'fr', 'de']:
+            self.assertTrue(subtitles.get(lang) is not None, 'Subtitles for \'%s\' not extracted' % lang)
 
     def test_youtube_subtitles_sbv_format(self):
         self.DL.params['writesubtitles'] = True
@@ -86,12 +79,6 @@ class TestYoutubeSubtitles(BaseTestSubtitles):
         subtitles = self.getSubtitles()
         self.assertEqual(md5(subtitles['en']), '3cb210999d3e021bd6c7f0ea751eab06')
 
-    def test_youtube_list_subtitles(self):
-        self.DL.expect_warning('Video doesn\'t have automatic captions')
-        self.DL.params['listsubtitles'] = True
-        info_dict = self.getInfoDict()
-        self.assertEqual(info_dict, None)
-
     def test_youtube_automatic_captions(self):
         self.url = '8YoUxe5ncPo'
         self.DL.params['writeautomaticsub'] = True
@@ -115,36 +102,20 @@ class TestYoutubeSubtitles(BaseTestSubtitles):
         subtitles = self.getSubtitles()
         self.assertFalse(subtitles)
 
-    def test_youtube_multiple_langs(self):
-        self.url = 'QRS8MkLhQmM'
-        self.DL.params['writesubtitles'] = True
-        langs = ['it', 'fr', 'de']
-        self.DL.params['subtitleslangs'] = langs
-        subtitles = self.getSubtitles()
-        for lang in langs:
-            self.assertTrue(subtitles.get(lang) is not None, 'Subtitles for \'%s\' not extracted' % lang)
-
 
 class TestDailymotionSubtitles(BaseTestSubtitles):
     url = 'http://www.dailymotion.com/video/xczg00'
     IE = DailymotionIE
 
-    def test_subtitles(self):
-        self.DL.params['writesubtitles'] = True
-        subtitles = self.getSubtitles()
-        self.assertEqual(md5(subtitles['en']), '976553874490cba125086bbfea3ff76f')
-
-    def test_subtitles_lang(self):
-        self.DL.params['writesubtitles'] = True
-        self.DL.params['subtitleslangs'] = ['fr']
-        subtitles = self.getSubtitles()
-        self.assertEqual(md5(subtitles['fr']), '594564ec7d588942e384e920e5341792')
-
     def test_allsubtitles(self):
         self.DL.params['writesubtitles'] = True
         self.DL.params['allsubtitles'] = True
         subtitles = self.getSubtitles()
         self.assertTrue(len(subtitles.keys()) >= 6)
+        self.assertEqual(md5(subtitles['en']), '976553874490cba125086bbfea3ff76f')
+        self.assertEqual(md5(subtitles['fr']), '594564ec7d588942e384e920e5341792')
+        for lang in ['es', 'fr', 'de']:
+            self.assertTrue(subtitles.get(lang) is not None, 'Subtitles for \'%s\' not extracted' % lang)
 
     def test_nosubtitles(self):
         self.DL.expect_warning('video doesn\'t have subtitles')
@@ -154,51 +125,19 @@ class TestDailymotionSubtitles(BaseTestSubtitles):
         subtitles = self.getSubtitles()
         self.assertFalse(subtitles)
 
-    def test_multiple_langs(self):
-        self.DL.params['writesubtitles'] = True
-        langs = ['es', 'fr', 'de']
-        self.DL.params['subtitleslangs'] = langs
-        subtitles = self.getSubtitles()
-        for lang in langs:
-            self.assertTrue(subtitles.get(lang) is not None, 'Subtitles for \'%s\' not extracted' % lang)
-
 
 class TestTedSubtitles(BaseTestSubtitles):
     url = 'http://www.ted.com/talks/dan_dennett_on_our_consciousness.html'
     IE = TEDIE
 
-    def test_no_writesubtitles(self):
-        subtitles = self.getSubtitles()
-        self.assertFalse(subtitles)
-
-    def test_subtitles(self):
-        self.DL.params['writesubtitles'] = True
-        subtitles = self.getSubtitles()
-        self.assertEqual(md5(subtitles['en']), '4262c1665ff928a2dada178f62cb8d14')
-
-    def test_subtitles_lang(self):
-        self.DL.params['writesubtitles'] = True
-        self.DL.params['subtitleslangs'] = ['fr']
-        subtitles = self.getSubtitles()
-        self.assertEqual(md5(subtitles['fr']), '66a63f7f42c97a50f8c0e90bc7797bb5')
-
     def test_allsubtitles(self):
         self.DL.params['writesubtitles'] = True
         self.DL.params['allsubtitles'] = True
         subtitles = self.getSubtitles()
         self.assertTrue(len(subtitles.keys()) >= 28)
-
-    def test_list_subtitles(self):
-        self.DL.params['listsubtitles'] = True
-        info_dict = self.getInfoDict()
-        self.assertEqual(info_dict, None)
-
-    def test_multiple_langs(self):
-        self.DL.params['writesubtitles'] = True
-        langs = ['es', 'fr', 'de']
-        self.DL.params['subtitleslangs'] = langs
-        subtitles = self.getSubtitles()
-        for lang in langs:
+        self.assertEqual(md5(subtitles['en']), '4262c1665ff928a2dada178f62cb8d14')
+        self.assertEqual(md5(subtitles['fr']), '66a63f7f42c97a50f8c0e90bc7797bb5')
+        for lang in ['es', 'fr', 'de']:
             self.assertTrue(subtitles.get(lang) is not None, 'Subtitles for \'%s\' not extracted' % lang)
 
 
@@ -206,11 +145,6 @@ class TestBlipTVSubtitles(BaseTestSubtitles):
     url = 'http://blip.tv/a/a-6603250'
     IE = BlipTVIE
 
-    def test_list_subtitles(self):
-        self.DL.params['listsubtitles'] = True
-        info_dict = self.getInfoDict()
-        self.assertEqual(info_dict, None)
-
     def test_allsubtitles(self):
         self.DL.params['writesubtitles'] = True
         self.DL.params['allsubtitles'] = True
@@ -223,22 +157,13 @@ class TestVimeoSubtitles(BaseTestSubtitles):
     url = 'http://vimeo.com/76979871'
     IE = VimeoIE
 
-    def test_subtitles(self):
-        self.DL.params['writesubtitles'] = True
-        subtitles = self.getSubtitles()
-        self.assertEqual(md5(subtitles['en']), '8062383cf4dec168fc40a088aa6d5888')
-
-    def test_subtitles_lang(self):
-        self.DL.params['writesubtitles'] = True
-        self.DL.params['subtitleslangs'] = ['fr']
-        subtitles = self.getSubtitles()
-        self.assertEqual(md5(subtitles['fr']), 'b6191146a6c5d3a452244d853fde6dc8')
-
     def test_allsubtitles(self):
         self.DL.params['writesubtitles'] = True
         self.DL.params['allsubtitles'] = True
         subtitles = self.getSubtitles()
         self.assertEqual(set(subtitles.keys()), set(['de', 'en', 'es', 'fr']))
+        self.assertEqual(md5(subtitles['en']), '8062383cf4dec168fc40a088aa6d5888')
+        self.assertEqual(md5(subtitles['fr']), 'b6191146a6c5d3a452244d853fde6dc8')
 
     def test_nosubtitles(self):
         self.DL.expect_warning('video doesn\'t have subtitles')
@@ -248,25 +173,11 @@ class TestVimeoSubtitles(BaseTestSubtitles):
         subtitles = self.getSubtitles()
         self.assertFalse(subtitles)
 
-    def test_multiple_langs(self):
-        self.DL.params['writesubtitles'] = True
-        langs = ['es', 'fr', 'de']
-        self.DL.params['subtitleslangs'] = langs
-        subtitles = self.getSubtitles()
-        for lang in langs:
-            self.assertTrue(subtitles.get(lang) is not None, 'Subtitles for \'%s\' not extracted' % lang)
-
 
 class TestWallaSubtitles(BaseTestSubtitles):
     url = 'http://vod.walla.co.il/movie/2705958/the-yes-men'
     IE = WallaIE
 
-    def test_list_subtitles(self):
-        self.DL.expect_warning('Automatic Captions not supported by this server')
-        self.DL.params['listsubtitles'] = True
-        info_dict = self.getInfoDict()
-        self.assertEqual(info_dict, None)
-
     def test_allsubtitles(self):
         self.DL.expect_warning('Automatic Captions not supported by this server')
         self.DL.params['writesubtitles'] = True
@@ -288,12 +199,6 @@ class TestCeskaTelevizeSubtitles(BaseTestSubtitles):
     url = 'http://www.ceskatelevize.cz/ivysilani/10600540290-u6-uzasny-svet-techniky'
     IE = CeskaTelevizeIE
 
-    def test_list_subtitles(self):
-        self.DL.expect_warning('Automatic Captions not supported by this server')
-        self.DL.params['listsubtitles'] = True
-        info_dict = self.getInfoDict()
-        self.assertEqual(info_dict, None)
-
     def test_allsubtitles(self):
         self.DL.expect_warning('Automatic Captions not supported by this server')
         self.DL.params['writesubtitles'] = True

From 80970e531bd377e1952ac358e7e345cfbf23593d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 22 Feb 2015 11:52:22 +0100
Subject: [PATCH 0480/6219] [test/subtitles] Update checksum for Viki

---
 test/test_subtitles.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/test_subtitles.py b/test/test_subtitles.py
index aa4e2bec4..7f93f0a75 100644
--- a/test/test_subtitles.py
+++ b/test/test_subtitles.py
@@ -288,7 +288,7 @@ class TestVikiSubtitles(BaseTestSubtitles):
         self.DL.params['allsubtitles'] = True
         subtitles = self.getSubtitles()
         self.assertEqual(set(subtitles.keys()), set(['en']))
-        self.assertEqual(md5(subtitles['en']), 'b0b781eeb45efd3f6398a925b259150b')
+        self.assertEqual(md5(subtitles['en']), '53cb083a5914b2d84ef1ab67b880d18a')
 
 
 class TestThePlatformSubtitles(BaseTestSubtitles):

From f311cfa23153fee51f94f14d1ab1f7f8b6a74702 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Feb 2015 19:53:32 +0600
Subject: [PATCH 0481/6219] [appletrailers] Extend _VALID_URL (Closes #5027)

---
 youtube_dl/extractor/appletrailers.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/appletrailers.py b/youtube_dl/extractor/appletrailers.py
index 43e82847f..9c718ea66 100644
--- a/youtube_dl/extractor/appletrailers.py
+++ b/youtube_dl/extractor/appletrailers.py
@@ -11,7 +11,7 @@ from ..utils import (
 
 
 class AppleTrailersIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?trailers\.apple\.com/trailers/(?P<company>[^/]+)/(?P<movie>[^/]+)'
+    _VALID_URL = r'https?://(?:www\.)?trailers\.apple\.com/(?:trailers|ca)/(?P<company>[^/]+)/(?P<movie>[^/]+)'
     _TEST = {
         "url": "http://trailers.apple.com/trailers/wb/manofsteel/",
         'info_dict': {

From 35b798230334e984977090ae03a307eaf7eedbc1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Feb 2015 19:58:39 +0600
Subject: [PATCH 0482/6219] [appletrailers] Add test (#5027)

---
 youtube_dl/extractor/appletrailers.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/appletrailers.py b/youtube_dl/extractor/appletrailers.py
index 9c718ea66..576f03b5b 100644
--- a/youtube_dl/extractor/appletrailers.py
+++ b/youtube_dl/extractor/appletrailers.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 class AppleTrailersIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?trailers\.apple\.com/(?:trailers|ca)/(?P<company>[^/]+)/(?P<movie>[^/]+)'
-    _TEST = {
+    _TESTS = [{
         "url": "http://trailers.apple.com/trailers/wb/manofsteel/",
         'info_dict': {
             'id': 'manofsteel',
@@ -63,7 +63,10 @@ class AppleTrailersIE(InfoExtractor):
                 },
             },
         ]
-    }
+    }, {
+        'url': 'http://trailers.apple.com/ca/metropole/autrui/',
+        'only_matching': True,
+    }]
 
     _JSON_RE = r'iTunes.playURL\((.*?)\);'
 

From c010af6f195c2e84aec7d0ddec060fcbe9c45089 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Feb 2015 23:11:33 +0600
Subject: [PATCH 0483/6219] [escapist] Make regexes more robust (Closes #5028)

---
 youtube_dl/extractor/escapist.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/escapist.py b/youtube_dl/extractor/escapist.py
index 6b693b3b6..b49b9869f 100644
--- a/youtube_dl/extractor/escapist.py
+++ b/youtube_dl/extractor/escapist.py
@@ -31,10 +31,10 @@ class EscapistIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         uploader_id = self._html_search_regex(
-            r"<h1 class='headline'><a href='/videos/view/(.*?)'",
+            r"<h1\s+class='headline'>\s*<a\s+href='/videos/view/(.*?)'",
             webpage, 'uploader ID', fatal=False)
         uploader = self._html_search_regex(
-            r"<h1 class='headline'>(.*?)</a>",
+            r"<h1\s+class='headline'>(.*?)</a>",
             webpage, 'uploader', fatal=False)
         description = self._html_search_meta('description', webpage)
 
@@ -42,7 +42,7 @@ class EscapistIE(InfoExtractor):
         title = raw_title.partition(' : ')[2]
 
         config_url = compat_urllib_parse.unquote(self._html_search_regex(
-            r'<param name="flashvars" value="config=([^"&]+)', webpage, 'config URL'))
+            r'<param\s+name="flashvars"\s+value="config=([^"&]+)', webpage, 'config URL'))
 
         formats = []
 

From 9fe6ef7ab249af1bd8f302d37dc6c13a2e4817f1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 23 Feb 2015 03:30:10 +0600
Subject: [PATCH 0484/6219] [extractor/common] Fix preference for m3u8 quality
 selection URL

---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index d3f86cf4a..79f6d199b 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -833,7 +833,7 @@ class InfoExtractor(object):
             'url': m3u8_url,
             'ext': ext,
             'protocol': 'm3u8',
-            'preference': -1,
+            'preference': preference - 1 if preference else -1,
             'resolution': 'multiple',
             'format_note': 'Quality selection URL',
         }]

From fcc3e6138b372e13578949dc724f456ae76dd065 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 23 Feb 2015 03:32:53 +0600
Subject: [PATCH 0485/6219] [r7] Add extractor (Closes #4405, closes #5004)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/r7.py       | 88 ++++++++++++++++++++++++++++++++
 2 files changed, 89 insertions(+)
 create mode 100644 youtube_dl/extractor/r7.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index ef0adfd87..4d3e79de9 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -364,6 +364,7 @@ from .promptfile import PromptFileIE
 from .prosiebensat1 import ProSiebenSat1IE
 from .pyvideo import PyvideoIE
 from .quickvid import QuickVidIE
+from .r7 import R7IE
 from .radiode import RadioDeIE
 from .radiobremen import RadioBremenIE
 from .radiofrance import RadioFranceIE
diff --git a/youtube_dl/extractor/r7.py b/youtube_dl/extractor/r7.py
new file mode 100644
index 000000000..976c8feec
--- /dev/null
+++ b/youtube_dl/extractor/r7.py
@@ -0,0 +1,88 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    js_to_json,
+    unescapeHTML,
+    int_or_none,
+)
+
+
+class R7IE(InfoExtractor):
+    _VALID_URL = r'''(?x)https?://
+                        (?:
+                            (?:[a-zA-Z]+)\.r7\.com(?:/[^/]+)+/idmedia/|
+                            noticias\.r7\.com(?:/[^/]+)+/[^/]+-|
+                            player\.r7\.com/video/i/
+                        )
+                        (?P<id>[\da-f]{24})
+                        '''
+    _TESTS = [{
+        'url': 'http://videos.r7.com/policiais-humilham-suspeito-a-beira-da-morte-morre-com-dignidade-/idmedia/54e7050b0cf2ff57e0279389.html',
+        'md5': '403c4e393617e8e8ddc748978ee8efde',
+        'info_dict': {
+            'id': '54e7050b0cf2ff57e0279389',
+            'ext': 'mp4',
+            'title': 'Policiais humilham suspeito à beira da morte: "Morre com dignidade"',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 98,
+            'like_count': int,
+            'view_count': int,
+        },
+    }, {
+        'url': 'http://esportes.r7.com/videos/cigano-manda-recado-aos-fas/idmedia/4e176727b51a048ee6646a1b.html',
+        'only_matching': True,
+    }, {
+        'url': 'http://noticias.r7.com/record-news/video/representante-do-instituto-sou-da-paz-fala-sobre-fim-do-estatuto-do-desarmamento-5480fc580cf2285b117f438d/',
+        'only_matching': True,
+    }, {
+        'url': 'http://player.r7.com/video/i/54e7050b0cf2ff57e0279389?play=true&video=http://vsh.r7.com/54e7050b0cf2ff57e0279389/ER7_RE_BG_MORTE_JOVENS_570kbps_2015-02-2009f17818-cc82-4c8f-86dc-89a66934e633-ATOS_copy.mp4&linkCallback=http://videos.r7.com/policiais-humilham-suspeito-a-beira-da-morte-morre-com-dignidade-/idmedia/54e7050b0cf2ff57e0279389.html&thumbnail=http://vtb.r7.com/ER7_RE_BG_MORTE_JOVENS_570kbps_2015-02-2009f17818-cc82-4c8f-86dc-89a66934e633-thumb.jpg&idCategory=192&share=true&layout=full&full=true',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(
+            'http://player.r7.com/video/i/%s' % video_id, video_id)
+
+        item = self._parse_json(js_to_json(self._search_regex(
+            r'(?s)var\s+item\s*=\s*({.+?});', webpage, 'player')), video_id)
+
+        title = unescapeHTML(item['title'])
+        thumbnail = item.get('init', {}).get('thumbUri')
+        duration = None
+
+        statistics = item.get('statistics', {})
+        like_count = int_or_none(statistics.get('likes'))
+        view_count = int_or_none(statistics.get('views'))
+
+        formats = []
+        for format_key, format_dict in item['playlist'][0].items():
+            src = format_dict.get('src')
+            if not src:
+                continue
+            format_id = format_dict.get('format') or format_key
+            if duration is None:
+                duration = format_dict.get('duration')
+            if '.f4m' in src:
+                formats.extend(self._extract_f4m_formats(src, video_id, preference=-1))
+            elif src.endswith('.m3u8'):
+                formats.extend(self._extract_m3u8_formats(src, video_id, 'mp4', preference=-2))
+            else:
+                formats.append({
+                    'url': src,
+                    'format_id': format_id,
+                })
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'thumbnail': thumbnail,
+            'duration': duration,
+            'like_count': like_count,
+            'view_count': view_count,
+            'formats': formats,
+        }

From 1b40dc92eb27b2a3f299157f83bfc8e95ca42268 Mon Sep 17 00:00:00 2001
From: Duncan Keall <duncan@duncankeall.com>
Date: Mon, 23 Feb 2015 16:10:08 +1300
Subject: [PATCH 0486/6219] [airmozilla] Add new extractor

---
 youtube_dl/extractor/__init__.py   |  1 +
 youtube_dl/extractor/airmozilla.py | 73 ++++++++++++++++++++++++++++++
 2 files changed, 74 insertions(+)
 create mode 100644 youtube_dl/extractor/airmozilla.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index ef0adfd87..7e1864315 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -8,6 +8,7 @@ from .adobetv import AdobeTVIE
 from .adultswim import AdultSwimIE
 from .aftenposten import AftenpostenIE
 from .aftonbladet import AftonbladetIE
+from .airmozilla import AirMozillaIE
 from .aljazeera import AlJazeeraIE
 from .alphaporno import AlphaPornoIE
 from .anitube import AnitubeIE
diff --git a/youtube_dl/extractor/airmozilla.py b/youtube_dl/extractor/airmozilla.py
new file mode 100644
index 000000000..44c20f886
--- /dev/null
+++ b/youtube_dl/extractor/airmozilla.py
@@ -0,0 +1,73 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import parse_iso8601
+
+
+class AirMozillaIE(InfoExtractor):
+    _VALID_URL = r'https?://air\.mozilla\.org/(?P<id>[0-9a-z-]+)/?'
+    _TEST = {
+        'url': 'https://air.mozilla.org/privacy-lab-a-meetup-for-privacy-minded-people-in-san-francisco/',
+        'md5': '2e3e7486ba5d180e829d453875b9b8bf',
+        'info_dict': {
+            'id': '6x4q2w',
+            'ext': 'mp4',
+            'title': 'Privacy Lab - a meetup for privacy minded people in San Francisco',
+            'thumbnail': 're:https://\w+\.cloudfront\.net/6x4q2w/poster\.jpg\?t=\d+',
+            'description': 'Brings together privacy professionals and others interested in privacy at for-profits, non-profits, and NGOs in an effort to contribute to the state of the ecosystem...',
+            'timestamp': 1422487800,
+            'upload_date': '20150128',
+            'location': 'SFO Commons',
+            'duration': 3780,
+            'view_count': int,
+            'categories': ['Main'],
+        }
+    }
+
+    _QUALITY_MAP = {
+        '360p': 0,
+        '576p': 1,
+        '640p': 2,
+        '720p': 3,
+    }
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+        video_id = self._html_search_regex(r'//vid.ly/(.*?)/embed', webpage, 'id')
+
+        embed_script = self._download_webpage('https://vid.ly/{0}/embed'.format(video_id), video_id)
+        jwconfig = self._search_regex(r'\svar jwconfig = (\{.*?\});\s', embed_script, 'metadata')
+        metadata = self._parse_json(jwconfig, video_id)
+
+        formats = []
+        for source in metadata['playlist'][0]['sources']:
+            fmt = {
+                'url': source['file'],
+                'ext': source['type'],
+                'format_id': self._search_regex(r'&format=(.*)$', source['file'], 'video format'),
+                'resolution': source['label'],
+                'quality': self._QUALITY_MAP.get(source['label'], -1),
+            }
+            formats.append(fmt)
+        self._sort_formats(formats)
+
+        duration_match = re.search(r'Duration:(?: (?P<H>\d+) hours?)?(?: (?P<M>\d+) minutes?)?', webpage)
+
+        return {
+            'id': video_id,
+            'title': self._og_search_title(webpage),
+            'formats': formats,
+            'url': self._og_search_url(webpage),
+            'display_id': display_id,
+            'thumbnail': metadata['playlist'][0]['image'],
+            'description': self._og_search_description(webpage),
+            'timestamp': parse_iso8601(self._html_search_regex(r'<time datetime="(.*?)"', webpage, 'timestamp')),
+            'location': self._html_search_regex(r'Location: (.*)', webpage, 'location', default=None),
+            'duration': int(duration_match.groupdict()['H'] or 0) * 3600 + int(duration_match.groupdict()['M'] or 0) * 60,
+            'view_count': int(self._html_search_regex(r'Views since archived: ([0-9]+)', webpage, 'view count')),
+            'categories': re.findall(r'<a href=".*?" class="channel">(.*?)</a>', webpage),
+        }

From ddc369f073fda4ddd429c2d9a104e561cefd417f Mon Sep 17 00:00:00 2001
From: "Leslie P. Polzer" <polzer@gnu.org>
Date: Mon, 23 Feb 2015 12:00:43 +0100
Subject: [PATCH 0487/6219] [chirbit] fix profile downloader regex.

---
 youtube_dl/extractor/chirbit.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/chirbit.py b/youtube_dl/extractor/chirbit.py
index 47ce94aa0..443192f43 100644
--- a/youtube_dl/extractor/chirbit.py
+++ b/youtube_dl/extractor/chirbit.py
@@ -37,7 +37,7 @@ class ChirbitIE(InfoExtractor):
         }
 
 class ChirbitProfileIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?chirbit.com/(?P<id>[^/]+)'
+    _VALID_URL = r'https?://(?:www\.)?chirbit.com/(?P<id>[^/]+)/?$'
     _TEST = {
         'url': 'http://chirbit.com/ScarletBeauty',
         'playlist_count': 3,

From 93b5071f73738d788c878b38a57f2b6efe0da883 Mon Sep 17 00:00:00 2001
From: "Leslie P. Polzer" <polzer@gnu.org>
Date: Mon, 23 Feb 2015 12:11:19 +0100
Subject: [PATCH 0488/6219] [soundgasm] add profile IE.

---
 youtube_dl/extractor/__init__.py  |  5 ++++-
 youtube_dl/extractor/soundgasm.py | 36 +++++++++++++++++++++++++++++++
 2 files changed, 40 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 94e150826..cf58f0800 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -425,7 +425,10 @@ from .soundcloud import (
     SoundcloudUserIE,
     SoundcloudPlaylistIE
 )
-from .soundgasm import SoundgasmIE
+from .soundgasm import (
+    SoundgasmIE,
+    SoundgasmProfileIE
+)
 from .southpark import (
     SouthParkIE,
     SouthparkDeIE,
diff --git a/youtube_dl/extractor/soundgasm.py b/youtube_dl/extractor/soundgasm.py
index a4f8ce6c3..e568ff18c 100644
--- a/youtube_dl/extractor/soundgasm.py
+++ b/youtube_dl/extractor/soundgasm.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..utils import clean_html
 
 
 class SoundgasmIE(InfoExtractor):
@@ -38,3 +39,38 @@ class SoundgasmIE(InfoExtractor):
             'title': audio_title,
             'description': description
         }
+
+class SoundgasmProfileIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?soundgasm\.net/u/(?P<id>[0-9a-zA-Z_\-]+)/?$'
+    _TEST = {
+        'url': 'http://soundgasm.net/u/ytdl',
+        'playlist_count': 1,
+        'info_dict': {
+            '_type': 'playlist',
+            'id': 'ytdl',
+            'title': 'ytdl'
+        }
+    }
+
+    def _real_extract(self, url):
+        profile_id = self._match_id(url)
+        webpage = self._download_webpage(url, profile_id)
+
+        ids = re.findall(r'''<a\s+href=".+?/u/%s/([^/]+)">''' % re.escape(profile_id), webpage)
+        ids = [clean_html(id) for id in ids]
+
+        entries = []
+        for id in ids:
+            entries.append({
+                '_type': 'url',
+                'url': ('http://soundgasm.net/u/%s/%s' % (profile_id, id))
+            })
+
+        info_dict = {
+            '_type': 'playlist',
+            'id': profile_id,
+            'title': profile_id,
+            'entries': entries
+        }
+
+        return info_dict;

From b531cfc019576b682f930bd269f68eb87cfd5abf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 23 Feb 2015 16:12:35 +0100
Subject: [PATCH 0489/6219] [YoutubeDL] remove compatiblity with the old
 subtitles system

---
 youtube_dl/YoutubeDL.py | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 7319323e5..70b364c9b 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1189,14 +1189,6 @@ class YoutubeDL(object):
             if formats is None:
                 self.report_warning('%s subtitles not available for %s' % (lang, video_id))
                 continue
-            if isinstance(formats, compat_str):
-                # TODO: convert all IE with subtitles support to the new format
-                # and remove this
-                subs[lang] = {
-                    'ext': formats_preference[0],
-                    'data': formats,
-                }
-                continue
             for ext in formats_preference:
                 if ext == 'best':
                     f = formats[-1]

From a65d4e7f1458a681f250d6e2e0190644b50d6793 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 23 Feb 2015 21:15:16 +0600
Subject: [PATCH 0490/6219] [chirbit] Simplify and extract profile from RSS
 (#5032)

---
 youtube_dl/extractor/__init__.py |   5 +-
 youtube_dl/extractor/chirbit.py  | 113 ++++++++++++++-----------------
 2 files changed, 53 insertions(+), 65 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index c3088fba2..40fc92cf7 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -63,7 +63,10 @@ from .ccc import CCCIE
 from .ceskatelevize import CeskaTelevizeIE
 from .channel9 import Channel9IE
 from .chilloutzone import ChilloutzoneIE
-from .chirbit import ChirbitIE, ChirbitProfileIE
+from .chirbit import (
+    ChirbitIE,
+    ChirbitProfileIE,
+)
 from .cinchcast import CinchcastIE
 from .clipfish import ClipfishIE
 from .cliphunter import CliphunterIE
diff --git a/youtube_dl/extractor/chirbit.py b/youtube_dl/extractor/chirbit.py
index 443192f43..124307b7c 100644
--- a/youtube_dl/extractor/chirbit.py
+++ b/youtube_dl/extractor/chirbit.py
@@ -1,97 +1,82 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
-from ..utils import clean_html
+from ..utils import (
+    parse_duration,
+    int_or_none,
+)
 
 
 class ChirbitIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?chirb\.it/(?P<id>[^/]+)'
-    _TEST = {
+    _VALID_URL = r'https?://(?:www\.)?chirb\.it/(?:(?:wp|pl)/|fb_chirbit_player\.swf\?key=)?(?P<id>[\da-zA-Z]+)'
+    _TESTS = [{
         'url': 'http://chirb.it/PrIPv5',
         'md5': '9847b0dad6ac3e074568bf2cfb197de8',
         'info_dict': {
             'id': 'PrIPv5',
-            'display_id': 'kukushtv_1423231243',
             'ext': 'mp3',
             'title': 'Фасадстрой',
-            'url': 'http://audio.chirbit.com/kukushtv_1423231243.mp3'
+            'duration': 52,
+            'view_count': int,
+            'comment_count': int,
         }
-    }
+    }, {
+        'url': 'https://chirb.it/fb_chirbit_player.swf?key=PrIPv5',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
-        audio_linkid = self._match_id(url)
-        webpage = self._download_webpage(url, audio_linkid)
+        audio_id = self._match_id(url)
 
-        audio_title = self._html_search_regex(r'<h2\s+itemprop="name">(.*?)</h2>', webpage, 'title')
-        audio_id = self._html_search_regex(r'\("setFile",\s+"http://audio.chirbit.com/(.*?).mp3"\)', webpage, 'audio ID')
-        audio_url = 'http://audio.chirbit.com/' + audio_id + '.mp3';
+        webpage = self._download_webpage(
+            'http://chirb.it/%s' % audio_id, audio_id)
+
+        audio_url = self._search_regex(
+            r'"setFile"\s*,\s*"([^"]+)"', webpage, 'audio url')
+
+        title = self._search_regex(
+            r'itemprop="name">([^<]+)', webpage, 'title')
+        duration = parse_duration(self._html_search_meta(
+            'duration', webpage, 'duration', fatal=False))
+        view_count = int_or_none(self._search_regex(
+            r'itemprop="playCount"\s*>(\d+)', webpage,
+            'listen count', fatal=False))
+        comment_count = int_or_none(self._search_regex(
+            r'>(\d+) Comments?:', webpage,
+            'comment count', fatal=False))
 
         return {
-            'id': audio_linkid,
-            'display_id': audio_id,
-            'title': audio_title,
-            'url': audio_url
+            'id': audio_id,
+            'url': audio_url,
+            'title': title,
+            'duration': duration,
+            'view_count': view_count,
+            'comment_count': comment_count,
         }
 
+
 class ChirbitProfileIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?chirbit.com/(?P<id>[^/]+)/?$'
+    _VALID_URL = r'https?://(?:www\.)?chirbit.com/(?:rss/)?(?P<id>[^/]+)'
     _TEST = {
         'url': 'http://chirbit.com/ScarletBeauty',
-        'playlist_count': 3,
         'info_dict': {
-            '_type': 'playlist',
-            'title': 'ScarletBeauty',
-            'id': 'ScarletBeauty'
-        }
+            'id': 'ScarletBeauty',
+            'title': 'Chirbits by ScarletBeauty',
+        },
+        'playlist_mincount': 3,
     }
 
     def _real_extract(self, url):
         profile_id = self._match_id(url)
 
-        # Chirbit has a pretty weird "Last Page" navigation behavior.
-        # We grab the profile's oldest entry to determine when to
-        # stop fetching entries.
-        oldestpage = self._download_webpage(url + '/24599', profile_id)
-        oldest_page_entries = re.findall(
-            r'''soundFile:\s*"http://audio.chirbit.com/(.*?).mp3"''',
-            oldestpage);
-        oldestentry = clean_html(oldest_page_entries[-1]);
+        rss = self._download_xml(
+            'http://chirbit.com/rss/%s' % profile_id, profile_id)
 
-        ids = []
-        titles = []
-        n = 0
-        while True:
-            page = self._download_webpage(url + '/' + str(n), profile_id)
-            page_ids = re.findall(
-                r'''soundFile:\s*"http://audio.chirbit.com/(.*?).mp3"''',
-                page);
-            page_titles = re.findall(
-                r'''<div\s+class="chirbit_title"\s*>(.*?)</div>''',
-                page);
-            ids += page_ids
-            titles += page_titles
-            if oldestentry in page_ids:
-                break
-            n += 1
+        entries = [
+            self.url_result(audio_url.text, 'Chirbit')
+            for audio_url in rss.findall('./channel/item/link')]
 
-        entries = []
-        i = 0
-        for id in ids:
-            entries.append({
-                'id': id,
-                'title': titles[i],
-                'url': 'http://audio.chirbit.com/' + id + '.mp3'
-            });
-            i += 1
+        title = rss.find('./channel/title').text
 
-        info_dict = {
-            '_type': 'playlist',
-            'id': profile_id,
-            'title': profile_id,
-            'entries': entries
-        }
-
-        return info_dict;
+        return self.playlist_result(entries, profile_id, title)

From 3cc57f96455ce14cc5c72264a25b8d434174f7dd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 23 Feb 2015 21:27:24 +0600
Subject: [PATCH 0491/6219] [soundgasm:profile] Simplify

---
 youtube_dl/extractor/soundgasm.py | 30 ++++++++----------------------
 1 file changed, 8 insertions(+), 22 deletions(-)

diff --git a/youtube_dl/extractor/soundgasm.py b/youtube_dl/extractor/soundgasm.py
index e568ff18c..e11d999f3 100644
--- a/youtube_dl/extractor/soundgasm.py
+++ b/youtube_dl/extractor/soundgasm.py
@@ -41,36 +41,22 @@ class SoundgasmIE(InfoExtractor):
         }
 
 class SoundgasmProfileIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?soundgasm\.net/u/(?P<id>[0-9a-zA-Z_\-]+)/?$'
+    _VALID_URL = r'https?://(?:www\.)?soundgasm\.net/u/(?P<id>[^/]+)'
     _TEST = {
         'url': 'http://soundgasm.net/u/ytdl',
-        'playlist_count': 1,
         'info_dict': {
-            '_type': 'playlist',
             'id': 'ytdl',
-            'title': 'ytdl'
-        }
+        },
+        'playlist_count': 1,
     }
 
     def _real_extract(self, url):
         profile_id = self._match_id(url)
+
         webpage = self._download_webpage(url, profile_id)
 
-        ids = re.findall(r'''<a\s+href=".+?/u/%s/([^/]+)">''' % re.escape(profile_id), webpage)
-        ids = [clean_html(id) for id in ids]
+        entries = [
+            self.url_result(audio_url, 'Soundgasm')
+            for audio_url in re.findall(r'href="([^"]+/u/%s/[^"]+)' % profile_id, webpage)]
 
-        entries = []
-        for id in ids:
-            entries.append({
-                '_type': 'url',
-                'url': ('http://soundgasm.net/u/%s/%s' % (profile_id, id))
-            })
-
-        info_dict = {
-            '_type': 'playlist',
-            'id': profile_id,
-            'title': profile_id,
-            'entries': entries
-        }
-
-        return info_dict;
+        return self.playlist_result(entries, profile_id)

From 80af2b73ab0b51e4416500301948caa71ec39cb8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 23 Feb 2015 21:27:56 +0600
Subject: [PATCH 0492/6219] [soundgasm] Clarify extractors' IE_NAMEs

---
 youtube_dl/extractor/soundgasm.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/soundgasm.py b/youtube_dl/extractor/soundgasm.py
index e11d999f3..26e96a120 100644
--- a/youtube_dl/extractor/soundgasm.py
+++ b/youtube_dl/extractor/soundgasm.py
@@ -8,6 +8,7 @@ from ..utils import clean_html
 
 
 class SoundgasmIE(InfoExtractor):
+    IE_NAME = 'soundgasm'
     _VALID_URL = r'https?://(?:www\.)?soundgasm\.net/u/(?P<user>[0-9a-zA-Z_\-]+)/(?P<title>[0-9a-zA-Z_\-]+)'
     _TEST = {
         'url': 'http://soundgasm.net/u/ytdl/Piano-sample',
@@ -41,6 +42,7 @@ class SoundgasmIE(InfoExtractor):
         }
 
 class SoundgasmProfileIE(InfoExtractor):
+    IE_NAME = 'soundgasm:profile'
     _VALID_URL = r'https?://(?:www\.)?soundgasm\.net/u/(?P<id>[^/]+)'
     _TEST = {
         'url': 'http://soundgasm.net/u/ytdl',

From 04e8c1108023d9fe5c466d16f988a469e04f326e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 23 Feb 2015 21:28:14 +0600
Subject: [PATCH 0493/6219] [chirbit] Clarify extractors' IE_NAMEs

---
 youtube_dl/extractor/chirbit.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/chirbit.py b/youtube_dl/extractor/chirbit.py
index 124307b7c..b1eeaf101 100644
--- a/youtube_dl/extractor/chirbit.py
+++ b/youtube_dl/extractor/chirbit.py
@@ -9,6 +9,7 @@ from ..utils import (
 
 
 class ChirbitIE(InfoExtractor):
+    IE_NAME = 'chirbit'
     _VALID_URL = r'https?://(?:www\.)?chirb\.it/(?:(?:wp|pl)/|fb_chirbit_player\.swf\?key=)?(?P<id>[\da-zA-Z]+)'
     _TESTS = [{
         'url': 'http://chirb.it/PrIPv5',
@@ -57,6 +58,7 @@ class ChirbitIE(InfoExtractor):
 
 
 class ChirbitProfileIE(InfoExtractor):
+    IE_NAME = 'chirbit:profile'
     _VALID_URL = r'https?://(?:www\.)?chirbit.com/(?:rss/)?(?P<id>[^/]+)'
     _TEST = {
         'url': 'http://chirbit.com/ScarletBeauty',

From 409693984f0acb8fbbf006c0d7965bc138211ac6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 23 Feb 2015 21:30:30 +0600
Subject: [PATCH 0494/6219] [soundgasm:profile] Fix _VALID_URL

---
 youtube_dl/extractor/soundgasm.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/soundgasm.py b/youtube_dl/extractor/soundgasm.py
index 26e96a120..9e992c9b7 100644
--- a/youtube_dl/extractor/soundgasm.py
+++ b/youtube_dl/extractor/soundgasm.py
@@ -43,7 +43,7 @@ class SoundgasmIE(InfoExtractor):
 
 class SoundgasmProfileIE(InfoExtractor):
     IE_NAME = 'soundgasm:profile'
-    _VALID_URL = r'https?://(?:www\.)?soundgasm\.net/u/(?P<id>[^/]+)'
+    _VALID_URL = r'https?://(?:www\.)?soundgasm\.net/u/(?P<id>[^/]+)/?(?:\#.*)?$'
     _TEST = {
         'url': 'http://soundgasm.net/u/ytdl',
         'info_dict': {

From 09c200acf258de115caeda210741a59f2b971b65 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 23 Feb 2015 21:31:57 +0600
Subject: [PATCH 0495/6219] Credit @skypher for chirbit and soundgasm:profile
 (#5032)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 47f12a9ee..bdd2a15dc 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -111,3 +111,4 @@ Paul Hartmann
 Frans de Jonge
 Robin de Rooij
 Ryan Schmidt
+Leslie P. Polzer

From 3438e7acd27d89d83d41e722d21d7660dbad7eea Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 23 Feb 2015 21:40:50 +0600
Subject: [PATCH 0496/6219] [soundgasm] Remove unused import

---
 youtube_dl/extractor/soundgasm.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/soundgasm.py b/youtube_dl/extractor/soundgasm.py
index 9e992c9b7..1c48478a6 100644
--- a/youtube_dl/extractor/soundgasm.py
+++ b/youtube_dl/extractor/soundgasm.py
@@ -4,7 +4,6 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import clean_html
 
 
 class SoundgasmIE(InfoExtractor):

From bd61a9e770506283f82e2ddf9e53b587169c2f04 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 23 Feb 2015 16:47:19 +0100
Subject: [PATCH 0497/6219] release 2015.02.23

---
 docs/supportedsites.md | 6 +++++-
 youtube_dl/version.py  | 2 +-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 5fe3e47cd..9f70db80a 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -72,6 +72,8 @@
  - **CeskaTelevize**
  - **channel9**: Channel 9
  - **Chilloutzone**
+ - **chirbit**
+ - **chirbit:profile**
  - **Cinchcast**
  - **Cinemassacre**
  - **clipfish**
@@ -330,6 +332,7 @@
  - **prosiebensat1**: ProSiebenSat.1 Digital
  - **Pyvideo**
  - **QuickVid**
+ - **R7**
  - **radio.de**
  - **radiobremen**
  - **radiofrance**
@@ -385,7 +388,8 @@
  - **soundcloud:playlist**
  - **soundcloud:set**
  - **soundcloud:user**
- - **Soundgasm**
+ - **soundgasm**
+ - **soundgasm:profile**
  - **southpark.cc.com**
  - **southpark.de**
  - **Space**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 7c8b29c3b..17317b29c 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.21'
+__version__ = '2015.02.23'

From 5bca2424bc2dfb15b5394a51fa5befd7148edc41 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 23 Feb 2015 16:51:09 +0100
Subject: [PATCH 0498/6219] [gdcvault] Remove dead code

---
 youtube_dl/extractor/gdcvault.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/gdcvault.py b/youtube_dl/extractor/gdcvault.py
index 05f58f1af..e5011a5dc 100644
--- a/youtube_dl/extractor/gdcvault.py
+++ b/youtube_dl/extractor/gdcvault.py
@@ -66,7 +66,6 @@ class GDCVaultIE(InfoExtractor):
 
     def _parse_flv(self, xml_description):
         video_formats = []
-        akami_url = xml_description.find('./metadata/akamaiHost').text
         slide_video_path = xml_description.find('./metadata/slideVideo').text
         video_formats.append({
             'url': 'rtmp://%s/ondemand?ovpfv=1.1' % 'fms.digitallyspeaking.com/cfx/st',

From 591ab1dff913d7ff88f30487c54c1e9c5d44d0cb Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 23 Feb 2015 16:51:21 +0100
Subject: [PATCH 0499/6219] [soundgasm] PEP8

---
 youtube_dl/extractor/soundgasm.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/soundgasm.py b/youtube_dl/extractor/soundgasm.py
index 1c48478a6..3a4ddf57e 100644
--- a/youtube_dl/extractor/soundgasm.py
+++ b/youtube_dl/extractor/soundgasm.py
@@ -40,6 +40,7 @@ class SoundgasmIE(InfoExtractor):
             'description': description
         }
 
+
 class SoundgasmProfileIE(InfoExtractor):
     IE_NAME = 'soundgasm:profile'
     _VALID_URL = r'https?://(?:www\.)?soundgasm\.net/u/(?P<id>[^/]+)/?(?:\#.*)?$'

From 4432db35d9ddd0e6777df6c596d8637514ba0b56 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 23 Feb 2015 21:59:11 +0600
Subject: [PATCH 0500/6219] [gdcvault] Restore akamai host for rtmp videos

---
 youtube_dl/extractor/gdcvault.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/gdcvault.py b/youtube_dl/extractor/gdcvault.py
index e5011a5dc..f7b467b0a 100644
--- a/youtube_dl/extractor/gdcvault.py
+++ b/youtube_dl/extractor/gdcvault.py
@@ -66,9 +66,10 @@ class GDCVaultIE(InfoExtractor):
 
     def _parse_flv(self, xml_description):
         video_formats = []
+        akamai_url = xml_description.find('./metadata/akamaiHost').text
         slide_video_path = xml_description.find('./metadata/slideVideo').text
         video_formats.append({
-            'url': 'rtmp://%s/ondemand?ovpfv=1.1' % 'fms.digitallyspeaking.com/cfx/st',
+            'url': 'rtmp://%s/ondemand?ovpfv=1.1' % akamai_url,
             'play_path': remove_end(slide_video_path, '.flv'),
             'ext': 'flv',
             'format_note': 'slide deck video',
@@ -78,7 +79,7 @@ class GDCVaultIE(InfoExtractor):
         })
         speaker_video_path = xml_description.find('./metadata/speakerVideo').text
         video_formats.append({
-            'url': 'rtmp://%s/ondemand?ovpfv=1.1' % 'fms.digitallyspeaking.com/cfx/st',
+            'url': 'rtmp://%s/ondemand?ovpfv=1.1' % akamai_url,
             'play_path': remove_end(speaker_video_path, '.flv'),
             'ext': 'flv',
             'format_note': 'speaker video',

From 459e5fbd5fa61064076534c4d6d8a1d010acb1b3 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 23 Feb 2015 18:17:39 +0100
Subject: [PATCH 0501/6219] release 2015.02.23.1

---
 README.md             | 4 ++--
 youtube_dl/version.py | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index 8ea31d605..699401b49 100644
--- a/README.md
+++ b/README.md
@@ -351,8 +351,8 @@ which means you can modify it, redistribute it or use it however you like.
     --all-subs                       downloads all the available subtitles of
                                      the video
     --list-subs                      lists all available subtitles for the video
-    --sub-format FORMAT              subtitle format (default=srt) ([sbv/vtt]
-                                     youtube only)
+    --sub-format FORMAT              subtitle format, accepts formats
+                                     preference, for example: "ass/srt/best"
     --sub-lang LANGS                 languages of the subtitles to download
                                      (optional) separated by commas, use IETF
                                      language tags like 'en,pt'
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 17317b29c..1852d834b 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.23'
+__version__ = '2015.02.23.1'

From ffdf972b9115d6d8c86439bf0828e945823bdcf8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 23 Feb 2015 18:54:15 +0100
Subject: [PATCH 0502/6219] [facebook] Extract all the formats (closes #5037)

---
 youtube_dl/extractor/facebook.py | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index 1ad4e77a8..f0e575320 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -126,11 +126,17 @@ class FacebookIE(InfoExtractor):
         params_raw = compat_urllib_parse.unquote(data['params'])
         params = json.loads(params_raw)
         video_data = params['video_data'][0]
-        video_url = video_data.get('hd_src')
-        if not video_url:
-            video_url = video_data['sd_src']
-        if not video_url:
-            raise ExtractorError('Cannot find video URL')
+
+        formats = []
+        for quality in ['sd', 'hd']:
+            src = video_data.get('%s_src' % quality)
+            if src is not None:
+                formats.append({
+                    'format_id': quality,
+                    'url': src,
+                })
+        if not formats:
+            raise ExtractorError('Cannot find video formats')
 
         video_title = self._html_search_regex(
             r'<h2 class="uiHeaderTitle">([^<]*)</h2>', webpage, 'title',
@@ -146,7 +152,7 @@ class FacebookIE(InfoExtractor):
         return {
             'id': video_id,
             'title': video_title,
-            'url': video_url,
+            'formats': formats,
             'duration': int_or_none(video_data.get('video_duration')),
             'thumbnail': video_data.get('thumbnail_src'),
         }

From 3037b91e05e68a4ab3420cbbdb23cfb0739011d3 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Mon, 23 Feb 2015 20:45:36 +0200
Subject: [PATCH 0503/6219] [laola1tv] Improve extraction and update test case
 (#3742)

---
 youtube_dl/extractor/laola1tv.py | 47 +++++++++++++++++++++-----------
 1 file changed, 31 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/extractor/laola1tv.py b/youtube_dl/extractor/laola1tv.py
index 2fd3b4699..135421406 100644
--- a/youtube_dl/extractor/laola1tv.py
+++ b/youtube_dl/extractor/laola1tv.py
@@ -1,23 +1,26 @@
+# -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
 import random
 import re
 
 from .common import InfoExtractor
-from ..utils import ExtractorError
+from ..utils import (
+    ExtractorError,
+    xpath_text,
+)
 
 
 class Laola1TvIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?laola1\.tv/(?P<lang>[a-z]+)-(?P<portal>[a-z]+)/.*?/(?P<id>[0-9]+)\.html'
     _TEST = {
-        'url': 'http://www.laola1.tv/de-de/live/bwf-bitburger-open-grand-prix-gold-court-1/250019.html',
+        'url': 'http://www.laola1.tv/de-de/video/straubing-tigers-koelner-haie/227883.html',
         'info_dict': {
-            'id': '250019',
+            'id': '227883',
             'ext': 'mp4',
-            'title': 'Bitburger Open Grand Prix Gold - Court 1',
-            'categories': ['Badminton'],
-            'uploader': 'BWF - Badminton World Federation',
-            'is_live': True,
+            'title': 'Straubing Tigers - Kölner Haie',
+            'categories': ['Eishockey'],
+            'is_live': False,
         },
         'params': {
             'skip_download': True,
@@ -43,15 +46,26 @@ class Laola1TvIE(InfoExtractor):
             r'flashvars\.([_a-zA-Z0-9]+)\s*=\s*"([^"]*)";', iframe)
         flashvars = dict((m[0], m[1]) for m in flashvars_m)
 
+        partner_id = self._search_regex(
+            'partnerid\s*:\s*"([^"]+)"', iframe, 'partner id')
+
         xml_url = ('http://www.laola1.tv/server/hd_video.php?' +
-                   'play=%s&partner=1&portal=%s&v5ident=&lang=%s' % (
-                       video_id, portal, lang))
+                   'play=%s&partner=%s&portal=%s&v5ident=&lang=%s' % (
+                       video_id, partner_id, portal, lang))
         hd_doc = self._download_xml(xml_url, video_id)
 
-        title = hd_doc.find('.//video/title').text
-        flash_url = hd_doc.find('.//video/url').text
-        categories = hd_doc.find('.//video/meta_sports').text.split(',')
-        uploader = hd_doc.find('.//video/meta_organistation').text
+        title = xpath_text(hd_doc, './/video/title', fatal=True)
+        flash_url = xpath_text(hd_doc, './/video/url', fatal=True)
+        uploader = xpath_text(hd_doc, './/video/meta_organistation')
+
+        is_live = xpath_text(hd_doc, './/video/islive') == 'true'
+        if is_live:
+            raise ExtractorError(
+                'Live streams are not supported by the f4m downloader.')
+
+        categories = xpath_text(hd_doc, './/video/meta_sports')
+        if categories:
+            categories = categories.split(',')
 
         ident = random.randint(10000000, 99999999)
         token_url = '%s&ident=%s&klub=0&unikey=0&timestamp=%s&auth=%s' % (
@@ -60,15 +74,16 @@ class Laola1TvIE(InfoExtractor):
         token_doc = self._download_xml(
             token_url, video_id, note='Downloading token')
         token_attrib = token_doc.find('.//token').attrib
-        if token_attrib.get('auth') == 'blocked':
-            raise ExtractorError('Token error: ' % token_attrib.get('comment'))
+        if token_attrib.get('auth') in ('blocked', 'restricted'):
+            raise ExtractorError(
+                'Token error: %s' % token_attrib.get('comment'), expected=True)
 
         video_url = '%s?hdnea=%s&hdcore=3.2.0' % (
             token_attrib['url'], token_attrib['auth'])
 
         return {
             'id': video_id,
-            'is_live': True,
+            'is_live': is_live,
             'title': title,
             'url': video_url,
             'uploader': uploader,

From 1fbaa0a5210976a2a8fc0c20207708c35621416a Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Mon, 23 Feb 2015 20:51:30 +0200
Subject: [PATCH 0504/6219] [laola1tv] Use raw strings for regular expressions

Oops
---
 youtube_dl/extractor/laola1tv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/laola1tv.py b/youtube_dl/extractor/laola1tv.py
index 135421406..e8ca49fd1 100644
--- a/youtube_dl/extractor/laola1tv.py
+++ b/youtube_dl/extractor/laola1tv.py
@@ -47,7 +47,7 @@ class Laola1TvIE(InfoExtractor):
         flashvars = dict((m[0], m[1]) for m in flashvars_m)
 
         partner_id = self._search_regex(
-            'partnerid\s*:\s*"([^"]+)"', iframe, 'partner id')
+            r'partnerid\s*:\s*"([^"]+)"', iframe, 'partner id')
 
         xml_url = ('http://www.laola1.tv/server/hd_video.php?' +
                    'play=%s&partner=%s&portal=%s&v5ident=&lang=%s' % (

From 99209c2916753799e9c68e8d466c5253113f25bc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 24 Feb 2015 01:35:15 +0600
Subject: [PATCH 0505/6219] [youtube] Extract UL playlists as mixes (Closes
 #5040)

---
 youtube_dl/extractor/youtube.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 22db896b1..3690f8021 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1153,13 +1153,13 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
                         |  p/
                         )
                         (
-                            (?:PL|LL|EC|UU|FL|RD)?[0-9A-Za-z-_]{10,}
+                            (?:PL|LL|EC|UU|FL|RD|UL)?[0-9A-Za-z-_]{10,}
                             # Top tracks, they can also include dots
                             |(?:MC)[\w\.]*
                         )
                         .*
                      |
-                        ((?:PL|LL|EC|UU|FL|RD)[0-9A-Za-z-_]{10,})
+                        ((?:PL|LL|EC|UU|FL|RD|UL)[0-9A-Za-z-_]{10,})
                      )"""
     _TEMPLATE_URL = 'https://www.youtube.com/playlist?list=%s'
     _VIDEO_RE = r'href="\s*/watch\?v=(?P<id>[0-9A-Za-z_-]{11})&amp;[^"]*?index=(?P<index>\d+)'
@@ -1244,7 +1244,7 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
             for vid_id in ids]
 
     def _extract_mix(self, playlist_id):
-        # The mixes are generated from a a single video
+        # The mixes are generated from a single video
         # the id of the playlist is just 'RD' + video_id
         url = 'https://youtube.com/watch?v=%s&list=%s' % (playlist_id[-11:], playlist_id)
         webpage = self._download_webpage(
@@ -1280,7 +1280,7 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
             else:
                 self.to_screen('Downloading playlist %s - add --no-playlist to just download video %s' % (playlist_id, video_id))
 
-        if playlist_id.startswith('RD'):
+        if playlist_id.startswith('RD') or playlist_id.startswith('UL'):
             # Mixes require a custom extraction process
             return self._extract_mix(playlist_id)
 

From 25ac63ed71bdc2a82842a593db9a150a0b8b7a6e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 23 Feb 2015 21:52:07 +0100
Subject: [PATCH 0506/6219] [rtve] Extract subtitles

---
 test/test_subtitles.py       | 15 +++++++++++++++
 youtube_dl/extractor/rtve.py | 12 ++++++++++++
 2 files changed, 27 insertions(+)

diff --git a/test/test_subtitles.py b/test/test_subtitles.py
index 7f93f0a75..3f2d8a2ba 100644
--- a/test/test_subtitles.py
+++ b/test/test_subtitles.py
@@ -25,6 +25,7 @@ from youtube_dl.extractor import (
     RaiIE,
     VikiIE,
     ThePlatformIE,
+    RTVEALaCartaIE,
 )
 
 
@@ -305,5 +306,19 @@ class TestThePlatformSubtitles(BaseTestSubtitles):
         self.assertEqual(md5(subtitles['en']), '97e7670cbae3c4d26ae8bcc7fdd78d4b')
 
 
+class TestRtveSubtitles(BaseTestSubtitles):
+    url = 'http://www.rtve.es/alacarta/videos/los-misterios-de-laura/misterios-laura-capitulo-32-misterio-del-numero-17-2-parte/2428621/'
+    IE = RTVEALaCartaIE
+
+    def test_allsubtitles(self):
+        print('Skipping, only available from Spain')
+        return
+        self.DL.params['writesubtitles'] = True
+        self.DL.params['allsubtitles'] = True
+        subtitles = self.getSubtitles()
+        self.assertEqual(set(subtitles.keys()), set(['es']))
+        self.assertEqual(md5(subtitles['es']), '69e70cae2d40574fb7316f31d6eb7fca')
+
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/extractor/rtve.py b/youtube_dl/extractor/rtve.py
index e60f85b5b..27cd34b7d 100644
--- a/youtube_dl/extractor/rtve.py
+++ b/youtube_dl/extractor/rtve.py
@@ -102,14 +102,26 @@ class RTVEALaCartaIE(InfoExtractor):
             video_url = compat_urlparse.urljoin(
                 'http://mvod1.akcdn.rtve.es/', video_path)
 
+        subtitles = None
+        if info.get('sbtFile') is not None:
+            subtitles = self.extract_subtitles(video_id, info['sbtFile'])
+
         return {
             'id': video_id,
             'title': info['title'],
             'url': video_url,
             'thumbnail': info.get('image'),
             'page_url': url,
+            'subtitles': subtitles,
         }
 
+    def _get_subtitles(self, video_id, sub_file):
+        subs = self._download_json(
+            sub_file + '.json', video_id,
+            'Downloading subtitles info')['page']['items']
+        return dict((s['lang'], [{'ext': 'vtt', 'url': s['src']}])
+            for s in subs)
+
 
 class RTVELiveIE(InfoExtractor):
     IE_NAME = 'rtve.es:live'

From ec5913b5cd92dfb8607ec535e02b04bdc09ff804 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 24 Feb 2015 11:08:00 +0100
Subject: [PATCH 0507/6219] [bloomberg] Modernize

---
 youtube_dl/extractor/bloomberg.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/bloomberg.py b/youtube_dl/extractor/bloomberg.py
index c51a97ce4..4a88ccd13 100644
--- a/youtube_dl/extractor/bloomberg.py
+++ b/youtube_dl/extractor/bloomberg.py
@@ -6,7 +6,7 @@ from .common import InfoExtractor
 
 
 class BloombergIE(InfoExtractor):
-    _VALID_URL = r'https?://www\.bloomberg\.com/video/(?P<name>.+?)\.html'
+    _VALID_URL = r'https?://www\.bloomberg\.com/video/(?P<id>.+?)\.html'
 
     _TEST = {
         'url': 'http://www.bloomberg.com/video/shah-s-presentation-on-foreign-exchange-strategies-qurhIVlJSB6hzkVi229d8g.html',
@@ -20,9 +20,9 @@ class BloombergIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        name = mobj.group('name')
+        name = self._match_id(url)
         webpage = self._download_webpage(url, name)
+
         f4m_url = self._search_regex(
             r'<source src="(https?://[^"]+\.f4m.*?)"', webpage,
             'f4m url')

From b665ba6aa6551243aa1a5b707ee7034be356f1bb Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 24 Feb 2015 11:24:26 +0100
Subject: [PATCH 0508/6219] release 2015.02.24

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 1852d834b..589f38834 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.23.1'
+__version__ = '2015.02.24'

From 9c665ab72e5fc99989800109cdada5acc3af56c5 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 24 Feb 2015 11:37:27 +0100
Subject: [PATCH 0509/6219] [rtve] PEP8

---
 youtube_dl/extractor/rtve.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/rtve.py b/youtube_dl/extractor/rtve.py
index 27cd34b7d..c0fd23ff1 100644
--- a/youtube_dl/extractor/rtve.py
+++ b/youtube_dl/extractor/rtve.py
@@ -119,7 +119,8 @@ class RTVEALaCartaIE(InfoExtractor):
         subs = self._download_json(
             sub_file + '.json', video_id,
             'Downloading subtitles info')['page']['items']
-        return dict((s['lang'], [{'ext': 'vtt', 'url': s['src']}])
+        return dict(
+            (s['lang'], [{'ext': 'vtt', 'url': s['src']}])
             for s in subs)
 
 

From 5a42414b9c4718f83f28fbc0e5a4a01ab67f23f6 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 24 Feb 2015 11:38:01 +0100
Subject: [PATCH 0510/6219] [utils] Prevent hyphen at beginning of filename
 (Fixes #5035)

---
 test/test_utils.py  | 2 ++
 youtube_dl/utils.py | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/test/test_utils.py b/test/test_utils.py
index c7373af1e..2f8996d7b 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -85,6 +85,8 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(
             sanitize_filename('New World record at 0:12:34'),
             'New World record at 0_12_34')
+        self.assertEqual(sanitize_filename('--gasdgf'), '_-gasdgf')
+        self.assertEqual(sanitize_filename('--gasdgf', is_id=True), '--gasdgf')
 
         forbidden = '"\0\\/'
         for fc in forbidden:
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 475fad3c9..e2631dccd 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -304,6 +304,8 @@ def sanitize_filename(s, restricted=False, is_id=False):
         # Common case of "Foreign band name - English song title"
         if restricted and result.startswith('-_'):
             result = result[2:]
+        if result.startswith('-'):
+            result = '_' + result[len('-'):]
         if not result:
             result = '_'
     return result

From db8e13ef714544574691f3dd4255dd0f12c1cf77 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 24 Feb 2015 11:38:21 +0100
Subject: [PATCH 0511/6219] release 2015.02.24.1

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 589f38834..a420860ed 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.24'
+__version__ = '2015.02.24.1'

From 54233c9080c1956f53802988fd8d5328cb38b7d7 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 24 Feb 2015 16:33:07 +0100
Subject: [PATCH 0512/6219] [escapist] Support JavaScript player (Fixes #5034)

---
 youtube_dl/extractor/escapist.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/escapist.py b/youtube_dl/extractor/escapist.py
index b49b9869f..51ffec7ee 100644
--- a/youtube_dl/extractor/escapist.py
+++ b/youtube_dl/extractor/escapist.py
@@ -42,7 +42,14 @@ class EscapistIE(InfoExtractor):
         title = raw_title.partition(' : ')[2]
 
         config_url = compat_urllib_parse.unquote(self._html_search_regex(
-            r'<param\s+name="flashvars"\s+value="config=([^"&]+)', webpage, 'config URL'))
+            r'''(?x)
+            (?:
+                <param\s+name="flashvars"\s+value="config=|
+                flashvars=&quot;config=
+            )
+            ([^"&]+)
+            ''',
+            webpage, 'config URL'))
 
         formats = []
 

From 4f3b21e1c738a7dacd514eb59242da43e81b5ae1 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 24 Feb 2015 16:34:42 +0100
Subject: [PATCH 0513/6219] release 2015.02.24.2

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index a420860ed..d23c6ae3d 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.24.1'
+__version__ = '2015.02.24.2'

From 7f09a662a0db2ad0a338645e4f2dbd43056a6fa6 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 24 Feb 2015 23:55:57 +0800
Subject: [PATCH 0514/6219] [Letv] Add new extractor. Single video only

---
 youtube_dl/extractor/__init__.py |   1 +
 youtube_dl/extractor/letv.py     | 120 +++++++++++++++++++++++++++++++
 2 files changed, 121 insertions(+)
 create mode 100644 youtube_dl/extractor/letv.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 40fc92cf7..829ab18a9 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -237,6 +237,7 @@ from .krasview import KrasViewIE
 from .ku6 import Ku6IE
 from .la7 import LA7IE
 from .laola1tv import Laola1TvIE
+from .letv import LetvIE
 from .lifenews import LifeNewsIE
 from .liveleak import LiveLeakIE
 from .livestream import (
diff --git a/youtube_dl/extractor/letv.py b/youtube_dl/extractor/letv.py
new file mode 100644
index 000000000..3eb20678e
--- /dev/null
+++ b/youtube_dl/extractor/letv.py
@@ -0,0 +1,120 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import os.path
+import time
+import datetime
+
+from .common import InfoExtractor
+from ..compat import (compat_urlparse, compat_urllib_parse)
+from ..utils import (ExtractorError, parse_iso8601)
+
+
+class LetvIE(InfoExtractor):
+    _VALID_URL = r'http://www.letv.com/ptv/vplay/(?P<id>\d+).html'
+
+    _TESTS = [{
+        'url': 'http://www.letv.com/ptv/vplay/22005890.html',
+        'md5': 'cab23bd68d5a8db9be31c9a222c1e8df',
+        'info_dict': {
+            'id': '22005890',
+            'ext': 'mp4',
+            'title': '第87届奥斯卡颁奖礼完美落幕 《鸟人》成最大赢家',
+            'timestamp': 1424747397,
+            'upload_date': '20150224',
+        }
+    }, {
+        'url': 'http://www.letv.com/ptv/vplay/1118082.html',
+        'info_dict': {
+            'id': '1118082',
+            'ext': 'mp4',
+        }
+    }]
+
+    @staticmethod
+    def urshift(val, n):
+        return val >> n if val >= 0 else (val + 0x100000000) >> n
+
+    # ror() and calcTimeKey() are reversed from a embedded swf file in KLetvPlayer.swf
+    def ror(self, param1, param2):
+        _loc3_ = 0
+        while _loc3_ < param2:
+            param1 = self.urshift(param1, 1) + ((param1 & 1) << 31)
+            _loc3_ += 1
+        return param1
+
+    def calcTimeKey(self, param1):
+        _loc2_ = 773625421
+        _loc3_ = self.ror(param1, _loc2_ % 13)
+        _loc3_ = _loc3_ ^ _loc2_
+        _loc3_ = self.ror(_loc3_, _loc2_ % 17)
+        return _loc3_
+
+    def _real_extract(self, url):
+        media_id = self._match_id(url)
+        page = self._download_webpage(url, media_id)
+        params = {
+            'id': media_id,
+            'platid': 1,
+            'splatid': 101,
+            'format': 1,
+            'tkey': self.calcTimeKey(int(time.time())),
+            'domain': 'www.letv.com'
+        }
+        play_json = self._download_json(
+            'http://api.letv.com/mms/out/video/playJson?' + compat_urllib_parse.urlencode(params),
+            media_id, 'playJson data')
+
+        # Check for errors
+        playstatus = play_json['playstatus']
+        if playstatus['status'] == 0:
+            flag = playstatus['flag']
+            if flag == 1:
+                msg = 'Country %s auth error' % playstatus['country']
+            else:
+                msg = 'Generic error. flag = %d' % flag
+            raise ExtractorError(msg, expected=True)
+
+        playurl = play_json['playurl']
+
+        formats = ['350', '1000', '1300', '720p', '1080p']
+        dispatch = playurl['dispatch']
+
+        urls = []
+        for format_id in formats:
+            if format_id in dispatch:
+                media_url = playurl['domain'][0] + dispatch[format_id][0]
+
+                # Mimic what flvxz.com do
+                url_parts = list(compat_urlparse.urlparse(media_url))
+                qs = dict(compat_urlparse.parse_qs(url_parts[4]))
+                qs.update({
+                    'platid': '14',
+                    'splatid': '1401',
+                    'tss': 'no',
+                    'retry': 1
+                })
+                url_parts[4] = compat_urllib_parse.urlencode(qs)
+                media_url = compat_urlparse.urlunparse(url_parts)
+
+                url_info_dict = {
+                    'url': media_url,
+                    'ext': os.path.splitext(dispatch[format_id][1])[1][1:]
+                }
+
+                if format_id[-1:] == 'p':
+                    url_info_dict['height'] = format_id[:-1]
+
+                urls.append(url_info_dict)
+
+        publish_time = parse_iso8601(self._html_search_regex(
+            r'发布时间&nbsp;([^<>]+) ', page, 'publish time', fatal=False),
+            delimiter=' ', timezone=datetime.timedelta(hours=8))
+
+        return {
+            'id': media_id,
+            'formats': urls,
+            'title': playurl['title'],
+            'thumbnail': playurl['pic'],
+            'timestamp': publish_time,
+        }

From df4bd0d53ff4baff6ce25ad04a1e87f37777c3ff Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 24 Feb 2015 17:25:02 +0100
Subject: [PATCH 0515/6219] [options] Add --yes-playlist as inverse of
 --no-playlist (Fixes #5051)

---
 youtube_dl/options.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 5c2d153b1..886ce9613 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -272,6 +272,10 @@ def parseOpts(overrideArguments=None):
         '--no-playlist',
         action='store_true', dest='noplaylist', default=False,
         help='If the URL refers to a video and a playlist, download only the video.')
+    selection.add_option(
+        '--yes-playlist',
+        action='store_false', dest='noplaylist', default=False,
+        help='If the URL refers to a video and a playlist, download the playlist.')
     selection.add_option(
         '--age-limit',
         metavar='YEARS', dest='age_limit', default=None, type=int,

From 41b264e77cd357444b632a132ea11ff7ddc3de1f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 24 Feb 2015 23:06:44 +0600
Subject: [PATCH 0516/6219] [nrktv] Workaround subtitles conversion issues on
 python 2.6 (Closes #5036)

---
 youtube_dl/extractor/nrk.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index 46f493cfc..1e4cfa2e7 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import compat_str
 from ..utils import (
     ExtractorError,
     float_or_none,
@@ -158,7 +159,9 @@ class NRKTVIE(InfoExtractor):
     def _get_subtitles(self, subtitlesurl, video_id, baseurl):
         url = "%s%s" % (baseurl, subtitlesurl)
         self._debug_print('%s: Subtitle url: %s' % (video_id, url))
-        captions = self._download_xml(url, video_id, 'Downloading subtitles')
+        captions = self._download_xml(
+            url, video_id, 'Downloading subtitles',
+            transform_source=lambda s: s.replace(r'<br />', '\r\n'))
         lang = captions.get('lang', 'no')
         ps = captions.findall('./{0}body/{0}div/{0}p'.format('{http://www.w3.org/ns/ttml}'))
         srt = ''
@@ -167,8 +170,7 @@ class NRKTVIE(InfoExtractor):
             duration = parse_duration(p.get('dur'))
             starttime = self._seconds2str(begin)
             endtime = self._seconds2str(begin + duration)
-            text = '\n'.join(p.itertext())
-            srt += '%s\r\n%s --> %s\r\n%s\r\n\r\n' % (str(pos), starttime, endtime, text)
+            srt += '%s\r\n%s --> %s\r\n%s\r\n\r\n' % (compat_str(pos), starttime, endtime, p.text)
         return {lang: [
             {'ext': 'ttml', 'url': url},
             {'ext': 'srt', 'data': srt},

From 570311610e33eb67d5a65a86705d54daaee17361 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 25 Feb 2015 00:45:11 +0800
Subject: [PATCH 0517/6219] [Letv] Add playlist support

---
 youtube_dl/extractor/__init__.py |  6 +++-
 youtube_dl/extractor/letv.py     | 54 +++++++++++++++++++++++++++++++-
 2 files changed, 58 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 829ab18a9..7b7d41adf 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -237,7 +237,11 @@ from .krasview import KrasViewIE
 from .ku6 import Ku6IE
 from .la7 import LA7IE
 from .laola1tv import Laola1TvIE
-from .letv import LetvIE
+from .letv import (
+    LetvIE,
+    LetvTvIE,
+    LetvPlaylistIE
+)
 from .lifenews import LifeNewsIE
 from .liveleak import LiveLeakIE
 from .livestream import (
diff --git a/youtube_dl/extractor/letv.py b/youtube_dl/extractor/letv.py
index 3eb20678e..d7d315634 100644
--- a/youtube_dl/extractor/letv.py
+++ b/youtube_dl/extractor/letv.py
@@ -2,6 +2,7 @@
 from __future__ import unicode_literals
 
 import os.path
+import re
 import time
 import datetime
 
@@ -11,7 +12,7 @@ from ..utils import (ExtractorError, parse_iso8601)
 
 
 class LetvIE(InfoExtractor):
-    _VALID_URL = r'http://www.letv.com/ptv/vplay/(?P<id>\d+).html'
+    _VALID_URL = r'http://www\.letv\.com/ptv/vplay/(?P<id>\d+).html'
 
     _TESTS = [{
         'url': 'http://www.letv.com/ptv/vplay/22005890.html',
@@ -118,3 +119,54 @@ class LetvIE(InfoExtractor):
             'thumbnail': playurl['pic'],
             'timestamp': publish_time,
         }
+
+
+class LetvTvIE(InfoExtractor):
+    _VALID_URL = r'http://www.letv.com/tv/(?P<id>\d+).html'
+    _TESTS = [{
+        'url': 'http://www.letv.com/tv/46177.html',
+        'info_dict': {
+            'id': '46177',
+            'title': '美人天下',
+            'description': 'md5:395666ff41b44080396e59570dbac01c'
+        },
+        'playlist_count': 35
+    }]
+
+    def _real_extract(self, url):
+        playlist_id = self._match_id(url)
+        page = self._download_webpage(url, playlist_id)
+
+        media_urls = list(set(re.findall(
+            r'http://www.letv.com/ptv/vplay/\d+.html', page)))
+        entries = [self.url_result(media_url, ie='Letv')
+                   for media_url in media_urls]
+
+        title = self._html_search_meta('keywords', page, fatal=False).split('，')[0]
+        description = self._html_search_meta('description', page, fatal=False)
+
+        return self.playlist_result(entries, playlist_id, playlist_title=title,
+                                    playlist_description=description)
+
+
+class LetvPlaylistIE(LetvTvIE):
+    _VALID_URL = r'http://tv.letv.com/[a-z]+/(?P<id>[a-z]+)/index.s?html'
+    _TESTS = [{
+        'url': 'http://tv.letv.com/izt/wuzetian/index.html',
+        'info_dict': {
+            'id': 'wuzetian',
+            'title': '武媚娘传奇',
+            'description': 'md5:e12499475ab3d50219e5bba00b3cb248'
+        },
+        'playlist_count': 96
+    }, {
+        'url': 'http://tv.letv.com/pzt/lswjzzjc/index.shtml',
+        'info_dict': {
+            'id': 'lswjzzjc',
+            # should be "劲舞青春", but I can't find a simple way to determine
+            # the playlist title
+            'title': '乐视午间自制剧场',
+            'description': 'md5:b1eef244f45589a7b5b1af9ff25a4489'
+        },
+        'playlist_mincount': 7
+    }]

From 59c7cbd482ba82248cd1bdca3569da6035720f21 Mon Sep 17 00:00:00 2001
From: logon84 <rubenlogon@yahoo.es>
Date: Tue, 24 Feb 2015 18:58:32 +0100
Subject: [PATCH 0518/6219] Update eporner.py

Updated to work. Old version shows an error about being unable to extract "redirect_code"
---
 youtube_dl/extractor/eporner.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/eporner.py b/youtube_dl/extractor/eporner.py
index 4de8d4bc5..9ae28855b 100644
--- a/youtube_dl/extractor/eporner.py
+++ b/youtube_dl/extractor/eporner.py
@@ -35,10 +35,7 @@ class EpornerIE(InfoExtractor):
         title = self._html_search_regex(
             r'<title>(.*?) - EPORNER', webpage, 'title')
 
-        redirect_code = self._html_search_regex(
-            r'<script type="text/javascript" src="/config5/%s/([a-f\d]+)/">' % video_id,
-            webpage, 'redirect_code')
-        redirect_url = 'http://www.eporner.com/config5/%s/%s' % (video_id, redirect_code)
+        redirect_url = 'http://www.eporner.com/config5/%s/%s' % (video_id, display_id)
         player_code = self._download_webpage(
             redirect_url, display_id, note='Downloading player config')
 

From 677063594e28d11ada9d2bfd3a30397d4de24360 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 25 Feb 2015 02:10:55 +0800
Subject: [PATCH 0519/6219] [Letv] Update testcases

---
 youtube_dl/extractor/letv.py | 26 +++++++++++++++++++-------
 1 file changed, 19 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/letv.py b/youtube_dl/extractor/letv.py
index d7d315634..d5839263c 100644
--- a/youtube_dl/extractor/letv.py
+++ b/youtube_dl/extractor/letv.py
@@ -23,14 +23,22 @@ class LetvIE(InfoExtractor):
             'title': '第87届奥斯卡颁奖礼完美落幕 《鸟人》成最大赢家',
             'timestamp': 1424747397,
             'upload_date': '20150224',
+            'description': 'md5:a9cb175fd753e2962176b7beca21a47c',
         }
     }, {
-        'url': 'http://www.letv.com/ptv/vplay/1118082.html',
+        'url': 'http://www.letv.com/ptv/vplay/1415246.html',
         'info_dict': {
-            'id': '1118082',
+            'id': '1415246',
             'ext': 'mp4',
-        }
+            'title': '美人天下01',
+            'description': 'md5:f88573d9d7225ada1359eaf0dbf8bcda',
+        },
+        'expected_warnings': [
+            'publish time'
+        ]
     }]
+    # http://www.letv.com/ptv/vplay/1118082.html
+    # This video is available only in Mainland China
 
     @staticmethod
     def urshift(val, n):
@@ -111,12 +119,14 @@ class LetvIE(InfoExtractor):
         publish_time = parse_iso8601(self._html_search_regex(
             r'发布时间&nbsp;([^<>]+) ', page, 'publish time', fatal=False),
             delimiter=' ', timezone=datetime.timedelta(hours=8))
+        description = self._html_search_meta('description', page, fatal=False)
 
         return {
             'id': media_id,
             'formats': urls,
             'title': playurl['title'],
             'thumbnail': playurl['pic'],
+            'description': description,
             'timestamp': publish_time,
         }
 
@@ -142,7 +152,8 @@ class LetvTvIE(InfoExtractor):
         entries = [self.url_result(media_url, ie='Letv')
                    for media_url in media_urls]
 
-        title = self._html_search_meta('keywords', page, fatal=False).split('，')[0]
+        title = self._html_search_meta('keywords', page,
+                                       fatal=False).split('，')[0]
         description = self._html_search_meta('description', page, fatal=False)
 
         return self.playlist_result(entries, playlist_id, playlist_title=title,
@@ -158,13 +169,14 @@ class LetvPlaylistIE(LetvTvIE):
             'title': '武媚娘传奇',
             'description': 'md5:e12499475ab3d50219e5bba00b3cb248'
         },
-        'playlist_count': 96
+        # This playlist contains some extra videos other than the drama itself
+        'playlist_mincount': 96
     }, {
         'url': 'http://tv.letv.com/pzt/lswjzzjc/index.shtml',
         'info_dict': {
             'id': 'lswjzzjc',
-            # should be "劲舞青春", but I can't find a simple way to determine
-            # the playlist title
+            # The title should be "劲舞青春", but I can't find a simple way to
+            # determine the playlist title
             'title': '乐视午间自制剧场',
             'description': 'md5:b1eef244f45589a7b5b1af9ff25a4489'
         },

From e765ed3a9c4ba52d4709a4a696881eae3401efa0 Mon Sep 17 00:00:00 2001
From: logon84 <rubenlogon@yahoo.es>
Date: Tue, 24 Feb 2015 19:41:46 +0100
Subject: [PATCH 0520/6219] [eporner] Fix redirect_code error

---
 youtube_dl/extractor/eporner.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/eporner.py b/youtube_dl/extractor/eporner.py
index 9ae28855b..f5943caa5 100644
--- a/youtube_dl/extractor/eporner.py
+++ b/youtube_dl/extractor/eporner.py
@@ -35,6 +35,7 @@ class EpornerIE(InfoExtractor):
         title = self._html_search_regex(
             r'<title>(.*?) - EPORNER', webpage, 'title')
 
+
         redirect_url = 'http://www.eporner.com/config5/%s/%s' % (video_id, display_id)
         player_code = self._download_webpage(
             redirect_url, display_id, note='Downloading player config')

From 37f885650c323e040a200bda9376bc7dbdf2ca25 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 25 Feb 2015 01:08:54 +0600
Subject: [PATCH 0521/6219] [eporner] Simplify and hardcode age limit

---
 youtube_dl/extractor/eporner.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/eporner.py b/youtube_dl/extractor/eporner.py
index f5943caa5..e006921ec 100644
--- a/youtube_dl/extractor/eporner.py
+++ b/youtube_dl/extractor/eporner.py
@@ -35,8 +35,7 @@ class EpornerIE(InfoExtractor):
         title = self._html_search_regex(
             r'<title>(.*?) - EPORNER', webpage, 'title')
 
-
-        redirect_url = 'http://www.eporner.com/config5/%s/%s' % (video_id, display_id)
+        redirect_url = 'http://www.eporner.com/config5/%s' % video_id
         player_code = self._download_webpage(
             redirect_url, display_id, note='Downloading player config')
 
@@ -67,5 +66,5 @@ class EpornerIE(InfoExtractor):
             'duration': duration,
             'view_count': view_count,
             'formats': formats,
-            'age_limit': self._rta_search(webpage),
+            'age_limit': 18,
         }

From 637570326bfa12575fe210e52e2a39d6585891d8 Mon Sep 17 00:00:00 2001
From: Antti Ajanki <antti.ajanki@iki.fi>
Date: Sun, 22 Feb 2015 10:16:51 +0200
Subject: [PATCH 0522/6219] [extractor/common] Extract the first of a seq of
 videos in a .smil file

---
 youtube_dl/extractor/common.py | 68 +++++++++++++++++++---------------
 1 file changed, 39 insertions(+), 29 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 87fce9cd8..4fe99d25d 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -921,39 +921,49 @@ class InfoExtractor(object):
 
         formats = []
         rtmp_count = 0
-        for video in smil.findall('./body/switch/video'):
-            src = video.get('src')
-            if not src:
-                continue
-            bitrate = int_or_none(video.get('system-bitrate') or video.get('systemBitrate'), 1000)
-            width = int_or_none(video.get('width'))
-            height = int_or_none(video.get('height'))
-            proto = video.get('proto')
-            if not proto:
-                if base:
-                    if base.startswith('rtmp'):
-                        proto = 'rtmp'
-                    elif base.startswith('http'):
-                        proto = 'http'
-            ext = video.get('ext')
-            if proto == 'm3u8':
-                formats.extend(self._extract_m3u8_formats(src, video_id, ext))
-            elif proto == 'rtmp':
-                rtmp_count += 1
-                streamer = video.get('streamer') or base
-                formats.append({
-                    'url': streamer,
-                    'play_path': src,
-                    'ext': 'flv',
-                    'format_id': 'rtmp-%d' % (rtmp_count if bitrate is None else bitrate),
-                    'tbr': bitrate,
-                    'width': width,
-                    'height': height,
-                })
+        if smil.findall('./body/seq/video'):
+            video = smil.findall('./body/seq/video')[0]
+            fmts, rtmp_count = self._parse_smil_video(video, base, rtmp_count)
+            formats.extend(fmts)
+        else:
+            for video in smil.findall('./body/switch/video'):
+                fmts, rtmp_count = self._parse_smil_video(video, base, rtmp_count)
+                formats.extend(fmts)
+
         self._sort_formats(formats)
 
         return formats
 
+    def _parse_smil_video(self, video, base, rtmp_count):
+        src = video.get('src')
+        if not src:
+            return ([], rtmp_count)
+        bitrate = int_or_none(video.get('system-bitrate') or video.get('systemBitrate'), 1000)
+        width = int_or_none(video.get('width'))
+        height = int_or_none(video.get('height'))
+        proto = video.get('proto')
+        if not proto:
+            if base:
+                if base.startswith('rtmp'):
+                    proto = 'rtmp'
+                elif base.startswith('http'):
+                    proto = 'http'
+        ext = video.get('ext')
+        if proto == 'm3u8':
+            return (self._extract_m3u8_formats(src, video_id, ext), rtmp_count)
+        elif proto == 'rtmp':
+            rtmp_count += 1
+            streamer = video.get('streamer') or base
+            return ([{
+                'url': streamer,
+                'play_path': src,
+                'ext': 'flv',
+                'format_id': 'rtmp-%d' % (rtmp_count if bitrate is None else bitrate),
+                'tbr': bitrate,
+                'width': width,
+                'height': height,
+            }], rtmp_count)
+
     def _live_title(self, name):
         """ Generate the title for a live video """
         now = datetime.datetime.now()

From 6f4ba54079893a09c6aa78fe3420523fb96df858 Mon Sep 17 00:00:00 2001
From: Antti Ajanki <antti.ajanki@iki.fi>
Date: Sun, 22 Feb 2015 10:18:36 +0200
Subject: [PATCH 0523/6219] [extractor/common] Extract HTTP (possibly f4m) URLs
 from a .smil file

---
 youtube_dl/extractor/common.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 4fe99d25d..313688208 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -963,6 +963,14 @@ class InfoExtractor(object):
                 'width': width,
                 'height': height,
             }], rtmp_count)
+        elif proto.startswith('http'):
+            return ([{
+                'url': base + src,
+                'ext': ext or 'flv',
+                'tbr': bitrate,
+                'width': width,
+                'height': height,
+            }], rtmp_count)
 
     def _live_title(self, name):
         """ Generate the title for a live video """

From c4f8c453ae2f735fc2320856e15e66510d74fd72 Mon Sep 17 00:00:00 2001
From: Antti Ajanki <antti.ajanki@iki.fi>
Date: Sun, 22 Feb 2015 21:03:49 +0200
Subject: [PATCH 0524/6219] [f4m] Refresh fragment list periodically on live
 streams

---
 youtube_dl/downloader/f4m.py | 59 ++++++++++++++++++++++++++++++------
 1 file changed, 49 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index 7b8fe8cf5..1df9ebe5b 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -121,7 +121,8 @@ class FlvReader(io.BytesIO):
 
         self.read_unsigned_int()  # BootstrapinfoVersion
         # Profile,Live,Update,Reserved
-        self.read(1)
+        flags = self.read_unsigned_char()
+        live = flags & 0x20 != 0
         # time scale
         self.read_unsigned_int()
         # CurrentMediaTime
@@ -160,6 +161,7 @@ class FlvReader(io.BytesIO):
         return {
             'segments': segments,
             'fragments': fragments,
+            'live': live,
         }
 
     def read_bootstrap_info(self):
@@ -182,6 +184,10 @@ def build_fragments_list(boot_info):
     for segment, fragments_count in segment_run_table['segment_run']:
         for _ in range(fragments_count):
             res.append((segment, next(fragments_counter)))
+
+    if boot_info['live']:
+        res = res[-2:]
+
     return res
 
 
@@ -246,6 +252,38 @@ class F4mFD(FileDownloader):
             self.report_error('Unsupported DRM')
         return media
 
+    def _get_bootstrap_from_url(self, bootstrap_url):
+        bootstrap = self.ydl.urlopen(bootstrap_url).read()
+        return read_bootstrap_info(bootstrap)
+
+    def _update_live_fragments(self, bootstrap_url, latest_fragment):
+        fragments_list = []
+        retries = 30
+        while (not fragments_list) and (retries > 0):
+            boot_info = self._get_bootstrap_from_url(bootstrap_url)
+            fragments_list = build_fragments_list(boot_info)
+            fragments_list = [f for f in fragments_list if f[1] > latest_fragment]
+            if not fragments_list:
+                # Retry after a while
+                time.sleep(5.0)
+                retries -= 1
+
+        if not fragments_list:
+            self.report_error('Failed to update fragments')
+
+        return fragments_list
+
+    def _parse_bootstrap_node(self, node, base_url):
+        if node.text is None:
+            bootstrap_url = compat_urlparse.urljoin(
+                base_url, node.attrib['url'])
+            boot_info = self._get_bootstrap_from_url(bootstrap_url)
+        else:
+            bootstrap_url = None
+            bootstrap = base64.b64decode(node.text)
+            boot_info = read_bootstrap_info(bootstrap)
+        return (boot_info, bootstrap_url)
+
     def real_download(self, filename, info_dict):
         man_url = info_dict['url']
         requested_bitrate = info_dict.get('tbr')
@@ -265,18 +303,13 @@ class F4mFD(FileDownloader):
 
         base_url = compat_urlparse.urljoin(man_url, media.attrib['url'])
         bootstrap_node = doc.find(_add_ns('bootstrapInfo'))
-        if bootstrap_node.text is None:
-            bootstrap_url = compat_urlparse.urljoin(
-                base_url, bootstrap_node.attrib['url'])
-            bootstrap = self.ydl.urlopen(bootstrap_url).read()
-        else:
-            bootstrap = base64.b64decode(bootstrap_node.text)
+        boot_info, bootstrap_url = self._parse_bootstrap_node(bootstrap_node, base_url)
+        live = boot_info['live']
         metadata_node = media.find(_add_ns('metadata'))
         if metadata_node is not None:
             metadata = base64.b64decode(metadata_node.text)
         else:
             metadata = None
-        boot_info = read_bootstrap_info(bootstrap)
 
         fragments_list = build_fragments_list(boot_info)
         if self.params.get('test', False):
@@ -301,7 +334,8 @@ class F4mFD(FileDownloader):
         (dest_stream, tmpfilename) = sanitize_open(tmpfilename, 'wb')
 
         write_flv_header(dest_stream)
-        write_metadata_tag(dest_stream, metadata)
+        if not live:
+            write_metadata_tag(dest_stream, metadata)
 
         # This dict stores the download progress, it's updated by the progress
         # hook
@@ -348,7 +382,8 @@ class F4mFD(FileDownloader):
         http_dl.add_progress_hook(frag_progress_hook)
 
         frags_filenames = []
-        for (seg_i, frag_i) in fragments_list:
+        while fragments_list:
+            seg_i, frag_i = fragments_list.pop(0)
             name = 'Seg%d-Frag%d' % (seg_i, frag_i)
             url = base_url + name
             if akamai_pv:
@@ -367,6 +402,10 @@ class F4mFD(FileDownloader):
                         break
             frags_filenames.append(frag_filename)
 
+            if not fragments_list and live and bootstrap_url:
+                fragments_list = self._update_live_fragments(bootstrap_url, frag_i)
+                self.to_screen('Updated available fragments: %d' % len(fragments_list))
+
         dest_stream.close()
 
         elapsed = time.time() - start

From 5eaaeb7c317a543af4bde5eb7d465f3695fc97d9 Mon Sep 17 00:00:00 2001
From: Antti Ajanki <antti.ajanki@iki.fi>
Date: Mon, 23 Feb 2015 21:56:35 +0200
Subject: [PATCH 0525/6219] [f4m] Tolerate missed fragments on live streams

---
 youtube_dl/downloader/f4m.py | 43 +++++++++++++++++++++++++-----------
 1 file changed, 30 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index 1df9ebe5b..3dc796faa 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -11,6 +11,7 @@ from .common import FileDownloader
 from .http import HttpFD
 from ..compat import (
     compat_urlparse,
+    compat_urllib_error,
 )
 from ..utils import (
     struct_pack,
@@ -389,22 +390,38 @@ class F4mFD(FileDownloader):
             if akamai_pv:
                 url += '?' + akamai_pv.strip(';')
             frag_filename = '%s-%s' % (tmpfilename, name)
-            success = http_dl.download(frag_filename, {'url': url})
-            if not success:
-                return False
-            with open(frag_filename, 'rb') as down:
-                down_data = down.read()
-                reader = FlvReader(down_data)
-                while True:
-                    _, box_type, box_data = reader.read_box_info()
-                    if box_type == b'mdat':
-                        dest_stream.write(box_data)
-                        break
-            frags_filenames.append(frag_filename)
+            try:
+                success = http_dl.download(frag_filename, {'url': url})
+                if not success:
+                    return False
+                with open(frag_filename, 'rb') as down:
+                    down_data = down.read()
+                    reader = FlvReader(down_data)
+                    while True:
+                        _, box_type, box_data = reader.read_box_info()
+                        if box_type == b'mdat':
+                            dest_stream.write(box_data)
+                            break
+                if live:
+                    os.remove(frag_filename)
+                else:
+                    frags_filenames.append(frag_filename)
+            except (compat_urllib_error.HTTPError, ) as err:
+                if live and (err.code == 404 or err.code == 410):
+                    # We didn't keep up with the live window. Continue
+                    # with the next available fragment.
+                    msg = 'Fragment %d unavailable' % frag_i
+                    self.report_warning(msg)
+                    fragments_list = []
+                else:
+                    raise
 
             if not fragments_list and live and bootstrap_url:
                 fragments_list = self._update_live_fragments(bootstrap_url, frag_i)
-                self.to_screen('Updated available fragments: %d' % len(fragments_list))
+                total_frags += len(fragments_list)
+                if fragments_list and (fragments_list[0][1] > frag_i + 1):
+                    msg = 'Missed %d fragments' % (fragments_list[0][1] - (frag_i + 1))
+                    self.report_warning(msg)
 
         dest_stream.close()
 

From b8988b63a6baa206b7f5e35d99a9f4eff6ec7b5e Mon Sep 17 00:00:00 2001
From: Antti Ajanki <antti.ajanki@iki.fi>
Date: Tue, 24 Feb 2015 21:23:59 +0200
Subject: [PATCH 0526/6219] [wdr] Download a live stream

---
 youtube_dl/extractor/wdr.py | 30 ++++++++++++++++++++++++++++++
 1 file changed, 30 insertions(+)

diff --git a/youtube_dl/extractor/wdr.py b/youtube_dl/extractor/wdr.py
index c90488500..b46802306 100644
--- a/youtube_dl/extractor/wdr.py
+++ b/youtube_dl/extractor/wdr.py
@@ -28,6 +28,7 @@ class WDRIE(InfoExtractor):
                 'title': 'Servicezeit',
                 'description': 'md5:c8f43e5e815eeb54d0b96df2fba906cb',
                 'upload_date': '20140310',
+                'is_live': False
             },
             'params': {
                 'skip_download': True,
@@ -41,6 +42,7 @@ class WDRIE(InfoExtractor):
                 'title': 'Marga Spiegel ist tot',
                 'description': 'md5:2309992a6716c347891c045be50992e4',
                 'upload_date': '20140311',
+                'is_live': False
             },
             'params': {
                 'skip_download': True,
@@ -55,6 +57,7 @@ class WDRIE(InfoExtractor):
                 'title': 'Erlebte Geschichten: Marga Spiegel (29.11.2009)',
                 'description': 'md5:2309992a6716c347891c045be50992e4',
                 'upload_date': '20091129',
+                'is_live': False
             },
         },
         {
@@ -66,6 +69,7 @@ class WDRIE(InfoExtractor):
                 'title': 'Flavia Coelho: Amar é Amar',
                 'description': 'md5:7b29e97e10dfb6e265238b32fa35b23a',
                 'upload_date': '20140717',
+                'is_live': False
             },
         },
         {
@@ -74,6 +78,20 @@ class WDRIE(InfoExtractor):
             'info_dict': {
                 'id': 'mediathek/video/sendungen/quarks_und_co/filterseite-quarks-und-co100',
             }
+        },
+        {
+            'url': 'http://www1.wdr.de/mediathek/video/livestream/index.html',
+            'info_dict': {
+                'id': 'mdb-103364',
+                'title': 're:^WDR Fernsehen [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
+                'description': 'md5:ae2ff888510623bf8d4b115f95a9b7c9',
+                'ext': 'flv',
+                'upload_date': '20150212',
+                'is_live': True
+            },
+            'params': {
+                'skip_download': True,
+            },
         }
     ]
 
@@ -119,6 +137,10 @@ class WDRIE(InfoExtractor):
         video_url = flashvars['dslSrc'][0]
         title = flashvars['trackerClipTitle'][0]
         thumbnail = flashvars['startPicture'][0] if 'startPicture' in flashvars else None
+        is_live = flashvars.get('isLive', ['0'])[0] == '1'
+
+        if is_live:
+            title = self._live_title(title)
 
         if 'trackerClipAirTime' in flashvars:
             upload_date = flashvars['trackerClipAirTime'][0]
@@ -131,6 +153,13 @@ class WDRIE(InfoExtractor):
         if video_url.endswith('.f4m'):
             video_url += '?hdcore=3.2.0&plugin=aasp-3.2.0.77.18'
             ext = 'flv'
+        elif video_url.endswith('.smil'):
+            fmt = self._extract_smil_formats(video_url, page_id)[0]
+            video_url = fmt['url']
+            sep = '&' if '?' in video_url else '?'
+            video_url += sep
+            video_url += 'hdcore=3.3.0&plugin=aasp-3.3.0.99.43'
+            ext = fmt['ext']
         else:
             ext = determine_ext(video_url)
 
@@ -144,6 +173,7 @@ class WDRIE(InfoExtractor):
             'description': description,
             'thumbnail': thumbnail,
             'upload_date': upload_date,
+            'is_live': is_live
         }
 
 

From 13d8fbef30f1d7608b02e78818fa2ec211d69e21 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 25 Feb 2015 17:56:51 +0100
Subject: [PATCH 0527/6219] [generic] Don't set the 'title' if it's not defined
 in the entry (closes #5061)

Some of them may be an 'url' result, which in general don't have the 'title' field.
---
 youtube_dl/extractor/generic.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 875e1bf05..3aff57e30 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1208,7 +1208,9 @@ class GenericIE(InfoExtractor):
             return entries[0]
         else:
             for num, e in enumerate(entries, start=1):
-                e['title'] = '%s (%d)' % (e['title'], num)
+                # 'url' results don't have a title
+                if e.get('title') is not None:
+                    e['title'] = '%s (%d)' % (e['title'], num)
             return {
                 '_type': 'playlist',
                 'entries': entries,

From 9504fc21b583d6ff968eec9d5843fc40b48725e7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Markus=20M=C3=BCller?= <mmu@grummel.net>
Date: Wed, 25 Feb 2015 23:27:19 +0100
Subject: [PATCH 0528/6219] Fix the RTL extractor for new episodes by using a
 different hostname

---
 youtube_dl/extractor/rtlnow.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/rtlnow.py b/youtube_dl/extractor/rtlnow.py
index fd93cc66f..785a8045e 100644
--- a/youtube_dl/extractor/rtlnow.py
+++ b/youtube_dl/extractor/rtlnow.py
@@ -146,7 +146,7 @@ class RTLnowIE(InfoExtractor):
                 mobj = re.search(r'.*/(?P<hoster>[^/]+)/videos/(?P<play_path>.+)\.f4m', filename.text)
                 if mobj:
                     fmt = {
-                        'url': 'rtmpe://fmspay-fra2.rtl.de/' + mobj.group('hoster'),
+                        'url': 'rtmpe://fms.rtl.de/' + mobj.group('hoster'),
                         'play_path': 'mp4:' + mobj.group('play_path'),
                         'page_url': url,
                         'player_url': video_page_url + 'includes/vodplayer.swf',

From c10ea454dc7a06bf4d911774fd12b49ce7550845 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 25 Feb 2015 23:52:54 +0100
Subject: [PATCH 0529/6219] [telecinco] Recognize more urls (closes #5065)

---
 youtube_dl/extractor/mitele.py    | 4 ++--
 youtube_dl/extractor/telecinco.py | 9 ++++++---
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/mitele.py b/youtube_dl/extractor/mitele.py
index 256758323..d8897eb90 100644
--- a/youtube_dl/extractor/mitele.py
+++ b/youtube_dl/extractor/mitele.py
@@ -18,7 +18,7 @@ class MiTeleIE(InfoExtractor):
     IE_NAME = 'mitele.es'
     _VALID_URL = r'http://www\.mitele\.es/[^/]+/[^/]+/[^/]+/(?P<id>[^/]+)/'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.mitele.es/programas-tv/diario-de/la-redaccion/programa-144/',
         'md5': '6a75fe9d0d3275bead0cb683c616fddb',
         'info_dict': {
@@ -29,7 +29,7 @@ class MiTeleIE(InfoExtractor):
             'display_id': 'programa-144',
             'duration': 2913,
         },
-    }
+    }]
 
     def _real_extract(self, url):
         episode = self._match_id(url)
diff --git a/youtube_dl/extractor/telecinco.py b/youtube_dl/extractor/telecinco.py
index be3f72df7..251a68680 100644
--- a/youtube_dl/extractor/telecinco.py
+++ b/youtube_dl/extractor/telecinco.py
@@ -6,9 +6,9 @@ from .mitele import MiTeleIE
 
 class TelecincoIE(MiTeleIE):
     IE_NAME = 'telecinco.es'
-    _VALID_URL = r'https?://www\.telecinco\.es/[^/]+/[^/]+/[^/]+/(?P<id>.*?)\.html'
+    _VALID_URL = r'https?://www\.telecinco\.es/[^/]+/[^/]+/(?:[^/]+/)?(?P<id>.*?)\.html'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.telecinco.es/robinfood/temporada-01/t01xp14/Bacalao-cocochas-pil-pil_0_1876350223.html',
         'info_dict': {
             'id': 'MDSVID20141015_0058',
@@ -16,4 +16,7 @@ class TelecincoIE(MiTeleIE):
             'title': 'Con Martín Berasategui, hacer un bacalao al ...',
             'duration': 662,
         },
-    }
+    }, {
+        'url': 'http://www.telecinco.es/informativos/nacional/Pablo_Iglesias-Informativos_Telecinco-entrevista-Pedro_Piqueras_2_1945155182.html',
+        'only_matching': True,
+    }]

From 1c69bca25871ebb8b54ebc5cb9bb4b2124bb12ab Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 26 Feb 2015 00:24:54 +0100
Subject: [PATCH 0530/6219] [escapist] Fix config URL matching

---
 youtube_dl/extractor/escapist.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/escapist.py b/youtube_dl/extractor/escapist.py
index 51ffec7ee..3244306dc 100644
--- a/youtube_dl/extractor/escapist.py
+++ b/youtube_dl/extractor/escapist.py
@@ -44,10 +44,10 @@ class EscapistIE(InfoExtractor):
         config_url = compat_urllib_parse.unquote(self._html_search_regex(
             r'''(?x)
             (?:
-                <param\s+name="flashvars"\s+value="config=|
+                <param\s+name="flashvars".*?\s+value="config=|
                 flashvars=&quot;config=
             )
-            ([^"&]+)
+            (https?://[^"&]+)
             ''',
             webpage, 'config URL'))
 

From 183139340b4841f22357fcf2b6a5ce2dedd71146 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 26 Feb 2015 00:40:12 +0100
Subject: [PATCH 0531/6219] [utils] Bump our user agent

---
 youtube_dl/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index e2631dccd..506c896de 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -54,7 +54,7 @@ from .compat import (
 compiled_regex_type = type(re.compile(''))
 
 std_headers = {
-    'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20100101 Firefox/10.0 (Chrome)',
+    'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20150101 Firefox/20.0 (Chrome)',
     'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
     'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
     'Accept-Encoding': 'gzip, deflate',

From 13cd97f3dfd87373cfc501224692352cb20e8194 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 26 Feb 2015 00:42:02 +0100
Subject: [PATCH 0532/6219] release 2015.02.26

---
 README.md             | 2 ++
 youtube_dl/version.py | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 699401b49..2c53e2211 100644
--- a/README.md
+++ b/README.md
@@ -139,6 +139,8 @@ which means you can modify it, redistribute it or use it however you like.
                                      dislike_count <? 50 & description" .
     --no-playlist                    If the URL refers to a video and a
                                      playlist, download only the video.
+    --yes-playlist                   If the URL refers to a video and a
+                                     playlist, download the playlist.
     --age-limit YEARS                download only videos suitable for the given
                                      age
     --download-archive FILE          Download only videos not listed in the
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index d23c6ae3d..0cbf66ed1 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.24.2'
+__version__ = '2015.02.26'

From 9a48926a5734e0c1530c4788680961610090a837 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 26 Feb 2015 00:59:53 +0100
Subject: [PATCH 0533/6219] [escapist] Add support for advertisements

---
 youtube_dl/extractor/escapist.py | 41 ++++++++++++++++++++++++--------
 1 file changed, 31 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/escapist.py b/youtube_dl/extractor/escapist.py
index 3244306dc..b45c1dbd0 100644
--- a/youtube_dl/extractor/escapist.py
+++ b/youtube_dl/extractor/escapist.py
@@ -52,6 +52,7 @@ class EscapistIE(InfoExtractor):
             webpage, 'config URL'))
 
         formats = []
+        ad_formats = []
 
         def _add_format(name, cfgurl, quality):
             config = self._download_json(
@@ -61,14 +62,19 @@ class EscapistIE(InfoExtractor):
                 transform_source=js_to_json)
 
             playlist = config['playlist']
-            video_url = next(
-                p['url'] for p in playlist
-                if p.get('eventCategory') == 'Video')
-            formats.append({
-                'url': video_url,
-                'format_id': name,
-                'quality': quality,
-            })
+            for p in playlist:
+                if p.get('eventCategory') == 'Video':
+                    ar = formats
+                elif p.get('eventCategory') == 'Video Postroll':
+                    ar = ad_formats
+                else:
+                    continue
+
+                ar.append({
+                    'url': p['url'],
+                    'format_id': name,
+                    'quality': quality,
+                })
 
         _add_format('normal', config_url, quality=0)
         hq_url = (config_url +
@@ -77,10 +83,9 @@ class EscapistIE(InfoExtractor):
             _add_format('hq', hq_url, quality=1)
         except ExtractorError:
             pass  # That's fine, we'll just use normal quality
-
         self._sort_formats(formats)
 
-        return {
+        res = {
             'id': video_id,
             'formats': formats,
             'uploader': uploader,
@@ -89,3 +94,19 @@ class EscapistIE(InfoExtractor):
             'thumbnail': self._og_search_thumbnail(webpage),
             'description': description,
         }
+
+        if self._downloader.params.get('include_ads') and ad_formats:
+            self._sort_formats(ad_formats)
+            ad_res = {
+                'id': '%s-ad' % video_id,
+                'title': '%s (Postroll)' % title,
+                'formats': ad_formats,
+            }
+            return {
+                '_type': 'playlist',
+                'entries': [res, ad_res],
+                'title': title,
+                'id': video_id,
+            }
+
+        return res

From 3e675fabe027ecf20e662454ff7a174596801256 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 26 Feb 2015 01:25:00 +0100
Subject: [PATCH 0534/6219] [airmozilla] Be more tolerant when nonessential
 items are missing (#5030)

---
 test/test_utils.py                 |  1 +
 youtube_dl/extractor/airmozilla.py | 47 +++++++++++++++---------------
 youtube_dl/utils.py                |  5 ++++
 3 files changed, 30 insertions(+), 23 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 2f8996d7b..3fba8ae11 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -246,6 +246,7 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(parse_duration('2.5 hours'), 9000)
         self.assertEqual(parse_duration('02:03:04'), 7384)
         self.assertEqual(parse_duration('01:02:03:04'), 93784)
+        self.assertEqual(parse_duration('1 hour 3 minutes'), 3780)
 
     def test_fix_xml_ampersands(self):
         self.assertEqual(
diff --git a/youtube_dl/extractor/airmozilla.py b/youtube_dl/extractor/airmozilla.py
index 44c20f886..611ad1e9d 100644
--- a/youtube_dl/extractor/airmozilla.py
+++ b/youtube_dl/extractor/airmozilla.py
@@ -4,7 +4,11 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import parse_iso8601
+from ..utils import (
+    int_or_none,
+    parse_duration,
+    parse_iso8601,
+)
 
 
 class AirMozillaIE(InfoExtractor):
@@ -27,13 +31,6 @@ class AirMozillaIE(InfoExtractor):
         }
     }
 
-    _QUALITY_MAP = {
-        '360p': 0,
-        '576p': 1,
-        '640p': 2,
-        '720p': 3,
-    }
-
     def _real_extract(self, url):
         display_id = self._match_id(url)
         webpage = self._download_webpage(url, display_id)
@@ -43,19 +40,23 @@ class AirMozillaIE(InfoExtractor):
         jwconfig = self._search_regex(r'\svar jwconfig = (\{.*?\});\s', embed_script, 'metadata')
         metadata = self._parse_json(jwconfig, video_id)
 
-        formats = []
-        for source in metadata['playlist'][0]['sources']:
-            fmt = {
-                'url': source['file'],
-                'ext': source['type'],
-                'format_id': self._search_regex(r'&format=(.*)$', source['file'], 'video format'),
-                'resolution': source['label'],
-                'quality': self._QUALITY_MAP.get(source['label'], -1),
-            }
-            formats.append(fmt)
+        formats = [{
+            'url': source['file'],
+            'ext': source['type'],
+            'format_id': self._search_regex(r'&format=(.*)$', source['file'], 'video format'),
+            'format': source['label'],
+            'height': int(source['label'].rstrip('p')),
+        } for source in metadata['playlist'][0]['sources']]
         self._sort_formats(formats)
 
-        duration_match = re.search(r'Duration:(?: (?P<H>\d+) hours?)?(?: (?P<M>\d+) minutes?)?', webpage)
+        view_count = int_or_none(self._html_search_regex(
+            r'Views since archived: ([0-9]+)',
+            webpage, 'view count', fatal=False))
+        timestamp = parse_iso8601(self._html_search_regex(
+            r'<time datetime="(.*?)"', webpage, 'timestamp', fatal=False))
+        duration = parse_duration(self._search_regex(
+            r'Duration:\s*(\d+\s*hours?\s*\d+\s*minutes?)',
+            webpage, 'duration', fatal=False))
 
         return {
             'id': video_id,
@@ -63,11 +64,11 @@ class AirMozillaIE(InfoExtractor):
             'formats': formats,
             'url': self._og_search_url(webpage),
             'display_id': display_id,
-            'thumbnail': metadata['playlist'][0]['image'],
+            'thumbnail': metadata['playlist'][0].get('image'),
             'description': self._og_search_description(webpage),
-            'timestamp': parse_iso8601(self._html_search_regex(r'<time datetime="(.*?)"', webpage, 'timestamp')),
+            'timestamp': timestamp,
             'location': self._html_search_regex(r'Location: (.*)', webpage, 'location', default=None),
-            'duration': int(duration_match.groupdict()['H'] or 0) * 3600 + int(duration_match.groupdict()['M'] or 0) * 60,
-            'view_count': int(self._html_search_regex(r'Views since archived: ([0-9]+)', webpage, 'view count')),
+            'duration': duration,
+            'view_count': view_count,
             'categories': re.findall(r'<a href=".*?" class="channel">(.*?)</a>', webpage),
         }
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 506c896de..1f3bfef7d 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1290,6 +1290,7 @@ def parse_duration(s):
             (?P<only_mins>[0-9.]+)\s*(?:mins?|minutes?)\s*|
             (?P<only_hours>[0-9.]+)\s*(?:hours?)|
 
+            \s*(?P<hours_reversed>[0-9]+)\s*(?:[:h]|hours?)\s*(?P<mins_reversed>[0-9]+)\s*(?:[:m]|mins?|minutes?)\s*|
             (?:
                 (?:
                     (?:(?P<days>[0-9]+)\s*(?:[:d]|days?)\s*)?
@@ -1308,10 +1309,14 @@ def parse_duration(s):
         return float_or_none(m.group('only_hours'), invscale=60 * 60)
     if m.group('secs'):
         res += int(m.group('secs'))
+    if m.group('mins_reversed'):
+        res += int(m.group('mins_reversed')) * 60
     if m.group('mins'):
         res += int(m.group('mins')) * 60
     if m.group('hours'):
         res += int(m.group('hours')) * 60 * 60
+    if m.group('hours_reversed'):
+        res += int(m.group('hours_reversed')) * 60 * 60
     if m.group('days'):
         res += int(m.group('days')) * 24 * 60 * 60
     if m.group('ms'):

From 84be7c230cd7b17c54926928e4108a1118f9892e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 26 Feb 2015 01:25:54 +0100
Subject: [PATCH 0535/6219] Cred @duncankl for airmozilla

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index bdd2a15dc..4674a5af3 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -112,3 +112,4 @@ Frans de Jonge
 Robin de Rooij
 Ryan Schmidt
 Leslie P. Polzer
+Duncan Keall

From 265bfa2c79abc8f233132126be313ed2d4b18dc8 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 26 Feb 2015 01:30:18 +0100
Subject: [PATCH 0536/6219] [letv] Simplify

---
 youtube_dl/extractor/letv.py | 22 ++++++++++++++--------
 1 file changed, 14 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/letv.py b/youtube_dl/extractor/letv.py
index d5839263c..583ce35b9 100644
--- a/youtube_dl/extractor/letv.py
+++ b/youtube_dl/extractor/letv.py
@@ -1,14 +1,20 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import os.path
+import datetime
 import re
 import time
-import datetime
 
 from .common import InfoExtractor
-from ..compat import (compat_urlparse, compat_urllib_parse)
-from ..utils import (ExtractorError, parse_iso8601)
+from ..compat import (
+    compat_urlparse,
+    compat_urllib_parse,
+)
+from ..utils import (
+    determine_ext,
+    ExtractorError,
+    parse_iso8601,
+)
 
 
 class LetvIE(InfoExtractor):
@@ -44,7 +50,7 @@ class LetvIE(InfoExtractor):
     def urshift(val, n):
         return val >> n if val >= 0 else (val + 0x100000000) >> n
 
-    # ror() and calcTimeKey() are reversed from a embedded swf file in KLetvPlayer.swf
+    # ror() and calc_time_key() are reversed from a embedded swf file in KLetvPlayer.swf
     def ror(self, param1, param2):
         _loc3_ = 0
         while _loc3_ < param2:
@@ -52,7 +58,7 @@ class LetvIE(InfoExtractor):
             _loc3_ += 1
         return param1
 
-    def calcTimeKey(self, param1):
+    def calc_time_key(self, param1):
         _loc2_ = 773625421
         _loc3_ = self.ror(param1, _loc2_ % 13)
         _loc3_ = _loc3_ ^ _loc2_
@@ -67,7 +73,7 @@ class LetvIE(InfoExtractor):
             'platid': 1,
             'splatid': 101,
             'format': 1,
-            'tkey': self.calcTimeKey(int(time.time())),
+            'tkey': self.calc_time_key(int(time.time())),
             'domain': 'www.letv.com'
         }
         play_json = self._download_json(
@@ -108,7 +114,7 @@ class LetvIE(InfoExtractor):
 
                 url_info_dict = {
                     'url': media_url,
-                    'ext': os.path.splitext(dispatch[format_id][1])[1][1:]
+                    'ext': determine_ext(dispatch[format_id][1])
                 }
 
                 if format_id[-1:] == 'p':

From 72a406e7aa6ef751f5f2ad4b110b749ce40d70a4 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 26 Feb 2015 01:35:43 +0100
Subject: [PATCH 0537/6219] [extractor/common] Pass in video_id (#5057)

---
 youtube_dl/extractor/common.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 313688208..7977fa8d0 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -923,18 +923,18 @@ class InfoExtractor(object):
         rtmp_count = 0
         if smil.findall('./body/seq/video'):
             video = smil.findall('./body/seq/video')[0]
-            fmts, rtmp_count = self._parse_smil_video(video, base, rtmp_count)
+            fmts, rtmp_count = self._parse_smil_video(video, video_id, base, rtmp_count)
             formats.extend(fmts)
         else:
             for video in smil.findall('./body/switch/video'):
-                fmts, rtmp_count = self._parse_smil_video(video, base, rtmp_count)
+                fmts, rtmp_count = self._parse_smil_video(video, video_id, base, rtmp_count)
                 formats.extend(fmts)
 
         self._sort_formats(formats)
 
         return formats
 
-    def _parse_smil_video(self, video, base, rtmp_count):
+    def _parse_smil_video(self, video, video_id, base, rtmp_count):
         src = video.get('src')
         if not src:
             return ([], rtmp_count)

From 2a15a98a6a077381cba52207b791d03506b0416c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 26 Feb 2015 01:44:20 +0100
Subject: [PATCH 0538/6219] [rmtp] Encode filename before invoking subprocess

This fixes #5066.
Reproducible with
LC_ALL=C youtube-dl "http://www.prosieben.de/tv/germanys-next-topmodel/video/playlist/ganze-folge-episode-2-das-casting-in-muenchen"
---
 youtube_dl/downloader/rtmp.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/rtmp.py b/youtube_dl/downloader/rtmp.py
index 0a52c34c7..89e98ae61 100644
--- a/youtube_dl/downloader/rtmp.py
+++ b/youtube_dl/downloader/rtmp.py
@@ -119,7 +119,9 @@ class RtmpFD(FileDownloader):
         # Download using rtmpdump. rtmpdump returns exit code 2 when
         # the connection was interrumpted and resuming appears to be
         # possible. This is part of rtmpdump's normal usage, AFAIK.
-        basic_args = ['rtmpdump', '--verbose', '-r', url, '-o', tmpfilename]
+        basic_args = [
+            'rtmpdump', '--verbose', '-r', url,
+            '-o', encodeFilename(tmpfilename, True)]
         if player_url is not None:
             basic_args += ['--swfVfy', player_url]
         if page_url is not None:

From 1c6510f57a458aa9634923c0f48596b57d084649 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 26 Feb 2015 01:47:12 +0100
Subject: [PATCH 0539/6219] [Makefile] clean pyc files in clean target

---
 Makefile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Makefile b/Makefile
index 708732956..c6c76274f 100644
--- a/Makefile
+++ b/Makefile
@@ -2,6 +2,7 @@ all: youtube-dl README.md CONTRIBUTING.md README.txt youtube-dl.1 youtube-dl.bas
 
 clean:
 	rm -rf youtube-dl.1.temp.md youtube-dl.1 youtube-dl.bash-completion README.txt MANIFEST build/ dist/ .coverage cover/ youtube-dl.tar.gz youtube-dl.zsh youtube-dl.fish *.dump *.part *.info.json *.mp4 *.flv *.mp3 *.avi CONTRIBUTING.md.tmp youtube-dl youtube-dl.exe
+	find -name "*.pyc" -delete
 
 PREFIX ?= /usr/local
 BINDIR ?= $(PREFIX)/bin

From 5c340b038740061c61f55be1935e364a883a3138 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 26 Feb 2015 01:47:16 +0100
Subject: [PATCH 0540/6219] release 2015.02.26.1

---
 docs/supportedsites.md | 4 ++++
 youtube_dl/version.py  | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 9f70db80a..49b4ac8c1 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -17,6 +17,7 @@
  - **AdultSwim**
  - **Aftenposten**
  - **Aftonbladet**
+ - **AirMozilla**
  - **AlJazeera**
  - **Allocine**
  - **AlphaPorno**
@@ -220,6 +221,9 @@
  - **Ku6**
  - **la7.tv**
  - **Laola1Tv**
+ - **Letv**
+ - **LetvPlaylist**
+ - **LetvTv**
  - **lifenews**: LIFE | NEWS
  - **LiveLeak**
  - **livestream**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 0cbf66ed1..9043ca695 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.26'
+__version__ = '2015.02.26.1'

From 63a562f95ee62e05e689bf7fbf8e923749248b05 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 26 Feb 2015 09:19:26 +0100
Subject: [PATCH 0541/6219] [escapist] Detect IP blocking and use another UA
 (Fixes #5069)

---
 youtube_dl/extractor/escapist.py | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/escapist.py b/youtube_dl/extractor/escapist.py
index b45c1dbd0..80e9084f4 100644
--- a/youtube_dl/extractor/escapist.py
+++ b/youtube_dl/extractor/escapist.py
@@ -3,6 +3,7 @@ from __future__ import unicode_literals
 from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse,
+    compat_urllib_request,
 )
 from ..utils import (
     ExtractorError,
@@ -12,6 +13,7 @@ from ..utils import (
 
 class EscapistIE(InfoExtractor):
     _VALID_URL = r'https?://?(www\.)?escapistmagazine\.com/videos/view/[^/?#]+/(?P<id>[0-9]+)-[^/?#]*(?:$|[?#])'
+    _USER_AGENT = 'Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko'
     _TEST = {
         'url': 'http://www.escapistmagazine.com/videos/view/the-escapist-presents/6618-Breaking-Down-Baldurs-Gate',
         'md5': 'ab3a706c681efca53f0a35f1415cf0d1',
@@ -28,7 +30,9 @@ class EscapistIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
+        webpage_req = compat_urllib_request.Request(url)
+        webpage_req.add_header('User-Agent', self._USER_AGENT)
+        webpage = self._download_webpage(webpage_req, video_id)
 
         uploader_id = self._html_search_regex(
             r"<h1\s+class='headline'>\s*<a\s+href='/videos/view/(.*?)'",
@@ -54,9 +58,11 @@ class EscapistIE(InfoExtractor):
         formats = []
         ad_formats = []
 
-        def _add_format(name, cfgurl, quality):
+        def _add_format(name, cfg_url, quality):
+            cfg_req = compat_urllib_request.Request(cfg_url)
+            cfg_req.add_header('User-Agent', self._USER_AGENT)
             config = self._download_json(
-                cfgurl, video_id,
+                cfg_req, video_id,
                 'Downloading ' + name + ' configuration',
                 'Unable to download ' + name + ' configuration',
                 transform_source=js_to_json)
@@ -74,6 +80,9 @@ class EscapistIE(InfoExtractor):
                     'url': p['url'],
                     'format_id': name,
                     'quality': quality,
+                    'http_headers': {
+                        'User-Agent': self._USER_AGENT,
+                    },
                 })
 
         _add_format('normal', config_url, quality=0)
@@ -85,6 +94,9 @@ class EscapistIE(InfoExtractor):
             pass  # That's fine, we'll just use normal quality
         self._sort_formats(formats)
 
+        if '/escapist/sales-marketing/' in formats[-1]['url']:
+            raise ExtractorError('This IP address has been blocked by The Escapist', expected=True)
+
         res = {
             'id': video_id,
             'formats': formats,

From 9724e5d33661b6d6c84b6da64b78c0b96221ab24 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 26 Feb 2015 09:45:11 +0100
Subject: [PATCH 0542/6219] release 2015.02.26.2

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 9043ca695..cf3e28bbe 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.26.1'
+__version__ = '2015.02.26.2'

From 2e241242a3ee6338cafd515c9cd7481eb5c6f928 Mon Sep 17 00:00:00 2001
From: "PishPosh.McGee" <pishposh.mcgee@gmail.com>
Date: Thu, 26 Feb 2015 03:59:35 -0600
Subject: [PATCH 0543/6219] Adding subtitles

---
 youtube_dl/extractor/comedycentral.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/comedycentral.py b/youtube_dl/extractor/comedycentral.py
index b24538981..e5edcc84b 100644
--- a/youtube_dl/extractor/comedycentral.py
+++ b/youtube_dl/extractor/comedycentral.py
@@ -250,6 +250,8 @@ class ComedyCentralShowsIE(MTVServicesInfoExtractor):
                 })
                 self._sort_formats(formats)
 
+            subtitles = self._extract_subtitles(cdoc, guid)
+
             virtual_id = show_name + ' ' + epTitle + ' part ' + compat_str(part_num + 1)
             entries.append({
                 'id': guid,
@@ -260,6 +262,7 @@ class ComedyCentralShowsIE(MTVServicesInfoExtractor):
                 'duration': duration,
                 'thumbnail': thumbnail,
                 'description': description,
+                'subtitles': subtitles,
             })
 
         return {

From e129c5bc0de4913564c1e1a62baae4bd0073824c Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Thu, 26 Feb 2015 14:35:48 +0200
Subject: [PATCH 0544/6219] [laola1tv] Allow live stream downloads

---
 youtube_dl/extractor/laola1tv.py | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/youtube_dl/extractor/laola1tv.py b/youtube_dl/extractor/laola1tv.py
index e8ca49fd1..b459559b0 100644
--- a/youtube_dl/extractor/laola1tv.py
+++ b/youtube_dl/extractor/laola1tv.py
@@ -27,8 +27,6 @@ class Laola1TvIE(InfoExtractor):
         }
     }
 
-    _BROKEN = True  # Not really - extractor works fine, but f4m downloader does not support live streams yet.
-
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
@@ -57,11 +55,7 @@ class Laola1TvIE(InfoExtractor):
         title = xpath_text(hd_doc, './/video/title', fatal=True)
         flash_url = xpath_text(hd_doc, './/video/url', fatal=True)
         uploader = xpath_text(hd_doc, './/video/meta_organistation')
-
         is_live = xpath_text(hd_doc, './/video/islive') == 'true'
-        if is_live:
-            raise ExtractorError(
-                'Live streams are not supported by the f4m downloader.')
 
         categories = xpath_text(hd_doc, './/video/meta_sports')
         if categories:

From 7ab7c9e93290f575747c1790c4aecf2055227a8d Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Thu, 26 Feb 2015 16:22:05 +0200
Subject: [PATCH 0545/6219] [gamestar] Fix title extraction

---
 youtube_dl/extractor/gamestar.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/gamestar.py b/youtube_dl/extractor/gamestar.py
index 7591a151e..590ccf526 100644
--- a/youtube_dl/extractor/gamestar.py
+++ b/youtube_dl/extractor/gamestar.py
@@ -1,6 +1,8 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..utils import (
     int_or_none,
@@ -31,7 +33,7 @@ class GameStarIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         og_title = self._og_search_title(webpage)
-        title = og_title.replace(' - Video bei GameStar.de', '').strip()
+        title = re.sub(r'\s*- Video (bei|-) GameStar\.de$', '', og_title)
 
         url = 'http://gamestar.de/_misc/videos/portal/getVideoUrl.cfm?premium=0&videoId=' + video_id
 

From 6317a3e9da42f5d60e514fddcb89eec7f5768294 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 26 Feb 2015 21:10:49 +0600
Subject: [PATCH 0546/6219] [mpora] Fix extraction

---
 youtube_dl/extractor/mpora.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mpora.py b/youtube_dl/extractor/mpora.py
index 6db3c67a5..a07d28928 100644
--- a/youtube_dl/extractor/mpora.py
+++ b/youtube_dl/extractor/mpora.py
@@ -25,7 +25,9 @@ class MporaIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         data_json = self._search_regex(
-            r"new FM\.Player\('[^']+',\s*(\{.*?)\).player;", webpage, 'json')
+            [r"new FM\.Player\('[^']+',\s*(\{.*?)\).player;",
+             r"new\s+FM\.Kaltura\.Player\('[^']+'\s*,\s*({.+?})\);"],
+            webpage, 'json')
         data = self._parse_json(data_json, video_id)
 
         uploader = data['info_overlay'].get('username')

From 250a9bdfe27ecf3d0acf179b4c21ff2126c64990 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 26 Feb 2015 21:16:35 +0600
Subject: [PATCH 0547/6219] [mpora] Improve _VALID_URL

---
 youtube_dl/extractor/mpora.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mpora.py b/youtube_dl/extractor/mpora.py
index a07d28928..5a1bee5c8 100644
--- a/youtube_dl/extractor/mpora.py
+++ b/youtube_dl/extractor/mpora.py
@@ -5,7 +5,7 @@ from ..utils import int_or_none
 
 
 class MporaIE(InfoExtractor):
-    _VALID_URL = r'https?://(www\.)?mpora\.(?:com|de)/videos/(?P<id>[^?#/]+)'
+    _VALID_URL = r'https?://(?:www\.)?mpora\.(?:com|de)/videos/(?P<id>[^?#/]+)'
     IE_NAME = 'MPORA'
 
     _TEST = {

From 1a2313a6f2d83c9e0a5c0c25bf5a2e3167994dbe Mon Sep 17 00:00:00 2001
From: anovicecodemonkey <anovicecodemonkey13435@mailinator.com>
Date: Fri, 27 Feb 2015 02:36:45 +1030
Subject: [PATCH 0548/6219] [TheChiveIE] added support for TheChive.com (Closes
 #5016)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/thechive.py | 60 ++++++++++++++++++++++++++++++++
 2 files changed, 61 insertions(+)
 create mode 100644 youtube_dl/extractor/thechive.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index ddb9d6670..b064a3a7c 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -481,6 +481,7 @@ from .tenplay import TenPlayIE
 from .testurl import TestURLIE
 from .testtube import TestTubeIE
 from .tf1 import TF1IE
+from .thechive import TheChiveIE
 from .theonion import TheOnionIE
 from .theplatform import ThePlatformIE
 from .thesixtyone import TheSixtyOneIE
diff --git a/youtube_dl/extractor/thechive.py b/youtube_dl/extractor/thechive.py
new file mode 100644
index 000000000..df1a7998d
--- /dev/null
+++ b/youtube_dl/extractor/thechive.py
@@ -0,0 +1,60 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import unified_strdate
+
+
+
+class TheChiveIE(InfoExtractor):
+    _VALID_URL = r'http://(www\.)?thechive\.com/[^/]+/[^/]+/[^/]+/(?P<video_id>[A-Za-z\-]+)'
+    _TEST = {
+        'url': "http://thechive.com/2015/02/20/so-thats-what-a-set-of-redneck-bagpipes-sound-like-video/",
+        'md5': "366710dda77cfa727bdef3523ba8466f",
+        'info_dict': {
+            'id': "so-thats-what-a-set-of-redneck-bagpipes-sound-like-video",
+            'title': "So that's what a set of redneck bagpipes sound like... (Video)",
+            'description': "Okay that was pretty good. Now play Freebird!...",
+            'thumbnail': "https://thechive.files.wordpress.com/2015/02/0_07dghz0w-thumbnail2.jpg",
+            'author': "Ben",
+            'upload_date': "20150220",
+            'ext': "mp4"
+        }
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('video_id')
+        webpage = self._download_webpage(url, video_id)
+
+        title = self._og_search_title(webpage)
+        description = self._html_search_regex(r'(?s)<meta name="description" content="(.*?)" />', webpage, 'description')
+        thumbnail = self._og_search_thumbnail(webpage)
+        author = self._html_search_regex(
+            r'(?s)itemprop="author">(.+?)</span>', webpage, 'author', fatal=False).capitalize() 
+        upload_date = unified_strdate(self._html_search_regex(
+            r'(?s)itemprop="datePublished" datetime="(.+?)">', webpage, 'upload_date', fatal=False))
+
+        # Adapted from extractor/musicvault.py
+        VIDEO_URL_TEMPLATE = 'http://cdnapi.kaltura.com/p/%(uid)s/sp/%(wid)s/playManifest/entryId/%(entry_id)s/format/url/protocol/http'
+
+        kaltura_id = self._search_regex(
+            r'entry_id=([^"]+)',
+            webpage, 'kaltura ID')
+        video_url = VIDEO_URL_TEMPLATE % {
+            'entry_id': kaltura_id,
+            'wid': self._search_regex(r'partner_id/([0-9]+)\?', webpage, 'wid'),
+            'uid': self._search_regex(r'uiconf_id/([0-9]+)/', webpage, 'uid'),
+        }
+
+        return {
+            'url': video_url,
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'author': author,
+            'upload_date': upload_date,
+            'ext': 'mp4'
+        }
\ No newline at end of file

From 0d97ef43bec006157870fd4a5cedfac1eaebf3a9 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Thu, 26 Feb 2015 23:45:54 +0200
Subject: [PATCH 0549/6219] [kaltura] Add new extractor

---
 youtube_dl/extractor/__init__.py |   1 +
 youtube_dl/extractor/kaltura.py  | 134 +++++++++++++++++++++++++++++++
 2 files changed, 135 insertions(+)
 create mode 100644 youtube_dl/extractor/kaltura.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index ddb9d6670..e3b2cb54f 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -227,6 +227,7 @@ from .jeuxvideo import JeuxVideoIE
 from .jove import JoveIE
 from .jukebox import JukeboxIE
 from .jpopsukitv import JpopsukiIE
+from .kaltura import KalturaIE
 from .kankan import KankanIE
 from .karaoketv import KaraoketvIE
 from .keezmovies import KeezMoviesIE
diff --git a/youtube_dl/extractor/kaltura.py b/youtube_dl/extractor/kaltura.py
new file mode 100644
index 000000000..2aff410c5
--- /dev/null
+++ b/youtube_dl/extractor/kaltura.py
@@ -0,0 +1,134 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..compat import compat_urllib_parse
+from ..utils import (
+    ExtractorError,
+    int_or_none,
+)
+
+
+class KalturaIE(InfoExtractor):
+    _VALID_URL = r'''(?x)
+    (?:kaltura:|
+       https?://(:?www\.)?kaltura\.com/index\.php/kwidget/(?:[^/]+/)*?wid/_
+    )(?P<partner_id>\d+)
+    (?::|
+       /(?:[^/]+/)*?entry_id/
+    )(?P<id>[0-9a-z_]+)'''
+    _API_BASE = 'http://cdnapi.kaltura.com/api_v3/index.php?'
+    _TESTS = [
+        {
+            'url': 'kaltura:269692:1_1jc2y3e4',
+            'md5': '3adcbdb3dcc02d647539e53f284ba171',
+            'info_dict': {
+                'id': '1_1jc2y3e4',
+                'ext': 'mp4',
+                'title': 'Track 4',
+                'upload_date': '20131219',
+                'uploader_id': 'mlundberg@wolfgangsvault.com',
+                'description': 'The Allman Brothers Band, 12/16/1981',
+                'thumbnail': 're:^https?://.*/thumbnail/.*',
+                'timestamp': int,
+            },
+        },
+        {
+            'url': 'http://www.kaltura.com/index.php/kwidget/cache_st/1300318621/wid/_269692/uiconf_id/3873291/entry_id/1_1jc2y3e4',
+            'only_matching': True,
+        },
+    ]
+
+    def _kaltura_api_call(self, video_id, actions, *args, **kwargs):
+        params = actions[0]
+        if len(actions) > 1:
+            for i, a in enumerate(actions[1:], start=1):
+                for k, v in a.items():
+                    params['%d:%s' % (i, k)] = v
+
+        query = compat_urllib_parse.urlencode(params)
+        url = self._API_BASE + query
+        data = self._download_json(url, video_id, *args, **kwargs)
+
+        status = data if len(actions) == 1 else data[0]
+        if status.get('objectType') == 'KalturaAPIException':
+            raise ExtractorError(
+                '%s said: %s' % (self.IE_NAME, status['message']))
+
+        return data
+
+    def _get_kaltura_signature(self, video_id, partner_id):
+        actions = [{
+            'apiVersion': '3.1',
+            'expiry': 86400,
+            'format': 1,
+            'service': 'session',
+            'action': 'startWidgetSession',
+            'widgetId': '_%s' % partner_id,
+        }]
+        return self._kaltura_api_call(
+            video_id, actions, note='Downloading Kaltura signature')['ks']
+
+    def _get_video_info(self, video_id, partner_id):
+        signature = self._get_kaltura_signature(video_id, partner_id)
+        actions = [
+            {
+                'action': 'null',
+                'apiVersion': '3.1.5',
+                'clientTag': 'kdp:v3.8.5',
+                'format': 1,  # JSON, 2 = XML, 3 = PHP
+                'service': 'multirequest',
+                'ks': signature,
+            },
+            {
+                'action': 'get',
+                'entryId': video_id,
+                'service': 'baseentry',
+                'version': '-1',
+            },
+            {
+                'action': 'getContextData',
+                'contextDataParams:objectType': 'KalturaEntryContextDataParams',
+                'contextDataParams:referrer': 'http://www.kaltura.com/',
+                'contextDataParams:streamerType': 'http',
+                'entryId': video_id,
+                'service': 'baseentry',
+            },
+        ]
+        return self._kaltura_api_call(
+            video_id, actions, note='Downloading video info JSON')
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        mobj = re.match(self._VALID_URL, url)
+        partner_id, entry_id = mobj.group('partner_id'), mobj.group('id')
+
+        info, source_data = self._get_video_info(entry_id, partner_id)
+
+        formats = [{
+            'format_id': '%(fileExt)s-%(bitrate)s' % f,
+            'ext': f['fileExt'],
+            'tbr': f['bitrate'],
+            'fps': f.get('frameRate'),
+            'filesize_approx': int_or_none(f.get('size'), invscale=1024),
+            'container': f.get('containerFormat'),
+            'vcodec': f.get('videoCodecId'),
+            'height': f.get('height'),
+            'width': f.get('width'),
+            'url': '%s/flavorId/%s' % (info['dataUrl'], f['id']),
+        } for f in source_data['flavorAssets']]
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': info['name'],
+            'formats': formats,
+            'description': info.get('description'),
+            'thumbnail': info.get('thumbnailUrl'),
+            'duration': info.get('duration'),
+            'timestamp': info.get('createdAt'),
+            'uploader_id': info.get('userId'),
+            'view_count': info.get('plays'),
+        }

From da419e23320aef560c05f543c5ee375e8956c95a Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Thu, 26 Feb 2015 23:47:45 +0200
Subject: [PATCH 0550/6219] [musicvault] Use the Kaltura extractor

---
 youtube_dl/extractor/musicvault.py | 27 +++++++--------------------
 1 file changed, 7 insertions(+), 20 deletions(-)

diff --git a/youtube_dl/extractor/musicvault.py b/youtube_dl/extractor/musicvault.py
index ebb1eb8e9..0e46ac7c1 100644
--- a/youtube_dl/extractor/musicvault.py
+++ b/youtube_dl/extractor/musicvault.py
@@ -3,17 +3,13 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import (
-    parse_duration,
-    unified_strdate,
-)
 
 
 class MusicVaultIE(InfoExtractor):
     _VALID_URL = r'https?://www\.musicvault\.com/(?P<uploader_id>[^/?#]*)/video/(?P<display_id>[^/?#]*)_(?P<id>[0-9]+)\.html'
     _TEST = {
         'url': 'http://www.musicvault.com/the-allman-brothers-band/video/straight-from-the-heart_1010863.html',
-        'md5': '2cdbb3ae75f7fb3519821507d2fb3c15',
+        'md5': '3adcbdb3dcc02d647539e53f284ba171',
         'info_dict': {
             'id': '1010863',
             'ext': 'mp4',
@@ -22,9 +18,10 @@ class MusicVaultIE(InfoExtractor):
             'duration': 244,
             'uploader': 'The Allman Brothers Band',
             'thumbnail': 're:^https?://.*/thumbnail/.*',
-            'upload_date': '19811216',
+            'upload_date': '20131219',
             'location': 'Capitol Theatre (Passaic, NJ)',
             'description': 'Listen to The Allman Brothers Band perform Straight from the Heart at Capitol Theatre (Passaic, NJ) on Dec 16, 1981',
+            'timestamp': int,
         }
     }
 
@@ -43,34 +40,24 @@ class MusicVaultIE(InfoExtractor):
             r'<h1.*?>(.*?)</h1>', data_div, 'uploader', fatal=False)
         title = self._html_search_regex(
             r'<h2.*?>(.*?)</h2>', data_div, 'title')
-        upload_date = unified_strdate(self._html_search_regex(
-            r'<h3.*?>(.*?)</h3>', data_div, 'uploader', fatal=False))
         location = self._html_search_regex(
             r'<h4.*?>(.*?)</h4>', data_div, 'location', fatal=False)
 
-        duration = parse_duration(self._html_search_meta('duration', webpage))
-
-        VIDEO_URL_TEMPLATE = 'http://cdnapi.kaltura.com/p/%(uid)s/sp/%(wid)s/playManifest/entryId/%(entry_id)s/format/url/protocol/http'
         kaltura_id = self._search_regex(
             r'<div id="video-detail-player" data-kaltura-id="([^"]+)"',
             webpage, 'kaltura ID')
-        video_url = VIDEO_URL_TEMPLATE % {
-            'entry_id': kaltura_id,
-            'wid': self._search_regex(r'/wid/_([0-9]+)/', webpage, 'wid'),
-            'uid': self._search_regex(r'uiconf_id/([0-9]+)/', webpage, 'uid'),
-        }
+        wid = self._search_regex(r'/wid/_([0-9]+)/', webpage, 'wid')
 
         return {
             'id': mobj.group('id'),
-            'url': video_url,
-            'ext': 'mp4',
+            '_type': 'url_transparent',
+            'url': 'kaltura:%s:%s' % (wid, kaltura_id),
+            'ie_key': 'Kaltura',
             'display_id': display_id,
             'uploader_id': mobj.group('uploader_id'),
             'thumbnail': thumbnail,
             'description': self._html_search_meta('description', webpage),
-            'upload_date': upload_date,
             'location': location,
             'title': title,
             'uploader': uploader,
-            'duration': duration,
         }

From e3216b82bf6ef54db63984f7fece4e95fbc3b981 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Fri, 27 Feb 2015 00:34:19 +0200
Subject: [PATCH 0551/6219] [generic] Support dynamic Kaltura embeds (#5016)
 (#5073)

---
 youtube_dl/extractor/generic.py | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 3aff57e30..27e2bc300 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -557,6 +557,18 @@ class GenericIE(InfoExtractor):
                 'title': 'EP3S5 - Bon Appétit - Baqueira Mi Corazon !',
             }
         },
+        # Kaltura embed
+        {
+            'url': 'http://www.monumentalnetwork.com/videos/john-carlson-postgame-2-25-15',
+            'info_dict': {
+                'id': '1_eergr3h1',
+                'ext': 'mp4',
+                'upload_date': '20150226',
+                'uploader_id': 'MonumentalSports-Kaltura@perfectsensedigital.com',
+                'timestamp': int,
+                'title': 'John Carlson Postgame 2/25/15',
+            },
+        },
     ]
 
     def report_following_redirect(self, new_url):
@@ -1113,6 +1125,12 @@ class GenericIE(InfoExtractor):
         if mobj is not None:
             return self.url_result(mobj.group('url'), 'Zapiks')
 
+        # Look for Kaltura embeds
+        mobj = re.search(
+            r"(?s)kWidget\.(?:thumb)?[Ee]mbed\(\{.*?'wid'\s*:\s*'_?(?P<partner_id>[^']+)',.*?'entry_id'\s*:\s*'(?P<id>[^']+)',", webpage)
+        if mobj is not None:
+            return self.url_result('kaltura:%(partner_id)s:%(id)s' % mobj.groupdict(), 'Kaltura')
+
         def check_video(vurl):
             if YoutubeIE.suitable(vurl):
                 return True

From 0eba1e178230a88d1b316f54edbb671d216c1d02 Mon Sep 17 00:00:00 2001
From: Sergey <Ftornik@users.noreply.github.com>
Date: Fri, 27 Feb 2015 00:51:22 +0200
Subject: [PATCH 0552/6219] [lynda] Fixed subtitles broken file

---
 youtube_dl/extractor/lynda.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index 109055e72..8bb21ee1d 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -152,7 +152,7 @@ class LyndaIE(InfoExtractor):
                 continue
             appear_time = m_current.group('timecode')
             disappear_time = m_next.group('timecode')
-            text = seq_current['Caption']
+            text = seq_current['Caption'].strip()
             srt += '%s\r\n%s --> %s\r\n%s' % (str(pos), appear_time, disappear_time, text)
         if srt:
             return srt

From f3bff94cf96587462baa8da6d7e5f17801381753 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 27 Feb 2015 12:24:51 +0100
Subject: [PATCH 0553/6219] [rtve] Extract duration

---
 youtube_dl/extractor/rtve.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/rtve.py b/youtube_dl/extractor/rtve.py
index c0fd23ff1..b42442d12 100644
--- a/youtube_dl/extractor/rtve.py
+++ b/youtube_dl/extractor/rtve.py
@@ -8,8 +8,9 @@ import time
 from .common import InfoExtractor
 from ..compat import compat_urlparse
 from ..utils import (
-    struct_unpack,
+    float_or_none,
     remove_end,
+    struct_unpack,
 )
 
 
@@ -67,6 +68,7 @@ class RTVEALaCartaIE(InfoExtractor):
             'id': '2491869',
             'ext': 'mp4',
             'title': 'Balonmano - Swiss Cup masculina. Final: España-Suecia',
+            'duration': 5024.566,
         },
     }, {
         'note': 'Live stream',
@@ -113,6 +115,7 @@ class RTVEALaCartaIE(InfoExtractor):
             'thumbnail': info.get('image'),
             'page_url': url,
             'subtitles': subtitles,
+            'duration': float_or_none(info.get('duration'), scale=1000),
         }
 
     def _get_subtitles(self, video_id, sub_file):

From 7862ad88b725daae957ad27ae60993e360c01e13 Mon Sep 17 00:00:00 2001
From: HanYOLO <HanYOLO@users.noreply.github.com>
Date: Fri, 27 Feb 2015 15:41:58 +0100
Subject: [PATCH 0554/6219] puls4 Add new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/puls4.py    | 61 ++++++++++++++++++++++++++++++++
 2 files changed, 62 insertions(+)
 create mode 100644 youtube_dl/extractor/puls4.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index e3b2cb54f..d137e1104 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -373,6 +373,7 @@ from .pornotube import PornotubeIE
 from .pornoxo import PornoXOIE
 from .promptfile import PromptFileIE
 from .prosiebensat1 import ProSiebenSat1IE
+from .puls4 import Puls4IE
 from .pyvideo import PyvideoIE
 from .quickvid import QuickVidIE
 from .r7 import R7IE
diff --git a/youtube_dl/extractor/puls4.py b/youtube_dl/extractor/puls4.py
new file mode 100644
index 000000000..70dedbff3
--- /dev/null
+++ b/youtube_dl/extractor/puls4.py
@@ -0,0 +1,61 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+import re
+
+
+class Puls4IE(InfoExtractor):
+
+    _VALID_URL = r'https?://www.puls4.com/video/.+?/play/(?P<id>[0-9]+)'
+    _TESTS = [{
+        'url': 'http://www.puls4.com/video/pro-und-contra/play/2716816',
+        'md5': '49f6a6629747eeec43cef6a46b5df81d',
+        'info_dict': {
+            'id': '2716816',
+            'ext': 'mp4',
+            'title': 'Pro und Contra vom 23.02.2015'}},
+        {
+        'url': 'http://www.puls4.com/video/kult-spielfilme/play/1298106',
+        'md5': '6a48316c8903ece8dab9b9a7bf7a59ec',
+        'info_dict': {
+            'id': '1298106',
+            'ext': 'mp4',
+            'title': 'Lucky Fritz'}}
+    ]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        # if fsk-button
+        real_url = self._html_search_regex(r'\"fsk-button\".+?href=\"([^"]+)',
+                                           webpage, 'fsk_button', default=None)
+        if real_url:
+            webpage = self._download_webpage(real_url, video_id)
+
+        title = self._html_search_regex(
+            r'<div id="bg_brandableContent">.+?<h1>(.+?)</h1>',
+            webpage, 'title', flags=re.DOTALL)
+
+        sd_url = self._html_search_regex(
+            r'{\"url\":\"([^"]+?)\",\"hd\":false',
+            webpage, 'sd_url').replace('\\', '')
+
+        formats = [{'format_id': 'sd', 'url': sd_url, 'quality': -2}]
+
+        hd_url = self._html_search_regex(
+            r'{\"url\":\"([^"]+?)\",\"hd\":true',
+            webpage, 'hd_url', default=None)
+        if hd_url:
+            hd_url = hd_url.replace('\\', '')
+            formats.append({'format_id': 'hd', 'url': hd_url, 'quality': -1})
+
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': formats,
+            'ext': 'mp4'
+        }

From a0d646135aaf417e0aa000419974c676335d164a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 27 Feb 2015 20:56:06 +0600
Subject: [PATCH 0555/6219] [lynda] Extend _VALID_URL

---
 youtube_dl/extractor/lynda.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index 109055e72..e7e9d80de 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -18,7 +18,7 @@ from ..utils import (
 class LyndaIE(InfoExtractor):
     IE_NAME = 'lynda'
     IE_DESC = 'lynda.com videos'
-    _VALID_URL = r'https?://www\.lynda\.com/[^/]+/[^/]+/\d+/(\d+)-\d\.html'
+    _VALID_URL = r'https?://www\.lynda\.com/(?:[^/]+/[^/]+/\d+|player/embed)/(\d+)'
     _LOGIN_URL = 'https://www.lynda.com/login/login.aspx'
     _NETRC_MACHINE = 'lynda'
 
@@ -27,7 +27,7 @@ class LyndaIE(InfoExtractor):
 
     ACCOUNT_CREDENTIALS_HINT = 'Use --username and --password options to provide lynda.com account credentials.'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.lynda.com/Bootstrap-tutorials/Using-exercise-files/110885/114408-4.html',
         'md5': 'ecfc6862da89489161fb9cd5f5a6fac1',
         'info_dict': {
@@ -36,7 +36,10 @@ class LyndaIE(InfoExtractor):
             'title': 'Using the exercise files',
             'duration': 68
         }
-    }
+    }, {
+        'url': 'https://www.lynda.com/player/embed/133770?tr=foo=1;bar=g;fizz=rt&fs=0',
+        'only_matching': True,
+    }]
 
     def _real_initialize(self):
         self._login()

From 781a7ef60ab03cf1e646a80ecf4cbd70f1532db6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 27 Feb 2015 16:18:18 +0100
Subject: [PATCH 0556/6219] [lynda] Use 'lstrip' for the subtitles

The newlines at the end are important, they separate each piece of text.
---
 youtube_dl/extractor/lynda.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index bfd9b73d2..5dc22da22 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -155,7 +155,7 @@ class LyndaIE(InfoExtractor):
                 continue
             appear_time = m_current.group('timecode')
             disappear_time = m_next.group('timecode')
-            text = seq_current['Caption'].strip()
+            text = seq_current['Caption'].lstrip()
             srt += '%s\r\n%s --> %s\r\n%s' % (str(pos), appear_time, disappear_time, text)
         if srt:
             return srt

From 4ffbf77886b1c6c3f8d767223ca088395d08ab14 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 28 Feb 2015 00:15:03 +0600
Subject: [PATCH 0557/6219] [odnoklassniki] Add extractor (Closes #5075)

---
 youtube_dl/extractor/__init__.py      |  1 +
 youtube_dl/extractor/odnoklassniki.py | 85 +++++++++++++++++++++++++++
 2 files changed, 86 insertions(+)
 create mode 100644 youtube_dl/extractor/odnoklassniki.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index e3b2cb54f..aecb67bf4 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -346,6 +346,7 @@ from .ntvde import NTVDeIE
 from .ntvru import NTVRuIE
 from .nytimes import NYTimesIE
 from .nuvid import NuvidIE
+from .odnoklassniki import OdnoklassnikiIE
 from .oktoberfesttv import OktoberfestTVIE
 from .ooyala import OoyalaIE
 from .openfilm import OpenFilmIE
diff --git a/youtube_dl/extractor/odnoklassniki.py b/youtube_dl/extractor/odnoklassniki.py
new file mode 100644
index 000000000..155d0ee6a
--- /dev/null
+++ b/youtube_dl/extractor/odnoklassniki.py
@@ -0,0 +1,85 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    unified_strdate,
+    int_or_none,
+    qualities,
+)
+
+
+class OdnoklassnikiIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:odnoklassniki|ok)\.ru/(?:video|web-api/video/moviePlayer)/(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'http://ok.ru/video/20079905452',
+        'md5': '8e24ad2da6f387948e7a7d44eb8668fe',
+        'info_dict': {
+            'id': '20079905452',
+            'ext': 'mp4',
+            'title': 'Культура меняет нас (прекрасный ролик!))',
+            'duration': 100,
+            'upload_date': '20141207',
+            'uploader_id': '330537914540',
+            'uploader': 'Виталий Добровольский',
+            'like_count': int,
+            'age_limit': 0,
+        },
+    }, {
+        'url': 'http://ok.ru/web-api/video/moviePlayer/20079905452',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        player = self._parse_json(
+            self._search_regex(
+                r"OKVideo\.start\(({.+?})\s*,\s*'VideoAutoplay_player'", webpage, 'player'),
+            video_id)
+
+        metadata = self._parse_json(player['flashvars']['metadata'], video_id)
+
+        movie = metadata['movie']
+        title = movie['title']
+        thumbnail = movie.get('poster')
+        duration = int_or_none(movie.get('duration'))
+
+        author = metadata.get('author', {})
+        uploader_id = author.get('id')
+        uploader = author.get('name')
+
+        upload_date = unified_strdate(self._html_search_meta(
+            'ya:ovs:upload_date', webpage, 'upload date'))
+
+        age_limit = None
+        adult = self._html_search_meta(
+            'ya:ovs:adult', webpage, 'age limit')
+        if adult:
+            age_limit = 18 if adult == 'true' else 0
+
+        like_count = int_or_none(metadata.get('likeCount'))
+
+        quality = qualities(('mobile', 'lowest', 'low', 'sd', 'hd'))
+
+        formats = [{
+            'url': f['url'],
+            'ext': 'mp4',
+            'format_id': f['name'],
+            'quality': quality(f['name']),
+        } for f in metadata['videos']]
+
+        return {
+            'id': video_id,
+            'title': title,
+            'thumbnail': thumbnail,
+            'duration': duration,
+            'upload_date': upload_date,
+            'uploader': uploader,
+            'uploader_id': uploader_id,
+            'like_count': like_count,
+            'age_limit': age_limit,
+            'formats': formats,
+        }

From bd3749ed693ae96becd3832f20e765e1efe01476 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 28 Feb 2015 00:19:31 +0600
Subject: [PATCH 0558/6219] [kaltura] Extend _VALID_URL (Closes #5081)

---
 youtube_dl/extractor/kaltura.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/kaltura.py b/youtube_dl/extractor/kaltura.py
index 2aff410c5..d28730492 100644
--- a/youtube_dl/extractor/kaltura.py
+++ b/youtube_dl/extractor/kaltura.py
@@ -14,7 +14,7 @@ from ..utils import (
 class KalturaIE(InfoExtractor):
     _VALID_URL = r'''(?x)
     (?:kaltura:|
-       https?://(:?www\.)?kaltura\.com/index\.php/kwidget/(?:[^/]+/)*?wid/_
+       https?://(:?(?:www|cdnapisec)\.)?kaltura\.com/index\.php/kwidget/(?:[^/]+/)*?wid/_
     )(?P<partner_id>\d+)
     (?::|
        /(?:[^/]+/)*?entry_id/
@@ -39,6 +39,10 @@ class KalturaIE(InfoExtractor):
             'url': 'http://www.kaltura.com/index.php/kwidget/cache_st/1300318621/wid/_269692/uiconf_id/3873291/entry_id/1_1jc2y3e4',
             'only_matching': True,
         },
+        {
+            'url': 'https://cdnapisec.kaltura.com/index.php/kwidget/wid/_557781/uiconf_id/22845202/entry_id/1_plr1syf3',
+            'only_matching': True,
+        },
     ]
 
     def _kaltura_api_call(self, video_id, actions, *args, **kwargs):

From 40b077bc7ee2722463f23974ce488acb7f11815a Mon Sep 17 00:00:00 2001
From: thc202 <thc202@gmail.com>
Date: Fri, 27 Feb 2015 22:27:30 +0000
Subject: [PATCH 0559/6219] [oppetarkiv] Add new extractor

Some, if not all, of the videos appear to be geo-blocked (Sweden).
Test might fail (403 Forbidden) if not run through a Swedish connection.
---
 youtube_dl/extractor/__init__.py   |  1 +
 youtube_dl/extractor/oppetarkiv.py | 56 ++++++++++++++++++++++++++++++
 2 files changed, 57 insertions(+)
 create mode 100644 youtube_dl/extractor/oppetarkiv.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index aecb67bf4..1544f1059 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -350,6 +350,7 @@ from .odnoklassniki import OdnoklassnikiIE
 from .oktoberfesttv import OktoberfestTVIE
 from .ooyala import OoyalaIE
 from .openfilm import OpenFilmIE
+from .oppetarkiv import OppetArkivIE
 from .orf import (
     ORFTVthekIE,
     ORFOE1IE,
diff --git a/youtube_dl/extractor/oppetarkiv.py b/youtube_dl/extractor/oppetarkiv.py
new file mode 100644
index 000000000..6dd1fad3f
--- /dev/null
+++ b/youtube_dl/extractor/oppetarkiv.py
@@ -0,0 +1,56 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    determine_ext,
+)
+
+
+class OppetArkivIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?oppetarkiv.se/video/(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'http://www.oppetarkiv.se/video/1058509/rederiet-sasong-1-avsnitt-1-av-318',
+        'md5': '7b95ca9bedeead63012b2d7c3992c28f',
+        'info_dict': {
+            'id': '1058509',
+            'ext': 'mp4',
+            'title': 'Farlig kryssning',
+            'duration': 2566,
+            'thumbnail': 're:^https?://.*[\.-]jpg$',
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        info = self._download_json(
+            'http://www.oppetarkiv.se/video/%s?output=json' % video_id, video_id)
+
+        title = info['context']['title']
+        thumbnail = info['context'].get('thumbnailImage')
+
+        video_info = info['video']
+        formats = []
+        for vr in video_info['videoReferences']:
+            vurl = vr['url']
+            if determine_ext(vurl) == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    vurl, video_id,
+                    ext='mp4', entry_protocol='m3u8_native',
+                    m3u8_id=vr.get('playerType')))
+            else:
+                formats.append({
+                    'format_id': vr.get('playerType'),
+                    'url': vurl,
+                })
+        self._sort_formats(formats)
+
+        duration = video_info.get('materialLength')
+
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': formats,
+            'thumbnail': thumbnail,
+            'duration': duration,
+        }

From 0f2c0d335b14ba1596e6608db7a6f29a0d9e1c86 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 28 Feb 2015 14:03:27 +0100
Subject: [PATCH 0560/6219] [YoutubeDL] Use the InfoExtractor._download_webpage
 method for getting the subtitles

It handles encodings better, for example for 'http://www.npo.nl/nos-journaal/14-02-2015/POW_00942207'
---
 youtube_dl/YoutubeDL.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 76fc394bc..74e426168 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1300,17 +1300,18 @@ class YoutubeDL(object):
             # subtitles download errors are already managed as troubles in relevant IE
             # that way it will silently go on when used with unsupporting IE
             subtitles = info_dict['requested_subtitles']
+            ie = self.get_info_extractor(info_dict['extractor_key'])
             for sub_lang, sub_info in subtitles.items():
                 sub_format = sub_info['ext']
                 if sub_info.get('data') is not None:
                     sub_data = sub_info['data']
                 else:
                     try:
-                        uf = self.urlopen(sub_info['url'])
-                        sub_data = uf.read().decode('utf-8')
-                    except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
+                        sub_data = ie._download_webpage(
+                            sub_info['url'], info_dict['id'], note=False)
+                    except ExtractorError as err:
                         self.report_warning('Unable to download subtitle for "%s": %s' %
-                                            (sub_lang, compat_str(err)))
+                                            (sub_lang, compat_str(err.cause)))
                         continue
                 try:
                     sub_filename = subtitles_filename(filename, sub_lang, sub_format)

From e9fade72f347cbcce779ff176d516467a425b43a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 28 Feb 2015 14:43:24 +0100
Subject: [PATCH 0561/6219] Add postprocessor for converting subtitles (closes
 #4954)

---
 youtube_dl/__init__.py               |  8 ++++++
 youtube_dl/options.py                |  4 +++
 youtube_dl/postprocessor/__init__.py |  2 ++
 youtube_dl/postprocessor/ffmpeg.py   | 38 ++++++++++++++++++++++++++++
 4 files changed, 52 insertions(+)

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 5ce201800..49f382695 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -170,6 +170,9 @@ def _real_main(argv=None):
     if opts.recodevideo is not None:
         if opts.recodevideo not in ['mp4', 'flv', 'webm', 'ogg', 'mkv']:
             parser.error('invalid video recode format specified')
+    if opts.convertsubtitles is not None:
+        if opts.convertsubtitles not in ['srt', 'vtt', 'ass']:
+            parser.error('invalid subtitle format specified')
 
     if opts.date is not None:
         date = DateRange.day(opts.date)
@@ -223,6 +226,11 @@ def _real_main(argv=None):
             'key': 'FFmpegVideoConvertor',
             'preferedformat': opts.recodevideo,
         })
+    if opts.convertsubtitles:
+        postprocessors.append({
+            'key': 'FFmpegSubtitlesConvertor',
+            'format': opts.convertsubtitles,
+        })
     if opts.embedsubtitles:
         postprocessors.append({
             'key': 'FFmpegEmbedSubtitle',
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 886ce9613..58f811162 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -751,6 +751,10 @@ def parseOpts(overrideArguments=None):
         '--exec',
         metavar='CMD', dest='exec_cmd',
         help='Execute a command on the file after downloading, similar to find\'s -exec syntax. Example: --exec \'adb push {} /sdcard/Music/ && rm {}\'')
+    postproc.add_option(
+        '--convert-subtitles', '--convert-subs',
+        metavar='FORMAT', dest='convertsubtitles', default=None,
+        help='Convert the subtitles to other format (currently supported: srt|ass|vtt)')
 
     parser.add_option_group(general)
     parser.add_option_group(network)
diff --git a/youtube_dl/postprocessor/__init__.py b/youtube_dl/postprocessor/__init__.py
index 0ffbca258..708df3dd4 100644
--- a/youtube_dl/postprocessor/__init__.py
+++ b/youtube_dl/postprocessor/__init__.py
@@ -11,6 +11,7 @@ from .ffmpeg import (
     FFmpegMergerPP,
     FFmpegMetadataPP,
     FFmpegVideoConvertorPP,
+    FFmpegSubtitlesConvertorPP,
 )
 from .xattrpp import XAttrMetadataPP
 from .execafterdownload import ExecAfterDownloadPP
@@ -31,6 +32,7 @@ __all__ = [
     'FFmpegMergerPP',
     'FFmpegMetadataPP',
     'FFmpegPostProcessor',
+    'FFmpegSubtitlesConvertorPP',
     'FFmpegVideoConvertorPP',
     'XAttrMetadataPP',
 ]
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 398fe050e..30094c2f3 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -1,5 +1,6 @@
 from __future__ import unicode_literals
 
+import io
 import os
 import subprocess
 import sys
@@ -635,3 +636,40 @@ class FFmpegFixupM4aPP(FFmpegPostProcessor):
         os.rename(encodeFilename(temp_filename), encodeFilename(filename))
 
         return True, info
+
+
+class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor):
+    def __init__(self, downloader=None, format=None):
+        super(FFmpegSubtitlesConvertorPP, self).__init__(downloader)
+        self.format = format
+
+    def run(self, info):
+        subs = info.get('requested_subtitles')
+        filename = info['filepath']
+        new_ext = self.format
+        new_format = new_ext
+        if new_format == 'vtt':
+            new_format = 'webvtt'
+        if subs is None:
+            self._downloader.to_screen('[ffmpeg] There aren\'t any subtitles to convert')
+            return True, info
+        self._downloader.to_screen('[ffmpeg] Converting subtitles')
+        for lang, sub in subs.items():
+            ext = sub['ext']
+            if ext == new_ext:
+                self._downloader.to_screen(
+                    '[ffmpeg] Subtitle file for %s is already in the requested'
+                    'format' % new_ext)
+                continue
+            new_file = subtitles_filename(filename, lang, new_ext)
+            self.run_ffmpeg(
+                subtitles_filename(filename, lang, ext),
+                new_file, ['-f', new_format])
+
+            with io.open(new_file, 'rt', encoding='utf-8') as f:
+                subs[lang] = {
+                    'ext': ext,
+                    'data': f.read(),
+                }
+
+        return True, info

From e143f5dae9c767529b8b522a9df63ac0ee8fc356 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 28 Feb 2015 21:12:06 +0600
Subject: [PATCH 0562/6219] [oppetarkiv] Extract f4m formats and age limit

---
 youtube_dl/extractor/oppetarkiv.py | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/oppetarkiv.py b/youtube_dl/extractor/oppetarkiv.py
index 6dd1fad3f..ae6a28308 100644
--- a/youtube_dl/extractor/oppetarkiv.py
+++ b/youtube_dl/extractor/oppetarkiv.py
@@ -8,17 +8,19 @@ from ..utils import (
 
 
 class OppetArkivIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?oppetarkiv.se/video/(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?oppetarkiv\.se/video/(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://www.oppetarkiv.se/video/1058509/rederiet-sasong-1-avsnitt-1-av-318',
-        'md5': '7b95ca9bedeead63012b2d7c3992c28f',
+        'md5': '5c1eb616e59f733d4af77edc5177d2fe',
         'info_dict': {
             'id': '1058509',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': 'Farlig kryssning',
             'duration': 2566,
             'thumbnail': 're:^https?://.*[\.-]jpg$',
+            'age_limit': 0,
         },
+        'skip': 'Only works from Sweden',
     }
 
     def _real_extract(self, url):
@@ -33,11 +35,16 @@ class OppetArkivIE(InfoExtractor):
         formats = []
         for vr in video_info['videoReferences']:
             vurl = vr['url']
-            if determine_ext(vurl) == 'm3u8':
+            ext = determine_ext(vurl)
+            if ext == 'm3u8':
                 formats.extend(self._extract_m3u8_formats(
                     vurl, video_id,
                     ext='mp4', entry_protocol='m3u8_native',
                     m3u8_id=vr.get('playerType')))
+            elif ext == 'f4m':
+                formats.extend(self._extract_f4m_formats(
+                    vurl + '?hdcore=3.3.0', video_id,
+                    f4m_id=vr.get('playerType')))
             else:
                 formats.append({
                     'format_id': vr.get('playerType'),
@@ -47,10 +54,13 @@ class OppetArkivIE(InfoExtractor):
 
         duration = video_info.get('materialLength')
 
+        age_limit = 18 if video_info.get('inappropriateForChildren') else 0
+
         return {
             'id': video_id,
             'title': title,
             'formats': formats,
             'thumbnail': thumbnail,
             'duration': duration,
+            'age_limit': age_limit,
         }

From df5ae3eb16effec4dc2609ed3520ddc068305efe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 28 Feb 2015 21:25:04 +0600
Subject: [PATCH 0563/6219] [oppetarkiv] Merge with svtplay

---
 youtube_dl/extractor/__init__.py   |  1 -
 youtube_dl/extractor/oppetarkiv.py | 66 ------------------------------
 youtube_dl/extractor/svtplay.py    | 42 +++++++++++++++----
 3 files changed, 34 insertions(+), 75 deletions(-)
 delete mode 100644 youtube_dl/extractor/oppetarkiv.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 1544f1059..aecb67bf4 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -350,7 +350,6 @@ from .odnoklassniki import OdnoklassnikiIE
 from .oktoberfesttv import OktoberfestTVIE
 from .ooyala import OoyalaIE
 from .openfilm import OpenFilmIE
-from .oppetarkiv import OppetArkivIE
 from .orf import (
     ORFTVthekIE,
     ORFOE1IE,
diff --git a/youtube_dl/extractor/oppetarkiv.py b/youtube_dl/extractor/oppetarkiv.py
deleted file mode 100644
index ae6a28308..000000000
--- a/youtube_dl/extractor/oppetarkiv.py
+++ /dev/null
@@ -1,66 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-from .common import InfoExtractor
-from ..utils import (
-    determine_ext,
-)
-
-
-class OppetArkivIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?oppetarkiv\.se/video/(?P<id>[0-9]+)'
-    _TEST = {
-        'url': 'http://www.oppetarkiv.se/video/1058509/rederiet-sasong-1-avsnitt-1-av-318',
-        'md5': '5c1eb616e59f733d4af77edc5177d2fe',
-        'info_dict': {
-            'id': '1058509',
-            'ext': 'flv',
-            'title': 'Farlig kryssning',
-            'duration': 2566,
-            'thumbnail': 're:^https?://.*[\.-]jpg$',
-            'age_limit': 0,
-        },
-        'skip': 'Only works from Sweden',
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        info = self._download_json(
-            'http://www.oppetarkiv.se/video/%s?output=json' % video_id, video_id)
-
-        title = info['context']['title']
-        thumbnail = info['context'].get('thumbnailImage')
-
-        video_info = info['video']
-        formats = []
-        for vr in video_info['videoReferences']:
-            vurl = vr['url']
-            ext = determine_ext(vurl)
-            if ext == 'm3u8':
-                formats.extend(self._extract_m3u8_formats(
-                    vurl, video_id,
-                    ext='mp4', entry_protocol='m3u8_native',
-                    m3u8_id=vr.get('playerType')))
-            elif ext == 'f4m':
-                formats.extend(self._extract_f4m_formats(
-                    vurl + '?hdcore=3.3.0', video_id,
-                    f4m_id=vr.get('playerType')))
-            else:
-                formats.append({
-                    'format_id': vr.get('playerType'),
-                    'url': vurl,
-                })
-        self._sort_formats(formats)
-
-        duration = video_info.get('materialLength')
-
-        age_limit = 18 if video_info.get('inappropriateForChildren') else 0
-
-        return {
-            'id': video_id,
-            'title': title,
-            'formats': formats,
-            'thumbnail': thumbnail,
-            'duration': duration,
-            'age_limit': age_limit,
-        }
diff --git a/youtube_dl/extractor/svtplay.py b/youtube_dl/extractor/svtplay.py
index eadb9ccb4..433dfd1cb 100644
--- a/youtube_dl/extractor/svtplay.py
+++ b/youtube_dl/extractor/svtplay.py
@@ -1,6 +1,8 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..utils import (
     determine_ext,
@@ -8,23 +10,40 @@ from ..utils import (
 
 
 class SVTPlayIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?svtplay\.se/video/(?P<id>[0-9]+)'
-    _TEST = {
+    IE_DESC = 'SVT Play and Öppet arkiv'
+    _VALID_URL = r'https?://(?:www\.)?(?P<host>svtplay|oppetarkiv)\.se/video/(?P<id>[0-9]+)'
+    _TESTS = [{
         'url': 'http://www.svtplay.se/video/2609989/sm-veckan/sm-veckan-rally-final-sasong-1-sm-veckan-rally-final',
-        'md5': 'f4a184968bc9c802a9b41316657aaa80',
+        'md5': 'ade3def0643fa1c40587a422f98edfd9',
         'info_dict': {
             'id': '2609989',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': 'SM veckan vinter, Örebro - Rally, final',
             'duration': 4500,
             'thumbnail': 're:^https?://.*[\.-]jpg$',
+            'age_limit': 0,
         },
-    }
+    }, {
+        'url': 'http://www.oppetarkiv.se/video/1058509/rederiet-sasong-1-avsnitt-1-av-318',
+        'md5': 'c3101a17ce9634f4c1f9800f0746c187',
+        'info_dict': {
+            'id': '1058509',
+            'ext': 'flv',
+            'title': 'Farlig kryssning',
+            'duration': 2566,
+            'thumbnail': 're:^https?://.*[\.-]jpg$',
+            'age_limit': 0,
+        },
+        'skip': 'Only works from Sweden',
+    }]
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        host = mobj.group('host')
+
         info = self._download_json(
-            'http://www.svtplay.se/video/%s?output=json' % video_id, video_id)
+            'http://www.%s.se/video/%s?output=json' % (host, video_id), video_id)
 
         title = info['context']['title']
         thumbnail = info['context'].get('thumbnailImage')
@@ -33,11 +52,16 @@ class SVTPlayIE(InfoExtractor):
         formats = []
         for vr in video_info['videoReferences']:
             vurl = vr['url']
-            if determine_ext(vurl) == 'm3u8':
+            ext = determine_ext(vurl)
+            if ext == 'm3u8':
                 formats.extend(self._extract_m3u8_formats(
                     vurl, video_id,
                     ext='mp4', entry_protocol='m3u8_native',
                     m3u8_id=vr.get('playerType')))
+            elif ext == 'f4m':
+                formats.extend(self._extract_f4m_formats(
+                    vurl + '?hdcore=3.3.0', video_id,
+                    f4m_id=vr.get('playerType')))
             else:
                 formats.append({
                     'format_id': vr.get('playerType'),
@@ -46,6 +70,7 @@ class SVTPlayIE(InfoExtractor):
         self._sort_formats(formats)
 
         duration = video_info.get('materialLength')
+        age_limit = 18 if video_info.get('inappropriateForChildren') else 0
 
         return {
             'id': video_id,
@@ -53,4 +78,5 @@ class SVTPlayIE(InfoExtractor):
             'formats': formats,
             'thumbnail': thumbnail,
             'duration': duration,
+            'age_limit': age_limit,
         }

From 6c87c2eea8b7d14c4178aaae3d74559347a772e6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 28 Feb 2015 22:25:57 +0600
Subject: [PATCH 0564/6219] [puls4] Improve and extract more metadata

---
 youtube_dl/extractor/puls4.py | 89 +++++++++++++++++++++++------------
 1 file changed, 58 insertions(+), 31 deletions(-)

diff --git a/youtube_dl/extractor/puls4.py b/youtube_dl/extractor/puls4.py
index 70dedbff3..cce84b9e4 100644
--- a/youtube_dl/extractor/puls4.py
+++ b/youtube_dl/extractor/puls4.py
@@ -1,61 +1,88 @@
 # -*- coding: utf-8 -*-
-
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-
-import re
+from ..utils import (
+    ExtractorError,
+    unified_strdate,
+    int_or_none,
+)
 
 
 class Puls4IE(InfoExtractor):
-
-    _VALID_URL = r'https?://www.puls4.com/video/.+?/play/(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?puls4\.com/video/[^/]+/play/(?P<id>[0-9]+)'
     _TESTS = [{
         'url': 'http://www.puls4.com/video/pro-und-contra/play/2716816',
         'md5': '49f6a6629747eeec43cef6a46b5df81d',
         'info_dict': {
             'id': '2716816',
             'ext': 'mp4',
-            'title': 'Pro und Contra vom 23.02.2015'}},
-        {
+            'title': 'Pro und Contra vom 23.02.2015',
+            'description': 'md5:293e44634d9477a67122489994675db6',
+            'duration': 2989,
+            'upload_date': '20150224',
+            'uploader': 'PULS_4',
+        },
+        'skip': 'Only works from Germany',
+    }, {
         'url': 'http://www.puls4.com/video/kult-spielfilme/play/1298106',
         'md5': '6a48316c8903ece8dab9b9a7bf7a59ec',
         'info_dict': {
             'id': '1298106',
             'ext': 'mp4',
-            'title': 'Lucky Fritz'}}
-    ]
+            'title': 'Lucky Fritz',
+        },
+        'skip': 'Only works from Germany',
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        # if fsk-button
-        real_url = self._html_search_regex(r'\"fsk-button\".+?href=\"([^"]+)',
-                                           webpage, 'fsk_button', default=None)
+        error_message = self._html_search_regex(
+            r'<div class="message-error">(.+?)</div>',
+            webpage, 'error message', default=None)
+        if error_message:
+            raise ExtractorError(
+                '%s returned error: %s' % (self.IE_NAME, error_message), expected=True)
+
+        real_url = self._html_search_regex(
+            r'\"fsk-button\".+?href=\"([^"]+)',
+            webpage, 'fsk_button', default=None)
         if real_url:
             webpage = self._download_webpage(real_url, video_id)
 
-        title = self._html_search_regex(
-            r'<div id="bg_brandableContent">.+?<h1>(.+?)</h1>',
-            webpage, 'title', flags=re.DOTALL)
+        player = self._search_regex(
+            r'p4_video_player(?:_iframe)?\("video_\d+_container"\s*,(.+?)\);\s*\}',
+            webpage, 'player')
 
-        sd_url = self._html_search_regex(
-            r'{\"url\":\"([^"]+?)\",\"hd\":false',
-            webpage, 'sd_url').replace('\\', '')
+        player_json = self._parse_json(
+            '[%s]' % player, video_id,
+            transform_source=lambda s: s.replace('undefined,', ''))
 
-        formats = [{'format_id': 'sd', 'url': sd_url, 'quality': -2}]
+        formats = None
+        result = None
 
-        hd_url = self._html_search_regex(
-            r'{\"url\":\"([^"]+?)\",\"hd\":true',
-            webpage, 'hd_url', default=None)
-        if hd_url:
-            hd_url = hd_url.replace('\\', '')
-            formats.append({'format_id': 'hd', 'url': hd_url, 'quality': -1})
+        for v in player_json:
+            if isinstance(v, list) and not formats:
+                formats = [{
+                    'url': f['url'],
+                    'format': 'hd' if f.get('hd') else 'sd',
+                    'width': int_or_none(f.get('size_x')),
+                    'height': int_or_none(f.get('size_y')),
+                    'tbr': int_or_none(f.get('bitrate')),
+                } for f in v]
+                self._sort_formats(formats)
+            elif isinstance(v, dict) and not result:
+                result = {
+                    'id': video_id,
+                    'title': v['videopartname'].strip(),
+                    'description': v.get('videotitle'),
+                    'duration': int_or_none(v.get('videoduration') or v.get('episodeduration')),
+                    'upload_date': unified_strdate(v.get('clipreleasetime')),
+                    'uploader': v.get('channel'),
+                }
 
-        return {
-            'id': video_id,
-            'title': title,
-            'formats': formats,
-            'ext': 'mp4'
-        }
+        result['formats'] = formats
+
+        return result

From 0d103de3b0b03c5027f0015327c2c44b9073513a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 28 Feb 2015 22:59:55 +0600
Subject: [PATCH 0565/6219] [twitch] Pass api_token along with every request
 (Closes #3986)

---
 youtube_dl/extractor/twitch.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 4b0d8988d..4b0ce54df 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -34,7 +34,15 @@ class TwitchBaseIE(InfoExtractor):
                 expected=True)
 
     def _download_json(self, url, video_id, note='Downloading JSON metadata'):
-        response = super(TwitchBaseIE, self)._download_json(url, video_id, note)
+        headers = {
+            'Referer': 'http://api.twitch.tv/crossdomain/receiver.html?v=2',
+            'X-Requested-With': 'XMLHttpRequest',
+        }
+        for cookie in self._downloader.cookiejar:
+            if cookie.name == 'api_token':
+                headers['Twitch-Api-Token'] = cookie.value
+        request = compat_urllib_request.Request(url, headers=headers)
+        response = super(TwitchBaseIE, self)._download_json(request, video_id, note)
         self._handle_error(response)
         return response
 

From 8237bec4f05d6930146a7f0b087ae1e259917799 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 28 Feb 2015 20:52:52 +0100
Subject: [PATCH 0566/6219] [escapist] Extract duration

---
 youtube_dl/extractor/escapist.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/escapist.py b/youtube_dl/extractor/escapist.py
index 80e9084f4..e47f3e27a 100644
--- a/youtube_dl/extractor/escapist.py
+++ b/youtube_dl/extractor/escapist.py
@@ -8,6 +8,7 @@ from ..compat import (
 from ..utils import (
     ExtractorError,
     js_to_json,
+    parse_duration,
 )
 
 
@@ -25,6 +26,7 @@ class EscapistIE(InfoExtractor):
             'uploader': 'The Escapist Presents',
             'title': "Breaking Down Baldur's Gate",
             'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 264,
         }
     }
 
@@ -41,6 +43,7 @@ class EscapistIE(InfoExtractor):
             r"<h1\s+class='headline'>(.*?)</a>",
             webpage, 'uploader', fatal=False)
         description = self._html_search_meta('description', webpage)
+        duration = parse_duration(self._html_search_meta('duration', webpage))
 
         raw_title = self._html_search_meta('title', webpage, fatal=True)
         title = raw_title.partition(' : ')[2]
@@ -105,6 +108,7 @@ class EscapistIE(InfoExtractor):
             'title': title,
             'thumbnail': self._og_search_thumbnail(webpage),
             'description': description,
+            'duration': duration,
         }
 
         if self._downloader.params.get('include_ads') and ad_formats:

From eee6293d572b85c700ef645ac96b1816cb45901d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 28 Feb 2015 20:55:44 +0100
Subject: [PATCH 0567/6219] [thechive] remove in favor of Kaltura (#5072)

---
 youtube_dl/extractor/__init__.py |  1 -
 youtube_dl/extractor/thechive.py | 60 --------------------------------
 2 files changed, 61 deletions(-)
 delete mode 100644 youtube_dl/extractor/thechive.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index f3967ff7b..ffcc7d9ab 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -484,7 +484,6 @@ from .tenplay import TenPlayIE
 from .testurl import TestURLIE
 from .testtube import TestTubeIE
 from .tf1 import TF1IE
-from .thechive import TheChiveIE
 from .theonion import TheOnionIE
 from .theplatform import ThePlatformIE
 from .thesixtyone import TheSixtyOneIE
diff --git a/youtube_dl/extractor/thechive.py b/youtube_dl/extractor/thechive.py
deleted file mode 100644
index df1a7998d..000000000
--- a/youtube_dl/extractor/thechive.py
+++ /dev/null
@@ -1,60 +0,0 @@
-from __future__ import unicode_literals
-
-import re
-
-from .common import InfoExtractor
-from ..utils import unified_strdate
-
-
-
-class TheChiveIE(InfoExtractor):
-    _VALID_URL = r'http://(www\.)?thechive\.com/[^/]+/[^/]+/[^/]+/(?P<video_id>[A-Za-z\-]+)'
-    _TEST = {
-        'url': "http://thechive.com/2015/02/20/so-thats-what-a-set-of-redneck-bagpipes-sound-like-video/",
-        'md5': "366710dda77cfa727bdef3523ba8466f",
-        'info_dict': {
-            'id': "so-thats-what-a-set-of-redneck-bagpipes-sound-like-video",
-            'title': "So that's what a set of redneck bagpipes sound like... (Video)",
-            'description': "Okay that was pretty good. Now play Freebird!...",
-            'thumbnail': "https://thechive.files.wordpress.com/2015/02/0_07dghz0w-thumbnail2.jpg",
-            'author': "Ben",
-            'upload_date': "20150220",
-            'ext': "mp4"
-        }
-    }
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('video_id')
-        webpage = self._download_webpage(url, video_id)
-
-        title = self._og_search_title(webpage)
-        description = self._html_search_regex(r'(?s)<meta name="description" content="(.*?)" />', webpage, 'description')
-        thumbnail = self._og_search_thumbnail(webpage)
-        author = self._html_search_regex(
-            r'(?s)itemprop="author">(.+?)</span>', webpage, 'author', fatal=False).capitalize() 
-        upload_date = unified_strdate(self._html_search_regex(
-            r'(?s)itemprop="datePublished" datetime="(.+?)">', webpage, 'upload_date', fatal=False))
-
-        # Adapted from extractor/musicvault.py
-        VIDEO_URL_TEMPLATE = 'http://cdnapi.kaltura.com/p/%(uid)s/sp/%(wid)s/playManifest/entryId/%(entry_id)s/format/url/protocol/http'
-
-        kaltura_id = self._search_regex(
-            r'entry_id=([^"]+)',
-            webpage, 'kaltura ID')
-        video_url = VIDEO_URL_TEMPLATE % {
-            'entry_id': kaltura_id,
-            'wid': self._search_regex(r'partner_id/([0-9]+)\?', webpage, 'wid'),
-            'uid': self._search_regex(r'uiconf_id/([0-9]+)/', webpage, 'uid'),
-        }
-
-        return {
-            'url': video_url,
-            'id': video_id,
-            'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
-            'author': author,
-            'upload_date': upload_date,
-            'ext': 'mp4'
-        }
\ No newline at end of file

From 01349011088b10861e283c245f5da56aa3d5fba0 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 28 Feb 2015 21:24:25 +0100
Subject: [PATCH 0568/6219] release 2015.02.28

---
 README.md              | 2 ++
 docs/supportedsites.md | 5 ++++-
 youtube_dl/version.py  | 2 +-
 3 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 2c53e2211..04f664cd3 100644
--- a/README.md
+++ b/README.md
@@ -408,6 +408,8 @@ which means you can modify it, redistribute it or use it however you like.
                                      downloading, similar to find's -exec
                                      syntax. Example: --exec 'adb push {}
                                      /sdcard/Music/ && rm {}'
+    --convert-subtitles FORMAT       Convert the subtitles to other format
+                                     (currently supported: srt|ass|vtt)
 
 # CONFIGURATION
 
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 49b4ac8c1..062cb3d62 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -210,6 +210,7 @@
  - **Jove**
  - **jpopsuki.tv**
  - **Jukebox**
+ - **Kaltura**
  - **Kankan**
  - **Karaoketv**
  - **keek**
@@ -308,6 +309,7 @@
  - **Nuvid**
  - **NYTimes**
  - **ocw.mit.edu**
+ - **Odnoklassniki**
  - **OktoberfestTV**
  - **on.aol.com**
  - **Ooyala**
@@ -334,6 +336,7 @@
  - **PornoXO**
  - **PromptFile**
  - **prosiebensat1**: ProSiebenSat.1 Digital
+ - **Puls4**
  - **Pyvideo**
  - **QuickVid**
  - **R7**
@@ -412,7 +415,7 @@
  - **StreamCZ**
  - **StreetVoice**
  - **SunPorno**
- - **SVTPlay**
+ - **SVTPlay**: SVT Play and Öppet arkiv
  - **SWRMediathek**
  - **Syfy**
  - **SztvHu**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index cf3e28bbe..5582348ba 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.26.2'
+__version__ = '2015.02.28'

From 003c69a84b68cadb46aeb8e03115848a722fd675 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 28 Feb 2015 21:42:16 +0100
Subject: [PATCH 0569/6219] Use shutil.get_terminal_size for getting the
 terminal width if it's available (python >= 3.3)

---
 youtube_dl/YoutubeDL.py |  4 ++--
 youtube_dl/compat.py    | 30 ++++++++++++++++++++++++++++++
 youtube_dl/options.py   |  4 ++--
 youtube_dl/utils.py     | 17 -----------------
 4 files changed, 34 insertions(+), 21 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 74e426168..d7c6db0ff 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -28,6 +28,7 @@ from .compat import (
     compat_basestring,
     compat_cookiejar,
     compat_expanduser,
+    compat_get_terminal_size,
     compat_http_client,
     compat_kwargs,
     compat_str,
@@ -46,7 +47,6 @@ from .utils import (
     ExtractorError,
     format_bytes,
     formatSeconds,
-    get_term_width,
     locked_file,
     make_HTTPS_handler,
     MaxDownloadsReached,
@@ -284,7 +284,7 @@ class YoutubeDL(object):
             try:
                 import pty
                 master, slave = pty.openpty()
-                width = get_term_width()
+                width = compat_get_terminal_size().columns
                 if width is None:
                     width_args = []
                 else:
diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index e989cdbbd..b2bf149ef 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -1,9 +1,11 @@
 from __future__ import unicode_literals
 
+import collections
 import getpass
 import optparse
 import os
 import re
+import shutil
 import socket
 import subprocess
 import sys
@@ -364,6 +366,33 @@ def workaround_optparse_bug9161():
             return real_add_option(self, *bargs, **bkwargs)
         optparse.OptionGroup.add_option = _compat_add_option
 
+if hasattr(shutil, 'get_terminal_size'):  # Python >= 3.3
+    compat_get_terminal_size = shutil.get_terminal_size
+else:
+    _terminal_size = collections.namedtuple('terminal_size', ['columns', 'lines'])
+
+    def compat_get_terminal_size():
+        columns = compat_getenv('COLUMNS', None)
+        if columns:
+            columns = int(columns)
+        else:
+            columns = None
+        lines = compat_getenv('LINES', None)
+        if lines:
+            lines = int(lines)
+        else:
+            lines = None
+
+        try:
+            sp = subprocess.Popen(
+                ['stty', 'size'],
+                stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+            out, err = sp.communicate()
+            lines, columns = map(int, out.split())
+        except:
+            pass
+        return _terminal_size(columns, lines)
+
 
 __all__ = [
     'compat_HTTPError',
@@ -371,6 +400,7 @@ __all__ = [
     'compat_chr',
     'compat_cookiejar',
     'compat_expanduser',
+    'compat_get_terminal_size',
     'compat_getenv',
     'compat_getpass',
     'compat_html_entities',
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 58f811162..a2ffe96bc 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -8,11 +8,11 @@ import sys
 from .downloader.external import list_external_downloaders
 from .compat import (
     compat_expanduser,
+    compat_get_terminal_size,
     compat_getenv,
     compat_kwargs,
 )
 from .utils import (
-    get_term_width,
     write_string,
 )
 from .version import __version__
@@ -100,7 +100,7 @@ def parseOpts(overrideArguments=None):
         return opts
 
     # No need to wrap help messages if we're on a wide console
-    columns = get_term_width()
+    columns = compat_get_terminal_size().columns
     max_width = columns if columns else 80
     max_help_position = 80
 
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 1f3bfef7d..d4938ec36 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -35,7 +35,6 @@ import zlib
 from .compat import (
     compat_basestring,
     compat_chr,
-    compat_getenv,
     compat_html_entities,
     compat_http_client,
     compat_parse_qs,
@@ -1173,22 +1172,6 @@ def parse_filesize(s):
     return int(float(num_str) * mult)
 
 
-def get_term_width():
-    columns = compat_getenv('COLUMNS', None)
-    if columns:
-        return int(columns)
-
-    try:
-        sp = subprocess.Popen(
-            ['stty', 'size'],
-            stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        out, err = sp.communicate()
-        return int(out.split()[1])
-    except:
-        pass
-    return None
-
-
 def month_by_name(name):
     """ Return the number of a month by (locale-independently) English name """
 

From 31bd39256b153391f521169ae449fd8c7c6e51d7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 1 Mar 2015 11:46:57 +0100
Subject: [PATCH 0570/6219] --load-info: Use the fileinput module

It automatically handles the '-' filename as stdin
---
 youtube_dl/YoutubeDL.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index d7c6db0ff..15367c4e3 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -4,8 +4,10 @@
 from __future__ import absolute_import, unicode_literals
 
 import collections
+import contextlib
 import datetime
 import errno
+import fileinput
 import io
 import itertools
 import json
@@ -1452,8 +1454,11 @@ class YoutubeDL(object):
         return self._download_retcode
 
     def download_with_info_file(self, info_filename):
-        with io.open(info_filename, 'r', encoding='utf-8') as f:
-            info = json.load(f)
+        with contextlib.closing(fileinput.FileInput(
+                [info_filename], mode='r',
+                openhook=fileinput.hook_encoded('utf-8'))) as f:
+            # FileInput doesn't have a read method, we can't call json.load
+            info = json.loads('\n'.join(f))
         try:
             self.process_ie_result(info, download=True)
         except DownloadError:

From 66dc9a3701cd97af44cf478e813d4f7eb0947edc Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Mar 2015 12:04:39 +0100
Subject: [PATCH 0571/6219] [README] Document HTTP 429 (Closes #5092)

---
 README.md | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/README.md b/README.md
index 04f664cd3..f2909e8d6 100644
--- a/README.md
+++ b/README.md
@@ -529,6 +529,10 @@ YouTube requires an additional signature since September 2012 which is not suppo
 
 In February 2015, the new YouTube player contained a character sequence in a string that was misinterpreted by old versions of youtube-dl. See [above](#how-do-i-update-youtube-dl) for how to update youtube-dl.
 
+### HTTP Error 429: Too Many Requests or 402: Payment Required
+
+These two error codes indicate that the service is blocking your IP address because of overuse. Contact the service and ask them to unblock your IP address, or - if you have acquired a whitelisted IP address already - use the [`--proxy` or `--network-address` options](#network-options) to select another IP address.
+
 ### SyntaxError: Non-ASCII character ###
 
 The error

From 1aa5172f56eca82ccb473859c6850ada677e6551 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 1 Mar 2015 21:55:43 +0600
Subject: [PATCH 0572/6219] [vk] Catch temporarily unavailable video error
 message

---
 youtube_dl/extractor/vk.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 7dea8c59d..08c3830a2 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -152,7 +152,10 @@ class VKIE(InfoExtractor):
             'use --username and --password options to provide account credentials.',
 
             r'<!>Unknown error':
-            'Video %s does not exist.'
+            'Video %s does not exist.',
+
+            r'<!>Видео временно недоступно':
+            'Video %s is temporarily unavailable.',
         }
 
         for error_re, error_msg in ERRORS.items():

From 4e01501bbf36519ab94dd16ba3486947b2785ee8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 1 Mar 2015 21:56:30 +0600
Subject: [PATCH 0573/6219] [vk] Fix extraction (Closes #4967, closes #4686)

---
 youtube_dl/extractor/vk.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 08c3830a2..842263f34 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -140,7 +140,7 @@ class VKIE(InfoExtractor):
         if not video_id:
             video_id = '%s_%s' % (mobj.group('oid'), mobj.group('id'))
 
-        info_url = 'http://vk.com/al_video.php?act=show&al=1&video=%s' % video_id
+        info_url = 'http://vk.com/al_video.php?act=show&al=1&module=video&video=%s' % video_id
         info_page = self._download_webpage(info_url, video_id)
 
         ERRORS = {

From 36300346092937a8320fa4ae7303bc1b746a69ab Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 2 Mar 2015 03:30:18 +0600
Subject: [PATCH 0574/6219] [vk] Fix test (Closes #5100)

---
 youtube_dl/extractor/vk.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 842263f34..cc384adbf 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -31,7 +31,7 @@ class VKIE(InfoExtractor):
                 'id': '162222515',
                 'ext': 'flv',
                 'title': 'ProtivoGunz - Хуёвая песня',
-                'uploader': 're:Noize MC.*',
+                'uploader': 're:(?:Noize MC|Alexander Ilyashenko).*',
                 'duration': 195,
                 'upload_date': '20120212',
             },

From 7594be85ffebee0c1ba457855ee7f4917e96499b Mon Sep 17 00:00:00 2001
From: Sergey <Ftornik@users.noreply.github.com>
Date: Mon, 2 Mar 2015 11:49:39 +0200
Subject: [PATCH 0575/6219] [lynda] Check for the empty subtitle

---
 youtube_dl/extractor/lynda.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index 5dc22da22..a84019bdf 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -144,6 +144,7 @@ class LyndaIE(InfoExtractor):
 
     def _fix_subtitles(self, subs):
         srt = ''
+        seq_counter = 0
         for pos in range(0, len(subs) - 1):
             seq_current = subs[pos]
             m_current = re.match(self._TIMECODE_REGEX, seq_current['Timecode'])
@@ -155,8 +156,10 @@ class LyndaIE(InfoExtractor):
                 continue
             appear_time = m_current.group('timecode')
             disappear_time = m_next.group('timecode')
-            text = seq_current['Caption'].lstrip()
-            srt += '%s\r\n%s --> %s\r\n%s' % (str(pos), appear_time, disappear_time, text)
+            text = seq_current['Caption'].strip()
+            if text:
+                seq_counter += 1
+                srt += '%s\r\n%s --> %s\r\n%s\r\n\r\n' % (seq_counter, appear_time, disappear_time, text)
         if srt:
             return srt
 

From 30cbd4e0d68361a20b036fc90bd53a8fa9a463dd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 2 Mar 2015 22:12:10 +0600
Subject: [PATCH 0576/6219] [lynda] Completely skip videos we don't have access
 to, extract base class and modernize (Closes #5093)

---
 youtube_dl/extractor/lynda.py | 149 ++++++++++++++++++----------------
 1 file changed, 81 insertions(+), 68 deletions(-)

diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index a84019bdf..1f02bef44 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -15,18 +15,72 @@ from ..utils import (
 )
 
 
-class LyndaIE(InfoExtractor):
+class LyndaBaseIE(InfoExtractor):
+    _LOGIN_URL = 'https://www.lynda.com/login/login.aspx'
+    _SUCCESSFUL_LOGIN_REGEX = r'isLoggedIn: true'
+    _ACCOUNT_CREDENTIALS_HINT = 'Use --username and --password options to provide lynda.com account credentials.'
+
+    def _real_initialize(self):
+        self._login()
+
+    def _login(self):
+        (username, password) = self._get_login_info()
+        if username is None:
+            return
+
+        login_form = {
+            'username': username,
+            'password': password,
+            'remember': 'false',
+            'stayPut': 'false'
+        }
+        request = compat_urllib_request.Request(
+            self._LOGIN_URL, compat_urllib_parse.urlencode(login_form))
+        login_page = self._download_webpage(
+            request, None, 'Logging in as %s' % username)
+
+        # Not (yet) logged in
+        m = re.search(r'loginResultJson = \'(?P<json>[^\']+)\';', login_page)
+        if m is not None:
+            response = m.group('json')
+            response_json = json.loads(response)
+            state = response_json['state']
+
+            if state == 'notlogged':
+                raise ExtractorError(
+                    'Unable to login, incorrect username and/or password',
+                    expected=True)
+
+            # This is when we get popup:
+            # > You're already logged in to lynda.com on two devices.
+            # > If you log in here, we'll log you out of another device.
+            # So, we need to confirm this.
+            if state == 'conflicted':
+                confirm_form = {
+                    'username': '',
+                    'password': '',
+                    'resolve': 'true',
+                    'remember': 'false',
+                    'stayPut': 'false',
+                }
+                request = compat_urllib_request.Request(
+                    self._LOGIN_URL, compat_urllib_parse.urlencode(confirm_form))
+                login_page = self._download_webpage(
+                    request, None,
+                    'Confirming log in and log out from another device')
+
+        if re.search(self._SUCCESSFUL_LOGIN_REGEX, login_page) is None:
+            raise ExtractorError('Unable to log in')
+
+
+class LyndaIE(LyndaBaseIE):
     IE_NAME = 'lynda'
     IE_DESC = 'lynda.com videos'
-    _VALID_URL = r'https?://www\.lynda\.com/(?:[^/]+/[^/]+/\d+|player/embed)/(\d+)'
-    _LOGIN_URL = 'https://www.lynda.com/login/login.aspx'
+    _VALID_URL = r'https?://www\.lynda\.com/(?:[^/]+/[^/]+/\d+|player/embed)/(?P<id>\d+)'
     _NETRC_MACHINE = 'lynda'
 
-    _SUCCESSFUL_LOGIN_REGEX = r'isLoggedIn: true'
     _TIMECODE_REGEX = r'\[(?P<timecode>\d+:\d+:\d+[\.,]\d+)\]'
 
-    ACCOUNT_CREDENTIALS_HINT = 'Use --username and --password options to provide lynda.com account credentials.'
-
     _TESTS = [{
         'url': 'http://www.lynda.com/Bootstrap-tutorials/Using-exercise-files/110885/114408-4.html',
         'md5': 'ecfc6862da89489161fb9cd5f5a6fac1',
@@ -41,23 +95,22 @@ class LyndaIE(InfoExtractor):
         'only_matching': True,
     }]
 
-    def _real_initialize(self):
-        self._login()
-
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group(1)
+        video_id = self._match_id(url)
 
-        page = self._download_webpage('http://www.lynda.com/ajax/player?videoId=%s&type=video' % video_id, video_id,
-                                      'Downloading video JSON')
+        page = self._download_webpage(
+            'http://www.lynda.com/ajax/player?videoId=%s&type=video' % video_id,
+            video_id, 'Downloading video JSON')
         video_json = json.loads(page)
 
         if 'Status' in video_json:
-            raise ExtractorError('lynda returned error: %s' % video_json['Message'], expected=True)
+            raise ExtractorError(
+                'lynda returned error: %s' % video_json['Message'], expected=True)
 
         if video_json['HasAccess'] is False:
             raise ExtractorError(
-                'Video %s is only available for members. ' % video_id + self.ACCOUNT_CREDENTIALS_HINT, expected=True)
+                'Video %s is only available for members. '
+                % video_id + self._ACCOUNT_CREDENTIALS_HINT, expected=True)
 
         video_id = compat_str(video_json['ID'])
         duration = video_json['DurationInSeconds']
@@ -100,48 +153,6 @@ class LyndaIE(InfoExtractor):
             'formats': formats
         }
 
-    def _login(self):
-        (username, password) = self._get_login_info()
-        if username is None:
-            return
-
-        login_form = {
-            'username': username,
-            'password': password,
-            'remember': 'false',
-            'stayPut': 'false'
-        }
-        request = compat_urllib_request.Request(self._LOGIN_URL, compat_urllib_parse.urlencode(login_form))
-        login_page = self._download_webpage(request, None, 'Logging in as %s' % username)
-
-        # Not (yet) logged in
-        m = re.search(r'loginResultJson = \'(?P<json>[^\']+)\';', login_page)
-        if m is not None:
-            response = m.group('json')
-            response_json = json.loads(response)
-            state = response_json['state']
-
-            if state == 'notlogged':
-                raise ExtractorError('Unable to login, incorrect username and/or password', expected=True)
-
-            # This is when we get popup:
-            # > You're already logged in to lynda.com on two devices.
-            # > If you log in here, we'll log you out of another device.
-            # So, we need to confirm this.
-            if state == 'conflicted':
-                confirm_form = {
-                    'username': '',
-                    'password': '',
-                    'resolve': 'true',
-                    'remember': 'false',
-                    'stayPut': 'false',
-                }
-                request = compat_urllib_request.Request(self._LOGIN_URL, compat_urllib_parse.urlencode(confirm_form))
-                login_page = self._download_webpage(request, None, 'Confirming log in and log out from another device')
-
-        if re.search(self._SUCCESSFUL_LOGIN_REGEX, login_page) is None:
-            raise ExtractorError('Unable to log in')
-
     def _fix_subtitles(self, subs):
         srt = ''
         seq_counter = 0
@@ -172,7 +183,7 @@ class LyndaIE(InfoExtractor):
             return {}
 
 
-class LyndaCourseIE(InfoExtractor):
+class LyndaCourseIE(LyndaBaseIE):
     IE_NAME = 'lynda:course'
     IE_DESC = 'lynda.com online courses'
 
@@ -185,35 +196,37 @@ class LyndaCourseIE(InfoExtractor):
         course_path = mobj.group('coursepath')
         course_id = mobj.group('courseid')
 
-        page = self._download_webpage('http://www.lynda.com/ajax/player?courseId=%s&type=course' % course_id,
-                                      course_id, 'Downloading course JSON')
+        page = self._download_webpage(
+            'http://www.lynda.com/ajax/player?courseId=%s&type=course' % course_id,
+            course_id, 'Downloading course JSON')
         course_json = json.loads(page)
 
         if 'Status' in course_json and course_json['Status'] == 'NotFound':
-            raise ExtractorError('Course %s does not exist' % course_id, expected=True)
+            raise ExtractorError(
+                'Course %s does not exist' % course_id, expected=True)
 
         unaccessible_videos = 0
         videos = []
-        (username, _) = self._get_login_info()
 
         # Might want to extract videos right here from video['Formats'] as it seems 'Formats' is not provided
         # by single video API anymore
 
         for chapter in course_json['Chapters']:
             for video in chapter['Videos']:
-                if username is None and video['HasAccess'] is False:
+                if video['HasAccess'] is False:
                     unaccessible_videos += 1
                     continue
                 videos.append(video['ID'])
 
         if unaccessible_videos > 0:
-            self._downloader.report_warning('%s videos are only available for members and will not be downloaded. '
-                                            % unaccessible_videos + LyndaIE.ACCOUNT_CREDENTIALS_HINT)
+            self._downloader.report_warning(
+                '%s videos are only available for members (or paid members) and will not be downloaded. '
+                % unaccessible_videos + self._ACCOUNT_CREDENTIALS_HINT)
 
         entries = [
-            self.url_result('http://www.lynda.com/%s/%s-4.html' %
-                            (course_path, video_id),
-                            'Lynda')
+            self.url_result(
+                'http://www.lynda.com/%s/%s-4.html' % (course_path, video_id),
+                'Lynda')
             for video_id in videos]
 
         course_title = course_json['Title']

From 2f0f6578c3e4b82a5decc6f906a5f58b7b34c1a9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 2 Mar 2015 22:38:44 +0600
Subject: [PATCH 0577/6219] [extractor/common] Assume non HTTP(S) URLs valid

---
 youtube_dl/extractor/common.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 7977fa8d0..cf39c0c21 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -767,6 +767,10 @@ class InfoExtractor(object):
                 formats)
 
     def _is_valid_url(self, url, video_id, item='video'):
+        url = self._proto_relative_url(url, scheme='http:')
+        # For now assume non HTTP(S) URLs always valid
+        if not (url.startswith('http://') or url.startswith('https://')):
+            return True
         try:
             self._request_webpage(url, video_id, 'Checking %s URL' % item)
             return True

From 562ceab13dc3a19243d5ab9d4d5927031a608334 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 2 Mar 2015 22:39:32 +0600
Subject: [PATCH 0578/6219] [soundcloud] Check direct links validity (Closes
 #5101)

---
 youtube_dl/extractor/soundcloud.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index c5284fa67..95788098e 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -200,8 +200,9 @@ class SoundcloudIE(InfoExtractor):
                 if f['format_id'].startswith('rtmp'):
                     f['protocol'] = 'rtmp'
 
-            self._sort_formats(formats)
-            result['formats'] = formats
+        self._check_formats(formats, track_id)
+        self._sort_formats(formats)
+        result['formats'] = formats
 
         return result
 

From 295df4edb97e3c0dc0ecd95746bb2c455607a4a3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 2 Mar 2015 22:47:07 +0600
Subject: [PATCH 0579/6219] [soundcloud] Fix glitches (#5101)

---
 youtube_dl/extractor/soundcloud.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 95788098e..9d4505972 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -180,7 +180,7 @@ class SoundcloudIE(InfoExtractor):
                     'format_id': key,
                     'url': url,
                     'play_path': 'mp3:' + path,
-                    'ext': ext,
+                    'ext': 'flv',
                     'vcodec': 'none',
                 })
 

From c75f0b361a1b00f6ac1298615d6fee101994b2b9 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 2 Mar 2015 15:06:09 +0100
Subject: [PATCH 0580/6219] [downloader/external] Add support for custom
 options (Fixes #4885, closes #5098)

---
 youtube_dl/YoutubeDL.py           |  4 ++--
 youtube_dl/__init__.py            |  5 +++++
 youtube_dl/downloader/common.py   |  2 ++
 youtube_dl/downloader/external.py | 15 ++++++++++++---
 youtube_dl/options.py             |  6 +++++-
 5 files changed, 26 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 15367c4e3..e0baa9872 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -249,10 +249,10 @@ class YoutubeDL(object):
     hls_prefer_native: Use the native HLS downloader instead of ffmpeg/avconv.
 
     The following parameters are not used by YoutubeDL itself, they are used by
-    the FileDownloader:
+    the downloader (see youtube_dl/downloader/common.py):
     nopart, updatetime, buffersize, ratelimit, min_filesize, max_filesize, test,
     noresizebuffer, retries, continuedl, noprogress, consoletitle,
-    xattr_set_filesize.
+    xattr_set_filesize, external_downloader_args.
 
     The following options are used by the post processors:
     prefer_ffmpeg:     If True, use ffmpeg instead of avconv if both are available,
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 49f382695..6056da1be 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -9,6 +9,7 @@ import codecs
 import io
 import os
 import random
+import shlex
 import sys
 
 
@@ -255,6 +256,9 @@ def _real_main(argv=None):
             xattr  # Confuse flake8
         except ImportError:
             parser.error('setting filesize xattr requested but python-xattr is not available')
+    external_downloader_args = None
+    if opts.external_downloader_args:
+        external_downloader_args = shlex.split(opts.external_downloader_args)
     match_filter = (
         None if opts.match_filter is None
         else match_filter_func(opts.match_filter))
@@ -359,6 +363,7 @@ def _real_main(argv=None):
         'no_color': opts.no_color,
         'ffmpeg_location': opts.ffmpeg_location,
         'hls_prefer_native': opts.hls_prefer_native,
+        'external_downloader_args': external_downloader_args,
     }
 
     with YoutubeDL(ydl_opts) as ydl:
diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index 3ae90021a..8ed5c19a6 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -42,6 +42,8 @@ class FileDownloader(object):
     max_filesize:       Skip files larger than this size
     xattr_set_filesize: Set ytdl.filesize user xattribute with expected size.
                         (experimenatal)
+    external_downloader_args:  A list of additional command-line arguments for the
+                        external downloader.
 
     Subclasses of this one must re-define the real_download method.
     """
diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index 51c41c704..1673b2382 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -51,6 +51,13 @@ class ExternalFD(FileDownloader):
             return []
         return [command_option, source_address]
 
+    def _configuration_args(self, default=[]):
+        ex_args = self.params.get('external_downloader_args')
+        if ex_args is None:
+            return default
+        assert isinstance(ex_args, list)
+        return ex_args
+
     def _call_downloader(self, tmpfilename, info_dict):
         """ Either overwrite this or implement _make_cmd """
         cmd = self._make_cmd(tmpfilename, info_dict)
@@ -79,6 +86,7 @@ class CurlFD(ExternalFD):
         for key, val in info_dict['http_headers'].items():
             cmd += ['--header', '%s: %s' % (key, val)]
         cmd += self._source_address('--interface')
+        cmd += self._configuration_args()
         cmd += ['--', info_dict['url']]
         return cmd
 
@@ -89,15 +97,16 @@ class WgetFD(ExternalFD):
         for key, val in info_dict['http_headers'].items():
             cmd += ['--header', '%s: %s' % (key, val)]
         cmd += self._source_address('--bind-address')
+        cmd += self._configuration_args()
         cmd += ['--', info_dict['url']]
         return cmd
 
 
 class Aria2cFD(ExternalFD):
     def _make_cmd(self, tmpfilename, info_dict):
-        cmd = [
-            self.exe, '-c',
-            '--min-split-size', '1M', '--max-connection-per-server', '4']
+        cmd = [self.exe, '-c']
+        cmd += self._configuration_args([
+            '--min-split-size', '1M', '--max-connection-per-server', '4'])
         dn = os.path.dirname(tmpfilename)
         if dn:
             cmd += ['--dir', dn]
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index a2ffe96bc..df2be7b74 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -435,8 +435,12 @@ def parseOpts(overrideArguments=None):
     downloader.add_option(
         '--external-downloader',
         dest='external_downloader', metavar='COMMAND',
-        help='(experimental) Use the specified external downloader. '
+        help='Use the specified external downloader. '
              'Currently supports %s' % ','.join(list_external_downloaders()))
+    downloader.add_option(
+        '--external-downloader-args',
+        dest='external_downloader_args', metavar='ARGS',
+        help='Give these arguments to the external downloader.')
 
     workarounds = optparse.OptionGroup(parser, 'Workarounds')
     workarounds.add_option(

From 76c73715fb1e0eee61ace5ff7855d8237abdcd54 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 2 Mar 2015 15:21:11 +0100
Subject: [PATCH 0581/6219] [generic] Parse RSS enclosure URLs (Fixes #5091)

---
 youtube_dl/extractor/generic.py | 34 ++++++++++++++++++++++++++++-----
 1 file changed, 29 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 27e2bc300..5dc53685c 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -26,6 +26,7 @@ from ..utils import (
     unsmuggle_url,
     UnsupportedError,
     url_basename,
+    xpath_text,
 )
 from .brightcove import BrightcoveIE
 from .ooyala import OoyalaIE
@@ -569,6 +570,16 @@ class GenericIE(InfoExtractor):
                 'title': 'John Carlson Postgame 2/25/15',
             },
         },
+        # RSS feed with enclosure
+        {
+            'url': 'http://podcastfeeds.nbcnews.com/audio/podcast/MSNBC-MADDOW-NETCAST-M4V.xml',
+            'info_dict': {
+                'id': 'pdv_maddow_netcast_m4v-02-27-2015-201624',
+                'ext': 'm4v',
+                'upload_date': '20150228',
+                'title': 'pdv_maddow_netcast_m4v-02-27-2015-201624',
+            }
+        }
     ]
 
     def report_following_redirect(self, new_url):
@@ -580,11 +591,24 @@ class GenericIE(InfoExtractor):
         playlist_desc_el = doc.find('./channel/description')
         playlist_desc = None if playlist_desc_el is None else playlist_desc_el.text
 
-        entries = [{
-            '_type': 'url',
-            'url': e.find('link').text,
-            'title': e.find('title').text,
-        } for e in doc.findall('./channel/item')]
+        entries = []
+        for it in doc.findall('./channel/item'):
+            next_url = xpath_text(it, 'link', fatal=False)
+            if not next_url:
+                enclosure_nodes = it.findall('./enclosure')
+                for e in enclosure_nodes:
+                    next_url = e.attrib.get('url')
+                    if next_url:
+                        break
+
+            if not next_url:
+                continue
+
+            entries.append({
+                '_type': 'url',
+                'url': next_url,
+                'title': it.find('title').text,
+            })
 
         return {
             '_type': 'playlist',

From a7440261c52d6518a429210aaae598095112c61b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 2 Mar 2015 19:07:17 +0100
Subject: [PATCH 0582/6219] [utils] Streap leading dots

Fixes #2865, closes #5087
---
 test/test_utils.py  | 3 +++
 youtube_dl/utils.py | 1 +
 2 files changed, 4 insertions(+)

diff --git a/test/test_utils.py b/test/test_utils.py
index 3fba8ae11..64fad58ad 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -85,8 +85,11 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(
             sanitize_filename('New World record at 0:12:34'),
             'New World record at 0_12_34')
+
         self.assertEqual(sanitize_filename('--gasdgf'), '_-gasdgf')
         self.assertEqual(sanitize_filename('--gasdgf', is_id=True), '--gasdgf')
+        self.assertEqual(sanitize_filename('.gasdgf'), 'gasdgf')
+        self.assertEqual(sanitize_filename('.gasdgf', is_id=True), '.gasdgf')
 
         forbidden = '"\0\\/'
         for fc in forbidden:
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index d4938ec36..1d3401bc2 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -305,6 +305,7 @@ def sanitize_filename(s, restricted=False, is_id=False):
             result = result[2:]
         if result.startswith('-'):
             result = '_' + result[len('-'):]
+        result = result.lstrip('.')
         if not result:
             result = '_'
     return result

From 91410c9bfa9fd8f01fb817474bcc7b0db5cabf95 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 3 Mar 2015 00:03:06 +0100
Subject: [PATCH 0583/6219] [letv] Add --cn-verification-proxy (Closes #5077)

---
 youtube_dl/YoutubeDL.py      |  5 ++++-
 youtube_dl/__init__.py       |  1 +
 youtube_dl/extractor/letv.py | 34 ++++++++++++++++++++++++++++------
 youtube_dl/options.py        |  6 ++++++
 youtube_dl/utils.py          | 10 ++++++++++
 5 files changed, 49 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index e0baa9872..915963d96 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -54,6 +54,7 @@ from .utils import (
     MaxDownloadsReached,
     PagedList,
     parse_filesize,
+    PerRequestProxyHandler,
     PostProcessingError,
     platform_name,
     preferredencoding,
@@ -183,6 +184,8 @@ class YoutubeDL(object):
     prefer_insecure:   Use HTTP instead of HTTPS to retrieve information.
                        At the moment, this is only supported by YouTube.
     proxy:             URL of the proxy server to use
+    cn_verification_proxy:  URL of the proxy to use for IP address verification
+                       on Chinese sites. (Experimental)
     socket_timeout:    Time to wait for unresponsive hosts, in seconds
     bidi_workaround:   Work around buggy terminals without bidirectional text
                        support, using fridibi
@@ -1762,7 +1765,7 @@ class YoutubeDL(object):
             # Set HTTPS proxy to HTTP one if given (https://github.com/rg3/youtube-dl/issues/805)
             if 'http' in proxies and 'https' not in proxies:
                 proxies['https'] = proxies['http']
-        proxy_handler = compat_urllib_request.ProxyHandler(proxies)
+        proxy_handler = PerRequestProxyHandler(proxies)
 
         debuglevel = 1 if self.params.get('debug_printtraffic') else 0
         https_handler = make_HTTPS_handler(self.params, debuglevel=debuglevel)
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 6056da1be..a08ddd670 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -364,6 +364,7 @@ def _real_main(argv=None):
         'ffmpeg_location': opts.ffmpeg_location,
         'hls_prefer_native': opts.hls_prefer_native,
         'external_downloader_args': external_downloader_args,
+        'cn_verification_proxy': opts.cn_verification_proxy,
     }
 
     with YoutubeDL(ydl_opts) as ydl:
diff --git a/youtube_dl/extractor/letv.py b/youtube_dl/extractor/letv.py
index 583ce35b9..fd5fd260e 100644
--- a/youtube_dl/extractor/letv.py
+++ b/youtube_dl/extractor/letv.py
@@ -7,8 +7,9 @@ import time
 
 from .common import InfoExtractor
 from ..compat import (
-    compat_urlparse,
     compat_urllib_parse,
+    compat_urllib_request,
+    compat_urlparse,
 )
 from ..utils import (
     determine_ext,
@@ -42,9 +43,23 @@ class LetvIE(InfoExtractor):
         'expected_warnings': [
             'publish time'
         ]
+    }, {
+        'note': 'This video is available only in Mainland China, thus a proxy is needed',
+        'url': 'http://www.letv.com/ptv/vplay/1118082.html',
+        'md5': 'f80936fbe20fb2f58648e81386ff7927',
+        'info_dict': {
+            'id': '1118082',
+            'ext': 'mp4',
+            'title': '与龙共舞 完整版',
+            'description': 'md5:7506a5eeb1722bb9d4068f85024e3986',
+        },
+        'expected_warnings': [
+            'publish time'
+        ],
+        'params': {
+            'cn_verification_proxy': 'proxy.uku.im:8888'
+        },
     }]
-    # http://www.letv.com/ptv/vplay/1118082.html
-    # This video is available only in Mainland China
 
     @staticmethod
     def urshift(val, n):
@@ -76,8 +91,14 @@ class LetvIE(InfoExtractor):
             'tkey': self.calc_time_key(int(time.time())),
             'domain': 'www.letv.com'
         }
+        play_json_req = compat_urllib_request.Request(
+            'http://api.letv.com/mms/out/video/playJson?' + compat_urllib_parse.urlencode(params)
+        )
+        play_json_req.add_header(
+            'Ytdl-Request-Proxy',
+            self._downloader.params.get('cn_verification_proxy'))
         play_json = self._download_json(
-            'http://api.letv.com/mms/out/video/playJson?' + compat_urllib_parse.urlencode(params),
+            play_json_req,
             media_id, 'playJson data')
 
         # Check for errors
@@ -114,7 +135,8 @@ class LetvIE(InfoExtractor):
 
                 url_info_dict = {
                     'url': media_url,
-                    'ext': determine_ext(dispatch[format_id][1])
+                    'ext': determine_ext(dispatch[format_id][1]),
+                    'format_id': format_id,
                 }
 
                 if format_id[-1:] == 'p':
@@ -123,7 +145,7 @@ class LetvIE(InfoExtractor):
                 urls.append(url_info_dict)
 
         publish_time = parse_iso8601(self._html_search_regex(
-            r'发布时间&nbsp;([^<>]+) ', page, 'publish time', fatal=False),
+            r'发布时间&nbsp;([^<>]+) ', page, 'publish time', default=None),
             delimiter=' ', timezone=datetime.timedelta(hours=8))
         description = self._html_search_meta('description', page, fatal=False)
 
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index df2be7b74..a4ca8adc4 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -195,6 +195,12 @@ def parseOpts(overrideArguments=None):
         action='store_const', const='::', dest='source_address',
         help='Make all connections via IPv6 (experimental)',
     )
+    network.add_option(
+        '--cn-verification-proxy',
+        dest='cn_verification_proxy', default=None, metavar='URL',
+        help='Use this proxy to verify the IP address for some Chinese sites. '
+        'The default proxy specified by --proxy (or none, if the options is not present) is used for the actual downloading. (experimental)'
+    )
 
     selection = optparse.OptionGroup(parser, 'Video Selection')
     selection.add_option(
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 1d3401bc2..b568288fa 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1768,3 +1768,13 @@ def match_filter_func(filter_str):
             video_title = info_dict.get('title', info_dict.get('id', 'video'))
             return '%s does not pass filter %s, skipping ..' % (video_title, filter_str)
     return _match_func
+
+
+class PerRequestProxyHandler(compat_urllib_request.ProxyHandler):
+    def proxy_open(self, req, proxy, type):
+        req_proxy = req.headers.get('Ytdl-Request-Proxy')
+        if req_proxy is not None:
+            proxy = req_proxy
+            del req.headers['Ytdl-Request-Proxy']
+        return compat_urllib_request.ProxyHandler.proxy_open(
+            self, req, proxy, type)

From 07490f8017a83b7607686499074c41212fa0a44f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 3 Mar 2015 00:05:05 +0100
Subject: [PATCH 0584/6219] release 2015.03.03

---
 README.md             | 411 ++++++++++++------------------------------
 youtube_dl/version.py |   2 +-
 2 files changed, 119 insertions(+), 294 deletions(-)

diff --git a/README.md b/README.md
index f2909e8d6..d05549125 100644
--- a/README.md
+++ b/README.md
@@ -47,211 +47,107 @@ which means you can modify it, redistribute it or use it however you like.
 # OPTIONS
     -h, --help                       print this help text and exit
     --version                        print program version and exit
-    -U, --update                     update this program to latest version. Make
-                                     sure that you have sufficient permissions
-                                     (run with sudo if needed)
-    -i, --ignore-errors              continue on download errors, for example to
-                                     skip unavailable videos in a playlist
-    --abort-on-error                 Abort downloading of further videos (in the
-                                     playlist or the command line) if an error
-                                     occurs
+    -U, --update                     update this program to latest version. Make sure that you have sufficient permissions (run with sudo if needed)
+    -i, --ignore-errors              continue on download errors, for example to skip unavailable videos in a playlist
+    --abort-on-error                 Abort downloading of further videos (in the playlist or the command line) if an error occurs
     --dump-user-agent                display the current browser identification
-    --list-extractors                List all supported extractors and the URLs
-                                     they would handle
-    --extractor-descriptions         Output descriptions of all supported
-                                     extractors
-    --default-search PREFIX          Use this prefix for unqualified URLs. For
-                                     example "gvsearch2:" downloads two videos
-                                     from google videos for  youtube-dl "large
-                                     apple". Use the value "auto" to let
-                                     youtube-dl guess ("auto_warning" to emit a
-                                     warning when guessing). "error" just throws
-                                     an error. The default value "fixup_error"
-                                     repairs broken URLs, but emits an error if
-                                     this is not possible instead of searching.
-    --ignore-config                  Do not read configuration files. When given
-                                     in the global configuration file /etc
-                                     /youtube-dl.conf: Do not read the user
-                                     configuration in ~/.config/youtube-
-                                     dl/config (%APPDATA%/youtube-dl/config.txt
-                                     on Windows)
-    --flat-playlist                  Do not extract the videos of a playlist,
-                                     only list them.
+    --list-extractors                List all supported extractors and the URLs they would handle
+    --extractor-descriptions         Output descriptions of all supported extractors
+    --default-search PREFIX          Use this prefix for unqualified URLs. For example "gvsearch2:" downloads two videos from google videos for  youtube-dl "large apple".
+                                     Use the value "auto" to let youtube-dl guess ("auto_warning" to emit a warning when guessing). "error" just throws an error. The
+                                     default value "fixup_error" repairs broken URLs, but emits an error if this is not possible instead of searching.
+    --ignore-config                  Do not read configuration files. When given in the global configuration file /etc/youtube-dl.conf: Do not read the user configuration
+                                     in ~/.config/youtube-dl/config (%APPDATA%/youtube-dl/config.txt on Windows)
+    --flat-playlist                  Do not extract the videos of a playlist, only list them.
     --no-color                       Do not emit color codes in output.
 
 ## Network Options:
-    --proxy URL                      Use the specified HTTP/HTTPS proxy. Pass in
-                                     an empty string (--proxy "") for direct
-                                     connection
+    --proxy URL                      Use the specified HTTP/HTTPS proxy. Pass in an empty string (--proxy "") for direct connection
     --socket-timeout SECONDS         Time to wait before giving up, in seconds
-    --source-address IP              Client-side IP address to bind to
-                                     (experimental)
-    -4, --force-ipv4                 Make all connections via IPv4
-                                     (experimental)
-    -6, --force-ipv6                 Make all connections via IPv6
-                                     (experimental)
+    --source-address IP              Client-side IP address to bind to (experimental)
+    -4, --force-ipv4                 Make all connections via IPv4 (experimental)
+    -6, --force-ipv6                 Make all connections via IPv6 (experimental)
 
 ## Video Selection:
     --playlist-start NUMBER          playlist video to start at (default is 1)
     --playlist-end NUMBER            playlist video to end at (default is last)
-    --playlist-items ITEM_SPEC       playlist video items to download. Specify
-                                     indices of the videos in the playlist
-                                     seperated by commas like: "--playlist-items
-                                     1,2,5,8" if you want to download videos
-                                     indexed 1, 2, 5, 8 in the playlist. You can
-                                     specify range: "--playlist-items
-                                     1-3,7,10-13", it will download the videos
-                                     at index 1, 2, 3, 7, 10, 11, 12 and 13.
-    --match-title REGEX              download only matching titles (regex or
-                                     caseless sub-string)
-    --reject-title REGEX             skip download for matching titles (regex or
-                                     caseless sub-string)
+    --playlist-items ITEM_SPEC       playlist video items to download. Specify indices of the videos in the playlist seperated by commas like: "--playlist-items 1,2,5,8"
+                                     if you want to download videos indexed 1, 2, 5, 8 in the playlist. You can specify range: "--playlist-items 1-3,7,10-13", it will
+                                     download the videos at index 1, 2, 3, 7, 10, 11, 12 and 13.
+    --match-title REGEX              download only matching titles (regex or caseless sub-string)
+    --reject-title REGEX             skip download for matching titles (regex or caseless sub-string)
     --max-downloads NUMBER           Abort after downloading NUMBER files
-    --min-filesize SIZE              Do not download any videos smaller than
-                                     SIZE (e.g. 50k or 44.6m)
-    --max-filesize SIZE              Do not download any videos larger than SIZE
-                                     (e.g. 50k or 44.6m)
+    --min-filesize SIZE              Do not download any videos smaller than SIZE (e.g. 50k or 44.6m)
+    --max-filesize SIZE              Do not download any videos larger than SIZE (e.g. 50k or 44.6m)
     --date DATE                      download only videos uploaded in this date
-    --datebefore DATE                download only videos uploaded on or before
-                                     this date (i.e. inclusive)
-    --dateafter DATE                 download only videos uploaded on or after
-                                     this date (i.e. inclusive)
-    --min-views COUNT                Do not download any videos with less than
-                                     COUNT views
-    --max-views COUNT                Do not download any videos with more than
-                                     COUNT views
-    --match-filter FILTER            (Experimental) Generic video filter.
-                                     Specify any key (see help for -o for a list
-                                     of available keys) to match if the key is
-                                     present, !key to check if the key is not
-                                     present,key > NUMBER (like "comment_count >
-                                     12", also works with >=, <, <=, !=, =) to
-                                     compare against a number, and & to require
-                                     multiple matches. Values which are not
-                                     known are excluded unless you put a
-                                     question mark (?) after the operator.For
-                                     example, to only match videos that have
-                                     been liked more than 100 times and disliked
-                                     less than 50 times (or the dislike
-                                     functionality is not available at the given
-                                     service), but who also have a description,
-                                     use  --match-filter "like_count > 100 &
+    --datebefore DATE                download only videos uploaded on or before this date (i.e. inclusive)
+    --dateafter DATE                 download only videos uploaded on or after this date (i.e. inclusive)
+    --min-views COUNT                Do not download any videos with less than COUNT views
+    --max-views COUNT                Do not download any videos with more than COUNT views
+    --match-filter FILTER            (Experimental) Generic video filter. Specify any key (see help for -o for a list of available keys) to match if the key is present,
+                                     !key to check if the key is not present,key > NUMBER (like "comment_count > 12", also works with >=, <, <=, !=, =) to compare against
+                                     a number, and & to require multiple matches. Values which are not known are excluded unless you put a question mark (?) after the
+                                     operator.For example, to only match videos that have been liked more than 100 times and disliked less than 50 times (or the dislike
+                                     functionality is not available at the given service), but who also have a description, use  --match-filter "like_count > 100 &
                                      dislike_count <? 50 & description" .
-    --no-playlist                    If the URL refers to a video and a
-                                     playlist, download only the video.
-    --yes-playlist                   If the URL refers to a video and a
-                                     playlist, download the playlist.
-    --age-limit YEARS                download only videos suitable for the given
-                                     age
-    --download-archive FILE          Download only videos not listed in the
-                                     archive file. Record the IDs of all
-                                     downloaded videos in it.
-    --include-ads                    Download advertisements as well
-                                     (experimental)
+    --no-playlist                    If the URL refers to a video and a playlist, download only the video.
+    --yes-playlist                   If the URL refers to a video and a playlist, download the playlist.
+    --age-limit YEARS                download only videos suitable for the given age
+    --download-archive FILE          Download only videos not listed in the archive file. Record the IDs of all downloaded videos in it.
+    --include-ads                    Download advertisements as well (experimental)
 
 ## Download Options:
-    -r, --rate-limit LIMIT           maximum download rate in bytes per second
-                                     (e.g. 50K or 4.2M)
-    -R, --retries RETRIES            number of retries (default is 10), or
-                                     "infinite".
-    --buffer-size SIZE               size of download buffer (e.g. 1024 or 16K)
-                                     (default is 1024)
-    --no-resize-buffer               do not automatically adjust the buffer
-                                     size. By default, the buffer size is
-                                     automatically resized from an initial value
-                                     of SIZE.
+    -r, --rate-limit LIMIT           maximum download rate in bytes per second (e.g. 50K or 4.2M)
+    -R, --retries RETRIES            number of retries (default is 10), or "infinite".
+    --buffer-size SIZE               size of download buffer (e.g. 1024 or 16K) (default is 1024)
+    --no-resize-buffer               do not automatically adjust the buffer size. By default, the buffer size is automatically resized from an initial value of SIZE.
     --playlist-reverse               Download playlist videos in reverse order
-    --xattr-set-filesize             (experimental) set file xattribute
-                                     ytdl.filesize with expected filesize
-    --hls-prefer-native              (experimental) Use the native HLS
-                                     downloader instead of ffmpeg.
-    --external-downloader COMMAND    (experimental) Use the specified external
-                                     downloader. Currently supports
-                                     aria2c,curl,wget
+    --xattr-set-filesize             (experimental) set file xattribute ytdl.filesize with expected filesize
+    --hls-prefer-native              (experimental) Use the native HLS downloader instead of ffmpeg.
+    --external-downloader COMMAND    Use the specified external downloader. Currently supports aria2c,curl,wget
+    --external-downloader-args ARGS  Give these arguments to the external downloader.
 
 ## Filesystem Options:
-    -a, --batch-file FILE            file containing URLs to download ('-' for
-                                     stdin)
+    -a, --batch-file FILE            file containing URLs to download ('-' for stdin)
     --id                             use only video ID in file name
-    -o, --output TEMPLATE            output filename template. Use %(title)s to
-                                     get the title, %(uploader)s for the
-                                     uploader name, %(uploader_id)s for the
-                                     uploader nickname if different,
-                                     %(autonumber)s to get an automatically
-                                     incremented number, %(ext)s for the
-                                     filename extension, %(format)s for the
-                                     format description (like "22 - 1280x720" or
-                                     "HD"), %(format_id)s for the unique id of
-                                     the format (like Youtube's itags: "137"),
-                                     %(upload_date)s for the upload date
-                                     (YYYYMMDD), %(extractor)s for the provider
-                                     (youtube, metacafe, etc), %(id)s for the
-                                     video id, %(playlist_title)s,
-                                     %(playlist_id)s, or %(playlist)s (=title if
-                                     present, ID otherwise) for the playlist the
-                                     video is in, %(playlist_index)s for the
-                                     position in the playlist. %(height)s and
-                                     %(width)s for the width and height of the
-                                     video format. %(resolution)s for a textual
-                                     description of the resolution of the video
-                                     format. %% for a literal percent. Use - to
-                                     output to stdout. Can also be used to
-                                     download to a different directory, for
-                                     example with -o '/my/downloads/%(uploader)s
-                                     /%(title)s-%(id)s.%(ext)s' .
-    --autonumber-size NUMBER         Specifies the number of digits in
-                                     %(autonumber)s when it is present in output
-                                     filename template or --auto-number option
-                                     is given
-    --restrict-filenames             Restrict filenames to only ASCII
-                                     characters, and avoid "&" and spaces in
-                                     filenames
-    -A, --auto-number                [deprecated; use  -o
-                                     "%(autonumber)s-%(title)s.%(ext)s" ] number
-                                     downloaded files starting from 00000
-    -t, --title                      [deprecated] use title in file name
-                                     (default)
+    -o, --output TEMPLATE            output filename template. Use %(title)s to get the title, %(uploader)s for the uploader name, %(uploader_id)s for the uploader
+                                     nickname if different, %(autonumber)s to get an automatically incremented number, %(ext)s for the filename extension, %(format)s for
+                                     the format description (like "22 - 1280x720" or "HD"), %(format_id)s for the unique id of the format (like Youtube's itags: "137"),
+                                     %(upload_date)s for the upload date (YYYYMMDD), %(extractor)s for the provider (youtube, metacafe, etc), %(id)s for the video id,
+                                     %(playlist_title)s, %(playlist_id)s, or %(playlist)s (=title if present, ID otherwise) for the playlist the video is in,
+                                     %(playlist_index)s for the position in the playlist. %(height)s and %(width)s for the width and height of the video format.
+                                     %(resolution)s for a textual description of the resolution of the video format. %% for a literal percent. Use - to output to stdout.
+                                     Can also be used to download to a different directory, for example with -o '/my/downloads/%(uploader)s/%(title)s-%(id)s.%(ext)s' .
+    --autonumber-size NUMBER         Specifies the number of digits in %(autonumber)s when it is present in output filename template or --auto-number option is given
+    --restrict-filenames             Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames
+    -A, --auto-number                [deprecated; use  -o "%(autonumber)s-%(title)s.%(ext)s" ] number downloaded files starting from 00000
+    -t, --title                      [deprecated] use title in file name (default)
     -l, --literal                    [deprecated] alias of --title
     -w, --no-overwrites              do not overwrite files
-    -c, --continue                   force resume of partially downloaded files.
-                                     By default, youtube-dl will resume
-                                     downloads if possible.
-    --no-continue                    do not resume partially downloaded files
-                                     (restart from beginning)
-    --no-part                        do not use .part files - write directly
-                                     into output file
-    --no-mtime                       do not use the Last-modified header to set
-                                     the file modification time
-    --write-description              write video description to a .description
-                                     file
+    -c, --continue                   force resume of partially downloaded files. By default, youtube-dl will resume downloads if possible.
+    --no-continue                    do not resume partially downloaded files (restart from beginning)
+    --no-part                        do not use .part files - write directly into output file
+    --no-mtime                       do not use the Last-modified header to set the file modification time
+    --write-description              write video description to a .description file
     --write-info-json                write video metadata to a .info.json file
-    --write-annotations              write video annotations to a .annotation
-                                     file
-    --load-info FILE                 json file containing the video information
-                                     (created with the "--write-json" option)
-    --cookies FILE                   file to read cookies from and dump cookie
-                                     jar in
-    --cache-dir DIR                  Location in the filesystem where youtube-dl
-                                     can store some downloaded information
-                                     permanently. By default $XDG_CACHE_HOME
-                                     /youtube-dl or ~/.cache/youtube-dl . At the
-                                     moment, only YouTube player files (for
-                                     videos with obfuscated signatures) are
-                                     cached, but that may change.
+    --write-annotations              write video annotations to a .annotation file
+    --load-info FILE                 json file containing the video information (created with the "--write-json" option)
+    --cookies FILE                   file to read cookies from and dump cookie jar in
+    --cache-dir DIR                  Location in the filesystem where youtube-dl can store some downloaded information permanently. By default $XDG_CACHE_HOME/youtube-dl
+                                     or ~/.cache/youtube-dl . At the moment, only YouTube player files (for videos with obfuscated signatures) are cached, but that may
+                                     change.
     --no-cache-dir                   Disable filesystem caching
     --rm-cache-dir                   Delete all filesystem cache files
 
 ## Thumbnail images:
     --write-thumbnail                write thumbnail image to disk
     --write-all-thumbnails           write all thumbnail image formats to disk
-    --list-thumbnails                Simulate and list all available thumbnail
-                                     formats
+    --list-thumbnails                Simulate and list all available thumbnail formats
 
 ## Verbosity / Simulation Options:
     -q, --quiet                      activates quiet mode
     --no-warnings                    Ignore warnings
-    -s, --simulate                   do not download the video and do not write
-                                     anything to disk
+    -s, --simulate                   do not download the video and do not write anything to disk
     --skip-download                  do not download the video
     -g, --get-url                    simulate, quiet but print URL
     -e, --get-title                  simulate, quiet but print title
@@ -261,155 +157,84 @@ which means you can modify it, redistribute it or use it however you like.
     --get-duration                   simulate, quiet but print video length
     --get-filename                   simulate, quiet but print output filename
     --get-format                     simulate, quiet but print output format
-    -j, --dump-json                  simulate, quiet but print JSON information.
-                                     See --output for a description of available
-                                     keys.
-    -J, --dump-single-json           simulate, quiet but print JSON information
-                                     for each command-line argument. If the URL
-                                     refers to a playlist, dump the whole
-                                     playlist information in a single line.
-    --print-json                     Be quiet and print the video information as
-                                     JSON (video is still being downloaded).
+    -j, --dump-json                  simulate, quiet but print JSON information. See --output for a description of available keys.
+    -J, --dump-single-json           simulate, quiet but print JSON information for each command-line argument. If the URL refers to a playlist, dump the whole playlist
+                                     information in a single line.
+    --print-json                     Be quiet and print the video information as JSON (video is still being downloaded).
     --newline                        output progress bar as new lines
     --no-progress                    do not print progress bar
     --console-title                  display progress in console titlebar
     -v, --verbose                    print various debugging information
-    --dump-intermediate-pages        print downloaded pages to debug problems
-                                     (very verbose)
-    --write-pages                    Write downloaded intermediary pages to
-                                     files in the current directory to debug
-                                     problems
+    --dump-intermediate-pages        print downloaded pages to debug problems (very verbose)
+    --write-pages                    Write downloaded intermediary pages to files in the current directory to debug problems
     --print-traffic                  Display sent and read HTTP traffic
-    -C, --call-home                  Contact the youtube-dl server for
-                                     debugging.
-    --no-call-home                   Do NOT contact the youtube-dl server for
-                                     debugging.
+    -C, --call-home                  Contact the youtube-dl server for debugging.
+    --no-call-home                   Do NOT contact the youtube-dl server for debugging.
 
 ## Workarounds:
     --encoding ENCODING              Force the specified encoding (experimental)
     --no-check-certificate           Suppress HTTPS certificate validation.
-    --prefer-insecure                Use an unencrypted connection to retrieve
-                                     information about the video. (Currently
-                                     supported only for YouTube)
+    --prefer-insecure                Use an unencrypted connection to retrieve information about the video. (Currently supported only for YouTube)
     --user-agent UA                  specify a custom user agent
-    --referer URL                    specify a custom referer, use if the video
-                                     access is restricted to one domain
-    --add-header FIELD:VALUE         specify a custom HTTP header and its value,
-                                     separated by a colon ':'. You can use this
-                                     option multiple times
-    --bidi-workaround                Work around terminals that lack
-                                     bidirectional text support. Requires bidiv
-                                     or fribidi executable in PATH
-    --sleep-interval SECONDS         Number of seconds to sleep before each
-                                     download.
+    --referer URL                    specify a custom referer, use if the video access is restricted to one domain
+    --add-header FIELD:VALUE         specify a custom HTTP header and its value, separated by a colon ':'. You can use this option multiple times
+    --bidi-workaround                Work around terminals that lack bidirectional text support. Requires bidiv or fribidi executable in PATH
+    --sleep-interval SECONDS         Number of seconds to sleep before each download.
 
 ## Video Format Options:
-    -f, --format FORMAT              video format code, specify the order of
-                                     preference using slashes, as in -f 22/17/18
-                                     .  Instead of format codes, you can select
-                                     by extension for the extensions aac, m4a,
-                                     mp3, mp4, ogg, wav, webm. You can also use
-                                     the special names "best", "bestvideo",
-                                     "bestaudio", "worst".  You can filter the
-                                     video results by putting a condition in
-                                     brackets, as in -f "best[height=720]" (or
-                                     -f "[filesize>10M]").  This works for
-                                     filesize, height, width, tbr, abr, vbr,
-                                     asr, and fps and the comparisons <, <=, >,
-                                     >=, =, != and for ext, acodec, vcodec,
-                                     container, and protocol and the comparisons
-                                     =, != . Formats for which the value is not
-                                     known are excluded unless you put a
-                                     question mark (?) after the operator. You
-                                     can combine format filters, so  -f "[height
-                                     <=? 720][tbr>500]" selects up to 720p
-                                     videos (or videos where the height is not
-                                     known) with a bitrate of at least 500
-                                     KBit/s. By default, youtube-dl will pick
-                                     the best quality. Use commas to download
-                                     multiple audio formats, such as -f
-                                     136/137/mp4/bestvideo,140/m4a/bestaudio.
-                                     You can merge the video and audio of two
-                                     formats into a single file using -f <video-
-                                     format>+<audio-format> (requires ffmpeg or
-                                     avconv), for example -f
+    -f, --format FORMAT              video format code, specify the order of preference using slashes, as in -f 22/17/18 .  Instead of format codes, you can select by
+                                     extension for the extensions aac, m4a, mp3, mp4, ogg, wav, webm. You can also use the special names "best", "bestvideo", "bestaudio",
+                                     "worst".  You can filter the video results by putting a condition in brackets, as in -f "best[height=720]" (or -f "[filesize>10M]").
+                                     This works for filesize, height, width, tbr, abr, vbr, asr, and fps and the comparisons <, <=, >, >=, =, != and for ext, acodec,
+                                     vcodec, container, and protocol and the comparisons =, != . Formats for which the value is not known are excluded unless you put a
+                                     question mark (?) after the operator. You can combine format filters, so  -f "[height <=? 720][tbr>500]" selects up to 720p videos
+                                     (or videos where the height is not known) with a bitrate of at least 500 KBit/s. By default, youtube-dl will pick the best quality.
+                                     Use commas to download multiple audio formats, such as -f  136/137/mp4/bestvideo,140/m4a/bestaudio. You can merge the video and audio
+                                     of two formats into a single file using -f <video-format>+<audio-format> (requires ffmpeg or avconv), for example -f
                                      bestvideo+bestaudio.
     --all-formats                    download all available video formats
-    --prefer-free-formats            prefer free video formats unless a specific
-                                     one is requested
+    --prefer-free-formats            prefer free video formats unless a specific one is requested
     --max-quality FORMAT             highest quality format to download
     -F, --list-formats               list all available formats
-    --youtube-skip-dash-manifest     Do not download the DASH manifest on
-                                     YouTube videos
-    --merge-output-format FORMAT     If a merge is required (e.g.
-                                     bestvideo+bestaudio), output to given
-                                     container format. One of mkv, mp4, ogg,
-                                     webm, flv.Ignored if no merge is required
+    --youtube-skip-dash-manifest     Do not download the DASH manifest on YouTube videos
+    --merge-output-format FORMAT     If a merge is required (e.g. bestvideo+bestaudio), output to given container format. One of mkv, mp4, ogg, webm, flv.Ignored if no
+                                     merge is required
 
 ## Subtitle Options:
     --write-sub                      write subtitle file
-    --write-auto-sub                 write automatic subtitle file (youtube
-                                     only)
-    --all-subs                       downloads all the available subtitles of
-                                     the video
+    --write-auto-sub                 write automatic subtitle file (youtube only)
+    --all-subs                       downloads all the available subtitles of the video
     --list-subs                      lists all available subtitles for the video
-    --sub-format FORMAT              subtitle format, accepts formats
-                                     preference, for example: "ass/srt/best"
-    --sub-lang LANGS                 languages of the subtitles to download
-                                     (optional) separated by commas, use IETF
-                                     language tags like 'en,pt'
+    --sub-format FORMAT              subtitle format, accepts formats preference, for example: "ass/srt/best"
+    --sub-lang LANGS                 languages of the subtitles to download (optional) separated by commas, use IETF language tags like 'en,pt'
 
 ## Authentication Options:
     -u, --username USERNAME          login with this account ID
-    -p, --password PASSWORD          account password. If this option is left
-                                     out, youtube-dl will ask interactively.
+    -p, --password PASSWORD          account password. If this option is left out, youtube-dl will ask interactively.
     -2, --twofactor TWOFACTOR        two-factor auth code
     -n, --netrc                      use .netrc authentication data
     --video-password PASSWORD        video password (vimeo, smotri)
 
 ## Post-processing Options:
-    -x, --extract-audio              convert video files to audio-only files
-                                     (requires ffmpeg or avconv and ffprobe or
-                                     avprobe)
-    --audio-format FORMAT            "best", "aac", "vorbis", "mp3", "m4a",
-                                     "opus", or "wav"; "best" by default
-    --audio-quality QUALITY          ffmpeg/avconv audio quality specification,
-                                     insert a value between 0 (better) and 9
-                                     (worse) for VBR or a specific bitrate like
-                                     128K (default 5)
-    --recode-video FORMAT            Encode the video to another format if
-                                     necessary (currently supported:
-                                     mp4|flv|ogg|webm|mkv)
-    -k, --keep-video                 keeps the video file on disk after the
-                                     post-processing; the video is erased by
-                                     default
-    --no-post-overwrites             do not overwrite post-processed files; the
-                                     post-processed files are overwritten by
-                                     default
-    --embed-subs                     embed subtitles in the video (only for mp4
-                                     videos)
+    -x, --extract-audio              convert video files to audio-only files (requires ffmpeg or avconv and ffprobe or avprobe)
+    --audio-format FORMAT            "best", "aac", "vorbis", "mp3", "m4a", "opus", or "wav"; "best" by default
+    --audio-quality QUALITY          ffmpeg/avconv audio quality specification, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K
+                                     (default 5)
+    --recode-video FORMAT            Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm|mkv)
+    -k, --keep-video                 keeps the video file on disk after the post-processing; the video is erased by default
+    --no-post-overwrites             do not overwrite post-processed files; the post-processed files are overwritten by default
+    --embed-subs                     embed subtitles in the video (only for mp4 videos)
     --embed-thumbnail                embed thumbnail in the audio as cover art
     --add-metadata                   write metadata to the video file
-    --xattrs                         write metadata to the video file's xattrs
-                                     (using dublin core and xdg standards)
-    --fixup POLICY                   Automatically correct known faults of the
-                                     file. One of never (do nothing), warn (only
-                                     emit a warning), detect_or_warn(the
-                                     default; fix file if we can, warn
-                                     otherwise)
-    --prefer-avconv                  Prefer avconv over ffmpeg for running the
-                                     postprocessors (default)
-    --prefer-ffmpeg                  Prefer ffmpeg over avconv for running the
-                                     postprocessors
-    --ffmpeg-location PATH           Location of the ffmpeg/avconv binary;
-                                     either the path to the binary or its
-                                     containing directory.
-    --exec CMD                       Execute a command on the file after
-                                     downloading, similar to find's -exec
-                                     syntax. Example: --exec 'adb push {}
-                                     /sdcard/Music/ && rm {}'
-    --convert-subtitles FORMAT       Convert the subtitles to other format
-                                     (currently supported: srt|ass|vtt)
+    --xattrs                         write metadata to the video file's xattrs (using dublin core and xdg standards)
+    --fixup POLICY                   Automatically correct known faults of the file. One of never (do nothing), warn (only emit a warning), detect_or_warn(the default;
+                                     fix file if we can, warn otherwise)
+    --prefer-avconv                  Prefer avconv over ffmpeg for running the postprocessors (default)
+    --prefer-ffmpeg                  Prefer ffmpeg over avconv for running the postprocessors
+    --ffmpeg-location PATH           Location of the ffmpeg/avconv binary; either the path to the binary or its containing directory.
+    --exec CMD                       Execute a command on the file after downloading, similar to find's -exec syntax. Example: --exec 'adb push {} /sdcard/Music/ && rm
+                                     {}'
+    --convert-subtitles FORMAT       Convert the subtitles to other format (currently supported: srt|ass|vtt)
 
 # CONFIGURATION
 
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 5582348ba..3a49e52c9 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.28'
+__version__ = '2015.03.03'

From 499bfcbfd09e85f053d7e8943a8d47fed9349b0e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 3 Mar 2015 12:59:17 +0100
Subject: [PATCH 0585/6219] Make sure netrc works for all extractors with login
 support

Fixes #5112
---
 test/test_netrc.py                  | 26 ++++++++++++++++++++++++++
 youtube_dl/extractor/atresplayer.py |  1 +
 youtube_dl/extractor/crunchyroll.py |  1 +
 youtube_dl/extractor/gdcvault.py    |  1 +
 youtube_dl/extractor/lynda.py       |  1 +
 youtube_dl/extractor/twitch.py      |  1 +
 6 files changed, 31 insertions(+)
 create mode 100644 test/test_netrc.py

diff --git a/test/test_netrc.py b/test/test_netrc.py
new file mode 100644
index 000000000..7cf3a6a2e
--- /dev/null
+++ b/test/test_netrc.py
@@ -0,0 +1,26 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import os
+import sys
+import unittest
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+
+from youtube_dl.extractor import (
+    gen_extractors,
+)
+
+
+class TestNetRc(unittest.TestCase):
+    def test_netrc_present(self):
+        for ie in gen_extractors():
+            if not hasattr(ie, '_login'):
+                continue
+            self.assertTrue(
+                hasattr(ie, '_NETRC_MACHINE'),
+                'Extractor %s supports login, but is missing a _NETRC_MACHINE property' % ie.IE_NAME)
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/youtube_dl/extractor/atresplayer.py b/youtube_dl/extractor/atresplayer.py
index 7669e0e3d..29f8795d3 100644
--- a/youtube_dl/extractor/atresplayer.py
+++ b/youtube_dl/extractor/atresplayer.py
@@ -19,6 +19,7 @@ from ..utils import (
 
 class AtresPlayerIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?atresplayer\.com/television/[^/]+/[^/]+/[^/]+/(?P<id>.+?)_\d+\.html'
+    _NETRC_MACHINE = 'atresplayer'
     _TESTS = [
         {
             'url': 'http://www.atresplayer.com/television/programas/el-club-de-la-comedia/temporada-4/capitulo-10-especial-solidario-nochebuena_2014122100174.html',
diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index f1da7d09b..e64b88fbc 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -29,6 +29,7 @@ from ..aes import (
 
 class CrunchyrollIE(InfoExtractor):
     _VALID_URL = r'https?://(?:(?P<prefix>www|m)\.)?(?P<url>crunchyroll\.(?:com|fr)/(?:[^/]*/[^/?&]*?|media/\?id=)(?P<video_id>[0-9]+))(?:[/?&]|$)'
+    _NETRC_MACHINE = 'crunchyroll'
     _TESTS = [{
         'url': 'http://www.crunchyroll.com/wanna-be-the-strongest-in-the-world/episode-1-an-idol-wrestler-is-born-645513',
         'info_dict': {
diff --git a/youtube_dl/extractor/gdcvault.py b/youtube_dl/extractor/gdcvault.py
index f7b467b0a..51796f3a4 100644
--- a/youtube_dl/extractor/gdcvault.py
+++ b/youtube_dl/extractor/gdcvault.py
@@ -12,6 +12,7 @@ from ..utils import remove_end
 
 class GDCVaultIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?gdcvault\.com/play/(?P<id>\d+)/(?P<name>(\w|-)+)'
+    _NETRC_MACHINE = 'gdcvault'
     _TESTS = [
         {
             'url': 'http://www.gdcvault.com/play/1019721/Doki-Doki-Universe-Sweet-Simple',
diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index 1f02bef44..cfd3b14f4 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -19,6 +19,7 @@ class LyndaBaseIE(InfoExtractor):
     _LOGIN_URL = 'https://www.lynda.com/login/login.aspx'
     _SUCCESSFUL_LOGIN_REGEX = r'isLoggedIn: true'
     _ACCOUNT_CREDENTIALS_HINT = 'Use --username and --password options to provide lynda.com account credentials.'
+    _NETRC_MACHINE = 'lynda'
 
     def _real_initialize(self):
         self._login()
diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 4b0ce54df..8af136147 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -23,6 +23,7 @@ class TwitchBaseIE(InfoExtractor):
     _API_BASE = 'https://api.twitch.tv'
     _USHER_BASE = 'http://usher.twitch.tv'
     _LOGIN_URL = 'https://secure.twitch.tv/user/login'
+    _NETRC_MACHINE = 'twitch'
 
     def _handle_error(self, response):
         if not isinstance(response, dict):

From 2461f79d2ad9eee44644f6187e366125a29aa70f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 3 Mar 2015 13:56:06 +0100
Subject: [PATCH 0586/6219] [utils] Correct per-request proxy handling

---
 youtube_dl/YoutubeDL.py      |  3 ++-
 youtube_dl/extractor/letv.py | 10 ++--------
 youtube_dl/utils.py          | 15 +++++++++++++--
 3 files changed, 17 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 915963d96..df2aebb59 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1771,7 +1771,8 @@ class YoutubeDL(object):
         https_handler = make_HTTPS_handler(self.params, debuglevel=debuglevel)
         ydlh = YoutubeDLHandler(self.params, debuglevel=debuglevel)
         opener = compat_urllib_request.build_opener(
-            https_handler, proxy_handler, cookie_processor, ydlh)
+            proxy_handler, https_handler, cookie_processor, ydlh)
+
         # Delete the default user-agent header, which would otherwise apply in
         # cases where our custom HTTP handler doesn't come into play
         # (See https://github.com/rg3/youtube-dl/issues/1309 for details)
diff --git a/youtube_dl/extractor/letv.py b/youtube_dl/extractor/letv.py
index fd5fd260e..85eee141b 100644
--- a/youtube_dl/extractor/letv.py
+++ b/youtube_dl/extractor/letv.py
@@ -40,9 +40,6 @@ class LetvIE(InfoExtractor):
             'title': '美人天下01',
             'description': 'md5:f88573d9d7225ada1359eaf0dbf8bcda',
         },
-        'expected_warnings': [
-            'publish time'
-        ]
     }, {
         'note': 'This video is available only in Mainland China, thus a proxy is needed',
         'url': 'http://www.letv.com/ptv/vplay/1118082.html',
@@ -53,11 +50,8 @@ class LetvIE(InfoExtractor):
             'title': '与龙共舞 完整版',
             'description': 'md5:7506a5eeb1722bb9d4068f85024e3986',
         },
-        'expected_warnings': [
-            'publish time'
-        ],
         'params': {
-            'cn_verification_proxy': 'proxy.uku.im:8888'
+            'cn_verification_proxy': 'http://proxy.uku.im:8888'
         },
     }]
 
@@ -95,7 +89,7 @@ class LetvIE(InfoExtractor):
             'http://api.letv.com/mms/out/video/playJson?' + compat_urllib_parse.urlencode(params)
         )
         play_json_req.add_header(
-            'Ytdl-Request-Proxy',
+            'Ytdl-request-proxy',
             self._downloader.params.get('cn_verification_proxy'))
         play_json = self._download_json(
             play_json_req,
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index b568288fa..7426e2a1f 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1771,10 +1771,21 @@ def match_filter_func(filter_str):
 
 
 class PerRequestProxyHandler(compat_urllib_request.ProxyHandler):
+    def __init__(self, proxies=None):
+        # Set default handlers
+        for type in ('http', 'https'):
+            setattr(self, '%s_open' % type,
+                    lambda r, proxy='__noproxy__', type=type, meth=self.proxy_open:
+                        meth(r, proxy, type))
+        return compat_urllib_request.ProxyHandler.__init__(self, proxies)
+
     def proxy_open(self, req, proxy, type):
-        req_proxy = req.headers.get('Ytdl-Request-Proxy')
+        req_proxy = req.headers.get('Ytdl-request-proxy')
         if req_proxy is not None:
             proxy = req_proxy
-            del req.headers['Ytdl-Request-Proxy']
+            del req.headers['Ytdl-request-proxy']
+
+        if proxy == '__noproxy__':
+            return None  # No Proxy
         return compat_urllib_request.ProxyHandler.proxy_open(
             self, req, proxy, type)

From 7fde87c77da41f24faf34b37f5d05c029ac23d4f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 3 Mar 2015 13:59:38 +0100
Subject: [PATCH 0587/6219] release 2015.03.03.1

---
 README.md             | 2 ++
 youtube_dl/version.py | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index d05549125..5b9dd2cea 100644
--- a/README.md
+++ b/README.md
@@ -67,6 +67,8 @@ which means you can modify it, redistribute it or use it however you like.
     --source-address IP              Client-side IP address to bind to (experimental)
     -4, --force-ipv4                 Make all connections via IPv4 (experimental)
     -6, --force-ipv6                 Make all connections via IPv6 (experimental)
+    --cn-verification-proxy URL      Use this proxy to verify the IP address for some Chinese sites. The default proxy specified by --proxy (or none, if the options is
+                                     not present) is used for the actual downloading. (experimental)
 
 ## Video Selection:
     --playlist-start NUMBER          playlist video to start at (default is 1)
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 3a49e52c9..252933993 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.03.03'
+__version__ = '2015.03.03.1'

From e0d0572b731b79393a5c5121acf4a43017160936 Mon Sep 17 00:00:00 2001
From: chaos33 <919946h43k48>
Date: Tue, 3 Mar 2015 22:53:05 +0800
Subject: [PATCH 0588/6219] fix youporn extractor's json search regex

---
 youtube_dl/extractor/youporn.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youporn.py b/youtube_dl/extractor/youporn.py
index 107c9ac36..a278386d4 100644
--- a/youtube_dl/extractor/youporn.py
+++ b/youtube_dl/extractor/youporn.py
@@ -47,7 +47,7 @@ class YouPornIE(InfoExtractor):
 
         # Get JSON parameters
         json_params = self._search_regex(
-            r'var currentVideo = new Video\((.*)\)[,;]',
+            r'var videoJason = (.*)[,;]',
             webpage, 'JSON parameters')
         try:
             params = json.loads(json_params)

From 63fc80005782430180ed0d13ac6ab5ca497d333a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 3 Mar 2015 23:17:19 +0800
Subject: [PATCH 0589/6219] [Letv] Fix test_Letv and test_Letv_1 failures in
 python 3

---
 youtube_dl/extractor/letv.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/letv.py b/youtube_dl/extractor/letv.py
index 85eee141b..9ed81a199 100644
--- a/youtube_dl/extractor/letv.py
+++ b/youtube_dl/extractor/letv.py
@@ -88,9 +88,10 @@ class LetvIE(InfoExtractor):
         play_json_req = compat_urllib_request.Request(
             'http://api.letv.com/mms/out/video/playJson?' + compat_urllib_parse.urlencode(params)
         )
-        play_json_req.add_header(
-            'Ytdl-request-proxy',
-            self._downloader.params.get('cn_verification_proxy'))
+        cn_verification_proxy = self._downloader.params.get('cn_verification_proxy')
+        if cn_verification_proxy:
+            play_json_req.add_header('Ytdl-request-proxy', cn_verification_proxy)
+
         play_json = self._download_json(
             play_json_req,
             media_id, 'playJson data')

From 50c9949d7ae8f76d7a105481e0e692de3448c385 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 3 Mar 2015 21:39:04 +0600
Subject: [PATCH 0590/6219] [youporn] Imrove JSON regex and preserve the old
 one

---
 youtube_dl/extractor/youporn.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youporn.py b/youtube_dl/extractor/youporn.py
index a278386d4..e4c855ee0 100644
--- a/youtube_dl/extractor/youporn.py
+++ b/youtube_dl/extractor/youporn.py
@@ -47,7 +47,8 @@ class YouPornIE(InfoExtractor):
 
         # Get JSON parameters
         json_params = self._search_regex(
-            r'var videoJason = (.*)[,;]',
+            [r'var\s+videoJa?son\s*=\s*({.+?});',
+             r'var\s+currentVideo\s*=\s*new\s+Video\((.+?)\)[,;]'],
             webpage, 'JSON parameters')
         try:
             params = json.loads(json_params)

From 22d362831978ea2b83ea0c21a634237dc2406ab6 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Tue, 3 Mar 2015 18:39:23 +0200
Subject: [PATCH 0591/6219] [tvplay] Adapt _VALID_URL and test case to domain
 name change

---
 youtube_dl/extractor/tvplay.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tvplay.py b/youtube_dl/extractor/tvplay.py
index 9a53a3c74..e83e31a31 100644
--- a/youtube_dl/extractor/tvplay.py
+++ b/youtube_dl/extractor/tvplay.py
@@ -16,6 +16,7 @@ class TVPlayIE(InfoExtractor):
     _VALID_URL = r'''(?x)http://(?:www\.)?
         (?:tvplay\.lv/parraides|
            tv3play\.lt/programos|
+           play\.tv3\.lt/programos|
            tv3play\.ee/sisu|
            tv3play\.se/program|
            tv6play\.se/program|
@@ -45,7 +46,7 @@ class TVPlayIE(InfoExtractor):
             },
         },
         {
-            'url': 'http://www.tv3play.lt/programos/moterys-meluoja-geriau/409229?autostart=true',
+            'url': 'http://play.tv3.lt/programos/moterys-meluoja-geriau/409229?autostart=true',
             'info_dict': {
                 'id': '409229',
                 'ext': 'flv',

From dc570c4951d397ded7535393a9f0972c9cd0bc04 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Tue, 3 Mar 2015 18:41:34 +0200
Subject: [PATCH 0592/6219] [lrt] Pass --realtime to rtmpdump

---
 youtube_dl/extractor/lrt.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/lrt.py b/youtube_dl/extractor/lrt.py
index 9c2fbdd96..e3236f7b5 100644
--- a/youtube_dl/extractor/lrt.py
+++ b/youtube_dl/extractor/lrt.py
@@ -52,6 +52,7 @@ class LRTIE(InfoExtractor):
                     'url': data['streamer'],
                     'play_path': 'mp4:%s' % data['file'],
                     'preference': -1,
+                    'rtmp_real_time': True,
                 })
             else:
                 formats.extend(

From 123397317c9906bf53a23b6b426db0b08c70da32 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 3 Mar 2015 18:45:56 +0100
Subject: [PATCH 0593/6219] [downloader/http] Remove wrong '_hook_progress'
 call (fixes #5117)

---
 youtube_dl/downloader/http.py | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index 2e3dac825..5e9982799 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -218,12 +218,6 @@ class HttpFD(FileDownloader):
         if tmpfilename != '-':
             stream.close()
 
-        self._hook_progress({
-            'downloaded_bytes': byte_counter,
-            'total_bytes': data_len,
-            'tmpfilename': tmpfilename,
-            'status': 'error',
-        })
         if data_len is not None and byte_counter != data_len:
             raise ContentTooShortError(byte_counter, int(data_len))
         self.try_rename(tmpfilename, filename)

From f28fe66970370b5df649aa457e52cf99f6a49905 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 4 Mar 2015 12:14:38 +0100
Subject: [PATCH 0594/6219] [downloader/http] Add missing fields for
 _hook_progress call

It would fail if you run 'youtube-dl --no-part URL' a second time when the file has already been downloaded.

(Reported in Fedora: https://bugzilla.redhat.com/show_bug.cgi?id=1195779)
---
 youtube_dl/downloader/http.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index 5e9982799..4047d7167 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -92,6 +92,8 @@ class HttpFD(FileDownloader):
                             self._hook_progress({
                                 'filename': filename,
                                 'status': 'finished',
+                                'downloaded_bytes': resume_len,
+                                'total_bytes': resume_len,
                             })
                             return True
                         else:

From 12a129ec6dde840c23e8b1e5d7552de93fce7be4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 5 Mar 2015 02:36:53 +0600
Subject: [PATCH 0595/6219] [playwire] Add extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/playwire.py | 78 ++++++++++++++++++++++++++++++++
 2 files changed, 79 insertions(+)
 create mode 100644 youtube_dl/extractor/playwire.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index ffcc7d9ab..5ca534cdf 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -364,6 +364,7 @@ from .planetaplay import PlanetaPlayIE
 from .played import PlayedIE
 from .playfm import PlayFMIE
 from .playvid import PlayvidIE
+from .playwire import PlaywireIE
 from .podomatic import PodomaticIE
 from .pornhd import PornHdIE
 from .pornhub import (
diff --git a/youtube_dl/extractor/playwire.py b/youtube_dl/extractor/playwire.py
new file mode 100644
index 000000000..bdc71017b
--- /dev/null
+++ b/youtube_dl/extractor/playwire.py
@@ -0,0 +1,78 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    xpath_text,
+    float_or_none,
+    int_or_none,
+)
+
+
+class PlaywireIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:config|cdn)\.playwire\.com(?:/v2)?/(?P<publisher_id>\d+)/(?:videos/v2|embed|config)/(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'http://config.playwire.com/14907/videos/v2/3353705/player.json',
+        'md5': 'e6398701e3595888125729eaa2329ed9',
+        'info_dict': {
+            'id': '3353705',
+            'ext': 'mp4',
+            'title': 'S04_RM_UCL_Rus',
+            'thumbnail': 're:^http://.*\.png$',
+            'duration': 145.94,
+        },
+    }, {
+        'url': 'http://cdn.playwire.com/11625/embed/85228.html',
+        'only_matching': True,
+    }, {
+        'url': 'http://config.playwire.com/12421/videos/v2/3389892/zeus.json',
+        'only_matching': True,
+    }, {
+        'url': 'http://cdn.playwire.com/v2/12342/config/1532636.json',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        publisher_id, video_id = mobj.group('publisher_id'), mobj.group('id')
+
+        player = self._download_json(
+            'http://config.playwire.com/%s/videos/v2/%s/zeus.json' % (publisher_id, video_id),
+            video_id)
+
+        title = player['settings']['title']
+        duration = float_or_none(player.get('duration'), 1000)
+
+        content = player['content']
+        thumbnail = content.get('poster')
+        src = content['media']['f4m']
+
+        f4m = self._download_xml(src, video_id)
+        base_url = xpath_text(f4m, './{http://ns.adobe.com/f4m/1.0}baseURL', 'base url', fatal=True)
+        formats = []
+        for media in f4m.findall('./{http://ns.adobe.com/f4m/1.0}media'):
+            media_url = media.get('url')
+            if not media_url:
+                continue
+            tbr = int_or_none(media.get('bitrate'))
+            width = int_or_none(media.get('width'))
+            height = int_or_none(media.get('height'))
+            f = {
+                'url': '%s/%s' % (base_url, media.attrib['url']),
+                'tbr': tbr,
+                'width': width,
+                'height': height,
+            }
+            if not (tbr or width or height):
+                f['quality'] = 1 if '-hd.' in media_url else 0
+            formats.append(f)
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'thumbnail': thumbnail,
+            'duration': duration,
+            'formats': formats,
+        }

From c2ebea6580481a89c35f9447898b17241ad71ac0 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Thu, 5 Mar 2015 14:45:38 +0200
Subject: [PATCH 0596/6219] [extremetube] Fix extraction (Closes #5127)

---
 youtube_dl/extractor/extremetube.py | 30 ++++++++++++++++++-----------
 1 file changed, 19 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/extremetube.py b/youtube_dl/extractor/extremetube.py
index 36ba33128..c826a5404 100644
--- a/youtube_dl/extractor/extremetube.py
+++ b/youtube_dl/extractor/extremetube.py
@@ -4,11 +4,11 @@ import re
 
 from .common import InfoExtractor
 from ..compat import (
-    compat_urllib_parse_urlparse,
+    compat_parse_qs,
     compat_urllib_request,
-    compat_urllib_parse,
 )
 from ..utils import (
+    qualities,
     str_to_int,
 )
 
@@ -17,7 +17,7 @@ class ExtremeTubeIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?(?P<url>extremetube\.com/.*?video/.+?(?P<id>[0-9]+))(?:[/?&]|$)'
     _TESTS = [{
         'url': 'http://www.extremetube.com/video/music-video-14-british-euro-brit-european-cumshots-swallow-652431',
-        'md5': '1fb9228f5e3332ec8c057d6ac36f33e0',
+        'md5': '344d0c6d50e2f16b06e49ca011d8ac69',
         'info_dict': {
             'id': '652431',
             'ext': 'mp4',
@@ -49,19 +49,27 @@ class ExtremeTubeIE(InfoExtractor):
             r'Views:\s*</strong>\s*<span>([\d,\.]+)</span>',
             webpage, 'view count', fatal=False))
 
-        video_url = compat_urllib_parse.unquote(self._html_search_regex(
-            r'video_url=(.+?)&amp;', webpage, 'video_url'))
-        path = compat_urllib_parse_urlparse(video_url).path
-        format = path.split('/')[5].split('_')[:2]
-        format = "-".join(format)
+        flash_vars = compat_parse_qs(self._search_regex(
+            r'<param[^>]+?name="flashvars"[^>]+?value="([^"]+)"', webpage, 'flash vars'))
+
+        formats = []
+        quality = qualities(['180p', '240p', '360p', '480p', '720p', '1080p'])
+        for k, vals in flash_vars.items():
+            m = re.match(r'quality_(?P<quality>[0-9]+p)$', k)
+            if m is not None:
+                formats.append({
+                    'format_id': m.group('quality'),
+                    'quality': quality(m.group('quality')),
+                    'url': vals[0],
+                })
+
+        self._sort_formats(formats)
 
         return {
             'id': video_id,
             'title': video_title,
+            'formats': formats,
             'uploader': uploader,
             'view_count': view_count,
-            'url': video_url,
-            'format': format,
-            'format_id': format,
             'age_limit': 18,
         }

From 5ee6fc974e617ce2f8d9d62c416091a1daa6d802 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 6 Mar 2015 02:43:05 +0800
Subject: [PATCH 0597/6219] [sohu] Fix info extractor and add tests

---
 youtube_dl/extractor/sohu.py | 46 +++++++++++++++++++++++++++++-------
 1 file changed, 38 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/sohu.py b/youtube_dl/extractor/sohu.py
index c04791997..ef7ec51df 100644
--- a/youtube_dl/extractor/sohu.py
+++ b/youtube_dl/extractor/sohu.py
@@ -4,22 +4,42 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from .common import compat_str
+from ..utils import compat_str
+from ..compat import compat_urllib_request
 
 
 class SohuIE(InfoExtractor):
     _VALID_URL = r'https?://(?P<mytv>my\.)?tv\.sohu\.com/.+?/(?(mytv)|n)(?P<id>\d+)\.shtml.*?'
 
-    _TEST = {
+    _TESTS = [{
+        'note': 'This video is available only in Mainland China',
         'url': 'http://tv.sohu.com/20130724/n382479172.shtml#super',
-        'md5': 'bde8d9a6ffd82c63a1eefaef4eeefec7',
+        'md5': '29175c8cadd8b5cc4055001e85d6b372',
         'info_dict': {
             'id': '382479172',
             'ext': 'mp4',
             'title': 'MV：Far East Movement《The Illest》',
         },
-        'skip': 'Only available from China',
-    }
+        'params': {
+            'cn_verification_proxy': 'proxy.uku.im:8888'
+        }
+    }, {
+        'url': 'http://tv.sohu.com/20150305/n409385080.shtml',
+        'md5': '699060e75cf58858dd47fb9c03c42cfb',
+        'info_dict': {
+            'id': '409385080',
+            'ext': 'mp4',
+            'title': '《2015湖南卫视羊年元宵晚会》唐嫣《花好月圆》',
+        }
+    }, {
+        'url': 'http://my.tv.sohu.com/us/232799889/78693464.shtml',
+        'md5': '9bf34be48f2f4dadcb226c74127e203c',
+        'info_dict': {
+            'id': '78693464',
+            'ext': 'mp4',
+            'title': '【爱范品】第31期：MWC见不到的奇葩手机',
+        }
+    }]
 
     def _real_extract(self, url):
 
@@ -29,9 +49,14 @@ class SohuIE(InfoExtractor):
             else:
                 base_data_url = 'http://hot.vrs.sohu.com/vrs_flash.action?vid='
 
-            return self._download_json(
-                base_data_url + vid_id, video_id,
-                'Downloading JSON data for %s' % vid_id)
+            req = compat_urllib_request.Request(base_data_url + vid_id)
+
+            cn_verification_proxy = self._downloader.params.get('cn_verification_proxy')
+            if cn_verification_proxy:
+                req.add_header('Ytdl-request-proxy', cn_verification_proxy)
+
+            return self._download_json(req, video_id,
+                                       'Downloading JSON data for %s' % vid_id)
 
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
@@ -77,6 +102,11 @@ class SohuIE(InfoExtractor):
                     % (format_id, i + 1, part_count))
 
                 part_info = part_str.split('|')
+
+                # Sanitize URL to prevent download failure
+                if part_info[0][-1] == '/' and su[i][0] == '/':
+                    su[i] = su[i][1:]
+
                 video_url = '%s%s?key=%s' % (part_info[0], su[i], part_info[3])
 
                 formats.append({

From 5c7495a19429e3b27c003a4bd5bb96ed1e3a4932 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 6 Mar 2015 02:48:27 +0800
Subject: [PATCH 0598/6219] [sohu] Correct wrong imports

---
 youtube_dl/extractor/sohu.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/sohu.py b/youtube_dl/extractor/sohu.py
index ef7ec51df..335e84fab 100644
--- a/youtube_dl/extractor/sohu.py
+++ b/youtube_dl/extractor/sohu.py
@@ -4,8 +4,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import compat_str
-from ..compat import compat_urllib_request
+from ..compat import (
+    compat_str,
+    compat_urllib_request
+)
 
 
 class SohuIE(InfoExtractor):

From 9609f02e3ca87e56fc5571e41040ddbcf2a79a73 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 5 Mar 2015 22:34:56 +0100
Subject: [PATCH 0599/6219] [vidme] Modernize

---
 youtube_dl/extractor/vidme.py | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/vidme.py b/youtube_dl/extractor/vidme.py
index 5c89824c1..339c3d897 100644
--- a/youtube_dl/extractor/vidme.py
+++ b/youtube_dl/extractor/vidme.py
@@ -1,7 +1,5 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..utils import (
     int_or_none,
@@ -28,12 +26,11 @@ class VidmeIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        video_url = self._html_search_regex(r'<source src="([^"]+)"', webpage, 'video URL')
+        video_url = self._html_search_regex(
+            r'<source src="([^"]+)"', webpage, 'video URL')
 
         title = self._og_search_title(webpage)
         description = self._og_search_description(webpage, default='')

From 55969016e96fded28b97b2ef3bbf66efa83d6afb Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 6 Mar 2015 12:43:49 +0800
Subject: [PATCH 0600/6219] [utils] Add a function to sanitize consecutive
 slashes in URLs

---
 test/test_utils.py           | 16 ++++++++++++++++
 youtube_dl/extractor/sohu.py |  8 +++-----
 youtube_dl/utils.py          | 15 +++++++++++++++
 3 files changed, 34 insertions(+), 5 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 64fad58ad..e02069c4d 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -54,6 +54,7 @@ from youtube_dl.utils import (
     xpath_with_ns,
     render_table,
     match_str,
+    url_sanitize_consecutive_slashes,
 )
 
 
@@ -501,6 +502,21 @@ ffmpeg version 2.4.4 Copyright (c) 2000-2014 the FFmpeg ...'''), '2.4.4')
             'like_count > 100 & dislike_count <? 50 & description',
             {'like_count': 190, 'dislike_count': 10}))
 
+    def test_url_sanitize_consecutive_slashes(self):
+        self.assertEqual(url_sanitize_consecutive_slashes(
+            'http://hostname/foo//bar/filename.html'),
+            'http://hostname/foo/bar/filename.html')
+        self.assertEqual(url_sanitize_consecutive_slashes(
+            'http://hostname//foo/bar/filename.html'),
+            'http://hostname/foo/bar/filename.html')
+        self.assertEqual(url_sanitize_consecutive_slashes(
+            'http://hostname//'), 'http://hostname/')
+        self.assertEqual(url_sanitize_consecutive_slashes(
+            'http://hostname/foo/bar/filename.html'),
+            'http://hostname/foo/bar/filename.html')
+        self.assertEqual(url_sanitize_consecutive_slashes(
+            'http://hostname/'), 'http://hostname/')
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/extractor/sohu.py b/youtube_dl/extractor/sohu.py
index 335e84fab..5adc734d9 100644
--- a/youtube_dl/extractor/sohu.py
+++ b/youtube_dl/extractor/sohu.py
@@ -8,6 +8,7 @@ from ..compat import (
     compat_str,
     compat_urllib_request
 )
+from ..utils import url_sanitize_consecutive_slashes
 
 
 class SohuIE(InfoExtractor):
@@ -105,11 +106,8 @@ class SohuIE(InfoExtractor):
 
                 part_info = part_str.split('|')
 
-                # Sanitize URL to prevent download failure
-                if part_info[0][-1] == '/' and su[i][0] == '/':
-                    su[i] = su[i][1:]
-
-                video_url = '%s%s?key=%s' % (part_info[0], su[i], part_info[3])
+                video_url = url_sanitize_consecutive_slashes(
+                    '%s%s?key=%s' % (part_info[0], su[i], part_info[3]))
 
                 formats.append({
                     'url': video_url,
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 7426e2a1f..ef14f9a36 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1789,3 +1789,18 @@ class PerRequestProxyHandler(compat_urllib_request.ProxyHandler):
             return None  # No Proxy
         return compat_urllib_request.ProxyHandler.proxy_open(
             self, req, proxy, type)
+
+
+def url_sanitize_consecutive_slashes(url):
+    """Sanitize URLs with consecutive slashes
+
+    For example, transform both
+        http://hostname/foo//bar/filename.html
+    and
+        http://hostname//foo/bar/filename.html
+    into
+        http://hostname/foo/bar/filename.html
+    """
+    parsed_url = list(compat_urlparse.urlparse(url))
+    parsed_url[2] = re.sub(r'/{2,}', '/', parsed_url[2])
+    return compat_urlparse.urlunparse(parsed_url)

From 7feddd9fc7a4a482edbc9a07d6295d418989bd67 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 6 Mar 2015 10:44:24 +0100
Subject: [PATCH 0601/6219] [travis] Declare 3.2 (Fixes #5144)

---
 .travis.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.travis.yml b/.travis.yml
index fb34299fc..511bee64c 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -2,6 +2,7 @@ language: python
 python:
   - "2.6"
   - "2.7"
+  - "3.2"
   - "3.3"
   - "3.4"
 before_install:

From a0bb7c5593b08339f34dc43d9bc1b199ddc3ff7d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 6 Mar 2015 10:49:42 +0100
Subject: [PATCH 0602/6219] [extractor/common] Improve m3u format IDs (#5143)

---
 youtube_dl/extractor/common.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index cf39c0c21..8ff76342f 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -883,8 +883,12 @@ class InfoExtractor(object):
                     formats.append({'url': format_url(line)})
                     continue
                 tbr = int_or_none(last_info.get('BANDWIDTH'), scale=1000)
+                format_id = last_media.get('NAME')
+                if not format_id:
+                    format_id = '-'.join(filter(None, [
+                        m3u8_id, 'm3u8-%d' % (tbr if tbr else len(formats))]))
                 f = {
-                    'format_id': '-'.join(filter(None, [m3u8_id, 'm3u8-%d' % (tbr if tbr else len(formats))])),
+                    'format_id': format_id,
                     'url': format_url(line.strip()),
                     'tbr': tbr,
                     'ext': ext,

From d0e958c71c8c561d1c3917d114370fcb85972181 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 6 Mar 2015 10:53:49 +0100
Subject: [PATCH 0603/6219] [twitch:vod] Prefer source stream (Fixes #5143)

---
 youtube_dl/extractor/twitch.py | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 8af136147..b058891bd 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -85,6 +85,14 @@ class TwitchBaseIE(InfoExtractor):
             raise ExtractorError(
                 'Unable to login: %s' % m.group('msg').strip(), expected=True)
 
+    def _prefer_source(self, formats):
+        try:
+            source = next(f for f in formats if f['format_id'] == 'Source')
+            source['preference'] = 10
+        except StopIteration:
+            pass  # No Source stream present
+        self._sort_formats(formats)
+
 
 class TwitchItemBaseIE(TwitchBaseIE):
     def _download_info(self, item, item_id):
@@ -209,6 +217,7 @@ class TwitchVodIE(TwitchItemBaseIE):
             '%s/vod/%s?nauth=%s&nauthsig=%s'
             % (self._USHER_BASE, item_id, access_token['token'], access_token['sig']),
             item_id, 'mp4')
+        self._prefer_source(formats)
         info['formats'] = formats
         return info
 
@@ -357,13 +366,7 @@ class TwitchStreamIE(TwitchBaseIE):
             '%s/api/channel/hls/%s.m3u8?%s'
             % (self._USHER_BASE, channel_id, compat_urllib_parse.urlencode(query).encode('utf-8')),
             channel_id, 'mp4')
-
-        # prefer the 'source' stream, the others are limited to 30 fps
-        def _sort_source(f):
-            if f.get('m3u8_media') is not None and f['m3u8_media'].get('NAME') == 'Source':
-                return 1
-            return 0
-        formats = sorted(formats, key=_sort_source)
+        self._prefer_source(formats)
 
         view_count = stream.get('viewers')
         timestamp = parse_iso8601(stream.get('created_at'))

From 8dc9d361c22754e6b6b52136d496d754dea65b46 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 6 Mar 2015 22:52:50 +0600
Subject: [PATCH 0604/6219] [extractor/common] Fix format_id when `last_media`
 is None and always include `m3u8_id` if present

The rationale behind `m3u8_id` was to resolve duplicates when processing several m3u8 playlists within the same media that give equal resulting `format_id`'s,
e.g. `youtube-dl http://www.rts.ch/play/tv/passe-moi-les-jumelles/video/la-fee-des-bois-mustang-les-chemins-du-vent?id=3854925 -F`
---
 youtube_dl/extractor/common.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 8ff76342f..06bce12e0 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -883,12 +883,13 @@ class InfoExtractor(object):
                     formats.append({'url': format_url(line)})
                     continue
                 tbr = int_or_none(last_info.get('BANDWIDTH'), scale=1000)
-                format_id = last_media.get('NAME')
-                if not format_id:
-                    format_id = '-'.join(filter(None, [
-                        m3u8_id, 'm3u8-%d' % (tbr if tbr else len(formats))]))
+                format_id = []
+                if m3u8_id:
+                    format_id.append(m3u8_id)
+                last_media_name = last_media.get('NAME') if last_media else None
+                format_id.append(last_media_name if last_media_name else '%d' % (tbr if tbr else len(formats)))
                 f = {
-                    'format_id': format_id,
+                    'format_id': '-'.join(format_id),
                     'url': format_url(line.strip()),
                     'tbr': tbr,
                     'ext': ext,

From bd05aa4e24774a9aa9f68a8379900e6b756aae11 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 7 Mar 2015 00:53:52 +0800
Subject: [PATCH 0605/6219] [Yam] Add an error detection and update test cases

---
 youtube_dl/extractor/yam.py | 34 ++++++++++++++++++++++++++++------
 1 file changed, 28 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/yam.py b/youtube_dl/extractor/yam.py
index b294767c5..19ad74d04 100644
--- a/youtube_dl/extractor/yam.py
+++ b/youtube_dl/extractor/yam.py
@@ -8,6 +8,7 @@ from ..compat import compat_urlparse
 from ..utils import (
     float_or_none,
     month_by_abbreviation,
+    ExtractorError,
 )
 
 
@@ -28,23 +29,44 @@ class YamIE(InfoExtractor):
         }
     }, {
         # An external video hosted on YouTube
-        'url': 'http://mymedia.yam.com/m/3598173',
-        'md5': '0238ceec479c654e8c2f1223755bf3e9',
+        'url': 'http://mymedia.yam.com/m/3599430',
+        'md5': '03127cf10d8f35d120a9e8e52e3b17c6',
         'info_dict': {
-            'id': 'pJ2Deys283c',
+            'id': 'CNpEoQlrIgA',
             'ext': 'mp4',
-            'upload_date': '20150202',
+            'upload_date': '20150306',
             'uploader': '新莊社大瑜伽社',
-            'description': 'md5:f5cc72f0baf259a70fb731654b0d2eff',
+            'description': 'md5:11e2e405311633ace874f2e6226c8b17',
             'uploader_id': '2323agoy',
-            'title': '外婆的澎湖灣KTV-潘安邦',
+            'title': '20090412陽明山二子坪-1',
         }
+    }, {
+        'url': 'http://mymedia.yam.com/m/3598173',
+        'info_dict': {
+            'id': '3598173',
+            'ext': 'mp4',
+        },
+        'skip': 'cause Yam system error',
+    }, {
+        'url': 'http://mymedia.yam.com/m/3599437',
+        'info_dict': {
+            'id': '3599437',
+            'ext': 'mp4',
+        },
+        'skip': 'invalid YouTube URL',
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
         page = self._download_webpage(url, video_id)
 
+        # Check for errors
+        system_msg = self._html_search_regex(
+            r'系統訊息(?:<br>|\n|\r)*([^<>]+)<br>', page, 'system message',
+            default=None)
+        if system_msg:
+            raise ExtractorError(system_msg, expected=True)
+
         # Is it hosted externally on YouTube?
         youtube_url = self._html_search_regex(
             r'<embed src="(http://www.youtube.com/[^"]+)"',

From f207019ce51e18f7151dc9ab332cc8d0933021b6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 6 Mar 2015 22:53:53 +0600
Subject: [PATCH 0606/6219] [extractor/common] Remove 'm3u8' from quality
 selection URL

---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 06bce12e0..f9e8e2bad 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -839,7 +839,7 @@ class InfoExtractor(object):
                               m3u8_id=None):
 
         formats = [{
-            'format_id': '-'.join(filter(None, [m3u8_id, 'm3u8-meta'])),
+            'format_id': '-'.join(filter(None, [m3u8_id, 'meta'])),
             'url': m3u8_url,
             'ext': ext,
             'protocol': 'm3u8',

From 3c6f245083db85d7ba9ae5444561339dc1b02671 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 6 Mar 2015 18:16:56 +0100
Subject: [PATCH 0607/6219] [vimeo] Fix upload date extraction

---
 youtube_dl/extractor/vimeo.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 8f540f578..3771769e1 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -20,6 +20,7 @@ from ..utils import (
     RegexNotFoundError,
     smuggle_url,
     std_headers,
+    unified_strdate,
     unsmuggle_url,
     urlencode_postdata,
 )
@@ -140,6 +141,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
                 'description': 'md5:8678b246399b070816b12313e8b4eb5c',
                 'uploader_id': 'atencio',
                 'uploader': 'Peter Atencio',
+                'upload_date': '20130927',
                 'duration': 187,
             },
         },
@@ -323,9 +325,9 @@ class VimeoIE(VimeoBaseInfoExtractor):
 
         # Extract upload date
         video_upload_date = None
-        mobj = re.search(r'<meta itemprop="dateCreated" content="(\d{4})-(\d{2})-(\d{2})T', webpage)
+        mobj = re.search(r'<time[^>]+datetime="([^"]+)"', webpage)
         if mobj is not None:
-            video_upload_date = mobj.group(1) + mobj.group(2) + mobj.group(3)
+            video_upload_date = unified_strdate(mobj.group(1))
 
         try:
             view_count = int(self._search_regex(r'UserPlays:(\d+)', webpage, 'view count'))

From 9c85b5376d3b784d1bfdead7bc473b12d7349614 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 6 Mar 2015 19:08:27 +0100
Subject: [PATCH 0608/6219] [vimeo] Fix and use '_verify_video_password'
 (#5001)

It only supports verifying the password over https now.

Use it instead of manually setting the 'password' cookie because it allows to check if the password is correct.
---
 youtube_dl/extractor/vimeo.py | 20 ++++++--------------
 1 file changed, 6 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 3771769e1..61f2d1ee8 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -4,7 +4,6 @@ from __future__ import unicode_literals
 import json
 import re
 import itertools
-import hashlib
 
 from .common import InfoExtractor
 from ..compat import (
@@ -178,17 +177,15 @@ class VimeoIE(VimeoBaseInfoExtractor):
         password = self._downloader.params.get('videopassword', None)
         if password is None:
             raise ExtractorError('This video is protected by a password, use the --video-password option', expected=True)
-        token = self._search_regex(r'xsrft: \'(.*?)\'', webpage, 'login token')
-        data = compat_urllib_parse.urlencode({
+        token = self._search_regex(r'xsrft = \'(.*?)\'', webpage, 'login token')
+        data = urlencode_postdata({
             'password': password,
             'token': token,
         })
-        # I didn't manage to use the password with https
-        if url.startswith('https'):
-            pass_url = url.replace('https', 'http')
-        else:
-            pass_url = url
-        password_request = compat_urllib_request.Request(pass_url + '/password', data)
+        if url.startswith('http://'):
+            # vimeo only supports https now, but the user can give an http url
+            url = url.replace('http://', 'https://')
+        password_request = compat_urllib_request.Request(url + '/password', data)
         password_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
         password_request.add_header('Cookie', 'xsrft=%s' % token)
         return self._download_webpage(
@@ -227,11 +224,6 @@ class VimeoIE(VimeoBaseInfoExtractor):
         if mobj.group('pro') or mobj.group('player'):
             url = 'http://player.vimeo.com/video/' + video_id
 
-        password = self._downloader.params.get('videopassword', None)
-        if password:
-            headers['Cookie'] = '%s_password=%s' % (
-                video_id, hashlib.md5(password.encode('utf-8')).hexdigest())
-
         # Retrieve video webpage to extract further information
         request = compat_urllib_request.Request(url, None, headers)
         try:

From d1508cd68d0a3e470860dc7f225509b2e422d34b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 6 Mar 2015 22:16:26 +0100
Subject: [PATCH 0609/6219] [vimeo:album] Fix password protected videos

Since it only uses https now, don't recognize http urls.
---
 youtube_dl/extractor/vimeo.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 61f2d1ee8..0d1c92633 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -429,10 +429,10 @@ class VimeoChannelIE(InfoExtractor):
             name="([^"]+)"\s+
             value="([^"]*)"
             ''', login_form))
-        token = self._search_regex(r'xsrft: \'(.*?)\'', webpage, 'login token')
+        token = self._search_regex(r'xsrft = \'(.*?)\'', webpage, 'login token')
         fields['token'] = token
         fields['password'] = password
-        post = compat_urllib_parse.urlencode(fields)
+        post = urlencode_postdata(fields)
         password_path = self._search_regex(
             r'action="([^"]+)"', login_form, 'password URL')
         password_url = compat_urlparse.urljoin(page_url, password_path)
@@ -494,10 +494,10 @@ class VimeoUserIE(VimeoChannelIE):
 
 class VimeoAlbumIE(VimeoChannelIE):
     IE_NAME = 'vimeo:album'
-    _VALID_URL = r'https?://vimeo\.com/album/(?P<id>\d+)'
+    _VALID_URL = r'https://vimeo\.com/album/(?P<id>\d+)'
     _TITLE_RE = r'<header id="page_header">\n\s*<h1>(.*?)</h1>'
     _TESTS = [{
-        'url': 'http://vimeo.com/album/2632481',
+        'url': 'https://vimeo.com/album/2632481',
         'info_dict': {
             'id': '2632481',
             'title': 'Staff Favorites: November 2013',
@@ -521,7 +521,7 @@ class VimeoAlbumIE(VimeoChannelIE):
 
     def _real_extract(self, url):
         album_id = self._match_id(url)
-        return self._extract_videos(album_id, 'http://vimeo.com/album/%s' % album_id)
+        return self._extract_videos(album_id, 'https://vimeo.com/album/%s' % album_id)
 
 
 class VimeoGroupsIE(VimeoAlbumIE):

From 61e00a97756de178db3e2a55b4292273532c9420 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 6 Mar 2015 22:39:05 +0100
Subject: [PATCH 0610/6219] [vimeo] Use https for player.vimeo.com urls (closes
 #5147)

---
 youtube_dl/extractor/vimeo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 0d1c92633..b84a83ba6 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -222,7 +222,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
         video_id = mobj.group('id')
         orig_url = url
         if mobj.group('pro') or mobj.group('player'):
-            url = 'http://player.vimeo.com/video/' + video_id
+            url = 'https://player.vimeo.com/video/' + video_id
 
         # Retrieve video webpage to extract further information
         request = compat_urllib_request.Request(url, None, headers)

From 23ba76bc0eae4931c1adbd00de45a9ed41848283 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 6 Mar 2015 22:45:05 +0100
Subject: [PATCH 0611/6219] [dailymotion] Replace test

It has been removed.
---
 youtube_dl/extractor/dailymotion.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index 42b20a46d..4f67c3aac 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -46,13 +46,13 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
 
     _TESTS = [
         {
-            'url': 'http://www.dailymotion.com/video/x33vw9_tutoriel-de-youtubeur-dl-des-video_tech',
-            'md5': '392c4b85a60a90dc4792da41ce3144eb',
+            'url': 'https://www.dailymotion.com/video/x2iuewm_steam-machine-models-pricing-listed-on-steam-store-ign-news_videogames',
+            'md5': '2137c41a8e78554bb09225b8eb322406',
             'info_dict': {
-                'id': 'x33vw9',
+                'id': 'x2iuewm',
                 'ext': 'mp4',
-                'uploader': 'Amphora Alex and Van .',
-                'title': 'Tutoriel de Youtubeur"DL DES VIDEO DE YOUTUBE"',
+                'uploader': 'IGN',
+                'title': 'Steam Machine Models, Pricing Listed on Steam Store - IGN News',
             }
         },
         # Vevo video

From a172d96292abda037b23b1f2b14dd2c6e65b56bb Mon Sep 17 00:00:00 2001
From: bonfy <foreverbonfy@163.com>
Date: Sat, 7 Mar 2015 14:05:56 +0800
Subject: [PATCH 0612/6219] [douyutv] Add new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/douyutv.py  | 59 ++++++++++++++++++++++++++++++++
 2 files changed, 60 insertions(+)
 create mode 100644 youtube_dl/extractor/douyutv.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index ffcc7d9ab..7d0798176 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -106,6 +106,7 @@ from .dctp import DctpTvIE
 from .deezer import DeezerPlaylistIE
 from .dfb import DFBIE
 from .dotsub import DotsubIE
+from .douyutv import DouyutvIE
 from .dreisat import DreiSatIE
 from .drbonanza import DRBonanzaIE
 from .drtuber import DrTuberIE
diff --git a/youtube_dl/extractor/douyutv.py b/youtube_dl/extractor/douyutv.py
new file mode 100644
index 000000000..e9b92eb3b
--- /dev/null
+++ b/youtube_dl/extractor/douyutv.py
@@ -0,0 +1,59 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+from ..utils import (
+    ExtractorError,
+)
+
+class DouyutvIE(InfoExtractor):
+    _VALID_URL = r'http://(?:www\.)?douyutv\.com/(?P<id>[A-Za-z0-9]+)'
+
+    '''
+    show_status: 1 直播中 ，2 没有直播
+    '''
+
+    _TEST = {
+        'url': 'http://www.douyutv.com/iseven',
+        'info_dict': {
+            'id': 'iseven',
+            'title': '清晨醒脑！T-ara根本停不下来！',
+            'ext': 'flv',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'is_live': True,
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        info_url = 'http://www.douyutv.com/api/client/room/' + video_id
+
+        config = self._download_json(info_url, video_id)
+
+        error_code = config.get('error')
+        show_status = config['data'].get('show_status')
+        if error_code is not 0:
+            raise ExtractorError('Server reported error %i' % error_code,
+                                 expected=True)
+
+        if show_status == '2':
+            raise ExtractorError('The live show has not yet started',
+                                 expected=True)
+
+        title = config['data'].get('room_name')
+        rtmp_url = config['data'].get('rtmp_url')
+        rtmp_live = config['data'].get('rtmp_live')
+        thumbnail = config['data'].get('room_src') 
+
+        url = rtmp_url+'/'+rtmp_live
+
+        return {
+            'id': video_id,
+            'title': title,
+            'ext':'flv',
+            'url': url,
+            'thumbnail': thumbnail,
+            'is_live': True,
+            # TODO more properties (see youtube_dl/extractor/common.py)
+        }
\ No newline at end of file

From 14137b5781305d3c9db523b14283128335f62135 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 7 Mar 2015 17:31:03 +0600
Subject: [PATCH 0613/6219] [orf:iptv] Add extractor (Closes #5140)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/orf.py      | 94 ++++++++++++++++++++++++++++++++
 2 files changed, 95 insertions(+)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 5ca534cdf..f72d06fc4 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -354,6 +354,7 @@ from .orf import (
     ORFTVthekIE,
     ORFOE1IE,
     ORFFM4IE,
+    ORFIPTVIE,
 )
 from .parliamentliveuk import ParliamentLiveUKIE
 from .patreon import PatreonIE
diff --git a/youtube_dl/extractor/orf.py b/youtube_dl/extractor/orf.py
index 4e293392b..ca1a5bb3c 100644
--- a/youtube_dl/extractor/orf.py
+++ b/youtube_dl/extractor/orf.py
@@ -11,6 +11,11 @@ from ..utils import (
     HEADRequest,
     unified_strdate,
     ExtractorError,
+    strip_jsonp,
+    int_or_none,
+    float_or_none,
+    determine_ext,
+    remove_end,
 )
 
 
@@ -197,3 +202,92 @@ class ORFFM4IE(InfoExtractor):
             'description': data['subtitle'],
             'entries': entries
         }
+
+
+class ORFIPTVIE(InfoExtractor):
+    IE_NAME = 'orf:iptv'
+    IE_DESC = 'iptv.ORF.at'
+    _VALID_URL = r'http://iptv\.orf\.at/(?:#/)?stories/(?P<id>\d+)'
+
+    _TEST = {
+        'url': 'http://iptv.orf.at/stories/2267952',
+        'md5': '26ffa4bab6dbce1eee78bbc7021016cd',
+        'info_dict': {
+            'id': '339775',
+            'ext': 'flv',
+            'title': 'Kreml-Kritiker Nawalny wieder frei',
+            'description': 'md5:6f24e7f546d364dacd0e616a9e409236',
+            'duration': 84.729,
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'upload_date': '20150306',
+        },
+    }
+
+    def _real_extract(self, url):
+        story_id = self._match_id(url)
+
+        webpage = self._download_webpage(
+            'http://iptv.orf.at/stories/%s' % story_id, story_id)
+
+        video_id = self._search_regex(
+            r'data-video(?:id)?="(\d+)"', webpage, 'video id')
+
+        data = self._download_json(
+            'http://bits.orf.at/filehandler/static-api/json/current/data.json?file=%s' % video_id,
+            video_id)[0]
+
+        duration = float_or_none(data['duration'], 1000)
+
+        video = data['sources']['default']
+        load_balancer_url = video['loadBalancerUrl']
+        abr = int_or_none(video.get('audioBitrate'))
+        vbr = int_or_none(video.get('bitrate'))
+        fps = int_or_none(video.get('videoFps'))
+        width = int_or_none(video.get('videoWidth'))
+        height = int_or_none(video.get('videoHeight'))
+        thumbnail = video.get('preview')
+
+        rendition = self._download_json(
+            load_balancer_url, video_id, transform_source=strip_jsonp)
+
+        f = {
+            'abr': abr,
+            'vbr': vbr,
+            'fps': fps,
+            'width': width,
+            'height': height,
+        }
+
+        formats = []
+        for format_id, format_url in rendition['redirect'].items():
+            if format_id == 'rtmp':
+                ff = f.copy()
+                ff.update({
+                    'url': format_url,
+                    'format_id': format_id,
+                })
+                formats.append(ff)
+            elif determine_ext(format_url) == 'f4m':
+                formats.extend(self._extract_f4m_formats(
+                    format_url, video_id, f4m_id=format_id))
+            elif determine_ext(format_url) == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    format_url, video_id, 'mp4', m3u8_id=format_id))
+            else:
+                continue
+        self._sort_formats(formats)
+
+        title = remove_end(self._og_search_title(webpage), ' - iptv.ORF.at')
+        description = self._og_search_description(webpage)
+        upload_date = unified_strdate(self._html_search_meta(
+            'dc.date', webpage, 'upload date'))
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'duration': duration,
+            'thumbnail': thumbnail,
+            'upload_date': upload_date,
+            'formats': formats,
+        }

From 98998cded6e179b35055177f75fc921d99ee5938 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 7 Mar 2015 18:59:06 +0600
Subject: [PATCH 0614/6219] [youtube:search_url] Fix extraction (Closes #5155)

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 3690f8021..27c8c4453 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1532,7 +1532,7 @@ class YoutubeSearchURLIE(InfoExtractor):
 
         webpage = self._download_webpage(url, query)
         result_code = self._search_regex(
-            r'(?s)<ol class="item-section"(.*?)</ol>', webpage, 'result HTML')
+            r'(?s)<ol[^>]+class="item-section"(.*?)</ol>', webpage, 'result HTML')
 
         part_codes = re.findall(
             r'(?s)<h3 class="yt-lockup-title">(.*?)</h3>', result_code)

From 0bf79ac455ff5dc52b61f26c254f83e7c041d75c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 7 Mar 2015 22:16:23 +0600
Subject: [PATCH 0615/6219] [eagleplatform] Add extractor

---
 youtube_dl/extractor/__init__.py      |  1 +
 youtube_dl/extractor/eagleplatform.py | 99 +++++++++++++++++++++++++++
 2 files changed, 100 insertions(+)
 create mode 100644 youtube_dl/extractor/eagleplatform.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index f72d06fc4..b489d5770 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -116,6 +116,7 @@ from .defense import DefenseGouvFrIE
 from .discovery import DiscoveryIE
 from .divxstage import DivxStageIE
 from .dropbox import DropboxIE
+from .eagleplatform import EaglePlatformIE
 from .ebaumsworld import EbaumsWorldIE
 from .echomsk import EchoMskIE
 from .ehow import EHowIE
diff --git a/youtube_dl/extractor/eagleplatform.py b/youtube_dl/extractor/eagleplatform.py
new file mode 100644
index 000000000..654f746fe
--- /dev/null
+++ b/youtube_dl/extractor/eagleplatform.py
@@ -0,0 +1,99 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    int_or_none,
+)
+
+
+class EaglePlatformIE(InfoExtractor):
+    _VALID_URL = r'''(?x)
+                    (?:
+                        eagleplatform:(?P<custom_host>[^/]+):|
+                        https?://(?P<host>.+?\.media\.eagleplatform\.com)/index/player\?.*\brecord_id=
+                    )
+                    (?P<id>\d+)
+                '''
+    _TESTS = [{
+        # http://lenta.ru/news/2015/03/06/navalny/
+        'url': 'http://lentaru.media.eagleplatform.com/index/player?player=new&record_id=227304&player_template_id=5201',
+        'md5': '0b7994faa2bd5c0f69a3db6db28d078d',
+        'info_dict': {
+            'id': '227304',
+            'ext': 'mp4',
+            'title': 'Навальный вышел на свободу',
+            'description': 'md5:d97861ac9ae77377f3f20eaf9d04b4f5',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 87,
+            'view_count': int,
+            'age_limit': 0,
+        },
+    }, {
+        # http://muz-tv.ru/play/7129/
+        # http://media.clipyou.ru/index/player?record_id=12820&width=730&height=415&autoplay=true
+        'url': 'eagleplatform:media.clipyou.ru:12820',
+        'md5': '6c2ebeab03b739597ce8d86339d5a905',
+        'info_dict': {
+            'id': '12820',
+            'ext': 'mp4',
+            'title': "'O Sole Mio",
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 216,
+            'view_count': int,
+        },
+    }]
+
+    def _handle_error(self, response):
+        print(response['status'])
+        status = int_or_none(response.get('status', 200))
+        if status != 200:
+            raise ExtractorError(' '.join(response['errors']), expected=True)
+
+    def _download_json(self, url_or_request, video_id, note='Downloading JSON metadata'):
+        response = super(EaglePlatformIE, self)._download_json(url_or_request, video_id, note)
+        self._handle_error(response)
+        return response
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        host, video_id = mobj.group('custom_host') or mobj.group('host'), mobj.group('id')
+
+        player_data = self._download_json(
+            'http://%s/api/player_data?id=%s' % (host, video_id), video_id)
+
+        media = player_data['data']['playlist']['viewports'][0]['medialist'][0]
+
+        title = media['title']
+        description = media.get('description')
+        thumbnail = media.get('snapshot')
+        duration = int_or_none(media.get('duration'))
+        view_count = int_or_none(media.get('views'))
+
+        age_restriction = media.get('age_restriction')
+        age_limit = None
+        if age_restriction:
+            age_limit = 0 if age_restriction == 'allow_all' else 18
+
+        m3u8_data = self._download_json(
+            media['sources']['secure_m3u8']['auto'],
+            video_id, 'Downloading m3u8 JSON')
+
+        formats = self._extract_m3u8_formats(
+            m3u8_data['data'][0], video_id,
+            'mp4', entry_protocol='m3u8_native')
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'duration': duration,
+            'view_count': view_count,
+            'age_limit': age_limit,
+            'formats': formats,
+        }

From 135c9c42bf272c667f38a0a42e97e381fe53df91 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 7 Mar 2015 22:22:57 +0600
Subject: [PATCH 0616/6219] [eagleplatform] Add support for embeds

---
 youtube_dl/extractor/generic.py | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 5dc53685c..a12b29da1 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -570,6 +570,20 @@ class GenericIE(InfoExtractor):
                 'title': 'John Carlson Postgame 2/25/15',
             },
         },
+        # Eagle.Platform embed (generic URL)
+        {
+            'url': 'http://lenta.ru/news/2015/03/06/navalny/',
+            'info_dict': {
+                'id': '227304',
+                'ext': 'mp4',
+                'title': 'Навальный вышел на свободу',
+                'description': 'md5:d97861ac9ae77377f3f20eaf9d04b4f5',
+                'thumbnail': 're:^https?://.*\.jpg$',
+                'duration': 87,
+                'view_count': int,
+                'age_limit': 0,
+            },
+        },
         # RSS feed with enclosure
         {
             'url': 'http://podcastfeeds.nbcnews.com/audio/podcast/MSNBC-MADDOW-NETCAST-M4V.xml',
@@ -1155,6 +1169,12 @@ class GenericIE(InfoExtractor):
         if mobj is not None:
             return self.url_result('kaltura:%(partner_id)s:%(id)s' % mobj.groupdict(), 'Kaltura')
 
+        # Look for Eagle.Platform embeds
+        mobj = re.search(
+            r'<iframe[^>]+src="(?P<url>https?://.+?\.media\.eagleplatform\.com/index/player\?.+?)"', webpage)
+        if mobj is not None:
+            return self.url_result(mobj.group('url'), 'EaglePlatform')
+
         def check_video(vurl):
             if YoutubeIE.suitable(vurl):
                 return True

From d47ae7f620e3b7be2387da2a61b618a3730d932a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 7 Mar 2015 22:34:44 +0600
Subject: [PATCH 0617/6219] [eagleplatform] Add support for ClipYou embeds

---
 youtube_dl/extractor/generic.py | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index a12b29da1..013198b0d 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -584,6 +584,18 @@ class GenericIE(InfoExtractor):
                 'age_limit': 0,
             },
         },
+        # ClipYou (Eagle.Platform) embed (custom URL)
+        {
+            'url': 'http://muz-tv.ru/play/7129/',
+            'info_dict': {
+                'id': '12820',
+                'ext': 'mp4',
+                'title': "'O Sole Mio",
+                'thumbnail': 're:^https?://.*\.jpg$',
+                'duration': 216,
+                'view_count': int,
+            },
+        },
         # RSS feed with enclosure
         {
             'url': 'http://podcastfeeds.nbcnews.com/audio/podcast/MSNBC-MADDOW-NETCAST-M4V.xml',
@@ -1175,6 +1187,12 @@ class GenericIE(InfoExtractor):
         if mobj is not None:
             return self.url_result(mobj.group('url'), 'EaglePlatform')
 
+        # Look for ClipYou (uses Eagle.Platform) embeds
+        mobj = re.search(
+            r'<iframe[^>]+src="https?://(?P<host>media\.clipyou\.ru)/index/player\?.*\brecord_id=(?P<id>\d+).*"', webpage)
+        if mobj is not None:
+            return self.url_result('eagleplatform:%(host)s:%(id)s' % mobj.groupdict(), 'EaglePlatform')
+
         def check_video(vurl):
             if YoutubeIE.suitable(vurl):
                 return True

From 8a48223a7ba5a3d1e47a36c30dd925e11183ae63 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 7 Mar 2015 22:35:36 +0600
Subject: [PATCH 0618/6219] [eagleplatform] Remove debug output

---
 youtube_dl/extractor/eagleplatform.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/eagleplatform.py b/youtube_dl/extractor/eagleplatform.py
index 654f746fe..7173371ee 100644
--- a/youtube_dl/extractor/eagleplatform.py
+++ b/youtube_dl/extractor/eagleplatform.py
@@ -48,7 +48,6 @@ class EaglePlatformIE(InfoExtractor):
     }]
 
     def _handle_error(self, response):
-        print(response['status'])
         status = int_or_none(response.get('status', 200))
         if status != 200:
             raise ExtractorError(' '.join(response['errors']), expected=True)

From 7dcad95d4faa91adfaa4d87d411c4bc55ab000e4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 8 Mar 2015 01:13:23 +0600
Subject: [PATCH 0619/6219] [YoutubeDL] Sanitize outtmpl as it may contain
 forbidden characters

---
 youtube_dl/YoutubeDL.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index df2aebb59..bae52e9c7 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -553,16 +553,20 @@ class YoutubeDL(object):
                 elif template_dict.get('width'):
                     template_dict['resolution'] = '?x%d' % template_dict['width']
 
+            restrict_filenames = self.params.get('restrictfilenames')
+
             sanitize = lambda k, v: sanitize_filename(
                 compat_str(v),
-                restricted=self.params.get('restrictfilenames'),
+                restricted=restrict_filenames,
                 is_id=(k == 'id'))
             template_dict = dict((k, sanitize(k, v))
                                  for k, v in template_dict.items()
                                  if v is not None)
             template_dict = collections.defaultdict(lambda: 'NA', template_dict)
 
-            outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
+            outtmpl = sanitize_filename(
+                self.params.get('outtmpl', DEFAULT_OUTTMPL),
+                restricted=restrict_filenames)
             tmpl = compat_expanduser(outtmpl)
             filename = tmpl % template_dict
             # Temporary fix for #4787

From ab205b9dc8dcee37786dac597f2c3760c89b191c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 7 Mar 2015 22:18:16 +0100
Subject: [PATCH 0620/6219] Revert "[YoutubeDL] Sanitize outtmpl as it may
 contain forbidden characters"

This reverts commit 7dcad95d4faa91adfaa4d87d411c4bc55ab000e4.

The output template is most definitly allowed to contain forbidden characters; otherwise -o /foo/bar/vid.mp4 wouldn't work.
---
 youtube_dl/YoutubeDL.py | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index bae52e9c7..df2aebb59 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -553,20 +553,16 @@ class YoutubeDL(object):
                 elif template_dict.get('width'):
                     template_dict['resolution'] = '?x%d' % template_dict['width']
 
-            restrict_filenames = self.params.get('restrictfilenames')
-
             sanitize = lambda k, v: sanitize_filename(
                 compat_str(v),
-                restricted=restrict_filenames,
+                restricted=self.params.get('restrictfilenames'),
                 is_id=(k == 'id'))
             template_dict = dict((k, sanitize(k, v))
                                  for k, v in template_dict.items()
                                  if v is not None)
             template_dict = collections.defaultdict(lambda: 'NA', template_dict)
 
-            outtmpl = sanitize_filename(
-                self.params.get('outtmpl', DEFAULT_OUTTMPL),
-                restricted=restrict_filenames)
+            outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
             tmpl = compat_expanduser(outtmpl)
             filename = tmpl % template_dict
             # Temporary fix for #4787

From d34e79492d1d9d9babf85af3737b90c4fe55eb42 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 8 Mar 2015 16:54:11 +0600
Subject: [PATCH 0621/6219] [twitch] Fix live streams (Closes #5158)

---
 youtube_dl/extractor/twitch.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index b058891bd..cbdaf9c7a 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -358,13 +358,12 @@ class TwitchStreamIE(TwitchBaseIE):
             'p': random.randint(1000000, 10000000),
             'player': 'twitchweb',
             'segment_preference': '4',
-            'sig': access_token['sig'],
-            'token': access_token['token'],
+            'sig': access_token['sig'].encode('utf-8'),
+            'token': access_token['token'].encode('utf-8'),
         }
-
         formats = self._extract_m3u8_formats(
             '%s/api/channel/hls/%s.m3u8?%s'
-            % (self._USHER_BASE, channel_id, compat_urllib_parse.urlencode(query).encode('utf-8')),
+            % (self._USHER_BASE, channel_id, compat_urllib_parse.urlencode(query)),
             channel_id, 'mp4')
         self._prefer_source(formats)
 

From 1132eae56d0f693b5dc55529d5cfadf32b32700d Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Sun, 8 Mar 2015 13:54:01 +0200
Subject: [PATCH 0622/6219] [gazeta] Add new extractor (Closes #4222)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/gazeta.py   | 35 ++++++++++++++++++++++++++++++++
 2 files changed, 36 insertions(+)
 create mode 100644 youtube_dl/extractor/gazeta.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index b489d5770..e2475a634 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -175,6 +175,7 @@ from .gameone import (
 from .gamespot import GameSpotIE
 from .gamestar import GameStarIE
 from .gametrailers import GametrailersIE
+from .gazeta import GazetaIE
 from .gdcvault import GDCVaultIE
 from .generic import GenericIE
 from .giantbomb import GiantBombIE
diff --git a/youtube_dl/extractor/gazeta.py b/youtube_dl/extractor/gazeta.py
new file mode 100644
index 000000000..3f9e58061
--- /dev/null
+++ b/youtube_dl/extractor/gazeta.py
@@ -0,0 +1,35 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+
+class GazetaIE(InfoExtractor):
+    _VALID_URL = r'(?P<url>https?://(?:www\.)?gazeta\.ru/(?:(?P<category>[^/]*)/)?video/(?:main/)?(?P<id>[A-Za-z0-9-_]+)\.s?html)'
+    _TEST = {
+        'url': 'http://www.gazeta.ru/video/main/zadaite_vopros_vladislavu_yurevichu.shtml',
+        'md5': 'd49c9bdc6e5a7888f27475dc215ee789',
+        'info_dict': {
+            'id': '205566',
+            'ext': 'mp4',
+            'title': '«70–80 процентов гражданских в Донецке на грани голода»',
+            'description': 'md5:38617526050bd17b234728e7f9620a71',
+            'thumbnail': 're:^https?://.*\.jpg',
+        },
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+
+        display_id = mobj.group('id')
+        embed_url = '%s?p=embed' % mobj.group('url')
+        embed_page = self._download_webpage(
+            embed_url, display_id, 'Downloading embed page')
+
+        video_id = self._search_regex(
+            r'<div[^>]*?class="eagleplayer"[^>]*?data-id="([^"]+)"', embed_page, 'video id')
+
+        return self.url_result(
+            'eagleplatform:gazeta.media.eagleplatform.com:%s' % video_id, 'EaglePlatform')

From 28778d6bae2ff2a0ad57d2c5a694e22ac4ead749 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 8 Mar 2015 18:03:12 +0600
Subject: [PATCH 0623/6219] [pladform] Add extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/pladform.py | 86 ++++++++++++++++++++++++++++++++
 2 files changed, 87 insertions(+)
 create mode 100644 youtube_dl/extractor/pladform.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index e2475a634..14172ca56 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -364,6 +364,7 @@ from .pbs import PBSIE
 from .phoenix import PhoenixIE
 from .photobucket import PhotobucketIE
 from .planetaplay import PlanetaPlayIE
+from .pladform import PladformIE
 from .played import PlayedIE
 from .playfm import PlayFMIE
 from .playvid import PlayvidIE
diff --git a/youtube_dl/extractor/pladform.py b/youtube_dl/extractor/pladform.py
new file mode 100644
index 000000000..926e368a2
--- /dev/null
+++ b/youtube_dl/extractor/pladform.py
@@ -0,0 +1,86 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    int_or_none,
+    xpath_text,
+)
+
+
+class PladformIE(InfoExtractor):
+    _VALID_URL = r'''(?x)
+                    https?://
+                        (?:
+                            (?:
+                                out\.pladform\.ru/player|
+                                static\.pladform\.ru/player\.swf
+                            )
+                            \?.*\bvideoid=|
+                            video\.pladform\.ru/catalog/video/videoid/
+                        )
+                        (?P<id>\d+)
+                    '''
+    _TESTS = [{
+        # http://muz-tv.ru/kinozal/view/7400/
+        'url': 'http://out.pladform.ru/player?pl=24822&videoid=100183293',
+        'md5': '61f37b575dd27f1bb2e1854777fe31f4',
+        'info_dict': {
+            'id': '100183293',
+            'ext': 'mp4',
+            'title': 'Тайны перевала Дятлова • Тайна перевала Дятлова 1 серия 2 часть',
+            'description': 'Документальный сериал-расследование одной из самых жутких тайн ХХ века',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 694,
+            'age_limit': 0,
+        },
+    }, {
+        'url': 'http://static.pladform.ru/player.swf?pl=21469&videoid=100183293&vkcid=0',
+        'only_matching': True,
+    }, {
+        'url': 'http://video.pladform.ru/catalog/video/videoid/100183293/vkcid/0',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        video = self._download_xml(
+            'http://out.pladform.ru/getVideo?pl=1&videoid=%s' % video_id,
+            video_id)
+
+        if video.tag == 'error':
+            raise ExtractorError(
+                '%s returned error: %s' % (self.IE_NAME, video.text),
+                expected=True)
+
+        formats = [{
+            'url': src.text,
+            'format_id': src.get('quality'),
+        } for src in video.findall('./src')]
+        self._sort_formats(formats)
+
+        webpage = self._download_webpage(
+            'http://video.pladform.ru/catalog/video/videoid/%s' % video_id,
+            video_id)
+
+        title = self._og_search_title(webpage, fatal=False) or xpath_text(
+            video, './/title', 'title', fatal=True)
+        description = self._search_regex(
+            r'</h3>\s*<p>([^<]+)</p>', webpage, 'description', fatal=False)
+        thumbnail = self._og_search_thumbnail(webpage) or xpath_text(
+            video, './/cover', 'cover')
+
+        duration = int_or_none(xpath_text(video, './/time', 'duration'))
+        age_limit = int_or_none(xpath_text(video, './/age18', 'age limit'))
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'duration': duration,
+            'age_limit': age_limit,
+            'formats': formats,
+        }

From f8388757269de8a5e87b9a507db55021b3090980 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 8 Mar 2015 18:07:10 +0600
Subject: [PATCH 0624/6219] [pladform] Add support for embeds

---
 youtube_dl/extractor/generic.py | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 013198b0d..4e6927b08 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -596,6 +596,19 @@ class GenericIE(InfoExtractor):
                 'view_count': int,
             },
         },
+        # Pladform embed
+        {
+            'url': 'http://muz-tv.ru/kinozal/view/7400/',
+            'info_dict': {
+                'id': '100183293',
+                'ext': 'mp4',
+                'title': 'Тайны перевала Дятлова • Тайна перевала Дятлова 1 серия 2 часть',
+                'description': 'Документальный сериал-расследование одной из самых жутких тайн ХХ века',
+                'thumbnail': 're:^https?://.*\.jpg$',
+                'duration': 694,
+                'age_limit': 0,
+            },
+        },
         # RSS feed with enclosure
         {
             'url': 'http://podcastfeeds.nbcnews.com/audio/podcast/MSNBC-MADDOW-NETCAST-M4V.xml',
@@ -1193,6 +1206,12 @@ class GenericIE(InfoExtractor):
         if mobj is not None:
             return self.url_result('eagleplatform:%(host)s:%(id)s' % mobj.groupdict(), 'EaglePlatform')
 
+        # Look for Pladform embeds
+        mobj = re.search(
+            r'<iframe[^>]+src="(?P<url>https?://out\.pladform\.ru/player\?.+?)"', webpage)
+        if mobj is not None:
+            return self.url_result(mobj.group('url'), 'Pladform')
+
         def check_video(vurl):
             if YoutubeIE.suitable(vurl):
                 return True

From 11101076a1b0eb36dd56b1b5aa1b7559f2d230b9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 8 Mar 2015 18:09:47 +0600
Subject: [PATCH 0625/6219] [pladform] Fix format quality sorting

---
 youtube_dl/extractor/pladform.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/pladform.py b/youtube_dl/extractor/pladform.py
index 926e368a2..abde34b94 100644
--- a/youtube_dl/extractor/pladform.py
+++ b/youtube_dl/extractor/pladform.py
@@ -6,6 +6,7 @@ from ..utils import (
     ExtractorError,
     int_or_none,
     xpath_text,
+    qualities,
 )
 
 
@@ -55,9 +56,12 @@ class PladformIE(InfoExtractor):
                 '%s returned error: %s' % (self.IE_NAME, video.text),
                 expected=True)
 
+        quality = qualities(('ld', 'sd', 'hd'))
+
         formats = [{
             'url': src.text,
             'format_id': src.get('quality'),
+            'quality': quality(src.get('quality')),
         } for src in video.findall('./src')]
         self._sort_formats(formats)
 

From 24993e3b396407af9ccdf8dd893588df31aff8af Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Sun, 8 Mar 2015 14:08:45 +0200
Subject: [PATCH 0626/6219] [vidme] Fix view_count extraction and remove
 comment_count extraction (Fixes #5133)

Comment counts seem to no longer be listed on vid.me
---
 youtube_dl/extractor/vidme.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/vidme.py b/youtube_dl/extractor/vidme.py
index 339c3d897..bd953fb4c 100644
--- a/youtube_dl/extractor/vidme.py
+++ b/youtube_dl/extractor/vidme.py
@@ -41,13 +41,10 @@ class VidmeIE(InfoExtractor):
         duration = float_or_none(self._html_search_regex(
             r'data-duration="([^"]+)"', webpage, 'duration', fatal=False))
         view_count = str_to_int(self._html_search_regex(
-            r'<span class="video_views">\s*([\d,\.]+)\s*plays?', webpage, 'view count', fatal=False))
+            r'<(?:li|span) class="video_views">\s*([\d,\.]+)\s*plays?', webpage, 'view count', fatal=False))
         like_count = str_to_int(self._html_search_regex(
             r'class="score js-video-vote-score"[^>]+data-score="([\d,\.\s]+)">',
             webpage, 'like count', fatal=False))
-        comment_count = str_to_int(self._html_search_regex(
-            r'class="js-comment-count"[^>]+data-count="([\d,\.\s]+)">',
-            webpage, 'comment count', fatal=False))
 
         return {
             'id': video_id,
@@ -61,5 +58,4 @@ class VidmeIE(InfoExtractor):
             'duration': duration,
             'view_count': view_count,
             'like_count': like_count,
-            'comment_count': comment_count,
         }

From 8b910bda0cdba859c1971a410aa5d2a51c6e4918 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Sun, 8 Mar 2015 14:28:53 +0200
Subject: [PATCH 0627/6219] [teamcoco] Fix extraction

---
 youtube_dl/extractor/teamcoco.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/teamcoco.py b/youtube_dl/extractor/teamcoco.py
index 5793dbc10..7cb06f351 100644
--- a/youtube_dl/extractor/teamcoco.py
+++ b/youtube_dl/extractor/teamcoco.py
@@ -53,10 +53,10 @@ class TeamcocoIE(InfoExtractor):
         embed = self._download_webpage(
             embed_url, video_id, 'Downloading embed page')
 
-        encoded_data = self._search_regex(
-            r'"preload"\s*:\s*"([^"]+)"', embed, 'encoded data')
+        player_data = self._parse_json(self._search_regex(
+            r'Y\.Ginger\.Module\.Player\((\{.*?\})\);', embed, 'player data'), video_id)
         data = self._parse_json(
-            base64.b64decode(encoded_data.encode('ascii')).decode('utf-8'), video_id)
+            base64.b64decode(player_data['preload'].encode('ascii')).decode('utf-8'), video_id)
 
         formats = []
         get_quality = qualities(['500k', '480p', '1000k', '720p', '1080p'])

From bdf6eee0aeed8df586569982eaaac04eecc0062d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 8 Mar 2015 19:17:54 +0600
Subject: [PATCH 0628/6219] [gazeta] Extend _VALID_URL

---
 youtube_dl/extractor/gazeta.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/gazeta.py b/youtube_dl/extractor/gazeta.py
index 3f9e58061..ea32b621c 100644
--- a/youtube_dl/extractor/gazeta.py
+++ b/youtube_dl/extractor/gazeta.py
@@ -7,8 +7,8 @@ from .common import InfoExtractor
 
 
 class GazetaIE(InfoExtractor):
-    _VALID_URL = r'(?P<url>https?://(?:www\.)?gazeta\.ru/(?:(?P<category>[^/]*)/)?video/(?:main/)?(?P<id>[A-Za-z0-9-_]+)\.s?html)'
-    _TEST = {
+    _VALID_URL = r'(?P<url>https?://(?:www\.)?gazeta\.ru/(?:[^/]+/)?video/(?:(?:main|\d{4}/\d{2}/\d{2})/)?(?P<id>[A-Za-z0-9-_.]+)\.s?html)'
+    _TESTS = [{
         'url': 'http://www.gazeta.ru/video/main/zadaite_vopros_vladislavu_yurevichu.shtml',
         'md5': 'd49c9bdc6e5a7888f27475dc215ee789',
         'info_dict': {
@@ -18,7 +18,10 @@ class GazetaIE(InfoExtractor):
             'description': 'md5:38617526050bd17b234728e7f9620a71',
             'thumbnail': 're:^https?://.*\.jpg',
         },
-    }
+    }, {
+        'url': 'http://www.gazeta.ru/lifestyle/video/2015/03/08/master-klass_krasivoi_byt._delaem_vesennii_makiyazh.shtml',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)

From a2aaf4dbc6e5f5d345329b5a845111851453b6a6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 8 Mar 2015 20:55:22 +0600
Subject: [PATCH 0629/6219] [utils] Add sanitize_path

---
 test/test_utils.py  | 21 +++++++++++++++++++++
 youtube_dl/utils.py | 18 ++++++++++++++++++
 2 files changed, 39 insertions(+)

diff --git a/test/test_utils.py b/test/test_utils.py
index 64fad58ad..5ebb8d498 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -38,6 +38,7 @@ from youtube_dl.utils import (
     parse_iso8601,
     read_batch_urls,
     sanitize_filename,
+    sanitize_path,
     shell_quote,
     smuggle_url,
     str_to_int,
@@ -131,6 +132,26 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(sanitize_filename('_BD_eEpuzXw', is_id=True), '_BD_eEpuzXw')
         self.assertEqual(sanitize_filename('N0Y__7-UOdI', is_id=True), 'N0Y__7-UOdI')
 
+    def test_sanitize_path(self):
+        if sys.platform != 'win32':
+            return
+
+        self.assertEqual(sanitize_path('abc'), 'abc')
+        self.assertEqual(sanitize_path('abc/def'), 'abc\\def')
+        self.assertEqual(sanitize_path('abc\\def'), 'abc\\def')
+        self.assertEqual(sanitize_path('abc|def'), 'abc#def')
+        self.assertEqual(sanitize_path('<>:"|?*'), '#######')
+        self.assertEqual(sanitize_path('C:/abc/def'), 'C:\\abc\\def')
+        self.assertEqual(sanitize_path('C?:/abc/def'), 'C##\\abc\\def')
+
+        self.assertEqual(sanitize_path('\\\\?\\UNC\\ComputerName\\abc'), '\\\\?\\UNC\\ComputerName\\abc')
+        self.assertEqual(sanitize_path('\\\\?\\UNC/ComputerName/abc'), '\\\\?\\UNC\\ComputerName\\abc')
+
+        self.assertEqual(sanitize_path('\\\\?\\C:\\abc'), '\\\\?\\C:\\abc')
+        self.assertEqual(sanitize_path('\\\\?\\C:/abc'), '\\\\?\\C:\\abc')
+        self.assertEqual(sanitize_path('\\\\?\\C:\\ab?c\\de:f'), '\\\\?\\C:\\ab#c\\de#f')
+        self.assertEqual(sanitize_path('\\\\?\\C:\\abc'), '\\\\?\\C:\\abc')
+
     def test_ordered_set(self):
         self.assertEqual(orderedSet([1, 1, 2, 3, 4, 4, 5, 6, 7, 3, 5]), [1, 2, 3, 4, 5, 6, 7])
         self.assertEqual(orderedSet([]), [])
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 7426e2a1f..0f49d602e 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -311,6 +311,24 @@ def sanitize_filename(s, restricted=False, is_id=False):
     return result
 
 
+def sanitize_path(s):
+    """Sanitizes and normalizes path on Windows"""
+    if sys.platform != 'win32':
+        return s
+    drive, _ = os.path.splitdrive(s)
+    unc, _ = os.path.splitunc(s)
+    unc_or_drive = unc or drive
+    norm_path = os.path.normpath(remove_start(s, unc_or_drive)).split(os.path.sep)
+    if unc_or_drive:
+        norm_path.pop(0)
+    sanitized_path = [
+        re.sub('[/<>:"\\|\\\\?\\*]', '#', path_part)
+        for path_part in norm_path]
+    if unc_or_drive:
+        sanitized_path.insert(0, unc_or_drive + os.path.sep)
+    return os.path.join(*sanitized_path)
+
+
 def orderedSet(iterable):
     """ Remove all duplicates from the input iterable """
     res = []

From d55de57b67bceca5d9116ddcc2ada2fa1957d89d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 8 Mar 2015 20:56:28 +0600
Subject: [PATCH 0630/6219] [utils] Fix sanitize_open

---
 youtube_dl/utils.py | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 0f49d602e..e511232ca 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -252,15 +252,12 @@ def sanitize_open(filename, open_mode):
             raise
 
         # In case of error, try to remove win32 forbidden chars
-        alt_filename = os.path.join(
-            re.sub('[/<>:"\\|\\\\?\\*]', '#', path_part)
-            for path_part in os.path.split(filename)
-        )
+        alt_filename = sanitize_path(filename)
         if alt_filename == filename:
             raise
         else:
             # An exception here should be caught in the caller
-            stream = open(encodeFilename(filename), open_mode)
+            stream = open(encodeFilename(alt_filename), open_mode)
             return (stream, alt_filename)
 
 

From 1bb5c511a551ba7c5d3179d2d3a124f0977e74b4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 8 Mar 2015 20:57:30 +0600
Subject: [PATCH 0631/6219] [YoutubeDL] Sanitize outtmpl as path

---
 youtube_dl/YoutubeDL.py | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index bae52e9c7..4b9151163 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -61,6 +61,7 @@ from .utils import (
     render_table,
     SameFileError,
     sanitize_filename,
+    sanitize_path,
     std_headers,
     subtitles_filename,
     takewhile_inclusive,
@@ -553,20 +554,16 @@ class YoutubeDL(object):
                 elif template_dict.get('width'):
                     template_dict['resolution'] = '?x%d' % template_dict['width']
 
-            restrict_filenames = self.params.get('restrictfilenames')
-
             sanitize = lambda k, v: sanitize_filename(
                 compat_str(v),
-                restricted=restrict_filenames,
+                restricted=self.params.get('restrictfilenames'),
                 is_id=(k == 'id'))
             template_dict = dict((k, sanitize(k, v))
                                  for k, v in template_dict.items()
                                  if v is not None)
             template_dict = collections.defaultdict(lambda: 'NA', template_dict)
 
-            outtmpl = sanitize_filename(
-                self.params.get('outtmpl', DEFAULT_OUTTMPL),
-                restricted=restrict_filenames)
+            outtmpl = sanitize_path(self.params.get('outtmpl', DEFAULT_OUTTMPL))
             tmpl = compat_expanduser(outtmpl)
             filename = tmpl % template_dict
             # Temporary fix for #4787

From f18ef2d14463a13d80e967d1b18ece6a076f60fa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 8 Mar 2015 22:08:48 +0600
Subject: [PATCH 0632/6219] [utils] Disallow trailing dot in sanitize_path for
 a path part

---
 test/test_utils.py  | 11 +++++++++++
 youtube_dl/utils.py |  2 +-
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 5ebb8d498..28bda654e 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -152,6 +152,17 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(sanitize_path('\\\\?\\C:\\ab?c\\de:f'), '\\\\?\\C:\\ab#c\\de#f')
         self.assertEqual(sanitize_path('\\\\?\\C:\\abc'), '\\\\?\\C:\\abc')
 
+        self.assertEqual(
+            sanitize_path('youtube/%(uploader)s/%(autonumber)s-%(title)s-%(upload_date)s.%(ext)s'),
+            'youtube\\%(uploader)s\\%(autonumber)s-%(title)s-%(upload_date)s.%(ext)s')
+
+        self.assertEqual(
+            sanitize_path('youtube/TheWreckingYard ./00001-Not bad, Especially for Free! (1987 Yamaha 700)-20141116.mp4.part'),
+            'youtube\\TheWreckingYard #\\00001-Not bad, Especially for Free! (1987 Yamaha 700)-20141116.mp4.part')
+        self.assertEqual(sanitize_path('abc/def...'), 'abc\\def..#')
+        self.assertEqual(sanitize_path('abc.../def'), 'abc..#\\def')
+        self.assertEqual(sanitize_path('abc.../def...'), 'abc..#\\def..#')
+
     def test_ordered_set(self):
         self.assertEqual(orderedSet([1, 1, 2, 3, 4, 4, 5, 6, 7, 3, 5]), [1, 2, 3, 4, 5, 6, 7])
         self.assertEqual(orderedSet([]), [])
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index e511232ca..d5597d514 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -319,7 +319,7 @@ def sanitize_path(s):
     if unc_or_drive:
         norm_path.pop(0)
     sanitized_path = [
-        re.sub('[/<>:"\\|\\\\?\\*]', '#', path_part)
+        re.sub('(?:[/<>:"\\|\\\\?\\*]|\.$)', '#', path_part)
         for path_part in norm_path]
     if unc_or_drive:
         sanitized_path.insert(0, unc_or_drive + os.path.sep)

From e5a11a2293069b1acfa5eb5d2694ad4082dd9755 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 8 Mar 2015 22:09:42 +0600
Subject: [PATCH 0633/6219] [YoutubeDL] Sanitize path before creating
 non-existent paths (Closes #4324)

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 4b9151163..bce7587fd 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1262,7 +1262,7 @@ class YoutubeDL(object):
             return
 
         try:
-            dn = os.path.dirname(encodeFilename(filename))
+            dn = os.path.dirname(sanitize_path(encodeFilename(filename)))
             if dn and not os.path.exists(dn):
                 os.makedirs(dn)
         except (OSError, IOError) as err:

From 43d6280d0a03335ec5143383f15e2ca9a49f4046 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 8 Mar 2015 18:25:11 +0100
Subject: [PATCH 0634/6219] [downloader/f4m] Fix use of base64 in python 3.2
 (fixes #5132)

b64decode needs a byte string, but on 3.4 it also accepts strings.
---
 youtube_dl/downloader/f4m.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index 3dc796faa..4ab000d67 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -281,7 +281,7 @@ class F4mFD(FileDownloader):
             boot_info = self._get_bootstrap_from_url(bootstrap_url)
         else:
             bootstrap_url = None
-            bootstrap = base64.b64decode(node.text)
+            bootstrap = base64.b64decode(node.text.encode('ascii'))
             boot_info = read_bootstrap_info(bootstrap)
         return (boot_info, bootstrap_url)
 
@@ -308,7 +308,7 @@ class F4mFD(FileDownloader):
         live = boot_info['live']
         metadata_node = media.find(_add_ns('metadata'))
         if metadata_node is not None:
-            metadata = base64.b64decode(metadata_node.text)
+            metadata = base64.b64decode(metadata_node.text.encode('ascii'))
         else:
             metadata = None
 

From 8bba753ccaab38a837efd5d334eff8f1b1c48fad Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 8 Mar 2015 18:37:43 +0100
Subject: [PATCH 0635/6219] [options] Rename --dump-intermediate-pages to
 --dump-pages for consistence with --write-pages

---
 youtube_dl/options.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index a4ca8adc4..eefe008d5 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -563,7 +563,7 @@ def parseOpts(overrideArguments=None):
         action='store_true', dest='verbose', default=False,
         help='print various debugging information')
     verbosity.add_option(
-        '--dump-intermediate-pages',
+        '--dump-pages', '--dump-intermediate-pages',
         action='store_true', dest='dump_intermediate_pages', default=False,
         help='print downloaded pages to debug problems (very verbose)')
     verbosity.add_option(

From cc08b11d16efd125d765cc1a69ac795a592f012c Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Sun, 8 Mar 2015 21:32:42 +0200
Subject: [PATCH 0636/6219] [adultswim] Improve video_info extraction (Fixes
 #5152)

Look for video_info inside `slugged_video`, if slug is not found among collections.
Also, simplify a bit.
---
 youtube_dl/extractor/adultswim.py | 44 ++++++++++++++++++++++---------
 1 file changed, 32 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/adultswim.py b/youtube_dl/extractor/adultswim.py
index 34b8b0115..39335b827 100644
--- a/youtube_dl/extractor/adultswim.py
+++ b/youtube_dl/extractor/adultswim.py
@@ -2,13 +2,12 @@
 from __future__ import unicode_literals
 
 import re
-import json
 
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
-    xpath_text,
     float_or_none,
+    xpath_text,
 )
 
 
@@ -60,6 +59,24 @@ class AdultSwimIE(InfoExtractor):
             'title': 'American Dad - Putting Francine Out of Business',
             'description': 'Stan hatches a plan to get Francine out of the real estate business.Watch more American Dad on [adult swim].'
         },
+    }, {
+        'url': 'http://www.adultswim.com/videos/tim-and-eric-awesome-show-great-job/dr-steve-brule-for-your-wine/',
+        'playlist': [
+            {
+                'md5': '3e346a2ab0087d687a05e1e7f3b3e529',
+                'info_dict': {
+                    'id': 'sY3cMUR_TbuE4YmdjzbIcQ-0',
+                    'ext': 'flv',
+                    'title': 'Tim and Eric Awesome Show Great Job! - Dr. Steve Brule, For Your Wine',
+                    'description': 'Dr. Brule reports live from Wine Country with a special report on wines.  \r\nWatch Tim and Eric Awesome Show Great Job! episode #20, "Embarrassed" on Adult Swim.\r\n\r\n',
+                },
+            }
+        ],
+        'info_dict': {
+            'id': 'sY3cMUR_TbuE4YmdjzbIcQ',
+            'title': 'Tim and Eric Awesome Show Great Job! - Dr. Steve Brule, For Your Wine',
+            'description': 'Dr. Brule reports live from Wine Country with a special report on wines.  \r\nWatch Tim and Eric Awesome Show Great Job! episode #20, "Embarrassed" on Adult Swim.\r\n\r\n',
+        },
     }]
 
     @staticmethod
@@ -80,6 +97,7 @@ class AdultSwimIE(InfoExtractor):
             for video in collection.get('videos'):
                 if video.get('slug') == slug:
                     return collection, video
+        return None, None
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
@@ -90,28 +108,30 @@ class AdultSwimIE(InfoExtractor):
         webpage = self._download_webpage(url, episode_path)
 
         # Extract the value of `bootstrappedData` from the Javascript in the page.
-        bootstrappedDataJS = self._search_regex(r'var bootstrappedData = ({.*});', webpage, episode_path)
-
-        try:
-            bootstrappedData = json.loads(bootstrappedDataJS)
-        except ValueError as ve:
-            errmsg = '%s: Failed to parse JSON ' % episode_path
-            raise ExtractorError(errmsg, cause=ve)
+        bootstrapped_data = self._parse_json(self._search_regex(
+            r'var bootstrappedData = ({.*});', webpage, 'bootstraped data'), episode_path)
 
         # Downloading videos from a /videos/playlist/ URL needs to be handled differently.
         # NOTE: We are only downloading one video (the current one) not the playlist
         if is_playlist:
-            collections = bootstrappedData['playlists']['collections']
+            collections = bootstrapped_data['playlists']['collections']
             collection = self.find_collection_by_linkURL(collections, show_path)
             video_info = self.find_video_info(collection, episode_path)
 
             show_title = video_info['showTitle']
             segment_ids = [video_info['videoPlaybackID']]
         else:
-            collections = bootstrappedData['show']['collections']
+            collections = bootstrapped_data['show']['collections']
             collection, video_info = self.find_collection_containing_video(collections, episode_path)
 
-            show = bootstrappedData['show']
+            # Video wasn't found in the collections, let's try `slugged_video`.
+            if video_info is None:
+                if bootstrapped_data.get('slugged_video', {}).get('slug') == episode_path:
+                    video_info = bootstrapped_data['slugged_video']
+                else:
+                    raise ExtractorError('Unable to find video info')
+
+            show = bootstrapped_data['show']
             show_title = show['title']
             segment_ids = [clip['videoPlaybackID'] for clip in video_info['clips']]
 

From dd7831fe94a0fb8270e7fa3699677c7476a5cd83 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 9 Mar 2015 04:55:35 +0600
Subject: [PATCH 0637/6219] [breakcom] Process only play purpose media formats
 (Closes #5164)

---
 youtube_dl/extractor/breakcom.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/breakcom.py b/youtube_dl/extractor/breakcom.py
index 4bcc897c9..809287d14 100644
--- a/youtube_dl/extractor/breakcom.py
+++ b/youtube_dl/extractor/breakcom.py
@@ -41,7 +41,7 @@ class BreakIE(InfoExtractor):
             'tbr': media['bitRate'],
             'width': media['width'],
             'height': media['height'],
-        } for media in info['media']]
+        } for media in info['media'] if media.get('mediaPurpose') == 'play']
 
         if not formats:
             formats.append({

From d475b3384cd372d89c35b9b1c26499f1e9cc1915 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 9 Mar 2015 03:00:01 +0100
Subject: [PATCH 0638/6219] [README] Better bug reporting instructions

Also address private emails which I get more and more these days.
---
 README.md | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 5b9dd2cea..1a5a90c2e 100644
--- a/README.md
+++ b/README.md
@@ -404,6 +404,18 @@ A note on the service that they don't host the infringing content, but just link
 
 Support requests for services that **do** purchase the rights to distribute their content are perfectly fine though. If in doubt, you can simply include a source that mentions the legitimate purchase of content.
 
+### How can I speed up work on my issue?
+
+(Also known as: Help, my important issue not being solved!) The youtube-dl core developer team is quite small. While we do our best to solve as many issues as possible, sometimes that can take quite a while. To speed up your issue, here's what you can do:
+
+First of all, please do report the issue [at our issue tracker](https://yt-dl.org/bugs). That allows us to coordinate all efforts by users and developers, and serves as a unified point. Unfortunately, the youtube-dl project has grown too large to use personal email as an effective communication channel.
+
+Please read the [bug reporting instructions](#bugs) below. A lot of bugs lack all the necessary information. If you can, offer proxy, VPN, or shell access to the youtube-dl developers. If you are able to, test the issue from multiple computers in multiple countries to exclude local censorship or misconfiguration issues.
+
+If nobody is interested in solving your issue, you are welcome to take matters into your own hands and submit a pull request (or coerce/pay somebody else to do so).
+
+Feel free to bump the issue from time to time by writing a small comment ("Issue is still present in youtube-dl version ...from France, but fixed from Belgium"), but please not more than once a month. Please do not declare your issue as `important` or `urgent`.
+
 ### How can I detect whether a given URL is supported by youtube-dl?
 
 For one, have a look at the [list of supported sites](docs/supportedsites.md). Note that it can sometimes happen that the site changes its URL scheme (say, from http://example.com/video/1234567 to http://example.com/v/1234567 ) and youtube-dl reports an URL of a service in that list as unsupported. In that case, simply report a bug.
@@ -572,7 +584,9 @@ If your report is shorter than two lines, it is almost certainly missing some of
 
 For bug reports, this means that your report should contain the *complete* output of youtube-dl when called with the -v flag. The error message you get for (most) bugs even says so, but you would not believe how many of our bug reports do not contain this information.
 
-Site support requests **must contain an example URL**. An example URL is a URL you might want to download, like http://www.youtube.com/watch?v=BaW_jenozKc . There should be an obvious video present. Except under very special circumstances, the main page of a video service (e.g. http://www.youtube.com/ ) is *not* an example URL.
+If your server has multiple IPs or you suspect censorship, adding --call-home may be a good idea to get more diagnostics. If the error is `ERROR: Unable to extract ...` and you cannot reproduce it from multiple countries, add `--dump-pages` (warning: this will yield a rather large output, redirect it to the file `log.txt` by adding `>log.txt 2>&1` to your command-line) or upload the `.dump` files you get when you add `--write-pages` [somewhere](https://gist.github.com/).
+
+**Site support requests must contain an example URL**. An example URL is a URL you might want to download, like http://www.youtube.com/watch?v=BaW_jenozKc . There should be an obvious video present. Except under very special circumstances, the main page of a video service (e.g. http://www.youtube.com/ ) is *not* an example URL.
 
 ###  Are you using the latest version?
 

From f848215dfce232dfc6618c1cc49da44c3f7df75a Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 9 Mar 2015 03:02:03 +0100
Subject: [PATCH 0639/6219] release 2015.03.09

---
 CONTRIBUTING.md        | 4 +++-
 README.md              | 2 +-
 docs/supportedsites.md | 5 +++++
 youtube_dl/version.py  | 2 +-
 4 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 351229f21..588b15bde 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -18,7 +18,9 @@ If your report is shorter than two lines, it is almost certainly missing some of
 
 For bug reports, this means that your report should contain the *complete* output of youtube-dl when called with the -v flag. The error message you get for (most) bugs even says so, but you would not believe how many of our bug reports do not contain this information.
 
-Site support requests **must contain an example URL**. An example URL is a URL you might want to download, like http://www.youtube.com/watch?v=BaW_jenozKc . There should be an obvious video present. Except under very special circumstances, the main page of a video service (e.g. http://www.youtube.com/ ) is *not* an example URL.
+If your server has multiple IPs or you suspect censorship, adding --call-home may be a good idea to get more diagnostics. If the error is `ERROR: Unable to extract ...` and you cannot reproduce it from multiple countries, add `--dump-pages` (warning: this will yield a rather large output, redirect it to the file `log.txt` by adding `>log.txt 2>&1` to your command-line) or upload the `.dump` files you get when you add `--write-pages` [somewhere](https://gist.github.com/).
+
+**Site support requests must contain an example URL**. An example URL is a URL you might want to download, like http://www.youtube.com/watch?v=BaW_jenozKc . There should be an obvious video present. Except under very special circumstances, the main page of a video service (e.g. http://www.youtube.com/ ) is *not* an example URL.
 
 ###  Are you using the latest version?
 
diff --git a/README.md b/README.md
index 1a5a90c2e..f41e7ecee 100644
--- a/README.md
+++ b/README.md
@@ -167,7 +167,7 @@ which means you can modify it, redistribute it or use it however you like.
     --no-progress                    do not print progress bar
     --console-title                  display progress in console titlebar
     -v, --verbose                    print various debugging information
-    --dump-intermediate-pages        print downloaded pages to debug problems (very verbose)
+    --dump-pages                     print downloaded pages to debug problems (very verbose)
     --write-pages                    Write downloaded intermediary pages to files in the current directory to debug problems
     --print-traffic                  Display sent and read HTTP traffic
     -C, --call-home                  Contact the youtube-dl server for debugging.
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 062cb3d62..80a696ee3 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -117,6 +117,7 @@
  - **DRTV**
  - **Dump**
  - **dvtv**: http://video.aktualne.cz/
+ - **EaglePlatform**
  - **EbaumsWorld**
  - **EchoMsk**
  - **eHow**
@@ -161,6 +162,7 @@
  - **GameSpot**
  - **GameStar**
  - **Gametrailers**
+ - **Gazeta**
  - **GDCVault**
  - **generic**: Generic downloader that works on some sites
  - **GiantBomb**
@@ -315,6 +317,7 @@
  - **Ooyala**
  - **OpenFilm**
  - **orf:fm4**: radio FM4
+ - **orf:iptv**: iptv.ORF.at
  - **orf:oe1**: Radio Österreich 1
  - **orf:tvthek**: ORF TVthek
  - **parliamentlive.tv**: UK parliament videos
@@ -322,10 +325,12 @@
  - **PBS**
  - **Phoenix**
  - **Photobucket**
+ - **Pladform**
  - **PlanetaPlay**
  - **play.fm**
  - **played.to**
  - **Playvid**
+ - **Playwire**
  - **plus.google**: Google Plus
  - **pluzz.francetv.fr**
  - **podomatic**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 252933993..1f0c88a4d 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.03.03.1'
+__version__ = '2015.03.09'

From 4c6039385466e4be3a0e20459e75886052653de7 Mon Sep 17 00:00:00 2001
From: Mamay Alexander <alexander@mamay.su>
Date: Mon, 9 Mar 2015 19:06:49 +0600
Subject: [PATCH 0640/6219] [YandexMusic] Add new extractor

---
 youtube_dl/extractor/__init__.py |   5 ++
 youtube_dl/extractor/yamusic.py  | 104 +++++++++++++++++++++++++++++++
 2 files changed, 109 insertions(+)
 create mode 100644 youtube_dl/extractor/yamusic.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 14172ca56..2db938516 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -611,6 +611,11 @@ from .yahoo import (
     YahooSearchIE,
 )
 from .yam import YamIE
+from .yamusic import (
+    YandexMusicTrackIE,
+    YandexMusicAlbumIE,
+    YandexMusicPlaylistIE,
+)
 from .yesjapan import YesJapanIE
 from .ynet import YnetIE
 from .youjizz import YouJizzIE
diff --git a/youtube_dl/extractor/yamusic.py b/youtube_dl/extractor/yamusic.py
new file mode 100644
index 000000000..5af6df89d
--- /dev/null
+++ b/youtube_dl/extractor/yamusic.py
@@ -0,0 +1,104 @@
+# coding=utf-8
+from __future__ import unicode_literals
+
+import re
+import hashlib
+import time
+
+from .common import InfoExtractor
+
+class YandexMusicAlbumIE(InfoExtractor):
+    _VALID_URL = r'http://music.yandex.ru/album/(?P<id>\d+)'
+
+    def _get_track_url(self, storage_dir, track_id):
+        data = self._download_json('http://music.yandex.ru/api/v1.5/handlers/api-jsonp.jsx?requestId=2&nc=%d&action=getTrackSrc&p=download-info/%s/2.mp3' % (time.time(), storage_dir), track_id)
+
+        hsh = hashlib.md5()
+        hsh.update('XGRlBW9FXlekgbPrRHuSiA' + data['path'][1:] + data['s'])
+        hash = hsh.hexdigest()
+        storage = storage_dir.split('.')
+
+        return 'http://%s/get-mp3/%s/%s?track-id=%s&from=service-10-track&similarities-experiment=default' % (data['host'], hash, data['ts'] + data['path'], storage[1])
+
+    def _get_album_id_and_data(self, url):
+        matched = re.match(self._VALID_URL, url)
+        id = matched.group('id')
+
+        webpage = self._download_webpage(url, id)
+        data = self._parse_json(
+            self._search_regex(
+                r'var\s+Mu\s+=\s+(.+?);\s+<\/script>', webpage, 'player'),
+            id)
+        return id, data['pageData']
+
+    def _real_extract(self, url):
+
+        id, data = self._get_album_id_and_data(url)
+
+        entries = []
+
+        for track in data['volumes'][0]:
+            entries.append({
+                'id': track['id'],
+                'ext': 'mp3',
+                'url': self._get_track_url(track['storageDir'], track['id']),
+                'title': track['artists'][0]['name'] + ' - ' + track['title'],
+            })
+
+        return {
+            '_type': 'playlist',
+            'entries': entries,
+            'id': id,
+            'title': data['title'],
+        }
+
+class YandexMusicPlaylistIE(YandexMusicAlbumIE):
+    _VALID_URL = r'http://music.yandex.ru/users/(?P<user_name>[^/]+)/playlists/(?P<id>\d+)'
+
+    def _real_extract(self, url):
+        id, data = self._get_album_id_and_data(url)
+        data = data['playlist']
+
+        entries = []
+
+        for track in data['tracks']:
+            entries.append({
+                'id': track['id'],
+                'ext': 'mp3',
+                'url': self._get_track_url(track['storageDir'], track['id']),
+                'title': track['artists'][0]['name'] + ' - ' + track['title'],
+            })
+
+        return {
+            '_type': 'playlist',
+            'entries': entries,
+            'id': id,
+            'title': data['title'],
+        }
+
+class YandexMusicTrackIE(YandexMusicAlbumIE):
+    _VALID_URL = r'http://music.yandex.ru/album/(?P<album_id>\d+)/track/(?P<id>\d+)'
+    _TEST = {
+        'url': 'http://music.yandex.ru/album/540508/track/4878838',
+        'info_dict': {
+            'id': '4878838',
+            'ext': 'mp3',
+            'title': 'Carlo Ambrosio - Gypsy Eyes 1',
+        }
+    }
+
+    def _real_extract(self, url):
+
+        id, data = self._get_album_id_and_data(url)
+
+        for track in data['volumes'][0]:
+            if track['id'] == id:
+                track_url = self._get_track_url(track['storageDir'], id)
+                break
+
+        return {
+            'id': id,
+            'ext': 'mp3',
+            'url': track_url,
+            'title': track['artists'][0]['name'] + ' - ' + track['title'],
+        }

From 47fe42e1ab3e182aacbc0c749b9a4f17f1b9dad2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 9 Mar 2015 21:43:46 +0600
Subject: [PATCH 0641/6219] [yamusic] Improve, simplify, fix python3 issues and
 add tests

---
 youtube_dl/extractor/yamusic.py | 168 ++++++++++++++++++--------------
 1 file changed, 96 insertions(+), 72 deletions(-)

diff --git a/youtube_dl/extractor/yamusic.py b/youtube_dl/extractor/yamusic.py
index 5af6df89d..e25c0be44 100644
--- a/youtube_dl/extractor/yamusic.py
+++ b/youtube_dl/extractor/yamusic.py
@@ -3,102 +3,126 @@ from __future__ import unicode_literals
 
 import re
 import hashlib
-import time
 
 from .common import InfoExtractor
+from ..compat import compat_str
+from ..utils import (
+    int_or_none,
+    float_or_none,
+)
 
-class YandexMusicAlbumIE(InfoExtractor):
-    _VALID_URL = r'http://music.yandex.ru/album/(?P<id>\d+)'
 
+class YandexMusicBaseIE(InfoExtractor):
     def _get_track_url(self, storage_dir, track_id):
-        data = self._download_json('http://music.yandex.ru/api/v1.5/handlers/api-jsonp.jsx?requestId=2&nc=%d&action=getTrackSrc&p=download-info/%s/2.mp3' % (time.time(), storage_dir), track_id)
+        data = self._download_json(
+            'http://music.yandex.ru/api/v1.5/handlers/api-jsonp.jsx?action=getTrackSrc&p=download-info/%s'
+            % storage_dir,
+            track_id, 'Downloading track location JSON')
 
-        hsh = hashlib.md5()
-        hsh.update('XGRlBW9FXlekgbPrRHuSiA' + data['path'][1:] + data['s'])
-        hash = hsh.hexdigest()
+        key = hashlib.md5(('XGRlBW9FXlekgbPrRHuSiA' + data['path'][1:] + data['s']).encode('utf-8')).hexdigest()
         storage = storage_dir.split('.')
 
-        return 'http://%s/get-mp3/%s/%s?track-id=%s&from=service-10-track&similarities-experiment=default' % (data['host'], hash, data['ts'] + data['path'], storage[1])
-
-    def _get_album_id_and_data(self, url):
-        matched = re.match(self._VALID_URL, url)
-        id = matched.group('id')
-
-        webpage = self._download_webpage(url, id)
-        data = self._parse_json(
-            self._search_regex(
-                r'var\s+Mu\s+=\s+(.+?);\s+<\/script>', webpage, 'player'),
-            id)
-        return id, data['pageData']
-
-    def _real_extract(self, url):
-
-        id, data = self._get_album_id_and_data(url)
-
-        entries = []
-
-        for track in data['volumes'][0]:
-            entries.append({
-                'id': track['id'],
-                'ext': 'mp3',
-                'url': self._get_track_url(track['storageDir'], track['id']),
-                'title': track['artists'][0]['name'] + ' - ' + track['title'],
-            })
+        return ('http://%s/get-mp3/%s/%s?track-id=%s&from=service-10-track&similarities-experiment=default'
+                % (data['host'], key, data['ts'] + data['path'], storage[1]))
 
+    def _get_track_info(self, track):
         return {
-            '_type': 'playlist',
-            'entries': entries,
-            'id': id,
-            'title': data['title'],
+            'id': track['id'],
+            'ext': 'mp3',
+            'url': self._get_track_url(track['storageDir'], track['id']),
+            'title': '%s - %s' % (track['artists'][0]['name'], track['title']),
+            'filesize': int_or_none(track.get('fileSize')),
+            'duration': float_or_none(track.get('durationMs'), 1000),
         }
 
-class YandexMusicPlaylistIE(YandexMusicAlbumIE):
-    _VALID_URL = r'http://music.yandex.ru/users/(?P<user_name>[^/]+)/playlists/(?P<id>\d+)'
 
-    def _real_extract(self, url):
-        id, data = self._get_album_id_and_data(url)
-        data = data['playlist']
+class YandexMusicTrackIE(YandexMusicBaseIE):
+    IE_NAME = 'yandexmusic:track'
+    IE_DESC = 'Яндекс.Музыка - Трек'
+    _VALID_URL = r'https?://music\.yandex\.ru/album/(?P<album_id>\d+)/track/(?P<id>\d+)'
 
-        entries = []
-
-        for track in data['tracks']:
-            entries.append({
-                'id': track['id'],
-                'ext': 'mp3',
-                'url': self._get_track_url(track['storageDir'], track['id']),
-                'title': track['artists'][0]['name'] + ' - ' + track['title'],
-            })
-
-        return {
-            '_type': 'playlist',
-            'entries': entries,
-            'id': id,
-            'title': data['title'],
-        }
-
-class YandexMusicTrackIE(YandexMusicAlbumIE):
-    _VALID_URL = r'http://music.yandex.ru/album/(?P<album_id>\d+)/track/(?P<id>\d+)'
     _TEST = {
         'url': 'http://music.yandex.ru/album/540508/track/4878838',
+        'md5': 'f496818aa2f60b6c0062980d2e00dc20',
         'info_dict': {
             'id': '4878838',
             'ext': 'mp3',
             'title': 'Carlo Ambrosio - Gypsy Eyes 1',
+            'filesize': 4628061,
+            'duration': 193.04,
         }
     }
 
     def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        album_id, track_id = mobj.group('album_id'), mobj.group('id')
 
-        id, data = self._get_album_id_and_data(url)
+        track = self._download_json(
+            'http://music.yandex.ru/handlers/track.jsx?track=%s:%s' % (track_id, album_id),
+            track_id, 'Downloading track JSON')['track']
 
-        for track in data['volumes'][0]:
-            if track['id'] == id:
-                track_url = self._get_track_url(track['storageDir'], id)
-                break
+        return self._get_track_info(track)
+
+
+class YandexMusicAlbumIE(YandexMusicBaseIE):
+    IE_NAME = 'yandexmusic:album'
+    IE_DESC = 'Яндекс.Музыка - Альбом'
+    _VALID_URL = r'https?://music\.yandex\.ru/album/(?P<id>\d+)'
+
+    _TEST = {
+        'url': 'http://music.yandex.ru/album/540508',
+        'info_dict': {
+            'id': '540508',
+            'title': 'Carlo Ambrosio - Gypsy Soul (2009)',
+        },
+        'playlist_count': 50,
+    }
+
+    def _real_extract(self, url):
+        album_id = self._match_id(url)
+
+        album = self._download_json(
+            'http://music.yandex.ru/handlers/album.jsx?album=%s' % album_id,
+            album_id, 'Downloading album JSON')
+
+        entries = [self._get_track_info(track) for track in album['volumes'][0]]
+
+        title = '%s - %s' % (album['artists'][0]['name'], album['title'])
+        year = album.get('year')
+        if year:
+            title += ' (%s)' % year
+
+        return self.playlist_result(entries, compat_str(album['id']), title)
+
+
+class YandexMusicPlaylistIE(YandexMusicBaseIE):
+    IE_NAME = 'yandexmusic:playlist'
+    IE_DESC = 'Яндекс.Музыка - Плейлист'
+    _VALID_URL = r'https?://music\.yandex\.ru/users/[^/]+/playlists/(?P<id>\d+)'
+
+    _TEST = {
+        'url': 'http://music.yandex.ru/users/music.partners/playlists/1245',
+        'info_dict': {
+            'id': '1245',
+            'title': 'Что слушают Enter Shikari',
+            'description': 'md5:3b9f27b0efbe53f2ee1e844d07155cc9',
+        },
+        'playlist_count': 6,
+    }
+
+    def _real_extract(self, url):
+        playlist_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, playlist_id)
+
+        playlist = self._parse_json(
+            self._search_regex(
+                r'var\s+Mu\s*=\s*({.+?});\s*</script>', webpage, 'player'),
+            playlist_id)['pageData']['playlist']
+
+        entries = [self._get_track_info(track) for track in playlist['tracks']]
+
+        return self.playlist_result(
+            entries, compat_str(playlist_id),
+            playlist['title'], playlist.get('description'))
 
-        return {
-            'id': id,
-            'ext': 'mp3',
-            'url': track_url,
-            'title': track['artists'][0]['name'] + ' - ' + track['title'],
-        }

From d1e2e8f583524e837d452d37fa7175a55ebc80ba Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 9 Mar 2015 21:44:59 +0600
Subject: [PATCH 0642/6219] [yamusic] Rename to yandexmusic

---
 youtube_dl/extractor/__init__.py                    | 2 +-
 youtube_dl/extractor/{yamusic.py => yandexmusic.py} | 0
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename youtube_dl/extractor/{yamusic.py => yandexmusic.py} (100%)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 2db938516..20bc73dce 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -611,7 +611,7 @@ from .yahoo import (
     YahooSearchIE,
 )
 from .yam import YamIE
-from .yamusic import (
+from .yandexmusic import (
     YandexMusicTrackIE,
     YandexMusicAlbumIE,
     YandexMusicPlaylistIE,
diff --git a/youtube_dl/extractor/yamusic.py b/youtube_dl/extractor/yandexmusic.py
similarity index 100%
rename from youtube_dl/extractor/yamusic.py
rename to youtube_dl/extractor/yandexmusic.py

From 41b2194f86c9440d2d095db3eaec90784703fa7b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 9 Mar 2015 21:46:31 +0600
Subject: [PATCH 0643/6219] Credit @MamayAlexander for yandexmusic (#5168)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 4674a5af3..421df69a6 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -113,3 +113,4 @@ Robin de Rooij
 Ryan Schmidt
 Leslie P. Polzer
 Duncan Keall
+Alexander Mamay

From dd77f14c641da26429ba32d2464563bf04e1741c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 9 Mar 2015 18:07:31 +0100
Subject: [PATCH 0644/6219] [yandexmusic] PEP8: remove blank line at the end of
 file

---
 youtube_dl/extractor/yandexmusic.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/yandexmusic.py b/youtube_dl/extractor/yandexmusic.py
index e25c0be44..1ee8d1bc0 100644
--- a/youtube_dl/extractor/yandexmusic.py
+++ b/youtube_dl/extractor/yandexmusic.py
@@ -125,4 +125,3 @@ class YandexMusicPlaylistIE(YandexMusicBaseIE):
         return self.playlist_result(
             entries, compat_str(playlist_id),
             playlist['title'], playlist.get('description'))
-

From 937daef4a7725aeecc4b2ce0caa29c6f1aaf0b87 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 9 Mar 2015 18:12:41 +0100
Subject: [PATCH 0645/6219] [niconico] Use '_match_id'

---
 youtube_dl/extractor/niconico.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/niconico.py b/youtube_dl/extractor/niconico.py
index 4c1890416..7fb4e57df 100644
--- a/youtube_dl/extractor/niconico.py
+++ b/youtube_dl/extractor/niconico.py
@@ -41,7 +41,7 @@ class NiconicoIE(InfoExtractor):
         },
     }
 
-    _VALID_URL = r'https?://(?:www\.|secure\.)?nicovideo\.jp/watch/((?:[a-z]{2})?[0-9]+)'
+    _VALID_URL = r'https?://(?:www\.|secure\.)?nicovideo\.jp/watch/(?P<id>(?:[a-z]{2})?[0-9]+)'
     _NETRC_MACHINE = 'niconico'
     # Determine whether the downloader used authentication to download video
     _AUTHENTICATED = False
@@ -76,8 +76,7 @@ class NiconicoIE(InfoExtractor):
         return True
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group(1)
+        video_id = self._match_id(url)
 
         # Get video webpage. We are not actually interested in it, but need
         # the cookies in order to be able to download the info webpage

From f5d8f58a170c9585e0913dbe1e50e0ed05afb698 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 9 Mar 2015 18:17:22 +0100
Subject: [PATCH 0646/6219] [yandexmusic:album] Improve _VALID_URL to avoid
 matching tracks urls

---
 youtube_dl/extractor/yandexmusic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/yandexmusic.py b/youtube_dl/extractor/yandexmusic.py
index 1ee8d1bc0..b47aecb15 100644
--- a/youtube_dl/extractor/yandexmusic.py
+++ b/youtube_dl/extractor/yandexmusic.py
@@ -67,7 +67,7 @@ class YandexMusicTrackIE(YandexMusicBaseIE):
 class YandexMusicAlbumIE(YandexMusicBaseIE):
     IE_NAME = 'yandexmusic:album'
     IE_DESC = 'Яндекс.Музыка - Альбом'
-    _VALID_URL = r'https?://music\.yandex\.ru/album/(?P<id>\d+)'
+    _VALID_URL = r'https?://music\.yandex\.ru/album/(?P<id>\d+)/?(\?|$)'
 
     _TEST = {
         'url': 'http://music.yandex.ru/album/540508',

From 2ebfeacabc7b74d03fa7cb096d8b9d1ecbdbb6e4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 10 Mar 2015 00:50:11 +0600
Subject: [PATCH 0647/6219] [utils] Keep dot and dotdot unmodified (Closes
 #5171)

---
 test/test_utils.py  | 5 +++++
 youtube_dl/utils.py | 2 +-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 28bda654e..8f790bf0a 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -163,6 +163,11 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(sanitize_path('abc.../def'), 'abc..#\\def')
         self.assertEqual(sanitize_path('abc.../def...'), 'abc..#\\def..#')
 
+        self.assertEqual(sanitize_path('../abc'), '..\\abc')
+        self.assertEqual(sanitize_path('../../abc'), '..\\..\\abc')
+        self.assertEqual(sanitize_path('./abc'), 'abc')
+        self.assertEqual(sanitize_path('./../abc'), '..\\abc')
+
     def test_ordered_set(self):
         self.assertEqual(orderedSet([1, 1, 2, 3, 4, 4, 5, 6, 7, 3, 5]), [1, 2, 3, 4, 5, 6, 7])
         self.assertEqual(orderedSet([]), [])
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index d5597d514..c3135effc 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -319,7 +319,7 @@ def sanitize_path(s):
     if unc_or_drive:
         norm_path.pop(0)
     sanitized_path = [
-        re.sub('(?:[/<>:"\\|\\\\?\\*]|\.$)', '#', path_part)
+        path_part if path_part in ['.', '..'] else re.sub('(?:[/<>:"\\|\\\\?\\*]|\.$)', '#', path_part)
         for path_part in norm_path]
     if unc_or_drive:
         sanitized_path.insert(0, unc_or_drive + os.path.sep)

From 614a7e1e230e095d9a11b59b20f4ff7462be8b21 Mon Sep 17 00:00:00 2001
From: "PishPosh.McGee" <pishposhmcgee@users.noreply.github.com>
Date: Tue, 10 Mar 2015 15:22:46 -0500
Subject: [PATCH 0648/6219] Added subtitles for FunnyOrDie

---
 youtube_dl/extractor/funnyordie.py | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/funnyordie.py b/youtube_dl/extractor/funnyordie.py
index a49fc1151..f832ac5c3 100644
--- a/youtube_dl/extractor/funnyordie.py
+++ b/youtube_dl/extractor/funnyordie.py
@@ -50,7 +50,6 @@ class FunnyOrDieIE(InfoExtractor):
         bitrates.sort()
 
         formats = []
-
         for bitrate in bitrates:
             for link in links:
                 formats.append({
@@ -59,6 +58,16 @@ class FunnyOrDieIE(InfoExtractor):
                     'vbr': bitrate,
                 })
 
+        subtitles={}
+        subtitle_matches=re.findall(r'<track kind="captions" src="([^"]+)" srclang="([^"]+)"', webpage)
+        for match in subtitle_matches:
+          (suburl,sublang)=match
+          if not sublang in subtitles.keys():
+            subtitles[sublang]=[]
+          subext=suburl.split('/')[-1]
+          print subext
+          subtitles[sublang].append({'url': suburl,'ext': subext})
+
         post_json = self._search_regex(
             r'fb_post\s*=\s*(\{.*?\});', webpage, 'post details')
         post = json.loads(post_json)
@@ -69,4 +78,5 @@ class FunnyOrDieIE(InfoExtractor):
             'description': post.get('description'),
             'thumbnail': post.get('picture'),
             'formats': formats,
+            'subtitles': subtitles,
         }

From 1593194c63acd8dc8aa125edbde443680fabd2f7 Mon Sep 17 00:00:00 2001
From: pishposhmcgee <pishposh.mcgee@gmail.com>
Date: Tue, 10 Mar 2015 15:35:35 -0500
Subject: [PATCH 0649/6219] Update funnyordie.py

---
 youtube_dl/extractor/funnyordie.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/funnyordie.py b/youtube_dl/extractor/funnyordie.py
index f832ac5c3..0a22d99d2 100644
--- a/youtube_dl/extractor/funnyordie.py
+++ b/youtube_dl/extractor/funnyordie.py
@@ -65,8 +65,7 @@ class FunnyOrDieIE(InfoExtractor):
           if not sublang in subtitles.keys():
             subtitles[sublang]=[]
           subext=suburl.split('/')[-1]
-          print subext
-          subtitles[sublang].append({'url': suburl,'ext': subext})
+          subtitles[sublang].append({'url': 'http://www.funnyordie.com'+suburl,'ext': subext})
 
         post_json = self._search_regex(
             r'fb_post\s*=\s*(\{.*?\});', webpage, 'post details')

From 32aaeca775c03ce784b8a30ea987c03a25a00e98 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 11 Mar 2015 20:34:32 +0600
Subject: [PATCH 0650/6219] [npo] Improve smooth stream skipping and set low
 preference for streams other than hds ans hls (Closes #5175)

---
 youtube_dl/extractor/npo.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index 9c01eb0af..557dffa46 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -219,7 +219,8 @@ class NPOLiveIE(NPOBaseIE):
         if streams:
             for stream in streams:
                 stream_type = stream.get('type').lower()
-                if stream_type == 'ss':
+                # smooth streaming is not supported
+                if stream_type in ['ss', 'ms']:
                     continue
                 stream_info = self._download_json(
                     'http://ida.omroep.nl/aapi/?stream=%s&token=%s&type=jsonp'
@@ -242,6 +243,7 @@ class NPOLiveIE(NPOBaseIE):
                 else:
                     formats.append({
                         'url': stream_url,
+                        'preference': -10,
                     })
 
         self._sort_formats(formats)

From c792b5011fe56af654d99378c1e77837475d7b69 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 11 Mar 2015 21:15:36 +0600
Subject: [PATCH 0651/6219] [ssa] Add extractor (Closes #5169)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/ssa.py      | 58 ++++++++++++++++++++++++++++++++
 2 files changed, 59 insertions(+)
 create mode 100644 youtube_dl/extractor/ssa.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 20bc73dce..7adcc4dbf 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -460,6 +460,7 @@ from .sport5 import Sport5IE
 from .sportbox import SportBoxIE
 from .sportdeutschland import SportDeutschlandIE
 from .srmediathek import SRMediathekIE
+from .ssa import SSAIE
 from .stanfordoc import StanfordOpenClassroomIE
 from .steam import SteamIE
 from .streamcloud import StreamcloudIE
diff --git a/youtube_dl/extractor/ssa.py b/youtube_dl/extractor/ssa.py
new file mode 100644
index 000000000..13101c714
--- /dev/null
+++ b/youtube_dl/extractor/ssa.py
@@ -0,0 +1,58 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    unescapeHTML,
+    parse_duration,
+)
+
+
+class SSAIE(InfoExtractor):
+    _VALID_URL = r'http://ssa\.nls\.uk/film/(?P<id>\d+)'
+    _TEST = {
+        'url': 'http://ssa.nls.uk/film/3561',
+        'info_dict': {
+            'id': '3561',
+            'ext': 'flv',
+            'title': 'SHETLAND WOOL',
+            'description': 'md5:c5afca6871ad59b4271e7704fe50ab04',
+            'duration': 900,
+            'thumbnail': 're:^https?://.*\.jpg$',
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        streamer = self._search_regex(
+            r"'streamer'\s*,\S*'(rtmp[^']+)'", webpage, 'streamer')
+        play_path = self._search_regex(
+            r"'file'\s*,\s*'([^']+)'", webpage, 'file').rpartition('.')[0]
+
+        def search_field(field_name, fatal=False):
+            return self._search_regex(
+                r'<span\s+class="field_title">%s:</span>\s*<span\s+class="field_content">([^<]+)</span>' % field_name,
+                webpage, 'title', fatal=fatal)
+
+        title = unescapeHTML(search_field('Title', fatal=True)).strip('()[]')
+        description = unescapeHTML(search_field('Description'))
+        duration = parse_duration(search_field('Running time'))
+        thumbnail = self._search_regex(
+            r"'image'\s*,\s*'([^']+)'", webpage, 'thumbnails', fatal=False)
+
+        return {
+            'id': video_id,
+            'url': streamer,
+            'play_path': play_path,
+            'ext': 'flv',
+            'title': title,
+            'description': description,
+            'duration': duration,
+            'thumbnail': thumbnail,
+        }

From a3fbd188241b6151b024f6e3ea21c0edc60fae09 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 11 Mar 2015 21:56:22 +0600
Subject: [PATCH 0652/6219] [funnyordie] Simplify subtitles

---
 youtube_dl/extractor/funnyordie.py | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/funnyordie.py b/youtube_dl/extractor/funnyordie.py
index 0a22d99d2..dd87257c4 100644
--- a/youtube_dl/extractor/funnyordie.py
+++ b/youtube_dl/extractor/funnyordie.py
@@ -58,14 +58,12 @@ class FunnyOrDieIE(InfoExtractor):
                     'vbr': bitrate,
                 })
 
-        subtitles={}
-        subtitle_matches=re.findall(r'<track kind="captions" src="([^"]+)" srclang="([^"]+)"', webpage)
-        for match in subtitle_matches:
-          (suburl,sublang)=match
-          if not sublang in subtitles.keys():
-            subtitles[sublang]=[]
-          subext=suburl.split('/')[-1]
-          subtitles[sublang].append({'url': 'http://www.funnyordie.com'+suburl,'ext': subext})
+        subtitles = {}
+        for src, src_lang in re.findall(r'<track kind="captions" src="([^"]+)" srclang="([^"]+)"', webpage):
+            subtitles[src_lang] = [{
+                'ext': src.split('/')[-1],
+                'url': 'http://www.funnyordie.com%s' % src,
+            }]
 
         post_json = self._search_regex(
             r'fb_post\s*=\s*(\{.*?\});', webpage, 'post details')

From 7e195d0e92772b2ea3c748e319574a81f807e807 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 11 Mar 2015 22:00:37 +0600
Subject: [PATCH 0653/6219] [funnyordie] Add subtitles test

---
 test/test_subtitles.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/test/test_subtitles.py b/test/test_subtitles.py
index 3f2d8a2ba..891ee620b 100644
--- a/test/test_subtitles.py
+++ b/test/test_subtitles.py
@@ -26,6 +26,7 @@ from youtube_dl.extractor import (
     VikiIE,
     ThePlatformIE,
     RTVEALaCartaIE,
+    FunnyOrDieIE,
 )
 
 
@@ -320,5 +321,17 @@ class TestRtveSubtitles(BaseTestSubtitles):
         self.assertEqual(md5(subtitles['es']), '69e70cae2d40574fb7316f31d6eb7fca')
 
 
+class TestFunnyOrDieSubtitles(BaseTestSubtitles):
+    url = 'http://www.funnyordie.com/videos/224829ff6d/judd-apatow-will-direct-your-vine'
+    IE = FunnyOrDieIE
+
+    def test_allsubtitles(self):
+        self.DL.params['writesubtitles'] = True
+        self.DL.params['allsubtitles'] = True
+        subtitles = self.getSubtitles()
+        self.assertEqual(set(subtitles.keys()), set(['en']))
+        self.assertEqual(md5(subtitles['en']), 'c5593c193eacd353596c11c2d4f9ecc4')
+
+
 if __name__ == '__main__':
     unittest.main()

From 3a77719c5ac60f7e634a7a8def82acb08ccc9ffc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 11 Mar 2015 17:38:35 +0100
Subject: [PATCH 0654/6219] Don't accept '-1' as format, 'all' is clearer

---
 youtube_dl/YoutubeDL.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index bce7587fd..be5b3c1ab 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1086,8 +1086,7 @@ class YoutubeDL(object):
         if req_format is None:
             req_format = 'best'
         formats_to_download = []
-        # The -1 is for supporting YoutubeIE
-        if req_format in ('-1', 'all'):
+        if req_format == 'all':
             formats_to_download = formats
         else:
             for rfstr in req_format.split(','):

From 71705fa70d3f8b410a25a9c1fb01c33dd5bb6026 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 12 Mar 2015 21:56:56 +0600
Subject: [PATCH 0655/6219] [footyroom] Add extractor (Closes #5000)

---
 youtube_dl/extractor/__init__.py  |  1 +
 youtube_dl/extractor/footyroom.py | 41 +++++++++++++++++++++++++++++++
 2 files changed, 42 insertions(+)
 create mode 100644 youtube_dl/extractor/footyroom.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 7adcc4dbf..7f9523c2b 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -151,6 +151,7 @@ from .fktv import (
 )
 from .flickr import FlickrIE
 from .folketinget import FolketingetIE
+from .footyroom import FootyRoomIE
 from .fourtube import FourTubeIE
 from .foxgay import FoxgayIE
 from .foxnews import FoxNewsIE
diff --git a/youtube_dl/extractor/footyroom.py b/youtube_dl/extractor/footyroom.py
new file mode 100644
index 000000000..2b4691ae8
--- /dev/null
+++ b/youtube_dl/extractor/footyroom.py
@@ -0,0 +1,41 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class FootyRoomIE(InfoExtractor):
+    _VALID_URL = r'http://footyroom\.com/(?P<id>[^/]+)'
+    _TEST = {
+        'url': 'http://footyroom.com/schalke-04-0-2-real-madrid-2015-02/',
+        'info_dict': {
+            'id': 'schalke-04-0-2-real-madrid-2015-02',
+            'title': 'Schalke 04 0 – 2 Real Madrid',
+        },
+        'playlist_count': 3,
+    }
+
+    def _real_extract(self, url):
+        playlist_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, playlist_id)
+
+        playlist = self._parse_json(
+            self._search_regex(
+                r'VideoSelector\.load\((\[.+?\])\);', webpage, 'video selector'),
+            playlist_id)
+
+        playlist_title = self._og_search_title(webpage)
+
+        entries = []
+        for video in playlist:
+            payload = video.get('payload')
+            if not payload:
+                continue
+            playwire_url = self._search_regex(
+                r'data-config="([^"]+)"', payload,
+                'playwire url', default=None)
+            if playwire_url:
+                entries.append(self.url_result(playwire_url, 'Playwire'))
+
+        return self.playlist_result(entries, playlist_id, playlist_title)

From 1138491631c43c2f758d933675131f0bd8249920 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 12 Mar 2015 21:59:46 +0600
Subject: [PATCH 0656/6219] [yam] Skip test

---
 youtube_dl/extractor/yam.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/yam.py b/youtube_dl/extractor/yam.py
index 19ad74d04..19f8762ae 100644
--- a/youtube_dl/extractor/yam.py
+++ b/youtube_dl/extractor/yam.py
@@ -39,7 +39,8 @@ class YamIE(InfoExtractor):
             'description': 'md5:11e2e405311633ace874f2e6226c8b17',
             'uploader_id': '2323agoy',
             'title': '20090412陽明山二子坪-1',
-        }
+        },
+        'skip': 'Video does not exist',
     }, {
         'url': 'http://mymedia.yam.com/m/3598173',
         'info_dict': {

From d7d79106c7463b455ecc967c5f68b744c1375147 Mon Sep 17 00:00:00 2001
From: Leonardo Amaral <contato@leonardoamaral.com.br>
Date: Thu, 12 Mar 2015 14:23:42 -0300
Subject: [PATCH 0657/6219] * Change globo.py flash ver to 17.0.0.132 - Chrome
 42.0.2311.22

---
 youtube_dl/extractor/globo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/globo.py b/youtube_dl/extractor/globo.py
index 29638a194..8a95793ca 100644
--- a/youtube_dl/extractor/globo.py
+++ b/youtube_dl/extractor/globo.py
@@ -20,7 +20,7 @@ class GloboIE(InfoExtractor):
     _VALID_URL = 'https?://.+?\.globo\.com/(?P<id>.+)'
 
     _API_URL_TEMPLATE = 'http://api.globovideos.com/videos/%s/playlist'
-    _SECURITY_URL_TEMPLATE = 'http://security.video.globo.com/videos/%s/hash?player=flash&version=2.9.9.50&resource_id=%s'
+    _SECURITY_URL_TEMPLATE = 'http://security.video.globo.com/videos/%s/hash?player=flash&version=17.0.0.132&resource_id=%s'
 
     _VIDEOID_REGEXES = [
         r'\bdata-video-id="(\d+)"',

From b84037013edf25866772ba12db3b0c55f94a3e12 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 12 Mar 2015 18:45:00 +0100
Subject: [PATCH 0658/6219] [vimeo] Fix login (#3886)

---
 youtube_dl/extractor/vimeo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index b84a83ba6..fc720e7e7 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -38,7 +38,7 @@ class VimeoBaseInfoExtractor(InfoExtractor):
         self.report_login()
         login_url = 'https://vimeo.com/log_in'
         webpage = self._download_webpage(login_url, None, False)
-        token = self._search_regex(r'xsrft: \'(.*?)\'', webpage, 'login token')
+        token = self._search_regex(r'xsrft = \'(.*?)\'', webpage, 'login token')
         data = urlencode_postdata({
             'email': username,
             'password': password,

From 3946864c8a752c068b188126013ddb747b53385d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 12 Mar 2015 19:08:16 +0100
Subject: [PATCH 0659/6219] [vimeo] Use https for all vimeo.com urls

Unfortunately vimeopro.com doesn't support it yet.
---
 test/test_all_urls.py         | 10 +++++-----
 youtube_dl/extractor/vimeo.py | 36 +++++++++++++++++------------------
 2 files changed, 23 insertions(+), 23 deletions(-)

diff --git a/test/test_all_urls.py b/test/test_all_urls.py
index e66264b4b..6ae168b7f 100644
--- a/test/test_all_urls.py
+++ b/test/test_all_urls.py
@@ -104,11 +104,11 @@ class TestAllURLsMatching(unittest.TestCase):
         self.assertMatch(':tds', ['ComedyCentralShows'])
 
     def test_vimeo_matching(self):
-        self.assertMatch('http://vimeo.com/channels/tributes', ['vimeo:channel'])
-        self.assertMatch('http://vimeo.com/channels/31259', ['vimeo:channel'])
-        self.assertMatch('http://vimeo.com/channels/31259/53576664', ['vimeo'])
-        self.assertMatch('http://vimeo.com/user7108434', ['vimeo:user'])
-        self.assertMatch('http://vimeo.com/user7108434/videos', ['vimeo:user'])
+        self.assertMatch('https://vimeo.com/channels/tributes', ['vimeo:channel'])
+        self.assertMatch('https://vimeo.com/channels/31259', ['vimeo:channel'])
+        self.assertMatch('https://vimeo.com/channels/31259/53576664', ['vimeo'])
+        self.assertMatch('https://vimeo.com/user7108434', ['vimeo:user'])
+        self.assertMatch('https://vimeo.com/user7108434/videos', ['vimeo:user'])
         self.assertMatch('https://vimeo.com/user21297594/review/75524534/3c257a1b5d', ['vimeo:review'])
 
     # https://github.com/rg3/youtube-dl/issues/1930
diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index fc720e7e7..bd09652cd 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -373,7 +373,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
             for tt in text_tracks:
                 subtitles[tt['lang']] = [{
                     'ext': 'vtt',
-                    'url': 'http://vimeo.com' + tt['url'],
+                    'url': 'https://vimeo.com' + tt['url'],
                 }]
 
         return {
@@ -396,11 +396,11 @@ class VimeoIE(VimeoBaseInfoExtractor):
 
 class VimeoChannelIE(InfoExtractor):
     IE_NAME = 'vimeo:channel'
-    _VALID_URL = r'https?://vimeo\.com/channels/(?P<id>[^/?#]+)/?(?:$|[?#])'
+    _VALID_URL = r'https://vimeo\.com/channels/(?P<id>[^/?#]+)/?(?:$|[?#])'
     _MORE_PAGES_INDICATOR = r'<a.+?rel="next"'
     _TITLE_RE = r'<link rel="alternate"[^>]+?title="(.*?)"'
     _TESTS = [{
-        'url': 'http://vimeo.com/channels/tributes',
+        'url': 'https://vimeo.com/channels/tributes',
         'info_dict': {
             'id': 'tributes',
             'title': 'Vimeo Tributes',
@@ -459,7 +459,7 @@ class VimeoChannelIE(InfoExtractor):
             if re.search(self._MORE_PAGES_INDICATOR, webpage, re.DOTALL) is None:
                 break
 
-        entries = [self.url_result('http://vimeo.com/%s' % video_id, 'Vimeo')
+        entries = [self.url_result('https://vimeo.com/%s' % video_id, 'Vimeo')
                    for video_id in video_ids]
         return {'_type': 'playlist',
                 'id': list_id,
@@ -470,15 +470,15 @@ class VimeoChannelIE(InfoExtractor):
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         channel_id = mobj.group('id')
-        return self._extract_videos(channel_id, 'http://vimeo.com/channels/%s' % channel_id)
+        return self._extract_videos(channel_id, 'https://vimeo.com/channels/%s' % channel_id)
 
 
 class VimeoUserIE(VimeoChannelIE):
     IE_NAME = 'vimeo:user'
-    _VALID_URL = r'https?://vimeo\.com/(?![0-9]+(?:$|[?#/]))(?P<name>[^/]+)(?:/videos|[#?]|$)'
+    _VALID_URL = r'https://vimeo\.com/(?![0-9]+(?:$|[?#/]))(?P<name>[^/]+)(?:/videos|[#?]|$)'
     _TITLE_RE = r'<a[^>]+?class="user">([^<>]+?)</a>'
     _TESTS = [{
-        'url': 'http://vimeo.com/nkistudio/videos',
+        'url': 'https://vimeo.com/nkistudio/videos',
         'info_dict': {
             'title': 'Nki',
             'id': 'nkistudio',
@@ -489,7 +489,7 @@ class VimeoUserIE(VimeoChannelIE):
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         name = mobj.group('name')
-        return self._extract_videos(name, 'http://vimeo.com/%s' % name)
+        return self._extract_videos(name, 'https://vimeo.com/%s' % name)
 
 
 class VimeoAlbumIE(VimeoChannelIE):
@@ -526,9 +526,9 @@ class VimeoAlbumIE(VimeoChannelIE):
 
 class VimeoGroupsIE(VimeoAlbumIE):
     IE_NAME = 'vimeo:group'
-    _VALID_URL = r'(?:https?://)?vimeo\.com/groups/(?P<name>[^/]+)'
+    _VALID_URL = r'https://vimeo\.com/groups/(?P<name>[^/]+)'
     _TESTS = [{
-        'url': 'http://vimeo.com/groups/rolexawards',
+        'url': 'https://vimeo.com/groups/rolexawards',
         'info_dict': {
             'id': 'rolexawards',
             'title': 'Rolex Awards for Enterprise',
@@ -542,13 +542,13 @@ class VimeoGroupsIE(VimeoAlbumIE):
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         name = mobj.group('name')
-        return self._extract_videos(name, 'http://vimeo.com/groups/%s' % name)
+        return self._extract_videos(name, 'https://vimeo.com/groups/%s' % name)
 
 
 class VimeoReviewIE(InfoExtractor):
     IE_NAME = 'vimeo:review'
     IE_DESC = 'Review pages on vimeo'
-    _VALID_URL = r'https?://vimeo\.com/[^/]+/review/(?P<id>[^/]+)'
+    _VALID_URL = r'https://vimeo\.com/[^/]+/review/(?P<id>[^/]+)'
     _TESTS = [{
         'url': 'https://vimeo.com/user21297594/review/75524534/3c257a1b5d',
         'md5': 'c507a72f780cacc12b2248bb4006d253',
@@ -560,7 +560,7 @@ class VimeoReviewIE(InfoExtractor):
         }
     }, {
         'note': 'video player needs Referer',
-        'url': 'http://vimeo.com/user22258446/review/91613211/13f927e053',
+        'url': 'https://vimeo.com/user22258446/review/91613211/13f927e053',
         'md5': '6295fdab8f4bf6a002d058b2c6dce276',
         'info_dict': {
             'id': '91613211',
@@ -582,11 +582,11 @@ class VimeoReviewIE(InfoExtractor):
 class VimeoWatchLaterIE(VimeoBaseInfoExtractor, VimeoChannelIE):
     IE_NAME = 'vimeo:watchlater'
     IE_DESC = 'Vimeo watch later list, "vimeowatchlater" keyword (requires authentication)'
-    _VALID_URL = r'https?://vimeo\.com/home/watchlater|:vimeowatchlater'
+    _VALID_URL = r'https://vimeo\.com/home/watchlater|:vimeowatchlater'
     _LOGIN_REQUIRED = True
     _TITLE_RE = r'href="/home/watchlater".*?>(.*?)<'
     _TESTS = [{
-        'url': 'http://vimeo.com/home/watchlater',
+        'url': 'https://vimeo.com/home/watchlater',
         'only_matching': True,
     }]
 
@@ -606,7 +606,7 @@ class VimeoWatchLaterIE(VimeoBaseInfoExtractor, VimeoChannelIE):
 
 
 class VimeoLikesIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?vimeo\.com/user(?P<id>[0-9]+)/likes/?(?:$|[?#]|sort:)'
+    _VALID_URL = r'https://(?:www\.)?vimeo\.com/user(?P<id>[0-9]+)/likes/?(?:$|[?#]|sort:)'
     IE_NAME = 'vimeo:likes'
     IE_DESC = 'Vimeo user likes'
     _TEST = {
@@ -634,8 +634,8 @@ class VimeoLikesIE(InfoExtractor):
         description = self._html_search_meta('description', webpage)
 
         def _get_page(idx):
-            page_url = '%s//vimeo.com/user%s/likes/page:%d/sort:date' % (
-                self.http_scheme(), user_id, idx + 1)
+            page_url = 'https://vimeo.com/user%s/likes/page:%d/sort:date' % (
+                user_id, idx + 1)
             webpage = self._download_webpage(
                 page_url, user_id,
                 note='Downloading page %d/%d' % (idx + 1, page_count))

From 11984c7467184100ca4a61ae939a8c260480f42c Mon Sep 17 00:00:00 2001
From: "Devin J. Pohly" <djpohly@gmail.com>
Date: Thu, 12 Mar 2015 15:43:13 -0400
Subject: [PATCH 0660/6219] [BeatportPro] Add new extractor

This extractor is for Beatport's 2-minute, low-quality track previews
only.  To obtain an entire track, you obviously have to purchase and
download it normally through the Beatport store!

Possible future improvements:
- Playlists for albums or other track-list pages
- User login to play from My Beatport, Hold Bin, or Cart
---
 youtube_dl/extractor/__init__.py    |   1 +
 youtube_dl/extractor/beatportpro.py | 101 ++++++++++++++++++++++++++++
 2 files changed, 102 insertions(+)
 create mode 100644 youtube_dl/extractor/beatportpro.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 7f9523c2b..ac765fdb8 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -37,6 +37,7 @@ from .bandcamp import BandcampIE, BandcampAlbumIE
 from .bbccouk import BBCCoUkIE
 from .beeg import BeegIE
 from .behindkink import BehindKinkIE
+from .beatportpro import BeatportProIE
 from .bet import BetIE
 from .bild import BildIE
 from .bilibili import BiliBiliIE
diff --git a/youtube_dl/extractor/beatportpro.py b/youtube_dl/extractor/beatportpro.py
new file mode 100644
index 000000000..21048b732
--- /dev/null
+++ b/youtube_dl/extractor/beatportpro.py
@@ -0,0 +1,101 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+import re
+import json
+
+
+class BeatportProIE(InfoExtractor):
+    _VALID_URL = r'https?://pro\.beatport\.com/track/.*/(?P<id>[0-9]+)'
+    _TESTS = [{
+        'url': 'https://pro.beatport.com/track/synesthesia-original-mix/5379371',
+        'md5': 'b3c34d8639a2f6a7f734382358478887',
+        'info_dict': {
+            'id': 5379371,
+            'display-id': 'synesthesia-original-mix',
+            'ext': 'mp4',
+            'title': 'Froxic - Synesthesia (Original Mix)',
+        },
+    }, {
+        'url': 'https://pro.beatport.com/track/love-and-war-original-mix/3756896',
+        'md5': 'e44c3025dfa38c6577fbaeb43da43514',
+        'info_dict': {
+            'id': 3756896,
+            'display-id': 'love-and-war-original-mix',
+            'ext': 'mp3',
+            'title': 'Wolfgang Gartner - Love & War (Original Mix)',
+        },
+    }, {
+        'url': 'https://pro.beatport.com/track/birds-original-mix/4991738',
+        'md5': 'a1fd8e8046de3950fd039304c186c05f',
+        'info_dict': {
+            'id': 4991738,
+            'display-id': 'birds-original-mix',
+            'ext': 'mp4',
+            'title': "Tos, Middle Milk, Mumblin' Johnsson - Birds (Original Mix)",
+        }
+    }]
+
+    def _real_extract(self, url):
+        track_id = self._match_id(url)
+        webpage = self._download_webpage(url, track_id)
+
+        # Extract "Playables" JSON information from the page
+        playables = self._search_regex(r'window\.Playables = ({.*?});', webpage,
+                                       'playables info', flags=re.DOTALL)
+        playables = json.loads(playables)
+
+        # Find first track with matching ID (always the first one listed?)
+        track = next(filter(lambda t: t['id'] == int(track_id), playables['tracks']))
+
+        # Construct title from artist(s), track name, and mix name
+        title = ', '.join((a['name'] for a in track['artists'])) + ' - ' + track['name']
+        if track['mix']:
+            title += ' (' + track['mix'] + ')'
+
+        # Get format information
+        formats = []
+        for ext, info in track['preview'].items():
+            if info['url'] is None:
+                continue
+            fmt = {
+                'url': info['url'],
+                'ext': ext,
+                'format_id': ext,
+                'vcodec': 'none',
+            }
+            if ext == 'mp3':
+                fmt['preference'] = 0
+                fmt['acodec'] = 'mp3'
+                fmt['abr'] = 96
+                fmt['asr'] = 44100
+            elif ext == 'mp4':
+                fmt['preference'] = 1
+                fmt['acodec'] = 'aac'
+                fmt['abr'] = 96
+                fmt['asr'] = 44100
+            formats += [fmt]
+        formats.sort(key=lambda f: f['preference'])
+
+        # Get album art as thumbnails
+        imgs = []
+        for name, info in track['images'].items():
+            if name == 'dynamic' or info['url'] is None:
+                continue
+            img = {
+                'id': name,
+                'url': info['url'],
+                'height': info['height'],
+                'width': info['width'],
+            }
+            imgs += [img]
+
+        return {
+            'id': track['id'],
+            'display-id': track['slug'],
+            'title': title,
+            'formats': formats,
+            'thumbnails': imgs,
+        }

From 65c5e044c7ab6d3140d30c98abda07785f2974c6 Mon Sep 17 00:00:00 2001
From: "Devin J. Pohly" <djpohly@gmail.com>
Date: Thu, 12 Mar 2015 16:42:55 -0400
Subject: [PATCH 0661/6219] fix python2

---
 youtube_dl/extractor/beatportpro.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/beatportpro.py b/youtube_dl/extractor/beatportpro.py
index 21048b732..c3c70fb33 100644
--- a/youtube_dl/extractor/beatportpro.py
+++ b/youtube_dl/extractor/beatportpro.py
@@ -48,7 +48,7 @@ class BeatportProIE(InfoExtractor):
         playables = json.loads(playables)
 
         # Find first track with matching ID (always the first one listed?)
-        track = next(filter(lambda t: t['id'] == int(track_id), playables['tracks']))
+        track = next(t for t in playables['tracks'] if t['id'] == int(track_id))
 
         # Construct title from artist(s), track name, and mix name
         title = ', '.join((a['name'] for a in track['artists'])) + ' - ' + track['name']

From 054b99a33079dcc4755e46aaf588424b4bb12020 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 12 Mar 2015 22:33:59 +0100
Subject: [PATCH 0662/6219] [jeuxvideo] Fix extraction (fixes #5190)

---
 youtube_dl/extractor/jeuxvideo.py | 36 ++++++++++++++++---------------
 1 file changed, 19 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/jeuxvideo.py b/youtube_dl/extractor/jeuxvideo.py
index 8094cc2e4..d0720ff56 100644
--- a/youtube_dl/extractor/jeuxvideo.py
+++ b/youtube_dl/extractor/jeuxvideo.py
@@ -2,7 +2,6 @@
 
 from __future__ import unicode_literals
 
-import json
 import re
 
 from .common import InfoExtractor
@@ -15,10 +14,10 @@ class JeuxVideoIE(InfoExtractor):
         'url': 'http://www.jeuxvideo.com/reportages-videos-jeux/0004/00046170/tearaway-playstation-vita-gc-2013-tearaway-nous-presente-ses-papiers-d-identite-00115182.htm',
         'md5': '046e491afb32a8aaac1f44dd4ddd54ee',
         'info_dict': {
-            'id': '5182',
+            'id': '114765',
             'ext': 'mp4',
-            'title': 'GC 2013 : Tearaway nous présente ses papiers d\'identité',
-            'description': 'Lorsque les développeurs de LittleBigPlanet proposent un nouveau titre, on ne peut que s\'attendre à un résultat original et fort attrayant.\n',
+            'title': 'Tearaway : GC 2013 : Tearaway nous présente ses papiers d\'identité',
+            'description': 'Lorsque les développeurs de LittleBigPlanet proposent un nouveau titre, on ne peut que s\'attendre à un résultat original et fort attrayant.',
         },
     }
 
@@ -26,26 +25,29 @@ class JeuxVideoIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         title = mobj.group(1)
         webpage = self._download_webpage(url, title)
-        xml_link = self._html_search_regex(
-            r'<param name="flashvars" value="config=(.*?)" />',
+        title = self._html_search_meta('name', webpage)
+        config_url = self._html_search_regex(
+            r'data-src="(/contenu/medias/video.php.*?)"',
             webpage, 'config URL')
+        config_url = 'http://www.jeuxvideo.com' + config_url
 
         video_id = self._search_regex(
-            r'http://www\.jeuxvideo\.com/config/\w+/\d+/(.*?)/\d+_player\.xml',
-            xml_link, 'video ID')
+            r'id=(\d+)',
+            config_url, 'video ID')
 
-        config = self._download_xml(
-            xml_link, title, 'Downloading XML config')
-        info_json = config.find('format.json').text
-        info = json.loads(info_json)['versions'][0]
+        config = self._download_json(
+            config_url, title, 'Downloading JSON config')
 
-        video_url = 'http://video720.jeuxvideo.com/' + info['file']
+        formats = [{
+            'url': source['file'],
+            'format_id': source['label'],
+            'resolution': source['label'],
+        } for source in reversed(config['sources'])]
 
         return {
             'id': video_id,
-            'title': config.find('titre_video').text,
-            'ext': 'mp4',
-            'url': video_url,
+            'title': title,
+            'formats': formats,
             'description': self._og_search_description(webpage),
-            'thumbnail': config.find('image').text,
+            'thumbnail': config.get('image'),
         }

From 486dd09e0bd8032a3234363231c2b592f725cad6 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 13 Mar 2015 08:40:20 +0100
Subject: [PATCH 0663/6219] [YoutubeDL] Check for bytes instead of unicode
 output templates (#5192)

Also adapt the embedding examples for those poor souls still using 2.x.
---
 README.md               | 2 ++
 youtube_dl/YoutubeDL.py | 5 +++++
 2 files changed, 7 insertions(+)

diff --git a/README.md b/README.md
index f41e7ecee..f4fbf0034 100644
--- a/README.md
+++ b/README.md
@@ -515,6 +515,7 @@ youtube-dl makes the best effort to be a good command-line program, and thus sho
 From a Python program, you can embed youtube-dl in a more powerful fashion, like this:
 
 ```python
+from __future__ import unicode_literals
 import youtube_dl
 
 ydl_opts = {}
@@ -527,6 +528,7 @@ Most likely, you'll want to use various options. For a list of what can be done,
 Here's a more complete example of a program that outputs only errors (and a short message after the download is finished), and downloads/converts the video to an mp3 file:
 
 ```python
+from __future__ import unicode_literals
 import youtube_dl
 
 
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index be5b3c1ab..e779fc9a8 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -323,6 +323,11 @@ class YoutubeDL(object):
                 'Set the LC_ALL environment variable to fix this.')
             self.params['restrictfilenames'] = True
 
+        if isinstance(params.get('outtmpl'), bytes):
+            self.report_warning(
+                'Parameter outtmpl is bytes, but should be a unicode string. '
+                'Put  from __future__ import unicode_literals  at the top of your code file or consider switching to Python 3.x.')
+
         if '%(stitle)s' in self.params.get('outtmpl', ''):
             self.report_warning('%(stitle)s is deprecated. Use the %(title)s and the --restrict-filenames flag(which also secures %(uploader)s et al) instead.')
 

From fb7e68833cc67c23428462bf6c1306cbce207369 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 13 Mar 2015 20:51:44 +0600
Subject: [PATCH 0664/6219] [kanalplay] Add extractor (Closes #5188)

---
 youtube_dl/extractor/__init__.py  |  1 +
 youtube_dl/extractor/kanalplay.py | 72 +++++++++++++++++++++++++++++++
 2 files changed, 73 insertions(+)
 create mode 100644 youtube_dl/extractor/kanalplay.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 7f9523c2b..5dc56e330 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -231,6 +231,7 @@ from .jove import JoveIE
 from .jukebox import JukeboxIE
 from .jpopsukitv import JpopsukiIE
 from .kaltura import KalturaIE
+from .kanalplay import KanalPlayIE
 from .kankan import KankanIE
 from .karaoketv import KaraoketvIE
 from .keezmovies import KeezMoviesIE
diff --git a/youtube_dl/extractor/kanalplay.py b/youtube_dl/extractor/kanalplay.py
new file mode 100644
index 000000000..869757ec0
--- /dev/null
+++ b/youtube_dl/extractor/kanalplay.py
@@ -0,0 +1,72 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    float_or_none,
+)
+
+
+class KanalPlayIE(InfoExtractor):
+    IE_DESC = 'Kanal 5/9/11 Play'
+    _VALID_URL = r'https?://(?:www\.)?kanal(?P<channel_id>5|9|11)play\.se/(?:#!/)?(?:play/)?program/\d+/video/(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'http://www.kanal5play.se/#!/play/program/3060212363/video/3270012277',
+        'md5': '',
+        'info_dict': {
+            'id': '2609989',
+            'ext': 'flv',
+            'title': 'Saknar både dusch och avlopp',
+            'description': 'md5:',
+            'duration': 2636.36,
+        },
+    }, {
+        'url': 'http://www.kanal9play.se/#!/play/program/335032/video/246042',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.kanal11play.se/#!/play/program/232835958/video/367135199',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        channel_id = mobj.group('channel_id')
+
+        video = self._download_json(
+            'http://www.kanal%splay.se/api/getVideo?format=FLASH&videoId=%s' % (channel_id, video_id),
+            video_id)
+
+        reasons_for_no_streams = video.get('reasonsForNoStreams')
+        if reasons_for_no_streams:
+            raise ExtractorError(
+                '%s returned error: %s' % (self.IE_NAME, '\n'.join(reasons_for_no_streams)),
+                expected=True)
+
+        title = video['title']
+        description = video.get('description')
+        duration = float_or_none(video.get('length'), 1000)
+        thumbnail = video.get('posterUrl')
+
+        stream_base_url = video['streamBaseUrl']
+
+        formats = [{
+            'url': stream_base_url,
+            'play_path': stream['source'],
+            'ext': 'flv',
+            'tbr': float_or_none(stream.get('bitrate'), 1000),
+            'rtmp_real_time': True,
+        } for stream in video['streams']]
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'duration': duration,
+            'formats': formats,
+        }

From 4a34f69ea6213ffcf5785b30a126b439ceef77e1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 13 Mar 2015 21:38:28 +0600
Subject: [PATCH 0665/6219] [extractor/common] Add subtitles timecode formatter

---
 youtube_dl/extractor/common.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index f9e8e2bad..e5245ec3f 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1062,6 +1062,9 @@ class InfoExtractor(object):
     def _get_automatic_captions(self, *args, **kwargs):
         raise NotImplementedError("This method must be implemented by subclasses")
 
+    def _subtitles_timecode(self, seconds):
+        return '%02d:%02d:%02d.%03d' % (seconds / 3600, (seconds % 3600) / 60, seconds % 60, (seconds % 1) * 1000)
+
 
 class SearchInfoExtractor(InfoExtractor):
     """

From 3f4327520c316043912ca9a8cd6f70d8121c3aa1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 13 Mar 2015 21:39:29 +0600
Subject: [PATCH 0666/6219] [kanalplay] Extract subtitles

---
 youtube_dl/extractor/kanalplay.py | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/youtube_dl/extractor/kanalplay.py b/youtube_dl/extractor/kanalplay.py
index 869757ec0..772d5aca4 100644
--- a/youtube_dl/extractor/kanalplay.py
+++ b/youtube_dl/extractor/kanalplay.py
@@ -31,6 +31,22 @@ class KanalPlayIE(InfoExtractor):
         'only_matching': True,
     }]
 
+    def _fix_subtitles(self, subs):
+        return '\r\n\r\n'.join(
+            '%s\r\n%s --> %s\r\n%s'
+            % (
+                num,
+                self._subtitles_timecode(item['startMillis'] / 1000.0),
+                self._subtitles_timecode(item['endMillis'] / 1000.0),
+                item['text'],
+            ) for num, item in enumerate(subs, 1))
+
+    def _get_subtitles(self, channel_id, video_id):
+        subs = self._download_json(
+            'http://www.kanal%splay.se/api/subtitles/%s' % (channel_id, video_id),
+            video_id, 'Downloading subtitles JSON', fatal=False)
+        return {'se': [{'ext': 'srt', 'data': self._fix_subtitles(subs)}]} if subs else {}
+
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
@@ -62,6 +78,10 @@ class KanalPlayIE(InfoExtractor):
         } for stream in video['streams']]
         self._sort_formats(formats)
 
+        subtitles = {}
+        if video.get('hasSubtitle'):
+            subtitles = self.extract_subtitles(channel_id, video_id)
+
         return {
             'id': video_id,
             'title': title,
@@ -69,4 +89,5 @@ class KanalPlayIE(InfoExtractor):
             'thumbnail': thumbnail,
             'duration': duration,
             'formats': formats,
+            'subtitles': subtitles,
         }

From 545315a9856743541f7f7b9b0ab42f2af9dd05c7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 13 Mar 2015 21:40:34 +0600
Subject: [PATCH 0667/6219] [nrk] Use generic subtitles timecode formatter

---
 youtube_dl/extractor/nrk.py | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index 1e4cfa2e7..bff36f9d3 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -149,9 +149,6 @@ class NRKTVIE(InfoExtractor):
         }
     ]
 
-    def _seconds2str(self, s):
-        return '%02d:%02d:%02d.%03d' % (s / 3600, (s % 3600) / 60, s % 60, (s % 1) * 1000)
-
     def _debug_print(self, txt):
         if self._downloader.params.get('verbose', False):
             self.to_screen('[debug] %s' % txt)
@@ -168,8 +165,8 @@ class NRKTVIE(InfoExtractor):
         for pos, p in enumerate(ps):
             begin = parse_duration(p.get('begin'))
             duration = parse_duration(p.get('dur'))
-            starttime = self._seconds2str(begin)
-            endtime = self._seconds2str(begin + duration)
+            starttime = self._subtitles_timecode(begin)
+            endtime = self._subtitles_timecode(begin + duration)
             srt += '%s\r\n%s --> %s\r\n%s\r\n\r\n' % (compat_str(pos), starttime, endtime, p.text)
         return {lang: [
             {'ext': 'ttml', 'url': url},

From 1b53778175e43e2bf2cb71885a760d96727ee837 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 13 Mar 2015 21:51:49 +0600
Subject: [PATCH 0668/6219] [beatenpro] Use generic format sort

---
 youtube_dl/extractor/beatportpro.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/beatportpro.py b/youtube_dl/extractor/beatportpro.py
index c3c70fb33..bc201572e 100644
--- a/youtube_dl/extractor/beatportpro.py
+++ b/youtube_dl/extractor/beatportpro.py
@@ -77,7 +77,7 @@ class BeatportProIE(InfoExtractor):
                 fmt['abr'] = 96
                 fmt['asr'] = 44100
             formats += [fmt]
-        formats.sort(key=lambda f: f['preference'])
+        self._sort_formats(formats)
 
         # Get album art as thumbnails
         imgs = []

From 517bcca29925548f9b9b121beec1391ef3ecedec Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 13 Mar 2015 22:01:15 +0600
Subject: [PATCH 0669/6219] [beatenpro] Simplify and improve

---
 youtube_dl/extractor/beatportpro.py | 34 ++++++++++++++---------------
 1 file changed, 16 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/beatportpro.py b/youtube_dl/extractor/beatportpro.py
index bc201572e..69657cbde 100644
--- a/youtube_dl/extractor/beatportpro.py
+++ b/youtube_dl/extractor/beatportpro.py
@@ -1,14 +1,15 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-from .common import InfoExtractor
-
 import re
 import json
 
+from .common import InfoExtractor
+from ..utils import int_or_none
+
 
 class BeatportProIE(InfoExtractor):
-    _VALID_URL = r'https?://pro\.beatport\.com/track/.*/(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://pro\.beatport\.com/track/.+/(?P<id>[0-9]+)'
     _TESTS = [{
         'url': 'https://pro.beatport.com/track/synesthesia-original-mix/5379371',
         'md5': 'b3c34d8639a2f6a7f734382358478887',
@@ -42,20 +43,17 @@ class BeatportProIE(InfoExtractor):
         track_id = self._match_id(url)
         webpage = self._download_webpage(url, track_id)
 
-        # Extract "Playables" JSON information from the page
-        playables = self._search_regex(r'window\.Playables = ({.*?});', webpage,
-                                       'playables info', flags=re.DOTALL)
+        playables = self._search_regex(
+            r'window\.Playables\s*=\s*({.*?});', webpage,
+            'playables info', flags=re.DOTALL)
         playables = json.loads(playables)
 
-        # Find first track with matching ID (always the first one listed?)
         track = next(t for t in playables['tracks'] if t['id'] == int(track_id))
 
-        # Construct title from artist(s), track name, and mix name
         title = ', '.join((a['name'] for a in track['artists'])) + ' - ' + track['name']
         if track['mix']:
             title += ' (' + track['mix'] + ')'
 
-        # Get format information
         formats = []
         for ext, info in track['preview'].items():
             if info['url'] is None:
@@ -76,26 +74,26 @@ class BeatportProIE(InfoExtractor):
                 fmt['acodec'] = 'aac'
                 fmt['abr'] = 96
                 fmt['asr'] = 44100
-            formats += [fmt]
+            formats.append(fmt)
         self._sort_formats(formats)
 
-        # Get album art as thumbnails
-        imgs = []
+        images = []
         for name, info in track['images'].items():
-            if name == 'dynamic' or info['url'] is None:
+            image_url = info.get('url')
+            if name == 'dynamic' or not image_url:
                 continue
             img = {
                 'id': name,
-                'url': info['url'],
-                'height': info['height'],
-                'width': info['width'],
+                'url': image_url,
+                'height': int_or_none(info.get('height')),
+                'width': int_or_none(info.get('width')),
             }
-            imgs += [img]
+            images.append(img)
 
         return {
             'id': track['id'],
             'display-id': track['slug'],
             'title': title,
             'formats': formats,
-            'thumbnails': imgs,
+            'thumbnails': images,
         }

From ba1d4c04883cafb55e40734776d9d8ba2ef85582 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 13 Mar 2015 22:03:58 +0600
Subject: [PATCH 0670/6219] [beatenpro] Improve display_id

---
 youtube_dl/extractor/beatportpro.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/beatportpro.py b/youtube_dl/extractor/beatportpro.py
index 69657cbde..5c072b131 100644
--- a/youtube_dl/extractor/beatportpro.py
+++ b/youtube_dl/extractor/beatportpro.py
@@ -9,7 +9,7 @@ from ..utils import int_or_none
 
 
 class BeatportProIE(InfoExtractor):
-    _VALID_URL = r'https?://pro\.beatport\.com/track/.+/(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://pro\.beatport\.com/track/(?P<display_id>[^/]+)/(?P<id>[0-9]+)'
     _TESTS = [{
         'url': 'https://pro.beatport.com/track/synesthesia-original-mix/5379371',
         'md5': 'b3c34d8639a2f6a7f734382358478887',
@@ -40,8 +40,11 @@ class BeatportProIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        track_id = self._match_id(url)
-        webpage = self._download_webpage(url, track_id)
+        mobj = re.match(self._VALID_URL, url)
+        track_id = mobj.group('id')
+        display_id = mobj.group('display_id')
+
+        webpage = self._download_webpage(url, display_id)
 
         playables = self._search_regex(
             r'window\.Playables\s*=\s*({.*?});', webpage,
@@ -92,7 +95,7 @@ class BeatportProIE(InfoExtractor):
 
         return {
             'id': track['id'],
-            'display-id': track['slug'],
+            'display_id': track.get('slug') or display_id,
             'title': title,
             'formats': formats,
             'thumbnails': images,

From fcd877013e4a8f654c7778019055b57031492889 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 13 Mar 2015 22:11:56 +0600
Subject: [PATCH 0671/6219] [beatenpro] Simplify

---
 youtube_dl/extractor/beatportpro.py | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/beatportpro.py b/youtube_dl/extractor/beatportpro.py
index 5c072b131..12a7faa4f 100644
--- a/youtube_dl/extractor/beatportpro.py
+++ b/youtube_dl/extractor/beatportpro.py
@@ -2,9 +2,9 @@
 from __future__ import unicode_literals
 
 import re
-import json
 
 from .common import InfoExtractor
+from ..compat import compat_str
 from ..utils import int_or_none
 
 
@@ -46,10 +46,11 @@ class BeatportProIE(InfoExtractor):
 
         webpage = self._download_webpage(url, display_id)
 
-        playables = self._search_regex(
-            r'window\.Playables\s*=\s*({.*?});', webpage,
-            'playables info', flags=re.DOTALL)
-        playables = json.loads(playables)
+        playables = self._parse_json(
+            self._search_regex(
+                r'window\.Playables\s*=\s*({.+?});', webpage,
+                'playables info', flags=re.DOTALL),
+            track_id)
 
         track = next(t for t in playables['tracks'] if t['id'] == int(track_id))
 
@@ -59,7 +60,7 @@ class BeatportProIE(InfoExtractor):
 
         formats = []
         for ext, info in track['preview'].items():
-            if info['url'] is None:
+            if not info['url']:
                 continue
             fmt = {
                 'url': info['url'],
@@ -85,16 +86,16 @@ class BeatportProIE(InfoExtractor):
             image_url = info.get('url')
             if name == 'dynamic' or not image_url:
                 continue
-            img = {
+            image = {
                 'id': name,
                 'url': image_url,
                 'height': int_or_none(info.get('height')),
                 'width': int_or_none(info.get('width')),
             }
-            images.append(img)
+            images.append(image)
 
         return {
-            'id': track['id'],
+            'id': compat_str(track.get('id')) or track_id,
             'display_id': track.get('slug') or display_id,
             'title': title,
             'formats': formats,

From bba3fc7960c8cd6f0752c31c55ada804ba7e2ae2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 13 Mar 2015 22:13:50 +0600
Subject: [PATCH 0672/6219] [beatenpro] Fix tests

---
 youtube_dl/extractor/beatportpro.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/beatportpro.py b/youtube_dl/extractor/beatportpro.py
index 12a7faa4f..3c7775d3e 100644
--- a/youtube_dl/extractor/beatportpro.py
+++ b/youtube_dl/extractor/beatportpro.py
@@ -14,8 +14,8 @@ class BeatportProIE(InfoExtractor):
         'url': 'https://pro.beatport.com/track/synesthesia-original-mix/5379371',
         'md5': 'b3c34d8639a2f6a7f734382358478887',
         'info_dict': {
-            'id': 5379371,
-            'display-id': 'synesthesia-original-mix',
+            'id': '5379371',
+            'display_id': 'synesthesia-original-mix',
             'ext': 'mp4',
             'title': 'Froxic - Synesthesia (Original Mix)',
         },
@@ -23,8 +23,8 @@ class BeatportProIE(InfoExtractor):
         'url': 'https://pro.beatport.com/track/love-and-war-original-mix/3756896',
         'md5': 'e44c3025dfa38c6577fbaeb43da43514',
         'info_dict': {
-            'id': 3756896,
-            'display-id': 'love-and-war-original-mix',
+            'id': '3756896',
+            'display_id': 'love-and-war-original-mix',
             'ext': 'mp3',
             'title': 'Wolfgang Gartner - Love & War (Original Mix)',
         },
@@ -32,8 +32,8 @@ class BeatportProIE(InfoExtractor):
         'url': 'https://pro.beatport.com/track/birds-original-mix/4991738',
         'md5': 'a1fd8e8046de3950fd039304c186c05f',
         'info_dict': {
-            'id': 4991738,
-            'display-id': 'birds-original-mix',
+            'id': '4991738',
+            'display_id': 'birds-original-mix',
             'ext': 'mp4',
             'title': "Tos, Middle Milk, Mumblin' Johnsson - Birds (Original Mix)",
         }

From 28c6411e4959e342ec8bc016eb8ca5dbcbdc5d7b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 13 Mar 2015 22:14:51 +0600
Subject: [PATCH 0673/6219] Credit @djpohly for BeatportPro (#5189)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 421df69a6..c10f03b98 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -114,3 +114,4 @@ Ryan Schmidt
 Leslie P. Polzer
 Duncan Keall
 Alexander Mamay
+Devin J. Pohly

From 13598940e3e019ce19eaf87ae872ea52b7cb8740 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Mar 2015 01:27:21 +0600
Subject: [PATCH 0674/6219] [kanalplay] Fix test

---
 youtube_dl/extractor/kanalplay.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/kanalplay.py b/youtube_dl/extractor/kanalplay.py
index 772d5aca4..2bb078036 100644
--- a/youtube_dl/extractor/kanalplay.py
+++ b/youtube_dl/extractor/kanalplay.py
@@ -15,14 +15,17 @@ class KanalPlayIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?kanal(?P<channel_id>5|9|11)play\.se/(?:#!/)?(?:play/)?program/\d+/video/(?P<id>\d+)'
     _TESTS = [{
         'url': 'http://www.kanal5play.se/#!/play/program/3060212363/video/3270012277',
-        'md5': '',
         'info_dict': {
-            'id': '2609989',
+            'id': '3270012277',
             'ext': 'flv',
             'title': 'Saknar både dusch och avlopp',
-            'description': 'md5:',
+            'description': 'md5:6023a95832a06059832ae93bc3c7efb7',
             'duration': 2636.36,
         },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        }
     }, {
         'url': 'http://www.kanal9play.se/#!/play/program/335032/video/246042',
         'only_matching': True,

From 3647136f24b332be81f9866ae5e2d596ca5c884a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Mar 2015 02:12:11 +0600
Subject: [PATCH 0675/6219] [viewster] Add extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/viewster.py | 53 ++++++++++++++++++++++++++++++++
 2 files changed, 54 insertions(+)
 create mode 100644 youtube_dl/extractor/viewster.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 4682996cd..ad133603f 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -559,6 +559,7 @@ from .videoweed import VideoWeedIE
 from .vidme import VidmeIE
 from .vidzi import VidziIE
 from .vier import VierIE, VierVideosIE
+from .viewster import ViewsterIE
 from .vimeo import (
     VimeoIE,
     VimeoAlbumIE,
diff --git a/youtube_dl/extractor/viewster.py b/youtube_dl/extractor/viewster.py
new file mode 100644
index 000000000..d7864d6a6
--- /dev/null
+++ b/youtube_dl/extractor/viewster.py
@@ -0,0 +1,53 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..compat import compat_urllib_request
+
+
+class ViewsterIE(InfoExtractor):
+    _VALID_URL = r'http://(?:www\.)?viewster\.com/movie/(?P<id>\d+-\d+-\d+)'
+    _TEST = {
+        'url': 'http://www.viewster.com/movie/1293-19341-000/hout-wood/',
+        'md5': '8f9d94b282d80c42b378dffdbb11caf3',
+        'info_dict': {
+            'id': '1293-19341-000',
+            'ext': 'flv',
+            'title': "'Hout' (Wood)",
+            'description': 'md5:925733185a9242ef96f436937683f33b',
+        },
+    }
+
+    _ACCEPT_HEADER = 'application/json, text/javascript, */*; q=0.01'
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        request = compat_urllib_request.Request(
+            'http://api.live.viewster.com/api/v1/movielink?movieid=%s&action=movierent&paymethod=fre&price=0&currency=&language=en&subtitlelanguage=x&ischromecast=false' % video_id)
+        request.add_header('Accept', self._ACCEPT_HEADER)
+
+        movie_link = self._download_json(
+            request, video_id, 'Downloading movie link JSON')
+
+        formats = self._extract_f4m_formats(
+            movie_link['url'] + '&hdcore=3.2.0&plugin=flowplayer-3.2.0.1', video_id)
+        self._sort_formats(formats)
+
+        request = compat_urllib_request.Request(
+            'http://api.live.viewster.com/api/v1/movie/%s' % video_id)
+        request.add_header('Accept', self._ACCEPT_HEADER)
+
+        movie = self._download_json(
+            request, video_id, 'Downloading movie metadata JSON')
+
+        title = movie.get('title') or movie['original_title']
+        description = movie.get('synopsis')
+        thumbnail = movie.get('large_artwork') or movie.get('artwork')
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'formats': formats,
+        }

From 7be5a62ed71d574132b648864ce3b741635604f9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Mar 2015 03:18:04 +0600
Subject: [PATCH 0676/6219] [viewster] Improve extraction

---
 youtube_dl/extractor/viewster.py | 122 +++++++++++++++++++++++++------
 1 file changed, 99 insertions(+), 23 deletions(-)

diff --git a/youtube_dl/extractor/viewster.py b/youtube_dl/extractor/viewster.py
index d7864d6a6..1742e66f4 100644
--- a/youtube_dl/extractor/viewster.py
+++ b/youtube_dl/extractor/viewster.py
@@ -6,33 +6,75 @@ from ..compat import compat_urllib_request
 
 class ViewsterIE(InfoExtractor):
     _VALID_URL = r'http://(?:www\.)?viewster\.com/movie/(?P<id>\d+-\d+-\d+)'
-    _TEST = {
+    _TESTS = [{
+        # movielink, paymethod=fre
         'url': 'http://www.viewster.com/movie/1293-19341-000/hout-wood/',
-        'md5': '8f9d94b282d80c42b378dffdbb11caf3',
+        'playlist': [{
+            'md5': '8f9d94b282d80c42b378dffdbb11caf3',
+            'info_dict': {
+                'id': '1293-19341-000-movie',
+                'ext': 'flv',
+                'title': "'Hout' (Wood) - Movie",
+            },
+        }],
         'info_dict': {
             'id': '1293-19341-000',
-            'ext': 'flv',
             'title': "'Hout' (Wood)",
             'description': 'md5:925733185a9242ef96f436937683f33b',
-        },
-    }
+        }
+    }, {
+        # movielink, paymethod=adv
+        'url': 'http://www.viewster.com/movie/1140-11855-000/the-listening-project/',
+        'playlist': [{
+            'md5': '77a005453ca7396cbe3d35c9bea30aef',
+            'info_dict': {
+                'id': '1140-11855-000-movie',
+                'ext': 'flv',
+                'title': "THE LISTENING PROJECT - Movie",
+            },
+        }],
+        'info_dict': {
+            'id': '1140-11855-000',
+            'title': "THE LISTENING PROJECT",
+            'description': 'md5:714421ae9957e112e672551094bf3b08',
+        }
+    }, {
+        # direct links, no movielink
+        'url': 'http://www.viewster.com/movie/1198-56411-000/sinister/',
+        'playlist': [{
+            'md5': '0307b7eac6bfb21ab0577a71f6eebd8f',
+            'info_dict': {
+                'id': '1198-56411-000-trailer',
+                'ext': 'mp4',
+                'title': "Sinister - Trailer",
+            },
+        }, {
+            'md5': '80b9ee3ad69fb368f104cb5d9732ae95',
+            'info_dict': {
+                'id': '1198-56411-000-behind-scenes',
+                'ext': 'mp4',
+                'title': "Sinister - Behind Scenes",
+            },
+        }, {
+            'md5': '3b3ea897ecaa91fca57a8a94ac1b15c5',
+            'info_dict': {
+                'id': '1198-56411-000-scene-from-movie',
+                'ext': 'mp4',
+                'title': "Sinister - Scene from movie",
+            },
+        }],
+        'info_dict': {
+            'id': '1198-56411-000',
+            'title': "Sinister",
+            'description': 'md5:014c40b0488848de9683566a42e33372',
+        }
+    }]
 
     _ACCEPT_HEADER = 'application/json, text/javascript, */*; q=0.01'
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        request = compat_urllib_request.Request(
-            'http://api.live.viewster.com/api/v1/movielink?movieid=%s&action=movierent&paymethod=fre&price=0&currency=&language=en&subtitlelanguage=x&ischromecast=false' % video_id)
-        request.add_header('Accept', self._ACCEPT_HEADER)
-
-        movie_link = self._download_json(
-            request, video_id, 'Downloading movie link JSON')
-
-        formats = self._extract_f4m_formats(
-            movie_link['url'] + '&hdcore=3.2.0&plugin=flowplayer-3.2.0.1', video_id)
-        self._sort_formats(formats)
-
         request = compat_urllib_request.Request(
             'http://api.live.viewster.com/api/v1/movie/%s' % video_id)
         request.add_header('Accept', self._ACCEPT_HEADER)
@@ -44,10 +86,44 @@ class ViewsterIE(InfoExtractor):
         description = movie.get('synopsis')
         thumbnail = movie.get('large_artwork') or movie.get('artwork')
 
-        return {
-            'id': video_id,
-            'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
-            'formats': formats,
-        }
+        entries = []
+        for clip in movie['play_list']:
+            entry = None
+
+            # movielink api
+            link_request = clip.get('link_request')
+            if link_request:
+                request = compat_urllib_request.Request(
+                    'http://api.live.viewster.com/api/v1/movielink?movieid=%(movieid)s&action=%(action)s&paymethod=%(paymethod)s&price=%(price)s&currency=%(currency)s&language=%(language)s&subtitlelanguage=%(subtitlelanguage)s&ischromecast=%(ischromecast)s'
+                    % link_request)
+                request.add_header('Accept', self._ACCEPT_HEADER)
+
+                movie_link = self._download_json(
+                    request, video_id, 'Downloading movie link JSON', fatal=False)
+
+                if movie_link:
+                    formats = self._extract_f4m_formats(
+                        movie_link['url'] + '&hdcore=3.2.0&plugin=flowplayer-3.2.0.1', video_id)
+                    self._sort_formats(formats)
+                    entry = {
+                        'formats': formats,
+                    }
+
+            # direct link
+            clip_url = clip.get('clip_data', {}).get('url')
+            if clip_url:
+                entry = {
+                    'url': clip_url,
+                    'ext': 'mp4',
+                }
+
+            if entry:
+                entry.update({
+                    'id': '%s-%s' % (video_id, clip['canonical_title']),
+                    'title': '%s - %s' % (title, clip['title']),
+                })
+                entries.append(entry)
+
+        playlist = self.playlist_result(entries, video_id, title, description)
+        playlist['thumbnail'] = thumbnail
+        return playlist

From 29171bc2d2f07a4eebb3b353fab989d7652a2083 Mon Sep 17 00:00:00 2001
From: Mamay Alexander <alexander@mamay.su>
Date: Sat, 14 Mar 2015 13:56:04 +0600
Subject: [PATCH 0677/6219] [yandexmusic] Site mirrors

---
 youtube_dl/extractor/yandexmusic.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/yandexmusic.py b/youtube_dl/extractor/yandexmusic.py
index b47aecb15..f4c0f5702 100644
--- a/youtube_dl/extractor/yandexmusic.py
+++ b/youtube_dl/extractor/yandexmusic.py
@@ -39,7 +39,7 @@ class YandexMusicBaseIE(InfoExtractor):
 class YandexMusicTrackIE(YandexMusicBaseIE):
     IE_NAME = 'yandexmusic:track'
     IE_DESC = 'Яндекс.Музыка - Трек'
-    _VALID_URL = r'https?://music\.yandex\.ru/album/(?P<album_id>\d+)/track/(?P<id>\d+)'
+    _VALID_URL = r'https?://music\.yandex\.(?:ru|kz|ua|by)/album/(?P<album_id>\d+)/track/(?P<id>\d+)'
 
     _TEST = {
         'url': 'http://music.yandex.ru/album/540508/track/4878838',
@@ -67,7 +67,7 @@ class YandexMusicTrackIE(YandexMusicBaseIE):
 class YandexMusicAlbumIE(YandexMusicBaseIE):
     IE_NAME = 'yandexmusic:album'
     IE_DESC = 'Яндекс.Музыка - Альбом'
-    _VALID_URL = r'https?://music\.yandex\.ru/album/(?P<id>\d+)/?(\?|$)'
+    _VALID_URL = r'https?://music\.yandex\.(?:ru|kz|ua|by)/album/(?P<id>\d+)/?(\?|$)'
 
     _TEST = {
         'url': 'http://music.yandex.ru/album/540508',
@@ -98,7 +98,7 @@ class YandexMusicAlbumIE(YandexMusicBaseIE):
 class YandexMusicPlaylistIE(YandexMusicBaseIE):
     IE_NAME = 'yandexmusic:playlist'
     IE_DESC = 'Яндекс.Музыка - Плейлист'
-    _VALID_URL = r'https?://music\.yandex\.ru/users/[^/]+/playlists/(?P<id>\d+)'
+    _VALID_URL = r'https?://music\.yandex\.(?:ru|kz|ua|by)/users/[^/]+/playlists/(?P<id>\d+)'
 
     _TEST = {
         'url': 'http://music.yandex.ru/users/music.partners/playlists/1245',

From 85741b998660bfb1564ca2c02e233db42059be05 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Mar 2015 15:52:06 +0600
Subject: [PATCH 0678/6219] [8tracks] Use predefined avg duration when duration
 is negative (Closes #5200)

---
 youtube_dl/extractor/eighttracks.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/eighttracks.py b/youtube_dl/extractor/eighttracks.py
index fb5dbbe2b..869ff72d6 100644
--- a/youtube_dl/extractor/eighttracks.py
+++ b/youtube_dl/extractor/eighttracks.py
@@ -117,6 +117,9 @@ class EightTracksIE(InfoExtractor):
         track_count = data['tracks_count']
         duration = data['duration']
         avg_song_duration = float(duration) / track_count
+        # duration is sometimes negative, use predefined avg duration
+        if avg_song_duration <= 0:
+            avg_song_duration = 300
         first_url = 'http://8tracks.com/sets/%s/play?player=sm&mix_id=%s&format=jsonh' % (session, mix_id)
         next_url = first_url
         entries = []

From 05be67e77dcf5c2a87d0549f6ae0f805ce38f758 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Mar 2015 15:54:23 +0600
Subject: [PATCH 0679/6219] [8tracks] Improve extraction

---
 youtube_dl/extractor/eighttracks.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/eighttracks.py b/youtube_dl/extractor/eighttracks.py
index 869ff72d6..49ec2be4c 100644
--- a/youtube_dl/extractor/eighttracks.py
+++ b/youtube_dl/extractor/eighttracks.py
@@ -108,9 +108,10 @@ class EightTracksIE(InfoExtractor):
 
         webpage = self._download_webpage(url, playlist_id)
 
-        json_like = self._search_regex(
-            r"(?s)PAGE.mix = (.*?);\n", webpage, 'trax information')
-        data = json.loads(json_like)
+        data = self._parse_json(
+            self._search_regex(
+                r"(?s)PAGE\.mix\s*=\s*({.+?});\n", webpage, 'trax information'),
+            playlist_id)
 
         session = str(random.randint(0, 1000000000))
         mix_id = data['id']

From a7e01c438d2f7eb28a44056a1feb1b48f51e1096 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Mar 2015 15:55:21 +0600
Subject: [PATCH 0680/6219] [8tracks] Modernize

---
 youtube_dl/extractor/eighttracks.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/eighttracks.py b/youtube_dl/extractor/eighttracks.py
index 49ec2be4c..c84d16142 100644
--- a/youtube_dl/extractor/eighttracks.py
+++ b/youtube_dl/extractor/eighttracks.py
@@ -103,8 +103,7 @@ class EightTracksIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        playlist_id = mobj.group('id')
+        playlist_id = self._match_id(url)
 
         webpage = self._download_webpage(url, playlist_id)
 

From 9202b1b7878e2763a4406ebe2d59962202f6004b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 14 Mar 2015 12:04:49 +0100
Subject: [PATCH 0681/6219] [eighttracks] Remove unused import

---
 youtube_dl/extractor/eighttracks.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/eighttracks.py b/youtube_dl/extractor/eighttracks.py
index c84d16142..0b61ea0ba 100644
--- a/youtube_dl/extractor/eighttracks.py
+++ b/youtube_dl/extractor/eighttracks.py
@@ -3,7 +3,6 @@ from __future__ import unicode_literals
 
 import json
 import random
-import re
 
 from .common import InfoExtractor
 from ..compat import (

From 082b1155a36dc9b51424151f80860e52ee30b55e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 14 Mar 2015 12:06:01 +0100
Subject: [PATCH 0682/6219] [livestream] Extract all videos in events (fixes
 #5198)

The webpage only contains the most recent ones, but if you scroll down more will appear.
---
 youtube_dl/extractor/livestream.py | 41 ++++++++++++++++++++++++++----
 1 file changed, 36 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/livestream.py b/youtube_dl/extractor/livestream.py
index 3642089f7..2467f8bdd 100644
--- a/youtube_dl/extractor/livestream.py
+++ b/youtube_dl/extractor/livestream.py
@@ -2,6 +2,7 @@ from __future__ import unicode_literals
 
 import re
 import json
+import itertools
 
 from .common import InfoExtractor
 from ..compat import (
@@ -40,6 +41,13 @@ class LivestreamIE(InfoExtractor):
             'id': '2245590',
         },
         'playlist_mincount': 4,
+    }, {
+        'url': 'http://new.livestream.com/chess24/tatasteelchess',
+        'info_dict': {
+            'title': 'Tata Steel Chess',
+            'id': '3705884',
+        },
+        'playlist_mincount': 60,
     }, {
         'url': 'https://new.livestream.com/accounts/362/events/3557232/videos/67864563/player?autoPlay=false&height=360&mute=false&width=640',
         'only_matching': True,
@@ -117,6 +125,30 @@ class LivestreamIE(InfoExtractor):
             'view_count': video_data.get('views'),
         }
 
+    def _extract_event(self, info):
+        event_id = compat_str(info['id'])
+        account = compat_str(info['owner_account_id'])
+        root_url = (
+            'https://new.livestream.com/api/accounts/{account}/events/{event}/'
+            'feed.json'.format(account=account, event=event_id))
+
+        def _extract_videos():
+            last_video = None
+            for i in itertools.count(1):
+                if last_video is None:
+                    info_url = root_url
+                else:
+                    info_url = '{root}?&id={id}&newer=-1&type=video'.format(
+                        root=root_url, id=last_video)
+                videos_info = self._download_json(info_url, event_id, 'Downloading page {0}'.format(i))['data']
+                videos_info = [v['data'] for v in videos_info if v['type'] == 'video']
+                if not videos_info:
+                    break
+                for v in videos_info:
+                    yield self._extract_video_info(v)
+                last_video = videos_info[-1]['id']
+        return self.playlist_result(_extract_videos(), event_id, info['full_name'])
+
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
@@ -144,14 +176,13 @@ class LivestreamIE(InfoExtractor):
                 result = result and compat_str(vdata['data']['id']) == vid
             return result
 
-        videos = [self._extract_video_info(video_data['data'])
-                  for video_data in info['feed']['data']
-                  if is_relevant(video_data, video_id)]
         if video_id is None:
             # This is an event page:
-            return self.playlist_result(
-                videos, '%s' % info['id'], info['full_name'])
+            return self._extract_event(info)
         else:
+            videos = [self._extract_video_info(video_data['data'])
+                      for video_data in info['feed']['data']
+                      if is_relevant(video_data, video_id)]
             if not videos:
                 raise ExtractorError('Cannot find video %s' % video_id)
             return videos[0]

From cd65491c306f644d7bb3c7ad98795a3f8660be49 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 15 Mar 2015 00:59:49 +0800
Subject: [PATCH 0683/6219] [Sohu] Add a multiplart video test case

---
 youtube_dl/extractor/sohu.py | 31 +++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/youtube_dl/extractor/sohu.py b/youtube_dl/extractor/sohu.py
index 5adc734d9..7db5b2f13 100644
--- a/youtube_dl/extractor/sohu.py
+++ b/youtube_dl/extractor/sohu.py
@@ -42,6 +42,37 @@ class SohuIE(InfoExtractor):
             'ext': 'mp4',
             'title': '【爱范品】第31期：MWC见不到的奇葩手机',
         }
+    }, {
+        'note': 'Multipart video',
+        'url': 'http://my.tv.sohu.com/pl/8384802/78910339.shtml',
+        'info_dict': {
+            'id': '78910339',
+        },
+        'playlist': [{
+            'md5': 'bdbfb8f39924725e6589c146bc1883ad',
+            'info_dict': {
+                'id': '78910339_part1',
+                'ext': 'mp4',
+                'duration': 294,
+                'title': '【神探苍实战秘籍】第13期 战争之影 赫卡里姆',
+            }
+        }, {
+            'md5': '3e1f46aaeb95354fd10e7fca9fc1804e',
+            'info_dict': {
+                'id': '78910339_part2',
+                'ext': 'mp4',
+                'duration': 300,
+                'title': '【神探苍实战秘籍】第13期 战争之影 赫卡里姆',
+            }
+        }, {
+            'md5': '8407e634175fdac706766481b9443450',
+            'info_dict': {
+                'id': '78910339_part3',
+                'ext': 'mp4',
+                'duration': 150,
+                'title': '【神探苍实战秘籍】第13期 战争之影 赫卡里姆',
+            }
+        }]
     }]
 
     def _real_extract(self, url):

From 2cb434e53ee861c8bcbd538455be107085f444ae Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 15 Mar 2015 01:05:01 +0800
Subject: [PATCH 0684/6219] [Sohu] Fix title extraction

---
 youtube_dl/extractor/sohu.py | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/sohu.py b/youtube_dl/extractor/sohu.py
index 7db5b2f13..ea5cc06b9 100644
--- a/youtube_dl/extractor/sohu.py
+++ b/youtube_dl/extractor/sohu.py
@@ -73,6 +73,17 @@ class SohuIE(InfoExtractor):
                 'title': '【神探苍实战秘籍】第13期 战争之影 赫卡里姆',
             }
         }]
+    }, {
+        'info': 'Video with title containing dash',
+        'url': 'http://my.tv.sohu.com/us/249884221/78932792.shtml',
+        'info_dict': {
+            'id': '78932792',
+            'ext': 'mp4',
+            'title': 'youtube-dl testing video',
+        },
+        'params': {
+            'skip_download': True
+        }
     }]
 
     def _real_extract(self, url):
@@ -97,10 +108,8 @@ class SohuIE(InfoExtractor):
         mytv = mobj.group('mytv') is not None
 
         webpage = self._download_webpage(url, video_id)
-        raw_title = self._html_search_regex(
-            r'(?s)<title>(.+?)</title>',
-            webpage, 'video title')
-        title = raw_title.partition('-')[0].strip()
+
+        title = self._og_search_title(webpage)
 
         vid = self._html_search_regex(
             r'var vid ?= ?["\'](\d+)["\']',

From e7db87f7000143341505cff812d1fa0371ac901e Mon Sep 17 00:00:00 2001
From: phiresky <phiresky@users.noreply.github.com>
Date: Wed, 4 Mar 2015 22:33:56 +0100
Subject: [PATCH 0685/6219] Add metadata from title parser

(Closes #5125)
---
 youtube_dl/__init__.py                        |  5 ++
 youtube_dl/options.py                         |  9 ++++
 youtube_dl/postprocessor/__init__.py          |  2 +
 youtube_dl/postprocessor/ffmpeg.py            | 10 +++-
 youtube_dl/postprocessor/metadatafromtitle.py | 48 +++++++++++++++++++
 5 files changed, 72 insertions(+), 2 deletions(-)
 create mode 100644 youtube_dl/postprocessor/metadatafromtitle.py

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index a08ddd670..852b2fc3d 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -213,6 +213,11 @@ def _real_main(argv=None):
     # PostProcessors
     postprocessors = []
     # Add the metadata pp first, the other pps will copy it
+    if opts.metafromtitle:
+        postprocessors.append({
+            'key': 'MetadataFromTitle',
+            'titleformat': opts.metafromtitle
+        })
     if opts.addmetadata:
         postprocessors.append({'key': 'FFmpegMetadata'})
     if opts.extractaudio:
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index eefe008d5..be9402fdb 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -735,6 +735,15 @@ def parseOpts(overrideArguments=None):
         '--add-metadata',
         action='store_true', dest='addmetadata', default=False,
         help='write metadata to the video file')
+    postproc.add_option(
+        '--metadata-from-title',
+        metavar='FORMAT', dest='metafromtitle',
+        help='parse additional metadata like song title / artist from the video title. \n'
+             'The format syntax is the same as --output, '
+             'the parsed parameters replace existing values.\n'
+             'Additional templates: %(songtitle), %(album), %(artist). \n'
+             'Example: --metadata-from-title "%(artist)s - %(title)s" matches a title like '
+             '"Coldplay - Paradise"')
     postproc.add_option(
         '--xattrs',
         action='store_true', dest='xattrs', default=False,
diff --git a/youtube_dl/postprocessor/__init__.py b/youtube_dl/postprocessor/__init__.py
index 708df3dd4..f39acadce 100644
--- a/youtube_dl/postprocessor/__init__.py
+++ b/youtube_dl/postprocessor/__init__.py
@@ -15,6 +15,7 @@ from .ffmpeg import (
 )
 from .xattrpp import XAttrMetadataPP
 from .execafterdownload import ExecAfterDownloadPP
+from .metadatafromtitle import MetadataFromTitlePP
 
 
 def get_postprocessor(key):
@@ -34,5 +35,6 @@ __all__ = [
     'FFmpegPostProcessor',
     'FFmpegSubtitlesConvertorPP',
     'FFmpegVideoConvertorPP',
+    'MetadataFromTitlePP',
     'XAttrMetadataPP',
 ]
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 30094c2f3..a17113cbf 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -541,11 +541,15 @@ class FFmpegEmbedSubtitlePP(FFmpegPostProcessor):
 class FFmpegMetadataPP(FFmpegPostProcessor):
     def run(self, info):
         metadata = {}
-        if info.get('title') is not None:
+        if info.get('songtitle') is not None:
+            metadata['title'] = info['songtitle']
+        elif info.get('title') is not None:
             metadata['title'] = info['title']
         if info.get('upload_date') is not None:
             metadata['date'] = info['upload_date']
-        if info.get('uploader') is not None:
+        if info.get('artist') is not None:
+            metadata['artist'] = info['artist']
+        elif info.get('uploader') is not None:
             metadata['artist'] = info['uploader']
         elif info.get('uploader_id') is not None:
             metadata['artist'] = info['uploader_id']
@@ -554,6 +558,8 @@ class FFmpegMetadataPP(FFmpegPostProcessor):
             metadata['comment'] = info['description']
         if info.get('webpage_url') is not None:
             metadata['purl'] = info['webpage_url']
+        if info.get('album') is not None:
+            metadata['album'] = info['album']
 
         if not metadata:
             self._downloader.to_screen('[ffmpeg] There isn\'t any metadata to add')
diff --git a/youtube_dl/postprocessor/metadatafromtitle.py b/youtube_dl/postprocessor/metadatafromtitle.py
new file mode 100644
index 000000000..4c9d3aafe
--- /dev/null
+++ b/youtube_dl/postprocessor/metadatafromtitle.py
@@ -0,0 +1,48 @@
+# -*- coding: utf-8 -*-
+
+import re
+
+from .common import PostProcessor
+from ..utils import PostProcessingError
+
+
+class MetadataFromTitlePPError(PostProcessingError):
+    pass
+
+
+class MetadataFromTitlePP(PostProcessor):
+    def __init__(self, downloader, titleformat):
+        self._titleformat = titleformat
+        self._titleregex = self.fmtToRegex(titleformat)
+
+    def fmtToRegex(self, fmt):
+        """
+        Converts a string like
+           '%(title)s - %(artist)s'
+        to a regex like
+           '(?P<title>.+)\ \-\ (?P<artist>.+)'
+        and a list of the named groups [title, artist]
+        """
+        lastpos = 0
+        regex = ""
+        groups = []
+        # replace %(..)s with regex group and escape other string parts
+        for match in re.finditer(r'%\((\w+)\)s', fmt):
+            regex += re.escape(fmt[lastpos:match.start()])
+            regex += r'(?P<' + match.group(1) + '>.+)'
+            lastpos = match.end()
+        if lastpos < len(fmt):
+            regex += re.escape(fmt[lastpos:len(fmt)])
+        return regex
+
+    def run(self, info):
+        title = info['title']
+        match = re.match(self._titleregex, title)
+        if match is None:
+            raise MetadataFromTitlePPError('Could not interpret title of video as "%s"' % self._titleformat)
+        for attribute, value in match.groupdict().items():
+            value = match.group(attribute)
+            info[attribute] = value
+            self._downloader.to_screen('[fromtitle] parsed ' + attribute + ': ' + value)
+
+        return True, info

From 88cf6fb3685c4e012c9f574cbc5f1836c42fc06d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 14 Mar 2015 19:55:42 +0100
Subject: [PATCH 0686/6219] [metadatafromtitle] Some improvements and cleanup

* Remove the 'songtitle' field, 'title' can be used instead.
* Remove newlines in the help text, for consistency with other options.
* Add 'from __future__ import unicode_literals'.
* Call '__init__' from the parent class.
* Add test for the format_to_regex method
---
 test/test_postprocessors.py                   | 17 +++++++++++++++++
 youtube_dl/options.py                         |  6 +++---
 youtube_dl/postprocessor/ffmpeg.py            |  4 +---
 youtube_dl/postprocessor/metadatafromtitle.py |  9 ++++-----
 4 files changed, 25 insertions(+), 11 deletions(-)
 create mode 100644 test/test_postprocessors.py

diff --git a/test/test_postprocessors.py b/test/test_postprocessors.py
new file mode 100644
index 000000000..addb69d6f
--- /dev/null
+++ b/test/test_postprocessors.py
@@ -0,0 +1,17 @@
+#!/usr/bin/env python
+
+from __future__ import unicode_literals
+
+# Allow direct execution
+import os
+import sys
+import unittest
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from youtube_dl.postprocessor import MetadataFromTitlePP
+
+
+class TestMetadataFromTitle(unittest.TestCase):
+    def test_format_to_regex(self):
+        pp = MetadataFromTitlePP(None, '%(title)s - %(artist)s')
+        self.assertEqual(pp._titleregex, '(?P<title>.+)\ \-\ (?P<artist>.+)')
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index be9402fdb..4e6e47d6f 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -738,10 +738,10 @@ def parseOpts(overrideArguments=None):
     postproc.add_option(
         '--metadata-from-title',
         metavar='FORMAT', dest='metafromtitle',
-        help='parse additional metadata like song title / artist from the video title. \n'
+        help='parse additional metadata like song title / artist from the video title. '
              'The format syntax is the same as --output, '
-             'the parsed parameters replace existing values.\n'
-             'Additional templates: %(songtitle), %(album), %(artist). \n'
+             'the parsed parameters replace existing values. '
+             'Additional templates: %(album), %(artist). '
              'Example: --metadata-from-title "%(artist)s - %(title)s" matches a title like '
              '"Coldplay - Paradise"')
     postproc.add_option(
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index a17113cbf..b6f51cfd5 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -541,9 +541,7 @@ class FFmpegEmbedSubtitlePP(FFmpegPostProcessor):
 class FFmpegMetadataPP(FFmpegPostProcessor):
     def run(self, info):
         metadata = {}
-        if info.get('songtitle') is not None:
-            metadata['title'] = info['songtitle']
-        elif info.get('title') is not None:
+        if info.get('title') is not None:
             metadata['title'] = info['title']
         if info.get('upload_date') is not None:
             metadata['date'] = info['upload_date']
diff --git a/youtube_dl/postprocessor/metadatafromtitle.py b/youtube_dl/postprocessor/metadatafromtitle.py
index 4c9d3aafe..5019433d3 100644
--- a/youtube_dl/postprocessor/metadatafromtitle.py
+++ b/youtube_dl/postprocessor/metadatafromtitle.py
@@ -1,4 +1,4 @@
-# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
 
 import re
 
@@ -12,20 +12,19 @@ class MetadataFromTitlePPError(PostProcessingError):
 
 class MetadataFromTitlePP(PostProcessor):
     def __init__(self, downloader, titleformat):
+        super(MetadataFromTitlePP, self).__init__(downloader)
         self._titleformat = titleformat
-        self._titleregex = self.fmtToRegex(titleformat)
+        self._titleregex = self.format_to_regex(titleformat)
 
-    def fmtToRegex(self, fmt):
+    def format_to_regex(self, fmt):
         """
         Converts a string like
            '%(title)s - %(artist)s'
         to a regex like
            '(?P<title>.+)\ \-\ (?P<artist>.+)'
-        and a list of the named groups [title, artist]
         """
         lastpos = 0
         regex = ""
-        groups = []
         # replace %(..)s with regex group and escape other string parts
         for match in re.finditer(r'%\((\w+)\)s', fmt):
             regex += re.escape(fmt[lastpos:match.start()])

From 4d1652484f4a81b8d014159e680f7e20feae85e1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 14 Mar 2015 20:25:37 +0100
Subject: [PATCH 0687/6219] [test/unicode_literals] Don't look into the .git
 and .tox directories

The .tox directory contains python code that we can't control
---
 test/test_unicode_literals.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/test/test_unicode_literals.py b/test/test_unicode_literals.py
index 7f816698e..6c1b7ec91 100644
--- a/test/test_unicode_literals.py
+++ b/test/test_unicode_literals.py
@@ -17,13 +17,22 @@ IGNORED_FILES = [
     'buildserver.py',
 ]
 
+IGNORED_DIRS = [
+    '.git',
+    '.tox',
+]
 
 from test.helper import assertRegexpMatches
 
 
 class TestUnicodeLiterals(unittest.TestCase):
     def test_all_files(self):
-        for dirpath, _, filenames in os.walk(rootDir):
+        for dirpath, dirnames, filenames in os.walk(rootDir):
+            for ignore_dir in IGNORED_DIRS:
+                if ignore_dir in dirnames:
+                    # If we remove the directory from dirnames os.walk won't
+                    # recurse into it
+                    dirnames.remove(ignore_dir)
             for basename in filenames:
                 if not basename.endswith('.py'):
                     continue

From 8508557e77362bf1e5bcc919e89e6b4aa5d101ba Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 14 Mar 2015 20:51:42 +0100
Subject: [PATCH 0688/6219] [test/YoutubeDL] Use valid urls

It failed on python 3.4 when building the http_headers field
---
 test/test_YoutubeDL.py | 46 ++++++++++++++++++++++--------------------
 1 file changed, 24 insertions(+), 22 deletions(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 055e42555..db8a47d2d 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -15,6 +15,8 @@ from youtube_dl import YoutubeDL
 from youtube_dl.extractor import YoutubeIE
 from youtube_dl.postprocessor.common import PostProcessor
 
+TEST_URL = 'http://localhost/sample.mp4'
+
 
 class YDL(FakeYDL):
     def __init__(self, *args, **kwargs):
@@ -46,8 +48,8 @@ class TestFormatSelection(unittest.TestCase):
         ydl = YDL()
         ydl.params['prefer_free_formats'] = True
         formats = [
-            {'ext': 'webm', 'height': 460, 'url': 'x'},
-            {'ext': 'mp4', 'height': 460, 'url': 'y'},
+            {'ext': 'webm', 'height': 460, 'url': TEST_URL},
+            {'ext': 'mp4', 'height': 460, 'url': TEST_URL},
         ]
         info_dict = _make_result(formats)
         yie = YoutubeIE(ydl)
@@ -60,8 +62,8 @@ class TestFormatSelection(unittest.TestCase):
         ydl = YDL()
         ydl.params['prefer_free_formats'] = True
         formats = [
-            {'ext': 'webm', 'height': 720, 'url': 'a'},
-            {'ext': 'mp4', 'height': 1080, 'url': 'b'},
+            {'ext': 'webm', 'height': 720, 'url': TEST_URL},
+            {'ext': 'mp4', 'height': 1080, 'url': TEST_URL},
         ]
         info_dict['formats'] = formats
         yie = YoutubeIE(ydl)
@@ -74,9 +76,9 @@ class TestFormatSelection(unittest.TestCase):
         ydl = YDL()
         ydl.params['prefer_free_formats'] = False
         formats = [
-            {'ext': 'webm', 'height': 720, 'url': '_'},
-            {'ext': 'mp4', 'height': 720, 'url': '_'},
-            {'ext': 'flv', 'height': 720, 'url': '_'},
+            {'ext': 'webm', 'height': 720, 'url': TEST_URL},
+            {'ext': 'mp4', 'height': 720, 'url': TEST_URL},
+            {'ext': 'flv', 'height': 720, 'url': TEST_URL},
         ]
         info_dict['formats'] = formats
         yie = YoutubeIE(ydl)
@@ -88,8 +90,8 @@ class TestFormatSelection(unittest.TestCase):
         ydl = YDL()
         ydl.params['prefer_free_formats'] = False
         formats = [
-            {'ext': 'flv', 'height': 720, 'url': '_'},
-            {'ext': 'webm', 'height': 720, 'url': '_'},
+            {'ext': 'flv', 'height': 720, 'url': TEST_URL},
+            {'ext': 'webm', 'height': 720, 'url': TEST_URL},
         ]
         info_dict['formats'] = formats
         yie = YoutubeIE(ydl)
@@ -133,10 +135,10 @@ class TestFormatSelection(unittest.TestCase):
 
     def test_format_selection(self):
         formats = [
-            {'format_id': '35', 'ext': 'mp4', 'preference': 1, 'url': '_'},
-            {'format_id': '45', 'ext': 'webm', 'preference': 2, 'url': '_'},
-            {'format_id': '47', 'ext': 'webm', 'preference': 3, 'url': '_'},
-            {'format_id': '2', 'ext': 'flv', 'preference': 4, 'url': '_'},
+            {'format_id': '35', 'ext': 'mp4', 'preference': 1, 'url': TEST_URL},
+            {'format_id': '45', 'ext': 'webm', 'preference': 2, 'url': TEST_URL},
+            {'format_id': '47', 'ext': 'webm', 'preference': 3, 'url': TEST_URL},
+            {'format_id': '2', 'ext': 'flv', 'preference': 4, 'url': TEST_URL},
         ]
         info_dict = _make_result(formats)
 
@@ -167,10 +169,10 @@ class TestFormatSelection(unittest.TestCase):
 
     def test_format_selection_audio(self):
         formats = [
-            {'format_id': 'audio-low', 'ext': 'webm', 'preference': 1, 'vcodec': 'none', 'url': '_'},
-            {'format_id': 'audio-mid', 'ext': 'webm', 'preference': 2, 'vcodec': 'none', 'url': '_'},
-            {'format_id': 'audio-high', 'ext': 'flv', 'preference': 3, 'vcodec': 'none', 'url': '_'},
-            {'format_id': 'vid', 'ext': 'mp4', 'preference': 4, 'url': '_'},
+            {'format_id': 'audio-low', 'ext': 'webm', 'preference': 1, 'vcodec': 'none', 'url': TEST_URL},
+            {'format_id': 'audio-mid', 'ext': 'webm', 'preference': 2, 'vcodec': 'none', 'url': TEST_URL},
+            {'format_id': 'audio-high', 'ext': 'flv', 'preference': 3, 'vcodec': 'none', 'url': TEST_URL},
+            {'format_id': 'vid', 'ext': 'mp4', 'preference': 4, 'url': TEST_URL},
         ]
         info_dict = _make_result(formats)
 
@@ -185,8 +187,8 @@ class TestFormatSelection(unittest.TestCase):
         self.assertEqual(downloaded['format_id'], 'audio-low')
 
         formats = [
-            {'format_id': 'vid-low', 'ext': 'mp4', 'preference': 1, 'url': '_'},
-            {'format_id': 'vid-high', 'ext': 'mp4', 'preference': 2, 'url': '_'},
+            {'format_id': 'vid-low', 'ext': 'mp4', 'preference': 1, 'url': TEST_URL},
+            {'format_id': 'vid-high', 'ext': 'mp4', 'preference': 2, 'url': TEST_URL},
         ]
         info_dict = _make_result(formats)
 
@@ -228,9 +230,9 @@ class TestFormatSelection(unittest.TestCase):
 
     def test_format_selection_video(self):
         formats = [
-            {'format_id': 'dash-video-low', 'ext': 'mp4', 'preference': 1, 'acodec': 'none', 'url': '_'},
-            {'format_id': 'dash-video-high', 'ext': 'mp4', 'preference': 2, 'acodec': 'none', 'url': '_'},
-            {'format_id': 'vid', 'ext': 'mp4', 'preference': 3, 'url': '_'},
+            {'format_id': 'dash-video-low', 'ext': 'mp4', 'preference': 1, 'acodec': 'none', 'url': TEST_URL},
+            {'format_id': 'dash-video-high', 'ext': 'mp4', 'preference': 2, 'acodec': 'none', 'url': TEST_URL},
+            {'format_id': 'vid', 'ext': 'mp4', 'preference': 3, 'url': TEST_URL},
         ]
         info_dict = _make_result(formats)
 

From 873383e9bd2496a31870e6456007463c1f873b33 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 14 Mar 2015 21:41:15 +0100
Subject: [PATCH 0689/6219] tox.ini: Run the same command as 'make offlinetest'
 by default

---
 tox.ini | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tox.ini b/tox.ini
index ed01e3386..6fe7fc8f5 100644
--- a/tox.ini
+++ b/tox.ini
@@ -4,5 +4,8 @@ envlist = py26,py27,py33
 deps =
    nose
    coverage
-commands = nosetests --verbose {posargs:test}  # --with-coverage --cover-package=youtube_dl --cover-html
+defaultargs = test --exclude test_download.py --exclude test_age_restriction.py
+    --exclude test_subtitles.py --exclude test_write_annotations.py
+    --exclude test_youtube_lists.py
+commands = nosetests --verbose {posargs:{[testenv]defaultargs}}  # --with-coverage --cover-package=youtube_dl --cover-html
                                                # test.test_download:TestDownload.test_NowVideo

From 7c42327e0e2f62a64771769e17cda33ea2a4a10d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 14 Mar 2015 21:41:56 +0100
Subject: [PATCH 0690/6219] tox.ini: Add python 3.4

---
 tox.ini | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tox.ini b/tox.ini
index 6fe7fc8f5..00c6e00e3 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,5 +1,5 @@
 [tox]
-envlist = py26,py27,py33
+envlist = py26,py27,py33,py34
 [testenv]
 deps =
    nose

From 8f4cc22455f1c08f8e95f867f40c9f378bfe7a49 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 15 Mar 2015 10:08:14 +0600
Subject: [PATCH 0691/6219] [aftenposten] Adapt to new URL format

---
 youtube_dl/extractor/aftenposten.py | 11 +++--------
 1 file changed, 3 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/aftenposten.py b/youtube_dl/extractor/aftenposten.py
index 2b257ede7..e15c015fb 100644
--- a/youtube_dl/extractor/aftenposten.py
+++ b/youtube_dl/extractor/aftenposten.py
@@ -14,10 +14,10 @@ from ..utils import (
 
 
 class AftenpostenIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?aftenposten\.no/webtv/([^/]+/)*(?P<id>[^/]+)-\d+\.html'
+    _VALID_URL = r'https?://(?:www\.)?aftenposten\.no/webtv/(?:#!/)?video/(?P<id>\d+)'
 
     _TEST = {
-        'url': 'http://www.aftenposten.no/webtv/serier-og-programmer/sweatshopenglish/TRAILER-SWEATSHOP---I-cant-take-any-more-7800835.html?paging=&section=webtv_serierogprogrammer_sweatshop_sweatshopenglish',
+        'url': 'http://www.aftenposten.no/webtv/#!/video/21039/trailer-sweatshop-i-can-t-take-any-more',
         'md5': 'fd828cd29774a729bf4d4425fe192972',
         'info_dict': {
             'id': '21039',
@@ -30,12 +30,7 @@ class AftenpostenIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        display_id = self._match_id(url)
-
-        webpage = self._download_webpage(url, display_id)
-
-        video_id = self._html_search_regex(
-            r'data-xs-id="(\d+)"', webpage, 'video id')
+        video_id = self._match_id(url)
 
         data = self._download_xml(
             'http://frontend.xstream.dk/ap/feed/video/?platform=web&id=%s' % video_id, video_id)

From 613b2d9dc6eaf515898516c654f79cee8951a269 Mon Sep 17 00:00:00 2001
From: Kang Hyojun <admire9@gmail.com>
Date: Sun, 15 Mar 2015 20:18:23 +0900
Subject: [PATCH 0692/6219] Fix mistyped docstring indent

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index e779fc9a8..5a83bc956 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -635,7 +635,7 @@ class YoutubeDL(object):
         Returns a list with a dictionary for each video we find.
         If 'download', also downloads the videos.
         extra_info is a dict containing the extra values to add to each result
-         '''
+        '''
 
         if ie_key:
             ies = [self.get_info_extractor(ie_key)]

From ec1b9577ba31b7b078eec370680f84909b2e70e3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 15 Mar 2015 22:42:13 +0600
Subject: [PATCH 0693/6219] [cloudy] Fix key extraction (Closes #5211)

---
 youtube_dl/extractor/cloudy.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/cloudy.py b/youtube_dl/extractor/cloudy.py
index abf8cc280..0fa720ee8 100644
--- a/youtube_dl/extractor/cloudy.py
+++ b/youtube_dl/extractor/cloudy.py
@@ -105,6 +105,7 @@ class CloudyIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         file_key = self._search_regex(
-            r'filekey\s*=\s*"([^"]+)"', webpage, 'file_key')
+            [r'key\s*:\s*"([^"]+)"', r'filekey\s*=\s*"([^"]+)"'],
+            webpage, 'file_key')
 
         return self._extract_video(video_host, video_id, file_key)

From 39aa42ffbbe4df95ded3e70d4126808898a43f24 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 16 Mar 2015 00:21:38 +0600
Subject: [PATCH 0694/6219] [ard] Capture and output time restricted videos
 (Closes #5213)

---
 youtube_dl/extractor/ard.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/ard.py b/youtube_dl/extractor/ard.py
index 783b53e23..6a35ea463 100644
--- a/youtube_dl/extractor/ard.py
+++ b/youtube_dl/extractor/ard.py
@@ -50,6 +50,9 @@ class ARDMediathekIE(InfoExtractor):
         if '>Der gewünschte Beitrag ist nicht mehr verfügbar.<' in webpage:
             raise ExtractorError('Video %s is no longer available' % video_id, expected=True)
 
+        if 'Diese Sendung ist für Jugendliche unter 12 Jahren nicht geeignet. Der Clip ist deshalb nur von 20 bis 6 Uhr verfügbar.' in webpage:
+            raise ExtractorError('This program is only suitable for those aged 12 and older. Video %s is therefore only available between 20 pm and 6 am.' % video_id, expected=True)
+
         if re.search(r'[\?&]rss($|[=&])', url):
             doc = parse_xml(webpage)
             if doc.tag == 'rss':

From 1de4ac1385bf0bedd753f41496055c53a14c20ca Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 15 Mar 2015 19:38:50 +0100
Subject: [PATCH 0695/6219] release 2015.03.15

---
 README.md              | 3 +++
 docs/supportedsites.md | 8 ++++++++
 youtube_dl/version.py  | 2 +-
 3 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index f4fbf0034..4f9fc8174 100644
--- a/README.md
+++ b/README.md
@@ -228,6 +228,9 @@ which means you can modify it, redistribute it or use it however you like.
     --embed-subs                     embed subtitles in the video (only for mp4 videos)
     --embed-thumbnail                embed thumbnail in the audio as cover art
     --add-metadata                   write metadata to the video file
+    --metadata-from-title FORMAT     parse additional metadata like song title / artist from the video title. The format syntax is the same as --output, the parsed
+                                     parameters replace existing values. Additional templates: %(album), %(artist). Example: --metadata-from-title "%(artist)s -
+                                     %(title)s" matches a title like "Coldplay - Paradise"
     --xattrs                         write metadata to the video file's xattrs (using dublin core and xdg standards)
     --fixup POLICY                   Automatically correct known faults of the file. One of never (do nothing), warn (only emit a warning), detect_or_warn(the default;
                                      fix file if we can, warn otherwise)
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 80a696ee3..d6a1e67c6 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -47,6 +47,7 @@
  - **Bandcamp**
  - **Bandcamp:album**
  - **bbc.co.uk**: BBC iPlayer
+ - **BeatportPro**
  - **Beeg**
  - **BehindKink**
  - **Bet**
@@ -145,6 +146,7 @@
  - **Firstpost**
  - **Flickr**
  - **Folketinget**: Folketinget (ft.dk; Danish parliament)
+ - **FootyRoom**
  - **Foxgay**
  - **FoxNews**
  - **france2.fr:generation-quoi**
@@ -213,6 +215,7 @@
  - **jpopsuki.tv**
  - **Jukebox**
  - **Kaltura**
+ - **KanalPlay**: Kanal 5/9/11 Play
  - **Kankan**
  - **Karaoketv**
  - **keek**
@@ -414,6 +417,7 @@
  - **SportBox**
  - **SportDeutschland**
  - **SRMediathek**: Saarländischer Rundfunk
+ - **SSA**
  - **stanfordoc**: Stanford Open ClassRoom
  - **Steam**
  - **streamcloud.eu**
@@ -510,6 +514,7 @@
  - **Vidzi**
  - **vier**
  - **vier:videos**
+ - **Viewster**
  - **viki**
  - **vimeo**
  - **vimeo:album**
@@ -556,6 +561,9 @@
  - **XXXYMovies**
  - **Yahoo**: Yahoo screen and movies
  - **Yam**
+ - **yandexmusic:album**: Яндекс.Музыка - Альбом
+ - **yandexmusic:playlist**: Яндекс.Музыка - Плейлист
+ - **yandexmusic:track**: Яндекс.Музыка - Трек
  - **YesJapan**
  - **Ynet**
  - **YouJizz**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 1f0c88a4d..7ed07c375 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.03.09'
+__version__ = '2015.03.15'

From d5b559393b793f21f6566321fa08a6bbbe6f524e Mon Sep 17 00:00:00 2001
From: Eduardo Ferro <eduardo.ferro.aldama@gmail.com>
Date: Sun, 15 Mar 2015 21:45:14 +0100
Subject: [PATCH 0696/6219] [rtve] Add new extractor for rtve infantil

---
 youtube_dl/extractor/__init__.py |  2 +-
 youtube_dl/extractor/rtve.py     | 43 ++++++++++++++++++++++++++++++++
 2 files changed, 44 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index ad133603f..1bb3e1a1c 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -405,7 +405,7 @@ from .rtlnow import RTLnowIE
 from .rtl2 import RTL2IE
 from .rtp import RTPIE
 from .rts import RTSIE
-from .rtve import RTVEALaCartaIE, RTVELiveIE
+from .rtve import RTVEALaCartaIE, RTVELiveIE, RTVEInfantilIE
 from .ruhd import RUHDIE
 from .rutube import (
     RutubeIE,
diff --git a/youtube_dl/extractor/rtve.py b/youtube_dl/extractor/rtve.py
index b42442d12..b701a322a 100644
--- a/youtube_dl/extractor/rtve.py
+++ b/youtube_dl/extractor/rtve.py
@@ -126,6 +126,49 @@ class RTVEALaCartaIE(InfoExtractor):
             (s['lang'], [{'ext': 'vtt', 'url': s['src']}])
             for s in subs)
 
+class RTVEInfantilIE(InfoExtractor):
+    IE_NAME = 'rtve.es:alacarta'
+    IE_DESC = 'RTVE a la carta'
+    _VALID_URL = r'https?://(?:www\.)?rtve\.es/infantil/serie/(?P<show>[^/]*)/video/(?P<short_tittle>[^/]*)/(?P<id>[0-9]+)/'
+
+    _TESTS = [{
+        'url': 'http://www.rtve.es/infantil/serie/cleo/video/maneras-vivir/3040283/',
+        'md5': '915319587b33720b8e0357caaa6617e6',
+        'info_dict': {
+            'id': '3040283',
+            'ext': 'mp4',
+            'title': 'Maneras de vivir',
+            'thumbnail': 'http://www.rtve.es/resources/jpg/6/5/1426182947956.JPG',
+            'duration': 357.958,
+        },
+    },]
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        short_tittle = mobj.group('short_tittle')
+        info = self._download_json(
+            'http://www.rtve.es/api/videos/%s/config/alacarta_videos.json' % video_id,
+            video_id)['page']['items'][0]
+
+        webpage = self._download_webpage(url, video_id)
+        vidplayer_id = self._search_regex(
+            r' id="vidplayer([0-9]+)"', webpage, 'internal video ID')
+
+        png_url = 'http://www.rtve.es/ztnr/movil/thumbnail/default/videos/%s.png' % vidplayer_id
+        png = self._download_webpage(png_url, video_id, 'Downloading url information')
+        video_url = _decrypt_url(png)
+
+        return {
+            'id': video_id,
+            'ext': 'mp4',
+            'title': info['title'],
+            'url': video_url,
+            'thumbnail': info.get('image'),
+            'duration': float_or_none(info.get('duration'), scale=1000),
+        }
+
+
 
 class RTVELiveIE(InfoExtractor):
     IE_NAME = 'rtve.es:live'

From b68eedba23cbb4cc419e3568055bfb86c5975054 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 15 Mar 2015 22:17:40 +0100
Subject: [PATCH 0697/6219] [rtve.es:infantil] Minor fixes (closes #5214)

---
 youtube_dl/extractor/rtve.py | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/rtve.py b/youtube_dl/extractor/rtve.py
index b701a322a..13f071077 100644
--- a/youtube_dl/extractor/rtve.py
+++ b/youtube_dl/extractor/rtve.py
@@ -126,10 +126,11 @@ class RTVEALaCartaIE(InfoExtractor):
             (s['lang'], [{'ext': 'vtt', 'url': s['src']}])
             for s in subs)
 
+
 class RTVEInfantilIE(InfoExtractor):
-    IE_NAME = 'rtve.es:alacarta'
-    IE_DESC = 'RTVE a la carta'
-    _VALID_URL = r'https?://(?:www\.)?rtve\.es/infantil/serie/(?P<show>[^/]*)/video/(?P<short_tittle>[^/]*)/(?P<id>[0-9]+)/'
+    IE_NAME = 'rtve.es:infantil'
+    IE_DESC = 'RTVE infantil'
+    _VALID_URL = r'https?://(?:www\.)?rtve\.es/infantil/serie/(?P<show>[^/]*)/video/(?P<short_title>[^/]*)/(?P<id>[0-9]+)/'
 
     _TESTS = [{
         'url': 'http://www.rtve.es/infantil/serie/cleo/video/maneras-vivir/3040283/',
@@ -141,12 +142,10 @@ class RTVEInfantilIE(InfoExtractor):
             'thumbnail': 'http://www.rtve.es/resources/jpg/6/5/1426182947956.JPG',
             'duration': 357.958,
         },
-    },]
+    }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        short_tittle = mobj.group('short_tittle')
+        video_id = self._match_id(url)
         info = self._download_json(
             'http://www.rtve.es/api/videos/%s/config/alacarta_videos.json' % video_id,
             video_id)['page']['items'][0]
@@ -169,7 +168,6 @@ class RTVEInfantilIE(InfoExtractor):
         }
 
 
-
 class RTVELiveIE(InfoExtractor):
     IE_NAME = 'rtve.es:live'
     IE_DESC = 'RTVE.es live streams'

From 90183a46d85758c59814187d9c777e4237a66afe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 15 Mar 2015 22:49:03 +0100
Subject: [PATCH 0698/6219] Credit @eferro for the rtve.es:infantil extractor
 (#5214)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index c10f03b98..872da6071 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -115,3 +115,4 @@ Leslie P. Polzer
 Duncan Keall
 Alexander Mamay
 Devin J. Pohly
+Eduardo Ferro Aldama

From 2e90dff2c2ecade8afb444b086fbc0ad6d2c812d Mon Sep 17 00:00:00 2001
From: felix <m.p.isaev@yandex.com>
Date: Mon, 16 Mar 2015 20:05:02 +0100
Subject: [PATCH 0699/6219] The Daily Show Podcast support

---
 youtube_dl/extractor/__init__.py      |  3 +-
 youtube_dl/extractor/comedycentral.py | 21 ++++++++++++++
 youtube_dl/extractor/libsyn.py        | 41 +++++++++++++++++++++++++++
 3 files changed, 64 insertions(+), 1 deletion(-)
 create mode 100644 youtube_dl/extractor/libsyn.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 1bb3e1a1c..e94779d40 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -84,7 +84,7 @@ from .cnn import (
 )
 from .collegehumor import CollegeHumorIE
 from .collegerama import CollegeRamaIE
-from .comedycentral import ComedyCentralIE, ComedyCentralShowsIE
+from .comedycentral import ComedyCentralIE, ComedyCentralShowsIE, TheDailyShowPodcastIE
 from .comcarcoff import ComCarCoffIE
 from .commonmistakes import CommonMistakesIE, UnicodeBOMIE
 from .condenast import CondeNastIE
@@ -250,6 +250,7 @@ from .letv import (
     LetvPlaylistIE
 )
 from .lifenews import LifeNewsIE
+from .libsyn import LibsynIE
 from .liveleak import LiveLeakIE
 from .livestream import (
     LivestreamIE,
diff --git a/youtube_dl/extractor/comedycentral.py b/youtube_dl/extractor/comedycentral.py
index e5edcc84b..e427b9821 100644
--- a/youtube_dl/extractor/comedycentral.py
+++ b/youtube_dl/extractor/comedycentral.py
@@ -2,6 +2,7 @@ from __future__ import unicode_literals
 
 import re
 
+from .common import InfoExtractor
 from .mtv import MTVServicesInfoExtractor
 from ..compat import (
     compat_str,
@@ -272,3 +273,23 @@ class ComedyCentralShowsIE(MTVServicesInfoExtractor):
             'title': show_name + ' ' + title,
             'description': description,
         }
+
+class TheDailyShowPodcastIE(InfoExtractor):
+    _VALID_URL = r'(?P<scheme>https?:)?//thedailyshow\.cc\.com/podcast/(?P<id>[a-z\-]+)'
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+
+        player_url = self._search_regex(r'<iframe(?:\s+[^>]+)?\s*src="((?:https?:)?//html5-player\.libsyn\.com/embed/episode/id/[0-9]+)', webpage, 'player URL')
+        if player_url.startswith('//'):
+            mobj = re.match(self._VALID_URL, url)
+            scheme = mobj.group('scheme')
+            if not scheme:
+                scheme = 'https:'
+            player_url = scheme + player_url
+
+        return {
+            '_type': 'url_transparent',
+            'url': player_url,
+        }
diff --git a/youtube_dl/extractor/libsyn.py b/youtube_dl/extractor/libsyn.py
new file mode 100644
index 000000000..4b5029f89
--- /dev/null
+++ b/youtube_dl/extractor/libsyn.py
@@ -0,0 +1,41 @@
+# encoding: utf-8
+from .common import InfoExtractor
+from ..utils import (
+    unified_strdate,
+)
+
+class LibsynIE(InfoExtractor):
+    _VALID_URL = r'(?:https?:)?//html5-player\.libsyn\.com/embed/episode/id/(?P<id>[0-9]+)(?:/.*)?'
+
+    def _real_extract(self, url):
+        if url.startswith('//'):
+            url = 'https:' + url
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+
+        podcast_title         = self._search_regex(r'<h2>(.*?)</h2>', webpage, 'show title')
+        podcast_episode_title = self._search_regex(r'<h3>(.*?)</h3>', webpage, 'episode title')
+        podcast_date          = unified_strdate(self._search_regex(r'<div class="release_date">Released: (.*?)</div>', webpage, 'release date'))
+        podcast_description   = self._search_regex(r'<div id="info_text_body">(.*?)</div>', webpage, 'description')
+
+        url0 = self._search_regex(r'var mediaURLLibsyn = "(?P<url0>https?://.*)";', webpage, 'first media URL')
+        url1 = self._search_regex(r'var mediaURL = "(?P<url1>https?://.*)";', webpage, 'second media URL')
+
+        if url0 != url1:
+            formats = [{
+                'url': url0
+            }, {
+                'url': url1
+            }]
+        else:
+            formats = [{
+                'url': url0
+            }]
+
+        return {
+            'id': display_id,
+            'title': podcast_episode_title,
+            'description': podcast_description,
+            'upload_date': podcast_date,
+            'formats': formats,
+        }

From c06a9f8730f41fb24835d482114403f227bc8421 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 17 Mar 2015 19:42:50 +0600
Subject: [PATCH 0700/6219] [arte+7] Check formats (Closes #5224)

---
 youtube_dl/extractor/arte.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index 929dd3cc5..8273bd6c9 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -146,6 +146,7 @@ class ArteTVPlus7IE(InfoExtractor):
 
             formats.append(format)
 
+        self._check_formats(formats, video_id)
         self._sort_formats(formats)
 
         info_dict['formats'] = formats

From 576904bce64ca036a7b21b43fa3f8c023e0bcdb1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 17 Mar 2015 20:01:31 +0600
Subject: [PATCH 0701/6219] [letv] Clarify download message

---
 youtube_dl/extractor/letv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/letv.py b/youtube_dl/extractor/letv.py
index 9ed81a199..1484ac0d2 100644
--- a/youtube_dl/extractor/letv.py
+++ b/youtube_dl/extractor/letv.py
@@ -94,7 +94,7 @@ class LetvIE(InfoExtractor):
 
         play_json = self._download_json(
             play_json_req,
-            media_id, 'playJson data')
+            media_id, 'Downloading playJson data')
 
         # Check for errors
         playstatus = play_json['playstatus']

From 733be371af58ec63be5faa52ee24cab4dd85d388 Mon Sep 17 00:00:00 2001
From: Jeff Buchbinder <jeff@ourexchange.net>
Date: Thu, 15 Jan 2015 21:28:57 -0500
Subject: [PATCH 0702/6219] Add megavideoz.eu support.

---
 youtube_dl/extractor/__init__.py     |  1 +
 youtube_dl/extractor/megavideozeu.py | 39 ++++++++++++++++++++++++++++
 2 files changed, 40 insertions(+)
 create mode 100644 youtube_dl/extractor/megavideozeu.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 1bb3e1a1c..5316af2d1 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -267,6 +267,7 @@ from .macgamestore import MacGameStoreIE
 from .mailru import MailRuIE
 from .malemotion import MalemotionIE
 from .mdr import MDRIE
+from .megavideozeu import MegavideozeuIE
 from .metacafe import MetacafeIE
 from .metacritic import MetacriticIE
 from .mgoon import MgoonIE
diff --git a/youtube_dl/extractor/megavideozeu.py b/youtube_dl/extractor/megavideozeu.py
new file mode 100644
index 000000000..e77b5f734
--- /dev/null
+++ b/youtube_dl/extractor/megavideozeu.py
@@ -0,0 +1,39 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    parse_filesize,
+    unified_strdate,
+)
+
+
+class MegavideozeuIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?megavideoz\.eu/video/(?P<id>.*)(?:.*)'
+
+    def _real_extract(self, url):
+        tmp_video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, tmp_video_id)
+
+        config_php = self._html_search_regex(
+            r'var cnf = \'([^\']+)\'', webpage, 'config.php url')
+
+	configpage = self._download_webpage(config_php, tmp_video_id)
+
+        video_id = self._html_search_regex(
+            r'<mediaid>([^<]+)', configpage, 'video id')
+        video_url = self._html_search_regex(
+            r'<file>([^<]+)', configpage, 'video URL')
+        title = self._html_search_regex(
+            r'<title><!\[CDATA\[([^\]]+)', configpage, 'title')
+        duration = int_or_none(self._html_search_regex(
+            r'<duration>([0-9]+)', configpage, 'duration', fatal=False))
+
+        return {
+            'id': video_id,
+            'url': video_url,
+            'title': title,
+            'duration': duration
+        }

From d41a3fa1b47fc050e206bd16feed02b6a62e7d2d Mon Sep 17 00:00:00 2001
From: Jeff Buchbinder <jeff@ourexchange.net>
Date: Wed, 4 Mar 2015 15:02:13 -0500
Subject: [PATCH 0703/6219] [Primesharetv] Add primeshare.tv extractor, still
 need test data

---
 youtube_dl/extractor/__init__.py     |  1 +
 youtube_dl/extractor/primesharetv.py | 46 ++++++++++++++++++++++++++++
 2 files changed, 47 insertions(+)
 create mode 100644 youtube_dl/extractor/primesharetv.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 5316af2d1..b02812365 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -381,6 +381,7 @@ from .pornhub import (
 )
 from .pornotube import PornotubeIE
 from .pornoxo import PornoXOIE
+from .primesharetv import PrimesharetvIE
 from .promptfile import PromptFileIE
 from .prosiebensat1 import ProSiebenSat1IE
 from .puls4 import Puls4IE
diff --git a/youtube_dl/extractor/primesharetv.py b/youtube_dl/extractor/primesharetv.py
new file mode 100644
index 000000000..967125abc
--- /dev/null
+++ b/youtube_dl/extractor/primesharetv.py
@@ -0,0 +1,46 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    parse_filesize,
+    unified_strdate,
+    urlencode_postdata,
+)
+from ..compat import (
+    compat_urllib_request,
+)
+
+class PrimesharetvIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?primeshare\.tv/download/(?P<id>.*)(?:.*)'
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+       
+        self._sleep(9, video_id)
+        
+        hashtoken = self._search_regex(r' name="hash" value="(.*?)" ', webpage, 'hash token')
+        data = urlencode_postdata({
+            'hash': hashtoken,
+        })
+        headers = {
+            'Referer': url,
+            'Content-Type': 'application/x-www-form-urlencoded',
+        }
+        video_page_request = compat_urllib_request.Request(url, data, headers=headers)
+        video_page = self._download_webpage(video_page_request, None, False, '')
+
+        video_url = self._html_search_regex(
+            r'url: \'(http://l\.primeshare\.tv[^\']+)\',', video_page, 'video url')
+
+        title = self._html_search_regex(
+            r'<h1>Watch&nbsp;[^\(]+\(([^/)]+)\)&nbsp;', video_page, 'title')
+
+        return {
+            'id': video_id,
+            'url': video_url,
+            'title': title,
+            'ext': 'mp4',
+        }

From af69cab21d48da78ff7d2b54effb9dfd5fcfd1d8 Mon Sep 17 00:00:00 2001
From: Jeff Buchbinder <jeff@ourexchange.net>
Date: Wed, 4 Mar 2015 15:18:06 -0500
Subject: [PATCH 0704/6219] [Primesharetv] Add public domain example video

---
 youtube_dl/extractor/primesharetv.py | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/primesharetv.py b/youtube_dl/extractor/primesharetv.py
index 967125abc..7c545761b 100644
--- a/youtube_dl/extractor/primesharetv.py
+++ b/youtube_dl/extractor/primesharetv.py
@@ -15,6 +15,19 @@ from ..compat import (
 class PrimesharetvIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?primeshare\.tv/download/(?P<id>.*)(?:.*)'
 
+    _TESTS = [
+        {
+            'url': 'http://primeshare.tv/download/238790B611',
+            'md5': 'bb41f9f6c0dd434c729f04ce5b677192',
+            'info_dict': {
+                'id': '238790B611',
+                'ext': 'mp4',
+                "title": "Public Domain - 1960s Commercial - Crest Toothpaste-YKsuFona [...]",
+                "duration": 10,
+            },
+        }
+    ]
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
@@ -33,7 +46,7 @@ class PrimesharetvIE(InfoExtractor):
         video_page = self._download_webpage(video_page_request, None, False, '')
 
         video_url = self._html_search_regex(
-            r'url: \'(http://l\.primeshare\.tv[^\']+)\',', video_page, 'video url')
+            r'url: \'(http://[a-z0-9]+\.primeshare\.tv:443/file/get/[^\']+)\',', video_page, 'video url')
 
         title = self._html_search_regex(
             r'<h1>Watch&nbsp;[^\(]+\(([^/)]+)\)&nbsp;', video_page, 'title')

From 13047f41359e32bf56bdc8dd555bae6c1065512d Mon Sep 17 00:00:00 2001
From: Jeff Buchbinder <jeff@ourexchange.net>
Date: Wed, 4 Mar 2015 17:38:21 -0500
Subject: [PATCH 0705/6219] [Primesharetv] Handle file not existing properly.

---
 youtube_dl/extractor/primesharetv.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/primesharetv.py b/youtube_dl/extractor/primesharetv.py
index 7c545761b..570fd2210 100644
--- a/youtube_dl/extractor/primesharetv.py
+++ b/youtube_dl/extractor/primesharetv.py
@@ -1,8 +1,11 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..utils import (
+    ExtractorError,
     int_or_none,
     parse_filesize,
     unified_strdate,
@@ -31,10 +34,12 @@ class PrimesharetvIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
+        if re.search(r'<h1>File not exist</h1>', webpage) is not None:
+            raise ExtractorError('The file does not exist', expected=True)
+        hashtoken = self._search_regex(r' name="hash" value="(.*?)" ', webpage, 'hash token')
        
         self._sleep(9, video_id)
         
-        hashtoken = self._search_regex(r' name="hash" value="(.*?)" ', webpage, 'hash token')
         data = urlencode_postdata({
             'hash': hashtoken,
         })
@@ -44,7 +49,6 @@ class PrimesharetvIE(InfoExtractor):
         }
         video_page_request = compat_urllib_request.Request(url, data, headers=headers)
         video_page = self._download_webpage(video_page_request, None, False, '')
-
         video_url = self._html_search_regex(
             r'url: \'(http://[a-z0-9]+\.primeshare\.tv:443/file/get/[^\']+)\',', video_page, 'video url')
 
@@ -57,3 +61,8 @@ class PrimesharetvIE(InfoExtractor):
             'title': title,
             'ext': 'mp4',
         }
+    
+    def _debug_print(self, txt):
+        if self._downloader.params.get('verbose'):
+            self.to_screen('[debug] %s' % txt)
+

From 0499cd866e7e746658f33e6c2f44f7e1e699ad1b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 17 Mar 2015 21:06:38 +0600
Subject: [PATCH 0706/6219] [primesharetv] Clean up

---
 youtube_dl/extractor/__init__.py     |  2 +-
 youtube_dl/extractor/primesharetv.py | 83 ++++++++++++++--------------
 2 files changed, 43 insertions(+), 42 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index b02812365..3c5401145 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -381,7 +381,7 @@ from .pornhub import (
 )
 from .pornotube import PornotubeIE
 from .pornoxo import PornoXOIE
-from .primesharetv import PrimesharetvIE
+from .primesharetv import PrimeShareTVIE
 from .promptfile import PromptFileIE
 from .prosiebensat1 import ProSiebenSat1IE
 from .puls4 import Puls4IE
diff --git a/youtube_dl/extractor/primesharetv.py b/youtube_dl/extractor/primesharetv.py
index 570fd2210..01cc3d9ea 100644
--- a/youtube_dl/extractor/primesharetv.py
+++ b/youtube_dl/extractor/primesharetv.py
@@ -1,59 +1,65 @@
-# encoding: utf-8
 from __future__ import unicode_literals
 
 import re
 
 from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
-    int_or_none,
-    parse_filesize,
-    unified_strdate,
-    urlencode_postdata,
-)
 from ..compat import (
+    compat_urllib_parse,
     compat_urllib_request,
 )
+from ..utils import ExtractorError
 
-class PrimesharetvIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?primeshare\.tv/download/(?P<id>.*)(?:.*)'
 
-    _TESTS = [
-        {
-            'url': 'http://primeshare.tv/download/238790B611',
-            'md5': 'bb41f9f6c0dd434c729f04ce5b677192',
-            'info_dict': {
-                'id': '238790B611',
-                'ext': 'mp4',
-                "title": "Public Domain - 1960s Commercial - Crest Toothpaste-YKsuFona [...]",
-                "duration": 10,
-            },
-        }
-    ]
+class PrimeShareTVIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?primeshare\.tv/download/(?P<id>[\da-zA-Z]+)'
+
+    _TEST = {
+        'url': 'http://primeshare.tv/download/238790B611',
+        'md5': 'b92d9bf5461137c36228009f31533fbc',
+        'info_dict': {
+            'id': '238790B611',
+            'ext': 'mp4',
+            'title': 'Public Domain - 1960s Commercial - Crest Toothpaste-YKsuFona',
+        },
+    }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
+
         webpage = self._download_webpage(url, video_id)
-        if re.search(r'<h1>File not exist</h1>', webpage) is not None:
-            raise ExtractorError('The file does not exist', expected=True)
-        hashtoken = self._search_regex(r' name="hash" value="(.*?)" ', webpage, 'hash token')
-       
-        self._sleep(9, video_id)
-        
-        data = urlencode_postdata({
-            'hash': hashtoken,
-        })
+
+        if '>File not exist<' in webpage:
+            raise ExtractorError('Video %s does not exist' % video_id, expected=True)
+
+        fields = dict(re.findall(r'''(?x)<input\s+
+            type="hidden"\s+
+            name="([^"]+)"\s+
+            (?:id="[^"]+"\s+)?
+            value="([^"]*)"
+            ''', webpage))
+
         headers = {
             'Referer': url,
             'Content-Type': 'application/x-www-form-urlencoded',
         }
-        video_page_request = compat_urllib_request.Request(url, data, headers=headers)
-        video_page = self._download_webpage(video_page_request, None, False, '')
-        video_url = self._html_search_regex(
-            r'url: \'(http://[a-z0-9]+\.primeshare\.tv:443/file/get/[^\']+)\',', video_page, 'video url')
+
+        wait_time = int(self._search_regex(
+            r'var\s+cWaitTime\s*=\s*(\d+)',
+            webpage, 'wait time', default=7)) + 1
+        self._sleep(wait_time, video_id)
+
+        req = compat_urllib_request.Request(
+            url, compat_urllib_parse.urlencode(fields), headers)
+        video_page = self._download_webpage(
+            req, video_id, 'Downloading video page')
+
+        video_url = self._search_regex(
+            r"url\s*:\s*'([^']+\.primeshare\.tv(?::443)?/file/[^']+)'",
+            video_page, 'video url')
 
         title = self._html_search_regex(
-            r'<h1>Watch&nbsp;[^\(]+\(([^/)]+)\)&nbsp;', video_page, 'title')
+            r'<h1>Watch\s*(?:&nbsp;)?\s*\((.+?)(?:\s*\[\.\.\.\])?\)\s*(?:&nbsp;)?\s*<strong>',
+            video_page, 'title')
 
         return {
             'id': video_id,
@@ -61,8 +67,3 @@ class PrimesharetvIE(InfoExtractor):
             'title': title,
             'ext': 'mp4',
         }
-    
-    def _debug_print(self, txt):
-        if self._downloader.params.get('verbose'):
-            self.to_screen('[debug] %s' % txt)
-

From 219da6bb685765186b7ffb878399c32f44351802 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 17 Mar 2015 21:13:42 +0600
Subject: [PATCH 0707/6219] [megavideoeu] Remove extractor

---
 youtube_dl/extractor/__init__.py     |  1 -
 youtube_dl/extractor/megavideozeu.py | 39 ----------------------------
 2 files changed, 40 deletions(-)
 delete mode 100644 youtube_dl/extractor/megavideozeu.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 3c5401145..bceed92e1 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -267,7 +267,6 @@ from .macgamestore import MacGameStoreIE
 from .mailru import MailRuIE
 from .malemotion import MalemotionIE
 from .mdr import MDRIE
-from .megavideozeu import MegavideozeuIE
 from .metacafe import MetacafeIE
 from .metacritic import MetacriticIE
 from .mgoon import MgoonIE
diff --git a/youtube_dl/extractor/megavideozeu.py b/youtube_dl/extractor/megavideozeu.py
deleted file mode 100644
index e77b5f734..000000000
--- a/youtube_dl/extractor/megavideozeu.py
+++ /dev/null
@@ -1,39 +0,0 @@
-# encoding: utf-8
-from __future__ import unicode_literals
-
-from .common import InfoExtractor
-from ..utils import (
-    int_or_none,
-    parse_filesize,
-    unified_strdate,
-)
-
-
-class MegavideozeuIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?megavideoz\.eu/video/(?P<id>.*)(?:.*)'
-
-    def _real_extract(self, url):
-        tmp_video_id = self._match_id(url)
-
-        webpage = self._download_webpage(url, tmp_video_id)
-
-        config_php = self._html_search_regex(
-            r'var cnf = \'([^\']+)\'', webpage, 'config.php url')
-
-	configpage = self._download_webpage(config_php, tmp_video_id)
-
-        video_id = self._html_search_regex(
-            r'<mediaid>([^<]+)', configpage, 'video id')
-        video_url = self._html_search_regex(
-            r'<file>([^<]+)', configpage, 'video URL')
-        title = self._html_search_regex(
-            r'<title><!\[CDATA\[([^\]]+)', configpage, 'title')
-        duration = int_or_none(self._html_search_regex(
-            r'<duration>([0-9]+)', configpage, 'duration', fatal=False))
-
-        return {
-            'id': video_id,
-            'url': video_url,
-            'title': title,
-            'duration': duration
-        }

From 92a4793b3cac36677bed5423b12b3d97d15c6036 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 17 Mar 2015 21:34:22 +0600
Subject: [PATCH 0708/6219] [utils] Place sanitize url function near other
 sanitizing functions

---
 test/test_utils.py  | 37 +++++++++++++++++++++----------------
 youtube_dl/utils.py | 22 +++++++---------------
 2 files changed, 28 insertions(+), 31 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 4f0ffd482..3431ad24e 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -39,6 +39,7 @@ from youtube_dl.utils import (
     read_batch_urls,
     sanitize_filename,
     sanitize_path,
+    sanitize_url_path_consecutive_slashes,
     shell_quote,
     smuggle_url,
     str_to_int,
@@ -55,7 +56,6 @@ from youtube_dl.utils import (
     xpath_with_ns,
     render_table,
     match_str,
-    url_sanitize_consecutive_slashes,
 )
 
 
@@ -169,6 +169,26 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(sanitize_path('./abc'), 'abc')
         self.assertEqual(sanitize_path('./../abc'), '..\\abc')
 
+    def test_sanitize_url_path_consecutive_slashes(self):
+        self.assertEqual(
+            sanitize_url_path_consecutive_slashes('http://hostname/foo//bar/filename.html'),
+            'http://hostname/foo/bar/filename.html')
+        self.assertEqual(
+            sanitize_url_path_consecutive_slashes('http://hostname//foo/bar/filename.html'),
+            'http://hostname/foo/bar/filename.html')
+        self.assertEqual(
+            sanitize_url_path_consecutive_slashes('http://hostname//'),
+            'http://hostname/')
+        self.assertEqual(
+            sanitize_url_path_consecutive_slashes('http://hostname/foo/bar/filename.html'),
+            'http://hostname/foo/bar/filename.html')
+        self.assertEqual(
+            sanitize_url_path_consecutive_slashes('http://hostname/'),
+            'http://hostname/')
+        self.assertEqual(
+            sanitize_url_path_consecutive_slashes('http://hostname/abc//'),
+            'http://hostname/abc/')
+
     def test_ordered_set(self):
         self.assertEqual(orderedSet([1, 1, 2, 3, 4, 4, 5, 6, 7, 3, 5]), [1, 2, 3, 4, 5, 6, 7])
         self.assertEqual(orderedSet([]), [])
@@ -539,21 +559,6 @@ ffmpeg version 2.4.4 Copyright (c) 2000-2014 the FFmpeg ...'''), '2.4.4')
             'like_count > 100 & dislike_count <? 50 & description',
             {'like_count': 190, 'dislike_count': 10}))
 
-    def test_url_sanitize_consecutive_slashes(self):
-        self.assertEqual(url_sanitize_consecutive_slashes(
-            'http://hostname/foo//bar/filename.html'),
-            'http://hostname/foo/bar/filename.html')
-        self.assertEqual(url_sanitize_consecutive_slashes(
-            'http://hostname//foo/bar/filename.html'),
-            'http://hostname/foo/bar/filename.html')
-        self.assertEqual(url_sanitize_consecutive_slashes(
-            'http://hostname//'), 'http://hostname/')
-        self.assertEqual(url_sanitize_consecutive_slashes(
-            'http://hostname/foo/bar/filename.html'),
-            'http://hostname/foo/bar/filename.html')
-        self.assertEqual(url_sanitize_consecutive_slashes(
-            'http://hostname/'), 'http://hostname/')
-
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index e82e3998a..472d4df41 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -326,6 +326,13 @@ def sanitize_path(s):
     return os.path.join(*sanitized_path)
 
 
+def sanitize_url_path_consecutive_slashes(url):
+    """Collapses consecutive slashes in URLs' path"""
+    parsed_url = list(compat_urlparse.urlparse(url))
+    parsed_url[2] = re.sub(r'/{2,}', '/', parsed_url[2])
+    return compat_urlparse.urlunparse(parsed_url)
+
+
 def orderedSet(iterable):
     """ Remove all duplicates from the input iterable """
     res = []
@@ -1804,18 +1811,3 @@ class PerRequestProxyHandler(compat_urllib_request.ProxyHandler):
             return None  # No Proxy
         return compat_urllib_request.ProxyHandler.proxy_open(
             self, req, proxy, type)
-
-
-def url_sanitize_consecutive_slashes(url):
-    """Sanitize URLs with consecutive slashes
-
-    For example, transform both
-        http://hostname/foo//bar/filename.html
-    and
-        http://hostname//foo/bar/filename.html
-    into
-        http://hostname/foo/bar/filename.html
-    """
-    parsed_url = list(compat_urlparse.urlparse(url))
-    parsed_url[2] = re.sub(r'/{2,}', '/', parsed_url[2])
-    return compat_urlparse.urlunparse(parsed_url)

From cd459b1d490ca8c0639220a835f5e6bee3e9a80d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 17 Mar 2015 21:39:31 +0600
Subject: [PATCH 0709/6219] [sohu] Fix test's note info

---
 youtube_dl/extractor/sohu.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/sohu.py b/youtube_dl/extractor/sohu.py
index ea5cc06b9..11edf616a 100644
--- a/youtube_dl/extractor/sohu.py
+++ b/youtube_dl/extractor/sohu.py
@@ -8,7 +8,7 @@ from ..compat import (
     compat_str,
     compat_urllib_request
 )
-from ..utils import url_sanitize_consecutive_slashes
+from ..utils import sanitize_url_path_consecutive_slashes
 
 
 class SohuIE(InfoExtractor):
@@ -74,7 +74,7 @@ class SohuIE(InfoExtractor):
             }
         }]
     }, {
-        'info': 'Video with title containing dash',
+        'note': 'Video with title containing dash',
         'url': 'http://my.tv.sohu.com/us/249884221/78932792.shtml',
         'info_dict': {
             'id': '78932792',
@@ -100,8 +100,9 @@ class SohuIE(InfoExtractor):
             if cn_verification_proxy:
                 req.add_header('Ytdl-request-proxy', cn_verification_proxy)
 
-            return self._download_json(req, video_id,
-                                       'Downloading JSON data for %s' % vid_id)
+            return self._download_json(
+                req, video_id,
+                'Downloading JSON data for %s' % vid_id)
 
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
@@ -146,7 +147,7 @@ class SohuIE(InfoExtractor):
 
                 part_info = part_str.split('|')
 
-                video_url = url_sanitize_consecutive_slashes(
+                video_url = sanitize_url_path_consecutive_slashes(
                     '%s%s?key=%s' % (part_info[0], su[i], part_info[3]))
 
                 formats.append({

From 2ca1c5aa9f8d7db08608de0bd99f6c994ab496d3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 17 Mar 2015 22:27:33 +0600
Subject: [PATCH 0710/6219] [douyutv] Improve and extract all formats

---
 youtube_dl/extractor/__init__.py |  2 +-
 youtube_dl/extractor/douyutv.py  | 72 ++++++++++++++++++++------------
 2 files changed, 46 insertions(+), 28 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index f3901bfc3..73c17aa84 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -107,7 +107,7 @@ from .dctp import DctpTvIE
 from .deezer import DeezerPlaylistIE
 from .dfb import DFBIE
 from .dotsub import DotsubIE
-from .douyutv import DouyutvIE
+from .douyutv import DouyuTVIE
 from .dreisat import DreiSatIE
 from .drbonanza import DRBonanzaIE
 from .drtuber import DrTuberIE
diff --git a/youtube_dl/extractor/douyutv.py b/youtube_dl/extractor/douyutv.py
index e9b92eb3b..d7956e6e4 100644
--- a/youtube_dl/extractor/douyutv.py
+++ b/youtube_dl/extractor/douyutv.py
@@ -2,58 +2,76 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..utils import ExtractorError
 
-from ..utils import (
-    ExtractorError,
-)
 
-class DouyutvIE(InfoExtractor):
+class DouyuTVIE(InfoExtractor):
     _VALID_URL = r'http://(?:www\.)?douyutv\.com/(?P<id>[A-Za-z0-9]+)'
-
-    '''
-    show_status: 1 直播中 ，2 没有直播
-    '''
-
     _TEST = {
         'url': 'http://www.douyutv.com/iseven',
         'info_dict': {
             'id': 'iseven',
-            'title': '清晨醒脑！T-ara根本停不下来！',
             'ext': 'flv',
+            'title': 're:^清晨醒脑！T-ara根本停不下来！ [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
+            'description': 'md5:9e525642c25a0a24302869937cf69d17',
             'thumbnail': 're:^https?://.*\.jpg$',
+            'uploader': '7师傅',
+            'uploader_id': '431925',
             'is_live': True,
+        },
+        'params': {
+            'skip_download': True,
         }
     }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        info_url = 'http://www.douyutv.com/api/client/room/' + video_id
 
-        config = self._download_json(info_url, video_id)
+        config = self._download_json(
+            'http://www.douyutv.com/api/client/room/%s' % video_id, video_id)
 
-        error_code = config.get('error')
-        show_status = config['data'].get('show_status')
+        data = config['data']
+
+        error_code = config.get('error', 0)
+        show_status = data.get('show_status')
         if error_code is not 0:
-            raise ExtractorError('Server reported error %i' % error_code,
-                                 expected=True)
+            raise ExtractorError(
+                'Server reported error %i' % error_code, expected=True)
 
+        # 1 = live, 2 = offline
         if show_status == '2':
-            raise ExtractorError('The live show has not yet started',
-                                 expected=True)
+            raise ExtractorError(
+                'Live stream is offline', expected=True)
 
-        title = config['data'].get('room_name')
-        rtmp_url = config['data'].get('rtmp_url')
-        rtmp_live = config['data'].get('rtmp_live')
-        thumbnail = config['data'].get('room_src') 
+        base_url = data['rtmp_url']
+        live_path = data['rtmp_live']
 
-        url = rtmp_url+'/'+rtmp_live
+        title = self._live_title(data['room_name'])
+        description = data.get('show_details')
+        thumbnail = data.get('room_src')
+
+        uploader = data.get('nickname')
+        uploader_id = data.get('owner_uid')
+
+        multi_formats = data.get('rtmp_multi_bitrate')
+        if not isinstance(multi_formats, dict):
+            multi_formats = {}
+        multi_formats['live'] = live_path
+
+        formats = [{
+            'url': '%s/%s' % (base_url, format_path),
+            'format_id': format_id,
+            'preference': 1 if format_id == 'live' else 0,
+        } for format_id, format_path in multi_formats.items()]
+        self._sort_formats(formats)
 
         return {
             'id': video_id,
             'title': title,
-            'ext':'flv',
-            'url': url,
+            'description': description,
             'thumbnail': thumbnail,
+            'uploader': uploader,
+            'uploader_id': uploader_id,
+            'formats': formats,
             'is_live': True,
-            # TODO more properties (see youtube_dl/extractor/common.py)
-        }
\ No newline at end of file
+        }

From 98f02fdde25bf00fbccfb211abb574f80819a177 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 17 Mar 2015 22:33:05 +0600
Subject: [PATCH 0711/6219] Credit @jbuchbinder for primesharetv (#5123)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 872da6071..512469f4c 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -116,3 +116,4 @@ Duncan Keall
 Alexander Mamay
 Devin J. Pohly
 Eduardo Ferro Aldama
+Jeff Buchbinder

From b1337948ebeeacfcf104d513fac5913d3a5818a7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 17 Mar 2015 23:13:43 +0600
Subject: [PATCH 0712/6219] [grooveshark] Fix extraction

---
 youtube_dl/extractor/grooveshark.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/grooveshark.py b/youtube_dl/extractor/grooveshark.py
index 848d17beb..36ad4915c 100644
--- a/youtube_dl/extractor/grooveshark.py
+++ b/youtube_dl/extractor/grooveshark.py
@@ -140,9 +140,9 @@ class GroovesharkIE(InfoExtractor):
 
         if webpage is not None:
             o = GroovesharkHtmlParser.extract_object_tags(webpage)
-            return (webpage, [x for x in o if x['attrs']['id'] == 'jsPlayerEmbed'])
+            return webpage, [x for x in o if x['attrs']['id'] == 'jsPlayerEmbed']
 
-        return (webpage, None)
+        return webpage, None
 
     def _real_initialize(self):
         self.ts = int(time.time() * 1000)  # timestamp in millis
@@ -154,7 +154,7 @@ class GroovesharkIE(InfoExtractor):
         swf_referer = None
         if self.do_playerpage_request:
             (_, player_objs) = self._get_playerpage(url)
-            if player_objs is not None:
+            if player_objs:
                 swf_referer = self._build_swf_referer(url, player_objs[0])
                 self.to_screen('SWF Referer: %s' % swf_referer)
 

From 84f810160667f9129e6a8d841fae2d0e2e1fec86 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 17 Mar 2015 23:51:40 +0600
Subject: [PATCH 0713/6219] [generic] Follow redirects specified by `Refresh`
 HTTP header

---
 youtube_dl/extractor/generic.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 4e6927b08..0b8d96c27 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1270,8 +1270,14 @@ class GenericIE(InfoExtractor):
         if not found:
             found = re.search(
                 r'(?i)<meta\s+(?=(?:[a-z-]+="[^"]+"\s+)*http-equiv="refresh")'
-                r'(?:[a-z-]+="[^"]+"\s+)*?content="[0-9]{,2};url=\'?([^\'"]+)',
+                r'(?:[a-z-]+="[^"]+"\s+)*?content="[0-9]{,2};\s*(?:URL|url)=\'?([^\'"]+)',
                 webpage)
+            if not found:
+                # Look also in Refresh HTTP header
+                refresh_header = head_response.headers.get('Refresh')
+                if refresh_header:
+                    found = re.search(
+                        r'[0-9]{,2};\s*(?:URL|url)=(.+)', refresh_header)
             if found:
                 new_url = found.group(1)
                 self.report_following_redirect(new_url)

From 9ef4f12b534578ae3d3e47815492c90826c03c36 Mon Sep 17 00:00:00 2001
From: felix <m.p.isaev@yandex.com>
Date: Tue, 17 Mar 2015 18:54:36 +0100
Subject: [PATCH 0714/6219] testcases for libsyn and The Daily Show Podcast
 extractors

---
 youtube_dl/extractor/comedycentral.py | 4 ++++
 youtube_dl/extractor/libsyn.py        | 9 +++++++++
 2 files changed, 13 insertions(+)

diff --git a/youtube_dl/extractor/comedycentral.py b/youtube_dl/extractor/comedycentral.py
index e427b9821..bd3817b56 100644
--- a/youtube_dl/extractor/comedycentral.py
+++ b/youtube_dl/extractor/comedycentral.py
@@ -276,6 +276,10 @@ class ComedyCentralShowsIE(MTVServicesInfoExtractor):
 
 class TheDailyShowPodcastIE(InfoExtractor):
     _VALID_URL = r'(?P<scheme>https?:)?//thedailyshow\.cc\.com/podcast/(?P<id>[a-z\-]+)'
+    _TESTS = [{
+        "url": "http://thedailyshow.cc.com/podcast/episodetwelve",
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
diff --git a/youtube_dl/extractor/libsyn.py b/youtube_dl/extractor/libsyn.py
index 4b5029f89..88379f276 100644
--- a/youtube_dl/extractor/libsyn.py
+++ b/youtube_dl/extractor/libsyn.py
@@ -6,6 +6,15 @@ from ..utils import (
 
 class LibsynIE(InfoExtractor):
     _VALID_URL = r'(?:https?:)?//html5-player\.libsyn\.com/embed/episode/id/(?P<id>[0-9]+)(?:/.*)?'
+    _TESTS = [{
+        'url': "http://html5-player.libsyn.com/embed/episode/id/3377616/",
+        'info_dict': {
+            'id': "3377616",
+            'ext': "mp3",
+            'title': "Episode 12: Bassem Youssef: Egypt's Jon Stewart",
+            'description': "<p>Bassem Youssef joins executive producer Steve Bodow and senior producer Sara Taksler for a conversation about how&nbsp;<em style=\"font-family: Tahoma, Geneva, sans-serif; font-size: 12.8000001907349px;\">The Daily Show</em>&nbsp;inspired Bassem to create&nbsp;<em style=\"font-family: Tahoma, Geneva, sans-serif; font-size: 12.8000001907349px;\">Al-Bernameg</em>, his massively popular (and now banned) Egyptian news satire program. Sara discusses her soon-to-be-released documentary,&nbsp;<em style=\"font-family: Tahoma, Geneva, sans-serif; font-size: 12.8000001907349px;\">Tickling Giants</em>, which chronicles how Bassem and his staff risked their safety every day to tell jokes.</p>",
+        },
+    }]
 
     def _real_extract(self, url):
         if url.startswith('//'):

From ed9a25dd612fb06d9cf007a6491ac9982535a8f9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 18 Mar 2015 00:05:40 +0600
Subject: [PATCH 0715/6219] [generic] Generalize redirect regex

---
 youtube_dl/extractor/generic.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 0b8d96c27..dc5755d12 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1268,16 +1268,16 @@ class GenericIE(InfoExtractor):
             # HTML5 video
             found = re.findall(r'(?s)<video[^<]*(?:>.*?<source[^>]*)?\s+src=["\'](.*?)["\']', webpage)
         if not found:
+            REDIRECT_REGEX = r'[0-9]{,2};\s*(?:URL|url)=\'?([^\'"]+)'
             found = re.search(
                 r'(?i)<meta\s+(?=(?:[a-z-]+="[^"]+"\s+)*http-equiv="refresh")'
-                r'(?:[a-z-]+="[^"]+"\s+)*?content="[0-9]{,2};\s*(?:URL|url)=\'?([^\'"]+)',
+                r'(?:[a-z-]+="[^"]+"\s+)*?content="%s' % REDIRECT_REGEX,
                 webpage)
             if not found:
                 # Look also in Refresh HTTP header
                 refresh_header = head_response.headers.get('Refresh')
                 if refresh_header:
-                    found = re.search(
-                        r'[0-9]{,2};\s*(?:URL|url)=(.+)', refresh_header)
+                    found = re.search(REDIRECT_REGEX, refresh_header)
             if found:
                 new_url = found.group(1)
                 self.report_following_redirect(new_url)

From c3c5c31517ce121740666440a71becaba11176ac Mon Sep 17 00:00:00 2001
From: Todoroki <todoroki.auone@gmail.com>
Date: Wed, 18 Mar 2015 22:19:55 +0900
Subject: [PATCH 0716/6219] fix nm video DL issue when logged in

---
 youtube_dl/extractor/niconico.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/niconico.py b/youtube_dl/extractor/niconico.py
index 7fb4e57df..a48df7f62 100644
--- a/youtube_dl/extractor/niconico.py
+++ b/youtube_dl/extractor/niconico.py
@@ -89,7 +89,7 @@ class NiconicoIE(InfoExtractor):
         if self._AUTHENTICATED:
             # Get flv info
             flv_info_webpage = self._download_webpage(
-                'http://flapi.nicovideo.jp/api/getflv?v=' + video_id,
+                'http://flapi.nicovideo.jp/api/getflv/' + video_id + '?as3=1',
                 video_id, 'Downloading flv info')
         else:
             # Get external player info
@@ -97,7 +97,6 @@ class NiconicoIE(InfoExtractor):
                 'http://ext.nicovideo.jp/thumb_watch/' + video_id, video_id)
             thumb_play_key = self._search_regex(
                 r'\'thumbPlayKey\'\s*:\s*\'(.*?)\'', ext_player_info, 'thumbPlayKey')
-
             # Get flv info
             flv_info_data = compat_urllib_parse.urlencode({
                 'k': thumb_play_key,

From ee580538fa741fc35faa8210e0245e252bf23e40 Mon Sep 17 00:00:00 2001
From: Todoroki <todoroki.auone@gmail.com>
Date: Wed, 18 Mar 2015 22:24:17 +0900
Subject: [PATCH 0717/6219] fix nm video DL issue when logged in

---
 youtube_dl/extractor/niconico.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/niconico.py b/youtube_dl/extractor/niconico.py
index a48df7f62..4d57ac703 100644
--- a/youtube_dl/extractor/niconico.py
+++ b/youtube_dl/extractor/niconico.py
@@ -97,6 +97,7 @@ class NiconicoIE(InfoExtractor):
                 'http://ext.nicovideo.jp/thumb_watch/' + video_id, video_id)
             thumb_play_key = self._search_regex(
                 r'\'thumbPlayKey\'\s*:\s*\'(.*?)\'', ext_player_info, 'thumbPlayKey')
+
             # Get flv info
             flv_info_data = compat_urllib_parse.urlencode({
                 'k': thumb_play_key,

From 2c2c06e359dc3843a86585a9a4b6419f03af510e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 18 Mar 2015 20:28:00 +0600
Subject: [PATCH 0718/6219] [krasview] Fix extraction (Closes #5228)

---
 youtube_dl/extractor/krasview.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/krasview.py b/youtube_dl/extractor/krasview.py
index e46954b47..96f95979a 100644
--- a/youtube_dl/extractor/krasview.py
+++ b/youtube_dl/extractor/krasview.py
@@ -40,8 +40,10 @@ class KrasViewIE(InfoExtractor):
         description = self._og_search_description(webpage, default=None)
         thumbnail = flashvars.get('image') or self._og_search_thumbnail(webpage)
         duration = int_or_none(flashvars.get('duration'))
-        width = int_or_none(self._og_search_property('video:width', webpage, 'video width'))
-        height = int_or_none(self._og_search_property('video:height', webpage, 'video height'))
+        width = int_or_none(self._og_search_property(
+            'video:width', webpage, 'video width', default=None))
+        height = int_or_none(self._og_search_property(
+            'video:height', webpage, 'video height', default=None))
 
         return {
             'id': video_id,

From fa8ce2690447fa06a10398efd5b6dfa96c2ed1b4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Roman=20Le=20N=C3=A9grate?= <roman.lenegrate@gmail.com>
Date: Sun, 15 Mar 2015 22:32:06 +0100
Subject: [PATCH 0719/6219] [mixcloud] Fix extraction like-count

---
 youtube_dl/extractor/mixcloud.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dl/extractor/mixcloud.py
index 1831c6749..7c6c885f9 100644
--- a/youtube_dl/extractor/mixcloud.py
+++ b/youtube_dl/extractor/mixcloud.py
@@ -99,8 +99,7 @@ class MixcloudIE(InfoExtractor):
             r'\s+"profile": "([^"]+)",', webpage, 'uploader id', fatal=False)
         description = self._og_search_description(webpage)
         like_count = str_to_int(self._search_regex(
-            [r'<meta itemprop="interactionCount" content="UserLikes:([0-9]+)"',
-             r'/favorites/?">([0-9]+)<'],
+            r'\bbutton-favorite\b.+m-ajax-toggle-count="([^"]+)"',
             webpage, 'like count', fatal=False))
         view_count = str_to_int(self._search_regex(
             [r'<meta itemprop="interactionCount" content="UserPlays:([0-9]+)"',

From 7a757b7194b1f1151857503baed3fa51d57397aa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 18 Mar 2015 16:50:23 +0100
Subject: [PATCH 0720/6219] [mixcloud] Fix extraction of some metadata

The second test had some wrong info.
I couldn't find the timestamp, so I have removed it.
---
 youtube_dl/extractor/mixcloud.py | 17 +++++------------
 1 file changed, 5 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dl/extractor/mixcloud.py
index 7c6c885f9..2fe55179a 100644
--- a/youtube_dl/extractor/mixcloud.py
+++ b/youtube_dl/extractor/mixcloud.py
@@ -10,7 +10,6 @@ from ..utils import (
     ExtractorError,
     HEADRequest,
     str_to_int,
-    parse_iso8601,
 )
 
 
@@ -27,8 +26,6 @@ class MixcloudIE(InfoExtractor):
             'description': 'After quite a long silence from myself, finally another Drum\'n\'Bass mix with my favourite current dance floor bangers.',
             'uploader': 'Daniel Holbach',
             'uploader_id': 'dholbach',
-            'upload_date': '20111115',
-            'timestamp': 1321359578,
             'thumbnail': 're:https?://.*\.jpg',
             'view_count': int,
             'like_count': int,
@@ -37,12 +34,12 @@ class MixcloudIE(InfoExtractor):
         'url': 'http://www.mixcloud.com/gillespeterson/caribou-7-inch-vinyl-mix-chat/',
         'info_dict': {
             'id': 'gillespeterson-caribou-7-inch-vinyl-mix-chat',
-            'ext': 'm4a',
-            'title': 'Electric Relaxation vol. 3',
+            'ext': 'mp3',
+            'title': 'Caribou 7 inch Vinyl Mix & Chat',
             'description': 'md5:2b8aec6adce69f9d41724647c65875e8',
-            'uploader': 'Daniel Drumz',
+            'uploader': 'Gilles Peterson Worldwide',
             'uploader_id': 'gillespeterson',
-            'thumbnail': 're:https?://.*\.jpg',
+            'thumbnail': 're:https?://.*/images/',
             'view_count': int,
             'like_count': int,
         },
@@ -85,7 +82,7 @@ class MixcloudIE(InfoExtractor):
             raise ExtractorError('Unable to extract track url')
 
         PREFIX = (
-            r'<span class="play-button[^"]*?"'
+            r'm-play-on-spacebar[^>]+'
             r'(?:\s+[a-zA-Z0-9-]+(?:="[^"]+")?)*?\s+')
         title = self._html_search_regex(
             PREFIX + r'm-title="([^"]+)"', webpage, 'title')
@@ -105,9 +102,6 @@ class MixcloudIE(InfoExtractor):
             [r'<meta itemprop="interactionCount" content="UserPlays:([0-9]+)"',
              r'/listeners/?">([0-9,.]+)</a>'],
             webpage, 'play count', fatal=False))
-        timestamp = parse_iso8601(self._search_regex(
-            r'<time itemprop="dateCreated" datetime="([^"]+)">',
-            webpage, 'upload date', default=None))
 
         return {
             'id': track_id,
@@ -117,7 +111,6 @@ class MixcloudIE(InfoExtractor):
             'thumbnail': thumbnail,
             'uploader': uploader,
             'uploader_id': uploader_id,
-            'timestamp': timestamp,
             'view_count': view_count,
             'like_count': like_count,
         }

From aae53774f247edcca7671434c16f7931d5c7b9da Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Roman=20Le=20N=C3=A9grate?= <roman.lenegrate@gmail.com>
Date: Mon, 16 Mar 2015 00:20:06 +0100
Subject: [PATCH 0721/6219] [mixcloud] Try preview server first, then further
 numbers

---
 youtube_dl/extractor/mixcloud.py | 49 ++++++++++++++++++++++++++------
 1 file changed, 41 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dl/extractor/mixcloud.py
index 2fe55179a..21aea0c55 100644
--- a/youtube_dl/extractor/mixcloud.py
+++ b/youtube_dl/extractor/mixcloud.py
@@ -1,6 +1,7 @@
 from __future__ import unicode_literals
 
 import re
+import itertools
 
 from .common import InfoExtractor
 from ..compat import (
@@ -45,20 +46,19 @@ class MixcloudIE(InfoExtractor):
         },
     }]
 
-    def _get_url(self, track_id, template_url):
-        server_count = 30
-        for i in range(server_count):
-            url = template_url % i
+    def _get_url(self, track_id, template_url, server_number):
+        boundaries = (1, 30)
+        for nr in server_numbers(server_number, boundaries):
+            url = template_url % nr
             try:
                 # We only want to know if the request succeed
                 # don't download the whole file
                 self._request_webpage(
                     HEADRequest(url), track_id,
-                    'Checking URL %d/%d ...' % (i + 1, server_count + 1))
+                    'Checking URL %d/%d ...' % (nr, boundaries[-1]))
                 return url
             except ExtractorError:
                 pass
-
         return None
 
     def _real_extract(self, url):
@@ -72,12 +72,13 @@ class MixcloudIE(InfoExtractor):
         preview_url = self._search_regex(
             r'\s(?:data-preview-url|m-preview)="([^"]+)"', webpage, 'preview url')
         song_url = preview_url.replace('/previews/', '/c/originals/')
+        server_number = int(self._search_regex(r'stream(\d+)', song_url, 'server number'))
         template_url = re.sub(r'(stream\d*)', 'stream%d', song_url)
-        final_song_url = self._get_url(track_id, template_url)
+        final_song_url = self._get_url(track_id, template_url, server_number)
         if final_song_url is None:
             self.to_screen('Trying with m4a extension')
             template_url = template_url.replace('.mp3', '.m4a').replace('originals/', 'm4a/64/')
-            final_song_url = self._get_url(track_id, template_url)
+            final_song_url = self._get_url(track_id, template_url, server_number)
         if final_song_url is None:
             raise ExtractorError('Unable to extract track url')
 
@@ -114,3 +115,35 @@ class MixcloudIE(InfoExtractor):
             'view_count': view_count,
             'like_count': like_count,
         }
+
+
+def server_numbers(first, boundaries):
+    """ Server numbers to try in descending order of probable availability.
+    Starting from first (i.e. the number of the server hosting the preview file)
+    and going further and further up to the higher boundary and down to the
+    lower one in an alternating fashion. Namely:
+
+        server_numbers(2, (1, 5))
+
+        # Where the preview server is 2, min number is 1 and max is 5.
+        # Yields: 2, 3, 1, 4, 5
+
+    Why not random numbers or increasing sequences? Since from what I've seen,
+    full length files seem to be hosted on servers whose number is closer to
+    that of the preview; to be confirmed.
+    """
+    zip_longest = getattr(itertools, 'zip_longest', None)
+    if zip_longest is None:
+        # python 2.x
+        zip_longest = itertools.izip_longest
+
+    if len(boundaries) != 2:
+        raise ValueError("boundaries should be a two-element tuple")
+    min, max = boundaries
+    highs = range(first + 1, max + 1)
+    lows = range(first - 1, min - 1, -1)
+    rest = filter(
+        None, itertools.chain.from_iterable(zip_longest(highs, lows)))
+    yield first
+    for n in rest:
+        yield n

From 3073a6d5e9036b0b613f57bc08099862a2af87f8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 18 Mar 2015 23:08:18 +0600
Subject: [PATCH 0722/6219] [ultimedia] Add extractor

Sponsored by thankyoumotion.com
---
 youtube_dl/extractor/__init__.py  |  1 +
 youtube_dl/extractor/ultimedia.py | 91 +++++++++++++++++++++++++++++++
 2 files changed, 92 insertions(+)
 create mode 100644 youtube_dl/extractor/ultimedia.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 73c17aa84..867e7c935 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -534,6 +534,7 @@ from .twitch import (
     TwitchStreamIE,
 )
 from .ubu import UbuIE
+from .ultimedia import UltimediaIE
 from .udemy import (
     UdemyIE,
     UdemyCourseIE
diff --git a/youtube_dl/extractor/ultimedia.py b/youtube_dl/extractor/ultimedia.py
new file mode 100644
index 000000000..97e4445d4
--- /dev/null
+++ b/youtube_dl/extractor/ultimedia.py
@@ -0,0 +1,91 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    qualities,
+    unified_strdate,
+    clean_html,
+)
+
+
+class UltimediaIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?ultimedia\.com/default/index/video[^/]+/id/(?P<id>[\d+a-z]+)'
+    _TESTS = [{
+        # news
+        'url': 'https://www.ultimedia.com/default/index/videogeneric/id/s8uk0r',
+        'md5': '276a0e49de58c7e85d32b057837952a2',
+        'info_dict': {
+            'id': 's8uk0r',
+            'ext': 'mp4',
+            'title': 'Loi sur la fin de vie: le texte prévoit un renforcement des directives anticipées',
+            'description': 'md5:3e5c8fd65791487333dda5db8aed32af',
+            'thumbnail': 're:^https?://.*\.jpg',
+            'upload_date': '20150317',
+        },
+    }, {
+        # music
+        'url': 'https://www.ultimedia.com/default/index/videomusic/id/xvpfp8',
+        'md5': '2ea3513813cf230605c7e2ffe7eca61c',
+        'info_dict': {
+            'id': 'xvpfp8',
+            'ext': 'mp4',
+            'title': "Two - C'est la vie (Clip)",
+            'description': 'Two',
+            'thumbnail': 're:^https?://.*\.jpg',
+            'upload_date': '20150224',
+        },
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        deliver_url = self._search_regex(
+            r'<iframe[^>]+src="(https?://(?:www\.)?ultimedia\.com/deliver/[^"]+)"',
+            webpage, 'deliver URL')
+
+        deliver_page = self._download_webpage(
+            deliver_url, video_id, 'Downloading iframe page')
+
+        player = self._parse_json(
+            self._search_regex(
+                r"jwplayer\('player(?:_temp)?'\)\.setup\(({.+?})\)\.on", deliver_page, 'player'),
+            video_id)
+
+        quality = qualities(['flash', 'html5'])
+
+        formats = [{
+            'url': mode['config']['file'],
+            'format_id': mode.get('type'),
+            'quality': quality(mode.get('type')),
+        } for mode in player['modes']]
+        self._sort_formats(formats)
+
+        thumbnail = player.get('image')
+
+        title = clean_html((
+            self._html_search_regex(
+                r'(?s)<div\s+id="catArticle">.+?</div>(.+?)</h1>',
+                webpage, 'title', default=None)
+            or self._search_regex(
+                r"var\s+nameVideo\s*=\s*'([^']+)'",
+                deliver_page, 'title')))
+
+        description = clean_html(self._html_search_regex(
+            r'(?s)<span>Description</span>(.+?)</p>', webpage,
+            'description', fatal=False))
+
+        upload_date = unified_strdate(self._search_regex(
+            r'Ajouté le\s*<span>([^<]+)', webpage,
+            'upload date', fatal=False))
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'upload_date': upload_date,
+            'formats': formats,
+        }

From d1dc7e39918ddfc3402a8ffd669e6c84ac971803 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 18 Mar 2015 23:11:48 +0600
Subject: [PATCH 0723/6219] [ultimedia] Fix alphabetic order

---
 youtube_dl/extractor/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 867e7c935..7eb9b4fbb 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -534,11 +534,11 @@ from .twitch import (
     TwitchStreamIE,
 )
 from .ubu import UbuIE
-from .ultimedia import UltimediaIE
 from .udemy import (
     UdemyIE,
     UdemyCourseIE
 )
+from .ultimedia import UltimediaIE
 from .unistra import UnistraIE
 from .urort import UrortIE
 from .ustream import UstreamIE, UstreamChannelIE

From 73900846b16d33f769d35ac945a97a66cc17fd5c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 19 Mar 2015 00:53:26 +0600
Subject: [PATCH 0724/6219] [ultimedia] Capture and output unavailable video
 message

---
 youtube_dl/extractor/ultimedia.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/ultimedia.py b/youtube_dl/extractor/ultimedia.py
index 97e4445d4..0c1b08d7d 100644
--- a/youtube_dl/extractor/ultimedia.py
+++ b/youtube_dl/extractor/ultimedia.py
@@ -3,6 +3,7 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..utils import (
+    ExtractorError,
     qualities,
     unified_strdate,
     clean_html,
@@ -49,6 +50,10 @@ class UltimediaIE(InfoExtractor):
         deliver_page = self._download_webpage(
             deliver_url, video_id, 'Downloading iframe page')
 
+        if '>This video is currently not available' in deliver_page:
+            raise ExtractorError(
+                'Video %s is currently not available' % video_id, expected=True)
+
         player = self._parse_json(
             self._search_regex(
                 r"jwplayer\('player(?:_temp)?'\)\.setup\(({.+?})\)\.on", deliver_page, 'player'),

From b30ef07c6ccb982cff623c34e7c5cec5d8eb9bb9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 19 Mar 2015 01:06:39 +0600
Subject: [PATCH 0725/6219] [ultimedia] Handle youtube embeds

---
 youtube_dl/extractor/ultimedia.py | 20 ++++++++++++++------
 1 file changed, 14 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/ultimedia.py b/youtube_dl/extractor/ultimedia.py
index 0c1b08d7d..06554a1be 100644
--- a/youtube_dl/extractor/ultimedia.py
+++ b/youtube_dl/extractor/ultimedia.py
@@ -1,6 +1,8 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
@@ -60,12 +62,18 @@ class UltimediaIE(InfoExtractor):
             video_id)
 
         quality = qualities(['flash', 'html5'])
-
-        formats = [{
-            'url': mode['config']['file'],
-            'format_id': mode.get('type'),
-            'quality': quality(mode.get('type')),
-        } for mode in player['modes']]
+        formats = []
+        for mode in player['modes']:
+            video_url = mode.get('config', {}).get('file')
+            if not video_url:
+                continue
+            if re.match(r'https?://www\.youtube\.com/.+?', video_url):
+                return self.url_result(video_url, 'Youtube')
+            formats.append({
+                'url': video_url,
+                'format_id': mode.get('type'),
+                'quality': quality(mode.get('type')),
+            })
         self._sort_formats(formats)
 
         thumbnail = player.get('image')

From cbc3cfcab41599f1b52e328878e19d15be1792d4 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 18 Mar 2015 22:02:39 +0100
Subject: [PATCH 0726/6219] release 2015.03.18

---
 docs/supportedsites.md | 4 ++++
 youtube_dl/version.py  | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index d6a1e67c6..72b365305 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -112,6 +112,7 @@
  - **Discovery**
  - **divxstage**: DivxStage
  - **Dotsub**
+ - **DouyuTV**
  - **DRBonanza**
  - **Dropbox**
  - **DrTuber**
@@ -342,6 +343,7 @@
  - **PornHubPlaylist**
  - **Pornotube**
  - **PornoXO**
+ - **PrimeShareTV**
  - **PromptFile**
  - **prosiebensat1**: ProSiebenSat.1 Digital
  - **Puls4**
@@ -367,6 +369,7 @@
  - **RTP**
  - **RTS**: RTS.ch
  - **rtve.es:alacarta**: RTVE a la carta
+ - **rtve.es:infantil**: RTVE infantil
  - **rtve.es:live**: RTVE.es live streams
  - **RUHD**
  - **rutube**: Rutube videos
@@ -487,6 +490,7 @@
  - **Ubu**
  - **udemy**
  - **udemy:course**
+ - **Ultimedia**
  - **Unistra**
  - **Urort**: NRK P3 Urørt
  - **ustream**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 7ed07c375..51b4260aa 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.03.15'
+__version__ = '2015.03.18'

From 0ae8bbac2d12a883b8eb7a941c65c7b87987d213 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 19 Mar 2015 21:17:04 +0600
Subject: [PATCH 0727/6219] [nytimes] Support embed URL

---
 youtube_dl/extractor/nytimes.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nytimes.py b/youtube_dl/extractor/nytimes.py
index 56e1cad3b..5b57e95b6 100644
--- a/youtube_dl/extractor/nytimes.py
+++ b/youtube_dl/extractor/nytimes.py
@@ -7,7 +7,7 @@ from ..utils import parse_iso8601
 
 
 class NYTimesIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?nytimes\.com/video/(?:[^/]+/)+(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:(?:www\.)?nytimes\.com/video/(?:[^/]+/)+|graphics8\.nytimes\.com/bcvideo/\d+(?:\.\d+)?/iframe/embed\.html\?videoId=)(?P<id>\d+)'
 
     _TEST = {
         'url': 'http://www.nytimes.com/video/opinion/100000002847155/verbatim-what-is-a-photocopier.html?playlistId=100000001150263',

From f3c0c667a6d8d1b58ca198b72a353887ea8c4f38 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 19 Mar 2015 21:23:52 +0600
Subject: [PATCH 0728/6219] [nytimes] Modernize

---
 youtube_dl/extractor/nytimes.py | 31 +++++++++++++++++--------------
 1 file changed, 17 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/nytimes.py b/youtube_dl/extractor/nytimes.py
index 5b57e95b6..d1cf8f4f3 100644
--- a/youtube_dl/extractor/nytimes.py
+++ b/youtube_dl/extractor/nytimes.py
@@ -1,9 +1,11 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
-from ..utils import parse_iso8601
+from ..utils import (
+    float_or_none,
+    int_or_none,
+    parse_iso8601,
+)
 
 
 class NYTimesIE(InfoExtractor):
@@ -25,15 +27,15 @@ class NYTimesIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         video_data = self._download_json(
-            'http://www.nytimes.com/svc/video/api/v2/video/%s' % video_id, video_id, 'Downloading video JSON')
+            'http://www.nytimes.com/svc/video/api/v2/video/%s' % video_id,
+            video_id, 'Downloading video JSON')
 
         title = video_data['headline']
-        description = video_data['summary']
-        duration = video_data['duration'] / 1000.0
+        description = video_data.get('summary')
+        duration = float_or_none(video_data.get('duration'), 1000)
 
         uploader = video_data['byline']
         timestamp = parse_iso8601(video_data['publication_date'][:-8])
@@ -49,11 +51,11 @@ class NYTimesIE(InfoExtractor):
         formats = [
             {
                 'url': video['url'],
-                'format_id': video['type'],
-                'vcodec': video['video_codec'],
-                'width': video['width'],
-                'height': video['height'],
-                'filesize': get_file_size(video['fileSize']),
+                'format_id': video.get('type'),
+                'vcodec': video.get('video_codec'),
+                'width': int_or_none(video.get('width')),
+                'height': int_or_none(video.get('height')),
+                'filesize': get_file_size(video.get('fileSize')),
             } for video in video_data['renditions']
         ]
         self._sort_formats(formats)
@@ -61,7 +63,8 @@ class NYTimesIE(InfoExtractor):
         thumbnails = [
             {
                 'url': 'http://www.nytimes.com/%s' % image['url'],
-                'resolution': '%dx%d' % (image['width'], image['height']),
+                'width': int_or_none(image.get('width')),
+                'height': int_or_none(image.get('height')),
             } for image in video_data['images']
         ]
 

From 3378d67a18d2d06574698168120bf8c7f7b9e172 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 19 Mar 2015 21:26:57 +0600
Subject: [PATCH 0729/6219] [generic] Add support for nytimes embeds (Closes
 #5234)

---
 youtube_dl/extractor/generic.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index dc5755d12..8716e4503 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1006,6 +1006,13 @@ class GenericIE(InfoExtractor):
         if mobj is not None:
             return self.url_result(mobj.group('url'))
 
+        # Look for NYTimes player
+        mobj = re.search(
+            r'<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//graphics8\.nytimes\.com/bcvideo/[^/]+/iframe/embed\.html.+?)\1>',
+            webpage)
+        if mobj is not None:
+            return self.url_result(mobj.group('url'))
+
         # Look for Ooyala videos
         mobj = (re.search(r'player\.ooyala\.com/[^"?]+\?[^"]*?(?:embedCode|ec)=(?P<ec>[^"&]+)', webpage) or
                 re.search(r'OO\.Player\.create\([\'"].*?[\'"],\s*[\'"](?P<ec>.{32})[\'"]', webpage) or

From eecc0685c976abc8fdcebe24e5c3a76d89268c54 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Thu, 19 Mar 2015 19:37:39 +0200
Subject: [PATCH 0730/6219] [videomega] Fix extraction and update test (Fixes
 #5235)

---
 youtube_dl/extractor/videomega.py | 45 ++++++++++---------------------
 1 file changed, 14 insertions(+), 31 deletions(-)

diff --git a/youtube_dl/extractor/videomega.py b/youtube_dl/extractor/videomega.py
index 273030316..eb309a7cd 100644
--- a/youtube_dl/extractor/videomega.py
+++ b/youtube_dl/extractor/videomega.py
@@ -4,28 +4,21 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse,
-    compat_urllib_request,
-)
-from ..utils import (
-    ExtractorError,
-    remove_start,
-)
+from ..compat import compat_urllib_request
 
 
 class VideoMegaIE(InfoExtractor):
     _VALID_URL = r'''(?x)https?://
         (?:www\.)?videomega\.tv/
-        (?:iframe\.php)?\?ref=(?P<id>[A-Za-z0-9]+)
+        (?:iframe\.php|cdn\.php)?\?ref=(?P<id>[A-Za-z0-9]+)
         '''
     _TEST = {
-        'url': 'http://videomega.tv/?ref=QR0HCUHI1661IHUCH0RQ',
+        'url': 'http://videomega.tv/?ref=4GNA688SU99US886ANG4',
         'md5': 'bf5c2f95c4c917536e80936af7bc51e1',
         'info_dict': {
-            'id': 'QR0HCUHI1661IHUCH0RQ',
+            'id': '4GNA688SU99US886ANG4',
             'ext': 'mp4',
-            'title': 'Big Buck Bunny',
+            'title': 'BigBuckBunny_320x180',
             'thumbnail': 're:^https?://.*\.jpg$',
         }
     }
@@ -33,34 +26,24 @@ class VideoMegaIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        iframe_url = 'http://videomega.tv/iframe.php?ref={0:}'.format(video_id)
+        iframe_url = 'http://videomega.tv/cdn.php?ref=%s' % video_id
         req = compat_urllib_request.Request(iframe_url)
         req.add_header('Referer', url)
         webpage = self._download_webpage(req, video_id)
 
-        try:
-            escaped_data = re.findall(r'unescape\("([^"]+)"\)', webpage)[-1]
-        except IndexError:
-            raise ExtractorError('Unable to extract escaped data')
-
-        playlist = compat_urllib_parse.unquote(escaped_data)
-
+        title = self._html_search_regex(
+            r'<title>(.*?)</title>', webpage, 'title')
+        title = re.sub(
+            r'(?:^[Vv]ideo[Mm]ega\.tv\s-\s?|\s?-\svideomega\.tv$)', '', title)
         thumbnail = self._search_regex(
-            r'image:\s*"([^"]+)"', playlist, 'thumbnail', fatal=False)
-        video_url = self._search_regex(r'file:\s*"([^"]+)"', playlist, 'URL')
-        title = remove_start(self._html_search_regex(
-            r'<title>(.*?)</title>', webpage, 'title'), 'VideoMega.tv - ')
-
-        formats = [{
-            'format_id': 'sd',
-            'url': video_url,
-        }]
-        self._sort_formats(formats)
+            r'<video[^>]+?poster="([^"]+)"', webpage, 'thumbnail', fatal=False)
+        video_url = self._search_regex(
+            r'<source[^>]+?src="([^"]+)"', webpage, 'video URL')
 
         return {
             'id': video_id,
             'title': title,
-            'formats': formats,
+            'url': video_url,
             'thumbnail': thumbnail,
             'http_headers': {
                 'Referer': iframe_url,

From ccf3960eeca3ca8b75da709c8c3019531101f1e7 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Thu, 19 Mar 2015 20:55:05 +0200
Subject: [PATCH 0731/6219] [nytimes] Improve _VALID_URL (Fixes #5238)

---
 youtube_dl/extractor/nytimes.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/nytimes.py b/youtube_dl/extractor/nytimes.py
index d1cf8f4f3..03f0a4de6 100644
--- a/youtube_dl/extractor/nytimes.py
+++ b/youtube_dl/extractor/nytimes.py
@@ -9,9 +9,9 @@ from ..utils import (
 
 
 class NYTimesIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:(?:www\.)?nytimes\.com/video/(?:[^/]+/)+|graphics8\.nytimes\.com/bcvideo/\d+(?:\.\d+)?/iframe/embed\.html\?videoId=)(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:(?:www\.)?nytimes\.com/video/(?:[^/]+/)+?|graphics8\.nytimes\.com/bcvideo/\d+(?:\.\d+)?/iframe/embed\.html\?videoId=)(?P<id>\d+)'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.nytimes.com/video/opinion/100000002847155/verbatim-what-is-a-photocopier.html?playlistId=100000001150263',
         'md5': '18a525a510f942ada2720db5f31644c0',
         'info_dict': {
@@ -24,7 +24,10 @@ class NYTimesIE(InfoExtractor):
             'uploader': 'Brett Weiner',
             'duration': 419,
         }
-    }
+    }, {
+        'url': 'http://www.nytimes.com/video/travel/100000003550828/36-hours-in-dubai.html',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From 2684871bc10fe2b34f58312b6f75d3237a786732 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Fri, 20 Mar 2015 01:50:36 +0200
Subject: [PATCH 0732/6219] [vine] Fix formats extraction (Closes #5239)

---
 youtube_dl/extractor/vine.py | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/vine.py b/youtube_dl/extractor/vine.py
index 0b58fe0fe..c3187cfeb 100644
--- a/youtube_dl/extractor/vine.py
+++ b/youtube_dl/extractor/vine.py
@@ -33,14 +33,13 @@ class VineIE(InfoExtractor):
             r'window\.POST_DATA = { %s: ({.+?}) }' % video_id, webpage, 'vine data'))
 
         formats = [{
-            'url': data['videoLowURL'],
-            'ext': 'mp4',
-            'format_id': 'low',
-        }, {
-            'url': data['videoUrl'],
-            'ext': 'mp4',
-            'format_id': 'standard',
-        }]
+            'format_id': '%(format)s-%(rate)s' % f,
+            'vcodec': f['format'],
+            'quality': f['rate'],
+            'url': f['videoUrl'],
+        } for f in data['videoUrls'] if f.get('rate')]
+
+        self._sort_formats(formats)
 
         return {
             'id': video_id,

From 01218f919b3bb5d85449806ea832b29251dc13ca Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 20 Mar 2015 14:59:38 +0100
Subject: [PATCH 0733/6219] [test/http] Add test for proxy support

---
 test/test_http.py | 49 ++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 48 insertions(+), 1 deletion(-)

diff --git a/test/test_http.py b/test/test_http.py
index bd4d46fef..f2e305b6f 100644
--- a/test/test_http.py
+++ b/test/test_http.py
@@ -8,7 +8,7 @@ import unittest
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 from youtube_dl import YoutubeDL
-from youtube_dl.compat import compat_http_server
+from youtube_dl.compat import compat_http_server, compat_urllib_request
 import ssl
 import threading
 
@@ -68,5 +68,52 @@ class TestHTTP(unittest.TestCase):
         r = ydl.extract_info('https://localhost:%d/video.html' % self.port)
         self.assertEqual(r['url'], 'https://localhost:%d/vid.mp4' % self.port)
 
+
+def _build_proxy_handler(name):
+    class HTTPTestRequestHandler(compat_http_server.BaseHTTPRequestHandler):
+        proxy_name = name
+
+        def log_message(self, format, *args):
+            pass
+
+        def do_GET(self):
+            self.send_response(200)
+            self.send_header('Content-Type', 'text/plain; charset=utf-8')
+            self.end_headers()
+            self.wfile.write('{self.proxy_name}: {self.path}'.format(self=self).encode('utf-8'))
+    return HTTPTestRequestHandler
+
+
+class TestProxy(unittest.TestCase):
+    def setUp(self):
+        self.proxy = compat_http_server.HTTPServer(
+            ('localhost', 0), _build_proxy_handler('normal'))
+        self.port = self.proxy.socket.getsockname()[1]
+        self.proxy_thread = threading.Thread(target=self.proxy.serve_forever)
+        self.proxy_thread.daemon = True
+        self.proxy_thread.start()
+
+        self.cn_proxy = compat_http_server.HTTPServer(
+            ('localhost', 0), _build_proxy_handler('cn'))
+        self.cn_port = self.cn_proxy.socket.getsockname()[1]
+        self.cn_proxy_thread = threading.Thread(target=self.cn_proxy.serve_forever)
+        self.cn_proxy_thread.daemon = True
+        self.cn_proxy_thread.start()
+
+    def test_proxy(self):
+        cn_proxy = 'localhost:{0}'.format(self.cn_port)
+        ydl = YoutubeDL({
+            'proxy': 'localhost:{0}'.format(self.port),
+            'cn_verification_proxy': cn_proxy,
+        })
+        url = 'http://foo.com/bar'
+        response = ydl.urlopen(url).read().decode('utf-8')
+        self.assertEqual(response, 'normal: {0}'.format(url))
+
+        req = compat_urllib_request.Request(url)
+        req.add_header('Ytdl-request-proxy', cn_proxy)
+        response = ydl.urlopen(req).read().decode('utf-8')
+        self.assertEqual(response, 'cn: {0}'.format(url))
+
 if __name__ == '__main__':
     unittest.main()

From f20bf146e217731016b32af598b177952fa1340a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 20 Mar 2015 15:14:25 +0100
Subject: [PATCH 0734/6219] [test/YoutubeDL] split in two classes

The name was misleading
---
 test/test_YoutubeDL.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index db8a47d2d..2597a1993 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -339,6 +339,8 @@ class TestFormatSelection(unittest.TestCase):
         downloaded = ydl.downloaded_info_dicts[0]
         self.assertEqual(downloaded['format_id'], 'G')
 
+
+class TestYoutubeDL(unittest.TestCase):
     def test_subtitles(self):
         def s_formats(lang, autocaption=False):
             return [{

From 1c9a1457fc154697b327a87d9afe674425740287 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 20 Mar 2015 20:53:14 +0600
Subject: [PATCH 0735/6219] [niconico] Add nm video test

---
 youtube_dl/extractor/niconico.py | 21 +++++++++++++++++++--
 1 file changed, 19 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/niconico.py b/youtube_dl/extractor/niconico.py
index 4d57ac703..ddec7b338 100644
--- a/youtube_dl/extractor/niconico.py
+++ b/youtube_dl/extractor/niconico.py
@@ -22,7 +22,7 @@ class NiconicoIE(InfoExtractor):
     IE_NAME = 'niconico'
     IE_DESC = 'ニコニコ動画'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.nicovideo.jp/watch/sm22312215',
         'md5': 'd1a75c0823e2f629128c43e1212760f9',
         'info_dict': {
@@ -39,7 +39,24 @@ class NiconicoIE(InfoExtractor):
             'username': 'ydl.niconico@gmail.com',
             'password': 'youtube-dl',
         },
-    }
+    }, {
+        'url': 'http://www.nicovideo.jp/watch/nm14296458',
+        'md5': '8db08e0158457cf852a31519fceea5bc',
+        'info_dict': {
+            'id': 'nm14296458',
+            'ext': 'swf',
+            'title': '【鏡音リン】Dance on media【オリジナル】take2!',
+            'description': 'md5:',
+            'uploader': 'りょうた',
+            'uploader_id': '18822557',
+            'upload_date': '20110429',
+            'duration': 209,
+        },
+        'params': {
+            'username': 'ydl.niconico@gmail.com',
+            'password': 'youtube-dl',
+        },
+    }]
 
     _VALID_URL = r'https?://(?:www\.|secure\.)?nicovideo\.jp/watch/(?P<id>(?:[a-z]{2})?[0-9]+)'
     _NETRC_MACHINE = 'niconico'

From 1887ecd4d629f44370846d372155512ec29e1f27 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 20 Mar 2015 21:45:09 +0600
Subject: [PATCH 0736/6219] [twitch] Fix login

---
 youtube_dl/extractor/twitch.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index cbdaf9c7a..aad2bf222 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -23,6 +23,7 @@ class TwitchBaseIE(InfoExtractor):
     _API_BASE = 'https://api.twitch.tv'
     _USHER_BASE = 'http://usher.twitch.tv'
     _LOGIN_URL = 'https://secure.twitch.tv/user/login'
+    _LOGIN_POST_URL = 'https://secure-login.twitch.tv/login'
     _NETRC_MACHINE = 'twitch'
 
     def _handle_error(self, response):
@@ -67,14 +68,14 @@ class TwitchBaseIE(InfoExtractor):
             'authenticity_token': authenticity_token,
             'redirect_on_login': '',
             'embed_form': 'false',
-            'mp_source_action': '',
+            'mp_source_action': 'login-button',
             'follow': '',
-            'user[login]': username,
-            'user[password]': password,
+            'login': username,
+            'password': password,
         }
 
         request = compat_urllib_request.Request(
-            self._LOGIN_URL, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
+            self._LOGIN_POST_URL, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
         request.add_header('Referer', self._LOGIN_URL)
         response = self._download_webpage(
             request, None, 'Logging in as %s' % username)

From 531980d89c37427c663152a525fd795dc0b040e4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 20 Mar 2015 17:05:28 +0100
Subject: [PATCH 0737/6219] [test/YoutubeDL] test match_filter

---
 test/test_YoutubeDL.py | 68 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 68 insertions(+)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 2597a1993..652519831 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -14,6 +14,7 @@ from test.helper import FakeYDL, assertRegexpMatches
 from youtube_dl import YoutubeDL
 from youtube_dl.extractor import YoutubeIE
 from youtube_dl.postprocessor.common import PostProcessor
+from youtube_dl.utils import match_filter_func
 
 TEST_URL = 'http://localhost/sample.mp4'
 
@@ -463,6 +464,73 @@ class TestYoutubeDL(unittest.TestCase):
         self.assertTrue(os.path.exists(audiofile), '%s doesn\'t exist' % audiofile)
         os.unlink(audiofile)
 
+    def test_match_filter(self):
+        class FilterYDL(YDL):
+            def __init__(self, *args, **kwargs):
+                super(FilterYDL, self).__init__(*args, **kwargs)
+                self.params['simulate'] = True
+
+            def process_info(self, info_dict):
+                super(YDL, self).process_info(info_dict)
+
+            def _match_entry(self, info_dict, incomplete):
+                res = super(FilterYDL, self)._match_entry(info_dict, incomplete)
+                if res is None:
+                    self.downloaded_info_dicts.append(info_dict)
+                return res
+
+        first = {
+            'id': '1',
+            'url': TEST_URL,
+            'title': 'one',
+            'extractor': 'TEST',
+            'duration': 30,
+            'filesize': 10 * 1024,
+        }
+        second = {
+            'id': '2',
+            'url': TEST_URL,
+            'title': 'two',
+            'extractor': 'TEST',
+            'duration': 10,
+            'description': 'foo',
+            'filesize': 5 * 1024,
+        }
+        videos = [first, second]
+
+        def get_videos(filter_=None):
+            ydl = FilterYDL({'match_filter': filter_})
+            for v in videos:
+                ydl.process_ie_result(v, download=True)
+            return [v['id'] for v in ydl.downloaded_info_dicts]
+
+        res = get_videos()
+        self.assertEqual(res, ['1', '2'])
+
+        def f(v):
+            if v['id'] == '1':
+                return None
+            else:
+                return 'Video id is not 1'
+        res = get_videos(f)
+        self.assertEqual(res, ['1'])
+
+        f = match_filter_func('duration < 30')
+        res = get_videos(f)
+        self.assertEqual(res, ['2'])
+
+        f = match_filter_func('description = foo')
+        res = get_videos(f)
+        self.assertEqual(res, ['2'])
+
+        f = match_filter_func('description =? foo')
+        res = get_videos(f)
+        self.assertEqual(res, ['1', '2'])
+
+        f = match_filter_func('filesize > 5KiB')
+        res = get_videos(f)
+        self.assertEqual(res, ['1'])
+
 
 if __name__ == '__main__':
     unittest.main()

From a7d9ded45dac709fbb9140ab3525a10a2a6d3c85 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 21 Mar 2015 12:07:23 +0100
Subject: [PATCH 0738/6219] [test] Add tests for aes

---
 devscripts/generate_aes_testdata.py | 36 ++++++++++++++++++++++
 test/test_aes.py                    | 47 +++++++++++++++++++++++++++++
 2 files changed, 83 insertions(+)
 create mode 100644 devscripts/generate_aes_testdata.py
 create mode 100644 test/test_aes.py

diff --git a/devscripts/generate_aes_testdata.py b/devscripts/generate_aes_testdata.py
new file mode 100644
index 000000000..ff66449eb
--- /dev/null
+++ b/devscripts/generate_aes_testdata.py
@@ -0,0 +1,36 @@
+from __future__ import unicode_literals
+
+import codecs
+import subprocess
+
+import os
+import sys
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from youtube_dl.utils import intlist_to_bytes
+from youtube_dl.aes import aes_encrypt, key_expansion
+
+secret_msg = b'Secret message goes here'
+
+
+def hex_str(int_list):
+    return codecs.encode(intlist_to_bytes(int_list), 'hex')
+
+
+def openssl_encode(algo, key, iv):
+    cmd = ['openssl', 'enc', '-e', '-' + algo, '-K', hex_str(key), '-iv', hex_str(iv)]
+    prog = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+    out, _ = prog.communicate(secret_msg)
+    return out
+
+iv = key = [0x20, 0x15] + 14 * [0]
+
+r = openssl_encode('aes-128-cbc', key, iv)
+print('aes_cbc_decrypt')
+print(repr(r))
+
+password = key
+new_key = aes_encrypt(password, key_expansion(password))
+r = openssl_encode('aes-128-ctr', new_key, iv)
+print('aes_decrypt_text')
+print(repr(r))
diff --git a/test/test_aes.py b/test/test_aes.py
new file mode 100644
index 000000000..111b902e1
--- /dev/null
+++ b/test/test_aes.py
@@ -0,0 +1,47 @@
+#!/usr/bin/env python
+
+from __future__ import unicode_literals
+
+# Allow direct execution
+import os
+import sys
+import unittest
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from youtube_dl.aes import aes_decrypt, aes_encrypt, aes_cbc_decrypt, aes_decrypt_text
+from youtube_dl.utils import bytes_to_intlist, intlist_to_bytes
+import base64
+
+# the encrypted data can be generate with 'devscripts/generate_aes_testdata.py'
+
+
+class TestAES(unittest.TestCase):
+    def setUp(self):
+        self.key = self.iv = [0x20, 0x15] + 14 * [0]
+        self.secret_msg = b'Secret message goes here'
+
+    def test_encrypt(self):
+        msg = b'message'
+        key = list(range(16))
+        encrypted = aes_encrypt(bytes_to_intlist(msg), key)
+        decrypted = intlist_to_bytes(aes_decrypt(encrypted, key))
+        self.assertEqual(decrypted, msg)
+
+    def test_cbc_decrypt(self):
+        data = bytes_to_intlist(
+            b"\x97\x92+\xe5\x0b\xc3\x18\x91ky9m&\xb3\xb5@\xe6'\xc2\x96.\xc8u\x88\xab9-[\x9e|\xf1\xcd"
+        )
+        decrypted = intlist_to_bytes(aes_cbc_decrypt(data, self.key, self.iv))
+        self.assertEqual(decrypted.rstrip(b'\x08'), self.secret_msg)
+
+    def test_decrypt_text(self):
+        password = intlist_to_bytes(self.key).decode('utf-8')
+        encrypted = base64.b64encode(
+            intlist_to_bytes(self.iv[:8]) +
+            b'\x17\x15\x93\xab\x8d\x80V\xcdV\xe0\t\xcdo\xc2\xa5\xd8ksM\r\xe27N\xae'
+        )
+        decrypted = (aes_decrypt_text(encrypted, password, 16))
+        self.assertEqual(decrypted, self.secret_msg)
+
+if __name__ == '__main__':
+    unittest.main()

From 85698c508651cc047b0ba2181580d99ed057ef0d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 21 Mar 2015 12:18:33 +0100
Subject: [PATCH 0739/6219] [crunchyroll] Remove unused class

---
 youtube_dl/extractor/crunchyroll.py | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index e64b88fbc..6ded723c9 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -23,7 +23,6 @@ from ..utils import (
 )
 from ..aes import (
     aes_cbc_decrypt,
-    inc,
 )
 
 
@@ -102,13 +101,6 @@ class CrunchyrollIE(InfoExtractor):
 
         key = obfuscate_key(id)
 
-        class Counter:
-            __value = iv
-
-            def next_value(self):
-                temp = self.__value
-                self.__value = inc(self.__value)
-                return temp
         decrypted_data = intlist_to_bytes(aes_cbc_decrypt(data, key, iv))
         return zlib.decompress(decrypted_data)
 

From 179d6678b1ce5ecdf22e2ae254f451183d836e75 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 21 Mar 2015 12:34:44 +0100
Subject: [PATCH 0740/6219] Remove the 'stitle' field

A warning has been printed for more than 2 years (since 97cd3afc7525394c46398f1526d412d081c02085)
---
 youtube_dl/YoutubeDL.py | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 5a83bc956..b5ef5e009 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -328,9 +328,6 @@ class YoutubeDL(object):
                 'Parameter outtmpl is bytes, but should be a unicode string. '
                 'Put  from __future__ import unicode_literals  at the top of your code file or consider switching to Python 3.x.')
 
-        if '%(stitle)s' in self.params.get('outtmpl', ''):
-            self.report_warning('%(stitle)s is deprecated. Use the %(title)s and the --restrict-filenames flag(which also secures %(uploader)s et al) instead.')
-
         self._setup_opener()
 
         if auto_init:
@@ -1218,9 +1215,6 @@ class YoutubeDL(object):
         if len(info_dict['title']) > 200:
             info_dict['title'] = info_dict['title'][:197] + '...'
 
-        # Keep for backwards compatibility
-        info_dict['stitle'] = info_dict['title']
-
         if 'format' not in info_dict:
             info_dict['format'] = info_dict['ext']
 

From 4e6a2286899ab156ad342e3e9003b0c3b239e0ae Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Mar 2015 18:20:49 +0600
Subject: [PATCH 0741/6219] [nrk] Adapt to new URL format

---
 youtube_dl/extractor/nrk.py | 28 +++++++++++++---------------
 1 file changed, 13 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index bff36f9d3..d17b7ed49 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -14,22 +14,22 @@ from ..utils import (
 
 
 class NRKIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?nrk\.no/(?:video|lyd)/[^/]+/(?P<id>[\dA-F]{16})'
+    _VALID_URL = r'(?:nrk:|http://(?:www\.)?nrk\.no/video/PS\*)(?P<id>\d+)'
 
     _TESTS = [
         {
-            'url': 'http://www.nrk.no/video/dompap_og_andre_fugler_i_piip_show/D0FA54B5C8B6CE59/emne/piipshow/',
-            'md5': 'a6eac35052f3b242bb6bb7f43aed5886',
+            'url': 'http://www.nrk.no/video/PS*150533',
+            'md5': 'bccd850baebefe23b56d708a113229c2',
             'info_dict': {
                 'id': '150533',
                 'ext': 'flv',
                 'title': 'Dompap og andre fugler i Piip-Show',
-                'description': 'md5:d9261ba34c43b61c812cb6b0269a5c8f'
+                'description': 'md5:d9261ba34c43b61c812cb6b0269a5c8f',
             }
         },
         {
-            'url': 'http://www.nrk.no/lyd/lyd_av_oppleser_for_blinde/AEFDDD5473BA0198/',
-            'md5': '3471f2a51718195164e88f46bf427668',
+            'url': 'http://www.nrk.no/video/PS*154915',
+            'md5': '0b1493ba1aae7d9579a5ad5531bc395a',
             'info_dict': {
                 'id': '154915',
                 'ext': 'flv',
@@ -40,20 +40,18 @@ class NRKIE(InfoExtractor):
     ]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
-        page = self._download_webpage(url, video_id)
-
-        video_id = self._html_search_regex(r'<div class="nrk-video" data-nrk-id="(\d+)">', page, 'video id')
+        video_id = self._match_id(url)
 
         data = self._download_json(
-            'http://v7.psapi.nrk.no/mediaelement/%s' % video_id, video_id, 'Downloading media JSON')
+            'http://v8.psapi.nrk.no/mediaelement/%s' % video_id,
+            video_id, 'Downloading media JSON')
 
         if data['usageRights']['isGeoBlocked']:
-            raise ExtractorError('NRK har ikke rettig-heter til å vise dette programmet utenfor Norge', expected=True)
+            raise ExtractorError(
+                'NRK har ikke rettig-heter til å vise dette programmet utenfor Norge',
+                expected=True)
 
-        video_url = data['mediaUrl'] + '?hdcore=3.1.1&plugin=aasp-3.1.1.69.124'
+        video_url = data['mediaUrl'] + '?hdcore=3.5.0&plugin=aasp-3.5.0.151.81'
 
         images = data.get('images')
         if images:

From 393d9fc6d28704ef7b60acb1dfc785493ecbe0eb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Mar 2015 18:21:19 +0600
Subject: [PATCH 0742/6219] [nrk] Extract duration

---
 youtube_dl/extractor/nrk.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index d17b7ed49..117a78aa1 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -25,6 +25,7 @@ class NRKIE(InfoExtractor):
                 'ext': 'flv',
                 'title': 'Dompap og andre fugler i Piip-Show',
                 'description': 'md5:d9261ba34c43b61c812cb6b0269a5c8f',
+                'duration': 263,
             }
         },
         {
@@ -35,6 +36,7 @@ class NRKIE(InfoExtractor):
                 'ext': 'flv',
                 'title': 'Slik høres internett ut når du er blind',
                 'description': 'md5:a621f5cc1bd75c8d5104cb048c6b8568',
+                'duration': 20,
             }
         },
     ]
@@ -53,6 +55,8 @@ class NRKIE(InfoExtractor):
 
         video_url = data['mediaUrl'] + '?hdcore=3.5.0&plugin=aasp-3.5.0.151.81'
 
+        duration = parse_duration(data.get('duration'))
+
         images = data.get('images')
         if images:
             thumbnails = images['webImages']
@@ -67,6 +71,7 @@ class NRKIE(InfoExtractor):
             'ext': 'flv',
             'title': data['title'],
             'description': data['description'],
+            'duration': duration,
             'thumbnail': thumbnail,
         }
 

From faa1b5c2920f1d50f01b83e96b5b0f136edc74f7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Mar 2015 18:22:08 +0600
Subject: [PATCH 0743/6219] [nrk:playlist] Add extractor (Closes #5245)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/nrk.py      | 31 +++++++++++++++++++++++++++++++
 2 files changed, 32 insertions(+)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 7eb9b4fbb..e25e4a582 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -346,6 +346,7 @@ from .npo import (
 )
 from .nrk import (
     NRKIE,
+    NRKPlaylistIE,
     NRKTVIE,
 )
 from .ntvde import NTVDeIE
diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index 117a78aa1..c2a8202dd 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -76,6 +76,37 @@ class NRKIE(InfoExtractor):
         }
 
 
+class NRKPlaylistIE(InfoExtractor):
+    _VALID_URL = r'http://(?:www\.)?nrk\.no/(?:[^/]+/)*(?P<id>[^/]+)'
+
+    _TEST = {
+        'url': 'http://www.nrk.no/troms/gjenopplev-den-historiske-solformorkelsen-1.12270763',
+        'info_dict': {
+            'id': 'gjenopplev-den-historiske-solformorkelsen-1.12270763',
+            'title': 'Gjenopplev den historiske solformørkelsen',
+            'description': 'md5:c2df8ea3bac5654a26fc2834a542feed',
+        },
+        'playlist_mincount': 2,
+    }
+
+    def _real_extract(self, url):
+        playlist_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, playlist_id)
+
+        entries = [
+            self.url_result('nrk:%s' % video_id, 'NRK')
+            for video_id in re.findall(
+                r'class="[^"]*\brich\b[^"]*"[^>]+data-video-id="(\d+)"', webpage)
+        ]
+
+        playlist_title = self._og_search_title(webpage)
+        playlist_description = self._og_search_description(webpage)
+
+        return self.playlist_result(
+            entries, playlist_id, playlist_title, playlist_description)
+
+
 class NRKTVIE(InfoExtractor):
     _VALID_URL = r'(?P<baseurl>http://tv\.nrk(?:super)?\.no/)(?:serie/[^/]+|program)/(?P<id>[a-zA-Z]{4}\d{8})(?:/\d{2}-\d{2}-\d{4})?(?:#del=(?P<part_id>\d+))?'
 

From c9450c7ab19bbd24a4ff8643370cb4f235b0b380 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 21 Mar 2015 14:00:37 +0100
Subject: [PATCH 0744/6219] [nrk:playlist] Restrict _VALID_URL

It would also match /videos/PS... urls
---
 youtube_dl/extractor/nrk.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index c2a8202dd..1355ecea2 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -77,7 +77,7 @@ class NRKIE(InfoExtractor):
 
 
 class NRKPlaylistIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?nrk\.no/(?:[^/]+/)*(?P<id>[^/]+)'
+    _VALID_URL = r'http://(?:www\.)?nrk\.no/(?!video)[^/]+/(?P<id>[^/]+)'
 
     _TEST = {
         'url': 'http://www.nrk.no/troms/gjenopplev-den-historiske-solformorkelsen-1.12270763',

From 5379a2d40db3674fead7c4239afdb65ff7b389c0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 21 Mar 2015 14:12:43 +0100
Subject: [PATCH 0745/6219] [test/utils] Test xpath_text

---
 test/test_utils.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/test/test_utils.py b/test/test_utils.py
index 3431ad24e..a8ab87685 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -24,6 +24,7 @@ from youtube_dl.utils import (
     encodeFilename,
     escape_rfc3986,
     escape_url,
+    ExtractorError,
     find_xpath_attr,
     fix_xml_ampersands,
     InAdvancePagedList,
@@ -54,6 +55,7 @@ from youtube_dl.utils import (
     urlencode_postdata,
     version_tuple,
     xpath_with_ns,
+    xpath_text,
     render_table,
     match_str,
 )
@@ -250,6 +252,17 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(find('media:song/media:author').text, 'The Author')
         self.assertEqual(find('media:song/url').text, 'http://server.com/download.mp3')
 
+    def test_xpath_text(self):
+        testxml = '''<root>
+            <div>
+                <p>Foo</p>
+            </div>
+        </root>'''
+        doc = xml.etree.ElementTree.fromstring(testxml)
+        self.assertEqual(xpath_text(doc, 'div/p'), 'Foo')
+        self.assertTrue(xpath_text(doc, 'div/bar') is None)
+        self.assertRaises(ExtractorError, xpath_text, doc, 'div/bar', fatal=True)
+
     def test_smuggle_url(self):
         data = {"ö": "ö", "abc": [3]}
         url = 'https://foo.bar/baz?x=y#a'

From a09141548aa31db7c7d9457b10f5c84e6e32beba Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Mar 2015 20:42:48 +0600
Subject: [PATCH 0746/6219] [nrk:playlist] Relax video id regex and improve
 _VALID_URL

---
 youtube_dl/extractor/nrk.py | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index 1355ecea2..e91d3a248 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -77,17 +77,25 @@ class NRKIE(InfoExtractor):
 
 
 class NRKPlaylistIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?nrk\.no/(?!video)[^/]+/(?P<id>[^/]+)'
+    _VALID_URL = r'http://(?:www\.)?nrk\.no/(?!video)(?:[^/]+/)+(?P<id>[^/]+)'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.nrk.no/troms/gjenopplev-den-historiske-solformorkelsen-1.12270763',
         'info_dict': {
             'id': 'gjenopplev-den-historiske-solformorkelsen-1.12270763',
             'title': 'Gjenopplev den historiske solformørkelsen',
             'description': 'md5:c2df8ea3bac5654a26fc2834a542feed',
         },
-        'playlist_mincount': 2,
-    }
+        'playlist_count': 2,
+    }, {
+        'url': 'http://www.nrk.no/kultur/bok/rivertonprisen-til-karin-fossum-1.12266449',
+        'info_dict': {
+            'id': 'rivertonprisen-til-karin-fossum-1.12266449',
+            'title': 'Rivertonprisen til Karin Fossum',
+            'description': 'Første kvinne på 15 år til å vinne krimlitteraturprisen.',
+        },
+        'playlist_count': 5,
+    }]
 
     def _real_extract(self, url):
         playlist_id = self._match_id(url)
@@ -97,7 +105,8 @@ class NRKPlaylistIE(InfoExtractor):
         entries = [
             self.url_result('nrk:%s' % video_id, 'NRK')
             for video_id in re.findall(
-                r'class="[^"]*\brich\b[^"]*"[^>]+data-video-id="(\d+)"', webpage)
+                r'class="[^"]*\brich\b[^"]*"[^>]+data-video-id="([^"]+)"',
+                webpage)
         ]
 
         playlist_title = self._og_search_title(webpage)

From 49aeedb8cb4dcf317c970a58c590d42e37904720 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Mar 2015 08:11:10 +0600
Subject: [PATCH 0747/6219] [libsyn] Improve and simplify

---
 youtube_dl/extractor/libsyn.py | 81 +++++++++++++++++++---------------
 1 file changed, 45 insertions(+), 36 deletions(-)

diff --git a/youtube_dl/extractor/libsyn.py b/youtube_dl/extractor/libsyn.py
index 88379f276..6bf741db8 100644
--- a/youtube_dl/extractor/libsyn.py
+++ b/youtube_dl/extractor/libsyn.py
@@ -1,50 +1,59 @@
-# encoding: utf-8
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
 from .common import InfoExtractor
-from ..utils import (
-    unified_strdate,
-)
+from ..utils import unified_strdate
+
 
 class LibsynIE(InfoExtractor):
-    _VALID_URL = r'(?:https?:)?//html5-player\.libsyn\.com/embed/episode/id/(?P<id>[0-9]+)(?:/.*)?'
-    _TESTS = [{
-        'url': "http://html5-player.libsyn.com/embed/episode/id/3377616/",
+    _VALID_URL = r'https?://html5-player\.libsyn\.com/embed/episode/id/(?P<id>[0-9]+)'
+
+    _TEST = {
+        'url': 'http://html5-player.libsyn.com/embed/episode/id/3377616/',
+        'md5': '443360ee1b58007bc3dcf09b41d093bb',
         'info_dict': {
-            'id': "3377616",
-            'ext': "mp3",
-            'title': "Episode 12: Bassem Youssef: Egypt's Jon Stewart",
-            'description': "<p>Bassem Youssef joins executive producer Steve Bodow and senior producer Sara Taksler for a conversation about how&nbsp;<em style=\"font-family: Tahoma, Geneva, sans-serif; font-size: 12.8000001907349px;\">The Daily Show</em>&nbsp;inspired Bassem to create&nbsp;<em style=\"font-family: Tahoma, Geneva, sans-serif; font-size: 12.8000001907349px;\">Al-Bernameg</em>, his massively popular (and now banned) Egyptian news satire program. Sara discusses her soon-to-be-released documentary,&nbsp;<em style=\"font-family: Tahoma, Geneva, sans-serif; font-size: 12.8000001907349px;\">Tickling Giants</em>, which chronicles how Bassem and his staff risked their safety every day to tell jokes.</p>",
+            'id': '3377616',
+            'ext': 'mp3',
+            'title': "The Daily Show Podcast without Jon Stewart - Episode 12: Bassem Youssef: Egypt's Jon Stewart",
+            'description': 'md5:601cb790edd05908957dae8aaa866465',
+            'upload_date': '20150220',
         },
-    }]
+    }
 
     def _real_extract(self, url):
-        if url.startswith('//'):
-            url = 'https:' + url
-        display_id = self._match_id(url)
-        webpage = self._download_webpage(url, display_id)
+        video_id = self._match_id(url)
 
-        podcast_title         = self._search_regex(r'<h2>(.*?)</h2>', webpage, 'show title')
-        podcast_episode_title = self._search_regex(r'<h3>(.*?)</h3>', webpage, 'episode title')
-        podcast_date          = unified_strdate(self._search_regex(r'<div class="release_date">Released: (.*?)</div>', webpage, 'release date'))
-        podcast_description   = self._search_regex(r'<div id="info_text_body">(.*?)</div>', webpage, 'description')
+        webpage = self._download_webpage(url, video_id)
 
-        url0 = self._search_regex(r'var mediaURLLibsyn = "(?P<url0>https?://.*)";', webpage, 'first media URL')
-        url1 = self._search_regex(r'var mediaURL = "(?P<url1>https?://.*)";', webpage, 'second media URL')
+        formats = [{
+            'url': media_url,
+        } for media_url in set(re.findall('var\s+mediaURL(?:Libsyn)?\s*=\s*"([^"]+)"', webpage))]
 
-        if url0 != url1:
-            formats = [{
-                'url': url0
-            }, {
-                'url': url1
-            }]
-        else:
-            formats = [{
-                'url': url0
-            }]
+        podcast_title = self._search_regex(
+            r'<h2>([^<]+)</h2>', webpage, 'title')
+        episode_title = self._search_regex(
+            r'<h3>([^<]+)</h3>', webpage, 'title', default=None)
+
+        title = '%s - %s' %(podcast_title, episode_title) if podcast_title else episode_title
+
+        description = self._html_search_regex(
+            r'<div id="info_text_body">(.+?)</div>', webpage,
+            'description', fatal=False)
+
+        thumbnail = self._search_regex(
+            r'<img[^>]+class="info-show-icon"[^>]+src="([^"]+)"',
+            webpage, 'thumbnail', fatal=False)
+
+        release_date = unified_strdate(self._search_regex(
+            r'<div class="release_date">Released: ([^<]+)<', webpage, 'release date', fatal=False))
 
         return {
-            'id': display_id,
-            'title': podcast_episode_title,
-            'description': podcast_description,
-            'upload_date': podcast_date,
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'upload_date': release_date,
             'formats': formats,
         }

From a1d0aa7b882484685a1a02185d0dafd51c545701 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Mar 2015 08:11:47 +0600
Subject: [PATCH 0748/6219] [libsyn] Fix extractor alphabetic order

---
 youtube_dl/extractor/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index a20492fc3..82b75a144 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -250,8 +250,8 @@ from .letv import (
     LetvTvIE,
     LetvPlaylistIE
 )
-from .lifenews import LifeNewsIE
 from .libsyn import LibsynIE
+from .lifenews import LifeNewsIE
 from .liveleak import LiveLeakIE
 from .livestream import (
     LivestreamIE,

From cefdf970ccd8017cd67e949004e5e4c770aacdb1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Mar 2015 08:18:13 +0600
Subject: [PATCH 0749/6219] [extractor/generic] Support Libsyn embeds

---
 youtube_dl/extractor/generic.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 8716e4503..84e8f14b2 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1013,6 +1013,12 @@ class GenericIE(InfoExtractor):
         if mobj is not None:
             return self.url_result(mobj.group('url'))
 
+        # Look for Libsyn player
+        mobj = re.search(
+            r'<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//html5-player\.libsyn\.com/embed/.+?)\1', webpage)
+        if mobj is not None:
+            return self.url_result(mobj.group('url'))
+
         # Look for Ooyala videos
         mobj = (re.search(r'player\.ooyala\.com/[^"?]+\?[^"]*?(?:embedCode|ec)=(?P<ec>[^"&]+)', webpage) or
                 re.search(r'OO\.Player\.create\([\'"].*?[\'"],\s*[\'"](?P<ec>.{32})[\'"]', webpage) or

From 2051acdeb2ed9a0edf3b6b70682699c37d19d851 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Mar 2015 08:20:27 +0600
Subject: [PATCH 0750/6219] [extractor/generic] Add test for Libsyn embed

---
 youtube_dl/extractor/generic.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 84e8f14b2..8a49b0b54 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -527,6 +527,17 @@ class GenericIE(InfoExtractor):
             },
             'add_ie': ['Viddler'],
         },
+        # Libsyn embed
+        {
+            'url': 'http://thedailyshow.cc.com/podcast/episodetwelve',
+            'info_dict': {
+                'id': '3377616',
+                'ext': 'mp3',
+                'title': "The Daily Show Podcast without Jon Stewart - Episode 12: Bassem Youssef: Egypt's Jon Stewart",
+                'description': 'md5:601cb790edd05908957dae8aaa866465',
+                'upload_date': '20150220',
+            },
+        },
         # jwplayer YouTube
         {
             'url': 'http://media.nationalarchives.gov.uk/index.php/webinar-using-discovery-national-archives-online-catalogue/',

From cf2e2eb1c0b626f2d5f210ffd14642aceb0358e6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Mar 2015 08:23:20 +0600
Subject: [PATCH 0751/6219] [comedycentral] Drop thedailyshow podcast extractor

Generic extractor is just fine for Libsyn embeds
---
 youtube_dl/extractor/__init__.py      |  2 +-
 youtube_dl/extractor/comedycentral.py | 24 ------------------------
 2 files changed, 1 insertion(+), 25 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 82b75a144..d73826d44 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -84,7 +84,7 @@ from .cnn import (
 )
 from .collegehumor import CollegeHumorIE
 from .collegerama import CollegeRamaIE
-from .comedycentral import ComedyCentralIE, ComedyCentralShowsIE, TheDailyShowPodcastIE
+from .comedycentral import ComedyCentralIE, ComedyCentralShowsIE
 from .comcarcoff import ComCarCoffIE
 from .commonmistakes import CommonMistakesIE, UnicodeBOMIE
 from .condenast import CondeNastIE
diff --git a/youtube_dl/extractor/comedycentral.py b/youtube_dl/extractor/comedycentral.py
index bd3817b56..648a6f990 100644
--- a/youtube_dl/extractor/comedycentral.py
+++ b/youtube_dl/extractor/comedycentral.py
@@ -273,27 +273,3 @@ class ComedyCentralShowsIE(MTVServicesInfoExtractor):
             'title': show_name + ' ' + title,
             'description': description,
         }
-
-class TheDailyShowPodcastIE(InfoExtractor):
-    _VALID_URL = r'(?P<scheme>https?:)?//thedailyshow\.cc\.com/podcast/(?P<id>[a-z\-]+)'
-    _TESTS = [{
-        "url": "http://thedailyshow.cc.com/podcast/episodetwelve",
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        display_id = self._match_id(url)
-        webpage = self._download_webpage(url, display_id)
-
-        player_url = self._search_regex(r'<iframe(?:\s+[^>]+)?\s*src="((?:https?:)?//html5-player\.libsyn\.com/embed/episode/id/[0-9]+)', webpage, 'player URL')
-        if player_url.startswith('//'):
-            mobj = re.match(self._VALID_URL, url)
-            scheme = mobj.group('scheme')
-            if not scheme:
-                scheme = 'https:'
-            player_url = scheme + player_url
-
-        return {
-            '_type': 'url_transparent',
-            'url': player_url,
-        }

From 1a4123de04d0168ef4a14e6064148eb248d65dc4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Mar 2015 08:23:38 +0600
Subject: [PATCH 0752/6219] [comedycentral] Remove unused import

---
 youtube_dl/extractor/comedycentral.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/comedycentral.py b/youtube_dl/extractor/comedycentral.py
index 648a6f990..e5edcc84b 100644
--- a/youtube_dl/extractor/comedycentral.py
+++ b/youtube_dl/extractor/comedycentral.py
@@ -2,7 +2,6 @@ from __future__ import unicode_literals
 
 import re
 
-from .common import InfoExtractor
 from .mtv import MTVServicesInfoExtractor
 from ..compat import (
     compat_str,

From 336d19044c84128cc3bdcb62882372243dfa32c8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 22 Mar 2015 11:03:52 +0100
Subject: [PATCH 0753/6219] [lybsyn] pep8: add space around operator

---
 youtube_dl/extractor/libsyn.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/libsyn.py b/youtube_dl/extractor/libsyn.py
index 6bf741db8..9ab1416f5 100644
--- a/youtube_dl/extractor/libsyn.py
+++ b/youtube_dl/extractor/libsyn.py
@@ -36,7 +36,7 @@ class LibsynIE(InfoExtractor):
         episode_title = self._search_regex(
             r'<h3>([^<]+)</h3>', webpage, 'title', default=None)
 
-        title = '%s - %s' %(podcast_title, episode_title) if podcast_title else episode_title
+        title = '%s - %s' % (podcast_title, episode_title) if podcast_title else episode_title
 
         description = self._html_search_regex(
             r'<div id="info_text_body">(.+?)</div>', webpage,

From f9544f6e8fe4697a4a48364b6d7a7dc7d93cabf5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 22 Mar 2015 12:09:58 +0100
Subject: [PATCH 0754/6219] [test/aes] Test aes_decrypt_text with 256 bit

---
 devscripts/generate_aes_testdata.py | 8 +++++++-
 test/test_aes.py                    | 8 ++++++++
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/devscripts/generate_aes_testdata.py b/devscripts/generate_aes_testdata.py
index ff66449eb..2e389fc8e 100644
--- a/devscripts/generate_aes_testdata.py
+++ b/devscripts/generate_aes_testdata.py
@@ -32,5 +32,11 @@ print(repr(r))
 password = key
 new_key = aes_encrypt(password, key_expansion(password))
 r = openssl_encode('aes-128-ctr', new_key, iv)
-print('aes_decrypt_text')
+print('aes_decrypt_text 16')
+print(repr(r))
+
+password = key + 16 * [0]
+new_key = aes_encrypt(password, key_expansion(password)) * (32 // 16)
+r = openssl_encode('aes-256-ctr', new_key, iv)
+print('aes_decrypt_text 32')
 print(repr(r))
diff --git a/test/test_aes.py b/test/test_aes.py
index 111b902e1..4dc7de7b5 100644
--- a/test/test_aes.py
+++ b/test/test_aes.py
@@ -43,5 +43,13 @@ class TestAES(unittest.TestCase):
         decrypted = (aes_decrypt_text(encrypted, password, 16))
         self.assertEqual(decrypted, self.secret_msg)
 
+        password = intlist_to_bytes(self.key).decode('utf-8')
+        encrypted = base64.b64encode(
+            intlist_to_bytes(self.iv[:8]) +
+            b'\x0b\xe6\xa4\xd9z\x0e\xb8\xb9\xd0\xd4i_\x85\x1d\x99\x98_\xe5\x80\xe7.\xbf\xa5\x83'
+        )
+        decrypted = (aes_decrypt_text(encrypted, password, 32))
+        self.assertEqual(decrypted, self.secret_msg)
+
 if __name__ == '__main__':
     unittest.main()

From 93f787070ff2de48023c5bfeaebd153b3e29137a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 22 Mar 2015 15:39:35 +0100
Subject: [PATCH 0755/6219] [twitch] Only match digits for the video id

Urls can also contain contain a query (for example a timestamp '?t=foo')
---
 youtube_dl/extractor/twitch.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index aad2bf222..94bd6345d 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -149,7 +149,7 @@ class TwitchItemBaseIE(TwitchBaseIE):
 
 class TwitchVideoIE(TwitchItemBaseIE):
     IE_NAME = 'twitch:video'
-    _VALID_URL = r'%s/[^/]+/b/(?P<id>[^/]+)' % TwitchBaseIE._VALID_URL_BASE
+    _VALID_URL = r'%s/[^/]+/b/(?P<id>\d+)' % TwitchBaseIE._VALID_URL_BASE
     _ITEM_TYPE = 'video'
     _ITEM_SHORTCUT = 'a'
 
@@ -165,7 +165,7 @@ class TwitchVideoIE(TwitchItemBaseIE):
 
 class TwitchChapterIE(TwitchItemBaseIE):
     IE_NAME = 'twitch:chapter'
-    _VALID_URL = r'%s/[^/]+/c/(?P<id>[^/]+)' % TwitchBaseIE._VALID_URL_BASE
+    _VALID_URL = r'%s/[^/]+/c/(?P<id>\d+)' % TwitchBaseIE._VALID_URL_BASE
     _ITEM_TYPE = 'chapter'
     _ITEM_SHORTCUT = 'c'
 
@@ -184,7 +184,7 @@ class TwitchChapterIE(TwitchItemBaseIE):
 
 class TwitchVodIE(TwitchItemBaseIE):
     IE_NAME = 'twitch:vod'
-    _VALID_URL = r'%s/[^/]+/v/(?P<id>[^/]+)' % TwitchBaseIE._VALID_URL_BASE
+    _VALID_URL = r'%s/[^/]+/v/(?P<id>\d+)' % TwitchBaseIE._VALID_URL_BASE
     _ITEM_TYPE = 'vod'
     _ITEM_SHORTCUT = 'v'
 

From 32d687f55e103963a2cb8d8f3f88bb31b9cb8fb6 Mon Sep 17 00:00:00 2001
From: zx8 <zx8@mac>
Date: Sun, 22 Mar 2015 18:03:40 +0000
Subject: [PATCH 0756/6219] [safari] Add safaribooksonline extractor

---
 youtube_dl/extractor/__init__.py |   4 +
 youtube_dl/extractor/safari.py   | 144 +++++++++++++++++++++++++++++++
 2 files changed, 148 insertions(+)
 create mode 100644 youtube_dl/extractor/safari.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index d73826d44..3a0c42ded 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -420,6 +420,10 @@ from .rutube import (
 )
 from .rutv import RUTVIE
 from .sandia import SandiaIE
+from .safari import (
+    SafariIE,
+    SafariCourseIE,
+)
 from .sapo import SapoIE
 from .savefrom import SaveFromIE
 from .sbs import SBSIE
diff --git a/youtube_dl/extractor/safari.py b/youtube_dl/extractor/safari.py
new file mode 100644
index 000000000..3e494b960
--- /dev/null
+++ b/youtube_dl/extractor/safari.py
@@ -0,0 +1,144 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+import re
+import json
+
+from .common import InfoExtractor
+from .brightcove import BrightcoveIE
+
+from ..compat import (
+    compat_urllib_parse,
+    compat_urllib_request,
+)
+from ..utils import (
+    ExtractorError,
+    smuggle_url,
+    std_headers,
+)
+
+
+class SafariBaseIE(InfoExtractor):
+    _LOGIN_URL = 'https://www.safaribooksonline.com/accounts/login/'
+    _SUCCESSFUL_LOGIN_REGEX = r'<a href="/accounts/logout/"[^>]+>Sign Out</a>'
+    _ACCOUNT_CREDENTIALS_HINT = ('Use --username and --password options to '
+                                 'supply credentials for safaribooksonline.com ')
+    _NETRC_MACHINE = 'safaribooksonline'
+
+    LOGGED_IN = False
+
+    def _real_initialize(self):
+        # We only need to log in once for courses or individual videos
+        if not SafariBaseIE.LOGGED_IN:
+            self._login()
+            SafariBaseIE.LOGGED_IN = True
+
+    def _login(self):
+        (username, password) = self._get_login_info()
+        if username is None:
+            raise ExtractorError(
+                self._ACCOUNT_CREDENTIALS_HINT,
+                expected=True)
+
+        headers = std_headers
+        if 'Referer' not in headers:
+            headers['Referer'] = self._LOGIN_URL
+
+        login_page = self._download_webpage(
+            self._LOGIN_URL, None,
+            'Downloading login form')
+
+        csrf = self._html_search_regex(
+            r"<input +type='hidden' +name='csrfmiddlewaretoken' +value='([^']+)' +/>",
+            login_page, 'csrf token')
+
+        login_form = {
+            'csrfmiddlewaretoken': csrf,
+            'email': username,
+            'password1': password,
+            'login': 'Sign In',
+            'next': '',
+        }
+
+        request = compat_urllib_request.Request(
+            self._LOGIN_URL, compat_urllib_parse.urlencode(login_form), headers=headers)
+        login_page = self._download_webpage(
+            request, None, 'Logging in as %s' % username)
+
+        if re.search(self._SUCCESSFUL_LOGIN_REGEX, login_page) is None:
+            raise ExtractorError('Login failed; make sure your credentials are correct and '
+                                 'try again.', expected=True)
+
+        self.to_screen('Login successful')
+
+
+class SafariIE(SafariBaseIE):
+    IE_NAME = 'safari'
+    IE_DESC = 'safaribooksonline.com online video'
+    _VALID_URL = (r'https?://(?:www\.)?safaribooksonline\.com/library/view/[^/]+/'
+                  '(?P<id>\d+)/(?P<part>part\d+)\.html')
+    _TEST = {
+        'url': ('https://www.safaribooksonline.com/library/view/'
+                'hadoop-fundamentals-livelessons/9780133392838/part00.html'),
+        'md5': '5b0c4cc1b3c1ba15dda7344085aa5592',
+        'info_dict': {
+            'id': '9780133392838',
+            'ext': 'mp4',
+            'title': 'Introduction',
+        }
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        part = mobj.group('part')
+
+        webpage = self._download_webpage(url, part)
+        bc_url = BrightcoveIE._extract_brightcove_url(webpage)
+        if not bc_url:
+            raise ExtractorError('Could not extract Brightcove URL from %s' % url, expected=True)
+
+        return {
+            '_type': 'url',
+            'url': smuggle_url(bc_url, {'Referer': url}),
+            'ie_key': 'Brightcove'
+        }
+
+
+class SafariCourseIE(SafariBaseIE):
+    IE_NAME = 'safari:course'
+    IE_DESC = 'safaribooksonline.com online courses'
+
+    _VALID_URL = (r'https?://(?:www\.)?safaribooksonline\.com/library/view/'
+                  '(?P<course_path>[^/]+)/(?P<id>\d+)/?$')
+
+    _API_BASE = 'https://www.safaribooksonline.com/api/v1/book'
+    _API_FORMAT = 'json'
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        course_path = mobj.group('course_path')
+        course_id = mobj.group('id')
+
+        webpage = self._download_webpage(
+            '%s/%s/?override_format=%s' % (self._API_BASE, course_id, self._API_FORMAT),
+            course_path, 'Downloading course JSON')
+
+        course_json = json.loads(webpage)
+
+        if 'chapters' not in course_json:
+            raise ExtractorError('No chapters found for course %s' % course_id, expected=True)
+
+        num_parts = len(course_json['chapters'])
+        parts = ['%02d' % part for part in range(num_parts)]
+
+        entries = [
+            self.url_result(
+                'https://www.safaribooksonline.com/library/view/%s/%s/part%s.html' % (course_path,
+                                                                                      course_id,
+                                                                                      part_id),
+                'Safari')
+            for part_id in parts]
+
+        course_title = course_json['title']
+
+        return self.playlist_result(entries, course_id, course_title)

From 575dad3c9842f333c4af27563a26bddaf0015fa2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Roman=20Le=20N=C3=A9grate?= <roman.lenegrate@gmail.com>
Date: Sun, 22 Mar 2015 20:25:44 +0100
Subject: [PATCH 0757/6219] [pornovoisines] Add extractor

---
 youtube_dl/extractor/__init__.py      |   1 +
 youtube_dl/extractor/pornovoisines.py | 101 ++++++++++++++++++++++++++
 2 files changed, 102 insertions(+)
 create mode 100644 youtube_dl/extractor/pornovoisines.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index d73826d44..17d075ec8 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -382,6 +382,7 @@ from .pornhub import (
     PornHubPlaylistIE,
 )
 from .pornotube import PornotubeIE
+from .pornovoisines import PornoVoisinesIE
 from .pornoxo import PornoXOIE
 from .primesharetv import PrimeShareTVIE
 from .promptfile import PromptFileIE
diff --git a/youtube_dl/extractor/pornovoisines.py b/youtube_dl/extractor/pornovoisines.py
new file mode 100644
index 000000000..efbb6a818
--- /dev/null
+++ b/youtube_dl/extractor/pornovoisines.py
@@ -0,0 +1,101 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+import datetime
+import random
+
+from ..compat import compat_urllib_parse
+from .common import InfoExtractor
+
+class PornoVoisinesIE(InfoExtractor):
+    _VALID_URL = r'^((?:http://)?(?:www\.)?pornovoisines.com)/showvideo/(\d+)/([^/]+)'
+
+    VIDEO_URL_TEMPLATE = 'http://stream%d.pornovoisines.com' \
+        '/static/media/video/transcoded/%s-640x360-1000-trscded.mp4'
+
+    SERVER_NUMBERS = (1, 2)
+
+    _TEST = {
+        'url': 'http://www.pornovoisines.com/showvideo/1285/recherche-appartement/',
+        'md5': '5ac670803bc12e9e7f9f662ce64cf1d1',
+        'info_dict': {
+            'id': '1285',
+            'display_id': 'recherche-appartement',
+            'ext': 'mp4',
+            'title': "Recherche appartement",
+            'upload_date': '20140925',
+            'view_count': int,
+            'duration': 120,
+            'categories': ["Débutante", "Scénario", "Sodomie"],
+            'description': 're:^Pour la .+ original...$',
+            'thumbnail': 're:^http://',
+            'uploader': "JMTV",
+            'average_rating': float,
+            'comment_count': int,
+            'age_limit': 18,
+        }
+    }
+
+    @classmethod
+    def build_video_url(cls, id):
+        server_nr = random.choice(cls.SERVER_NUMBERS)
+        return cls.VIDEO_URL_TEMPLATE % (server_nr, id)
+
+    @staticmethod
+    def parse_upload_date(str):
+        return datetime.datetime.strptime(str, "%d-%m-%Y").strftime("%Y%m%d")
+
+    @staticmethod
+    def parse_categories(str):
+        return map(lambda s: s.strip(), str.split(','))
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        url_prefix = mobj.group(1)
+        id = mobj.group(2)
+        display_id = mobj.group(3)
+
+        webpage = self._download_webpage(url, id)
+
+        title = self._html_search_regex(r'<h1>(.+?)</h1>', webpage, 'title',
+            flags=re.DOTALL)
+        url = self.build_video_url(id)
+        upload_date = self.parse_upload_date(
+            self._search_regex(r'Publié le (\d\d-\d\d-\d{4})', webpage,
+            'upload date'))
+        view_count = int(self._search_regex(r'(\d+) vues', webpage, 'view count'))
+        duration = int(self._search_regex('Durée (\d+)', webpage, 'duration'))
+        categories = self.parse_categories(self._html_search_regex(
+            r'<li class="categorie">(.+?)</li>', webpage, "categories",
+            flags=re.DOTALL))
+        description = self._html_search_regex(
+            r'<article id="descriptif">(.+?)</article>', webpage, "description",
+            flags=re.DOTALL)
+        thumbnail = url_prefix + self._html_search_regex(re.compile(
+            '<div id="mediaspace' + id + '">.*?<img src="(.+?)"', re.DOTALL),
+            webpage, "thumbnail")
+        uploader = re.sub(r' *\| *$', '',
+            self._html_search_regex(r'<li class="auteur">(.+?)</li>', webpage,
+            "uploader", flags=re.DOTALL))
+        average_rating = float(self._search_regex(r'Note : (\d+,\d+)',
+            webpage, "average rating").replace(',', '.'))
+        comment_count = int(self._search_regex(r'\((\d+)\)', webpage,
+            "comment count"))
+
+        return {
+            'id': id,
+            'display_id': display_id,
+            'url': url,
+            'title': title,
+            'upload_date': upload_date,
+            'view_count': view_count,
+            'duration': duration,
+            'categories': categories,
+            'description': description,
+            'thumbnail': thumbnail,
+            'uploader': uploader,
+            'average_rating': average_rating,
+            'comment_count': comment_count,
+            'age_limit': 18,
+        }

From c41a2ec4af9fa76b04b6d9f50d9a895d124ea14c Mon Sep 17 00:00:00 2001
From: tiktok <tiktok7@users.noreply.github.com>
Date: Mon, 23 Mar 2015 01:42:17 +0100
Subject: [PATCH 0758/6219] [MiomioTv] Add new extractor

---
 docs/supportedsites.md            |  1 +
 youtube_dl/extractor/__init__.py  |  1 +
 youtube_dl/extractor/miomio_tv.py | 70 +++++++++++++++++++++++++++++++
 3 files changed, 72 insertions(+)
 create mode 100644 youtube_dl/extractor/miomio_tv.py

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 062cb3d62..53d280677 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -244,6 +244,7 @@
  - **Mgoon**
  - **Minhateca**
  - **MinistryGrid**
+ - **Miomio.tv**
  - **mitele.es**
  - **mixcloud**
  - **MLB**
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index ffcc7d9ab..370154773 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -265,6 +265,7 @@ from .mdr import MDRIE
 from .metacafe import MetacafeIE
 from .metacritic import MetacriticIE
 from .mgoon import MgoonIE
+from .miomio_tv import MiomioTvIE
 from .minhateca import MinhatecaIE
 from .ministrygrid import MinistryGridIE
 from .mit import TechTVMITIE, MITIE, OCWMITIE
diff --git a/youtube_dl/extractor/miomio_tv.py b/youtube_dl/extractor/miomio_tv.py
new file mode 100644
index 000000000..355774f54
--- /dev/null
+++ b/youtube_dl/extractor/miomio_tv.py
@@ -0,0 +1,70 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class MiomioTvIE(InfoExtractor):
+    IE_NAME = 'miomio.tv'
+    _VALID_URL = r'https?://(?:www\.)?miomio\.tv/watch/cc(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'http://www.miomio.tv/watch/cc179734/',
+        'md5': '48de02137d0739c15b440a224ad364b9',
+        'info_dict': {
+            'id': '179734',
+            'title': u'\u624b\u7ed8\u52a8\u6f2b\u9b3c\u6ce3\u4f46\u4e01\u5168\u7a0b\u753b\u6cd5',
+            'ext': 'flv'
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        title = self._html_search_regex(r'<meta\s+name="description"\s+content="\s*([^"]*)\s*"', webpage, 'title')
+        ref_path = self._search_regex(r'src="(/mioplayer/.*?)"', webpage, 'ref_path')
+        referer = 'http://www.miomio.tv{}'.format(ref_path)
+        xml_config = self._search_regex(r'flashvars="type=sina&amp;(.*?)&amp;cid=', webpage, 'xml config')
+        self._request_webpage("http://www.miomio.tv/mioplayer/mioplayerconfigfiles/xml.php?id={}&r=cc{}".format(id, 945), video_id)
+        xml_url = 'http://www.miomio.tv/mioplayer/mioplayerconfigfiles/sina.php?{}'.format(xml_config)
+        vidconfig = self._download_xml(xml_url, video_id)
+
+        file_els = vidconfig.findall('.//durl')
+
+        entries = []
+
+        for file_el in file_els:
+            segment_id = file_el.find('order').text.strip()
+            segment_title = '_'.join([title, segment_id])
+            segment_duration = file_el.find('length').text.strip()
+            segment_url = file_el.find('url').text.strip()
+
+            entries.append({
+                'id': segment_id,
+                'title': segment_title,
+                'duration': segment_duration,
+                'url': segment_url
+            })
+
+        http_headers = {
+            'Referer': referer,
+            'Accept-Language': 'en,en-US;q=0.7,de;q=0.3',
+            'Accept-Encoding': 'gzip, deflate',
+            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
+        }
+
+        if len(entries) == 1:
+            return {
+                'id': video_id,
+                'title': title,
+                'url': entries[0]['url'],
+                'http_headers': http_headers
+            }
+
+        return {
+            '_type': 'multi_video',
+            'id': video_id,
+            'title': title,
+            'entries': entries,
+            'http_headers': http_headers
+        }

From 9f790b9901445d532564b3c5c90c0ecfaea4805d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 23 Mar 2015 21:23:57 +0600
Subject: [PATCH 0759/6219] [mlb] Improve _VALID_URL (Closes #5260)

---
 youtube_dl/extractor/mlb.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mlb.py b/youtube_dl/extractor/mlb.py
index 1a241aca7..e369551c2 100644
--- a/youtube_dl/extractor/mlb.py
+++ b/youtube_dl/extractor/mlb.py
@@ -10,7 +10,7 @@ from ..utils import (
 
 
 class MLBIE(InfoExtractor):
-    _VALID_URL = r'https?://m(?:lb)?\.mlb\.com/(?:(?:.*?/)?video/(?:topic/[\da-z_-]+/)?v|(?:shared/video/embed/embed\.html|[^/]+/video/play\.jsp)\?.*?\bcontent_id=)(?P<id>n?\d+)'
+    _VALID_URL = r'https?://m(?:lb)?\.(?:[\da-z_-]+\.)?mlb\.com/(?:(?:.*?/)?video/(?:topic/[\da-z_-]+/)?v|(?:shared/video/embed/embed\.html|[^/]+/video/play\.jsp)\?.*?\bcontent_id=)(?P<id>n?\d+)'
     _TESTS = [
         {
             'url': 'http://m.mlb.com/sea/video/topic/51231442/v34698933/nymsea-ackley-robs-a-home-run-with-an-amazing-catch/?c_id=sea',
@@ -80,6 +80,10 @@ class MLBIE(InfoExtractor):
             'url': 'http://mlb.mlb.com/es/video/play.jsp?content_id=36599553',
             'only_matching': True,
         },
+        {
+            'url': 'http://m.cardinals.mlb.com/stl/video/v51175783/atlstl-piscotty-makes-great-sliding-catch-on-line/?partnerId=as_mlb_20150321_42500876&adbid=579409712979910656&adbpl=tw&adbpr=52847728',
+            'only_matching': True,
+        }
     ]
 
     def _real_extract(self, url):

From b0872c19ea6fb5dcc20d695d6faead4af42da364 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 23 Mar 2015 22:15:01 +0600
Subject: [PATCH 0760/6219] [npo] Skip broken URL links (Closes #5266)

---
 youtube_dl/extractor/npo.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index 557dffa46..5d8448571 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -231,7 +231,10 @@ class NPOLiveIE(NPOBaseIE):
                 stream_url = self._download_json(
                     stream_info['stream'], display_id,
                     'Downloading %s URL' % stream_type,
-                    transform_source=strip_jsonp)
+                    'Unable to download %s URL' % stream_type,
+                    transform_source=strip_jsonp, fatal=False)
+                if not stream_url:
+                    continue
                 if stream_type == 'hds':
                     f4m_formats = self._extract_f4m_formats(stream_url, display_id)
                     # f4m downloader downloads only piece of live stream

From 5d1f0e607b3dfe7b1adea09a1188011e57acf0fb Mon Sep 17 00:00:00 2001
From: tiktok <tiktok7@users.noreply.github.com>
Date: Mon, 23 Mar 2015 23:16:50 +0100
Subject: [PATCH 0761/6219] [MiomioTv] updated based on feedback to merge
 request:     1) added comment to explain extra xml link download     2)
 changed {} entries to {0}, {1} etc     3) removed redundant language header
 (the others are required)     4) checked out the old version of the supported
 sites md (the change was     not required)

---
 docs/supportedsites.md            |  1 -
 youtube_dl/extractor/miomio_tv.py | 12 ++++++++----
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 53d280677..062cb3d62 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -244,7 +244,6 @@
  - **Mgoon**
  - **Minhateca**
  - **MinistryGrid**
- - **Miomio.tv**
  - **mitele.es**
  - **mixcloud**
  - **MLB**
diff --git a/youtube_dl/extractor/miomio_tv.py b/youtube_dl/extractor/miomio_tv.py
index 355774f54..ae20a32fa 100644
--- a/youtube_dl/extractor/miomio_tv.py
+++ b/youtube_dl/extractor/miomio_tv.py
@@ -23,10 +23,15 @@ class MiomioTvIE(InfoExtractor):
 
         title = self._html_search_regex(r'<meta\s+name="description"\s+content="\s*([^"]*)\s*"', webpage, 'title')
         ref_path = self._search_regex(r'src="(/mioplayer/.*?)"', webpage, 'ref_path')
-        referer = 'http://www.miomio.tv{}'.format(ref_path)
+        referer = 'http://www.miomio.tv{0}'.format(ref_path)
         xml_config = self._search_regex(r'flashvars="type=sina&amp;(.*?)&amp;cid=', webpage, 'xml config')
-        self._request_webpage("http://www.miomio.tv/mioplayer/mioplayerconfigfiles/xml.php?id={}&r=cc{}".format(id, 945), video_id)
-        xml_url = 'http://www.miomio.tv/mioplayer/mioplayerconfigfiles/sina.php?{}'.format(xml_config)
+        
+        # skipping the following page causes lags and eventually connection drop-outs
+        # id is normally a rotating three digit value but a fixed value always appears to work
+        self._request_webpage("http://www.miomio.tv/mioplayer/mioplayerconfigfiles/xml.php?id={0}&r=cc{1}".format(id, 945), video_id)
+
+        # the following xml contains the actual configuration information on the video file(s)
+        xml_url = 'http://www.miomio.tv/mioplayer/mioplayerconfigfiles/sina.php?{0}'.format(xml_config)
         vidconfig = self._download_xml(xml_url, video_id)
 
         file_els = vidconfig.findall('.//durl')
@@ -48,7 +53,6 @@ class MiomioTvIE(InfoExtractor):
 
         http_headers = {
             'Referer': referer,
-            'Accept-Language': 'en,en-US;q=0.7,de;q=0.3',
             'Accept-Encoding': 'gzip, deflate',
             'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
         }

From f5e2efbbf0a4a1fa02df09a12ce9ec69970ab14c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 24 Mar 2015 16:39:46 +0100
Subject: [PATCH 0762/6219] [options] Handle special characters in argv (Fixes
 #5157)

---
 test/test_execution.py | 9 +++++++++
 youtube_dl/options.py  | 5 +++++
 2 files changed, 14 insertions(+)

diff --git a/test/test_execution.py b/test/test_execution.py
index 60df187de..f31e51558 100644
--- a/test/test_execution.py
+++ b/test/test_execution.py
@@ -1,4 +1,6 @@
 #!/usr/bin/env python
+# coding: utf-8
+
 from __future__ import unicode_literals
 
 import unittest
@@ -27,5 +29,12 @@ class TestExecution(unittest.TestCase):
     def test_main_exec(self):
         subprocess.check_call([sys.executable, 'youtube_dl/__main__.py', '--version'], cwd=rootDir, stdout=_DEV_NULL)
 
+    def test_cmdline_umlauts(self):
+        p = subprocess.Popen(
+            [sys.executable, 'youtube_dl/__main__.py', 'ä', '--version'],
+            cwd=rootDir, stdout=_DEV_NULL, stderr=subprocess.PIPE)
+        _, stderr = p.communicate()
+        self.assertFalse(stderr)
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 4e6e47d6f..35c7e5fb3 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -794,6 +794,11 @@ def parseOpts(overrideArguments=None):
             write_string('[debug] Override config: ' + repr(overrideArguments) + '\n')
     else:
         command_line_conf = sys.argv[1:]
+        # Workaround for Python 2.x, where argv is a byte list
+        if sys.version_info < (3,):
+            command_line_conf = [
+                a.decode('utf-8', 'replace') for a in command_line_conf]
+
         if '--ignore-config' in command_line_conf:
             system_conf = []
             user_conf = []

From 48c971e07336f1e5d33a760e454c52fac83392d5 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 24 Mar 2015 16:39:53 +0100
Subject: [PATCH 0763/6219] release 2015.03.24

---
 docs/supportedsites.md | 2 ++
 youtube_dl/version.py  | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 72b365305..baf7b3880 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -231,6 +231,7 @@
  - **Letv**
  - **LetvPlaylist**
  - **LetvTv**
+ - **Libsyn**
  - **lifenews**: LIFE | NEWS
  - **LiveLeak**
  - **livestream**
@@ -310,6 +311,7 @@
  - **npo.nl:radio**
  - **npo.nl:radio:fragment**
  - **NRK**
+ - **NRKPlaylist**
  - **NRKTV**
  - **ntv.ru**
  - **Nuvid**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 51b4260aa..039ceadf2 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.03.18'
+__version__ = '2015.03.24'

From 17941321ab0b3f9548d1f65e3f9d69e8cd01c0a3 Mon Sep 17 00:00:00 2001
From: testbonn <andreas.beer@gmail.com>
Date: Wed, 25 Mar 2015 11:02:55 +0100
Subject: [PATCH 0764/6219] Clean up of --help output

For consistency and readability
---
 youtube_dl/options.py | 196 +++++++++++++++++++++---------------------
 1 file changed, 98 insertions(+), 98 deletions(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 35c7e5fb3..68193a271 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -120,19 +120,19 @@ def parseOpts(overrideArguments=None):
     general.add_option(
         '-h', '--help',
         action='help',
-        help='print this help text and exit')
+        help='Print this help text and exit')
     general.add_option(
         '-v', '--version',
         action='version',
-        help='print program version and exit')
+        help='Print program version and exit')
     general.add_option(
         '-U', '--update',
         action='store_true', dest='update_self',
-        help='update this program to latest version. Make sure that you have sufficient permissions (run with sudo if needed)')
+        help='Update this program to latest version. Make sure that you have sufficient permissions (run with sudo if needed)')
     general.add_option(
         '-i', '--ignore-errors',
         action='store_true', dest='ignoreerrors', default=False,
-        help='continue on download errors, for example to skip unavailable videos in a playlist')
+        help='Continue on download errors, for example to skip unavailable videos in a playlist')
     general.add_option(
         '--abort-on-error',
         action='store_false', dest='ignoreerrors',
@@ -140,7 +140,7 @@ def parseOpts(overrideArguments=None):
     general.add_option(
         '--dump-user-agent',
         action='store_true', dest='dump_user_agent', default=False,
-        help='display the current browser identification')
+        help='Display the current browser identification')
     general.add_option(
         '--list-extractors',
         action='store_true', dest='list_extractors', default=False,
@@ -152,7 +152,7 @@ def parseOpts(overrideArguments=None):
     general.add_option(
         '--default-search',
         dest='default_search', metavar='PREFIX',
-        help='Use this prefix for unqualified URLs. For example "gvsearch2:" downloads two videos from google videos for  youtube-dl "large apple". Use the value "auto" to let youtube-dl guess ("auto_warning" to emit a warning when guessing). "error" just throws an error. The default value "fixup_error" repairs broken URLs, but emits an error if this is not possible instead of searching.')
+        help='Use this prefix for unqualified URLs. For example "gvsearch2:" downloads two videos from google videos for youtube-dl "large apple". Use the value "auto" to let youtube-dl guess ("auto_warning" to emit a warning when guessing). "error" just throws an error. The default value "fixup_error" repairs broken URLs, but emits an error if this is not possible instead of searching.')
     general.add_option(
         '--ignore-config',
         action='store_true',
@@ -169,7 +169,7 @@ def parseOpts(overrideArguments=None):
         '--no-color', '--no-colors',
         action='store_true', dest='no_color',
         default=False,
-        help='Do not emit color codes in output.')
+        help='Do not emit color codes in output')
 
     network = optparse.OptionGroup(parser, 'Network Options')
     network.add_option(
@@ -206,23 +206,23 @@ def parseOpts(overrideArguments=None):
     selection.add_option(
         '--playlist-start',
         dest='playliststart', metavar='NUMBER', default=1, type=int,
-        help='playlist video to start at (default is %default)')
+        help='Playlist video to start at (default is %default)')
     selection.add_option(
         '--playlist-end',
         dest='playlistend', metavar='NUMBER', default=None, type=int,
-        help='playlist video to end at (default is last)')
+        help='Playlist video to end at (default is last)')
     selection.add_option(
         '--playlist-items',
         dest='playlist_items', metavar='ITEM_SPEC', default=None,
-        help='playlist video items to download. Specify indices of the videos in the playlist seperated by commas like: "--playlist-items 1,2,5,8" if you want to download videos indexed 1, 2, 5, 8 in the playlist. You can specify range: "--playlist-items 1-3,7,10-13", it will download the videos at index 1, 2, 3, 7, 10, 11, 12 and 13.')
+        help='Playlist video items to download. Specify indices of the videos in the playlist seperated by commas like: "--playlist-items 1,2,5,8" if you want to download videos indexed 1, 2, 5, 8 in the playlist. You can specify range: "--playlist-items 1-3,7,10-13", it will download the videos at index 1, 2, 3, 7, 10, 11, 12 and 13.')
     selection.add_option(
         '--match-title',
         dest='matchtitle', metavar='REGEX',
-        help='download only matching titles (regex or caseless sub-string)')
+        help='Download only matching titles (regex or caseless sub-string)')
     selection.add_option(
         '--reject-title',
         dest='rejecttitle', metavar='REGEX',
-        help='skip download for matching titles (regex or caseless sub-string)')
+        help='Skip download for matching titles (regex or caseless sub-string)')
     selection.add_option(
         '--max-downloads',
         dest='max_downloads', metavar='NUMBER', type=int, default=None,
@@ -238,19 +238,19 @@ def parseOpts(overrideArguments=None):
     selection.add_option(
         '--date',
         metavar='DATE', dest='date', default=None,
-        help='download only videos uploaded in this date')
+        help='Download only videos uploaded in this date')
     selection.add_option(
         '--datebefore',
         metavar='DATE', dest='datebefore', default=None,
-        help='download only videos uploaded on or before this date (i.e. inclusive)')
+        help='Download only videos uploaded on or before this date (i.e. inclusive)')
     selection.add_option(
         '--dateafter',
         metavar='DATE', dest='dateafter', default=None,
-        help='download only videos uploaded on or after this date (i.e. inclusive)')
+        help='Download only videos uploaded on or after this date (i.e. inclusive)')
     selection.add_option(
         '--min-views',
         metavar='COUNT', dest='min_views', default=None, type=int,
-        help='Do not download any videos with less than COUNT views',)
+        help='Do not download any videos with less than COUNT views')
     selection.add_option(
         '--max-views',
         metavar='COUNT', dest='max_views', default=None, type=int,
@@ -259,7 +259,7 @@ def parseOpts(overrideArguments=None):
         '--match-filter',
         metavar='FILTER', dest='match_filter', default=None,
         help=(
-            '(Experimental) Generic video filter. '
+            'Generic video filter (experimental). '
             'Specify any key (see help for -o for a list of available keys) to'
             ' match if the key is present, '
             '!key to check if the key is not present,'
@@ -277,15 +277,15 @@ def parseOpts(overrideArguments=None):
     selection.add_option(
         '--no-playlist',
         action='store_true', dest='noplaylist', default=False,
-        help='If the URL refers to a video and a playlist, download only the video.')
+        help='Download only the video, if the URL refers to a video and a playlist.')
     selection.add_option(
         '--yes-playlist',
         action='store_false', dest='noplaylist', default=False,
-        help='If the URL refers to a video and a playlist, download the playlist.')
+        help='Download the playlist, if the URL refers to a video and a playlist.')
     selection.add_option(
         '--age-limit',
         metavar='YEARS', dest='age_limit', default=None, type=int,
-        help='download only videos suitable for the given age')
+        help='Download only videos suitable for the given age')
     selection.add_option(
         '--download-archive', metavar='FILE',
         dest='download_archive',
@@ -299,30 +299,30 @@ def parseOpts(overrideArguments=None):
     authentication.add_option(
         '-u', '--username',
         dest='username', metavar='USERNAME',
-        help='login with this account ID')
+        help='Login with this account ID')
     authentication.add_option(
         '-p', '--password',
         dest='password', metavar='PASSWORD',
-        help='account password. If this option is left out, youtube-dl will ask interactively.')
+        help='Account password. If this option is left out, youtube-dl will ask interactively.')
     authentication.add_option(
         '-2', '--twofactor',
         dest='twofactor', metavar='TWOFACTOR',
-        help='two-factor auth code')
+        help='Two-factor auth code')
     authentication.add_option(
         '-n', '--netrc',
         action='store_true', dest='usenetrc', default=False,
-        help='use .netrc authentication data')
+        help='Use .netrc authentication data')
     authentication.add_option(
         '--video-password',
         dest='videopassword', metavar='PASSWORD',
-        help='video password (vimeo, smotri)')
+        help='Video password (vimeo, smotri)')
 
     video_format = optparse.OptionGroup(parser, 'Video Format Options')
     video_format.add_option(
         '-f', '--format',
         action='store', dest='format', metavar='FORMAT', default=None,
         help=(
-            'video format code, specify the order of preference using'
+            'Video format code, specify the order of preference using'
             ' slashes, as in -f 22/17/18 . '
             ' Instead of format codes, you can select by extension for the '
             'extensions aac, m4a, mp3, mp4, ogg, wav, webm. '
@@ -350,19 +350,19 @@ def parseOpts(overrideArguments=None):
     video_format.add_option(
         '--all-formats',
         action='store_const', dest='format', const='all',
-        help='download all available video formats')
+        help='Download all available video formats')
     video_format.add_option(
         '--prefer-free-formats',
         action='store_true', dest='prefer_free_formats', default=False,
-        help='prefer free video formats unless a specific one is requested')
+        help='Prefer free video formats unless a specific one is requested')
     video_format.add_option(
         '--max-quality',
         action='store', dest='format_limit', metavar='FORMAT',
-        help='highest quality format to download')
+        help='Specify highest quality format to download')
     video_format.add_option(
         '-F', '--list-formats',
         action='store_true', dest='listformats',
-        help='list all available formats')
+        help='List all available formats')
     video_format.add_option(
         '--youtube-include-dash-manifest',
         action='store_true', dest='youtube_include_dash_manifest', default=True,
@@ -382,46 +382,46 @@ def parseOpts(overrideArguments=None):
     subtitles.add_option(
         '--write-sub', '--write-srt',
         action='store_true', dest='writesubtitles', default=False,
-        help='write subtitle file')
+        help='Write subtitle file')
     subtitles.add_option(
         '--write-auto-sub', '--write-automatic-sub',
         action='store_true', dest='writeautomaticsub', default=False,
-        help='write automatic subtitle file (youtube only)')
+        help='Write automatic subtitle file (YouTube only)')
     subtitles.add_option(
         '--all-subs',
         action='store_true', dest='allsubtitles', default=False,
-        help='downloads all the available subtitles of the video')
+        help='Download all the available subtitles of the video')
     subtitles.add_option(
         '--list-subs',
         action='store_true', dest='listsubtitles', default=False,
-        help='lists all available subtitles for the video')
+        help='List all available subtitles for the video')
     subtitles.add_option(
         '--sub-format',
         action='store', dest='subtitlesformat', metavar='FORMAT', default='best',
-        help='subtitle format, accepts formats preference, for example: "ass/srt/best"')
+        help='Specify subtitle format preference, for example: "srt" or "ass/srt/best"')
     subtitles.add_option(
         '--sub-lang', '--sub-langs', '--srt-lang',
         action='callback', dest='subtitleslangs', metavar='LANGS', type='str',
         default=[], callback=_comma_separated_values_options_callback,
-        help='languages of the subtitles to download (optional) separated by commas, use IETF language tags like \'en,pt\'')
+        help='Languages of the subtitles to download (optional) separated by commas, use IETF language tags like \'en,pt\'')
 
     downloader = optparse.OptionGroup(parser, 'Download Options')
     downloader.add_option(
         '-r', '--rate-limit',
         dest='ratelimit', metavar='LIMIT',
-        help='maximum download rate in bytes per second (e.g. 50K or 4.2M)')
+        help='Maximum download rate in bytes per second (e.g. 50K or 4.2M)')
     downloader.add_option(
         '-R', '--retries',
         dest='retries', metavar='RETRIES', default=10,
-        help='number of retries (default is %default), or "infinite".')
+        help='Number of retries (default is %default), or "infinite".')
     downloader.add_option(
         '--buffer-size',
         dest='buffersize', metavar='SIZE', default='1024',
-        help='size of download buffer (e.g. 1024 or 16K) (default is %default)')
+        help='Size of download buffer (e.g. 1024 or 16K) (default is %default)')
     downloader.add_option(
         '--no-resize-buffer',
         action='store_true', dest='noresizebuffer', default=False,
-        help='do not automatically adjust the buffer size. By default, the buffer size is automatically resized from an initial value of SIZE.')
+        help='Do not automatically adjust the buffer size. By default, the buffer size is automatically resized from an initial value of SIZE.')
     downloader.add_option(
         '--test',
         action='store_true', dest='test', default=False,
@@ -433,11 +433,11 @@ def parseOpts(overrideArguments=None):
     downloader.add_option(
         '--xattr-set-filesize',
         dest='xattr_set_filesize', action='store_true',
-        help='(experimental) set file xattribute ytdl.filesize with expected filesize')
+        help='Set file xattribute ytdl.filesize with expected filesize (experimental)')
     downloader.add_option(
         '--hls-prefer-native',
         dest='hls_prefer_native', action='store_true',
-        help='(experimental) Use the native HLS downloader instead of ffmpeg.')
+        help='Use the native HLS downloader instead of ffmpeg (experimental)')
     downloader.add_option(
         '--external-downloader',
         dest='external_downloader', metavar='COMMAND',
@@ -446,7 +446,7 @@ def parseOpts(overrideArguments=None):
     downloader.add_option(
         '--external-downloader-args',
         dest='external_downloader_args', metavar='ARGS',
-        help='Give these arguments to the external downloader.')
+        help='Give these arguments to the external downloader')
 
     workarounds = optparse.OptionGroup(parser, 'Workarounds')
     workarounds.add_option(
@@ -456,7 +456,7 @@ def parseOpts(overrideArguments=None):
     workarounds.add_option(
         '--no-check-certificate',
         action='store_true', dest='no_check_certificate', default=False,
-        help='Suppress HTTPS certificate validation.')
+        help='Suppress HTTPS certificate validation')
     workarounds.add_option(
         '--prefer-insecure',
         '--prefer-unsecure', action='store_true', dest='prefer_insecure',
@@ -464,16 +464,16 @@ def parseOpts(overrideArguments=None):
     workarounds.add_option(
         '--user-agent',
         metavar='UA', dest='user_agent',
-        help='specify a custom user agent')
+        help='Specify a custom user agent')
     workarounds.add_option(
         '--referer',
         metavar='URL', dest='referer', default=None,
-        help='specify a custom referer, use if the video access is restricted to one domain',
+        help='Specify a custom referer, use if the video access is restricted to one domain',
     )
     workarounds.add_option(
         '--add-header',
         metavar='FIELD:VALUE', dest='headers', action='append',
-        help='specify a custom HTTP header and its value, separated by a colon \':\'. You can use this option multiple times',
+        help='Specify a custom HTTP header and its value, separated by a colon \':\'. You can use this option multiple times',
     )
     workarounds.add_option(
         '--bidi-workaround',
@@ -488,7 +488,7 @@ def parseOpts(overrideArguments=None):
     verbosity.add_option(
         '-q', '--quiet',
         action='store_true', dest='quiet', default=False,
-        help='activates quiet mode')
+        help='Activate quiet mode')
     verbosity.add_option(
         '--no-warnings',
         dest='no_warnings', action='store_true', default=False,
@@ -496,51 +496,51 @@ def parseOpts(overrideArguments=None):
     verbosity.add_option(
         '-s', '--simulate',
         action='store_true', dest='simulate', default=False,
-        help='do not download the video and do not write anything to disk',)
+        help='Do not download the video and do not write anything to disk')
     verbosity.add_option(
         '--skip-download',
         action='store_true', dest='skip_download', default=False,
-        help='do not download the video',)
+        help='Do not download the video')
     verbosity.add_option(
         '-g', '--get-url',
         action='store_true', dest='geturl', default=False,
-        help='simulate, quiet but print URL')
+        help='Simulate, quiet but print URL')
     verbosity.add_option(
         '-e', '--get-title',
         action='store_true', dest='gettitle', default=False,
-        help='simulate, quiet but print title')
+        help='Simulate, quiet but print title')
     verbosity.add_option(
         '--get-id',
         action='store_true', dest='getid', default=False,
-        help='simulate, quiet but print id')
+        help='Simulate, quiet but print id')
     verbosity.add_option(
         '--get-thumbnail',
         action='store_true', dest='getthumbnail', default=False,
-        help='simulate, quiet but print thumbnail URL')
+        help='Simulate, quiet but print thumbnail URL')
     verbosity.add_option(
         '--get-description',
         action='store_true', dest='getdescription', default=False,
-        help='simulate, quiet but print video description')
+        help='Simulate, quiet but print video description')
     verbosity.add_option(
         '--get-duration',
         action='store_true', dest='getduration', default=False,
-        help='simulate, quiet but print video length')
+        help='Simulate, quiet but print video length')
     verbosity.add_option(
         '--get-filename',
         action='store_true', dest='getfilename', default=False,
-        help='simulate, quiet but print output filename')
+        help='Simulate, quiet but print output filename')
     verbosity.add_option(
         '--get-format',
         action='store_true', dest='getformat', default=False,
-        help='simulate, quiet but print output format')
+        help='Simulate, quiet but print output format')
     verbosity.add_option(
         '-j', '--dump-json',
         action='store_true', dest='dumpjson', default=False,
-        help='simulate, quiet but print JSON information. See --output for a description of available keys.')
+        help='Simulate, quiet but print JSON information. See --output for a description of available keys.')
     verbosity.add_option(
         '-J', '--dump-single-json',
         action='store_true', dest='dump_single_json', default=False,
-        help='simulate, quiet but print JSON information for each command-line argument. If the URL refers to a playlist, dump the whole playlist information in a single line.')
+        help='Simulate, quiet but print JSON information for each command-line argument. If the URL refers to a playlist, dump the whole playlist information in a single line.')
     verbosity.add_option(
         '--print-json',
         action='store_true', dest='print_json', default=False,
@@ -549,23 +549,23 @@ def parseOpts(overrideArguments=None):
     verbosity.add_option(
         '--newline',
         action='store_true', dest='progress_with_newline', default=False,
-        help='output progress bar as new lines')
+        help='Output progress bar as new lines')
     verbosity.add_option(
         '--no-progress',
         action='store_true', dest='noprogress', default=False,
-        help='do not print progress bar')
+        help='Do not print progress bar')
     verbosity.add_option(
         '--console-title',
         action='store_true', dest='consoletitle', default=False,
-        help='display progress in console titlebar')
+        help='Display progress in console titlebar')
     verbosity.add_option(
         '-v', '--verbose',
         action='store_true', dest='verbose', default=False,
-        help='print various debugging information')
+        help='Print various debugging information')
     verbosity.add_option(
         '--dump-pages', '--dump-intermediate-pages',
         action='store_true', dest='dump_intermediate_pages', default=False,
-        help='print downloaded pages to debug problems (very verbose)')
+        help='Print downloaded pages to debug problems (very verbose)')
     verbosity.add_option(
         '--write-pages',
         action='store_true', dest='write_pages', default=False,
@@ -581,31 +581,31 @@ def parseOpts(overrideArguments=None):
     verbosity.add_option(
         '-C', '--call-home',
         dest='call_home', action='store_true', default=False,
-        help='Contact the youtube-dl server for debugging.')
+        help='Contact the youtube-dl server for debugging')
     verbosity.add_option(
         '--no-call-home',
         dest='call_home', action='store_false', default=False,
-        help='Do NOT contact the youtube-dl server for debugging.')
+        help='Do NOT contact the youtube-dl server for debugging')
 
     filesystem = optparse.OptionGroup(parser, 'Filesystem Options')
     filesystem.add_option(
         '-a', '--batch-file',
         dest='batchfile', metavar='FILE',
-        help='file containing URLs to download (\'-\' for stdin)')
+        help='File containing URLs to download (\'-\' for stdin)')
     filesystem.add_option(
         '--id', default=False,
-        action='store_true', dest='useid', help='use only video ID in file name')
+        action='store_true', dest='useid', help='Use only video ID in file name')
     filesystem.add_option(
         '-o', '--output',
         dest='outtmpl', metavar='TEMPLATE',
-        help=('output filename template. Use %(title)s to get the title, '
+        help=('Output filename template. Use %(title)s to get the title, '
               '%(uploader)s for the uploader name, %(uploader_id)s for the uploader nickname if different, '
               '%(autonumber)s to get an automatically incremented number, '
               '%(ext)s for the filename extension, '
               '%(format)s for the format description (like "22 - 1280x720" or "HD"), '
-              '%(format_id)s for the unique id of the format (like Youtube\'s itags: "137"), '
+              '%(format_id)s for the unique id of the format (like YouTube\'s itags: "137"), '
               '%(upload_date)s for the upload date (YYYYMMDD), '
-              '%(extractor)s for the provider (youtube, metacafe, etc), '
+              '%(extractor)s for the provider (YouTube, metacafe, etc), '
               '%(id)s for the video id, '
               '%(playlist_title)s, %(playlist_id)s, or %(playlist)s (=title if present, ID otherwise) for the playlist the video is in, '
               '%(playlist_index)s for the position in the playlist. '
@@ -617,7 +617,7 @@ def parseOpts(overrideArguments=None):
     filesystem.add_option(
         '--autonumber-size',
         dest='autonumber_size', metavar='NUMBER',
-        help='Specifies the number of digits in %(autonumber)s when it is present in output filename template or --auto-number option is given')
+        help='Specify the number of digits in %(autonumber)s when it is present in output filename template or --auto-number option is given')
     filesystem.add_option(
         '--restrict-filenames',
         action='store_true', dest='restrictfilenames', default=False,
@@ -625,55 +625,55 @@ def parseOpts(overrideArguments=None):
     filesystem.add_option(
         '-A', '--auto-number',
         action='store_true', dest='autonumber', default=False,
-        help='[deprecated; use  -o "%(autonumber)s-%(title)s.%(ext)s" ] number downloaded files starting from 00000')
+        help='[deprecated; use  -o "%(autonumber)s-%(title)s.%(ext)s" ] Number of downloaded files starting from 00000')
     filesystem.add_option(
         '-t', '--title',
         action='store_true', dest='usetitle', default=False,
-        help='[deprecated] use title in file name (default)')
+        help='[deprecated] Use title in file name (default)')
     filesystem.add_option(
         '-l', '--literal', default=False,
         action='store_true', dest='usetitle',
-        help='[deprecated] alias of --title')
+        help='[deprecated] Alias of --title')
     filesystem.add_option(
         '-w', '--no-overwrites',
         action='store_true', dest='nooverwrites', default=False,
-        help='do not overwrite files')
+        help='Do not overwrite files')
     filesystem.add_option(
         '-c', '--continue',
         action='store_true', dest='continue_dl', default=True,
-        help='force resume of partially downloaded files. By default, youtube-dl will resume downloads if possible.')
+        help='Force resume of partially downloaded files. By default, youtube-dl will resume downloads if possible.')
     filesystem.add_option(
         '--no-continue',
         action='store_false', dest='continue_dl',
-        help='do not resume partially downloaded files (restart from beginning)')
+        help='Do not resume partially downloaded files (restart from beginning)')
     filesystem.add_option(
         '--no-part',
         action='store_true', dest='nopart', default=False,
-        help='do not use .part files - write directly into output file')
+        help='Do not use .part files - write directly into output file')
     filesystem.add_option(
         '--no-mtime',
         action='store_false', dest='updatetime', default=True,
-        help='do not use the Last-modified header to set the file modification time')
+        help='Do not use the Last-modified header to set the file modification time')
     filesystem.add_option(
         '--write-description',
         action='store_true', dest='writedescription', default=False,
-        help='write video description to a .description file')
+        help='Write video description to a .description file')
     filesystem.add_option(
         '--write-info-json',
         action='store_true', dest='writeinfojson', default=False,
-        help='write video metadata to a .info.json file')
+        help='Write video metadata to a .info.json file')
     filesystem.add_option(
         '--write-annotations',
         action='store_true', dest='writeannotations', default=False,
-        help='write video annotations to a .annotation file')
+        help='Write video annotations to a .annotation file')
     filesystem.add_option(
         '--load-info',
         dest='load_info_filename', metavar='FILE',
-        help='json file containing the video information (created with the "--write-json" option)')
+        help='Specify JSON file containing the video information (created with the "--write-json" option)')
     filesystem.add_option(
         '--cookies',
         dest='cookiefile', metavar='FILE',
-        help='file to read cookies from and dump cookie jar in')
+        help='File to read cookies from and dump cookie jar in')
     filesystem.add_option(
         '--cache-dir', dest='cachedir', default=None, metavar='DIR',
         help='Location in the filesystem where youtube-dl can store some downloaded information permanently. By default $XDG_CACHE_HOME/youtube-dl or ~/.cache/youtube-dl . At the moment, only YouTube player files (for videos with obfuscated signatures) are cached, but that may change.')
@@ -689,11 +689,11 @@ def parseOpts(overrideArguments=None):
     thumbnail.add_option(
         '--write-thumbnail',
         action='store_true', dest='writethumbnail', default=False,
-        help='write thumbnail image to disk')
+        help='Write thumbnail image to disk')
     thumbnail.add_option(
         '--write-all-thumbnails',
         action='store_true', dest='write_all_thumbnails', default=False,
-        help='write all thumbnail image formats to disk')
+        help='Write all thumbnail image formats to disk')
     thumbnail.add_option(
         '--list-thumbnails',
         action='store_true', dest='list_thumbnails', default=False,
@@ -703,14 +703,14 @@ def parseOpts(overrideArguments=None):
     postproc.add_option(
         '-x', '--extract-audio',
         action='store_true', dest='extractaudio', default=False,
-        help='convert video files to audio-only files (requires ffmpeg or avconv and ffprobe or avprobe)')
+        help='Convert video files to audio-only files (requires ffmpeg or avconv and ffprobe or avprobe)')
     postproc.add_option(
         '--audio-format', metavar='FORMAT', dest='audioformat', default='best',
-        help='"best", "aac", "vorbis", "mp3", "m4a", "opus", or "wav"; "%default" by default')
+        help='Specify audio format: "best", "aac", "vorbis", "mp3", "m4a", "opus", or "wav"; "%default" by default')
     postproc.add_option(
         '--audio-quality', metavar='QUALITY',
         dest='audioquality', default='5',
-        help='ffmpeg/avconv audio quality specification, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K (default %default)')
+        help='Specify ffmpeg/avconv audio quality, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K (default %default)')
     postproc.add_option(
         '--recode-video',
         metavar='FORMAT', dest='recodevideo', default=None,
@@ -718,27 +718,27 @@ def parseOpts(overrideArguments=None):
     postproc.add_option(
         '-k', '--keep-video',
         action='store_true', dest='keepvideo', default=False,
-        help='keeps the video file on disk after the post-processing; the video is erased by default')
+        help='Keep the video file on disk after the post-processing; the video is erased by default')
     postproc.add_option(
         '--no-post-overwrites',
         action='store_true', dest='nopostoverwrites', default=False,
-        help='do not overwrite post-processed files; the post-processed files are overwritten by default')
+        help='Do not overwrite post-processed files; the post-processed files are overwritten by default')
     postproc.add_option(
         '--embed-subs',
         action='store_true', dest='embedsubtitles', default=False,
-        help='embed subtitles in the video (only for mp4 videos)')
+        help='Embed subtitles in the video (only for mp4 videos)')
     postproc.add_option(
         '--embed-thumbnail',
         action='store_true', dest='embedthumbnail', default=False,
-        help='embed thumbnail in the audio as cover art')
+        help='Embed thumbnail in the audio as cover art')
     postproc.add_option(
         '--add-metadata',
         action='store_true', dest='addmetadata', default=False,
-        help='write metadata to the video file')
+        help='Write metadata to the video file')
     postproc.add_option(
         '--metadata-from-title',
         metavar='FORMAT', dest='metafromtitle',
-        help='parse additional metadata like song title / artist from the video title. '
+        help='Parse additional metadata like song title / artist from the video title. '
              'The format syntax is the same as --output, '
              'the parsed parameters replace existing values. '
              'Additional templates: %(album), %(artist). '
@@ -747,7 +747,7 @@ def parseOpts(overrideArguments=None):
     postproc.add_option(
         '--xattrs',
         action='store_true', dest='xattrs', default=False,
-        help='write metadata to the video file\'s xattrs (using dublin core and xdg standards)')
+        help='Write metadata to the video file\'s xattrs (using dublin core and xdg standards)')
     postproc.add_option(
         '--fixup',
         metavar='POLICY', dest='fixup', default='detect_or_warn',

From 2676caf344c1e22f5b01588716471edeeebf0ea4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 25 Mar 2015 20:08:35 +0600
Subject: [PATCH 0765/6219] [redtube] Capture and output removed video message
 (#5281)

---
 youtube_dl/extractor/redtube.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/redtube.py b/youtube_dl/extractor/redtube.py
index 846b76c81..2e3db6887 100644
--- a/youtube_dl/extractor/redtube.py
+++ b/youtube_dl/extractor/redtube.py
@@ -1,6 +1,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..utils import ExtractorError
 
 
 class RedTubeIE(InfoExtractor):
@@ -19,6 +20,9 @@ class RedTubeIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
+        if any(s in webpage for s in ['video-deleted-info', '>This video has been removed']):
+            raise ExtractorError('Video %s has been removed' % video_id, expected=True)
+
         video_url = self._html_search_regex(
             r'<source src="(.+?)" type="video/mp4">', webpage, 'video URL')
         video_title = self._html_search_regex(

From 838b93405bc31654b07e120795115aec6e883855 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 25 Mar 2015 20:09:01 +0600
Subject: [PATCH 0766/6219] [redtube] Fix test

---
 youtube_dl/extractor/redtube.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/redtube.py b/youtube_dl/extractor/redtube.py
index 2e3db6887..d6054d717 100644
--- a/youtube_dl/extractor/redtube.py
+++ b/youtube_dl/extractor/redtube.py
@@ -8,11 +8,12 @@ class RedTubeIE(InfoExtractor):
     _VALID_URL = r'http://(?:www\.)?redtube\.com/(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://www.redtube.com/66418',
+        'md5': '7b8c22b5e7098a3e1c09709df1126d2d',
         'info_dict': {
             'id': '66418',
             'ext': 'mp4',
-            "title": "Sucked on a toilet",
-            "age_limit": 18,
+            'title': 'Sucked on a toilet',
+            'age_limit': 18,
         }
     }
 

From 5c19d18cbfb6a0271a9d0b7ff516e1e2e0eeccfb Mon Sep 17 00:00:00 2001
From: Amish Bhadeshia <amishb@gmail.com>
Date: Mon, 23 Mar 2015 13:02:44 +0000
Subject: [PATCH 0767/6219] [22Tracks] Add new extractor

Conflicts:
	youtube_dl/extractor/__init__.py
---
 youtube_dl/extractor/twentytwotracks.py | 111 ++++++++++++++++++++++++
 1 file changed, 111 insertions(+)
 create mode 100644 youtube_dl/extractor/twentytwotracks.py

diff --git a/youtube_dl/extractor/twentytwotracks.py b/youtube_dl/extractor/twentytwotracks.py
new file mode 100644
index 000000000..b655a7503
--- /dev/null
+++ b/youtube_dl/extractor/twentytwotracks.py
@@ -0,0 +1,111 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+# 22Tracks regularly replace the audio tracks that can be streamed on their
+# site. The tracks usually expire after 1 months, so we can't add tests.
+
+
+class TwentyTwoTracksIE(InfoExtractor):
+    _VALID_URL = r'http://22tracks\.com/([a-z]+)/([a-z]+[2]*)/(\d+)'
+    IE_NAME = 'TwentyTwoTracks:Tracks'
+
+    def _extract_info(self, city, genre, track=''):
+        self._base_url = "http://22tracks.com/api/"
+
+        if track == '':
+            itemid = genre
+        else:
+            itemid = track
+
+        cities = self._download_json(
+            self._base_url + 'cities', itemid,
+            'Downloading city info', 'Cannot download city info')
+        city_id = [x['id'] for x in cities if x['slug'] == city]
+
+        genres = self._download_json(
+            self._base_url + 'genres/' + str(city_id[0]), itemid,
+            'Downloading genre info', 'Cannot download genre info')
+        genre_id = [x['id'] for x in genres if x['slug'] == genre]
+
+        tracks = self._download_json(
+            self._base_url + 'tracks/' + str(genre_id[0]),
+            itemid, 'Downloading track info', 'Cannot download track info')
+
+        if track == '':
+            return [[x['title'] for x in genres if x['slug'] == genre][0],
+                    tracks]
+        else:
+            return [x for x in tracks if x['id'] == itemid][0]
+
+    def _get_token(self, filename, track_id):
+        token = self._download_json(
+            'http://22tracks.com/token.php?desktop=true&u=%2F128%2f{0}'.format(
+                filename), track_id, 'Finding download link...')
+
+        down_url = 'http://audio.22tracks.com{0}?st={1}&e={2}'.format(
+            token['filename'],
+            token['st'],
+            token['e'])
+
+        return down_url
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+
+        city_id = mobj.group(1)
+        genre_id = mobj.group(2)
+        track_id = mobj.group(3)
+
+        self.to_screen(':: Track ID found! - Downloading single track')
+
+        track_info = self._extract_info(city_id, genre_id, track_id)
+
+        download_url = self._get_token(track_info['filename'], track_id)
+        title = '{0}-{1}'.format(
+            track_info['artist'].strip(), track_info['title'].strip())
+
+        return {
+            'id': track_id,
+            'url': download_url,
+            'ext': 'mp3',
+            'title': title,
+            'duration': track_info['duration']
+        }
+
+
+class TwentyTwoTracksGenreIE(TwentyTwoTracksIE):
+    _VALID_URL = r'http://22tracks\.com/([a-z]+)/([a-z]+[2]*)/?'
+    IE_NAME = 'TwentyTwoTracks:Genre'
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+
+        city_id = mobj.group(1)
+        genre_id = mobj.group(2)
+
+        self.to_screen(':: Track ID not found! - Downloading entire genre')
+
+        playlist_info = self._extract_info(city_id, genre_id)
+
+        entries = []
+        for track in playlist_info[1]:
+            title = '{0}-{1}'.format(
+                track['artist'].strip(), track['title'].strip())
+            entries.append({
+                'id': track['id'],
+                'url': self._get_token(track['filename'], track['id']),
+                'ext': 'mp3',
+                'title': title
+            })
+
+        self.to_screen(':: Links found - Downloading Playlist')
+
+        return {
+            '_type': 'playlist',
+            'id': genre_id,
+            'title': playlist_info[0],
+            'entries': entries
+        }

From 8f76df7f3791bb085199eb90c3b4125931a44e55 Mon Sep 17 00:00:00 2001
From: Amish Bhadeshia <amishb@gmail.com>
Date: Tue, 24 Mar 2015 18:34:33 +0000
Subject: [PATCH 0768/6219] Updated init to add 22tracks

---
 youtube_dl/extractor/__init__.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index d73826d44..b0df52afd 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -526,6 +526,10 @@ from .tvp import TvpIE, TvpSeriesIE
 from .tvplay import TVPlayIE
 from .tweakers import TweakersIE
 from .twentyfourvideo import TwentyFourVideoIE
+from .twentytwotracks import (
+    TwentyTwoTracksIE,
+    TwentyTwoTracksGenreIE
+)
 from .twitch import (
     TwitchVideoIE,
     TwitchChapterIE,

From ae67d082fe0469109033714360b2b1ae92d7d86e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 25 Mar 2015 22:26:02 +0600
Subject: [PATCH 0769/6219] [22tracks] Improve and simplify

---
 youtube_dl/extractor/twentytwotracks.py | 121 ++++++++++--------------
 1 file changed, 48 insertions(+), 73 deletions(-)

diff --git a/youtube_dl/extractor/twentytwotracks.py b/youtube_dl/extractor/twentytwotracks.py
index b655a7503..d6c0ab184 100644
--- a/youtube_dl/extractor/twentytwotracks.py
+++ b/youtube_dl/extractor/twentytwotracks.py
@@ -3,109 +3,84 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..utils import int_or_none
 
 # 22Tracks regularly replace the audio tracks that can be streamed on their
 # site. The tracks usually expire after 1 months, so we can't add tests.
 
 
 class TwentyTwoTracksIE(InfoExtractor):
-    _VALID_URL = r'http://22tracks\.com/([a-z]+)/([a-z]+[2]*)/(\d+)'
-    IE_NAME = 'TwentyTwoTracks:Tracks'
+    _VALID_URL = r'https?://22tracks\.com/(?P<city>[a-z]+)/(?P<genre>[\da-z]+)/(?P<id>\d+)'
+    IE_NAME = '22tracks:track'
 
-    def _extract_info(self, city, genre, track=''):
-        self._base_url = "http://22tracks.com/api/"
+    _API_BASE = 'http://22tracks.com/api'
 
-        if track == '':
-            itemid = genre
-        else:
-            itemid = track
+    def _extract_info(self, city, genre_name, track_id=None):
+        item_id = track_id if track_id else genre_name
 
         cities = self._download_json(
-            self._base_url + 'cities', itemid,
-            'Downloading city info', 'Cannot download city info')
-        city_id = [x['id'] for x in cities if x['slug'] == city]
+            '%s/cities' % self._API_BASE, item_id,
+            'Downloading cities info',
+            'Unable to download cities info')
+        city_id = [x['id'] for x in cities if x['slug'] == city][0]
 
         genres = self._download_json(
-            self._base_url + 'genres/' + str(city_id[0]), itemid,
-            'Downloading genre info', 'Cannot download genre info')
-        genre_id = [x['id'] for x in genres if x['slug'] == genre]
+            '%s/genres/%s' % (self._API_BASE, city_id), item_id,
+            'Downloading %s genres info' % city,
+            'Unable to download %s genres info' % city)
+        genre = [x for x in genres if x['slug'] == genre_name][0]
+        genre_id = genre['id']
 
         tracks = self._download_json(
-            self._base_url + 'tracks/' + str(genre_id[0]),
-            itemid, 'Downloading track info', 'Cannot download track info')
+            '%s/tracks/%s' % (self._API_BASE, genre_id), item_id,
+            'Downloading %s genre tracks info' % genre_name,
+            'Unable to download track info')
 
-        if track == '':
-            return [[x['title'] for x in genres if x['slug'] == genre][0],
-                    tracks]
-        else:
-            return [x for x in tracks if x['id'] == itemid][0]
+        return [x for x in tracks if x['id'] == item_id][0] if track_id else [genre['title'], tracks]
 
-    def _get_token(self, filename, track_id):
+    def _get_track_url(self, filename, track_id):
         token = self._download_json(
-            'http://22tracks.com/token.php?desktop=true&u=%2F128%2f{0}'.format(
-                filename), track_id, 'Finding download link...')
-
-        down_url = 'http://audio.22tracks.com{0}?st={1}&e={2}'.format(
-            token['filename'],
-            token['st'],
-            token['e'])
-
-        return down_url
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-
-        city_id = mobj.group(1)
-        genre_id = mobj.group(2)
-        track_id = mobj.group(3)
-
-        self.to_screen(':: Track ID found! - Downloading single track')
-
-        track_info = self._extract_info(city_id, genre_id, track_id)
-
-        download_url = self._get_token(track_info['filename'], track_id)
-        title = '{0}-{1}'.format(
-            track_info['artist'].strip(), track_info['title'].strip())
+            'http://22tracks.com/token.php?desktop=true&u=/128/%s' % filename,
+            track_id, 'Downloading token', 'Unable to download token')
+        return 'http://audio.22tracks.com%s?st=%s&e=%d' % (token['filename'], token['st'], token['e'])
 
+    def _extract_track_info(self, track_info, track_id):
+        download_url = self._get_track_url(track_info['filename'], track_id)
+        title = '%s - %s' % (track_info['artist'].strip(), track_info['title'].strip())
         return {
             'id': track_id,
             'url': download_url,
             'ext': 'mp3',
             'title': title,
-            'duration': track_info['duration']
+            'duration': int_or_none(track_info.get('duration')),
+            'timestamp': int_or_none(track_info.get('published_at') or track_info.get('created'))
         }
 
-
-class TwentyTwoTracksGenreIE(TwentyTwoTracksIE):
-    _VALID_URL = r'http://22tracks\.com/([a-z]+)/([a-z]+[2]*)/?'
-    IE_NAME = 'TwentyTwoTracks:Genre'
-
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
 
-        city_id = mobj.group(1)
-        genre_id = mobj.group(2)
+        city = mobj.group('city')
+        genre = mobj.group('genre')
+        track_id = mobj.group('id')
 
-        self.to_screen(':: Track ID not found! - Downloading entire genre')
+        track_info = self._extract_info(city, genre, track_id)
+        return self._extract_track_info(track_info, track_id)
 
-        playlist_info = self._extract_info(city_id, genre_id)
 
-        entries = []
-        for track in playlist_info[1]:
-            title = '{0}-{1}'.format(
-                track['artist'].strip(), track['title'].strip())
-            entries.append({
-                'id': track['id'],
-                'url': self._get_token(track['filename'], track['id']),
-                'ext': 'mp3',
-                'title': title
-            })
+class TwentyTwoTracksGenreIE(TwentyTwoTracksIE):
+    _VALID_URL = r'https?://22tracks\.com/(?P<city>[a-z]+)/(?P<genre>[\da-z]+)/?$'
+    IE_NAME = '22tracks:genre'
 
-        self.to_screen(':: Links found - Downloading Playlist')
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
 
-        return {
-            '_type': 'playlist',
-            'id': genre_id,
-            'title': playlist_info[0],
-            'entries': entries
-        }
+        city = mobj.group('city')
+        genre = mobj.group('genre')
+
+        genre_title, tracks = self._extract_info(city, genre)
+
+        entries = [
+            self._extract_track_info(track_info, track_info['id'])
+            for track_info in tracks]
+
+        return self.playlist_result(entries, genre, genre_title)

From 094ce39c45422d4675228337f115a110c718d68a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 25 Mar 2015 22:27:20 +0600
Subject: [PATCH 0770/6219] Credit @amishb for 22tracks (#5276)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 512469f4c..59f1b5f21 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -117,3 +117,4 @@ Alexander Mamay
 Devin J. Pohly
 Eduardo Ferro Aldama
 Jeff Buchbinder
+Amish Bhadeshia

From db40364b8767839cc1abf5ece53511167d70f792 Mon Sep 17 00:00:00 2001
From: Mohammad Teimori Pabandi <mtp1376@gmail.com>
Date: Thu, 26 Mar 2015 18:17:21 +0430
Subject: [PATCH 0771/6219] [Varzesh3] Add new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/varzesh3.py | 63 ++++++++++++++++++++++++++++++++
 2 files changed, 64 insertions(+)
 create mode 100644 youtube_dl/extractor/varzesh3.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index b0df52afd..3011b784d 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -548,6 +548,7 @@ from .ultimedia import UltimediaIE
 from .unistra import UnistraIE
 from .urort import UrortIE
 from .ustream import UstreamIE, UstreamChannelIE
+from .varzesh3 import Varzesh3IE
 from .vbox7 import Vbox7IE
 from .veehd import VeeHDIE
 from .veoh import VeohIE
diff --git a/youtube_dl/extractor/varzesh3.py b/youtube_dl/extractor/varzesh3.py
new file mode 100644
index 000000000..85dcd8734
--- /dev/null
+++ b/youtube_dl/extractor/varzesh3.py
@@ -0,0 +1,63 @@
+# coding: utf-8
+from __future__ import unicode_literals
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+)
+import re
+
+
+class Varzesh3IE(InfoExtractor):
+    _VALID_URL = r'(?P<url>(https?://(?:www\.)?video\.varzesh3\.com)/(?P<id>.+))'
+    _TESTS = [
+    {
+        'url': 'http://video.varzesh3.com/video/%D8%AF%D8%A7%D9%86%D9%85%D8%A7%D8%B1%DA%A9-3-2-%D8%A2%D9%85%D8%B1%DB%8C%DA%A9%D8%A7/',
+        'md5': 'c4b850780df9374b078463c5cf6b3318',
+        'info_dict': {
+            'url': 'http://dl1.video.varzesh3.com/video/clip94/1/international/friendly/usa_2_3_danmark.mp4',
+            'id': '76464',
+            'ext': 'mp4',
+            'title': u'دانمارک 	۳-۲ 	آمریکا (گلهای بازی)',
+            'thumbnail': 'http://video.varzesh3.com/wp-content/uploads/250315_usa_danmark_site.jpg',
+            'description': u'بازی دوستانه ۲۰۱۵',
+
+        }
+    },
+    {
+        'url': 'http://video.varzesh3.com/germany/bundesliga/5-%D9%88%D8%A7%DA%A9%D9%86%D8%B4-%D8%A8%D8%B1%D8%AA%D8%B1-%D8%AF%D8%B1%D9%88%D8%A7%D8%B2%D9%87%E2%80%8C%D8%A8%D8%A7%D9%86%D8%A7%D9%86%D8%9B%D9%87%D9%81%D8%AA%D9%87-26-%D8%A8%D9%88%D9%86%D8%AF%D8%B3/',
+        'md5': '2a933874cb7dce4366075281eb49e855',
+        'info_dict': {
+            'url': 'http://dl1.video.varzesh3.com/video/clip94/1/video/namayeshi/saves_week26.mp4',
+            'id': '76337',
+            'ext': 'mp4',
+            'title': u'۵ واکنش برتر دروازه‌بانان؛هفته ۲۶ بوندسلیگا',
+            'thumbnail': 'http://video.varzesh3.com/wp-content/uploads/230315_saves_week26.jpg',
+            'description': u'فصل ۲۰۱۵-۲۰۱۴',
+        }
+    },
+    ]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        if not 'shortlink' in webpage:
+            raise ExtractorError('URL has no videos or there is a problem.')
+
+        title = self._html_search_regex(r'meta[^>]+property="og:title"[^>]+content="([^"]+)"', webpage, 'title')
+        video_link = self._html_search_regex(r'source[^>]+src="([^"]+)"', webpage, 'video_link')
+        vid_id = self._html_search_regex(r"link[^>]+rel='canonical'[^>]+href='\/\?p=([^']+)'\/>", webpage, 'vid_id')
+        try:
+            description = self._html_search_regex(r'<div class="matn">(.*?)</div>', webpage, 'description', flags=re.DOTALL)
+        except:
+            description = title
+        thumbnail = self._html_search_regex(r'link[^>]+rel="image_src"[^>]+href="([^"]+)"', webpage, 'thumbnail')
+
+        return {
+            'url': video_link,
+            'id': vid_id,
+            'title': title,
+            'ext': video_link.split(".")[-1],
+            'description': description,
+            'thumbnail': thumbnail,
+        }

From fbfcc2972b3b24bda092eaed92b81113154c4327 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Thu, 26 Mar 2015 16:13:53 +0200
Subject: [PATCH 0772/6219] [teamcoco] Fix extraction

---
 youtube_dl/extractor/teamcoco.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/teamcoco.py b/youtube_dl/extractor/teamcoco.py
index 7cb06f351..a46a7ecba 100644
--- a/youtube_dl/extractor/teamcoco.py
+++ b/youtube_dl/extractor/teamcoco.py
@@ -54,7 +54,7 @@ class TeamcocoIE(InfoExtractor):
             embed_url, video_id, 'Downloading embed page')
 
         player_data = self._parse_json(self._search_regex(
-            r'Y\.Ginger\.Module\.Player\((\{.*?\})\);', embed, 'player data'), video_id)
+            r'Y\.Ginger\.Module\.Player(?:;var\s*player\s*=\s*new\s*m)?\((\{.*?\})\);', embed, 'player data'), video_id)
         data = self._parse_json(
             base64.b64decode(player_data['preload'].encode('ascii')).decode('utf-8'), video_id)
 

From 91757b0f373ec3201f95066eeb0e09ebdcc1a067 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Thu, 26 Mar 2015 17:15:27 +0200
Subject: [PATCH 0773/6219] [utils] Escape all HTML entities written in
 hexadecimal form

---
 test/test_utils.py  | 2 ++
 youtube_dl/utils.py | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index a8ab87685..abaf1ab73 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -200,6 +200,8 @@ class TestUtil(unittest.TestCase):
 
     def test_unescape_html(self):
         self.assertEqual(unescapeHTML('%20;'), '%20;')
+        self.assertEqual(unescapeHTML('&#x2F;'), '/')
+        self.assertEqual(unescapeHTML('&#47;'), '/')
         self.assertEqual(
             unescapeHTML('&eacute;'), 'é')
 
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 472d4df41..245d623d8 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -348,7 +348,7 @@ def _htmlentity_transform(entity):
     if entity in compat_html_entities.name2codepoint:
         return compat_chr(compat_html_entities.name2codepoint[entity])
 
-    mobj = re.match(r'#(x?[0-9]+)', entity)
+    mobj = re.match(r'#(x[0-9a-fA-F]+|[0-9]+)', entity)
     if mobj is not None:
         numstr = mobj.group(1)
         if numstr.startswith('x'):

From a7fce980adeb12afd94e4414cc6a6b37021f6aa4 Mon Sep 17 00:00:00 2001
From: Mohammad Teimori Pabandi <mtp1376@gmail.com>
Date: Thu, 26 Mar 2015 19:47:34 +0430
Subject: [PATCH 0774/6219] removed one of tests that made problem with testing
 server

---
 youtube_dl/extractor/varzesh3.py | 19 ++-----------------
 1 file changed, 2 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/varzesh3.py b/youtube_dl/extractor/varzesh3.py
index 85dcd8734..cb3fdacff 100644
--- a/youtube_dl/extractor/varzesh3.py
+++ b/youtube_dl/extractor/varzesh3.py
@@ -9,21 +9,7 @@ import re
 
 class Varzesh3IE(InfoExtractor):
     _VALID_URL = r'(?P<url>(https?://(?:www\.)?video\.varzesh3\.com)/(?P<id>.+))'
-    _TESTS = [
-    {
-        'url': 'http://video.varzesh3.com/video/%D8%AF%D8%A7%D9%86%D9%85%D8%A7%D8%B1%DA%A9-3-2-%D8%A2%D9%85%D8%B1%DB%8C%DA%A9%D8%A7/',
-        'md5': 'c4b850780df9374b078463c5cf6b3318',
-        'info_dict': {
-            'url': 'http://dl1.video.varzesh3.com/video/clip94/1/international/friendly/usa_2_3_danmark.mp4',
-            'id': '76464',
-            'ext': 'mp4',
-            'title': u'دانمارک 	۳-۲ 	آمریکا (گلهای بازی)',
-            'thumbnail': 'http://video.varzesh3.com/wp-content/uploads/250315_usa_danmark_site.jpg',
-            'description': u'بازی دوستانه ۲۰۱۵',
-
-        }
-    },
-    {
+    _TEST ={
         'url': 'http://video.varzesh3.com/germany/bundesliga/5-%D9%88%D8%A7%DA%A9%D9%86%D8%B4-%D8%A8%D8%B1%D8%AA%D8%B1-%D8%AF%D8%B1%D9%88%D8%A7%D8%B2%D9%87%E2%80%8C%D8%A8%D8%A7%D9%86%D8%A7%D9%86%D8%9B%D9%87%D9%81%D8%AA%D9%87-26-%D8%A8%D9%88%D9%86%D8%AF%D8%B3/',
         'md5': '2a933874cb7dce4366075281eb49e855',
         'info_dict': {
@@ -34,8 +20,7 @@ class Varzesh3IE(InfoExtractor):
             'thumbnail': 'http://video.varzesh3.com/wp-content/uploads/230315_saves_week26.jpg',
             'description': u'فصل ۲۰۱۵-۲۰۱۴',
         }
-    },
-    ]
+    }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From 8896b614a9257ba33346f49a135371d460bfb311 Mon Sep 17 00:00:00 2001
From: Mohammad Teimori Pabandi <mtp1376@gmail.com>
Date: Thu, 26 Mar 2015 20:06:50 +0430
Subject: [PATCH 0775/6219] removing unicode literal because it is imported :))

---
 youtube_dl/extractor/varzesh3.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/varzesh3.py b/youtube_dl/extractor/varzesh3.py
index cb3fdacff..62bcb9118 100644
--- a/youtube_dl/extractor/varzesh3.py
+++ b/youtube_dl/extractor/varzesh3.py
@@ -16,7 +16,7 @@ class Varzesh3IE(InfoExtractor):
             'url': 'http://dl1.video.varzesh3.com/video/clip94/1/video/namayeshi/saves_week26.mp4',
             'id': '76337',
             'ext': 'mp4',
-            'title': u'۵ واکنش برتر دروازه‌بانان؛هفته ۲۶ بوندسلیگا',
+            'title': '۵ واکنش برتر دروازه‌بانان؛هفته ۲۶ بوندسلیگا',
             'thumbnail': 'http://video.varzesh3.com/wp-content/uploads/230315_saves_week26.jpg',
             'description': u'فصل ۲۰۱۵-۲۰۱۴',
         }

From 448830ce7b180250a9f2cbafc1625a20584e59d3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 26 Mar 2015 21:41:09 +0600
Subject: [PATCH 0776/6219] [youtube:watchlater] Extract watchlater as playlist
 (Closes #5280)

---
 youtube_dl/extractor/youtube.py | 52 ++++++++++++++++++---------------
 1 file changed, 29 insertions(+), 23 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 27c8c4453..e0f9228a4 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1263,27 +1263,7 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
 
         return self.playlist_result(url_results, playlist_id, title)
 
-    def _real_extract(self, url):
-        # Extract playlist id
-        mobj = re.match(self._VALID_URL, url)
-        if mobj is None:
-            raise ExtractorError('Invalid URL: %s' % url)
-        playlist_id = mobj.group(1) or mobj.group(2)
-
-        # Check if it's a video-specific URL
-        query_dict = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
-        if 'v' in query_dict:
-            video_id = query_dict['v'][0]
-            if self._downloader.params.get('noplaylist'):
-                self.to_screen('Downloading just video %s because of --no-playlist' % video_id)
-                return self.url_result(video_id, 'Youtube', video_id=video_id)
-            else:
-                self.to_screen('Downloading playlist %s - add --no-playlist to just download video %s' % (playlist_id, video_id))
-
-        if playlist_id.startswith('RD') or playlist_id.startswith('UL'):
-            # Mixes require a custom extraction process
-            return self._extract_mix(playlist_id)
-
+    def _extract_playlist(self, playlist_id):
         url = self._TEMPLATE_URL % playlist_id
         page = self._download_webpage(url, playlist_id)
         more_widget_html = content_html = page
@@ -1327,6 +1307,29 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
         url_results = self._ids_to_results(ids)
         return self.playlist_result(url_results, playlist_id, playlist_title)
 
+    def _real_extract(self, url):
+        # Extract playlist id
+        mobj = re.match(self._VALID_URL, url)
+        if mobj is None:
+            raise ExtractorError('Invalid URL: %s' % url)
+        playlist_id = mobj.group(1) or mobj.group(2)
+
+        # Check if it's a video-specific URL
+        query_dict = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
+        if 'v' in query_dict:
+            video_id = query_dict['v'][0]
+            if self._downloader.params.get('noplaylist'):
+                self.to_screen('Downloading just video %s because of --no-playlist' % video_id)
+                return self.url_result(video_id, 'Youtube', video_id=video_id)
+            else:
+                self.to_screen('Downloading playlist %s - add --no-playlist to just download video %s' % (playlist_id, video_id))
+
+        if playlist_id.startswith('RD') or playlist_id.startswith('UL'):
+            # Mixes require a custom extraction process
+            return self._extract_mix(playlist_id)
+
+        return self._extract_playlist(playlist_id)
+
 
 class YoutubeChannelIE(InfoExtractor):
     IE_DESC = 'YouTube.com channels'
@@ -1649,13 +1652,16 @@ class YoutubeRecommendedIE(YoutubeFeedsInfoExtractor):
     _PLAYLIST_TITLE = 'Youtube Recommended videos'
 
 
-class YoutubeWatchLaterIE(YoutubeFeedsInfoExtractor):
+class YoutubeWatchLaterIE(YoutubePlaylistIE):
     IE_DESC = 'Youtube watch later list, ":ytwatchlater" for short (requires authentication)'
-    _VALID_URL = r'https?://www\.youtube\.com/feed/watch_later|:ytwatchlater'
+    _VALID_URL = r'https?://www\.youtube\.com/(?:feed/watch_later|playlist\?list=WL)|:ytwatchlater'
     _FEED_NAME = 'watch_later'
     _PLAYLIST_TITLE = 'Youtube Watch Later'
     _PERSONAL_FEED = True
 
+    def _real_extract(self, url):
+        return self._extract_playlist('WL')
+
 
 class YoutubeHistoryIE(YoutubeFeedsInfoExtractor):
     IE_DESC = 'Youtube watch history, ":ythistory" for short (requires authentication)'

From 7e17ec8c717796ec304b205059c741e7b47474dd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 26 Mar 2015 21:42:28 +0600
Subject: [PATCH 0777/6219] [youtube] Clarify some IE_NAMEs

---
 youtube_dl/extractor/youtube.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index e0f9228a4..eba699c3a 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1646,6 +1646,7 @@ class YoutubeFeedsInfoExtractor(YoutubeBaseInfoExtractor):
 
 
 class YoutubeRecommendedIE(YoutubeFeedsInfoExtractor):
+    IE_NAME = 'youtube:recommended'
     IE_DESC = 'YouTube.com recommended videos, ":ytrec" for short (requires authentication)'
     _VALID_URL = r'https?://www\.youtube\.com/feed/recommended|:ytrec(?:ommended)?'
     _FEED_NAME = 'recommended'
@@ -1653,6 +1654,7 @@ class YoutubeRecommendedIE(YoutubeFeedsInfoExtractor):
 
 
 class YoutubeWatchLaterIE(YoutubePlaylistIE):
+    IE_NAME = 'youtube:watchlater'
     IE_DESC = 'Youtube watch later list, ":ytwatchlater" for short (requires authentication)'
     _VALID_URL = r'https?://www\.youtube\.com/(?:feed/watch_later|playlist\?list=WL)|:ytwatchlater'
     _FEED_NAME = 'watch_later'
@@ -1664,6 +1666,7 @@ class YoutubeWatchLaterIE(YoutubePlaylistIE):
 
 
 class YoutubeHistoryIE(YoutubeFeedsInfoExtractor):
+    IE_NAME = 'youtube:history'
     IE_DESC = 'Youtube watch history, ":ythistory" for short (requires authentication)'
     _VALID_URL = 'https?://www\.youtube\.com/feed/history|:ythistory'
     _FEED_NAME = 'history'

From 425142be601adfb4218f78c10aedbed14ad1facd Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Thu, 26 Mar 2015 17:46:20 +0200
Subject: [PATCH 0778/6219] [slideshare] Fix extraction (#5279)

---
 youtube_dl/extractor/slideshare.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/slideshare.py b/youtube_dl/extractor/slideshare.py
index 9f79ff5c1..0b717a1e4 100644
--- a/youtube_dl/extractor/slideshare.py
+++ b/youtube_dl/extractor/slideshare.py
@@ -30,7 +30,7 @@ class SlideshareIE(InfoExtractor):
         page_title = mobj.group('title')
         webpage = self._download_webpage(url, page_title)
         slideshare_obj = self._search_regex(
-            r'var\s+slideshare_object\s*=\s*({.*?});\s*var\s+user_info\s*=',
+            r'\$\.extend\(slideshare_object,\s*(\{.*?\})\);',
             webpage, 'slideshare object')
         info = json.loads(slideshare_obj)
         if info['slideshow']['type'] != 'video':

From 223b27f46cdf3cde9c9556dac19055ad5d8bff5f Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Thu, 26 Mar 2015 19:48:22 +0200
Subject: [PATCH 0779/6219] [vessel] Add new extractor (Closes #5275)

---
 youtube_dl/extractor/__init__.py |   1 +
 youtube_dl/extractor/vessel.py   | 127 +++++++++++++++++++++++++++++++
 2 files changed, 128 insertions(+)
 create mode 100644 youtube_dl/extractor/vessel.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index b0df52afd..053eb32ae 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -551,6 +551,7 @@ from .ustream import UstreamIE, UstreamChannelIE
 from .vbox7 import Vbox7IE
 from .veehd import VeeHDIE
 from .veoh import VeohIE
+from .vessel import VesselIE
 from .vesti import VestiIE
 from .vevo import VevoIE
 from .vgtv import VGTVIE
diff --git a/youtube_dl/extractor/vessel.py b/youtube_dl/extractor/vessel.py
new file mode 100644
index 000000000..123d9470e
--- /dev/null
+++ b/youtube_dl/extractor/vessel.py
@@ -0,0 +1,127 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import json
+
+from .common import InfoExtractor
+from ..compat import compat_urllib_request
+from ..utils import (
+    ExtractorError,
+    parse_iso8601,
+)
+
+
+class VesselIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?vessel\.com/videos/(?P<id>[0-9a-zA-Z]+)'
+    _API_URL_TEMPLATE = 'https://www.vessel.com/api/view/items/%s'
+    _LOGIN_URL = 'https://www.vessel.com/api/account/login'
+    _NETRC_MACHINE = 'vessel'
+    _TEST = {
+        'url': 'https://www.vessel.com/videos/HDN7G5UMs',
+        'md5': '455cdf8beb71c6dd797fd2f3818d05c4',
+        'info_dict': {
+            'id': 'HDN7G5UMs',
+            'ext': 'mp4',
+            'title': 'Nvidia GeForce GTX Titan X - The Best Video Card on the Market?',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'upload_date': '20150317',
+            'description': 'Did Nvidia pull out all the stops on the Titan X, or does its performance leave something to be desired?',
+            'timestamp': int,
+        },
+    }
+
+    @staticmethod
+    def make_json_request(url, data):
+        payload = json.dumps(data).encode('utf-8')
+        req = compat_urllib_request.Request(url, payload)
+        req.add_header('Content-Type', 'application/json; charset=utf-8')
+        return req
+
+    @staticmethod
+    def find_assets(data, asset_type):
+        for asset in data.get('assets', []):
+            if asset.get('type') == asset_type:
+                yield asset
+
+    def _check_access_rights(self, data):
+        access_info = data.get('__view', {})
+        if access_info.get('allow_access') == False:
+            err_code = access_info.get('error_code') or ''
+            if err_code == 'ITEM_PAID_ONLY':
+                raise ExtractorError(
+                    'This video requires subscription.', expected=True)
+            else:
+                raise ExtractorError(
+                    'Access to this content is restricted. (%s said: %s)' % (self.IE_NAME, err_code), expected=True)
+
+    def _login(self):
+        (username, password) = self._get_login_info()
+        if username is None:
+            return
+        self.report_login()
+        data = {
+            'client_id': 'web',
+            'type': 'password',
+            'user_key': username,
+            'password': password,
+        }
+        login_request = VesselIE.make_json_request(self._LOGIN_URL, data)
+        self._download_webpage(login_request, None, False, 'Wrong login info')
+
+    def _real_initialize(self):
+        self._login()
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+        data = self._parse_json(self._search_regex(
+            r'App\.bootstrapData\((.*?)\);', webpage, 'data'), video_id)
+        asset_id = data['model']['data']['id']
+
+        req = VesselIE.make_json_request(
+            self._API_URL_TEMPLATE % asset_id, {'client': 'web'})
+        data = self._download_json(req, video_id)
+
+        self._check_access_rights(data)
+
+        try:
+            video_asset = next(VesselIE.find_assets(data, 'video'))
+        except StopIteration:
+            raise ExtractorError('No video assets found')
+
+        formats = []
+        for f in video_asset.get('sources', []):
+            if f['name'] == 'hls-index':
+                formats.extend(self._extract_m3u8_formats(
+                    f['location'], video_id, ext='mp4', m3u8_id='m3u8'))
+            else:
+                formats.append({
+                    'format_id': f['name'],
+                    'tbr': f.get('bitrate'),
+                    'height': f.get('height'),
+                    'width': f.get('width'),
+                    'url': f['location'],
+                })
+        self._sort_formats(formats)
+
+        thumbnails = []
+        for im_asset in VesselIE.find_assets(data, 'image'):
+            thumbnails.append({
+                'url': im_asset['location'],
+                'width': im_asset.get('width', 0),
+                'height': im_asset.get('height', 0),
+            })
+
+        return {
+            'id': video_id,
+            'title': data['title'],
+            'formats': formats,
+            'thumbnails': thumbnails,
+            'description': data.get('short_description'),
+            'duration': data.get('duration'),
+            'comment_count': data.get('comment_count'),
+            'like_count': data.get('like_count'),
+            'view_count': data.get('view_count'),
+            'timestamp': parse_iso8601(data.get('released_at')),
+        }

From 31c48098276100a3fa2529b1443687e56838f707 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 26 Mar 2015 23:57:46 +0600
Subject: [PATCH 0780/6219] [safari] Improve and simplify

---
 youtube_dl/extractor/safari.py | 101 +++++++++++++++++++--------------
 1 file changed, 57 insertions(+), 44 deletions(-)

diff --git a/youtube_dl/extractor/safari.py b/youtube_dl/extractor/safari.py
index 3e494b960..10251f29e 100644
--- a/youtube_dl/extractor/safari.py
+++ b/youtube_dl/extractor/safari.py
@@ -2,7 +2,6 @@
 from __future__ import unicode_literals
 
 import re
-import json
 
 from .common import InfoExtractor
 from .brightcove import BrightcoveIE
@@ -20,16 +19,18 @@ from ..utils import (
 
 class SafariBaseIE(InfoExtractor):
     _LOGIN_URL = 'https://www.safaribooksonline.com/accounts/login/'
-    _SUCCESSFUL_LOGIN_REGEX = r'<a href="/accounts/logout/"[^>]+>Sign Out</a>'
-    _ACCOUNT_CREDENTIALS_HINT = ('Use --username and --password options to '
-                                 'supply credentials for safaribooksonline.com ')
-    _NETRC_MACHINE = 'safaribooksonline'
+    _SUCCESSFUL_LOGIN_REGEX = r'<a href="/accounts/logout/"[^>]*>Sign Out</a>'
+    _ACCOUNT_CREDENTIALS_HINT = 'Use --username and --password options to supply credentials for safaribooksonline.com'
+    _NETRC_MACHINE = 'safari'
+
+    _API_BASE = 'https://www.safaribooksonline.com/api/v1/book'
+    _API_FORMAT = 'json'
 
     LOGGED_IN = False
 
     def _real_initialize(self):
         # We only need to log in once for courses or individual videos
-        if not SafariBaseIE.LOGGED_IN:
+        if not self.LOGGED_IN:
             self._login()
             SafariBaseIE.LOGGED_IN = True
 
@@ -49,7 +50,7 @@ class SafariBaseIE(InfoExtractor):
             'Downloading login form')
 
         csrf = self._html_search_regex(
-            r"<input +type='hidden' +name='csrfmiddlewaretoken' +value='([^']+)' +/>",
+            r"name='csrfmiddlewaretoken'\s+value='([^']+)'",
             login_page, 'csrf token')
 
         login_form = {
@@ -66,8 +67,9 @@ class SafariBaseIE(InfoExtractor):
             request, None, 'Logging in as %s' % username)
 
         if re.search(self._SUCCESSFUL_LOGIN_REGEX, login_page) is None:
-            raise ExtractorError('Login failed; make sure your credentials are correct and '
-                                 'try again.', expected=True)
+            raise ExtractorError(
+                'Login failed; make sure your credentials are correct and try again.',
+                expected=True)
 
         self.to_screen('Login successful')
 
@@ -75,69 +77,80 @@ class SafariBaseIE(InfoExtractor):
 class SafariIE(SafariBaseIE):
     IE_NAME = 'safari'
     IE_DESC = 'safaribooksonline.com online video'
-    _VALID_URL = (r'https?://(?:www\.)?safaribooksonline\.com/library/view/[^/]+/'
-                  '(?P<id>\d+)/(?P<part>part\d+)\.html')
-    _TEST = {
-        'url': ('https://www.safaribooksonline.com/library/view/'
-                'hadoop-fundamentals-livelessons/9780133392838/part00.html'),
+    _VALID_URL = r'''(?x)https?://
+                            (?:www\.)?safaribooksonline\.com/
+                                (?:
+                                    library/view/[^/]+|
+                                    api/v1/book
+                                )/
+                                (?P<course_id>\d+)/
+                                    (?:chapter(?:-content)?/)?
+                                (?P<part>part\d+)\.html
+    '''
+
+    _TESTS = [{
+        'url': 'https://www.safaribooksonline.com/library/view/hadoop-fundamentals-livelessons/9780133392838/part00.html',
         'md5': '5b0c4cc1b3c1ba15dda7344085aa5592',
         'info_dict': {
-            'id': '9780133392838',
+            'id': '2842601850001',
             'ext': 'mp4',
             'title': 'Introduction',
-        }
-    }
+        },
+        'skip': 'Requires safaribooksonline account credentials',
+    }, {
+        'url': 'https://www.safaribooksonline.com/api/v1/book/9780133392838/chapter/part00.html',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
+        course_id = mobj.group('course_id')
         part = mobj.group('part')
 
-        webpage = self._download_webpage(url, part)
+        webpage = self._download_webpage(
+            '%s/%s/chapter-content/%s.html' % (self._API_BASE, course_id, part),
+            part)
+
         bc_url = BrightcoveIE._extract_brightcove_url(webpage)
         if not bc_url:
             raise ExtractorError('Could not extract Brightcove URL from %s' % url, expected=True)
 
-        return {
-            '_type': 'url',
-            'url': smuggle_url(bc_url, {'Referer': url}),
-            'ie_key': 'Brightcove'
-        }
+        return self.url_result(smuggle_url(bc_url, {'Referer': url}), 'Brightcove')
 
 
 class SafariCourseIE(SafariBaseIE):
     IE_NAME = 'safari:course'
     IE_DESC = 'safaribooksonline.com online courses'
 
-    _VALID_URL = (r'https?://(?:www\.)?safaribooksonline\.com/library/view/'
-                  '(?P<course_path>[^/]+)/(?P<id>\d+)/?$')
+    _VALID_URL = r'https?://(?:www\.)?safaribooksonline\.com/(?:library/view/[^/]+|api/v1/book)/(?P<id>\d+)/?(?:[#?]|$)'
 
-    _API_BASE = 'https://www.safaribooksonline.com/api/v1/book'
-    _API_FORMAT = 'json'
+    _TESTS = [{
+        'url': 'https://www.safaribooksonline.com/library/view/hadoop-fundamentals-livelessons/9780133392838/',
+        'info_dict': {
+            'id': '9780133392838',
+            'title': 'Hadoop Fundamentals LiveLessons',
+        },
+        'playlist_count': 22,
+        'skip': 'Requires safaribooksonline account credentials',
+    }, {
+        'url': 'https://www.safaribooksonline.com/api/v1/book/9781449396459/?override_format=json',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        course_path = mobj.group('course_path')
-        course_id = mobj.group('id')
+        course_id = self._match_id(url)
 
-        webpage = self._download_webpage(
+        course_json = self._download_json(
             '%s/%s/?override_format=%s' % (self._API_BASE, course_id, self._API_FORMAT),
-            course_path, 'Downloading course JSON')
-
-        course_json = json.loads(webpage)
+            course_id, 'Downloading course JSON')
 
         if 'chapters' not in course_json:
-            raise ExtractorError('No chapters found for course %s' % course_id, expected=True)
-
-        num_parts = len(course_json['chapters'])
-        parts = ['%02d' % part for part in range(num_parts)]
+            raise ExtractorError(
+                'No chapters found for course %s' % course_id, expected=True)
 
         entries = [
-            self.url_result(
-                'https://www.safaribooksonline.com/library/view/%s/%s/part%s.html' % (course_path,
-                                                                                      course_id,
-                                                                                      part_id),
-                'Safari')
-            for part_id in parts]
+            self.url_result(chapter, 'Safari')
+            for chapter in course_json['chapters']]
 
         course_title = course_json['title']
 

From c496ec084823436fa27eda0a12887c028f572ed3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 26 Mar 2015 19:51:40 +0100
Subject: [PATCH 0781/6219] [vessel] Fix pep8 issue

---
 youtube_dl/extractor/vessel.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vessel.py b/youtube_dl/extractor/vessel.py
index 123d9470e..6215f0642 100644
--- a/youtube_dl/extractor/vessel.py
+++ b/youtube_dl/extractor/vessel.py
@@ -45,7 +45,7 @@ class VesselIE(InfoExtractor):
 
     def _check_access_rights(self, data):
         access_info = data.get('__view', {})
-        if access_info.get('allow_access') == False:
+        if not access_info.get('allow_access', True):
             err_code = access_info.get('error_code') or ''
             if err_code == 'ITEM_PAID_ONLY':
                 raise ExtractorError(

From 157e9e5aa5e2e461ec0e4f1e0875125a440d6d60 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 26 Mar 2015 20:03:31 +0100
Subject: [PATCH 0782/6219] [youtube:watchlater] Remove unused properties and
 fix tests

---
 test/test_all_urls.py           | 2 +-
 youtube_dl/extractor/youtube.py | 5 ++---
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/test/test_all_urls.py b/test/test_all_urls.py
index 6ae168b7f..a9db42b30 100644
--- a/test/test_all_urls.py
+++ b/test/test_all_urls.py
@@ -59,7 +59,7 @@ class TestAllURLsMatching(unittest.TestCase):
         self.assertMatch('www.youtube.com/NASAgovVideo/videos', ['youtube:user'])
 
     def test_youtube_feeds(self):
-        self.assertMatch('https://www.youtube.com/feed/watch_later', ['youtube:watch_later'])
+        self.assertMatch('https://www.youtube.com/feed/watch_later', ['youtube:watchlater'])
         self.assertMatch('https://www.youtube.com/feed/subscriptions', ['youtube:subscriptions'])
         self.assertMatch('https://www.youtube.com/feed/recommended', ['youtube:recommended'])
         self.assertMatch('https://www.youtube.com/my_favorites', ['youtube:favorites'])
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index eba699c3a..5488101e1 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1657,9 +1657,8 @@ class YoutubeWatchLaterIE(YoutubePlaylistIE):
     IE_NAME = 'youtube:watchlater'
     IE_DESC = 'Youtube watch later list, ":ytwatchlater" for short (requires authentication)'
     _VALID_URL = r'https?://www\.youtube\.com/(?:feed/watch_later|playlist\?list=WL)|:ytwatchlater'
-    _FEED_NAME = 'watch_later'
-    _PLAYLIST_TITLE = 'Youtube Watch Later'
-    _PERSONAL_FEED = True
+
+    _TESTS = []  # override PlaylistIE tests
 
     def _real_extract(self, url):
         return self._extract_playlist('WL')

From 2315fb5e5fb65271519fc018bb1df5a7269f0a6a Mon Sep 17 00:00:00 2001
From: Mohammad Teimori Pabandi <mtp1376@gmail.com>
Date: Thu, 26 Mar 2015 23:53:57 +0430
Subject: [PATCH 0783/6219] unicde :(

---
 youtube_dl/extractor/varzesh3.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/varzesh3.py b/youtube_dl/extractor/varzesh3.py
index 62bcb9118..eb49586cc 100644
--- a/youtube_dl/extractor/varzesh3.py
+++ b/youtube_dl/extractor/varzesh3.py
@@ -18,7 +18,7 @@ class Varzesh3IE(InfoExtractor):
             'ext': 'mp4',
             'title': '۵ واکنش برتر دروازه‌بانان؛هفته ۲۶ بوندسلیگا',
             'thumbnail': 'http://video.varzesh3.com/wp-content/uploads/230315_saves_week26.jpg',
-            'description': u'فصل ۲۰۱۵-۲۰۱۴',
+            'description': 'فصل ۲۰۱۵-۲۰۱۴',
         }
     }
 

From 998e6cdba01ff54e2009d50c3026e7af0bb1c11a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 27 Mar 2015 03:05:08 +0600
Subject: [PATCH 0784/6219] [vimeo] Capture and output error message (#5294)

---
 youtube_dl/extractor/vimeo.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index bd09652cd..28bcc89cd 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -244,6 +244,16 @@ class VimeoIE(VimeoBaseInfoExtractor):
         # and latter we extract those that are Vimeo specific.
         self.report_extraction(video_id)
 
+        vimeo_config = self._search_regex(
+            r'vimeo\.config\s*=\s*({.+?});', webpage,
+            'vimeo config', default=None)
+        if vimeo_config:
+            seed_status = self._parse_json(vimeo_config, video_id).get('seed_status', {})
+            if seed_status.get('state') == 'failed':
+                raise ExtractorError(
+                    '%s returned error: %s' % (self.IE_NAME, seed_status['title']),
+                    expected=True)
+
         # Extract the config JSON
         try:
             try:

From af140002158c1079b1365392c6a48ea06ad23c82 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Thu, 26 Mar 2015 23:24:15 +0200
Subject: [PATCH 0785/6219] [eroprofile] Add login support (#5269)

---
 youtube_dl/extractor/eroprofile.py | 52 ++++++++++++++++++++++++++++--
 1 file changed, 50 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/eroprofile.py b/youtube_dl/extractor/eroprofile.py
index 79e2fbd39..0cbca90b0 100644
--- a/youtube_dl/extractor/eroprofile.py
+++ b/youtube_dl/extractor/eroprofile.py
@@ -1,11 +1,17 @@
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
+from ..compat import compat_urllib_parse
+from ..utils import ExtractorError
 
 
 class EroProfileIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?eroprofile\.com/m/videos/view/(?P<id>[^/]+)'
-    _TEST = {
+    _LOGIN_URL = 'http://www.eroprofile.com/auth/auth.php?'
+    _NETRC_MACHINE = 'eroprofile'
+    _TESTS = [{
         'url': 'http://www.eroprofile.com/m/videos/view/sexy-babe-softcore',
         'md5': 'c26f351332edf23e1ea28ce9ec9de32f',
         'info_dict': {
@@ -16,13 +22,55 @@ class EroProfileIE(InfoExtractor):
             'thumbnail': 're:https?://.*\.jpg',
             'age_limit': 18,
         }
-    }
+    }, {
+        'url': 'http://www.eroprofile.com/m/videos/view/Try-It-On-Pee_cut_2-wmv-4shared-com-file-sharing-download-movie-file',
+        'md5': '1baa9602ede46ce904c431f5418d8916',
+        'info_dict': {
+            'id': '1133519',
+            'ext': 'm4v',
+            'title': 'Try It On Pee_cut_2.wmv - 4shared.com - file sharing - download movie file',
+            'thumbnail': 're:https?://.*\.jpg',
+            'age_limit': 18,
+        },
+        'skip': 'Requires login',
+    }]
+
+    def _login(self):
+        (username, password) = self._get_login_info()
+        if username is None:
+            return
+
+        query = compat_urllib_parse.urlencode({
+            'username': username,
+            'password': password,
+            'url': 'http://www.eroprofile.com/',
+        })
+        login_url = self._LOGIN_URL + query
+        login_page = self._download_webpage(login_url, None, False)
+
+        m = re.search(r'Your username or password was incorrect\.', login_page)
+        if m:
+            raise ExtractorError(
+                'Wrong username and/or password.', expected=True)
+
+        self.report_login()
+        redirect_url = self._search_regex(
+            r'<script[^>]+?src="([^"]+)"', login_page, 'login redirect url')
+        self._download_webpage(redirect_url, None, False)
+
+    def _real_initialize(self):
+        self._login()
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
 
         webpage = self._download_webpage(url, display_id)
 
+        m = re.search(r'You must be logged in to view this video\.', webpage)
+        if m:
+            raise ExtractorError(
+                'This video requires login. Please specify a username and password and try again.', expected=True)
+
         video_id = self._search_regex(
             [r"glbUpdViews\s*\('\d*','(\d+)'", r'p/report/video/(\d+)'],
             webpage, 'video id', default=None)

From 70a1165b32acf253905109e9b4f245295d67af1f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 27 Mar 2015 13:02:20 +0100
Subject: [PATCH 0786/6219] Don't use bare 'except:'

They catch any exception, including KeyboardInterrupt, we don't want to catch it.
---
 devscripts/check-porn.py           |  2 +-
 youtube_dl/YoutubeDL.py            |  4 ++--
 youtube_dl/compat.py               |  2 +-
 youtube_dl/downloader/common.py    |  2 +-
 youtube_dl/extractor/youporn.py    |  2 +-
 youtube_dl/postprocessor/ffmpeg.py | 15 ++++++---------
 youtube_dl/update.py               |  4 ++--
 youtube_dl/utils.py                |  4 ++--
 8 files changed, 16 insertions(+), 19 deletions(-)

diff --git a/devscripts/check-porn.py b/devscripts/check-porn.py
index 6a5bd9eda..7a219ebe9 100644
--- a/devscripts/check-porn.py
+++ b/devscripts/check-porn.py
@@ -28,7 +28,7 @@ for test in get_testcases():
     if METHOD == 'EURISTIC':
         try:
             webpage = compat_urllib_request.urlopen(test['url'], timeout=10).read()
-        except:
+        except Exception:
             print('\nFail: {0}'.format(test['name']))
             continue
 
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index b5ef5e009..640b8c99d 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1701,10 +1701,10 @@ class YoutubeDL(object):
             out = out.decode().strip()
             if re.match('[0-9a-f]+', out):
                 self._write_string('[debug] Git HEAD: ' + out + '\n')
-        except:
+        except Exception:
             try:
                 sys.exc_clear()
-            except:
+            except Exception:
                 pass
         self._write_string('[debug] Python version %s - %s\n' % (
             platform.python_version(), platform_name()))
diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index b2bf149ef..973bcd320 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -389,7 +389,7 @@ else:
                 stdout=subprocess.PIPE, stderr=subprocess.PIPE)
             out, err = sp.communicate()
             lines, columns = map(int, out.split())
-        except:
+        except Exception:
             pass
         return _terminal_size(columns, lines)
 
diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index 8ed5c19a6..ca14d64bc 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -204,7 +204,7 @@ class FileDownloader(object):
             return
         try:
             os.utime(filename, (time.time(), filetime))
-        except:
+        except Exception:
             pass
         return filetime
 
diff --git a/youtube_dl/extractor/youporn.py b/youtube_dl/extractor/youporn.py
index e4c855ee0..6abe72f73 100644
--- a/youtube_dl/extractor/youporn.py
+++ b/youtube_dl/extractor/youporn.py
@@ -52,7 +52,7 @@ class YouPornIE(InfoExtractor):
             webpage, 'JSON parameters')
         try:
             params = json.loads(json_params)
-        except:
+        except ValueError:
             raise ExtractorError('Invalid JSON')
 
         self.report_extraction(video_id)
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index b6f51cfd5..55adf9685 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -3,7 +3,6 @@ from __future__ import unicode_literals
 import io
 import os
 import subprocess
-import sys
 import time
 
 
@@ -269,19 +268,17 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
             else:
                 self._downloader.to_screen('[' + self.basename + '] Destination: ' + new_path)
                 self.run_ffmpeg(path, new_path, acodec, more_opts)
-        except:
-            etype, e, tb = sys.exc_info()
-            if isinstance(e, AudioConversionError):
-                msg = 'audio conversion failed: ' + e.msg
-            else:
-                msg = 'error running ' + self.basename
-            raise PostProcessingError(msg)
+        except AudioConversionError as e:
+            raise PostProcessingError(
+                'audio conversion failed: ' + e.msg)
+        except Exception:
+            raise PostProcessingError('error running ' + self.basename)
 
         # Try to update the date time for extracted audio file.
         if information.get('filetime') is not None:
             try:
                 os.utime(encodeFilename(new_path), (time.time(), information['filetime']))
-            except:
+            except Exception:
                 self._downloader.report_warning('Cannot update utime of audio file')
 
         information['filepath'] = new_path
diff --git a/youtube_dl/update.py b/youtube_dl/update.py
index d8be4049f..de3169eef 100644
--- a/youtube_dl/update.py
+++ b/youtube_dl/update.py
@@ -65,7 +65,7 @@ def update_self(to_screen, verbose):
     # Check if there is a new version
     try:
         newversion = opener.open(VERSION_URL).read().decode('utf-8').strip()
-    except:
+    except Exception:
         if verbose:
             to_screen(compat_str(traceback.format_exc()))
         to_screen('ERROR: can\'t find the current version. Please try again later.')
@@ -78,7 +78,7 @@ def update_self(to_screen, verbose):
     try:
         versions_info = opener.open(JSON_URL).read().decode('utf-8')
         versions_info = json.loads(versions_info)
-    except:
+    except Exception:
         if verbose:
             to_screen(compat_str(traceback.format_exc()))
         to_screen('ERROR: can\'t obtain versions info. Please try again later.')
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 245d623d8..90e0ed9ab 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -75,7 +75,7 @@ def preferredencoding():
     try:
         pref = locale.getpreferredencoding()
         'TEST'.encode(pref)
-    except:
+    except Exception:
         pref = 'UTF-8'
 
     return pref
@@ -127,7 +127,7 @@ def write_json_file(obj, fn):
             except OSError:
                 pass
         os.rename(tf.name, fn)
-    except:
+    except Exception:
         try:
             os.remove(tf.name)
         except OSError:

From 8e678af4ba873c348ec2634ab22f84c61546c093 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 27 Mar 2015 14:21:53 +0100
Subject: [PATCH 0787/6219] Makefile: fix 'find' command

It worked with the GNU version, but not with the BSD version.
---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index c6c76274f..fdb1abb60 100644
--- a/Makefile
+++ b/Makefile
@@ -2,7 +2,7 @@ all: youtube-dl README.md CONTRIBUTING.md README.txt youtube-dl.1 youtube-dl.bas
 
 clean:
 	rm -rf youtube-dl.1.temp.md youtube-dl.1 youtube-dl.bash-completion README.txt MANIFEST build/ dist/ .coverage cover/ youtube-dl.tar.gz youtube-dl.zsh youtube-dl.fish *.dump *.part *.info.json *.mp4 *.flv *.mp3 *.avi CONTRIBUTING.md.tmp youtube-dl youtube-dl.exe
-	find -name "*.pyc" -delete
+	find . -name "*.pyc" -delete
 
 PREFIX ?= /usr/local
 BINDIR ?= $(PREFIX)/bin

From c59e701e35dedb40da9d1e88c051141e63ded550 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 28 Mar 2015 08:11:32 +0100
Subject: [PATCH 0788/6219] Default to continuedl=True

We already do this in the CLI interface, so it should be just fine.
---
 youtube_dl/downloader/common.py | 2 +-
 youtube_dl/downloader/http.py   | 2 +-
 youtube_dl/downloader/rtmp.py   | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index ca14d64bc..a0fc5ead0 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -318,7 +318,7 @@ class FileDownloader(object):
         )
 
         continuedl_and_exists = (
-            self.params.get('continuedl', False) and
+            self.params.get('continuedl', True) and
             os.path.isfile(encodeFilename(filename)) and
             not self.params.get('nopart', False)
         )
diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index 4047d7167..d136bebd1 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -49,7 +49,7 @@ class HttpFD(FileDownloader):
 
         open_mode = 'wb'
         if resume_len != 0:
-            if self.params.get('continuedl', False):
+            if self.params.get('continuedl', True):
                 self.report_resuming_byte(resume_len)
                 request.add_header('Range', 'bytes=%d-' % resume_len)
                 open_mode = 'ab'
diff --git a/youtube_dl/downloader/rtmp.py b/youtube_dl/downloader/rtmp.py
index 89e98ae61..ddf5724ae 100644
--- a/youtube_dl/downloader/rtmp.py
+++ b/youtube_dl/downloader/rtmp.py
@@ -105,7 +105,7 @@ class RtmpFD(FileDownloader):
         protocol = info_dict.get('rtmp_protocol', None)
         real_time = info_dict.get('rtmp_real_time', False)
         no_resume = info_dict.get('no_resume', False)
-        continue_dl = info_dict.get('continuedl', False)
+        continue_dl = info_dict.get('continuedl', True)
 
         self.report_destination(filename)
         tmpfilename = self.temp_name(filename)

From 4747e2183acbddea61a3f10fb4ba1ed5b70c8bd1 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 28 Mar 2015 08:12:05 +0100
Subject: [PATCH 0789/6219] release 2015.03.28

---
 docs/supportedsites.md | 7 ++++++-
 youtube_dl/version.py  | 2 +-
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index baf7b3880..fd59cc2be 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -2,6 +2,8 @@
  - **1tv**: Первый канал
  - **1up.com**
  - **220.ro**
+ - **22tracks:genre**
+ - **22tracks:track**
  - **24video**
  - **3sat**
  - **4tube**
@@ -380,6 +382,8 @@
  - **rutube:movie**: Rutube movies
  - **rutube:person**: Rutube person videos
  - **RUTV**: RUTV.RU
+ - **safari**: safaribooksonline.com online video
+ - **safari:course**: safaribooksonline.com online courses
  - **Sandia**: Sandia National Laboratories
  - **Sapo**: SAPO Vídeos
  - **savefrom.net**
@@ -500,6 +504,7 @@
  - **Vbox7**
  - **VeeHD**
  - **Veoh**
+ - **Vessel**
  - **Vesti**: Вести.Ru
  - **Vevo**
  - **VGTV**
@@ -588,7 +593,7 @@
  - **youtube:show**: YouTube.com (multi-season) shows
  - **youtube:subscriptions**: YouTube.com subscriptions feed, "ytsubs" keyword (requires authentication)
  - **youtube:user**: YouTube.com user videos (URL or "ytuser" keyword)
- - **youtube:watch_later**: Youtube watch later list, ":ytwatchlater" for short (requires authentication)
+ - **youtube:watchlater**: Youtube watch later list, ":ytwatchlater" for short (requires authentication)
  - **Zapiks**
  - **ZDF**
  - **ZDFChannel**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 039ceadf2..dd93e295a 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.03.24'
+__version__ = '2015.03.28'

From 643fe72717e6b9c45af4528e46dfc181cad7aebb Mon Sep 17 00:00:00 2001
From: Oskar Jauch <oskar.jauch@gmail.com>
Date: Sat, 28 Mar 2015 10:38:52 +0100
Subject: [PATCH 0790/6219] [DHM] Add new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/dhm.py      | 52 ++++++++++++++++++++++++++++++++
 2 files changed, 53 insertions(+)
 create mode 100644 youtube_dl/extractor/dhm.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index d56eb6448..a65c0c25b 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -106,6 +106,7 @@ from .dbtv import DBTVIE
 from .dctp import DctpTvIE
 from .deezer import DeezerPlaylistIE
 from .dfb import DFBIE
+from .dhm import DHMIE
 from .dotsub import DotsubIE
 from .douyutv import DouyuTVIE
 from .dreisat import DreiSatIE
diff --git a/youtube_dl/extractor/dhm.py b/youtube_dl/extractor/dhm.py
new file mode 100644
index 000000000..d379c9d53
--- /dev/null
+++ b/youtube_dl/extractor/dhm.py
@@ -0,0 +1,52 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+import urllib2
+import xml.etree.ElementTree as ET
+import re
+
+
+class DHMIE(InfoExtractor):
+    _VALID_URL = r'http://www\.dhm\.de/filmarchiv/(?P<id>.*?)'
+
+    _TEST = {
+        'url': 'http://www.dhm.de/filmarchiv/die-filme/the-marshallplan-at-work-in-west-germany/',
+        'md5': '11c475f670209bf6acca0b2b7ef51827',
+        'info_dict': {
+            'id': 'marshallwg',
+            'ext': 'flv',
+            'title': 'MARSHALL PLAN AT WORK IN WESTERN GERMANY, THE',
+            'thumbnail': 'http://www.dhm.de/filmarchiv/video/mpworkwg.jpg',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = ''
+        webpage = self._download_webpage(url, video_id)
+
+        title = self._html_search_regex(
+            r'dc:title=\"(.*?)\"', webpage, 'title')
+
+        playlist_url = self._html_search_regex(
+            r'file: \'(.*?)\'', webpage, 'playlist URL')
+
+        xml_file = urllib2.urlopen(playlist_url)
+        data = xml_file.read()
+        xml_file.close()
+
+        root = ET.fromstring(data)
+        video_url = root[0][0][0].text
+        thumbnail = root[0][0][2].text
+
+        m = re.search('video/(.+?).flv', video_url)
+        if m:
+            video_id = m.group(1)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'url': video_url,
+            'thumbnail': thumbnail,
+        }

From ff79552f13efbfa87a814dd0a950aa797b08a5cd Mon Sep 17 00:00:00 2001
From: Oskar Jauch <oskar.jauch@gmail.com>
Date: Sat, 28 Mar 2015 10:42:35 +0100
Subject: [PATCH 0791/6219] [DHM] Add extractor description

---
 youtube_dl/extractor/dhm.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/dhm.py b/youtube_dl/extractor/dhm.py
index d379c9d53..a0a584f6a 100644
--- a/youtube_dl/extractor/dhm.py
+++ b/youtube_dl/extractor/dhm.py
@@ -9,6 +9,7 @@ import re
 
 
 class DHMIE(InfoExtractor):
+    IE_DESC = 'Deutsches Historisches Museum'
     _VALID_URL = r'http://www\.dhm\.de/filmarchiv/(?P<id>.*?)'
 
     _TEST = {

From cb88671e37a9fbb964ee0b2ab46031195f2aab1e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 28 Mar 2015 14:18:11 +0100
Subject: [PATCH 0792/6219] [nbc] Recognize https urls (fixes #5300)

---
 youtube_dl/extractor/nbc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index 3645d3033..80a01c778 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -14,7 +14,7 @@ from ..utils import (
 
 
 class NBCIE(InfoExtractor):
-    _VALID_URL = r'http://www\.nbc\.com/(?:[^/]+/)+(?P<id>n?\d+)'
+    _VALID_URL = r'https?://www\.nbc\.com/(?:[^/]+/)+(?P<id>n?\d+)'
 
     _TESTS = [
         {

From af8c93086c45cf61cadc8571644713927659a65e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 28 Mar 2015 22:30:13 +0600
Subject: [PATCH 0793/6219] [dhm] Simplify

---
 youtube_dl/extractor/dhm.py | 59 ++++++++++++++++++++++---------------
 1 file changed, 35 insertions(+), 24 deletions(-)

diff --git a/youtube_dl/extractor/dhm.py b/youtube_dl/extractor/dhm.py
index a0a584f6a..80ee40018 100644
--- a/youtube_dl/extractor/dhm.py
+++ b/youtube_dl/extractor/dhm.py
@@ -1,53 +1,64 @@
-# coding: utf-8
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-
-import urllib2
-import xml.etree.ElementTree as ET
-import re
+from ..utils import (
+    xpath_text,
+    parse_duration,
+)
 
 
 class DHMIE(InfoExtractor):
-    IE_DESC = 'Deutsches Historisches Museum'
-    _VALID_URL = r'http://www\.dhm\.de/filmarchiv/(?P<id>.*?)'
+    IE_DESC = 'Filmarchiv - Deutsches Historisches Museum'
+    _VALID_URL = r'http://www\.dhm\.de/filmarchiv/die-filme/(?P<id>[^/]+)'
 
     _TEST = {
         'url': 'http://www.dhm.de/filmarchiv/die-filme/the-marshallplan-at-work-in-west-germany/',
         'md5': '11c475f670209bf6acca0b2b7ef51827',
         'info_dict': {
-            'id': 'marshallwg',
+            'id': 'the-marshallplan-at-work-in-west-germany',
             'ext': 'flv',
             'title': 'MARSHALL PLAN AT WORK IN WESTERN GERMANY, THE',
-            'thumbnail': 'http://www.dhm.de/filmarchiv/video/mpworkwg.jpg',
+            'description': 'md5:1fabd480c153f97b07add61c44407c82',
+            'duration': 660,
+            'thumbnail': 're:^https?://.*\.jpg$',
         }
     }
 
     def _real_extract(self, url):
-        video_id = ''
+        video_id = self._match_id(url)
+
         webpage = self._download_webpage(url, video_id)
 
-        title = self._html_search_regex(
-            r'dc:title=\"(.*?)\"', webpage, 'title')
+        playlist_url = self._search_regex(
+            r"file\s*:\s*'([^']+)'", webpage, 'playlist url')
 
-        playlist_url = self._html_search_regex(
-            r'file: \'(.*?)\'', webpage, 'playlist URL')
+        playlist = self._download_xml(playlist_url, video_id)
 
-        xml_file = urllib2.urlopen(playlist_url)
-        data = xml_file.read()
-        xml_file.close()
+        track = playlist.find(
+            './{http://xspf.org/ns/0/}trackList/{http://xspf.org/ns/0/}track')
 
-        root = ET.fromstring(data)
-        video_url = root[0][0][0].text
-        thumbnail = root[0][0][2].text
+        video_url = xpath_text(
+            track, './{http://xspf.org/ns/0/}location',
+            'video url', fatal=True)
+        thumbnail = xpath_text(
+            track, './{http://xspf.org/ns/0/}image',
+            'thumbnail')
 
-        m = re.search('video/(.+?).flv', video_url)
-        if m:
-            video_id = m.group(1)
+        title = self._search_regex(
+            [r'dc:title="([^"]+)"', r'<title> &raquo;([^<]+)</title>'],
+            webpage, 'title').strip()
+        description = self._html_search_regex(
+            r'<p><strong>Description:</strong>(.+?)</p>',
+            webpage, 'description', fatal=False)
+        duration = parse_duration(self._search_regex(
+            r'<em>Length\s*</em>\s*:\s*</strong>([^<]+)',
+            webpage, 'duration', fatal=False))
 
         return {
             'id': video_id,
-            'title': title,
             'url': video_url,
+            'title': title,
+            'description': description,
+            'duration': duration,
             'thumbnail': thumbnail,
         }

From b7a2268e7b52fbedd1630ad101460d76cca9dcdd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 28 Mar 2015 23:43:15 +0600
Subject: [PATCH 0794/6219] Credit @ossi96 for dhm (#5305)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 59f1b5f21..b36086448 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -118,3 +118,4 @@ Devin J. Pohly
 Eduardo Ferro Aldama
 Jeff Buchbinder
 Amish Bhadeshia
+Oskar Jauch

From 5a3b315b5fcc1b1e153baeaed669e64d137d4047 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 28 Mar 2015 23:55:15 +0600
Subject: [PATCH 0795/6219] [dhm] Improve _VALID_URL and add test

---
 youtube_dl/extractor/dhm.py | 21 +++++++++++++++------
 1 file changed, 15 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/dhm.py b/youtube_dl/extractor/dhm.py
index 80ee40018..3ed1f1663 100644
--- a/youtube_dl/extractor/dhm.py
+++ b/youtube_dl/extractor/dhm.py
@@ -9,9 +9,9 @@ from ..utils import (
 
 class DHMIE(InfoExtractor):
     IE_DESC = 'Filmarchiv - Deutsches Historisches Museum'
-    _VALID_URL = r'http://www\.dhm\.de/filmarchiv/die-filme/(?P<id>[^/]+)'
+    _VALID_URL = r'https?://(?:www\.)?dhm\.de/filmarchiv/(?:[^/]+/)+(?P<id>[^/]+)'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.dhm.de/filmarchiv/die-filme/the-marshallplan-at-work-in-west-germany/',
         'md5': '11c475f670209bf6acca0b2b7ef51827',
         'info_dict': {
@@ -21,8 +21,17 @@ class DHMIE(InfoExtractor):
             'description': 'md5:1fabd480c153f97b07add61c44407c82',
             'duration': 660,
             'thumbnail': 're:^https?://.*\.jpg$',
-        }
-    }
+        },
+    }, {
+        'url': 'http://www.dhm.de/filmarchiv/02-mapping-the-wall/peter-g/rolle-1/',
+        'md5': '09890226332476a3e3f6f2cb74734aa5',
+        'info_dict': {
+            'id': 'rolle-1',
+            'ext': 'flv',
+            'title': 'ROLLE 1',
+            'thumbnail': 're:^https?://.*\.jpg$',
+        },
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -49,10 +58,10 @@ class DHMIE(InfoExtractor):
             webpage, 'title').strip()
         description = self._html_search_regex(
             r'<p><strong>Description:</strong>(.+?)</p>',
-            webpage, 'description', fatal=False)
+            webpage, 'description', default=None)
         duration = parse_duration(self._search_regex(
             r'<em>Length\s*</em>\s*:\s*</strong>([^<]+)',
-            webpage, 'duration', fatal=False))
+            webpage, 'duration', default=None))
 
         return {
             'id': video_id,

From 616af2f4b9458dd15152ddf6a7905f7818ae583a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 29 Mar 2015 00:03:59 +0600
Subject: [PATCH 0796/6219] Unduplicate @ossi96

---
 AUTHORS | 1 -
 1 file changed, 1 deletion(-)

diff --git a/AUTHORS b/AUTHORS
index b36086448..59f1b5f21 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -118,4 +118,3 @@ Devin J. Pohly
 Eduardo Ferro Aldama
 Jeff Buchbinder
 Amish Bhadeshia
-Oskar Jauch

From 5f88e028183896f67e35d2d82d9596213cd2c73e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 28 Mar 2015 23:35:55 +0100
Subject: [PATCH 0797/6219] [ultimedia] PEP8

---
 youtube_dl/extractor/ultimedia.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/ultimedia.py b/youtube_dl/extractor/ultimedia.py
index 06554a1be..96c809eaf 100644
--- a/youtube_dl/extractor/ultimedia.py
+++ b/youtube_dl/extractor/ultimedia.py
@@ -42,7 +42,6 @@ class UltimediaIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-
         webpage = self._download_webpage(url, video_id)
 
         deliver_url = self._search_regex(
@@ -81,8 +80,8 @@ class UltimediaIE(InfoExtractor):
         title = clean_html((
             self._html_search_regex(
                 r'(?s)<div\s+id="catArticle">.+?</div>(.+?)</h1>',
-                webpage, 'title', default=None)
-            or self._search_regex(
+                webpage, 'title', default=None) or
+            self._search_regex(
                 r"var\s+nameVideo\s*=\s*'([^']+)'",
                 deliver_page, 'title')))
 

From ecb750a446dd4805467904864b99eab705e866a8 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 28 Mar 2015 23:39:41 +0100
Subject: [PATCH 0798/6219] [cnn] Match more URLs

---
 youtube_dl/extractor/cnn.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/cnn.py b/youtube_dl/extractor/cnn.py
index 90ea07438..0a77e951c 100644
--- a/youtube_dl/extractor/cnn.py
+++ b/youtube_dl/extractor/cnn.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 class CNNIE(InfoExtractor):
     _VALID_URL = r'''(?x)https?://(?:(?:edition|www)\.)?cnn\.com/video/(?:data/.+?|\?)/
-        (?P<path>.+?/(?P<title>[^/]+?)(?:\.(?:cnn|hln)(?:-ap)?|(?=&)))'''
+        (?P<path>.+?/(?P<title>[^/]+?)(?:\.(?:cnn|hln|ktvk)(?:-ap)?|(?=&)))'''
 
     _TESTS = [{
         'url': 'http://edition.cnn.com/video/?/video/sports/2013/06/09/nadal-1-on-1.cnn',
@@ -45,6 +45,9 @@ class CNNIE(InfoExtractor):
             'description': 'md5:e7223a503315c9f150acac52e76de086',
             'upload_date': '20141222',
         }
+    }, {
+        'url': 'http://cnn.com/video/?/video/politics/2015/03/27/pkg-arizona-senator-church-attendance-mandatory.ktvk',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From ad320e9b8376221a3eda935a358886b6e7ab7bf6 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Sun, 29 Mar 2015 04:57:37 +0300
Subject: [PATCH 0799/6219] [generic] Add support for 5min embeds (#5310)

---
 youtube_dl/extractor/generic.py | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 8a49b0b54..042d23a13 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -620,6 +620,16 @@ class GenericIE(InfoExtractor):
                 'age_limit': 0,
             },
         },
+        # 5min embed
+        {
+            'url': 'http://techcrunch.com/video/facebook-creates-on-this-day-crunch-report/518726732/',
+            'md5': '4c6f127a30736b59b3e2c19234ee2bf7',
+            'info_dict': {
+                'id': '518726732',
+                'ext': 'mp4',
+                'title': 'Facebook Creates "On This Day" | Crunch Report',
+            },
+        },
         # RSS feed with enclosure
         {
             'url': 'http://podcastfeeds.nbcnews.com/audio/podcast/MSNBC-MADDOW-NETCAST-M4V.xml',
@@ -1236,6 +1246,12 @@ class GenericIE(InfoExtractor):
         if mobj is not None:
             return self.url_result(mobj.group('url'), 'Pladform')
 
+        # Look for 5min embeds
+        mobj = re.search(
+            r'<meta[^>]+property="og:video"[^>]+content="https?://embed\.5min\.com/(?P<id>[0-9]+)/?', webpage)
+        if mobj is not None:
+            return self.url_result('5min:%s' % mobj.group('id'), 'FiveMin')
+
         def check_video(vurl):
             if YoutubeIE.suitable(vurl):
                 return True

From 8343a03357ec4edb49b29bc841f2410ee4b610d1 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 29 Mar 2015 14:26:28 +0800
Subject: [PATCH 0800/6219] [douyutv] Fix extractor and improve error handling

---
 youtube_dl/extractor/douyutv.py | 57 ++++++++++++++++++++++++++-------
 1 file changed, 46 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/douyutv.py b/youtube_dl/extractor/douyutv.py
index d7956e6e4..3a1665ab4 100644
--- a/youtube_dl/extractor/douyutv.py
+++ b/youtube_dl/extractor/douyutv.py
@@ -1,19 +1,23 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import hashlib
+import time
 from .common import InfoExtractor
-from ..utils import ExtractorError
+from ..utils import (ExtractorError, unescapeHTML)
+from ..compat import (compat_str, compat_basestring)
 
 
 class DouyuTVIE(InfoExtractor):
     _VALID_URL = r'http://(?:www\.)?douyutv\.com/(?P<id>[A-Za-z0-9]+)'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.douyutv.com/iseven',
         'info_dict': {
-            'id': 'iseven',
+            'id': '17732',
+            'display_id': 'iseven',
             'ext': 'flv',
             'title': 're:^清晨醒脑！T-ara根本停不下来！ [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
-            'description': 'md5:9e525642c25a0a24302869937cf69d17',
+            'description': 'md5:c93d6692dde6fe33809a46edcbecca44',
             'thumbnail': 're:^https?://.*\.jpg$',
             'uploader': '7师傅',
             'uploader_id': '431925',
@@ -22,22 +26,52 @@ class DouyuTVIE(InfoExtractor):
         'params': {
             'skip_download': True,
         }
-    }
+    }, {
+        'url': 'http://www.douyutv.com/85982',
+        'info_dict': {
+            'id': '85982',
+            'display_id': '85982',
+            'ext': 'flv',
+            'title': 're:^小漠从零单排记！——CSOL2躲猫猫 [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
+            'description': 'md5:746a2f7a253966a06755a912f0acc0d2',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'uploader': 'douyu小漠',
+            'uploader_id': '3769985',
+            'is_live': True,
+        },
+        'params': {
+            'skip_download': True,
+        }
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
+        if video_id.isdigit():
+            room_id = video_id
+        else:
+            page = self._download_webpage(url, video_id)
+            room_id = self._html_search_regex(
+                r'"room_id"\s*:\s*(\d+),', page, 'room id')
+
+        prefix = 'room/%s?aid=android&client_sys=android&time=%d' % (
+            room_id, int(time.time()))
+
+        auth = hashlib.md5((prefix + '1231').encode('ascii')).hexdigest()
         config = self._download_json(
-            'http://www.douyutv.com/api/client/room/%s' % video_id, video_id)
+            'http://www.douyutv.com/api/v1/%s&auth=%s' % (prefix, auth),
+            video_id)
 
         data = config['data']
 
         error_code = config.get('error', 0)
-        show_status = data.get('show_status')
         if error_code is not 0:
-            raise ExtractorError(
-                'Server reported error %i' % error_code, expected=True)
+            error_desc = 'Server reported error %i' % error_code
+            if isinstance(data, compat_str) or isinstance(data, compat_basestring):
+                error_desc += ': ' + data
+            raise ExtractorError(error_desc, expected=True)
 
+        show_status = data.get('show_status')
         # 1 = live, 2 = offline
         if show_status == '2':
             raise ExtractorError(
@@ -46,7 +80,7 @@ class DouyuTVIE(InfoExtractor):
         base_url = data['rtmp_url']
         live_path = data['rtmp_live']
 
-        title = self._live_title(data['room_name'])
+        title = self._live_title(unescapeHTML(data['room_name']))
         description = data.get('show_details')
         thumbnail = data.get('room_src')
 
@@ -66,7 +100,8 @@ class DouyuTVIE(InfoExtractor):
         self._sort_formats(formats)
 
         return {
-            'id': video_id,
+            'id': room_id,
+            'display_id': video_id,
             'title': title,
             'description': description,
             'thumbnail': thumbnail,

From 2ddf08358861a3c0f8724020c4eaf119e6be21a2 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 29 Mar 2015 18:17:48 +0800
Subject: [PATCH 0801/6219] [douyutv] Simplify usage of isinstance

---
 youtube_dl/extractor/douyutv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/douyutv.py b/youtube_dl/extractor/douyutv.py
index 3a1665ab4..479430c51 100644
--- a/youtube_dl/extractor/douyutv.py
+++ b/youtube_dl/extractor/douyutv.py
@@ -67,7 +67,7 @@ class DouyuTVIE(InfoExtractor):
         error_code = config.get('error', 0)
         if error_code is not 0:
             error_desc = 'Server reported error %i' % error_code
-            if isinstance(data, compat_str) or isinstance(data, compat_basestring):
+            if isinstance(data, (compat_str, compat_basestring)):
                 error_desc += ': ' + data
             raise ExtractorError(error_desc, expected=True)
 

From 9b4774b21bb26eea9eb8042b2ff248a176595c5a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 29 Mar 2015 20:40:05 +0800
Subject: [PATCH 0802/6219] [Xuite] Fix extraction on python 3.2

base64.b64decode() accept only binary types in Python 3.2
---
 youtube_dl/extractor/xuite.py | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/xuite.py b/youtube_dl/extractor/xuite.py
index 4971965f9..81d885fdc 100644
--- a/youtube_dl/extractor/xuite.py
+++ b/youtube_dl/extractor/xuite.py
@@ -69,18 +69,26 @@ class XuiteIE(InfoExtractor):
         'only_matching': True,
     }]
 
+    @staticmethod
+    def base64_decode_utf8(data):
+        return base64.b64decode(data.encode('utf-8')).decode('utf-8')
+
+    @staticmethod
+    def base64_encode_utf8(data):
+        return base64.b64encode(data.encode('utf-8')).decode('utf-8')
+
     def _extract_flv_config(self, media_id):
-        base64_media_id = base64.b64encode(media_id.encode('utf-8')).decode('utf-8')
+        base64_media_id = self.base64_encode_utf8(media_id)
         flv_config = self._download_xml(
             'http://vlog.xuite.net/flash/player?media=%s' % base64_media_id,
             'flv config')
         prop_dict = {}
         for prop in flv_config.findall('./property'):
-            prop_id = base64.b64decode(prop.attrib['id']).decode('utf-8')
+            prop_id = self.base64_decode_utf8(prop.attrib['id'])
             # CDATA may be empty in flv config
             if not prop.text:
                 continue
-            encoded_content = base64.b64decode(prop.text).decode('utf-8')
+            encoded_content = self.base64_decode_utf8(prop.text)
             prop_dict[prop_id] = compat_urllib_parse_unquote(encoded_content)
         return prop_dict
 

From 4d5d14f5cf096f4bd90ad373cad687dc82bf8a8f Mon Sep 17 00:00:00 2001
From: Joram Schrijver <i@joram.io>
Date: Sun, 29 Mar 2015 23:41:06 +0200
Subject: [PATCH 0803/6219] [Dumpert] Add new extractor

Add support for the Dutch video site Dumpert. http://www.dumpert.nl/
---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/dumpert.py  | 47 ++++++++++++++++++++++++++++++++
 2 files changed, 48 insertions(+)
 create mode 100644 youtube_dl/extractor/dumpert.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index a65c0c25b..43bac0252 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -115,6 +115,7 @@ from .drtuber import DrTuberIE
 from .drtv import DRTVIE
 from .dvtv import DVTVIE
 from .dump import DumpIE
+from .dumpert import DumpertIE
 from .defense import DefenseGouvFrIE
 from .discovery import DiscoveryIE
 from .divxstage import DivxStageIE
diff --git a/youtube_dl/extractor/dumpert.py b/youtube_dl/extractor/dumpert.py
new file mode 100644
index 000000000..52d07deac
--- /dev/null
+++ b/youtube_dl/extractor/dumpert.py
@@ -0,0 +1,47 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import base64
+
+from .common import InfoExtractor
+
+
+class DumpertIE(InfoExtractor):
+    _VALID_URL = (r'https?://(?:www\.)?dumpert\.nl/mediabase/'
+                  r'(?P<id>[0-9]+/[0-9a-zA-Z]+)/?.*')
+    _TEST = {
+        'url': 'http://www.dumpert.nl/mediabase/6646981/951bc60f/',
+        'md5': '1b9318d7d5054e7dcb9dc7654f21d643',
+        'info_dict': {
+            'id': '6646981/951bc60f',
+            'ext': 'mp4',
+            'title': 'Ik heb nieuws voor je',
+            'description': 'Niet schrikken hoor'
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        title = self._html_search_meta('title', webpage)
+        description = self._html_search_meta('description', webpage)
+
+        files_base64 = self._html_search_regex(r'data-files="(.*?)"',
+                                               webpage,
+                                               'files')
+        files_json = base64.b64decode(files_base64).decode('iso-8859-1')
+        files = self._parse_json(files_json, video_id)
+
+        format_names = ['flv', 'mobile', 'tablet', '720p']
+        formats = [{'format_id': name,
+                    'url': files[name].replace(r'\/', '/')}
+                   for name in format_names
+                   if name in files]
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'formats': formats
+        }

From 7700207ec7d39e1594d9963a5014ddcb30c7301a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 30 Mar 2015 19:41:04 +0600
Subject: [PATCH 0804/6219] [pornhub] Fix comment count extraction (Closes
 #5320)

---
 youtube_dl/extractor/pornhub.py | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index 3a27e3789..0c8b731cf 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -33,10 +33,8 @@ class PornHubIE(InfoExtractor):
     }
 
     def _extract_count(self, pattern, webpage, name):
-        count = self._html_search_regex(pattern, webpage, '%s count' % name, fatal=False)
-        if count:
-            count = str_to_int(count)
-        return count
+        return str_to_int(self._search_regex(
+            pattern, webpage, '%s count' % name, fatal=False))
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -62,11 +60,14 @@ class PornHubIE(InfoExtractor):
         if thumbnail:
             thumbnail = compat_urllib_parse.unquote(thumbnail)
 
-        view_count = self._extract_count(r'<span class="count">([\d,\.]+)</span> views', webpage, 'view')
-        like_count = self._extract_count(r'<span class="votesUp">([\d,\.]+)</span>', webpage, 'like')
-        dislike_count = self._extract_count(r'<span class="votesDown">([\d,\.]+)</span>', webpage, 'dislike')
+        view_count = self._extract_count(
+            r'<span class="count">([\d,\.]+)</span> views', webpage, 'view')
+        like_count = self._extract_count(
+            r'<span class="votesUp">([\d,\.]+)</span>', webpage, 'like')
+        dislike_count = self._extract_count(
+            r'<span class="votesDown">([\d,\.]+)</span>', webpage, 'dislike')
         comment_count = self._extract_count(
-            r'All comments \(<var class="videoCommentCount">([\d,\.]+)</var>', webpage, 'comment')
+            r'All Comments\s*<span>\(([\d,.]+)\)', webpage, 'comment')
 
         video_urls = list(map(compat_urllib_parse.unquote, re.findall(r'"quality_[0-9]{3}p":"([^"]+)', webpage)))
         if webpage.find('"encrypted":true') != -1:

From 87270c84166bb014ba1043a6eeee7330694b0649 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 30 Mar 2015 20:11:51 +0600
Subject: [PATCH 0805/6219] [dumpert] Simplify and fix python 3.2

---
 youtube_dl/extractor/dumpert.py | 39 ++++++++++++++++++++-------------
 1 file changed, 24 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/dumpert.py b/youtube_dl/extractor/dumpert.py
index 52d07deac..e43bc81b2 100644
--- a/youtube_dl/extractor/dumpert.py
+++ b/youtube_dl/extractor/dumpert.py
@@ -4,11 +4,11 @@ from __future__ import unicode_literals
 import base64
 
 from .common import InfoExtractor
+from ..utils import qualities
 
 
 class DumpertIE(InfoExtractor):
-    _VALID_URL = (r'https?://(?:www\.)?dumpert\.nl/mediabase/'
-                  r'(?P<id>[0-9]+/[0-9a-zA-Z]+)/?.*')
+    _VALID_URL = r'https?://(?:www\.)?dumpert\.nl/mediabase/(?P<id>[0-9]+/[0-9a-zA-Z]+)'
     _TEST = {
         'url': 'http://www.dumpert.nl/mediabase/6646981/951bc60f/',
         'md5': '1b9318d7d5054e7dcb9dc7654f21d643',
@@ -16,7 +16,8 @@ class DumpertIE(InfoExtractor):
             'id': '6646981/951bc60f',
             'ext': 'mp4',
             'title': 'Ik heb nieuws voor je',
-            'description': 'Niet schrikken hoor'
+            'description': 'Niet schrikken hoor',
+            'thumbnail': 're:^https?://.*\.jpg$',
         }
     }
 
@@ -24,24 +25,32 @@ class DumpertIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        title = self._html_search_meta('title', webpage)
-        description = self._html_search_meta('description', webpage)
+        files_base64 = self._search_regex(
+            r'data-files="([^"]+)"', webpage, 'data files')
 
-        files_base64 = self._html_search_regex(r'data-files="(.*?)"',
-                                               webpage,
-                                               'files')
-        files_json = base64.b64decode(files_base64).decode('iso-8859-1')
-        files = self._parse_json(files_json, video_id)
+        files = self._parse_json(
+            base64.b64decode(files_base64.encode('utf-8')).decode('utf-8'),
+            video_id)
 
-        format_names = ['flv', 'mobile', 'tablet', '720p']
-        formats = [{'format_id': name,
-                    'url': files[name].replace(r'\/', '/')}
-                   for name in format_names
-                   if name in files]
+        quality = qualities(['flv', 'mobile', 'tablet', '720p'])
+
+        formats = [{
+            'url': video_url,
+            'format_id': format_id,
+            'quality': quality(format_id),
+        } for format_id, video_url in files.items() if format_id != 'still']
+        self._sort_formats(formats)
+
+        title = self._html_search_meta(
+            'title', webpage) or self._og_search_title(webpage)
+        description = self._html_search_meta(
+            'description', webpage) or self._og_search_description(webpage)
+        thumbnail = files.get('still') or self._og_search_thumbnail(webpage)
 
         return {
             'id': video_id,
             'title': title,
             'description': description,
+            'thumbnail': thumbnail,
             'formats': formats
         }

From fd203fe35774f6e8a5ce17ad78650089d2536c97 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 30 Mar 2015 20:12:55 +0600
Subject: [PATCH 0806/6219] Credit @jorams or dumpert.nl (#5319)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 59f1b5f21..48be31e29 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -118,3 +118,4 @@ Devin J. Pohly
 Eduardo Ferro Aldama
 Jeff Buchbinder
 Amish Bhadeshia
+Joram Schrijver

From c808ef81bb67f737b89671ce882abfca666e0139 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 30 Mar 2015 21:03:38 +0600
Subject: [PATCH 0807/6219] [soundcloud:set:user] Support mobile URLs (Closes
 #5323)

---
 youtube_dl/extractor/soundcloud.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 9d4505972..316b2c90f 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -242,7 +242,7 @@ class SoundcloudIE(InfoExtractor):
 
 
 class SoundcloudSetIE(SoundcloudIE):
-    _VALID_URL = r'https?://(?:www\.)?soundcloud\.com/(?P<uploader>[\w\d-]+)/sets/(?P<slug_title>[\w\d-]+)(?:/(?P<token>[^?/]+))?'
+    _VALID_URL = r'https?://(?:(?:www|m)\.)?soundcloud\.com/(?P<uploader>[\w\d-]+)/sets/(?P<slug_title>[\w\d-]+)(?:/(?P<token>[^?/]+))?'
     IE_NAME = 'soundcloud:set'
     _TESTS = [{
         'url': 'https://soundcloud.com/the-concept-band/sets/the-royal-concept-ep',
@@ -287,7 +287,7 @@ class SoundcloudSetIE(SoundcloudIE):
 
 
 class SoundcloudUserIE(SoundcloudIE):
-    _VALID_URL = r'https?://(www\.)?soundcloud\.com/(?P<user>[^/]+)/?((?P<rsrc>tracks|likes)/?)?(\?.*)?$'
+    _VALID_URL = r'https?://(?:(?:www|m)\.)?soundcloud\.com/(?P<user>[^/]+)/?((?P<rsrc>tracks|likes)/?)?(\?.*)?$'
     IE_NAME = 'soundcloud:user'
     _TESTS = [{
         'url': 'https://soundcloud.com/the-concept-band',

From edd7344820303e550c9daae1ef591be8068ac47e Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Mon, 30 Mar 2015 18:15:08 +0300
Subject: [PATCH 0808/6219] [phoenix] Extend _VALID_URL (#5322)

---
 youtube_dl/extractor/phoenix.py | 40 ++++++++++++++++++++++-----------
 1 file changed, 27 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/phoenix.py b/youtube_dl/extractor/phoenix.py
index a20672c0c..46cebc0d7 100644
--- a/youtube_dl/extractor/phoenix.py
+++ b/youtube_dl/extractor/phoenix.py
@@ -5,19 +5,33 @@ from .zdf import extract_from_xml_url
 
 
 class PhoenixIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?phoenix\.de/content/(?P<id>[0-9]+)'
-    _TEST = {
-        'url': 'http://www.phoenix.de/content/884301',
-        'md5': 'ed249f045256150c92e72dbb70eadec6',
-        'info_dict': {
-            'id': '884301',
-            'ext': 'mp4',
-            'title': 'Michael Krons mit Hans-Werner Sinn',
-            'description': 'Im Dialog - Sa. 25.10.14, 00.00 - 00.35 Uhr',
-            'upload_date': '20141025',
-            'uploader': 'Im Dialog',
-        }
-    }
+    _VALID_URL = r'''(?x)https?://(?:www\.)?phoenix\.de/content/
+        (?:
+            phoenix/die_sendungen/(?:[^/]+/)?
+        )?
+        (?P<id>[0-9]+)'''
+    _TESTS = [
+        {
+            'url': 'http://www.phoenix.de/content/884301',
+            'md5': 'ed249f045256150c92e72dbb70eadec6',
+            'info_dict': {
+                'id': '884301',
+                'ext': 'mp4',
+                'title': 'Michael Krons mit Hans-Werner Sinn',
+                'description': 'Im Dialog - Sa. 25.10.14, 00.00 - 00.35 Uhr',
+                'upload_date': '20141025',
+                'uploader': 'Im Dialog',
+            }
+        },
+        {
+            'url': 'http://www.phoenix.de/content/phoenix/die_sendungen/869815',
+            'only_matching': True,
+        },
+        {
+            'url': 'http://www.phoenix.de/content/phoenix/die_sendungen/diskussionen/928234',
+            'only_matching': True,
+        },
+    ]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From a28ccbabc60c81016c851ae46365be377ea83795 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 31 Mar 2015 02:21:27 +0800
Subject: [PATCH 0809/6219] [Yahoo/NBCSports] Fix #5226

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/nbc.py      | 21 +++++++++++++++++++++
 youtube_dl/extractor/yahoo.py    | 14 ++++++++++++++
 3 files changed, 36 insertions(+)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 43bac0252..5d0d2a9bc 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -310,6 +310,7 @@ from .naver import NaverIE
 from .nba import NBAIE
 from .nbc import (
     NBCIE,
+    NBCSportsIE,
     NBCNewsIE,
 )
 from .ndr import NDRIE
diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index 80a01c778..033bf71f0 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -50,6 +50,27 @@ class NBCIE(InfoExtractor):
         return self.url_result(theplatform_url)
 
 
+class NBCSportsIE(InfoExtractor):
+    _VALID_URL = r'https?://vplayer\.nbcsports\.com/(?:[^/]+/)+(?P<id>[0-9a-zA-Z]+)'
+
+    _TEST = {
+        'url': 'https://vplayer.nbcsports.com/p/BxmELC/nbcsports_share/select/9CsDKds0kvHI',
+        'md5': 'ceae8dced5c14a1c1ffcb7a32194cca5',
+        'info_dict': {
+            'id': '9CsDKds0kvHI',
+            'ext': 'flv',
+            'description': 'md5:df390f70a9ba7c95ff1daace988f0d8d',
+            'title': 'Tyler Kalinoski hits buzzer-beater to lift Davidson',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        theplatform_url = self._og_search_video_url(webpage)
+        return self.url_result(theplatform_url, 'ThePlatform')
+
+
 class NBCNewsIE(InfoExtractor):
     _VALID_URL = r'''(?x)https?://(?:www\.)?nbcnews\.com/
         (?:video/.+?/(?P<id>\d+)|
diff --git a/youtube_dl/extractor/yahoo.py b/youtube_dl/extractor/yahoo.py
index 97dbac4cc..6e72f1e55 100644
--- a/youtube_dl/extractor/yahoo.py
+++ b/youtube_dl/extractor/yahoo.py
@@ -129,6 +129,15 @@ class YahooIE(InfoExtractor):
         }, {
             'url': 'https://gma.yahoo.com/pizza-delivery-man-surprised-huge-tip-college-kids-195200785.html',
             'only_matching': True,
+        }, {
+            'note': 'NBC Sports embeds',
+            'url': 'http://sports.yahoo.com/blogs/ncaab-the-dagger/tyler-kalinoski-s-buzzer-beater-caps-davidson-s-comeback-win-185609842.html?guid=nbc_cbk_davidsonbuzzerbeater_150313',
+            'info_dict': {
+                'id': '9CsDKds0kvHI',
+                'ext': 'flv',
+                'description': 'md5:df390f70a9ba7c95ff1daace988f0d8d',
+                'title': 'Tyler Kalinoski hits buzzer-beater to lift Davidson',
+            }
         }
     ]
 
@@ -151,6 +160,11 @@ class YahooIE(InfoExtractor):
                 items = json.loads(items_json)
                 video_id = items[0]['id']
                 return self._get_info(video_id, display_id, webpage)
+        # Look for NBCSports iframes
+        iframe_m = re.search(
+            r'<iframe[^>]+src="(?P<url>https?://vplayer\.nbcsports\.com/[^"]+)"', webpage)
+        if iframe_m:
+            return self.url_result(iframe_m.group('url'), 'NBCSports')
 
         items_json = self._search_regex(
             r'mediaItems: ({.*?})$', webpage, 'items', flags=re.MULTILINE,

From a2a4d5fa313d5244d24fa70d5db91971a7583d79 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 31 Mar 2015 02:47:18 +0800
Subject: [PATCH 0810/6219] [Yahoo/NBCSports] Generalize NBC sports info
 extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/nbc.py      | 31 ++++++++++++++++++++++++++++++-
 youtube_dl/extractor/yahoo.py    | 10 ++++++----
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 5d0d2a9bc..b113aaec6 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -310,6 +310,7 @@ from .naver import NaverIE
 from .nba import NBAIE
 from .nbc import (
     NBCIE,
+    NBCSportsVPlayerIE,
     NBCSportsIE,
     NBCNewsIE,
 )
diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index 033bf71f0..c8dd72ab4 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -50,7 +50,7 @@ class NBCIE(InfoExtractor):
         return self.url_result(theplatform_url)
 
 
-class NBCSportsIE(InfoExtractor):
+class NBCSportsVPlayerIE(InfoExtractor):
     _VALID_URL = r'https?://vplayer\.nbcsports\.com/(?:[^/]+/)+(?P<id>[0-9a-zA-Z]+)'
 
     _TEST = {
@@ -64,6 +64,13 @@ class NBCSportsIE(InfoExtractor):
         }
     }
 
+    @staticmethod
+    def _extract_url(webpage):
+        iframe_m = re.search(
+            r'<iframe[^>]+src="(?P<url>https?://vplayer\.nbcsports\.com/[^"]+)"', webpage)
+        if iframe_m:
+            return iframe_m.group('url')
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
@@ -71,6 +78,28 @@ class NBCSportsIE(InfoExtractor):
         return self.url_result(theplatform_url, 'ThePlatform')
 
 
+class NBCSportsIE(InfoExtractor):
+    # Does not include https becuase its certificate is invalid
+    _VALID_URL = r'http://www\.nbcsports\.com//?(?:[^/]+/)+(?P<id>[0-9a-z-]+)'
+
+    _TEST = {
+        'url': 'http://www.nbcsports.com//college-basketball/ncaab/tom-izzo-michigan-st-has-so-much-respect-duke',
+        'md5': 'ba6c93f96b67bf05344f78bd523dac0f',
+        'info_dict': {
+            'id': 'PHJSaFWbrTY9',
+            'ext': 'flv',
+            'title': 'Tom Izzo, Michigan St. has \'so much respect\' for Duke',
+            'description': 'md5:ecb459c9d59e0766ac9c7d5d0eda8113',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        return self.url_result(
+            NBCSportsVPlayerIE._extract_url(webpage), 'NBCSportsVPlayer')
+
+
 class NBCNewsIE(InfoExtractor):
     _VALID_URL = r'''(?x)https?://(?:www\.)?nbcnews\.com/
         (?:video/.+?/(?P<id>\d+)|
diff --git a/youtube_dl/extractor/yahoo.py b/youtube_dl/extractor/yahoo.py
index 6e72f1e55..43776d1e6 100644
--- a/youtube_dl/extractor/yahoo.py
+++ b/youtube_dl/extractor/yahoo.py
@@ -17,6 +17,8 @@ from ..utils import (
     int_or_none,
 )
 
+from .nbc import NBCSportsVPlayerIE
+
 
 class YahooIE(InfoExtractor):
     IE_DESC = 'Yahoo screen and movies'
@@ -132,6 +134,7 @@ class YahooIE(InfoExtractor):
         }, {
             'note': 'NBC Sports embeds',
             'url': 'http://sports.yahoo.com/blogs/ncaab-the-dagger/tyler-kalinoski-s-buzzer-beater-caps-davidson-s-comeback-win-185609842.html?guid=nbc_cbk_davidsonbuzzerbeater_150313',
+            'md5': 'ceae8dced5c14a1c1ffcb7a32194cca5',
             'info_dict': {
                 'id': '9CsDKds0kvHI',
                 'ext': 'flv',
@@ -161,10 +164,9 @@ class YahooIE(InfoExtractor):
                 video_id = items[0]['id']
                 return self._get_info(video_id, display_id, webpage)
         # Look for NBCSports iframes
-        iframe_m = re.search(
-            r'<iframe[^>]+src="(?P<url>https?://vplayer\.nbcsports\.com/[^"]+)"', webpage)
-        if iframe_m:
-            return self.url_result(iframe_m.group('url'), 'NBCSports')
+        nbc_sports_url = NBCSportsVPlayerIE._extract_url(webpage)
+        if nbc_sports_url:
+            return self.url_result(nbc_sports_url, 'NBCSportsVPlayer')
 
         items_json = self._search_regex(
             r'mediaItems: ({.*?})$', webpage, 'items', flags=re.MULTILINE,

From 1d31e7a2fc2fb78c792754578a8a58b056811b84 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 31 Mar 2015 02:51:11 +0800
Subject: [PATCH 0811/6219] [NBCSports] Move imports alphabetically

---
 youtube_dl/extractor/__init__.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index b113aaec6..9fddb8e32 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -310,9 +310,9 @@ from .naver import NaverIE
 from .nba import NBAIE
 from .nbc import (
     NBCIE,
-    NBCSportsVPlayerIE,
-    NBCSportsIE,
     NBCNewsIE,
+    NBCSportsIE,
+    NBCSportsVPlayerIE,
 )
 from .ndr import NDRIE
 from .ndtv import NDTVIE

From a2edf2e7ff314eaa3124c1da1b962d054b6d9fff Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 31 Mar 2015 03:36:09 +0800
Subject: [PATCH 0812/6219] [NBC/ThePlatform/Generic] Add a generic detector
 for NBCSportsVPlayer and enhance error detection in ThePlatformIE

---
 youtube_dl/extractor/generic.py     | 15 +++++++++++++++
 youtube_dl/extractor/nbc.py         |  2 +-
 youtube_dl/extractor/theplatform.py |  2 +-
 3 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 042d23a13..9ddf36f6b 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -29,6 +29,7 @@ from ..utils import (
     xpath_text,
 )
 from .brightcove import BrightcoveIE
+from .nbc import NBCSportsVPlayerIE
 from .ooyala import OoyalaIE
 from .rutv import RUTVIE
 from .smotri import SmotriIE
@@ -639,6 +640,15 @@ class GenericIE(InfoExtractor):
                 'upload_date': '20150228',
                 'title': 'pdv_maddow_netcast_m4v-02-27-2015-201624',
             }
+        },
+        # NBC Sports vplayer embeds
+        {
+            'url': 'http://bbs.clutchfans.net/showthread.php?t=244180',
+            'info_dict': {
+                'id': '_hqLjQ95yx8Z',
+                'ext': 'flv'
+            },
+            'skip': 'This content expired on 9/17/14 12:23 PM',
         }
     ]
 
@@ -1252,6 +1262,11 @@ class GenericIE(InfoExtractor):
         if mobj is not None:
             return self.url_result('5min:%s' % mobj.group('id'), 'FiveMin')
 
+        # Look for NBC Sports VPlayer embeds
+        nbc_sports_url = NBCSportsVPlayerIE._extract_url(webpage)
+        if nbc_sports_url:
+            return self.url_result(nbc_sports_url, 'NBCSportsVPlayer')
+
         def check_video(vurl):
             if YoutubeIE.suitable(vurl):
                 return True
diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index c8dd72ab4..be9969d12 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -51,7 +51,7 @@ class NBCIE(InfoExtractor):
 
 
 class NBCSportsVPlayerIE(InfoExtractor):
-    _VALID_URL = r'https?://vplayer\.nbcsports\.com/(?:[^/]+/)+(?P<id>[0-9a-zA-Z]+)'
+    _VALID_URL = r'https?://vplayer\.nbcsports\.com/(?:[^/]+/)+(?P<id>[0-9a-zA-Z_]+)'
 
     _TEST = {
         'url': 'https://vplayer.nbcsports.com/p/BxmELC/nbcsports_share/select/9CsDKds0kvHI',
diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index feac666f7..0e3e627f4 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -92,7 +92,7 @@ class ThePlatformIE(InfoExtractor):
             error_msg = next(
                 n.attrib['abstract']
                 for n in meta.findall(_x('.//smil:ref'))
-                if n.attrib.get('title') == 'Geographic Restriction')
+                if n.attrib.get('title') == 'Geographic Restriction' or n.attrib.get('title') == 'Expired')
         except StopIteration:
             pass
         else:

From 5cbb2699ee04535449e37a07dd9cac9bfd224fe3 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 31 Mar 2015 03:38:45 +0800
Subject: [PATCH 0813/6219] [NBCSports] Add a test case for extended _VALID_URL

---
 youtube_dl/extractor/nbc.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index be9969d12..395f53df3 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -53,7 +53,7 @@ class NBCIE(InfoExtractor):
 class NBCSportsVPlayerIE(InfoExtractor):
     _VALID_URL = r'https?://vplayer\.nbcsports\.com/(?:[^/]+/)+(?P<id>[0-9a-zA-Z_]+)'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'https://vplayer.nbcsports.com/p/BxmELC/nbcsports_share/select/9CsDKds0kvHI',
         'md5': 'ceae8dced5c14a1c1ffcb7a32194cca5',
         'info_dict': {
@@ -62,7 +62,11 @@ class NBCSportsVPlayerIE(InfoExtractor):
             'description': 'md5:df390f70a9ba7c95ff1daace988f0d8d',
             'title': 'Tyler Kalinoski hits buzzer-beater to lift Davidson',
         }
-    }
+    }, {
+        'note': 'This video is already expired. It\'s for testing _VALID_URL',
+        'url': 'http://vplayer.nbcsports.com/p/BxmELC/nbc_embedshare/select/_hqLjQ95yx8Z',
+        'only_matching': True,
+    }]
 
     @staticmethod
     def _extract_url(webpage):

From e15307a612ea588b504f1f03ba0201612df66b35 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 31 Mar 2015 13:13:29 +0800
Subject: [PATCH 0814/6219] [NBCSports/Yahoo] Comment out some MD5 checksums

They seems to change constantly
---
 youtube_dl/extractor/nbc.py   | 2 --
 youtube_dl/extractor/yahoo.py | 1 -
 2 files changed, 3 deletions(-)

diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index 395f53df3..b7f6a5366 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -55,7 +55,6 @@ class NBCSportsVPlayerIE(InfoExtractor):
 
     _TESTS = [{
         'url': 'https://vplayer.nbcsports.com/p/BxmELC/nbcsports_share/select/9CsDKds0kvHI',
-        'md5': 'ceae8dced5c14a1c1ffcb7a32194cca5',
         'info_dict': {
             'id': '9CsDKds0kvHI',
             'ext': 'flv',
@@ -88,7 +87,6 @@ class NBCSportsIE(InfoExtractor):
 
     _TEST = {
         'url': 'http://www.nbcsports.com//college-basketball/ncaab/tom-izzo-michigan-st-has-so-much-respect-duke',
-        'md5': 'ba6c93f96b67bf05344f78bd523dac0f',
         'info_dict': {
             'id': 'PHJSaFWbrTY9',
             'ext': 'flv',
diff --git a/youtube_dl/extractor/yahoo.py b/youtube_dl/extractor/yahoo.py
index 43776d1e6..b777159c5 100644
--- a/youtube_dl/extractor/yahoo.py
+++ b/youtube_dl/extractor/yahoo.py
@@ -134,7 +134,6 @@ class YahooIE(InfoExtractor):
         }, {
             'note': 'NBC Sports embeds',
             'url': 'http://sports.yahoo.com/blogs/ncaab-the-dagger/tyler-kalinoski-s-buzzer-beater-caps-davidson-s-comeback-win-185609842.html?guid=nbc_cbk_davidsonbuzzerbeater_150313',
-            'md5': 'ceae8dced5c14a1c1ffcb7a32194cca5',
             'info_dict': {
                 'id': '9CsDKds0kvHI',
                 'ext': 'flv',

From facecb84a11ba59a67d01c659ec3944179774418 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 31 Mar 2015 20:11:14 +0600
Subject: [PATCH 0815/6219] [generic] Add working NBC Sports vplayer test

---
 youtube_dl/extractor/generic.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 9ddf36f6b..2ff002643 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -641,14 +641,15 @@ class GenericIE(InfoExtractor):
                 'title': 'pdv_maddow_netcast_m4v-02-27-2015-201624',
             }
         },
-        # NBC Sports vplayer embeds
+        # NBC Sports vplayer embed
         {
-            'url': 'http://bbs.clutchfans.net/showthread.php?t=244180',
+            'url': 'http://www.riderfans.com/forum/showthread.php?121827-Freeman&s=e98fa1ea6dc08e886b1678d35212494a',
             'info_dict': {
-                'id': '_hqLjQ95yx8Z',
-                'ext': 'flv'
+                'id': 'ln7x1qSThw4k',
+                'ext': 'flv',
+                'title': "PFT Live: New leader in the 'new-look' defense",
+                'description': 'md5:65a19b4bbfb3b0c0c5768bed1dfad74e',
             },
-            'skip': 'This content expired on 9/17/14 12:23 PM',
         }
     ]
 

From c89fbfb385c4989af3bf2eb45e300e01c385cfc7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 31 Mar 2015 20:14:37 +0600
Subject: [PATCH 0816/6219] [nbc] Remove redundant note

This is already supposed by `only_matching`
---
 youtube_dl/extractor/nbc.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index b7f6a5366..ecd0ac8b1 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -62,7 +62,6 @@ class NBCSportsVPlayerIE(InfoExtractor):
             'title': 'Tyler Kalinoski hits buzzer-beater to lift Davidson',
         }
     }, {
-        'note': 'This video is already expired. It\'s for testing _VALID_URL',
         'url': 'http://vplayer.nbcsports.com/p/BxmELC/nbc_embedshare/select/_hqLjQ95yx8Z',
         'only_matching': True,
     }]

From 2a0c2ca2b8be753f13f9b9dfdcce55560e4953e8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 31 Mar 2015 20:55:21 +0600
Subject: [PATCH 0817/6219] [dailymotion] Fix ff cookie and use it for embed
 page (Closes #5330)

---
 youtube_dl/extractor/dailymotion.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index 4f67c3aac..47d58330b 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -25,8 +25,7 @@ class DailymotionBaseInfoExtractor(InfoExtractor):
     def _build_request(url):
         """Build a request with the family filter disabled"""
         request = compat_urllib_request.Request(url)
-        request.add_header('Cookie', 'family_filter=off')
-        request.add_header('Cookie', 'ff=off')
+        request.add_header('Cookie', 'family_filter=off; ff=off')
         return request
 
 
@@ -112,8 +111,9 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
             video_upload_date = mobj.group(3) + mobj.group(2) + mobj.group(1)
 
         embed_url = 'http://www.dailymotion.com/embed/video/%s' % video_id
-        embed_page = self._download_webpage(embed_url, video_id,
-                                            'Downloading embed page')
+        embed_request = self._build_request(embed_url)
+        embed_page = self._download_webpage(
+            embed_request, video_id, 'Downloading embed page')
         info = self._search_regex(r'var info = ({.*?}),$', embed_page,
                                   'video info', flags=re.MULTILINE)
         info = json.loads(info)

From 55cde6ef3c046dfd5f6ef84b908d38ee4375cf08 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 1 Apr 2015 22:02:55 +0600
Subject: [PATCH 0818/6219] [varzesh3] Simplify

---
 youtube_dl/extractor/varzesh3.py | 43 +++++++++++++++-----------------
 1 file changed, 20 insertions(+), 23 deletions(-)

diff --git a/youtube_dl/extractor/varzesh3.py b/youtube_dl/extractor/varzesh3.py
index eb49586cc..9369abaf8 100644
--- a/youtube_dl/extractor/varzesh3.py
+++ b/youtube_dl/extractor/varzesh3.py
@@ -1,48 +1,45 @@
 # coding: utf-8
 from __future__ import unicode_literals
+
 from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
-)
-import re
 
 
 class Varzesh3IE(InfoExtractor):
-    _VALID_URL = r'(?P<url>(https?://(?:www\.)?video\.varzesh3\.com)/(?P<id>.+))'
-    _TEST ={
+    _VALID_URL = r'https?://(?:www\.)?video\.varzesh3\.com/(?:[^/]+/)+(?P<id>[^/]+)/?'
+    _TEST = {
         'url': 'http://video.varzesh3.com/germany/bundesliga/5-%D9%88%D8%A7%DA%A9%D9%86%D8%B4-%D8%A8%D8%B1%D8%AA%D8%B1-%D8%AF%D8%B1%D9%88%D8%A7%D8%B2%D9%87%E2%80%8C%D8%A8%D8%A7%D9%86%D8%A7%D9%86%D8%9B%D9%87%D9%81%D8%AA%D9%87-26-%D8%A8%D9%88%D9%86%D8%AF%D8%B3/',
         'md5': '2a933874cb7dce4366075281eb49e855',
         'info_dict': {
-            'url': 'http://dl1.video.varzesh3.com/video/clip94/1/video/namayeshi/saves_week26.mp4',
             'id': '76337',
             'ext': 'mp4',
             'title': '۵ واکنش برتر دروازه‌بانان؛هفته ۲۶ بوندسلیگا',
-            'thumbnail': 'http://video.varzesh3.com/wp-content/uploads/230315_saves_week26.jpg',
             'description': 'فصل ۲۰۱۵-۲۰۱۴',
+            'thumbnail': 're:^https?://.*\.jpg$',
         }
     }
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
+        display_id = self._match_id(url)
 
-        if not 'shortlink' in webpage:
-            raise ExtractorError('URL has no videos or there is a problem.')
+        webpage = self._download_webpage(url, display_id)
 
-        title = self._html_search_regex(r'meta[^>]+property="og:title"[^>]+content="([^"]+)"', webpage, 'title')
-        video_link = self._html_search_regex(r'source[^>]+src="([^"]+)"', webpage, 'video_link')
-        vid_id = self._html_search_regex(r"link[^>]+rel='canonical'[^>]+href='\/\?p=([^']+)'\/>", webpage, 'vid_id')
-        try:
-            description = self._html_search_regex(r'<div class="matn">(.*?)</div>', webpage, 'description', flags=re.DOTALL)
-        except:
-            description = title
-        thumbnail = self._html_search_regex(r'link[^>]+rel="image_src"[^>]+href="([^"]+)"', webpage, 'thumbnail')
+        video_url = self._search_regex(
+            r'<source[^>]+src="([^"]+)"', webpage, 'video url')
+
+        title = self._og_search_title(webpage)
+        description = self._html_search_regex(
+            r'(?s)<div class="matn">(.+?)</div>',
+            webpage, 'description', fatal=False)
+        thumbnail = self._og_search_thumbnail(webpage)
+
+        video_id = self._search_regex(
+            r"<link[^>]+rel='(?:canonical|shortlink)'[^>]+href='/\?p=([^']+)'",
+            webpage, display_id, default=display_id)
 
         return {
-            'url': video_link,
-            'id': vid_id,
+            'url': video_url,
+            'id': video_id,
             'title': title,
-            'ext': video_link.split(".")[-1],
             'description': description,
             'thumbnail': thumbnail,
         }

From 1a944d8a2ae8756b78a709862cccae58720445ba Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 2 Apr 2015 14:09:55 +0200
Subject: [PATCH 0819/6219] Print a warning if no ssl certificates are loaded

---
 youtube_dl/YoutubeDL.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 640b8c99d..4fa2223ad 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1768,6 +1768,12 @@ class YoutubeDL(object):
 
         debuglevel = 1 if self.params.get('debug_printtraffic') else 0
         https_handler = make_HTTPS_handler(self.params, debuglevel=debuglevel)
+        # The ssl context is only available in python 2.7.9 and 3.x
+        if hasattr(https_handler, '_context'):
+            if len(https_handler._context.get_ca_certs()) == 0:
+                self.report_warning(
+                    'No ssl certificates were loaded, urls that use https '
+                    'won\'t work')
         ydlh = YoutubeDLHandler(self.params, debuglevel=debuglevel)
         opener = compat_urllib_request.build_opener(
             proxy_handler, https_handler, cookie_processor, ydlh)

From 8075d4f99dbbf330c4a44d58b535055853b99aab Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 2 Apr 2015 20:26:05 +0600
Subject: [PATCH 0820/6219] [playfm] Adapt to v2api (Closes #5344)

---
 youtube_dl/extractor/playfm.py | 87 +++++++++++++++-------------------
 1 file changed, 37 insertions(+), 50 deletions(-)

diff --git a/youtube_dl/extractor/playfm.py b/youtube_dl/extractor/playfm.py
index 9576aed0e..e766ccca3 100644
--- a/youtube_dl/extractor/playfm.py
+++ b/youtube_dl/extractor/playfm.py
@@ -4,85 +4,72 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse,
-    compat_urllib_request,
-)
+from ..compat import compat_str
 from ..utils import (
     ExtractorError,
-    float_or_none,
     int_or_none,
-    str_to_int,
+    parse_iso8601,
 )
 
 
 class PlayFMIE(InfoExtractor):
     IE_NAME = 'play.fm'
-    _VALID_URL = r'https?://(?:www\.)?play\.fm/[^?#]*(?P<upload_date>[0-9]{8})(?P<id>[0-9]{6})(?:$|[?#])'
+    _VALID_URL = r'https?://(?:www\.)?play\.fm/(?P<slug>(?:[^/]+/)+(?P<id>[^/]+))/?(?:$|[?#])'
 
     _TEST = {
-        'url': 'http://www.play.fm/recording/leipzigelectronicmusicbatofarparis_fr20140712137220',
+        'url': 'https://www.play.fm/dan-drastic/sven-tasnadi-leipzig-electronic-music-batofar-paris-fr-2014-07-12',
         'md5': 'c505f8307825a245d0c7ad1850001f22',
         'info_dict': {
-            'id': '137220',
+            'id': '71276',
             'ext': 'mp3',
-            'title': 'LEIPZIG ELECTRONIC MUSIC @ Batofar (Paris,FR) - 2014-07-12',
-            'uploader': 'Sven Tasnadi',
-            'uploader_id': 'sventasnadi',
-            'duration': 5627.428,
-            'upload_date': '20140712',
+            'title': 'Sven Tasnadi - LEIPZIG ELECTRONIC MUSIC @ Batofar (Paris,FR) - 2014-07-12',
+            'description': '',
+            'duration': 5627,
+            'timestamp': 1406033781,
+            'upload_date': '20140722',
+            'uploader': 'Dan Drastic',
+            'uploader_id': '71170',
             'view_count': int,
             'comment_count': int,
-            'thumbnail': 're:^https?://.*\.jpg$',
         },
     }
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
-        upload_date = mobj.group('upload_date')
+        slug = mobj.group('slug')
 
-        rec_data = compat_urllib_parse.urlencode({'rec_id': video_id})
-        req = compat_urllib_request.Request(
-            'http://www.play.fm/flexRead/recording', data=rec_data)
-        req.add_header('Content-Type', 'application/x-www-form-urlencoded')
-        rec_doc = self._download_xml(req, video_id)
+        recordings = self._download_json(
+            'http://v2api.play.fm/recordings/slug/%s' % slug, video_id)
 
-        error_node = rec_doc.find('./error')
-        if error_node is not None:
-            raise ExtractorError('An error occured: %s (code %s)' % (
-                error_node.text, rec_doc.find('./status').text))
+        error = recordings.get('error')
+        if isinstance(error, dict):
+            raise ExtractorError(
+                '%s returned error: %s' % (self.IE_NAME, error.get('message')),
+                expected=True)
 
-        recording = rec_doc.find('./recording')
-        title = recording.find('./title').text
-        view_count = str_to_int(recording.find('./stats/playcount').text)
-        comment_count = str_to_int(recording.find('./stats/comments').text)
-        duration = float_or_none(recording.find('./duration').text, scale=1000)
-        thumbnail = recording.find('./image').text
-
-        artist = recording.find('./artists/artist')
-        uploader = artist.find('./name').text
-        uploader_id = artist.find('./slug').text
-
-        video_url = '%s//%s/%s/%s/offset/0/sh/%s/rec/%s/jingle/%s/loc/%s' % (
-            'http:', recording.find('./url').text,
-            recording.find('./_class').text, recording.find('./file_id').text,
-            rec_doc.find('./uuid').text, video_id,
-            rec_doc.find('./jingle/file_id').text,
-            'http%3A%2F%2Fwww.play.fm%2Fplayer',
-        )
+        audio_url = recordings['audio']
+        video_id = compat_str(recordings.get('id') or video_id)
+        title = recordings['title']
+        description = recordings.get('description')
+        duration = int_or_none(recordings.get('recordingDuration'))
+        timestamp = parse_iso8601(recordings.get('created_at'))
+        uploader = recordings.get('page', {}).get('title')
+        uploader_id = compat_str(recordings.get('page', {}).get('id'))
+        view_count = int_or_none(recordings.get('playCount'))
+        comment_count = int_or_none(recordings.get('commentCount'))
+        categories = [tag['name'] for tag in recordings.get('tags', []) if tag.get('name')]
 
         return {
             'id': video_id,
-            'url': video_url,
-            'ext': 'mp3',
-            'filesize': int_or_none(recording.find('./size').text),
+            'url': audio_url,
             'title': title,
-            'upload_date': upload_date,
-            'view_count': view_count,
-            'comment_count': comment_count,
+            'description': description,
             'duration': duration,
-            'thumbnail': thumbnail,
+            'timestamp': timestamp,
             'uploader': uploader,
             'uploader_id': uploader_id,
+            'view_count': view_count,
+            'comment_count': comment_count,
+            'categories': categories,
         }

From f5b669113fc530922280987ff9e19afa780b8844 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 2 Apr 2015 22:32:16 +0600
Subject: [PATCH 0821/6219] [miomio] Simplify and fix python 2.6 issue

---
 youtube_dl/extractor/miomio_tv.py | 101 ++++++++++++++++++------------
 1 file changed, 60 insertions(+), 41 deletions(-)

diff --git a/youtube_dl/extractor/miomio_tv.py b/youtube_dl/extractor/miomio_tv.py
index ae20a32fa..dc2ba7cb4 100644
--- a/youtube_dl/extractor/miomio_tv.py
+++ b/youtube_dl/extractor/miomio_tv.py
@@ -1,74 +1,93 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import random
+
 from .common import InfoExtractor
+from ..utils import (
+    xpath_text,
+    int_or_none,
+)
 
 
 class MiomioTvIE(InfoExtractor):
     IE_NAME = 'miomio.tv'
     _VALID_URL = r'https?://(?:www\.)?miomio\.tv/watch/cc(?P<id>[0-9]+)'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.miomio.tv/watch/cc179734/',
         'md5': '48de02137d0739c15b440a224ad364b9',
         'info_dict': {
             'id': '179734',
-            'title': u'\u624b\u7ed8\u52a8\u6f2b\u9b3c\u6ce3\u4f46\u4e01\u5168\u7a0b\u753b\u6cd5',
-            'ext': 'flv'
-        }
-    }
+            'ext': 'flv',
+            'title': '手绘动漫鬼泣但丁全程画法',
+            'duration': 354,
+        },
+    }, {
+        'url': 'http://www.miomio.tv/watch/cc184024/',
+        'info_dict': {
+            'id': '43729',
+            'title': '《动漫同人插画绘制》',
+        },
+        'playlist_mincount': 86,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        title = self._html_search_regex(r'<meta\s+name="description"\s+content="\s*([^"]*)\s*"', webpage, 'title')
-        ref_path = self._search_regex(r'src="(/mioplayer/.*?)"', webpage, 'ref_path')
-        referer = 'http://www.miomio.tv{0}'.format(ref_path)
-        xml_config = self._search_regex(r'flashvars="type=sina&amp;(.*?)&amp;cid=', webpage, 'xml config')
+        title = self._html_search_meta(
+            'description', webpage, 'title', fatal=True)
+
+        mioplayer_path = self._search_regex(
+            r'src="(/mioplayer/[^"]+)"', webpage, 'ref_path')
+
+        xml_config = self._search_regex(
+            r'flashvars="type=sina&amp;(.+?)&amp;',
+            webpage, 'xml config')
         
         # skipping the following page causes lags and eventually connection drop-outs
-        # id is normally a rotating three digit value but a fixed value always appears to work
-        self._request_webpage("http://www.miomio.tv/mioplayer/mioplayerconfigfiles/xml.php?id={0}&r=cc{1}".format(id, 945), video_id)
+        self._request_webpage(
+            'http://www.miomio.tv/mioplayer/mioplayerconfigfiles/xml.php?id=%s&r=%s' % (id, random.randint(100, 999)),
+            video_id)
 
         # the following xml contains the actual configuration information on the video file(s)
-        xml_url = 'http://www.miomio.tv/mioplayer/mioplayerconfigfiles/sina.php?{0}'.format(xml_config)
-        vidconfig = self._download_xml(xml_url, video_id)
-
-        file_els = vidconfig.findall('.//durl')
-
-        entries = []
-
-        for file_el in file_els:
-            segment_id = file_el.find('order').text.strip()
-            segment_title = '_'.join([title, segment_id])
-            segment_duration = file_el.find('length').text.strip()
-            segment_url = file_el.find('url').text.strip()
-
-            entries.append({
-                'id': segment_id,
-                'title': segment_title,
-                'duration': segment_duration,
-                'url': segment_url
-            })
+        vid_config = self._download_xml(
+            'http://www.miomio.tv/mioplayer/mioplayerconfigfiles/sina.php?{0}'.format(xml_config),
+            video_id)
 
         http_headers = {
-            'Referer': referer,
-            'Accept-Encoding': 'gzip, deflate',
-            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
+            'Referer': 'http://www.miomio.tv%s' % mioplayer_path,
         }
 
+        entries = []
+        for f in vid_config.findall('./durl'):
+            segment_url = xpath_text(f, 'url', 'video url')
+            if not segment_url:
+                continue
+            order = xpath_text(f, 'order', 'order')
+            segment_id = video_id
+            segment_title = title
+            if order:
+                segment_id += '-%s' % order
+                segment_title += ' part %s' % order
+            entries.append({
+                'id': segment_id,
+                'url': segment_url,
+                'title': segment_title,
+                'duration': int_or_none(xpath_text(f, 'length', 'duration'), 1000),
+                'http_headers': http_headers,
+            })
+
         if len(entries) == 1:
-            return {
-                'id': video_id,
-                'title': title,
-                'url': entries[0]['url'],
-                'http_headers': http_headers
-            }
+            segment = entries[0]
+            segment['id'] = video_id
+            segment['title'] = title
+            return segment
 
         return {
             '_type': 'multi_video',
             'id': video_id,
-            'title': title,
             'entries': entries,
-            'http_headers': http_headers
+            'title': title,
+            'http_headers': http_headers,
         }

From e03bfb30ce0062c048759ed01f5a2fe4190d66fd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 2 Apr 2015 22:33:30 +0600
Subject: [PATCH 0822/6219] [miomio] Rename extractor

---
 youtube_dl/extractor/__init__.py                 | 2 +-
 youtube_dl/extractor/{miomio_tv.py => miomio.py} | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)
 rename youtube_dl/extractor/{miomio_tv.py => miomio.py} (98%)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 0b9736f2d..9700d81f5 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -274,7 +274,7 @@ from .mdr import MDRIE
 from .metacafe import MetacafeIE
 from .metacritic import MetacriticIE
 from .mgoon import MgoonIE
-from .miomio_tv import MiomioTvIE
+from .miomio import MioMioIE
 from .minhateca import MinhatecaIE
 from .ministrygrid import MinistryGridIE
 from .mit import TechTVMITIE, MITIE, OCWMITIE
diff --git a/youtube_dl/extractor/miomio_tv.py b/youtube_dl/extractor/miomio.py
similarity index 98%
rename from youtube_dl/extractor/miomio_tv.py
rename to youtube_dl/extractor/miomio.py
index dc2ba7cb4..11608f730 100644
--- a/youtube_dl/extractor/miomio_tv.py
+++ b/youtube_dl/extractor/miomio.py
@@ -10,7 +10,7 @@ from ..utils import (
 )
 
 
-class MiomioTvIE(InfoExtractor):
+class MioMioIE(InfoExtractor):
     IE_NAME = 'miomio.tv'
     _VALID_URL = r'https?://(?:www\.)?miomio\.tv/watch/cc(?P<id>[0-9]+)'
     _TESTS = [{

From 2ec8e04cac895121a71f11a44b855b1bf8a0195e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 2 Apr 2015 22:34:08 +0600
Subject: [PATCH 0823/6219] [miomio] Fix alphabetic order

---
 youtube_dl/extractor/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 9700d81f5..aae4aae4c 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -274,9 +274,9 @@ from .mdr import MDRIE
 from .metacafe import MetacafeIE
 from .metacritic import MetacriticIE
 from .mgoon import MgoonIE
-from .miomio import MioMioIE
 from .minhateca import MinhatecaIE
 from .ministrygrid import MinistryGridIE
+from .miomio import MioMioIE
 from .mit import TechTVMITIE, MITIE, OCWMITIE
 from .mitele import MiTeleIE
 from .mixcloud import MixcloudIE

From a3c7019e065298103026960e00fff6b80b6bcf62 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 2 Apr 2015 22:50:10 +0600
Subject: [PATCH 0824/6219] [YoutubeDL] Check for `get_ca_certs` availability

`get_ca_certs` is not available in python <3.4
---
 youtube_dl/YoutubeDL.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 4fa2223ad..ce4b72fd3 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1770,7 +1770,9 @@ class YoutubeDL(object):
         https_handler = make_HTTPS_handler(self.params, debuglevel=debuglevel)
         # The ssl context is only available in python 2.7.9 and 3.x
         if hasattr(https_handler, '_context'):
-            if len(https_handler._context.get_ca_certs()) == 0:
+            ctx = https_handler._context
+            # get_ca_certs is unavailable prior to python 3.4
+            if hasattr(ctx, 'get_ca_certs') and len(ctx.get_ca_certs()) == 0:
                 self.report_warning(
                     'No ssl certificates were loaded, urls that use https '
                     'won\'t work')

From 6b7556a554a490a21307fec65cdaefc3331027c6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 3 Apr 2015 01:47:18 +0600
Subject: [PATCH 0825/6219] Credit @tiktok7 for miomio.tv (#5265)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 48be31e29..48769320a 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -119,3 +119,4 @@ Eduardo Ferro Aldama
 Jeff Buchbinder
 Amish Bhadeshia
 Joram Schrijver
+Will W.

From a9cbab173584c716219b348ff36ccd5274f75249 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 3 Apr 2015 10:22:25 +0200
Subject: [PATCH 0826/6219] release 2015.04.03

---
 docs/supportedsites.md | 6 ++++++
 youtube_dl/version.py  | 2 +-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index fd59cc2be..2785b9587 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -111,6 +111,7 @@
  - **DctpTv**
  - **DeezerPlaylist**
  - **defense.gouv.fr**
+ - **DHM**: Filmarchiv - Deutsches Historisches Museum
  - **Discovery**
  - **divxstage**: DivxStage
  - **Dotsub**
@@ -120,6 +121,7 @@
  - **DrTuber**
  - **DRTV**
  - **Dump**
+ - **Dumpert**
  - **dvtv**: http://video.aktualne.cz/
  - **EaglePlatform**
  - **EbaumsWorld**
@@ -253,6 +255,7 @@
  - **Mgoon**
  - **Minhateca**
  - **MinistryGrid**
+ - **miomio.tv**
  - **mitele.es**
  - **mixcloud**
  - **MLB**
@@ -286,6 +289,8 @@
  - **NBA**
  - **NBC**
  - **NBCNews**
+ - **NBCSports**
+ - **NBCSportsVPlayer**
  - **ndr**: NDR.de - Mediathek
  - **NDTV**
  - **NerdCubedFeed**
@@ -501,6 +506,7 @@
  - **Urort**: NRK P3 Urørt
  - **ustream**
  - **ustream:channel**
+ - **Varzesh3**
  - **Vbox7**
  - **VeeHD**
  - **Veoh**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index dd93e295a..e1c385bec 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.03.28'
+__version__ = '2015.04.03'

From 4bbeb19fc77a49af763ce3443293b29b8450d686 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 3 Apr 2015 14:09:07 +0200
Subject: [PATCH 0827/6219] [miomio] pep8: remove whitespaces in empty line

---
 youtube_dl/extractor/miomio.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/miomio.py b/youtube_dl/extractor/miomio.py
index 11608f730..cc3f27194 100644
--- a/youtube_dl/extractor/miomio.py
+++ b/youtube_dl/extractor/miomio.py
@@ -44,7 +44,7 @@ class MioMioIE(InfoExtractor):
         xml_config = self._search_regex(
             r'flashvars="type=sina&amp;(.+?)&amp;',
             webpage, 'xml config')
-        
+
         # skipping the following page causes lags and eventually connection drop-outs
         self._request_webpage(
             'http://www.miomio.tv/mioplayer/mioplayerconfigfiles/xml.php?id=%s&r=%s' % (id, random.randint(100, 999)),

From 3da4b31359c8dbbad3477fbe8341e6fc293cda82 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 3 Apr 2015 14:09:50 +0200
Subject: [PATCH 0828/6219] [postprocessor/ffmpeg] Fix crash when
 ffprobe/avprobe are not installed (closes #5349)

'self.probe_basename' was None, so 'probe_executable' raised a KeyError exception
---
 youtube_dl/postprocessor/ffmpeg.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 55adf9685..0b60ac7e7 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -116,6 +116,10 @@ class FFmpegPostProcessor(PostProcessor):
     def executable(self):
         return self._paths[self.basename]
 
+    @property
+    def probe_available(self):
+        return self.probe_basename is not None
+
     @property
     def probe_executable(self):
         return self._paths[self.probe_basename]
@@ -168,7 +172,7 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
 
     def get_audio_codec(self, path):
 
-        if not self.probe_executable:
+        if not self.probe_available:
             raise PostProcessingError('ffprobe or avprobe not found. Please install one.')
         try:
             cmd = [

From ff2be6e180f1af471dd6d533719d9c595c756557 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 3 Apr 2015 15:01:17 +0200
Subject: [PATCH 0829/6219] [bloomberg] Adapt to website changes (fixes #5347)

---
 youtube_dl/extractor/bloomberg.py | 25 ++++++++++++++++---------
 1 file changed, 16 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/bloomberg.py b/youtube_dl/extractor/bloomberg.py
index 4a88ccd13..0dca29b71 100644
--- a/youtube_dl/extractor/bloomberg.py
+++ b/youtube_dl/extractor/bloomberg.py
@@ -6,32 +6,39 @@ from .common import InfoExtractor
 
 
 class BloombergIE(InfoExtractor):
-    _VALID_URL = r'https?://www\.bloomberg\.com/video/(?P<id>.+?)\.html'
+    _VALID_URL = r'https?://www\.bloomberg\.com/news/videos/[^/]+/(?P<id>[^/?#]+)'
 
     _TEST = {
-        'url': 'http://www.bloomberg.com/video/shah-s-presentation-on-foreign-exchange-strategies-qurhIVlJSB6hzkVi229d8g.html',
+        'url': 'http://www.bloomberg.com/news/videos/b/aaeae121-5949-481e-a1ce-4562db6f5df2',
         # The md5 checksum changes
         'info_dict': {
             'id': 'qurhIVlJSB6hzkVi229d8g',
             'ext': 'flv',
             'title': 'Shah\'s Presentation on Foreign-Exchange Strategies',
-            'description': 'md5:0681e0d30dcdfc6abf34594961d8ea88',
+            'description': 'md5:a8ba0302912d03d246979735c17d2761',
         },
     }
 
     def _real_extract(self, url):
         name = self._match_id(url)
         webpage = self._download_webpage(url, name)
-
-        f4m_url = self._search_regex(
-            r'<source src="(https?://[^"]+\.f4m.*?)"', webpage,
-            'f4m url')
+        video_id = self._search_regex(r'"bmmrId":"(.+?)"', webpage, 'id')
         title = re.sub(': Video$', '', self._og_search_title(webpage))
 
+        embed_info = self._download_json(
+            'http://www.bloomberg.com/api/embed?id=%s' % video_id, video_id)
+        formats = []
+        for stream in embed_info['streams']:
+            if stream["muxing_format"] == "TS":
+                formats.extend(self._extract_m3u8_formats(stream['url'], video_id))
+            else:
+                formats.extend(self._extract_f4m_formats(stream['url'], video_id))
+        self._sort_formats(formats)
+
         return {
-            'id': name.split('-')[-1],
+            'id': video_id,
             'title': title,
-            'formats': self._extract_f4m_formats(f4m_url, name),
+            'formats': formats,
             'description': self._og_search_description(webpage),
             'thumbnail': self._og_search_thumbnail(webpage),
         }

From e81a47460365738a0add4d4da52a712c0091704f Mon Sep 17 00:00:00 2001
From: snipem <mail@matthias-kuech.de>
Date: Fri, 3 Apr 2015 15:34:49 +0200
Subject: [PATCH 0830/6219] [Gamersyde] Add new extractor

---
 youtube_dl/extractor/__init__.py  |  1 +
 youtube_dl/extractor/gamersyde.py | 64 +++++++++++++++++++++++++++++++
 2 files changed, 65 insertions(+)
 create mode 100644 youtube_dl/extractor/gamersyde.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index aae4aae4c..2935d5b33 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -177,6 +177,7 @@ from .gameone import (
     GameOneIE,
     GameOnePlaylistIE,
 )
+from .gamersyde import GamersydeIE
 from .gamespot import GameSpotIE
 from .gamestar import GameStarIE
 from .gametrailers import GametrailersIE
diff --git a/youtube_dl/extractor/gamersyde.py b/youtube_dl/extractor/gamersyde.py
new file mode 100644
index 000000000..c40106216
--- /dev/null
+++ b/youtube_dl/extractor/gamersyde.py
@@ -0,0 +1,64 @@
+# coding: utf-8
+from __future__ import unicode_literals
+import re
+import json
+import time
+from .common import InfoExtractor
+
+
+class GamersydeIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?gamersyde\.com/hqstream_'
+    _TEST = {
+        'url': 'http://www.gamersyde.com/hqstream_bloodborne_birth_of_a_hero-34371_en.html',
+        'md5': 'f38d400d32f19724570040d5ce3a505f',
+        'info_dict': {
+            'id': '34371',
+            'ext': 'mp4',
+            'title': 'Bloodborne - Birth of a hero',
+            'thumbnail': 're:^https?://.*\.jpg$',
+        }
+    }
+
+    def _calculateDuration(self, durationString):
+        duration = time.strptime(durationString, "%M minutes %S seconds")
+        return duration.tm_min * 60 + duration.tm_sec
+
+    def _fixJsonSyntax(self, json):
+
+        json = re.sub(r"{\s*(\w)", r'{"\1', json)
+        json = re.sub(r",\s*(\w)", r',"\1', json)
+        json = re.sub(r"(\w): ", r'\1":', json)
+        json = re.sub(r",\s*}", "}", json, flags=re.DOTALL)
+        json = re.sub(r",\s*]", "]", json, flags=re.DOTALL)
+
+        return json
+
+    def _real_extract(self, url):
+
+        video_id = self._search_regex(r'-(.*?)_[a-z]{2}.html$', url, 'video_id')
+        webpage = self._download_webpage(url, video_id)
+
+        filesJson = self._search_regex(r'playlist: (.*?)\}\);', webpage, 'files', flags=re.DOTALL)
+        filesJson = self._fixJsonSyntax(filesJson)
+
+        data = json.loads(filesJson)
+        playlist = data[0]
+
+        formats = []
+
+        title = re.sub(r"[0-9]+ - ", "", playlist['title'])
+
+        for playlistEntry in playlist['sources']:
+            format = {
+                'url': playlistEntry['file'],
+                'format_id': playlistEntry['label']
+            }
+
+            formats.append(format)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': formats,
+            'thumbnail': playlist['image']
+            }

From 185a7e25e7c18b0dff17bdb9ae828616a5ac17d4 Mon Sep 17 00:00:00 2001
From: Mohammad Teimori Pabandi <mtp1376@gmail.com>
Date: Fri, 3 Apr 2015 20:55:39 +0430
Subject: [PATCH 0831/6219] [RadioJavan] Add new extractor

---
 youtube_dl/extractor/__init__.py   |  1 +
 youtube_dl/extractor/radiojavan.py | 71 ++++++++++++++++++++++++++++++
 2 files changed, 72 insertions(+)
 create mode 100644 youtube_dl/extractor/radiojavan.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 3011b784d..df4a7419a 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -391,6 +391,7 @@ from .pyvideo import PyvideoIE
 from .quickvid import QuickVidIE
 from .r7 import R7IE
 from .radiode import RadioDeIE
+from .radiojavan import RadioJavanIE
 from .radiobremen import RadioBremenIE
 from .radiofrance import RadioFranceIE
 from .rai import RaiIE
diff --git a/youtube_dl/extractor/radiojavan.py b/youtube_dl/extractor/radiojavan.py
new file mode 100644
index 000000000..de90f9270
--- /dev/null
+++ b/youtube_dl/extractor/radiojavan.py
@@ -0,0 +1,71 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import(
+    parse_duration,
+    str_to_int
+)
+
+class RadioJavanIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?radiojavan\.com/videos/video/(?P<id>[^/]+)/?'
+    _TEST = {
+        'url': 'http://www.radiojavan.com/videos/video/chaartaar-ashoobam',
+        'md5': 'e85208ffa3ca8b83534fca9fe19af95b',
+        'info_dict': {
+            'id': 'chaartaar-ashoobam',
+            'ext': 'mp4',
+            'title': 'Chaartaar - Ashoobam',
+            'description': 'Chaartaar - Ashoobam',
+            'thumbnail': 're:^https?://.*\.jpe?g$',
+        }
+    }
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+        urls = list()
+        prefix = 'https://media.rdjavan.com/media/music_video/'
+
+        video_url_480 = self._search_regex(
+            r'RJ\.video480p = \'([^\']+)\'', webpage, '480 video url', fatal= False)
+        video_url_720 = self._search_regex(
+            r'RJ\.video720p = \'([^\']+)\'', webpage, '720 video url', fatal= False)
+        video_url_1080 = self._search_regex(
+            r'RJ\.video1080p = \'([^\']+)\'', webpage, '1080 video url', fatal= False)
+
+        if video_url_480:
+            urls.append({'url': prefix + video_url_480, 'format': '480p'})
+        if video_url_720:
+            urls.append({'url': prefix + video_url_720, 'format': '720p'})
+        if video_url_1080:
+            urls.append({'url': prefix + video_url_1080, 'format': '1080p'})
+
+        title = self._og_search_title(webpage)
+        thumbnail = self._og_search_thumbnail(webpage)
+        formats = [{
+            'url': url['url'],
+            'format': url['format']
+        } for url in urls]
+
+        likes = self._search_regex(
+            r'<span class="rating">([\d,]+)\s*likes</span>', webpage, 'Likes Count', fatal=False )
+        likes = likes.replace(',', '')
+        dislikes = self._search_regex(
+            r'<span class="rating">([\d,]+)\s*dislikes</span>', webpage, 'Dislikes Count', fatal=False )
+        dislikes = dislikes.replace(',', '')
+
+        plays = self._search_regex(
+            r'views_publish[">\s]*<span[^>]+class="views">Plays: ([\d,]+)</span>', webpage, 'Play Count', fatal=False )
+        plays = plays.replace(',', '')
+
+        return {
+            'formats': formats,
+            'id': display_id,
+            'title': title,
+            'description': title, # no description provided in RadioJavan
+            'thumbnail': thumbnail,
+            'like_count': str_to_int(likes),
+            'dislike_count': str_to_int(dislikes),
+            'viewCount': str_to_int(plays)
+        }
\ No newline at end of file

From cd341b6e0679c11b3698191615b18dbaaf2b0a76 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 3 Apr 2015 19:37:35 +0200
Subject: [PATCH 0832/6219] [mixcloud] Fix extraction of like count (reported
 in #5231)

---
 youtube_dl/extractor/mixcloud.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dl/extractor/mixcloud.py
index 21aea0c55..84f291558 100644
--- a/youtube_dl/extractor/mixcloud.py
+++ b/youtube_dl/extractor/mixcloud.py
@@ -97,7 +97,7 @@ class MixcloudIE(InfoExtractor):
             r'\s+"profile": "([^"]+)",', webpage, 'uploader id', fatal=False)
         description = self._og_search_description(webpage)
         like_count = str_to_int(self._search_regex(
-            r'\bbutton-favorite\b.+m-ajax-toggle-count="([^"]+)"',
+            r'\bbutton-favorite\b[^>]+m-ajax-toggle-count="([^"]+)"',
             webpage, 'like count', fatal=False))
         view_count = str_to_int(self._search_regex(
             [r'<meta itemprop="interactionCount" content="UserPlays:([0-9]+)"',

From 01534bf54f7eb80e4414cba86f1ac6fa608fc6e0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 3 Apr 2015 23:42:53 +0600
Subject: [PATCH 0833/6219] [prosiebensat1] Fix bitrate (Closes #5350 closes
 #5351)

---
 youtube_dl/extractor/prosiebensat1.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/prosiebensat1.py b/youtube_dl/extractor/prosiebensat1.py
index 385681d06..c46aaada6 100644
--- a/youtube_dl/extractor/prosiebensat1.py
+++ b/youtube_dl/extractor/prosiebensat1.py
@@ -10,6 +10,7 @@ from ..compat import (
 )
 from ..utils import (
     unified_strdate,
+    int_or_none,
 )
 
 
@@ -266,6 +267,9 @@ class ProSiebenSat1IE(InfoExtractor):
             urls_sources = urls_sources.values()
 
         def fix_bitrate(bitrate):
+            bitrate = int_or_none(bitrate)
+            if not bitrate:
+                return None
             return (bitrate // 1000) if bitrate % 1000 == 0 else bitrate
 
         for source in urls_sources:

From 16fa01291bd94703e2258a68bef1491d57f0dabc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 3 Apr 2015 23:44:13 +0600
Subject: [PATCH 0834/6219] [prosiebensat1] Fix test

---
 youtube_dl/extractor/prosiebensat1.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/prosiebensat1.py b/youtube_dl/extractor/prosiebensat1.py
index c46aaada6..7cc799664 100644
--- a/youtube_dl/extractor/prosiebensat1.py
+++ b/youtube_dl/extractor/prosiebensat1.py
@@ -25,7 +25,7 @@ class ProSiebenSat1IE(InfoExtractor):
             'info_dict': {
                 'id': '2104602',
                 'ext': 'mp4',
-                'title': 'Staffel 2, Episode 18 - Jahresrückblick',
+                'title': 'Episode 18 - Staffel 2',
                 'description': 'md5:8733c81b702ea472e069bc48bb658fc1',
                 'upload_date': '20131231',
                 'duration': 5845.04,

From ff556f5c09ec8700bb012a58a5e39505b887b774 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Apr 2015 00:30:37 +0600
Subject: [PATCH 0835/6219] Do not encode outtmpl twice (Closes #5288)

---
 youtube_dl/__init__.py | 4 ----
 youtube_dl/options.py  | 3 ++-
 2 files changed, 2 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 852b2fc3d..1c8b411b7 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -189,10 +189,6 @@ def _real_main(argv=None):
     if opts.allsubtitles and not opts.writeautomaticsub:
         opts.writesubtitles = True
 
-    if sys.version_info < (3,):
-        # In Python 2, sys.argv is a bytestring (also note http://bugs.python.org/issue2128 for Windows systems)
-        if opts.outtmpl is not None:
-            opts.outtmpl = opts.outtmpl.decode(preferredencoding())
     outtmpl = ((opts.outtmpl is not None and opts.outtmpl) or
                (opts.format == '-1' and opts.usetitle and '%(title)s-%(id)s-%(format)s.%(ext)s') or
                (opts.format == '-1' and '%(id)s-%(format)s.%(ext)s') or
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 35c7e5fb3..8e80e3759 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -13,6 +13,7 @@ from .compat import (
     compat_kwargs,
 )
 from .utils import (
+    preferredencoding,
     write_string,
 )
 from .version import __version__
@@ -797,7 +798,7 @@ def parseOpts(overrideArguments=None):
         # Workaround for Python 2.x, where argv is a byte list
         if sys.version_info < (3,):
             command_line_conf = [
-                a.decode('utf-8', 'replace') for a in command_line_conf]
+                a.decode(preferredencoding(), 'replace') for a in command_line_conf]
 
         if '--ignore-config' in command_line_conf:
             system_conf = []

From 115c281672bd7479f87c48249f6a0186ac7d19cc Mon Sep 17 00:00:00 2001
From: snipem <mail@matthias-kuech.de>
Date: Sat, 4 Apr 2015 12:31:48 +0200
Subject: [PATCH 0836/6219] [Gamersyde] Improved robustness, added duration and
 tests

Fix for Json syntax is now less error prone for Json syntax inside of
values. Extractor is now also using native Json handling. Added tests
for several videos that were producing errors in the first place.
---
 youtube_dl/extractor/gamersyde.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/gamersyde.py b/youtube_dl/extractor/gamersyde.py
index c40106216..5c68a6891 100644
--- a/youtube_dl/extractor/gamersyde.py
+++ b/youtube_dl/extractor/gamersyde.py
@@ -8,7 +8,6 @@ from .common import InfoExtractor
 
 class GamersydeIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?gamersyde\.com/hqstream_'
-    _TEST = {
         'url': 'http://www.gamersyde.com/hqstream_bloodborne_birth_of_a_hero-34371_en.html',
         'md5': 'f38d400d32f19724570040d5ce3a505f',
         'info_dict': {
@@ -17,6 +16,11 @@ class GamersydeIE(InfoExtractor):
             'title': 'Bloodborne - Birth of a hero',
             'thumbnail': 're:^https?://.*\.jpg$',
         }
+    },
+    {
+        'url': 'http://www.gamersyde.com/hqstream_dark_souls_ii_scholar_of_the_first_sin_gameplay_part_1-34417_en.html',
+        'info_dict': {
+            'ext': 'mp4',
     }
 
     def _calculateDuration(self, durationString):
@@ -27,7 +31,6 @@ class GamersydeIE(InfoExtractor):
 
         json = re.sub(r"{\s*(\w)", r'{"\1', json)
         json = re.sub(r",\s*(\w)", r',"\1', json)
-        json = re.sub(r"(\w): ", r'\1":', json)
         json = re.sub(r",\s*}", "}", json, flags=re.DOTALL)
         json = re.sub(r",\s*]", "]", json, flags=re.DOTALL)
 
@@ -40,7 +43,6 @@ class GamersydeIE(InfoExtractor):
 
         filesJson = self._search_regex(r'playlist: (.*?)\}\);', webpage, 'files', flags=re.DOTALL)
         filesJson = self._fixJsonSyntax(filesJson)
-
         data = json.loads(filesJson)
         playlist = data[0]
 

From 3d24d997ae1f92686aa7edd0bfeed28353fbfb2e Mon Sep 17 00:00:00 2001
From: snipem <mail@matthias-kuech.de>
Date: Sat, 4 Apr 2015 12:42:14 +0200
Subject: [PATCH 0837/6219] Fixed intendation of test cases

Leaded to error on Linux machine
---
 youtube_dl/extractor/gamersyde.py | 45 ++++++++++++++++++++++++-------
 1 file changed, 36 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/gamersyde.py b/youtube_dl/extractor/gamersyde.py
index 5c68a6891..cc6fa4037 100644
--- a/youtube_dl/extractor/gamersyde.py
+++ b/youtube_dl/extractor/gamersyde.py
@@ -1,39 +1,62 @@
 # coding: utf-8
 from __future__ import unicode_literals
 import re
-import json
 import time
+
 from .common import InfoExtractor
 
 
 class GamersydeIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?gamersyde\.com/hqstream_'
+    _TESTS = [{
         'url': 'http://www.gamersyde.com/hqstream_bloodborne_birth_of_a_hero-34371_en.html',
         'md5': 'f38d400d32f19724570040d5ce3a505f',
         'info_dict': {
             'id': '34371',
             'ext': 'mp4',
+            'duration': 372,
             'title': 'Bloodborne - Birth of a hero',
             'thumbnail': 're:^https?://.*\.jpg$',
         }
-    },
-    {
+    }, {
         'url': 'http://www.gamersyde.com/hqstream_dark_souls_ii_scholar_of_the_first_sin_gameplay_part_1-34417_en.html',
+        'md5': '94bd7c3feff3275576cf5cb6c8a3a720',
         'info_dict': {
+            'id': '34417',
             'ext': 'mp4',
+            'duration': 270,
+            'title': 'Dark Souls II: Scholar of the First Sin - Gameplay - Part 1',
+            'thumbnail': 're:^https?://.*\.jpg$',
+        }
+    }, {
+        'url': 'http://www.gamersyde.com/hqstream_grand_theft_auto_v_heists_trailer-33786_en.html',
+        'md5': '65e442f5f340d571ece8c80d50700369',
+        'info_dict': {
+            'id': '33786',
+            'ext': 'mp4',
+            'duration': 59,
+            'title': 'Grand Theft Auto V - Heists Trailer',
+            'thumbnail': 're:^https?://.*\.jpg$',
+        }
     }
+    ]
 
     def _calculateDuration(self, durationString):
-        duration = time.strptime(durationString, "%M minutes %S seconds")
+        if (durationString.find("minutes") > -1):
+            duration = time.strptime(durationString, "%M minutes %S seconds")
+        else:
+            duration = time.strptime(durationString, "%S seconds")
         return duration.tm_min * 60 + duration.tm_sec
 
     def _fixJsonSyntax(self, json):
 
-        json = re.sub(r"{\s*(\w)", r'{"\1', json)
-        json = re.sub(r",\s*(\w)", r',"\1', json)
         json = re.sub(r",\s*}", "}", json, flags=re.DOTALL)
         json = re.sub(r",\s*]", "]", json, flags=re.DOTALL)
-
+        json = json.replace('file: "', '"file": "')
+        json = json.replace('title: "', '"title": "')
+        json = json.replace('label: "', '"label": "')
+        json = json.replace('image: "', '"image": "')
+        json = json.replace('sources: [', '"sources": [')
         return json
 
     def _real_extract(self, url):
@@ -42,13 +65,16 @@ class GamersydeIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         filesJson = self._search_regex(r'playlist: (.*?)\}\);', webpage, 'files', flags=re.DOTALL)
-        filesJson = self._fixJsonSyntax(filesJson)
-        data = json.loads(filesJson)
+        data = self._parse_json(filesJson,video_id, transform_source=self._fixJsonSyntax)
+        
         playlist = data[0]
 
         formats = []
 
         title = re.sub(r"[0-9]+ - ", "", playlist['title'])
+        
+        length = self._search_regex(r'(([0-9]{1,2} minutes ){0,1}[0-9]{1,2} seconds)', webpage, 'length')
+        duration = self._calculateDuration(length)
 
         for playlistEntry in playlist['sources']:
             format = {
@@ -62,5 +88,6 @@ class GamersydeIE(InfoExtractor):
             'id': video_id,
             'title': title,
             'formats': formats,
+            'duration': duration,
             'thumbnail': playlist['image']
             }

From 7cf97daf77f6419f2b965a199a3fb1e63b8771b5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Apr 2015 16:45:41 +0600
Subject: [PATCH 0838/6219] [radiojavan] Simplify and extract upload date

---
 youtube_dl/extractor/radiojavan.py | 75 ++++++++++++++----------------
 1 file changed, 35 insertions(+), 40 deletions(-)

diff --git a/youtube_dl/extractor/radiojavan.py b/youtube_dl/extractor/radiojavan.py
index de90f9270..73ab78d6d 100644
--- a/youtube_dl/extractor/radiojavan.py
+++ b/youtube_dl/extractor/radiojavan.py
@@ -1,12 +1,14 @@
-# coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..utils import(
-    parse_duration,
-    str_to_int
+    unified_strdate,
+    str_to_int,
 )
 
+
 class RadioJavanIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?radiojavan\.com/videos/video/(?P<id>[^/]+)/?'
     _TEST = {
@@ -16,56 +18,49 @@ class RadioJavanIE(InfoExtractor):
             'id': 'chaartaar-ashoobam',
             'ext': 'mp4',
             'title': 'Chaartaar - Ashoobam',
-            'description': 'Chaartaar - Ashoobam',
             'thumbnail': 're:^https?://.*\.jpe?g$',
+            'upload_date': '20150215',
+            'view_count': int,
+            'like_count': int,
+            'dislike_count': int,
         }
     }
 
     def _real_extract(self, url):
-        display_id = self._match_id(url)
-        webpage = self._download_webpage(url, display_id)
-        urls = list()
-        prefix = 'https://media.rdjavan.com/media/music_video/'
+        video_id = self._match_id(url)
 
-        video_url_480 = self._search_regex(
-            r'RJ\.video480p = \'([^\']+)\'', webpage, '480 video url', fatal= False)
-        video_url_720 = self._search_regex(
-            r'RJ\.video720p = \'([^\']+)\'', webpage, '720 video url', fatal= False)
-        video_url_1080 = self._search_regex(
-            r'RJ\.video1080p = \'([^\']+)\'', webpage, '1080 video url', fatal= False)
+        webpage = self._download_webpage(url, video_id)
 
-        if video_url_480:
-            urls.append({'url': prefix + video_url_480, 'format': '480p'})
-        if video_url_720:
-            urls.append({'url': prefix + video_url_720, 'format': '720p'})
-        if video_url_1080:
-            urls.append({'url': prefix + video_url_1080, 'format': '1080p'})
+        formats = [{
+            'url': 'https://media.rdjavan.com/media/music_video/%s' % video_path,
+            'format_id': '%sp' % height,
+            'height': height,
+        } for height, video_path in re.findall(r"RJ\.video(\d+)p\s*=\s*'/?([^']+)'", webpage)]
 
         title = self._og_search_title(webpage)
         thumbnail = self._og_search_thumbnail(webpage)
-        formats = [{
-            'url': url['url'],
-            'format': url['format']
-        } for url in urls]
 
-        likes = self._search_regex(
-            r'<span class="rating">([\d,]+)\s*likes</span>', webpage, 'Likes Count', fatal=False )
-        likes = likes.replace(',', '')
-        dislikes = self._search_regex(
-            r'<span class="rating">([\d,]+)\s*dislikes</span>', webpage, 'Dislikes Count', fatal=False )
-        dislikes = dislikes.replace(',', '')
+        upload_date = unified_strdate(self._search_regex(
+            r'class="date_added">Date added: ([^<]+)<',
+            webpage, 'upload date', fatal=False))
 
-        plays = self._search_regex(
-            r'views_publish[">\s]*<span[^>]+class="views">Plays: ([\d,]+)</span>', webpage, 'Play Count', fatal=False )
-        plays = plays.replace(',', '')
+        view_count = str_to_int(self._search_regex(
+            r'class="views">Plays: ([\d,]+)',
+            webpage, 'view count', fatal=False))
+        like_count = str_to_int(self._search_regex(
+            r'class="rating">([\d,]+) likes',
+            webpage, 'like count', fatal=False))
+        dislike_count = str_to_int(self._search_regex(
+            r'class="rating">([\d,]+) dislikes',
+            webpage, 'dislike count', fatal=False))
 
         return {
-            'formats': formats,
-            'id': display_id,
+            'id': video_id,
             'title': title,
-            'description': title, # no description provided in RadioJavan
             'thumbnail': thumbnail,
-            'like_count': str_to_int(likes),
-            'dislike_count': str_to_int(dislikes),
-            'viewCount': str_to_int(plays)
-        }
\ No newline at end of file
+            'upload_date': upload_date,
+            'view_count': view_count,
+            'like_count': like_count,
+            'dislike_count': dislike_count,
+            'formats': formats,
+        }

From 6e617ed0b6b5bb932f928f63c2bda36f5317468d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Apr 2015 16:47:09 +0600
Subject: [PATCH 0839/6219] Credit @mtp1376 for varzesh3 and radiojavan

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 48769320a..cf238176b 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -120,3 +120,4 @@ Jeff Buchbinder
 Amish Bhadeshia
 Joram Schrijver
 Will W.
+Mohammad Teimori Pabandi

From e9f65f87496d740fbb61e036c710bf2c174f1cc0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 4 Apr 2015 13:11:55 +0200
Subject: [PATCH 0840/6219] [rtve] Extract a better quality video

---
 youtube_dl/extractor/rtve.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/rtve.py b/youtube_dl/extractor/rtve.py
index 13f071077..8d9be1b98 100644
--- a/youtube_dl/extractor/rtve.py
+++ b/youtube_dl/extractor/rtve.py
@@ -10,6 +10,7 @@ from ..compat import compat_urlparse
 from ..utils import (
     float_or_none,
     remove_end,
+    std_headers,
     struct_unpack,
 )
 
@@ -84,13 +85,20 @@ class RTVEALaCartaIE(InfoExtractor):
         'only_matching': True,
     }]
 
+    def _real_initialize(self):
+        user_agent_b64 = base64.b64encode(std_headers['User-Agent'].encode('utf-8')).decode('utf-8')
+        manager_info = self._download_json(
+            'http://www.rtve.es/odin/loki/' + user_agent_b64,
+            None, 'Fetching manager info')
+        self._manager = manager_info['manager']
+
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
         info = self._download_json(
             'http://www.rtve.es/api/videos/%s/config/alacarta_videos.json' % video_id,
             video_id)['page']['items'][0]
-        png_url = 'http://www.rtve.es/ztnr/movil/thumbnail/default/videos/%s.png' % video_id
+        png_url = 'http://www.rtve.es/ztnr/movil/thumbnail/%s/videos/%s.png' % (self._manager, video_id)
         png = self._download_webpage(png_url, video_id, 'Downloading url information')
         video_url = _decrypt_url(png)
         if not video_url.endswith('.f4m'):

From ba9e68f40261355ceae5bb87c5707adc7f7beb2e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Apr 2015 17:48:55 +0600
Subject: [PATCH 0841/6219] [utils] Drop trailing comma before closing brace

---
 test/test_utils.py  | 6 ++++++
 youtube_dl/utils.py | 2 +-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index abaf1ab73..4e524aca3 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -470,6 +470,12 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(d['x'], 1)
         self.assertEqual(d['y'], 'a')
 
+        on = js_to_json('["abc", "def",]')
+        self.assertEqual(json.loads(on), ['abc', 'def'])
+
+        on = js_to_json('{"abc": "def",}')
+        self.assertEqual(json.loads(on), {'abc': 'def'})
+
     def test_clean_html(self):
         self.assertEqual(clean_html('a:\nb'), 'a: b')
         self.assertEqual(clean_html('a:\n   "b"'), 'a:    "b"')
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 90e0ed9ab..e1761265c 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1577,7 +1577,7 @@ def js_to_json(code):
         '(?:[^'\\]*(?:\\\\|\\['"nu]))*[^'\\]*'|
         [a-zA-Z_][.a-zA-Z_0-9]*
         ''', fix_kv, code)
-    res = re.sub(r',(\s*\])', lambda m: m.group(1), res)
+    res = re.sub(r',(\s*[\]}])', lambda m: m.group(1), res)
     return res
 
 

From 5c29dbd0c76083eaf596f623fabb612575f71861 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Apr 2015 17:53:22 +0600
Subject: [PATCH 0842/6219] [gamersyde] Simplify

---
 youtube_dl/extractor/gamersyde.py | 103 ++++++++++++------------------
 1 file changed, 40 insertions(+), 63 deletions(-)

diff --git a/youtube_dl/extractor/gamersyde.py b/youtube_dl/extractor/gamersyde.py
index cc6fa4037..d545e01bb 100644
--- a/youtube_dl/extractor/gamersyde.py
+++ b/youtube_dl/extractor/gamersyde.py
@@ -1,14 +1,18 @@
-# coding: utf-8
 from __future__ import unicode_literals
+
 import re
-import time
 
 from .common import InfoExtractor
+from ..utils import (
+    js_to_json,
+    parse_duration,
+    remove_start,
+)
 
 
 class GamersydeIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?gamersyde\.com/hqstream_'
-    _TESTS = [{
+    _VALID_URL = r'https?://(?:www\.)?gamersyde\.com/hqstream_(?P<display_id>[\da-z_]+)-(?P<id>\d+)_[a-z]{2}\.html'
+    _TEST = {
         'url': 'http://www.gamersyde.com/hqstream_bloodborne_birth_of_a_hero-34371_en.html',
         'md5': 'f38d400d32f19724570040d5ce3a505f',
         'info_dict': {
@@ -18,76 +22,49 @@ class GamersydeIE(InfoExtractor):
             'title': 'Bloodborne - Birth of a hero',
             'thumbnail': 're:^https?://.*\.jpg$',
         }
-    }, {
-        'url': 'http://www.gamersyde.com/hqstream_dark_souls_ii_scholar_of_the_first_sin_gameplay_part_1-34417_en.html',
-        'md5': '94bd7c3feff3275576cf5cb6c8a3a720',
-        'info_dict': {
-            'id': '34417',
-            'ext': 'mp4',
-            'duration': 270,
-            'title': 'Dark Souls II: Scholar of the First Sin - Gameplay - Part 1',
-            'thumbnail': 're:^https?://.*\.jpg$',
-        }
-    }, {
-        'url': 'http://www.gamersyde.com/hqstream_grand_theft_auto_v_heists_trailer-33786_en.html',
-        'md5': '65e442f5f340d571ece8c80d50700369',
-        'info_dict': {
-            'id': '33786',
-            'ext': 'mp4',
-            'duration': 59,
-            'title': 'Grand Theft Auto V - Heists Trailer',
-            'thumbnail': 're:^https?://.*\.jpg$',
-        }
     }
-    ]
-
-    def _calculateDuration(self, durationString):
-        if (durationString.find("minutes") > -1):
-            duration = time.strptime(durationString, "%M minutes %S seconds")
-        else:
-            duration = time.strptime(durationString, "%S seconds")
-        return duration.tm_min * 60 + duration.tm_sec
-
-    def _fixJsonSyntax(self, json):
-
-        json = re.sub(r",\s*}", "}", json, flags=re.DOTALL)
-        json = re.sub(r",\s*]", "]", json, flags=re.DOTALL)
-        json = json.replace('file: "', '"file": "')
-        json = json.replace('title: "', '"title": "')
-        json = json.replace('label: "', '"label": "')
-        json = json.replace('image: "', '"image": "')
-        json = json.replace('sources: [', '"sources": [')
-        return json
 
     def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        display_id = mobj.group('display_id')
 
-        video_id = self._search_regex(r'-(.*?)_[a-z]{2}.html$', url, 'video_id')
-        webpage = self._download_webpage(url, video_id)
+        webpage = self._download_webpage(url, display_id)
 
-        filesJson = self._search_regex(r'playlist: (.*?)\}\);', webpage, 'files', flags=re.DOTALL)
-        data = self._parse_json(filesJson,video_id, transform_source=self._fixJsonSyntax)
-        
-        playlist = data[0]
+        playlist = self._parse_json(
+            self._search_regex(
+                r'(?s)playlist: \[({.+?})\]\s*}\);', webpage, 'files'),
+            display_id, transform_source=js_to_json)
 
         formats = []
-
-        title = re.sub(r"[0-9]+ - ", "", playlist['title'])
-        
-        length = self._search_regex(r'(([0-9]{1,2} minutes ){0,1}[0-9]{1,2} seconds)', webpage, 'length')
-        duration = self._calculateDuration(length)
-
-        for playlistEntry in playlist['sources']:
-            format = {
-                'url': playlistEntry['file'],
-                'format_id': playlistEntry['label']
+        for source in playlist['sources']:
+            video_url = source.get('file')
+            if not video_url:
+                continue
+            format_id = source.get('label')
+            f = {
+                'url': video_url,
+                'format_id': format_id,
             }
+            m = re.search(r'^(?P<height>\d+)[pP](?P<fps>\d+)fps', format_id)
+            if m:
+                f.update({
+                    'height': int(m.group('height')),
+                    'fps': int(m.group('fps')),
+                })
+            formats.append(f)
+        self._sort_formats(formats)
 
-            formats.append(format)
+        title = remove_start(playlist['title'], '%s - ' % video_id)
+        thumbnail = playlist.get('image')
+        duration = parse_duration(self._search_regex(
+            r'Length:</label>([^<]+)<', webpage, 'duration', fatal=False))
 
         return {
             'id': video_id,
+            'display_id': display_id,
             'title': title,
-            'formats': formats,
+            'thumbnail': thumbnail,
             'duration': duration,
-            'thumbnail': playlist['image']
-            }
+            'formats': formats,
+        }

From 79c21abba7c9902f00ddac83a2af29c36fe0e122 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Apr 2015 18:45:46 +0600
Subject: [PATCH 0843/6219] [utils] Add one more template to unified_strdate

---
 youtube_dl/utils.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index e1761265c..be3f62da7 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -759,6 +759,7 @@ def unified_strdate(date_str, day_first=True):
     ]
     if day_first:
         format_expressions.extend([
+            '%d-%m-%Y',
             '%d.%m.%Y',
             '%d/%m/%Y',
             '%d/%m/%y',
@@ -766,6 +767,7 @@ def unified_strdate(date_str, day_first=True):
         ])
     else:
         format_expressions.extend([
+            '%m-%d-%Y',
             '%m.%d.%Y',
             '%m/%d/%Y',
             '%m/%d/%y',

From 15ac8413c78b991f2e99b6bdc538bc8c5ae8e8a0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Apr 2015 19:08:48 +0600
Subject: [PATCH 0844/6219] [utils] Avoid treating `*-%Y` date template as UTC
 offset

---
 youtube_dl/utils.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index be3f62da7..52f0dd09a 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -730,7 +730,8 @@ def unified_strdate(date_str, day_first=True):
     # Replace commas
     date_str = date_str.replace(',', ' ')
     # %z (UTC offset) is only supported in python>=3.2
-    date_str = re.sub(r' ?(\+|-)[0-9]{2}:?[0-9]{2}$', '', date_str)
+    if not re.match(r'^[0-9]{1,2}-[0-9]{1,2}-[0-9]{4}$', date_str):
+        date_str = re.sub(r' ?(\+|-)[0-9]{2}:?[0-9]{2}$', '', date_str)
     # Remove AM/PM + timezone
     date_str = re.sub(r'(?i)\s*(?:AM|PM)(?:\s+[A-Z]+)?', '', date_str)
 

From 8cf70de428c3fef910ba966fb56d39478226acc8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Apr 2015 19:11:01 +0600
Subject: [PATCH 0845/6219] [test_utils] Add test for unified_strdate

---
 test/test_utils.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/test/test_utils.py b/test/test_utils.py
index 4e524aca3..2e3a6480c 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -227,6 +227,7 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(
             unified_strdate('2/2/2015 6:47:40 PM', day_first=False),
             '20150202')
+        self.assertEqual(unified_strdate('25-09-2014'), '20140925')
 
     def test_find_xpath_attr(self):
         testxml = '''<root>

From 7c39a65543b809b681434246b84710349f5837aa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Apr 2015 19:13:37 +0600
Subject: [PATCH 0846/6219] [pornovoisines] Simplify

---
 youtube_dl/extractor/pornovoisines.py | 111 ++++++++++++--------------
 1 file changed, 53 insertions(+), 58 deletions(-)

diff --git a/youtube_dl/extractor/pornovoisines.py b/youtube_dl/extractor/pornovoisines.py
index efbb6a818..9688ed948 100644
--- a/youtube_dl/extractor/pornovoisines.py
+++ b/youtube_dl/extractor/pornovoisines.py
@@ -2,19 +2,23 @@
 from __future__ import unicode_literals
 
 import re
-import datetime
 import random
 
-from ..compat import compat_urllib_parse
 from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    float_or_none,
+    unified_strdate,
+)
+
 
 class PornoVoisinesIE(InfoExtractor):
-    _VALID_URL = r'^((?:http://)?(?:www\.)?pornovoisines.com)/showvideo/(\d+)/([^/]+)'
+    _VALID_URL = r'http://(?:www\.)?pornovoisines\.com/showvideo/(?P<id>\d+)/(?P<display_id>[^/]+)'
 
-    VIDEO_URL_TEMPLATE = 'http://stream%d.pornovoisines.com' \
+    _VIDEO_URL_TEMPLATE = 'http://stream%d.pornovoisines.com' \
         '/static/media/video/transcoded/%s-640x360-1000-trscded.mp4'
 
-    SERVER_NUMBERS = (1, 2)
+    _SERVER_NUMBERS = (1, 2)
 
     _TEST = {
         'url': 'http://www.pornovoisines.com/showvideo/1285/recherche-appartement/',
@@ -23,79 +27,70 @@ class PornoVoisinesIE(InfoExtractor):
             'id': '1285',
             'display_id': 'recherche-appartement',
             'ext': 'mp4',
-            'title': "Recherche appartement",
+            'title': 'Recherche appartement',
+            'description': 'md5:819ea0b785e2a04667a1a01cdc89594e',
+            'thumbnail': 're:^https?://.*\.jpg$',
             'upload_date': '20140925',
-            'view_count': int,
             'duration': 120,
-            'categories': ["Débutante", "Scénario", "Sodomie"],
-            'description': 're:^Pour la .+ original...$',
-            'thumbnail': 're:^http://',
-            'uploader': "JMTV",
+            'view_count': int,
             'average_rating': float,
-            'comment_count': int,
+            'categories': ['Débutante', 'Scénario', 'Sodomie'],
             'age_limit': 18,
         }
     }
 
     @classmethod
-    def build_video_url(cls, id):
-        server_nr = random.choice(cls.SERVER_NUMBERS)
-        return cls.VIDEO_URL_TEMPLATE % (server_nr, id)
-
-    @staticmethod
-    def parse_upload_date(str):
-        return datetime.datetime.strptime(str, "%d-%m-%Y").strftime("%Y%m%d")
-
-    @staticmethod
-    def parse_categories(str):
-        return map(lambda s: s.strip(), str.split(','))
+    def build_video_url(cls, num):
+        return cls._VIDEO_URL_TEMPLATE % (random.choice(cls._SERVER_NUMBERS), num)
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        url_prefix = mobj.group(1)
-        id = mobj.group(2)
-        display_id = mobj.group(3)
+        video_id = mobj.group('id')
+        display_id = mobj.group('display_id')
 
-        webpage = self._download_webpage(url, id)
+        webpage = self._download_webpage(url, video_id)
 
-        title = self._html_search_regex(r'<h1>(.+?)</h1>', webpage, 'title',
-            flags=re.DOTALL)
-        url = self.build_video_url(id)
-        upload_date = self.parse_upload_date(
-            self._search_regex(r'Publié le (\d\d-\d\d-\d{4})', webpage,
-            'upload date'))
-        view_count = int(self._search_regex(r'(\d+) vues', webpage, 'view count'))
-        duration = int(self._search_regex('Durée (\d+)', webpage, 'duration'))
-        categories = self.parse_categories(self._html_search_regex(
-            r'<li class="categorie">(.+?)</li>', webpage, "categories",
-            flags=re.DOTALL))
+        video_url = self.build_video_url(video_id)
+
+        title = self._html_search_regex(
+            r'<h1>(.+?)</h1>', webpage, 'title', flags=re.DOTALL)
         description = self._html_search_regex(
-            r'<article id="descriptif">(.+?)</article>', webpage, "description",
-            flags=re.DOTALL)
-        thumbnail = url_prefix + self._html_search_regex(re.compile(
-            '<div id="mediaspace' + id + '">.*?<img src="(.+?)"', re.DOTALL),
-            webpage, "thumbnail")
-        uploader = re.sub(r' *\| *$', '',
-            self._html_search_regex(r'<li class="auteur">(.+?)</li>', webpage,
-            "uploader", flags=re.DOTALL))
-        average_rating = float(self._search_regex(r'Note : (\d+,\d+)',
-            webpage, "average rating").replace(',', '.'))
-        comment_count = int(self._search_regex(r'\((\d+)\)', webpage,
-            "comment count"))
+            r'<article id="descriptif">(.+?)</article>',
+            webpage, "description", fatal=False, flags=re.DOTALL)
+
+        thumbnail = self._search_regex(
+            r'<div id="mediaspace%s">\s*<img src="/?([^"]+)"' % video_id,
+            webpage, 'thumbnail', fatal=False)
+        if thumbnail:
+            thumbnail = 'http://www.pornovoisines.com/%s' % thumbnail
+
+        upload_date = unified_strdate(self._search_regex(
+            r'Publié le ([\d-]+)', webpage, 'upload date', fatal=False))
+        duration = int_or_none(self._search_regex(
+            'Durée (\d+)', webpage, 'duration', fatal=False))
+        view_count = int_or_none(self._search_regex(
+            r'(\d+) vues', webpage, 'view count', fatal=False))
+        average_rating = self._search_regex(
+            r'Note : (\d+,\d+)', webpage, 'average rating', fatal=False)
+        if average_rating:
+            average_rating = float_or_none(average_rating.replace(',', '.'))
+
+        categories = self._html_search_meta(
+            'keywords', webpage, 'categories', fatal=False)
+        if categories:
+            categories = [category.strip() for category in categories.split(',')]
 
         return {
-            'id': id,
+            'id': video_id,
             'display_id': display_id,
-            'url': url,
+            'url': video_url,
             'title': title,
-            'upload_date': upload_date,
-            'view_count': view_count,
-            'duration': duration,
-            'categories': categories,
             'description': description,
             'thumbnail': thumbnail,
-            'uploader': uploader,
+            'upload_date': upload_date,
+            'duration': duration,
+            'view_count': view_count,
             'average_rating': average_rating,
-            'comment_count': comment_count,
+            'categories': categories,
             'age_limit': 18,
         }

From 424266abb10eafe6c57c5d391a947fa190a365e0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Apr 2015 19:16:18 +0600
Subject: [PATCH 0847/6219] Credit @Roman2K for pornovoisines (#5264)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index cf238176b..9c65dc1d4 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -121,3 +121,4 @@ Amish Bhadeshia
 Joram Schrijver
 Will W.
 Mohammad Teimori Pabandi
+Roman Le Négrate

From ff02a228e35ab11c9cfa6e0d000b7fd6de52a0c6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Apr 2015 19:21:50 +0600
Subject: [PATCH 0848/6219] [test_execution] Fix test under python 2 @ windows

---
 test/test_execution.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/test/test_execution.py b/test/test_execution.py
index f31e51558..620db080e 100644
--- a/test/test_execution.py
+++ b/test/test_execution.py
@@ -8,6 +8,9 @@ import unittest
 import sys
 import os
 import subprocess
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from youtube_dl.utils import encodeArgument
 
 rootDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 
@@ -31,7 +34,7 @@ class TestExecution(unittest.TestCase):
 
     def test_cmdline_umlauts(self):
         p = subprocess.Popen(
-            [sys.executable, 'youtube_dl/__main__.py', 'ä', '--version'],
+            [sys.executable, 'youtube_dl/__main__.py', encodeArgument('ä'), '--version'],
             cwd=rootDir, stdout=_DEV_NULL, stderr=subprocess.PIPE)
         _, stderr = p.communicate()
         self.assertFalse(stderr)

From 4e8cc1e973da2656c46c5df84d4e85c5d78836ac Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Apr 2015 19:24:37 +0600
Subject: [PATCH 0849/6219] [radiojavan] Fix height

---
 youtube_dl/extractor/radiojavan.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/radiojavan.py b/youtube_dl/extractor/radiojavan.py
index 73ab78d6d..c9eda9b53 100644
--- a/youtube_dl/extractor/radiojavan.py
+++ b/youtube_dl/extractor/radiojavan.py
@@ -34,7 +34,7 @@ class RadioJavanIE(InfoExtractor):
         formats = [{
             'url': 'https://media.rdjavan.com/media/music_video/%s' % video_path,
             'format_id': '%sp' % height,
-            'height': height,
+            'height': int(height),
         } for height, video_path in re.findall(r"RJ\.video(\d+)p\s*=\s*'/?([^']+)'", webpage)]
 
         title = self._og_search_title(webpage)

From 8fb2e5a4f5b9604f93964f9b6ae7062830e3bab2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Apr 2015 19:25:08 +0600
Subject: [PATCH 0850/6219] [radiojavan] Sort formats

---
 youtube_dl/extractor/radiojavan.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/radiojavan.py b/youtube_dl/extractor/radiojavan.py
index c9eda9b53..884c28420 100644
--- a/youtube_dl/extractor/radiojavan.py
+++ b/youtube_dl/extractor/radiojavan.py
@@ -36,6 +36,7 @@ class RadioJavanIE(InfoExtractor):
             'format_id': '%sp' % height,
             'height': int(height),
         } for height, video_path in re.findall(r"RJ\.video(\d+)p\s*=\s*'/?([^']+)'", webpage)]
+        self._sort_formats(formats)
 
         title = self._og_search_title(webpage)
         thumbnail = self._og_search_thumbnail(webpage)

From 4a3cdf81af9c22c45912b0b4c5845531d52d3a0c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Apr 2015 20:00:23 +0600
Subject: [PATCH 0851/6219] [options] Restore some strings

---
 youtube_dl/options.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 9bded4521..d861ac458 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -359,7 +359,7 @@ def parseOpts(overrideArguments=None):
     video_format.add_option(
         '--max-quality',
         action='store', dest='format_limit', metavar='FORMAT',
-        help='Specify highest quality format to download')
+        help='Highest quality format to download')
     video_format.add_option(
         '-F', '--list-formats',
         action='store_true', dest='listformats',
@@ -399,7 +399,7 @@ def parseOpts(overrideArguments=None):
     subtitles.add_option(
         '--sub-format',
         action='store', dest='subtitlesformat', metavar='FORMAT', default='best',
-        help='Specify subtitle format preference, for example: "srt" or "ass/srt/best"')
+        help='Subtitle format, accepts formats preference, for example: "srt" or "ass/srt/best"')
     subtitles.add_option(
         '--sub-lang', '--sub-langs', '--srt-lang',
         action='callback', dest='subtitleslangs', metavar='LANGS', type='str',

From f01855813b364dbd3e0c7fecacda84410d2780bf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Apr 2015 20:01:24 +0600
Subject: [PATCH 0852/6219] [options] extractor is lowercase

---
 youtube_dl/options.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index d861ac458..ed2216d40 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -606,7 +606,7 @@ def parseOpts(overrideArguments=None):
               '%(format)s for the format description (like "22 - 1280x720" or "HD"), '
               '%(format_id)s for the unique id of the format (like YouTube\'s itags: "137"), '
               '%(upload_date)s for the upload date (YYYYMMDD), '
-              '%(extractor)s for the provider (YouTube, metacafe, etc), '
+              '%(extractor)s for the provider (youtube, metacafe, etc), '
               '%(id)s for the video id, '
               '%(playlist_title)s, %(playlist_id)s, or %(playlist)s (=title if present, ID otherwise) for the playlist the video is in, '
               '%(playlist_index)s for the position in the playlist. '

From 6b70a4eb7d4bcbe6812f78876b4aa9aa44a58fef Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Apr 2015 20:02:29 +0600
Subject: [PATCH 0853/6219] [options] `Number` is a verb here

---
 youtube_dl/options.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index ed2216d40..2097a9436 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -626,7 +626,7 @@ def parseOpts(overrideArguments=None):
     filesystem.add_option(
         '-A', '--auto-number',
         action='store_true', dest='autonumber', default=False,
-        help='[deprecated; use  -o "%(autonumber)s-%(title)s.%(ext)s" ] Number of downloaded files starting from 00000')
+        help='[deprecated; use  -o "%(autonumber)s-%(title)s.%(ext)s" ] Number downloaded files starting from 00000')
     filesystem.add_option(
         '-t', '--title',
         action='store_true', dest='usetitle', default=False,

From 1a48181a9ff872e4b8428603f70851c386a2790d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Apr 2015 20:09:11 +0600
Subject: [PATCH 0854/6219] [options] Fix load info help string

---
 youtube_dl/options.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 2097a9436..5720fb424 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -670,7 +670,7 @@ def parseOpts(overrideArguments=None):
     filesystem.add_option(
         '--load-info',
         dest='load_info_filename', metavar='FILE',
-        help='Specify JSON file containing the video information (created with the "--write-json" option)')
+        help='JSON file containing the video information (created with the "--write-info-json" option)')
     filesystem.add_option(
         '--cookies',
         dest='cookiefile', metavar='FILE',

From 1a68d39211cab61994c8717cce296b0baae8095a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Apr 2015 22:15:59 +0600
Subject: [PATCH 0855/6219] [aftonbladet] Fix extraction

---
 youtube_dl/extractor/aftonbladet.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/aftonbladet.py b/youtube_dl/extractor/aftonbladet.py
index 8442019ea..4675585ca 100644
--- a/youtube_dl/extractor/aftonbladet.py
+++ b/youtube_dl/extractor/aftonbladet.py
@@ -5,7 +5,7 @@ from .common import InfoExtractor
 
 
 class AftonbladetIE(InfoExtractor):
-    _VALID_URL = r'^http://tv\.aftonbladet\.se/webbtv.+?(?P<video_id>article[0-9]+)\.ab(?:$|[?#])'
+    _VALID_URL = r'http://tv\.aftonbladet\.se/webbtv.+?(?P<id>article[0-9]+)\.ab(?:$|[?#])'
     _TEST = {
         'url': 'http://tv.aftonbladet.se/webbtv/nyheter/vetenskap/rymden/article36015.ab',
         'info_dict': {

From 8e1f93747338d64f6855c0f7f9467714bf56db93 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Apr 2015 22:19:34 +0600
Subject: [PATCH 0856/6219] [aftonbladet] Modernize

---
 youtube_dl/extractor/aftonbladet.py | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/aftonbladet.py b/youtube_dl/extractor/aftonbladet.py
index 4675585ca..a117502bc 100644
--- a/youtube_dl/extractor/aftonbladet.py
+++ b/youtube_dl/extractor/aftonbladet.py
@@ -2,6 +2,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..utils import int_or_none
 
 
 class AftonbladetIE(InfoExtractor):
@@ -43,9 +44,9 @@ class AftonbladetIE(InfoExtractor):
             formats.append({
                 'url': 'http://%s:%d/%s/%s' % (p['address'], p['port'], p['path'], p['filename']),
                 'ext': 'mp4',
-                'width': fmt['width'],
-                'height': fmt['height'],
-                'tbr': fmt['bitrate'],
+                'width': int_or_none(fmt.get('width')),
+                'height': int_or_none(fmt.get('height')),
+                'tbr': int_or_none(fmt.get('bitrate')),
                 'protocol': 'http',
             })
         self._sort_formats(formats)
@@ -54,9 +55,9 @@ class AftonbladetIE(InfoExtractor):
             'id': video_id,
             'title': internal_meta_json['title'],
             'formats': formats,
-            'thumbnail': internal_meta_json['imageUrl'],
-            'description': internal_meta_json['shortPreamble'],
-            'timestamp': internal_meta_json['timePublished'],
-            'duration': internal_meta_json['duration'],
-            'view_count': internal_meta_json['views'],
+            'thumbnail': internal_meta_json.get('imageUrl'),
+            'description': internal_meta_json.get('shortPreamble'),
+            'timestamp': int_or_none(internal_meta_json.get('timePublished')),
+            'duration': int_or_none(internal_meta_json.get('duration')),
+            'view_count': int_or_none(internal_meta_json.get('views')),
         }

From ed676e8c0ab087acb8e5e26a2a8d94a47fe10c33 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Apr 2015 22:27:25 +0600
Subject: [PATCH 0857/6219] [bliptv] Check format URLs

Some formats are now 404
---
 youtube_dl/extractor/bliptv.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/bliptv.py b/youtube_dl/extractor/bliptv.py
index 8c7ba4b91..b632ce967 100644
--- a/youtube_dl/extractor/bliptv.py
+++ b/youtube_dl/extractor/bliptv.py
@@ -172,6 +172,7 @@ class BlipTVIE(InfoExtractor):
                     'width': int_or_none(media_content.get('width')),
                     'height': int_or_none(media_content.get('height')),
                 })
+        self._check_formats(formats, video_id)
         self._sort_formats(formats)
 
         subtitles = self.extract_subtitles(video_id, subtitles_urls)

From 184a1974414bc91c5804251a33b8dd5cba1f75d9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Apr 2015 22:43:34 +0600
Subject: [PATCH 0858/6219] [culturebox] Check for unavailable videos

---
 youtube_dl/extractor/francetv.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index 170d68075..20acc96bd 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -275,7 +275,12 @@ class CultureboxIE(FranceTVBaseInfoExtractor):
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         name = mobj.group('name')
+
         webpage = self._download_webpage(url, name)
+
+        if ">Ce live n'est plus disponible en replay<" in webpage:
+            raise ExtractorError('Video %s is not available' % name, expected=True)
+
         video_id, catalogue = self._search_regex(
             r'"http://videos\.francetv\.fr/video/([^@]+@[^"]+)"', webpage, 'video id').split('@')
 

From aed2d4b31e331422fefa304ab3fa49c050ea13e8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Apr 2015 22:50:13 +0600
Subject: [PATCH 0859/6219] [culturebox] Replace test

---
 youtube_dl/extractor/francetv.py | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index 20acc96bd..55b8e9d9e 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -260,15 +260,18 @@ class CultureboxIE(FranceTVBaseInfoExtractor):
     _VALID_URL = r'https?://(?:m\.)?culturebox\.francetvinfo\.fr/(?P<name>.*?)(\?|$)'
 
     _TEST = {
-        'url': 'http://culturebox.francetvinfo.fr/festivals/dans-les-jardins-de-william-christie/dans-les-jardins-de-william-christie-le-camus-162553',
-        'md5': '5ad6dec1ffb2a3fbcb20cc4b744be8d6',
+        'url': 'http://culturebox.francetvinfo.fr/live/musique/musique-classique/le-livre-vermeil-de-montserrat-a-la-cathedrale-delne-214511',
         'info_dict': {
-            'id': 'EV_22853',
-            'ext': 'flv',
-            'title': 'Dans les jardins de William Christie - Le Camus',
-            'description': 'md5:4710c82315c40f0c865ca8b9a68b5299',
-            'upload_date': '20140829',
-            'timestamp': 1409317200,
+            'id': 'EV_50111',
+            'ext': 'mp4',
+            'title': "Le Livre Vermeil de Montserrat à la Cathédrale d'Elne",
+            'description': 'md5:f8a4ad202e8fe533e2c493cc12e739d9',
+            'upload_date': '20150320',
+            'timestamp': 1426892400,
+            'duration': 2760.9,
+        },
+        'params': {
+            'skip_download': True,
         },
     }
 

From f05d0e73c6d38d86393f8f552fbfcdda80f37607 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Apr 2015 22:52:25 +0600
Subject: [PATCH 0860/6219] [francetv] Fix duration

---
 youtube_dl/extractor/francetv.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index 55b8e9d9e..0d92ef9c4 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -14,6 +14,7 @@ from ..utils import (
     clean_html,
     ExtractorError,
     int_or_none,
+    float_or_none,
     parse_duration,
 )
 
@@ -86,7 +87,7 @@ class FranceTVBaseInfoExtractor(InfoExtractor):
             'title': info['titre'],
             'description': clean_html(info['synopsis']),
             'thumbnail': compat_urlparse.urljoin('http://pluzz.francetv.fr', info['image']),
-            'duration': parse_duration(info['duree']),
+            'duration': float_or_none(info.get('real_duration'), 1000) or parse_duration(info['duree']),
             'timestamp': int_or_none(info['diffusion']['timestamp']),
             'formats': formats,
         }

From bc03228ab52672666b79c9fadfbf886f8d8bf5d0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Apr 2015 23:02:04 +0600
Subject: [PATCH 0861/6219] [francetv] Improve formats extraction

---
 youtube_dl/extractor/francetv.py | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index 0d92ef9c4..fd3e7aa7b 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -16,6 +16,7 @@ from ..utils import (
     int_or_none,
     float_or_none,
     parse_duration,
+    determine_ext,
 )
 
 
@@ -51,7 +52,8 @@ class FranceTVBaseInfoExtractor(InfoExtractor):
             if not video_url:
                 continue
             format_id = video['format']
-            if video_url.endswith('.f4m'):
+            ext = determine_ext(video_url)
+            if ext == 'f4m':
                 if georestricted:
                     # See https://github.com/rg3/youtube-dl/issues/3963
                     # m3u8 urls work fine
@@ -61,12 +63,9 @@ class FranceTVBaseInfoExtractor(InfoExtractor):
                     'http://hdfauth.francetv.fr/esi/urltokengen2.html?url=%s' % video_url_parsed.path,
                     video_id, 'Downloading f4m manifest token', fatal=False)
                 if f4m_url:
-                    f4m_formats = self._extract_f4m_formats(f4m_url, video_id)
-                    for f4m_format in f4m_formats:
-                        f4m_format['preference'] = 1
-                    formats.extend(f4m_formats)
-            elif video_url.endswith('.m3u8'):
-                formats.extend(self._extract_m3u8_formats(video_url, video_id, 'mp4'))
+                    formats.extend(self._extract_f4m_formats(f4m_url, video_id, 1, format_id))
+            elif ext == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(video_url, video_id, 'mp4', m3u8_id=format_id))
             elif video_url.startswith('rtmp'):
                 formats.append({
                     'url': video_url,

From e21a55abcc502abac559027551751ff84d215077 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Apr 2015 23:05:25 +0600
Subject: [PATCH 0862/6219] [extractor/common] Remove f4m section

It's now provided by `f4m_id`
---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index e5245ec3f..530c449c1 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -822,7 +822,7 @@ class InfoExtractor(object):
                                 (media_el.attrib.get('href') or media_el.attrib.get('url')))
             tbr = int_or_none(media_el.attrib.get('bitrate'))
             formats.append({
-                'format_id': '-'.join(filter(None, [f4m_id, 'f4m-%d' % (i if tbr is None else tbr)])),
+                'format_id': '-'.join(filter(None, [f4m_id, compat_str(i if tbr is None else tbr)])),
                 'url': manifest_url,
                 'ext': 'flv',
                 'tbr': tbr,

From ac651e974ef8da74df35e5ac5464cc03e35bb2d1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Apr 2015 23:06:16 +0600
Subject: [PATCH 0863/6219] [culturebox] Fix test

---
 youtube_dl/extractor/francetv.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index fd3e7aa7b..edf555b29 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -261,18 +261,16 @@ class CultureboxIE(FranceTVBaseInfoExtractor):
 
     _TEST = {
         'url': 'http://culturebox.francetvinfo.fr/live/musique/musique-classique/le-livre-vermeil-de-montserrat-a-la-cathedrale-delne-214511',
+        'md5': '9b88dc156781c4dbebd4c3e066e0b1d6',
         'info_dict': {
             'id': 'EV_50111',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': "Le Livre Vermeil de Montserrat à la Cathédrale d'Elne",
             'description': 'md5:f8a4ad202e8fe533e2c493cc12e739d9',
             'upload_date': '20150320',
             'timestamp': 1426892400,
             'duration': 2760.9,
         },
-        'params': {
-            'skip_download': True,
-        },
     }
 
     def _real_extract(self, url):

From aff84bec07fc1919591827543845790074b0194f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Apr 2015 23:17:09 +0600
Subject: [PATCH 0864/6219] [drtv] Check for unavailable videos

---
 youtube_dl/extractor/drtv.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/drtv.py b/youtube_dl/extractor/drtv.py
index 8257e35a4..bd1109549 100644
--- a/youtube_dl/extractor/drtv.py
+++ b/youtube_dl/extractor/drtv.py
@@ -26,6 +26,10 @@ class DRTVIE(InfoExtractor):
 
         webpage = self._download_webpage(url, video_id)
 
+        if '>Programmet er ikke længere tilgængeligt' in webpage:
+            raise ExtractorError(
+                'Video %s is not available' % video_id, expected=True)
+
         video_id = self._search_regex(
             r'data-(?:material-identifier|episode-slug)="([^"]+)"',
             webpage, 'video id')

From 7d2546397209deab14a0ebad6c933ed97e73fe41 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Apr 2015 23:19:28 +0600
Subject: [PATCH 0865/6219] [drtv] Update test

---
 youtube_dl/extractor/drtv.py | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/drtv.py b/youtube_dl/extractor/drtv.py
index bd1109549..f25ab319e 100644
--- a/youtube_dl/extractor/drtv.py
+++ b/youtube_dl/extractor/drtv.py
@@ -1,3 +1,4 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
 from .common import InfoExtractor, ExtractorError
@@ -8,16 +9,16 @@ class DRTVIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?dr\.dk/tv/se/(?:[^/]+/)*(?P<id>[\da-z-]+)(?:[/#?]|$)'
 
     _TEST = {
-        'url': 'http://www.dr.dk/tv/se/partiets-mand/partiets-mand-7-8',
-        'md5': '4a7e1dd65cdb2643500a3f753c942f25',
+        'url': 'https://www.dr.dk/tv/se/boern/ultra/panisk-paske/panisk-paske-5',
+        'md5': 'dc515a9ab50577fa14cc4e4b0265168f',
         'info_dict': {
-            'id': 'partiets-mand-7-8',
+            'id': 'panisk-paske-5',
             'ext': 'mp4',
-            'title': 'Partiets mand (7:8)',
-            'description': 'md5:a684b90a8f9336cd4aab94b7647d7862',
-            'timestamp': 1403047940,
-            'upload_date': '20140617',
-            'duration': 1299.040,
+            'title': 'Panisk Påske (5)',
+            'description': 'md5:ca14173c5ab24cd26b0fcc074dff391c',
+            'timestamp': 1426984612,
+            'upload_date': '20150322',
+            'duration': 1455,
         },
     }
 

From 218d6bcc05bd84d8f69a7b764702dc24acb2f761 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Apr 2015 23:28:47 +0600
Subject: [PATCH 0866/6219] [dreisat] Capture status errors

---
 youtube_dl/extractor/dreisat.py | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/dreisat.py b/youtube_dl/extractor/dreisat.py
index 69ca75423..b88460a23 100644
--- a/youtube_dl/extractor/dreisat.py
+++ b/youtube_dl/extractor/dreisat.py
@@ -3,7 +3,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import unified_strdate
+from ..utils import (
+    ExtractorError,
+    unified_strdate,
+)
 
 
 class DreiSatIE(InfoExtractor):
@@ -28,6 +31,15 @@ class DreiSatIE(InfoExtractor):
         details_url = 'http://www.3sat.de/mediathek/xmlservice/web/beitragsDetails?ak=web&id=%s' % video_id
         details_doc = self._download_xml(details_url, video_id, 'Downloading video details')
 
+        status_code = details_doc.find('./status/statuscode')
+        if status_code is not None and status_code.text != 'ok':
+            code = status_code.text
+            if code == 'notVisibleAnymore':
+                message = 'Video %s is not available' % video_id
+            else:
+                message = '%s returned error: %s' % (self.IE_NAME, code)
+            raise ExtractorError(message, expected=True)
+
         thumbnail_els = details_doc.findall('.//teaserimage')
         thumbnails = [{
             'width': int(te.attrib['key'].partition('x')[0]),

From a319c33d8b8c4a7ac1d2f8dd739508b041d960b0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Apr 2015 23:30:38 +0600
Subject: [PATCH 0867/6219] [dreisat] Update test

---
 youtube_dl/extractor/dreisat.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/dreisat.py b/youtube_dl/extractor/dreisat.py
index b88460a23..05bb22ddf 100644
--- a/youtube_dl/extractor/dreisat.py
+++ b/youtube_dl/extractor/dreisat.py
@@ -13,15 +13,15 @@ class DreiSatIE(InfoExtractor):
     IE_NAME = '3sat'
     _VALID_URL = r'(?:http://)?(?:www\.)?3sat\.de/mediathek/(?:index\.php)?\?(?:(?:mode|display)=[^&]+&)*obj=(?P<id>[0-9]+)$'
     _TEST = {
-        'url': 'http://www.3sat.de/mediathek/index.php?obj=36983',
-        'md5': '9dcfe344732808dbfcc901537973c922',
+        'url': 'http://www.3sat.de/mediathek/index.php?mode=play&obj=45918',
+        'md5': 'be37228896d30a88f315b638900a026e',
         'info_dict': {
-            'id': '36983',
+            'id': '45918',
             'ext': 'mp4',
-            'title': 'Kaffeeland Schweiz',
-            'description': 'md5:cc4424b18b75ae9948b13929a0814033',
+            'title': 'Waidmannsheil',
+            'description': 'md5:cce00ca1d70e21425e72c86a98a56817',
             'uploader': '3sat',
-            'upload_date': '20130622'
+            'upload_date': '20140913'
         }
     }
 

From fefc9d121d32321d3609e131e488c443d7af962a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Apr 2015 23:33:07 +0600
Subject: [PATCH 0868/6219] [dump] Fix title extraction

---
 youtube_dl/extractor/dump.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/dump.py b/youtube_dl/extractor/dump.py
index 6b651778a..ff78d4fd2 100644
--- a/youtube_dl/extractor/dump.py
+++ b/youtube_dl/extractor/dump.py
@@ -28,12 +28,12 @@ class DumpIE(InfoExtractor):
         video_url = self._search_regex(
             r's1.addVariable\("file",\s*"([^"]+)"', webpage, 'video URL')
 
-        thumb = self._og_search_thumbnail(webpage)
-        title = self._search_regex(r'<b>([^"]+)</b>', webpage, 'title')
+        title = self._og_search_title(webpage)
+        thumbnail = self._og_search_thumbnail(webpage)
 
         return {
             'id': video_id,
             'title': title,
             'url': video_url,
-            'thumbnail': thumb,
+            'thumbnail': thumbnail,
         }

From f67dcc09f5e2f68032e379133e53b07e58c544dd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Apr 2015 23:36:45 +0600
Subject: [PATCH 0869/6219] [eagleplatform] Skip georestricted test

---
 youtube_dl/extractor/eagleplatform.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/eagleplatform.py b/youtube_dl/extractor/eagleplatform.py
index 7173371ee..688dfc2f7 100644
--- a/youtube_dl/extractor/eagleplatform.py
+++ b/youtube_dl/extractor/eagleplatform.py
@@ -45,6 +45,7 @@ class EaglePlatformIE(InfoExtractor):
             'duration': 216,
             'view_count': int,
         },
+        'skip': 'Georestricted',
     }]
 
     def _handle_error(self, response):

From 27fe5e347350484009e79251ec7ef97484219481 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 5 Apr 2015 00:00:04 +0600
Subject: [PATCH 0870/6219] [ellentv] Make video url extraction fatal

---
 youtube_dl/extractor/ellentv.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/ellentv.py b/youtube_dl/extractor/ellentv.py
index fc92ff825..3a7962144 100644
--- a/youtube_dl/extractor/ellentv.py
+++ b/youtube_dl/extractor/ellentv.py
@@ -40,14 +40,15 @@ class EllenTVIE(InfoExtractor):
         video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id)
-        video_url = self._html_search_meta('VideoURL', webpage, 'url')
+
+        video_url = self._html_search_meta('VideoURL', webpage, 'url', fatal=True)
         title = self._og_search_title(webpage, default=None) or self._search_regex(
             r'pageName\s*=\s*"([^"]+)"', webpage, 'title')
         description = self._html_search_meta(
             'description', webpage, 'description') or self._og_search_description(webpage)
         timestamp = parse_iso8601(self._search_regex(
             r'<span class="publish-date"><time datetime="([^"]+)">',
-            webpage, 'timestamp'))
+            webpage, 'timestamp', fatal=False))
 
         return {
             'id': video_id,

From 0de9312a7e7c15262ce4a59249d3c4294989757c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 5 Apr 2015 00:01:55 +0600
Subject: [PATCH 0871/6219] [ellentv] Replace test

---
 youtube_dl/extractor/ellentv.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/ellentv.py b/youtube_dl/extractor/ellentv.py
index 3a7962144..5154bbd7f 100644
--- a/youtube_dl/extractor/ellentv.py
+++ b/youtube_dl/extractor/ellentv.py
@@ -13,15 +13,15 @@ from ..utils import (
 class EllenTVIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?(?:ellentv|ellentube)\.com/videos/(?P<id>[a-z0-9_-]+)'
     _TESTS = [{
-        'url': 'http://www.ellentv.com/videos/0-7jqrsr18/',
-        'md5': 'e4af06f3bf0d5f471921a18db5764642',
+        'url': 'http://www.ellentv.com/videos/0-ipq1gsai/',
+        'md5': '8e3c576bf2e9bfff4d76565f56f94c9c',
         'info_dict': {
-            'id': '0-7jqrsr18',
+            'id': '0-ipq1gsai',
             'ext': 'mp4',
-            'title': 'What\'s Wrong with These Photos? A Whole Lot',
-            'description': 'md5:35f152dc66b587cf13e6d2cf4fa467f6',
-            'timestamp': 1406876400,
-            'upload_date': '20140801',
+            'title': 'Fast Fingers of Fate',
+            'description': 'md5:686114ced0a032926935e9015ee794ac',
+            'timestamp': 1428033600,
+            'upload_date': '20150403',
         }
     }, {
         'url': 'http://ellentube.com/videos/0-dvzmabd5/',

From 3a9fadd6dfc127ed0707b218b11ac10c654af1e2 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 5 Apr 2015 22:29:06 +0800
Subject: [PATCH 0872/6219] [youtube] Enhance url_encoded_fmt_stream_map
 checking (fix #5361)

---
 youtube_dl/extractor/youtube.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 5488101e1..f7f701cc5 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -855,7 +855,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 args = ytplayer_config['args']
                 # Convert to the same format returned by compat_parse_qs
                 video_info = dict((k, [v]) for k, v in args.items())
-                if 'url_encoded_fmt_stream_map' not in args:
+                if ('url_encoded_fmt_stream_map' not in args or
+                        args['url_encoded_fmt_stream_map'] == ''):
                     raise ValueError('No stream_map present')  # caught below
             except ValueError:
                 # We fallback to the get_video_info pages (used by the embed page)

From 06b491eb7b9459f92484f83973c17d46dba59f1f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 6 Apr 2015 00:35:55 +0600
Subject: [PATCH 0873/6219] [youtube] Add test for #5361

---
 youtube_dl/extractor/youtube.py | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index f7f701cc5..198fe84ef 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -495,7 +495,23 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader': '孫艾倫',
                 'title': '[A-made] 變態妍字幕版 太妍 我就是這樣的人',
             },
-        }
+        },
+        # url_encoded_fmt_stream_map is empty string
+        {
+            'url': 'qEJwOuvDf7I',
+            'info_dict': {
+                'id': 'qEJwOuvDf7I',
+                'ext': 'mp4',
+                'title': 'Обсуждение судебной практики по выборам 14 сентября 2014 года в Санкт-Петербурге',
+                'description': '',
+                'upload_date': '20150404',
+                'uploader_id': 'spbelect',
+                'uploader': 'Наблюдатели Петербурга',
+            },
+            'params': {
+                'skip_download': 'requires avconv',
+            }
+        },
     ]
 
     def __init__(self, *args, **kwargs):

From e40bd5f06ba397f6a3f9f21ab1df01c120d90eb7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 6 Apr 2015 00:45:57 +0600
Subject: [PATCH 0874/6219] [youtube] Simplify url_encoded_fmt_stream_map check

---
 youtube_dl/extractor/youtube.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 198fe84ef..79ce39aa4 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -871,8 +871,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 args = ytplayer_config['args']
                 # Convert to the same format returned by compat_parse_qs
                 video_info = dict((k, [v]) for k, v in args.items())
-                if ('url_encoded_fmt_stream_map' not in args or
-                        args['url_encoded_fmt_stream_map'] == ''):
+                if not args.get('url_encoded_fmt_stream_map'):
                     raise ValueError('No stream_map present')  # caught below
             except ValueError:
                 # We fallback to the get_video_info pages (used by the embed page)

From 64102296818f94d3814a8183daa5d92cbdd952fd Mon Sep 17 00:00:00 2001
From: newtonelectron <newton.electron@gmail.com>
Date: Sun, 5 Apr 2015 12:50:21 -0700
Subject: [PATCH 0875/6219] [SpankBang] Add new extractor

---
 youtube_dl/extractor/__init__.py  |  1 +
 youtube_dl/extractor/spankbang.py | 38 +++++++++++++++++++++++++++++++
 2 files changed, 39 insertions(+)
 create mode 100644 youtube_dl/extractor/spankbang.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 0f7d44616..e6fdf1297 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -471,6 +471,7 @@ from .southpark import (
     SouthparkDeIE,
 )
 from .space import SpaceIE
+from .spankbang import SpankBangIE
 from .spankwire import SpankwireIE
 from .spiegel import SpiegelIE, SpiegelArticleIE
 from .spiegeltv import SpiegeltvIE
diff --git a/youtube_dl/extractor/spankbang.py b/youtube_dl/extractor/spankbang.py
new file mode 100644
index 000000000..8e845ef26
--- /dev/null
+++ b/youtube_dl/extractor/spankbang.py
@@ -0,0 +1,38 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+import re
+
+class SpankBangIE(InfoExtractor):
+    """Extractor for http://spankbang.com"""
+    
+    _VALID_URL = r"https?://(?:www\.)?spankbang\.com/(?P<id>\w+)/video/.*"
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        title = self._html_search_regex(r"<h1>(?:<img.+?>)?(.*?)</h1>", webpage, "title")
+        
+        stream_key = self._html_search_regex(r"""var\s+stream_key\s*[=]\s*['"](.+?)['"]\s*;""", webpage, "stream_key")
+        
+        qualities = re.findall(r"<span.+?>([0-9]+p).*?</span>", webpage)
+        
+        formats = []
+        for q in sorted(qualities):
+            formats.append({
+                "format_id": q,
+                "format": q,
+                "ext": "mp4",
+                "url": "http://spankbang.com/_{}/{}/title/{}__mp4".format(video_id, stream_key, q)
+            })
+
+        return {
+            "id": video_id,
+            "title": title,
+            "description": self._og_search_description(webpage),
+            "formats": formats
+        }
+
+# vim: tabstop=4 expandtab

From 2e7daef50220ee90e8a2e2b979600f8bd4a3e40e Mon Sep 17 00:00:00 2001
From: newtonelectron <newton.electron@gmail.com>
Date: Sun, 5 Apr 2015 13:43:21 -0700
Subject: [PATCH 0876/6219] [SpankBang] Use python2.6 compatible string
 formatting spec

---
 youtube_dl/extractor/spankbang.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/spankbang.py b/youtube_dl/extractor/spankbang.py
index 8e845ef26..61fd64d17 100644
--- a/youtube_dl/extractor/spankbang.py
+++ b/youtube_dl/extractor/spankbang.py
@@ -25,7 +25,7 @@ class SpankBangIE(InfoExtractor):
                 "format_id": q,
                 "format": q,
                 "ext": "mp4",
-                "url": "http://spankbang.com/_{}/{}/title/{}__mp4".format(video_id, stream_key, q)
+                "url": "http://spankbang.com/_{0}/{1}/title/{2}__mp4".format(video_id, stream_key, q)
             })
 
         return {

From 5c1d459ae91d2681be88023e9056dcae3f48a70a Mon Sep 17 00:00:00 2001
From: newtonelectron <newton.electron@gmail.com>
Date: Sun, 5 Apr 2015 13:57:59 -0700
Subject: [PATCH 0877/6219] [SpankBang] Add test

---
 youtube_dl/extractor/spankbang.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/youtube_dl/extractor/spankbang.py b/youtube_dl/extractor/spankbang.py
index 61fd64d17..2e20a5ad5 100644
--- a/youtube_dl/extractor/spankbang.py
+++ b/youtube_dl/extractor/spankbang.py
@@ -8,6 +8,20 @@ class SpankBangIE(InfoExtractor):
     """Extractor for http://spankbang.com"""
     
     _VALID_URL = r"https?://(?:www\.)?spankbang\.com/(?P<id>\w+)/video/.*"
+    
+    _TEST = {
+        "url": "http://spankbang.com/3vvn/video/fantasy+solo",
+        "md5": "1cc433e1d6aa14bc376535b8679302f7",
+        "info_dict": {
+            "id": "3vvn",
+            "title": "fantasy solo",
+            "description": "Watch fantasy solo free HD porn video - 05 minutes - dillion harper masturbates on a bed free adult movies.",
+            "format": "720p",
+            "format_id": "720p",
+            "ext": "mp4",
+            "url": "re:http://spankbang.com/_3vvn/IjE0MjgyNjY5MTcuMzUi.IaGrcF-vDrvktMhjd-1fWixiCzU/title/720p__mp4"
+        }
+    }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From c7ac5dce8c692f82f10363e40a7085ac53113bc8 Mon Sep 17 00:00:00 2001
From: newtonelectron <newton.electron@gmail.com>
Date: Sun, 5 Apr 2015 14:02:05 -0700
Subject: [PATCH 0878/6219] [SpankBang] Remove regexp type prefix from _TEST
 url.

---
 youtube_dl/extractor/spankbang.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/spankbang.py b/youtube_dl/extractor/spankbang.py
index 2e20a5ad5..d0b5ba278 100644
--- a/youtube_dl/extractor/spankbang.py
+++ b/youtube_dl/extractor/spankbang.py
@@ -19,7 +19,7 @@ class SpankBangIE(InfoExtractor):
             "format": "720p",
             "format_id": "720p",
             "ext": "mp4",
-            "url": "re:http://spankbang.com/_3vvn/IjE0MjgyNjY5MTcuMzUi.IaGrcF-vDrvktMhjd-1fWixiCzU/title/720p__mp4"
+            "url": "http://spankbang.com/_3vvn/IjE0MjgyNjY5MTcuMzUi.IaGrcF-vDrvktMhjd-1fWixiCzU/title/720p__mp4"
         }
     }
 

From d2272fcf6e2796583b516f4786733577459dec43 Mon Sep 17 00:00:00 2001
From: felix <m.p.isaev@yandex.com>
Date: Mon, 6 Apr 2015 09:54:19 +0200
Subject: [PATCH 0879/6219] crooksandliars.com extractor

---
 youtube_dl/extractor/__init__.py       |  1 +
 youtube_dl/extractor/crooksandliars.py | 71 ++++++++++++++++++++++++++
 2 files changed, 72 insertions(+)
 create mode 100644 youtube_dl/extractor/crooksandliars.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 7eb9b4fbb..dc272af82 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -90,6 +90,7 @@ from .commonmistakes import CommonMistakesIE, UnicodeBOMIE
 from .condenast import CondeNastIE
 from .cracked import CrackedIE
 from .criterion import CriterionIE
+from .crooksandliars import CrooksAndLiarsIE, CrooksAndLiarsArticleIE
 from .crunchyroll import (
     CrunchyrollIE,
     CrunchyrollShowPlaylistIE
diff --git a/youtube_dl/extractor/crooksandliars.py b/youtube_dl/extractor/crooksandliars.py
new file mode 100644
index 000000000..afccca354
--- /dev/null
+++ b/youtube_dl/extractor/crooksandliars.py
@@ -0,0 +1,71 @@
+from __future__ import unicode_literals
+
+import json
+
+from .common import InfoExtractor
+from ..utils import (
+    mimetype2ext,
+)
+
+
+class CrooksAndLiarsIE(InfoExtractor):
+    _VALID_URL = r'(?:https?:)?//embed.crooksandliars.com/embed/(?P<id>[A-Za-z0-9]+)(?:$|[?#])'
+
+    _TESTS = [{
+        'url': 'https://embed.crooksandliars.com/embed/8RUoRhRi',
+        'info_dict': {
+            'id': 'https://embed.crooksandliars.com/embed/8RUoRhRi',
+            'title': "Fox & Friends Says Protecting Atheists From Discrimination Is Anti-Christian!",
+            'description': "Fox News, Fox & Friends Weekend, April 4, 2015. Read more... http://crooksandliars.com/2015/04/fox-friends-says-protecting-atheists",
+            'timestamp': 1428207000,
+            'thumbnail': '//crooksandliars.com/files/mediaposters/2015/04/31235.jpg?ts=1428207050',
+            'uploader': "Heather",
+        }
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        manifest = json.loads(self._html_search_regex(r'var manifest = ({.*?})\n', webpage, 'manifest JSON'))
+
+        formats = []
+        for item in manifest['flavors']:
+            if not item['mime'].startswith('video/'): # XXX: or item['exclude']?
+                continue
+            formats.append({
+                'format_id': item['type'],
+                'ext': mimetype2ext(item['mime']),
+                'url': item['url'],
+            })
+
+        # XXX: manifest['url']?
+        return {
+            'url': url,
+            'id': video_id,
+            'uploader': manifest['author'],
+            'title': manifest['title'],
+            'description': manifest['description'],
+            'thumbnail': manifest['poster'],
+            'duration': manifest['duration'],
+            'timestamp': int(manifest['created']),
+            'formats': formats,
+        }
+
+class CrooksAndLiarsArticleIE(InfoExtractor):
+    _VALID_URL = r'(?:https?:)?//crooksandliars.com/\d+/\d+/(?P<id>[a-z\-]+)(?:/|$)'
+
+    _TESTS = [{
+        'url': 'http://crooksandliars.com/2015/04/fox-friends-says-protecting-atheists',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        player_url = self._proto_relative_url(self._html_search_regex(r'<iframe src="(//embed.crooksandliars.com/.*)"', webpage, 'embedded player'))
+
+        return {
+            '_type': 'url',
+            'url': player_url
+        }

From 6e53c91608d1c43a9fe1614f13a15db74e877a91 Mon Sep 17 00:00:00 2001
From: felix <m.p.isaev@yandex.com>
Date: Mon, 6 Apr 2015 10:12:43 +0200
Subject: [PATCH 0880/6219] [crooksandliars] resolve protocol-relative URLs

---
 youtube_dl/extractor/crooksandliars.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/crooksandliars.py b/youtube_dl/extractor/crooksandliars.py
index afccca354..cee0603f4 100644
--- a/youtube_dl/extractor/crooksandliars.py
+++ b/youtube_dl/extractor/crooksandliars.py
@@ -18,7 +18,7 @@ class CrooksAndLiarsIE(InfoExtractor):
             'title': "Fox & Friends Says Protecting Atheists From Discrimination Is Anti-Christian!",
             'description': "Fox News, Fox & Friends Weekend, April 4, 2015. Read more... http://crooksandliars.com/2015/04/fox-friends-says-protecting-atheists",
             'timestamp': 1428207000,
-            'thumbnail': '//crooksandliars.com/files/mediaposters/2015/04/31235.jpg?ts=1428207050',
+            'thumbnail': 'https://crooksandliars.com/files/mediaposters/2015/04/31235.jpg?ts=1428207050',
             'uploader': "Heather",
         }
     }]
@@ -46,7 +46,7 @@ class CrooksAndLiarsIE(InfoExtractor):
             'uploader': manifest['author'],
             'title': manifest['title'],
             'description': manifest['description'],
-            'thumbnail': manifest['poster'],
+            'thumbnail': self._proto_relative_url(manifest['poster']),
             'duration': manifest['duration'],
             'timestamp': int(manifest['created']),
             'formats': formats,

From d97aae75724fc301243a00c5a71ac93b235d62fe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 6 Apr 2015 21:24:17 +0600
Subject: [PATCH 0881/6219] [spankbang] Improve and simplify

---
 youtube_dl/extractor/spankbang.py | 80 +++++++++++++++++--------------
 1 file changed, 44 insertions(+), 36 deletions(-)

diff --git a/youtube_dl/extractor/spankbang.py b/youtube_dl/extractor/spankbang.py
index d0b5ba278..7f060b15b 100644
--- a/youtube_dl/extractor/spankbang.py
+++ b/youtube_dl/extractor/spankbang.py
@@ -1,25 +1,23 @@
-# coding: utf-8
 from __future__ import unicode_literals
 
-from .common import InfoExtractor
 import re
 
+from .common import InfoExtractor
+
+
 class SpankBangIE(InfoExtractor):
-    """Extractor for http://spankbang.com"""
-    
-    _VALID_URL = r"https?://(?:www\.)?spankbang\.com/(?P<id>\w+)/video/.*"
-    
+    _VALID_URL = r'https?://(?:(?:www|[a-z]{2})\.)?spankbang\.com/(?P<id>[\da-z]+)/video'
     _TEST = {
-        "url": "http://spankbang.com/3vvn/video/fantasy+solo",
-        "md5": "1cc433e1d6aa14bc376535b8679302f7",
-        "info_dict": {
-            "id": "3vvn",
-            "title": "fantasy solo",
-            "description": "Watch fantasy solo free HD porn video - 05 minutes - dillion harper masturbates on a bed free adult movies.",
-            "format": "720p",
-            "format_id": "720p",
-            "ext": "mp4",
-            "url": "http://spankbang.com/_3vvn/IjE0MjgyNjY5MTcuMzUi.IaGrcF-vDrvktMhjd-1fWixiCzU/title/720p__mp4"
+        'url': 'http://spankbang.com/3vvn/video/fantasy+solo',
+        'md5': '1cc433e1d6aa14bc376535b8679302f7',
+        'info_dict': {
+            'id': '3vvn',
+            'ext': 'mp4',
+            'title': 'fantasy solo',
+            'description': 'dillion harper masturbates on a bed',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'uploader': 'silly2587',
+            'age_limit': 18,
         }
     }
 
@@ -27,26 +25,36 @@ class SpankBangIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        title = self._html_search_regex(r"<h1>(?:<img.+?>)?(.*?)</h1>", webpage, "title")
-        
-        stream_key = self._html_search_regex(r"""var\s+stream_key\s*[=]\s*['"](.+?)['"]\s*;""", webpage, "stream_key")
-        
-        qualities = re.findall(r"<span.+?>([0-9]+p).*?</span>", webpage)
-        
-        formats = []
-        for q in sorted(qualities):
-            formats.append({
-                "format_id": q,
-                "format": q,
-                "ext": "mp4",
-                "url": "http://spankbang.com/_{0}/{1}/title/{2}__mp4".format(video_id, stream_key, q)
-            })
+        stream_key = self._html_search_regex(
+            r'''var\s+stream_key\s*=\s*['"](.+?)['"]''',
+            webpage, 'stream key')
+
+        formats = [{
+            'url': 'http://spankbang.com/_%s/%s/title/%sp__mp4' % (video_id, stream_key, height),
+            'ext': 'mp4',
+            'format_id': '%sp' % height,
+            'height': int(height),
+        } for height in re.findall(r'<span[^>]+q_(\d+)p', webpage)]
+        self._sort_formats(formats)
+
+        title = self._html_search_regex(
+            r'(?s)<h1>(.+?)</h1>', webpage, 'title')
+        description = self._search_regex(
+            r'class="desc"[^>]*>([^<]+)',
+            webpage, 'description', default=None)
+        thumbnail = self._og_search_thumbnail(webpage)
+        uploader = self._search_regex(
+            r'class="user"[^>]*>([^<]+)',
+            webpage, 'uploader', fatal=False)
+
+        age_limit = self._rta_search(webpage)
 
         return {
-            "id": video_id,
-            "title": title,
-            "description": self._og_search_description(webpage),
-            "formats": formats
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'uploader': uploader,
+            'formats': formats,
+            'age_limit': age_limit,
         }
-
-# vim: tabstop=4 expandtab

From 8e4b83b96b8c2bc45aa1a9daa87c04853d3e7ade Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 6 Apr 2015 22:18:08 +0200
Subject: [PATCH 0882/6219] Remove check for ssl certs

When it uses a capath instead of a cafile, 'get_ca_certs' or 'cert_store_stats' only returns certificates already used in a connection.
(see #5364)
---
 youtube_dl/YoutubeDL.py | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index ce4b72fd3..640b8c99d 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1768,14 +1768,6 @@ class YoutubeDL(object):
 
         debuglevel = 1 if self.params.get('debug_printtraffic') else 0
         https_handler = make_HTTPS_handler(self.params, debuglevel=debuglevel)
-        # The ssl context is only available in python 2.7.9 and 3.x
-        if hasattr(https_handler, '_context'):
-            ctx = https_handler._context
-            # get_ca_certs is unavailable prior to python 3.4
-            if hasattr(ctx, 'get_ca_certs') and len(ctx.get_ca_certs()) == 0:
-                self.report_warning(
-                    'No ssl certificates were loaded, urls that use https '
-                    'won\'t work')
         ydlh = YoutubeDLHandler(self.params, debuglevel=debuglevel)
         opener = compat_urllib_request.build_opener(
             proxy_handler, https_handler, cookie_processor, ydlh)

From 29713e4268623ce5d471d5d25beea9ef433a2216 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 7 Apr 2015 14:59:13 +0200
Subject: [PATCH 0883/6219] [cnn] Match more affilliates

---
 youtube_dl/extractor/cnn.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/cnn.py b/youtube_dl/extractor/cnn.py
index 0a77e951c..5efc5f4fe 100644
--- a/youtube_dl/extractor/cnn.py
+++ b/youtube_dl/extractor/cnn.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 class CNNIE(InfoExtractor):
     _VALID_URL = r'''(?x)https?://(?:(?:edition|www)\.)?cnn\.com/video/(?:data/.+?|\?)/
-        (?P<path>.+?/(?P<title>[^/]+?)(?:\.(?:cnn|hln|ktvk)(?:-ap)?|(?=&)))'''
+        (?P<path>.+?/(?P<title>[^/]+?)(?:\.(?:[a-z]{3,5})(?:-ap)?|(?=&)))'''
 
     _TESTS = [{
         'url': 'http://edition.cnn.com/video/?/video/sports/2013/06/09/nadal-1-on-1.cnn',
@@ -48,6 +48,9 @@ class CNNIE(InfoExtractor):
     }, {
         'url': 'http://cnn.com/video/?/video/politics/2015/03/27/pkg-arizona-senator-church-attendance-mandatory.ktvk',
         'only_matching': True,
+    }, {
+        'url': 'http://cnn.com/video/?/video/us/2015/04/06/dnt-baker-refuses-anti-gay-order.wkmg',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From beb10f843f245efab24bec8ac6fead2834c5cce3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 7 Apr 2015 21:00:22 +0600
Subject: [PATCH 0884/6219] [addanime] Add format quality (Closes #5371)

---
 youtube_dl/extractor/addanime.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/addanime.py b/youtube_dl/extractor/addanime.py
index 203936e54..d2a60a056 100644
--- a/youtube_dl/extractor/addanime.py
+++ b/youtube_dl/extractor/addanime.py
@@ -11,6 +11,7 @@ from ..compat import (
 )
 from ..utils import (
     ExtractorError,
+    qualities,
 )
 
 
@@ -63,8 +64,10 @@ class AddAnimeIE(InfoExtractor):
                 note='Confirming after redirect')
             webpage = self._download_webpage(url, video_id)
 
+        FORMATS = ('normal', 'hq')
+        quality = qualities(FORMATS)
         formats = []
-        for format_id in ('normal', 'hq'):
+        for format_id in FORMATS:
             rex = r"var %s_video_file = '(.*?)';" % re.escape(format_id)
             video_url = self._search_regex(rex, webpage, 'video file URLx',
                                            fatal=False)
@@ -73,6 +76,7 @@ class AddAnimeIE(InfoExtractor):
             formats.append({
                 'format_id': format_id,
                 'url': video_url,
+                'quality': quality(format_id),
             })
         self._sort_formats(formats)
         video_title = self._og_search_title(webpage)

From 5f4b5cf044cff5263fa062596ace6180d42a4534 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 7 Apr 2015 21:00:52 +0600
Subject: [PATCH 0885/6219] [addanime] Extend _VALID_URL (Closes #5372)

---
 youtube_dl/extractor/addanime.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/addanime.py b/youtube_dl/extractor/addanime.py
index d2a60a056..28180bc3f 100644
--- a/youtube_dl/extractor/addanime.py
+++ b/youtube_dl/extractor/addanime.py
@@ -16,7 +16,7 @@ from ..utils import (
 
 
 class AddAnimeIE(InfoExtractor):
-    _VALID_URL = r'^http://(?:\w+\.)?add-anime\.net/watch_video\.php\?(?:.*?)v=(?P<id>[\w_]+)(?:.*)'
+    _VALID_URL = r'http://(?:\w+\.)?add-anime\.net/(?:watch_video\.php\?(?:.*?)v=|video/)(?P<id>[\w_]+)'
     _TEST = {
         'url': 'http://www.add-anime.net/watch_video.php?v=24MR3YO5SAS9',
         'md5': '72954ea10bc979ab5e2eb288b21425a0',

From a35099bd331e42d96e647865f4a644b217bba62a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 7 Apr 2015 21:01:35 +0600
Subject: [PATCH 0886/6219] [addanime] Add test for #5372

---
 youtube_dl/extractor/addanime.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/addanime.py b/youtube_dl/extractor/addanime.py
index 28180bc3f..e3e6d2113 100644
--- a/youtube_dl/extractor/addanime.py
+++ b/youtube_dl/extractor/addanime.py
@@ -17,7 +17,7 @@ from ..utils import (
 
 class AddAnimeIE(InfoExtractor):
     _VALID_URL = r'http://(?:\w+\.)?add-anime\.net/(?:watch_video\.php\?(?:.*?)v=|video/)(?P<id>[\w_]+)'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.add-anime.net/watch_video.php?v=24MR3YO5SAS9',
         'md5': '72954ea10bc979ab5e2eb288b21425a0',
         'info_dict': {
@@ -26,7 +26,10 @@ class AddAnimeIE(InfoExtractor):
             'description': 'One Piece 606',
             'title': 'One Piece 606',
         }
-    }
+    }, {
+        'url': 'http://add-anime.net/video/MDUGWYKNGBD8/One-Piece-687',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From de5c54564874fd870fdfe3fd24f47e3e5f6cedf7 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 8 Apr 2015 03:45:02 +0800
Subject: [PATCH 0887/6219] [youtube] Skip WebVTT in DASH manifest (#5297)

---
 youtube_dl/extractor/youtube.py | 62 +++++++++++++++++++--------------
 1 file changed, 35 insertions(+), 27 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 79ce39aa4..2774ec30b 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -788,33 +788,41 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             errnote='Could not download DASH manifest')
 
         formats = []
-        for r in dash_doc.findall('.//{urn:mpeg:DASH:schema:MPD:2011}Representation'):
-            url_el = r.find('{urn:mpeg:DASH:schema:MPD:2011}BaseURL')
-            if url_el is None:
-                continue
-            format_id = r.attrib['id']
-            video_url = url_el.text
-            filesize = int_or_none(url_el.attrib.get('{http://youtube.com/yt/2012/10/10}contentLength'))
-            f = {
-                'format_id': format_id,
-                'url': video_url,
-                'width': int_or_none(r.attrib.get('width')),
-                'height': int_or_none(r.attrib.get('height')),
-                'tbr': int_or_none(r.attrib.get('bandwidth'), 1000),
-                'asr': int_or_none(r.attrib.get('audioSamplingRate')),
-                'filesize': filesize,
-                'fps': int_or_none(r.attrib.get('frameRate')),
-            }
-            try:
-                existing_format = next(
-                    fo for fo in formats
-                    if fo['format_id'] == format_id)
-            except StopIteration:
-                full_info = self._formats.get(format_id, {}).copy()
-                full_info.update(f)
-                formats.append(full_info)
-            else:
-                existing_format.update(f)
+        for a in dash_doc.findall('.//{urn:mpeg:DASH:schema:MPD:2011}AdaptationSet'):
+            mime_type = a.attrib.get('mimeType')
+            for r in a.findall('{urn:mpeg:DASH:schema:MPD:2011}Representation'):
+                url_el = r.find('{urn:mpeg:DASH:schema:MPD:2011}BaseURL')
+                if url_el is None:
+                    continue
+                if mime_type == 'text/vtt':
+                    # TODO implement WebVTT downloading
+                    pass
+                elif mime_type.startswith('audio/') or mime_type.startswith('video/'):
+                    format_id = r.attrib['id']
+                    video_url = url_el.text
+                    filesize = int_or_none(url_el.attrib.get('{http://youtube.com/yt/2012/10/10}contentLength'))
+                    f = {
+                        'format_id': format_id,
+                        'url': video_url,
+                        'width': int_or_none(r.attrib.get('width')),
+                        'height': int_or_none(r.attrib.get('height')),
+                        'tbr': int_or_none(r.attrib.get('bandwidth'), 1000),
+                        'asr': int_or_none(r.attrib.get('audioSamplingRate')),
+                        'filesize': filesize,
+                        'fps': int_or_none(r.attrib.get('frameRate')),
+                    }
+                    try:
+                        existing_format = next(
+                            fo for fo in formats
+                            if fo['format_id'] == format_id)
+                    except StopIteration:
+                        full_info = self._formats.get(format_id, {}).copy()
+                        full_info.update(f)
+                        formats.append(full_info)
+                    else:
+                        existing_format.update(f)
+                else:
+                    self.report_warning('Unknown MIME type %s in DASH manifest' % mime_type)
         return formats
 
     def _real_extract(self, url):

From cc55d08832f5008efb56228900a0dbacb428ed7f Mon Sep 17 00:00:00 2001
From: Pete Hemery <petehemery@hotmail.com>
Date: Tue, 7 Apr 2015 22:33:18 +0100
Subject: [PATCH 0888/6219] [ffmpeg] adding exception catching for call to
 os.utime in run_ffmpeg_multiple_files

---
 youtube_dl/postprocessor/ffmpeg.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 0b60ac7e7..5ef5e0e54 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -146,7 +146,11 @@ class FFmpegPostProcessor(PostProcessor):
             stderr = stderr.decode('utf-8', 'replace')
             msg = stderr.strip().split('\n')[-1]
             raise FFmpegPostProcessorError(msg)
-        os.utime(encodeFilename(out_path), (oldest_mtime, oldest_mtime))
+        try:
+            os.utime(encodeFilename(out_path), (oldest_mtime, oldest_mtime))
+        except Exception:
+            self._downloader.report_warning('Cannot update utime of file')
+
         if self._deletetempfiles:
             for ipath in input_paths:
                 os.remove(ipath)

From 418c5cc3fc3ea99b791ad1774a6b03504eab7086 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 8 Apr 2015 17:26:51 +0800
Subject: [PATCH 0889/6219] [udn] Add new extractor

---
 test/test_utils.py               |  5 +++
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/generic.py  | 20 ++++++++++
 youtube_dl/extractor/udn.py      | 66 ++++++++++++++++++++++++++++++++
 youtube_dl/utils.py              | 11 ++++++
 5 files changed, 103 insertions(+)
 create mode 100644 youtube_dl/extractor/udn.py

diff --git a/test/test_utils.py b/test/test_utils.py
index 2e3a6480c..8291edd9a 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -53,6 +53,7 @@ from youtube_dl.utils import (
     uppercase_escape,
     url_basename,
     urlencode_postdata,
+    url_infer_protocol,
     version_tuple,
     xpath_with_ns,
     xpath_text,
@@ -296,6 +297,10 @@ class TestUtil(unittest.TestCase):
             url_basename('http://media.w3.org/2010/05/sintel/trailer.mp4'),
             'trailer.mp4')
 
+    def test_url_infer_protocol(self):
+        self.assertEqual(url_infer_protocol('http://foo.com/', '//bar.com/'), 'http://bar.com/')
+        self.assertEqual(url_infer_protocol('http://foo.com/', 'https://bar.com/'), 'https://bar.com/')
+
     def test_parse_duration(self):
         self.assertEqual(parse_duration(None), None)
         self.assertEqual(parse_duration(False), None)
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index e6fdf1297..8df1db83e 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -557,6 +557,7 @@ from .udemy import (
     UdemyIE,
     UdemyCourseIE
 )
+from .udn import UDNEmbedIE
 from .ultimedia import UltimediaIE
 from .unistra import UnistraIE
 from .urort import UrortIE
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 2ff002643..9178d2b7b 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -26,6 +26,7 @@ from ..utils import (
     unsmuggle_url,
     UnsupportedError,
     url_basename,
+    url_infer_protocol,
     xpath_text,
 )
 from .brightcove import BrightcoveIE
@@ -34,6 +35,7 @@ from .ooyala import OoyalaIE
 from .rutv import RUTVIE
 from .smotri import SmotriIE
 from .condenast import CondeNastIE
+from .udn import UDNEmbedIE
 
 
 class GenericIE(InfoExtractor):
@@ -650,6 +652,17 @@ class GenericIE(InfoExtractor):
                 'title': "PFT Live: New leader in the 'new-look' defense",
                 'description': 'md5:65a19b4bbfb3b0c0c5768bed1dfad74e',
             },
+        },
+        # UDN embed
+        {
+            'url': 'http://www.udn.com/news/story/7314/822787',
+            'md5': 'de06b4c90b042c128395a88f0384817e',
+            'info_dict': {
+                'id': '300040',
+                'ext': 'mp4',
+                'title': '生物老師男變女 全校挺"做自己"',
+                'thumbnail': 're:^https?://.*\.jpg$',
+            }
         }
     ]
 
@@ -1268,6 +1281,13 @@ class GenericIE(InfoExtractor):
         if nbc_sports_url:
             return self.url_result(nbc_sports_url, 'NBCSportsVPlayer')
 
+        # Look for UDN embeds
+        mobj = re.search(
+            r'<iframe[^>]+src="(?P<url>%s)"' % UDNEmbedIE._VALID_URL, webpage)
+        if mobj is not None:
+            return self.url_result(
+                url_infer_protocol(url, mobj.group('url')), 'UDNEmbed')
+
         def check_video(vurl):
             if YoutubeIE.suitable(vurl):
                 return True
diff --git a/youtube_dl/extractor/udn.py b/youtube_dl/extractor/udn.py
new file mode 100644
index 000000000..2b9a733e8
--- /dev/null
+++ b/youtube_dl/extractor/udn.py
@@ -0,0 +1,66 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import json
+from .common import InfoExtractor
+from ..utils import (
+    url_infer_protocol,
+    js_to_json
+)
+
+
+class UDNEmbedIE(InfoExtractor):
+    _VALID_URL = r'(?:https?:)?//video\.udn\.com/embed/news/(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'http://video.udn.com/embed/news/300040',
+        'md5': 'de06b4c90b042c128395a88f0384817e',
+        'info_dict': {
+            'id': '300040',
+            'ext': 'mp4',
+            'title': '生物老師男變女 全校挺"做自己"',
+            'thumbnail': 're:^https?://.*\.jpg$',
+        }
+    }, {
+        'url': '//video.udn.com/embed/news/300040',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        page = self._download_webpage(url, video_id)
+
+        options = json.loads(js_to_json(self._html_search_regex(
+            r'var options\s*=\s*([^;]+);', page, 'video urls dictionary')))
+
+        video_urls = options['video']
+
+        if video_urls.get('youtube'):
+            return self.url_result(video_urls.get('youtube'), 'Youtube')
+
+        try:
+            del video_urls['youtube']
+        except KeyError:
+            pass
+
+        formats = [{
+            'url': self._download_webpage(
+                url_infer_protocol(url, api_url), video_id,
+                'retrieve url for %s video' % video_type),
+            'format_id': video_type,
+            'preference': 0 if video_type == 'mp4' else -1,
+        } for video_type, api_url in video_urls.items()]
+
+        self._sort_formats(formats)
+
+        thumbnail = None
+
+        if options.get('gallery') and len(options['gallery']):
+            thumbnail = options['gallery'][0].get('original')
+
+        return {
+            'id': video_id,
+            'formats': formats,
+            'title': options['title'],
+            'thumbnail': thumbnail
+        }
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 52f0dd09a..f3b8d9f81 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1711,6 +1711,17 @@ def determine_protocol(info_dict):
     return compat_urllib_parse_urlparse(url).scheme
 
 
+def url_infer_protocol(ref_url, target_url):
+    """ Infer protocol for protocol independent target urls """
+    parsed_target_url = list(compat_urllib_parse_urlparse(target_url))
+    if parsed_target_url[0]:
+        return target_url
+
+    parsed_target_url[0] = compat_urllib_parse_urlparse(ref_url).scheme
+
+    return compat_urlparse.urlunparse(parsed_target_url)
+
+
 def render_table(header_row, data):
     """ Render a list of rows, each as a list of values """
     table = [header_row] + data

From 4a20c9f628b6038766a76ed1c3e37aa6a2b34718 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 8 Apr 2015 17:42:26 +0800
Subject: [PATCH 0890/6219] [livestream] Extend _VALID_URL (fixes #5375)

---
 youtube_dl/extractor/livestream.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/livestream.py b/youtube_dl/extractor/livestream.py
index 2467f8bdd..ec309dadd 100644
--- a/youtube_dl/extractor/livestream.py
+++ b/youtube_dl/extractor/livestream.py
@@ -21,7 +21,7 @@ from ..utils import (
 
 class LivestreamIE(InfoExtractor):
     IE_NAME = 'livestream'
-    _VALID_URL = r'https?://new\.livestream\.com/.*?/(?P<event_name>.*?)(/videos/(?P<id>[0-9]+)(?:/player)?)?/?(?:$|[?#])'
+    _VALID_URL = r'https?://(?:new\.)?livestream\.com/.*?/(?P<event_name>.*?)(/videos/(?P<id>[0-9]+)(?:/player)?)?/?(?:$|[?#])'
     _TESTS = [{
         'url': 'http://new.livestream.com/CoheedandCambria/WebsterHall/videos/4719370',
         'md5': '53274c76ba7754fb0e8d072716f2292b',
@@ -51,6 +51,9 @@ class LivestreamIE(InfoExtractor):
     }, {
         'url': 'https://new.livestream.com/accounts/362/events/3557232/videos/67864563/player?autoPlay=false&height=360&mute=false&width=640',
         'only_matching': True,
+    }, {
+        'url': 'http://livestream.com/bsww/concacafbeachsoccercampeonato2015',
+        'only_matching': True,
     }]
 
     def _parse_smil(self, video_id, smil_url):

From bd7a6478a2db228c7325a48e13e0e699502f56f4 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 8 Apr 2015 19:20:34 +0800
Subject: [PATCH 0891/6219] [theplatform] Fix video url extraction (fixes
 #5340)

In SMIL 2.1, <switch> nodes may be enclosed in <par>. See
http://www.w3.org/TR/SMIL2/smil-timing.html#edef-par
---
 youtube_dl/extractor/theplatform.py | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index 0e3e627f4..a7d060459 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -28,7 +28,7 @@ class ThePlatformIE(InfoExtractor):
            (?P<config>(?:[^/\?]+/(?:swf|config)|onsite)/select/)?
          |theplatform:)(?P<id>[^/\?&]+)'''
 
-    _TEST = {
+    _TESTS = [{
         # from http://www.metacafe.com/watch/cb-e9I_cZgTgIPd/blackberrys_big_bold_z30/
         'url': 'http://link.theplatform.com/s/dJ5BDC/e9I_cZgTgIPd/meta.smil?format=smil&Tracking=true&mbr=true',
         'info_dict': {
@@ -42,7 +42,20 @@ class ThePlatformIE(InfoExtractor):
             # rtmp download
             'skip_download': True,
         },
-    }
+        # from http://www.cnet.com/videos/tesla-model-s-a-second-step-towards-a-cleaner-motoring-future/
+    }, {
+        'url': 'http://link.theplatform.com/s/kYEXFC/22d_qsQ6MIRT',
+        'info_dict': {
+            'id': '22d_qsQ6MIRT',
+            'ext': 'flv',
+            'description': 'md5:ac330c9258c04f9d7512cf26b9595409',
+            'title': 'Tesla Model S: A second step towards a cleaner motoring future',
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        }
+    }]
 
     @staticmethod
     def _sign_url(url, sig_key, sig_secret, life=600, include_qs=False):
@@ -126,7 +139,7 @@ class ThePlatformIE(InfoExtractor):
             formats = self._extract_f4m_formats(f4m_url, video_id)
         else:
             formats = []
-            switch = body.find(_x('smil:switch'))
+            switch = body.find(_x('.//smil:switch'))
             if switch is not None:
                 base_url = head.find(_x('smil:meta')).attrib['base']
                 for f in switch.findall(_x('smil:video')):

From a662163fd5b3c9b1221b8aeaf54ed9083af8574f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 8 Apr 2015 20:21:34 +0800
Subject: [PATCH 0892/6219] [theplatform] Rework on <switch> inside <par>

---
 youtube_dl/extractor/theplatform.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index a7d060459..bcddd23e4 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -139,7 +139,9 @@ class ThePlatformIE(InfoExtractor):
             formats = self._extract_f4m_formats(f4m_url, video_id)
         else:
             formats = []
-            switch = body.find(_x('.//smil:switch'))
+            switch = body.find(_x('smil:switch'))
+            if switch is None:
+                switch = body.find(_x('smil:par//smil:switch'))
             if switch is not None:
                 base_url = head.find(_x('smil:meta')).attrib['base']
                 for f in switch.findall(_x('smil:video')):

From 0a1603634bcf799eeb769d95b6e716e66123b77f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 8 Apr 2015 21:39:34 +0800
Subject: [PATCH 0893/6219] [utils] Remove url_infer_protocol

---
 test/test_utils.py              |  5 -----
 youtube_dl/extractor/generic.py |  3 +--
 youtube_dl/extractor/udn.py     |  8 +++-----
 youtube_dl/utils.py             | 11 -----------
 4 files changed, 4 insertions(+), 23 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 8291edd9a..2e3a6480c 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -53,7 +53,6 @@ from youtube_dl.utils import (
     uppercase_escape,
     url_basename,
     urlencode_postdata,
-    url_infer_protocol,
     version_tuple,
     xpath_with_ns,
     xpath_text,
@@ -297,10 +296,6 @@ class TestUtil(unittest.TestCase):
             url_basename('http://media.w3.org/2010/05/sintel/trailer.mp4'),
             'trailer.mp4')
 
-    def test_url_infer_protocol(self):
-        self.assertEqual(url_infer_protocol('http://foo.com/', '//bar.com/'), 'http://bar.com/')
-        self.assertEqual(url_infer_protocol('http://foo.com/', 'https://bar.com/'), 'https://bar.com/')
-
     def test_parse_duration(self):
         self.assertEqual(parse_duration(None), None)
         self.assertEqual(parse_duration(False), None)
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 9178d2b7b..6c212efac 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -26,7 +26,6 @@ from ..utils import (
     unsmuggle_url,
     UnsupportedError,
     url_basename,
-    url_infer_protocol,
     xpath_text,
 )
 from .brightcove import BrightcoveIE
@@ -1286,7 +1285,7 @@ class GenericIE(InfoExtractor):
             r'<iframe[^>]+src="(?P<url>%s)"' % UDNEmbedIE._VALID_URL, webpage)
         if mobj is not None:
             return self.url_result(
-                url_infer_protocol(url, mobj.group('url')), 'UDNEmbed')
+                compat_urlparse.urljoin(url, mobj.group('url')), 'UDNEmbed')
 
         def check_video(vurl):
             if YoutubeIE.suitable(vurl):
diff --git a/youtube_dl/extractor/udn.py b/youtube_dl/extractor/udn.py
index 2b9a733e8..bba25bb58 100644
--- a/youtube_dl/extractor/udn.py
+++ b/youtube_dl/extractor/udn.py
@@ -3,10 +3,8 @@ from __future__ import unicode_literals
 
 import json
 from .common import InfoExtractor
-from ..utils import (
-    url_infer_protocol,
-    js_to_json
-)
+from ..utils import js_to_json
+from ..compat import compat_urlparse
 
 
 class UDNEmbedIE(InfoExtractor):
@@ -45,7 +43,7 @@ class UDNEmbedIE(InfoExtractor):
 
         formats = [{
             'url': self._download_webpage(
-                url_infer_protocol(url, api_url), video_id,
+                compat_urlparse.urljoin(url, api_url), video_id,
                 'retrieve url for %s video' % video_type),
             'format_id': video_type,
             'preference': 0 if video_type == 'mp4' else -1,
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index f3b8d9f81..52f0dd09a 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1711,17 +1711,6 @@ def determine_protocol(info_dict):
     return compat_urllib_parse_urlparse(url).scheme
 
 
-def url_infer_protocol(ref_url, target_url):
-    """ Infer protocol for protocol independent target urls """
-    parsed_target_url = list(compat_urllib_parse_urlparse(target_url))
-    if parsed_target_url[0]:
-        return target_url
-
-    parsed_target_url[0] = compat_urllib_parse_urlparse(ref_url).scheme
-
-    return compat_urlparse.urlunparse(parsed_target_url)
-
-
 def render_table(header_row, data):
     """ Render a list of rows, each as a list of values """
     table = [header_row] + data

From aef8fdba1172d60983ba9685249c03b66e7a94f0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 8 Apr 2015 21:03:11 +0600
Subject: [PATCH 0894/6219] [theplatform] Allow <par> without <swtich> at all

Bare `wget` on http://link.theplatform.com/s/kYEXFC/22d_qsQ6MIRTl results in an XML without <switch> at all
but with <par> and <video> inside it. Let's handle this possible outcome as well.
---
 youtube_dl/extractor/theplatform.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index bcddd23e4..2d2178331 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -142,6 +142,8 @@ class ThePlatformIE(InfoExtractor):
             switch = body.find(_x('smil:switch'))
             if switch is None:
                 switch = body.find(_x('smil:par//smil:switch'))
+            if switch is None:
+                switch = body.find(_x('smil:par'))
             if switch is not None:
                 base_url = head.find(_x('smil:meta')).attrib['base']
                 for f in switch.findall(_x('smil:video')):

From dd29eb7f816bae7cf7807db9f26b7f5621b8d557 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 8 Apr 2015 21:40:31 +0600
Subject: [PATCH 0895/6219] [postprocessor/common:postprocessor/ffmpeg]
 Generalize utime

---
 youtube_dl/postprocessor/common.py | 13 ++++++++++++-
 youtube_dl/postprocessor/ffmpeg.py | 12 ++++--------
 2 files changed, 16 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/postprocessor/common.py b/youtube_dl/postprocessor/common.py
index e54ae678d..ef9fdfa19 100644
--- a/youtube_dl/postprocessor/common.py
+++ b/youtube_dl/postprocessor/common.py
@@ -1,6 +1,11 @@
 from __future__ import unicode_literals
 
-from ..utils import PostProcessingError
+import os
+
+from ..utils import (
+    PostProcessingError,
+    encodeFilename,
+)
 
 
 class PostProcessor(object):
@@ -46,6 +51,12 @@ class PostProcessor(object):
         """
         return None, information  # by default, keep file and do nothing
 
+    def try_utime(self, path, atime, mtime, errnote='Cannot update utime of file'):
+        try:
+            os.utime(encodeFilename(path), (atime, mtime))
+        except Exception:
+            self._downloader.report_warning(errnote)
+
 
 class AudioConversionError(PostProcessingError):
     pass
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 5ef5e0e54..8e99a3c2c 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -146,10 +146,7 @@ class FFmpegPostProcessor(PostProcessor):
             stderr = stderr.decode('utf-8', 'replace')
             msg = stderr.strip().split('\n')[-1]
             raise FFmpegPostProcessorError(msg)
-        try:
-            os.utime(encodeFilename(out_path), (oldest_mtime, oldest_mtime))
-        except Exception:
-            self._downloader.report_warning('Cannot update utime of file')
+        self.try_utime(out_path, oldest_mtime, oldest_mtime)
 
         if self._deletetempfiles:
             for ipath in input_paths:
@@ -284,10 +281,9 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
 
         # Try to update the date time for extracted audio file.
         if information.get('filetime') is not None:
-            try:
-                os.utime(encodeFilename(new_path), (time.time(), information['filetime']))
-            except Exception:
-                self._downloader.report_warning('Cannot update utime of audio file')
+            self.try_utime(
+                new_path, time.time(), information['filetime'],
+                errnote='Cannot update utime of audio file')
 
         information['filepath'] = new_path
         return self._nopostoverwrites, information

From 372f08c99057b6a994c72cf2591e184231b3e850 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 8 Apr 2015 22:27:25 +0600
Subject: [PATCH 0896/6219] [theplatform] Fix for python 2.6

At least single depth level extraction...
---
 youtube_dl/extractor/theplatform.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index 2d2178331..50c51d825 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -42,8 +42,8 @@ class ThePlatformIE(InfoExtractor):
             # rtmp download
             'skip_download': True,
         },
-        # from http://www.cnet.com/videos/tesla-model-s-a-second-step-towards-a-cleaner-motoring-future/
     }, {
+        # from http://www.cnet.com/videos/tesla-model-s-a-second-step-towards-a-cleaner-motoring-future/
         'url': 'http://link.theplatform.com/s/kYEXFC/22d_qsQ6MIRT',
         'info_dict': {
             'id': '22d_qsQ6MIRT',
@@ -128,7 +128,7 @@ class ThePlatformIE(InfoExtractor):
         head = meta.find(_x('smil:head'))
         body = meta.find(_x('smil:body'))
 
-        f4m_node = body.find(_x('smil:seq//smil:video'))
+        f4m_node = body.find(_x('smil:seq//smil:video')) or body.find(_x('smil:seq/smil:video'))
         if f4m_node is not None and '.f4m' in f4m_node.attrib['src']:
             f4m_url = f4m_node.attrib['src']
             if 'manifest.f4m?' not in f4m_url:
@@ -141,7 +141,7 @@ class ThePlatformIE(InfoExtractor):
             formats = []
             switch = body.find(_x('smil:switch'))
             if switch is None:
-                switch = body.find(_x('smil:par//smil:switch'))
+                switch = body.find(_x('smil:par//smil:switch')) or body.find(_x('smil:par/smil:switch'))
             if switch is None:
                 switch = body.find(_x('smil:par'))
             if switch is not None:
@@ -162,7 +162,7 @@ class ThePlatformIE(InfoExtractor):
                         'vbr': vbr,
                     })
             else:
-                switch = body.find(_x('smil:seq//smil:switch'))
+                switch = body.find(_x('smil:seq//smil:switch')) or body.find(_x('smil:seq/smil:switch'))
                 for f in switch.findall(_x('smil:video')):
                     attr = f.attrib
                     vbr = int(attr['system-bitrate']) // 1000

From 402a3efc927538684fb69e23c65a97b19ce4f663 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 8 Apr 2015 22:29:10 +0600
Subject: [PATCH 0897/6219] [theplatform] Modernize

---
 youtube_dl/extractor/theplatform.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index 50c51d825..6a006b2d2 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -17,6 +17,7 @@ from ..utils import (
     ExtractorError,
     xpath_with_ns,
     unsmuggle_url,
+    int_or_none,
 )
 
 _x = lambda p: xpath_with_ns(p, {'smil': 'http://www.w3.org/2005/SMIL21/Language'})
@@ -148,9 +149,9 @@ class ThePlatformIE(InfoExtractor):
                 base_url = head.find(_x('smil:meta')).attrib['base']
                 for f in switch.findall(_x('smil:video')):
                     attr = f.attrib
-                    width = int(attr['width'])
-                    height = int(attr['height'])
-                    vbr = int(attr['system-bitrate']) // 1000
+                    width = int_or_none(attr.get('width'))
+                    height = int_or_none(attr.get('height'))
+                    vbr = int_or_none(attr.get('system-bitrate'), 1000)
                     format_id = '%dx%d_%dk' % (width, height, vbr)
                     formats.append({
                         'format_id': format_id,
@@ -165,7 +166,7 @@ class ThePlatformIE(InfoExtractor):
                 switch = body.find(_x('smil:seq//smil:switch')) or body.find(_x('smil:seq/smil:switch'))
                 for f in switch.findall(_x('smil:video')):
                     attr = f.attrib
-                    vbr = int(attr['system-bitrate']) // 1000
+                    vbr = int_or_none(attr.get('system-bitrate'), 1000)
                     ext = determine_ext(attr['src'])
                     if ext == 'once':
                         ext = 'mp4'
@@ -184,5 +185,5 @@ class ThePlatformIE(InfoExtractor):
             'formats': formats,
             'description': info['description'],
             'thumbnail': info['defaultThumbnailUrl'],
-            'duration': info['duration'] // 1000,
+            'duration': int_or_none(info.get('duration'), 1000),
         }

From af76e8174d1f73e0c44e6bead56a780c665e299d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 9 Apr 2015 02:25:31 +0600
Subject: [PATCH 0898/6219] [dailymotion:user] Improve _VALID_URL (Closes
 #5380)

---
 youtube_dl/extractor/dailymotion.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index 47d58330b..7615ecd4b 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -224,7 +224,7 @@ class DailymotionPlaylistIE(DailymotionBaseInfoExtractor):
 
 class DailymotionUserIE(DailymotionPlaylistIE):
     IE_NAME = 'dailymotion:user'
-    _VALID_URL = r'https?://(?:www\.)?dailymotion\.[a-z]{2,3}/user/(?P<user>[^/]+)'
+    _VALID_URL = r'https?://(?:www\.)?dailymotion\.[a-z]{2,3}/(?:old/)?user/(?P<user>[^/]+)'
     _PAGE_TEMPLATE = 'http://www.dailymotion.com/user/%s/%s'
     _TESTS = [{
         'url': 'https://www.dailymotion.com/user/nqtv',

From 1dc2726f8da5ec7726e3c45ee718ce4464b83ead Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 9 Apr 2015 00:21:19 +0200
Subject: [PATCH 0899/6219] release 2015.04.09

---
 README.md              | 194 ++++++++++++++++++++---------------------
 docs/supportedsites.md |   5 ++
 youtube_dl/version.py  |   2 +-
 3 files changed, 103 insertions(+), 98 deletions(-)

diff --git a/README.md b/README.md
index 4f9fc8174..caa1478d9 100644
--- a/README.md
+++ b/README.md
@@ -45,21 +45,21 @@ which means you can modify it, redistribute it or use it however you like.
     youtube-dl [OPTIONS] URL [URL...]
 
 # OPTIONS
-    -h, --help                       print this help text and exit
-    --version                        print program version and exit
-    -U, --update                     update this program to latest version. Make sure that you have sufficient permissions (run with sudo if needed)
-    -i, --ignore-errors              continue on download errors, for example to skip unavailable videos in a playlist
+    -h, --help                       Print this help text and exit
+    --version                        Print program version and exit
+    -U, --update                     Update this program to latest version. Make sure that you have sufficient permissions (run with sudo if needed)
+    -i, --ignore-errors              Continue on download errors, for example to skip unavailable videos in a playlist
     --abort-on-error                 Abort downloading of further videos (in the playlist or the command line) if an error occurs
-    --dump-user-agent                display the current browser identification
+    --dump-user-agent                Display the current browser identification
     --list-extractors                List all supported extractors and the URLs they would handle
     --extractor-descriptions         Output descriptions of all supported extractors
-    --default-search PREFIX          Use this prefix for unqualified URLs. For example "gvsearch2:" downloads two videos from google videos for  youtube-dl "large apple".
+    --default-search PREFIX          Use this prefix for unqualified URLs. For example "gvsearch2:" downloads two videos from google videos for youtube-dl "large apple".
                                      Use the value "auto" to let youtube-dl guess ("auto_warning" to emit a warning when guessing). "error" just throws an error. The
                                      default value "fixup_error" repairs broken URLs, but emits an error if this is not possible instead of searching.
     --ignore-config                  Do not read configuration files. When given in the global configuration file /etc/youtube-dl.conf: Do not read the user configuration
                                      in ~/.config/youtube-dl/config (%APPDATA%/youtube-dl/config.txt on Windows)
     --flat-playlist                  Do not extract the videos of a playlist, only list them.
-    --no-color                       Do not emit color codes in output.
+    --no-color                       Do not emit color codes in output
 
 ## Network Options:
     --proxy URL                      Use the specified HTTP/HTTPS proxy. Pass in an empty string (--proxy "") for direct connection
@@ -71,70 +71,70 @@ which means you can modify it, redistribute it or use it however you like.
                                      not present) is used for the actual downloading. (experimental)
 
 ## Video Selection:
-    --playlist-start NUMBER          playlist video to start at (default is 1)
-    --playlist-end NUMBER            playlist video to end at (default is last)
-    --playlist-items ITEM_SPEC       playlist video items to download. Specify indices of the videos in the playlist seperated by commas like: "--playlist-items 1,2,5,8"
+    --playlist-start NUMBER          Playlist video to start at (default is 1)
+    --playlist-end NUMBER            Playlist video to end at (default is last)
+    --playlist-items ITEM_SPEC       Playlist video items to download. Specify indices of the videos in the playlist seperated by commas like: "--playlist-items 1,2,5,8"
                                      if you want to download videos indexed 1, 2, 5, 8 in the playlist. You can specify range: "--playlist-items 1-3,7,10-13", it will
                                      download the videos at index 1, 2, 3, 7, 10, 11, 12 and 13.
-    --match-title REGEX              download only matching titles (regex or caseless sub-string)
-    --reject-title REGEX             skip download for matching titles (regex or caseless sub-string)
+    --match-title REGEX              Download only matching titles (regex or caseless sub-string)
+    --reject-title REGEX             Skip download for matching titles (regex or caseless sub-string)
     --max-downloads NUMBER           Abort after downloading NUMBER files
     --min-filesize SIZE              Do not download any videos smaller than SIZE (e.g. 50k or 44.6m)
     --max-filesize SIZE              Do not download any videos larger than SIZE (e.g. 50k or 44.6m)
-    --date DATE                      download only videos uploaded in this date
-    --datebefore DATE                download only videos uploaded on or before this date (i.e. inclusive)
-    --dateafter DATE                 download only videos uploaded on or after this date (i.e. inclusive)
+    --date DATE                      Download only videos uploaded in this date
+    --datebefore DATE                Download only videos uploaded on or before this date (i.e. inclusive)
+    --dateafter DATE                 Download only videos uploaded on or after this date (i.e. inclusive)
     --min-views COUNT                Do not download any videos with less than COUNT views
     --max-views COUNT                Do not download any videos with more than COUNT views
-    --match-filter FILTER            (Experimental) Generic video filter. Specify any key (see help for -o for a list of available keys) to match if the key is present,
+    --match-filter FILTER            Generic video filter (experimental). Specify any key (see help for -o for a list of available keys) to match if the key is present,
                                      !key to check if the key is not present,key > NUMBER (like "comment_count > 12", also works with >=, <, <=, !=, =) to compare against
                                      a number, and & to require multiple matches. Values which are not known are excluded unless you put a question mark (?) after the
                                      operator.For example, to only match videos that have been liked more than 100 times and disliked less than 50 times (or the dislike
                                      functionality is not available at the given service), but who also have a description, use  --match-filter "like_count > 100 &
                                      dislike_count <? 50 & description" .
-    --no-playlist                    If the URL refers to a video and a playlist, download only the video.
-    --yes-playlist                   If the URL refers to a video and a playlist, download the playlist.
-    --age-limit YEARS                download only videos suitable for the given age
+    --no-playlist                    Download only the video, if the URL refers to a video and a playlist.
+    --yes-playlist                   Download the playlist, if the URL refers to a video and a playlist.
+    --age-limit YEARS                Download only videos suitable for the given age
     --download-archive FILE          Download only videos not listed in the archive file. Record the IDs of all downloaded videos in it.
     --include-ads                    Download advertisements as well (experimental)
 
 ## Download Options:
-    -r, --rate-limit LIMIT           maximum download rate in bytes per second (e.g. 50K or 4.2M)
-    -R, --retries RETRIES            number of retries (default is 10), or "infinite".
-    --buffer-size SIZE               size of download buffer (e.g. 1024 or 16K) (default is 1024)
-    --no-resize-buffer               do not automatically adjust the buffer size. By default, the buffer size is automatically resized from an initial value of SIZE.
+    -r, --rate-limit LIMIT           Maximum download rate in bytes per second (e.g. 50K or 4.2M)
+    -R, --retries RETRIES            Number of retries (default is 10), or "infinite".
+    --buffer-size SIZE               Size of download buffer (e.g. 1024 or 16K) (default is 1024)
+    --no-resize-buffer               Do not automatically adjust the buffer size. By default, the buffer size is automatically resized from an initial value of SIZE.
     --playlist-reverse               Download playlist videos in reverse order
-    --xattr-set-filesize             (experimental) set file xattribute ytdl.filesize with expected filesize
-    --hls-prefer-native              (experimental) Use the native HLS downloader instead of ffmpeg.
+    --xattr-set-filesize             Set file xattribute ytdl.filesize with expected filesize (experimental)
+    --hls-prefer-native              Use the native HLS downloader instead of ffmpeg (experimental)
     --external-downloader COMMAND    Use the specified external downloader. Currently supports aria2c,curl,wget
-    --external-downloader-args ARGS  Give these arguments to the external downloader.
+    --external-downloader-args ARGS  Give these arguments to the external downloader
 
 ## Filesystem Options:
-    -a, --batch-file FILE            file containing URLs to download ('-' for stdin)
-    --id                             use only video ID in file name
-    -o, --output TEMPLATE            output filename template. Use %(title)s to get the title, %(uploader)s for the uploader name, %(uploader_id)s for the uploader
+    -a, --batch-file FILE            File containing URLs to download ('-' for stdin)
+    --id                             Use only video ID in file name
+    -o, --output TEMPLATE            Output filename template. Use %(title)s to get the title, %(uploader)s for the uploader name, %(uploader_id)s for the uploader
                                      nickname if different, %(autonumber)s to get an automatically incremented number, %(ext)s for the filename extension, %(format)s for
-                                     the format description (like "22 - 1280x720" or "HD"), %(format_id)s for the unique id of the format (like Youtube's itags: "137"),
+                                     the format description (like "22 - 1280x720" or "HD"), %(format_id)s for the unique id of the format (like YouTube's itags: "137"),
                                      %(upload_date)s for the upload date (YYYYMMDD), %(extractor)s for the provider (youtube, metacafe, etc), %(id)s for the video id,
                                      %(playlist_title)s, %(playlist_id)s, or %(playlist)s (=title if present, ID otherwise) for the playlist the video is in,
                                      %(playlist_index)s for the position in the playlist. %(height)s and %(width)s for the width and height of the video format.
                                      %(resolution)s for a textual description of the resolution of the video format. %% for a literal percent. Use - to output to stdout.
                                      Can also be used to download to a different directory, for example with -o '/my/downloads/%(uploader)s/%(title)s-%(id)s.%(ext)s' .
-    --autonumber-size NUMBER         Specifies the number of digits in %(autonumber)s when it is present in output filename template or --auto-number option is given
+    --autonumber-size NUMBER         Specify the number of digits in %(autonumber)s when it is present in output filename template or --auto-number option is given
     --restrict-filenames             Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames
-    -A, --auto-number                [deprecated; use  -o "%(autonumber)s-%(title)s.%(ext)s" ] number downloaded files starting from 00000
-    -t, --title                      [deprecated] use title in file name (default)
-    -l, --literal                    [deprecated] alias of --title
-    -w, --no-overwrites              do not overwrite files
-    -c, --continue                   force resume of partially downloaded files. By default, youtube-dl will resume downloads if possible.
-    --no-continue                    do not resume partially downloaded files (restart from beginning)
-    --no-part                        do not use .part files - write directly into output file
-    --no-mtime                       do not use the Last-modified header to set the file modification time
-    --write-description              write video description to a .description file
-    --write-info-json                write video metadata to a .info.json file
-    --write-annotations              write video annotations to a .annotation file
-    --load-info FILE                 json file containing the video information (created with the "--write-json" option)
-    --cookies FILE                   file to read cookies from and dump cookie jar in
+    -A, --auto-number                [deprecated; use  -o "%(autonumber)s-%(title)s.%(ext)s" ] Number downloaded files starting from 00000
+    -t, --title                      [deprecated] Use title in file name (default)
+    -l, --literal                    [deprecated] Alias of --title
+    -w, --no-overwrites              Do not overwrite files
+    -c, --continue                   Force resume of partially downloaded files. By default, youtube-dl will resume downloads if possible.
+    --no-continue                    Do not resume partially downloaded files (restart from beginning)
+    --no-part                        Do not use .part files - write directly into output file
+    --no-mtime                       Do not use the Last-modified header to set the file modification time
+    --write-description              Write video description to a .description file
+    --write-info-json                Write video metadata to a .info.json file
+    --write-annotations              Write video annotations to a .annotation file
+    --load-info FILE                 JSON file containing the video information (created with the "--write-info-json" option)
+    --cookies FILE                   File to read cookies from and dump cookie jar in
     --cache-dir DIR                  Location in the filesystem where youtube-dl can store some downloaded information permanently. By default $XDG_CACHE_HOME/youtube-dl
                                      or ~/.cache/youtube-dl . At the moment, only YouTube player files (for videos with obfuscated signatures) are cached, but that may
                                      change.
@@ -142,49 +142,49 @@ which means you can modify it, redistribute it or use it however you like.
     --rm-cache-dir                   Delete all filesystem cache files
 
 ## Thumbnail images:
-    --write-thumbnail                write thumbnail image to disk
-    --write-all-thumbnails           write all thumbnail image formats to disk
+    --write-thumbnail                Write thumbnail image to disk
+    --write-all-thumbnails           Write all thumbnail image formats to disk
     --list-thumbnails                Simulate and list all available thumbnail formats
 
 ## Verbosity / Simulation Options:
-    -q, --quiet                      activates quiet mode
+    -q, --quiet                      Activate quiet mode
     --no-warnings                    Ignore warnings
-    -s, --simulate                   do not download the video and do not write anything to disk
-    --skip-download                  do not download the video
-    -g, --get-url                    simulate, quiet but print URL
-    -e, --get-title                  simulate, quiet but print title
-    --get-id                         simulate, quiet but print id
-    --get-thumbnail                  simulate, quiet but print thumbnail URL
-    --get-description                simulate, quiet but print video description
-    --get-duration                   simulate, quiet but print video length
-    --get-filename                   simulate, quiet but print output filename
-    --get-format                     simulate, quiet but print output format
-    -j, --dump-json                  simulate, quiet but print JSON information. See --output for a description of available keys.
-    -J, --dump-single-json           simulate, quiet but print JSON information for each command-line argument. If the URL refers to a playlist, dump the whole playlist
+    -s, --simulate                   Do not download the video and do not write anything to disk
+    --skip-download                  Do not download the video
+    -g, --get-url                    Simulate, quiet but print URL
+    -e, --get-title                  Simulate, quiet but print title
+    --get-id                         Simulate, quiet but print id
+    --get-thumbnail                  Simulate, quiet but print thumbnail URL
+    --get-description                Simulate, quiet but print video description
+    --get-duration                   Simulate, quiet but print video length
+    --get-filename                   Simulate, quiet but print output filename
+    --get-format                     Simulate, quiet but print output format
+    -j, --dump-json                  Simulate, quiet but print JSON information. See --output for a description of available keys.
+    -J, --dump-single-json           Simulate, quiet but print JSON information for each command-line argument. If the URL refers to a playlist, dump the whole playlist
                                      information in a single line.
     --print-json                     Be quiet and print the video information as JSON (video is still being downloaded).
-    --newline                        output progress bar as new lines
-    --no-progress                    do not print progress bar
-    --console-title                  display progress in console titlebar
-    -v, --verbose                    print various debugging information
-    --dump-pages                     print downloaded pages to debug problems (very verbose)
+    --newline                        Output progress bar as new lines
+    --no-progress                    Do not print progress bar
+    --console-title                  Display progress in console titlebar
+    -v, --verbose                    Print various debugging information
+    --dump-pages                     Print downloaded pages to debug problems (very verbose)
     --write-pages                    Write downloaded intermediary pages to files in the current directory to debug problems
     --print-traffic                  Display sent and read HTTP traffic
-    -C, --call-home                  Contact the youtube-dl server for debugging.
-    --no-call-home                   Do NOT contact the youtube-dl server for debugging.
+    -C, --call-home                  Contact the youtube-dl server for debugging
+    --no-call-home                   Do NOT contact the youtube-dl server for debugging
 
 ## Workarounds:
     --encoding ENCODING              Force the specified encoding (experimental)
-    --no-check-certificate           Suppress HTTPS certificate validation.
+    --no-check-certificate           Suppress HTTPS certificate validation
     --prefer-insecure                Use an unencrypted connection to retrieve information about the video. (Currently supported only for YouTube)
-    --user-agent UA                  specify a custom user agent
-    --referer URL                    specify a custom referer, use if the video access is restricted to one domain
-    --add-header FIELD:VALUE         specify a custom HTTP header and its value, separated by a colon ':'. You can use this option multiple times
+    --user-agent UA                  Specify a custom user agent
+    --referer URL                    Specify a custom referer, use if the video access is restricted to one domain
+    --add-header FIELD:VALUE         Specify a custom HTTP header and its value, separated by a colon ':'. You can use this option multiple times
     --bidi-workaround                Work around terminals that lack bidirectional text support. Requires bidiv or fribidi executable in PATH
     --sleep-interval SECONDS         Number of seconds to sleep before each download.
 
 ## Video Format Options:
-    -f, --format FORMAT              video format code, specify the order of preference using slashes, as in -f 22/17/18 .  Instead of format codes, you can select by
+    -f, --format FORMAT              Video format code, specify the order of preference using slashes, as in -f 22/17/18 .  Instead of format codes, you can select by
                                      extension for the extensions aac, m4a, mp3, mp4, ogg, wav, webm. You can also use the special names "best", "bestvideo", "bestaudio",
                                      "worst".  You can filter the video results by putting a condition in brackets, as in -f "best[height=720]" (or -f "[filesize>10M]").
                                      This works for filesize, height, width, tbr, abr, vbr, asr, and fps and the comparisons <, <=, >, >=, =, != and for ext, acodec,
@@ -194,44 +194,44 @@ which means you can modify it, redistribute it or use it however you like.
                                      Use commas to download multiple audio formats, such as -f  136/137/mp4/bestvideo,140/m4a/bestaudio. You can merge the video and audio
                                      of two formats into a single file using -f <video-format>+<audio-format> (requires ffmpeg or avconv), for example -f
                                      bestvideo+bestaudio.
-    --all-formats                    download all available video formats
-    --prefer-free-formats            prefer free video formats unless a specific one is requested
-    --max-quality FORMAT             highest quality format to download
-    -F, --list-formats               list all available formats
+    --all-formats                    Download all available video formats
+    --prefer-free-formats            Prefer free video formats unless a specific one is requested
+    --max-quality FORMAT             Highest quality format to download
+    -F, --list-formats               List all available formats
     --youtube-skip-dash-manifest     Do not download the DASH manifest on YouTube videos
     --merge-output-format FORMAT     If a merge is required (e.g. bestvideo+bestaudio), output to given container format. One of mkv, mp4, ogg, webm, flv.Ignored if no
                                      merge is required
 
 ## Subtitle Options:
-    --write-sub                      write subtitle file
-    --write-auto-sub                 write automatic subtitle file (youtube only)
-    --all-subs                       downloads all the available subtitles of the video
-    --list-subs                      lists all available subtitles for the video
-    --sub-format FORMAT              subtitle format, accepts formats preference, for example: "ass/srt/best"
-    --sub-lang LANGS                 languages of the subtitles to download (optional) separated by commas, use IETF language tags like 'en,pt'
+    --write-sub                      Write subtitle file
+    --write-auto-sub                 Write automatic subtitle file (YouTube only)
+    --all-subs                       Download all the available subtitles of the video
+    --list-subs                      List all available subtitles for the video
+    --sub-format FORMAT              Subtitle format, accepts formats preference, for example: "srt" or "ass/srt/best"
+    --sub-lang LANGS                 Languages of the subtitles to download (optional) separated by commas, use IETF language tags like 'en,pt'
 
 ## Authentication Options:
-    -u, --username USERNAME          login with this account ID
-    -p, --password PASSWORD          account password. If this option is left out, youtube-dl will ask interactively.
-    -2, --twofactor TWOFACTOR        two-factor auth code
-    -n, --netrc                      use .netrc authentication data
-    --video-password PASSWORD        video password (vimeo, smotri)
+    -u, --username USERNAME          Login with this account ID
+    -p, --password PASSWORD          Account password. If this option is left out, youtube-dl will ask interactively.
+    -2, --twofactor TWOFACTOR        Two-factor auth code
+    -n, --netrc                      Use .netrc authentication data
+    --video-password PASSWORD        Video password (vimeo, smotri)
 
 ## Post-processing Options:
-    -x, --extract-audio              convert video files to audio-only files (requires ffmpeg or avconv and ffprobe or avprobe)
-    --audio-format FORMAT            "best", "aac", "vorbis", "mp3", "m4a", "opus", or "wav"; "best" by default
-    --audio-quality QUALITY          ffmpeg/avconv audio quality specification, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K
-                                     (default 5)
+    -x, --extract-audio              Convert video files to audio-only files (requires ffmpeg or avconv and ffprobe or avprobe)
+    --audio-format FORMAT            Specify audio format: "best", "aac", "vorbis", "mp3", "m4a", "opus", or "wav"; "best" by default
+    --audio-quality QUALITY          Specify ffmpeg/avconv audio quality, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K (default
+                                     5)
     --recode-video FORMAT            Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm|mkv)
-    -k, --keep-video                 keeps the video file on disk after the post-processing; the video is erased by default
-    --no-post-overwrites             do not overwrite post-processed files; the post-processed files are overwritten by default
-    --embed-subs                     embed subtitles in the video (only for mp4 videos)
-    --embed-thumbnail                embed thumbnail in the audio as cover art
-    --add-metadata                   write metadata to the video file
-    --metadata-from-title FORMAT     parse additional metadata like song title / artist from the video title. The format syntax is the same as --output, the parsed
+    -k, --keep-video                 Keep the video file on disk after the post-processing; the video is erased by default
+    --no-post-overwrites             Do not overwrite post-processed files; the post-processed files are overwritten by default
+    --embed-subs                     Embed subtitles in the video (only for mp4 videos)
+    --embed-thumbnail                Embed thumbnail in the audio as cover art
+    --add-metadata                   Write metadata to the video file
+    --metadata-from-title FORMAT     Parse additional metadata like song title / artist from the video title. The format syntax is the same as --output, the parsed
                                      parameters replace existing values. Additional templates: %(album), %(artist). Example: --metadata-from-title "%(artist)s -
                                      %(title)s" matches a title like "Coldplay - Paradise"
-    --xattrs                         write metadata to the video file's xattrs (using dublin core and xdg standards)
+    --xattrs                         Write metadata to the video file's xattrs (using dublin core and xdg standards)
     --fixup POLICY                   Automatically correct known faults of the file. One of never (do nothing), warn (only emit a warning), detect_or_warn(the default;
                                      fix file if we can, warn otherwise)
     --prefer-avconv                  Prefer avconv over ffmpeg for running the postprocessors (default)
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 2785b9587..c85a39918 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -166,6 +166,7 @@
  - **Gamekings**
  - **GameOne**
  - **gameone:playlist**
+ - **Gamersyde**
  - **GameSpot**
  - **GameStar**
  - **Gametrailers**
@@ -351,6 +352,7 @@
  - **PornHub**
  - **PornHubPlaylist**
  - **Pornotube**
+ - **PornoVoisines**
  - **PornoXO**
  - **PrimeShareTV**
  - **PromptFile**
@@ -362,6 +364,7 @@
  - **radio.de**
  - **radiobremen**
  - **radiofrance**
+ - **RadioJavan**
  - **Rai**
  - **RBMARadio**
  - **RedTube**
@@ -422,6 +425,7 @@
  - **southpark.cc.com**
  - **southpark.de**
  - **Space**
+ - **SpankBang**
  - **Spankwire**
  - **Spiegel**
  - **Spiegel:Article**: Articles on spiegel.de
@@ -501,6 +505,7 @@
  - **Ubu**
  - **udemy**
  - **udemy:course**
+ - **UDNEmbed**
  - **Ultimedia**
  - **Unistra**
  - **Urort**: NRK P3 Urørt
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index e1c385bec..1095fea2f 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.04.03'
+__version__ = '2015.04.09'

From ce73839fe4bc1ac43d7a6540df040139f82948b1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 9 Apr 2015 14:01:33 +0200
Subject: [PATCH 0900/6219] [rtve] Detect videos that are no longer available

---
 youtube_dl/extractor/rtve.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/rtve.py b/youtube_dl/extractor/rtve.py
index 8d9be1b98..849300140 100644
--- a/youtube_dl/extractor/rtve.py
+++ b/youtube_dl/extractor/rtve.py
@@ -8,6 +8,7 @@ import time
 from .common import InfoExtractor
 from ..compat import compat_urlparse
 from ..utils import (
+    ExtractorError,
     float_or_none,
     remove_end,
     std_headers,
@@ -98,6 +99,8 @@ class RTVEALaCartaIE(InfoExtractor):
         info = self._download_json(
             'http://www.rtve.es/api/videos/%s/config/alacarta_videos.json' % video_id,
             video_id)['page']['items'][0]
+        if info['state'] == 'DESPU':
+            raise ExtractorError('The video is no longer available', expected=True)
         png_url = 'http://www.rtve.es/ztnr/movil/thumbnail/%s/videos/%s.png' % (self._manager, video_id)
         png = self._download_webpage(png_url, video_id, 'Downloading url information')
         video_url = _decrypt_url(png)

From aa2af7ba7469370f987aa178ac031fdad00aae3d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 9 Apr 2015 19:53:00 +0600
Subject: [PATCH 0901/6219] [dumpert] Add nsfw cookie (Closes #5382)

---
 youtube_dl/extractor/dumpert.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/dumpert.py b/youtube_dl/extractor/dumpert.py
index e43bc81b2..9c594b757 100644
--- a/youtube_dl/extractor/dumpert.py
+++ b/youtube_dl/extractor/dumpert.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 import base64
 
 from .common import InfoExtractor
+from ..compat import compat_urllib_request
 from ..utils import qualities
 
 
@@ -23,7 +24,10 @@ class DumpertIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
+
+        req = compat_urllib_request.Request(url)
+        req.add_header('Cookie', 'nsfw=1')
+        webpage = self._download_webpage(req, video_id)
 
         files_base64 = self._search_regex(
             r'data-files="([^"]+)"', webpage, 'data files')

From 6ac41a4ef50029fa3b0eee5083805d0133a4d1dd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 9 Apr 2015 22:32:22 +0600
Subject: [PATCH 0902/6219] [vine] Zero rate videos is perfectly valid (#5389)

---
 youtube_dl/extractor/vine.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vine.py b/youtube_dl/extractor/vine.py
index c3187cfeb..353791e1d 100644
--- a/youtube_dl/extractor/vine.py
+++ b/youtube_dl/extractor/vine.py
@@ -37,7 +37,7 @@ class VineIE(InfoExtractor):
             'vcodec': f['format'],
             'quality': f['rate'],
             'url': f['videoUrl'],
-        } for f in data['videoUrls'] if f.get('rate')]
+        } for f in data['videoUrls']]
 
         self._sort_formats(formats)
 

From 58a9f1b86404e3a6c26c7143cb408db793aa5946 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 9 Apr 2015 22:32:48 +0600
Subject: [PATCH 0903/6219] [vine] Fix post data regex (Closes #5389)

---
 youtube_dl/extractor/vine.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vine.py b/youtube_dl/extractor/vine.py
index 353791e1d..13c13152e 100644
--- a/youtube_dl/extractor/vine.py
+++ b/youtube_dl/extractor/vine.py
@@ -30,7 +30,8 @@ class VineIE(InfoExtractor):
         webpage = self._download_webpage('https://vine.co/v/' + video_id, video_id)
 
         data = json.loads(self._html_search_regex(
-            r'window\.POST_DATA = { %s: ({.+?}) }' % video_id, webpage, 'vine data'))
+            r'window\.POST_DATA = { %s: ({.+?}) };\s*</script>' % video_id,
+            webpage, 'vine data'))
 
         formats = [{
             'format_id': '%(format)s-%(rate)s' % f,

From 3359fb661fdf3f9308611dc10c5b579b7a7ef56e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 9 Apr 2015 22:37:54 +0600
Subject: [PATCH 0904/6219] [vine] Add tests for #5389

---
 youtube_dl/extractor/vine.py | 30 ++++++++++++++++++++++++++++--
 1 file changed, 28 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vine.py b/youtube_dl/extractor/vine.py
index 13c13152e..804adbdb0 100644
--- a/youtube_dl/extractor/vine.py
+++ b/youtube_dl/extractor/vine.py
@@ -10,7 +10,7 @@ from ..utils import unified_strdate
 
 class VineIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?vine\.co/v/(?P<id>\w+)'
-    _TEST = {
+    _TESTS = [{
         'url': 'https://vine.co/v/b9KOOWX7HUx',
         'md5': '2f36fed6235b16da96ce9b4dc890940d',
         'info_dict': {
@@ -23,7 +23,33 @@ class VineIE(InfoExtractor):
             'uploader': 'Jack Dorsey',
             'uploader_id': '76',
         },
-    }
+    }, {
+        'url': 'https://vine.co/v/MYxVapFvz2z',
+        'md5': '7b9a7cbc76734424ff942eb52c8f1065',
+        'info_dict': {
+            'id': 'MYxVapFvz2z',
+            'ext': 'mp4',
+            'title': 'Fuck Da Police #Mikebrown #justice #ferguson #prayforferguson #protesting #NMOS14',
+            'alt_title': 'Vine by Luna',
+            'description': 'Fuck Da Police #Mikebrown #justice #ferguson #prayforferguson #protesting #NMOS14',
+            'upload_date': '20140815',
+            'uploader': 'Luna',
+            'uploader_id': '1102363502380728320',
+        },
+    }, {
+        'url': 'https://vine.co/v/bxVjBbZlPUH',
+        'md5': 'ea27decea3fa670625aac92771a96b73',
+        'info_dict': {
+            'id': 'bxVjBbZlPUH',
+            'ext': 'mp4',
+            'title': '#mw3 #ac130 #killcam #angelofdeath',
+            'alt_title': 'Vine by Z3k3',
+            'description': '#mw3 #ac130 #killcam #angelofdeath',
+            'upload_date': '20130430',
+            'uploader': 'Z3k3',
+            'uploader_id': '936470460173008896',
+        },
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From 64f1aba8f1a6dba88e0dd0edc799fee978c7ce76 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 9 Apr 2015 22:40:18 +0600
Subject: [PATCH 0905/6219] [vine] Extend _VALID_URL

---
 youtube_dl/extractor/vine.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vine.py b/youtube_dl/extractor/vine.py
index 804adbdb0..a4d5af147 100644
--- a/youtube_dl/extractor/vine.py
+++ b/youtube_dl/extractor/vine.py
@@ -9,7 +9,7 @@ from ..utils import unified_strdate
 
 
 class VineIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?vine\.co/v/(?P<id>\w+)'
+    _VALID_URL = r'https?://(?:www\.)?vine\.co/(?:v|oembed)/(?P<id>\w+)'
     _TESTS = [{
         'url': 'https://vine.co/v/b9KOOWX7HUx',
         'md5': '2f36fed6235b16da96ce9b4dc890940d',
@@ -49,6 +49,9 @@ class VineIE(InfoExtractor):
             'uploader': 'Z3k3',
             'uploader_id': '936470460173008896',
         },
+    }, {
+        'url': 'https://vine.co/oembed/MYxVapFvz2z.json',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 4c4780c25e20d6ad281faff5d6aede3df4b58f66 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 9 Apr 2015 22:41:41 +0600
Subject: [PATCH 0906/6219] [vine] Modernize

---
 youtube_dl/extractor/vine.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/vine.py b/youtube_dl/extractor/vine.py
index a4d5af147..d4f5a991e 100644
--- a/youtube_dl/extractor/vine.py
+++ b/youtube_dl/extractor/vine.py
@@ -58,9 +58,11 @@ class VineIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage('https://vine.co/v/' + video_id, video_id)
 
-        data = json.loads(self._html_search_regex(
-            r'window\.POST_DATA = { %s: ({.+?}) };\s*</script>' % video_id,
-            webpage, 'vine data'))
+        data = self._parse_json(
+            self._html_search_regex(
+                r'window\.POST_DATA = { %s: ({.+?}) };\s*</script>' % video_id,
+                webpage, 'vine data'),
+            video_id)
 
         formats = [{
             'format_id': '%(format)s-%(rate)s' % f,

From ce9f47de99f3f607ad940897e457d02ea084795d Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Thu, 9 Apr 2015 23:54:53 +0300
Subject: [PATCH 0907/6219] [teamcoco] Fix extraction

---
 youtube_dl/extractor/teamcoco.py | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/teamcoco.py b/youtube_dl/extractor/teamcoco.py
index a46a7ecba..b3cfe4e4f 100644
--- a/youtube_dl/extractor/teamcoco.py
+++ b/youtube_dl/extractor/teamcoco.py
@@ -4,7 +4,10 @@ import base64
 import re
 
 from .common import InfoExtractor
-from ..utils import qualities
+from ..utils import (
+    ExtractorError,
+    qualities,
+)
 
 
 class TeamcocoIE(InfoExtractor):
@@ -49,14 +52,12 @@ class TeamcocoIE(InfoExtractor):
             video_id = self._html_search_regex(
                 self._VIDEO_ID_REGEXES, webpage, 'video id')
 
-        embed_url = 'http://teamcoco.com/embed/v/%s' % video_id
-        embed = self._download_webpage(
-            embed_url, video_id, 'Downloading embed page')
-
-        player_data = self._parse_json(self._search_regex(
-            r'Y\.Ginger\.Module\.Player(?:;var\s*player\s*=\s*new\s*m)?\((\{.*?\})\);', embed, 'player data'), video_id)
+        preloads = re.findall(r'"preload":\s*"([^"]+)"', webpage)
+        if not preloads:
+            raise ExtractorError('Preload information could not be extracted')
+        preload = max([(len(p), p) for p in preloads])[1]
         data = self._parse_json(
-            base64.b64decode(player_data['preload'].encode('ascii')).decode('utf-8'), video_id)
+            base64.b64decode(preload.encode('ascii')).decode('utf-8'), video_id)
 
         formats = []
         get_quality = qualities(['500k', '480p', '1000k', '720p', '1080p'])

From 5bb6328cb944f67f7981eddc40d92998a153f00d Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Thu, 9 Apr 2015 23:57:51 +0300
Subject: [PATCH 0908/6219] [teamcoco] Extract m3u8 URLs

---
 youtube_dl/extractor/teamcoco.py | 34 ++++++++++++++++++--------------
 1 file changed, 19 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/teamcoco.py b/youtube_dl/extractor/teamcoco.py
index b3cfe4e4f..41677503c 100644
--- a/youtube_dl/extractor/teamcoco.py
+++ b/youtube_dl/extractor/teamcoco.py
@@ -62,23 +62,27 @@ class TeamcocoIE(InfoExtractor):
         formats = []
         get_quality = qualities(['500k', '480p', '1000k', '720p', '1080p'])
         for filed in data['files']:
-            m_format = re.search(r'(\d+(k|p))\.mp4', filed['url'])
-            if m_format is not None:
-                format_id = m_format.group(1)
+            if filed['type'] == 'hls':
+                formats.extend(self._extract_m3u8_formats(
+                    filed['url'], video_id, ext='mp4'))
             else:
-                format_id = filed['bitrate']
-            tbr = (
-                int(filed['bitrate'])
-                if filed['bitrate'].isdigit()
-                else None)
+                m_format = re.search(r'(\d+(k|p))\.mp4', filed['url'])
+                if m_format is not None:
+                    format_id = m_format.group(1)
+                else:
+                    format_id = filed['bitrate']
+                tbr = (
+                    int(filed['bitrate'])
+                    if filed['bitrate'].isdigit()
+                    else None)
 
-            formats.append({
-                'url': filed['url'],
-                'ext': 'mp4',
-                'tbr': tbr,
-                'format_id': format_id,
-                'quality': get_quality(format_id),
-            })
+                formats.append({
+                    'url': filed['url'],
+                    'ext': 'mp4',
+                    'tbr': tbr,
+                    'format_id': format_id,
+                    'quality': get_quality(format_id),
+                })
 
         self._sort_formats(formats)
 

From 7088f5b5fa07381ec9d484c8ef83616724654e3f Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Fri, 10 Apr 2015 02:03:38 +0300
Subject: [PATCH 0909/6219] [teamcoco] Extract duration

---
 youtube_dl/extractor/teamcoco.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/teamcoco.py b/youtube_dl/extractor/teamcoco.py
index 41677503c..1caf08cb7 100644
--- a/youtube_dl/extractor/teamcoco.py
+++ b/youtube_dl/extractor/teamcoco.py
@@ -21,6 +21,7 @@ class TeamcocoIE(InfoExtractor):
                 'ext': 'mp4',
                 'title': 'Conan Becomes A Mary Kay Beauty Consultant',
                 'description': 'Mary Kay is perhaps the most trusted name in female beauty, so of course Conan is a natural choice to sell their products.',
+                'duration': 504,
                 'age_limit': 0,
             }
         }, {
@@ -31,6 +32,7 @@ class TeamcocoIE(InfoExtractor):
                 'ext': 'mp4',
                 'description': 'Louis C.K. got starstruck by George W. Bush, so what? Part one.',
                 'title': 'Louis C.K. Interview Pt. 1 11/3/11',
+                'duration': 288,
                 'age_limit': 0,
             }
         }
@@ -93,5 +95,6 @@ class TeamcocoIE(InfoExtractor):
             'title': data['title'],
             'thumbnail': data.get('thumb', {}).get('href'),
             'description': data.get('teaser'),
+            'duration': data.get('duration'),
             'age_limit': self._family_friendly_search(webpage),
         }

From 8749477ed0a3cbc85d1726b6526fa5e794ce6072 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Apr 2015 22:27:16 +0600
Subject: [PATCH 0910/6219] [rai] Fix extraction (Closes #5396)

---
 youtube_dl/extractor/rai.py | 72 ++++++++++++++++++++++++++-----------
 1 file changed, 52 insertions(+), 20 deletions(-)

diff --git a/youtube_dl/extractor/rai.py b/youtube_dl/extractor/rai.py
index 144e33982..115cc64cc 100644
--- a/youtube_dl/extractor/rai.py
+++ b/youtube_dl/extractor/rai.py
@@ -13,7 +13,7 @@ from ..utils import (
 
 
 class RaiIE(InfoExtractor):
-    _VALID_URL = r'(?P<url>http://(?:.+?\.)?(?:rai\.it|rai\.tv|rainews\.it)/dl/.+?-(?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})(?:-.+?)?\.html)'
+    _VALID_URL = r'(?P<url>(?P<host>http://(?:.+?\.)?(?:rai\.it|rai\.tv|rainews\.it))/dl/.+?-(?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})(?:-.+?)?\.html)'
     _TESTS = [
         {
             'url': 'http://www.rai.tv/dl/RaiTV/programmi/media/ContentItem-cb27157f-9dd0-4aee-b788-b1f67643a391.html',
@@ -64,32 +64,65 @@ class RaiIE(InfoExtractor):
         },
     ]
 
+    def _extract_relinker_url(self, webpage):
+        return self._proto_relative_url(self._search_regex(
+            [r'name="videourl" content="([^"]+)"', r'var\s+videoURL(?:_MP4)?\s*=\s*"([^"]+)"'],
+            webpage, 'relinker url', default=None))
+
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
+        host = mobj.group('host')
 
-        media = self._download_json('%s?json' % mobj.group('url'), video_id, 'Downloading video JSON')
+        webpage = self._download_webpage(url, video_id)
 
-        title = media.get('name')
-        description = media.get('desc')
-        thumbnail = media.get('image_300') or media.get('image_medium') or media.get('image')
-        duration = parse_duration(media.get('length'))
-        uploader = media.get('author')
-        upload_date = unified_strdate(media.get('date'))
+        relinker_url = self._extract_relinker_url(webpage)
 
-        formats = []
+        if not relinker_url:
+            iframe_path = self._search_regex(
+                r'<iframe[^>]+src="/?(dl/[^"]+\?iframe\b[^"]*)"',
+                webpage, 'iframe')
+            iframe_page = self._download_webpage(
+                '%s/%s' % (host, iframe_path), video_id)
+            relinker_url = self._extract_relinker_url(iframe_page)
 
-        for format_id in ['wmv', 'm3u8', 'mediaUri', 'h264']:
-            media_url = media.get(format_id)
-            if not media_url:
-                continue
-            formats.append({
+        relinker = self._download_json(
+            '%s&output=47' % relinker_url, video_id)
+
+        media_url = relinker['video'][0]
+        ct = relinker.get('ct')
+        if ct == 'f4m':
+            formats = self._extract_f4m_formats(
+                media_url + '&hdcore=3.7.0&plugin=aasp-3.7.0.39.44', video_id)
+        else:
+            formats = [{
                 'url': media_url,
-                'format_id': format_id,
-                'ext': 'mp4',
-            })
+                'format_id': ct,
+            }]
 
-        subtitles = self.extract_subtitles(video_id, url)
+        json_link = self._html_search_meta(
+            'jsonlink', webpage, 'JSON link', default=None)
+        if json_link:
+            media = self._download_json(
+                host + json_link, video_id, 'Downloading video JSON')
+            title = media.get('name')
+            description = media.get('desc')
+            thumbnail = media.get('image_300') or media.get('image_medium') or media.get('image')
+            duration = parse_duration(media.get('length'))
+            uploader = media.get('author')
+            upload_date = unified_strdate(media.get('date'))
+        else:
+            title = self._search_regex(
+                r'var\s+videoTitolo\s*=\s*"([^"]+)";',
+                webpage, 'title', default=None) or self._og_search_title(webpage)
+            description = self._og_search_description(webpage)
+            thumbnail = self._og_search_thumbnail(webpage)
+            duration = None
+            uploader = self._html_search_meta('Editore', webpage, 'uploader')
+            upload_date = unified_strdate(self._html_search_meta(
+                'item-date', webpage, 'upload date'))
+
+        subtitles = self.extract_subtitles(video_id, webpage)
 
         return {
             'id': video_id,
@@ -103,8 +136,7 @@ class RaiIE(InfoExtractor):
             'subtitles': subtitles,
         }
 
-    def _get_subtitles(self, video_id, url):
-        webpage = self._download_webpage(url, video_id)
+    def _get_subtitles(self, video_id, webpage):
         subtitles = {}
         m = re.search(r'<meta name="closedcaption" content="(?P<captions>[^"]+)"', webpage)
         if m:

From d7c78decb0fb4adcf84e5e74cd0d4f858d48e2ee Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Apr 2015 22:41:26 +0600
Subject: [PATCH 0911/6219] [rai] Improve extraction

---
 youtube_dl/extractor/rai.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/rai.py b/youtube_dl/extractor/rai.py
index 115cc64cc..12127c634 100644
--- a/youtube_dl/extractor/rai.py
+++ b/youtube_dl/extractor/rai.py
@@ -82,9 +82,9 @@ class RaiIE(InfoExtractor):
             iframe_path = self._search_regex(
                 r'<iframe[^>]+src="/?(dl/[^"]+\?iframe\b[^"]*)"',
                 webpage, 'iframe')
-            iframe_page = self._download_webpage(
+            webpage = self._download_webpage(
                 '%s/%s' % (host, iframe_path), video_id)
-            relinker_url = self._extract_relinker_url(iframe_page)
+            relinker_url = self._extract_relinker_url(webpage)
 
         relinker = self._download_json(
             '%s&output=47' % relinker_url, video_id)
@@ -112,15 +112,15 @@ class RaiIE(InfoExtractor):
             uploader = media.get('author')
             upload_date = unified_strdate(media.get('date'))
         else:
-            title = self._search_regex(
-                r'var\s+videoTitolo\s*=\s*"([^"]+)";',
-                webpage, 'title', default=None) or self._og_search_title(webpage)
+            title = (self._search_regex(
+                r'var\s+videoTitolo\s*=\s*"(.+?)";',
+                webpage, 'title', default=None) or self._og_search_title(webpage)).replace('\\"', '"')
             description = self._og_search_description(webpage)
             thumbnail = self._og_search_thumbnail(webpage)
             duration = None
             uploader = self._html_search_meta('Editore', webpage, 'uploader')
             upload_date = unified_strdate(self._html_search_meta(
-                'item-date', webpage, 'upload date'))
+                'item-date', webpage, 'upload date', default=None))
 
         subtitles = self.extract_subtitles(video_id, webpage)
 

From cd47a628fcfb4ec49d0559bdedf792d99a52d53d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Apr 2015 22:41:52 +0600
Subject: [PATCH 0912/6219] [rai] Add test for #5396

---
 youtube_dl/extractor/rai.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/youtube_dl/extractor/rai.py b/youtube_dl/extractor/rai.py
index 12127c634..1631faf29 100644
--- a/youtube_dl/extractor/rai.py
+++ b/youtube_dl/extractor/rai.py
@@ -62,6 +62,17 @@ class RaiIE(InfoExtractor):
                 'description': 'Edizione delle ore 20:30 ',
             }
         },
+        {
+            'url': 'http://www.ilcandidato.rai.it/dl/ray/media/Il-Candidato---Primo-episodio-Le-Primarie-28e5525a-b495-45e8-a7c3-bc48ba45d2b6.html',
+            'md5': '02b64456f7cc09f96ff14e7dd489017e',
+            'info_dict': {
+                'id': '28e5525a-b495-45e8-a7c3-bc48ba45d2b6',
+                'ext': 'flv',
+                'title': 'Il Candidato - Primo episodio: "Le Primarie"',
+                'description': 'Primo appuntamento con "Il candidato" con Filippo Timi, alias Piero Zucca presidente!',
+                'uploader': 'RaiTre',
+            }
+        }
     ]
 
     def _extract_relinker_url(self, webpage):

From 66ee7b3234482ddcb24849d3ab64db382fff5bfd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Apr 2015 23:36:28 +0600
Subject: [PATCH 0913/6219] [ted] Extract all formats (Closes #5397)

---
 youtube_dl/extractor/ted.py | 51 +++++++++++++++++++++++++++----------
 1 file changed, 37 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/ted.py b/youtube_dl/extractor/ted.py
index 4cec06f8b..a2dc14c2b 100644
--- a/youtube_dl/extractor/ted.py
+++ b/youtube_dl/extractor/ted.py
@@ -5,9 +5,8 @@ import re
 
 from .common import InfoExtractor
 
-from ..compat import (
-    compat_str,
-)
+from ..compat import compat_str
+from ..utils import int_or_none
 
 
 class TEDIE(InfoExtractor):
@@ -170,17 +169,41 @@ class TEDIE(InfoExtractor):
                 finfo = self._NATIVE_FORMATS.get(f['format_id'])
                 if finfo:
                     f.update(finfo)
-        else:
-            # Use rtmp downloads
-            formats = [{
-                'format_id': f['name'],
-                'url': talk_info['streamer'],
-                'play_path': f['file'],
-                'ext': 'flv',
-                'width': f['width'],
-                'height': f['height'],
-                'tbr': f['bitrate'],
-            } for f in talk_info['resources']['rtmp']]
+
+        for format_id, resources in talk_info['resources'].items():
+            if format_id == 'h264':
+                for resource in resources:
+                    bitrate = int_or_none(resource.get('bitrate'))
+                    formats.append({
+                        'url': resource['file'],
+                        'format_id': '%s-%sk' % (format_id, bitrate),
+                        'tbr': bitrate,
+                    })
+            elif format_id == 'rtmp':
+                streamer = talk_info.get('streamer')
+                if not streamer:
+                    continue
+                for resource in resources:
+                    formats.append({
+                        'format_id': '%s-%s' % (format_id, resource.get('name')),
+                        'url': streamer,
+                        'play_path': resource['file'],
+                        'ext': 'flv',
+                        'width': int_or_none(resource.get('width')),
+                        'height': int_or_none(resource.get('height')),
+                        'tbr': int_or_none(resource.get('bitrate')),
+                    })
+            elif format_id == 'hls':
+                formats.extend(self._extract_m3u8_formats(
+                    resources.get('stream'), video_name, 'mp4', m3u8_id=format_id))
+
+        audio_download = talk_info.get('audioDownload')
+        if audio_download:
+            formats.append({
+                'url': audio_download,
+                'format_id': 'audio',
+            })
+
         self._sort_formats(formats)
 
         video_id = compat_str(talk_info['id'])

From 65939effb55087f584ecd5d4b304eadbdef875d1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 11 Apr 2015 18:52:41 +0600
Subject: [PATCH 0914/6219] [hitbox:live] Fix hls extration (Closes #5315)

---
 youtube_dl/extractor/hitbox.py | 24 ++++++++++++++++++++----
 1 file changed, 20 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/hitbox.py b/youtube_dl/extractor/hitbox.py
index 84bd7c080..d606429ca 100644
--- a/youtube_dl/extractor/hitbox.py
+++ b/youtube_dl/extractor/hitbox.py
@@ -10,6 +10,7 @@ from ..utils import (
     float_or_none,
     int_or_none,
     compat_str,
+    determine_ext,
 )
 
 
@@ -147,12 +148,27 @@ class HitboxLiveIE(HitboxIE):
                 servers.append(base_url)
                 for stream in cdn.get('bitrates'):
                     label = stream.get('label')
-                    if label != 'Auto':
+                    if label == 'Auto':
+                        continue
+                    stream_url = stream.get('url')
+                    if not stream_url:
+                        continue
+                    bitrate = int_or_none(stream.get('bitrate'))
+                    if stream.get('provider') == 'hls' or determine_ext(stream_url) == 'm3u8':
+                        if not stream_url.startswith('http'):
+                            continue
                         formats.append({
-                            'url': '%s/%s' % (base_url, stream.get('url')),
+                            'url': stream_url,
                             'ext': 'mp4',
-                            'vbr': stream.get('bitrate'),
-                            'resolution': label,
+                            'tbr': bitrate,
+                            'format_note': label,
+                            'rtmp_live': True,
+                        })
+                    else:
+                        formats.append({
+                            'url': '%s/%s' % (base_url, stream_url),
+                            'ext': 'mp4',
+                            'tbr': bitrate,
                             'rtmp_live': True,
                             'format_note': host,
                             'page_url': url,

From 7a91d1fc43f8873e6636c316006c908d7efe2f07 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 11 Apr 2015 20:03:12 +0600
Subject: [PATCH 0915/6219] [crooksandliars] Improve embed extractor and remove
 article extractor

---
 youtube_dl/extractor/__init__.py       |  2 +-
 youtube_dl/extractor/crooksandliars.py | 87 +++++++++++---------------
 2 files changed, 39 insertions(+), 50 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 894aa5b43..bbf3be41d 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -90,7 +90,7 @@ from .commonmistakes import CommonMistakesIE, UnicodeBOMIE
 from .condenast import CondeNastIE
 from .cracked import CrackedIE
 from .criterion import CriterionIE
-from .crooksandliars import CrooksAndLiarsIE, CrooksAndLiarsArticleIE
+from .crooksandliars import CrooksAndLiarsIE
 from .crunchyroll import (
     CrunchyrollIE,
     CrunchyrollShowPlaylistIE
diff --git a/youtube_dl/extractor/crooksandliars.py b/youtube_dl/extractor/crooksandliars.py
index cee0603f4..143509004 100644
--- a/youtube_dl/extractor/crooksandliars.py
+++ b/youtube_dl/extractor/crooksandliars.py
@@ -1,71 +1,60 @@
 from __future__ import unicode_literals
 
-import json
-
 from .common import InfoExtractor
 from ..utils import (
-    mimetype2ext,
+    int_or_none,
+    qualities,
 )
 
 
 class CrooksAndLiarsIE(InfoExtractor):
-    _VALID_URL = r'(?:https?:)?//embed.crooksandliars.com/embed/(?P<id>[A-Za-z0-9]+)(?:$|[?#])'
-
+    _VALID_URL = r'https?://embed\.crooksandliars\.com/(?:embed|v)/(?P<id>[A-Za-z0-9]+)'
     _TESTS = [{
         'url': 'https://embed.crooksandliars.com/embed/8RUoRhRi',
         'info_dict': {
-            'id': 'https://embed.crooksandliars.com/embed/8RUoRhRi',
+            'id': '8RUoRhRi',
+            'ext': 'mp4',
             'title': "Fox & Friends Says Protecting Atheists From Discrimination Is Anti-Christian!",
-            'description': "Fox News, Fox & Friends Weekend, April 4, 2015. Read more... http://crooksandliars.com/2015/04/fox-friends-says-protecting-atheists",
+            'description': 'md5:e1a46ad1650e3a5ec7196d432799127f',
+            'thumbnail': 're:^https?://.*\.jpg',
             'timestamp': 1428207000,
-            'thumbnail': 'https://crooksandliars.com/files/mediaposters/2015/04/31235.jpg?ts=1428207050',
-            'uploader': "Heather",
+            'upload_date': '20150405',
+            'uploader': 'Heather',
+            'duration': 236,
         }
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-
-        manifest = json.loads(self._html_search_regex(r'var manifest = ({.*?})\n', webpage, 'manifest JSON'))
-
-        formats = []
-        for item in manifest['flavors']:
-            if not item['mime'].startswith('video/'): # XXX: or item['exclude']?
-                continue
-            formats.append({
-                'format_id': item['type'],
-                'ext': mimetype2ext(item['mime']),
-                'url': item['url'],
-            })
-
-        # XXX: manifest['url']?
-        return {
-            'url': url,
-            'id': video_id,
-            'uploader': manifest['author'],
-            'title': manifest['title'],
-            'description': manifest['description'],
-            'thumbnail': self._proto_relative_url(manifest['poster']),
-            'duration': manifest['duration'],
-            'timestamp': int(manifest['created']),
-            'formats': formats,
-        }
-
-class CrooksAndLiarsArticleIE(InfoExtractor):
-    _VALID_URL = r'(?:https?:)?//crooksandliars.com/\d+/\d+/(?P<id>[a-z\-]+)(?:/|$)'
-
-    _TESTS = [{
-        'url': 'http://crooksandliars.com/2015/04/fox-friends-says-protecting-atheists',
+    }, {
+        'url': 'http://embed.crooksandliars.com/v/MTE3MjUtMzQ2MzA',
         'only_matching': True,
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-        player_url = self._proto_relative_url(self._html_search_regex(r'<iframe src="(//embed.crooksandliars.com/.*)"', webpage, 'embedded player'))
+
+        webpage = self._download_webpage(
+            'http://embed.crooksandliars.com/embed/%s' % video_id, video_id)
+
+        manifest = self._parse_json(
+            self._search_regex(
+                r'var\s+manifest\s*=\s*({.+?})\n', webpage, 'manifest JSON'),
+            video_id)
+
+        quality = qualities(('webm_low', 'mp4_low', 'webm_high', 'mp4_high'))
+
+        formats = [{
+            'url': item['url'],
+            'format_id': item['type'],
+            'quality': quality(item['type']),
+        } for item in manifest['flavors'] if item['mime'].startswith('video/')]
+        self._sort_formats(formats)
 
         return {
-            '_type': 'url',
-            'url': player_url
+            'url': url,
+            'id': video_id,
+            'title': manifest['title'],
+            'description': manifest.get('description'),
+            'thumbnail': self._proto_relative_url(manifest.get('poster')),
+            'timestamp': int_or_none(manifest.get('created')),
+            'uploader': manifest.get('author'),
+            'duration': int_or_none(manifest.get('duration')),
+            'formats': formats,
         }

From 18153f1b3249701a82c24ec492f5d58929292caf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 11 Apr 2015 20:20:20 +0600
Subject: [PATCH 0916/6219] [generic] Add support for Crooks and Liars embeds

---
 youtube_dl/extractor/generic.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 6c212efac..0dc52ed23 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1275,6 +1275,12 @@ class GenericIE(InfoExtractor):
         if mobj is not None:
             return self.url_result('5min:%s' % mobj.group('id'), 'FiveMin')
 
+        # Look for Crooks and Liars embeds
+        mobj = re.search(
+            r'<(?:iframe[^>]+src|param[^>]+value)=(["\'])(?P<url>(?:https?:)?//embed\.crooksandliars\.com/(?:embed|v)/.+?)\1', webpage)
+        if mobj is not None:
+            return self.url_result(mobj.group('url'))
+
         # Look for NBC Sports VPlayer embeds
         nbc_sports_url = NBCSportsVPlayerIE._extract_url(webpage)
         if nbc_sports_url:

From a4257017ef2ee665d35dd71905db03ca3913c92e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 11 Apr 2015 20:26:42 +0600
Subject: [PATCH 0917/6219] [generic] Add tests for Crooks and Liars embeds

---
 youtube_dl/extractor/generic.py | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 0dc52ed23..7ad555e9f 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -642,6 +642,32 @@ class GenericIE(InfoExtractor):
                 'title': 'pdv_maddow_netcast_m4v-02-27-2015-201624',
             }
         },
+        # Crooks and Liars embed
+        {
+            'url': 'http://crooksandliars.com/2015/04/fox-friends-says-protecting-atheists',
+            'info_dict': {
+                'id': '8RUoRhRi',
+                'ext': 'mp4',
+                'title': "Fox & Friends Says Protecting Atheists From Discrimination Is Anti-Christian!",
+                'description': 'md5:e1a46ad1650e3a5ec7196d432799127f',
+                'timestamp': 1428207000,
+                'upload_date': '20150405',
+                'uploader': 'Heather',
+            },
+        },
+        # Crooks and Liars external embed
+        {
+            'url': 'http://theothermccain.com/2010/02/02/video-proves-that-bill-kristol-has-been-watching-glenn-beck/comment-page-1/',
+            'info_dict': {
+                'id': 'MTE3MjUtMzQ2MzA',
+                'ext': 'mp4',
+                'title': 'md5:5e3662a81a4014d24c250d76d41a08d5',
+                'description': 'md5:9b8e9542d6c3c5de42d6451b7d780cec',
+                'timestamp': 1265032391,
+                'upload_date': '20100201',
+                'uploader': 'Heather',
+            },
+        },
         # NBC Sports vplayer embed
         {
             'url': 'http://www.riderfans.com/forum/showthread.php?121827-Freeman&s=e98fa1ea6dc08e886b1678d35212494a',

From ed5641e2499de9dfbfa78c6684e92e9581402f16 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 11 Apr 2015 20:27:39 +0600
Subject: [PATCH 0918/6219] [crooksandliars] Quotes consistency

---
 youtube_dl/extractor/crooksandliars.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/crooksandliars.py b/youtube_dl/extractor/crooksandliars.py
index 143509004..443eb7691 100644
--- a/youtube_dl/extractor/crooksandliars.py
+++ b/youtube_dl/extractor/crooksandliars.py
@@ -14,7 +14,7 @@ class CrooksAndLiarsIE(InfoExtractor):
         'info_dict': {
             'id': '8RUoRhRi',
             'ext': 'mp4',
-            'title': "Fox & Friends Says Protecting Atheists From Discrimination Is Anti-Christian!",
+            'title': 'Fox & Friends Says Protecting Atheists From Discrimination Is Anti-Christian!',
             'description': 'md5:e1a46ad1650e3a5ec7196d432799127f',
             'thumbnail': 're:^https?://.*\.jpg',
             'timestamp': 1428207000,

From b04b94da5fbe944e2a9d2946ab8b3acb212f9f70 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 12 Apr 2015 03:57:56 +0600
Subject: [PATCH 0919/6219] [options] Fix file based configurations for python
 2 (Closes #5401)

---
 youtube_dl/options.py | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 5720fb424..11603f60d 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -794,21 +794,22 @@ def parseOpts(overrideArguments=None):
         if opts.verbose:
             write_string('[debug] Override config: ' + repr(overrideArguments) + '\n')
     else:
-        command_line_conf = sys.argv[1:]
-        # Workaround for Python 2.x, where argv is a byte list
-        if sys.version_info < (3,):
-            command_line_conf = [
-                a.decode(preferredencoding(), 'replace') for a in command_line_conf]
+        def compat_conf(conf):
+            if sys.version_info < (3,):
+                return [a.decode(preferredencoding(), 'replace') for a in conf]
+            return conf
+
+        command_line_conf = compat_conf(sys.argv[1:])
 
         if '--ignore-config' in command_line_conf:
             system_conf = []
             user_conf = []
         else:
-            system_conf = _readOptions('/etc/youtube-dl.conf')
+            system_conf = compat_conf(_readOptions('/etc/youtube-dl.conf'))
             if '--ignore-config' in system_conf:
                 user_conf = []
             else:
-                user_conf = _readUserConf()
+                user_conf = compat_conf(_readUserConf())
         argv = system_conf + user_conf + command_line_conf
 
         opts, args = parser.parse_args(argv)

From 7d2ba6394c489879578e316e533eeb282942e54b Mon Sep 17 00:00:00 2001
From: snipem <mail@matthias-kuech.de>
Date: Sun, 12 Apr 2015 11:23:01 +0200
Subject: [PATCH 0920/6219] [FootyRoom] Fixed missing http prefix
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

For some reason FootyTube is missing the „http:“ prefix on some
Playwire links for some videos
---
 youtube_dl/extractor/footyroom.py | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/footyroom.py b/youtube_dl/extractor/footyroom.py
index 2b4691ae8..dc784db96 100644
--- a/youtube_dl/extractor/footyroom.py
+++ b/youtube_dl/extractor/footyroom.py
@@ -6,14 +6,25 @@ from .common import InfoExtractor
 
 class FootyRoomIE(InfoExtractor):
     _VALID_URL = r'http://footyroom\.com/(?P<id>[^/]+)'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://footyroom.com/schalke-04-0-2-real-madrid-2015-02/',
+        'md5': 'f38d400d32f19724570040d5ce3a505f',
         'info_dict': {
             'id': 'schalke-04-0-2-real-madrid-2015-02',
             'title': 'Schalke 04 0 – 2 Real Madrid',
         },
         'playlist_count': 3,
-    }
+    },
+    {
+        'url': 'http://footyroom.com/georgia-0-2-germany-2015-03/',
+        'info_dict': {
+            'id': 'georgia-0-2-germany-2015-03',
+            'title': 'Georgia 0 – 2 Germany',
+        },
+        'playlist_count': 1,
+    },
+
+    ]
 
     def _real_extract(self, url):
         playlist_id = self._match_id(url)
@@ -35,6 +46,8 @@ class FootyRoomIE(InfoExtractor):
             playwire_url = self._search_regex(
                 r'data-config="([^"]+)"', payload,
                 'playwire url', default=None)
+            if not playwire_url.startswith("http:"):
+                playwire_url = "http:" + playwire_url
             if playwire_url:
                 entries.append(self.url_result(playwire_url, 'Playwire'))
 

From 9a4d8fae82f10afe8b2d0611f2f054af60dc7acc Mon Sep 17 00:00:00 2001
From: snipem <mail@matthias-kuech.de>
Date: Sun, 12 Apr 2015 11:31:58 +0200
Subject: [PATCH 0921/6219] [FootyTube] Fixed wrong md5 checksum

---
 youtube_dl/extractor/footyroom.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/footyroom.py b/youtube_dl/extractor/footyroom.py
index dc784db96..70dfb0659 100644
--- a/youtube_dl/extractor/footyroom.py
+++ b/youtube_dl/extractor/footyroom.py
@@ -8,7 +8,6 @@ class FootyRoomIE(InfoExtractor):
     _VALID_URL = r'http://footyroom\.com/(?P<id>[^/]+)'
     _TESTS = [{
         'url': 'http://footyroom.com/schalke-04-0-2-real-madrid-2015-02/',
-        'md5': 'f38d400d32f19724570040d5ce3a505f',
         'info_dict': {
             'id': 'schalke-04-0-2-real-madrid-2015-02',
             'title': 'Schalke 04 0 – 2 Real Madrid',

From 504c1cedfeb1562b089fa83ca258966e22acc773 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 12 Apr 2015 17:09:52 +0600
Subject: [PATCH 0922/6219] [footyroom] Improve

---
 youtube_dl/extractor/footyroom.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/footyroom.py b/youtube_dl/extractor/footyroom.py
index 70dfb0659..385a99b13 100644
--- a/youtube_dl/extractor/footyroom.py
+++ b/youtube_dl/extractor/footyroom.py
@@ -45,9 +45,8 @@ class FootyRoomIE(InfoExtractor):
             playwire_url = self._search_regex(
                 r'data-config="([^"]+)"', payload,
                 'playwire url', default=None)
-            if not playwire_url.startswith("http:"):
-                playwire_url = "http:" + playwire_url
             if playwire_url:
-                entries.append(self.url_result(playwire_url, 'Playwire'))
+                entries.append(self.url_result(self._proto_relative_url(
+                    playwire_url, 'http:'), 'Playwire'))
 
         return self.playlist_result(entries, playlist_id, playlist_title)

From ac58e68bc396904b8e7afa21c9d65a55d2a4852b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 12 Apr 2015 17:11:11 +0600
Subject: [PATCH 0923/6219] [footyroom] Remove superfluous whitespace

---
 youtube_dl/extractor/footyroom.py | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/footyroom.py b/youtube_dl/extractor/footyroom.py
index 385a99b13..4c7dbca40 100644
--- a/youtube_dl/extractor/footyroom.py
+++ b/youtube_dl/extractor/footyroom.py
@@ -13,17 +13,14 @@ class FootyRoomIE(InfoExtractor):
             'title': 'Schalke 04 0 – 2 Real Madrid',
         },
         'playlist_count': 3,
-    },
-    {
+    }, {
         'url': 'http://footyroom.com/georgia-0-2-germany-2015-03/',
         'info_dict': {
             'id': 'georgia-0-2-germany-2015-03',
             'title': 'Georgia 0 – 2 Germany',
         },
         'playlist_count': 1,
-    },
-
-    ]
+    }]
 
     def _real_extract(self, url):
         playlist_id = self._match_id(url)

From e91b2d14e3100801269ec9e5e853fd8a21c29443 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 12 Apr 2015 17:17:31 +0600
Subject: [PATCH 0924/6219] Credit @snipem for gamersyde (#5352)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 9c65dc1d4..db3f42b26 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -122,3 +122,4 @@ Joram Schrijver
 Will W.
 Mohammad Teimori Pabandi
 Roman Le Négrate
+Matthias Küch

From c36a95954947fd5d9fb8df2539115a7f9a6b3a59 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 12 Apr 2015 17:36:29 +0600
Subject: [PATCH 0925/6219] [YoutubeDL] Try to download worst audio+video
 served by a single file first (Closes #5408)

---
 youtube_dl/YoutubeDL.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 640b8c99d..a68b24ab4 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -919,6 +919,11 @@ class YoutubeDL(object):
         if format_spec == 'best' or format_spec is None:
             return available_formats[-1]
         elif format_spec == 'worst':
+            audiovideo_formats = [
+                f for f in available_formats
+                if f.get('vcodec') != 'none' and f.get('acodec') != 'none']
+            if audiovideo_formats:
+                return audiovideo_formats[0]
             return available_formats[0]
         elif format_spec == 'bestaudio':
             audio_formats = [

From 830d53bfae7a665b55656dd50c9f35f0d0b0161d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 12 Apr 2015 23:11:47 +0600
Subject: [PATCH 0926/6219] [utils] Add `video_title` for `url_result`

---
 youtube_dl/extractor/common.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 530c449c1..8ed97f8dd 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -492,7 +492,7 @@ class InfoExtractor(object):
 
     # Methods for following #608
     @staticmethod
-    def url_result(url, ie=None, video_id=None):
+    def url_result(url, ie=None, video_id=None, video_title=None):
         """Returns a url that points to a page that should be processed"""
         # TODO: ie should be the class used for getting the info
         video_info = {'_type': 'url',
@@ -500,6 +500,8 @@ class InfoExtractor(object):
                       'ie_key': ie}
         if video_id is not None:
             video_info['id'] = video_id
+        if video_title is not None:
+            video_info['title'] = video_title
         return video_info
 
     @staticmethod

From fb69240ca0934299583bf6c7a855d5c602a4a7e0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 12 Apr 2015 23:19:00 +0600
Subject: [PATCH 0927/6219] [youtube] Extract video titles for channel playlist
 if possible (Closes #4971)

---
 youtube_dl/extractor/youtube.py | 29 ++++++++++++++++++++---------
 1 file changed, 20 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 2774ec30b..791e1fe62 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1370,10 +1370,18 @@ class YoutubeChannelIE(InfoExtractor):
 
     def extract_videos_from_page(self, page):
         ids_in_page = []
-        for mobj in re.finditer(r'href="/watch\?v=([0-9A-Za-z_-]+)&?', page):
-            if mobj.group(1) not in ids_in_page:
-                ids_in_page.append(mobj.group(1))
-        return ids_in_page
+        titles_in_page = []
+        for mobj in re.finditer(r'(?:title="(?P<title>[^"]+)"[^>]+)?href="/watch\?v=(?P<id>[0-9A-Za-z_-]+)&?', page):
+            video_id = mobj.group('id')
+            video_title = unescapeHTML(mobj.group('title'))
+            try:
+                idx = ids_in_page.index(video_id)
+                if video_title and not titles_in_page[idx]:
+                    titles_in_page[idx] = video_title
+            except ValueError:
+                ids_in_page.append(video_id)
+                titles_in_page.append(video_title)
+        return zip(ids_in_page, titles_in_page)
 
     def _real_extract(self, url):
         channel_id = self._match_id(url)
@@ -1390,10 +1398,12 @@ class YoutubeChannelIE(InfoExtractor):
         if autogenerated:
             # The videos are contained in a single page
             # the ajax pages can't be used, they are empty
-            video_ids = self.extract_videos_from_page(channel_page)
+            videos = self.extract_videos_from_page(channel_page)
             entries = [
-                self.url_result(video_id, 'Youtube', video_id=video_id)
-                for video_id in video_ids]
+                self.url_result(
+                    video_id, 'Youtube', video_id=video_id,
+                    video_title=video_title)
+                for video_id, video_title in videos]
             return self.playlist_result(entries, channel_id)
 
         def _entries():
@@ -1401,9 +1411,10 @@ class YoutubeChannelIE(InfoExtractor):
             for pagenum in itertools.count(1):
 
                 ids_in_page = self.extract_videos_from_page(content_html)
-                for video_id in ids_in_page:
+                for video_id, video_title in ids_in_page:
                     yield self.url_result(
-                        video_id, 'Youtube', video_id=video_id)
+                        video_id, 'Youtube', video_id=video_id,
+                        video_title=video_title)
 
                 mobj = re.search(
                     r'data-uix-load-more-href="/?(?P<more>[^"]+)"',

From 7bd930368c48222b0b14211840ac7951fd62f3cf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 13 Apr 2015 00:08:39 +0600
Subject: [PATCH 0928/6219] [youtube] Remove unused variable

---
 youtube_dl/extractor/youtube.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 791e1fe62..dc9e15e98 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1386,7 +1386,6 @@ class YoutubeChannelIE(InfoExtractor):
     def _real_extract(self, url):
         channel_id = self._match_id(url)
 
-        video_ids = []
         url = 'https://www.youtube.com/channel/%s/videos' % channel_id
         channel_page = self._download_webpage(url, channel_id)
         autogenerated = re.search(r'''(?x)

From 51f1244600beb8c3182dc0df756f61cbcfb6c13c Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 13 Apr 2015 19:26:15 +0800
Subject: [PATCH 0929/6219] [vine] flake8

---
 youtube_dl/extractor/vine.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/vine.py b/youtube_dl/extractor/vine.py
index d4f5a991e..65c459fad 100644
--- a/youtube_dl/extractor/vine.py
+++ b/youtube_dl/extractor/vine.py
@@ -1,7 +1,6 @@
 from __future__ import unicode_literals
 
 import re
-import json
 import itertools
 
 from .common import InfoExtractor

From 8f02ad4f12549865a2a4436328075f4b20b906ef Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 13 Apr 2015 20:28:16 +0600
Subject: [PATCH 0930/6219] [youtube] Simplify

---
 youtube_dl/extractor/youtube.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index dc9e15e98..52909b0da 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1397,20 +1397,18 @@ class YoutubeChannelIE(InfoExtractor):
         if autogenerated:
             # The videos are contained in a single page
             # the ajax pages can't be used, they are empty
-            videos = self.extract_videos_from_page(channel_page)
             entries = [
                 self.url_result(
                     video_id, 'Youtube', video_id=video_id,
                     video_title=video_title)
-                for video_id, video_title in videos]
+                for video_id, video_title in self.extract_videos_from_page(channel_page)]
             return self.playlist_result(entries, channel_id)
 
         def _entries():
             more_widget_html = content_html = channel_page
             for pagenum in itertools.count(1):
 
-                ids_in_page = self.extract_videos_from_page(content_html)
-                for video_id, video_title in ids_in_page:
+                for video_id, video_title in self.extract_videos_from_page(content_html):
                     yield self.url_result(
                         video_id, 'Youtube', video_id=video_id,
                         video_title=video_title)

From 37b44fe7c1e462b748171c4af743f30c01c95fe1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 13 Apr 2015 22:50:40 +0200
Subject: [PATCH 0931/6219] [postprocessor/atomicparsley] Don't try to remove
 the temporary and original files if the format is unsupported (fixes #5419)

---
 youtube_dl/postprocessor/atomicparsley.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/postprocessor/atomicparsley.py b/youtube_dl/postprocessor/atomicparsley.py
index 448ccc5f3..a5dfc136a 100644
--- a/youtube_dl/postprocessor/atomicparsley.py
+++ b/youtube_dl/postprocessor/atomicparsley.py
@@ -50,8 +50,13 @@ class AtomicParsleyPP(PostProcessor):
             msg = stderr.decode('utf-8', 'replace').strip()
             raise AtomicParsleyPPError(msg)
 
-        os.remove(encodeFilename(filename))
         os.remove(encodeFilename(temp_thumbnail))
-        os.rename(encodeFilename(temp_filename), encodeFilename(filename))
+        # for formats that don't support thumbnails (like 3gp) AtomicParsley
+        # won't create to the temporary file
+        if b'No changes' in stdout:
+            self._downloader.report_warning('The file format doesn\'t support embedding a thumbnail')
+        else:
+            os.remove(encodeFilename(filename))
+            os.rename(encodeFilename(temp_filename), encodeFilename(filename))
 
         return True, info

From edfcf7abe2f31bb8309ca032e738c90dc3a5722e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 14 Apr 2015 12:45:43 +0800
Subject: [PATCH 0932/6219] [generic] Support another type of Ooyala embedded
 video

---
 youtube_dl/extractor/generic.py | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 7ad555e9f..92a62e866 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -688,6 +688,20 @@ class GenericIE(InfoExtractor):
                 'title': '生物老師男變女 全校挺"做自己"',
                 'thumbnail': 're:^https?://.*\.jpg$',
             }
+        },
+        # Ooyala embed
+        {
+            'url': 'http://www.businessinsider.com/excel-index-match-vlookup-video-how-to-2015-2?IR=T',
+            'info_dict': {
+                'id': '50YnY4czr4ms1vJ7yz3xzq0excz_pUMs',
+                'ext': 'mp4',
+                'description': 'VIDEO: Index/Match versus VLOOKUP.',
+                'title': 'This is what separates the Excel masters from the wannabes',
+            },
+            'params': {
+                # m3u8 downloads
+                'skip_download': True,
+            }
         }
     ]
 
@@ -1092,7 +1106,8 @@ class GenericIE(InfoExtractor):
         # Look for Ooyala videos
         mobj = (re.search(r'player\.ooyala\.com/[^"?]+\?[^"]*?(?:embedCode|ec)=(?P<ec>[^"&]+)', webpage) or
                 re.search(r'OO\.Player\.create\([\'"].*?[\'"],\s*[\'"](?P<ec>.{32})[\'"]', webpage) or
-                re.search(r'SBN\.VideoLinkset\.ooyala\([\'"](?P<ec>.{32})[\'"]\)', webpage))
+                re.search(r'SBN\.VideoLinkset\.ooyala\([\'"](?P<ec>.{32})[\'"]\)', webpage) or
+                re.search(r'data-ooyala-video-id\s*=\s*[\'"](?P<ec>.{32})[\'"]', webpage))
         if mobj is not None:
             return OoyalaIE._build_url_result(mobj.group('ec'))
 

From 01c58f84738e056733717174d1076ec465c62500 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 14 Apr 2015 13:10:10 +0800
Subject: [PATCH 0933/6219] [generic] Fix test generic_51

The website replaced the original video with a new one
---
 youtube_dl/extractor/generic.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 92a62e866..eaf9c769a 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -681,11 +681,11 @@ class GenericIE(InfoExtractor):
         # UDN embed
         {
             'url': 'http://www.udn.com/news/story/7314/822787',
-            'md5': 'de06b4c90b042c128395a88f0384817e',
+            'md5': 'fd2060e988c326991037b9aff9df21a6',
             'info_dict': {
-                'id': '300040',
+                'id': '300346',
                 'ext': 'mp4',
-                'title': '生物老師男變女 全校挺"做自己"',
+                'title': '中一中男師變性 全校師生力挺',
                 'thumbnail': 're:^https?://.*\.jpg$',
             }
         },

From 8da1bb04186d4147f89923abc09a9db0fa2a4fec Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 14 Apr 2015 15:27:56 +0800
Subject: [PATCH 0934/6219] [miomio] Enhance error checking and replace dead
 test case

---
 youtube_dl/extractor/miomio.py | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/miomio.py b/youtube_dl/extractor/miomio.py
index cc3f27194..d41195a96 100644
--- a/youtube_dl/extractor/miomio.py
+++ b/youtube_dl/extractor/miomio.py
@@ -7,6 +7,7 @@ from .common import InfoExtractor
 from ..utils import (
     xpath_text,
     int_or_none,
+    ExtractorError,
 )
 
 
@@ -14,13 +15,14 @@ class MioMioIE(InfoExtractor):
     IE_NAME = 'miomio.tv'
     _VALID_URL = r'https?://(?:www\.)?miomio\.tv/watch/cc(?P<id>[0-9]+)'
     _TESTS = [{
-        'url': 'http://www.miomio.tv/watch/cc179734/',
-        'md5': '48de02137d0739c15b440a224ad364b9',
+        # "type=video" in flashvars
+        'url': 'http://www.miomio.tv/watch/cc88912/',
+        'md5': '317a5f7f6b544ce8419b784ca8edae65',
         'info_dict': {
-            'id': '179734',
+            'id': '88912',
             'ext': 'flv',
-            'title': '手绘动漫鬼泣但丁全程画法',
-            'duration': 354,
+            'title': '【SKY】字幕 铠武昭和VS平成 假面骑士大战FEAT战队 魔星字幕组 字幕',
+            'duration': 5923,
         },
     }, {
         'url': 'http://www.miomio.tv/watch/cc184024/',
@@ -42,7 +44,7 @@ class MioMioIE(InfoExtractor):
             r'src="(/mioplayer/[^"]+)"', webpage, 'ref_path')
 
         xml_config = self._search_regex(
-            r'flashvars="type=sina&amp;(.+?)&amp;',
+            r'flashvars="type=(?:sina|video)&amp;(.+?)&amp;',
             webpage, 'xml config')
 
         # skipping the following page causes lags and eventually connection drop-outs
@@ -59,6 +61,9 @@ class MioMioIE(InfoExtractor):
             'Referer': 'http://www.miomio.tv%s' % mioplayer_path,
         }
 
+        if not int_or_none(xpath_text(vid_config, 'timelength')):
+            raise ExtractorError('Unable to load videos!', expected=True)
+
         entries = []
         for f in vid_config.findall('./durl'):
             segment_url = xpath_text(f, 'url', 'video url')

From 476e1095fac398b648f953a826ed3b191ce8d5d7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 14 Apr 2015 17:48:41 +0600
Subject: [PATCH 0935/6219] [brightcove] Improve brightcove experience regex
 (Closes #5421)

---
 youtube_dl/extractor/brightcove.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 0733bece7..b37857b2e 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -183,7 +183,7 @@ class BrightcoveIE(InfoExtractor):
             (?:
                 [^>]+?class=[\'"][^>]*?BrightcoveExperience.*?[\'"] |
                 [^>]*?>\s*<param\s+name="movie"\s+value="https?://[^/]*brightcove\.com/
-            ).+?</object>''',
+            ).+?>\s*</object>''',
             webpage)
         return [cls._build_brighcove_url(m) for m in matches]
 

From 94c1255782414cdcda565717e31c91734ec00990 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 14 Apr 2015 17:50:53 +0600
Subject: [PATCH 0936/6219] [brightcove] Handle non well-formed XMLs (#5421)

---
 youtube_dl/extractor/brightcove.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index b37857b2e..117cb00e6 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -117,7 +117,10 @@ class BrightcoveIE(InfoExtractor):
         object_str = re.sub(r'(<object[^>]*)(xmlns=".*?")', r'\1', object_str)
         object_str = fix_xml_ampersands(object_str)
 
-        object_doc = xml.etree.ElementTree.fromstring(object_str.encode('utf-8'))
+        try:
+            object_doc = xml.etree.ElementTree.fromstring(object_str.encode('utf-8'))
+        except xml.etree.ElementTree.ParseError:
+            return
 
         fv_el = find_xpath_attr(object_doc, './param', 'name', 'flashVars')
         if fv_el is not None:
@@ -185,7 +188,7 @@ class BrightcoveIE(InfoExtractor):
                 [^>]*?>\s*<param\s+name="movie"\s+value="https?://[^/]*brightcove\.com/
             ).+?>\s*</object>''',
             webpage)
-        return [cls._build_brighcove_url(m) for m in matches]
+        return filter(None, [cls._build_brighcove_url(m) for m in matches])
 
     def _real_extract(self, url):
         url, smuggled_data = unsmuggle_url(url, {})

From ae849ca1701dc5bdf25e648a57a1e6ec64f2f968 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 15 Apr 2015 17:45:28 +0800
Subject: [PATCH 0937/6219] [tumblr] Dismiss warnings for optional fields
 (fixes #5202)

---
 youtube_dl/extractor/tumblr.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/tumblr.py b/youtube_dl/extractor/tumblr.py
index 2a1ae5a71..828c808a6 100644
--- a/youtube_dl/extractor/tumblr.py
+++ b/youtube_dl/extractor/tumblr.py
@@ -56,6 +56,6 @@ class TumblrIE(InfoExtractor):
             'url': video_url,
             'ext': 'mp4',
             'title': video_title,
-            'description': self._og_search_description(webpage),
-            'thumbnail': self._og_search_thumbnail(webpage),
+            'description': self._og_search_description(webpage, default=None),
+            'thumbnail': self._og_search_thumbnail(webpage, default=None),
         }

From 9c5335a0270042b7e84f02e550676447ccf603b1 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 15 Apr 2015 19:56:21 +0800
Subject: [PATCH 0938/6219] [teamcoco] Fix "preload" data extraction (fixes
 #5179)

---
 youtube_dl/extractor/teamcoco.py | 31 ++++++++++++++++++++++++++++---
 1 file changed, 28 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/teamcoco.py b/youtube_dl/extractor/teamcoco.py
index 1caf08cb7..2381676b4 100644
--- a/youtube_dl/extractor/teamcoco.py
+++ b/youtube_dl/extractor/teamcoco.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
 import base64
@@ -35,6 +36,17 @@ class TeamcocoIE(InfoExtractor):
                 'duration': 288,
                 'age_limit': 0,
             }
+        }, {
+            'url': 'http://teamcoco.com/video/timothy-olyphant-drinking-whiskey',
+            'info_dict': {
+                'id': '88748',
+                'ext': 'mp4',
+                'title': 'Timothy Olyphant Raises A Toast To “Justified”',
+                'description': 'md5:15501f23f020e793aeca761205e42c24',
+            },
+            'params': {
+                'skip_download': True,  # m3u8 downloads
+            }
         }
     ]
     _VIDEO_ID_REGEXES = (
@@ -54,10 +66,23 @@ class TeamcocoIE(InfoExtractor):
             video_id = self._html_search_regex(
                 self._VIDEO_ID_REGEXES, webpage, 'video id')
 
+        preload = None
         preloads = re.findall(r'"preload":\s*"([^"]+)"', webpage)
-        if not preloads:
-            raise ExtractorError('Preload information could not be extracted')
-        preload = max([(len(p), p) for p in preloads])[1]
+        if preloads:
+            preload = max([(len(p), p) for p in preloads])[1]
+
+        if not preload:
+            preload = ''.join(re.findall(r'this\.push\("([^"]+)"\);', webpage))
+
+        if not preload:
+            preload = self._html_search_regex([
+                r'player,\[?"([^"]+)"\]?', r'player.init\(\[?"([^"]+)"\]?\)'
+            ], webpage.replace('","', ''), 'preload data', default=None)
+
+        if not preload:
+            raise ExtractorError(
+                'Preload information could not be extracted', expected=True)
+
         data = self._parse_json(
             base64.b64decode(preload.encode('ascii')).decode('utf-8'), video_id)
 

From 4d1cdb5bfebcc93d73ffb710ec7af0752d698dca Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 15 Apr 2015 20:58:48 +0600
Subject: [PATCH 0939/6219] [spike] Extend _VALID_URL (Closes #5420)

---
 youtube_dl/extractor/spike.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/spike.py b/youtube_dl/extractor/spike.py
index e529bb55c..182f286df 100644
--- a/youtube_dl/extractor/spike.py
+++ b/youtube_dl/extractor/spike.py
@@ -5,7 +5,7 @@ from .mtv import MTVServicesInfoExtractor
 
 class SpikeIE(MTVServicesInfoExtractor):
     _VALID_URL = r'''(?x)https?://
-        (?:www\.spike\.com/(?:video-clips|(?:full-)?episodes)/.+|
+        (?:www\.spike\.com/(?:video-(?:clips|playlists)|(?:full-)?episodes)/.+|
          m\.spike\.com/videos/video\.rbml\?id=(?P<id>[^&]+))
         '''
     _TEST = {

From 0dfe9bc9d286ceb0fd35b0fc8857ba1a86cecdd3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 15 Apr 2015 21:02:34 +0600
Subject: [PATCH 0940/6219] [mtv] Capture and output error message (#5420)

---
 youtube_dl/extractor/mtv.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index c11de1cb6..4430b3416 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -118,6 +118,14 @@ class MTVServicesInfoExtractor(InfoExtractor):
         mediagen_doc = self._download_xml(mediagen_url, video_id,
                                           'Downloading video urls')
 
+        item = mediagen_doc.find('./video/item')
+        if item is not None and item.get('type') == 'text':
+            message = '%s returned error: ' % self.IE_NAME
+            if item.get('code') is not None:
+                message += '%s - ' % item.get('code')
+            message += item.text
+            raise ExtractorError(message, expected=True)
+
         description_node = itemdoc.find('description')
         if description_node is not None:
             description = description_node.text.strip()

From 2dcc114f84ebd6ea18b60a57070bcb81861d8f84 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 15 Apr 2015 22:10:08 +0600
Subject: [PATCH 0941/6219] [generic] Add support for playwire embeds (Closes
 #5430)

---
 youtube_dl/extractor/generic.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index eaf9c769a..ab3538694 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1310,6 +1310,12 @@ class GenericIE(InfoExtractor):
         if mobj is not None:
             return self.url_result(mobj.group('url'), 'Pladform')
 
+        # Look for Playwire embeds
+        mobj = re.search(
+            r'<script[^>]+data-config=(["\'])(?P<url>(?:https?:)?//config\.playwire\.com/.+?)\1', webpage)
+        if mobj is not None:
+            return self.url_result(mobj.group('url'))
+
         # Look for 5min embeds
         mobj = re.search(
             r'<meta[^>]+property="og:video"[^>]+content="https?://embed\.5min\.com/(?P<id>[0-9]+)/?', webpage)

From c798f15b989bc8c3578c5b0baf75f4fb4760ba81 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 15 Apr 2015 22:13:01 +0600
Subject: [PATCH 0942/6219] [generic] Add test for playwire embed (#5430)

---
 youtube_dl/extractor/generic.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index ab3538694..7f2faa935 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -622,6 +622,17 @@ class GenericIE(InfoExtractor):
                 'age_limit': 0,
             },
         },
+        # Playwire embed
+        {
+            'url': 'http://www.cinemablend.com/new/First-Joe-Dirt-2-Trailer-Teaser-Stupid-Greatness-70874.html',
+            'info_dict': {
+                'id': '3519514',
+                'ext': 'mp4',
+                'title': 'Joe Dirt 2 Beautiful Loser Teaser Trailer',
+                'thumbnail': 're:^https?://.*\.png$',
+                'duration': 45.115,
+            },
+        },
         # 5min embed
         {
             'url': 'http://techcrunch.com/video/facebook-creates-on-this-day-crunch-report/518726732/',

From 9fc03aa87c74cb199f9e1e146ac5cb71b166d3e7 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 16 Apr 2015 00:27:39 +0800
Subject: [PATCH 0943/6219] [brightcove] Always return lists from
 _extract_brightcove_urls

In Python 3, filter() returns an iterable object, which is equivalently
to True even for an empty result set. It causes false positive playlists
in generic extraction logic.
---
 youtube_dl/extractor/brightcove.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 117cb00e6..4f60d5366 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -188,7 +188,7 @@ class BrightcoveIE(InfoExtractor):
                 [^>]*?>\s*<param\s+name="movie"\s+value="https?://[^/]*brightcove\.com/
             ).+?>\s*</object>''',
             webpage)
-        return filter(None, [cls._build_brighcove_url(m) for m in matches])
+        return list(filter(None, [cls._build_brighcove_url(m) for m in matches]))
 
     def _real_extract(self, url):
         url, smuggled_data = unsmuggle_url(url, {})

From afe4a8c7699e70ecaa7f273b55a038b4d8a1050f Mon Sep 17 00:00:00 2001
From: FireDart <firedartonline@gmail.com>
Date: Wed, 15 Apr 2015 22:17:45 -0400
Subject: [PATCH 0944/6219] [gfycat] Add new extractor

---
 youtube_dl/extractor/__init__.py |   1 +
 youtube_dl/extractor/gfycat.py   | 107 +++++++++++++++++++++++++++++++
 2 files changed, 108 insertions(+)
 create mode 100644 youtube_dl/extractor/gfycat.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index bbf3be41d..d32f1cbd2 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -185,6 +185,7 @@ from .gametrailers import GametrailersIE
 from .gazeta import GazetaIE
 from .gdcvault import GDCVaultIE
 from .generic import GenericIE
+from .gfycat import GfycatIE
 from .giantbomb import GiantBombIE
 from .giga import GigaIE
 from .glide import GlideIE
diff --git a/youtube_dl/extractor/gfycat.py b/youtube_dl/extractor/gfycat.py
new file mode 100644
index 000000000..d103693cd
--- /dev/null
+++ b/youtube_dl/extractor/gfycat.py
@@ -0,0 +1,107 @@
+# coding: utf-8
+
+from __future__ import unicode_literals
+
+import datetime
+
+from .common import InfoExtractor
+
+class GfycatIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?gfycat\.com/(?P<id>[^/?#]+)'
+    _TESTS = [{
+        'url': 'http://gfycat.com/DeadlyDecisiveGermanpinscher',
+        'info_dict': {
+            'id':          'DeadlyDecisiveGermanpinscher',
+            'title':       'Ghost in the Shell',
+            'ext':         'mp4',
+            'upload_date': '20140913'
+        }
+    },{
+        'url': 'http://gfycat.com/pleasinghilariouskusimanse',
+        'info_dict': {
+            'id':          'pleasinghilariouskusimanse',
+            'title':       'PleasingHilariousKusimanse',
+            'ext':         'webm',
+            'upload_date': '20150412'
+        }
+    },{
+        'url': 'http://gfycat.com/requiredunkemptbuzzard',
+        'info_dict': {
+            'id':          'requiredunkemptbuzzard',
+            'title':       'Headshot!',
+            'ext':         'gif',
+            'upload_date': '20150130'
+        }
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        json     = self._download_json("http://gfycat.com/cajax/get/" + video_id, video_id, 'Downloading video info')['gfyItem']
+        
+        # Title
+        # Use user title first, else fallback to url formated name
+        if json['title']:
+            video_title = json['title']
+        else:
+            video_title = json['gfyName']
+        
+        # Formats
+        # Pref: mp4, webm, gif
+        formats = [{
+            'format_id':  'mp4',
+            'ext':        'mp4',
+            'url':        json['mp4Url'],
+            'width':      json['width'],
+            'height':     json['height'],
+            'fps':        json['frameRate'],
+            'filesize':   json['mp4Size'],
+            'preference': '-1'
+        }, {
+            'format_id': 'webm',
+            'ext':       'webm',
+            'url':        json['webmUrl'],
+            'width':      json['width'],
+            'height':     json['height'],
+            'fps':        json['frameRate'],
+            'filesize':   json['webmSize'],
+            'preference': 0
+        }, {
+            'format_id':  'gif',
+            'ext':        'gif',
+            'url':        json['gifUrl'],
+            'width':      json['width'],
+            'height':     json['height'],
+            'fps':        json['frameRate'],
+            'filesize':   json['gifSize'],
+            'preference': 1
+        }]
+        
+        self._sort_formats(formats)
+        
+        # Date
+        date = datetime.datetime.fromtimestamp(
+            int(json['createDate'])
+        ).strftime('%Y%m%d')
+        
+        # Length
+        duration = json['numFrames'] / json['frameRate']
+        
+        # Age limit
+        # 1 = nsfw / 0 = sfw
+        if json['nsfw'] == 1:
+            age_limit = 18
+        else:
+            age_limit = 0
+        
+        return {
+            'id':          video_id,
+            'title':       video_title,
+            'formats':     formats,
+            'creator':     json['userName'],
+            'description': json['description'],
+            'upload_date': date,
+            'categories':  json['tags'],
+            'age_limit':   age_limit,
+            'duration':    duration,
+            'view_count':  json['views']
+        }

From d0eb724e22dc2e48f206dac45f9db9c17dcb26e1 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 16 Apr 2015 17:04:53 +0800
Subject: [PATCH 0945/6219] [UDNEmbed] Enhance error checking and extend
 _VALID_URL

---
 youtube_dl/extractor/udn.py | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/udn.py b/youtube_dl/extractor/udn.py
index bba25bb58..c08428acf 100644
--- a/youtube_dl/extractor/udn.py
+++ b/youtube_dl/extractor/udn.py
@@ -3,12 +3,15 @@ from __future__ import unicode_literals
 
 import json
 from .common import InfoExtractor
-from ..utils import js_to_json
+from ..utils import (
+    js_to_json,
+    ExtractorError,
+)
 from ..compat import compat_urlparse
 
 
 class UDNEmbedIE(InfoExtractor):
-    _VALID_URL = r'(?:https?:)?//video\.udn\.com/embed/news/(?P<id>\d+)'
+    _VALID_URL = r'https?://video\.udn\.com/(?:embed|play)/news/(?P<id>\d+)'
     _TESTS = [{
         'url': 'http://video.udn.com/embed/news/300040',
         'md5': 'de06b4c90b042c128395a88f0384817e',
@@ -19,7 +22,11 @@ class UDNEmbedIE(InfoExtractor):
             'thumbnail': 're:^https?://.*\.jpg$',
         }
     }, {
-        'url': '//video.udn.com/embed/news/300040',
+        'url': 'https://video.udn.com/embed/news/300040',
+        'only_matching': True,
+    }, {
+        # From https://video.udn.com/news/303776
+        'url': 'https://video.udn.com/play/news/303776',
         'only_matching': True,
     }]
 
@@ -47,7 +54,10 @@ class UDNEmbedIE(InfoExtractor):
                 'retrieve url for %s video' % video_type),
             'format_id': video_type,
             'preference': 0 if video_type == 'mp4' else -1,
-        } for video_type, api_url in video_urls.items()]
+        } for video_type, api_url in video_urls.items() if api_url]
+
+        if not formats:
+            raise ExtractorError('No videos found', expected=True)
 
         self._sort_formats(formats)
 

From d6fd958c5f6847f39f4ed653e82832f2f44657ba Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 16 Apr 2015 17:16:11 +0800
Subject: [PATCH 0946/6219] [generic] Extract videos from SMIL manifests
 (closes #5145 and fixes #5135)

---
 youtube_dl/extractor/generic.py | 37 ++++++++++++++++++++++++++-------
 1 file changed, 30 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 7f2faa935..8c859f068 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -713,6 +713,20 @@ class GenericIE(InfoExtractor):
                 # m3u8 downloads
                 'skip_download': True,
             }
+        },
+        # Contains a SMIL manifest
+        {
+            'url': 'http://www.telewebion.com/fa/1263668/%D9%82%D8%B1%D8%B9%D9%87%E2%80%8C%DA%A9%D8%B4%DB%8C-%D9%84%DB%8C%DA%AF-%D9%82%D9%87%D8%B1%D9%85%D8%A7%D9%86%D8%A7%D9%86-%D8%A7%D8%B1%D9%88%D9%BE%D8%A7/%2B-%D9%81%D9%88%D8%AA%D8%A8%D8%A7%D9%84.html',
+            'info_dict': {
+                'id': 'file',
+                'ext': 'flv',
+                'title': '+ Football: Lottery Champions League Europe',
+                'uploader': 'www.telewebion.com',
+            },
+            'params': {
+                # rtmpe downloads
+                'skip_download': True,
+            }
         }
     ]
 
@@ -1440,13 +1454,22 @@ class GenericIE(InfoExtractor):
             # here's a fun little line of code for you:
             video_id = os.path.splitext(video_id)[0]
 
-            entries.append({
-                'id': video_id,
-                'url': video_url,
-                'uploader': video_uploader,
-                'title': video_title,
-                'age_limit': age_limit,
-            })
+            if determine_ext(video_url) == 'smil':
+                entries.append({
+                    'id': video_id,
+                    'formats': self._extract_smil_formats(video_url, video_id),
+                    'uploader': video_uploader,
+                    'title': video_title,
+                    'age_limit': age_limit,
+                })
+            else:
+                entries.append({
+                    'id': video_id,
+                    'url': video_url,
+                    'uploader': video_uploader,
+                    'title': video_title,
+                    'age_limit': age_limit,
+                })
 
         if len(entries) == 1:
             return entries[0]

From 5d98908b26a4bf593abd6340d34d983f8d0980df Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 17 Mar 2015 19:03:29 +0800
Subject: [PATCH 0947/6219] [QQMusic] Add new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/qqmusic.py  | 56 ++++++++++++++++++++++++++++++++
 2 files changed, 57 insertions(+)
 create mode 100644 youtube_dl/extractor/qqmusic.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index bbf3be41d..63bcf666f 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -396,6 +396,7 @@ from .promptfile import PromptFileIE
 from .prosiebensat1 import ProSiebenSat1IE
 from .puls4 import Puls4IE
 from .pyvideo import PyvideoIE
+from .qqmusic import QQMusicIE
 from .quickvid import QuickVidIE
 from .r7 import R7IE
 from .radiode import RadioDeIE
diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
new file mode 100644
index 000000000..3dc637392
--- /dev/null
+++ b/youtube_dl/extractor/qqmusic.py
@@ -0,0 +1,56 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import strip_jsonp
+
+# guid is a random number generated in javascript, but seems a fixed number
+# also works
+guid = '1'
+
+
+class QQMusicIE(InfoExtractor):
+    _VALID_URL = r'http://y.qq.com/#type=song&mid=(?P<id>[0-9A-Za-z]+)'
+    _TESTS = [{
+        'url': 'http://y.qq.com/#type=song&mid=004295Et37taLD',
+        'md5': 'bed90b6db2a7a7a7e11bc585f471f63a',
+        'info_dict': {
+            'id': '004295Et37taLD',
+            'ext': 'm4a',
+            'title': '可惜没如果',
+            'upload_date': '20141227',
+            'creator': '林俊杰',
+        }
+    }]
+
+    def _real_extract(self, url):
+        mid = self._match_id(url)
+
+        detail_info_page = self._download_webpage(
+            'http://s.plcloud.music.qq.com/fcgi-bin/fcg_yqq_song_detail_info.fcg?songmid=%s&play=0' % mid,
+            mid, note='Download sont detail info',
+            errnote='Unable to get song detail info')
+
+        song_name = self._html_search_regex(
+            r"songname:\s*'([^']+)'", detail_info_page, 'song name')
+
+        publish_time = self._html_search_regex(
+            r'发行时间：(\d{4}-\d{2}-\d{2})', detail_info_page,
+            'publish time').replace('-', '')
+
+        singer = self._html_search_regex(
+            r"singer:\s*'([^']+)", detail_info_page, 'singer')
+
+        vkey = self._download_json(
+            'http://base.music.qq.com/fcgi-bin/fcg_musicexpress.fcg?json=3&guid=%s' % guid,
+            mid, note='Retrieve vkey', errnote='Unable to get vkey',
+            transform_source=strip_jsonp)['key']
+        song_url = 'http://cc.stream.qqmusic.qq.com/C200%s.m4a?vkey=%s&guid=%s&fromtag=0' % (mid, vkey, guid)
+
+        return {
+            'id': mid,
+            'url': song_url,
+            'title': song_name,
+            'upload_date': publish_time,
+            'creator': singer,
+        }

From a2043572aa99c7517c84c31fe8a7e2051d27bf09 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 18 Mar 2015 13:56:02 +0800
Subject: [PATCH 0948/6219] [QQMusic] Implement the guid algorithm

---
 youtube_dl/extractor/qqmusic.py | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index 3dc637392..93440b954 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -1,13 +1,12 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import random
+import time
+
 from .common import InfoExtractor
 from ..utils import strip_jsonp
 
-# guid is a random number generated in javascript, but seems a fixed number
-# also works
-guid = '1'
-
 
 class QQMusicIE(InfoExtractor):
     _VALID_URL = r'http://y.qq.com/#type=song&mid=(?P<id>[0-9A-Za-z]+)'
@@ -23,6 +22,13 @@ class QQMusicIE(InfoExtractor):
         }
     }]
 
+    # Reference: m_r_GetRUin() in top_player.js
+    # http://imgcache.gtimg.cn/music/portal_v3/y/top_player.js
+    @staticmethod
+    def m_r_get_ruin():
+        curMs = int(time.time() * 1000) % 1000
+        return int(round(random.random() * 2147483647) * curMs % 1E10)
+
     def _real_extract(self, url):
         mid = self._match_id(url)
 
@@ -41,6 +47,8 @@ class QQMusicIE(InfoExtractor):
         singer = self._html_search_regex(
             r"singer:\s*'([^']+)", detail_info_page, 'singer')
 
+        guid = self.m_r_get_ruin()
+
         vkey = self._download_json(
             'http://base.music.qq.com/fcgi-bin/fcg_musicexpress.fcg?json=3&guid=%s' % guid,
             mid, note='Retrieve vkey', errnote='Unable to get vkey',

From 8afff9f849e43c9c4194e302c06b5bec38edd83f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 18 Mar 2015 14:59:33 +0800
Subject: [PATCH 0949/6219] [QQMusic] Add singer info extractor

---
 youtube_dl/extractor/__init__.py |  5 ++-
 youtube_dl/extractor/qqmusic.py  | 53 +++++++++++++++++++++++++++++++-
 2 files changed, 56 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 63bcf666f..65dbcb6cf 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -396,7 +396,10 @@ from .promptfile import PromptFileIE
 from .prosiebensat1 import ProSiebenSat1IE
 from .puls4 import Puls4IE
 from .pyvideo import PyvideoIE
-from .qqmusic import QQMusicIE
+from .qqmusic import (
+    QQMusicIE,
+    QQMusicSingerIE
+)
 from .quickvid import QuickVidIE
 from .r7 import R7IE
 from .radiode import RadioDeIE
diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index 93440b954..1c22943a5 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -3,9 +3,11 @@ from __future__ import unicode_literals
 
 import random
 import time
+import re
 
 from .common import InfoExtractor
 from ..utils import strip_jsonp
+from ..compat import compat_urllib_request
 
 
 class QQMusicIE(InfoExtractor):
@@ -34,7 +36,7 @@ class QQMusicIE(InfoExtractor):
 
         detail_info_page = self._download_webpage(
             'http://s.plcloud.music.qq.com/fcgi-bin/fcg_yqq_song_detail_info.fcg?songmid=%s&play=0' % mid,
-            mid, note='Download sont detail info',
+            mid, note='Download song detail info',
             errnote='Unable to get song detail info')
 
         song_name = self._html_search_regex(
@@ -62,3 +64,52 @@ class QQMusicIE(InfoExtractor):
             'upload_date': publish_time,
             'creator': singer,
         }
+
+
+class QQMusicSingerIE(InfoExtractor):
+    _VALID_URL = r'http://y.qq.com/#type=singer&mid=(?P<id>[0-9A-Za-z]+)'
+    _TEST = {
+        'url': 'http://y.qq.com/#type=singer&mid=001BLpXF2DyJe2',
+        'info_dict': {
+            'id': '001BLpXF2DyJe2',
+            'title': '林俊杰',
+            'description': 'md5:2a222d89ba4455a3af19940c0481bb78',
+        },
+        'playlist_count': 12,
+    }
+
+    def _real_extract(self, url):
+        mid = self._match_id(url)
+
+        singer_page = self._download_webpage(
+            'http://y.qq.com/y/static/singer/%s/%s/%s.html' % (mid[-2], mid[-1], mid),
+            'Download singer page')
+
+        entries = []
+
+        for item in re.findall(r'<span class="data">([^<>]+)</span>', singer_page):
+            song_mid = item.split('|')[-5]
+            entries.append(self.url_result(
+                'http://y.qq.com/#type=song&mid=' + song_mid, 'QQMusic', song_mid))
+
+        singer_name = self._html_search_regex(
+            r"singername\s*:\s*'([^']+)'", singer_page, 'singer name',
+            default=None)
+
+        singer_id = self._html_search_regex(
+            r"singerid\s*:\s*'([0-9]+)'", singer_page, 'singer id',
+            default=None)
+
+        singer_desc = None
+
+        if singer_id:
+            req = compat_urllib_request.Request(
+                'http://s.plcloud.music.qq.com/fcgi-bin/fcg_get_singer_desc.fcg?utf8=1&outCharset=utf-8&format=xml&singerid=%s' % singer_id)
+            req.add_header(
+                'Referer', 'http://s.plcloud.music.qq.com/xhr_proxy_utf8.html')
+            singer_desc_page = self._download_xml(
+                req, 'Donwload singer description XML')
+
+            singer_desc = singer_desc_page.find('./data/info/desc').text
+
+        return self.playlist_result(entries, mid, singer_name, singer_desc)

From 5edea45fab73874c269655b4cf08da0bbc5ea479 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 19 Mar 2015 01:47:07 +0800
Subject: [PATCH 0950/6219] [QQMusic] Add album info extractor

---
 youtube_dl/extractor/__init__.py |  3 +-
 youtube_dl/extractor/qqmusic.py  | 72 +++++++++++++++++++++++++++-----
 2 files changed, 63 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 65dbcb6cf..36860f72c 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -398,7 +398,8 @@ from .puls4 import Puls4IE
 from .pyvideo import PyvideoIE
 from .qqmusic import (
     QQMusicIE,
-    QQMusicSingerIE
+    QQMusicSingerIE,
+    QQMusicAlbumIE,
 )
 from .quickvid import QuickVidIE
 from .r7 import R7IE
diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index 1c22943a5..d0ea4a769 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -6,7 +6,10 @@ import time
 import re
 
 from .common import InfoExtractor
-from ..utils import strip_jsonp
+from ..utils import (
+    strip_jsonp,
+    unescapeHTML,
+)
 from ..compat import compat_urllib_request
 
 
@@ -66,7 +69,28 @@ class QQMusicIE(InfoExtractor):
         }
 
 
-class QQMusicSingerIE(InfoExtractor):
+class QQPlaylistBaseIE(InfoExtractor):
+    @staticmethod
+    def qq_static_url(category, mid):
+        return 'http://y.qq.com/y/static/%s/%s/%s/%s.html' % (category, mid[-2], mid[-1], mid)
+
+    @staticmethod
+    def qq_song_url(mid):
+        return 'http://y.qq.com/#type=song&mid=%s' % mid
+
+    @classmethod
+    def get_entries_from_page(cls, page):
+        entries = []
+
+        for item in re.findall(r'class="data"[^<>]*>([^<>]+)</', page):
+            song_mid = unescapeHTML(item).split('|')[-5]
+            entries.append(cls.url_result(
+                cls.qq_song_url(song_mid), 'QQMusic', song_mid))
+
+        return entries
+
+
+class QQMusicSingerIE(QQPlaylistBaseIE):
     _VALID_URL = r'http://y.qq.com/#type=singer&mid=(?P<id>[0-9A-Za-z]+)'
     _TEST = {
         'url': 'http://y.qq.com/#type=singer&mid=001BLpXF2DyJe2',
@@ -82,15 +106,9 @@ class QQMusicSingerIE(InfoExtractor):
         mid = self._match_id(url)
 
         singer_page = self._download_webpage(
-            'http://y.qq.com/y/static/singer/%s/%s/%s.html' % (mid[-2], mid[-1], mid),
-            'Download singer page')
+            self.qq_static_url('singer', mid), mid, 'Download singer page')
 
-        entries = []
-
-        for item in re.findall(r'<span class="data">([^<>]+)</span>', singer_page):
-            song_mid = item.split('|')[-5]
-            entries.append(self.url_result(
-                'http://y.qq.com/#type=song&mid=' + song_mid, 'QQMusic', song_mid))
+        entries = self.get_entries_from_page(singer_page)
 
         singer_name = self._html_search_regex(
             r"singername\s*:\s*'([^']+)'", singer_page, 'singer name',
@@ -108,8 +126,40 @@ class QQMusicSingerIE(InfoExtractor):
             req.add_header(
                 'Referer', 'http://s.plcloud.music.qq.com/xhr_proxy_utf8.html')
             singer_desc_page = self._download_xml(
-                req, 'Donwload singer description XML')
+                req, mid, 'Donwload singer description XML')
 
             singer_desc = singer_desc_page.find('./data/info/desc').text
 
         return self.playlist_result(entries, mid, singer_name, singer_desc)
+
+
+class QQMusicAlbumIE(QQPlaylistBaseIE):
+    _VALID_URL = r'http://y.qq.com/#type=album&mid=(?P<id>[0-9A-Za-z]+)'
+
+    _TEST = {
+        'url': 'http://y.qq.com/#type=album&mid=000gXCTb2AhRR1&play=0',
+        'info_dict': {
+            'id': '000gXCTb2AhRR1',
+            'title': '我们都是这样长大的',
+            'description': 'md5:d216c55a2d4b3537fe4415b8767d74d6',
+        },
+        'playlist_count': 4,
+    }
+
+    def _real_extract(self, url):
+        mid = self._match_id(url)
+
+        album_page = self._download_webpage(
+            self.qq_static_url('album', mid), mid, 'Download album page')
+
+        entries = self.get_entries_from_page(album_page)
+
+        album_name = self._html_search_regex(
+            r"albumname\s*:\s*'([^']+)',", album_page, 'album name',
+            default=None)
+
+        album_detail = self._html_search_regex(
+            r'<div class="album_detail close_detail">\s*<p>((?:[^<>]+(?:<br />)?)+)</p>',
+            album_page, 'album details', default=None)
+
+        return self.playlist_result(entries, mid, album_name, album_detail)

From a685ae511a7f2148afb765d0a30bca42a75cf861 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 21 Mar 2015 11:38:53 +0800
Subject: [PATCH 0951/6219] [QQMusic] Song extractor: Add lyrics as description

Note: Test fails on python 3 due to encoding issues
---
 youtube_dl/extractor/qqmusic.py | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index d0ea4a769..e8aacbc3d 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -24,6 +24,7 @@ class QQMusicIE(InfoExtractor):
             'title': '可惜没如果',
             'upload_date': '20141227',
             'creator': '林俊杰',
+            'description': 'md5:242c97c2847e0495583b7b13764f7106',
         }
     }]
 
@@ -47,10 +48,16 @@ class QQMusicIE(InfoExtractor):
 
         publish_time = self._html_search_regex(
             r'发行时间：(\d{4}-\d{2}-\d{2})', detail_info_page,
-            'publish time').replace('-', '')
+            'publish time', default=None)
+        if publish_time:
+            publish_time = publish_time.replace('-', '')
 
         singer = self._html_search_regex(
-            r"singer:\s*'([^']+)", detail_info_page, 'singer')
+            r"singer:\s*'([^']+)", detail_info_page, 'singer', default=None)
+
+        lrc_content = self._html_search_regex(
+            r'<div class="content" id="lrc_content"[^<>]*>([^<>]+)</div>',
+            detail_info_page, 'LRC lyrics', default=None)
 
         guid = self.m_r_get_ruin()
 
@@ -66,6 +73,7 @@ class QQMusicIE(InfoExtractor):
             'title': song_name,
             'upload_date': publish_time,
             'creator': singer,
+            'description': lrc_content,
         }
 
 
@@ -74,10 +82,6 @@ class QQPlaylistBaseIE(InfoExtractor):
     def qq_static_url(category, mid):
         return 'http://y.qq.com/y/static/%s/%s/%s/%s.html' % (category, mid[-2], mid[-1], mid)
 
-    @staticmethod
-    def qq_song_url(mid):
-        return 'http://y.qq.com/#type=song&mid=%s' % mid
-
     @classmethod
     def get_entries_from_page(cls, page):
         entries = []
@@ -85,7 +89,8 @@ class QQPlaylistBaseIE(InfoExtractor):
         for item in re.findall(r'class="data"[^<>]*>([^<>]+)</', page):
             song_mid = unescapeHTML(item).split('|')[-5]
             entries.append(cls.url_result(
-                cls.qq_song_url(song_mid), 'QQMusic', song_mid))
+                'http://y.qq.com/#type=song&mid=' + song_mid, 'QQMusic',
+                song_mid))
 
         return entries
 

From c9a779695da56c1f9561af8586ecc586103dc254 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 21 Mar 2015 12:21:27 +0800
Subject: [PATCH 0952/6219] [extractor/common] Add the encoding parameter

The QQMusic info extractor need forced encoding for correct working.
---
 youtube_dl/extractor/common.py  | 34 ++++++++++++++++++++-------------
 youtube_dl/extractor/qqmusic.py |  4 ++--
 2 files changed, 23 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 8ed97f8dd..28f672e42 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -324,7 +324,7 @@ class InfoExtractor(object):
                 self._downloader.report_warning(errmsg)
                 return False
 
-    def _download_webpage_handle(self, url_or_request, video_id, note=None, errnote=None, fatal=True):
+    def _download_webpage_handle(self, url_or_request, video_id, note=None, errnote=None, fatal=True, encoding=None):
         """ Returns a tuple (page content as string, URL handle) """
         # Strip hashes from the URL (#1038)
         if isinstance(url_or_request, (compat_str, str)):
@@ -334,14 +334,11 @@ class InfoExtractor(object):
         if urlh is False:
             assert not fatal
             return False
-        content = self._webpage_read_content(urlh, url_or_request, video_id, note, errnote, fatal)
+        content = self._webpage_read_content(urlh, url_or_request, video_id, note, errnote, fatal, encoding=encoding)
         return (content, urlh)
 
-    def _webpage_read_content(self, urlh, url_or_request, video_id, note=None, errnote=None, fatal=True, prefix=None):
-        content_type = urlh.headers.get('Content-Type', '')
-        webpage_bytes = urlh.read()
-        if prefix is not None:
-            webpage_bytes = prefix + webpage_bytes
+    @staticmethod
+    def _guess_encoding_from_content(content_type, webpage_bytes):
         m = re.match(r'[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+\s*;\s*charset=(.+)', content_type)
         if m:
             encoding = m.group(1)
@@ -354,6 +351,16 @@ class InfoExtractor(object):
                 encoding = 'utf-16'
             else:
                 encoding = 'utf-8'
+
+        return encoding
+
+    def _webpage_read_content(self, urlh, url_or_request, video_id, note=None, errnote=None, fatal=True, prefix=None, encoding=None):
+        content_type = urlh.headers.get('Content-Type', '')
+        webpage_bytes = urlh.read()
+        if prefix is not None:
+            webpage_bytes = prefix + webpage_bytes
+        if not encoding:
+            encoding = self._guess_encoding_from_content(content_type, webpage_bytes)
         if self._downloader.params.get('dump_intermediate_pages', False):
             try:
                 url = url_or_request.get_full_url()
@@ -410,13 +417,13 @@ class InfoExtractor(object):
 
         return content
 
-    def _download_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True, tries=1, timeout=5):
+    def _download_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True, tries=1, timeout=5, encoding=None):
         """ Returns the data of the page as a string """
         success = False
         try_count = 0
         while success is False:
             try:
-                res = self._download_webpage_handle(url_or_request, video_id, note, errnote, fatal)
+                res = self._download_webpage_handle(url_or_request, video_id, note, errnote, fatal, encoding=encoding)
                 success = True
             except compat_http_client.IncompleteRead as e:
                 try_count += 1
@@ -431,10 +438,10 @@ class InfoExtractor(object):
 
     def _download_xml(self, url_or_request, video_id,
                       note='Downloading XML', errnote='Unable to download XML',
-                      transform_source=None, fatal=True):
+                      transform_source=None, fatal=True, encoding=None):
         """Return the xml as an xml.etree.ElementTree.Element"""
         xml_string = self._download_webpage(
-            url_or_request, video_id, note, errnote, fatal=fatal)
+            url_or_request, video_id, note, errnote, fatal=fatal, encoding=encoding)
         if xml_string is False:
             return xml_string
         if transform_source:
@@ -445,9 +452,10 @@ class InfoExtractor(object):
                        note='Downloading JSON metadata',
                        errnote='Unable to download JSON metadata',
                        transform_source=None,
-                       fatal=True):
+                       fatal=True, encoding=None):
         json_string = self._download_webpage(
-            url_or_request, video_id, note, errnote, fatal=fatal)
+            url_or_request, video_id, note, errnote, fatal=fatal,
+            encoding=encoding)
         if (not fatal) and json_string is False:
             return None
         return self._parse_json(
diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index e8aacbc3d..174c8e0ae 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -24,7 +24,7 @@ class QQMusicIE(InfoExtractor):
             'title': '可惜没如果',
             'upload_date': '20141227',
             'creator': '林俊杰',
-            'description': 'md5:242c97c2847e0495583b7b13764f7106',
+            'description': 'md5:4348ff1dd24036906baa7b6f973f8d30',
         }
     }]
 
@@ -41,7 +41,7 @@ class QQMusicIE(InfoExtractor):
         detail_info_page = self._download_webpage(
             'http://s.plcloud.music.qq.com/fcgi-bin/fcg_yqq_song_detail_info.fcg?songmid=%s&play=0' % mid,
             mid, note='Download song detail info',
-            errnote='Unable to get song detail info')
+            errnote='Unable to get song detail info', encoding='gbk')
 
         song_name = self._html_search_regex(
             r"songname:\s*'([^']+)'", detail_info_page, 'song name')

From c052ce6cde53f25949c21d00e4243669f523c0a7 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 16 Apr 2015 22:00:45 +0800
Subject: [PATCH 0953/6219] [Srf] Add new extractor (fixes #981)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/srf.py      | 86 ++++++++++++++++++++++++++++++++
 2 files changed, 87 insertions(+)
 create mode 100644 youtube_dl/extractor/srf.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 36860f72c..830090346 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -485,6 +485,7 @@ from .spike import SpikeIE
 from .sport5 import Sport5IE
 from .sportbox import SportBoxIE
 from .sportdeutschland import SportDeutschlandIE
+from .srf import SrfIE
 from .srmediathek import SRMediathekIE
 from .ssa import SSAIE
 from .stanfordoc import StanfordOpenClassroomIE
diff --git a/youtube_dl/extractor/srf.py b/youtube_dl/extractor/srf.py
new file mode 100644
index 000000000..87b4a676a
--- /dev/null
+++ b/youtube_dl/extractor/srf.py
@@ -0,0 +1,86 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+from .common import InfoExtractor
+from ..utils import (
+    determine_ext,
+    parse_iso8601,
+    xpath_text,
+)
+
+
+class SrfIE(InfoExtractor):
+    _VALID_URL = r'http://www\.srf\.ch/play(?:er)?/tv/[^/]+/video/(?P<display_id>[^?]+)\?id=(?P<id>[0-9a-f\-]{36})'
+    _TESTS = [{
+        'url': 'http://www.srf.ch/play/tv/10vor10/video/snowden-beantragt-asyl-in-russland?id=28e1a57d-5b76-4399-8ab3-9097f071e6c5',
+        'md5': '4cd93523723beff51bb4bee974ee238d',
+        'info_dict': {
+            'id': '28e1a57d-5b76-4399-8ab3-9097f071e6c5',
+            'display_id': 'snowden-beantragt-asyl-in-russland',
+            'ext': 'm4v',
+            'upload_date': '20130701',
+            'title': 'Snowden beantragt Asyl in Russland',
+            'timestamp': 1372713995,
+        }
+    }, {
+        # No Speichern (Save) button
+        'url': 'http://www.srf.ch/play/tv/top-gear/video/jaguar-xk120-shadow-und-tornado-dampflokomotive?id=677f5829-e473-4823-ac83-a1087fe97faa',
+        'info_dict': {
+            'id': '677f5829-e473-4823-ac83-a1087fe97faa',
+            'display_id': 'jaguar-xk120-shadow-und-tornado-dampflokomotive',
+            'ext': 'mp4',
+            'upload_date': '20130710',
+            'title': 'Jaguar XK120, Shadow und Tornado-Dampflokomotive',
+            'timestamp': 1373493600,
+        },
+        'params': {
+            # Require ffmpeg/avconv
+            'skip_download': True,
+        }
+    }, {
+        'url': 'http://www.srf.ch/player/tv/10vor10/video/snowden-beantragt-asyl-in-russland?id=28e1a57d-5b76-4399-8ab3-9097f071e6c5',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        video_data = self._download_xml(
+            'http://il.srgssr.ch/integrationlayer/1.0/ue/srf/video/play/%s.xml' % video_id,
+            video_id)
+
+        display_id = re.match(self._VALID_URL, url).group('display_id')
+        title = xpath_text(
+            video_data, './AssetMetadatas/AssetMetadata/title', fatal=True)
+        thumbnails = [{
+            'url': s.text
+        } for s in video_data.findall('.//ImageRepresentation/url')]
+        timestamp = parse_iso8601(xpath_text(video_data, './createdDate'))
+        # The <duration> field in XML is different from the exact duration, skipping
+
+        formats = []
+        for item in video_data.findall('./Playlists/Playlist') + video_data.findall('./Downloads/Download'):
+            url_node = item.find('url')
+            quality = url_node.attrib['quality']
+            full_url = url_node.text
+            original_ext = determine_ext(full_url)
+            if original_ext == 'f4m':
+                full_url += '?hdcore=3.4.0'  # Without this, you get a 403 error
+            formats.append({
+                'url': full_url,
+                'ext': 'mp4' if original_ext == 'm3u8' else original_ext,
+                'format_id': '%s-%s' % (quality, item.attrib['protocol']),
+                'preference': 0 if 'HD' in quality else -1,
+            })
+
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            'formats': formats,
+            'title': title,
+            'thumbnails': thumbnails,
+            'timestamp': timestamp,
+        }

From 355c524bfaec10aa8f5ff31bc95d2c50ef1a3113 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 16 Apr 2015 20:31:02 +0600
Subject: [PATCH 0954/6219] [srf] Extract all formats and prefer direct links
 over hls and hds

---
 youtube_dl/extractor/srf.py | 31 +++++++++++++++++++------------
 1 file changed, 19 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/srf.py b/youtube_dl/extractor/srf.py
index 87b4a676a..4be329a2f 100644
--- a/youtube_dl/extractor/srf.py
+++ b/youtube_dl/extractor/srf.py
@@ -61,18 +61,25 @@ class SrfIE(InfoExtractor):
 
         formats = []
         for item in video_data.findall('./Playlists/Playlist') + video_data.findall('./Downloads/Download'):
-            url_node = item.find('url')
-            quality = url_node.attrib['quality']
-            full_url = url_node.text
-            original_ext = determine_ext(full_url)
-            if original_ext == 'f4m':
-                full_url += '?hdcore=3.4.0'  # Without this, you get a 403 error
-            formats.append({
-                'url': full_url,
-                'ext': 'mp4' if original_ext == 'm3u8' else original_ext,
-                'format_id': '%s-%s' % (quality, item.attrib['protocol']),
-                'preference': 0 if 'HD' in quality else -1,
-            })
+            for url_node in item.findall('url'):
+                quality = url_node.attrib['quality']
+                full_url = url_node.text
+                original_ext = determine_ext(full_url)
+                format_id = '%s-%s' % (quality, item.attrib['protocol'])
+                if original_ext == 'f4m':
+                    formats.extend(self._extract_f4m_formats(
+                        full_url + '?hdcore=3.4.0', video_id, f4m_id=format_id))
+                elif original_ext == 'm3u8':
+                    formats.extend(self._extract_m3u8_formats(
+                        full_url, video_id, 'mp4', m3u8_id=format_id))
+                else:
+                    formats.append({
+                        'url': full_url,
+                        'ext': 'mp4' if original_ext == 'm3u8' else original_ext,
+                        'format_id': format_id,
+                        'quality': 0 if 'HD' in quality else -1,
+                        'preference': 1,
+                    })
 
         self._sort_formats(formats)
 

From 820b06480493ab1b3c146c3edfc2c89fc80061cc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 16 Apr 2015 20:48:17 +0600
Subject: [PATCH 0955/6219] [srf] Extract subtitles

---
 youtube_dl/extractor/srf.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/youtube_dl/extractor/srf.py b/youtube_dl/extractor/srf.py
index 4be329a2f..a080eb7ca 100644
--- a/youtube_dl/extractor/srf.py
+++ b/youtube_dl/extractor/srf.py
@@ -83,6 +83,16 @@ class SrfIE(InfoExtractor):
 
         self._sort_formats(formats)
 
+        subtitles = {}
+        subtitles_data = video_data.find('Subtitles')
+        if subtitles_data is not None:
+            subtitles_list = [{
+                'url': sub.text,
+                'ext': determine_ext(sub.text),
+            } for sub in subtitles_data]
+            if subtitles_list:
+                subtitles['de'] = subtitles_list
+
         return {
             'id': video_id,
             'display_id': display_id,
@@ -90,4 +100,5 @@ class SrfIE(InfoExtractor):
             'title': title,
             'thumbnails': thumbnails,
             'timestamp': timestamp,
+            'subtitles': subtitles,
         }

From fbbb2194097afcc3d3fb4f28888279f1b970b253 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 16 Apr 2015 21:28:21 +0600
Subject: [PATCH 0956/6219] [srf] Fix direct links ext

---
 youtube_dl/extractor/srf.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/srf.py b/youtube_dl/extractor/srf.py
index a080eb7ca..2ac55dcd1 100644
--- a/youtube_dl/extractor/srf.py
+++ b/youtube_dl/extractor/srf.py
@@ -75,7 +75,7 @@ class SrfIE(InfoExtractor):
                 else:
                     formats.append({
                         'url': full_url,
-                        'ext': 'mp4' if original_ext == 'm3u8' else original_ext,
+                        'ext': original_ext,
                         'format_id': format_id,
                         'quality': 0 if 'HD' in quality else -1,
                         'preference': 1,

From 89c09e2a0884301e405d014fdc9706cf1750f798 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 16 Apr 2015 21:30:13 +0600
Subject: [PATCH 0957/6219] [srf] Update test

---
 youtube_dl/extractor/srf.py | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/srf.py b/youtube_dl/extractor/srf.py
index 2ac55dcd1..e30351602 100644
--- a/youtube_dl/extractor/srf.py
+++ b/youtube_dl/extractor/srf.py
@@ -26,18 +26,15 @@ class SrfIE(InfoExtractor):
     }, {
         # No Speichern (Save) button
         'url': 'http://www.srf.ch/play/tv/top-gear/video/jaguar-xk120-shadow-und-tornado-dampflokomotive?id=677f5829-e473-4823-ac83-a1087fe97faa',
+        'md5': 'd97e236e80d1d24729e5d0953d276a4f',
         'info_dict': {
             'id': '677f5829-e473-4823-ac83-a1087fe97faa',
             'display_id': 'jaguar-xk120-shadow-und-tornado-dampflokomotive',
-            'ext': 'mp4',
+            'ext': 'flv',
             'upload_date': '20130710',
             'title': 'Jaguar XK120, Shadow und Tornado-Dampflokomotive',
             'timestamp': 1373493600,
         },
-        'params': {
-            # Require ffmpeg/avconv
-            'skip_download': True,
-        }
     }, {
         'url': 'http://www.srf.ch/player/tv/10vor10/video/snowden-beantragt-asyl-in-russland?id=28e1a57d-5b76-4399-8ab3-9097f071e6c5',
         'only_matching': True,

From 5cb91ceaa50ffe5b8551075765a8ad920efd0b59 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 16 Apr 2015 21:33:01 +0600
Subject: [PATCH 0958/6219] [pladform] Update test

---
 youtube_dl/extractor/pladform.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pladform.py b/youtube_dl/extractor/pladform.py
index abde34b94..551c8c9f0 100644
--- a/youtube_dl/extractor/pladform.py
+++ b/youtube_dl/extractor/pladform.py
@@ -30,7 +30,7 @@ class PladformIE(InfoExtractor):
         'info_dict': {
             'id': '100183293',
             'ext': 'mp4',
-            'title': 'Тайны перевала Дятлова • Тайна перевала Дятлова 1 серия 2 часть',
+            'title': 'Тайны перевала Дятлова • 1 серия 2 часть',
             'description': 'Документальный сериал-расследование одной из самых жутких тайн ХХ века',
             'thumbnail': 're:^https?://.*\.jpg$',
             'duration': 694,

From 62259846816568917d3cff41dbbce24a706c9fc1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 16 Apr 2015 21:37:15 +0600
Subject: [PATCH 0959/6219] [generic] Update pladform embed test

---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 8c859f068..e645d1bb3 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -615,7 +615,7 @@ class GenericIE(InfoExtractor):
             'info_dict': {
                 'id': '100183293',
                 'ext': 'mp4',
-                'title': 'Тайны перевала Дятлова • Тайна перевала Дятлова 1 серия 2 часть',
+                'title': 'Тайны перевала Дятлова • 1 серия 2 часть',
                 'description': 'Документальный сериал-расследование одной из самых жутких тайн ХХ века',
                 'thumbnail': 're:^https?://.*\.jpg$',
                 'duration': 694,

From 5141249c5930943cbf6890db0e36ba50f06a20ee Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 16 Apr 2015 21:47:42 +0600
Subject: [PATCH 0960/6219] [srf] Extend _VALID_URL

---
 youtube_dl/extractor/srf.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/srf.py b/youtube_dl/extractor/srf.py
index e30351602..25b589cb9 100644
--- a/youtube_dl/extractor/srf.py
+++ b/youtube_dl/extractor/srf.py
@@ -11,7 +11,7 @@ from ..utils import (
 
 
 class SrfIE(InfoExtractor):
-    _VALID_URL = r'http://www\.srf\.ch/play(?:er)?/tv/[^/]+/video/(?P<display_id>[^?]+)\?id=(?P<id>[0-9a-f\-]{36})'
+    _VALID_URL = r'https?://(?:www\.srf\.ch/play(?:er)?/tv/[^/]+/video/(?P<display_id>[^?]+)\?id=|tp\.srgssr\.ch/p/flash\?urn=urn:srf:ais:video:)(?P<id>[0-9a-f\-]{36})'
     _TESTS = [{
         'url': 'http://www.srf.ch/play/tv/10vor10/video/snowden-beantragt-asyl-in-russland?id=28e1a57d-5b76-4399-8ab3-9097f071e6c5',
         'md5': '4cd93523723beff51bb4bee974ee238d',
@@ -38,6 +38,9 @@ class SrfIE(InfoExtractor):
     }, {
         'url': 'http://www.srf.ch/player/tv/10vor10/video/snowden-beantragt-asyl-in-russland?id=28e1a57d-5b76-4399-8ab3-9097f071e6c5',
         'only_matching': True,
+    }, {
+        'url': 'https://tp.srgssr.ch/p/flash?urn=urn:srf:ais:video:28e1a57d-5b76-4399-8ab3-9097f071e6c5',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 65c1a750f5e5327a2ec8f702509eba05364ca4fa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 16 Apr 2015 21:48:22 +0600
Subject: [PATCH 0961/6219] [srf] Show display_id when present

---
 youtube_dl/extractor/srf.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/srf.py b/youtube_dl/extractor/srf.py
index 25b589cb9..77eec0bc7 100644
--- a/youtube_dl/extractor/srf.py
+++ b/youtube_dl/extractor/srf.py
@@ -45,12 +45,12 @@ class SrfIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
+        display_id = re.match(self._VALID_URL, url).group('display_id') or video_id
 
         video_data = self._download_xml(
             'http://il.srgssr.ch/integrationlayer/1.0/ue/srf/video/play/%s.xml' % video_id,
-            video_id)
+            display_id)
 
-        display_id = re.match(self._VALID_URL, url).group('display_id')
         title = xpath_text(
             video_data, './AssetMetadatas/AssetMetadata/title', fatal=True)
         thumbnails = [{
@@ -68,10 +68,10 @@ class SrfIE(InfoExtractor):
                 format_id = '%s-%s' % (quality, item.attrib['protocol'])
                 if original_ext == 'f4m':
                     formats.extend(self._extract_f4m_formats(
-                        full_url + '?hdcore=3.4.0', video_id, f4m_id=format_id))
+                        full_url + '?hdcore=3.4.0', display_id, f4m_id=format_id))
                 elif original_ext == 'm3u8':
                     formats.extend(self._extract_m3u8_formats(
-                        full_url, video_id, 'mp4', m3u8_id=format_id))
+                        full_url, display_id, 'mp4', m3u8_id=format_id))
                 else:
                     formats.append({
                         'url': full_url,

From be531ef1ec90cb5ee5b3f02d817a140313b76412 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 16 Apr 2015 22:12:38 +0600
Subject: [PATCH 0962/6219] [utils] Fix splitunc deprecation warning

---
 youtube_dl/utils.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 52f0dd09a..e628fac81 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -312,17 +312,17 @@ def sanitize_path(s):
     """Sanitizes and normalizes path on Windows"""
     if sys.platform != 'win32':
         return s
-    drive, _ = os.path.splitdrive(s)
-    unc, _ = os.path.splitunc(s)
-    unc_or_drive = unc or drive
-    norm_path = os.path.normpath(remove_start(s, unc_or_drive)).split(os.path.sep)
-    if unc_or_drive:
+    drive_or_unc, _ = os.path.splitdrive(s)
+    if sys.version_info < (2, 7) and not drive_or_unc:
+        drive_or_unc, _ = os.path.splitunc(s)
+    norm_path = os.path.normpath(remove_start(s, drive_or_unc)).split(os.path.sep)
+    if drive_or_unc:
         norm_path.pop(0)
     sanitized_path = [
         path_part if path_part in ['.', '..'] else re.sub('(?:[/<>:"\\|\\\\?\\*]|\.$)', '#', path_part)
         for path_part in norm_path]
-    if unc_or_drive:
-        sanitized_path.insert(0, unc_or_drive + os.path.sep)
+    if drive_or_unc:
+        sanitized_path.insert(0, drive_or_unc + os.path.sep)
     return os.path.join(*sanitized_path)
 
 

From 4aec95f3c932ee7042ca4dcae9fcd8c57341bb55 Mon Sep 17 00:00:00 2001
From: FireDart <firedartonline@gmail.com>
Date: Thu, 16 Apr 2015 18:10:53 -0400
Subject: [PATCH 0963/6219] [gfycat] Updated tests.

---
 youtube_dl/extractor/gfycat.py | 58 +++++++++++++++++++---------------
 1 file changed, 33 insertions(+), 25 deletions(-)

diff --git a/youtube_dl/extractor/gfycat.py b/youtube_dl/extractor/gfycat.py
index d103693cd..6de78c49d 100644
--- a/youtube_dl/extractor/gfycat.py
+++ b/youtube_dl/extractor/gfycat.py
@@ -8,31 +8,39 @@ from .common import InfoExtractor
 
 class GfycatIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?gfycat\.com/(?P<id>[^/?#]+)'
-    _TESTS = [{
-        'url': 'http://gfycat.com/DeadlyDecisiveGermanpinscher',
-        'info_dict': {
-            'id':          'DeadlyDecisiveGermanpinscher',
-            'title':       'Ghost in the Shell',
-            'ext':         'mp4',
-            'upload_date': '20140913'
-        }
-    },{
-        'url': 'http://gfycat.com/pleasinghilariouskusimanse',
-        'info_dict': {
-            'id':          'pleasinghilariouskusimanse',
-            'title':       'PleasingHilariousKusimanse',
-            'ext':         'webm',
-            'upload_date': '20150412'
-        }
-    },{
-        'url': 'http://gfycat.com/requiredunkemptbuzzard',
-        'info_dict': {
-            'id':          'requiredunkemptbuzzard',
-            'title':       'Headshot!',
-            'ext':         'gif',
-            'upload_date': '20150130'
-        }
-    }]
+    _TESTS = [
+        {
+            'url': 'http://gfycat.com/DeadlyDecisiveGermanpinscher',
+            'info_dict': {
+                'id':          'DeadlyDecisiveGermanpinscher',
+                'title':       'Ghost in the Shell',
+                'ext':         'mp4',
+                'upload_date': '20140913'
+            }
+        },{
+            'url': 'http://gfycat.com/pleasinghilariouskusimanse',
+            'info_dict': {
+                'id':          'pleasinghilariouskusimanse',
+                'title':       'PleasingHilariousKusimanse',
+                'ext':         'webm',
+                'upload_date': '20150412'
+            },
+            'params': {
+                'format': 'webm',
+            },
+        },{
+            'url': 'http://gfycat.com/requiredunkemptbuzzard',
+            'info_dict': {
+                'id':          'requiredunkemptbuzzard',
+                'title':       'Headshot!',
+                'ext':         'gif',
+                'upload_date': '20150129'
+            },
+            'params': {
+                'format': 'gif',
+            },
+        },
+    ]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From 954352c4c08dab0dd2d9ca20f5a414a307cea96f Mon Sep 17 00:00:00 2001
From: FireDart <firedartonline@gmail.com>
Date: Thu, 16 Apr 2015 18:11:30 -0400
Subject: [PATCH 0964/6219] [gfycat] Fixed preferences.

---
 youtube_dl/extractor/gfycat.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/gfycat.py b/youtube_dl/extractor/gfycat.py
index 6de78c49d..5e70ed3f6 100644
--- a/youtube_dl/extractor/gfycat.py
+++ b/youtube_dl/extractor/gfycat.py
@@ -63,7 +63,7 @@ class GfycatIE(InfoExtractor):
             'height':     json['height'],
             'fps':        json['frameRate'],
             'filesize':   json['mp4Size'],
-            'preference': '-1'
+            'preference': 2
         }, {
             'format_id': 'webm',
             'ext':       'webm',
@@ -72,7 +72,7 @@ class GfycatIE(InfoExtractor):
             'height':     json['height'],
             'fps':        json['frameRate'],
             'filesize':   json['webmSize'],
-            'preference': 0
+            'preference': 1
         }, {
             'format_id':  'gif',
             'ext':        'gif',
@@ -81,7 +81,7 @@ class GfycatIE(InfoExtractor):
             'height':     json['height'],
             'fps':        json['frameRate'],
             'filesize':   json['gifSize'],
-            'preference': 1
+            'preference': 0
         }]
         
         self._sort_formats(formats)

From f11554092b419baa919875432fe6ebc1f22f5307 Mon Sep 17 00:00:00 2001
From: Tjark Saul <tjark.saul@wachplan.net>
Date: Fri, 17 Apr 2015 09:21:54 +0200
Subject: [PATCH 0965/6219] [Lecture2Go] Add new extractor

---
 youtube_dl/extractor/__init__.py   |  1 +
 youtube_dl/extractor/lecture2go.py | 33 ++++++++++++++++++++++++++++++
 2 files changed, 34 insertions(+)
 create mode 100644 youtube_dl/extractor/lecture2go.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index bbf3be41d..3d6e981b2 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -249,6 +249,7 @@ from .krasview import KrasViewIE
 from .ku6 import Ku6IE
 from .la7 import LA7IE
 from .laola1tv import Laola1TvIE
+from .lecture2go import Lecture2GoIE
 from .letv import (
     LetvIE,
     LetvTvIE,
diff --git a/youtube_dl/extractor/lecture2go.py b/youtube_dl/extractor/lecture2go.py
new file mode 100644
index 000000000..9cf28e31c
--- /dev/null
+++ b/youtube_dl/extractor/lecture2go.py
@@ -0,0 +1,33 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class Lecture2GoIE(InfoExtractor):
+    _VALID_URL = r'https?://lecture2go.uni-hamburg.de/veranstaltungen/-/v/(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'https://lecture2go.uni-hamburg.de/veranstaltungen/-/v/17473',
+        'md5': 'a9e76f83b3ef58019c4b7dbc35f406c1',
+        'info_dict': {
+            'id': '17473',
+            'ext': 'mp4',
+            'url': 'https://fms1.rrz.uni-hamburg.de/abo/64.050_FrankHeitmann_2015-04-13_14-35.mp4',
+            'title': '2 - Endliche Automaten und reguläre Sprachen'
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        title = self._html_search_regex(r'<em class="title">(.*?)</em>', webpage, 'title')
+        video_url = self._search_regex(r'b.isFirefox..a.useHTML5\).b.setOption.a,"src","(.*.mp4)"\).else', webpage, 'video_url')
+        creator = self._html_search_regex(r'<div id="description">(.*)</div>', webpage, 'creator')
+
+        return {
+            'id': video_id,
+            'title': title,
+            'url': video_url,
+            'creator': creator
+        }

From 024ebb270663fbe27cfab52b1a8b9a21f227d985 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 17 Apr 2015 10:46:25 +0200
Subject: [PATCH 0966/6219] [soundcloud] Handle 'secret_token' for
 'w.soundcloud.com/player/?url=*' urls (fixes #5453)

---
 youtube_dl/extractor/soundcloud.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 316b2c90f..7efc6aff1 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -221,7 +221,12 @@ class SoundcloudIE(InfoExtractor):
                 info_json_url += "&secret_token=" + token
         elif mobj.group('player'):
             query = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
-            return self.url_result(query['url'][0])
+            real_url = query['url'][0]
+            # If the token is in the query of the original url we have to
+            # manually add it
+            if 'secret_token' in query:
+                real_url += '?secret_token=' + query['secret_token'][0]
+            return self.url_result(real_url)
         else:
             # extract uploader (which is in the url)
             uploader = mobj.group('uploader')

From 3220c50f9af3a1df239656025aebfe00fec00e45 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 17 Apr 2015 11:14:25 +0200
Subject: [PATCH 0967/6219] release 2015.04.17

---
 docs/supportedsites.md | 5 +++++
 youtube_dl/version.py  | 2 +-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index c85a39918..80e86c1b6 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -98,6 +98,7 @@
  - **CondeNast**: Condé Nast media group: Condé Nast, GQ, Glamour, Vanity Fair, Vogue, W Magazine, WIRED
  - **Cracked**
  - **Criterion**
+ - **CrooksAndLiars**
  - **Crunchyroll**
  - **crunchyroll:playlist**
  - **CSpan**: C-SPAN
@@ -359,6 +360,9 @@
  - **prosiebensat1**: ProSiebenSat.1 Digital
  - **Puls4**
  - **Pyvideo**
+ - **QQMusic**
+ - **QQMusicAlbum**
+ - **QQMusicSinger**
  - **QuickVid**
  - **R7**
  - **radio.de**
@@ -434,6 +438,7 @@
  - **Sport5**
  - **SportBox**
  - **SportDeutschland**
+ - **Srf**
  - **SRMediathek**: Saarländischer Rundfunk
  - **SSA**
  - **stanfordoc**: Stanford Open ClassRoom
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 1095fea2f..3fd0e7e56 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.04.09'
+__version__ = '2015.04.17'

From 08f2a92c9c23cf460b00a290b5b3819c7972231b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 17 Apr 2015 14:55:24 +0200
Subject: [PATCH 0968/6219] InfoExtractor._search_regex: Suggest updating when
 the regex is not found (suggested in #5442)

Reuse the same message from ExtractorError
---
 youtube_dl/extractor/common.py |  4 ++--
 youtube_dl/utils.py            | 19 ++++++++++++-------
 2 files changed, 14 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 28f672e42..7757bf950 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -23,6 +23,7 @@ from ..compat import (
 )
 from ..utils import (
     age_restricted,
+    bug_reports_message,
     clean_html,
     compiled_regex_type,
     ExtractorError,
@@ -556,8 +557,7 @@ class InfoExtractor(object):
         elif fatal:
             raise RegexNotFoundError('Unable to extract %s' % _name)
         else:
-            self._downloader.report_warning('unable to extract %s; '
-                                            'please report this issue on http://yt-dl.org/bug' % _name)
+            self._downloader.report_warning('unable to extract %s' % _name + bug_reports_message())
             return None
 
     def _html_search_regex(self, pattern, string, name, default=_NO_DEFAULT, fatal=True, flags=0, group=None):
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index e628fac81..edeee1853 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -452,6 +452,17 @@ def make_HTTPS_handler(params, **kwargs):
         return YoutubeDLHTTPSHandler(params, context=context, **kwargs)
 
 
+def bug_reports_message():
+    if ytdl_is_updateable():
+        update_cmd = 'type  youtube-dl -U  to update'
+    else:
+        update_cmd = 'see  https://yt-dl.org/update  on how to update'
+    msg = '; please report this issue on https://yt-dl.org/bug .'
+    msg += ' Make sure you are using the latest version; %s.' % update_cmd
+    msg += ' Be sure to call youtube-dl with the --verbose flag and include its complete output.'
+    return msg
+
+
 class ExtractorError(Exception):
     """Error during info extraction."""
 
@@ -467,13 +478,7 @@ class ExtractorError(Exception):
         if cause:
             msg += ' (caused by %r)' % cause
         if not expected:
-            if ytdl_is_updateable():
-                update_cmd = 'type  youtube-dl -U  to update'
-            else:
-                update_cmd = 'see  https://yt-dl.org/update  on how to update'
-            msg += '; please report this issue on https://yt-dl.org/bug .'
-            msg += ' Make sure you are using the latest version; %s.' % update_cmd
-            msg += ' Be sure to call youtube-dl with the --verbose flag and include its complete output.'
+            msg += bug_reports_message()
         super(ExtractorError, self).__init__(msg)
 
         self.traceback = tb

From d8e7ef04dcb583f3271a6b6a099a3da2e650fb45 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Apr 2015 22:56:26 +0600
Subject: [PATCH 0969/6219] [vimple] Fix extraction (Closes #5448)

---
 youtube_dl/extractor/vimple.py | 71 ++++++++++++----------------------
 1 file changed, 25 insertions(+), 46 deletions(-)

diff --git a/youtube_dl/extractor/vimple.py b/youtube_dl/extractor/vimple.py
index ee3d86117..aa3d6ddfd 100644
--- a/youtube_dl/extractor/vimple.py
+++ b/youtube_dl/extractor/vimple.py
@@ -1,75 +1,54 @@
-# coding: utf-8
 from __future__ import unicode_literals
 
-import base64
-import re
-import xml.etree.ElementTree
-import zlib
-
 from .common import InfoExtractor
 from ..utils import int_or_none
 
 
 class VimpleIE(InfoExtractor):
-    IE_DESC = 'Vimple.ru'
-    _VALID_URL = r'https?://(player.vimple.ru/iframe|vimple.ru)/(?P<id>[a-f0-9]{10,})'
+    IE_DESC = 'Vimple - one-click video hosting'
+    _VALID_URL = r'https?://(?:player\.vimple\.ru/iframe|vimple\.ru)/(?P<id>[\da-f-]{32,36})'
     _TESTS = [
         {
             'url': 'http://vimple.ru/c0f6b1687dcd4000a97ebe70068039cf',
             'md5': '2e750a330ed211d3fd41821c6ad9a279',
             'info_dict': {
-                'id': 'c0f6b1687dcd4000a97ebe70068039cf',
+                'id': 'c0f6b168-7dcd-4000-a97e-be70068039cf',
                 'ext': 'mp4',
                 'title': 'Sunset',
                 'duration': 20,
                 'thumbnail': 're:https?://.*?\.jpg',
             },
-        },
+        }, {
+            'url': 'http://player.vimple.ru/iframe/52e1beec-1314-4a83-aeac-c61562eadbf9',
+            'only_matching': True,
+        }
     ]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
-        iframe_url = 'http://player.vimple.ru/iframe/%s' % video_id
+        webpage = self._download_webpage(
+            'http://player.vimple.ru/iframe/%s' % video_id, video_id)
 
-        iframe = self._download_webpage(
-            iframe_url, video_id,
-            note='Downloading iframe', errnote='unable to fetch iframe')
-        player_url = self._html_search_regex(
-            r'"(http://player.vimple.ru/flash/.+?)"', iframe, 'player url')
+        playlist = self._parse_json(
+            self._search_regex(
+                r'sprutoData\s*:\s*({.+?}),\r\n', webpage, 'spruto data'),
+            video_id)['playlist'][0]
 
-        player = self._request_webpage(
-            player_url, video_id, note='Downloading swf player').read()
+        title = playlist['title']
+        video_id = playlist.get('videoId') or video_id
+        thumbnail = playlist.get('posterUrl') or playlist.get('thumbnailUrl')
+        duration = int_or_none(playlist.get('duration'))
 
-        player = zlib.decompress(player[8:])
-
-        xml_pieces = re.findall(b'([a-zA-Z0-9 =+/]{500})', player)
-        xml_pieces = [piece[1:-1] for piece in xml_pieces]
-
-        xml_data = b''.join(xml_pieces)
-        xml_data = base64.b64decode(xml_data)
-
-        xml_data = xml.etree.ElementTree.fromstring(xml_data)
-
-        video = xml_data.find('Video')
-        quality = video.get('quality')
-        q_tag = video.find(quality.capitalize())
-
-        formats = [
-            {
-                'url': q_tag.get('url'),
-                'tbr': int(q_tag.get('bitrate')),
-                'filesize': int(q_tag.get('filesize')),
-                'format_id': quality,
-            },
-        ]
+        formats = [{
+            'url': f['url'],
+        } for f in playlist['video']]
+        self._sort_formats(formats)
 
         return {
             'id': video_id,
-            'title': video.find('Title').text,
+            'title': title,
+            'thumbnail': thumbnail,
+            'duration': duration,
             'formats': formats,
-            'thumbnail': video.find('Poster').get('url'),
-            'duration': int_or_none(video.get('duration')),
-            'webpage_url': video.find('Share').get('videoPageUrl'),
         }

From c5826a491b7b214a7e81030ad53103c4aca04dc2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 17 Apr 2015 19:02:49 +0200
Subject: [PATCH 0970/6219] [mixcloud] Simplify url extraction

On the tracks I tested the server number in the url from the webpage is valid
for the mp3 or the m4a file and any other number is invalid, it's a
waste of time to check them.
---
 youtube_dl/extractor/mixcloud.py | 72 +++++++-------------------------
 1 file changed, 15 insertions(+), 57 deletions(-)

diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dl/extractor/mixcloud.py
index 84f291558..425a4ccf1 100644
--- a/youtube_dl/extractor/mixcloud.py
+++ b/youtube_dl/extractor/mixcloud.py
@@ -1,7 +1,6 @@
 from __future__ import unicode_literals
 
 import re
-import itertools
 
 from .common import InfoExtractor
 from ..compat import (
@@ -46,20 +45,16 @@ class MixcloudIE(InfoExtractor):
         },
     }]
 
-    def _get_url(self, track_id, template_url, server_number):
-        boundaries = (1, 30)
-        for nr in server_numbers(server_number, boundaries):
-            url = template_url % nr
-            try:
-                # We only want to know if the request succeed
-                # don't download the whole file
-                self._request_webpage(
-                    HEADRequest(url), track_id,
-                    'Checking URL %d/%d ...' % (nr, boundaries[-1]))
-                return url
-            except ExtractorError:
-                pass
-        return None
+    def _check_url(self, url, track_id, ext):
+        try:
+            # We only want to know if the request succeed
+            # don't download the whole file
+            self._request_webpage(
+                HEADRequest(url), track_id,
+                'Trying %s URL' % ext)
+            return True
+        except ExtractorError:
+            return False
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
@@ -72,15 +67,10 @@ class MixcloudIE(InfoExtractor):
         preview_url = self._search_regex(
             r'\s(?:data-preview-url|m-preview)="([^"]+)"', webpage, 'preview url')
         song_url = preview_url.replace('/previews/', '/c/originals/')
-        server_number = int(self._search_regex(r'stream(\d+)', song_url, 'server number'))
-        template_url = re.sub(r'(stream\d*)', 'stream%d', song_url)
-        final_song_url = self._get_url(track_id, template_url, server_number)
-        if final_song_url is None:
-            self.to_screen('Trying with m4a extension')
-            template_url = template_url.replace('.mp3', '.m4a').replace('originals/', 'm4a/64/')
-            final_song_url = self._get_url(track_id, template_url, server_number)
-        if final_song_url is None:
-            raise ExtractorError('Unable to extract track url')
+        if not self._check_url(song_url, track_id, 'mp3'):
+            song_url = song_url.replace('.mp3', '.m4a').replace('originals/', 'm4a/64/')
+            if not self._check_url(song_url, track_id, 'm4a'):
+                raise ExtractorError('Unable to extract track url')
 
         PREFIX = (
             r'm-play-on-spacebar[^>]+'
@@ -107,7 +97,7 @@ class MixcloudIE(InfoExtractor):
         return {
             'id': track_id,
             'title': title,
-            'url': final_song_url,
+            'url': song_url,
             'description': description,
             'thumbnail': thumbnail,
             'uploader': uploader,
@@ -115,35 +105,3 @@ class MixcloudIE(InfoExtractor):
             'view_count': view_count,
             'like_count': like_count,
         }
-
-
-def server_numbers(first, boundaries):
-    """ Server numbers to try in descending order of probable availability.
-    Starting from first (i.e. the number of the server hosting the preview file)
-    and going further and further up to the higher boundary and down to the
-    lower one in an alternating fashion. Namely:
-
-        server_numbers(2, (1, 5))
-
-        # Where the preview server is 2, min number is 1 and max is 5.
-        # Yields: 2, 3, 1, 4, 5
-
-    Why not random numbers or increasing sequences? Since from what I've seen,
-    full length files seem to be hosted on servers whose number is closer to
-    that of the preview; to be confirmed.
-    """
-    zip_longest = getattr(itertools, 'zip_longest', None)
-    if zip_longest is None:
-        # python 2.x
-        zip_longest = itertools.izip_longest
-
-    if len(boundaries) != 2:
-        raise ValueError("boundaries should be a two-element tuple")
-    min, max = boundaries
-    highs = range(first + 1, max + 1)
-    lows = range(first - 1, min - 1, -1)
-    rest = filter(
-        None, itertools.chain.from_iterable(zip_longest(highs, lows)))
-    yield first
-    for n in rest:
-        yield n

From 214e74bf6f6eee15cf85c9d5a4defce14b56d586 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 17 Apr 2015 19:24:30 +0200
Subject: [PATCH 0971/6219] [soundcloud] Raise an error instead of calling
 'report_error'

---
 youtube_dl/extractor/soundcloud.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 7efc6aff1..183ff50f4 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -279,9 +279,8 @@ class SoundcloudSetIE(SoundcloudIE):
         info = self._download_json(resolv_url, full_title)
 
         if 'errors' in info:
-            for err in info['errors']:
-                self._downloader.report_error('unable to download video webpage: %s' % compat_str(err['error_message']))
-            return
+            msgs = (compat_str(err['error_message']) for err in info['errors'])
+            raise ExtractorError('unable to download video webpage: %s' % ','.join(msgs))
 
         return {
             '_type': 'playlist',

From 7691a7a3bd77dd2c169a9a8592283d99a9266973 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Apr 2015 23:41:07 +0600
Subject: [PATCH 0972/6219] [comedycentral] Fix feed uri request (Closes #5449,
 closes #5455)

---
 youtube_dl/extractor/comedycentral.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/comedycentral.py b/youtube_dl/extractor/comedycentral.py
index e5edcc84b..91ebb0ce5 100644
--- a/youtube_dl/extractor/comedycentral.py
+++ b/youtube_dl/extractor/comedycentral.py
@@ -201,7 +201,7 @@ class ComedyCentralShowsIE(MTVServicesInfoExtractor):
 
         uri = mMovieParams[0][1]
         # Correct cc.com in uri
-        uri = re.sub(r'(episode:[^.]+)(\.cc)?\.com', r'\1.cc.com', uri)
+        uri = re.sub(r'(episode:[^.]+)(\.cc)?\.com', r'\1.com', uri)
 
         index_url = 'http://%s.cc.com/feeds/mrss?%s' % (show_name, compat_urllib_parse.urlencode({'uri': uri}))
         idoc = self._download_xml(

From ce81b1411d182fbfe7bd6da7b875d50f37ae38d6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 17 Apr 2015 22:29:30 +0200
Subject: [PATCH 0973/6219] FFmpegExtractAudioPP: Simplify handling of already
 existing files

---
 youtube_dl/postprocessor/ffmpeg.py | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 8e99a3c2c..4c4a038f9 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -264,15 +264,14 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
         new_path = prefix + sep + extension
 
         # If we download foo.mp3 and convert it to... foo.mp3, then don't delete foo.mp3, silly.
-        if new_path == path:
-            self._nopostoverwrites = True
+        if (new_path == path or
+                (self._nopostoverwrites and os.path.exists(encodeFilename(new_path)))):
+            self._downloader.to_screen('[youtube] Post-process file %s exists, skipping' % new_path)
+            return True, information
 
         try:
-            if self._nopostoverwrites and os.path.exists(encodeFilename(new_path)):
-                self._downloader.to_screen('[youtube] Post-process file %s exists, skipping' % new_path)
-            else:
-                self._downloader.to_screen('[' + self.basename + '] Destination: ' + new_path)
-                self.run_ffmpeg(path, new_path, acodec, more_opts)
+            self._downloader.to_screen('[' + self.basename + '] Destination: ' + new_path)
+            self.run_ffmpeg(path, new_path, acodec, more_opts)
         except AudioConversionError as e:
             raise PostProcessingError(
                 'audio conversion failed: ' + e.msg)
@@ -286,7 +285,7 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
                 errnote='Cannot update utime of audio file')
 
         information['filepath'] = new_path
-        return self._nopostoverwrites, information
+        return False, information
 
 
 class FFmpegVideoConvertorPP(FFmpegPostProcessor):

From ecc6bd1341cd03fd335058371c3efa63dfd31152 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 17 Apr 2015 22:38:14 +0200
Subject: [PATCH 0974/6219] YoutubeDL.post_process: simplify keep_video
 handling

Since keep_video started as None we always set it to keep_video_wish unless it was None, so in the end keep_video == keep_video_wish. This should have been changed in f3ff1a3696c4080468e2cc5810c34273b148bd3e, but I didn't notice it.
---
 youtube_dl/YoutubeDL.py | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index a68b24ab4..6ac85f4e7 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1486,16 +1486,9 @@ class YoutubeDL(object):
             pps_chain.extend(ie_info['__postprocessors'])
         pps_chain.extend(self._pps)
         for pp in pps_chain:
-            keep_video = None
             old_filename = info['filepath']
             try:
-                keep_video_wish, info = pp.run(info)
-                if keep_video_wish is not None:
-                    if keep_video_wish:
-                        keep_video = keep_video_wish
-                    elif keep_video is None:
-                        # No clear decision yet, let IE decide
-                        keep_video = keep_video_wish
+                keep_video, info = pp.run(info)
             except PostProcessingError as e:
                 self.report_error(e.msg)
             if keep_video is False and not self.params.get('keepvideo', False):

From f52e66505a9f9403d2278d22732ed98b711292fc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Apr 2015 03:50:22 +0600
Subject: [PATCH 0975/6219] [gfycat] Simplify (Closes #5439, Closes #5394)

---
 youtube_dl/extractor/gfycat.py | 179 ++++++++++++++-------------------
 1 file changed, 77 insertions(+), 102 deletions(-)

diff --git a/youtube_dl/extractor/gfycat.py b/youtube_dl/extractor/gfycat.py
index 5e70ed3f6..397f1d42e 100644
--- a/youtube_dl/extractor/gfycat.py
+++ b/youtube_dl/extractor/gfycat.py
@@ -1,115 +1,90 @@
 # coding: utf-8
-
 from __future__ import unicode_literals
 
-import datetime
-
 from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    float_or_none,
+    qualities,
+)
+
 
 class GfycatIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?gfycat\.com/(?P<id>[^/?#]+)'
-    _TESTS = [
-        {
-            'url': 'http://gfycat.com/DeadlyDecisiveGermanpinscher',
-            'info_dict': {
-                'id':          'DeadlyDecisiveGermanpinscher',
-                'title':       'Ghost in the Shell',
-                'ext':         'mp4',
-                'upload_date': '20140913'
-            }
-        },{
-            'url': 'http://gfycat.com/pleasinghilariouskusimanse',
-            'info_dict': {
-                'id':          'pleasinghilariouskusimanse',
-                'title':       'PleasingHilariousKusimanse',
-                'ext':         'webm',
-                'upload_date': '20150412'
-            },
-            'params': {
-                'format': 'webm',
-            },
-        },{
-            'url': 'http://gfycat.com/requiredunkemptbuzzard',
-            'info_dict': {
-                'id':          'requiredunkemptbuzzard',
-                'title':       'Headshot!',
-                'ext':         'gif',
-                'upload_date': '20150129'
-            },
-            'params': {
-                'format': 'gif',
-            },
-        },
-    ]
+    _TEST = {
+        'url': 'http://gfycat.com/DeadlyDecisiveGermanpinscher',
+        'info_dict': {
+            'id': 'DeadlyDecisiveGermanpinscher',
+            'ext': 'mp4',
+            'title': 'Ghost in the Shell',
+            'timestamp': 1410656006,
+            'upload_date': '20140914',
+            'uploader': 'anonymous',
+            'duration': 10.4,
+            'view_count': int,
+            'like_count': int,
+            'dislike_count': int,
+            'categories': list,
+            'age_limit': 0,
+        }
+    }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        json     = self._download_json("http://gfycat.com/cajax/get/" + video_id, video_id, 'Downloading video info')['gfyItem']
-        
-        # Title
-        # Use user title first, else fallback to url formated name
-        if json['title']:
-            video_title = json['title']
-        else:
-            video_title = json['gfyName']
-        
-        # Formats
-        # Pref: mp4, webm, gif
-        formats = [{
-            'format_id':  'mp4',
-            'ext':        'mp4',
-            'url':        json['mp4Url'],
-            'width':      json['width'],
-            'height':     json['height'],
-            'fps':        json['frameRate'],
-            'filesize':   json['mp4Size'],
-            'preference': 2
-        }, {
-            'format_id': 'webm',
-            'ext':       'webm',
-            'url':        json['webmUrl'],
-            'width':      json['width'],
-            'height':     json['height'],
-            'fps':        json['frameRate'],
-            'filesize':   json['webmSize'],
-            'preference': 1
-        }, {
-            'format_id':  'gif',
-            'ext':        'gif',
-            'url':        json['gifUrl'],
-            'width':      json['width'],
-            'height':     json['height'],
-            'fps':        json['frameRate'],
-            'filesize':   json['gifSize'],
-            'preference': 0
-        }]
-        
+
+        gfy = self._download_json(
+            'http://gfycat.com/cajax/get/%s' % video_id,
+            video_id, 'Downloading video info')['gfyItem']
+
+        title = gfy.get('title') or gfy['gfyName']
+        description = gfy.get('description')
+        timestamp = int_or_none(gfy.get('createDate'))
+        uploader = gfy.get('userName')
+        view_count = int_or_none(gfy.get('views'))
+        like_count = int_or_none(gfy.get('likes'))
+        dislike_count = int_or_none(gfy.get('dislikes'))
+        age_limit = 18 if gfy.get('nsfw') == '1' else 0
+
+        width = int_or_none(gfy.get('width'))
+        height = int_or_none(gfy.get('height'))
+        fps = int_or_none(gfy.get('frameRate'))
+        num_frames = int_or_none(gfy.get('numFrames'))
+
+        duration = float_or_none(num_frames, fps) if num_frames and fps else None
+
+        categories = gfy.get('tags') or gfy.get('extraLemmas') or []
+
+        FORMATS = ('gif', 'webm', 'mp4')
+        quality = qualities(FORMATS)
+
+        formats = []
+        for format_id in FORMATS:
+            video_url = gfy.get('%sUrl' % format_id)
+            if not video_url:
+                continue
+            filesize = gfy.get('%sSize' % format_id)
+            formats.append({
+                'url': video_url,
+                'format_id': format_id,
+                'width': width,
+                'height': height,
+                'fps': fps,
+                'filesize': filesize,
+                'quality': quality(format_id),
+            })
         self._sort_formats(formats)
-        
-        # Date
-        date = datetime.datetime.fromtimestamp(
-            int(json['createDate'])
-        ).strftime('%Y%m%d')
-        
-        # Length
-        duration = json['numFrames'] / json['frameRate']
-        
-        # Age limit
-        # 1 = nsfw / 0 = sfw
-        if json['nsfw'] == 1:
-            age_limit = 18
-        else:
-            age_limit = 0
-        
+
         return {
-            'id':          video_id,
-            'title':       video_title,
-            'formats':     formats,
-            'creator':     json['userName'],
-            'description': json['description'],
-            'upload_date': date,
-            'categories':  json['tags'],
-            'age_limit':   age_limit,
-            'duration':    duration,
-            'view_count':  json['views']
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'timestamp': timestamp,
+            'uploader': uploader,
+            'duration': duration,
+            'view_count': view_count,
+            'like_count': like_count,
+            'dislike_count': dislike_count,
+            'categories': categories,
+            'age_limit': age_limit,
+            'formats': formats,
         }

From bf12cbe07c3a22deb46848df3bd3242da645eab1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Apr 2015 03:51:21 +0600
Subject: [PATCH 0976/6219] Credit @julianrichen for gfycat (#5440)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index db3f42b26..267b8da1e 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -123,3 +123,4 @@ Will W.
 Mohammad Teimori Pabandi
 Roman Le Négrate
 Matthias Küch
+Julian Richen

From c62566971fb8877d5ac52bc613d3bd53fe87a8d8 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 17 Apr 2015 00:08:52 +0800
Subject: [PATCH 0977/6219] [facebook] Extend _VALID_URL

---
 youtube_dl/extractor/facebook.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index f0e575320..f3a68b21d 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -24,8 +24,12 @@ class FacebookIE(InfoExtractor):
     _VALID_URL = r'''(?x)
         https?://(?:\w+\.)?facebook\.com/
         (?:[^#]*?\#!/)?
-        (?:video/video\.php|photo\.php|video\.php|video/embed)\?(?:.*?)
-        (?:v|video_id)=(?P<id>[0-9]+)
+        (?:
+            (?:video/video\.php|photo\.php|video\.php|video/embed)\?(?:.*?)
+            (?:v|video_id)=|
+            [^/]+/videos/
+        )
+        (?P<id>[0-9]+)
         (?:.*)'''
     _LOGIN_URL = 'https://www.facebook.com/login.php?next=http%3A%2F%2Ffacebook.com%2Fhome.php&login_attempt=1'
     _CHECKPOINT_URL = 'https://www.facebook.com/checkpoint/?next=http%3A%2F%2Ffacebook.com%2Fhome.php&_fb_noscript=1'
@@ -50,6 +54,9 @@ class FacebookIE(InfoExtractor):
     }, {
         'url': 'https://www.facebook.com/video.php?v=10204634152394104',
         'only_matching': True,
+    }, {
+        'url': 'https://www.facebook.com/amogood/videos/1618742068337349/?fref=nf',
+        'only_matching': True,
     }]
 
     def _login(self):

From 53faa3ca5f62c46bb56c0a85d1ed87b911b7ffa4 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 18 Apr 2015 16:08:24 +0800
Subject: [PATCH 0978/6219] [facebook] Extend _VALID_URL take 2 (#5120)

---
 youtube_dl/extractor/facebook.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index f3a68b21d..937b28fcc 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -27,7 +27,7 @@ class FacebookIE(InfoExtractor):
         (?:
             (?:video/video\.php|photo\.php|video\.php|video/embed)\?(?:.*?)
             (?:v|video_id)=|
-            [^/]+/videos/
+            [^/]+/videos/(?:[^/]+/)?
         )
         (?P<id>[0-9]+)
         (?:.*)'''
@@ -57,6 +57,9 @@ class FacebookIE(InfoExtractor):
     }, {
         'url': 'https://www.facebook.com/amogood/videos/1618742068337349/?fref=nf',
         'only_matching': True,
+    }, {
+        'url': 'https://www.facebook.com/ChristyClarkForBC/videos/vb.22819070941/10153870694020942/?type=2&theater',
+        'only_matching': True,
     }]
 
     def _login(self):

From 592e97e8550389e22b716eb33c30584aa3a8d656 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 18 Apr 2015 11:36:42 +0200
Subject: [PATCH 0979/6219] Postprocessors: use a list for the files that can
 be deleted

We could only know if we had to delete the original file, but this system allows to specify us more files (like subtitles).
---
 test/test_YoutubeDL.py                        | 21 ++++++++----
 youtube_dl/YoutubeDL.py                       | 13 +++----
 youtube_dl/postprocessor/atomicparsley.py     |  2 +-
 youtube_dl/postprocessor/common.py            |  8 ++---
 youtube_dl/postprocessor/execafterdownload.py |  2 +-
 youtube_dl/postprocessor/ffmpeg.py            | 34 +++++++++----------
 youtube_dl/postprocessor/metadatafromtitle.py |  2 +-
 youtube_dl/postprocessor/xattrpp.py           |  4 +--
 8 files changed, 48 insertions(+), 38 deletions(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 652519831..820e55ec2 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -443,27 +443,36 @@ class TestYoutubeDL(unittest.TestCase):
             def run(self, info):
                 with open(audiofile, 'wt') as f:
                     f.write('EXAMPLE')
-                info['filepath']
-                return False, info
+                return [info['filepath']], info
 
-        def run_pp(params):
+        def run_pp(params, PP):
             with open(filename, 'wt') as f:
                 f.write('EXAMPLE')
             ydl = YoutubeDL(params)
-            ydl.add_post_processor(SimplePP())
+            ydl.add_post_processor(PP())
             ydl.post_process(filename, {'filepath': filename})
 
-        run_pp({'keepvideo': True})
+        run_pp({'keepvideo': True}, SimplePP)
         self.assertTrue(os.path.exists(filename), '%s doesn\'t exist' % filename)
         self.assertTrue(os.path.exists(audiofile), '%s doesn\'t exist' % audiofile)
         os.unlink(filename)
         os.unlink(audiofile)
 
-        run_pp({'keepvideo': False})
+        run_pp({'keepvideo': False}, SimplePP)
         self.assertFalse(os.path.exists(filename), '%s exists' % filename)
         self.assertTrue(os.path.exists(audiofile), '%s doesn\'t exist' % audiofile)
         os.unlink(audiofile)
 
+        class ModifierPP(PostProcessor):
+            def run(self, info):
+                with open(info['filepath'], 'wt') as f:
+                    f.write('MODIFIED')
+                return [], info
+
+        run_pp({'keepvideo': False}, ModifierPP)
+        self.assertTrue(os.path.exists(filename), '%s doesn\'t exist' % filename)
+        os.unlink(filename)
+
     def test_match_filter(self):
         class FilterYDL(YDL):
             def __init__(self, *args, **kwargs):
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 6ac85f4e7..8d8b146b2 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1488,15 +1488,16 @@ class YoutubeDL(object):
         for pp in pps_chain:
             old_filename = info['filepath']
             try:
-                keep_video, info = pp.run(info)
+                files_to_delete, info = pp.run(info)
             except PostProcessingError as e:
                 self.report_error(e.msg)
-            if keep_video is False and not self.params.get('keepvideo', False):
-                try:
+            if files_to_delete and not self.params.get('keepvideo', False):
+                for old_filename in files_to_delete:
                     self.to_screen('Deleting original file %s (pass -k to keep)' % old_filename)
-                    os.remove(encodeFilename(old_filename))
-                except (IOError, OSError):
-                    self.report_warning('Unable to remove downloaded video file')
+                    try:
+                        os.remove(encodeFilename(old_filename))
+                    except (IOError, OSError):
+                        self.report_warning('Unable to remove downloaded original file')
 
     def _make_archive_id(self, info_dict):
         # Future-proof against any change in case
diff --git a/youtube_dl/postprocessor/atomicparsley.py b/youtube_dl/postprocessor/atomicparsley.py
index a5dfc136a..e4e198695 100644
--- a/youtube_dl/postprocessor/atomicparsley.py
+++ b/youtube_dl/postprocessor/atomicparsley.py
@@ -59,4 +59,4 @@ class AtomicParsleyPP(PostProcessor):
             os.remove(encodeFilename(filename))
             os.rename(encodeFilename(temp_filename), encodeFilename(filename))
 
-        return True, info
+        return [], info
diff --git a/youtube_dl/postprocessor/common.py b/youtube_dl/postprocessor/common.py
index ef9fdfa19..3b0e8ddd8 100644
--- a/youtube_dl/postprocessor/common.py
+++ b/youtube_dl/postprocessor/common.py
@@ -42,14 +42,14 @@ class PostProcessor(object):
         one has an extra field called "filepath" that points to the
         downloaded file.
 
-        This method returns a tuple, the first element of which describes
-        whether the original file should be kept (i.e. not deleted - None for
-        no preference), and the second of which is the updated information.
+        This method returns a tuple, the first element is a list of the files
+        that can be deleted, and the second of which is the updated
+        information.
 
         In addition, this method may raise a PostProcessingError
         exception if post processing fails.
         """
-        return None, information  # by default, keep file and do nothing
+        return [], information  # by default, keep file and do nothing
 
     def try_utime(self, path, atime, mtime, errnote='Cannot update utime of file'):
         try:
diff --git a/youtube_dl/postprocessor/execafterdownload.py b/youtube_dl/postprocessor/execafterdownload.py
index 75c0f7bbe..341437575 100644
--- a/youtube_dl/postprocessor/execafterdownload.py
+++ b/youtube_dl/postprocessor/execafterdownload.py
@@ -25,4 +25,4 @@ class ExecAfterDownloadPP(PostProcessor):
             raise PostProcessingError(
                 'Command returned error code %d' % retCode)
 
-        return None, information  # by default, keep file and do nothing
+        return [], information
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 4c4a038f9..4cdbfce63 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -267,7 +267,7 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
         if (new_path == path or
                 (self._nopostoverwrites and os.path.exists(encodeFilename(new_path)))):
             self._downloader.to_screen('[youtube] Post-process file %s exists, skipping' % new_path)
-            return True, information
+            return [], information
 
         try:
             self._downloader.to_screen('[' + self.basename + '] Destination: ' + new_path)
@@ -285,7 +285,7 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
                 errnote='Cannot update utime of audio file')
 
         information['filepath'] = new_path
-        return False, information
+        return [path], information
 
 
 class FFmpegVideoConvertorPP(FFmpegPostProcessor):
@@ -299,13 +299,13 @@ class FFmpegVideoConvertorPP(FFmpegPostProcessor):
         outpath = prefix + sep + self._preferedformat
         if information['ext'] == self._preferedformat:
             self._downloader.to_screen('[ffmpeg] Not converting video file %s - already is in target format %s' % (path, self._preferedformat))
-            return True, information
+            return [], information
         self._downloader.to_screen('[' + 'ffmpeg' + '] Converting video from %s to %s, Destination: ' % (information['ext'], self._preferedformat) + outpath)
         self.run_ffmpeg(path, outpath, [])
         information['filepath'] = outpath
         information['format'] = self._preferedformat
         information['ext'] = self._preferedformat
-        return False, information
+        return [path], information
 
 
 class FFmpegEmbedSubtitlePP(FFmpegPostProcessor):
@@ -505,11 +505,11 @@ class FFmpegEmbedSubtitlePP(FFmpegPostProcessor):
     def run(self, information):
         if information['ext'] != 'mp4':
             self._downloader.to_screen('[ffmpeg] Subtitles can only be embedded in mp4 files')
-            return True, information
+            return [], information
         subtitles = information.get('requested_subtitles')
         if not subtitles:
             self._downloader.to_screen('[ffmpeg] There aren\'t any subtitles to embed')
-            return True, information
+            return [], information
 
         sub_langs = list(subtitles.keys())
         filename = information['filepath']
@@ -535,7 +535,7 @@ class FFmpegEmbedSubtitlePP(FFmpegPostProcessor):
         os.remove(encodeFilename(filename))
         os.rename(encodeFilename(temp_filename), encodeFilename(filename))
 
-        return True, information
+        return [], information
 
 
 class FFmpegMetadataPP(FFmpegPostProcessor):
@@ -561,7 +561,7 @@ class FFmpegMetadataPP(FFmpegPostProcessor):
 
         if not metadata:
             self._downloader.to_screen('[ffmpeg] There isn\'t any metadata to add')
-            return True, info
+            return [], info
 
         filename = info['filepath']
         temp_filename = prepend_extension(filename, 'temp')
@@ -578,7 +578,7 @@ class FFmpegMetadataPP(FFmpegPostProcessor):
         self.run_ffmpeg(filename, temp_filename, options)
         os.remove(encodeFilename(filename))
         os.rename(encodeFilename(temp_filename), encodeFilename(filename))
-        return True, info
+        return [], info
 
 
 class FFmpegMergerPP(FFmpegPostProcessor):
@@ -587,7 +587,7 @@ class FFmpegMergerPP(FFmpegPostProcessor):
         args = ['-c', 'copy', '-map', '0:v:0', '-map', '1:a:0']
         self._downloader.to_screen('[ffmpeg] Merging formats into "%s"' % filename)
         self.run_ffmpeg_multiple_files(info['__files_to_merge'], filename, args)
-        return True, info
+        return [], info
 
 
 class FFmpegAudioFixPP(FFmpegPostProcessor):
@@ -602,14 +602,14 @@ class FFmpegAudioFixPP(FFmpegPostProcessor):
         os.remove(encodeFilename(filename))
         os.rename(encodeFilename(temp_filename), encodeFilename(filename))
 
-        return True, info
+        return [], info
 
 
 class FFmpegFixupStretchedPP(FFmpegPostProcessor):
     def run(self, info):
         stretched_ratio = info.get('stretched_ratio')
         if stretched_ratio is None or stretched_ratio == 1:
-            return True, info
+            return [], info
 
         filename = info['filepath']
         temp_filename = prepend_extension(filename, 'temp')
@@ -621,13 +621,13 @@ class FFmpegFixupStretchedPP(FFmpegPostProcessor):
         os.remove(encodeFilename(filename))
         os.rename(encodeFilename(temp_filename), encodeFilename(filename))
 
-        return True, info
+        return [], info
 
 
 class FFmpegFixupM4aPP(FFmpegPostProcessor):
     def run(self, info):
         if info.get('container') != 'm4a_dash':
-            return True, info
+            return [], info
 
         filename = info['filepath']
         temp_filename = prepend_extension(filename, 'temp')
@@ -639,7 +639,7 @@ class FFmpegFixupM4aPP(FFmpegPostProcessor):
         os.remove(encodeFilename(filename))
         os.rename(encodeFilename(temp_filename), encodeFilename(filename))
 
-        return True, info
+        return [], info
 
 
 class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor):
@@ -656,7 +656,7 @@ class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor):
             new_format = 'webvtt'
         if subs is None:
             self._downloader.to_screen('[ffmpeg] There aren\'t any subtitles to convert')
-            return True, info
+            return [], info
         self._downloader.to_screen('[ffmpeg] Converting subtitles')
         for lang, sub in subs.items():
             ext = sub['ext']
@@ -676,4 +676,4 @@ class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor):
                     'data': f.read(),
                 }
 
-        return True, info
+        return [], info
diff --git a/youtube_dl/postprocessor/metadatafromtitle.py b/youtube_dl/postprocessor/metadatafromtitle.py
index 5019433d3..a56077f20 100644
--- a/youtube_dl/postprocessor/metadatafromtitle.py
+++ b/youtube_dl/postprocessor/metadatafromtitle.py
@@ -44,4 +44,4 @@ class MetadataFromTitlePP(PostProcessor):
             info[attribute] = value
             self._downloader.to_screen('[fromtitle] parsed ' + attribute + ': ' + value)
 
-        return True, info
+        return [], info
diff --git a/youtube_dl/postprocessor/xattrpp.py b/youtube_dl/postprocessor/xattrpp.py
index f6c63fe97..0cba99fc3 100644
--- a/youtube_dl/postprocessor/xattrpp.py
+++ b/youtube_dl/postprocessor/xattrpp.py
@@ -105,8 +105,8 @@ class XAttrMetadataPP(PostProcessor):
                     byte_value = value.encode('utf-8')
                     write_xattr(filename, xattrname, byte_value)
 
-            return True, info
+            return [], info
 
         except (subprocess.CalledProcessError, OSError):
             self._downloader.report_error("This filesystem doesn't support extended attributes. (You may have to enable them in your /etc/fstab)")
-            return False, info
+            return [], info

From 14523ed9695975704fae441a3518daf8b0e382fe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 18 Apr 2015 11:44:42 +0200
Subject: [PATCH 0980/6219] FFmpegEmbedSubtitlePP: remove the subtitle files if
 '--keep-video' is not given (closes #5435)

---
 youtube_dl/postprocessor/ffmpeg.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 4cdbfce63..852dc3e44 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -513,7 +513,8 @@ class FFmpegEmbedSubtitlePP(FFmpegPostProcessor):
 
         sub_langs = list(subtitles.keys())
         filename = information['filepath']
-        input_files = [filename] + [subtitles_filename(filename, lang, sub_info['ext']) for lang, sub_info in subtitles.items()]
+        sub_filenames = [subtitles_filename(filename, lang, sub_info['ext']) for lang, sub_info in subtitles.items()]
+        input_files = [filename] + sub_filenames
 
         opts = [
             '-map', '0',
@@ -535,7 +536,7 @@ class FFmpegEmbedSubtitlePP(FFmpegPostProcessor):
         os.remove(encodeFilename(filename))
         os.rename(encodeFilename(temp_filename), encodeFilename(filename))
 
-        return [], information
+        return sub_filenames, information
 
 
 class FFmpegMetadataPP(FFmpegPostProcessor):

From d47aeb2252aa0289199bc3ea9bb533997051ec0c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 18 Apr 2015 11:52:36 +0200
Subject: [PATCH 0981/6219] FFmpegMergerPP: use the new system for specifying
 which files can be delete

---
 youtube_dl/YoutubeDL.py            | 2 +-
 youtube_dl/postprocessor/ffmpeg.py | 9 ++-------
 2 files changed, 3 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 8d8b146b2..944571881 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1361,7 +1361,7 @@ class YoutubeDL(object):
                 if info_dict.get('requested_formats') is not None:
                     downloaded = []
                     success = True
-                    merger = FFmpegMergerPP(self, not self.params.get('keepvideo'))
+                    merger = FFmpegMergerPP(self)
                     if not merger.available:
                         postprocessors = []
                         self.report_warning('You have requested multiple '
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 852dc3e44..4d619236e 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -28,9 +28,8 @@ class FFmpegPostProcessorError(PostProcessingError):
 
 
 class FFmpegPostProcessor(PostProcessor):
-    def __init__(self, downloader=None, deletetempfiles=False):
+    def __init__(self, downloader=None):
         PostProcessor.__init__(self, downloader)
-        self._deletetempfiles = deletetempfiles
         self._determine_executables()
 
     def check_version(self):
@@ -148,10 +147,6 @@ class FFmpegPostProcessor(PostProcessor):
             raise FFmpegPostProcessorError(msg)
         self.try_utime(out_path, oldest_mtime, oldest_mtime)
 
-        if self._deletetempfiles:
-            for ipath in input_paths:
-                os.remove(ipath)
-
     def run_ffmpeg(self, path, out_path, opts):
         self.run_ffmpeg_multiple_files([path], out_path, opts)
 
@@ -588,7 +583,7 @@ class FFmpegMergerPP(FFmpegPostProcessor):
         args = ['-c', 'copy', '-map', '0:v:0', '-map', '1:a:0']
         self._downloader.to_screen('[ffmpeg] Merging formats into "%s"' % filename)
         self.run_ffmpeg_multiple_files(info['__files_to_merge'], filename, args)
-        return [], info
+        return info['__files_to_merge'], info
 
 
 class FFmpegAudioFixPP(FFmpegPostProcessor):

From cc36e2295a19796380e4a4f2abf618e727617efe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 18 Apr 2015 13:27:35 +0200
Subject: [PATCH 0982/6219] [ign] Fix extraction of some videos in articles

Give higher preference to the hero-poster regex because some articles may contain other videos
---
 youtube_dl/extractor/ign.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/ign.py b/youtube_dl/extractor/ign.py
index 3aade9e74..bf2d2041b 100644
--- a/youtube_dl/extractor/ign.py
+++ b/youtube_dl/extractor/ign.py
@@ -61,7 +61,7 @@ class IGNIE(InfoExtractor):
         },
         {
             'url': 'http://www.ign.com/articles/2014/08/15/rewind-theater-wild-trailer-gamescom-2014?watch',
-            'md5': '4e9a0bda1e5eebd31ddcf86ec0b9b3c7',
+            'md5': '618fedb9c901fd086f6f093564ef8558',
             'info_dict': {
                 'id': '078fdd005f6d3c02f63d795faa1b984f',
                 'ext': 'mp4',
@@ -77,10 +77,10 @@ class IGNIE(InfoExtractor):
     def _find_video_id(self, webpage):
         res_id = [
             r'"video_id"\s*:\s*"(.*?)"',
+            r'class="hero-poster[^"]*?"[^>]*id="(.+?)"',
             r'data-video-id="(.+?)"',
             r'<object id="vid_(.+?)"',
             r'<meta name="og:image" content=".*/(.+?)-(.+?)/.+.jpg"',
-            r'class="hero-poster[^"]*?"[^>]*id="(.+?)"',
         ]
         return self._search_regex(res_id, webpage, 'video id')
 

From 8f4e8bf28080a7d6e969fea66e59afd3a1b2f085 Mon Sep 17 00:00:00 2001
From: hedii <hedi.chaibs@gmail.com>
Date: Sat, 18 Apr 2015 15:40:40 +0200
Subject: [PATCH 0983/6219] Update wat.py

line 116, modify 'Downloding' to 'Downloading'.
It looks like nothing, but it is very annoying when youtube-dl command's output is parsed to find progress on a php (or other language) website for example.
---
 youtube_dl/extractor/wat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/wat.py b/youtube_dl/extractor/wat.py
index bf9e40bad..affcc52f6 100644
--- a/youtube_dl/extractor/wat.py
+++ b/youtube_dl/extractor/wat.py
@@ -113,7 +113,7 @@ class WatIE(InfoExtractor):
             video_url = self._download_webpage(
                 'http://www.wat.tv/get%s?token=%s&getURL=1&country=%s' % (webid, compute_token(webid), country),
                 real_id,
-                'Downloding %s video URL' % fmt[0],
+                'Downloading %s video URL' % fmt[0],
                 'Failed to download %s video URL' % fmt[0],
                 False)
             if not video_url:

From fec2d97ca2f1aa8b64c24b28d8c63cab052e9db4 Mon Sep 17 00:00:00 2001
From: Jeff Buchbinder <jeff@ourexchange.net>
Date: Thu, 15 Jan 2015 21:28:57 -0500
Subject: [PATCH 0984/6219] Add megavideoz.eu support.

---
 youtube_dl/extractor/__init__.py     |  1 +
 youtube_dl/extractor/megavideozeu.py | 39 ++++++++++++++++++++++++++++
 2 files changed, 40 insertions(+)
 create mode 100644 youtube_dl/extractor/megavideozeu.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 9e9e20589..6657695cb 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -274,6 +274,7 @@ from .macgamestore import MacGameStoreIE
 from .mailru import MailRuIE
 from .malemotion import MalemotionIE
 from .mdr import MDRIE
+from .megavideozeu import MegavideozeuIE
 from .metacafe import MetacafeIE
 from .metacritic import MetacriticIE
 from .mgoon import MgoonIE
diff --git a/youtube_dl/extractor/megavideozeu.py b/youtube_dl/extractor/megavideozeu.py
new file mode 100644
index 000000000..e77b5f734
--- /dev/null
+++ b/youtube_dl/extractor/megavideozeu.py
@@ -0,0 +1,39 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    parse_filesize,
+    unified_strdate,
+)
+
+
+class MegavideozeuIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?megavideoz\.eu/video/(?P<id>.*)(?:.*)'
+
+    def _real_extract(self, url):
+        tmp_video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, tmp_video_id)
+
+        config_php = self._html_search_regex(
+            r'var cnf = \'([^\']+)\'', webpage, 'config.php url')
+
+	configpage = self._download_webpage(config_php, tmp_video_id)
+
+        video_id = self._html_search_regex(
+            r'<mediaid>([^<]+)', configpage, 'video id')
+        video_url = self._html_search_regex(
+            r'<file>([^<]+)', configpage, 'video URL')
+        title = self._html_search_regex(
+            r'<title><!\[CDATA\[([^\]]+)', configpage, 'title')
+        duration = int_or_none(self._html_search_regex(
+            r'<duration>([0-9]+)', configpage, 'duration', fatal=False))
+
+        return {
+            'id': video_id,
+            'url': video_url,
+            'title': title,
+            'duration': duration
+        }

From f32cb5cb147e8e6f05625ed6b5880eb566101f03 Mon Sep 17 00:00:00 2001
From: Jeff Buchbinder <jeff@ourexchange.net>
Date: Fri, 17 Apr 2015 11:25:01 -0400
Subject: [PATCH 0985/6219] [megavideoez] Add working test

---
 youtube_dl/extractor/megavideozeu.py | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/megavideozeu.py b/youtube_dl/extractor/megavideozeu.py
index e77b5f734..ee26b0f2e 100644
--- a/youtube_dl/extractor/megavideozeu.py
+++ b/youtube_dl/extractor/megavideozeu.py
@@ -11,6 +11,18 @@ from ..utils import (
 
 class MegavideozeuIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?megavideoz\.eu/video/(?P<id>.*)(?:.*)'
+    _TESTS = [
+        {
+            'url': 'http://megavideoz.eu/video/WM6UB919XMXH/SMPTE-Universal-Film-Leader',
+            'info_dict': {
+                'id': '48723',
+                'ext': 'mp4',
+                'duration': '10',
+                'title': 'SMPTE Universal Film Leader',
+            }
+        }
+    ]
+
 
     def _real_extract(self, url):
         tmp_video_id = self._match_id(url)
@@ -29,7 +41,7 @@ class MegavideozeuIE(InfoExtractor):
         title = self._html_search_regex(
             r'<title><!\[CDATA\[([^\]]+)', configpage, 'title')
         duration = int_or_none(self._html_search_regex(
-            r'<duration>([0-9]+)', configpage, 'duration', fatal=False))
+            r'<duration>([0-9\.]+)', configpage, 'duration', fatal=False))
 
         return {
             'id': video_id,

From 31f224008e39b629d793a8fc3f8fafc7a4baf417 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 19 Apr 2015 04:07:45 +0600
Subject: [PATCH 0986/6219] [megavideozeu] Simplify (Closes #5454)

---
 youtube_dl/extractor/megavideozeu.py | 59 ++++++++++++++--------------
 1 file changed, 30 insertions(+), 29 deletions(-)

diff --git a/youtube_dl/extractor/megavideozeu.py b/youtube_dl/extractor/megavideozeu.py
index ee26b0f2e..f98080caa 100644
--- a/youtube_dl/extractor/megavideozeu.py
+++ b/youtube_dl/extractor/megavideozeu.py
@@ -1,51 +1,52 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..utils import (
-    int_or_none,
-    parse_filesize,
-    unified_strdate,
+    float_or_none,
+    xpath_text,
 )
 
 
 class MegavideozeuIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?megavideoz\.eu/video/(?P<id>.*)(?:.*)'
-    _TESTS = [
-        {
-            'url': 'http://megavideoz.eu/video/WM6UB919XMXH/SMPTE-Universal-Film-Leader',
-            'info_dict': {
-                'id': '48723',
-                'ext': 'mp4',
-                'duration': '10',
-                'title': 'SMPTE Universal Film Leader',
-            }
+    _VALID_URL = r'https?://(?:www\.)?megavideoz\.eu/video/(?P<id>[^/]+)(?:/(?P<display_id>[^/]+))?'
+    _TEST = {
+        'url': 'http://megavideoz.eu/video/WM6UB919XMXH/SMPTE-Universal-Film-Leader',
+        'info_dict': {
+            'id': '48723',
+            'display_id': 'SMPTE-Universal-Film-Leader',
+            'ext': 'mp4',
+            'title': 'SMPTE Universal Film Leader',
+            'thumbnail': 're:https?://.*?\.jpg',
+            'duration': 10.93,
         }
-    ]
-
+    }
 
     def _real_extract(self, url):
-        tmp_video_id = self._match_id(url)
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        display_id = mobj.group('display_id') or video_id
 
-        webpage = self._download_webpage(url, tmp_video_id)
+        webpage = self._download_webpage(url, display_id)
 
-        config_php = self._html_search_regex(
-            r'var cnf = \'([^\']+)\'', webpage, 'config.php url')
+        config = self._download_xml(
+            self._search_regex(
+                r"var\s+cnf\s*=\s*'([^']+)'", webpage, 'cnf url'),
+            display_id)
 
-	configpage = self._download_webpage(config_php, tmp_video_id)
-
-        video_id = self._html_search_regex(
-            r'<mediaid>([^<]+)', configpage, 'video id')
-        video_url = self._html_search_regex(
-            r'<file>([^<]+)', configpage, 'video URL')
-        title = self._html_search_regex(
-            r'<title><!\[CDATA\[([^\]]+)', configpage, 'title')
-        duration = int_or_none(self._html_search_regex(
-            r'<duration>([0-9\.]+)', configpage, 'duration', fatal=False))
+        video_url = xpath_text(config, './file', 'video url', fatal=True)
+        title = xpath_text(config, './title', 'title', fatal=True)
+        thumbnail = xpath_text(config, './image', 'thumbnail')
+        duration = float_or_none(xpath_text(config, './duration', 'duration'))
+        video_id = xpath_text(config, './mediaid', 'video id') or video_id
 
         return {
             'id': video_id,
+            'display_id': display_id,
             'url': video_url,
             'title': title,
+            'thumbnail': thumbnail,
             'duration': duration
         }

From cc9b9df0b680da1cf610da57fbf397d8ebd75e74 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 19 Apr 2015 04:08:29 +0600
Subject: [PATCH 0987/6219] [megavideozeu] Rename extractor

---
 youtube_dl/extractor/__init__.py                        | 2 +-
 youtube_dl/extractor/{megavideozeu.py => megavideoz.py} | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)
 rename youtube_dl/extractor/{megavideozeu.py => megavideoz.py} (97%)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 6657695cb..1dabea7d6 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -274,7 +274,7 @@ from .macgamestore import MacGameStoreIE
 from .mailru import MailRuIE
 from .malemotion import MalemotionIE
 from .mdr import MDRIE
-from .megavideozeu import MegavideozeuIE
+from .megavideoz import MegaVideozIE
 from .metacafe import MetacafeIE
 from .metacritic import MetacriticIE
 from .mgoon import MgoonIE
diff --git a/youtube_dl/extractor/megavideozeu.py b/youtube_dl/extractor/megavideoz.py
similarity index 97%
rename from youtube_dl/extractor/megavideozeu.py
rename to youtube_dl/extractor/megavideoz.py
index f98080caa..d80f3633e 100644
--- a/youtube_dl/extractor/megavideozeu.py
+++ b/youtube_dl/extractor/megavideoz.py
@@ -10,7 +10,7 @@ from ..utils import (
 )
 
 
-class MegavideozeuIE(InfoExtractor):
+class MegaVideozIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?megavideoz\.eu/video/(?P<id>[^/]+)(?:/(?P<display_id>[^/]+))?'
     _TEST = {
         'url': 'http://megavideoz.eu/video/WM6UB919XMXH/SMPTE-Universal-Film-Leader',

From 6e218b3f9a57f42756aa3016fa9dd530fb58c452 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 19 Apr 2015 04:09:01 +0600
Subject: [PATCH 0988/6219] [megavideoz] Check non-existing videos

---
 youtube_dl/extractor/megavideoz.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/megavideoz.py b/youtube_dl/extractor/megavideoz.py
index d80f3633e..a15b6a32a 100644
--- a/youtube_dl/extractor/megavideoz.py
+++ b/youtube_dl/extractor/megavideoz.py
@@ -5,6 +5,7 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
+    ExtractorError,
     float_or_none,
     xpath_text,
 )
@@ -31,6 +32,9 @@ class MegaVideozIE(InfoExtractor):
 
         webpage = self._download_webpage(url, display_id)
 
+        if '>Video Not Found<' in webpage:
+            raise ExtractorError('Video %s does not exist' % video_id, expected=True)
+
         config = self._download_xml(
             self._search_regex(
                 r"var\s+cnf\s*=\s*'([^']+)'", webpage, 'cnf url'),

From 163965d86188e5545caae4b5e33ff25278fee5e7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 19 Apr 2015 04:14:58 +0600
Subject: [PATCH 0989/6219] [megavideoz] Improve non-existing videos check

---
 youtube_dl/extractor/megavideoz.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/megavideoz.py b/youtube_dl/extractor/megavideoz.py
index a15b6a32a..af7ff07ea 100644
--- a/youtube_dl/extractor/megavideoz.py
+++ b/youtube_dl/extractor/megavideoz.py
@@ -32,7 +32,7 @@ class MegaVideozIE(InfoExtractor):
 
         webpage = self._download_webpage(url, display_id)
 
-        if '>Video Not Found<' in webpage:
+        if any(p in webpage for p in ('>Video Not Found<', '>404 Error<')):
             raise ExtractorError('Video %s does not exist' % video_id, expected=True)
 
         config = self._download_xml(

From 880ee801cf8a67643cd0a1130ea46cf3be046bf1 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 19 Apr 2015 19:08:37 +0800
Subject: [PATCH 0990/6219] [tests] Allow multi_video to be tested as playlists

---
 test/helper.py        | 2 +-
 test/test_download.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/helper.py b/test/helper.py
index 12afdf184..e1129e58f 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -150,7 +150,7 @@ def expect_info_dict(self, got_dict, expected_dict):
                              'invalid value for field %s, expected %r, got %r' % (info_field, expected, got))
 
     # Check for the presence of mandatory fields
-    if got_dict.get('_type') != 'playlist':
+    if got_dict.get('_type') not in ('playlist', 'multi_video'):
         for key in ('id', 'url', 'title', 'ext'):
             self.assertTrue(got_dict.get(key), 'Missing mandatory field %s' % key)
     # Check for mandatory fields that are automatically set by YoutubeDL
diff --git a/test/test_download.py b/test/test_download.py
index 6a149ae4f..1110357a7 100644
--- a/test/test_download.py
+++ b/test/test_download.py
@@ -153,7 +153,7 @@ def generator(test_case):
                     break
 
             if is_playlist:
-                self.assertEqual(res_dict['_type'], 'playlist')
+                self.assertTrue(res_dict['_type'] in ['playlist', 'multi_video'])
                 self.assertTrue('entries' in res_dict)
                 expect_info_dict(self, res_dict, test_case.get('info_dict', {}))
 

From 8b0e8990c241789d7deb1f5e27fc1aea00b4fa5e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 19 Apr 2015 19:12:23 +0800
Subject: [PATCH 0991/6219] [miomio] Replace the slow test case

MioMio_1 takes about 25~35 seconds on information retrieval
---
 youtube_dl/extractor/miomio.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/youtube_dl/extractor/miomio.py b/youtube_dl/extractor/miomio.py
index d41195a96..a784fc5fb 100644
--- a/youtube_dl/extractor/miomio.py
+++ b/youtube_dl/extractor/miomio.py
@@ -31,6 +31,14 @@ class MioMioIE(InfoExtractor):
             'title': '《动漫同人插画绘制》',
         },
         'playlist_mincount': 86,
+        'skip': 'This video takes time too long for retrieving the URL',
+    }, {
+        'url': 'http://www.miomio.tv/watch/cc173113/',
+        'info_dict': {
+            'id': '173113',
+            'title': 'The New Macbook 2015 上手试玩与简评'
+        },
+        'playlist_mincount': 2,
     }]
 
     def _real_extract(self, url):

From f158799bbe72e1fe99ec057cc968d3ec874fb1dd Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 19 Apr 2015 19:19:44 +0800
Subject: [PATCH 0992/6219] [Sohu] Fix title extraction

---
 youtube_dl/extractor/sohu.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/sohu.py b/youtube_dl/extractor/sohu.py
index 11edf616a..f8a4840f7 100644
--- a/youtube_dl/extractor/sohu.py
+++ b/youtube_dl/extractor/sohu.py
@@ -47,6 +47,7 @@ class SohuIE(InfoExtractor):
         'url': 'http://my.tv.sohu.com/pl/8384802/78910339.shtml',
         'info_dict': {
             'id': '78910339',
+            'title': '【神探苍实战秘籍】第13期 战争之影 赫卡里姆',
         },
         'playlist': [{
             'md5': 'bdbfb8f39924725e6589c146bc1883ad',
@@ -110,7 +111,7 @@ class SohuIE(InfoExtractor):
 
         webpage = self._download_webpage(url, video_id)
 
-        title = self._og_search_title(webpage)
+        title = re.sub(r' - 搜狐视频$', '', self._og_search_title(webpage))
 
         vid = self._html_search_regex(
             r'var vid ?= ?["\'](\d+)["\']',
@@ -172,9 +173,10 @@ class SohuIE(InfoExtractor):
             info['id'] = video_id
         else:
             info = {
-                '_type': 'playlist',
+                '_type': 'multi_video',
                 'entries': playlist,
                 'id': video_id,
+                'title': title,
             }
 
         return info

From 5b5fbc0867f0eb73416c10c9d692fceee92b5766 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 19 Apr 2015 16:56:22 +0200
Subject: [PATCH 0993/6219] Detect already merged videos

Without the '--keep-video' option the two files would be downloaded again and even using the option, ffmpeg would be run again, which for some videos can take a long time.
We use a temporary file with ffmpeg so that the final file only exists if it success
---
 youtube_dl/YoutubeDL.py            | 25 +++++++++++++++----------
 youtube_dl/postprocessor/ffmpeg.py |  4 +++-
 2 files changed, 18 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 944571881..28cf1662e 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1369,16 +1369,21 @@ class YoutubeDL(object):
                                             ' The formats won\'t be merged')
                     else:
                         postprocessors = [merger]
-                    for f in info_dict['requested_formats']:
-                        new_info = dict(info_dict)
-                        new_info.update(f)
-                        fname = self.prepare_filename(new_info)
-                        fname = prepend_extension(fname, 'f%s' % f['format_id'])
-                        downloaded.append(fname)
-                        partial_success = dl(fname, new_info)
-                        success = success and partial_success
-                    info_dict['__postprocessors'] = postprocessors
-                    info_dict['__files_to_merge'] = downloaded
+                    if os.path.exists(encodeFilename(filename)):
+                        self.to_screen(
+                            '[download] %s has already been downloaded and '
+                            'merged' % filename)
+                    else:
+                        for f in info_dict['requested_formats']:
+                            new_info = dict(info_dict)
+                            new_info.update(f)
+                            fname = self.prepare_filename(new_info)
+                            fname = prepend_extension(fname, 'f%s' % f['format_id'])
+                            downloaded.append(fname)
+                            partial_success = dl(fname, new_info)
+                            success = success and partial_success
+                        info_dict['__postprocessors'] = postprocessors
+                        info_dict['__files_to_merge'] = downloaded
                 else:
                     # Just a single file
                     success = dl(filename, info_dict)
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 4d619236e..df6fb6665 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -580,9 +580,11 @@ class FFmpegMetadataPP(FFmpegPostProcessor):
 class FFmpegMergerPP(FFmpegPostProcessor):
     def run(self, info):
         filename = info['filepath']
+        temp_filename = prepend_extension(filename, 'temp')
         args = ['-c', 'copy', '-map', '0:v:0', '-map', '1:a:0']
         self._downloader.to_screen('[ffmpeg] Merging formats into "%s"' % filename)
-        self.run_ffmpeg_multiple_files(info['__files_to_merge'], filename, args)
+        self.run_ffmpeg_multiple_files(info['__files_to_merge'], temp_filename, args)
+        os.rename(encodeFilename(temp_filename), encodeFilename(filename))
         return info['__files_to_merge'], info
 
 

From feccf29c876869f44a9c983977371073b9801a51 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Apr 2015 01:14:10 +0600
Subject: [PATCH 0994/6219] [YoutubeDL] Make `bestvideo+bestaudio/best` default
 format when merger is available

---
 youtube_dl/YoutubeDL.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 28cf1662e..e5d497b3f 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1091,7 +1091,11 @@ class YoutubeDL(object):
 
         req_format = self.params.get('format')
         if req_format is None:
-            req_format = 'best'
+            req_format_list = []
+            if info_dict['extractor'] == 'youtube' and FFmpegMergerPP(self).available:
+                req_format_list.append('bestvideo+bestaudio')
+            req_format_list.append('best')
+            req_format = '/'.join(req_format_list)
         formats_to_download = []
         if req_format == 'all':
             formats_to_download = formats

From 81cd954a512e48fb5cccc3159cd0581088bff0b3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Apr 2015 03:00:35 +0600
Subject: [PATCH 0995/6219] [YoutubeDL] Merge incompatible formats into mkv
 (#5456)

---
 youtube_dl/YoutubeDL.py | 24 +++++++++++++++++++++++-
 1 file changed, 23 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index e5d497b3f..5dd9d2430 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1373,12 +1373,34 @@ class YoutubeDL(object):
                                             ' The formats won\'t be merged')
                     else:
                         postprocessors = [merger]
+
+                    def compatible_formats(formats):
+                        video, audio = formats
+                        # Check extension
+                        video_ext, audio_ext = audio.get('ext'), video.get('ext')
+                        if video_ext and audio_ext:
+                            COMPATIBLE_EXTS = (
+                                ('mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v'),
+                                ('webm')
+                            )
+                            for exts in COMPATIBLE_EXTS:
+                                if video_ext in exts and audio_ext in exts:
+                                    return True
+                        # TODO: Check acodec/vcodec
+                        return False
+
+                    requested_formats = info_dict['requested_formats']
+                    # Merge incompatible formats into mkv
+                    if not compatible_formats(requested_formats):
+                        filename = os.path.splitext(filename)[0] + '.mkv'
+                        self.report_warning('You have requested formats uncompatible for merge. '
+                                            'The formats will be merged into mkv')
                     if os.path.exists(encodeFilename(filename)):
                         self.to_screen(
                             '[download] %s has already been downloaded and '
                             'merged' % filename)
                     else:
-                        for f in info_dict['requested_formats']:
+                        for f in requested_formats:
                             new_info = dict(info_dict)
                             new_info.update(f)
                             fname = self.prepare_filename(new_info)

From 70947ea7b13d22a55756dc2b6d086e058edd91b1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Apr 2015 03:07:59 +0600
Subject: [PATCH 0996/6219] [parameters.json] Set default `format` parameter to
 `best`

---
 test/parameters.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/parameters.json b/test/parameters.json
index cbff9bd16..48b5a062e 100644
--- a/test/parameters.json
+++ b/test/parameters.json
@@ -7,7 +7,7 @@
     "forcethumbnail": false, 
     "forcetitle": false, 
     "forceurl": false, 
-    "format": null, 
+    "format": "best",
     "format_limit": null, 
     "ignoreerrors": false, 
     "listformats": null, 

From c0dea0a782a1035225ad25d556e398e2909f62db Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 19 Apr 2015 22:33:52 +0600
Subject: [PATCH 0997/6219] [YoutubeDL] Respect explicit
 `--merge-format-output` for uncompatible formats as well

---
 youtube_dl/YoutubeDL.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 5dd9d2430..3bb350e2a 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1390,8 +1390,7 @@ class YoutubeDL(object):
                         return False
 
                     requested_formats = info_dict['requested_formats']
-                    # Merge incompatible formats into mkv
-                    if not compatible_formats(requested_formats):
+                    if self.params.get('merge_output_format') is None and not compatible_formats(requested_formats):
                         filename = os.path.splitext(filename)[0] + '.mkv'
                         self.report_warning('You have requested formats uncompatible for merge. '
                                             'The formats will be merged into mkv')

From a38050925958c48175bf56a988e9714edef8f12f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 19 Apr 2015 18:37:38 +0200
Subject: [PATCH 0998/6219] Move the documentation for the `--format` option to
 the manpage

It's too big for beeing embedded in the help message and it's easier to edit in the markdown file.
---
 README.md             | 19 +++++++++----------
 youtube_dl/options.py | 27 +--------------------------
 2 files changed, 10 insertions(+), 36 deletions(-)

diff --git a/README.md b/README.md
index caa1478d9..68426b685 100644
--- a/README.md
+++ b/README.md
@@ -5,6 +5,7 @@ youtube-dl - download videos from youtube.com or other video platforms
 - [OPTIONS](#options)
 - [CONFIGURATION](#configuration)
 - [OUTPUT TEMPLATE](#output-template)
+- [FORMAT SELECTION](#format-selection)
 - [VIDEO SELECTION](#video-selection)
 - [FAQ](#faq)
 - [DEVELOPER INSTRUCTIONS](#developer-instructions)
@@ -184,16 +185,7 @@ which means you can modify it, redistribute it or use it however you like.
     --sleep-interval SECONDS         Number of seconds to sleep before each download.
 
 ## Video Format Options:
-    -f, --format FORMAT              Video format code, specify the order of preference using slashes, as in -f 22/17/18 .  Instead of format codes, you can select by
-                                     extension for the extensions aac, m4a, mp3, mp4, ogg, wav, webm. You can also use the special names "best", "bestvideo", "bestaudio",
-                                     "worst".  You can filter the video results by putting a condition in brackets, as in -f "best[height=720]" (or -f "[filesize>10M]").
-                                     This works for filesize, height, width, tbr, abr, vbr, asr, and fps and the comparisons <, <=, >, >=, =, != and for ext, acodec,
-                                     vcodec, container, and protocol and the comparisons =, != . Formats for which the value is not known are excluded unless you put a
-                                     question mark (?) after the operator. You can combine format filters, so  -f "[height <=? 720][tbr>500]" selects up to 720p videos
-                                     (or videos where the height is not known) with a bitrate of at least 500 KBit/s. By default, youtube-dl will pick the best quality.
-                                     Use commas to download multiple audio formats, such as -f  136/137/mp4/bestvideo,140/m4a/bestaudio. You can merge the video and audio
-                                     of two formats into a single file using -f <video-format>+<audio-format> (requires ffmpeg or avconv), for example -f
-                                     bestvideo+bestaudio.
+    -f, --format FORMAT              Video format code, see the "FORMAT SELECTION" for all the info
     --all-formats                    Download all available video formats
     --prefer-free-formats            Prefer free video formats unless a specific one is requested
     --max-quality FORMAT             Highest quality format to download
@@ -271,6 +263,13 @@ $ youtube-dl --get-filename -o "%(title)s.%(ext)s" BaW_jenozKc --restrict-filena
 youtube-dl_test_video_.mp4          # A simple file name
 ```
 
+# FORMAT SELECTION
+
+By default youtube-dl tries to download the best quality, but sometimes you may want to download other format.
+The simplest case is requesting a specific format, for example `-f 22`. You can get the list of available formats using `--list-formats`, you can also use a file extension (currently it supports aac, m4a, mp3, mp4, ogg, wav, webm) or the special names `best`, `bestvideo`, `bestaudio` and `worst`.
+
+If you are want to download multiple videos and they don't have the same formats available, you can specify the order of preference using slashes, as in `-f 22/17/18`. You can also filter the video results by putting a condition in brackets, as in `-f "best[height=720]"` (or `-f "[filesize>10M]"`).  This works for filesize, height, width, tbr, abr, vbr, asr, and fps and the comparisons <, <=, >, >=, =, != and for ext, acodec, vcodec, container, and protocol and the comparisons =, != . Formats for which the value is not known are excluded unless you put a question mark (?) after the operator. You can combine format filters, so  `-f "[height <=? 720][tbr>500]"` selects up to 720p videos (or videos where the height is not known) with a bitrate of at least 500 KBit/s. Use commas to download multiple formats, such as `-f 136/137/mp4/bestvideo,140/m4a/bestaudio`. You can merge the video and audio of two formats into a single file using `-f <video-format>+<audio-format>` (requires ffmpeg or avconv), for example `-f bestvideo+bestaudio`.
+
 # VIDEO SELECTION
 
 Videos can be filtered by their upload date using the options `--date`, `--datebefore` or `--dateafter`, they accept dates in two formats:
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 11603f60d..39c38c980 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -322,32 +322,7 @@ def parseOpts(overrideArguments=None):
     video_format.add_option(
         '-f', '--format',
         action='store', dest='format', metavar='FORMAT', default=None,
-        help=(
-            'Video format code, specify the order of preference using'
-            ' slashes, as in -f 22/17/18 . '
-            ' Instead of format codes, you can select by extension for the '
-            'extensions aac, m4a, mp3, mp4, ogg, wav, webm. '
-            'You can also use the special names "best",'
-            ' "bestvideo", "bestaudio", "worst". '
-            ' You can filter the video results by putting a condition in'
-            ' brackets, as in -f "best[height=720]"'
-            ' (or -f "[filesize>10M]"). '
-            ' This works for filesize, height, width, tbr, abr, vbr, asr, and fps'
-            ' and the comparisons <, <=, >, >=, =, !='
-            ' and for ext, acodec, vcodec, container, and protocol'
-            ' and the comparisons =, != .'
-            ' Formats for which the value is not known are excluded unless you'
-            ' put a question mark (?) after the operator.'
-            ' You can combine format filters, so  '
-            '-f "[height <=? 720][tbr>500]" '
-            'selects up to 720p videos (or videos where the height is not '
-            'known) with a bitrate of at least 500 KBit/s.'
-            ' By default, youtube-dl will pick the best quality.'
-            ' Use commas to download multiple audio formats, such as'
-            ' -f  136/137/mp4/bestvideo,140/m4a/bestaudio.'
-            ' You can merge the video and audio of two formats into a single'
-            ' file using -f <video-format>+<audio-format> (requires ffmpeg or'
-            ' avconv), for example -f bestvideo+bestaudio.'))
+        help='Video format code, see the "FORMAT SELECTION" for all the info')
     video_format.add_option(
         '--all-formats',
         action='store_const', dest='format', const='all',

From 7b071e317bd79c81fac0c529d021da381801c3ea Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 19 Apr 2015 18:52:01 +0200
Subject: [PATCH 0999/6219] README: document bestvideo+bestaudio/best (#5447)

---
 README.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/README.md b/README.md
index 68426b685..7dbe66995 100644
--- a/README.md
+++ b/README.md
@@ -270,6 +270,8 @@ The simplest case is requesting a specific format, for example `-f 22`. You can
 
 If you are want to download multiple videos and they don't have the same formats available, you can specify the order of preference using slashes, as in `-f 22/17/18`. You can also filter the video results by putting a condition in brackets, as in `-f "best[height=720]"` (or `-f "[filesize>10M]"`).  This works for filesize, height, width, tbr, abr, vbr, asr, and fps and the comparisons <, <=, >, >=, =, != and for ext, acodec, vcodec, container, and protocol and the comparisons =, != . Formats for which the value is not known are excluded unless you put a question mark (?) after the operator. You can combine format filters, so  `-f "[height <=? 720][tbr>500]"` selects up to 720p videos (or videos where the height is not known) with a bitrate of at least 500 KBit/s. Use commas to download multiple formats, such as `-f 136/137/mp4/bestvideo,140/m4a/bestaudio`. You can merge the video and audio of two formats into a single file using `-f <video-format>+<audio-format>` (requires ffmpeg or avconv), for example `-f bestvideo+bestaudio`.
 
+youtube-dl uses `-f bestvideo+bestaudio/best` if ffmpeg or avconv are installed (`best` is needed for videos that don't come from YouTube because they don't provide the audio and video in two different files). If you want to only download some dash formats (for example if you are not interested in getting videos with a resolution higher than 1080p), you can add `-f bestvideo[height<=?1080]+bestaudio/best` to your configuration file.
+
 # VIDEO SELECTION
 
 Videos can be filtered by their upload date using the options `--date`, `--datebefore` or `--dateafter`, they accept dates in two formats:

From b524a001d68767298272ef59274c826a6ea6a1b7 Mon Sep 17 00:00:00 2001
From: Quentin Rameau <quinq@quinq.eu.org>
Date: Mon, 20 Apr 2015 06:25:55 +0200
Subject: [PATCH 1000/6219] [bandcamp] fix video_id parsing (fixes #4861)

---
 youtube_dl/extractor/bandcamp.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bandcamp.py b/youtube_dl/extractor/bandcamp.py
index 869294967..505877b77 100644
--- a/youtube_dl/extractor/bandcamp.py
+++ b/youtube_dl/extractor/bandcamp.py
@@ -72,7 +72,7 @@ class BandcampIE(InfoExtractor):
 
         download_link = m_download.group(1)
         video_id = self._search_regex(
-            r'(?ms)var TralbumData = {.*?id: (?P<id>\d+),?$',
+            r'(?ms)var TralbumData = .*?[{,]\s*id: (?P<id>\d+),?$',
             webpage, 'video id')
 
         download_webpage = self._download_webpage(download_link, video_id, 'Downloading free downloads page')

From 3ded7bac166c71f99b8d8036c072552c483d2364 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 20 Apr 2015 21:13:31 +0600
Subject: [PATCH 1001/6219] [extractor/common] Add ability to specify custom
 field preference for `_sort_formats`

---
 youtube_dl/extractor/common.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 7757bf950..71230323c 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -708,7 +708,7 @@ class InfoExtractor(object):
         return self._html_search_meta('twitter:player', html,
                                       'twitter card player')
 
-    def _sort_formats(self, formats):
+    def _sort_formats(self, formats, field_preference=None):
         if not formats:
             raise ExtractorError('No video formats found')
 
@@ -718,6 +718,9 @@ class InfoExtractor(object):
             if not f.get('ext') and 'url' in f:
                 f['ext'] = determine_ext(f['url'])
 
+            if isinstance(field_preference, (list, tuple)):
+                return tuple(f.get(field) if f.get(field) is not None else -1 for field in field_preference)
+
             preference = f.get('preference')
             if preference is None:
                 proto = f.get('protocol')

From 736785ab63104b4d285545e4755fa03ef4496a47 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 20 Apr 2015 21:42:20 +0600
Subject: [PATCH 1002/6219] [ted] Clarify audio/video-only formats

---
 youtube_dl/extractor/ted.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/ted.py b/youtube_dl/extractor/ted.py
index a2dc14c2b..2c8acfef6 100644
--- a/youtube_dl/extractor/ted.py
+++ b/youtube_dl/extractor/ted.py
@@ -194,14 +194,18 @@ class TEDIE(InfoExtractor):
                         'tbr': int_or_none(resource.get('bitrate')),
                     })
             elif format_id == 'hls':
-                formats.extend(self._extract_m3u8_formats(
-                    resources.get('stream'), video_name, 'mp4', m3u8_id=format_id))
+                hls_formats = self._extract_m3u8_formats(
+                    resources.get('stream'), video_name, 'mp4', m3u8_id=format_id)
+                for f in hls_formats:
+                    f['acodec'] = 'none'
+                formats.extend(hls_formats)
 
         audio_download = talk_info.get('audioDownload')
         if audio_download:
             formats.append({
                 'url': audio_download,
                 'format_id': 'audio',
+                'vcodec': 'none',
             })
 
         self._sort_formats(formats)

From cfbee8a431fdc22ff98dc115a59e0a48cace2c0e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 20 Apr 2015 21:42:42 +0600
Subject: [PATCH 1003/6219] [ted] Clarify IE_NAME

---
 youtube_dl/extractor/ted.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/ted.py b/youtube_dl/extractor/ted.py
index 2c8acfef6..e2cbbc121 100644
--- a/youtube_dl/extractor/ted.py
+++ b/youtube_dl/extractor/ted.py
@@ -10,6 +10,7 @@ from ..utils import int_or_none
 
 
 class TEDIE(InfoExtractor):
+    IE_NAME = 'ted'
     _VALID_URL = r'''(?x)
         (?P<proto>https?://)
         (?P<type>www|embed(?:-ssl)?)(?P<urlmain>\.ted\.com/

From 17c8675853f679d3671c1558f6b2f060f2c8e23a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 20 Apr 2015 21:58:29 +0600
Subject: [PATCH 1004/6219] [YoutubeDL] Allow bestvideo+bestaudio/best strategy
 for ted extractor

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 3bb350e2a..b5fddb8e7 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1092,7 +1092,7 @@ class YoutubeDL(object):
         req_format = self.params.get('format')
         if req_format is None:
             req_format_list = []
-            if info_dict['extractor'] == 'youtube' and FFmpegMergerPP(self).available:
+            if info_dict['extractor'] in ['youtube', 'ted'] and FFmpegMergerPP(self).available:
                 req_format_list.append('bestvideo+bestaudio')
             req_format_list.append('best')
             req_format = '/'.join(req_format_list)

From 6728187ac0b1d5d083e8654e35a18d33485f2614 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 20 Apr 2015 21:58:46 +0600
Subject: [PATCH 1005/6219] [YoutubeDL] mp3 is compatible with mp4

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index b5fddb8e7..5b2c3aa38 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1380,7 +1380,7 @@ class YoutubeDL(object):
                         video_ext, audio_ext = audio.get('ext'), video.get('ext')
                         if video_ext and audio_ext:
                             COMPATIBLE_EXTS = (
-                                ('mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v'),
+                                ('mp3', 'mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v'),
                                 ('webm')
                             )
                             for exts in COMPATIBLE_EXTS:

From 0f0b5736dab5e45189d1b35e0b7074585edd057b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 20 Apr 2015 22:01:02 +0600
Subject: [PATCH 1006/6219] [ted] Fix hls audio/video-only formats

---
 youtube_dl/extractor/ted.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ted.py b/youtube_dl/extractor/ted.py
index e2cbbc121..c788feb5d 100644
--- a/youtube_dl/extractor/ted.py
+++ b/youtube_dl/extractor/ted.py
@@ -198,7 +198,10 @@ class TEDIE(InfoExtractor):
                 hls_formats = self._extract_m3u8_formats(
                     resources.get('stream'), video_name, 'mp4', m3u8_id=format_id)
                 for f in hls_formats:
-                    f['acodec'] = 'none'
+                    if not f.get('height'):
+                        f['vcodec'] = 'none'
+                    else:
+                        f['acodec'] = 'none'
                 formats.extend(hls_formats)
 
         audio_download = talk_info.get('audioDownload')

From 14f7abfa71bc71f46a5e3306062b1feccc4ccd26 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 20 Apr 2015 22:04:17 +0600
Subject: [PATCH 1007/6219] [ted] Lower preference for direct audio since it's
 mono

---
 youtube_dl/extractor/ted.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/ted.py b/youtube_dl/extractor/ted.py
index c788feb5d..8aa27d282 100644
--- a/youtube_dl/extractor/ted.py
+++ b/youtube_dl/extractor/ted.py
@@ -210,6 +210,7 @@ class TEDIE(InfoExtractor):
                 'url': audio_download,
                 'format_id': 'audio',
                 'vcodec': 'none',
+                'preference': -0.5,
             })
 
         self._sort_formats(formats)

From 6621ca39a3f03e44648b54bd8ff57814362f5dbc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 20 Apr 2015 22:04:42 +0600
Subject: [PATCH 1008/6219] [ted] Skip hls quality selection format

---
 youtube_dl/extractor/ted.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/ted.py b/youtube_dl/extractor/ted.py
index 8aa27d282..a48d77c30 100644
--- a/youtube_dl/extractor/ted.py
+++ b/youtube_dl/extractor/ted.py
@@ -198,6 +198,8 @@ class TEDIE(InfoExtractor):
                 hls_formats = self._extract_m3u8_formats(
                     resources.get('stream'), video_name, 'mp4', m3u8_id=format_id)
                 for f in hls_formats:
+                    if f.get('format_id') == 'hls-meta':
+                        continue
                     if not f.get('height'):
                         f['vcodec'] = 'none'
                     else:

From bda44f31a1e18d71dc1cc7a2a3a754593ecd63d3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 20 Apr 2015 22:33:35 +0600
Subject: [PATCH 1009/6219] [bambuser] Modernize

---
 youtube_dl/extractor/bambuser.py | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/bambuser.py b/youtube_dl/extractor/bambuser.py
index c193e66ca..12673e766 100644
--- a/youtube_dl/extractor/bambuser.py
+++ b/youtube_dl/extractor/bambuser.py
@@ -35,12 +35,11 @@ class BambuserIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        info_url = ('http://player-c.api.bambuser.com/getVideo.json?'
-                    '&api_key=%s&vid=%s' % (self._API_KEY, video_id))
-        info_json = self._download_webpage(info_url, video_id)
-        info = json.loads(info_json)['result']
+        video_id = self._match_id(url)
+
+        info = self._download_json(
+            'http://player-c.api.bambuser.com/getVideo.json?api_key=%s&vid=%s'
+            % (self._API_KEY, video_id), video_id)['result']
 
         return {
             'id': video_id,

From ae8953409e30ef9d7dd6bc40350fccc2a29ffb4b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 20 Apr 2015 22:35:53 +0600
Subject: [PATCH 1010/6219] [bambuser] Capture and output error message (#5478)

---
 youtube_dl/extractor/bambuser.py | 29 +++++++++++++++++------------
 1 file changed, 17 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/bambuser.py b/youtube_dl/extractor/bambuser.py
index 12673e766..d52302ebc 100644
--- a/youtube_dl/extractor/bambuser.py
+++ b/youtube_dl/extractor/bambuser.py
@@ -1,13 +1,11 @@
 from __future__ import unicode_literals
 
 import re
-import json
 import itertools
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_request,
-)
+from ..compat import compat_urllib_request
+from ..utils import ExtractorError
 
 
 class BambuserIE(InfoExtractor):
@@ -39,17 +37,24 @@ class BambuserIE(InfoExtractor):
 
         info = self._download_json(
             'http://player-c.api.bambuser.com/getVideo.json?api_key=%s&vid=%s'
-            % (self._API_KEY, video_id), video_id)['result']
+            % (self._API_KEY, video_id), video_id)
+
+        error = info.get('error')
+        if error:
+            raise ExtractorError(
+                '%s returned error: %s' % (self.IE_NAME, error), expected=True)
+
+        result = info['result']
 
         return {
             'id': video_id,
-            'title': info['title'],
-            'url': info['url'],
-            'thumbnail': info.get('preview'),
-            'duration': int(info['length']),
-            'view_count': int(info['views_total']),
-            'uploader': info['username'],
-            'uploader_id': info['owner']['uid'],
+            'title': result['title'],
+            'url': result['url'],
+            'thumbnail': result.get('preview'),
+            'duration': int(result['length']),
+            'view_count': int(result['views_total']),
+            'uploader': result['username'],
+            'uploader_id': result['owner']['uid'],
         }
 
 

From edf421611928e48ac4fda778d6de8ea83585019d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 20 Apr 2015 22:46:01 +0600
Subject: [PATCH 1011/6219] [bambuser] Modernize and extract more metadata

---
 youtube_dl/extractor/bambuser.py | 25 +++++++++++++++++++------
 1 file changed, 19 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/bambuser.py b/youtube_dl/extractor/bambuser.py
index d52302ebc..93913c3f4 100644
--- a/youtube_dl/extractor/bambuser.py
+++ b/youtube_dl/extractor/bambuser.py
@@ -4,8 +4,15 @@ import re
 import itertools
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_request
-from ..utils import ExtractorError
+from ..compat import (
+    compat_urllib_request,
+    compat_str,
+)
+from ..utils import (
+    ExtractorError,
+    int_or_none,
+    float_or_none,
+)
 
 
 class BambuserIE(InfoExtractor):
@@ -24,6 +31,9 @@ class BambuserIE(InfoExtractor):
             'duration': 3741,
             'uploader': 'pixelversity',
             'uploader_id': '344706',
+            'timestamp': 1382976692,
+            'upload_date': '20131028',
+            'view_count': int,
         },
         'params': {
             # It doesn't respect the 'Range' header, it would download the whole video
@@ -51,10 +61,13 @@ class BambuserIE(InfoExtractor):
             'title': result['title'],
             'url': result['url'],
             'thumbnail': result.get('preview'),
-            'duration': int(result['length']),
-            'view_count': int(result['views_total']),
-            'uploader': result['username'],
-            'uploader_id': result['owner']['uid'],
+            'duration': int_or_none(result.get('length')),
+            'uploader': result.get('username'),
+            'uploader_id': compat_str(result.get('owner', {}).get('uid')),
+            'timestamp': int_or_none(result.get('created')),
+            'fps': float_or_none(result.get('framerate')),
+            'view_count': int_or_none(result.get('views_total')),
+            'comment_count': int_or_none(result.get('comment_count')),
         }
 
 

From 006ce15a0cfbf79dedd2c59b1e17344b1a130e81 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 20 Apr 2015 23:00:37 +0600
Subject: [PATCH 1012/6219] [bambuser] Add support for authentication (#5478)

---
 youtube_dl/extractor/bambuser.py | 31 +++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/youtube_dl/extractor/bambuser.py b/youtube_dl/extractor/bambuser.py
index 93913c3f4..8dff1d6e3 100644
--- a/youtube_dl/extractor/bambuser.py
+++ b/youtube_dl/extractor/bambuser.py
@@ -5,6 +5,7 @@ import itertools
 
 from .common import InfoExtractor
 from ..compat import (
+    compat_urllib_parse,
     compat_urllib_request,
     compat_str,
 )
@@ -19,6 +20,8 @@ class BambuserIE(InfoExtractor):
     IE_NAME = 'bambuser'
     _VALID_URL = r'https?://bambuser\.com/v/(?P<id>\d+)'
     _API_KEY = '005f64509e19a868399060af746a00aa'
+    _LOGIN_URL = 'https://bambuser.com/user'
+    _NETRC_MACHINE = 'bambuser'
 
     _TEST = {
         'url': 'http://bambuser.com/v/4050584',
@@ -42,6 +45,34 @@ class BambuserIE(InfoExtractor):
         },
     }
 
+    def _login(self):
+        (username, password) = self._get_login_info()
+        if username is None:
+            return
+
+        login_form = {
+            'form_id': 'user_login',
+            'op': 'Log in',
+            'name': username,
+            'pass': password,
+        }
+
+        request = compat_urllib_request.Request(
+            self._LOGIN_URL, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
+        request.add_header('Referer', self._LOGIN_URL)
+        response = self._download_webpage(
+            request, None, 'Logging in as %s' % username)
+
+        login_error = self._html_search_regex(
+            r'(?s)<div class="messages error">(.+?)</div>',
+            response, 'login error', default=None)
+        if login_error:
+            raise ExtractorError(
+                'Unable to login: %s' % login_error, expected=True)
+
+    def _real_initialize(self):
+        self._login()
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
 

From c6391cd587a26eb3d9bba7296be804f14612e919 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 21 Apr 2015 02:29:56 +0800
Subject: [PATCH 1013/6219] [Senate] Add new extractor (#5302)

---
 youtube_dl/downloader/f4m.py       |   2 +
 youtube_dl/extractor/__init__.py   |   1 +
 youtube_dl/extractor/senateisvp.py | 129 +++++++++++++++++++++++++++++
 3 files changed, 132 insertions(+)
 create mode 100644 youtube_dl/extractor/senateisvp.py

diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index 4ab000d67..b1a858c45 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -389,6 +389,8 @@ class F4mFD(FileDownloader):
             url = base_url + name
             if akamai_pv:
                 url += '?' + akamai_pv.strip(';')
+            if info_dict.get('extra_param_to_segment_url'):
+                url += info_dict.get('extra_param_to_segment_url')
             frag_filename = '%s-%s' % (tmpfilename, name)
             try:
                 success = http_dl.download(frag_filename, {'url': url})
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 1dabea7d6..3bd6d1697 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -447,6 +447,7 @@ from .scivee import SciVeeIE
 from .screencast import ScreencastIE
 from .screencastomatic import ScreencastOMaticIE
 from .screenwavemedia import CinemassacreIE, ScreenwaveMediaIE, TeamFourIE
+from .senateisvp import SenateISVPIE
 from .servingsys import ServingSysIE
 from .sexu import SexuIE
 from .sexykarma import SexyKarmaIE
diff --git a/youtube_dl/extractor/senateisvp.py b/youtube_dl/extractor/senateisvp.py
new file mode 100644
index 000000000..807979d13
--- /dev/null
+++ b/youtube_dl/extractor/senateisvp.py
@@ -0,0 +1,129 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+from .common import InfoExtractor
+from ..utils import ExtractorError
+from ..compat import (
+    compat_parse_qs,
+    compat_urlparse,
+)
+
+
+class SenateISVPIE(InfoExtractor):
+    _COMM_MAP = [
+        ["ag", "76440", "http://ag-f.akamaihd.net"],
+        ["aging", "76442", "http://aging-f.akamaihd.net"],
+        ["approps", "76441", "http://approps-f.akamaihd.net"],
+        ["armed", "76445", "http://armed-f.akamaihd.net"],
+        ["banking", "76446", "http://banking-f.akamaihd.net"],
+        ["budget", "76447", "http://budget-f.akamaihd.net"],
+        ["cecc", "76486", "http://srs-f.akamaihd.net"],
+        ["commerce", "80177", "http://commerce1-f.akamaihd.net"],
+        ["csce", "75229", "http://srs-f.akamaihd.net"],
+        ["dpc", "76590", "http://dpc-f.akamaihd.net"],
+        ["energy", "76448", "http://energy-f.akamaihd.net"],
+        ["epw", "76478", "http://epw-f.akamaihd.net"],
+        ["ethics", "76449", "http://ethics-f.akamaihd.net"],
+        ["finance", "76450", "http://finance-f.akamaihd.net"],
+        ["foreign", "76451", "http://foreign-f.akamaihd.net"],
+        ["govtaff", "76453", "http://govtaff-f.akamaihd.net"],
+        ["help", "76452", "http://help-f.akamaihd.net"],
+        ["indian", "76455", "http://indian-f.akamaihd.net"],
+        ["intel", "76456", "http://intel-f.akamaihd.net"],
+        ["intlnarc", "76457", "http://intlnarc-f.akamaihd.net"],
+        ["jccic", "85180", "http://jccic-f.akamaihd.net"],
+        ["jec", "76458", "http://jec-f.akamaihd.net"],
+        ["judiciary", "76459", "http://judiciary-f.akamaihd.net"],
+        ["rpc", "76591", "http://rpc-f.akamaihd.net"],
+        ["rules", "76460", "http://rules-f.akamaihd.net"],
+        ["saa", "76489", "http://srs-f.akamaihd.net"],
+        ["smbiz", "76461", "http://smbiz-f.akamaihd.net"],
+        ["srs", "75229", "http://srs-f.akamaihd.net"],
+        ["uscc", "76487", "http://srs-f.akamaihd.net"],
+        ["vetaff", "76462", "http://vetaff-f.akamaihd.net"],
+        ["arch", "", "http://ussenate-f.akamaihd.net/"]
+    ]
+    _IE_NAME = 'senate.gov'
+    _VALID_URL = r'http://www\.senate\.gov/isvp/\?(?P<qs>.+)'
+    _TESTS = [{
+        'url': 'http://www.senate.gov/isvp/?comm=judiciary&type=live&stt=&filename=judiciary031715&auto_play=false&wmode=transparent&poster=http%3A%2F%2Fwww.judiciary.senate.gov%2Fthemes%2Fjudiciary%2Fimages%2Fvideo-poster-flash-fit.png',
+        'md5': '7314c4b96dad66dd8e63dc3518ceaa6f',
+        'info_dict': {
+            'id': 'judiciary031715',
+            'ext': 'flv',
+            'title': 'Integrated Senate Video Player',
+        }
+    }, {
+        'url': 'http://www.senate.gov/isvp/?type=live&comm=commerce&filename=commerce011514.mp4&auto_play=false',
+        'md5': '2917c827513700aa9b70eaebf25116da',
+        'info_dict': {
+            'id': 'commerce011514',
+            'ext': 'flv',
+            'title': 'Integrated Senate Video Player'
+        }
+    }, {
+        'url': 'http://www.senate.gov/isvp/?type=arch&comm=intel&filename=intel090613&hc_location=ufi',
+        # checksum differs each time
+        'info_dict': {
+            'id': 'intel090613',
+            'ext': 'mp4',
+            'title': 'Integrated Senate Video Player'
+        }
+    }]
+
+    def _get_info_for_comm(self, committee):
+        for entry in self._COMM_MAP:
+            if entry[0] == committee:
+                return entry[1:]
+
+    def _real_extract(self, url):
+        qs = compat_parse_qs(re.match(self._VALID_URL, url).group('qs'))
+        if not qs.get('filename') or not qs.get('type') or not qs.get('comm'):
+            raise ExtractorError('Invalid URL', expected=True)
+
+        video_id = re.sub(r'.mp4$', '', qs['filename'][0])
+
+        webpage = self._download_webpage(url, video_id)
+
+        title = self._html_search_regex(r'<title>([^<]+)</title>', webpage, video_id)
+
+        video_type = qs['type'][0]
+        committee = video_type if video_type == 'arch' else qs['comm'][0]
+        stream_num, domain = self._get_info_for_comm(committee)
+
+        formats = []
+        if video_type == 'arch':
+            filename = video_id if '.' in video_id else video_id + '.mp4'
+            formats = [{
+                # All parameters in the query string are necessary to prevent a 403 error
+                'url': compat_urlparse.urljoin(domain, filename) + '?v=3.1.0&fp=&r=&g=',
+            }]
+        else:
+            hdcore_sign = '?hdcore=3.1.0'
+            url_params = (domain, video_id, stream_num)
+            f4m_url = '%s/z/%s_1@%s/manifest.f4m' % url_params + hdcore_sign
+            m3u8_url = '%s/i/%s_1@%s/master.m3u8' % url_params
+            for entry in self._extract_f4m_formats(f4m_url, video_id, f4m_id='f4m'):
+                # URLs without the extra param induce an 404 error
+                entry.update({'extra_param_to_segment_url': hdcore_sign})
+                formats.append(entry)
+            for entry in self._extract_m3u8_formats(m3u8_url, video_id, ext='mp4', m3u8_id='m3u8'):
+                mobj = re.search(r'(?P<tag>(?:-p|-b)).m3u8', entry['url'])
+                if mobj:
+                    entry['format_id'] += mobj.group('tag')
+                formats.append(entry)
+
+            self._sort_formats(formats)
+
+        info_dict = {
+            'id': video_id,
+            'title': title,
+        }
+
+        if len(formats) >= 1:
+            info_dict.update({'formats': formats})
+        else:
+            info_dict.update(formats[0])
+
+        return info_dict

From 24e21613b670efefd2f284c8e7027023d0a64399 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 21 Apr 2015 02:32:10 +0800
Subject: [PATCH 1014/6219] [bilibili] Capture the video-not-exist message

---
 youtube_dl/extractor/bilibili.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/bilibili.py b/youtube_dl/extractor/bilibili.py
index 75d744852..904d9a8b4 100644
--- a/youtube_dl/extractor/bilibili.py
+++ b/youtube_dl/extractor/bilibili.py
@@ -7,6 +7,7 @@ from .common import InfoExtractor
 from ..utils import (
     int_or_none,
     unified_strdate,
+    ExtractorError,
 )
 
 
@@ -30,6 +31,8 @@ class BiliBiliIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
+        if self._search_regex(r'(此视频不存在或被删除)', webpage, 'error message', default=None):
+            raise ExtractorError('The video does not exist or was deleted', expected=True)
         video_code = self._search_regex(
             r'(?s)<div itemprop="video".*?>(.*?)</div>', webpage, 'video code')
 

From f91e1a8739a59bca1ced0bbc70f8cf9c3a33f778 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 21 Apr 2015 02:57:32 +0800
Subject: [PATCH 1015/6219] [Senate] Try to capture thumbnails

---
 youtube_dl/extractor/senateisvp.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/senateisvp.py b/youtube_dl/extractor/senateisvp.py
index 807979d13..a93874cad 100644
--- a/youtube_dl/extractor/senateisvp.py
+++ b/youtube_dl/extractor/senateisvp.py
@@ -53,6 +53,7 @@ class SenateISVPIE(InfoExtractor):
             'id': 'judiciary031715',
             'ext': 'flv',
             'title': 'Integrated Senate Video Player',
+            'thumbnail': 're:^https?://.*\.(?:jpg|png)$',
         }
     }, {
         'url': 'http://www.senate.gov/isvp/?type=live&comm=commerce&filename=commerce011514.mp4&auto_play=false',
@@ -87,6 +88,9 @@ class SenateISVPIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         title = self._html_search_regex(r'<title>([^<]+)</title>', webpage, video_id)
+        poster = qs.get('poster')
+        if poster:
+            thumbnail = poster[0]
 
         video_type = qs['type'][0]
         committee = video_type if video_type == 'arch' else qs['comm'][0]
@@ -119,6 +123,7 @@ class SenateISVPIE(InfoExtractor):
         info_dict = {
             'id': video_id,
             'title': title,
+            'thumbnail': thumbnail,
         }
 
         if len(formats) >= 1:

From 2fe1b5bd2add12d70717878704cd3f811af5d22c Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 21 Apr 2015 03:18:38 +0800
Subject: [PATCH 1016/6219] [CSpan] Add detection for Senate ISVP. Closes #5302

---
 youtube_dl/extractor/cspan.py      | 18 +++++++++++++++++-
 youtube_dl/extractor/generic.py    |  6 ++++++
 youtube_dl/extractor/senateisvp.py | 20 ++++++++++++++++++--
 3 files changed, 41 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/cspan.py b/youtube_dl/extractor/cspan.py
index 955119d40..7377ac7b9 100644
--- a/youtube_dl/extractor/cspan.py
+++ b/youtube_dl/extractor/cspan.py
@@ -7,7 +7,9 @@ from ..utils import (
     int_or_none,
     unescapeHTML,
     find_xpath_attr,
+    smuggle_url,
 )
+from .senateisvp import SenateISVPIE
 
 
 class CSpanIE(InfoExtractor):
@@ -40,6 +42,15 @@ class CSpanIE(InfoExtractor):
             'title': 'General Motors Ignition Switch Recall',
         },
         'playlist_duration_sum': 14855,
+    }, {
+        # Video from senate.gov
+        'url': 'http://www.c-span.org/video/?104517-1/immigration-reforms-needed-protect-skilled-american-workers',
+        'md5': '7314c4b96dad66dd8e63dc3518ceaa6f',
+        'info_dict': {
+            'id': 'judiciary031715',
+            'ext': 'flv',
+            'title': 'Immigration Reforms Needed to Protect Skilled American Workers',
+        }
     }]
 
     def _real_extract(self, url):
@@ -56,7 +67,7 @@ class CSpanIE(InfoExtractor):
                 # present, otherwise this is a stripped version
                 r'<p class=\'initial\'>(.*?)</p>'
             ],
-            webpage, 'description', flags=re.DOTALL)
+            webpage, 'description', flags=re.DOTALL, default=None)
 
         info_url = 'http://c-spanvideo.org/videoLibrary/assets/player/ajax-player.php?os=android&html5=program&id=' + video_id
         data = self._download_json(info_url, video_id)
@@ -68,6 +79,11 @@ class CSpanIE(InfoExtractor):
         title = find_xpath_attr(doc, './/string', 'name', 'title').text
         thumbnail = find_xpath_attr(doc, './/string', 'name', 'poster').text
 
+        senate_isvp_url = SenateISVPIE._search_iframe_url(webpage)
+        if senate_isvp_url:
+            surl = smuggle_url(senate_isvp_url, {'force_title': title})
+            return self.url_result(surl, 'SenateISVP', video_id, title)
+
         files = data['video']['files']
 
         entries = [{
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index e645d1bb3..ec4d0c210 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -35,6 +35,7 @@ from .rutv import RUTVIE
 from .smotri import SmotriIE
 from .condenast import CondeNastIE
 from .udn import UDNEmbedIE
+from .senateisvp import SenateISVPIE
 
 
 class GenericIE(InfoExtractor):
@@ -1365,6 +1366,11 @@ class GenericIE(InfoExtractor):
             return self.url_result(
                 compat_urlparse.urljoin(url, mobj.group('url')), 'UDNEmbed')
 
+        # Look for Senate ISVP iframe
+        senate_isvp_url = SenateISVPIE._search_iframe_url(webpage)
+        if senate_isvp_url:
+            return self.url_result(surl, 'SenateISVP')
+
         def check_video(vurl):
             if YoutubeIE.suitable(vurl):
                 return True
diff --git a/youtube_dl/extractor/senateisvp.py b/youtube_dl/extractor/senateisvp.py
index a93874cad..23e1cd944 100644
--- a/youtube_dl/extractor/senateisvp.py
+++ b/youtube_dl/extractor/senateisvp.py
@@ -3,7 +3,10 @@ from __future__ import unicode_literals
 
 import re
 from .common import InfoExtractor
-from ..utils import ExtractorError
+from ..utils import (
+    ExtractorError,
+    unsmuggle_url,
+)
 from ..compat import (
     compat_parse_qs,
     compat_urlparse,
@@ -73,12 +76,22 @@ class SenateISVPIE(InfoExtractor):
         }
     }]
 
+    @staticmethod
+    def _search_iframe_url(webpage):
+        mobj = re.search(
+            r"<iframe[^>]+src=['\"](?P<url>http://www\.senate\.gov/isvp/\?[^'\"]+)['\"]",
+            webpage)
+        if mobj:
+            return mobj.group('url')
+
     def _get_info_for_comm(self, committee):
         for entry in self._COMM_MAP:
             if entry[0] == committee:
                 return entry[1:]
 
     def _real_extract(self, url):
+        url, smuggled_data = unsmuggle_url(url, {})
+
         qs = compat_parse_qs(re.match(self._VALID_URL, url).group('qs'))
         if not qs.get('filename') or not qs.get('type') or not qs.get('comm'):
             raise ExtractorError('Invalid URL', expected=True)
@@ -87,7 +100,10 @@ class SenateISVPIE(InfoExtractor):
 
         webpage = self._download_webpage(url, video_id)
 
-        title = self._html_search_regex(r'<title>([^<]+)</title>', webpage, video_id)
+        if smuggled_data.get('force_title'):
+            title = smuggled_data['force_title']
+        else:
+            title = self._html_search_regex(r'<title>([^<]+)</title>', webpage, video_id)
         poster = qs.get('poster')
         if poster:
             thumbnail = poster[0]

From 92dcba1e1cc411eecb64792e78ad8f3b125691c8 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 21 Apr 2015 03:30:54 +0800
Subject: [PATCH 1017/6219] [CSpan] Fix test cases CSpan_1 and CSpan_2

---
 youtube_dl/extractor/cspan.py | 22 +++++++++++++++-------
 1 file changed, 15 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/cspan.py b/youtube_dl/extractor/cspan.py
index 7377ac7b9..6bebcc65c 100644
--- a/youtube_dl/extractor/cspan.py
+++ b/youtube_dl/extractor/cspan.py
@@ -37,11 +37,14 @@ class CSpanIE(InfoExtractor):
         }
     }, {
         'url': 'http://www.c-span.org/video/?318608-1/gm-ignition-switch-recall',
+        'md5': '446562a736c6bf97118e389433ed88d4',
         'info_dict': {
             'id': '342759',
+            'ext': 'mp4',
             'title': 'General Motors Ignition Switch Recall',
+            'duration': 14848,
+            'description': 'md5:70c7c3b8fa63fa60d42772440596034c'
         },
-        'playlist_duration_sum': 14855,
     }, {
         # Video from senate.gov
         'url': 'http://www.c-span.org/video/?104517-1/immigration-reforms-needed-protect-skilled-american-workers',
@@ -97,9 +100,14 @@ class CSpanIE(InfoExtractor):
             'duration': int_or_none(f.get('length', {}).get('#text')),
         } for partnum, f in enumerate(files)]
 
-        return {
-            '_type': 'playlist',
-            'entries': entries,
-            'title': title,
-            'id': video_id,
-        }
+        if len(entries) == 1:
+            entry = dict(entries[0])
+            entry['id'] = video_id
+            return entry
+        else:
+            return {
+                '_type': 'playlist',
+                'entries': entries,
+                'title': title,
+                'id': video_id,
+            }

From 13a11b195ff00648ab5631db5b1ae7c46548c7fb Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 21 Apr 2015 05:13:25 +0800
Subject: [PATCH 1018/6219] [SenateISVP] Fix tests

Remove md5 sums. They differs from my PC and the travis worker.
---
 youtube_dl/extractor/senateisvp.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/senateisvp.py b/youtube_dl/extractor/senateisvp.py
index 23e1cd944..fa6610261 100644
--- a/youtube_dl/extractor/senateisvp.py
+++ b/youtube_dl/extractor/senateisvp.py
@@ -51,7 +51,6 @@ class SenateISVPIE(InfoExtractor):
     _VALID_URL = r'http://www\.senate\.gov/isvp/\?(?P<qs>.+)'
     _TESTS = [{
         'url': 'http://www.senate.gov/isvp/?comm=judiciary&type=live&stt=&filename=judiciary031715&auto_play=false&wmode=transparent&poster=http%3A%2F%2Fwww.judiciary.senate.gov%2Fthemes%2Fjudiciary%2Fimages%2Fvideo-poster-flash-fit.png',
-        'md5': '7314c4b96dad66dd8e63dc3518ceaa6f',
         'info_dict': {
             'id': 'judiciary031715',
             'ext': 'flv',
@@ -60,7 +59,6 @@ class SenateISVPIE(InfoExtractor):
         }
     }, {
         'url': 'http://www.senate.gov/isvp/?type=live&comm=commerce&filename=commerce011514.mp4&auto_play=false',
-        'md5': '2917c827513700aa9b70eaebf25116da',
         'info_dict': {
             'id': 'commerce011514',
             'ext': 'flv',
@@ -105,8 +103,7 @@ class SenateISVPIE(InfoExtractor):
         else:
             title = self._html_search_regex(r'<title>([^<]+)</title>', webpage, video_id)
         poster = qs.get('poster')
-        if poster:
-            thumbnail = poster[0]
+        thumbnail = poster[0] if poster else None
 
         video_type = qs['type'][0]
         committee = video_type if video_type == 'arch' else qs['comm'][0]

From da55dac047d63d48fd12247369b2fb858c4210ef Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 21 Apr 2015 05:22:23 +0800
Subject: [PATCH 1019/6219] [CSpan] Removed the md5 sum of CSpan_3

---
 youtube_dl/extractor/cspan.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/cspan.py b/youtube_dl/extractor/cspan.py
index 6bebcc65c..d516b1402 100644
--- a/youtube_dl/extractor/cspan.py
+++ b/youtube_dl/extractor/cspan.py
@@ -48,7 +48,6 @@ class CSpanIE(InfoExtractor):
     }, {
         # Video from senate.gov
         'url': 'http://www.c-span.org/video/?104517-1/immigration-reforms-needed-protect-skilled-american-workers',
-        'md5': '7314c4b96dad66dd8e63dc3518ceaa6f',
         'info_dict': {
             'id': 'judiciary031715',
             'ext': 'flv',

From 0954cd8aa4421b1844a30e99702c364f1fffc15f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 21 Apr 2015 13:48:02 +0800
Subject: [PATCH 1020/6219] [Cinemassacre] Add detection for videos from
 blip.tv

---
 youtube_dl/extractor/bliptv.py          |  9 +++++++++
 youtube_dl/extractor/generic.py         | 10 ++++------
 youtube_dl/extractor/screenwavemedia.py | 23 ++++++++++++++++++++++-
 3 files changed, 35 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/bliptv.py b/youtube_dl/extractor/bliptv.py
index b632ce967..fb56cd78d 100644
--- a/youtube_dl/extractor/bliptv.py
+++ b/youtube_dl/extractor/bliptv.py
@@ -102,6 +102,15 @@ class BlipTVIE(InfoExtractor):
         },
     ]
 
+    @staticmethod
+    def _extract_url(webpage):
+        mobj = re.search(r'<meta\s[^>]*https?://api\.blip\.tv/\w+/redirect/\w+/(\d+)', webpage)
+        if mobj:
+            return 'http://blip.tv/a/a-' + mobj.group(1)
+        mobj = re.search(r'<(?:iframe|embed|object)\s[^>]*(https?://(?:\w+\.)?blip\.tv/(?:play/|api\.swf#)[a-zA-Z0-9_]+)', webpage)
+        if mobj:
+            return mobj.group(1)
+
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         lookup_id = mobj.group('lookup_id')
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index ec4d0c210..4946cc132 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -36,6 +36,7 @@ from .smotri import SmotriIE
 from .condenast import CondeNastIE
 from .udn import UDNEmbedIE
 from .senateisvp import SenateISVPIE
+from .bliptv import BlipTVIE
 
 
 class GenericIE(InfoExtractor):
@@ -1073,12 +1074,9 @@ class GenericIE(InfoExtractor):
             }
 
         # Look for embedded blip.tv player
-        mobj = re.search(r'<meta\s[^>]*https?://api\.blip\.tv/\w+/redirect/\w+/(\d+)', webpage)
-        if mobj:
-            return self.url_result('http://blip.tv/a/a-' + mobj.group(1), 'BlipTV')
-        mobj = re.search(r'<(?:iframe|embed|object)\s[^>]*(https?://(?:\w+\.)?blip\.tv/(?:play/|api\.swf#)[a-zA-Z0-9_]+)', webpage)
-        if mobj:
-            return self.url_result(mobj.group(1), 'BlipTV')
+        bliptv_url = BlipTVIE._extract_url(webpage)
+        if bliptv_url:
+            return self.url_result(bliptv_url, 'BlipTV')
 
         # Look for embedded condenast player
         matches = re.findall(
diff --git a/youtube_dl/extractor/screenwavemedia.py b/youtube_dl/extractor/screenwavemedia.py
index 6c9fdb7c1..b515b11b4 100644
--- a/youtube_dl/extractor/screenwavemedia.py
+++ b/youtube_dl/extractor/screenwavemedia.py
@@ -7,7 +7,9 @@ from .common import InfoExtractor
 from ..utils import (
     int_or_none,
     unified_strdate,
+    ExtractorError
 )
+from .bliptv import BlipTVIE
 
 
 class ScreenwaveMediaIE(InfoExtractor):
@@ -104,6 +106,20 @@ class CinemassacreIE(InfoExtractor):
                 'upload_date': '20131002',
                 'title': 'The Mummy’s Hand (1940)',
             },
+        },
+        {
+            'url': 'http://cinemassacre.com/2006/12/07/chronologically-confused-about-bad-movie-and-video-game-sequel-titles/',
+            'md5': 'ca9b3c8dd5a66f9375daeb5135f5a3de',
+            'info_dict': {
+                'id': '4065369',
+                'ext': 'flv',
+                'title': 'AVGN: Chronologically Confused about Bad Movie and Video Game Sequel Titles',
+                'upload_date': '20061207',
+                'uploader': 'cinemassacre',
+                'uploader_id': '250778',
+                'timestamp': 1283233867,
+                'description': 'md5:0a108c78d130676b207d0f6d029ecffd',
+            }
         }
     ]
 
@@ -116,7 +132,12 @@ class CinemassacreIE(InfoExtractor):
 
         playerdata_url = self._search_regex(
             r'src="(http://player\.screenwavemedia\.com/play/[a-zA-Z]+\.php\?[^"]*\bid=.+?)"',
-            webpage, 'player data URL')
+            webpage, 'player data URL', default=None)
+        if not playerdata_url:
+            playerdata_url = BlipTVIE._extract_url(webpage)
+        if not playerdata_url:
+            raise ExtractorError('Unable to find player data')
+
         video_title = self._html_search_regex(
             r'<title>(?P<title>.+?)\|', webpage, 'title')
         video_description = self._html_search_regex(

From e94443de80d20a62fccda23fa25abcfa3798243a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 21 Apr 2015 15:10:27 +0800
Subject: [PATCH 1021/6219] [Cinemassacre] Move to a standalone module

---
 youtube_dl/extractor/__init__.py        |  3 +-
 youtube_dl/extractor/cinemassacre.py    | 81 +++++++++++++++++++++++++
 youtube_dl/extractor/screenwavemedia.py | 75 -----------------------
 3 files changed, 83 insertions(+), 76 deletions(-)
 create mode 100644 youtube_dl/extractor/cinemassacre.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 3bd6d1697..a64afa1da 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -70,6 +70,7 @@ from .chirbit import (
     ChirbitProfileIE,
 )
 from .cinchcast import CinchcastIE
+from .cinemassacre import CinemassacreIE
 from .clipfish import ClipfishIE
 from .cliphunter import CliphunterIE
 from .clipsyndicate import ClipsyndicateIE
@@ -446,7 +447,7 @@ from .sbs import SBSIE
 from .scivee import SciVeeIE
 from .screencast import ScreencastIE
 from .screencastomatic import ScreencastOMaticIE
-from .screenwavemedia import CinemassacreIE, ScreenwaveMediaIE, TeamFourIE
+from .screenwavemedia import ScreenwaveMediaIE, TeamFourIE
 from .senateisvp import SenateISVPIE
 from .servingsys import ServingSysIE
 from .sexu import SexuIE
diff --git a/youtube_dl/extractor/cinemassacre.py b/youtube_dl/extractor/cinemassacre.py
new file mode 100644
index 000000000..c8bbebe1a
--- /dev/null
+++ b/youtube_dl/extractor/cinemassacre.py
@@ -0,0 +1,81 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import ExtractorError
+from .bliptv import BlipTVIE
+
+
+class CinemassacreIE(InfoExtractor):
+    _VALID_URL = 'https?://(?:www\.)?cinemassacre\.com/(?P<date_y>[0-9]{4})/(?P<date_m>[0-9]{2})/(?P<date_d>[0-9]{2})/(?P<display_id>[^?#/]+)'
+    _TESTS = [
+        {
+            'url': 'http://cinemassacre.com/2012/11/10/avgn-the-movie-trailer/',
+            'md5': 'fde81fbafaee331785f58cd6c0d46190',
+            'info_dict': {
+                'id': 'Cinemassacre-19911',
+                'ext': 'mp4',
+                'upload_date': '20121110',
+                'title': '“Angry Video Game Nerd: The Movie” – Trailer',
+                'description': 'md5:fb87405fcb42a331742a0dce2708560b',
+            },
+        },
+        {
+            'url': 'http://cinemassacre.com/2013/10/02/the-mummys-hand-1940',
+            'md5': 'd72f10cd39eac4215048f62ab477a511',
+            'info_dict': {
+                'id': 'Cinemassacre-521be8ef82b16',
+                'ext': 'mp4',
+                'upload_date': '20131002',
+                'title': 'The Mummy’s Hand (1940)',
+            },
+        },
+        {
+            'url': 'http://cinemassacre.com/2006/12/07/chronologically-confused-about-bad-movie-and-video-game-sequel-titles/',
+            'md5': 'ca9b3c8dd5a66f9375daeb5135f5a3de',
+            'info_dict': {
+                'id': '4065369',
+                'ext': 'flv',
+                'title': 'AVGN: Chronologically Confused about Bad Movie and Video Game Sequel Titles',
+                'upload_date': '20061207',
+                'uploader': 'cinemassacre',
+                'uploader_id': '250778',
+                'timestamp': 1283233867,
+                'description': 'md5:0a108c78d130676b207d0f6d029ecffd',
+            }
+        }
+    ]
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        display_id = mobj.group('display_id')
+        video_date = mobj.group('date_y') + mobj.group('date_m') + mobj.group('date_d')
+
+        webpage = self._download_webpage(url, display_id)
+
+        playerdata_url = self._search_regex(
+            r'src="(http://player\.screenwavemedia\.com/play/[a-zA-Z]+\.php\?[^"]*\bid=.+?)"',
+            webpage, 'player data URL', default=None)
+        if not playerdata_url:
+            playerdata_url = BlipTVIE._extract_url(webpage)
+        if not playerdata_url:
+            raise ExtractorError('Unable to find player data')
+
+        video_title = self._html_search_regex(
+            r'<title>(?P<title>.+?)\|', webpage, 'title')
+        video_description = self._html_search_regex(
+            r'<div class="entry-content">(?P<description>.+?)</div>',
+            webpage, 'description', flags=re.DOTALL, fatal=False)
+        video_thumbnail = self._og_search_thumbnail(webpage)
+
+        return {
+            '_type': 'url_transparent',
+            'display_id': display_id,
+            'title': video_title,
+            'description': video_description,
+            'upload_date': video_date,
+            'thumbnail': video_thumbnail,
+            'url': playerdata_url,
+        }
diff --git a/youtube_dl/extractor/screenwavemedia.py b/youtube_dl/extractor/screenwavemedia.py
index b515b11b4..74fb1983a 100644
--- a/youtube_dl/extractor/screenwavemedia.py
+++ b/youtube_dl/extractor/screenwavemedia.py
@@ -7,9 +7,7 @@ from .common import InfoExtractor
 from ..utils import (
     int_or_none,
     unified_strdate,
-    ExtractorError
 )
-from .bliptv import BlipTVIE
 
 
 class ScreenwaveMediaIE(InfoExtractor):
@@ -83,79 +81,6 @@ class ScreenwaveMediaIE(InfoExtractor):
         }
 
 
-class CinemassacreIE(InfoExtractor):
-    _VALID_URL = 'https?://(?:www\.)?cinemassacre\.com/(?P<date_y>[0-9]{4})/(?P<date_m>[0-9]{2})/(?P<date_d>[0-9]{2})/(?P<display_id>[^?#/]+)'
-    _TESTS = [
-        {
-            'url': 'http://cinemassacre.com/2012/11/10/avgn-the-movie-trailer/',
-            'md5': 'fde81fbafaee331785f58cd6c0d46190',
-            'info_dict': {
-                'id': 'Cinemassacre-19911',
-                'ext': 'mp4',
-                'upload_date': '20121110',
-                'title': '“Angry Video Game Nerd: The Movie” – Trailer',
-                'description': 'md5:fb87405fcb42a331742a0dce2708560b',
-            },
-        },
-        {
-            'url': 'http://cinemassacre.com/2013/10/02/the-mummys-hand-1940',
-            'md5': 'd72f10cd39eac4215048f62ab477a511',
-            'info_dict': {
-                'id': 'Cinemassacre-521be8ef82b16',
-                'ext': 'mp4',
-                'upload_date': '20131002',
-                'title': 'The Mummy’s Hand (1940)',
-            },
-        },
-        {
-            'url': 'http://cinemassacre.com/2006/12/07/chronologically-confused-about-bad-movie-and-video-game-sequel-titles/',
-            'md5': 'ca9b3c8dd5a66f9375daeb5135f5a3de',
-            'info_dict': {
-                'id': '4065369',
-                'ext': 'flv',
-                'title': 'AVGN: Chronologically Confused about Bad Movie and Video Game Sequel Titles',
-                'upload_date': '20061207',
-                'uploader': 'cinemassacre',
-                'uploader_id': '250778',
-                'timestamp': 1283233867,
-                'description': 'md5:0a108c78d130676b207d0f6d029ecffd',
-            }
-        }
-    ]
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        display_id = mobj.group('display_id')
-        video_date = mobj.group('date_y') + mobj.group('date_m') + mobj.group('date_d')
-
-        webpage = self._download_webpage(url, display_id)
-
-        playerdata_url = self._search_regex(
-            r'src="(http://player\.screenwavemedia\.com/play/[a-zA-Z]+\.php\?[^"]*\bid=.+?)"',
-            webpage, 'player data URL', default=None)
-        if not playerdata_url:
-            playerdata_url = BlipTVIE._extract_url(webpage)
-        if not playerdata_url:
-            raise ExtractorError('Unable to find player data')
-
-        video_title = self._html_search_regex(
-            r'<title>(?P<title>.+?)\|', webpage, 'title')
-        video_description = self._html_search_regex(
-            r'<div class="entry-content">(?P<description>.+?)</div>',
-            webpage, 'description', flags=re.DOTALL, fatal=False)
-        video_thumbnail = self._og_search_thumbnail(webpage)
-
-        return {
-            '_type': 'url_transparent',
-            'display_id': display_id,
-            'title': video_title,
-            'description': video_description,
-            'upload_date': video_date,
-            'thumbnail': video_thumbnail,
-            'url': playerdata_url,
-        }
-
-
 class TeamFourIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?teamfourstar\.com/video/(?P<id>[a-z0-9\-]+)/?'
     _TEST = {

From 757cda0a96e5efd0c1fb86824a99e558a7c797b0 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 21 Apr 2015 15:20:40 +0800
Subject: [PATCH 1022/6219] [Cinemassacre] Support Youtube embedded videos
 (fixes #5131)

---
 youtube_dl/extractor/cinemassacre.py | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/cinemassacre.py b/youtube_dl/extractor/cinemassacre.py
index c8bbebe1a..cf0a7551b 100644
--- a/youtube_dl/extractor/cinemassacre.py
+++ b/youtube_dl/extractor/cinemassacre.py
@@ -33,6 +33,7 @@ class CinemassacreIE(InfoExtractor):
             },
         },
         {
+            # blip.tv embedded video
             'url': 'http://cinemassacre.com/2006/12/07/chronologically-confused-about-bad-movie-and-video-game-sequel-titles/',
             'md5': 'ca9b3c8dd5a66f9375daeb5135f5a3de',
             'info_dict': {
@@ -45,6 +46,20 @@ class CinemassacreIE(InfoExtractor):
                 'timestamp': 1283233867,
                 'description': 'md5:0a108c78d130676b207d0f6d029ecffd',
             }
+        },
+        {
+            # Youtube embedded video
+            'url': 'http://cinemassacre.com/2006/09/01/mckids/',
+            'md5': '6eb30961fa795fedc750eac4881ad2e1',
+            'info_dict': {
+                'id': 'FnxsNhuikpo',
+                'ext': 'mp4',
+                'upload_date': '20060901',
+                'uploader': 'Cinemassacre Extras',
+                'description': 'md5:de9b751efa9e45fbaafd9c8a1123ed53',
+                'uploader_id': 'Cinemassacre',
+                'title': 'AVGN: McKids',
+            }
         }
     ]
 
@@ -56,7 +71,10 @@ class CinemassacreIE(InfoExtractor):
         webpage = self._download_webpage(url, display_id)
 
         playerdata_url = self._search_regex(
-            r'src="(http://player\.screenwavemedia\.com/play/[a-zA-Z]+\.php\?[^"]*\bid=.+?)"',
+            [
+                r'src="(http://player\.screenwavemedia\.com/play/[a-zA-Z]+\.php\?[^"]*\bid=.+?)"',
+                r'<iframe[^>]+src="((?:https?:)?//(?:[^.]+\.)?youtube\.com/.+?)"',
+            ],
             webpage, 'player data URL', default=None)
         if not playerdata_url:
             playerdata_url = BlipTVIE._extract_url(webpage)

From 5c1e6f69c421c244139b1f4c6d2759dadfbc1abd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 21 Apr 2015 15:04:55 +0200
Subject: [PATCH 1023/6219] [senate] Simplify

There isn't any problem if the 'formats' field only has one element
---
 youtube_dl/extractor/senateisvp.py | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/senateisvp.py b/youtube_dl/extractor/senateisvp.py
index fa6610261..d3b8a1be4 100644
--- a/youtube_dl/extractor/senateisvp.py
+++ b/youtube_dl/extractor/senateisvp.py
@@ -133,15 +133,9 @@ class SenateISVPIE(InfoExtractor):
 
             self._sort_formats(formats)
 
-        info_dict = {
+        return {
             'id': video_id,
             'title': title,
+            'formats': formats,
             'thumbnail': thumbnail,
         }
-
-        if len(formats) >= 1:
-            info_dict.update({'formats': formats})
-        else:
-            info_dict.update(formats[0])
-
-        return info_dict

From ed553379dfd4d564f8335defc1067eeecd536f04 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 21 Apr 2015 20:55:05 +0600
Subject: [PATCH 1024/6219] [youtube:ytsearch] Temporary workaround (#5483)

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 52909b0da..4ec39c589 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1517,7 +1517,7 @@ class YoutubeSearchIE(SearchInfoExtractor):
         while (PAGE_SIZE * pagenum) < limit:
             result_url = self._API_URL % (
                 compat_urllib_parse.quote_plus(query.encode('utf-8')),
-                (PAGE_SIZE * pagenum) + 1)
+                max((PAGE_SIZE * pagenum) + 1), 2)
             data_json = self._download_webpage(
                 result_url, video_id='query "%s"' % query,
                 note='Downloading page %s' % (pagenum + 1),

From eb0f3e7ec080549c1df6a104fc59400efd9a992a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 21 Apr 2015 22:36:41 +0600
Subject: [PATCH 1025/6219] [youtube:user] Extract in terms of
 `load_more_widget_html`

---
 youtube_dl/extractor/youtube.py | 49 +++------------------------------
 1 file changed, 4 insertions(+), 45 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 4ec39c589..0b4038038 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1358,6 +1358,7 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
 class YoutubeChannelIE(InfoExtractor):
     IE_DESC = 'YouTube.com channels'
     _VALID_URL = r'https?://(?:youtu\.be|(?:\w+\.)?youtube(?:-nocookie)?\.com)/channel/(?P<id>[0-9A-Za-z_-]+)'
+    _TEMPLATE_URL = 'https://www.youtube.com/channel/%s/videos'
     IE_NAME = 'youtube:channel'
     _TESTS = [{
         'note': 'paginated channel',
@@ -1386,7 +1387,7 @@ class YoutubeChannelIE(InfoExtractor):
     def _real_extract(self, url):
         channel_id = self._match_id(url)
 
-        url = 'https://www.youtube.com/channel/%s/videos' % channel_id
+        url = self._TEMPLATE_URL % channel_id
         channel_page = self._download_webpage(url, channel_id)
         autogenerated = re.search(r'''(?x)
                 class="[^"]*?(?:
@@ -1429,12 +1430,10 @@ class YoutubeChannelIE(InfoExtractor):
         return self.playlist_result(_entries(), channel_id)
 
 
-class YoutubeUserIE(InfoExtractor):
+class YoutubeUserIE(YoutubeChannelIE):
     IE_DESC = 'YouTube.com user videos (URL or "ytuser" keyword)'
     _VALID_URL = r'(?:(?:(?:https?://)?(?:\w+\.)?youtube\.com/(?:user/)?(?!(?:attribution_link|watch|results)(?:$|[^a-z_A-Z0-9-])))|ytuser:)(?!feed/)(?P<id>[A-Za-z0-9_-]+)'
-    _TEMPLATE_URL = 'https://gdata.youtube.com/feeds/api/users/%s'
-    _GDATA_PAGE_SIZE = 50
-    _GDATA_URL = 'https://gdata.youtube.com/feeds/api/users/%s/uploads?max-results=%d&start-index=%d&alt=json'
+    _TEMPLATE_URL = 'https://www.youtube.com/user/%s/videos'
     IE_NAME = 'youtube:user'
 
     _TESTS = [{
@@ -1458,46 +1457,6 @@ class YoutubeUserIE(InfoExtractor):
         else:
             return super(YoutubeUserIE, cls).suitable(url)
 
-    def _real_extract(self, url):
-        username = self._match_id(url)
-
-        # Download video ids using YouTube Data API. Result size per
-        # query is limited (currently to 50 videos) so we need to query
-        # page by page until there are no video ids - it means we got
-        # all of them.
-
-        def download_page(pagenum):
-            start_index = pagenum * self._GDATA_PAGE_SIZE + 1
-
-            gdata_url = self._GDATA_URL % (username, self._GDATA_PAGE_SIZE, start_index)
-            page = self._download_webpage(
-                gdata_url, username,
-                'Downloading video ids from %d to %d' % (
-                    start_index, start_index + self._GDATA_PAGE_SIZE))
-
-            try:
-                response = json.loads(page)
-            except ValueError as err:
-                raise ExtractorError('Invalid JSON in API response: ' + compat_str(err))
-            if 'entry' not in response['feed']:
-                return
-
-            # Extract video identifiers
-            entries = response['feed']['entry']
-            for entry in entries:
-                title = entry['title']['$t']
-                video_id = entry['id']['$t'].split('/')[-1]
-                yield {
-                    '_type': 'url',
-                    'url': video_id,
-                    'ie_key': 'Youtube',
-                    'id': video_id,
-                    'title': title,
-                }
-        url_results = OnDemandPagedList(download_page, self._GDATA_PAGE_SIZE)
-
-        return self.playlist_result(url_results, playlist_title=username)
-
 
 class YoutubeSearchIE(SearchInfoExtractor):
     IE_DESC = 'YouTube.com searches'

From 60bf45c80d377a38b00b9ec1426c4cc1d9003742 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 21 Apr 2015 22:37:45 +0600
Subject: [PATCH 1026/6219] [youtube:channel] Specify first page download
 message

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 0b4038038..1469b932f 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1388,7 +1388,7 @@ class YoutubeChannelIE(InfoExtractor):
         channel_id = self._match_id(url)
 
         url = self._TEMPLATE_URL % channel_id
-        channel_page = self._download_webpage(url, channel_id)
+        channel_page = self._download_webpage(url, channel_id, 'Downloading page #1')
         autogenerated = re.search(r'''(?x)
                 class="[^"]*?(?:
                     channel-header-autogenerated-label|

From 6de5dbafeebd2c54670e4cbf2833f9f10f3c2032 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 21 Apr 2015 22:42:21 +0600
Subject: [PATCH 1027/6219] [youtube:channel] Make `extract_videos_from_page`
 static

---
 youtube_dl/extractor/youtube.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 1469b932f..8bf54931c 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1369,7 +1369,8 @@ class YoutubeChannelIE(InfoExtractor):
         }
     }]
 
-    def extract_videos_from_page(self, page):
+    @staticmethod
+    def extract_videos_from_page(page):
         ids_in_page = []
         titles_in_page = []
         for mobj in re.finditer(r'(?:title="(?P<title>[^"]+)"[^>]+)?href="/watch\?v=(?P<id>[0-9A-Za-z_-]+)&?', page):

From cc38fa6cfbdab2ca77ecb1155d64574ab0004bb5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 21 Apr 2015 22:55:59 +0600
Subject: [PATCH 1028/6219] [youtube] Remove unused import

---
 youtube_dl/extractor/youtube.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 8bf54931c..b2ae08418 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -28,7 +28,6 @@ from ..utils import (
     get_element_by_attribute,
     get_element_by_id,
     int_or_none,
-    OnDemandPagedList,
     orderedSet,
     unescapeHTML,
     unified_strdate,

From b4c08069638896e6c565536d52ab8fc9226c91bd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 21 Apr 2015 19:30:31 +0200
Subject: [PATCH 1029/6219] [youtube:ytsearch] Use the same system as the
 search webpage (fixes #5483)

The gdata api V2 was deprecated and according to http://youtube-eng.blogspot.com.es/2014/03/committing-to-youtube-data-api-v3_4.html remains available until April 20, 2015.
---
 youtube_dl/extractor/youtube.py | 51 +++++++++++++++++----------------
 1 file changed, 26 insertions(+), 25 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index b2ae08418..f1a5c0077 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1458,54 +1458,55 @@ class YoutubeUserIE(YoutubeChannelIE):
             return super(YoutubeUserIE, cls).suitable(url)
 
 
-class YoutubeSearchIE(SearchInfoExtractor):
+class YoutubeSearchIE(SearchInfoExtractor, YoutubePlaylistIE):
     IE_DESC = 'YouTube.com searches'
-    _API_URL = 'https://gdata.youtube.com/feeds/api/videos?q=%s&start-index=%i&max-results=50&v=2&alt=jsonc'
-    _MAX_RESULTS = 1000
+    # there doesn't appear to be a real limit, for example if you search for
+    # 'python' you get more than 8.000.000 results
+    _MAX_RESULTS = float('inf')
     IE_NAME = 'youtube:search'
     _SEARCH_KEY = 'ytsearch'
+    _EXTRA_QUERY_ARGS = {}
 
     def _get_n_results(self, query, n):
         """Get a specified number of results for a query"""
 
-        video_ids = []
-        pagenum = 0
+        videos = []
         limit = n
-        PAGE_SIZE = 50
 
-        while (PAGE_SIZE * pagenum) < limit:
-            result_url = self._API_URL % (
-                compat_urllib_parse.quote_plus(query.encode('utf-8')),
-                max((PAGE_SIZE * pagenum) + 1), 2)
-            data_json = self._download_webpage(
+        for pagenum in itertools.count(1):
+            url_query = {
+                'search_query': query,
+                'page': pagenum,
+                'spf': 'navigate',
+            }
+            url_query.update(self._EXTRA_QUERY_ARGS)
+            result_url = 'https://www.youtube.com/results?' + compat_urllib_parse.urlencode(url_query)
+            data = self._download_json(
                 result_url, video_id='query "%s"' % query,
-                note='Downloading page %s' % (pagenum + 1),
+                note='Downloading page %s' % pagenum,
                 errnote='Unable to download API page')
-            data = json.loads(data_json)
-            api_response = data['data']
+            html_content = data[1]['body']['content']
 
-            if 'items' not in api_response:
+            if 'class="search-message' in html_content:
                 raise ExtractorError(
                     '[youtube] No video results', expected=True)
 
-            new_ids = list(video['id'] for video in api_response['items'])
-            video_ids += new_ids
+            new_videos = self._ids_to_results(orderedSet(re.findall(
+                r'href="/watch\?v=(.{11})', html_content)))
+            videos += new_videos
+            if not new_videos or len(videos) > limit:
+                break
 
-            limit = min(n, api_response['totalItems'])
-            pagenum += 1
-
-        if len(video_ids) > n:
-            video_ids = video_ids[:n]
-        videos = [self.url_result(video_id, 'Youtube', video_id=video_id)
-                  for video_id in video_ids]
+        if len(videos) > n:
+            videos = videos[:n]
         return self.playlist_result(videos, query)
 
 
 class YoutubeSearchDateIE(YoutubeSearchIE):
     IE_NAME = YoutubeSearchIE.IE_NAME + ':date'
-    _API_URL = 'https://gdata.youtube.com/feeds/api/videos?q=%s&start-index=%i&max-results=50&v=2&alt=jsonc&orderby=published'
     _SEARCH_KEY = 'ytsearchdate'
     IE_DESC = 'YouTube.com searches, newest videos first'
+    _EXTRA_QUERY_ARGS = {'search_sort': 'video_date_uploaded'}
 
 
 class YoutubeSearchURLIE(InfoExtractor):

From 8be2bdfabd7a24936697f6681bd417de414a554a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 22 Apr 2015 15:05:35 +0800
Subject: [PATCH 1030/6219] [YoutubeDL] Remove the redundant assignment to
 old_filename

Caused by commmit 592e97e8550389e22b716eb33c30584aa3a8d656
---
 youtube_dl/YoutubeDL.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 5b2c3aa38..0fdcf1b0b 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1516,7 +1516,6 @@ class YoutubeDL(object):
             pps_chain.extend(ie_info['__postprocessors'])
         pps_chain.extend(self._pps)
         for pp in pps_chain:
-            old_filename = info['filepath']
             try:
                 files_to_delete, info = pp.run(info)
             except PostProcessingError as e:

From 9dd8e46a2d0860421b4bb4f616f05e5ebd686380 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 22 Apr 2015 20:28:33 +0600
Subject: [PATCH 1031/6219] [youtube:search] Cancel out _TESTS

---
 youtube_dl/extractor/youtube.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index f1a5c0077..07c0f6ef9 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1466,6 +1466,7 @@ class YoutubeSearchIE(SearchInfoExtractor, YoutubePlaylistIE):
     IE_NAME = 'youtube:search'
     _SEARCH_KEY = 'ytsearch'
     _EXTRA_QUERY_ARGS = {}
+    _TESTS = []
 
     def _get_n_results(self, query, n):
         """Get a specified number of results for a query"""

From bc94bd510bfb8a98f7456912f930c69c21bd6e15 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 22 Apr 2015 21:01:25 +0600
Subject: [PATCH 1032/6219] [hitbox] Extract all formats (Closes #5494)

---
 youtube_dl/extractor/hitbox.py | 32 ++++++++++++++++++++++++++------
 1 file changed, 26 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/hitbox.py b/youtube_dl/extractor/hitbox.py
index d606429ca..d7cd8bb7e 100644
--- a/youtube_dl/extractor/hitbox.py
+++ b/youtube_dl/extractor/hitbox.py
@@ -96,13 +96,33 @@ class HitboxIE(InfoExtractor):
             'https://www.hitbox.tv/api/player/config/video/%s' % video_id,
             video_id)
 
-        clip = player_config.get('clip')
-        video_url = clip.get('url')
-        res = clip.get('bitrates', [])[0].get('label')
+        formats = []
+        for video in player_config['clip']['bitrates']:
+            label = video.get('label')
+            if label == 'Auto':
+                continue
+            video_url = video.get('url')
+            if not video_url:
+                continue
+            bitrate = int_or_none(video.get('bitrate'))
+            if determine_ext(video_url) == 'm3u8':
+                if not video_url.startswith('http'):
+                    continue
+                formats.append({
+                    'url': video_url,
+                    'ext': 'mp4',
+                    'tbr': bitrate,
+                    'format_note': label,
+                    'protocol': 'm3u8_native',
+                })
+            else:
+                formats.append({
+                    'url': video_url,
+                    'tbr': bitrate,
+                    'format_note': label,
+                })
 
-        metadata['resolution'] = res
-        metadata['url'] = video_url
-        metadata['protocol'] = 'm3u8'
+        metadata['formats'] = formats
 
         return metadata
 

From 29492f33329c6d6fd4be7f2da7dcaff14bc531ba Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 22 Apr 2015 21:01:52 +0600
Subject: [PATCH 1033/6219] [hitbox] Sort formats

---
 youtube_dl/extractor/hitbox.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/hitbox.py b/youtube_dl/extractor/hitbox.py
index d7cd8bb7e..6701fa48d 100644
--- a/youtube_dl/extractor/hitbox.py
+++ b/youtube_dl/extractor/hitbox.py
@@ -121,6 +121,7 @@ class HitboxIE(InfoExtractor):
                     'tbr': bitrate,
                     'format_note': label,
                 })
+        self._sort_formats(formats)
 
         metadata['formats'] = formats
 

From 008bee0f50ea27e4fde4ba4d0cdb9043783299c0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 22 Apr 2015 21:03:56 +0600
Subject: [PATCH 1034/6219] [hitbox] Extract formats before metadata

---
 youtube_dl/extractor/hitbox.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/hitbox.py b/youtube_dl/extractor/hitbox.py
index 6701fa48d..c51bc7ca4 100644
--- a/youtube_dl/extractor/hitbox.py
+++ b/youtube_dl/extractor/hitbox.py
@@ -88,10 +88,6 @@ class HitboxIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        metadata = self._extract_metadata(
-            'https://www.hitbox.tv/api/media/video',
-            video_id)
-
         player_config = self._download_json(
             'https://www.hitbox.tv/api/player/config/video/%s' % video_id,
             video_id)
@@ -123,6 +119,9 @@ class HitboxIE(InfoExtractor):
                 })
         self._sort_formats(formats)
 
+        metadata = self._extract_metadata(
+            'https://www.hitbox.tv/api/media/video',
+            video_id)
         metadata['formats'] = formats
 
         return metadata

From 14f41bc2fb01266243e763ddc80840b895acd294 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 22 Apr 2015 21:05:08 +0600
Subject: [PATCH 1035/6219] [hitbox:live] Extract formats before metadata

---
 youtube_dl/extractor/hitbox.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/hitbox.py b/youtube_dl/extractor/hitbox.py
index c51bc7ca4..2c440898e 100644
--- a/youtube_dl/extractor/hitbox.py
+++ b/youtube_dl/extractor/hitbox.py
@@ -150,10 +150,6 @@ class HitboxLiveIE(HitboxIE):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        metadata = self._extract_metadata(
-            'https://www.hitbox.tv/api/media/live',
-            video_id)
-
         player_config = self._download_json(
             'https://www.hitbox.tv/api/player/config/live/%s' % video_id,
             video_id)
@@ -194,9 +190,13 @@ class HitboxLiveIE(HitboxIE):
                             'page_url': url,
                             'player_url': 'http://www.hitbox.tv/static/player/flowplayer/flowplayer.commercial-3.2.16.swf',
                         })
-
         self._sort_formats(formats)
+
+        metadata = self._extract_metadata(
+            'https://www.hitbox.tv/api/media/live',
+            video_id)
         metadata['formats'] = formats
         metadata['is_live'] = True
         metadata['title'] = self._live_title(metadata.get('title'))
+
         return metadata

From 33b066bda0491bca54cc09d7d117867f885aa5f4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 22 Apr 2015 21:09:21 +0600
Subject: [PATCH 1036/6219] [hitbox] Clarify download messages

---
 youtube_dl/extractor/hitbox.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/hitbox.py b/youtube_dl/extractor/hitbox.py
index 2c440898e..421f55bbe 100644
--- a/youtube_dl/extractor/hitbox.py
+++ b/youtube_dl/extractor/hitbox.py
@@ -43,7 +43,8 @@ class HitboxIE(InfoExtractor):
     def _extract_metadata(self, url, video_id):
         thumb_base = 'https://edge.sf.hitbox.tv'
         metadata = self._download_json(
-            '%s/%s' % (url, video_id), video_id)
+            '%s/%s' % (url, video_id), video_id,
+            'Downloading metadata JSON')
 
         date = 'media_live_since'
         media_type = 'livestream'
@@ -90,7 +91,7 @@ class HitboxIE(InfoExtractor):
 
         player_config = self._download_json(
             'https://www.hitbox.tv/api/player/config/video/%s' % video_id,
-            video_id)
+            video_id, 'Downloading video JSON')
 
         formats = []
         for video in player_config['clip']['bitrates']:

From f8e51f60b302357b43ef15cf479da3ce09643f9a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 22 Apr 2015 19:24:14 +0200
Subject: [PATCH 1037/6219] [flickr] Fix extraction (fixes #5501)

---
 youtube_dl/extractor/flickr.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/flickr.py b/youtube_dl/extractor/flickr.py
index 0c858b654..adffe4857 100644
--- a/youtube_dl/extractor/flickr.py
+++ b/youtube_dl/extractor/flickr.py
@@ -3,6 +3,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import compat_urllib_request
 from ..utils import (
     ExtractorError,
     unescapeHTML,
@@ -29,9 +30,14 @@ class FlickrIE(InfoExtractor):
         video_id = mobj.group('id')
         video_uploader_id = mobj.group('uploader_id')
         webpage_url = 'http://www.flickr.com/photos/' + video_uploader_id + '/' + video_id
-        webpage = self._download_webpage(webpage_url, video_id)
+        req = compat_urllib_request.Request(webpage_url)
+        req.add_header(
+            'User-Agent',
+            # it needs a more recent version
+            'Mozilla/5.0 (X11; Linux x86_64; rv:38.0) Gecko/20150101 Firefox/38.0 (Chrome)')
+        webpage = self._download_webpage(req, video_id)
 
-        secret = self._search_regex(r"photo_secret: '(\w+)'", webpage, 'secret')
+        secret = self._search_regex(r'secret"\s*:\s*"(\w+)"', webpage, 'secret')
 
         first_url = 'https://secure.flickr.com/apps/video/video_mtl_xml.gne?v=x&photo_id=' + video_id + '&secret=' + secret + '&bitrate=700&target=_self'
         first_xml = self._download_webpage(first_url, video_id, 'Downloading first data webpage')

From c04c3e334cc7ff0bbd2cbb8167f5cd2794c29d29 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 22 Apr 2015 19:58:39 +0200
Subject: [PATCH 1038/6219] [flickr] Don't use regex for extracting the info
 from the xml files

---
 youtube_dl/extractor/flickr.py | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/flickr.py b/youtube_dl/extractor/flickr.py
index adffe4857..2fe76d661 100644
--- a/youtube_dl/extractor/flickr.py
+++ b/youtube_dl/extractor/flickr.py
@@ -6,7 +6,7 @@ from .common import InfoExtractor
 from ..compat import compat_urllib_request
 from ..utils import (
     ExtractorError,
-    unescapeHTML,
+    find_xpath_attr,
 )
 
 
@@ -40,20 +40,21 @@ class FlickrIE(InfoExtractor):
         secret = self._search_regex(r'secret"\s*:\s*"(\w+)"', webpage, 'secret')
 
         first_url = 'https://secure.flickr.com/apps/video/video_mtl_xml.gne?v=x&photo_id=' + video_id + '&secret=' + secret + '&bitrate=700&target=_self'
-        first_xml = self._download_webpage(first_url, video_id, 'Downloading first data webpage')
+        first_xml = self._download_xml(first_url, video_id, 'Downloading first data webpage')
 
-        node_id = self._html_search_regex(r'<Item id="id">(\d+-\d+)</Item>',
-                                          first_xml, 'node_id')
+        node_id = find_xpath_attr(
+            first_xml, './/{http://video.yahoo.com/YEP/1.0/}Item', 'id',
+            'id').text
 
         second_url = 'https://secure.flickr.com/video_playlist.gne?node_id=' + node_id + '&tech=flash&mode=playlist&bitrate=700&secret=' + secret + '&rd=video.yahoo.com&noad=1'
-        second_xml = self._download_webpage(second_url, video_id, 'Downloading second data webpage')
+        second_xml = self._download_xml(second_url, video_id, 'Downloading second data webpage')
 
         self.report_extraction(video_id)
 
-        mobj = re.search(r'<STREAM APP="(.+?)" FULLPATH="(.+?)"', second_xml)
-        if mobj is None:
+        stream = second_xml.find('.//STREAM')
+        if stream is None:
             raise ExtractorError('Unable to extract video url')
-        video_url = mobj.group(1) + unescapeHTML(mobj.group(2))
+        video_url = stream.attrib['APP'] + stream.attrib['FULLPATH']
 
         return {
             'id': video_id,

From 7513f298b04eb11ab6c0bd0845b7a3b221bee317 Mon Sep 17 00:00:00 2001
From: pulpe <Pulpan3@gmail.com>
Date: Wed, 22 Apr 2015 23:50:11 +0200
Subject: [PATCH 1039/6219] [vimeo] Fix login token (fixes #5082)

---
 youtube_dl/extractor/vimeo.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 28bcc89cd..55c1574bb 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -38,7 +38,7 @@ class VimeoBaseInfoExtractor(InfoExtractor):
         self.report_login()
         login_url = 'https://vimeo.com/log_in'
         webpage = self._download_webpage(login_url, None, False)
-        token = self._search_regex(r'xsrft = \'(.*?)\'', webpage, 'login token')
+        token = self._search_regex(r'xsrft":"(.*?)"', webpage, 'login token')
         data = urlencode_postdata({
             'email': username,
             'password': password,
@@ -177,7 +177,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
         password = self._downloader.params.get('videopassword', None)
         if password is None:
             raise ExtractorError('This video is protected by a password, use the --video-password option', expected=True)
-        token = self._search_regex(r'xsrft = \'(.*?)\'', webpage, 'login token')
+        token = self._search_regex(r'xsrft":"(.*?)"', webpage, 'login token')
         data = urlencode_postdata({
             'password': password,
             'token': token,

From 14a2d6789f24ce6f6565747a48ffe7728d5a2251 Mon Sep 17 00:00:00 2001
From: pulpe <Pulpan3@gmail.com>
Date: Wed, 22 Apr 2015 23:55:19 +0200
Subject: [PATCH 1040/6219] [vimeo] one token overlooked

---
 youtube_dl/extractor/vimeo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 55c1574bb..9027f9dd6 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -439,7 +439,7 @@ class VimeoChannelIE(InfoExtractor):
             name="([^"]+)"\s+
             value="([^"]*)"
             ''', login_form))
-        token = self._search_regex(r'xsrft = \'(.*?)\'', webpage, 'login token')
+        token = self._search_regex(r'xsrft":"(.*?)"', webpage, 'login token')
         fields['token'] = token
         fields['password'] = password
         post = urlencode_postdata(fields)

From 8c8826176d4562dce9558d121a955ed71509315a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 23 Apr 2015 13:49:06 +0800
Subject: [PATCH 1041/6219] [xattr] Add version detection for python-pyxattr

For more information, see #5498 and changes to convertObj() in
iustin/pyxattr@cc84e466f63906d32ec1bf4a4fcae6a7bce9a4c8
---
 youtube_dl/postprocessor/xattrpp.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/youtube_dl/postprocessor/xattrpp.py b/youtube_dl/postprocessor/xattrpp.py
index 0cba99fc3..b74adff43 100644
--- a/youtube_dl/postprocessor/xattrpp.py
+++ b/youtube_dl/postprocessor/xattrpp.py
@@ -11,6 +11,7 @@ from ..compat import (
 from ..utils import (
     check_executable,
     hyphenate_date,
+    version_tuple,
 )
 
 
@@ -36,6 +37,19 @@ class XAttrMetadataPP(PostProcessor):
             # try the pyxattr module...
             import xattr
 
+            # Unicode arguments are not supported in python-pyxattr until
+            # version 0.5.0
+            # See https://github.com/rg3/youtube-dl/issues/5498
+            pyxattr_required_version = '0.5.0'
+            if version_tuple(xattr.__version__) < version_tuple(pyxattr_required_version):
+                self._downloader.report_warning(
+                    'python-pyxattr is detected but is too old. '
+                    'yourube-dl requires %s or above while your version is %s. '
+                    'Falling back to other xattr implementations' % (
+                        pyxattr_required_version, xattr.__version__))
+
+                raise ImportError
+
             def write_xattr(path, key, value):
                 return xattr.setxattr(path, key, value)
 

From 3eec9fef302929438c321adfc8ec16adcc6ffb6d Mon Sep 17 00:00:00 2001
From: pulpe <Pulpan3@gmail.com>
Date: Thu, 23 Apr 2015 11:41:21 +0200
Subject: [PATCH 1042/6219] [realvid] Add extractor for realvid.net (closes
 #5504)

---
 youtube_dl/extractor/gorillavid.py | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/gorillavid.py b/youtube_dl/extractor/gorillavid.py
index ae24aff84..ed2623456 100644
--- a/youtube_dl/extractor/gorillavid.py
+++ b/youtube_dl/extractor/gorillavid.py
@@ -15,10 +15,10 @@ from ..utils import (
 
 
 class GorillaVidIE(InfoExtractor):
-    IE_DESC = 'GorillaVid.in, daclips.in, movpod.in and fastvideo.in'
+    IE_DESC = 'GorillaVid.in, daclips.in, movpod.in, fastvideo.in and realvid.net'
     _VALID_URL = r'''(?x)
         https?://(?P<host>(?:www\.)?
-            (?:daclips\.in|gorillavid\.in|movpod\.in|fastvideo\.in))/
+            (?:daclips\.in|gorillavid\.in|movpod\.in|fastvideo\.in|realvid\.net))/
         (?:embed-)?(?P<id>[0-9a-zA-Z]+)(?:-[0-9]+x[0-9]+\.html)?
     '''
 
@@ -61,6 +61,15 @@ class GorillaVidIE(InfoExtractor):
             'title': 'Man of Steel - Trailer',
             'thumbnail': 're:http://.*\.jpg',
         },
+    }, {
+        'url': 'http://realvid.net/ctn2y6p2eviw',
+        'md5': 'b2166d2cf192efd6b6d764c18fd3710e',
+        'info_dict': {
+            'id': 'ctn2y6p2eviw',
+            'ext': 'flv',
+            'title': 'rdx 1955',
+            'thumbnail': 're:http://.*\.jpg',
+        },
     }, {
         'url': 'http://movpod.in/0wguyyxi1yca',
         'only_matching': True,
@@ -97,7 +106,7 @@ class GorillaVidIE(InfoExtractor):
             webpage = self._download_webpage(req, video_id, 'Downloading video page')
 
         title = self._search_regex(
-            r'style="z-index: [0-9]+;">([^<]+)</span>',
+            [r'style="z-index: [0-9]+;">([^<]+)</span>', r'>Watch (.+) '],
             webpage, 'title', default=None) or self._og_search_title(webpage)
         video_url = self._search_regex(
             r'file\s*:\s*["\'](http[^"\']+)["\'],', webpage, 'file url')

From f8610ba1ca4705dcae5c45c5ac66a99b174ebed3 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 23 Apr 2015 17:57:49 +0800
Subject: [PATCH 1043/6219] [ustream] Fix extraction (closes #3998)

---
 youtube_dl/extractor/ustream.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ustream.py b/youtube_dl/extractor/ustream.py
index 68d03b999..d559e228a 100644
--- a/youtube_dl/extractor/ustream.py
+++ b/youtube_dl/extractor/ustream.py
@@ -1,5 +1,6 @@
 from __future__ import unicode_literals
 
+import json
 import re
 
 from .common import InfoExtractor
@@ -39,7 +40,15 @@ class UstreamIE(InfoExtractor):
             desktop_url = 'http://www.ustream.tv/recorded/' + desktop_video_id
             return self.url_result(desktop_url, 'Ustream')
 
-        video_url = 'http://tcdn.ustream.tv/video/%s' % video_id
+        params = self._download_json(
+            'http://cdngw.ustream.tv/rgwjson/Viewer.getVideo/' + json.dumps({
+                'brandId': 1,
+                'videoId': int(video_id),
+                'autoplay': False,
+            }), video_id)
+
+        video_url = params['flv']
+
         webpage = self._download_webpage(url, video_id)
 
         self.report_extraction(video_id)

From 762155cc90754fc29c68bf0f0256988f37862c3b Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 23 Apr 2015 18:07:09 +0800
Subject: [PATCH 1044/6219] [ustream] Checking errors

---
 youtube_dl/extractor/ustream.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/ustream.py b/youtube_dl/extractor/ustream.py
index d559e228a..f8893b6cd 100644
--- a/youtube_dl/extractor/ustream.py
+++ b/youtube_dl/extractor/ustream.py
@@ -7,6 +7,7 @@ from .common import InfoExtractor
 from ..compat import (
     compat_urlparse,
 )
+from ..utils import ExtractorError
 
 
 class UstreamIE(InfoExtractor):
@@ -47,6 +48,9 @@ class UstreamIE(InfoExtractor):
                 'autoplay': False,
             }), video_id)
 
+        if 'error' in params:
+            raise ExtractorError(params['error']['message'], expected=True)
+
         video_url = params['flv']
 
         webpage = self._download_webpage(url, video_id)

From 2a8137272d7d463648ccf53d25d78660209ad928 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 23 Apr 2015 18:24:44 +0800
Subject: [PATCH 1045/6219] [ustream] Add an alternative approach to extract
 title (fixes #5128)

---
 youtube_dl/extractor/ustream.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ustream.py b/youtube_dl/extractor/ustream.py
index f8893b6cd..a395dc5da 100644
--- a/youtube_dl/extractor/ustream.py
+++ b/youtube_dl/extractor/ustream.py
@@ -58,7 +58,16 @@ class UstreamIE(InfoExtractor):
         self.report_extraction(video_id)
 
         video_title = self._html_search_regex(r'data-title="(?P<title>.+)"',
-                                              webpage, 'title')
+                                              webpage, 'title', default=None)
+
+        if not video_title:
+            try:
+                video_title = params['moduleConfig']['meta']['title']
+            except KeyError:
+                pass
+
+        if not video_title:
+            video_title = 'Ustream video ' + video_id
 
         uploader = self._html_search_regex(r'data-content-type="channel".*?>(?P<uploader>.*?)</a>',
                                            webpage, 'uploader', fatal=False, flags=re.DOTALL)

From cd9fdccde04799910fc3ec36fb1e8baa26a5b3e0 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 23 Apr 2015 18:33:25 +0800
Subject: [PATCH 1046/6219] [ustream] Try to extract uploader from JSON data
 (#5128)

---
 youtube_dl/extractor/ustream.py | 23 ++++++++++++++++++++---
 1 file changed, 20 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/ustream.py b/youtube_dl/extractor/ustream.py
index a395dc5da..c39c278ab 100644
--- a/youtube_dl/extractor/ustream.py
+++ b/youtube_dl/extractor/ustream.py
@@ -13,7 +13,7 @@ from ..utils import ExtractorError
 class UstreamIE(InfoExtractor):
     _VALID_URL = r'https?://www\.ustream\.tv/(?P<type>recorded|embed|embed/recorded)/(?P<videoID>\d+)'
     IE_NAME = 'ustream'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.ustream.tv/recorded/20274954',
         'md5': '088f151799e8f572f84eb62f17d73e5c',
         'info_dict': {
@@ -22,7 +22,18 @@ class UstreamIE(InfoExtractor):
             'uploader': 'Young Americans for Liberty',
             'title': 'Young Americans for Liberty February 7, 2012 2:28 AM',
         },
-    }
+    }, {
+        # From http://sportscanada.tv/canadagames/index.php/week2/figure-skating/444
+        # Title and uploader available only from params JSON
+        'url': 'http://www.ustream.tv/embed/recorded/59307601?ub=ff0000&lc=ff0000&oc=ffffff&uc=ffffff&v=3&wmode=direct',
+        'md5': '5a2abf40babeac9812ed20ae12d34e10',
+        'info_dict': {
+            'id': '59307601',
+            'ext': 'flv',
+            'title': '-CG11- Canada Games Figure Skating',
+            'uploader': 'sportscanadatv',
+        }
+    }]
 
     def _real_extract(self, url):
         m = re.match(self._VALID_URL, url)
@@ -70,7 +81,13 @@ class UstreamIE(InfoExtractor):
             video_title = 'Ustream video ' + video_id
 
         uploader = self._html_search_regex(r'data-content-type="channel".*?>(?P<uploader>.*?)</a>',
-                                           webpage, 'uploader', fatal=False, flags=re.DOTALL)
+                                           webpage, 'uploader', fatal=False, flags=re.DOTALL, default=None)
+
+        if not uploader:
+            try:
+                uploader = params['moduleConfig']['meta']['userName']
+            except KeyError:
+                uploader = None
 
         thumbnail = self._html_search_regex(r'<link rel="image_src" href="(?P<thumb>.*?)"',
                                             webpage, 'thumbnail', fatal=False)

From b46ed49996669a5e602042ae4d357f2ad952af58 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 23 Apr 2015 21:44:51 +0600
Subject: [PATCH 1047/6219] [cracked] Fix extraction

---
 youtube_dl/extractor/cracked.py | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/cracked.py b/youtube_dl/extractor/cracked.py
index cf763ee7e..e579863df 100644
--- a/youtube_dl/extractor/cracked.py
+++ b/youtube_dl/extractor/cracked.py
@@ -33,17 +33,25 @@ class CrackedIE(InfoExtractor):
         video_url = self._html_search_regex(
             [r'var\s+CK_vidSrc\s*=\s*"([^"]+)"', r'<video\s+src="([^"]+)"'], webpage, 'video URL')
 
-        title = self._og_search_title(webpage)
-        description = self._og_search_description(webpage)
+        title = self._search_regex(
+            [r'property="?og:title"?\s+content="([^"]+)"', r'class="?title"?>([^<]+)'],
+            webpage, 'title')
 
-        timestamp = self._html_search_regex(r'<time datetime="([^"]+)"', webpage, 'upload date', fatal=False)
+        description = self._search_regex(
+            r'name="?(?:og:)?description"?\s+content="([^"]+)"',
+            webpage, 'description', default=None)
+
+        timestamp = self._html_search_regex(
+            r'"date"\s*:\s*"([^"]+)"', webpage, 'upload date', fatal=False)
         if timestamp:
             timestamp = parse_iso8601(timestamp[:-6])
 
         view_count = str_to_int(self._html_search_regex(
-            r'<span class="views" id="viewCounts">([\d,\.]+) Views</span>', webpage, 'view count', fatal=False))
+            r'<span\s+class="?views"? id="?viewCounts"?>([\d,\.]+) Views</span>',
+            webpage, 'view count', fatal=False))
         comment_count = str_to_int(self._html_search_regex(
-            r'<span id="commentCounts">([\d,\.]+)</span>', webpage, 'comment count', fatal=False))
+            r'<span\s+id="?commentCounts"?>([\d,\.]+)</span>',
+            webpage, 'comment count', fatal=False))
 
         m = re.search(r'_(?P<width>\d+)X(?P<height>\d+)\.mp4$', video_url)
         if m:

From 6447353f52dc4b2b5926d3d9a3a28d3a7a242b4d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 23 Apr 2015 21:49:54 +0600
Subject: [PATCH 1048/6219] [cracked] Add support for youtube embeds

---
 youtube_dl/extractor/cracked.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/cracked.py b/youtube_dl/extractor/cracked.py
index e579863df..8b381dd3a 100644
--- a/youtube_dl/extractor/cracked.py
+++ b/youtube_dl/extractor/cracked.py
@@ -30,8 +30,15 @@ class CrackedIE(InfoExtractor):
 
         webpage = self._download_webpage(url, video_id)
 
+        youtube_url = self._search_regex(
+            r'<iframe[^>]+src="((?:https?:)?//www\.youtube\.com/embed/[^"]+)"',
+            webpage, 'youtube url', default=None)
+        if youtube_url:
+            return self.url_result(youtube_url)
+
         video_url = self._html_search_regex(
-            [r'var\s+CK_vidSrc\s*=\s*"([^"]+)"', r'<video\s+src="([^"]+)"'], webpage, 'video URL')
+            [r'var\s+CK_vidSrc\s*=\s*"([^"]+)"', r'<video\s+src="([^"]+)"'],
+            webpage, 'video URL')
 
         title = self._search_regex(
             [r'property="?og:title"?\s+content="([^"]+)"', r'class="?title"?>([^<]+)'],

From c610f38ba9586f21f632e115cbc15e172d60ee40 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 23 Apr 2015 21:58:50 +0600
Subject: [PATCH 1049/6219] [cracked] Update tests

---
 youtube_dl/extractor/cracked.py | 32 ++++++++++++++++++++++----------
 1 file changed, 22 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/cracked.py b/youtube_dl/extractor/cracked.py
index 8b381dd3a..b2b151558 100644
--- a/youtube_dl/extractor/cracked.py
+++ b/youtube_dl/extractor/cracked.py
@@ -11,18 +11,30 @@ from ..utils import (
 
 class CrackedIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?cracked\.com/video_(?P<id>\d+)_[\da-z-]+\.html'
-    _TEST = {
-        'url': 'http://www.cracked.com/video_19006_4-plot-holes-you-didnt-notice-in-your-favorite-movies.html',
-        'md5': '4b29a5eeec292cd5eca6388c7558db9e',
+    _TESTS = [{
+        'url': 'http://www.cracked.com/video_19070_if-animal-actors-got-e21-true-hollywood-stories.html',
+        'md5': '89b90b9824e3806ca95072c4d78f13f7',
         'info_dict': {
-            'id': '19006',
+            'id': '19070',
             'ext': 'mp4',
-            'title': '4 Plot Holes You Didn\'t Notice in Your Favorite Movies',
-            'description': 'md5:3b909e752661db86007d10e5ec2df769',
-            'timestamp': 1405659600,
-            'upload_date': '20140718',
+            'title': 'If Animal Actors Got E! True Hollywood Stories',
+            'timestamp': 1404954000,
+            'upload_date': '20140710',
         }
-    }
+    }, {
+        # youtube embed
+        'url': 'http://www.cracked.com/video_19006_4-plot-holes-you-didnt-notice-in-your-favorite-movies.html',
+        'md5': 'ccd52866b50bde63a6ef3b35016ba8c7',
+        'info_dict': {
+            'id': 'EjI00A3rZD0',
+            'ext': 'mp4',
+            'title': "4 Plot Holes You Didn't Notice in Your Favorite Movies - The Spit Take",
+            'description': 'md5:c603708c718b796fe6079e2b3351ffc7',
+            'upload_date': '20140725',
+            'uploader_id': 'Cracked',
+            'uploader': 'Cracked',
+        }
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
@@ -34,7 +46,7 @@ class CrackedIE(InfoExtractor):
             r'<iframe[^>]+src="((?:https?:)?//www\.youtube\.com/embed/[^"]+)"',
             webpage, 'youtube url', default=None)
         if youtube_url:
-            return self.url_result(youtube_url)
+            return self.url_result(youtube_url, 'Youtube')
 
         video_url = self._html_search_regex(
             [r'var\s+CK_vidSrc\s*=\s*"([^"]+)"', r'<video\s+src="([^"]+)"'],

From d7403332248a3893810f0461a682229552e1fd9e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 23 Apr 2015 21:59:18 +0600
Subject: [PATCH 1050/6219] [cracked] Modernize

---
 youtube_dl/extractor/cracked.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/cracked.py b/youtube_dl/extractor/cracked.py
index b2b151558..94d03ce2a 100644
--- a/youtube_dl/extractor/cracked.py
+++ b/youtube_dl/extractor/cracked.py
@@ -37,8 +37,7 @@ class CrackedIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id)
 

From 4515cb43cadd583de40b38021202595248080fb8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 23 Apr 2015 22:11:09 +0600
Subject: [PATCH 1051/6219] [xattrpp] Fix typo

---
 youtube_dl/postprocessor/xattrpp.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/postprocessor/xattrpp.py b/youtube_dl/postprocessor/xattrpp.py
index b74adff43..93d0abcf6 100644
--- a/youtube_dl/postprocessor/xattrpp.py
+++ b/youtube_dl/postprocessor/xattrpp.py
@@ -44,7 +44,7 @@ class XAttrMetadataPP(PostProcessor):
             if version_tuple(xattr.__version__) < version_tuple(pyxattr_required_version):
                 self._downloader.report_warning(
                     'python-pyxattr is detected but is too old. '
-                    'yourube-dl requires %s or above while your version is %s. '
+                    'youtube-dl requires %s or above while your version is %s. '
                     'Falling back to other xattr implementations' % (
                         pyxattr_required_version, xattr.__version__))
 

From 02f502f435f01d82b71d52e8b5fdb9369d76873d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 24 Apr 2015 01:34:57 +0600
Subject: [PATCH 1052/6219] [README] Document on how to enable old format
 selection behavior (#5510, #5511)

---
 README.md | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 7dbe66995..3ea69b637 100644
--- a/README.md
+++ b/README.md
@@ -270,7 +270,9 @@ The simplest case is requesting a specific format, for example `-f 22`. You can
 
 If you are want to download multiple videos and they don't have the same formats available, you can specify the order of preference using slashes, as in `-f 22/17/18`. You can also filter the video results by putting a condition in brackets, as in `-f "best[height=720]"` (or `-f "[filesize>10M]"`).  This works for filesize, height, width, tbr, abr, vbr, asr, and fps and the comparisons <, <=, >, >=, =, != and for ext, acodec, vcodec, container, and protocol and the comparisons =, != . Formats for which the value is not known are excluded unless you put a question mark (?) after the operator. You can combine format filters, so  `-f "[height <=? 720][tbr>500]"` selects up to 720p videos (or videos where the height is not known) with a bitrate of at least 500 KBit/s. Use commas to download multiple formats, such as `-f 136/137/mp4/bestvideo,140/m4a/bestaudio`. You can merge the video and audio of two formats into a single file using `-f <video-format>+<audio-format>` (requires ffmpeg or avconv), for example `-f bestvideo+bestaudio`.
 
-youtube-dl uses `-f bestvideo+bestaudio/best` if ffmpeg or avconv are installed (`best` is needed for videos that don't come from YouTube because they don't provide the audio and video in two different files). If you want to only download some dash formats (for example if you are not interested in getting videos with a resolution higher than 1080p), you can add `-f bestvideo[height<=?1080]+bestaudio/best` to your configuration file.
+Since the end of April 2015 youtube-dl uses `-f bestvideo+bestaudio/best` as default format selection (see #5447, #5456). If ffmpeg or avconv are installed this results in downloading `bestvideo` and `bestaudio` separately and muxing them together into a single file giving the best overall quality available. Otherwise it falls back to `best` and results in downloading best available quality served as a single file. `best` is also needed for videos that don't come from YouTube because they don't provide the audio and video in two different files. If you want to only download some dash formats (for example if you are not interested in getting videos with a resolution higher than 1080p), you can add `-f bestvideo[height<=?1080]+bestaudio/best` to your configuration file.
+
+If you want to preserve the old format selection behavior (pre-April 2015), i.e. you want to download best available quality media served as a single file, you should explicitly specify your choice with `-f best`. You may want to add it to the [configuration file](https://github.com/rg3/youtube-dl/blob/master/README.md#configuration) in order not to type it every time you run youtube-dl.
 
 # VIDEO SELECTION
 

From 7fb993e1f4105d573a5e42f71a2b3c51c0794a9d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 24 Apr 2015 01:38:02 +0600
Subject: [PATCH 1053/6219] [README] Fix configuration file link and typo

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 3ea69b637..2b3bdde22 100644
--- a/README.md
+++ b/README.md
@@ -268,11 +268,11 @@ youtube-dl_test_video_.mp4          # A simple file name
 By default youtube-dl tries to download the best quality, but sometimes you may want to download other format.
 The simplest case is requesting a specific format, for example `-f 22`. You can get the list of available formats using `--list-formats`, you can also use a file extension (currently it supports aac, m4a, mp3, mp4, ogg, wav, webm) or the special names `best`, `bestvideo`, `bestaudio` and `worst`.
 
-If you are want to download multiple videos and they don't have the same formats available, you can specify the order of preference using slashes, as in `-f 22/17/18`. You can also filter the video results by putting a condition in brackets, as in `-f "best[height=720]"` (or `-f "[filesize>10M]"`).  This works for filesize, height, width, tbr, abr, vbr, asr, and fps and the comparisons <, <=, >, >=, =, != and for ext, acodec, vcodec, container, and protocol and the comparisons =, != . Formats for which the value is not known are excluded unless you put a question mark (?) after the operator. You can combine format filters, so  `-f "[height <=? 720][tbr>500]"` selects up to 720p videos (or videos where the height is not known) with a bitrate of at least 500 KBit/s. Use commas to download multiple formats, such as `-f 136/137/mp4/bestvideo,140/m4a/bestaudio`. You can merge the video and audio of two formats into a single file using `-f <video-format>+<audio-format>` (requires ffmpeg or avconv), for example `-f bestvideo+bestaudio`.
+If you want to download multiple videos and they don't have the same formats available, you can specify the order of preference using slashes, as in `-f 22/17/18`. You can also filter the video results by putting a condition in brackets, as in `-f "best[height=720]"` (or `-f "[filesize>10M]"`).  This works for filesize, height, width, tbr, abr, vbr, asr, and fps and the comparisons <, <=, >, >=, =, != and for ext, acodec, vcodec, container, and protocol and the comparisons =, != . Formats for which the value is not known are excluded unless you put a question mark (?) after the operator. You can combine format filters, so  `-f "[height <=? 720][tbr>500]"` selects up to 720p videos (or videos where the height is not known) with a bitrate of at least 500 KBit/s. Use commas to download multiple formats, such as `-f 136/137/mp4/bestvideo,140/m4a/bestaudio`. You can merge the video and audio of two formats into a single file using `-f <video-format>+<audio-format>` (requires ffmpeg or avconv), for example `-f bestvideo+bestaudio`.
 
 Since the end of April 2015 youtube-dl uses `-f bestvideo+bestaudio/best` as default format selection (see #5447, #5456). If ffmpeg or avconv are installed this results in downloading `bestvideo` and `bestaudio` separately and muxing them together into a single file giving the best overall quality available. Otherwise it falls back to `best` and results in downloading best available quality served as a single file. `best` is also needed for videos that don't come from YouTube because they don't provide the audio and video in two different files. If you want to only download some dash formats (for example if you are not interested in getting videos with a resolution higher than 1080p), you can add `-f bestvideo[height<=?1080]+bestaudio/best` to your configuration file.
 
-If you want to preserve the old format selection behavior (pre-April 2015), i.e. you want to download best available quality media served as a single file, you should explicitly specify your choice with `-f best`. You may want to add it to the [configuration file](https://github.com/rg3/youtube-dl/blob/master/README.md#configuration) in order not to type it every time you run youtube-dl.
+If you want to preserve the old format selection behavior (pre-April 2015), i.e. you want to download best available quality media served as a single file, you should explicitly specify your choice with `-f best`. You may want to add it to the [configuration file](https://github.com/rg3/youtube-dl#configuration) in order not to type it every time you run youtube-dl.
 
 # VIDEO SELECTION
 

From 4d6a3ff411d494e2f3168bca0331317718b5d9a9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 24 Apr 2015 01:41:36 +0600
Subject: [PATCH 1054/6219] [README] Finally fix configuration file link

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 2b3bdde22..7040be1ec 100644
--- a/README.md
+++ b/README.md
@@ -272,7 +272,7 @@ If you want to download multiple videos and they don't have the same formats ava
 
 Since the end of April 2015 youtube-dl uses `-f bestvideo+bestaudio/best` as default format selection (see #5447, #5456). If ffmpeg or avconv are installed this results in downloading `bestvideo` and `bestaudio` separately and muxing them together into a single file giving the best overall quality available. Otherwise it falls back to `best` and results in downloading best available quality served as a single file. `best` is also needed for videos that don't come from YouTube because they don't provide the audio and video in two different files. If you want to only download some dash formats (for example if you are not interested in getting videos with a resolution higher than 1080p), you can add `-f bestvideo[height<=?1080]+bestaudio/best` to your configuration file.
 
-If you want to preserve the old format selection behavior (pre-April 2015), i.e. you want to download best available quality media served as a single file, you should explicitly specify your choice with `-f best`. You may want to add it to the [configuration file](https://github.com/rg3/youtube-dl#configuration) in order not to type it every time you run youtube-dl.
+If you want to preserve the old format selection behavior (pre-April 2015), i.e. you want to download best available quality media served as a single file, you should explicitly specify your choice with `-f best`. You may want to add it to the [configuration file](#configuration) in order not to type it every time you run youtube-dl.
 
 # VIDEO SELECTION
 

From a9b0d4e1f4936d93f277f098495eb4d5b770056f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 24 Apr 2015 14:09:35 +0800
Subject: [PATCH 1055/6219] [Crunchyroll] Fix extraction on Python 2.6

XPath with recursive children selection not supported
---
 youtube_dl/extractor/crunchyroll.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 6ded723c9..1c77df47e 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -263,8 +263,8 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
             streamdata = self._download_xml(
                 streamdata_req, video_id,
                 note='Downloading media info for %s' % video_format)
-            video_url = streamdata.find('.//host').text
-            video_play_path = streamdata.find('.//file').text
+            video_url = streamdata.find('./host').text
+            video_play_path = streamdata.find('./file').text
             formats.append({
                 'url': video_url,
                 'play_path': video_play_path,

From ddbed36455f8b3053f38d84b2e62e2fb5cd66eac Mon Sep 17 00:00:00 2001
From: pulpe <Pulpan3@gmail.com>
Date: Fri, 24 Apr 2015 08:48:49 +0200
Subject: [PATCH 1056/6219] [embedthumbnail] Add support for mp3 cover
 embedding

---
 youtube_dl/__init__.py                     |  4 +-
 youtube_dl/postprocessor/__init__.py       |  4 +-
 youtube_dl/postprocessor/atomicparsley.py  | 62 ---------------
 youtube_dl/postprocessor/embedthumbnail.py | 87 ++++++++++++++++++++++
 youtube_dl/postprocessor/ffmpeg.py         |  2 +
 5 files changed, 92 insertions(+), 67 deletions(-)
 delete mode 100644 youtube_dl/postprocessor/atomicparsley.py
 create mode 100644 youtube_dl/postprocessor/embedthumbnail.py

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 1c8b411b7..d7759db68 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -240,9 +240,7 @@ def _real_main(argv=None):
     if opts.xattrs:
         postprocessors.append({'key': 'XAttrMetadata'})
     if opts.embedthumbnail:
-        if not opts.addmetadata:
-            postprocessors.append({'key': 'FFmpegAudioFix'})
-        postprocessors.append({'key': 'AtomicParsley'})
+        postprocessors.append({'key': 'EmbedThumbnail'})
     # Please keep ExecAfterDownload towards the bottom as it allows the user to modify the final file in any way.
     # So if the user is able to remove the file before your postprocessor runs it might cause a few problems.
     if opts.exec_cmd:
diff --git a/youtube_dl/postprocessor/__init__.py b/youtube_dl/postprocessor/__init__.py
index f39acadce..ab7f1a29a 100644
--- a/youtube_dl/postprocessor/__init__.py
+++ b/youtube_dl/postprocessor/__init__.py
@@ -1,6 +1,6 @@
 from __future__ import unicode_literals
 
-from .atomicparsley import AtomicParsleyPP
+from .embedthumbnail import EmbedThumbnailPP
 from .ffmpeg import (
     FFmpegPostProcessor,
     FFmpegAudioFixPP,
@@ -23,7 +23,7 @@ def get_postprocessor(key):
 
 
 __all__ = [
-    'AtomicParsleyPP',
+    'EmbedThumbnailPP',
     'ExecAfterDownloadPP',
     'FFmpegAudioFixPP',
     'FFmpegEmbedSubtitlePP',
diff --git a/youtube_dl/postprocessor/atomicparsley.py b/youtube_dl/postprocessor/atomicparsley.py
deleted file mode 100644
index e4e198695..000000000
--- a/youtube_dl/postprocessor/atomicparsley.py
+++ /dev/null
@@ -1,62 +0,0 @@
-# -*- coding: utf-8 -*-
-from __future__ import unicode_literals
-
-
-import os
-import subprocess
-
-from .common import PostProcessor
-from ..compat import (
-    compat_urlretrieve,
-)
-from ..utils import (
-    check_executable,
-    encodeFilename,
-    PostProcessingError,
-    prepend_extension,
-    shell_quote
-)
-
-
-class AtomicParsleyPPError(PostProcessingError):
-    pass
-
-
-class AtomicParsleyPP(PostProcessor):
-    def run(self, info):
-        if not check_executable('AtomicParsley', ['-v']):
-            raise AtomicParsleyPPError('AtomicParsley was not found. Please install.')
-
-        filename = info['filepath']
-        temp_filename = prepend_extension(filename, 'temp')
-        temp_thumbnail = prepend_extension(filename, 'thumb')
-
-        if not info.get('thumbnail'):
-            raise AtomicParsleyPPError('Thumbnail was not found. Nothing to do.')
-
-        compat_urlretrieve(info['thumbnail'], temp_thumbnail)
-
-        cmd = ['AtomicParsley', filename, '--artwork', temp_thumbnail, '-o', temp_filename]
-
-        self._downloader.to_screen('[atomicparsley] Adding thumbnail to "%s"' % filename)
-
-        if self._downloader.params.get('verbose', False):
-            self._downloader.to_screen('[debug] AtomicParsley command line: %s' % shell_quote(cmd))
-
-        p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        stdout, stderr = p.communicate()
-
-        if p.returncode != 0:
-            msg = stderr.decode('utf-8', 'replace').strip()
-            raise AtomicParsleyPPError(msg)
-
-        os.remove(encodeFilename(temp_thumbnail))
-        # for formats that don't support thumbnails (like 3gp) AtomicParsley
-        # won't create to the temporary file
-        if b'No changes' in stdout:
-            self._downloader.report_warning('The file format doesn\'t support embedding a thumbnail')
-        else:
-            os.remove(encodeFilename(filename))
-            os.rename(encodeFilename(temp_filename), encodeFilename(filename))
-
-        return [], info
diff --git a/youtube_dl/postprocessor/embedthumbnail.py b/youtube_dl/postprocessor/embedthumbnail.py
new file mode 100644
index 000000000..b6507db27
--- /dev/null
+++ b/youtube_dl/postprocessor/embedthumbnail.py
@@ -0,0 +1,87 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+
+import os
+import subprocess
+
+from .common import PostProcessor
+from ..compat import (
+    compat_urlretrieve,
+)
+from ..utils import (
+    check_executable,
+    encodeFilename,
+    PostProcessingError,
+    prepend_extension,
+    shell_quote
+)
+
+
+class EmbedThumbnailPPError(PostProcessingError):
+    pass
+
+
+class EmbedThumbnailPP(PostProcessor):
+    def run(self, info):
+        filename = info['filepath']
+        temp_filename = prepend_extension(filename, 'temp')
+        temp_thumbnail = prepend_extension(filename, 'thumb')
+
+        if not info.get('thumbnail'):
+            raise EmbedThumbnailPPError('Thumbnail was not found. Nothing to do.')
+
+        compat_urlretrieve(info['thumbnail'], temp_thumbnail)
+
+        if info['ext'] == 'mp3':
+            if not check_executable('ffmpeg', ['-version']):
+                raise AtomicParsleyPPError('FFmpeg was not found. Please install.')
+
+            cmd = ['ffmpeg', '-i', filename, '-i', temp_thumbnail, '-c', 'copy', '-map', '0', '-map', '1', '-metadata:s:v', 'title="Album cover"', '-metadata:s:v', 'comment="Cover (Front)"', temp_filename]
+
+            self._downloader.to_screen('[ffmpeg] Adding thumbnail to "%s"' % filename)
+
+            if self._downloader.params.get('verbose', False):
+                self._downloader.to_screen('[debug] FFmpeg command line: %s' % shell_quote(cmd))
+
+            p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+            stdout, stderr = p.communicate()
+
+            if p.returncode != 0:
+                msg = stderr.decode('utf-8', 'replace').strip()
+                raise EmbedThumbnailPPError(msg)
+
+            os.remove(encodeFilename(temp_thumbnail))
+            os.remove(encodeFilename(filename))
+            os.rename(encodeFilename(temp_filename), encodeFilename(filename))
+
+        elif info['ext'] == 'm4a':
+            if not check_executable('AtomicParsley', ['-v']):
+                raise EmbedThumbnailPPError('AtomicParsley was not found. Please install.')
+
+            cmd = ['AtomicParsley', filename, '--artwork', temp_thumbnail, '-o', temp_filename]
+
+            self._downloader.to_screen('[atomicparsley] Adding thumbnail to "%s"' % filename)
+
+            if self._downloader.params.get('verbose', False):
+                self._downloader.to_screen('[debug] AtomicParsley command line: %s' % shell_quote(cmd))
+
+            p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+            stdout, stderr = p.communicate()
+
+            if p.returncode != 0:
+                msg = stderr.decode('utf-8', 'replace').strip()
+                raise EmbedThumbnailPPError(msg)
+
+            os.remove(encodeFilename(temp_thumbnail))
+            # for formats that don't support thumbnails (like 3gp) AtomicParsley
+            # won't create to the temporary file
+            if b'No changes' in stdout:
+                self._downloader.report_warning('The file format doesn\'t support embedding a thumbnail')
+            else:
+                os.remove(encodeFilename(filename))
+                os.rename(encodeFilename(temp_filename), encodeFilename(filename))
+        else:
+            raise EmbedThumbnailPPError('Only mp3 and m4a are supported for thumbnail embedding for now.')
+
+        return [], info
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index df6fb6665..c01f2eedd 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -280,6 +280,8 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
                 errnote='Cannot update utime of audio file')
 
         information['filepath'] = new_path
+        information['ext'] = extension
+
         return [path], information
 
 

From 31fd9c76013faaa86b0c515d9305de548856ef84 Mon Sep 17 00:00:00 2001
From: pulpe <Pulpan3@gmail.com>
Date: Fri, 24 Apr 2015 09:08:57 +0200
Subject: [PATCH 1057/6219] [embedthumbnail] use FFmpegPostProcessor for mp3

---
 youtube_dl/postprocessor/embedthumbnail.py | 21 ++++++---------------
 1 file changed, 6 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/postprocessor/embedthumbnail.py b/youtube_dl/postprocessor/embedthumbnail.py
index b6507db27..a2d6b14db 100644
--- a/youtube_dl/postprocessor/embedthumbnail.py
+++ b/youtube_dl/postprocessor/embedthumbnail.py
@@ -5,7 +5,8 @@ from __future__ import unicode_literals
 import os
 import subprocess
 
-from .common import PostProcessor
+from .ffmpeg import FFmpegPostProcessor
+
 from ..compat import (
     compat_urlretrieve,
 )
@@ -22,7 +23,7 @@ class EmbedThumbnailPPError(PostProcessingError):
     pass
 
 
-class EmbedThumbnailPP(PostProcessor):
+class EmbedThumbnailPP(FFmpegPostProcessor):
     def run(self, info):
         filename = info['filepath']
         temp_filename = prepend_extension(filename, 'temp')
@@ -34,22 +35,12 @@ class EmbedThumbnailPP(PostProcessor):
         compat_urlretrieve(info['thumbnail'], temp_thumbnail)
 
         if info['ext'] == 'mp3':
-            if not check_executable('ffmpeg', ['-version']):
-                raise AtomicParsleyPPError('FFmpeg was not found. Please install.')
-
-            cmd = ['ffmpeg', '-i', filename, '-i', temp_thumbnail, '-c', 'copy', '-map', '0', '-map', '1', '-metadata:s:v', 'title="Album cover"', '-metadata:s:v', 'comment="Cover (Front)"', temp_filename]
+            options = ['-i', temp_thumbnail, '-c', 'copy', '-map', '0', '-map', '1',
+                '-metadata:s:v', 'title="Album cover"', '-metadata:s:v', 'comment="Cover (Front)"']
 
             self._downloader.to_screen('[ffmpeg] Adding thumbnail to "%s"' % filename)
 
-            if self._downloader.params.get('verbose', False):
-                self._downloader.to_screen('[debug] FFmpeg command line: %s' % shell_quote(cmd))
-
-            p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-            stdout, stderr = p.communicate()
-
-            if p.returncode != 0:
-                msg = stderr.decode('utf-8', 'replace').strip()
-                raise EmbedThumbnailPPError(msg)
+            self.run_ffmpeg(filename, temp_filename, options)
 
             os.remove(encodeFilename(temp_thumbnail))
             os.remove(encodeFilename(filename))

From c0ea8ebb9bc628e3787e739dbf58bd0e59c83f87 Mon Sep 17 00:00:00 2001
From: pulpe <Pulpan3@gmail.com>
Date: Fri, 24 Apr 2015 09:11:39 +0200
Subject: [PATCH 1058/6219] [ffmpeg] Remove unneeded class

---
 youtube_dl/postprocessor/ffmpeg.py | 15 ---------------
 1 file changed, 15 deletions(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index c01f2eedd..7a952963e 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -590,21 +590,6 @@ class FFmpegMergerPP(FFmpegPostProcessor):
         return info['__files_to_merge'], info
 
 
-class FFmpegAudioFixPP(FFmpegPostProcessor):
-    def run(self, info):
-        filename = info['filepath']
-        temp_filename = prepend_extension(filename, 'temp')
-
-        options = ['-vn', '-acodec', 'copy']
-        self._downloader.to_screen('[ffmpeg] Fixing audio file "%s"' % filename)
-        self.run_ffmpeg(filename, temp_filename, options)
-
-        os.remove(encodeFilename(filename))
-        os.rename(encodeFilename(temp_filename), encodeFilename(filename))
-
-        return [], info
-
-
 class FFmpegFixupStretchedPP(FFmpegPostProcessor):
     def run(self, info):
         stretched_ratio = info.get('stretched_ratio')

From 10fb7710e85a240cc4bd065b98fd3f5a0c3d10ca Mon Sep 17 00:00:00 2001
From: pulpe <Pulpan3@gmail.com>
Date: Fri, 24 Apr 2015 09:17:46 +0200
Subject: [PATCH 1059/6219] Forgot to clean the remains of class

---
 youtube_dl/postprocessor/__init__.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/youtube_dl/postprocessor/__init__.py b/youtube_dl/postprocessor/__init__.py
index ab7f1a29a..0d8ef6ca2 100644
--- a/youtube_dl/postprocessor/__init__.py
+++ b/youtube_dl/postprocessor/__init__.py
@@ -3,7 +3,6 @@ from __future__ import unicode_literals
 from .embedthumbnail import EmbedThumbnailPP
 from .ffmpeg import (
     FFmpegPostProcessor,
-    FFmpegAudioFixPP,
     FFmpegEmbedSubtitlePP,
     FFmpegExtractAudioPP,
     FFmpegFixupStretchedPP,
@@ -25,7 +24,6 @@ def get_postprocessor(key):
 __all__ = [
     'EmbedThumbnailPP',
     'ExecAfterDownloadPP',
-    'FFmpegAudioFixPP',
     'FFmpegEmbedSubtitlePP',
     'FFmpegExtractAudioPP',
     'FFmpegFixupM4aPP',

From 25f7d1beba61ad9a5d013d423f169eafb05d76a5 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 24 Apr 2015 22:25:34 +0800
Subject: [PATCH 1060/6219] [gdcvault] Extend _VALID_URL (fixes #5236)

---
 youtube_dl/extractor/gdcvault.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/gdcvault.py b/youtube_dl/extractor/gdcvault.py
index 51796f3a4..f634e6285 100644
--- a/youtube_dl/extractor/gdcvault.py
+++ b/youtube_dl/extractor/gdcvault.py
@@ -11,7 +11,7 @@ from ..utils import remove_end
 
 
 class GDCVaultIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?gdcvault\.com/play/(?P<id>\d+)/(?P<name>(\w|-)+)'
+    _VALID_URL = r'https?://(?:www\.)?gdcvault\.com/play/(?P<id>\d+)/(?P<name>(\w|-)+)?'
     _NETRC_MACHINE = 'gdcvault'
     _TESTS = [
         {
@@ -43,6 +43,10 @@ class GDCVaultIE(InfoExtractor):
                 'title': 'Thexder Meets Windows 95, or Writing Great Games in the Windows 95 Environment',
             },
             'skip': 'Requires login',
+        },
+        {
+            'url': 'http://gdcvault.com/play/1020791/',
+            'only_matching': True,
         }
     ]
 

From c8ff645766aa56742045adcf0c64b92617334eb5 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 24 Apr 2015 22:43:33 +0800
Subject: [PATCH 1061/6219] [gdcvault] Add display_id

---
 youtube_dl/extractor/gdcvault.py | 21 ++++++++++++++-------
 1 file changed, 14 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/gdcvault.py b/youtube_dl/extractor/gdcvault.py
index f634e6285..43f916412 100644
--- a/youtube_dl/extractor/gdcvault.py
+++ b/youtube_dl/extractor/gdcvault.py
@@ -19,6 +19,7 @@ class GDCVaultIE(InfoExtractor):
             'md5': '7ce8388f544c88b7ac11c7ab1b593704',
             'info_dict': {
                 'id': '1019721',
+                'display_id': 'Doki-Doki-Universe-Sweet-Simple',
                 'ext': 'mp4',
                 'title': 'Doki-Doki Universe: Sweet, Simple and Genuine (GDC Next 10)'
             }
@@ -27,6 +28,7 @@ class GDCVaultIE(InfoExtractor):
             'url': 'http://www.gdcvault.com/play/1015683/Embracing-the-Dark-Art-of',
             'info_dict': {
                 'id': '1015683',
+                'display_id': 'Embracing-the-Dark-Art-of',
                 'ext': 'flv',
                 'title': 'Embracing the Dark Art of Mathematical Modeling in AI'
             },
@@ -39,6 +41,7 @@ class GDCVaultIE(InfoExtractor):
             'md5': 'a5eb77996ef82118afbbe8e48731b98e',
             'info_dict': {
                 'id': '1015301',
+                'display_id': 'Thexder-Meets-Windows-95-or',
                 'ext': 'flv',
                 'title': 'Thexder Meets Windows 95, or Writing Great Games in the Windows 95 Environment',
             },
@@ -94,7 +97,7 @@ class GDCVaultIE(InfoExtractor):
         })
         return video_formats
 
-    def _login(self, webpage_url, video_id):
+    def _login(self, webpage_url, display_id):
         (username, password) = self._get_login_info()
         if username is None or password is None:
             self.report_warning('It looks like ' + webpage_url + ' requires a login. Try specifying a username and password and try again.')
@@ -111,9 +114,9 @@ class GDCVaultIE(InfoExtractor):
 
         request = compat_urllib_request.Request(login_url, compat_urllib_parse.urlencode(login_form))
         request.add_header('Content-Type', 'application/x-www-form-urlencoded')
-        self._download_webpage(request, video_id, 'Logging in')
-        start_page = self._download_webpage(webpage_url, video_id, 'Getting authenticated video page')
-        self._download_webpage(logout_url, video_id, 'Logging out')
+        self._download_webpage(request, display_id, 'Logging in')
+        start_page = self._download_webpage(webpage_url, display_id, 'Getting authenticated video page')
+        self._download_webpage(logout_url, display_id, 'Logging out')
 
         return start_page
 
@@ -121,8 +124,10 @@ class GDCVaultIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
 
         video_id = mobj.group('id')
+        display_id = mobj.group('name') or video_id
+
         webpage_url = 'http://www.gdcvault.com/play/' + video_id
-        start_page = self._download_webpage(webpage_url, video_id)
+        start_page = self._download_webpage(webpage_url, display_id)
 
         direct_url = self._search_regex(
             r's1\.addVariable\("file",\s*encodeURIComponent\("(/[^"]+)"\)\);',
@@ -135,6 +140,7 @@ class GDCVaultIE(InfoExtractor):
 
             return {
                 'id': video_id,
+                'display_id': display_id,
                 'url': video_url,
                 'ext': 'flv',
                 'title': title,
@@ -145,7 +151,7 @@ class GDCVaultIE(InfoExtractor):
             start_page, 'xml root', default=None)
         if xml_root is None:
             # Probably need to authenticate
-            login_res = self._login(webpage_url, video_id)
+            login_res = self._login(webpage_url, display_id)
             if login_res is None:
                 self.report_warning('Could not login.')
             else:
@@ -163,7 +169,7 @@ class GDCVaultIE(InfoExtractor):
             xml_name = self._html_search_regex(r'<iframe src=".*?\?xmlURL=xml/(?P<xml_file>.+?\.xml).*?".*?</iframe>', start_page, 'xml filename')
 
         xml_decription_url = xml_root + 'xml/' + xml_name
-        xml_description = self._download_xml(xml_decription_url, video_id)
+        xml_description = self._download_xml(xml_decription_url, display_id)
 
         video_title = xml_description.find('./metadata/title').text
         video_formats = self._parse_mp4(xml_description)
@@ -172,6 +178,7 @@ class GDCVaultIE(InfoExtractor):
 
         return {
             'id': video_id,
+            'display_id': display_id,
             'title': video_title,
             'formats': video_formats,
         }

From 5090d93f2c7e5d40cd6d7a8c9eda789f67bd1eb8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 24 Apr 2015 21:47:13 +0600
Subject: [PATCH 1062/6219] [dotsub] Fix extraction

---
 youtube_dl/extractor/dotsub.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/dotsub.py b/youtube_dl/extractor/dotsub.py
index f51d88a98..e9ca236d4 100644
--- a/youtube_dl/extractor/dotsub.py
+++ b/youtube_dl/extractor/dotsub.py
@@ -36,7 +36,8 @@ class DotsubIE(InfoExtractor):
         if not video_url:
             webpage = self._download_webpage(url, video_id)
             video_url = self._search_regex(
-                r'"file"\s*:\s*\'([^\']+)', webpage, 'video url')
+                [r'<source[^>]+src="([^"]+)"', r'"file"\s*:\s*\'([^\']+)'],
+                webpage, 'video url')
 
         return {
             'id': video_id,

From 2ad978532bd0a94fb7529f429a26e9b9966b2e1a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 24 Apr 2015 22:03:14 +0600
Subject: [PATCH 1063/6219] [ellentv] Fix extraction

---
 youtube_dl/extractor/ellentv.py | 28 ++++++++++++----------------
 1 file changed, 12 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/extractor/ellentv.py b/youtube_dl/extractor/ellentv.py
index 5154bbd7f..93affaa8f 100644
--- a/youtube_dl/extractor/ellentv.py
+++ b/youtube_dl/extractor/ellentv.py
@@ -39,24 +39,20 @@ class EllenTVIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        webpage = self._download_webpage(url, video_id)
+        webpage = self._download_webpage(
+            'http://widgets.ellentube.com/videos/%s' % video_id,
+            video_id)
 
-        video_url = self._html_search_meta('VideoURL', webpage, 'url', fatal=True)
-        title = self._og_search_title(webpage, default=None) or self._search_regex(
-            r'pageName\s*=\s*"([^"]+)"', webpage, 'title')
-        description = self._html_search_meta(
-            'description', webpage, 'description') or self._og_search_description(webpage)
-        timestamp = parse_iso8601(self._search_regex(
-            r'<span class="publish-date"><time datetime="([^"]+)">',
-            webpage, 'timestamp', fatal=False))
+        partner_id = self._search_regex(
+            r"var\s+partnerId\s*=\s*'([^']+)", webpage, 'partner id')
 
-        return {
-            'id': video_id,
-            'url': video_url,
-            'title': title,
-            'description': description,
-            'timestamp': timestamp,
-        }
+        kaltura_id = self._search_regex(
+            [r'id="kaltura_player_([^"]+)"',
+             r"_wb_entry_id\s*:\s*'([^']+)",
+             r'data-kaltura-entry-id="([^"]+)'],
+            webpage, 'kaltura id')
+
+        return self.url_result('kaltura:%s:%s' % (partner_id, kaltura_id), 'Kaltura')
 
 
 class EllenTVClipsIE(InfoExtractor):

From 870744ce8f03bdd50a1d4d4ba87913aecba35716 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 24 Apr 2015 22:07:15 +0600
Subject: [PATCH 1064/6219] [ellentv] Fix tests

---
 youtube_dl/extractor/ellentv.py | 22 ++++++----------------
 1 file changed, 6 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/extractor/ellentv.py b/youtube_dl/extractor/ellentv.py
index 93affaa8f..464aeabdb 100644
--- a/youtube_dl/extractor/ellentv.py
+++ b/youtube_dl/extractor/ellentv.py
@@ -12,29 +12,19 @@ from ..utils import (
 
 class EllenTVIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?(?:ellentv|ellentube)\.com/videos/(?P<id>[a-z0-9_-]+)'
-    _TESTS = [{
+    _TEST = {
         'url': 'http://www.ellentv.com/videos/0-ipq1gsai/',
         'md5': '8e3c576bf2e9bfff4d76565f56f94c9c',
         'info_dict': {
-            'id': '0-ipq1gsai',
+            'id': '0_ipq1gsai',
             'ext': 'mp4',
             'title': 'Fast Fingers of Fate',
-            'description': 'md5:686114ced0a032926935e9015ee794ac',
-            'timestamp': 1428033600,
+            'description': 'md5:587e79fbbd0d73b148bc596d99ce48e6',
+            'timestamp': 1428035648,
             'upload_date': '20150403',
+            'uploader_id': 'batchUser',
         }
-    }, {
-        'url': 'http://ellentube.com/videos/0-dvzmabd5/',
-        'md5': '98238118eaa2bbdf6ad7f708e3e4f4eb',
-        'info_dict': {
-            'id': '0-dvzmabd5',
-            'ext': 'mp4',
-            'title': '1 year old twin sister makes her brother laugh',
-            'description': '1 year old twin sister makes her brother laugh',
-            'timestamp': 1419542075,
-            'upload_date': '20141225',
-        }
-    }]
+    }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From 66be4b89d73adcfeb9cd31cd0c903ecca40e2152 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 24 Apr 2015 22:09:54 +0600
Subject: [PATCH 1065/6219] [ellentv:clips] Fix extraction

---
 youtube_dl/extractor/ellentv.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ellentv.py b/youtube_dl/extractor/ellentv.py
index 464aeabdb..54601786a 100644
--- a/youtube_dl/extractor/ellentv.py
+++ b/youtube_dl/extractor/ellentv.py
@@ -78,4 +78,8 @@ class EllenTVClipsIE(InfoExtractor):
             raise ExtractorError('Failed to download JSON', cause=ve)
 
     def _extract_entries(self, playlist):
-        return [self.url_result(item['url'], 'EllenTV') for item in playlist]
+        return [
+            self.url_result(
+                'kaltura:%s:%s' % (item['kaltura_partner_id'], item['kaltura_entry_id']),
+                'Kaltura')
+            for item in playlist]

From d0aefec99aa7bf2cd307b700f7c0d2c268d6762d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 24 Apr 2015 22:10:27 +0600
Subject: [PATCH 1066/6219] [ellentv:clips] Fix test

---
 youtube_dl/extractor/ellentv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ellentv.py b/youtube_dl/extractor/ellentv.py
index 54601786a..74b50bca2 100644
--- a/youtube_dl/extractor/ellentv.py
+++ b/youtube_dl/extractor/ellentv.py
@@ -54,7 +54,7 @@ class EllenTVClipsIE(InfoExtractor):
             'id': 'meryl-streep-vanessa-hudgens',
             'title': 'Meryl Streep, Vanessa Hudgens',
         },
-        'playlist_mincount': 9,
+        'playlist_mincount': 7,
     }
 
     def _real_extract(self, url):

From db37e0c273d9d139d1d6a8541146d929b659610d Mon Sep 17 00:00:00 2001
From: mrkrossxdx <mrkrossxdx@libero.it>
Date: Fri, 24 Apr 2015 20:50:34 +0200
Subject: [PATCH 1067/6219] Added support for mpv if mplayer is not available

---
 youtube_dl/downloader/mplayer.py | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/downloader/mplayer.py b/youtube_dl/downloader/mplayer.py
index 72cef30ea..551c4ae94 100644
--- a/youtube_dl/downloader/mplayer.py
+++ b/youtube_dl/downloader/mplayer.py
@@ -16,12 +16,19 @@ class MplayerFD(FileDownloader):
         self.report_destination(filename)
         tmpfilename = self.temp_name(filename)
 
-        args = [
-            'mplayer', '-really-quiet', '-vo', 'null', '-vc', 'dummy',
-            '-dumpstream', '-dumpfile', tmpfilename, url]
+        args = []
         # Check for mplayer first
-        if not check_executable('mplayer', ['-h']):
-            self.report_error('MMS or RTSP download detected but "%s" could not be run' % args[0])
+        if check_executable('mplayer', ['-h']):
+            args = [
+                'mplayer', '-really-quiet', '-vo', 'null', '-vc', 'dummy',
+                '-dumpstream', '-dumpfile', tmpfilename, url]
+                        
+        # Check for mpv
+        elif check_executable('mpv', ['-h']):
+            args = [
+                'mpv', '-really-quiet', '--vo=null', '--stream-dump=' + tmpfilename, url]
+        else:
+            self.report_error('MMS or RTSP download detected but neither "mplayer" nor "mpv" could be run')
             return False
 
         # Download using mplayer.

From a4196c3ea57cf06b6d1f961b9c40f65e173c9876 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 24 Apr 2015 22:06:22 +0200
Subject: [PATCH 1068/6219] [ellentv] Remove unused import

---
 youtube_dl/extractor/ellentv.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/ellentv.py b/youtube_dl/extractor/ellentv.py
index 74b50bca2..02c6a4615 100644
--- a/youtube_dl/extractor/ellentv.py
+++ b/youtube_dl/extractor/ellentv.py
@@ -6,7 +6,6 @@ import json
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
-    parse_iso8601,
 )
 
 

From 92995e6265d2287ee67af869e36a56a05fb86a06 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 24 Apr 2015 22:08:00 +0200
Subject: [PATCH 1069/6219] [postprocessor/embedthumbnail] Style fix

---
 youtube_dl/postprocessor/embedthumbnail.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/postprocessor/embedthumbnail.py b/youtube_dl/postprocessor/embedthumbnail.py
index a2d6b14db..7ba98a0ea 100644
--- a/youtube_dl/postprocessor/embedthumbnail.py
+++ b/youtube_dl/postprocessor/embedthumbnail.py
@@ -35,7 +35,8 @@ class EmbedThumbnailPP(FFmpegPostProcessor):
         compat_urlretrieve(info['thumbnail'], temp_thumbnail)
 
         if info['ext'] == 'mp3':
-            options = ['-i', temp_thumbnail, '-c', 'copy', '-map', '0', '-map', '1',
+            options = [
+                '-i', temp_thumbnail, '-c', 'copy', '-map', '0', '-map', '1',
                 '-metadata:s:v', 'title="Album cover"', '-metadata:s:v', 'comment="Cover (Front)"']
 
             self._downloader.to_screen('[ffmpeg] Adding thumbnail to "%s"' % filename)

From 9f3fa89f7c6300c8a7372cf7f694fd8a659785cb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 25 Apr 2015 11:59:54 +0200
Subject: [PATCH 1070/6219] Remove the --max-quality option

It doesn't work well with 'bestvideo' and 'bestaudio' because they are usually before the max quality.
Format filters should be used instead, they are more flexible and don't require the requested quality to exist for each video.
---
 test/parameters.json    |  1 -
 test/test_YoutubeDL.py  | 33 ---------------------------------
 youtube_dl/YoutubeDL.py |  8 --------
 youtube_dl/__init__.py  |  1 -
 youtube_dl/options.py   |  4 ----
 youtube_dl/utils.py     |  9 ---------
 6 files changed, 56 deletions(-)

diff --git a/test/parameters.json b/test/parameters.json
index 48b5a062e..7bf59c25f 100644
--- a/test/parameters.json
+++ b/test/parameters.json
@@ -8,7 +8,6 @@
     "forcetitle": false, 
     "forceurl": false, 
     "format": "best",
-    "format_limit": null, 
     "ignoreerrors": false, 
     "listformats": null, 
     "logtostderr": false, 
diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 820e55ec2..bb4a65ee1 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -101,39 +101,6 @@ class TestFormatSelection(unittest.TestCase):
         downloaded = ydl.downloaded_info_dicts[0]
         self.assertEqual(downloaded['ext'], 'flv')
 
-    def test_format_limit(self):
-        formats = [
-            {'format_id': 'meh', 'url': 'http://example.com/meh', 'preference': 1},
-            {'format_id': 'good', 'url': 'http://example.com/good', 'preference': 2},
-            {'format_id': 'great', 'url': 'http://example.com/great', 'preference': 3},
-            {'format_id': 'excellent', 'url': 'http://example.com/exc', 'preference': 4},
-        ]
-        info_dict = _make_result(formats)
-
-        ydl = YDL()
-        ydl.process_ie_result(info_dict)
-        downloaded = ydl.downloaded_info_dicts[0]
-        self.assertEqual(downloaded['format_id'], 'excellent')
-
-        ydl = YDL({'format_limit': 'good'})
-        assert ydl.params['format_limit'] == 'good'
-        ydl.process_ie_result(info_dict.copy())
-        downloaded = ydl.downloaded_info_dicts[0]
-        self.assertEqual(downloaded['format_id'], 'good')
-
-        ydl = YDL({'format_limit': 'great', 'format': 'all'})
-        ydl.process_ie_result(info_dict.copy())
-        self.assertEqual(ydl.downloaded_info_dicts[0]['format_id'], 'meh')
-        self.assertEqual(ydl.downloaded_info_dicts[1]['format_id'], 'good')
-        self.assertEqual(ydl.downloaded_info_dicts[2]['format_id'], 'great')
-        self.assertTrue('3' in ydl.msgs[0])
-
-        ydl = YDL()
-        ydl.params['format_limit'] = 'excellent'
-        ydl.process_ie_result(info_dict.copy())
-        downloaded = ydl.downloaded_info_dicts[0]
-        self.assertEqual(downloaded['format_id'], 'excellent')
-
     def test_format_selection(self):
         formats = [
             {'format_id': '35', 'ext': 'mp4', 'preference': 1, 'url': TEST_URL},
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 0fdcf1b0b..977141881 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -64,7 +64,6 @@ from .utils import (
     sanitize_path,
     std_headers,
     subtitles_filename,
-    takewhile_inclusive,
     UnavailableVideoError,
     url_basename,
     version_tuple,
@@ -135,7 +134,6 @@ class YoutubeDL(object):
                        (or video) as a single JSON line.
     simulate:          Do not download the video files.
     format:            Video format code. See options.py for more information.
-    format_limit:      Highest quality format to try.
     outtmpl:           Template for output names.
     restrictfilenames: Do not allow "&" and spaces in file names
     ignoreerrors:      Do not stop on download errors.
@@ -1068,12 +1066,6 @@ class YoutubeDL(object):
             full_format_info.update(format)
             format['http_headers'] = self._calc_headers(full_format_info)
 
-        format_limit = self.params.get('format_limit', None)
-        if format_limit:
-            formats = list(takewhile_inclusive(
-                lambda f: f['format_id'] != format_limit, formats
-            ))
-
         # TODO Central sorting goes here
 
         if formats[0] is not info_dict:
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index d7759db68..c88489f29 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -283,7 +283,6 @@ def _real_main(argv=None):
         'simulate': opts.simulate or any_getting,
         'skip_download': opts.skip_download,
         'format': opts.format,
-        'format_limit': opts.format_limit,
         'listformats': opts.listformats,
         'outtmpl': outtmpl,
         'autonumber_size': opts.autonumber_size,
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 39c38c980..4c9d39d9a 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -331,10 +331,6 @@ def parseOpts(overrideArguments=None):
         '--prefer-free-formats',
         action='store_true', dest='prefer_free_formats', default=False,
         help='Prefer free video formats unless a specific one is requested')
-    video_format.add_option(
-        '--max-quality',
-        action='store', dest='format_limit', metavar='FORMAT',
-        help='Highest quality format to download')
     video_format.add_option(
         '-F', '--list-formats',
         action='store_true', dest='listformats',
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index edeee1853..c69d3e165 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1109,15 +1109,6 @@ def shell_quote(args):
     return ' '.join(quoted_args)
 
 
-def takewhile_inclusive(pred, seq):
-    """ Like itertools.takewhile, but include the latest evaluated element
-        (the first element so that Not pred(e)) """
-    for e in seq:
-        yield e
-        if not pred(e):
-            return
-
-
 def smuggle_url(url, data):
     """ Pass additional data in a URL for internal use. """
 

From 0d1bd5d62f29cc1a02c4dd5cea55fe018f9fea97 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 25 Apr 2015 15:14:16 +0200
Subject: [PATCH 1071/6219] README: remove --max-quality

---
 README.md | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index 7040be1ec..948e0a4b9 100644
--- a/README.md
+++ b/README.md
@@ -188,7 +188,6 @@ which means you can modify it, redistribute it or use it however you like.
     -f, --format FORMAT              Video format code, see the "FORMAT SELECTION" for all the info
     --all-formats                    Download all available video formats
     --prefer-free-formats            Prefer free video formats unless a specific one is requested
-    --max-quality FORMAT             Highest quality format to download
     -F, --list-formats               List all available formats
     --youtube-skip-dash-manifest     Do not download the DASH manifest on YouTube videos
     --merge-output-format FORMAT     If a merge is required (e.g. bestvideo+bestaudio), output to given container format. One of mkv, mp4, ogg, webm, flv.Ignored if no
@@ -324,9 +323,9 @@ YouTube changed their playlist format in March 2014 and later on, so you'll need
 
 If you have installed youtube-dl with a package manager, pip, setup.py or a tarball, please use that to update. Note that Ubuntu packages do not seem to get updated anymore. Since we are not affiliated with Ubuntu, there is little we can do. Feel free to [report bugs](https://bugs.launchpad.net/ubuntu/+source/youtube-dl/+filebug) to the [Ubuntu packaging guys](mailto:ubuntu-motu@lists.ubuntu.com?subject=outdated%20version%20of%20youtube-dl) - all they have to do is update the package to a somewhat recent version. See above for a way to update.
 
-### Do I always have to pass in `--max-quality FORMAT`, or `-citw`?
+### Do I always have to pass `-citw`?
 
-By default, youtube-dl intends to have the best options (incidentally, if you have a convincing case that these should be different, [please file an issue where you explain that](https://yt-dl.org/bug)). Therefore, it is unnecessary and sometimes harmful to copy long option strings from webpages. In particular, `--max-quality` *limits* the video quality (so if you want the best quality, do NOT pass it in), and the only option out of `-citw` that is regularly useful is `-i`.
+By default, youtube-dl intends to have the best options (incidentally, if you have a convincing case that these should be different, [please file an issue where you explain that](https://yt-dl.org/bug)). Therefore, it is unnecessary and sometimes harmful to copy long option strings from webpages. In particular, the only option out of `-citw` that is regularly useful is `-i`.
 
 ### Can you please put the -b option back?
 

From a542e372ab3dc36d2320ba588d5bfa8011e1df5a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 25 Apr 2015 20:22:20 +0600
Subject: [PATCH 1072/6219] [mtv] Stuff lang into info URL when available

---
 youtube_dl/extractor/mtv.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index 4430b3416..b48fac5e3 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -25,6 +25,7 @@ def _media_xml_tag(tag):
 
 class MTVServicesInfoExtractor(InfoExtractor):
     _MOBILE_TEMPLATE = None
+    _LANG = None
 
     @staticmethod
     def _id_from_uri(uri):
@@ -169,8 +170,12 @@ class MTVServicesInfoExtractor(InfoExtractor):
         video_id = self._id_from_uri(uri)
         feed_url = self._get_feed_url(uri)
         data = compat_urllib_parse.urlencode({'uri': uri})
+        info_url = feed_url + '?'
+        if self._LANG:
+            info_url += 'lang=%s&' % self._LANG
+        info_url += data
         idoc = self._download_xml(
-            feed_url + '?' + data, video_id,
+            info_url, video_id,
             'Downloading info', transform_source=fix_xml_ampersands)
         return self.playlist_result(
             [self._get_video_info(item) for item in idoc.findall('.//item')])

From e4a5e772f2e691a2f462796ad08dea6e04938697 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 25 Apr 2015 20:23:42 +0600
Subject: [PATCH 1073/6219] [southpark:espanol] Add extractor (Closes #5525)

---
 youtube_dl/extractor/__init__.py  |  1 +
 youtube_dl/extractor/southpark.py | 11 +++++++++++
 2 files changed, 12 insertions(+)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index a64afa1da..ab80fd5e0 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -478,6 +478,7 @@ from .soundgasm import (
 )
 from .southpark import (
     SouthParkIE,
+    SouthParkEsIE,
     SouthparkDeIE,
 )
 from .space import SpaceIE
diff --git a/youtube_dl/extractor/southpark.py b/youtube_dl/extractor/southpark.py
index c20397b3d..5c653c8bc 100644
--- a/youtube_dl/extractor/southpark.py
+++ b/youtube_dl/extractor/southpark.py
@@ -20,6 +20,17 @@ class SouthParkIE(MTVServicesInfoExtractor):
     }]
 
 
+class SouthParkEsIE(SouthParkIE):
+    IE_NAME = 'southpark.cc.com:espanol'
+    _VALID_URL = r'https?://(?:www\.)?(?P<url>southpark\.cc\.com/episodios-en-espanol/(?P<id>.+?)(\?|#|$))'
+    _LANG = 'es'
+
+    _TESTS = [{
+        'url': 'http://southpark.cc.com/episodios-en-espanol/s01e01-cartman-consigue-una-sonda-anal#source=351c1323-0b96-402d-a8b9-40d01b2e9bde&position=1&sort=!airdate',
+        'playlist_count': 4,
+    }]
+
+
 class SouthparkDeIE(SouthParkIE):
     IE_NAME = 'southpark.de'
     _VALID_URL = r'https?://(www\.)?(?P<url>southpark\.de/(clips|alle-episoden)/(?P<id>.+?)(\?|#|$))'

From 857f00ed94ea04575a19155d8813b7b599bbd6a5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 25 Apr 2015 20:24:15 +0600
Subject: [PATCH 1074/6219] [southpark] Improve some _VALID_URL's

---
 youtube_dl/extractor/southpark.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/southpark.py b/youtube_dl/extractor/southpark.py
index 5c653c8bc..77758bbed 100644
--- a/youtube_dl/extractor/southpark.py
+++ b/youtube_dl/extractor/southpark.py
@@ -5,7 +5,7 @@ from .mtv import MTVServicesInfoExtractor
 
 class SouthParkIE(MTVServicesInfoExtractor):
     IE_NAME = 'southpark.cc.com'
-    _VALID_URL = r'https?://(www\.)?(?P<url>southpark\.cc\.com/(clips|full-episodes)/(?P<id>.+?)(\?|#|$))'
+    _VALID_URL = r'https?://(?:www\.)?(?P<url>southpark\.cc\.com/(?:clips|full-episodes)/(?P<id>.+?)(\?|#|$))'
 
     _FEED_URL = 'http://www.southparkstudios.com/feeds/video-player/mrss'
 
@@ -33,7 +33,7 @@ class SouthParkEsIE(SouthParkIE):
 
 class SouthparkDeIE(SouthParkIE):
     IE_NAME = 'southpark.de'
-    _VALID_URL = r'https?://(www\.)?(?P<url>southpark\.de/(clips|alle-episoden)/(?P<id>.+?)(\?|#|$))'
+    _VALID_URL = r'https?://(?:www\.)?(?P<url>southpark\.de/(?:clips|alle-episoden)/(?P<id>.+?)(\?|#|$))'
     _FEED_URL = 'http://www.southpark.de/feeds/video-player/mrss/'
 
     _TESTS = [{

From 529d26c3e191a3fc24b9c7618310c47929a0c11f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 25 Apr 2015 21:06:27 +0600
Subject: [PATCH 1075/6219] [orf:iptv] Update test

---
 youtube_dl/extractor/orf.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/orf.py b/youtube_dl/extractor/orf.py
index ca1a5bb3c..2e6c9872b 100644
--- a/youtube_dl/extractor/orf.py
+++ b/youtube_dl/extractor/orf.py
@@ -210,16 +210,16 @@ class ORFIPTVIE(InfoExtractor):
     _VALID_URL = r'http://iptv\.orf\.at/(?:#/)?stories/(?P<id>\d+)'
 
     _TEST = {
-        'url': 'http://iptv.orf.at/stories/2267952',
-        'md5': '26ffa4bab6dbce1eee78bbc7021016cd',
+        'url': 'http://iptv.orf.at/stories/2275236/',
+        'md5': 'c8b22af4718a4b4af58342529453e3e5',
         'info_dict': {
-            'id': '339775',
+            'id': '350612',
             'ext': 'flv',
-            'title': 'Kreml-Kritiker Nawalny wieder frei',
-            'description': 'md5:6f24e7f546d364dacd0e616a9e409236',
-            'duration': 84.729,
+            'title': 'Weitere Evakuierungen um Vulkan Calbuco',
+            'description': 'md5:d689c959bdbcf04efeddedbf2299d633',
+            'duration': 68.197,
             'thumbnail': 're:^https?://.*\.jpg$',
-            'upload_date': '20150306',
+            'upload_date': '20150425',
         },
     }
 

From 672f1bd8497f43179dcd01f8b4831564f0b42356 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 24 Apr 2015 23:46:51 +0800
Subject: [PATCH 1076/6219] [cspan] Extract subtitles

---
 youtube_dl/extractor/cspan.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/youtube_dl/extractor/cspan.py b/youtube_dl/extractor/cspan.py
index d516b1402..fbefd37d0 100644
--- a/youtube_dl/extractor/cspan.py
+++ b/youtube_dl/extractor/cspan.py
@@ -8,6 +8,7 @@ from ..utils import (
     unescapeHTML,
     find_xpath_attr,
     smuggle_url,
+    determine_ext,
 )
 from .senateisvp import SenateISVPIE
 
@@ -87,6 +88,10 @@ class CSpanIE(InfoExtractor):
             return self.url_result(surl, 'SenateISVP', video_id, title)
 
         files = data['video']['files']
+        try:
+            capfile = data['video']['capfile']['#text']
+        except KeyError:
+            capfile = None
 
         entries = [{
             'id': '%s_%d' % (video_id, partnum + 1),
@@ -97,6 +102,12 @@ class CSpanIE(InfoExtractor):
             'description': description,
             'thumbnail': thumbnail,
             'duration': int_or_none(f.get('length', {}).get('#text')),
+            'subtitles': {
+                'en': [{
+                    'url': capfile,
+                    'ext': determine_ext(capfile, 'dfxp')
+                }],
+            } if capfile else None,
         } for partnum, f in enumerate(files)]
 
         if len(entries) == 1:

From bf6427d2fbcbd95cd1cb640e8b894c18782a2a12 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 25 Apr 2015 23:15:05 +0800
Subject: [PATCH 1077/6219] [ffmpeg] Add dfxp (TTML) subtitles support (#3432,
 #5146)

---
 test/test_utils.py                 | 38 +++++++++++++++++++++
 youtube_dl/postprocessor/ffmpeg.py | 25 ++++++++++++++
 youtube_dl/utils.py                | 53 ++++++++++++++++++++++++++++++
 3 files changed, 116 insertions(+)

diff --git a/test/test_utils.py b/test/test_utils.py
index 2e3a6480c..17017a8c0 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -58,6 +58,8 @@ from youtube_dl.utils import (
     xpath_text,
     render_table,
     match_str,
+    parse_dfxp_time_expr,
+    dfxp2srt,
 )
 
 
@@ -581,6 +583,42 @@ ffmpeg version 2.4.4 Copyright (c) 2000-2014 the FFmpeg ...'''), '2.4.4')
             'like_count > 100 & dislike_count <? 50 & description',
             {'like_count': 190, 'dislike_count': 10}))
 
+    def test_parse_dfxp_time_expr(self):
+        self.assertEqual(parse_dfxp_time_expr(None), 0.0)
+        self.assertEqual(parse_dfxp_time_expr(''), 0.0)
+        self.assertEqual(parse_dfxp_time_expr('0.1'), 0.1)
+        self.assertEqual(parse_dfxp_time_expr('0.1s'), 0.1)
+        self.assertEqual(parse_dfxp_time_expr('00:00:01'), 1.0)
+        self.assertEqual(parse_dfxp_time_expr('00:00:01.100'), 1.1)
+
+    def test_dfxp2srt(self):
+        dfxp_data = '''<?xml version="1.0" encoding="UTF-8"?>
+            <tt xmlns="http://www.w3.org/ns/ttml" xml:lang="en" xmlns:tts="http://www.w3.org/ns/ttml#parameter">
+            <body>
+                <div xml:lang="en">
+                    <p begin="0" end="1">The following line contains Chinese characters and special symbols</p>
+                    <p begin="1" end="2">第二行<br/>♪♪</p>
+                    <p begin="2" end="3"><span>Third<br/>Line</span></p>
+                </div>
+            </body>
+            </tt>'''
+        srt_data = '''1
+00:00:00,000 --> 00:00:01,000
+The following line contains Chinese characters and special symbols
+
+2
+00:00:01,000 --> 00:00:02,000
+第二行
+♪♪
+
+3
+00:00:02,000 --> 00:00:03,000
+Third
+Line
+
+'''
+        self.assertEqual(dfxp2srt(dfxp_data), srt_data)
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 7a952963e..1765f4969 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -20,6 +20,7 @@ from ..utils import (
     prepend_extension,
     shell_quote,
     subtitles_filename,
+    dfxp2srt,
 )
 
 
@@ -651,6 +652,30 @@ class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor):
                     'format' % new_ext)
                 continue
             new_file = subtitles_filename(filename, lang, new_ext)
+
+            if ext == 'dfxp' or ext == 'ttml':
+                self._downloader.report_warning(
+                    'You have requested to convert dfxp (TTML) subtitles into another format, '
+                    'which results in style information loss')
+
+                dfxp_file = subtitles_filename(filename, lang, ext)
+                srt_file = subtitles_filename(filename, lang, 'srt')
+
+                with io.open(dfxp_file, 'rt', encoding='utf-8') as f:
+                    srt_data = dfxp2srt(f.read())
+
+                with io.open(srt_file, 'wt', encoding='utf-8') as f:
+                    f.write(srt_data)
+
+                ext = 'srt'
+                subs[lang] = {
+                    'ext': 'srt',
+                    'data': srt_data
+                }
+
+                if new_ext == 'srt':
+                    continue
+
             self.run_ffmpeg(
                 subtitles_filename(filename, lang, ext),
                 new_file, ['-f', new_format])
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index edeee1853..5e1c4525d 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1800,6 +1800,59 @@ def match_filter_func(filter_str):
     return _match_func
 
 
+def parse_dfxp_time_expr(time_expr):
+    if not time_expr:
+        return 0.0
+
+    mobj = re.match(r'^(?P<time_offset>\d+(?:\.\d+)?)s?$', time_expr)
+    if mobj:
+        return float(mobj.group('time_offset'))
+
+    mobj = re.match(r'^(\d+):(\d\d):(\d\d(?:\.\d+)?)$', time_expr)
+    if mobj:
+        return 3600 * int(mobj.group(1)) + 60 * int(mobj.group(2)) + float(mobj.group(3))
+
+
+def format_srt_time(seconds):
+    (mins, secs) = divmod(seconds, 60)
+    (hours, mins) = divmod(mins, 60)
+    millisecs = (secs - int(secs)) * 1000
+    secs = int(secs)
+    return '%02d:%02d:%02d,%03d' % (hours, mins, secs, millisecs)
+
+
+def dfxp2srt(dfxp_data):
+    _x = functools.partial(xpath_with_ns, ns_map={'ttml': 'http://www.w3.org/ns/ttml'})
+
+    def parse_node(node):
+        str_or_empty = functools.partial(str_or_none, default='')
+
+        out = str_or_empty(node.text)
+
+        for child in node:
+            if child.tag == _x('ttml:br'):
+                out += '\n' + str_or_empty(child.tail)
+            elif child.tag == _x('ttml:span'):
+                out += str_or_empty(parse_node(child))
+            else:
+                out += str_or_empty(xml.etree.ElementTree.tostring(child))
+
+        return out
+
+    dfxp = xml.etree.ElementTree.fromstring(dfxp_data.encode('utf-8'))
+    out = []
+    paras = dfxp.findall(_x('.//ttml:p'))
+
+    for para, index in zip(paras, itertools.count(1)):
+        out.append('%d\n%s --> %s\n%s\n\n' % (
+            index,
+            format_srt_time(parse_dfxp_time_expr(para.attrib.get('begin'))),
+            format_srt_time(parse_dfxp_time_expr(para.attrib.get('end'))),
+            parse_node(para)))
+
+    return ''.join(out)
+
+
 class PerRequestProxyHandler(compat_urllib_request.ProxyHandler):
     def __init__(self, proxies=None):
         # Set default handlers

From f54bab4d6777535390d64502bd20bb5a81b5468a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 25 Apr 2015 22:39:50 +0600
Subject: [PATCH 1078/6219] [instagram] Improve _VALID_URL

---
 youtube_dl/extractor/instagram.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/instagram.py b/youtube_dl/extractor/instagram.py
index b020e2621..d219ceb55 100644
--- a/youtube_dl/extractor/instagram.py
+++ b/youtube_dl/extractor/instagram.py
@@ -9,7 +9,7 @@ from ..utils import (
 
 
 class InstagramIE(InfoExtractor):
-    _VALID_URL = r'http://instagram\.com/p/(?P<id>.*?)/'
+    _VALID_URL = r'https?://instagram\.com/p/(?P<id>[\da-zA-Z]+)'
     _TEST = {
         'url': 'http://instagram.com/p/aye83DjauH/?foo=bar#abc',
         'md5': '0d2da106a9d2631273e192b372806516',

From d2d8248f68c029472bb81eacf4aa15987729977f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 25 Apr 2015 22:42:15 +0600
Subject: [PATCH 1079/6219] [instagram] Modernize

---
 youtube_dl/extractor/instagram.py | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/instagram.py b/youtube_dl/extractor/instagram.py
index d219ceb55..65f6ca103 100644
--- a/youtube_dl/extractor/instagram.py
+++ b/youtube_dl/extractor/instagram.py
@@ -3,9 +3,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import (
-    int_or_none,
-)
+from ..utils import int_or_none
 
 
 class InstagramIE(InfoExtractor):
@@ -23,8 +21,8 @@ class InstagramIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
+
         webpage = self._download_webpage(url, video_id)
         uploader_id = self._search_regex(r'"owner":{"username":"(.+?)"',
                                          webpage, 'uploader id', fatal=False)

From b860f5dfd45f58b2c0a92cecd28c7eb05a7a2ff4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 26 Apr 2015 00:22:13 +0600
Subject: [PATCH 1080/6219] [mplayer] Clarify error message

---
 youtube_dl/downloader/mplayer.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/mplayer.py b/youtube_dl/downloader/mplayer.py
index 551c4ae94..b1ea8daa6 100644
--- a/youtube_dl/downloader/mplayer.py
+++ b/youtube_dl/downloader/mplayer.py
@@ -28,7 +28,7 @@ class MplayerFD(FileDownloader):
             args = [
                 'mpv', '-really-quiet', '--vo=null', '--stream-dump=' + tmpfilename, url]
         else:
-            self.report_error('MMS or RTSP download detected but neither "mplayer" nor "mpv" could be run')
+            self.report_error('MMS or RTSP download detected but neither "mplayer" nor "mpv" could be run. Please install any.')
             return False
 
         # Download using mplayer.

From b874495b1fa339126c306df54aa316043ac0047b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 26 Apr 2015 00:23:16 +0600
Subject: [PATCH 1081/6219] [mplayer] Simplify

---
 youtube_dl/downloader/mplayer.py | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/youtube_dl/downloader/mplayer.py b/youtube_dl/downloader/mplayer.py
index b1ea8daa6..0cad5a148 100644
--- a/youtube_dl/downloader/mplayer.py
+++ b/youtube_dl/downloader/mplayer.py
@@ -16,14 +16,10 @@ class MplayerFD(FileDownloader):
         self.report_destination(filename)
         tmpfilename = self.temp_name(filename)
 
-        args = []
-        # Check for mplayer first
         if check_executable('mplayer', ['-h']):
             args = [
                 'mplayer', '-really-quiet', '-vo', 'null', '-vc', 'dummy',
                 '-dumpstream', '-dumpfile', tmpfilename, url]
-                        
-        # Check for mpv
         elif check_executable('mpv', ['-h']):
             args = [
                 'mpv', '-really-quiet', '--vo=null', '--stream-dump=' + tmpfilename, url]
@@ -31,7 +27,6 @@ class MplayerFD(FileDownloader):
             self.report_error('MMS or RTSP download detected but neither "mplayer" nor "mpv" could be run. Please install any.')
             return False
 
-        # Download using mplayer.
         retval = subprocess.call(args)
         if retval == 0:
             fsize = os.path.getsize(encodeFilename(tmpfilename))
@@ -46,5 +41,5 @@ class MplayerFD(FileDownloader):
             return True
         else:
             self.to_stderr('\n')
-            self.report_error('mplayer exited with code %d' % retval)
+            self.report_error('%s exited with code %d' % (args[0], retval))
             return False

From a5ebf77d877ededed377db920faf4f31644573cf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 26 Apr 2015 00:25:51 +0600
Subject: [PATCH 1082/6219] [mplayer] Rename to RTSP

---
 youtube_dl/downloader/__init__.py             | 6 +++---
 youtube_dl/downloader/{mplayer.py => rtsp.py} | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)
 rename youtube_dl/downloader/{mplayer.py => rtsp.py} (97%)

diff --git a/youtube_dl/downloader/__init__.py b/youtube_dl/downloader/__init__.py
index 9fb66e2f7..f110830c4 100644
--- a/youtube_dl/downloader/__init__.py
+++ b/youtube_dl/downloader/__init__.py
@@ -6,7 +6,7 @@ from .f4m import F4mFD
 from .hls import HlsFD
 from .hls import NativeHlsFD
 from .http import HttpFD
-from .mplayer import MplayerFD
+from .rtsp import RtspFD
 from .rtmp import RtmpFD
 
 from ..utils import (
@@ -17,8 +17,8 @@ PROTOCOL_MAP = {
     'rtmp': RtmpFD,
     'm3u8_native': NativeHlsFD,
     'm3u8': HlsFD,
-    'mms': MplayerFD,
-    'rtsp': MplayerFD,
+    'mms': RtspFD,
+    'rtsp': RtspFD,
     'f4m': F4mFD,
 }
 
diff --git a/youtube_dl/downloader/mplayer.py b/youtube_dl/downloader/rtsp.py
similarity index 97%
rename from youtube_dl/downloader/mplayer.py
rename to youtube_dl/downloader/rtsp.py
index 0cad5a148..3eb29526c 100644
--- a/youtube_dl/downloader/mplayer.py
+++ b/youtube_dl/downloader/rtsp.py
@@ -10,7 +10,7 @@ from ..utils import (
 )
 
 
-class MplayerFD(FileDownloader):
+class RtspFD(FileDownloader):
     def real_download(self, filename, info_dict):
         url = info_dict['url']
         self.report_destination(filename)

From 2a09c1b8ab348c43ece3cf246645e7335a802ca9 Mon Sep 17 00:00:00 2001
From: pulpe <Pulpan3@gmail.com>
Date: Sat, 25 Apr 2015 20:41:15 +0200
Subject: [PATCH 1083/6219] [postprocessor/embedthumbnail] Fix mp3 embedding
 with avconv (fixes #5526)

---
 youtube_dl/postprocessor/embedthumbnail.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/postprocessor/embedthumbnail.py b/youtube_dl/postprocessor/embedthumbnail.py
index 7ba98a0ea..4868a42fd 100644
--- a/youtube_dl/postprocessor/embedthumbnail.py
+++ b/youtube_dl/postprocessor/embedthumbnail.py
@@ -11,6 +11,7 @@ from ..compat import (
     compat_urlretrieve,
 )
 from ..utils import (
+    determine_ext,
     check_executable,
     encodeFilename,
     PostProcessingError,
@@ -27,7 +28,7 @@ class EmbedThumbnailPP(FFmpegPostProcessor):
     def run(self, info):
         filename = info['filepath']
         temp_filename = prepend_extension(filename, 'temp')
-        temp_thumbnail = prepend_extension(filename, 'thumb')
+        temp_thumbnail = filename + '.' + determine_ext(info['thumbnail'])
 
         if not info.get('thumbnail'):
             raise EmbedThumbnailPPError('Thumbnail was not found. Nothing to do.')

From 642f23bd81ae31bb9b5647ff515db66e3023c35f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 25 Apr 2015 22:34:28 +0200
Subject: [PATCH 1084/6219] =?UTF-8?q?[southpark]=20Use=20'=C3=B1'=20in=20t?=
 =?UTF-8?q?he=20spanish=20extractor=20name?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

IE_NAME can contain unicode characters, so it shouldn't be a problem.
---
 youtube_dl/extractor/southpark.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/southpark.py b/youtube_dl/extractor/southpark.py
index 77758bbed..e3b73295c 100644
--- a/youtube_dl/extractor/southpark.py
+++ b/youtube_dl/extractor/southpark.py
@@ -1,3 +1,4 @@
+# encoding: utf-8
 from __future__ import unicode_literals
 
 from .mtv import MTVServicesInfoExtractor
@@ -21,7 +22,7 @@ class SouthParkIE(MTVServicesInfoExtractor):
 
 
 class SouthParkEsIE(SouthParkIE):
-    IE_NAME = 'southpark.cc.com:espanol'
+    IE_NAME = 'southpark.cc.com:español'
     _VALID_URL = r'https?://(?:www\.)?(?P<url>southpark\.cc\.com/episodios-en-espanol/(?P<id>.+?)(\?|#|$))'
     _LANG = 'es'
 

From aa49acd15a92faa5cfc1d2876821743f86440c13 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 26 Apr 2015 04:29:41 +0600
Subject: [PATCH 1085/6219] [utils] Add `get_subprocess_encoding` and
 filename/argument decode counterparts

---
 youtube_dl/utils.py | 49 +++++++++++++++++++++++++++++++--------------
 1 file changed, 34 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 5e1c4525d..7de7742e3 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -371,6 +371,18 @@ def unescapeHTML(s):
         r'&([^;]+);', lambda m: _htmlentity_transform(m.group(1)), s)
 
 
+def get_subprocess_encoding():
+    if sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
+        # For subprocess calls, encode with locale encoding
+        # Refer to http://stackoverflow.com/a/9951851/35070
+        encoding = preferredencoding()
+    else:
+        encoding = sys.getfilesystemencoding()
+    if encoding is None:
+        encoding = 'utf-8'
+    return encoding
+
+
 def encodeFilename(s, for_subprocess=False):
     """
     @param s The name of the file
@@ -382,21 +394,24 @@ def encodeFilename(s, for_subprocess=False):
     if sys.version_info >= (3, 0):
         return s
 
-    if sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
-        # Pass '' directly to use Unicode APIs on Windows 2000 and up
-        # (Detecting Windows NT 4 is tricky because 'major >= 4' would
-        # match Windows 9x series as well. Besides, NT 4 is obsolete.)
-        if not for_subprocess:
-            return s
-        else:
-            # For subprocess calls, encode with locale encoding
-            # Refer to http://stackoverflow.com/a/9951851/35070
-            encoding = preferredencoding()
-    else:
-        encoding = sys.getfilesystemencoding()
-    if encoding is None:
-        encoding = 'utf-8'
-    return s.encode(encoding, 'ignore')
+    # Pass '' directly to use Unicode APIs on Windows 2000 and up
+    # (Detecting Windows NT 4 is tricky because 'major >= 4' would
+    # match Windows 9x series as well. Besides, NT 4 is obsolete.)
+    if not for_subprocess and sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
+        return s
+
+    return s.encode(get_subprocess_encoding(), 'ignore')
+
+
+def decodeFilename(b, for_subprocess=False):
+
+    if sys.version_info >= (3, 0):
+        return b
+
+    if not isinstance(b, bytes):
+        return b
+
+    return b.decode(get_subprocess_encoding(), 'ignore')
 
 
 def encodeArgument(s):
@@ -408,6 +423,10 @@ def encodeArgument(s):
     return encodeFilename(s, True)
 
 
+def decodeArgument(b):
+    return decodeFilename(b, True)
+
+
 def decodeOption(optval):
     if optval is None:
         return optval

From cd8a07a7649cbd0feea476c89882c5c85130724f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 26 Apr 2015 04:30:45 +0600
Subject: [PATCH 1086/6219] [downloader/common] Use decodeArgument

---
 youtube_dl/downloader/common.py | 15 ++++++---------
 1 file changed, 6 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index a0fc5ead0..97e755d4b 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -8,6 +8,7 @@ import time
 from ..compat import compat_str
 from ..utils import (
     encodeFilename,
+    decodeArgument,
     format_bytes,
     timeconvert,
 )
@@ -353,19 +354,15 @@ class FileDownloader(object):
         # this interface
         self._progress_hooks.append(ph)
 
-    def _debug_cmd(self, args, subprocess_encoding, exe=None):
+    def _debug_cmd(self, args, exe=None):
         if not self.params.get('verbose', False):
             return
 
-        if exe is None:
-            exe = os.path.basename(args[0])
+        str_args = [decodeArgument(a) for a in args]
+
+        if exe is None:
+            exe = os.path.basename(str_args[0])
 
-        if subprocess_encoding:
-            str_args = [
-                a.decode(subprocess_encoding) if isinstance(a, bytes) else a
-                for a in args]
-        else:
-            str_args = args
         try:
             import pipes
             shell_quote = lambda args: ' '.join(map(pipes.quote, str_args))

From 9e105a858c2aa0089764326cc3fca808c51a9ffd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 26 Apr 2015 04:32:54 +0600
Subject: [PATCH 1087/6219] [downloader/rtmp] Fix arguments encoding and
 simplify retry logic (Closes #5528)

---
 youtube_dl/downloader/rtmp.py | 21 +++++++++------------
 1 file changed, 9 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/downloader/rtmp.py b/youtube_dl/downloader/rtmp.py
index ddf5724ae..6865b5e2f 100644
--- a/youtube_dl/downloader/rtmp.py
+++ b/youtube_dl/downloader/rtmp.py
@@ -3,7 +3,6 @@ from __future__ import unicode_literals
 import os
 import re
 import subprocess
-import sys
 import time
 
 from .common import FileDownloader
@@ -11,6 +10,7 @@ from ..compat import compat_str
 from ..utils import (
     check_executable,
     encodeFilename,
+    encodeArgument,
     get_exe_version,
 )
 
@@ -121,7 +121,7 @@ class RtmpFD(FileDownloader):
         # possible. This is part of rtmpdump's normal usage, AFAIK.
         basic_args = [
             'rtmpdump', '--verbose', '-r', url,
-            '-o', encodeFilename(tmpfilename, True)]
+            '-o', tmpfilename]
         if player_url is not None:
             basic_args += ['--swfVfy', player_url]
         if page_url is not None:
@@ -154,16 +154,9 @@ class RtmpFD(FileDownloader):
         if not live and continue_dl:
             args += ['--skip', '1']
 
-        if sys.platform == 'win32' and sys.version_info < (3, 0):
-            # Windows subprocess module does not actually support Unicode
-            # on Python 2.x
-            # See http://stackoverflow.com/a/9951851/35070
-            subprocess_encoding = sys.getfilesystemencoding()
-            args = [a.encode(subprocess_encoding, 'ignore') for a in args]
-        else:
-            subprocess_encoding = None
+        args = [encodeArgument(a) for a in args]
 
-        self._debug_cmd(args, subprocess_encoding, exe='rtmpdump')
+        self._debug_cmd(args, exe='rtmpdump')
 
         RD_SUCCESS = 0
         RD_FAILED = 1
@@ -180,7 +173,11 @@ class RtmpFD(FileDownloader):
             prevsize = os.path.getsize(encodeFilename(tmpfilename))
             self.to_screen('[rtmpdump] %s bytes' % prevsize)
             time.sleep(5.0)  # This seems to be needed
-            retval = run_rtmpdump(basic_args + ['-e'] + [[], ['-k', '1']][retval == RD_FAILED])
+            args = basic_args + ['--resume']
+            if retval == RD_FAILED:
+                args += ['--skip', '1']
+            args = [encodeArgument(a) for a in args]
+            retval = run_rtmpdump(args)
             cursize = os.path.getsize(encodeFilename(tmpfilename))
             if prevsize == cursize and retval == RD_FAILED:
                 break

From 74f8654a53ed56dd813ace704151ea575963cabb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 26 Apr 2015 04:33:43 +0600
Subject: [PATCH 1088/6219] [downloader/external] Use encodeArgument

---
 youtube_dl/downloader/external.py | 14 +++-----------
 1 file changed, 3 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index 1673b2382..7ca2d3143 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -2,11 +2,11 @@ from __future__ import unicode_literals
 
 import os.path
 import subprocess
-import sys
 
 from .common import FileDownloader
 from ..utils import (
     encodeFilename,
+    encodeArgument,
 )
 
 
@@ -60,17 +60,9 @@ class ExternalFD(FileDownloader):
 
     def _call_downloader(self, tmpfilename, info_dict):
         """ Either overwrite this or implement _make_cmd """
-        cmd = self._make_cmd(tmpfilename, info_dict)
+        cmd = [encodeArgument(a) for a in self._make_cmd(tmpfilename, info_dict)]
 
-        if sys.platform == 'win32' and sys.version_info < (3, 0):
-            # Windows subprocess module does not actually support Unicode
-            # on Python 2.x
-            # See http://stackoverflow.com/a/9951851/35070
-            subprocess_encoding = sys.getfilesystemencoding()
-            cmd = [a.encode(subprocess_encoding, 'ignore') for a in cmd]
-        else:
-            subprocess_encoding = None
-        self._debug_cmd(cmd, subprocess_encoding)
+        self._debug_cmd(cmd)
 
         p = subprocess.Popen(
             cmd, stderr=subprocess.PIPE)

From 06d07c400088d35096df1406a3434c996795250d Mon Sep 17 00:00:00 2001
From: felix <m.p.isaev@yandex.com>
Date: Sun, 26 Apr 2015 14:15:29 +0200
Subject: [PATCH 1089/6219] New extractor: live.philharmoniedeparis.fr

---
 youtube_dl/extractor/__init__.py            |  1 +
 youtube_dl/extractor/philharmoniedeparis.py | 77 +++++++++++++++++++++
 2 files changed, 78 insertions(+)
 create mode 100644 youtube_dl/extractor/philharmoniedeparis.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index ab80fd5e0..641c45f43 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -377,6 +377,7 @@ from .orf import (
 from .parliamentliveuk import ParliamentLiveUKIE
 from .patreon import PatreonIE
 from .pbs import PBSIE
+from .philharmoniedeparis import PhilharmonieDeParisIE
 from .phoenix import PhoenixIE
 from .photobucket import PhotobucketIE
 from .planetaplay import PlanetaPlayIE
diff --git a/youtube_dl/extractor/philharmoniedeparis.py b/youtube_dl/extractor/philharmoniedeparis.py
new file mode 100644
index 000000000..7fc2f32ab
--- /dev/null
+++ b/youtube_dl/extractor/philharmoniedeparis.py
@@ -0,0 +1,77 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    determine_ext,
+    int_or_none,
+    parse_iso8601,
+    unified_strdate,
+)
+
+class PhilharmonieDeParisIE(InfoExtractor):
+    _VALID_URL = r'http://live\.philharmoniedeparis\.fr/concert/(?P<id>\d+)(?:/|\.html)'
+    _TESTS = [{
+        'url': 'http://live.philharmoniedeparis.fr/concert/1032066.html',
+        'info_dict': {
+            'id': '1032066',
+            'ext': 'mp4',
+            'title': "Week-end Bach. Passion selon saint Jean. Akademie für alte Musik Berlin, Rias Kammerchor, René Jacobs",
+            'upload_date': '20150404',
+        }
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+        fichier_nom = self._html_search_regex(r'\sflashvars\s*:\s*\{\s*fichier_nom\s*:\s*\'(.*?)\'\s*,', webpage, 'fichier_nom')
+
+        playlist = self._download_xml('http://live.philharmoniedeparis.fr' + fichier_nom, video_id)
+
+        concert = playlist.find('.//concert')
+
+        formats = []
+        info_dict = {
+            'id': video_id,
+            'title': concert.find('./titre').text,
+            'formats': formats,
+        }
+
+        if concert.attrib.get('heure'):
+            info_dict['timestamp'] = parse_iso8601(('%s-%s-%s%s') % (
+                concert.attrib['date'][0:4],
+                concert.attrib['date'][4:6],
+                concert.attrib['date'][6:8],
+                concert.attrib['heure']
+            ))
+        else:
+            info_dict['upload_date'] = concert.attrib['date']
+
+        fichiers = concert.find('./fichiers')
+        for fichier in fichiers.findall('./fichier'):
+            # Sometimes <ficher>s have no attributes at all. Skip them.
+            if 'url' not in fichier.attrib:
+                continue
+
+            formats.append({
+                'format_id': 'lq',
+                'url': fichiers.attrib['serveurstream'],
+                'ext': determine_ext(fichier.attrib['url']),
+                'play_path': fichier.attrib['url'],
+                'width': int_or_none(concert.attrib['largeur']),
+                'height': int_or_none(concert.attrib['hauteur']),
+                'quality': 1,
+            })
+
+            formats.append({
+                'format_id': 'hq',
+                'url': fichiers.attrib['serveurstream'],
+                'ext': determine_ext(fichier.attrib['url_hd']),
+                'play_path': fichier.attrib['url_hd'],
+                'width': int_or_none(concert.attrib['largeur_hd']),
+                'height': int_or_none(concert.attrib['hauteur_hd']),
+                'quality': 2,
+            })
+
+        return info_dict

From 4eb5c65beec671e534ad4859c073adb362e537ec Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 26 Apr 2015 22:45:20 +0200
Subject: [PATCH 1090/6219] release 2015.04.26

---
 docs/supportedsites.md | 10 +++++++---
 youtube_dl/version.py  |  2 +-
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 80e86c1b6..cc1564b7b 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -174,6 +174,7 @@
  - **Gazeta**
  - **GDCVault**
  - **generic**: Generic downloader that works on some sites
+ - **Gfycat**
  - **GiantBomb**
  - **Giga**
  - **Glide**: Glide mobile video messages (glide.me)
@@ -181,7 +182,7 @@
  - **GodTube**
  - **GoldenMoustache**
  - **Golem**
- - **GorillaVid**: GorillaVid.in, daclips.in, movpod.in and fastvideo.in
+ - **GorillaVid**: GorillaVid.in, daclips.in, movpod.in, fastvideo.in and realvid.net
  - **Goshgay**
  - **Grooveshark**
  - **Groupon**
@@ -252,6 +253,7 @@
  - **Malemotion**
  - **MDR**
  - **media.ccc.de**
+ - **MegaVideoz**
  - **metacafe**
  - **Metacritic**
  - **Mgoon**
@@ -405,6 +407,7 @@
  - **Screencast**
  - **ScreencastOMatic**
  - **ScreenwaveMedia**
+ - **SenateISVP**
  - **ServingSys**
  - **Sexu**
  - **SexyKarma**: Sexy Karma and Watch Indian Porn
@@ -427,6 +430,7 @@
  - **soundgasm**
  - **soundgasm:profile**
  - **southpark.cc.com**
+ - **southpark.cc.com:español**
  - **southpark.de**
  - **Space**
  - **SpankBang**
@@ -461,7 +465,7 @@
  - **TeamFour**
  - **TechTalks**
  - **techtv.mit.edu**
- - **TED**
+ - **ted**
  - **tegenlicht.vpro.nl**
  - **TeleBruxelles**
  - **telecinco.es**
@@ -551,7 +555,7 @@
  - **vimeo:review**: Review pages on vimeo
  - **vimeo:user**
  - **vimeo:watchlater**: Vimeo watch later list, "vimeowatchlater" keyword (requires authentication)
- - **Vimple**: Vimple.ru
+ - **Vimple**: Vimple - one-click video hosting
  - **Vine**
  - **vine:user**
  - **vk.com**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 3fd0e7e56..dc7d666dd 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.04.17'
+__version__ = '2015.04.26'

From a01cfc2951745c36cb3e8664bacff3ff8a61d4d4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 27 Apr 2015 03:36:32 +0600
Subject: [PATCH 1091/6219] [philharmoniedeparis] Fix extraction and tests,
 improve, simplify

---
 youtube_dl/extractor/philharmoniedeparis.py | 89 +++++++++++----------
 1 file changed, 45 insertions(+), 44 deletions(-)

diff --git a/youtube_dl/extractor/philharmoniedeparis.py b/youtube_dl/extractor/philharmoniedeparis.py
index 7fc2f32ab..6e60e5fe9 100644
--- a/youtube_dl/extractor/philharmoniedeparis.py
+++ b/youtube_dl/extractor/philharmoniedeparis.py
@@ -3,75 +3,76 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..utils import (
-    determine_ext,
+    float_or_none,
     int_or_none,
     parse_iso8601,
-    unified_strdate,
+    xpath_text,
 )
 
+
 class PhilharmonieDeParisIE(InfoExtractor):
-    _VALID_URL = r'http://live\.philharmoniedeparis\.fr/concert/(?P<id>\d+)(?:/|\.html)'
+    IE_DESC = 'Philharmonie de Paris'
+    _VALID_URL = r'http://live\.philharmoniedeparis\.fr/(?:[Cc]oncert/|misc/Playlist\.ashx\?id=)(?P<id>\d+)'
     _TESTS = [{
         'url': 'http://live.philharmoniedeparis.fr/concert/1032066.html',
         'info_dict': {
             'id': '1032066',
-            'ext': 'mp4',
-            'title': "Week-end Bach. Passion selon saint Jean. Akademie für alte Musik Berlin, Rias Kammerchor, René Jacobs",
+            'ext': 'flv',
+            'title': 'md5:d1f5585d87d041d07ce9434804bc8425',
+            'timestamp': 1428179400,
             'upload_date': '20150404',
+            'duration': 6592.278,
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
         }
+    }, {
+        'url': 'http://live.philharmoniedeparis.fr/Concert/1030324.html',
+        'only_matching': True,
+    }, {
+        'url': 'http://live.philharmoniedeparis.fr/misc/Playlist.ashx?id=1030324&track=&lang=fr',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        webpage = self._download_webpage(url, video_id)
-        fichier_nom = self._html_search_regex(r'\sflashvars\s*:\s*\{\s*fichier_nom\s*:\s*\'(.*?)\'\s*,', webpage, 'fichier_nom')
-
-        playlist = self._download_xml('http://live.philharmoniedeparis.fr' + fichier_nom, video_id)
-
-        concert = playlist.find('.//concert')
+        concert = self._download_xml(
+            'http://live.philharmoniedeparis.fr/misc/Playlist.ashx?id=%s' % video_id,
+            video_id).find('./concert')
 
         formats = []
         info_dict = {
             'id': video_id,
-            'title': concert.find('./titre').text,
+            'title': xpath_text(concert, './titre', 'title', fatal=True),
             'formats': formats,
         }
 
-        if concert.attrib.get('heure'):
-            info_dict['timestamp'] = parse_iso8601(('%s-%s-%s%s') % (
-                concert.attrib['date'][0:4],
-                concert.attrib['date'][4:6],
-                concert.attrib['date'][6:8],
-                concert.attrib['heure']
-            ))
-        else:
-            info_dict['upload_date'] = concert.attrib['date']
-
         fichiers = concert.find('./fichiers')
+        stream = fichiers.attrib['serveurstream']
         for fichier in fichiers.findall('./fichier'):
-            # Sometimes <ficher>s have no attributes at all. Skip them.
-            if 'url' not in fichier.attrib:
-                continue
+            info_dict['duration'] = float_or_none(fichier.get('timecodefin'))
+            for quality, (format_id, suffix) in enumerate([('lq', ''), ('hq', '_hd')]):
+                format_url = fichier.get('url%s' % suffix)
+                if not format_url:
+                    continue
+                formats.append({
+                    'url': stream,
+                    'play_path': format_url,
+                    'ext': 'flv',
+                    'format_id': format_id,
+                    'width': int_or_none(concert.get('largeur%s' % suffix)),
+                    'height': int_or_none(concert.get('hauteur%s' % suffix)),
+                    'quality': quality,
+                })
+        self._sort_formats(formats)
 
-            formats.append({
-                'format_id': 'lq',
-                'url': fichiers.attrib['serveurstream'],
-                'ext': determine_ext(fichier.attrib['url']),
-                'play_path': fichier.attrib['url'],
-                'width': int_or_none(concert.attrib['largeur']),
-                'height': int_or_none(concert.attrib['hauteur']),
-                'quality': 1,
-            })
-
-            formats.append({
-                'format_id': 'hq',
-                'url': fichiers.attrib['serveurstream'],
-                'ext': determine_ext(fichier.attrib['url_hd']),
-                'play_path': fichier.attrib['url_hd'],
-                'width': int_or_none(concert.attrib['largeur_hd']),
-                'height': int_or_none(concert.attrib['hauteur_hd']),
-                'quality': 2,
-            })
+        date, hour = concert.get('date'), concert.get('heure')
+        if date and hour:
+            info_dict['timestamp'] = parse_iso8601(
+                '%s-%s-%sT%s:00' % (date[0:4], date[4:6], date[6:8], hour))
+        elif date:
+            info_dict['upload_date'] = date
 
         return info_dict

From 053c94f1b358a44789c38efd916904ce2733fd51 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 27 Apr 2015 15:21:51 +0600
Subject: [PATCH 1092/6219] [README] Clarify youtube-dl version when format
 selection changed to bestvideo+bestaudio/best

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 7040be1ec..b4e4b5db6 100644
--- a/README.md
+++ b/README.md
@@ -270,9 +270,9 @@ The simplest case is requesting a specific format, for example `-f 22`. You can
 
 If you want to download multiple videos and they don't have the same formats available, you can specify the order of preference using slashes, as in `-f 22/17/18`. You can also filter the video results by putting a condition in brackets, as in `-f "best[height=720]"` (or `-f "[filesize>10M]"`).  This works for filesize, height, width, tbr, abr, vbr, asr, and fps and the comparisons <, <=, >, >=, =, != and for ext, acodec, vcodec, container, and protocol and the comparisons =, != . Formats for which the value is not known are excluded unless you put a question mark (?) after the operator. You can combine format filters, so  `-f "[height <=? 720][tbr>500]"` selects up to 720p videos (or videos where the height is not known) with a bitrate of at least 500 KBit/s. Use commas to download multiple formats, such as `-f 136/137/mp4/bestvideo,140/m4a/bestaudio`. You can merge the video and audio of two formats into a single file using `-f <video-format>+<audio-format>` (requires ffmpeg or avconv), for example `-f bestvideo+bestaudio`.
 
-Since the end of April 2015 youtube-dl uses `-f bestvideo+bestaudio/best` as default format selection (see #5447, #5456). If ffmpeg or avconv are installed this results in downloading `bestvideo` and `bestaudio` separately and muxing them together into a single file giving the best overall quality available. Otherwise it falls back to `best` and results in downloading best available quality served as a single file. `best` is also needed for videos that don't come from YouTube because they don't provide the audio and video in two different files. If you want to only download some dash formats (for example if you are not interested in getting videos with a resolution higher than 1080p), you can add `-f bestvideo[height<=?1080]+bestaudio/best` to your configuration file.
+Since the end of April 2015 and version 2015.04.26 youtube-dl uses `-f bestvideo+bestaudio/best` as default format selection (see #5447, #5456). If ffmpeg or avconv are installed this results in downloading `bestvideo` and `bestaudio` separately and muxing them together into a single file giving the best overall quality available. Otherwise it falls back to `best` and results in downloading best available quality served as a single file. `best` is also needed for videos that don't come from YouTube because they don't provide the audio and video in two different files. If you want to only download some dash formats (for example if you are not interested in getting videos with a resolution higher than 1080p), you can add `-f bestvideo[height<=?1080]+bestaudio/best` to your configuration file.
 
-If you want to preserve the old format selection behavior (pre-April 2015), i.e. you want to download best available quality media served as a single file, you should explicitly specify your choice with `-f best`. You may want to add it to the [configuration file](#configuration) in order not to type it every time you run youtube-dl.
+If you want to preserve the old format selection behavior (prior to youtube-dl 2015.04.26), i.e. you want to download best available quality media served as a single file, you should explicitly specify your choice with `-f best`. You may want to add it to the [configuration file](#configuration) in order not to type it every time you run youtube-dl.
 
 # VIDEO SELECTION
 

From c8183e661d629bd7d86d38eda9e0013e2aad682e Mon Sep 17 00:00:00 2001
From: "Sergey M." <dstftw@gmail.com>
Date: Mon, 27 Apr 2015 16:01:30 +0600
Subject: [PATCH 1093/6219] [README] Document special characters escaping
 (#5538)

---
 README.md | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/README.md b/README.md
index b4e4b5db6..5fb992241 100644
--- a/README.md
+++ b/README.md
@@ -358,6 +358,22 @@ YouTube has switched to a new video info format in July 2011 which is not suppor
 
 YouTube requires an additional signature since September 2012 which is not supported by old versions of youtube-dl. See [above](#how-do-i-update-youtube-dl) for how to update youtube-dl.
 
+### Video URL contains an ampersand and I'm getting some strange output `[1] 2839` or `'v' is not recognized as an internal or external command` ###
+
+That's actually the output from your shell. Since ampersand is one of the special shell characters it's interpreted by shell preventing you from passing the whole URL to youtube-dl. To disable your shell from interpreting the ampersands (or any other special characters) you have to either put the whole URL in quotes or escape them with a backslash (which approach will work depends on your shell).
+
+For example if your URL is https://www.youtube.com/watch?t=48&v=e2CXaJv0cMw you should end up with following command:
+
+```youtube-dl 'https://www.youtube.com/watch?t=48&v=e2CXaJv0cMw'```
+
+or
+
+```youtube-dl https://www.youtube.com/watch?t=48\&v=e2CXaJv0cMw```
+
+For Windows you have to use the double quotes:
+
+```youtube-dl "https://www.youtube.com/watch?t=48&v=e2CXaJv0cMw"```
+
 ### ExtractorError: Could not find JS function u'OF'
 
 In February 2015, the new YouTube player contained a character sequence in a string that was misinterpreted by old versions of youtube-dl. See [above](#how-do-i-update-youtube-dl) for how to update youtube-dl.

From 189ba90996d810ea00b165d20d320869efe14518 Mon Sep 17 00:00:00 2001
From: "Sergey M." <dstftw@gmail.com>
Date: Mon, 27 Apr 2015 16:05:01 +0600
Subject: [PATCH 1094/6219] [README] Use youtube-dl test video URL

---
 README.md | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index 5fb992241..f67744f0f 100644
--- a/README.md
+++ b/README.md
@@ -362,17 +362,17 @@ YouTube requires an additional signature since September 2012 which is not suppo
 
 That's actually the output from your shell. Since ampersand is one of the special shell characters it's interpreted by shell preventing you from passing the whole URL to youtube-dl. To disable your shell from interpreting the ampersands (or any other special characters) you have to either put the whole URL in quotes or escape them with a backslash (which approach will work depends on your shell).
 
-For example if your URL is https://www.youtube.com/watch?t=48&v=e2CXaJv0cMw you should end up with following command:
+For example if your URL is https://www.youtube.com/watch?t=4&v=BaW_jenozKc you should end up with following command:
 
-```youtube-dl 'https://www.youtube.com/watch?t=48&v=e2CXaJv0cMw'```
+```youtube-dl 'https://www.youtube.com/watch?t=4&v=BaW_jenozKc'```
 
 or
 
-```youtube-dl https://www.youtube.com/watch?t=48\&v=e2CXaJv0cMw```
+```youtube-dl https://www.youtube.com/watch?t=4\&v=BaW_jenozKc```
 
 For Windows you have to use the double quotes:
 
-```youtube-dl "https://www.youtube.com/watch?t=48&v=e2CXaJv0cMw"```
+```youtube-dl "https://www.youtube.com/watch?t=4&v=BaW_jenozKc"```
 
 ### ExtractorError: Could not find JS function u'OF'
 

From c86b61428b127ee29793f005c5e45e321696d7e9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 27 Apr 2015 20:00:18 +0600
Subject: [PATCH 1095/6219] [utils] Fix another old python 2.6 kwargs issue
 (Closes #5539)

---
 youtube_dl/utils.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 7d15eab64..f07679c76 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -37,6 +37,7 @@ from .compat import (
     compat_chr,
     compat_html_entities,
     compat_http_client,
+    compat_kwargs,
     compat_parse_qs,
     compat_socket_create_connection,
     compat_str,
@@ -114,7 +115,7 @@ def write_json_file(obj, fn):
             'encoding': 'utf-8',
         })
 
-    tf = tempfile.NamedTemporaryFile(**args)
+    tf = tempfile.NamedTemporaryFile(**compat_kwargs(args))
 
     try:
         with tf:

From e2dc351d25d1b05b26cba654a22b0d10ed47dec9 Mon Sep 17 00:00:00 2001
From: pulpe <Pulpan3@gmail.com>
Date: Mon, 27 Apr 2015 17:44:13 +0200
Subject: [PATCH 1096/6219] [escapist] Fix extractor (fixes #5090)

---
 youtube_dl/extractor/escapist.py | 156 +++++++++++++------------------
 1 file changed, 65 insertions(+), 91 deletions(-)

diff --git a/youtube_dl/extractor/escapist.py b/youtube_dl/extractor/escapist.py
index e47f3e27a..a01a05b06 100644
--- a/youtube_dl/extractor/escapist.py
+++ b/youtube_dl/extractor/escapist.py
@@ -1,128 +1,102 @@
 from __future__ import unicode_literals
 
+import json
+
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse,
-    compat_urllib_request,
-)
+from ..compat import compat_urllib_request
+
 from ..utils import (
-    ExtractorError,
-    js_to_json,
-    parse_duration,
+    determine_ext,
+    clean_html,
+    qualities,
 )
 
 
+def _decrypt_config(key, string):
+    a = ''
+    i = ''
+    r = ''
+
+    while len(a) < (len(string) / 2):
+        a += key
+
+    a = a[0:int(len(string) / 2)]
+
+    t = 0
+    while t < len(string):
+        i += chr(int(string[t] + string[t + 1], 16))
+        t += 2
+
+    icko = [s for s in i]
+
+    for t, c in enumerate(a):
+        r += chr(ord(c) ^ ord(icko[t]))
+
+    return r
+
+
 class EscapistIE(InfoExtractor):
     _VALID_URL = r'https?://?(www\.)?escapistmagazine\.com/videos/view/[^/?#]+/(?P<id>[0-9]+)-[^/?#]*(?:$|[?#])'
-    _USER_AGENT = 'Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.escapistmagazine.com/videos/view/the-escapist-presents/6618-Breaking-Down-Baldurs-Gate',
-        'md5': 'ab3a706c681efca53f0a35f1415cf0d1',
+        'md5': 'c6793dbda81388f4264c1ba18684a74d',
         'info_dict': {
             'id': '6618',
             'ext': 'mp4',
             'description': "Baldur's Gate: Original, Modded or Enhanced Edition? I'll break down what you can expect from the new Baldur's Gate: Enhanced Edition.",
-            'uploader_id': 'the-escapist-presents',
-            'uploader': 'The Escapist Presents',
             'title': "Breaking Down Baldur's Gate",
             'thumbnail': 're:^https?://.*\.jpg$',
             'duration': 264,
         }
-    }
+    }, {
+        'url': 'http://www.escapistmagazine.com/videos/view/zero-punctuation/10044-Evolve-One-vs-Multiplayer',
+        'md5': 'cf8842a8a46444d241f9a9980d7874f2',
+        'info_dict': {
+            'id': '10044',
+            'ext': 'mp4',
+            'description': 'This week, Zero Punctuation reviews Evolve.',
+            'title': 'Evolve - One vs Multiplayer',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 304,
+        }
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        webpage_req = compat_urllib_request.Request(url)
-        webpage_req.add_header('User-Agent', self._USER_AGENT)
-        webpage = self._download_webpage(webpage_req, video_id)
+        webpage = self._download_webpage(url, video_id)
 
-        uploader_id = self._html_search_regex(
-            r"<h1\s+class='headline'>\s*<a\s+href='/videos/view/(.*?)'",
-            webpage, 'uploader ID', fatal=False)
-        uploader = self._html_search_regex(
-            r"<h1\s+class='headline'>(.*?)</a>",
-            webpage, 'uploader', fatal=False)
-        description = self._html_search_meta('description', webpage)
-        duration = parse_duration(self._html_search_meta('duration', webpage))
+        imsVideo = json.loads(self._search_regex(r'imsVideo\.play\(([^\)]+)\);', webpage, 'imsVideo'))
+        video_id = imsVideo['videoID']
+        key = imsVideo['hash']
 
-        raw_title = self._html_search_meta('title', webpage, fatal=True)
-        title = raw_title.partition(' : ')[2]
-
-        config_url = compat_urllib_parse.unquote(self._html_search_regex(
-            r'''(?x)
-            (?:
-                <param\s+name="flashvars".*?\s+value="config=|
-                flashvars=&quot;config=
-            )
-            (https?://[^"&]+)
-            ''',
-            webpage, 'config URL'))
+        quality = qualities(['lq', 'hq', 'hd'])
 
         formats = []
-        ad_formats = []
+        for q in ['lq', 'hq', 'hd']:
+            config_req = compat_urllib_request.Request('http://www.escapistmagazine.com/videos/'
+                'vidconfig.php?videoID=%s&hash=%s&quality=%s' % (video_id, key, 'mp4_' + q))
+            config_req.add_header('Referer', url)
+            config = self._download_webpage(config_req, video_id, 'Downloading video config ' + q.upper())
 
-        def _add_format(name, cfg_url, quality):
-            cfg_req = compat_urllib_request.Request(cfg_url)
-            cfg_req.add_header('User-Agent', self._USER_AGENT)
-            config = self._download_json(
-                cfg_req, video_id,
-                'Downloading ' + name + ' configuration',
-                'Unable to download ' + name + ' configuration',
-                transform_source=js_to_json)
+            data = json.loads(_decrypt_config(key, config))
 
-            playlist = config['playlist']
-            for p in playlist:
-                if p.get('eventCategory') == 'Video':
-                    ar = formats
-                elif p.get('eventCategory') == 'Video Postroll':
-                    ar = ad_formats
-                else:
-                    continue
+            title = clean_html(data['videoData']['title'])
+            duration = data['videoData']['duration'] / 1000
 
-                ar.append({
-                    'url': p['url'],
-                    'format_id': name,
-                    'quality': quality,
-                    'http_headers': {
-                        'User-Agent': self._USER_AGENT,
-                    },
-                })
+            for i, v in enumerate(data['files']['videos']):
 
-        _add_format('normal', config_url, quality=0)
-        hq_url = (config_url +
-                  ('&hq=1' if '?' in config_url else config_url + '?hq=1'))
-        try:
-            _add_format('hq', hq_url, quality=1)
-        except ExtractorError:
-            pass  # That's fine, we'll just use normal quality
-        self._sort_formats(formats)
+                formats.append({
+                    'url': v,
+                    'format_id': determine_ext(v) + '_' + q + str(i),
+                    'quality': quality(q),
+                    })
 
-        if '/escapist/sales-marketing/' in formats[-1]['url']:
-            raise ExtractorError('This IP address has been blocked by The Escapist', expected=True)
 
-        res = {
+        return {
             'id': video_id,
             'formats': formats,
-            'uploader': uploader,
-            'uploader_id': uploader_id,
             'title': title,
             'thumbnail': self._og_search_thumbnail(webpage),
-            'description': description,
+            'description': self._og_search_description(webpage),
             'duration': duration,
         }
-
-        if self._downloader.params.get('include_ads') and ad_formats:
-            self._sort_formats(ad_formats)
-            ad_res = {
-                'id': '%s-ad' % video_id,
-                'title': '%s (Postroll)' % title,
-                'formats': ad_formats,
-            }
-            return {
-                '_type': 'playlist',
-                'entries': [res, ad_res],
-                'title': title,
-                'id': video_id,
-            }
-
-        return res

From 290a5a8d85e8f7514099fa305c8f0b26d1b35a78 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 27 Apr 2015 22:17:51 +0600
Subject: [PATCH 1097/6219] [escapist] Fix imsVideo regex (#5090)

---
 youtube_dl/extractor/escapist.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/escapist.py b/youtube_dl/extractor/escapist.py
index a01a05b06..600ebf078 100644
--- a/youtube_dl/extractor/escapist.py
+++ b/youtube_dl/extractor/escapist.py
@@ -65,7 +65,10 @@ class EscapistIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        imsVideo = json.loads(self._search_regex(r'imsVideo\.play\(([^\)]+)\);', webpage, 'imsVideo'))
+        imsVideo = self._parse_json(
+            self._search_regex(
+                r'imsVideo\.play\(({.+?})\);', webpage, 'imsVideo'),
+            video_id)
         video_id = imsVideo['videoID']
         key = imsVideo['hash']
 

From e206740fd7454bb03e2819365a257bce59f76162 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 27 Apr 2015 23:44:05 +0600
Subject: [PATCH 1098/6219] [moniker] Capture and output error message (#5541)

---
 youtube_dl/extractor/moniker.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/youtube_dl/extractor/moniker.py b/youtube_dl/extractor/moniker.py
index 5de719bdc..801abe0e1 100644
--- a/youtube_dl/extractor/moniker.py
+++ b/youtube_dl/extractor/moniker.py
@@ -9,6 +9,7 @@ from ..compat import (
     compat_urllib_parse,
     compat_urllib_request,
 )
+from ..utils import ExtractorError
 
 
 class MonikerIE(InfoExtractor):
@@ -40,6 +41,12 @@ class MonikerIE(InfoExtractor):
         video_id = self._match_id(url)
         orig_webpage = self._download_webpage(url, video_id)
 
+        error = self._search_regex(
+            r'class="err">([^<]+)<', orig_webpage, 'error', default=None)
+        if error:
+            raise ExtractorError(
+                '%s returned error: %s' % (self.IE_NAME, error), expected=True)
+
         fields = re.findall(r'type="hidden" name="(.+?)"\s* value="?(.+?)">', orig_webpage)
         data = dict(fields)
 

From 2419a376b911df61bec6dae7178fc342a4e218f5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 27 Apr 2015 23:46:16 +0600
Subject: [PATCH 1099/6219] [moniker] Check not found error (#5541)

---
 youtube_dl/extractor/moniker.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/moniker.py b/youtube_dl/extractor/moniker.py
index 801abe0e1..88dcd4f73 100644
--- a/youtube_dl/extractor/moniker.py
+++ b/youtube_dl/extractor/moniker.py
@@ -41,6 +41,9 @@ class MonikerIE(InfoExtractor):
         video_id = self._match_id(url)
         orig_webpage = self._download_webpage(url, video_id)
 
+        if '>File Not Found<' in orig_webpage:
+            raise ExtractorError('Video %s does not exist' % video_id, expected=True)
+
         error = self._search_regex(
             r'class="err">([^<]+)<', orig_webpage, 'error', default=None)
         if error:

From 3a0f0c263a9c6e2a3e81fc97f59a5960fd5ee994 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 28 Apr 2015 09:11:18 +0200
Subject: [PATCH 1100/6219] release 2015.04.28

---
 docs/supportedsites.md | 1 +
 youtube_dl/version.py  | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index cc1564b7b..d1d676cdd 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -340,6 +340,7 @@
  - **parliamentlive.tv**: UK parliament videos
  - **Patreon**
  - **PBS**
+ - **PhilharmonieDeParis**: Philharmonie de Paris
  - **Phoenix**
  - **Photobucket**
  - **Pladform**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index dc7d666dd..b88ea85e8 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.04.26'
+__version__ = '2015.04.28'

From 10831b5ec9158f2de5b16b66520d9dde02b97d77 Mon Sep 17 00:00:00 2001
From: pulpe <Pulpan3@gmail.com>
Date: Tue, 28 Apr 2015 14:56:48 +0200
Subject: [PATCH 1101/6219] [vimeo] Fix redirection

---
 youtube_dl/extractor/vimeo.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 9027f9dd6..398ca67c1 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -223,6 +223,8 @@ class VimeoIE(VimeoBaseInfoExtractor):
         orig_url = url
         if mobj.group('pro') or mobj.group('player'):
             url = 'https://player.vimeo.com/video/' + video_id
+        else:
+            url = 'https://vimeo.com/' + video_id
 
         # Retrieve video webpage to extract further information
         request = compat_urllib_request.Request(url, None, headers)

From 2edce52584ecd14ee626c6a1b6bbaeabb3985cc0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 28 Apr 2015 15:05:41 +0200
Subject: [PATCH 1102/6219] [vimeo] Fix password protected videos again (#5082)

Since they have changed again to the previous format, I've modified the regex to match both formats.
---
 youtube_dl/extractor/vimeo.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 398ca67c1..f300c7ca4 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -177,7 +177,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
         password = self._downloader.params.get('videopassword', None)
         if password is None:
             raise ExtractorError('This video is protected by a password, use the --video-password option', expected=True)
-        token = self._search_regex(r'xsrft":"(.*?)"', webpage, 'login token')
+        token = self._search_regex(r'xsrft[\s=:"\']+([^"\']+)', webpage, 'login token')
         data = urlencode_postdata({
             'password': password,
             'token': token,
@@ -441,7 +441,7 @@ class VimeoChannelIE(InfoExtractor):
             name="([^"]+)"\s+
             value="([^"]*)"
             ''', login_form))
-        token = self._search_regex(r'xsrft":"(.*?)"', webpage, 'login token')
+        token = self._search_regex(r'xsrft[\s=:"\']+([^"\']+)', webpage, 'login token')
         fields['token'] = token
         fields['password'] = password
         post = urlencode_postdata(fields)

From 39b62db1160f5a4770348f1d01daeb0ce049c28c Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 28 Apr 2015 23:07:56 +0800
Subject: [PATCH 1103/6219] [youtube] Catch more alert messages (closes #5074)

---
 youtube_dl/extractor/youtube.py | 22 ++++++++++++++++------
 1 file changed, 16 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 07c0f6ef9..0869c9fd4 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1291,12 +1291,22 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
         page = self._download_webpage(url, playlist_id)
         more_widget_html = content_html = page
 
-        # Check if the playlist exists or is private
-        if re.search(r'<div class="yt-alert-message">[^<]*?(The|This) playlist (does not exist|is private)[^<]*?</div>', page) is not None:
-            raise ExtractorError(
-                'The playlist doesn\'t exist or is private, use --username or '
-                '--netrc to access it.',
-                expected=True)
+        for match in re.findall(r'<div class="yt-alert-message">([^<]+)</div>', page):
+            match = match.strip()
+            # Check if the playlist exists or is private
+            if re.match(r'[^<]*(The|This) playlist (does not exist|is private)[^<]*', match):
+                raise ExtractorError(
+                    'The playlist doesn\'t exist or is private, use --username or '
+                    '--netrc to access it.',
+                    expected=True)
+            elif re.match(r'[^<]*Invalid parameters[^<]*', match):
+                raise ExtractorError(
+                    'Invalid parameters. Maybe URL is incorrect.',
+                    expected=True)
+            elif re.match(r'[^<]*Choose your language[^<]*', match):
+                continue
+            else:
+                self.report_warning('Youtube gives an alert message: ' + match)
 
         # Extract the video ids from the playlist pages
         ids = []

From e70c7568c0226449190b57b988c1df10bf6508db Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 29 Apr 2015 00:01:09 +0800
Subject: [PATCH 1104/6219] [testtube] Detect Youtube iframes (fixes #4867)

---
 youtube_dl/extractor/testtube.py | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/youtube_dl/extractor/testtube.py b/youtube_dl/extractor/testtube.py
index 6a7b5e49d..cbada136c 100644
--- a/youtube_dl/extractor/testtube.py
+++ b/youtube_dl/extractor/testtube.py
@@ -22,12 +22,30 @@ class TestTubeIE(InfoExtractor):
             'uploader': 'DNews',
             'uploader_id': 'dnews',
         },
+    }, {
+        'url': 'https://testtube.com/iflscience/insane-jet-ski-flipping',
+        'info_dict': {
+            'id': 'fAGfJ4YjVus',
+            'ext': 'mp4',
+            'title': 'Flipping Jet-Ski Skills | Outrageous Acts of Science',
+            'uploader': 'Science Channel',
+            'uploader_id': 'ScienceChannel',
+            'upload_date': '20150203',
+            'description': 'md5:e61374030015bae1d2e22f096d4769d6',
+        }
     }]
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
 
         webpage = self._download_webpage(url, display_id)
+
+        youtube_url = self._html_search_regex(
+            r'<iframe[^>]+src="((?:https?:)?//www.youtube.com/embed/[^"]+)"',
+            webpage, 'youtube iframe', default=None)
+        if youtube_url:
+            return self.url_result(youtube_url, 'Youtube', video_id=display_id)
+
         video_id = self._search_regex(
             r"player\.loadRevision3Item\('video_id',\s*([0-9]+)\);",
             webpage, 'video ID')

From ec7c1e85e0b26e67752bbd3d25b8d5ed23ae0992 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 29 Apr 2015 00:24:58 +0800
Subject: [PATCH 1105/6219] [testtube] Fix test case 1

Seems the site now provides webm with higher bitrates
---
 youtube_dl/extractor/testtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/testtube.py b/youtube_dl/extractor/testtube.py
index cbada136c..26655d690 100644
--- a/youtube_dl/extractor/testtube.py
+++ b/youtube_dl/extractor/testtube.py
@@ -15,7 +15,7 @@ class TestTubeIE(InfoExtractor):
             'id': '60163',
             'display_id': '5-weird-ways-plants-can-eat-animals',
             'duration': 275,
-            'ext': 'mp4',
+            'ext': 'webm',
             'title': '5 Weird Ways Plants Can Eat Animals',
             'description': 'Why have some plants evolved to eat meat?',
             'thumbnail': 're:^https?://.*\.jpg$',

From 9d8ba307ef18e737efc0be5164b53e0546288a4d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 29 Apr 2015 04:03:07 +0800
Subject: [PATCH 1106/6219] [yourupload] Fix extraction

---
 youtube_dl/extractor/yourupload.py | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/yourupload.py b/youtube_dl/extractor/yourupload.py
index 40fc4165f..2064f4a69 100644
--- a/youtube_dl/extractor/yourupload.py
+++ b/youtube_dl/extractor/yourupload.py
@@ -16,7 +16,7 @@ class YourUploadIE(InfoExtractor):
     _TESTS = [
         {
             'url': 'http://yourupload.com/watch/14i14h',
-            'md5': 'bf5c2f95c4c917536e80936af7bc51e1',
+            'md5': '5e2c63385454c557f97c4c4131a393cd',
             'info_dict': {
                 'id': '14i14h',
                 'ext': 'mp4',
@@ -38,16 +38,16 @@ class YourUploadIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
 
-        url = 'http://embed.yucache.net/{0:}'.format(video_id)
-        webpage = self._download_webpage(url, video_id)
+        embed_url = 'http://embed.yucache.net/{0:}'.format(video_id)
+        webpage = self._download_webpage(embed_url, video_id)
 
         title = self._og_search_title(webpage)
         thumbnail = self._og_search_thumbnail(webpage)
-        url = self._og_search_video_url(webpage)
+        video_url = self._og_search_video_url(webpage)
 
         formats = [{
             'format_id': 'sd',
-            'url': url,
+            'url': video_url,
         }]
 
         return {
@@ -55,4 +55,7 @@ class YourUploadIE(InfoExtractor):
             'title': title,
             'formats': formats,
             'thumbnail': thumbnail,
+            'http_headers': {
+                'Referer': embed_url,
+            },
         }

From cbbece96a25443dbf5cb66819883efa2e9b66de4 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 29 Apr 2015 04:05:14 +0800
Subject: [PATCH 1107/6219] [yourupload] Simplify

---
 youtube_dl/extractor/yourupload.py | 14 +++-----------
 1 file changed, 3 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/yourupload.py b/youtube_dl/extractor/yourupload.py
index 2064f4a69..4e25d6f22 100644
--- a/youtube_dl/extractor/yourupload.py
+++ b/youtube_dl/extractor/yourupload.py
@@ -1,8 +1,6 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 
 
@@ -35,25 +33,19 @@ class YourUploadIE(InfoExtractor):
     ]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         embed_url = 'http://embed.yucache.net/{0:}'.format(video_id)
         webpage = self._download_webpage(embed_url, video_id)
 
         title = self._og_search_title(webpage)
-        thumbnail = self._og_search_thumbnail(webpage)
         video_url = self._og_search_video_url(webpage)
-
-        formats = [{
-            'format_id': 'sd',
-            'url': video_url,
-        }]
+        thumbnail = self._og_search_thumbnail(webpage, default=None)
 
         return {
             'id': video_id,
             'title': title,
-            'formats': formats,
+            'url': video_url,
             'thumbnail': thumbnail,
             'http_headers': {
                 'Referer': embed_url,

From 5456d78f0c177bbdd3ad0b6221ff95b706104441 Mon Sep 17 00:00:00 2001
From: zouhair <zouhair@users.noreply.github.com>
Date: Wed, 29 Apr 2015 10:07:49 -0400
Subject: [PATCH 1108/6219] Typo "incompatible" instead of "uncompatible"

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 977141881..827c88e0d 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1384,7 +1384,7 @@ class YoutubeDL(object):
                     requested_formats = info_dict['requested_formats']
                     if self.params.get('merge_output_format') is None and not compatible_formats(requested_formats):
                         filename = os.path.splitext(filename)[0] + '.mkv'
-                        self.report_warning('You have requested formats uncompatible for merge. '
+                        self.report_warning('You have requested formats incompatible for merge. '
                                             'The formats will be merged into mkv')
                     if os.path.exists(encodeFilename(filename)):
                         self.to_screen(

From cf0649f8b76879f9d32ceab91e87d659ea8ec398 Mon Sep 17 00:00:00 2001
From: zouhair <zouhair@users.noreply.github.com>
Date: Wed, 29 Apr 2015 11:03:10 -0400
Subject: [PATCH 1109/6219] Typo: twice "the the" to "the"

---
 youtube_dl/extractor/common.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 71230323c..48f58aae3 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -47,7 +47,7 @@ class InfoExtractor(object):
     information possibly downloading the video to the file system, among
     other possible outcomes.
 
-    The type field determines the the type of the result.
+    The type field determines the type of the result.
     By far the most common value (and the default if _type is missing) is
     "video", which indicates a single video.
 
@@ -572,7 +572,7 @@ class InfoExtractor(object):
 
     def _get_login_info(self):
         """
-        Get the the login info as (username, password)
+        Get the login info as (username, password)
         It will look in the netrc file using the _NETRC_MACHINE value
         If there's no info available, return (None, None)
         """

From 9ee53a49f0f040f07e0f675e2077f22ace2acefe Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 29 Apr 2015 04:18:53 +0800
Subject: [PATCH 1110/6219] [YouPorn] Fix extractor

---
 youtube_dl/extractor/youporn.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youporn.py b/youtube_dl/extractor/youporn.py
index 6abe72f73..4ba7c36db 100644
--- a/youtube_dl/extractor/youporn.py
+++ b/youtube_dl/extractor/youporn.py
@@ -47,7 +47,7 @@ class YouPornIE(InfoExtractor):
 
         # Get JSON parameters
         json_params = self._search_regex(
-            [r'var\s+videoJa?son\s*=\s*({.+?});',
+            [r'videoJa?son\s*=\s*({.+})',
              r'var\s+currentVideo\s*=\s*new\s+Video\((.+?)\)[,;]'],
             webpage, 'JSON parameters')
         try:

From bb865f3a5e7290f0da43d180f068f8715c39e3ae Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 30 Apr 2015 00:24:24 +0800
Subject: [PATCH 1111/6219] [niconico] Fix extraction and update tests (closes
 #5511)

---
 youtube_dl/extractor/niconico.py | 72 +++++++++++++++++++++++---------
 1 file changed, 52 insertions(+), 20 deletions(-)

diff --git a/youtube_dl/extractor/niconico.py b/youtube_dl/extractor/niconico.py
index ddec7b338..0ca046ac2 100644
--- a/youtube_dl/extractor/niconico.py
+++ b/youtube_dl/extractor/niconico.py
@@ -14,7 +14,9 @@ from ..utils import (
     ExtractorError,
     int_or_none,
     parse_duration,
-    unified_strdate,
+    parse_iso8601,
+    xpath_text,
+    determine_ext,
 )
 
 
@@ -32,6 +34,7 @@ class NiconicoIE(InfoExtractor):
             'uploader': 'takuya0301',
             'uploader_id': '2698420',
             'upload_date': '20131123',
+            'timestamp': 1385182762,
             'description': '(c) copyright 2008, Blender Foundation / www.bigbuckbunny.org',
             'duration': 33,
         },
@@ -46,16 +49,31 @@ class NiconicoIE(InfoExtractor):
             'id': 'nm14296458',
             'ext': 'swf',
             'title': '【鏡音リン】Dance on media【オリジナル】take2!',
-            'description': 'md5:',
+            'description': 'md5:689f066d74610b3b22e0f1739add0f58',
             'uploader': 'りょうた',
             'uploader_id': '18822557',
             'upload_date': '20110429',
+            'timestamp': 1304065916,
             'duration': 209,
         },
         'params': {
             'username': 'ydl.niconico@gmail.com',
             'password': 'youtube-dl',
         },
+    }, {
+        # 'video exists but is marked as "deleted"
+        'url': 'http://www.nicovideo.jp/watch/sm10000',
+        'md5': '38e53c9aad548f3ecf01ca7680b59b08',
+        'info_dict': {
+            'id': 'sm10000',
+            'ext': 'unknown_video',
+            'description': 'deleted',
+            'title': 'ドラえもんエターナル第3話「決戦第3新東京市」＜前編＞',
+        },
+        'params': {
+            'username': 'ydl.niconico@gmail.com',
+            'password': 'youtube-dl',
+        }
     }]
 
     _VALID_URL = r'https?://(?:www\.|secure\.)?nicovideo\.jp/watch/(?P<id>(?:[a-z]{2})?[0-9]+)'
@@ -95,9 +113,10 @@ class NiconicoIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        # Get video webpage. We are not actually interested in it, but need
-        # the cookies in order to be able to download the info webpage
-        self._download_webpage('http://www.nicovideo.jp/watch/' + video_id, video_id)
+        # Get video webpage. We are not actually interested in it for normal
+        # cases, but need the cookies in order to be able to download the
+        # info webpage
+        webpage = self._download_webpage('http://www.nicovideo.jp/watch/' + video_id, video_id)
 
         video_info = self._download_xml(
             'http://ext.nicovideo.jp/api/getthumbinfo/' + video_id, video_id,
@@ -127,22 +146,34 @@ class NiconicoIE(InfoExtractor):
                 flv_info_request, video_id,
                 note='Downloading flv info', errnote='Unable to download flv info')
 
-        if 'deleted=' in flv_info_webpage:
-            raise ExtractorError('The video has been deleted.',
-                                 expected=True)
-        video_real_url = compat_urlparse.parse_qs(flv_info_webpage)['url'][0]
+        flv_info = compat_urlparse.parse_qs(flv_info_webpage)
+        if 'url' not in flv_info:
+            if 'deleted' in flv_info:
+                raise ExtractorError('The video has been deleted.',
+                                     expected=True)
+            else:
+                raise ExtractorError('Unable to find video URL')
+
+        video_real_url = flv_info['url'][0]
 
         # Start extracting information
-        title = video_info.find('.//title').text
-        extension = video_info.find('.//movie_type').text
+        title = xpath_text(video_info, './/title')
+        if not title:
+            title = self._html_search_regex(
+                r'<span[^>]+class="videoHeaderTitle"[^>]*>([^<]+)</span>',
+                webpage, 'video title')
+
+        extension = xpath_text(video_info, './/movie_type')
+        if not extension:
+            extension = determine_ext(video_real_url)
         video_format = extension.upper()
-        thumbnail = video_info.find('.//thumbnail_url').text
-        description = video_info.find('.//description').text
-        upload_date = unified_strdate(video_info.find('.//first_retrieve').text.split('+')[0])
-        view_count = int_or_none(video_info.find('.//view_counter').text)
-        comment_count = int_or_none(video_info.find('.//comment_num').text)
-        duration = parse_duration(video_info.find('.//length').text)
-        webpage_url = video_info.find('.//watch_url').text
+        thumbnail = xpath_text(video_info, './/thumbnail_url')
+        description = xpath_text(video_info, './/description')
+        timestamp = parse_iso8601(xpath_text(video_info, './/first_retrieve'))
+        view_count = int_or_none(xpath_text(video_info, './/view_counter'))
+        comment_count = int_or_none(xpath_text(video_info, './/comment_num'))
+        duration = parse_duration(xpath_text(video_info, './/length'))
+        webpage_url = xpath_text(video_info, './/watch_url')
 
         if video_info.find('.//ch_id') is not None:
             uploader_id = video_info.find('.//ch_id').text
@@ -153,7 +184,7 @@ class NiconicoIE(InfoExtractor):
         else:
             uploader_id = uploader = None
 
-        return {
+        ret = {
             'id': video_id,
             'url': video_real_url,
             'title': title,
@@ -162,13 +193,14 @@ class NiconicoIE(InfoExtractor):
             'thumbnail': thumbnail,
             'description': description,
             'uploader': uploader,
-            'upload_date': upload_date,
+            'timestamp': timestamp,
             'uploader_id': uploader_id,
             'view_count': view_count,
             'comment_count': comment_count,
             'duration': duration,
             'webpage_url': webpage_url,
         }
+        return dict((k, v) for k, v in ret.items() if v is not None)
 
 
 class NiconicoPlaylistIE(InfoExtractor):

From 59d814f79341341e6390392a09e628ee12a6f18d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 30 Apr 2015 00:47:52 +0800
Subject: [PATCH 1112/6219] [niconico] Remove credentials from tests and
 enhance title extraction

All test videos can be downloaded without username and password now.
---
 youtube_dl/extractor/niconico.py | 17 ++++-------------
 1 file changed, 4 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/niconico.py b/youtube_dl/extractor/niconico.py
index 0ca046ac2..a9b770cb3 100644
--- a/youtube_dl/extractor/niconico.py
+++ b/youtube_dl/extractor/niconico.py
@@ -38,13 +38,10 @@ class NiconicoIE(InfoExtractor):
             'description': '(c) copyright 2008, Blender Foundation / www.bigbuckbunny.org',
             'duration': 33,
         },
-        'params': {
-            'username': 'ydl.niconico@gmail.com',
-            'password': 'youtube-dl',
-        },
     }, {
+        # File downloaded with and without credentials are different, so omit
+        # the md5 field
         'url': 'http://www.nicovideo.jp/watch/nm14296458',
-        'md5': '8db08e0158457cf852a31519fceea5bc',
         'info_dict': {
             'id': 'nm14296458',
             'ext': 'swf',
@@ -56,10 +53,6 @@ class NiconicoIE(InfoExtractor):
             'timestamp': 1304065916,
             'duration': 209,
         },
-        'params': {
-            'username': 'ydl.niconico@gmail.com',
-            'password': 'youtube-dl',
-        },
     }, {
         # 'video exists but is marked as "deleted"
         'url': 'http://www.nicovideo.jp/watch/sm10000',
@@ -70,10 +63,6 @@ class NiconicoIE(InfoExtractor):
             'description': 'deleted',
             'title': 'ドラえもんエターナル第3話「決戦第3新東京市」＜前編＞',
         },
-        'params': {
-            'username': 'ydl.niconico@gmail.com',
-            'password': 'youtube-dl',
-        }
     }]
 
     _VALID_URL = r'https?://(?:www\.|secure\.)?nicovideo\.jp/watch/(?P<id>(?:[a-z]{2})?[0-9]+)'
@@ -158,6 +147,8 @@ class NiconicoIE(InfoExtractor):
 
         # Start extracting information
         title = xpath_text(video_info, './/title')
+        if not title:
+            title = self._og_search_title(webpage, default=None)
         if not title:
             title = self._html_search_regex(
                 r'<span[^>]+class="videoHeaderTitle"[^>]*>([^<]+)</span>',

From b2e8e7dab567ed9b27817c5dd0cf173bc7fb8cfa Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 30 Apr 2015 02:24:05 +0800
Subject: [PATCH 1113/6219] [niconico] Try to extract all optional fields from
 various sources

---
 youtube_dl/extractor/niconico.py | 57 ++++++++++++++++++++++++++++----
 1 file changed, 51 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/niconico.py b/youtube_dl/extractor/niconico.py
index a9b770cb3..dd16d0042 100644
--- a/youtube_dl/extractor/niconico.py
+++ b/youtube_dl/extractor/niconico.py
@@ -3,6 +3,7 @@ from __future__ import unicode_literals
 
 import re
 import json
+import datetime
 
 from .common import InfoExtractor
 from ..compat import (
@@ -55,13 +56,16 @@ class NiconicoIE(InfoExtractor):
         },
     }, {
         # 'video exists but is marked as "deleted"
+        # md5 is unstable
         'url': 'http://www.nicovideo.jp/watch/sm10000',
-        'md5': '38e53c9aad548f3ecf01ca7680b59b08',
         'info_dict': {
             'id': 'sm10000',
             'ext': 'unknown_video',
             'description': 'deleted',
             'title': 'ドラえもんエターナル第3話「決戦第3新東京市」＜前編＞',
+            'upload_date': '20071224',
+            'timestamp': 1198527840,  # timestamp field has different value if logged in
+            'duration': 304,
         },
     }]
 
@@ -154,17 +158,59 @@ class NiconicoIE(InfoExtractor):
                 r'<span[^>]+class="videoHeaderTitle"[^>]*>([^<]+)</span>',
                 webpage, 'video title')
 
+        watch_api_data_string = self._html_search_regex(
+            r'<div[^>]+id="watchAPIDataContainer"[^>]+>([^<]+)</div>',
+            webpage, 'watch api data', default=None)
+        watch_api_data = self._parse_json(watch_api_data_string, video_id) if watch_api_data_string else {}
+        video_detail = watch_api_data.get('videoDetail', {})
+
         extension = xpath_text(video_info, './/movie_type')
         if not extension:
             extension = determine_ext(video_real_url)
         video_format = extension.upper()
-        thumbnail = xpath_text(video_info, './/thumbnail_url')
+
+        thumbnail = (
+            xpath_text(video_info, './/thumbnail_url') or
+            self._html_search_meta('image', webpage, 'thumbnail', default=None) or
+            video_detail.get('thumbnail'))
+
         description = xpath_text(video_info, './/description')
+
         timestamp = parse_iso8601(xpath_text(video_info, './/first_retrieve'))
+        if not timestamp:
+            match = self._html_search_meta('datePublished', webpage, 'date published', default=None)
+            if match:
+                timestamp = parse_iso8601(match.replace('+', ':00+'))
+        if not timestamp and video_detail.get('postedAt'):
+            timestamp = parse_iso8601(
+                video_detail['postedAt'].replace('/', '-'),
+                delimiter=' ', timezone=datetime.timedelta(hours=9))
+
         view_count = int_or_none(xpath_text(video_info, './/view_counter'))
+        if not view_count:
+            match = self._html_search_regex(
+                r'>Views: <strong[^>]*>([^<]+)</strong>',
+                webpage, 'view count', default=None)
+            if match:
+                view_count = int_or_none(match.replace(',', ''))
+        view_count = view_count or video_detail.get('viewCount')
+
         comment_count = int_or_none(xpath_text(video_info, './/comment_num'))
-        duration = parse_duration(xpath_text(video_info, './/length'))
-        webpage_url = xpath_text(video_info, './/watch_url')
+        if not comment_count:
+            match = self._html_search_regex(
+                r'>Comments: <strong[^>]*>([^<]+)</strong>',
+                webpage, 'comment count', default=None)
+            if match:
+                comment_count = int_or_none(match.replace(',', ''))
+        comment_count = comment_count or video_detail.get('commentCount')
+
+        duration = (parse_duration(
+            xpath_text(video_info, './/length') or
+            self._html_search_meta(
+                'video:duration', webpage, 'video duration', default=None)) or
+            video_detail.get('length'))
+
+        webpage_url = xpath_text(video_info, './/watch_url') or url
 
         if video_info.find('.//ch_id') is not None:
             uploader_id = video_info.find('.//ch_id').text
@@ -175,7 +221,7 @@ class NiconicoIE(InfoExtractor):
         else:
             uploader_id = uploader = None
 
-        ret = {
+        return {
             'id': video_id,
             'url': video_real_url,
             'title': title,
@@ -191,7 +237,6 @@ class NiconicoIE(InfoExtractor):
             'duration': duration,
             'webpage_url': webpage_url,
         }
-        return dict((k, v) for k, v in ret.items() if v is not None)
 
 
 class NiconicoPlaylistIE(InfoExtractor):

From 965cb8d530e4ced61a9bc42530f7f91b67c709e6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 29 Apr 2015 22:46:19 +0200
Subject: [PATCH 1114/6219] [escapist] pep8 fixes

---
 youtube_dl/extractor/escapist.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/escapist.py b/youtube_dl/extractor/escapist.py
index 600ebf078..8facf1185 100644
--- a/youtube_dl/extractor/escapist.py
+++ b/youtube_dl/extractor/escapist.py
@@ -76,7 +76,8 @@ class EscapistIE(InfoExtractor):
 
         formats = []
         for q in ['lq', 'hq', 'hd']:
-            config_req = compat_urllib_request.Request('http://www.escapistmagazine.com/videos/'
+            config_req = compat_urllib_request.Request(
+                'http://www.escapistmagazine.com/videos/'
                 'vidconfig.php?videoID=%s&hash=%s&quality=%s' % (video_id, key, 'mp4_' + q))
             config_req.add_header('Referer', url)
             config = self._download_webpage(config_req, video_id, 'Downloading video config ' + q.upper())
@@ -92,8 +93,7 @@ class EscapistIE(InfoExtractor):
                     'url': v,
                     'format_id': determine_ext(v) + '_' + q + str(i),
                     'quality': quality(q),
-                    })
-
+                })
 
         return {
             'id': video_id,

From 8dd5418803a25de89d08cdb9d32f80f71c5d6c47 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 29 Apr 2015 22:53:18 +0200
Subject: [PATCH 1115/6219] Make 'best' format only match non-DASH formats
 (closes #5554)

Otherwise it's impossible to only download non-DASH formats, for example `best[height=?480]/best` would download a DASH video if it's the only one with height=480, instead for falling back to the second format specifier.
For audio only urls (soundcloud, bandcamp ...), the best audio will be downloaded as before.
---
 test/test_YoutubeDL.py  | 4 ++--
 youtube_dl/YoutubeDL.py | 9 ++++++++-
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index bb4a65ee1..82b827536 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -237,7 +237,7 @@ class TestFormatSelection(unittest.TestCase):
             f2['url'] = 'url:' + f2id
 
             info_dict = _make_result([f1, f2], extractor='youtube')
-            ydl = YDL()
+            ydl = YDL({'format': 'best/bestvideo'})
             yie = YoutubeIE(ydl)
             yie._sort_formats(info_dict['formats'])
             ydl.process_ie_result(info_dict)
@@ -245,7 +245,7 @@ class TestFormatSelection(unittest.TestCase):
             self.assertEqual(downloaded['format_id'], f1id)
 
             info_dict = _make_result([f2, f1], extractor='youtube')
-            ydl = YDL()
+            ydl = YDL({'format': 'best/bestvideo'})
             yie = YoutubeIE(ydl)
             yie._sort_formats(info_dict['formats'])
             ydl.process_ie_result(info_dict)
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 827c88e0d..eee9c0154 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -915,7 +915,14 @@ class YoutubeDL(object):
             return None
 
         if format_spec == 'best' or format_spec is None:
-            return available_formats[-1]
+            audiovideo_formats = [
+                f for f in available_formats
+                if f.get('vcodec') != 'none' and f.get('acodec') != 'none']
+            if audiovideo_formats:
+                return audiovideo_formats[-1]
+            # for audio only urls, 'best' selects the best audio format
+            elif all(f.get('acodec') != 'none' for f in available_formats):
+                return available_formats[-1]
         elif format_spec == 'worst':
             audiovideo_formats = [
                 f for f in available_formats

From 621ffe7bf420aa1a227e823edf2f1acbc67660d0 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 30 Apr 2015 17:05:02 +0800
Subject: [PATCH 1116/6219] [niconico] Fix so* video extraction (fixes #4874)
 (#2087)

---
 youtube_dl/extractor/niconico.py | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/niconico.py b/youtube_dl/extractor/niconico.py
index dd16d0042..3cecebf95 100644
--- a/youtube_dl/extractor/niconico.py
+++ b/youtube_dl/extractor/niconico.py
@@ -67,6 +67,18 @@ class NiconicoIE(InfoExtractor):
             'timestamp': 1198527840,  # timestamp field has different value if logged in
             'duration': 304,
         },
+    }, {
+        'url': 'http://www.nicovideo.jp/watch/so22543406',
+        'info_dict': {
+            'id': '1388129933',
+            'ext': 'mp4',
+            'title': '【第1回】RADIOアニメロミックス ラブライブ！～のぞえりRadio Garden～',
+            'description': 'md5:b27d224bb0ff53d3c8269e9f8b561cf1',
+            'timestamp': 1388851200,
+            'upload_date': '20140104',
+            'uploader': 'アニメロチャンネル',
+            'uploader_id': '312',
+        }
     }]
 
     _VALID_URL = r'https?://(?:www\.|secure\.)?nicovideo\.jp/watch/(?P<id>(?:[a-z]{2})?[0-9]+)'
@@ -109,7 +121,10 @@ class NiconicoIE(InfoExtractor):
         # Get video webpage. We are not actually interested in it for normal
         # cases, but need the cookies in order to be able to download the
         # info webpage
-        webpage = self._download_webpage('http://www.nicovideo.jp/watch/' + video_id, video_id)
+        webpage, handle = self._download_webpage_handle(
+            'http://www.nicovideo.jp/watch/' + video_id, video_id)
+        if video_id.startswith('so'):
+            video_id = self._match_id(handle.geturl())
 
         video_info = self._download_xml(
             'http://ext.nicovideo.jp/api/getthumbinfo/' + video_id, video_id,

From c4a21bc9db1868e8be114f496899f6786b9982ec Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 30 Apr 2015 18:23:35 +0800
Subject: [PATCH 1117/6219] [bilibili] Extract multipart videos (closes #3250)

---
 youtube_dl/extractor/bilibili.py | 74 +++++++++++++++++++++-----------
 1 file changed, 49 insertions(+), 25 deletions(-)

diff --git a/youtube_dl/extractor/bilibili.py b/youtube_dl/extractor/bilibili.py
index 904d9a8b4..7ca835e31 100644
--- a/youtube_dl/extractor/bilibili.py
+++ b/youtube_dl/extractor/bilibili.py
@@ -2,6 +2,7 @@
 from __future__ import unicode_literals
 
 import re
+import itertools
 
 from .common import InfoExtractor
 from ..utils import (
@@ -14,18 +15,25 @@ from ..utils import (
 class BiliBiliIE(InfoExtractor):
     _VALID_URL = r'http://www\.bilibili\.(?:tv|com)/video/av(?P<id>[0-9]+)/'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.bilibili.tv/video/av1074402/',
         'md5': '2c301e4dab317596e837c3e7633e7d86',
         'info_dict': {
-            'id': '1074402',
+            'id': '1074402_part1',
             'ext': 'flv',
             'title': '【金坷垃】金泡沫',
             'duration': 308,
             'upload_date': '20140420',
             'thumbnail': 're:^https?://.+\.jpg',
         },
-    }
+    }, {
+        'url': 'http://www.bilibili.com/video/av1041170/',
+        'info_dict': {
+            'id': '1041170',
+            'title': '【BD1080P】刀语【诸神&异域】',
+        },
+        'playlist_count': 9,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -57,19 +65,14 @@ class BiliBiliIE(InfoExtractor):
 
         cid = self._search_regex(r'cid=(\d+)', webpage, 'cid')
 
+        entries = []
+
         lq_doc = self._download_xml(
             'http://interface.bilibili.com/v_cdn_play?appkey=1&cid=%s' % cid,
             video_id,
             note='Downloading LQ video info'
         )
-        lq_durl = lq_doc.find('./durl')
-        formats = [{
-            'format_id': 'lq',
-            'quality': 1,
-            'url': lq_durl.find('./url').text,
-            'filesize': int_or_none(
-                lq_durl.find('./size'), get_attr='text'),
-        }]
+        lq_durls = lq_doc.findall('./durl')
 
         hq_doc = self._download_xml(
             'http://interface.bilibili.com/playurl?appkey=1&cid=%s' % cid,
@@ -77,23 +80,44 @@ class BiliBiliIE(InfoExtractor):
             note='Downloading HQ video info',
             fatal=False,
         )
-        if hq_doc is not False:
-            hq_durl = hq_doc.find('./durl')
-            formats.append({
-                'format_id': 'hq',
-                'quality': 2,
-                'ext': 'flv',
-                'url': hq_durl.find('./url').text,
+        hq_durls = hq_doc.findall('./durl') if hq_doc is not False else itertools.repeat(None)
+
+        assert len(lq_durls) == len(hq_durls)
+
+        i = 1
+        for lq_durl, hq_durl in zip(lq_durls, hq_durls):
+            formats = [{
+                'format_id': 'lq',
+                'quality': 1,
+                'url': lq_durl.find('./url').text,
                 'filesize': int_or_none(
-                    hq_durl.find('./size'), get_attr='text'),
+                    lq_durl.find('./size'), get_attr='text'),
+            }]
+            if hq_durl:
+                formats.append({
+                    'format_id': 'hq',
+                    'quality': 2,
+                    'ext': 'flv',
+                    'url': hq_durl.find('./url').text,
+                    'filesize': int_or_none(
+                        hq_durl.find('./size'), get_attr='text'),
+                })
+            self._sort_formats(formats)
+
+            entries.append({
+                'id': '%s_part%d' % (video_id, i),
+                'title': title,
+                'formats': formats,
+                'duration': duration,
+                'upload_date': upload_date,
+                'thumbnail': thumbnail,
             })
 
-        self._sort_formats(formats)
+            i += 1
+
         return {
+            '_type': 'multi_video',
+            'entries': entries,
             'id': video_id,
-            'title': title,
-            'formats': formats,
-            'duration': duration,
-            'upload_date': upload_date,
-            'thumbnail': thumbnail,
+            'title': title
         }

From f7f1df1d82d556c0726898b9de2f7f3824c1be5a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 30 Apr 2015 22:37:41 +0800
Subject: [PATCH 1118/6219] [VeeHD] Enhance extraction and fix tests (fixes
 #4965)

---
 youtube_dl/extractor/veehd.py | 36 +++++++++++++++++++++++++++++++----
 1 file changed, 32 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/veehd.py b/youtube_dl/extractor/veehd.py
index 96353f525..7fdeb784d 100644
--- a/youtube_dl/extractor/veehd.py
+++ b/youtube_dl/extractor/veehd.py
@@ -17,7 +17,9 @@ from ..utils import (
 class VeeHDIE(InfoExtractor):
     _VALID_URL = r'https?://veehd\.com/video/(?P<id>\d+)'
 
-    _TEST = {
+    # Seems VeeHD videos have multiple copies on several servers, all of
+    # whom have different MD5 checksums, so omit md5 field in all tests
+    _TESTS = [{
         'url': 'http://veehd.com/video/4639434_Solar-Sinter',
         'info_dict': {
             'id': '4639434',
@@ -26,7 +28,26 @@ class VeeHDIE(InfoExtractor):
             'uploader_id': 'VideoEyes',
             'description': 'md5:46a840e8692ddbaffb5f81d9885cb457',
         },
-    }
+        'skip': 'Video deleted',
+    }, {
+        'url': 'http://veehd.com/video/4905758_Elysian-Fields-Channeling',
+        'info_dict': {
+            'id': '4905758',
+            'ext': 'mp4',
+            'title': 'Elysian Fields - Channeling',
+            'description': 'md5:360e4e95fdab58aefbea0f2a19e5604b',
+            'uploader_id': 'spotted',
+        }
+    }, {
+        'url': 'http://veehd.com/video/4665804_Tell-No-One-Ne-le-dis-a-personne-2006-French-EngSoftSubs-Re-Up',
+        'info_dict': {
+            'id': '4665804',
+            'ext': 'avi',
+            'title': 'Tell No One (Ne le dis a personne) 2006 French(EngSoftSubs) Re-Up',
+            'description': 'md5:d660cca685549776f37165e9a10b60ba',
+            'uploader_id': 'belial2549',
+        }
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -48,13 +69,21 @@ class VeeHDIE(InfoExtractor):
         player_page = self._download_webpage(
             player_url, video_id, 'Downloading player page')
 
+        video_url = None
+
         config_json = self._search_regex(
             r'value=\'config=({.+?})\'', player_page, 'config json', default=None)
 
         if config_json:
             config = json.loads(config_json)
             video_url = compat_urlparse.unquote(config['clip']['url'])
-        else:
+
+        if not video_url:
+            video_url = self._html_search_regex(
+                r'<embed[^>]+type="video/divx"[^>]+src="([^"]+)"',
+                player_page, 'video url', default=None)
+
+        if not video_url:
             iframe_src = self._search_regex(
                 r'<iframe[^>]+src="/?([^"]+)"', player_page, 'iframe url')
             iframe_url = 'http://veehd.com/%s' % iframe_src
@@ -82,7 +111,6 @@ class VeeHDIE(InfoExtractor):
             'id': video_id,
             'title': title,
             'url': video_url,
-            'ext': 'mp4',
             'uploader_id': uploader_id,
             'thumbnail': thumbnail,
             'description': description,

From e01c56f9e12d7abb2e3b548818a689146092806e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 30 Apr 2015 21:06:51 +0600
Subject: [PATCH 1119/6219] [YoutubeDL] Generalize best/worst format match
 behavior

---
 youtube_dl/YoutubeDL.py | 16 +++++-----------
 1 file changed, 5 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index eee9c0154..9d4a2dce8 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -914,22 +914,16 @@ class YoutubeDL(object):
         if not available_formats:
             return None
 
-        if format_spec == 'best' or format_spec is None:
+        if format_spec in ['best', 'worst', None]:
+            format_idx = 0 if format_spec == 'worst' else -1
             audiovideo_formats = [
                 f for f in available_formats
                 if f.get('vcodec') != 'none' and f.get('acodec') != 'none']
             if audiovideo_formats:
-                return audiovideo_formats[-1]
-            # for audio only urls, 'best' selects the best audio format
+                return audiovideo_formats[format_idx]
+            # for audio only urls, select the best/worst audio format
             elif all(f.get('acodec') != 'none' for f in available_formats):
-                return available_formats[-1]
-        elif format_spec == 'worst':
-            audiovideo_formats = [
-                f for f in available_formats
-                if f.get('vcodec') != 'none' and f.get('acodec') != 'none']
-            if audiovideo_formats:
-                return audiovideo_formats[0]
-            return available_formats[0]
+                return available_formats[format_idx]
         elif format_spec == 'bestaudio':
             audio_formats = [
                 f for f in available_formats

From cd298882cd6f8ed2571dd372f684ec17e992fd9d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 30 Apr 2015 21:25:17 +0600
Subject: [PATCH 1120/6219] [vporn] Fix metadata extraction (#5560)

---
 youtube_dl/extractor/vporn.py | 22 ++++++++++------------
 1 file changed, 10 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/vporn.py b/youtube_dl/extractor/vporn.py
index 2d23effcc..04ed0b381 100644
--- a/youtube_dl/extractor/vporn.py
+++ b/youtube_dl/extractor/vporn.py
@@ -64,29 +64,29 @@ class VpornIE(InfoExtractor):
         title = self._html_search_regex(
             r'videoname\s*=\s*\'([^\']+)\'', webpage, 'title').strip()
         description = self._html_search_regex(
-            r'<div class="description_txt">(.*?)</div>', webpage, 'description', fatal=False)
+            r'class="(?:descr|description_txt)">(.*?)</div>',
+            webpage, 'description', fatal=False)
         thumbnail = self._html_search_regex(
             r'flashvars\.imageUrl\s*=\s*"([^"]+)"', webpage, 'description', fatal=False, default=None)
         if thumbnail:
             thumbnail = 'http://www.vporn.com' + thumbnail
 
         uploader = self._html_search_regex(
-            r'(?s)UPLOADED BY.*?<a href="/user/[^"]+">([^<]+)</a>',
+            r'(?s)Uploaded by:.*?<a href="/user/[^"]+">([^<]+)</a>',
             webpage, 'uploader', fatal=False)
 
         categories = re.findall(r'<a href="/cat/[^"]+">([^<]+)</a>', webpage)
 
         duration = parse_duration(self._search_regex(
-            r'duration (\d+ min \d+ sec)', webpage, 'duration', fatal=False))
+            r'Runtime:\s*</span>\s*(\d+ min \d+ sec)',
+            webpage, 'duration', fatal=False))
 
-        view_count = str_to_int(self._html_search_regex(
-            r'<span>([\d,\.]+) VIEWS</span>', webpage, 'view count', fatal=False))
-        like_count = str_to_int(self._html_search_regex(
-            r'<span id="like" class="n">([\d,\.]+)</span>', webpage, 'like count', fatal=False))
-        dislike_count = str_to_int(self._html_search_regex(
-            r'<span id="dislike" class="n">([\d,\.]+)</span>', webpage, 'dislike count', fatal=False))
+        view_count = str_to_int(self._search_regex(
+            r'class="views">([\d,\.]+) [Vv]iews<',
+            webpage, 'view count', fatal=False))
         comment_count = str_to_int(self._html_search_regex(
-            r'<h4>Comments \(<b>([\d,\.]+)</b>\)</h4>', webpage, 'comment count', fatal=False))
+            r"'Comments \(([\d,\.]+)\)'",
+            webpage, 'comment count', default=None))
 
         formats = []
 
@@ -117,8 +117,6 @@ class VpornIE(InfoExtractor):
             'categories': categories,
             'duration': duration,
             'view_count': view_count,
-            'like_count': like_count,
-            'dislike_count': dislike_count,
             'comment_count': comment_count,
             'age_limit': 18,
             'formats': formats,

From 482a1258de6af0a15b6e7859d244f9125cadef47 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 30 Apr 2015 22:58:03 +0800
Subject: [PATCH 1121/6219] [VeeHD] Replace the third test case due to
 copyright issues

---
 youtube_dl/extractor/veehd.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/veehd.py b/youtube_dl/extractor/veehd.py
index 7fdeb784d..346edf485 100644
--- a/youtube_dl/extractor/veehd.py
+++ b/youtube_dl/extractor/veehd.py
@@ -39,13 +39,13 @@ class VeeHDIE(InfoExtractor):
             'uploader_id': 'spotted',
         }
     }, {
-        'url': 'http://veehd.com/video/4665804_Tell-No-One-Ne-le-dis-a-personne-2006-French-EngSoftSubs-Re-Up',
+        'url': 'http://veehd.com/video/2046729_2012-2009-DivX-Trailer',
         'info_dict': {
-            'id': '4665804',
+            'id': '2046729',
             'ext': 'avi',
-            'title': 'Tell No One (Ne le dis a personne) 2006 French(EngSoftSubs) Re-Up',
-            'description': 'md5:d660cca685549776f37165e9a10b60ba',
-            'uploader_id': 'belial2549',
+            'title': '2012 (2009) DivX Trailer',
+            'description': 'md5:75435ee95255e6a9838ac6f6f3a2396b',
+            'uploader_id': 'Movie_Trailers',
         }
     }]
 

From 7a03280df4555998fc99399907062b62383db2c4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 30 Apr 2015 21:31:38 +0600
Subject: [PATCH 1122/6219] [vporn] More metadata extraction fixes and tests
 update (#5560)

---
 youtube_dl/extractor/vporn.py | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/vporn.py b/youtube_dl/extractor/vporn.py
index 04ed0b381..92c90e517 100644
--- a/youtube_dl/extractor/vporn.py
+++ b/youtube_dl/extractor/vporn.py
@@ -27,9 +27,6 @@ class VpornIE(InfoExtractor):
                 'duration': 393,
                 'age_limit': 18,
                 'view_count': int,
-                'like_count': int,
-                'dislike_count': int,
-                'comment_count': int,
             }
         },
         {
@@ -47,9 +44,6 @@ class VpornIE(InfoExtractor):
                 'duration': 588,
                 'age_limit': 18,
                 'view_count': int,
-                'like_count': int,
-                'dislike_count': int,
-                'comment_count': int,
             }
         },
     ]
@@ -72,10 +66,10 @@ class VpornIE(InfoExtractor):
             thumbnail = 'http://www.vporn.com' + thumbnail
 
         uploader = self._html_search_regex(
-            r'(?s)Uploaded by:.*?<a href="/user/[^"]+">([^<]+)</a>',
+            r'(?s)Uploaded by:.*?<a href="/user/[^"]+"[^>]*>(.+?)</a>',
             webpage, 'uploader', fatal=False)
 
-        categories = re.findall(r'<a href="/cat/[^"]+">([^<]+)</a>', webpage)
+        categories = re.findall(r'<a href="/cat/[^"]+"[^>]*>([^<]+)</a>', webpage)
 
         duration = parse_duration(self._search_regex(
             r'Runtime:\s*</span>\s*(\d+ min \d+ sec)',

From 4070b458ece46a29dad9be2312a7daa48bb2f1d7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 30 Apr 2015 23:55:05 +0600
Subject: [PATCH 1123/6219] [YoutubeDL] Do not write requested info in info
 JSON file (Closes #5562, closes #5564)

---
 youtube_dl/YoutubeDL.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 9d4a2dce8..e747c6892 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1337,8 +1337,11 @@ class YoutubeDL(object):
                 self.to_screen('[info] Video description metadata is already present')
             else:
                 self.to_screen('[info] Writing video description metadata as JSON to: ' + infofn)
+                filtered_info_dict = dict(
+                    (k, v) for k, v in info_dict.items()
+                    if not k in ['requested_formats', 'requested_subtitles'])
                 try:
-                    write_json_file(info_dict, infofn)
+                    write_json_file(filtered_info_dict, infofn)
                 except (OSError, IOError):
                     self.report_error('Cannot write metadata to JSON file ' + infofn)
                     return

From df8301fef55f9144f06337c10b8570b6560caa24 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 30 Apr 2015 20:18:42 +0200
Subject: [PATCH 1124/6219] [YoutubeDL] pep8: use 'k not in' instead of 'not k
 in'

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index e747c6892..584dbf8a6 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1339,7 +1339,7 @@ class YoutubeDL(object):
                 self.to_screen('[info] Writing video description metadata as JSON to: ' + infofn)
                 filtered_info_dict = dict(
                     (k, v) for k, v in info_dict.items()
-                    if not k in ['requested_formats', 'requested_subtitles'])
+                    if k not in ['requested_formats', 'requested_subtitles'])
                 try:
                     write_json_file(filtered_info_dict, infofn)
                 except (OSError, IOError):

From 67fc8ecd53b4ffe5375a741bf0b1282f7a44587d Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Thu, 30 Apr 2015 21:26:55 +0300
Subject: [PATCH 1125/6219] [dreisat] Extend _VALID_URL (Closes #5548)

---
 youtube_dl/extractor/dreisat.py | 32 +++++++++++++++++++-------------
 1 file changed, 19 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/dreisat.py b/youtube_dl/extractor/dreisat.py
index 05bb22ddf..8ac8587be 100644
--- a/youtube_dl/extractor/dreisat.py
+++ b/youtube_dl/extractor/dreisat.py
@@ -11,19 +11,25 @@ from ..utils import (
 
 class DreiSatIE(InfoExtractor):
     IE_NAME = '3sat'
-    _VALID_URL = r'(?:http://)?(?:www\.)?3sat\.de/mediathek/(?:index\.php)?\?(?:(?:mode|display)=[^&]+&)*obj=(?P<id>[0-9]+)$'
-    _TEST = {
-        'url': 'http://www.3sat.de/mediathek/index.php?mode=play&obj=45918',
-        'md5': 'be37228896d30a88f315b638900a026e',
-        'info_dict': {
-            'id': '45918',
-            'ext': 'mp4',
-            'title': 'Waidmannsheil',
-            'description': 'md5:cce00ca1d70e21425e72c86a98a56817',
-            'uploader': '3sat',
-            'upload_date': '20140913'
-        }
-    }
+    _VALID_URL = r'(?:http://)?(?:www\.)?3sat\.de/mediathek/(?:index\.php|mediathek\.php)?\?(?:(?:mode|display)=[^&]+&)*obj=(?P<id>[0-9]+)$'
+    _TESTS = [
+        {
+            'url': 'http://www.3sat.de/mediathek/index.php?mode=play&obj=45918',
+            'md5': 'be37228896d30a88f315b638900a026e',
+            'info_dict': {
+                'id': '45918',
+                'ext': 'mp4',
+                'title': 'Waidmannsheil',
+                'description': 'md5:cce00ca1d70e21425e72c86a98a56817',
+                'uploader': '3sat',
+                'upload_date': '20140913'
+            }
+        },
+        {
+            'url': 'http://www.3sat.de/mediathek/mediathek.php?mode=play&obj=51066',
+            'only_matching': True,
+        },
+    ]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)

From cb202fd28635bf82836a025c631339665ba610af Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 1 May 2015 00:44:34 +0600
Subject: [PATCH 1126/6219] [YoutubeDL] Filter requested info fields on
 `--load-info` as well

In order to properly handle JSON info files generated by youtube-dl versions prior to 4070b458ece46a29dad9be2312a7daa48bb2f1d7
---
 youtube_dl/YoutubeDL.py | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 584dbf8a6..55b429f31 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1337,11 +1337,8 @@ class YoutubeDL(object):
                 self.to_screen('[info] Video description metadata is already present')
             else:
                 self.to_screen('[info] Writing video description metadata as JSON to: ' + infofn)
-                filtered_info_dict = dict(
-                    (k, v) for k, v in info_dict.items()
-                    if k not in ['requested_formats', 'requested_subtitles'])
                 try:
-                    write_json_file(filtered_info_dict, infofn)
+                    write_json_file(self.filter_requested_info(info_dict), infofn)
                 except (OSError, IOError):
                     self.report_error('Cannot write metadata to JSON file ' + infofn)
                     return
@@ -1491,7 +1488,7 @@ class YoutubeDL(object):
                 [info_filename], mode='r',
                 openhook=fileinput.hook_encoded('utf-8'))) as f:
             # FileInput doesn't have a read method, we can't call json.load
-            info = json.loads('\n'.join(f))
+            info = self.filter_requested_info(json.loads('\n'.join(f)))
         try:
             self.process_ie_result(info, download=True)
         except DownloadError:
@@ -1503,6 +1500,12 @@ class YoutubeDL(object):
                 raise
         return self._download_retcode
 
+    @staticmethod
+    def filter_requested_info(info_dict):
+        return dict(
+            (k, v) for k, v in info_dict.items()
+            if k not in ['requested_formats', 'requested_subtitles'])
+
     def post_process(self, filename, ie_info):
         """Run all the postprocessors on the given file."""
         info = dict(ie_info)

From 6a8422b942f5140238106c43e27d869d70126446 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 1 May 2015 02:49:06 +0600
Subject: [PATCH 1127/6219] [foxsports] Add extractor (Closes #5517)

---
 youtube_dl/extractor/__init__.py  |  1 +
 youtube_dl/extractor/foxsports.py | 32 +++++++++++++++++++++++++++++++
 2 files changed, 33 insertions(+)
 create mode 100644 youtube_dl/extractor/foxsports.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 641c45f43..fced42bd9 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -161,6 +161,7 @@ from .footyroom import FootyRoomIE
 from .fourtube import FourTubeIE
 from .foxgay import FoxgayIE
 from .foxnews import FoxNewsIE
+from .foxsports import FoxSportsIE
 from .franceculture import FranceCultureIE
 from .franceinter import FranceInterIE
 from .francetv import (
diff --git a/youtube_dl/extractor/foxsports.py b/youtube_dl/extractor/foxsports.py
new file mode 100644
index 000000000..363866b64
--- /dev/null
+++ b/youtube_dl/extractor/foxsports.py
@@ -0,0 +1,32 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import smuggle_url
+
+
+class FoxSportsIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?foxsports\.com/video\?vid=(?P<id>\d+)'
+
+    _TEST = {
+        'url': 'http://www.foxsports.com/video?vid=432609859715',
+        'info_dict': {
+            'id': 'gA0bHB3Ladz3',
+            'ext': 'flv',
+            'title': 'Courtney Lee on going up 2-0 in series vs. Blazers',
+            'description': 'Courtney Lee talks about Memphis being focused.',
+        },
+        'add_ie': ['ThePlatform'],
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        config = self._parse_json(
+            self._search_regex(
+                r"data-player-config='([^']+)'", webpage, 'data player config'),
+            video_id)
+
+        return self.url_result(smuggle_url(
+            config['releaseURL'] + '&manifest=f4m', {'force_smil_url': True}))

From 1dbd717eb49d075fa1efabc674e8074fd165eb0a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 1 May 2015 02:51:55 +0600
Subject: [PATCH 1128/6219] [theplaform] Fix FutureWarning

---
 youtube_dl/extractor/theplatform.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index 6a006b2d2..92731ad3d 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -129,7 +129,9 @@ class ThePlatformIE(InfoExtractor):
         head = meta.find(_x('smil:head'))
         body = meta.find(_x('smil:body'))
 
-        f4m_node = body.find(_x('smil:seq//smil:video')) or body.find(_x('smil:seq/smil:video'))
+        f4m_node = body.find(_x('smil:seq//smil:video'))
+        if f4m_node is None:
+            f4m_node = body.find(_x('smil:seq/smil:video'))
         if f4m_node is not None and '.f4m' in f4m_node.attrib['src']:
             f4m_url = f4m_node.attrib['src']
             if 'manifest.f4m?' not in f4m_url:
@@ -142,7 +144,9 @@ class ThePlatformIE(InfoExtractor):
             formats = []
             switch = body.find(_x('smil:switch'))
             if switch is None:
-                switch = body.find(_x('smil:par//smil:switch')) or body.find(_x('smil:par/smil:switch'))
+                switch = body.find(_x('smil:par//smil:switch'))
+            if switch is None:
+                switch = body.find(_x('smil:par/smil:switch'))
             if switch is None:
                 switch = body.find(_x('smil:par'))
             if switch is not None:
@@ -163,7 +167,9 @@ class ThePlatformIE(InfoExtractor):
                         'vbr': vbr,
                     })
             else:
-                switch = body.find(_x('smil:seq//smil:switch')) or body.find(_x('smil:seq/smil:switch'))
+                switch = body.find(_x('smil:seq//smil:switch'))
+                if switch is None:
+                    switch = body.find(_x('smil:seq/smil:switch'))
                 for f in switch.findall(_x('smil:video')):
                     attr = f.attrib
                     vbr = int_or_none(attr.get('system-bitrate'), 1000)

From 8683b4d8d91a7c6b72ca4a12bf6b538cbb4b2a68 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 1 May 2015 03:59:13 +0600
Subject: [PATCH 1129/6219] [bbccouk] Improve extraction (Closes #5530)

---
 youtube_dl/extractor/bbccouk.py | 35 ++++++++++++++++++++++++---------
 1 file changed, 26 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/bbccouk.py b/youtube_dl/extractor/bbccouk.py
index abc34a576..22c2843be 100644
--- a/youtube_dl/extractor/bbccouk.py
+++ b/youtube_dl/extractor/bbccouk.py
@@ -3,7 +3,10 @@ from __future__ import unicode_literals
 import xml.etree.ElementTree
 
 from .common import InfoExtractor
-from ..utils import ExtractorError
+from ..utils import (
+    ExtractorError,
+    int_or_none,
+)
 from ..compat import compat_HTTPError
 
 
@@ -326,16 +329,29 @@ class BBCCoUkIE(InfoExtractor):
 
         webpage = self._download_webpage(url, group_id, 'Downloading video page')
 
-        programme_id = self._search_regex(
-            r'"vpid"\s*:\s*"([\da-z]{8})"', webpage, 'vpid', fatal=False, default=None)
+        thumbnail = self._og_search_thumbnail(webpage)
+
+        programme_id = None
+
+        tviplayer = self._search_regex(
+            r'mediator\.bind\(({.+?})\s*,\s*document\.getElementById',
+            webpage, 'player', default=None)
+
+        if tviplayer:
+            player = self._parse_json(tviplayer, group_id).get('player', {})
+            duration = int_or_none(player.get('duration'))
+            programme_id = player.get('vpid')
+
+        if not programme_id:
+            programme_id = self._search_regex(
+                r'"vpid"\s*:\s*"([\da-z]{8})"', webpage, 'vpid', fatal=False, default=None)
+
         if programme_id:
-            player = self._download_json(
-                'http://www.bbc.co.uk/iplayer/episode/%s.json' % group_id,
-                group_id)['jsConf']['player']
-            title = player['title']
-            description = player['subtitle']
-            duration = player['duration']
             formats, subtitles = self._download_media_selector(programme_id)
+            title = self._og_search_title(webpage)
+            description = self._search_regex(
+                r'<p class="medium-description">([^<]+)</p>',
+                webpage, 'description', fatal=False)
         else:
             programme_id, title, description, duration, formats, subtitles = self._download_playlist(group_id)
 
@@ -345,6 +361,7 @@ class BBCCoUkIE(InfoExtractor):
             'id': programme_id,
             'title': title,
             'description': description,
+            'thumbnail': thumbnail,
             'duration': duration,
             'formats': formats,
             'subtitles': subtitles,

From e68ae99a417f39db269dcffb5011cfcc8341552d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 1 May 2015 04:02:56 +0600
Subject: [PATCH 1130/6219] [bbccouk] Add test for #5530

---
 youtube_dl/extractor/bbccouk.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/youtube_dl/extractor/bbccouk.py b/youtube_dl/extractor/bbccouk.py
index 22c2843be..dbfbbb5ca 100644
--- a/youtube_dl/extractor/bbccouk.py
+++ b/youtube_dl/extractor/bbccouk.py
@@ -115,6 +115,20 @@ class BBCCoUkIE(InfoExtractor):
                 # rtmp download
                 'skip_download': True,
             }
+        }, {
+            'url': 'http://www.bbc.co.uk/iplayer/episode/b054fn09/ad/natural-world-20152016-2-super-powered-owls',
+            'info_dict': {
+                'id': 'p02n76xf',
+                'ext': 'flv',
+                'title': 'Natural World, 2015-2016: 2. Super Powered Owls',
+                'description': 'md5:e4db5c937d0e95a7c6b5e654d429183d',
+                'duration': 3540,
+            },
+            'params': {
+                # rtmp download
+                'skip_download': True,
+            },
+            'skip': 'geolocation',
         }, {
             'url': 'http://www.bbc.co.uk/iplayer/playlist/p01dvks4',
             'only_matching': True,

From 650cfd0cb0e330c8e6b1a5cc43a5a20d54b4714c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 1 May 2015 04:07:30 +0600
Subject: [PATCH 1131/6219] [bbccouk] Mute thumbnail

---
 youtube_dl/extractor/bbccouk.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/bbccouk.py b/youtube_dl/extractor/bbccouk.py
index dbfbbb5ca..249bc6bbd 100644
--- a/youtube_dl/extractor/bbccouk.py
+++ b/youtube_dl/extractor/bbccouk.py
@@ -343,8 +343,6 @@ class BBCCoUkIE(InfoExtractor):
 
         webpage = self._download_webpage(url, group_id, 'Downloading video page')
 
-        thumbnail = self._og_search_thumbnail(webpage)
-
         programme_id = None
 
         tviplayer = self._search_regex(
@@ -375,7 +373,7 @@ class BBCCoUkIE(InfoExtractor):
             'id': programme_id,
             'title': title,
             'description': description,
-            'thumbnail': thumbnail,
+            'thumbnail': self._og_search_thumbnail(webpage, default=None),
             'duration': duration,
             'formats': formats,
             'subtitles': subtitles,

From 861e65eb0573c824cf82e1f31b7169df2efa74ab Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 1 May 2015 12:31:31 +0800
Subject: [PATCH 1132/6219] [yahoo] Extend _VALID_URL

---
 youtube_dl/extractor/yahoo.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/yahoo.py b/youtube_dl/extractor/yahoo.py
index b777159c5..bf4e659ac 100644
--- a/youtube_dl/extractor/yahoo.py
+++ b/youtube_dl/extractor/yahoo.py
@@ -22,7 +22,7 @@ from .nbc import NBCSportsVPlayerIE
 
 class YahooIE(InfoExtractor):
     IE_DESC = 'Yahoo screen and movies'
-    _VALID_URL = r'(?P<url>(?P<host>https?://(?:[a-zA-Z]{2}\.)?[\da-zA-Z_-]+\.yahoo\.com)/(?:[^/]+/)*(?P<display_id>.+?)-(?P<id>[0-9]+)(?:-[a-z]+)?\.html)'
+    _VALID_URL = r'(?P<url>(?P<host>https?://(?:[a-zA-Z]{2}\.)?[\da-zA-Z_-]+\.yahoo\.com)/(?:[^/]+/)*(?P<display_id>.+)?-(?P<id>[0-9]+)(?:-[a-z]+)?\.html)'
     _TESTS = [
         {
             'url': 'http://screen.yahoo.com/julian-smith-travis-legg-watch-214727115.html',
@@ -140,12 +140,15 @@ class YahooIE(InfoExtractor):
                 'description': 'md5:df390f70a9ba7c95ff1daace988f0d8d',
                 'title': 'Tyler Kalinoski hits buzzer-beater to lift Davidson',
             }
+        }, {
+            'url': 'https://tw.news.yahoo.com/-100120367.html',
+            'only_matching': True,
         }
     ]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        display_id = mobj.group('display_id')
+        display_id = mobj.group('display_id') or self._match_id(url)
         page_id = mobj.group('id')
         url = mobj.group('url')
         host = mobj.group('host')

From 083c1bb960715031aed63dfb834a5bdc5ac6ff9a Mon Sep 17 00:00:00 2001
From: Nikoli <nikoli@gmx.us>
Date: Wed, 15 Apr 2015 20:27:40 +0300
Subject: [PATCH 1133/6219] Add ability to embed subtitles in mkv files (closes
 #5434)

---
 README.md                          | 2 +-
 youtube_dl/options.py              | 2 +-
 youtube_dl/postprocessor/ffmpeg.py | 7 ++++---
 3 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/README.md b/README.md
index e1f30ca47..3432546fc 100644
--- a/README.md
+++ b/README.md
@@ -216,7 +216,7 @@ which means you can modify it, redistribute it or use it however you like.
     --recode-video FORMAT            Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm|mkv)
     -k, --keep-video                 Keep the video file on disk after the post-processing; the video is erased by default
     --no-post-overwrites             Do not overwrite post-processed files; the post-processed files are overwritten by default
-    --embed-subs                     Embed subtitles in the video (only for mp4 videos)
+    --embed-subs                     Embed subtitles in the video (only for mkv and mp4 videos)
     --embed-thumbnail                Embed thumbnail in the audio as cover art
     --add-metadata                   Write metadata to the video file
     --metadata-from-title FORMAT     Parse additional metadata like song title / artist from the video title. The format syntax is the same as --output, the parsed
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 4c9d39d9a..d0aa8296d 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -698,7 +698,7 @@ def parseOpts(overrideArguments=None):
     postproc.add_option(
         '--embed-subs',
         action='store_true', dest='embedsubtitles', default=False,
-        help='Embed subtitles in the video (only for mp4 videos)')
+        help='Embed subtitles in the video (only for mkv and mp4 videos)')
     postproc.add_option(
         '--embed-thumbnail',
         action='store_true', dest='embedthumbnail', default=False,
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 1765f4969..214de39f9 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -501,8 +501,8 @@ class FFmpegEmbedSubtitlePP(FFmpegPostProcessor):
         return cls._lang_map.get(code[:2])
 
     def run(self, information):
-        if information['ext'] != 'mp4':
-            self._downloader.to_screen('[ffmpeg] Subtitles can only be embedded in mp4 files')
+        if information['ext'] not in ['mp4', 'mkv']:
+            self._downloader.to_screen('[ffmpeg] Subtitles can only be embedded in mp4 or mkv files')
             return [], information
         subtitles = information.get('requested_subtitles')
         if not subtitles:
@@ -520,8 +520,9 @@ class FFmpegEmbedSubtitlePP(FFmpegPostProcessor):
             # Don't copy the existing subtitles, we may be running the
             # postprocessor a second time
             '-map', '-0:s',
-            '-c:s', 'mov_text',
         ]
+        if information['ext'] == 'mp4':
+            opts += ['-c:s', 'mov_text']
         for (i, lang) in enumerate(sub_langs):
             opts.extend(['-map', '%d:0' % (i + 1)])
             lang_code = self._conver_lang_code(lang)

From 5890eef6b021845cb68882107364f1b04d773913 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 1 May 2015 17:43:06 +0600
Subject: [PATCH 1134/6219] [pbs] Add support for HD (Closes #3564, closes
 #5390)

---
 youtube_dl/extractor/pbs.py | 56 ++++++++++++++++++++++++++-----------
 1 file changed, 40 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index afce732e1..761bd6d8d 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -5,6 +5,8 @@ import re
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
+    determine_ext,
+    int_or_none,
     unified_strdate,
     US_RATINGS,
 )
@@ -149,21 +151,44 @@ class PBSIE(InfoExtractor):
                 for vid_id in video_id]
             return self.playlist_result(entries, display_id)
 
-        info_url = 'http://video.pbs.org/videoInfo/%s?format=json' % video_id
-        info = self._download_json(info_url, display_id)
+        info = self._download_json(
+            'http://video.pbs.org/videoInfo/%s?format=json&type=partner' % video_id,
+            display_id)
 
-        redirect_url = info['alternate_encoding']['url']
-        redirect_info = self._download_json(
-            redirect_url + '?format=json', display_id,
-            'Downloading video url info')
-        if redirect_info['status'] == 'error':
-            if redirect_info['http_code'] == 403:
-                message = (
-                    'The video is not available in your region due to '
-                    'right restrictions')
+        formats = []
+        for encoding_name in ('recommended_encoding', 'alternate_encoding'):
+            redirect = info.get(encoding_name)
+            if not redirect:
+                continue
+            redirect_url = redirect.get('url')
+            if not redirect_url:
+                continue
+
+            redirect_info = self._download_json(
+                redirect_url + '?format=json', display_id,
+                'Downloading %s video url info' % encoding_name)
+
+            if redirect_info['status'] == 'error':
+                if redirect_info['http_code'] == 403:
+                    message = (
+                        'The video is not available in your region due to '
+                        'right restrictions')
+                else:
+                    message = redirect_info['message']
+                raise ExtractorError(message, expected=True)
+
+            format_url = redirect_info.get('url')
+            if not format_url:
+                continue
+
+            if determine_ext(format_url) == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    format_url, display_id, 'mp4', preference=1, m3u8_id='hls'))
             else:
-                message = redirect_info['message']
-            raise ExtractorError(message, expected=True)
+                formats.append({
+                    'url': format_url,
+                })
+        self._sort_formats(formats)
 
         rating_str = info.get('rating')
         if rating_str is not None:
@@ -174,11 +199,10 @@ class PBSIE(InfoExtractor):
             'id': video_id,
             'display_id': display_id,
             'title': info['title'],
-            'url': redirect_info['url'],
-            'ext': 'mp4',
             'description': info['program'].get('description'),
             'thumbnail': info.get('image_url'),
-            'duration': info.get('duration'),
+            'duration': int_or_none(info.get('duration')),
             'age_limit': age_limit,
             'upload_date': upload_date,
+            'formats': formats,
         }

From 8e3df9dfeef8503e9a8c01fcf42008d376d8d64d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 2 May 2015 00:08:38 +0800
Subject: [PATCH 1135/6219] [viki] Fix extractor and add a global availble test
 case

---
 youtube_dl/extractor/viki.py | 28 +++++++++++++++++++++++-----
 1 file changed, 23 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index 6816dacb6..957e3c01e 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -2,7 +2,10 @@ from __future__ import unicode_literals
 
 import re
 
-from ..compat import compat_urlparse
+from ..compat import (
+    compat_urlparse,
+    compat_urllib_request,
+)
 from ..utils import (
     ExtractorError,
     unescapeHTML,
@@ -15,8 +18,11 @@ from .common import InfoExtractor
 class VikiIE(InfoExtractor):
     IE_NAME = 'viki'
 
+    # iPad2
+    _USER_AGENT = 'Mozilla/5.0(iPad; U; CPU OS 4_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8F191 Safari/6533.18.5'
+
     _VALID_URL = r'^https?://(?:www\.)?viki\.com/videos/(?P<id>[0-9]+v)'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.viki.com/videos/1023585v-heirs-episode-14',
         'info_dict': {
             'id': '1023585v',
@@ -28,7 +34,17 @@ class VikiIE(InfoExtractor):
             'age_limit': 13,
         },
         'skip': 'Blocked in the US',
-    }
+    }, {
+        'url': 'http://www.viki.com/videos/1067139v-the-avengers-age-of-ultron-press-conference',
+        'md5': 'ca6493e6f0a6ec07da9aa8d6304b4b2c',
+        'info_dict': {
+            'id': '1067139v',
+            'ext': 'mp4',
+            'description': 'md5:d70b2f9428f5488321bfe1db10d612ea',
+            'upload_date': '20150430',
+            'title': '\'The Avengers: Age of Ultron\' Press Conference',
+        }
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -50,9 +66,11 @@ class VikiIE(InfoExtractor):
             'rating information', default='').strip()
         age_limit = US_RATINGS.get(rating_str)
 
-        info_url = 'http://www.viki.com/player5_fragment/%s?action=show&controller=videos' % video_id
+        req = compat_urllib_request.Request(
+            'http://www.viki.com/player5_fragment/%s?action=show&controller=videos' % video_id)
+        req.add_header('User-Agent', self._USER_AGENT)
         info_webpage = self._download_webpage(
-            info_url, video_id, note='Downloading info page')
+            req, video_id, note='Downloading info page')
         if re.match(r'\s*<div\s+class="video-error', info_webpage):
             raise ExtractorError(
                 'Video %s is blocked from your location.' % video_id,

From 89966a5aeacb70cc19b0a87a0514be824d2409af Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 2 May 2015 00:32:46 +0800
Subject: [PATCH 1136/6219] [viki] Enhance error message handling (#3774)

---
 youtube_dl/extractor/viki.py | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index 957e3c01e..0fc1ceb19 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -11,6 +11,7 @@ from ..utils import (
     unescapeHTML,
     unified_strdate,
     US_RATINGS,
+    clean_html,
 )
 from .common import InfoExtractor
 
@@ -71,10 +72,15 @@ class VikiIE(InfoExtractor):
         req.add_header('User-Agent', self._USER_AGENT)
         info_webpage = self._download_webpage(
             req, video_id, note='Downloading info page')
-        if re.match(r'\s*<div\s+class="video-error', info_webpage):
-            raise ExtractorError(
-                'Video %s is blocked from your location.' % video_id,
-                expected=True)
+        err_msg = self._html_search_regex(r'<div[^>]+class="video-error[^>]+>(.+)</div>', info_webpage, 'error message', default=None)
+        if err_msg:
+            err_msg = clean_html(err_msg)
+            if 'not available in your region' in err_msg:
+                raise ExtractorError(
+                    'Video %s is blocked from your location.' % video_id,
+                    expected=True)
+            else:
+                raise ExtractorError('Viki said: ' + err_msg)
         video_url = self._html_search_regex(
             r'<source[^>]+src="([^"]+)"', info_webpage, 'video URL')
 

From d948e09b6174179adb43ac10cdb95dffeb4854dd Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 2 May 2015 01:19:06 +0800
Subject: [PATCH 1137/6219] [viki] Extract m3u8 videos (#4855)

---
 youtube_dl/extractor/viki.py | 34 +++++++++++++++++++++++++++++++---
 1 file changed, 31 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index 0fc1ceb19..f85e43042 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -12,6 +12,8 @@ from ..utils import (
     unified_strdate,
     US_RATINGS,
     clean_html,
+    determine_ext,
+    mimetype2ext,
 )
 from .common import InfoExtractor
 
@@ -45,6 +47,19 @@ class VikiIE(InfoExtractor):
             'upload_date': '20150430',
             'title': '\'The Avengers: Age of Ultron\' Press Conference',
         }
+    }, {
+        'url': 'http://www.viki.com/videos/1048879v-ankhon-dekhi',
+        'info_dict': {
+            'id': '1048879v',
+            'ext': 'mp4',
+            'upload_date': '20140820',
+            'description': 'md5:54ff56d51bdfc7a30441ec967394e91c',
+            'title': 'Ankhon Dekhi',
+        },
+        'params': {
+            # requires ffmpeg
+            'skip_download': True,
+        }
     }]
 
     def _real_extract(self, url):
@@ -81,8 +96,21 @@ class VikiIE(InfoExtractor):
                     expected=True)
             else:
                 raise ExtractorError('Viki said: ' + err_msg)
-        video_url = self._html_search_regex(
-            r'<source[^>]+src="([^"]+)"', info_webpage, 'video URL')
+        mobj = re.search(
+            r'<source[^>]+type="(?P<mime_type>[^"]+)"[^>]+src="(?P<url>[^"]+)"', info_webpage)
+        if not mobj:
+            raise ExtractorError('Unable to find video URL')
+        video_url = unescapeHTML(mobj.group('url'))
+        video_ext = mimetype2ext(mobj.group('mime_type'))
+
+        if determine_ext(video_url) == 'm3u8':
+            formats = self._extract_m3u8_formats(
+                video_url, video_id, ext=video_ext)
+        else:
+            formats = [{
+                'url': video_url,
+                'ext': video_ext,
+            }]
 
         upload_date_str = self._html_search_regex(
             r'"created_at":"([^"]+)"', info_webpage, 'upload date')
@@ -98,7 +126,7 @@ class VikiIE(InfoExtractor):
         return {
             'id': video_id,
             'title': title,
-            'url': video_url,
+            'formats': formats,
             'description': description,
             'thumbnail': thumbnail,
             'age_limit': age_limit,

From 2eb0192155d53d70ae89ab8553ead49feb860ea4 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 2 May 2015 01:35:46 +0800
Subject: [PATCH 1138/6219] [viki] Remove clean_html call

---
 youtube_dl/extractor/viki.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index f85e43042..cf6af1e5c 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -11,7 +11,6 @@ from ..utils import (
     unescapeHTML,
     unified_strdate,
     US_RATINGS,
-    clean_html,
     determine_ext,
     mimetype2ext,
 )
@@ -89,7 +88,6 @@ class VikiIE(InfoExtractor):
             req, video_id, note='Downloading info page')
         err_msg = self._html_search_regex(r'<div[^>]+class="video-error[^>]+>(.+)</div>', info_webpage, 'error message', default=None)
         if err_msg:
-            err_msg = clean_html(err_msg)
             if 'not available in your region' in err_msg:
                 raise ExtractorError(
                     'Video %s is blocked from your location.' % video_id,

From c938c35f957ea069eed824131ca908608853abe9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 2 May 2015 07:18:22 +0600
Subject: [PATCH 1139/6219] [iconosquare] Fix extraction

---
 youtube_dl/extractor/iconosquare.py | 57 ++++++++++++++++++++++++-----
 1 file changed, 48 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/iconosquare.py b/youtube_dl/extractor/iconosquare.py
index 370e86e5a..70e4c0d41 100644
--- a/youtube_dl/extractor/iconosquare.py
+++ b/youtube_dl/extractor/iconosquare.py
@@ -1,36 +1,75 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..utils import int_or_none
 
 
 class IconosquareIE(InfoExtractor):
-    _VALID_URL = r'https?://(www\.)?(?:iconosquare\.com|statigr\.am)/p/(?P<id>[^/]+)'
+    _VALID_URL = r'https?://(?:www\.)?(?:iconosquare\.com|statigr\.am)/p/(?P<id>[^/]+)'
     _TEST = {
         'url': 'http://statigr.am/p/522207370455279102_24101272',
         'md5': '6eb93b882a3ded7c378ee1d6884b1814',
         'info_dict': {
             'id': '522207370455279102_24101272',
             'ext': 'mp4',
-            'uploader_id': 'aguynamedpatrick',
-            'title': 'Instagram photo by @aguynamedpatrick (Patrick Janelle)',
+            'title': 'Instagram media by @aguynamedpatrick (Patrick Janelle)',
             'description': 'md5:644406a9ec27457ed7aa7a9ebcd4ce3d',
+            'timestamp': 1376471991,
+            'upload_date': '20130814',
+            'uploader': 'aguynamedpatrick',
+            'uploader_id': '24101272',
+            'comment_count': int,
+            'like_count': int,
         },
     }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
+
         webpage = self._download_webpage(url, video_id)
+
+        media = self._parse_json(
+            self._search_regex(
+                r'window\.media\s*=\s*({.+?});\n', webpage, 'media'),
+            video_id)
+
+        formats = [{
+            'url': f['url'],
+            'format_id': format_id,
+            'width': int_or_none(f.get('width')),
+            'height': int_or_none(f.get('height'))
+        } for format_id, f in media['videos'].items()]
+        self._sort_formats(formats)
+
         title = self._html_search_regex(
             r'<title>(.+?)(?: *\(Videos?\))? \| (?:Iconosquare|Statigram)</title>',
             webpage, 'title')
-        uploader_id = self._html_search_regex(
-            r'@([^ ]+)', title, 'uploader name', fatal=False)
+
+        timestamp = int_or_none(media.get('created_time') or media.get('caption', {}).get('created_time'))
+        description = media.get('caption', {}).get('text')
+
+        uploader = media.get('user', {}).get('username')
+        uploader_id = media.get('user', {}).get('id')
+
+        comment_count = int_or_none(media.get('comments', {}).get('count'))
+        like_count = int_or_none(media.get('likes', {}).get('count'))
+
+        thumbnails = [{
+            'url': t['url'],
+            'id': thumbnail_id,
+            'width': int_or_none(t.get('width')),
+            'height': int_or_none(t.get('height'))
+        } for thumbnail_id, t in media.get('images', {}).items()]
 
         return {
             'id': video_id,
-            'url': self._og_search_video_url(webpage),
             'title': title,
-            'description': self._og_search_description(webpage),
-            'thumbnail': self._og_search_thumbnail(webpage),
-            'uploader_id': uploader_id
+            'description': description,
+            'thumbnails': thumbnails,
+            'timestamp': timestamp,
+            'uploader': uploader,
+            'uploader_id': uploader_id,
+            'comment_count': comment_count,
+            'like_count': like_count,
+            'formats': formats,
         }

From dd8920653c7249545bbcf5a690ffca702c9f29f2 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 2 May 2015 21:45:25 +0800
Subject: [PATCH 1140/6219] [Grooveshark] Remove the extractor

grooveshark.com was shut down on 2015/04/30
---
 youtube_dl/extractor/__init__.py    |   1 -
 youtube_dl/extractor/grooveshark.py | 191 ----------------------------
 2 files changed, 192 deletions(-)
 delete mode 100644 youtube_dl/extractor/grooveshark.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index fced42bd9..43f3e6808 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -199,7 +199,6 @@ from .googleplus import GooglePlusIE
 from .googlesearch import GoogleSearchIE
 from .gorillavid import GorillaVidIE
 from .goshgay import GoshgayIE
-from .grooveshark import GroovesharkIE
 from .groupon import GrouponIE
 from .hark import HarkIE
 from .hearthisat import HearThisAtIE
diff --git a/youtube_dl/extractor/grooveshark.py b/youtube_dl/extractor/grooveshark.py
deleted file mode 100644
index 36ad4915c..000000000
--- a/youtube_dl/extractor/grooveshark.py
+++ /dev/null
@@ -1,191 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-import time
-import math
-import os.path
-import re
-
-
-from .common import InfoExtractor
-from ..compat import (
-    compat_html_parser,
-    compat_urllib_parse,
-    compat_urllib_request,
-    compat_urlparse,
-)
-from ..utils import ExtractorError
-
-
-class GroovesharkHtmlParser(compat_html_parser.HTMLParser):
-    def __init__(self):
-        self._current_object = None
-        self.objects = []
-        compat_html_parser.HTMLParser.__init__(self)
-
-    def handle_starttag(self, tag, attrs):
-        attrs = dict((k, v) for k, v in attrs)
-        if tag == 'object':
-            self._current_object = {'attrs': attrs, 'params': []}
-        elif tag == 'param':
-            self._current_object['params'].append(attrs)
-
-    def handle_endtag(self, tag):
-        if tag == 'object':
-            self.objects.append(self._current_object)
-            self._current_object = None
-
-    @classmethod
-    def extract_object_tags(cls, html):
-        p = cls()
-        p.feed(html)
-        p.close()
-        return p.objects
-
-
-class GroovesharkIE(InfoExtractor):
-    _VALID_URL = r'https?://(www\.)?grooveshark\.com/#!/s/([^/]+)/([^/]+)'
-    _TEST = {
-        'url': 'http://grooveshark.com/#!/s/Jolene+Tenth+Key+Remix+Ft+Will+Sessions/6SS1DW?src=5',
-        'md5': '7ecf8aefa59d6b2098517e1baa530023',
-        'info_dict': {
-            'id': '6SS1DW',
-            'title': 'Jolene (Tenth Key Remix ft. Will Sessions)',
-            'ext': 'mp3',
-            'duration': 227,
-        }
-    }
-
-    do_playerpage_request = True
-    do_bootstrap_request = True
-
-    def _parse_target(self, target):
-        uri = compat_urlparse.urlparse(target)
-        hash = uri.fragment[1:].split('?')[0]
-        token = os.path.basename(hash.rstrip('/'))
-        return (uri, hash, token)
-
-    def _build_bootstrap_url(self, target):
-        (uri, hash, token) = self._parse_target(target)
-        query = 'getCommunicationToken=1&hash=%s&%d' % (compat_urllib_parse.quote(hash, safe=''), self.ts)
-        return (compat_urlparse.urlunparse((uri.scheme, uri.netloc, '/preload.php', None, query, None)), token)
-
-    def _build_meta_url(self, target):
-        (uri, hash, token) = self._parse_target(target)
-        query = 'hash=%s&%d' % (compat_urllib_parse.quote(hash, safe=''), self.ts)
-        return (compat_urlparse.urlunparse((uri.scheme, uri.netloc, '/preload.php', None, query, None)), token)
-
-    def _build_stream_url(self, meta):
-        return compat_urlparse.urlunparse(('http', meta['streamKey']['ip'], '/stream.php', None, None, None))
-
-    def _build_swf_referer(self, target, obj):
-        (uri, _, _) = self._parse_target(target)
-        return compat_urlparse.urlunparse((uri.scheme, uri.netloc, obj['attrs']['data'], None, None, None))
-
-    def _transform_bootstrap(self, js):
-        return re.split('(?m)^\s*try\s*\{', js)[0] \
-                 .split(' = ', 1)[1].strip().rstrip(';')
-
-    def _transform_meta(self, js):
-        return js.split('\n')[0].split('=')[1].rstrip(';')
-
-    def _get_meta(self, target):
-        (meta_url, token) = self._build_meta_url(target)
-        self.to_screen('Metadata URL: %s' % meta_url)
-
-        headers = {'Referer': compat_urlparse.urldefrag(target)[0]}
-        req = compat_urllib_request.Request(meta_url, headers=headers)
-        res = self._download_json(req, token,
-                                  transform_source=self._transform_meta)
-
-        if 'getStreamKeyWithSong' not in res:
-            raise ExtractorError(
-                'Metadata not found. URL may be malformed, or Grooveshark API may have changed.')
-
-        if res['getStreamKeyWithSong'] is None:
-            raise ExtractorError(
-                'Metadata download failed, probably due to Grooveshark anti-abuse throttling. Wait at least an hour before retrying from this IP.',
-                expected=True)
-
-        return res['getStreamKeyWithSong']
-
-    def _get_bootstrap(self, target):
-        (bootstrap_url, token) = self._build_bootstrap_url(target)
-
-        headers = {'Referer': compat_urlparse.urldefrag(target)[0]}
-        req = compat_urllib_request.Request(bootstrap_url, headers=headers)
-        res = self._download_json(req, token, fatal=False,
-                                  note='Downloading player bootstrap data',
-                                  errnote='Unable to download player bootstrap data',
-                                  transform_source=self._transform_bootstrap)
-        return res
-
-    def _get_playerpage(self, target):
-        (_, _, token) = self._parse_target(target)
-
-        webpage = self._download_webpage(
-            target, token,
-            note='Downloading player page',
-            errnote='Unable to download player page',
-            fatal=False)
-
-        if webpage is not None:
-            # Search (for example German) error message
-            error_msg = self._html_search_regex(
-                r'<div id="content">\s*<h2>(.*?)</h2>', webpage,
-                'error message', default=None)
-            if error_msg is not None:
-                error_msg = error_msg.replace('\n', ' ')
-                raise ExtractorError('Grooveshark said: %s' % error_msg)
-
-        if webpage is not None:
-            o = GroovesharkHtmlParser.extract_object_tags(webpage)
-            return webpage, [x for x in o if x['attrs']['id'] == 'jsPlayerEmbed']
-
-        return webpage, None
-
-    def _real_initialize(self):
-        self.ts = int(time.time() * 1000)  # timestamp in millis
-
-    def _real_extract(self, url):
-        (target_uri, _, token) = self._parse_target(url)
-
-        # 1. Fill cookiejar by making a request to the player page
-        swf_referer = None
-        if self.do_playerpage_request:
-            (_, player_objs) = self._get_playerpage(url)
-            if player_objs:
-                swf_referer = self._build_swf_referer(url, player_objs[0])
-                self.to_screen('SWF Referer: %s' % swf_referer)
-
-        # 2. Ask preload.php for swf bootstrap data to better mimic webapp
-        if self.do_bootstrap_request:
-            bootstrap = self._get_bootstrap(url)
-            self.to_screen('CommunicationToken: %s' % bootstrap['getCommunicationToken'])
-
-        # 3. Ask preload.php for track metadata.
-        meta = self._get_meta(url)
-
-        # 4. Construct stream request for track.
-        stream_url = self._build_stream_url(meta)
-        duration = int(math.ceil(float(meta['streamKey']['uSecs']) / 1000000))
-        post_dict = {'streamKey': meta['streamKey']['streamKey']}
-        post_data = compat_urllib_parse.urlencode(post_dict).encode('utf-8')
-        headers = {
-            'Content-Length': len(post_data),
-            'Content-Type': 'application/x-www-form-urlencoded'
-        }
-        if swf_referer is not None:
-            headers['Referer'] = swf_referer
-
-        return {
-            'id': token,
-            'title': meta['song']['Name'],
-            'http_method': 'POST',
-            'url': stream_url,
-            'ext': 'mp3',
-            'format': 'mp3 audio',
-            'duration': duration,
-            'http_post_data': post_data,
-            'http_headers': headers,
-        }

From 2ddcd88129e633ead8c5faee6c930b66dddb1b32 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 2 May 2015 17:29:56 +0200
Subject: [PATCH 1141/6219] Remove code that was only used by the Grooveshark
 extractor

---
 youtube_dl/compat.py           | 6 ------
 youtube_dl/downloader/http.py  | 9 ++-------
 youtube_dl/extractor/common.py | 3 ---
 3 files changed, 2 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 973bcd320..f9529210d 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -46,11 +46,6 @@ try:
 except ImportError:  # Python 2
     import htmlentitydefs as compat_html_entities
 
-try:
-    import html.parser as compat_html_parser
-except ImportError:  # Python 2
-    import HTMLParser as compat_html_parser
-
 try:
     import http.client as compat_http_client
 except ImportError:  # Python 2
@@ -404,7 +399,6 @@ __all__ = [
     'compat_getenv',
     'compat_getpass',
     'compat_html_entities',
-    'compat_html_parser',
     'compat_http_client',
     'compat_http_server',
     'compat_kwargs',
diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index d136bebd1..b7f144af9 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -28,13 +28,8 @@ class HttpFD(FileDownloader):
         add_headers = info_dict.get('http_headers')
         if add_headers:
             headers.update(add_headers)
-        data = info_dict.get('http_post_data')
-        http_method = info_dict.get('http_method')
-        basic_request = compat_urllib_request.Request(url, data, headers)
-        request = compat_urllib_request.Request(url, data, headers)
-        if http_method is not None:
-            basic_request.get_method = lambda: http_method
-            request.get_method = lambda: http_method
+        basic_request = compat_urllib_request.Request(url, None, headers)
+        request = compat_urllib_request.Request(url, None, headers)
 
         is_test = self.params.get('test', False)
 
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 48f58aae3..3ae5d5212 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -111,11 +111,8 @@ class InfoExtractor(object):
                                   (quality takes higher priority)
                                  -1 for default (order by other properties),
                                  -2 or smaller for less than default.
-                    * http_method  HTTP method to use for the download.
                     * http_headers  A dictionary of additional HTTP headers
                                  to add to the request.
-                    * http_post_data  Additional data to send with a POST
-                                 request.
                     * stretched_ratio  If given and not 1, indicates that the
                                  video's pixels are not square.
                                  width : height ratio as float.

From 38c6902b9051fa7e1dbda79808a90dbf2aac37ca Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 2 May 2015 22:52:21 +0600
Subject: [PATCH 1142/6219] [YoutubeDL] Ensure correct extension is always
 present for a merged file (Closes #5535)

---
 youtube_dl/YoutubeDL.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 55b429f31..eaa436bf9 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1382,11 +1382,18 @@ class YoutubeDL(object):
                         # TODO: Check acodec/vcodec
                         return False
 
+                    filename_real_ext = os.path.splitext(filename)[1][1:]
+                    filename_wo_ext = (
+                        os.path.splitext(filename)[0]
+                        if filename_real_ext == info_dict['ext']
+                        else filename)
                     requested_formats = info_dict['requested_formats']
                     if self.params.get('merge_output_format') is None and not compatible_formats(requested_formats):
-                        filename = os.path.splitext(filename)[0] + '.mkv'
+                        info_dict['ext'] = 'mkv'
                         self.report_warning('You have requested formats incompatible for merge. '
                                             'The formats will be merged into mkv')
+                    # Ensure filename always has a correct extension for successful merge
+                    filename = '%s.%s' % (filename_wo_ext, info_dict['ext'])
                     if os.path.exists(encodeFilename(filename)):
                         self.to_screen(
                             '[download] %s has already been downloaded and '

From 21f6330274c6f87c796bd9248ed82d2bc73de969 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 3 May 2015 00:51:24 +0800
Subject: [PATCH 1143/6219] [baiduvideo] Add new extractor (closes #4563)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/baidu.py    | 68 ++++++++++++++++++++++++++++++++
 2 files changed, 69 insertions(+)
 create mode 100644 youtube_dl/extractor/baidu.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 43f3e6808..41af925cc 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -32,6 +32,7 @@ from .atresplayer import AtresPlayerIE
 from .atttechchannel import ATTTechChannelIE
 from .audiomack import AudiomackIE, AudiomackAlbumIE
 from .azubu import AzubuIE
+from .baidu import BaiduVideoIE
 from .bambuser import BambuserIE, BambuserChannelIE
 from .bandcamp import BandcampIE, BandcampAlbumIE
 from .bbccouk import BBCCoUkIE
diff --git a/youtube_dl/extractor/baidu.py b/youtube_dl/extractor/baidu.py
new file mode 100644
index 000000000..7f7a88ffe
--- /dev/null
+++ b/youtube_dl/extractor/baidu.py
@@ -0,0 +1,68 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..compat import compat_urlparse
+
+
+class BaiduVideoIE(InfoExtractor):
+    _VALID_URL = r'http://v\.baidu\.com/(?P<type>[a-z]+)/(?P<id>\d+).htm'
+    _TESTS = [{
+        'url': 'http://v.baidu.com/comic/1069.htm?frp=bdbrand&q=%E4%B8%AD%E5%8D%8E%E5%B0%8F%E5%BD%93%E5%AE%B6',
+        'info_dict': {
+            'id': '1069',
+            'title': '中华小当家 TV版 (全52集)',
+            'description': 'md5:395a419e41215e531c857bb037bbaf80',
+        },
+        'playlist_count': 52,
+    }, {
+        'url': 'http://v.baidu.com/show/11595.htm?frp=bdbrand',
+        'info_dict': {
+            'id': '11595',
+            'title': 're:^奔跑吧兄弟',
+            'description': 'md5:1bf88bad6d850930f542d51547c089b8',
+        },
+        'playlist_mincount': 3,
+    }]
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        playlist_id = mobj.group('id')
+        category = category2 = mobj.group('type')
+        if category == 'show':
+            category2 = 'tvshow'
+
+        webpage = self._download_webpage(url, playlist_id)
+
+        playlist_title = self._html_search_regex(
+            r'title\s*:\s*(["\'])(?P<title>[^\']+)\1', webpage,
+            'playlist title', group='title')
+        playlist_description = self._html_search_regex(
+            r'<input[^>]+class="j-data-intro"[^>]+value="([^"]+)"/>', webpage,
+            playlist_id, 'playlist description')
+
+        site = self._html_search_regex(
+            r'filterSite\s*:\s*["\']([^"]*)["\']', webpage,
+            'primary provider site')
+        api_result = self._download_json(
+            'http://v.baidu.com/%s_intro/?dtype=%sPlayUrl&id=%s&site=%s' % (
+                category, category2, playlist_id, site),
+            playlist_id, 'Get playlist links')
+
+        entries = []
+        for episode in api_result[0]['episodes']:
+            episode_id = '%s_%s' % (playlist_id, episode['episode'])
+
+            redirect_page = self._download_webpage(
+                compat_urlparse.urljoin(url, episode['url']), episode_id,
+                note='Download Baidu redirect page')
+            real_url = self._html_search_regex(
+                r'location\.replace\("([^"]+)"\)', redirect_page, 'real URL')
+
+            entries.append(self.url_result(
+                real_url, video_title=episode['single_title']))
+
+        return self.playlist_result(
+            entries, playlist_id, playlist_title, playlist_description)

From e65e4c88748f2d245aa683116a2c6de907186751 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 2 May 2015 23:06:01 +0600
Subject: [PATCH 1144/6219] [utils] Improve prepend_extension

Now `ext` is appended to filename if real extension != expected extension.
---
 youtube_dl/utils.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index f07679c76..b3abfbc11 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1349,9 +1349,12 @@ def parse_duration(s):
     return res
 
 
-def prepend_extension(filename, ext):
+def prepend_extension(filename, ext, expected_real_ext=None):
     name, real_ext = os.path.splitext(filename)
-    return '{0}.{1}{2}'.format(name, ext, real_ext)
+    return (
+        '{0}.{1}{2}'.format(name, ext, real_ext)
+        if not expected_real_ext or real_ext[1:] == expected_real_ext
+        else '{0}.{1}'.format(filename, ext))
 
 
 def check_executable(exe, args=[]):

From a4bcaad77314c714d2304cb21ffdd87a9b84316b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 2 May 2015 23:10:48 +0600
Subject: [PATCH 1145/6219] [test_utils] Add tests for prepend_extension

---
 test/test_utils.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/test/test_utils.py b/test/test_utils.py
index 17017a8c0..a9464f2e4 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -41,6 +41,7 @@ from youtube_dl.utils import (
     sanitize_filename,
     sanitize_path,
     sanitize_url_path_consecutive_slashes,
+    prepend_extension,
     shell_quote,
     smuggle_url,
     str_to_int,
@@ -193,6 +194,14 @@ class TestUtil(unittest.TestCase):
             sanitize_url_path_consecutive_slashes('http://hostname/abc//'),
             'http://hostname/abc/')
 
+    def test_prepend_extension(self):
+        self.assertEqual(prepend_extension('abc.ext', 'temp'), 'abc.temp.ext')
+        self.assertEqual(prepend_extension('abc.ext', 'temp', 'ext'), 'abc.temp.ext')
+        self.assertEqual(prepend_extension('abc.unexpected_ext', 'temp', 'ext'), 'abc.unexpected_ext.temp')
+        self.assertEqual(prepend_extension('abc', 'temp'), 'abc.temp')
+        self.assertEqual(prepend_extension('.abc', 'temp'), '.abc.temp')
+        self.assertEqual(prepend_extension('.abc.ext', 'temp'), '.abc.temp.ext')
+
     def test_ordered_set(self):
         self.assertEqual(orderedSet([1, 1, 2, 3, 4, 4, 5, 6, 7, 3, 5]), [1, 2, 3, 4, 5, 6, 7])
         self.assertEqual(orderedSet([]), [])

From 666a9a2b954bb6c75a5fcdb9fbb18842038c188a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 2 May 2015 23:11:34 +0600
Subject: [PATCH 1146/6219] [YoutubeDL] Improve audio/video-only file naming

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index eaa436bf9..06d04c8f0 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1403,7 +1403,7 @@ class YoutubeDL(object):
                             new_info = dict(info_dict)
                             new_info.update(f)
                             fname = self.prepare_filename(new_info)
-                            fname = prepend_extension(fname, 'f%s' % f['format_id'])
+                            fname = prepend_extension(fname, 'f%s' % f['format_id'], new_info['ext'])
                             downloaded.append(fname)
                             partial_success = dl(fname, new_info)
                             success = success and partial_success

From b3ed15b7604ce83e85b791ed329c5725a436b805 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 2 May 2015 23:23:06 +0600
Subject: [PATCH 1147/6219] [utils] Add replace_extension

---
 test/test_utils.py  | 9 +++++++++
 youtube_dl/utils.py | 7 +++++++
 2 files changed, 16 insertions(+)

diff --git a/test/test_utils.py b/test/test_utils.py
index a9464f2e4..6906a65c2 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -42,6 +42,7 @@ from youtube_dl.utils import (
     sanitize_path,
     sanitize_url_path_consecutive_slashes,
     prepend_extension,
+    replace_extension,
     shell_quote,
     smuggle_url,
     str_to_int,
@@ -202,6 +203,14 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(prepend_extension('.abc', 'temp'), '.abc.temp')
         self.assertEqual(prepend_extension('.abc.ext', 'temp'), '.abc.temp.ext')
 
+    def test_replace_extension(self):
+        self.assertEqual(replace_extension('abc.ext', 'temp'), 'abc.temp')
+        self.assertEqual(replace_extension('abc.ext', 'temp', 'ext'), 'abc.temp')
+        self.assertEqual(replace_extension('abc.unexpected_ext', 'temp', 'ext'), 'abc.unexpected_ext.temp')
+        self.assertEqual(replace_extension('abc', 'temp'), 'abc.temp')
+        self.assertEqual(replace_extension('.abc', 'temp'), '.abc.temp')
+        self.assertEqual(replace_extension('.abc.ext', 'temp'), '.abc.temp')
+
     def test_ordered_set(self):
         self.assertEqual(orderedSet([1, 1, 2, 3, 4, 4, 5, 6, 7, 3, 5]), [1, 2, 3, 4, 5, 6, 7])
         self.assertEqual(orderedSet([]), [])
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index b3abfbc11..a5a5c317e 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1357,6 +1357,13 @@ def prepend_extension(filename, ext, expected_real_ext=None):
         else '{0}.{1}'.format(filename, ext))
 
 
+def replace_extension(filename, ext, expected_real_ext=None):
+    name, real_ext = os.path.splitext(filename)
+    return '{0}.{1}'.format(
+        name if not expected_real_ext or real_ext[1:] == expected_real_ext else filename,
+        ext)
+
+
 def check_executable(exe, args=[]):
     """ Checks if the given binary is installed somewhere in PATH, and returns its name.
     args can be a list of arguments for a short output (like -version) """

From b29e0000e69458252f73ffe62a466da5d1449863 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 2 May 2015 23:23:44 +0600
Subject: [PATCH 1148/6219] [YoutubeDL] Improve JSON info file naming

---
 youtube_dl/YoutubeDL.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 06d04c8f0..0330b0b34 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -71,6 +71,7 @@ from .utils import (
     write_string,
     YoutubeDLHandler,
     prepend_extension,
+    replace_extension,
     args_to_str,
     age_restricted,
 )
@@ -1332,7 +1333,7 @@ class YoutubeDL(object):
                     return
 
         if self.params.get('writeinfojson', False):
-            infofn = os.path.splitext(filename)[0] + '.info.json'
+            infofn = replace_extension(filename, 'info.json', info_dict.get('ext'))
             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(infofn)):
                 self.to_screen('[info] Video description metadata is already present')
             else:

From 98727e123f6d4881920bcb9fa57ea7956afb9a5a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 2 May 2015 23:35:18 +0600
Subject: [PATCH 1149/6219] [YoutubeDL] Improve annotations file naming

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 0330b0b34..27c1e726f 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1286,7 +1286,7 @@ class YoutubeDL(object):
                     return
 
         if self.params.get('writeannotations', False):
-            annofn = filename + '.annotations.xml'
+            annofn = replace_extension(filename, 'annotations.xml', info_dict.get('ext'))
             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(annofn)):
                 self.to_screen('[info] Video annotations are already present')
             else:

From 2699da8041cf10a636e0ac99efb91854b2c3aa58 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 2 May 2015 23:36:55 +0600
Subject: [PATCH 1150/6219] [YoutubeDL] Improve description file naming

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 27c1e726f..eb7470f72 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1271,7 +1271,7 @@ class YoutubeDL(object):
             return
 
         if self.params.get('writedescription', False):
-            descfn = filename + '.description'
+            descfn = replace_extension(filename, 'description', info_dict.get('ext'))
             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(descfn)):
                 self.to_screen('[info] Video description is already present')
             elif info_dict.get('description') is None:

From 0669c89c55a77eeda2b9767e096c465b12c67b8c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 2 May 2015 23:38:30 +0600
Subject: [PATCH 1151/6219] [options] Clarify `--write-annotations` help

---
 youtube_dl/options.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index d0aa8296d..22dbc3aec 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -637,7 +637,7 @@ def parseOpts(overrideArguments=None):
     filesystem.add_option(
         '--write-annotations',
         action='store_true', dest='writeannotations', default=False,
-        help='Write video annotations to a .annotation file')
+        help='Write video annotations to a .annotations.xml file')
     filesystem.add_option(
         '--load-info',
         dest='load_info_filename', metavar='FILE',

From 0a64aa73552d3fc3ae6e24a56e2f8e78c960603c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 3 May 2015 00:58:42 +0600
Subject: [PATCH 1152/6219] [vgtv] Fix _VALID_URL (Closes #5578)

---
 youtube_dl/extractor/vgtv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vgtv.py b/youtube_dl/extractor/vgtv.py
index 2f111bf7e..69dc9a759 100644
--- a/youtube_dl/extractor/vgtv.py
+++ b/youtube_dl/extractor/vgtv.py
@@ -8,7 +8,7 @@ from ..utils import float_or_none
 
 
 class VGTVIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?vgtv\.no/#!/(?:.*)/(?P<id>[0-9]+)'
+    _VALID_URL = r'http://(?:www\.)?vgtv\.no/#!/[^/]+/(?P<id>[0-9]+)'
     _TESTS = [
         {
             # streamType: vod

From 963aea5279691affca34d220452cecfd0e291a77 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 3 May 2015 07:45:15 +0600
Subject: [PATCH 1153/6219] [baiduvideo] Improve _VALID_URL

---
 youtube_dl/extractor/baidu.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/baidu.py b/youtube_dl/extractor/baidu.py
index 7f7a88ffe..906895c1e 100644
--- a/youtube_dl/extractor/baidu.py
+++ b/youtube_dl/extractor/baidu.py
@@ -8,7 +8,7 @@ from ..compat import compat_urlparse
 
 
 class BaiduVideoIE(InfoExtractor):
-    _VALID_URL = r'http://v\.baidu\.com/(?P<type>[a-z]+)/(?P<id>\d+).htm'
+    _VALID_URL = r'http://v\.baidu\.com/(?P<type>[a-z]+)/(?P<id>\d+)\.htm'
     _TESTS = [{
         'url': 'http://v.baidu.com/comic/1069.htm?frp=bdbrand&q=%E4%B8%AD%E5%8D%8E%E5%B0%8F%E5%BD%93%E5%AE%B6',
         'info_dict': {

From 8dab1e9072037daa9c6cab3da4a5dbd4daaae4c8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 3 May 2015 09:56:03 +0600
Subject: [PATCH 1154/6219] [rutv] Recognize live streams (#5584)

---
 youtube_dl/extractor/rutv.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/rutv.py b/youtube_dl/extractor/rutv.py
index ef766237b..169f7c032 100644
--- a/youtube_dl/extractor/rutv.py
+++ b/youtube_dl/extractor/rutv.py
@@ -181,12 +181,15 @@ class RUTVIE(InfoExtractor):
 
         self._sort_formats(formats)
 
+        is_live = video_type == 'live'
+
         return {
             'id': video_id,
-            'title': title,
+            'title': self._live_title(title) if is_live else title,
             'description': description,
             'thumbnail': thumbnail,
             'view_count': view_count,
             'duration': duration,
             'formats': formats,
+            'is_live': is_live,
         }

From d0fd305023b37f7776485679a74e422eade26c13 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 3 May 2015 10:00:34 +0600
Subject: [PATCH 1155/6219] [rutv] Add test for #5584

---
 youtube_dl/extractor/rutv.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/rutv.py b/youtube_dl/extractor/rutv.py
index 169f7c032..1ec2c86e5 100644
--- a/youtube_dl/extractor/rutv.py
+++ b/youtube_dl/extractor/rutv.py
@@ -84,11 +84,20 @@ class RUTVIE(InfoExtractor):
                 'title': 'Сочи-2014. Биатлон. Индивидуальная гонка. Мужчины ',
                 'description': 'md5:9e0ed5c9d2fa1efbfdfed90c9a6d179c',
             },
+            'skip': 'Translation has finished',
+        },
+        {
+            'url': 'http://live.russia.tv/index/index/channel_id/3',
+            'info_dict': {
+                'id': '21',
+                'ext': 'mp4',
+                'title': 're:^Россия 24. Прямой эфир [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
+                'is_live': True,
+            },
             'params': {
-                # rtmp download
+                # m3u8 download
                 'skip_download': True,
             },
-            'skip': 'Translation has finished',
         },
     ]
 

From 233c1c0e76d64c9e13dc8968bfd8a014c49e66a8 Mon Sep 17 00:00:00 2001
From: Antti Ajanki <antti.ajanki@iki.fi>
Date: Sun, 3 May 2015 11:04:14 +0300
Subject: [PATCH 1156/6219] [downloader/f4m] Fragment filenames must be
 sanitized

because the fragment was written to a file with a sanitized name by
http_dl.download()
---
 youtube_dl/downloader/f4m.py | 23 ++++++++++++-----------
 1 file changed, 12 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index b1a858c45..3cb07e15f 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -396,18 +396,19 @@ class F4mFD(FileDownloader):
                 success = http_dl.download(frag_filename, {'url': url})
                 if not success:
                     return False
-                with open(frag_filename, 'rb') as down:
-                    down_data = down.read()
-                    reader = FlvReader(down_data)
-                    while True:
-                        _, box_type, box_data = reader.read_box_info()
-                        if box_type == b'mdat':
-                            dest_stream.write(box_data)
-                            break
+                (down, frag_sanitized) = sanitize_open(frag_filename, 'rb')
+                down_data = down.read()
+                down.close()
+                reader = FlvReader(down_data)
+                while True:
+                    _, box_type, box_data = reader.read_box_info()
+                    if box_type == b'mdat':
+                        dest_stream.write(box_data)
+                        break
                 if live:
-                    os.remove(frag_filename)
+                    os.remove(encodeFilename(frag_sanitized))
                 else:
-                    frags_filenames.append(frag_filename)
+                    frags_filenames.append(frag_sanitized)
             except (compat_urllib_error.HTTPError, ) as err:
                 if live and (err.code == 404 or err.code == 410):
                     # We didn't keep up with the live window. Continue
@@ -430,7 +431,7 @@ class F4mFD(FileDownloader):
         elapsed = time.time() - start
         self.try_rename(tmpfilename, filename)
         for frag_file in frags_filenames:
-            os.remove(frag_file)
+            os.remove(encodeFilename(frag_file))
 
         fsize = os.path.getsize(encodeFilename(filename))
         self._hook_progress({

From 5477ca82395545e577afb269b910336cd98de5b5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 3 May 2015 16:59:14 +0200
Subject: [PATCH 1157/6219] [dailymotion] Use https urls

The video url still redirects to an http url, but it doesn't explicitly contain the video id.
---
 youtube_dl/extractor/dailymotion.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index 7615ecd4b..aa595af20 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -85,7 +85,7 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        url = 'http://www.dailymotion.com/video/%s' % video_id
+        url = 'https://www.dailymotion.com/video/%s' % video_id
 
         # Retrieve video webpage to extract further information
         request = self._build_request(url)
@@ -110,7 +110,7 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
         if mobj is not None:
             video_upload_date = mobj.group(3) + mobj.group(2) + mobj.group(1)
 
-        embed_url = 'http://www.dailymotion.com/embed/video/%s' % video_id
+        embed_url = 'https://www.dailymotion.com/embed/video/%s' % video_id
         embed_request = self._build_request(embed_url)
         embed_page = self._download_webpage(
             embed_request, video_id, 'Downloading embed page')

From 1748d67aea66b6d63bc203ada1036110e7fd2402 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 4 May 2015 01:11:23 +0800
Subject: [PATCH 1158/6219] [lifenews] Fix view count and comment count

---
 youtube_dl/extractor/lifenews.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/lifenews.py b/youtube_dl/extractor/lifenews.py
index 1dfe7f77f..3c00ed333 100644
--- a/youtube_dl/extractor/lifenews.py
+++ b/youtube_dl/extractor/lifenews.py
@@ -47,9 +47,9 @@ class LifeNewsIE(InfoExtractor):
         description = self._og_search_description(webpage)
 
         view_count = self._html_search_regex(
-            r'<div class=\'views\'>(\d+)</div>', webpage, 'view count', fatal=False)
+            r'<div class=\'views\'>\s*(\d+)\s*</div>', webpage, 'view count', fatal=False)
         comment_count = self._html_search_regex(
-            r'<div class=\'comments\'>\s*<span class=\'counter\'>(\d+)</span>', webpage, 'comment count', fatal=False)
+            r'<div class=\'comments\'>\s*<span class=\'counter\'>\s*(\d+)\s*</span>', webpage, 'comment count', fatal=False)
 
         upload_date = self._html_search_regex(
             r'<time datetime=\'([^\']+)\'>', webpage, 'upload date', fatal=False)

From 848edeab898ee45e45a8fcdb355f35be8b73dd5d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 4 May 2015 01:24:19 +0800
Subject: [PATCH 1159/6219] [lifenews] Detect <iframe> (fixes #5346)

---
 youtube_dl/extractor/lifenews.py | 47 ++++++++++++++++++++++++++------
 1 file changed, 38 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/lifenews.py b/youtube_dl/extractor/lifenews.py
index 3c00ed333..330138692 100644
--- a/youtube_dl/extractor/lifenews.py
+++ b/youtube_dl/extractor/lifenews.py
@@ -16,7 +16,7 @@ class LifeNewsIE(InfoExtractor):
     IE_DESC = 'LIFE | NEWS'
     _VALID_URL = r'http://lifenews\.ru/(?:mobile/)?news/(?P<id>\d+)'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://lifenews.ru/news/126342',
         'md5': 'e1b50a5c5fb98a6a544250f2e0db570a',
         'info_dict': {
@@ -27,7 +27,19 @@ class LifeNewsIE(InfoExtractor):
             'thumbnail': 're:http://.*\.jpg',
             'upload_date': '20140130',
         }
-    }
+    }, {
+        # video in <iframe>
+        'url': 'http://lifenews.ru/news/152125',
+        'md5': '77d19a6f0886cd76bdbf44b4d971a273',
+        'info_dict': {
+            'id': '152125',
+            'ext': 'mp4',
+            'title': 'В Сети появилось видео захвата «Правым сектором» колхозных полей ',
+            'description': 'Жители двух поселков Днепропетровской области не простили радикалам угрозу лишения плодородных земель и пошли в лобовую. ',
+            'upload_date': '20150402',
+            'uploader': 'embed.life.ru',
+        }
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
@@ -36,7 +48,9 @@ class LifeNewsIE(InfoExtractor):
         webpage = self._download_webpage('http://lifenews.ru/news/%s' % video_id, video_id, 'Downloading page')
 
         videos = re.findall(r'<video.*?poster="(?P<poster>[^"]+)".*?src="(?P<video>[^"]+)".*?></video>', webpage)
-        if not videos:
+        iframe_link = self._html_search_regex(
+            '<iframe[^>]+src="([^"]+)', webpage, 'iframe link', default=None)
+        if not videos and not iframe_link:
             raise ExtractorError('No media links available for %s' % video_id)
 
         title = self._og_search_title(webpage)
@@ -56,17 +70,32 @@ class LifeNewsIE(InfoExtractor):
         if upload_date is not None:
             upload_date = unified_strdate(upload_date)
 
+        common_info = {
+            'description': description,
+            'view_count': int_or_none(view_count),
+            'comment_count': int_or_none(comment_count),
+            'upload_date': upload_date,
+        }
+
         def make_entry(video_id, media, video_number=None):
-            return {
+            cur_info = dict(common_info)
+            cur_info.update({
                 'id': video_id,
                 'url': media[1],
                 'thumbnail': media[0],
                 'title': title if video_number is None else '%s-video%s' % (title, video_number),
-                'description': description,
-                'view_count': int_or_none(view_count),
-                'comment_count': int_or_none(comment_count),
-                'upload_date': upload_date,
-            }
+            })
+            return cur_info
+
+        if iframe_link:
+            cur_info = dict(common_info)
+            cur_info.update({
+                '_type': 'url_transparent',
+                'id': video_id,
+                'title': title,
+                'url': iframe_link,
+            })
+            return cur_info
 
         if len(videos) == 1:
             return make_entry(video_id, videos[0])

From 3e7202c1bca0618fef04c60a0f5603d50f09b76f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 4 May 2015 01:59:26 +0800
Subject: [PATCH 1160/6219] [MLB] Extend _VALID_URL (#5443)

---
 youtube_dl/extractor/mlb.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mlb.py b/youtube_dl/extractor/mlb.py
index e369551c2..ee9ff73bf 100644
--- a/youtube_dl/extractor/mlb.py
+++ b/youtube_dl/extractor/mlb.py
@@ -10,7 +10,7 @@ from ..utils import (
 
 
 class MLBIE(InfoExtractor):
-    _VALID_URL = r'https?://m(?:lb)?\.(?:[\da-z_-]+\.)?mlb\.com/(?:(?:.*?/)?video/(?:topic/[\da-z_-]+/)?v|(?:shared/video/embed/embed\.html|[^/]+/video/play\.jsp)\?.*?\bcontent_id=)(?P<id>n?\d+)'
+    _VALID_URL = r'https?://m(?:lb)?\.(?:[\da-z_-]+\.)?mlb\.com/(?:(?:.*?/)?video/(?:topic/[\da-z_-]+/)?v|(?:shared/video/embed/(?:embed|m-internal-embed)\.html|[^/]+/video/play\.jsp)\?.*?\bcontent_id=)(?P<id>n?\d+)'
     _TESTS = [
         {
             'url': 'http://m.mlb.com/sea/video/topic/51231442/v34698933/nymsea-ackley-robs-a-home-run-with-an-amazing-catch/?c_id=sea',
@@ -83,6 +83,11 @@ class MLBIE(InfoExtractor):
         {
             'url': 'http://m.cardinals.mlb.com/stl/video/v51175783/atlstl-piscotty-makes-great-sliding-catch-on-line/?partnerId=as_mlb_20150321_42500876&adbid=579409712979910656&adbpl=tw&adbpr=52847728',
             'only_matching': True,
+        },
+        {
+            # From http://m.mlb.com/news/article/118550098/blue-jays-kevin-pillar-goes-spidey-up-the-wall-to-rob-tim-beckham-of-a-homer
+            'url': 'http://mlb.mlb.com/shared/video/embed/m-internal-embed.html?content_id=75609783&property=mlb&autoplay=true&hashmode=false&siteSection=mlb/multimedia/article_118550098/article_embed&club=mlb',
+            'only_matching': True,
         }
     ]
 

From 8001607e903a5ccaff76908f23bece08818743c0 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 4 May 2015 02:20:07 +0800
Subject: [PATCH 1161/6219] [generic] Detect more MLB videos (fixes #5443)

---
 youtube_dl/extractor/generic.py | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 4946cc132..d09e85665 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -413,6 +413,19 @@ class GenericIE(InfoExtractor):
                 'thumbnail': 're:^https?://.*\.jpg$',
             },
         },
+        # MLB articles
+        {
+            'url': 'http://m.mlb.com/news/article/118550098/blue-jays-kevin-pillar-goes-spidey-up-the-wall-to-rob-tim-beckham-of-a-homer',
+            'md5': 'b190e70141fb9a1552a85426b4da1b5d',
+            'info_dict': {
+                'id': '75609783',
+                'ext': 'mp4',
+                'title': 'Must C: Pillar climbs for catch',
+                'description': '4/15/15: Blue Jays outfielder Kevin Pillar continues his defensive dominance by climbing the wall in left to rob Tim Beckham of a home run',
+                'timestamp': 1429124820,
+                'upload_date': '20150415',
+            }
+        },
         # Wistia embed
         {
             'url': 'http://education-portal.com/academy/lesson/north-american-exploration-failed-colonies-of-spain-france-england.html#lesson',
@@ -1289,6 +1302,10 @@ class GenericIE(InfoExtractor):
         mobj = re.search(
             r'<iframe[^>]+?src=(["\'])(?P<url>https?://m(?:lb)?\.mlb\.com/shared/video/embed/embed\.html\?.+?)\1',
             webpage)
+        if not mobj:
+            mobj = re.search(
+                r'data-video-link=["\'](?P<url>http://m.mlb.com/video/[^"\']+)',
+                webpage)
         if mobj is not None:
             return self.url_result(mobj.group('url'), 'MLB')
 

From 957b794c2622fdc3696c245f1fa1eebfdb0f8bdf Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 3 May 2015 22:31:39 +0200
Subject: [PATCH 1162/6219] release 2015.05.03

---
 README.md              | 2 +-
 docs/supportedsites.md | 3 ++-
 youtube_dl/version.py  | 2 +-
 3 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index 3432546fc..9aeb114f3 100644
--- a/README.md
+++ b/README.md
@@ -133,7 +133,7 @@ which means you can modify it, redistribute it or use it however you like.
     --no-mtime                       Do not use the Last-modified header to set the file modification time
     --write-description              Write video description to a .description file
     --write-info-json                Write video metadata to a .info.json file
-    --write-annotations              Write video annotations to a .annotation file
+    --write-annotations              Write video annotations to a .annotations.xml file
     --load-info FILE                 JSON file containing the video information (created with the "--write-info-json" option)
     --cookies FILE                   File to read cookies from and dump cookie jar in
     --cache-dir DIR                  Location in the filesystem where youtube-dl can store some downloaded information permanently. By default $XDG_CACHE_HOME/youtube-dl
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index d1d676cdd..6d2e496a8 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -44,6 +44,7 @@
  - **audiomack**
  - **audiomack:album**
  - **Azubu**
+ - **BaiduVideo**
  - **bambuser**
  - **bambuser:channel**
  - **Bandcamp**
@@ -155,6 +156,7 @@
  - **FootyRoom**
  - **Foxgay**
  - **FoxNews**
+ - **FoxSports**
  - **france2.fr:generation-quoi**
  - **FranceCulture**
  - **FranceInter**
@@ -184,7 +186,6 @@
  - **Golem**
  - **GorillaVid**: GorillaVid.in, daclips.in, movpod.in, fastvideo.in and realvid.net
  - **Goshgay**
- - **Grooveshark**
  - **Groupon**
  - **Hark**
  - **HearThisAt**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index b88ea85e8..6950afc47 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.04.28'
+__version__ = '2015.05.03'

From b9b3ab45ea95e4c48f3c87d3584f8180ff460be6 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 4 May 2015 19:09:18 +0800
Subject: [PATCH 1163/6219] [NBC] Enhance extraction of ThePlatform URL (fixes
 #5470)

---
 youtube_dl/extractor/nbc.py | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index ecd0ac8b1..6cbe03d0f 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -37,13 +37,26 @@ class NBCIE(InfoExtractor):
             },
             'skip': 'Only works from US',
         },
+        {
+            'url': 'http://www.nbc.com/saturday-night-live/video/star-wars-teaser/2832821',
+            'info_dict': {
+                'id': '8iUuyzWDdYUZ',
+                'ext': 'flv',
+                'title': 'Star Wars Teaser',
+                'description': 'md5:0b40f9cbde5b671a7ff62fceccc4f442',
+            },
+            'skip': 'Only works from US',
+        }
     ]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
         theplatform_url = self._search_regex(
-            '(?:class="video-player video-player-full" data-mpx-url|class="player" src)="(.*?)"',
+            [
+                r'(?:class="video-player video-player-full" data-mpx-url|class="player" src)="(.*?)"',
+                r'"embedURL"\s*:\s*"([^"]+)"'
+            ],
             webpage, 'theplatform url').replace('_no_endcard', '')
         if theplatform_url.startswith('//'):
             theplatform_url = 'http:' + theplatform_url

From 71fa56b887b7c4d1a3ba16c7127736fcaec34b4d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 4 May 2015 18:59:22 +0600
Subject: [PATCH 1164/6219] [escapist] Fix formats extraction

---
 youtube_dl/extractor/escapist.py | 35 ++++++++++++++------------------
 1 file changed, 15 insertions(+), 20 deletions(-)

diff --git a/youtube_dl/extractor/escapist.py b/youtube_dl/extractor/escapist.py
index 8facf1185..802943dc2 100644
--- a/youtube_dl/extractor/escapist.py
+++ b/youtube_dl/extractor/escapist.py
@@ -8,7 +8,7 @@ from ..compat import compat_urllib_request
 from ..utils import (
     determine_ext,
     clean_html,
-    qualities,
+    int_or_none,
 )
 
 
@@ -72,28 +72,23 @@ class EscapistIE(InfoExtractor):
         video_id = imsVideo['videoID']
         key = imsVideo['hash']
 
-        quality = qualities(['lq', 'hq', 'hd'])
+        config_req = compat_urllib_request.Request(
+            'http://www.escapistmagazine.com/videos/'
+            'vidconfig.php?videoID=%s&hash=%s' % (video_id, key))
+        config_req.add_header('Referer', url)
+        config = self._download_webpage(config_req, video_id, 'Downloading video config')
 
-        formats = []
-        for q in ['lq', 'hq', 'hd']:
-            config_req = compat_urllib_request.Request(
-                'http://www.escapistmagazine.com/videos/'
-                'vidconfig.php?videoID=%s&hash=%s&quality=%s' % (video_id, key, 'mp4_' + q))
-            config_req.add_header('Referer', url)
-            config = self._download_webpage(config_req, video_id, 'Downloading video config ' + q.upper())
+        data = json.loads(_decrypt_config(key, config))
 
-            data = json.loads(_decrypt_config(key, config))
+        title = clean_html(data['videoData']['title'])
+        duration = data['videoData']['duration'] / 1000
 
-            title = clean_html(data['videoData']['title'])
-            duration = data['videoData']['duration'] / 1000
-
-            for i, v in enumerate(data['files']['videos']):
-
-                formats.append({
-                    'url': v,
-                    'format_id': determine_ext(v) + '_' + q + str(i),
-                    'quality': quality(q),
-                })
+        formats = [{
+            'url': video['src'],
+            'format_id': '%s-%sp' % (determine_ext(video['src']), video['res']),
+            'height': int_or_none(video.get('res')),
+        } for video in data['files']['videos']]
+        self._sort_formats(formats)
 
         return {
             'id': video_id,

From cec04ef3a6cfd43036f7463dc1fa890af5b6ee11 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 4 May 2015 19:00:34 +0600
Subject: [PATCH 1165/6219] [escapist] Update tests' checksums

---
 youtube_dl/extractor/escapist.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/escapist.py b/youtube_dl/extractor/escapist.py
index 802943dc2..daf859385 100644
--- a/youtube_dl/extractor/escapist.py
+++ b/youtube_dl/extractor/escapist.py
@@ -39,7 +39,7 @@ class EscapistIE(InfoExtractor):
     _VALID_URL = r'https?://?(www\.)?escapistmagazine\.com/videos/view/[^/?#]+/(?P<id>[0-9]+)-[^/?#]*(?:$|[?#])'
     _TESTS = [{
         'url': 'http://www.escapistmagazine.com/videos/view/the-escapist-presents/6618-Breaking-Down-Baldurs-Gate',
-        'md5': 'c6793dbda81388f4264c1ba18684a74d',
+        'md5': 'ab3a706c681efca53f0a35f1415cf0d1',
         'info_dict': {
             'id': '6618',
             'ext': 'mp4',
@@ -50,7 +50,7 @@ class EscapistIE(InfoExtractor):
         }
     }, {
         'url': 'http://www.escapistmagazine.com/videos/view/zero-punctuation/10044-Evolve-One-vs-Multiplayer',
-        'md5': 'cf8842a8a46444d241f9a9980d7874f2',
+        'md5': '9e8c437b0dbb0387d3bd3255ca77f6bf',
         'info_dict': {
             'id': '10044',
             'ext': 'mp4',

From 90b4b0eabeabbdad3bfece8254692d34a3dcba95 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 4 May 2015 19:01:08 +0600
Subject: [PATCH 1166/6219] [escapist] Improve _VALID_URL

---
 youtube_dl/extractor/escapist.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/escapist.py b/youtube_dl/extractor/escapist.py
index daf859385..dadfaa6a5 100644
--- a/youtube_dl/extractor/escapist.py
+++ b/youtube_dl/extractor/escapist.py
@@ -36,7 +36,7 @@ def _decrypt_config(key, string):
 
 
 class EscapistIE(InfoExtractor):
-    _VALID_URL = r'https?://?(www\.)?escapistmagazine\.com/videos/view/[^/?#]+/(?P<id>[0-9]+)-[^/?#]*(?:$|[?#])'
+    _VALID_URL = r'https?://?(?:www\.)?escapistmagazine\.com/videos/view/[^/?#]+/(?P<id>[0-9]+)-[^/?#]*(?:$|[?#])'
     _TESTS = [{
         'url': 'http://www.escapistmagazine.com/videos/view/the-escapist-presents/6618-Breaking-Down-Baldurs-Gate',
         'md5': 'ab3a706c681efca53f0a35f1415cf0d1',

From 782e0568ef28ebd7d7a11273d0a5af065575379a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 4 May 2015 19:04:49 +0600
Subject: [PATCH 1167/6219] [escapist] Modernize

---
 youtube_dl/extractor/escapist.py | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/escapist.py b/youtube_dl/extractor/escapist.py
index dadfaa6a5..2cd3af142 100644
--- a/youtube_dl/extractor/escapist.py
+++ b/youtube_dl/extractor/escapist.py
@@ -9,6 +9,7 @@ from ..utils import (
     determine_ext,
     clean_html,
     int_or_none,
+    float_or_none,
 )
 
 
@@ -65,12 +66,12 @@ class EscapistIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        imsVideo = self._parse_json(
+        ims_video = self._parse_json(
             self._search_regex(
                 r'imsVideo\.play\(({.+?})\);', webpage, 'imsVideo'),
             video_id)
-        video_id = imsVideo['videoID']
-        key = imsVideo['hash']
+        video_id = ims_video['videoID']
+        key = ims_video['hash']
 
         config_req = compat_urllib_request.Request(
             'http://www.escapistmagazine.com/videos/'
@@ -80,8 +81,11 @@ class EscapistIE(InfoExtractor):
 
         data = json.loads(_decrypt_config(key, config))
 
-        title = clean_html(data['videoData']['title'])
-        duration = data['videoData']['duration'] / 1000
+        video_data = data['videoData']
+
+        title = clean_html(video_data['title'])
+        duration = float_or_none(video_data.get('duration'), 1000)
+        uploader = video_data.get('publisher')
 
         formats = [{
             'url': video['src'],
@@ -97,4 +101,5 @@ class EscapistIE(InfoExtractor):
             'thumbnail': self._og_search_thumbnail(webpage),
             'description': self._og_search_description(webpage),
             'duration': duration,
+            'uploader': uploader,
         }

From b2f82360d78696daca576f558f585b2285db0341 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 4 May 2015 19:06:07 +0600
Subject: [PATCH 1168/6219] [escapist] Add uploader to tests

---
 youtube_dl/extractor/escapist.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/escapist.py b/youtube_dl/extractor/escapist.py
index 2cd3af142..c85b4c458 100644
--- a/youtube_dl/extractor/escapist.py
+++ b/youtube_dl/extractor/escapist.py
@@ -48,6 +48,7 @@ class EscapistIE(InfoExtractor):
             'title': "Breaking Down Baldur's Gate",
             'thumbnail': 're:^https?://.*\.jpg$',
             'duration': 264,
+            'uploader': 'The Escapist',
         }
     }, {
         'url': 'http://www.escapistmagazine.com/videos/view/zero-punctuation/10044-Evolve-One-vs-Multiplayer',
@@ -59,6 +60,7 @@ class EscapistIE(InfoExtractor):
             'title': 'Evolve - One vs Multiplayer',
             'thumbnail': 're:^https?://.*\.jpg$',
             'duration': 304,
+            'uploader': 'The Escapist',
         }
     }]
 

From dc1eed93be7ac2afb3f52237ae0034d24715b4bd Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 4 May 2015 15:12:48 +0200
Subject: [PATCH 1169/6219] release 2015.05.04

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 6950afc47..a5a81bcd2 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.05.03'
+__version__ = '2015.05.04'

From 0fe2ff78e68ec03d56bf3d9434eb612ffb683977 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 4 May 2015 21:53:05 +0800
Subject: [PATCH 1170/6219] [NBC] Enhance embedURL extraction (closes #2549)

---
 test/test_utils.py          |  5 +++++
 youtube_dl/extractor/nbc.py | 11 +++++++++--
 youtube_dl/utils.py         |  8 ++++++++
 3 files changed, 22 insertions(+), 2 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 6906a65c2..032d3656a 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -53,6 +53,7 @@ from youtube_dl.utils import (
     unified_strdate,
     unsmuggle_url,
     uppercase_escape,
+    lowercase_escape,
     url_basename,
     urlencode_postdata,
     version_tuple,
@@ -418,6 +419,10 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(uppercase_escape('aä'), 'aä')
         self.assertEqual(uppercase_escape('\\U0001d550'), '𝕐')
 
+    def test_lowercase_escape(self):
+        self.assertEqual(lowercase_escape('aä'), 'aä')
+        self.assertEqual(lowercase_escape('\\u0026'), '&')
+
     def test_limit_length(self):
         self.assertEqual(limit_length(None, 12), None)
         self.assertEqual(limit_length('foo', 12), 'foo')
diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index 6cbe03d0f..dc2091be0 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -10,6 +10,8 @@ from ..compat import (
 from ..utils import (
     ExtractorError,
     find_xpath_attr,
+    lowercase_escape,
+    unescapeHTML,
 )
 
 
@@ -46,18 +48,23 @@ class NBCIE(InfoExtractor):
                 'description': 'md5:0b40f9cbde5b671a7ff62fceccc4f442',
             },
             'skip': 'Only works from US',
+        },
+        {
+            # This video has expired but with an escaped embedURL
+            'url': 'http://www.nbc.com/parenthood/episode-guide/season-5/just-like-at-home/515',
+            'skip': 'Expired'
         }
     ]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
-        theplatform_url = self._search_regex(
+        theplatform_url = unescapeHTML(lowercase_escape(self._html_search_regex(
             [
                 r'(?:class="video-player video-player-full" data-mpx-url|class="player" src)="(.*?)"',
                 r'"embedURL"\s*:\s*"([^"]+)"'
             ],
-            webpage, 'theplatform url').replace('_no_endcard', '')
+            webpage, 'theplatform url').replace('_no_endcard', '').replace('\\/', '/')))
         if theplatform_url.startswith('//'):
             theplatform_url = 'http:' + theplatform_url
         return self.url_result(theplatform_url)
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index a5a5c317e..1013f7c18 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1486,6 +1486,14 @@ def uppercase_escape(s):
         s)
 
 
+def lowercase_escape(s):
+    unicode_escape = codecs.getdecoder('unicode_escape')
+    return re.sub(
+        r'\\u[0-9a-fA-F]{4}',
+        lambda m: unicode_escape(m.group(0))[0],
+        s)
+
+
 def escape_rfc3986(s):
     """Escape non-ASCII characters as suggested by RFC 3986"""
     if sys.version_info < (3, 0) and isinstance(s, compat_str):

From 883340c10758abf76a250172deb12b528b8da7b8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 4 May 2015 16:50:22 +0200
Subject: [PATCH 1171/6219] [livestream:original] Fix extraction (fixes #4702)

---
 youtube_dl/extractor/livestream.py | 16 +++++-----------
 1 file changed, 5 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/livestream.py b/youtube_dl/extractor/livestream.py
index ec309dadd..5be59a92f 100644
--- a/youtube_dl/extractor/livestream.py
+++ b/youtube_dl/extractor/livestream.py
@@ -202,13 +202,9 @@ class LivestreamOriginalIE(InfoExtractor):
         'url': 'http://www.livestream.com/dealbook/video?clipId=pla_8aa4a3f1-ba15-46a4-893b-902210e138fb',
         'info_dict': {
             'id': 'pla_8aa4a3f1-ba15-46a4-893b-902210e138fb',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Spark 1 (BitCoin) with Cameron Winklevoss & Tyler Winklevoss of Winklevoss Capital',
         },
-        'params': {
-            # rtmp
-            'skip_download': True,
-        },
     }, {
         'url': 'https://www.livestream.com/newplay/folder?dirId=a07bf706-d0e4-4e75-a747-b021d84f2fd3',
         'info_dict': {
@@ -221,19 +217,17 @@ class LivestreamOriginalIE(InfoExtractor):
         api_url = 'http://x{0}x.api.channel.livestream.com/2.0/clipdetails?extendedInfo=true&id={1}'.format(user, video_id)
 
         info = self._download_xml(api_url, video_id)
+        # this url is used on mobile devices
+        stream_url = 'http://x{0}x.api.channel.livestream.com/3.0/getstream.json?id={1}'.format(user, video_id)
+        stream_info = self._download_json(stream_url, video_id)
         item = info.find('channel').find('item')
         ns = {'media': 'http://search.yahoo.com/mrss'}
         thumbnail_url = item.find(xpath_with_ns('media:thumbnail', ns)).attrib['url']
-        # Remove the extension and number from the path (like 1.jpg)
-        path = self._search_regex(r'(user-files/.+)_.*?\.jpg$', thumbnail_url, 'path')
 
         return {
             'id': video_id,
             'title': item.find('title').text,
-            'url': 'rtmp://extondemand.livestream.com/ondemand',
-            'play_path': 'trans/dv15/mogulus-{0}'.format(path),
-            'player_url': 'http://static.livestream.com/chromelessPlayer/v21/playerapi.swf?hash=5uetk&v=0803&classid=D27CDB6E-AE6D-11cf-96B8-444553540000&jsEnabled=false&wmode=opaque',
-            'ext': 'flv',
+            'url': stream_info['progressiveUrl'],
             'thumbnail': thumbnail_url,
         }
 

From a90552663e555c89638706141c31b0022a9ba27b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 4 May 2015 16:54:01 +0200
Subject: [PATCH 1172/6219] [livestream:original] Update url format (fixes
 #5598)

---
 youtube_dl/extractor/livestream.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/livestream.py b/youtube_dl/extractor/livestream.py
index 5be59a92f..6d7733e41 100644
--- a/youtube_dl/extractor/livestream.py
+++ b/youtube_dl/extractor/livestream.py
@@ -194,19 +194,19 @@ class LivestreamIE(InfoExtractor):
 # The original version of Livestream uses a different system
 class LivestreamOriginalIE(InfoExtractor):
     IE_NAME = 'livestream:original'
-    _VALID_URL = r'''(?x)https?://www\.livestream\.com/
+    _VALID_URL = r'''(?x)https?://original\.livestream\.com/
         (?P<user>[^/]+)/(?P<type>video|folder)
         (?:\?.*?Id=|/)(?P<id>.*?)(&|$)
         '''
     _TESTS = [{
-        'url': 'http://www.livestream.com/dealbook/video?clipId=pla_8aa4a3f1-ba15-46a4-893b-902210e138fb',
+        'url': 'http://original.livestream.com/dealbook/video?clipId=pla_8aa4a3f1-ba15-46a4-893b-902210e138fb',
         'info_dict': {
             'id': 'pla_8aa4a3f1-ba15-46a4-893b-902210e138fb',
             'ext': 'mp4',
             'title': 'Spark 1 (BitCoin) with Cameron Winklevoss & Tyler Winklevoss of Winklevoss Capital',
         },
     }, {
-        'url': 'https://www.livestream.com/newplay/folder?dirId=a07bf706-d0e4-4e75-a747-b021d84f2fd3',
+        'url': 'https://original.livestream.com/newplay/folder?dirId=a07bf706-d0e4-4e75-a747-b021d84f2fd3',
         'info_dict': {
             'id': 'a07bf706-d0e4-4e75-a747-b021d84f2fd3',
         },

From 50aa43b3ae074781bb7c7ad55a85f54b1fd84146 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 4 May 2015 22:32:57 +0800
Subject: [PATCH 1173/6219] [nytimes] Implement extracting videos from articles
 (closes  #5436)

---
 youtube_dl/extractor/nytimes.py | 79 +++++++++++++++++++++++----------
 1 file changed, 55 insertions(+), 24 deletions(-)

diff --git a/youtube_dl/extractor/nytimes.py b/youtube_dl/extractor/nytimes.py
index 03f0a4de6..266d8d400 100644
--- a/youtube_dl/extractor/nytimes.py
+++ b/youtube_dl/extractor/nytimes.py
@@ -8,30 +8,8 @@ from ..utils import (
 )
 
 
-class NYTimesIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:(?:www\.)?nytimes\.com/video/(?:[^/]+/)+?|graphics8\.nytimes\.com/bcvideo/\d+(?:\.\d+)?/iframe/embed\.html\?videoId=)(?P<id>\d+)'
-
-    _TESTS = [{
-        'url': 'http://www.nytimes.com/video/opinion/100000002847155/verbatim-what-is-a-photocopier.html?playlistId=100000001150263',
-        'md5': '18a525a510f942ada2720db5f31644c0',
-        'info_dict': {
-            'id': '100000002847155',
-            'ext': 'mov',
-            'title': 'Verbatim: What Is a Photocopier?',
-            'description': 'md5:93603dada88ddbda9395632fdc5da260',
-            'timestamp': 1398631707,
-            'upload_date': '20140427',
-            'uploader': 'Brett Weiner',
-            'duration': 419,
-        }
-    }, {
-        'url': 'http://www.nytimes.com/video/travel/100000003550828/36-hours-in-dubai.html',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
+class NYTimesBaseIE(InfoExtractor):
+    def _extract_video_from_id(self, video_id):
         video_data = self._download_json(
             'http://www.nytimes.com/svc/video/api/v2/video/%s' % video_id,
             video_id, 'Downloading video JSON')
@@ -81,3 +59,56 @@ class NYTimesIE(InfoExtractor):
             'formats': formats,
             'thumbnails': thumbnails,
         }
+
+
+class NYTimesIE(NYTimesBaseIE):
+    _VALID_URL = r'https?://(?:(?:www\.)?nytimes\.com/video/(?:[^/]+/)+?|graphics8\.nytimes\.com/bcvideo/\d+(?:\.\d+)?/iframe/embed\.html\?videoId=)(?P<id>\d+)'
+
+    _TESTS = [{
+        'url': 'http://www.nytimes.com/video/opinion/100000002847155/verbatim-what-is-a-photocopier.html?playlistId=100000001150263',
+        'md5': '18a525a510f942ada2720db5f31644c0',
+        'info_dict': {
+            'id': '100000002847155',
+            'ext': 'mov',
+            'title': 'Verbatim: What Is a Photocopier?',
+            'description': 'md5:93603dada88ddbda9395632fdc5da260',
+            'timestamp': 1398631707,
+            'upload_date': '20140427',
+            'uploader': 'Brett Weiner',
+            'duration': 419,
+        }
+    }, {
+        'url': 'http://www.nytimes.com/video/travel/100000003550828/36-hours-in-dubai.html',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        return self._extract_video_from_id(video_id)
+
+
+class NYTimesArticleIE(NYTimesBaseIE):
+    _VALID_URL = r'https?://(?:www)?\.nytimes\.com/\d{4}/\d{2}/\d{2}/(?:[^/]+/)*(?P<id>[^.]+)\.html'
+    _TEST = {
+        'url': 'http://www.nytimes.com/2015/04/14/business/owner-of-gravity-payments-a-credit-card-processor-is-setting-a-new-minimum-wage-70000-a-year.html?_r=0',
+        'md5': 'e2076d58b4da18e6a001d53fd56db3c9',
+        'info_dict': {
+            'id': '100000003628438',
+            'ext': 'mov',
+            'title': 'New Minimum Wage: $70,000 a Year',
+            'description': 'Dan Price, C.E.O. of Gravity Payments, surprised his 120-person staff by announcing that he planned over the next three years to raise the salary of every employee to $70,000 a year.',
+            'timestamp': 1429033037,
+            'upload_date': '20150414',
+            'uploader': 'Matthew Williams',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        video_id = self._html_search_regex(r'data-videoid="(\d+)"', webpage, 'video id')
+
+        return self._extract_video_from_id(video_id)

From df8418ffcff60c5a90e3138a39959d36cebcee7f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 4 May 2015 23:00:09 +0800
Subject: [PATCH 1174/6219] [nytimes] Extend _VALID_URL (#2754)

---
 youtube_dl/extractor/__init__.py | 5 ++++-
 youtube_dl/extractor/nytimes.py  | 9 ++++++---
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 41af925cc..1c3a46dd8 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -363,7 +363,10 @@ from .nrk import (
 )
 from .ntvde import NTVDeIE
 from .ntvru import NTVRuIE
-from .nytimes import NYTimesIE
+from .nytimes import (
+    NYTimesIE,
+    NYTimesArticleIE,
+)
 from .nuvid import NuvidIE
 from .odnoklassniki import OdnoklassnikiIE
 from .oktoberfesttv import OktoberfestTVIE
diff --git a/youtube_dl/extractor/nytimes.py b/youtube_dl/extractor/nytimes.py
index 266d8d400..6ffbe3863 100644
--- a/youtube_dl/extractor/nytimes.py
+++ b/youtube_dl/extractor/nytimes.py
@@ -89,8 +89,8 @@ class NYTimesIE(NYTimesBaseIE):
 
 
 class NYTimesArticleIE(NYTimesBaseIE):
-    _VALID_URL = r'https?://(?:www)?\.nytimes\.com/\d{4}/\d{2}/\d{2}/(?:[^/]+/)*(?P<id>[^.]+)\.html'
-    _TEST = {
+    _VALID_URL = r'https?://(?:www)?\.nytimes\.com/(.(?<!video))*?/(?:[^/]+/)*(?P<id>[^.]+)(?:\.html)?'
+    _TESTS = [{
         'url': 'http://www.nytimes.com/2015/04/14/business/owner-of-gravity-payments-a-credit-card-processor-is-setting-a-new-minimum-wage-70000-a-year.html?_r=0',
         'md5': 'e2076d58b4da18e6a001d53fd56db3c9',
         'info_dict': {
@@ -102,7 +102,10 @@ class NYTimesArticleIE(NYTimesBaseIE):
             'upload_date': '20150414',
             'uploader': 'Matthew Williams',
         }
-    }
+    }, {
+        'url': 'http://www.nytimes.com/news/minute/2014/03/17/times-minute-whats-next-in-crimea/?_php=true&_type=blogs&_php=true&_type=blogs&_r=1',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From dfad3aac981a311de89a5ae29121bcb6255902f1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 4 May 2015 21:23:26 +0600
Subject: [PATCH 1175/6219] [rutv] Fix live stream test URL

---
 youtube_dl/extractor/rutv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/rutv.py b/youtube_dl/extractor/rutv.py
index 1ec2c86e5..6b56b1d49 100644
--- a/youtube_dl/extractor/rutv.py
+++ b/youtube_dl/extractor/rutv.py
@@ -87,7 +87,7 @@ class RUTVIE(InfoExtractor):
             'skip': 'Translation has finished',
         },
         {
-            'url': 'http://live.russia.tv/index/index/channel_id/3',
+            'url': 'http://player.rutv.ru/iframe/live/id/21/showZoomBtn/false/isPlay/true/',
             'info_dict': {
                 'id': '21',
                 'ext': 'mp4',

From e038d5c4e323bdfa9de02ad59d1caa2c819d9cf5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 4 May 2015 21:29:32 +0600
Subject: [PATCH 1176/6219] [rutv] Fix preference

---
 youtube_dl/extractor/rutv.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/rutv.py b/youtube_dl/extractor/rutv.py
index 6b56b1d49..a5e98415f 100644
--- a/youtube_dl/extractor/rutv.py
+++ b/youtube_dl/extractor/rutv.py
@@ -128,8 +128,10 @@ class RUTVIE(InfoExtractor):
         elif video_path.startswith('index/iframe/cast_id'):
             video_type = 'live'
 
+        is_live = video_type == 'live'
+
         json_data = self._download_json(
-            'http://player.rutv.ru/iframe/%splay/id/%s' % ('live-' if video_type == 'live' else '', video_id),
+            'http://player.rutv.ru/iframe/%splay/id/%s' % ('live-' if is_live else '', video_id),
             video_id, 'Downloading JSON')
 
         if json_data['errors']:
@@ -156,6 +158,7 @@ class RUTVIE(InfoExtractor):
 
         for transport, links in media['sources'].items():
             for quality, url in links.items():
+                preference = -1 if priority_transport == transport else -2
                 if transport == 'rtmp':
                     mobj = re.search(r'^(?P<url>rtmp://[^/]+/(?P<app>.+))/(?P<playpath>.+)$', url)
                     if not mobj:
@@ -169,9 +172,11 @@ class RUTVIE(InfoExtractor):
                         'rtmp_live': True,
                         'ext': 'flv',
                         'vbr': int(quality),
+                        'preference': preference,
                     }
                 elif transport == 'm3u8':
-                    formats.extend(self._extract_m3u8_formats(url, video_id, 'mp4'))
+                    formats.extend(self._extract_m3u8_formats(
+                        url, video_id, 'mp4', preference=preference, m3u8_id='hls'))
                     continue
                 else:
                     fmt = {
@@ -181,7 +186,6 @@ class RUTVIE(InfoExtractor):
                     'width': width,
                     'height': height,
                     'format_id': '%s-%s' % (transport, quality),
-                    'preference': -1 if priority_transport == transport else -2,
                 })
                 formats.append(fmt)
 
@@ -190,8 +194,6 @@ class RUTVIE(InfoExtractor):
 
         self._sort_formats(formats)
 
-        is_live = video_type == 'live'
-
         return {
             'id': video_id,
             'title': self._live_title(title) if is_live else title,

From 1aa43d77c084bc2dd735fcd3d03b8b2fc3407c51 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 4 May 2015 21:29:56 +0600
Subject: [PATCH 1177/6219] [rutv] Remove superfluous check

---
 youtube_dl/extractor/rutv.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/youtube_dl/extractor/rutv.py b/youtube_dl/extractor/rutv.py
index a5e98415f..55604637d 100644
--- a/youtube_dl/extractor/rutv.py
+++ b/youtube_dl/extractor/rutv.py
@@ -189,9 +189,6 @@ class RUTVIE(InfoExtractor):
                 })
                 formats.append(fmt)
 
-        if not formats:
-            raise ExtractorError('No media links available for %s' % video_id)
-
         self._sort_formats(formats)
 
         return {

From 7212560f4d6b0de5b76eb41090c639855915946e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aur=C3=A9lien=20Dunand?= <aurelien@tassatux.net>
Date: Wed, 29 Apr 2015 01:07:33 +0200
Subject: [PATCH 1178/6219] [noco] Retrieve video language according to user
 options

---
 youtube_dl/extractor/noco.py | 22 ++++++++++++++++------
 1 file changed, 16 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/noco.py b/youtube_dl/extractor/noco.py
index 251e6da07..20a658149 100644
--- a/youtube_dl/extractor/noco.py
+++ b/youtube_dl/extractor/noco.py
@@ -86,22 +86,36 @@ class NocoIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
 
+        options = self._call_api('users/init', None, 'Downloading user options JSON')['options']
+        audio_lang = options.get('audio_language', 'fr')
+
         medias = self._call_api(
             'shows/%s/medias' % video_id,
             video_id, 'Downloading video JSON')
 
+        show = self._call_api(
+            'shows/by_id/%s' % video_id,
+            video_id, 'Downloading show JSON')[0]
+
+        if audio_lang == 'original':
+            audio_lang = show['original_lang']
+        if len(medias) == 1:
+            audio_lang = list(medias.keys())[0]
+        elif not audio_lang in medias:
+            audio_lang = 'fr'
+
         qualities = self._call_api(
             'qualities',
             video_id, 'Downloading qualities JSON')
 
         formats = []
 
-        for lang, lang_dict in medias['fr']['video_list'].items():
+        for lang, lang_dict in medias[audio_lang]['video_list'].items():
             for format_id, fmt in lang_dict['quality_list'].items():
                 format_id_extended = '%s-%s' % (lang, format_id) if lang != 'none' else format_id
 
                 video = self._call_api(
-                    'shows/%s/video/%s/fr' % (video_id, format_id.lower()),
+                    'shows/%s/video/%s/%s' % (video_id, format_id.lower(), audio_lang),
                     video_id, 'Downloading %s video JSON' % format_id_extended,
                     lang if lang != 'none' else None)
 
@@ -127,10 +141,6 @@ class NocoIE(InfoExtractor):
 
         self._sort_formats(formats)
 
-        show = self._call_api(
-            'shows/by_id/%s' % video_id,
-            video_id, 'Downloading show JSON')[0]
-
         upload_date = unified_strdate(show['online_date_start_utc'])
         uploader = show['partner_name']
         uploader_id = show['partner_key']

From ff9d68e7be2dd01a21cc0ed90aaa594b5d36697f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aur=C3=A9lien=20Dunand?= <aurelien@tassatux.net>
Date: Mon, 4 May 2015 19:55:29 +0200
Subject: [PATCH 1179/6219] [noco] Add test for multi languages video

---
 youtube_dl/extractor/noco.py | 45 +++++++++++++++++++++++++-----------
 1 file changed, 31 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/noco.py b/youtube_dl/extractor/noco.py
index 20a658149..f86d210ee 100644
--- a/youtube_dl/extractor/noco.py
+++ b/youtube_dl/extractor/noco.py
@@ -25,21 +25,38 @@ class NocoIE(InfoExtractor):
     _SUB_LANG_TEMPLATE = '&sub_lang=%s'
     _NETRC_MACHINE = 'noco'
 
-    _TEST = {
-        'url': 'http://noco.tv/emission/11538/nolife/ami-ami-idol-hello-france/',
-        'md5': '0a993f0058ddbcd902630b2047ef710e',
-        'info_dict': {
-            'id': '11538',
-            'ext': 'mp4',
-            'title': 'Ami Ami Idol - Hello! France',
-            'description': 'md5:4eaab46ab68fa4197a317a88a53d3b86',
-            'upload_date': '20140412',
-            'uploader': 'Nolife',
-            'uploader_id': 'NOL',
-            'duration': 2851.2,
+    _TESTS = [
+        {
+            'url': 'http://noco.tv/emission/11538/nolife/ami-ami-idol-hello-france/',
+            'md5': '0a993f0058ddbcd902630b2047ef710e',
+            'info_dict': {
+                'id': '11538',
+                'ext': 'mp4',
+                'title': 'Ami Ami Idol - Hello! France',
+                'description': 'md5:4eaab46ab68fa4197a317a88a53d3b86',
+                'upload_date': '20140412',
+                'uploader': 'Nolife',
+                'uploader_id': 'NOL',
+                'duration': 2851.2,
+            },
+            'skip': 'Requires noco account',
         },
-        'skip': 'Requires noco account',
-    }
+        {
+            'url': 'http://noco.tv/emission/12610/lbl42/the-guild/s01e01-wake-up-call',
+            'md5': 'c190f1f48e313c55838f1f412225934d',
+            'info_dict': {
+                'id': '12610',
+                'ext': 'mp4',
+                'title': 'The Guild #1 - Wake-Up Call',
+                'description': '',
+                'upload_date': '20140627',
+                'uploader': 'LBL42',
+                'uploader_id': 'LBL',
+                'duration': 233.023,
+            },
+            'skip': 'Requires noco account',
+        }
+    ]
 
     def _real_initialize(self):
         self._login()

From 6568382d6f3a986a773aa4d92b6bcbd367ccb794 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 5 May 2015 02:27:24 +0600
Subject: [PATCH 1180/6219] [noco] Extract all variations of audio/subtitles
 media

---
 youtube_dl/extractor/noco.py | 69 +++++++++++++++++++-----------------
 1 file changed, 37 insertions(+), 32 deletions(-)

diff --git a/youtube_dl/extractor/noco.py b/youtube_dl/extractor/noco.py
index f86d210ee..e44ece5a2 100644
--- a/youtube_dl/extractor/noco.py
+++ b/youtube_dl/extractor/noco.py
@@ -103,9 +103,6 @@ class NocoIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
 
-        options = self._call_api('users/init', None, 'Downloading user options JSON')['options']
-        audio_lang = options.get('audio_language', 'fr')
-
         medias = self._call_api(
             'shows/%s/medias' % video_id,
             video_id, 'Downloading video JSON')
@@ -114,12 +111,17 @@ class NocoIE(InfoExtractor):
             'shows/by_id/%s' % video_id,
             video_id, 'Downloading show JSON')[0]
 
-        if audio_lang == 'original':
-            audio_lang = show['original_lang']
+        options = self._call_api(
+            'users/init', video_id,
+            'Downloading user options JSON')['options']
+        audio_lang_pref = options.get('audio_language') or options.get('language', 'fr')
+
+        if audio_lang_pref == 'original':
+            audio_lang_pref = show['original_lang']
         if len(medias) == 1:
-            audio_lang = list(medias.keys())[0]
-        elif not audio_lang in medias:
-            audio_lang = 'fr'
+            audio_lang_pref = list(medias.keys())[0]
+        elif audio_lang_pref not in medias:
+            audio_lang_pref = 'fr'
 
         qualities = self._call_api(
             'qualities',
@@ -127,34 +129,37 @@ class NocoIE(InfoExtractor):
 
         formats = []
 
-        for lang, lang_dict in medias[audio_lang]['video_list'].items():
-            for format_id, fmt in lang_dict['quality_list'].items():
-                format_id_extended = '%s-%s' % (lang, format_id) if lang != 'none' else format_id
+        for audio_lang, audio_lang_dict in medias.items():
+            preference = 1 if audio_lang == audio_lang_pref else 0
+            for sub_lang, lang_dict in audio_lang_dict['video_list'].items():
+                for format_id, fmt in lang_dict['quality_list'].items():
+                    format_id_extended = 'audio-%s_sub-%s_%s' % (audio_lang, sub_lang, format_id)
 
-                video = self._call_api(
-                    'shows/%s/video/%s/%s' % (video_id, format_id.lower(), audio_lang),
-                    video_id, 'Downloading %s video JSON' % format_id_extended,
-                    lang if lang != 'none' else None)
+                    video = self._call_api(
+                        'shows/%s/video/%s/%s' % (video_id, format_id.lower(), audio_lang),
+                        video_id, 'Downloading %s video JSON' % format_id_extended,
+                        sub_lang if sub_lang != 'none' else None)
 
-                file_url = video['file']
-                if not file_url:
-                    continue
+                    file_url = video['file']
+                    if not file_url:
+                        continue
 
-                if file_url in ['forbidden', 'not found']:
-                    popmessage = video['popmessage']
-                    self._raise_error(popmessage['title'], popmessage['message'])
+                    if file_url in ['forbidden', 'not found']:
+                        popmessage = video['popmessage']
+                        self._raise_error(popmessage['title'], popmessage['message'])
 
-                formats.append({
-                    'url': file_url,
-                    'format_id': format_id_extended,
-                    'width': fmt['res_width'],
-                    'height': fmt['res_lines'],
-                    'abr': fmt['audiobitrate'],
-                    'vbr': fmt['videobitrate'],
-                    'filesize': fmt['filesize'],
-                    'format_note': qualities[format_id]['quality_name'],
-                    'preference': qualities[format_id]['priority'],
-                })
+                    formats.append({
+                        'url': file_url,
+                        'format_id': format_id_extended,
+                        'width': fmt['res_width'],
+                        'height': fmt['res_lines'],
+                        'abr': fmt['audiobitrate'],
+                        'vbr': fmt['videobitrate'],
+                        'filesize': fmt['filesize'],
+                        'format_note': qualities[format_id]['quality_name'],
+                        'quality': qualities[format_id]['priority'],
+                        'preference': preference,
+                    })
 
         self._sort_formats(formats)
 

From 815ac0293ee9d7f3771499cd71d833c83575bec9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 5 May 2015 02:38:13 +0600
Subject: [PATCH 1181/6219] [noco] Modernize

---
 youtube_dl/extractor/noco.py | 27 +++++++++++++++------------
 1 file changed, 15 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/noco.py b/youtube_dl/extractor/noco.py
index e44ece5a2..098c564bd 100644
--- a/youtube_dl/extractor/noco.py
+++ b/youtube_dl/extractor/noco.py
@@ -14,6 +14,9 @@ from ..compat import (
 from ..utils import (
     clean_html,
     ExtractorError,
+    int_or_none,
+    float_or_none,
+    parse_iso8601,
     unified_strdate,
 )
 
@@ -151,22 +154,22 @@ class NocoIE(InfoExtractor):
                     formats.append({
                         'url': file_url,
                         'format_id': format_id_extended,
-                        'width': fmt['res_width'],
-                        'height': fmt['res_lines'],
-                        'abr': fmt['audiobitrate'],
-                        'vbr': fmt['videobitrate'],
-                        'filesize': fmt['filesize'],
-                        'format_note': qualities[format_id]['quality_name'],
-                        'quality': qualities[format_id]['priority'],
+                        'width': int_or_none(fmt.get('res_width')),
+                        'height': int_or_none(fmt.get('res_lines')),
+                        'abr': int_or_none(fmt.get('audiobitrate')),
+                        'vbr': int_or_none(fmt.get('videobitrate')),
+                        'filesize': int_or_none(fmt.get('filesize')),
+                        'format_note': qualities[format_id].get('quality_name'),
+                        'quality': qualities[format_id].get('priority'),
                         'preference': preference,
                     })
 
         self._sort_formats(formats)
 
-        upload_date = unified_strdate(show['online_date_start_utc'])
-        uploader = show['partner_name']
-        uploader_id = show['partner_key']
-        duration = show['duration_ms'] / 1000.0
+        timestamp = parse_iso8601(show.get('online_date_start_utc'), ' ')
+        uploader = show.get('partner_name')
+        uploader_id = show.get('partner_key')
+        duration = float_or_none(show.get('duration_ms'), 1000)
 
         thumbnails = []
         for thumbnail_key, thumbnail_url in show.items():
@@ -198,7 +201,7 @@ class NocoIE(InfoExtractor):
             'title': title,
             'description': description,
             'thumbnails': thumbnails,
-            'upload_date': upload_date,
+            'timestamp': timestamp,
             'uploader': uploader,
             'uploader_id': uploader_id,
             'duration': duration,

From 01e4b1ee14a3e9dedcb6a156c6eaf1603a8a0592 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 5 May 2015 02:50:39 +0600
Subject: [PATCH 1182/6219] [noco] Update tests

---
 youtube_dl/extractor/noco.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/noco.py b/youtube_dl/extractor/noco.py
index 098c564bd..5674ee2a4 100644
--- a/youtube_dl/extractor/noco.py
+++ b/youtube_dl/extractor/noco.py
@@ -51,7 +51,7 @@ class NocoIE(InfoExtractor):
                 'id': '12610',
                 'ext': 'mp4',
                 'title': 'The Guild #1 - Wake-Up Call',
-                'description': '',
+                'timestamp': 1403863200,
                 'upload_date': '20140627',
                 'uploader': 'LBL42',
                 'uploader_id': 'LBL',

From ce5c1ae51703d1be4e486cbe8d3dfc2ab16372a8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 5 May 2015 02:52:21 +0600
Subject: [PATCH 1183/6219] [noco] Remove unused import

---
 youtube_dl/extractor/noco.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/noco.py b/youtube_dl/extractor/noco.py
index 5674ee2a4..664dc81d4 100644
--- a/youtube_dl/extractor/noco.py
+++ b/youtube_dl/extractor/noco.py
@@ -17,7 +17,6 @@ from ..utils import (
     int_or_none,
     float_or_none,
     parse_iso8601,
-    unified_strdate,
 )
 
 

From e10dc0e1f0f6b33b60a601bf7855e76e7813ecde Mon Sep 17 00:00:00 2001
From: rrooij <rderooij685@gmail.com>
Date: Tue, 5 May 2015 08:59:09 +0200
Subject: [PATCH 1184/6219] [southparknl] Add extractor for southpark.nl

---
 youtube_dl/extractor/__init__.py  |  1 +
 youtube_dl/extractor/southpark.py | 15 +++++++++++++++
 2 files changed, 16 insertions(+)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 1c3a46dd8..b042e9215 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -485,6 +485,7 @@ from .southpark import (
     SouthParkIE,
     SouthParkEsIE,
     SouthparkDeIE,
+    SouthParkNlIE
 )
 from .space import SpaceIE
 from .spankbang import SpankBangIE
diff --git a/youtube_dl/extractor/southpark.py b/youtube_dl/extractor/southpark.py
index e3b73295c..ebcd2ed04 100644
--- a/youtube_dl/extractor/southpark.py
+++ b/youtube_dl/extractor/southpark.py
@@ -46,3 +46,18 @@ class SouthparkDeIE(SouthParkIE):
             'description': 'Cartman explains the benefits of "Shitter" to Stan, Kyle and Craig.',
         },
     }]
+
+
+class SouthParkNlIE(SouthParkIE):
+    IE_NAME = 'southpark.nl'
+    _VALID_URL = r'https?://(?:www\.)?(?P<url>southpark\.nl/(?:clips|full-episodes)/(?P<id>.+?)(\?|#|$))'
+    _FEED_URL = 'http://www.southpark.nl/feeds/video-player/mrss/'
+
+    _TESTS = [{
+        'url': 'http://www.southpark.nl/full-episodes/s18e06-freemium-isnt-free',
+        'info_dict': {
+            'id': 'c56b9a1f-0e42-4942-89e0-8e721d814c5c',
+            'ext': 'mp4',
+            'title': 'South Park 1806: Freemium Isn\'t Free - Act 3',
+        },
+    }]

From 3408f6e64a7438dbc1bcc087a00e8540ef660eb3 Mon Sep 17 00:00:00 2001
From: rrooij <rderooij685@gmail.com>
Date: Tue, 5 May 2015 09:01:07 +0200
Subject: [PATCH 1185/6219] [southparkde] Fix naming inconsistency

The class was first called 'SouthparkDe'. It is now changed to
'SouthParkDe' to match the name of the other extractors.
---
 youtube_dl/extractor/__init__.py  | 2 +-
 youtube_dl/extractor/southpark.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index b042e9215..5dfa781f8 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -484,7 +484,7 @@ from .soundgasm import (
 from .southpark import (
     SouthParkIE,
     SouthParkEsIE,
-    SouthparkDeIE,
+    SouthParkDeIE,
     SouthParkNlIE
 )
 from .space import SpaceIE
diff --git a/youtube_dl/extractor/southpark.py b/youtube_dl/extractor/southpark.py
index ebcd2ed04..2b12f2ad3 100644
--- a/youtube_dl/extractor/southpark.py
+++ b/youtube_dl/extractor/southpark.py
@@ -32,7 +32,7 @@ class SouthParkEsIE(SouthParkIE):
     }]
 
 
-class SouthparkDeIE(SouthParkIE):
+class SouthParkDeIE(SouthParkIE):
     IE_NAME = 'southpark.de'
     _VALID_URL = r'https?://(?:www\.)?(?P<url>southpark\.de/(?:clips|alle-episoden)/(?P<id>.+?)(\?|#|$))'
     _FEED_URL = 'http://www.southpark.de/feeds/video-player/mrss/'

From dcf807790673ef8adb22ed6ff93f6232903d75f6 Mon Sep 17 00:00:00 2001
From: rrooij <rderooij685@gmail.com>
Date: Tue, 5 May 2015 09:17:21 +0200
Subject: [PATCH 1186/6219] [southparknl] Fix test to match playlist tests

---
 youtube_dl/extractor/southpark.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/southpark.py b/youtube_dl/extractor/southpark.py
index 2b12f2ad3..59e31198c 100644
--- a/youtube_dl/extractor/southpark.py
+++ b/youtube_dl/extractor/southpark.py
@@ -55,9 +55,5 @@ class SouthParkNlIE(SouthParkIE):
 
     _TESTS = [{
         'url': 'http://www.southpark.nl/full-episodes/s18e06-freemium-isnt-free',
-        'info_dict': {
-            'id': 'c56b9a1f-0e42-4942-89e0-8e721d814c5c',
-            'ext': 'mp4',
-            'title': 'South Park 1806: Freemium Isn\'t Free - Act 3',
-        },
+        'playlist_count': 4,
     }]

From 07d2921c6d13ce43adb9347677e50c36268fe54f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 5 May 2015 23:39:54 +0800
Subject: [PATCH 1187/6219] [lifenews] Correctly determine iframe links (fixes
 #5618)

---
 youtube_dl/extractor/lifenews.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/youtube_dl/extractor/lifenews.py b/youtube_dl/extractor/lifenews.py
index 330138692..963f16e1a 100644
--- a/youtube_dl/extractor/lifenews.py
+++ b/youtube_dl/extractor/lifenews.py
@@ -39,6 +39,17 @@ class LifeNewsIE(InfoExtractor):
             'upload_date': '20150402',
             'uploader': 'embed.life.ru',
         }
+    }, {
+        'url': 'http://lifenews.ru/news/153461',
+        'md5': '9b6ef8bc0ffa25aebc8bdb40d89ab795',
+        'info_dict': {
+            'id': '153461',
+            'ext': 'mp4',
+            'title': 'В Москве спасли потерявшегося медвежонка, который спрятался на дереве',
+            'description': 'Маленький хищник не смог найти дорогу домой и обрел временное убежище на тополе недалеко от жилого массива, пока его не нашла соседская собака.',
+            'upload_date': '20150505',
+            'uploader': 'embed.life.ru',
+        }
     }]
 
     def _real_extract(self, url):
@@ -88,6 +99,8 @@ class LifeNewsIE(InfoExtractor):
             return cur_info
 
         if iframe_link:
+            if iframe_link.startswith('//'):
+                iframe_link = 'http:' + iframe_link
             cur_info = dict(common_info)
             cur_info.update({
                 '_type': 'url_transparent',

From b326b07adc49466a4bdd0ea3c63f329a9e523121 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 5 May 2015 21:49:36 +0600
Subject: [PATCH 1188/6219] [lifenews] Use `_proto_relative_url`

---
 youtube_dl/extractor/lifenews.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/lifenews.py b/youtube_dl/extractor/lifenews.py
index 963f16e1a..081016b80 100644
--- a/youtube_dl/extractor/lifenews.py
+++ b/youtube_dl/extractor/lifenews.py
@@ -99,8 +99,7 @@ class LifeNewsIE(InfoExtractor):
             return cur_info
 
         if iframe_link:
-            if iframe_link.startswith('//'):
-                iframe_link = 'http:' + iframe_link
+            iframe_link = self._proto_relative_url(iframe_link, 'http:')
             cur_info = dict(common_info)
             cur_info.update({
                 '_type': 'url_transparent',

From d6a1738892038940ef4af59a33aeddc99ef0c966 Mon Sep 17 00:00:00 2001
From: blissland <blissland.house@googlemail.com>
Date: Wed, 6 May 2015 11:48:36 +0100
Subject: [PATCH 1189/6219] [archive.org] Fix incorrect url condition (closes
 #5628)

The condition for assigning to json_url is the wrong way round:

currently for url: aaa.com/xxx

we get:

aaa.com/xxx&output=json

instead of the correct value:

aaa.com/xxx?output=json
---
 youtube_dl/extractor/archiveorg.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/archiveorg.py b/youtube_dl/extractor/archiveorg.py
index 9fc35a42b..8feb7cb74 100644
--- a/youtube_dl/extractor/archiveorg.py
+++ b/youtube_dl/extractor/archiveorg.py
@@ -33,7 +33,7 @@ class ArchiveOrgIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        json_url = url + ('?' if '?' in url else '&') + 'output=json'
+        json_url = url + ('&' if '?' in url else '?') + 'output=json'
         data = self._download_json(json_url, video_id)
 
         def get_optional(data_dict, field):

From 74f728249fcf58b0c99eb69606b04b304d882423 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 6 May 2015 21:24:24 +0600
Subject: [PATCH 1190/6219] [extractor/common] Fallback to empty string for
 (yet) missing `format_id` in `_sort_formats` (Closes #5624)

---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 3ae5d5212..34e27a35a 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -764,7 +764,7 @@ class InfoExtractor(object):
                 f.get('fps') if f.get('fps') is not None else -1,
                 f.get('filesize_approx') if f.get('filesize_approx') is not None else -1,
                 f.get('source_preference') if f.get('source_preference') is not None else -1,
-                f.get('format_id'),
+                f.get('format_id') if f.get('format_id') is not None else '',
             )
         formats.sort(key=_formats_key)
 

From e8b9ab8957084d1b7ab60f14574e5d14d9417909 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 6 May 2015 21:31:25 +0600
Subject: [PATCH 1191/6219] [pbs] Add `format_id` for direct links

---
 youtube_dl/extractor/pbs.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 761bd6d8d..143a76696 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -187,6 +187,7 @@ class PBSIE(InfoExtractor):
             else:
                 formats.append({
                     'url': format_url,
+                    'format_id': redirect.get('eeid'),
                 })
         self._sort_formats(formats)
 

From 6a8f9cd22efa7db844e035a3e40571f7d05eebea Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 6 May 2015 21:39:53 +0600
Subject: [PATCH 1192/6219] [giga] Fix view count extraction

---
 youtube_dl/extractor/giga.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/giga.py b/youtube_dl/extractor/giga.py
index 775890112..28eb733e2 100644
--- a/youtube_dl/extractor/giga.py
+++ b/youtube_dl/extractor/giga.py
@@ -85,7 +85,8 @@ class GigaIE(InfoExtractor):
             r'class="author">([^<]+)</a>', webpage, 'uploader', fatal=False)
 
         view_count = str_to_int(self._search_regex(
-            r'<span class="views"><strong>([\d.]+)</strong>', webpage, 'view count', fatal=False))
+            r'<span class="views"><strong>([\d.,]+)</strong>',
+            webpage, 'view count', fatal=False))
 
         return {
             'id': video_id,

From 1ed34f3dd6f534a3a5a3d6808d8d3466a9e5dea2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 6 May 2015 21:43:36 +0600
Subject: [PATCH 1193/6219] [gorillavid] Switch 404 test to only matching

---
 youtube_dl/extractor/gorillavid.py | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/gorillavid.py b/youtube_dl/extractor/gorillavid.py
index ed2623456..6147596e4 100644
--- a/youtube_dl/extractor/gorillavid.py
+++ b/youtube_dl/extractor/gorillavid.py
@@ -35,13 +35,7 @@ class GorillaVidIE(InfoExtractor):
         },
     }, {
         'url': 'http://gorillavid.in/embed-z08zf8le23c6-960x480.html',
-        'md5': 'c9e293ca74d46cad638e199c3f3fe604',
-        'info_dict': {
-            'id': 'z08zf8le23c6',
-            'ext': 'mp4',
-            'title': 'Say something nice',
-            'thumbnail': 're:http://.*\.jpg',
-        },
+        'only_matching': True,
     }, {
         'url': 'http://daclips.in/3rso4kdn6f9m',
         'md5': '1ad8fd39bb976eeb66004d3a4895f106',

From ad0c0ad3b4db546dc2078391f397aa5b44c8150b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 6 May 2015 21:52:26 +0600
Subject: [PATCH 1194/6219] [historicfilms] Fix tape id extraction

---
 youtube_dl/extractor/historicfilms.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/historicfilms.py b/youtube_dl/extractor/historicfilms.py
index 40afbe537..6a36933ac 100644
--- a/youtube_dl/extractor/historicfilms.py
+++ b/youtube_dl/extractor/historicfilms.py
@@ -25,7 +25,8 @@ class HistoricFilmsIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         tape_id = self._search_regex(
-            r'class="tapeId">([^<]+)<', webpage, 'tape id')
+            [r'class="tapeId"[^>]*>([^<]+)<', r'tapeId\s*:\s*"([^"]+)"'],
+            webpage, 'tape id')
 
         title = self._og_search_title(webpage)
         description = self._og_search_description(webpage)

From ac6c358c2ab882427f74af47fe5df762dd348c20 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 7 May 2015 02:08:47 +0800
Subject: [PATCH 1195/6219] [teamcoco] Fix extracting preload data again

---
 youtube_dl/extractor/teamcoco.py | 24 +++++++++++++++++++++---
 1 file changed, 21 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/teamcoco.py b/youtube_dl/extractor/teamcoco.py
index 2381676b4..95d58ddd0 100644
--- a/youtube_dl/extractor/teamcoco.py
+++ b/youtube_dl/extractor/teamcoco.py
@@ -2,6 +2,7 @@
 from __future__ import unicode_literals
 
 import base64
+import binascii
 import re
 
 from .common import InfoExtractor
@@ -9,6 +10,7 @@ from ..utils import (
     ExtractorError,
     qualities,
 )
+from ..compat import compat_ord
 
 
 class TeamcocoIE(InfoExtractor):
@@ -66,7 +68,7 @@ class TeamcocoIE(InfoExtractor):
             video_id = self._html_search_regex(
                 self._VIDEO_ID_REGEXES, webpage, 'video id')
 
-        preload = None
+        data = preload = None
         preloads = re.findall(r'"preload":\s*"([^"]+)"', webpage)
         if preloads:
             preload = max([(len(p), p) for p in preloads])[1]
@@ -80,11 +82,27 @@ class TeamcocoIE(InfoExtractor):
             ], webpage.replace('","', ''), 'preload data', default=None)
 
         if not preload:
+            preload_codes = self._html_search_regex(
+                r'(function.+)setTimeout\(function\(\)\{playlist',
+                webpage, 'preload codes')
+            base64_fragments = re.findall(r'"([a-zA-z0-9+/=]+)"', preload_codes)
+            base64_fragments.remove('init')
+            for i in range(len(base64_fragments)):
+                cur_sequence = (''.join(base64_fragments[i:] + base64_fragments[:i])).encode('ascii')
+                try:
+                    raw_data = base64.b64decode(cur_sequence)
+                except (TypeError, binascii.Error):
+                    continue
+                if compat_ord(raw_data[0]) == compat_ord('{'):
+                    data = self._parse_json(raw_data.decode('utf-8'), video_id, fatal=False)
+
+        if not preload and not data:
             raise ExtractorError(
                 'Preload information could not be extracted', expected=True)
 
-        data = self._parse_json(
-            base64.b64decode(preload.encode('ascii')).decode('utf-8'), video_id)
+        if not data:
+            data = self._parse_json(
+                base64.b64decode(preload.encode('ascii')).decode('utf-8'), video_id)
 
         formats = []
         get_quality = qualities(['500k', '480p', '1000k', '720p', '1080p'])

From d9a743d9178b0ed1e44168e42e8cec2d7dd8d63e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 7 May 2015 18:05:37 +0800
Subject: [PATCH 1196/6219] [vice] Remove a redundant print

---
 youtube_dl/extractor/vice.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/vice.py b/youtube_dl/extractor/vice.py
index 71f520fb5..04e2b0ba7 100644
--- a/youtube_dl/extractor/vice.py
+++ b/youtube_dl/extractor/vice.py
@@ -31,7 +31,6 @@ class ViceIE(InfoExtractor):
                 r'embedCode=([^&\'"]+)', webpage,
                 'ooyala embed code')
             ooyala_url = OoyalaIE._url_for_embed_code(embed_code)
-            print(ooyala_url)
         except ExtractorError:
             raise ExtractorError('The page doesn\'t contain a video', expected=True)
         return self.url_result(ooyala_url, ie='Ooyala')

From 05d5392cdaa558dba285c328182d4f3e82fb8e8b Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 7 May 2015 18:06:22 +0800
Subject: [PATCH 1197/6219] [common] Ignore subtitles in m3u8

---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 34e27a35a..981e34bc7 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -896,7 +896,7 @@ class InfoExtractor(object):
                 format_id = []
                 if m3u8_id:
                     format_id.append(m3u8_id)
-                last_media_name = last_media.get('NAME') if last_media else None
+                last_media_name = last_media.get('NAME') if last_media and last_media.get('TYPE') != 'SUBTITLES' else None
                 format_id.append(last_media_name if last_media_name else '%d' % (tbr if tbr else len(formats)))
                 f = {
                     'format_id': '-'.join(format_id),

From 84bf31aaf8b9b7397de5f3189295d93e8e93e5e2 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 7 May 2015 18:12:01 +0800
Subject: [PATCH 1198/6219] [ooyala] Extract m3u8 information (#2292)

---
 youtube_dl/extractor/ooyala.py | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/ooyala.py b/youtube_dl/extractor/ooyala.py
index d5b05c18f..b33e8230d 100644
--- a/youtube_dl/extractor/ooyala.py
+++ b/youtube_dl/extractor/ooyala.py
@@ -6,6 +6,7 @@ from .common import InfoExtractor
 from ..utils import (
     unescapeHTML,
     ExtractorError,
+    determine_ext,
 )
 
 
@@ -44,11 +45,21 @@ class OoyalaIE(InfoExtractor):
                               ie=cls.ie_key())
 
     def _extract_result(self, info, more_info):
+        embedCode = info['embedCode']
+        video_url = info.get('ipad_url') or info['url']
+
+        if determine_ext(video_url) == 'm3u8':
+            formats = self._extract_m3u8_formats(video_url, embedCode, ext='mp4')
+        else:
+            formats = [{
+                'url': video_url,
+                'ext': 'mp4',
+            }]
+
         return {
-            'id': info['embedCode'],
-            'ext': 'mp4',
+            'id': embedCode,
             'title': unescapeHTML(info['title']),
-            'url': info.get('ipad_url') or info['url'],
+            'formats': formats,
             'description': unescapeHTML(more_info['description']),
             'thumbnail': more_info['promo'],
         }

From c09593c04e0b345df02cae663dc064d29e241cba Mon Sep 17 00:00:00 2001
From: blissland <blissland.house@googlemail.com>
Date: Thu, 7 May 2015 15:07:11 +0100
Subject: [PATCH 1199/6219] [BildIE] Escape ampersands in xml and update test
 thumbnail

---
 youtube_dl/extractor/bild.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/bild.py b/youtube_dl/extractor/bild.py
index 77b562d99..ba0c185eb 100644
--- a/youtube_dl/extractor/bild.py
+++ b/youtube_dl/extractor/bild.py
@@ -2,7 +2,10 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..utils import int_or_none
+from ..utils import (
+  int_or_none,
+  fix_xml_ampersands,
+)
 
 
 class BildIE(InfoExtractor):
@@ -15,7 +18,7 @@ class BildIE(InfoExtractor):
             'id': '38184146',
             'ext': 'mp4',
             'title': 'BILD hat sie getestet',
-            'thumbnail': 'http://bilder.bild.de/fotos/stand-das-koennen-die-neuen-ipads-38184138/Bild/1.bild.jpg',
+            'thumbnail': 'http://bilder.bild.de/fotos/bild-hat-sie-getestet-das-koennen-apples-neue-ipads-38184138/Bild/1.bild.jpg',
             'duration': 196,
             'description': 'Mit dem iPad Air 2 und dem iPad Mini 3 hat Apple zwei neue Tablet-Modelle präsentiert. BILD-Reporter Sven Stein durfte die Geräte bereits testen. ',
         }
@@ -25,7 +28,7 @@ class BildIE(InfoExtractor):
         video_id = self._match_id(url)
 
         xml_url = url.split(".bild.html")[0] + ",view=xml.bild.xml"
-        doc = self._download_xml(xml_url, video_id)
+        doc = self._download_xml(xml_url, video_id, transform_source=fix_xml_ampersands)
 
         duration = int_or_none(doc.attrib.get('duration'), scale=1000)
 

From aafe2739909882931f7f624e83fe532af0bfafc1 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 7 May 2015 22:07:32 +0800
Subject: [PATCH 1200/6219] [ooyala] Use SAS API to extract info (fixes #4336)

---
 youtube_dl/extractor/ooyala.py | 43 ++++++++++++++++++++++++++++++++++
 1 file changed, 43 insertions(+)

diff --git a/youtube_dl/extractor/ooyala.py b/youtube_dl/extractor/ooyala.py
index b33e8230d..0b049274a 100644
--- a/youtube_dl/extractor/ooyala.py
+++ b/youtube_dl/extractor/ooyala.py
@@ -1,12 +1,14 @@
 from __future__ import unicode_literals
 import re
 import json
+import base64
 
 from .common import InfoExtractor
 from ..utils import (
     unescapeHTML,
     ExtractorError,
     determine_ext,
+    int_or_none,
 )
 
 
@@ -33,6 +35,17 @@ class OoyalaIE(InfoExtractor):
                 'description': '',
             },
         },
+        {
+            # Information available only through SAS api
+            # From http://community.plm.automation.siemens.com/t5/News-NX-Manufacturing/Tool-Path-Divide/ba-p/4187
+            'url': 'http://player.ooyala.com/player.js?embedCode=FiOG81ZTrvckcchQxmalf4aQj590qTEx',
+            'md5': 'a84001441b35ea492bc03736e59e7935',
+            'info_dict': {
+                'id': 'FiOG81ZTrvckcchQxmalf4aQj590qTEx',
+                'ext': 'mp4',
+                'title': 'Ooyala video',
+            }
+        }
     ]
 
     @staticmethod
@@ -88,6 +101,36 @@ class OoyalaIE(InfoExtractor):
                 mobile_player, 'info', fatal=False, default=None)
             if videos_info:
                 break
+
+        if not videos_info:
+            formats = []
+            auth_data = self._download_json(
+                'http://player.ooyala.com/sas/player_api/v1/authorization/embed_code/%s/%s?domain=www.example.org&supportedFormats=mp4,webm' % (embedCode, embedCode),
+                embedCode)
+
+            cur_auth_data = auth_data['authorization_data'][embedCode]
+
+            for stream in cur_auth_data['streams']:
+                formats.append({
+                    'url': base64.b64decode(stream['url']['data'].encode('ascii')).decode('utf-8'),
+                    'ext': stream.get('delivery_type'),
+                    'format': stream.get('video_codec'),
+                    'format_id': stream.get('profile'),
+                    'width': int_or_none(stream.get('width')),
+                    'height': int_or_none(stream.get('height')),
+                    'abr': int_or_none(stream.get('audio_bitrate')),
+                    'vbr': int_or_none(stream.get('video_bitrate')),
+                })
+            if len(formats):
+                return {
+                    'id': embedCode,
+                    'formats': formats,
+                    'title': 'Ooyala video',
+                }
+
+            if not cur_auth_data['authorized']:
+                raise ExtractorError(cur_auth_data['message'], expected=True)
+
         if not videos_info:
             raise ExtractorError('Unable to extract info')
         videos_info = videos_info.replace('\\"', '"')

From bc08873cff6d36ba175e5121b0ba1ad270c664c9 Mon Sep 17 00:00:00 2001
From: blissland <blissland.house@googlemail.com>
Date: Thu, 7 May 2015 15:09:27 +0100
Subject: [PATCH 1201/6219] Fix indents

---
 youtube_dl/extractor/bild.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/bild.py b/youtube_dl/extractor/bild.py
index ba0c185eb..41dd1cbc1 100644
--- a/youtube_dl/extractor/bild.py
+++ b/youtube_dl/extractor/bild.py
@@ -3,8 +3,8 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..utils import (
-  int_or_none,
-  fix_xml_ampersands,
+    int_or_none,
+    fix_xml_ampersands,
 )
 
 

From f22834a3723dd59f1e04c27d0bc3a373ebc17183 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 7 May 2015 20:20:43 +0600
Subject: [PATCH 1202/6219] [bild] Relax thumbnail test check

---
 youtube_dl/extractor/bild.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bild.py b/youtube_dl/extractor/bild.py
index 41dd1cbc1..4d8cce1ef 100644
--- a/youtube_dl/extractor/bild.py
+++ b/youtube_dl/extractor/bild.py
@@ -18,7 +18,7 @@ class BildIE(InfoExtractor):
             'id': '38184146',
             'ext': 'mp4',
             'title': 'BILD hat sie getestet',
-            'thumbnail': 'http://bilder.bild.de/fotos/bild-hat-sie-getestet-das-koennen-apples-neue-ipads-38184138/Bild/1.bild.jpg',
+            'thumbnail': 're:^https?://.*\.jpg$',
             'duration': 196,
             'description': 'Mit dem iPad Air 2 und dem iPad Mini 3 hat Apple zwei neue Tablet-Modelle präsentiert. BILD-Reporter Sven Stein durfte die Geräte bereits testen. ',
         }

From 3799834dcff27ffeea66e3ec96166f8da8fa73ab Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 7 May 2015 20:46:11 +0600
Subject: [PATCH 1203/6219] [YoutubeDL] Do not force bestvideo+bestaudio when
 outtmpl is stdout (#5627)

---
 youtube_dl/YoutubeDL.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index eb7470f72..d8583a8eb 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1086,7 +1086,9 @@ class YoutubeDL(object):
         req_format = self.params.get('format')
         if req_format is None:
             req_format_list = []
-            if info_dict['extractor'] in ['youtube', 'ted'] and FFmpegMergerPP(self).available:
+            if (self.params.get('outtmpl', DEFAULT_OUTTMPL) != '-'
+                    and info_dict['extractor'] in ['youtube', 'ted']
+                    and FFmpegMergerPP(self).available):
                 req_format_list.append('bestvideo+bestaudio')
             req_format_list.append('best')
             req_format = '/'.join(req_format_list)

From 406224be5231e602b543579706ad6056b75fbe68 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 7 May 2015 21:02:59 +0600
Subject: [PATCH 1204/6219] [extractor/generic] Fix following incomplete
 redirects (#5640)

---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index d09e85665..cd7c47d6d 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1453,7 +1453,7 @@ class GenericIE(InfoExtractor):
                 if refresh_header:
                     found = re.search(REDIRECT_REGEX, refresh_header)
             if found:
-                new_url = found.group(1)
+                new_url = compat_urlparse.urljoin(url, found.group(1))
                 self.report_following_redirect(new_url)
                 return {
                     '_type': 'url',

From 5268a05e4722d74e125a97b023d92943745bb249 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 7 May 2015 17:04:15 +0200
Subject: [PATCH 1205/6219] [ooyala] Style fix

---
 youtube_dl/extractor/ooyala.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ooyala.py b/youtube_dl/extractor/ooyala.py
index 0b049274a..c0e6d643d 100644
--- a/youtube_dl/extractor/ooyala.py
+++ b/youtube_dl/extractor/ooyala.py
@@ -121,7 +121,7 @@ class OoyalaIE(InfoExtractor):
                     'abr': int_or_none(stream.get('audio_bitrate')),
                     'vbr': int_or_none(stream.get('video_bitrate')),
                 })
-            if len(formats):
+            if formats:
                 return {
                     'id': embedCode,
                     'formats': formats,

From 09b412dafa4bc95b9850d77b3bce5f7eac47a578 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 8 May 2015 02:12:28 +0800
Subject: [PATCH 1206/6219] [nhl] Partial support for hlg id (fixes #4285)

---
 youtube_dl/extractor/nhl.py | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nhl.py b/youtube_dl/extractor/nhl.py
index 407465998..b572370c2 100644
--- a/youtube_dl/extractor/nhl.py
+++ b/youtube_dl/extractor/nhl.py
@@ -21,6 +21,9 @@ class NHLBaseInfoExtractor(InfoExtractor):
         return json_string.replace('\\\'', '\'')
 
     def _real_extract_video(self, video_id):
+        vid_parts = video_id.split(',')
+        if len(vid_parts) == 3:
+            video_id = '%s0%s%s-X-h' % (vid_parts[0][:4], vid_parts[1], vid_parts[2].rjust(4, '0'))
         json_url = 'http://video.nhl.com/videocenter/servlets/playlist?ids=%s&format=json' % video_id
         data = self._download_json(
             json_url, video_id, transform_source=self._fix_json)
@@ -60,7 +63,7 @@ class NHLBaseInfoExtractor(InfoExtractor):
 
 class NHLIE(NHLBaseInfoExtractor):
     IE_NAME = 'nhl.com'
-    _VALID_URL = r'https?://video(?P<team>\.[^.]*)?\.nhl\.com/videocenter/(?:console)?(?:\?(?:.*?[?&])?)id=(?P<id>[-0-9a-zA-Z]+)'
+    _VALID_URL = r'https?://video(?P<team>\.[^.]*)?\.nhl\.com/videocenter/(?:console)?(?:\?(?:.*?[?&])?)(?:id|hlg)=(?P<id>[-0-9a-zA-Z,]+)'
 
     _TESTS = [{
         'url': 'http://video.canucks.nhl.com/videocenter/console?catid=6?id=453614',
@@ -101,6 +104,17 @@ class NHLIE(NHLBaseInfoExtractor):
     }, {
         'url': 'http://video.nhl.com/videocenter/?id=736722',
         'only_matching': True,
+    }, {
+        'url': 'http://video.nhl.com/videocenter/console?hlg=20142015,2,299&lang=en',
+        'md5': '076fcb88c255154aacbf0a7accc3f340',
+        'info_dict': {
+            'id': '2014020299-X-h',
+            'ext': 'mp4',
+            'title': 'Penguins at Islanders / Game Highlights',
+            'description': 'Home broadcast - Pittsburgh Penguins at New York Islanders - November 22, 2014',
+            'duration': 268,
+            'upload_date': '20141122',
+        }
     }]
 
     def _real_extract(self, url):

From 46be82b811d91be0b0876cf141e6a94e65b8fd7f Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Thu, 7 May 2015 21:58:03 +0300
Subject: [PATCH 1207/6219] [vessel] Use `main_video_asset` when searching for
 video_asset (Fixes #5623)

---
 youtube_dl/extractor/vessel.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/vessel.py b/youtube_dl/extractor/vessel.py
index 6215f0642..3c8d2a943 100644
--- a/youtube_dl/extractor/vessel.py
+++ b/youtube_dl/extractor/vessel.py
@@ -38,9 +38,13 @@ class VesselIE(InfoExtractor):
         return req
 
     @staticmethod
-    def find_assets(data, asset_type):
+    def find_assets(data, asset_type, asset_id=None):
         for asset in data.get('assets', []):
-            if asset.get('type') == asset_type:
+            if not asset.get('type') == asset_type:
+                continue
+            elif asset_id is not None and not asset.get('id') == asset_id:
+                continue
+            else:
                 yield asset
 
     def _check_access_rights(self, data):
@@ -82,11 +86,13 @@ class VesselIE(InfoExtractor):
         req = VesselIE.make_json_request(
             self._API_URL_TEMPLATE % asset_id, {'client': 'web'})
         data = self._download_json(req, video_id)
+        video_asset_id = data.get('main_video_asset')
 
         self._check_access_rights(data)
 
         try:
-            video_asset = next(VesselIE.find_assets(data, 'video'))
+            video_asset = next(
+                VesselIE.find_assets(data, 'video', asset_id=video_asset_id))
         except StopIteration:
             raise ExtractorError('No video assets found')
 

From 156fc83a55b14258bb4a2fa1ec3b02d4db679603 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 8 May 2015 03:02:34 +0800
Subject: [PATCH 1208/6219] [downloader/rtmp] Fix a typo

---
 youtube_dl/downloader/rtmp.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/rtmp.py b/youtube_dl/downloader/rtmp.py
index 6865b5e2f..7d19bb808 100644
--- a/youtube_dl/downloader/rtmp.py
+++ b/youtube_dl/downloader/rtmp.py
@@ -131,7 +131,7 @@ class RtmpFD(FileDownloader):
         if play_path is not None:
             basic_args += ['--playpath', play_path]
         if tc_url is not None:
-            basic_args += ['--tcUrl', url]
+            basic_args += ['--tcUrl', tc_url]
         if test:
             basic_args += ['--stop', '1']
         if flash_version is not None:

From 7ef00afe9da87c7d0fdbea93af39b47d5447f1a0 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 8 May 2015 03:09:19 +0800
Subject: [PATCH 1209/6219] [nhl] Support RTMP videos (fixes #4481)

---
 youtube_dl/extractor/nhl.py | 23 ++++++++++++++++++++++-
 1 file changed, 22 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nhl.py b/youtube_dl/extractor/nhl.py
index b572370c2..279b18386 100644
--- a/youtube_dl/extractor/nhl.py
+++ b/youtube_dl/extractor/nhl.py
@@ -50,7 +50,7 @@ class NHLBaseInfoExtractor(InfoExtractor):
             video_url = initial_video_url
 
         join = compat_urlparse.urljoin
-        return {
+        ret = {
             'id': video_id,
             'title': info['name'],
             'url': video_url,
@@ -59,6 +59,15 @@ class NHLBaseInfoExtractor(InfoExtractor):
             'thumbnail': join(join(video_url, '/u/'), info['bigImage']),
             'upload_date': unified_strdate(info['releaseDate'].split('.')[0]),
         }
+        if video_url.startswith('rtmp:'):
+            mobj = re.match(r'(?P<tc_url>rtmp://[^/]+/(?P<app>[a-z0-9/]+))/(?P<play_path>mp4:.*)', video_url)
+            ret.update({
+                'tc_url': mobj.group('tc_url'),
+                'play_path': mobj.group('play_path'),
+                'app': mobj.group('app'),
+                'no_resume': True,
+            })
+        return ret
 
 
 class NHLIE(NHLBaseInfoExtractor):
@@ -115,6 +124,18 @@ class NHLIE(NHLBaseInfoExtractor):
             'duration': 268,
             'upload_date': '20141122',
         }
+    }, {
+        'url': 'http://video.oilers.nhl.com/videocenter/console?id=691469&catid=4',
+        'info_dict': {
+            'id': '691469',
+            'ext': 'mp4',
+            'title': 'RAW | Craig MacTavish Full Press Conference',
+            'description': 'Oilers GM Craig MacTavish addresses the media at Rexall Place on Friday.',
+            'upload_date': '20141205',
+        },
+        'params': {
+            'skip_download': True,  # Requires rtmpdump
+        }
     }]
 
     def _real_extract(self, url):

From a745475808e125a590afb14df48c565309d3f75c Mon Sep 17 00:00:00 2001
From: Behrooz <behrooz.abbasy@gmail.com>
Date: Fri, 8 May 2015 02:50:46 +0200
Subject: [PATCH 1210/6219] Ir90Tv Add new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/ir90tv.py   | 41 ++++++++++++++++++++++++++++++++
 2 files changed, 42 insertions(+)
 create mode 100644 youtube_dl/extractor/ir90tv.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 5dfa781f8..ee05a6958 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -229,6 +229,7 @@ from .infoq import InfoQIE
 from .instagram import InstagramIE, InstagramUserIE
 from .internetvideoarchive import InternetVideoArchiveIE
 from .iprima import IPrimaIE
+from .ir90tv import Ir90TvIE
 from .ivi import (
     IviIE,
     IviCompilationIE
diff --git a/youtube_dl/extractor/ir90tv.py b/youtube_dl/extractor/ir90tv.py
new file mode 100644
index 000000000..5aa9d6ff4
--- /dev/null
+++ b/youtube_dl/extractor/ir90tv.py
@@ -0,0 +1,41 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class Ir90TvIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?90tv\.ir/video/(?P<id>[0-9]+)/.*'
+    _TEST = {
+        'url': 'http://90tv.ir/video/95719/%D8%B4%D8%A7%DB%8C%D8%B9%D8%A7%D8%AA-%D9%86%D9%82%D9%84-%D9%88-%D8%A7%D9%86%D8%AA%D9%82%D8%A7%D9%84%D8%A7%D8%AA-%D9%85%D9%87%D9%85-%D9%81%D9%88%D8%AA%D8%A8%D8%A7%D9%84-%D8%A7%D8%B1%D9%88%D9%BE%D8%A7-940218',
+        'md5': '411dbd94891381960cb9e13daa47a869',
+        'info_dict': {
+            'id': '95719',
+            'ext': 'mp4',
+            'title': 'شایعات نقل و انتقالات مهم فوتبال اروپا 94/02/18',
+            'thumbnail': 're:^https?://.*\.jpg$',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        # TODO more code goes here, for example ...
+        title = self._html_search_regex(
+            r'<title>\n90tv.ir :: (.*?)</title>', webpage, 'title')
+
+        video_url = self._search_regex(
+            r'<source[^>]+src="([^"]+)"', webpage, 'video url')
+
+        thumbnail = self._search_regex(r'poster="([^"]+)"', webpage, 'thumbnail url')
+        print thumbnail
+
+
+        return {
+            'url': video_url,
+            'id': video_id,
+            'title': title,
+            'video_url' : video_url,
+            'thumbnail' : thumbnail,
+        }
\ No newline at end of file

From 54b31d149e7be08eb7be9981a9eec398d11f17ef Mon Sep 17 00:00:00 2001
From: Behrooz <behrooz.abbasy@gmail.com>
Date: Fri, 8 May 2015 02:55:01 +0200
Subject: [PATCH 1211/6219] Ir90Tv Add new extractor

---
 youtube_dl/extractor/ir90tv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ir90tv.py b/youtube_dl/extractor/ir90tv.py
index 5aa9d6ff4..3a3cb4887 100644
--- a/youtube_dl/extractor/ir90tv.py
+++ b/youtube_dl/extractor/ir90tv.py
@@ -38,4 +38,4 @@ class Ir90TvIE(InfoExtractor):
             'title': title,
             'video_url' : video_url,
             'thumbnail' : thumbnail,
-        }
\ No newline at end of file
+        }

From a650110ba762b2658c64392317c1afd2a284dd3d Mon Sep 17 00:00:00 2001
From: Behrooz <behrooz.abbasy@gmail.com>
Date: Fri, 8 May 2015 04:32:08 +0200
Subject: [PATCH 1212/6219] remove print

---
 youtube_dl/extractor/ir90tv.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/youtube_dl/extractor/ir90tv.py b/youtube_dl/extractor/ir90tv.py
index 3a3cb4887..b79529b1b 100644
--- a/youtube_dl/extractor/ir90tv.py
+++ b/youtube_dl/extractor/ir90tv.py
@@ -21,7 +21,6 @@ class Ir90TvIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        # TODO more code goes here, for example ...
         title = self._html_search_regex(
             r'<title>\n90tv.ir :: (.*?)</title>', webpage, 'title')
 
@@ -29,8 +28,6 @@ class Ir90TvIE(InfoExtractor):
             r'<source[^>]+src="([^"]+)"', webpage, 'video url')
 
         thumbnail = self._search_regex(r'poster="([^"]+)"', webpage, 'thumbnail url')
-        print thumbnail
-
 
         return {
             'url': video_url,

From 541168039d8f3e7680a15cc366fcc94335308d81 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 8 May 2015 11:01:24 +0200
Subject: [PATCH 1213/6219] [utils] get_exe_version: encode executable name
 (fixes #5647)

It failed in python 2.x when $PATH contains a directory with non-ascii characters.
---
 youtube_dl/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 1013f7c18..de09b53b2 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1380,7 +1380,7 @@ def get_exe_version(exe, args=['--version'],
     or False if the executable is not present """
     try:
         out, _ = subprocess.Popen(
-            [exe] + args,
+            [encodeArgument(exe)] + args,
             stdout=subprocess.PIPE, stderr=subprocess.STDOUT).communicate()
     except OSError:
         return False

From 249962ffa2b155795ebfe0a267eb025cd5e30c56 Mon Sep 17 00:00:00 2001
From: blissland <blissland.house@googlemail.com>
Date: Thu, 7 May 2015 16:56:15 +0100
Subject: [PATCH 1214/6219] [bet] Use unique part of xml url as the video id
 and fix tests (closes #5642)

The guid changes often.
---
 youtube_dl/extractor/bet.py | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/bet.py b/youtube_dl/extractor/bet.py
index d2abd4d77..26b934543 100644
--- a/youtube_dl/extractor/bet.py
+++ b/youtube_dl/extractor/bet.py
@@ -16,11 +16,11 @@ class BetIE(InfoExtractor):
         {
             'url': 'http://www.bet.com/news/politics/2014/12/08/in-bet-exclusive-obama-talks-race-and-racism.html',
             'info_dict': {
-                'id': '740ab250-bb94-4a8a-8787-fe0de7c74471',
+                'id': 'news/national/2014/a-conversation-with-president-obama',
                 'display_id': 'in-bet-exclusive-obama-talks-race-and-racism',
                 'ext': 'flv',
-                'title': 'BET News Presents: A Conversation With President Obama',
-                'description': 'md5:5a88d8ae912c1b33e090290af7ec33c6',
+                'title': 'A Conversation With President Obama',
+                'description': 'md5:699d0652a350cf3e491cd15cc745b5da',
                 'duration': 1534,
                 'timestamp': 1418075340,
                 'upload_date': '20141208',
@@ -35,7 +35,7 @@ class BetIE(InfoExtractor):
         {
             'url': 'http://www.bet.com/video/news/national/2014/justice-for-ferguson-a-community-reacts.html',
             'info_dict': {
-                'id': 'bcd1b1df-673a-42cf-8d01-b282db608f2d',
+                'id': 'news/national/2014/justice-for-ferguson-a-community-reacts',
                 'display_id': 'justice-for-ferguson-a-community-reacts',
                 'ext': 'flv',
                 'title': 'Justice for Ferguson: A Community Reacts',
@@ -61,6 +61,9 @@ class BetIE(InfoExtractor):
             [r'mediaURL\s*:\s*"([^"]+)"', r"var\s+mrssMediaUrl\s*=\s*'([^']+)'"],
             webpage, 'media URL'))
 
+        video_id = self._search_regex(
+            r'/video/(.*)/_jcr_content/', media_url, 'video id')
+
         mrss = self._download_xml(media_url, display_id)
 
         item = mrss.find('./channel/item')
@@ -75,8 +78,6 @@ class BetIE(InfoExtractor):
         description = xpath_text(
             item, './description', 'description', fatal=False)
 
-        video_id = xpath_text(item, './guid', 'video id', fatal=False)
-
         timestamp = parse_iso8601(xpath_text(
             item, xpath_with_ns('./dc:date', NS_MAP),
             'upload date', fatal=False))

From 43837189c18af635cfb1cd8fe503265b4b218c32 Mon Sep 17 00:00:00 2001
From: blissland <blissland.house@googlemail.com>
Date: Fri, 8 May 2015 10:40:25 +0100
Subject: [PATCH 1215/6219] Fix URL template extraction for netzkino. Fixes
 #5614

---
 youtube_dl/extractor/netzkino.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/netzkino.py b/youtube_dl/extractor/netzkino.py
index bc17e20aa..0d165a82a 100644
--- a/youtube_dl/extractor/netzkino.py
+++ b/youtube_dl/extractor/netzkino.py
@@ -49,7 +49,7 @@ class NetzkinoIE(InfoExtractor):
             'http://www.netzkino.de/beta/dist/production.min.js', video_id,
             note='Downloading player code')
         avo_js = self._search_regex(
-            r'window\.avoCore\s*=.*?urlTemplate:\s*(\{.*?"\})',
+            r'var urlTemplate=(\{.*?"\})',
             production_js, 'URL templates')
         templates = self._parse_json(
             avo_js, video_id, transform_source=js_to_json)

From d1feb308116f57ceae3888db5e1b93394300f564 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 8 May 2015 20:07:53 +0600
Subject: [PATCH 1216/6219] [mlb] Fallback to extracting video id from webpage
 for all URLs that does not contain it explicitly (Closes #5630)

---
 youtube_dl/extractor/mlb.py | 22 +++++++++++++++++++++-
 1 file changed, 21 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mlb.py b/youtube_dl/extractor/mlb.py
index ee9ff73bf..109eecefd 100644
--- a/youtube_dl/extractor/mlb.py
+++ b/youtube_dl/extractor/mlb.py
@@ -10,7 +10,21 @@ from ..utils import (
 
 
 class MLBIE(InfoExtractor):
-    _VALID_URL = r'https?://m(?:lb)?\.(?:[\da-z_-]+\.)?mlb\.com/(?:(?:.*?/)?video/(?:topic/[\da-z_-]+/)?v|(?:shared/video/embed/(?:embed|m-internal-embed)\.html|[^/]+/video/play\.jsp)\?.*?\bcontent_id=)(?P<id>n?\d+)'
+    _VALID_URL = r'''(?x)
+                    https?://
+                        m(?:lb)?\.(?:[\da-z_-]+\.)?mlb\.com/
+                        (?:
+                            (?:
+                                (?:.*?/)?video/(?:topic/[\da-z_-]+/)?v|
+                                (?:
+                                    shared/video/embed/(?:embed|m-internal-embed)\.html|
+                                    [^/]+/video/play\.jsp
+                                )\?.*?\bcontent_id=
+                            )
+                            (?P<id>n?\d+)|
+                            (?P<path>.+?)
+                        )
+                    '''
     _TESTS = [
         {
             'url': 'http://m.mlb.com/sea/video/topic/51231442/v34698933/nymsea-ackley-robs-a-home-run-with-an-amazing-catch/?c_id=sea',
@@ -95,6 +109,12 @@ class MLBIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
 
+        if not video_id:
+            video_path = mobj.group('path')
+            webpage = self._download_webpage(url, video_path)
+            video_id = self._search_regex(
+                r'data-videoid="(\d+)"', webpage, 'video id')
+
         detail = self._download_xml(
             'http://m.mlb.com/gen/multimedia/detail/%s/%s/%s/%s.xml'
             % (video_id[-3], video_id[-2], video_id[-1], video_id), video_id)

From 4e6e9d21bd62c4e2ab2576347e066891092a5783 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 8 May 2015 21:48:47 +0600
Subject: [PATCH 1217/6219] [mlb] Improve _VALID_URL

---
 youtube_dl/extractor/mlb.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mlb.py b/youtube_dl/extractor/mlb.py
index 109eecefd..4e054fb53 100644
--- a/youtube_dl/extractor/mlb.py
+++ b/youtube_dl/extractor/mlb.py
@@ -22,7 +22,7 @@ class MLBIE(InfoExtractor):
                                 )\?.*?\bcontent_id=
                             )
                             (?P<id>n?\d+)|
-                            (?P<path>.+?)
+                            (?:[^/]+/)*(?P<path>[^/]+)
                         )
                     '''
     _TESTS = [

From 34e7dc81a94d39d48c5b4aac8cddcca46edba94d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 8 May 2015 22:03:03 +0600
Subject: [PATCH 1218/6219] [vgtv] Add support for generic bt.no URLs (#5620)

---
 youtube_dl/extractor/vgtv.py | 20 +++++++++++++++++---
 1 file changed, 17 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/vgtv.py b/youtube_dl/extractor/vgtv.py
index 69dc9a759..b0f0b3bc2 100644
--- a/youtube_dl/extractor/vgtv.py
+++ b/youtube_dl/extractor/vgtv.py
@@ -8,7 +8,8 @@ from ..utils import float_or_none
 
 
 class VGTVIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?vgtv\.no/#!/[^/]+/(?P<id>[0-9]+)'
+    IE_DESC = 'VGTV and BTTV'
+    _VALID_URL = r'http://(?:www\.)?(?P<host>vgtv|bt)\.no/(?:(?:tv/)?#!/(?:video|live)/(?P<id>[0-9]+)|(?:[^/]+/)*(?P<path>[^/]+))'
     _TESTS = [
         {
             # streamType: vod
@@ -64,12 +65,25 @@ class VGTVIE(InfoExtractor):
                 'skip_download': True,
             },
         },
+        {
+            'url': 'http://www.bt.no/tv/#!/video/100250/norling-dette-er-forskjellen-paa-1-divisjon-og-eliteserien',
+            'only_matching': True,
+        },
     ]
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        host = mobj.group('host')
+
+        HOST_WEBSITES = {
+            'vgtv': 'vgtv',
+            'bt': 'bttv',
+        }
+
         data = self._download_json(
-            'http://svp.vg.no/svp/api/v1/vgtv/assets/%s?appName=vgtv-website' % video_id,
+            'http://svp.vg.no/svp/api/v1/%s/assets/%s?appName=%s-website'
+            % (host, video_id, HOST_WEBSITES[host]),
             video_id, 'Downloading media JSON')
 
         streams = data['streamUrls']

From 0ceab8474924c4e7a6e28497c8da40cc5002c8d3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 8 May 2015 22:18:43 +0600
Subject: [PATCH 1219/6219] [vgtv] Add support for bt.no articles (#5620)

---
 youtube_dl/extractor/__init__.py |  5 +++-
 youtube_dl/extractor/vgtv.py     | 39 +++++++++++++++++++++++++++++++-
 2 files changed, 42 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 5dfa781f8..587a45940 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -587,7 +587,10 @@ from .veoh import VeohIE
 from .vessel import VesselIE
 from .vesti import VestiIE
 from .vevo import VevoIE
-from .vgtv import VGTVIE
+from .vgtv import (
+    BTArticleIE,
+    VGTVIE,
+)
 from .vh1 import VH1IE
 from .vice import ViceIE
 from .viddler import ViddlerIE
diff --git a/youtube_dl/extractor/vgtv.py b/youtube_dl/extractor/vgtv.py
index b0f0b3bc2..ad07e54c9 100644
--- a/youtube_dl/extractor/vgtv.py
+++ b/youtube_dl/extractor/vgtv.py
@@ -9,7 +9,18 @@ from ..utils import float_or_none
 
 class VGTVIE(InfoExtractor):
     IE_DESC = 'VGTV and BTTV'
-    _VALID_URL = r'http://(?:www\.)?(?P<host>vgtv|bt)\.no/(?:(?:tv/)?#!/(?:video|live)/(?P<id>[0-9]+)|(?:[^/]+/)*(?P<path>[^/]+))'
+    _VALID_URL = r'''(?x)
+                    (?:
+                        vgtv:|
+                        http://(?:www\.)?
+                    )
+                    (?P<host>vgtv|bt)
+                    (?:
+                        :|
+                        \.no/(?:tv/)?#!/(?:video|live)/
+                    )
+                    (?P<id>[0-9]+)
+                    '''
     _TESTS = [
         {
             # streamType: vod
@@ -129,3 +140,29 @@ class VGTVIE(InfoExtractor):
             'view_count': data['displays'],
             'formats': formats,
         }
+
+
+class BTArticleIE(InfoExtractor):
+    IE_DESC = 'Bergens Tidende'
+    _VALID_URL = 'http://(?:www\.)?bt\.no/(?:[^/]+/)+(?P<id>[^/]+)-\d+\.html'
+    _TEST = {
+        'url': 'http://www.bt.no/nyheter/lokalt/Kjemper-for-internatet-1788214.html',
+        'md5': 'd055e8ee918ef2844745fcfd1a4175fb',
+        'info_dict': {
+            'id': '23199',
+            'ext': 'mp4',
+            'title': 'Alrekstad internat',
+            'description': 'md5:dc81a9056c874fedb62fc48a300dac58',
+            'thumbnail': 're:^https?://.*\.jpg',
+            'duration': 191,
+            'timestamp': 1289991323,
+            'upload_date': '20101117',
+            'view_count': int,
+        },
+    }
+
+    def _real_extract(self, url):
+        webpage = self._download_webpage(url, self._match_id(url))
+        video_id = self._search_regex(
+            r'SVP\.Player\.load\(\s*(\d+)', webpage, 'video id')
+        return self.url_result('vgtv:bt:%s' % video_id, 'VGTV')

From 2c0c9dc46cda490137b6788d6d66f31ca092f58f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 8 May 2015 22:50:01 +0600
Subject: [PATCH 1220/6219] [xstream] Move xstream to separate extractor

---
 youtube_dl/extractor/__init__.py |   1 +
 youtube_dl/extractor/xstream.py  | 115 +++++++++++++++++++++++++++++++
 2 files changed, 116 insertions(+)
 create mode 100644 youtube_dl/extractor/xstream.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 587a45940..5cc35c8eb 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -653,6 +653,7 @@ from .xboxclips import XboxClipsIE
 from .xhamster import XHamsterIE
 from .xminus import XMinusIE
 from .xnxx import XNXXIE
+from .xstream import XstreamIE
 from .xvideos import XVideosIE
 from .xtube import XTubeUserIE, XTubeIE
 from .xuite import XuiteIE
diff --git a/youtube_dl/extractor/xstream.py b/youtube_dl/extractor/xstream.py
new file mode 100644
index 000000000..71584c291
--- /dev/null
+++ b/youtube_dl/extractor/xstream.py
@@ -0,0 +1,115 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    parse_iso8601,
+    xpath_with_ns,
+    xpath_text,
+    find_xpath_attr,
+)
+
+
+class XstreamIE(InfoExtractor):
+    _VALID_URL = r'''(?x)
+                    (?:
+                        xstream:|
+                        https?://frontend\.xstream\.(?:dk|net)/
+                    )
+                    (?P<partner_id>[^/]+)
+                    (?:
+                        :|
+                        /feed/video/\?.*?\bid=
+                    )
+                    (?P<id>\d+)
+                    '''
+    _TESTS = [{
+        'url': 'http://frontend.xstream.dk/btno/feed/video/?platform=web&id=86588',
+        'md5': 'd7d17e3337dc80de6d3a540aefbe441b',
+        'info_dict': {
+            'id': '86588',
+            'ext': 'mov',
+            'title': 'Otto Wollertsen',
+            'description': 'Vestlendingen Otto Fredrik Wollertsen',
+            'timestamp': 1430473209,
+            'upload_date': '20150501',
+        },
+    }, {
+        'url': 'http://frontend.xstream.dk/ap/feed/video/?platform=web&id=21039',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        partner_id = mobj.group('partner_id')
+        video_id = mobj.group('id')
+
+        data = self._download_xml(
+            'http://frontend.xstream.dk/%s/feed/video/?platform=web&id=%s'
+            % (partner_id, video_id),
+            video_id)
+
+        NS_MAP = {
+            'atom': 'http://www.w3.org/2005/Atom',
+            'xt': 'http://xstream.dk/',
+            'media': 'http://search.yahoo.com/mrss/',
+        }
+
+        entry = data.find(xpath_with_ns('./atom:entry', NS_MAP))
+
+        title = xpath_text(
+            entry, xpath_with_ns('./atom:title', NS_MAP), 'title')
+        description = xpath_text(
+            entry, xpath_with_ns('./atom:summary', NS_MAP), 'description')
+        timestamp = parse_iso8601(xpath_text(
+            entry, xpath_with_ns('./atom:published', NS_MAP), 'upload date'))
+
+        formats = []
+        media_group = entry.find(xpath_with_ns('./media:group', NS_MAP))
+        for media_content in media_group.findall(xpath_with_ns('./media:content', NS_MAP)):
+            media_url = media_content.get('url')
+            if not media_url:
+                continue
+            tbr = int_or_none(media_content.get('bitrate'))
+            mobj = re.search(r'^(?P<url>rtmp://[^/]+/(?P<app>[^/]+))/(?P<playpath>.+)$', media_url)
+            if mobj:
+                formats.append({
+                    'url': mobj.group('url'),
+                    'play_path': 'mp4:%s' % mobj.group('playpath'),
+                    'app': mobj.group('app'),
+                    'ext': 'flv',
+                    'tbr': tbr,
+                    'format_id': 'rtmp-%d' % tbr,
+                })
+            else:
+                formats.append({
+                    'url': media_url,
+                    'tbr': tbr,
+                })
+        self._sort_formats(formats)
+
+        link = find_xpath_attr(
+            entry, xpath_with_ns('./atom:link', NS_MAP), 'rel', 'original')
+        if link is not None:
+            formats.append({
+                'url': link.get('href'),
+                'format_id': link.get('rel'),
+            })
+
+        thumbnails = [{
+            'url': splash.get('url'),
+            'width': int_or_none(splash.get('width')),
+            'height': int_or_none(splash.get('height')),
+        } for splash in media_group.findall(xpath_with_ns('./xt:splash', NS_MAP))]
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'timestamp': timestamp,
+            'formats': formats,
+            'thumbnails': thumbnails,
+        }

From cbe443362f91ab111e2a01fe8246e17a98668f88 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 8 May 2015 22:52:20 +0600
Subject: [PATCH 1221/6219] [aftenposten] Implement in terms of xtream
 extractor

---
 youtube_dl/extractor/aftenposten.py | 77 +----------------------------
 1 file changed, 1 insertion(+), 76 deletions(-)

diff --git a/youtube_dl/extractor/aftenposten.py b/youtube_dl/extractor/aftenposten.py
index e15c015fb..0c00acfb5 100644
--- a/youtube_dl/extractor/aftenposten.py
+++ b/youtube_dl/extractor/aftenposten.py
@@ -1,21 +1,11 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
-from ..utils import (
-    int_or_none,
-    parse_iso8601,
-    xpath_with_ns,
-    xpath_text,
-    find_xpath_attr,
-)
 
 
 class AftenpostenIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?aftenposten\.no/webtv/(?:#!/)?video/(?P<id>\d+)'
-
     _TEST = {
         'url': 'http://www.aftenposten.no/webtv/#!/video/21039/trailer-sweatshop-i-can-t-take-any-more',
         'md5': 'fd828cd29774a729bf4d4425fe192972',
@@ -30,69 +20,4 @@ class AftenpostenIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        data = self._download_xml(
-            'http://frontend.xstream.dk/ap/feed/video/?platform=web&id=%s' % video_id, video_id)
-
-        NS_MAP = {
-            'atom': 'http://www.w3.org/2005/Atom',
-            'xt': 'http://xstream.dk/',
-            'media': 'http://search.yahoo.com/mrss/',
-        }
-
-        entry = data.find(xpath_with_ns('./atom:entry', NS_MAP))
-
-        title = xpath_text(
-            entry, xpath_with_ns('./atom:title', NS_MAP), 'title')
-        description = xpath_text(
-            entry, xpath_with_ns('./atom:summary', NS_MAP), 'description')
-        timestamp = parse_iso8601(xpath_text(
-            entry, xpath_with_ns('./atom:published', NS_MAP), 'upload date'))
-
-        formats = []
-        media_group = entry.find(xpath_with_ns('./media:group', NS_MAP))
-        for media_content in media_group.findall(xpath_with_ns('./media:content', NS_MAP)):
-            media_url = media_content.get('url')
-            if not media_url:
-                continue
-            tbr = int_or_none(media_content.get('bitrate'))
-            mobj = re.search(r'^(?P<url>rtmp://[^/]+/(?P<app>[^/]+))/(?P<playpath>.+)$', media_url)
-            if mobj:
-                formats.append({
-                    'url': mobj.group('url'),
-                    'play_path': 'mp4:%s' % mobj.group('playpath'),
-                    'app': mobj.group('app'),
-                    'ext': 'flv',
-                    'tbr': tbr,
-                    'format_id': 'rtmp-%d' % tbr,
-                })
-            else:
-                formats.append({
-                    'url': media_url,
-                    'tbr': tbr,
-                })
-        self._sort_formats(formats)
-
-        link = find_xpath_attr(
-            entry, xpath_with_ns('./atom:link', NS_MAP), 'rel', 'original')
-        if link is not None:
-            formats.append({
-                'url': link.get('href'),
-                'format_id': link.get('rel'),
-            })
-
-        thumbnails = [{
-            'url': splash.get('url'),
-            'width': int_or_none(splash.get('width')),
-            'height': int_or_none(splash.get('height')),
-        } for splash in media_group.findall(xpath_with_ns('./xt:splash', NS_MAP))]
-
-        return {
-            'id': video_id,
-            'title': title,
-            'description': description,
-            'timestamp': timestamp,
-            'formats': formats,
-            'thumbnails': thumbnails,
-        }
+        return self.url_result('xstream:ap:%s' % self._match_id(url), 'Xstream')

From fe373287ebdda002ed84dca1d8b9d6f8a5686138 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 8 May 2015 22:59:50 +0600
Subject: [PATCH 1222/6219] [vgtv] Add support for bt vestlendingen (Closes
 #5620)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/vgtv.py     | 26 +++++++++++++++++++++++++-
 2 files changed, 26 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 5cc35c8eb..96cf28efe 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -589,6 +589,7 @@ from .vesti import VestiIE
 from .vevo import VevoIE
 from .vgtv import (
     BTArticleIE,
+    BTVestlendingenIE,
     VGTVIE,
 )
 from .vh1 import VH1IE
diff --git a/youtube_dl/extractor/vgtv.py b/youtube_dl/extractor/vgtv.py
index ad07e54c9..db7a4bdb1 100644
--- a/youtube_dl/extractor/vgtv.py
+++ b/youtube_dl/extractor/vgtv.py
@@ -143,7 +143,8 @@ class VGTVIE(InfoExtractor):
 
 
 class BTArticleIE(InfoExtractor):
-    IE_DESC = 'Bergens Tidende'
+    IE_NAME = 'bt:article'
+    IE_DESC = 'Bergens Tidende Articles'
     _VALID_URL = 'http://(?:www\.)?bt\.no/(?:[^/]+/)+(?P<id>[^/]+)-\d+\.html'
     _TEST = {
         'url': 'http://www.bt.no/nyheter/lokalt/Kjemper-for-internatet-1788214.html',
@@ -166,3 +167,26 @@ class BTArticleIE(InfoExtractor):
         video_id = self._search_regex(
             r'SVP\.Player\.load\(\s*(\d+)', webpage, 'video id')
         return self.url_result('vgtv:bt:%s' % video_id, 'VGTV')
+
+
+class BTVestlendingenIE(InfoExtractor):
+    IE_NAME = 'bt:vestlendingen'
+    IE_DESC = 'Bergens Tidende - Vestlendingen'
+    _VALID_URL = 'http://(?:www\.)?bt\.no/spesial/vestlendingen/#!/(?P<id>\d+)'
+    _TEST = {
+        'url': 'http://www.bt.no/spesial/vestlendingen/#!/86588',
+        'md5': 'd7d17e3337dc80de6d3a540aefbe441b',
+        'info_dict': {
+            'id': '86588',
+            'ext': 'mov',
+            'title': 'Otto Wollertsen',
+            'description': 'Vestlendingen Otto Fredrik Wollertsen',
+            'timestamp': 1430473209,
+            'upload_date': '20150501',
+        },
+    }
+
+    def _real_extract(self, url):
+        return self.url_result('xstream:btno:%s' % self._match_id(url), 'Xstream')
+
+

From 4384cf9e7d59492141ebd45f77830238097c695c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 8 May 2015 23:04:27 +0600
Subject: [PATCH 1223/6219] [extractor/__init__] Fix alphabetic order

---
 youtube_dl/extractor/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 96cf28efe..7f0070784 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -655,9 +655,9 @@ from .xhamster import XHamsterIE
 from .xminus import XMinusIE
 from .xnxx import XNXXIE
 from .xstream import XstreamIE
-from .xvideos import XVideosIE
 from .xtube import XTubeUserIE, XTubeIE
 from .xuite import XuiteIE
+from .xvideos import XVideosIE
 from .xxxymovies import XXXYMoviesIE
 from .yahoo import (
     YahooIE,

From bb03fdae0d4da9c591c2967044c5e30bf797c22a Mon Sep 17 00:00:00 2001
From: "Sergey M." <dstftw@gmail.com>
Date: Fri, 8 May 2015 23:19:57 +0600
Subject: [PATCH 1224/6219] [README.md] Clarify format selection when streaming
 to stdout

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 9aeb114f3..b6e9429df 100644
--- a/README.md
+++ b/README.md
@@ -269,7 +269,7 @@ The simplest case is requesting a specific format, for example `-f 22`. You can
 
 If you want to download multiple videos and they don't have the same formats available, you can specify the order of preference using slashes, as in `-f 22/17/18`. You can also filter the video results by putting a condition in brackets, as in `-f "best[height=720]"` (or `-f "[filesize>10M]"`).  This works for filesize, height, width, tbr, abr, vbr, asr, and fps and the comparisons <, <=, >, >=, =, != and for ext, acodec, vcodec, container, and protocol and the comparisons =, != . Formats for which the value is not known are excluded unless you put a question mark (?) after the operator. You can combine format filters, so  `-f "[height <=? 720][tbr>500]"` selects up to 720p videos (or videos where the height is not known) with a bitrate of at least 500 KBit/s. Use commas to download multiple formats, such as `-f 136/137/mp4/bestvideo,140/m4a/bestaudio`. You can merge the video and audio of two formats into a single file using `-f <video-format>+<audio-format>` (requires ffmpeg or avconv), for example `-f bestvideo+bestaudio`.
 
-Since the end of April 2015 and version 2015.04.26 youtube-dl uses `-f bestvideo+bestaudio/best` as default format selection (see #5447, #5456). If ffmpeg or avconv are installed this results in downloading `bestvideo` and `bestaudio` separately and muxing them together into a single file giving the best overall quality available. Otherwise it falls back to `best` and results in downloading best available quality served as a single file. `best` is also needed for videos that don't come from YouTube because they don't provide the audio and video in two different files. If you want to only download some dash formats (for example if you are not interested in getting videos with a resolution higher than 1080p), you can add `-f bestvideo[height<=?1080]+bestaudio/best` to your configuration file.
+Since the end of April 2015 and version 2015.04.26 youtube-dl uses `-f bestvideo+bestaudio/best` as default format selection (see #5447, #5456). If ffmpeg or avconv are installed this results in downloading `bestvideo` and `bestaudio` separately and muxing them together into a single file giving the best overall quality available. Otherwise it falls back to `best` and results in downloading best available quality served as a single file. `best` is also needed for videos that don't come from YouTube because they don't provide the audio and video in two different files. If you want to only download some dash formats (for example if you are not interested in getting videos with a resolution higher than 1080p), you can add `-f bestvideo[height<=?1080]+bestaudio/best` to your configuration file. Note that if you use youtube-dl to stream to `stdout` (and most likely to pipe it to your media player then), i.e. you explicitly specify output template as `-o -`, youtube-dl still uses `-f best` format selection in order to start content delivery immediately to your player and not to wait until `bestvideo` and `bestaudio` are downloaded an muxed.
 
 If you want to preserve the old format selection behavior (prior to youtube-dl 2015.04.26), i.e. you want to download best available quality media served as a single file, you should explicitly specify your choice with `-f best`. You may want to add it to the [configuration file](#configuration) in order not to type it every time you run youtube-dl.
 

From 50b901306406d5c37f31880860e2a4dbb5e0a165 Mon Sep 17 00:00:00 2001
From: "Sergey M." <dstftw@gmail.com>
Date: Fri, 8 May 2015 23:21:23 +0600
Subject: [PATCH 1225/6219] [README.md] Fix typo

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index b6e9429df..3d9436456 100644
--- a/README.md
+++ b/README.md
@@ -269,7 +269,7 @@ The simplest case is requesting a specific format, for example `-f 22`. You can
 
 If you want to download multiple videos and they don't have the same formats available, you can specify the order of preference using slashes, as in `-f 22/17/18`. You can also filter the video results by putting a condition in brackets, as in `-f "best[height=720]"` (or `-f "[filesize>10M]"`).  This works for filesize, height, width, tbr, abr, vbr, asr, and fps and the comparisons <, <=, >, >=, =, != and for ext, acodec, vcodec, container, and protocol and the comparisons =, != . Formats for which the value is not known are excluded unless you put a question mark (?) after the operator. You can combine format filters, so  `-f "[height <=? 720][tbr>500]"` selects up to 720p videos (or videos where the height is not known) with a bitrate of at least 500 KBit/s. Use commas to download multiple formats, such as `-f 136/137/mp4/bestvideo,140/m4a/bestaudio`. You can merge the video and audio of two formats into a single file using `-f <video-format>+<audio-format>` (requires ffmpeg or avconv), for example `-f bestvideo+bestaudio`.
 
-Since the end of April 2015 and version 2015.04.26 youtube-dl uses `-f bestvideo+bestaudio/best` as default format selection (see #5447, #5456). If ffmpeg or avconv are installed this results in downloading `bestvideo` and `bestaudio` separately and muxing them together into a single file giving the best overall quality available. Otherwise it falls back to `best` and results in downloading best available quality served as a single file. `best` is also needed for videos that don't come from YouTube because they don't provide the audio and video in two different files. If you want to only download some dash formats (for example if you are not interested in getting videos with a resolution higher than 1080p), you can add `-f bestvideo[height<=?1080]+bestaudio/best` to your configuration file. Note that if you use youtube-dl to stream to `stdout` (and most likely to pipe it to your media player then), i.e. you explicitly specify output template as `-o -`, youtube-dl still uses `-f best` format selection in order to start content delivery immediately to your player and not to wait until `bestvideo` and `bestaudio` are downloaded an muxed.
+Since the end of April 2015 and version 2015.04.26 youtube-dl uses `-f bestvideo+bestaudio/best` as default format selection (see #5447, #5456). If ffmpeg or avconv are installed this results in downloading `bestvideo` and `bestaudio` separately and muxing them together into a single file giving the best overall quality available. Otherwise it falls back to `best` and results in downloading best available quality served as a single file. `best` is also needed for videos that don't come from YouTube because they don't provide the audio and video in two different files. If you want to only download some dash formats (for example if you are not interested in getting videos with a resolution higher than 1080p), you can add `-f bestvideo[height<=?1080]+bestaudio/best` to your configuration file. Note that if you use youtube-dl to stream to `stdout` (and most likely to pipe it to your media player then), i.e. you explicitly specify output template as `-o -`, youtube-dl still uses `-f best` format selection in order to start content delivery immediately to your player and not to wait until `bestvideo` and `bestaudio` are downloaded and muxed.
 
 If you want to preserve the old format selection behavior (prior to youtube-dl 2015.04.26), i.e. you want to download best available quality media served as a single file, you should explicitly specify your choice with `-f best`. You may want to add it to the [configuration file](#configuration) in order not to type it every time you run youtube-dl.
 

From 79998cd5afb2e16fe14cebdbec81b21c45c24c32 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 9 May 2015 00:12:42 +0600
Subject: [PATCH 1226/6219] [svtplay] Generalize svt extractors and add svt.se
 extractor

---
 youtube_dl/extractor/__init__.py |  5 +-
 youtube_dl/extractor/svtplay.py  | 98 ++++++++++++++++++++------------
 2 files changed, 67 insertions(+), 36 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 7f0070784..79236c6c1 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -505,7 +505,10 @@ from .streamcloud import StreamcloudIE
 from .streamcz import StreamCZIE
 from .streetvoice import StreetVoiceIE
 from .sunporno import SunPornoIE
-from .svtplay import SVTPlayIE
+from .svtplay import (
+    SVTIE,
+    SVTPlayIE,
+)
 from .swrmediathek import SWRMediathekIE
 from .syfy import SyfyIE
 from .sztvhu import SztvHuIE
diff --git a/youtube_dl/extractor/svtplay.py b/youtube_dl/extractor/svtplay.py
index 433dfd1cb..732f02048 100644
--- a/youtube_dl/extractor/svtplay.py
+++ b/youtube_dl/extractor/svtplay.py
@@ -9,41 +9,9 @@ from ..utils import (
 )
 
 
-class SVTPlayIE(InfoExtractor):
-    IE_DESC = 'SVT Play and Öppet arkiv'
-    _VALID_URL = r'https?://(?:www\.)?(?P<host>svtplay|oppetarkiv)\.se/video/(?P<id>[0-9]+)'
-    _TESTS = [{
-        'url': 'http://www.svtplay.se/video/2609989/sm-veckan/sm-veckan-rally-final-sasong-1-sm-veckan-rally-final',
-        'md5': 'ade3def0643fa1c40587a422f98edfd9',
-        'info_dict': {
-            'id': '2609989',
-            'ext': 'flv',
-            'title': 'SM veckan vinter, Örebro - Rally, final',
-            'duration': 4500,
-            'thumbnail': 're:^https?://.*[\.-]jpg$',
-            'age_limit': 0,
-        },
-    }, {
-        'url': 'http://www.oppetarkiv.se/video/1058509/rederiet-sasong-1-avsnitt-1-av-318',
-        'md5': 'c3101a17ce9634f4c1f9800f0746c187',
-        'info_dict': {
-            'id': '1058509',
-            'ext': 'flv',
-            'title': 'Farlig kryssning',
-            'duration': 2566,
-            'thumbnail': 're:^https?://.*[\.-]jpg$',
-            'age_limit': 0,
-        },
-        'skip': 'Only works from Sweden',
-    }]
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        host = mobj.group('host')
-
-        info = self._download_json(
-            'http://www.%s.se/video/%s?output=json' % (host, video_id), video_id)
+class SVTBaseIE(InfoExtractor):
+    def _extract_video(self, url, video_id):
+        info = self._download_json(url, video_id)
 
         title = info['context']['title']
         thumbnail = info['context'].get('thumbnailImage')
@@ -80,3 +48,63 @@ class SVTPlayIE(InfoExtractor):
             'duration': duration,
             'age_limit': age_limit,
         }
+
+
+class SVTIE(SVTBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?svt\.se/wd\?(?:.*?&)?widgetId=(?P<widget_id>\d+)&.*?\barticleId=(?P<id>\d+)'
+    _TEST = {
+        'url': 'http://www.svt.se/wd?widgetId=23991&sectionId=541&articleId=2900353&type=embed&contextSectionId=123&autostart=false',
+        'md5': '9648197555fc1b49e3dc22db4af51d46',
+        'info_dict': {
+            'id': '2900353',
+            'ext': 'flv',
+            'title': 'Här trycker Jagr till Giroux (under SVT-intervjun)',
+            'duration': 27,
+            'age_limit': 0,
+        },
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        widget_id = mobj.group('widget_id')
+        article_id = mobj.group('id')
+        return self._extract_video(
+            'http://www.svt.se/wd?widgetId=%s&articleId=%s&format=json&type=embed&output=json' % (widget_id, article_id),
+            article_id)
+
+
+class SVTPlayIE(SVTBaseIE):
+    IE_DESC = 'SVT Play and Öppet arkiv'
+    _VALID_URL = r'https?://(?:www\.)?(?P<host>svtplay|oppetarkiv)\.se/video/(?P<id>[0-9]+)'
+    _TESTS = [{
+        'url': 'http://www.svtplay.se/video/2609989/sm-veckan/sm-veckan-rally-final-sasong-1-sm-veckan-rally-final',
+        'md5': 'ade3def0643fa1c40587a422f98edfd9',
+        'info_dict': {
+            'id': '2609989',
+            'ext': 'flv',
+            'title': 'SM veckan vinter, Örebro - Rally, final',
+            'duration': 4500,
+            'thumbnail': 're:^https?://.*[\.-]jpg$',
+            'age_limit': 0,
+        },
+    }, {
+        'url': 'http://www.oppetarkiv.se/video/1058509/rederiet-sasong-1-avsnitt-1-av-318',
+        'md5': 'c3101a17ce9634f4c1f9800f0746c187',
+        'info_dict': {
+            'id': '1058509',
+            'ext': 'flv',
+            'title': 'Farlig kryssning',
+            'duration': 2566,
+            'thumbnail': 're:^https?://.*[\.-]jpg$',
+            'age_limit': 0,
+        },
+        'skip': 'Only works from Sweden',
+    }]
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        host = mobj.group('host')
+        return self._extract_video(
+            'http://www.%s.se/video/%s?output=json' % (host, video_id),
+            video_id)

From 322915014f0378e2675a2a17cd67fe89a6e6a7d7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 9 May 2015 00:13:40 +0600
Subject: [PATCH 1227/6219] [svtplay] Rename to svt

---
 youtube_dl/extractor/__init__.py            | 2 +-
 youtube_dl/extractor/{svtplay.py => svt.py} | 0
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename youtube_dl/extractor/{svtplay.py => svt.py} (100%)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 79236c6c1..0a18dba5c 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -505,7 +505,7 @@ from .streamcloud import StreamcloudIE
 from .streamcz import StreamCZIE
 from .streetvoice import StreetVoiceIE
 from .sunporno import SunPornoIE
-from .svtplay import (
+from .svt import (
     SVTIE,
     SVTPlayIE,
 )
diff --git a/youtube_dl/extractor/svtplay.py b/youtube_dl/extractor/svt.py
similarity index 100%
rename from youtube_dl/extractor/svtplay.py
rename to youtube_dl/extractor/svt.py

From bab19a8e91153705d6600fe1d1a0b0aa0bf93bb3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 9 May 2015 00:23:35 +0600
Subject: [PATCH 1228/6219] [extractor/generic] Add support for svt embeds
 (Closes #5622)

---
 youtube_dl/extractor/generic.py | 6 ++++++
 youtube_dl/extractor/svt.py     | 7 +++++++
 2 files changed, 13 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index cd7c47d6d..046bcb0f0 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -37,6 +37,7 @@ from .condenast import CondeNastIE
 from .udn import UDNEmbedIE
 from .senateisvp import SenateISVPIE
 from .bliptv import BlipTVIE
+from .svt import SVTIE
 
 
 class GenericIE(InfoExtractor):
@@ -1091,6 +1092,11 @@ class GenericIE(InfoExtractor):
         if bliptv_url:
             return self.url_result(bliptv_url, 'BlipTV')
 
+        # Look for SVT player
+        svt_url = SVTIE._extract_url(webpage)
+        if svt_url:
+            return self.url_result(svt_url, 'SVT')
+
         # Look for embedded condenast player
         matches = re.findall(
             r'<iframe\s+(?:[a-zA-Z-]+="[^"]+"\s+)*?src="(https?://player\.cnevids\.com/embed/[^"]+")',
diff --git a/youtube_dl/extractor/svt.py b/youtube_dl/extractor/svt.py
index 732f02048..fc20f664b 100644
--- a/youtube_dl/extractor/svt.py
+++ b/youtube_dl/extractor/svt.py
@@ -64,6 +64,13 @@ class SVTIE(SVTBaseIE):
         },
     }
 
+    @staticmethod
+    def _extract_url(webpage):
+        mobj = re.search(
+            r'(?:<iframe src|href)="(?P<url>%s[^"]*)"' % SVTIE._VALID_URL, webpage)
+        if mobj:
+            return mobj.group('url')
+
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         widget_id = mobj.group('widget_id')

From dc455a5f88d6bb9bd5a44352e2b67ef7dd5fd683 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 9 May 2015 00:27:37 +0600
Subject: [PATCH 1229/6219] [extractor/generic] Add test for svt embed

---
 youtube_dl/extractor/generic.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 046bcb0f0..7c38bce7c 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -659,6 +659,17 @@ class GenericIE(InfoExtractor):
                 'title': 'Facebook Creates "On This Day" | Crunch Report',
             },
         },
+        # SVT embed
+        {
+            'url': 'http://www.svt.se/sport/ishockey/jagr-tacklar-giroux-under-intervjun',
+            'info_dict': {
+                'id': '2900353',
+                'ext': 'flv',
+                'title': 'Här trycker Jagr till Giroux (under SVT-intervjun)',
+                'duration': 27,
+                'age_limit': 0,
+            },
+        },
         # RSS feed with enclosure
         {
             'url': 'http://podcastfeeds.nbcnews.com/audio/podcast/MSNBC-MADDOW-NETCAST-M4V.xml',

From de765f6c3188802bb2dea704a645f539fa61c8aa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 9 May 2015 02:15:51 +0600
Subject: [PATCH 1230/6219] [foxsports] Support some more URLs (#5611)

---
 youtube_dl/extractor/foxsports.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/foxsports.py b/youtube_dl/extractor/foxsports.py
index 363866b64..df7665176 100644
--- a/youtube_dl/extractor/foxsports.py
+++ b/youtube_dl/extractor/foxsports.py
@@ -5,7 +5,7 @@ from ..utils import smuggle_url
 
 
 class FoxSportsIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?foxsports\.com/video\?vid=(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?foxsports\.com/(?:[^/]+/)*(?P<id>[^/]+)'
 
     _TEST = {
         'url': 'http://www.foxsports.com/video?vid=432609859715',

From 3dbec410a0e195036025aa3a3792932783f371d2 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 9 May 2015 13:19:54 +0800
Subject: [PATCH 1231/6219] [sohu] Enhance error handling

---
 youtube_dl/extractor/sohu.py | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/sohu.py b/youtube_dl/extractor/sohu.py
index f8a4840f7..13b9e9133 100644
--- a/youtube_dl/extractor/sohu.py
+++ b/youtube_dl/extractor/sohu.py
@@ -8,7 +8,10 @@ from ..compat import (
     compat_str,
     compat_urllib_request
 )
-from ..utils import sanitize_url_path_consecutive_slashes
+from ..utils import (
+    sanitize_url_path_consecutive_slashes,
+    ExtractorError,
+)
 
 
 class SohuIE(InfoExtractor):
@@ -117,6 +120,15 @@ class SohuIE(InfoExtractor):
             r'var vid ?= ?["\'](\d+)["\']',
             webpage, 'video path')
         vid_data = _fetch_data(vid, mytv)
+        if vid_data['play'] != 1:
+            if vid_data.get('status') == 12:
+                raise ExtractorError(
+                    'Sohu said: There\'s something wrong in the video.',
+                    expected=True)
+            else:
+                raise ExtractorError(
+                    'Sohu said: The video is only licensed to users in Mainland China.',
+                    expected=True)
 
         formats_json = {}
         for format_id in ('nor', 'high', 'super', 'ori', 'h2644k', 'h2654k'):

From 32060c6d6b618fa858b2ce43db34d02fd43bc542 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 9 May 2015 13:54:28 +0800
Subject: [PATCH 1232/6219] [sohu] Update extractor

The original extraction logic always fails for all test videos
---
 youtube_dl/extractor/sohu.py | 44 ++++++++++++++++--------------------
 1 file changed, 19 insertions(+), 25 deletions(-)

diff --git a/youtube_dl/extractor/sohu.py b/youtube_dl/extractor/sohu.py
index 13b9e9133..eab4adfca 100644
--- a/youtube_dl/extractor/sohu.py
+++ b/youtube_dl/extractor/sohu.py
@@ -8,10 +8,7 @@ from ..compat import (
     compat_str,
     compat_urllib_request
 )
-from ..utils import (
-    sanitize_url_path_consecutive_slashes,
-    ExtractorError,
-)
+from ..utils import ExtractorError
 
 
 class SohuIE(InfoExtractor):
@@ -31,7 +28,7 @@ class SohuIE(InfoExtractor):
         }
     }, {
         'url': 'http://tv.sohu.com/20150305/n409385080.shtml',
-        'md5': '699060e75cf58858dd47fb9c03c42cfb',
+        'md5': 'ac9a5d322b4bf9ae184d53e4711e4f1a',
         'info_dict': {
             'id': '409385080',
             'ext': 'mp4',
@@ -39,7 +36,7 @@ class SohuIE(InfoExtractor):
         }
     }, {
         'url': 'http://my.tv.sohu.com/us/232799889/78693464.shtml',
-        'md5': '9bf34be48f2f4dadcb226c74127e203c',
+        'md5': '49308ff6dafde5ece51137d04aec311e',
         'info_dict': {
             'id': '78693464',
             'ext': 'mp4',
@@ -53,7 +50,7 @@ class SohuIE(InfoExtractor):
             'title': '【神探苍实战秘籍】第13期 战争之影 赫卡里姆',
         },
         'playlist': [{
-            'md5': 'bdbfb8f39924725e6589c146bc1883ad',
+            'md5': '492923eac023ba2f13ff69617c32754a',
             'info_dict': {
                 'id': '78910339_part1',
                 'ext': 'mp4',
@@ -61,7 +58,7 @@ class SohuIE(InfoExtractor):
                 'title': '【神探苍实战秘籍】第13期 战争之影 赫卡里姆',
             }
         }, {
-            'md5': '3e1f46aaeb95354fd10e7fca9fc1804e',
+            'md5': 'de604848c0e8e9c4a4dde7e1347c0637',
             'info_dict': {
                 'id': '78910339_part2',
                 'ext': 'mp4',
@@ -69,7 +66,7 @@ class SohuIE(InfoExtractor):
                 'title': '【神探苍实战秘籍】第13期 战争之影 赫卡里姆',
             }
         }, {
-            'md5': '8407e634175fdac706766481b9443450',
+            'md5': '93584716ee0657c0b205b8aa3d27aa13',
             'info_dict': {
                 'id': '78910339_part3',
                 'ext': 'mp4',
@@ -144,24 +141,21 @@ class SohuIE(InfoExtractor):
         for i in range(part_count):
             formats = []
             for format_id, format_data in formats_json.items():
-                allot = format_data['allot']
-                prot = format_data['prot']
-
                 data = format_data['data']
-                clips_url = data['clipsURL']
-                su = data['su']
 
-                part_str = self._download_webpage(
-                    'http://%s/?prot=%s&file=%s&new=%s' %
-                    (allot, prot, clips_url[i], su[i]),
-                    video_id,
-                    'Downloading %s video URL part %d of %d'
-                    % (format_id, i + 1, part_count))
-
-                part_info = part_str.split('|')
-
-                video_url = sanitize_url_path_consecutive_slashes(
-                    '%s%s?key=%s' % (part_info[0], su[i], part_info[3]))
+                # URLs starts with http://newflv.sohu.ccgslb.net/ is not usable
+                # so retry until got a working URL
+                video_url = 'newflv.sohu.ccgslb.net'
+                retries = 0
+                while 'newflv.sohu.ccgslb.net' in video_url and retries < 5:
+                    download_note = 'Download information from CDN gateway for format ' + format_id
+                    if retries > 0:
+                        download_note += ' (retry #%d)' % retries
+                    retries += 1
+                    cdn_info = self._download_json(
+                        'http://data.vod.itc.cn/cdnList?new=' + data['su'][i],
+                        video_id, download_note)
+                    video_url = cdn_info['url']
 
                 formats.append({
                     'url': video_url,

From 6d14d08e062ff3d6e0fd17f04cb341099097902c Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 9 May 2015 17:36:07 +0800
Subject: [PATCH 1233/6219] [yam] Fix title and uploader id

---
 youtube_dl/extractor/yam.py | 26 ++++++++++++++++++++++----
 1 file changed, 22 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/yam.py b/youtube_dl/extractor/yam.py
index 19f8762ae..9d851bae3 100644
--- a/youtube_dl/extractor/yam.py
+++ b/youtube_dl/extractor/yam.py
@@ -9,6 +9,7 @@ from ..utils import (
     float_or_none,
     month_by_abbreviation,
     ExtractorError,
+    get_element_by_attribute,
 )
 
 
@@ -23,6 +24,7 @@ class YamIE(InfoExtractor):
             'id': '2283921',
             'ext': 'mp3',
             'title': '發現 - 趙薇 京華煙雲主題曲',
+            'description': '發現 - 趙薇 京華煙雲主題曲',
             'uploader_id': 'princekt',
             'upload_date': '20080807',
             'duration': 313.0,
@@ -55,6 +57,17 @@ class YamIE(InfoExtractor):
             'ext': 'mp4',
         },
         'skip': 'invalid YouTube URL',
+    }, {
+        'url': 'http://mymedia.yam.com/m/2373534',
+        'md5': '7ff74b91b7a817269d83796f8c5890b1',
+        'info_dict': {
+            'id': '2373534',
+            'ext': 'mp3',
+            'title': '林俊傑&蔡卓妍-小酒窩',
+            'description': 'md5:904003395a0fcce6cfb25028ff468420',
+            'upload_date': '20080928',
+            'uploader_id': 'onliner2',
+        }
     }]
 
     def _real_extract(self, url):
@@ -75,15 +88,19 @@ class YamIE(InfoExtractor):
         if youtube_url:
             return self.url_result(youtube_url, 'Youtube')
 
+        title = self._html_search_regex(
+            r'<h1[^>]+class="heading"[^>]*>\s*(.+)\s*</h1>', page, 'title')
+
         api_page = self._download_webpage(
             'http://mymedia.yam.com/api/a/?pID=' + video_id, video_id,
             note='Downloading API page')
         api_result_obj = compat_urlparse.parse_qs(api_page)
 
+        info_table = get_element_by_attribute('class', 'info', page)
         uploader_id = self._html_search_regex(
-            r'<!-- 發表作者 -->：[\n ]+<a href="/([a-z]+)"',
-            page, 'uploader id', fatal=False)
-        mobj = re.search(r'<!-- 發表於 -->(?P<mon>[A-Z][a-z]{2})  ' +
+            r'<!-- 發表作者 -->：[\n ]+<a href="/([a-z0-9]+)"',
+            info_table, 'uploader id', fatal=False)
+        mobj = re.search(r'<!-- 發表於 -->(?P<mon>[A-Z][a-z]{2})\s+' +
                          r'(?P<day>\d{1,2}), (?P<year>\d{4})', page)
         if mobj:
             upload_date = '%s%02d%02d' % (
@@ -97,7 +114,8 @@ class YamIE(InfoExtractor):
         return {
             'id': video_id,
             'url': api_result_obj['mp3file'][0],
-            'title': self._html_search_meta('description', page),
+            'title': title,
+            'description': self._html_search_meta('description', page),
             'duration': duration,
             'uploader_id': uploader_id,
             'upload_date': upload_date,

From d39e0f05db226ef5691f5730d40da796aec6bac6 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 9 May 2015 17:37:39 +0800
Subject: [PATCH 1234/6219] [utils] Remove
 sanitize_url_path_consecutive_slashes()

This function is used only in SohuIE, which is updated to use a new
extraction logic.
---
 test/test_utils.py  | 21 ---------------------
 youtube_dl/utils.py |  7 -------
 2 files changed, 28 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 032d3656a..86b110a7d 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -40,7 +40,6 @@ from youtube_dl.utils import (
     read_batch_urls,
     sanitize_filename,
     sanitize_path,
-    sanitize_url_path_consecutive_slashes,
     prepend_extension,
     replace_extension,
     shell_quote,
@@ -176,26 +175,6 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(sanitize_path('./abc'), 'abc')
         self.assertEqual(sanitize_path('./../abc'), '..\\abc')
 
-    def test_sanitize_url_path_consecutive_slashes(self):
-        self.assertEqual(
-            sanitize_url_path_consecutive_slashes('http://hostname/foo//bar/filename.html'),
-            'http://hostname/foo/bar/filename.html')
-        self.assertEqual(
-            sanitize_url_path_consecutive_slashes('http://hostname//foo/bar/filename.html'),
-            'http://hostname/foo/bar/filename.html')
-        self.assertEqual(
-            sanitize_url_path_consecutive_slashes('http://hostname//'),
-            'http://hostname/')
-        self.assertEqual(
-            sanitize_url_path_consecutive_slashes('http://hostname/foo/bar/filename.html'),
-            'http://hostname/foo/bar/filename.html')
-        self.assertEqual(
-            sanitize_url_path_consecutive_slashes('http://hostname/'),
-            'http://hostname/')
-        self.assertEqual(
-            sanitize_url_path_consecutive_slashes('http://hostname/abc//'),
-            'http://hostname/abc/')
-
     def test_prepend_extension(self):
         self.assertEqual(prepend_extension('abc.ext', 'temp'), 'abc.temp.ext')
         self.assertEqual(prepend_extension('abc.ext', 'temp', 'ext'), 'abc.temp.ext')
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index de09b53b2..d73efcf25 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -327,13 +327,6 @@ def sanitize_path(s):
     return os.path.join(*sanitized_path)
 
 
-def sanitize_url_path_consecutive_slashes(url):
-    """Collapses consecutive slashes in URLs' path"""
-    parsed_url = list(compat_urlparse.urlparse(url))
-    parsed_url[2] = re.sub(r'/{2,}', '/', parsed_url[2])
-    return compat_urlparse.urlunparse(parsed_url)
-
-
 def orderedSet(iterable):
     """ Remove all duplicates from the input iterable """
     res = []

From 5c0b2c16a80c509dbcee48f48da3de0bf9912cda Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 9 May 2015 12:34:45 +0200
Subject: [PATCH 1235/6219] [vgtv] Escape '#' in _VALID_URL and remove empty
 newlines at the end

In verbose mode, '#' is interpreted as the start of a comment.
---
 youtube_dl/extractor/vgtv.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/vgtv.py b/youtube_dl/extractor/vgtv.py
index db7a4bdb1..eb2652fb1 100644
--- a/youtube_dl/extractor/vgtv.py
+++ b/youtube_dl/extractor/vgtv.py
@@ -17,7 +17,7 @@ class VGTVIE(InfoExtractor):
                     (?P<host>vgtv|bt)
                     (?:
                         :|
-                        \.no/(?:tv/)?#!/(?:video|live)/
+                        \.no/(?:tv/)?\#!/(?:video|live)/
                     )
                     (?P<id>[0-9]+)
                     '''
@@ -188,5 +188,3 @@ class BTVestlendingenIE(InfoExtractor):
 
     def _real_extract(self, url):
         return self.url_result('xstream:btno:%s' % self._match_id(url), 'Xstream')
-
-

From 3b5f65a64c06859cdee0b93f319c80d5c116cedd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 9 May 2015 12:41:56 +0200
Subject: [PATCH 1236/6219] [mlb] Fix extraction of articles

And move test from generic, since it's directly handled by MLBIE
---
 youtube_dl/extractor/generic.py | 13 -------------
 youtube_dl/extractor/mlb.py     | 14 +++++++++++++-
 2 files changed, 13 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 7c38bce7c..3d756e848 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -414,19 +414,6 @@ class GenericIE(InfoExtractor):
                 'thumbnail': 're:^https?://.*\.jpg$',
             },
         },
-        # MLB articles
-        {
-            'url': 'http://m.mlb.com/news/article/118550098/blue-jays-kevin-pillar-goes-spidey-up-the-wall-to-rob-tim-beckham-of-a-homer',
-            'md5': 'b190e70141fb9a1552a85426b4da1b5d',
-            'info_dict': {
-                'id': '75609783',
-                'ext': 'mp4',
-                'title': 'Must C: Pillar climbs for catch',
-                'description': '4/15/15: Blue Jays outfielder Kevin Pillar continues his defensive dominance by climbing the wall in left to rob Tim Beckham of a home run',
-                'timestamp': 1429124820,
-                'upload_date': '20150415',
-            }
-        },
         # Wistia embed
         {
             'url': 'http://education-portal.com/academy/lesson/north-american-exploration-failed-colonies-of-spain-france-england.html#lesson',
diff --git a/youtube_dl/extractor/mlb.py b/youtube_dl/extractor/mlb.py
index 4e054fb53..40c9ecb35 100644
--- a/youtube_dl/extractor/mlb.py
+++ b/youtube_dl/extractor/mlb.py
@@ -82,6 +82,18 @@ class MLBIE(InfoExtractor):
                 'thumbnail': 're:^https?://.*\.jpg$',
             },
         },
+        {
+            'url': 'http://m.mlb.com/news/article/118550098/blue-jays-kevin-pillar-goes-spidey-up-the-wall-to-rob-tim-beckham-of-a-homer',
+            'md5': 'b190e70141fb9a1552a85426b4da1b5d',
+            'info_dict': {
+                'id': '75609783',
+                'ext': 'mp4',
+                'title': 'Must C: Pillar climbs for catch',
+                'description': '4/15/15: Blue Jays outfielder Kevin Pillar continues his defensive dominance by climbing the wall in left to rob Tim Beckham of a home run',
+                'timestamp': 1429124820,
+                'upload_date': '20150415',
+            }
+        },
         {
             'url': 'http://m.mlb.com/shared/video/embed/embed.html?content_id=35692085&topic_id=6479266&width=400&height=224&property=mlb',
             'only_matching': True,
@@ -113,7 +125,7 @@ class MLBIE(InfoExtractor):
             video_path = mobj.group('path')
             webpage = self._download_webpage(url, video_path)
             video_id = self._search_regex(
-                r'data-videoid="(\d+)"', webpage, 'video id')
+                r'data-video-?id="(\d+)"', webpage, 'video id')
 
         detail = self._download_xml(
             'http://m.mlb.com/gen/multimedia/detail/%s/%s/%s/%s.xml'

From d592b42f5c37c4b0d9f1587dd76f225b0287959f Mon Sep 17 00:00:00 2001
From: blissland <blissland.house@googlemail.com>
Date: Sat, 9 May 2015 15:26:00 +0100
Subject: [PATCH 1237/6219] Updated two tests for BRIE

---
 youtube_dl/extractor/br.py | 30 +++++++++++++++---------------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/br.py b/youtube_dl/extractor/br.py
index 45ba51732..04a3ecd17 100644
--- a/youtube_dl/extractor/br.py
+++ b/youtube_dl/extractor/br.py
@@ -16,27 +16,27 @@ class BRIE(InfoExtractor):
 
     _TESTS = [
         {
-            'url': 'http://www.br.de/mediathek/video/sendungen/heimatsound/heimatsound-festival-2014-trailer-100.html',
-            'md5': '93556dd2bcb2948d9259f8670c516d59',
+            'url': 'http://www.br.de/mediathek/video/sendungen/abendschau/betriebliche-altersvorsorge-104.html',
+            'md5': '83a0477cf0b8451027eb566d88b51106',
             'info_dict': {
-                'id': '25e279aa-1ffd-40fd-9955-5325bd48a53a',
+                'id': '48f656ef-287e-486f-be86-459122db22cc',
                 'ext': 'mp4',
-                'title': 'Wenn das Traditions-Theater wackelt',
-                'description': 'Heimatsound-Festival 2014: Wenn das Traditions-Theater wackelt',
-                'duration': 34,
-                'uploader': 'BR',
-                'upload_date': '20140802',
+                'title': 'Die böse Überraschung',
+                'description': 'Betriebliche Altersvorsorge: Die böse Überraschung',
+                'duration': 180,
+                'uploader': 'Reinhard Weber',
+                'upload_date': '20150422',
             }
         },
         {
-            'url': 'http://www.br.de/nachrichten/schaeuble-haushaltsentwurf-bundestag-100.html',
-            'md5': '3db0df1a9a9cd9fa0c70e6ea8aa8e820',
+            'url': 'http://www.br.de/nachrichten/oberbayern/inhalt/muenchner-polizeipraesident-schreiber-gestorben-100.html',
+            'md5': 'a44396d73ab6a68a69a568fae10705bb',
             'info_dict': {
-                'id': 'c6aae3de-2cf9-43f2-957f-f17fef9afaab',
-                'ext': 'aac',
-                'title': '"Keine neuen Schulden im nächsten Jahr"',
-                'description': 'Haushaltsentwurf: "Keine neuen Schulden im nächsten Jahr"',
-                'duration': 64,
+                'id': 'a4b83e34-123d-4b81-9f4e-c0d3121a4e05',
+                'ext': 'mp4',
+                'title': 'Manfred Schreiber ist tot',
+                'description': 'Abendschau kompakt: Manfred Schreiber ist tot',
+                'duration': 26,
             }
         },
         {

From 0892090a56e04726175c247d13ecce7f6c9cb839 Mon Sep 17 00:00:00 2001
From: blissland <blissland.house@googlemail.com>
Date: Sat, 9 May 2015 16:02:07 +0100
Subject: [PATCH 1238/6219] Added audio test for BRIE

---
 youtube_dl/extractor/br.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/youtube_dl/extractor/br.py b/youtube_dl/extractor/br.py
index 04a3ecd17..66e394e10 100644
--- a/youtube_dl/extractor/br.py
+++ b/youtube_dl/extractor/br.py
@@ -39,6 +39,17 @@ class BRIE(InfoExtractor):
                 'duration': 26,
             }
         },
+        {
+            'url': 'http://www.br.de/radio/br-klassik/sendungen/allegro/premiere-urauffuehrung-the-land-2015-dance-festival-muenchen-100.html',
+            'md5': '8b5b27c0b090f3b35eac4ab3f7a73d3d',
+            'info_dict': {
+                'id': '74c603c9-26d3-48bb-b85b-079aeed66e0b',
+                'ext': 'aac',
+                'title': 'Kurzweilig und sehr bewegend',
+                'description': '"The Land" von Peeping Tom: Kurzweilig und sehr bewegend',
+                'duration': 296,
+            }
+        },
         {
             'url': 'http://www.br.de/radio/bayern1/service/team/videos/team-video-erdelt100.html',
             'md5': 'dbab0aef2e047060ea7a21fc1ce1078a',

From 32fffff2ccc044c639c8723281981aa347423762 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 9 May 2015 21:19:09 +0600
Subject: [PATCH 1239/6219] [eroprofile] Fix video URL extraction (Closes
 #5657)

---
 youtube_dl/extractor/eroprofile.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/eroprofile.py b/youtube_dl/extractor/eroprofile.py
index 0cbca90b0..316033cf1 100644
--- a/youtube_dl/extractor/eroprofile.py
+++ b/youtube_dl/extractor/eroprofile.py
@@ -4,7 +4,10 @@ import re
 
 from .common import InfoExtractor
 from ..compat import compat_urllib_parse
-from ..utils import ExtractorError
+from ..utils import (
+    ExtractorError,
+    unescapeHTML
+)
 
 
 class EroProfileIE(InfoExtractor):
@@ -75,8 +78,8 @@ class EroProfileIE(InfoExtractor):
             [r"glbUpdViews\s*\('\d*','(\d+)'", r'p/report/video/(\d+)'],
             webpage, 'video id', default=None)
 
-        video_url = self._search_regex(
-            r'<source src="([^"]+)', webpage, 'video url')
+        video_url = unescapeHTML(self._search_regex(
+            r'<source src="([^"]+)', webpage, 'video url'))
         title = self._html_search_regex(
             r'Title:</th><td>([^<]+)</td>', webpage, 'title')
         thumbnail = self._search_regex(

From f2e0056579ac507b776ce2c86b5281fc28bbc275 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 9 May 2015 21:23:09 +0600
Subject: [PATCH 1240/6219] [vgtv] Avoid duplicate format_id

---
 youtube_dl/extractor/vgtv.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vgtv.py b/youtube_dl/extractor/vgtv.py
index eb2652fb1..e6ee1e471 100644
--- a/youtube_dl/extractor/vgtv.py
+++ b/youtube_dl/extractor/vgtv.py
@@ -103,11 +103,14 @@ class VGTVIE(InfoExtractor):
 
         hls_url = streams.get('hls')
         if hls_url:
-            formats.extend(self._extract_m3u8_formats(hls_url, video_id, 'mp4'))
+            formats.extend(self._extract_m3u8_formats(
+                hls_url, video_id, 'mp4', m3u8_id='hls'))
 
         hds_url = streams.get('hds')
         if hds_url:
-            formats.extend(self._extract_f4m_formats(hds_url + '?hdcore=3.2.0&plugin=aasp-3.2.0.77.18', video_id))
+            formats.extend(self._extract_f4m_formats(
+                hds_url + '?hdcore=3.2.0&plugin=aasp-3.2.0.77.18',
+                video_id, f4m_id='hds'))
 
         mp4_url = streams.get('mp4')
         if mp4_url:

From 480065172d4c97f00973b3f0bf24cd1b8e567627 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 10 May 2015 00:26:42 +0600
Subject: [PATCH 1241/6219] [lifenews] Add support for video URLs (Closes
 #5660)

---
 youtube_dl/extractor/lifenews.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/lifenews.py b/youtube_dl/extractor/lifenews.py
index 081016b80..92031e843 100644
--- a/youtube_dl/extractor/lifenews.py
+++ b/youtube_dl/extractor/lifenews.py
@@ -14,7 +14,7 @@ from ..utils import (
 class LifeNewsIE(InfoExtractor):
     IE_NAME = 'lifenews'
     IE_DESC = 'LIFE | NEWS'
-    _VALID_URL = r'http://lifenews\.ru/(?:mobile/)?news/(?P<id>\d+)'
+    _VALID_URL = r'http://lifenews\.ru/(?:mobile/)?(?P<section>news|video)/(?P<id>\d+)'
 
     _TESTS = [{
         'url': 'http://lifenews.ru/news/126342',
@@ -55,12 +55,15 @@ class LifeNewsIE(InfoExtractor):
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
+        section = mobj.group('section')
 
-        webpage = self._download_webpage('http://lifenews.ru/news/%s' % video_id, video_id, 'Downloading page')
+        webpage = self._download_webpage(
+            'http://lifenews.ru/%s/%s' % (section, video_id),
+            video_id, 'Downloading page')
 
         videos = re.findall(r'<video.*?poster="(?P<poster>[^"]+)".*?src="(?P<video>[^"]+)".*?></video>', webpage)
         iframe_link = self._html_search_regex(
-            '<iframe[^>]+src="([^"]+)', webpage, 'iframe link', default=None)
+            '<iframe[^>]+src=["\']([^"\']+)["\']', webpage, 'iframe link', default=None)
         if not videos and not iframe_link:
             raise ExtractorError('No media links available for %s' % video_id)
 

From 057ebeaca3da40b901b2592e2302a0d4bbab48e3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 10 May 2015 00:27:49 +0600
Subject: [PATCH 1242/6219] [lifenews] Add test for #5660

---
 youtube_dl/extractor/lifenews.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/lifenews.py b/youtube_dl/extractor/lifenews.py
index 92031e843..7d5b8621b 100644
--- a/youtube_dl/extractor/lifenews.py
+++ b/youtube_dl/extractor/lifenews.py
@@ -50,6 +50,9 @@ class LifeNewsIE(InfoExtractor):
             'upload_date': '20150505',
             'uploader': 'embed.life.ru',
         }
+    }, {
+        'url': 'http://lifenews.ru/video/13035',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 754270313a2b337eda98fa95232bd1064e294173 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 10 May 2015 01:03:26 +0600
Subject: [PATCH 1243/6219] [life:embed] Move to separated extractor and
 extract m3u8 formats

---
 youtube_dl/extractor/__init__.py |  5 +++-
 youtube_dl/extractor/lifenews.py | 47 ++++++++++++++++++++++++++++++++
 2 files changed, 51 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 0a18dba5c..f117578a2 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -258,7 +258,10 @@ from .letv import (
     LetvPlaylistIE
 )
 from .libsyn import LibsynIE
-from .lifenews import LifeNewsIE
+from .lifenews import (
+    LifeNewsIE,
+    LifeEmbedIE,
+)
 from .liveleak import LiveLeakIE
 from .livestream import (
     LivestreamIE,
diff --git a/youtube_dl/extractor/lifenews.py b/youtube_dl/extractor/lifenews.py
index 7d5b8621b..7f39fa4cf 100644
--- a/youtube_dl/extractor/lifenews.py
+++ b/youtube_dl/extractor/lifenews.py
@@ -4,7 +4,9 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import compat_urlparse
 from ..utils import (
+    determine_ext,
     int_or_none,
     unified_strdate,
     ExtractorError,
@@ -119,3 +121,48 @@ class LifeNewsIE(InfoExtractor):
             return make_entry(video_id, videos[0])
         else:
             return [make_entry(video_id, media, video_number + 1) for video_number, media in enumerate(videos)]
+
+
+class LifeEmbedIE(InfoExtractor):
+    IE_NAME = 'life:embed'
+    _VALID_URL = r'http://embed\.life\.ru/embed/(?P<id>[\da-f]{32})'
+
+    _TEST = {
+        'url': 'http://embed.life.ru/embed/e50c2dec2867350528e2574c899b8291',
+        'md5': 'b889715c9e49cb1981281d0e5458fbbe',
+        'info_dict': {
+            'id': 'e50c2dec2867350528e2574c899b8291',
+            'ext': 'mp4',
+            'title': 'e50c2dec2867350528e2574c899b8291',
+            'thumbnail': 're:http://.*\.jpg',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        formats = []
+        for video_url in re.findall(r'"file"\s*:\s*"([^"]+)', webpage):
+            video_url = compat_urlparse.urljoin(url, video_url)
+            ext = determine_ext(video_url)
+            if ext == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    video_url, video_id, 'mp4', m3u8_id='m3u8'))
+            else:
+                formats.append({
+                    'url': video_url,
+                    'format_id': ext,
+                    'preference': 1,
+                })
+
+        thumbnail = self._search_regex(
+            r'"image"\s*:\s*"([^"]+)', webpage, 'thumbnail', default=None)
+
+        return {
+            'id': video_id,
+            'title': video_id,
+            'thumbnail': thumbnail,
+            'formats': formats,
+        }

From 69fe3a5f0961c0ae602da531d2b0fb3f11b9d7c9 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 10 May 2015 01:05:24 +0200
Subject: [PATCH 1244/6219] release 2015.05.10

---
 docs/supportedsites.md | 9 ++++++++-
 youtube_dl/version.py  | 2 +-
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 6d2e496a8..98b625380 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -64,6 +64,8 @@
  - **BR**: Bayerischer Rundfunk Mediathek
  - **Break**
  - **Brightcove**
+ - **bt:article**: Bergens Tidende Articles
+ - **bt:vestlendingen**: Bergens Tidende - Vestlendingen
  - **BuzzFeed**
  - **BYUtv**
  - **Camdemy**
@@ -240,6 +242,7 @@
  - **LetvPlaylist**
  - **LetvTv**
  - **Libsyn**
+ - **life:embed**
  - **lifenews**: LIFE | NEWS
  - **LiveLeak**
  - **livestream**
@@ -328,6 +331,7 @@
  - **ntv.ru**
  - **Nuvid**
  - **NYTimes**
+ - **NYTimesArticle**
  - **ocw.mit.edu**
  - **Odnoklassniki**
  - **OktoberfestTV**
@@ -434,6 +438,7 @@
  - **southpark.cc.com**
  - **southpark.cc.com:español**
  - **southpark.de**
+ - **southpark.nl**
  - **Space**
  - **SpankBang**
  - **Spankwire**
@@ -453,6 +458,7 @@
  - **StreamCZ**
  - **StreetVoice**
  - **SunPorno**
+ - **SVT**
  - **SVTPlay**: SVT Play and Öppet arkiv
  - **SWRMediathek**
  - **Syfy**
@@ -529,7 +535,7 @@
  - **Vessel**
  - **Vesti**: Вести.Ru
  - **Vevo**
- - **VGTV**
+ - **VGTV**: VGTV and BTTV
  - **vh1.com**
  - **Vice**
  - **Viddler**
@@ -587,6 +593,7 @@
  - **XHamster**
  - **XMinus**
  - **XNXX**
+ - **Xstream**
  - **XTube**
  - **XTubeUser**: XTube user profile
  - **Xuite**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index a5a81bcd2..83c5a1659 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.05.04'
+__version__ = '2015.05.10'

From 3800b908b1976f242b41d5a2d114418559ce3b48 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 10 May 2015 06:14:34 +0600
Subject: [PATCH 1245/6219] [mlb] Fix #5663

---
 youtube_dl/extractor/mlb.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/mlb.py b/youtube_dl/extractor/mlb.py
index 40c9ecb35..e242b897f 100644
--- a/youtube_dl/extractor/mlb.py
+++ b/youtube_dl/extractor/mlb.py
@@ -12,13 +12,13 @@ from ..utils import (
 class MLBIE(InfoExtractor):
     _VALID_URL = r'''(?x)
                     https?://
-                        m(?:lb)?\.(?:[\da-z_-]+\.)?mlb\.com/
+                        (?:[\da-z_-]+\.)*mlb\.com/
                         (?:
                             (?:
                                 (?:.*?/)?video/(?:topic/[\da-z_-]+/)?v|
                                 (?:
                                     shared/video/embed/(?:embed|m-internal-embed)\.html|
-                                    [^/]+/video/play\.jsp
+                                    (?:[^/]+/)+(?:play|index)\.jsp|
                                 )\?.*?\bcontent_id=
                             )
                             (?P<id>n?\d+)|
@@ -114,6 +114,10 @@ class MLBIE(InfoExtractor):
             # From http://m.mlb.com/news/article/118550098/blue-jays-kevin-pillar-goes-spidey-up-the-wall-to-rob-tim-beckham-of-a-homer
             'url': 'http://mlb.mlb.com/shared/video/embed/m-internal-embed.html?content_id=75609783&property=mlb&autoplay=true&hashmode=false&siteSection=mlb/multimedia/article_118550098/article_embed&club=mlb',
             'only_matching': True,
+        },
+        {
+            'url': 'http://washington.nationals.mlb.com/mlb/gameday/index.jsp?c_id=was&gid=2015_05_09_atlmlb_wasmlb_1&lang=en&content_id=108309983&mode=video#',
+            'only_matching': True,
         }
     ]
 
@@ -125,7 +129,7 @@ class MLBIE(InfoExtractor):
             video_path = mobj.group('path')
             webpage = self._download_webpage(url, video_path)
             video_id = self._search_regex(
-                r'data-video-?id="(\d+)"', webpage, 'video id')
+                [r'data-video-?id="(\d+)"', r'content_id=(\d+)'], webpage, 'video id')
 
         detail = self._download_xml(
             'http://m.mlb.com/gen/multimedia/detail/%s/%s/%s/%s.xml'

From c6ddbdb66c5d6ead5e198013c54ef53d641063f1 Mon Sep 17 00:00:00 2001
From: Duncan <duncan@vtllf.org>
Date: Sun, 10 May 2015 12:30:07 +1200
Subject: [PATCH 1246/6219] [voicerepublic] Add new extractor

---
 youtube_dl/extractor/__init__.py      |  1 +
 youtube_dl/extractor/voicerepublic.py | 55 +++++++++++++++++++++++++++
 2 files changed, 56 insertions(+)
 create mode 100644 youtube_dl/extractor/voicerepublic.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index f117578a2..5cb3c304d 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -634,6 +634,7 @@ from .vk import (
     VKUserVideosIE,
 )
 from .vodlocker import VodlockerIE
+from .voicerepublic import VoiceRepublicIE
 from .vporn import VpornIE
 from .vrt import VRTIE
 from .vube import VubeIE
diff --git a/youtube_dl/extractor/voicerepublic.py b/youtube_dl/extractor/voicerepublic.py
new file mode 100644
index 000000000..1a90693cb
--- /dev/null
+++ b/youtube_dl/extractor/voicerepublic.py
@@ -0,0 +1,55 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+from ..compat import (
+    compat_urllib_request,
+)
+
+
+class VoiceRepublicIE(InfoExtractor):
+    _VALID_URL = r'https?://voicerepublic\.com/talks/(?P<id>[0-9a-z-]+)'
+    _TEST = {
+        'url': 'https://voicerepublic.com/talks/watching-the-watchers-building-a-sousveillance-state',
+        'md5': '0554a24d1657915aa8e8f84e15dc9353',
+        'info_dict': {
+            'id': '2296',
+            'ext': 'm4a',
+            'title': 'Watching the Watchers: Building a Sousveillance State',
+            'thumbnail': 'https://voicerepublic.com/system/flyer/2296.png',
+            'description': 'md5:715ba964958afa2398df615809cfecb1',
+            'creator': 'M. C. McGrath',
+        }
+    }
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        req = compat_urllib_request.Request(url)
+        # Older versions of Firefox get redirected to an "upgrade browser" page
+        req.add_header('User-Agent', 'youtube-dl')
+        webpage = self._download_webpage(req, display_id)
+        thumbnail = self._og_search_thumbnail(webpage)
+        video_id = self._search_regex(r'/(\d+)\.png', thumbnail, 'id')
+
+        if '<div class=\'vr-player jp-jplayer\'' in webpage:
+            formats = [{
+                'url': 'https://voicerepublic.com/vrmedia/{}-clean.{}'.format(video_id, ext),
+                'ext': ext,
+                'format_id': ext,
+                'vcodec': 'none',
+            } for ext in ['m4a', 'mp3', 'ogg']]
+            self._sort_formats(formats)
+        else:
+            # Audio is still queued for processing
+            formats = []
+
+        return {
+            'id': video_id,
+            'title': self._og_search_title(webpage),
+            'formats': formats,
+            'url': self._og_search_url(webpage),
+            'thumbnail': thumbnail,
+            'description': self._og_search_description(webpage),
+            'creator': self._search_regex(r'<meta content=\'([^\']*?)\' name=\'author\'>', webpage, 'author', fatal=False),
+        }

From 95eb1adda8692a61db639fb21344ad22d1847044 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 10 May 2015 08:54:50 +0600
Subject: [PATCH 1247/6219] [life:embed] Sort formats

---
 youtube_dl/extractor/lifenews.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/lifenews.py b/youtube_dl/extractor/lifenews.py
index 7f39fa4cf..42cb6e35f 100644
--- a/youtube_dl/extractor/lifenews.py
+++ b/youtube_dl/extractor/lifenews.py
@@ -156,6 +156,7 @@ class LifeEmbedIE(InfoExtractor):
                     'format_id': ext,
                     'preference': 1,
                 })
+        self._sort_formats(formats)
 
         thumbnail = self._search_regex(
             r'"image"\s*:\s*"([^"]+)', webpage, 'thumbnail', default=None)

From f900dc3fb9e17e399b0f33925ee239696cc46010 Mon Sep 17 00:00:00 2001
From: Duncan <duncan@vtllf.org>
Date: Sun, 10 May 2015 15:01:58 +1200
Subject: [PATCH 1248/6219] [voicerepublic] Extract author using
 _html_search_meta

---
 youtube_dl/extractor/voicerepublic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/voicerepublic.py b/youtube_dl/extractor/voicerepublic.py
index 1a90693cb..7d255d6fa 100644
--- a/youtube_dl/extractor/voicerepublic.py
+++ b/youtube_dl/extractor/voicerepublic.py
@@ -51,5 +51,5 @@ class VoiceRepublicIE(InfoExtractor):
             'url': self._og_search_url(webpage),
             'thumbnail': thumbnail,
             'description': self._og_search_description(webpage),
-            'creator': self._search_regex(r'<meta content=\'([^\']*?)\' name=\'author\'>', webpage, 'author', fatal=False),
+            'creator': self._html_search_meta('author', webpage),
         }

From 03f760b1c0478c1f65cf6e978d7592be46873313 Mon Sep 17 00:00:00 2001
From: Duncan <duncan@vtllf.org>
Date: Sun, 10 May 2015 15:40:09 +1200
Subject: [PATCH 1249/6219] [voicerepublic] Remove creator field

---
 youtube_dl/extractor/voicerepublic.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/youtube_dl/extractor/voicerepublic.py b/youtube_dl/extractor/voicerepublic.py
index 7d255d6fa..960974e16 100644
--- a/youtube_dl/extractor/voicerepublic.py
+++ b/youtube_dl/extractor/voicerepublic.py
@@ -19,7 +19,6 @@ class VoiceRepublicIE(InfoExtractor):
             'title': 'Watching the Watchers: Building a Sousveillance State',
             'thumbnail': 'https://voicerepublic.com/system/flyer/2296.png',
             'description': 'md5:715ba964958afa2398df615809cfecb1',
-            'creator': 'M. C. McGrath',
         }
     }
 
@@ -51,5 +50,4 @@ class VoiceRepublicIE(InfoExtractor):
             'url': self._og_search_url(webpage),
             'thumbnail': thumbnail,
             'description': self._og_search_description(webpage),
-            'creator': self._html_search_meta('author', webpage),
         }

From f03a8a3c4ec4dc95164c12181ffc1ddcb7583ef6 Mon Sep 17 00:00:00 2001
From: Duncan <duncan@vtllf.org>
Date: Sun, 10 May 2015 15:12:29 +1200
Subject: [PATCH 1250/6219] [voicerepublic] Raise ExtractorError if audio is
 still being processed

---
 youtube_dl/extractor/voicerepublic.py | 27 ++++++++++++---------------
 1 file changed, 12 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/voicerepublic.py b/youtube_dl/extractor/voicerepublic.py
index 960974e16..d3e35a815 100644
--- a/youtube_dl/extractor/voicerepublic.py
+++ b/youtube_dl/extractor/voicerepublic.py
@@ -2,10 +2,8 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-
-from ..compat import (
-    compat_urllib_request,
-)
+from ..compat import compat_urllib_request
+from ..utils import ExtractorError
 
 
 class VoiceRepublicIE(InfoExtractor):
@@ -31,17 +29,16 @@ class VoiceRepublicIE(InfoExtractor):
         thumbnail = self._og_search_thumbnail(webpage)
         video_id = self._search_regex(r'/(\d+)\.png', thumbnail, 'id')
 
-        if '<div class=\'vr-player jp-jplayer\'' in webpage:
-            formats = [{
-                'url': 'https://voicerepublic.com/vrmedia/{}-clean.{}'.format(video_id, ext),
-                'ext': ext,
-                'format_id': ext,
-                'vcodec': 'none',
-            } for ext in ['m4a', 'mp3', 'ogg']]
-            self._sort_formats(formats)
-        else:
-            # Audio is still queued for processing
-            formats = []
+        if '<a>Queued for processing, please stand by...</a>' in webpage:
+            raise ExtractorError('Audio is still queued for processing')
+
+        formats = [{
+            'url': 'https://voicerepublic.com/vrmedia/{}-clean.{}'.format(video_id, ext),
+            'ext': ext,
+            'format_id': ext,
+            'vcodec': 'none',
+        } for ext in ['m4a', 'mp3', 'ogg']]
+        self._sort_formats(formats)
 
         return {
             'id': video_id,

From 28ebef0b1b1b7b97137fbd8e093c09cb51954606 Mon Sep 17 00:00:00 2001
From: Duncan <duncan@vtllf.org>
Date: Sun, 10 May 2015 16:03:09 +1200
Subject: [PATCH 1251/6219] [voicerepublic] Detect list of available formats
 from the web page

---
 youtube_dl/extractor/voicerepublic.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/voicerepublic.py b/youtube_dl/extractor/voicerepublic.py
index d3e35a815..d150b5b5e 100644
--- a/youtube_dl/extractor/voicerepublic.py
+++ b/youtube_dl/extractor/voicerepublic.py
@@ -1,6 +1,8 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..compat import compat_urllib_request
 from ..utils import ExtractorError
@@ -32,12 +34,15 @@ class VoiceRepublicIE(InfoExtractor):
         if '<a>Queued for processing, please stand by...</a>' in webpage:
             raise ExtractorError('Audio is still queued for processing')
 
+        ext_matches = re.finditer(r'data-\w+=\'/vrmedia/\d+-clean\.(\w+)\'', webpage)
+        exts = [match.group(1) for match in ext_matches]
+
         formats = [{
             'url': 'https://voicerepublic.com/vrmedia/{}-clean.{}'.format(video_id, ext),
             'ext': ext,
             'format_id': ext,
             'vcodec': 'none',
-        } for ext in ['m4a', 'mp3', 'ogg']]
+        } for ext in exts]
         self._sort_formats(formats)
 
         return {

From 1dcb52188d3709711b3ea5ae1ff6bdb985e79c62 Mon Sep 17 00:00:00 2001
From: Duncan <duncan@vtllf.org>
Date: Sun, 10 May 2015 16:38:26 +1200
Subject: [PATCH 1252/6219] [voicerepublic] Remove hardcoded paths to media
 files

---
 youtube_dl/extractor/voicerepublic.py | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/voicerepublic.py b/youtube_dl/extractor/voicerepublic.py
index d150b5b5e..a3e40b940 100644
--- a/youtube_dl/extractor/voicerepublic.py
+++ b/youtube_dl/extractor/voicerepublic.py
@@ -34,15 +34,12 @@ class VoiceRepublicIE(InfoExtractor):
         if '<a>Queued for processing, please stand by...</a>' in webpage:
             raise ExtractorError('Audio is still queued for processing')
 
-        ext_matches = re.finditer(r'data-\w+=\'/vrmedia/\d+-clean\.(\w+)\'', webpage)
-        exts = [match.group(1) for match in ext_matches]
-
         formats = [{
-            'url': 'https://voicerepublic.com/vrmedia/{}-clean.{}'.format(video_id, ext),
+            'url': 'https://voicerepublic.com' + path,
             'ext': ext,
             'format_id': ext,
             'vcodec': 'none',
-        } for ext in exts]
+        } for ext, path in re.findall(r"data-([^=]+)='(/[^']+\.\1)'", webpage)]
         self._sort_formats(formats)
 
         return {

From a909e6ad43f9d9661691739a810d7b8853e17175 Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Sun, 10 May 2015 15:27:55 +0800
Subject: [PATCH 1253/6219] [dailymotion] Patch upload_date detection.

(closes #5665)
---
 youtube_dl/extractor/dailymotion.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index aa595af20..db10b8d00 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -52,6 +52,7 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
                 'ext': 'mp4',
                 'uploader': 'IGN',
                 'title': 'Steam Machine Models, Pricing Listed on Steam Store - IGN News',
+                'upload_date': '20150306',
             }
         },
         # Vevo video
@@ -106,9 +107,9 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
         age_limit = self._rta_search(webpage)
 
         video_upload_date = None
-        mobj = re.search(r'<div class="[^"]*uploaded_cont[^"]*" title="[^"]*">([0-9]{2})-([0-9]{2})-([0-9]{4})</div>', webpage)
+        mobj = re.search(r'<meta property="video:release_date" content="([0-9]{4})-([0-9]{2})-([0-9]{2}).+?"/>', webpage)
         if mobj is not None:
-            video_upload_date = mobj.group(3) + mobj.group(2) + mobj.group(1)
+            video_upload_date = mobj.group(1) + mobj.group(2) + mobj.group(3)
 
         embed_url = 'https://www.dailymotion.com/embed/video/%s' % video_id
         embed_request = self._build_request(embed_url)

From 1934f3a0eaf16ae1d1644178b7128806b8629867 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 10 May 2015 18:22:07 +0800
Subject: [PATCH 1254/6219] [ndr] Extended to support n-joy.de as well (closes
 #4527)

According to http://en.wikipedia.org/wiki/N-Joy, n-joy.de is a service
hosted by NDR, so I put them together.
---
 youtube_dl/extractor/__init__.py |  5 +-
 youtube_dl/extractor/ndr.py      | 90 ++++++++++++++++++++------------
 2 files changed, 61 insertions(+), 34 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index f117578a2..66adb4de7 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -324,7 +324,10 @@ from .nbc import (
     NBCSportsIE,
     NBCSportsVPlayerIE,
 )
-from .ndr import NDRIE
+from .ndr import (
+    NDRIE,
+    NJoyIE,
+)
 from .ndtv import NDTVIE
 from .netzkino import NetzkinoIE
 from .nerdcubed import NerdCubedFeedIE
diff --git a/youtube_dl/extractor/ndr.py b/youtube_dl/extractor/ndr.py
index f49c66690..afb9eda27 100644
--- a/youtube_dl/extractor/ndr.py
+++ b/youtube_dl/extractor/ndr.py
@@ -8,41 +8,11 @@ from ..utils import (
     ExtractorError,
     int_or_none,
     qualities,
+    parse_duration,
 )
 
 
-class NDRIE(InfoExtractor):
-    IE_NAME = 'ndr'
-    IE_DESC = 'NDR.de - Mediathek'
-    _VALID_URL = r'https?://www\.ndr\.de/.+?(?P<id>\d+)\.html'
-
-    _TESTS = [
-        {
-            'url': 'http://www.ndr.de/fernsehen/sendungen/nordmagazin/Kartoffeltage-in-der-Lewitz,nordmagazin25866.html',
-            'md5': '5bc5f5b92c82c0f8b26cddca34f8bb2c',
-            'note': 'Video file',
-            'info_dict': {
-                'id': '25866',
-                'ext': 'mp4',
-                'title': 'Kartoffeltage in der Lewitz',
-                'description': 'md5:48c4c04dde604c8a9971b3d4e3b9eaa8',
-                'duration': 166,
-            }
-        },
-        {
-            'url': 'http://www.ndr.de/info/audio51535.html',
-            'md5': 'bb3cd38e24fbcc866d13b50ca59307b8',
-            'note': 'Audio file',
-            'info_dict': {
-                'id': '51535',
-                'ext': 'mp3',
-                'title': 'La Valette entgeht der Hinrichtung',
-                'description': 'md5:22f9541913a40fe50091d5cdd7c9f536',
-                'duration': 884,
-            }
-        }
-    ]
-
+class NDRBaseIE(InfoExtractor):
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
@@ -54,7 +24,11 @@ class NDRIE(InfoExtractor):
         if description:
             description = description.strip()
 
-        duration = int_or_none(self._html_search_regex(r'duration: (\d+),\n', page, 'duration', fatal=False))
+        duration = int_or_none(self._html_search_regex(r'duration: (\d+),\n', page, 'duration', default=None))
+        if not duration:
+            duration = parse_duration(self._html_search_regex(
+                r'(<span class="min">\d+</span>:<span class="sec">\d+</span>)',
+                page, 'duration', default=None))
 
         formats = []
 
@@ -92,3 +66,53 @@ class NDRIE(InfoExtractor):
             'duration': duration,
             'formats': formats,
         }
+
+
+class NDRIE(NDRBaseIE):
+    IE_NAME = 'ndr'
+    IE_DESC = 'NDR.de - Mediathek'
+    _VALID_URL = r'https?://www\.ndr\.de/.+?(?P<id>\d+)\.html'
+
+    _TESTS = [
+        {
+            'url': 'http://www.ndr.de/fernsehen/sendungen/nordmagazin/Kartoffeltage-in-der-Lewitz,nordmagazin25866.html',
+            'md5': '5bc5f5b92c82c0f8b26cddca34f8bb2c',
+            'note': 'Video file',
+            'info_dict': {
+                'id': '25866',
+                'ext': 'mp4',
+                'title': 'Kartoffeltage in der Lewitz',
+                'description': 'md5:48c4c04dde604c8a9971b3d4e3b9eaa8',
+                'duration': 166,
+            }
+        },
+        {
+            'url': 'http://www.ndr.de/info/audio51535.html',
+            'md5': 'bb3cd38e24fbcc866d13b50ca59307b8',
+            'note': 'Audio file',
+            'info_dict': {
+                'id': '51535',
+                'ext': 'mp3',
+                'title': 'La Valette entgeht der Hinrichtung',
+                'description': 'md5:22f9541913a40fe50091d5cdd7c9f536',
+                'duration': 884,
+            }
+        }
+    ]
+
+
+class NJoyIE(NDRBaseIE):
+    IE_NAME = 'N-JOY'
+    _VALID_URL = r'https?://www\.n-joy\.de/.+?(?P<id>\d+)\.html'
+
+    _TEST = {
+        'url': 'http://www.n-joy.de/entertainment/comedy/comedy_contest/Benaissa-beim-NDR-Comedy-Contest,comedycontest2480.html',
+        'md5': 'cb63be60cd6f9dd75218803146d8dc67',
+        'info_dict': {
+            'id': '2480',
+            'ext': 'mp4',
+            'title': 'Benaissa beim NDR Comedy Contest',
+            'description': 'Von seinem sehr "behaarten" Leben lässt sich Benaissa trotz aller Schwierigkeiten nicht unterkriegen.',
+            'duration': 654,
+        }
+    }

From 63cbd19f500eb4d90c1fc7c09f04de5df43a6a04 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 10 May 2015 18:30:26 +0800
Subject: [PATCH 1255/6219] [ndr] Replace the 404 test case

---
 youtube_dl/extractor/ndr.py | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ndr.py b/youtube_dl/extractor/ndr.py
index afb9eda27..79a13958b 100644
--- a/youtube_dl/extractor/ndr.py
+++ b/youtube_dl/extractor/ndr.py
@@ -84,7 +84,19 @@ class NDRIE(NDRBaseIE):
                 'title': 'Kartoffeltage in der Lewitz',
                 'description': 'md5:48c4c04dde604c8a9971b3d4e3b9eaa8',
                 'duration': 166,
-            }
+            },
+            'skip': '404 Not found',
+        },
+        {
+            'url': 'http://www.ndr.de/fernsehen/Party-Poette-und-Parade,hafengeburtstag988.html',
+            'md5': 'dadc003c55ae12a5d2f6bd436cd73f59',
+            'info_dict': {
+                'id': '988',
+                'ext': 'mp4',
+                'title': 'Party, Pötte und Parade',
+                'description': 'Hunderttausende feiern zwischen Speicherstadt und St. Pauli den 826. Hafengeburtstag. Die NDR Sondersendung zeigt die schönsten und spektakulärsten Bilder vom Auftakt.',
+                'duration': 3498,
+            },
         },
         {
             'url': 'http://www.ndr.de/info/audio51535.html',

From a6762c4a22325b5b69770de82df8725d2eb5c3df Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 10 May 2015 18:29:15 +0600
Subject: [PATCH 1256/6219] [voicerepublic] Make more robust and extract more
 metadata

---
 youtube_dl/extractor/voicerepublic.py | 95 ++++++++++++++++++++-------
 1 file changed, 71 insertions(+), 24 deletions(-)

diff --git a/youtube_dl/extractor/voicerepublic.py b/youtube_dl/extractor/voicerepublic.py
index a3e40b940..1106c655b 100644
--- a/youtube_dl/extractor/voicerepublic.py
+++ b/youtube_dl/extractor/voicerepublic.py
@@ -1,52 +1,99 @@
-# coding: utf-8
 from __future__ import unicode_literals
 
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_request
-from ..utils import ExtractorError
+from ..compat import (
+    compat_urllib_request,
+    compat_urlparse,
+)
+from ..utils import (
+    ExtractorError,
+    determine_ext,
+    int_or_none,
+)
 
 
 class VoiceRepublicIE(InfoExtractor):
-    _VALID_URL = r'https?://voicerepublic\.com/talks/(?P<id>[0-9a-z-]+)'
-    _TEST = {
-        'url': 'https://voicerepublic.com/talks/watching-the-watchers-building-a-sousveillance-state',
+    _VALID_URL = r'https?://voicerepublic\.com/(?:talks|embed)/(?P<id>[0-9a-z-]+)'
+    _TESTS = [{
+        'url': 'http://voicerepublic.com/talks/watching-the-watchers-building-a-sousveillance-state',
         'md5': '0554a24d1657915aa8e8f84e15dc9353',
         'info_dict': {
             'id': '2296',
+            'display_id': 'watching-the-watchers-building-a-sousveillance-state',
             'ext': 'm4a',
             'title': 'Watching the Watchers: Building a Sousveillance State',
-            'thumbnail': 'https://voicerepublic.com/system/flyer/2296.png',
             'description': 'md5:715ba964958afa2398df615809cfecb1',
+            'thumbnail': 're:^https?://.*\.(?:png|jpg)$',
+            'duration': 1800,
+            'view_count': int,
         }
-    }
+    }, {
+        'url': 'http://voicerepublic.com/embed/watching-the-watchers-building-a-sousveillance-state',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
-        req = compat_urllib_request.Request(url)
+
+        req = compat_urllib_request.Request(
+            compat_urlparse.urljoin(url, '/talks/%s' % display_id))
         # Older versions of Firefox get redirected to an "upgrade browser" page
         req.add_header('User-Agent', 'youtube-dl')
         webpage = self._download_webpage(req, display_id)
-        thumbnail = self._og_search_thumbnail(webpage)
-        video_id = self._search_regex(r'/(\d+)\.png', thumbnail, 'id')
 
-        if '<a>Queued for processing, please stand by...</a>' in webpage:
-            raise ExtractorError('Audio is still queued for processing')
+        if '>Queued for processing, please stand by...<' in webpage:
+            raise ExtractorError(
+                'Audio is still queued for processing', expected=True)
 
-        formats = [{
-            'url': 'https://voicerepublic.com' + path,
-            'ext': ext,
-            'format_id': ext,
-            'vcodec': 'none',
-        } for ext, path in re.findall(r"data-([^=]+)='(/[^']+\.\1)'", webpage)]
+        data = self._parse_json(
+            self._search_regex(
+                r'(?s)return ({.+?});\s*\n', webpage,
+                'data', default=None),
+            display_id, fatal=False)
+
+        if data:
+            title = data['title']
+            description = data.get('teaser')
+            talk_id = data.get('talk_id') or display_id
+            talk = data['talk']
+            duration = int_or_none(talk.get('duration'))
+            formats = [{
+                'url': compat_urlparse.urljoin(url, talk_url),
+                'format_id': format_id,
+                'ext': determine_ext(talk_url) or format_id,
+                'vcodec': 'none',
+            } for format_id, talk_url in talk['links'].items()]
+        else:
+            title = self._og_search_title(webpage)
+            description = self._html_search_regex(
+                r"(?s)<div class='talk-teaser'[^>]*>(.+?)</div>",
+                webpage, 'description', fatal=False)
+            talk_id = self._search_regex(
+                [r"id='jc-(\d+)'", r"data-shareable-id='(\d+)'"],
+                webpage, 'talk id', default=None) or display_id
+            duration = None
+            formats = [{
+                'url': compat_urlparse.urljoin(url, talk_url),
+                'format_id': format_id,
+                'ext': determine_ext(talk_url) or format_id,
+                'vcodec': 'none',
+            } for format_id, talk_url in re.findall(r"data-([^=]+)='([^']+)'", webpage)]
         self._sort_formats(formats)
 
+        thumbnail = self._og_search_thumbnail(webpage)
+        view_count = int_or_none(self._search_regex(
+            r"class='play-count[^']*'>\s*(\d+) plays",
+            webpage, 'play count', fatal=False))
+
         return {
-            'id': video_id,
-            'title': self._og_search_title(webpage),
-            'formats': formats,
-            'url': self._og_search_url(webpage),
+            'id': talk_id,
+            'display_id': display_id,
+            'title': title,
+            'description': description,
             'thumbnail': thumbnail,
-            'description': self._og_search_description(webpage),
+            'duration': duration,
+            'view_count': view_count,
+            'formats': formats,
         }

From 370b39e8ece9f475d489eda721130eec9a9f15e9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 10 May 2015 18:37:52 +0600
Subject: [PATCH 1257/6219] [voicerepublic] Fix fallback branch formats
 extraction

---
 youtube_dl/extractor/voicerepublic.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/voicerepublic.py b/youtube_dl/extractor/voicerepublic.py
index 1106c655b..254383d6c 100644
--- a/youtube_dl/extractor/voicerepublic.py
+++ b/youtube_dl/extractor/voicerepublic.py
@@ -47,12 +47,10 @@ class VoiceRepublicIE(InfoExtractor):
             raise ExtractorError(
                 'Audio is still queued for processing', expected=True)
 
-        data = self._parse_json(
-            self._search_regex(
-                r'(?s)return ({.+?});\s*\n', webpage,
-                'data', default=None),
-            display_id, fatal=False)
-
+        config = self._search_regex(
+            r'(?s)return ({.+?});\s*\n', webpage,
+            'data', default=None)
+        data = self._parse_json(config, display_id, fatal=False) if config else None
         if data:
             title = data['title']
             description = data.get('teaser')
@@ -74,12 +72,14 @@ class VoiceRepublicIE(InfoExtractor):
                 [r"id='jc-(\d+)'", r"data-shareable-id='(\d+)'"],
                 webpage, 'talk id', default=None) or display_id
             duration = None
+            player = self._search_regex(
+                r"class='vr-player jp-jplayer'([^>]+)>", webpage, 'player')
             formats = [{
                 'url': compat_urlparse.urljoin(url, talk_url),
                 'format_id': format_id,
                 'ext': determine_ext(talk_url) or format_id,
                 'vcodec': 'none',
-            } for format_id, talk_url in re.findall(r"data-([^=]+)='([^']+)'", webpage)]
+            } for format_id, talk_url in re.findall(r"data-([^=]+)='([^']+)'", player)]
         self._sort_formats(formats)
 
         thumbnail = self._og_search_thumbnail(webpage)

From 95c5534f8ed016a81f715f291ab09c4ea2c3679c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 10 May 2015 17:41:11 +0200
Subject: [PATCH 1258/6219] ExecAfterDownloadPP, YoutubeDL: remove unused
 parameters

---
 youtube_dl/YoutubeDL.py                       | 1 -
 youtube_dl/__init__.py                        | 2 --
 youtube_dl/postprocessor/execafterdownload.py | 3 +--
 3 files changed, 1 insertion(+), 5 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index d8583a8eb..4cf83c510 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -260,7 +260,6 @@ class YoutubeDL(object):
     The following options are used by the post processors:
     prefer_ffmpeg:     If True, use ffmpeg instead of avconv if both are available,
                        otherwise prefer avconv.
-    exec_cmd:          Arbitrary command to run after downloading
     """
 
     params = None
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index c88489f29..9cc9f851f 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -246,7 +246,6 @@ def _real_main(argv=None):
     if opts.exec_cmd:
         postprocessors.append({
             'key': 'ExecAfterDownload',
-            'verboseOutput': opts.verbose,
             'exec_cmd': opts.exec_cmd,
         })
     if opts.xattr_set_filesize:
@@ -345,7 +344,6 @@ def _real_main(argv=None):
         'default_search': opts.default_search,
         'youtube_include_dash_manifest': opts.youtube_include_dash_manifest,
         'encoding': opts.encoding,
-        'exec_cmd': opts.exec_cmd,
         'extract_flat': opts.extract_flat,
         'merge_output_format': opts.merge_output_format,
         'postprocessors': postprocessors,
diff --git a/youtube_dl/postprocessor/execafterdownload.py b/youtube_dl/postprocessor/execafterdownload.py
index 341437575..765fd8fe4 100644
--- a/youtube_dl/postprocessor/execafterdownload.py
+++ b/youtube_dl/postprocessor/execafterdownload.py
@@ -8,8 +8,7 @@ from ..utils import PostProcessingError
 
 
 class ExecAfterDownloadPP(PostProcessor):
-    def __init__(self, downloader=None, verboseOutput=None, exec_cmd=None):
-        self.verboseOutput = verboseOutput
+    def __init__(self, downloader=None, exec_cmd=None):
         self.exec_cmd = exec_cmd
 
     def run(self, information):

From 69b46b3d956220e4b3a3d5eda55768753a67f19d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 10 May 2015 17:47:49 +0200
Subject: [PATCH 1259/6219] ExecAfterDownloadPP: fix __init__ method

---
 youtube_dl/postprocessor/execafterdownload.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/postprocessor/execafterdownload.py b/youtube_dl/postprocessor/execafterdownload.py
index 765fd8fe4..13794b7ba 100644
--- a/youtube_dl/postprocessor/execafterdownload.py
+++ b/youtube_dl/postprocessor/execafterdownload.py
@@ -8,7 +8,8 @@ from ..utils import PostProcessingError
 
 
 class ExecAfterDownloadPP(PostProcessor):
-    def __init__(self, downloader=None, exec_cmd=None):
+    def __init__(self, downloader, exec_cmd):
+        super(ExecAfterDownloadPP, self).__init__(downloader)
         self.exec_cmd = exec_cmd
 
     def run(self, information):

From 70484b9f8ae629ccb87e8c0569f8f4bf2dfdb0ac Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 11 May 2015 00:26:39 +0600
Subject: [PATCH 1260/6219] [postprocessor/ffmpeg] Extract `check_outdated`
 method

---
 youtube_dl/postprocessor/ffmpeg.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 214de39f9..211faf69a 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -36,7 +36,9 @@ class FFmpegPostProcessor(PostProcessor):
     def check_version(self):
         if not self.available:
             raise FFmpegPostProcessorError('ffmpeg or avconv not found. Please install one.')
+        self.check_outdated()
 
+    def check_outdated(self):
         required_version = '10-0' if self.basename == 'avconv' else '1.0'
         if is_outdated_version(
                 self._versions[self.basename], required_version):
@@ -44,6 +46,8 @@ class FFmpegPostProcessor(PostProcessor):
                 self.basename, self.basename, required_version)
             if self._downloader:
                 self._downloader.report_warning(warning)
+            return True
+        return False
 
     @staticmethod
     def get_versions(downloader=None):

From 7fcb605b82796e79a5f559624808ca9404df1154 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 11 May 2015 00:27:29 +0600
Subject: [PATCH 1261/6219] [YoutubeDL] Fallback to `-f best` when merger is
 outdated

---
 youtube_dl/YoutubeDL.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 4cf83c510..7c3bdb964 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1086,9 +1086,10 @@ class YoutubeDL(object):
         if req_format is None:
             req_format_list = []
             if (self.params.get('outtmpl', DEFAULT_OUTTMPL) != '-'
-                    and info_dict['extractor'] in ['youtube', 'ted']
-                    and FFmpegMergerPP(self).available):
-                req_format_list.append('bestvideo+bestaudio')
+                    and info_dict['extractor'] in ['youtube', 'ted']):
+                merger = FFmpegMergerPP(self)
+                if merger.available and not merger.check_outdated():
+                    req_format_list.append('bestvideo+bestaudio')
             req_format_list.append('best')
             req_format = '/'.join(req_format_list)
         formats_to_download = []

From 13763ce599c8fbba43e57d2d79a9b007cfbd4ced Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 11 May 2015 02:00:31 +0600
Subject: [PATCH 1262/6219] [postprocessor/ffmpeg] Add `can_merge` method

---
 youtube_dl/postprocessor/ffmpeg.py | 21 +++++++++++++++++----
 1 file changed, 17 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 211faf69a..cc65b34e7 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -36,9 +36,7 @@ class FFmpegPostProcessor(PostProcessor):
     def check_version(self):
         if not self.available:
             raise FFmpegPostProcessorError('ffmpeg or avconv not found. Please install one.')
-        self.check_outdated()
 
-    def check_outdated(self):
         required_version = '10-0' if self.basename == 'avconv' else '1.0'
         if is_outdated_version(
                 self._versions[self.basename], required_version):
@@ -46,8 +44,6 @@ class FFmpegPostProcessor(PostProcessor):
                 self.basename, self.basename, required_version)
             if self._downloader:
                 self._downloader.report_warning(warning)
-            return True
-        return False
 
     @staticmethod
     def get_versions(downloader=None):
@@ -595,6 +591,23 @@ class FFmpegMergerPP(FFmpegPostProcessor):
         os.rename(encodeFilename(temp_filename), encodeFilename(filename))
         return info['__files_to_merge'], info
 
+    def can_merge(self):
+        # TODO: figure out merge-capable ffmpeg version
+        if self.basename != 'avconv':
+            return True
+
+        required_version = '10-0'
+        if is_outdated_version(
+                self._versions[self.basename], required_version):
+            warning = ('Your copy of %s is outdated and unable to properly mux separate video and audio files, '
+                       'youtube-dl will download single file media. '
+                       'Update %s to version %s or newer to fix this.') % (
+                           self.basename, self.basename, required_version)
+            if self._downloader:
+                self._downloader.report_warning(warning)
+            return False
+        return True
+
 
 class FFmpegFixupStretchedPP(FFmpegPostProcessor):
     def run(self, info):

From 97fcf1bbd07ae0c5b6e530dcf2623d199452a76c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 11 May 2015 02:01:16 +0600
Subject: [PATCH 1263/6219] [YoutubeDL] Check if merger can actually merge

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 7c3bdb964..00f86b342 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1088,7 +1088,7 @@ class YoutubeDL(object):
             if (self.params.get('outtmpl', DEFAULT_OUTTMPL) != '-'
                     and info_dict['extractor'] in ['youtube', 'ted']):
                 merger = FFmpegMergerPP(self)
-                if merger.available and not merger.check_outdated():
+                if merger.available and merger.can_merge():
                     req_format_list.append('bestvideo+bestaudio')
             req_format_list.append('best')
             req_format = '/'.join(req_format_list)

From e41f450f2860ab5aa3f3a04bc646594c6dbc6714 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 11 May 2015 20:04:05 +0800
Subject: [PATCH 1264/6219] [tmz] Add support for articles (fixes #5477)

---
 youtube_dl/extractor/__init__.py |  5 ++++-
 youtube_dl/extractor/tmz.py      | 28 ++++++++++++++++++++++++++++
 2 files changed, 32 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index e808f2734..b376fd279 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -543,7 +543,10 @@ from .thesixtyone import TheSixtyOneIE
 from .thisav import ThisAVIE
 from .tinypic import TinyPicIE
 from .tlc import TlcIE, TlcDeIE
-from .tmz import TMZIE
+from .tmz import (
+    TMZIE,
+    TMZArticleIE,
+)
 from .tnaflix import TNAFlixIE
 from .thvideo import (
     THVideoIE,
diff --git a/youtube_dl/extractor/tmz.py b/youtube_dl/extractor/tmz.py
index c5c6fdc51..7dbe68b5c 100644
--- a/youtube_dl/extractor/tmz.py
+++ b/youtube_dl/extractor/tmz.py
@@ -30,3 +30,31 @@ class TMZIE(InfoExtractor):
             'description': self._og_search_description(webpage),
             'thumbnail': self._html_search_meta('ThumbURL', webpage),
         }
+
+
+class TMZArticleIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?tmz\.com/\d{4}/\d{2}/\d{2}/(?P<id>[^/]+)/?'
+    _TEST = {
+        'url': 'http://www.tmz.com/2015/04/19/bobby-brown-bobbi-kristina-awake-video-concert',
+        'md5': 'e482a414a38db73087450e3a6ce69d00',
+        'info_dict': {
+            'id': '0_6snoelag',
+            'ext': 'mp4',
+            'title': 'Bobby Brown Tells Crowd ... Bobbi Kristina is Awake',
+            'description': 'Bobby Brown stunned his audience during a concert Saturday night, when he told the crowd, "Bobbi is awake.  She\'s watching me."',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+        embedded_video_info_str = self._html_search_regex(
+            r'tmzVideoEmbedV2\("([^)]+)"\);', webpage, 'embedded video info')
+
+        embedded_video_info = self._parse_json(
+            embedded_video_info_str, video_id,
+            transform_source=lambda s: s.replace('\\', ''))
+
+        return self.url_result(
+            'http://www.tmz.com/videos/%s/' % embedded_video_info['id'])

From 1f92865494c6efa1a0d5d90ffa849e85b80c8248 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 11 May 2015 21:05:39 +0600
Subject: [PATCH 1265/6219] [dumpert] Add cpc cookie (Closes #5672)

---
 youtube_dl/extractor/dumpert.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/dumpert.py b/youtube_dl/extractor/dumpert.py
index 9c594b757..999fb5620 100644
--- a/youtube_dl/extractor/dumpert.py
+++ b/youtube_dl/extractor/dumpert.py
@@ -26,7 +26,7 @@ class DumpertIE(InfoExtractor):
         video_id = self._match_id(url)
 
         req = compat_urllib_request.Request(url)
-        req.add_header('Cookie', 'nsfw=1')
+        req.add_header('Cookie', 'nsfw=1; cpc=10')
         webpage = self._download_webpage(req, video_id)
 
         files_base64 = self._search_regex(

From 511565282861e26913caddc1bcc0c865a9eec786 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 11 May 2015 21:31:36 +0600
Subject: [PATCH 1266/6219] [zingmp3] Capture error message

---
 youtube_dl/extractor/zingmp3.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/zingmp3.py b/youtube_dl/extractor/zingmp3.py
index 1afbe68ed..7dc1e2f2b 100644
--- a/youtube_dl/extractor/zingmp3.py
+++ b/youtube_dl/extractor/zingmp3.py
@@ -4,12 +4,18 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..utils import ExtractorError
 
 
 class ZingMp3BaseInfoExtractor(InfoExtractor):
 
-    @staticmethod
-    def _extract_item(item):
+    def _extract_item(self, item):
+        error_message = item.find('./errormessage').text
+        if error_message:
+            raise ExtractorError(
+                '%s returned error: %s' % (self.IE_NAME, error_message),
+                expected=True)
+
         title = item.find('./title').text.strip()
         source = item.find('./source').text
         extension = item.attrib['type']

From 81ed3bb9c0edb9a11b43964459ef57cca5683461 Mon Sep 17 00:00:00 2001
From: rrooij <rderooij685@gmail.com>
Date: Mon, 11 May 2015 17:38:08 +0200
Subject: [PATCH 1267/6219] [southpark] Sort alphabetically

---
 youtube_dl/extractor/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index b376fd279..3368edf7c 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -489,8 +489,8 @@ from .soundgasm import (
 )
 from .southpark import (
     SouthParkIE,
-    SouthParkEsIE,
     SouthParkDeIE,
+    SouthParkEsIE,
     SouthParkNlIE
 )
 from .space import SpaceIE

From 968ee176777a1bf4e33cfb849a7241b0ac45d254 Mon Sep 17 00:00:00 2001
From: rrooij <rderooij685@gmail.com>
Date: Mon, 11 May 2015 18:02:25 +0200
Subject: [PATCH 1268/6219] [southparkdk] Add extractor

---
 youtube_dl/extractor/__init__.py  |  1 +
 youtube_dl/extractor/southpark.py | 10 ++++++++++
 2 files changed, 11 insertions(+)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 3368edf7c..de19dfd7a 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -490,6 +490,7 @@ from .soundgasm import (
 from .southpark import (
     SouthParkIE,
     SouthParkDeIE,
+    SouthParkDkIE,
     SouthParkEsIE,
     SouthParkNlIE
 )
diff --git a/youtube_dl/extractor/southpark.py b/youtube_dl/extractor/southpark.py
index 59e31198c..83e5a7659 100644
--- a/youtube_dl/extractor/southpark.py
+++ b/youtube_dl/extractor/southpark.py
@@ -57,3 +57,13 @@ class SouthParkNlIE(SouthParkIE):
         'url': 'http://www.southpark.nl/full-episodes/s18e06-freemium-isnt-free',
         'playlist_count': 4,
     }]
+
+class SouthParkDkIE(SouthParkIE):
+    IE_NAME = 'southpark.dk'
+    _VALID_URL = r'https?://(?:www\.)?(?P<url>southparkstudios\.dk/(?:clips|full-episodes)/(?P<id>.+?)(\?|#|$))'
+    _FEED_URL = 'http://www.southparkstudios.dk/feeds/video-player/mrss/'
+
+    _TESTS = [{
+        'url': 'http://www.southparkstudios.dk/full-episodes/s18e07-grounded-vindaloop',
+        'playlist_count': 4,
+    }]

From 6d3f5935e516760964052718e6b90324c6f07391 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 11 May 2015 23:47:50 +0600
Subject: [PATCH 1269/6219] [southpark] Fix IE_NAME

---
 youtube_dl/extractor/southpark.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/southpark.py b/youtube_dl/extractor/southpark.py
index 83e5a7659..7fb165a87 100644
--- a/youtube_dl/extractor/southpark.py
+++ b/youtube_dl/extractor/southpark.py
@@ -58,8 +58,9 @@ class SouthParkNlIE(SouthParkIE):
         'playlist_count': 4,
     }]
 
+
 class SouthParkDkIE(SouthParkIE):
-    IE_NAME = 'southpark.dk'
+    IE_NAME = 'southparkstudios.dk'
     _VALID_URL = r'https?://(?:www\.)?(?P<url>southparkstudios\.dk/(?:clips|full-episodes)/(?P<id>.+?)(\?|#|$))'
     _FEED_URL = 'http://www.southparkstudios.dk/feeds/video-player/mrss/'
 

From d4b963d0a68f81f4fef5495af14e2e41add21a0f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 12 May 2015 01:54:56 +0600
Subject: [PATCH 1270/6219] [vine] Relax `alt_title` (Closes #5677)

---
 youtube_dl/extractor/vine.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vine.py b/youtube_dl/extractor/vine.py
index 65c459fad..c733a48fa 100644
--- a/youtube_dl/extractor/vine.py
+++ b/youtube_dl/extractor/vine.py
@@ -75,7 +75,7 @@ class VineIE(InfoExtractor):
         return {
             'id': video_id,
             'title': self._og_search_title(webpage),
-            'alt_title': self._og_search_description(webpage),
+            'alt_title': self._og_search_description(webpage, default=None),
             'description': data['description'],
             'thumbnail': data['thumbnailUrl'],
             'upload_date': unified_strdate(data['created']),

From 5332fd91bf16867b6777bd6cfd0b5086f84112c5 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 12 May 2015 12:42:13 +0800
Subject: [PATCH 1271/6219] [nytimes] Correct _VALID_URL of NYTimesArticleIE

---
 youtube_dl/extractor/nytimes.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nytimes.py b/youtube_dl/extractor/nytimes.py
index 6ffbe3863..7f254b867 100644
--- a/youtube_dl/extractor/nytimes.py
+++ b/youtube_dl/extractor/nytimes.py
@@ -89,7 +89,7 @@ class NYTimesIE(NYTimesBaseIE):
 
 
 class NYTimesArticleIE(NYTimesBaseIE):
-    _VALID_URL = r'https?://(?:www)?\.nytimes\.com/(.(?<!video))*?/(?:[^/]+/)*(?P<id>[^.]+)(?:\.html)?'
+    _VALID_URL = r'https?://(?:www\.)?nytimes\.com/(.(?<!video))*?/(?:[^/]+/)*(?P<id>[^.]+)(?:\.html)?'
     _TESTS = [{
         'url': 'http://www.nytimes.com/2015/04/14/business/owner-of-gravity-payments-a-credit-card-processor-is-setting-a-new-minimum-wage-70000-a-year.html?_r=0',
         'md5': 'e2076d58b4da18e6a001d53fd56db3c9',

From 7dff03636a843a6990e52200edb3ecca1246b3df Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 12 May 2015 12:47:37 +0800
Subject: [PATCH 1272/6219] [utils] Support 'dur' field in TTML

---
 test/test_utils.py  | 2 +-
 youtube_dl/utils.py | 8 ++++++--
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 86b110a7d..b40107037 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -600,7 +600,7 @@ ffmpeg version 2.4.4 Copyright (c) 2000-2014 the FFmpeg ...'''), '2.4.4')
                 <div xml:lang="en">
                     <p begin="0" end="1">The following line contains Chinese characters and special symbols</p>
                     <p begin="1" end="2">第二行<br/>♪♪</p>
-                    <p begin="2" end="3"><span>Third<br/>Line</span></p>
+                    <p begin="2" dur="1"><span>Third<br/>Line</span></p>
                 </div>
             </body>
             </tt>'''
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index d73efcf25..5439fcb35 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1866,10 +1866,14 @@ def dfxp2srt(dfxp_data):
     paras = dfxp.findall(_x('.//ttml:p'))
 
     for para, index in zip(paras, itertools.count(1)):
+        begin_time = parse_dfxp_time_expr(para.attrib['begin'])
+        end_time = parse_dfxp_time_expr(para.attrib.get('end'))
+        if not end_time:
+            end_time = begin_time + parse_dfxp_time_expr(para.attrib['dur'])
         out.append('%d\n%s --> %s\n%s\n\n' % (
             index,
-            format_srt_time(parse_dfxp_time_expr(para.attrib.get('begin'))),
-            format_srt_time(parse_dfxp_time_expr(para.attrib.get('end'))),
+            format_srt_time(begin_time),
+            format_srt_time(end_time),
             parse_node(para)))
 
     return ''.join(out)

From 1c7e2e64f6328024711d5fa999d4498396f4cb5c Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 12 May 2015 12:55:14 +0800
Subject: [PATCH 1273/6219] [nrk] Remove TTML to srt conversion codes

A common routine is implemented in utils.py and can be used via
--convert-subtitles.
---
 youtube_dl/extractor/nrk.py | 13 +------------
 1 file changed, 1 insertion(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index e91d3a248..cc70c2950 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -4,7 +4,6 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_str
 from ..utils import (
     ExtractorError,
     float_or_none,
@@ -200,20 +199,10 @@ class NRKTVIE(InfoExtractor):
         url = "%s%s" % (baseurl, subtitlesurl)
         self._debug_print('%s: Subtitle url: %s' % (video_id, url))
         captions = self._download_xml(
-            url, video_id, 'Downloading subtitles',
-            transform_source=lambda s: s.replace(r'<br />', '\r\n'))
+            url, video_id, 'Downloading subtitles')
         lang = captions.get('lang', 'no')
-        ps = captions.findall('./{0}body/{0}div/{0}p'.format('{http://www.w3.org/ns/ttml}'))
-        srt = ''
-        for pos, p in enumerate(ps):
-            begin = parse_duration(p.get('begin'))
-            duration = parse_duration(p.get('dur'))
-            starttime = self._subtitles_timecode(begin)
-            endtime = self._subtitles_timecode(begin + duration)
-            srt += '%s\r\n%s --> %s\r\n%s\r\n\r\n' % (compat_str(pos), starttime, endtime, p.text)
         return {lang: [
             {'ext': 'ttml', 'url': url},
-            {'ext': 'srt', 'data': srt},
         ]}
 
     def _extract_f4m(self, manifest_url, video_id):

From c1c924abfeda45f29b991bb74f315f0e79dcf126 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 12 May 2015 13:04:54 +0800
Subject: [PATCH 1274/6219] [utils,common] Merge format_srt_time and
 _subtitles_timecode

format_srt_time uses a comma as the delimiter between seconds and
milliseconds while _subtitles_timecode uses a dot. All .srt examples I
found on the Internet uses a comma, so I use a comma in the merged
version. See http://matroska.org/technical/specs/subtitles/srt.html and
http://devel.aegisub.org/wiki/SubtitleFormats/SRT
---
 youtube_dl/extractor/common.py    |  3 ---
 youtube_dl/extractor/kanalplay.py |  5 +++--
 youtube_dl/utils.py               | 12 ++++--------
 3 files changed, 7 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 981e34bc7..65bb77086 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1072,9 +1072,6 @@ class InfoExtractor(object):
     def _get_automatic_captions(self, *args, **kwargs):
         raise NotImplementedError("This method must be implemented by subclasses")
 
-    def _subtitles_timecode(self, seconds):
-        return '%02d:%02d:%02d.%03d' % (seconds / 3600, (seconds % 3600) / 60, seconds % 60, (seconds % 1) * 1000)
-
 
 class SearchInfoExtractor(InfoExtractor):
     """
diff --git a/youtube_dl/extractor/kanalplay.py b/youtube_dl/extractor/kanalplay.py
index 2bb078036..4597d1b96 100644
--- a/youtube_dl/extractor/kanalplay.py
+++ b/youtube_dl/extractor/kanalplay.py
@@ -7,6 +7,7 @@ from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
     float_or_none,
+    srt_subtitles_timecode,
 )
 
 
@@ -39,8 +40,8 @@ class KanalPlayIE(InfoExtractor):
             '%s\r\n%s --> %s\r\n%s'
             % (
                 num,
-                self._subtitles_timecode(item['startMillis'] / 1000.0),
-                self._subtitles_timecode(item['endMillis'] / 1000.0),
+                srt_subtitles_timecode(item['startMillis'] / 1000.0),
+                srt_subtitles_timecode(item['endMillis'] / 1000.0),
                 item['text'],
             ) for num, item in enumerate(subs, 1))
 
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 5439fcb35..ed9ed9ed6 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1835,12 +1835,8 @@ def parse_dfxp_time_expr(time_expr):
         return 3600 * int(mobj.group(1)) + 60 * int(mobj.group(2)) + float(mobj.group(3))
 
 
-def format_srt_time(seconds):
-    (mins, secs) = divmod(seconds, 60)
-    (hours, mins) = divmod(mins, 60)
-    millisecs = (secs - int(secs)) * 1000
-    secs = int(secs)
-    return '%02d:%02d:%02d,%03d' % (hours, mins, secs, millisecs)
+def srt_subtitles_timecode(seconds):
+    return '%02d:%02d:%02d,%03d' % (seconds / 3600, (seconds % 3600) / 60, seconds % 60, (seconds % 1) * 1000)
 
 
 def dfxp2srt(dfxp_data):
@@ -1872,8 +1868,8 @@ def dfxp2srt(dfxp_data):
             end_time = begin_time + parse_dfxp_time_expr(para.attrib['dur'])
         out.append('%d\n%s --> %s\n%s\n\n' % (
             index,
-            format_srt_time(begin_time),
-            format_srt_time(end_time),
+            srt_subtitles_timecode(begin_time),
+            srt_subtitles_timecode(end_time),
             parse_node(para)))
 
     return ''.join(out)

From 41333b97b9471316cf0f395db59196e6571fc776 Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Tue, 12 May 2015 22:35:16 +0800
Subject: [PATCH 1275/6219] [qqmusic] Add support for charts / top lists

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/qqmusic.py  | 55 ++++++++++++++++++++++++++++++++
 2 files changed, 56 insertions(+)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index de19dfd7a..8ec0c1032 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -414,6 +414,7 @@ from .qqmusic import (
     QQMusicIE,
     QQMusicSingerIE,
     QQMusicAlbumIE,
+    QQMusicToplistIE,
 )
 from .quickvid import QuickVidIE
 from .r7 import R7IE
diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index 174c8e0ae..d4a85d8c3 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -9,6 +9,7 @@ from .common import InfoExtractor
 from ..utils import (
     strip_jsonp,
     unescapeHTML,
+    js_to_json,
 )
 from ..compat import compat_urllib_request
 
@@ -168,3 +169,57 @@ class QQMusicAlbumIE(QQPlaylistBaseIE):
             album_page, 'album details', default=None)
 
         return self.playlist_result(entries, mid, album_name, album_detail)
+
+
+class QQMusicToplistIE(QQPlaylistBaseIE):
+    _VALID_URL = r'http://y\.qq\.com/#type=toplist&p=(?P<id>(top|global)_[0-9]+)'
+    
+    _TESTS = [{
+        'url': 'http://y.qq.com/#type=toplist&p=global_12',
+        'info_dict': {
+            'id': 'global_12',
+            'title': 'itunes榜',
+        },
+        'playlist_count': 10,
+    }, {
+        'url': 'http://y.qq.com/#type=toplist&p=top_6',
+        'info_dict': {
+            'id': 'top_6',
+            'title': 'QQ音乐巅峰榜·欧美',
+        },
+        'playlist_count': 100,
+    }]
+
+    @staticmethod
+    def strip_qq_jsonp(code):
+        return js_to_json(re.sub(r'^MusicJsonCallback\((.*?)\)/\*.+?\*/$', r'\1', code))
+    
+    def _real_extract(self, url):
+        list_id = self._match_id(url)
+
+        list_type = list_id.split("_")[0]
+        num_id = list_id.split("_")[1]
+
+        list_page = self._download_webpage("http://y.qq.com/y/static/toplist/index/%s.html" % list_id, list_id, 'Download toplist page')
+        entries = []
+        if list_type == 'top':
+            list = self._download_json(
+                "http://y.qq.com/y/static/toplist/json/top/%s/1.js" % num_id,
+                list_id, note='Retrieve toplist json', errnote='Unable to get toplist json', transform_source=self.strip_qq_jsonp)
+
+            for song in list['l']:
+                s = song['s']
+                song_mid = s.split("|")[20]
+                entries.append(self.url_result(
+                    'http://y.qq.com/#type=song&mid=' + song_mid, 'QQMusic',
+                    song_mid))
+
+        elif list_type == 'global':
+            entries = self.get_entries_from_page(list_page)
+
+        list_name = self._html_search_regex(
+            r'<h2 id="top_name">([^\']+)</h2>', list_page, 'top list name',
+            default=None)
+        list_desc = None
+
+        return self.playlist_result(entries, list_id, list_name, list_desc)
\ No newline at end of file

From b480e7874b45862eae343ab8484aa43381cd28fa Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Tue, 12 May 2015 22:41:37 +0800
Subject: [PATCH 1276/6219] [qqmusic] Fix code formatting

---
 youtube_dl/extractor/qqmusic.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index d4a85d8c3..bca4a8f90 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -200,12 +200,15 @@ class QQMusicToplistIE(QQPlaylistBaseIE):
         list_type = list_id.split("_")[0]
         num_id = list_id.split("_")[1]
 
-        list_page = self._download_webpage("http://y.qq.com/y/static/toplist/index/%s.html" % list_id, list_id, 'Download toplist page')
+        list_page = self._download_webpage(
+            "http://y.qq.com/y/static/toplist/index/%s.html" % list_id, 
+            list_id, 'Download toplist page')
         entries = []
         if list_type == 'top':
             list = self._download_json(
                 "http://y.qq.com/y/static/toplist/json/top/%s/1.js" % num_id,
-                list_id, note='Retrieve toplist json', errnote='Unable to get toplist json', transform_source=self.strip_qq_jsonp)
+                list_id, note='Retrieve toplist json', errnote='Unable to get toplist json', 
+                transform_source=self.strip_qq_jsonp)
 
             for song in list['l']:
                 s = song['s']
@@ -222,4 +225,5 @@ class QQMusicToplistIE(QQPlaylistBaseIE):
             default=None)
         list_desc = None
 
-        return self.playlist_result(entries, list_id, list_name, list_desc)
\ No newline at end of file
+        return self.playlist_result(entries, list_id, list_name, list_desc)
+        
\ No newline at end of file

From fd4eefed39595850b864d3be9711224e4e8e9dd4 Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Wed, 13 May 2015 01:14:02 +0800
Subject: [PATCH 1277/6219] [qqmusic] Fix extraction for global list

---
 youtube_dl/extractor/qqmusic.py | 34 ++++++++++++++++++++-------------
 1 file changed, 21 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index bca4a8f90..3401dcaef 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -188,6 +188,13 @@ class QQMusicToplistIE(QQPlaylistBaseIE):
             'title': 'QQ音乐巅峰榜·欧美',
         },
         'playlist_count': 100,
+    }, {
+        'url': 'http://y.qq.com/#type=toplist&p=global_5',
+        'info_dict': {
+            'id': 'global_5',
+            'title': '韩国mnet排行榜',
+        },
+        'playlist_count': 50,
     }]
 
     @staticmethod
@@ -203,22 +210,23 @@ class QQMusicToplistIE(QQPlaylistBaseIE):
         list_page = self._download_webpage(
             "http://y.qq.com/y/static/toplist/index/%s.html" % list_id, 
             list_id, 'Download toplist page')
+
         entries = []
+        jsonp_url = ""
         if list_type == 'top':
-            list = self._download_json(
-                "http://y.qq.com/y/static/toplist/json/top/%s/1.js" % num_id,
-                list_id, note='Retrieve toplist json', errnote='Unable to get toplist json', 
-                transform_source=self.strip_qq_jsonp)
-
-            for song in list['l']:
-                s = song['s']
-                song_mid = s.split("|")[20]
-                entries.append(self.url_result(
-                    'http://y.qq.com/#type=song&mid=' + song_mid, 'QQMusic',
-                    song_mid))
-
+            jsonp_url = "http://y.qq.com/y/static/toplist/json/top/%s/1.js" % num_id
         elif list_type == 'global':
-            entries = self.get_entries_from_page(list_page)
+            jsonp_url = "http://y.qq.com/y/static/toplist/json/global/%s/1_1.js" % num_id
+        
+        list = self._download_json(jsonp_url, list_id, note='Retrieve toplist json', 
+            errnote='Unable to get toplist json', transform_source=self.strip_qq_jsonp)
+            
+        for song in list['l']:
+            s = song['s']
+            song_mid = s.split("|")[20]
+            entries.append(self.url_result(
+                'http://y.qq.com/#type=song&mid=' + song_mid, 'QQMusic',
+                song_mid))
 
         list_name = self._html_search_regex(
             r'<h2 id="top_name">([^\']+)</h2>', list_page, 'top list name',

From 86ec1e487c4908f4d0d0ece512007a2e5fedc593 Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Wed, 13 May 2015 01:37:56 +0800
Subject: [PATCH 1278/6219] [qqmusic] Code fixes

---
 youtube_dl/extractor/qqmusic.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index 3401dcaef..bae2ce31a 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -212,10 +212,9 @@ class QQMusicToplistIE(QQPlaylistBaseIE):
             list_id, 'Download toplist page')
 
         entries = []
-        jsonp_url = ""
         if list_type == 'top':
             jsonp_url = "http://y.qq.com/y/static/toplist/json/top/%s/1.js" % num_id
-        elif list_type == 'global':
+        else:
             jsonp_url = "http://y.qq.com/y/static/toplist/json/global/%s/1_1.js" % num_id
         
         list = self._download_json(jsonp_url, list_id, note='Retrieve toplist json', 

From 0b4253fa3710c656e12b6147ed7c1f7843bb9aae Mon Sep 17 00:00:00 2001
From: blissland <blissland.house@googlemail.com>
Date: Tue, 12 May 2015 18:57:06 +0100
Subject: [PATCH 1279/6219] [BYUtvIE] Change thumbnail regex so test does not
 fail

---
 youtube_dl/extractor/byutv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/byutv.py b/youtube_dl/extractor/byutv.py
index 6252be05b..3b2de517e 100644
--- a/youtube_dl/extractor/byutv.py
+++ b/youtube_dl/extractor/byutv.py
@@ -16,7 +16,7 @@ class BYUtvIE(InfoExtractor):
             'ext': 'mp4',
             'description': 'md5:5438d33774b6bdc662f9485a340401cc',
             'title': 'Season 5 Episode 5',
-            'thumbnail': 're:^https?://.*promo.*'
+            'thumbnail': 're:^https?://.*\.jpg$'
         },
         'params': {
             'skip_download': True,

From 3749e36e9f0e6be2a3a3ab1b15c0c02be5a50e2f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 13 May 2015 21:16:45 +0800
Subject: [PATCH 1280/6219] [YoutubeDL] Fix PEP8 W503

---
 youtube_dl/YoutubeDL.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 4cf83c510..84d50dab7 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1085,9 +1085,9 @@ class YoutubeDL(object):
         req_format = self.params.get('format')
         if req_format is None:
             req_format_list = []
-            if (self.params.get('outtmpl', DEFAULT_OUTTMPL) != '-'
-                    and info_dict['extractor'] in ['youtube', 'ted']
-                    and FFmpegMergerPP(self).available):
+            if (self.params.get('outtmpl', DEFAULT_OUTTMPL) != '-' and
+                    info_dict['extractor'] in ['youtube', 'ted'] and
+                    FFmpegMergerPP(self).available):
                 req_format_list.append('bestvideo+bestaudio')
             req_format_list.append('best')
             req_format = '/'.join(req_format_list)

From 372744c544ec3de1b35583e7d6fc2cbc4cc39f3a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 13 May 2015 22:26:30 +0600
Subject: [PATCH 1281/6219] [odnoklassniki] Fix extraction (Closes #5671)

---
 youtube_dl/extractor/odnoklassniki.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/odnoklassniki.py b/youtube_dl/extractor/odnoklassniki.py
index 155d0ee6a..fbc521d1a 100644
--- a/youtube_dl/extractor/odnoklassniki.py
+++ b/youtube_dl/extractor/odnoklassniki.py
@@ -6,6 +6,7 @@ from ..utils import (
     unified_strdate,
     int_or_none,
     qualities,
+    unescapeHTML,
 )
 
 
@@ -36,8 +37,8 @@ class OdnoklassnikiIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         player = self._parse_json(
-            self._search_regex(
-                r"OKVideo\.start\(({.+?})\s*,\s*'VideoAutoplay_player'", webpage, 'player'),
+            unescapeHTML(self._search_regex(
+                r'data-attributes="([^"]+)"', webpage, 'player')),
             video_id)
 
         metadata = self._parse_json(player['flashvars']['metadata'], video_id)

From 8e595397529abc71093264e3695fb00d95be4d78 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 14 May 2015 02:32:00 +0800
Subject: [PATCH 1282/6219] [postprocessor/embedthumbnail] Use thumbnails
 downloaded by YoutubeDL

---
 youtube_dl/YoutubeDL.py                    |  2 +-
 youtube_dl/__init__.py                     |  8 +++++++-
 youtube_dl/postprocessor/embedthumbnail.py | 23 +++++++++++-----------
 3 files changed, 20 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 84d50dab7..0fbfe9642 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1848,7 +1848,7 @@ class YoutubeDL(object):
             thumb_ext = determine_ext(t['url'], 'jpg')
             suffix = '_%s' % t['id'] if len(thumbnails) > 1 else ''
             thumb_display_id = '%s ' % t['id'] if len(thumbnails) > 1 else ''
-            thumb_filename = os.path.splitext(filename)[0] + suffix + '.' + thumb_ext
+            t['filename'] = thumb_filename = os.path.splitext(filename)[0] + suffix + '.' + thumb_ext
 
             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(thumb_filename)):
                 self.to_screen('[%s] %s: Thumbnail %sis already present' %
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 9cc9f851f..ace17857c 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -240,7 +240,13 @@ def _real_main(argv=None):
     if opts.xattrs:
         postprocessors.append({'key': 'XAttrMetadata'})
     if opts.embedthumbnail:
-        postprocessors.append({'key': 'EmbedThumbnail'})
+        already_have_thumbnail = opts.writethumbnail or opts.write_all_thumbnails
+        postprocessors.append({
+            'key': 'EmbedThumbnail',
+            'already_have_thumbnail': already_have_thumbnail
+        })
+        if not already_have_thumbnail:
+            opts.writethumbnail = True
     # Please keep ExecAfterDownload towards the bottom as it allows the user to modify the final file in any way.
     # So if the user is able to remove the file before your postprocessor runs it might cause a few problems.
     if opts.exec_cmd:
diff --git a/youtube_dl/postprocessor/embedthumbnail.py b/youtube_dl/postprocessor/embedthumbnail.py
index 4868a42fd..95c52f65f 100644
--- a/youtube_dl/postprocessor/embedthumbnail.py
+++ b/youtube_dl/postprocessor/embedthumbnail.py
@@ -7,11 +7,7 @@ import subprocess
 
 from .ffmpeg import FFmpegPostProcessor
 
-from ..compat import (
-    compat_urlretrieve,
-)
 from ..utils import (
-    determine_ext,
     check_executable,
     encodeFilename,
     PostProcessingError,
@@ -25,26 +21,30 @@ class EmbedThumbnailPPError(PostProcessingError):
 
 
 class EmbedThumbnailPP(FFmpegPostProcessor):
+    def __init__(self, downloader=None, already_have_thumbnail=False):
+        super(EmbedThumbnailPP, self).__init__(downloader)
+        self._already_have_thumbnail = already_have_thumbnail
+
     def run(self, info):
         filename = info['filepath']
         temp_filename = prepend_extension(filename, 'temp')
-        temp_thumbnail = filename + '.' + determine_ext(info['thumbnail'])
 
-        if not info.get('thumbnail'):
+        if not info.get('thumbnails'):
             raise EmbedThumbnailPPError('Thumbnail was not found. Nothing to do.')
 
-        compat_urlretrieve(info['thumbnail'], temp_thumbnail)
+        thumbnail_filename = info['thumbnails'][-1]['filename']
 
         if info['ext'] == 'mp3':
             options = [
-                '-i', temp_thumbnail, '-c', 'copy', '-map', '0', '-map', '1',
+                '-i', thumbnail_filename, '-c', 'copy', '-map', '0', '-map', '1',
                 '-metadata:s:v', 'title="Album cover"', '-metadata:s:v', 'comment="Cover (Front)"']
 
             self._downloader.to_screen('[ffmpeg] Adding thumbnail to "%s"' % filename)
 
             self.run_ffmpeg(filename, temp_filename, options)
 
-            os.remove(encodeFilename(temp_thumbnail))
+            if not self._already_have_thumbnail:
+                os.remove(encodeFilename(thumbnail_filename))
             os.remove(encodeFilename(filename))
             os.rename(encodeFilename(temp_filename), encodeFilename(filename))
 
@@ -52,7 +52,7 @@ class EmbedThumbnailPP(FFmpegPostProcessor):
             if not check_executable('AtomicParsley', ['-v']):
                 raise EmbedThumbnailPPError('AtomicParsley was not found. Please install.')
 
-            cmd = ['AtomicParsley', filename, '--artwork', temp_thumbnail, '-o', temp_filename]
+            cmd = ['AtomicParsley', filename, '--artwork', thumbnail_filename, '-o', temp_filename]
 
             self._downloader.to_screen('[atomicparsley] Adding thumbnail to "%s"' % filename)
 
@@ -66,7 +66,8 @@ class EmbedThumbnailPP(FFmpegPostProcessor):
                 msg = stderr.decode('utf-8', 'replace').strip()
                 raise EmbedThumbnailPPError(msg)
 
-            os.remove(encodeFilename(temp_thumbnail))
+            if not self._already_have_thumbnail:
+                os.remove(encodeFilename(thumbnail_filename))
             # for formats that don't support thumbnails (like 3gp) AtomicParsley
             # won't create to the temporary file
             if b'No changes' in stdout:

From bb8ca1d112e95cd3fe48fff5af980a62a9db2572 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 14 May 2015 02:35:28 +0800
Subject: [PATCH 1283/6219] [postprocessor/embedthumbnail] Use
 run_ffmpeg_multiple_files

---
 youtube_dl/postprocessor/embedthumbnail.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/postprocessor/embedthumbnail.py b/youtube_dl/postprocessor/embedthumbnail.py
index 95c52f65f..4e08c2709 100644
--- a/youtube_dl/postprocessor/embedthumbnail.py
+++ b/youtube_dl/postprocessor/embedthumbnail.py
@@ -36,12 +36,12 @@ class EmbedThumbnailPP(FFmpegPostProcessor):
 
         if info['ext'] == 'mp3':
             options = [
-                '-i', thumbnail_filename, '-c', 'copy', '-map', '0', '-map', '1',
+                '-c', 'copy', '-map', '0', '-map', '1',
                 '-metadata:s:v', 'title="Album cover"', '-metadata:s:v', 'comment="Cover (Front)"']
 
             self._downloader.to_screen('[ffmpeg] Adding thumbnail to "%s"' % filename)
 
-            self.run_ffmpeg(filename, temp_filename, options)
+            self.run_ffmpeg_multiple_files([filename, thumbnail_filename], temp_filename, options)
 
             if not self._already_have_thumbnail:
                 os.remove(encodeFilename(thumbnail_filename))

From 2cc6d135479c5dbd6e715a1e767c5be163cd22ce Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 14 May 2015 04:41:30 +0800
Subject: [PATCH 1284/6219] [postprocessor/embedthumbnail] Encode arguments in
 calling AtomicParsley

---
 youtube_dl/postprocessor/embedthumbnail.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/postprocessor/embedthumbnail.py b/youtube_dl/postprocessor/embedthumbnail.py
index 4e08c2709..8f825f785 100644
--- a/youtube_dl/postprocessor/embedthumbnail.py
+++ b/youtube_dl/postprocessor/embedthumbnail.py
@@ -9,6 +9,7 @@ from .ffmpeg import FFmpegPostProcessor
 
 from ..utils import (
     check_executable,
+    encodeArgument,
     encodeFilename,
     PostProcessingError,
     prepend_extension,
@@ -52,7 +53,12 @@ class EmbedThumbnailPP(FFmpegPostProcessor):
             if not check_executable('AtomicParsley', ['-v']):
                 raise EmbedThumbnailPPError('AtomicParsley was not found. Please install.')
 
-            cmd = ['AtomicParsley', filename, '--artwork', thumbnail_filename, '-o', temp_filename]
+            cmd = [encodeFilename('AtomicParsley', True),
+                   encodeFilename(filename, True),
+                   encodeArgument('--artwork'),
+                   encodeFilename(thumbnail_filename, True),
+                   encodeArgument('-o'),
+                   encodeFilename(temp_filename, True)]
 
             self._downloader.to_screen('[atomicparsley] Adding thumbnail to "%s"' % filename)
 

From 86c7fdb17c0dcbff88a8daa131fddc57b6304b83 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 14 May 2015 14:26:47 +0800
Subject: [PATCH 1285/6219] [xattr] Enhance error handling to catch ENOSPC

Fixes #5589
---
 youtube_dl/postprocessor/xattrpp.py | 61 +++++++++++++++++++++++------
 1 file changed, 50 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/postprocessor/xattrpp.py b/youtube_dl/postprocessor/xattrpp.py
index 93d0abcf6..16f2966e9 100644
--- a/youtube_dl/postprocessor/xattrpp.py
+++ b/youtube_dl/postprocessor/xattrpp.py
@@ -3,18 +3,32 @@ from __future__ import unicode_literals
 import os
 import subprocess
 import sys
+import errno
 
 from .common import PostProcessor
-from ..compat import (
-    subprocess_check_output
-)
 from ..utils import (
     check_executable,
     hyphenate_date,
     version_tuple,
+    PostProcessingError,
+    encodeArgument,
+    encodeFilename,
 )
 
 
+class XAttrMetadataError(PostProcessingError):
+    def __init__(self, code=None, msg='Unknown error'):
+        super(XAttrMetadataError, self).__init__(msg)
+        self.code = code
+
+        # Parsing code and msg
+        if (self.code in (errno.ENOSPC, errno.EDQUOT) or
+                'No space left' in self.msg or 'Disk quota excedded' in self.msg):
+            self.reason = 'NO_SPACE'
+        else:
+            self.reason = 'NOT_SUPPORTED'
+
+
 class XAttrMetadataPP(PostProcessor):
 
     #
@@ -51,7 +65,10 @@ class XAttrMetadataPP(PostProcessor):
                 raise ImportError
 
             def write_xattr(path, key, value):
-                return xattr.setxattr(path, key, value)
+                try:
+                    xattr.set(path, key, value)
+                except EnvironmentError as e:
+                    raise XAttrMetadataError(e.errno, e.strerror)
 
         except ImportError:
             if os.name == 'nt':
@@ -62,8 +79,11 @@ class XAttrMetadataPP(PostProcessor):
                     assert os.path.exists(path)
 
                     ads_fn = path + ":" + key
-                    with open(ads_fn, "wb") as f:
-                        f.write(value)
+                    try:
+                        with open(ads_fn, "wb") as f:
+                            f.write(value)
+                    except EnvironmentError as e:
+                        raise XAttrMetadataError(e.errno, e.strerror)
             else:
                 user_has_setfattr = check_executable("setfattr", ['--version'])
                 user_has_xattr = check_executable("xattr", ['-h'])
@@ -71,12 +91,24 @@ class XAttrMetadataPP(PostProcessor):
                 if user_has_setfattr or user_has_xattr:
 
                     def write_xattr(path, key, value):
+                        value = value.decode('utf-8')
                         if user_has_setfattr:
-                            cmd = ['setfattr', '-n', key, '-v', value, path]
+                            executable = 'setfattr'
+                            opts = ['-n', key, '-v', value]
                         elif user_has_xattr:
-                            cmd = ['xattr', '-w', key, value, path]
+                            executable = 'xattr'
+                            opts = ['-w', key, value]
 
-                        subprocess_check_output(cmd)
+                        cmd = ([encodeFilename(executable, True)] +
+                               [encodeArgument(o) for o in opts] +
+                               [encodeFilename(path, True)])
+
+                        p = subprocess.Popen(
+                            cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
+                        stdout, stderr = p.communicate()
+                        stderr = stderr.decode('utf-8', 'replace')
+                        if p.returncode != 0:
+                            raise XAttrMetadataError(p.returncode, stderr)
 
                 else:
                     # On Unix, and can't find pyxattr, setfattr, or xattr.
@@ -121,6 +153,13 @@ class XAttrMetadataPP(PostProcessor):
 
             return [], info
 
-        except (subprocess.CalledProcessError, OSError):
-            self._downloader.report_error("This filesystem doesn't support extended attributes. (You may have to enable them in your /etc/fstab)")
+        except XAttrMetadataError as e:
+            if e.reason == 'NO_SPACE':
+                self._downloader.report_warning(
+                    'There\'s no disk space left or disk quota exceeded. ' +
+                    'Extended attributes are not written.')
+            else:
+                self._downloader.report_error(
+                    'This filesystem doesn\'t support extended attributes. ' +
+                    '(You may have to enable them in your /etc/fstab)')
             return [], info

From fbff30d2dbc6462c628384ea5960c2461e7cdcca Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 14 May 2015 14:51:00 +0800
Subject: [PATCH 1286/6219] [xattr] Catch 'Argument list too long'

---
 youtube_dl/postprocessor/xattrpp.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/postprocessor/xattrpp.py b/youtube_dl/postprocessor/xattrpp.py
index 16f2966e9..27e273000 100644
--- a/youtube_dl/postprocessor/xattrpp.py
+++ b/youtube_dl/postprocessor/xattrpp.py
@@ -25,6 +25,8 @@ class XAttrMetadataError(PostProcessingError):
         if (self.code in (errno.ENOSPC, errno.EDQUOT) or
                 'No space left' in self.msg or 'Disk quota excedded' in self.msg):
             self.reason = 'NO_SPACE'
+        elif self.code == errno.E2BIG or 'Argument list too long' in self.msg:
+            self.reason = 'VALUE_TOO_LONG'
         else:
             self.reason = 'NOT_SUPPORTED'
 
@@ -103,8 +105,11 @@ class XAttrMetadataPP(PostProcessor):
                                [encodeArgument(o) for o in opts] +
                                [encodeFilename(path, True)])
 
-                        p = subprocess.Popen(
-                            cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
+                        try:
+                            p = subprocess.Popen(
+                                cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
+                        except EnvironmentError as e:
+                            raise XAttrMetadataError(e.errno, e.strerror)
                         stdout, stderr = p.communicate()
                         stderr = stderr.decode('utf-8', 'replace')
                         if p.returncode != 0:
@@ -158,6 +163,9 @@ class XAttrMetadataPP(PostProcessor):
                 self._downloader.report_warning(
                     'There\'s no disk space left or disk quota exceeded. ' +
                     'Extended attributes are not written.')
+            elif e.reason == 'VALUE_TOO_LONG':
+                self._downloader.report_warning(
+                    'Unable to write extended attributes due to too long values.')
             else:
                 self._downloader.report_error(
                     'This filesystem doesn\'t support extended attributes. ' +

From 509c630db8cdaff473f95805cda1ae350107e36b Mon Sep 17 00:00:00 2001
From: blissland <blissland.house@googlemail.com>
Date: Thu, 14 May 2015 08:09:56 +0100
Subject: [PATCH 1287/6219] [CanalplusIE] Update tests that were no longer
 working

---
 youtube_dl/extractor/canalplus.py | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/canalplus.py b/youtube_dl/extractor/canalplus.py
index 1b14471e5..71801488a 100644
--- a/youtube_dl/extractor/canalplus.py
+++ b/youtube_dl/extractor/canalplus.py
@@ -25,14 +25,13 @@ class CanalplusIE(InfoExtractor):
     }
 
     _TESTS = [{
-        'url': 'http://www.canalplus.fr/c-infos-documentaires/pid1830-c-zapping.html?vid=922470',
-        'md5': '3db39fb48b9685438ecf33a1078023e4',
+        'url': 'http://www.canalplus.fr/c-emissions/pid1830-c-zapping.html?vid=1263092',
         'info_dict': {
-            'id': '922470',
+            'id': '1263092',
             'ext': 'flv',
-            'title': 'Zapping - 26/08/13',
-            'description': 'Le meilleur de toutes les chaînes, tous les jours.\nEmission du 26 août 2013',
-            'upload_date': '20130826',
+            'title': 'Le Zapping - 13/05/15',
+            'description': 'md5:09738c0d06be4b5d06a0940edb0da73f',
+            'upload_date': '20150513',
         },
     }, {
         'url': 'http://www.piwiplus.fr/videos-piwi/pid1405-le-labyrinthe-boing-super-ranger.html?vid=1108190',
@@ -56,7 +55,6 @@ class CanalplusIE(InfoExtractor):
         'skip': 'videos get deleted after a while',
     }, {
         'url': 'http://www.itele.fr/france/video/aubervilliers-un-lycee-en-colere-111559',
-        'md5': '65aa83ad62fe107ce29e564bb8712580',
         'info_dict': {
             'id': '1213714',
             'ext': 'flv',

From c827d4cfdb9ce47d13ccbec32d2b32dfb429ea8a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 14 May 2015 16:53:10 +0800
Subject: [PATCH 1288/6219] [xattr] Enhanced error messages on Windows

---
 youtube_dl/postprocessor/xattrpp.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/postprocessor/xattrpp.py b/youtube_dl/postprocessor/xattrpp.py
index 27e273000..7d88e1308 100644
--- a/youtube_dl/postprocessor/xattrpp.py
+++ b/youtube_dl/postprocessor/xattrpp.py
@@ -167,7 +167,10 @@ class XAttrMetadataPP(PostProcessor):
                 self._downloader.report_warning(
                     'Unable to write extended attributes due to too long values.')
             else:
-                self._downloader.report_error(
-                    'This filesystem doesn\'t support extended attributes. ' +
-                    '(You may have to enable them in your /etc/fstab)')
+                msg = 'This filesystem doesn\'t support extended attributes. '
+                if os.name == 'nt':
+                    msg += 'You need to use NTFS.'
+                else:
+                    msg += '(You may have to enable them in your /etc/fstab)'
+                self._downloader.report_error(msg)
             return [], info

From 7d57d2e18be416faa593364966ccf667243fd3ea Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 14 May 2015 14:59:27 +0600
Subject: [PATCH 1289/6219] [canalplus] Restore checksums in tests

---
 youtube_dl/extractor/canalplus.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/canalplus.py b/youtube_dl/extractor/canalplus.py
index 71801488a..699b4f7d0 100644
--- a/youtube_dl/extractor/canalplus.py
+++ b/youtube_dl/extractor/canalplus.py
@@ -26,6 +26,7 @@ class CanalplusIE(InfoExtractor):
 
     _TESTS = [{
         'url': 'http://www.canalplus.fr/c-emissions/pid1830-c-zapping.html?vid=1263092',
+        'md5': 'b3481d7ca972f61e37420798d0a9d934',
         'info_dict': {
             'id': '1263092',
             'ext': 'flv',
@@ -55,6 +56,7 @@ class CanalplusIE(InfoExtractor):
         'skip': 'videos get deleted after a while',
     }, {
         'url': 'http://www.itele.fr/france/video/aubervilliers-un-lycee-en-colere-111559',
+        'md5': 'f3a46edcdf28006598ffaf5b30e6a2d4',
         'info_dict': {
             'id': '1213714',
             'ext': 'flv',

From 82245a6de77f4755b063310258c5611c15f5ffbd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 14 May 2015 15:21:27 +0600
Subject: [PATCH 1290/6219] [YoutubeDL] Restore filename for thumbnails

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 4c8196d08..691f3e09f 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1849,7 +1849,7 @@ class YoutubeDL(object):
             thumb_ext = determine_ext(t['url'], 'jpg')
             suffix = '_%s' % t['id'] if len(thumbnails) > 1 else ''
             thumb_display_id = '%s ' % t['id'] if len(thumbnails) > 1 else ''
-            thumb_filename = os.path.splitext(filename)[0] + suffix + '.' + thumb_ext
+            t['filename'] = thumb_filename = os.path.splitext(filename)[0] + suffix + '.' + thumb_ext
 
             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(thumb_filename)):
                 self.to_screen('[%s] %s: Thumbnail %sis already present' %

From fa6a16996e4a1aeee4e421b172efc6c351b1b123 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 14 May 2015 18:00:57 +0800
Subject: [PATCH 1291/6219] [worldstarhiphop] Support Android URLs (fixes
 #5629)

---
 youtube_dl/extractor/worldstarhiphop.py | 23 +++++++++++++++++------
 1 file changed, 17 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/worldstarhiphop.py b/youtube_dl/extractor/worldstarhiphop.py
index d5c26a032..a3ea26feb 100644
--- a/youtube_dl/extractor/worldstarhiphop.py
+++ b/youtube_dl/extractor/worldstarhiphop.py
@@ -6,8 +6,8 @@ from .common import InfoExtractor
 
 
 class WorldStarHipHopIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www|m)\.worldstar(?:candy|hiphop)\.com/videos/video\.php\?v=(?P<id>.*)'
-    _TEST = {
+    _VALID_URL = r'https?://(?:www|m)\.worldstar(?:candy|hiphop)\.com/(?:videos|android)/video\.php\?v=(?P<id>.*)'
+    _TESTS = [{
         "url": "http://www.worldstarhiphop.com/videos/video.php?v=wshh6a7q1ny0G34ZwuIO",
         "md5": "9d04de741161603bf7071bbf4e883186",
         "info_dict": {
@@ -15,7 +15,15 @@ class WorldStarHipHopIE(InfoExtractor):
             "ext": "mp4",
             "title": "KO Of The Week: MMA Fighter Gets Knocked Out By Swift Head Kick!"
         }
-    }
+    }, {
+        'url': 'http://m.worldstarhiphop.com/android/video.php?v=wshh6a7q1ny0G34ZwuIO',
+        'md5': 'dc1c76c83ecc4190bb1eb143899b87d3',
+        'info_dict': {
+            'id': 'wshh6a7q1ny0G34ZwuIO',
+            'ext': 'mp4',
+            "title": "KO Of The Week: MMA Fighter Gets Knocked Out By Swift Head Kick!"
+        }
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -26,19 +34,22 @@ class WorldStarHipHopIE(InfoExtractor):
             return self.url_result('vevo:%s' % m_vevo_id.group(1), ie='Vevo')
 
         video_url = self._search_regex(
-            r'so\.addVariable\("file","(.*?)"\)', webpage, 'video URL')
+            [r'so\.addVariable\("file","(.*?)"\)',
+             r'<div class="artlist">\s*<a[^>]+href="([^"]+)">'],
+            webpage, 'video URL')
 
         if 'youtube' in video_url:
             return self.url_result(video_url, ie='Youtube')
 
         video_title = self._html_search_regex(
-            r'(?s)<div class="content-heading">\s*<h1>(.*?)</h1>',
+            [r'(?s)<div class="content-heading">\s*<h1>(.*?)</h1>',
+             r'<span[^>]+class="tc-sp-pinned-title">(.*)</span>'],
             webpage, 'title')
 
         # Getting thumbnail and if not thumbnail sets correct title for WSHH candy video.
         thumbnail = self._html_search_regex(
             r'rel="image_src" href="(.*)" />', webpage, 'thumbnail',
-            fatal=False)
+            default=None)
         if not thumbnail:
             _title = r'candytitles.*>(.*)</span>'
             mobj = re.search(_title, webpage)

From 7a012d5a16632a103466f9e9794dd98ad573ce88 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 14 May 2015 16:39:35 +0600
Subject: [PATCH 1292/6219] [screenwavemedia] Add support for player2 URLs
 (Closes #5696)

---
 youtube_dl/extractor/screenwavemedia.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/screenwavemedia.py b/youtube_dl/extractor/screenwavemedia.py
index 74fb1983a..d1ab66b32 100644
--- a/youtube_dl/extractor/screenwavemedia.py
+++ b/youtube_dl/extractor/screenwavemedia.py
@@ -11,7 +11,7 @@ from ..utils import (
 
 
 class ScreenwaveMediaIE(InfoExtractor):
-    _VALID_URL = r'http://player\.screenwavemedia\.com/play/[a-zA-Z]+\.php\?[^"]*\bid=(?P<id>.+)'
+    _VALID_URL = r'http://player\d?\.screenwavemedia\.com/(?:play/)?[a-zA-Z]+\.php\?[^"]*\bid=(?P<id>.+)'
 
     _TESTS = [{
         'url': 'http://player.screenwavemedia.com/play/play.php?playerdiv=videoarea&companiondiv=squareAd&id=Cinemassacre-19911',
@@ -20,7 +20,10 @@ class ScreenwaveMediaIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        playerdata = self._download_webpage(url, video_id, 'Downloading player webpage')
+
+        playerdata = self._download_webpage(
+            'http://player.screenwavemedia.com/play/player.php?id=%s' % video_id,
+            video_id, 'Downloading player webpage')
 
         vidtitle = self._search_regex(
             r'\'vidtitle\'\s*:\s*"([^"]+)"', playerdata, 'vidtitle').replace('\\/', '/')
@@ -99,7 +102,7 @@ class TeamFourIE(InfoExtractor):
         webpage = self._download_webpage(url, display_id)
 
         playerdata_url = self._search_regex(
-            r'src="(http://player\.screenwavemedia\.com/play/[a-zA-Z]+\.php\?[^"]*\bid=.+?)"',
+            r'src="(http://player\d?\.screenwavemedia\.com/(?:play/)?[a-zA-Z]+\.php\?[^"]*\bid=.+?)"',
             webpage, 'player data URL')
 
         video_title = self._html_search_regex(

From 548897396158d7822020f45c10301e9ca3c46453 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 14 May 2015 23:25:43 +0800
Subject: [PATCH 1293/6219] [qqmusic] flake8

---
 youtube_dl/extractor/qqmusic.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index bae2ce31a..5ddbb183e 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -173,7 +173,7 @@ class QQMusicAlbumIE(QQPlaylistBaseIE):
 
 class QQMusicToplistIE(QQPlaylistBaseIE):
     _VALID_URL = r'http://y\.qq\.com/#type=toplist&p=(?P<id>(top|global)_[0-9]+)'
-    
+
     _TESTS = [{
         'url': 'http://y.qq.com/#type=toplist&p=global_12',
         'info_dict': {
@@ -200,7 +200,7 @@ class QQMusicToplistIE(QQPlaylistBaseIE):
     @staticmethod
     def strip_qq_jsonp(code):
         return js_to_json(re.sub(r'^MusicJsonCallback\((.*?)\)/\*.+?\*/$', r'\1', code))
-    
+
     def _real_extract(self, url):
         list_id = self._match_id(url)
 
@@ -208,7 +208,7 @@ class QQMusicToplistIE(QQPlaylistBaseIE):
         num_id = list_id.split("_")[1]
 
         list_page = self._download_webpage(
-            "http://y.qq.com/y/static/toplist/index/%s.html" % list_id, 
+            "http://y.qq.com/y/static/toplist/index/%s.html" % list_id,
             list_id, 'Download toplist page')
 
         entries = []
@@ -216,10 +216,11 @@ class QQMusicToplistIE(QQPlaylistBaseIE):
             jsonp_url = "http://y.qq.com/y/static/toplist/json/top/%s/1.js" % num_id
         else:
             jsonp_url = "http://y.qq.com/y/static/toplist/json/global/%s/1_1.js" % num_id
-        
-        list = self._download_json(jsonp_url, list_id, note='Retrieve toplist json', 
+
+        list = self._download_json(
+            jsonp_url, list_id, note='Retrieve toplist json',
             errnote='Unable to get toplist json', transform_source=self.strip_qq_jsonp)
-            
+
         for song in list['l']:
             s = song['s']
             song_mid = s.split("|")[20]
@@ -233,4 +234,3 @@ class QQMusicToplistIE(QQPlaylistBaseIE):
         list_desc = None
 
         return self.playlist_result(entries, list_id, list_name, list_desc)
-        
\ No newline at end of file

From 29ea57283e473e94c72cf9cbc065c3c05a14830f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 14 May 2015 23:28:42 +0800
Subject: [PATCH 1294/6219] [qqmusic] Refactoring QQMusicToplistIE

---
 youtube_dl/extractor/qqmusic.py | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index 5ddbb183e..59e93a1dd 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -204,8 +204,7 @@ class QQMusicToplistIE(QQPlaylistBaseIE):
     def _real_extract(self, url):
         list_id = self._match_id(url)
 
-        list_type = list_id.split("_")[0]
-        num_id = list_id.split("_")[1]
+        list_type, num_id = list_id.split("_")
 
         list_page = self._download_webpage(
             "http://y.qq.com/y/static/toplist/index/%s.html" % list_id,
@@ -217,11 +216,11 @@ class QQMusicToplistIE(QQPlaylistBaseIE):
         else:
             jsonp_url = "http://y.qq.com/y/static/toplist/json/global/%s/1_1.js" % num_id
 
-        list = self._download_json(
+        toplist_json = self._download_json(
             jsonp_url, list_id, note='Retrieve toplist json',
             errnote='Unable to get toplist json', transform_source=self.strip_qq_jsonp)
 
-        for song in list['l']:
+        for song in toplist_json['l']:
             s = song['s']
             song_mid = s.split("|")[20]
             entries.append(self.url_result(
@@ -231,6 +230,5 @@ class QQMusicToplistIE(QQPlaylistBaseIE):
         list_name = self._html_search_regex(
             r'<h2 id="top_name">([^\']+)</h2>', list_page, 'top list name',
             default=None)
-        list_desc = None
 
-        return self.playlist_result(entries, list_id, list_name, list_desc)
+        return self.playlist_result(entries, list_id, list_name)

From 7ec676bb3dd6cba4b56fccb2d5aae08e66086b4e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 14 May 2015 23:32:36 +0800
Subject: [PATCH 1295/6219] [qqmusic] Add IE_NAME for all extractors

---
 youtube_dl/extractor/qqmusic.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index 59e93a1dd..13113820b 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -15,6 +15,7 @@ from ..compat import compat_urllib_request
 
 
 class QQMusicIE(InfoExtractor):
+    IE_NAME = 'qqmusic'
     _VALID_URL = r'http://y.qq.com/#type=song&mid=(?P<id>[0-9A-Za-z]+)'
     _TESTS = [{
         'url': 'http://y.qq.com/#type=song&mid=004295Et37taLD',
@@ -97,6 +98,7 @@ class QQPlaylistBaseIE(InfoExtractor):
 
 
 class QQMusicSingerIE(QQPlaylistBaseIE):
+    IE_NAME = 'qqmusic:singer'
     _VALID_URL = r'http://y.qq.com/#type=singer&mid=(?P<id>[0-9A-Za-z]+)'
     _TEST = {
         'url': 'http://y.qq.com/#type=singer&mid=001BLpXF2DyJe2',
@@ -140,6 +142,7 @@ class QQMusicSingerIE(QQPlaylistBaseIE):
 
 
 class QQMusicAlbumIE(QQPlaylistBaseIE):
+    IE_NAME = 'qqmusic:album'
     _VALID_URL = r'http://y.qq.com/#type=album&mid=(?P<id>[0-9A-Za-z]+)'
 
     _TEST = {
@@ -172,6 +175,7 @@ class QQMusicAlbumIE(QQPlaylistBaseIE):
 
 
 class QQMusicToplistIE(QQPlaylistBaseIE):
+    IE_NAME = 'qqmusic:toplist'
     _VALID_URL = r'http://y\.qq\.com/#type=toplist&p=(?P<id>(top|global)_[0-9]+)'
 
     _TESTS = [{

From 1ae72fb23df709687091133602fd715ab6cb7b2b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 14 May 2015 22:28:42 +0600
Subject: [PATCH 1296/6219] [soundcloud:user] Defer download link resolve
 (Closes #5248)

Looks like final download links can expire before downloading process reach them. So, resolving download links right before actual downloading.
---
 youtube_dl/extractor/soundcloud.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 183ff50f4..c23c5ee0f 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -336,7 +336,7 @@ class SoundcloudUserIE(SoundcloudIE):
             if len(new_entries) == 0:
                 self.to_screen('%s: End page received' % uploader)
                 break
-            entries.extend(self._extract_info_dict(e, quiet=True) for e in new_entries)
+            entries.extend(self.url_result(e['permalink_url'], 'Soundcloud') for e in new_entries)
 
         return {
             '_type': 'playlist',

From 3a105f7b20e8a3f742ac86cc1a6b02935b831778 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 15 May 2015 02:17:22 +0800
Subject: [PATCH 1297/6219] [teamcoco] Rewrite preload data extraction

Idea: "puncture" some consecutive fragments and check whether the
b64decode result of a punctured string is a valid JSON or not.

It's a O(N^3) algorithm, but should be fast for a small N (less than 30
fragments in all test cases)
---
 youtube_dl/extractor/teamcoco.py | 53 ++++++++++++++++----------------
 1 file changed, 26 insertions(+), 27 deletions(-)

diff --git a/youtube_dl/extractor/teamcoco.py b/youtube_dl/extractor/teamcoco.py
index 95d58ddd0..410eb7d3a 100644
--- a/youtube_dl/extractor/teamcoco.py
+++ b/youtube_dl/extractor/teamcoco.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 import base64
 import binascii
 import re
+import json
 
 from .common import InfoExtractor
 from ..utils import (
@@ -68,41 +69,39 @@ class TeamcocoIE(InfoExtractor):
             video_id = self._html_search_regex(
                 self._VIDEO_ID_REGEXES, webpage, 'video id')
 
-        data = preload = None
-        preloads = re.findall(r'"preload":\s*"([^"]+)"', webpage)
-        if preloads:
-            preload = max([(len(p), p) for p in preloads])[1]
+        data = None
 
-        if not preload:
-            preload = ''.join(re.findall(r'this\.push\("([^"]+)"\);', webpage))
+        preload_codes = self._html_search_regex(
+            r'(function.+)setTimeout\(function\(\)\{playlist',
+            webpage, 'preload codes')
+        base64_fragments = re.findall(r'"([a-zA-z0-9+/=]+)"', preload_codes)
+        base64_fragments.remove('init')
 
-        if not preload:
-            preload = self._html_search_regex([
-                r'player,\[?"([^"]+)"\]?', r'player.init\(\[?"([^"]+)"\]?\)'
-            ], webpage.replace('","', ''), 'preload data', default=None)
-
-        if not preload:
-            preload_codes = self._html_search_regex(
-                r'(function.+)setTimeout\(function\(\)\{playlist',
-                webpage, 'preload codes')
-            base64_fragments = re.findall(r'"([a-zA-z0-9+/=]+)"', preload_codes)
-            base64_fragments.remove('init')
-            for i in range(len(base64_fragments)):
-                cur_sequence = (''.join(base64_fragments[i:] + base64_fragments[:i])).encode('ascii')
+        def _check_sequence(cur_fragments):
+            if not cur_fragments:
+                return
+            for i in range(len(cur_fragments)):
+                cur_sequence = (''.join(cur_fragments[i:] + cur_fragments[:i])).encode('ascii')
                 try:
                     raw_data = base64.b64decode(cur_sequence)
-                except (TypeError, binascii.Error):
+                    if compat_ord(raw_data[0]) == compat_ord('{'):
+                        return json.loads(raw_data.decode('utf-8'))
+                except (TypeError, binascii.Error, UnicodeDecodeError, ValueError):
                     continue
-                if compat_ord(raw_data[0]) == compat_ord('{'):
-                    data = self._parse_json(raw_data.decode('utf-8'), video_id, fatal=False)
 
-        if not preload and not data:
-            raise ExtractorError(
-                'Preload information could not be extracted', expected=True)
+        def _check_data():
+            for i in range(len(base64_fragments) + 1):
+                for j in range(i, len(base64_fragments) + 1):
+                    data = _check_sequence(base64_fragments[:i] + base64_fragments[j:])
+                    if data:
+                        return data
+
+        self.to_screen('Try to compute possible data sequence. This may take some time.')
+        data = _check_data()
 
         if not data:
-            data = self._parse_json(
-                base64.b64decode(preload.encode('ascii')).decode('utf-8'), video_id)
+            raise ExtractorError(
+                'Preload information could not be extracted', expected=True)
 
         formats = []
         get_quality = qualities(['500k', '480p', '1000k', '720p', '1080p'])

From 12675275a1d2158fbe409361888569e4cb52ef07 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 15 May 2015 02:27:41 +0800
Subject: [PATCH 1298/6219] [teamcoco] Detect expired videos (#5626)

---
 youtube_dl/extractor/teamcoco.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/teamcoco.py b/youtube_dl/extractor/teamcoco.py
index 410eb7d3a..56be52638 100644
--- a/youtube_dl/extractor/teamcoco.py
+++ b/youtube_dl/extractor/teamcoco.py
@@ -62,7 +62,9 @@ class TeamcocoIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
 
         display_id = mobj.group('display_id')
-        webpage = self._download_webpage(url, display_id)
+        webpage, urlh = self._download_webpage_handle(url, display_id)
+        if 'src=expired' in urlh.geturl():
+            raise ExtractorError('This video is expired.', expected=True)
 
         video_id = mobj.group('video_id')
         if not video_id:

From 2bc43303031215436b201e656094b60ab3ec7e9e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 14 May 2015 23:41:27 +0200
Subject: [PATCH 1299/6219] [youtube:history] Fix extraction (fixes #5702)

It uses the same method as YoutubeSubscriptionsIE, if other feed starts using it we should consider using base class.
---
 youtube_dl/extractor/youtube.py | 37 +++++++++++++++++++++++++++++----
 1 file changed, 33 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 0869c9fd4..e58184adc 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1667,13 +1667,42 @@ class YoutubeWatchLaterIE(YoutubePlaylistIE):
         return self._extract_playlist('WL')
 
 
-class YoutubeHistoryIE(YoutubeFeedsInfoExtractor):
+class YoutubeHistoryIE(YoutubePlaylistIE):
     IE_NAME = 'youtube:history'
     IE_DESC = 'Youtube watch history, ":ythistory" for short (requires authentication)'
     _VALID_URL = 'https?://www\.youtube\.com/feed/history|:ythistory'
-    _FEED_NAME = 'history'
-    _PERSONAL_FEED = True
-    _PLAYLIST_TITLE = 'Youtube Watch History'
+    _TESTS = []
+
+    def _real_extract(self, url):
+        title = 'Youtube History'
+        page = self._download_webpage('https://www.youtube.com/feed/history', title)
+
+        # The extraction process is the same as for playlists, but the regex
+        # for the video ids doesn't contain an index
+        ids = []
+        more_widget_html = content_html = page
+
+        for page_num in itertools.count(1):
+            matches = re.findall(r'href="\s*/watch\?v=([0-9A-Za-z_-]{11})', content_html)
+            new_ids = orderedSet(matches)
+            ids.extend(new_ids)
+
+            mobj = re.search(r'data-uix-load-more-href="/?(?P<more>[^"]+)"', more_widget_html)
+            if not mobj:
+                break
+
+            more = self._download_json(
+                'https://youtube.com/%s' % mobj.group('more'), title,
+                'Downloading page #%s' % page_num,
+                transform_source=uppercase_escape)
+            content_html = more['content_html']
+            more_widget_html = more['load_more_widget_html']
+
+        return {
+            '_type': 'playlist',
+            'title': title,
+            'entries': self._ids_to_results(ids),
+        }
 
 
 class YoutubeFavouritesIE(YoutubeBaseInfoExtractor):

From c4fc559f45ea5c40409eab44867ff2b4f08976c2 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 15 May 2015 10:13:43 +0200
Subject: [PATCH 1300/6219] release 2015.05.15

---
 docs/supportedsites.md | 11 ++++++++---
 youtube_dl/version.py  |  2 +-
 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 98b625380..43fbe8b1d 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -291,6 +291,7 @@
  - **MySpass**
  - **myvideo**
  - **MyVidster**
+ - **N-JOY**
  - **n-tv.de**
  - **NationalGeographic**
  - **Naver**
@@ -368,9 +369,10 @@
  - **prosiebensat1**: ProSiebenSat.1 Digital
  - **Puls4**
  - **Pyvideo**
- - **QQMusic**
- - **QQMusicAlbum**
- - **QQMusicSinger**
+ - **qqmusic**
+ - **qqmusic:album**
+ - **qqmusic:singer**
+ - **qqmusic:toplist**
  - **QuickVid**
  - **R7**
  - **radio.de**
@@ -439,6 +441,7 @@
  - **southpark.cc.com:español**
  - **southpark.de**
  - **southpark.nl**
+ - **southparkstudios.dk**
  - **Space**
  - **SpankBang**
  - **Spankwire**
@@ -492,6 +495,7 @@
  - **tlc.com**
  - **tlc.de**
  - **TMZ**
+ - **TMZArticle**
  - **TNAFlix**
  - **tou.tv**
  - **Toypics**: Toypics user profile
@@ -569,6 +573,7 @@
  - **vk.com**
  - **vk.com:user-videos**: vk.com:All of a user's videos
  - **Vodlocker**
+ - **VoiceRepublic**
  - **Vporn**
  - **VRT**
  - **vube**: Vube.com
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 83c5a1659..38f00bc9b 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.05.10'
+__version__ = '2015.05.15'

From 3884dcf313223040049e4153e0c398fbc36b5117 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 15 May 2015 14:03:00 +0200
Subject: [PATCH 1301/6219] YoutubeDL: ignore indexes from 'playlist_items'
 that are not in the list (fixes #5706)

We ignore them instead of failing to match the behaviour of the 'playliststart' parameter.
---
 youtube_dl/YoutubeDL.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 691f3e09f..5df889945 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -759,7 +759,9 @@ class YoutubeDL(object):
             if isinstance(ie_entries, list):
                 n_all_entries = len(ie_entries)
                 if playlistitems:
-                    entries = [ie_entries[i - 1] for i in playlistitems]
+                    entries = [
+                        ie_entries[i - 1] for i in playlistitems
+                        if -n_all_entries <= i - 1 < n_all_entries]
                 else:
                     entries = ie_entries[playliststart:playlistend]
                 n_entries = len(entries)

From e9eaf3fbcf497e76a55d2ba15d5880af83a065d4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 15 May 2015 14:06:19 +0200
Subject: [PATCH 1302/6219] [test/YoutubeDL] Add tests for 'playliststart',
 'playlistend' and 'playlist_items'

---
 test/test_YoutubeDL.py | 46 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 46 insertions(+)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 82b827536..a13c09ef4 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -12,6 +12,7 @@ import copy
 
 from test.helper import FakeYDL, assertRegexpMatches
 from youtube_dl import YoutubeDL
+from youtube_dl.compat import compat_str
 from youtube_dl.extractor import YoutubeIE
 from youtube_dl.postprocessor.common import PostProcessor
 from youtube_dl.utils import match_filter_func
@@ -507,6 +508,51 @@ class TestYoutubeDL(unittest.TestCase):
         res = get_videos(f)
         self.assertEqual(res, ['1'])
 
+    def test_playlist_items_selection(self):
+        entries = [{
+            'id': compat_str(i),
+            'title': compat_str(i),
+            'url': TEST_URL,
+        } for i in range(1, 5)]
+        playlist = {
+            '_type': 'playlist',
+            'id': 'test',
+            'entries': entries,
+            'extractor': 'test:playlist',
+            'extractor_key': 'test:playlist',
+            'webpage_url': 'http://example.com',
+        }
+
+        def get_ids(params):
+            ydl = YDL(params)
+            # make a copy because the dictionary can be modified
+            ydl.process_ie_result(playlist.copy())
+            return [int(v['id']) for v in ydl.downloaded_info_dicts]
+
+        result = get_ids({})
+        self.assertEqual(result, [1, 2, 3, 4])
+
+        result = get_ids({'playlistend': 10})
+        self.assertEqual(result, [1, 2, 3, 4])
+
+        result = get_ids({'playlistend': 2})
+        self.assertEqual(result, [1, 2])
+
+        result = get_ids({'playliststart': 10})
+        self.assertEqual(result, [])
+
+        result = get_ids({'playliststart': 2})
+        self.assertEqual(result, [2, 3, 4])
+
+        result = get_ids({'playlist_items': '2-4'})
+        self.assertEqual(result, [2, 3, 4])
+
+        result = get_ids({'playlist_items': '2,4'})
+        self.assertEqual(result, [2, 4])
+
+        result = get_ids({'playlist_items': '10'})
+        self.assertEqual(result, [])
+
 
 if __name__ == '__main__':
     unittest.main()

From 15da7ce7fb89203247f4c959a748281ecf353e2a Mon Sep 17 00:00:00 2001
From: blissland <blissland.house@googlemail.com>
Date: Fri, 15 May 2015 12:28:10 +0100
Subject: [PATCH 1303/6219] Fix file format extraction regex and update test
 file checksum

---
 youtube_dl/extractor/ccc.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/ccc.py b/youtube_dl/extractor/ccc.py
index 2a5d4be18..6924eac70 100644
--- a/youtube_dl/extractor/ccc.py
+++ b/youtube_dl/extractor/ccc.py
@@ -16,7 +16,7 @@ class CCCIE(InfoExtractor):
 
     _TEST = {
         'url': 'http://media.ccc.de/browse/congress/2013/30C3_-_5443_-_en_-_saal_g_-_201312281830_-_introduction_to_processor_design_-_byterazor.html#video',
-        'md5': '205a365d0d57c0b1e43a12c9ffe8f9be',
+        'md5': '3a1eda8f3a29515d27f5adb967d7e740',
         'info_dict': {
             'id': '20131228183',
             'ext': 'mp4',
@@ -51,7 +51,7 @@ class CCCIE(InfoExtractor):
 
         matches = re.finditer(r'''(?xs)
             <(?:span|div)\s+class='label\s+filetype'>(?P<format>.*?)</(?:span|div)>\s*
-            <a\s+href='(?P<http_url>[^']+)'>\s*
+            <a\s+download\s+href='(?P<http_url>[^']+)'>\s*
             (?:
                 .*?
                 <a\s+href='(?P<torrent_url>[^']+\.torrent)'

From a7b8467ac0baecd02a815b1f57731ae9bb10ab87 Mon Sep 17 00:00:00 2001
From: Vitaliy Syrchikov <maddoger@gmail.com>
Date: Fri, 15 May 2015 16:52:11 +0400
Subject: [PATCH 1304/6219] Sportbox extractor fix.

---
 youtube_dl/extractor/sportbox.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/sportbox.py b/youtube_dl/extractor/sportbox.py
index becdf658f..830220543 100644
--- a/youtube_dl/extractor/sportbox.py
+++ b/youtube_dl/extractor/sportbox.py
@@ -11,7 +11,7 @@ from ..utils import (
 
 
 class SportBoxIE(InfoExtractor):
-    _VALID_URL = r'https?://news\.sportbox\.ru/Vidy_sporta/(?:[^/]+/)+spbvideo_NI\d+_(?P<display_id>.+)'
+    _VALID_URL = r'https?://news\.sportbox\.ru/(?:[^/]+/)+spbvideo_NI\d+_(?P<display_id>.+)'
     _TESTS = [
         {
             'url': 'http://news.sportbox.ru/Vidy_sporta/Avtosport/Rossijskij/spbvideo_NI483529_Gonka-2-zaezd-Obyedinenniy-2000-klassi-Turing-i-S',
@@ -50,7 +50,7 @@ class SportBoxIE(InfoExtractor):
             display_id, 'Downloading player webpage')
 
         hls = self._search_regex(
-            r"var\s+original_hls_file\s*=\s*'([^']+)'", player, 'hls file')
+            r"sportboxPlayer\.jwplayer_common_params\.file\s*=\s*['\"]+([^\"]+)['\"]+", player, 'hls file')
 
         formats = self._extract_m3u8_formats(hls, display_id, 'mp4')
 

From ae670a6ed8019f1b69bbe345621f51c8b32789ec Mon Sep 17 00:00:00 2001
From: Vitaliy Syrchikov <maddoger@gmail.com>
Date: Fri, 15 May 2015 17:53:05 +0400
Subject: [PATCH 1305/6219] Sportbox source fix. HD videos support.

---
 youtube_dl/extractor/sportbox.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/sportbox.py b/youtube_dl/extractor/sportbox.py
index 830220543..695b3ff82 100644
--- a/youtube_dl/extractor/sportbox.py
+++ b/youtube_dl/extractor/sportbox.py
@@ -14,7 +14,7 @@ class SportBoxIE(InfoExtractor):
     _VALID_URL = r'https?://news\.sportbox\.ru/(?:[^/]+/)+spbvideo_NI\d+_(?P<display_id>.+)'
     _TESTS = [
         {
-            'url': 'http://news.sportbox.ru/Vidy_sporta/Avtosport/Rossijskij/spbvideo_NI483529_Gonka-2-zaezd-Obyedinenniy-2000-klassi-Turing-i-S',
+            'url': 'http://news.sportbox.ru/Vidy_sporta/Avtosport/Rossijskij/spbvideo_NI483529_Gonka-2-zaezd-Obyedinenniy-2000-klassi-Turing-i-S',            
             'md5': 'ff56a598c2cf411a9a38a69709e97079',
             'info_dict': {
                 'id': '80822',
@@ -42,11 +42,15 @@ class SportBoxIE(InfoExtractor):
 
         webpage = self._download_webpage(url, display_id)
 
-        video_id = self._search_regex(
-            r'src="/vdl/player/media/(\d+)"', webpage, 'video id')
+        sobj = re.search(r'src="/vdl/player/(?P<media_type>\w+)/(?P<video_id>\d+)"', webpage)
+        if (sobj):
+            video_id = sobj.group('video_id')
+            media_type = sobj.group('media_type')
+        else:
+            raise RegexNotFoundError('Unable to extract video_id')
 
         player = self._download_webpage(
-            'http://news.sportbox.ru/vdl/player/media/%s' % video_id,
+            'http://news.sportbox.ru/vdl/player/%s/%s' % (media_type, video_id),
             display_id, 'Downloading player webpage')
 
         hls = self._search_regex(

From 25f14e9f93295a787e0cb436a5f6179d6174733d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 15 May 2015 21:06:59 +0600
Subject: [PATCH 1306/6219] [youtube] Separate feed extractor

---
 youtube_dl/extractor/youtube.py | 143 +++++++++-----------------------
 1 file changed, 37 insertions(+), 106 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index e58184adc..9096a2975 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -49,6 +49,11 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
             # YouTube sets the expire time to about two months
             expire_time=time.time() + 2 * 30 * 24 * 3600)
 
+    def _ids_to_results(self, ids):
+        return [
+            self.url_result(vid_id, 'Youtube', video_id=vid_id)
+            for vid_id in ids]
+
     def _login(self):
         """
         Attempt to log in to YouTube.
@@ -1261,11 +1266,6 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
     def _real_initialize(self):
         self._login()
 
-    def _ids_to_results(self, ids):
-        return [
-            self.url_result(vid_id, 'Youtube', video_id=vid_id)
-            for vid_id in ids]
-
     def _extract_mix(self, playlist_id):
         # The mixes are generated from a single video
         # the id of the playlist is just 'RD' + video_id
@@ -1601,20 +1601,10 @@ class YoutubeShowIE(InfoExtractor):
 
 class YoutubeFeedsInfoExtractor(YoutubeBaseInfoExtractor):
     """
-    Base class for extractors that fetch info from
-    http://www.youtube.com/feed_ajax
+    Base class for feed extractors
     Subclasses must define the _FEED_NAME and _PLAYLIST_TITLE properties.
     """
     _LOGIN_REQUIRED = True
-    # use action_load_personal_feed instead of action_load_system_feed
-    _PERSONAL_FEED = False
-
-    @property
-    def _FEED_TEMPLATE(self):
-        action = 'action_load_system_feed'
-        if self._PERSONAL_FEED:
-            action = 'action_load_personal_feed'
-        return 'https://www.youtube.com/feed_ajax?%s=1&feed_name=%s&paging=%%s' % (action, self._FEED_NAME)
 
     @property
     def IE_NAME(self):
@@ -1624,58 +1614,8 @@ class YoutubeFeedsInfoExtractor(YoutubeBaseInfoExtractor):
         self._login()
 
     def _real_extract(self, url):
-        feed_entries = []
-        paging = 0
-        for i in itertools.count(1):
-            info = self._download_json(
-                self._FEED_TEMPLATE % paging,
-                '%s feed' % self._FEED_NAME,
-                'Downloading page %s' % i,
-                transform_source=uppercase_escape)
-            feed_html = info.get('feed_html') or info.get('content_html')
-            load_more_widget_html = info.get('load_more_widget_html') or feed_html
-            m_ids = re.finditer(r'"/watch\?v=(.*?)["&]', feed_html)
-            ids = orderedSet(m.group(1) for m in m_ids)
-            feed_entries.extend(
-                self.url_result(video_id, 'Youtube', video_id=video_id)
-                for video_id in ids)
-            mobj = re.search(
-                r'data-uix-load-more-href="/?[^"]+paging=(?P<paging>\d+)',
-                load_more_widget_html)
-            if mobj is None:
-                break
-            paging = mobj.group('paging')
-        return self.playlist_result(feed_entries, playlist_title=self._PLAYLIST_TITLE)
-
-
-class YoutubeRecommendedIE(YoutubeFeedsInfoExtractor):
-    IE_NAME = 'youtube:recommended'
-    IE_DESC = 'YouTube.com recommended videos, ":ytrec" for short (requires authentication)'
-    _VALID_URL = r'https?://www\.youtube\.com/feed/recommended|:ytrec(?:ommended)?'
-    _FEED_NAME = 'recommended'
-    _PLAYLIST_TITLE = 'Youtube Recommended videos'
-
-
-class YoutubeWatchLaterIE(YoutubePlaylistIE):
-    IE_NAME = 'youtube:watchlater'
-    IE_DESC = 'Youtube watch later list, ":ytwatchlater" for short (requires authentication)'
-    _VALID_URL = r'https?://www\.youtube\.com/(?:feed/watch_later|playlist\?list=WL)|:ytwatchlater'
-
-    _TESTS = []  # override PlaylistIE tests
-
-    def _real_extract(self, url):
-        return self._extract_playlist('WL')
-
-
-class YoutubeHistoryIE(YoutubePlaylistIE):
-    IE_NAME = 'youtube:history'
-    IE_DESC = 'Youtube watch history, ":ythistory" for short (requires authentication)'
-    _VALID_URL = 'https?://www\.youtube\.com/feed/history|:ythistory'
-    _TESTS = []
-
-    def _real_extract(self, url):
-        title = 'Youtube History'
-        page = self._download_webpage('https://www.youtube.com/feed/history', title)
+        page = self._download_webpage(
+            'https://www.youtube.com/feed/%s' % self._FEED_NAME, self._PLAYLIST_TITLE)
 
         # The extraction process is the same as for playlists, but the regex
         # for the video ids doesn't contain an index
@@ -1692,17 +1632,25 @@ class YoutubeHistoryIE(YoutubePlaylistIE):
                 break
 
             more = self._download_json(
-                'https://youtube.com/%s' % mobj.group('more'), title,
+                'https://youtube.com/%s' % mobj.group('more'), self._PLAYLIST_TITLE,
                 'Downloading page #%s' % page_num,
                 transform_source=uppercase_escape)
             content_html = more['content_html']
             more_widget_html = more['load_more_widget_html']
 
-        return {
-            '_type': 'playlist',
-            'title': title,
-            'entries': self._ids_to_results(ids),
-        }
+        return self.playlist_result(
+            self._ids_to_results(ids), playlist_title=self._PLAYLIST_TITLE)
+
+
+class YoutubeWatchLaterIE(YoutubePlaylistIE):
+    IE_NAME = 'youtube:watchlater'
+    IE_DESC = 'Youtube watch later list, ":ytwatchlater" for short (requires authentication)'
+    _VALID_URL = r'https?://www\.youtube\.com/(?:feed/watch_later|playlist\?list=WL)|:ytwatchlater'
+
+    _TESTS = []  # override PlaylistIE tests
+
+    def _real_extract(self, url):
+        return self._extract_playlist('WL')
 
 
 class YoutubeFavouritesIE(YoutubeBaseInfoExtractor):
@@ -1717,42 +1665,25 @@ class YoutubeFavouritesIE(YoutubeBaseInfoExtractor):
         return self.url_result(playlist_id, 'YoutubePlaylist')
 
 
-class YoutubeSubscriptionsIE(YoutubePlaylistIE):
-    IE_NAME = 'youtube:subscriptions'
+class YoutubeRecommendedIE(YoutubeFeedsInfoExtractor):
+    IE_DESC = 'YouTube.com recommended videos, ":ytrec" for short (requires authentication)'
+    _VALID_URL = r'https?://www\.youtube\.com/feed/recommended|:ytrec(?:ommended)?'
+    _FEED_NAME = 'recommended'
+    _PLAYLIST_TITLE = 'Youtube Recommended videos'
+
+
+class YoutubeSubscriptionsIE(YoutubeFeedsInfoExtractor):
     IE_DESC = 'YouTube.com subscriptions feed, "ytsubs" keyword (requires authentication)'
     _VALID_URL = r'https?://www\.youtube\.com/feed/subscriptions|:ytsubs(?:criptions)?'
-    _TESTS = []
+    _FEED_NAME = 'subscriptions'
+    _PLAYLIST_TITLE = 'Youtube Subscriptions'
 
-    def _real_extract(self, url):
-        title = 'Youtube Subscriptions'
-        page = self._download_webpage('https://www.youtube.com/feed/subscriptions', title)
 
-        # The extraction process is the same as for playlists, but the regex
-        # for the video ids doesn't contain an index
-        ids = []
-        more_widget_html = content_html = page
-
-        for page_num in itertools.count(1):
-            matches = re.findall(r'href="\s*/watch\?v=([0-9A-Za-z_-]{11})', content_html)
-            new_ids = orderedSet(matches)
-            ids.extend(new_ids)
-
-            mobj = re.search(r'data-uix-load-more-href="/?(?P<more>[^"]+)"', more_widget_html)
-            if not mobj:
-                break
-
-            more = self._download_json(
-                'https://youtube.com/%s' % mobj.group('more'), title,
-                'Downloading page #%s' % page_num,
-                transform_source=uppercase_escape)
-            content_html = more['content_html']
-            more_widget_html = more['load_more_widget_html']
-
-        return {
-            '_type': 'playlist',
-            'title': title,
-            'entries': self._ids_to_results(ids),
-        }
+class YoutubeHistoryIE(YoutubeFeedsInfoExtractor):
+    IE_DESC = 'Youtube watch history, ":ythistory" for short (requires authentication)'
+    _VALID_URL = 'https?://www\.youtube\.com/feed/history|:ythistory'
+    _FEED_NAME = 'history'
+    _PLAYLIST_TITLE = 'Youtube History'
 
 
 class YoutubeTruncatedURLIE(InfoExtractor):

From 62c95fd5fcb8dbea2faeb4edac4c5177cbac5912 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 15 May 2015 21:42:34 +0600
Subject: [PATCH 1307/6219] [youtube:feed] Check each 'load more' portion for
 unique video ids

---
 youtube_dl/extractor/youtube.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 9096a2975..1f9940cf5 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1621,10 +1621,16 @@ class YoutubeFeedsInfoExtractor(YoutubeBaseInfoExtractor):
         # for the video ids doesn't contain an index
         ids = []
         more_widget_html = content_html = page
-
         for page_num in itertools.count(1):
             matches = re.findall(r'href="\s*/watch\?v=([0-9A-Za-z_-]{11})', content_html)
-            new_ids = orderedSet(matches)
+
+            # 'recommended' feed has infinite 'load more' and each new portion spins
+            # the same videos in (sometimes) slightly different order, so we'll check
+            # for unicity and break when portion has no new videos
+            new_ids = filter(lambda video_id: video_id not in ids, orderedSet(matches))
+            if not new_ids:
+                break
+
             ids.extend(new_ids)
 
             mobj = re.search(r'data-uix-load-more-href="/?(?P<more>[^"]+)"', more_widget_html)

From e9ca615a9872e85a6986061fdf54257244ce1f77 Mon Sep 17 00:00:00 2001
From: Vitaliy Syrchikov <maddoger@gmail.com>
Date: Fri, 15 May 2015 19:57:54 +0400
Subject: [PATCH 1308/6219] New test

---
 youtube_dl/extractor/sportbox.py | 23 +++++++++++++++++++++--
 1 file changed, 21 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/sportbox.py b/youtube_dl/extractor/sportbox.py
index 695b3ff82..cb1515eff 100644
--- a/youtube_dl/extractor/sportbox.py
+++ b/youtube_dl/extractor/sportbox.py
@@ -30,10 +30,29 @@ class SportBoxIE(InfoExtractor):
                 # m3u8 download
                 'skip_download': True,
             },
-        }, {
+        }, 
+        {
+            'url': 'http://news.sportbox.ru/video/no_ads/spbvideo_NI536574_V_Novorossijske_proshel_detskij_turnir_Pole_slavy_bojevoj?ci=211355',            
+            'md5': 'ff56a598c2cf411a9a38a69709e97079',
+            'info_dict': {
+                'id': '211355',
+                'ext': 'mp4',
+                'title': 'В Новороссийске прошел детский турнир «Поле славы боевой»',
+                'description': '16 детских коллективов приняли участие в суперфинале турнира «Поле славы боевой».',
+                'thumbnail': 're:^https?://.*\.jpg$',
+                'timestamp': 1426237001,
+                'upload_date': '20150313',
+                'duration': 292,
+            },
+            'params': {
+                # m3u8 download
+                'skip_download': True,
+            },
+        }, 
+        {
             'url': 'http://news.sportbox.ru/Vidy_sporta/billiard/spbvideo_NI486287_CHempionat-mira-po-dinamichnoy-piramide-4',
             'only_matching': True,
-        }
+        },
     ]
 
     def _real_extract(self, url):

From 34fe5a94baf9e7ea437de68621a5fa73780c0f17 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 15 May 2015 18:42:59 +0200
Subject: [PATCH 1309/6219] [gamespot] Add support for videos that don't use
 'f4m_stream' (fixes #5707)

---
 youtube_dl/extractor/gamespot.py | 60 +++++++++++++++++++++-----------
 1 file changed, 40 insertions(+), 20 deletions(-)

diff --git a/youtube_dl/extractor/gamespot.py b/youtube_dl/extractor/gamespot.py
index 47373e215..5927455f6 100644
--- a/youtube_dl/extractor/gamespot.py
+++ b/youtube_dl/extractor/gamespot.py
@@ -15,7 +15,7 @@ from ..utils import (
 
 class GameSpotIE(InfoExtractor):
     _VALID_URL = r'(?:http://)?(?:www\.)?gamespot\.com/.*-(?P<id>\d+)/?'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.gamespot.com/videos/arma-3-community-guide-sitrep-i/2300-6410818/',
         'md5': 'b2a30deaa8654fcccd43713a6b6a4825',
         'info_dict': {
@@ -23,8 +23,16 @@ class GameSpotIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Arma 3 - Community Guide: SITREP I',
             'description': 'Check out this video where some of the basics of Arma 3 is explained.',
-        }
-    }
+        },
+    }, {
+        'url': 'http://www.gamespot.com/videos/the-witcher-3-wild-hunt-xbox-one-now-playing/2300-6424837/',
+        'info_dict': {
+            'id': 'gs-2300-6424837',
+            'ext': 'flv',
+            'title': 'The Witcher 3: Wild Hunt [Xbox ONE]  - Now Playing',
+            'description': 'Join us as we take a look at the early hours of The Witcher 3: Wild Hunt and more.',
+        },
+    }]
 
     def _real_extract(self, url):
         page_id = self._match_id(url)
@@ -32,25 +40,37 @@ class GameSpotIE(InfoExtractor):
         data_video_json = self._search_regex(
             r'data-video=["\'](.*?)["\']', webpage, 'data video')
         data_video = json.loads(unescapeHTML(data_video_json))
+        streams = data_video['videoStreams']
 
-        # Transform the manifest url to a link to the mp4 files
-        # they are used in mobile devices.
-        f4m_url = data_video['videoStreams']['f4m_stream']
-        f4m_path = compat_urlparse.urlparse(f4m_url).path
-        QUALITIES_RE = r'((,\d+)+,?)'
-        qualities = self._search_regex(QUALITIES_RE, f4m_path, 'qualities').strip(',').split(',')
-        http_path = f4m_path[1:].split('/', 1)[1]
-        http_template = re.sub(QUALITIES_RE, r'%s', http_path)
-        http_template = http_template.replace('.csmil/manifest.f4m', '')
-        http_template = compat_urlparse.urljoin(
-            'http://video.gamespotcdn.com/', http_template)
         formats = []
-        for q in qualities:
-            formats.append({
-                'url': http_template % q,
-                'ext': 'mp4',
-                'format_id': q,
-            })
+        f4m_url = streams.get('f4m_stream')
+        if f4m_url is not None:
+            # Transform the manifest url to a link to the mp4 files
+            # they are used in mobile devices.
+            f4m_path = compat_urlparse.urlparse(f4m_url).path
+            QUALITIES_RE = r'((,\d+)+,?)'
+            qualities = self._search_regex(QUALITIES_RE, f4m_path, 'qualities').strip(',').split(',')
+            http_path = f4m_path[1:].split('/', 1)[1]
+            http_template = re.sub(QUALITIES_RE, r'%s', http_path)
+            http_template = http_template.replace('.csmil/manifest.f4m', '')
+            http_template = compat_urlparse.urljoin(
+                'http://video.gamespotcdn.com/', http_template)
+            for q in qualities:
+                formats.append({
+                    'url': http_template % q,
+                    'ext': 'mp4',
+                    'format_id': q,
+                })
+        else:
+            for quality in ['sd', 'hd']:
+                # It's actually a link to a flv file
+                flv_url = streams.get('f4m_{0}'.format(quality))
+                if flv_url is not None:
+                    formats.append({
+                        'url': flv_url,
+                        'ext': 'flv',
+                        'format_id': quality,
+                    })
 
         return {
             'id': data_video['guid'],

From eeb23eb7ea6953d7e90ccf669cd0e636d10b2b91 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 15 May 2015 18:44:08 +0200
Subject: [PATCH 1310/6219] [gamespot] The protocol is not optional

---
 youtube_dl/extractor/gamespot.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/gamespot.py b/youtube_dl/extractor/gamespot.py
index 5927455f6..2d33fa7f5 100644
--- a/youtube_dl/extractor/gamespot.py
+++ b/youtube_dl/extractor/gamespot.py
@@ -14,7 +14,7 @@ from ..utils import (
 
 
 class GameSpotIE(InfoExtractor):
-    _VALID_URL = r'(?:http://)?(?:www\.)?gamespot\.com/.*-(?P<id>\d+)/?'
+    _VALID_URL = r'http://(?:www\.)?gamespot\.com/.*-(?P<id>\d+)/?'
     _TESTS = [{
         'url': 'http://www.gamespot.com/videos/arma-3-community-guide-sitrep-i/2300-6410818/',
         'md5': 'b2a30deaa8654fcccd43713a6b6a4825',

From 3a7382950b6f498f50173c8813f6cb1db3739277 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 15 May 2015 22:50:44 +0600
Subject: [PATCH 1311/6219] [sportbox:embed] Add extractor

---
 youtube_dl/extractor/__init__.py |   5 +-
 youtube_dl/extractor/sportbox.py | 138 +++++++++++++++++--------------
 2 files changed, 82 insertions(+), 61 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 8ec0c1032..f293bc2a4 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -502,7 +502,10 @@ from .spiegel import SpiegelIE, SpiegelArticleIE
 from .spiegeltv import SpiegeltvIE
 from .spike import SpikeIE
 from .sport5 import Sport5IE
-from .sportbox import SportBoxIE
+from .sportbox import (
+    SportBoxIE,
+    SportBoxEmbedIE,
+)
 from .sportdeutschland import SportDeutschlandIE
 from .srf import SrfIE
 from .srmediathek import SRMediathekIE
diff --git a/youtube_dl/extractor/sportbox.py b/youtube_dl/extractor/sportbox.py
index cb1515eff..10c45eb74 100644
--- a/youtube_dl/extractor/sportbox.py
+++ b/youtube_dl/extractor/sportbox.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import compat_urlparse
 from ..utils import (
     parse_duration,
     parse_iso8601,
@@ -12,48 +13,30 @@ from ..utils import (
 
 class SportBoxIE(InfoExtractor):
     _VALID_URL = r'https?://news\.sportbox\.ru/(?:[^/]+/)+spbvideo_NI\d+_(?P<display_id>.+)'
-    _TESTS = [
-        {
-            'url': 'http://news.sportbox.ru/Vidy_sporta/Avtosport/Rossijskij/spbvideo_NI483529_Gonka-2-zaezd-Obyedinenniy-2000-klassi-Turing-i-S',            
-            'md5': 'ff56a598c2cf411a9a38a69709e97079',
-            'info_dict': {
-                'id': '80822',
-                'ext': 'mp4',
-                'title': 'Гонка 2  заезд ««Объединенный 2000»: классы Туринг и Супер-продакшн',
-                'description': 'md5:81715fa9c4ea3d9e7915dc8180c778ed',
-                'thumbnail': 're:^https?://.*\.jpg$',
-                'timestamp': 1411896237,
-                'upload_date': '20140928',
-                'duration': 4846,
-            },
-            'params': {
-                # m3u8 download
-                'skip_download': True,
-            },
-        }, 
-        {
-            'url': 'http://news.sportbox.ru/video/no_ads/spbvideo_NI536574_V_Novorossijske_proshel_detskij_turnir_Pole_slavy_bojevoj?ci=211355',            
-            'md5': 'ff56a598c2cf411a9a38a69709e97079',
-            'info_dict': {
-                'id': '211355',
-                'ext': 'mp4',
-                'title': 'В Новороссийске прошел детский турнир «Поле славы боевой»',
-                'description': '16 детских коллективов приняли участие в суперфинале турнира «Поле славы боевой».',
-                'thumbnail': 're:^https?://.*\.jpg$',
-                'timestamp': 1426237001,
-                'upload_date': '20150313',
-                'duration': 292,
-            },
-            'params': {
-                # m3u8 download
-                'skip_download': True,
-            },
-        }, 
-        {
-            'url': 'http://news.sportbox.ru/Vidy_sporta/billiard/spbvideo_NI486287_CHempionat-mira-po-dinamichnoy-piramide-4',
-            'only_matching': True,
+    _TESTS = [{
+        'url': 'http://news.sportbox.ru/Vidy_sporta/Avtosport/Rossijskij/spbvideo_NI483529_Gonka-2-zaezd-Obyedinenniy-2000-klassi-Turing-i-S',
+        'md5': 'ff56a598c2cf411a9a38a69709e97079',
+        'info_dict': {
+            'id': '80822',
+            'ext': 'mp4',
+            'title': 'Гонка 2  заезд ««Объединенный 2000»: классы Туринг и Супер-продакшн',
+            'description': 'md5:81715fa9c4ea3d9e7915dc8180c778ed',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'timestamp': 1411896237,
+            'upload_date': '20140928',
+            'duration': 4846,
         },
-    ]
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }, {
+        'url': 'http://news.sportbox.ru/Vidy_sporta/billiard/spbvideo_NI486287_CHempionat-mira-po-dinamichnoy-piramide-4',
+        'only_matching': True,
+    }, {
+        'url': 'http://news.sportbox.ru/video/no_ads/spbvideo_NI536574_V_Novorossijske_proshel_detskij_turnir_Pole_slavy_bojevoj?ci=211355',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
@@ -61,39 +44,74 @@ class SportBoxIE(InfoExtractor):
 
         webpage = self._download_webpage(url, display_id)
 
-        sobj = re.search(r'src="/vdl/player/(?P<media_type>\w+)/(?P<video_id>\d+)"', webpage)
-        if (sobj):
-            video_id = sobj.group('video_id')
-            media_type = sobj.group('media_type')
-        else:
-            raise RegexNotFoundError('Unable to extract video_id')
-
-        player = self._download_webpage(
-            'http://news.sportbox.ru/vdl/player/%s/%s' % (media_type, video_id),
-            display_id, 'Downloading player webpage')
-
-        hls = self._search_regex(
-            r"sportboxPlayer\.jwplayer_common_params\.file\s*=\s*['\"]+([^\"]+)['\"]+", player, 'hls file')
-
-        formats = self._extract_m3u8_formats(hls, display_id, 'mp4')
+        player = self._search_regex(
+            r'src="/?(vdl/player/[^"]+)"', webpage, 'player')
 
         title = self._html_search_regex(
             r'<h1 itemprop="name">([^<]+)</h1>', webpage, 'title')
         description = self._html_search_regex(
-            r'(?s)<div itemprop="description">(.+?)</div>', webpage, 'description', fatal=False)
+            r'(?s)<div itemprop="description">(.+?)</div>',
+            webpage, 'description', fatal=False)
         thumbnail = self._og_search_thumbnail(webpage)
         timestamp = parse_iso8601(self._search_regex(
-            r'<span itemprop="uploadDate">([^<]+)</span>', webpage, 'timestamp', fatal=False))
+            r'<span itemprop="uploadDate">([^<]+)</span>',
+            webpage, 'timestamp', fatal=False))
         duration = parse_duration(self._html_search_regex(
-            r'<meta itemprop="duration" content="PT([^"]+)">', webpage, 'duration', fatal=False))
+            r'<meta itemprop="duration" content="PT([^"]+)">',
+            webpage, 'duration', fatal=False))
 
         return {
-            'id': video_id,
+            '_type': 'url_transparent',
+            'url': compat_urlparse.urljoin(url, '/%s' % player),
             'display_id': display_id,
             'title': title,
             'description': description,
             'thumbnail': thumbnail,
             'timestamp': timestamp,
             'duration': duration,
+        }
+
+
+class SportBoxEmbedIE(InfoExtractor):
+    _VALID_URL = r'https?://news\.sportbox\.ru/vdl/player(?:/[^/]+/|\?.*?\bn?id=)(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'http://news.sportbox.ru/vdl/player/ci/211355',
+        'info_dict': {
+            'id': '211355',
+            'ext': 'mp4',
+            'title': 'В Новороссийске прошел детский турнир «Поле славы боевой»',
+            'thumbnail': 're:^https?://.*\.jpg$',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }, {
+        'url': 'http://news.sportbox.ru/vdl/player?nid=370908&only_player=1&autostart=false&playeri=2&height=340&width=580',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        hls = self._search_regex(
+            r"sportboxPlayer\.jwplayer_common_params\.file\s*=\s*['\"]([^'\"]+)['\"]",
+            webpage, 'hls file')
+
+        formats = self._extract_m3u8_formats(hls, video_id, 'mp4')
+
+        title = self._search_regex(
+            r'sportboxPlayer\.node_title\s*=\s*"([^"]+)"', webpage, 'title')
+
+        thumbnail = self._search_regex(
+            r'sportboxPlayer\.jwplayer_common_params\.image\s*=\s*"([^"]+)"',
+            webpage, 'thumbnail', fatal=False)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'thumbnail': thumbnail,
             'formats': formats,
         }

From e8cfacae3710c2c225488e4b2d41b84268217a55 Mon Sep 17 00:00:00 2001
From: blissland <blissland.house@googlemail.com>
Date: Fri, 15 May 2015 17:57:32 +0100
Subject: [PATCH 1312/6219] [CBSNewsIE] Relax thumbnail regex so test passes

---
 youtube_dl/extractor/cbsnews.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/cbsnews.py b/youtube_dl/extractor/cbsnews.py
index 7e47960ab..52e61d85b 100644
--- a/youtube_dl/extractor/cbsnews.py
+++ b/youtube_dl/extractor/cbsnews.py
@@ -32,7 +32,7 @@ class CBSNewsIE(InfoExtractor):
                 'id': 'fort-hood-shooting-army-downplays-mental-illness-as-cause-of-attack',
                 'ext': 'flv',
                 'title': 'Fort Hood shooting: Army downplays mental illness as cause of attack',
-                'thumbnail': 'http://cbsnews2.cbsistatic.com/hub/i/r/2014/04/04/0c9fbc66-576b-41ca-8069-02d122060dd2/thumbnail/140x90/6dad7a502f88875ceac38202984b6d58/en-0404-werner-replace-640x360.jpg',
+                'thumbnail': 're:^https?://.*\.jpg$',
                 'duration': 205,
             },
             'params': {

From 1436a6835e0f3489a4c37cca3da5087567b68158 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 15 May 2015 23:08:44 +0600
Subject: [PATCH 1313/6219] [sportbox:embed] Add `_extract_urls`

---
 youtube_dl/extractor/sportbox.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/sportbox.py b/youtube_dl/extractor/sportbox.py
index 10c45eb74..a869a1b25 100644
--- a/youtube_dl/extractor/sportbox.py
+++ b/youtube_dl/extractor/sportbox.py
@@ -91,6 +91,12 @@ class SportBoxEmbedIE(InfoExtractor):
         'only_matching': True,
     }]
 
+    @staticmethod
+    def _extract_urls(webpage):
+        return re.findall(
+            r'<iframe[^>]+src="(https?://news\.sportbox\.ru/vdl/player[^"]+)"',
+            webpage)
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
 

From ef28a6cb26630f8f198a72eee34a2b5c8bd2f802 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 15 May 2015 23:09:10 +0600
Subject: [PATCH 1314/6219] [sportbox:embed] Relax thumbnail

---
 youtube_dl/extractor/sportbox.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/sportbox.py b/youtube_dl/extractor/sportbox.py
index a869a1b25..8686f9d11 100644
--- a/youtube_dl/extractor/sportbox.py
+++ b/youtube_dl/extractor/sportbox.py
@@ -113,7 +113,7 @@ class SportBoxEmbedIE(InfoExtractor):
 
         thumbnail = self._search_regex(
             r'sportboxPlayer\.jwplayer_common_params\.image\s*=\s*"([^"]+)"',
-            webpage, 'thumbnail', fatal=False)
+            webpage, 'thumbnail', default=None)
 
         return {
             'id': video_id,

From d40a3b5b55973d7ed65538179b71990c1828845a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 15 May 2015 23:09:34 +0600
Subject: [PATCH 1315/6219] [generic] Add support for sportbox embeds

---
 youtube_dl/extractor/generic.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 3d756e848..9230c3bb0 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -32,6 +32,7 @@ from .brightcove import BrightcoveIE
 from .nbc import NBCSportsVPlayerIE
 from .ooyala import OoyalaIE
 from .rutv import RUTVIE
+from .sportbox import SportBoxEmbedIE
 from .smotri import SmotriIE
 from .condenast import CondeNastIE
 from .udn import UDNEmbedIE
@@ -1229,6 +1230,11 @@ class GenericIE(InfoExtractor):
         if rutv_url:
             return self.url_result(rutv_url, 'RUTV')
 
+        # Look for embedded SportBox player
+        sportbox_urls = SportBoxEmbedIE._extract_urls(webpage)
+        if sportbox_urls:
+            return _playlist_from_matches(sportbox_urls, ie='SportBoxEmbed')
+
         # Look for embedded TED player
         mobj = re.search(
             r'<iframe[^>]+?src=(["\'])(?P<url>https?://embed(?:-ssl)?\.ted\.com/.+?)\1', webpage)

From b827a6015c145d67a4d4e9ea38aa54ebe347d3fe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 15 May 2015 23:18:21 +0600
Subject: [PATCH 1316/6219] [generic] Add test for sportbox embeds

---
 youtube_dl/extractor/generic.py | 31 +++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 9230c3bb0..610e33091 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -225,6 +225,37 @@ class GenericIE(InfoExtractor):
                 'skip_download': True,
             },
         },
+        # SportBox embed
+        {
+            'url': 'http://www.vestifinance.ru/articles/25753',
+            'info_dict': {
+                'id': '25753',
+                'title': 'Вести Экономика ― Прямые трансляции с Форума-выставки "Госзаказ-2013"',
+            },
+            'playlist': [{
+                'info_dict': {
+                    'id': '370908',
+                    'title': 'Госзаказ. День 3',
+                    'ext': 'mp4',
+                }
+            }, {
+                'info_dict': {
+                    'id': '370905',
+                    'title': 'Госзаказ. День 2',
+                    'ext': 'mp4',
+                }
+            }, {
+                'info_dict': {
+                    'id': '370902',
+                    'title': 'Госзаказ. День 1',
+                    'ext': 'mp4',
+                }
+            }],
+            'params': {
+                # m3u8 download
+                'skip_download': True,
+            },
+        },
         # Embedded TED video
         {
             'url': 'http://en.support.wordpress.com/videos/ted-talks/',

From 25c3a7348f7971c0af32dcea2d7fd57bd5c63f05 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 15 May 2015 23:23:51 +0600
Subject: [PATCH 1317/6219] [generic] Fix typo

---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 610e33091..9a7b0d25d 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1425,7 +1425,7 @@ class GenericIE(InfoExtractor):
         # Look for Senate ISVP iframe
         senate_isvp_url = SenateISVPIE._search_iframe_url(webpage)
         if senate_isvp_url:
-            return self.url_result(surl, 'SenateISVP')
+            return self.url_result(senate_isvp_url, 'SenateISVP')
 
         def check_video(vurl):
             if YoutubeIE.suitable(vurl):

From 70d0d43b5eeff04b41b089e499401e38c115e456 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 15 May 2015 23:32:25 +0600
Subject: [PATCH 1318/6219] [rts] Check formats (Closes #5711)

---
 youtube_dl/extractor/rts.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/rts.py b/youtube_dl/extractor/rts.py
index d0981115d..9fbe239d8 100644
--- a/youtube_dl/extractor/rts.py
+++ b/youtube_dl/extractor/rts.py
@@ -190,6 +190,7 @@ class RTSIE(InfoExtractor):
                 'tbr': media['rate'] or extract_bitrate(media['url']),
             } for media in info['media'] if media.get('rate')])
 
+        self._check_formats(formats, video_id)
         self._sort_formats(formats)
 
         return {

From 0d7f03642976e7859e290b06db41d20a4bfd3a38 Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Sat, 16 May 2015 15:43:13 +0800
Subject: [PATCH 1319/6219] [viki] Add support for shows

---
 youtube_dl/extractor/__init__.py |  5 ++++-
 youtube_dl/extractor/viki.py     | 33 ++++++++++++++++++++++++++++++++
 2 files changed, 37 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index f293bc2a4..cb6635610 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -639,7 +639,10 @@ from .vine import (
     VineIE,
     VineUserIE,
 )
-from .viki import VikiIE
+from .viki import (
+    VikiIE,
+    VikiShowIE,
+)
 from .vk import (
     VKIE,
     VKUserVideosIE,
diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index cf6af1e5c..4d185c0e6 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -145,3 +145,36 @@ class VikiIE(InfoExtractor):
                 'ext': 'vtt',
             }]
         return res
+
+
+class VikiShowIE(InfoExtractor):
+    IE_NAME = 'viki:show'
+    _VALID_URL = r'^https?://(?:www\.)?viki\.com/tv/(?P<id>[0-9]+c)'
+    _TESTS = [{
+        'url': 'http://www.viki.com/tv/50c-boys-over-flowers',
+        'info_dict': {
+            'id': '50c',
+            'title': 'Boys Over Flowers',
+            'description': 'md5:ecd3cff47967fe193cff37c0bec52790',
+        },
+        'playlist_count': 25,
+    }]
+
+    def _real_extract(self, url):
+        show_id = self._match_id(url)
+        show_page = self._download_webpage(url, show_id, 'Download show page')
+
+        title = self._og_search_title(show_page)
+        description = self._og_search_description(show_page)
+
+        show_json = self._download_json(
+            'http://api.viki.io/v4/containers/%s/episodes.json?app=100000a&per_page=999&sort=number&direction=asc' % show_id,
+            show_id, note='Retrieve show json', errnote='Unable to get show json'
+        )
+        entries = []
+        for video in show_json['response']:
+            video_id = video['id']
+            entries.append(self.url_result(
+                'http://www.viki.com/videos/%s' % video_id, 'Viki', video_id))
+
+        return self.playlist_result(entries, show_id, title, description)

From 2f3bdab2b90c6695c0a478f352967b0c9da4f23f Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Sat, 16 May 2015 15:56:37 +0800
Subject: [PATCH 1320/6219] [viki] Fix code format

---
 youtube_dl/extractor/viki.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index 4d185c0e6..40a73f561 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -169,8 +169,7 @@ class VikiShowIE(InfoExtractor):
 
         show_json = self._download_json(
             'http://api.viki.io/v4/containers/%s/episodes.json?app=100000a&per_page=999&sort=number&direction=asc' % show_id,
-            show_id, note='Retrieve show json', errnote='Unable to get show json'
-        )
+            show_id, note='Retrieve show json', errnote='Unable to get show json')
         entries = []
         for video in show_json['response']:
             video_id = video['id']

From 1c97b0a777f52c520587e93e7e61721fa6195977 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 16 May 2015 20:00:40 +0600
Subject: [PATCH 1321/6219] [ooyala:external] Add extractor

---
 youtube_dl/extractor/__init__.py |   5 +-
 youtube_dl/extractor/ooyala.py   | 221 +++++++++++++++++++------------
 2 files changed, 137 insertions(+), 89 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index f293bc2a4..1731f4fb2 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -376,7 +376,10 @@ from .nytimes import (
 from .nuvid import NuvidIE
 from .odnoklassniki import OdnoklassnikiIE
 from .oktoberfesttv import OktoberfestTVIE
-from .ooyala import OoyalaIE
+from .ooyala import (
+    OoyalaIE,
+    OoyalaExternalIE,
+)
 from .openfilm import OpenFilmIE
 from .orf import (
     ORFTVthekIE,
diff --git a/youtube_dl/extractor/ooyala.py b/youtube_dl/extractor/ooyala.py
index c0e6d643d..9f4fe5b29 100644
--- a/youtube_dl/extractor/ooyala.py
+++ b/youtube_dl/extractor/ooyala.py
@@ -12,7 +12,100 @@ from ..utils import (
 )
 
 
-class OoyalaIE(InfoExtractor):
+class OoyalaBaseIE(InfoExtractor):
+
+    def _extract_result(self, info, more_info):
+        embedCode = info['embedCode']
+        video_url = info.get('ipad_url') or info['url']
+
+        if determine_ext(video_url) == 'm3u8':
+            formats = self._extract_m3u8_formats(video_url, embedCode, ext='mp4')
+        else:
+            formats = [{
+                'url': video_url,
+                'ext': 'mp4',
+            }]
+
+        return {
+            'id': embedCode,
+            'title': unescapeHTML(info['title']),
+            'formats': formats,
+            'description': unescapeHTML(more_info['description']),
+            'thumbnail': more_info['promo'],
+        }
+
+    def _extract(self, player_url, video_id):
+        player = self._download_webpage(player_url, video_id)
+        mobile_url = self._search_regex(r'mobile_player_url="(.+?)&device="',
+                                        player, 'mobile player url')
+        # Looks like some videos are only available for particular devices
+        # (e.g. http://player.ooyala.com/player.js?embedCode=x1b3lqZDq9y_7kMyC2Op5qo-p077tXD0
+        # is only available for ipad)
+        # Working around with fetching URLs for all the devices found starting with 'unknown'
+        # until we succeed or eventually fail for each device.
+        devices = re.findall(r'device\s*=\s*"([^"]+)";', player)
+        devices.remove('unknown')
+        devices.insert(0, 'unknown')
+        for device in devices:
+            mobile_player = self._download_webpage(
+                '%s&device=%s' % (mobile_url, device), video_id,
+                'Downloading mobile player JS for %s device' % device)
+            videos_info = self._search_regex(
+                r'var streams=window.oo_testEnv\?\[\]:eval\("\((\[{.*?}\])\)"\);',
+                mobile_player, 'info', fatal=False, default=None)
+            if videos_info:
+                break
+
+        if not videos_info:
+            formats = []
+            auth_data = self._download_json(
+                'http://player.ooyala.com/sas/player_api/v1/authorization/embed_code/%s/%s?domain=www.example.org&supportedFormats=mp4,webm' % (embedCode, embedCode),
+                video_id)
+
+            cur_auth_data = auth_data['authorization_data'][video_id]
+
+            for stream in cur_auth_data['streams']:
+                formats.append({
+                    'url': base64.b64decode(stream['url']['data'].encode('ascii')).decode('utf-8'),
+                    'ext': stream.get('delivery_type'),
+                    'format': stream.get('video_codec'),
+                    'format_id': stream.get('profile'),
+                    'width': int_or_none(stream.get('width')),
+                    'height': int_or_none(stream.get('height')),
+                    'abr': int_or_none(stream.get('audio_bitrate')),
+                    'vbr': int_or_none(stream.get('video_bitrate')),
+                })
+            if formats:
+                return {
+                    'id': video_id,
+                    'formats': formats,
+                    'title': 'Ooyala video',
+                }
+
+            if not cur_auth_data['authorized']:
+                raise ExtractorError(cur_auth_data['message'], expected=True)
+
+        if not videos_info:
+            raise ExtractorError('Unable to extract info')
+        videos_info = videos_info.replace('\\"', '"')
+        videos_more_info = self._search_regex(
+            r'eval\("\(({.*?\\"promo\\".*?})\)"', mobile_player, 'more info').replace('\\"', '"')
+        videos_info = json.loads(videos_info)
+        videos_more_info = json.loads(videos_more_info)
+
+        if videos_more_info.get('lineup'):
+            videos = [self._extract_result(info, more_info) for (info, more_info) in zip(videos_info, videos_more_info['lineup'])]
+            return {
+                '_type': 'playlist',
+                'id': video_id,
+                'title': unescapeHTML(videos_more_info['title']),
+                'entries': videos,
+            }
+        else:
+            return self._extract_result(videos_info[0], videos_more_info)
+
+
+class OoyalaIE(OoyalaBaseIE):
     _VALID_URL = r'(?:ooyala:|https?://.+?\.ooyala\.com/.*?(?:embedCode|ec)=)(?P<id>.+?)(&|$)'
 
     _TESTS = [
@@ -57,95 +150,47 @@ class OoyalaIE(InfoExtractor):
         return cls.url_result(cls._url_for_embed_code(embed_code),
                               ie=cls.ie_key())
 
-    def _extract_result(self, info, more_info):
-        embedCode = info['embedCode']
-        video_url = info.get('ipad_url') or info['url']
+    def _real_extract(self, url):
+        embed_code = self._match_id(url)
+        player_url = 'http://player.ooyala.com/player.js?embedCode=%s' % embed_code
+        return self._extract(player_url, embed_code)
 
-        if determine_ext(video_url) == 'm3u8':
-            formats = self._extract_m3u8_formats(video_url, embedCode, ext='mp4')
-        else:
-            formats = [{
-                'url': video_url,
-                'ext': 'mp4',
-            }]
 
-        return {
-            'id': embedCode,
-            'title': unescapeHTML(info['title']),
-            'formats': formats,
-            'description': unescapeHTML(more_info['description']),
-            'thumbnail': more_info['promo'],
-        }
+class OoyalaExternalIE(OoyalaBaseIE):
+    _VALID_URL = r'''(?x)
+                    (?:
+                        ooyalaexternal:|
+                        https?://.+?\.ooyala\.com/.*?\bexternalId=
+                    )
+                    (?P<partner_id>[^:]+)
+                    :
+                    (?P<id>.+)
+                    (?:
+                        :|
+                        .*?&pcode=
+                    )
+                    (?P<pcode>.+?)
+                    (&|$)
+                    '''
+
+    _TEST = {
+        'url': 'https://player.ooyala.com/player.js?externalId=espn:10365079&pcode=1kNG061cgaoolOncv54OAO1ceO-I&adSetCode=91cDU6NuXTGKz3OdjOxFdAgJVtQcKJnI&callback=handleEvents&hasModuleParams=1&height=968&playerBrandingId=7af3bd04449c444c964f347f11873075&targetReplaceId=videoPlayer&width=1656&wmode=opaque&allowScriptAccess=always',
+        'info_dict': {
+            'id': 'FkYWtmazr6Ed8xmvILvKLWjd4QvYZpzG',
+            'ext': 'mp4',
+            'title': 'dm_140128_30for30Shorts___JudgingJewellv2',
+            'description': '',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        embedCode = mobj.group('id')
-        player_url = 'http://player.ooyala.com/player.js?embedCode=%s' % embedCode
-        player = self._download_webpage(player_url, embedCode)
-        mobile_url = self._search_regex(r'mobile_player_url="(.+?)&device="',
-                                        player, 'mobile player url')
-        # Looks like some videos are only available for particular devices
-        # (e.g. http://player.ooyala.com/player.js?embedCode=x1b3lqZDq9y_7kMyC2Op5qo-p077tXD0
-        # is only available for ipad)
-        # Working around with fetching URLs for all the devices found starting with 'unknown'
-        # until we succeed or eventually fail for each device.
-        devices = re.findall(r'device\s*=\s*"([^"]+)";', player)
-        devices.remove('unknown')
-        devices.insert(0, 'unknown')
-        for device in devices:
-            mobile_player = self._download_webpage(
-                '%s&device=%s' % (mobile_url, device), embedCode,
-                'Downloading mobile player JS for %s device' % device)
-            videos_info = self._search_regex(
-                r'var streams=window.oo_testEnv\?\[\]:eval\("\((\[{.*?}\])\)"\);',
-                mobile_player, 'info', fatal=False, default=None)
-            if videos_info:
-                break
-
-        if not videos_info:
-            formats = []
-            auth_data = self._download_json(
-                'http://player.ooyala.com/sas/player_api/v1/authorization/embed_code/%s/%s?domain=www.example.org&supportedFormats=mp4,webm' % (embedCode, embedCode),
-                embedCode)
-
-            cur_auth_data = auth_data['authorization_data'][embedCode]
-
-            for stream in cur_auth_data['streams']:
-                formats.append({
-                    'url': base64.b64decode(stream['url']['data'].encode('ascii')).decode('utf-8'),
-                    'ext': stream.get('delivery_type'),
-                    'format': stream.get('video_codec'),
-                    'format_id': stream.get('profile'),
-                    'width': int_or_none(stream.get('width')),
-                    'height': int_or_none(stream.get('height')),
-                    'abr': int_or_none(stream.get('audio_bitrate')),
-                    'vbr': int_or_none(stream.get('video_bitrate')),
-                })
-            if formats:
-                return {
-                    'id': embedCode,
-                    'formats': formats,
-                    'title': 'Ooyala video',
-                }
-
-            if not cur_auth_data['authorized']:
-                raise ExtractorError(cur_auth_data['message'], expected=True)
-
-        if not videos_info:
-            raise ExtractorError('Unable to extract info')
-        videos_info = videos_info.replace('\\"', '"')
-        videos_more_info = self._search_regex(
-            r'eval\("\(({.*?\\"promo\\".*?})\)"', mobile_player, 'more info').replace('\\"', '"')
-        videos_info = json.loads(videos_info)
-        videos_more_info = json.loads(videos_more_info)
-
-        if videos_more_info.get('lineup'):
-            videos = [self._extract_result(info, more_info) for (info, more_info) in zip(videos_info, videos_more_info['lineup'])]
-            return {
-                '_type': 'playlist',
-                'id': embedCode,
-                'title': unescapeHTML(videos_more_info['title']),
-                'entries': videos,
-            }
-        else:
-            return self._extract_result(videos_info[0], videos_more_info)
+        partner_id = mobj.group('partner_id')
+        video_id = mobj.group('id')
+        pcode = mobj.group('pcode')
+        player_url = 'http://player.ooyala.com/player.js?externalId=%s:%s&pcode=%s' % (partner_id, video_id, pcode)
+        return self._extract(player_url, video_id)

From 9354a5fad4521687eb9f08c1a42848621857400b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 16 May 2015 20:15:31 +0600
Subject: [PATCH 1322/6219] [ooyala] Fix unresolved reference

---
 youtube_dl/extractor/ooyala.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ooyala.py b/youtube_dl/extractor/ooyala.py
index 9f4fe5b29..a262a9f6d 100644
--- a/youtube_dl/extractor/ooyala.py
+++ b/youtube_dl/extractor/ooyala.py
@@ -59,7 +59,7 @@ class OoyalaBaseIE(InfoExtractor):
         if not videos_info:
             formats = []
             auth_data = self._download_json(
-                'http://player.ooyala.com/sas/player_api/v1/authorization/embed_code/%s/%s?domain=www.example.org&supportedFormats=mp4,webm' % (embedCode, embedCode),
+                'http://player.ooyala.com/sas/player_api/v1/authorization/embed_code/%s/%s?domain=www.example.org&supportedFormats=mp4,webm' % (video_id, video_id),
                 video_id)
 
             cur_auth_data = auth_data['authorization_data'][video_id]

From ef2dcbe4adce4478d409397faaae7ec6453ecf7d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 16 May 2015 21:07:29 +0600
Subject: [PATCH 1323/6219] [sbs] Fix extraction (Closes #5725)

---
 youtube_dl/extractor/sbs.py | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/sbs.py b/youtube_dl/extractor/sbs.py
index b8775c2f9..3073e5e86 100644
--- a/youtube_dl/extractor/sbs.py
+++ b/youtube_dl/extractor/sbs.py
@@ -33,16 +33,18 @@ class SBSIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
+
         webpage = self._download_webpage(url, video_id)
 
-        release_urls_json = js_to_json(self._search_regex(
+        player = self._search_regex(
             r'(?s)playerParams\.releaseUrls\s*=\s*(\{.*?\n\});\n',
-            webpage, ''))
-        release_urls = json.loads(release_urls_json)
-        theplatform_url = (
-            release_urls.get('progressive') or release_urls.get('standard'))
+            webpage, 'player')
+        player = re.sub(r"'\s*\+\s*[\da-zA-Z_]+\s*\+\s*'", '', player)
+
+        release_urls = self._parse_json(js_to_json(player), video_id)
+
+        theplatform_url = release_urls.get('progressive') or release_urls['standard']
 
         title = remove_end(self._og_search_title(webpage), ' (The Feed)')
         description = self._html_search_meta('description', webpage)
@@ -52,7 +54,6 @@ class SBSIE(InfoExtractor):
             '_type': 'url_transparent',
             'id': video_id,
             'url': theplatform_url,
-
             'title': title,
             'description': description,
             'thumbnail': thumbnail,

From 7e760fc18897663db7c0717434e28a8cca9f3810 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 16 May 2015 21:14:19 +0600
Subject: [PATCH 1324/6219] [espn] Add extractor (#4396)

Unfinished
---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/espn.py     | 55 ++++++++++++++++++++++++++++++++
 2 files changed, 56 insertions(+)
 create mode 100644 youtube_dl/extractor/espn.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 1731f4fb2..6b19eb6f8 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -141,6 +141,7 @@ from .engadget import EngadgetIE
 from .eporner import EpornerIE
 from .eroprofile import EroProfileIE
 from .escapist import EscapistIE
+from .espn import ESPNIE
 from .everyonesmixtape import EveryonesMixtapeIE
 from .exfm import ExfmIE
 from .expotv import ExpoTVIE
diff --git a/youtube_dl/extractor/espn.py b/youtube_dl/extractor/espn.py
new file mode 100644
index 000000000..e6f8f0337
--- /dev/null
+++ b/youtube_dl/extractor/espn.py
@@ -0,0 +1,55 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class ESPNIE(InfoExtractor):
+    _VALID_URL = r'https?://espn\.go\.com/(?:[^/]+/)*(?P<id>[^/]+)'
+    _WORKING = False
+    _TESTS = [{
+        'url': 'http://espn.go.com/video/clip?id=10365079',
+        'info_dict': {
+            'id': 'FkYWtmazr6Ed8xmvILvKLWjd4QvYZpzG',
+            'ext': 'mp4',
+            'title': 'dm_140128_30for30Shorts___JudgingJewellv2',
+            'description': '',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }, {
+        'url': 'https://espn.go.com/video/iframe/twitter/?cms=espn&id=10365079',
+        'only_matching': True,
+    }, {
+        'url': 'http://espn.go.com/nba/recap?gameId=400793786',
+        'only_matching': True,
+    }, {
+        'url': 'http://espn.go.com/blog/golden-state-warriors/post/_/id/593/how-warriors-rapidly-regained-a-winning-edge',
+        'only_matching': True,
+    }, {
+        'url': 'http://espn.go.com/sports/endurance/story/_/id/12893522/dzhokhar-tsarnaev-sentenced-role-boston-marathon-bombings',
+        'only_matching': True,
+    }, {
+        'url': 'http://espn.go.com/nba/playoffs/2015/story/_/id/12887571/john-wall-washington-wizards-no-swelling-left-hand-wrist-game-5-return',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        video_id = self._search_regex(
+            r'class="video-play-button"[^>]+data-id="(\d+)',
+            webpage, 'video id')
+
+        player = self._download_webpage(
+            'https://espn.go.com/video/iframe/twitter/?id=%s' % video_id, video_id)
+
+        pcode = self._search_regex(
+            r'["\']pcode=([^"\']+)["\']', player, 'pcode')
+
+        return self.url_result(
+            'ooyalaexternal:espn:%s:%s' % (video_id, pcode),
+            'OoyalaExternal')

From 4d52f2eb7f4b16ea5491f20abf0b29a1fcb24a02 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 16 May 2015 18:38:28 +0200
Subject: [PATCH 1325/6219] [sbs] Remove unused import

---
 youtube_dl/extractor/sbs.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/sbs.py b/youtube_dl/extractor/sbs.py
index 3073e5e86..d4bd1a0d7 100644
--- a/youtube_dl/extractor/sbs.py
+++ b/youtube_dl/extractor/sbs.py
@@ -1,7 +1,6 @@
 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
-import json
 import re
 from .common import InfoExtractor
 from ..utils import (

From 1c18de00192d195357989861563cc1fad9256128 Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Sun, 17 May 2015 01:38:50 +0800
Subject: [PATCH 1326/6219] [viki] Add proper paging and include clips

---
 youtube_dl/extractor/viki.py | 29 ++++++++++++++++++++---------
 1 file changed, 20 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index 40a73f561..4d477b03c 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -93,7 +93,7 @@ class VikiIE(InfoExtractor):
                     'Video %s is blocked from your location.' % video_id,
                     expected=True)
             else:
-                raise ExtractorError('Viki said: ' + err_msg)
+                raise ExtractorError('Viki said: %s %s' % (err_msg, url))
         mobj = re.search(
             r'<source[^>]+type="(?P<mime_type>[^"]+)"[^>]+src="(?P<url>[^"]+)"', info_webpage)
         if not mobj:
@@ -157,7 +157,15 @@ class VikiShowIE(InfoExtractor):
             'title': 'Boys Over Flowers',
             'description': 'md5:ecd3cff47967fe193cff37c0bec52790',
         },
-        'playlist_count': 25,
+        'playlist_count': 70,
+    }, {
+        'url': 'http://www.viki.com/tv/1354c-poor-nastya-complete',
+        'info_dict': {
+            'id': '1354c',
+            'title': 'Poor Nastya [COMPLETE]',
+            'description': 'md5:05bf5471385aa8b21c18ad450e350525',
+        },
+        'playlist_count': 127,
     }]
 
     def _real_extract(self, url):
@@ -167,13 +175,16 @@ class VikiShowIE(InfoExtractor):
         title = self._og_search_title(show_page)
         description = self._og_search_description(show_page)
 
-        show_json = self._download_json(
-            'http://api.viki.io/v4/containers/%s/episodes.json?app=100000a&per_page=999&sort=number&direction=asc' % show_id,
-            show_id, note='Retrieve show json', errnote='Unable to get show json')
         entries = []
-        for video in show_json['response']:
-            video_id = video['id']
-            entries.append(self.url_result(
-                'http://www.viki.com/videos/%s' % video_id, 'Viki', video_id))
+        for video_type in ['episodes', 'clips']:
+            json_url = 'http://api.viki.io/v4/containers/%s/%s.json?app=100000a&per_page=25&sort=number&direction=asc&with_paging=true&page=1' % (show_id, video_type)
+            while json_url is not None:
+                show_json = self._download_json(
+                    json_url, show_id, note='Retrieve show json', errnote='Unable to get show json')
+                for video in show_json['response']:
+                    video_id = video['id']
+                    entries.append(self.url_result(
+                        'http://www.viki.com/videos/%s' % video_id, 'Viki', video_id))
+                json_url = show_json['pagination']['next']
 
         return self.playlist_result(entries, show_id, title, description)

From baa43cbaf01a575eacb8e1bb39c7200f68c36daa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 17 May 2015 02:59:35 +0600
Subject: [PATCH 1327/6219] [extractor/common] Relax valid url check verbosity

---
 youtube_dl/extractor/common.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 65bb77086..cecf917ff 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -786,8 +786,8 @@ class InfoExtractor(object):
             return True
         except ExtractorError as e:
             if isinstance(e.cause, compat_HTTPError):
-                self.report_warning(
-                    '%s URL is invalid, skipping' % item, video_id)
+                self.to_screen(
+                    '%s: %s URL is invalid, skipping' % (video_id, item))
                 return False
             raise
 

From bc0f937b55aae6ce731d259a7658b0281c2e62ec Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 17 May 2015 03:01:52 +0600
Subject: [PATCH 1328/6219] [tv2] Add extractor (#5724)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/tv2.py      | 93 ++++++++++++++++++++++++++++++++
 2 files changed, 94 insertions(+)
 create mode 100644 youtube_dl/extractor/tv2.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 6b19eb6f8..fb4f63ca3 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -572,6 +572,7 @@ from .tumblr import TumblrIE
 from .tunein import TuneInIE
 from .turbo import TurboIE
 from .tutv import TutvIE
+from .tv2 import TV2IE
 from .tv4 import TV4IE
 from .tvigle import TvigleIE
 from .tvp import TvpIE, TvpSeriesIE
diff --git a/youtube_dl/extractor/tv2.py b/youtube_dl/extractor/tv2.py
new file mode 100644
index 000000000..2dcc0e971
--- /dev/null
+++ b/youtube_dl/extractor/tv2.py
@@ -0,0 +1,93 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    determine_ext,
+    int_or_none,
+    float_or_none,
+    parse_iso8601,
+)
+
+
+class TV2IE(InfoExtractor):
+    _VALID_URL = 'http://(?:www\.)?tv2\.no/v/(?P<id>\d+)'
+    _TEST = {
+        'url': 'http://www.tv2.no/v/916509/',
+        'md5': '9cb9e3410b18b515d71892f27856e9b1',
+        'info_dict': {
+            'id': '916509',
+            'ext': 'flv',
+            'title': 'Se Gryttens hyllest av Steven Gerrard',
+            'description': 'TV 2 Sportens huspoet tar avskjed med Liverpools kaptein Steven Gerrard.',
+            'timestamp': 1431715610,
+            'upload_date': '20150515',
+            'duration': 156.967,
+            'view_count': int,
+            'categories': list,
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        formats = []
+        format_urls = []
+        for protocol in ('HDS', 'HLS'):
+            data = self._download_json(
+                'http://sumo.tv2.no/api/web/asset/%s/play.json?protocol=%s&videoFormat=SMIL+ISMUSP' % (video_id, protocol),
+                video_id, 'Downloading play JSON')['playback']
+            for item in data['items']['item']:
+                video_url = item.get('url')
+                if not video_url or video_url in format_urls:
+                    continue
+                format_id = '%s-%s' % (protocol.lower(), item.get('mediaFormat'))
+                if not self._is_valid_url(video_url, video_id, format_id):
+                    continue
+                format_urls.append(video_url)
+                ext = determine_ext(video_url)
+                if ext == 'f4m':
+                    formats.extend(self._extract_f4m_formats(
+                        video_url, video_id, f4m_id=format_id))
+                elif ext == 'm3u8':
+                    formats.extend(self._extract_m3u8_formats(
+                        video_url, video_id, 'mp4', m3u8_id=format_id))
+                elif ext == 'ism' or video_url.endswith('.ism/Manifest'):
+                    pass
+                else:
+                    formats.append({
+                        'url': video_url,
+                        'format_id': format_id,
+                        'tbr': int_or_none(item.get('bitrate')),
+                        'filesize': int_or_none(item.get('fileSize')),
+                    })
+        self._sort_formats(formats)
+
+        asset = self._download_json(
+            'http://sumo.tv2.no/api/web/asset/%s.json' % video_id,
+            video_id, 'Downloading metadata JSON')['asset']
+
+        title = asset['title']
+        description = asset.get('description')
+        timestamp = parse_iso8601(asset.get('createTime'))
+        duration = float_or_none(asset.get('accurateDuration') or asset.get('duration'))
+        view_count = int_or_none(asset.get('views'))
+        categories = asset.get('keywords', '').split(',')
+
+        thumbnails = [{
+            'id': thumbnail.get('@type'),
+            'url': thumbnail.get('url'),
+        } for _, thumbnail in asset.get('imageVersions', {}).items()]
+
+        return {
+            'id': video_id,
+            'url': video_url,
+            'title': title,
+            'description': description,
+            'thumbnails': thumbnails,
+            'timestamp': timestamp,
+            'duration': duration,
+            'view_count': view_count,
+            'categories': categories,
+            'formats': formats,
+        }

From 588b82bbf8c90981c54f180eca40e6c743f8f89f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 17 May 2015 03:32:53 +0600
Subject: [PATCH 1329/6219] [tv2:article] Add extractor (Closes #5724)

---
 youtube_dl/extractor/__init__.py |  5 ++++-
 youtube_dl/extractor/tv2.py      | 33 ++++++++++++++++++++++++++++++++
 2 files changed, 37 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index fb4f63ca3..6f8c261d5 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -572,7 +572,10 @@ from .tumblr import TumblrIE
 from .tunein import TuneInIE
 from .turbo import TurboIE
 from .tutv import TutvIE
-from .tv2 import TV2IE
+from .tv2 import (
+    TV2IE,
+    TV2ArticleIE,
+)
 from .tv4 import TV4IE
 from .tvigle import TvigleIE
 from .tvp import TvpIE, TvpSeriesIE
diff --git a/youtube_dl/extractor/tv2.py b/youtube_dl/extractor/tv2.py
index 2dcc0e971..fa338b936 100644
--- a/youtube_dl/extractor/tv2.py
+++ b/youtube_dl/extractor/tv2.py
@@ -1,12 +1,15 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..utils import (
     determine_ext,
     int_or_none,
     float_or_none,
     parse_iso8601,
+    remove_end,
 )
 
 
@@ -91,3 +94,33 @@ class TV2IE(InfoExtractor):
             'categories': categories,
             'formats': formats,
         }
+
+
+class TV2ArticleIE(InfoExtractor):
+    _VALID_URL = 'http://(?:www\.)?tv2\.no/(?:a|\d{4}/\d{2}/\d{2}(/[^/]+)+)/(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'http://www.tv2.no/2015/05/16/nyheter/alesund/krim/pingvin/6930542',
+        'info_dict': {
+            'id': '6930542',
+            'title': 'Russen hetses etter pingvintyveri – innrømmer å ha åpnet luken på buret',
+            'description': 'md5:339573779d3eea3542ffe12006190954',
+        },
+        'playlist_count': 2,
+    }, {
+        'url': 'http://www.tv2.no/a/6930542',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        playlist_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, playlist_id)
+
+        entries = [
+            self.url_result('http://www.tv2.no/v/%s' % video_id, 'TV2')
+            for video_id in re.findall(r'data-assetid="(\d+)"', webpage)]
+
+        title = remove_end(self._og_search_title(webpage), ' - TV2.no')
+        description = remove_end(self._og_search_description(webpage), ' - TV2.no')
+
+        return self.playlist_result(entries, playlist_id, title, description)

From 8da0e0e94682faa0463f33d991df70a2402b5a86 Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Sun, 17 May 2015 06:19:38 +0800
Subject: [PATCH 1330/6219] [viki] Change IE name to channel, better message
 output

---
 youtube_dl/extractor/__init__.py |  2 +-
 youtube_dl/extractor/viki.py     | 11 +++++++----
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index cb6635610..21f7b7290 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -641,7 +641,7 @@ from .vine import (
 )
 from .viki import (
     VikiIE,
-    VikiShowIE,
+    VikiChannelIE,
 )
 from .vk import (
     VKIE,
diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index 4d477b03c..9bdbdc3e4 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -147,8 +147,8 @@ class VikiIE(InfoExtractor):
         return res
 
 
-class VikiShowIE(InfoExtractor):
-    IE_NAME = 'viki:show'
+class VikiChannelIE(InfoExtractor):
+    IE_NAME = 'viki:channel'
     _VALID_URL = r'^https?://(?:www\.)?viki\.com/tv/(?P<id>[0-9]+c)'
     _TESTS = [{
         'url': 'http://www.viki.com/tv/50c-boys-over-flowers',
@@ -167,6 +167,7 @@ class VikiShowIE(InfoExtractor):
         },
         'playlist_count': 127,
     }]
+    _PER_PAGE = 25
 
     def _real_extract(self, url):
         show_id = self._match_id(url)
@@ -177,10 +178,12 @@ class VikiShowIE(InfoExtractor):
 
         entries = []
         for video_type in ['episodes', 'clips']:
-            json_url = 'http://api.viki.io/v4/containers/%s/%s.json?app=100000a&per_page=25&sort=number&direction=asc&with_paging=true&page=1' % (show_id, video_type)
+            json_url = 'http://api.viki.io/v4/containers/%s/%s.json?app=100000a&per_page=%d&sort=number&direction=asc&with_paging=true&page=1' % (show_id, video_type, self._PER_PAGE)
             while json_url is not None:
                 show_json = self._download_json(
-                    json_url, show_id, note='Retrieve show json', errnote='Unable to get show json')
+                    json_url, show_id,
+                    note='Downloading %s json page #%s' %
+                         (video_type, re.search(r'[?&]page=([0-9]+)', json_url).group(1)))
                 for video in show_json['response']:
                     video_id = video['id']
                     entries.append(self.url_result(

From 725652e9247e1171110b624d748e20fa1c88260e Mon Sep 17 00:00:00 2001
From: Mister Hat <misterhat144@gmail.com>
Date: Sat, 16 May 2015 19:50:58 -0500
Subject: [PATCH 1331/6219] [karrierevideos] add support for
 www.karrierevideos.at (closes #5354)

---
 youtube_dl/extractor/__init__.py       |  1 +
 youtube_dl/extractor/karrierevideos.py | 52 ++++++++++++++++++++++++++
 2 files changed, 53 insertions(+)
 create mode 100644 youtube_dl/extractor/karrierevideos.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index fb4f63ca3..d131d3ec3 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -244,6 +244,7 @@ from .kaltura import KalturaIE
 from .kanalplay import KanalPlayIE
 from .kankan import KankanIE
 from .karaoketv import KaraoketvIE
+from .karrierevideos import KarriereVideosIE
 from .keezmovies import KeezMoviesIE
 from .khanacademy import KhanAcademyIE
 from .kickstarter import KickStarterIE
diff --git a/youtube_dl/extractor/karrierevideos.py b/youtube_dl/extractor/karrierevideos.py
new file mode 100644
index 000000000..59d29e845
--- /dev/null
+++ b/youtube_dl/extractor/karrierevideos.py
@@ -0,0 +1,52 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class KarriereVideosIE(InfoExtractor):
+    _VALID_URL = r'http://(?:www\.)?karrierevideos\.at/berufsvideos/([a-z-]+)/(?P<id>[a-z-]+)'
+    _TEST = {
+        'url': 'http://www.karrierevideos.at/berufsvideos/mittlere-hoehere-schulen/altenpflegerin',
+        'info_dict': {
+            'id': 'altenpflegerin',
+            'ext': 'mp4',
+            'title': 'AltenpflegerIn',
+            'thumbnail': 're:^http://.*\.png\?v=[0-9]+',
+            'description': 'md5:dbadd1259fde2159a9b28667cb664ae2'
+        },
+        'params': {
+            'skip_download': 'requires rtmpdump'
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        description = self._html_search_regex(
+            r'<div class="leadtext">\n{0,}?\s{0,}<p>(.*?)</p>',
+            webpage, 'description')
+
+        playlist = self._html_search_regex(r'/config/video/(.*?)\.xml', webpage, 'playlist')
+        playlist = self._download_xml(
+            'http://www.karrierevideos.at/player-playlist.xml.php?p=%s' % playlist,
+            video_id)
+
+        namespace = {
+            'jwplayer': 'http://developer.longtailvideo.com/trac/wiki/FlashFormats'
+        }
+
+        item = playlist.find('tracklist/item')
+        streamer = item.find('jwplayer:streamer', namespace).text
+
+        return {
+            'id': video_id,
+            'title': self._html_search_meta('title', webpage),
+            'description': description,
+            'thumbnail': 'http://www.karrierevideos.at' + self._html_search_meta('thumbnail', webpage),
+            'protocol': 'rtmp',
+            'url': streamer.replace('rtmpt', 'http'),
+            'play_path': 'mp4:' + item.find('jwplayer:file', namespace).text,
+            'tc_url': streamer,
+            'ext': 'mp4'
+        }

From ba9d16291b8ace3bd412bcfc0c128c047545e509 Mon Sep 17 00:00:00 2001
From: Mister Hat <misterhat144@gmail.com>
Date: Sun, 17 May 2015 03:35:08 -0500
Subject: [PATCH 1332/6219] manually specify namespace

---
 youtube_dl/extractor/karrierevideos.py | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/karrierevideos.py b/youtube_dl/extractor/karrierevideos.py
index 59d29e845..a05e8ab76 100644
--- a/youtube_dl/extractor/karrierevideos.py
+++ b/youtube_dl/extractor/karrierevideos.py
@@ -29,15 +29,13 @@ class KarriereVideosIE(InfoExtractor):
 
         playlist = self._html_search_regex(r'/config/video/(.*?)\.xml', webpage, 'playlist')
         playlist = self._download_xml(
-            'http://www.karrierevideos.at/player-playlist.xml.php?p=%s' % playlist,
+            'http://www.karrierevideos.at/player-playlist.xml.php?p=' + playlist,
             video_id)
 
-        namespace = {
-            'jwplayer': 'http://developer.longtailvideo.com/trac/wiki/FlashFormats'
-        }
+        namespace = 'http://developer.longtailvideo.com/trac/wiki/FlashFormats'
 
         item = playlist.find('tracklist/item')
-        streamer = item.find('jwplayer:streamer', namespace).text
+        streamer = item.find('{%s}streamer' % namespace).text
 
         return {
             'id': video_id,
@@ -46,7 +44,7 @@ class KarriereVideosIE(InfoExtractor):
             'thumbnail': 'http://www.karrierevideos.at' + self._html_search_meta('thumbnail', webpage),
             'protocol': 'rtmp',
             'url': streamer.replace('rtmpt', 'http'),
-            'play_path': 'mp4:' + item.find('jwplayer:file', namespace).text,
+            'play_path': 'mp4:' + item.find('{%s}file' % namespace).text,
             'tc_url': streamer,
             'ext': 'mp4'
         }

From 4a5a898a8fa392d02102672f9767f33a39a73066 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 17 May 2015 20:56:03 +0600
Subject: [PATCH 1333/6219] [YoutubeDL] Clarify incompatible formats merge
 message

When `-f` is not specified it's misleading to see `You have requested ...` as user did not actually request any formats.
---
 youtube_dl/YoutubeDL.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 5df889945..58b34e087 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1368,7 +1368,7 @@ class YoutubeDL(object):
                         postprocessors = []
                         self.report_warning('You have requested multiple '
                                             'formats but ffmpeg or avconv are not installed.'
-                                            ' The formats won\'t be merged')
+                                            ' The formats won\'t be merged.')
                     else:
                         postprocessors = [merger]
 
@@ -1395,8 +1395,8 @@ class YoutubeDL(object):
                     requested_formats = info_dict['requested_formats']
                     if self.params.get('merge_output_format') is None and not compatible_formats(requested_formats):
                         info_dict['ext'] = 'mkv'
-                        self.report_warning('You have requested formats incompatible for merge. '
-                                            'The formats will be merged into mkv')
+                        self.report_warning(
+                            'Requested formats are incompatible for merge and will be merged into mkv.')
                     # Ensure filename always has a correct extension for successful merge
                     filename = '%s.%s' % (filename_wo_ext, info_dict['ext'])
                     if os.path.exists(encodeFilename(filename)):

From fc6e75dd57f3497b99def659b3d0f173b195b7d0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 18 May 2015 11:21:09 +0200
Subject: [PATCH 1334/6219] [instagram] Only recognize https urls (fixes #5739)

http urls redirect to them.
---
 youtube_dl/extractor/instagram.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/instagram.py b/youtube_dl/extractor/instagram.py
index 65f6ca103..b10755788 100644
--- a/youtube_dl/extractor/instagram.py
+++ b/youtube_dl/extractor/instagram.py
@@ -7,9 +7,9 @@ from ..utils import int_or_none
 
 
 class InstagramIE(InfoExtractor):
-    _VALID_URL = r'https?://instagram\.com/p/(?P<id>[\da-zA-Z]+)'
+    _VALID_URL = r'https://instagram\.com/p/(?P<id>[\da-zA-Z]+)'
     _TEST = {
-        'url': 'http://instagram.com/p/aye83DjauH/?foo=bar#abc',
+        'url': 'https://instagram.com/p/aye83DjauH/?foo=bar#abc',
         'md5': '0d2da106a9d2631273e192b372806516',
         'info_dict': {
             'id': 'aye83DjauH',
@@ -41,11 +41,11 @@ class InstagramIE(InfoExtractor):
 
 
 class InstagramUserIE(InfoExtractor):
-    _VALID_URL = r'http://instagram\.com/(?P<username>[^/]{2,})/?(?:$|[?#])'
+    _VALID_URL = r'https://instagram\.com/(?P<username>[^/]{2,})/?(?:$|[?#])'
     IE_DESC = 'Instagram user profile'
     IE_NAME = 'instagram:user'
     _TEST = {
-        'url': 'http://instagram.com/porsche',
+        'url': 'https://instagram.com/porsche',
         'info_dict': {
             'id': 'porsche',
             'title': 'porsche',

From 5bdc520cf19f404247ec2be1ffc1e83449fa2375 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 18 May 2015 21:23:05 +0600
Subject: [PATCH 1335/6219] [xminus] Fix extraction

---
 youtube_dl/extractor/xminus.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/xminus.py b/youtube_dl/extractor/xminus.py
index 8c6241aed..7c9d8af6f 100644
--- a/youtube_dl/extractor/xminus.py
+++ b/youtube_dl/extractor/xminus.py
@@ -43,7 +43,7 @@ class XMinusIE(InfoExtractor):
             r'minus_track\.dur_sec=\'([0-9]*?)\'',
             webpage, 'duration', fatal=False))
         filesize_approx = parse_filesize(self._html_search_regex(
-            r'<div class="filesize[^"]*"></div>\s*([0-9.]+\s*[a-zA-Z][bB])',
+            r'<div id="finfo"[^>]*>\s*↓\s*([0-9.]+\s*[a-zA-Z][bB])',
             webpage, 'approximate filesize', fatal=False))
         tbr = int_or_none(self._html_search_regex(
             r'<div class="quality[^"]*"></div>\s*([0-9]+)\s*kbps',
@@ -58,7 +58,7 @@ class XMinusIE(InfoExtractor):
             description = re.sub(' *\r *', '\n', description)
 
         enc_token = self._html_search_regex(
-            r'minus_track\.tkn="(.+?)"', webpage, 'enc_token')
+            r'minus_track\.s?tkn="(.+?)"', webpage, 'enc_token')
         token = ''.join(
             c if pos == 3 else compat_chr(compat_ord(c) - 1)
             for pos, c in enumerate(reversed(enc_token)))

From 4f514c7e88d2ce8ebe9c2478183e8797cfb2a4c4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 18 May 2015 21:29:41 +0600
Subject: [PATCH 1336/6219] [wimp] Fix youtube extraction (Closes #5690)

---
 youtube_dl/extractor/wimp.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/wimp.py b/youtube_dl/extractor/wimp.py
index d6dec25ca..f69d46a28 100644
--- a/youtube_dl/extractor/wimp.py
+++ b/youtube_dl/extractor/wimp.py
@@ -37,7 +37,8 @@ class WimpIE(InfoExtractor):
         video_id = mobj.group(1)
         webpage = self._download_webpage(url, video_id)
         video_url = self._search_regex(
-            r"[\"']file[\"']\s*[:,]\s*[\"'](.+?)[\"']", webpage, 'video URL')
+            [r"[\"']file[\"']\s*[:,]\s*[\"'](.+?)[\"']", r"videoId\s*:\s*[\"']([^\"']+)[\"']"],
+            webpage, 'video URL')
         if YoutubeIE.suitable(video_url):
             self.to_screen('Found YouTube video')
             return {

From 2328f2fe684f9a9025217c6f149e92a403a4c437 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 18 May 2015 21:34:20 +0600
Subject: [PATCH 1337/6219] [vulture] Fix extraction

---
 youtube_dl/extractor/vulture.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vulture.py b/youtube_dl/extractor/vulture.py
index 1eb24a3d6..faa167e65 100644
--- a/youtube_dl/extractor/vulture.py
+++ b/youtube_dl/extractor/vulture.py
@@ -44,7 +44,7 @@ class VultureIE(InfoExtractor):
         query_webpage = self._download_webpage(
             query_url, display_id, note='Downloading query page')
         params_json = self._search_regex(
-            r'(?sm)new MagnifyEmbeddablePlayer\({.*?contentItem:\s*(\{.*?\})\n,\n',
+            r'(?sm)new MagnifyEmbeddablePlayer\({.*?contentItem:\s*(\{.*?\})\n?,\n',
             query_webpage,
             'player params')
         params = json.loads(params_json)

From 5d8dcb5342c97b05c037c8c4e80002540db261b8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 18 May 2015 21:39:15 +0600
Subject: [PATCH 1338/6219] [vuclip] Fix extraction

---
 youtube_dl/extractor/vuclip.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vuclip.py b/youtube_dl/extractor/vuclip.py
index c3fde53f5..a6d9b5fee 100644
--- a/youtube_dl/extractor/vuclip.py
+++ b/youtube_dl/extractor/vuclip.py
@@ -49,7 +49,7 @@ class VuClipIE(InfoExtractor):
         links_code = self._search_regex(
             r'''(?xs)
                 (?:
-                    <img\s+src="/im/play.gif".*?>|
+                    <img\s+src="[^"]*/play.gif".*?>|
                     <!--\ player\ end\ -->\s*</div><!--\ thumb\ end-->
                 )
                 (.*?)

From 484c9d2d5b669220c24c865947c3f65049916b56 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 18 May 2015 21:43:54 +0600
Subject: [PATCH 1339/6219] [vier] Fix extraction

---
 youtube_dl/extractor/vier.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/vier.py b/youtube_dl/extractor/vier.py
index 619039e51..15377097e 100644
--- a/youtube_dl/extractor/vier.py
+++ b/youtube_dl/extractor/vier.py
@@ -38,11 +38,14 @@ class VierIE(InfoExtractor):
         webpage = self._download_webpage(url, display_id)
 
         video_id = self._search_regex(
-            r'"nid"\s*:\s*"(\d+)"', webpage, 'video id')
+            [r'data-nid="(\d+)"', r'"nid"\s*:\s*"(\d+)"'],
+            webpage, 'video id')
         application = self._search_regex(
-            r'"application"\s*:\s*"([^"]+)"', webpage, 'application', default='vier_vod')
+            [r'data-application="([^"]+)"', r'"application"\s*:\s*"([^"]+)"'],
+            webpage, 'application', default='vier_vod')
         filename = self._search_regex(
-            r'"filename"\s*:\s*"([^"]+)"', webpage, 'filename')
+            [r'data-filename="([^"]+)"', r'"filename"\s*:\s*"([^"]+)"'],
+            webpage, 'filename')
 
         playlist_url = 'http://vod.streamcloud.be/%s/mp4:_definst_/%s.mp4/playlist.m3u8' % (application, filename)
         formats = self._extract_m3u8_formats(playlist_url, display_id, 'mp4')

From 2aa64b89b3ac8f387d4c0c27ce7de64bc0ff68de Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 18 May 2015 17:58:53 +0200
Subject: [PATCH 1340/6219] tox: Pass HOME environment variable

Since version 2.0 it only passes a limited set of variables and we need HOME for the tests
---
 tox.ini | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tox.ini b/tox.ini
index 00c6e00e3..cd805fe8a 100644
--- a/tox.ini
+++ b/tox.ini
@@ -4,6 +4,8 @@ envlist = py26,py27,py33,py34
 deps =
    nose
    coverage
+# We need a valid $HOME for test_compat_expanduser
+passenv = HOME
 defaultargs = test --exclude test_download.py --exclude test_age_restriction.py
     --exclude test_subtitles.py --exclude test_write_annotations.py
     --exclude test_youtube_lists.py

From 1b0427e6c433c0b6db5e210db6e3173e19e702ed Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 19 May 2015 00:45:01 +0800
Subject: [PATCH 1341/6219] [utils] Support TTML without default namespace

In a strict sense such TTML is invalid, but Yahoo uses it.
---
 test/test_utils.py  | 15 +++++++++++++++
 youtube_dl/utils.py |  9 ++++++---
 2 files changed, 21 insertions(+), 3 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index b40107037..e13e11b59 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -621,6 +621,21 @@ Line
 '''
         self.assertEqual(dfxp2srt(dfxp_data), srt_data)
 
+        dfxp_data_no_default_namespace = '''<?xml version="1.0" encoding="UTF-8"?>
+            <tt xml:lang="en" xmlns:tts="http://www.w3.org/ns/ttml#parameter">
+            <body>
+                <div xml:lang="en">
+                    <p begin="0" end="1">The first line</p>
+                </div>
+            </body>
+            </tt>'''
+        srt_data = '''1
+00:00:00,000 --> 00:00:01,000
+The first line
+
+'''
+        self.assertEqual(dfxp2srt(dfxp_data_no_default_namespace), srt_data)
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index ed9ed9ed6..507f07383 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1848,9 +1848,9 @@ def dfxp2srt(dfxp_data):
         out = str_or_empty(node.text)
 
         for child in node:
-            if child.tag == _x('ttml:br'):
+            if child.tag in (_x('ttml:br'), 'br'):
                 out += '\n' + str_or_empty(child.tail)
-            elif child.tag == _x('ttml:span'):
+            elif child.tag in (_x('ttml:span'), 'span'):
                 out += str_or_empty(parse_node(child))
             else:
                 out += str_or_empty(xml.etree.ElementTree.tostring(child))
@@ -1859,7 +1859,10 @@ def dfxp2srt(dfxp_data):
 
     dfxp = xml.etree.ElementTree.fromstring(dfxp_data.encode('utf-8'))
     out = []
-    paras = dfxp.findall(_x('.//ttml:p'))
+    paras = dfxp.findall(_x('.//ttml:p')) or dfxp.findall('.//p')
+
+    if not paras:
+        raise ValueError('Invalid dfxp/TTML subtitle')
 
     for para, index in zip(paras, itertools.count(1)):
         begin_time = parse_dfxp_time_expr(para.attrib['begin'])

From ecee5724110847b832a6074c66ca4a63758100f4 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 19 May 2015 00:50:24 +0800
Subject: [PATCH 1342/6219] [yahoo] Add support for closed captions (closes
 #5714)

---
 youtube_dl/extractor/yahoo.py | 18 ++++++++++++++++++
 youtube_dl/utils.py           |  1 +
 2 files changed, 19 insertions(+)

diff --git a/youtube_dl/extractor/yahoo.py b/youtube_dl/extractor/yahoo.py
index bf4e659ac..f9afbdbab 100644
--- a/youtube_dl/extractor/yahoo.py
+++ b/youtube_dl/extractor/yahoo.py
@@ -15,6 +15,7 @@ from ..utils import (
     unescapeHTML,
     ExtractorError,
     int_or_none,
+    mimetype2ext,
 )
 
 from .nbc import NBCSportsVPlayerIE
@@ -236,6 +237,22 @@ class YahooIE(InfoExtractor):
 
         self._sort_formats(formats)
 
+        closed_captions = self._html_search_regex(
+            r'"closedcaptions":(\[[^\]]+\])', webpage, 'closed captions',
+            default='[]')
+
+        cc_json = self._parse_json(closed_captions, video_id, fatal=False)
+        subtitles = {}
+        if cc_json:
+            for closed_caption in cc_json:
+                lang = closed_caption['lang']
+                if lang not in subtitles:
+                    subtitles[lang] = []
+                subtitles[lang].append({
+                    'url': closed_caption['url'],
+                    'ext': mimetype2ext(closed_caption['content_type']),
+                })
+
         return {
             'id': video_id,
             'display_id': display_id,
@@ -244,6 +261,7 @@ class YahooIE(InfoExtractor):
             'description': clean_html(meta['description']),
             'thumbnail': meta['thumbnail'] if meta.get('thumbnail') else self._og_search_thumbnail(webpage),
             'duration': int_or_none(meta.get('duration')),
+            'subtitles': subtitles,
         }
 
 
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 507f07383..52d198fa3 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1665,6 +1665,7 @@ def mimetype2ext(mt):
     return {
         'x-ms-wmv': 'wmv',
         'x-mp4-fragmented': 'mp4',
+        'ttml+xml': 'ttml',
     }.get(res, res)
 
 

From b813d8caf1b23821036b77b851e42ba0a0ad35a7 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 19 May 2015 01:01:42 +0800
Subject: [PATCH 1343/6219] [qqmusic] Unescape '\\n' in description (#5705)

---
 youtube_dl/extractor/qqmusic.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index 13113820b..b540033e2 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -26,7 +26,7 @@ class QQMusicIE(InfoExtractor):
             'title': '可惜没如果',
             'upload_date': '20141227',
             'creator': '林俊杰',
-            'description': 'md5:4348ff1dd24036906baa7b6f973f8d30',
+            'description': 'md5:d327722d0361576fde558f1ac68a7065',
         }
     }]
 
@@ -60,6 +60,8 @@ class QQMusicIE(InfoExtractor):
         lrc_content = self._html_search_regex(
             r'<div class="content" id="lrc_content"[^<>]*>([^<>]+)</div>',
             detail_info_page, 'LRC lyrics', default=None)
+        if lrc_content:
+            lrc_content = lrc_content.replace('\\n', '\n')
 
         guid = self.m_r_get_ruin()
 

From d9d747a06ab3b4c36c6063074ffb42aeb185431f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 19 May 2015 21:28:41 +0600
Subject: [PATCH 1344/6219] [ultimedia] Fix extraction

---
 youtube_dl/extractor/ultimedia.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/ultimedia.py b/youtube_dl/extractor/ultimedia.py
index 96c809eaf..c4751050e 100644
--- a/youtube_dl/extractor/ultimedia.py
+++ b/youtube_dl/extractor/ultimedia.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import compat_urllib_parse_urlparse
 from ..utils import (
     ExtractorError,
     qualities,
@@ -44,9 +45,9 @@ class UltimediaIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        deliver_url = self._search_regex(
-            r'<iframe[^>]+src="(https?://(?:www\.)?ultimedia\.com/deliver/[^"]+)"',
-            webpage, 'deliver URL')
+        deliver_url = self._proto_relative_url(self._search_regex(
+            r'<iframe[^>]+src="((?:https?:)?//(?:www\.)?ultimedia\.com/deliver/[^"]+)"',
+            webpage, 'deliver URL'), compat_urllib_parse_urlparse(url).scheme + ':')
 
         deliver_page = self._download_webpage(
             deliver_url, video_id, 'Downloading iframe page')
@@ -57,7 +58,8 @@ class UltimediaIE(InfoExtractor):
 
         player = self._parse_json(
             self._search_regex(
-                r"jwplayer\('player(?:_temp)?'\)\.setup\(({.+?})\)\.on", deliver_page, 'player'),
+                r"jwplayer\('player(?:_temp)?'\)\.setup\(({.+?})\)\.on",
+                deliver_page, 'player'),
             video_id)
 
         quality = qualities(['flash', 'html5'])

From f670ef1c8ebe0329a68b3a3d5c2b7e07ae5c9425 Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Wed, 20 May 2015 13:51:43 +0800
Subject: [PATCH 1345/6219] [dramafever] Add new extractor for dramafever.com

---
 youtube_dl/extractor/__init__.py   |   4 +
 youtube_dl/extractor/dramafever.py | 131 +++++++++++++++++++++++++++++
 2 files changed, 135 insertions(+)
 create mode 100644 youtube_dl/extractor/dramafever.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 6f8c261d5..ca857a75f 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -112,6 +112,10 @@ from .dfb import DFBIE
 from .dhm import DHMIE
 from .dotsub import DotsubIE
 from .douyutv import DouyuTVIE
+from .dramafever import (
+    DramaFeverIE,
+    DramaFeverSeriesIE,
+)
 from .dreisat import DreiSatIE
 from .drbonanza import DRBonanzaIE
 from .drtuber import DrTuberIE
diff --git a/youtube_dl/extractor/dramafever.py b/youtube_dl/extractor/dramafever.py
new file mode 100644
index 000000000..8fac99cc5
--- /dev/null
+++ b/youtube_dl/extractor/dramafever.py
@@ -0,0 +1,131 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+
+class DramaFeverIE(InfoExtractor):
+    IE_NAME = 'dramafever'
+    _VALID_URL = r'^https?://(?:www\.)?dramafever\.com/drama/(?P<id>[0-9]+/[0-9]+)/'
+    _TESTS = [{
+        'url': 'http://www.dramafever.com/drama/4512/1/Cooking_with_Shin/',
+        'info_dict': {
+            'id': '4512.1',
+            'ext': 'flv',
+            'title': 'Cooking with Shin 4512.1',
+            'upload_date': '20140702',
+            'description': 'Served at all special occasions and featured in the hit drama Heirs, Shin cooks Red Bean Rice.',
+        }
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url).replace("/", ".")
+
+        consumer_secret = self._get_consumer_secret(video_id)
+
+        ep_json = self._download_json(
+            "http://www.dramafever.com/amp/episode/feed.json?guid=%s" % video_id,
+            video_id, note='Downloading episode metadata',
+            errnote="Video may not be available for your location")["channel"]["item"]
+
+        title = ep_json["media-group"]["media-title"]
+        description = ep_json["media-group"]["media-description"]
+        thumbnail = ep_json["media-group"]["media-thumbnail"]["@attributes"]["url"]
+        duration = int(ep_json["media-group"]["media-content"][0]["@attributes"]["duration"])
+        mobj = re.match(r"([0-9]{4})-([0-9]{2})-([0-9]{2})", ep_json["pubDate"])
+        upload_date = mobj.group(1) + mobj.group(2) + mobj.group(3) if mobj is not None else None
+
+        formats = []
+        for vid_format in ep_json["media-group"]["media-content"]:
+            src = vid_format["@attributes"]["url"]
+            if '.f4m' in src:
+                formats.extend(self._extract_f4m_formats(src, video_id))
+
+        self._sort_formats(formats)
+        video_subtitles = self.extract_subtitles(video_id, consumer_secret)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'upload_date': upload_date,
+            'duration': duration,
+            'formats': formats,
+            'subtitles': video_subtitles,
+        }
+
+    def _get_consumer_secret(self, video_id):
+        df_js = self._download_webpage(
+            "http://www.dramafever.com/static/126960d/v2/js/plugins/jquery.threadedcomments.js", video_id)
+        return self._search_regex(r"'cs': '([0-9a-zA-Z]+)'", df_js, "cs")
+
+    def _get_episodes(self, series_id, consumer_secret, episode_filter=None):
+        _PAGE_SIZE = 60
+
+        curr_page = 1
+        max_pages = curr_page + 1
+        results = []
+        while max_pages >= curr_page:
+            page_url = "http://www.dramafever.com/api/4/episode/series/?cs=%s&series_id=%s&page_size=%d&page_number=%d" % \
+                       (consumer_secret, series_id, _PAGE_SIZE, curr_page)
+            series = self._download_json(
+                page_url, series_id, note="Downloading series json page #%d" % curr_page)
+            max_pages = series['num_pages']
+            results.extend([ep for ep in series['value'] if episode_filter is None or episode_filter(ep)])
+            curr_page += 1
+        return results
+
+    def _get_subtitles(self, video_id, consumer_secret):
+
+        def match_episode(ep):
+            return ep['guid'] == video_id
+
+        res = None
+        info = self._get_episodes(
+            video_id.split(".")[0], consumer_secret, episode_filter=match_episode)
+        if len(info) == 1 and info[0]['subfile'] != '':
+            res = {'en': [{'url': info[0]['subfile'], 'ext': 'srt'}]}
+        return res
+
+
+class DramaFeverSeriesIE(DramaFeverIE):
+    IE_NAME = 'dramafever:series'
+    _VALID_URL = r'^https?://(?:www\.)?dramafever\.com/drama/(?P<id>[0-9]+)/\d*[a-zA-Z_][a-zA-Z0-9_]*/'
+    _TESTS = [{
+        'url': 'http://www.dramafever.com/drama/4512/Cooking_with_Shin/',
+        'info_dict': {
+            'id': '4512',
+            'title': 'Cooking with Shin',
+            'description': 'Professional chef and cooking instructor Shin Kim takes some of the delicious dishes featured in your favorite dramas and shows you how to make them right at home.',
+        },
+        'playlist_count': 4,
+    }, {
+        'url': 'http://www.dramafever.com/drama/124/IRIS/',
+        'info_dict': {
+            'id': '124',
+            'title': 'IRIS',
+            'description': 'Lee Byung Hun and Kim Tae Hee star in this powerhouse drama and ratings megahit of action, intrigue and romance.',
+        },
+        'playlist_count': 20,
+    }]
+
+    def _real_extract(self, url):
+        series_id = self._match_id(url)
+        consumer_secret = self._get_consumer_secret(series_id)
+
+        series_json = self._download_json(
+            "http://www.dramafever.com/api/4/series/query/?cs=%s&series_id=%s" % (consumer_secret, series_id),
+            series_id, note='Downloading series metadata')["series"][series_id]
+
+        title = series_json["name"]
+        description = series_json["description_short"]
+
+        episodes = self._get_episodes(series_id, consumer_secret)
+        entries = []
+        for ep in episodes:
+            entries.append(self.url_result(
+                'http://www.dramafever.com%s' % ep['episode_url'], 'DramaFever', ep['guid']))
+        return self.playlist_result(entries, series_id, title, description)

From 051df9ad99d0a29d9eb984970e3e431795b6e445 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 20 May 2015 14:08:23 +0800
Subject: [PATCH 1346/6219] [letv/sohu] Skip tests relying on external proxies

The proxy is currently broken. See #5655 and zhuzhuor/Unblock-Youku#427
---
 youtube_dl/extractor/letv.py | 4 +---
 youtube_dl/extractor/sohu.py | 4 +---
 2 files changed, 2 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/letv.py b/youtube_dl/extractor/letv.py
index 1484ac0d2..da896caf1 100644
--- a/youtube_dl/extractor/letv.py
+++ b/youtube_dl/extractor/letv.py
@@ -50,9 +50,7 @@ class LetvIE(InfoExtractor):
             'title': '与龙共舞 完整版',
             'description': 'md5:7506a5eeb1722bb9d4068f85024e3986',
         },
-        'params': {
-            'cn_verification_proxy': 'http://proxy.uku.im:8888'
-        },
+        'skip': 'Only available in China',
     }]
 
     @staticmethod
diff --git a/youtube_dl/extractor/sohu.py b/youtube_dl/extractor/sohu.py
index eab4adfca..29bd9ce6f 100644
--- a/youtube_dl/extractor/sohu.py
+++ b/youtube_dl/extractor/sohu.py
@@ -23,9 +23,7 @@ class SohuIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'MV：Far East Movement《The Illest》',
         },
-        'params': {
-            'cn_verification_proxy': 'proxy.uku.im:8888'
-        }
+        'skip': 'On available in China',
     }, {
         'url': 'http://tv.sohu.com/20150305/n409385080.shtml',
         'md5': 'ac9a5d322b4bf9ae184d53e4711e4f1a',

From 137597b0ea88a92d174341b44b8f395b8897a2bf Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Wed, 20 May 2015 15:15:28 +0800
Subject: [PATCH 1347/6219] [dramafever] Streamline code

---
 youtube_dl/extractor/dramafever.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/dramafever.py b/youtube_dl/extractor/dramafever.py
index 8fac99cc5..40787ffcd 100644
--- a/youtube_dl/extractor/dramafever.py
+++ b/youtube_dl/extractor/dramafever.py
@@ -80,12 +80,11 @@ class DramaFeverIE(InfoExtractor):
 
     def _get_subtitles(self, video_id, consumer_secret):
 
-        def match_episode(ep):
-            return ep['guid'] == video_id
-
         res = None
         info = self._get_episodes(
-            video_id.split(".")[0], consumer_secret, episode_filter=match_episode)
+            video_id.split(".")[0], consumer_secret,
+            episode_filter=lambda x: x['guid'] == video_id)
+
         if len(info) == 1 and info[0]['subfile'] != '':
             res = {'en': [{'url': info[0]['subfile'], 'ext': 'srt'}]}
         return res

From 2632941f327c8b013e5fbc736317fc897876ab73 Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Wed, 20 May 2015 15:53:45 +0800
Subject: [PATCH 1348/6219] [soompi] Add new extractor for tv.soompi.com

---
 youtube_dl/extractor/__init__.py |   4 +
 youtube_dl/extractor/soompi.py   | 130 +++++++++++++++++++++++++++++++
 2 files changed, 134 insertions(+)
 create mode 100644 youtube_dl/extractor/soompi.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 6f8c261d5..2a5cf9547 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -482,6 +482,10 @@ from .smotri import (
 from .snotr import SnotrIE
 from .sockshare import SockshareIE
 from .sohu import SohuIE
+from .soompi import (
+    SoompiIE,
+    SoompiShowIE,
+)
 from .soundcloud import (
     SoundcloudIE,
     SoundcloudSetIE,
diff --git a/youtube_dl/extractor/soompi.py b/youtube_dl/extractor/soompi.py
new file mode 100644
index 000000000..5ecf40b7f
--- /dev/null
+++ b/youtube_dl/extractor/soompi.py
@@ -0,0 +1,130 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+import re
+import json
+import base64
+import xml.etree.ElementTree
+
+# Soompi uses the same subtitle encryption as crunchyroll
+from .crunchyroll import CrunchyrollIE
+
+
+class SoompiIE(CrunchyrollIE):
+    IE_NAME = 'soompi'
+    _VALID_URL = r'^https?://tv\.soompi\.com/en/watch/(?P<id>[0-9]+)'
+    _TESTS = [{
+        'url': 'http://tv.soompi.com/en/watch/23363',
+        'info_dict': {
+            'id': '23363',
+            'ext': 'mp4',
+            'title': 'Liar Game CM1',
+            'description': '15sec'
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }]
+
+    def _get_episodes(self, webpage, episode_filter=None):
+        episodes = json.loads(
+            self._search_regex(r'\s+VIDEOS\s+= (\[.+?\]);', webpage, "episodes meta"))
+        return [ep for ep in episodes if episode_filter is None or episode_filter(ep)]
+
+    def _get_subtitles(self, video_id, show_format_xml):
+        subtitles = {}
+        subtitle_info_nodes = show_format_xml.findall('./{default}preload/subtitles/subtitle')
+        subtitle_nodes = show_format_xml.findall('./{default}preload/subtitle')
+
+        sub_langs = {}
+        for i in subtitle_info_nodes:
+            sub_langs[i.attrib["id"]] = i.attrib["title"]
+
+        for s in subtitle_nodes:
+            lang_code = sub_langs.get(s.attrib["id"], None)
+            if lang_code is None:
+                continue
+
+            sub_id = int(s.attrib["id"])
+            iv = base64.b64decode(s.find("iv").text)
+            data = base64.b64decode(s.find("data").text)
+            subtitle = self._decrypt_subtitles(data, iv, sub_id).decode('utf-8')
+            sub_root = xml.etree.ElementTree.fromstring(subtitle)
+
+            subtitles[lang_code] = [{
+                'ext': 'srt', 'data': self._convert_subtitles_to_srt(sub_root)
+            }, {
+                'ext': 'ass', 'data': self._convert_subtitles_to_ass(sub_root)
+            }]
+        return subtitles
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(
+            url, video_id, note="Downloading episode page",
+            errnote="Video may not be available for your location")
+        vid_formats = re.findall(r"\?quality=q([0-9]+)", webpage)
+
+        show_meta = json.loads(
+            self._search_regex(r'\s+var show = (\{.+?\});', webpage, "show meta"))
+        episodes = self._get_episodes(
+            webpage, episode_filter=lambda x: x['id'] == video_id)
+
+        title = episodes[0]["name"]
+        description = episodes[0]["description"]
+        duration = int(episodes[0]["duration"])
+        slug = show_meta["slug"]
+
+        formats = []
+        show_format_xml = None
+        for vf in vid_formats:
+            show_format_url = "http://tv.soompi.com/en/show/%s/%s-config.xml?mode=hls&quality=q%s" \
+                              % (slug, video_id, vf)
+            show_format_xml = self._download_xml(
+                show_format_url, video_id, note="Downloading q%s show xml" % vf)
+            avail_formats = self._extract_m3u8_formats(
+                show_format_xml.find('./{default}preload/stream_info/file').text,
+                video_id, ext="mp4", m3u8_id=vf, preference=int(vf))
+            formats.extend(avail_formats)
+        self._sort_formats(formats)
+
+        subtitles = self.extract_subtitles(video_id, show_format_xml)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'duration': duration,
+            'formats': formats,
+            'subtitles': subtitles
+        }
+
+
+class SoompiShowIE(SoompiIE):
+    IE_NAME = 'soompi:show'
+    _VALID_URL = r'^https?://tv\.soompi\.com/en/shows/(?P<id>[0-9a-zA-Z\-_]+)'
+    _TESTS = [{
+        'url': 'http://tv.soompi.com/en/shows/liar-game',
+        'info_dict': {
+            'id': 'liar-game',
+            'title': 'Liar Game',
+            'description': 'md5:52c02bce0c1a622a95823591d0589b66',
+        },
+        'playlist_count': 14,
+    }]
+
+    def _real_extract(self, url):
+        show_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, show_id, note="Downloading show page")
+        title = self._og_search_title(webpage).replace("SoompiTV | ", "")
+        description = self._og_search_description(webpage)
+
+        episodes = self._get_episodes(webpage)
+        entries = []
+        for ep in episodes:
+            entries.append(self.url_result(
+                'http://tv.soompi.com/en/watch/%s' % ep['id'], 'Soompi', ep['id']))
+
+        return self.playlist_result(entries, show_id, title, description)

From 0b9f7cd074786abafcd35b26db4ecb4d92814393 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 20 May 2015 10:01:48 +0200
Subject: [PATCH 1349/6219] release 2015.05.20

---
 docs/supportedsites.md | 5 +++++
 youtube_dl/version.py  | 2 +-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 43fbe8b1d..a4879bd9a 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -142,6 +142,7 @@
  - **Eporner**
  - **EroProfile**
  - **Escapist**
+ - **ESPN** (Currently broken)
  - **EveryonesMixtape**
  - **exfm**: ex.fm
  - **ExpoTV**
@@ -338,6 +339,7 @@
  - **OktoberfestTV**
  - **on.aol.com**
  - **Ooyala**
+ - **OoyalaExternal**
  - **OpenFilm**
  - **orf:fm4**: radio FM4
  - **orf:iptv**: iptv.ORF.at
@@ -451,6 +453,7 @@
  - **Spike**
  - **Sport5**
  - **SportBox**
+ - **SportBoxEmbed**
  - **SportDeutschland**
  - **Srf**
  - **SRMediathek**: Saarländischer Rundfunk
@@ -510,6 +513,8 @@
  - **Turbo**
  - **Tutv**
  - **tv.dfb.de**
+ - **TV2**
+ - **TV2Article**
  - **TV4**: tv4.se and tv4play.se
  - **tvigle**: Интернет-телевидение Tvigle.ru
  - **tvp.pl**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 38f00bc9b..b33385153 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.05.15'
+__version__ = '2015.05.20'

From 5137adb94dcce98a3c14fb3892c5c72f70ff34ea Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Wed, 20 May 2015 16:16:10 +0800
Subject: [PATCH 1350/6219] [soompi] Switch to non-geoblocked test video

---
 youtube_dl/extractor/soompi.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/soompi.py b/youtube_dl/extractor/soompi.py
index 5ecf40b7f..4726872dc 100644
--- a/youtube_dl/extractor/soompi.py
+++ b/youtube_dl/extractor/soompi.py
@@ -14,12 +14,12 @@ class SoompiIE(CrunchyrollIE):
     IE_NAME = 'soompi'
     _VALID_URL = r'^https?://tv\.soompi\.com/en/watch/(?P<id>[0-9]+)'
     _TESTS = [{
-        'url': 'http://tv.soompi.com/en/watch/23363',
+        'url': 'http://tv.soompi.com/en/watch/29235',
         'info_dict': {
-            'id': '23363',
+            'id': '29235',
             'ext': 'mp4',
-            'title': 'Liar Game CM1',
-            'description': '15sec'
+            'title': 'Episode 1096',
+            'description': '2015-05-20'
         },
         'params': {
             'skip_download': True,

From b0d619fde2b187f2b36b077a1eb11d766429f88c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 20 May 2015 21:28:04 +0600
Subject: [PATCH 1351/6219] [viki:channel] Extract title from JSON

---
 youtube_dl/extractor/viki.py | 40 +++++++++++++++++++++---------------
 1 file changed, 24 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index 9bdbdc3e4..fc585c299 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -23,7 +23,7 @@ class VikiIE(InfoExtractor):
     # iPad2
     _USER_AGENT = 'Mozilla/5.0(iPad; U; CPU OS 4_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8F191 Safari/6533.18.5'
 
-    _VALID_URL = r'^https?://(?:www\.)?viki\.com/videos/(?P<id>[0-9]+v)'
+    _VALID_URL = r'https?://(?:www\.)?viki\.com/videos/(?P<id>[0-9]+v)'
     _TESTS = [{
         'url': 'http://www.viki.com/videos/1023585v-heirs-episode-14',
         'info_dict': {
@@ -149,7 +149,7 @@ class VikiIE(InfoExtractor):
 
 class VikiChannelIE(InfoExtractor):
     IE_NAME = 'viki:channel'
-    _VALID_URL = r'^https?://(?:www\.)?viki\.com/tv/(?P<id>[0-9]+c)'
+    _VALID_URL = r'https?://(?:www\.)?viki\.com/tv/(?P<id>[0-9]+c)'
     _TESTS = [{
         'url': 'http://www.viki.com/tv/50c-boys-over-flowers',
         'info_dict': {
@@ -167,27 +167,35 @@ class VikiChannelIE(InfoExtractor):
         },
         'playlist_count': 127,
     }]
+    _API_BASE = 'http://api.viki.io/v4/containers'
+    _APP = '100000a'
     _PER_PAGE = 25
 
     def _real_extract(self, url):
-        show_id = self._match_id(url)
-        show_page = self._download_webpage(url, show_id, 'Download show page')
+        channel_id = self._match_id(url)
 
-        title = self._og_search_title(show_page)
-        description = self._og_search_description(show_page)
+        channel = self._download_json(
+            '%s/%s.json?app=%s' % (self._API_BASE, channel_id, self._APP),
+            channel_id, 'Downloading channel JSON')
+
+        titles = channel['titles']
+        title = titles.get('en') or titles[titles.keys()[0]]
+
+        descriptions = channel['descriptions']
+        description = descriptions.get('en') or descriptions[descriptions.keys()[0]]
 
         entries = []
-        for video_type in ['episodes', 'clips']:
-            json_url = 'http://api.viki.io/v4/containers/%s/%s.json?app=100000a&per_page=%d&sort=number&direction=asc&with_paging=true&page=1' % (show_id, video_type, self._PER_PAGE)
-            while json_url is not None:
-                show_json = self._download_json(
-                    json_url, show_id,
-                    note='Downloading %s json page #%s' %
-                         (video_type, re.search(r'[?&]page=([0-9]+)', json_url).group(1)))
-                for video in show_json['response']:
+        for video_type in ('episodes', 'clips'):
+            page_url = '%s/%s/%s.json?app=%s&per_page=%d&sort=number&direction=asc&with_paging=true&page=1' % (self._API_BASE, channel_id, video_type, self._APP, self._PER_PAGE)
+            while page_url:
+                page = self._download_json(
+                    page_url, channel_id,
+                    'Downloading %s JSON page #%s'
+                    % (video_type, re.search(r'[?&]page=([0-9]+)', page_url).group(1)))
+                for video in page['response']:
                     video_id = video['id']
                     entries.append(self.url_result(
                         'http://www.viki.com/videos/%s' % video_id, 'Viki', video_id))
-                json_url = show_json['pagination']['next']
+                page_url = page['pagination']['next']
 
-        return self.playlist_result(entries, show_id, title, description)
+        return self.playlist_result(entries, channel_id, title, description)

From 1a83c731bd58ed85f6f7695cee9c88d09a224bc2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 21 May 2015 01:44:05 +0600
Subject: [PATCH 1352/6219] [viki] Switch extraction to API

---
 youtube_dl/extractor/viki.py | 230 +++++++++++++++++++++--------------
 1 file changed, 142 insertions(+), 88 deletions(-)

diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index fc585c299..234649ca8 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -1,29 +1,64 @@
 from __future__ import unicode_literals
 
 import re
+import time
+import hmac
+import hashlib
 
-from ..compat import (
-    compat_urlparse,
-    compat_urllib_request,
-)
 from ..utils import (
     ExtractorError,
-    unescapeHTML,
-    unified_strdate,
-    US_RATINGS,
-    determine_ext,
-    mimetype2ext,
+    int_or_none,
+    parse_age_limit,
+    parse_iso8601,
 )
 from .common import InfoExtractor
 
 
-class VikiIE(InfoExtractor):
+class VikiBaseIE(InfoExtractor):
+    _API_QUERY_TEMPLATE = '/v4/%sapp=%s&t=%s&site=www.viki.com'
+    _API_URL_TEMPLATE = 'http://api.viki.io%s&sig=%s'
+
+    _APP = '65535a'
+    _APP_VERSION = '2.2.5.1428709186'
+    _APP_SECRET = '-$iJ}@p7!G@SyU/je1bEyWg}upLu-6V6-Lg9VD(]siH,r.,m-r|ulZ,U4LC/SeR)'
+
+    def _prepare_call(self, path, timestamp=None):
+        path += '?' if '?' not in path else '&'
+        if not timestamp:
+            timestamp = int(time.time())
+        query = self._API_QUERY_TEMPLATE % (path, self._APP, timestamp)
+        sig = hmac.new(
+            self._APP_SECRET.encode('ascii'),
+            query.encode('ascii'),
+            hashlib.sha1
+        ).hexdigest()
+        return self._API_URL_TEMPLATE % (query, sig)
+
+    def _call_api(self, path, video_id, note, timestamp=None):
+        resp = self._download_json(
+            self._prepare_call(path, timestamp), video_id, note)
+
+        error = resp.get('error')
+        if error:
+            if error == 'invalid timestamp':
+                resp = self._download_json(
+                    self._prepare_call(path, int(resp['current_timestamp'])),
+                    video_id, '%s (retry)' % note)
+                error = resp.get('error')
+            if error:
+                self._raise_error(resp['error'])
+
+        return resp
+
+    def _raise_error(self, error):
+        raise ExtractorError(
+            '%s returned error: %s' % (self.IE_NAME, error),
+            expected=True)
+
+
+class VikiIE(VikiBaseIE):
     IE_NAME = 'viki'
-
-    # iPad2
-    _USER_AGENT = 'Mozilla/5.0(iPad; U; CPU OS 4_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8F191 Safari/6533.18.5'
-
-    _VALID_URL = r'https?://(?:www\.)?viki\.com/videos/(?P<id>[0-9]+v)'
+    _VALID_URL = r'https?://(?:www\.)?viki\.com/(?:videos|player)/(?P<id>[0-9]+v)'
     _TESTS = [{
         'url': 'http://www.viki.com/videos/1023585v-heirs-episode-14',
         'info_dict': {
@@ -37,115 +72,134 @@ class VikiIE(InfoExtractor):
         },
         'skip': 'Blocked in the US',
     }, {
+        # clip
         'url': 'http://www.viki.com/videos/1067139v-the-avengers-age-of-ultron-press-conference',
-        'md5': 'ca6493e6f0a6ec07da9aa8d6304b4b2c',
+        'md5': '86c0b5dbd4d83a6611a79987cc7a1989',
         'info_dict': {
             'id': '1067139v',
             'ext': 'mp4',
+            'title': "'The Avengers: Age of Ultron' Press Conference",
             'description': 'md5:d70b2f9428f5488321bfe1db10d612ea',
+            'duration': 352,
+            'timestamp': 1430380829,
             'upload_date': '20150430',
-            'title': '\'The Avengers: Age of Ultron\' Press Conference',
+            'uploader': 'Arirang TV',
+            'like_count': int,
+            'age_limit': 0,
         }
     }, {
         'url': 'http://www.viki.com/videos/1048879v-ankhon-dekhi',
         'info_dict': {
             'id': '1048879v',
             'ext': 'mp4',
-            'upload_date': '20140820',
-            'description': 'md5:54ff56d51bdfc7a30441ec967394e91c',
             'title': 'Ankhon Dekhi',
+            'duration': 6512,
+            'timestamp': 1408532356,
+            'upload_date': '20140820',
+            'uploader': 'Spuul',
+            'like_count': int,
+            'age_limit': 13,
         },
         'params': {
-            # requires ffmpeg
+            # m3u8 download
             'skip_download': True,
         }
+    }, {
+        # episode
+        'url': 'http://www.viki.com/videos/44699v-boys-over-flowers-episode-1',
+        'md5': '190f3ef426005ba3a080a63325955bc3',
+        'info_dict': {
+            'id': '44699v',
+            'ext': 'mp4',
+            'title': 'Boys Over Flowers - Episode 1',
+            'description': 'md5:52617e4f729c7d03bfd4bcbbb6e946f2',
+            'duration': 4155,
+            'timestamp': 1270496524,
+            'upload_date': '20100405',
+            'uploader': 'group8',
+            'like_count': int,
+            'age_limit': 13,
+        }
+    }, {
+        'url': 'http://www.viki.com/player/44699v',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        webpage = self._download_webpage(url, video_id)
-        title = self._og_search_title(webpage)
-        description = self._og_search_description(webpage)
-        thumbnail = self._og_search_thumbnail(webpage)
+        streams = self._call_api(
+            'videos/%s/streams.json' % video_id, video_id,
+            'Downloading video streams JSON')
 
-        uploader_m = re.search(
-            r'<strong>Broadcast Network: </strong>\s*([^<]*)<', webpage)
-        if uploader_m is None:
-            uploader = None
-        else:
-            uploader = uploader_m.group(1).strip()
+        formats = []
+        for format_id, stream_dict in streams.items():
+            height = self._search_regex(
+                r'^(\d+)[pP]$', format_id, 'height', default=None)
+            for protocol, format_dict in stream_dict.items():
+                if format_id == 'm3u8':
+                    formats = self._extract_m3u8_formats(
+                        format_dict['url'], video_id, 'mp4', m3u8_id='m3u8-%s' % protocol)
+                else:
+                    formats.append({
+                        'url': format_dict['url'],
+                        'format_id': '%s-%s' % (format_id, protocol),
+                        'height': height,
+                    })
+        self._sort_formats(formats)
 
-        rating_str = self._html_search_regex(
-            r'<strong>Rating: </strong>\s*([^<]*)<', webpage,
-            'rating information', default='').strip()
-        age_limit = US_RATINGS.get(rating_str)
+        video = self._call_api(
+            'videos/%s.json' % video_id, video_id, 'Downloading video JSON')
 
-        req = compat_urllib_request.Request(
-            'http://www.viki.com/player5_fragment/%s?action=show&controller=videos' % video_id)
-        req.add_header('User-Agent', self._USER_AGENT)
-        info_webpage = self._download_webpage(
-            req, video_id, note='Downloading info page')
-        err_msg = self._html_search_regex(r'<div[^>]+class="video-error[^>]+>(.+)</div>', info_webpage, 'error message', default=None)
-        if err_msg:
-            if 'not available in your region' in err_msg:
-                raise ExtractorError(
-                    'Video %s is blocked from your location.' % video_id,
-                    expected=True)
-            else:
-                raise ExtractorError('Viki said: %s %s' % (err_msg, url))
-        mobj = re.search(
-            r'<source[^>]+type="(?P<mime_type>[^"]+)"[^>]+src="(?P<url>[^"]+)"', info_webpage)
-        if not mobj:
-            raise ExtractorError('Unable to find video URL')
-        video_url = unescapeHTML(mobj.group('url'))
-        video_ext = mimetype2ext(mobj.group('mime_type'))
+        title = None
+        titles = video.get('titles')
+        if titles:
+            title = titles.get('en') or titles[titles.keys()[0]]
+        if not title:
+            title = 'Episode %d' % video.get('number') if video.get('type') == 'episode' else video.get('id') or video_id
+            container_titles = video.get('container', {}).get('titles')
+            if container_titles:
+                container_title = container_titles.get('en') or container_titles[titles.keys()[0]]
+                title = '%s - %s' % (container_title, title)
 
-        if determine_ext(video_url) == 'm3u8':
-            formats = self._extract_m3u8_formats(
-                video_url, video_id, ext=video_ext)
-        else:
-            formats = [{
-                'url': video_url,
-                'ext': video_ext,
-            }]
+        descriptions = video.get('descriptions')
+        description = descriptions.get('en') or descriptions[titles.keys()[0]] if descriptions else None
 
-        upload_date_str = self._html_search_regex(
-            r'"created_at":"([^"]+)"', info_webpage, 'upload date')
-        upload_date = (
-            unified_strdate(upload_date_str)
-            if upload_date_str is not None
-            else None
-        )
+        duration = int_or_none(video.get('duration'))
+        timestamp = parse_iso8601(video.get('created_at'))
+        uploader = video.get('author')
+        like_count = int_or_none(video.get('likes', {}).get('count'))
+        age_limit = parse_age_limit(video.get('rating'))
 
-        # subtitles
-        video_subtitles = self.extract_subtitles(video_id, info_webpage)
+        thumbnails = []
+        for thumbnail_id, thumbnail in video.get('images', {}).items():
+            thumbnails.append({
+                'id': thumbnail_id,
+                'url': thumbnail.get('url'),
+            })
+
+        subtitles = {}
+        for subtitle_lang, _ in video.get('subtitle_completions', {}).items():
+            subtitles[subtitle_lang] = [{
+                'ext': subtitles_format,
+                'url': self._prepare_call(
+                    'videos/%s/subtitles/%s.%s' % (video_id, subtitle_lang, subtitles_format)),
+            } for subtitles_format in ('srt', 'vtt')]
 
         return {
             'id': video_id,
             'title': title,
-            'formats': formats,
             'description': description,
-            'thumbnail': thumbnail,
-            'age_limit': age_limit,
+            'duration': duration,
+            'timestamp': timestamp,
             'uploader': uploader,
-            'subtitles': video_subtitles,
-            'upload_date': upload_date,
+            'like_count': like_count,
+            'age_limit': age_limit,
+            'thumbnails': thumbnails,
+            'formats': formats,
+            'subtitles': subtitles,
         }
 
-    def _get_subtitles(self, video_id, info_webpage):
-        res = {}
-        for sturl_html in re.findall(r'<track src="([^"]+)"', info_webpage):
-            sturl = unescapeHTML(sturl_html)
-            m = re.search(r'/(?P<lang>[a-z]+)\.vtt', sturl)
-            if not m:
-                continue
-            res[m.group('lang')] = [{
-                'url': compat_urlparse.urljoin('http://www.viki.com', sturl),
-                'ext': 'vtt',
-            }]
-        return res
-
 
 class VikiChannelIE(InfoExtractor):
     IE_NAME = 'viki:channel'

From ac20d95f9766aa130748aac07fa90ee5dfa566d4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 21 May 2015 01:56:02 +0600
Subject: [PATCH 1353/6219] [viki] Add support for youtube externals

---
 youtube_dl/extractor/viki.py | 70 ++++++++++++++++++++++++------------
 1 file changed, 48 insertions(+), 22 deletions(-)

diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index 234649ca8..68d5cac6e 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -120,6 +120,23 @@ class VikiIE(VikiBaseIE):
             'like_count': int,
             'age_limit': 13,
         }
+    }, {
+        # youtube external
+        'url': 'http://www.viki.com/videos/50562v-poor-nastya-complete-episode-1',
+        'md5': '216d1afdc0c64d1febc1e9f2bd4b864b',
+        'info_dict': {
+            'id': '50562v',
+            'ext': 'mp4',
+            'title': 'Poor Nastya [COMPLETE] - Episode 1',
+            'description': '',
+            'duration': 607,
+            'timestamp': 1274949505,
+            'upload_date': '20101213',
+            'uploader': 'ad14065n',
+            'uploader_id': 'ad14065n',
+            'like_count': int,
+            'age_limit': 13,
+        }
     }, {
         'url': 'http://www.viki.com/player/44699v',
         'only_matching': True,
@@ -128,26 +145,6 @@ class VikiIE(VikiBaseIE):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        streams = self._call_api(
-            'videos/%s/streams.json' % video_id, video_id,
-            'Downloading video streams JSON')
-
-        formats = []
-        for format_id, stream_dict in streams.items():
-            height = self._search_regex(
-                r'^(\d+)[pP]$', format_id, 'height', default=None)
-            for protocol, format_dict in stream_dict.items():
-                if format_id == 'm3u8':
-                    formats = self._extract_m3u8_formats(
-                        format_dict['url'], video_id, 'mp4', m3u8_id='m3u8-%s' % protocol)
-                else:
-                    formats.append({
-                        'url': format_dict['url'],
-                        'format_id': '%s-%s' % (format_id, protocol),
-                        'height': height,
-                    })
-        self._sort_formats(formats)
-
         video = self._call_api(
             'videos/%s.json' % video_id, video_id, 'Downloading video JSON')
 
@@ -186,7 +183,7 @@ class VikiIE(VikiBaseIE):
                     'videos/%s/subtitles/%s.%s' % (video_id, subtitle_lang, subtitles_format)),
             } for subtitles_format in ('srt', 'vtt')]
 
-        return {
+        result = {
             'id': video_id,
             'title': title,
             'description': description,
@@ -196,10 +193,39 @@ class VikiIE(VikiBaseIE):
             'like_count': like_count,
             'age_limit': age_limit,
             'thumbnails': thumbnails,
-            'formats': formats,
             'subtitles': subtitles,
         }
 
+        streams = self._call_api(
+            'videos/%s/streams.json' % video_id, video_id,
+            'Downloading video streams JSON')
+
+        if 'external' in streams:
+            result.update({
+                '_type': 'url_transparent',
+                'url': streams['external']['url'],
+            })
+            return result
+
+        formats = []
+        for format_id, stream_dict in streams.items():
+            height = self._search_regex(
+                r'^(\d+)[pP]$', format_id, 'height', default=None)
+            for protocol, format_dict in stream_dict.items():
+                if format_id == 'm3u8':
+                    formats = self._extract_m3u8_formats(
+                        format_dict['url'], video_id, 'mp4', m3u8_id='m3u8-%s' % protocol)
+                else:
+                    formats.append({
+                        'url': format_dict['url'],
+                        'format_id': '%s-%s' % (format_id, protocol),
+                        'height': height,
+                    })
+        self._sort_formats(formats)
+
+        result['formats'] = formats
+        return result
+
 
 class VikiChannelIE(InfoExtractor):
     IE_NAME = 'viki:channel'

From bc56355ec6bc823fe96e31688cd3123dc18ae627 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 21 May 2015 02:08:13 +0600
Subject: [PATCH 1354/6219] [viki:channel] Switch to API

---
 youtube_dl/extractor/viki.py | 28 ++++++++++++++--------------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index 68d5cac6e..071e280fb 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -4,6 +4,7 @@ import re
 import time
 import hmac
 import hashlib
+import itertools
 
 from ..utils import (
     ExtractorError,
@@ -227,7 +228,7 @@ class VikiIE(VikiBaseIE):
         return result
 
 
-class VikiChannelIE(InfoExtractor):
+class VikiChannelIE(VikiBaseIE):
     IE_NAME = 'viki:channel'
     _VALID_URL = r'https?://(?:www\.)?viki\.com/tv/(?P<id>[0-9]+c)'
     _TESTS = [{
@@ -247,16 +248,15 @@ class VikiChannelIE(InfoExtractor):
         },
         'playlist_count': 127,
     }]
-    _API_BASE = 'http://api.viki.io/v4/containers'
-    _APP = '100000a'
+
     _PER_PAGE = 25
 
     def _real_extract(self, url):
         channel_id = self._match_id(url)
 
-        channel = self._download_json(
-            '%s/%s.json?app=%s' % (self._API_BASE, channel_id, self._APP),
-            channel_id, 'Downloading channel JSON')
+        channel = self._call_api(
+            'containers/%s.json' % channel_id, channel_id,
+            'Downloading channel JSON')
 
         titles = channel['titles']
         title = titles.get('en') or titles[titles.keys()[0]]
@@ -266,16 +266,16 @@ class VikiChannelIE(InfoExtractor):
 
         entries = []
         for video_type in ('episodes', 'clips'):
-            page_url = '%s/%s/%s.json?app=%s&per_page=%d&sort=number&direction=asc&with_paging=true&page=1' % (self._API_BASE, channel_id, video_type, self._APP, self._PER_PAGE)
-            while page_url:
-                page = self._download_json(
-                    page_url, channel_id,
-                    'Downloading %s JSON page #%s'
-                    % (video_type, re.search(r'[?&]page=([0-9]+)', page_url).group(1)))
+            for page_num in itertools.count(1):
+                page = self._call_api(
+                    'containers/%s/%s.json?per_page=%d&sort=number&direction=asc&with_paging=true&page=%d'
+                    % (channel_id, video_type, self._PER_PAGE, page_num), channel_id,
+                    'Downloading %s JSON page #%d' % (video_type, page_num))
                 for video in page['response']:
                     video_id = video['id']
                     entries.append(self.url_result(
-                        'http://www.viki.com/videos/%s' % video_id, 'Viki', video_id))
-                page_url = page['pagination']['next']
+                        'http://www.viki.com/videos/%s' % video_id, 'Viki'))
+                if not page['pagination']['next']:
+                    break
 
         return self.playlist_result(entries, channel_id, title, description)

From d01924f48810db69d572bc121ab98021f04ac957 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 21 May 2015 02:30:04 +0600
Subject: [PATCH 1355/6219] [viki:channel] Extend matching URLs and extract
 movies

---
 youtube_dl/extractor/viki.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index 071e280fb..3acb481f9 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -230,7 +230,7 @@ class VikiIE(VikiBaseIE):
 
 class VikiChannelIE(VikiBaseIE):
     IE_NAME = 'viki:channel'
-    _VALID_URL = r'https?://(?:www\.)?viki\.com/tv/(?P<id>[0-9]+c)'
+    _VALID_URL = r'https?://(?:www\.)?viki\.com/(?:tv|news|movies|artists)/(?P<id>[0-9]+c)'
     _TESTS = [{
         'url': 'http://www.viki.com/tv/50c-boys-over-flowers',
         'info_dict': {
@@ -247,6 +247,15 @@ class VikiChannelIE(VikiBaseIE):
             'description': 'md5:05bf5471385aa8b21c18ad450e350525',
         },
         'playlist_count': 127,
+    }, {
+        'url': 'http://www.viki.com/news/24569c-showbiz-korea',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.viki.com/movies/22047c-pride-and-prejudice-2005',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.viki.com/artists/2141c-shinee',
+        'only_matching': True,
     }]
 
     _PER_PAGE = 25
@@ -265,7 +274,7 @@ class VikiChannelIE(VikiBaseIE):
         description = descriptions.get('en') or descriptions[descriptions.keys()[0]]
 
         entries = []
-        for video_type in ('episodes', 'clips'):
+        for video_type in ('episodes', 'clips', 'movies'):
             for page_num in itertools.count(1):
                 page = self._call_api(
                     'containers/%s/%s.json?per_page=%d&sort=number&direction=asc&with_paging=true&page=%d'

From 4d8ee01389c4229f14fad45f0aa7b033a2509aef Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 21 May 2015 02:38:43 +0600
Subject: [PATCH 1356/6219] [viki] Fix typo

---
 youtube_dl/extractor/viki.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index 3acb481f9..0ec8ef0ef 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -157,7 +157,7 @@ class VikiIE(VikiBaseIE):
             title = 'Episode %d' % video.get('number') if video.get('type') == 'episode' else video.get('id') or video_id
             container_titles = video.get('container', {}).get('titles')
             if container_titles:
-                container_title = container_titles.get('en') or container_titles[titles.keys()[0]]
+                container_title = container_titles.get('en') or container_titles[container_titles.keys()[0]]
                 title = '%s - %s' % (container_title, title)
 
         descriptions = video.get('descriptions')

From 4d2f42361e02bb67de7c2017c6817b46ff3b2bd2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 21 May 2015 11:42:20 +0200
Subject: [PATCH 1357/6219] [viki] remove unused import

---
 youtube_dl/extractor/viki.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index 0ec8ef0ef..fe7229952 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -1,6 +1,5 @@
 from __future__ import unicode_literals
 
-import re
 import time
 import hmac
 import hashlib

From e7752cd57853718c6875b02517613d14c4c7221d Mon Sep 17 00:00:00 2001
From: frenchy1983 <frenchy1983@zoho.com>
Date: Thu, 21 May 2015 11:47:16 +0200
Subject: [PATCH 1358/6219] [TNAFlix] Allow dot (and more) in cat_id and
 display_id

URLs with dots were raising a "UnsupportedError: Unsupported URL" error.
---
 youtube_dl/extractor/tnaflix.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/tnaflix.py b/youtube_dl/extractor/tnaflix.py
index d48cbbf14..725edd3c7 100644
--- a/youtube_dl/extractor/tnaflix.py
+++ b/youtube_dl/extractor/tnaflix.py
@@ -10,23 +10,23 @@ from ..utils import (
 
 
 class TNAFlixIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?tnaflix\.com/(?P<cat_id>[\w-]+)/(?P<display_id>[\w-]+)/video(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?tnaflix\.com/(?P<cat_id>[^/]+)/(?P<display_id>[^/]+)/video(?P<id>\d+)'
 
     _TITLE_REGEX = r'<title>(.+?) - TNAFlix Porn Videos</title>'
     _DESCRIPTION_REGEX = r'<h3 itemprop="description">([^<]+)</h3>'
     _CONFIG_REGEX = r'flashvars\.config\s*=\s*escape\("([^"]+)"'
 
     _TEST = {
-        'url': 'http://www.tnaflix.com/porn-stars/Carmella-Decesare-striptease/video553878',
-        'md5': 'ecf3498417d09216374fc5907f9c6ec0',
+        'url': 'https://www.tnaflix.com/amateur-porn/bunzHD-Ms.Donk/video358632',
+        'md5': '6c431ea56756497e227fb3f01a687869',
         'info_dict': {
-            'id': '553878',
-            'display_id': 'Carmella-Decesare-striptease',
+            'id': '358632',
+            'display_id': 'bunzHD-Ms.Donk',
             'ext': 'mp4',
-            'title': 'Carmella Decesare - striptease',
-            'description': '',
+            'title': 'bunzHD Ms.Donk',
+            'description': 'bubble booty ebony teen goddess Ms.Donk has a firm ass and acts like she is shy but really she is a freak in the sheets watch her 20 min XX rated vid at bunzHD.com click on the catalog link',
             'thumbnail': 're:https?://.*\.jpg$',
-            'duration': 91,
+            'duration': 394,
             'age_limit': 18,
         }
     }

From 6ad9cb224a7d9156109fe0b0100d277b954063d3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 21 May 2015 12:02:53 +0200
Subject: [PATCH 1359/6219] [mitele] It now uses m3u8 (#5764)

It should also be possible to use Adobe HDS, but it would require more work.
---
 youtube_dl/extractor/mitele.py    | 9 +++++++--
 youtube_dl/extractor/telecinco.py | 4 ++++
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/mitele.py b/youtube_dl/extractor/mitele.py
index d8897eb90..7091f3335 100644
--- a/youtube_dl/extractor/mitele.py
+++ b/youtube_dl/extractor/mitele.py
@@ -20,7 +20,6 @@ class MiTeleIE(InfoExtractor):
 
     _TESTS = [{
         'url': 'http://www.mitele.es/programas-tv/diario-de/la-redaccion/programa-144/',
-        'md5': '6a75fe9d0d3275bead0cb683c616fddb',
         'info_dict': {
             'id': '0fce117d',
             'ext': 'mp4',
@@ -29,6 +28,10 @@ class MiTeleIE(InfoExtractor):
             'display_id': 'programa-144',
             'duration': 2913,
         },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
     }]
 
     def _real_extract(self, url):
@@ -56,12 +59,14 @@ class MiTeleIE(InfoExtractor):
             episode,
             transform_source=strip_jsonp
         )
+        formats = self._extract_m3u8_formats(
+            token_info['tokenizedUrl'], episode, ext='mp4')
 
         return {
             'id': embed_data['videoId'],
             'display_id': episode,
             'title': info_el.find('title').text,
-            'url': token_info['tokenizedUrl'],
+            'formats': formats,
             'description': get_element_by_attribute('class', 'text', webpage),
             'thumbnail': info_el.find('thumb').text,
             'duration': parse_duration(info_el.find('duration').text),
diff --git a/youtube_dl/extractor/telecinco.py b/youtube_dl/extractor/telecinco.py
index 251a68680..a0c744fd1 100644
--- a/youtube_dl/extractor/telecinco.py
+++ b/youtube_dl/extractor/telecinco.py
@@ -16,6 +16,10 @@ class TelecincoIE(MiTeleIE):
             'title': 'Con Martín Berasategui, hacer un bacalao al ...',
             'duration': 662,
         },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
     }, {
         'url': 'http://www.telecinco.es/informativos/nacional/Pablo_Iglesias-Informativos_Telecinco-entrevista-Pedro_Piqueras_2_1945155182.html',
         'only_matching': True,

From 663004ac2b001b9be03bd951d539a62cf83c58ac Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 21 May 2015 22:06:25 +0600
Subject: [PATCH 1360/6219] [options] Clarify `--metadata-from-title`
 additional templates

---
 youtube_dl/options.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 22dbc3aec..dd07266b7 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -713,7 +713,7 @@ def parseOpts(overrideArguments=None):
         help='Parse additional metadata like song title / artist from the video title. '
              'The format syntax is the same as --output, '
              'the parsed parameters replace existing values. '
-             'Additional templates: %(album), %(artist). '
+             'Additional templates: %(album)s, %(artist)s. '
              'Example: --metadata-from-title "%(artist)s - %(title)s" matches a title like '
              '"Coldplay - Paradise"')
     postproc.add_option(

From 53de95da5e40aa1a465668977e507ccc914099f9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 21 May 2015 22:27:22 +0600
Subject: [PATCH 1361/6219] [viki] Extend _VALID_URLs

---
 youtube_dl/extractor/viki.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index fe7229952..7f2fb1ca8 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -15,6 +15,7 @@ from .common import InfoExtractor
 
 
 class VikiBaseIE(InfoExtractor):
+    _VALID_URL_BASE = r'https?://(?:www\.)?viki\.(?:com|net|mx|jp|fr)/'
     _API_QUERY_TEMPLATE = '/v4/%sapp=%s&t=%s&site=www.viki.com'
     _API_URL_TEMPLATE = 'http://api.viki.io%s&sig=%s'
 
@@ -58,7 +59,7 @@ class VikiBaseIE(InfoExtractor):
 
 class VikiIE(VikiBaseIE):
     IE_NAME = 'viki'
-    _VALID_URL = r'https?://(?:www\.)?viki\.com/(?:videos|player)/(?P<id>[0-9]+v)'
+    _VALID_URL = r'%s(?:videos|player)/(?P<id>[0-9]+v)' % VikiBaseIE._VALID_URL_BASE
     _TESTS = [{
         'url': 'http://www.viki.com/videos/1023585v-heirs-episode-14',
         'info_dict': {
@@ -229,7 +230,7 @@ class VikiIE(VikiBaseIE):
 
 class VikiChannelIE(VikiBaseIE):
     IE_NAME = 'viki:channel'
-    _VALID_URL = r'https?://(?:www\.)?viki\.com/(?:tv|news|movies|artists)/(?P<id>[0-9]+c)'
+    _VALID_URL = r'%s(?:tv|news|movies|artists)/(?P<id>[0-9]+c)' % VikiBaseIE._VALID_URL_BASE
     _TESTS = [{
         'url': 'http://www.viki.com/tv/50c-boys-over-flowers',
         'info_dict': {

From 5cd47a5e4f54033bcf6d80908e00eff4c75a51c5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 21 May 2015 23:58:46 +0600
Subject: [PATCH 1362/6219] [videott] Fix for python 3.2

---
 youtube_dl/extractor/videott.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/videott.py b/youtube_dl/extractor/videott.py
index ececc7ee0..591024ead 100644
--- a/youtube_dl/extractor/videott.py
+++ b/youtube_dl/extractor/videott.py
@@ -43,7 +43,7 @@ class VideoTtIE(InfoExtractor):
 
         formats = [
             {
-                'url': base64.b64decode(res['u']).decode('utf-8'),
+                'url': base64.b64decode(res['u'].encode('utf-8')).decode('utf-8'),
                 'ext': 'flv',
                 'format_id': res['l'],
             } for res in settings['res'] if res['u']

From 06947add03b6b619292812a771993d3365b0e7e5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 22 May 2015 00:03:47 +0600
Subject: [PATCH 1363/6219] [chilloutzone] Fix for python 3.2

---
 youtube_dl/extractor/chilloutzone.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/chilloutzone.py b/youtube_dl/extractor/chilloutzone.py
index c922f6959..0206d96db 100644
--- a/youtube_dl/extractor/chilloutzone.py
+++ b/youtube_dl/extractor/chilloutzone.py
@@ -57,7 +57,7 @@ class ChilloutzoneIE(InfoExtractor):
 
         base64_video_info = self._html_search_regex(
             r'var cozVidData = "(.+?)";', webpage, 'video data')
-        decoded_video_info = base64.b64decode(base64_video_info).decode("utf-8")
+        decoded_video_info = base64.b64decode(base64_video_info.encode('utf-8')).decode('utf-8')
         video_info_dict = json.loads(decoded_video_info)
 
         # get video information from dict

From 878563c847fa5248eedbd44187536dec04643eaf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 22 May 2015 00:06:10 +0600
Subject: [PATCH 1364/6219] [aes] Fix for python 3.2

---
 youtube_dl/aes.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/aes.py b/youtube_dl/aes.py
index 07224d508..7817adcfd 100644
--- a/youtube_dl/aes.py
+++ b/youtube_dl/aes.py
@@ -152,7 +152,7 @@ def aes_decrypt_text(data, password, key_size_bytes):
     """
     NONCE_LENGTH_BYTES = 8
 
-    data = bytes_to_intlist(base64.b64decode(data))
+    data = bytes_to_intlist(base64.b64decode(data.encode('utf-8')))
     password = bytes_to_intlist(password.encode('utf-8'))
 
     key = password[:key_size_bytes] + [0] * (key_size_bytes - len(password))

From afe8b594be53161f68189e15a65b4e9c6eba0b35 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 22 May 2015 00:09:15 +0600
Subject: [PATCH 1365/6219] [rtve.es:alacarta] Fix for python 3.2

---
 youtube_dl/extractor/rtve.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/rtve.py b/youtube_dl/extractor/rtve.py
index 849300140..82cd98ac7 100644
--- a/youtube_dl/extractor/rtve.py
+++ b/youtube_dl/extractor/rtve.py
@@ -17,7 +17,7 @@ from ..utils import (
 
 
 def _decrypt_url(png):
-    encrypted_data = base64.b64decode(png)
+    encrypted_data = base64.b64decode(png.encode('utf-8'))
     text_index = encrypted_data.find(b'tEXt')
     text_chunk = encrypted_data[text_index - 4:]
     length = struct_unpack('!I', text_chunk[:4])[0]

From 43150d7ac36efda7bc60c694b8a18e1f720da04b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 22 May 2015 00:10:05 +0600
Subject: [PATCH 1366/6219] [shared] Fix for python 3.2

---
 youtube_dl/extractor/shared.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/shared.py b/youtube_dl/extractor/shared.py
index 26ced716e..9f3e944e7 100644
--- a/youtube_dl/extractor/shared.py
+++ b/youtube_dl/extractor/shared.py
@@ -47,7 +47,7 @@ class SharedIE(InfoExtractor):
         video_url = self._html_search_regex(
             r'data-url="([^"]+)"', video_page, 'video URL')
         title = base64.b64decode(self._html_search_meta(
-            'full:title', webpage, 'title')).decode('utf-8')
+            'full:title', webpage, 'title').encode('utf-8')).decode('utf-8')
         filesize = int_or_none(self._html_search_meta(
             'full:size', webpage, 'file size', fatal=False))
         thumbnail = self._html_search_regex(

From 0459432d962bf358566340eed00f6c1c56b7b732 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 22 May 2015 00:10:53 +0600
Subject: [PATCH 1367/6219] [shared] Fix for python 3.2

---
 youtube_dl/extractor/tutv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tutv.py b/youtube_dl/extractor/tutv.py
index 4de0aac52..fad720b68 100644
--- a/youtube_dl/extractor/tutv.py
+++ b/youtube_dl/extractor/tutv.py
@@ -26,7 +26,7 @@ class TutvIE(InfoExtractor):
 
         data_content = self._download_webpage(
             'http://tu.tv/flvurl.php?codVideo=%s' % internal_id, video_id, 'Downloading video info')
-        video_url = base64.b64decode(compat_parse_qs(data_content)['kpt'][0]).decode('utf-8')
+        video_url = base64.b64decode(compat_parse_qs(data_content)['kpt'][0].encode('utf-8')).decode('utf-8')
 
         return {
             'id': internal_id,

From 77d9cb2f04462677f2a36f487c20e7a7992a0a32 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 22 May 2015 00:45:33 +0600
Subject: [PATCH 1368/6219] [sportbox] Fix extraction

---
 youtube_dl/extractor/sportbox.py | 25 +++++++++----------------
 1 file changed, 9 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/extractor/sportbox.py b/youtube_dl/extractor/sportbox.py
index 8686f9d11..2ab3489e4 100644
--- a/youtube_dl/extractor/sportbox.py
+++ b/youtube_dl/extractor/sportbox.py
@@ -7,7 +7,7 @@ from .common import InfoExtractor
 from ..compat import compat_urlparse
 from ..utils import (
     parse_duration,
-    parse_iso8601,
+    unified_strdate,
 )
 
 
@@ -20,11 +20,9 @@ class SportBoxIE(InfoExtractor):
             'id': '80822',
             'ext': 'mp4',
             'title': 'Гонка 2  заезд ««Объединенный 2000»: классы Туринг и Супер-продакшн',
-            'description': 'md5:81715fa9c4ea3d9e7915dc8180c778ed',
+            'description': 'md5:3d72dc4a006ab6805d82f037fdc637ad',
             'thumbnail': 're:^https?://.*\.jpg$',
-            'timestamp': 1411896237,
             'upload_date': '20140928',
-            'duration': 4846,
         },
         'params': {
             # m3u8 download
@@ -48,17 +46,13 @@ class SportBoxIE(InfoExtractor):
             r'src="/?(vdl/player/[^"]+)"', webpage, 'player')
 
         title = self._html_search_regex(
-            r'<h1 itemprop="name">([^<]+)</h1>', webpage, 'title')
-        description = self._html_search_regex(
-            r'(?s)<div itemprop="description">(.+?)</div>',
-            webpage, 'description', fatal=False)
+            [r'"nodetitle"\s*:\s*"([^"]+)"', r'class="node-header_{1,2}title">([^<]+)'],
+            webpage, 'title')
+        description = self._og_search_description(webpage) or self._html_search_meta(
+            'description', webpage, 'description')
         thumbnail = self._og_search_thumbnail(webpage)
-        timestamp = parse_iso8601(self._search_regex(
-            r'<span itemprop="uploadDate">([^<]+)</span>',
-            webpage, 'timestamp', fatal=False))
-        duration = parse_duration(self._html_search_regex(
-            r'<meta itemprop="duration" content="PT([^"]+)">',
-            webpage, 'duration', fatal=False))
+        upload_date = unified_strdate(self._html_search_meta(
+            'dateCreated', webpage, 'upload date'))
 
         return {
             '_type': 'url_transparent',
@@ -67,8 +61,7 @@ class SportBoxIE(InfoExtractor):
             'title': title,
             'description': description,
             'thumbnail': thumbnail,
-            'timestamp': timestamp,
-            'duration': duration,
+            'upload_date': upload_date,
         }
 
 

From 8a278a1d7ef6134a5ac6b7dd31e3458d05f71225 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 22 May 2015 13:26:50 +0800
Subject: [PATCH 1369/6219] [nba] Fix duration extraction (fixes #5777)

---
 youtube_dl/extractor/nba.py | 20 ++++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/nba.py b/youtube_dl/extractor/nba.py
index 862b706bf..944096e1c 100644
--- a/youtube_dl/extractor/nba.py
+++ b/youtube_dl/extractor/nba.py
@@ -22,6 +22,18 @@ class NBAIE(InfoExtractor):
     }, {
         'url': 'http://www.nba.com/video/games/hornets/2014/12/05/0021400276-nyk-cha-play5.nba/',
         'only_matching': True,
+    }, {
+        'url': 'http://watch.nba.com/nba/video/channels/playoffs/2015/05/20/0041400301-cle-atl-recap.nba',
+        'info_dict': {
+            'id': '0041400301-cle-atl-recap.nba',
+            'ext': 'mp4',
+            'title': 'NBA GAME TIME | Video: Hawks vs. Cavaliers Game 1',
+            'description': 'md5:8094c3498d35a9bd6b1a8c396a071b4d',
+            'duration': 228,
+        },
+        'params': {
+            'skip_download': True,
+        }
     }]
 
     def _real_extract(self, url):
@@ -35,8 +47,12 @@ class NBAIE(InfoExtractor):
             self._og_search_title(webpage, default=shortened_video_id), ' : NBA.com')
 
         description = self._og_search_description(webpage)
-        duration = parse_duration(
-            self._html_search_meta('duration', webpage, 'duration'))
+        duration_str = self._html_search_meta(
+            'duration', webpage, 'duration', default=None)
+        if not duration_str:
+            duration_str = self._html_search_regex(
+                r'Duration:</b>\s*(\d+:\d+)', webpage, 'duration', fatal=False)
+        duration = parse_duration(duration_str)
 
         return {
             'id': shortened_video_id,

From ed5a637d62e8ede4a8cef75df4e5f341e3c667a1 Mon Sep 17 00:00:00 2001
From: frenchy1983 <frenchy1983@zoho.com>
Date: Fri, 22 May 2015 09:29:35 +0200
Subject: [PATCH 1370/6219] [TNAFlix] Restore test

See dstftw's comment in #5772
---
 youtube_dl/extractor/empflix.py | 26 ++++++++++++++------------
 youtube_dl/extractor/tnaflix.py | 32 +++++++++++++++++++-------------
 2 files changed, 33 insertions(+), 25 deletions(-)

diff --git a/youtube_dl/extractor/empflix.py b/youtube_dl/extractor/empflix.py
index 70f8efe27..0dc947c1d 100644
--- a/youtube_dl/extractor/empflix.py
+++ b/youtube_dl/extractor/empflix.py
@@ -10,16 +10,18 @@ class EMPFlixIE(TNAFlixIE):
     _DESCRIPTION_REGEX = r'name="description" value="([^"]*)"'
     _CONFIG_REGEX = r'flashvars\.config\s*=\s*escape\("([^"]+)"'
 
-    _TEST = {
-        'url': 'http://www.empflix.com/videos/Amateur-Finger-Fuck-33051.html',
-        'md5': 'b1bc15b6412d33902d6e5952035fcabc',
-        'info_dict': {
-            'id': '33051',
-            'display_id': 'Amateur-Finger-Fuck',
-            'ext': 'mp4',
-            'title': 'Amateur Finger Fuck',
-            'description': 'Amateur solo finger fucking.',
-            'thumbnail': 're:https?://.*\.jpg$',
-            'age_limit': 18,
+    _TESTS = [
+        {
+            'url': 'http://www.empflix.com/videos/Amateur-Finger-Fuck-33051.html',
+            'md5': 'b1bc15b6412d33902d6e5952035fcabc',
+            'info_dict': {
+                'id': '33051',
+                'display_id': 'Amateur-Finger-Fuck',
+                'ext': 'mp4',
+                'title': 'Amateur Finger Fuck',
+                'description': 'Amateur solo finger fucking.',
+                'thumbnail': 're:https?://.*\.jpg$',
+                'age_limit': 18,
+            }
         }
-    }
+    ]
diff --git a/youtube_dl/extractor/tnaflix.py b/youtube_dl/extractor/tnaflix.py
index 725edd3c7..79496039d 100644
--- a/youtube_dl/extractor/tnaflix.py
+++ b/youtube_dl/extractor/tnaflix.py
@@ -16,20 +16,26 @@ class TNAFlixIE(InfoExtractor):
     _DESCRIPTION_REGEX = r'<h3 itemprop="description">([^<]+)</h3>'
     _CONFIG_REGEX = r'flashvars\.config\s*=\s*escape\("([^"]+)"'
 
-    _TEST = {
-        'url': 'https://www.tnaflix.com/amateur-porn/bunzHD-Ms.Donk/video358632',
-        'md5': '6c431ea56756497e227fb3f01a687869',
-        'info_dict': {
-            'id': '358632',
-            'display_id': 'bunzHD-Ms.Donk',
-            'ext': 'mp4',
-            'title': 'bunzHD Ms.Donk',
-            'description': 'bubble booty ebony teen goddess Ms.Donk has a firm ass and acts like she is shy but really she is a freak in the sheets watch her 20 min XX rated vid at bunzHD.com click on the catalog link',
-            'thumbnail': 're:https?://.*\.jpg$',
-            'duration': 394,
-            'age_limit': 18,
+    _TESTS = [
+        {
+            'url': 'http://www.tnaflix.com/porn-stars/Carmella-Decesare-striptease/video553878',
+            'md5': 'ecf3498417d09216374fc5907f9c6ec0',
+            'info_dict': {
+                'id': '553878',
+                'display_id': 'Carmella-Decesare-striptease',
+                'ext': 'mp4',
+                'title': 'Carmella Decesare - striptease',
+                'description': '',
+                'thumbnail': 're:https?://.*\.jpg$',
+                'duration': 91,
+                'age_limit': 18,
+            }
+        },
+        {
+            'url': 'https://www.tnaflix.com/amateur-porn/bunzHD-Ms.Donk/video358632',
+            'matching_only': True,
         }
-    }
+    ]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)

From ba6454761687e099f960b50cb50a9b87f4ec6d17 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 22 May 2015 11:35:09 +0200
Subject: [PATCH 1371/6219] [sportbox] Remove unused import

---
 youtube_dl/extractor/sportbox.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/sportbox.py b/youtube_dl/extractor/sportbox.py
index 2ab3489e4..86d509ae5 100644
--- a/youtube_dl/extractor/sportbox.py
+++ b/youtube_dl/extractor/sportbox.py
@@ -6,7 +6,6 @@ import re
 from .common import InfoExtractor
 from ..compat import compat_urlparse
 from ..utils import (
-    parse_duration,
     unified_strdate,
 )
 

From 79979c689713fd28e8fdf08bd71eecb6798f23d9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 22 May 2015 16:14:55 +0200
Subject: [PATCH 1372/6219] Clarify that --dump-pages encodes the pages using
 base64 (#5781)

---
 youtube_dl/options.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index dd07266b7..5a2315bd9 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -537,7 +537,7 @@ def parseOpts(overrideArguments=None):
     verbosity.add_option(
         '--dump-pages', '--dump-intermediate-pages',
         action='store_true', dest='dump_intermediate_pages', default=False,
-        help='Print downloaded pages to debug problems (very verbose)')
+        help='Print downloaded pages encoded using base64 to debug problems (very verbose)')
     verbosity.add_option(
         '--write-pages',
         action='store_true', dest='write_pages', default=False,

From 69e0f1b445388e4b6f45868d53780d6f8937f56e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 May 2015 00:08:10 +0600
Subject: [PATCH 1373/6219] Credit @ping for viki:channel, qqmusic:toplist

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 267b8da1e..ebed7ebb3 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -124,3 +124,4 @@ Mohammad Teimori Pabandi
 Roman Le Négrate
 Matthias Küch
 Julian Richen
+Ping O.

From 685c74d315a54154c5a1d9ecee8b212dbee94bc7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 May 2015 01:01:47 +0600
Subject: [PATCH 1374/6219] [rutv] Extend embed URL (Closes #5782)

---
 youtube_dl/extractor/rutv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/rutv.py b/youtube_dl/extractor/rutv.py
index 55604637d..d9df06861 100644
--- a/youtube_dl/extractor/rutv.py
+++ b/youtube_dl/extractor/rutv.py
@@ -104,7 +104,7 @@ class RUTVIE(InfoExtractor):
     @classmethod
     def _extract_url(cls, webpage):
         mobj = re.search(
-            r'<iframe[^>]+?src=(["\'])(?P<url>https?://player\.rutv\.ru/(?:iframe/(?:swf|video|live)/id|index/iframe/cast_id)/.+?)\1', webpage)
+            r'<iframe[^>]+?src=(["\'])(?P<url>https?://player\.(?:rutv\.ru|vgtrk\.com)/(?:iframe/(?:swf|video|live)/id|index/iframe/cast_id)/.+?)\1', webpage)
         if mobj:
             return mobj.group('url')
 

From d386878af96f368ba4c2fc8bc9b078a69b79fdf3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 May 2015 21:25:53 +0600
Subject: [PATCH 1375/6219] [prosiebensat1] Add support for .at domain names
 (Closes #5786)

---
 youtube_dl/extractor/prosiebensat1.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/prosiebensat1.py b/youtube_dl/extractor/prosiebensat1.py
index 7cc799664..255d4abc1 100644
--- a/youtube_dl/extractor/prosiebensat1.py
+++ b/youtube_dl/extractor/prosiebensat1.py
@@ -17,7 +17,7 @@ from ..utils import (
 class ProSiebenSat1IE(InfoExtractor):
     IE_NAME = 'prosiebensat1'
     IE_DESC = 'ProSiebenSat.1 Digital'
-    _VALID_URL = r'https?://(?:www\.)?(?:(?:prosieben|prosiebenmaxx|sixx|sat1|kabeleins|ran|the-voice-of-germany)\.de|fem\.com)/(?P<id>.+)'
+    _VALID_URL = r'https?://(?:www\.)?(?:(?:prosieben|prosiebenmaxx|sixx|sat1|kabeleins|the-voice-of-germany)\.(?:de|at)|ran\.de|fem\.com)/(?P<id>.+)'
 
     _TESTS = [
         {

From abca34cbc04693662d913e19634c06c214a237f6 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 24 May 2015 02:04:02 +0800
Subject: [PATCH 1376/6219] [cnn] Relax _VALID_URL again (fixes #5737)

The problem is the same as test:CNN_1, so I didn't add the test case
---
 youtube_dl/extractor/cnn.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/cnn.py b/youtube_dl/extractor/cnn.py
index 5efc5f4fe..3b1bd4033 100644
--- a/youtube_dl/extractor/cnn.py
+++ b/youtube_dl/extractor/cnn.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 class CNNIE(InfoExtractor):
     _VALID_URL = r'''(?x)https?://(?:(?:edition|www)\.)?cnn\.com/video/(?:data/.+?|\?)/
-        (?P<path>.+?/(?P<title>[^/]+?)(?:\.(?:[a-z]{3,5})(?:-ap)?|(?=&)))'''
+        (?P<path>.+?/(?P<title>[^/]+?)(?:\.(?:[a-z\-]+)|(?=&)))'''
 
     _TESTS = [{
         'url': 'http://edition.cnn.com/video/?/video/sports/2013/06/09/nadal-1-on-1.cnn',

From 9bf87ae3aaac81df3efb92fd0a3247ccb522de2a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 24 May 2015 02:36:47 +0800
Subject: [PATCH 1377/6219] [nextmedia] Merge AppleDailyRealtimeNewsIE and
 AppleDailyAnimationNewsIE

---
 youtube_dl/extractor/__init__.py  |  3 +--
 youtube_dl/extractor/nextmedia.py | 37 +++++++++++++------------------
 2 files changed, 16 insertions(+), 24 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 24efb7ce5..8bb3926a0 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -338,8 +338,7 @@ from .newstube import NewstubeIE
 from .nextmedia import (
     NextMediaIE,
     NextMediaActionNewsIE,
-    AppleDailyRealtimeNewsIE,
-    AppleDailyAnimationNewsIE
+    AppleDailyIE,
 )
 from .nfb import NFBIE
 from .nfl import NFLIE
diff --git a/youtube_dl/extractor/nextmedia.py b/youtube_dl/extractor/nextmedia.py
index 02dba4ef6..c75ccafc7 100644
--- a/youtube_dl/extractor/nextmedia.py
+++ b/youtube_dl/extractor/nextmedia.py
@@ -89,8 +89,8 @@ class NextMediaActionNewsIE(NextMediaIE):
         return self._extract_from_nextmedia_page(news_id, url, article_page)
 
 
-class AppleDailyRealtimeNewsIE(NextMediaIE):
-    _VALID_URL = r'http://(www|ent).appledaily.com.tw/(realtimenews|enews)/[^/]+/[^/]+/(?P<date>\d+)/(?P<id>\d+)(/.*)?'
+class AppleDailyIE(NextMediaIE):
+    _VALID_URL = r'http://(www|ent).appledaily.com.tw/(animation|realtimenews|enews)/[^/]+/[^/]+/(?P<date>\d+)/(?P<id>\d+)(/.*)?'
     _TESTS = [{
         'url': 'http://ent.appledaily.com.tw/enews/article/entertainment/20150128/36354694',
         'md5': 'a843ab23d150977cc55ef94f1e2c1e4d',
@@ -99,7 +99,7 @@ class AppleDailyRealtimeNewsIE(NextMediaIE):
             'ext': 'mp4',
             'title': '周亭羽走過摩鐵陰霾2男陪吃 九把刀孤寒看醫生',
             'thumbnail': 're:^https?://.*\.jpg$',
-            'description': 'md5:b23787119933404ce515c6356a8c355c',
+            'description': 'md5:2acd430e59956dc47cd7f67cb3c003f4',
             'upload_date': '20150128',
         }
     }, {
@@ -110,26 +110,10 @@ class AppleDailyRealtimeNewsIE(NextMediaIE):
             'ext': 'mp4',
             'title': '不滿被踩腳　山東兩大媽一路打下車',
             'thumbnail': 're:^https?://.*\.jpg$',
-            'description': 'md5:2648aaf6fc4f401f6de35a91d111aa1d',
+            'description': 'md5:175b4260c1d7c085993474217e4ab1b4',
             'upload_date': '20150128',
         }
-    }]
-
-    _URL_PATTERN = r'\{url: \'(.+)\'\}'
-
-    def _fetch_title(self, page):
-        return self._html_search_regex(r'<h1 id="h1">([^<>]+)</h1>', page, 'news title')
-
-    def _fetch_thumbnail(self, page):
-        return self._html_search_regex(r"setInitialImage\(\'([^']+)'\)", page, 'video thumbnail', fatal=False)
-
-    def _fetch_timestamp(self, page):
-        return None
-
-
-class AppleDailyAnimationNewsIE(AppleDailyRealtimeNewsIE):
-    _VALID_URL = 'http://www.appledaily.com.tw/animation/[^/]+/[^/]+/(?P<date>\d+)/(?P<id>\d+)(/.*)?'
-    _TESTS = [{
+    }, {
         'url': 'http://www.appledaily.com.tw/animation/realtimenews/new/20150128/5003671',
         'md5': '03df296d95dedc2d5886debbb80cb43f',
         'info_dict': {
@@ -156,8 +140,17 @@ class AppleDailyAnimationNewsIE(AppleDailyRealtimeNewsIE):
         ]
     }]
 
+    _URL_PATTERN = r'\{url: \'(.+)\'\}'
+
     def _fetch_title(self, page):
-        return self._html_search_meta('description', page, 'news title')
+        return (self._html_search_regex(r'<h1 id="h1">([^<>]+)</h1>', page, 'news title', default=None) or
+                self._html_search_meta('description', page, 'news title'))
+
+    def _fetch_thumbnail(self, page):
+        return self._html_search_regex(r"setInitialImage\(\'([^']+)'\)", page, 'video thumbnail', fatal=False)
+
+    def _fetch_timestamp(self, page):
+        return None
 
     def _fetch_description(self, page):
         return self._html_search_meta('description', page, 'news description')

From 30455ce2554d00489901a398d457fac89456fe49 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 24 May 2015 02:42:01 +0800
Subject: [PATCH 1378/6219] [nextmedia] Extend and reorder _VALID_URL

---
 youtube_dl/extractor/nextmedia.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nextmedia.py b/youtube_dl/extractor/nextmedia.py
index c75ccafc7..d1b7cff4c 100644
--- a/youtube_dl/extractor/nextmedia.py
+++ b/youtube_dl/extractor/nextmedia.py
@@ -90,7 +90,7 @@ class NextMediaActionNewsIE(NextMediaIE):
 
 
 class AppleDailyIE(NextMediaIE):
-    _VALID_URL = r'http://(www|ent).appledaily.com.tw/(animation|realtimenews|enews)/[^/]+/[^/]+/(?P<date>\d+)/(?P<id>\d+)(/.*)?'
+    _VALID_URL = r'http://(www|ent).appledaily.com.tw/(?:animation|appledaily|enews|realtimenews)/[^/]+/[^/]+/(?P<date>\d+)/(?P<id>\d+)(/.*)?'
     _TESTS = [{
         'url': 'http://ent.appledaily.com.tw/enews/article/entertainment/20150128/36354694',
         'md5': 'a843ab23d150977cc55ef94f1e2c1e4d',
@@ -138,6 +138,9 @@ class AppleDailyIE(NextMediaIE):
         'expected_warnings': [
             'video thumbnail',
         ]
+    }, {
+        'url': 'http://www.appledaily.com.tw/appledaily/article/supplement/20140417/35770334/',
+        'only_matching': True,
     }]
 
     _URL_PATTERN = r'\{url: \'(.+)\'\}'

From 1335c3aca8f0cbddc0c521c73579eec2b9a5643c Mon Sep 17 00:00:00 2001
From: Sergey M? <dstftw@gmail.com>
Date: Sun, 24 May 2015 01:21:18 +0600
Subject: [PATCH 1379/6219] [drtv] Improve extraction (Closes #5792)

---
 youtube_dl/extractor/drtv.py | 37 +++++++++++++++++++++++++-----------
 1 file changed, 26 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/drtv.py b/youtube_dl/extractor/drtv.py
index f25ab319e..baa24c6d1 100644
--- a/youtube_dl/extractor/drtv.py
+++ b/youtube_dl/extractor/drtv.py
@@ -1,8 +1,11 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-from .common import InfoExtractor, ExtractorError
-from ..utils import parse_iso8601
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    parse_iso8601,
+)
 
 
 class DRTVIE(InfoExtractor):
@@ -60,19 +63,31 @@ class DRTVIE(InfoExtractor):
                 restricted_to_denmark = asset['RestrictedToDenmark']
                 spoken_subtitles = asset['Target'] == 'SpokenSubtitles'
                 for link in asset['Links']:
-                    target = link['Target']
                     uri = link['Uri']
+                    target = link['Target']
                     format_id = target
-                    preference = -1 if target == 'HDS' else -2
+                    preference = None
                     if spoken_subtitles:
-                        preference -= 2
+                        preference = -1
                         format_id += '-spoken-subtitles'
-                    formats.append({
-                        'url': uri + '?hdcore=3.3.0&plugin=aasp-3.3.0.99.43' if target == 'HDS' else uri,
-                        'format_id': format_id,
-                        'ext': link['FileFormat'],
-                        'preference': preference,
-                    })
+                    if target == 'HDS':
+                        formats.extend(self._extract_f4m_formats(
+                            uri + '?hdcore=3.3.0&plugin=aasp-3.3.0.99.43',
+                            video_id, preference, f4m_id=format_id))
+                    elif target == 'HLS':
+                        formats.extend(self._extract_m3u8_formats(
+                            uri, video_id, 'mp4', preference=preference,
+                            m3u8_id=format_id))
+                    else:
+                        bitrate = link.get('Bitrate')
+                        if bitrate:
+                            format_id += '-%s' % bitrate
+                        formats.append({
+                            'url': uri,
+                            'format_id': format_id,
+                            'tbr': bitrate,
+                            'ext': link.get('FileFormat'),
+                        })
                 subtitles_list = asset.get('SubtitlesList')
                 if isinstance(subtitles_list, list):
                     LANGS = {

From 71646e465348b25962a15f9a567f134514bde30a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 24 May 2015 04:14:01 +0600
Subject: [PATCH 1380/6219] [YoutubeDL] Initialize `files_to_delete` (Closes
 #5797)

---
 youtube_dl/YoutubeDL.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 58b34e087..d1953c18f 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1527,6 +1527,7 @@ class YoutubeDL(object):
             pps_chain.extend(ie_info['__postprocessors'])
         pps_chain.extend(self._pps)
         for pp in pps_chain:
+            files_to_delete = []
             try:
                 files_to_delete, info = pp.run(info)
             except PostProcessingError as e:

From 1807ae22dd93646ea4d7ba4bd28087bf1ef4857c Mon Sep 17 00:00:00 2001
From: WassimAttar <wassim.attar@free.fr>
Date: Sun, 24 May 2015 10:37:05 +0200
Subject: [PATCH 1381/6219] chmod error

After installing youtube-dl with this method
    sudo wget https://yt-dl.org/downloads/latest/youtube-dl -O /usr/local/bin/youtube-dl
    sudo chmod a+xr /usr/local/bin/youtube-dl
When i try to use it, i get this error
    python: can't open file '/usr/local/bin/youtube-dl': [Errno 13] Permission denied

The correct chmod is a+xr
---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 3d9436456..a29cccb3f 100644
--- a/README.md
+++ b/README.md
@@ -17,12 +17,12 @@ youtube-dl - download videos from youtube.com or other video platforms
 To install it right away for all UNIX users (Linux, OS X, etc.), type:
 
     sudo curl https://yt-dl.org/latest/youtube-dl -o /usr/local/bin/youtube-dl
-    sudo chmod a+x /usr/local/bin/youtube-dl
+    sudo chmod a+xr /usr/local/bin/youtube-dl
 
 If you do not have curl, you can alternatively use a recent wget:
 
     sudo wget https://yt-dl.org/downloads/latest/youtube-dl -O /usr/local/bin/youtube-dl
-    sudo chmod a+x /usr/local/bin/youtube-dl
+    sudo chmod a+xr /usr/local/bin/youtube-dl
 
 Windows users can [download a .exe file](https://yt-dl.org/latest/youtube-dl.exe) and place it in their home directory or any other location on their [PATH](http://en.wikipedia.org/wiki/PATH_%28variable%29).
 

From 23905927e19280d9217ecad377ef26ea9d5793fe Mon Sep 17 00:00:00 2001
From: "Sergey M." <dstftw@gmail.com>
Date: Sun, 24 May 2015 18:32:04 +0600
Subject: [PATCH 1382/6219] [README.md] Keep more idiomatic rwx order

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index a29cccb3f..e51bb5343 100644
--- a/README.md
+++ b/README.md
@@ -17,12 +17,12 @@ youtube-dl - download videos from youtube.com or other video platforms
 To install it right away for all UNIX users (Linux, OS X, etc.), type:
 
     sudo curl https://yt-dl.org/latest/youtube-dl -o /usr/local/bin/youtube-dl
-    sudo chmod a+xr /usr/local/bin/youtube-dl
+    sudo chmod a+rx /usr/local/bin/youtube-dl
 
 If you do not have curl, you can alternatively use a recent wget:
 
     sudo wget https://yt-dl.org/downloads/latest/youtube-dl -O /usr/local/bin/youtube-dl
-    sudo chmod a+xr /usr/local/bin/youtube-dl
+    sudo chmod a+rx /usr/local/bin/youtube-dl
 
 Windows users can [download a .exe file](https://yt-dl.org/latest/youtube-dl.exe) and place it in their home directory or any other location on their [PATH](http://en.wikipedia.org/wiki/PATH_%28variable%29).
 

From abac15f3c6915d176c37f7aa748b8a0f03db82a0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 24 May 2015 19:08:22 +0600
Subject: [PATCH 1383/6219] [tnaflix] Do not capture `cat_id`

---
 youtube_dl/extractor/tnaflix.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tnaflix.py b/youtube_dl/extractor/tnaflix.py
index 79496039d..59af9aba0 100644
--- a/youtube_dl/extractor/tnaflix.py
+++ b/youtube_dl/extractor/tnaflix.py
@@ -10,7 +10,7 @@ from ..utils import (
 
 
 class TNAFlixIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?tnaflix\.com/(?P<cat_id>[^/]+)/(?P<display_id>[^/]+)/video(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?tnaflix\.com/[^/]+/(?P<display_id>[^/]+)/video(?P<id>\d+)'
 
     _TITLE_REGEX = r'<title>(.+?) - TNAFlix Porn Videos</title>'
     _DESCRIPTION_REGEX = r'<h3 itemprop="description">([^<]+)</h3>'

From 34fb7e46ad3fa1a04635fa4876401aac881bb39b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 24 May 2015 19:10:03 +0600
Subject: [PATCH 1384/6219] [empflix] Relax _VALID_URL

---
 youtube_dl/extractor/empflix.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/empflix.py b/youtube_dl/extractor/empflix.py
index 0dc947c1d..9a5a8f4bb 100644
--- a/youtube_dl/extractor/empflix.py
+++ b/youtube_dl/extractor/empflix.py
@@ -4,7 +4,7 @@ from .tnaflix import TNAFlixIE
 
 
 class EMPFlixIE(TNAFlixIE):
-    _VALID_URL = r'^https?://www\.empflix\.com/videos/(?P<display_id>[0-9a-zA-Z-]+)-(?P<id>[0-9]+)\.html'
+    _VALID_URL = r'https?://(?:www\.)?empflix\.com/videos/(?P<display_id>.+?)-(?P<id>[0-9]+)\.html'
 
     _TITLE_REGEX = r'name="title" value="(?P<title>[^"]*)"'
     _DESCRIPTION_REGEX = r'name="description" value="([^"]*)"'
@@ -23,5 +23,9 @@ class EMPFlixIE(TNAFlixIE):
                 'thumbnail': 're:https?://.*\.jpg$',
                 'age_limit': 18,
             }
+        },
+        {
+            'url': 'http://www.empflix.com/videos/[AROMA][ARMD-718]-Aoi-Yoshino-Sawa-25826.html',
+            'matching_only': True,
         }
     ]

From d78c834ead934e5532d2f5bc221bb11eedaef0e5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 24 May 2015 20:04:13 +0600
Subject: [PATCH 1385/6219] [karrierevideos] Improve and simplify

---
 youtube_dl/extractor/karrierevideos.py | 96 +++++++++++++++++++-------
 1 file changed, 71 insertions(+), 25 deletions(-)

diff --git a/youtube_dl/extractor/karrierevideos.py b/youtube_dl/extractor/karrierevideos.py
index a05e8ab76..bed94bc93 100644
--- a/youtube_dl/extractor/karrierevideos.py
+++ b/youtube_dl/extractor/karrierevideos.py
@@ -1,50 +1,96 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..compat import compat_urlparse
+from ..utils import (
+    fix_xml_ampersands,
+    float_or_none,
+    xpath_with_ns,
+    xpath_text,
+)
 
 
 class KarriereVideosIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?karrierevideos\.at/berufsvideos/([a-z-]+)/(?P<id>[a-z-]+)'
-    _TEST = {
+    _VALID_URL = r'http://(?:www\.)?karrierevideos\.at(?:/[^/]+)+/(?P<id>[^/]+)'
+    _TESTS = [{
         'url': 'http://www.karrierevideos.at/berufsvideos/mittlere-hoehere-schulen/altenpflegerin',
         'info_dict': {
-            'id': 'altenpflegerin',
-            'ext': 'mp4',
+            'id': '32c91',
+            'ext': 'flv',
             'title': 'AltenpflegerIn',
-            'thumbnail': 're:^http://.*\.png\?v=[0-9]+',
-            'description': 'md5:dbadd1259fde2159a9b28667cb664ae2'
+            'description': 'md5:dbadd1259fde2159a9b28667cb664ae2',
+            'thumbnail': 're:^http://.*\.png',
         },
         'params': {
-            'skip_download': 'requires rtmpdump'
+            # rtmp download
+            'skip_download': True,
         }
-    }
+    }, {
+        # broken ampersands
+        'url': 'http://www.karrierevideos.at/orientierung/vaeterkarenz-und-neue-chancen-fuer-muetter-baby-was-nun',
+        'info_dict': {
+            'id': '5sniu',
+            'ext': 'flv',
+            'title': 'Väterkarenz und neue Chancen für Mütter - "Baby - was nun?"',
+            'description': 'md5:97092c6ad1fd7d38e9d6a5fdeb2bcc33',
+            'thumbnail': 're:^http://.*\.png',
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        }
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
+
         webpage = self._download_webpage(url, video_id)
 
+        title = (self._html_search_meta('title', webpage, default=None) or
+                 self._search_regex(r'<h1 class="title">([^<]+)</h1>'))
+
+        video_id = self._search_regex(
+            r'/config/video/(.+?)\.xml', webpage, 'video id')
+        playlist = self._download_xml(
+            'http://www.karrierevideos.at/player-playlist.xml.php?p=%s' % video_id,
+            video_id, transform_source=fix_xml_ampersands)
+
+        NS_MAP = {
+            'jwplayer': 'http://developer.longtailvideo.com/trac/wiki/FlashFormats'
+        }
+
+        def ns(path):
+            return xpath_with_ns(path, NS_MAP)
+
+        item = playlist.find('./tracklist/item')
+        video_file = xpath_text(
+            item, ns('./jwplayer:file'), 'video url', fatal=True)
+        streamer = xpath_text(
+            item, ns('./jwplayer:streamer'), 'streamer', fatal=True)
+
+        uploader = xpath_text(
+            item, ns('./jwplayer:author'), 'uploader')
+        duration = float_or_none(
+            xpath_text(item, ns('./jwplayer:duration'), 'duration'))
+
         description = self._html_search_regex(
-            r'<div class="leadtext">\n{0,}?\s{0,}<p>(.*?)</p>',
+            r'(?s)<div class="leadtext">(.+?)</div>',
             webpage, 'description')
 
-        playlist = self._html_search_regex(r'/config/video/(.*?)\.xml', webpage, 'playlist')
-        playlist = self._download_xml(
-            'http://www.karrierevideos.at/player-playlist.xml.php?p=' + playlist,
-            video_id)
-
-        namespace = 'http://developer.longtailvideo.com/trac/wiki/FlashFormats'
-
-        item = playlist.find('tracklist/item')
-        streamer = item.find('{%s}streamer' % namespace).text
+        thumbnail = self._html_search_meta(
+            'thumbnail', webpage, 'thumbnail')
+        if thumbnail:
+            thumbnail = compat_urlparse.urljoin(url, thumbnail)
 
         return {
             'id': video_id,
-            'title': self._html_search_meta('title', webpage),
+            'url': streamer.replace('rtmpt', 'rtmp'),
+            'play_path': 'mp4:%s' % video_file,
+            'ext': 'flv',
+            'title': title,
             'description': description,
-            'thumbnail': 'http://www.karrierevideos.at' + self._html_search_meta('thumbnail', webpage),
-            'protocol': 'rtmp',
-            'url': streamer.replace('rtmpt', 'http'),
-            'play_path': 'mp4:' + item.find('{%s}file' % namespace).text,
-            'tc_url': streamer,
-            'ext': 'mp4'
+            'thumbnail': thumbnail,
+            'uploader': uploader,
+            'duration': duration,
         }

From 63f3cab4aee5da45dc9b91a9661d5d52b5a72ec6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 24 May 2015 21:09:08 +0600
Subject: [PATCH 1386/6219] [rtbf] Fix extraction (Closes #5803)

---
 youtube_dl/extractor/rtbf.py | 30 +++++++++++++++---------------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/rtbf.py b/youtube_dl/extractor/rtbf.py
index dce64e151..5a381d9ce 100644
--- a/youtube_dl/extractor/rtbf.py
+++ b/youtube_dl/extractor/rtbf.py
@@ -1,10 +1,11 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
-import json
-
 from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    unescapeHTML,
+)
 
 
 class RTBFIE(InfoExtractor):
@@ -16,25 +17,24 @@ class RTBFIE(InfoExtractor):
             'id': '1921274',
             'ext': 'mp4',
             'title': 'Les Diables au coeur (épisode 2)',
-            'description': 'Football - Diables Rouges',
             'duration': 3099,
-            'timestamp': 1398456336,
-            'upload_date': '20140425',
         }
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
-        page = self._download_webpage('https://www.rtbf.be/video/embed?id=%s' % video_id, video_id)
+        webpage = self._download_webpage(
+            'http://www.rtbf.be/video/embed?id=%s' % video_id, video_id)
 
-        data = json.loads(self._html_search_regex(
-            r'<div class="js-player-embed(?: player-embed)?" data-video="([^"]+)"', page, 'data video'))['data']
+        data = self._parse_json(
+            unescapeHTML(self._search_regex(
+                r'data-video="([^"]+)"', webpage, 'data video')),
+            video_id)
 
         video_url = data.get('downloadUrl') or data.get('url')
 
-        if data['provider'].lower() == 'youtube':
+        if data.get('provider').lower() == 'youtube':
             return self.url_result(video_url, 'Youtube')
 
         return {
@@ -42,8 +42,8 @@ class RTBFIE(InfoExtractor):
             'url': video_url,
             'title': data['title'],
             'description': data.get('description') or data.get('subtitle'),
-            'thumbnail': data['thumbnail']['large'],
+            'thumbnail': data.get('thumbnail'),
             'duration': data.get('duration') or data.get('realDuration'),
-            'timestamp': data['created'],
-            'view_count': data['viewCount'],
+            'timestamp': int_or_none(data.get('created')),
+            'view_count': int_or_none(data.get('viewCount')),
         }

From 2ad5708c43a8672da547fa279e71b20c327793d2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 24 May 2015 21:25:00 +0600
Subject: [PATCH 1387/6219] [arte:future] Switch to `search_regex` for now
 (Closes #5801)

---
 youtube_dl/extractor/arte.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index 8273bd6c9..fce38248d 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -195,7 +195,9 @@ class ArteTVFutureIE(ArteTVPlus7IE):
     def _real_extract(self, url):
         anchor_id, lang = self._extract_url_info(url)
         webpage = self._download_webpage(url, anchor_id)
-        row = get_element_by_id(anchor_id, webpage)
+        row = self._search_regex(
+            r'(?s)id="%s"[^>]*>.+?(<div[^>]*arte_vp_url[^>]*>)' % anchor_id,
+            webpage, 'row')
         return self._extract_from_webpage(row, anchor_id, lang)
 
 

From 04b3b3df05a26a361441754afeb7ff24d0c1f559 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 22 May 2015 11:58:52 +0200
Subject: [PATCH 1388/6219] [youtube] Remove the nondash formats (fixes #5774)

Since we use fixed values for some fields like width and height they can be wrong, and would get picked by some formats filters.
For example for https://www.youtube.com/watch?v=EQCrhbBxsjA the biggest height is 720 and for nondash formats it's set to 1440, so -f 'bestvideo[height>=1200]+bestaudio' would incorrectly pick the nondash format, instead it should report that the requested format is not available.
---
 youtube_dl/extractor/youtube.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 1f9940cf5..0301682b8 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1126,12 +1126,13 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     self.report_warning(
                         'Skipping DASH manifest: %r' % e, video_id)
                 else:
-                    # Hide the formats we found through non-DASH
+                    # Remove the formats we found through non-DASH, they
+                    # contain less info and it can be wrong, because we use
+                    # fixed values (for example the resolution). See
+                    # https://github.com/rg3/youtube-dl/issues/5774 for an
+                    # example.
                     dash_keys = set(df['format_id'] for df in dash_formats)
-                    for f in formats:
-                        if f['format_id'] in dash_keys:
-                            f['format_id'] = 'nondash-%s' % f['format_id']
-                            f['preference'] = f.get('preference', 0) - 10000
+                    formats = [f for f in formats if f['format_id'] not in dash_keys]
                     formats.extend(dash_formats)
 
         # Check for malformed aspect ratio

From 4b4e1af059c0922da9770a79a68a471277303f21 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 24 May 2015 18:46:29 +0200
Subject: [PATCH 1389/6219] [arte] Remove unused import

---
 youtube_dl/extractor/arte.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index fce38248d..76de24477 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -7,7 +7,6 @@ from .common import InfoExtractor
 from ..utils import (
     find_xpath_attr,
     unified_strdate,
-    get_element_by_id,
     get_element_by_attribute,
     int_or_none,
     qualities,

From d41ebe146b693011eb1020ca9cd935e7db892d0b Mon Sep 17 00:00:00 2001
From: Sergey M? <dstftw@gmail.com>
Date: Sun, 24 May 2015 23:57:47 +0600
Subject: [PATCH 1390/6219] [tenplay] Fix formats and modernize (Closes #5806)

---
 youtube_dl/extractor/tenplay.py | 27 ++++++++++++++++++---------
 1 file changed, 18 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/tenplay.py b/youtube_dl/extractor/tenplay.py
index 466155ef8..f6694149b 100644
--- a/youtube_dl/extractor/tenplay.py
+++ b/youtube_dl/extractor/tenplay.py
@@ -2,6 +2,10 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    float_or_none,
+)
 
 
 class TenPlayIE(InfoExtractor):
@@ -49,18 +53,23 @@ class TenPlayIE(InfoExtractor):
             if protocol == 'rtmp':
                 url = url.replace('&mp4:', '')
 
+                tbr = int_or_none(rendition.get('encodingRate'), 1000)
+
             formats.append({
-                'format_id': '_'.join(['rtmp', rendition['videoContainer'].lower(), rendition['videoCodec'].lower()]),
-                'width': rendition['frameWidth'],
-                'height': rendition['frameHeight'],
-                'tbr': rendition['encodingRate'] / 1024,
-                'filesize': rendition['size'],
+                'format_id': '_'.join(
+                    ['rtmp', rendition['videoContainer'].lower(),
+                     rendition['videoCodec'].lower(), '%sk' % tbr]),
+                'width': int_or_none(rendition['frameWidth']),
+                'height': int_or_none(rendition['frameHeight']),
+                'tbr': tbr,
+                'filesize': int_or_none(rendition['size']),
                 'protocol': protocol,
                 'ext': ext,
                 'vcodec': rendition['videoCodec'].lower(),
                 'container': rendition['videoContainer'].lower(),
                 'url': url,
             })
+        self._sort_formats(formats)
 
         return {
             'id': video_id,
@@ -74,8 +83,8 @@ class TenPlayIE(InfoExtractor):
                 'url': json['thumbnailURL']
             }],
             'thumbnail': json['videoStillURL'],
-            'duration': json['length'] / 1000,
-            'timestamp': float(json['creationDate']) / 1000,
-            'uploader': json['customFields']['production_company_distributor'] if 'production_company_distributor' in json['customFields'] else 'TENplay',
-            'view_count': json['playsTotal']
+            'duration': float_or_none(json.get('length'), 1000),
+            'timestamp': float_or_none(json.get('creationDate'), 1000),
+            'uploader': json.get('customFields', {}).get('production_company_distributor') or 'TENplay',
+            'view_count': int_or_none(json.get('playsTotal')),
         }

From 7198063d96003050eccb0ea59cc938f0388c0606 Mon Sep 17 00:00:00 2001
From: Mister Hat <misterhat144@gmail.com>
Date: Sun, 24 May 2015 15:26:59 -0500
Subject: [PATCH 1391/6219] [pinkbike] new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/pinkbike.py | 78 ++++++++++++++++++++++++++++++++
 2 files changed, 79 insertions(+)
 create mode 100644 youtube_dl/extractor/pinkbike.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 79bcd9106..80bec39da 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -394,6 +394,7 @@ from .pbs import PBSIE
 from .philharmoniedeparis import PhilharmonieDeParisIE
 from .phoenix import PhoenixIE
 from .photobucket import PhotobucketIE
+from .pinkbike import PinkbikeIE
 from .planetaplay import PlanetaPlayIE
 from .pladform import PladformIE
 from .played import PlayedIE
diff --git a/youtube_dl/extractor/pinkbike.py b/youtube_dl/extractor/pinkbike.py
new file mode 100644
index 000000000..4a15c1835
--- /dev/null
+++ b/youtube_dl/extractor/pinkbike.py
@@ -0,0 +1,78 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+
+class PinkbikeIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?pinkbike\.com/video/(?P<id>[0-9]+)'
+    _TESTS = [{
+        'url': 'http://www.pinkbike.com/video/402811/',
+        'md5': '4814b8ca7651034cd87e3361d5c2155a',
+        'info_dict': {
+            'id': '402811',
+            'ext': 'mp4',
+            'title': 'Brandon Semenuk - RAW 100',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'location': 'Victoria, British Columbia, Canada',
+            'uploader_id': 'revelco',
+            'upload_date': '20150406',
+            'description': 'Official release: www.redbull.ca/rupertwalker',
+            'duration': '100'
+        }
+    }, {
+        'url': 'http://www.pinkbike.com/video/406629/',
+        'md5': 'c7a3e19a2bd5cde5a1cda6b2b46caa74',
+        'info_dict': {
+            'id': '406629',
+            'ext': 'mp4',
+            'title': 'Chromag: Reece Wallace in Utah',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'location': 'Whistler, British Columbia, Canada',
+            'uploader_id': 'Chromagbikes',
+            'upload_date': '20150505',
+            'description': 'Reece Wallace shredding Virgin, Utah. Video by Virtu Media.',
+            'duration': '180'
+        }
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        title = self._html_search_regex(r'<title>(.*?)</title>', webpage, 'title')
+        title = title[:-len(' Video - Pinkbike')]
+
+        description = self._html_search_meta('description', webpage, 'description')
+        description = description[len(title + '. '):]
+
+        uploader_id = self._html_search_regex(r'un:\s*"(.*?)"', webpage, 'uploader_id')
+
+        upload_date = self._html_search_regex(
+            r'class="fullTime"\s*title="([0-9]{4}(?:-[0-9]{2}){2})"',
+            webpage, 'upload_date')
+        upload_date = upload_date.replace('-', '')
+
+        location = self._html_search_regex(
+            r'<dt>Location</dt>\n?\s*<dd>\n?(.*?)\s*<img',
+            webpage, 'location')
+
+        formats = re.findall(
+            r'<source data-quality=\\"([0-9]+)p\\" src=\\"(.*?)\\">',
+            webpage)
+
+        formats = [{'url': fmt[1], 'height': fmt[0]} for fmt in formats]
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'duration': self._html_search_meta('video:duration', webpage, 'duration'),
+            'thumbnail': self._html_search_meta('og:image', webpage, 'thumbnail'),
+            'uploader_id': uploader_id,
+            'upload_date': upload_date,
+            'location': location,
+            'formats': formats
+        }

From 2c935c0c7224a3332ff9f0fd83e8c074cfbe2c9d Mon Sep 17 00:00:00 2001
From: Mister Hat <misterhat144@gmail.com>
Date: Sun, 24 May 2015 16:30:03 -0500
Subject: [PATCH 1392/6219] [pinkbike] converted duration to int

---
 youtube_dl/extractor/pinkbike.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/pinkbike.py b/youtube_dl/extractor/pinkbike.py
index 4a15c1835..66605ddbe 100644
--- a/youtube_dl/extractor/pinkbike.py
+++ b/youtube_dl/extractor/pinkbike.py
@@ -20,7 +20,7 @@ class PinkbikeIE(InfoExtractor):
             'uploader_id': 'revelco',
             'upload_date': '20150406',
             'description': 'Official release: www.redbull.ca/rupertwalker',
-            'duration': '100'
+            'duration': 100
         }
     }, {
         'url': 'http://www.pinkbike.com/video/406629/',
@@ -34,7 +34,7 @@ class PinkbikeIE(InfoExtractor):
             'uploader_id': 'Chromagbikes',
             'upload_date': '20150505',
             'description': 'Reece Wallace shredding Virgin, Utah. Video by Virtu Media.',
-            'duration': '180'
+            'duration': 180
         }
     }]
 
@@ -69,7 +69,7 @@ class PinkbikeIE(InfoExtractor):
             'id': video_id,
             'title': title,
             'description': description,
-            'duration': self._html_search_meta('video:duration', webpage, 'duration'),
+            'duration': int(self._html_search_meta('video:duration', webpage, 'duration')),
             'thumbnail': self._html_search_meta('og:image', webpage, 'thumbnail'),
             'uploader_id': uploader_id,
             'upload_date': upload_date,

From 680f9744c4e010ad5111c7711c58c341d5ba24dd Mon Sep 17 00:00:00 2001
From: Mister Hat <misterhat144@gmail.com>
Date: Sun, 24 May 2015 16:45:10 -0500
Subject: [PATCH 1393/6219] [pinkbike] used proper conversion methods

---
 youtube_dl/extractor/pinkbike.py | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/pinkbike.py b/youtube_dl/extractor/pinkbike.py
index 66605ddbe..45c0b1377 100644
--- a/youtube_dl/extractor/pinkbike.py
+++ b/youtube_dl/extractor/pinkbike.py
@@ -4,6 +4,11 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    remove_end,
+    remove_start
+)
 
 
 class PinkbikeIE(InfoExtractor):
@@ -43,10 +48,13 @@ class PinkbikeIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         title = self._html_search_regex(r'<title>(.*?)</title>', webpage, 'title')
-        title = title[:-len(' Video - Pinkbike')]
+        title = remove_end(title, ' Video - Pinkbike')
 
         description = self._html_search_meta('description', webpage, 'description')
-        description = description[len(title + '. '):]
+        description = remove_start(description, title + '. ')
+
+        duration = int_or_none(self._html_search_meta(
+            'video:duration', webpage, 'duration'))
 
         uploader_id = self._html_search_regex(r'un:\s*"(.*?)"', webpage, 'uploader_id')
 
@@ -63,13 +71,13 @@ class PinkbikeIE(InfoExtractor):
             r'<source data-quality=\\"([0-9]+)p\\" src=\\"(.*?)\\">',
             webpage)
 
-        formats = [{'url': fmt[1], 'height': fmt[0]} for fmt in formats]
+        formats = [{'url': fmt[1], 'height': int_or_none(fmt[0])} for fmt in formats]
 
         return {
             'id': video_id,
             'title': title,
             'description': description,
-            'duration': int(self._html_search_meta('video:duration', webpage, 'duration')),
+            'duration': duration,
             'thumbnail': self._html_search_meta('og:image', webpage, 'thumbnail'),
             'uploader_id': uploader_id,
             'upload_date': upload_date,

From b885bae6340b2aa9406501250fdebfbeea54e5e0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 25 May 2015 04:53:53 +0600
Subject: [PATCH 1394/6219] Credit @misterhat for karrierevideos (#5729)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index ebed7ebb3..3410e1fb9 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -125,3 +125,4 @@ Roman Le Négrate
 Matthias Küch
 Julian Richen
 Ping O.
+Mister Hat

From c6bbdadd79fac001cde15e8fd118b9535427474d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 25 May 2015 21:22:13 +0600
Subject: [PATCH 1395/6219] [odnoklassniki] Support extraction from metadata
 URL (Closes #5813)

---
 youtube_dl/extractor/odnoklassniki.py | 33 ++++++++++++++++++++++-----
 1 file changed, 27 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/odnoklassniki.py b/youtube_dl/extractor/odnoklassniki.py
index fbc521d1a..691f503f5 100644
--- a/youtube_dl/extractor/odnoklassniki.py
+++ b/youtube_dl/extractor/odnoklassniki.py
@@ -2,6 +2,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..compat import compat_urllib_parse
 from ..utils import (
     unified_strdate,
     int_or_none,
@@ -11,8 +12,9 @@ from ..utils import (
 
 
 class OdnoklassnikiIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:odnoklassniki|ok)\.ru/(?:video|web-api/video/moviePlayer)/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:odnoklassniki|ok)\.ru/(?:video|web-api/video/moviePlayer)/(?P<id>[\d-]+)'
     _TESTS = [{
+        # metadata in JSON
         'url': 'http://ok.ru/video/20079905452',
         'md5': '8e24ad2da6f387948e7a7d44eb8668fe',
         'info_dict': {
@@ -20,11 +22,22 @@ class OdnoklassnikiIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Культура меняет нас (прекрасный ролик!))',
             'duration': 100,
-            'upload_date': '20141207',
             'uploader_id': '330537914540',
             'uploader': 'Виталий Добровольский',
             'like_count': int,
-            'age_limit': 0,
+        },
+    }, {
+        # metadataUrl
+        'url': 'http://ok.ru/video/63567059965189-0',
+        'md5': '9676cf86eff5391d35dea675d224e131',
+        'info_dict': {
+            'id': '63567059965189-0',
+            'ext': 'mp4',
+            'title': 'Девушка без комплексов ...',
+            'duration': 191,
+            'uploader_id': '534380003155',
+            'uploader': 'Андрей Мещанинов',
+            'like_count': int,
         },
     }, {
         'url': 'http://ok.ru/web-api/video/moviePlayer/20079905452',
@@ -41,7 +54,15 @@ class OdnoklassnikiIE(InfoExtractor):
                 r'data-attributes="([^"]+)"', webpage, 'player')),
             video_id)
 
-        metadata = self._parse_json(player['flashvars']['metadata'], video_id)
+        flashvars = player['flashvars']
+
+        metadata = flashvars.get('metadata')
+        if metadata:
+            metadata = self._parse_json(metadata, video_id)
+        else:
+            metadata = self._download_json(
+                compat_urllib_parse.unquote(flashvars['metadataUrl']),
+                video_id, 'Downloading metadata JSON')
 
         movie = metadata['movie']
         title = movie['title']
@@ -53,11 +74,11 @@ class OdnoklassnikiIE(InfoExtractor):
         uploader = author.get('name')
 
         upload_date = unified_strdate(self._html_search_meta(
-            'ya:ovs:upload_date', webpage, 'upload date'))
+            'ya:ovs:upload_date', webpage, 'upload date', default=None))
 
         age_limit = None
         adult = self._html_search_meta(
-            'ya:ovs:adult', webpage, 'age limit')
+            'ya:ovs:adult', webpage, 'age limit', default=None)
         if adult:
             age_limit = 18 if adult == 'true' else 0
 

From ba2df04b41b62d08e1fd0efaaaf104467133e9a5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 25 May 2015 21:27:43 +0600
Subject: [PATCH 1396/6219] [odnoklassniki] Make URL explicit

---
 youtube_dl/extractor/odnoklassniki.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/odnoklassniki.py b/youtube_dl/extractor/odnoklassniki.py
index 691f503f5..6c7149fe3 100644
--- a/youtube_dl/extractor/odnoklassniki.py
+++ b/youtube_dl/extractor/odnoklassniki.py
@@ -47,7 +47,8 @@ class OdnoklassnikiIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        webpage = self._download_webpage(url, video_id)
+        webpage = self._download_webpage(
+            'http://ok.ru/video/%s' % video_id, video_id)
 
         player = self._parse_json(
             unescapeHTML(self._search_regex(

From 5d0a33eebcae821ac5d1124043a5ad77a58fa291 Mon Sep 17 00:00:00 2001
From: Alexander Kirk <alexander@kirk.at>
Date: Mon, 25 May 2015 20:12:18 +0200
Subject: [PATCH 1397/6219] rtlnow is now hosted at nowtv.de

---
 youtube_dl/extractor/__init__.py |   2 +-
 youtube_dl/extractor/nowtv.py    |  90 ++++++++++++++++
 youtube_dl/extractor/rtlnow.py   | 174 -------------------------------
 3 files changed, 91 insertions(+), 175 deletions(-)
 create mode 100644 youtube_dl/extractor/nowtv.py
 delete mode 100644 youtube_dl/extractor/rtlnow.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 79bcd9106..bfd07392e 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -355,6 +355,7 @@ from .normalboots import NormalbootsIE
 from .nosvideo import NosVideoIE
 from .novamov import NovaMovIE
 from .nowness import NownessIE
+from .nowtv import NowTvIE
 from .nowvideo import NowVideoIE
 from .npo import (
     NPOIE,
@@ -438,7 +439,6 @@ from .roxwel import RoxwelIE
 from .rtbf import RTBFIE
 from .rte import RteIE
 from .rtlnl import RtlNlIE
-from .rtlnow import RTLnowIE
 from .rtl2 import RTL2IE
 from .rtp import RTPIE
 from .rts import RTSIE
diff --git a/youtube_dl/extractor/nowtv.py b/youtube_dl/extractor/nowtv.py
new file mode 100644
index 000000000..bf97fe7f4
--- /dev/null
+++ b/youtube_dl/extractor/nowtv.py
@@ -0,0 +1,90 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    qualities,
+    unified_strdate,
+    int_or_none,
+)
+
+class NowTvIE(InfoExtractor):
+    """Information Extractor for RTL NOW, RTL2 NOW, RTL NITRO, SUPER RTL NOW, VOX NOW and n-tv NOW"""
+    _VALID_URL = r'''(?x)
+                        (?:https?://)?
+                        (
+                            (?:www\.)?nowtv\.de
+                            /(rtl|rtl2|rtlnitro||superrtl|ntv|vox)(?P<path>/.*?)/player
+                        )'''
+
+    _TESTS = [
+        {
+            'url': 'http://www.nowtv.de/vox/der-hundeprofi/buero-fall-chihuahua-joel/player',
+            'info_dict': {
+                'id': '128953',
+                'ext': 'mp4',
+                'title': 'B\u00fcro-Fall \/ Chihuahua \'Joel\'',
+                'description': 'md5:ce843b6b5901d9a7f7d04d1bbcdb12de',
+                'upload_date': '2015-05-23 19:10:00',
+                'duration': '00:51:32',
+            },
+            'params': {
+                'skip_download': True,
+            },
+            'skip': 'Only works from Germany',
+        },
+    ]
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        info_url = 'https://api.nowtv.de/v3/movies' + mobj.group('path') + '?fields=*,format,files,breakpoints,paymentPaytypes,trailers'
+        info = self._download_json(info_url, None)
+
+        video_id = info['id']
+        title = info['title']
+        description = info['articleShort']
+        duration = info['duration']
+        upload_date = unified_strdate(info['broadcastStartDate'])
+        free = info['free']
+        station = info['format']['station']
+        thumbnail = info['format']['defaultImage169Logo']
+
+        if station == 'rtl':
+            base_url = 'http://hls.fra.rtlnow.de/hls-vod-enc/'
+        elif station == 'rtl2':
+            base_url = 'http://hls.fra.rtl2now.de/hls-vod-enc/'
+        elif station == 'vox':
+            base_url = 'http://hls.fra.voxnow.de/hls-vod-enc/'
+        elif station == 'nitro':
+            base_url = 'http://hls.fra.rtlnitronow.de/hls-vod-enc/'
+        elif station == 'ntv':
+            base_url = 'http://hls.fra.n-tvnow.de/hls-vod-enc/'
+        elif station == 'superrtl':
+            base_url = 'http://hls.fra.superrtlnow.de/hls-vod-enc/'
+
+        formats = []
+        for item in info['files']['items']:
+            if item['type'] != 'video/x-abr':
+                continue
+
+            fmt = {
+                'url': base_url + item['path'] + '.m3u8',
+                'tbr': int_or_none(item['bitrate']),
+                'ext': 'mp4',
+                'format_id': int_or_none(item['id']),
+            }
+            formats.append(fmt)
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'upload_date': upload_date,
+            'duration': duration,
+            'formats': formats,
+        }
diff --git a/youtube_dl/extractor/rtlnow.py b/youtube_dl/extractor/rtlnow.py
deleted file mode 100644
index 785a8045e..000000000
--- a/youtube_dl/extractor/rtlnow.py
+++ /dev/null
@@ -1,174 +0,0 @@
-# encoding: utf-8
-from __future__ import unicode_literals
-
-import re
-
-from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
-    clean_html,
-    unified_strdate,
-    int_or_none,
-)
-
-
-class RTLnowIE(InfoExtractor):
-    """Information Extractor for RTL NOW, RTL2 NOW, RTL NITRO, SUPER RTL NOW, VOX NOW and n-tv NOW"""
-    _VALID_URL = r'''(?x)
-                        (?:https?://)?
-                        (?P<url>
-                            (?P<domain>
-                                rtl-now\.rtl\.de|
-                                rtl2now\.rtl2\.de|
-                                (?:www\.)?voxnow\.de|
-                                (?:www\.)?rtlnitronow\.de|
-                                (?:www\.)?superrtlnow\.de|
-                                (?:www\.)?n-tvnow\.de)
-                            /+[a-zA-Z0-9-]+/[a-zA-Z0-9-]+\.php\?
-                            (?:container_id|film_id)=(?P<video_id>[0-9]+)&
-                            player=1(?:&season=[0-9]+)?(?:&.*)?
-                        )'''
-
-    _TESTS = [
-        {
-            'url': 'http://rtl-now.rtl.de/ahornallee/folge-1.php?film_id=90419&player=1&season=1',
-            'info_dict': {
-                'id': '90419',
-                'ext': 'flv',
-                'title': 'Ahornallee - Folge 1 - Der Einzug',
-                'description': 'md5:ce843b6b5901d9a7f7d04d1bbcdb12de',
-                'upload_date': '20070416',
-                'duration': 1685,
-            },
-            'params': {
-                'skip_download': True,
-            },
-            'skip': 'Only works from Germany',
-        },
-        {
-            'url': 'http://rtl2now.rtl2.de/aerger-im-revier/episode-15-teil-1.php?film_id=69756&player=1&season=2&index=5',
-            'info_dict': {
-                'id': '69756',
-                'ext': 'flv',
-                'title': 'Ärger im Revier - Ein junger Ladendieb, ein handfester Streit u.a.',
-                'description': 'md5:3fb247005ed21a935ffc82b7dfa70cf0',
-                'thumbnail': 'http://autoimg.static-fra.de/rtl2now/219850/1500x1500/image2.jpg',
-                'upload_date': '20120519',
-                'duration': 1245,
-            },
-            'params': {
-                'skip_download': True,
-            },
-            'skip': 'Only works from Germany',
-        },
-        {
-            'url': 'http://www.voxnow.de/voxtours/suedafrika-reporter-ii.php?film_id=13883&player=1&season=17',
-            'info_dict': {
-                'id': '13883',
-                'ext': 'flv',
-                'title': 'Voxtours - Südafrika-Reporter II',
-                'description': 'md5:de7f8d56be6fd4fed10f10f57786db00',
-                'upload_date': '20090627',
-                'duration': 1800,
-            },
-            'params': {
-                'skip_download': True,
-            },
-        },
-        {
-            'url': 'http://superrtlnow.de/medicopter-117/angst.php?film_id=99205&player=1',
-            'info_dict': {
-                'id': '99205',
-                'ext': 'flv',
-                'title': 'Medicopter 117 - Angst!',
-                'description': 're:^Im Therapiezentrum \'Sonnalm\' kommen durch eine Unachtsamkeit die für die B.handlung mit Phobikern gehaltenen Voglespinnen frei\. Eine Ausreißerin',
-                'thumbnail': 'http://autoimg.static-fra.de/superrtlnow/287529/1500x1500/image2.jpg',
-                'upload_date': '20080928',
-                'duration': 2691,
-            },
-            'params': {
-                'skip_download': True,
-            },
-        },
-        {
-            'url': 'http://rtl-now.rtl.de/der-bachelor/folge-4.php?film_id=188729&player=1&season=5',
-            'info_dict': {
-                'id': '188729',
-                'ext': 'flv',
-                'upload_date': '20150204',
-                'description': 'md5:5e1ce23095e61a79c166d134b683cecc',
-                'title': 'Der Bachelor - Folge 4',
-            }
-        }, {
-            'url': 'http://www.n-tvnow.de/deluxe-alles-was-spass-macht/thema-ua-luxushotel-fuer-vierbeiner.php?container_id=153819&player=1&season=0',
-            'only_matching': True,
-        },
-    ]
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_page_url = 'http://%s/' % mobj.group('domain')
-        video_id = mobj.group('video_id')
-
-        webpage = self._download_webpage('http://' + mobj.group('url'), video_id)
-
-        mobj = re.search(r'(?s)<div style="margin-left: 20px; font-size: 13px;">(.*?)<div id="playerteaser">', webpage)
-        if mobj:
-            raise ExtractorError(clean_html(mobj.group(1)), expected=True)
-
-        title = self._og_search_title(webpage)
-        description = self._og_search_description(webpage)
-        thumbnail = self._og_search_thumbnail(webpage, default=None)
-
-        upload_date = unified_strdate(self._html_search_meta('uploadDate', webpage, 'upload date'))
-
-        mobj = re.search(r'<meta itemprop="duration" content="PT(?P<seconds>\d+)S" />', webpage)
-        duration = int(mobj.group('seconds')) if mobj else None
-
-        playerdata_url = self._html_search_regex(
-            r"'playerdata': '(?P<playerdata_url>[^']+)'", webpage, 'playerdata_url')
-
-        playerdata = self._download_xml(playerdata_url, video_id, 'Downloading player data XML')
-
-        videoinfo = playerdata.find('./playlist/videoinfo')
-
-        formats = []
-        for filename in videoinfo.findall('filename'):
-            mobj = re.search(r'(?P<url>rtmpe://(?:[^/]+/){2})(?P<play_path>.+)', filename.text)
-            if mobj:
-                fmt = {
-                    'url': mobj.group('url'),
-                    'play_path': 'mp4:' + mobj.group('play_path'),
-                    'page_url': video_page_url,
-                    'player_url': video_page_url + 'includes/vodplayer.swf',
-                }
-            else:
-                mobj = re.search(r'.*/(?P<hoster>[^/]+)/videos/(?P<play_path>.+)\.f4m', filename.text)
-                if mobj:
-                    fmt = {
-                        'url': 'rtmpe://fms.rtl.de/' + mobj.group('hoster'),
-                        'play_path': 'mp4:' + mobj.group('play_path'),
-                        'page_url': url,
-                        'player_url': video_page_url + 'includes/vodplayer.swf',
-                    }
-                else:
-                    fmt = {
-                        'url': filename.text,
-                    }
-            fmt.update({
-                'width': int_or_none(filename.get('width')),
-                'height': int_or_none(filename.get('height')),
-                'vbr': int_or_none(filename.get('bitrate')),
-                'ext': 'flv',
-            })
-            formats.append(fmt)
-
-        return {
-            'id': video_id,
-            'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
-            'upload_date': upload_date,
-            'duration': duration,
-            'formats': formats,
-        }

From 42833b44b5e3810a2875dfb130aefbf5db057c1e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 26 May 2015 13:32:43 +0800
Subject: [PATCH 1398/6219] [tf1] Extend _VALID_URL (fixes #5819)

---
 youtube_dl/extractor/tf1.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/tf1.py b/youtube_dl/extractor/tf1.py
index 025d0877c..656410528 100644
--- a/youtube_dl/extractor/tf1.py
+++ b/youtube_dl/extractor/tf1.py
@@ -6,8 +6,8 @@ from .common import InfoExtractor
 
 class TF1IE(InfoExtractor):
     """TF1 uses the wat.tv player."""
-    _VALID_URL = r'http://(?:videos\.tf1|www\.tfou)\.fr/.*?-(?P<id>\d+)(?:-\d+)?\.html'
-    _TESTS = {
+    _VALID_URL = r'http://(?:videos\.tf1|www\.tfou|www\.tf1)\.fr/.*?-(?P<id>\d+)(?:-\d+)?\.html'
+    _TESTS = [{
         'url': 'http://videos.tf1.fr/auto-moto/citroen-grand-c4-picasso-2013-presentation-officielle-8062060.html',
         'info_dict': {
             'id': '10635995',
@@ -32,7 +32,10 @@ class TF1IE(InfoExtractor):
             # Sometimes wat serves the whole file with the --test option
             'skip_download': True,
         },
-    }
+    }, {
+        'url': 'http://www.tf1.fr/tf1/koh-lanta/videos/replay-koh-lanta-22-mai-2015.html',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From db3ca36403930063b7df3b228a3f297bf278b43c Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 26 May 2015 13:37:15 +0800
Subject: [PATCH 1399/6219] [facebook] Move the title extraction warning below
 (fixes #5820)

---
 youtube_dl/extractor/facebook.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index 937b28fcc..e8d682716 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -50,7 +50,10 @@ class FacebookIE(InfoExtractor):
             'id': '274175099429670',
             'ext': 'mp4',
             'title': 'Facebook video #274175099429670',
-        }
+        },
+        'expected_warnings': [
+            'title'
+        ]
     }, {
         'url': 'https://www.facebook.com/video.php?v=10204634152394104',
         'only_matching': True,
@@ -150,11 +153,11 @@ class FacebookIE(InfoExtractor):
 
         video_title = self._html_search_regex(
             r'<h2 class="uiHeaderTitle">([^<]*)</h2>', webpage, 'title',
-            fatal=False)
+            default=None)
         if not video_title:
             video_title = self._html_search_regex(
                 r'(?s)<span class="fbPhotosPhotoCaption".*?id="fbPhotoPageCaption"><span class="hasCaption">(.*?)</span>',
-                webpage, 'alternative title', default=None)
+                webpage, 'alternative title', fatal=False)
             video_title = limit_length(video_title, 80)
         if not video_title:
             video_title = 'Facebook video #%s' % video_id

From efec4358b9b8ed5726c1f7d7939e8bce49f9100c Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 26 May 2015 13:54:41 +0800
Subject: [PATCH 1400/6219] [cinemassacre] Support an alternative form of
 screenwavemedia URL

fixes #5821
---
 youtube_dl/extractor/cinemassacre.py | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/cinemassacre.py b/youtube_dl/extractor/cinemassacre.py
index cf0a7551b..c949a4814 100644
--- a/youtube_dl/extractor/cinemassacre.py
+++ b/youtube_dl/extractor/cinemassacre.py
@@ -60,6 +60,17 @@ class CinemassacreIE(InfoExtractor):
                 'uploader_id': 'Cinemassacre',
                 'title': 'AVGN: McKids',
             }
+        },
+        {
+            'url': 'http://cinemassacre.com/2015/05/25/mario-kart-64-nintendo-64-james-mike-mondays/',
+            'md5': '1376908e49572389e7b06251a53cdd08',
+            'info_dict': {
+                'id': 'Cinemassacre-555779690c440',
+                'ext': 'mp4',
+                'description': 'Let’s Play Mario Kart 64 !! Mario Kart 64 is a classic go-kart racing game released for the Nintendo 64 (N64). Today James & Mike do 4 player Battle Mode with Kyle and Bootsy!',
+                'title': 'Mario Kart 64 (Nintendo 64) James & Mike Mondays',
+                'upload_date': '20150525',
+            }
         }
     ]
 
@@ -72,7 +83,7 @@ class CinemassacreIE(InfoExtractor):
 
         playerdata_url = self._search_regex(
             [
-                r'src="(http://player\.screenwavemedia\.com/play/[a-zA-Z]+\.php\?[^"]*\bid=.+?)"',
+                r'src="(http://(?:player2\.screenwavemedia\.com|player\.screenwavemedia\.com/play)/[a-zA-Z]+\.php\?[^"]*\bid=.+?)"',
                 r'<iframe[^>]+src="((?:https?:)?//(?:[^.]+\.)?youtube\.com/.+?)"',
             ],
             webpage, 'player data URL', default=None)

From ff305edd645d6f4307faa9307dea91694a1d217d Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Tue, 26 May 2015 13:43:00 +0300
Subject: [PATCH 1401/6219] [sockshare] Remove extractor

Haywire since last October.
---
 youtube_dl/extractor/sockshare.py | 83 -------------------------------
 1 file changed, 83 deletions(-)
 delete mode 100644 youtube_dl/extractor/sockshare.py

diff --git a/youtube_dl/extractor/sockshare.py b/youtube_dl/extractor/sockshare.py
deleted file mode 100644
index b5fa6f1da..000000000
--- a/youtube_dl/extractor/sockshare.py
+++ /dev/null
@@ -1,83 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-import re
-
-from ..compat import (
-    compat_urllib_parse,
-    compat_urllib_request,
-)
-from ..utils import (
-    determine_ext,
-    ExtractorError,
-)
-
-from .common import InfoExtractor
-
-
-class SockshareIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?sockshare\.com/file/(?P<id>[0-9A-Za-z]+)'
-    _FILE_DELETED_REGEX = r'This file doesn\'t exist, or has been removed\.</div>'
-    _TEST = {
-        'url': 'http://www.sockshare.com/file/437BE28B89D799D7',
-        'md5': '9d0bf1cfb6dbeaa8d562f6c97506c5bd',
-        'info_dict': {
-            'id': '437BE28B89D799D7',
-            'title': 'big_buck_bunny_720p_surround.avi',
-            'ext': 'avi',
-        }
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        url = 'http://sockshare.com/file/%s' % video_id
-        webpage = self._download_webpage(url, video_id)
-
-        if re.search(self._FILE_DELETED_REGEX, webpage) is not None:
-            raise ExtractorError('Video %s does not exist' % video_id,
-                                 expected=True)
-
-        confirm_hash = self._html_search_regex(r'''(?x)<input\s+
-            type="hidden"\s+
-            value="([^"]*)"\s+
-            name="hash"
-            ''', webpage, 'hash')
-
-        fields = {
-            "hash": confirm_hash.encode('utf-8'),
-            "confirm": "Continue as Free User"
-        }
-
-        post = compat_urllib_parse.urlencode(fields)
-        req = compat_urllib_request.Request(url, post)
-        # Apparently, this header is required for confirmation to work.
-        req.add_header('Host', 'www.sockshare.com')
-        req.add_header('Content-type', 'application/x-www-form-urlencoded')
-
-        webpage = self._download_webpage(
-            req, video_id, 'Downloading video page')
-
-        video_url = self._html_search_regex(
-            r'<a href="([^"]*)".+class="download_file_link"',
-            webpage, 'file url')
-        video_url = "http://www.sockshare.com" + video_url
-        title = self._html_search_regex((
-            r'<h1>(.+)<strong>',
-            r'var name = "([^"]+)";'),
-            webpage, 'title', default=None)
-        thumbnail = self._html_search_regex(
-            r'<img\s+src="([^"]*)".+?name="bg"',
-            webpage, 'thumbnail', default=None)
-
-        formats = [{
-            'format_id': 'sd',
-            'url': video_url,
-            'ext': determine_ext(title),
-        }]
-
-        return {
-            'id': video_id,
-            'title': title,
-            'thumbnail': thumbnail,
-            'formats': formats,
-        }

From 35a4f24a378419d41ea9d9fbcc9790fa457e83d2 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Tue, 26 May 2015 13:44:46 +0300
Subject: [PATCH 1402/6219] [firedrive] Remove extractor (Closes #3870)

Haywire since last October.
---
 youtube_dl/extractor/firedrive.py | 80 -------------------------------
 1 file changed, 80 deletions(-)
 delete mode 100644 youtube_dl/extractor/firedrive.py

diff --git a/youtube_dl/extractor/firedrive.py b/youtube_dl/extractor/firedrive.py
deleted file mode 100644
index 3191116d9..000000000
--- a/youtube_dl/extractor/firedrive.py
+++ /dev/null
@@ -1,80 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-import re
-
-from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse,
-    compat_urllib_request,
-)
-from ..utils import (
-    ExtractorError,
-)
-
-
-class FiredriveIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?firedrive\.com/' + \
-                 '(?:file|embed)/(?P<id>[0-9a-zA-Z]+)'
-    _FILE_DELETED_REGEX = r'<div class="removed_file_image">'
-
-    _TESTS = [{
-        'url': 'https://www.firedrive.com/file/FEB892FA160EBD01',
-        'md5': 'd5d4252f80ebeab4dc2d5ceaed1b7970',
-        'info_dict': {
-            'id': 'FEB892FA160EBD01',
-            'ext': 'flv',
-            'title': 'bbb_theora_486kbit.flv',
-            'thumbnail': 're:^http://.*\.jpg$',
-        },
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        url = 'http://firedrive.com/file/%s' % video_id
-        webpage = self._download_webpage(url, video_id)
-
-        if re.search(self._FILE_DELETED_REGEX, webpage) is not None:
-            raise ExtractorError('Video %s does not exist' % video_id,
-                                 expected=True)
-
-        fields = dict(re.findall(r'''(?x)<input\s+
-            type="hidden"\s+
-            name="([^"]+)"\s+
-            value="([^"]*)"
-            ''', webpage))
-
-        post = compat_urllib_parse.urlencode(fields)
-        req = compat_urllib_request.Request(url, post)
-        req.add_header('Content-type', 'application/x-www-form-urlencoded')
-
-        # Apparently, this header is required for confirmation to work.
-        req.add_header('Host', 'www.firedrive.com')
-
-        webpage = self._download_webpage(req, video_id,
-                                         'Downloading video page')
-
-        title = self._search_regex(r'class="external_title_left">(.+)</div>',
-                                   webpage, 'title')
-        thumbnail = self._search_regex(r'image:\s?"(//[^\"]+)', webpage,
-                                       'thumbnail', fatal=False)
-        if thumbnail is not None:
-            thumbnail = 'http:' + thumbnail
-
-        ext = self._search_regex(r'type:\s?\'([^\']+)\',',
-                                 webpage, 'extension', fatal=False)
-        video_url = self._search_regex(
-            r'file:\s?loadURL\(\'(http[^\']+)\'\),', webpage, 'file url')
-
-        formats = [{
-            'format_id': 'sd',
-            'url': video_url,
-            'ext': ext,
-        }]
-
-        return {
-            'id': video_id,
-            'title': title,
-            'thumbnail': thumbnail,
-            'formats': formats,
-        }

From 544a8693b7d9321d776987a5104889056955daa2 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Tue, 26 May 2015 13:53:14 +0300
Subject: [PATCH 1403/6219] Remove Firedrive and Sockshare imports

Oops
---
 youtube_dl/extractor/__init__.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 79bcd9106..80c9cb107 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -149,7 +149,6 @@ from .extremetube import ExtremeTubeIE
 from .facebook import FacebookIE
 from .faz import FazIE
 from .fc2 import FC2IE
-from .firedrive import FiredriveIE
 from .firstpost import FirstpostIE
 from .firsttv import FirstTVIE
 from .fivemin import FiveMinIE
@@ -480,7 +479,6 @@ from .smotri import (
     SmotriBroadcastIE,
 )
 from .snotr import SnotrIE
-from .sockshare import SockshareIE
 from .sohu import SohuIE
 from .soundcloud import (
     SoundcloudIE,

From 7d65242dc3c0c3306b775f0663d325ba55b62379 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 26 May 2015 22:12:26 +0600
Subject: [PATCH 1404/6219] [dailymotion:user] Process user home as user
 (Closes #5823)

---
 youtube_dl/extractor/dailymotion.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index db10b8d00..d8f9eb13f 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -225,7 +225,7 @@ class DailymotionPlaylistIE(DailymotionBaseInfoExtractor):
 
 class DailymotionUserIE(DailymotionPlaylistIE):
     IE_NAME = 'dailymotion:user'
-    _VALID_URL = r'https?://(?:www\.)?dailymotion\.[a-z]{2,3}/(?:old/)?user/(?P<user>[^/]+)'
+    _VALID_URL = r'https?://(?:www\.)?dailymotion\.[a-z]{2,3}/(?:(?:old/)?user/)?(?P<user>[^/]+)'
     _PAGE_TEMPLATE = 'http://www.dailymotion.com/user/%s/%s'
     _TESTS = [{
         'url': 'https://www.dailymotion.com/user/nqtv',
@@ -239,7 +239,8 @@ class DailymotionUserIE(DailymotionPlaylistIE):
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         user = mobj.group('user')
-        webpage = self._download_webpage(url, user)
+        webpage = self._download_webpage(
+            'https://www.dailymotion.com/user/%s' % user, user)
         full_user = unescapeHTML(self._html_search_regex(
             r'<a class="nav-image" title="([^"]+)" href="/%s">' % re.escape(user),
             webpage, 'user'))

From 5406af92bc3f88b9fe4f26fe20bdaaf0b4968c32 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 26 May 2015 22:16:47 +0600
Subject: [PATCH 1405/6219] [dailymotion:user] Fix _VALID_URL

---
 youtube_dl/extractor/dailymotion.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index d8f9eb13f..70aa4333c 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -225,7 +225,7 @@ class DailymotionPlaylistIE(DailymotionBaseInfoExtractor):
 
 class DailymotionUserIE(DailymotionPlaylistIE):
     IE_NAME = 'dailymotion:user'
-    _VALID_URL = r'https?://(?:www\.)?dailymotion\.[a-z]{2,3}/(?:(?:old/)?user/)?(?P<user>[^/]+)'
+    _VALID_URL = r'https?://(?:www\.)?dailymotion\.[a-z]{2,3}/(?:(?:old/)?user/)?(?P<user>[^/]+)$'
     _PAGE_TEMPLATE = 'http://www.dailymotion.com/user/%s/%s'
     _TESTS = [{
         'url': 'https://www.dailymotion.com/user/nqtv',

From 7a372b64dfa69d5b2cfd1514b89e8fc0ab7e5874 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 27 May 2015 01:41:00 +0600
Subject: [PATCH 1406/6219] [pornhub] Do not modify aes key string (Closes
 #5824)

---
 youtube_dl/extractor/pornhub.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index 0c8b731cf..daa284ea2 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -71,7 +71,8 @@ class PornHubIE(InfoExtractor):
 
         video_urls = list(map(compat_urllib_parse.unquote, re.findall(r'"quality_[0-9]{3}p":"([^"]+)', webpage)))
         if webpage.find('"encrypted":true') != -1:
-            password = compat_urllib_parse.unquote_plus(self._html_search_regex(r'"video_title":"([^"]+)', webpage, 'password'))
+            password = compat_urllib_parse.unquote_plus(
+                self._search_regex(r'"video_title":"([^"]+)', webpage, 'password'))
             video_urls = list(map(lambda s: aes_decrypt_text(s, password, 32).decode('utf-8'), video_urls))
 
         formats = []

From 1434184c577953ff6fe558ccc6751697791f4076 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 27 May 2015 01:42:53 +0600
Subject: [PATCH 1407/6219] [spankwire] Do not modify aes key string

---
 youtube_dl/extractor/spankwire.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/spankwire.py b/youtube_dl/extractor/spankwire.py
index b936202f6..06d6e6640 100644
--- a/youtube_dl/extractor/spankwire.py
+++ b/youtube_dl/extractor/spankwire.py
@@ -71,7 +71,7 @@ class SpankwireIE(InfoExtractor):
             compat_urllib_parse.unquote,
             re.findall(r'playerData\.cdnPath[0-9]{3,}\s*=\s*["\']([^"\']+)["\']', webpage)))
         if webpage.find('flashvars\.encrypted = "true"') != -1:
-            password = self._html_search_regex(
+            password = self._search_regex(
                 r'flashvars\.video_title = "([^"]+)',
                 webpage, 'password').replace('+', ' ')
             video_urls = list(map(

From b535170b218131afd1165776e611691479627ce8 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 27 May 2015 04:14:24 +0800
Subject: [PATCH 1408/6219] [bilibili] Skip assertion if HQ videos not
 available

---
 youtube_dl/extractor/bilibili.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/bilibili.py b/youtube_dl/extractor/bilibili.py
index 7ca835e31..a8bea2c10 100644
--- a/youtube_dl/extractor/bilibili.py
+++ b/youtube_dl/extractor/bilibili.py
@@ -80,9 +80,11 @@ class BiliBiliIE(InfoExtractor):
             note='Downloading HQ video info',
             fatal=False,
         )
-        hq_durls = hq_doc.findall('./durl') if hq_doc is not False else itertools.repeat(None)
-
-        assert len(lq_durls) == len(hq_durls)
+        if hq_doc is not False:
+            hq_durls = hq_doc.findall('./durl')
+            assert len(lq_durls) == len(hq_durls)
+        else:
+            hq_durls = itertools.repeat(None)
 
         i = 1
         for lq_durl, hq_durl in zip(lq_durls, hq_durls):

From 6d00a2dcd110f12a0aa110f5479df76613792fbd Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 27 May 2015 04:23:21 +0800
Subject: [PATCH 1409/6219] [bilibili] Catch API call failures

JSON are returned in a failed API call
---
 youtube_dl/extractor/bilibili.py | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bilibili.py b/youtube_dl/extractor/bilibili.py
index a8bea2c10..2103ed73a 100644
--- a/youtube_dl/extractor/bilibili.py
+++ b/youtube_dl/extractor/bilibili.py
@@ -3,6 +3,8 @@ from __future__ import unicode_literals
 
 import re
 import itertools
+import json
+import xml.etree.ElementTree as ET
 
 from .common import InfoExtractor
 from ..utils import (
@@ -67,11 +69,19 @@ class BiliBiliIE(InfoExtractor):
 
         entries = []
 
-        lq_doc = self._download_xml(
+        lq_page = self._download_webpage(
             'http://interface.bilibili.com/v_cdn_play?appkey=1&cid=%s' % cid,
             video_id,
             note='Downloading LQ video info'
         )
+        try:
+            err_info = json.loads(lq_page)
+            raise ExtractorError(
+                'BiliBili said: ' + err_info['error_text'], expected=True)
+        except ValueError:
+            pass
+
+        lq_doc = ET.fromstring(lq_page)
         lq_durls = lq_doc.findall('./durl')
 
         hq_doc = self._download_xml(

From c23848b3c5244f8ef1501adfd04a32111b12d7ff Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 27 May 2015 14:20:29 +0800
Subject: [PATCH 1410/6219] [naver] Enhanced error detection

---
 youtube_dl/extractor/naver.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/naver.py b/youtube_dl/extractor/naver.py
index c10405f04..fdd825784 100644
--- a/youtube_dl/extractor/naver.py
+++ b/youtube_dl/extractor/naver.py
@@ -35,7 +35,7 @@ class NaverIE(InfoExtractor):
                          webpage)
         if m_id is None:
             m_error = re.search(
-                r'(?s)<div class="nation_error">\s*(?:<!--.*?-->)?\s*<p class="[^"]+">(?P<msg>.+?)</p>\s*</div>',
+                r'(?s)<div class="(?:nation_error|nation_box)">\s*(?:<!--.*?-->)?\s*<p class="[^"]+">(?P<msg>.+?)</p>\s*</div>',
                 webpage)
             if m_error:
                 raise ExtractorError(clean_html(m_error.group('msg')), expected=True)

From f8d5e1cfb5d9a8c946a966452d9b86c45182a952 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 27 May 2015 14:44:08 +0800
Subject: [PATCH 1411/6219] [naver] Fix video url (fixes #5809)

RTMP urls in test:naver does not work. Need more investigation.
---
 youtube_dl/extractor/naver.py | 22 +++++++++++++++++++---
 1 file changed, 19 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/naver.py b/youtube_dl/extractor/naver.py
index fdd825784..925967753 100644
--- a/youtube_dl/extractor/naver.py
+++ b/youtube_dl/extractor/naver.py
@@ -6,6 +6,7 @@ import re
 from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse,
+    compat_urlparse,
 )
 from ..utils import (
     ExtractorError,
@@ -16,7 +17,7 @@ from ..utils import (
 class NaverIE(InfoExtractor):
     _VALID_URL = r'https?://(?:m\.)?tvcast\.naver\.com/v/(?P<id>\d+)'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://tvcast.naver.com/v/81652',
         'info_dict': {
             'id': '81652',
@@ -25,7 +26,18 @@ class NaverIE(InfoExtractor):
             'description': '합격불변의 법칙 메가스터디 | 메가스터디 수학 김상희 선생님이 9월 모의고사 수학A형 16번에서 20번까지 해설강의를 공개합니다.',
             'upload_date': '20130903',
         },
-    }
+    }, {
+        'url': 'http://tvcast.naver.com/v/395837',
+        'md5': '638ed4c12012c458fefcddfd01f173cd',
+        'info_dict': {
+            'id': '395837',
+            'ext': 'mp4',
+            'title': '9년이 지나도 아픈 기억, 전효성의 아버지',
+            'description': 'md5:5bf200dcbf4b66eb1b350d1eb9c753f7',
+            'upload_date': '20150519',
+        },
+        'skip': 'Georestricted',
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -58,14 +70,18 @@ class NaverIE(InfoExtractor):
         formats = []
         for format_el in urls.findall('EncodingOptions/EncodingOption'):
             domain = format_el.find('Domain').text
+            uri = format_el.find('uri').text
             f = {
-                'url': domain + format_el.find('uri').text,
+                'url': compat_urlparse.urljoin(domain, uri),
                 'ext': 'mp4',
                 'width': int(format_el.find('width').text),
                 'height': int(format_el.find('height').text),
             }
             if domain.startswith('rtmp'):
+                # urlparse does not support custom schemes
+                # https://bugs.python.org/issue18828
                 f.update({
+                    'url': domain + uri,
                     'ext': 'flv',
                     'rtmp_protocol': '1',  # rtmpt
                 })

From f9f3e3df9a9fff1b00c7184234c4f607ea3cec81 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 27 May 2015 14:51:18 +0800
Subject: [PATCH 1412/6219] [teamcoco] Use determine_ext to determine the video
 type

Some videos does not contain a 'type' field (#5798)
---
 youtube_dl/extractor/teamcoco.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/teamcoco.py b/youtube_dl/extractor/teamcoco.py
index 56be52638..b2a4b1fc0 100644
--- a/youtube_dl/extractor/teamcoco.py
+++ b/youtube_dl/extractor/teamcoco.py
@@ -10,6 +10,7 @@ from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
     qualities,
+    determine_ext,
 )
 from ..compat import compat_ord
 
@@ -108,7 +109,7 @@ class TeamcocoIE(InfoExtractor):
         formats = []
         get_quality = qualities(['500k', '480p', '1000k', '720p', '1080p'])
         for filed in data['files']:
-            if filed['type'] == 'hls':
+            if determine_ext(filed['url']) == 'm3u8':
                 formats.extend(self._extract_m3u8_formats(
                     filed['url'], video_id, ext='mp4'))
             else:

From f0bfaa2d7d9563975f1f6effa75d28dcdb1c23ce Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 27 May 2015 15:23:34 +0800
Subject: [PATCH 1413/6219] [nrk] Update subtitles test

Subtitle conversion routine is removed, so the subtitles are TTML now. See
1c7e2e64f6328024711d5fa999d4498396f4cb5c
---
 test/test_subtitles.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/test_subtitles.py b/test/test_subtitles.py
index 891ee620b..c4e3adb67 100644
--- a/test/test_subtitles.py
+++ b/test/test_subtitles.py
@@ -266,7 +266,7 @@ class TestNRKSubtitles(BaseTestSubtitles):
         self.DL.params['allsubtitles'] = True
         subtitles = self.getSubtitles()
         self.assertEqual(set(subtitles.keys()), set(['no']))
-        self.assertEqual(md5(subtitles['no']), '1d221e6458c95c5494dcd38e6a1f129a')
+        self.assertEqual(md5(subtitles['no']), '544fa917d3197fcbee64634559221cc2')
 
 
 class TestRaiSubtitles(BaseTestSubtitles):

From bf24c3d01798fad0a8344a642eb5d46231fd78c2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 27 May 2015 21:25:07 +0600
Subject: [PATCH 1414/6219] [facebook] Improve title regex (Closes #5816)

---
 youtube_dl/extractor/facebook.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index e8d682716..82dc27bc6 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -152,7 +152,7 @@ class FacebookIE(InfoExtractor):
             raise ExtractorError('Cannot find video formats')
 
         video_title = self._html_search_regex(
-            r'<h2 class="uiHeaderTitle">([^<]*)</h2>', webpage, 'title',
+            r'<h2\s+[^>]*class="uiHeaderTitle"[^>]*>([^<]*)</h2>', webpage, 'title',
             default=None)
         if not video_title:
             video_title = self._html_search_regex(

From d90b3854ca9e8602f440cc9439e1cba240192286 Mon Sep 17 00:00:00 2001
From: PeterDing <dfhayst@gmail.com>
Date: Thu, 28 May 2015 00:37:00 +0800
Subject: [PATCH 1415/6219] [porn91] Add new extractor for 91porn.com

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/porn91.py   | 62 ++++++++++++++++++++++++++++++++
 2 files changed, 63 insertions(+)
 create mode 100644 youtube_dl/extractor/porn91.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 80c9cb107..d20ad286d 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -400,6 +400,7 @@ from .playfm import PlayFMIE
 from .playvid import PlayvidIE
 from .playwire import PlaywireIE
 from .podomatic import PodomaticIE
+from .porn91 import Porn91IE
 from .pornhd import PornHdIE
 from .pornhub import (
     PornHubIE,
diff --git a/youtube_dl/extractor/porn91.py b/youtube_dl/extractor/porn91.py
new file mode 100644
index 000000000..af06af2b7
--- /dev/null
+++ b/youtube_dl/extractor/porn91.py
@@ -0,0 +1,62 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+import re
+import json
+
+from ..compat import compat_urllib_parse
+from .common import InfoExtractor
+from ..utils import ExtractorError
+
+
+class Porn91IE(InfoExtractor):
+    IE_NAME = '91porn'
+    _VALID_URL = r'(?:https?://)(?:www\.|)91porn\.com/.+?\?viewkey=(?P<id>[\w\d]+)'
+
+    _TEST = {
+            'url': 'http://91porn.com/view_video.php?viewkey=7e42283b4f5ab36da134',
+            'md5': '6df8f6d028bc8b14f5dbd73af742fb20',
+            'info_dict': {
+                'id': '7e42283b4f5ab36da134',
+                'title': '18岁大一漂亮学妹，水嫩性感，再爽一次！',
+                'ext': 'mp4'
+            }
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        url = 'http://91porn.com/view_video.php?viewkey=%s' % video_id
+        self._set_cookie('91porn.com', 'language', 'cn_CN')
+        webpage = self._download_webpage(url, video_id, "get HTML content")
+        title = re.search(
+            r'<div id="viewvideo-title">(.+?)</div>',
+            webpage,
+            re.DOTALL)
+        assert title
+        title = title.group(1).replace('\n', '')
+
+        # get real url
+        n1 = re.search(r'so.addVariable\(\'file\',\'(\d+)\'', webpage)
+        n2 = re.search(r'so.addVariable\(\'seccode\',\'(.+?)\'', webpage)
+        n3 = re.search(r'so.addVariable\(\'max_vid\',\'(\d+)\'', webpage)
+        if not (n1 and n2 and n3):
+            raise ExtractorError("You are Blocked by Server.")
+
+        url_params = compat_urllib_parse.urlencode({
+            'VID': n1.group(1),
+            'mp4': '1',
+            'seccode': n2.group(1),
+            'max_vid': n3.group(1),
+        })
+        t_url = 'http://91porn.com/getfile.php?' + url_params
+        info_cn = self._download_webpage(t_url, video_id, "get real video_url")
+        video_url = re.search(r'file=(http.+?)&', info_cn).group(1)
+
+        info = {
+            'id': video_id,
+            'title': title,
+            'url': video_url,
+        }
+
+        return info

From b25b645d5106e5b2bf33c640813fe744b63f4730 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 27 May 2015 23:20:32 +0600
Subject: [PATCH 1416/6219] [nowtv] Improve and simplify

---
 youtube_dl/extractor/__init__.py |   2 +-
 youtube_dl/extractor/nowtv.py    | 197 ++++++++++++++++++++++---------
 2 files changed, 139 insertions(+), 60 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index bfd07392e..17248ccea 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -355,7 +355,7 @@ from .normalboots import NormalbootsIE
 from .nosvideo import NosVideoIE
 from .novamov import NovaMovIE
 from .nowness import NownessIE
-from .nowtv import NowTvIE
+from .nowtv import NowTVIE
 from .nowvideo import NowVideoIE
 from .npo import (
     NPOIE,
diff --git a/youtube_dl/extractor/nowtv.py b/youtube_dl/extractor/nowtv.py
index bf97fe7f4..5c91acec6 100644
--- a/youtube_dl/extractor/nowtv.py
+++ b/youtube_dl/extractor/nowtv.py
@@ -1,90 +1,169 @@
-# encoding: utf-8
+# coding: utf-8
 from __future__ import unicode_literals
 
 import re
 
 from .common import InfoExtractor
+from ..compat import compat_str
 from ..utils import (
     ExtractorError,
-    qualities,
-    unified_strdate,
     int_or_none,
+    parse_iso8601,
+    parse_duration,
+    remove_start,
 )
 
-class NowTvIE(InfoExtractor):
-    """Information Extractor for RTL NOW, RTL2 NOW, RTL NITRO, SUPER RTL NOW, VOX NOW and n-tv NOW"""
-    _VALID_URL = r'''(?x)
-                        (?:https?://)?
-                        (
-                            (?:www\.)?nowtv\.de
-                            /(rtl|rtl2|rtlnitro||superrtl|ntv|vox)(?P<path>/.*?)/player
-                        )'''
 
-    _TESTS = [
-        {
-            'url': 'http://www.nowtv.de/vox/der-hundeprofi/buero-fall-chihuahua-joel/player',
-            'info_dict': {
-                'id': '128953',
-                'ext': 'mp4',
-                'title': 'B\u00fcro-Fall \/ Chihuahua \'Joel\'',
-                'description': 'md5:ce843b6b5901d9a7f7d04d1bbcdb12de',
-                'upload_date': '2015-05-23 19:10:00',
-                'duration': '00:51:32',
-            },
-            'params': {
-                'skip_download': True,
-            },
-            'skip': 'Only works from Germany',
+class NowTVIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?nowtv\.de/(?P<station>rtl|rtl2|rtlnitro|superrtl|ntv|vox)/(?P<id>.+?)/player'
+
+    _TESTS = [{
+        # rtl
+        'url': 'http://www.nowtv.de/rtl/bauer-sucht-frau/die-neuen-bauern-und-eine-hochzeit/player',
+        'info_dict': {
+            'id': '203519',
+            'display_id': 'bauer-sucht-frau/die-neuen-bauern-und-eine-hochzeit',
+            'ext': 'mp4',
+            'title': 'Die neuen Bauern und eine Hochzeit',
+            'description': 'md5:e234e1ed6d63cf06be5c070442612e7e',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'timestamp': 1432580700,
+            'upload_date': '20150525',
+            'duration': 2786,
         },
-    ]
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }, {
+        # rtl2
+        'url': 'http://www.nowtv.de/rtl2/berlin-tag-nacht/berlin-tag-nacht-folge-934/player',
+        'info_dict': {
+            'id': '203481',
+            'display_id': 'berlin-tag-nacht/berlin-tag-nacht-folge-934',
+            'ext': 'mp4',
+            'title': 'Berlin - Tag & Nacht (Folge 934)',
+            'description': 'md5:c85e88c2e36c552dfe63433bc9506dd0',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'timestamp': 1432666800,
+            'upload_date': '20150526',
+            'duration': 2641,
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }, {
+        # superrtl
+        'url': 'http://www.nowtv.de/superrtl/medicopter-117/angst/player',
+        'info_dict': {
+            'id': '99205',
+            'display_id': 'medicopter-117/angst',
+            'ext': 'mp4',
+            'title': 'Angst!',
+            'description': 'md5:30cbc4c0b73ec98bcd73c9f2a8c17c4e',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'timestamp': 1222632900,
+            'upload_date': '20080928',
+            'duration': 3025,
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }, {
+        # ntv
+        'url': 'http://www.nowtv.de/ntv/ratgeber-geld/thema-ua-der-erste-blick-die-apple-watch/player',
+        'info_dict': {
+            'id': '203521',
+            'display_id': 'ratgeber-geld/thema-ua-der-erste-blick-die-apple-watch',
+            'ext': 'mp4',
+            'title': 'Thema u.a.: Der erste Blick: Die Apple Watch',
+            'description': 'md5:4312b6c9d839ffe7d8caf03865a531af',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'timestamp': 1432751700,
+            'upload_date': '20150527',
+            'duration': 1083,
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }, {
+        # vox
+        'url': 'http://www.nowtv.de/vox/der-hundeprofi/buero-fall-chihuahua-joel/player',
+        'info_dict': {
+            'id': '128953',
+            'display_id': 'der-hundeprofi/buero-fall-chihuahua-joel',
+            'ext': 'mp4',
+            'title': "Büro-Fall / Chihuahua 'Joel'",
+            'description': 'md5:e62cb6bf7c3cc669179d4f1eb279ad8d',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'timestamp': 1432408200,
+            'upload_date': '20150523',
+            'duration': 3092,
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        info_url = 'https://api.nowtv.de/v3/movies' + mobj.group('path') + '?fields=*,format,files,breakpoints,paymentPaytypes,trailers'
-        info = self._download_json(info_url, None)
+        display_id = mobj.group('id')
+        station = mobj.group('station')
 
-        video_id = info['id']
-        title = info['title']
-        description = info['articleShort']
-        duration = info['duration']
-        upload_date = unified_strdate(info['broadcastStartDate'])
-        free = info['free']
-        station = info['format']['station']
-        thumbnail = info['format']['defaultImage169Logo']
+        info = self._download_json(
+            'https://api.nowtv.de/v3/movies/%s?fields=*,format,files,breakpoints,paymentPaytypes,trailers,pictures' % display_id,
+            display_id)
 
-        if station == 'rtl':
-            base_url = 'http://hls.fra.rtlnow.de/hls-vod-enc/'
-        elif station == 'rtl2':
-            base_url = 'http://hls.fra.rtl2now.de/hls-vod-enc/'
-        elif station == 'vox':
-            base_url = 'http://hls.fra.voxnow.de/hls-vod-enc/'
-        elif station == 'nitro':
-            base_url = 'http://hls.fra.rtlnitronow.de/hls-vod-enc/'
-        elif station == 'ntv':
-            base_url = 'http://hls.fra.n-tvnow.de/hls-vod-enc/'
-        elif station == 'superrtl':
-            base_url = 'http://hls.fra.superrtlnow.de/hls-vod-enc/'
+        video_id = compat_str(info['id'])
+
+        if info.get('geoblocked'):
+            raise ExtractorError(
+                'Video %s is not available from your location due to geo restriction' % video_id,
+                expected=True)
+
+        f = info.get('format', {})
+        station = f.get('station') or station
+
+        STATIONS = {
+            'rtl': 'rtlnow',
+            'rtl2': 'rtl2now',
+            'vox': 'voxnow',
+            'nitro': 'rtlnitronow',
+            'ntv': 'n-tvnow',
+            'superrtl': 'superrtlnow'
+        }
 
         formats = []
         for item in info['files']['items']:
-            if item['type'] != 'video/x-abr':
-                continue
-
-            fmt = {
-                'url': base_url + item['path'] + '.m3u8',
-                'tbr': int_or_none(item['bitrate']),
+            item_path = remove_start(item['path'], '/')
+            tbr = int_or_none(item['bitrate'])
+            m3u8_url = 'http://hls.fra.%s.de/hls-vod-enc/%s.m3u8' % (STATIONS[station], item_path)
+            m3u8_url = m3u8_url.replace('now/', 'now/videos/')
+            formats.append({
+                'url': m3u8_url,
+                'format_id': '%s-%sk' % (item['id'], tbr),
                 'ext': 'mp4',
-                'format_id': int_or_none(item['id']),
-            }
-            formats.append(fmt)
+                'tbr': tbr,
+            })
         self._sort_formats(formats)
 
+        title = info['title']
+        description = info.get('articleLong') or info.get('articleShort')
+        timestamp = parse_iso8601(info.get('broadcastStartDate'), ' ')
+        duration = parse_duration(info.get('duration'))
+        thumbnail = f.get('defaultImage169Format') or f.get('defaultImage169Logo')
+
         return {
             'id': video_id,
+            'display_id': display_id,
             'title': title,
             'description': description,
             'thumbnail': thumbnail,
-            'upload_date': upload_date,
+            'timestamp': timestamp,
             'duration': duration,
             'formats': formats,
         }

From 703d78bbf5edf73f60447ac273c0d303d28cc340 Mon Sep 17 00:00:00 2001
From: PeterDing <dfhayst@gmail.com>
Date: Thu, 28 May 2015 01:37:24 +0800
Subject: [PATCH 1417/6219] [porn91] change re to _search_regex

---
 youtube_dl/extractor/porn91.py | 29 +++++++++++++++--------------
 1 file changed, 15 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/porn91.py b/youtube_dl/extractor/porn91.py
index af06af2b7..f3a97df64 100644
--- a/youtube_dl/extractor/porn91.py
+++ b/youtube_dl/extractor/porn91.py
@@ -2,7 +2,6 @@
 from __future__ import unicode_literals
 
 import re
-import json
 
 from ..compat import compat_urllib_parse
 from .common import InfoExtractor
@@ -29,30 +28,32 @@ class Porn91IE(InfoExtractor):
         url = 'http://91porn.com/view_video.php?viewkey=%s' % video_id
         self._set_cookie('91porn.com', 'language', 'cn_CN')
         webpage = self._download_webpage(url, video_id, "get HTML content")
-        title = re.search(
-            r'<div id="viewvideo-title">(.+?)</div>',
-            webpage,
-            re.DOTALL)
+        title = self._search_regex(
+            r'<div id="viewvideo-title">(?P<title>.+?)</div>',
+            webpage, 'title', flags=re.DOTALL)
         assert title
-        title = title.group(1).replace('\n', '')
+        title = title.replace('\n', '')
 
         # get real url
-        n1 = re.search(r'so.addVariable\(\'file\',\'(\d+)\'', webpage)
-        n2 = re.search(r'so.addVariable\(\'seccode\',\'(.+?)\'', webpage)
-        n3 = re.search(r'so.addVariable\(\'max_vid\',\'(\d+)\'', webpage)
+        n1 = self._search_regex(
+            r'so.addVariable\(\'file\',\'(?P<n1>\d+)\'', webpage, 'n1')
+        n2 = self._search_regex(
+            r'so.addVariable\(\'seccode\',\'(?P<n2>.+?)\'', webpage, 'n2')
+        n3 = self._search_regex(
+            r'so.addVariable\(\'max_vid\',\'(?P<n3>\d+)\'', webpage, 'n3')
         if not (n1 and n2 and n3):
             raise ExtractorError("You are Blocked by Server.")
-
         url_params = compat_urllib_parse.urlencode({
-            'VID': n1.group(1),
+            'VID': n1,
             'mp4': '1',
-            'seccode': n2.group(1),
-            'max_vid': n3.group(1),
+            'seccode': n2,
+            'max_vid': n3,
         })
         t_url = 'http://91porn.com/getfile.php?' + url_params
         info_cn = self._download_webpage(t_url, video_id, "get real video_url")
-        video_url = re.search(r'file=(http.+?)&', info_cn).group(1)
+        video_url = self._search_regex(r'file=(?P<url>http.+?)&', info_cn, 'url')
 
+        # construct info
         info = {
             'id': video_id,
             'title': title,

From 9b254aa177d58b7b4c4f44dce8c38fa7978c7df6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 27 May 2015 23:41:43 +0600
Subject: [PATCH 1418/6219] [nowtv] Add non-free video check

---
 youtube_dl/extractor/nowtv.py | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/nowtv.py b/youtube_dl/extractor/nowtv.py
index 5c91acec6..295168432 100644
--- a/youtube_dl/extractor/nowtv.py
+++ b/youtube_dl/extractor/nowtv.py
@@ -120,10 +120,15 @@ class NowTVIE(InfoExtractor):
 
         video_id = compat_str(info['id'])
 
-        if info.get('geoblocked'):
-            raise ExtractorError(
-                'Video %s is not available from your location due to geo restriction' % video_id,
-                expected=True)
+        files = info['files']
+        if not files:
+            if info.get('geoblocked', False):
+                raise ExtractorError(
+                    'Video %s is not available from your location due to geo restriction' % video_id,
+                    expected=True)
+            if not info.get('free', True):
+                raise ExtractorError(
+                    'Video %s is not available for free' % video_id, expected=True)
 
         f = info.get('format', {})
         station = f.get('station') or station
@@ -138,7 +143,7 @@ class NowTVIE(InfoExtractor):
         }
 
         formats = []
-        for item in info['files']['items']:
+        for item in files['items']:
             item_path = remove_start(item['path'], '/')
             tbr = int_or_none(item['bitrate'])
             m3u8_url = 'http://hls.fra.%s.de/hls-vod-enc/%s.m3u8' % (STATIONS[station], item_path)

From ff4a1279f2d40fdba3287d4e7949bd8caa89eb04 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 28 May 2015 01:15:04 +0600
Subject: [PATCH 1419/6219] [nowtv] Do not request unnecessary metadata

---
 youtube_dl/extractor/nowtv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nowtv.py b/youtube_dl/extractor/nowtv.py
index 295168432..d39bbde99 100644
--- a/youtube_dl/extractor/nowtv.py
+++ b/youtube_dl/extractor/nowtv.py
@@ -115,7 +115,7 @@ class NowTVIE(InfoExtractor):
         station = mobj.group('station')
 
         info = self._download_json(
-            'https://api.nowtv.de/v3/movies/%s?fields=*,format,files,breakpoints,paymentPaytypes,trailers,pictures' % display_id,
+            'https://api.nowtv.de/v3/movies/%s?fields=*,format,files' % display_id,
             display_id)
 
         video_id = compat_str(info['id'])

From 9e0b5791281c68e5773555688928184064396011 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 28 May 2015 01:26:14 +0600
Subject: [PATCH 1420/6219] [nowtv] Add test for rtlnitro

---
 youtube_dl/extractor/nowtv.py | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/youtube_dl/extractor/nowtv.py b/youtube_dl/extractor/nowtv.py
index d39bbde99..173e46cd8 100644
--- a/youtube_dl/extractor/nowtv.py
+++ b/youtube_dl/extractor/nowtv.py
@@ -53,6 +53,24 @@ class NowTVIE(InfoExtractor):
             # m3u8 download
             'skip_download': True,
         },
+    }, {
+        # rtlnitro
+        'url': 'http://www.nowtv.de/rtlnitro/alarm-fuer-cobra-11-die-autobahnpolizei/hals-und-beinbruch-2014-08-23-21-10-00/player',
+        'info_dict': {
+            'id': '165780',
+            'display_id': 'alarm-fuer-cobra-11-die-autobahnpolizei/hals-und-beinbruch-2014-08-23-21-10-00',
+            'ext': 'mp4',
+            'title': 'Hals- und Beinbruch',
+            'description': 'md5:b50d248efffe244e6f56737f0911ca57',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'timestamp': 1432415400,
+            'upload_date': '20150523',
+            'duration': 2742,
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
     }, {
         # superrtl
         'url': 'http://www.nowtv.de/superrtl/medicopter-117/angst/player',

From f9355dc989362f31d8e21ccb8fa765546a2360f2 Mon Sep 17 00:00:00 2001
From: PeterDing <dfhayst@gmail.com>
Date: Thu, 28 May 2015 17:00:09 +0800
Subject: [PATCH 1421/6219] [youku] update youku

---
 youtube_dl/extractor/youku.py | 248 ++++++++++++++++++++++------------
 1 file changed, 162 insertions(+), 86 deletions(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index 97b98bbe8..8d86c3f45 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -1,123 +1,199 @@
 # coding: utf-8
-
 from __future__ import unicode_literals
 
-import math
-import random
 import re
-import time
+import base64
 
 from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
-)
-
+from ..utils import ExtractorError
 
 class YoukuIE(InfoExtractor):
+    IE_NAME = 'youku'
     _VALID_URL = r'''(?x)
         (?:
             http://(?:v|player)\.youku\.com/(?:v_show/id_|player\.php/sid/)|
             youku:)
         (?P<id>[A-Za-z0-9]+)(?:\.html|/v\.swf|)
     '''
+
     _TEST = {
-        'url': 'http://v.youku.com/v_show/id_XNDgyMDQ2NTQw.html',
-        'md5': 'ffe3f2e435663dc2d1eea34faeff5b5b',
-        'params': {
-            'test': False
-        },
-        'info_dict': {
-            'id': 'XNDgyMDQ2NTQw_part00',
-            'ext': 'flv',
-            'title': 'youtube-dl test video "\'/\\ä↭𝕐'
-        }
+            'url': 'http://v.youku.com/v_show/id_XMTc1ODE5Njcy.html',
+            'md5': '5f3af4192eabacc4501508d54a8cabd7',
+            'info_dict': {
+                'id': 'XMTc1ODE5Njcy',
+                'title': '★Smile﹗♡ Git Fresh -Booty Music舞蹈.',
+                'ext': 'flv'
+            }
     }
 
-    def _gen_sid(self):
-        nowTime = int(time.time() * 1000)
-        random1 = random.randint(1000, 1998)
-        random2 = random.randint(1000, 9999)
+    def construct_video_urls(self, data1, data2):
+        # get sid, token
+        def yk_t(s1, s2):
+            ls = list(range(256))
+            t = 0
+            for i in range(256):
+                t = (t + ls[i] + ord(s1[i%len(s1)])) % 256
+                ls[i], ls[t] = ls[t], ls[i]
+            s, x, y = '', 0, 0
+            for i in range(len(s2)):
+                y = (y + 1) % 256
+                x = (x + ls[y]) % 256
+                ls[x], ls[y] = ls[y], ls[x]
+                s += chr((s2[i] ^ ls[(ls[x]+ls[y]) % 256]))
+            return s
 
-        return "%d%d%d" % (nowTime, random1, random2)
+        sid, token = yk_t(
+            'becaf9be', base64.b64decode(bytes(data2['ep'], 'ascii'))
+        ).split('_')
 
-    def _get_file_ID_mix_string(self, seed):
-        mixed = []
-        source = list("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ/\:._-1234567890")
-        seed = float(seed)
-        for i in range(len(source)):
-            seed = (seed * 211 + 30031) % 65536
-            index = math.floor(seed / 65536 * len(source))
-            mixed.append(source[int(index)])
-            source.remove(source[int(index)])
-        # return ''.join(mixed)
-        return mixed
+        # get oip
+        oip = data2['ip']
 
-    def _get_file_id(self, fileId, seed):
-        mixed = self._get_file_ID_mix_string(seed)
-        ids = fileId.split('*')
-        realId = []
-        for ch in ids:
-            if ch:
-                realId.append(mixed[int(ch)])
-        return ''.join(realId)
+        # get fileid
+        string_ls = list(
+            'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ/\:._-1234567890')
+        shuffled_string_ls = []
+        seed = data1['seed']
+        N = len(string_ls)
+        for ii in range(N):
+            seed = (seed * 0xd3 + 0x754f) % 0x10000
+            idx = seed * len(string_ls) // 0x10000
+            shuffled_string_ls.append(string_ls[idx])
+            del string_ls[idx]
+
+        fileid_dict = {}
+        for format in data1['streamtypes']:
+            streamfileid = [
+                int(i) for i in data1['streamfileids'][format].strip('*').split('*')]
+            fileid = ''.join(
+                [shuffled_string_ls[i] for i in streamfileid])
+            fileid_dict[format] = fileid[:8] + '%s' + fileid[10:]
+
+        def get_fileid(format, n):
+            fileid = fileid_dict[format] % hex(int(n))[2:].upper().zfill(2)
+            return fileid
+
+        # get ep
+        def generate_ep(format, n):
+            fileid = get_fileid(format, n)
+            ep_t = yk_t(
+                'bf7e5f01',
+                bytes('%s_%s_%s' % (sid, fileid, token), 'ascii'))
+            ep = base64.b64encode(bytes(ep_t, 'latin')).decode()
+            ep = ep.replace('+', '%2B')
+            ep = ep.replace('/', '%2F')
+            ep = ep.replace('=', '%2D')
+            return ep
+
+        # generate video_urls
+        video_urls_dict = {}
+        for format in data1['streamtypes']:
+            video_urls = []
+            for dt in data1['segs'][format]:
+                n = str(int(dt['no']))
+                video_url = \
+                    'http://k.youku.com/player/getFlvPath/' + \
+                    'sid/' + sid + \
+                    '_' + str(int(n)+1).zfill(2) + \
+                    '/st/' + self.parse_ext_l(format) + \
+                    '/fileid/' + get_fileid(format, n)  + '?' + \
+                    'K=' + str(dt['k']) + \
+                    '&hd=' + self.get_hd(format) + \
+                    '&myp=0' + \
+                    '&ts=' + str(dt['seconds']) + \
+                    '&ypp=0&ctype=12&ev=1' + \
+                    '&token=' + str(token) + \
+                    '&oip=' + str(oip) + \
+                    '&ep=' + generate_ep(format, n)
+                video_urls.append(video_url)
+            video_urls_dict[format] = video_urls
+
+        return video_urls_dict
+
+    def get_hd(self, fm):
+        hd_id_dict = {
+            'flv': '0',
+            'mp4': '1',
+            'hd2': '2',
+            'hd3': '3',
+            '3gp': '0',
+            '3gphd': '1'
+        }
+        return hd_id_dict[fm]
+
+    def parse_ext_l(self, fm):
+        ext_dict = {
+            'flv': 'flv',
+            'mp4': 'mp4',
+            'hd2': 'flv',
+            'hd3': 'flv',
+            '3gp': 'flv',
+            '3gphd': 'mp4',
+        }
+        return ext_dict[fm]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
 
-        info_url = 'http://v.youku.com/player/getPlayList/VideoIDS/' + video_id
+        # request basic data
+        data1_url = 'http://v.youku.com/player/getPlayList/VideoIDS/%s' % video_id
+        data2_url = 'http://v.youku.com/player/getPlayList/VideoIDS/%s/Pf/4/ctype/12/ev/1' % video_id
 
-        config = self._download_json(info_url, video_id)
+        raw_data1 = self._download_json(data1_url, video_id)
+        raw_data2 = self._download_json(data2_url, video_id)
+        data1 = raw_data1['data'][0]
+        data2 = raw_data2['data'][0]
 
-        error_code = config['data'][0].get('error_code')
+        error_code = data1.get('error_code')
         if error_code:
             # -8 means blocked outside China.
-            error = config['data'][0].get('error')  # Chinese and English, separated by newline.
-            raise ExtractorError(error or 'Server reported error %i' % error_code,
-                                 expected=True)
+            # Chinese and English, separated by newline.
+            error = data1.get('error')
+            raise ExtractorError(
+                error or 'Server reported error %i' %
+                error_code,
+                expected=True)
 
-        video_title = config['data'][0]['title']
-        seed = config['data'][0]['seed']
+        title = data1['title']
 
-        format = self._downloader.params.get('format', None)
-        supported_format = list(config['data'][0]['streamfileids'].keys())
+        # generate video_urls_dict
+        video_urls_dict = self.construct_video_urls(data1, data2)
 
-        # TODO proper format selection
-        if format is None or format == 'best':
-            if 'hd2' in supported_format:
-                format = 'hd2'
-            else:
-                format = 'flv'
-            ext = 'flv'
-        elif format == 'worst':
-            format = 'mp4'
-            ext = 'mp4'
-        else:
-            format = 'flv'
-            ext = 'flv'
+        # construct info
+        entries = []
+        for fm in data1['streamtypes']:
+            #formats = []
+            video_urls = video_urls_dict[fm]
+            for i in range(len(video_urls)):
+                if len(entries) < i+1:
+                    entries.append({'formats': []})
+                entries[i]['formats'].append(
+                    {
+                        'url': video_urls[i],
+                        'format_id': fm,
+                        'ext': self.parse_ext_l(fm),
+                        'filesize': int(data1['segs'][fm][i]['size'])
+                    }
+                )
 
-        fileid = config['data'][0]['streamfileids'][format]
-        keys = [s['k'] for s in config['data'][0]['segs'][format]]
-        # segs is usually a dictionary, but an empty *list* if an error occured.
-
-        files_info = []
-        sid = self._gen_sid()
-        fileid = self._get_file_id(fileid, seed)
-
-        # column 8,9 of fileid represent the segment number
-        # fileid[7:9] should be changed
-        for index, key in enumerate(keys):
-            temp_fileid = '%s%02X%s' % (fileid[0:8], index, fileid[10:])
-            download_url = 'http://k.youku.com/player/getFlvPath/sid/%s_%02X/st/flv/fileid/%s?k=%s' % (sid, index, temp_fileid, key)
+        for i in range(len(entries)):
+            entries[i].update(
+                {
+                    'id': '_part%d' % (i+1),
+                    'title': title,
+                }
+            )
 
+        if len(entries) > 1:
             info = {
-                'id': '%s_part%02d' % (video_id, index),
-                'url': download_url,
-                'uploader': None,
-                'upload_date': None,
-                'title': video_title,
-                'ext': ext,
+                '_type': 'multi_video',
+                'id': video_id,
+                'title': title,
+                'entries': entries,
             }
-            files_info.append(info)
+        else:
+            info = entries[0]
+            info['id'] = video_id
 
-        return files_info
+        return info

From ca45246627f5a67a7c82cd40a11e5c4ff5f68871 Mon Sep 17 00:00:00 2001
From: PeterDing <dfhayst@gmail.com>
Date: Thu, 28 May 2015 21:04:58 +0800
Subject: [PATCH 1422/6219] [youku] compatible for python > 3.3 or > 2.7

---
 youtube_dl/extractor/youku.py | 26 +++++++++++++++++++++-----
 1 file changed, 21 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index 8d86c3f45..7a07c8a5f 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -1,6 +1,8 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import sys
+pyvs = sys.version_info[0]
 import re
 import base64
 
@@ -34,16 +36,23 @@ class YoukuIE(InfoExtractor):
             for i in range(256):
                 t = (t + ls[i] + ord(s1[i%len(s1)])) % 256
                 ls[i], ls[t] = ls[t], ls[i]
-            s, x, y = '', 0, 0
+            s = '' if pyvs == 3 else b''
+            x, y = 0, 0
             for i in range(len(s2)):
                 y = (y + 1) % 256
                 x = (x + ls[y]) % 256
                 ls[x], ls[y] = ls[y], ls[x]
-                s += chr((s2[i] ^ ls[(ls[x]+ls[y]) % 256]))
+                if isinstance(s2[i], int):
+                    s += chr(s2[i] ^ ls[(ls[x]+ls[y]) % 256])
+                else:
+                    s += chr(ord(s2[i]) ^ ls[(ls[x]+ls[y]) % 256])
             return s
 
         sid, token = yk_t(
-            'becaf9be', base64.b64decode(bytes(data2['ep'], 'ascii'))
+            'becaf9be',
+            base64.b64decode(bytes(data2['ep'], 'ascii')) \
+                if pyvs == 3 \
+                else base64.b64decode(data2['ep'])
         ).split('_')
 
         # get oip
@@ -78,8 +87,15 @@ class YoukuIE(InfoExtractor):
             fileid = get_fileid(format, n)
             ep_t = yk_t(
                 'bf7e5f01',
-                bytes('%s_%s_%s' % (sid, fileid, token), 'ascii'))
-            ep = base64.b64encode(bytes(ep_t, 'latin')).decode()
+                bytes('%s_%s_%s' % (sid, fileid, token), 'ascii') \
+                if pyvs == 3 \
+                else ('%s_%s_%s' % (sid, fileid, token))
+            )
+            ep = base64.b64encode(
+                bytes(ep_t, 'latin') \
+                if pyvs == 3 \
+                else ep_t
+            ).decode()
             ep = ep.replace('+', '%2B')
             ep = ep.replace('/', '%2F')
             ep = ep.replace('=', '%2D')

From 806598b94dec1268566ae71d671116060f7971d6 Mon Sep 17 00:00:00 2001
From: PeterDing <dfhayst@gmail.com>
Date: Fri, 29 May 2015 08:21:24 +0800
Subject: [PATCH 1423/6219] [porn91] the one that _search_regex returns not
 needs to be checked

---
 youtube_dl/extractor/porn91.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/youtube_dl/extractor/porn91.py b/youtube_dl/extractor/porn91.py
index f3a97df64..b62eec92d 100644
--- a/youtube_dl/extractor/porn91.py
+++ b/youtube_dl/extractor/porn91.py
@@ -31,7 +31,6 @@ class Porn91IE(InfoExtractor):
         title = self._search_regex(
             r'<div id="viewvideo-title">(?P<title>.+?)</div>',
             webpage, 'title', flags=re.DOTALL)
-        assert title
         title = title.replace('\n', '')
 
         # get real url
@@ -41,8 +40,6 @@ class Porn91IE(InfoExtractor):
             r'so.addVariable\(\'seccode\',\'(?P<n2>.+?)\'', webpage, 'n2')
         n3 = self._search_regex(
             r'so.addVariable\(\'max_vid\',\'(?P<n3>\d+)\'', webpage, 'n3')
-        if not (n1 and n2 and n3):
-            raise ExtractorError("You are Blocked by Server.")
         url_params = compat_urllib_parse.urlencode({
             'VID': n1,
             'mp4': '1',

From 1498940b10a3f43490c05045ebe7a517267a2bff Mon Sep 17 00:00:00 2001
From: PeterDing <dfhayst@gmail.com>
Date: Fri, 29 May 2015 10:13:09 +0800
Subject: [PATCH 1424/6219] [youku] compare bytes and str for compatible; use
 compat_urllib_parse for making video_url

---
 youtube_dl/extractor/youku.py | 38 +++++++++++++++++++----------------
 1 file changed, 21 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index 7a07c8a5f..063f2e10e 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -1,14 +1,16 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import sys
-pyvs = sys.version_info[0]
 import re
 import base64
 
 from .common import InfoExtractor
 from ..utils import ExtractorError
 
+from ..compat import compat_urllib_parse
+
+bytes_is_str = (bytes == str)  # for compatible
+
 class YoukuIE(InfoExtractor):
     IE_NAME = 'youku'
     _VALID_URL = r'''(?x)
@@ -36,7 +38,7 @@ class YoukuIE(InfoExtractor):
             for i in range(256):
                 t = (t + ls[i] + ord(s1[i%len(s1)])) % 256
                 ls[i], ls[t] = ls[t], ls[i]
-            s = '' if pyvs == 3 else b''
+            s = '' if not bytes_is_str else b''
             x, y = 0, 0
             for i in range(len(s2)):
                 y = (y + 1) % 256
@@ -51,7 +53,7 @@ class YoukuIE(InfoExtractor):
         sid, token = yk_t(
             'becaf9be',
             base64.b64decode(bytes(data2['ep'], 'ascii')) \
-                if pyvs == 3 \
+                if not bytes_is_str \
                 else base64.b64decode(data2['ep'])
         ).split('_')
 
@@ -88,17 +90,14 @@ class YoukuIE(InfoExtractor):
             ep_t = yk_t(
                 'bf7e5f01',
                 bytes('%s_%s_%s' % (sid, fileid, token), 'ascii') \
-                if pyvs == 3 \
+                if not bytes_is_str \
                 else ('%s_%s_%s' % (sid, fileid, token))
             )
             ep = base64.b64encode(
                 bytes(ep_t, 'latin') \
-                if pyvs == 3 \
+                if not bytes_is_str \
                 else ep_t
             ).decode()
-            ep = ep.replace('+', '%2B')
-            ep = ep.replace('/', '%2F')
-            ep = ep.replace('=', '%2D')
             return ep
 
         # generate video_urls
@@ -107,20 +106,25 @@ class YoukuIE(InfoExtractor):
             video_urls = []
             for dt in data1['segs'][format]:
                 n = str(int(dt['no']))
+                param = {
+                    'K': dt['k'],
+                    'hd': self.get_hd(format),
+                    'myp': 0,
+                    'ts': dt['seconds'],
+                    'ypp': 0,
+                    'ctype': 12,
+                    'ev': 1,
+                    'token': token,
+                    'oip': oip,
+                    'ep': generate_ep(format, n)
+                }
                 video_url = \
                     'http://k.youku.com/player/getFlvPath/' + \
                     'sid/' + sid + \
                     '_' + str(int(n)+1).zfill(2) + \
                     '/st/' + self.parse_ext_l(format) + \
                     '/fileid/' + get_fileid(format, n)  + '?' + \
-                    'K=' + str(dt['k']) + \
-                    '&hd=' + self.get_hd(format) + \
-                    '&myp=0' + \
-                    '&ts=' + str(dt['seconds']) + \
-                    '&ypp=0&ctype=12&ev=1' + \
-                    '&token=' + str(token) + \
-                    '&oip=' + str(oip) + \
-                    '&ep=' + generate_ep(format, n)
+                    compat_urllib_parse.urlencode(param)
                 video_urls.append(video_url)
             video_urls_dict[format] = video_urls
 

From 84e1e036c2cb7311cdea14763bec3322403a8d54 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 29 May 2015 12:44:31 +0800
Subject: [PATCH 1425/6219] [senate] Extend _VALID_URL (fixes #5836)

---
 youtube_dl/extractor/senateisvp.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/senateisvp.py b/youtube_dl/extractor/senateisvp.py
index d3b8a1be4..9c53704ea 100644
--- a/youtube_dl/extractor/senateisvp.py
+++ b/youtube_dl/extractor/senateisvp.py
@@ -48,7 +48,7 @@ class SenateISVPIE(InfoExtractor):
         ["arch", "", "http://ussenate-f.akamaihd.net/"]
     ]
     _IE_NAME = 'senate.gov'
-    _VALID_URL = r'http://www\.senate\.gov/isvp/\?(?P<qs>.+)'
+    _VALID_URL = r'http://www\.senate\.gov/isvp/?\?(?P<qs>.+)'
     _TESTS = [{
         'url': 'http://www.senate.gov/isvp/?comm=judiciary&type=live&stt=&filename=judiciary031715&auto_play=false&wmode=transparent&poster=http%3A%2F%2Fwww.judiciary.senate.gov%2Fthemes%2Fjudiciary%2Fimages%2Fvideo-poster-flash-fit.png',
         'info_dict': {
@@ -72,12 +72,16 @@ class SenateISVPIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Integrated Senate Video Player'
         }
+    }, {
+        # From http://www.c-span.org/video/?96791-1
+        'url': 'http://www.senate.gov/isvp?type=live&comm=banking&filename=banking012715',
+        'only_matching': True,
     }]
 
     @staticmethod
     def _search_iframe_url(webpage):
         mobj = re.search(
-            r"<iframe[^>]+src=['\"](?P<url>http://www\.senate\.gov/isvp/\?[^'\"]+)['\"]",
+            r"<iframe[^>]+src=['\"](?P<url>http://www\.senate\.gov/isvp/?\?[^'\"]+)['\"]",
             webpage)
         if mobj:
             return mobj.group('url')

From eb6cb9fbe934fe99a35af22065cf91063d416c12 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 29 May 2015 07:52:17 +0200
Subject: [PATCH 1426/6219] release 2015.05.29

---
 README.md              | 4 ++--
 docs/supportedsites.md | 9 ++++-----
 youtube_dl/version.py  | 2 +-
 3 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/README.md b/README.md
index e51bb5343..f3d83c89f 100644
--- a/README.md
+++ b/README.md
@@ -168,7 +168,7 @@ which means you can modify it, redistribute it or use it however you like.
     --no-progress                    Do not print progress bar
     --console-title                  Display progress in console titlebar
     -v, --verbose                    Print various debugging information
-    --dump-pages                     Print downloaded pages to debug problems (very verbose)
+    --dump-pages                     Print downloaded pages encoded using base64 to debug problems (very verbose)
     --write-pages                    Write downloaded intermediary pages to files in the current directory to debug problems
     --print-traffic                  Display sent and read HTTP traffic
     -C, --call-home                  Contact the youtube-dl server for debugging
@@ -220,7 +220,7 @@ which means you can modify it, redistribute it or use it however you like.
     --embed-thumbnail                Embed thumbnail in the audio as cover art
     --add-metadata                   Write metadata to the video file
     --metadata-from-title FORMAT     Parse additional metadata like song title / artist from the video title. The format syntax is the same as --output, the parsed
-                                     parameters replace existing values. Additional templates: %(album), %(artist). Example: --metadata-from-title "%(artist)s -
+                                     parameters replace existing values. Additional templates: %(album)s, %(artist)s. Example: --metadata-from-title "%(artist)s -
                                      %(title)s" matches a title like "Coldplay - Paradise"
     --xattrs                         Write metadata to the video file's xattrs (using dublin core and xdg standards)
     --fixup POLICY                   Automatically correct known faults of the file. One of never (do nothing), warn (only emit a warning), detect_or_warn(the default;
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index a4879bd9a..a421ae62b 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -26,8 +26,7 @@
  - **anitube.se**
  - **AnySex**
  - **Aparat**
- - **AppleDailyAnimationNews**
- - **AppleDailyRealtimeNews**
+ - **AppleDaily**
  - **AppleTrailers**
  - **archive.org**: archive.org videos
  - **ARD**
@@ -152,7 +151,6 @@
  - **fc2**
  - **fernsehkritik.tv**
  - **fernsehkritik.tv:postecke**
- - **Firedrive**
  - **Firstpost**
  - **Flickr**
  - **Folketinget**: Folketinget (ft.dk; Danish parliament)
@@ -230,6 +228,7 @@
  - **KanalPlay**: Kanal 5/9/11 Play
  - **Kankan**
  - **Karaoketv**
+ - **KarriereVideos**
  - **keek**
  - **KeezMovies**
  - **KhanAcademy**
@@ -322,6 +321,7 @@
  - **NosVideo**
  - **novamov**: NovaMov
  - **Nowness**
+ - **NowTV**
  - **nowvideo**: NowVideo
  - **npo.nl**
  - **npo.nl:live**
@@ -393,7 +393,6 @@
  - **Rte**
  - **rtl.nl**: rtl.nl and rtlxl.nl
  - **RTL2**
- - **RTLnow**
  - **RTP**
  - **RTS**: RTS.ch
  - **rtve.es:alacarta**: RTVE a la carta
@@ -431,7 +430,6 @@
  - **smotri:community**: Smotri.com community videos
  - **smotri:user**: Smotri.com user videos
  - **Snotr**
- - **Sockshare**
  - **Sohu**
  - **soundcloud**
  - **soundcloud:playlist**
@@ -564,6 +562,7 @@
  - **vier:videos**
  - **Viewster**
  - **viki**
+ - **viki:channel**
  - **vimeo**
  - **vimeo:album**
  - **vimeo:channel**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index b33385153..653710131 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.05.20'
+__version__ = '2015.05.29'

From d6aa68ce75ef4f4d27fbf7103edfda1f92ba70b2 Mon Sep 17 00:00:00 2001
From: pulpe <Pulpan3@gmail.com>
Date: Fri, 29 May 2015 12:47:20 +0200
Subject: [PATCH 1427/6219] [postprocessor/embedthumbnail] embed mp4 too (fixes
 #5840)

---
 youtube_dl/postprocessor/embedthumbnail.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/postprocessor/embedthumbnail.py b/youtube_dl/postprocessor/embedthumbnail.py
index 8f825f785..774494efd 100644
--- a/youtube_dl/postprocessor/embedthumbnail.py
+++ b/youtube_dl/postprocessor/embedthumbnail.py
@@ -49,7 +49,7 @@ class EmbedThumbnailPP(FFmpegPostProcessor):
             os.remove(encodeFilename(filename))
             os.rename(encodeFilename(temp_filename), encodeFilename(filename))
 
-        elif info['ext'] == 'm4a':
+        elif info['ext'] in ['m4a', 'mp4']:
             if not check_executable('AtomicParsley', ['-v']):
                 raise EmbedThumbnailPPError('AtomicParsley was not found. Please install.')
 
@@ -82,6 +82,6 @@ class EmbedThumbnailPP(FFmpegPostProcessor):
                 os.remove(encodeFilename(filename))
                 os.rename(encodeFilename(temp_filename), encodeFilename(filename))
         else:
-            raise EmbedThumbnailPPError('Only mp3 and m4a are supported for thumbnail embedding for now.')
+            raise EmbedThumbnailPPError('Only mp3 and m4a/mp4 are supported for thumbnail embedding for now.')
 
         return [], info

From 605ec701b7b4cd120a9acb33bfcc4306719b59b4 Mon Sep 17 00:00:00 2001
From: PeterDing <dfhayst@gmail.com>
Date: Fri, 29 May 2015 23:32:04 +0800
Subject: [PATCH 1428/6219] [iqiyi] Add new extractor for iqiyi.com

---
 youtube_dl/extractor/__init__.py |   1 +
 youtube_dl/extractor/iqiyi.py    | 214 +++++++++++++++++++++++++++++++
 2 files changed, 215 insertions(+)
 create mode 100644 youtube_dl/extractor/iqiyi.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 80c9cb107..85c1b1a3a 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -229,6 +229,7 @@ from .infoq import InfoQIE
 from .instagram import InstagramIE, InstagramUserIE
 from .internetvideoarchive import InternetVideoArchiveIE
 from .iprima import IPrimaIE
+from .iqiyi import IqiyiIE
 from .ivi import (
     IviIE,
     IviCompilationIE
diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
new file mode 100644
index 000000000..d96d13225
--- /dev/null
+++ b/youtube_dl/extractor/iqiyi.py
@@ -0,0 +1,214 @@
+# coding: utf-8
+
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+from ..compat import (
+    compat_chr,
+    compat_parse_qs,
+    compat_urllib_parse,
+    compat_urllib_request,
+    compat_urlparse,
+    compat_str,
+)
+
+from ..utils import ExtractorError
+
+import re
+import time
+import json
+import uuid
+import math
+import random
+import zlib
+import hashlib
+
+class IqiyiIE(InfoExtractor):
+    IE_NAME = 'iqiyi'
+
+    _VALID_URL = r'http://(?:www\.)iqiyi.com/.+?\.html'
+
+    _TEST = {
+            'url': 'http://www.iqiyi.com/v_19rrojlavg.html',
+            'md5': '260f0f59686e65e886995d0ba791ab83',
+            'info_dict': {
+                'id': '9c1fb1b99d192b21c559e5a1a2cb3c73',
+                'title': '美国德州空中惊现奇异云团 酷似UFO',
+                'ext': 'f4v'
+            }
+    }
+
+    def construct_video_urls(self, data, video_id, _uuid):
+        def do_xor(x, y):
+            a = y % 3
+            if a == 1:
+                return x ^ 121
+            if a == 2:
+                return x ^ 72
+            return x ^ 103
+
+        def get_encode_code(l):
+            a = 0
+            b = l.split('-')
+            c = len(b)
+            s = ''
+            for i in range(c - 1, -1, -1):
+                a = do_xor(int(b[c-i-1], 16), i)
+                s += chr(a)
+            return s[::-1]
+
+        def get_path_key(x):
+            mg = ')(*&^flash@#$%a'
+            tm = self._download_json(
+                'http://data.video.qiyi.com/t?tn=' + str(random.random()), video_id)['t']
+            t = str(int(math.floor(int(tm)/(600.0))))
+            return hashlib.md5(
+                 (t+mg+x).encode('utf8')).hexdigest()
+
+        video_urls_dict = {}
+        for i in data['vp']['tkl'][0]['vs']:
+            if 0 < int(i['bid']) <= 10:
+                format_id = self.get_format(i['bid'])
+
+            video_urls_info = i['fs']
+            if not i['fs'][0]['l'].startswith('/'):
+                t = get_encode_code(i['fs'][0]['l'])
+                if t.endswith('mp4'):
+                    video_urls_info = i['flvs']
+
+            video_urls = []
+            for ii in video_urls_info:
+                vl = ii['l']
+                if not vl.startswith('/'):
+                    vl = get_encode_code(vl)
+                key = get_path_key(
+                    vl.split('/')[-1].split('.')[0])
+                filesize = ii['b']
+                base_url = data['vp']['du'].split('/')
+                base_url.insert(-1, key)
+                base_url = '/'.join(base_url)
+                param = {
+                    'su': _uuid,
+                    'qyid': uuid.uuid4().hex,
+                    'client': '',
+                    'z': '',
+                    'bt': '',
+                    'ct': '',
+                    'tn': str(int(time.time()))
+                }
+                api_video_url = base_url + vl + '?' + \
+                    compat_urllib_parse.urlencode(param)
+                js = self._download_json(api_video_url, video_id)
+                video_url = js['l']
+                video_urls.append(
+                    (video_url, filesize))
+
+            video_urls_dict[format_id] = video_urls
+        return video_urls_dict
+
+    def get_format(self, bid):
+        bid_dict = {
+            '1': 'standard',
+            '2': 'high',
+            '3': 'super',
+            '4': 'suprt-high',
+            '5': 'fullhd',
+            '10': '4k'
+        }
+        return bid_dict[str(bid)]
+
+    def get_raw_data(self, tvid, video_id, enc_key, _uuid):
+        tm = str(int(time.time()))
+        param = {
+            'key': 'fvip',
+            'src': hashlib.md5(b'youtube-dl').hexdigest(),
+            'tvId': tvid,
+            'vid': video_id,
+            'vinfo': 1,
+            'tm': tm,
+            'enc': hashlib.md5(
+                (enc_key + tm + tvid).encode('utf8')).hexdigest(),
+            'qyid': _uuid,
+            'tn': random.random(),
+            'um': 0,
+            'authkey': hashlib.md5(
+                (tm + tvid).encode('utf8')).hexdigest()
+        }
+
+        api_url = 'http://cache.video.qiyi.com/vms' + '?' + \
+            compat_urllib_parse.urlencode(param)
+        raw_data = self._download_json(api_url, video_id)
+        return raw_data
+
+    def get_enc_key(self, swf_url, video_id):
+        req = self._request_webpage(
+            swf_url, video_id, note='download swf content')
+        cn = req.read()
+        cn = zlib.decompress(cn[8:])
+        pt = re.compile(b'MixerRemote\x08(?P<enc_key>.+?)\$&vv')
+        enc_key = self._search_regex(pt, cn, 'enc_key').decode('utf8')
+        return enc_key
+
+    def _real_extract(self, url):
+        webpage = self._download_webpage(
+            url, 'temp_id', note='download video page')
+        tvid = self._search_regex(
+            r'tvId ?= ?(\'|\")(?P<tvid>\d+)', webpage, 'tvid', flags=re.I, group='tvid')
+        video_id = self._search_regex(
+            r'videoId ?= ?(\'|\")(?P<video_id>[a-z\d]+)',
+            webpage, 'video_id', flags=re.I, group='video_id')
+        swf_url = self._search_regex(
+            r'(?P<swf>http://.+?MainPlayer.+?\.swf)', webpage, 'swf')
+        _uuid = uuid.uuid4().hex
+
+        enc_key = self.get_enc_key(swf_url, video_id)
+
+        raw_data = self.get_raw_data(tvid, video_id, enc_key, _uuid)
+        assert raw_data['code'] == 'A000000'
+        if not raw_data['data']['vp']['tkl']:
+            raise ExtractorError('No support iQiqy VIP video')
+
+        data = raw_data['data']
+
+        title = data['vi']['vn']
+
+        # generate video_urls_dict
+        video_urls_dict = self.construct_video_urls(data, video_id, _uuid)
+
+        # construct info
+        entries = []
+        for format_id in video_urls_dict:
+            video_urls = video_urls_dict[format_id]
+            for i, video_url_info in enumerate(video_urls):
+                if len(entries) < i+1:
+                    entries.append({'formats': []})
+                entries[i]['formats'].append(
+                    {
+                        'url': video_url_info[0],
+                        'filesize': video_url_info[-1],
+                        'format_id': format_id,
+                    }
+                )
+
+        for i in range(len(entries)):
+            entries[i].update(
+                {
+                    'id': '_part%d' % (i+1),
+                    'title': title,
+                }
+            )
+
+        if len(entries) > 1:
+            info = {
+                '_type': 'multi_video',
+                'id': video_id,
+                'title': title,
+                'entries': entries,
+            }
+        else:
+            info = entries[0]
+            info['id'] = video_id
+            info['title'] = title
+
+        return info

From 08f7db20c16743a2bd3040eb7dac11d675011eef Mon Sep 17 00:00:00 2001
From: PeterDing <dfhayst@gmail.com>
Date: Sat, 30 May 2015 10:03:32 +0800
Subject: [PATCH 1429/6219] [youku] change format_id

---
 youtube_dl/extractor/youku.py | 37 +++++++++++++++++++++++------------
 1 file changed, 24 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index 063f2e10e..aed6b960a 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -132,26 +132,37 @@ class YoukuIE(InfoExtractor):
 
     def get_hd(self, fm):
         hd_id_dict = {
-            'flv': '0',
-            'mp4': '1',
-            'hd2': '2',
-            'hd3': '3',
-            '3gp': '0',
-            '3gphd': '1'
+            'flv'   : '0',
+            'mp4'   : '1',
+            'hd2'   : '2',
+            'hd3'   : '3',
+            '3gp'   : '0',
+            '3gphd' : '1'
         }
         return hd_id_dict[fm]
 
     def parse_ext_l(self, fm):
         ext_dict = {
-            'flv': 'flv',
-            'mp4': 'mp4',
-            'hd2': 'flv',
-            'hd3': 'flv',
-            '3gp': 'flv',
-            '3gphd': 'mp4',
+            'flv'   : 'flv',
+            'mp4'   : 'mp4',
+            'hd2'   : 'flv',
+            'hd3'   : 'flv',
+            '3gp'   : 'flv',
+            '3gphd' : 'mp4'
         }
         return ext_dict[fm]
 
+    def get_format_name(self, fm):
+        _dict = {
+            '3gp'   : 'h6',
+            '3gphd' : 'h5',
+            'flv'   : 'h4',
+            'mp4'   : 'h3',
+            'hd2'   : 'h2',
+            'hd3'   : 'h1'
+        }
+        return _dict[fm]
+
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
@@ -191,7 +202,7 @@ class YoukuIE(InfoExtractor):
                 entries[i]['formats'].append(
                     {
                         'url': video_urls[i],
-                        'format_id': fm,
+                        'format_id': self.get_format_name(fm),
                         'ext': self.parse_ext_l(fm),
                         'filesize': int(data1['segs'][fm][i]['size'])
                     }

From 670861bd206ab4063baeb6b80d06a054ce4e1d62 Mon Sep 17 00:00:00 2001
From: PeterDing <dfhayst@gmail.com>
Date: Sat, 30 May 2015 10:37:54 +0800
Subject: [PATCH 1430/6219] [iqiyi] Do not request for unneeded formats

---
 youtube_dl/extractor/iqiyi.py | 72 ++++++++++++++++++++++++-----------
 1 file changed, 50 insertions(+), 22 deletions(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index d96d13225..747f3f902 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -4,20 +4,12 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 
-from ..compat import (
-    compat_chr,
-    compat_parse_qs,
-    compat_urllib_parse,
-    compat_urllib_request,
-    compat_urlparse,
-    compat_str,
-)
+from ..compat import compat_urllib_parse
 
 from ..utils import ExtractorError
 
 import re
 import time
-import json
 import uuid
 import math
 import random
@@ -31,15 +23,15 @@ class IqiyiIE(InfoExtractor):
 
     _TEST = {
             'url': 'http://www.iqiyi.com/v_19rrojlavg.html',
-            'md5': '260f0f59686e65e886995d0ba791ab83',
+            'md5': '2cb594dc2781e6c941a110d8f358118b',
             'info_dict': {
                 'id': '9c1fb1b99d192b21c559e5a1a2cb3c73',
                 'title': '美国德州空中惊现奇异云团 酷似UFO',
-                'ext': 'f4v'
+                'ext': 'f4v',
             }
     }
 
-    def construct_video_urls(self, data, video_id, _uuid):
+    def construct_video_urls(self, data, video_id, _uuid, bid):
         def do_xor(x, y):
             a = y % 3
             if a == 1:
@@ -66,10 +58,21 @@ class IqiyiIE(InfoExtractor):
             return hashlib.md5(
                  (t+mg+x).encode('utf8')).hexdigest()
 
+        # get accept format
+        # getting all format will spend minutes for a big video.
+        if bid == 'best':
+            bids = [int(i['bid']) for i in data['vp']['tkl'][0]['vs'] \
+                   if 0 < int(i['bid']) <= 10]
+            bid = str(max(bids))
+
         video_urls_dict = {}
         for i in data['vp']['tkl'][0]['vs']:
             if 0 < int(i['bid']) <= 10:
                 format_id = self.get_format(i['bid'])
+            else:
+                continue
+
+            video_urls = []
 
             video_urls_info = i['fs']
             if not i['fs'][0]['l'].startswith('/'):
@@ -77,7 +80,12 @@ class IqiyiIE(InfoExtractor):
                 if t.endswith('mp4'):
                     video_urls_info = i['flvs']
 
-            video_urls = []
+            if int(i['bid']) != int(bid):  # ignore missing match format
+                video_urls.extend(
+                    [('http://example.com/v.flv', ii['b']) for ii in video_urls_info])
+                video_urls_dict[format_id] = video_urls
+                continue
+
             for ii in video_urls_info:
                 vl = ii['l']
                 if not vl.startswith('/'):
@@ -108,15 +116,27 @@ class IqiyiIE(InfoExtractor):
         return video_urls_dict
 
     def get_format(self, bid):
-        bid_dict = {
-            '1': 'standard',
-            '2': 'high',
-            '3': 'super',
-            '4': 'suprt-high',
-            '5': 'fullhd',
-            '10': '4k'
+        _dict = {
+            '1'  : 'h6',
+            '2'  : 'h5',
+            '3'  : 'h4',
+            '4'  : 'h3',
+            '5'  : 'h2',
+            '10' : 'h1'
         }
-        return bid_dict[str(bid)]
+        return _dict.get(str(bid), None)
+
+    def get_bid(self, format_id):
+        _dict = {
+            'h6'   : '1',
+            'h5'   : '2',
+            'h4'   : '3',
+            'h3'   : '4',
+            'h2'   : '5',
+            'h1'   : '10',
+            'best' : 'best'
+        }
+        return _dict.get(format_id, None)
 
     def get_raw_data(self, tvid, video_id, enc_key, _uuid):
         tm = str(int(time.time()))
@@ -173,8 +193,14 @@ class IqiyiIE(InfoExtractor):
 
         title = data['vi']['vn']
 
+        format = self._downloader.params.get('format', None)
+        bid = self.get_bid(format) if format else 'best'
+        if not bid:
+            raise ExtractorError('Can\'t get format.')
+
         # generate video_urls_dict
-        video_urls_dict = self.construct_video_urls(data, video_id, _uuid)
+        video_urls_dict = self.construct_video_urls(
+            data, video_id, _uuid, bid)
 
         # construct info
         entries = []
@@ -188,10 +214,12 @@ class IqiyiIE(InfoExtractor):
                         'url': video_url_info[0],
                         'filesize': video_url_info[-1],
                         'format_id': format_id,
+                        'preference': int(self.get_bid(format_id))
                     }
                 )
 
         for i in range(len(entries)):
+            self._sort_formats(entries[i]['formats'])
             entries[i].update(
                 {
                     'id': '_part%d' % (i+1),

From fafec39d4177f4873bb2393749a46873c4ffda4a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 30 May 2015 13:23:09 +0800
Subject: [PATCH 1431/6219] [spiegeltv] Changed RTMP server (fixes #5788 and
 fixes #5843)

Thanks to @brickleroux for finding out the problem
---
 youtube_dl/extractor/spiegeltv.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/spiegeltv.py b/youtube_dl/extractor/spiegeltv.py
index 98cf92d89..359722ad6 100644
--- a/youtube_dl/extractor/spiegeltv.py
+++ b/youtube_dl/extractor/spiegeltv.py
@@ -51,9 +51,9 @@ class SpiegeltvIE(InfoExtractor):
         is_wide = media_json['is_wide']
 
         server_json = self._download_json(
-            'http://www.spiegel.tv/streaming_servers/', video_id,
-            note='Downloading server information')
-        server = server_json[0]['endpoint']
+            'http://spiegeltv-prod-static.s3.amazonaws.com/projectConfigs/projectConfig.json',
+            video_id, note='Downloading server information')
+        server = server_json['streamingserver'][0]['endpoint']
 
         thumbnails = []
         for image in media_json['images']:
@@ -76,5 +76,6 @@ class SpiegeltvIE(InfoExtractor):
             'ext': 'm4v',
             'description': description,
             'duration': duration,
-            'thumbnails': thumbnails
+            'thumbnails': thumbnails,
+            'rtmp_live': True,
         }

From 6ebdfe43e439239df051f6071a23c51705c150cf Mon Sep 17 00:00:00 2001
From: pulpe <Pulpan3@gmail.com>
Date: Sat, 30 May 2015 09:30:14 +0200
Subject: [PATCH 1432/6219] [tube8] fix extractor (fixes #5846)

---
 youtube_dl/extractor/tube8.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/tube8.py b/youtube_dl/extractor/tube8.py
index d73ad3762..6ca8840b0 100644
--- a/youtube_dl/extractor/tube8.py
+++ b/youtube_dl/extractor/tube8.py
@@ -47,7 +47,7 @@ class Tube8IE(InfoExtractor):
         webpage = self._download_webpage(req, display_id)
 
         flashvars = json.loads(self._html_search_regex(
-            r'var flashvars\s*=\s*({.+?})', webpage, 'flashvars'))
+            r'flashvars\s*=\s*({.+?})', webpage, 'flashvars'))
 
         video_url = flashvars['video_url']
         if flashvars.get('encrypted') is True:
@@ -58,19 +58,19 @@ class Tube8IE(InfoExtractor):
         thumbnail = flashvars.get('image_url')
 
         title = self._html_search_regex(
-            r'videotitle\s*=\s*"([^"]+)', webpage, 'title')
+            r'videoTitle\s*=\s*"([^"]+)', webpage, 'title')
         description = self._html_search_regex(
-            r'>Description:</strong>(.+?)<', webpage, 'description', fatal=False)
+            r'>Description:</strong>\s*(.+?)\s*<', webpage, 'description', fatal=False)
         uploader = self._html_search_regex(
-            r'<strong class="video-username">(?:<a href="[^"]+">)?([^<]+)(?:</a>)?</strong>',
+            r'<span class="username">\s*(.+?)\s*<',
             webpage, 'uploader', fatal=False)
 
         like_count = int_or_none(self._html_search_regex(
-            r"rupVar\s*=\s*'(\d+)'", webpage, 'like count', fatal=False))
+            r'rupVar\s*=\s*"(\d+)"', webpage, 'like count', fatal=False))
         dislike_count = int_or_none(self._html_search_regex(
-            r"rdownVar\s*=\s*'(\d+)'", webpage, 'dislike count', fatal=False))
+            r'rdownVar\s*=\s*"(\d+)"', webpage, 'dislike count', fatal=False))
         view_count = self._html_search_regex(
-            r'<strong>Views: </strong>([\d,\.]+)</li>', webpage, 'view count', fatal=False)
+            r'<strong>Views: </strong>([\d,\.]+)\s*</li>', webpage, 'view count', fatal=False)
         if view_count:
             view_count = str_to_int(view_count)
         comment_count = self._html_search_regex(

From 0385d642232ba4e8b455d0c4eb95c7985f22f276 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 30 May 2015 14:12:58 +0600
Subject: [PATCH 1433/6219] [crunchyroll] Extract subtitles extraction routine

---
 youtube_dl/extractor/crunchyroll.py | 30 +++++++++++++----------------
 1 file changed, 13 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 1c77df47e..4ac537a6d 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -76,8 +76,8 @@ class CrunchyrollIE(InfoExtractor):
         self._login()
 
     def _decrypt_subtitles(self, data, iv, id):
-        data = bytes_to_intlist(data)
-        iv = bytes_to_intlist(iv)
+        data = bytes_to_intlist(base64.b64decode(data))
+        iv = bytes_to_intlist(base64.b64decode(iv))
         id = int(id)
 
         def obfuscate_key_aux(count, modulo, start):
@@ -179,6 +179,16 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
 
         return output
 
+    def _extract_subtitles(self, subtitle):
+        sub_root = xml.etree.ElementTree.fromstring(subtitle)
+        return [{
+            'ext': 'srt',
+            'data': self._convert_subtitles_to_srt(sub_root),
+        }, {
+            'ext': 'ass',
+            'data': self._convert_subtitles_to_ass(sub_root),
+        }]
+
     def _get_subtitles(self, video_id, webpage):
         subtitles = {}
         for sub_id, sub_name in re.findall(r'\?ssid=([0-9]+)" title="([^"]+)', webpage):
@@ -190,25 +200,11 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
             data = self._search_regex(r'<data>([^<]+)', sub_page, 'subtitle_data', fatal=False)
             if not id or not iv or not data:
                 continue
-            id = int(id)
-            iv = base64.b64decode(iv)
-            data = base64.b64decode(data)
-
             subtitle = self._decrypt_subtitles(data, iv, id).decode('utf-8')
             lang_code = self._search_regex(r'lang_code=["\']([^"\']+)', subtitle, 'subtitle_lang_code', fatal=False)
             if not lang_code:
                 continue
-            sub_root = xml.etree.ElementTree.fromstring(subtitle)
-            subtitles[lang_code] = [
-                {
-                    'ext': 'srt',
-                    'data': self._convert_subtitles_to_srt(sub_root),
-                },
-                {
-                    'ext': 'ass',
-                    'data': self._convert_subtitles_to_ass(sub_root),
-                },
-            ]
+            subtitles[lang_code] = self._extract_subtitles(subtitle)
         return subtitles
 
     def _real_extract(self, url):

From b2cf6543b21bbe0954c45b35b1402eaca5187c0d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 30 May 2015 14:30:04 +0600
Subject: [PATCH 1434/6219] [soompi] Improve and simplify

---
 youtube_dl/extractor/soompi.py | 146 ++++++++++++++++++---------------
 1 file changed, 81 insertions(+), 65 deletions(-)

diff --git a/youtube_dl/extractor/soompi.py b/youtube_dl/extractor/soompi.py
index 4726872dc..5da66ca9e 100644
--- a/youtube_dl/extractor/soompi.py
+++ b/youtube_dl/extractor/soompi.py
@@ -2,17 +2,31 @@
 from __future__ import unicode_literals
 
 import re
-import json
-import base64
-import xml.etree.ElementTree
 
-# Soompi uses the same subtitle encryption as crunchyroll
 from .crunchyroll import CrunchyrollIE
 
+from .common import InfoExtractor
+from ..compat import compat_HTTPError
+from ..utils import (
+    ExtractorError,
+    int_or_none,
+    remove_start,
+    xpath_text,
+)
 
-class SoompiIE(CrunchyrollIE):
+
+class SoompiBaseIE(InfoExtractor):
+    def _get_episodes(self, webpage, episode_filter=None):
+        episodes = self._parse_json(
+            self._search_regex(
+                r'VIDEOS\s*=\s*(\[.+?\]);', webpage, 'episodes JSON'),
+            None)
+        return list(filter(episode_filter, episodes))
+
+
+class SoompiIE(SoompiBaseIE, CrunchyrollIE):
     IE_NAME = 'soompi'
-    _VALID_URL = r'^https?://tv\.soompi\.com/en/watch/(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://tv\.soompi\.com/(?:en/)?watch/(?P<id>[0-9]+)'
     _TESTS = [{
         'url': 'http://tv.soompi.com/en/watch/29235',
         'info_dict': {
@@ -26,84 +40,86 @@ class SoompiIE(CrunchyrollIE):
         },
     }]
 
-    def _get_episodes(self, webpage, episode_filter=None):
-        episodes = json.loads(
-            self._search_regex(r'\s+VIDEOS\s+= (\[.+?\]);', webpage, "episodes meta"))
-        return [ep for ep in episodes if episode_filter is None or episode_filter(ep)]
-
-    def _get_subtitles(self, video_id, show_format_xml):
-        subtitles = {}
-        subtitle_info_nodes = show_format_xml.findall('./{default}preload/subtitles/subtitle')
-        subtitle_nodes = show_format_xml.findall('./{default}preload/subtitle')
+    def _get_episode(self, webpage, video_id):
+        return self._get_episodes(webpage, lambda x: x['id'] == video_id)[0]
 
+    def _get_subtitles(self, config, video_id):
         sub_langs = {}
-        for i in subtitle_info_nodes:
-            sub_langs[i.attrib["id"]] = i.attrib["title"]
+        for subtitle in config.findall('./{default}preload/subtitles/subtitle'):
+            sub_langs[subtitle.attrib['id']] = subtitle.attrib['title']
 
-        for s in subtitle_nodes:
-            lang_code = sub_langs.get(s.attrib["id"], None)
-            if lang_code is None:
+        subtitles = {}
+        for s in config.findall('./{default}preload/subtitle'):
+            lang_code = sub_langs.get(s.attrib['id'])
+            if not lang_code:
+                continue
+            sub_id = s.get('id')
+            data = xpath_text(s, './data', 'data')
+            iv = xpath_text(s, './iv', 'iv')
+            if not id or not iv or not data:
                 continue
-
-            sub_id = int(s.attrib["id"])
-            iv = base64.b64decode(s.find("iv").text)
-            data = base64.b64decode(s.find("data").text)
             subtitle = self._decrypt_subtitles(data, iv, sub_id).decode('utf-8')
-            sub_root = xml.etree.ElementTree.fromstring(subtitle)
-
-            subtitles[lang_code] = [{
-                'ext': 'srt', 'data': self._convert_subtitles_to_srt(sub_root)
-            }, {
-                'ext': 'ass', 'data': self._convert_subtitles_to_ass(sub_root)
-            }]
+            subtitles[lang_code] = self._extract_subtitles(subtitle)
         return subtitles
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        webpage = self._download_webpage(
-            url, video_id, note="Downloading episode page",
-            errnote="Video may not be available for your location")
-        vid_formats = re.findall(r"\?quality=q([0-9]+)", webpage)
-
-        show_meta = json.loads(
-            self._search_regex(r'\s+var show = (\{.+?\});', webpage, "show meta"))
-        episodes = self._get_episodes(
-            webpage, episode_filter=lambda x: x['id'] == video_id)
-
-        title = episodes[0]["name"]
-        description = episodes[0]["description"]
-        duration = int(episodes[0]["duration"])
-        slug = show_meta["slug"]
+        try:
+            webpage = self._download_webpage(
+                url, video_id, 'Downloading episode page')
+        except ExtractorError as ee:
+            if isinstance(ee.cause, compat_HTTPError) and ee.cause.code == 403:
+                webpage = ee.cause.read()
+                block_message = self._html_search_regex(
+                    r'(?s)<div class="block-message">(.+?)</div>', webpage,
+                    'block message', default=None)
+                if block_message:
+                    raise ExtractorError(block_message, expected=True)
+            raise
 
         formats = []
-        show_format_xml = None
-        for vf in vid_formats:
-            show_format_url = "http://tv.soompi.com/en/show/%s/%s-config.xml?mode=hls&quality=q%s" \
-                              % (slug, video_id, vf)
-            show_format_xml = self._download_xml(
-                show_format_url, video_id, note="Downloading q%s show xml" % vf)
-            avail_formats = self._extract_m3u8_formats(
-                show_format_xml.find('./{default}preload/stream_info/file').text,
-                video_id, ext="mp4", m3u8_id=vf, preference=int(vf))
-            formats.extend(avail_formats)
+        config = None
+        for format_id in re.findall(r'\?quality=([0-9a-zA-Z]+)', webpage):
+            config = self._download_xml(
+                'http://tv.soompi.com/en/show/_/%s-config.xml?mode=hls&quality=%s' % (video_id, format_id),
+                video_id, 'Downloading %s XML' % format_id)
+            m3u8_url = xpath_text(
+                config, './{default}preload/stream_info/file',
+                '%s m3u8 URL' % format_id)
+            if not m3u8_url:
+                continue
+            formats.extend(self._extract_m3u8_formats(
+                m3u8_url, video_id, 'mp4', m3u8_id=format_id))
         self._sort_formats(formats)
 
-        subtitles = self.extract_subtitles(video_id, show_format_xml)
+        episode = self._get_episode(webpage, video_id)
+
+        title = episode['name']
+        description = episode.get('description')
+        duration = int_or_none(episode.get('duration'))
+
+        thumbnails = [{
+            'id': thumbnail_id,
+            'url': thumbnail_url,
+        } for thumbnail_id, thumbnail_url in episode.get('img_url', {}).items()]
+
+        subtitles = self.extract_subtitles(config, video_id)
 
         return {
             'id': video_id,
             'title': title,
             'description': description,
+            'thumbnails': thumbnails,
             'duration': duration,
             'formats': formats,
             'subtitles': subtitles
         }
 
 
-class SoompiShowIE(SoompiIE):
+class SoompiShowIE(SoompiBaseIE):
     IE_NAME = 'soompi:show'
-    _VALID_URL = r'^https?://tv\.soompi\.com/en/shows/(?P<id>[0-9a-zA-Z\-_]+)'
+    _VALID_URL = r'https?://tv\.soompi\.com/en/shows/(?P<id>[0-9a-zA-Z\-_]+)'
     _TESTS = [{
         'url': 'http://tv.soompi.com/en/shows/liar-game',
         'info_dict': {
@@ -117,14 +133,14 @@ class SoompiShowIE(SoompiIE):
     def _real_extract(self, url):
         show_id = self._match_id(url)
 
-        webpage = self._download_webpage(url, show_id, note="Downloading show page")
-        title = self._og_search_title(webpage).replace("SoompiTV | ", "")
+        webpage = self._download_webpage(
+            url, show_id, 'Downloading show page')
+
+        title = remove_start(self._og_search_title(webpage), 'SoompiTV | ')
         description = self._og_search_description(webpage)
 
-        episodes = self._get_episodes(webpage)
-        entries = []
-        for ep in episodes:
-            entries.append(self.url_result(
-                'http://tv.soompi.com/en/watch/%s' % ep['id'], 'Soompi', ep['id']))
+        entries = [
+            self.url_result('http://tv.soompi.com/en/watch/%s' % episode['id'], 'Soompi')
+            for episode in self._get_episodes(webpage)]
 
         return self.playlist_result(entries, show_id, title, description)

From 1a5b77dc21384c462e0be86a1638cafd15a6e236 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 30 May 2015 14:36:45 +0600
Subject: [PATCH 1435/6219] [crunchyroll] Fix python 3.2

---
 youtube_dl/extractor/crunchyroll.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 4ac537a6d..41f0c736d 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -76,8 +76,8 @@ class CrunchyrollIE(InfoExtractor):
         self._login()
 
     def _decrypt_subtitles(self, data, iv, id):
-        data = bytes_to_intlist(base64.b64decode(data))
-        iv = bytes_to_intlist(base64.b64decode(iv))
+        data = bytes_to_intlist(base64.b64decode(data.encode('utf-8')))
+        iv = bytes_to_intlist(base64.b64decode(iv.encode('utf-8')))
         id = int(id)
 
         def obfuscate_key_aux(count, modulo, start):

From 5c2191a6053cb5b1210cef68406e8a52e86fd9fd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 30 May 2015 15:14:10 +0600
Subject: [PATCH 1436/6219] [vgtv] Skip wasLive hds (Closes #5835)

---
 youtube_dl/extractor/vgtv.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vgtv.py b/youtube_dl/extractor/vgtv.py
index e6ee1e471..654298431 100644
--- a/youtube_dl/extractor/vgtv.py
+++ b/youtube_dl/extractor/vgtv.py
@@ -107,7 +107,8 @@ class VGTVIE(InfoExtractor):
                 hls_url, video_id, 'mp4', m3u8_id='hls'))
 
         hds_url = streams.get('hds')
-        if hds_url:
+        # wasLive hds are always 404
+        if hds_url and data.get('streamType') != 'wasLive':
             formats.extend(self._extract_f4m_formats(
                 hds_url + '?hdcore=3.2.0&plugin=aasp-3.2.0.77.18',
                 video_id, f4m_id='hds'))

From 4d454c5e4b7ecfae97ff109e05453f43d7cea0a2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 30 May 2015 15:15:42 +0600
Subject: [PATCH 1437/6219] [vgtv] Check for inactive videos

---
 youtube_dl/extractor/vgtv.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vgtv.py b/youtube_dl/extractor/vgtv.py
index 654298431..472feb7f0 100644
--- a/youtube_dl/extractor/vgtv.py
+++ b/youtube_dl/extractor/vgtv.py
@@ -4,7 +4,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import float_or_none
+from ..utils import (
+    ExtractorError,
+    float_or_none,
+)
 
 
 class VGTVIE(InfoExtractor):
@@ -97,6 +100,10 @@ class VGTVIE(InfoExtractor):
             % (host, video_id, HOST_WEBSITES[host]),
             video_id, 'Downloading media JSON')
 
+        if data.get('status') == 'inactive':
+            raise ExtractorError(
+                'Video %s is no longer available' % video_id, expected=True)
+
         streams = data['streamUrls']
 
         formats = []

From 181c7053e377700c1615bdff2b0fb19235762c57 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 30 May 2015 16:04:44 +0600
Subject: [PATCH 1438/6219] [YoutubeDL] Make sure all formats have unique
 format_id

---
 youtube_dl/YoutubeDL.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index d1953c18f..21d247f23 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1047,6 +1047,8 @@ class YoutubeDL(object):
         if not formats:
             raise ExtractorError('No video formats found!')
 
+        formats_dict = {}
+
         # We check that all the formats have the format and format_id fields
         for i, format in enumerate(formats):
             if 'url' not in format:
@@ -1054,6 +1056,18 @@ class YoutubeDL(object):
 
             if format.get('format_id') is None:
                 format['format_id'] = compat_str(i)
+            format_id = format['format_id']
+            if format_id not in formats_dict:
+                formats_dict[format_id] = []
+            formats_dict[format_id].append(format)
+
+        # Make sure all formats have unique format_id
+        for format_id, ambiguous_formats in formats_dict.items():
+            if len(ambiguous_formats) > 1:
+                for i, format in enumerate(ambiguous_formats):
+                    format['format_id'] = '%s-%d' % (format_id, i)
+
+        for i, format in enumerate(formats):
             if format.get('format') is None:
                 format['format'] = '{id} - {res}{note}'.format(
                     id=format['format_id'],

From b4dd98358f0a68650f6154e8de4e12b8881248aa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 30 May 2015 16:12:07 +0600
Subject: [PATCH 1439/6219] [vgtv] Properly handle lives

---
 youtube_dl/extractor/vgtv.py | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/vgtv.py b/youtube_dl/extractor/vgtv.py
index 472feb7f0..f38a72fde 100644
--- a/youtube_dl/extractor/vgtv.py
+++ b/youtube_dl/extractor/vgtv.py
@@ -62,16 +62,16 @@ class VGTVIE(InfoExtractor):
         },
         {
             # streamType: live
-            'url': 'http://www.vgtv.no/#!/live/100015/direkte-her-kan-du-se-laksen-live-fra-suldalslaagen',
+            'url': 'http://www.vgtv.no/#!/live/113063/direkte-v75-fra-solvalla',
             'info_dict': {
-                'id': '100015',
+                'id': '113063',
                 'ext': 'flv',
-                'title': 'DIREKTE: Her kan du se laksen live fra Suldalslågen!',
-                'description': 'md5:9a60cc23fa349f761628924e56eeec2d',
+                'title': 're:^DIREKTE: V75 fra Solvalla [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
+                'description': 'md5:b3743425765355855f88e096acc93231',
                 'thumbnail': 're:^https?://.*\.jpg',
                 'duration': 0,
-                'timestamp': 1407423348,
-                'upload_date': '20140807',
+                'timestamp': 1432975582,
+                'upload_date': '20150530',
                 'view_count': int,
             },
             'params': {
@@ -105,6 +105,7 @@ class VGTVIE(InfoExtractor):
                 'Video %s is no longer available' % video_id, expected=True)
 
         streams = data['streamUrls']
+        stream_type = data.get('streamType')
 
         formats = []
 
@@ -115,7 +116,7 @@ class VGTVIE(InfoExtractor):
 
         hds_url = streams.get('hds')
         # wasLive hds are always 404
-        if hds_url and data.get('streamType') != 'wasLive':
+        if hds_url and stream_type != 'wasLive':
             formats.extend(self._extract_f4m_formats(
                 hds_url + '?hdcore=3.2.0&plugin=aasp-3.2.0.77.18',
                 video_id, f4m_id='hds'))
@@ -143,13 +144,14 @@ class VGTVIE(InfoExtractor):
 
         return {
             'id': video_id,
-            'title': data['title'],
+            'title': self._live_title(data['title']),
             'description': data['description'],
             'thumbnail': data['images']['main'] + '?t[]=900x506q80',
             'timestamp': data['published'],
             'duration': float_or_none(data['duration'], 1000),
             'view_count': data['displays'],
             'formats': formats,
+            'is_live': True if stream_type == 'live' else False,
         }
 
 

From e6e63e91a70a9c2e4ab92b8afad6fac3b8bede18 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 30 May 2015 16:18:11 +0600
Subject: [PATCH 1440/6219] [tf1] Extend _VALID_URL (Closes #5848)

---
 youtube_dl/extractor/tf1.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tf1.py b/youtube_dl/extractor/tf1.py
index 656410528..3a68eaa80 100644
--- a/youtube_dl/extractor/tf1.py
+++ b/youtube_dl/extractor/tf1.py
@@ -6,7 +6,7 @@ from .common import InfoExtractor
 
 class TF1IE(InfoExtractor):
     """TF1 uses the wat.tv player."""
-    _VALID_URL = r'http://(?:videos\.tf1|www\.tfou|www\.tf1)\.fr/.*?-(?P<id>\d+)(?:-\d+)?\.html'
+    _VALID_URL = r'http://(?:(?:videos|www|lci)\.tf1|www\.tfou)\.fr/.*?-(?P<id>\d+)(?:-\d+)?\.html'
     _TESTS = [{
         'url': 'http://videos.tf1.fr/auto-moto/citroen-grand-c4-picasso-2013-presentation-officielle-8062060.html',
         'info_dict': {
@@ -35,6 +35,9 @@ class TF1IE(InfoExtractor):
     }, {
         'url': 'http://www.tf1.fr/tf1/koh-lanta/videos/replay-koh-lanta-22-mai-2015.html',
         'only_matching': True,
+    }, {
+        'url': 'http://lci.tf1.fr/sept-a-huit/videos/sept-a-huit-du-24-mai-2015-8611550.html',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 5196b988971716b9e9c5884d33c757a41aa4548a Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Sat, 30 May 2015 14:16:18 +0300
Subject: [PATCH 1441/6219] [tubitv] Add new extractor (Closes #5524)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/tubitv.py   | 79 ++++++++++++++++++++++++++++++++
 2 files changed, 80 insertions(+)
 create mode 100644 youtube_dl/extractor/tubitv.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index f73bf646b..e7e0a55f2 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -569,6 +569,7 @@ from .traileraddict import TrailerAddictIE
 from .trilulilu import TriluliluIE
 from .trutube import TruTubeIE
 from .tube8 import Tube8IE
+from .tubitv import TubiTvIE
 from .tudou import TudouIE
 from .tumblr import TumblrIE
 from .tunein import TuneInIE
diff --git a/youtube_dl/extractor/tubitv.py b/youtube_dl/extractor/tubitv.py
new file mode 100644
index 000000000..03e971e5e
--- /dev/null
+++ b/youtube_dl/extractor/tubitv.py
@@ -0,0 +1,79 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import codecs
+import re
+
+from .common import InfoExtractor
+from ..compat import (
+    compat_urllib_parse,
+    compat_urllib_request
+)
+from ..utils import (
+    ExtractorError,
+    int_or_none,
+)
+
+
+class TubiTvIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?tubitv\.com/video\?id=(?P<id>[0-9]+)'
+    _LOGIN_URL = 'http://tubitv.com/login'
+    _NETRC_MACHINE = 'tubitv'
+    _TEST = {
+        'url': 'http://tubitv.com/video?id=54411&title=The_Kitchen_Musical_-_EP01',
+        'info_dict': {
+            'id': '54411',
+            'ext': 'mp4',
+            'title': 'The Kitchen Musical - EP01',
+            'thumbnail': 're:^https?://.*\.png$',
+            'description': 'md5:37532716166069b353e8866e71fefae7',
+            'duration': 2407,
+        },
+        'params': {
+            'skip_download': 'HLS download',
+        },
+    }
+
+    def _login(self):
+        (username, password) = self._get_login_info()
+        if username is None:
+            return
+        self.report_login()
+        form_data = {
+            'username': username,
+            'password': password,
+        }
+        payload = compat_urllib_parse.urlencode(form_data).encode('utf-8')
+        request = compat_urllib_request.Request(self._LOGIN_URL, payload)
+        request.add_header('Content-Type', 'application/x-www-form-urlencoded')
+        login_page = self._download_webpage(
+            request, None, False, 'Wrong login info')
+        if not re.search(r'id="tubi-logout"', login_page):
+            raise ExtractorError(
+                'Login failed (invalid username/password)', expected=True)
+
+    def _real_initialize(self):
+        self._login()
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+        title = self._og_search_title(webpage)
+        description = self._og_search_description(webpage)
+        thumbnail = self._og_search_thumbnail(webpage)
+        duration = int_or_none(self._html_search_meta(
+            'video:duration', webpage, 'duration'))
+
+        apu = self._search_regex(r"apu='([^']+)'", webpage, 'apu')
+        m3u8_url = codecs.decode(apu, 'rot_13')[::-1]
+        formats = self._extract_m3u8_formats(m3u8_url, video_id, ext='mp4')
+
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': formats,
+            'thumbnail': thumbnail,
+            'description': description,
+            'duration': duration,
+        }

From 1ae7ff771b17d16540aa446aef4f10971465a249 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Sat, 30 May 2015 14:33:27 +0300
Subject: [PATCH 1442/6219] [tubitv] Add error message for videos that require
 login (#5524)

---
 youtube_dl/extractor/tubitv.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/tubitv.py b/youtube_dl/extractor/tubitv.py
index 03e971e5e..2c4b21807 100644
--- a/youtube_dl/extractor/tubitv.py
+++ b/youtube_dl/extractor/tubitv.py
@@ -59,6 +59,11 @@ class TubiTvIE(InfoExtractor):
         video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id)
+        if re.search(r"<(?:DIV|div) class='login-required-screen'>", webpage):
+            raise ExtractorError(
+                'This video requires login, use --username and --password '
+                'options to provide account credentials.', expected=True)
+
         title = self._og_search_title(webpage)
         description = self._og_search_description(webpage)
         thumbnail = self._og_search_thumbnail(webpage)

From 386bdfa698a7f06c43df91913677db3732e29900 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 30 May 2015 18:29:16 +0600
Subject: [PATCH 1443/6219] [youtube:user] Workaround 35 pages limitation
 (Closes #5778)

---
 youtube_dl/extractor/youtube.py | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 0301682b8..fcdbfe0bc 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1399,6 +1399,26 @@ class YoutubeChannelIE(InfoExtractor):
         channel_id = self._match_id(url)
 
         url = self._TEMPLATE_URL % channel_id
+
+        # Channel by page listing is restricted to 35 pages of 30 items, i.e. 1050 videos total (see #5778)
+        # Workaround by extracting as a playlist if managed to obtain channel playlist URL
+        # otherwise fallback on channel by page extraction
+        channel_page = self._download_webpage(
+            url + '?view=57', channel_id,
+            'Downloading channel page', fatal=False)
+        channel_playlist_id = self._search_regex(
+            [r'<meta itemprop="channelId" content="([^"]+)">',
+             r'data-channel-external-id="([^"]+)"'],
+            channel_page, 'channel id', default=None)
+        if channel_playlist_id and channel_playlist_id.startswith('UC'):
+            playlist_id = 'UU' + channel_playlist_id[2:]
+            channel_playlist = unescapeHTML(self._search_regex(
+                r'href="/?(watch\?v=[0-9A-Za-z_-]{11}&amp;list=%s)"' % playlist_id,
+                channel_page, 'channel playlist URL', default=None))
+            if channel_playlist:
+                return self.url_result(
+                    compat_urlparse.urljoin(url, '/%s' % channel_playlist), 'YoutubePlaylist')
+
         channel_page = self._download_webpage(url, channel_id, 'Downloading page #1')
         autogenerated = re.search(r'''(?x)
                 class="[^"]*?(?:

From 9ff811c5cddbf3481fdcd44e97cf3683a925b33f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 30 May 2015 23:35:55 +0800
Subject: [PATCH 1444/6219] [porn91] PEP8

---
 youtube_dl/extractor/porn91.py | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/porn91.py b/youtube_dl/extractor/porn91.py
index b62eec92d..cdf308f3d 100644
--- a/youtube_dl/extractor/porn91.py
+++ b/youtube_dl/extractor/porn91.py
@@ -5,7 +5,6 @@ import re
 
 from ..compat import compat_urllib_parse
 from .common import InfoExtractor
-from ..utils import ExtractorError
 
 
 class Porn91IE(InfoExtractor):
@@ -13,13 +12,13 @@ class Porn91IE(InfoExtractor):
     _VALID_URL = r'(?:https?://)(?:www\.|)91porn\.com/.+?\?viewkey=(?P<id>[\w\d]+)'
 
     _TEST = {
-            'url': 'http://91porn.com/view_video.php?viewkey=7e42283b4f5ab36da134',
-            'md5': '6df8f6d028bc8b14f5dbd73af742fb20',
-            'info_dict': {
-                'id': '7e42283b4f5ab36da134',
-                'title': '18岁大一漂亮学妹，水嫩性感，再爽一次！',
-                'ext': 'mp4'
-            }
+        'url': 'http://91porn.com/view_video.php?viewkey=7e42283b4f5ab36da134',
+        'md5': '6df8f6d028bc8b14f5dbd73af742fb20',
+        'info_dict': {
+            'id': '7e42283b4f5ab36da134',
+            'title': '18岁大一漂亮学妹，水嫩性感，再爽一次！',
+            'ext': 'mp4'
+        }
     }
 
     def _real_extract(self, url):

From 1c2223875664f99325b73fe7765677db9b87e105 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 31 May 2015 00:03:19 +0800
Subject: [PATCH 1445/6219] [porn91] Simplify

---
 youtube_dl/extractor/porn91.py | 38 ++++++++++++++--------------------
 1 file changed, 16 insertions(+), 22 deletions(-)

diff --git a/youtube_dl/extractor/porn91.py b/youtube_dl/extractor/porn91.py
index cdf308f3d..377ca2c77 100644
--- a/youtube_dl/extractor/porn91.py
+++ b/youtube_dl/extractor/porn91.py
@@ -1,8 +1,6 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from ..compat import compat_urllib_parse
 from .common import InfoExtractor
 
@@ -22,38 +20,34 @@ class Porn91IE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
         url = 'http://91porn.com/view_video.php?viewkey=%s' % video_id
         self._set_cookie('91porn.com', 'language', 'cn_CN')
         webpage = self._download_webpage(url, video_id, "get HTML content")
         title = self._search_regex(
-            r'<div id="viewvideo-title">(?P<title>.+?)</div>',
-            webpage, 'title', flags=re.DOTALL)
+            r'<div id="viewvideo-title">([^<]+)</div>', webpage, 'title')
         title = title.replace('\n', '')
 
         # get real url
-        n1 = self._search_regex(
-            r'so.addVariable\(\'file\',\'(?P<n1>\d+)\'', webpage, 'n1')
-        n2 = self._search_regex(
-            r'so.addVariable\(\'seccode\',\'(?P<n2>.+?)\'', webpage, 'n2')
-        n3 = self._search_regex(
-            r'so.addVariable\(\'max_vid\',\'(?P<n3>\d+)\'', webpage, 'n3')
+        file_id = self._search_regex(
+            r'so.addVariable\(\'file\',\'(\d+)\'', webpage, 'file id')
+        sec_code = self._search_regex(
+            r'so.addVariable\(\'seccode\',\'([^\']+)\'', webpage, 'sec code')
+        max_vid = self._search_regex(
+            r'so.addVariable\(\'max_vid\',\'(\d+)\'', webpage, 'max vid')
         url_params = compat_urllib_parse.urlencode({
-            'VID': n1,
+            'VID': file_id,
             'mp4': '1',
-            'seccode': n2,
-            'max_vid': n3,
+            'seccode': sec_code,
+            'max_vid': max_vid,
         })
-        t_url = 'http://91porn.com/getfile.php?' + url_params
-        info_cn = self._download_webpage(t_url, video_id, "get real video_url")
-        video_url = self._search_regex(r'file=(?P<url>http.+?)&', info_cn, 'url')
+        info_cn = self._download_webpage(
+            'http://91porn.com/getfile.php?' + url_params, video_id,
+            "get real video url")
+        video_url = self._search_regex(r'file=([^&]+)&', info_cn, 'url')
 
-        # construct info
-        info = {
+        return {
             'id': video_id,
             'title': title,
             'url': video_url,
         }
-
-        return info

From a80601f8d9789e27c0a916e63d7192c3f398d5d5 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 31 May 2015 00:20:37 +0800
Subject: [PATCH 1446/6219] [porn91] Extract more info

---
 youtube_dl/extractor/porn91.py | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/porn91.py b/youtube_dl/extractor/porn91.py
index 377ca2c77..ea1efc71b 100644
--- a/youtube_dl/extractor/porn91.py
+++ b/youtube_dl/extractor/porn91.py
@@ -3,6 +3,10 @@ from __future__ import unicode_literals
 
 from ..compat import compat_urllib_parse
 from .common import InfoExtractor
+from ..utils import (
+    parse_duration,
+    int_or_none,
+)
 
 
 class Porn91IE(InfoExtractor):
@@ -15,7 +19,8 @@ class Porn91IE(InfoExtractor):
         'info_dict': {
             'id': '7e42283b4f5ab36da134',
             'title': '18岁大一漂亮学妹，水嫩性感，再爽一次！',
-            'ext': 'mp4'
+            'ext': 'mp4',
+            'duration': 431,
         }
     }
 
@@ -46,8 +51,16 @@ class Porn91IE(InfoExtractor):
             "get real video url")
         video_url = self._search_regex(r'file=([^&]+)&', info_cn, 'url')
 
+        duration = parse_duration(self._search_regex(
+            r'时长:\s*</span>\s*(\d+:\d+)', webpage, 'duration', fatal=False))
+
+        comment_count = int_or_none(self._search_regex(
+            r'留言:\s*</span>\s*(\d+)', webpage, 'comment count', fatal=False))
+
         return {
             'id': video_id,
             'title': title,
             'url': video_url,
+            'duration': duration,
+            'comment_count': comment_count,
         }

From d05a1dbe7013d6314ec477b50d864726e509a872 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 31 May 2015 00:26:12 +0800
Subject: [PATCH 1447/6219] [porn91] Catch daily limit error

---
 youtube_dl/extractor/porn91.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/porn91.py b/youtube_dl/extractor/porn91.py
index ea1efc71b..c119c7e94 100644
--- a/youtube_dl/extractor/porn91.py
+++ b/youtube_dl/extractor/porn91.py
@@ -6,6 +6,7 @@ from .common import InfoExtractor
 from ..utils import (
     parse_duration,
     int_or_none,
+    ExtractorError,
 )
 
 
@@ -29,6 +30,10 @@ class Porn91IE(InfoExtractor):
         url = 'http://91porn.com/view_video.php?viewkey=%s' % video_id
         self._set_cookie('91porn.com', 'language', 'cn_CN')
         webpage = self._download_webpage(url, video_id, "get HTML content")
+
+        if '作为游客，你每天只可观看10个视频' in webpage:
+            raise ExtractorError('91 Porn says: Daily limit 10 videos exceeded', expected=True)
+
         title = self._search_regex(
             r'<div id="viewvideo-title">([^<]+)</div>', webpage, 'title')
         title = title.replace('\n', '')

From a2d971309b75c79f3f688a0c381707d828cb1026 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 31 May 2015 00:31:18 +0800
Subject: [PATCH 1448/6219] [porn91] Use single quotes

---
 youtube_dl/extractor/porn91.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/porn91.py b/youtube_dl/extractor/porn91.py
index c119c7e94..72d1b2718 100644
--- a/youtube_dl/extractor/porn91.py
+++ b/youtube_dl/extractor/porn91.py
@@ -29,7 +29,7 @@ class Porn91IE(InfoExtractor):
         video_id = self._match_id(url)
         url = 'http://91porn.com/view_video.php?viewkey=%s' % video_id
         self._set_cookie('91porn.com', 'language', 'cn_CN')
-        webpage = self._download_webpage(url, video_id, "get HTML content")
+        webpage = self._download_webpage(url, video_id, 'get HTML content')
 
         if '作为游客，你每天只可观看10个视频' in webpage:
             raise ExtractorError('91 Porn says: Daily limit 10 videos exceeded', expected=True)
@@ -53,7 +53,7 @@ class Porn91IE(InfoExtractor):
         })
         info_cn = self._download_webpage(
             'http://91porn.com/getfile.php?' + url_params, video_id,
-            "get real video url")
+            'get real video url')
         video_url = self._search_regex(r'file=([^&]+)&', info_cn, 'url')
 
         duration = parse_duration(self._search_regex(

From 931bc3c3a719fe33101c05b9fdc4e6ad8eb08bdc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 30 May 2015 22:52:02 +0600
Subject: [PATCH 1449/6219] [YoutubeDL] Do not loose request method information

---
 youtube_dl/YoutubeDL.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 21d247f23..5fc8754c6 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -49,6 +49,7 @@ from .utils import (
     ExtractorError,
     format_bytes,
     formatSeconds,
+    HEADRequest,
     locked_file,
     make_HTTPS_handler,
     MaxDownloadsReached,
@@ -1720,7 +1721,8 @@ class YoutubeDL(object):
             if req_is_string:
                 req = url_escaped
             else:
-                req = compat_urllib_request.Request(
+                req_type = HEADRequest if req.get_method() == 'HEAD' else compat_urllib_request.Request
+                req = req_type(
                     url_escaped, data=req.data, headers=req.headers,
                     origin_req_host=req.origin_req_host, unverifiable=req.unverifiable)
 

From 339516072be6865bf7e9316be81704ae69296c6b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 30 May 2015 23:16:14 +0600
Subject: [PATCH 1450/6219] [extractor/generic] Unescape video_id and title
 extracted from URL

---
 youtube_dl/extractor/generic.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 9a7b0d25d..c9c92d686 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -894,7 +894,7 @@ class GenericIE(InfoExtractor):
             force_videoid = smuggled_data['force_videoid']
             video_id = force_videoid
         else:
-            video_id = os.path.splitext(url.rstrip('/').split('/')[-1])[0]
+            video_id = os.path.splitext(compat_urllib_parse.unquote(url.rstrip('/').split('/')[-1]))[0]
 
         self.to_screen('%s: Requesting header' % video_id)
 
@@ -927,7 +927,7 @@ class GenericIE(InfoExtractor):
                 head_response.headers.get('Last-Modified'))
             return {
                 'id': video_id,
-                'title': os.path.splitext(url_basename(url))[0],
+                'title': os.path.splitext(compat_urllib_parse.unquote(url_basename(url)))[0],
                 'direct': True,
                 'formats': [{
                     'format_id': m.group('format_id'),
@@ -953,7 +953,7 @@ class GenericIE(InfoExtractor):
                 head_response.headers.get('Last-Modified'))
             return {
                 'id': video_id,
-                'title': os.path.splitext(url_basename(url))[0],
+                'title': os.path.splitext(compat_urllib_parse.unquote(url_basename(url)))[0],
                 'direct': True,
                 'url': url,
                 'upload_date': upload_date,

From 58bde34a236ff98f25fc109a94b3d393f0bbc9ee Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 31 May 2015 00:44:54 +0600
Subject: [PATCH 1451/6219] [extractor/generic] Force Accept-Encoding to any
 for extraction pass

---
 youtube_dl/extractor/generic.py | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index c9c92d686..ec1d9abbe 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -11,6 +11,7 @@ from ..compat import (
     compat_urllib_parse,
     compat_urlparse,
     compat_xml_parse_error,
+    compat_urllib_request,
 )
 from ..utils import (
     determine_ext,
@@ -916,7 +917,9 @@ class GenericIE(InfoExtractor):
 
         full_response = None
         if head_response is False:
-            full_response = self._request_webpage(url, video_id)
+            request = compat_urllib_request.Request(url)
+            request.add_header('Accept-Encoding', '*')
+            full_response = self._request_webpage(request, video_id)
             head_response = full_response
 
         # Check for direct link to a video
@@ -941,7 +944,17 @@ class GenericIE(InfoExtractor):
             self._downloader.report_warning('Falling back on generic information extractor.')
 
         if not full_response:
-            full_response = self._request_webpage(url, video_id)
+            request = compat_urllib_request.Request(url)
+            # Some webservers may serve compressed content of rather big size (e.g. gzipped flac)
+            # making it impossible to download only chunk of the file (yet we need only 512kB to
+            # test whether it's HTML or not). According to youtube-dl default Accept-Encoding
+            # that will always result in downloading the whole file that is not desirable.
+            # Therefore for extraction pass we have to override Accept-Encoding to any in order
+            # to accept raw bytes and being able to download only a chunk.
+            # It may probably better to solve this by checking Content-Type for application/octet-stream
+            # after HEAD request finishes, but not sure if we can rely on this.
+            request.add_header('Accept-Encoding', '*')
+            full_response = self._request_webpage(request, video_id)
 
         # Maybe it's a direct link to a video?
         # Be careful not to download the whole thing!

From 1ddb9456c4a63a207ec40bd74cdf0b36d8c68409 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 31 May 2015 01:23:58 +0600
Subject: [PATCH 1452/6219] [extractor/generic] Use compat_urllib_parse_unquote
 for unquoting video_id and title from URL

---
 youtube_dl/extractor/generic.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index ec1d9abbe..d9116ce10 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -9,9 +9,10 @@ from .common import InfoExtractor
 from .youtube import YoutubeIE
 from ..compat import (
     compat_urllib_parse,
+    compat_urllib_parse_unquote,
+    compat_urllib_request,
     compat_urlparse,
     compat_xml_parse_error,
-    compat_urllib_request,
 )
 from ..utils import (
     determine_ext,
@@ -895,7 +896,7 @@ class GenericIE(InfoExtractor):
             force_videoid = smuggled_data['force_videoid']
             video_id = force_videoid
         else:
-            video_id = os.path.splitext(compat_urllib_parse.unquote(url.rstrip('/').split('/')[-1]))[0]
+            video_id = compat_urllib_parse_unquote(os.path.splitext(url.rstrip('/').split('/')[-1])[0])
 
         self.to_screen('%s: Requesting header' % video_id)
 
@@ -930,7 +931,7 @@ class GenericIE(InfoExtractor):
                 head_response.headers.get('Last-Modified'))
             return {
                 'id': video_id,
-                'title': os.path.splitext(compat_urllib_parse.unquote(url_basename(url)))[0],
+                'title': compat_urllib_parse_unquote(os.path.splitext(url_basename(url))[0]),
                 'direct': True,
                 'formats': [{
                     'format_id': m.group('format_id'),
@@ -966,7 +967,7 @@ class GenericIE(InfoExtractor):
                 head_response.headers.get('Last-Modified'))
             return {
                 'id': video_id,
-                'title': os.path.splitext(compat_urllib_parse.unquote(url_basename(url)))[0],
+                'title': compat_urllib_parse_unquote(os.path.splitext(url_basename(url))[0]),
                 'direct': True,
                 'url': url,
                 'upload_date': upload_date,

From a074e922967fa571d4f1abb1773c711747060f00 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 31 May 2015 02:13:24 +0600
Subject: [PATCH 1453/6219] [extractor/generic] Add test for large compressed
 media

---
 youtube_dl/extractor/generic.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index d9116ce10..737141f95 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -138,6 +138,20 @@ class GenericIE(InfoExtractor):
                 'upload_date': '20100513',
             }
         },
+        # Direct link to a media delivered compressed (requires Accept-Encoding == *)
+        {
+            'url': 'http://calimero.tk/muzik/FictionJunction-Parallel_Hearts.flac',
+            'md5': '128c42e68b13950268b648275386fc74',
+            'info_dict': {
+                'id': 'FictionJunction-Parallel_Hearts',
+                'ext': 'flac',
+                'title': 'FictionJunction-Parallel_Hearts',
+                'upload_date': '20140522',
+            },
+            'expected_warnings': [
+                'URL could be a direct video link, returning it as such.'
+            ]
+        },
         # ooyala video
         {
             'url': 'http://www.rollingstone.com/music/videos/norwegian-dj-cashmere-cat-goes-spartan-on-with-me-premiere-20131219',

From c5fa81fe81ce05cd81c20ff4ea6dac3dccdcbf9d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 31 May 2015 02:22:29 +0600
Subject: [PATCH 1454/6219] [extractor/generic] Put all direct link tests near
 to each other for better navigation

---
 youtube_dl/extractor/generic.py | 182 ++++++++++++++++----------------
 1 file changed, 91 insertions(+), 91 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 737141f95..8f2e53063 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -48,6 +48,97 @@ class GenericIE(InfoExtractor):
     _VALID_URL = r'.*'
     IE_NAME = 'generic'
     _TESTS = [
+        # Direct link to a video
+        {
+            'url': 'http://media.w3.org/2010/05/sintel/trailer.mp4',
+            'md5': '67d406c2bcb6af27fa886f31aa934bbe',
+            'info_dict': {
+                'id': 'trailer',
+                'ext': 'mp4',
+                'title': 'trailer',
+                'upload_date': '20100513',
+            }
+        },
+        # Direct link to a media delivered compressed (requires Accept-Encoding == *)
+        {
+            'url': 'http://calimero.tk/muzik/FictionJunction-Parallel_Hearts.flac',
+            'md5': '128c42e68b13950268b648275386fc74',
+            'info_dict': {
+                'id': 'FictionJunction-Parallel_Hearts',
+                'ext': 'flac',
+                'title': 'FictionJunction-Parallel_Hearts',
+                'upload_date': '20140522',
+            },
+            'expected_warnings': [
+                'URL could be a direct video link, returning it as such.'
+            ]
+        },
+        # Direct download with broken HEAD
+        {
+            'url': 'http://ai-radio.org:8000/radio.opus',
+            'info_dict': {
+                'id': 'radio',
+                'ext': 'opus',
+                'title': 'radio',
+            },
+            'params': {
+                'skip_download': True,  # infinite live stream
+            },
+            'expected_warnings': [
+                r'501.*Not Implemented'
+            ],
+        },
+        # Direct link with incorrect MIME type
+        {
+            'url': 'http://ftp.nluug.nl/video/nluug/2014-11-20_nj14/zaal-2/5_Lennart_Poettering_-_Systemd.webm',
+            'md5': '4ccbebe5f36706d85221f204d7eb5913',
+            'info_dict': {
+                'url': 'http://ftp.nluug.nl/video/nluug/2014-11-20_nj14/zaal-2/5_Lennart_Poettering_-_Systemd.webm',
+                'id': '5_Lennart_Poettering_-_Systemd',
+                'ext': 'webm',
+                'title': '5_Lennart_Poettering_-_Systemd',
+                'upload_date': '20141120',
+            },
+            'expected_warnings': [
+                'URL could be a direct video link, returning it as such.'
+            ]
+        },
+        # RSS feed
+        {
+            'url': 'http://phihag.de/2014/youtube-dl/rss2.xml',
+            'info_dict': {
+                'id': 'http://phihag.de/2014/youtube-dl/rss2.xml',
+                'title': 'Zero Punctuation',
+                'description': 're:.*groundbreaking video review series.*'
+            },
+            'playlist_mincount': 11,
+        },
+        # RSS feed with enclosure
+        {
+            'url': 'http://podcastfeeds.nbcnews.com/audio/podcast/MSNBC-MADDOW-NETCAST-M4V.xml',
+            'info_dict': {
+                'id': 'pdv_maddow_netcast_m4v-02-27-2015-201624',
+                'ext': 'm4v',
+                'upload_date': '20150228',
+                'title': 'pdv_maddow_netcast_m4v-02-27-2015-201624',
+            }
+        },
+        # google redirect
+        {
+            'url': 'http://www.google.com/url?sa=t&rct=j&q=&esrc=s&source=web&cd=1&cad=rja&ved=0CCUQtwIwAA&url=http%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DcmQHVoWB5FY&ei=F-sNU-LLCaXk4QT52ICQBQ&usg=AFQjCNEw4hL29zgOohLXvpJ-Bdh2bils1Q&bvm=bv.61965928,d.bGE',
+            'info_dict': {
+                'id': 'cmQHVoWB5FY',
+                'ext': 'mp4',
+                'upload_date': '20130224',
+                'uploader_id': 'TheVerge',
+                'description': 're:^Chris Ziegler takes a look at the\.*',
+                'uploader': 'The Verge',
+                'title': 'First Firefox OS phones side-by-side',
+            },
+            'params': {
+                'skip_download': False,
+            }
+        },
         {
             'url': 'http://www.hodiho.fr/2013/02/regis-plante-sa-jeep.html',
             'md5': '85b90ccc9d73b4acd9138d3af4c27f89',
@@ -127,31 +218,6 @@ class GenericIE(InfoExtractor):
                 'skip_download': True,  # m3u8 download
             },
         },
-        # Direct link to a video
-        {
-            'url': 'http://media.w3.org/2010/05/sintel/trailer.mp4',
-            'md5': '67d406c2bcb6af27fa886f31aa934bbe',
-            'info_dict': {
-                'id': 'trailer',
-                'ext': 'mp4',
-                'title': 'trailer',
-                'upload_date': '20100513',
-            }
-        },
-        # Direct link to a media delivered compressed (requires Accept-Encoding == *)
-        {
-            'url': 'http://calimero.tk/muzik/FictionJunction-Parallel_Hearts.flac',
-            'md5': '128c42e68b13950268b648275386fc74',
-            'info_dict': {
-                'id': 'FictionJunction-Parallel_Hearts',
-                'ext': 'flac',
-                'title': 'FictionJunction-Parallel_Hearts',
-                'upload_date': '20140522',
-            },
-            'expected_warnings': [
-                'URL could be a direct video link, returning it as such.'
-            ]
-        },
         # ooyala video
         {
             'url': 'http://www.rollingstone.com/music/videos/norwegian-dj-cashmere-cat-goes-spartan-on-with-me-premiere-20131219',
@@ -176,22 +242,6 @@ class GenericIE(InfoExtractor):
             },
             'add_ie': ['Ooyala'],
         },
-        # google redirect
-        {
-            'url': 'http://www.google.com/url?sa=t&rct=j&q=&esrc=s&source=web&cd=1&cad=rja&ved=0CCUQtwIwAA&url=http%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DcmQHVoWB5FY&ei=F-sNU-LLCaXk4QT52ICQBQ&usg=AFQjCNEw4hL29zgOohLXvpJ-Bdh2bils1Q&bvm=bv.61965928,d.bGE',
-            'info_dict': {
-                'id': 'cmQHVoWB5FY',
-                'ext': 'mp4',
-                'upload_date': '20130224',
-                'uploader_id': 'TheVerge',
-                'description': 're:^Chris Ziegler takes a look at the\.*',
-                'uploader': 'The Verge',
-                'title': 'First Firefox OS phones side-by-side',
-            },
-            'params': {
-                'skip_download': False,
-            }
-        },
         # embed.ly video
         {
             'url': 'http://www.tested.com/science/weird/460206-tested-grinding-coffee-2000-frames-second/',
@@ -423,16 +473,6 @@ class GenericIE(InfoExtractor):
                 'title': 'Busty Blonde Siri Tit Fuck While Wank at HandjobHub.com',
             }
         },
-        # RSS feed
-        {
-            'url': 'http://phihag.de/2014/youtube-dl/rss2.xml',
-            'info_dict': {
-                'id': 'http://phihag.de/2014/youtube-dl/rss2.xml',
-                'title': 'Zero Punctuation',
-                'description': 're:.*groundbreaking video review series.*'
-            },
-            'playlist_mincount': 11,
-        },
         # Multiple brightcove videos
         # https://github.com/rg3/youtube-dl/issues/2283
         {
@@ -486,21 +526,6 @@ class GenericIE(InfoExtractor):
                 'uploader': 'thoughtworks.wistia.com',
             },
         },
-        # Direct download with broken HEAD
-        {
-            'url': 'http://ai-radio.org:8000/radio.opus',
-            'info_dict': {
-                'id': 'radio',
-                'ext': 'opus',
-                'title': 'radio',
-            },
-            'params': {
-                'skip_download': True,  # infinite live stream
-            },
-            'expected_warnings': [
-                r'501.*Not Implemented'
-            ],
-        },
         # Soundcloud embed
         {
             'url': 'http://nakedsecurity.sophos.com/2014/10/29/sscc-171-are-you-sure-that-1234-is-a-bad-password-podcast/',
@@ -532,21 +557,6 @@ class GenericIE(InfoExtractor):
             },
             'playlist_mincount': 2,
         },
-        # Direct link with incorrect MIME type
-        {
-            'url': 'http://ftp.nluug.nl/video/nluug/2014-11-20_nj14/zaal-2/5_Lennart_Poettering_-_Systemd.webm',
-            'md5': '4ccbebe5f36706d85221f204d7eb5913',
-            'info_dict': {
-                'url': 'http://ftp.nluug.nl/video/nluug/2014-11-20_nj14/zaal-2/5_Lennart_Poettering_-_Systemd.webm',
-                'id': '5_Lennart_Poettering_-_Systemd',
-                'ext': 'webm',
-                'title': '5_Lennart_Poettering_-_Systemd',
-                'upload_date': '20141120',
-            },
-            'expected_warnings': [
-                'URL could be a direct video link, returning it as such.'
-            ]
-        },
         # Cinchcast embed
         {
             'url': 'http://undergroundwellness.com/podcasts/306-5-steps-to-permanent-gut-healing/',
@@ -705,16 +715,6 @@ class GenericIE(InfoExtractor):
                 'age_limit': 0,
             },
         },
-        # RSS feed with enclosure
-        {
-            'url': 'http://podcastfeeds.nbcnews.com/audio/podcast/MSNBC-MADDOW-NETCAST-M4V.xml',
-            'info_dict': {
-                'id': 'pdv_maddow_netcast_m4v-02-27-2015-201624',
-                'ext': 'm4v',
-                'upload_date': '20150228',
-                'title': 'pdv_maddow_netcast_m4v-02-27-2015-201624',
-            }
-        },
         # Crooks and Liars embed
         {
             'url': 'http://crooksandliars.com/2015/04/fox-friends-says-protecting-atheists',

From c5138a7ce49db19b64adc11d81384595b966a7a9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 31 May 2015 02:36:20 +0600
Subject: [PATCH 1455/6219] [extractor/generic] Clarify test comment

---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 8f2e53063..96ca398de 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -59,7 +59,7 @@ class GenericIE(InfoExtractor):
                 'upload_date': '20100513',
             }
         },
-        # Direct link to a media delivered compressed (requires Accept-Encoding == *)
+        # Direct link to media delivered compressed (until Accept-Encoding is *)
         {
             'url': 'http://calimero.tk/muzik/FictionJunction-Parallel_Hearts.flac',
             'md5': '128c42e68b13950268b648275386fc74',

From d2a9de78dfc629aaaaf8a2a30432d5f02c949e9a Mon Sep 17 00:00:00 2001
From: Ivan Kozik <ivan@ludios.org>
Date: Sat, 30 May 2015 20:50:22 +0000
Subject: [PATCH 1456/6219] [youtube] Construct a playlist URL in case the page
 is missing one

This fixes jumping from user/channel -> playlist for some users like
https://www.youtube.com/user/BitcoinFoundation

This also removes the superfluous log message
"add --no-playlist to just download video VIDEOID"
when downloading a user/channel.
---
 youtube_dl/extractor/youtube.py | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index fcdbfe0bc..aacb999ce 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1412,12 +1412,8 @@ class YoutubeChannelIE(InfoExtractor):
             channel_page, 'channel id', default=None)
         if channel_playlist_id and channel_playlist_id.startswith('UC'):
             playlist_id = 'UU' + channel_playlist_id[2:]
-            channel_playlist = unescapeHTML(self._search_regex(
-                r'href="/?(watch\?v=[0-9A-Za-z_-]{11}&amp;list=%s)"' % playlist_id,
-                channel_page, 'channel playlist URL', default=None))
-            if channel_playlist:
-                return self.url_result(
-                    compat_urlparse.urljoin(url, '/%s' % channel_playlist), 'YoutubePlaylist')
+            return self.url_result(
+                compat_urlparse.urljoin(url, '/playlist?list=%s' % playlist_id), 'YoutubePlaylist')
 
         channel_page = self._download_webpage(url, channel_id, 'Downloading page #1')
         autogenerated = re.search(r'''(?x)

From eb47569f8a6017190d73429b3ef54c1ffaf201dc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 31 May 2015 03:00:13 +0600
Subject: [PATCH 1457/6219] [tvigle] Add support for m3u8

---
 youtube_dl/extractor/tvigle.py | 25 ++++++++++++++++---------
 1 file changed, 16 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/tvigle.py b/youtube_dl/extractor/tvigle.py
index 102362b29..4e95bd30f 100644
--- a/youtube_dl/extractor/tvigle.py
+++ b/youtube_dl/extractor/tvigle.py
@@ -6,6 +6,7 @@ import re
 from .common import InfoExtractor
 from ..utils import (
     float_or_none,
+    int_or_none,
     parse_age_limit,
 )
 
@@ -24,17 +25,17 @@ class TvigleIE(InfoExtractor):
                 'display_id': 'sokrat',
                 'ext': 'flv',
                 'title': 'Сократ',
-                'description': 'md5:a05bd01be310074d5833efc6743be95e',
+                'description': 'md5:d6b92ffb7217b4b8ebad2e7665253c17',
                 'duration': 6586,
-                'age_limit': 0,
+                'age_limit': 12,
             },
         },
         {
             'url': 'http://www.tvigle.ru/video/vladimir-vysotskii/vedushchii-teleprogrammy-60-minut-ssha-o-vladimire-vysotskom/',
-            'md5': 'd9012d7c7c598fe7a11d7fb46dc1f574',
+            'md5': 'e7efe5350dd5011d0de6550b53c3ba7b',
             'info_dict': {
                 'id': '5142516',
-                'ext': 'mp4',
+                'ext': 'flv',
                 'title': 'Ведущий телепрограммы «60 минут» (США) о Владимире Высоцком',
                 'description': 'md5:027f7dc872948f14c96d19b4178428a4',
                 'duration': 186.080,
@@ -54,7 +55,7 @@ class TvigleIE(InfoExtractor):
         if not video_id:
             webpage = self._download_webpage(url, display_id)
             video_id = self._html_search_regex(
-                r'<li class="video-preview current_playing" id="(\d+)">',
+                r'class="video-preview current_playing" id="(\d+)">',
                 webpage, 'video id')
 
         video_data = self._download_json(
@@ -70,13 +71,19 @@ class TvigleIE(InfoExtractor):
 
         formats = []
         for vcodec, fmts in item['videos'].items():
-            for quality, video_url in fmts.items():
+            for format_id, video_url in fmts.items():
+                if format_id == 'm3u8':
+                    formats.extend(self._extract_m3u8_formats(
+                        video_url, video_id, 'mp4', m3u8_id=vcodec))
+                    continue
+                height = self._search_regex(
+                    r'^(\d+)[pP]$', format_id, 'height', default=None)
                 formats.append({
                     'url': video_url,
-                    'format_id': '%s-%s' % (vcodec, quality),
+                    'format_id': '%s-%s' % (vcodec, format_id),
                     'vcodec': vcodec,
-                    'height': int(quality[:-1]),
-                    'filesize': item['video_files_size'][vcodec][quality],
+                    'height': int_or_none(height),
+                    'filesize': item['video_files_size'][vcodec][format_id],
                 })
         self._sort_formats(formats)
 

From 7584e38ce4e98e0e9abca146a513d215701308e2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 31 May 2015 03:01:41 +0600
Subject: [PATCH 1458/6219] [tvigle] Modernize

---
 youtube_dl/extractor/tvigle.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/tvigle.py b/youtube_dl/extractor/tvigle.py
index 4e95bd30f..a85693888 100644
--- a/youtube_dl/extractor/tvigle.py
+++ b/youtube_dl/extractor/tvigle.py
@@ -64,8 +64,8 @@ class TvigleIE(InfoExtractor):
         item = video_data['playlist']['items'][0]
 
         title = item['title']
-        description = item['description']
-        thumbnail = item['thumbnail']
+        description = item.get('description')
+        thumbnail = item.get('thumbnail')
         duration = float_or_none(item.get('durationMilliseconds'), 1000)
         age_limit = parse_age_limit(item.get('ageRestrictions'))
 
@@ -83,7 +83,7 @@ class TvigleIE(InfoExtractor):
                     'format_id': '%s-%s' % (vcodec, format_id),
                     'vcodec': vcodec,
                     'height': int_or_none(height),
-                    'filesize': item['video_files_size'][vcodec][format_id],
+                    'filesize': int_or_none(item.get('video_files_size', {}).get(vcodec, {}).get(format_id)),
                 })
         self._sort_formats(formats)
 

From df15ef8dab6df79eb076b3d06b3948917763ac3a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 31 May 2015 04:05:09 +0600
Subject: [PATCH 1459/6219] [YoutubeDL] Tweak select_format for video only
 media

---
 youtube_dl/YoutubeDL.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 5fc8754c6..aa6ec9d9a 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -924,8 +924,9 @@ class YoutubeDL(object):
                 if f.get('vcodec') != 'none' and f.get('acodec') != 'none']
             if audiovideo_formats:
                 return audiovideo_formats[format_idx]
-            # for audio only urls, select the best/worst audio format
-            elif all(f.get('acodec') != 'none' for f in available_formats):
+            # for audio only (soundcloud) or video only (imgur) urls, select the best/worst audio format
+            elif (all(f.get('acodec') != 'none' for f in available_formats) or
+                  all(f.get('vcodec') != 'none' for f in available_formats)):
                 return available_formats[format_idx]
         elif format_spec == 'bestaudio':
             audio_formats = [

From 96b9690985e9b9f4e50fde10bbc92e1a72df64e4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 31 May 2015 04:05:26 +0600
Subject: [PATCH 1460/6219] [imgur] Improve extraction

---
 youtube_dl/extractor/imgur.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/imgur.py b/youtube_dl/extractor/imgur.py
index fe5d95e2c..d692ea79a 100644
--- a/youtube_dl/extractor/imgur.py
+++ b/youtube_dl/extractor/imgur.py
@@ -3,6 +3,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import compat_urlparse
 from ..utils import (
     int_or_none,
     js_to_json,
@@ -12,7 +13,7 @@ from ..utils import (
 
 
 class ImgurIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:i\.)?imgur\.com/(?P<id>[a-zA-Z0-9]+)(?:\.mp4|\.gifv)?'
+    _VALID_URL = r'https?://(?:i\.)?imgur\.com/(?P<id>[a-zA-Z0-9]+)'
 
     _TESTS = [{
         'url': 'https://i.imgur.com/A61SaA1.gifv',
@@ -34,7 +35,8 @@ class ImgurIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
+        webpage = self._download_webpage(
+            compat_urlparse.urljoin(url, video_id), video_id)
 
         width = int_or_none(self._search_regex(
             r'<param name="width" value="([0-9]+)"',

From 47fd8c2f761c2073744cb041f9eccb7ed10f2470 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 1 Jun 2015 00:04:36 +0600
Subject: [PATCH 1461/6219] [patreon] Fix embeds extraction (Closes #5862)

---
 youtube_dl/extractor/patreon.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/patreon.py b/youtube_dl/extractor/patreon.py
index f179ea200..6cdc2638b 100644
--- a/youtube_dl/extractor/patreon.py
+++ b/youtube_dl/extractor/patreon.py
@@ -87,7 +87,7 @@ class PatreonIE(InfoExtractor):
             r'<div class="attach"><a target="_blank" href="([^"]+)">',
             webpage, 'attachment URL', default=None)
         embed = self._html_search_regex(
-            r'<div id="watchCreation">\s*<iframe class="embedly-embed" src="([^"]+)"',
+            r'<div[^>]+id="watchCreation"[^>]*>\s*<iframe[^>]+src="([^"]+)"',
             webpage, 'embedded URL', default=None)
 
         if attach_fn is not None:

From 4053ee9104cd7669f749a267dc2c2a1725ca188b Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 1 Jun 2015 14:43:20 +0800
Subject: [PATCH 1462/6219] Credit @PeterDing for 91porn extractor (#5830)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 3410e1fb9..bf2a25cb8 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -126,3 +126,4 @@ Matthias Küch
 Julian Richen
 Ping O.
 Mister Hat
+Peter Ding

From 866b296d0f156831cceccc967c34382a90b77422 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Mon, 1 Jun 2015 16:11:19 +0300
Subject: [PATCH 1463/6219] [aftonbladet] Fix extraction and update _VALID_URL
 (Fixes #5863)

---
 youtube_dl/extractor/aftonbladet.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/aftonbladet.py b/youtube_dl/extractor/aftonbladet.py
index a117502bc..e0518cf26 100644
--- a/youtube_dl/extractor/aftonbladet.py
+++ b/youtube_dl/extractor/aftonbladet.py
@@ -6,11 +6,11 @@ from ..utils import int_or_none
 
 
 class AftonbladetIE(InfoExtractor):
-    _VALID_URL = r'http://tv\.aftonbladet\.se/webbtv.+?(?P<id>article[0-9]+)\.ab(?:$|[?#])'
+    _VALID_URL = r'http://tv\.aftonbladet\.se/abtv/articles/(?P<id>[0-9]+)'
     _TEST = {
-        'url': 'http://tv.aftonbladet.se/webbtv/nyheter/vetenskap/rymden/article36015.ab',
+        'url': 'http://tv.aftonbladet.se/abtv/articles/36015',
         'info_dict': {
-            'id': 'article36015',
+            'id': '36015',
             'ext': 'mp4',
             'title': 'Vulkanutbrott i rymden - nu släpper NASA bilderna',
             'description': 'Jupiters måne mest aktiv av alla himlakroppar',
@@ -25,8 +25,9 @@ class AftonbladetIE(InfoExtractor):
 
         # find internal video meta data
         meta_url = 'http://aftonbladet-play.drlib.aptoma.no/video/%s.json'
-        internal_meta_id = self._html_search_regex(
-            r'data-aptomaId="([\w\d]+)"', webpage, 'internal_meta_id')
+        player_config = self._parse_json(self._html_search_regex(
+            r'data-player-config="([^"]+)"', webpage, 'player config'), video_id)
+        internal_meta_id = player_config['videoId']
         internal_meta_url = meta_url % internal_meta_id
         internal_meta_json = self._download_json(
             internal_meta_url, video_id, 'Downloading video meta data')

From 923e79e2e4d9cc0c24496614aab520737cdc89ec Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 2 Jun 2015 00:53:04 +0600
Subject: [PATCH 1464/6219] [nova] Add extractor

---
 youtube_dl/extractor/__init__.py |   1 +
 youtube_dl/extractor/nova.py     | 135 +++++++++++++++++++++++++++++++
 2 files changed, 136 insertions(+)
 create mode 100644 youtube_dl/extractor/nova.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 4dc07efe0..67eb96057 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -352,6 +352,7 @@ from .ninegag import NineGagIE
 from .noco import NocoIE
 from .normalboots import NormalbootsIE
 from .nosvideo import NosVideoIE
+from .nova import NovaIE
 from .novamov import NovaMovIE
 from .nowness import NownessIE
 from .nowtv import NowTVIE
diff --git a/youtube_dl/extractor/nova.py b/youtube_dl/extractor/nova.py
new file mode 100644
index 000000000..e93a7ffa8
--- /dev/null
+++ b/youtube_dl/extractor/nova.py
@@ -0,0 +1,135 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+
+class NovaIE(InfoExtractor):
+    IE_DESC = 'TN.cz, Prásk.tv, Nova.cz, Novaplus.cz, FANDA.tv, Krásná.cz and Doma.cz'
+    _VALID_URL = 'http://(?:[^.]+\.)?(?P<site>tv(?:noviny)?|tn|novaplus|vymena|fanda|krasna|doma|prask)\.nova\.cz/(?:[^/]+/)+(?P<id>[^/]+)(?:\.html|/?)'
+    _TESTS = [{
+        'url': 'http://tvnoviny.nova.cz/clanek/novinky/co-na-sebe-sportaci-praskli-vime-jestli-pujde-hrdlicka-na-materskou.html',
+        'info_dict': {
+            'id': '1608920',
+            'display_id': 'co-na-sebe-sportaci-praskli-vime-jestli-pujde-hrdlicka-na-materskou',
+            'ext': 'flv',
+            'title': 'Duel: Michal Hrdlička a Petr Suchoň',
+            'description': 'md5:d0cc509858eee1b1374111c588c6f5d5',
+            'thumbnail': 're:^https?://.*\.(?:jpg)',
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        }
+    }, {
+        'url': 'http://tn.nova.cz/clanek/tajemstvi-ukryte-v-podzemi-specialni-nemocnice-v-prazske-krci.html',
+        'md5': '1dd7b9d5ea27bc361f110cd855a19bd3',
+        'info_dict': {
+            'id': '1757139',
+            'display_id': 'tajemstvi-ukryte-v-podzemi-specialni-nemocnice-v-prazske-krci',
+            'ext': 'mp4',
+            'title': 'Podzemní nemocnice v pražské Krči',
+            'description': 'md5:f0a42dd239c26f61c28f19e62d20ef53',
+            'thumbnail': 're:^https?://.*\.(?:jpg)',
+        }
+    }, {
+        'url': 'http://novaplus.nova.cz/porad/policie-modrava/video/5591-policie-modrava-15-dil-blondynka-na-hrbitove/',
+        'info_dict': {
+            'id': '1756825',
+            'display_id': '5591-policie-modrava-15-dil-blondynka-na-hrbitove',
+            'ext': 'flv',
+            'title': 'Policie Modrava - 15. díl - Blondýnka na hřbitově',
+            'description': 'md5:d804ba6b30bc7da2705b1fea961bddfe',
+            'thumbnail': 're:^https?://.*\.(?:jpg)',
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        }
+    }, {
+        'url': 'http://sport.tn.nova.cz/clanek/sport/hokej/nhl/zivot-jde-dal-hodnotil-po-vyrazeni-z-playoff-jiri-sekac.html',
+        'only_matching': True,
+    }, {
+        'url': 'http://fanda.nova.cz/clanek/fun-and-games/krvavy-epos-zaklinac-3-divoky-hon-vychazi-vyhrajte-ho-pro-sebe.html',
+        'only_matching': True,
+    }, {
+        'url': 'http://doma.nova.cz/clanek/zdravi/prijdte-se-zapsat-do-registru-kostni-drene-jiz-ve-stredu-3-cervna.html',
+        'only_matching': True,
+    }, {
+        'url': 'http://prask.nova.cz/clanek/novinky/co-si-na-sobe-nase-hvezdy-nechaly-pojistit.html',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        display_id = mobj.group('id')
+        site = mobj.group('site')
+
+        webpage = self._download_webpage(url, display_id)
+
+        video_id = self._search_regex(
+            [r"(?:media|video_id)\s*:\s*'(\d+)'",
+             r'media=(\d+)',
+             r'id="article_video_(\d+)"',
+             r'id="player_(\d+)"'],
+            webpage, 'video id')
+
+        config_url = self._search_regex(
+            r'src="(http://tn\.nova\.cz/bin/player/videojs/config\.php\?[^"]+)"',
+            webpage, 'config url', default=None)
+
+        if not config_url:
+            DEFAULT_SITE_ID = '23000'
+            SITES = {
+                'tvnoviny': DEFAULT_SITE_ID,
+                'novaplus': DEFAULT_SITE_ID,
+                'vymena': DEFAULT_SITE_ID,
+                'krasna': DEFAULT_SITE_ID,
+                'fanda': '30',
+                'tn': '30',
+                'doma': '30',
+            }
+
+            site_id = self._search_regex(
+                r'site=(\d+)', webpage, 'site id', default=None) or SITES.get(site, DEFAULT_SITE_ID)
+
+            config_url = ('http://tn.nova.cz/bin/player/videojs/config.php?site=%s&media=%s&jsVar=vjsconfig'
+                          % (site_id, video_id))
+
+        config = self._download_json(
+            config_url, display_id,
+            'Downloading config JSON',
+            transform_source=lambda s: re.sub(r'var\s+[\da-zA-Z_]+\s*=\s*({.+?});', r'\1', s))
+
+        mediafile = config['mediafile']
+        video_url = mediafile['src']
+
+        m = re.search(r'^(?P<url>rtmpe?://[^/]+/(?P<app>[^/]+?))/&*(?P<playpath>.+)$', video_url)
+        if m:
+            formats = [{
+                'url': m.group('url'),
+                'app': m.group('app'),
+                'play_path': m.group('playpath'),
+                'player_path': 'http://tvnoviny.nova.cz/static/shared/app/videojs/video-js.swf',
+                'ext': 'flv',
+            }]
+        else:
+            formats = [{
+                'url': video_url,
+            }]
+        self._sort_formats(formats)
+
+        title = mediafile.get('meta', {}).get('title') or self._og_search_title(webpage)
+        description = self._og_search_description(webpage)
+        thumbnail = config.get('poster')
+
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'formats': formats,
+        }

From 60158217ef8da5f44ef316e50c8a5e2ac2e202c5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 2 Jun 2015 00:57:08 +0600
Subject: [PATCH 1465/6219] [nova] Add tv test

---
 youtube_dl/extractor/nova.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/nova.py b/youtube_dl/extractor/nova.py
index e93a7ffa8..4e999b237 100644
--- a/youtube_dl/extractor/nova.py
+++ b/youtube_dl/extractor/nova.py
@@ -60,6 +60,9 @@ class NovaIE(InfoExtractor):
     }, {
         'url': 'http://prask.nova.cz/clanek/novinky/co-si-na-sobe-nase-hvezdy-nechaly-pojistit.html',
         'only_matching': True,
+    }, {
+        'url': 'http://tv.nova.cz/clanek/novinky/zivot-je-zivot-bondovsky-trailer.html',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 9f4b9118ccaef5cd7c414a78c5622968e8c3343f Mon Sep 17 00:00:00 2001
From: Slava Shklyaev <shk.slava@gmail.com>
Date: Tue, 2 Jun 2015 12:47:52 +0300
Subject: [PATCH 1466/6219] [nova] Fix display_id extraction bug

Make id group non-greedy so that .html is not included in it.
---
 youtube_dl/extractor/nova.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nova.py b/youtube_dl/extractor/nova.py
index 4e999b237..1dd18511e 100644
--- a/youtube_dl/extractor/nova.py
+++ b/youtube_dl/extractor/nova.py
@@ -8,7 +8,7 @@ from .common import InfoExtractor
 
 class NovaIE(InfoExtractor):
     IE_DESC = 'TN.cz, Prásk.tv, Nova.cz, Novaplus.cz, FANDA.tv, Krásná.cz and Doma.cz'
-    _VALID_URL = 'http://(?:[^.]+\.)?(?P<site>tv(?:noviny)?|tn|novaplus|vymena|fanda|krasna|doma|prask)\.nova\.cz/(?:[^/]+/)+(?P<id>[^/]+)(?:\.html|/?)'
+    _VALID_URL = 'http://(?:[^.]+\.)?(?P<site>tv(?:noviny)?|tn|novaplus|vymena|fanda|krasna|doma|prask)\.nova\.cz/(?:[^/]+/)+(?P<id>[^/]+?)(?:\.html|/?)$'
     _TESTS = [{
         'url': 'http://tvnoviny.nova.cz/clanek/novinky/co-na-sebe-sportaci-praskli-vime-jestli-pujde-hrdlicka-na-materskou.html',
         'info_dict': {

From 9464a194dbf48989c486fa2de9e1aebc59e28ed4 Mon Sep 17 00:00:00 2001
From: Slava Shklyaev <shk.slava@gmail.com>
Date: Tue, 2 Jun 2015 12:52:39 +0300
Subject: [PATCH 1467/6219] [nova] Fix extension extraction bug

Replace the hardcoded flv with determine_ext. Let rtmpdump parse the url.
---
 youtube_dl/extractor/nova.py | 23 ++++++-----------------
 1 file changed, 6 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/nova.py b/youtube_dl/extractor/nova.py
index 1dd18511e..fd5f9cb0e 100644
--- a/youtube_dl/extractor/nova.py
+++ b/youtube_dl/extractor/nova.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..utils import determine_ext
 
 
 class NovaIE(InfoExtractor):
@@ -39,7 +40,7 @@ class NovaIE(InfoExtractor):
         'info_dict': {
             'id': '1756825',
             'display_id': '5591-policie-modrava-15-dil-blondynka-na-hrbitove',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Policie Modrava - 15. díl - Blondýnka na hřbitově',
             'description': 'md5:d804ba6b30bc7da2705b1fea961bddfe',
             'thumbnail': 're:^https?://.*\.(?:jpg)',
@@ -108,21 +109,8 @@ class NovaIE(InfoExtractor):
 
         mediafile = config['mediafile']
         video_url = mediafile['src']
-
-        m = re.search(r'^(?P<url>rtmpe?://[^/]+/(?P<app>[^/]+?))/&*(?P<playpath>.+)$', video_url)
-        if m:
-            formats = [{
-                'url': m.group('url'),
-                'app': m.group('app'),
-                'play_path': m.group('playpath'),
-                'player_path': 'http://tvnoviny.nova.cz/static/shared/app/videojs/video-js.swf',
-                'ext': 'flv',
-            }]
-        else:
-            formats = [{
-                'url': video_url,
-            }]
-        self._sort_formats(formats)
+        ext = determine_ext(video_url)
+        video_url = video_url.replace('&{}:'.format(ext), '')
 
         title = mediafile.get('meta', {}).get('title') or self._og_search_title(webpage)
         description = self._og_search_description(webpage)
@@ -134,5 +122,6 @@ class NovaIE(InfoExtractor):
             'title': title,
             'description': description,
             'thumbnail': thumbnail,
-            'formats': formats,
+            'url': video_url,
+            'ext': ext,
         }

From fcb04bcaca1b83cd3f13f494d7d775e35e0b6182 Mon Sep 17 00:00:00 2001
From: Slava Shklyaev <shk.slava@gmail.com>
Date: Tue, 2 Jun 2015 12:55:41 +0300
Subject: [PATCH 1468/6219] [nova] Extract upload_date in some cases

---
 youtube_dl/extractor/nova.py | 39 ++++++++++++++++++++++++++++++++++++
 1 file changed, 39 insertions(+)

diff --git a/youtube_dl/extractor/nova.py b/youtube_dl/extractor/nova.py
index fd5f9cb0e..30c64aaf8 100644
--- a/youtube_dl/extractor/nova.py
+++ b/youtube_dl/extractor/nova.py
@@ -49,6 +49,33 @@ class NovaIE(InfoExtractor):
             # rtmp download
             'skip_download': True,
         }
+    }, {
+        'url': 'http://novaplus.nova.cz/porad/televizni-noviny/video/5585-televizni-noviny-30-5-2015/',
+        'info_dict': {
+            'id': '1756858',
+            'ext': 'mp4',
+            'title': 'Televizní noviny - 30. 5. 2015',
+            'thumbnail': 're:^https?://.*\.(?:jpg)',
+            'upload_date': '20150530',
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        }
+    }, {
+        'url': 'http://fanda.nova.cz/clanek/fun-and-games/krvavy-epos-zaklinac-3-divoky-hon-vychazi-vyhrajte-ho-pro-sebe.html',
+        'info_dict': {
+            'id': '1753621',
+            'ext': 'mp4',
+            'title': 'Zaklínač 3: Divoký hon',
+            'description': 're:.*Pokud se stejně jako my nemůžete.*',
+            'thumbnail': 're:https?://.*\.jpg(\?.*)?',
+            'upload_date': '20150521',
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        }
     }, {
         'url': 'http://sport.tn.nova.cz/clanek/sport/hokej/nhl/zivot-jde-dal-hodnotil-po-vyrazeni-z-playoff-jiri-sekac.html',
         'only_matching': True,
@@ -116,11 +143,23 @@ class NovaIE(InfoExtractor):
         description = self._og_search_description(webpage)
         thumbnail = config.get('poster')
 
+        mobj = None
+        if site == 'novaplus':
+            mobj = re.search(r'(?P<day>\d{1,2})-(?P<month>\d{1,2})-(?P<year>\d{4})$', display_id)
+        if site == 'fanda':
+            mobj = re.search(
+                r'<span class="date_time">(?P<day>\d{1,2})\.(?P<month>\d{1,2})\.(?P<year>\d{4})\b', webpage)
+        if mobj:
+            upload_date = '{}{:02d}{:02d}'.format(mobj.group('year'), int(mobj.group('month')), int(mobj.group('day')))
+        else:
+            upload_date = None
+
         return {
             'id': video_id,
             'display_id': display_id,
             'title': title,
             'description': description,
+            'upload_date': upload_date,
             'thumbnail': thumbnail,
             'url': video_url,
             'ext': ext,

From 34c0f95db273ac5e7a7f8a6d23a3f90ceadf4695 Mon Sep 17 00:00:00 2001
From: Slava Shklyaev <shk.slava@gmail.com>
Date: Tue, 2 Jun 2015 12:56:36 +0300
Subject: [PATCH 1469/6219] [nova] Remove html tags from description

---
 youtube_dl/extractor/nova.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/nova.py b/youtube_dl/extractor/nova.py
index 30c64aaf8..140312f9c 100644
--- a/youtube_dl/extractor/nova.py
+++ b/youtube_dl/extractor/nova.py
@@ -4,7 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import determine_ext
+from ..utils import clean_html, determine_ext
 
 
 class NovaIE(InfoExtractor):
@@ -42,7 +42,7 @@ class NovaIE(InfoExtractor):
             'display_id': '5591-policie-modrava-15-dil-blondynka-na-hrbitove',
             'ext': 'mp4',
             'title': 'Policie Modrava - 15. díl - Blondýnka na hřbitově',
-            'description': 'md5:d804ba6b30bc7da2705b1fea961bddfe',
+            'description': 'md5:dc24e50be5908df83348e50d1431295e',
             'thumbnail': 're:^https?://.*\.(?:jpg)',
         },
         'params': {
@@ -140,7 +140,7 @@ class NovaIE(InfoExtractor):
         video_url = video_url.replace('&{}:'.format(ext), '')
 
         title = mediafile.get('meta', {}).get('title') or self._og_search_title(webpage)
-        description = self._og_search_description(webpage)
+        description = clean_html(self._og_search_description(webpage, default=None))
         thumbnail = config.get('poster')
 
         mobj = None

From a00234f1c517d077a237da576be638fef980d79e Mon Sep 17 00:00:00 2001
From: Slava Shklyaev <shk.slava@gmail.com>
Date: Tue, 2 Jun 2015 12:57:03 +0300
Subject: [PATCH 1470/6219] [nova] Minor style improvement

---
 youtube_dl/extractor/nova.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nova.py b/youtube_dl/extractor/nova.py
index 140312f9c..10957e5fa 100644
--- a/youtube_dl/extractor/nova.py
+++ b/youtube_dl/extractor/nova.py
@@ -132,7 +132,7 @@ class NovaIE(InfoExtractor):
         config = self._download_json(
             config_url, display_id,
             'Downloading config JSON',
-            transform_source=lambda s: re.sub(r'var\s+[\da-zA-Z_]+\s*=\s*({.+?});', r'\1', s))
+            transform_source=lambda s: s[s.index('{'):s.rindex('}') + 1])
 
         mediafile = config['mediafile']
         video_url = mediafile['src']

From bc03e58565d99677a643e0a058d25c7ee9b265d6 Mon Sep 17 00:00:00 2001
From: Slava Shklyaev <shk.slava@gmail.com>
Date: Tue, 2 Jun 2015 13:16:58 +0300
Subject: [PATCH 1471/6219] [iprima] Update

---
 youtube_dl/extractor/iprima.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/iprima.py b/youtube_dl/extractor/iprima.py
index 8529bedfc..f3536893a 100644
--- a/youtube_dl/extractor/iprima.py
+++ b/youtube_dl/extractor/iprima.py
@@ -15,7 +15,7 @@ from ..utils import (
 
 
 class IPrimaIE(InfoExtractor):
-    _VALID_URL = r'https?://play\.iprima\.cz/[^?#]+/(?P<id>[^?#]+)'
+    _VALID_URL = r'https?://play\.iprima\.cz/([^/]+/)*(?P<id>[^?#]+)'
 
     _TESTS = [{
         'url': 'http://play.iprima.cz/particka/particka-92',
@@ -23,7 +23,7 @@ class IPrimaIE(InfoExtractor):
             'id': '39152',
             'ext': 'flv',
             'title': 'Partička (92)',
-            'description': 'md5:3740fda51464da35a2d4d0670b8e4fd6',
+            'description': 'md5:db00b9bc10ffd26fb148fa6a3a67c40b',
             'thumbnail': 'http://play.iprima.cz/sites/default/files/image_crops/image_620x349/3/491483_particka-92_image_620x349.jpg',
         },
         'params': {
@@ -35,13 +35,16 @@ class IPrimaIE(InfoExtractor):
             'id': '9718337',
             'ext': 'flv',
             'title': 'Tchibo Partička - Jarní móda',
-            'description': 'md5:589f8f59f414220621ff8882eb3ce7be',
+            'description': 'md5:db00b9bc10ffd26fb148fa6a3a67c40b',
             'thumbnail': 're:^http:.*\.jpg$',
         },
         'params': {
             'skip_download': True,  # requires rtmpdump
         },
         'skip': 'Do not have permission to access this page',
+    }, {
+        'url': 'http://play.iprima.cz/zpravy-ftv-prima-2752015',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
@@ -102,7 +105,7 @@ class IPrimaIE(InfoExtractor):
 
         return {
             'id': real_id,
-            'title': self._og_search_title(webpage),
+            'title': self._og_search_title(webpage).replace(' | Prima PLAY', ''),
             'thumbnail': self._og_search_thumbnail(webpage),
             'formats': formats,
             'description': self._og_search_description(webpage),

From b5597738d4de35fd6f2be7bf1cb6a32c754d873f Mon Sep 17 00:00:00 2001
From: Slava Shklyaev <shk.slava@gmail.com>
Date: Tue, 2 Jun 2015 17:28:14 +0300
Subject: [PATCH 1472/6219] [iprima] Comply with review

---
 youtube_dl/extractor/iprima.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/iprima.py b/youtube_dl/extractor/iprima.py
index f3536893a..502507551 100644
--- a/youtube_dl/extractor/iprima.py
+++ b/youtube_dl/extractor/iprima.py
@@ -11,11 +11,12 @@ from ..compat import (
 )
 from ..utils import (
     ExtractorError,
+    remove_end,
 )
 
 
 class IPrimaIE(InfoExtractor):
-    _VALID_URL = r'https?://play\.iprima\.cz/([^/]+/)*(?P<id>[^?#]+)'
+    _VALID_URL = r'https?://play\.iprima\.cz/(?:[^/]+/)*(?P<id>[^?#]+)'
 
     _TESTS = [{
         'url': 'http://play.iprima.cz/particka/particka-92',
@@ -105,7 +106,7 @@ class IPrimaIE(InfoExtractor):
 
         return {
             'id': real_id,
-            'title': self._og_search_title(webpage).replace(' | Prima PLAY', ''),
+            'title': remove_end(self._og_search_title(webpage), ' | Prima PLAY'),
             'thumbnail': self._og_search_thumbnail(webpage),
             'formats': formats,
             'description': self._og_search_description(webpage),

From d23da75b32e02963f988bad962b3f5259e4a6d31 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 2 Jun 2015 21:10:18 +0600
Subject: [PATCH 1473/6219] [iprima] Fix description extraction

`og:description` does not contain actual description anymore.
---
 youtube_dl/extractor/iprima.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/iprima.py b/youtube_dl/extractor/iprima.py
index 502507551..821c8ec10 100644
--- a/youtube_dl/extractor/iprima.py
+++ b/youtube_dl/extractor/iprima.py
@@ -24,7 +24,7 @@ class IPrimaIE(InfoExtractor):
             'id': '39152',
             'ext': 'flv',
             'title': 'Partička (92)',
-            'description': 'md5:db00b9bc10ffd26fb148fa6a3a67c40b',
+            'description': 'md5:74e9617e51bca67c3ecfb2c6f9766f45',
             'thumbnail': 'http://play.iprima.cz/sites/default/files/image_crops/image_620x349/3/491483_particka-92_image_620x349.jpg',
         },
         'params': {
@@ -36,13 +36,11 @@ class IPrimaIE(InfoExtractor):
             'id': '9718337',
             'ext': 'flv',
             'title': 'Tchibo Partička - Jarní móda',
-            'description': 'md5:db00b9bc10ffd26fb148fa6a3a67c40b',
             'thumbnail': 're:^http:.*\.jpg$',
         },
         'params': {
             'skip_download': True,  # requires rtmpdump
         },
-        'skip': 'Do not have permission to access this page',
     }, {
         'url': 'http://play.iprima.cz/zpravy-ftv-prima-2752015',
         'only_matching': True,
@@ -109,5 +107,7 @@ class IPrimaIE(InfoExtractor):
             'title': remove_end(self._og_search_title(webpage), ' | Prima PLAY'),
             'thumbnail': self._og_search_thumbnail(webpage),
             'formats': formats,
-            'description': self._og_search_description(webpage),
+            'description': self._search_regex(
+                r'<p[^>]+itemprop="description"[^>]*>([^<]+)',
+                webpage, 'description', default=None),
         }

From 4b5fe1349f5568f3b9b939520db0a1ddc598b4b3 Mon Sep 17 00:00:00 2001
From: Slava Shklyaev <shk.slava@gmail.com>
Date: Tue, 2 Jun 2015 18:15:05 +0300
Subject: [PATCH 1474/6219] [nova] Comply with review

---
 youtube_dl/extractor/nova.py | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/nova.py b/youtube_dl/extractor/nova.py
index 10957e5fa..4a2d76506 100644
--- a/youtube_dl/extractor/nova.py
+++ b/youtube_dl/extractor/nova.py
@@ -4,7 +4,11 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import clean_html, determine_ext
+from ..utils import (
+    clean_html,
+    determine_ext,
+    unified_strdate,
+)
 
 
 class NovaIE(InfoExtractor):
@@ -143,14 +147,12 @@ class NovaIE(InfoExtractor):
         description = clean_html(self._og_search_description(webpage, default=None))
         thumbnail = config.get('poster')
 
-        mobj = None
         if site == 'novaplus':
-            mobj = re.search(r'(?P<day>\d{1,2})-(?P<month>\d{1,2})-(?P<year>\d{4})$', display_id)
-        if site == 'fanda':
-            mobj = re.search(
-                r'<span class="date_time">(?P<day>\d{1,2})\.(?P<month>\d{1,2})\.(?P<year>\d{4})\b', webpage)
-        if mobj:
-            upload_date = '{}{:02d}{:02d}'.format(mobj.group('year'), int(mobj.group('month')), int(mobj.group('day')))
+            upload_date = unified_strdate(self._search_regex(
+                r'(\d{1,2}-\d{1,2}-\d{4})$', display_id, 'upload date', default=None))
+        elif site == 'fanda':
+            upload_date = unified_strdate(self._search_regex(
+                r'<span class="date_time">(\d{1,2}\.\d{1,2}\.\d{4})', webpage, 'upload date', default=None))
         else:
             upload_date = None
 

From 08b7968e2873b45dafe465ec04541db8fcd4967d Mon Sep 17 00:00:00 2001
From: Slava Shklyaev <shk.slava@gmail.com>
Date: Tue, 2 Jun 2015 17:49:15 +0300
Subject: [PATCH 1475/6219] [nova] Fix display_id extraction bug

---
 youtube_dl/extractor/nova.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/nova.py b/youtube_dl/extractor/nova.py
index 4a2d76506..8360a65d9 100644
--- a/youtube_dl/extractor/nova.py
+++ b/youtube_dl/extractor/nova.py
@@ -13,9 +13,9 @@ from ..utils import (
 
 class NovaIE(InfoExtractor):
     IE_DESC = 'TN.cz, Prásk.tv, Nova.cz, Novaplus.cz, FANDA.tv, Krásná.cz and Doma.cz'
-    _VALID_URL = 'http://(?:[^.]+\.)?(?P<site>tv(?:noviny)?|tn|novaplus|vymena|fanda|krasna|doma|prask)\.nova\.cz/(?:[^/]+/)+(?P<id>[^/]+?)(?:\.html|/?)$'
+    _VALID_URL = 'http://(?:[^.]+\.)?(?P<site>tv(?:noviny)?|tn|novaplus|vymena|fanda|krasna|doma|prask)\.nova\.cz/(?:[^/]+/)+(?P<id>[^/]+?)(?:\.html|/|$)'
     _TESTS = [{
-        'url': 'http://tvnoviny.nova.cz/clanek/novinky/co-na-sebe-sportaci-praskli-vime-jestli-pujde-hrdlicka-na-materskou.html',
+        'url': 'http://tvnoviny.nova.cz/clanek/novinky/co-na-sebe-sportaci-praskli-vime-jestli-pujde-hrdlicka-na-materskou.html?utm_source=tvnoviny&utm_medium=cpfooter&utm_campaign=novaplus',
         'info_dict': {
             'id': '1608920',
             'display_id': 'co-na-sebe-sportaci-praskli-vime-jestli-pujde-hrdlicka-na-materskou',
@@ -29,7 +29,7 @@ class NovaIE(InfoExtractor):
             'skip_download': True,
         }
     }, {
-        'url': 'http://tn.nova.cz/clanek/tajemstvi-ukryte-v-podzemi-specialni-nemocnice-v-prazske-krci.html',
+        'url': 'http://tn.nova.cz/clanek/tajemstvi-ukryte-v-podzemi-specialni-nemocnice-v-prazske-krci.html#player_13260',
         'md5': '1dd7b9d5ea27bc361f110cd855a19bd3',
         'info_dict': {
             'id': '1757139',
@@ -40,7 +40,7 @@ class NovaIE(InfoExtractor):
             'thumbnail': 're:^https?://.*\.(?:jpg)',
         }
     }, {
-        'url': 'http://novaplus.nova.cz/porad/policie-modrava/video/5591-policie-modrava-15-dil-blondynka-na-hrbitove/',
+        'url': 'http://novaplus.nova.cz/porad/policie-modrava/video/5591-policie-modrava-15-dil-blondynka-na-hrbitove',
         'info_dict': {
             'id': '1756825',
             'display_id': '5591-policie-modrava-15-dil-blondynka-na-hrbitove',

From b0cda32f726443d464a68a34b22a2e02ef8b29b0 Mon Sep 17 00:00:00 2001
From: Slava Shklyaev <shk.slava@gmail.com>
Date: Tue, 2 Jun 2015 18:17:33 +0300
Subject: [PATCH 1476/6219] [nova] Fix Python 2.6 compatability issue

---
 youtube_dl/extractor/nova.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nova.py b/youtube_dl/extractor/nova.py
index 8360a65d9..7e3498eea 100644
--- a/youtube_dl/extractor/nova.py
+++ b/youtube_dl/extractor/nova.py
@@ -141,7 +141,7 @@ class NovaIE(InfoExtractor):
         mediafile = config['mediafile']
         video_url = mediafile['src']
         ext = determine_ext(video_url)
-        video_url = video_url.replace('&{}:'.format(ext), '')
+        video_url = video_url.replace('&%s:' % ext, '')
 
         title = mediafile.get('meta', {}).get('title') or self._og_search_title(webpage)
         description = clean_html(self._og_search_description(webpage, default=None))

From fa971259e69a8031c384754b6238cfff71bea773 Mon Sep 17 00:00:00 2001
From: Slava Shklyaev <shk.slava@gmail.com>
Date: Tue, 2 Jun 2015 19:09:47 +0300
Subject: [PATCH 1477/6219] [nova] Add a comment about html in description

---
 youtube_dl/extractor/nova.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nova.py b/youtube_dl/extractor/nova.py
index 7e3498eea..85253b6ed 100644
--- a/youtube_dl/extractor/nova.py
+++ b/youtube_dl/extractor/nova.py
@@ -46,7 +46,7 @@ class NovaIE(InfoExtractor):
             'display_id': '5591-policie-modrava-15-dil-blondynka-na-hrbitove',
             'ext': 'mp4',
             'title': 'Policie Modrava - 15. díl - Blondýnka na hřbitově',
-            'description': 'md5:dc24e50be5908df83348e50d1431295e',
+            'description': 'md5:dc24e50be5908df83348e50d1431295e',  # Make sure this description is clean of html tags
             'thumbnail': 're:^https?://.*\.(?:jpg)',
         },
         'params': {

From 23dd1fc74c36329ed40855301ac499a0ad2a0009 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 3 Jun 2015 10:21:03 +0800
Subject: [PATCH 1478/6219] [vidme] Always use the non-embedded page

For example, https://vid.me/Wmur contains more information than
https://vid.me/e/Wmur
---
 youtube_dl/extractor/vidme.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vidme.py b/youtube_dl/extractor/vidme.py
index bd953fb4c..e0b55078b 100644
--- a/youtube_dl/extractor/vidme.py
+++ b/youtube_dl/extractor/vidme.py
@@ -10,7 +10,7 @@ from ..utils import (
 
 class VidmeIE(InfoExtractor):
     _VALID_URL = r'https?://vid\.me/(?:e/)?(?P<id>[\da-zA-Z]+)'
-    _TEST = {
+    _TESTS = [{
         'url': 'https://vid.me/QNB',
         'md5': 'f42d05e7149aeaec5c037b17e5d3dc82',
         'info_dict': {
@@ -23,9 +23,14 @@ class VidmeIE(InfoExtractor):
             'upload_date': '20140725',
             'thumbnail': 're:^https?://.*\.jpg',
         },
-    }
+    }, {
+        # From http://naked-yogi.tumblr.com/post/118312946248/naked-smoking-stretching
+        'url': 'https://vid.me/e/Wmur',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
+        url = url.replace('vid.me/e/', 'vid.me/')
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 

From 8f9478412424b87e4fb77be53d239c13932b078a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 3 Jun 2015 10:26:39 +0800
Subject: [PATCH 1479/6219] [tumblr] Detect vid.me embeds (fixes #5883)

---
 youtube_dl/extractor/tumblr.py | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/youtube_dl/extractor/tumblr.py b/youtube_dl/extractor/tumblr.py
index 828c808a6..e6218808f 100644
--- a/youtube_dl/extractor/tumblr.py
+++ b/youtube_dl/extractor/tumblr.py
@@ -28,6 +28,17 @@ class TumblrIE(InfoExtractor):
             'description': 'md5:dba62ac8639482759c8eb10ce474586a',
             'thumbnail': 're:http://.*\.jpg',
         }
+    }, {
+        'url': 'http://naked-yogi.tumblr.com/post/118312946248/naked-smoking-stretching',
+        'md5': 'de07e5211d60d4f3a2c3df757ea9f6ab',
+        'info_dict': {
+            'id': 'Wmur',
+            'ext': 'mp4',
+            'title': 'naked smoking & stretching',
+            'upload_date': '20150506',
+            'timestamp': 1430931613,
+        },
+        'add_ie': ['Vidme'],
     }]
 
     def _real_extract(self, url):
@@ -38,6 +49,12 @@ class TumblrIE(InfoExtractor):
         url = 'http://%s.tumblr.com/post/%s/' % (blog, video_id)
         webpage = self._download_webpage(url, video_id)
 
+        vid_me_embed_url = self._search_regex(
+            r'src=[\'"](https?://vid\.me/[^\'"]+)[\'"]',
+            webpage, 'vid.me embed', default=None)
+        if vid_me_embed_url is not None:
+            return self.url_result(vid_me_embed_url, 'Vidme')
+
         iframe_url = self._search_regex(
             r'src=\'(https?://www\.tumblr\.com/video/[^\']+)\'',
             webpage, 'iframe url')

From 687cb3ad35ac49f1053c1ea52e3b6db18b3aa1cf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 3 Jun 2015 20:47:11 +0600
Subject: [PATCH 1480/6219] [24video] Fix uploader extraction

---
 youtube_dl/extractor/twentyfourvideo.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/twentyfourvideo.py b/youtube_dl/extractor/twentyfourvideo.py
index 67e8bfea0..c1ee1decc 100644
--- a/youtube_dl/extractor/twentyfourvideo.py
+++ b/youtube_dl/extractor/twentyfourvideo.py
@@ -15,7 +15,7 @@ class TwentyFourVideoIE(InfoExtractor):
     _TESTS = [
         {
             'url': 'http://www.24video.net/video/view/1044982',
-            'md5': '48dd7646775690a80447a8dca6a2df76',
+            'md5': 'd041af8b5b4246ea466226a0d6693345',
             'info_dict': {
                 'id': '1044982',
                 'ext': 'mp4',
@@ -54,7 +54,7 @@ class TwentyFourVideoIE(InfoExtractor):
             webpage, 'upload date'))
 
         uploader = self._html_search_regex(
-            r'Загрузил\s*<a href="/jsecUser/movies/[^"]+" class="link">([^<]+)</a>',
+            r'class="video-uploaded"[^>]*>\s*<a href="/jsecUser/movies/[^"]+"[^>]*>([^<]+)</a>',
             webpage, 'uploader', fatal=False)
 
         view_count = int_or_none(self._html_search_regex(

From 15b74b94beb8720a4f6d7ee076c123dd8ae05309 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 3 Jun 2015 20:52:47 +0600
Subject: [PATCH 1481/6219] [tvigle] Capture error message

---
 youtube_dl/extractor/tvigle.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/youtube_dl/extractor/tvigle.py b/youtube_dl/extractor/tvigle.py
index a85693888..aa07c8251 100644
--- a/youtube_dl/extractor/tvigle.py
+++ b/youtube_dl/extractor/tvigle.py
@@ -5,6 +5,7 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
+    ExtractorError,
     float_or_none,
     int_or_none,
     parse_age_limit,
@@ -63,6 +64,13 @@ class TvigleIE(InfoExtractor):
 
         item = video_data['playlist']['items'][0]
 
+        videos = item.get('videos')
+
+        error_message = item.get('errorMessage')
+        if not videos and error_message:
+            raise ExtractorError(
+                '%s returned error: %s' % (self.IE_NAME, error_message), expected=True)
+
         title = item['title']
         description = item.get('description')
         thumbnail = item.get('thumbnail')

From 3153a2c98d7201b1ae8104c346db58e19f322cc5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 3 Jun 2015 20:53:54 +0600
Subject: [PATCH 1482/6219] [tvigle] Skip tests

---
 youtube_dl/extractor/tvigle.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/tvigle.py b/youtube_dl/extractor/tvigle.py
index aa07c8251..dc3a8334a 100644
--- a/youtube_dl/extractor/tvigle.py
+++ b/youtube_dl/extractor/tvigle.py
@@ -30,6 +30,7 @@ class TvigleIE(InfoExtractor):
                 'duration': 6586,
                 'age_limit': 12,
             },
+            'skip': 'georestricted',
         },
         {
             'url': 'http://www.tvigle.ru/video/vladimir-vysotskii/vedushchii-teleprogrammy-60-minut-ssha-o-vladimire-vysotskom/',
@@ -42,6 +43,7 @@ class TvigleIE(InfoExtractor):
                 'duration': 186.080,
                 'age_limit': 0,
             },
+            'skip': 'georestricted',
         }, {
             'url': 'https://cloud.tvigle.ru/video/5267604/',
             'only_matching': True,

From 6800d3372f35e08dcc4d34d06601815bf0cb0a3d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 3 Jun 2015 23:10:18 +0800
Subject: [PATCH 1483/6219] [YoutubeDL] Support DASH manifest downloading

---
 youtube_dl/downloader/dash.py   | 50 +++++++++++++++++++++++++++++++++
 youtube_dl/downloader/http.py   |  4 +++
 youtube_dl/extractor/youtube.py |  6 ++++
 3 files changed, 60 insertions(+)
 create mode 100644 youtube_dl/downloader/dash.py

diff --git a/youtube_dl/downloader/dash.py b/youtube_dl/downloader/dash.py
new file mode 100644
index 000000000..18eca2c04
--- /dev/null
+++ b/youtube_dl/downloader/dash.py
@@ -0,0 +1,50 @@
+from __future__ import unicode_literals
+from .common import FileDownloader
+from ..compat import compat_urllib_request
+
+import re
+
+
+class DashSegmentsFD(FileDownloader):
+    """
+    Download segments in a DASH manifest
+    """
+    def real_download(self, filename, info_dict):
+        self.report_destination(filename)
+        tmpfilename = self.temp_name(filename)
+        base_url = info_dict['url']
+        segment_urls = info_dict['segment_urls']
+
+        self.byte_counter = 0
+
+        def append_url_to_file(outf, target_url, target_name):
+            self.to_screen('[DashSegments] %s: Downloading %s' % (info_dict['id'], target_name))
+            req = compat_urllib_request.Request(target_url)
+            data = self.ydl.urlopen(req).read()
+            outf.write(data)
+            self.byte_counter += len(data)
+
+        def combine_url(base_url, target_url):
+            if re.match(r'^https?://', target_url):
+                return target_url
+            return '%s/%s' % (base_url, target_url)
+
+        with open(tmpfilename, 'wb') as outf:
+            append_url_to_file(
+                outf, combine_url(base_url, info_dict['initialization_url']),
+                'initialization segment')
+            for i, segment_url in enumerate(segment_urls):
+                append_url_to_file(
+                    outf, combine_url(base_url, segment_url),
+                    'segment %d / %d' % (i + 1, len(segment_urls)))
+
+        self.try_rename(tmpfilename, filename)
+
+        self._hook_progress({
+            'downloaded_bytes': self.byte_counter,
+            'total_bytes': self.byte_counter,
+            'filename': filename,
+            'status': 'finished',
+        })
+
+        return True
diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index b7f144af9..ceacb8522 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -6,6 +6,7 @@ import socket
 import time
 
 from .common import FileDownloader
+from .dash import DashSegmentsFD
 from ..compat import (
     compat_urllib_request,
     compat_urllib_error,
@@ -19,6 +20,9 @@ from ..utils import (
 
 class HttpFD(FileDownloader):
     def real_download(self, filename, info_dict):
+        if info_dict.get('initialization_url') and list(filter(None, info_dict.get('segment_urls', []))):
+            return DashSegmentsFD(self.ydl, self.params).real_download(filename, info_dict)
+
         url = info_dict['url']
         tmpfilename = self.temp_name(filename)
         stream = None
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index aacb999ce..5d1297e0d 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -802,6 +802,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     # TODO implement WebVTT downloading
                     pass
                 elif mime_type.startswith('audio/') or mime_type.startswith('video/'):
+                    segment_list = r.find('{urn:mpeg:DASH:schema:MPD:2011}SegmentList')
                     format_id = r.attrib['id']
                     video_url = url_el.text
                     filesize = int_or_none(url_el.attrib.get('{http://youtube.com/yt/2012/10/10}contentLength'))
@@ -815,6 +816,11 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                         'filesize': filesize,
                         'fps': int_or_none(r.attrib.get('frameRate')),
                     }
+                    if segment_list:
+                        f.update({
+                            'initialization_url': segment_list.find('{urn:mpeg:DASH:schema:MPD:2011}Initialization').attrib['sourceURL'],
+                            'segment_urls': [segment.attrib.get('media') for segment in segment_list.findall('{urn:mpeg:DASH:schema:MPD:2011}SegmentURL')]
+                        })
                     try:
                         existing_format = next(
                             fo for fo in formats

From e4ac7bb1e598b0317742737ea06c162fa7f22cd4 Mon Sep 17 00:00:00 2001
From: Slava Shklyaev <shk.slava@gmail.com>
Date: Wed, 3 Jun 2015 19:18:41 +0300
Subject: [PATCH 1484/6219] [nova] Revert "Fix extension extraction bug"

This reverts commit 9464a194dbf48989c486fa2de9e1aebc59e28ed4.
---
 youtube_dl/extractor/nova.py | 25 ++++++++++++++++++-------
 1 file changed, 18 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/nova.py b/youtube_dl/extractor/nova.py
index 85253b6ed..3f9c776ef 100644
--- a/youtube_dl/extractor/nova.py
+++ b/youtube_dl/extractor/nova.py
@@ -6,7 +6,6 @@ import re
 from .common import InfoExtractor
 from ..utils import (
     clean_html,
-    determine_ext,
     unified_strdate,
 )
 
@@ -44,7 +43,7 @@ class NovaIE(InfoExtractor):
         'info_dict': {
             'id': '1756825',
             'display_id': '5591-policie-modrava-15-dil-blondynka-na-hrbitove',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': 'Policie Modrava - 15. díl - Blondýnka na hřbitově',
             'description': 'md5:dc24e50be5908df83348e50d1431295e',  # Make sure this description is clean of html tags
             'thumbnail': 're:^https?://.*\.(?:jpg)',
@@ -57,7 +56,7 @@ class NovaIE(InfoExtractor):
         'url': 'http://novaplus.nova.cz/porad/televizni-noviny/video/5585-televizni-noviny-30-5-2015/',
         'info_dict': {
             'id': '1756858',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': 'Televizní noviny - 30. 5. 2015',
             'thumbnail': 're:^https?://.*\.(?:jpg)',
             'upload_date': '20150530',
@@ -140,8 +139,21 @@ class NovaIE(InfoExtractor):
 
         mediafile = config['mediafile']
         video_url = mediafile['src']
-        ext = determine_ext(video_url)
-        video_url = video_url.replace('&%s:' % ext, '')
+
+        m = re.search(r'^(?P<url>rtmpe?://[^/]+/(?P<app>[^/]+?))/&*(?P<playpath>.+)$', video_url)
+        if m:
+            formats = [{
+                'url': m.group('url'),
+                'app': m.group('app'),
+                'play_path': m.group('playpath'),
+                'player_path': 'http://tvnoviny.nova.cz/static/shared/app/videojs/video-js.swf',
+                'ext': 'flv',
+            }]
+        else:
+            formats = [{
+                'url': video_url,
+            }]
+        self._sort_formats(formats)
 
         title = mediafile.get('meta', {}).get('title') or self._og_search_title(webpage)
         description = clean_html(self._og_search_description(webpage, default=None))
@@ -163,6 +175,5 @@ class NovaIE(InfoExtractor):
             'description': description,
             'upload_date': upload_date,
             'thumbnail': thumbnail,
-            'url': video_url,
-            'ext': ext,
+            'formats': formats,
         }

From 4c8fea92f350d5a3f33d505980ac750b05a9cd34 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 3 Jun 2015 23:50:38 +0200
Subject: [PATCH 1485/6219] [test/aes] Fix on python 3.3 and higher

Since 878563c847fa5248eedbd44187536dec04643eaf the aes functions only accepts the base64 data as a unicode string.
---
 test/test_aes.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/test_aes.py b/test/test_aes.py
index 4dc7de7b5..315a3f5ae 100644
--- a/test/test_aes.py
+++ b/test/test_aes.py
@@ -39,7 +39,7 @@ class TestAES(unittest.TestCase):
         encrypted = base64.b64encode(
             intlist_to_bytes(self.iv[:8]) +
             b'\x17\x15\x93\xab\x8d\x80V\xcdV\xe0\t\xcdo\xc2\xa5\xd8ksM\r\xe27N\xae'
-        )
+        ).decode('utf-8')
         decrypted = (aes_decrypt_text(encrypted, password, 16))
         self.assertEqual(decrypted, self.secret_msg)
 
@@ -47,7 +47,7 @@ class TestAES(unittest.TestCase):
         encrypted = base64.b64encode(
             intlist_to_bytes(self.iv[:8]) +
             b'\x0b\xe6\xa4\xd9z\x0e\xb8\xb9\xd0\xd4i_\x85\x1d\x99\x98_\xe5\x80\xe7.\xbf\xa5\x83'
-        )
+        ).decode('utf-8')
         decrypted = (aes_decrypt_text(encrypted, password, 32))
         self.assertEqual(decrypted, self.secret_msg)
 

From eedda32e6bc4620d704eabab7a2c8f4b1f1a9169 Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Thu, 4 Jun 2015 11:27:18 +0800
Subject: [PATCH 1486/6219] [qqmusic] Fix toplist

---
 youtube_dl/extractor/qqmusic.py | 58 +++++++++++++--------------------
 1 file changed, 23 insertions(+), 35 deletions(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index b540033e2..48f28ffe9 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -181,60 +181,48 @@ class QQMusicToplistIE(QQPlaylistBaseIE):
     _VALID_URL = r'http://y\.qq\.com/#type=toplist&p=(?P<id>(top|global)_[0-9]+)'
 
     _TESTS = [{
-        'url': 'http://y.qq.com/#type=toplist&p=global_12',
+        'url': 'http://y.qq.com/#type=toplist&p=global_123',
         'info_dict': {
-            'id': 'global_12',
-            'title': 'itunes榜',
+            'id': 'global_123',
+            'title': '美国iTunes榜',
         },
         'playlist_count': 10,
     }, {
-        'url': 'http://y.qq.com/#type=toplist&p=top_6',
+        'url': 'http://y.qq.com/#type=toplist&p=top_3',
         'info_dict': {
-            'id': 'top_6',
+            'id': 'top_3',
             'title': 'QQ音乐巅峰榜·欧美',
+            'description': 'QQ音乐巅峰榜·欧美根据用户收听行为自动生成，集结当下最流行的欧美新歌！:更新时间：每周四22点|统'
+                           '计周期：一周（上周四至本周三）|统计对象：三个月内发行的欧美歌曲|统计数量：100首|统计算法：根据'
+                           '歌曲在一周内的有效播放次数，由高到低取前100名（同一歌手最多允许5首歌曲同时上榜）|有效播放次数：'
+                           '登录用户完整播放一首歌曲，记为一次有效播放；同一用户收听同一首歌曲，每天记录为1次有效播放'
         },
         'playlist_count': 100,
     }, {
-        'url': 'http://y.qq.com/#type=toplist&p=global_5',
+        'url': 'http://y.qq.com/#type=toplist&p=global_106',
         'info_dict': {
-            'id': 'global_5',
-            'title': '韩国mnet排行榜',
+            'id': 'global_106',
+            'title': '韩国Mnet榜',
         },
         'playlist_count': 50,
     }]
 
-    @staticmethod
-    def strip_qq_jsonp(code):
-        return js_to_json(re.sub(r'^MusicJsonCallback\((.*?)\)/\*.+?\*/$', r'\1', code))
-
     def _real_extract(self, url):
         list_id = self._match_id(url)
 
         list_type, num_id = list_id.split("_")
 
-        list_page = self._download_webpage(
-            "http://y.qq.com/y/static/toplist/index/%s.html" % list_id,
+        toplist_json = self._download_json(
+            'http://i.y.qq.com/v8/fcg-bin/fcg_v8_toplist_cp.fcg?type=%s&topid=%s&format=json'
+            % (list_type, num_id),
             list_id, 'Download toplist page')
 
-        entries = []
-        if list_type == 'top':
-            jsonp_url = "http://y.qq.com/y/static/toplist/json/top/%s/1.js" % num_id
-        else:
-            jsonp_url = "http://y.qq.com/y/static/toplist/json/global/%s/1_1.js" % num_id
+        entries = [
+            self.url_result(
+                'http://y.qq.com/#type=song&mid=' + song['data']['songmid'], 'QQMusic', song['data']['songmid']
+            ) for song in toplist_json['songlist']
+        ]
 
-        toplist_json = self._download_json(
-            jsonp_url, list_id, note='Retrieve toplist json',
-            errnote='Unable to get toplist json', transform_source=self.strip_qq_jsonp)
-
-        for song in toplist_json['l']:
-            s = song['s']
-            song_mid = s.split("|")[20]
-            entries.append(self.url_result(
-                'http://y.qq.com/#type=song&mid=' + song_mid, 'QQMusic',
-                song_mid))
-
-        list_name = self._html_search_regex(
-            r'<h2 id="top_name">([^\']+)</h2>', list_page, 'top list name',
-            default=None)
-
-        return self.playlist_result(entries, list_id, list_name)
+        list_name = toplist_json['topinfo']['ListName']
+        list_description = toplist_json['topinfo']['info']
+        return self.playlist_result(entries, list_id, list_name, list_description)

From ed15e9ba02382fc7db22e6176068d2220c00a32e Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Thu, 4 Jun 2015 17:32:06 +0800
Subject: [PATCH 1487/6219] [qqmusic] Remove unused import

---
 youtube_dl/extractor/qqmusic.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index 48f28ffe9..9943fcddb 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -9,7 +9,6 @@ from .common import InfoExtractor
 from ..utils import (
     strip_jsonp,
     unescapeHTML,
-    js_to_json,
 )
 from ..compat import compat_urllib_request
 

From 55e5841f14131ab61359535fdcc44e1564d555b8 Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Thu, 4 Jun 2015 17:41:29 +0800
Subject: [PATCH 1488/6219] [qqmusic] Extract additional formats (mp3-128,
 mp3-320)

---
 youtube_dl/extractor/qqmusic.py | 22 ++++++++++++++++++----
 1 file changed, 18 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index b540033e2..4b4ef4993 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -19,10 +19,10 @@ class QQMusicIE(InfoExtractor):
     _VALID_URL = r'http://y.qq.com/#type=song&mid=(?P<id>[0-9A-Za-z]+)'
     _TESTS = [{
         'url': 'http://y.qq.com/#type=song&mid=004295Et37taLD',
-        'md5': 'bed90b6db2a7a7a7e11bc585f471f63a',
+        'md5': '9ce1c1c8445f561506d2e3cfb0255705',
         'info_dict': {
             'id': '004295Et37taLD',
-            'ext': 'm4a',
+            'ext': 'mp3',
             'title': '可惜没如果',
             'upload_date': '20141227',
             'creator': '林俊杰',
@@ -30,6 +30,12 @@ class QQMusicIE(InfoExtractor):
         }
     }]
 
+    _FORMATS = {
+        'mp3-320': {'prefix': 'M800', 'ext': 'mp3', 'preference': 40},
+        'mp3-128': {'prefix': 'M500', 'ext': 'mp3', 'preference': 30},
+        'm4a': {'prefix': 'C200', 'ext': 'm4a', 'preference': 10}
+    }
+
     # Reference: m_r_GetRUin() in top_player.js
     # http://imgcache.gtimg.cn/music/portal_v3/y/top_player.js
     @staticmethod
@@ -69,11 +75,19 @@ class QQMusicIE(InfoExtractor):
             'http://base.music.qq.com/fcgi-bin/fcg_musicexpress.fcg?json=3&guid=%s' % guid,
             mid, note='Retrieve vkey', errnote='Unable to get vkey',
             transform_source=strip_jsonp)['key']
-        song_url = 'http://cc.stream.qqmusic.qq.com/C200%s.m4a?vkey=%s&guid=%s&fromtag=0' % (mid, vkey, guid)
+
+        formats = []
+        for k, sf in self._FORMATS.items():
+            formats.append({
+                'url': 'http://cc.stream.qqmusic.qq.com/%s%s.%s?vkey=%s&guid=%s&fromtag=0'
+                       % (sf['prefix'], mid, sf['ext'], vkey, guid),
+                'format': k, 'format_id': k, 'preference': sf['preference']
+            })
+        self._sort_formats(formats)
 
         return {
             'id': mid,
-            'url': song_url,
+            'formats': formats,
             'title': song_name,
             'upload_date': publish_time,
             'creator': singer,

From b9258c61789388b49792ebdceb5d804217a36da5 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 4 Jun 2015 22:05:33 +0800
Subject: [PATCH 1489/6219] [YoutubeDL] Change how DashSegmentsFD is selected

---
 youtube_dl/downloader/__init__.py | 2 ++
 youtube_dl/downloader/http.py     | 4 ----
 youtube_dl/extractor/youtube.py   | 3 ++-
 3 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/downloader/__init__.py b/youtube_dl/downloader/__init__.py
index f110830c4..1b618ab54 100644
--- a/youtube_dl/downloader/__init__.py
+++ b/youtube_dl/downloader/__init__.py
@@ -8,6 +8,7 @@ from .hls import NativeHlsFD
 from .http import HttpFD
 from .rtsp import RtspFD
 from .rtmp import RtmpFD
+from .dash import DashSegmentsFD
 
 from ..utils import (
     determine_protocol,
@@ -20,6 +21,7 @@ PROTOCOL_MAP = {
     'mms': RtspFD,
     'rtsp': RtspFD,
     'f4m': F4mFD,
+    'dash_segments': DashSegmentsFD,
 }
 
 
diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index ceacb8522..b7f144af9 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -6,7 +6,6 @@ import socket
 import time
 
 from .common import FileDownloader
-from .dash import DashSegmentsFD
 from ..compat import (
     compat_urllib_request,
     compat_urllib_error,
@@ -20,9 +19,6 @@ from ..utils import (
 
 class HttpFD(FileDownloader):
     def real_download(self, filename, info_dict):
-        if info_dict.get('initialization_url') and list(filter(None, info_dict.get('segment_urls', []))):
-            return DashSegmentsFD(self.ydl, self.params).real_download(filename, info_dict)
-
         url = info_dict['url']
         tmpfilename = self.temp_name(filename)
         stream = None
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 5d1297e0d..692d4d8db 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -819,7 +819,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     if segment_list:
                         f.update({
                             'initialization_url': segment_list.find('{urn:mpeg:DASH:schema:MPD:2011}Initialization').attrib['sourceURL'],
-                            'segment_urls': [segment.attrib.get('media') for segment in segment_list.findall('{urn:mpeg:DASH:schema:MPD:2011}SegmentURL')]
+                            'segment_urls': [segment.attrib.get('media') for segment in segment_list.findall('{urn:mpeg:DASH:schema:MPD:2011}SegmentURL')],
+                            'protocol': 'dash_segments',
                         })
                     try:
                         existing_format = next(

From 453a1617aac6e8000ed947cad7d88817c5740ede Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 4 Jun 2015 22:12:05 +0800
Subject: [PATCH 1490/6219] [downloader/dash] Reorder imports

---
 youtube_dl/downloader/dash.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/downloader/dash.py b/youtube_dl/downloader/dash.py
index 18eca2c04..5f14658ba 100644
--- a/youtube_dl/downloader/dash.py
+++ b/youtube_dl/downloader/dash.py
@@ -1,9 +1,10 @@
 from __future__ import unicode_literals
-from .common import FileDownloader
-from ..compat import compat_urllib_request
 
 import re
 
+from .common import FileDownloader
+from ..compat import compat_urllib_request
+
 
 class DashSegmentsFD(FileDownloader):
     """

From 423d2be5f8c5e70d202ddfa63f3e5365e6afe823 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 4 Jun 2015 22:27:29 +0800
Subject: [PATCH 1491/6219] [downloader/dash] Rename the protocol

'http_dash_segments' looks more like a protocol name than
'dash_segments'
---
 youtube_dl/downloader/__init__.py | 2 +-
 youtube_dl/extractor/youtube.py   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/downloader/__init__.py b/youtube_dl/downloader/__init__.py
index 1b618ab54..dccc59212 100644
--- a/youtube_dl/downloader/__init__.py
+++ b/youtube_dl/downloader/__init__.py
@@ -21,7 +21,7 @@ PROTOCOL_MAP = {
     'mms': RtspFD,
     'rtsp': RtspFD,
     'f4m': F4mFD,
-    'dash_segments': DashSegmentsFD,
+    'http_dash_segments': DashSegmentsFD,
 }
 
 
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 692d4d8db..6d288e848 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -820,7 +820,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                         f.update({
                             'initialization_url': segment_list.find('{urn:mpeg:DASH:schema:MPD:2011}Initialization').attrib['sourceURL'],
                             'segment_urls': [segment.attrib.get('media') for segment in segment_list.findall('{urn:mpeg:DASH:schema:MPD:2011}SegmentURL')],
-                            'protocol': 'dash_segments',
+                            'protocol': 'http_dash_segments',
                         })
                     try:
                         existing_format = next(

From 56c837ccb75b639d362397095f33300229c4bd1c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 4 Jun 2015 20:34:48 +0600
Subject: [PATCH 1492/6219] [tnaflix] Fix typo

---
 youtube_dl/extractor/tnaflix.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tnaflix.py b/youtube_dl/extractor/tnaflix.py
index 59af9aba0..bc51bae37 100644
--- a/youtube_dl/extractor/tnaflix.py
+++ b/youtube_dl/extractor/tnaflix.py
@@ -33,7 +33,7 @@ class TNAFlixIE(InfoExtractor):
         },
         {
             'url': 'https://www.tnaflix.com/amateur-porn/bunzHD-Ms.Donk/video358632',
-            'matching_only': True,
+            'only_matching': True,
         }
     ]
 

From e52c0bd0eb853d4d242872e1d9ff5426a35dd30c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 4 Jun 2015 20:37:05 +0600
Subject: [PATCH 1493/6219] [tnaflix] Modernize

---
 youtube_dl/extractor/tnaflix.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/tnaflix.py b/youtube_dl/extractor/tnaflix.py
index bc51bae37..3e335d653 100644
--- a/youtube_dl/extractor/tnaflix.py
+++ b/youtube_dl/extractor/tnaflix.py
@@ -51,9 +51,8 @@ class TNAFlixIE(InfoExtractor):
 
         age_limit = self._rta_search(webpage)
 
-        duration = self._html_search_meta('duration', webpage, 'duration', default=None)
-        if duration:
-            duration = parse_duration(duration[1:])
+        duration = parse_duration(self._html_search_meta(
+            'duration', webpage, 'duration', default=None))
 
         cfg_url = self._proto_relative_url(self._html_search_regex(
             self._CONFIG_REGEX, webpage, 'flashvars.config'), 'http:')

From 3ce9bc712acd88df8499dd0982277c8f64b0d15a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 4 Jun 2015 20:39:03 +0600
Subject: [PATCH 1494/6219] [empflix] Fix typo

---
 youtube_dl/extractor/empflix.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/empflix.py b/youtube_dl/extractor/empflix.py
index 9a5a8f4bb..4827022e0 100644
--- a/youtube_dl/extractor/empflix.py
+++ b/youtube_dl/extractor/empflix.py
@@ -26,6 +26,6 @@ class EMPFlixIE(TNAFlixIE):
         },
         {
             'url': 'http://www.empflix.com/videos/[AROMA][ARMD-718]-Aoi-Yoshino-Sawa-25826.html',
-            'matching_only': True,
+            'only_matching': True,
         }
     ]

From 3d6388e34ea41d937f39e561b7731f1389971a66 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 4 Jun 2015 20:42:37 +0600
Subject: [PATCH 1495/6219] [tnaflix] Fix relative URLs (empflix)

---
 youtube_dl/extractor/tnaflix.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/tnaflix.py b/youtube_dl/extractor/tnaflix.py
index 3e335d653..c282865b2 100644
--- a/youtube_dl/extractor/tnaflix.py
+++ b/youtube_dl/extractor/tnaflix.py
@@ -61,14 +61,15 @@ class TNAFlixIE(InfoExtractor):
             cfg_url, display_id, note='Downloading metadata',
             transform_source=fix_xml_ampersands)
 
-        thumbnail = cfg_xml.find('./startThumb').text
+        thumbnail = self._proto_relative_url(
+            cfg_xml.find('./startThumb').text, 'http:')
 
         formats = []
         for item in cfg_xml.findall('./quality/item'):
             video_url = re.sub('speed=\d+', 'speed=', item.find('videoLink').text)
             format_id = item.find('res').text
             fmt = {
-                'url': video_url,
+                'url': self._proto_relative_url(video_url, 'http:'),
                 'format_id': format_id,
             }
             m = re.search(r'^(\d+)', format_id)

From 9d4f213f90d6024c7748f4defdc7b45f2351b0da Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 5 Jun 2015 00:52:18 +0800
Subject: [PATCH 1496/6219] [qqmusic:toplist] List name and description are
 optional

---
 youtube_dl/extractor/qqmusic.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index 9943fcddb..f773332a8 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -222,6 +222,7 @@ class QQMusicToplistIE(QQPlaylistBaseIE):
             ) for song in toplist_json['songlist']
         ]
 
-        list_name = toplist_json['topinfo']['ListName']
-        list_description = toplist_json['topinfo']['info']
+        topinfo = toplist_json.get('topinfo', {})
+        list_name = topinfo.get('ListName')
+        list_description = topinfo.get('info')
         return self.playlist_result(entries, list_id, list_name, list_description)

From f5c78d118ba2d7e5e4a1ccd40c97fc1bf85a8dcf Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 4 Jun 2015 21:49:02 +0200
Subject: [PATCH 1497/6219] release 2015.06.04

---
 docs/supportedsites.md | 5 +++++
 youtube_dl/version.py  | 2 +-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index a421ae62b..d147b53fe 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -10,6 +10,7 @@
  - **56.com**
  - **5min**
  - **8tracks**
+ - **91porn**
  - **9gag**
  - **abc.net.au**
  - **Abc7News**
@@ -319,6 +320,7 @@
  - **Noco**
  - **Normalboots**
  - **NosVideo**
+ - **Nova**: TN.cz, Prásk.tv, Nova.cz, Novaplus.cz, FANDA.tv, Krásná.cz and Doma.cz
  - **novamov**: NovaMov
  - **Nowness**
  - **NowTV**
@@ -431,6 +433,8 @@
  - **smotri:user**: Smotri.com user videos
  - **Snotr**
  - **Sohu**
+ - **soompi**
+ - **soompi:show**
  - **soundcloud**
  - **soundcloud:playlist**
  - **soundcloud:set**
@@ -505,6 +509,7 @@
  - **Trilulilu**
  - **TruTube**
  - **Tube8**
+ - **TubiTv**
  - **Tudou**
  - **Tumblr**
  - **TuneIn**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 653710131..84224b7a7 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.05.29'
+__version__ = '2015.06.04'

From 0e805e782bd05951ca3f420cf2a050e2ac3ae846 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 4 Jun 2015 21:54:33 +0200
Subject: [PATCH 1498/6219] release 2015.06.04.1

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 84224b7a7..9cf84ff71 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.06.04'
+__version__ = '2015.06.04.1'

From 8b8cde21406b53f5aeb6586dab03a9d78d62e631 Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Fri, 5 Jun 2015 06:04:26 +0800
Subject: [PATCH 1499/6219] [qqmusic] Set abr for mp3 formats

---
 youtube_dl/extractor/qqmusic.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index 4b4ef4993..dc300e189 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -31,8 +31,8 @@ class QQMusicIE(InfoExtractor):
     }]
 
     _FORMATS = {
-        'mp3-320': {'prefix': 'M800', 'ext': 'mp3', 'preference': 40},
-        'mp3-128': {'prefix': 'M500', 'ext': 'mp3', 'preference': 30},
+        'mp3-320': {'prefix': 'M800', 'ext': 'mp3', 'preference': 40, 'abr': 320},
+        'mp3-128': {'prefix': 'M500', 'ext': 'mp3', 'preference': 30, 'abr': 128},
         'm4a': {'prefix': 'C200', 'ext': 'm4a', 'preference': 10}
     }
 
@@ -77,11 +77,12 @@ class QQMusicIE(InfoExtractor):
             transform_source=strip_jsonp)['key']
 
         formats = []
-        for k, sf in self._FORMATS.items():
+        for k, f in self._FORMATS.items():
             formats.append({
                 'url': 'http://cc.stream.qqmusic.qq.com/%s%s.%s?vkey=%s&guid=%s&fromtag=0'
-                       % (sf['prefix'], mid, sf['ext'], vkey, guid),
-                'format': k, 'format_id': k, 'preference': sf['preference']
+                       % (f['prefix'], mid, f['ext'], vkey, guid),
+                'format': k, 'format_id': k, 'preference': f['preference'],
+                'abr': f.get('abr')
             })
         self._sort_formats(formats)
 

From d31573fa37c8db7133492baf0a6be3ece643f8ff Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 5 Jun 2015 22:55:29 +0800
Subject: [PATCH 1500/6219] [teamcoco] Handle incomplete m3u8 URLs (fixes
 #5798)

There are 2 TODOs. I don't know how to handle these cases correctly.
---
 youtube_dl/extractor/teamcoco.py | 29 +++++++++++++++++++++++++++--
 1 file changed, 27 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/teamcoco.py b/youtube_dl/extractor/teamcoco.py
index b2a4b1fc0..d1b7264b4 100644
--- a/youtube_dl/extractor/teamcoco.py
+++ b/youtube_dl/extractor/teamcoco.py
@@ -51,6 +51,17 @@ class TeamcocoIE(InfoExtractor):
             'params': {
                 'skip_download': True,  # m3u8 downloads
             }
+        }, {
+            'url': 'http://teamcoco.com/video/full-episode-mon-6-1-joel-mchale-jake-tapper-and-musical-guest-courtney-barnett?playlist=x;eyJ0eXBlIjoidGFnIiwiaWQiOjl9',
+            'info_dict': {
+                'id': '89341',
+                'ext': 'mp4',
+                'title': 'Full Episode - Mon. 6/1 - Joel McHale, Jake Tapper, And Musical Guest Courtney Barnett',
+                'description': 'Guests: Joel McHale, Jake Tapper, And Musical Guest Courtney Barnett',
+            },
+            'params': {
+                'skip_download': True,  # m3u8 downloads
+            }
         }
     ]
     _VIDEO_ID_REGEXES = (
@@ -110,9 +121,23 @@ class TeamcocoIE(InfoExtractor):
         get_quality = qualities(['500k', '480p', '1000k', '720p', '1080p'])
         for filed in data['files']:
             if determine_ext(filed['url']) == 'm3u8':
-                formats.extend(self._extract_m3u8_formats(
-                    filed['url'], video_id, ext='mp4'))
+                # compat_urllib_parse.urljoin does not work here
+                if filed['url'].startswith('/'):
+                    m3u8_url = 'http://ht.cdn.turner.com/tbs/big/teamcoco' + filed['url']
+                else:
+                    m3u8_url = filed['url']
+                m3u8_formats = self._extract_m3u8_formats(
+                    m3u8_url, video_id, ext='mp4')
+                for m3u8_format in m3u8_formats:
+                    if m3u8_format not in formats:
+                        formats.append(m3u8_format)
+            elif determine_ext(filed['url']) == 'f4m':
+                # TODO Correct f4m extraction
+                continue
             else:
+                if filed['url'].startswith('/mp4:protected/'):
+                    # TODO Correct extraction for these files
+                    continue
                 m_format = re.search(r'(\d+(k|p))\.mp4', filed['url'])
                 if m_format is not None:
                     format_id = m_format.group(1)

From f00a650705a5e5b4f2b540ea8133a1752e63dd81 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 5 Jun 2015 23:16:34 +0800
Subject: [PATCH 1501/6219] [qqmusic] Rearrange codes

---
 youtube_dl/extractor/qqmusic.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index e24ddaefe..c903bee58 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -80,7 +80,9 @@ class QQMusicIE(InfoExtractor):
             formats.append({
                 'url': 'http://cc.stream.qqmusic.qq.com/%s%s.%s?vkey=%s&guid=%s&fromtag=0'
                        % (f['prefix'], mid, f['ext'], vkey, guid),
-                'format': k, 'format_id': k, 'preference': f['preference'],
+                'format': k,
+                'format_id': k,
+                'preference': f['preference'],
                 'abr': f.get('abr')
             })
         self._sort_formats(formats)

From e8ac61e840b5c02e406b910f5f0eed3d8b331969 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 5 Jun 2015 23:19:25 +0800
Subject: [PATCH 1502/6219] [qqmusic] Use meaningful variable names

---
 youtube_dl/extractor/qqmusic.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index c903bee58..bafa81c21 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -76,14 +76,14 @@ class QQMusicIE(InfoExtractor):
             transform_source=strip_jsonp)['key']
 
         formats = []
-        for k, f in self._FORMATS.items():
+        for format_id, details in self._FORMATS.items():
             formats.append({
                 'url': 'http://cc.stream.qqmusic.qq.com/%s%s.%s?vkey=%s&guid=%s&fromtag=0'
-                       % (f['prefix'], mid, f['ext'], vkey, guid),
-                'format': k,
-                'format_id': k,
-                'preference': f['preference'],
-                'abr': f.get('abr')
+                       % (details['prefix'], mid, details['ext'], vkey, guid),
+                'format': format_id,
+                'format_id': format_id,
+                'preference': details['preference'],
+                'abr': details.get('abr'),
             })
         self._sort_formats(formats)
 

From dfe7dd9bdb45ec765c9b335c149e9913cf7e413f Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Sat, 6 Jun 2015 02:54:57 +0300
Subject: [PATCH 1503/6219] [izlesene] Unquote video URLs and simplify

---
 youtube_dl/extractor/izlesene.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/izlesene.py b/youtube_dl/extractor/izlesene.py
index 99a1361f8..753cb98ea 100644
--- a/youtube_dl/extractor/izlesene.py
+++ b/youtube_dl/extractor/izlesene.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import compat_urllib_parse_unquote
 from ..utils import (
     determine_ext,
     float_or_none,
@@ -67,9 +68,9 @@ class IzleseneIE(InfoExtractor):
 
         uploader = self._html_search_regex(
             r"adduserUsername\s*=\s*'([^']+)';",
-            webpage, 'uploader', fatal=False, default='')
+            webpage, 'uploader', fatal=False)
         timestamp = parse_iso8601(self._html_search_meta(
-            'uploadDate', webpage, 'upload date', fatal=False))
+            'uploadDate', webpage, 'upload date'))
 
         duration = float_or_none(self._html_search_regex(
             r'"videoduration"\s*:\s*"([^"]+)"',
@@ -86,8 +87,7 @@ class IzleseneIE(InfoExtractor):
 
         # Might be empty for some videos.
         streams = self._html_search_regex(
-            r'"qualitylevel"\s*:\s*"([^"]+)"',
-            webpage, 'streams', fatal=False, default='')
+            r'"qualitylevel"\s*:\s*"([^"]+)"', webpage, 'streams', default='')
 
         formats = []
         if streams:
@@ -95,15 +95,15 @@ class IzleseneIE(InfoExtractor):
                 quality, url = re.search(r'\[(\w+)\](.+)', stream).groups()
                 formats.append({
                     'format_id': '%sp' % quality if quality else 'sd',
-                    'url': url,
+                    'url': compat_urllib_parse_unquote(url),
                     'ext': ext,
                 })
         else:
             stream_url = self._search_regex(
-                r'"streamurl"\s?:\s?"([^"]+)"', webpage, 'stream URL')
+                r'"streamurl"\s*:\s*"([^"]+)"', webpage, 'stream URL')
             formats.append({
                 'format_id': 'sd',
-                'url': stream_url,
+                'url': compat_urllib_parse_unquote(stream_url),
                 'ext': ext,
             })
 

From c33c547d66b20064f83932cdaa04823b17a96b70 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Sat, 6 Jun 2015 02:57:12 +0300
Subject: [PATCH 1504/6219] [izlesene] Avoid timestamp differences in tests due
 to DST

---
 youtube_dl/extractor/izlesene.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/izlesene.py b/youtube_dl/extractor/izlesene.py
index 753cb98ea..bc226fa67 100644
--- a/youtube_dl/extractor/izlesene.py
+++ b/youtube_dl/extractor/izlesene.py
@@ -31,7 +31,7 @@ class IzleseneIE(InfoExtractor):
                 'description': 'md5:253753e2655dde93f59f74b572454f6d',
                 'thumbnail': 're:^http://.*\.jpg',
                 'uploader_id': 'pelikzzle',
-                'timestamp': 1404302298,
+                'timestamp': int,
                 'upload_date': '20140702',
                 'duration': 95.395,
                 'age_limit': 0,
@@ -47,7 +47,7 @@ class IzleseneIE(InfoExtractor):
                 'description': 'Tarkan Dortmund 2006 Konseri',
                 'thumbnail': 're:^http://.*\.jpg',
                 'uploader_id': 'parlayankiz',
-                'timestamp': 1163322193,
+                'timestamp': int,
                 'upload_date': '20061112',
                 'duration': 253.666,
                 'age_limit': 0,

From 54eb81a087516e9d040bc1ad274c0a64b51dd1d1 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Sat, 6 Jun 2015 03:11:43 +0300
Subject: [PATCH 1505/6219] [pornovoisines] Improve average_rating extraction
 and update test case

---
 youtube_dl/extractor/pornovoisines.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/pornovoisines.py b/youtube_dl/extractor/pornovoisines.py
index 9688ed948..eba4dfbb3 100644
--- a/youtube_dl/extractor/pornovoisines.py
+++ b/youtube_dl/extractor/pornovoisines.py
@@ -34,7 +34,7 @@ class PornoVoisinesIE(InfoExtractor):
             'duration': 120,
             'view_count': int,
             'average_rating': float,
-            'categories': ['Débutante', 'Scénario', 'Sodomie'],
+            'categories': ['Débutantes', 'Scénario', 'Sodomie'],
             'age_limit': 18,
         }
     }
@@ -71,7 +71,7 @@ class PornoVoisinesIE(InfoExtractor):
         view_count = int_or_none(self._search_regex(
             r'(\d+) vues', webpage, 'view count', fatal=False))
         average_rating = self._search_regex(
-            r'Note : (\d+,\d+)', webpage, 'average rating', fatal=False)
+            r'Note\s*:\s*(\d+(?:,\d+)?)', webpage, 'average rating', fatal=False)
         if average_rating:
             average_rating = float_or_none(average_rating.replace(',', '.'))
 

From 3d8e9573a470594df6fa471dc33c4c4b938b668a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 6 Jun 2015 06:25:37 +0600
Subject: [PATCH 1506/6219] [youtube:channel] Improve channel id extraction
 (#5904)

---
 youtube_dl/extractor/youtube.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index aacb999ce..419f7b019 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1406,10 +1406,12 @@ class YoutubeChannelIE(InfoExtractor):
         channel_page = self._download_webpage(
             url + '?view=57', channel_id,
             'Downloading channel page', fatal=False)
-        channel_playlist_id = self._search_regex(
-            [r'<meta itemprop="channelId" content="([^"]+)">',
-             r'data-channel-external-id="([^"]+)"'],
-            channel_page, 'channel id', default=None)
+        channel_playlist_id = self._html_search_meta(
+            'channelId', channel_page, 'channel id', default=None)
+        if not channel_playlist_id:
+            channel_playlist_id = self._search_regex(
+                r'data-channel-external-id="([^"]+)"',
+                channel_page, 'channel id', default=None)
         if channel_playlist_id and channel_playlist_id.startswith('UC'):
             playlist_id = 'UU' + channel_playlist_id[2:]
             return self.url_result(

From 223544552fcfec0c5c6a83326520c614e4489cbb Mon Sep 17 00:00:00 2001
From: Hannu Lintala <hannu.lintala@gmail.com>
Date: Sat, 9 May 2015 03:53:43 +0300
Subject: [PATCH 1507/6219] [Ruutu] Add new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/ruutu.py    | 88 ++++++++++++++++++++++++++++++++
 2 files changed, 89 insertions(+)
 create mode 100644 youtube_dl/extractor/ruutu.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 8ec0c1032..860023d14 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -448,6 +448,7 @@ from .rutube import (
     RutubePersonIE,
 )
 from .rutv import RUTVIE
+from .ruutu import RuutuIE
 from .sandia import SandiaIE
 from .safari import (
     SafariIE,
diff --git a/youtube_dl/extractor/ruutu.py b/youtube_dl/extractor/ruutu.py
new file mode 100644
index 000000000..e346434f9
--- /dev/null
+++ b/youtube_dl/extractor/ruutu.py
@@ -0,0 +1,88 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..compat import compat_urllib_parse_urlparse
+import re
+
+
+class RuutuIE(InfoExtractor):
+    _VALID_URL = r'http://(www\.)?ruutu\.fi/ohjelmat/(?:[^/]+/)?(?P<id>.*)$'
+    _TESTS = [
+        {
+            'url': 'http://www.ruutu.fi/ohjelmat/oletko-aina-halunnut-tietaa-mita-tapahtuu-vain-hetki-ennen-lahetysta-nyt-se-selvisi',
+            'md5': 'ab2093f39be1ca8581963451b3c0234f',
+            'info_dict': {
+                'id': 'oletko-aina-halunnut-tietaa-mita-tapahtuu-vain-hetki-ennen-lahetysta-nyt-se-selvisi',
+                'ext': 'mp4',
+                'title': 'Oletko aina halunnut tietää mitä tapahtuu vain hetki ennen lähetystä? - Nyt se selvisi!',
+                'description': 'Toinen toistaan huikeampia ohjelmaideoita ja täysin päätöntä sekoilua? No sitä juuri nimenomaan. Metro Helsingin Iltapäivän vieraaksi saapui Tuomas Kauhanen ja he Petra Kalliomaan kanssa keskustelivat hieman ennen lähetyksen alkua, mutta kamerat olivatkin jo päällä.',
+            },
+            'params': {
+                'format': 'http-1000',
+            }
+        },
+        {
+            'url': 'http://www.ruutu.fi/ohjelmat/superpesis/superpesis-katso-koko-kausi-ruudussa',
+            'md5': '065a10ae4d5b8cfd9d0c3d332465e3d9',
+            'info_dict': {
+                'id': 'superpesis-katso-koko-kausi-ruudussa',
+                'ext': 'mp4',
+                'title': 'Superpesis: katso koko kausi Ruudussa',
+                'description': 'Huippujännittävän Superpesiksen suoria ottelulähetyksiä seurataan Ruudussa kauden alusta viimeiseen finaaliin asti. Katso lisätiedot osoitteesta ruutu.fi/superpesis.',
+            },
+            'params': {
+                'format': 'http-1000',
+            }
+        },
+    ]
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+
+        video_id = mobj.group('id')
+        webpage = self._download_webpage(url, video_id)
+        media_id = self._html_search_regex(r'data-media-id="(\d+)"', webpage, 'media_id')
+        media_json = self._parse_json(self._search_regex(r'jQuery.extend\([^,]+, (.*)\);', webpage, 'media_data'), video_id)
+        xml_url = media_json['ruutuplayer']['xmlUrl'].replace('{ID}', media_id)
+        media_xml = self._download_xml(xml_url, media_id)
+
+        formats = []
+        parsed_urls = []
+        for fmt in media_xml.findall('.//Clip//'):
+            url = fmt.text
+            if not fmt.tag.endswith('File') or url in parsed_urls or \
+                    'NOT_USED' in url:
+                continue
+
+            if url.endswith('m3u8'):
+                formats.extend(self._extract_m3u8_formats(url, media_id, m3u8_id='hls'))
+                parsed_urls.append(url)
+            elif url.endswith('f4m'):
+                formats.extend(self._extract_f4m_formats(url, media_id, f4m_id='hds'))
+                parsed_urls.append(url)
+            else:
+                proto = compat_urllib_parse_urlparse(url).scheme
+                width_str, height_str = fmt.get('resolution').split('x')
+                tbr = int(fmt.get('bitrate', 0))
+                formats.append({
+                    'format_id': '%s-%d' % (proto, tbr),
+                    'url': url,
+                    'width': int(width_str),
+                    'height': int(height_str),
+                    'tbr': tbr,
+                    'ext': url.rsplit('.', 1)[-1],
+                    'live': True,
+                    'protocol': proto,
+                })
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': self._og_search_title(webpage),
+            'formats': formats,
+            'description': self._og_search_description(webpage),
+            'thumbnail': self._og_search_thumbnail(webpage),
+            'duration': int(media_xml.find('.//Runtime').text),
+            'age_limit': int(media_xml.find('.//AgeLimit').text),
+        }

From a9e58ecd3fdfa93cdc8a7f9fc852dbbd0814d6a4 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Sat, 6 Jun 2015 13:56:46 +0300
Subject: [PATCH 1508/6219] [turbo] Improve description extraction

`og:description` is empty for some videos.
---
 youtube_dl/extractor/turbo.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/turbo.py b/youtube_dl/extractor/turbo.py
index 29703a8a9..7ae63a499 100644
--- a/youtube_dl/extractor/turbo.py
+++ b/youtube_dl/extractor/turbo.py
@@ -23,7 +23,7 @@ class TurboIE(InfoExtractor):
             'ext': 'mp4',
             'duration': 3715,
             'title': 'Turbo du 07/09/2014 : Renault Twingo 3, Bentley Continental GT Speed, CES, Guide Achat Dacia... ',
-            'description': 'Retrouvez dans cette rubrique toutes les vidéos de l\'Turbo du 07/09/2014 : Renault Twingo 3, Bentley Continental GT Speed, CES, Guide Achat Dacia... ',
+            'description': 'Turbo du 07/09/2014 : Renault Twingo 3, Bentley Continental GT Speed, CES, Guide Achat Dacia...',
             'thumbnail': 're:^https?://.*\.jpg$',
         }
     }
@@ -42,7 +42,7 @@ class TurboIE(InfoExtractor):
         title = xpath_text(item, './title', 'title')
         duration = int_or_none(xpath_text(item, './durate', 'duration'))
         thumbnail = xpath_text(item, './visuel_clip', 'thumbnail')
-        description = self._og_search_description(webpage)
+        description = self._html_search_meta('description', webpage)
 
         formats = []
         get_quality = qualities(['3g', 'sd', 'hq'])

From 05aa9c82d90644af406519e5e25fefb0884d504e Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Sat, 6 Jun 2015 13:58:20 +0300
Subject: [PATCH 1509/6219] [sunporno] Fix view_count extraction

---
 youtube_dl/extractor/sunporno.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/sunporno.py b/youtube_dl/extractor/sunporno.py
index 854d01bee..e527aa971 100644
--- a/youtube_dl/extractor/sunporno.py
+++ b/youtube_dl/extractor/sunporno.py
@@ -44,7 +44,7 @@ class SunPornoIE(InfoExtractor):
             webpage, 'duration', fatal=False))
 
         view_count = int_or_none(self._html_search_regex(
-            r'class="views">\s*(\d+)\s*<',
+            r'class="views">(?:<noscript>)?\s*(\d+)\s*<',
             webpage, 'view count', fatal=False))
         comment_count = int_or_none(self._html_search_regex(
             r'(\d+)</b> Comments?',

From 4da31bd56629054497634d041035e4bd6fcfacbb Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 6 Jun 2015 22:22:26 +0800
Subject: [PATCH 1510/6219] [youtube] Fix a FutureWarning from
 xml.etree.ElementTree

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 6d288e848..2424ac2c0 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -816,7 +816,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                         'filesize': filesize,
                         'fps': int_or_none(r.attrib.get('frameRate')),
                     }
-                    if segment_list:
+                    if len(segment_list):
                         f.update({
                             'initialization_url': segment_list.find('{urn:mpeg:DASH:schema:MPD:2011}Initialization').attrib['sourceURL'],
                             'segment_urls': [segment.attrib.get('media') for segment in segment_list.findall('{urn:mpeg:DASH:schema:MPD:2011}SegmentURL')],

From f1da861018924e6f442ffedd9a5682055c79aea6 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 7 Jun 2015 00:37:29 +0800
Subject: [PATCH 1511/6219] [iqiyi] PEP8

---
 youtube_dl/extractor/iqiyi.py | 56 +++++++++++++++++------------------
 1 file changed, 28 insertions(+), 28 deletions(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index 747f3f902..597441baf 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -16,19 +16,20 @@ import random
 import zlib
 import hashlib
 
+
 class IqiyiIE(InfoExtractor):
     IE_NAME = 'iqiyi'
 
     _VALID_URL = r'http://(?:www\.)iqiyi.com/.+?\.html'
 
     _TEST = {
-            'url': 'http://www.iqiyi.com/v_19rrojlavg.html',
-            'md5': '2cb594dc2781e6c941a110d8f358118b',
-            'info_dict': {
-                'id': '9c1fb1b99d192b21c559e5a1a2cb3c73',
-                'title': '美国德州空中惊现奇异云团 酷似UFO',
-                'ext': 'f4v',
-            }
+        'url': 'http://www.iqiyi.com/v_19rrojlavg.html',
+        'md5': '2cb594dc2781e6c941a110d8f358118b',
+        'info_dict': {
+            'id': '9c1fb1b99d192b21c559e5a1a2cb3c73',
+            'title': '美国德州空中惊现奇异云团 酷似UFO',
+            'ext': 'f4v',
+        }
     }
 
     def construct_video_urls(self, data, video_id, _uuid, bid):
@@ -46,7 +47,7 @@ class IqiyiIE(InfoExtractor):
             c = len(b)
             s = ''
             for i in range(c - 1, -1, -1):
-                a = do_xor(int(b[c-i-1], 16), i)
+                a = do_xor(int(b[c - i - 1], 16), i)
                 s += chr(a)
             return s[::-1]
 
@@ -54,15 +55,14 @@ class IqiyiIE(InfoExtractor):
             mg = ')(*&^flash@#$%a'
             tm = self._download_json(
                 'http://data.video.qiyi.com/t?tn=' + str(random.random()), video_id)['t']
-            t = str(int(math.floor(int(tm)/(600.0))))
-            return hashlib.md5(
-                 (t+mg+x).encode('utf8')).hexdigest()
+            t = str(int(math.floor(int(tm) / (600.0))))
+            return hashlib.md5((t + mg + x).encode('utf8')).hexdigest()
 
         # get accept format
         # getting all format will spend minutes for a big video.
         if bid == 'best':
-            bids = [int(i['bid']) for i in data['vp']['tkl'][0]['vs'] \
-                   if 0 < int(i['bid']) <= 10]
+            bids = [int(i['bid']) for i in data['vp']['tkl'][0]['vs']
+                    if 0 < int(i['bid']) <= 10]
             bid = str(max(bids))
 
         video_urls_dict = {}
@@ -117,24 +117,24 @@ class IqiyiIE(InfoExtractor):
 
     def get_format(self, bid):
         _dict = {
-            '1'  : 'h6',
-            '2'  : 'h5',
-            '3'  : 'h4',
-            '4'  : 'h3',
-            '5'  : 'h2',
-            '10' : 'h1'
+            '1': 'h6',
+            '2': 'h5',
+            '3': 'h4',
+            '4': 'h3',
+            '5': 'h2',
+            '10': 'h1'
         }
         return _dict.get(str(bid), None)
 
     def get_bid(self, format_id):
         _dict = {
-            'h6'   : '1',
-            'h5'   : '2',
-            'h4'   : '3',
-            'h3'   : '4',
-            'h2'   : '5',
-            'h1'   : '10',
-            'best' : 'best'
+            'h6': '1',
+            'h5': '2',
+            'h4': '3',
+            'h3': '4',
+            'h2': '5',
+            'h1': '10',
+            'best': 'best'
         }
         return _dict.get(format_id, None)
 
@@ -207,7 +207,7 @@ class IqiyiIE(InfoExtractor):
         for format_id in video_urls_dict:
             video_urls = video_urls_dict[format_id]
             for i, video_url_info in enumerate(video_urls):
-                if len(entries) < i+1:
+                if len(entries) < i + 1:
                     entries.append({'formats': []})
                 entries[i]['formats'].append(
                     {
@@ -222,7 +222,7 @@ class IqiyiIE(InfoExtractor):
             self._sort_formats(entries[i]['formats'])
             entries[i].update(
                 {
-                    'id': '_part%d' % (i+1),
+                    'id': '_part%d' % (i + 1),
                     'title': title,
                 }
             )

From 7012620e2b9355d25ddfc855fc5990af938f04d8 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 7 Jun 2015 00:44:54 +0800
Subject: [PATCH 1512/6219] [iqiyi] Remove format selection codes

---
 youtube_dl/extractor/iqiyi.py | 22 ++--------------------
 1 file changed, 2 insertions(+), 20 deletions(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index 597441baf..5645fb6ee 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -32,7 +32,7 @@ class IqiyiIE(InfoExtractor):
         }
     }
 
-    def construct_video_urls(self, data, video_id, _uuid, bid):
+    def construct_video_urls(self, data, video_id, _uuid):
         def do_xor(x, y):
             a = y % 3
             if a == 1:
@@ -58,13 +58,6 @@ class IqiyiIE(InfoExtractor):
             t = str(int(math.floor(int(tm) / (600.0))))
             return hashlib.md5((t + mg + x).encode('utf8')).hexdigest()
 
-        # get accept format
-        # getting all format will spend minutes for a big video.
-        if bid == 'best':
-            bids = [int(i['bid']) for i in data['vp']['tkl'][0]['vs']
-                    if 0 < int(i['bid']) <= 10]
-            bid = str(max(bids))
-
         video_urls_dict = {}
         for i in data['vp']['tkl'][0]['vs']:
             if 0 < int(i['bid']) <= 10:
@@ -80,12 +73,6 @@ class IqiyiIE(InfoExtractor):
                 if t.endswith('mp4'):
                     video_urls_info = i['flvs']
 
-            if int(i['bid']) != int(bid):  # ignore missing match format
-                video_urls.extend(
-                    [('http://example.com/v.flv', ii['b']) for ii in video_urls_info])
-                video_urls_dict[format_id] = video_urls
-                continue
-
             for ii in video_urls_info:
                 vl = ii['l']
                 if not vl.startswith('/'):
@@ -193,14 +180,9 @@ class IqiyiIE(InfoExtractor):
 
         title = data['vi']['vn']
 
-        format = self._downloader.params.get('format', None)
-        bid = self.get_bid(format) if format else 'best'
-        if not bid:
-            raise ExtractorError('Can\'t get format.')
-
         # generate video_urls_dict
         video_urls_dict = self.construct_video_urls(
-            data, video_id, _uuid, bid)
+            data, video_id, _uuid)
 
         # construct info
         entries = []

From 29e7e0781b1b8e276c28a079bc5b18e1b0db2d5e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 7 Jun 2015 00:56:08 +0800
Subject: [PATCH 1513/6219] [iqiyi] Simplify and improve regex patterns

See the comments in #5849
---
 youtube_dl/extractor/iqiyi.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index 5645fb6ee..18a7587a2 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -161,12 +161,11 @@ class IqiyiIE(InfoExtractor):
         webpage = self._download_webpage(
             url, 'temp_id', note='download video page')
         tvid = self._search_regex(
-            r'tvId ?= ?(\'|\")(?P<tvid>\d+)', webpage, 'tvid', flags=re.I, group='tvid')
+            r'data-player-tvid\s*=\s*[\'"](\d+)', webpage, 'tvid')
         video_id = self._search_regex(
-            r'videoId ?= ?(\'|\")(?P<video_id>[a-z\d]+)',
-            webpage, 'video_id', flags=re.I, group='video_id')
+            r'data-player-videoid\s*=\s*[\'"]([a-f\d]+)', webpage, 'video_id')
         swf_url = self._search_regex(
-            r'(?P<swf>http://.+?MainPlayer.+?\.swf)', webpage, 'swf')
+            r'(http://.+?MainPlayer.+?\.swf)', webpage, 'swf player URL')
         _uuid = uuid.uuid4().hex
 
         enc_key = self.get_enc_key(swf_url, video_id)

From aacda28b28c1804866d634c5c5086b3d53cb2b2f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 7 Jun 2015 01:32:03 +0800
Subject: [PATCH 1514/6219] [iqiyi] Give error message for assertion failures

---
 youtube_dl/extractor/iqiyi.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index 18a7587a2..dc35c3380 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -171,7 +171,10 @@ class IqiyiIE(InfoExtractor):
         enc_key = self.get_enc_key(swf_url, video_id)
 
         raw_data = self.get_raw_data(tvid, video_id, enc_key, _uuid)
-        assert raw_data['code'] == 'A000000'
+
+        if raw_data['code'] != 'A000000':
+            raise ExtractorError('Unable to load data. Error code: ' + raw_data['code'])
+
         if not raw_data['data']['vp']['tkl']:
             raise ExtractorError('No support iQiqy VIP video')
 

From 958d0b659b80d4493d045d4da82074ed68ed6c4e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 7 Jun 2015 01:35:09 +0800
Subject: [PATCH 1515/6219] [iqiyi] Reorder imports

---
 youtube_dl/extractor/iqiyi.py | 17 +++++++----------
 1 file changed, 7 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index dc35c3380..36029361a 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -1,20 +1,17 @@
 # coding: utf-8
-
 from __future__ import unicode_literals
 
-from .common import InfoExtractor
-
-from ..compat import compat_urllib_parse
-
-from ..utils import ExtractorError
-
+import hashlib
+import math
+import random
 import re
 import time
 import uuid
-import math
-import random
 import zlib
-import hashlib
+
+from .common import InfoExtractor
+from ..compat import compat_urllib_parse
+from ..utils import ExtractorError
 
 
 class IqiyiIE(InfoExtractor):

From ffba4edb067238b593b98c71f4293e9b60ba95ce Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 7 Jun 2015 01:52:51 +0800
Subject: [PATCH 1516/6219] [iqiyi] Improve some variable names and add
 download notes

---
 youtube_dl/extractor/iqiyi.py | 32 ++++++++++++++++++--------------
 1 file changed, 18 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index 36029361a..c17e1fde4 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -48,35 +48,37 @@ class IqiyiIE(InfoExtractor):
                 s += chr(a)
             return s[::-1]
 
-        def get_path_key(x):
+        def get_path_key(x, format_id, segment_index):
             mg = ')(*&^flash@#$%a'
             tm = self._download_json(
-                'http://data.video.qiyi.com/t?tn=' + str(random.random()), video_id)['t']
+                'http://data.video.qiyi.com/t?tn=' + str(random.random()), video_id,
+                note='Download path key of segment %d for format %s' % (segment_index + 1, format_id)
+            )['t']
             t = str(int(math.floor(int(tm) / (600.0))))
             return hashlib.md5((t + mg + x).encode('utf8')).hexdigest()
 
         video_urls_dict = {}
-        for i in data['vp']['tkl'][0]['vs']:
-            if 0 < int(i['bid']) <= 10:
-                format_id = self.get_format(i['bid'])
+        for format_item in data['vp']['tkl'][0]['vs']:
+            if 0 < int(format_item['bid']) <= 10:
+                format_id = self.get_format(format_item['bid'])
             else:
                 continue
 
             video_urls = []
 
-            video_urls_info = i['fs']
-            if not i['fs'][0]['l'].startswith('/'):
-                t = get_encode_code(i['fs'][0]['l'])
+            video_urls_info = format_item['fs']
+            if not format_item['fs'][0]['l'].startswith('/'):
+                t = get_encode_code(format_item['fs'][0]['l'])
                 if t.endswith('mp4'):
-                    video_urls_info = i['flvs']
+                    video_urls_info = format_item['flvs']
 
-            for ii in video_urls_info:
-                vl = ii['l']
+            for segment_index, segment in enumerate(video_urls_info):
+                vl = segment['l']
                 if not vl.startswith('/'):
                     vl = get_encode_code(vl)
                 key = get_path_key(
-                    vl.split('/')[-1].split('.')[0])
-                filesize = ii['b']
+                    vl.split('/')[-1].split('.')[0], format_id, segment_index)
+                filesize = segment['b']
                 base_url = data['vp']['du'].split('/')
                 base_url.insert(-1, key)
                 base_url = '/'.join(base_url)
@@ -91,7 +93,9 @@ class IqiyiIE(InfoExtractor):
                 }
                 api_video_url = base_url + vl + '?' + \
                     compat_urllib_parse.urlencode(param)
-                js = self._download_json(api_video_url, video_id)
+                js = self._download_json(
+                    api_video_url, video_id,
+                    note='Download video info of segment %d for format %s' % (segment_index + 1, format_id))
                 video_url = js['l']
                 video_urls.append(
                     (video_url, filesize))

From c4ee87022bd18863fc3f22f80064453e272d956f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 7 Jun 2015 01:57:05 +0800
Subject: [PATCH 1517/6219] [iqiyi] Change id for multipart videos

---
 youtube_dl/extractor/iqiyi.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index c17e1fde4..840cc9a4d 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -207,7 +207,7 @@ class IqiyiIE(InfoExtractor):
             self._sort_formats(entries[i]['formats'])
             entries[i].update(
                 {
-                    'id': '_part%d' % (i + 1),
+                    'id': '%s_part%d' % (video_id, i + 1),
                     'title': title,
                 }
             )

From 99481135907b5fa3558d4f176fd02acbdafccdb6 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 7 Jun 2015 02:09:33 +0800
Subject: [PATCH 1518/6219] [iqiyi] Add a multipart test case

---
 youtube_dl/extractor/iqiyi.py | 67 +++++++++++++++++++++++++++++++++--
 1 file changed, 65 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index 840cc9a4d..d73687d88 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -19,7 +19,7 @@ class IqiyiIE(InfoExtractor):
 
     _VALID_URL = r'http://(?:www\.)iqiyi.com/.+?\.html'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.iqiyi.com/v_19rrojlavg.html',
         'md5': '2cb594dc2781e6c941a110d8f358118b',
         'info_dict': {
@@ -27,7 +27,70 @@ class IqiyiIE(InfoExtractor):
             'title': '美国德州空中惊现奇异云团 酷似UFO',
             'ext': 'f4v',
         }
-    }
+    }, {
+        'url': 'http://www.iqiyi.com/v_19rrhnnclk.html',
+        'info_dict': {
+            'id': 'e3f585b550a280af23c98b6cb2be19fb',
+            'title': '名侦探柯南第752集',
+        },
+        'playlist': [{
+            'md5': '7e49376fecaffa115d951634917fe105',
+            'info_dict': {
+                'id': 'e3f585b550a280af23c98b6cb2be19fb_part1',
+                'ext': 'f4v',
+                'title': '名侦探柯南第752集',
+            },
+        }, {
+            'md5': '41b75ba13bb7ac0e411131f92bc4f6ca',
+            'info_dict': {
+                'id': 'e3f585b550a280af23c98b6cb2be19fb_part2',
+                'ext': 'f4v',
+                'title': '名侦探柯南第752集',
+            },
+        }, {
+            'md5': '0cee1dd0a3d46a83e71e2badeae2aab0',
+            'info_dict': {
+                'id': 'e3f585b550a280af23c98b6cb2be19fb_part3',
+                'ext': 'f4v',
+                'title': '名侦探柯南第752集',
+            },
+        }, {
+            'md5': '4f8ad72373b0c491b582e7c196b0b1f9',
+            'info_dict': {
+                'id': 'e3f585b550a280af23c98b6cb2be19fb_part4',
+                'ext': 'f4v',
+                'title': '名侦探柯南第752集',
+            },
+        }, {
+            'md5': 'd89ad028bcfad282918e8098e811711d',
+            'info_dict': {
+                'id': 'e3f585b550a280af23c98b6cb2be19fb_part5',
+                'ext': 'f4v',
+                'title': '名侦探柯南第752集',
+            },
+        }, {
+            'md5': '9cb1e5c95da25dff0660c32ae50903b7',
+            'info_dict': {
+                'id': 'e3f585b550a280af23c98b6cb2be19fb_part6',
+                'ext': 'f4v',
+                'title': '名侦探柯南第752集',
+            },
+        }, {
+            'md5': '155116e0ff1867bbc9b98df294faabc9',
+            'info_dict': {
+                'id': 'e3f585b550a280af23c98b6cb2be19fb_part7',
+                'ext': 'f4v',
+                'title': '名侦探柯南第752集',
+            },
+        }, {
+            'md5': '53f5db77622ae14fa493ed2a278a082b',
+            'info_dict': {
+                'id': 'e3f585b550a280af23c98b6cb2be19fb_part8',
+                'ext': 'f4v',
+                'title': '名侦探柯南第752集',
+            },
+        }],
+    }]
 
     def construct_video_urls(self, data, video_id, _uuid):
         def do_xor(x, y):

From 865ab62f43eb94a9f4f757a464df147e983cb439 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 7 Jun 2015 02:13:22 +0800
Subject: [PATCH 1519/6219] [iqiyi] Make _VALID_URL more accurate

v_* urls are individual videos, while a_* urls are playlists, which are
not supported yet.
---
 youtube_dl/extractor/iqiyi.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index d73687d88..f0d423331 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -17,7 +17,7 @@ from ..utils import ExtractorError
 class IqiyiIE(InfoExtractor):
     IE_NAME = 'iqiyi'
 
-    _VALID_URL = r'http://(?:www\.)iqiyi.com/.+?\.html'
+    _VALID_URL = r'http://(?:www\.)iqiyi.com/v_.+?\.html'
 
     _TESTS = [{
         'url': 'http://www.iqiyi.com/v_19rrojlavg.html',

From 08bb8ef2011d795948d8e89478bf3afe4b99405f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 7 Jun 2015 02:25:00 +0800
Subject: [PATCH 1520/6219] [iqiyi] Unify get_format() and get_bid()

---
 youtube_dl/extractor/iqiyi.py | 32 +++++++++++++-------------------
 1 file changed, 13 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index f0d423331..122f33692 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -92,6 +92,15 @@ class IqiyiIE(InfoExtractor):
         }],
     }]
 
+    _FORMATS_MAP = [
+        ('1', 'h6'),
+        ('2', 'h5'),
+        ('3', 'h4'),
+        ('4', 'h3'),
+        ('5', 'h2'),
+        ('10', 'h1'),
+    ]
+
     def construct_video_urls(self, data, video_id, _uuid):
         def do_xor(x, y):
             a = y % 3
@@ -167,27 +176,12 @@ class IqiyiIE(InfoExtractor):
         return video_urls_dict
 
     def get_format(self, bid):
-        _dict = {
-            '1': 'h6',
-            '2': 'h5',
-            '3': 'h4',
-            '4': 'h3',
-            '5': 'h2',
-            '10': 'h1'
-        }
-        return _dict.get(str(bid), None)
+        matched_format_ids = [_format_id for _bid, _format_id in self._FORMATS_MAP if _bid == str(bid)]
+        return matched_format_ids[0] if len(matched_format_ids) else None
 
     def get_bid(self, format_id):
-        _dict = {
-            'h6': '1',
-            'h5': '2',
-            'h4': '3',
-            'h3': '4',
-            'h2': '5',
-            'h1': '10',
-            'best': 'best'
-        }
-        return _dict.get(format_id, None)
+        matched_bids = [_bid for _bid, _format_id in self._FORMATS_MAP if _format_id == format_id]
+        return matched_bids[0] if len(matched_bids) else None
 
     def get_raw_data(self, tvid, video_id, enc_key, _uuid):
         tm = str(int(time.time()))

From 9c5f685ef14a8b44d17b897ba8ae2da051011c35 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 7 Jun 2015 02:39:03 +0800
Subject: [PATCH 1521/6219] [iqiyi] Improve regex pattern again

---
 youtube_dl/extractor/iqiyi.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index 122f33692..15481b84b 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -223,7 +223,7 @@ class IqiyiIE(InfoExtractor):
         video_id = self._search_regex(
             r'data-player-videoid\s*=\s*[\'"]([a-f\d]+)', webpage, 'video_id')
         swf_url = self._search_regex(
-            r'(http://.+?MainPlayer.+?\.swf)', webpage, 'swf player URL')
+            r'(http://[^\'"]+MainPlayer[^.]+\.swf)', webpage, 'swf player URL')
         _uuid = uuid.uuid4().hex
 
         enc_key = self.get_enc_key(swf_url, video_id)

From b5a3c7f10927c9d55f6fdad5f5c002e02338642e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 7 Jun 2015 02:47:36 +0800
Subject: [PATCH 1522/6219] [iqiyi] Cache encryption keys

---
 youtube_dl/extractor/iqiyi.py | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index 15481b84b..9106dd074 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -3,6 +3,7 @@ from __future__ import unicode_literals
 
 import hashlib
 import math
+import os.path
 import random
 import re
 import time
@@ -11,7 +12,10 @@ import zlib
 
 from .common import InfoExtractor
 from ..compat import compat_urllib_parse
-from ..utils import ExtractorError
+from ..utils import (
+    ExtractorError,
+    url_basename,
+)
 
 
 class IqiyiIE(InfoExtractor):
@@ -207,12 +211,20 @@ class IqiyiIE(InfoExtractor):
         return raw_data
 
     def get_enc_key(self, swf_url, video_id):
+        filename, _ = os.path.splitext(url_basename(swf_url))
+        enc_key_json = self._downloader.cache.load('iqiyi-enc-key', filename)
+        if enc_key_json is not None:
+            return enc_key_json[0]
+
         req = self._request_webpage(
             swf_url, video_id, note='download swf content')
         cn = req.read()
         cn = zlib.decompress(cn[8:])
         pt = re.compile(b'MixerRemote\x08(?P<enc_key>.+?)\$&vv')
         enc_key = self._search_regex(pt, cn, 'enc_key').decode('utf8')
+
+        self._downloader.cache.store('iqiyi-enc-key', filename, [enc_key])
+
         return enc_key
 
     def _real_extract(self, url):

From d00735a0c5aabd38b37bfea76a93ae8c47a8d419 Mon Sep 17 00:00:00 2001
From: Hannu Lintala <hannu.lintala@gmail.com>
Date: Sat, 6 Jun 2015 23:01:23 +0300
Subject: [PATCH 1523/6219] [ruutu] Don't use fallback for DASH and other
 non-HTTP urls

---
 youtube_dl/extractor/ruutu.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/ruutu.py b/youtube_dl/extractor/ruutu.py
index e346434f9..59e0b12fd 100644
--- a/youtube_dl/extractor/ruutu.py
+++ b/youtube_dl/extractor/ruutu.py
@@ -62,6 +62,8 @@ class RuutuIE(InfoExtractor):
                 formats.extend(self._extract_f4m_formats(url, media_id, f4m_id='hds'))
                 parsed_urls.append(url)
             else:
+                if not fmt.tag.startswith('HTTP'):
+                    continue
                 proto = compat_urllib_parse_urlparse(url).scheme
                 width_str, height_str = fmt.get('resolution').split('x')
                 tbr = int(fmt.get('bitrate', 0))

From de390ea0771a0e35c0c2970bc00f5fa2dd9d3eac Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 7 Jun 2015 00:19:45 +0200
Subject: [PATCH 1524/6219] update: Use https for getting the version info
 (fixes #5909)

---
 youtube_dl/update.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/update.py b/youtube_dl/update.py
index de3169eef..fc7ac8305 100644
--- a/youtube_dl/update.py
+++ b/youtube_dl/update.py
@@ -50,7 +50,7 @@ def rsa_verify(message, signature, key):
 def update_self(to_screen, verbose):
     """Update the program file with the latest version from the repository"""
 
-    UPDATE_URL = "http://rg3.github.io/youtube-dl/update/"
+    UPDATE_URL = "https://rg3.github.io/youtube-dl/update/"
     VERSION_URL = UPDATE_URL + 'LATEST_VERSION'
     JSON_URL = UPDATE_URL + 'versions.json'
     UPDATES_RSA_KEY = (0x9d60ee4d8f805312fdb15a62f87b95bd66177b91df176765d13514a0f1754bcd2057295c5b6f1d35daa6742c3ffc9a82d3e118861c207995a8031e151d863c9927e304576bc80692bc8e094896fcf11b66f3e29e04e3a71e9a11558558acea1840aec37fc396fb6b65dc81a1c4144e03bd1c011de62e3f1357b327d08426fe93, 65537)

From 9414338a48ca815fd666aad496ebabd6d0c76e5c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 7 Jun 2015 05:37:29 +0600
Subject: [PATCH 1525/6219] [ruutu] Improve, make more robust and fix python
 2.6 support

---
 youtube_dl/extractor/ruutu.py | 129 +++++++++++++++++++++-------------
 1 file changed, 79 insertions(+), 50 deletions(-)

diff --git a/youtube_dl/extractor/ruutu.py b/youtube_dl/extractor/ruutu.py
index 59e0b12fd..4e22628d0 100644
--- a/youtube_dl/extractor/ruutu.py
+++ b/youtube_dl/extractor/ruutu.py
@@ -3,88 +3,117 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..compat import compat_urllib_parse_urlparse
-import re
+from ..utils import (
+    determine_ext,
+    int_or_none,
+    xpath_text,
+)
 
 
 class RuutuIE(InfoExtractor):
-    _VALID_URL = r'http://(www\.)?ruutu\.fi/ohjelmat/(?:[^/]+/)?(?P<id>.*)$'
+    _VALID_URL = r'http://(?:www\.)?ruutu\.fi/ohjelmat/(?:[^/?#]+/)*(?P<id>[^/?#]+)'
     _TESTS = [
         {
             'url': 'http://www.ruutu.fi/ohjelmat/oletko-aina-halunnut-tietaa-mita-tapahtuu-vain-hetki-ennen-lahetysta-nyt-se-selvisi',
             'md5': 'ab2093f39be1ca8581963451b3c0234f',
             'info_dict': {
-                'id': 'oletko-aina-halunnut-tietaa-mita-tapahtuu-vain-hetki-ennen-lahetysta-nyt-se-selvisi',
+                'id': '2058907',
+                'display_id': 'oletko-aina-halunnut-tietaa-mita-tapahtuu-vain-hetki-ennen-lahetysta-nyt-se-selvisi',
                 'ext': 'mp4',
                 'title': 'Oletko aina halunnut tietää mitä tapahtuu vain hetki ennen lähetystä? - Nyt se selvisi!',
-                'description': 'Toinen toistaan huikeampia ohjelmaideoita ja täysin päätöntä sekoilua? No sitä juuri nimenomaan. Metro Helsingin Iltapäivän vieraaksi saapui Tuomas Kauhanen ja he Petra Kalliomaan kanssa keskustelivat hieman ennen lähetyksen alkua, mutta kamerat olivatkin jo päällä.',
+                'description': 'md5:cfc6ccf0e57a814360df464a91ff67d6',
+                'thumbnail': 're:^https?://.*\.jpg$',
+                'duration': 114,
+                'age_limit': 0,
             },
-            'params': {
-                'format': 'http-1000',
-            }
         },
         {
             'url': 'http://www.ruutu.fi/ohjelmat/superpesis/superpesis-katso-koko-kausi-ruudussa',
             'md5': '065a10ae4d5b8cfd9d0c3d332465e3d9',
             'info_dict': {
-                'id': 'superpesis-katso-koko-kausi-ruudussa',
+                'id': '2057306',
+                'display_id': 'superpesis-katso-koko-kausi-ruudussa',
                 'ext': 'mp4',
                 'title': 'Superpesis: katso koko kausi Ruudussa',
-                'description': 'Huippujännittävän Superpesiksen suoria ottelulähetyksiä seurataan Ruudussa kauden alusta viimeiseen finaaliin asti. Katso lisätiedot osoitteesta ruutu.fi/superpesis.',
+                'description': 'md5:44c44a99fdbe5b380ab74ebd75f0af77',
+                'thumbnail': 're:^https?://.*\.jpg$',
+                'duration': 40,
+                'age_limit': 0,
             },
-            'params': {
-                'format': 'http-1000',
-            }
         },
     ]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
+        display_id = self._match_id(url)
 
-        video_id = mobj.group('id')
-        webpage = self._download_webpage(url, video_id)
-        media_id = self._html_search_regex(r'data-media-id="(\d+)"', webpage, 'media_id')
-        media_json = self._parse_json(self._search_regex(r'jQuery.extend\([^,]+, (.*)\);', webpage, 'media_data'), video_id)
-        xml_url = media_json['ruutuplayer']['xmlUrl'].replace('{ID}', media_id)
-        media_xml = self._download_xml(xml_url, media_id)
+        webpage = self._download_webpage(url, display_id)
+
+        video_id = self._search_regex(
+            r'data-media-id="(\d+)"', webpage, 'media id')
+
+        video_xml_url = None
+
+        media_data = self._search_regex(
+            r'jQuery\.extend\([^,]+,\s*(.+?)\);', webpage,
+            'media data', default=None)
+        if media_data:
+            media_json = self._parse_json(media_data, display_id, fatal=False)
+            if media_json:
+                xml_url = media_json.get('ruutuplayer', {}).get('xmlUrl')
+                if xml_url:
+                    video_xml_url = xml_url.replace('{ID}', video_id)
+
+        if not video_xml_url:
+            video_xml_url = 'http://gatling.ruutu.fi/media-xml-cache?id=%s' % video_id
+
+        video_xml = self._download_xml(video_xml_url, video_id)
 
         formats = []
-        parsed_urls = []
-        for fmt in media_xml.findall('.//Clip//'):
-            url = fmt.text
-            if not fmt.tag.endswith('File') or url in parsed_urls or \
-                    'NOT_USED' in url:
-                continue
+        processed_urls = []
 
-            if url.endswith('m3u8'):
-                formats.extend(self._extract_m3u8_formats(url, media_id, m3u8_id='hls'))
-                parsed_urls.append(url)
-            elif url.endswith('f4m'):
-                formats.extend(self._extract_f4m_formats(url, media_id, f4m_id='hds'))
-                parsed_urls.append(url)
-            else:
-                if not fmt.tag.startswith('HTTP'):
-                    continue
-                proto = compat_urllib_parse_urlparse(url).scheme
-                width_str, height_str = fmt.get('resolution').split('x')
-                tbr = int(fmt.get('bitrate', 0))
-                formats.append({
-                    'format_id': '%s-%d' % (proto, tbr),
-                    'url': url,
-                    'width': int(width_str),
-                    'height': int(height_str),
-                    'tbr': tbr,
-                    'ext': url.rsplit('.', 1)[-1],
-                    'live': True,
-                    'protocol': proto,
-                })
+        def extract_formats(node):
+            for child in node:
+                if child.tag.endswith('Files'):
+                    extract_formats(child)
+                elif child.tag.endswith('File'):
+                    video_url = child.text
+                    if not video_url or video_url in processed_urls or 'NOT_USED' in video_url:
+                        return
+                    processed_urls.append(video_url)
+                    ext = determine_ext(video_url)
+                    if ext == 'm3u8':
+                        formats.extend(self._extract_m3u8_formats(
+                            video_url, video_id, 'mp4', m3u8_id='hls'))
+                    elif ext == 'f4m':
+                        formats.extend(self._extract_f4m_formats(
+                            video_url, video_id, f4m_id='hds'))
+                    else:
+                        proto = compat_urllib_parse_urlparse(video_url).scheme
+                        if not child.tag.startswith('HTTP') and proto != 'rtmp':
+                            continue
+                        preference = -1 if proto == 'rtmp' else 1
+                        label = child.get('label')
+                        tbr = int_or_none(child.get('bitrate'))
+                        width, height = [int_or_none(x) for x in child.get('resolution', '').split('x')]
+                        formats.append({
+                            'format_id': '%s-%s' % (proto, label if label else tbr),
+                            'url': video_url,
+                            'width': width,
+                            'height': height,
+                            'tbr': tbr,
+                            'preference': preference,
+                        })
+
+        extract_formats(video_xml.find('./Clip'))
         self._sort_formats(formats)
 
         return {
             'id': video_id,
+            'display_id': display_id,
             'title': self._og_search_title(webpage),
-            'formats': formats,
             'description': self._og_search_description(webpage),
             'thumbnail': self._og_search_thumbnail(webpage),
-            'duration': int(media_xml.find('.//Runtime').text),
-            'age_limit': int(media_xml.find('.//AgeLimit').text),
+            'duration': int_or_none(xpath_text(video_xml, './/Runtime', 'duration')),
+            'age_limit': int_or_none(xpath_text(video_xml, './/AgeLimit', 'age limit')),
+            'formats': formats,
         }

From 9836cfb8d682c91036ce417fa31200673b52115b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 7 Jun 2015 08:12:21 +0600
Subject: [PATCH 1526/6219] [options] Clarify `--list-extractors` (Closes
 #5916)

---
 youtube_dl/options.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 5a2315bd9..689fa7595 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -145,7 +145,7 @@ def parseOpts(overrideArguments=None):
     general.add_option(
         '--list-extractors',
         action='store_true', dest='list_extractors', default=False,
-        help='List all supported extractors and the URLs they would handle')
+        help='List all supported extractors')
     general.add_option(
         '--extractor-descriptions',
         action='store_true', dest='list_extractor_descriptions', default=False,

From b26733ba7f376f8c9285ac7928534286622bbc7c Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 7 Jun 2015 15:29:17 +0800
Subject: [PATCH 1527/6219] [brightcove] Allow single quotes in Brightcove URLs
 (fixes #5901)

---
 youtube_dl/extractor/brightcove.py |  2 +-
 youtube_dl/extractor/generic.py    | 12 ++++++++++++
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 4f60d5366..c1d4320e1 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -172,7 +172,7 @@ class BrightcoveIE(InfoExtractor):
         """Return a list of all Brightcove URLs from the webpage """
 
         url_m = re.search(
-            r'<meta\s+property="og:video"\s+content="(https?://(?:secure|c)\.brightcove.com/[^"]+)"',
+            r'<meta\s+property=[\'"]og:video[\'"]\s+content=[\'"](https?://(?:secure|c)\.brightcove.com/[^\'"]+)[\'"]',
             webpage)
         if url_m:
             url = unescapeHTML(url_m.group(1))
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 96ca398de..759691365 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -789,6 +789,18 @@ class GenericIE(InfoExtractor):
                 # rtmpe downloads
                 'skip_download': True,
             }
+        },
+        # Brightcove URL in single quotes
+        {
+            'url': 'http://www.sportsnet.ca/baseball/mlb/sn-presents-russell-martin-world-citizen/',
+            'md5': '4ae374f1f8b91c889c4b9203c8c752af',
+            'info_dict': {
+                'id': '4255764656001',
+                'ext': 'mp4',
+                'title': 'SN Presents: Russell Martin, World Citizen',
+                'description': 'To understand why he was the Toronto Blue Jays’ top off-season priority is to appreciate his background and upbringing in Montreal, where he first developed his baseball skills. Written and narrated by Stephen Brunt.',
+                'uploader': 'Rogers Sportsnet',
+            },
         }
     ]
 

From 621ed9f5f4d9d82659272ebe01e740e9196fad61 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 7 Jun 2015 16:33:22 +0800
Subject: [PATCH 1528/6219] [common] Add note and errnote field for
 _extract_m3u8_formats

---
 youtube_dl/extractor/common.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index cecf917ff..49e4dc710 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -846,7 +846,7 @@ class InfoExtractor(object):
 
     def _extract_m3u8_formats(self, m3u8_url, video_id, ext=None,
                               entry_protocol='m3u8', preference=None,
-                              m3u8_id=None):
+                              m3u8_id=None, note=None, errnote=None):
 
         formats = [{
             'format_id': '-'.join(filter(None, [m3u8_id, 'meta'])),
@@ -865,8 +865,8 @@ class InfoExtractor(object):
 
         m3u8_doc = self._download_webpage(
             m3u8_url, video_id,
-            note='Downloading m3u8 information',
-            errnote='Failed to download m3u8 information')
+            note=note or 'Downloading m3u8 information',
+            errnote=errnote or 'Failed to download m3u8 information')
         last_info = None
         last_media = None
         kv_rex = re.compile(

From 65ba8b23f471b96e6f937f2754c729e22bf2cf0a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 7 Jun 2015 16:34:19 +0800
Subject: [PATCH 1529/6219] [discovery] Rewrite DiscoveryIE (fixes #5898)

Discovery.com now uses a completely different approach for serving
videos. At least in both test cases brightcove are involved. However,
AMF support is necessary for these brightcove videos. As a result, I
try to extract videos from the info page ('?flat=1'). The downloaded
file can be different from the one in browsers.
---
 youtube_dl/extractor/discovery.py | 52 ++++++++++++++++++++-----------
 1 file changed, 33 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/extractor/discovery.py b/youtube_dl/extractor/discovery.py
index d3e667528..d6723ecf2 100644
--- a/youtube_dl/extractor/discovery.py
+++ b/youtube_dl/extractor/discovery.py
@@ -2,19 +2,19 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..utils import (
+    parse_duration,
     parse_iso8601,
-    int_or_none,
 )
+from ..compat import compat_str
 
 
 class DiscoveryIE(InfoExtractor):
     _VALID_URL = r'http://www\.discovery\.com\/[a-zA-Z0-9\-]*/[a-zA-Z0-9\-]*/videos/(?P<id>[a-zA-Z0-9_\-]*)(?:\.htm)?'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.discovery.com/tv-shows/mythbusters/videos/mission-impossible-outtakes.htm',
-        'md5': '3c69d77d9b0d82bfd5e5932a60f26504',
         'info_dict': {
-            'id': 'mission-impossible-outtakes',
-            'ext': 'flv',
+            'id': '20769',
+            'ext': 'mp4',
             'title': 'Mission Impossible Outtakes',
             'description': ('Watch Jamie Hyneman and Adam Savage practice being'
                             ' each other -- to the point of confusing Jamie\'s dog -- and '
@@ -24,22 +24,36 @@ class DiscoveryIE(InfoExtractor):
             'timestamp': 1303099200,
             'upload_date': '20110418',
         },
-    }
+        'params': {
+            'skip_download': True,  # requires ffmpeg
+        }
+    }, {
+        'url': 'http://www.discovery.com/tv-shows/mythbusters/videos/mythbusters-the-simpsons',
+        'info_dict': {
+            'id': 'mythbusters-the-simpsons',
+            'title': 'MythBusters: The Simpsons',
+        },
+        'playlist_count': 9,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
+        info = self._download_json(url + '?flat=1', video_id)
 
-        info = self._parse_json(self._search_regex(
-            r'(?s)<script type="application/ld\+json">(.*?)</script>',
-            webpage, 'video info'), video_id)
+        video_title = info.get('playlist_title') or info.get('video_title')
 
-        return {
-            'id': video_id,
-            'title': info['name'],
-            'url': info['contentURL'],
-            'description': info.get('description'),
-            'thumbnail': info.get('thumbnailUrl'),
-            'timestamp': parse_iso8601(info.get('uploadDate')),
-            'duration': int_or_none(info.get('duration')),
-        }
+        entries = [{
+            'id': compat_str(video_info['id']),
+            'formats': self._extract_m3u8_formats(
+                video_info['src'], video_id, ext='mp4',
+                note='Download m3u8 information for video %d' % (idx + 1)),
+            'title': video_info['title'],
+            'description': video_info.get('description'),
+            'duration': parse_duration(video_info.get('video_length')),
+            'webpage_url': video_info.get('href'),
+            'thumbnail': video_info.get('thumbnailURL'),
+            'alt_title': video_info.get('secondary_title'),
+            'timestamp': parse_iso8601(video_info.get('publishedDate')),
+        } for idx, video_info in enumerate(info['playlist'])]
+
+        return self.playlist_result(entries, video_id, video_title)

From 68477c3dab97733eb7a2feb8fcc90f648c29c2b4 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 7 Jun 2015 16:38:39 +0800
Subject: [PATCH 1530/6219] [tlc] Fix test failure due to DiscoveryIE changes

---
 youtube_dl/extractor/tlc.py | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/tlc.py b/youtube_dl/extractor/tlc.py
index 9f9e388c5..13263614c 100644
--- a/youtube_dl/extractor/tlc.py
+++ b/youtube_dl/extractor/tlc.py
@@ -12,17 +12,22 @@ class TlcIE(DiscoveryIE):
     IE_NAME = 'tlc.com'
     _VALID_URL = r'http://www\.tlc\.com\/[a-zA-Z0-9\-]*/[a-zA-Z0-9\-]*/videos/(?P<id>[a-zA-Z0-9\-]*)(.htm)?'
 
-    _TEST = {
+    # DiscoveryIE has _TESTS
+    _TESTS = [{
         'url': 'http://www.tlc.com/tv-shows/cake-boss/videos/too-big-to-fly.htm',
-        'md5': 'c4038f4a9b44d0b5d74caaa64ed2a01a',
         'info_dict': {
-            'id': '853232',
+            'id': '104493',
             'ext': 'mp4',
-            'title': 'Cake Boss: Too Big to Fly',
+            'title': 'Too Big to Fly',
             'description': 'Buddy has taken on a high flying task.',
             'duration': 119,
+            'timestamp': 1393365060,
+            'upload_date': '20140225',
         },
-    }
+        'params': {
+            'skip_download': True,  # requires ffmpef
+        },
+    }]
 
 
 class TlcDeIE(InfoExtractor):

From edb99d4c18475ba27fae4f7d0ec6e3db9b574885 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 8 Jun 2015 01:17:21 +0800
Subject: [PATCH 1531/6219] [instagram] Handling null values (fixes #5919)

I didn't add the test case here because it takes too much time. (7
minutes on my machine)
---
 youtube_dl/extractor/instagram.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/instagram.py b/youtube_dl/extractor/instagram.py
index b10755788..b92367a9d 100644
--- a/youtube_dl/extractor/instagram.py
+++ b/youtube_dl/extractor/instagram.py
@@ -100,7 +100,9 @@ class InstagramUserIE(InfoExtractor):
                 thumbnails_el = it.get('images', {})
                 thumbnail = thumbnails_el.get('thumbnail', {}).get('url')
 
-                title = it.get('caption', {}).get('text', it['id'])
+                # In some cases caption is null, which corresponds to None
+                # in python. As a result, it.get('caption', {}) gives None
+                title = (it.get('caption') or {}).get('text', it['id'])
 
                 entries.append({
                     'id': it['id'],

From e1ec93304dfcf385380feb95a3777c796cc49420 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 8 Jun 2015 01:46:33 +0800
Subject: [PATCH 1532/6219] [instagram:user] Truncate title to 80 characters
 (#5919)

This is a workaround. Currently YoutubeDL.process_info() truncates
info_dict['title'] to 200 characters, but the implementation can't
handle wide characters.
---
 youtube_dl/extractor/instagram.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/instagram.py b/youtube_dl/extractor/instagram.py
index b92367a9d..3d78f78c4 100644
--- a/youtube_dl/extractor/instagram.py
+++ b/youtube_dl/extractor/instagram.py
@@ -3,7 +3,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import int_or_none
+from ..utils import (
+    int_or_none,
+    limit_length,
+)
 
 
 class InstagramIE(InfoExtractor):
@@ -106,7 +109,7 @@ class InstagramUserIE(InfoExtractor):
 
                 entries.append({
                     'id': it['id'],
-                    'title': title,
+                    'title': limit_length(title, 80),
                     'formats': formats,
                     'thumbnail': thumbnail,
                     'webpage_url': it.get('link'),

From 788be3313df7ad020dc0a98bd5ed43a60120fb3b Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 8 Jun 2015 13:32:04 +0800
Subject: [PATCH 1533/6219] [cnet] Fix theplatform vid extraction (fixes #5924)

---
 youtube_dl/extractor/cnet.py | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/cnet.py b/youtube_dl/extractor/cnet.py
index 3145b3051..5dd69bff7 100644
--- a/youtube_dl/extractor/cnet.py
+++ b/youtube_dl/extractor/cnet.py
@@ -11,7 +11,7 @@ from ..utils import (
 
 class CNETIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?cnet\.com/videos/(?P<id>[^/]+)/'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.cnet.com/videos/hands-on-with-microsofts-windows-8-1-update/',
         'info_dict': {
             'id': '56f4ea68-bd21-4852-b08c-4de5b8354c60',
@@ -25,7 +25,20 @@ class CNETIE(InfoExtractor):
         'params': {
             'skip_download': 'requires rtmpdump',
         }
-    }
+    }, {
+        'url': 'http://www.cnet.com/videos/whiny-pothole-tweets-at-local-government-when-hit-by-cars-tomorrow-daily-187/',
+        'info_dict': {
+            'id': '56527b93-d25d-44e3-b738-f989ce2e49ba',
+            'ext': 'flv',
+            'description': 'Khail and Ashley wonder what other civic woes can be solved by self-tweeting objects, investigate a new kind of VR camera and watch an origami robot self-assemble, walk, climb, dig and dissolve. #TDPothole',
+            'uploader_id': 'b163284d-6b73-44fc-b3e6-3da66c392d40',
+            'uploader': 'Ashley Esqueda',
+            'title': 'Whiny potholes tweet at local government when hit by cars (Tomorrow Daily 187)',
+        },
+        'params': {
+            'skip_download': True,  # requires rtmpdump
+        },
+    }]
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
@@ -42,7 +55,7 @@ class CNETIE(InfoExtractor):
             raise ExtractorError('Cannot find video data')
 
         mpx_account = data['config']['players']['default']['mpx_account']
-        vid = vdata['files']['rtmp']
+        vid = vdata['files'].get('rtmp', vdata['files']['hds'])
         tp_link = 'http://link.theplatform.com/s/%s/%s' % (mpx_account, vid)
 
         video_id = vdata['id']

From 01e21b89eefc32bcc4a92c3a82658cee139b6b2c Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 8 Jun 2015 17:39:55 +0800
Subject: [PATCH 1534/6219] [noco] Skip invalid timestamps (closes #5826)

---
 youtube_dl/extractor/noco.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/noco.py b/youtube_dl/extractor/noco.py
index 664dc81d4..5bbd2dcf6 100644
--- a/youtube_dl/extractor/noco.py
+++ b/youtube_dl/extractor/noco.py
@@ -166,6 +166,10 @@ class NocoIE(InfoExtractor):
         self._sort_formats(formats)
 
         timestamp = parse_iso8601(show.get('online_date_start_utc'), ' ')
+
+        if timestamp is not None and timestamp < 0:
+            timestamp = None
+
         uploader = show.get('partner_name')
         uploader_id = show.get('partner_key')
         duration = float_or_none(show.get('duration_ms'), 1000)

From a55e36f48d1f0dc5454b144c7373361f284b9236 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 8 Jun 2015 21:05:17 +0600
Subject: [PATCH 1535/6219] [YoutubeDL] Handle out-of-range timestamps (#5826)

---
 youtube_dl/YoutubeDL.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index aa6ec9d9a..b1f792d4e 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1016,13 +1016,13 @@ class YoutubeDL(object):
             info_dict['display_id'] = info_dict['id']
 
         if info_dict.get('upload_date') is None and info_dict.get('timestamp') is not None:
-            # Working around negative timestamps in Windows
-            # (see http://bugs.python.org/issue1646728)
-            if info_dict['timestamp'] < 0 and os.name == 'nt':
-                info_dict['timestamp'] = 0
-            upload_date = datetime.datetime.utcfromtimestamp(
-                info_dict['timestamp'])
-            info_dict['upload_date'] = upload_date.strftime('%Y%m%d')
+            # Working around out-of-range timestamp values (e.g. negative ones on Windows,
+            # see http://bugs.python.org/issue1646728)
+            try:
+                upload_date = datetime.datetime.utcfromtimestamp(info_dict['timestamp'])
+                info_dict['upload_date'] = upload_date.strftime('%Y%m%d')
+            except (ValueError, OverflowError, OSError):
+                pass
 
         if self.params.get('listsubtitles', False):
             if 'automatic_captions' in info_dict:

From 627b96482567c1525dddfcceae2c16ff53c18b6a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 9 Jun 2015 11:41:17 +0800
Subject: [PATCH 1536/6219] [kickstarted] Extract thumbnails in embedded videos
 (#5929)

---
 youtube_dl/extractor/kickstarter.py | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/kickstarter.py b/youtube_dl/extractor/kickstarter.py
index 7d4b57056..1d391e69f 100644
--- a/youtube_dl/extractor/kickstarter.py
+++ b/youtube_dl/extractor/kickstarter.py
@@ -28,6 +28,14 @@ class KickStarterIE(InfoExtractor):
             'uploader': 'Pebble Technology',
             'title': 'Pebble iOS Notifications',
         }
+    }, {
+        'url': 'https://www.kickstarter.com/projects/1420158244/power-drive-2000/widget/video.html',
+        'info_dict': {
+            'id': '1420158244',
+            'ext': 'mp4',
+            'title': 'Power Drive 2000',
+        },
+        'expected_warnings': ['OpenGraph description'],
     }]
 
     def _real_extract(self, url):
@@ -48,10 +56,15 @@ class KickStarterIE(InfoExtractor):
                 'title': title,
             }
 
+        thumbnail = self._og_search_thumbnail(webpage, default=None)
+        if thumbnail is None:
+            thumbnail = self._html_search_regex(
+                r'<img[^>]+class="[^"]+\s*poster\s*[^"]+"[^>]+src="([^"]+)"',
+                webpage, 'thumbnail image', fatal=False)
         return {
             'id': video_id,
             'url': video_url,
             'title': title,
             'description': self._og_search_description(webpage),
-            'thumbnail': self._og_search_thumbnail(webpage),
+            'thumbnail': thumbnail,
         }

From e1b9322b091122b6f6832c70e3a845a84ee1764e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 9 Jun 2015 14:48:18 +0800
Subject: [PATCH 1537/6219] [youtube] Restricter DASH signature pattern

A problematic DASH url is:
https://manifest.googlevideo.com/api/manifest/dash/mm/35/key/yt5/ip/140.112.247.145/ms/pm/mv/s/mt/1433794435/id/o-AD2Od_dsOlAUYPu03ZsVWKSbGEbCJJrMp9vnXGhnyRhd/mn/sn-aigllm7r/sparams/as%2Chfr%2Cid%2Cip%2Cipbits%2Citag%2Cmm%2Cmn%2Cms%2Cmv%2Cnh%2Cpl%2Cplayback_host%2Crequiressl%2Csource%2Cexpire/fexp/9406009%2C9406821%2C9407575%2C9408142%2C9408420%2C9408710%2C9409121%2C9409208%2C9412514%2C9412780%2C9413208%2C9413426%2C9413476%2C9413503%2C9415304%2C9415753/upn/viDQrs8SnmE/as/fmp4_audio_clear%2Cwebm_audio_clear%2Cfmp4_sd_hd_clear%2Cwebm_sd_hd_clear%2Cwebm2_sd_hd_clear/playback_host/r4---sn-aigllm7r.googlevideo.com/ipbits/0/requiressl/yes/pl/20/itag/0/source/youtube/expire/1433824806/nh/EAQ/signature/81ABE6391E351BA495F5B041B00FF1257A353318.1A6E48ABB74E8F4AE73CA2CB1F963FC34E33DEE7/sver/3/hfr/1
---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 419f7b019..083da777d 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -785,7 +785,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             s = mobj.group(1)
             dec_s = self._decrypt_signature(s, video_id, player_url, age_gate)
             return '/signature/%s' % dec_s
-        dash_manifest_url = re.sub(r'/s/([\w\.]+)', decrypt_sig, dash_manifest_url)
+        dash_manifest_url = re.sub(r'/s/([a-fA-F0-9\.]+)', decrypt_sig, dash_manifest_url)
         dash_doc = self._download_xml(
             dash_manifest_url, video_id,
             note='Downloading DASH manifest',

From d9cf48e81e38f4bf151a8648c48d6e5233325b40 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 9 Jun 2015 20:36:08 +0600
Subject: [PATCH 1538/6219] [spiegeltv] Extract all formats and prefer hls
 (Closes #5843)

---
 youtube_dl/extractor/spiegeltv.py | 45 ++++++++++++++++++++++++-------
 1 file changed, 36 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/spiegeltv.py b/youtube_dl/extractor/spiegeltv.py
index 359722ad6..08a5c4314 100644
--- a/youtube_dl/extractor/spiegeltv.py
+++ b/youtube_dl/extractor/spiegeltv.py
@@ -2,7 +2,11 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..utils import float_or_none
+from ..compat import compat_urllib_parse_urlparse
+from ..utils import (
+    determine_ext,
+    float_or_none,
+)
 
 
 class SpiegeltvIE(InfoExtractor):
@@ -17,7 +21,7 @@ class SpiegeltvIE(InfoExtractor):
             'thumbnail': 're:http://.*\.jpg$',
         },
         'params': {
-            # rtmp download
+            # m3u8 download
             'skip_download': True,
         }
     }, {
@@ -53,7 +57,35 @@ class SpiegeltvIE(InfoExtractor):
         server_json = self._download_json(
             'http://spiegeltv-prod-static.s3.amazonaws.com/projectConfigs/projectConfig.json',
             video_id, note='Downloading server information')
-        server = server_json['streamingserver'][0]['endpoint']
+
+        format = '16x9' if is_wide else '4x3'
+
+        formats = []
+        for streamingserver in server_json['streamingserver']:
+            endpoint = streamingserver.get('endpoint')
+            if not endpoint:
+                continue
+            play_path = 'mp4:%s_spiegeltv_0500_%s.m4v' % (uuid, format)
+            if endpoint.startswith('rtmp'):
+                formats.append({
+                    'url': endpoint,
+                    'format_id': 'rtmp',
+                    'app': compat_urllib_parse_urlparse(endpoint).path[1:],
+                    'play_path': play_path,
+                    'player_path': 'http://prod-static.spiegel.tv/frontend-076.swf',
+                    'ext': 'flv',
+                    'rtmp_live': True,
+                })
+            elif determine_ext(endpoint) == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    endpoint.replace('[video]', play_path),
+                    video_id, 'm4v',
+                    preference=1,  # Prefer hls since it allows to workaround georestriction
+                    m3u8_id='hls'))
+            else:
+                formats.append({
+                    'url': endpoint,
+                })
 
         thumbnails = []
         for image in media_json['images']:
@@ -65,17 +97,12 @@ class SpiegeltvIE(InfoExtractor):
 
         description = media_json['subtitle']
         duration = float_or_none(media_json.get('duration_in_ms'), scale=1000)
-        format = '16x9' if is_wide else '4x3'
-
-        url = server + 'mp4:' + uuid + '_spiegeltv_0500_' + format + '.m4v'
 
         return {
             'id': video_id,
             'title': title,
-            'url': url,
-            'ext': 'm4v',
             'description': description,
             'duration': duration,
             'thumbnails': thumbnails,
-            'rtmp_live': True,
+            'formats': formats,
         }

From 9bf99891d08d166ac1d81b652dc487bb940fa685 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 9 Jun 2015 21:23:53 +0600
Subject: [PATCH 1539/6219] [cbs] Add support for colbertlateshow (Closes
 #5888)

---
 youtube_dl/extractor/cbs.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/cbs.py b/youtube_dl/extractor/cbs.py
index 1ceb9d8d9..89614a3c9 100644
--- a/youtube_dl/extractor/cbs.py
+++ b/youtube_dl/extractor/cbs.py
@@ -4,7 +4,7 @@ from .common import InfoExtractor
 
 
 class CBSIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?cbs\.com/shows/[^/]+/(?:video|artist)/(?P<id>[^/]+)/.*'
+    _VALID_URL = r'https?://(?:(?:www\.)?cbs\.com/shows/[^/]+/(?:video|artist)|colbertlateshow\.com/(?:video|podcasts))/(?P<id>[^/]+)/.*'
 
     _TESTS = [{
         'url': 'http://www.cbs.com/shows/garth-brooks/video/_u7W953k6la293J7EPTd9oHkSPs6Xn6_/connect-chat-feat-garth-brooks/',
@@ -34,12 +34,18 @@ class CBSIE(InfoExtractor):
             'skip_download': True,
         },
         '_skip': 'Blocked outside the US',
+    }, {
+        'url': 'http://colbertlateshow.com/video/8GmB0oY0McANFvp2aEffk9jZZZ2YyXxy/the-colbeard/',
+        'only_matching': True,
+    }, {
+        'url': 'http://colbertlateshow.com/podcasts/dYSwjqPs_X1tvbV_P2FcPWRa_qT6akTC/in-the-bad-room-with-stephen/',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
         real_id = self._search_regex(
-            r"video\.settings\.pid\s*=\s*'([^']+)';",
+            [r"video\.settings\.pid\s*=\s*'([^']+)';", r"cbsplayer\.pid\s*=\s*'([^']+)';"],
             webpage, 'real video ID')
         return self.url_result('theplatform:%s' % real_id)

From 9d581f3d5224140ca35ebd06d614b929e9252cd4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 9 Jun 2015 21:39:45 +0600
Subject: [PATCH 1540/6219] [cbs] Extract display_id

---
 youtube_dl/extractor/cbs.py | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/cbs.py b/youtube_dl/extractor/cbs.py
index 89614a3c9..75fffb156 100644
--- a/youtube_dl/extractor/cbs.py
+++ b/youtube_dl/extractor/cbs.py
@@ -4,12 +4,13 @@ from .common import InfoExtractor
 
 
 class CBSIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:(?:www\.)?cbs\.com/shows/[^/]+/(?:video|artist)|colbertlateshow\.com/(?:video|podcasts))/(?P<id>[^/]+)/.*'
+    _VALID_URL = r'https?://(?:www\.)?(?:cbs\.com/shows/[^/]+/(?:video|artist)|colbertlateshow\.com/(?:video|podcasts))/[^/]+/(?P<id>[^/]+)'
 
     _TESTS = [{
         'url': 'http://www.cbs.com/shows/garth-brooks/video/_u7W953k6la293J7EPTd9oHkSPs6Xn6_/connect-chat-feat-garth-brooks/',
         'info_dict': {
             'id': '4JUVEwq3wUT7',
+            'display_id': 'connect-chat-feat-garth-brooks',
             'ext': 'flv',
             'title': 'Connect Chat feat. Garth Brooks',
             'description': 'Connect with country music singer Garth Brooks, as he chats with fans on Wednesday November 27, 2013. Be sure to tune in to Garth Brooks: Live from Las Vegas, Friday November 29, at 9/8c on CBS!',
@@ -24,6 +25,7 @@ class CBSIE(InfoExtractor):
         'url': 'http://www.cbs.com/shows/liveonletterman/artist/221752/st-vincent/',
         'info_dict': {
             'id': 'WWF_5KqY3PK1',
+            'display_id': 'st-vincent',
             'ext': 'flv',
             'title': 'Live on Letterman - St. Vincent',
             'description': 'Live On Letterman: St. Vincent in concert from New York\'s Ed Sullivan Theater on Tuesday, July 16, 2014.',
@@ -38,14 +40,19 @@ class CBSIE(InfoExtractor):
         'url': 'http://colbertlateshow.com/video/8GmB0oY0McANFvp2aEffk9jZZZ2YyXxy/the-colbeard/',
         'only_matching': True,
     }, {
-        'url': 'http://colbertlateshow.com/podcasts/dYSwjqPs_X1tvbV_P2FcPWRa_qT6akTC/in-the-bad-room-with-stephen/',
+        'url': 'http://www.colbertlateshow.com/podcasts/dYSwjqPs_X1tvbV_P2FcPWRa_qT6akTC/in-the-bad-room-with-stephen/',
         'only_matching': True,
     }]
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
         real_id = self._search_regex(
             [r"video\.settings\.pid\s*=\s*'([^']+)';", r"cbsplayer\.pid\s*=\s*'([^']+)';"],
             webpage, 'real video ID')
-        return self.url_result('theplatform:%s' % real_id)
+        return {
+            '_type': 'url_transparent',
+            'ie_key': 'ThePlatform',
+            'url': 'theplatform:%s' % real_id,
+            'display_id': display_id,
+        }

From 6e054aacca2ac44413ed37ee8b1d63a09c8b4ae2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 9 Jun 2015 23:07:22 +0600
Subject: [PATCH 1541/6219] [theplatform] Take care of /select/media URLs
 (Closes #5746)

---
 youtube_dl/extractor/theplatform.py | 23 +++++++++++++++++++----
 1 file changed, 19 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index 92731ad3d..48c6ff03f 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -26,7 +26,7 @@ _x = lambda p: xpath_with_ns(p, {'smil': 'http://www.w3.org/2005/SMIL21/Language
 class ThePlatformIE(InfoExtractor):
     _VALID_URL = r'''(?x)
         (?:https?://(?:link|player)\.theplatform\.com/[sp]/(?P<provider_id>[^/]+)/
-           (?P<config>(?:[^/\?]+/(?:swf|config)|onsite)/select/)?
+           (?:(?P<config>(?:[^/\?]+/(?:swf|config)|onsite)/select/)|(?P<media>(?:[^/]+/)+select/media/))?
          |theplatform:)(?P<id>[^/\?&]+)'''
 
     _TESTS = [{
@@ -56,6 +56,17 @@ class ThePlatformIE(InfoExtractor):
             # rtmp download
             'skip_download': True,
         }
+    }, {
+        'url': 'https://player.theplatform.com/p/D6x-PC/pulse_preview/embed/select/media/yMBg9E8KFxZD',
+        'info_dict': {
+            'id': 'yMBg9E8KFxZD',
+            'ext': 'mp4',
+            'description': 'md5:644ad9188d655b742f942bf2e06b002d',
+            'title': 'HIGHLIGHTS: USA bag first ever series Cup win',
+        }
+    }, {
+        'url': 'http://player.theplatform.com/p/NnzsPC/widget/select/media/4Y0TlYUr_ZT7',
+        'only_matching': True,
     }]
 
     @staticmethod
@@ -85,6 +96,11 @@ class ThePlatformIE(InfoExtractor):
         if not provider_id:
             provider_id = 'dJ5BDC'
 
+        path = provider_id
+        if mobj.group('media'):
+            path += '/media'
+        path += '/' + video_id
+
         if smuggled_data.get('force_smil_url', False):
             smil_url = url
         elif mobj.group('config'):
@@ -94,8 +110,7 @@ class ThePlatformIE(InfoExtractor):
             config = self._download_json(config_url, video_id, 'Downloading config')
             smil_url = config['releaseUrl'] + '&format=SMIL&formats=MPEG4&manifest=f4m'
         else:
-            smil_url = ('http://link.theplatform.com/s/{0}/{1}/meta.smil?'
-                        'format=smil&mbr=true'.format(provider_id, video_id))
+            smil_url = 'http://link.theplatform.com/s/%s/meta.smil?format=smil&mbr=true' % path
 
         sig = smuggled_data.get('sig')
         if sig:
@@ -112,7 +127,7 @@ class ThePlatformIE(InfoExtractor):
         else:
             raise ExtractorError(error_msg, expected=True)
 
-        info_url = 'http://link.theplatform.com/s/{0}/{1}?format=preview'.format(provider_id, video_id)
+        info_url = 'http://link.theplatform.com/s/%s?format=preview' % path
         info_json = self._download_webpage(info_url, video_id)
         info = json.loads(info_json)
 

From bd5bc0cd5af257abf7a1a4c14a9dd39c4f97e622 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 9 Jun 2015 23:12:13 +0600
Subject: [PATCH 1542/6219] [theplatform] Check for /select/media URLs first
 (#5746)

---
 youtube_dl/extractor/theplatform.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index 48c6ff03f..83d833e30 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -26,7 +26,7 @@ _x = lambda p: xpath_with_ns(p, {'smil': 'http://www.w3.org/2005/SMIL21/Language
 class ThePlatformIE(InfoExtractor):
     _VALID_URL = r'''(?x)
         (?:https?://(?:link|player)\.theplatform\.com/[sp]/(?P<provider_id>[^/]+)/
-           (?:(?P<config>(?:[^/\?]+/(?:swf|config)|onsite)/select/)|(?P<media>(?:[^/]+/)+select/media/))?
+           (?:(?P<media>(?:[^/]+/)+select/media/)|(?P<config>(?:[^/\?]+/(?:swf|config)|onsite)/select/))?
          |theplatform:)(?P<id>[^/\?&]+)'''
 
     _TESTS = [{

From 70219b0f4371fe54cc72d025ce06fc4691ba12fa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 9 Jun 2015 23:49:11 +0200
Subject: [PATCH 1543/6219] [youtube:playlist] Use an iterator for the entries
 (closes #5935)

So that '--playlist-end' downloads only the required pages.
---
 youtube_dl/extractor/youtube.py | 47 ++++++++++++++++-----------------
 1 file changed, 23 insertions(+), 24 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 083da777d..3448bec4f 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1290,7 +1290,6 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
     def _extract_playlist(self, playlist_id):
         url = self._TEMPLATE_URL % playlist_id
         page = self._download_webpage(url, playlist_id)
-        more_widget_html = content_html = page
 
         for match in re.findall(r'<div class="yt-alert-message">([^<]+)</div>', page):
             match = match.strip()
@@ -1310,36 +1309,36 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
                 self.report_warning('Youtube gives an alert message: ' + match)
 
         # Extract the video ids from the playlist pages
-        ids = []
+        def _entries():
+            more_widget_html = content_html = page
+            for page_num in itertools.count(1):
+                matches = re.finditer(self._VIDEO_RE, content_html)
+                # We remove the duplicates and the link with index 0
+                # (it's not the first video of the playlist)
+                new_ids = orderedSet(m.group('id') for m in matches if m.group('index') != '0')
+                for vid_id in new_ids:
+                    yield self.url_result(vid_id, 'Youtube', video_id=vid_id)
 
-        for page_num in itertools.count(1):
-            matches = re.finditer(self._VIDEO_RE, content_html)
-            # We remove the duplicates and the link with index 0
-            # (it's not the first video of the playlist)
-            new_ids = orderedSet(m.group('id') for m in matches if m.group('index') != '0')
-            ids.extend(new_ids)
+                mobj = re.search(r'data-uix-load-more-href="/?(?P<more>[^"]+)"', more_widget_html)
+                if not mobj:
+                    break
 
-            mobj = re.search(r'data-uix-load-more-href="/?(?P<more>[^"]+)"', more_widget_html)
-            if not mobj:
-                break
-
-            more = self._download_json(
-                'https://youtube.com/%s' % mobj.group('more'), playlist_id,
-                'Downloading page #%s' % page_num,
-                transform_source=uppercase_escape)
-            content_html = more['content_html']
-            if not content_html.strip():
-                # Some webpages show a "Load more" button but they don't
-                # have more videos
-                break
-            more_widget_html = more['load_more_widget_html']
+                more = self._download_json(
+                    'https://youtube.com/%s' % mobj.group('more'), playlist_id,
+                    'Downloading page #%s' % page_num,
+                    transform_source=uppercase_escape)
+                content_html = more['content_html']
+                if not content_html.strip():
+                    # Some webpages show a "Load more" button but they don't
+                    # have more videos
+                    break
+                more_widget_html = more['load_more_widget_html']
 
         playlist_title = self._html_search_regex(
             r'(?s)<h1 class="pl-header-title[^"]*">\s*(.*?)\s*</h1>',
             page, 'title')
 
-        url_results = self._ids_to_results(ids)
-        return self.playlist_result(url_results, playlist_id, playlist_title)
+        return self.playlist_result(_entries(), playlist_id, playlist_title)
 
     def _real_extract(self, url):
         # Extract playlist id

From d84f1d14b526c4a5359117a58f25691a3da4c97e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aur=C3=A9lio=20A=2E=20Heckert?= <aurelio@colivre.coop.br>
Date: Tue, 9 Jun 2015 22:08:16 -0300
Subject: [PATCH 1544/6219] Adds support for XviD output with extra
 parametrization

As the "LG Time Machine" (a (not so) smart TV) has a limitation for video dimensions (as for codecs), I take to implement an extra parameter `--pp-params` where we can send extra parameterization for the video converter (post-processor).

Example:
```
$ youtube-dl --recode-video=xvid --pp-params='-s 720x480' -c https://www.youtube.com/watch?v=BE7Qoe2ZiXE
```
That works fine on a 4yo LG Time Machine.

Closes #5733
---
 README.md                          |  3 ++-
 youtube_dl/__init__.py             |  5 ++++-
 youtube_dl/options.py              |  6 +++++-
 youtube_dl/postprocessor/ffmpeg.py | 14 ++++++++++----
 4 files changed, 21 insertions(+), 7 deletions(-)

diff --git a/README.md b/README.md
index f3d83c89f..726ec9cf2 100644
--- a/README.md
+++ b/README.md
@@ -213,7 +213,8 @@ which means you can modify it, redistribute it or use it however you like.
     --audio-format FORMAT            Specify audio format: "best", "aac", "vorbis", "mp3", "m4a", "opus", or "wav"; "best" by default
     --audio-quality QUALITY          Specify ffmpeg/avconv audio quality, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K (default
                                      5)
-    --recode-video FORMAT            Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm|mkv)
+    --recode-video FORMAT            Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm|mkv|xvid)
+    --pp-params                      Extra parameters for video post-processor. The params will be splited on spaces.
     -k, --keep-video                 Keep the video file on disk after the post-processing; the video is erased by default
     --no-post-overwrites             Do not overwrite post-processed files; the post-processed files are overwritten by default
     --embed-subs                     Embed subtitles in the video (only for mkv and mp4 videos)
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index ace17857c..5b28e4817 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -169,8 +169,10 @@ def _real_main(argv=None):
         if not opts.audioquality.isdigit():
             parser.error('invalid audio quality specified')
     if opts.recodevideo is not None:
-        if opts.recodevideo not in ['mp4', 'flv', 'webm', 'ogg', 'mkv']:
+        if opts.recodevideo not in ['mp4', 'flv', 'webm', 'ogg', 'mkv', 'xvid']:
             parser.error('invalid video recode format specified')
+    if opts.pp_params is not None:
+        opts.pp_params = opts.pp_params.split()
     if opts.convertsubtitles is not None:
         if opts.convertsubtitles not in ['srt', 'vtt', 'ass']:
             parser.error('invalid subtitle format specified')
@@ -227,6 +229,7 @@ def _real_main(argv=None):
         postprocessors.append({
             'key': 'FFmpegVideoConvertor',
             'preferedformat': opts.recodevideo,
+            'extra_params': opts.pp_params
         })
     if opts.convertsubtitles:
         postprocessors.append({
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 689fa7595..ceb4b5f38 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -686,7 +686,11 @@ def parseOpts(overrideArguments=None):
     postproc.add_option(
         '--recode-video',
         metavar='FORMAT', dest='recodevideo', default=None,
-        help='Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm|mkv)')
+        help='Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm|mkv|xvid)')
+    postproc.add_option(
+        '--pp-params',
+        dest='pp_params', default=None,
+        help='Extra parameters for video post-processor. The params will be splited on spaces.')
     postproc.add_option(
         '-k', '--keep-video',
         action='store_true', dest='keepvideo', default=False,
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index cc65b34e7..a696b12b4 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -287,22 +287,28 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
 
 
 class FFmpegVideoConvertorPP(FFmpegPostProcessor):
-    def __init__(self, downloader=None, preferedformat=None):
+    def __init__(self, downloader=None, preferedformat=None, extra_params=[]):
         super(FFmpegVideoConvertorPP, self).__init__(downloader)
         self._preferedformat = preferedformat
+        self._extra_params = extra_params
 
     def run(self, information):
         path = information['filepath']
         prefix, sep, ext = path.rpartition('.')
-        outpath = prefix + sep + self._preferedformat
+        ext = self._preferedformat
+        options = self._extra_params
+        if self._preferedformat == 'xvid':
+            ext = 'avi'
+            options.extend(['-c:v', 'libxvid', '-vtag', 'XVID'])
+        outpath = prefix + sep + ext
         if information['ext'] == self._preferedformat:
             self._downloader.to_screen('[ffmpeg] Not converting video file %s - already is in target format %s' % (path, self._preferedformat))
             return [], information
         self._downloader.to_screen('[' + 'ffmpeg' + '] Converting video from %s to %s, Destination: ' % (information['ext'], self._preferedformat) + outpath)
-        self.run_ffmpeg(path, outpath, [])
+        self.run_ffmpeg(path, outpath, options)
         information['filepath'] = outpath
         information['format'] = self._preferedformat
-        information['ext'] = self._preferedformat
+        information['ext'] = ext
         return [path], information
 
 

From 0c8662d2b6f033ad42f1cc97989d4975629b524b Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 10 Jun 2015 13:40:41 +0800
Subject: [PATCH 1545/6219] [youtube] Fix a TypeError caused by
 4da31bd56629054497634d041035e4bd6fcfacbb

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 2424ac2c0..a1906eef6 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -816,7 +816,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                         'filesize': filesize,
                         'fps': int_or_none(r.attrib.get('frameRate')),
                     }
-                    if len(segment_list):
+                    if segment_list is not None:
                         f.update({
                             'initialization_url': segment_list.find('{urn:mpeg:DASH:schema:MPD:2011}Initialization').attrib['sourceURL'],
                             'segment_urls': [segment.attrib.get('media') for segment in segment_list.findall('{urn:mpeg:DASH:schema:MPD:2011}SegmentURL')],

From 93dfcb9357b400b4d7e353d0a9db0e0194135b19 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 10 Jun 2015 13:44:54 +0800
Subject: [PATCH 1546/6219] [downloader/dash] Do not pollute ```self```

---
 youtube_dl/downloader/dash.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/downloader/dash.py b/youtube_dl/downloader/dash.py
index 5f14658ba..cd84e0b07 100644
--- a/youtube_dl/downloader/dash.py
+++ b/youtube_dl/downloader/dash.py
@@ -16,14 +16,14 @@ class DashSegmentsFD(FileDownloader):
         base_url = info_dict['url']
         segment_urls = info_dict['segment_urls']
 
-        self.byte_counter = 0
+        byte_counter = 0
 
         def append_url_to_file(outf, target_url, target_name):
             self.to_screen('[DashSegments] %s: Downloading %s' % (info_dict['id'], target_name))
             req = compat_urllib_request.Request(target_url)
             data = self.ydl.urlopen(req).read()
             outf.write(data)
-            self.byte_counter += len(data)
+            return len(data)
 
         def combine_url(base_url, target_url):
             if re.match(r'^https?://', target_url):
@@ -35,15 +35,16 @@ class DashSegmentsFD(FileDownloader):
                 outf, combine_url(base_url, info_dict['initialization_url']),
                 'initialization segment')
             for i, segment_url in enumerate(segment_urls):
-                append_url_to_file(
+                segment_len = append_url_to_file(
                     outf, combine_url(base_url, segment_url),
                     'segment %d / %d' % (i + 1, len(segment_urls)))
+                byte_counter += segment_len
 
         self.try_rename(tmpfilename, filename)
 
         self._hook_progress({
-            'downloaded_bytes': self.byte_counter,
-            'total_bytes': self.byte_counter,
+            'downloaded_bytes': byte_counter,
+            'total_bytes': byte_counter,
             'filename': filename,
             'status': 'finished',
         })

From 7ebd5376feb493edd0bc04abd07bba89397b7307 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 10 Jun 2015 14:15:20 +0800
Subject: [PATCH 1547/6219] [nfl] Relax _VALID_URL (fixes #5940)

---
 youtube_dl/extractor/nfl.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nfl.py b/youtube_dl/extractor/nfl.py
index 2684dd250..dc54634a5 100644
--- a/youtube_dl/extractor/nfl.py
+++ b/youtube_dl/extractor/nfl.py
@@ -19,7 +19,7 @@ class NFLIE(InfoExtractor):
     _VALID_URL = r'''(?x)https?://
         (?P<host>(?:www\.)?(?:nfl\.com|.*?\.clubs\.nfl\.com))/
         (?:.+?/)*
-        (?P<id>(?:\d[a-z]{2}\d{13}|\w{8}\-(?:\w{4}\-){3}\w{12}))'''
+        (?P<id>(?:[a-z0-9]{16}|\w{8}\-(?:\w{4}\-){3}\w{12}))'''
     _TESTS = [
         {
             'url': 'http://www.nfl.com/videos/nfl-game-highlights/0ap3000000398478/Week-3-Redskins-vs-Eagles-highlights',
@@ -58,6 +58,10 @@ class NFLIE(InfoExtractor):
                 'upload_date': '20150202',
             },
         },
+        {
+            'url': 'http://www.nfl.com/videos/nfl-network-top-ten/09000d5d810a6bd4/Top-10-Gutsiest-Performances-Jack-Youngblood',
+            'only_matching': True,
+        }
     ]
 
     @staticmethod

From 5bf3276e8d6ee7d017c8be04414398752cd9cdf3 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 10 Jun 2015 14:45:54 +0800
Subject: [PATCH 1548/6219] [downloader/dash] Add testing facility

---
 youtube_dl/downloader/dash.py | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/downloader/dash.py b/youtube_dl/downloader/dash.py
index cd84e0b07..a4685d307 100644
--- a/youtube_dl/downloader/dash.py
+++ b/youtube_dl/downloader/dash.py
@@ -16,12 +16,21 @@ class DashSegmentsFD(FileDownloader):
         base_url = info_dict['url']
         segment_urls = info_dict['segment_urls']
 
+        is_test = self.params.get('test', False)
+        remaining_bytes = self._TEST_FILE_SIZE if is_test else None
         byte_counter = 0
 
-        def append_url_to_file(outf, target_url, target_name):
+        def append_url_to_file(outf, target_url, target_name, remaining_bytes=None):
             self.to_screen('[DashSegments] %s: Downloading %s' % (info_dict['id'], target_name))
             req = compat_urllib_request.Request(target_url)
+            if remaining_bytes is not None:
+                req.add_header('Range', 'bytes=0-%d' % (remaining_bytes - 1))
+
             data = self.ydl.urlopen(req).read()
+
+            if remaining_bytes is not None:
+                data = data[:remaining_bytes]
+
             outf.write(data)
             return len(data)
 
@@ -37,8 +46,13 @@ class DashSegmentsFD(FileDownloader):
             for i, segment_url in enumerate(segment_urls):
                 segment_len = append_url_to_file(
                     outf, combine_url(base_url, segment_url),
-                    'segment %d / %d' % (i + 1, len(segment_urls)))
+                    'segment %d / %d' % (i + 1, len(segment_urls)),
+                    remaining_bytes)
                 byte_counter += segment_len
+                if remaining_bytes is not None:
+                    remaining_bytes -= segment_len
+                    if remaining_bytes <= 0:
+                        break
 
         self.try_rename(tmpfilename, filename)
 

From 8a1a26ce4c64d7a2c142718fc56f46d9a1c2c4f2 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 10 Jun 2015 14:47:02 +0800
Subject: [PATCH 1549/6219] [youtube] Add a test for the DASH segment
 downloader

---
 youtube_dl/extractor/youtube.py | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index a1906eef6..939f5e61f 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -516,6 +516,24 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'skip_download': 'requires avconv',
             }
         },
+        # DASH manifest with segment_list
+        {
+            'url': 'https://www.youtube.com/embed/CsmdDsKjzN8',
+            'md5': '8ce563a1d667b599d21064e982ab9e31',
+            'info_dict': {
+                'id': 'CsmdDsKjzN8',
+                'ext': 'mp4',
+                'upload_date': '20150510',
+                'uploader': 'Airtek',
+                'description': 'Retransmisión en directo de la XVIII media maratón de Zaragoza.',
+                'uploader_id': 'UCzTzUmjXxxacNnL8I3m4LnQ',
+                'title': 'Retransmisión XVIII Media maratón Zaragoza 2015',
+            },
+            'params': {
+                'youtube_include_dash_manifest': True,
+                'format': '135',  # bestvideo
+            }
+        }
     ]
 
     def __init__(self, *args, **kwargs):

From eb8be1fe76a9fbc285e6c957b3fdd5c05135ae3f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 10 Jun 2015 14:12:43 +0200
Subject: [PATCH 1550/6219] [rtbf] Extract all formats (closes #5947)

---
 youtube_dl/extractor/rtbf.py | 20 +++++++++++++++++---
 1 file changed, 17 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/rtbf.py b/youtube_dl/extractor/rtbf.py
index 5a381d9ce..e4215d546 100644
--- a/youtube_dl/extractor/rtbf.py
+++ b/youtube_dl/extractor/rtbf.py
@@ -21,6 +21,13 @@ class RTBFIE(InfoExtractor):
         }
     }
 
+    _QUALITIES = [
+        ('mobile', 'mobile'),
+        ('web', 'SD'),
+        ('url', 'MD'),
+        ('high', 'HD'),
+    ]
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
@@ -32,14 +39,21 @@ class RTBFIE(InfoExtractor):
                 r'data-video="([^"]+)"', webpage, 'data video')),
             video_id)
 
-        video_url = data.get('downloadUrl') or data.get('url')
-
         if data.get('provider').lower() == 'youtube':
+            video_url = data.get('downloadUrl') or data.get('url')
             return self.url_result(video_url, 'Youtube')
+        formats = []
+        for key, format_id in self._QUALITIES:
+            format_url = data['sources'].get(key)
+            if format_url:
+                formats.append({
+                    'format_id': format_id,
+                    'url': format_url,
+                })
 
         return {
             'id': video_id,
-            'url': video_url,
+            'formats': formats,
             'title': data['title'],
             'description': data.get('description') or data.get('subtitle'),
             'thumbnail': data.get('thumbnail'),

From f98470df690d053e45691ede2751ab6a4063082b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 10 Jun 2015 23:01:12 +0600
Subject: [PATCH 1551/6219] [bilibili] Fix FutureWarning

---
 youtube_dl/extractor/bilibili.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bilibili.py b/youtube_dl/extractor/bilibili.py
index 2103ed73a..bf60450c2 100644
--- a/youtube_dl/extractor/bilibili.py
+++ b/youtube_dl/extractor/bilibili.py
@@ -105,7 +105,7 @@ class BiliBiliIE(InfoExtractor):
                 'filesize': int_or_none(
                     lq_durl.find('./size'), get_attr='text'),
             }]
-            if hq_durl:
+            if hq_durl is not None:
                 formats.append({
                     'format_id': 'hq',
                     'quality': 2,

From a9d56c684319eaf8b9494bd8d2dc9d0f40485254 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 11 Jun 2015 19:03:22 +0600
Subject: [PATCH 1552/6219] [rtlnl] Improve _VALID_URL (#5950)

---
 youtube_dl/extractor/rtlnl.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/rtlnl.py b/youtube_dl/extractor/rtlnl.py
index cfce4550a..41d202c28 100644
--- a/youtube_dl/extractor/rtlnl.py
+++ b/youtube_dl/extractor/rtlnl.py
@@ -12,10 +12,10 @@ class RtlNlIE(InfoExtractor):
     IE_NAME = 'rtl.nl'
     IE_DESC = 'rtl.nl and rtlxl.nl'
     _VALID_URL = r'''(?x)
-        https?://(www\.)?
+        https?://(?:www\.)?
         (?:
             rtlxl\.nl/\#!/[^/]+/|
-            rtl\.nl/system/videoplayer/[^?#]+?/video_embed\.html\#uuid=
+            rtl\.nl/system/videoplayer/(?:[^/]+/)+(?:video_)?embed\.html\b.+?\buuid=
         )
         (?P<id>[0-9a-f-]+)'''
 
@@ -43,6 +43,9 @@ class RtlNlIE(InfoExtractor):
             'upload_date': '20150215',
             'description': 'Er zijn nieuwe beelden vrijgegeven die vlak na de aanslag in Kopenhagen zijn gemaakt. Op de video is goed te zien hoe omstanders zich bekommeren om één van de slachtoffers, terwijl de eerste agenten ter plaatse komen.',
         }
+    }, {
+        'url': 'http://www.rtl.nl/system/videoplayer/derden/embed.html#!/uuid=bb0353b0-d6a4-1dad-90e9-18fe75b8d1f0',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 97b570a94cc2387153af525f781e144bb4bb791e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 11 Jun 2015 19:04:12 +0600
Subject: [PATCH 1553/6219] [generic] Improve rtl.nl embeds detection (Closes
 #5950)

---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 759691365..75526384f 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1073,7 +1073,7 @@ class GenericIE(InfoExtractor):
 
         # Look for embedded rtl.nl player
         matches = re.findall(
-            r'<iframe\s+(?:[a-zA-Z-]+="[^"]+"\s+)*?src="((?:https?:)?//(?:www\.)?rtl\.nl/system/videoplayer/[^"]+video_embed[^"]+)"',
+            r'<iframe[^>]+?src="((?:https?:)?//(?:www\.)?rtl\.nl/system/videoplayer/[^"]+(?:video_)?embed[^"]+)"',
             webpage)
         if matches:
             return _playlist_from_matches(matches, ie='RtlNl')

From ff0f0b9172e432ebbfca88da91278554eb47c307 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 11 Jun 2015 22:18:08 +0600
Subject: [PATCH 1554/6219] [tube8] Fix extraction (Closes #5952)

---
 youtube_dl/extractor/tube8.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tube8.py b/youtube_dl/extractor/tube8.py
index 6ca8840b0..c9cb69333 100644
--- a/youtube_dl/extractor/tube8.py
+++ b/youtube_dl/extractor/tube8.py
@@ -47,7 +47,7 @@ class Tube8IE(InfoExtractor):
         webpage = self._download_webpage(req, display_id)
 
         flashvars = json.loads(self._html_search_regex(
-            r'flashvars\s*=\s*({.+?})', webpage, 'flashvars'))
+            r'flashvars\s*=\s*({.+?});\r?\n', webpage, 'flashvars'))
 
         video_url = flashvars['video_url']
         if flashvars.get('encrypted') is True:

From 99ac0390f559aa6dd09ffd8a15b9b562fda5f363 Mon Sep 17 00:00:00 2001
From: Hannu Lintala <hannu.lintala@gmail.com>
Date: Mon, 8 Jun 2015 05:58:41 +0300
Subject: [PATCH 1555/6219] [fivetv] Add extractor (Closes #5794)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/fivetv.py   | 67 ++++++++++++++++++++++++++++++++
 2 files changed, 68 insertions(+)
 create mode 100644 youtube_dl/extractor/fivetv.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 67eb96057..d10275d03 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -152,6 +152,7 @@ from .fc2 import FC2IE
 from .firstpost import FirstpostIE
 from .firsttv import FirstTVIE
 from .fivemin import FiveMinIE
+from .fivetv import FiveTVIE
 from .fktv import (
     FKTVIE,
     FKTVPosteckeIE,
diff --git a/youtube_dl/extractor/fivetv.py b/youtube_dl/extractor/fivetv.py
new file mode 100644
index 000000000..e47383b39
--- /dev/null
+++ b/youtube_dl/extractor/fivetv.py
@@ -0,0 +1,67 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+)
+
+
+class FiveTVIE(InfoExtractor):
+    _VALID_URL = r'http://(?:www\.)?5-tv\.ru/[^/]*/(?P<id>\d+)'
+    _TESTS = [
+        {
+            'url': 'http://5-tv.ru/news/96814/',
+            'md5': 'bbff554ad415ecf5416a2f48c22d9283',
+            'info_dict': {
+                'id': '96814',
+                'ext': 'mp4',
+                'title': 'Россияне выбрали имя для общенациональной платежной системы',
+                'description': 'md5:a8aa13e2b7ad36789e9f77a74b6de660',
+                'thumbnail': 're:^https?://.*\.jpg$',
+                'width': 480,
+                'height': 360,
+                'duration': 180,
+            },
+        },
+        {
+            'url': 'http://5-tv.ru/video/1021729/',
+            'md5': '299c8b72960efc9990acd2c784dc2296',
+            'info_dict': {
+                'id': '1021729',
+                'ext': 'mp4',
+                'title': '3D принтер',
+                'description': 'md5:d76c736d29ef7ec5c0cf7d7c65ffcb41',
+                'thumbnail': 're:^https?://.*\.jpg$',
+                'width': 480,
+                'height': 360,
+                'duration': 180,
+            },
+        },
+    ]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        video_link = self._search_regex(
+            r'(<a.*?class="videoplayer">)', webpage, 'video link')
+
+        url = self._search_regex(r'href="([^"]+)"', video_link, 'video url')
+        width = int_or_none(self._search_regex(
+            r'width:(\d+)px', video_link, 'width', default=None, fatal=False))
+        height = int_or_none(self._search_regex(
+            r'height:(\d+)px', video_link, 'height', default=None, fatal=False))
+        duration = int_or_none(self._og_search_property(
+            'video:duration', webpage, 'duration'))
+        return {
+            'id': video_id,
+            'url': url,
+            'width': width,
+            'height': height,
+            'title': self._og_search_title(webpage),
+            'description': self._og_search_description(webpage),
+            'thumbnail': self._og_search_thumbnail(webpage),
+            'duration': duration,
+        }

From 87446dc6186c7e4247fd7f9bc1046ef41f5d1a0f Mon Sep 17 00:00:00 2001
From: Hannu Lintala <hannu.lintala@gmail.com>
Date: Sun, 7 Jun 2015 17:25:30 +0300
Subject: [PATCH 1556/6219] [tvc] Add extractor (Closes #5795)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/tvc.py      | 79 ++++++++++++++++++++++++++++++++
 2 files changed, 80 insertions(+)
 create mode 100644 youtube_dl/extractor/tvc.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 67eb96057..8c4e12904 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -582,6 +582,7 @@ from .tv2 import (
     TV2ArticleIE,
 )
 from .tv4 import TV4IE
+from .tvc import TVCIE
 from .tvigle import TvigleIE
 from .tvp import TvpIE, TvpSeriesIE
 from .tvplay import TVPlayIE
diff --git a/youtube_dl/extractor/tvc.py b/youtube_dl/extractor/tvc.py
new file mode 100644
index 000000000..b62ab857c
--- /dev/null
+++ b/youtube_dl/extractor/tvc.py
@@ -0,0 +1,79 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    str_or_none,
+)
+
+
+class TVCIE(InfoExtractor):
+    _VALID_URL = r'http://(?:www\.)?tvc\.ru/.*/show/.*id/(?P<id>\d+)'
+    _TESTS = [
+        {
+            'url': 'http://www.tvc.ru/channel/brand/id/29/show/episodes/episode_id/39702/',
+            'md5': 'aa6fb3cf384e18a0ad3b30ee2898beba',
+            'info_dict': {
+                'id': '74622',
+                'display_id': '39702',
+                'ext': 'mp4',
+                'title': 'События. "События". Эфир от 22.05.2015 14:30',
+                'description': 'md5:ad7aa7db22903f983e687b8a3e98c6dd',
+                'thumbnail': 're:^https?://.*\.jpg$',
+                'duration': 1122,
+            },
+        },
+        {
+            'url': 'http://www.tvc.ru/news/show/id/69944',
+            'md5': 'b173128ee7b88b5b06c84e5f7880909f',
+            'info_dict': {
+                'id': '75399',
+                'display_id': '69944',
+                'ext': 'mp4',
+                'title': 'Эксперты: в столице встал вопрос о максимально безопасных остановках',
+                'description': 'md5:f675c8eaf23aab9df542d31773ed6518',
+                'thumbnail': 're:^https?://.*\.jpg$',
+                'duration': 278,
+            },
+        },
+    ]
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, display_id)
+
+        video_url = self._og_search_video_url(webpage)
+
+        video_id = self._search_regex(
+            r'video/iframe/id/(\d+)/', video_url, 'video id')
+
+        video_json_url = 'http://www.tvc.ru/video/json/id/%s' % (video_id)
+
+        video_json = self._download_json(video_json_url, video_id)
+
+        formats = []
+        for info in video_json.get('path', {}).get('quality', []):
+            format_id = self._search_regex(
+                r'cdnvideo/([^-]+)-[^/]+/', info.get('url'), 'format id',
+                fatal=False)
+            formats.append({
+                'format_id': str_or_none(format_id),
+                'url': info.get('url'),
+                'width': int_or_none(info.get('width')),
+                'height': int_or_none(info.get('height')),
+                'tbr': int_or_none(info.get('bitrate')),
+            })
+
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            'title': self._og_search_title(webpage),
+            'description': self._og_search_description(webpage),
+            'thumbnail': self._og_search_thumbnail(webpage),
+            'duration': int_or_none(video_json.get('duration')),
+            'formats': formats,
+        }

From 9f15bdabc85add582d78a6dd57cfbb56cb33baff Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 12 Jun 2015 16:13:36 +0600
Subject: [PATCH 1557/6219] [tvc] Separate embed extractor

---
 youtube_dl/extractor/__init__.py |   5 +-
 youtube_dl/extractor/tvc.py      | 125 ++++++++++++++++++-------------
 2 files changed, 77 insertions(+), 53 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 6dc3cbff4..a8d3a8928 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -584,7 +584,10 @@ from .tv2 import (
     TV2ArticleIE,
 )
 from .tv4 import TV4IE
-from .tvc import TVCIE
+from .tvc import (
+    TVCIE,
+    TVCEmbedIE,
+)
 from .tvigle import TvigleIE
 from .tvp import TvpIE, TvpSeriesIE
 from .tvplay import TVPlayIE
diff --git a/youtube_dl/extractor/tvc.py b/youtube_dl/extractor/tvc.py
index b62ab857c..0055f9598 100644
--- a/youtube_dl/extractor/tvc.py
+++ b/youtube_dl/extractor/tvc.py
@@ -3,77 +3,98 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..utils import (
+    clean_html,
     int_or_none,
-    str_or_none,
 )
 
 
-class TVCIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?tvc\.ru/.*/show/.*id/(?P<id>\d+)'
-    _TESTS = [
-        {
-            'url': 'http://www.tvc.ru/channel/brand/id/29/show/episodes/episode_id/39702/',
-            'md5': 'aa6fb3cf384e18a0ad3b30ee2898beba',
-            'info_dict': {
-                'id': '74622',
-                'display_id': '39702',
-                'ext': 'mp4',
-                'title': 'События. "События". Эфир от 22.05.2015 14:30',
-                'description': 'md5:ad7aa7db22903f983e687b8a3e98c6dd',
-                'thumbnail': 're:^https?://.*\.jpg$',
-                'duration': 1122,
-            },
+class TVCEmbedIE(InfoExtractor):
+    _VALID_URL = r'http://(?:www\.)?tvc\.ru/video/iframe/id/(?P<id>\d+)'
+    _TEST = {
+        'url': 'http://www.tvc.ru/video/iframe/id/74622/isPlay/false/id_stat/channel/?acc_video_id=/channel/brand/id/17/show/episodes/episode_id/39702',
+        'md5': 'bbc5ff531d1e90e856f60fc4b3afd708',
+        'info_dict': {
+            'id': '74622',
+            'ext': 'mp4',
+            'title': 'События. "События". Эфир от 22.05.2015 14:30',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 1122,
         },
-        {
-            'url': 'http://www.tvc.ru/news/show/id/69944',
-            'md5': 'b173128ee7b88b5b06c84e5f7880909f',
-            'info_dict': {
-                'id': '75399',
-                'display_id': '69944',
-                'ext': 'mp4',
-                'title': 'Эксперты: в столице встал вопрос о максимально безопасных остановках',
-                'description': 'md5:f675c8eaf23aab9df542d31773ed6518',
-                'thumbnail': 're:^https?://.*\.jpg$',
-                'duration': 278,
-            },
-        },
-    ]
+    }
 
     def _real_extract(self, url):
-        display_id = self._match_id(url)
+        video_id = self._match_id(url)
 
-        webpage = self._download_webpage(url, display_id)
-
-        video_url = self._og_search_video_url(webpage)
-
-        video_id = self._search_regex(
-            r'video/iframe/id/(\d+)/', video_url, 'video id')
-
-        video_json_url = 'http://www.tvc.ru/video/json/id/%s' % (video_id)
-
-        video_json = self._download_json(video_json_url, video_id)
+        video = self._download_json(
+            'http://www.tvc.ru/video/json/id/%s' % video_id, video_id)
 
         formats = []
-        for info in video_json.get('path', {}).get('quality', []):
+        for info in video.get('path', {}).get('quality', []):
+            video_url = info.get('url')
+            if not video_url:
+                continue
             format_id = self._search_regex(
-                r'cdnvideo/([^-]+)-[^/]+/', info.get('url'), 'format id',
-                fatal=False)
+                r'cdnvideo/([^/]+?)(?:-[^/]+?)?/', video_url,
+                'format id', default=None)
             formats.append({
-                'format_id': str_or_none(format_id),
-                'url': info.get('url'),
+                'url': video_url,
+                'format_id': format_id,
                 'width': int_or_none(info.get('width')),
                 'height': int_or_none(info.get('height')),
                 'tbr': int_or_none(info.get('bitrate')),
             })
-
         self._sort_formats(formats)
 
         return {
             'id': video_id,
-            'display_id': display_id,
-            'title': self._og_search_title(webpage),
-            'description': self._og_search_description(webpage),
-            'thumbnail': self._og_search_thumbnail(webpage),
-            'duration': int_or_none(video_json.get('duration')),
+            'title': video['title'],
+            'thumbnail': video.get('picture'),
+            'duration': int_or_none(video.get('duration')),
             'formats': formats,
         }
+
+
+class TVCIE(InfoExtractor):
+    _VALID_URL = r'http://(?:www\.)?tvc\.ru/(?!video/iframe/id/)(?P<id>[^?#]+)'
+    _TESTS = [{
+        'url': 'http://www.tvc.ru/channel/brand/id/29/show/episodes/episode_id/39702/',
+        'info_dict': {
+            'id': '74622',
+            'ext': 'mp4',
+            'title': 'События. "События". Эфир от 22.05.2015 14:30',
+            'description': 'md5:ad7aa7db22903f983e687b8a3e98c6dd',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 1122,
+        },
+    }, {
+        'url': 'http://www.tvc.ru/news/show/id/69944',
+        'info_dict': {
+            'id': '75399',
+            'ext': 'mp4',
+            'title': 'Эксперты: в столице встал вопрос о максимально безопасных остановках',
+            'description': 'md5:f2098f71e21f309e89f69b525fd9846e',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 278,
+        },
+    }, {
+        'url': 'http://www.tvc.ru/channel/brand/id/47/show/episodes#',
+        'info_dict': {
+            'id': '2185',
+            'ext': 'mp4',
+            'title': 'Ещё не поздно. Эфир от 03.08.2013',
+            'description': 'md5:51fae9f3f8cfe67abce014e428e5b027',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 3316,
+        },
+    }]
+
+    def _real_extract(self, url):
+        webpage = self._download_webpage(url, self._match_id(url))
+        return {
+            '_type': 'url_transparent',
+            'ie_key': 'TVCEmbed',
+            'url': self._og_search_video_url(webpage),
+            'title': clean_html(self._og_search_title(webpage)),
+            'description': clean_html(self._og_search_description(webpage)),
+            'thumbnail': self._og_search_thumbnail(webpage),
+        }

From 29902c8ec016a7128557d47a7413e82d4e022f01 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 12 Jun 2015 16:22:23 +0600
Subject: [PATCH 1558/6219] [tvc:embed] Add embed extraction routine

---
 youtube_dl/extractor/tvc.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/youtube_dl/extractor/tvc.py b/youtube_dl/extractor/tvc.py
index 0055f9598..756fec732 100644
--- a/youtube_dl/extractor/tvc.py
+++ b/youtube_dl/extractor/tvc.py
@@ -1,6 +1,8 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..utils import (
     clean_html,
@@ -22,6 +24,13 @@ class TVCEmbedIE(InfoExtractor):
         },
     }
 
+    @classmethod
+    def _extract_url(cls, webpage):
+        mobj = re.search(
+            r'<iframe[^>]+?src=(["\'])(?P<url>(?:http://)?(?:www\.)?tvc\.ru/video/iframe/id/[^"]+)\1', webpage)
+        if mobj:
+            return mobj.group('url')
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
 

From 494f20cbdca8e76e3cb452bb0feabcb855d9b4a7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 12 Jun 2015 16:22:46 +0600
Subject: [PATCH 1559/6219] [extractor/generic] Add support for tvc embeds

---
 youtube_dl/extractor/generic.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 75526384f..c797c4b52 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -34,6 +34,7 @@ from .brightcove import BrightcoveIE
 from .nbc import NBCSportsVPlayerIE
 from .ooyala import OoyalaIE
 from .rutv import RUTVIE
+from .tvc import TVCEmbedIE
 from .sportbox import SportBoxEmbedIE
 from .smotri import SmotriIE
 from .condenast import CondeNastIE
@@ -1301,6 +1302,11 @@ class GenericIE(InfoExtractor):
         if rutv_url:
             return self.url_result(rutv_url, 'RUTV')
 
+        # Look for embedded TVC player
+        rutv_url = TVCEmbedIE._extract_url(webpage)
+        if rutv_url:
+            return self.url_result(rutv_url, 'TVCEmbed')
+
         # Look for embedded SportBox player
         sportbox_urls = SportBoxEmbedIE._extract_urls(webpage)
         if sportbox_urls:

From 954c1d05299ae7c6a51db46c1ac33ddf150266c6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 12 Jun 2015 16:24:13 +0600
Subject: [PATCH 1560/6219] [tvc] Refactor extractor names

---
 youtube_dl/extractor/__init__.py | 2 +-
 youtube_dl/extractor/generic.py  | 4 ++--
 youtube_dl/extractor/tvc.py      | 4 ++--
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index a8d3a8928..18b1c5e54 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -586,7 +586,7 @@ from .tv2 import (
 from .tv4 import TV4IE
 from .tvc import (
     TVCIE,
-    TVCEmbedIE,
+    TVCArticleIE,
 )
 from .tvigle import TvigleIE
 from .tvp import TvpIE, TvpSeriesIE
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index c797c4b52..507e4a571 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -34,7 +34,7 @@ from .brightcove import BrightcoveIE
 from .nbc import NBCSportsVPlayerIE
 from .ooyala import OoyalaIE
 from .rutv import RUTVIE
-from .tvc import TVCEmbedIE
+from .tvc import TVCIE
 from .sportbox import SportBoxEmbedIE
 from .smotri import SmotriIE
 from .condenast import CondeNastIE
@@ -1303,7 +1303,7 @@ class GenericIE(InfoExtractor):
             return self.url_result(rutv_url, 'RUTV')
 
         # Look for embedded TVC player
-        rutv_url = TVCEmbedIE._extract_url(webpage)
+        rutv_url = TVCIE._extract_url(webpage)
         if rutv_url:
             return self.url_result(rutv_url, 'TVCEmbed')
 
diff --git a/youtube_dl/extractor/tvc.py b/youtube_dl/extractor/tvc.py
index 756fec732..36c2a3196 100644
--- a/youtube_dl/extractor/tvc.py
+++ b/youtube_dl/extractor/tvc.py
@@ -10,7 +10,7 @@ from ..utils import (
 )
 
 
-class TVCEmbedIE(InfoExtractor):
+class TVCIE(InfoExtractor):
     _VALID_URL = r'http://(?:www\.)?tvc\.ru/video/iframe/id/(?P<id>\d+)'
     _TEST = {
         'url': 'http://www.tvc.ru/video/iframe/id/74622/isPlay/false/id_stat/channel/?acc_video_id=/channel/brand/id/17/show/episodes/episode_id/39702',
@@ -63,7 +63,7 @@ class TVCEmbedIE(InfoExtractor):
         }
 
 
-class TVCIE(InfoExtractor):
+class TVCArticleIE(InfoExtractor):
     _VALID_URL = r'http://(?:www\.)?tvc\.ru/(?!video/iframe/id/)(?P<id>[^?#]+)'
     _TESTS = [{
         'url': 'http://www.tvc.ru/channel/brand/id/29/show/episodes/episode_id/39702/',

From 5ccddb7ecfb1015038f2616dd7e0da78a4365c89 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 12 Jun 2015 16:25:26 +0600
Subject: [PATCH 1561/6219] [tvc] Fix ie_key

---
 youtube_dl/extractor/tvc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tvc.py b/youtube_dl/extractor/tvc.py
index 36c2a3196..6b5d80aee 100644
--- a/youtube_dl/extractor/tvc.py
+++ b/youtube_dl/extractor/tvc.py
@@ -101,7 +101,7 @@ class TVCArticleIE(InfoExtractor):
         webpage = self._download_webpage(url, self._match_id(url))
         return {
             '_type': 'url_transparent',
-            'ie_key': 'TVCEmbed',
+            'ie_key': 'TVC',
             'url': self._og_search_video_url(webpage),
             'title': clean_html(self._og_search_title(webpage)),
             'description': clean_html(self._og_search_description(webpage)),

From 2da09ff8b0de3c27a16d9096f5d28d03f44fcf70 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 12 Jun 2015 16:26:31 +0600
Subject: [PATCH 1562/6219] [extractor/generic] Fix tvc ie_key

---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 507e4a571..66aceefb8 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1305,7 +1305,7 @@ class GenericIE(InfoExtractor):
         # Look for embedded TVC player
         rutv_url = TVCIE._extract_url(webpage)
         if rutv_url:
-            return self.url_result(rutv_url, 'TVCEmbed')
+            return self.url_result(rutv_url, 'TVC')
 
         # Look for embedded SportBox player
         sportbox_urls = SportBoxEmbedIE._extract_urls(webpage)

From f37bdbe537134f1ece0819e2aa677b1fec0c1cc8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 12 Jun 2015 16:28:45 +0600
Subject: [PATCH 1563/6219] [extractor/generic] Add test for tvc embed

---
 youtube_dl/extractor/generic.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 66aceefb8..6be9e6329 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -292,6 +292,15 @@ class GenericIE(InfoExtractor):
                 'skip_download': True,
             },
         },
+        # TVC embed
+        {
+            'url': 'http://sch1298sz.mskobr.ru/dou_edu/karamel_ki/filial_galleries/video/iframe_src_http_tvc_ru_video_iframe_id_55304_isplay_false_acc_video_id_channel_brand_id_11_show_episodes_episode_id_32307_frameb/',
+            'info_dict': {
+                'id': '55304',
+                'ext': 'mp4',
+                'title': 'Дошкольное воспитание',
+            },
+        },
         # SportBox embed
         {
             'url': 'http://www.vestifinance.ru/articles/25753',

From 499a077761b1577857952dc3b541c9f61a8bcade Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 12 Jun 2015 17:48:42 +0600
Subject: [PATCH 1564/6219] [5tv] Improve

---
 youtube_dl/extractor/fivetv.py | 115 +++++++++++++++++++--------------
 1 file changed, 68 insertions(+), 47 deletions(-)

diff --git a/youtube_dl/extractor/fivetv.py b/youtube_dl/extractor/fivetv.py
index e47383b39..13fbc4da2 100644
--- a/youtube_dl/extractor/fivetv.py
+++ b/youtube_dl/extractor/fivetv.py
@@ -1,67 +1,88 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
-from ..utils import (
-    int_or_none,
-)
+from ..utils import int_or_none
 
 
 class FiveTVIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?5-tv\.ru/[^/]*/(?P<id>\d+)'
-    _TESTS = [
-        {
-            'url': 'http://5-tv.ru/news/96814/',
-            'md5': 'bbff554ad415ecf5416a2f48c22d9283',
-            'info_dict': {
-                'id': '96814',
-                'ext': 'mp4',
-                'title': 'Россияне выбрали имя для общенациональной платежной системы',
-                'description': 'md5:a8aa13e2b7ad36789e9f77a74b6de660',
-                'thumbnail': 're:^https?://.*\.jpg$',
-                'width': 480,
-                'height': 360,
-                'duration': 180,
-            },
+    _VALID_URL = r'''(?x)
+                    http://
+                        (?:www\.)?5-tv\.ru/
+                        (?:
+                            (?:[^/]+/)+(?P<id>\d+)|
+                            (?P<path>[^/?#]+)(?:[/?#])?
+                        )
+                    '''
+
+    _TESTS = [{
+        'url': 'http://5-tv.ru/news/96814/',
+        'md5': 'bbff554ad415ecf5416a2f48c22d9283',
+        'info_dict': {
+            'id': '96814',
+            'ext': 'mp4',
+            'title': 'Россияне выбрали имя для общенациональной платежной системы',
+            'description': 'md5:a8aa13e2b7ad36789e9f77a74b6de660',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 180,
         },
-        {
-            'url': 'http://5-tv.ru/video/1021729/',
-            'md5': '299c8b72960efc9990acd2c784dc2296',
-            'info_dict': {
-                'id': '1021729',
-                'ext': 'mp4',
-                'title': '3D принтер',
-                'description': 'md5:d76c736d29ef7ec5c0cf7d7c65ffcb41',
-                'thumbnail': 're:^https?://.*\.jpg$',
-                'width': 480,
-                'height': 360,
-                'duration': 180,
-            },
+    }, {
+        'url': 'http://5-tv.ru/video/1021729/',
+        'info_dict': {
+            'id': '1021729',
+            'ext': 'mp4',
+            'title': '3D принтер',
+            'description': 'md5:d76c736d29ef7ec5c0cf7d7c65ffcb41',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 180,
         },
-    ]
+    }, {
+        'url': 'http://www.5-tv.ru/glavnoe/#itemDetails',
+        'info_dict': {
+            'id': 'glavnoe',
+            'ext': 'mp4',
+            'title': 'Итоги недели с 8 по 14 июня 2015 года',
+            'thumbnail': 're:^https?://.*\.jpg$',
+        },
+    }, {
+        'url': 'http://www.5-tv.ru/glavnoe/broadcasts/508645/',
+        'only_matching': True,
+    }, {
+        'url': 'http://5-tv.ru/films/1507502/',
+        'only_matching': True,
+    }, {
+        'url': 'http://5-tv.ru/programs/broadcast/508713/',
+        'only_matching': True,
+    }, {
+        'url': 'http://5-tv.ru/angel/',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.5-tv.ru/schedule/?iframe=true&width=900&height=450',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id') or mobj.group('path')
 
         webpage = self._download_webpage(url, video_id)
 
-        video_link = self._search_regex(
-            r'(<a.*?class="videoplayer">)', webpage, 'video link')
+        video_url = self._search_regex(
+            r'<a[^>]+?href="([^"]+)"[^>]+?class="videoplayer"',
+            webpage, 'video url')
 
-        url = self._search_regex(r'href="([^"]+)"', video_link, 'video url')
-        width = int_or_none(self._search_regex(
-            r'width:(\d+)px', video_link, 'width', default=None, fatal=False))
-        height = int_or_none(self._search_regex(
-            r'height:(\d+)px', video_link, 'height', default=None, fatal=False))
+        title = self._og_search_title(webpage, default=None) or self._search_regex(
+            r'<title>([^<]+)</title>', webpage, 'title')
         duration = int_or_none(self._og_search_property(
-            'video:duration', webpage, 'duration'))
+            'video:duration', webpage, 'duration', default=None))
+
         return {
             'id': video_id,
-            'url': url,
-            'width': width,
-            'height': height,
-            'title': self._og_search_title(webpage),
-            'description': self._og_search_description(webpage),
-            'thumbnail': self._og_search_thumbnail(webpage),
+            'url': video_url,
+            'title': title,
+            'description': self._og_search_description(webpage, default=None),
+            'thumbnail': self._og_search_thumbnail(webpage, default=None),
             'duration': duration,
         }

From b859971873915df55668a59a18ccfd259c20800e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 12 Jun 2015 18:15:30 +0600
Subject: [PATCH 1565/6219] [extractor/generic] Rename tvc embed url variable

---
 youtube_dl/extractor/generic.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 6be9e6329..357d58cea 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1312,9 +1312,9 @@ class GenericIE(InfoExtractor):
             return self.url_result(rutv_url, 'RUTV')
 
         # Look for embedded TVC player
-        rutv_url = TVCIE._extract_url(webpage)
-        if rutv_url:
-            return self.url_result(rutv_url, 'TVC')
+        tvc_url = TVCIE._extract_url(webpage)
+        if tvc_url:
+            return self.url_result(tvc_url, 'TVC')
 
         # Look for embedded SportBox player
         sportbox_urls = SportBoxEmbedIE._extract_urls(webpage)

From 9872d3110c0d3027dac856e005299f3682ef23ed Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 12 Jun 2015 18:37:09 +0600
Subject: [PATCH 1566/6219] [extractor/generic] Add support for tvigle embeds

---
 youtube_dl/extractor/generic.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 357d58cea..40d869c53 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1321,6 +1321,12 @@ class GenericIE(InfoExtractor):
         if sportbox_urls:
             return _playlist_from_matches(sportbox_urls, ie='SportBoxEmbed')
 
+        # Look for embedded Tvigle player
+        mobj = re.search(
+            r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//cloud\.tvigle\.ru/video/.+?)\1', webpage)
+        if mobj is not None:
+            return self.url_result(mobj.group('url'), 'Tvigle')
+
         # Look for embedded TED player
         mobj = re.search(
             r'<iframe[^>]+?src=(["\'])(?P<url>https?://embed(?:-ssl)?\.ted\.com/.+?)\1', webpage)

From d22dec74ffa2a53a1c04770af37d39f384f3d56c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 12 Jun 2015 19:20:12 +0600
Subject: [PATCH 1567/6219] Add `--force-generic-extractor`

For some extractors that are hard to workout a good _VALID_URL we use very vague and unrestrictive ones,
e.g. just allowing anything after hostname and capturing part of URL as id.
If some of these extractors happen to have an video embed of some different hoster or platform
and this scenario was not handled in extractor itself we end up with inability to download this embed
until extractor is fixed to support embed of this kind.
Forcing downloader to use the generic extractor can be a neat temporary solution for this problem.
Example: FiveTV extractor with Tvigle embed - http://www.5-tv.ru/rabota/broadcasts/48/
---
 youtube_dl/YoutubeDL.py         | 6 ++++++
 youtube_dl/__init__.py          | 1 +
 youtube_dl/extractor/generic.py | 4 +++-
 youtube_dl/options.py           | 4 ++++
 4 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index b1f792d4e..4b801a917 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -139,6 +139,7 @@ class YoutubeDL(object):
     outtmpl:           Template for output names.
     restrictfilenames: Do not allow "&" and spaces in file names
     ignoreerrors:      Do not stop on download errors.
+    force_generic_extractor: Force downloader to use the generic extractor
     nooverwrites:      Prevent overwriting files.
     playliststart:     Playlist item to start at.
     playlistend:       Playlist item to end at.
@@ -282,6 +283,7 @@ class YoutubeDL(object):
         self._num_downloads = 0
         self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
         self._err_file = sys.stderr
+        self._force_generic_extractor_required = params.get('force_generic_extractor', False)
         self.params = params
         self.cache = Cache(self)
 
@@ -633,6 +635,10 @@ class YoutubeDL(object):
         extra_info is a dict containing the extra values to add to each result
         '''
 
+        if not ie_key and self._force_generic_extractor_required:
+            self._force_generic_extractor_required = False
+            ie_key = 'Generic'
+
         if ie_key:
             ies = [self.get_info_extractor(ie_key)]
         else:
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index ace17857c..215b616de 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -293,6 +293,7 @@ def _real_main(argv=None):
         'autonumber_size': opts.autonumber_size,
         'restrictfilenames': opts.restrictfilenames,
         'ignoreerrors': opts.ignoreerrors,
+        'force_generic_extractor': opts.force_generic_extractor,
         'ratelimit': opts.ratelimit,
         'nooverwrites': opts.nooverwrites,
         'retries': opts_retries,
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 40d869c53..3d672197c 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -977,7 +977,9 @@ class GenericIE(InfoExtractor):
                 'upload_date': upload_date,
             }
 
-        if not self._downloader.params.get('test', False) and not is_intentional:
+        if (not self._downloader.params.get('test', False) and
+                not is_intentional and
+                not self._downloader.params.get('force_generic_extractor', False)):
             self._downloader.report_warning('Falling back on generic information extractor.')
 
         if not full_response:
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 689fa7595..096ab6137 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -150,6 +150,10 @@ def parseOpts(overrideArguments=None):
         '--extractor-descriptions',
         action='store_true', dest='list_extractor_descriptions', default=False,
         help='Output descriptions of all supported extractors')
+    general.add_option(
+        '--force-generic-extractor',
+        action='store_true', dest='force_generic_extractor', default=False,
+        help='Force extraction to use the generic extractor')
     general.add_option(
         '--default-search',
         dest='default_search', metavar='PREFIX',

From 3d535e047162af021b3df6086f9a90d0cb0b6100 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 12 Jun 2015 19:31:52 +0600
Subject: [PATCH 1568/6219] [tvc] Fix embed regex

---
 youtube_dl/extractor/tvc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tvc.py b/youtube_dl/extractor/tvc.py
index 6b5d80aee..3a4f393fc 100644
--- a/youtube_dl/extractor/tvc.py
+++ b/youtube_dl/extractor/tvc.py
@@ -27,7 +27,7 @@ class TVCIE(InfoExtractor):
     @classmethod
     def _extract_url(cls, webpage):
         mobj = re.search(
-            r'<iframe[^>]+?src=(["\'])(?P<url>(?:http://)?(?:www\.)?tvc\.ru/video/iframe/id/[^"]+)\1', webpage)
+            r'<iframe[^>]+?src=(["\'])(?P<url>(?:http:)?//(?:www\.)?tvc\.ru/video/iframe/id/[^"]+)\1', webpage)
         if mobj:
             return mobj.group('url')
 

From 185dbc49749ca81fbb0a61a78c6dd35f2c32b15f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 12 Jun 2015 21:13:14 +0600
Subject: [PATCH 1569/6219] [prosiebensat1] Fix rtmp extraction (Closes #5962)

---
 youtube_dl/extractor/prosiebensat1.py | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/prosiebensat1.py b/youtube_dl/extractor/prosiebensat1.py
index 255d4abc1..6b13eb605 100644
--- a/youtube_dl/extractor/prosiebensat1.py
+++ b/youtube_dl/extractor/prosiebensat1.py
@@ -206,8 +206,8 @@ class ProSiebenSat1IE(InfoExtractor):
     def _extract_clip(self, url, webpage):
         clip_id = self._html_search_regex(self._CLIPID_REGEXES, webpage, 'clip id')
 
-        access_token = 'testclient'
-        client_name = 'kolibri-1.2.5'
+        access_token = 'prosieben'
+        client_name = 'kolibri-1.12.6'
         client_location = url
 
         videos_api_url = 'http://vas.sim-technik.de/vas/live/v2/videos?%s' % compat_urllib_parse.urlencode({
@@ -275,13 +275,17 @@ class ProSiebenSat1IE(InfoExtractor):
         for source in urls_sources:
             protocol = source['protocol']
             if protocol == 'rtmp' or protocol == 'rtmpe':
-                mobj = re.search(r'^(?P<url>rtmpe?://[^/]+/(?P<app>[^/]+))/(?P<playpath>.+)$', source['url'])
+                mobj = re.search(r'^(?P<url>rtmpe?://[^/]+)/(?P<path>.+)$', source['url'])
                 if not mobj:
                     continue
+                path = mobj.group('path')
+                mp4colon_index = path.rfind('mp4:')
+                app = path[:mp4colon_index]
+                play_path = path[mp4colon_index:]
                 formats.append({
-                    'url': mobj.group('url'),
-                    'app': mobj.group('app'),
-                    'play_path': mobj.group('playpath'),
+                    'url': '%s/%s' % (mobj.group('url'), app),
+                    'app': app,
+                    'play_path': play_path,
                     'player_url': 'http://livepassdl.conviva.com/hf/ver/2.79.0.17083/LivePassModuleMain.swf',
                     'page_url': 'http://www.prosieben.de',
                     'vbr': fix_bitrate(source['bitrate']),

From 8b6c896c4b60fe13b30227071aba2783222132a0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 12 Jun 2015 21:18:13 +0600
Subject: [PATCH 1570/6219] [prosiebensat1] Add title regex

---
 youtube_dl/extractor/prosiebensat1.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/prosiebensat1.py b/youtube_dl/extractor/prosiebensat1.py
index 6b13eb605..536a42dc8 100644
--- a/youtube_dl/extractor/prosiebensat1.py
+++ b/youtube_dl/extractor/prosiebensat1.py
@@ -177,6 +177,7 @@ class ProSiebenSat1IE(InfoExtractor):
         r'<header class="clearfix">\s*<h3>(.+?)</h3>',
         r'<!-- start video -->\s*<h1>(.+?)</h1>',
         r'<h1 class="att-name">\s*(.+?)</h1>',
+        r'<header class="module_header">\s*<h2>([^<]+)</h2>\s*</header>',
     ]
     _DESCRIPTION_REGEXES = [
         r'<p itemprop="description">\s*(.+?)</p>',

From 9f4323252abade4f10b0884682f92cedc78b4d4a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 12 Jun 2015 21:56:50 +0600
Subject: [PATCH 1571/6219] [YoutubeDL] Fix for multiple URLs

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 4b801a917..8dbad7cf8 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -283,7 +283,6 @@ class YoutubeDL(object):
         self._num_downloads = 0
         self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
         self._err_file = sys.stderr
-        self._force_generic_extractor_required = params.get('force_generic_extractor', False)
         self.params = params
         self.cache = Cache(self)
 
@@ -1504,6 +1503,7 @@ class YoutubeDL(object):
 
         for url in url_list:
             try:
+                self._force_generic_extractor_required = self.params.get('force_generic_extractor', False)
                 # It also downloads the videos
                 res = self.extract_info(url)
             except UnavailableVideoError:

From 61aa5ba36eea3b7cf8c3570ab33604dd2c13b855 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 13 Jun 2015 02:05:21 +0600
Subject: [PATCH 1572/6219] [YoutubeDL] Remove global state for
 force_generic_extractor flag in favor of passing argument

---
 youtube_dl/YoutubeDL.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 8dbad7cf8..dd2d8cb3c 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -627,15 +627,14 @@ class YoutubeDL(object):
             info_dict.setdefault(key, value)
 
     def extract_info(self, url, download=True, ie_key=None, extra_info={},
-                     process=True):
+                     process=True, force_generic_extractor=False):
         '''
         Returns a list with a dictionary for each video we find.
         If 'download', also downloads the videos.
         extra_info is a dict containing the extra values to add to each result
         '''
 
-        if not ie_key and self._force_generic_extractor_required:
-            self._force_generic_extractor_required = False
+        if not ie_key and force_generic_extractor:
             ie_key = 'Generic'
 
         if ie_key:
@@ -663,7 +662,7 @@ class YoutubeDL(object):
                     }
                 self.add_default_extra_info(ie_result, ie, url)
                 if process:
-                    return self.process_ie_result(ie_result, download, extra_info)
+                    return self.process_ie_result(ie_result, download, extra_info, force_generic_extractor=False)
                 else:
                     return ie_result
             except ExtractorError as de:  # An error we somewhat expected
@@ -688,7 +687,7 @@ class YoutubeDL(object):
             'extractor_key': ie.ie_key(),
         })
 
-    def process_ie_result(self, ie_result, download=True, extra_info={}):
+    def process_ie_result(self, ie_result, download=True, extra_info={}, force_generic_extractor=False):
         """
         Take the result of the ie(may be modified) and resolve all unresolved
         references (URLs, playlist items).
@@ -716,7 +715,8 @@ class YoutubeDL(object):
             return self.extract_info(ie_result['url'],
                                      download,
                                      ie_key=ie_result.get('ie_key'),
-                                     extra_info=extra_info)
+                                     extra_info=extra_info,
+                                     force_generic_extractor=force_generic_extractor)
         elif result_type == 'url_transparent':
             # Use the information from the embedding page
             info = self.extract_info(
@@ -1503,9 +1503,9 @@ class YoutubeDL(object):
 
         for url in url_list:
             try:
-                self._force_generic_extractor_required = self.params.get('force_generic_extractor', False)
                 # It also downloads the videos
-                res = self.extract_info(url)
+                res = self.extract_info(
+                    url, force_generic_extractor=self.params.get('force_generic_extractor', False))
             except UnavailableVideoError:
                 self.report_error('unable to download video')
             except MaxDownloadsReached:

From 0072afca8e02052c77dc3b7009e51114887e31b4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 13 Jun 2015 02:21:29 +0600
Subject: [PATCH 1573/6219] [YoutubeDL] Remove force_generic_extractor arg from
 process_ie_result

---
 youtube_dl/YoutubeDL.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index dd2d8cb3c..a7d3a1c01 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -662,7 +662,7 @@ class YoutubeDL(object):
                     }
                 self.add_default_extra_info(ie_result, ie, url)
                 if process:
-                    return self.process_ie_result(ie_result, download, extra_info, force_generic_extractor=False)
+                    return self.process_ie_result(ie_result, download, extra_info)
                 else:
                     return ie_result
             except ExtractorError as de:  # An error we somewhat expected
@@ -687,7 +687,7 @@ class YoutubeDL(object):
             'extractor_key': ie.ie_key(),
         })
 
-    def process_ie_result(self, ie_result, download=True, extra_info={}, force_generic_extractor=False):
+    def process_ie_result(self, ie_result, download=True, extra_info={}):
         """
         Take the result of the ie(may be modified) and resolve all unresolved
         references (URLs, playlist items).
@@ -715,8 +715,7 @@ class YoutubeDL(object):
             return self.extract_info(ie_result['url'],
                                      download,
                                      ie_key=ie_result.get('ie_key'),
-                                     extra_info=extra_info,
-                                     force_generic_extractor=force_generic_extractor)
+                                     extra_info=extra_info)
         elif result_type == 'url_transparent':
             # Use the information from the embedding page
             info = self.extract_info(

From 4f3bf679f5a764f7a26d3c45c82be43e34a3cc4d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 13 Jun 2015 03:09:35 +0600
Subject: [PATCH 1574/6219] [vk] Fix authentication for non-ASCII
 login/password

---
 youtube_dl/extractor/vk.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index cc384adbf..d0e772108 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -119,8 +119,8 @@ class VKIE(InfoExtractor):
             'act': 'login',
             'role': 'al_frame',
             'expire': '1',
-            'email': username,
-            'pass': password,
+            'email': username.encode('cp1251'),
+            'pass': password.encode('cp1251'),
         }
 
         request = compat_urllib_request.Request('https://login.vk.com/?act=login',

From 9fcbd5db2abb5a56beefe4a64486da692705ad12 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 13 Jun 2015 03:24:36 +0600
Subject: [PATCH 1575/6219] [pornhub] Add support for embeds

---
 youtube_dl/extractor/pornhub.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index daa284ea2..3c99b4def 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -19,7 +19,7 @@ from ..aes import (
 
 
 class PornHubIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?pornhub\.com/view_video\.php\?viewkey=(?P<id>[0-9a-f]+)'
+    _VALID_URL = r'https?://(?:www\.)?pornhub\.com/(?:view_video\.php\?viewkey=|embed/)(?P<id>[0-9a-f]+)'
     _TEST = {
         'url': 'http://www.pornhub.com/view_video.php?viewkey=648719015',
         'md5': '882f488fa1f0026f023f33576004a2ed',
@@ -39,7 +39,8 @@ class PornHubIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        req = compat_urllib_request.Request(url)
+        req = compat_urllib_request.Request(
+            'http://www.pornhub.com/view_video.php?viewkey=%s' % video_id)
         req.add_header('Cookie', 'age_verified=1')
         webpage = self._download_webpage(req, video_id)
 

From 65d161c480e9964026e618a2e95f9fc9eb8119e7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 13 Jun 2015 03:36:16 +0600
Subject: [PATCH 1576/6219] [extractor/generic] Add support for pornhub embeds

---
 youtube_dl/extractor/generic.py | 5 +++++
 youtube_dl/extractor/pornhub.py | 7 +++++++
 2 files changed, 12 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 40d869c53..f683760e4 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -42,6 +42,7 @@ from .udn import UDNEmbedIE
 from .senateisvp import SenateISVPIE
 from .bliptv import BlipTVIE
 from .svt import SVTIE
+from .pornhub import PornHubIE
 
 
 class GenericIE(InfoExtractor):
@@ -1321,6 +1322,10 @@ class GenericIE(InfoExtractor):
         if sportbox_urls:
             return _playlist_from_matches(sportbox_urls, ie='SportBoxEmbed')
 
+        pornhub_url = PornHubIE._extract_url(webpage)
+        if pornhub_url:
+            return self.url_result(pornhub_url, 'PornHub')
+
         # Look for embedded Tvigle player
         mobj = re.search(
             r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//cloud\.tvigle\.ru/video/.+?)\1', webpage)
diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index 3c99b4def..8565d7551 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -32,6 +32,13 @@ class PornHubIE(InfoExtractor):
         }
     }
 
+    @classmethod
+    def _extract_url(cls, webpage):
+        mobj = re.search(
+            r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?pornhub\.com/embed/\d+)\1', webpage)
+        if mobj:
+            return mobj.group('url')
+
     def _extract_count(self, pattern, webpage, name):
         return str_to_int(self._search_regex(
             pattern, webpage, '%s count' % name, fatal=False))

From 78e2b74bb945dc7f1724f7486405dd523486d634 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 13 Jun 2015 03:39:14 +0600
Subject: [PATCH 1577/6219] [tumblr] Add support for pornhub embeds (Closes
 #5963)

---
 youtube_dl/extractor/generic.py | 1 +
 youtube_dl/extractor/tumblr.py  | 5 +++++
 2 files changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index f683760e4..f6b984300 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1322,6 +1322,7 @@ class GenericIE(InfoExtractor):
         if sportbox_urls:
             return _playlist_from_matches(sportbox_urls, ie='SportBoxEmbed')
 
+        # Look for embedded PornHub player
         pornhub_url = PornHubIE._extract_url(webpage)
         if pornhub_url:
             return self.url_result(pornhub_url, 'PornHub')
diff --git a/youtube_dl/extractor/tumblr.py b/youtube_dl/extractor/tumblr.py
index e6218808f..63c20310d 100644
--- a/youtube_dl/extractor/tumblr.py
+++ b/youtube_dl/extractor/tumblr.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from .pornhub import PornHubIE
 
 
 class TumblrIE(InfoExtractor):
@@ -55,6 +56,10 @@ class TumblrIE(InfoExtractor):
         if vid_me_embed_url is not None:
             return self.url_result(vid_me_embed_url, 'Vidme')
 
+        pornhub_url = PornHubIE._extract_url(webpage)
+        if pornhub_url:
+            return self.url_result(pornhub_url, 'PornHub')
+
         iframe_url = self._search_regex(
             r'src=\'(https?://www\.tumblr\.com/video/[^\']+)\'',
             webpage, 'iframe url')

From b4e1576aee7cf18f5842714c87985ae0b72f1546 Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Sat, 13 Jun 2015 06:09:44 -0500
Subject: [PATCH 1578/6219] Brightcove extractor: support
 customBC.createVideo(...); method found in
 http://www.americanbar.org/groups/family_law.html and
 http://america.aljazeera.com/watch/shows/america-tonight/2015/6/exclusive-hunting-isil-with-the-pkk.html

---
 youtube_dl/extractor/brightcove.py | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index c1d4320e1..20a6ed965 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -188,7 +188,19 @@ class BrightcoveIE(InfoExtractor):
                 [^>]*?>\s*<param\s+name="movie"\s+value="https?://[^/]*brightcove\.com/
             ).+?>\s*</object>''',
             webpage)
-        return list(filter(None, [cls._build_brighcove_url(m) for m in matches]))
+        if matches:
+            return list(filter(None, [cls._build_brighcove_url(m) for m in matches]))
+
+        custombcs = re.findall(r'customBC.\createVideo\((.+?)\);',webpage)
+        if custombcs:
+            urls = []
+            for match in custombcs:
+                # brightcove playerkey begins with AQ and is 50 characters in length,
+                # however it's appended to itself in places, so truncate.
+                f = re.search(r'["\'](AQ[^"\']{48}).*?["\'](\d+)["\']', match)
+                if f:
+                    urls.append('brightcove:playerKey='+f.group(1)+'&%40videoPlayer='+f.group(2))
+            return urls
 
     def _real_extract(self, url):
         url, smuggled_data = unsmuggle_url(url, {})

From af9cdee9cba610aa3924f90a8a3fcd7dd43c65eb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 13 Jun 2015 19:53:32 +0600
Subject: [PATCH 1579/6219] [brightcove] Improve and generalize brightcove URL
 extraction from JS

---
 youtube_dl/extractor/brightcove.py | 35 +++++++++++++++++++++---------
 1 file changed, 25 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 20a6ed965..d768f99e6 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -156,6 +156,28 @@ class BrightcoveIE(InfoExtractor):
         linkBase = find_param('linkBaseURL')
         if linkBase is not None:
             params['linkBaseURL'] = linkBase
+        return cls._make_brightcove_url(params)
+
+    @classmethod
+    def _build_brighcove_url_from_js(cls, object_js):
+        # The layout of JS is as follows:
+        # customBC.createVideo = function (width, height, playerID, playerKey, videoPlayer, VideoRandomID) {
+        #   // build Brightcove <object /> XML
+        # }
+        m = re.search(
+            r'''(?x)customBC.\createVideo\(
+                .*?                                                  # skipping width and height
+                ["\'](?P<playerID>\d+)["\']\s*,\s*                   # playerID
+                ["\'](?P<playerKey>AQ[^"\']{48})[^"\']*["\']\s*,\s*  # playerKey begins with AQ and is 50 characters
+                                                                     # in length, however it's appended to itself
+                                                                     # in places, so truncate
+                ["\'](?P<videoID>\d+)["\']                           # @videoPlayer
+            ''', object_js)
+        if m:
+            return cls._make_brightcove_url(m.groupdict())
+
+    @classmethod
+    def _make_brightcove_url(cls, params):
         data = compat_urllib_parse.urlencode(params)
         return cls._FEDERATED_URL_TEMPLATE % data
 
@@ -191,16 +213,9 @@ class BrightcoveIE(InfoExtractor):
         if matches:
             return list(filter(None, [cls._build_brighcove_url(m) for m in matches]))
 
-        custombcs = re.findall(r'customBC.\createVideo\((.+?)\);',webpage)
-        if custombcs:
-            urls = []
-            for match in custombcs:
-                # brightcove playerkey begins with AQ and is 50 characters in length,
-                # however it's appended to itself in places, so truncate.
-                f = re.search(r'["\'](AQ[^"\']{48}).*?["\'](\d+)["\']', match)
-                if f:
-                    urls.append('brightcove:playerKey='+f.group(1)+'&%40videoPlayer='+f.group(2))
-            return urls
+        return list(filter(None, [
+            cls._build_brighcove_url_from_js(custom_bc)
+            for custom_bc in re.findall(r'(customBC\.createVideo\(.+?\);)', webpage)]))
 
     def _real_extract(self, url):
         url, smuggled_data = unsmuggle_url(url, {})

From 0029071adbdc0e1469d76cdc7e058c2f77299610 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 14 Jun 2015 07:43:14 +0600
Subject: [PATCH 1580/6219] [dramefever] Improve and simplify

---
 youtube_dl/extractor/dramafever.py | 172 +++++++++++++++++------------
 1 file changed, 101 insertions(+), 71 deletions(-)

diff --git a/youtube_dl/extractor/dramafever.py b/youtube_dl/extractor/dramafever.py
index 40787ffcd..0f33a61a3 100644
--- a/youtube_dl/extractor/dramafever.py
+++ b/youtube_dl/extractor/dramafever.py
@@ -1,104 +1,111 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
-import re
+import itertools
 
 from .common import InfoExtractor
+from ..compat import (
+    compat_HTTPError,
+    compat_urlparse,
+)
+from ..utils import (
+    ExtractorError,
+    clean_html,
+    determine_ext,
+    int_or_none,
+    parse_iso8601,
+)
 
 
 class DramaFeverIE(InfoExtractor):
     IE_NAME = 'dramafever'
-    _VALID_URL = r'^https?://(?:www\.)?dramafever\.com/drama/(?P<id>[0-9]+/[0-9]+)/'
-    _TESTS = [{
+    _VALID_URL = r'https?://(?:www\.)?dramafever\.com/drama/(?P<id>[0-9]+/[0-9]+)'
+    _TEST = {
         'url': 'http://www.dramafever.com/drama/4512/1/Cooking_with_Shin/',
         'info_dict': {
             'id': '4512.1',
             'ext': 'flv',
             'title': 'Cooking with Shin 4512.1',
+            'description': 'md5:a8eec7942e1664a6896fcd5e1287bfd0',
+            'thumbnail': 're:^https?://.*\.jpg',
+            'timestamp': 1404336058,
             'upload_date': '20140702',
-            'description': 'Served at all special occasions and featured in the hit drama Heirs, Shin cooks Red Bean Rice.',
+            'duration': 343,
         }
-    }]
+    }
 
     def _real_extract(self, url):
-        video_id = self._match_id(url).replace("/", ".")
+        video_id = self._match_id(url).replace('/', '.')
 
-        consumer_secret = self._get_consumer_secret(video_id)
+        try:
+            feed = self._download_json(
+                'http://www.dramafever.com/amp/episode/feed.json?guid=%s' % video_id,
+                video_id, 'Downloading episode JSON')['channel']['item']
+        except ExtractorError as e:
+            if isinstance(e.cause, compat_HTTPError):
+                raise ExtractorError(
+                    'Currently unavailable in your country.', expected=True)
+            raise
 
-        ep_json = self._download_json(
-            "http://www.dramafever.com/amp/episode/feed.json?guid=%s" % video_id,
-            video_id, note='Downloading episode metadata',
-            errnote="Video may not be available for your location")["channel"]["item"]
-
-        title = ep_json["media-group"]["media-title"]
-        description = ep_json["media-group"]["media-description"]
-        thumbnail = ep_json["media-group"]["media-thumbnail"]["@attributes"]["url"]
-        duration = int(ep_json["media-group"]["media-content"][0]["@attributes"]["duration"])
-        mobj = re.match(r"([0-9]{4})-([0-9]{2})-([0-9]{2})", ep_json["pubDate"])
-        upload_date = mobj.group(1) + mobj.group(2) + mobj.group(3) if mobj is not None else None
+        media_group = feed.get('media-group', {})
 
         formats = []
-        for vid_format in ep_json["media-group"]["media-content"]:
-            src = vid_format["@attributes"]["url"]
-            if '.f4m' in src:
-                formats.extend(self._extract_f4m_formats(src, video_id))
-
+        for media_content in media_group['media-content']:
+            src = media_content.get('@attributes', {}).get('url')
+            if not src:
+                continue
+            ext = determine_ext(src)
+            if ext == 'f4m':
+                formats.extend(self._extract_f4m_formats(
+                    src, video_id, f4m_id='hds'))
+            elif ext == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    src, video_id, 'mp4', m3u8_id='hls'))
+            else:
+                formats.append({
+                    'url': src,
+                })
         self._sort_formats(formats)
-        video_subtitles = self.extract_subtitles(video_id, consumer_secret)
+
+        title = media_group.get('media-title')
+        description = media_group.get('media-description')
+        duration = int_or_none(media_group['media-content'][0].get('@attributes', {}).get('duration'))
+        thumbnail = self._proto_relative_url(
+            media_group.get('media-thumbnail', {}).get('@attributes', {}).get('url'))
+        timestamp = parse_iso8601(feed.get('pubDate'), ' ')
+
+        subtitles = {}
+        for media_subtitle in media_group.get('media-subTitle', []):
+            lang = media_subtitle.get('@attributes', {}).get('lang')
+            href = media_subtitle.get('@attributes', {}).get('href')
+            if not lang or not href:
+                continue
+            subtitles[lang] = [{
+                'ext': 'ttml',
+                'url': href,
+            }]
 
         return {
             'id': video_id,
             'title': title,
             'description': description,
             'thumbnail': thumbnail,
-            'upload_date': upload_date,
+            'timestamp': timestamp,
             'duration': duration,
             'formats': formats,
-            'subtitles': video_subtitles,
+            'subtitles': subtitles,
         }
 
-    def _get_consumer_secret(self, video_id):
-        df_js = self._download_webpage(
-            "http://www.dramafever.com/static/126960d/v2/js/plugins/jquery.threadedcomments.js", video_id)
-        return self._search_regex(r"'cs': '([0-9a-zA-Z]+)'", df_js, "cs")
 
-    def _get_episodes(self, series_id, consumer_secret, episode_filter=None):
-        _PAGE_SIZE = 60
-
-        curr_page = 1
-        max_pages = curr_page + 1
-        results = []
-        while max_pages >= curr_page:
-            page_url = "http://www.dramafever.com/api/4/episode/series/?cs=%s&series_id=%s&page_size=%d&page_number=%d" % \
-                       (consumer_secret, series_id, _PAGE_SIZE, curr_page)
-            series = self._download_json(
-                page_url, series_id, note="Downloading series json page #%d" % curr_page)
-            max_pages = series['num_pages']
-            results.extend([ep for ep in series['value'] if episode_filter is None or episode_filter(ep)])
-            curr_page += 1
-        return results
-
-    def _get_subtitles(self, video_id, consumer_secret):
-
-        res = None
-        info = self._get_episodes(
-            video_id.split(".")[0], consumer_secret,
-            episode_filter=lambda x: x['guid'] == video_id)
-
-        if len(info) == 1 and info[0]['subfile'] != '':
-            res = {'en': [{'url': info[0]['subfile'], 'ext': 'srt'}]}
-        return res
-
-
-class DramaFeverSeriesIE(DramaFeverIE):
+class DramaFeverSeriesIE(InfoExtractor):
     IE_NAME = 'dramafever:series'
-    _VALID_URL = r'^https?://(?:www\.)?dramafever\.com/drama/(?P<id>[0-9]+)/\d*[a-zA-Z_][a-zA-Z0-9_]*/'
+    _VALID_URL = r'https?://(?:www\.)?dramafever\.com/drama/(?P<id>[0-9]+)(?:/(?:(?!\d).+)?)?$'
     _TESTS = [{
         'url': 'http://www.dramafever.com/drama/4512/Cooking_with_Shin/',
         'info_dict': {
             'id': '4512',
             'title': 'Cooking with Shin',
-            'description': 'Professional chef and cooking instructor Shin Kim takes some of the delicious dishes featured in your favorite dramas and shows you how to make them right at home.',
+            'description': 'md5:84a3f26e3cdc3fb7f500211b3593b5c1',
         },
         'playlist_count': 4,
     }, {
@@ -106,25 +113,48 @@ class DramaFeverSeriesIE(DramaFeverIE):
         'info_dict': {
             'id': '124',
             'title': 'IRIS',
-            'description': 'Lee Byung Hun and Kim Tae Hee star in this powerhouse drama and ratings megahit of action, intrigue and romance.',
+            'description': 'md5:b3a30e587cf20c59bd1c01ec0ee1b862',
         },
         'playlist_count': 20,
     }]
 
+    _CONSUMER_SECRET = 'DA59dtVXYLxajktV'
+    _PAGE_SIZE = 5  # max is 60 (see http://api.drama9.com/#get--api-4-episode-series-)
+
+    def _get_consumer_secret(self, video_id):
+        mainjs = self._download_webpage(
+            'http://www.dramafever.com/static/51afe95/df2014/scripts/main.js',
+            video_id, 'Downloading main.js', fatal=False)
+        if not mainjs:
+            return self._CONSUMER_SECRET
+        return self._search_regex(
+            r"var\s+cs\s*=\s*'([^']+)'", mainjs,
+            'consumer secret', default=self._CONSUMER_SECRET)
+
     def _real_extract(self, url):
         series_id = self._match_id(url)
+
         consumer_secret = self._get_consumer_secret(series_id)
 
-        series_json = self._download_json(
-            "http://www.dramafever.com/api/4/series/query/?cs=%s&series_id=%s" % (consumer_secret, series_id),
-            series_id, note='Downloading series metadata')["series"][series_id]
+        series = self._download_json(
+            'http://www.dramafever.com/api/4/series/query/?cs=%s&series_id=%s'
+            % (consumer_secret, series_id),
+            series_id, 'Downloading series JSON')['series'][series_id]
 
-        title = series_json["name"]
-        description = series_json["description_short"]
+        title = clean_html(series['name'])
+        description = clean_html(series.get('description') or series.get('description_short'))
 
-        episodes = self._get_episodes(series_id, consumer_secret)
         entries = []
-        for ep in episodes:
-            entries.append(self.url_result(
-                'http://www.dramafever.com%s' % ep['episode_url'], 'DramaFever', ep['guid']))
+        for page_num in itertools.count(1):
+            episodes = self._download_json(
+                'http://www.dramafever.com/api/4/episode/series/?cs=%s&series_id=%s&page_size=%d&page_number=%d'
+                % (consumer_secret, series_id, self._PAGE_SIZE, page_num),
+                series_id, 'Downloading episodes JSON page #%d' % page_num)
+            for episode in episodes.get('value', []):
+                entries.append(self.url_result(
+                    compat_urlparse.urljoin(url, episode['episode_url']),
+                    'DramaFever', episode.get('guid')))
+            if page_num == episodes['num_pages']:
+                break
+
         return self.playlist_result(entries, series_id, title, description)

From 70a2002399b46aa0cde2879d856d1bb68e2c6f67 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 14 Jun 2015 09:50:23 +0600
Subject: [PATCH 1581/6219] [dramafever:series] Fix _VALID_URL (Closes #5973)

---
 youtube_dl/extractor/dramafever.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/dramafever.py b/youtube_dl/extractor/dramafever.py
index 0f33a61a3..42e0df24e 100644
--- a/youtube_dl/extractor/dramafever.py
+++ b/youtube_dl/extractor/dramafever.py
@@ -99,7 +99,7 @@ class DramaFeverIE(InfoExtractor):
 
 class DramaFeverSeriesIE(InfoExtractor):
     IE_NAME = 'dramafever:series'
-    _VALID_URL = r'https?://(?:www\.)?dramafever\.com/drama/(?P<id>[0-9]+)(?:/(?:(?!\d).+)?)?$'
+    _VALID_URL = r'https?://(?:www\.)?dramafever\.com/drama/(?P<id>[0-9]+)(?:/(?:(?!\d+(?:/|$)).+)?)?$'
     _TESTS = [{
         'url': 'http://www.dramafever.com/drama/4512/Cooking_with_Shin/',
         'info_dict': {

From 463b2e5542a85d5cd41b255a71833fec7b4f51e0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 14 Jun 2015 09:51:07 +0600
Subject: [PATCH 1582/6219] [dramafever:series] Rollback _PAGE_SIZE to max
 possible

---
 youtube_dl/extractor/dramafever.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/dramafever.py b/youtube_dl/extractor/dramafever.py
index 42e0df24e..c4b7c0b68 100644
--- a/youtube_dl/extractor/dramafever.py
+++ b/youtube_dl/extractor/dramafever.py
@@ -119,7 +119,7 @@ class DramaFeverSeriesIE(InfoExtractor):
     }]
 
     _CONSUMER_SECRET = 'DA59dtVXYLxajktV'
-    _PAGE_SIZE = 5  # max is 60 (see http://api.drama9.com/#get--api-4-episode-series-)
+    _PAGE_SIZE = 60  # max is 60 (see http://api.drama9.com/#get--api-4-episode-series-)
 
     def _get_consumer_secret(self, video_id):
         mainjs = self._download_webpage(

From 450d89ddc12d80a500a2429632d35a0045cf630b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 14 Jun 2015 09:58:26 +0600
Subject: [PATCH 1583/6219] [dramafever] Improve _VALID_URL

---
 youtube_dl/extractor/dramafever.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/dramafever.py b/youtube_dl/extractor/dramafever.py
index c4b7c0b68..a34aad486 100644
--- a/youtube_dl/extractor/dramafever.py
+++ b/youtube_dl/extractor/dramafever.py
@@ -19,7 +19,7 @@ from ..utils import (
 
 class DramaFeverIE(InfoExtractor):
     IE_NAME = 'dramafever'
-    _VALID_URL = r'https?://(?:www\.)?dramafever\.com/drama/(?P<id>[0-9]+/[0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?dramafever\.com/drama/(?P<id>[0-9]+/[0-9]+)(?:/|$)'
     _TEST = {
         'url': 'http://www.dramafever.com/drama/4512/1/Cooking_with_Shin/',
         'info_dict': {

From 976b03c56bddf20c978820474e307457523f4c05 Mon Sep 17 00:00:00 2001
From: chaoskagami <chaos.kagami@gmail.com>
Date: Sun, 14 Jun 2015 00:18:40 -0400
Subject: [PATCH 1584/6219] Quality note for niconico - at least notify whether
 you'll get low or src

---
 youtube_dl/extractor/niconico.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/niconico.py b/youtube_dl/extractor/niconico.py
index 3cecebf95..e10348004 100644
--- a/youtube_dl/extractor/niconico.py
+++ b/youtube_dl/extractor/niconico.py
@@ -184,6 +184,11 @@ class NiconicoIE(InfoExtractor):
             extension = determine_ext(video_real_url)
         video_format = extension.upper()
 
+        if video_real_url.endswith('low'):
+            format_note = 'low'
+        else:
+            format_note = 'src'
+
         thumbnail = (
             xpath_text(video_info, './/thumbnail_url') or
             self._html_search_meta('image', webpage, 'thumbnail', default=None) or
@@ -242,6 +247,7 @@ class NiconicoIE(InfoExtractor):
             'title': title,
             'ext': extension,
             'format': video_format,
+            'format_note' : format_note,
             'thumbnail': thumbnail,
             'description': description,
             'uploader': uploader,

From 180940e02df60129bce36035b4a2fd79c0c60995 Mon Sep 17 00:00:00 2001
From: Shrimadhav U K <SpEcHiDe@users.noreply.github.com>
Date: Sun, 14 Jun 2015 11:19:42 +0530
Subject: [PATCH 1585/6219] spelling mistake corrected

acces changed to accessing
---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index b1f792d4e..aacec2958 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -119,7 +119,7 @@ class YoutubeDL(object):
 
     username:          Username for authentication purposes.
     password:          Password for authentication purposes.
-    videopassword:     Password for acces a video.
+    videopassword:     Password for accessing a video.
     usenetrc:          Use netrc for authentication instead.
     verbose:           Print additional info to stdout.
     quiet:             Do not print messages to stdout.

From 755a9d3d1a8f99b061c8d29525d629b8ad6061a4 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Sun, 14 Jun 2015 20:58:15 +0300
Subject: [PATCH 1586/6219] [tvplay] Add support for NovaTv

---
 youtube_dl/extractor/tvplay.py | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/youtube_dl/extractor/tvplay.py b/youtube_dl/extractor/tvplay.py
index e83e31a31..79863e781 100644
--- a/youtube_dl/extractor/tvplay.py
+++ b/youtube_dl/extractor/tvplay.py
@@ -26,6 +26,7 @@ class TVPlayIE(InfoExtractor):
            viasat4play\.no/programmer|
            tv6play\.no/programmer|
            tv3play\.dk/programmer|
+           play\.novatv\.bg/programi
         )/[^/]+/(?P<id>\d+)
         '''
     _TESTS = [
@@ -173,6 +174,22 @@ class TVPlayIE(InfoExtractor):
                 'skip_download': True,
             },
         },
+        {
+            'url': 'http://play.novatv.bg/programi/zdravei-bulgariya/624952?autostart=true',
+            'info_dict': {
+                'id': '624952',
+                'ext': 'flv',
+                'title': 'Здравей, България (12.06.2015 г.) ',
+                'description': 'md5:99f3700451ac5bb71a260268b8daefd7',
+                'duration': 8838,
+                'timestamp': 1434100372,
+                'upload_date': '20150612',
+            },
+            'params': {
+                # rtmp download
+                'skip_download': True,
+            },
+        },
     ]
 
     def _real_extract(self, url):

From 9fd24e3a227a059eed07b679dac858e5bd747123 Mon Sep 17 00:00:00 2001
From: jomo <github@jomo.tv>
Date: Sun, 14 Jun 2015 21:50:03 +0200
Subject: [PATCH 1587/6219] LiveLeak: support more original videos

some (old?) videos use ...mp4.h264_270p.mp4... instead of ...mp4.h264_base.mp4...
This is an addition to #4768
---
 youtube_dl/extractor/liveleak.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/liveleak.py b/youtube_dl/extractor/liveleak.py
index 35822067f..431f2e85d 100644
--- a/youtube_dl/extractor/liveleak.py
+++ b/youtube_dl/extractor/liveleak.py
@@ -85,7 +85,7 @@ class LiveLeakIE(InfoExtractor):
             'url': s['file'],
         } for i, s in enumerate(sources)]
         for i, s in enumerate(sources):
-            orig_url = s['file'].replace('.h264_base.mp4', '')
+            orig_url = re.sub(r'.h264_.+\.mp4', '', s['file'])
             if s['file'] != orig_url:
                 formats.append({
                     'format_id': 'original-%s' % i,

From 8f75761f24f9f2599efe100b5a094182af6403d0 Mon Sep 17 00:00:00 2001
From: jomo <github@jomo.tv>
Date: Sun, 14 Jun 2015 22:41:44 +0200
Subject: [PATCH 1588/6219] LiveLak: add test for URLs with 'h264_270p'

---
 youtube_dl/extractor/liveleak.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/youtube_dl/extractor/liveleak.py b/youtube_dl/extractor/liveleak.py
index 431f2e85d..c658cc92b 100644
--- a/youtube_dl/extractor/liveleak.py
+++ b/youtube_dl/extractor/liveleak.py
@@ -40,6 +40,16 @@ class LiveLeakIE(InfoExtractor):
             'title': 'Man is Fatally Struck by Reckless Car While Packing up a Moving Truck',
             'age_limit': 18,
         }
+    }, {
+        'url': 'http://www.liveleak.com/view?i=801_1409392012',
+        'md5': '0b3bec2d888c20728ca2ad3642f0ef15',
+        'info_dict': {
+            'id': '801_1409392012',
+            'ext': 'mp4',
+            'description': "Happened on 27.7.2014. \r\nAt 0:53 you can see people still swimming at near beach.",
+            'uploader': 'bony333',
+            'title': 'Crazy Hungarian tourist films close call waterspout in Croatia'
+        }
     }]
 
     def _real_extract(self, url):

From 00ac23e6e06bf6de59d5d5b3f42ff64ce039fee2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 15 Jun 2015 02:51:21 +0600
Subject: [PATCH 1589/6219] [liveleak] Improve regex for restoring original
 video URL

---
 youtube_dl/extractor/liveleak.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/liveleak.py b/youtube_dl/extractor/liveleak.py
index c658cc92b..e82f21ea7 100644
--- a/youtube_dl/extractor/liveleak.py
+++ b/youtube_dl/extractor/liveleak.py
@@ -95,7 +95,7 @@ class LiveLeakIE(InfoExtractor):
             'url': s['file'],
         } for i, s in enumerate(sources)]
         for i, s in enumerate(sources):
-            orig_url = re.sub(r'.h264_.+\.mp4', '', s['file'])
+            orig_url = re.sub(r'\.h264_.+?\.mp4', '', s['file'])
             if s['file'] != orig_url:
                 formats.append({
                     'format_id': 'original-%s' % i,

From afa1ded425ffe12a5b90bcd4d316c43941a5dc1a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 15 Jun 2015 02:54:05 +0600
Subject: [PATCH 1590/6219] [liveleak] Clarify rationale for restoring raw
 video

---
 youtube_dl/extractor/liveleak.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/liveleak.py b/youtube_dl/extractor/liveleak.py
index e82f21ea7..0a4e473d6 100644
--- a/youtube_dl/extractor/liveleak.py
+++ b/youtube_dl/extractor/liveleak.py
@@ -95,6 +95,9 @@ class LiveLeakIE(InfoExtractor):
             'url': s['file'],
         } for i, s in enumerate(sources)]
         for i, s in enumerate(sources):
+            # Removing '.h264_*.mp4' gives the raw video, which is essentially
+            # the same video without the LiveLeak logo at the top (see
+            # https://github.com/rg3/youtube-dl/pull/4768)
             orig_url = re.sub(r'\.h264_.+?\.mp4', '', s['file'])
             if s['file'] != orig_url:
                 formats.append({

From b95cfa917025750805fb873fc4e2eb161241b22b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 15 Jun 2015 02:54:49 +0600
Subject: [PATCH 1591/6219] [liveleak] Clarify test

---
 youtube_dl/extractor/liveleak.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/liveleak.py b/youtube_dl/extractor/liveleak.py
index 0a4e473d6..857edfde2 100644
--- a/youtube_dl/extractor/liveleak.py
+++ b/youtube_dl/extractor/liveleak.py
@@ -41,6 +41,7 @@ class LiveLeakIE(InfoExtractor):
             'age_limit': 18,
         }
     }, {
+        # Covers https://github.com/rg3/youtube-dl/pull/5983
         'url': 'http://www.liveleak.com/view?i=801_1409392012',
         'md5': '0b3bec2d888c20728ca2ad3642f0ef15',
         'info_dict': {

From 5774ef35c4d167f7c959041bf4efc5581a98f0a4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 15 Jun 2015 02:57:07 +0600
Subject: [PATCH 1592/6219] [options] Add missing whitespace for --fixup
 description

---
 youtube_dl/options.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 689fa7595..740458e51 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -725,7 +725,7 @@ def parseOpts(overrideArguments=None):
         metavar='POLICY', dest='fixup', default='detect_or_warn',
         help='Automatically correct known faults of the file. '
              'One of never (do nothing), warn (only emit a warning), '
-             'detect_or_warn(the default; fix file if we can, warn otherwise)')
+             'detect_or_warn (the default; fix file if we can, warn otherwise)')
     postproc.add_option(
         '--prefer-avconv',
         action='store_false', dest='prefer_ffmpeg',

From 67d95f177c7ffedfc8f8b086535013a1a7a48b29 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 15 Jun 2015 03:43:33 +0600
Subject: [PATCH 1593/6219] [niconico] Simplify format info

---
 youtube_dl/extractor/niconico.py | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/niconico.py b/youtube_dl/extractor/niconico.py
index e10348004..0f8aa5ada 100644
--- a/youtube_dl/extractor/niconico.py
+++ b/youtube_dl/extractor/niconico.py
@@ -182,12 +182,6 @@ class NiconicoIE(InfoExtractor):
         extension = xpath_text(video_info, './/movie_type')
         if not extension:
             extension = determine_ext(video_real_url)
-        video_format = extension.upper()
-
-        if video_real_url.endswith('low'):
-            format_note = 'low'
-        else:
-            format_note = 'src'
 
         thumbnail = (
             xpath_text(video_info, './/thumbnail_url') or
@@ -246,8 +240,7 @@ class NiconicoIE(InfoExtractor):
             'url': video_real_url,
             'title': title,
             'ext': extension,
-            'format': video_format,
-            'format_note' : format_note,
+            'format_id': 'economy' if video_real_url.endswith('low') else 'normal',
             'thumbnail': thumbnail,
             'description': description,
             'uploader': uploader,

From 2a0fcf6113c3f0c8d0510167fd7017cc0fdfa622 Mon Sep 17 00:00:00 2001
From: zx8 <zx8@mac>
Date: Mon, 15 Jun 2015 00:27:43 +0100
Subject: [PATCH 1594/6219] [safari] make url regex more lenient

---
 youtube_dl/extractor/safari.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/safari.py b/youtube_dl/extractor/safari.py
index 10251f29e..20ba6fa33 100644
--- a/youtube_dl/extractor/safari.py
+++ b/youtube_dl/extractor/safari.py
@@ -83,7 +83,7 @@ class SafariIE(SafariBaseIE):
                                     library/view/[^/]+|
                                     api/v1/book
                                 )/
-                                (?P<course_id>\d+)/
+                                (?P<course_id>[^/]+)/
                                     (?:chapter(?:-content)?/)?
                                 (?P<part>part\d+)\.html
     '''
@@ -122,7 +122,7 @@ class SafariCourseIE(SafariBaseIE):
     IE_NAME = 'safari:course'
     IE_DESC = 'safaribooksonline.com online courses'
 
-    _VALID_URL = r'https?://(?:www\.)?safaribooksonline\.com/(?:library/view/[^/]+|api/v1/book)/(?P<id>\d+)/?(?:[#?]|$)'
+    _VALID_URL = r'https?://(?:www\.)?safaribooksonline\.com/(?:library/view/[^/]+|api/v1/book)/(?P<id>[^/]+)/?(?:[#?]|$)'
 
     _TESTS = [{
         'url': 'https://www.safaribooksonline.com/library/view/hadoop-fundamentals-livelessons/9780133392838/',

From 4b9f9010b0d744969189c383e98e8729f9fe9623 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 15 Jun 2015 01:35:50 +0200
Subject: [PATCH 1595/6219] release 2015.06.15

---
 README.md              | 4 ++--
 docs/supportedsites.md | 7 +++++++
 youtube_dl/version.py  | 2 +-
 3 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index f3d83c89f..5f3a08f5a 100644
--- a/README.md
+++ b/README.md
@@ -52,7 +52,7 @@ which means you can modify it, redistribute it or use it however you like.
     -i, --ignore-errors              Continue on download errors, for example to skip unavailable videos in a playlist
     --abort-on-error                 Abort downloading of further videos (in the playlist or the command line) if an error occurs
     --dump-user-agent                Display the current browser identification
-    --list-extractors                List all supported extractors and the URLs they would handle
+    --list-extractors                List all supported extractors
     --extractor-descriptions         Output descriptions of all supported extractors
     --default-search PREFIX          Use this prefix for unqualified URLs. For example "gvsearch2:" downloads two videos from google videos for youtube-dl "large apple".
                                      Use the value "auto" to let youtube-dl guess ("auto_warning" to emit a warning when guessing). "error" just throws an error. The
@@ -223,7 +223,7 @@ which means you can modify it, redistribute it or use it however you like.
                                      parameters replace existing values. Additional templates: %(album)s, %(artist)s. Example: --metadata-from-title "%(artist)s -
                                      %(title)s" matches a title like "Coldplay - Paradise"
     --xattrs                         Write metadata to the video file's xattrs (using dublin core and xdg standards)
-    --fixup POLICY                   Automatically correct known faults of the file. One of never (do nothing), warn (only emit a warning), detect_or_warn(the default;
+    --fixup POLICY                   Automatically correct known faults of the file. One of never (do nothing), warn (only emit a warning), detect_or_warn (the default;
                                      fix file if we can, warn otherwise)
     --prefer-avconv                  Prefer avconv over ffmpeg for running the postprocessors (default)
     --prefer-ffmpeg                  Prefer ffmpeg over avconv for running the postprocessors
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index d147b53fe..220e52b98 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -120,6 +120,8 @@
  - **divxstage**: DivxStage
  - **Dotsub**
  - **DouyuTV**
+ - **dramafever**
+ - **dramafever:series**
  - **DRBonanza**
  - **Dropbox**
  - **DrTuber**
@@ -153,6 +155,7 @@
  - **fernsehkritik.tv**
  - **fernsehkritik.tv:postecke**
  - **Firstpost**
+ - **FiveTV**
  - **Flickr**
  - **Folketinget**: Folketinget (ft.dk; Danish parliament)
  - **FootyRoom**
@@ -217,6 +220,7 @@
  - **instagram:user**: Instagram user profile
  - **InternetVideoArchive**
  - **IPrima**
+ - **iqiyi**
  - **ivi**: ivi.ru
  - **ivi:compilation**: ivi.ru compilations
  - **Izlesene**
@@ -407,6 +411,7 @@
  - **rutube:movie**: Rutube movies
  - **rutube:person**: Rutube person videos
  - **RUTV**: RUTV.RU
+ - **Ruutu**
  - **safari**: safaribooksonline.com online video
  - **safari:course**: safaribooksonline.com online courses
  - **Sandia**: Sandia National Laboratories
@@ -519,6 +524,8 @@
  - **TV2**
  - **TV2Article**
  - **TV4**: tv4.se and tv4play.se
+ - **TVC**
+ - **TVCArticle**
  - **tvigle**: Интернет-телевидение Tvigle.ru
  - **tvp.pl**
  - **tvp.pl:Series**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 9cf84ff71..34a13cb81 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.06.04.1'
+__version__ = '2015.06.15'

From 4af98ecdfb896c16e73cb9f7306908cc686782e8 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 15 Jun 2015 18:49:27 +0800
Subject: [PATCH 1596/6219] [vbox7] Fix extraction (fixes #5967)

---
 youtube_dl/extractor/vbox7.py | 21 ++++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/vbox7.py b/youtube_dl/extractor/vbox7.py
index dd026748d..722eb5236 100644
--- a/youtube_dl/extractor/vbox7.py
+++ b/youtube_dl/extractor/vbox7.py
@@ -5,6 +5,7 @@ from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse,
     compat_urllib_request,
+    compat_urlparse,
 )
 from ..utils import (
     ExtractorError,
@@ -26,11 +27,21 @@ class Vbox7IE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        redirect_page, urlh = self._download_webpage_handle(url, video_id)
-        new_location = self._search_regex(r'window\.location = \'(.*)\';',
-                                          redirect_page, 'redirect location')
-        redirect_url = urlh.geturl() + new_location
-        webpage = self._download_webpage(redirect_url, video_id,
+        # need to get the page 3 times for the correct jsSecretToken cookie
+        # which is necessary for the correct title
+        def get_session_id():
+            redirect_page = self._download_webpage(url, video_id)
+            session_id_url = self._search_regex(
+                r'var\s*url\s*=\s*\'([^\']+)\';', redirect_page,
+                'session id url')
+            self._download_webpage(
+                compat_urlparse.urljoin(url, session_id_url), video_id,
+                'Getting session id')
+
+        get_session_id()
+        get_session_id()
+
+        webpage = self._download_webpage(url, video_id,
                                          'Downloading redirect page')
 
         title = self._html_search_regex(r'<title>(.*)</title>',

From aed473ccf9d9da73b1b80ee8b06d00ee66a3769d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 15 Jun 2015 22:41:24 +0800
Subject: [PATCH 1597/6219] [youku] PEP8

---
 youtube_dl/extractor/youku.py | 80 +++++++++++++++++------------------
 1 file changed, 40 insertions(+), 40 deletions(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index aed6b960a..4e47fca8a 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -11,6 +11,7 @@ from ..compat import compat_urllib_parse
 
 bytes_is_str = (bytes == str)  # for compatible
 
+
 class YoukuIE(InfoExtractor):
     IE_NAME = 'youku'
     _VALID_URL = r'''(?x)
@@ -21,13 +22,13 @@ class YoukuIE(InfoExtractor):
     '''
 
     _TEST = {
-            'url': 'http://v.youku.com/v_show/id_XMTc1ODE5Njcy.html',
-            'md5': '5f3af4192eabacc4501508d54a8cabd7',
-            'info_dict': {
-                'id': 'XMTc1ODE5Njcy',
-                'title': '★Smile﹗♡ Git Fresh -Booty Music舞蹈.',
-                'ext': 'flv'
-            }
+        'url': 'http://v.youku.com/v_show/id_XMTc1ODE5Njcy.html',
+        'md5': '5f3af4192eabacc4501508d54a8cabd7',
+        'info_dict': {
+            'id': 'XMTc1ODE5Njcy',
+            'title': '★Smile﹗♡ Git Fresh -Booty Music舞蹈.',
+            'ext': 'flv'
+        }
     }
 
     def construct_video_urls(self, data1, data2):
@@ -36,7 +37,7 @@ class YoukuIE(InfoExtractor):
             ls = list(range(256))
             t = 0
             for i in range(256):
-                t = (t + ls[i] + ord(s1[i%len(s1)])) % 256
+                t = (t + ls[i] + ord(s1[i % len(s1)])) % 256
                 ls[i], ls[t] = ls[t], ls[i]
             s = '' if not bytes_is_str else b''
             x, y = 0, 0
@@ -45,16 +46,16 @@ class YoukuIE(InfoExtractor):
                 x = (x + ls[y]) % 256
                 ls[x], ls[y] = ls[y], ls[x]
                 if isinstance(s2[i], int):
-                    s += chr(s2[i] ^ ls[(ls[x]+ls[y]) % 256])
+                    s += chr(s2[i] ^ ls[(ls[x] + ls[y]) % 256])
                 else:
-                    s += chr(ord(s2[i]) ^ ls[(ls[x]+ls[y]) % 256])
+                    s += chr(ord(s2[i]) ^ ls[(ls[x] + ls[y]) % 256])
             return s
 
         sid, token = yk_t(
             'becaf9be',
-            base64.b64decode(bytes(data2['ep'], 'ascii')) \
-                if not bytes_is_str \
-                else base64.b64decode(data2['ep'])
+            base64.b64decode(bytes(data2['ep'], 'ascii'))
+            if not bytes_is_str
+            else base64.b64decode(data2['ep'])
         ).split('_')
 
         # get oip
@@ -89,13 +90,13 @@ class YoukuIE(InfoExtractor):
             fileid = get_fileid(format, n)
             ep_t = yk_t(
                 'bf7e5f01',
-                bytes('%s_%s_%s' % (sid, fileid, token), 'ascii') \
-                if not bytes_is_str \
+                bytes('%s_%s_%s' % (sid, fileid, token), 'ascii')
+                if not bytes_is_str
                 else ('%s_%s_%s' % (sid, fileid, token))
             )
             ep = base64.b64encode(
-                bytes(ep_t, 'latin') \
-                if not bytes_is_str \
+                bytes(ep_t, 'latin')
+                if not bytes_is_str
                 else ep_t
             ).decode()
             return ep
@@ -121,9 +122,9 @@ class YoukuIE(InfoExtractor):
                 video_url = \
                     'http://k.youku.com/player/getFlvPath/' + \
                     'sid/' + sid + \
-                    '_' + str(int(n)+1).zfill(2) + \
+                    '_' + str(int(n) + 1).zfill(2) + \
                     '/st/' + self.parse_ext_l(format) + \
-                    '/fileid/' + get_fileid(format, n)  + '?' + \
+                    '/fileid/' + get_fileid(format, n) + '?' + \
                     compat_urllib_parse.urlencode(param)
                 video_urls.append(video_url)
             video_urls_dict[format] = video_urls
@@ -132,34 +133,34 @@ class YoukuIE(InfoExtractor):
 
     def get_hd(self, fm):
         hd_id_dict = {
-            'flv'   : '0',
-            'mp4'   : '1',
-            'hd2'   : '2',
-            'hd3'   : '3',
-            '3gp'   : '0',
-            '3gphd' : '1'
+            'flv': '0',
+            'mp4': '1',
+            'hd2': '2',
+            'hd3': '3',
+            '3gp': '0',
+            '3gphd': '1'
         }
         return hd_id_dict[fm]
 
     def parse_ext_l(self, fm):
         ext_dict = {
-            'flv'   : 'flv',
-            'mp4'   : 'mp4',
-            'hd2'   : 'flv',
-            'hd3'   : 'flv',
-            '3gp'   : 'flv',
-            '3gphd' : 'mp4'
+            'flv': 'flv',
+            'mp4': 'mp4',
+            'hd2': 'flv',
+            'hd3': 'flv',
+            '3gp': 'flv',
+            '3gphd': 'mp4'
         }
         return ext_dict[fm]
 
     def get_format_name(self, fm):
         _dict = {
-            '3gp'   : 'h6',
-            '3gphd' : 'h5',
-            'flv'   : 'h4',
-            'mp4'   : 'h3',
-            'hd2'   : 'h2',
-            'hd3'   : 'h1'
+            '3gp': 'h6',
+            '3gphd': 'h5',
+            'flv': 'h4',
+            'mp4': 'h3',
+            'hd2': 'h2',
+            'hd3': 'h1'
         }
         return _dict[fm]
 
@@ -194,10 +195,9 @@ class YoukuIE(InfoExtractor):
         # construct info
         entries = []
         for fm in data1['streamtypes']:
-            #formats = []
             video_urls = video_urls_dict[fm]
             for i in range(len(video_urls)):
-                if len(entries) < i+1:
+                if len(entries) < i + 1:
                     entries.append({'formats': []})
                 entries[i]['formats'].append(
                     {
@@ -211,7 +211,7 @@ class YoukuIE(InfoExtractor):
         for i in range(len(entries)):
             entries[i].update(
                 {
-                    'id': '_part%d' % (i+1),
+                    'id': '_part%d' % (i + 1),
                     'title': title,
                 }
             )

From 054932f4035d606946f0c054c02cf87496b753f6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 15 Jun 2015 20:46:10 +0600
Subject: [PATCH 1598/6219] [vk] Fix extraction (Closes #5987)

---
 youtube_dl/extractor/vk.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index d0e772108..6aeba109d 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -175,16 +175,16 @@ class VKIE(InfoExtractor):
                 m_rutube.group(1).replace('\\', ''))
             return self.url_result(rutube_url)
 
-        m_opts = re.search(r'(?s)var\s+opts\s*=\s*({.*?});', info_page)
+        m_opts = re.search(r'(?s)var\s+opts\s*=\s*({.+?});', info_page)
         if m_opts:
-            m_opts_url = re.search(r"url\s*:\s*'([^']+)", m_opts.group(1))
+            m_opts_url = re.search(r"url\s*:\s*'((?!/\b)[^']+)", m_opts.group(1))
             if m_opts_url:
                 opts_url = m_opts_url.group(1)
                 if opts_url.startswith('//'):
                     opts_url = 'http:' + opts_url
                 return self.url_result(opts_url)
 
-        data_json = self._search_regex(r'var vars = ({.*?});', info_page, 'vars')
+        data_json = self._search_regex(r'var\s+vars\s*=\s*({.+?});', info_page, 'vars')
         data = json.loads(data_json)
 
         # Extract upload date

From 7c7dd9dc7fe8139196c7bd1c512301a61f9f362b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 15 Jun 2015 20:47:01 +0600
Subject: [PATCH 1599/6219] [vk] Fix upload date extraction

---
 youtube_dl/extractor/vk.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 6aeba109d..f974f8fef 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -189,7 +189,7 @@ class VKIE(InfoExtractor):
 
         # Extract upload date
         upload_date = None
-        mobj = re.search(r'id="mv_date_wrap".*?Added ([a-zA-Z]+ [0-9]+), ([0-9]+) at', info_page)
+        mobj = re.search(r'id="mv_date(?:_views)?_wrap"[^>]*>([a-zA-Z]+ [0-9]+), ([0-9]+) at', info_page)
         if mobj is not None:
             mobj.group(1) + ' ' + mobj.group(2)
             upload_date = unified_strdate(mobj.group(1) + ' ' + mobj.group(2))

From 8117df4cd9e49a3c7369db3cab6c0b94365c7786 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 15 Jun 2015 20:55:25 +0600
Subject: [PATCH 1600/6219] [vk] Extract view count

---
 youtube_dl/extractor/vk.py | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index f974f8fef..38ff3c1a9 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -13,6 +13,7 @@ from ..compat import (
 from ..utils import (
     ExtractorError,
     orderedSet,
+    str_to_int,
     unescapeHTML,
     unified_strdate,
 )
@@ -34,6 +35,7 @@ class VKIE(InfoExtractor):
                 'uploader': 're:(?:Noize MC|Alexander Ilyashenko).*',
                 'duration': 195,
                 'upload_date': '20120212',
+                'view_count': int,
             },
         },
         {
@@ -45,7 +47,8 @@ class VKIE(InfoExtractor):
                 'uploader': 'Tom Cruise',
                 'title': 'No name',
                 'duration': 9,
-                'upload_date': '20130721'
+                'upload_date': '20130721',
+                'view_count': int,
             }
         },
         {
@@ -59,6 +62,7 @@ class VKIE(InfoExtractor):
                 'title': 'Lin Dan',
                 'duration': 101,
                 'upload_date': '20120730',
+                'view_count': int,
             }
         },
         {
@@ -73,7 +77,8 @@ class VKIE(InfoExtractor):
                 'uploader': 'Триллеры',
                 'title': '► Бойцовский клуб / Fight Club 1999 [HD 720]',
                 'duration': 8352,
-                'upload_date': '20121218'
+                'upload_date': '20121218',
+                'view_count': int,
             },
             'skip': 'Requires vk account credentials',
         },
@@ -100,6 +105,7 @@ class VKIE(InfoExtractor):
                 'title': 'Книга Илая',
                 'duration': 6771,
                 'upload_date': '20140626',
+                'view_count': int,
             },
             'skip': 'Only works from Russia',
         },
@@ -194,6 +200,10 @@ class VKIE(InfoExtractor):
             mobj.group(1) + ' ' + mobj.group(2)
             upload_date = unified_strdate(mobj.group(1) + ' ' + mobj.group(2))
 
+        view_count = str_to_int(self._search_regex(
+            r'"mv_views_count_number"[^>]*>([\d,.]+) views<',
+            info_page, 'view count', fatal=False))
+
         formats = [{
             'format_id': k,
             'url': v,
@@ -210,6 +220,7 @@ class VKIE(InfoExtractor):
             'uploader': data.get('md_author'),
             'duration': data.get('duration'),
             'upload_date': upload_date,
+            'view_count': view_count,
         }
 
 

From 02175a7986c4223a0ed27a872c1ca16926913e05 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 15 Jun 2015 21:01:26 +0600
Subject: [PATCH 1601/6219] [youtube:search] Fix search query (Closes #5988)

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 3448bec4f..9e2671192 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1504,7 +1504,7 @@ class YoutubeSearchIE(SearchInfoExtractor, YoutubePlaylistIE):
 
         for pagenum in itertools.count(1):
             url_query = {
-                'search_query': query,
+                'search_query': query.encode('utf-8'),
                 'page': pagenum,
                 'spf': 'navigate',
             }

From c203be3fb4f00388c81564dc0c85ff8a10ff4553 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 15 Jun 2015 23:28:59 +0800
Subject: [PATCH 1602/6219] [youku] Better handling for Python 2/3
 compatibility

---
 youtube_dl/extractor/youku.py | 37 ++++++++++++-----------------------
 1 file changed, 13 insertions(+), 24 deletions(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index 4e47fca8a..26e5baadc 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -7,9 +7,10 @@ import base64
 from .common import InfoExtractor
 from ..utils import ExtractorError
 
-from ..compat import compat_urllib_parse
-
-bytes_is_str = (bytes == str)  # for compatible
+from ..compat import (
+    compat_urllib_parse,
+    compat_ord,
+)
 
 
 class YoukuIE(InfoExtractor):
@@ -37,26 +38,20 @@ class YoukuIE(InfoExtractor):
             ls = list(range(256))
             t = 0
             for i in range(256):
-                t = (t + ls[i] + ord(s1[i % len(s1)])) % 256
+                t = (t + ls[i] + compat_ord(s1[i % len(s1)])) % 256
                 ls[i], ls[t] = ls[t], ls[i]
-            s = '' if not bytes_is_str else b''
+            s = bytearray()
             x, y = 0, 0
             for i in range(len(s2)):
                 y = (y + 1) % 256
                 x = (x + ls[y]) % 256
                 ls[x], ls[y] = ls[y], ls[x]
-                if isinstance(s2[i], int):
-                    s += chr(s2[i] ^ ls[(ls[x] + ls[y]) % 256])
-                else:
-                    s += chr(ord(s2[i]) ^ ls[(ls[x] + ls[y]) % 256])
-            return s
+                s.append(compat_ord(s2[i]) ^ ls[(ls[x] + ls[y]) % 256])
+            return bytes(s)
 
         sid, token = yk_t(
-            'becaf9be',
-            base64.b64decode(bytes(data2['ep'], 'ascii'))
-            if not bytes_is_str
-            else base64.b64decode(data2['ep'])
-        ).split('_')
+            b'becaf9be', base64.b64decode(data2['ep'].encode('ascii'))
+        ).decode('ascii').split('_')
 
         # get oip
         oip = data2['ip']
@@ -89,16 +84,10 @@ class YoukuIE(InfoExtractor):
         def generate_ep(format, n):
             fileid = get_fileid(format, n)
             ep_t = yk_t(
-                'bf7e5f01',
-                bytes('%s_%s_%s' % (sid, fileid, token), 'ascii')
-                if not bytes_is_str
-                else ('%s_%s_%s' % (sid, fileid, token))
+                b'bf7e5f01',
+                ('%s_%s_%s' % (sid, fileid, token)).encode('ascii')
             )
-            ep = base64.b64encode(
-                bytes(ep_t, 'latin')
-                if not bytes_is_str
-                else ep_t
-            ).decode()
+            ep = base64.b64encode(ep_t).decode('ascii')
             return ep
 
         # generate video_urls

From 99e6833c85868b78df7c810603ffdccdaeb4eaf0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 15 Jun 2015 21:30:27 +0600
Subject: [PATCH 1603/6219] [francetv] Update f4m manifest token URL (Closes
 #5981, Closes #5989)

---
 youtube_dl/extractor/francetv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index edf555b29..db0bbec1e 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -60,7 +60,7 @@ class FranceTVBaseInfoExtractor(InfoExtractor):
                     continue
                 video_url_parsed = compat_urllib_parse_urlparse(video_url)
                 f4m_url = self._download_webpage(
-                    'http://hdfauth.francetv.fr/esi/urltokengen2.html?url=%s' % video_url_parsed.path,
+                    'http://hdfauth.francetv.fr/esi/TA?url=%s' % video_url_parsed.path,
                     video_id, 'Downloading f4m manifest token', fatal=False)
                 if f4m_url:
                     formats.extend(self._extract_f4m_formats(f4m_url, video_id, 1, format_id))

From 9383e66f9475eca0e64c09972c1392d92d17570c Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 15 Jun 2015 23:31:30 +0800
Subject: [PATCH 1604/6219] [youku] Use _match_id

---
 youtube_dl/extractor/youku.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index 26e5baadc..e41b48369 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -1,7 +1,6 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
 import base64
 
 from .common import InfoExtractor
@@ -154,8 +153,7 @@ class YoukuIE(InfoExtractor):
         return _dict[fm]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         # request basic data
         data1_url = 'http://v.youku.com/player/getPlayList/VideoIDS/%s' % video_id

From ee69799262e8344742b9d8b492fe792b4d586f6a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 15 Jun 2015 23:36:28 +0800
Subject: [PATCH 1605/6219] [youku] Add a v.swf test case

---
 youtube_dl/extractor/youku.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index e41b48369..d8162a0c5 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -21,7 +21,7 @@ class YoukuIE(InfoExtractor):
         (?P<id>[A-Za-z0-9]+)(?:\.html|/v\.swf|)
     '''
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://v.youku.com/v_show/id_XMTc1ODE5Njcy.html',
         'md5': '5f3af4192eabacc4501508d54a8cabd7',
         'info_dict': {
@@ -29,7 +29,10 @@ class YoukuIE(InfoExtractor):
             'title': '★Smile﹗♡ Git Fresh -Booty Music舞蹈.',
             'ext': 'flv'
         }
-    }
+    }, {
+        'url': 'http://player.youku.com/player.php/sid/XNDgyMDQ2NTQw/v.swf',
+        'only_matching': True,
+    }]
 
     def construct_video_urls(self, data1, data2):
         # get sid, token

From 4fd35ee072a39654f11e794db5c50ee375c9a7c6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 15 Jun 2015 21:36:30 +0600
Subject: [PATCH 1606/6219] [safari] Add test for #5985

---
 youtube_dl/extractor/safari.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/safari.py b/youtube_dl/extractor/safari.py
index 20ba6fa33..f3c80708c 100644
--- a/youtube_dl/extractor/safari.py
+++ b/youtube_dl/extractor/safari.py
@@ -100,6 +100,10 @@ class SafariIE(SafariBaseIE):
     }, {
         'url': 'https://www.safaribooksonline.com/api/v1/book/9780133392838/chapter/part00.html',
         'only_matching': True,
+    }, {
+        # non-digits in course id
+        'url': 'https://www.safaribooksonline.com/library/view/create-a-nodejs/100000006A0210/part00.html',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From f1e66cb2eb40b48c6508acbe57207a2d99792bf0 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 15 Jun 2015 23:46:07 +0800
Subject: [PATCH 1607/6219] [youku] Change video_id and add a multipart test
 case

---
 youtube_dl/extractor/youku.py | 29 +++++++++++++++--------------
 1 file changed, 15 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index d8162a0c5..d5b73ebce 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -25,13 +25,20 @@ class YoukuIE(InfoExtractor):
         'url': 'http://v.youku.com/v_show/id_XMTc1ODE5Njcy.html',
         'md5': '5f3af4192eabacc4501508d54a8cabd7',
         'info_dict': {
-            'id': 'XMTc1ODE5Njcy',
+            'id': 'XMTc1ODE5Njcy_part1',
             'title': '★Smile﹗♡ Git Fresh -Booty Music舞蹈.',
             'ext': 'flv'
         }
     }, {
         'url': 'http://player.youku.com/player.php/sid/XNDgyMDQ2NTQw/v.swf',
         'only_matching': True,
+    }, {
+        'url': 'http://v.youku.com/v_show/id_XODgxNjg1Mzk2_ev_1.html',
+        'info_dict': {
+            'id': 'XODgxNjg1Mzk2',
+            'title': '武媚娘传奇 85',
+        },
+        'playlist_count': 11,
     }]
 
     def construct_video_urls(self, data1, data2):
@@ -201,20 +208,14 @@ class YoukuIE(InfoExtractor):
         for i in range(len(entries)):
             entries[i].update(
                 {
-                    'id': '_part%d' % (i + 1),
+                    'id': '%s_part%d' % (video_id, i + 1),
                     'title': title,
                 }
             )
 
-        if len(entries) > 1:
-            info = {
-                '_type': 'multi_video',
-                'id': video_id,
-                'title': title,
-                'entries': entries,
-            }
-        else:
-            info = entries[0]
-            info['id'] = video_id
-
-        return info
+        return {
+            '_type': 'multi_video',
+            'id': video_id,
+            'title': title,
+            'entries': entries,
+        }

From 04e7596680bce28beae2436bac0f6d1f01a45210 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 15 Jun 2015 23:54:55 +0800
Subject: [PATCH 1608/6219] [youku] Better error handling

---
 youtube_dl/extractor/youku.py | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index d5b73ebce..91f9f6bff 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -176,13 +176,15 @@ class YoukuIE(InfoExtractor):
 
         error_code = data1.get('error_code')
         if error_code:
-            # -8 means blocked outside China.
-            # Chinese and English, separated by newline.
             error = data1.get('error')
-            raise ExtractorError(
-                error or 'Server reported error %i' %
-                error_code,
-                expected=True)
+            if error is not None and '因版权原因无法观看此视频' in error:
+                raise ExtractorError(
+                    'Youku said: Sorry, this video is available in China only', expected=True)
+            else:
+                msg = 'Youku server reported error %i' % error_code
+                if error is not None:
+                    msg += ': ' + error
+                raise ExtractorError(msg)
 
         title = data1['title']
 

From 5228b756af2c2bfc2962a5b1bb6db1e6a41c9e05 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 16 Jun 2015 00:06:23 +0800
Subject: [PATCH 1609/6219] [youku] Add cn_verification_proxy support and add a
 georestricted test case

---
 youtube_dl/extractor/youku.py | 32 +++++++++++++++++++++++++-------
 1 file changed, 25 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index 91f9f6bff..ea37dc8b2 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -9,6 +9,7 @@ from ..utils import ExtractorError
 from ..compat import (
     compat_urllib_parse,
     compat_ord,
+    compat_urllib_request,
 )
 
 
@@ -39,6 +40,14 @@ class YoukuIE(InfoExtractor):
             'title': '武媚娘传奇 85',
         },
         'playlist_count': 11,
+    }, {
+        'url': 'http://v.youku.com/v_show/id_XMTI1OTczNDM5Mg==.html',
+        'info_dict': {
+            'id': 'XMTI1OTczNDM5Mg',
+            'title': '花千骨 04',
+        },
+        'playlist_count': 13,
+        'skip': 'Available in China only',
     }]
 
     def construct_video_urls(self, data1, data2):
@@ -165,14 +174,23 @@ class YoukuIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        # request basic data
-        data1_url = 'http://v.youku.com/player/getPlayList/VideoIDS/%s' % video_id
-        data2_url = 'http://v.youku.com/player/getPlayList/VideoIDS/%s/Pf/4/ctype/12/ev/1' % video_id
+        def retrieve_data(req_url, note):
+            req = compat_urllib_request.Request(req_url)
 
-        raw_data1 = self._download_json(data1_url, video_id)
-        raw_data2 = self._download_json(data2_url, video_id)
-        data1 = raw_data1['data'][0]
-        data2 = raw_data2['data'][0]
+            cn_verification_proxy = self._downloader.params.get('cn_verification_proxy')
+            if cn_verification_proxy:
+                req.add_header('Ytdl-request-proxy', cn_verification_proxy)
+
+            raw_data = self._download_json(req, video_id, note=note)
+            return raw_data['data'][0]
+
+        # request basic data
+        data1 = retrieve_data(
+            'http://v.youku.com/player/getPlayList/VideoIDS/%s' % video_id,
+            'Downloading JSON metadata 1')
+        data2 = retrieve_data(
+            'http://v.youku.com/player/getPlayList/VideoIDS/%s/Pf/4/ctype/12/ev/1' % video_id,
+            'Downloading JSON metadata 2')
 
         error_code = data1.get('error_code')
         if error_code:

From a155b7e76c5a71c650f62c4716d23a24943fc373 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 16 Jun 2015 00:15:09 +0800
Subject: [PATCH 1610/6219] [youku] Coding style

---
 youtube_dl/extractor/youku.py | 24 ++++++++++--------------
 1 file changed, 10 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index ea37dc8b2..cab5be3a4 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -216,22 +216,18 @@ class YoukuIE(InfoExtractor):
             for i in range(len(video_urls)):
                 if len(entries) < i + 1:
                     entries.append({'formats': []})
-                entries[i]['formats'].append(
-                    {
-                        'url': video_urls[i],
-                        'format_id': self.get_format_name(fm),
-                        'ext': self.parse_ext_l(fm),
-                        'filesize': int(data1['segs'][fm][i]['size'])
-                    }
-                )
+                entries[i]['formats'].append({
+                    'url': video_urls[i],
+                    'format_id': self.get_format_name(fm),
+                    'ext': self.parse_ext_l(fm),
+                    'filesize': int(data1['segs'][fm][i]['size'])
+                })
 
         for i in range(len(entries)):
-            entries[i].update(
-                {
-                    'id': '%s_part%d' % (video_id, i + 1),
-                    'title': title,
-                }
-            )
+            entries[i].update({
+                'id': '%s_part%d' % (video_id, i + 1),
+                'title': title,
+            })
 
         return {
             '_type': 'multi_video',

From 0501bfa159db5b5e8ed7fd1ed966b9989becb3e9 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 16 Jun 2015 00:15:30 +0800
Subject: [PATCH 1611/6219] [YoutubeDL] Youku extractor now uses the standard
 format selection

---
 youtube_dl/YoutubeDL.py | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index aacec2958..6e4b6f566 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1033,12 +1033,6 @@ class YoutubeDL(object):
             info_dict['id'], info_dict.get('subtitles'),
             info_dict.get('automatic_captions'))
 
-        # This extractors handle format selection themselves
-        if info_dict['extractor'] in ['Youku']:
-            if download:
-                self.process_info(info_dict)
-            return info_dict
-
         # We now pick which formats have to be downloaded
         if info_dict.get('formats') is None:
             # There's only one format available

From 7f0172b3e5e0da2a19708fdf3ec1b521a6e2656f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 15 Jun 2015 22:29:41 +0600
Subject: [PATCH 1612/6219] Credit @jackyzy823 for iqiyi

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index bf2a25cb8..889d599a2 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -127,3 +127,4 @@ Julian Richen
 Ping O.
 Mister Hat
 Peter Ding
+jackyzy823

From f3aecb27a4d7b178ae66b4a294cff5dbe9bb2b18 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 16 Jun 2015 14:41:52 +0200
Subject: [PATCH 1613/6219] [youku] Simplify a bit the 'entries' construction

Mainly avoid having to use an index.
---
 youtube_dl/extractor/youku.py | 24 +++++++++++-------------
 1 file changed, 11 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index cab5be3a4..ced3a10cd 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -210,25 +210,23 @@ class YoukuIE(InfoExtractor):
         video_urls_dict = self.construct_video_urls(data1, data2)
 
         # construct info
-        entries = []
+        entries = [{
+            'id': '%s_part%d' % (video_id, i + 1),
+            'title': title,
+            'formats': [],
+            # some formats are not available for all parts, we have to detect
+            # which one has all
+        } for i in range(max(len(v) for v in data1['segs'].values()))]
         for fm in data1['streamtypes']:
             video_urls = video_urls_dict[fm]
-            for i in range(len(video_urls)):
-                if len(entries) < i + 1:
-                    entries.append({'formats': []})
-                entries[i]['formats'].append({
-                    'url': video_urls[i],
+            for video_url, seg, entry in zip(video_urls, data1['segs'][fm], entries):
+                entry['formats'].append({
+                    'url': video_url,
                     'format_id': self.get_format_name(fm),
                     'ext': self.parse_ext_l(fm),
-                    'filesize': int(data1['segs'][fm][i]['size'])
+                    'filesize': int(seg['size']),
                 })
 
-        for i in range(len(entries)):
-            entries[i].update({
-                'id': '%s_part%d' % (video_id, i + 1),
-                'title': title,
-            })
-
         return {
             '_type': 'multi_video',
             'id': video_id,

From 447053668fbed993f6f4fd2e06d9282ea30224bb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 16 Jun 2015 21:19:18 +0600
Subject: [PATCH 1614/6219] [spankwire] Fix extraction

---
 youtube_dl/extractor/spankwire.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/spankwire.py b/youtube_dl/extractor/spankwire.py
index 06d6e6640..bff75d6b2 100644
--- a/youtube_dl/extractor/spankwire.py
+++ b/youtube_dl/extractor/spankwire.py
@@ -27,7 +27,7 @@ class SpankwireIE(InfoExtractor):
             'description': 'Crazy Bitch X rated music video.',
             'uploader': 'oreusz',
             'uploader_id': '124697',
-            'upload_date': '20070508',
+            'upload_date': '20070507',
             'age_limit': 18,
         }
     }
@@ -44,7 +44,7 @@ class SpankwireIE(InfoExtractor):
         title = self._html_search_regex(
             r'<h1>([^<]+)', webpage, 'title')
         description = self._html_search_regex(
-            r'<div\s+id="descriptionContent">([^<]+)<',
+            r'(?s)<div\s+id="descriptionContent">(.+?)</div>',
             webpage, 'description', fatal=False)
         thumbnail = self._html_search_regex(
             r'playerData\.screenShot\s*=\s*["\']([^"\']+)["\']',
@@ -64,12 +64,12 @@ class SpankwireIE(InfoExtractor):
             r'<div id="viewsCounter"><span>([\d,\.]+)</span> views</div>',
             webpage, 'view count', fatal=False))
         comment_count = str_to_int(self._html_search_regex(
-            r'Comments<span[^>]+>\s*\(([\d,\.]+)\)</span>',
+            r'<span\s+id="spCommentCount"[^>]*>([\d,\.]+)</span>',
             webpage, 'comment count', fatal=False))
 
         video_urls = list(map(
             compat_urllib_parse.unquote,
-            re.findall(r'playerData\.cdnPath[0-9]{3,}\s*=\s*["\']([^"\']+)["\']', webpage)))
+            re.findall(r'playerData\.cdnPath[0-9]{3,}\s*=\s*(?:encodeURIComponent\()?["\']([^"\']+)["\']', webpage)))
         if webpage.find('flashvars\.encrypted = "true"') != -1:
             password = self._search_regex(
                 r'flashvars\.video_title = "([^"]+)',

From 14835de9fb41798c8e6e731a3f07ae871770666f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aur=C3=A9lio=20A=2E=20Heckert?= <aurelio@colivre.coop.br>
Date: Tue, 16 Jun 2015 18:10:31 -0300
Subject: [PATCH 1615/6219] Use shlex.split for --pp-params and update related
 docs.

---
 README.md                          | 2 +-
 youtube_dl/YoutubeDL.py            | 1 +
 youtube_dl/__init__.py             | 6 ++++--
 youtube_dl/options.py              | 4 ++--
 youtube_dl/postprocessor/common.py | 3 ++-
 5 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/README.md b/README.md
index 726ec9cf2..813ac4a15 100644
--- a/README.md
+++ b/README.md
@@ -214,7 +214,7 @@ which means you can modify it, redistribute it or use it however you like.
     --audio-quality QUALITY          Specify ffmpeg/avconv audio quality, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K (default
                                      5)
     --recode-video FORMAT            Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm|mkv|xvid)
-    --pp-params                      Extra parameters for video post-processor. The params will be splited on spaces.
+    --pp-params                      Extra parameters for video post-processor.
     -k, --keep-video                 Keep the video file on disk after the post-processing; the video is erased by default
     --no-post-overwrites             Do not overwrite post-processed files; the post-processed files are overwritten by default
     --embed-subs                     Embed subtitles in the video (only for mkv and mp4 videos)
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index b1f792d4e..3bfe30c76 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -261,6 +261,7 @@ class YoutubeDL(object):
     The following options are used by the post processors:
     prefer_ffmpeg:     If True, use ffmpeg instead of avconv if both are available,
                        otherwise prefer avconv.
+    pp_params:         Extra parameters for external apps, like avconv.
     """
 
     params = None
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 5b28e4817..8b54d4ae2 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -171,8 +171,10 @@ def _real_main(argv=None):
     if opts.recodevideo is not None:
         if opts.recodevideo not in ['mp4', 'flv', 'webm', 'ogg', 'mkv', 'xvid']:
             parser.error('invalid video recode format specified')
-    if opts.pp_params is not None:
-        opts.pp_params = opts.pp_params.split()
+    if opts.pp_params is None:
+        opts.pp_params = []
+    else:
+        opts.pp_params = shlex.split(opts.pp_params)
     if opts.convertsubtitles is not None:
         if opts.convertsubtitles not in ['srt', 'vtt', 'ass']:
             parser.error('invalid subtitle format specified')
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index ceb4b5f38..fbba9b9d8 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -689,8 +689,8 @@ def parseOpts(overrideArguments=None):
         help='Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm|mkv|xvid)')
     postproc.add_option(
         '--pp-params',
-        dest='pp_params', default=None,
-        help='Extra parameters for video post-processor. The params will be splited on spaces.')
+        dest='pp_params', default=None, metavar='ARGS',
+        help='Extra parameters for video post-processor.')
     postproc.add_option(
         '-k', '--keep-video',
         action='store_true', dest='keepvideo', default=False,
diff --git a/youtube_dl/postprocessor/common.py b/youtube_dl/postprocessor/common.py
index 3b0e8ddd8..d944d9367 100644
--- a/youtube_dl/postprocessor/common.py
+++ b/youtube_dl/postprocessor/common.py
@@ -22,7 +22,8 @@ class PostProcessor(object):
     of the chain is reached.
 
     PostProcessor objects follow a "mutual registration" process similar
-    to InfoExtractor objects.
+    to InfoExtractor objects. And it can receive parameters from CLI trough
+    --pp-params.
     """
 
     _downloader = None

From 028a33d7f2a0bc028f533530d2722b57b31dabdc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 17 Jun 2015 20:27:38 +0600
Subject: [PATCH 1616/6219] [lifenews] Fix extraction

---
 youtube_dl/extractor/lifenews.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/lifenews.py b/youtube_dl/extractor/lifenews.py
index 42cb6e35f..373122c93 100644
--- a/youtube_dl/extractor/lifenews.py
+++ b/youtube_dl/extractor/lifenews.py
@@ -82,10 +82,11 @@ class LifeNewsIE(InfoExtractor):
         view_count = self._html_search_regex(
             r'<div class=\'views\'>\s*(\d+)\s*</div>', webpage, 'view count', fatal=False)
         comment_count = self._html_search_regex(
-            r'<div class=\'comments\'>\s*<span class=\'counter\'>\s*(\d+)\s*</span>', webpage, 'comment count', fatal=False)
+            r'=\'commentCount\'[^>]*>\s*(\d+)\s*<',
+            webpage, 'comment count', fatal=False)
 
         upload_date = self._html_search_regex(
-            r'<time datetime=\'([^\']+)\'>', webpage, 'upload date', fatal=False)
+            r'<time[^>]*datetime=\'([^\']+)\'', webpage, 'upload date', fatal=False)
         if upload_date is not None:
             upload_date = unified_strdate(upload_date)
 

From 062a3fdf36dee6515c9e3b103c39ac2691196e95 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 17 Jun 2015 20:28:47 +0600
Subject: [PATCH 1617/6219] [lifenews] Modernize

---
 youtube_dl/extractor/lifenews.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/lifenews.py b/youtube_dl/extractor/lifenews.py
index 373122c93..1f07db3ec 100644
--- a/youtube_dl/extractor/lifenews.py
+++ b/youtube_dl/extractor/lifenews.py
@@ -8,6 +8,7 @@ from ..compat import compat_urlparse
 from ..utils import (
     determine_ext,
     int_or_none,
+    remove_end,
     unified_strdate,
     ExtractorError,
 )
@@ -72,10 +73,9 @@ class LifeNewsIE(InfoExtractor):
         if not videos and not iframe_link:
             raise ExtractorError('No media links available for %s' % video_id)
 
-        title = self._og_search_title(webpage)
-        TITLE_SUFFIX = ' - Первый по срочным новостям — LIFE | NEWS'
-        if title.endswith(TITLE_SUFFIX):
-            title = title[:-len(TITLE_SUFFIX)]
+        title = remove_end(
+            self._og_search_title(webpage),
+            ' - Первый по срочным новостям — LIFE | NEWS')
 
         description = self._og_search_description(webpage)
 

From a69e8bfdd9e94537eb72a358c1b62dbea8a59a04 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 17 Jun 2015 20:29:15 +0600
Subject: [PATCH 1618/6219] [lifenews] Fix tests

---
 youtube_dl/extractor/lifenews.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/youtube_dl/extractor/lifenews.py b/youtube_dl/extractor/lifenews.py
index 1f07db3ec..f8cbca7b3 100644
--- a/youtube_dl/extractor/lifenews.py
+++ b/youtube_dl/extractor/lifenews.py
@@ -40,7 +40,6 @@ class LifeNewsIE(InfoExtractor):
             'title': 'В Сети появилось видео захвата «Правым сектором» колхозных полей ',
             'description': 'Жители двух поселков Днепропетровской области не простили радикалам угрозу лишения плодородных земель и пошли в лобовую. ',
             'upload_date': '20150402',
-            'uploader': 'embed.life.ru',
         }
     }, {
         'url': 'http://lifenews.ru/news/153461',
@@ -51,7 +50,6 @@ class LifeNewsIE(InfoExtractor):
             'title': 'В Москве спасли потерявшегося медвежонка, который спрятался на дереве',
             'description': 'Маленький хищник не смог найти дорогу домой и обрел временное убежище на тополе недалеко от жилого массива, пока его не нашла соседская собака.',
             'upload_date': '20150505',
-            'uploader': 'embed.life.ru',
         }
     }, {
         'url': 'http://lifenews.ru/video/13035',

From 0d0d5d37174ed611dd823c5be025c49e73d83d1d Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Thu, 18 Jun 2015 13:59:37 +0800
Subject: [PATCH 1619/6219] [qqmusic] Add support for playlists

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/qqmusic.py  | 34 ++++++++++++++++++++++++++++++++
 2 files changed, 35 insertions(+)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 6fdaf90b2..d03577cdf 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -427,6 +427,7 @@ from .qqmusic import (
     QQMusicSingerIE,
     QQMusicAlbumIE,
     QQMusicToplistIE,
+    QQMusicPlaylistIE,
 )
 from .quickvid import QuickVidIE
 from .r7 import R7IE
diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index bafa81c21..f9aafcd28 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -9,6 +9,7 @@ from .common import InfoExtractor
 from ..utils import (
     strip_jsonp,
     unescapeHTML,
+    clean_html,
 )
 from ..compat import compat_urllib_request
 
@@ -243,3 +244,36 @@ class QQMusicToplistIE(QQPlaylistBaseIE):
         list_name = topinfo.get('ListName')
         list_description = topinfo.get('info')
         return self.playlist_result(entries, list_id, list_name, list_description)
+
+
+class QQMusicPlaylistIE(QQPlaylistBaseIE):
+    IE_NAME = 'qqmusic:playlist'
+    _VALID_URL = r'http://y\.qq\.com/#type=taoge&id=(?P<id>[0-9]+)'
+
+    _TEST = {
+        'url': 'http://y.qq.com/#type=taoge&id=3462654915',
+        'info_dict': {
+            'id': '3462654915',
+            'title': '韩国5月新歌精选下旬',
+            'description': 'md5:d2c9d758a96b9888cf4fe82f603121d4',
+        },
+        'playlist_count': 40,
+    }
+
+    def _real_extract(self, url):
+        list_id = self._match_id(url)
+
+        list_json = self._download_json(
+            'http://i.y.qq.com/qzone-music/fcg-bin/fcg_ucc_getcdinfo_byids_cp.fcg?type=1&json=1&utf8=1&onlysong=0&disstid=%s'
+            % list_id, list_id, 'Download list page',
+            transform_source=strip_jsonp)['cdlist'][0]
+
+        entries = [
+            self.url_result(
+                'http://y.qq.com/#type=song&mid=' + song['songmid'], 'QQMusic', song['songmid']
+            ) for song in list_json['songlist']
+        ]
+
+        list_name = list_json['dissname']
+        list_description = clean_html(unescapeHTML(list_json.get('desc')))
+        return self.playlist_result(entries, list_id, list_name, list_description)

From 8f73e89ca0ecde0a8bbd1f1463e9a06a53c6b573 Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Thu, 18 Jun 2015 14:15:54 +0800
Subject: [PATCH 1620/6219] [kuwo] New extractor for kuwo.cn

---
 youtube_dl/extractor/__init__.py |   9 +
 youtube_dl/extractor/kuwo.py     | 326 +++++++++++++++++++++++++++++++
 2 files changed, 335 insertions(+)
 create mode 100644 youtube_dl/extractor/kuwo.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 6fdaf90b2..82cd85c44 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -257,6 +257,15 @@ from .keek import KeekIE
 from .kontrtube import KontrTubeIE
 from .krasview import KrasViewIE
 from .ku6 import Ku6IE
+from .kuwo import (
+    KuwoIE,
+    KuwoAlbumIE,
+    KuwoChartIE,
+    KuwoSingerIE,
+    KuwoSingerMusicIE,
+    KuwoCategoryIE,
+    KuwoMvIE,
+)
 from .la7 import LA7IE
 from .laola1tv import Laola1TvIE
 from .letv import (
diff --git a/youtube_dl/extractor/kuwo.py b/youtube_dl/extractor/kuwo.py
new file mode 100644
index 000000000..6a96a1aa4
--- /dev/null
+++ b/youtube_dl/extractor/kuwo.py
@@ -0,0 +1,326 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+import itertools
+
+from .common import InfoExtractor
+from ..utils import (
+    get_element_by_id,
+    clean_html,
+    ExtractorError,
+)
+
+
+class KuwoIE(InfoExtractor):
+    IE_NAME = 'kuwo:song'
+    _VALID_URL = r'http://www\.kuwo\.cn/yinyue/(?P<id>[0-9]+?)/'
+    _TESTS = [{
+        'url': 'http://www.kuwo.cn/yinyue/635632/',
+        'info_dict': {
+            'id': '635632',
+            'ext': 'ape',
+            'title': '爱我别走',
+            'creator': '张震岳',
+            'upload_date': '20080122',
+            'description': 'md5:ed13f58e3c3bf3f7fd9fbc4e5a7aa75c'
+        },
+    }, {
+        'url': 'http://www.kuwo.cn/yinyue/6446136/',
+        'info_dict': {
+            'id': '6446136',
+            'ext': 'mp3',
+            'title': '心',
+            'creator': 'IU',
+            'upload_date': '20150518',
+        },
+        'params': {
+            'format': 'mp3-320'
+        },
+    }]
+    _FORMATS = [
+        {'format': 'ape', 'ext': 'ape', 'preference': 100},
+        {'format': 'mp3-320', 'ext': 'mp3', 'br': '320kmp3', 'abr': 320, 'preference': 80},
+        {'format': 'mp3-192', 'ext': 'mp3', 'br': '192kmp3', 'abr': 192, 'preference': 70},
+        {'format': 'mp3-128', 'ext': 'mp3', 'br': '128kmp3', 'abr': 128, 'preference': 60},
+        {'format': 'wma', 'ext': 'wma', 'preference': 20},
+        {'format': 'aac', 'ext': 'aac', 'abr': 48, 'preference': 10}
+    ]
+
+    def _get_formats(self, song_id):
+        formats = []
+        for file_format in self._FORMATS:
+            song_url = self._download_webpage(
+                "http://antiserver.kuwo.cn/anti.s?format=%s&br=%s&rid=MUSIC_%s&type=convert_url&response=url" %
+                (file_format['ext'], file_format.get('br', ''), song_id),
+                song_id, note="Download %s url info" % file_format["format"],
+            )
+            if song_url.startswith('http://') or song_url.startswith('https://'):
+                formats.append({
+                    'url': song_url,
+                    'format_id': file_format['format'],
+                    'format': file_format['format'],
+                    'preference': file_format['preference'],
+                    'abr': file_format.get('abr'),
+                })
+        self._sort_formats(formats)
+        return formats
+
+    def _real_extract(self, url):
+        song_id = self._match_id(url)
+        webpage = self._download_webpage(
+            url, song_id, note='Download song detail info',
+            errnote='Unable to get song detail info')
+
+        song_name = self._html_search_regex(
+            r'<h1 title="(.+?)">', webpage, 'song name')
+        singer_name = self._html_search_regex(
+            r'<div class="s_img">.+?title="(.+?)".+?</div>', webpage, 'singer name',
+            flags=re.DOTALL, default=None)
+        lrc_content = clean_html(get_element_by_id("lrcContent", webpage))
+        if lrc_content == '暂无':     # indicates no lyrics
+            lrc_content = None
+
+        formats = self._get_formats(song_id)
+
+        album_id = self._html_search_regex(
+            r'<p class="album" title=".+?">.+?<a href="http://www\.kuwo\.cn/album/([0-9]+)/" ',
+            webpage, 'album id', default=None, fatal=False)
+
+        publish_time = None
+        if album_id is not None:
+            album_info_page = self._download_webpage(
+                "http://www.kuwo.cn/album/%s/" % album_id, song_id,
+                note='Download album detail info',
+                errnote='Unable to get album detail info')
+
+            publish_time = self._html_search_regex(
+                r'发行时间：(\d{4}-\d{2}-\d{2})', album_info_page,
+                'publish time', default=None)
+            if publish_time:
+                publish_time = publish_time.replace('-', '')
+
+        return {
+            'id': song_id,
+            'title': song_name,
+            'creator': singer_name,
+            'upload_date': publish_time,
+            'description': lrc_content,
+            'formats': formats,
+        }
+
+
+class KuwoAlbumIE(InfoExtractor):
+    IE_NAME = 'kuwo:album'
+    _VALID_URL = r'http://www\.kuwo\.cn/album/(?P<id>[0-9]+?)/'
+    _TEST = {
+        'url': 'http://www.kuwo.cn/album/502294/',
+        'info_dict': {
+            'id': '502294',
+            'title': 'M',
+            'description': 'md5:6a7235a84cc6400ec3b38a7bdaf1d60c',
+        },
+        'playlist_count': 2,
+    }
+
+    def _real_extract(self, url):
+        album_id = self._match_id(url)
+
+        webpage = self._download_webpage(
+            url, album_id, note='Download album info',
+            errnote='Unable to get album info')
+
+        album_name = self._html_search_regex(
+            r'<div class="comm".+?<h1 title="(.+?)">.+?</h1>', webpage,
+            'album name', flags=re.DOTALL)
+        album_intro = clean_html(
+            re.sub(r'^.+简介：', '', get_element_by_id("intro", webpage).strip()))
+
+        entries = [
+            self.url_result("http://www.kuwo.cn/yinyue/%s/" % song_id, 'Kuwo', song_id)
+            for song_id in re.findall(
+                r'<p class="listen"><a href="http://www\.kuwo\.cn/yinyue/([0-9]+)/" target="_blank" title="试听.*?"></a></p>',
+                webpage)
+        ]
+        return self.playlist_result(entries, album_id, album_name, album_intro)
+
+
+class KuwoChartIE(InfoExtractor):
+    IE_NAME = 'kuwo:chart'
+    _VALID_URL = r'http://yinyue\.kuwo\.cn/billboard_(?P<id>.+?).htm'
+    _TEST = {
+        'url': 'http://yinyue.kuwo.cn/billboard_香港中文龙虎榜.htm',
+        'info_dict': {
+            'id': '香港中文龙虎榜',
+            'title': '香港中文龙虎榜',
+            'description': 're:[0-9]{4}第[0-9]{2}期',
+        },
+        'playlist_mincount': 10,
+    }
+
+    def _real_extract(self, url):
+        chart_id = self._match_id(url)
+        webpage = self._download_webpage(
+            url, chart_id, note='Download chart info',
+            errnote='Unable to get chart info')
+
+        chart_name = self._html_search_regex(
+            r'<h1 class="unDis">(.+?)</h1>', webpage, 'chart name')
+
+        chart_desc = self._html_search_regex(
+            r'<p class="tabDef">([0-9]{4}第[0-9]{2}期)</p>', webpage, 'chart desc')
+
+        entries = [
+            self.url_result("http://www.kuwo.cn/yinyue/%s/" % song_id, 'Kuwo', song_id)
+            for song_id in re.findall(
+                r'<a href="http://www\.kuwo\.cn/yinyue/([0-9]+)/" .+?>.+?</a>', webpage)
+        ]
+        return self.playlist_result(entries, chart_id, chart_name, chart_desc)
+
+
+class KuwoSingerIE(InfoExtractor):
+    IE_NAME = 'kuwo:singer'
+    _VALID_URL = r'http://www\.kuwo\.cn/mingxing/(?P<id>[^/]+?)/$'
+    _TEST = {
+        'url': 'http://www.kuwo.cn/mingxing/bruno+mars/',
+        'info_dict': {
+            'id': 'bruno+mars',
+            'title': 'Bruno Mars',
+        },
+        'playlist_count': 10,
+    }
+
+    def _real_extract(self, url):
+        singer_id = self._match_id(url)
+        webpage = self._download_webpage(
+            url, singer_id, note='Download singer info',
+            errnote='Unable to get singer info')
+
+        singer_name = self._html_search_regex(
+            r'姓名：<span>(.+?)</span>', webpage, 'singer name')
+
+        entries = [
+            self.url_result("http://www.kuwo.cn/yinyue/%s/" % song_id, 'Kuwo', song_id)
+            for song_id in re.findall(
+                r'<a href="http://www\.kuwo\.cn/yinyue/([0-9]+)/" .+?>.+?</a>',
+                webpage, flags=re.DOTALL)
+        ]
+        return self.playlist_result(entries, singer_id, singer_name)
+
+
+class KuwoSingerMusicIE(InfoExtractor):
+    IE_NAME = 'kuwo:singermusic'
+    _VALID_URL = r'http://www\.kuwo\.cn/mingxing/(?P<id>[^/]+?)/music(_[0-9]+)?.htm'
+    _TEST = {
+        'url': 'http://www.kuwo.cn/mingxing/Ali/music.htm',
+        'info_dict': {
+            'id': 'Ali',
+            'title': 'Ali的热门歌曲',
+        },
+        'playlist_mincount': 95,
+    }
+
+    def _real_extract(self, url):
+        singer_id = self._match_id(url)
+
+        list_name = None
+        entries = []
+        for page_num in itertools.count(1):
+            webpage = self._download_webpage(
+                'http://www.kuwo.cn/mingxing/%s/music_%d.htm' % (singer_id, page_num),
+                singer_id, note='Download song list page #%d' % page_num,
+                errnote='Unable to get song list page #%d' % page_num)
+
+            if list_name is None:
+                list_name = self._html_search_regex(
+                    r'<h1>([^<>]+)<span>', webpage, 'list name')
+
+            entries.extend([
+                self.url_result("http://www.kuwo.cn/yinyue/%s/" % song_id, 'Kuwo', song_id)
+                for song_id in re.findall(
+                    r'<p class="m_name"><a href="http://www\.kuwo\.cn/yinyue/([0-9]+)/',
+                    webpage)
+            ])
+            if not re.search(r'<a href="[^"]+">下一页</a>', webpage):
+                break
+
+        return self.playlist_result(entries, singer_id, list_name)
+
+
+class KuwoCategoryIE(InfoExtractor):
+    IE_NAME = 'kuwo:category'
+    _VALID_URL = r'http://yinyue\.kuwo\.cn/yy/cinfo_(?P<id>[0-9]+?).htm'
+    _TEST = {
+        'url': 'http://yinyue.kuwo.cn/yy/cinfo_86375.htm',
+        'info_dict': {
+            'id': '86375',
+            'title': '八十年代精选',
+            'description': '这些都是属于八十年代的回忆！',
+        },
+        'playlist_count': 30,
+    }
+
+    def _real_extract(self, url):
+        category_id = self._match_id(url)
+        webpage = self._download_webpage(
+            url, category_id, note='Download category info',
+            errnote='Unable to get category info')
+
+        category_name = self._html_search_regex(
+            r'<h1 title="([^<>]+?)">[^<>]+?</h1>', webpage, 'category name')
+        
+        category_desc = re.sub(
+            r'^.+简介：', '', get_element_by_id("intro", webpage).strip())
+        
+        jsonm = self._parse_json(self._html_search_regex(
+            r'var jsonm = (\{.+?\});', webpage, 'category songs'), category_id)
+
+        entries = [
+            self.url_result(
+                "http://www.kuwo.cn/yinyue/%s/" % song['musicrid'],
+                'Kuwo', song['musicrid'])
+            for song in jsonm['musiclist']
+        ]
+        return self.playlist_result(entries, category_id, category_name, category_desc)
+
+
+class KuwoMvIE(KuwoIE):
+    IE_NAME = 'kuwo:mv'
+    _VALID_URL = r'http://www\.kuwo\.cn/mv/(?P<id>[0-9]+?)/'
+    _TESTS = [{
+        'url': 'http://www.kuwo.cn/mv/6480076/',
+        'info_dict': {
+            'id': '6480076',
+            'ext': 'mkv',
+            'title': '我们家MV',
+            'creator': '2PM',
+        },
+    }]
+    _FORMATS = KuwoIE._FORMATS + [
+        {'format': 'mkv', 'ext': 'mkv', 'preference': 250},
+        {'format': 'mp4', 'ext': 'mp4', 'preference': 200},
+    ]
+
+    def _real_extract(self, url):
+        song_id = self._match_id(url)
+        webpage = self._download_webpage(
+            url, song_id, note='Download mv detail info: %s' % song_id,
+            errnote='Unable to get mv detail info: %s' % song_id)
+
+        mobj = re.search(
+            r'<h1 title="(?P<song>.+?)">[^<>]+<span .+?title="(?P<singer>.+?)".+?>[^<>]+</span></h1>',
+            webpage)
+        if mobj:
+            song_name = mobj.group('song')
+            singer_name = mobj.group('singer')
+        else:
+            raise ExtractorError("Unable to find song or singer names")
+
+        formats = self._get_formats(song_id)
+
+        return {
+            'id': song_id,
+            'title': song_name,
+            'creator': singer_name,
+            'formats': formats,
+        }

From 29b809de68aeefb5e991c75929ed3d03fb40c1f1 Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Thu, 18 Jun 2015 15:52:04 +0800
Subject: [PATCH 1621/6219] [qqmusic] Fix album extraction

---
 youtube_dl/extractor/qqmusic.py | 37 ++++++++++++++++++++-------------
 1 file changed, 22 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index bafa81c21..d9a783f8a 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -163,31 +163,38 @@ class QQMusicAlbumIE(QQPlaylistBaseIE):
     IE_NAME = 'qqmusic:album'
     _VALID_URL = r'http://y.qq.com/#type=album&mid=(?P<id>[0-9A-Za-z]+)'
 
-    _TEST = {
-        'url': 'http://y.qq.com/#type=album&mid=000gXCTb2AhRR1&play=0',
+    _TESTS = [{
+        'url': 'http://y.qq.com/#type=album&mid=000gXCTb2AhRR1',
         'info_dict': {
             'id': '000gXCTb2AhRR1',
             'title': '我们都是这样长大的',
-            'description': 'md5:d216c55a2d4b3537fe4415b8767d74d6',
+            'description': 'md5:712f0cdbfc7e776820d08150e6df593d',
         },
         'playlist_count': 4,
-    }
+    }, {
+        'url': 'http://y.qq.com/#type=album&mid=002Y5a3b3AlCu3',
+        'info_dict': {
+            'id': '002Y5a3b3AlCu3',
+            'title': '그리고...',
+            'description': 'md5:b1d133b8c9bac8fed4e1a97df759f4cf',
+        },
+        'playlist_count': 8,
+    }]
 
     def _real_extract(self, url):
         mid = self._match_id(url)
 
-        album_page = self._download_webpage(
-            self.qq_static_url('album', mid), mid, 'Download album page')
+        album = self._download_json(
+            'http://i.y.qq.com/v8/fcg-bin/fcg_v8_album_info_cp.fcg?albummid=%s&format=json' % mid,
+            mid, 'Download album page')['data']
 
-        entries = self.get_entries_from_page(album_page)
-
-        album_name = self._html_search_regex(
-            r"albumname\s*:\s*'([^']+)',", album_page, 'album name',
-            default=None)
-
-        album_detail = self._html_search_regex(
-            r'<div class="album_detail close_detail">\s*<p>((?:[^<>]+(?:<br />)?)+)</p>',
-            album_page, 'album details', default=None)
+        entries = [
+            self.url_result(
+                'http://y.qq.com/#type=song&mid=' + song['songmid'], 'QQMusic', song['songmid']
+            ) for song in album['list']
+        ]
+        album_name = album['name']
+        album_detail = album.get('desc')
 
         return self.playlist_result(entries, mid, album_name, album_detail)
 

From 5e3915cbe3fabd2dbb633131b851b1158c0bba7b Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Thu, 18 Jun 2015 21:06:25 +0800
Subject: [PATCH 1622/6219] [qqmusic] Fix song extraction when certain formats
 are unavailable

---
 youtube_dl/extractor/qqmusic.py | 55 +++++++++++++++++++++++++++------
 1 file changed, 46 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index bafa81c21..7ddc4ca25 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -9,8 +9,13 @@ from .common import InfoExtractor
 from ..utils import (
     strip_jsonp,
     unescapeHTML,
+    HEADRequest,
+    ExtractorError,
+)
+from ..compat import (
+    compat_urllib_request,
+    compat_HTTPError,
 )
-from ..compat import compat_urllib_request
 
 
 class QQMusicIE(InfoExtractor):
@@ -26,6 +31,20 @@ class QQMusicIE(InfoExtractor):
             'upload_date': '20141227',
             'creator': '林俊杰',
             'description': 'md5:d327722d0361576fde558f1ac68a7065',
+            'thumbnail': 'http://i.gtimg.cn/music/photo/mid_album_500/7/p/001IV22P1RDX7p.jpg',
+        }
+    }, {
+        'note': 'There is no mp3-320 version of this song.',
+        'url': 'http://y.qq.com/#type=song&mid=004MsGEo3DdNxV',
+        'md5': 'fa3926f0c585cda0af8fa4f796482e3e',
+        'info_dict': {
+            'id': '004MsGEo3DdNxV',
+            'ext': 'mp3',
+            'title': '如果',
+            'upload_date': '20050626',
+            'creator': '李季美',
+            'description': 'md5:46857d5ed62bc4ba84607a805dccf437',
+            'thumbnail': 'http://i.gtimg.cn/music/photo/mid_album_500/r/Q/0042owYj46IxrQ.jpg',
         }
     }]
 
@@ -68,6 +87,13 @@ class QQMusicIE(InfoExtractor):
         if lrc_content:
             lrc_content = lrc_content.replace('\\n', '\n')
 
+        thumbnail_url = None
+        albummid = self._search_regex(
+            [r'albummid:\'([0-9a-zA-Z]+)\'', r'"albummid":"([0-9a-zA-Z]+)"'], detail_info_page, 'album mid', default=None)
+        if albummid:
+            thumbnail_url = "http://i.gtimg.cn/music/photo/mid_album_500/%s/%s/%s.jpg" \
+                            % (albummid[-2:-1], albummid[-1], albummid)
+
         guid = self.m_r_get_ruin()
 
         vkey = self._download_json(
@@ -77,14 +103,24 @@ class QQMusicIE(InfoExtractor):
 
         formats = []
         for format_id, details in self._FORMATS.items():
-            formats.append({
-                'url': 'http://cc.stream.qqmusic.qq.com/%s%s.%s?vkey=%s&guid=%s&fromtag=0'
-                       % (details['prefix'], mid, details['ext'], vkey, guid),
-                'format': format_id,
-                'format_id': format_id,
-                'preference': details['preference'],
-                'abr': details.get('abr'),
-            })
+            video_url = 'http://cc.stream.qqmusic.qq.com/%s%s.%s?vkey=%s&guid=%s&fromtag=0' \
+                        % (details['prefix'], mid, details['ext'], vkey, guid)
+            req = HEADRequest(video_url)
+            try:
+                res = self._request_webpage(
+                    req, mid, note='Testing %s video URL' % format_id, fatal=False)
+            except ExtractorError as e:
+                if isinstance(e.cause, compat_HTTPError) and e.cause.code in [400, 404]:
+                    self.report_warning('Invalid %s video URL' % format_id, mid)
+            else:
+                if res:
+                    formats.append({
+                        'url': video_url,
+                        'format': format_id,
+                        'format_id': format_id,
+                        'preference': details['preference'],
+                        'abr': details.get('abr'),
+                    })
         self._sort_formats(formats)
 
         return {
@@ -94,6 +130,7 @@ class QQMusicIE(InfoExtractor):
             'upload_date': publish_time,
             'creator': singer,
             'description': lrc_content,
+            'thumbnail': thumbnail_url,
         }
 
 

From 0392ac98d2c5c5a6fd2ab51c51096f82312a287c Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Thu, 18 Jun 2015 21:13:03 +0800
Subject: [PATCH 1623/6219] [qqmusic] Fix code formatting

---
 youtube_dl/extractor/qqmusic.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index 7ddc4ca25..5a18191bc 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -89,7 +89,8 @@ class QQMusicIE(InfoExtractor):
 
         thumbnail_url = None
         albummid = self._search_regex(
-            [r'albummid:\'([0-9a-zA-Z]+)\'', r'"albummid":"([0-9a-zA-Z]+)"'], detail_info_page, 'album mid', default=None)
+            [r'albummid:\'([0-9a-zA-Z]+)\'', r'"albummid":"([0-9a-zA-Z]+)"'],
+            detail_info_page, 'album mid', default=None)
         if albummid:
             thumbnail_url = "http://i.gtimg.cn/music/photo/mid_album_500/%s/%s/%s.jpg" \
                             % (albummid[-2:-1], albummid[-1], albummid)

From 974a6146fe71029a8dedf670fd72501da5ec3809 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 18 Jun 2015 21:40:45 +0800
Subject: [PATCH 1624/6219] [bbccouk] Fix error handling

---
 youtube_dl/extractor/bbccouk.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbccouk.py b/youtube_dl/extractor/bbccouk.py
index 249bc6bbd..4b25b8c38 100644
--- a/youtube_dl/extractor/bbccouk.py
+++ b/youtube_dl/extractor/bbccouk.py
@@ -267,7 +267,7 @@ class BBCCoUkIE(InfoExtractor):
                 programme_id, 'Downloading media selection XML')
         except ExtractorError as ee:
             if isinstance(ee.cause, compat_HTTPError) and ee.cause.code == 403:
-                media_selection = xml.etree.ElementTree.fromstring(ee.cause.read().encode('utf-8'))
+                media_selection = xml.etree.ElementTree.fromstring(ee.cause.read().decode('utf-8'))
             else:
                 raise
 

From 25fa8d66e653d443522bf10140106f88711d1d66 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 18 Jun 2015 22:00:13 +0800
Subject: [PATCH 1625/6219] [bbccouk] Fix description (closes #6006)

---
 youtube_dl/extractor/bbccouk.py | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbccouk.py b/youtube_dl/extractor/bbccouk.py
index 4b25b8c38..0305f88b5 100644
--- a/youtube_dl/extractor/bbccouk.py
+++ b/youtube_dl/extractor/bbccouk.py
@@ -129,6 +129,20 @@ class BBCCoUkIE(InfoExtractor):
                 'skip_download': True,
             },
             'skip': 'geolocation',
+        }, {
+            'url': 'http://www.bbc.co.uk/iplayer/episode/b05zmgwn/royal-academy-summer-exhibition',
+            'info_dict': {
+                'id': 'b05zmgw1',
+                'ext': 'flv',
+                'description': 'Kirsty Wark and Morgan Quaintance visit the Royal Academy as it prepares for its annual artistic extravaganza, meeting people who have come together to make the show unique.',
+                'title': 'Royal Academy Summer Exhibition',
+                'duration': 3540,
+            },
+            'params': {
+                # rtmp download
+                'skip_download': True,
+            },
+            'skip': 'geolocation',
         }, {
             'url': 'http://www.bbc.co.uk/iplayer/playlist/p01dvks4',
             'only_matching': True,
@@ -362,7 +376,7 @@ class BBCCoUkIE(InfoExtractor):
             formats, subtitles = self._download_media_selector(programme_id)
             title = self._og_search_title(webpage)
             description = self._search_regex(
-                r'<p class="medium-description">([^<]+)</p>',
+                r'<p class="[^"]*medium-description[^"]*">([^<]+)</p>',
                 webpage, 'description', fatal=False)
         else:
             programme_id, title, description, duration, formats, subtitles = self._download_playlist(group_id)

From c33a8639a7c1f783297f29282b48fceec9a0b84a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 18 Jun 2015 22:05:02 +0800
Subject: [PATCH 1626/6219] [postprocessor/embedthumbnail] Skipping if
 thumbnail file missing

Fixes #5996
---
 youtube_dl/postprocessor/embedthumbnail.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/postprocessor/embedthumbnail.py b/youtube_dl/postprocessor/embedthumbnail.py
index 774494efd..e19dbf73d 100644
--- a/youtube_dl/postprocessor/embedthumbnail.py
+++ b/youtube_dl/postprocessor/embedthumbnail.py
@@ -35,6 +35,11 @@ class EmbedThumbnailPP(FFmpegPostProcessor):
 
         thumbnail_filename = info['thumbnails'][-1]['filename']
 
+        if not os.path.exists(encodeFilename(thumbnail_filename)):
+            self._downloader.report_warning(
+                'Skipping embedding the thumbnail because the file is missing.')
+            return [], info
+
         if info['ext'] == 'mp3':
             options = [
                 '-c', 'copy', '-map', '0', '-map', '1',

From 4d58b24c15ea0efc699a7ad7ee468245029da4e3 Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Thu, 18 Jun 2015 23:09:04 +0800
Subject: [PATCH 1627/6219] [qqmusic] Use _check_formats instead

---
 youtube_dl/extractor/qqmusic.py | 34 ++++++++++-----------------------
 1 file changed, 10 insertions(+), 24 deletions(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index 5a18191bc..7183c2bb1 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -9,13 +9,8 @@ from .common import InfoExtractor
 from ..utils import (
     strip_jsonp,
     unescapeHTML,
-    HEADRequest,
-    ExtractorError,
-)
-from ..compat import (
-    compat_urllib_request,
-    compat_HTTPError,
 )
+from ..compat import compat_urllib_request
 
 
 class QQMusicIE(InfoExtractor):
@@ -104,24 +99,15 @@ class QQMusicIE(InfoExtractor):
 
         formats = []
         for format_id, details in self._FORMATS.items():
-            video_url = 'http://cc.stream.qqmusic.qq.com/%s%s.%s?vkey=%s&guid=%s&fromtag=0' \
-                        % (details['prefix'], mid, details['ext'], vkey, guid)
-            req = HEADRequest(video_url)
-            try:
-                res = self._request_webpage(
-                    req, mid, note='Testing %s video URL' % format_id, fatal=False)
-            except ExtractorError as e:
-                if isinstance(e.cause, compat_HTTPError) and e.cause.code in [400, 404]:
-                    self.report_warning('Invalid %s video URL' % format_id, mid)
-            else:
-                if res:
-                    formats.append({
-                        'url': video_url,
-                        'format': format_id,
-                        'format_id': format_id,
-                        'preference': details['preference'],
-                        'abr': details.get('abr'),
-                    })
+            formats.append({
+                'url': 'http://cc.stream.qqmusic.qq.com/%s%s.%s?vkey=%s&guid=%s&fromtag=0'
+                       % (details['prefix'], mid, details['ext'], vkey, guid),
+                'format': format_id,
+                'format_id': format_id,
+                'preference': details['preference'],
+                'abr': details.get('abr'),
+            })
+        self._check_formats(formats, mid)
         self._sort_formats(formats)
 
         return {

From accf79b1079302a339a3565e98ecfea5b33b5528 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 18 Jun 2015 21:17:26 +0600
Subject: [PATCH 1628/6219] [viki] Add support for authentication (Closes
 #6005)

---
 youtube_dl/extractor/viki.py | 34 +++++++++++++++++++++++++++++-----
 1 file changed, 29 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index 7f2fb1ca8..52d10d242 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -1,5 +1,7 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
+import json
 import time
 import hmac
 import hashlib
@@ -11,6 +13,7 @@ from ..utils import (
     parse_age_limit,
     parse_iso8601,
 )
+from ..compat import compat_urllib_request
 from .common import InfoExtractor
 
 
@@ -23,7 +26,9 @@ class VikiBaseIE(InfoExtractor):
     _APP_VERSION = '2.2.5.1428709186'
     _APP_SECRET = '-$iJ}@p7!G@SyU/je1bEyWg}upLu-6V6-Lg9VD(]siH,r.,m-r|ulZ,U4LC/SeR)'
 
-    def _prepare_call(self, path, timestamp=None):
+    _NETRC_MACHINE = 'viki'
+
+    def _prepare_call(self, path, timestamp=None, post_data=None):
         path += '?' if '?' not in path else '&'
         if not timestamp:
             timestamp = int(time.time())
@@ -33,17 +38,19 @@ class VikiBaseIE(InfoExtractor):
             query.encode('ascii'),
             hashlib.sha1
         ).hexdigest()
-        return self._API_URL_TEMPLATE % (query, sig)
+        url = self._API_URL_TEMPLATE % (query, sig)
+        return compat_urllib_request.Request(
+            url, json.dumps(post_data).encode('utf-8')) if post_data else url
 
-    def _call_api(self, path, video_id, note, timestamp=None):
+    def _call_api(self, path, video_id, note, timestamp=None, post_data=None):
         resp = self._download_json(
-            self._prepare_call(path, timestamp), video_id, note)
+            self._prepare_call(path, timestamp, post_data), video_id, note)
 
         error = resp.get('error')
         if error:
             if error == 'invalid timestamp':
                 resp = self._download_json(
-                    self._prepare_call(path, int(resp['current_timestamp'])),
+                    self._prepare_call(path, int(resp['current_timestamp']), post_data),
                     video_id, '%s (retry)' % note)
                 error = resp.get('error')
             if error:
@@ -56,6 +63,23 @@ class VikiBaseIE(InfoExtractor):
             '%s returned error: %s' % (self.IE_NAME, error),
             expected=True)
 
+    def _real_initialize(self):
+        self._login()
+
+    def _login(self):
+        (username, password) = self._get_login_info()
+        if username is None:
+            return
+
+        login_form = {
+            'login_id': username,
+            'password': password,
+        }
+
+        self._call_api(
+            'sessions.json', None,
+            'Logging in as %s' % username, post_data=login_form)
+
 
 class VikiIE(VikiBaseIE):
     IE_NAME = 'viki'

From 360075e28a4c009babb62e0ed8141ed5244cb3f5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 18 Jun 2015 22:26:17 +0600
Subject: [PATCH 1629/6219] [pornhub] Extend _VALID_URL (Closes #6019)

---
 youtube_dl/extractor/pornhub.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index 8565d7551..8172bc997 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -19,8 +19,8 @@ from ..aes import (
 
 
 class PornHubIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?pornhub\.com/(?:view_video\.php\?viewkey=|embed/)(?P<id>[0-9a-f]+)'
-    _TEST = {
+    _VALID_URL = r'https?://(?:www\.)?pornhub\.com/(?:view_video\.php\?viewkey=|embed/)(?P<id>[0-9a-z]+)'
+    _TESTS = [{
         'url': 'http://www.pornhub.com/view_video.php?viewkey=648719015',
         'md5': '882f488fa1f0026f023f33576004a2ed',
         'info_dict': {
@@ -30,7 +30,10 @@ class PornHubIE(InfoExtractor):
             "title": "Seductive Indian beauty strips down and fingers her pink pussy",
             "age_limit": 18
         }
-    }
+    }, {
+        'url': 'http://www.pornhub.com/view_video.php?viewkey=ph557bbb6676d2d',
+        'only_matching': True,
+    }]
 
     @classmethod
     def _extract_url(cls, webpage):

From 9e96dc8b3561c1e6e62ce6a34efba485e5e49054 Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Fri, 19 Jun 2015 01:36:59 -0500
Subject: [PATCH 1630/6219] Support BBC News (bbc.com/news)

---
 docs/supportedsites.md           |   1 +
 youtube_dl/extractor/__init__.py |   1 +
 youtube_dl/extractor/bbcnews.py  | 162 +++++++++++++++++++++++++++++++
 3 files changed, 164 insertions(+)
 create mode 100644 youtube_dl/extractor/bbcnews.py

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 220e52b98..d4ccbbd3a 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -50,6 +50,7 @@
  - **Bandcamp**
  - **Bandcamp:album**
  - **bbc.co.uk**: BBC iPlayer
+ - **bbc.com**: BBC news videos
  - **BeatportPro**
  - **Beeg**
  - **BehindKink**
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 6fdaf90b2..51d2d20e9 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -36,6 +36,7 @@ from .baidu import BaiduVideoIE
 from .bambuser import BambuserIE, BambuserChannelIE
 from .bandcamp import BandcampIE, BandcampAlbumIE
 from .bbccouk import BBCCoUkIE
+from .bbcnews import BBCNewsIE
 from .beeg import BeegIE
 from .behindkink import BehindKinkIE
 from .beatportpro import BeatportProIE
diff --git a/youtube_dl/extractor/bbcnews.py b/youtube_dl/extractor/bbcnews.py
new file mode 100644
index 000000000..b10e30a81
--- /dev/null
+++ b/youtube_dl/extractor/bbcnews.py
@@ -0,0 +1,162 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    int_or_none,
+)
+from ..compat import compat_HTTPError
+import re
+from .bbccouk import BBCCoUkIE
+
+class BBCNewsIE(BBCCoUkIE):
+    IE_NAME = 'bbc.com'
+    IE_DESC = 'BBC news'
+    _VALID_URL = r'https?://(?:www\.)?(?:bbc\.co\.uk|bbc\.com)/news/(?P<id>[^/]+)'
+
+    _TESTS = [{
+        'url': 'http://www.bbc.com/news/world-europe-32668511',
+        'info_dict': {
+            'id': 'world-europe-32668511',
+            'title': 'Russia stages massive WW2 parade despite Western boycott',
+        },
+        'playlist_count': 2,
+    },{
+        'url': 'http://www.bbc.com/news/business-28299555',
+        'info_dict': {
+            'id': 'business-28299555',
+            'title': 'Farnborough Airshow: Video highlights',
+        },
+        'playlist_count': 9,
+    },{
+        'url': 'http://www.bbc.com/news/world-europe-32041533',
+        'note': 'Video',
+        'info_dict': {
+            'id': 'p02mprgb',
+            'ext': 'mp4',
+            'title': 'Aerial footage showed the site of the crash in the Alps - courtesy BFM TV',
+            'description': 'Germanwings plane crash site in aerial video - Aerial footage showed the site of the crash in the Alps - courtesy BFM TV',
+            'duration': 47,
+        },
+        'params': {
+            'skip_download': True,
+        }
+    }]
+
+    def _duration_str2int(self, str):
+        if not str:
+            return None
+        ret = re.match(r'^\d+$', str)
+        if ret:
+            return int(ret.group(0))
+        ret = re.match(r'PT((?P<h>\d+)H)?((?P<m>\d+)M)?(?P<s>\d+)S$', str)
+        if ret:
+            total=int(ret.group('s'))
+            if ret.group('m'):
+                total+=(int(ret.group('m'))*60)
+            if ret.group('h'):
+                total+=(int(ret.group('h'))*3600)
+            return total
+        return None
+
+    def _download_media_selector(self, programme_id):
+        # bbc news uses http://open.live.bbc.co.uk/mediaselector/4/mtis/stream/ not
+        # http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/pc/vpid/
+        # Could add third urlspec arg to BBCCoUkIE._download_media_selector instead of duplicating it
+
+        try:
+            media_selection = self._download_xml(
+               'http://open.live.bbc.co.uk/mediaselector/4/mtis/stream/%s' % programme_id,
+                programme_id, 'Downloading media selection XML')
+        except ExtractorError as ee:
+            if isinstance(ee.cause, compat_HTTPError) and ee.cause.code == 403:
+                media_selection = xml.etree.ElementTree.fromstring(ee.cause.read().encode('utf-8'))
+            else:
+                raise
+        formats = []
+        subtitles = None
+
+        for media in self._extract_medias(media_selection):
+            kind = media.get('kind')
+            if kind == 'audio':
+                formats.extend(self._extract_audio(media, programme_id))
+            elif kind == 'video':
+                formats.extend(self._extract_video(media, programme_id))
+            elif kind == 'captions':
+                subtitles = self.extract_subtitles(media, programme_id)
+
+        formats = []
+        subtitles = None
+
+        for media in self._extract_medias(media_selection):
+            kind = media.get('kind')
+            if kind == 'audio':
+                formats.extend(self._extract_audio(media, programme_id))
+            elif kind == 'video':
+                formats.extend(self._extract_video(media, programme_id))
+            elif kind == 'captions':
+                subtitles = self.extract_subtitles(media, programme_id)
+
+        return formats, subtitles
+
+    def _real_extract(self, url):
+        list_id = self._match_id(url)
+        webpage = self._download_webpage(url, list_id)
+
+        list_title = self._html_search_regex(r'<title>(.*?)(?:\s*-\s*BBC News)?</title>', webpage, 'list title')
+
+        pubdate = self._html_search_regex(r'"datePublished":\s*"(\d+-\d+-\d+)', webpage, 'date', default=None)
+        if pubdate:
+           pubdate = pubdate.replace('-','')
+
+        ret = []
+        # works with bbc.com/news/something-something-123456 articles
+        matches = re.findall(r"data-media-meta='({[^']+})'", webpage)
+        if not matches:
+           # stubbornly generic extractor for {json with "image":{allvideoshavethis},etc}
+           # in http://www.bbc.com/news/video_and_audio/international
+           matches = re.findall(r"({[^{}]+image\":{[^}]+}[^}]+})", webpage)
+        if not matches:
+           raise ExtractorError('No video found', expected=True)
+
+        for ent in matches:
+            jent = self._parse_json(ent,list_id)
+
+            programme_id = jent.get('externalId',None)
+            xml_url = jent.get('href', None)
+
+            title = jent['caption']
+            duration = self._duration_str2int(jent.get('duration',None))
+            description = list_title + ' - ' + jent.get('caption','')
+            thumbnail = None
+            if jent.has_key('image'):
+               thumbnail=jent['image'].get('href',None)
+
+            if programme_id:
+               formats, subtitles = self._download_media_selector(programme_id)
+            elif xml_url:
+               # Cheap fallback
+               # http://playlists.bbc.co.uk/news/(list_id)[ABC..]/playlist.sxml
+               xml = self._download_webpage(xml_url, programme_id, 'Downloading playlist.sxml for externalId (fallback)')
+               programme_id = self._search_regex(r'<mediator [^>]*identifier="(.+?)"', xml, 'playlist.sxml (externalId fallback)')
+               formats, subtitles = self._download_media_selector(programme_id)
+            else:
+               raise ExtractorError('data-media-meta entry has no externalId or href value.')
+               
+            self._sort_formats(formats)
+
+            ret.append( {
+                'id': programme_id,
+                'uploader': 'BBC News',
+                'upload_date': pubdate,
+                'title': title,
+                'description': description,
+                'thumbnail': thumbnail,
+                'duration': duration,
+                'formats': formats,
+                'subtitles': subtitles,
+            } )
+
+        if len(ret) > 0:
+           return self.playlist_result(ret, list_id, list_title)
+        raise ExtractorError('No video found', expected=True)

From a8b081a0523c412fd4e01d5cddec7ae382c4793e Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Fri, 19 Jun 2015 01:52:25 -0500
Subject: [PATCH 1631/6219] BBCNewsIE: eliminate redundant function.
 BBCCoUkIE._download_media_selector: use class variable instead of hardcoded
 string for mediaselector_url template.

---
 youtube_dl/extractor/bbccouk.py |  4 +++-
 youtube_dl/extractor/bbcnews.py | 42 ++-------------------------------
 2 files changed, 5 insertions(+), 41 deletions(-)

diff --git a/youtube_dl/extractor/bbccouk.py b/youtube_dl/extractor/bbccouk.py
index 0305f88b5..dcc5fc2fa 100644
--- a/youtube_dl/extractor/bbccouk.py
+++ b/youtube_dl/extractor/bbccouk.py
@@ -15,6 +15,8 @@ class BBCCoUkIE(InfoExtractor):
     IE_DESC = 'BBC iPlayer'
     _VALID_URL = r'https?://(?:www\.)?bbc\.co\.uk/(?:(?:(?:programmes|iplayer(?:/[^/]+)?/(?:episode|playlist))/)|music/clips[/#])(?P<id>[\da-z]{8})'
 
+    mediaselector_url = 'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/pc/vpid/%s'
+
     _TESTS = [
         {
             'url': 'http://www.bbc.co.uk/programmes/b039g8p7',
@@ -277,7 +279,7 @@ class BBCCoUkIE(InfoExtractor):
     def _download_media_selector(self, programme_id):
         try:
             media_selection = self._download_xml(
-                'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/pc/vpid/%s' % programme_id,
+                self.mediaselector_url % programme_id,
                 programme_id, 'Downloading media selection XML')
         except ExtractorError as ee:
             if isinstance(ee.cause, compat_HTTPError) and ee.cause.code == 403:
diff --git a/youtube_dl/extractor/bbcnews.py b/youtube_dl/extractor/bbcnews.py
index b10e30a81..9bb8d42e6 100644
--- a/youtube_dl/extractor/bbcnews.py
+++ b/youtube_dl/extractor/bbcnews.py
@@ -14,6 +14,8 @@ class BBCNewsIE(BBCCoUkIE):
     IE_DESC = 'BBC news'
     _VALID_URL = r'https?://(?:www\.)?(?:bbc\.co\.uk|bbc\.com)/news/(?P<id>[^/]+)'
 
+    mediaselector_url = 'http://open.live.bbc.co.uk/mediaselector/4/mtis/stream/%s'
+
     _TESTS = [{
         'url': 'http://www.bbc.com/news/world-europe-32668511',
         'info_dict': {
@@ -59,46 +61,6 @@ class BBCNewsIE(BBCCoUkIE):
             return total
         return None
 
-    def _download_media_selector(self, programme_id):
-        # bbc news uses http://open.live.bbc.co.uk/mediaselector/4/mtis/stream/ not
-        # http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/pc/vpid/
-        # Could add third urlspec arg to BBCCoUkIE._download_media_selector instead of duplicating it
-
-        try:
-            media_selection = self._download_xml(
-               'http://open.live.bbc.co.uk/mediaselector/4/mtis/stream/%s' % programme_id,
-                programme_id, 'Downloading media selection XML')
-        except ExtractorError as ee:
-            if isinstance(ee.cause, compat_HTTPError) and ee.cause.code == 403:
-                media_selection = xml.etree.ElementTree.fromstring(ee.cause.read().encode('utf-8'))
-            else:
-                raise
-        formats = []
-        subtitles = None
-
-        for media in self._extract_medias(media_selection):
-            kind = media.get('kind')
-            if kind == 'audio':
-                formats.extend(self._extract_audio(media, programme_id))
-            elif kind == 'video':
-                formats.extend(self._extract_video(media, programme_id))
-            elif kind == 'captions':
-                subtitles = self.extract_subtitles(media, programme_id)
-
-        formats = []
-        subtitles = None
-
-        for media in self._extract_medias(media_selection):
-            kind = media.get('kind')
-            if kind == 'audio':
-                formats.extend(self._extract_audio(media, programme_id))
-            elif kind == 'video':
-                formats.extend(self._extract_video(media, programme_id))
-            elif kind == 'captions':
-                subtitles = self.extract_subtitles(media, programme_id)
-
-        return formats, subtitles
-
     def _real_extract(self, url):
         list_id = self._match_id(url)
         webpage = self._download_webpage(url, list_id)

From d5552a3477a0970f4aaaa746ce07c816267bb9cf Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Fri, 19 Jun 2015 06:25:50 -0500
Subject: [PATCH 1632/6219] bbcnews: Switch to parse_duration, revert change to
 docs/supportedsites.md

---
 docs/supportedsites.md          |  1 -
 youtube_dl/extractor/bbcnews.py | 19 ++-----------------
 2 files changed, 2 insertions(+), 18 deletions(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index d4ccbbd3a..220e52b98 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -50,7 +50,6 @@
  - **Bandcamp**
  - **Bandcamp:album**
  - **bbc.co.uk**: BBC iPlayer
- - **bbc.com**: BBC news videos
  - **BeatportPro**
  - **Beeg**
  - **BehindKink**
diff --git a/youtube_dl/extractor/bbcnews.py b/youtube_dl/extractor/bbcnews.py
index 9bb8d42e6..fd4a5e38f 100644
--- a/youtube_dl/extractor/bbcnews.py
+++ b/youtube_dl/extractor/bbcnews.py
@@ -3,6 +3,7 @@ from __future__ import unicode_literals
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
+    parse_duration,
     int_or_none,
 )
 from ..compat import compat_HTTPError
@@ -45,22 +46,6 @@ class BBCNewsIE(BBCCoUkIE):
         }
     }]
 
-    def _duration_str2int(self, str):
-        if not str:
-            return None
-        ret = re.match(r'^\d+$', str)
-        if ret:
-            return int(ret.group(0))
-        ret = re.match(r'PT((?P<h>\d+)H)?((?P<m>\d+)M)?(?P<s>\d+)S$', str)
-        if ret:
-            total=int(ret.group('s'))
-            if ret.group('m'):
-                total+=(int(ret.group('m'))*60)
-            if ret.group('h'):
-                total+=(int(ret.group('h'))*3600)
-            return total
-        return None
-
     def _real_extract(self, url):
         list_id = self._match_id(url)
         webpage = self._download_webpage(url, list_id)
@@ -88,7 +73,7 @@ class BBCNewsIE(BBCCoUkIE):
             xml_url = jent.get('href', None)
 
             title = jent['caption']
-            duration = self._duration_str2int(jent.get('duration',None))
+            duration = parse_duration(jent.get('duration',None))
             description = list_title + ' - ' + jent.get('caption','')
             thumbnail = None
             if jent.has_key('image'):

From c9bebed294dd29d9188265c8f7bfb0e1b43406ff Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 19 Jun 2015 20:52:44 +0600
Subject: [PATCH 1633/6219] [youtube] Add itag 59 and 78 (Closes #5979)

---
 youtube_dl/extractor/youtube.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 9e2671192..a3da56c14 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -234,6 +234,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         '44': {'ext': 'webm', 'width': 854, 'height': 480},
         '45': {'ext': 'webm', 'width': 1280, 'height': 720},
         '46': {'ext': 'webm', 'width': 1920, 'height': 1080},
+        '59': {'ext': 'mp4', 'width': 854, 'height': 480},
+        '78': {'ext': 'mp4', 'width': 854, 'height': 480},
 
 
         # 3d videos

From cbcd1a5474dd8b39e68b0d2bbc493701c655a2d9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 19 Jun 2015 21:57:31 +0600
Subject: [PATCH 1634/6219] [dramafever] Add support for authentication (Closes
 #6017)

---
 youtube_dl/extractor/dramafever.py | 38 ++++++++++++++++++++++++++++--
 1 file changed, 36 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/dramafever.py b/youtube_dl/extractor/dramafever.py
index a34aad486..cfbcddcef 100644
--- a/youtube_dl/extractor/dramafever.py
+++ b/youtube_dl/extractor/dramafever.py
@@ -6,6 +6,8 @@ import itertools
 from .common import InfoExtractor
 from ..compat import (
     compat_HTTPError,
+    compat_urllib_parse,
+    compat_urllib_request,
     compat_urlparse,
 )
 from ..utils import (
@@ -17,7 +19,39 @@ from ..utils import (
 )
 
 
-class DramaFeverIE(InfoExtractor):
+class DramaFeverBaseIE(InfoExtractor):
+    _LOGIN_URL = 'https://www.dramafever.com/accounts/login/'
+    _NETRC_MACHINE = 'dramafever'
+
+    def _real_initialize(self):
+        self._login()
+
+    def _login(self):
+        (username, password) = self._get_login_info()
+        if username is None:
+            return
+
+        login_form = {
+            'username': username,
+            'password': password,
+        }
+
+        request = compat_urllib_request.Request(
+            self._LOGIN_URL, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
+        response = self._download_webpage(
+            request, None, 'Logging in as %s' % username)
+
+        if all(logout_pattern not in response
+               for logout_pattern in ['href="/accounts/logout/"', '>Log out<']):
+            error = self._html_search_regex(
+                r'(?s)class="hidden-xs prompt"[^>]*>(.+?)<',
+                response, 'error message', default=None)
+            if error:
+                raise ExtractorError('Unable to login: %s' % error, expected=True)
+            raise ExtractorError('Unable to log in')
+
+
+class DramaFeverIE(DramaFeverBaseIE):
     IE_NAME = 'dramafever'
     _VALID_URL = r'https?://(?:www\.)?dramafever\.com/drama/(?P<id>[0-9]+/[0-9]+)(?:/|$)'
     _TEST = {
@@ -97,7 +131,7 @@ class DramaFeverIE(InfoExtractor):
         }
 
 
-class DramaFeverSeriesIE(InfoExtractor):
+class DramaFeverSeriesIE(DramaFeverBaseIE):
     IE_NAME = 'dramafever:series'
     _VALID_URL = r'https?://(?:www\.)?dramafever\.com/drama/(?P<id>[0-9]+)(?:/(?:(?!\d+(?:/|$)).+)?)?$'
     _TESTS = [{

From 10464af5d1d03a3461286a601ae7db91c5a8141c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 19 Jun 2015 22:02:07 +0600
Subject: [PATCH 1635/6219] [dramafever:series] Fix extraction while
 authenticated

---
 youtube_dl/extractor/dramafever.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/dramafever.py b/youtube_dl/extractor/dramafever.py
index cfbcddcef..ca41a3abf 100644
--- a/youtube_dl/extractor/dramafever.py
+++ b/youtube_dl/extractor/dramafever.py
@@ -185,8 +185,11 @@ class DramaFeverSeriesIE(DramaFeverBaseIE):
                 % (consumer_secret, series_id, self._PAGE_SIZE, page_num),
                 series_id, 'Downloading episodes JSON page #%d' % page_num)
             for episode in episodes.get('value', []):
+                episode_url = episode.get('episode_url')
+                if not episode_url:
+                    continue
                 entries.append(self.url_result(
-                    compat_urlparse.urljoin(url, episode['episode_url']),
+                    compat_urlparse.urljoin(url, episode_url),
                     'DramaFever', episode.get('guid')))
             if page_num == episodes['num_pages']:
                 break

From 385c3e5e91680dcc6573f05e6b30fdf45048503e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 20 Jun 2015 00:10:08 +0600
Subject: [PATCH 1636/6219] [pinkbike] Improve and simplify

---
 youtube_dl/extractor/pinkbike.py | 88 ++++++++++++++++++--------------
 1 file changed, 49 insertions(+), 39 deletions(-)

diff --git a/youtube_dl/extractor/pinkbike.py b/youtube_dl/extractor/pinkbike.py
index 45c0b1377..745433b48 100644
--- a/youtube_dl/extractor/pinkbike.py
+++ b/youtube_dl/extractor/pinkbike.py
@@ -7,12 +7,14 @@ from .common import InfoExtractor
 from ..utils import (
     int_or_none,
     remove_end,
-    remove_start
+    remove_start,
+    str_to_int,
+    unified_strdate,
 )
 
 
 class PinkbikeIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?pinkbike\.com/video/(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://(?:(?:www\.)?pinkbike\.com/video/|es\.pinkbike\.org/i/kvid/kvid-y5\.swf\?id=)(?P<id>[0-9]+)'
     _TESTS = [{
         'url': 'http://www.pinkbike.com/video/402811/',
         'md5': '4814b8ca7651034cd87e3361d5c2155a',
@@ -20,67 +22,75 @@ class PinkbikeIE(InfoExtractor):
             'id': '402811',
             'ext': 'mp4',
             'title': 'Brandon Semenuk - RAW 100',
-            'thumbnail': 're:^https?://.*\.jpg$',
-            'location': 'Victoria, British Columbia, Canada',
-            'uploader_id': 'revelco',
-            'upload_date': '20150406',
             'description': 'Official release: www.redbull.ca/rupertwalker',
-            'duration': 100
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 100,
+            'upload_date': '20150406',
+            'uploader': 'revelco',
+            'location': 'Victoria, British Columbia, Canada',
+            'view_count': int,
+            'comment_count': int,
         }
     }, {
-        'url': 'http://www.pinkbike.com/video/406629/',
-        'md5': 'c7a3e19a2bd5cde5a1cda6b2b46caa74',
-        'info_dict': {
-            'id': '406629',
-            'ext': 'mp4',
-            'title': 'Chromag: Reece Wallace in Utah',
-            'thumbnail': 're:^https?://.*\.jpg$',
-            'location': 'Whistler, British Columbia, Canada',
-            'uploader_id': 'Chromagbikes',
-            'upload_date': '20150505',
-            'description': 'Reece Wallace shredding Virgin, Utah. Video by Virtu Media.',
-            'duration': 180
-        }
+        'url': 'http://es.pinkbike.org/i/kvid/kvid-y5.swf?id=406629',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
 
-        title = self._html_search_regex(r'<title>(.*?)</title>', webpage, 'title')
-        title = remove_end(title, ' Video - Pinkbike')
+        webpage = self._download_webpage(
+            'http://www.pinkbike.com/video/%s' % video_id, video_id)
 
-        description = self._html_search_meta('description', webpage, 'description')
-        description = remove_start(description, title + '. ')
+        formats = []
+        for _, format_id, src in re.findall(
+            r'data-quality=((?:\\)?["\'])(.+?)\1[^>]+src=\1(.+?)\1', webpage):
+            height = int_or_none(self._search_regex(
+                r'^(\d+)[pP]$', format_id, 'height', default=None))
+            formats.append({
+                'url': src,
+                'format_id': format_id,
+                'height': height,
+            })
+        self._sort_formats(formats)
 
+        title = remove_end(self._og_search_title(webpage), ' Video - Pinkbike')
+        description = self._html_search_regex(
+            r'(?s)id="media-description"[^>]*>(.+?)<',
+            webpage, 'description', default=None) or remove_start(
+            self._og_search_description(webpage), title + '. ')
+        thumbnail = self._og_search_thumbnail(webpage)
         duration = int_or_none(self._html_search_meta(
             'video:duration', webpage, 'duration'))
 
-        uploader_id = self._html_search_regex(r'un:\s*"(.*?)"', webpage, 'uploader_id')
-
-        upload_date = self._html_search_regex(
-            r'class="fullTime"\s*title="([0-9]{4}(?:-[0-9]{2}){2})"',
-            webpage, 'upload_date')
-        upload_date = upload_date.replace('-', '')
+        uploader = self._search_regex(
+            r'un:\s*"([^"]+)"', webpage, 'uploader', fatal=False)
+        upload_date = unified_strdate(self._search_regex(
+            r'class="fullTime"[^>]+title="([^"]+)"',
+            webpage, 'upload date', fatal=False))
 
         location = self._html_search_regex(
-            r'<dt>Location</dt>\n?\s*<dd>\n?(.*?)\s*<img',
-            webpage, 'location')
+            r'(?s)<dt>Location</dt>\s*<dd>(.+?)<',
+            webpage, 'location', fatal=False)
 
-        formats = re.findall(
-            r'<source data-quality=\\"([0-9]+)p\\" src=\\"(.*?)\\">',
-            webpage)
+        def extract_count(webpage, label):
+            return str_to_int(self._search_regex(
+                r'<span[^>]+class="stat-num"[^>]*>([\d,.]+)</span>\s*<span[^>]+class="stat-label"[^>]*>%s' % label,
+                webpage, label, fatal=False))
 
-        formats = [{'url': fmt[1], 'height': int_or_none(fmt[0])} for fmt in formats]
+        view_count = extract_count(webpage, 'Views')
+        comment_count = extract_count(webpage, 'Comments')
 
         return {
             'id': video_id,
             'title': title,
             'description': description,
+            'thumbnail': thumbnail,
             'duration': duration,
-            'thumbnail': self._html_search_meta('og:image', webpage, 'thumbnail'),
-            'uploader_id': uploader_id,
             'upload_date': upload_date,
+            'uploader': uploader,
             'location': location,
+            'view_count': view_count,
+            'comment_count': comment_count,
             'formats': formats
         }

From 16d6973f8a9eb5a70c12d82aa40f57c2b4aa8c6f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 20 Jun 2015 00:49:28 +0600
Subject: [PATCH 1637/6219] [viki] Pass session token around (#6005)

---
 youtube_dl/extractor/viki.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index 52d10d242..51cdc6b65 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -28,11 +28,15 @@ class VikiBaseIE(InfoExtractor):
 
     _NETRC_MACHINE = 'viki'
 
+    _token = None
+
     def _prepare_call(self, path, timestamp=None, post_data=None):
         path += '?' if '?' not in path else '&'
         if not timestamp:
             timestamp = int(time.time())
         query = self._API_QUERY_TEMPLATE % (path, self._APP, timestamp)
+        if self._token:
+            query += '&token=%s' % self._token
         sig = hmac.new(
             self._APP_SECRET.encode('ascii'),
             query.encode('ascii'),
@@ -76,10 +80,14 @@ class VikiBaseIE(InfoExtractor):
             'password': password,
         }
 
-        self._call_api(
+        login = self._call_api(
             'sessions.json', None,
             'Logging in as %s' % username, post_data=login_form)
 
+        self._token = login.get('token')
+        if not self._token:
+            self.report_warning('Unable to get session token, login has probably failed')
+
 
 class VikiIE(VikiBaseIE):
     IE_NAME = 'viki'

From 10273d6e0846cd8f3762e3777712d5cd2a0cafcd Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Sat, 20 Jun 2015 08:22:13 -0500
Subject: [PATCH 1638/6219] toss new stuff into old file

---
 youtube_dl/extractor/__init__.py |   3 +-
 youtube_dl/extractor/bbccouk.py  | 101 ++++++++++++++++++++++++++++
 youtube_dl/extractor/bbcnews.py  | 109 -------------------------------
 3 files changed, 102 insertions(+), 111 deletions(-)
 delete mode 100644 youtube_dl/extractor/bbcnews.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 51d2d20e9..f9f7bdfaf 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -35,8 +35,7 @@ from .azubu import AzubuIE
 from .baidu import BaiduVideoIE
 from .bambuser import BambuserIE, BambuserChannelIE
 from .bandcamp import BandcampIE, BandcampAlbumIE
-from .bbccouk import BBCCoUkIE
-from .bbcnews import BBCNewsIE
+from .bbccouk import BBCCoUkIE, BBCNewsIE
 from .beeg import BeegIE
 from .behindkink import BehindKinkIE
 from .beatportpro import BeatportProIE
diff --git a/youtube_dl/extractor/bbccouk.py b/youtube_dl/extractor/bbccouk.py
index dcc5fc2fa..ea682fb6f 100644
--- a/youtube_dl/extractor/bbccouk.py
+++ b/youtube_dl/extractor/bbccouk.py
@@ -5,9 +5,11 @@ import xml.etree.ElementTree
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
+    parse_duration,
     int_or_none,
 )
 from ..compat import compat_HTTPError
+import re
 
 
 class BBCCoUkIE(InfoExtractor):
@@ -394,3 +396,102 @@ class BBCCoUkIE(InfoExtractor):
             'formats': formats,
             'subtitles': subtitles,
         }
+
+
+class BBCNewsIE(BBCCoUkIE):
+    IE_NAME = 'bbc.com'
+    IE_DESC = 'BBC news'
+    _VALID_URL = r'https?://(?:www\.)?(?:bbc\.co\.uk|bbc\.com)/news/(?P<id>[^/]+)'
+
+    mediaselector_url = 'http://open.live.bbc.co.uk/mediaselector/4/mtis/stream/%s'
+
+    _TESTS = [{
+        'url': 'http://www.bbc.com/news/world-europe-32668511',
+        'info_dict': {
+            'id': 'world-europe-32668511',
+            'title': 'Russia stages massive WW2 parade despite Western boycott',
+        },
+        'playlist_count': 2,
+    },{
+        'url': 'http://www.bbc.com/news/business-28299555',
+        'info_dict': {
+            'id': 'business-28299555',
+            'title': 'Farnborough Airshow: Video highlights',
+        },
+        'playlist_count': 9,
+    },{
+        'url': 'http://www.bbc.com/news/world-europe-32041533',
+        'note': 'Video',
+        'info_dict': {
+            'id': 'p02mprgb',
+            'ext': 'mp4',
+            'title': 'Aerial footage showed the site of the crash in the Alps - courtesy BFM TV',
+            'description': 'Germanwings plane crash site in aerial video - Aerial footage showed the site of the crash in the Alps - courtesy BFM TV',
+            'duration': 47,
+        },
+        'params': {
+            'skip_download': True,
+        }
+    }]
+
+    def _real_extract(self, url):
+        list_id = self._match_id(url)
+        webpage = self._download_webpage(url, list_id)
+
+        list_title = self._html_search_regex(r'<title>(.*?)(?:\s*-\s*BBC News)?</title>', webpage, 'list title')
+
+        pubdate = self._html_search_regex(r'"datePublished":\s*"(\d+-\d+-\d+)', webpage, 'date', default=None)
+        if pubdate:
+           pubdate = pubdate.replace('-','')
+
+        ret = []
+        # works with bbc.com/news/something-something-123456 articles
+        matches = re.findall(r"data-media-meta='({[^']+})'", webpage)
+        if not matches:
+           # stubbornly generic extractor for {json with "image":{allvideoshavethis},etc}
+           # in http://www.bbc.com/news/video_and_audio/international
+           matches = re.findall(r"({[^{}]+image\":{[^}]+}[^}]+})", webpage)
+        if not matches:
+           raise ExtractorError('No video found', expected=True)
+
+        for ent in matches:
+            jent = self._parse_json(ent,list_id)
+
+            programme_id = jent.get('externalId',None)
+            xml_url = jent.get('href', None)
+
+            title = jent['caption']
+            duration = parse_duration(jent.get('duration',None))
+            description = list_title + ' - ' + jent.get('caption','')
+            thumbnail = None
+            if jent.has_key('image'):
+               thumbnail=jent['image'].get('href',None)
+
+            if programme_id:
+               formats, subtitles = self._download_media_selector(programme_id)
+            elif xml_url:
+               # Cheap fallback
+               # http://playlists.bbc.co.uk/news/(list_id)[ABC..]/playlist.sxml
+               xml = self._download_webpage(xml_url, programme_id, 'Downloading playlist.sxml for externalId (fallback)')
+               programme_id = self._search_regex(r'<mediator [^>]*identifier="(.+?)"', xml, 'playlist.sxml (externalId fallback)')
+               formats, subtitles = self._download_media_selector(programme_id)
+            else:
+               raise ExtractorError('data-media-meta entry has no externalId or href value.')
+               
+            self._sort_formats(formats)
+
+            ret.append( {
+                'id': programme_id,
+                'uploader': 'BBC News',
+                'upload_date': pubdate,
+                'title': title,
+                'description': description,
+                'thumbnail': thumbnail,
+                'duration': duration,
+                'formats': formats,
+                'subtitles': subtitles,
+            } )
+
+        if len(ret) > 0:
+           return self.playlist_result(ret, list_id, list_title)
+        raise ExtractorError('No video found', expected=True)
diff --git a/youtube_dl/extractor/bbcnews.py b/youtube_dl/extractor/bbcnews.py
deleted file mode 100644
index fd4a5e38f..000000000
--- a/youtube_dl/extractor/bbcnews.py
+++ /dev/null
@@ -1,109 +0,0 @@
-from __future__ import unicode_literals
-
-from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
-    parse_duration,
-    int_or_none,
-)
-from ..compat import compat_HTTPError
-import re
-from .bbccouk import BBCCoUkIE
-
-class BBCNewsIE(BBCCoUkIE):
-    IE_NAME = 'bbc.com'
-    IE_DESC = 'BBC news'
-    _VALID_URL = r'https?://(?:www\.)?(?:bbc\.co\.uk|bbc\.com)/news/(?P<id>[^/]+)'
-
-    mediaselector_url = 'http://open.live.bbc.co.uk/mediaselector/4/mtis/stream/%s'
-
-    _TESTS = [{
-        'url': 'http://www.bbc.com/news/world-europe-32668511',
-        'info_dict': {
-            'id': 'world-europe-32668511',
-            'title': 'Russia stages massive WW2 parade despite Western boycott',
-        },
-        'playlist_count': 2,
-    },{
-        'url': 'http://www.bbc.com/news/business-28299555',
-        'info_dict': {
-            'id': 'business-28299555',
-            'title': 'Farnborough Airshow: Video highlights',
-        },
-        'playlist_count': 9,
-    },{
-        'url': 'http://www.bbc.com/news/world-europe-32041533',
-        'note': 'Video',
-        'info_dict': {
-            'id': 'p02mprgb',
-            'ext': 'mp4',
-            'title': 'Aerial footage showed the site of the crash in the Alps - courtesy BFM TV',
-            'description': 'Germanwings plane crash site in aerial video - Aerial footage showed the site of the crash in the Alps - courtesy BFM TV',
-            'duration': 47,
-        },
-        'params': {
-            'skip_download': True,
-        }
-    }]
-
-    def _real_extract(self, url):
-        list_id = self._match_id(url)
-        webpage = self._download_webpage(url, list_id)
-
-        list_title = self._html_search_regex(r'<title>(.*?)(?:\s*-\s*BBC News)?</title>', webpage, 'list title')
-
-        pubdate = self._html_search_regex(r'"datePublished":\s*"(\d+-\d+-\d+)', webpage, 'date', default=None)
-        if pubdate:
-           pubdate = pubdate.replace('-','')
-
-        ret = []
-        # works with bbc.com/news/something-something-123456 articles
-        matches = re.findall(r"data-media-meta='({[^']+})'", webpage)
-        if not matches:
-           # stubbornly generic extractor for {json with "image":{allvideoshavethis},etc}
-           # in http://www.bbc.com/news/video_and_audio/international
-           matches = re.findall(r"({[^{}]+image\":{[^}]+}[^}]+})", webpage)
-        if not matches:
-           raise ExtractorError('No video found', expected=True)
-
-        for ent in matches:
-            jent = self._parse_json(ent,list_id)
-
-            programme_id = jent.get('externalId',None)
-            xml_url = jent.get('href', None)
-
-            title = jent['caption']
-            duration = parse_duration(jent.get('duration',None))
-            description = list_title + ' - ' + jent.get('caption','')
-            thumbnail = None
-            if jent.has_key('image'):
-               thumbnail=jent['image'].get('href',None)
-
-            if programme_id:
-               formats, subtitles = self._download_media_selector(programme_id)
-            elif xml_url:
-               # Cheap fallback
-               # http://playlists.bbc.co.uk/news/(list_id)[ABC..]/playlist.sxml
-               xml = self._download_webpage(xml_url, programme_id, 'Downloading playlist.sxml for externalId (fallback)')
-               programme_id = self._search_regex(r'<mediator [^>]*identifier="(.+?)"', xml, 'playlist.sxml (externalId fallback)')
-               formats, subtitles = self._download_media_selector(programme_id)
-            else:
-               raise ExtractorError('data-media-meta entry has no externalId or href value.')
-               
-            self._sort_formats(formats)
-
-            ret.append( {
-                'id': programme_id,
-                'uploader': 'BBC News',
-                'upload_date': pubdate,
-                'title': title,
-                'description': description,
-                'thumbnail': thumbnail,
-                'duration': duration,
-                'formats': formats,
-                'subtitles': subtitles,
-            } )
-
-        if len(ret) > 0:
-           return self.playlist_result(ret, list_id, list_title)
-        raise ExtractorError('No video found', expected=True)

From 75ab0ebcf593ec91a46d83e69854ffa313d33309 Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Sat, 20 Jun 2015 08:24:02 -0500
Subject: [PATCH 1639/6219] no .get('..',None)

---
 youtube_dl/extractor/bbccouk.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/bbccouk.py b/youtube_dl/extractor/bbccouk.py
index ea682fb6f..de4d7f9c0 100644
--- a/youtube_dl/extractor/bbccouk.py
+++ b/youtube_dl/extractor/bbccouk.py
@@ -457,15 +457,15 @@ class BBCNewsIE(BBCCoUkIE):
         for ent in matches:
             jent = self._parse_json(ent,list_id)
 
-            programme_id = jent.get('externalId',None)
-            xml_url = jent.get('href', None)
+            programme_id = jent.get('externalId')
+            xml_url = jent.get('href')
 
             title = jent['caption']
-            duration = parse_duration(jent.get('duration',None))
+            duration = parse_duration(jent.get('duration')
             description = list_title + ' - ' + jent.get('caption','')
             thumbnail = None
             if jent.has_key('image'):
-               thumbnail=jent['image'].get('href',None)
+               thumbnail=jent['image'].get('href')
 
             if programme_id:
                formats, subtitles = self._download_media_selector(programme_id)

From 77c975f536befbe89bf718e86282958d391d9ffe Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Sat, 20 Jun 2015 08:28:14 -0500
Subject: [PATCH 1640/6219] typofix

---
 youtube_dl/extractor/bbccouk.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbccouk.py b/youtube_dl/extractor/bbccouk.py
index de4d7f9c0..f9404f3fa 100644
--- a/youtube_dl/extractor/bbccouk.py
+++ b/youtube_dl/extractor/bbccouk.py
@@ -461,7 +461,7 @@ class BBCNewsIE(BBCCoUkIE):
             xml_url = jent.get('href')
 
             title = jent['caption']
-            duration = parse_duration(jent.get('duration')
+            duration = parse_duration(jent.get('duration'))
             description = list_title + ' - ' + jent.get('caption','')
             thumbnail = None
             if jent.has_key('image'):

From de939d89eb83c851c6db66933e5fc0c401a1a679 Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Sat, 20 Jun 2015 11:04:46 -0500
Subject: [PATCH 1641/6219] Support BBC news in other languages,
 non-mediaselector videos

---
 youtube_dl/extractor/bbccouk.py | 87 +++++++++++++++++++++++++++------
 1 file changed, 73 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/bbccouk.py b/youtube_dl/extractor/bbccouk.py
index f9404f3fa..72e20857b 100644
--- a/youtube_dl/extractor/bbccouk.py
+++ b/youtube_dl/extractor/bbccouk.py
@@ -401,7 +401,7 @@ class BBCCoUkIE(InfoExtractor):
 class BBCNewsIE(BBCCoUkIE):
     IE_NAME = 'bbc.com'
     IE_DESC = 'BBC news'
-    _VALID_URL = r'https?://(?:www\.)?(?:bbc\.co\.uk|bbc\.com)/news/(?P<id>[^/]+)'
+    _VALID_URL = r'https?://(?:www\.)?bbc\.com/.+?/(?P<id>[^/]+)$'
 
     mediaselector_url = 'http://open.live.bbc.co.uk/mediaselector/4/mtis/stream/%s'
 
@@ -432,56 +432,115 @@ class BBCNewsIE(BBCCoUkIE):
         'params': {
             'skip_download': True,
         }
+    },{
+        'url': 'http://www.bbc.com/turkce/haberler/2015/06/150615_telabyad_kentin_cogu',
+        'note': 'Video',
+        'info_dict': {
+            'id': 'NA',
+            'ext': 'mp4',
+            'title': 'YPG - Tel Abyad..n tamam. kontrol.m.zde',
+            'duration': 47,
+        },
+        'params': {
+            'skip_download': True,
+        }
+    },{
+        'url': 'http://www.bbc.com/mundo/video_fotos/2015/06/150619_video_honduras_militares_hospitales_corrupcion_aw',
+        'note': 'Video',
+        'info_dict': {
+            'id': '39275083',
+            'ext': 'mp4',
+            'title': 'Honduras militariza sus hospitales por nuevo esc.ndalo de corrupci.n',
+            'duration': 87,
+        },
+        'params': {
+            'skip_download': True,
+        }
     }]
 
     def _real_extract(self, url):
         list_id = self._match_id(url)
         webpage = self._download_webpage(url, list_id)
 
-        list_title = self._html_search_regex(r'<title>(.*?)(?:\s*-\s*BBC News)?</title>', webpage, 'list title')
+        list_title = self._html_search_regex(r'<title>(.*?)(?:\s*-\s*BBC [^ ]+)?</title>', webpage, 'list title')
 
         pubdate = self._html_search_regex(r'"datePublished":\s*"(\d+-\d+-\d+)', webpage, 'date', default=None)
         if pubdate:
            pubdate = pubdate.replace('-','')
 
         ret = []
+        jsent = []
+
         # works with bbc.com/news/something-something-123456 articles
-        matches = re.findall(r"data-media-meta='({[^']+})'", webpage)
-        if not matches:
+        jsent = map(
+           lambda m: self._parse_json(m,list_id),
+           re.findall(r"data-media-meta='({[^']+})'", webpage)
+        )
+
+        if len(jsent) == 0:
+           # http://www.bbc.com/news/video_and_audio/international
+           # and single-video articles
+           masset = self._html_search_regex(r'mediaAssetPage\.init\(\s*({.+?}), "/', webpage, 'mediaassets', default=None)
+           if masset:
+              jmasset = self._parse_json(masset,list_id)
+              for key, val in jmasset.get('videos',{}).items():
+                  for skey, sval in val.items():
+                      sval['id'] = skey
+                      jsent.append(sval)
+
+        if len(jsent) == 0:
            # stubbornly generic extractor for {json with "image":{allvideoshavethis},etc}
            # in http://www.bbc.com/news/video_and_audio/international
-           matches = re.findall(r"({[^{}]+image\":{[^}]+}[^}]+})", webpage)
-        if not matches:
+           # prone to breaking if entries have sourceFiles list
+           jsent = map(
+               lambda m: self._parse_json(m,list_id),
+               re.findall(r"({[^{}]+image\":{[^}]+}[^}]+})", webpage)
+           )          
+
+        if len(jsent) == 0:
            raise ExtractorError('No video found', expected=True)
 
-        for ent in matches:
-            jent = self._parse_json(ent,list_id)
-
+        for jent in jsent:
             programme_id = jent.get('externalId')
-            xml_url = jent.get('href')
+            xml_url = jent.get('hxref')
+
+            title = jent.get('caption',list_title)
 
-            title = jent['caption']
             duration = parse_duration(jent.get('duration'))
             description = list_title + ' - ' + jent.get('caption','')
             thumbnail = None
             if jent.has_key('image'):
                thumbnail=jent['image'].get('href')
 
+            formats = []
+            subtitles = []
+
             if programme_id:
                formats, subtitles = self._download_media_selector(programme_id)
+            elif jent.has_key('sourceFiles'):
+               # mediaselector not used at
+               # http://www.bbc.com/turkce/haberler/2015/06/150615_telabyad_kentin_cogu
+               for key, val in jent['sourceFiles'].items():
+                  formats.append( {
+                     'ext': val.get('encoding'),
+                     'url': val.get('url'),
+                     'filesize': int(val.get('filesize')),
+                     'format_id': key
+                  } )
             elif xml_url:
                # Cheap fallback
                # http://playlists.bbc.co.uk/news/(list_id)[ABC..]/playlist.sxml
                xml = self._download_webpage(xml_url, programme_id, 'Downloading playlist.sxml for externalId (fallback)')
                programme_id = self._search_regex(r'<mediator [^>]*identifier="(.+?)"', xml, 'playlist.sxml (externalId fallback)')
                formats, subtitles = self._download_media_selector(programme_id)
-            else:
-               raise ExtractorError('data-media-meta entry has no externalId or href value.')
+
+            if len(formats) == 0:
+               raise ExtractorError('unsupported json media entry.\n    '+str(jent)+'\n')
                
             self._sort_formats(formats)
 
             ret.append( {
-                'id': programme_id,
+                'id': jent.get('programme_id',jent.get('id')),
                 'uploader': 'BBC News',
                 'upload_date': pubdate,
                 'title': title,

From 7bb23aeca4e9076528e3d31d501a9a288dcd444c Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Sat, 20 Jun 2015 11:08:13 -0500
Subject: [PATCH 1642/6219] rename bbccouk.py -> bbc.py

---
 youtube_dl/extractor/__init__.py            | 2 +-
 youtube_dl/extractor/{bbccouk.py => bbc.py} | 0
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename youtube_dl/extractor/{bbccouk.py => bbc.py} (100%)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index f9f7bdfaf..a48346e60 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -35,7 +35,7 @@ from .azubu import AzubuIE
 from .baidu import BaiduVideoIE
 from .bambuser import BambuserIE, BambuserChannelIE
 from .bandcamp import BandcampIE, BandcampAlbumIE
-from .bbccouk import BBCCoUkIE, BBCNewsIE
+from .bbc import BBCCoUkIE, BBCNewsIE
 from .beeg import BeegIE
 from .behindkink import BehindKinkIE
 from .beatportpro import BeatportProIE
diff --git a/youtube_dl/extractor/bbccouk.py b/youtube_dl/extractor/bbc.py
similarity index 100%
rename from youtube_dl/extractor/bbccouk.py
rename to youtube_dl/extractor/bbc.py

From 2a282a3b5f366ba0569bae477d5060329ba254fb Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Sat, 20 Jun 2015 11:11:41 -0500
Subject: [PATCH 1643/6219] Unbreak breakage that was broken to test breakage

---
 youtube_dl/extractor/bbc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 72e20857b..310db9d1d 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -502,7 +502,7 @@ class BBCNewsIE(BBCCoUkIE):
 
         for jent in jsent:
             programme_id = jent.get('externalId')
-            xml_url = jent.get('hxref')
+            xml_url = jent.get('href')
 
             title = jent.get('caption',list_title)
 

From 964afd0689bdd7140b8ab182273d6379fe7b0548 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 21 Jun 2015 03:12:17 +0800
Subject: [PATCH 1644/6219] [xvideos] Support lower-quality formats found on
 Android

Closes #5968
---
 youtube_dl/extractor/xvideos.py | 27 ++++++++++++++++++++++++++-
 1 file changed, 26 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/xvideos.py b/youtube_dl/extractor/xvideos.py
index 2a45dc574..d8415bed4 100644
--- a/youtube_dl/extractor/xvideos.py
+++ b/youtube_dl/extractor/xvideos.py
@@ -5,10 +5,12 @@ import re
 from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse,
+    compat_urllib_request,
 )
 from ..utils import (
     clean_html,
     ExtractorError,
+    determine_ext,
 )
 
 
@@ -25,6 +27,8 @@ class XVideosIE(InfoExtractor):
         }
     }
 
+    _ANDROID_USER_AGENT = 'Mozilla/5.0 (Linux; Android 4.0.4; Galaxy Nexus Build/IMM76B) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.133 Mobile Safari/535.19'
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
@@ -40,9 +44,30 @@ class XVideosIE(InfoExtractor):
         video_thumbnail = self._search_regex(
             r'url_bigthumb=(.+?)&amp', webpage, 'thumbnail', fatal=False)
 
+        formats = [{
+            'url': video_url,
+        }]
+
+        android_req = compat_urllib_request.Request(url)
+        android_req.add_header('User-Agent', self._ANDROID_USER_AGENT)
+        android_webpage = self._download_webpage(android_req, video_id, fatal=False)
+
+        if android_webpage is not None:
+            player_params_str = self._search_regex(
+                'mobileReplacePlayerDivTwoQual\(([^)]+)\)',
+                android_webpage, 'player parameters', default='')
+            player_params = list(map(lambda s: s.strip(' \''), player_params_str.split(',')))
+            if player_params:
+                formats.extend([{
+                    'url': param,
+                    'preference': -10,
+                } for param in player_params if determine_ext(param) == 'mp4'])
+
+        self._sort_formats(formats)
+
         return {
             'id': video_id,
-            'url': video_url,
+            'formats': formats,
             'title': video_title,
             'ext': 'flv',
             'thumbnail': video_thumbnail,

From c9ac7fa909fb969ac21a6d168d09803119b018c3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 21 Jun 2015 04:17:54 +0600
Subject: [PATCH 1645/6219] [imdb] Fix extraction

---
 youtube_dl/extractor/imdb.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/imdb.py b/youtube_dl/extractor/imdb.py
index f29df36b5..4bb574cf3 100644
--- a/youtube_dl/extractor/imdb.py
+++ b/youtube_dl/extractor/imdb.py
@@ -46,7 +46,7 @@ class ImdbIE(InfoExtractor):
             format_info = info['videoPlayerObject']['video']
             formats.append({
                 'format_id': f_id,
-                'url': format_info['url'],
+                'url': format_info['videoInfoList'][0]['videoUrl'],
             })
 
         return {

From af1fa6234e9623a1a09e66c24c0dedc6996645c9 Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Sun, 21 Jun 2015 11:12:31 +0800
Subject: [PATCH 1646/6219] [neteasemusic] Add new extractor for music.163.com

---
 youtube_dl/extractor/__init__.py     |   9 +
 youtube_dl/extractor/neteasemusic.py | 434 +++++++++++++++++++++++++++
 2 files changed, 443 insertions(+)
 create mode 100644 youtube_dl/extractor/neteasemusic.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 6c548d8e9..c8692ffa9 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -339,6 +339,15 @@ from .ndtv import NDTVIE
 from .netzkino import NetzkinoIE
 from .nerdcubed import NerdCubedFeedIE
 from .nerdist import NerdistIE
+from .neteasemusic import (
+    NetEaseMusicIE,
+    NetEaseMusicAlbumIE,
+    NetEaseMusicSingerIE,
+    NetEaseMusicListIE,
+    NetEaseMusicMvIE,
+    NetEaseMusicProgramIE,
+    NetEaseMusicDjRadioIE,
+)
 from .newgrounds import NewgroundsIE
 from .newstube import NewstubeIE
 from .nextmedia import (
diff --git a/youtube_dl/extractor/neteasemusic.py b/youtube_dl/extractor/neteasemusic.py
new file mode 100644
index 000000000..ebed5d3b4
--- /dev/null
+++ b/youtube_dl/extractor/neteasemusic.py
@@ -0,0 +1,434 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from hashlib import md5
+from datetime import datetime
+import itertools
+import re
+
+from .common import InfoExtractor
+from ..compat import (
+    compat_urllib_request,
+    compat_urllib_parse,
+)
+
+
+class NetEaseMusicBaseIE(InfoExtractor):
+    _FORMATS = ['bMusic', 'mMusic', 'hMusic']
+    _NETEASE_SALT = '3go8&$8*3*3h0k(2)2'
+    _API_BASE = 'http://music.163.com/api/'
+
+    @classmethod
+    def _encrypt(cls, dfsid):
+        salt_bytes = bytearray(str(cls._NETEASE_SALT))
+        string_bytes = bytearray(str(dfsid))
+        salt_len = len(salt_bytes)
+        for i in xrange(len(string_bytes)):
+            string_bytes[i] = string_bytes[i] ^ salt_bytes[i % salt_len]
+        m = md5()
+        m.update(string_bytes)
+        result = m.digest().encode('base64')[:-1]
+        return result.replace('/', '_').replace('+', '-')
+
+    @classmethod
+    def extract_formats(cls, info):
+        formats = []
+        for song_format in cls._FORMATS:
+            details = info.get(song_format)
+            if not details:
+                continue
+            formats.append({
+                'url': 'http://m1.music.126.net/%s/%s.%s' %
+                       (cls._encrypt(details['dfsId']), details['dfsId'],
+                        details['extension']),
+                'ext': details['extension'],
+                'abr': details['bitrate'] / 1000,
+                'preference': details['bitrate'],
+                'format_id': song_format,
+                'filesize': details['size'],
+                'asr': details['sr']
+            })
+        return formats
+
+    def query_api(self, endpoint, video_id, note):
+        req = compat_urllib_request.Request('%s%s' % (self._API_BASE, endpoint))
+        req.add_header('Referer', self._API_BASE)
+        return self._download_json(req, video_id, note)
+
+
+class NetEaseMusicIE(NetEaseMusicBaseIE):
+    IE_NAME = 'netease:song'
+    _VALID_URL = r'https?://music\.163\.com/(#/)?song\?id=(?P<id>[0-9]+)'
+    _TESTS = [{
+        'url': 'http://music.163.com/#/song?id=32102397',
+        'md5': 'f2e97280e6345c74ba9d5677dd5dcb45',
+        'info_dict': {
+            'id': '32102397',
+            'ext': 'mp3',
+            'title': 'Bad Blood (feat. Kendrick Lamar)',
+            'creator': 'Taylor Swift / Kendrick Lamar',
+            'upload_date': '20150517',
+            'timestamp': 1431878400,
+            'description': 'md5:a10a54589c2860300d02e1de821eb2ef',
+        },
+    }, {
+        'note': 'No lyrics translation.',
+        'url': 'http://music.163.com/#/song?id=29822014',
+        'info_dict': {
+            'id': '29822014',
+            'ext': 'mp3',
+            'title': '听见下雨的声音',
+            'creator': '周杰伦',
+            'upload_date': '20141225',
+            'timestamp': 1419523200,
+            'description': 'md5:a4d8d89f44656af206b7b2555c0bce6c',
+        },
+    }, {
+        'note': 'No lyrics.',
+        'url': 'http://music.163.com/song?id=17241424',
+        'info_dict': {
+            'id': '17241424',
+            'ext': 'mp3',
+            'title': 'Opus 28',
+            'creator': 'Dustin O\'Halloran',
+            'upload_date': '20080211',
+            'timestamp': 1202745600,
+        },
+    }]
+
+    def _process_lyrics(self, lyrics_info):
+        original = lyrics_info.get('lrc', {}).get('lyric')
+        translated = lyrics_info.get('tlyric', {}).get('lyric')
+
+        if not translated:
+            return original
+
+        lyrics_expr = r'(\[[0-9]{2}:[0-9]{2}\.[0-9]{2,}\])([^\n]+)'
+        original_ts_texts = re.findall(lyrics_expr, original)
+        translation_ts_dict = {
+            time_stamp: text for time_stamp, text in re.findall(lyrics_expr, translated)
+        }
+
+        lyrics = '\n'.join([
+            '%s%s / %s' % (time_stamp, text, translation_ts_dict.get(time_stamp, ''))
+            for time_stamp, text in original_ts_texts
+        ])
+        return lyrics
+
+    def _real_extract(self, url):
+        song_id = self._match_id(url)
+
+        params = {
+            'id': song_id,
+            'ids': '[%s]' % song_id
+        }
+        info = self.query_api(
+            'song/detail?' + compat_urllib_parse.urlencode(params),
+            song_id, 'Downloading song info')['songs'][0]
+
+        formats = self.extract_formats(info)
+        self._sort_formats(formats)
+
+        lyrics_info = self.query_api(
+            'song/lyric?id=%s&lv=-1&tv=-1' % song_id, 
+            song_id, 'Downloading lyrics data')
+        lyrics = self._process_lyrics(lyrics_info)
+
+        alt_title = None
+        if info.get('alias'):
+            alt_title = '/'.join(info.get('alias'))
+
+        return {
+            'id': song_id,
+            'title': info['name'],
+            'alt_title': alt_title,
+            'creator': ' / '.join([artist['name'] for artist in info.get('artists', [])]),
+            'timestamp': int(info.get('album', {}).get('publishTime')/1000),
+            'thumbnail': info.get('album', {}).get('picUrl'),
+            'duration': int(info.get('duration', 0)/1000),
+            'description': lyrics,
+            'formats': formats,
+        }
+
+
+class NetEaseMusicAlbumIE(NetEaseMusicBaseIE):
+    IE_NAME = 'netease:album'
+    _VALID_URL = r'https?://music\.163\.com/(#/)?album\?id=(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'http://music.163.com/#/album?id=220780',
+        'info_dict': {
+            'id': '220780',
+            'title': 'B\'day',
+        },
+        'playlist_count': 23,
+    }
+
+    def _real_extract(self, url):
+        album_id = self._match_id(url)
+
+        info = self.query_api(
+            'album/%s?id=%s' % (album_id, album_id), 
+            album_id, 'Downloading album data')['album']
+
+        name = info['name']
+        desc = info.get('description')
+        entries = [
+            self.url_result('http://music.163.com/#/song?id=%s' % song['id'],
+                            'NetEaseMusic', song['id'])
+            for song in info['songs']
+        ]
+        return self.playlist_result(entries, album_id, name, desc)
+
+
+class NetEaseMusicSingerIE(NetEaseMusicBaseIE):
+    IE_NAME = 'netease:singer'
+    _VALID_URL = r'https?://music\.163\.com/(#/)?artist\?id=(?P<id>[0-9]+)'
+    _TESTS = [{
+        'note': 'Singer has aliases.',
+        'url': 'http://music.163.com/#/artist?id=10559',
+        'info_dict': {
+            'id': '10559',
+            'title': '张惠妹 - aMEI;阿密特',
+        },
+        'playlist_count': 50,
+    }, {
+        'note': 'Singer has translated name.',
+        'url': 'http://music.163.com/#/artist?id=124098',
+        'info_dict': {
+            'id': '124098',
+            'title': '李昇基 - 이승기',
+        },
+        'playlist_count': 50,
+    }]
+
+    def _real_extract(self, url):
+        singer_id = self._match_id(url)
+
+        info = self.query_api(
+            'artist/%s?id=%s' % (singer_id, singer_id), 
+            singer_id, 'Downloading singer data')
+
+        name = info['artist']['name']
+        if info['artist']['trans']:
+            name = '%s - %s' % (name, info['artist']['trans'])
+        if info['artist']['alias']:
+            name = '%s - %s' % (name, ";".join(info['artist']['alias']))
+
+        entries = [
+            self.url_result('http://music.163.com/#/song?id=%s' % song['id'],
+                            'NetEaseMusic', song['id'])
+            for song in info['hotSongs']
+        ]
+        return self.playlist_result(entries, singer_id, name)
+
+
+class NetEaseMusicListIE(NetEaseMusicBaseIE):
+    IE_NAME = 'netease:playlist'
+    _VALID_URL = r'https?://music\.163\.com/(#/)?(playlist|discover/toplist)\?id=(?P<id>[0-9]+)'
+    _TESTS = [{
+        'url': 'http://music.163.com/#/playlist?id=79177352',
+        'info_dict': {
+            'id': '79177352',
+            'title': 'Billboard 2007 Top 100',
+            'description': 'md5:12fd0819cab2965b9583ace0f8b7b022'
+        },
+        'playlist_count': 99,
+    }, {
+        'note': 'Toplist/Charts sample',
+        'url': 'http://music.163.com/#/discover/toplist?id=3733003',
+        'info_dict': {
+            'id': '3733003',
+            'title': 're:韩国Melon排行榜周榜 [0-9]{4}-[0-9]{2}-[0-9]{2}',
+            'description': 'md5:73ec782a612711cadc7872d9c1e134fc',
+        },
+        'playlist_count': 50,
+    }]
+
+    def _real_extract(self, url):
+        list_id = self._match_id(url)
+
+        info = self.query_api(
+            'playlist/detail?id=%s&lv=-1&tv=-1' % list_id, 
+            list_id, 'Downloading playlist data')['result']
+
+        name = info['name']
+        desc = info.get('description')
+
+        if info.get('specialType') == 10:  # is a chart/toplist
+            datestamp = datetime.fromtimestamp(info['updateTime']/1000).strftime('%Y-%m-%d')
+            name = '%s %s' % (name, datestamp)
+
+        entries = [
+            self.url_result('http://music.163.com/#/song?id=%s' % song['id'],
+                            'NetEaseMusic', song['id'])
+            for song in info['tracks']
+        ]
+        return self.playlist_result(entries, list_id, name, desc)
+
+
+class NetEaseMusicMvIE(NetEaseMusicBaseIE):
+    IE_NAME = 'netease:mv'
+    _VALID_URL = r'https?://music\.163\.com/(#/)?mv\?id=(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'http://music.163.com/#/mv?id=415350',
+        'info_dict': {
+            'id': '415350',
+            'ext': 'mp4',
+            'title': '이럴거면 그러지말지',
+            'description': '白雅言自作曲唱甜蜜爱情',
+            'creator': '白雅言',
+            'upload_date': '20150520',
+        },
+    }
+
+    def _real_extract(self, url):
+        mv_id = self._match_id(url)
+
+        info = self.query_api(
+            'mv/detail?id=%s&type=mp4' % mv_id,
+            mv_id, 'Downloading mv info')['data']
+
+        formats = [
+            {'url': mv_url, 'ext': 'mp4', 'format_id': '%sp' % brs, 'preference': int(brs)}
+            for brs, mv_url in info['brs'].items()
+        ]
+        self._sort_formats(formats)
+
+        return {
+            'id': mv_id,
+            'title': info['name'],
+            'description': info.get('desc') or info.get('briefDesc'),
+            'creator': info['artistName'],
+            'upload_date': info['publishTime'].replace('-', ''),
+            'formats': formats,
+            'thumbnail': info.get('cover'),
+            'duration': int(info.get('duration', 0)/1000),
+        }
+
+
+class NetEaseMusicProgramIE(NetEaseMusicBaseIE):
+    IE_NAME = 'netease:program'
+    _VALID_URL = r'https?://music\.163\.com/(#/?)program\?id=(?P<id>[0-9]+)'
+    _TESTS = [{
+        'url': 'http://music.163.com/#/program?id=10109055',
+        'info_dict': {
+            'id': '10109055',
+            'ext': 'mp3',
+            'title': '不丹足球背后的故事',
+            'description': '喜马拉雅人的足球梦 ...',
+            'creator': '大话西藏',
+            'timestamp': 1434179341,
+            'upload_date': '20150613',
+            'duration': 900,
+        },
+    }, {
+        'note': 'This program has accompanying songs.',
+        'url': 'http://music.163.com/#/program?id=10141022',
+        'info_dict': {
+            'id': '10141022',
+            'title': '25岁，你是自在如风的少年<27°C>',
+            'description': 'md5:8d594db46cc3e6509107ede70a4aaa3b',
+        },
+        'playlist_count': 4,
+    }, {
+        'note': 'This program has accompanying songs.',
+        'url': 'http://music.163.com/#/program?id=10141022',
+        'info_dict': {
+            'id': '10141022',
+            'ext': 'mp3',
+            'title': '25岁，你是自在如风的少年<27°C>',
+            'description': 'md5:8d594db46cc3e6509107ede70a4aaa3b',
+            'timestamp': 1434450840,
+            'upload_date': '20150616',
+        },
+        'params': {
+            'noplaylist': True
+        }
+    }]
+
+    def _real_extract(self, url):
+        program_id = self._match_id(url)
+
+        info = self.query_api(
+            'dj/program/detail?id=%s' % program_id,
+            program_id, 'Downloading program info')['program']
+
+        name = info['name']
+        description = info['description']
+
+        if not info['songs'] or self._downloader.params.get('noplaylist'):
+            if info['songs']:
+                self.to_screen(
+                    'Downloading just the main audio %s because of --no-playlist'
+                    % info['mainSong']['id'])
+            
+            formats = self.extract_formats(info['mainSong'])
+            self._sort_formats(formats)
+
+            return {
+                'id': program_id,
+                'title': name,
+                'description': description,
+                'creator': info['dj']['brand'],
+                'timestamp': int(info['createTime']/1000),
+                'thumbnail': info['coverUrl'],
+                'duration': int(info.get('duration', 0)/1000),
+                'formats': formats,
+            }
+
+        self.to_screen(
+            'Downloading playlist %s - add --no-playlist to just download the main audio %s'
+            % (program_id, info['mainSong']['id']))
+
+        song_ids = [info['mainSong']['id']]
+        song_ids.extend([song['id'] for song in info['songs']])
+        entries = [
+            self.url_result('http://music.163.com/#/song?id=%s' % song_id,
+                            'NetEaseMusic', song_id)
+            for song_id in song_ids
+        ]
+        return self.playlist_result(entries, program_id, name, description)
+
+
+class NetEaseMusicDjRadioIE(NetEaseMusicBaseIE):
+    IE_NAME = 'netease:djradio'
+    _VALID_URL = r'https?://music\.163\.com/(#/)?djradio\?id=(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'http://music.163.com/#/djradio?id=42',
+        'info_dict': {
+            'id': '42',
+            'title': '声音蔓延',
+            'description': 'md5:766220985cbd16fdd552f64c578a6b15'
+        },
+        'playlist_mincount': 40,
+    }
+    _PAGE_SIZE = 1000
+
+    def _real_extract(self, url):
+        dj_id = self._match_id(url)
+
+        name = None
+        desc = None
+        entries = []
+        for offset in itertools.count(start=0, step=self._PAGE_SIZE):
+            info = self.query_api(
+                'dj/program/byradio?asc=false&limit=%d&radioId=%s&offset=%d' 
+                % (self._PAGE_SIZE, dj_id, offset),
+                dj_id, 'Downloading dj programs - %d' % offset)
+
+            entries.extend([
+                self.url_result(
+                    'http://music.163.com/#/program?id=%s' % program['id'],
+                    'NetEaseMusicProgram', program['id'])
+                for program in info['programs']
+            ])
+
+            if name is None:
+                radio = info['programs'][0]['radio']
+                name = radio['name']
+                desc = radio['desc']
+
+            if not info['more']:
+                break
+
+        return self.playlist_result(entries, dj_id, name, desc)

From 2da0cad6ae08e3c78f85a340260aef80a464cb20 Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Sun, 21 Jun 2015 11:44:50 +0800
Subject: [PATCH 1647/6219] [neteasemusic] Do proper rounding conversion of
 millisecond timestamps/durations

---
 youtube_dl/extractor/neteasemusic.py | 21 +++++++++++++--------
 1 file changed, 13 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/neteasemusic.py b/youtube_dl/extractor/neteasemusic.py
index ebed5d3b4..a70c65ca5 100644
--- a/youtube_dl/extractor/neteasemusic.py
+++ b/youtube_dl/extractor/neteasemusic.py
@@ -50,6 +50,10 @@ class NetEaseMusicBaseIE(InfoExtractor):
             })
         return formats
 
+    @classmethod
+    def convert_milliseconds(cls, ms):
+        return int(round(ms/1000.0))
+
     def query_api(self, endpoint, video_id, note):
         req = compat_urllib_request.Request('%s%s' % (self._API_BASE, endpoint))
         req.add_header('Referer', self._API_BASE)
@@ -143,9 +147,9 @@ class NetEaseMusicIE(NetEaseMusicBaseIE):
             'title': info['name'],
             'alt_title': alt_title,
             'creator': ' / '.join([artist['name'] for artist in info.get('artists', [])]),
-            'timestamp': int(info.get('album', {}).get('publishTime')/1000),
+            'timestamp': self.convert_milliseconds(info.get('album', {}).get('publishTime')),
             'thumbnail': info.get('album', {}).get('picUrl'),
-            'duration': int(info.get('duration', 0)/1000),
+            'duration': self.convert_milliseconds(info.get('duration', 0)),
             'description': lyrics,
             'formats': formats,
         }
@@ -255,7 +259,8 @@ class NetEaseMusicListIE(NetEaseMusicBaseIE):
         desc = info.get('description')
 
         if info.get('specialType') == 10:  # is a chart/toplist
-            datestamp = datetime.fromtimestamp(info['updateTime']/1000).strftime('%Y-%m-%d')
+            datestamp = datetime.fromtimestamp(
+                self.convert_milliseconds(info['updateTime'])).strftime('%Y-%m-%d')
             name = '%s %s' % (name, datestamp)
 
         entries = [
@@ -302,7 +307,7 @@ class NetEaseMusicMvIE(NetEaseMusicBaseIE):
             'upload_date': info['publishTime'].replace('-', ''),
             'formats': formats,
             'thumbnail': info.get('cover'),
-            'duration': int(info.get('duration', 0)/1000),
+            'duration': self.convert_milliseconds(info.get('duration', 0)),
         }
 
 
@@ -317,7 +322,7 @@ class NetEaseMusicProgramIE(NetEaseMusicBaseIE):
             'title': '不丹足球背后的故事',
             'description': '喜马拉雅人的足球梦 ...',
             'creator': '大话西藏',
-            'timestamp': 1434179341,
+            'timestamp': 1434179342,
             'upload_date': '20150613',
             'duration': 900,
         },
@@ -338,7 +343,7 @@ class NetEaseMusicProgramIE(NetEaseMusicBaseIE):
             'ext': 'mp3',
             'title': '25岁，你是自在如风的少年<27°C>',
             'description': 'md5:8d594db46cc3e6509107ede70a4aaa3b',
-            'timestamp': 1434450840,
+            'timestamp': 1434450841,
             'upload_date': '20150616',
         },
         'params': {
@@ -370,9 +375,9 @@ class NetEaseMusicProgramIE(NetEaseMusicBaseIE):
                 'title': name,
                 'description': description,
                 'creator': info['dj']['brand'],
-                'timestamp': int(info['createTime']/1000),
+                'timestamp': self.convert_milliseconds(info['createTime']),
                 'thumbnail': info['coverUrl'],
-                'duration': int(info.get('duration', 0)/1000),
+                'duration': self.convert_milliseconds(info.get('duration', 0)),
                 'formats': formats,
             }
 

From 6a745c2c0fa2d627b46f2d4d8013fa69276c4fac Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 21 Jun 2015 18:22:19 +0800
Subject: [PATCH 1648/6219] [pinkbike] PEP8

---
 youtube_dl/extractor/pinkbike.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pinkbike.py b/youtube_dl/extractor/pinkbike.py
index 745433b48..a52210fab 100644
--- a/youtube_dl/extractor/pinkbike.py
+++ b/youtube_dl/extractor/pinkbike.py
@@ -44,7 +44,7 @@ class PinkbikeIE(InfoExtractor):
 
         formats = []
         for _, format_id, src in re.findall(
-            r'data-quality=((?:\\)?["\'])(.+?)\1[^>]+src=\1(.+?)\1', webpage):
+                r'data-quality=((?:\\)?["\'])(.+?)\1[^>]+src=\1(.+?)\1', webpage):
             height = int_or_none(self._search_regex(
                 r'^(\d+)[pP]$', format_id, 'height', default=None))
             formats.append({

From b407e173e44041b1a92fb61e316f92d19834a40a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 21 Jun 2015 18:23:58 +0800
Subject: [PATCH 1649/6219] [vimeo/generic] Move detection logic from GenericIE
 to VimeoIE

---
 youtube_dl/extractor/generic.py | 16 ++++------------
 youtube_dl/extractor/vimeo.py   | 16 ++++++++++++++++
 2 files changed, 20 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index f6b984300..bf689f531 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -43,6 +43,7 @@ from .senateisvp import SenateISVPIE
 from .bliptv import BlipTVIE
 from .svt import SVTIE
 from .pornhub import PornHubIE
+from .vimeo import VimeoIE
 
 
 class GenericIE(InfoExtractor):
@@ -1089,18 +1090,9 @@ class GenericIE(InfoExtractor):
         if matches:
             return _playlist_from_matches(matches, ie='RtlNl')
 
-        # Look for embedded (iframe) Vimeo player
-        mobj = re.search(
-            r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//player\.vimeo\.com/video/.+?)\1', webpage)
-        if mobj:
-            player_url = unescapeHTML(mobj.group('url'))
-            surl = smuggle_url(player_url, {'Referer': url})
-            return self.url_result(surl)
-        # Look for embedded (swf embed) Vimeo player
-        mobj = re.search(
-            r'<embed[^>]+?src="((?:https?:)?//(?:www\.)?vimeo\.com/moogaloop\.swf.+?)"', webpage)
-        if mobj:
-            return self.url_result(mobj.group(1))
+        vimeo_url = VimeoIE._extract_vimeo_url(url, webpage)
+        if vimeo_url is not None:
+            return self.url_result(vimeo_url)
 
         # Look for embedded YouTube player
         matches = re.findall(r'''(?x)
diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index f300c7ca4..cae90205d 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -22,6 +22,7 @@ from ..utils import (
     unified_strdate,
     unsmuggle_url,
     urlencode_postdata,
+    unescapeHTML,
 )
 
 
@@ -173,6 +174,21 @@ class VimeoIE(VimeoBaseInfoExtractor):
         },
     ]
 
+    @staticmethod
+    def _extract_vimeo_url(url, webpage):
+        # Look for embedded (iframe) Vimeo player
+        mobj = re.search(
+            r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//player\.vimeo\.com/video/.+?)\1', webpage)
+        if mobj:
+            player_url = unescapeHTML(mobj.group('url'))
+            surl = smuggle_url(player_url, {'Referer': url})
+            return surl
+        # Look for embedded (swf embed) Vimeo player
+        mobj = re.search(
+            r'<embed[^>]+?src="((?:https?:)?//(?:www\.)?vimeo\.com/moogaloop\.swf.+?)"', webpage)
+        if mobj:
+            return mobj.group(1)
+
     def _verify_video_password(self, url, video_id, webpage):
         password = self._downloader.params.get('videopassword', None)
         if password is None:

From c5895d5dbdc33fbad1c91f448704d7711448220d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 21 Jun 2015 18:30:38 +0800
Subject: [PATCH 1650/6219] [tumblr] Support Vimeo embeds (fixes #5969)

---
 youtube_dl/extractor/tumblr.py | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/youtube_dl/extractor/tumblr.py b/youtube_dl/extractor/tumblr.py
index 63c20310d..9ead13a91 100644
--- a/youtube_dl/extractor/tumblr.py
+++ b/youtube_dl/extractor/tumblr.py
@@ -5,6 +5,7 @@ import re
 
 from .common import InfoExtractor
 from .pornhub import PornHubIE
+from .vimeo import VimeoIE
 
 
 class TumblrIE(InfoExtractor):
@@ -40,6 +41,17 @@ class TumblrIE(InfoExtractor):
             'timestamp': 1430931613,
         },
         'add_ie': ['Vidme'],
+    }, {
+        'url': 'http://camdamage.tumblr.com/post/98846056295/',
+        'md5': 'a9e0c8371ea1ca306d6554e3fecf50b6',
+        'info_dict': {
+            'id': '105463834',
+            'ext': 'mp4',
+            'title': 'Cam Damage-HD 720p',
+            'uploader': 'John Moyer',
+            'uploader_id': 'user32021558',
+        },
+        'add_ie': ['Vimeo'],
     }]
 
     def _real_extract(self, url):
@@ -60,6 +72,10 @@ class TumblrIE(InfoExtractor):
         if pornhub_url:
             return self.url_result(pornhub_url, 'PornHub')
 
+        vimeo_url = VimeoIE._extract_vimeo_url(url, webpage)
+        if vimeo_url:
+            return self.url_result(vimeo_url, 'Vimeo')
+
         iframe_url = self._search_regex(
             r'src=\'(https?://www\.tumblr\.com/video/[^\']+)\'',
             webpage, 'iframe url')

From 396726244a9096f142f5420ba5f3a1a36abb9a86 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 21 Jun 2015 18:53:17 +0800
Subject: [PATCH 1651/6219] [utils/ffmpeg] Move ISO 639 related codes to utils

---
 youtube_dl/postprocessor/ffmpeg.py | 196 +---------------------------
 youtube_dl/utils.py                | 202 +++++++++++++++++++++++++++++
 2 files changed, 204 insertions(+), 194 deletions(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index cc65b34e7..fe7e0a8ee 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -21,6 +21,7 @@ from ..utils import (
     shell_quote,
     subtitles_filename,
     dfxp2srt,
+    ISO639Utils,
 )
 
 
@@ -307,199 +308,6 @@ class FFmpegVideoConvertorPP(FFmpegPostProcessor):
 
 
 class FFmpegEmbedSubtitlePP(FFmpegPostProcessor):
-    # See http://www.loc.gov/standards/iso639-2/ISO-639-2_utf-8.txt
-    _lang_map = {
-        'aa': 'aar',
-        'ab': 'abk',
-        'ae': 'ave',
-        'af': 'afr',
-        'ak': 'aka',
-        'am': 'amh',
-        'an': 'arg',
-        'ar': 'ara',
-        'as': 'asm',
-        'av': 'ava',
-        'ay': 'aym',
-        'az': 'aze',
-        'ba': 'bak',
-        'be': 'bel',
-        'bg': 'bul',
-        'bh': 'bih',
-        'bi': 'bis',
-        'bm': 'bam',
-        'bn': 'ben',
-        'bo': 'bod',
-        'br': 'bre',
-        'bs': 'bos',
-        'ca': 'cat',
-        'ce': 'che',
-        'ch': 'cha',
-        'co': 'cos',
-        'cr': 'cre',
-        'cs': 'ces',
-        'cu': 'chu',
-        'cv': 'chv',
-        'cy': 'cym',
-        'da': 'dan',
-        'de': 'deu',
-        'dv': 'div',
-        'dz': 'dzo',
-        'ee': 'ewe',
-        'el': 'ell',
-        'en': 'eng',
-        'eo': 'epo',
-        'es': 'spa',
-        'et': 'est',
-        'eu': 'eus',
-        'fa': 'fas',
-        'ff': 'ful',
-        'fi': 'fin',
-        'fj': 'fij',
-        'fo': 'fao',
-        'fr': 'fra',
-        'fy': 'fry',
-        'ga': 'gle',
-        'gd': 'gla',
-        'gl': 'glg',
-        'gn': 'grn',
-        'gu': 'guj',
-        'gv': 'glv',
-        'ha': 'hau',
-        'he': 'heb',
-        'hi': 'hin',
-        'ho': 'hmo',
-        'hr': 'hrv',
-        'ht': 'hat',
-        'hu': 'hun',
-        'hy': 'hye',
-        'hz': 'her',
-        'ia': 'ina',
-        'id': 'ind',
-        'ie': 'ile',
-        'ig': 'ibo',
-        'ii': 'iii',
-        'ik': 'ipk',
-        'io': 'ido',
-        'is': 'isl',
-        'it': 'ita',
-        'iu': 'iku',
-        'ja': 'jpn',
-        'jv': 'jav',
-        'ka': 'kat',
-        'kg': 'kon',
-        'ki': 'kik',
-        'kj': 'kua',
-        'kk': 'kaz',
-        'kl': 'kal',
-        'km': 'khm',
-        'kn': 'kan',
-        'ko': 'kor',
-        'kr': 'kau',
-        'ks': 'kas',
-        'ku': 'kur',
-        'kv': 'kom',
-        'kw': 'cor',
-        'ky': 'kir',
-        'la': 'lat',
-        'lb': 'ltz',
-        'lg': 'lug',
-        'li': 'lim',
-        'ln': 'lin',
-        'lo': 'lao',
-        'lt': 'lit',
-        'lu': 'lub',
-        'lv': 'lav',
-        'mg': 'mlg',
-        'mh': 'mah',
-        'mi': 'mri',
-        'mk': 'mkd',
-        'ml': 'mal',
-        'mn': 'mon',
-        'mr': 'mar',
-        'ms': 'msa',
-        'mt': 'mlt',
-        'my': 'mya',
-        'na': 'nau',
-        'nb': 'nob',
-        'nd': 'nde',
-        'ne': 'nep',
-        'ng': 'ndo',
-        'nl': 'nld',
-        'nn': 'nno',
-        'no': 'nor',
-        'nr': 'nbl',
-        'nv': 'nav',
-        'ny': 'nya',
-        'oc': 'oci',
-        'oj': 'oji',
-        'om': 'orm',
-        'or': 'ori',
-        'os': 'oss',
-        'pa': 'pan',
-        'pi': 'pli',
-        'pl': 'pol',
-        'ps': 'pus',
-        'pt': 'por',
-        'qu': 'que',
-        'rm': 'roh',
-        'rn': 'run',
-        'ro': 'ron',
-        'ru': 'rus',
-        'rw': 'kin',
-        'sa': 'san',
-        'sc': 'srd',
-        'sd': 'snd',
-        'se': 'sme',
-        'sg': 'sag',
-        'si': 'sin',
-        'sk': 'slk',
-        'sl': 'slv',
-        'sm': 'smo',
-        'sn': 'sna',
-        'so': 'som',
-        'sq': 'sqi',
-        'sr': 'srp',
-        'ss': 'ssw',
-        'st': 'sot',
-        'su': 'sun',
-        'sv': 'swe',
-        'sw': 'swa',
-        'ta': 'tam',
-        'te': 'tel',
-        'tg': 'tgk',
-        'th': 'tha',
-        'ti': 'tir',
-        'tk': 'tuk',
-        'tl': 'tgl',
-        'tn': 'tsn',
-        'to': 'ton',
-        'tr': 'tur',
-        'ts': 'tso',
-        'tt': 'tat',
-        'tw': 'twi',
-        'ty': 'tah',
-        'ug': 'uig',
-        'uk': 'ukr',
-        'ur': 'urd',
-        'uz': 'uzb',
-        've': 'ven',
-        'vi': 'vie',
-        'vo': 'vol',
-        'wa': 'wln',
-        'wo': 'wol',
-        'xh': 'xho',
-        'yi': 'yid',
-        'yo': 'yor',
-        'za': 'zha',
-        'zh': 'zho',
-        'zu': 'zul',
-    }
-
-    @classmethod
-    def _conver_lang_code(cls, code):
-        """Convert language code from ISO 639-1 to ISO 639-2/T"""
-        return cls._lang_map.get(code[:2])
-
     def run(self, information):
         if information['ext'] not in ['mp4', 'mkv']:
             self._downloader.to_screen('[ffmpeg] Subtitles can only be embedded in mp4 or mkv files')
@@ -525,7 +333,7 @@ class FFmpegEmbedSubtitlePP(FFmpegPostProcessor):
             opts += ['-c:s', 'mov_text']
         for (i, lang) in enumerate(sub_langs):
             opts.extend(['-map', '%d:0' % (i + 1)])
-            lang_code = self._conver_lang_code(lang)
+            lang_code = ISO639Utils.short2long(lang)
             if lang_code is not None:
                 opts.extend(['-metadata:s:s:%d' % i, 'language=%s' % lang_code])
 
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 52d198fa3..259a9d634 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1879,6 +1879,208 @@ def dfxp2srt(dfxp_data):
     return ''.join(out)
 
 
+class ISO639Utils(object):
+    # See http://www.loc.gov/standards/iso639-2/ISO-639-2_utf-8.txt
+    _lang_map = {
+        'aa': 'aar',
+        'ab': 'abk',
+        'ae': 'ave',
+        'af': 'afr',
+        'ak': 'aka',
+        'am': 'amh',
+        'an': 'arg',
+        'ar': 'ara',
+        'as': 'asm',
+        'av': 'ava',
+        'ay': 'aym',
+        'az': 'aze',
+        'ba': 'bak',
+        'be': 'bel',
+        'bg': 'bul',
+        'bh': 'bih',
+        'bi': 'bis',
+        'bm': 'bam',
+        'bn': 'ben',
+        'bo': 'bod',
+        'br': 'bre',
+        'bs': 'bos',
+        'ca': 'cat',
+        'ce': 'che',
+        'ch': 'cha',
+        'co': 'cos',
+        'cr': 'cre',
+        'cs': 'ces',
+        'cu': 'chu',
+        'cv': 'chv',
+        'cy': 'cym',
+        'da': 'dan',
+        'de': 'deu',
+        'dv': 'div',
+        'dz': 'dzo',
+        'ee': 'ewe',
+        'el': 'ell',
+        'en': 'eng',
+        'eo': 'epo',
+        'es': 'spa',
+        'et': 'est',
+        'eu': 'eus',
+        'fa': 'fas',
+        'ff': 'ful',
+        'fi': 'fin',
+        'fj': 'fij',
+        'fo': 'fao',
+        'fr': 'fra',
+        'fy': 'fry',
+        'ga': 'gle',
+        'gd': 'gla',
+        'gl': 'glg',
+        'gn': 'grn',
+        'gu': 'guj',
+        'gv': 'glv',
+        'ha': 'hau',
+        'he': 'heb',
+        'hi': 'hin',
+        'ho': 'hmo',
+        'hr': 'hrv',
+        'ht': 'hat',
+        'hu': 'hun',
+        'hy': 'hye',
+        'hz': 'her',
+        'ia': 'ina',
+        'id': 'ind',
+        'ie': 'ile',
+        'ig': 'ibo',
+        'ii': 'iii',
+        'ik': 'ipk',
+        'io': 'ido',
+        'is': 'isl',
+        'it': 'ita',
+        'iu': 'iku',
+        'ja': 'jpn',
+        'jv': 'jav',
+        'ka': 'kat',
+        'kg': 'kon',
+        'ki': 'kik',
+        'kj': 'kua',
+        'kk': 'kaz',
+        'kl': 'kal',
+        'km': 'khm',
+        'kn': 'kan',
+        'ko': 'kor',
+        'kr': 'kau',
+        'ks': 'kas',
+        'ku': 'kur',
+        'kv': 'kom',
+        'kw': 'cor',
+        'ky': 'kir',
+        'la': 'lat',
+        'lb': 'ltz',
+        'lg': 'lug',
+        'li': 'lim',
+        'ln': 'lin',
+        'lo': 'lao',
+        'lt': 'lit',
+        'lu': 'lub',
+        'lv': 'lav',
+        'mg': 'mlg',
+        'mh': 'mah',
+        'mi': 'mri',
+        'mk': 'mkd',
+        'ml': 'mal',
+        'mn': 'mon',
+        'mr': 'mar',
+        'ms': 'msa',
+        'mt': 'mlt',
+        'my': 'mya',
+        'na': 'nau',
+        'nb': 'nob',
+        'nd': 'nde',
+        'ne': 'nep',
+        'ng': 'ndo',
+        'nl': 'nld',
+        'nn': 'nno',
+        'no': 'nor',
+        'nr': 'nbl',
+        'nv': 'nav',
+        'ny': 'nya',
+        'oc': 'oci',
+        'oj': 'oji',
+        'om': 'orm',
+        'or': 'ori',
+        'os': 'oss',
+        'pa': 'pan',
+        'pi': 'pli',
+        'pl': 'pol',
+        'ps': 'pus',
+        'pt': 'por',
+        'qu': 'que',
+        'rm': 'roh',
+        'rn': 'run',
+        'ro': 'ron',
+        'ru': 'rus',
+        'rw': 'kin',
+        'sa': 'san',
+        'sc': 'srd',
+        'sd': 'snd',
+        'se': 'sme',
+        'sg': 'sag',
+        'si': 'sin',
+        'sk': 'slk',
+        'sl': 'slv',
+        'sm': 'smo',
+        'sn': 'sna',
+        'so': 'som',
+        'sq': 'sqi',
+        'sr': 'srp',
+        'ss': 'ssw',
+        'st': 'sot',
+        'su': 'sun',
+        'sv': 'swe',
+        'sw': 'swa',
+        'ta': 'tam',
+        'te': 'tel',
+        'tg': 'tgk',
+        'th': 'tha',
+        'ti': 'tir',
+        'tk': 'tuk',
+        'tl': 'tgl',
+        'tn': 'tsn',
+        'to': 'ton',
+        'tr': 'tur',
+        'ts': 'tso',
+        'tt': 'tat',
+        'tw': 'twi',
+        'ty': 'tah',
+        'ug': 'uig',
+        'uk': 'ukr',
+        'ur': 'urd',
+        'uz': 'uzb',
+        've': 'ven',
+        'vi': 'vie',
+        'vo': 'vol',
+        'wa': 'wln',
+        'wo': 'wol',
+        'xh': 'xho',
+        'yi': 'yid',
+        'yo': 'yor',
+        'za': 'zha',
+        'zh': 'zho',
+        'zu': 'zul',
+    }
+
+    @classmethod
+    def short2long(cls, code):
+        """Convert language code from ISO 639-1 to ISO 639-2/T"""
+        return cls._lang_map.get(code[:2])
+
+    @classmethod
+    def long2short(cls, code):
+        """Convert language code from ISO 639-2/T to ISO 639-1"""
+        for short_name, long_name in cls._lang_map.items():
+            if long_name == code:
+                return short_name
+
+
 class PerRequestProxyHandler(compat_urllib_request.ProxyHandler):
     def __init__(self, proxies=None):
         # Set default handlers

From 607841af64d308eaf577e528fd7317a8b382b8e6 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 21 Jun 2015 18:55:26 +0800
Subject: [PATCH 1652/6219] [adobetv] Support embeddable videos (closes #6039)

---
 youtube_dl/extractor/__init__.py |  5 ++-
 youtube_dl/extractor/adobetv.py  | 59 ++++++++++++++++++++++++++++++++
 2 files changed, 63 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 6c548d8e9..0f4af88f0 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -4,7 +4,10 @@ from .abc import ABCIE
 from .abc7news import Abc7NewsIE
 from .academicearth import AcademicEarthCourseIE
 from .addanime import AddAnimeIE
-from .adobetv import AdobeTVIE
+from .adobetv import (
+    AdobeTVIE,
+    AdobeTVVideoIE,
+)
 from .adultswim import AdultSwimIE
 from .aftenposten import AftenpostenIE
 from .aftonbladet import AftonbladetIE
diff --git a/youtube_dl/extractor/adobetv.py b/youtube_dl/extractor/adobetv.py
index 97d128560..695a4a15c 100644
--- a/youtube_dl/extractor/adobetv.py
+++ b/youtube_dl/extractor/adobetv.py
@@ -5,6 +5,8 @@ from ..utils import (
     parse_duration,
     unified_strdate,
     str_to_int,
+    float_or_none,
+    ISO639Utils,
 )
 
 
@@ -69,3 +71,60 @@ class AdobeTVIE(InfoExtractor):
             'view_count': view_count,
             'formats': formats,
         }
+
+
+class AdobeTVVideoIE(InfoExtractor):
+    _VALID_URL = r'https?://video\.tv\.adobe\.com/v/(?P<id>\d+)'
+
+    _TEST = {
+        'url': 'https://video.tv.adobe.com/v/2456/',
+        'md5': '43662b577c018ad707a63766462b1e87',
+        'info_dict': {
+            'id': '2456',
+            'ext': 'mp4',
+            'title': 'New experience with Acrobat DC',
+            'description': 'New experience with Acrobat DC',
+            'duration': 248.667,
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        player_params = self._parse_json(self._search_regex(
+            r'var\s+bridge\s*=\s*([^;]+);', webpage, 'player parameters'),
+            video_id)
+
+        formats = [{
+            'url': source['src'],
+            'width': source.get('width'),
+            'height': source.get('height'),
+            'tbr': source.get('bitrate'),
+        } for source in player_params['sources']]
+
+        # For both metadata and downloaded files the duration varies among
+        # formats. I just pick the max one
+        duration = max(filter(None, [
+            float_or_none(source.get('duration'), scale=1000)
+            for source in player_params['sources']]))
+
+        subtitles = {}
+        for translation in player_params.get('translations', []):
+            lang_id = translation.get('language_w3c') or ISO639Utils.long2short(translation['language_medium'])
+            if lang_id not in subtitles:
+                subtitles[lang_id] = []
+            subtitles[lang_id].append({
+                'url': translation['vttPath'],
+                'ext': 'vtt',
+            })
+
+        return {
+            'id': video_id,
+            'formats': formats,
+            'title': player_params['title'],
+            'description': self._og_search_description(webpage),
+            'duration': duration,
+            'subtitles': subtitles,
+        }

From 4e3357717312ac56145ba166a1ae2806f6db8337 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 21 Jun 2015 19:16:59 +0800
Subject: [PATCH 1653/6219] [utils] Support ttaf1 namespace in TTML

It's found in bbc.co.uk. See #6038
---
 youtube_dl/utils.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 259a9d634..a2746b2d1 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1841,7 +1841,10 @@ def srt_subtitles_timecode(seconds):
 
 
 def dfxp2srt(dfxp_data):
-    _x = functools.partial(xpath_with_ns, ns_map={'ttml': 'http://www.w3.org/ns/ttml'})
+    _x = functools.partial(xpath_with_ns, ns_map={
+        'ttml': 'http://www.w3.org/ns/ttml',
+        'ttaf1': 'http://www.w3.org/2006/10/ttaf1',
+    })
 
     def parse_node(node):
         str_or_empty = functools.partial(str_or_none, default='')
@@ -1849,9 +1852,9 @@ def dfxp2srt(dfxp_data):
         out = str_or_empty(node.text)
 
         for child in node:
-            if child.tag in (_x('ttml:br'), 'br'):
+            if child.tag in (_x('ttml:br'), _x('ttaf1:br'), 'br'):
                 out += '\n' + str_or_empty(child.tail)
-            elif child.tag in (_x('ttml:span'), 'span'):
+            elif child.tag in (_x('ttml:span'), _x('ttaf1:span'), 'span'):
                 out += str_or_empty(parse_node(child))
             else:
                 out += str_or_empty(xml.etree.ElementTree.tostring(child))
@@ -1860,7 +1863,7 @@ def dfxp2srt(dfxp_data):
 
     dfxp = xml.etree.ElementTree.fromstring(dfxp_data.encode('utf-8'))
     out = []
-    paras = dfxp.findall(_x('.//ttml:p')) or dfxp.findall('.//p')
+    paras = dfxp.findall(_x('.//ttml:p')) or dfxp.findall(_x('.//ttaf1:p')) or dfxp.findall('.//p')
 
     if not paras:
         raise ValueError('Invalid dfxp/TTML subtitle')

From 78294e6a9ce2c9a294d663ac79936df7353b9980 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 21 Jun 2015 19:22:26 +0800
Subject: [PATCH 1654/6219] [bbccouk] Remove TTML to srt conversion codes

It's broken. See #6038
---
 youtube_dl/extractor/bbccouk.py | 15 ---------------
 1 file changed, 15 deletions(-)

diff --git a/youtube_dl/extractor/bbccouk.py b/youtube_dl/extractor/bbccouk.py
index 0305f88b5..5825d2867 100644
--- a/youtube_dl/extractor/bbccouk.py
+++ b/youtube_dl/extractor/bbccouk.py
@@ -251,26 +251,11 @@ class BBCCoUkIE(InfoExtractor):
         for connection in self._extract_connections(media):
             captions = self._download_xml(connection.get('href'), programme_id, 'Downloading captions')
             lang = captions.get('{http://www.w3.org/XML/1998/namespace}lang', 'en')
-            ps = captions.findall('./{0}body/{0}div/{0}p'.format('{http://www.w3.org/2006/10/ttaf1}'))
-            srt = ''
-
-            def _extract_text(p):
-                if p.text is not None:
-                    stripped_text = p.text.strip()
-                    if stripped_text:
-                        return stripped_text
-                return ' '.join(span.text.strip() for span in p.findall('{http://www.w3.org/2006/10/ttaf1}span'))
-            for pos, p in enumerate(ps):
-                srt += '%s\r\n%s --> %s\r\n%s\r\n\r\n' % (str(pos), p.get('begin'), p.get('end'), _extract_text(p))
             subtitles[lang] = [
                 {
                     'url': connection.get('href'),
                     'ext': 'ttml',
                 },
-                {
-                    'data': srt,
-                    'ext': 'srt',
-                },
             ]
         return subtitles
 

From 756f574e4e7160ca5b39c6e18ec5168beb4a8eb1 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 21 Jun 2015 21:30:34 +0800
Subject: [PATCH 1655/6219] [dailymotion/generic] Add DailymotionCloudIE

---
 youtube_dl/extractor/__init__.py    |  1 +
 youtube_dl/extractor/dailymotion.py | 42 +++++++++++++++++++++++++++++
 youtube_dl/extractor/generic.py     | 17 ++++++++++++
 3 files changed, 60 insertions(+)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 0f4af88f0..bd3c3193f 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -106,6 +106,7 @@ from .dailymotion import (
     DailymotionIE,
     DailymotionPlaylistIE,
     DailymotionUserIE,
+    DailymotionCloudIE,
 )
 from .daum import DaumIE
 from .dbtv import DBTVIE
diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index 70aa4333c..96f0ed9ad 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -251,3 +251,45 @@ class DailymotionUserIE(DailymotionPlaylistIE):
             'title': full_user,
             'entries': self._extract_entries(user),
         }
+
+
+class DailymotionCloudIE(DailymotionBaseInfoExtractor):
+    _VALID_URL = r'http://api\.dmcloud\.net/embed/[^/]+/(?P<id>[^/?]+)'
+
+    _TEST = {
+        # From http://www.francetvinfo.fr/economie/entreprises/les-entreprises-familiales-le-secret-de-la-reussite_933271.html
+        # Tested at FranceTvInfo_2
+        'url': 'http://api.dmcloud.net/embed/4e7343f894a6f677b10006b4/556e03339473995ee145930c?auth=1464865870-0-jyhsm84b-ead4c701fb750cf9367bf4447167a3db&autoplay=1',
+        'only_matching': True,
+    }
+
+    @classmethod
+    def _extract_dmcloud_url(self, webpage):
+        mobj = re.search(r'<iframe[^>]+src=[\'"](http://api\.dmcloud\.net/embed/[^/]+/[^\'"]+)[\'"]', webpage)
+        if mobj:
+            return mobj.group(1)
+
+        mobj = re.search(r'<input[^>]+id=[\'"]dmcloudUrlEmissionSelect[\'"][^>]+value=[\'"](http://api\.dmcloud\.net/embed/[^/]+/[^\'"]+)[\'"]', webpage)
+        if mobj:
+            return mobj.group(1)
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        request = self._build_request(url)
+        webpage = self._download_webpage(request, video_id)
+
+        title = self._html_search_regex(r'<title>([^>]+)</title>', webpage, 'title')
+
+        video_info = self._parse_json(self._search_regex(
+            r'var\s+info\s*=\s*([^;]+);', webpage, 'video info'), video_id)
+
+        # TODO: parse ios_url, which is in fact a manifest
+        video_url = video_info['mp4_url']
+
+        return {
+            'id': video_id,
+            'url': video_url,
+            'title': title,
+            'thumbnail': video_info.get('thumbnail_url'),
+        }
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index bf689f531..07939b196 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -44,6 +44,7 @@ from .bliptv import BlipTVIE
 from .svt import SVTIE
 from .pornhub import PornHubIE
 from .vimeo import VimeoIE
+from .dailymotion import DailymotionCloudIE
 
 
 class GenericIE(InfoExtractor):
@@ -813,6 +814,17 @@ class GenericIE(InfoExtractor):
                 'description': 'To understand why he was the Toronto Blue Jays’ top off-season priority is to appreciate his background and upbringing in Montreal, where he first developed his baseball skills. Written and narrated by Stephen Brunt.',
                 'uploader': 'Rogers Sportsnet',
             },
+        },
+        # Dailymotion Cloud video
+        {
+            'url': 'http://replay.publicsenat.fr/vod/le-debat/florent-kolandjian,dominique-cena,axel-decourtye,laurence-abeille,bruno-parmentier/175910',
+            'md5': '49444254273501a64675a7e68c502681',
+            'info_dict': {
+                'id': '5585de919473990de4bee11b',
+                'ext': 'mp4',
+                'title': 'Le débat',
+                'thumbnail': 're:^https?://.*\.jpe?g$',
+            }
         }
     ]
 
@@ -1486,6 +1498,11 @@ class GenericIE(InfoExtractor):
         if senate_isvp_url:
             return self.url_result(senate_isvp_url, 'SenateISVP')
 
+        # Look for Dailymotion Cloud videos
+        dmcloud_url = DailymotionCloudIE._extract_dmcloud_url(webpage)
+        if dmcloud_url:
+            return self.url_result(dmcloud_url, 'DailymotionCloud')
+
         def check_video(vurl):
             if YoutubeIE.suitable(vurl):
                 return True

From 6f96e308d0fa7674ac88e1e80fc602413f9a6b31 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 21 Jun 2015 21:31:33 +0800
Subject: [PATCH 1656/6219] [francetvinfo.fr] Support dmcloud embeds (fixes
 #6034)

---
 youtube_dl/extractor/francetv.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index db0bbec1e..b2c984bf2 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -18,6 +18,7 @@ from ..utils import (
     parse_duration,
     determine_ext,
 )
+from .dailymotion import DailymotionCloudIE
 
 
 class FranceTVBaseInfoExtractor(InfoExtractor):
@@ -131,12 +132,26 @@ class FranceTvInfoIE(FranceTVBaseInfoExtractor):
             'skip_download': 'HLS (reqires ffmpeg)'
         },
         'skip': 'Ce direct est terminé et sera disponible en rattrapage dans quelques minutes.',
+    }, {
+        'url': 'http://www.francetvinfo.fr/economie/entreprises/les-entreprises-familiales-le-secret-de-la-reussite_933271.html',
+        'md5': 'f485bda6e185e7d15dbc69b72bae993e',
+        'info_dict': {
+            'id': '556e03339473995ee145930c',
+            'ext': 'mp4',
+            'title': 'Les entreprises familiales : le secret de la réussite',
+            'thumbnail': 're:^https?://.*\.jpe?g$',
+        }
     }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         page_title = mobj.group('title')
         webpage = self._download_webpage(url, page_title)
+
+        dmcloud_url = DailymotionCloudIE._extract_dmcloud_url(webpage)
+        if dmcloud_url:
+            return self.url_result(dmcloud_url, 'DailymotionCloud')
+
         video_id, catalogue = self._search_regex(
             r'id-video=([^@]+@[^"]+)', webpage, 'video id').split('@')
         return self._extract_video(video_id, catalogue)

From 3f3308cd75fc068e4d67d00aa7d7892e02ab16e9 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 21 Jun 2015 23:29:40 +0800
Subject: [PATCH 1657/6219] Revert "[sohu] Update extractor"

This reverts commit 32060c6d6b618fa858b2ce43db34d02fd43bc542.
---
 youtube_dl/extractor/sohu.py | 46 ++++++++++++++++++++----------------
 1 file changed, 26 insertions(+), 20 deletions(-)

diff --git a/youtube_dl/extractor/sohu.py b/youtube_dl/extractor/sohu.py
index 29bd9ce6f..7644cc02d 100644
--- a/youtube_dl/extractor/sohu.py
+++ b/youtube_dl/extractor/sohu.py
@@ -8,7 +8,10 @@ from ..compat import (
     compat_str,
     compat_urllib_request
 )
-from ..utils import ExtractorError
+from ..utils import (
+    sanitize_url_path_consecutive_slashes,
+    ExtractorError,
+)
 
 
 class SohuIE(InfoExtractor):
@@ -26,7 +29,7 @@ class SohuIE(InfoExtractor):
         'skip': 'On available in China',
     }, {
         'url': 'http://tv.sohu.com/20150305/n409385080.shtml',
-        'md5': 'ac9a5d322b4bf9ae184d53e4711e4f1a',
+        'md5': '699060e75cf58858dd47fb9c03c42cfb',
         'info_dict': {
             'id': '409385080',
             'ext': 'mp4',
@@ -34,7 +37,7 @@ class SohuIE(InfoExtractor):
         }
     }, {
         'url': 'http://my.tv.sohu.com/us/232799889/78693464.shtml',
-        'md5': '49308ff6dafde5ece51137d04aec311e',
+        'md5': '9bf34be48f2f4dadcb226c74127e203c',
         'info_dict': {
             'id': '78693464',
             'ext': 'mp4',
@@ -48,7 +51,7 @@ class SohuIE(InfoExtractor):
             'title': '【神探苍实战秘籍】第13期 战争之影 赫卡里姆',
         },
         'playlist': [{
-            'md5': '492923eac023ba2f13ff69617c32754a',
+            'md5': 'bdbfb8f39924725e6589c146bc1883ad',
             'info_dict': {
                 'id': '78910339_part1',
                 'ext': 'mp4',
@@ -56,7 +59,7 @@ class SohuIE(InfoExtractor):
                 'title': '【神探苍实战秘籍】第13期 战争之影 赫卡里姆',
             }
         }, {
-            'md5': 'de604848c0e8e9c4a4dde7e1347c0637',
+            'md5': '3e1f46aaeb95354fd10e7fca9fc1804e',
             'info_dict': {
                 'id': '78910339_part2',
                 'ext': 'mp4',
@@ -64,7 +67,7 @@ class SohuIE(InfoExtractor):
                 'title': '【神探苍实战秘籍】第13期 战争之影 赫卡里姆',
             }
         }, {
-            'md5': '93584716ee0657c0b205b8aa3d27aa13',
+            'md5': '8407e634175fdac706766481b9443450',
             'info_dict': {
                 'id': '78910339_part3',
                 'ext': 'mp4',
@@ -139,21 +142,24 @@ class SohuIE(InfoExtractor):
         for i in range(part_count):
             formats = []
             for format_id, format_data in formats_json.items():
-                data = format_data['data']
+                allot = format_data['allot']
+                prot = format_data['prot']
 
-                # URLs starts with http://newflv.sohu.ccgslb.net/ is not usable
-                # so retry until got a working URL
-                video_url = 'newflv.sohu.ccgslb.net'
-                retries = 0
-                while 'newflv.sohu.ccgslb.net' in video_url and retries < 5:
-                    download_note = 'Download information from CDN gateway for format ' + format_id
-                    if retries > 0:
-                        download_note += ' (retry #%d)' % retries
-                    retries += 1
-                    cdn_info = self._download_json(
-                        'http://data.vod.itc.cn/cdnList?new=' + data['su'][i],
-                        video_id, download_note)
-                    video_url = cdn_info['url']
+                data = format_data['data']
+                clips_url = data['clipsURL']
+                su = data['su']
+
+                part_str = self._download_webpage(
+                    'http://%s/?prot=%s&file=%s&new=%s' %
+                    (allot, prot, clips_url[i], su[i]),
+                    video_id,
+                    'Downloading %s video URL part %d of %d'
+                    % (format_id, i + 1, part_count))
+
+                part_info = part_str.split('|')
+
+                video_url = sanitize_url_path_consecutive_slashes(
+                    '%s%s?key=%s' % (part_info[0], su[i], part_info[3]))
 
                 formats.append({
                     'url': video_url,

From 98ca102441624c2a1a66114c32e0142899f78dd3 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 22 Jun 2015 00:59:55 +0800
Subject: [PATCH 1658/6219] [sohu] Fix extraction again

---
 youtube_dl/extractor/sohu.py | 42 +++++++++++++++++++++++++-----------
 1 file changed, 30 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/sohu.py b/youtube_dl/extractor/sohu.py
index 7644cc02d..ba2d5e19b 100644
--- a/youtube_dl/extractor/sohu.py
+++ b/youtube_dl/extractor/sohu.py
@@ -6,10 +6,10 @@ import re
 from .common import InfoExtractor
 from ..compat import (
     compat_str,
-    compat_urllib_request
+    compat_urllib_request,
+    compat_urllib_parse,
 )
 from ..utils import (
-    sanitize_url_path_consecutive_slashes,
     ExtractorError,
 )
 
@@ -143,23 +143,41 @@ class SohuIE(InfoExtractor):
             formats = []
             for format_id, format_data in formats_json.items():
                 allot = format_data['allot']
-                prot = format_data['prot']
 
                 data = format_data['data']
                 clips_url = data['clipsURL']
                 su = data['su']
 
-                part_str = self._download_webpage(
-                    'http://%s/?prot=%s&file=%s&new=%s' %
-                    (allot, prot, clips_url[i], su[i]),
-                    video_id,
-                    'Downloading %s video URL part %d of %d'
-                    % (format_id, i + 1, part_count))
+                video_url = 'newflv.sohu.ccgslb.net'
+                cdnId = None
+                retries = 0
 
-                part_info = part_str.split('|')
+                while 'newflv.sohu.ccgslb.net' in video_url:
+                    params = {
+                        'prot': 9,
+                        'file': clips_url[i],
+                        'new': su[i],
+                        'prod': 'flash',
+                    }
 
-                video_url = sanitize_url_path_consecutive_slashes(
-                    '%s%s?key=%s' % (part_info[0], su[i], part_info[3]))
+                    if cdnId is not None:
+                        params['idc'] = cdnId
+
+                    download_note = 'Downloading %s video URL part %d of %d' % (
+                        format_id, i + 1, part_count)
+
+                    if retries > 0:
+                        download_note += ' (retry #%d)' % retries
+                    part_info = self._parse_json(self._download_webpage(
+                        'http://%s/?%s' % (allot, compat_urllib_parse.urlencode(params)),
+                        video_id, download_note), video_id)
+
+                    video_url = part_info['url']
+                    cdnId = part_info.get('nid')
+
+                    retries += 1
+                    if retries > 5:
+                        raise ExtractorError('Failed to get video URL')
 
                 formats.append({
                     'url': video_url,

From 0bbba43ed0c68b612fcafbdad460a93b733b9f87 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 21 Jun 2015 23:10:38 +0600
Subject: [PATCH 1659/6219] [xhamster:embed] Add extractor (Closes #6032)

---
 youtube_dl/extractor/__init__.py |  5 ++++-
 youtube_dl/extractor/xhamster.py | 28 +++++++++++++++++++++++++++-
 2 files changed, 31 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index bd3c3193f..dc1a302e6 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -701,7 +701,10 @@ from .wrzuta import WrzutaIE
 from .wsj import WSJIE
 from .xbef import XBefIE
 from .xboxclips import XboxClipsIE
-from .xhamster import XHamsterIE
+from .xhamster import (
+    XHamsterIE,
+    XHamsterEmbedIE,
+)
 from .xminus import XMinusIE
 from .xnxx import XNXXIE
 from .xstream import XstreamIE
diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index 4527567f8..ae3c5962b 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -13,7 +13,6 @@ from ..utils import (
 
 
 class XHamsterIE(InfoExtractor):
-    """Information Extractor for xHamster"""
     _VALID_URL = r'(?P<proto>https?)://(?:.+?\.)?xhamster\.com/movies/(?P<id>[0-9]+)/(?P<seo>.+?)\.html(?:\?.*)?'
     _TESTS = [
         {
@@ -133,3 +132,30 @@ class XHamsterIE(InfoExtractor):
             'age_limit': age_limit,
             'formats': formats,
         }
+
+
+class XHamsterEmbedIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?xhamster\.com/xembed\.php\?video=(?P<id>\d+)'
+    _TEST = {
+        'url': 'http://xhamster.com/xembed.php?video=3328539',
+        'info_dict': {
+            'id': '3328539',
+            'ext': 'mp4',
+            'title': 'Pen Masturbation',
+            'upload_date': '20140728',
+            'uploader_id': 'anonymous',
+            'duration': 5,
+            'age_limit': 18,
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        video_url = self._search_regex(
+            r'href="(https?://xhamster\.com/movies/%s/[^"]+\.html[^"]*)"' % video_id,
+            webpage, 'xhamster url')
+
+        return self.url_result(video_url, 'XHamster');

From 2bb5b6d0a1671957c7a2e6d6433901b2a1b8f48f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 21 Jun 2015 23:11:25 +0600
Subject: [PATCH 1660/6219] [generic] Add support for xhamster embeds

---
 youtube_dl/extractor/generic.py  | 6 ++++++
 youtube_dl/extractor/xhamster.py | 6 ++++++
 2 files changed, 12 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 07939b196..e108bde66 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -43,6 +43,7 @@ from .senateisvp import SenateISVPIE
 from .bliptv import BlipTVIE
 from .svt import SVTIE
 from .pornhub import PornHubIE
+from .xhamster import XHamsterEmbedIE
 from .vimeo import VimeoIE
 from .dailymotion import DailymotionCloudIE
 
@@ -1331,6 +1332,11 @@ class GenericIE(InfoExtractor):
         if pornhub_url:
             return self.url_result(pornhub_url, 'PornHub')
 
+        # Look for embedded XHamster player
+        xhamster_urls = XHamsterEmbedIE._extract_urls(webpage)
+        if xhamster_urls:
+            return _playlist_from_matches(xhamster_urls, ie='XHamsterEmbed')
+
         # Look for embedded Tvigle player
         mobj = re.search(
             r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//cloud\.tvigle\.ru/video/.+?)\1', webpage)
diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index ae3c5962b..725e01ac3 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -149,6 +149,12 @@ class XHamsterEmbedIE(InfoExtractor):
         }
     }
 
+    @staticmethod
+    def _extract_urls(webpage):
+        return [url for _, url in re.findall(
+            r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?xhamster\.com/xembed\.php\?video=\d+)\1',
+            webpage)]
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
 

From c76799c5553b3b48bb7cc73dec452c3637a8670a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 21 Jun 2015 23:18:28 +0600
Subject: [PATCH 1661/6219] [extractor/generic] Add test for xhamster embed

---
 youtube_dl/extractor/generic.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index e108bde66..20fcd8170 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -336,6 +336,15 @@ class GenericIE(InfoExtractor):
                 'skip_download': True,
             },
         },
+        # XHamster embed
+        {
+            'url': 'http://www.numisc.com/forum/showthread.php?11696-FM15-which-pumiscer-was-this-%28-vid-%29-%28-alfa-as-fuck-srx-%29&s=711f5db534502e22260dec8c5e2d66d8',
+            'info_dict': {
+                'id': 'showthread',
+                'title': '[NSFL] [FM15] which pumiscer was this ( vid ) ( alfa as fuck srx )',
+            },
+            'playlist_mincount': 7,
+        },
         # Embedded TED video
         {
             'url': 'http://en.support.wordpress.com/videos/ted-talks/',

From a5158f38a31e863a39de8f66c26469a5d4469280 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 22 Jun 2015 15:02:53 +0800
Subject: [PATCH 1662/6219] [generic/adobetv] Support AdobeTVVideo embeds
 (#6039)

---
 youtube_dl/extractor/adobetv.py |  1 +
 youtube_dl/extractor/generic.py | 21 +++++++++++++++++++++
 2 files changed, 22 insertions(+)

diff --git a/youtube_dl/extractor/adobetv.py b/youtube_dl/extractor/adobetv.py
index 695a4a15c..5e43adc51 100644
--- a/youtube_dl/extractor/adobetv.py
+++ b/youtube_dl/extractor/adobetv.py
@@ -77,6 +77,7 @@ class AdobeTVVideoIE(InfoExtractor):
     _VALID_URL = r'https?://video\.tv\.adobe\.com/v/(?P<id>\d+)'
 
     _TEST = {
+        # From https://helpx.adobe.com/acrobat/how-to/new-experience-acrobat-dc.html?set=acrobat--get-started--essential-beginners
         'url': 'https://video.tv.adobe.com/v/2456/',
         'md5': '43662b577c018ad707a63766462b1e87',
         'info_dict': {
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 20fcd8170..5c03fddc6 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -835,6 +835,18 @@ class GenericIE(InfoExtractor):
                 'title': 'Le débat',
                 'thumbnail': 're:^https?://.*\.jpe?g$',
             }
+        },
+        # AdobeTVVideo embed
+        {
+            'url': 'https://helpx.adobe.com/acrobat/how-to/new-experience-acrobat-dc.html?set=acrobat--get-started--essential-beginners',
+            'md5': '43662b577c018ad707a63766462b1e87',
+            'info_dict': {
+                'id': '2456',
+                'ext': 'mp4',
+                'title': 'New experience with Acrobat DC',
+                'description': 'New experience with Acrobat DC',
+                'duration': 248.667,
+            },
         }
     ]
 
@@ -1518,6 +1530,15 @@ class GenericIE(InfoExtractor):
         if dmcloud_url:
             return self.url_result(dmcloud_url, 'DailymotionCloud')
 
+        # Look for AdobeTVVideo embeds
+        mobj = re.search(
+            r'<iframe[^>]+src=[\'"]((?:https?:)?//video\.tv\.adobe\.com/v/\d+[^"]+)[\'"]',
+            webpage)
+        if mobj is not None:
+            return self.url_result(
+                self._proto_relative_url(unescapeHTML(mobj.group(1))),
+                'AdobeTVVideo')
+
         def check_video(vurl):
             if YoutubeIE.suitable(vurl):
                 return True

From 25701d5a2ca8c7a58c91e11c6a30d4e61b02e89c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 22 Jun 2015 11:18:52 +0200
Subject: [PATCH 1663/6219] [xhamster] pep8: remove trailing ';'

---
 youtube_dl/extractor/xhamster.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index 725e01ac3..b4ad513a0 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -164,4 +164,4 @@ class XHamsterEmbedIE(InfoExtractor):
             r'href="(https?://xhamster\.com/movies/%s/[^"]+\.html[^"]*)"' % video_id,
             webpage, 'xhamster url')
 
-        return self.url_result(video_url, 'XHamster');
+        return self.url_result(video_url, 'XHamster')

From 255f5694aabe07fa7f33978c6b97ced469e172db Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 22 Jun 2015 20:11:15 +0600
Subject: [PATCH 1664/6219] [faz] Extend _VALID_URL (Closes #6050)

---
 youtube_dl/extractor/faz.py | 21 ++++++++++++++++++---
 1 file changed, 18 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/faz.py b/youtube_dl/extractor/faz.py
index 3c39ca451..cebdd0193 100644
--- a/youtube_dl/extractor/faz.py
+++ b/youtube_dl/extractor/faz.py
@@ -6,9 +6,9 @@ from .common import InfoExtractor
 
 class FazIE(InfoExtractor):
     IE_NAME = 'faz.net'
-    _VALID_URL = r'https?://www\.faz\.net/multimedia/videos/.*?-(?P<id>\d+)\.html'
+    _VALID_URL = r'https?://(?:www\.)?faz\.net/(?:[^/]+/)*.*?-(?P<id>\d+)\.html'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.faz.net/multimedia/videos/stockholm-chemie-nobelpreis-fuer-drei-amerikanische-forscher-12610585.html',
         'info_dict': {
             'id': '12610585',
@@ -16,7 +16,22 @@ class FazIE(InfoExtractor):
             'title': 'Stockholm: Chemie-Nobelpreis für drei amerikanische Forscher',
             'description': 'md5:1453fbf9a0d041d985a47306192ea253',
         },
-    }
+    }, {
+        'url': 'http://www.faz.net/aktuell/politik/berlin-gabriel-besteht-zerreissprobe-ueber-datenspeicherung-13659345.html',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.faz.net/berlin-gabriel-besteht-zerreissprobe-ueber-datenspeicherung-13659345.html',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.faz.net/-13659345.html',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.faz.net/aktuell/politik/-13659345.html',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.faz.net/foobarblafasel-13659345.html',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From a9dcf4a860214e37971ab05f27f74bbae65ff8ae Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Tue, 23 Jun 2015 01:08:07 -0500
Subject: [PATCH 1665/6219] Prefer externalId over non-mediaserver-specific
 hashkey for video id.

---
 youtube_dl/extractor/bbc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 310db9d1d..fed344ea0 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -540,7 +540,7 @@ class BBCNewsIE(BBCCoUkIE):
             self._sort_formats(formats)
 
             ret.append( {
-                'id': jent.get('programme_id',jent.get('id')),
+                'id': jent.get('id') if programme_id == None else programme_id,
                 'uploader': 'BBC News',
                 'upload_date': pubdate,
                 'title': title,

From e20d0c1e69f66a82dd493680351538ea92393fd0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 23 Jun 2015 21:34:29 +0600
Subject: [PATCH 1666/6219] [brightcove] Use `compat_xml_parse_error` (Closes
 #6060)

---
 youtube_dl/extractor/brightcove.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index d768f99e6..4721c2293 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -13,6 +13,7 @@ from ..compat import (
     compat_urllib_parse_urlparse,
     compat_urllib_request,
     compat_urlparse,
+    compat_xml_parse_error,
 )
 from ..utils import (
     determine_ext,
@@ -119,7 +120,7 @@ class BrightcoveIE(InfoExtractor):
 
         try:
             object_doc = xml.etree.ElementTree.fromstring(object_str.encode('utf-8'))
-        except xml.etree.ElementTree.ParseError:
+        except compat_xml_parse_error:
             return
 
         fv_el = find_xpath_attr(object_doc, './param', 'name', 'flashVars')

From 2fece970b80022574a6b54c936820897cfd10d70 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 24 Jun 2015 03:08:24 +0600
Subject: [PATCH 1667/6219] [extractor/generic] Clarify generic extraction
 warning

---
 youtube_dl/extractor/generic.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 3d672197c..c8582bda9 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -977,10 +977,10 @@ class GenericIE(InfoExtractor):
                 'upload_date': upload_date,
             }
 
-        if (not self._downloader.params.get('test', False) and
-                not is_intentional and
-                not self._downloader.params.get('force_generic_extractor', False)):
-            self._downloader.report_warning('Falling back on generic information extractor.')
+        if not self._downloader.params.get('test', False) and not is_intentional:
+            force = self._downloader.params.get('force_generic_extractor', False)
+            self._downloader.report_warning(
+                '%s on generic information extractor.' % ('Forcing' if force else 'Falling back'))
 
         if not full_response:
             request = compat_urllib_request.Request(url)

From 18b5e1e5348ba3a6d1b6a98e97217eebb3d32a1e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 24 Jun 2015 16:00:12 +0800
Subject: [PATCH 1668/6219] [drbonanza] Fix extraction of videos

---
 youtube_dl/extractor/drbonanza.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/drbonanza.py b/youtube_dl/extractor/drbonanza.py
index 7626219ba..8b98b013a 100644
--- a/youtube_dl/extractor/drbonanza.py
+++ b/youtube_dl/extractor/drbonanza.py
@@ -15,7 +15,6 @@ class DRBonanzaIE(InfoExtractor):
 
     _TESTS = [{
         'url': 'http://www.dr.dk/bonanza/serie/portraetter/Talkshowet.htm?assetId=65517',
-        'md5': 'fe330252ddea607635cf2eb2c99a0af3',
         'info_dict': {
             'id': '65517',
             'ext': 'mp4',
@@ -26,6 +25,9 @@ class DRBonanzaIE(InfoExtractor):
             'upload_date': '20110120',
             'duration': 3664,
         },
+        'params': {
+            'skip_download': True,  # requires rtmp
+        },
     }, {
         'url': 'http://www.dr.dk/bonanza/radio/serie/sport/fodbold.htm?assetId=59410',
         'md5': '6dfe039417e76795fb783c52da3de11d',
@@ -93,6 +95,11 @@ class DRBonanzaIE(InfoExtractor):
                         'format_id': file['Type'].replace('Video', ''),
                         'preference': preferencemap.get(file['Type'], -10),
                     })
+                    if format['url'].startswith('rtmp'):
+                        rtmp_url = format['url']
+                        format['rtmp_live'] = True  # --resume does not work
+                        if '/bonanza/' in rtmp_url:
+                            format['play_path'] = rtmp_url.split('/bonanza/')[1]
                     formats.append(format)
                 elif file['Type'] == "Thumb":
                     thumbnail = file['Location']
@@ -111,9 +118,6 @@ class DRBonanzaIE(InfoExtractor):
         description = '%s\n%s\n%s\n' % (
             info['Description'], info['Actors'], info['Colophon'])
 
-        for f in formats:
-            f['url'] = f['url'].replace('rtmp://vod-bonanza.gss.dr.dk/bonanza/', 'http://vodfiles.dr.dk/')
-            f['url'] = f['url'].replace('mp4:bonanza', 'bonanza')
         self._sort_formats(formats)
 
         display_id = re.sub(r'[^\w\d-]', '', re.sub(r' ', '-', title.lower())) + '-' + asset_id

From 1a1251e87716ba02bfe6c4c9af2726e413e00208 Mon Sep 17 00:00:00 2001
From: Mitsukarenai <mitsu@suumitsu.eu>
Date: Wed, 24 Jun 2015 10:40:01 +0200
Subject: [PATCH 1669/6219] [noco.tv] Fix issue #6066: title interpreted as
 integer

---
 youtube_dl/extractor/noco.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/noco.py b/youtube_dl/extractor/noco.py
index 5bbd2dcf6..a53e27b27 100644
--- a/youtube_dl/extractor/noco.py
+++ b/youtube_dl/extractor/noco.py
@@ -195,7 +195,7 @@ class NocoIE(InfoExtractor):
         if episode_number:
             title += ' #' + compat_str(episode_number)
         if episode:
-            title += ' - ' + episode
+            title += ' - ' + compat_str(episode)
 
         description = show.get('show_resume') or show.get('family_resume')
 

From a219d175c66680968d5ce7de0a2e619623171a40 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 24 Jun 2015 20:48:57 +0600
Subject: [PATCH 1670/6219] [soundcloud] Improve _VALID_URL

Do not takeover soundcloud:user URLs
---
 youtube_dl/extractor/soundcloud.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index c23c5ee0f..e31b4596a 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -29,7 +29,7 @@ class SoundcloudIE(InfoExtractor):
     _VALID_URL = r'''(?x)^(?:https?://)?
                     (?:(?:(?:www\.|m\.)?soundcloud\.com/
                             (?P<uploader>[\w\d-]+)/
-                            (?!sets/|likes/?(?:$|[?#]))
+                            (?!sets/|(?:likes|tracks)/?(?:$|[?#]))
                             (?P<title>[\w\d-]+)/?
                             (?P<token>[^?]+?)?(?:[?].*)?$)
                        |(?:api\.soundcloud\.com/tracks/(?P<track_id>\d+)

From 03b9c94437756c3f715b29150daa59a8ea97eebd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 24 Jun 2015 20:49:22 +0600
Subject: [PATCH 1671/6219] [soundcloud:user] Add test

---
 youtube_dl/extractor/soundcloud.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index e31b4596a..118ca4832 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -307,6 +307,9 @@ class SoundcloudUserIE(SoundcloudIE):
             'title': 'The Royal Concept',
         },
         'playlist_mincount': 1,
+    }, {
+        'url': 'https://soundcloud.com/the-akashic-chronicler/tracks',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From f843300fe56ffbfc8e3005fd0f7a8237e5deaaae Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 24 Jun 2015 23:12:13 +0600
Subject: [PATCH 1672/6219] [onionstudios] Add extractor

---
 youtube_dl/extractor/__init__.py     |  1 +
 youtube_dl/extractor/onionstudios.py | 67 ++++++++++++++++++++++++++++
 2 files changed, 68 insertions(+)
 create mode 100644 youtube_dl/extractor/onionstudios.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index dc1a302e6..46cc4cd06 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -388,6 +388,7 @@ from .nytimes import (
 from .nuvid import NuvidIE
 from .odnoklassniki import OdnoklassnikiIE
 from .oktoberfesttv import OktoberfestTVIE
+from .onionstudios import OnionStudiosIE
 from .ooyala import (
     OoyalaIE,
     OoyalaExternalIE,
diff --git a/youtube_dl/extractor/onionstudios.py b/youtube_dl/extractor/onionstudios.py
new file mode 100644
index 000000000..d5d03fd44
--- /dev/null
+++ b/youtube_dl/extractor/onionstudios.py
@@ -0,0 +1,67 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import determine_ext
+
+
+class OnionStudiosIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?onionstudios\.com/(?:videos/[^/]+-|embed\?.*\bid=)(?P<id>\d+)(?!-)'
+
+    _TESTS = [{
+        'url': 'http://www.onionstudios.com/videos/hannibal-charges-forward-stops-for-a-cocktail-2937',
+        'md5': 'd4851405d31adfadf71cd7a487b765bb',
+        'info_dict': {
+            'id': '2937',
+            'ext': 'mp4',
+            'title': 'Hannibal charges forward, stops for a cocktail',
+            'description': 'md5:545299bda6abf87e5ec666548c6a9448',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'uploader': 'The A.V. Club',
+            'uploader_id': 'TheAVClub',
+        },
+    }, {
+        'url': 'http://www.onionstudios.com/embed?id=2855&autoplay=true',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(
+            'http://www.onionstudios.com/embed?id=%s' % video_id, video_id)
+
+        formats = []
+        for src in re.findall(r'<source[^>]+src="([^"]+)"', webpage):
+            if determine_ext(src) != 'm3u8':  # m3u8 always results in 403
+                formats.append({
+                    'url': src,
+                })
+        self._sort_formats(formats)
+
+        title = self._search_regex(
+            r'share_title\s*=\s*"([^"]+)"', webpage, 'title')
+        description = self._search_regex(
+            r'share_description\s*=\s*"([^"]+)"', webpage,
+            'description', default=None)
+        thumbnail = self._search_regex(
+            r'poster="([^"]+)"', webpage, 'thumbnail', default=False)
+
+        uploader_id = self._search_regex(
+            r'twitter_handle\s*=\s*"([^"]+)"',
+            webpage, 'uploader id', fatal=False)
+        uploader = self._search_regex(
+            r'window\.channelName\s*=\s*"Embedded:([^"]+)"',
+            webpage, 'uploader', default=False)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'uploader': uploader,
+            'uploader_id': uploader_id,
+            'formats': formats,
+        }

From d4f58034f7f3d959788d4fcf3812e24d3ba05f68 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 24 Jun 2015 23:16:33 +0600
Subject: [PATCH 1673/6219] [onionstudios] Add generic embed extraction routine

---
 youtube_dl/extractor/onionstudios.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/youtube_dl/extractor/onionstudios.py b/youtube_dl/extractor/onionstudios.py
index d5d03fd44..8fa507dec 100644
--- a/youtube_dl/extractor/onionstudios.py
+++ b/youtube_dl/extractor/onionstudios.py
@@ -27,6 +27,13 @@ class OnionStudiosIE(InfoExtractor):
         'only_matching': True,
     }]
 
+    @staticmethod
+    def _extract_url(webpage):
+        mobj = re.search(
+            r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?onionstudios\.com/embed.+?)\1', webpage)
+        if mobj:
+            return mobj.group('url')
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
 

From 1ac1c4c26ecdccb8c515aac634d965b89cbdabe7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 24 Jun 2015 23:19:50 +0600
Subject: [PATCH 1674/6219] [extractor/generic] Add support for OnionStudios
 embeds (Closes #5841)

---
 youtube_dl/extractor/generic.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 7769ffc5c..9c5125f38 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -46,6 +46,7 @@ from .pornhub import PornHubIE
 from .xhamster import XHamsterEmbedIE
 from .vimeo import VimeoIE
 from .dailymotion import DailymotionCloudIE
+from .onionstudios import OnionStudiosIE
 
 
 class GenericIE(InfoExtractor):
@@ -1532,6 +1533,11 @@ class GenericIE(InfoExtractor):
         if dmcloud_url:
             return self.url_result(dmcloud_url, 'DailymotionCloud')
 
+        # Look for OnionStudios embeds
+        onionstudios_url = OnionStudiosIE._extract_url(webpage)
+        if onionstudios_url:
+            return self.url_result(onionstudios_url)
+
         # Look for AdobeTVVideo embeds
         mobj = re.search(
             r'<iframe[^>]+src=[\'"]((?:https?:)?//video\.tv\.adobe\.com/v/\d+[^"]+)[\'"]',

From 8084be78c50e0e921508f13e69fda34a907dc5f8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 24 Jun 2015 23:23:16 +0600
Subject: [PATCH 1675/6219] [extractor/generic] Add test for OnionStudios
 embeds

---
 youtube_dl/extractor/generic.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 9c5125f38..42e4e7035 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -837,6 +837,18 @@ class GenericIE(InfoExtractor):
                 'thumbnail': 're:^https?://.*\.jpe?g$',
             }
         },
+        # OnionStudios embed
+        {
+            'url': 'http://www.clickhole.com/video/dont-understand-bitcoin-man-will-mumble-explanatio-2537',
+            'info_dict': {
+                'id': '2855',
+                'ext': 'mp4',
+                'title': 'Don’t Understand Bitcoin? This Man Will Mumble An Explanation At You',
+                'thumbnail': 're:^https?://.*\.jpe?g$',
+                'uploader': 'ClickHole',
+                'uploader_id': 'clickhole',
+            }
+        },
         # AdobeTVVideo embed
         {
             'url': 'https://helpx.adobe.com/acrobat/how-to/new-experience-acrobat-dc.html?set=acrobat--get-started--essential-beginners',

From 12e9e8445dc090a12c6f7643a346725788b0ca6b Mon Sep 17 00:00:00 2001
From: "Sergey M." <dstftw@gmail.com>
Date: Thu, 25 Jun 2015 01:10:42 +0600
Subject: [PATCH 1676/6219] [README.md] Fix typo

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 5f3a08f5a..10d2d4209 100644
--- a/README.md
+++ b/README.md
@@ -379,7 +379,7 @@ In February 2015, the new YouTube player contained a character sequence in a str
 
 ### HTTP Error 429: Too Many Requests or 402: Payment Required
 
-These two error codes indicate that the service is blocking your IP address because of overuse. Contact the service and ask them to unblock your IP address, or - if you have acquired a whitelisted IP address already - use the [`--proxy` or `--network-address` options](#network-options) to select another IP address.
+These two error codes indicate that the service is blocking your IP address because of overuse. Contact the service and ask them to unblock your IP address, or - if you have acquired a whitelisted IP address already - use the [`--proxy` or `--source-address` options](#network-options) to select another IP address.
 
 ### SyntaxError: Non-ASCII character ###
 

From da92eeae42f556926cb676b3c14e270603b7e38e Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Thu, 25 Jun 2015 00:31:32 -0500
Subject: [PATCH 1677/6219] Fix tests, description formatting

---
 youtube_dl/extractor/bbc.py | 22 ++++++++++++++++++----
 1 file changed, 18 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index fed344ea0..bb671d473 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -428,6 +428,8 @@ class BBCNewsIE(BBCCoUkIE):
             'title': 'Aerial footage showed the site of the crash in the Alps - courtesy BFM TV',
             'description': 'Germanwings plane crash site in aerial video - Aerial footage showed the site of the crash in the Alps - courtesy BFM TV',
             'duration': 47,
+            'upload_date': '20150324',
+            'uploader': 'BBC News',
         },
         'params': {
             'skip_download': True,
@@ -438,8 +440,11 @@ class BBCNewsIE(BBCCoUkIE):
         'info_dict': {
             'id': 'NA',
             'ext': 'mp4',
-            'title': 'YPG - Tel Abyad..n tamam. kontrol.m.zde',
+            'title': 'YPG: Tel Abyad\'\u0131n tamam\u0131 kontrol\xfcm\xfczde',
+            'description': 'YPG: Tel Abyad\'\u0131n tamam\u0131 kontrol\xfcm\xfczde',
             'duration': 47,
+            'upload_date': '20150615',
+            'uploader': 'BBC News',
         },
         'params': {
             'skip_download': True,
@@ -450,8 +455,11 @@ class BBCNewsIE(BBCCoUkIE):
         'info_dict': {
             'id': '39275083',
             'ext': 'mp4',
-            'title': 'Honduras militariza sus hospitales por nuevo esc.ndalo de corrupci.n',
+            'title': 'Honduras militariza sus hospitales por nuevo esc\xe1ndalo de corrupci\xf3n',
+            'description': 'Honduras militariza sus hospitales por nuevo esc\xe1ndalo de corrupci\xf3n',
             'duration': 87,
+            'upload_date': '20150619',
+            'uploader': 'BBC News',
         },
         'params': {
             'skip_download': True,
@@ -507,7 +515,9 @@ class BBCNewsIE(BBCCoUkIE):
             title = jent.get('caption',list_title)
 
             duration = parse_duration(jent.get('duration'))
-            description = list_title + ' - ' + jent.get('caption','')
+            description = list_title
+            if jent.get('caption'):
+               description += ' - ' + jent.get('caption')
             thumbnail = None
             if jent.has_key('image'):
                thumbnail=jent['image'].get('href')
@@ -539,8 +549,12 @@ class BBCNewsIE(BBCCoUkIE):
                
             self._sort_formats(formats)
 
+            id = jent.get('id') if programme_id == None else programme_id
+            if id == None:
+               id = 'NA'
+
             ret.append( {
-                'id': jent.get('id') if programme_id == None else programme_id,
+                'id': id,
                 'uploader': 'BBC News',
                 'upload_date': pubdate,
                 'title': title,

From a1593a4a0e0f278d15ebda603df8c69679bb34c9 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 25 Jun 2015 07:43:49 +0200
Subject: [PATCH 1678/6219] release 2015.06.25

---
 README.md              | 1 +
 docs/supportedsites.md | 7 ++++++-
 youtube_dl/version.py  | 2 +-
 3 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 10d2d4209..e3452c9e1 100644
--- a/README.md
+++ b/README.md
@@ -54,6 +54,7 @@ which means you can modify it, redistribute it or use it however you like.
     --dump-user-agent                Display the current browser identification
     --list-extractors                List all supported extractors
     --extractor-descriptions         Output descriptions of all supported extractors
+    --force-generic-extractor        Force extraction to use the generic extractor
     --default-search PREFIX          Use this prefix for unqualified URLs. For example "gvsearch2:" downloads two videos from google videos for youtube-dl "large apple".
                                      Use the value "auto" to let youtube-dl guess ("auto_warning" to emit a warning when guessing). "error" just throws an error. The
                                      default value "fixup_error" repairs broken URLs, but emits an error if this is not possible instead of searching.
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 220e52b98..9a50fbd1c 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -17,6 +17,7 @@
  - **AcademicEarth:Course**
  - **AddAnime**
  - **AdobeTV**
+ - **AdobeTVVideo**
  - **AdultSwim**
  - **Aftenposten**
  - **Aftonbladet**
@@ -110,6 +111,7 @@
  - **dailymotion**
  - **dailymotion:playlist**
  - **dailymotion:user**
+ - **DailymotionCloud**
  - **daum.net**
  - **DBTV**
  - **DctpTv**
@@ -344,6 +346,7 @@
  - **Odnoklassniki**
  - **OktoberfestTV**
  - **on.aol.com**
+ - **OnionStudios**
  - **Ooyala**
  - **OoyalaExternal**
  - **OpenFilm**
@@ -357,6 +360,7 @@
  - **PhilharmonieDeParis**: Philharmonie de Paris
  - **Phoenix**
  - **Photobucket**
+ - **Pinkbike**
  - **Pladform**
  - **PlanetaPlay**
  - **play.fm**
@@ -612,6 +616,7 @@
  - **XBef**
  - **XboxClips**
  - **XHamster**
+ - **XHamsterEmbed**
  - **XMinus**
  - **XNXX**
  - **Xstream**
@@ -628,7 +633,7 @@
  - **YesJapan**
  - **Ynet**
  - **YouJizz**
- - **Youku**
+ - **youku**
  - **YouPorn**
  - **YourUpload**
  - **youtube**: YouTube.com
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 34a13cb81..a225e03a1 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.06.15'
+__version__ = '2015.06.25'

From 2182ab5187736f806f1a2ce959c3893179bdd1a2 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 25 Jun 2015 16:15:13 +0800
Subject: [PATCH 1679/6219] [thesixtyone] Fix audio_server

Some of the songs are moved to Amazon AWS
---
 youtube_dl/extractor/thesixtyone.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/thesixtyone.py b/youtube_dl/extractor/thesixtyone.py
index a77c6a2fc..e1be978a4 100644
--- a/youtube_dl/extractor/thesixtyone.py
+++ b/youtube_dl/extractor/thesixtyone.py
@@ -17,7 +17,7 @@ class TheSixtyOneIE(InfoExtractor):
             song
         )/(?P<id>[A-Za-z0-9]+)/?$'''
     _SONG_URL_TEMPLATE = 'http://thesixtyone.com/s/{0:}'
-    _SONG_FILE_URL_TEMPLATE = 'http://{audio_server:}.thesixtyone.com/thesixtyone_production/audio/{0:}_stream'
+    _SONG_FILE_URL_TEMPLATE = 'http://{audio_server:}/thesixtyone_production/audio/{0:}_stream'
     _THUMBNAIL_URL_TEMPLATE = '{photo_base_url:}_desktop'
     _TESTS = [
         {
@@ -78,6 +78,12 @@ class TheSixtyOneIE(InfoExtractor):
 
         song_data = json.loads(self._search_regex(
             r'"%s":\s(\{.*?\})' % song_id, webpage, 'song_data'))
+
+        if self._search_regex(r'(t61\.s3_audio_load\s*=\s*1\.0;)', webpage, 's3_audio_load marker', default=None):
+            song_data['audio_server'] = 's3.amazonaws.com'
+        else:
+            song_data['audio_server'] = song_data['audio_server'] + '.thesixtyone.com'
+
         keys = [self._DECODE_MAP.get(s, s) for s in song_data['key']]
         url = self._SONG_FILE_URL_TEMPLATE.format(
             "".join(reversed(keys)), **song_data)

From 381c06775598da9667548fcc45d5df7a03748f7d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 25 Jun 2015 16:19:04 +0800
Subject: [PATCH 1680/6219] [thesixtyone] Modernize

---
 youtube_dl/extractor/thesixtyone.py | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/thesixtyone.py b/youtube_dl/extractor/thesixtyone.py
index e1be978a4..5d09eb9a8 100644
--- a/youtube_dl/extractor/thesixtyone.py
+++ b/youtube_dl/extractor/thesixtyone.py
@@ -1,9 +1,6 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import json
-import re
-
 from .common import InfoExtractor
 from ..utils import unified_strdate
 
@@ -70,14 +67,13 @@ class TheSixtyOneIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        song_id = mobj.group('id')
+        song_id = self._match_id(url)
 
         webpage = self._download_webpage(
             self._SONG_URL_TEMPLATE.format(song_id), song_id)
 
-        song_data = json.loads(self._search_regex(
-            r'"%s":\s(\{.*?\})' % song_id, webpage, 'song_data'))
+        song_data = self._parse_json(self._search_regex(
+            r'"%s":\s(\{.*?\})' % song_id, webpage, 'song_data'), song_id)
 
         if self._search_regex(r'(t61\.s3_audio_load\s*=\s*1\.0;)', webpage, 's3_audio_load marker', default=None):
             song_data['audio_server'] = 's3.amazonaws.com'

From b8091db6b9c75fbdc0ccaad3a9503ef091a7437c Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 25 Jun 2015 16:40:29 +0800
Subject: [PATCH 1681/6219] [planetaplay/quickvid/vube] Skip inaccessible sites

---
 youtube_dl/extractor/planetaplay.py | 3 ++-
 youtube_dl/extractor/quickvid.py    | 1 +
 youtube_dl/extractor/vube.py        | 1 +
 3 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/planetaplay.py b/youtube_dl/extractor/planetaplay.py
index 596c621d7..06505e96f 100644
--- a/youtube_dl/extractor/planetaplay.py
+++ b/youtube_dl/extractor/planetaplay.py
@@ -18,7 +18,8 @@ class PlanetaPlayIE(InfoExtractor):
             'id': '3586',
             'ext': 'flv',
             'title': 'md5:e829428ee28b1deed00de90de49d1da1',
-        }
+        },
+        'skip': 'Not accessible from Travis CI server',
     }
 
     _SONG_FORMATS = {
diff --git a/youtube_dl/extractor/quickvid.py b/youtube_dl/extractor/quickvid.py
index af7d76cf4..f414e2384 100644
--- a/youtube_dl/extractor/quickvid.py
+++ b/youtube_dl/extractor/quickvid.py
@@ -24,6 +24,7 @@ class QuickVidIE(InfoExtractor):
             'thumbnail': 're:^https?://.*\.(?:png|jpg|gif)$',
             'view_count': int,
         },
+        'skip': 'Not accessible from Travis CI server',
     }
 
     def _real_extract(self, url):
diff --git a/youtube_dl/extractor/vube.py b/youtube_dl/extractor/vube.py
index 405cb9db4..149e36467 100644
--- a/youtube_dl/extractor/vube.py
+++ b/youtube_dl/extractor/vube.py
@@ -36,6 +36,7 @@ class VubeIE(InfoExtractor):
                 'comment_count': int,
                 'categories': ['amazing', 'hd', 'best drummer ever', 'william wei', 'bucket drumming', 'street drummer', 'epic street drumming'],
             },
+            'skip': 'Not accessible from Travis CI server',
         }, {
             'url': 'http://vube.com/Chiara+Grispo+Video+Channel/YL2qNPkqon',
             'md5': 'db7aba89d4603dadd627e9d1973946fe',

From 906e2f0eac3729228644fc5e9a3e8dedbd74894e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 25 Jun 2015 15:48:04 +0200
Subject: [PATCH 1682/6219] [downloader/external] Add downloader for httpie
 (closes #6079)

---
 youtube_dl/downloader/external.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index 7ca2d3143..a57c15856 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -109,6 +109,14 @@ class Aria2cFD(ExternalFD):
         cmd += ['--', info_dict['url']]
         return cmd
 
+
+class HttpieFD(ExternalFD):
+    def _make_cmd(self, tmpfilename, info_dict):
+        cmd = ['http', '--download', '--output', tmpfilename, info_dict['url']]
+        for key, val in info_dict['http_headers'].items():
+            cmd += ['%s:%s' % (key, val)]
+        return cmd
+
 _BY_NAME = dict(
     (klass.get_basename(), klass)
     for name, klass in globals().items()

From 533f67d3fa76eb44799276656207a25ece6d20a3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 25 Jun 2015 19:54:44 +0600
Subject: [PATCH 1683/6219] [infoq] Relax _VALID_URL (Closes #6071)

---
 youtube_dl/extractor/infoq.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/infoq.py b/youtube_dl/extractor/infoq.py
index f25f43664..117a7faf6 100644
--- a/youtube_dl/extractor/infoq.py
+++ b/youtube_dl/extractor/infoq.py
@@ -9,9 +9,9 @@ from ..compat import (
 
 
 class InfoQIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?infoq\.com/[^/]+/(?P<id>[^/]+)$'
+    _VALID_URL = r'https?://(?:www\.)?infoq\.com/(?:[^/]+/)+(?P<id>[^/]+)'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.infoq.com/presentations/A-Few-of-My-Favorite-Python-Things',
         'md5': 'b5ca0e0a8c1fed93b0e65e48e462f9a2',
         'info_dict': {
@@ -20,7 +20,10 @@ class InfoQIE(InfoExtractor):
             'description': 'Mike Pirnat presents some tips and tricks, standard libraries and third party packages that make programming in Python a richer experience.',
             'title': 'A Few of My Favorite [Python] Things',
         },
-    }
+    }, {
+        'url': 'http://www.infoq.com/fr/presentations/changez-avis-sur-javascript',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From b4dea075a3f923b4d85d74729479ec4c3e3a313a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 26 Jun 2015 19:36:04 +0600
Subject: [PATCH 1684/6219] [lynda] Fix login request (Closes #6088)

---
 youtube_dl/extractor/lynda.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index cfd3b14f4..2ff902c65 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -36,7 +36,7 @@ class LyndaBaseIE(InfoExtractor):
             'stayPut': 'false'
         }
         request = compat_urllib_request.Request(
-            self._LOGIN_URL, compat_urllib_parse.urlencode(login_form))
+            self._LOGIN_URL, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
         login_page = self._download_webpage(
             request, None, 'Logging in as %s' % username)
 

From 62cca96b729424fa71a3b174af7b646cc1cf12f2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 26 Jun 2015 19:46:42 +0600
Subject: [PATCH 1685/6219] [lynda] Fix confirm login request (#6088)

---
 youtube_dl/extractor/lynda.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index 2ff902c65..18efe5a09 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -65,7 +65,7 @@ class LyndaBaseIE(InfoExtractor):
                     'stayPut': 'false',
                 }
                 request = compat_urllib_request.Request(
-                    self._LOGIN_URL, compat_urllib_parse.urlencode(confirm_form))
+                    self._LOGIN_URL, compat_urllib_parse.urlencode(confirm_form).encode('utf-8'))
                 login_page = self._download_webpage(
                     request, None,
                     'Confirming log in and log out from another device')

From 2988835af5cbe695e61d3b58bc65027f5b48fb63 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 26 Jun 2015 19:48:23 +0600
Subject: [PATCH 1686/6219] [lynda] Fix non-ASCII logins/passwords on python 2

---
 youtube_dl/extractor/lynda.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index 18efe5a09..a00f6e5e5 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -30,8 +30,8 @@ class LyndaBaseIE(InfoExtractor):
             return
 
         login_form = {
-            'username': username,
-            'password': password,
+            'username': username.encode('utf-8'),
+            'password': password.encode('utf-8'),
             'remember': 'false',
             'stayPut': 'false'
         }

From 2028c6e03d7e254831350081bb4b4741b0b47ac4 Mon Sep 17 00:00:00 2001
From: Shadab Zafar <dufferzafar0@gmail.com>
Date: Fri, 26 Jun 2015 21:27:43 +0530
Subject: [PATCH 1687/6219] Added a Playlist Info Extractor for WebOfStories

---
 youtube_dl/extractor/__init__.py     |  5 ++++-
 youtube_dl/extractor/webofstories.py | 25 +++++++++++++++++++++++++
 2 files changed, 29 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 46cc4cd06..c3f3a3e38 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -693,7 +693,10 @@ from .wdr import (
     WDRMobileIE,
     WDRMausIE,
 )
-from .webofstories import WebOfStoriesIE
+from .webofstories import (
+    WebOfStoriesIE,
+    WebOfStoriesPlaylistIE,
+)
 from .weibo import WeiboIE
 from .wimp import WimpIE
 from .wistia import WistiaIE
diff --git a/youtube_dl/extractor/webofstories.py b/youtube_dl/extractor/webofstories.py
index 73077a312..d70e30c00 100644
--- a/youtube_dl/extractor/webofstories.py
+++ b/youtube_dl/extractor/webofstories.py
@@ -1,6 +1,8 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..utils import int_or_none
 
@@ -98,3 +100,26 @@ class WebOfStoriesIE(InfoExtractor):
             'description': description,
             'duration': duration,
         }
+
+
+class WebOfStoriesPlaylistIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?webofstories\.com/playAll/(?P<id>[^/]+)'
+    _TESTS = []
+
+    def _real_extract(self, url):
+        playlist_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, playlist_id)
+
+        entries = [
+            self.url_result('http://www.webofstories.com/play/%s' % video_number, 'WebOfStories')
+            for video_number in set(re.findall('href="/playAll/%s\?sId=(\d+)"' % playlist_id, webpage))
+        ]
+
+        title = self._html_search_regex(
+            r'<title>([^<]+)\s*-\s*Web\sof\sStories</title>', webpage, 'title')
+
+        description = self._html_search_meta(
+            'description', webpage, 'description')
+
+        return self.playlist_result(entries, playlist_id, title, description)

From 03339b7b5bb8f563eace8826512e2f7a4baba415 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 26 Jun 2015 18:25:43 +0100
Subject: [PATCH 1688/6219] [snagfilms] Add new extractor

---
 youtube_dl/extractor/__init__.py  |  1 +
 youtube_dl/extractor/snagfilms.py | 67 +++++++++++++++++++++++++++++++
 2 files changed, 68 insertions(+)
 create mode 100644 youtube_dl/extractor/snagfilms.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index dc1a302e6..3b906b880 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -492,6 +492,7 @@ from .smotri import (
     SmotriUserIE,
     SmotriBroadcastIE,
 )
+from .snagfilms import SnagFilmsIE
 from .snotr import SnotrIE
 from .sohu import SohuIE
 from .soompi import (
diff --git a/youtube_dl/extractor/snagfilms.py b/youtube_dl/extractor/snagfilms.py
new file mode 100644
index 000000000..33832855f
--- /dev/null
+++ b/youtube_dl/extractor/snagfilms.py
@@ -0,0 +1,67 @@
+from .common import InfoExtractor
+from ..utils import js_to_json
+from re import DOTALL
+
+class SnagFilmsIE(InfoExtractor):
+	_VALID_URL = r'(?:https?://)?(?:www.)?snagfilms\.com/films/title/(?P<display_id>.+?)(?:/|$)'
+	_TEST = {
+		'url': 'http://www.snagfilms.com/films/title/lost_for_life',
+		'info_dict':
+		{
+			'id': '0000014c-de2f-d5d6-abcf-ffef58af0017',
+			'display_id': 'lost_for_life',
+			'ext': 'mp4',
+			'title': 'Lost for Life',
+			'duration': 4489,
+			'description': 'In the United States, more than 2500 individuals are serving life-without-parole sentences for crimes they committed when they were seventeen years old or younger. Children as young as thirteen are among the thousands serving these sentences. Directed by Joshua Rof&eacute; (who spent four intensive years on the project), LOST FOR LIFE tells the stories of these individuals, their families and the families of juvenile murder victims. This searingly powerful documentary tackles this contentious issue from multiple perspectives and explores the complexity of the lives of those affected. What is justice when a kid kills? Can a horrific act place a life beyond redemption? Could you forgive?<br />',
+			'categories': ['Documentary','Crime','Award Winning','Festivals']
+		}
+	}
+
+	def _real_extract(self, url):
+		display_id = self._search_regex(
+			self._VALID_URL,
+			url,
+			'display_id',
+			group='display_id'
+		)
+		webpage = self._download_webpage(url, display_id)
+
+		json_data = self._parse_json(self._html_search_regex(
+			r'"data":{"film":(?P<data>{.*?}})}',
+			webpage,
+			'data',
+			group='data'
+		), display_id)
+		title = json_data['title']
+		video_id = json_data['id']
+		duration = int(json_data['duration'])
+		description = json_data['synopsis']
+		categories = [category['title'] for category in json_data['categories']]
+
+		embed_webpage = self._download_webpage('http://www.snagfilms.com/embed/player?filmId=' + video_id, video_id)
+		sources = self._parse_json(js_to_json(self._html_search_regex(
+			r'sources: (?P<sources>\[.*?\])',
+			embed_webpage,
+			'sources',
+			group='sources',
+			flags=DOTALL
+		)), video_id)
+
+		formats = []
+		for source in sources:
+			if source['type'] == 'm3u8':
+				formats.extend(self._extract_m3u8_formats(source['file'], video_id))
+			else:
+				formats.append({'url': source['file'],'ext': source['type'], 'resolution': source['label']})
+		self._sort_formats(formats)
+
+		return {
+			'id': video_id,
+			'display_id': display_id,
+			'title': title,
+			'duration': duration,
+			'description': description,
+			'categories': categories,
+			'formats': formats,
+		}

From d8d24a922adb0646f9f78b2fc4a287a72fa7ff73 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Jun 2015 00:36:23 +0600
Subject: [PATCH 1689/6219] [youtube] Extract formats from multiple DASH
 manifests (Closes #6093)

DASH manifest pointed by dashmpd from the video webpage and one pointed by get_video_info may
be different (namely different itag set) - some itags are missing from DASH manifest pointed by
webpage's dashmpd, some - from DASH manifest pointed by get_video_info's dashmpd).
The general idea is to take a union of itags of both DASH manifests (for example video with such
'manifest behavior' see https://github.com/rg3/youtube-dl/issues/6093).
---
 youtube_dl/extractor/youtube.py | 77 ++++++++++++++++++++-------------
 1 file changed, 46 insertions(+), 31 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index a3da56c14..fa1a2e544 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -853,6 +853,13 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         else:
             player_url = None
 
+        dash_mpds = []
+
+        def add_dash_mpd(video_info):
+            dash_mpd = video_info.get('dashmpd')
+            if dash_mpd and dash_mpd[0] not in dash_mpds:
+                dash_mpds.append(dash_mpd[0])
+
         # Get video info
         embed_webpage = None
         if re.search(r'player-age-gate-content">', video_webpage) is not None:
@@ -873,34 +880,40 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 note='Refetching age-gated info webpage',
                 errnote='unable to download video info webpage')
             video_info = compat_parse_qs(video_info_webpage)
+            add_dash_mpd(video_info)
         else:
             age_gate = False
-            try:
-                # Try looking directly into the video webpage
-                mobj = re.search(r';ytplayer\.config\s*=\s*({.*?});', video_webpage)
-                if not mobj:
-                    raise ValueError('Could not find ytplayer.config')  # caught below
+            # Try looking directly into the video webpage
+            mobj = re.search(r';ytplayer\.config\s*=\s*({.*?});', video_webpage)
+            if mobj:
                 json_code = uppercase_escape(mobj.group(1))
                 ytplayer_config = json.loads(json_code)
                 args = ytplayer_config['args']
-                # Convert to the same format returned by compat_parse_qs
-                video_info = dict((k, [v]) for k, v in args.items())
-                if not args.get('url_encoded_fmt_stream_map'):
-                    raise ValueError('No stream_map present')  # caught below
-            except ValueError:
-                # We fallback to the get_video_info pages (used by the embed page)
-                self.report_video_info_webpage_download(video_id)
-                for el_type in ['&el=embedded', '&el=detailpage', '&el=vevo', '']:
-                    video_info_url = (
-                        '%s://www.youtube.com/get_video_info?&video_id=%s%s&ps=default&eurl=&gl=US&hl=en'
-                        % (proto, video_id, el_type))
-                    video_info_webpage = self._download_webpage(
-                        video_info_url,
-                        video_id, note=False,
-                        errnote='unable to download video info webpage')
-                    video_info = compat_parse_qs(video_info_webpage)
-                    if 'token' in video_info:
-                        break
+                if args.get('url_encoded_fmt_stream_map'):
+                    # Convert to the same format returned by compat_parse_qs
+                    video_info = dict((k, [v]) for k, v in args.items())
+                    add_dash_mpd(video_info)
+            # We also try looking in get_video_info since it may contain different dashmpd
+            # URL that points to a DASH manifest with possibly different itag set (some itags
+            # are missing from DASH manifest pointed by webpage's dashmpd, some - from DASH
+            # manifest pointed by get_video_info's dashmpd).
+            # The general idea is to take a union of itags of both DASH manifests (for example
+            # video with such 'manifest behavior' see https://github.com/rg3/youtube-dl/issues/6093)
+            self.report_video_info_webpage_download(video_id)
+            for el_type in ['&el=info', '&el=embedded', '&el=detailpage', '&el=vevo', '']:
+                video_info_url = (
+                    '%s://www.youtube.com/get_video_info?&video_id=%s%s&ps=default&eurl=&gl=US&hl=en'
+                    % (proto, video_id, el_type))
+                video_info_webpage = self._download_webpage(
+                    video_info_url,
+                    video_id, note=False,
+                    errnote='unable to download video info webpage')
+                get_video_info = compat_parse_qs(video_info_webpage)
+                add_dash_mpd(get_video_info)
+                if not video_info:
+                    video_info = get_video_info
+                if 'token' in get_video_info:
+                    break
         if 'token' not in video_info:
             if 'reason' in video_info:
                 raise ExtractorError(
@@ -1118,24 +1131,26 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
 
         # Look for the DASH manifest
         if self._downloader.params.get('youtube_include_dash_manifest', True):
-            dash_mpd = video_info.get('dashmpd')
-            if dash_mpd:
-                dash_manifest_url = dash_mpd[0]
+            for dash_manifest_url in dash_mpds:
+                dash_formats = {}
                 try:
-                    dash_formats = self._parse_dash_manifest(
-                        video_id, dash_manifest_url, player_url, age_gate)
+                    for df in self._parse_dash_manifest(
+                            video_id, dash_manifest_url, player_url, age_gate):
+                        # Do not overwrite DASH format found in some previous DASH manifest
+                        if df['format_id'] not in dash_formats:
+                            dash_formats[df['format_id']] = df
                 except (ExtractorError, KeyError) as e:
                     self.report_warning(
                         'Skipping DASH manifest: %r' % e, video_id)
-                else:
+                if dash_formats:
                     # Remove the formats we found through non-DASH, they
                     # contain less info and it can be wrong, because we use
                     # fixed values (for example the resolution). See
                     # https://github.com/rg3/youtube-dl/issues/5774 for an
                     # example.
-                    dash_keys = set(df['format_id'] for df in dash_formats)
+                    dash_keys = set(df['format_id'] for df in dash_formats.values())
                     formats = [f for f in formats if f['format_id'] not in dash_keys]
-                    formats.extend(dash_formats)
+                    formats.extend(dash_formats.values())
 
         # Check for malformed aspect ratio
         stretched_m = re.search(

From 1b5a1ae257c6d64fd69ea91eaf69328d4b6d17df Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Jun 2015 00:41:26 +0600
Subject: [PATCH 1690/6219] [youtube] Pick up codecs info from DASH manifest
 when not set explicitly

---
 youtube_dl/extractor/youtube.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index a3da56c14..c9d8e5125 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -824,6 +824,12 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     except StopIteration:
                         full_info = self._formats.get(format_id, {}).copy()
                         full_info.update(f)
+                        codecs = r.attrib.get('codecs')
+                        if codecs:
+                            if full_info.get('acodec') == 'none' and 'vcodec' not in full_info:
+                                full_info['vcodec'] = codecs
+                            elif full_info.get('vcodec') == 'none' and 'acodec' not in full_info:
+                                full_info['acodec'] = codecs
                         formats.append(full_info)
                     else:
                         existing_format.update(f)

From d80265ccd6ff08ef273d83aff847a457f051071f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Jun 2015 02:48:50 +0600
Subject: [PATCH 1691/6219] [youtube] Simplify non-DASH formats exclusion

---
 youtube_dl/extractor/youtube.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index fa1a2e544..46841617a 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1148,8 +1148,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     # fixed values (for example the resolution). See
                     # https://github.com/rg3/youtube-dl/issues/5774 for an
                     # example.
-                    dash_keys = set(df['format_id'] for df in dash_formats.values())
-                    formats = [f for f in formats if f['format_id'] not in dash_keys]
+                    formats = [f for f in formats if f['format_id'] not in dash_formats.keys()]
                     formats.extend(dash_formats.values())
 
         # Check for malformed aspect ratio

From 7e0480ae0e7a9d0b3c73366b8d9ecf86226ad5a8 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 26 Jun 2015 21:50:27 +0100
Subject: [PATCH 1692/6219] convert tabs to 4 spaces identation

---
 youtube_dl/extractor/snagfilms.py | 114 +++++++++++++++---------------
 1 file changed, 57 insertions(+), 57 deletions(-)

diff --git a/youtube_dl/extractor/snagfilms.py b/youtube_dl/extractor/snagfilms.py
index 33832855f..212fefef7 100644
--- a/youtube_dl/extractor/snagfilms.py
+++ b/youtube_dl/extractor/snagfilms.py
@@ -3,65 +3,65 @@ from ..utils import js_to_json
 from re import DOTALL
 
 class SnagFilmsIE(InfoExtractor):
-	_VALID_URL = r'(?:https?://)?(?:www.)?snagfilms\.com/films/title/(?P<display_id>.+?)(?:/|$)'
-	_TEST = {
-		'url': 'http://www.snagfilms.com/films/title/lost_for_life',
-		'info_dict':
-		{
-			'id': '0000014c-de2f-d5d6-abcf-ffef58af0017',
-			'display_id': 'lost_for_life',
-			'ext': 'mp4',
-			'title': 'Lost for Life',
-			'duration': 4489,
-			'description': 'In the United States, more than 2500 individuals are serving life-without-parole sentences for crimes they committed when they were seventeen years old or younger. Children as young as thirteen are among the thousands serving these sentences. Directed by Joshua Rof&eacute; (who spent four intensive years on the project), LOST FOR LIFE tells the stories of these individuals, their families and the families of juvenile murder victims. This searingly powerful documentary tackles this contentious issue from multiple perspectives and explores the complexity of the lives of those affected. What is justice when a kid kills? Can a horrific act place a life beyond redemption? Could you forgive?<br />',
-			'categories': ['Documentary','Crime','Award Winning','Festivals']
-		}
-	}
+    _VALID_URL = r'(?:https?://)?(?:www.)?snagfilms\.com/films/title/(?P<display_id>.+?)(?:/|$)'
+    _TEST = {
+        'url': 'http://www.snagfilms.com/films/title/lost_for_life',
+        'info_dict':
+        {
+            'id': '0000014c-de2f-d5d6-abcf-ffef58af0017',
+            'display_id': 'lost_for_life',
+            'ext': 'mp4',
+            'title': 'Lost for Life',
+            'duration': 4489,
+            'description': 'In the United States, more than 2500 individuals are serving life-without-parole sentences for crimes they committed when they were seventeen years old or younger. Children as young as thirteen are among the thousands serving these sentences. Directed by Joshua Rof&eacute; (who spent four intensive years on the project), LOST FOR LIFE tells the stories of these individuals, their families and the families of juvenile murder victims. This searingly powerful documentary tackles this contentious issue from multiple perspectives and explores the complexity of the lives of those affected. What is justice when a kid kills? Can a horrific act place a life beyond redemption? Could you forgive?<br />',
+            'categories': ['Documentary','Crime','Award Winning','Festivals']
+        }
+    }
 
-	def _real_extract(self, url):
-		display_id = self._search_regex(
-			self._VALID_URL,
-			url,
-			'display_id',
-			group='display_id'
-		)
-		webpage = self._download_webpage(url, display_id)
+    def _real_extract(self, url):
+        display_id = self._search_regex(
+            self._VALID_URL,
+            url,
+            'display_id',
+            group='display_id'
+        )
+        webpage = self._download_webpage(url, display_id)
 
-		json_data = self._parse_json(self._html_search_regex(
-			r'"data":{"film":(?P<data>{.*?}})}',
-			webpage,
-			'data',
-			group='data'
-		), display_id)
-		title = json_data['title']
-		video_id = json_data['id']
-		duration = int(json_data['duration'])
-		description = json_data['synopsis']
-		categories = [category['title'] for category in json_data['categories']]
+        json_data = self._parse_json(self._html_search_regex(
+            r'"data":{"film":(?P<data>{.*?}})}',
+            webpage,
+            'data',
+            group='data'
+        ), display_id)
+        title = json_data['title']
+        video_id = json_data['id']
+        duration = int(json_data['duration'])
+        description = json_data['synopsis']
+        categories = [category['title'] for category in json_data['categories']]
 
-		embed_webpage = self._download_webpage('http://www.snagfilms.com/embed/player?filmId=' + video_id, video_id)
-		sources = self._parse_json(js_to_json(self._html_search_regex(
-			r'sources: (?P<sources>\[.*?\])',
-			embed_webpage,
-			'sources',
-			group='sources',
-			flags=DOTALL
-		)), video_id)
+        embed_webpage = self._download_webpage('http://www.snagfilms.com/embed/player?filmId=' + video_id, video_id)
+        sources = self._parse_json(js_to_json(self._html_search_regex(
+            r'sources: (?P<sources>\[.*?\])',
+            embed_webpage,
+            'sources',
+            group='sources',
+            flags=DOTALL
+        )), video_id)
 
-		formats = []
-		for source in sources:
-			if source['type'] == 'm3u8':
-				formats.extend(self._extract_m3u8_formats(source['file'], video_id))
-			else:
-				formats.append({'url': source['file'],'ext': source['type'], 'resolution': source['label']})
-		self._sort_formats(formats)
+        formats = []
+        for source in sources:
+            if source['type'] == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(source['file'], video_id))
+            else:
+                formats.append({'url': source['file'],'ext': source['type'], 'resolution': source['label']})
+        self._sort_formats(formats)
 
-		return {
-			'id': video_id,
-			'display_id': display_id,
-			'title': title,
-			'duration': duration,
-			'description': description,
-			'categories': categories,
-			'formats': formats,
-		}
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            'title': title,
+            'duration': duration,
+            'description': description,
+            'categories': categories,
+            'formats': formats,
+        }

From fd40bdc0be8984bf6043e70796c1d465a0499d03 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 26 Jun 2015 21:56:15 +0100
Subject: [PATCH 1693/6219] remove unnecessary symbolic name for group

---
 youtube_dl/extractor/snagfilms.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/youtube_dl/extractor/snagfilms.py b/youtube_dl/extractor/snagfilms.py
index 212fefef7..60354b9af 100644
--- a/youtube_dl/extractor/snagfilms.py
+++ b/youtube_dl/extractor/snagfilms.py
@@ -31,7 +31,6 @@ class SnagFilmsIE(InfoExtractor):
             r'"data":{"film":(?P<data>{.*?}})}',
             webpage,
             'data',
-            group='data'
         ), display_id)
         title = json_data['title']
         video_id = json_data['id']
@@ -44,7 +43,6 @@ class SnagFilmsIE(InfoExtractor):
             r'sources: (?P<sources>\[.*?\])',
             embed_webpage,
             'sources',
-            group='sources',
             flags=DOTALL
         )), video_id)
 

From 7d7d4690259f343385a240efcef8d157fc99c72d Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 27 Jun 2015 00:13:14 +0100
Subject: [PATCH 1694/6219] add support for embed links

---
 youtube_dl/extractor/snagfilms.py | 48 ++++++++++++++++++++++---------
 1 file changed, 34 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/snagfilms.py b/youtube_dl/extractor/snagfilms.py
index 60354b9af..cd345474e 100644
--- a/youtube_dl/extractor/snagfilms.py
+++ b/youtube_dl/extractor/snagfilms.py
@@ -1,10 +1,10 @@
+from re import match,DOTALL
 from .common import InfoExtractor
 from ..utils import js_to_json
-from re import DOTALL
 
 class SnagFilmsIE(InfoExtractor):
-    _VALID_URL = r'(?:https?://)?(?:www.)?snagfilms\.com/films/title/(?P<display_id>.+?)(?:/|$)'
-    _TEST = {
+    _VALID_URL = r'(?:https?://)?(?:www.|embed.)?snagfilms\.com/(?:films/title/(?P<display_id>.+?)|embed/player\?.*filmId=(?P<id>.+?))(?:&|/|$)'
+    _TESTS = [{
         'url': 'http://www.snagfilms.com/films/title/lost_for_life',
         'info_dict':
         {
@@ -16,29 +16,48 @@ class SnagFilmsIE(InfoExtractor):
             'description': 'In the United States, more than 2500 individuals are serving life-without-parole sentences for crimes they committed when they were seventeen years old or younger. Children as young as thirteen are among the thousands serving these sentences. Directed by Joshua Rof&eacute; (who spent four intensive years on the project), LOST FOR LIFE tells the stories of these individuals, their families and the families of juvenile murder victims. This searingly powerful documentary tackles this contentious issue from multiple perspectives and explores the complexity of the lives of those affected. What is justice when a kid kills? Can a horrific act place a life beyond redemption? Could you forgive?<br />',
             'categories': ['Documentary','Crime','Award Winning','Festivals']
         }
-    }
+    },{
+        'url': 'http://embed.snagfilms.com/embed/player?filmId=74849a00-85a9-11e1-9660-123139220831',
+        'info_dict':
+        {
+            'id': '74849a00-85a9-11e1-9660-123139220831',
+            'display_id': 'while_we_watch',
+            'ext': 'mp4',
+            'title': '#whilewewatch',
+            'duration': 2311,
+            'description': 'A gripping portrait of the Occupy Wall Street media revolution,&nbsp;#WHILEWEWATCH is the first definitive film to emerge from Zuccotti Park—with full access and cooperation from masterminds who made #OccupyWallStreet a reality.&nbsp;The #OccupyWallStreet media team had no fear of a critical city government, big corporations, hostile police or a lagging mainstream media to tell their story. Through rain, snow, grueling days and sleeping on concrete, they pump out exhilarating ideas to the world. With little money, they rely on Twitter, texting, Wi-Fi, posters, Tumblr, live streams, YouTube, Facebook, dramatic marches, drumbeats and chants. As the film unfolds, we witness the burgeoning power of social media.<br />',
+            'categories': ['Documentary','Politics']
+        }
+    }]
 
     def _real_extract(self, url):
-        display_id = self._search_regex(
-            self._VALID_URL,
-            url,
-            'display_id',
-            group='display_id'
-        )
-        webpage = self._download_webpage(url, display_id)
+        display_id, video_id = match(self._VALID_URL,url).groups()
+        if display_id is None:
+            embed_webpage = self._download_webpage('http://www.snagfilms.com/embed/player?filmId=' + video_id, video_id)
+
+            display_id = self._html_search_regex(
+                r"snagfilms\.com/films/title/(?P<display_id>.+?)(?:/|')",
+                embed_webpage,
+                'display_id'
+            )
+        webpage = self._download_webpage('http://www.snagfilms.com/films/title/' + display_id, display_id)
 
         json_data = self._parse_json(self._html_search_regex(
             r'"data":{"film":(?P<data>{.*?}})}',
             webpage,
-            'data',
+            'data'
         ), display_id)
+
         title = json_data['title']
-        video_id = json_data['id']
         duration = int(json_data['duration'])
         description = json_data['synopsis']
         categories = [category['title'] for category in json_data['categories']]
+        thumbnail = json_data['image']
+
+        if video_id is None:
+            video_id = json_data['id']
+            embed_webpage = self._download_webpage('http://www.snagfilms.com/embed/player?filmId=' + video_id, video_id)
 
-        embed_webpage = self._download_webpage('http://www.snagfilms.com/embed/player?filmId=' + video_id, video_id)
         sources = self._parse_json(js_to_json(self._html_search_regex(
             r'sources: (?P<sources>\[.*?\])',
             embed_webpage,
@@ -61,5 +80,6 @@ class SnagFilmsIE(InfoExtractor):
             'duration': duration,
             'description': description,
             'categories': categories,
+            'thumbnail': thumbnail,
             'formats': formats,
         }

From 4eb10f66216b5239a9a8efcf950c9b16f5c8b5de Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 27 Jun 2015 13:13:57 +0800
Subject: [PATCH 1695/6219] [utils] Add ISO3166Utils

---
 youtube_dl/utils.py | 260 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 260 insertions(+)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index a2746b2d1..96490f112 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2084,6 +2084,266 @@ class ISO639Utils(object):
                 return short_name
 
 
+class ISO3166Utils(object):
+    # From http://data.okfn.org/data/core/country-list
+    _country_map = {
+        'AF': 'Afghanistan',
+        'AX': 'Åland Islands',
+        'AL': 'Albania',
+        'DZ': 'Algeria',
+        'AS': 'American Samoa',
+        'AD': 'Andorra',
+        'AO': 'Angola',
+        'AI': 'Anguilla',
+        'AQ': 'Antarctica',
+        'AG': 'Antigua and Barbuda',
+        'AR': 'Argentina',
+        'AM': 'Armenia',
+        'AW': 'Aruba',
+        'AU': 'Australia',
+        'AT': 'Austria',
+        'AZ': 'Azerbaijan',
+        'BS': 'Bahamas',
+        'BH': 'Bahrain',
+        'BD': 'Bangladesh',
+        'BB': 'Barbados',
+        'BY': 'Belarus',
+        'BE': 'Belgium',
+        'BZ': 'Belize',
+        'BJ': 'Benin',
+        'BM': 'Bermuda',
+        'BT': 'Bhutan',
+        'BO': 'Bolivia, Plurinational State of',
+        'BQ': 'Bonaire, Sint Eustatius and Saba',
+        'BA': 'Bosnia and Herzegovina',
+        'BW': 'Botswana',
+        'BV': 'Bouvet Island',
+        'BR': 'Brazil',
+        'IO': 'British Indian Ocean Territory',
+        'BN': 'Brunei Darussalam',
+        'BG': 'Bulgaria',
+        'BF': 'Burkina Faso',
+        'BI': 'Burundi',
+        'KH': 'Cambodia',
+        'CM': 'Cameroon',
+        'CA': 'Canada',
+        'CV': 'Cape Verde',
+        'KY': 'Cayman Islands',
+        'CF': 'Central African Republic',
+        'TD': 'Chad',
+        'CL': 'Chile',
+        'CN': 'China',
+        'CX': 'Christmas Island',
+        'CC': 'Cocos (Keeling) Islands',
+        'CO': 'Colombia',
+        'KM': 'Comoros',
+        'CG': 'Congo',
+        'CD': 'Congo, the Democratic Republic of the',
+        'CK': 'Cook Islands',
+        'CR': 'Costa Rica',
+        'CI': 'Côte d\'Ivoire',
+        'HR': 'Croatia',
+        'CU': 'Cuba',
+        'CW': 'Curaçao',
+        'CY': 'Cyprus',
+        'CZ': 'Czech Republic',
+        'DK': 'Denmark',
+        'DJ': 'Djibouti',
+        'DM': 'Dominica',
+        'DO': 'Dominican Republic',
+        'EC': 'Ecuador',
+        'EG': 'Egypt',
+        'SV': 'El Salvador',
+        'GQ': 'Equatorial Guinea',
+        'ER': 'Eritrea',
+        'EE': 'Estonia',
+        'ET': 'Ethiopia',
+        'FK': 'Falkland Islands (Malvinas)',
+        'FO': 'Faroe Islands',
+        'FJ': 'Fiji',
+        'FI': 'Finland',
+        'FR': 'France',
+        'GF': 'French Guiana',
+        'PF': 'French Polynesia',
+        'TF': 'French Southern Territories',
+        'GA': 'Gabon',
+        'GM': 'Gambia',
+        'GE': 'Georgia',
+        'DE': 'Germany',
+        'GH': 'Ghana',
+        'GI': 'Gibraltar',
+        'GR': 'Greece',
+        'GL': 'Greenland',
+        'GD': 'Grenada',
+        'GP': 'Guadeloupe',
+        'GU': 'Guam',
+        'GT': 'Guatemala',
+        'GG': 'Guernsey',
+        'GN': 'Guinea',
+        'GW': 'Guinea-Bissau',
+        'GY': 'Guyana',
+        'HT': 'Haiti',
+        'HM': 'Heard Island and McDonald Islands',
+        'VA': 'Holy See (Vatican City State)',
+        'HN': 'Honduras',
+        'HK': 'Hong Kong',
+        'HU': 'Hungary',
+        'IS': 'Iceland',
+        'IN': 'India',
+        'ID': 'Indonesia',
+        'IR': 'Iran, Islamic Republic of',
+        'IQ': 'Iraq',
+        'IE': 'Ireland',
+        'IM': 'Isle of Man',
+        'IL': 'Israel',
+        'IT': 'Italy',
+        'JM': 'Jamaica',
+        'JP': 'Japan',
+        'JE': 'Jersey',
+        'JO': 'Jordan',
+        'KZ': 'Kazakhstan',
+        'KE': 'Kenya',
+        'KI': 'Kiribati',
+        'KP': 'Korea, Democratic People\'s Republic of',
+        'KR': 'Korea, Republic of',
+        'KW': 'Kuwait',
+        'KG': 'Kyrgyzstan',
+        'LA': 'Lao People\'s Democratic Republic',
+        'LV': 'Latvia',
+        'LB': 'Lebanon',
+        'LS': 'Lesotho',
+        'LR': 'Liberia',
+        'LY': 'Libya',
+        'LI': 'Liechtenstein',
+        'LT': 'Lithuania',
+        'LU': 'Luxembourg',
+        'MO': 'Macao',
+        'MK': 'Macedonia, the Former Yugoslav Republic of',
+        'MG': 'Madagascar',
+        'MW': 'Malawi',
+        'MY': 'Malaysia',
+        'MV': 'Maldives',
+        'ML': 'Mali',
+        'MT': 'Malta',
+        'MH': 'Marshall Islands',
+        'MQ': 'Martinique',
+        'MR': 'Mauritania',
+        'MU': 'Mauritius',
+        'YT': 'Mayotte',
+        'MX': 'Mexico',
+        'FM': 'Micronesia, Federated States of',
+        'MD': 'Moldova, Republic of',
+        'MC': 'Monaco',
+        'MN': 'Mongolia',
+        'ME': 'Montenegro',
+        'MS': 'Montserrat',
+        'MA': 'Morocco',
+        'MZ': 'Mozambique',
+        'MM': 'Myanmar',
+        'NA': 'Namibia',
+        'NR': 'Nauru',
+        'NP': 'Nepal',
+        'NL': 'Netherlands',
+        'NC': 'New Caledonia',
+        'NZ': 'New Zealand',
+        'NI': 'Nicaragua',
+        'NE': 'Niger',
+        'NG': 'Nigeria',
+        'NU': 'Niue',
+        'NF': 'Norfolk Island',
+        'MP': 'Northern Mariana Islands',
+        'NO': 'Norway',
+        'OM': 'Oman',
+        'PK': 'Pakistan',
+        'PW': 'Palau',
+        'PS': 'Palestine, State of',
+        'PA': 'Panama',
+        'PG': 'Papua New Guinea',
+        'PY': 'Paraguay',
+        'PE': 'Peru',
+        'PH': 'Philippines',
+        'PN': 'Pitcairn',
+        'PL': 'Poland',
+        'PT': 'Portugal',
+        'PR': 'Puerto Rico',
+        'QA': 'Qatar',
+        'RE': 'Réunion',
+        'RO': 'Romania',
+        'RU': 'Russian Federation',
+        'RW': 'Rwanda',
+        'BL': 'Saint Barthélemy',
+        'SH': 'Saint Helena, Ascension and Tristan da Cunha',
+        'KN': 'Saint Kitts and Nevis',
+        'LC': 'Saint Lucia',
+        'MF': 'Saint Martin (French part)',
+        'PM': 'Saint Pierre and Miquelon',
+        'VC': 'Saint Vincent and the Grenadines',
+        'WS': 'Samoa',
+        'SM': 'San Marino',
+        'ST': 'Sao Tome and Principe',
+        'SA': 'Saudi Arabia',
+        'SN': 'Senegal',
+        'RS': 'Serbia',
+        'SC': 'Seychelles',
+        'SL': 'Sierra Leone',
+        'SG': 'Singapore',
+        'SX': 'Sint Maarten (Dutch part)',
+        'SK': 'Slovakia',
+        'SI': 'Slovenia',
+        'SB': 'Solomon Islands',
+        'SO': 'Somalia',
+        'ZA': 'South Africa',
+        'GS': 'South Georgia and the South Sandwich Islands',
+        'SS': 'South Sudan',
+        'ES': 'Spain',
+        'LK': 'Sri Lanka',
+        'SD': 'Sudan',
+        'SR': 'Suriname',
+        'SJ': 'Svalbard and Jan Mayen',
+        'SZ': 'Swaziland',
+        'SE': 'Sweden',
+        'CH': 'Switzerland',
+        'SY': 'Syrian Arab Republic',
+        'TW': 'Taiwan, Province of China',
+        'TJ': 'Tajikistan',
+        'TZ': 'Tanzania, United Republic of',
+        'TH': 'Thailand',
+        'TL': 'Timor-Leste',
+        'TG': 'Togo',
+        'TK': 'Tokelau',
+        'TO': 'Tonga',
+        'TT': 'Trinidad and Tobago',
+        'TN': 'Tunisia',
+        'TR': 'Turkey',
+        'TM': 'Turkmenistan',
+        'TC': 'Turks and Caicos Islands',
+        'TV': 'Tuvalu',
+        'UG': 'Uganda',
+        'UA': 'Ukraine',
+        'AE': 'United Arab Emirates',
+        'GB': 'United Kingdom',
+        'US': 'United States',
+        'UM': 'United States Minor Outlying Islands',
+        'UY': 'Uruguay',
+        'UZ': 'Uzbekistan',
+        'VU': 'Vanuatu',
+        'VE': 'Venezuela, Bolivarian Republic of',
+        'VN': 'Viet Nam',
+        'VG': 'Virgin Islands, British',
+        'VI': 'Virgin Islands, U.S.',
+        'WF': 'Wallis and Futuna',
+        'EH': 'Western Sahara',
+        'YE': 'Yemen',
+        'ZM': 'Zambia',
+        'ZW': 'Zimbabwe',
+    }
+
+    @classmethod
+    def short2full(cls, code):
+        """Convert an ISO 3166-2 country code to the corresponding full name"""
+        return cls._country_map.get(code.upper())
+
+
 class PerRequestProxyHandler(compat_urllib_request.ProxyHandler):
     def __init__(self, proxies=None):
         # Set default handlers

From af214c3a79bb598d3a13559c5ebddcca4584669a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 27 Jun 2015 13:15:57 +0800
Subject: [PATCH 1696/6219] [youtube] More useful messages for georestricted
 videos (#5716)

---
 youtube_dl/extractor/youtube.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index a3da56c14..d9240ff02 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -32,6 +32,7 @@ from ..utils import (
     unescapeHTML,
     unified_strdate,
     uppercase_escape,
+    ISO3166Utils,
 )
 
 
@@ -903,6 +904,12 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                         break
         if 'token' not in video_info:
             if 'reason' in video_info:
+                if 'The uploader has not made this video available in your country.' in video_info['reason']:
+                    regions_allowed = self._html_search_meta('regionsAllowed', video_webpage, default=None)
+                    if regions_allowed is not None:
+                        raise ExtractorError('YouTube said: This video is available in %s only' % (
+                            ', '.join(map(ISO3166Utils.short2full, regions_allowed.split(',')))),
+                            expected=True)
                 raise ExtractorError(
                     'YouTube said: %s' % video_info['reason'][0],
                     expected=True, video_id=video_id)

From bc93bdb5bbf48b121977a5088ab9607f0fbeb83e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Jun 2015 13:19:46 +0600
Subject: [PATCH 1697/6219] [youtube] Fix reference before assignment for
 video_info

---
 youtube_dl/extractor/youtube.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 46841617a..45e5cb80e 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -883,6 +883,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             add_dash_mpd(video_info)
         else:
             age_gate = False
+            video_info = None
             # Try looking directly into the video webpage
             mobj = re.search(r';ytplayer\.config\s*=\s*({.*?});', video_webpage)
             if mobj:

From 226b886ca85b5843d2cb9638fb6d88223e9dd5b2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Jun 2015 14:04:55 +0600
Subject: [PATCH 1698/6219] [vk] Fix authentication (Closes #6105)

---
 youtube_dl/extractor/vk.py | 25 ++++++++++++++++---------
 1 file changed, 16 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 38ff3c1a9..f2ae109f9 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -121,20 +121,27 @@ class VKIE(InfoExtractor):
         if username is None:
             return
 
-        login_form = {
-            'act': 'login',
-            'role': 'al_frame',
-            'expire': '1',
+        login_page = self._download_webpage(
+            'https://vk.com', None, 'Downloading login page')
+
+        login_form = dict(re.findall(
+            r'<input\s+type="hidden"\s+name="([^"]+)"\s+(?:id="[^"]+"\s+)?value="([^"]*)"',
+            login_page))
+
+        login_form.update({
             'email': username.encode('cp1251'),
             'pass': password.encode('cp1251'),
-        }
+        })
 
-        request = compat_urllib_request.Request('https://login.vk.com/?act=login',
-                                                compat_urllib_parse.urlencode(login_form).encode('utf-8'))
-        login_page = self._download_webpage(request, None, note='Logging in as %s' % username)
+        request = compat_urllib_request.Request(
+            'https://login.vk.com/?act=login',
+            compat_urllib_parse.urlencode(login_form).encode('utf-8'))
+        login_page = self._download_webpage(
+            request, None, note='Logging in as %s' % username)
 
         if re.search(r'onLoginFailed', login_page):
-            raise ExtractorError('Unable to login, incorrect username and/or password', expected=True)
+            raise ExtractorError(
+                'Unable to login, incorrect username and/or password', expected=True)
 
     def _real_initialize(self):
         self._login()

From 00334d0de01d46165880569159c534f9b7d927a6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Jun 2015 14:26:51 +0600
Subject: [PATCH 1699/6219] [options] Add missing whitespace and split lines

---
 youtube_dl/options.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 6aeca61ee..e7d067642 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -351,7 +351,8 @@ def parseOpts(overrideArguments=None):
         '--merge-output-format',
         action='store', dest='merge_output_format', metavar='FORMAT', default=None,
         help=(
-            'If a merge is required (e.g. bestvideo+bestaudio), output to given container format. One of mkv, mp4, ogg, webm, flv.'
+            'If a merge is required (e.g. bestvideo+bestaudio), '
+            'output to given container format. One of mkv, mp4, ogg, webm, flv. '
             'Ignored if no merge is required'))
 
     subtitles = optparse.OptionGroup(parser, 'Subtitle Options')

From 0a3cf9ad3dc1a038018ff347473c0e2d8fc10a69 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Jun 2015 14:31:18 +0600
Subject: [PATCH 1700/6219] [youtube] Skip get_video_info requests when
 --youtube-skip-dash-manifest is specified

---
 youtube_dl/extractor/youtube.py | 43 +++++++++++++++++----------------
 1 file changed, 22 insertions(+), 21 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 45e5cb80e..2c9ad5e92 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -894,27 +894,28 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     # Convert to the same format returned by compat_parse_qs
                     video_info = dict((k, [v]) for k, v in args.items())
                     add_dash_mpd(video_info)
-            # We also try looking in get_video_info since it may contain different dashmpd
-            # URL that points to a DASH manifest with possibly different itag set (some itags
-            # are missing from DASH manifest pointed by webpage's dashmpd, some - from DASH
-            # manifest pointed by get_video_info's dashmpd).
-            # The general idea is to take a union of itags of both DASH manifests (for example
-            # video with such 'manifest behavior' see https://github.com/rg3/youtube-dl/issues/6093)
-            self.report_video_info_webpage_download(video_id)
-            for el_type in ['&el=info', '&el=embedded', '&el=detailpage', '&el=vevo', '']:
-                video_info_url = (
-                    '%s://www.youtube.com/get_video_info?&video_id=%s%s&ps=default&eurl=&gl=US&hl=en'
-                    % (proto, video_id, el_type))
-                video_info_webpage = self._download_webpage(
-                    video_info_url,
-                    video_id, note=False,
-                    errnote='unable to download video info webpage')
-                get_video_info = compat_parse_qs(video_info_webpage)
-                add_dash_mpd(get_video_info)
-                if not video_info:
-                    video_info = get_video_info
-                if 'token' in get_video_info:
-                    break
+            if not video_info or self._downloader.params.get('youtube_include_dash_manifest', True):
+                # We also try looking in get_video_info since it may contain different dashmpd
+                # URL that points to a DASH manifest with possibly different itag set (some itags
+                # are missing from DASH manifest pointed by webpage's dashmpd, some - from DASH
+                # manifest pointed by get_video_info's dashmpd).
+                # The general idea is to take a union of itags of both DASH manifests (for example
+                # video with such 'manifest behavior' see https://github.com/rg3/youtube-dl/issues/6093)
+                self.report_video_info_webpage_download(video_id)
+                for el_type in ['&el=info', '&el=embedded', '&el=detailpage', '&el=vevo', '']:
+                    video_info_url = (
+                        '%s://www.youtube.com/get_video_info?&video_id=%s%s&ps=default&eurl=&gl=US&hl=en'
+                        % (proto, video_id, el_type))
+                    video_info_webpage = self._download_webpage(
+                        video_info_url,
+                        video_id, note=False,
+                        errnote='unable to download video info webpage')
+                    get_video_info = compat_parse_qs(video_info_webpage)
+                    add_dash_mpd(get_video_info)
+                    if not video_info:
+                        video_info = get_video_info
+                    if 'token' in get_video_info:
+                        break
         if 'token' not in video_info:
             if 'reason' in video_info:
                 raise ExtractorError(

From b2575b38e76e14b7552a43cd707a03fe78748f40 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Jun 2015 14:38:41 +0600
Subject: [PATCH 1701/6219] [options] Clarify --youtube-skip-dash-manifest

---
 youtube_dl/options.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 6aeca61ee..e3dfb7af9 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -346,7 +346,7 @@ def parseOpts(overrideArguments=None):
     video_format.add_option(
         '--youtube-skip-dash-manifest',
         action='store_false', dest='youtube_include_dash_manifest',
-        help='Do not download the DASH manifest on YouTube videos')
+        help='Do not download the DASH manifests and related data on YouTube videos')
     video_format.add_option(
         '--merge-output-format',
         action='store', dest='merge_output_format', metavar='FORMAT', default=None,

From da77d856a1310b52975abbad82121a1b4c6597a9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Jun 2015 14:55:46 +0600
Subject: [PATCH 1702/6219] [youtube] Add test for #6093

---
 youtube_dl/extractor/youtube.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 2c9ad5e92..20e1781f8 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -518,6 +518,20 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'skip_download': 'requires avconv',
             }
         },
+        # Extraction from multiple DASH manifests (https://github.com/rg3/youtube-dl/pull/6097)
+        {
+            'url': 'https://www.youtube.com/watch?v=FIl7x6_3R5Y',
+            'info_dict': {
+                'id': 'FIl7x6_3R5Y',
+                'ext': 'mp4',
+                'title': 'md5:7b81415841e02ecd4313668cde88737a',
+                'description': 'md5:116377fd2963b81ec4ce64b542173306',
+                'upload_date': '20150625',
+                'uploader_id': 'dorappi2000',
+                'uploader': 'dorappi2000',
+                'formats': 'mincount:33',
+            },
+        }
     ]
 
     def __init__(self, *args, **kwargs):

From f39eb98bab497d8e06b9f243a8240509326678f0 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 27 Jun 2015 10:55:25 +0100
Subject: [PATCH 1703/6219] download all pages before start extracting info

---
 youtube_dl/extractor/snagfilms.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/snagfilms.py b/youtube_dl/extractor/snagfilms.py
index cd345474e..74cd2698d 100644
--- a/youtube_dl/extractor/snagfilms.py
+++ b/youtube_dl/extractor/snagfilms.py
@@ -34,7 +34,6 @@ class SnagFilmsIE(InfoExtractor):
         display_id, video_id = match(self._VALID_URL,url).groups()
         if display_id is None:
             embed_webpage = self._download_webpage('http://www.snagfilms.com/embed/player?filmId=' + video_id, video_id)
-
             display_id = self._html_search_regex(
                 r"snagfilms\.com/films/title/(?P<display_id>.+?)(?:/|')",
                 embed_webpage,
@@ -48,16 +47,16 @@ class SnagFilmsIE(InfoExtractor):
             'data'
         ), display_id)
 
+        if video_id is None:
+            video_id = json_data['id']
+            embed_webpage = self._download_webpage('http://www.snagfilms.com/embed/player?filmId=' + video_id, video_id)
+
         title = json_data['title']
         duration = int(json_data['duration'])
         description = json_data['synopsis']
         categories = [category['title'] for category in json_data['categories']]
         thumbnail = json_data['image']
 
-        if video_id is None:
-            video_id = json_data['id']
-            embed_webpage = self._download_webpage('http://www.snagfilms.com/embed/player?filmId=' + video_id, video_id)
-
         sources = self._parse_json(js_to_json(self._html_search_regex(
             r'sources: (?P<sources>\[.*?\])',
             embed_webpage,

From 73c471e9efd3f888af060b391eb750b079a5822c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 27 Jun 2015 14:18:01 +0200
Subject: [PATCH 1704/6219] [newstube] Fix GUID extraction (fixes #6109)

---
 youtube_dl/extractor/newstube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/newstube.py b/youtube_dl/extractor/newstube.py
index 85fcad06b..fe29cfc3f 100644
--- a/youtube_dl/extractor/newstube.py
+++ b/youtube_dl/extractor/newstube.py
@@ -31,7 +31,7 @@ class NewstubeIE(InfoExtractor):
         page = self._download_webpage(url, video_id, 'Downloading page')
 
         video_guid = self._html_search_regex(
-            r'<meta property="og:video" content="https?://(?:www\.)?newstube\.ru/freshplayer\.swf\?guid=(?P<guid>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})',
+                r'<meta property="og:video:url" content="https?://(?:www\.)?newstube\.ru/freshplayer\.swf\?guid=(?P<guid>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})',
             page, 'video GUID')
 
         player = self._download_xml(

From cee16e0fa3c2030aa6e58669ca2253457f1f3e64 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 27 Jun 2015 14:20:33 +0200
Subject: [PATCH 1705/6219] [newstube] style: fix alignment

---
 youtube_dl/extractor/newstube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/newstube.py b/youtube_dl/extractor/newstube.py
index fe29cfc3f..5a9e73cd6 100644
--- a/youtube_dl/extractor/newstube.py
+++ b/youtube_dl/extractor/newstube.py
@@ -31,7 +31,7 @@ class NewstubeIE(InfoExtractor):
         page = self._download_webpage(url, video_id, 'Downloading page')
 
         video_guid = self._html_search_regex(
-                r'<meta property="og:video:url" content="https?://(?:www\.)?newstube\.ru/freshplayer\.swf\?guid=(?P<guid>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})',
+            r'<meta property="og:video:url" content="https?://(?:www\.)?newstube\.ru/freshplayer\.swf\?guid=(?P<guid>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})',
             page, 'video GUID')
 
         player = self._download_xml(

From 654fd03c73fa0e4407a71c07d821b45321c3cdb8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Jun 2015 18:20:42 +0600
Subject: [PATCH 1706/6219] [snagfilms] Improve and simplify

---
 youtube_dl/extractor/__init__.py  |   5 +-
 youtube_dl/extractor/snagfilms.py | 171 +++++++++++++++++++-----------
 2 files changed, 114 insertions(+), 62 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 59068a8b8..7e74a971d 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -493,7 +493,10 @@ from .smotri import (
     SmotriUserIE,
     SmotriBroadcastIE,
 )
-from .snagfilms import SnagFilmsIE
+from .snagfilms import (
+    SnagFilmsIE,
+    SnagFilmsEmbedIE,
+)
 from .snotr import SnotrIE
 from .sohu import SohuIE
 from .soompi import (
diff --git a/youtube_dl/extractor/snagfilms.py b/youtube_dl/extractor/snagfilms.py
index 74cd2698d..6e103bd49 100644
--- a/youtube_dl/extractor/snagfilms.py
+++ b/youtube_dl/extractor/snagfilms.py
@@ -1,84 +1,133 @@
-from re import match,DOTALL
+from __future__ import unicode_literals
+
+import re
+
 from .common import InfoExtractor
-from ..utils import js_to_json
+from ..utils import (
+    clean_html,
+    determine_ext,
+    int_or_none,
+    js_to_json,
+    parse_duration,
+)
+
+
+class SnagFilmsEmbedIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:(?:www|embed)\.)?snagfilms\.com/embed/player\?.*\bfilmId=(?P<id>[\da-f-]{36})'
+    _TESTS = [{
+        'url': 'http://embed.snagfilms.com/embed/player?filmId=74849a00-85a9-11e1-9660-123139220831&w=500',
+        'md5': '2924e9215c6eff7a55ed35b72276bd93',
+        'info_dict': {
+            'id': '74849a00-85a9-11e1-9660-123139220831',
+            'ext': 'mp4',
+            'title': '#whilewewatch',
+        }
+    }, {
+        'url': 'http://www.snagfilms.com/embed/player?filmId=0000014c-de2f-d5d6-abcf-ffef58af0017',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        formats = []
+        for source in self._parse_json(js_to_json(self._search_regex(
+                r'(?s)sources:\s*(\[.+?\]),', webpage, 'json')), video_id):
+            file_ = source.get('file')
+            if not file_:
+                continue
+            type_ = source.get('type')
+            format_id = source.get('label')
+            ext = determine_ext(file_)
+            if any(_ == 'm3u8' for _ in (type_, ext)):
+                formats.extend(self._extract_m3u8_formats(
+                    file_, video_id, 'mp4', m3u8_id='hls'))
+            else:
+                bitrate = int_or_none(self._search_regex(
+                    r'(\d+)kbps', file_, 'bitrate', default=None))
+                height = int_or_none(self._search_regex(
+                    r'^(\d+)[pP]$', format_id, 'height', default=None))
+                formats.append({
+                    'url': file_,
+                    'format_id': format_id,
+                    'tbr': bitrate,
+                    'height': height,
+                })
+        self._sort_formats(formats)
+
+        title = self._search_regex(
+            [r"title\s*:\s*'([^']+)'", r'<title>([^<]+)</title>'],
+            webpage, 'title')
+
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': formats,
+        }
+
 
 class SnagFilmsIE(InfoExtractor):
-    _VALID_URL = r'(?:https?://)?(?:www.|embed.)?snagfilms\.com/(?:films/title/(?P<display_id>.+?)|embed/player\?.*filmId=(?P<id>.+?))(?:&|/|$)'
-    _TESTS = [{
+    _VALID_URL = r'https?://(?:www\.)?snagfilms\.com/films/title/(?P<id>[^/]+)'
+    _TEST = {
         'url': 'http://www.snagfilms.com/films/title/lost_for_life',
-        'info_dict':
-        {
+        'md5': '19844f897b35af219773fd63bdec2942',
+        'info_dict': {
             'id': '0000014c-de2f-d5d6-abcf-ffef58af0017',
             'display_id': 'lost_for_life',
             'ext': 'mp4',
             'title': 'Lost for Life',
+            'description': 'md5:fbdacc8bb6b455e464aaf98bc02e1c82',
+            'thumbnail': 're:^https?://.*\.jpg',
             'duration': 4489,
-            'description': 'In the United States, more than 2500 individuals are serving life-without-parole sentences for crimes they committed when they were seventeen years old or younger. Children as young as thirteen are among the thousands serving these sentences. Directed by Joshua Rof&eacute; (who spent four intensive years on the project), LOST FOR LIFE tells the stories of these individuals, their families and the families of juvenile murder victims. This searingly powerful documentary tackles this contentious issue from multiple perspectives and explores the complexity of the lives of those affected. What is justice when a kid kills? Can a horrific act place a life beyond redemption? Could you forgive?<br />',
-            'categories': ['Documentary','Crime','Award Winning','Festivals']
+            'categories': ['Documentary', 'Crime', 'Award Winning', 'Festivals']
         }
-    },{
-        'url': 'http://embed.snagfilms.com/embed/player?filmId=74849a00-85a9-11e1-9660-123139220831',
-        'info_dict':
-        {
-            'id': '74849a00-85a9-11e1-9660-123139220831',
-            'display_id': 'while_we_watch',
-            'ext': 'mp4',
-            'title': '#whilewewatch',
-            'duration': 2311,
-            'description': 'A gripping portrait of the Occupy Wall Street media revolution,&nbsp;#WHILEWEWATCH is the first definitive film to emerge from Zuccotti Park—with full access and cooperation from masterminds who made #OccupyWallStreet a reality.&nbsp;The #OccupyWallStreet media team had no fear of a critical city government, big corporations, hostile police or a lagging mainstream media to tell their story. Through rain, snow, grueling days and sleeping on concrete, they pump out exhilarating ideas to the world. With little money, they rely on Twitter, texting, Wi-Fi, posters, Tumblr, live streams, YouTube, Facebook, dramatic marches, drumbeats and chants. As the film unfolds, we witness the burgeoning power of social media.<br />',
-            'categories': ['Documentary','Politics']
-        }
-    }]
+    }
 
     def _real_extract(self, url):
-        display_id, video_id = match(self._VALID_URL,url).groups()
-        if display_id is None:
-            embed_webpage = self._download_webpage('http://www.snagfilms.com/embed/player?filmId=' + video_id, video_id)
-            display_id = self._html_search_regex(
-                r"snagfilms\.com/films/title/(?P<display_id>.+?)(?:/|')",
-                embed_webpage,
-                'display_id'
-            )
-        webpage = self._download_webpage('http://www.snagfilms.com/films/title/' + display_id, display_id)
+        display_id = self._match_id(url)
 
-        json_data = self._parse_json(self._html_search_regex(
-            r'"data":{"film":(?P<data>{.*?}})}',
-            webpage,
-            'data'
-        ), display_id)
+        webpage = self._download_webpage(url, display_id)
 
-        if video_id is None:
-            video_id = json_data['id']
-            embed_webpage = self._download_webpage('http://www.snagfilms.com/embed/player?filmId=' + video_id, video_id)
+        film_id = self._search_regex(r'filmId=([\da-f-]{36})"', webpage, 'film id')
 
-        title = json_data['title']
-        duration = int(json_data['duration'])
-        description = json_data['synopsis']
-        categories = [category['title'] for category in json_data['categories']]
-        thumbnail = json_data['image']
+        snag = self._parse_json(
+            self._search_regex(
+                'Snag\.page\.data\s*=\s*(\[.+?\]);', webpage, 'snag'),
+            display_id)
 
-        sources = self._parse_json(js_to_json(self._html_search_regex(
-            r'sources: (?P<sources>\[.*?\])',
-            embed_webpage,
-            'sources',
-            flags=DOTALL
-        )), video_id)
-
-        formats = []
-        for source in sources:
-            if source['type'] == 'm3u8':
-                formats.extend(self._extract_m3u8_formats(source['file'], video_id))
-            else:
-                formats.append({'url': source['file'],'ext': source['type'], 'resolution': source['label']})
-        self._sort_formats(formats)
+        for item in snag:
+            if item.get('data', {}).get('film', {}).get('id') == film_id:
+                data = item['data']['film']
+                title = data['title']
+                description = clean_html(data.get('synopsis'))
+                thumbnail = data.get('image')
+                duration = int_or_none(data.get('duration') or data.get('runtime'))
+                categories = [
+                    category['title'] for category in data.get('categories', [])
+                    if category.get('title')]
+                break
+        else:
+            title = self._search_regex(
+                r'itemprop="title">([^<]+)<', webpage, 'title')
+            description = self._html_search_regex(
+                r'(?s)<div itemprop="description" class="film-synopsis-inner ">(.+?)</div>',
+                webpage, 'description', default=None) or self._og_search_description(webpage)
+            thumbnail = self._og_search_thumbnail(webpage)
+            duration = parse_duration(self._search_regex(
+                r'<span itemprop="duration" class="film-duration strong">([^<]+)<',
+                webpage, 'duration', fatal=False))
+            categories = re.findall(r'<a href="/movies/[^"]+">([^<]+)</a>', webpage)
 
         return {
-            'id': video_id,
+            '_type': 'url_transparent',
+            'url': 'http://embed.snagfilms.com/embed/player?filmId=%s' % film_id,
+            'id': film_id,
             'display_id': display_id,
             'title': title,
-            'duration': duration,
             'description': description,
-            'categories': categories,
             'thumbnail': thumbnail,
-            'formats': formats,
+            'duration': duration,
+            'categories': categories,
         }

From 7c197ad96dd0f36177eda66777c93502228fc36b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Jun 2015 18:25:50 +0600
Subject: [PATCH 1707/6219] [snagfilms] Add routine for generic embeds
 extractions

---
 youtube_dl/extractor/snagfilms.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/youtube_dl/extractor/snagfilms.py b/youtube_dl/extractor/snagfilms.py
index 6e103bd49..cb52eb72b 100644
--- a/youtube_dl/extractor/snagfilms.py
+++ b/youtube_dl/extractor/snagfilms.py
@@ -27,6 +27,13 @@ class SnagFilmsEmbedIE(InfoExtractor):
         'only_matching': True,
     }]
 
+    @staticmethod
+    def _extract_url(webpage):
+        mobj = re.search(
+            r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:embed\.)?snagfilms\.com/embed/player.+?)\1', webpage)
+        if mobj:
+            return mobj.group('url')
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
 

From eedd20ef9637660d5585b1b7d221d7f2d31630ac Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Jun 2015 18:26:14 +0600
Subject: [PATCH 1708/6219] [extractor/generic] Add support for snagfilms
 embeds

---
 youtube_dl/extractor/generic.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 42e4e7035..5b1da47e3 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -47,6 +47,7 @@ from .xhamster import XHamsterEmbedIE
 from .vimeo import VimeoIE
 from .dailymotion import DailymotionCloudIE
 from .onionstudios import OnionStudiosIE
+from .snagfilms import SnagFilmsEmbedIE
 
 
 class GenericIE(InfoExtractor):
@@ -1550,6 +1551,11 @@ class GenericIE(InfoExtractor):
         if onionstudios_url:
             return self.url_result(onionstudios_url)
 
+        # Look for SnagFilms embeds
+        snagfilms_url = SnagFilmsEmbedIE._extract_url(webpage)
+        if snagfilms_url:
+            return self.url_result(snagfilms_url)
+
         # Look for AdobeTVVideo embeds
         mobj = re.search(
             r'<iframe[^>]+src=[\'"]((?:https?:)?//video\.tv\.adobe\.com/v/\d+[^"]+)[\'"]',

From b8c1cc1a51c16682725cf382f0e498a390c62a2d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Jun 2015 18:28:10 +0600
Subject: [PATCH 1709/6219] [extractor/generic] Add test for snagfilms embeds

---
 youtube_dl/extractor/generic.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 5b1da47e3..32e41d13e 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -850,6 +850,15 @@ class GenericIE(InfoExtractor):
                 'uploader_id': 'clickhole',
             }
         },
+        # SnagFilms embed
+        {
+            'url': 'http://whilewewatch.blogspot.ru/2012/06/whilewewatch-whilewewatch-gripping.html',
+            'info_dict': {
+                'id': '74849a00-85a9-11e1-9660-123139220831',
+                'ext': 'mp4',
+                'title': '#whilewewatch',
+            }
+        },
         # AdobeTVVideo embed
         {
             'url': 'https://helpx.adobe.com/acrobat/how-to/new-experience-acrobat-dc.html?set=acrobat--get-started--essential-beginners',

From 242a998bdc5e808411870d7b13395b7739501f16 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Jun 2015 18:40:01 +0600
Subject: [PATCH 1710/6219] [snagfilms] Add support for shows

---
 youtube_dl/extractor/snagfilms.py | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/snagfilms.py b/youtube_dl/extractor/snagfilms.py
index cb52eb72b..f04ca6036 100644
--- a/youtube_dl/extractor/snagfilms.py
+++ b/youtube_dl/extractor/snagfilms.py
@@ -76,8 +76,8 @@ class SnagFilmsEmbedIE(InfoExtractor):
 
 
 class SnagFilmsIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?snagfilms\.com/films/title/(?P<id>[^/]+)'
-    _TEST = {
+    _VALID_URL = r'https?://(?:www\.)?snagfilms\.com/(?:films/title|show)/(?P<id>[^?#]+)'
+    _TESTS = [{
         'url': 'http://www.snagfilms.com/films/title/lost_for_life',
         'md5': '19844f897b35af219773fd63bdec2942',
         'info_dict': {
@@ -90,7 +90,20 @@ class SnagFilmsIE(InfoExtractor):
             'duration': 4489,
             'categories': ['Documentary', 'Crime', 'Award Winning', 'Festivals']
         }
-    }
+    }, {
+        'url': 'http://www.snagfilms.com/show/the_world_cut_project/india',
+        'md5': 'e6292e5b837642bbda82d7f8bf3fbdfd',
+        'info_dict': {
+            'id': '00000145-d75c-d96e-a9c7-ff5c67b20000',
+            'display_id': 'the_world_cut_project/india',
+            'ext': 'mp4',
+            'title': 'India',
+            'description': 'md5:5c168c5a8f4719c146aad2e0dfac6f5f',
+            'thumbnail': 're:^https?://.*\.jpg',
+            'duration': 979,
+            'categories': ['Documentary', 'Sports', 'Politics']
+        }
+    }]
 
     def _real_extract(self, url):
         display_id = self._match_id(url)

From 9fbfc9bd4dd076e2b00f68dc43ce4fa0ed72c949 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Jun 2015 18:50:26 +0600
Subject: [PATCH 1711/6219] [snagfilms:embed] Capture geolocation restriction
 error

---
 youtube_dl/extractor/snagfilms.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/snagfilms.py b/youtube_dl/extractor/snagfilms.py
index f04ca6036..2b1838bf9 100644
--- a/youtube_dl/extractor/snagfilms.py
+++ b/youtube_dl/extractor/snagfilms.py
@@ -4,6 +4,7 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
+    ExtractorError,
     clean_html,
     determine_ext,
     int_or_none,
@@ -30,7 +31,8 @@ class SnagFilmsEmbedIE(InfoExtractor):
     @staticmethod
     def _extract_url(webpage):
         mobj = re.search(
-            r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:embed\.)?snagfilms\.com/embed/player.+?)\1', webpage)
+            r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:embed\.)?snagfilms\.com/embed/player.+?)\1',
+            webpage)
         if mobj:
             return mobj.group('url')
 
@@ -39,6 +41,10 @@ class SnagFilmsEmbedIE(InfoExtractor):
 
         webpage = self._download_webpage(url, video_id)
 
+        if '>This film is not playable in your area.<' in webpage:
+            raise ExtractorError(
+                'This film is not playable in your area.', expected=True)
+
         formats = []
         for source in self._parse_json(js_to_json(self._search_regex(
                 r'(?s)sources:\s*(\[.+?\]),', webpage, 'json')), video_id):

From a42a1bb09dd1c86b4d35eedf1104d8e51b6c407b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Jun 2015 18:54:08 +0600
Subject: [PATCH 1712/6219] [snagfilms] Capture not available error

---
 youtube_dl/extractor/snagfilms.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/snagfilms.py b/youtube_dl/extractor/snagfilms.py
index 2b1838bf9..1be7dd13b 100644
--- a/youtube_dl/extractor/snagfilms.py
+++ b/youtube_dl/extractor/snagfilms.py
@@ -43,7 +43,7 @@ class SnagFilmsEmbedIE(InfoExtractor):
 
         if '>This film is not playable in your area.<' in webpage:
             raise ExtractorError(
-                'This film is not playable in your area.', expected=True)
+                'Film %s is not playable in your area.' % video_id, expected=True)
 
         formats = []
         for source in self._parse_json(js_to_json(self._search_regex(
@@ -116,6 +116,10 @@ class SnagFilmsIE(InfoExtractor):
 
         webpage = self._download_webpage(url, display_id)
 
+        if ">Sorry, the Film you're looking for is not available.<" in webpage:
+            raise ExtractorError(
+                'Film %s is not available.' % display_id, expected=True)
+
         film_id = self._search_regex(r'filmId=([\da-f-]{36})"', webpage, 'film id')
 
         snag = self._parse_json(

From a9de9517445a4971cb89c73d28a3c7b9ecf51b11 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Jun 2015 18:57:01 +0600
Subject: [PATCH 1713/6219] [snagfilms] More tests

---
 youtube_dl/extractor/snagfilms.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/youtube_dl/extractor/snagfilms.py b/youtube_dl/extractor/snagfilms.py
index 1be7dd13b..cf495f310 100644
--- a/youtube_dl/extractor/snagfilms.py
+++ b/youtube_dl/extractor/snagfilms.py
@@ -109,6 +109,14 @@ class SnagFilmsIE(InfoExtractor):
             'duration': 979,
             'categories': ['Documentary', 'Sports', 'Politics']
         }
+    }, {
+        # Film is not playable in your area.
+        'url': 'http://www.snagfilms.com/films/title/inside_mecca',
+        'only_matching': True,
+    }, {
+        # Film is not available.
+        'url': 'http://www.snagfilms.com/show/augie_alone/flirting',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 81103ef35d5148f52229f35e91b99ff8d9218b10 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Jun 2015 23:00:27 +0600
Subject: [PATCH 1714/6219] [smotri] Fix password protected video extraction

---
 youtube_dl/extractor/smotri.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/smotri.py b/youtube_dl/extractor/smotri.py
index 24746a09a..b8fa0457d 100644
--- a/youtube_dl/extractor/smotri.py
+++ b/youtube_dl/extractor/smotri.py
@@ -152,6 +152,10 @@ class SmotriIE(InfoExtractor):
             'getvideoinfo': '1',
         }
 
+        video_password = self._downloader.params.get('videopassword', None)
+        if video_password:
+            video_form['pass'] = hashlib.md5(video_password.encode('utf-8')).hexdigest()
+
         request = compat_urllib_request.Request(
             'http://smotri.com/video/view/url/bot/', compat_urllib_parse.urlencode(video_form))
         request.add_header('Content-Type', 'application/x-www-form-urlencoded')
@@ -161,13 +165,18 @@ class SmotriIE(InfoExtractor):
         video_url = video.get('_vidURL') or video.get('_vidURL_mp4')
 
         if not video_url:
-            if video.get('_moderate_no') or not video.get('moderated'):
+            if video.get('_moderate_no'):
                 raise ExtractorError(
                     'Video %s has not been approved by moderator' % video_id, expected=True)
 
             if video.get('error'):
                 raise ExtractorError('Video %s does not exist' % video_id, expected=True)
 
+            if video.get('_pass_protected') == 1:
+                msg = ('Invalid video password' if video_password
+                       else 'This video is protected by a password, use the --video-password option')
+                raise ExtractorError(msg, expected=True)
+
         title = video['title']
         thumbnail = video['_imgURL']
         upload_date = unified_strdate(video['added'])

From c3b7202f4fdb8a78a90a96f3cf6213f6ac215b3b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Jun 2015 23:03:26 +0600
Subject: [PATCH 1715/6219] [smotri] Remove non relevant test

---
 youtube_dl/extractor/smotri.py | 15 ---------------
 1 file changed, 15 deletions(-)

diff --git a/youtube_dl/extractor/smotri.py b/youtube_dl/extractor/smotri.py
index b8fa0457d..cc8dce3db 100644
--- a/youtube_dl/extractor/smotri.py
+++ b/youtube_dl/extractor/smotri.py
@@ -90,21 +90,6 @@ class SmotriIE(InfoExtractor):
             },
             'skip': 'Video is not approved by moderator',
         },
-        # not approved by moderator, but available
-        {
-            'url': 'http://smotri.com/video/view/?id=v28888533b73',
-            'md5': 'f44bc7adac90af518ef1ecf04893bb34',
-            'info_dict': {
-                'id': 'v28888533b73',
-                'ext': 'mp4',
-                'title': 'Russian Spies Killed By ISIL Child Soldier',
-                'uploader': 'Mopeder',
-                'uploader_id': 'mopeder',
-                'duration': 71,
-                'thumbnail': 'http://frame9.loadup.ru/d7/32/2888853.2.3.jpg',
-                'upload_date': '20150114',
-            },
-        },
         # swf player
         {
             'url': 'http://pics.smotri.com/scrubber_custom8.swf?file=v9188090500',

From 04c27802c08268c5c9e42ca226d2a848a657b82c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Jun 2015 23:31:27 +0600
Subject: [PATCH 1716/6219] [smotri] Add tests for password protected videos

---
 youtube_dl/extractor/smotri.py | 39 ++++++++++++++++++++++++++++++++--
 1 file changed, 37 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/smotri.py b/youtube_dl/extractor/smotri.py
index cc8dce3db..93a7cfe15 100644
--- a/youtube_dl/extractor/smotri.py
+++ b/youtube_dl/extractor/smotri.py
@@ -53,7 +53,7 @@ class SmotriIE(InfoExtractor):
                 'thumbnail': 'http://frame4.loadup.ru/03/ed/57591.2.3.jpg',
             },
         },
-        # video-password
+        # video-password, not approved by moderator
         {
             'url': 'http://smotri.com/video/view/?id=v1390466a13c',
             'md5': 'f6331cef33cad65a0815ee482a54440b',
@@ -71,7 +71,24 @@ class SmotriIE(InfoExtractor):
             },
             'skip': 'Video is not approved by moderator',
         },
-        # age limit + video-password
+        # video-password
+        {
+            'url': 'http://smotri.com/video/view/?id=v6984858774#',
+            'md5': 'f11e01d13ac676370fc3b95b9bda11b0',
+            'info_dict': {
+                'id': 'v6984858774',
+                'ext': 'mp4',
+                'title': 'Дача Солженицина ПАРОЛЬ 223322',
+                'uploader': 'psavari1',
+                'uploader_id': 'psavari1',
+                'upload_date': '20081103',
+                'thumbnail': 're:^https?://.*\.jpg$',
+            },
+            'params': {
+                'videopassword': '223322',
+            },
+        },
+        # age limit + video-password, not approved by moderator
         {
             'url': 'http://smotri.com/video/view/?id=v15408898bcf',
             'md5': '91e909c9f0521adf5ee86fbe073aad70',
@@ -90,6 +107,24 @@ class SmotriIE(InfoExtractor):
             },
             'skip': 'Video is not approved by moderator',
         },
+        # age limit + video-password
+        {
+            'url': 'http://smotri.com/video/view/?id=v7780025814',
+            'md5': 'b4599b068422559374a59300c5337d72',
+            'info_dict': {
+                'id': 'v7780025814',
+                'ext': 'mp4',
+                'title': 'Sexy Beach (пароль 123)',
+                'uploader': 'вАся',
+                'uploader_id': 'asya_prosto',
+                'upload_date': '20081218',
+                'thumbnail': 're:^https?://.*\.jpg$',
+                'age_limit': 18,
+            },
+            'params': {
+                'videopassword': '123'
+            },
+        },
         # swf player
         {
             'url': 'http://pics.smotri.com/scrubber_custom8.swf?file=v9188090500',

From 6c4d20cd6fae8e47c1f0c6771e2e91660bf10eb5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 28 Jun 2015 00:08:52 +0600
Subject: [PATCH 1717/6219] [downloader/external] Fix externals downloaders
 specified with extension on Windows

---
 youtube_dl/downloader/external.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index a57c15856..1d5cc9904 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -131,5 +131,6 @@ def list_external_downloaders():
 def get_external_downloader(external_downloader):
     """ Given the name of the executable, see whether we support the given
         downloader . """
-    bn = os.path.basename(external_downloader)
+    # Drop .exe extension on Windows
+    bn = os.path.splitext(os.path.basename(external_downloader))[0]
     return _BY_NAME[bn]

From 82ea1051b54134150f290d3d9f259a4baf54faf4 Mon Sep 17 00:00:00 2001
From: George Brighton <george@gebn.co.uk>
Date: Sat, 27 Jun 2015 05:11:23 +0100
Subject: [PATCH 1718/6219] [moviefap] Add new extractor

---
 youtube_dl/extractor/__init__.py |   1 +
 youtube_dl/extractor/moviefap.py | 117 +++++++++++++++++++++++++++++++
 2 files changed, 118 insertions(+)
 create mode 100644 youtube_dl/extractor/moviefap.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 7e74a971d..b70c1b57e 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -311,6 +311,7 @@ from .morningstar import MorningstarIE
 from .motherless import MotherlessIE
 from .motorsport import MotorsportIE
 from .movieclips import MovieClipsIE
+from .moviefap import MovieFapIE
 from .moviezine import MoviezineIE
 from .movshare import MovShareIE
 from .mtv import (
diff --git a/youtube_dl/extractor/moviefap.py b/youtube_dl/extractor/moviefap.py
new file mode 100644
index 000000000..49b8ab7d9
--- /dev/null
+++ b/youtube_dl/extractor/moviefap.py
@@ -0,0 +1,117 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import str_to_int
+
+
+class MovieFapIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?moviefap\.com/videos/(?P<id>[0-9a-f]+)/(?P<name>[a-z-_]+)'
+    _TESTS = [{
+        'url': 'http://www.moviefap.com/videos/e5da0d3edce5404418f5/jeune-couple-russe.html',
+        'md5': 'fa56683e291fc80635907168a743c9ad',
+        'info_dict': {
+            'id': 'e5da0d3edce5404418f5',
+            'ext': 'flv',
+            'title': 'Jeune Couple Russe',
+            'description': 'Amateur',
+            'thumbnail': 'http://pic.moviefap.com/thumbs/e5/949-18l.jpg',
+            'uploader_id': 'whiskeyjar',
+            'display_id': 'jeune-couple-russe'
+        }
+    }, {
+        'url': 'http://www.moviefap.com/videos/3080837f6712355015c2/busty-british-blonde-takes-backdoor-in-fake-taxi.html',
+        'md5': 'bedef72cb23d27a20755fc430a6d7a0e',
+        'info_dict': {
+            'id': '3080837f6712355015c2',
+            'ext': 'mp4',
+            'title': 'Busty British blonde takes backdoor in fake taxi',
+            'description': 'Big boobs British blonde flashing in fake taxi then giving titsjob and rimjob in the back seat before getting big cock up her tight ass',
+            'thumbnail': 'http://img.moviefap.com/a16:9w990r/thumbs/30/322021-18l.jpg',
+            'uploader_id': 'momcikoper',
+            'display_id': 'busty-british-blonde-takes-backdoor-in-fake-taxi'
+        }
+    }]
+
+    @staticmethod
+    def __get_thumbnail_data(xml):
+
+        """
+        Constructs a list of video thumbnails from timeline preview images.
+        :param xml: the information XML document to parse
+        """
+
+        timeline = xml.find('timeline')
+        if timeline is None:
+            # not all videos have the data - ah well
+            return []
+
+        # get the required information from the XML
+        attrs = {attr: str_to_int(timeline.find(attr).text)
+                 for attr in ['imageWidth', 'imageHeight', 'imageFirst', 'imageLast']}
+        pattern = timeline.find('imagePattern').text
+
+        # generate the list of thumbnail information dicts
+        thumbnails = []
+        for i in range(attrs['imageFirst'], attrs['imageLast'] + 1):
+            thumbnails.append({
+                'url': pattern.replace('#', str(i)),
+                'width': attrs['imageWidth'],
+                'height': attrs['imageHeight']
+            })
+        return thumbnails
+
+    def _real_extract(self, url):
+
+        # find the video ID
+        video_id = self._match_id(url)
+
+        # retrieve the page HTML
+        webpage = self._download_webpage(url, video_id)
+
+        # find the URL of the XML document detailing video download URLs
+        info_url = self._html_search_regex(r'flashvars\.config = escape\("(.+?)"', webpage, 'player parameters')
+
+        # download that XML
+        xml = self._download_xml(info_url, video_id)
+
+        # create dictionary of properties we know so far, or can find easily
+        info = {
+            'id': video_id,
+            'title': self._html_search_regex(r'<div id="view_title"><h1>(.*?)</h1>', webpage, 'title'),
+            'display_id': re.compile(self._VALID_URL).match(url).group('name'),
+            'thumbnails': self.__get_thumbnail_data(xml),
+            'thumbnail': xml.find('startThumb').text,
+            'description': self._html_search_regex(r'name="description" value="(.*?)"', webpage, 'description'),
+            'uploader_id': self._html_search_regex(r'name="username" value="(.*?)"', webpage, 'uploader_id'),
+            'view_count': str_to_int(self._html_search_regex(r'<br>Views <strong>([0-9]+)</strong>', webpage, 'view_count')),
+            'average_rating': float(self._html_search_regex(r'Current Rating<br> <strong>(.*?)</strong>', webpage, 'average_rating')),
+            'comment_count': str_to_int(self._html_search_regex(r'<span id="comCount">([0-9]+)</span>', webpage, 'comment_count')),
+            'age_limit': 18,
+            'webpage_url': self._html_search_regex(r'name="link" value="(.*?)"', webpage, 'webpage_url'),
+            'categories': self._html_search_regex(r'</div>\s*(.*?)\s*<br>', webpage, 'categories').split(', ')
+        }
+
+        # find and add the format
+        if xml.find('videoConfig') is not None:
+            info['ext'] = xml.find('videoConfig').find('type').text
+        else:
+            info['ext'] = 'flv'  # guess...
+
+        # work out the video URL(s)
+        if xml.find('videoLink') is not None:
+            # single format available
+            info['url'] = xml.find('videoLink').text
+        else:
+            # multiple formats available
+            info['formats'] = []
+
+            # N.B. formats are already in ascending order of quality
+            for item in xml.find('quality').findall('item'):
+                info['formats'].append({
+                    'url': item.find('videoLink').text,
+                    'resolution': item.find('res').text  # 480p etc.
+                })
+
+        return info

From 71f9e49e67bfe62c9f1a6d8f74b7d81ab820ba84 Mon Sep 17 00:00:00 2001
From: George Brighton <george@gebn.co.uk>
Date: Sat, 27 Jun 2015 05:22:35 +0100
Subject: [PATCH 1719/6219] [moviefap] Fix dictionary comprehension syntax
 incompatible with Python 2.6

---
 youtube_dl/extractor/moviefap.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/moviefap.py b/youtube_dl/extractor/moviefap.py
index 49b8ab7d9..88f9dab6f 100644
--- a/youtube_dl/extractor/moviefap.py
+++ b/youtube_dl/extractor/moviefap.py
@@ -48,17 +48,19 @@ class MovieFapIE(InfoExtractor):
             return []
 
         # get the required information from the XML
-        attrs = {attr: str_to_int(timeline.find(attr).text)
-                 for attr in ['imageWidth', 'imageHeight', 'imageFirst', 'imageLast']}
+        width = str_to_int(timeline.find('imageWidth').text)
+        height = str_to_int(timeline.find('imageHeight').text)
+        first = str_to_int(timeline.find('imageFirst').text)
+        last = str_to_int(timeline.find('imageLast').text)
         pattern = timeline.find('imagePattern').text
 
         # generate the list of thumbnail information dicts
         thumbnails = []
-        for i in range(attrs['imageFirst'], attrs['imageLast'] + 1):
+        for i in range(first, last + 1):
             thumbnails.append({
                 'url': pattern.replace('#', str(i)),
-                'width': attrs['imageWidth'],
-                'height': attrs['imageHeight']
+                'width': width,
+                'height': height
             })
         return thumbnails
 

From 802d74aa6ba2613f95d19c3be6c2480b9a2ebe5b Mon Sep 17 00:00:00 2001
From: George Brighton <george@gebn.co.uk>
Date: Sat, 27 Jun 2015 20:01:27 +0100
Subject: [PATCH 1720/6219] [moviefap] Swap test for an alternative
 non-copyrighted video

---
 youtube_dl/extractor/moviefap.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/moviefap.py b/youtube_dl/extractor/moviefap.py
index 88f9dab6f..880ea2247 100644
--- a/youtube_dl/extractor/moviefap.py
+++ b/youtube_dl/extractor/moviefap.py
@@ -21,16 +21,16 @@ class MovieFapIE(InfoExtractor):
             'display_id': 'jeune-couple-russe'
         }
     }, {
-        'url': 'http://www.moviefap.com/videos/3080837f6712355015c2/busty-british-blonde-takes-backdoor-in-fake-taxi.html',
-        'md5': 'bedef72cb23d27a20755fc430a6d7a0e',
+        'url': 'http://www.moviefap.com/videos/be9867c9416c19f54a4a/experienced-milf-amazing-handjob.html',
+        'md5': '26624b4e2523051b550067d547615906',
         'info_dict': {
-            'id': '3080837f6712355015c2',
+            'id': 'be9867c9416c19f54a4a',
             'ext': 'mp4',
-            'title': 'Busty British blonde takes backdoor in fake taxi',
-            'description': 'Big boobs British blonde flashing in fake taxi then giving titsjob and rimjob in the back seat before getting big cock up her tight ass',
-            'thumbnail': 'http://img.moviefap.com/a16:9w990r/thumbs/30/322021-18l.jpg',
-            'uploader_id': 'momcikoper',
-            'display_id': 'busty-british-blonde-takes-backdoor-in-fake-taxi'
+            'title': 'Experienced MILF Amazing Handjob',
+            'description': 'Experienced MILF giving an Amazing Handjob',
+            'thumbnail': 'http://img.moviefap.com/a16:9w990r/thumbs/be/322032-20l.jpg',
+            'uploader_id': 'darvinfred06',
+            'display_id': 'experienced-milf-amazing-handjob'
         }
     }]
 

From 9c494108983c2a951780217014c26d8f3b081682 Mon Sep 17 00:00:00 2001
From: George Brighton <george@gebn.co.uk>
Date: Sat, 27 Jun 2015 20:06:44 +0100
Subject: [PATCH 1721/6219] [moviefap] Add categories to tests

---
 youtube_dl/extractor/moviefap.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/moviefap.py b/youtube_dl/extractor/moviefap.py
index 880ea2247..6b79183c7 100644
--- a/youtube_dl/extractor/moviefap.py
+++ b/youtube_dl/extractor/moviefap.py
@@ -18,7 +18,8 @@ class MovieFapIE(InfoExtractor):
             'description': 'Amateur',
             'thumbnail': 'http://pic.moviefap.com/thumbs/e5/949-18l.jpg',
             'uploader_id': 'whiskeyjar',
-            'display_id': 'jeune-couple-russe'
+            'display_id': 'jeune-couple-russe',
+            'categories': ['Amateur', 'Teen']
         }
     }, {
         'url': 'http://www.moviefap.com/videos/be9867c9416c19f54a4a/experienced-milf-amazing-handjob.html',
@@ -30,7 +31,8 @@ class MovieFapIE(InfoExtractor):
             'description': 'Experienced MILF giving an Amazing Handjob',
             'thumbnail': 'http://img.moviefap.com/a16:9w990r/thumbs/be/322032-20l.jpg',
             'uploader_id': 'darvinfred06',
-            'display_id': 'experienced-milf-amazing-handjob'
+            'display_id': 'experienced-milf-amazing-handjob',
+            'categories': ['Amateur', 'Masturbation', 'Mature', 'Flashing']
         }
     }]
 

From a8e6f30d8ed13209f2e9815ae97b3edccd38561b Mon Sep 17 00:00:00 2001
From: George Brighton <george@gebn.co.uk>
Date: Sat, 27 Jun 2015 20:14:15 +0100
Subject: [PATCH 1722/6219] [moviefap] Swap and justify tests

---
 youtube_dl/extractor/moviefap.py | 28 +++++++++++++++-------------
 1 file changed, 15 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/moviefap.py b/youtube_dl/extractor/moviefap.py
index 6b79183c7..1985c53c0 100644
--- a/youtube_dl/extractor/moviefap.py
+++ b/youtube_dl/extractor/moviefap.py
@@ -9,19 +9,7 @@ from ..utils import str_to_int
 class MovieFapIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?moviefap\.com/videos/(?P<id>[0-9a-f]+)/(?P<name>[a-z-_]+)'
     _TESTS = [{
-        'url': 'http://www.moviefap.com/videos/e5da0d3edce5404418f5/jeune-couple-russe.html',
-        'md5': 'fa56683e291fc80635907168a743c9ad',
-        'info_dict': {
-            'id': 'e5da0d3edce5404418f5',
-            'ext': 'flv',
-            'title': 'Jeune Couple Russe',
-            'description': 'Amateur',
-            'thumbnail': 'http://pic.moviefap.com/thumbs/e5/949-18l.jpg',
-            'uploader_id': 'whiskeyjar',
-            'display_id': 'jeune-couple-russe',
-            'categories': ['Amateur', 'Teen']
-        }
-    }, {
+        # normal, multi-format video
         'url': 'http://www.moviefap.com/videos/be9867c9416c19f54a4a/experienced-milf-amazing-handjob.html',
         'md5': '26624b4e2523051b550067d547615906',
         'info_dict': {
@@ -34,6 +22,20 @@ class MovieFapIE(InfoExtractor):
             'display_id': 'experienced-milf-amazing-handjob',
             'categories': ['Amateur', 'Masturbation', 'Mature', 'Flashing']
         }
+    }, {
+        # quirky single-format case where the extension is given as fid, but the video is really an flv
+        'url': 'http://www.moviefap.com/videos/e5da0d3edce5404418f5/jeune-couple-russe.html',
+        'md5': 'fa56683e291fc80635907168a743c9ad',
+        'info_dict': {
+            'id': 'e5da0d3edce5404418f5',
+            'ext': 'flv',
+            'title': 'Jeune Couple Russe',
+            'description': 'Amateur',
+            'thumbnail': 'http://pic.moviefap.com/thumbs/e5/949-18l.jpg',
+            'uploader_id': 'whiskeyjar',
+            'display_id': 'jeune-couple-russe',
+            'categories': ['Amateur', 'Teen']
+        }
     }]
 
     @staticmethod

From 23e7cba87fbcec9aa2b1fbccf60c2d560df3d7ad Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 28 Jun 2015 01:22:25 +0600
Subject: [PATCH 1723/6219] [twitter:card] Add extractor (#5239)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/twitter.py  | 72 ++++++++++++++++++++++++++++++++
 2 files changed, 73 insertions(+)
 create mode 100644 youtube_dl/extractor/twitter.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 7e74a971d..c0a825e8a 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -621,6 +621,7 @@ from .twitch import (
     TwitchBookmarksIE,
     TwitchStreamIE,
 )
+from .twitter import TwitterCardIE
 from .ubu import UbuIE
 from .udemy import (
     UdemyIE,
diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
new file mode 100644
index 000000000..1aaa06305
--- /dev/null
+++ b/youtube_dl/extractor/twitter.py
@@ -0,0 +1,72 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..compat import compat_urllib_request
+from ..utils import (
+    float_or_none,
+    unescapeHTML,
+)
+
+
+class TwitterCardIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?twitter\.com/i/cards/tfw/v1/(?P<id>\d+)'
+    _TEST = {
+        'url': 'https://twitter.com/i/cards/tfw/v1/560070183650213889',
+        'md5': 'a74f50b310c83170319ba16de6955192',
+        'info_dict': {
+            'id': '560070183650213889',
+            'ext': 'mp4',
+            'title': 'TwitterCard',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 30.033,
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        # Different formats served for different User-Agents
+        USER_AGENTS = [
+            'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20150101 Firefox/20.0 (Chrome)',  # mp4
+            'Mozilla/5.0 (Windows NT 5.2; WOW64; rv:38.0) Gecko/20100101 Firefox/38.0',  # webm
+        ]
+
+        config = None
+        formats = []
+        for user_agent in USER_AGENTS:
+            request = compat_urllib_request.Request(url)
+            request.add_header('User-Agent', user_agent)
+            webpage = self._download_webpage(request, video_id)
+
+            config = self._parse_json(
+                unescapeHTML(self._search_regex(
+                    r'data-player-config="([^"]+)"', webpage, 'data player config')),
+                video_id)
+
+            video_url = config['playlist'][0]['source']
+
+            f = {
+                'url': video_url,
+            }
+
+            m = re.search(r'/(?P<width>\d+)x(?P<height>\d+)/', video_url)
+            if m:
+                f.update({
+                    'width': int(m.group('width')),
+                    'height': int(m.group('height')),
+                })
+            formats.append(f)
+        self._sort_formats(formats)
+
+        thumbnail = config.get('posterImageUrl')
+        duration = float_or_none(config.get('duration'))
+
+        return {
+            'id': video_id,
+            'title': 'TwitterCard',
+            'thumbnail': thumbnail,
+            'duration': duration,
+            'formats': formats,
+        }

From d16ef949ca94ffb998c4db5a8367f0b367d659e3 Mon Sep 17 00:00:00 2001
From: George Brighton <george@gebn.co.uk>
Date: Sat, 27 Jun 2015 20:36:46 +0100
Subject: [PATCH 1724/6219] [moviefap] Allow non-critical fields to change
 without breaking extraction

---
 youtube_dl/extractor/moviefap.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/moviefap.py b/youtube_dl/extractor/moviefap.py
index 1985c53c0..23575d30a 100644
--- a/youtube_dl/extractor/moviefap.py
+++ b/youtube_dl/extractor/moviefap.py
@@ -89,14 +89,14 @@ class MovieFapIE(InfoExtractor):
             'display_id': re.compile(self._VALID_URL).match(url).group('name'),
             'thumbnails': self.__get_thumbnail_data(xml),
             'thumbnail': xml.find('startThumb').text,
-            'description': self._html_search_regex(r'name="description" value="(.*?)"', webpage, 'description'),
-            'uploader_id': self._html_search_regex(r'name="username" value="(.*?)"', webpage, 'uploader_id'),
-            'view_count': str_to_int(self._html_search_regex(r'<br>Views <strong>([0-9]+)</strong>', webpage, 'view_count')),
-            'average_rating': float(self._html_search_regex(r'Current Rating<br> <strong>(.*?)</strong>', webpage, 'average_rating')),
-            'comment_count': str_to_int(self._html_search_regex(r'<span id="comCount">([0-9]+)</span>', webpage, 'comment_count')),
+            'description': self._html_search_regex(r'name="description" value="(.*?)"', webpage, 'description', fatal=False),
+            'uploader_id': self._html_search_regex(r'name="username" value="(.*?)"', webpage, 'uploader_id', fatal=False),
+            'view_count': str_to_int(self._html_search_regex(r'<br>Views <strong>([0-9]+)</strong>', webpage, 'view_count, fatal=False')),
+            'average_rating': float(self._html_search_regex(r'Current Rating<br> <strong>(.*?)</strong>', webpage, 'average_rating', fatal=False)),
+            'comment_count': str_to_int(self._html_search_regex(r'<span id="comCount">([0-9]+)</span>', webpage, 'comment_count', fatal=False)),
             'age_limit': 18,
-            'webpage_url': self._html_search_regex(r'name="link" value="(.*?)"', webpage, 'webpage_url'),
-            'categories': self._html_search_regex(r'</div>\s*(.*?)\s*<br>', webpage, 'categories').split(', ')
+            'webpage_url': self._html_search_regex(r'name="link" value="(.*?)"', webpage, 'webpage_url', fatal=False),
+            'categories': self._html_search_regex(r'</div>\s*(.*?)\s*<br>', webpage, 'categories', fatal=False).split(', ')
         }
 
         # find and add the format

From 62b742ece3ec6c7d7fd24898b5413b6b98a4ae8f Mon Sep 17 00:00:00 2001
From: George Brighton <george@gebn.co.uk>
Date: Sat, 27 Jun 2015 20:51:11 +0100
Subject: [PATCH 1725/6219] [moviefap] Remove redundant comments

---
 youtube_dl/extractor/moviefap.py | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/moviefap.py b/youtube_dl/extractor/moviefap.py
index 23575d30a..b38a8e71f 100644
--- a/youtube_dl/extractor/moviefap.py
+++ b/youtube_dl/extractor/moviefap.py
@@ -70,19 +70,13 @@ class MovieFapIE(InfoExtractor):
 
     def _real_extract(self, url):
 
-        # find the video ID
         video_id = self._match_id(url)
-
-        # retrieve the page HTML
         webpage = self._download_webpage(url, video_id)
 
-        # find the URL of the XML document detailing video download URLs
+        # find and retrieve the XML document detailing video download URLs
         info_url = self._html_search_regex(r'flashvars\.config = escape\("(.+?)"', webpage, 'player parameters')
-
-        # download that XML
         xml = self._download_xml(info_url, video_id)
 
-        # create dictionary of properties we know so far, or can find easily
         info = {
             'id': video_id,
             'title': self._html_search_regex(r'<div id="view_title"><h1>(.*?)</h1>', webpage, 'title'),

From 43b925ce74efd0a011f7880dcdcc90f4cf3b8f4b Mon Sep 17 00:00:00 2001
From: George Brighton <george@gebn.co.uk>
Date: Sat, 27 Jun 2015 20:52:12 +0100
Subject: [PATCH 1726/6219] [moviefap] Replace calls to `find()` with
 `util.xpath_text()`.

---
 youtube_dl/extractor/moviefap.py | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/moviefap.py b/youtube_dl/extractor/moviefap.py
index b38a8e71f..6da93dbc9 100644
--- a/youtube_dl/extractor/moviefap.py
+++ b/youtube_dl/extractor/moviefap.py
@@ -3,7 +3,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import str_to_int
+from ..utils import (
+    xpath_text,
+    str_to_int
+)
 
 
 class MovieFapIE(InfoExtractor):
@@ -82,7 +85,7 @@ class MovieFapIE(InfoExtractor):
             'title': self._html_search_regex(r'<div id="view_title"><h1>(.*?)</h1>', webpage, 'title'),
             'display_id': re.compile(self._VALID_URL).match(url).group('name'),
             'thumbnails': self.__get_thumbnail_data(xml),
-            'thumbnail': xml.find('startThumb').text,
+            'thumbnail': xpath_text(xml, 'startThumb', 'thumbnail'),
             'description': self._html_search_regex(r'name="description" value="(.*?)"', webpage, 'description', fatal=False),
             'uploader_id': self._html_search_regex(r'name="username" value="(.*?)"', webpage, 'uploader_id', fatal=False),
             'view_count': str_to_int(self._html_search_regex(r'<br>Views <strong>([0-9]+)</strong>', webpage, 'view_count, fatal=False')),
@@ -102,7 +105,7 @@ class MovieFapIE(InfoExtractor):
         # work out the video URL(s)
         if xml.find('videoLink') is not None:
             # single format available
-            info['url'] = xml.find('videoLink').text
+            info['url'] = xpath_text(xml, 'videoLink', 'url', True)
         else:
             # multiple formats available
             info['formats'] = []
@@ -110,8 +113,8 @@ class MovieFapIE(InfoExtractor):
             # N.B. formats are already in ascending order of quality
             for item in xml.find('quality').findall('item'):
                 info['formats'].append({
-                    'url': item.find('videoLink').text,
-                    'resolution': item.find('res').text  # 480p etc.
+                    'url': xpath_text(item, 'videoLink', 'url', True),
+                    'resolution': xpath_text(item, 'res', 'resolution', True)  # 480p etc.
                 })
 
         return info

From b971abe897ee17fed7e36868fdc8880f6b145d7b Mon Sep 17 00:00:00 2001
From: George Brighton <george@gebn.co.uk>
Date: Sat, 27 Jun 2015 21:04:53 +0100
Subject: [PATCH 1727/6219] [moviefap] Replace call to `str()` with
 `compat.compat_str()`

---
 youtube_dl/extractor/moviefap.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/moviefap.py b/youtube_dl/extractor/moviefap.py
index 6da93dbc9..20a78f3b2 100644
--- a/youtube_dl/extractor/moviefap.py
+++ b/youtube_dl/extractor/moviefap.py
@@ -7,6 +7,7 @@ from ..utils import (
     xpath_text,
     str_to_int
 )
+from ..compat import compat_str
 
 
 class MovieFapIE(InfoExtractor):
@@ -65,7 +66,7 @@ class MovieFapIE(InfoExtractor):
         thumbnails = []
         for i in range(first, last + 1):
             thumbnails.append({
-                'url': pattern.replace('#', str(i)),
+                'url': pattern.replace('#', compat_str(i)),
                 'width': width,
                 'height': height
             })

From 8a1b49ff19a8a1fdc2c30cf10cc0598ac9bc8819 Mon Sep 17 00:00:00 2001
From: George Brighton <george@gebn.co.uk>
Date: Sat, 27 Jun 2015 22:27:06 +0100
Subject: [PATCH 1728/6219] [moviefap] Explicitly sort formats to handle
 possible site changes

---
 youtube_dl/extractor/moviefap.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/moviefap.py b/youtube_dl/extractor/moviefap.py
index 20a78f3b2..295bfe3f0 100644
--- a/youtube_dl/extractor/moviefap.py
+++ b/youtube_dl/extractor/moviefap.py
@@ -111,11 +111,14 @@ class MovieFapIE(InfoExtractor):
             # multiple formats available
             info['formats'] = []
 
-            # N.B. formats are already in ascending order of quality
             for item in xml.find('quality').findall('item'):
+                resolution = xpath_text(item, 'res', 'resolution', True)  # 480p etc.
                 info['formats'].append({
                     'url': xpath_text(item, 'videoLink', 'url', True),
-                    'resolution': xpath_text(item, 'res', 'resolution', True)  # 480p etc.
+                    'resolution': resolution,
+                    'height': int(re.findall(r'\d+', resolution)[0])
                 })
 
+            self._sort_formats(info['formats'])
+
         return info

From 1a5fd4eebc2717b5173df50d65007f90cb05ee30 Mon Sep 17 00:00:00 2001
From: George Brighton <george@gebn.co.uk>
Date: Sat, 27 Jun 2015 22:32:56 +0100
Subject: [PATCH 1729/6219] [moviefap] Wrap long lines

---
 youtube_dl/extractor/moviefap.py | 27 ++++++++++++++++++---------
 1 file changed, 18 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/moviefap.py b/youtube_dl/extractor/moviefap.py
index 295bfe3f0..9de052a99 100644
--- a/youtube_dl/extractor/moviefap.py
+++ b/youtube_dl/extractor/moviefap.py
@@ -78,23 +78,32 @@ class MovieFapIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         # find and retrieve the XML document detailing video download URLs
-        info_url = self._html_search_regex(r'flashvars\.config = escape\("(.+?)"', webpage, 'player parameters')
+        info_url = self._html_search_regex( \
+                r'flashvars\.config = escape\("(.+?)"', webpage, 'player parameters')
         xml = self._download_xml(info_url, video_id)
 
         info = {
             'id': video_id,
-            'title': self._html_search_regex(r'<div id="view_title"><h1>(.*?)</h1>', webpage, 'title'),
+            'title': self._html_search_regex( \
+                    r'<div id="view_title"><h1>(.*?)</h1>', webpage, 'title'),
             'display_id': re.compile(self._VALID_URL).match(url).group('name'),
             'thumbnails': self.__get_thumbnail_data(xml),
             'thumbnail': xpath_text(xml, 'startThumb', 'thumbnail'),
-            'description': self._html_search_regex(r'name="description" value="(.*?)"', webpage, 'description', fatal=False),
-            'uploader_id': self._html_search_regex(r'name="username" value="(.*?)"', webpage, 'uploader_id', fatal=False),
-            'view_count': str_to_int(self._html_search_regex(r'<br>Views <strong>([0-9]+)</strong>', webpage, 'view_count, fatal=False')),
-            'average_rating': float(self._html_search_regex(r'Current Rating<br> <strong>(.*?)</strong>', webpage, 'average_rating', fatal=False)),
-            'comment_count': str_to_int(self._html_search_regex(r'<span id="comCount">([0-9]+)</span>', webpage, 'comment_count', fatal=False)),
+            'description': self._html_search_regex( \
+                    r'name="description" value="(.*?)"', webpage, 'description', fatal=False),
+            'uploader_id': self._html_search_regex( \
+                    r'name="username" value="(.*?)"', webpage, 'uploader_id', fatal=False),
+            'view_count': str_to_int(self._html_search_regex( \
+                    r'<br>Views <strong>([0-9]+)</strong>', webpage, 'view_count, fatal=False')),
+            'average_rating': float(self._html_search_regex( \
+                    r'Current Rating<br> <strong>(.*?)</strong>', webpage, 'average_rating', fatal=False)),
+            'comment_count': str_to_int(self._html_search_regex( \
+                    r'<span id="comCount">([0-9]+)</span>', webpage, 'comment_count', fatal=False)),
             'age_limit': 18,
-            'webpage_url': self._html_search_regex(r'name="link" value="(.*?)"', webpage, 'webpage_url', fatal=False),
-            'categories': self._html_search_regex(r'</div>\s*(.*?)\s*<br>', webpage, 'categories', fatal=False).split(', ')
+            'webpage_url': self._html_search_regex( \
+                    r'name="link" value="(.*?)"', webpage, 'webpage_url', fatal=False),
+            'categories': self._html_search_regex( \
+                    r'</div>\s*(.*?)\s*<br>', webpage, 'categories', fatal=False).split(', ')
         }
 
         # find and add the format

From 5a9cc19972fb3aae7a67470f65ec5cd30918f4e1 Mon Sep 17 00:00:00 2001
From: George Brighton <george@gebn.co.uk>
Date: Sat, 27 Jun 2015 23:03:06 +0100
Subject: [PATCH 1730/6219] [moviefap] Move flv videos to formats in the
 metadata

---
 youtube_dl/extractor/moviefap.py | 56 +++++++++++++++++---------------
 1 file changed, 29 insertions(+), 27 deletions(-)

diff --git a/youtube_dl/extractor/moviefap.py b/youtube_dl/extractor/moviefap.py
index 9de052a99..5e0c701d4 100644
--- a/youtube_dl/extractor/moviefap.py
+++ b/youtube_dl/extractor/moviefap.py
@@ -82,8 +82,36 @@ class MovieFapIE(InfoExtractor):
                 r'flashvars\.config = escape\("(.+?)"', webpage, 'player parameters')
         xml = self._download_xml(info_url, video_id)
 
-        info = {
+        # find the video container
+        if xml.find('videoConfig') is not None:
+            ext = xml.find('videoConfig').find('type').text
+        else:
+            ext = 'flv'  # guess...
+
+        # work out the video URL(s)
+        formats = []
+        if xml.find('videoLink') is not None:
+            # single format available
+            formats.append({
+                'url': xpath_text(xml, 'videoLink', 'url', True),
+                'ext': ext
+            })
+        else:
+            # multiple formats available
+            for item in xml.find('quality').findall('item'):
+                resolution = xpath_text(item, 'res', 'resolution', True)  # 480p etc.
+                formats.append({
+                    'url': xpath_text(item, 'videoLink', 'url', True),
+                    'ext': ext,
+                    'resolution': resolution,
+                    'height': int(re.findall(r'\d+', resolution)[0])
+                })
+
+            self._sort_formats(formats)
+
+        return {
             'id': video_id,
+            'formats': formats,
             'title': self._html_search_regex( \
                     r'<div id="view_title"><h1>(.*?)</h1>', webpage, 'title'),
             'display_id': re.compile(self._VALID_URL).match(url).group('name'),
@@ -105,29 +133,3 @@ class MovieFapIE(InfoExtractor):
             'categories': self._html_search_regex( \
                     r'</div>\s*(.*?)\s*<br>', webpage, 'categories', fatal=False).split(', ')
         }
-
-        # find and add the format
-        if xml.find('videoConfig') is not None:
-            info['ext'] = xml.find('videoConfig').find('type').text
-        else:
-            info['ext'] = 'flv'  # guess...
-
-        # work out the video URL(s)
-        if xml.find('videoLink') is not None:
-            # single format available
-            info['url'] = xpath_text(xml, 'videoLink', 'url', True)
-        else:
-            # multiple formats available
-            info['formats'] = []
-
-            for item in xml.find('quality').findall('item'):
-                resolution = xpath_text(item, 'res', 'resolution', True)  # 480p etc.
-                info['formats'].append({
-                    'url': xpath_text(item, 'videoLink', 'url', True),
-                    'resolution': resolution,
-                    'height': int(re.findall(r'\d+', resolution)[0])
-                })
-
-            self._sort_formats(info['formats'])
-
-        return info

From db652ea186586e3eda5006ee096161b1a867c0d0 Mon Sep 17 00:00:00 2001
From: George Brighton <george@gebn.co.uk>
Date: Sat, 27 Jun 2015 23:04:55 +0100
Subject: [PATCH 1731/6219] [moviefap] Fix `flake8` warnings introduced in
 1a5fd4e

---
 youtube_dl/extractor/moviefap.py | 36 ++++++++++++++++----------------
 1 file changed, 18 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/moviefap.py b/youtube_dl/extractor/moviefap.py
index 5e0c701d4..82b863539 100644
--- a/youtube_dl/extractor/moviefap.py
+++ b/youtube_dl/extractor/moviefap.py
@@ -78,8 +78,8 @@ class MovieFapIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         # find and retrieve the XML document detailing video download URLs
-        info_url = self._html_search_regex( \
-                r'flashvars\.config = escape\("(.+?)"', webpage, 'player parameters')
+        info_url = self._html_search_regex(
+            r'flashvars\.config = escape\("(.+?)"', webpage, 'player parameters')
         xml = self._download_xml(info_url, video_id)
 
         # find the video container
@@ -112,24 +112,24 @@ class MovieFapIE(InfoExtractor):
         return {
             'id': video_id,
             'formats': formats,
-            'title': self._html_search_regex( \
-                    r'<div id="view_title"><h1>(.*?)</h1>', webpage, 'title'),
+            'title': self._html_search_regex(
+                r'<div id="view_title"><h1>(.*?)</h1>', webpage, 'title'),
             'display_id': re.compile(self._VALID_URL).match(url).group('name'),
             'thumbnails': self.__get_thumbnail_data(xml),
             'thumbnail': xpath_text(xml, 'startThumb', 'thumbnail'),
-            'description': self._html_search_regex( \
-                    r'name="description" value="(.*?)"', webpage, 'description', fatal=False),
-            'uploader_id': self._html_search_regex( \
-                    r'name="username" value="(.*?)"', webpage, 'uploader_id', fatal=False),
-            'view_count': str_to_int(self._html_search_regex( \
-                    r'<br>Views <strong>([0-9]+)</strong>', webpage, 'view_count, fatal=False')),
-            'average_rating': float(self._html_search_regex( \
-                    r'Current Rating<br> <strong>(.*?)</strong>', webpage, 'average_rating', fatal=False)),
-            'comment_count': str_to_int(self._html_search_regex( \
-                    r'<span id="comCount">([0-9]+)</span>', webpage, 'comment_count', fatal=False)),
+            'description': self._html_search_regex(
+                r'name="description" value="(.*?)"', webpage, 'description', fatal=False),
+            'uploader_id': self._html_search_regex(
+                r'name="username" value="(.*?)"', webpage, 'uploader_id', fatal=False),
+            'view_count': str_to_int(self._html_search_regex(
+                r'<br>Views <strong>([0-9]+)</strong>', webpage, 'view_count, fatal=False')),
+            'average_rating': float(self._html_search_regex(
+                r'Current Rating<br> <strong>(.*?)</strong>', webpage, 'average_rating', fatal=False)),
+            'comment_count': str_to_int(self._html_search_regex(
+                r'<span id="comCount">([0-9]+)</span>', webpage, 'comment_count', fatal=False)),
             'age_limit': 18,
-            'webpage_url': self._html_search_regex( \
-                    r'name="link" value="(.*?)"', webpage, 'webpage_url', fatal=False),
-            'categories': self._html_search_regex( \
-                    r'</div>\s*(.*?)\s*<br>', webpage, 'categories', fatal=False).split(', ')
+            'webpage_url': self._html_search_regex(
+                r'name="link" value="(.*?)"', webpage, 'webpage_url', fatal=False),
+            'categories': self._html_search_regex(
+                r'</div>\s*(.*?)\s*<br>', webpage, 'categories', fatal=False).split(', ')
         }

From bb512e57dc138b261cf9c71a833b0df5d5ba849f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 28 Jun 2015 13:25:59 +0800
Subject: [PATCH 1732/6219] [twitch:vod] Fix 'Source' format in m3u8 (closes
 #6115)

---
 youtube_dl/extractor/twitch.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 94bd6345d..3e798e62d 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -215,7 +215,7 @@ class TwitchVodIE(TwitchItemBaseIE):
             '%s/api/vods/%s/access_token' % (self._API_BASE, item_id), item_id,
             'Downloading %s access token' % self._ITEM_TYPE)
         formats = self._extract_m3u8_formats(
-            '%s/vod/%s?nauth=%s&nauthsig=%s'
+            '%s/vod/%s?nauth=%s&nauthsig=%s&allow_source=true'
             % (self._USHER_BASE, item_id, access_token['token'], access_token['sig']),
             item_id, 'mp4')
         self._prefer_source(formats)

From ac0474f89d3e6f8c8c1fb3223a16a18a2fd02bcb Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 28 Jun 2015 13:31:37 +0800
Subject: [PATCH 1733/6219] [twitch:vod] Update _TEST

The original test case is gone
---
 youtube_dl/extractor/twitch.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 3e798e62d..b56ee2959 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -189,17 +189,17 @@ class TwitchVodIE(TwitchItemBaseIE):
     _ITEM_SHORTCUT = 'v'
 
     _TEST = {
-        'url': 'http://www.twitch.tv/ksptv/v/3622000',
+        'url': 'http://www.twitch.tv/riotgames/v/6528877',
         'info_dict': {
-            'id': 'v3622000',
+            'id': 'v6528877',
             'ext': 'mp4',
-            'title': '''KSPTV: Squadcast: "Everyone's on vacation so here's Dahud" Edition!''',
+            'title': 'LCK Summer Split - Week 6 Day 1',
             'thumbnail': 're:^https?://.*\.jpg$',
-            'duration': 6951,
-            'timestamp': 1419028564,
-            'upload_date': '20141219',
-            'uploader': 'KSPTV',
-            'uploader_id': 'ksptv',
+            'duration': 17208,
+            'timestamp': 1435131709,
+            'upload_date': '20150624',
+            'uploader': 'Riot Games',
+            'uploader_id': 'riotgames',
             'view_count': int,
         },
         'params': {

From 9603e8a7d998615d3da1af47461ec9c353ec4e7a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 28 Jun 2015 22:55:28 +0600
Subject: [PATCH 1734/6219] [YoutubeDL] Handle None width and height similarly
 to formats

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index ef0f71bad..411de9ac9 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1008,7 +1008,7 @@ class YoutubeDL(object):
                 t.get('preference'), t.get('width'), t.get('height'),
                 t.get('id'), t.get('url')))
             for i, t in enumerate(thumbnails):
-                if 'width' in t and 'height' in t:
+                if t.get('width') and t.get('height'):
                     t['resolution'] = '%dx%d' % (t['width'], t['height'])
                 if t.get('id') is None:
                     t['id'] = '%d' % i

From bf42a9906d9a066d32f1cc50e1b033e6676744ba Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 28 Jun 2015 22:56:07 +0600
Subject: [PATCH 1735/6219] [utils] Add default value for xpath_text

---
 youtube_dl/utils.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 96490f112..942f76d24 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -62,6 +62,8 @@ std_headers = {
 }
 
 
+NO_DEFAULT = object()
+
 ENGLISH_MONTH_NAMES = [
     'January', 'February', 'March', 'April', 'May', 'June',
     'July', 'August', 'September', 'October', 'November', 'December']
@@ -171,13 +173,15 @@ def xpath_with_ns(path, ns_map):
     return '/'.join(replaced)
 
 
-def xpath_text(node, xpath, name=None, fatal=False):
+def xpath_text(node, xpath, name=None, fatal=False, default=NO_DEFAULT):
     if sys.version_info < (2, 7):  # Crazy 2.6
         xpath = xpath.encode('ascii')
 
     n = node.find(xpath)
     if n is None or n.text is None:
-        if fatal:
+        if default is not NO_DEFAULT:
+            return default
+        elif fatal:
             name = xpath if name is None else name
             raise ExtractorError('Could not find XML element %s' % name)
         else:

From c342041fba9283ba5f05f48427aabf79adcf8647 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 28 Jun 2015 22:56:45 +0600
Subject: [PATCH 1736/6219] [extractor/common] Use NO_DEFAULT from utils

---
 youtube_dl/extractor/common.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 49e4dc710..7fa46d295 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -22,6 +22,7 @@ from ..compat import (
     compat_str,
 )
 from ..utils import (
+    NO_DEFAULT,
     age_restricted,
     bug_reports_message,
     clean_html,
@@ -33,7 +34,7 @@ from ..utils import (
     sanitize_filename,
     unescapeHTML,
 )
-_NO_DEFAULT = object()
+
 
 
 class InfoExtractor(object):
@@ -523,7 +524,7 @@ class InfoExtractor(object):
             video_info['description'] = playlist_description
         return video_info
 
-    def _search_regex(self, pattern, string, name, default=_NO_DEFAULT, fatal=True, flags=0, group=None):
+    def _search_regex(self, pattern, string, name, default=NO_DEFAULT, fatal=True, flags=0, group=None):
         """
         Perform a regex search on the given string, using a single or a list of
         patterns returning the first matching group.
@@ -549,7 +550,7 @@ class InfoExtractor(object):
                 return next(g for g in mobj.groups() if g is not None)
             else:
                 return mobj.group(group)
-        elif default is not _NO_DEFAULT:
+        elif default is not NO_DEFAULT:
             return default
         elif fatal:
             raise RegexNotFoundError('Unable to extract %s' % _name)
@@ -557,7 +558,7 @@ class InfoExtractor(object):
             self._downloader.report_warning('unable to extract %s' % _name + bug_reports_message())
             return None
 
-    def _html_search_regex(self, pattern, string, name, default=_NO_DEFAULT, fatal=True, flags=0, group=None):
+    def _html_search_regex(self, pattern, string, name, default=NO_DEFAULT, fatal=True, flags=0, group=None):
         """
         Like _search_regex, but strips HTML tags and unescapes entities.
         """

From d16154d16327907279eff48a4018c495726d401a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 28 Jun 2015 23:05:09 +0600
Subject: [PATCH 1737/6219] [tnaflix] Generalize tnaflix extractors

---
 youtube_dl/extractor/__init__.py |   8 +-
 youtube_dl/extractor/empflix.py  |  31 ----
 youtube_dl/extractor/moviefap.py | 135 ---------------
 youtube_dl/extractor/tnaflix.py  | 279 +++++++++++++++++++++++++------
 4 files changed, 234 insertions(+), 219 deletions(-)
 delete mode 100644 youtube_dl/extractor/empflix.py
 delete mode 100644 youtube_dl/extractor/moviefap.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index d41d277c9..d44339200 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -144,7 +144,6 @@ from .ellentv import (
 )
 from .elpais import ElPaisIE
 from .embedly import EmbedlyIE
-from .empflix import EMPFlixIE
 from .engadget import EngadgetIE
 from .eporner import EpornerIE
 from .eroprofile import EroProfileIE
@@ -311,7 +310,6 @@ from .morningstar import MorningstarIE
 from .motherless import MotherlessIE
 from .motorsport import MotorsportIE
 from .movieclips import MovieClipsIE
-from .moviefap import MovieFapIE
 from .moviezine import MoviezineIE
 from .movshare import MovShareIE
 from .mtv import (
@@ -578,7 +576,11 @@ from .tmz import (
     TMZIE,
     TMZArticleIE,
 )
-from .tnaflix import TNAFlixIE
+from .tnaflix import (
+    TNAFlixIE,
+    EMPFlixIE,
+    MovieFapIE,
+)
 from .thvideo import (
     THVideoIE,
     THVideoPlaylistIE
diff --git a/youtube_dl/extractor/empflix.py b/youtube_dl/extractor/empflix.py
deleted file mode 100644
index 4827022e0..000000000
--- a/youtube_dl/extractor/empflix.py
+++ /dev/null
@@ -1,31 +0,0 @@
-from __future__ import unicode_literals
-
-from .tnaflix import TNAFlixIE
-
-
-class EMPFlixIE(TNAFlixIE):
-    _VALID_URL = r'https?://(?:www\.)?empflix\.com/videos/(?P<display_id>.+?)-(?P<id>[0-9]+)\.html'
-
-    _TITLE_REGEX = r'name="title" value="(?P<title>[^"]*)"'
-    _DESCRIPTION_REGEX = r'name="description" value="([^"]*)"'
-    _CONFIG_REGEX = r'flashvars\.config\s*=\s*escape\("([^"]+)"'
-
-    _TESTS = [
-        {
-            'url': 'http://www.empflix.com/videos/Amateur-Finger-Fuck-33051.html',
-            'md5': 'b1bc15b6412d33902d6e5952035fcabc',
-            'info_dict': {
-                'id': '33051',
-                'display_id': 'Amateur-Finger-Fuck',
-                'ext': 'mp4',
-                'title': 'Amateur Finger Fuck',
-                'description': 'Amateur solo finger fucking.',
-                'thumbnail': 're:https?://.*\.jpg$',
-                'age_limit': 18,
-            }
-        },
-        {
-            'url': 'http://www.empflix.com/videos/[AROMA][ARMD-718]-Aoi-Yoshino-Sawa-25826.html',
-            'only_matching': True,
-        }
-    ]
diff --git a/youtube_dl/extractor/moviefap.py b/youtube_dl/extractor/moviefap.py
deleted file mode 100644
index 82b863539..000000000
--- a/youtube_dl/extractor/moviefap.py
+++ /dev/null
@@ -1,135 +0,0 @@
-from __future__ import unicode_literals
-
-import re
-
-from .common import InfoExtractor
-from ..utils import (
-    xpath_text,
-    str_to_int
-)
-from ..compat import compat_str
-
-
-class MovieFapIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?moviefap\.com/videos/(?P<id>[0-9a-f]+)/(?P<name>[a-z-_]+)'
-    _TESTS = [{
-        # normal, multi-format video
-        'url': 'http://www.moviefap.com/videos/be9867c9416c19f54a4a/experienced-milf-amazing-handjob.html',
-        'md5': '26624b4e2523051b550067d547615906',
-        'info_dict': {
-            'id': 'be9867c9416c19f54a4a',
-            'ext': 'mp4',
-            'title': 'Experienced MILF Amazing Handjob',
-            'description': 'Experienced MILF giving an Amazing Handjob',
-            'thumbnail': 'http://img.moviefap.com/a16:9w990r/thumbs/be/322032-20l.jpg',
-            'uploader_id': 'darvinfred06',
-            'display_id': 'experienced-milf-amazing-handjob',
-            'categories': ['Amateur', 'Masturbation', 'Mature', 'Flashing']
-        }
-    }, {
-        # quirky single-format case where the extension is given as fid, but the video is really an flv
-        'url': 'http://www.moviefap.com/videos/e5da0d3edce5404418f5/jeune-couple-russe.html',
-        'md5': 'fa56683e291fc80635907168a743c9ad',
-        'info_dict': {
-            'id': 'e5da0d3edce5404418f5',
-            'ext': 'flv',
-            'title': 'Jeune Couple Russe',
-            'description': 'Amateur',
-            'thumbnail': 'http://pic.moviefap.com/thumbs/e5/949-18l.jpg',
-            'uploader_id': 'whiskeyjar',
-            'display_id': 'jeune-couple-russe',
-            'categories': ['Amateur', 'Teen']
-        }
-    }]
-
-    @staticmethod
-    def __get_thumbnail_data(xml):
-
-        """
-        Constructs a list of video thumbnails from timeline preview images.
-        :param xml: the information XML document to parse
-        """
-
-        timeline = xml.find('timeline')
-        if timeline is None:
-            # not all videos have the data - ah well
-            return []
-
-        # get the required information from the XML
-        width = str_to_int(timeline.find('imageWidth').text)
-        height = str_to_int(timeline.find('imageHeight').text)
-        first = str_to_int(timeline.find('imageFirst').text)
-        last = str_to_int(timeline.find('imageLast').text)
-        pattern = timeline.find('imagePattern').text
-
-        # generate the list of thumbnail information dicts
-        thumbnails = []
-        for i in range(first, last + 1):
-            thumbnails.append({
-                'url': pattern.replace('#', compat_str(i)),
-                'width': width,
-                'height': height
-            })
-        return thumbnails
-
-    def _real_extract(self, url):
-
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-
-        # find and retrieve the XML document detailing video download URLs
-        info_url = self._html_search_regex(
-            r'flashvars\.config = escape\("(.+?)"', webpage, 'player parameters')
-        xml = self._download_xml(info_url, video_id)
-
-        # find the video container
-        if xml.find('videoConfig') is not None:
-            ext = xml.find('videoConfig').find('type').text
-        else:
-            ext = 'flv'  # guess...
-
-        # work out the video URL(s)
-        formats = []
-        if xml.find('videoLink') is not None:
-            # single format available
-            formats.append({
-                'url': xpath_text(xml, 'videoLink', 'url', True),
-                'ext': ext
-            })
-        else:
-            # multiple formats available
-            for item in xml.find('quality').findall('item'):
-                resolution = xpath_text(item, 'res', 'resolution', True)  # 480p etc.
-                formats.append({
-                    'url': xpath_text(item, 'videoLink', 'url', True),
-                    'ext': ext,
-                    'resolution': resolution,
-                    'height': int(re.findall(r'\d+', resolution)[0])
-                })
-
-            self._sort_formats(formats)
-
-        return {
-            'id': video_id,
-            'formats': formats,
-            'title': self._html_search_regex(
-                r'<div id="view_title"><h1>(.*?)</h1>', webpage, 'title'),
-            'display_id': re.compile(self._VALID_URL).match(url).group('name'),
-            'thumbnails': self.__get_thumbnail_data(xml),
-            'thumbnail': xpath_text(xml, 'startThumb', 'thumbnail'),
-            'description': self._html_search_regex(
-                r'name="description" value="(.*?)"', webpage, 'description', fatal=False),
-            'uploader_id': self._html_search_regex(
-                r'name="username" value="(.*?)"', webpage, 'uploader_id', fatal=False),
-            'view_count': str_to_int(self._html_search_regex(
-                r'<br>Views <strong>([0-9]+)</strong>', webpage, 'view_count, fatal=False')),
-            'average_rating': float(self._html_search_regex(
-                r'Current Rating<br> <strong>(.*?)</strong>', webpage, 'average_rating', fatal=False)),
-            'comment_count': str_to_int(self._html_search_regex(
-                r'<span id="comCount">([0-9]+)</span>', webpage, 'comment_count', fatal=False)),
-            'age_limit': 18,
-            'webpage_url': self._html_search_regex(
-                r'name="link" value="(.*?)"', webpage, 'webpage_url', fatal=False),
-            'categories': self._html_search_regex(
-                r'</div>\s*(.*?)\s*<br>', webpage, 'categories', fatal=False).split(', ')
-        }
diff --git a/youtube_dl/extractor/tnaflix.py b/youtube_dl/extractor/tnaflix.py
index c282865b2..49516abca 100644
--- a/youtube_dl/extractor/tnaflix.py
+++ b/youtube_dl/extractor/tnaflix.py
@@ -3,39 +3,70 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import compat_str
 from ..utils import (
-    parse_duration,
     fix_xml_ampersands,
+    float_or_none,
+    int_or_none,
+    parse_duration,
+    str_to_int,
+    xpath_text,
 )
 
 
-class TNAFlixIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?tnaflix\.com/[^/]+/(?P<display_id>[^/]+)/video(?P<id>\d+)'
-
-    _TITLE_REGEX = r'<title>(.+?) - TNAFlix Porn Videos</title>'
-    _DESCRIPTION_REGEX = r'<h3 itemprop="description">([^<]+)</h3>'
-    _CONFIG_REGEX = r'flashvars\.config\s*=\s*escape\("([^"]+)"'
-
-    _TESTS = [
-        {
-            'url': 'http://www.tnaflix.com/porn-stars/Carmella-Decesare-striptease/video553878',
-            'md5': 'ecf3498417d09216374fc5907f9c6ec0',
-            'info_dict': {
-                'id': '553878',
-                'display_id': 'Carmella-Decesare-striptease',
-                'ext': 'mp4',
-                'title': 'Carmella Decesare - striptease',
-                'description': '',
-                'thumbnail': 're:https?://.*\.jpg$',
-                'duration': 91,
-                'age_limit': 18,
-            }
-        },
-        {
-            'url': 'https://www.tnaflix.com/amateur-porn/bunzHD-Ms.Donk/video358632',
-            'only_matching': True,
-        }
+class TNAFlixNetworkBaseIE(InfoExtractor):
+    # May be overridden in descendants if necessary
+    _CONFIG_REGEX = [
+        r'flashvars\.config\s*=\s*escape\("([^"]+)"',
+        r'<input[^>]+name="config\d?" value="([^"]+)"',
     ]
+    _TITLE_REGEX = r'<input[^>]+name="title" value="([^"]+)"'
+    _DESCRIPTION_REGEX = r'<input[^>]+name="description" value="([^"]+)"'
+    _UPLOADER_REGEX = r'<input[^>]+name="username" value="([^"]+)"'
+    _VIEW_COUNT_REGEX = None
+    _COMMENT_COUNT_REGEX = None
+    _AVERAGE_RATING_REGEX = None
+    _CATEGORIES_REGEX = r'<li[^>]*>\s*<span[^>]+class="infoTitle"[^>]*>Categories:</span>\s*<span[^>]+class="listView"[^>]*>(.+?)</span>\s*</li>'
+
+    def _extract_thumbnails(self, flix_xml):
+
+        def get_child(elem, names):
+            for name in names:
+                child = elem.find(name)
+                if child is not None:
+                    return child
+
+        timeline = get_child(flix_xml, ['timeline', 'rolloverBarImage'])
+        if timeline is None:
+            return
+
+        pattern_el = get_child(timeline, ['imagePattern', 'pattern'])
+        if pattern_el is None or not pattern_el.text:
+            return
+
+        first_el = get_child(timeline, ['imageFirst', 'first'])
+        last_el = get_child(timeline, ['imageLast', 'last'])
+        if first_el is None or last_el is None:
+            return
+
+        first_text = first_el.text
+        last_text = last_el.text
+        if not first_text.isdigit() or not last_text.isdigit():
+            return
+
+        first = int(first_text)
+        last = int(last_text)
+        if first > last:
+            return
+
+        width = int_or_none(xpath_text(timeline, './imageWidth', 'thumbnail width'))
+        height = int_or_none(xpath_text(timeline, './imageHeight', 'thumbnail height'))
+
+        return [{
+            'url': self._proto_relative_url(pattern_el.text.replace('#', compat_str(i)), 'http:'),
+            'width': width,
+            'height': height,
+        } for i in range(first, last + 1)]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
@@ -44,39 +75,64 @@ class TNAFlixIE(InfoExtractor):
 
         webpage = self._download_webpage(url, display_id)
 
+        cfg_url = self._proto_relative_url(self._html_search_regex(
+            self._CONFIG_REGEX, webpage, 'flashvars.config'), 'http:')
+
+        cfg_xml = self._download_xml(
+            cfg_url, display_id, 'Downloading metadata',
+            transform_source=fix_xml_ampersands)
+
+        formats = []
+
+        def extract_video_url(vl):
+            return re.sub('speed=\d+', 'speed=', vl.text)
+
+        video_link = cfg_xml.find('./videoLink')
+        if video_link is not None:
+            formats.append({
+                'url': extract_video_url(video_link),
+                'ext': xpath_text(cfg_xml, './videoConfig/type', 'type', default='flv'),
+            })
+
+        for item in cfg_xml.findall('./quality/item'):
+            video_link = item.find('./videoLink')
+            if video_link is None:
+                continue
+            res = item.find('res')
+            format_id = None if res is None else res.text
+            height = int_or_none(self._search_regex(
+                r'^(\d+)[pP]', format_id, 'height', default=None))
+            formats.append({
+                'url': self._proto_relative_url(extract_video_url(video_link), 'http:'),
+                'format_id': format_id,
+                'height': height,
+            })
+
+        self._sort_formats(formats)
+
+        thumbnail = self._proto_relative_url(
+            xpath_text(cfg_xml, './startThumb', 'thumbnail'), 'http:')
+        thumbnails = self._extract_thumbnails(cfg_xml)
+
         title = self._html_search_regex(
             self._TITLE_REGEX, webpage, 'title') if self._TITLE_REGEX else self._og_search_title(webpage)
-        description = self._html_search_regex(
-            self._DESCRIPTION_REGEX, webpage, 'description', fatal=False, default='')
 
         age_limit = self._rta_search(webpage)
 
         duration = parse_duration(self._html_search_meta(
             'duration', webpage, 'duration', default=None))
 
-        cfg_url = self._proto_relative_url(self._html_search_regex(
-            self._CONFIG_REGEX, webpage, 'flashvars.config'), 'http:')
+        def extract_field(pattern, name):
+            return self._html_search_regex(pattern, webpage, name, default=None) if pattern else None
 
-        cfg_xml = self._download_xml(
-            cfg_url, display_id, note='Downloading metadata',
-            transform_source=fix_xml_ampersands)
+        description = extract_field(self._DESCRIPTION_REGEX, 'description')
+        uploader = extract_field(self._UPLOADER_REGEX, 'uploader')
+        view_count = str_to_int(extract_field(self._VIEW_COUNT_REGEX, 'view count'))
+        comment_count = str_to_int(extract_field(self._COMMENT_COUNT_REGEX, 'comment count'))
+        average_rating = float_or_none(extract_field(self._AVERAGE_RATING_REGEX, 'average rating'))
 
-        thumbnail = self._proto_relative_url(
-            cfg_xml.find('./startThumb').text, 'http:')
-
-        formats = []
-        for item in cfg_xml.findall('./quality/item'):
-            video_url = re.sub('speed=\d+', 'speed=', item.find('videoLink').text)
-            format_id = item.find('res').text
-            fmt = {
-                'url': self._proto_relative_url(video_url, 'http:'),
-                'format_id': format_id,
-            }
-            m = re.search(r'^(\d+)', format_id)
-            if m:
-                fmt['height'] = int(m.group(1))
-            formats.append(fmt)
-        self._sort_formats(formats)
+        categories_str = extract_field(self._CATEGORIES_REGEX, 'categories')
+        categories = categories_str.split(', ') if categories_str is not None else []
 
         return {
             'id': video_id,
@@ -84,7 +140,130 @@ class TNAFlixIE(InfoExtractor):
             'title': title,
             'description': description,
             'thumbnail': thumbnail,
+            'thumbnails': thumbnails,
             'duration': duration,
             'age_limit': age_limit,
+            'uploader': uploader,
+            'view_count': view_count,
+            'comment_count': comment_count,
+            'average_rating': average_rating,
+            'categories': categories,
             'formats': formats,
         }
+
+
+class TNAFlixIE(TNAFlixNetworkBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?tnaflix\.com/[^/]+/(?P<display_id>[^/]+)/video(?P<id>\d+)'
+
+    _TITLE_REGEX = r'<title>(.+?) - TNAFlix Porn Videos</title>'
+    _DESCRIPTION_REGEX = r'<h3 itemprop="description">([^<]+)</h3>'
+    _UPLOADER_REGEX = r'(?s)<span[^>]+class="infoTitle"[^>]*>Uploaded By:</span>(.+?)<div'
+
+    _TESTS = [{
+        # anonymous uploader, no categories
+        'url': 'http://www.tnaflix.com/porn-stars/Carmella-Decesare-striptease/video553878',
+        'md5': 'ecf3498417d09216374fc5907f9c6ec0',
+        'info_dict': {
+            'id': '553878',
+            'display_id': 'Carmella-Decesare-striptease',
+            'ext': 'mp4',
+            'title': 'Carmella Decesare - striptease',
+            'thumbnail': 're:https?://.*\.jpg$',
+            'duration': 91,
+            'age_limit': 18,
+            'uploader': 'Anonymous',
+            'categories': [],
+        }
+    }, {
+        # non-anonymous uploader, categories
+        'url': 'https://www.tnaflix.com/teen-porn/Educational-xxx-video/video6538',
+        'md5': '0f5d4d490dbfd117b8607054248a07c0',
+        'info_dict': {
+            'id': '6538',
+            'display_id': 'Educational-xxx-video',
+            'ext': 'mp4',
+            'title': 'Educational xxx video',
+            'description': 'md5:b4fab8f88a8621c8fabd361a173fe5b8',
+            'thumbnail': 're:https?://.*\.jpg$',
+            'duration': 164,
+            'age_limit': 18,
+            'uploader': 'bobwhite39',
+            'categories': ['Amateur Porn', 'Squirting Videos', 'Teen Girls 18+'],
+        }
+    }, {
+        'url': 'https://www.tnaflix.com/amateur-porn/bunzHD-Ms.Donk/video358632',
+        'only_matching': True,
+    }]
+
+
+class EMPFlixIE(TNAFlixNetworkBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?empflix\.com/videos/(?P<display_id>.+?)-(?P<id>[0-9]+)\.html'
+
+    _UPLOADER_REGEX = r'<span[^>]+class="infoTitle"[^>]*>Uploaded By:</span>(.+?)</li>'
+
+    _TESTS = [{
+        'url': 'http://www.empflix.com/videos/Amateur-Finger-Fuck-33051.html',
+        'md5': 'b1bc15b6412d33902d6e5952035fcabc',
+        'info_dict': {
+            'id': '33051',
+            'display_id': 'Amateur-Finger-Fuck',
+            'ext': 'mp4',
+            'title': 'Amateur Finger Fuck',
+            'description': 'Amateur solo finger fucking.',
+            'thumbnail': 're:https?://.*\.jpg$',
+            'duration': 83,
+            'age_limit': 18,
+            'uploader': 'cwbike',
+            'categories': ['Amateur', 'Anal', 'Fisting', 'Home made', 'Solo'],
+        }
+    }, {
+        'url': 'http://www.empflix.com/videos/[AROMA][ARMD-718]-Aoi-Yoshino-Sawa-25826.html',
+        'only_matching': True,
+    }]
+
+
+class MovieFapIE(TNAFlixNetworkBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?moviefap\.com/videos/(?P<id>[0-9a-f]+)/(?P<display_id>[^/]+)\.html'
+
+    _VIEW_COUNT_REGEX = r'<br>Views\s*<strong>([\d,.]+)</strong>'
+    _COMMENT_COUNT_REGEX = r'<span[^>]+id="comCount"[^>]*>([\d,.]+)</span>'
+    _AVERAGE_RATING_REGEX = r'Current Rating\s*<br>\s*<strong>([\d.]+)</strong>'
+    _CATEGORIES_REGEX = r'(?s)<div[^>]+id="vid_info"[^>]*>\s*<div[^>]*>.+?</div>(.*?)<br>'
+
+    _TESTS = [{
+        # normal, multi-format video
+        'url': 'http://www.moviefap.com/videos/be9867c9416c19f54a4a/experienced-milf-amazing-handjob.html',
+        'md5': '26624b4e2523051b550067d547615906',
+        'info_dict': {
+            'id': 'be9867c9416c19f54a4a',
+            'display_id': 'experienced-milf-amazing-handjob',
+            'ext': 'mp4',
+            'title': 'Experienced MILF Amazing Handjob',
+            'description': 'Experienced MILF giving an Amazing Handjob',
+            'thumbnail': 're:https?://.*\.jpg$',
+            'age_limit': 18,
+            'uploader': 'darvinfred06',
+            'view_count': int,
+            'comment_count': int,
+            'average_rating': float,
+            'categories': ['Amateur', 'Masturbation', 'Mature', 'Flashing'],
+        }
+    }, {
+        # quirky single-format case where the extension is given as fid, but the video is really an flv
+        'url': 'http://www.moviefap.com/videos/e5da0d3edce5404418f5/jeune-couple-russe.html',
+        'md5': 'fa56683e291fc80635907168a743c9ad',
+        'info_dict': {
+            'id': 'e5da0d3edce5404418f5',
+            'display_id': 'jeune-couple-russe',
+            'ext': 'flv',
+            'title': 'Jeune Couple Russe',
+            'description': 'Amateur',
+            'thumbnail': 're:https?://.*\.jpg$',
+            'age_limit': 18,
+            'uploader': 'whiskeyjar',
+            'view_count': int,
+            'comment_count': int,
+            'average_rating': float,
+            'categories': ['Amateur', 'Teen'],
+        }
+    }]

From 507683780eb14d012d4430044dc402d7e08e36b4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 28 Jun 2015 23:08:05 +0600
Subject: [PATCH 1738/6219] Credit @gebn for moviefap

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 889d599a2..117b9c219 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -128,3 +128,4 @@ Ping O.
 Mister Hat
 Peter Ding
 jackyzy823
+George Brighton

From c93d53f5e307dab1b5d03cd3c621a68f40ef840d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 29 Jun 2015 00:48:06 +0600
Subject: [PATCH 1739/6219] [youtube] Fix likes/dislike extraction

---
 youtube_dl/extractor/youtube.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index d9240ff02..8b43e274b 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -29,6 +29,7 @@ from ..utils import (
     get_element_by_id,
     int_or_none,
     orderedSet,
+    str_to_int,
     unescapeHTML,
     unified_strdate,
     uppercase_escape,
@@ -1005,12 +1006,11 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 video_description = ''
 
         def _extract_count(count_name):
-            count = self._search_regex(
-                r'id="watch-%s"[^>]*>.*?([\d,]+)\s*</span>' % re.escape(count_name),
-                video_webpage, count_name, default=None)
-            if count is not None:
-                return int(count.replace(',', ''))
-            return None
+            return str_to_int(self._search_regex(
+                r'-%s-button[^>]+><span[^>]+class="yt-uix-button-content"[^>]*>([\d,]+)</span>'
+                % re.escape(count_name),
+                video_webpage, count_name, default=None))
+
         like_count = _extract_count('like')
         dislike_count = _extract_count('dislike')
 

From 541462379153c19656aa52cc5796dbf05de874ab Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 29 Jun 2015 00:49:19 +0600
Subject: [PATCH 1740/6219] [extractor/common] Remove superfluous line

---
 youtube_dl/extractor/common.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 7fa46d295..81623bfe3 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -36,7 +36,6 @@ from ..utils import (
 )
 
 
-
 class InfoExtractor(object):
     """Information Extractor class.
 

From 67134eaba1a56cec4117000acb2fc9284c9cdd9a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 28 Jun 2015 22:08:29 +0200
Subject: [PATCH 1741/6219] [YoutubeDL] rework how the format spec is processed

The spec string is processed using 'tokenize.tokenize' to split it in words and operators, the filters are still processed using regular expressions.
This should make easier to allow grouping operators with parens.
---
 test/test_YoutubeDL.py  |  27 ++--
 youtube_dl/YoutubeDL.py | 298 +++++++++++++++++++++++++---------------
 youtube_dl/compat.py    |   5 +
 3 files changed, 209 insertions(+), 121 deletions(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index a13c09ef4..8f7aef512 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -229,21 +229,30 @@ class TestFormatSelection(unittest.TestCase):
             '141', '172', '140', '171', '139',
         ]
 
-        for f1id, f2id in zip(order, order[1:]):
-            f1 = YoutubeIE._formats[f1id].copy()
-            f1['format_id'] = f1id
-            f1['url'] = 'url:' + f1id
-            f2 = YoutubeIE._formats[f2id].copy()
-            f2['format_id'] = f2id
-            f2['url'] = 'url:' + f2id
+        def format_info(f_id):
+            info = YoutubeIE._formats[f_id].copy()
+            info['format_id'] = f_id
+            info['url'] = 'url:' + f_id
+            return info
+        formats_order = [format_info(f_id) for f_id in order]
 
+        info_dict = _make_result(list(formats_order), extractor='youtube')
+        ydl = YDL({'format': 'bestvideo+bestaudio'})
+        yie = YoutubeIE(ydl)
+        yie._sort_formats(info_dict['formats'])
+        ydl.process_ie_result(info_dict)
+        downloaded = ydl.downloaded_info_dicts[0]
+        self.assertEqual(downloaded['format_id'], '137+141')
+        self.assertEqual(downloaded['ext'], 'mp4')
+
+        for f1, f2 in zip(formats_order, formats_order[1:]):
             info_dict = _make_result([f1, f2], extractor='youtube')
             ydl = YDL({'format': 'best/bestvideo'})
             yie = YoutubeIE(ydl)
             yie._sort_formats(info_dict['formats'])
             ydl.process_ie_result(info_dict)
             downloaded = ydl.downloaded_info_dicts[0]
-            self.assertEqual(downloaded['format_id'], f1id)
+            self.assertEqual(downloaded['format_id'], f1['format_id'])
 
             info_dict = _make_result([f2, f1], extractor='youtube')
             ydl = YDL({'format': 'best/bestvideo'})
@@ -251,7 +260,7 @@ class TestFormatSelection(unittest.TestCase):
             yie._sort_formats(info_dict['formats'])
             ydl.process_ie_result(info_dict)
             downloaded = ydl.downloaded_info_dicts[0]
-            self.assertEqual(downloaded['format_id'], f1id)
+            self.assertEqual(downloaded['format_id'], f1['format_id'])
 
     def test_format_filtering(self):
         formats = [
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index ef0f71bad..17a5407b9 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -21,6 +21,7 @@ import subprocess
 import socket
 import sys
 import time
+import tokenize
 import traceback
 
 if os.name == 'nt':
@@ -34,6 +35,7 @@ from .compat import (
     compat_http_client,
     compat_kwargs,
     compat_str,
+    compat_tokenize_tokenize,
     compat_urllib_error,
     compat_urllib_request,
 )
@@ -851,8 +853,8 @@ class YoutubeDL(object):
         else:
             raise Exception('Invalid result type: %s' % result_type)
 
-    def _apply_format_filter(self, format_spec, available_formats):
-        " Returns a tuple of the remaining format_spec and filtered formats "
+    def _build_format_filter(self, filter_spec):
+        " Returns a function to filter the formats according to the filter_spec "
 
         OPERATORS = {
             '<': operator.lt,
@@ -862,13 +864,13 @@ class YoutubeDL(object):
             '=': operator.eq,
             '!=': operator.ne,
         }
-        operator_rex = re.compile(r'''(?x)\s*\[
+        operator_rex = re.compile(r'''(?x)\s*
             (?P<key>width|height|tbr|abr|vbr|asr|filesize|fps)
             \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
             (?P<value>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)
-            \]$
+            $
             ''' % '|'.join(map(re.escape, OPERATORS.keys())))
-        m = operator_rex.search(format_spec)
+        m = operator_rex.search(filter_spec)
         if m:
             try:
                 comparison_value = int(m.group('value'))
@@ -879,7 +881,7 @@ class YoutubeDL(object):
                 if comparison_value is None:
                     raise ValueError(
                         'Invalid value %r in format specification %r' % (
-                            m.group('value'), format_spec))
+                            m.group('value'), filter_spec))
             op = OPERATORS[m.group('op')]
 
         if not m:
@@ -887,85 +889,201 @@ class YoutubeDL(object):
                 '=': operator.eq,
                 '!=': operator.ne,
             }
-            str_operator_rex = re.compile(r'''(?x)\s*\[
+            str_operator_rex = re.compile(r'''(?x)
                 \s*(?P<key>ext|acodec|vcodec|container|protocol)
                 \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?
                 \s*(?P<value>[a-zA-Z0-9_-]+)
-                \s*\]$
+                \s*$
                 ''' % '|'.join(map(re.escape, STR_OPERATORS.keys())))
-            m = str_operator_rex.search(format_spec)
+            m = str_operator_rex.search(filter_spec)
             if m:
                 comparison_value = m.group('value')
                 op = STR_OPERATORS[m.group('op')]
 
         if not m:
-            raise ValueError('Invalid format specification %r' % format_spec)
+            raise ValueError('Invalid filter specification %r' % filter_spec)
 
         def _filter(f):
             actual_value = f.get(m.group('key'))
             if actual_value is None:
                 return m.group('none_inclusive')
             return op(actual_value, comparison_value)
-        new_formats = [f for f in available_formats if _filter(f)]
+        return _filter
 
-        new_format_spec = format_spec[:-len(m.group(0))]
-        if not new_format_spec:
-            new_format_spec = 'best'
+    def build_format_selector(self, format_spec):
+        def syntax_error(note, start):
+            message = (
+                'Invalid format specification: '
+                '{0}\n\t{1}\n\t{2}^'.format(note, format_spec, ' ' * start[1]))
+            return SyntaxError(message)
 
-        return (new_format_spec, new_formats)
+        PICKFIRST = 'PICKFIRST'
+        MERGE = 'MERGE'
+        SINGLE = 'SINGLE'
+        FormatSelector = collections.namedtuple('FormatSelector', ['type', 'selector', 'filters'])
 
-    def select_format(self, format_spec, available_formats):
-        while format_spec.endswith(']'):
-            format_spec, available_formats = self._apply_format_filter(
-                format_spec, available_formats)
-        if not available_formats:
-            return None
+        def _parse_filter(tokens):
+            filter_parts = []
+            for type, string, start, _, _ in tokens:
+                if type == tokenize.OP and string == ']':
+                    return ''.join(filter_parts)
+                else:
+                    filter_parts.append(string)
 
-        if format_spec in ['best', 'worst', None]:
-            format_idx = 0 if format_spec == 'worst' else -1
-            audiovideo_formats = [
-                f for f in available_formats
-                if f.get('vcodec') != 'none' and f.get('acodec') != 'none']
-            if audiovideo_formats:
-                return audiovideo_formats[format_idx]
-            # for audio only (soundcloud) or video only (imgur) urls, select the best/worst audio format
-            elif (all(f.get('acodec') != 'none' for f in available_formats) or
-                  all(f.get('vcodec') != 'none' for f in available_formats)):
-                return available_formats[format_idx]
-        elif format_spec == 'bestaudio':
-            audio_formats = [
-                f for f in available_formats
-                if f.get('vcodec') == 'none']
-            if audio_formats:
-                return audio_formats[-1]
-        elif format_spec == 'worstaudio':
-            audio_formats = [
-                f for f in available_formats
-                if f.get('vcodec') == 'none']
-            if audio_formats:
-                return audio_formats[0]
-        elif format_spec == 'bestvideo':
-            video_formats = [
-                f for f in available_formats
-                if f.get('acodec') == 'none']
-            if video_formats:
-                return video_formats[-1]
-        elif format_spec == 'worstvideo':
-            video_formats = [
-                f for f in available_formats
-                if f.get('acodec') == 'none']
-            if video_formats:
-                return video_formats[0]
-        else:
-            extensions = ['mp4', 'flv', 'webm', '3gp', 'm4a', 'mp3', 'ogg', 'aac', 'wav']
-            if format_spec in extensions:
-                filter_f = lambda f: f['ext'] == format_spec
-            else:
-                filter_f = lambda f: f['format_id'] == format_spec
-            matches = list(filter(filter_f, available_formats))
-            if matches:
-                return matches[-1]
-        return None
+        def _parse_format_selection(tokens, endwith=[]):
+            selectors = []
+            current_selector = None
+            for type, string, start, _, _ in tokens:
+                # ENCODING is only defined in python 3.x
+                if type == getattr(tokenize, 'ENCODING', None):
+                    continue
+                elif type in [tokenize.NAME, tokenize.NUMBER]:
+                    current_selector = FormatSelector(SINGLE, string, [])
+                elif type == tokenize.OP:
+                    if string in endwith:
+                        break
+                    if string == ',':
+                        selectors.append(current_selector)
+                        current_selector = None
+                    elif string == '/':
+                        first_choice = current_selector
+                        second_choice = _parse_format_selection(tokens, [','])
+                        current_selector = None
+                        selectors.append(FormatSelector(PICKFIRST, (first_choice, second_choice), []))
+                    elif string == '[':
+                        if not current_selector:
+                            current_selector = FormatSelector(SINGLE, 'best', [])
+                        format_filter = _parse_filter(tokens)
+                        current_selector.filters.append(format_filter)
+                    elif string == '+':
+                        video_selector = current_selector
+                        audio_selector = _parse_format_selection(tokens, [','])
+                        current_selector = None
+                        selectors.append(FormatSelector(MERGE, (video_selector, audio_selector), []))
+                    else:
+                        raise syntax_error('Operator not recognized: "{0}"'.format(string), start)
+                elif type == tokenize.ENDMARKER:
+                    break
+            if current_selector:
+                selectors.append(current_selector)
+            return selectors
+
+        def _build_selector_function(selector):
+            if isinstance(selector, list):
+                fs = [_build_selector_function(s) for s in selector]
+
+                def selector_function(formats):
+                    for f in fs:
+                        for format in f(formats):
+                            yield format
+                return selector_function
+            elif selector.type == PICKFIRST:
+                fs = [_build_selector_function(s) for s in selector.selector]
+
+                def selector_function(formats):
+                    for f in fs:
+                        picked_formats = list(f(formats))
+                        if picked_formats:
+                            return picked_formats
+                    return []
+            elif selector.type == SINGLE:
+                format_spec = selector.selector
+
+                def selector_function(formats):
+                    if format_spec in ['best', 'worst', None]:
+                        format_idx = 0 if format_spec == 'worst' else -1
+                        audiovideo_formats = [
+                            f for f in formats
+                            if f.get('vcodec') != 'none' and f.get('acodec') != 'none']
+                        if audiovideo_formats:
+                            yield audiovideo_formats[format_idx]
+                        # for audio only (soundcloud) or video only (imgur) urls, select the best/worst audio format
+                        elif (all(f.get('acodec') != 'none' for f in formats) or
+                              all(f.get('vcodec') != 'none' for f in formats)):
+                            yield formats[format_idx]
+                    elif format_spec == 'bestaudio':
+                        audio_formats = [
+                            f for f in formats
+                            if f.get('vcodec') == 'none']
+                        if audio_formats:
+                            yield audio_formats[-1]
+                    elif format_spec == 'worstaudio':
+                        audio_formats = [
+                            f for f in formats
+                            if f.get('vcodec') == 'none']
+                        if audio_formats:
+                            yield audio_formats[0]
+                    elif format_spec == 'bestvideo':
+                        video_formats = [
+                            f for f in formats
+                            if f.get('acodec') == 'none']
+                        if video_formats:
+                            yield video_formats[-1]
+                    elif format_spec == 'worstvideo':
+                        video_formats = [
+                            f for f in formats
+                            if f.get('acodec') == 'none']
+                        if video_formats:
+                            yield video_formats[0]
+                    else:
+                        extensions = ['mp4', 'flv', 'webm', '3gp', 'm4a', 'mp3', 'ogg', 'aac', 'wav']
+                        if format_spec in extensions:
+                            filter_f = lambda f: f['ext'] == format_spec
+                        else:
+                            filter_f = lambda f: f['format_id'] == format_spec
+                        matches = list(filter(filter_f, formats))
+                        if matches:
+                            yield matches[-1]
+            elif selector.type == MERGE:
+                def _merge(formats_info):
+                    format_1, format_2 = [f['format_id'] for f in formats_info]
+                    # The first format must contain the video and the
+                    # second the audio
+                    if formats_info[0].get('vcodec') == 'none':
+                        self.report_error('The first format must '
+                                          'contain the video, try using '
+                                          '"-f %s+%s"' % (format_2, format_1))
+                        return
+                    output_ext = (
+                        formats_info[0]['ext']
+                        if self.params.get('merge_output_format') is None
+                        else self.params['merge_output_format'])
+                    return {
+                        'requested_formats': formats_info,
+                        'format': '%s+%s' % (formats_info[0].get('format'),
+                                             formats_info[1].get('format')),
+                        'format_id': '%s+%s' % (formats_info[0].get('format_id'),
+                                                formats_info[1].get('format_id')),
+                        'width': formats_info[0].get('width'),
+                        'height': formats_info[0].get('height'),
+                        'resolution': formats_info[0].get('resolution'),
+                        'fps': formats_info[0].get('fps'),
+                        'vcodec': formats_info[0].get('vcodec'),
+                        'vbr': formats_info[0].get('vbr'),
+                        'stretched_ratio': formats_info[0].get('stretched_ratio'),
+                        'acodec': formats_info[1].get('acodec'),
+                        'abr': formats_info[1].get('abr'),
+                        'ext': output_ext,
+                    }
+                video_selector, audio_selector = map(_build_selector_function, selector.selector)
+
+                def selector_function(formats):
+                    formats = list(formats)
+                    for pair in itertools.product(video_selector(formats), audio_selector(formats)):
+                        yield _merge(pair)
+
+            filters = [self._build_format_filter(f) for f in selector.filters]
+
+            def final_selector(formats):
+                for _filter in filters:
+                    formats = list(filter(_filter, formats))
+                return selector_function(formats)
+            return final_selector
+
+        stream = io.BytesIO(format_spec.encode('utf-8'))
+        tokens = compat_tokenize_tokenize(stream.readline)
+        parsed_selector = _parse_format_selection(tokens)
+        return _build_selector_function(parsed_selector)
 
     def _calc_headers(self, info_dict):
         res = std_headers.copy()
@@ -1112,52 +1230,8 @@ class YoutubeDL(object):
         if req_format == 'all':
             formats_to_download = formats
         else:
-            for rfstr in req_format.split(','):
-                # We can accept formats requested in the format: 34/5/best, we pick
-                # the first that is available, starting from left
-                req_formats = rfstr.split('/')
-                for rf in req_formats:
-                    if re.match(r'.+?\+.+?', rf) is not None:
-                        # Two formats have been requested like '137+139'
-                        format_1, format_2 = rf.split('+')
-                        formats_info = (self.select_format(format_1, formats),
-                                        self.select_format(format_2, formats))
-                        if all(formats_info):
-                            # The first format must contain the video and the
-                            # second the audio
-                            if formats_info[0].get('vcodec') == 'none':
-                                self.report_error('The first format must '
-                                                  'contain the video, try using '
-                                                  '"-f %s+%s"' % (format_2, format_1))
-                                return
-                            output_ext = (
-                                formats_info[0]['ext']
-                                if self.params.get('merge_output_format') is None
-                                else self.params['merge_output_format'])
-                            selected_format = {
-                                'requested_formats': formats_info,
-                                'format': '%s+%s' % (formats_info[0].get('format'),
-                                                     formats_info[1].get('format')),
-                                'format_id': '%s+%s' % (formats_info[0].get('format_id'),
-                                                        formats_info[1].get('format_id')),
-                                'width': formats_info[0].get('width'),
-                                'height': formats_info[0].get('height'),
-                                'resolution': formats_info[0].get('resolution'),
-                                'fps': formats_info[0].get('fps'),
-                                'vcodec': formats_info[0].get('vcodec'),
-                                'vbr': formats_info[0].get('vbr'),
-                                'stretched_ratio': formats_info[0].get('stretched_ratio'),
-                                'acodec': formats_info[1].get('acodec'),
-                                'abr': formats_info[1].get('abr'),
-                                'ext': output_ext,
-                            }
-                        else:
-                            selected_format = None
-                    else:
-                        selected_format = self.select_format(rf, formats)
-                    if selected_format is not None:
-                        formats_to_download.append(selected_format)
-                        break
+            format_selector = self.build_format_selector(req_format)
+            formats_to_download = list(format_selector(formats))
         if not formats_to_download:
             raise ExtractorError('requested format not available',
                                  expected=True)
diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index f9529210d..bc218dd71 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -388,6 +388,10 @@ else:
             pass
         return _terminal_size(columns, lines)
 
+if sys.version_info >= (3, 0):
+    from tokenize import tokenize as compat_tokenize_tokenize
+else:
+    from tokenize import generate_tokens as compat_tokenize_tokenize
 
 __all__ = [
     'compat_HTTPError',
@@ -408,6 +412,7 @@ __all__ = [
     'compat_socket_create_connection',
     'compat_str',
     'compat_subprocess_get_DEVNULL',
+    'compat_tokenize_tokenize',
     'compat_urllib_error',
     'compat_urllib_parse',
     'compat_urllib_parse_unquote',

From 5acfa126c812c3ab7088af6c7df79697baee7831 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 28 Jun 2015 22:48:02 +0200
Subject: [PATCH 1742/6219] [YoutubeDL] format spec: treat 'all' like a normal
 specifier

So you can use filters with it, for example 'all[width>=400][width<=600]'.
---
 test/test_YoutubeDL.py  |  5 +++++
 youtube_dl/YoutubeDL.py | 13 ++++++-------
 2 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 8f7aef512..709e3100f 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -317,6 +317,11 @@ class TestFormatSelection(unittest.TestCase):
         downloaded = ydl.downloaded_info_dicts[0]
         self.assertEqual(downloaded['format_id'], 'G')
 
+        ydl = YDL({'format': 'all[width>=400][width<=600]'})
+        ydl.process_ie_result(info_dict)
+        downloaded_ids = [info['format_id'] for info in ydl.downloaded_info_dicts]
+        self.assertEqual(downloaded_ids, ['B', 'C', 'D'])
+
 
 class TestYoutubeDL(unittest.TestCase):
     def test_subtitles(self):
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 17a5407b9..258e612af 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -990,7 +990,10 @@ class YoutubeDL(object):
                 format_spec = selector.selector
 
                 def selector_function(formats):
-                    if format_spec in ['best', 'worst', None]:
+                    if format_spec == 'all':
+                        for f in formats:
+                            yield f
+                    elif format_spec in ['best', 'worst', None]:
                         format_idx = 0 if format_spec == 'worst' else -1
                         audiovideo_formats = [
                             f for f in formats
@@ -1226,12 +1229,8 @@ class YoutubeDL(object):
                     req_format_list.append('bestvideo+bestaudio')
             req_format_list.append('best')
             req_format = '/'.join(req_format_list)
-        formats_to_download = []
-        if req_format == 'all':
-            formats_to_download = formats
-        else:
-            format_selector = self.build_format_selector(req_format)
-            formats_to_download = list(format_selector(formats))
+        format_selector = self.build_format_selector(req_format)
+        formats_to_download = list(format_selector(formats))
         if not formats_to_download:
             raise ExtractorError('requested format not available',
                                  expected=True)

From c4bd188da46a837ddf8f8f8d4766eb799fa2b484 Mon Sep 17 00:00:00 2001
From: Anders Einar Hilden <hildenae@gmail.com>
Date: Mon, 29 Jun 2015 00:11:31 +0200
Subject: [PATCH 1743/6219] NRK now supports / requires HTTPS

Add s? to regexp to support new urls. Update testcases to use HTTPS.
---
 youtube_dl/extractor/nrk.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index cc70c2950..9e4581cf9 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -13,7 +13,7 @@ from ..utils import (
 
 
 class NRKIE(InfoExtractor):
-    _VALID_URL = r'(?:nrk:|http://(?:www\.)?nrk\.no/video/PS\*)(?P<id>\d+)'
+    _VALID_URL = r'(?:nrk:|https?://(?:www\.)?nrk\.no/video/PS\*)(?P<id>\d+)'
 
     _TESTS = [
         {
@@ -76,7 +76,7 @@ class NRKIE(InfoExtractor):
 
 
 class NRKPlaylistIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?nrk\.no/(?!video)(?:[^/]+/)+(?P<id>[^/]+)'
+    _VALID_URL = r'https?://(?:www\.)?nrk\.no/(?!video)(?:[^/]+/)+(?P<id>[^/]+)'
 
     _TESTS = [{
         'url': 'http://www.nrk.no/troms/gjenopplev-den-historiske-solformorkelsen-1.12270763',
@@ -116,11 +116,11 @@ class NRKPlaylistIE(InfoExtractor):
 
 
 class NRKTVIE(InfoExtractor):
-    _VALID_URL = r'(?P<baseurl>http://tv\.nrk(?:super)?\.no/)(?:serie/[^/]+|program)/(?P<id>[a-zA-Z]{4}\d{8})(?:/\d{2}-\d{2}-\d{4})?(?:#del=(?P<part_id>\d+))?'
+    _VALID_URL = r'(?P<baseurl>https?://tv\.nrk(?:super)?\.no/)(?:serie/[^/]+|program)/(?P<id>[a-zA-Z]{4}\d{8})(?:/\d{2}-\d{2}-\d{4})?(?:#del=(?P<part_id>\d+))?'
 
     _TESTS = [
         {
-            'url': 'http://tv.nrk.no/serie/20-spoersmaal-tv/MUHH48000314/23-05-2014',
+            'url': 'https://tv.nrk.no/serie/20-spoersmaal-tv/MUHH48000314/23-05-2014',
             'md5': 'adf2c5454fa2bf032f47a9f8fb351342',
             'info_dict': {
                 'id': 'MUHH48000314',
@@ -132,7 +132,7 @@ class NRKTVIE(InfoExtractor):
             },
         },
         {
-            'url': 'http://tv.nrk.no/program/mdfp15000514',
+            'url': 'https://tv.nrk.no/program/mdfp15000514',
             'md5': '383650ece2b25ecec996ad7b5bb2a384',
             'info_dict': {
                 'id': 'mdfp15000514',
@@ -145,7 +145,7 @@ class NRKTVIE(InfoExtractor):
         },
         {
             # single playlist video
-            'url': 'http://tv.nrk.no/serie/tour-de-ski/MSPO40010515/06-01-2015#del=2',
+            'url': 'https://tv.nrk.no/serie/tour-de-ski/MSPO40010515/06-01-2015#del=2',
             'md5': 'adbd1dbd813edaf532b0a253780719c2',
             'info_dict': {
                 'id': 'MSPO40010515-part2',
@@ -157,7 +157,7 @@ class NRKTVIE(InfoExtractor):
             'skip': 'Only works from Norway',
         },
         {
-            'url': 'http://tv.nrk.no/serie/tour-de-ski/MSPO40010515/06-01-2015',
+            'url': 'https://tv.nrk.no/serie/tour-de-ski/MSPO40010515/06-01-2015',
             'playlist': [
                 {
                     'md5': '9480285eff92d64f06e02a5367970a7a',

From bea41c7f3fa4f9072ad2f5354938ab1c8cef0a6d Mon Sep 17 00:00:00 2001
From: corone17 <reino@degeelebosch.nl>
Date: Mon, 29 Jun 2015 00:59:18 +0200
Subject: [PATCH 1744/6219] Update rtlnl.py

Better to extract 'http://manifest.us.rtl.nl' from the json, I'd say. And I think it's better to use the default json-url to make it more futureproof.
Succesfully tested with tarball.
---
 youtube_dl/extractor/rtlnl.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/rtlnl.py b/youtube_dl/extractor/rtlnl.py
index 41d202c28..e708e0093 100644
--- a/youtube_dl/extractor/rtlnl.py
+++ b/youtube_dl/extractor/rtlnl.py
@@ -51,7 +51,7 @@ class RtlNlIE(InfoExtractor):
     def _real_extract(self, url):
         uuid = self._match_id(url)
         info = self._download_json(
-            'http://www.rtl.nl/system/s4m/vfd/version=2/uuid=%s/fmt=flash/' % uuid,
+            'http://www.rtl.nl/system/s4m/vfd/version=2/uuid=%s/fmt=adaptive/' % uuid,
             uuid)
 
         material = info['material'][0]
@@ -60,8 +60,8 @@ class RtlNlIE(InfoExtractor):
         description = material.get('synopsis') or info['episodes'][0]['synopsis']
 
         # Use unencrypted m3u8 streams (See https://github.com/rg3/youtube-dl/issues/4118)
-        videopath = material['videopath'].replace('.f4m', '.m3u8')
-        m3u8_url = 'http://manifest.us.rtl.nl' + videopath
+        videopath = material['videopath'].replace('adaptive', 'flash')
+        m3u8_url = info['meta']['videohost'] + videopath
 
         formats = self._extract_m3u8_formats(m3u8_url, uuid, ext='mp4')
 

From 738b92632296a9fee3eb7e0c915f6ea6b395125f Mon Sep 17 00:00:00 2001
From: nawl <dylanlwrigh@gmail.com>
Date: Sun, 28 Jun 2015 17:24:00 -0600
Subject: [PATCH 1745/6219] [hentaistigma] Fix video extractor

---
 youtube_dl/extractor/hentaistigma.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/hentaistigma.py b/youtube_dl/extractor/hentaistigma.py
index 63d87b74c..225af8cb3 100644
--- a/youtube_dl/extractor/hentaistigma.py
+++ b/youtube_dl/extractor/hentaistigma.py
@@ -32,7 +32,7 @@ class HentaiStigmaIE(InfoExtractor):
         wrap_webpage = self._download_webpage(wrap_url, video_id)
 
         video_url = self._html_search_regex(
-            r'clip:\s*{\s*url: "([^"]*)"', wrap_webpage, 'video url')
+            r'file:"([^"]+)"', wrap_webpage, 'video url')
 
         return {
             'id': video_id,

From 0130afb76e5cb6f470f39f127c8d09eea3e82d0d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 29 Jun 2015 12:42:02 +0200
Subject: [PATCH 1746/6219] [YoutubeDL] format spec: allow grouping specifiers
 with parentheses

---
 test/test_YoutubeDL.py  | 24 ++++++++++++++++++++++++
 youtube_dl/YoutubeDL.py | 39 +++++++++++++++++++++++++++++++++++++--
 2 files changed, 61 insertions(+), 2 deletions(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 709e3100f..6f374d7ea 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -245,6 +245,30 @@ class TestFormatSelection(unittest.TestCase):
         self.assertEqual(downloaded['format_id'], '137+141')
         self.assertEqual(downloaded['ext'], 'mp4')
 
+        info_dict = _make_result(list(formats_order), extractor='youtube')
+        ydl = YDL({'format': '(bestvideo[ext=mp4],bestvideo[ext=webm])+bestaudio'})
+        yie = YoutubeIE(ydl)
+        yie._sort_formats(info_dict['formats'])
+        ydl.process_ie_result(info_dict)
+        downloaded_ids = [info['format_id'] for info in ydl.downloaded_info_dicts]
+        self.assertEqual(downloaded_ids, ['137+141', '248+141'])
+
+        info_dict = _make_result(list(formats_order), extractor='youtube')
+        ydl = YDL({'format': '(bestvideo[ext=mp4],bestvideo[ext=webm])[height<=720]+bestaudio'})
+        yie = YoutubeIE(ydl)
+        yie._sort_formats(info_dict['formats'])
+        ydl.process_ie_result(info_dict)
+        downloaded_ids = [info['format_id'] for info in ydl.downloaded_info_dicts]
+        self.assertEqual(downloaded_ids, ['136+141', '247+141'])
+
+        info_dict = _make_result(list(formats_order), extractor='youtube')
+        ydl = YDL({'format': '(bestvideo[ext=none]/bestvideo[ext=webm])+bestaudio'})
+        yie = YoutubeIE(ydl)
+        yie._sort_formats(info_dict['formats'])
+        ydl.process_ie_result(info_dict)
+        downloaded_ids = [info['format_id'] for info in ydl.downloaded_info_dicts]
+        self.assertEqual(downloaded_ids, ['248+141'])
+
         for f1, f2 in zip(formats_order, formats_order[1:]):
             info_dict = _make_result([f1, f2], extractor='youtube')
             ydl = YDL({'format': 'best/bestvideo'})
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 258e612af..e5b46f87e 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -920,6 +920,7 @@ class YoutubeDL(object):
         PICKFIRST = 'PICKFIRST'
         MERGE = 'MERGE'
         SINGLE = 'SINGLE'
+        GROUP = 'GROUP'
         FormatSelector = collections.namedtuple('FormatSelector', ['type', 'selector', 'filters'])
 
         def _parse_filter(tokens):
@@ -942,6 +943,10 @@ class YoutubeDL(object):
                 elif type == tokenize.OP:
                     if string in endwith:
                         break
+                    elif string == ')':
+                        # ')' will be handled by the parentheses group
+                        tokens.restore_last_token()
+                        break
                     if string == ',':
                         selectors.append(current_selector)
                         current_selector = None
@@ -955,6 +960,10 @@ class YoutubeDL(object):
                             current_selector = FormatSelector(SINGLE, 'best', [])
                         format_filter = _parse_filter(tokens)
                         current_selector.filters.append(format_filter)
+                    elif string == '(':
+                        if current_selector:
+                            raise syntax_error('Unexpected "("', start)
+                        current_selector = FormatSelector(GROUP, _parse_format_selection(tokens, [')']), [])
                     elif string == '+':
                         video_selector = current_selector
                         audio_selector = _parse_format_selection(tokens, [','])
@@ -977,6 +986,8 @@ class YoutubeDL(object):
                         for format in f(formats):
                             yield format
                 return selector_function
+            elif selector.type == GROUP:
+                selector_function = _build_selector_function(selector.selector)
             elif selector.type == PICKFIRST:
                 fs = [_build_selector_function(s) for s in selector.selector]
 
@@ -1084,8 +1095,32 @@ class YoutubeDL(object):
             return final_selector
 
         stream = io.BytesIO(format_spec.encode('utf-8'))
-        tokens = compat_tokenize_tokenize(stream.readline)
-        parsed_selector = _parse_format_selection(tokens)
+        try:
+            tokens = list(compat_tokenize_tokenize(stream.readline))
+        except tokenize.TokenError:
+            raise syntax_error('Missing closing/opening brackets or parenthesis', (0, len(format_spec)))
+
+        class TokenIterator(object):
+            def __init__(self, tokens):
+                self.tokens = tokens
+                self.counter = 0
+
+            def __iter__(self):
+                return self
+
+            def __next__(self):
+                if self.counter >= len(self.tokens):
+                    raise StopIteration()
+                value = self.tokens[self.counter]
+                self.counter += 1
+                return value
+
+            next = __next__
+
+            def restore_last_token(self):
+                self.counter -= 1
+
+        parsed_selector = _parse_format_selection(iter(TokenIterator(tokens)))
         return _build_selector_function(parsed_selector)
 
     def _calc_headers(self, info_dict):

From cf386750c9194839e419a0412f45f25f28236c77 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 29 Jun 2015 22:21:09 +0600
Subject: [PATCH 1747/6219] [hentaistigma] Modernize

---
 youtube_dl/extractor/hentaistigma.py | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/hentaistigma.py b/youtube_dl/extractor/hentaistigma.py
index 225af8cb3..f5aa73d18 100644
--- a/youtube_dl/extractor/hentaistigma.py
+++ b/youtube_dl/extractor/hentaistigma.py
@@ -1,7 +1,5 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 
 
@@ -19,20 +17,19 @@ class HentaiStigmaIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id)
 
         title = self._html_search_regex(
-            r'<h2 class="posttitle"><a[^>]*>([^<]+)</a>',
+            r'<h2[^>]+class="posttitle"[^>]*><a[^>]*>([^<]+)</a>',
             webpage, 'title')
         wrap_url = self._html_search_regex(
-            r'<iframe src="([^"]+mp4)"', webpage, 'wrapper url')
+            r'<iframe[^>]+src="([^"]+mp4)"', webpage, 'wrapper url')
         wrap_webpage = self._download_webpage(wrap_url, video_id)
 
         video_url = self._html_search_regex(
-            r'file:"([^"]+)"', wrap_webpage, 'video url')
+            r'file\s*:\s*"([^"]+)"', wrap_webpage, 'video url')
 
         return {
             'id': video_id,

From f0714c9f869a8b4c2032566af12c107491472361 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 30 Jun 2015 01:02:48 +0600
Subject: [PATCH 1748/6219] [youtube] Speed up upload date regex (#6125)

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index c28ca9319..ed382e10b 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1003,7 +1003,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         mobj = re.search(r'(?s)id="eow-date.*?>(.*?)</span>', video_webpage)
         if mobj is None:
             mobj = re.search(
-                r'(?s)id="watch-uploader-info".*?>.*?(?:Published|Uploaded|Streamed live) on (.*?)</strong>',
+                r'id="watch-uploader-info".*?>.*?(?:Published|Uploaded|Streamed live|Started) on (.*?)</strong>',
                 video_webpage)
         if mobj is not None:
             upload_date = ' '.join(re.sub(r'[/,-]', r' ', mobj.group(1)).split())

From c05724cb18eab295b4917a7e13f845e55cac56d9 Mon Sep 17 00:00:00 2001
From: alarig <alarig@swordarmor.fr>
Date: Tue, 30 Jun 2015 16:47:14 +0200
Subject: [PATCH 1749/6219] Add support of HTTPS for ina.fr

---
 youtube_dl/extractor/ina.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ina.py b/youtube_dl/extractor/ina.py
index 0847074ee..65712abc2 100644
--- a/youtube_dl/extractor/ina.py
+++ b/youtube_dl/extractor/ina.py
@@ -7,7 +7,7 @@ from .common import InfoExtractor
 
 
 class InaIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?ina\.fr/video/(?P<id>I?[A-Z0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?ina\.fr/video/(?P<id>I?[A-Z0-9]+)'
     _TEST = {
         'url': 'http://www.ina.fr/video/I12055569/francois-hollande-je-crois-que-c-est-clair-video.html',
         'md5': 'a667021bf2b41f8dc6049479d9bb38a3',

From 9d0b581feaa3adc72ed3a53b7e8c86fadfddb6e1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 30 Jun 2015 20:52:26 +0600
Subject: [PATCH 1750/6219] [youtube] Prefer meta for upload date and modernize

---
 youtube_dl/extractor/youtube.py | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index ed382e10b..6769a009d 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -999,15 +999,16 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             video_thumbnail = compat_urllib_parse.unquote_plus(video_info['thumbnail_url'][0])
 
         # upload date
-        upload_date = None
-        mobj = re.search(r'(?s)id="eow-date.*?>(.*?)</span>', video_webpage)
-        if mobj is None:
-            mobj = re.search(
-                r'id="watch-uploader-info".*?>.*?(?:Published|Uploaded|Streamed live|Started) on (.*?)</strong>',
-                video_webpage)
-        if mobj is not None:
-            upload_date = ' '.join(re.sub(r'[/,-]', r' ', mobj.group(1)).split())
-            upload_date = unified_strdate(upload_date)
+        upload_date = self._html_search_meta(
+            'datePublished', video_webpage, 'upload date', default=None)
+        if not upload_date:
+            upload_date = self._search_regex(
+                [r'(?s)id="eow-date.*?>(.*?)</span>',
+                 r'id="watch-uploader-info".*?>.*?(?:Published|Uploaded|Streamed live|Started) on (.+?)</strong>'],
+                video_webpage, 'upload date', default=None)
+            if upload_date:
+                upload_date = ' '.join(re.sub(r'[/,-]', r' ', mobj.group(1)).split())
+        upload_date = unified_strdate(upload_date)
 
         m_cat_container = self._search_regex(
             r'(?s)<h4[^>]*>\s*Category\s*</h4>\s*<ul[^>]*>(.*?)</ul>',

From cf2ac6df6896dac4d23918867bb86fac1e1088d9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 30 Jun 2015 19:45:42 +0200
Subject: [PATCH 1751/6219] [YoutubeDL] format spec: Fix handling of '+' with
 '/'

'bestvideo+bestaudio/best' was incorrectly interpreted as 'bestvideo+(bestaudio/best)', so it would fail if 'bestaudio' doesn't exist instead of falling back to 'best'.
---
 test/test_YoutubeDL.py  |  8 ++++++++
 youtube_dl/YoutubeDL.py | 25 +++++++++++++++----------
 2 files changed, 23 insertions(+), 10 deletions(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 6f374d7ea..1e4aaa559 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -245,6 +245,14 @@ class TestFormatSelection(unittest.TestCase):
         self.assertEqual(downloaded['format_id'], '137+141')
         self.assertEqual(downloaded['ext'], 'mp4')
 
+        info_dict = _make_result(list(formats_order), extractor='youtube')
+        ydl = YDL({'format': 'bestvideo[height>=999999]+bestaudio/best'})
+        yie = YoutubeIE(ydl)
+        yie._sort_formats(info_dict['formats'])
+        ydl.process_ie_result(info_dict)
+        downloaded = ydl.downloaded_info_dicts[0]
+        self.assertEqual(downloaded['format_id'], '38')
+
         info_dict = _make_result(list(formats_order), extractor='youtube')
         ydl = YDL({'format': '(bestvideo[ext=mp4],bestvideo[ext=webm])+bestaudio'})
         yie = YoutubeIE(ydl)
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index e5b46f87e..5deb4848e 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -931,7 +931,7 @@ class YoutubeDL(object):
                 else:
                     filter_parts.append(string)
 
-        def _parse_format_selection(tokens, endwith=[]):
+        def _parse_format_selection(tokens, inside_merge=False, inside_choice=False, inside_group=False):
             selectors = []
             current_selector = None
             for type, string, start, _, _ in tokens:
@@ -941,18 +941,23 @@ class YoutubeDL(object):
                 elif type in [tokenize.NAME, tokenize.NUMBER]:
                     current_selector = FormatSelector(SINGLE, string, [])
                 elif type == tokenize.OP:
-                    if string in endwith:
+                    if string == ')':
+                        if not inside_group:
+                            # ')' will be handled by the parentheses group
+                            tokens.restore_last_token()
                         break
-                    elif string == ')':
-                        # ')' will be handled by the parentheses group
+                    elif inside_merge and string in ['/', ',']:
                         tokens.restore_last_token()
                         break
-                    if string == ',':
+                    elif inside_choice and string == ',':
+                        tokens.restore_last_token()
+                        break
+                    elif string == ',':
                         selectors.append(current_selector)
                         current_selector = None
                     elif string == '/':
                         first_choice = current_selector
-                        second_choice = _parse_format_selection(tokens, [','])
+                        second_choice = _parse_format_selection(tokens, inside_choice=True)
                         current_selector = None
                         selectors.append(FormatSelector(PICKFIRST, (first_choice, second_choice), []))
                     elif string == '[':
@@ -963,12 +968,12 @@ class YoutubeDL(object):
                     elif string == '(':
                         if current_selector:
                             raise syntax_error('Unexpected "("', start)
-                        current_selector = FormatSelector(GROUP, _parse_format_selection(tokens, [')']), [])
+                        group = _parse_format_selection(tokens, inside_group=True)
+                        current_selector = FormatSelector(GROUP, group, [])
                     elif string == '+':
                         video_selector = current_selector
-                        audio_selector = _parse_format_selection(tokens, [','])
-                        current_selector = None
-                        selectors.append(FormatSelector(MERGE, (video_selector, audio_selector), []))
+                        audio_selector = _parse_format_selection(tokens, inside_merge=True)
+                        current_selector = FormatSelector(MERGE, (video_selector, audio_selector), [])
                     else:
                         raise syntax_error('Operator not recognized: "{0}"'.format(string), start)
                 elif type == tokenize.ENDMARKER:

From 1866432db74946c2b66263d38ed2c9d9d7e3177d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aur=C3=A9lio=20A=2E=20Heckert?= <aurelio@colivre.coop.br>
Date: Tue, 30 Jun 2015 16:22:09 -0300
Subject: [PATCH 1752/6219] Rename --pp-params to --postprocessor-args and
 access value as super class attribute

---
 README.md                          |  2 +-
 youtube_dl/YoutubeDL.py            |  2 +-
 youtube_dl/__init__.py             |  6 +-----
 youtube_dl/options.py              |  4 ++--
 youtube_dl/postprocessor/common.py |  6 ++++--
 youtube_dl/postprocessor/ffmpeg.py | 11 +++++------
 6 files changed, 14 insertions(+), 17 deletions(-)

diff --git a/README.md b/README.md
index 813ac4a15..7eb17a163 100644
--- a/README.md
+++ b/README.md
@@ -214,7 +214,7 @@ which means you can modify it, redistribute it or use it however you like.
     --audio-quality QUALITY          Specify ffmpeg/avconv audio quality, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K (default
                                      5)
     --recode-video FORMAT            Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm|mkv|xvid)
-    --pp-params                      Extra parameters for video post-processor.
+    --postprocessor-args             Extra parameters for video post-processor.
     -k, --keep-video                 Keep the video file on disk after the post-processing; the video is erased by default
     --no-post-overwrites             Do not overwrite post-processed files; the post-processed files are overwritten by default
     --embed-subs                     Embed subtitles in the video (only for mkv and mp4 videos)
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 3bfe30c76..ff95add78 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -261,7 +261,7 @@ class YoutubeDL(object):
     The following options are used by the post processors:
     prefer_ffmpeg:     If True, use ffmpeg instead of avconv if both are available,
                        otherwise prefer avconv.
-    pp_params:         Extra parameters for external apps, like avconv.
+    postprocessor_args: Extra parameters for external apps, like avconv.
     """
 
     params = None
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 8b54d4ae2..356697015 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -171,10 +171,6 @@ def _real_main(argv=None):
     if opts.recodevideo is not None:
         if opts.recodevideo not in ['mp4', 'flv', 'webm', 'ogg', 'mkv', 'xvid']:
             parser.error('invalid video recode format specified')
-    if opts.pp_params is None:
-        opts.pp_params = []
-    else:
-        opts.pp_params = shlex.split(opts.pp_params)
     if opts.convertsubtitles is not None:
         if opts.convertsubtitles not in ['srt', 'vtt', 'ass']:
             parser.error('invalid subtitle format specified')
@@ -231,7 +227,7 @@ def _real_main(argv=None):
         postprocessors.append({
             'key': 'FFmpegVideoConvertor',
             'preferedformat': opts.recodevideo,
-            'extra_params': opts.pp_params
+            'extra_cmd_args': opts.postprocessor_args,
         })
     if opts.convertsubtitles:
         postprocessors.append({
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index fbba9b9d8..3d88428c4 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -688,8 +688,8 @@ def parseOpts(overrideArguments=None):
         metavar='FORMAT', dest='recodevideo', default=None,
         help='Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm|mkv|xvid)')
     postproc.add_option(
-        '--pp-params',
-        dest='pp_params', default=None, metavar='ARGS',
+        '--postprocessor-args',
+        dest='postprocessor_args', default=None, metavar='ARGS',
         help='Extra parameters for video post-processor.')
     postproc.add_option(
         '-k', '--keep-video',
diff --git a/youtube_dl/postprocessor/common.py b/youtube_dl/postprocessor/common.py
index d944d9367..c44501b59 100644
--- a/youtube_dl/postprocessor/common.py
+++ b/youtube_dl/postprocessor/common.py
@@ -1,6 +1,7 @@
 from __future__ import unicode_literals
 
 import os
+import shlex
 
 from ..utils import (
     PostProcessingError,
@@ -23,12 +24,13 @@ class PostProcessor(object):
 
     PostProcessor objects follow a "mutual registration" process similar
     to InfoExtractor objects. And it can receive parameters from CLI trough
-    --pp-params.
+    --postprocessor-args.
     """
 
     _downloader = None
 
-    def __init__(self, downloader=None):
+    def __init__(self, downloader=None, extra_cmd_args=None):
+        self._extra_cmd_args = shlex.split(extra_cmd_args or '')
         self._downloader = downloader
 
     def set_downloader(self, downloader):
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index a696b12b4..891c72769 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -29,8 +29,8 @@ class FFmpegPostProcessorError(PostProcessingError):
 
 
 class FFmpegPostProcessor(PostProcessor):
-    def __init__(self, downloader=None):
-        PostProcessor.__init__(self, downloader)
+    def __init__(self, downloader=None, extra_cmd_args=None):
+        PostProcessor.__init__(self, downloader, extra_cmd_args)
         self._determine_executables()
 
     def check_version(self):
@@ -287,16 +287,15 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
 
 
 class FFmpegVideoConvertorPP(FFmpegPostProcessor):
-    def __init__(self, downloader=None, preferedformat=None, extra_params=[]):
-        super(FFmpegVideoConvertorPP, self).__init__(downloader)
+    def __init__(self, downloader=None, preferedformat=None, extra_cmd_args=None):
+        super(FFmpegVideoConvertorPP, self).__init__(downloader, extra_cmd_args)
         self._preferedformat = preferedformat
-        self._extra_params = extra_params
 
     def run(self, information):
         path = information['filepath']
         prefix, sep, ext = path.rpartition('.')
         ext = self._preferedformat
-        options = self._extra_params
+        options = self._extra_cmd_args
         if self._preferedformat == 'xvid':
             ext = 'avi'
             options.extend(['-c:v', 'libxvid', '-vtag', 'XVID'])

From 86511ea417ac22e02800fdc3efda1d73a87eb772 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 1 Jul 2015 21:47:56 +0600
Subject: [PATCH 1753/6219] [drtuber] Fix extraction

---
 youtube_dl/extractor/drtuber.py | 21 ++++++++++-----------
 1 file changed, 10 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/drtuber.py b/youtube_dl/extractor/drtuber.py
index 37c5c181f..639f9182c 100644
--- a/youtube_dl/extractor/drtuber.py
+++ b/youtube_dl/extractor/drtuber.py
@@ -36,25 +36,24 @@ class DrTuberIE(InfoExtractor):
             r'<source src="([^"]+)"', webpage, 'video URL')
 
         title = self._html_search_regex(
-            [r'class="hd_title" style="[^"]+">([^<]+)</h1>', r'<title>([^<]+) - \d+'],
+            [r'<p[^>]+class="title_substrate">([^<]+)</p>', r'<title>([^<]+) - \d+'],
             webpage, 'title')
 
         thumbnail = self._html_search_regex(
             r'poster="([^"]+)"',
             webpage, 'thumbnail', fatal=False)
 
-        like_count = str_to_int(self._html_search_regex(
-            r'<span id="rate_likes">\s*<img[^>]+>\s*<span>([\d,\.]+)</span>',
-            webpage, 'like count', fatal=False))
-        dislike_count = str_to_int(self._html_search_regex(
-            r'<span id="rate_dislikes">\s*<img[^>]+>\s*<span>([\d,\.]+)</span>',
-            webpage, 'like count', fatal=False))
-        comment_count = str_to_int(self._html_search_regex(
-            r'<span class="comments_count">([\d,\.]+)</span>',
-            webpage, 'comment count', fatal=False))
+        def extract_count(id_, name):
+            return str_to_int(self._html_search_regex(
+                r'<span[^>]+(?:class|id)="%s"[^>]*>([\d,\.]+)</span>' % id_,
+                webpage, '%s count' % name, fatal=False))
+
+        like_count = extract_count('rate_likes', 'like')
+        dislike_count = extract_count('rate_dislikes', 'dislike')
+        comment_count = extract_count('comments_count', 'comment')
 
         cats_str = self._search_regex(
-            r'<span>Categories:</span><div>(.+?)</div>', webpage, 'categories', fatal=False)
+            r'<div[^>]+class="categories_list">(.+?)</div>', webpage, 'categories', fatal=False)
         categories = [] if not cats_str else re.findall(r'<a title="([^"]+)"', cats_str)
 
         return {

From aa5d9a79d6b5c354ee4a6bfbb43f94c2485ab9b4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aur=C3=A9lio=20A=2E=20Heckert?= <aurelio@colivre.coop.br>
Date: Wed, 1 Jul 2015 20:12:26 -0300
Subject: [PATCH 1754/6219] Simplify `postprocessor_args` transmission to PP
 base class

* Remove `extra_cmd_args` transmission from sub to super class.
* Simplify params transmission through `downloader.params`.
---
 youtube_dl/__init__.py             | 2 +-
 youtube_dl/postprocessor/common.py | 5 ++---
 youtube_dl/postprocessor/ffmpeg.py | 8 ++++----
 3 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 356697015..249f76365 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -227,7 +227,6 @@ def _real_main(argv=None):
         postprocessors.append({
             'key': 'FFmpegVideoConvertor',
             'preferedformat': opts.recodevideo,
-            'extra_cmd_args': opts.postprocessor_args,
         })
     if opts.convertsubtitles:
         postprocessors.append({
@@ -354,6 +353,7 @@ def _real_main(argv=None):
         'extract_flat': opts.extract_flat,
         'merge_output_format': opts.merge_output_format,
         'postprocessors': postprocessors,
+        'postprocessor_args': shlex.split(opts.postprocessor_args or ''),
         'fixup': opts.fixup,
         'source_address': opts.source_address,
         'call_home': opts.call_home,
diff --git a/youtube_dl/postprocessor/common.py b/youtube_dl/postprocessor/common.py
index c44501b59..bee64c457 100644
--- a/youtube_dl/postprocessor/common.py
+++ b/youtube_dl/postprocessor/common.py
@@ -1,7 +1,6 @@
 from __future__ import unicode_literals
 
 import os
-import shlex
 
 from ..utils import (
     PostProcessingError,
@@ -29,8 +28,8 @@ class PostProcessor(object):
 
     _downloader = None
 
-    def __init__(self, downloader=None, extra_cmd_args=None):
-        self._extra_cmd_args = shlex.split(extra_cmd_args or '')
+    def __init__(self, downloader=None):
+        self._extra_cmd_args = downloader.params.get('postprocessor_args')
         self._downloader = downloader
 
     def set_downloader(self, downloader):
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 891c72769..de8c225da 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -29,8 +29,8 @@ class FFmpegPostProcessorError(PostProcessingError):
 
 
 class FFmpegPostProcessor(PostProcessor):
-    def __init__(self, downloader=None, extra_cmd_args=None):
-        PostProcessor.__init__(self, downloader, extra_cmd_args)
+    def __init__(self, downloader=None):
+        PostProcessor.__init__(self, downloader)
         self._determine_executables()
 
     def check_version(self):
@@ -287,8 +287,8 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
 
 
 class FFmpegVideoConvertorPP(FFmpegPostProcessor):
-    def __init__(self, downloader=None, preferedformat=None, extra_cmd_args=None):
-        super(FFmpegVideoConvertorPP, self).__init__(downloader, extra_cmd_args)
+    def __init__(self, downloader=None, preferedformat=None):
+        super(FFmpegVideoConvertorPP, self).__init__(downloader)
         self._preferedformat = preferedformat
 
     def run(self, information):

From dbe1a935269fe24fdbb894c325af4b8394c49c04 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 2 Jul 2015 13:05:43 +0800
Subject: [PATCH 1755/6219] [pbs] Fix player URL (closes #6139)

---
 youtube_dl/extractor/pbs.py | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 143a76696..289f930e0 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -90,6 +90,21 @@ class PBSIE(InfoExtractor):
                 'id': 'united-states-of-secrets',
             },
             'playlist_count': 2,
+        },
+        {
+            'url': 'http://www.pbs.org/wgbh/americanexperience/films/death/player/',
+            'info_dict': {
+                'id': '2280706814',
+                'display_id': 'player',
+                'ext': 'mp4',
+                'title': 'Death and the Civil War',
+                'description': 'American Experience, TV’s most-watched history series, brings to life the compelling stories from our past that inform our understanding of the world today.',
+                'duration': 6705,
+                'thumbnail': 're:^https?://.*\.jpg$',
+            },
+            'params': {
+                'skip_download': True,  # requires ffmpeg
+            }
         }
     ]
 
@@ -123,7 +138,7 @@ class PBSIE(InfoExtractor):
                 return media_id, presumptive_id, upload_date
 
             url = self._search_regex(
-                r'<iframe\s+(?:class|id)=["\']partnerPlayer["\'].*?\s+src=["\'](.*?)["\']>',
+                r'<iframe\s+[^>]*\s+src=["\']([^\'"]+partnerplayer[^\'"]+)["\']',
                 webpage, 'player URL')
             mobj = re.match(self._VALID_URL, url)
 

From 5b61070c700d2460e5aa8ea22f3aa806a2e17dc9 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 2 Jul 2015 13:08:48 +0800
Subject: [PATCH 1756/6219] [pbs] skip_download for m3u8 test cases

---
 youtube_dl/extractor/pbs.py | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 289f930e0..32c6adeba 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -35,6 +35,9 @@ class PBSIE(InfoExtractor):
                 'description': 'md5:ba0c207295339c8d6eced00b7c363c6a',
                 'duration': 3190,
             },
+            'params': {
+                'skip_download': True,  # requires ffmpeg
+            },
         },
         {
             'url': 'http://www.pbs.org/wgbh/pages/frontline/losing-iraq/',
@@ -46,6 +49,9 @@ class PBSIE(InfoExtractor):
                 'description': 'md5:f5bfbefadf421e8bb8647602011caf8e',
                 'duration': 5050,
             },
+            'params': {
+                'skip_download': True,  # requires ffmpeg
+            }
         },
         {
             'url': 'http://www.pbs.org/newshour/bb/education-jan-june12-cyberschools_02-23/',
@@ -68,7 +74,10 @@ class PBSIE(InfoExtractor):
                 'title': 'Dudamel Conducts Verdi Requiem at the Hollywood Bowl - Full',
                 'duration': 6559,
                 'thumbnail': 're:^https?://.*\.jpg$',
-            }
+            },
+            'params': {
+                'skip_download': True,  # requires ffmpeg
+            },
         },
         {
             'url': 'http://www.pbs.org/wgbh/nova/earth/killer-typhoon.html',
@@ -82,7 +91,10 @@ class PBSIE(InfoExtractor):
                 'duration': 3172,
                 'thumbnail': 're:^https?://.*\.jpg$',
                 'upload_date': '20140122',
-            }
+            },
+            'params': {
+                'skip_download': True,  # requires ffmpeg
+            },
         },
         {
             'url': 'http://www.pbs.org/wgbh/pages/frontline/united-states-of-secrets/',
@@ -104,7 +116,7 @@ class PBSIE(InfoExtractor):
             },
             'params': {
                 'skip_download': True,  # requires ffmpeg
-            }
+            },
         }
     ]
 

From 93f942099351895c16ef558366f3eab241cd8793 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 2 Jul 2015 13:13:27 +0800
Subject: [PATCH 1757/6219] [pbs] Add coding declaration

Python 2.x does not work without it.
---
 youtube_dl/extractor/pbs.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 32c6adeba..1e2b965f9 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -1,3 +1,4 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
 import re

From 8d06a62485ddd2cff396287f6b39614afd1e6033 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 2 Jul 2015 16:47:55 +0800
Subject: [PATCH 1758/6219] [npo] Decrypting token (closes #6136)

---
 youtube_dl/extractor/npo.py | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index 5d8448571..41d7e3fcc 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -16,8 +16,23 @@ class NPOBaseIE(InfoExtractor):
         token_page = self._download_webpage(
             'http://ida.omroep.nl/npoplayer/i.js',
             video_id, note='Downloading token')
-        return self._search_regex(
+        token = self._search_regex(
             r'npoplayer\.token = "(.+?)"', token_page, 'token')
+        token_l = list(token)
+        first = second = None
+        for i in range(5, len(token_l) - 4):
+            if token_l[i].isdigit():
+                if first is None:
+                    first = i
+                elif second is None:
+                    second = i
+        if first is None or second is None:
+            first = 12
+            second = 13
+
+        token_l[first], token_l[second] = token_l[second], token_l[first]
+
+        return ''.join(token_l)
 
 
 class NPOIE(NPOBaseIE):
@@ -92,7 +107,7 @@ class NPOIE(NPOBaseIE):
 
     def _get_info(self, video_id):
         metadata = self._download_json(
-            'http://e.omroep.nl/metadata/aflevering/%s' % video_id,
+            'http://e.omroep.nl/metadata/%s' % video_id,
             video_id,
             # We have to remove the javascript callback
             transform_source=strip_jsonp,

From f5ca97e393badab4efa07e358b051318955d0b6f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 2 Jul 2015 20:20:09 +0600
Subject: [PATCH 1759/6219] [npo] Clarify token decryption algorithm source

---
 youtube_dl/extractor/npo.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index 41d7e3fcc..62d12b7a6 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -18,6 +18,7 @@ class NPOBaseIE(InfoExtractor):
             video_id, note='Downloading token')
         token = self._search_regex(
             r'npoplayer\.token = "(.+?)"', token_page, 'token')
+        # Decryption algorithm extracted from http://npoplayer.omroep.nl/csjs/npoplayer-min.js
         token_l = list(token)
         first = second = None
         for i in range(5, len(token_l) - 4):

From 66e568de3b7ed73fd379b941f35ce2b3c7ad9664 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 2 Jul 2015 21:39:46 +0600
Subject: [PATCH 1760/6219] [extractor/generic] Improve kaltura embeds support
 (Closes #6137)

---
 youtube_dl/extractor/generic.py | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 32e41d13e..ea60d4a96 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -669,6 +669,18 @@ class GenericIE(InfoExtractor):
                 'title': 'John Carlson Postgame 2/25/15',
             },
         },
+        # Kaltura embed (different embed code)
+        {
+            'url': 'http://www.premierchristianradio.com/Shows/Saturday/Unbelievable/Conference-Videos/Os-Guinness-Is-It-Fools-Talk-Unbelievable-Conference-2014',
+            'info_dict': {
+                'id': '1_a52wc67y',
+                'ext': 'flv',
+                'upload_date': '20150127',
+                'uploader_id': 'PremierMedia',
+                'timestamp': int,
+                'title': 'Os Guinness // Is It Fools Talk? // Unbelievable? Conference 2014',
+            },
+        },
         # Eagle.Platform embed (generic URL)
         {
             'url': 'http://lenta.ru/news/2015/03/06/navalny/',
@@ -1492,8 +1504,8 @@ class GenericIE(InfoExtractor):
             return self.url_result(mobj.group('url'), 'Zapiks')
 
         # Look for Kaltura embeds
-        mobj = re.search(
-            r"(?s)kWidget\.(?:thumb)?[Ee]mbed\(\{.*?'wid'\s*:\s*'_?(?P<partner_id>[^']+)',.*?'entry_id'\s*:\s*'(?P<id>[^']+)',", webpage)
+        mobj = (re.search(r"(?s)kWidget\.(?:thumb)?[Ee]mbed\(\{.*?'wid'\s*:\s*'_?(?P<partner_id>[^']+)',.*?'entry_id'\s*:\s*'(?P<id>[^']+)',", webpage) or
+                re.search(r'(?s)(["\'])(?:https?:)?//cdnapisec\.kaltura\.com/.*?(?:p|partner_id)/(?P<partner_id>\d+).*?\1.*?entry_id\s*:\s*(["\'])(?P<id>[^\2]+?)\2', webpage))
         if mobj is not None:
             return self.url_result('kaltura:%(partner_id)s:%(id)s' % mobj.groupdict(), 'Kaltura')
 

From 91b21b2334a3f8ba54a2b5c5fe4322c46565b9f6 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 3 Jul 2015 11:53:37 +0800
Subject: [PATCH 1761/6219] [infoq] Fix extraction (closes #6141)

---
 youtube_dl/extractor/infoq.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/infoq.py b/youtube_dl/extractor/infoq.py
index 117a7faf6..91a1b3ccb 100644
--- a/youtube_dl/extractor/infoq.py
+++ b/youtube_dl/extractor/infoq.py
@@ -5,6 +5,7 @@ import base64
 from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse,
+    compat_urlparse,
 )
 
 
@@ -45,7 +46,7 @@ class InfoQIE(InfoExtractor):
         video_id, extension = video_filename.split('.')
 
         http_base = self._search_regex(
-            r'EXPRESSINSTALL_SWF\s*=\s*"(https?://[^/"]+/)', webpage,
+            r'EXPRESSINSTALL_SWF\s*=\s*[^"]*"((?:https?:)?//[^/"]+/)', webpage,
             'HTTP base URL')
 
         formats = [{
@@ -55,7 +56,7 @@ class InfoQIE(InfoExtractor):
             'play_path': playpath,
         }, {
             'format_id': 'http',
-            'url': http_base + real_id,
+            'url': compat_urlparse.urljoin(url, http_base) + real_id,
         }]
         self._sort_formats(formats)
 

From ddf0f74de77cba1d13445996725bfae4a66ac283 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 3 Jul 2015 22:32:56 +0600
Subject: [PATCH 1762/6219] [howcast] Fix extraction and modernize

---
 youtube_dl/extractor/howcast.py | 35 +++++++++++++++++----------------
 1 file changed, 18 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/howcast.py b/youtube_dl/extractor/howcast.py
index 3f7d6666c..16677f179 100644
--- a/youtube_dl/extractor/howcast.py
+++ b/youtube_dl/extractor/howcast.py
@@ -1,8 +1,7 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
+from ..utils import parse_iso8601
 
 
 class HowcastIE(InfoExtractor):
@@ -13,29 +12,31 @@ class HowcastIE(InfoExtractor):
         'info_dict': {
             'id': '390161',
             'ext': 'mp4',
-            'description': 'The square knot, also known as the reef knot, is one of the oldest, most basic knots to tie, and can be used in many different ways. Here\'s the proper way to tie a square knot.',
             'title': 'How to Tie a Square Knot Properly',
-        }
+            'description': 'md5:dbe792e5f6f1489027027bf2eba188a3',
+            'timestamp': 1276081287,
+            'upload_date': '20100609',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
+        video_id = self._match_id(url)
 
-        video_id = mobj.group('id')
         webpage = self._download_webpage(url, video_id)
 
-        self.report_extraction(video_id)
-
-        video_url = self._search_regex(r'\'?file\'?: "(http://mobile-media\.howcast\.com/[0-9]+\.mp4)',
-                                       webpage, 'video URL')
-
-        video_description = self._html_search_regex(r'<meta content=(?:"([^"]+)"|\'([^\']+)\') name=\'description\'',
-                                                    webpage, 'description', fatal=False)
+        embed_code = self._search_regex(
+            r'<iframe[^>]+src="[^"]+\bembed_code=([^\b]+)\b',
+            webpage, 'ooyala embed code')
 
         return {
+            '_type': 'url_transparent',
+            'ie_key': 'Ooyala',
+            'url': 'ooyala:%s' % embed_code,
             'id': video_id,
-            'url': video_url,
-            'title': self._og_search_title(webpage),
-            'description': video_description,
-            'thumbnail': self._og_search_thumbnail(webpage),
+            'timestamp': parse_iso8601(self._html_search_meta(
+                'article:published_time', webpage, 'timestamp')),
         }

From 0bcdc276530fe26725857eda5127aefc326ddc47 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 3 Jul 2015 22:47:52 +0600
Subject: [PATCH 1763/6219] [dailymotion:cloud] Extend _VALID_URL (Closes
 #6145)

---
 youtube_dl/extractor/dailymotion.py | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index 96f0ed9ad..8852f0add 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -254,22 +254,30 @@ class DailymotionUserIE(DailymotionPlaylistIE):
 
 
 class DailymotionCloudIE(DailymotionBaseInfoExtractor):
-    _VALID_URL = r'http://api\.dmcloud\.net/embed/[^/]+/(?P<id>[^/?]+)'
+    _VALID_URL_PREFIX = r'http://api\.dmcloud\.net/(?:player/)?embed/'
+    _VALID_URL = r'%s[^/]+/(?P<id>[^/?]+)' % _VALID_URL_PREFIX
+    _VALID_EMBED_URL = r'%s[^/]+/[^\'"]+' % _VALID_URL_PREFIX
 
-    _TEST = {
+    _TESTS = [{
         # From http://www.francetvinfo.fr/economie/entreprises/les-entreprises-familiales-le-secret-de-la-reussite_933271.html
         # Tested at FranceTvInfo_2
         'url': 'http://api.dmcloud.net/embed/4e7343f894a6f677b10006b4/556e03339473995ee145930c?auth=1464865870-0-jyhsm84b-ead4c701fb750cf9367bf4447167a3db&autoplay=1',
         'only_matching': True,
-    }
+    }, {
+        # http://www.francetvinfo.fr/societe/larguez-les-amarres-le-cobaturage-se-developpe_980101.html
+        'url': 'http://api.dmcloud.net/player/embed/4e7343f894a6f677b10006b4/559545469473996d31429f06?auth=1467430263-0-90tglw2l-a3a4b64ed41efe48d7fccad85b8b8fda&autoplay=1',
+        'only_matching': True,
+    }]
 
     @classmethod
     def _extract_dmcloud_url(self, webpage):
-        mobj = re.search(r'<iframe[^>]+src=[\'"](http://api\.dmcloud\.net/embed/[^/]+/[^\'"]+)[\'"]', webpage)
+        mobj = re.search(r'<iframe[^>]+src=[\'"](%s)[\'"]' % self._VALID_EMBED_URL, webpage)
         if mobj:
             return mobj.group(1)
 
-        mobj = re.search(r'<input[^>]+id=[\'"]dmcloudUrlEmissionSelect[\'"][^>]+value=[\'"](http://api\.dmcloud\.net/embed/[^/]+/[^\'"]+)[\'"]', webpage)
+        mobj = re.search(
+            r'<input[^>]+id=[\'"]dmcloudUrlEmissionSelect[\'"][^>]+value=[\'"](%s)[\'"]' % self._VALID_EMBED_URL,
+            webpage)
         if mobj:
             return mobj.group(1)
 

From 2a46a27e6c704f7b99242243747fa7d8ac68cefc Mon Sep 17 00:00:00 2001
From: Eric Wong <eric@taedium.com>
Date: Mon, 15 Jun 2015 01:07:48 -0700
Subject: [PATCH 1764/6219] [thisamericanlife] Add a new extractor

---
 youtube_dl/extractor/__init__.py         |  1 +
 youtube_dl/extractor/thisamericanlife.py | 32 ++++++++++++++++++++++++
 2 files changed, 33 insertions(+)
 create mode 100644 youtube_dl/extractor/thisamericanlife.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index d44339200..aba62db53 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -569,6 +569,7 @@ from .tf1 import TF1IE
 from .theonion import TheOnionIE
 from .theplatform import ThePlatformIE
 from .thesixtyone import TheSixtyOneIE
+from .thisamericanlife import ThisAmericanLifeIE
 from .thisav import ThisAVIE
 from .tinypic import TinyPicIE
 from .tlc import TlcIE, TlcDeIE
diff --git a/youtube_dl/extractor/thisamericanlife.py b/youtube_dl/extractor/thisamericanlife.py
new file mode 100644
index 000000000..6118afa1a
--- /dev/null
+++ b/youtube_dl/extractor/thisamericanlife.py
@@ -0,0 +1,32 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class ThisAmericanLifeIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?thisamericanlife\.org/radio-archives/episode/(?P<id>\d+)'
+    _TEST = {
+        'url': 'http://www.thisamericanlife.org/radio-archives/episode/487/harper-high-school-part-one',
+        'md5': '5cda28076c9f9d1fd0b0f5cff5959948',
+        'info_dict': {
+            'id': '487',
+            'title': '487: Harper High School, Part One',
+            'url' : 'http://stream.thisamericanlife.org/487/stream/487_64k.m3u8',
+            'ext': 'aac',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        title = self._html_search_regex(r'<h1[^>]*>(.*?)</h1>', webpage, 'title')
+        media_url = 'http://stream.thisamericanlife.org/' + video_id + '/stream/' + video_id + '_64k.m3u8'
+
+        return {
+            'id': video_id,
+            'title': title,
+            'url': media_url,
+            'ext': 'aac',
+        }

From e41840c522fa09a08812c6e6724063f955426f21 Mon Sep 17 00:00:00 2001
From: Eric Wong <eric@taedium.com>
Date: Mon, 15 Jun 2015 09:35:48 -0700
Subject: [PATCH 1765/6219] [thisamericanlife] get info from <meta> tags

---
 youtube_dl/extractor/thisamericanlife.py | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/thisamericanlife.py b/youtube_dl/extractor/thisamericanlife.py
index 6118afa1a..300884c1d 100644
--- a/youtube_dl/extractor/thisamericanlife.py
+++ b/youtube_dl/extractor/thisamericanlife.py
@@ -6,6 +6,7 @@ from .common import InfoExtractor
 
 class ThisAmericanLifeIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?thisamericanlife\.org/radio-archives/episode/(?P<id>\d+)'
+
     _TEST = {
         'url': 'http://www.thisamericanlife.org/radio-archives/episode/487/harper-high-school-part-one',
         'md5': '5cda28076c9f9d1fd0b0f5cff5959948',
@@ -14,19 +15,26 @@ class ThisAmericanLifeIE(InfoExtractor):
             'title': '487: Harper High School, Part One',
             'url' : 'http://stream.thisamericanlife.org/487/stream/487_64k.m3u8',
             'ext': 'aac',
-        }
+            'thumbnail': 'http://www.thisamericanlife.org/sites/default/files/imagecache/large_square/episodes/487_lg_2.jpg',
+            'description': 'We spent five months at Harper High School in Chicago, where last year alone 29 current and recent students were shot. 29. We went to get a sense of what it means to live in the midst of all this gun violence, how teens and adults navigate a world of funerals and Homecoming dances.',
+        },
+        'params': {
+            # m38u download
+            'skip_download': True,
+        },
     }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        title = self._html_search_regex(r'<h1[^>]*>(.*?)</h1>', webpage, 'title')
-        media_url = 'http://stream.thisamericanlife.org/' + video_id + '/stream/' + video_id + '_64k.m3u8'
+        # TODO check to see if there's a free mp3. if so, download that, otherwise get the m3u8 stream.
 
         return {
             'id': video_id,
-            'title': title,
-            'url': media_url,
+            'title': self._html_search_regex(r'<meta property="twitter:title" content="(.*?)"', webpage, 'title'),
+            'url': 'http://stream.thisamericanlife.org/' + video_id + '/stream/' + video_id + '_64k.m3u8',
             'ext': 'aac',
+            'thumbnail': self._html_search_regex(r'<meta property="og:image" content="(.*?)"', webpage, 'thumbnail'),
+            'description': self._html_search_regex(r'<meta name="description" content="(.*?)"', webpage, 'description'),
         }

From 95506e37af27ee779dd667395f525e4a28455fc2 Mon Sep 17 00:00:00 2001
From: Eric Wong <eric@taedium.com>
Date: Mon, 15 Jun 2015 09:50:39 -0700
Subject: [PATCH 1766/6219] [thisamericanlife] Remove unnecessary comment

---
 youtube_dl/extractor/thisamericanlife.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/youtube_dl/extractor/thisamericanlife.py b/youtube_dl/extractor/thisamericanlife.py
index 300884c1d..295523271 100644
--- a/youtube_dl/extractor/thisamericanlife.py
+++ b/youtube_dl/extractor/thisamericanlife.py
@@ -28,8 +28,6 @@ class ThisAmericanLifeIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        # TODO check to see if there's a free mp3. if so, download that, otherwise get the m3u8 stream.
-
         return {
             'id': video_id,
             'title': self._html_search_regex(r'<meta property="twitter:title" content="(.*?)"', webpage, 'title'),

From e56a4c9e9bc459346d23554207b910a1b772000d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Jul 2015 05:42:53 +0600
Subject: [PATCH 1767/6219] [thisamericanlife] Improve and simplify

---
 youtube_dl/extractor/thisamericanlife.py | 42 +++++++++++++-----------
 1 file changed, 22 insertions(+), 20 deletions(-)

diff --git a/youtube_dl/extractor/thisamericanlife.py b/youtube_dl/extractor/thisamericanlife.py
index 295523271..36493a5de 100644
--- a/youtube_dl/extractor/thisamericanlife.py
+++ b/youtube_dl/extractor/thisamericanlife.py
@@ -1,38 +1,40 @@
-# coding: utf-8
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
 
 
 class ThisAmericanLifeIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?thisamericanlife\.org/radio-archives/episode/(?P<id>\d+)'
-
-    _TEST = {
+    _VALID_URL = r'https?://(?:www\.)?thisamericanlife\.org/(?:radio-archives/episode/|play_full\.php\?play=)(?P<id>\d+)'
+    _TESTS = [{
         'url': 'http://www.thisamericanlife.org/radio-archives/episode/487/harper-high-school-part-one',
-        'md5': '5cda28076c9f9d1fd0b0f5cff5959948',
+        'md5': '8f7d2da8926298fdfca2ee37764c11ce',
         'info_dict': {
             'id': '487',
+            'ext': 'm4a',
             'title': '487: Harper High School, Part One',
-            'url' : 'http://stream.thisamericanlife.org/487/stream/487_64k.m3u8',
-            'ext': 'aac',
-            'thumbnail': 'http://www.thisamericanlife.org/sites/default/files/imagecache/large_square/episodes/487_lg_2.jpg',
-            'description': 'We spent five months at Harper High School in Chicago, where last year alone 29 current and recent students were shot. 29. We went to get a sense of what it means to live in the midst of all this gun violence, how teens and adults navigate a world of funerals and Homecoming dances.',
+            'description': 'md5:ee40bdf3fb96174a9027f76dbecea655',
+            'thumbnail': 're:^https?://.*\.jpg$',
         },
-        'params': {
-            # m38u download
-            'skip_download': True,
-        },
-    }
+    }, {
+        'url': 'http://www.thisamericanlife.org/play_full.php?play=487',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
+
+        webpage = self._download_webpage(
+            'http://www.thisamericanlife.org/radio-archives/episode/%s' % video_id, video_id)
 
         return {
             'id': video_id,
-            'title': self._html_search_regex(r'<meta property="twitter:title" content="(.*?)"', webpage, 'title'),
-            'url': 'http://stream.thisamericanlife.org/' + video_id + '/stream/' + video_id + '_64k.m3u8',
-            'ext': 'aac',
-            'thumbnail': self._html_search_regex(r'<meta property="og:image" content="(.*?)"', webpage, 'thumbnail'),
-            'description': self._html_search_regex(r'<meta name="description" content="(.*?)"', webpage, 'description'),
+            'url': 'http://stream.thisamericanlife.org/{0}/stream/{0}_64k.m3u8'.format(video_id),
+            'protocol': 'm3u8_native',
+            'ext': 'm4a',
+            'acodec': 'aac',
+            'vcodec': 'none',
+            'abr': 64,
+            'title': self._html_search_meta(r'twitter:title', webpage, 'title', fatal=True),
+            'description': self._html_search_meta(r'description', webpage, 'description'),
+            'thumbnail': self._og_search_thumbnail(webpage),
         }

From 35eb649e9d96027f4d3eec77841791524cc345e2 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 4 Jul 2015 09:24:00 +0200
Subject: [PATCH 1768/6219] release 2015.07.04

---
 README.md              | 6 +++---
 docs/supportedsites.md | 5 +++++
 youtube_dl/version.py  | 2 +-
 3 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index e3452c9e1..93e7fb06f 100644
--- a/README.md
+++ b/README.md
@@ -108,7 +108,7 @@ which means you can modify it, redistribute it or use it however you like.
     --playlist-reverse               Download playlist videos in reverse order
     --xattr-set-filesize             Set file xattribute ytdl.filesize with expected filesize (experimental)
     --hls-prefer-native              Use the native HLS downloader instead of ffmpeg (experimental)
-    --external-downloader COMMAND    Use the specified external downloader. Currently supports aria2c,curl,wget
+    --external-downloader COMMAND    Use the specified external downloader. Currently supports aria2c,curl,httpie,wget
     --external-downloader-args ARGS  Give these arguments to the external downloader
 
 ## Filesystem Options:
@@ -190,8 +190,8 @@ which means you can modify it, redistribute it or use it however you like.
     --all-formats                    Download all available video formats
     --prefer-free-formats            Prefer free video formats unless a specific one is requested
     -F, --list-formats               List all available formats
-    --youtube-skip-dash-manifest     Do not download the DASH manifest on YouTube videos
-    --merge-output-format FORMAT     If a merge is required (e.g. bestvideo+bestaudio), output to given container format. One of mkv, mp4, ogg, webm, flv.Ignored if no
+    --youtube-skip-dash-manifest     Do not download the DASH manifests and related data on YouTube videos
+    --merge-output-format FORMAT     If a merge is required (e.g. bestvideo+bestaudio), output to given container format. One of mkv, mp4, ogg, webm, flv. Ignored if no
                                      merge is required
 
 ## Subtitle Options:
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 9a50fbd1c..687936103 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -283,6 +283,7 @@
  - **Motherless**
  - **Motorsport**: motorsport.com
  - **MovieClips**
+ - **MovieFap**
  - **Moviezine**
  - **movshare**: MovShare
  - **MPORA**
@@ -440,6 +441,8 @@
  - **smotri:broadcast**: Smotri.com broadcasts
  - **smotri:community**: Smotri.com community videos
  - **smotri:user**: Smotri.com user videos
+ - **SnagFilms**
+ - **SnagFilmsEmbed**
  - **Snotr**
  - **Sohu**
  - **soompi**
@@ -502,6 +505,7 @@
  - **TheOnion**
  - **ThePlatform**
  - **TheSixtyOne**
+ - **ThisAmericanLife**
  - **ThisAV**
  - **THVideo**
  - **THVideoPlaylist**
@@ -542,6 +546,7 @@
  - **twitch:stream**
  - **twitch:video**
  - **twitch:vod**
+ - **TwitterCard**
  - **Ubu**
  - **udemy**
  - **udemy:course**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index a225e03a1..eff4aebeb 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.06.25'
+__version__ = '2015.07.04'

From d7c9a3e976b00068e926e18d963e9fdb7c3cd678 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Jul 2015 17:22:11 +0600
Subject: [PATCH 1769/6219] Credit @remitamine for snagfilms (#6096)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 117b9c219..d5418dd37 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -129,3 +129,4 @@ Mister Hat
 Peter Ding
 jackyzy823
 George Brighton
+Remita Amine

From f5f4a27a964b41646303921104f4d6d6fd2098e4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 4 Jul 2015 21:30:26 +0200
Subject: [PATCH 1770/6219] [YoutubeDL] format spec: fix handling of '/' with
 ','

When using 'bestvideo/best,bestaudio', 'bestvideo/best' must be set as the current_selector (instead of appending it to the selectors), otherwise when it gets the ',' it would append 'None' to the selectors.
---
 test/test_YoutubeDL.py  | 8 ++++++++
 youtube_dl/YoutubeDL.py | 3 +--
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 1e4aaa559..f103779d3 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -253,6 +253,14 @@ class TestFormatSelection(unittest.TestCase):
         downloaded = ydl.downloaded_info_dicts[0]
         self.assertEqual(downloaded['format_id'], '38')
 
+        info_dict = _make_result(list(formats_order), extractor='youtube')
+        ydl = YDL({'format': 'bestvideo/best,bestaudio'})
+        yie = YoutubeIE(ydl)
+        yie._sort_formats(info_dict['formats'])
+        ydl.process_ie_result(info_dict)
+        downloaded_ids = [info['format_id'] for info in ydl.downloaded_info_dicts]
+        self.assertEqual(downloaded_ids, ['137', '141'])
+
         info_dict = _make_result(list(formats_order), extractor='youtube')
         ydl = YDL({'format': '(bestvideo[ext=mp4],bestvideo[ext=webm])+bestaudio'})
         yie = YoutubeIE(ydl)
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 5deb4848e..5a79e5f1d 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -958,8 +958,7 @@ class YoutubeDL(object):
                     elif string == '/':
                         first_choice = current_selector
                         second_choice = _parse_format_selection(tokens, inside_choice=True)
-                        current_selector = None
-                        selectors.append(FormatSelector(PICKFIRST, (first_choice, second_choice), []))
+                        current_selector = FormatSelector(PICKFIRST, (first_choice, second_choice), [])
                     elif string == '[':
                         if not current_selector:
                             current_selector = FormatSelector(SINGLE, 'best', [])

From bb8e55366289e0c129ef85abb8c1ac1cbae86a66 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 4 Jul 2015 21:41:09 +0200
Subject: [PATCH 1771/6219] [YoutubeDL] format spec: Do not fail when a filter
 gives an empty result

For example with 'best[height<40]' we ended getting a 'IndexError: list index out of range'.
---
 test/test_YoutubeDL.py  | 9 ++++++++-
 youtube_dl/YoutubeDL.py | 3 +++
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index f103779d3..bf2baae07 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -15,7 +15,7 @@ from youtube_dl import YoutubeDL
 from youtube_dl.compat import compat_str
 from youtube_dl.extractor import YoutubeIE
 from youtube_dl.postprocessor.common import PostProcessor
-from youtube_dl.utils import match_filter_func
+from youtube_dl.utils import ExtractorError, match_filter_func
 
 TEST_URL = 'http://localhost/sample.mp4'
 
@@ -362,6 +362,13 @@ class TestFormatSelection(unittest.TestCase):
         downloaded_ids = [info['format_id'] for info in ydl.downloaded_info_dicts]
         self.assertEqual(downloaded_ids, ['B', 'C', 'D'])
 
+        ydl = YDL({'format': 'best[height<40]'})
+        try:
+            ydl.process_ie_result(info_dict)
+        except ExtractorError:
+            pass
+        self.assertEqual(ydl.downloaded_info_dicts, [])
+
 
 class TestYoutubeDL(unittest.TestCase):
     def test_subtitles(self):
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 5a79e5f1d..6478d05dc 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1005,6 +1005,9 @@ class YoutubeDL(object):
                 format_spec = selector.selector
 
                 def selector_function(formats):
+                    formats = list(formats)
+                    if not formats:
+                        return
                     if format_spec == 'all':
                         for f in formats:
                             yield f

From ede21449c8d87c2d000d16e5102ee85d87a9b14e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 5 Jul 2015 06:29:36 +0600
Subject: [PATCH 1772/6219] [crunchyroll] Fix extraction (Closes #5855, closes
 #5881)

---
 youtube_dl/extractor/crunchyroll.py | 31 ++++++++++++++++++++++-------
 1 file changed, 24 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 41f0c736d..73f1e22ef 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -27,7 +27,7 @@ from ..aes import (
 
 
 class CrunchyrollIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:(?P<prefix>www|m)\.)?(?P<url>crunchyroll\.(?:com|fr)/(?:[^/]*/[^/?&]*?|media/\?id=)(?P<video_id>[0-9]+))(?:[/?&]|$)'
+    _VALID_URL = r'https?://(?:(?P<prefix>www|m)\.)?(?P<url>crunchyroll\.(?:com|fr)/(?:media(?:-|/\?id=)|[^/]*/[^/?&]*?)(?P<video_id>[0-9]+))(?:[/?&]|$)'
     _NETRC_MACHINE = 'crunchyroll'
     _TESTS = [{
         'url': 'http://www.crunchyroll.com/wanna-be-the-strongest-in-the-world/episode-1-an-idol-wrestler-is-born-645513',
@@ -45,6 +45,22 @@ class CrunchyrollIE(InfoExtractor):
             # rtmp
             'skip_download': True,
         },
+    }, {
+        'url': 'http://www.crunchyroll.com/media-589804/culture-japan-1',
+        'info_dict': {
+            'id': '589804',
+            'ext': 'flv',
+            'title': 'Culture Japan Episode 1 – Rebuilding Japan after the 3.11',
+            'description': 'md5:fe2743efedb49d279552926d0bd0cd9e',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'uploader': 'Danny Choo Network',
+            'upload_date': '20120213',
+        },
+        'params': {
+            # rtmp
+            'skip_download': True,
+        },
+
     }, {
         'url': 'http://www.crunchyroll.fr/girl-friend-beta/episode-11-goodbye-la-mode-661697',
         'only_matching': True,
@@ -251,16 +267,17 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
         for fmt in re.findall(r'showmedia\.([0-9]{3,4})p', webpage):
             stream_quality, stream_format = self._FORMAT_IDS[fmt]
             video_format = fmt + 'p'
-            streamdata_req = compat_urllib_request.Request('http://www.crunchyroll.com/xml/')
-            # urlencode doesn't work!
-            streamdata_req.data = 'req=RpcApiVideoEncode%5FGetStreamInfo&video%5Fencode%5Fquality=' + stream_quality + '&media%5Fid=' + stream_id + '&video%5Fformat=' + stream_format
+            streamdata_req = compat_urllib_request.Request(
+                'http://www.crunchyroll.com/xml/?req=RpcApiVideoPlayer_GetStandardConfig&media_id=%s&video_format=%s&video_quality=%s'
+                % (stream_id, stream_format, stream_quality),
+                compat_urllib_parse.urlencode({'current_page': url}).encode('utf-8'))
             streamdata_req.add_header('Content-Type', 'application/x-www-form-urlencoded')
-            streamdata_req.add_header('Content-Length', str(len(streamdata_req.data)))
             streamdata = self._download_xml(
                 streamdata_req, video_id,
                 note='Downloading media info for %s' % video_format)
-            video_url = streamdata.find('./host').text
-            video_play_path = streamdata.find('./file').text
+            stream_info = streamdata.find('./{default}preload/stream_info')
+            video_url = stream_info.find('./host').text
+            video_play_path = stream_info.find('./file').text
             formats.append({
                 'url': video_url,
                 'play_path': video_play_path,

From 43232d5c143b1025671f70ff34f5c0f28ec56847 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 5 Jul 2015 19:01:07 +0600
Subject: [PATCH 1773/6219] [rtlnl] Improve

---
 youtube_dl/extractor/rtlnl.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/rtlnl.py b/youtube_dl/extractor/rtlnl.py
index e708e0093..d94861325 100644
--- a/youtube_dl/extractor/rtlnl.py
+++ b/youtube_dl/extractor/rtlnl.py
@@ -59,9 +59,11 @@ class RtlNlIE(InfoExtractor):
         subtitle = material['title'] or info['episodes'][0]['name']
         description = material.get('synopsis') or info['episodes'][0]['synopsis']
 
+        meta = info.get('meta', {})
+
         # Use unencrypted m3u8 streams (See https://github.com/rg3/youtube-dl/issues/4118)
-        videopath = material['videopath'].replace('adaptive', 'flash')
-        m3u8_url = info['meta']['videohost'] + videopath
+        videopath = material['videopath'].replace('/adaptive/', '/flash/')
+        m3u8_url = meta.get('videohost', 'http://manifest.us.rtl.nl') + videopath
 
         formats = self._extract_m3u8_formats(m3u8_url, uuid, ext='mp4')
 
@@ -82,7 +84,7 @@ class RtlNlIE(InfoExtractor):
         self._sort_formats(formats)
 
         thumbnails = []
-        meta = info.get('meta', {})
+
         for p in ('poster_base_url', '"thumb_base_url"'):
             if not meta.get(p):
                 continue

From 9dfc4fa1a1a922fb6c0212d09d28b4a8495c1031 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 5 Jul 2015 19:07:07 +0600
Subject: [PATCH 1774/6219] [rtlnl] Add test with encrypted m3u8 streams for
 reference

---
 youtube_dl/extractor/rtlnl.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/rtlnl.py b/youtube_dl/extractor/rtlnl.py
index d94861325..049deaf8d 100644
--- a/youtube_dl/extractor/rtlnl.py
+++ b/youtube_dl/extractor/rtlnl.py
@@ -43,6 +43,10 @@ class RtlNlIE(InfoExtractor):
             'upload_date': '20150215',
             'description': 'Er zijn nieuwe beelden vrijgegeven die vlak na de aanslag in Kopenhagen zijn gemaakt. Op de video is goed te zien hoe omstanders zich bekommeren om één van de slachtoffers, terwijl de eerste agenten ter plaatse komen.',
         }
+    }, {
+        # encrypted m3u8 streams
+        'url': 'http://www.rtlxl.nl/#!/afl-2-257632/52a74543-c504-4cde-8aa8-ec66fe8d68a7',
+        'only_matching': True,
     }, {
         'url': 'http://www.rtl.nl/system/videoplayer/derden/embed.html#!/uuid=bb0353b0-d6a4-1dad-90e9-18fe75b8d1f0',
         'only_matching': True,

From 89d42c2c758a3cb06b71145c24bfe8057419295a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 6 Jul 2015 02:58:02 +0600
Subject: [PATCH 1775/6219] [rtlnl] Clarify test

---
 youtube_dl/extractor/rtlnl.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/rtlnl.py b/youtube_dl/extractor/rtlnl.py
index 049deaf8d..1228245ad 100644
--- a/youtube_dl/extractor/rtlnl.py
+++ b/youtube_dl/extractor/rtlnl.py
@@ -44,7 +44,7 @@ class RtlNlIE(InfoExtractor):
             'description': 'Er zijn nieuwe beelden vrijgegeven die vlak na de aanslag in Kopenhagen zijn gemaakt. Op de video is goed te zien hoe omstanders zich bekommeren om één van de slachtoffers, terwijl de eerste agenten ter plaatse komen.',
         }
     }, {
-        # encrypted m3u8 streams
+        # encrypted m3u8 streams, georestricted
         'url': 'http://www.rtlxl.nl/#!/afl-2-257632/52a74543-c504-4cde-8aa8-ec66fe8d68a7',
         'only_matching': True,
     }, {

From 0c20ee7d4b0c98221c71769482980a0a218dadb4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 6 Jul 2015 04:16:56 +0600
Subject: [PATCH 1776/6219] [rtlnl] Clarify current adaptive -> flash
 workaround rationale

---
 youtube_dl/extractor/rtlnl.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/rtlnl.py b/youtube_dl/extractor/rtlnl.py
index 1228245ad..a4d3d73ff 100644
--- a/youtube_dl/extractor/rtlnl.py
+++ b/youtube_dl/extractor/rtlnl.py
@@ -66,6 +66,9 @@ class RtlNlIE(InfoExtractor):
         meta = info.get('meta', {})
 
         # Use unencrypted m3u8 streams (See https://github.com/rg3/youtube-dl/issues/4118)
+        # NB: nowadays, recent ffmpeg and avconv can handle these encrypted streams, so
+        # this adaptive -> flash workaround is not required in general, but it also
+        # allows bypassing georestriction therefore is retained for now.
         videopath = material['videopath'].replace('/adaptive/', '/flash/')
         m3u8_url = meta.get('videohost', 'http://manifest.us.rtl.nl') + videopath
 

From 13af92fdc443f753dcb3cb91736b09a4ae85c36a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 6 Jul 2015 08:39:38 +0800
Subject: [PATCH 1777/6219] [common] Add 'fatal' to _extract_m3u8_formats

---
 youtube_dl/extractor/common.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 81623bfe3..d859aea52 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -846,7 +846,8 @@ class InfoExtractor(object):
 
     def _extract_m3u8_formats(self, m3u8_url, video_id, ext=None,
                               entry_protocol='m3u8', preference=None,
-                              m3u8_id=None, note=None, errnote=None):
+                              m3u8_id=None, note=None, errnote=None,
+                              fatal=True):
 
         formats = [{
             'format_id': '-'.join(filter(None, [m3u8_id, 'meta'])),
@@ -866,7 +867,10 @@ class InfoExtractor(object):
         m3u8_doc = self._download_webpage(
             m3u8_url, video_id,
             note=note or 'Downloading m3u8 information',
-            errnote=errnote or 'Failed to download m3u8 information')
+            errnote=errnote or 'Failed to download m3u8 information',
+            fatal=fatal)
+        if m3u8_doc is False:
+            return m3u8_doc
         last_info = None
         last_media = None
         kv_rex = re.compile(

From 59a83d3e5b9c09d0c6e8fb430ea99d71ef6e2eba Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 6 Jul 2015 08:40:38 +0800
Subject: [PATCH 1778/6219] [spiegeltv] Skip invalid m3u8 manifests (closes
 #6157)

---
 youtube_dl/extractor/spiegeltv.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/spiegeltv.py b/youtube_dl/extractor/spiegeltv.py
index 08a5c4314..27f4033c5 100644
--- a/youtube_dl/extractor/spiegeltv.py
+++ b/youtube_dl/extractor/spiegeltv.py
@@ -77,11 +77,13 @@ class SpiegeltvIE(InfoExtractor):
                     'rtmp_live': True,
                 })
             elif determine_ext(endpoint) == 'm3u8':
-                formats.extend(self._extract_m3u8_formats(
+                m3u8_formats = self._extract_m3u8_formats(
                     endpoint.replace('[video]', play_path),
                     video_id, 'm4v',
                     preference=1,  # Prefer hls since it allows to workaround georestriction
-                    m3u8_id='hls'))
+                    m3u8_id='hls', fatal=False)
+                if m3u8_formats is not False:
+                    formats.extend(m3u8_formats)
             else:
                 formats.append({
                     'url': endpoint,

From 37c1e4025c6df834e93a64c1c13eebac23e90942 Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Mon, 6 Jul 2015 15:26:49 +0800
Subject: [PATCH 1779/6219] [yinyuetai] New extractor for yinyuetai.com

---
 youtube_dl/extractor/__init__.py  |  1 +
 youtube_dl/extractor/yinyuetai.py | 47 +++++++++++++++++++++++++++++++
 2 files changed, 48 insertions(+)
 create mode 100644 youtube_dl/extractor/yinyuetai.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index aba62db53..8665855eb 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -733,6 +733,7 @@ from .yandexmusic import (
     YandexMusicPlaylistIE,
 )
 from .yesjapan import YesJapanIE
+from .yinyuetai import YinYueTaiIE
 from .ynet import YnetIE
 from .youjizz import YouJizzIE
 from .youku import YoukuIE
diff --git a/youtube_dl/extractor/yinyuetai.py b/youtube_dl/extractor/yinyuetai.py
new file mode 100644
index 000000000..661c34602
--- /dev/null
+++ b/youtube_dl/extractor/yinyuetai.py
@@ -0,0 +1,47 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import ExtractorError
+
+
+class YinYueTaiIE(InfoExtractor):
+    IE_NAME = 'yinyuetai:video'
+    _VALID_URL = r'https?://v\.yinyuetai\.com/video(/h5)?/(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'http://v.yinyuetai.com/video/2322376',
+        'md5': '6e3abe28d38e3a54b591f9f040595ce0',
+        'info_dict': {
+            'id': '2322376',
+            'ext': 'mp4',
+            'title': '少女时代_PARTY_Music Video Teaser',
+            'creator': '少女时代',
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        info = self._download_json(
+            'http://ext.yinyuetai.com/main/get-h-mv-info?json=true&videoId=%s' % video_id, video_id,
+            'Downloading mv info')['videoInfo']['coreVideoInfo']
+
+        if info['error']:
+            raise ExtractorError(info['errorMsg'], expected=True)
+
+        formats = [
+            {'url': format_info['videoUrl'], 'format_id': format_info['qualityLevel'],
+             'format': format_info['qualityLevelName'], 'filesize': format_info['fileSize'],
+             'ext': 'mp4', 'preference': format_info['bitrate']}
+            for format_info in info['videoUrlModels']
+        ]
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': info['videoName'],
+            'thumbnail': info['bigHeadImage'],
+            'creator': info['artistNames'],
+            'duration': info['duration'],
+            'formats': formats,
+        }

From e9d33454b552c7a5a2e048f439b90a58aa5528ad Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 6 Jul 2015 16:19:49 +0800
Subject: [PATCH 1780/6219] [qqmusic:playlist] Playlist names are optional

---
 youtube_dl/extractor/qqmusic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index f9aafcd28..d6bc05b7b 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -274,6 +274,6 @@ class QQMusicPlaylistIE(QQPlaylistBaseIE):
             ) for song in list_json['songlist']
         ]
 
-        list_name = list_json['dissname']
+        list_name = list_json.get('dissname')
         list_description = clean_html(unescapeHTML(list_json.get('desc')))
         return self.playlist_result(entries, list_id, list_name, list_description)

From fc7ae675e26a98bc74918f311d22f515ec7e0477 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 6 Jul 2015 17:08:32 +0800
Subject: [PATCH 1781/6219] [qqmusic:album] Strip description

---
 youtube_dl/extractor/qqmusic.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index e704640e5..6d85d58e3 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -169,7 +169,7 @@ class QQMusicAlbumIE(QQPlaylistBaseIE):
         'info_dict': {
             'id': '000gXCTb2AhRR1',
             'title': '我们都是这样长大的',
-            'description': 'md5:712f0cdbfc7e776820d08150e6df593d',
+            'description': 'md5:179c5dce203a5931970d306aa9607ea6',
         },
         'playlist_count': 4,
     }, {
@@ -177,7 +177,7 @@ class QQMusicAlbumIE(QQPlaylistBaseIE):
         'info_dict': {
             'id': '002Y5a3b3AlCu3',
             'title': '그리고...',
-            'description': 'md5:b1d133b8c9bac8fed4e1a97df759f4cf',
+            'description': 'md5:a48823755615508a95080e81b51ba729',
         },
         'playlist_count': 8,
     }]
@@ -196,6 +196,8 @@ class QQMusicAlbumIE(QQPlaylistBaseIE):
         ]
         album_name = album['name']
         album_detail = album.get('desc')
+        if album_detail is not None:
+            album_detail = album_detail.strip()
 
         return self.playlist_result(entries, mid, album_name, album_detail)
 

From dfc4eca21f7d34b5e65f42b284b24077c8bbc109 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 6 Jul 2015 17:09:17 +0800
Subject: [PATCH 1782/6219] [qqmusic:album] Playlist names are optional

---
 youtube_dl/extractor/qqmusic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index 6d85d58e3..03e6c688f 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -194,7 +194,7 @@ class QQMusicAlbumIE(QQPlaylistBaseIE):
                 'http://y.qq.com/#type=song&mid=' + song['songmid'], 'QQMusic', song['songmid']
             ) for song in album['list']
         ]
-        album_name = album['name']
+        album_name = album.get('name')
         album_detail = album.get('desc')
         if album_detail is not None:
             album_detail = album_detail.strip()

From 85a064861f5213abf2b56dd671e1f45188a02adf Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 6 Jul 2015 17:54:41 +0800
Subject: [PATCH 1783/6219] [qqmusic] Use regex for thumbnails in test cases

---
 youtube_dl/extractor/qqmusic.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index 8a724ab51..476432330 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -27,7 +27,7 @@ class QQMusicIE(InfoExtractor):
             'upload_date': '20141227',
             'creator': '林俊杰',
             'description': 'md5:d327722d0361576fde558f1ac68a7065',
-            'thumbnail': 'http://i.gtimg.cn/music/photo/mid_album_500/7/p/001IV22P1RDX7p.jpg',
+            'thumbnail': 're:^https?://.*\.jpg$',
         }
     }, {
         'note': 'There is no mp3-320 version of this song.',
@@ -40,7 +40,7 @@ class QQMusicIE(InfoExtractor):
             'upload_date': '20050626',
             'creator': '李季美',
             'description': 'md5:46857d5ed62bc4ba84607a805dccf437',
-            'thumbnail': 'http://i.gtimg.cn/music/photo/mid_album_500/r/Q/0042owYj46IxrQ.jpg',
+            'thumbnail': 're:^https?://.*\.jpg$',
         }
     }]
 

From 275c0423aa85691fc78dda253fa5d00ab471e7b2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 7 Jul 2015 00:02:34 +0600
Subject: [PATCH 1784/6219] [vk] Fix extraction (Closes #6153)

---
 youtube_dl/extractor/vk.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index f2ae109f9..4ae28d4ca 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -153,7 +153,7 @@ class VKIE(InfoExtractor):
         if not video_id:
             video_id = '%s_%s' % (mobj.group('oid'), mobj.group('id'))
 
-        info_url = 'http://vk.com/al_video.php?act=show&al=1&module=video&video=%s' % video_id
+        info_url = 'https://vk.com/al_video.php?act=show&al=1&module=video&video=%s' % video_id
         info_page = self._download_webpage(info_url, video_id)
 
         ERRORS = {

From 7f220b2facecde21ac3257d20938b63c1b6b01fb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 7 Jul 2015 00:04:19 +0600
Subject: [PATCH 1785/6219] [vk] Catch ownership confirmation request

---
 youtube_dl/extractor/vk.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 4ae28d4ca..ccd897084 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -156,6 +156,11 @@ class VKIE(InfoExtractor):
         info_url = 'https://vk.com/al_video.php?act=show&al=1&module=video&video=%s' % video_id
         info_page = self._download_webpage(info_url, video_id)
 
+        if re.search(r'<!>/login\.php\?.*\bact=security_check', info_page):
+            raise ExtractorError(
+                'You are trying to log in from an unusual location. You should confirm ownership at vk.com to log in with this IP.',
+                expected=True)
+
         ERRORS = {
             r'>Видеозапись .*? была изъята из публичного доступа в связи с обращением правообладателя.<':
             'Video %s has been removed from public access due to rightholder complaint.',

From d7b4d5dd5072fc82a7f3d437e5990aa74a35a100 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 7 Jul 2015 14:16:56 +0800
Subject: [PATCH 1786/6219] [gfycat] Extract id correctly (fixes #6165)

---
 youtube_dl/extractor/gfycat.py | 22 +++++++++++++++++++---
 1 file changed, 19 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/gfycat.py b/youtube_dl/extractor/gfycat.py
index 397f1d42e..048ee31e2 100644
--- a/youtube_dl/extractor/gfycat.py
+++ b/youtube_dl/extractor/gfycat.py
@@ -10,8 +10,8 @@ from ..utils import (
 
 
 class GfycatIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?gfycat\.com/(?P<id>[^/?#]+)'
-    _TEST = {
+    _VALID_URL = r'https?://(?:www\.)?gfycat\.com/(?:ifr/)?(?P<id>[^/?#]+)'
+    _TESTS = [{
         'url': 'http://gfycat.com/DeadlyDecisiveGermanpinscher',
         'info_dict': {
             'id': 'DeadlyDecisiveGermanpinscher',
@@ -27,7 +27,23 @@ class GfycatIE(InfoExtractor):
             'categories': list,
             'age_limit': 0,
         }
-    }
+    }, {
+        'url': 'http://gfycat.com/ifr/JauntyTimelyAmazontreeboa',
+        'info_dict': {
+            'id': 'JauntyTimelyAmazontreeboa',
+            'ext': 'mp4',
+            'title': 'JauntyTimelyAmazontreeboa',
+            'timestamp': 1411720126,
+            'upload_date': '20140926',
+            'uploader': 'anonymous',
+            'duration': 3.52,
+            'view_count': int,
+            'like_count': int,
+            'dislike_count': int,
+            'categories': list,
+            'age_limit': 0,
+        }
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From 267dc07e6b71277b67abad08c515ca9c1bb09f61 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 7 Jul 2015 14:22:13 +0800
Subject: [PATCH 1787/6219] [gfycat] Catch errors

---
 youtube_dl/extractor/gfycat.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/gfycat.py b/youtube_dl/extractor/gfycat.py
index 048ee31e2..884700c52 100644
--- a/youtube_dl/extractor/gfycat.py
+++ b/youtube_dl/extractor/gfycat.py
@@ -6,6 +6,7 @@ from ..utils import (
     int_or_none,
     float_or_none,
     qualities,
+    ExtractorError,
 )
 
 
@@ -50,7 +51,10 @@ class GfycatIE(InfoExtractor):
 
         gfy = self._download_json(
             'http://gfycat.com/cajax/get/%s' % video_id,
-            video_id, 'Downloading video info')['gfyItem']
+            video_id, 'Downloading video info')
+        if 'error' in gfy:
+            raise ExtractorError('Gfycat said: ' + gfy['error'], expected=True)
+        gfy = gfy['gfyItem']
 
         title = gfy.get('title') or gfy['gfyName']
         description = gfy.get('description')

From 4dd09c9addeaccad6ad6d33f01d6240f3594fc84 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 7 Jul 2015 10:36:07 +0200
Subject: [PATCH 1788/6219] release 2015.07.07

---
 docs/supportedsites.md | 1 +
 youtube_dl/version.py  | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 687936103..0ca06c71d 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -384,6 +384,7 @@
  - **Pyvideo**
  - **qqmusic**
  - **qqmusic:album**
+ - **qqmusic:playlist**
  - **qqmusic:singer**
  - **qqmusic:toplist**
  - **QuickVid**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index eff4aebeb..3364647ed 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.07.04'
+__version__ = '2015.07.07'

From cbc1fadd6f544619b711209b68ea15a912ca0fa1 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 8 Jul 2015 13:40:21 +0800
Subject: [PATCH 1789/6219] [clipsyndicate] Support chic subdomain (fixes
 #6176)

---
 youtube_dl/extractor/clipsyndicate.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/clipsyndicate.py b/youtube_dl/extractor/clipsyndicate.py
index d07d544ea..b1e45a677 100644
--- a/youtube_dl/extractor/clipsyndicate.py
+++ b/youtube_dl/extractor/clipsyndicate.py
@@ -10,9 +10,9 @@ from ..utils import (
 
 
 class ClipsyndicateIE(InfoExtractor):
-    _VALID_URL = r'http://www\.clipsyndicate\.com/video/play(list/\d+)?/(?P<id>\d+)'
+    _VALID_URL = r'http://(?:chic|www)\.clipsyndicate\.com/video/play(list/\d+)?/(?P<id>\d+)'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.clipsyndicate.com/video/play/4629301/brick_briscoe',
         'md5': '4d7d549451bad625e0ff3d7bd56d776c',
         'info_dict': {
@@ -22,7 +22,10 @@ class ClipsyndicateIE(InfoExtractor):
             'duration': 612,
             'thumbnail': 're:^https?://.+\.jpg',
         },
-    }
+    }, {
+        'url': 'http://chic.clipsyndicate.com/video/play/5844117/shark_attack',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)

From 1316b54956ac871b4644b6760c5bf88207b2e58f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 8 Jul 2015 13:43:23 +0800
Subject: [PATCH 1790/6219] [clipsyndicate] Use _match_id

---
 youtube_dl/extractor/clipsyndicate.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/clipsyndicate.py b/youtube_dl/extractor/clipsyndicate.py
index b1e45a677..8306d6fb7 100644
--- a/youtube_dl/extractor/clipsyndicate.py
+++ b/youtube_dl/extractor/clipsyndicate.py
@@ -1,7 +1,5 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..utils import (
     find_xpath_attr,
@@ -28,8 +26,7 @@ class ClipsyndicateIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
         js_player = self._download_webpage(
             'http://eplayer.clipsyndicate.com/embed/player.js?va_id=%s' % video_id,
             video_id, 'Downlaoding player')

From cf9cf7dd04d8e15635084dcf9f7a76acff5ac218 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 8 Jul 2015 20:27:06 +0600
Subject: [PATCH 1791/6219] [vk] Extend _VALID_URL to handle biqle.ru (Closes
 #6179)

---
 youtube_dl/extractor/vk.py | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index ccd897084..0f7ce45ca 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -21,7 +21,17 @@ from ..utils import (
 
 class VKIE(InfoExtractor):
     IE_NAME = 'vk.com'
-    _VALID_URL = r'https?://(?:m\.)?vk\.com/(?:video_ext\.php\?.*?\boid=(?P<oid>-?\d+).*?\bid=(?P<id>\d+)|(?:.+?\?.*?z=)?video(?P<videoid>[^s].*?)(?:\?|%2F|$))'
+    _VALID_URL = r'''(?x)
+                    https?://
+                        (?:
+                            (?:m\.)?vk\.com/video_ext\.php\?.*?\boid=(?P<oid>-?\d+).*?\bid=(?P<id>\d+)|
+                            (?:
+                                (?:m\.)?vk\.com/(?:.+?\?.*?z=)?video|
+                                (?:www\.)?biqle\.ru/watch/
+                            )
+                            (?P<videoid>[^s].*?)(?:\?|%2F|$)
+                        )
+                    '''
     _NETRC_MACHINE = 'vk'
 
     _TESTS = [
@@ -114,6 +124,11 @@ class VKIE(InfoExtractor):
             'url': 'http://vk.com/feed?z=video-43215063_166094326%2Fbb50cacd3177146d7a',
             'only_matching': True,
         },
+        {
+            # vk wrapper
+            'url': 'http://www.biqle.ru/watch/847655_160197695',
+            'only_matching': True,
+        }
     ]
 
     def _login(self):

From 4647845679429739f72b0eba5b963e096ec5bf86 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 8 Jul 2015 20:34:50 +0600
Subject: [PATCH 1792/6219] [vk] Fix youtube extraction

---
 youtube_dl/extractor/vk.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 0f7ce45ca..697064175 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -195,10 +195,11 @@ class VKIE(InfoExtractor):
             if re.search(error_re, info_page):
                 raise ExtractorError(error_msg % video_id, expected=True)
 
-        m_yt = re.search(r'src="(http://www.youtube.com/.*?)"', info_page)
-        if m_yt is not None:
-            self.to_screen('Youtube video detected')
-            return self.url_result(m_yt.group(1), 'Youtube')
+        youtube_url = self._search_regex(
+            r'<iframe[^>]+src="((?:https?:)?//www.youtube.com/embed/[^"]+)"',
+            info_page, 'youtube iframe', default=None)
+        if youtube_url:
+            return self.url_result(youtube_url, 'Youtube', video_id)
 
         m_rutube = re.search(
             r'\ssrc="((?:https?:)?//rutube\.ru\\?/video\\?/embed(?:.*?))\\?"', info_page)

From 9281f6d25338c179e7bc2f48352da3311102158e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 8 Jul 2015 20:41:08 +0600
Subject: [PATCH 1793/6219] [vk] Add test for youtube embed

---
 youtube_dl/extractor/vk.py | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 697064175..23d153031 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -119,6 +119,21 @@ class VKIE(InfoExtractor):
             },
             'skip': 'Only works from Russia',
         },
+        {
+            # youtube embed
+            'url': 'https://vk.com/video276849682_170681728',
+            'info_dict': {
+                'id': 'V3K4mi0SYkc',
+                'ext': 'mp4',
+                'title': "DSWD Awards 'Children's Joy Foundation, Inc.' Certificate of Registration and License to Operate",
+                'description': 'md5:bf9c26cfa4acdfb146362682edd3827a',
+                'duration': 179,
+                'upload_date': '20130116',
+                'uploader': "Children's Joy Foundation",
+                'uploader_id': 'thecjf',
+                'view_count': int,
+            },
+        },
         {
             # removed video, just testing that we match the pattern
             'url': 'http://vk.com/feed?z=video-43215063_166094326%2Fbb50cacd3177146d7a',
@@ -199,7 +214,7 @@ class VKIE(InfoExtractor):
             r'<iframe[^>]+src="((?:https?:)?//www.youtube.com/embed/[^"]+)"',
             info_page, 'youtube iframe', default=None)
         if youtube_url:
-            return self.url_result(youtube_url, 'Youtube', video_id)
+            return self.url_result(youtube_url, 'Youtube')
 
         m_rutube = re.search(
             r'\ssrc="((?:https?:)?//rutube\.ru\\?/video\\?/embed(?:.*?))\\?"', info_page)

From e2082ea9422aadf7ae2580d9333008279cda51f0 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 9 Jul 2015 00:50:32 +0800
Subject: [PATCH 1794/6219] [yinyuetai] Add test for h5/ part in _VALID_URL

---
 youtube_dl/extractor/yinyuetai.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/yinyuetai.py b/youtube_dl/extractor/yinyuetai.py
index 661c34602..41ee89da4 100644
--- a/youtube_dl/extractor/yinyuetai.py
+++ b/youtube_dl/extractor/yinyuetai.py
@@ -8,7 +8,7 @@ from ..utils import ExtractorError
 class YinYueTaiIE(InfoExtractor):
     IE_NAME = 'yinyuetai:video'
     _VALID_URL = r'https?://v\.yinyuetai\.com/video(/h5)?/(?P<id>[0-9]+)'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://v.yinyuetai.com/video/2322376',
         'md5': '6e3abe28d38e3a54b591f9f040595ce0',
         'info_dict': {
@@ -17,7 +17,10 @@ class YinYueTaiIE(InfoExtractor):
             'title': '少女时代_PARTY_Music Video Teaser',
             'creator': '少女时代',
         },
-    }
+    }, {
+        'url': 'http://v.yinyuetai.com/video/h5/2322376',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From af0f9b0e95233862e758140b282497d04edfb885 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 9 Jul 2015 00:54:37 +0800
Subject: [PATCH 1795/6219] [yinyuetai] Style

---
 youtube_dl/extractor/yinyuetai.py | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/yinyuetai.py b/youtube_dl/extractor/yinyuetai.py
index 41ee89da4..a4ada4872 100644
--- a/youtube_dl/extractor/yinyuetai.py
+++ b/youtube_dl/extractor/yinyuetai.py
@@ -32,12 +32,14 @@ class YinYueTaiIE(InfoExtractor):
         if info['error']:
             raise ExtractorError(info['errorMsg'], expected=True)
 
-        formats = [
-            {'url': format_info['videoUrl'], 'format_id': format_info['qualityLevel'],
-             'format': format_info['qualityLevelName'], 'filesize': format_info['fileSize'],
-             'ext': 'mp4', 'preference': format_info['bitrate']}
-            for format_info in info['videoUrlModels']
-        ]
+        formats = [{
+            'url': format_info['videoUrl'],
+            'format_id': format_info['qualityLevel'],
+            'format': format_info['qualityLevelName'],
+            'filesize': format_info['fileSize'],
+            'ext': 'mp4',
+            'preference': format_info['bitrate'],
+        } for format_info in info['videoUrlModels']]
         self._sort_formats(formats)
 
         return {

From d76dea001b5365b7646986cba12f9908cd321f6a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 9 Jul 2015 01:07:45 +0800
Subject: [PATCH 1796/6219] [yinyuetai] Miscellaneous improvements

1. Include all fields in _TEST
2. Use .get() for optional fields
3. Clarify the intention of 'ext' in formats
---
 youtube_dl/extractor/yinyuetai.py | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/yinyuetai.py b/youtube_dl/extractor/yinyuetai.py
index a4ada4872..003df9233 100644
--- a/youtube_dl/extractor/yinyuetai.py
+++ b/youtube_dl/extractor/yinyuetai.py
@@ -16,6 +16,8 @@ class YinYueTaiIE(InfoExtractor):
             'ext': 'mp4',
             'title': '少女时代_PARTY_Music Video Teaser',
             'creator': '少女时代',
+            'duration': 25,
+            'thumbnail': 're:^https?://.*\.jpg$',
         },
     }, {
         'url': 'http://v.yinyuetai.com/video/h5/2322376',
@@ -35,18 +37,19 @@ class YinYueTaiIE(InfoExtractor):
         formats = [{
             'url': format_info['videoUrl'],
             'format_id': format_info['qualityLevel'],
-            'format': format_info['qualityLevelName'],
-            'filesize': format_info['fileSize'],
+            'format': format_info.get('qualityLevelName'),
+            'filesize': format_info.get('fileSize'),
+            # though URLs ends with .flv, the downloaded files are in fact mp4
             'ext': 'mp4',
-            'preference': format_info['bitrate'],
+            'tbr': format_info.get('bitrate'),
         } for format_info in info['videoUrlModels']]
         self._sort_formats(formats)
 
         return {
             'id': video_id,
             'title': info['videoName'],
-            'thumbnail': info['bigHeadImage'],
-            'creator': info['artistNames'],
-            'duration': info['duration'],
+            'thumbnail': info.get('bigHeadImage'),
+            'creator': info.get('artistNames'),
+            'duration': info.get('duration'),
             'formats': formats,
         }

From 082a0140eff05dfdac1020548a49541fcddd6ec6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 8 Jul 2015 23:40:19 +0600
Subject: [PATCH 1797/6219] [yinyuetai] Do not capture unused group

---
 youtube_dl/extractor/yinyuetai.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/yinyuetai.py b/youtube_dl/extractor/yinyuetai.py
index 003df9233..fa6b40816 100644
--- a/youtube_dl/extractor/yinyuetai.py
+++ b/youtube_dl/extractor/yinyuetai.py
@@ -7,7 +7,7 @@ from ..utils import ExtractorError
 
 class YinYueTaiIE(InfoExtractor):
     IE_NAME = 'yinyuetai:video'
-    _VALID_URL = r'https?://v\.yinyuetai\.com/video(/h5)?/(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://v\.yinyuetai\.com/video(?:/h5)?/(?P<id>[0-9]+)'
     _TESTS = [{
         'url': 'http://v.yinyuetai.com/video/2322376',
         'md5': '6e3abe28d38e3a54b591f9f040595ce0',

From 77c6fb5b24c477c9de022f252dc9958a4b3b8b42 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 9 Jul 2015 20:48:38 +0600
Subject: [PATCH 1798/6219] [youtube] Make further DASH manifests not fatal
 after succeeded one

---
 youtube_dl/extractor/youtube.py | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 6769a009d..3c629d38a 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -798,7 +798,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         return self._download_webpage(url, video_id, note='Searching for annotations.', errnote='Unable to download video annotations.')
 
     def _parse_dash_manifest(
-            self, video_id, dash_manifest_url, player_url, age_gate):
+            self, video_id, dash_manifest_url, player_url, age_gate, fatal=True):
         def decrypt_sig(mobj):
             s = mobj.group(1)
             dec_s = self._decrypt_signature(s, video_id, player_url, age_gate)
@@ -807,7 +807,11 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         dash_doc = self._download_xml(
             dash_manifest_url, video_id,
             note='Downloading DASH manifest',
-            errnote='Could not download DASH manifest')
+            errnote='Could not download DASH manifest',
+            fatal=fatal)
+
+        if dash_doc is False:
+            return []
 
         formats = []
         for a in dash_doc.findall('.//{urn:mpeg:DASH:schema:MPD:2011}AdaptationSet'):
@@ -1161,14 +1165,21 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
 
         # Look for the DASH manifest
         if self._downloader.params.get('youtube_include_dash_manifest', True):
+            dash_mpd_fatal = True
             for dash_manifest_url in dash_mpds:
                 dash_formats = {}
                 try:
                     for df in self._parse_dash_manifest(
-                            video_id, dash_manifest_url, player_url, age_gate):
+                            video_id, dash_manifest_url, player_url, age_gate, dash_mpd_fatal):
                         # Do not overwrite DASH format found in some previous DASH manifest
                         if df['format_id'] not in dash_formats:
                             dash_formats[df['format_id']] = df
+                        # Additional DASH manifests may end up in HTTP Error 403 therefore
+                        # allow them to fail without bug report message if we already have
+                        # some DASH manifest succeeded. This is temporary workaround to reduce
+                        # burst of bug reports until we figure out the reason and whether it
+                        # can be fixed at all.
+                        dash_mpd_fatal = False
                 except (ExtractorError, KeyError) as e:
                     self.report_warning(
                         'Skipping DASH manifest: %r' % e, video_id)

From 675e9f22ea3f43fbade9bbd13ee5de3eb45c538f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 9 Jul 2015 23:52:03 +0600
Subject: [PATCH 1799/6219] [vimple] Extract spruto player based extractor
 class

---
 youtube_dl/extractor/vimple.py | 46 +++++++++++++++++++---------------
 1 file changed, 26 insertions(+), 20 deletions(-)

diff --git a/youtube_dl/extractor/vimple.py b/youtube_dl/extractor/vimple.py
index aa3d6ddfd..92321d66e 100644
--- a/youtube_dl/extractor/vimple.py
+++ b/youtube_dl/extractor/vimple.py
@@ -4,7 +4,29 @@ from .common import InfoExtractor
 from ..utils import int_or_none
 
 
-class VimpleIE(InfoExtractor):
+class SprutoBaseIE(InfoExtractor):
+    def _extract_spruto(self, spruto, video_id):
+        playlist = spruto['playlist'][0]
+        title = playlist['title']
+        video_id = playlist.get('videoId') or video_id
+        thumbnail = playlist.get('posterUrl') or playlist.get('thumbnailUrl')
+        duration = int_or_none(playlist.get('duration'))
+
+        formats = [{
+            'url': f['url'],
+        } for f in playlist['video']]
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'thumbnail': thumbnail,
+            'duration': duration,
+            'formats': formats,
+        }
+
+
+class VimpleIE(SprutoBaseIE):
     IE_DESC = 'Vimple - one-click video hosting'
     _VALID_URL = r'https?://(?:player\.vimple\.ru/iframe|vimple\.ru)/(?P<id>[\da-f-]{32,36})'
     _TESTS = [
@@ -30,25 +52,9 @@ class VimpleIE(InfoExtractor):
         webpage = self._download_webpage(
             'http://player.vimple.ru/iframe/%s' % video_id, video_id)
 
-        playlist = self._parse_json(
+        spruto = self._parse_json(
             self._search_regex(
                 r'sprutoData\s*:\s*({.+?}),\r\n', webpage, 'spruto data'),
-            video_id)['playlist'][0]
+            video_id)
 
-        title = playlist['title']
-        video_id = playlist.get('videoId') or video_id
-        thumbnail = playlist.get('posterUrl') or playlist.get('thumbnailUrl')
-        duration = int_or_none(playlist.get('duration'))
-
-        formats = [{
-            'url': f['url'],
-        } for f in playlist['video']]
-        self._sort_formats(formats)
-
-        return {
-            'id': video_id,
-            'title': title,
-            'thumbnail': thumbnail,
-            'duration': duration,
-            'formats': formats,
-        }
+        return self._extract_spruto(spruto, video_id)

From 1c20ddc966a69a241027c2d9a132b9caf3d0ebde Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 9 Jul 2015 23:53:50 +0600
Subject: [PATCH 1800/6219] [myvi:embed] Add extractor (Closes #6167)

---
 youtube_dl/extractor/myvi.py | 44 ++++++++++++++++++++++++++++++++++++
 1 file changed, 44 insertions(+)
 create mode 100644 youtube_dl/extractor/myvi.py

diff --git a/youtube_dl/extractor/myvi.py b/youtube_dl/extractor/myvi.py
new file mode 100644
index 000000000..9f4330f50
--- /dev/null
+++ b/youtube_dl/extractor/myvi.py
@@ -0,0 +1,44 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .vimple import SprutoBaseIE
+
+
+class MyviEmbedIE(SprutoBaseIE):
+    _VALID_URL = r'''(?x)
+                    https?://
+                        myvi\.ru/player/
+                            (?:
+                                (?:
+                                    embed/html|
+                                    api/Video/Get
+                                )/|
+                                content/preloader\.swf\?.*\bid=
+                            )
+                            (?P<id>[\da-zA-Z_]+)
+                    '''
+    _TESTS = [{
+        'url': 'http://myvi.ru/player/embed/html/oOy4euHA6LVwNNAjhD9_Jq5Ha2Qf0rtVMVFMAZav8wObeRTZaCATzucDQIDph8hQU0',
+        'md5': '571bbdfba9f9ed229dc6d34cc0f335bf',
+        'info_dict': {
+            'id': 'f16b2bbd-cde8-481c-a981-7cd48605df43',
+            'ext': 'mp4',
+            'title': 'хозяин жизни',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 25,
+        },
+    }, {
+        'url': 'http://myvi.ru/player/content/preloader.swf?id=oOy4euHA6LVwNNAjhD9_Jq5Ha2Qf0rtVMVFMAZav8wOYf1WFpPfc_bWTKGVf_Zafr0',
+        'only_matching': True,
+    }, {
+        'url': 'http://myvi.ru/player/api/Video/Get/oOy4euHA6LVwNNAjhD9_Jq5Ha2Qf0rtVMVFMAZav8wObeRTZaCATzucDQIDph8hQU0',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        spruto = self._download_json(
+            'http://myvi.ru/player/api/Video/Get/%s?sig' % video_id, video_id)['sprutoData']
+
+        return self._extract_spruto(spruto, video_id)

From 83423254ccdf60cb8756aaf7900c929c1cf1a3ef Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Jul 2015 00:07:26 +0600
Subject: [PATCH 1801/6219] [myvi:embed] Extend _VALID_URL

---
 youtube_dl/extractor/myvi.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/myvi.py b/youtube_dl/extractor/myvi.py
index 9f4330f50..a14a5365b 100644
--- a/youtube_dl/extractor/myvi.py
+++ b/youtube_dl/extractor/myvi.py
@@ -7,7 +7,7 @@ from .vimple import SprutoBaseIE
 class MyviEmbedIE(SprutoBaseIE):
     _VALID_URL = r'''(?x)
                     https?://
-                        myvi\.ru/player/
+                        myvi\.(?:ru/player|tv)/
                             (?:
                                 (?:
                                     embed/html|
@@ -33,6 +33,9 @@ class MyviEmbedIE(SprutoBaseIE):
     }, {
         'url': 'http://myvi.ru/player/api/Video/Get/oOy4euHA6LVwNNAjhD9_Jq5Ha2Qf0rtVMVFMAZav8wObeRTZaCATzucDQIDph8hQU0',
         'only_matching': True,
+    }, {
+        'url': 'http://myvi.tv/embed/html/oTGTNWdyz4Zwy_u1nraolwZ1odenTd9WkTnRfIL9y8VOgHYqOHApE575x4_xxS9Vn0?ap=0',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From e6c2d9ad29bcc4eaa0eed03d3852588b6c7a10c0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Jul 2015 00:25:36 +0600
Subject: [PATCH 1802/6219] [extractor/generic:myvi] Add support for myvi
 embeds

---
 youtube_dl/extractor/generic.py | 6 ++++++
 youtube_dl/extractor/myvi.py    | 9 +++++++++
 2 files changed, 15 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index ea60d4a96..f8d6a8c76 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -37,6 +37,7 @@ from .rutv import RUTVIE
 from .tvc import TVCIE
 from .sportbox import SportBoxEmbedIE
 from .smotri import SmotriIE
+from .myvi import MyviEmbedIE
 from .condenast import CondeNastIE
 from .udn import UDNEmbedIE
 from .senateisvp import SenateISVPIE
@@ -1425,6 +1426,11 @@ class GenericIE(InfoExtractor):
         if smotri_url:
             return self.url_result(smotri_url, 'Smotri')
 
+        # Look for embedded Myvi.ru player
+        myvi_url = MyviEmbedIE._extract_url(webpage)
+        if myvi_url:
+            return self.url_result(myvi_url)
+
         # Look for embeded soundcloud player
         mobj = re.search(
             r'<iframe\s+(?:[a-zA-Z0-9_-]+="[^"]+"\s+)*src="(?P<url>https?://(?:w\.)?soundcloud\.com/player[^"]+)"',
diff --git a/youtube_dl/extractor/myvi.py b/youtube_dl/extractor/myvi.py
index a14a5365b..896080c1e 100644
--- a/youtube_dl/extractor/myvi.py
+++ b/youtube_dl/extractor/myvi.py
@@ -1,6 +1,8 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .vimple import SprutoBaseIE
 
 
@@ -38,6 +40,13 @@ class MyviEmbedIE(SprutoBaseIE):
         'only_matching': True,
     }]
 
+    @classmethod
+    def _extract_url(cls, webpage):
+        mobj = re.search(
+            r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//myvi\.(?:ru/player|tv)/embed/html/[^"]+)\1', webpage)
+        if mobj:
+            return mobj.group('url')
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
 

From f2f89c762a13392e8876c5e9dff8b418c3912ac6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Jul 2015 00:27:02 +0600
Subject: [PATCH 1803/6219] [myvi:embed] Improve _VALID_URL

---
 youtube_dl/extractor/myvi.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/myvi.py b/youtube_dl/extractor/myvi.py
index 896080c1e..cd80cc177 100644
--- a/youtube_dl/extractor/myvi.py
+++ b/youtube_dl/extractor/myvi.py
@@ -13,11 +13,12 @@ class MyviEmbedIE(SprutoBaseIE):
                             (?:
                                 (?:
                                     embed/html|
+                                    flash|
                                     api/Video/Get
                                 )/|
                                 content/preloader\.swf\?.*\bid=
                             )
-                            (?P<id>[\da-zA-Z_]+)
+                            (?P<id>[\da-zA-Z_-]+)
                     '''
     _TESTS = [{
         'url': 'http://myvi.ru/player/embed/html/oOy4euHA6LVwNNAjhD9_Jq5Ha2Qf0rtVMVFMAZav8wObeRTZaCATzucDQIDph8hQU0',
@@ -38,12 +39,15 @@ class MyviEmbedIE(SprutoBaseIE):
     }, {
         'url': 'http://myvi.tv/embed/html/oTGTNWdyz4Zwy_u1nraolwZ1odenTd9WkTnRfIL9y8VOgHYqOHApE575x4_xxS9Vn0?ap=0',
         'only_matching': True,
+    }, {
+        'url': 'http://myvi.ru/player/flash/ocp2qZrHI-eZnHKQBK4cZV60hslH8LALnk0uBfKsB-Q4WnY26SeGoYPi8HWHxu0O30',
+        'only_matching': True,
     }]
 
     @classmethod
     def _extract_url(cls, webpage):
         mobj = re.search(
-            r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//myvi\.(?:ru/player|tv)/embed/html/[^"]+)\1', webpage)
+            r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//myvi\.(?:ru/player|tv)/(?:embed/html|flash)/[^"]+)\1', webpage)
         if mobj:
             return mobj.group('url')
 

From 6dd94d3a79353f8e694efaf2fa27f4bb40227aff Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Jul 2015 00:27:44 +0600
Subject: [PATCH 1804/6219] [myvi:embed] Rename to myvi

---
 youtube_dl/extractor/__init__.py | 1 +
 youtube_dl/extractor/generic.py  | 4 ++--
 youtube_dl/extractor/myvi.py     | 2 +-
 3 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index a0e3b333d..a64b457bd 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -323,6 +323,7 @@ from .musicvault import MusicVaultIE
 from .muzu import MuzuTVIE
 from .myspace import MySpaceIE, MySpaceAlbumIE
 from .myspass import MySpassIE
+from .myvi import MyviIE
 from .myvideo import MyVideoIE
 from .myvidster import MyVidsterIE
 from .nationalgeographic import NationalGeographicIE
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index f8d6a8c76..7c604c554 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -37,7 +37,7 @@ from .rutv import RUTVIE
 from .tvc import TVCIE
 from .sportbox import SportBoxEmbedIE
 from .smotri import SmotriIE
-from .myvi import MyviEmbedIE
+from .myvi import MyviIE
 from .condenast import CondeNastIE
 from .udn import UDNEmbedIE
 from .senateisvp import SenateISVPIE
@@ -1427,7 +1427,7 @@ class GenericIE(InfoExtractor):
             return self.url_result(smotri_url, 'Smotri')
 
         # Look for embedded Myvi.ru player
-        myvi_url = MyviEmbedIE._extract_url(webpage)
+        myvi_url = MyviIE._extract_url(webpage)
         if myvi_url:
             return self.url_result(myvi_url)
 
diff --git a/youtube_dl/extractor/myvi.py b/youtube_dl/extractor/myvi.py
index cd80cc177..4c65be122 100644
--- a/youtube_dl/extractor/myvi.py
+++ b/youtube_dl/extractor/myvi.py
@@ -6,7 +6,7 @@ import re
 from .vimple import SprutoBaseIE
 
 
-class MyviEmbedIE(SprutoBaseIE):
+class MyviIE(SprutoBaseIE):
     _VALID_URL = r'''(?x)
                     https?://
                         myvi\.(?:ru/player|tv)/

From 06a12933f3621c4b5b84346edc0d5bd570f1aac3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Jul 2015 00:58:01 +0600
Subject: [PATCH 1805/6219] [pbs] Add support for subtitles (Closes #6184)

---
 youtube_dl/extractor/pbs.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 1e2b965f9..fec5d65ad 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -224,6 +224,14 @@ class PBSIE(InfoExtractor):
             rating_str = rating_str.rpartition('-')[2]
         age_limit = US_RATINGS.get(rating_str)
 
+        subtitles = {}
+        closed_captions_url = info.get('closed_captions_url')
+        if closed_captions_url:
+            subtitles['en'] = [{
+                'ext': 'ttml',
+                'url': closed_captions_url,
+            }]
+
         return {
             'id': video_id,
             'display_id': display_id,
@@ -234,4 +242,5 @@ class PBSIE(InfoExtractor):
             'age_limit': age_limit,
             'upload_date': upload_date,
             'formats': formats,
+            'subtitles': subtitles,
         }

From bf20b9c5405b276e6de29b1b28b2e3ad2182e891 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Jul 2015 01:15:55 +0600
Subject: [PATCH 1806/6219] [extractor/generic] Add test for myvi embed

---
 youtube_dl/extractor/generic.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 7c604c554..392ad3648 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -339,6 +339,17 @@ class GenericIE(InfoExtractor):
                 'skip_download': True,
             },
         },
+        # Myvi.ru embed
+        {
+            'url': 'http://www.kinomyvi.tv/news/detail/Pervij-dublirovannij-trejler--Uzhastikov-_nOw1',
+            'info_dict': {
+                'id': 'f4dafcad-ff21-423d-89b5-146cfd89fa1e',
+                'ext': 'mp4',
+                'title': 'Ужастики, русский трейлер (2015)',
+                'thumbnail': 're:^https?://.*\.jpg$',
+                'duration': 153,
+            }
+        },
         # XHamster embed
         {
             'url': 'http://www.numisc.com/forum/showthread.php?11696-FM15-which-pumiscer-was-this-%28-vid-%29-%28-alfa-as-fuck-srx-%29&s=711f5db534502e22260dec8c5e2d66d8',

From 02b386f80abc16f5334e980c2fb729b0cc69a0d6 Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Fri, 10 Jul 2015 13:29:57 +0800
Subject: [PATCH 1807/6219] [neteasemusic] Changes after review

---
 youtube_dl/extractor/neteasemusic.py | 42 ++++++++++++++++++----------
 1 file changed, 27 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/neteasemusic.py b/youtube_dl/extractor/neteasemusic.py
index a70c65ca5..419f3ecc9 100644
--- a/youtube_dl/extractor/neteasemusic.py
+++ b/youtube_dl/extractor/neteasemusic.py
@@ -2,6 +2,7 @@
 from __future__ import unicode_literals
 
 from hashlib import md5
+from base64 import b64encode
 from datetime import datetime
 import itertools
 import re
@@ -20,14 +21,14 @@ class NetEaseMusicBaseIE(InfoExtractor):
 
     @classmethod
     def _encrypt(cls, dfsid):
-        salt_bytes = bytearray(str(cls._NETEASE_SALT))
+        salt_bytes = bytearray(cls._NETEASE_SALT, 'utf-8')
         string_bytes = bytearray(str(dfsid))
         salt_len = len(salt_bytes)
-        for i in xrange(len(string_bytes)):
+        for i in range(len(string_bytes)):
             string_bytes[i] = string_bytes[i] ^ salt_bytes[i % salt_len]
         m = md5()
         m.update(string_bytes)
-        result = m.digest().encode('base64')[:-1]
+        result = b64encode(m.digest())
         return result.replace('/', '_').replace('+', '-')
 
     @classmethod
@@ -41,12 +42,11 @@ class NetEaseMusicBaseIE(InfoExtractor):
                 'url': 'http://m1.music.126.net/%s/%s.%s' %
                        (cls._encrypt(details['dfsId']), details['dfsId'],
                         details['extension']),
-                'ext': details['extension'],
-                'abr': details['bitrate'] / 1000,
-                'preference': details['bitrate'],
+                'ext': details.get('extension'),
+                'abr': details.get('bitrate', 0) / 1000,
                 'format_id': song_format,
-                'filesize': details['size'],
-                'asr': details['sr']
+                'filesize': details.get('size'),
+                'asr': details.get('sr')
             })
         return formats
 
@@ -98,6 +98,19 @@ class NetEaseMusicIE(NetEaseMusicBaseIE):
             'upload_date': '20080211',
             'timestamp': 1202745600,
         },
+    }, {
+        'note': 'Has translated name.',
+        'url': 'http://music.163.com/#/song?id=22735043',
+        'info_dict': {
+            'id': '22735043',
+            'ext': 'mp3',
+            'title': '소원을 말해봐 (Genie)',
+            'creator': '少女时代',
+            'description': 'md5:79d99cc560e4ca97e0c4d86800ee4184',
+            'upload_date': '20100127',
+            'timestamp': 1264608000,
+            'alt_title': '说出愿望吧(Genie)',
+        }
     }]
 
     def _process_lyrics(self, lyrics_info):
@@ -109,10 +122,9 @@ class NetEaseMusicIE(NetEaseMusicBaseIE):
 
         lyrics_expr = r'(\[[0-9]{2}:[0-9]{2}\.[0-9]{2,}\])([^\n]+)'
         original_ts_texts = re.findall(lyrics_expr, original)
-        translation_ts_dict = {
-            time_stamp: text for time_stamp, text in re.findall(lyrics_expr, translated)
-        }
-
+        translation_ts_dict = dict(
+            (time_stamp, text) for time_stamp, text in re.findall(lyrics_expr, translated)
+        )
         lyrics = '\n'.join([
             '%s%s / %s' % (time_stamp, text, translation_ts_dict.get(time_stamp, ''))
             for time_stamp, text in original_ts_texts
@@ -139,8 +151,8 @@ class NetEaseMusicIE(NetEaseMusicBaseIE):
         lyrics = self._process_lyrics(lyrics_info)
 
         alt_title = None
-        if info.get('alias'):
-            alt_title = '/'.join(info.get('alias'))
+        if info.get('transNames'):
+            alt_title = '/'.join(info.get('transNames'))
 
         return {
             'id': song_id,
@@ -294,7 +306,7 @@ class NetEaseMusicMvIE(NetEaseMusicBaseIE):
             mv_id, 'Downloading mv info')['data']
 
         formats = [
-            {'url': mv_url, 'ext': 'mp4', 'format_id': '%sp' % brs, 'preference': int(brs)}
+            {'url': mv_url, 'ext': 'mp4', 'format_id': '%sp' % brs, 'height': int(brs)}
             for brs, mv_url in info['brs'].items()
         ]
         self._sort_formats(formats)

From 2b0fa1f7dd8d158b69eec4d17b254b99e976bc5c Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Fri, 10 Jul 2015 15:09:12 +0800
Subject: [PATCH 1808/6219] [kuwo] Merge KuwoSingerMusicIE into KuwoSingerIE

---
 youtube_dl/extractor/__init__.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 82cd85c44..a348b3077 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -262,7 +262,6 @@ from .kuwo import (
     KuwoAlbumIE,
     KuwoChartIE,
     KuwoSingerIE,
-    KuwoSingerMusicIE,
     KuwoCategoryIE,
     KuwoMvIE,
 )

From 1633491bff3e393d7d095b7303f954dacce4f4da Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Fri, 10 Jul 2015 15:19:07 +0800
Subject: [PATCH 1809/6219] [kuwo] Merge KuwoSingerMusicIE into KuwoSingerIE
 (missed kuwo.py)

---
 youtube_dl/extractor/kuwo.py | 53 ++++++++++++------------------------
 1 file changed, 17 insertions(+), 36 deletions(-)

diff --git a/youtube_dl/extractor/kuwo.py b/youtube_dl/extractor/kuwo.py
index 6a96a1aa4..82d5f3f95 100644
--- a/youtube_dl/extractor/kuwo.py
+++ b/youtube_dl/extractor/kuwo.py
@@ -180,15 +180,22 @@ class KuwoChartIE(InfoExtractor):
 
 class KuwoSingerIE(InfoExtractor):
     IE_NAME = 'kuwo:singer'
-    _VALID_URL = r'http://www\.kuwo\.cn/mingxing/(?P<id>[^/]+?)/$'
-    _TEST = {
+    _VALID_URL = r'http://www\.kuwo\.cn/mingxing/(?P<id>[^/]+)'
+    _TESTS = [{
         'url': 'http://www.kuwo.cn/mingxing/bruno+mars/',
         'info_dict': {
             'id': 'bruno+mars',
             'title': 'Bruno Mars',
         },
         'playlist_count': 10,
-    }
+    }, {
+        'url': 'http://www.kuwo.cn/mingxing/Ali/music.htm',
+        'info_dict': {
+            'id': 'Ali',
+            'title': 'Ali',
+        },
+        'playlist_mincount': 95,
+    }]
 
     def _real_extract(self, url):
         singer_id = self._match_id(url)
@@ -197,54 +204,28 @@ class KuwoSingerIE(InfoExtractor):
             errnote='Unable to get singer info')
 
         singer_name = self._html_search_regex(
-            r'姓名：<span>(.+?)</span>', webpage, 'singer name')
+            r'<div class="title clearfix">[\n\s\t]*?<h1>(.+?)<span', webpage, 'singer name'
+        )
 
-        entries = [
-            self.url_result("http://www.kuwo.cn/yinyue/%s/" % song_id, 'Kuwo', song_id)
-            for song_id in re.findall(
-                r'<a href="http://www\.kuwo\.cn/yinyue/([0-9]+)/" .+?>.+?</a>',
-                webpage, flags=re.DOTALL)
-        ]
-        return self.playlist_result(entries, singer_id, singer_name)
-
-
-class KuwoSingerMusicIE(InfoExtractor):
-    IE_NAME = 'kuwo:singermusic'
-    _VALID_URL = r'http://www\.kuwo\.cn/mingxing/(?P<id>[^/]+?)/music(_[0-9]+)?.htm'
-    _TEST = {
-        'url': 'http://www.kuwo.cn/mingxing/Ali/music.htm',
-        'info_dict': {
-            'id': 'Ali',
-            'title': 'Ali的热门歌曲',
-        },
-        'playlist_mincount': 95,
-    }
-
-    def _real_extract(self, url):
-        singer_id = self._match_id(url)
-
-        list_name = None
         entries = []
+        first_page_only = False if re.match(r'.+/music(?:_[0-9]+)?\.htm', url) else True
         for page_num in itertools.count(1):
             webpage = self._download_webpage(
                 'http://www.kuwo.cn/mingxing/%s/music_%d.htm' % (singer_id, page_num),
                 singer_id, note='Download song list page #%d' % page_num,
                 errnote='Unable to get song list page #%d' % page_num)
 
-            if list_name is None:
-                list_name = self._html_search_regex(
-                    r'<h1>([^<>]+)<span>', webpage, 'list name')
-
             entries.extend([
                 self.url_result("http://www.kuwo.cn/yinyue/%s/" % song_id, 'Kuwo', song_id)
                 for song_id in re.findall(
                     r'<p class="m_name"><a href="http://www\.kuwo\.cn/yinyue/([0-9]+)/',
                     webpage)
-            ])
-            if not re.search(r'<a href="[^"]+">下一页</a>', webpage):
+            ][:10 if first_page_only else None])
+
+            if first_page_only or not re.search(r'<a href="[^"]+">下一页</a>', webpage):
                 break
 
-        return self.playlist_result(entries, singer_id, list_name)
+        return self.playlist_result(entries, singer_id, singer_name)
 
 
 class KuwoCategoryIE(InfoExtractor):

From 15830339eff9076b1c4ca2cb6322fc1ec55233a8 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 10 Jul 2015 18:30:23 +0800
Subject: [PATCH 1810/6219] [neteasemusic] PEP8

---
 youtube_dl/extractor/neteasemusic.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/neteasemusic.py b/youtube_dl/extractor/neteasemusic.py
index 419f3ecc9..e73c77f89 100644
--- a/youtube_dl/extractor/neteasemusic.py
+++ b/youtube_dl/extractor/neteasemusic.py
@@ -52,7 +52,7 @@ class NetEaseMusicBaseIE(InfoExtractor):
 
     @classmethod
     def convert_milliseconds(cls, ms):
-        return int(round(ms/1000.0))
+        return int(round(ms / 1000.0))
 
     def query_api(self, endpoint, video_id, note):
         req = compat_urllib_request.Request('%s%s' % (self._API_BASE, endpoint))
@@ -146,7 +146,7 @@ class NetEaseMusicIE(NetEaseMusicBaseIE):
         self._sort_formats(formats)
 
         lyrics_info = self.query_api(
-            'song/lyric?id=%s&lv=-1&tv=-1' % song_id, 
+            'song/lyric?id=%s&lv=-1&tv=-1' % song_id,
             song_id, 'Downloading lyrics data')
         lyrics = self._process_lyrics(lyrics_info)
 
@@ -183,7 +183,7 @@ class NetEaseMusicAlbumIE(NetEaseMusicBaseIE):
         album_id = self._match_id(url)
 
         info = self.query_api(
-            'album/%s?id=%s' % (album_id, album_id), 
+            'album/%s?id=%s' % (album_id, album_id),
             album_id, 'Downloading album data')['album']
 
         name = info['name']
@@ -221,7 +221,7 @@ class NetEaseMusicSingerIE(NetEaseMusicBaseIE):
         singer_id = self._match_id(url)
 
         info = self.query_api(
-            'artist/%s?id=%s' % (singer_id, singer_id), 
+            'artist/%s?id=%s' % (singer_id, singer_id),
             singer_id, 'Downloading singer data')
 
         name = info['artist']['name']
@@ -264,7 +264,7 @@ class NetEaseMusicListIE(NetEaseMusicBaseIE):
         list_id = self._match_id(url)
 
         info = self.query_api(
-            'playlist/detail?id=%s&lv=-1&tv=-1' % list_id, 
+            'playlist/detail?id=%s&lv=-1&tv=-1' % list_id,
             list_id, 'Downloading playlist data')['result']
 
         name = info['name']
@@ -378,7 +378,7 @@ class NetEaseMusicProgramIE(NetEaseMusicBaseIE):
                 self.to_screen(
                     'Downloading just the main audio %s because of --no-playlist'
                     % info['mainSong']['id'])
-            
+
             formats = self.extract_formats(info['mainSong'])
             self._sort_formats(formats)
 
@@ -429,7 +429,7 @@ class NetEaseMusicDjRadioIE(NetEaseMusicBaseIE):
         entries = []
         for offset in itertools.count(start=0, step=self._PAGE_SIZE):
             info = self.query_api(
-                'dj/program/byradio?asc=false&limit=%d&radioId=%s&offset=%d' 
+                'dj/program/byradio?asc=false&limit=%d&radioId=%s&offset=%d'
                 % (self._PAGE_SIZE, dj_id, offset),
                 dj_id, 'Downloading dj programs - %d' % offset)
 

From 397a8ea96e228f918f127313f837f7de7dd78aed Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 10 Jul 2015 18:43:38 +0800
Subject: [PATCH 1811/6219] [neteasemusic] Encoding fixes for Python 2.6 and
 3.x

---
 youtube_dl/extractor/neteasemusic.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/neteasemusic.py b/youtube_dl/extractor/neteasemusic.py
index e73c77f89..9e99ddbda 100644
--- a/youtube_dl/extractor/neteasemusic.py
+++ b/youtube_dl/extractor/neteasemusic.py
@@ -11,6 +11,7 @@ from .common import InfoExtractor
 from ..compat import (
     compat_urllib_request,
     compat_urllib_parse,
+    compat_str,
 )
 
 
@@ -21,14 +22,14 @@ class NetEaseMusicBaseIE(InfoExtractor):
 
     @classmethod
     def _encrypt(cls, dfsid):
-        salt_bytes = bytearray(cls._NETEASE_SALT, 'utf-8')
-        string_bytes = bytearray(str(dfsid))
+        salt_bytes = bytearray(cls._NETEASE_SALT.encode('utf-8'))
+        string_bytes = bytearray(compat_str(dfsid).encode('ascii'))
         salt_len = len(salt_bytes)
         for i in range(len(string_bytes)):
             string_bytes[i] = string_bytes[i] ^ salt_bytes[i % salt_len]
         m = md5()
-        m.update(string_bytes)
-        result = b64encode(m.digest())
+        m.update(bytes(string_bytes))
+        result = b64encode(m.digest()).decode('ascii')
         return result.replace('/', '_').replace('+', '-')
 
     @classmethod

From a0e060ac1e9871974c02fe7cb3e812a3cbe79e73 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 10 Jul 2015 18:58:12 +0800
Subject: [PATCH 1812/6219] [compat] Add compat_itertools_count

'step' parameter is added in Python 2.7
---
 youtube_dl/compat.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index f9529210d..c3783337a 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -9,6 +9,7 @@ import shutil
 import socket
 import subprocess
 import sys
+import itertools
 
 
 try:
@@ -388,6 +389,15 @@ else:
             pass
         return _terminal_size(columns, lines)
 
+try:
+    itertools.count(start=0, step=1)
+    compat_itertools_count = itertools.count
+except TypeError:  # Python 2.6
+    def compat_itertools_count(start=0, step=1):
+        n = start
+        while True:
+            yield n
+            n += step
 
 __all__ = [
     'compat_HTTPError',
@@ -401,6 +411,7 @@ __all__ = [
     'compat_html_entities',
     'compat_http_client',
     'compat_http_server',
+    'compat_itertools_count',
     'compat_kwargs',
     'compat_ord',
     'compat_parse_qs',

From 4eab60cbd27ea8ce31ae9c6bf6b3f6e0e469d3af Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 10 Jul 2015 18:59:12 +0800
Subject: [PATCH 1813/6219] [netease:djradio] Use compat_itertools_count

---
 youtube_dl/extractor/neteasemusic.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/neteasemusic.py b/youtube_dl/extractor/neteasemusic.py
index 9e99ddbda..bdfe7e63f 100644
--- a/youtube_dl/extractor/neteasemusic.py
+++ b/youtube_dl/extractor/neteasemusic.py
@@ -4,7 +4,6 @@ from __future__ import unicode_literals
 from hashlib import md5
 from base64 import b64encode
 from datetime import datetime
-import itertools
 import re
 
 from .common import InfoExtractor
@@ -12,6 +11,7 @@ from ..compat import (
     compat_urllib_request,
     compat_urllib_parse,
     compat_str,
+    compat_itertools_count,
 )
 
 
@@ -428,7 +428,7 @@ class NetEaseMusicDjRadioIE(NetEaseMusicBaseIE):
         name = None
         desc = None
         entries = []
-        for offset in itertools.count(start=0, step=self._PAGE_SIZE):
+        for offset in compat_itertools_count(start=0, step=self._PAGE_SIZE):
             info = self.query_api(
                 'dj/program/byradio?asc=false&limit=%d&radioId=%s&offset=%d'
                 % (self._PAGE_SIZE, dj_id, offset),

From a34af8d0667d8f4ceba3380f808a6d563ca01d77 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 10 Jul 2015 19:13:52 +0800
Subject: [PATCH 1814/6219] [kuwo] PEP8

---
 youtube_dl/extractor/kuwo.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/kuwo.py b/youtube_dl/extractor/kuwo.py
index 82d5f3f95..9c62191b5 100644
--- a/youtube_dl/extractor/kuwo.py
+++ b/youtube_dl/extractor/kuwo.py
@@ -249,10 +249,10 @@ class KuwoCategoryIE(InfoExtractor):
 
         category_name = self._html_search_regex(
             r'<h1 title="([^<>]+?)">[^<>]+?</h1>', webpage, 'category name')
-        
+
         category_desc = re.sub(
             r'^.+简介：', '', get_element_by_id("intro", webpage).strip())
-        
+
         jsonm = self._parse_json(self._html_search_regex(
             r'var jsonm = (\{.+?\});', webpage, 'category songs'), category_id)
 

From 9296e92e1cbb5788c18c9a09590ccbc0e3faec68 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Jul 2015 20:55:53 +0600
Subject: [PATCH 1815/6219] [twitch] Fix login (Closes #6186)

---
 youtube_dl/extractor/twitch.py | 20 +++++++-------------
 1 file changed, 7 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index b56ee2959..b7a72a7bd 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -22,8 +22,8 @@ class TwitchBaseIE(InfoExtractor):
 
     _API_BASE = 'https://api.twitch.tv'
     _USHER_BASE = 'http://usher.twitch.tv'
-    _LOGIN_URL = 'https://secure.twitch.tv/user/login'
-    _LOGIN_POST_URL = 'https://secure-login.twitch.tv/login'
+    _LOGIN_URL = 'https://secure.twitch.tv/login'
+    _LOGIN_POST_URL = 'https://passport.twitch.tv/authorize'
     _NETRC_MACHINE = 'twitch'
 
     def _handle_error(self, response):
@@ -59,20 +59,14 @@ class TwitchBaseIE(InfoExtractor):
         login_page = self._download_webpage(
             self._LOGIN_URL, None, 'Downloading login page')
 
-        authenticity_token = self._search_regex(
-            r'<input name="authenticity_token" type="hidden" value="([^"]+)"',
-            login_page, 'authenticity token')
+        login_form = dict(re.findall(
+            r'<input\s+type="hidden"\s+name="([^"]+)"\s+(?:id="[^"]+"\s+)?value="([^"]*)"',
+            login_page))
 
-        login_form = {
-            'utf8': '✓'.encode('utf-8'),
-            'authenticity_token': authenticity_token,
-            'redirect_on_login': '',
-            'embed_form': 'false',
-            'mp_source_action': 'login-button',
-            'follow': '',
+        login_form.update({
             'login': username,
             'password': password,
-        }
+        })
 
         request = compat_urllib_request.Request(
             self._LOGIN_POST_URL, compat_urllib_parse.urlencode(login_form).encode('utf-8'))

From 89a683ae745de77f96ac8e82c8ff16d2675fa398 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Jul 2015 21:15:09 +0600
Subject: [PATCH 1816/6219] [twitch] Fix error message regex

---
 youtube_dl/extractor/twitch.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index b7a72a7bd..f912d3825 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -74,11 +74,12 @@ class TwitchBaseIE(InfoExtractor):
         response = self._download_webpage(
             request, None, 'Logging in as %s' % username)
 
-        m = re.search(
-            r"id=([\"'])login_error_message\1[^>]*>(?P<msg>[^<]+)", response)
-        if m:
+        error_message = self._search_regex(
+            r'<div[^>]+class="subwindow_notice"[^>]*>([^<]+)</div>',
+            response, 'error message', default=None)
+        if error_message:
             raise ExtractorError(
-                'Unable to login: %s' % m.group('msg').strip(), expected=True)
+                'Unable to login. Twitch said: %s' % error_message, expected=True)
 
     def _prefer_source(self, formats):
         try:

From 17b41a3337b5326fd2c8dec8aa1ab285225e1daa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Jul 2015 21:16:42 +0600
Subject: [PATCH 1817/6219] [twitch] Show reset password request

---
 youtube_dl/extractor/twitch.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index f912d3825..df809ecfe 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -81,6 +81,9 @@ class TwitchBaseIE(InfoExtractor):
             raise ExtractorError(
                 'Unable to login. Twitch said: %s' % error_message, expected=True)
 
+        if '>Reset your password<' in response:
+            self.report_warning('Twitch asks you to reset your password, go to https://secure.twitch.tv/reset/submit')
+
     def _prefer_source(self, formats):
         try:
             source = next(f for f in formats if f['format_id'] == 'Source')

From a31e3e7dcb9d0471d90ec8562934a144d25d7132 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 10 Jul 2015 23:23:07 +0800
Subject: [PATCH 1818/6219] [kuwo] Regular expression improvements

1. Prevent .+ and .*
2. Use [^>]+ instead of spaces for HTML tags
3. Remove unnecessary trailing parts
---
 youtube_dl/extractor/kuwo.py         | 47 +++++++++++++++-------------
 youtube_dl/extractor/neteasemusic.py |  2 +-
 2 files changed, 26 insertions(+), 23 deletions(-)

diff --git a/youtube_dl/extractor/kuwo.py b/youtube_dl/extractor/kuwo.py
index 9c62191b5..1095a26e2 100644
--- a/youtube_dl/extractor/kuwo.py
+++ b/youtube_dl/extractor/kuwo.py
@@ -9,6 +9,7 @@ from ..utils import (
     get_element_by_id,
     clean_html,
     ExtractorError,
+    remove_start,
 )
 
 
@@ -73,10 +74,10 @@ class KuwoIE(InfoExtractor):
             errnote='Unable to get song detail info')
 
         song_name = self._html_search_regex(
-            r'<h1 title="(.+?)">', webpage, 'song name')
+            r'<h1[^>]+title="([^"]+)">', webpage, 'song name')
         singer_name = self._html_search_regex(
-            r'<div class="s_img">.+?title="(.+?)".+?</div>', webpage, 'singer name',
-            flags=re.DOTALL, default=None)
+            r'<div[^>]+class="s_img">\s*<a[^>]+title="([^>]+)"',
+            webpage, 'singer name', default=None)
         lrc_content = clean_html(get_element_by_id("lrcContent", webpage))
         if lrc_content == '暂无':     # indicates no lyrics
             lrc_content = None
@@ -84,7 +85,7 @@ class KuwoIE(InfoExtractor):
         formats = self._get_formats(song_id)
 
         album_id = self._html_search_regex(
-            r'<p class="album" title=".+?">.+?<a href="http://www\.kuwo\.cn/album/([0-9]+)/" ',
+            r'<p[^>]+class="album"[^<]+<a[^>]+href="http://www\.kuwo\.cn/album/(\d+)/"',
             webpage, 'album id', default=None, fatal=False)
 
         publish_time = None
@@ -131,15 +132,16 @@ class KuwoAlbumIE(InfoExtractor):
             errnote='Unable to get album info')
 
         album_name = self._html_search_regex(
-            r'<div class="comm".+?<h1 title="(.+?)">.+?</h1>', webpage,
-            'album name', flags=re.DOTALL)
-        album_intro = clean_html(
-            re.sub(r'^.+简介：', '', get_element_by_id("intro", webpage).strip()))
+            r'<div[^>]+class="comm"[^<]+<h1[^>]+title="([^"]+)"', webpage,
+            'album name')
+        album_intro = remove_start(
+            clean_html(get_element_by_id("intro", webpage)),
+            '%s简介：' % album_name)
 
         entries = [
             self.url_result("http://www.kuwo.cn/yinyue/%s/" % song_id, 'Kuwo', song_id)
             for song_id in re.findall(
-                r'<p class="listen"><a href="http://www\.kuwo\.cn/yinyue/([0-9]+)/" target="_blank" title="试听.*?"></a></p>',
+                r'<p[^>]+class="listen"><a[^>]+href="http://www\.kuwo\.cn/yinyue/(\d+)/"',
                 webpage)
         ]
         return self.playlist_result(entries, album_id, album_name, album_intro)
@@ -147,7 +149,7 @@ class KuwoAlbumIE(InfoExtractor):
 
 class KuwoChartIE(InfoExtractor):
     IE_NAME = 'kuwo:chart'
-    _VALID_URL = r'http://yinyue\.kuwo\.cn/billboard_(?P<id>.+?).htm'
+    _VALID_URL = r'http://yinyue\.kuwo\.cn/billboard_(?P<id>[^.]+).htm'
     _TEST = {
         'url': 'http://yinyue.kuwo.cn/billboard_香港中文龙虎榜.htm',
         'info_dict': {
@@ -165,15 +167,15 @@ class KuwoChartIE(InfoExtractor):
             errnote='Unable to get chart info')
 
         chart_name = self._html_search_regex(
-            r'<h1 class="unDis">(.+?)</h1>', webpage, 'chart name')
+            r'<h1[^>]+class="unDis">([^<]+)</h1>', webpage, 'chart name')
 
         chart_desc = self._html_search_regex(
-            r'<p class="tabDef">([0-9]{4}第[0-9]{2}期)</p>', webpage, 'chart desc')
+            r'<p[^>]+class="tabDef">(\d{4}第\d{2}期)</p>', webpage, 'chart desc')
 
         entries = [
             self.url_result("http://www.kuwo.cn/yinyue/%s/" % song_id, 'Kuwo', song_id)
             for song_id in re.findall(
-                r'<a href="http://www\.kuwo\.cn/yinyue/([0-9]+)/" .+?>.+?</a>', webpage)
+                r'<a[^>]+href="http://www\.kuwo\.cn/yinyue/(\d+)/"', webpage)
         ]
         return self.playlist_result(entries, chart_id, chart_name, chart_desc)
 
@@ -204,11 +206,11 @@ class KuwoSingerIE(InfoExtractor):
             errnote='Unable to get singer info')
 
         singer_name = self._html_search_regex(
-            r'<div class="title clearfix">[\n\s\t]*?<h1>(.+?)<span', webpage, 'singer name'
+            r'<div class="title clearfix">\s*<h1>([^<]+)<span', webpage, 'singer name'
         )
 
         entries = []
-        first_page_only = False if re.match(r'.+/music(?:_[0-9]+)?\.htm', url) else True
+        first_page_only = False if re.search(r'/music(?:_[0-9]+)?\.htm', url) else True
         for page_num in itertools.count(1):
             webpage = self._download_webpage(
                 'http://www.kuwo.cn/mingxing/%s/music_%d.htm' % (singer_id, page_num),
@@ -218,11 +220,11 @@ class KuwoSingerIE(InfoExtractor):
             entries.extend([
                 self.url_result("http://www.kuwo.cn/yinyue/%s/" % song_id, 'Kuwo', song_id)
                 for song_id in re.findall(
-                    r'<p class="m_name"><a href="http://www\.kuwo\.cn/yinyue/([0-9]+)/',
+                    r'<p[^>]+class="m_name"><a[^>]+href="http://www\.kuwo\.cn/yinyue/([0-9]+)/',
                     webpage)
             ][:10 if first_page_only else None])
 
-            if first_page_only or not re.search(r'<a href="[^"]+">下一页</a>', webpage):
+            if first_page_only or not re.search(r'<a[^>]+href="[^"]+">下一页</a>', webpage):
                 break
 
         return self.playlist_result(entries, singer_id, singer_name)
@@ -248,13 +250,14 @@ class KuwoCategoryIE(InfoExtractor):
             errnote='Unable to get category info')
 
         category_name = self._html_search_regex(
-            r'<h1 title="([^<>]+?)">[^<>]+?</h1>', webpage, 'category name')
+            r'<h1[^>]+title="([^<>]+?)">[^<>]+?</h1>', webpage, 'category name')
 
-        category_desc = re.sub(
-            r'^.+简介：', '', get_element_by_id("intro", webpage).strip())
+        category_desc = remove_start(
+            get_element_by_id("intro", webpage).strip(),
+            '%s简介：' % category_name)
 
         jsonm = self._parse_json(self._html_search_regex(
-            r'var jsonm = (\{.+?\});', webpage, 'category songs'), category_id)
+            r'var\s+jsonm\s*=\s*([^;]+);', webpage, 'category songs'), category_id)
 
         entries = [
             self.url_result(
@@ -289,7 +292,7 @@ class KuwoMvIE(KuwoIE):
             errnote='Unable to get mv detail info: %s' % song_id)
 
         mobj = re.search(
-            r'<h1 title="(?P<song>.+?)">[^<>]+<span .+?title="(?P<singer>.+?)".+?>[^<>]+</span></h1>',
+            r'<h1[^>]+title="(?P<song>[^"]+)">[^<]+<span[^>]+title="(?P<singer>[^"]+)"',
             webpage)
         if mobj:
             song_name = mobj.group('song')
diff --git a/youtube_dl/extractor/neteasemusic.py b/youtube_dl/extractor/neteasemusic.py
index bdfe7e63f..ee52efaee 100644
--- a/youtube_dl/extractor/neteasemusic.py
+++ b/youtube_dl/extractor/neteasemusic.py
@@ -229,7 +229,7 @@ class NetEaseMusicSingerIE(NetEaseMusicBaseIE):
         if info['artist']['trans']:
             name = '%s - %s' % (name, info['artist']['trans'])
         if info['artist']['alias']:
-            name = '%s - %s' % (name, ";".join(info['artist']['alias']))
+            name = '%s - %s' % (name, ';'.join(info['artist']['alias']))
 
         entries = [
             self.url_result('http://music.163.com/#/song?id=%s' % song['id'],

From c0bf5e1c4dc0a236402c946b0b1fd358f9e39a74 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Jul 2015 21:40:21 +0600
Subject: [PATCH 1819/6219] [twitch] Fix non-ASCII logins/passwords on python 2

---
 youtube_dl/extractor/twitch.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index df809ecfe..49535cd80 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -64,8 +64,8 @@ class TwitchBaseIE(InfoExtractor):
             login_page))
 
         login_form.update({
-            'login': username,
-            'password': password,
+            'login': username.encode('utf-8'),
+            'password': password.encode('utf-8'),
         })
 
         request = compat_urllib_request.Request(

From a9684c0dbf3879478fd223ce7594d58be7dffa4f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 10 Jul 2015 23:46:44 +0800
Subject: [PATCH 1820/6219] [kuwo] Add KuwoBaseIE

---
 youtube_dl/extractor/kuwo.py | 63 +++++++++++++++++++-----------------
 1 file changed, 33 insertions(+), 30 deletions(-)

diff --git a/youtube_dl/extractor/kuwo.py b/youtube_dl/extractor/kuwo.py
index 1095a26e2..928f7f62d 100644
--- a/youtube_dl/extractor/kuwo.py
+++ b/youtube_dl/extractor/kuwo.py
@@ -13,32 +13,7 @@ from ..utils import (
 )
 
 
-class KuwoIE(InfoExtractor):
-    IE_NAME = 'kuwo:song'
-    _VALID_URL = r'http://www\.kuwo\.cn/yinyue/(?P<id>[0-9]+?)/'
-    _TESTS = [{
-        'url': 'http://www.kuwo.cn/yinyue/635632/',
-        'info_dict': {
-            'id': '635632',
-            'ext': 'ape',
-            'title': '爱我别走',
-            'creator': '张震岳',
-            'upload_date': '20080122',
-            'description': 'md5:ed13f58e3c3bf3f7fd9fbc4e5a7aa75c'
-        },
-    }, {
-        'url': 'http://www.kuwo.cn/yinyue/6446136/',
-        'info_dict': {
-            'id': '6446136',
-            'ext': 'mp3',
-            'title': '心',
-            'creator': 'IU',
-            'upload_date': '20150518',
-        },
-        'params': {
-            'format': 'mp3-320'
-        },
-    }]
+class KuwoBaseIE(InfoExtractor):
     _FORMATS = [
         {'format': 'ape', 'ext': 'ape', 'preference': 100},
         {'format': 'mp3-320', 'ext': 'mp3', 'br': '320kmp3', 'abr': 320, 'preference': 80},
@@ -67,6 +42,34 @@ class KuwoIE(InfoExtractor):
         self._sort_formats(formats)
         return formats
 
+
+class KuwoIE(KuwoBaseIE):
+    IE_NAME = 'kuwo:song'
+    _VALID_URL = r'http://www\.kuwo\.cn/yinyue/(?P<id>[0-9]+?)/'
+    _TESTS = [{
+        'url': 'http://www.kuwo.cn/yinyue/635632/',
+        'info_dict': {
+            'id': '635632',
+            'ext': 'ape',
+            'title': '爱我别走',
+            'creator': '张震岳',
+            'upload_date': '20080122',
+            'description': 'md5:ed13f58e3c3bf3f7fd9fbc4e5a7aa75c'
+        },
+    }, {
+        'url': 'http://www.kuwo.cn/yinyue/6446136/',
+        'info_dict': {
+            'id': '6446136',
+            'ext': 'mp3',
+            'title': '心',
+            'creator': 'IU',
+            'upload_date': '20150518',
+        },
+        'params': {
+            'format': 'mp3-320'
+        },
+    }]
+
     def _real_extract(self, url):
         song_id = self._match_id(url)
         webpage = self._download_webpage(
@@ -268,10 +271,10 @@ class KuwoCategoryIE(InfoExtractor):
         return self.playlist_result(entries, category_id, category_name, category_desc)
 
 
-class KuwoMvIE(KuwoIE):
+class KuwoMvIE(KuwoBaseIE):
     IE_NAME = 'kuwo:mv'
     _VALID_URL = r'http://www\.kuwo\.cn/mv/(?P<id>[0-9]+?)/'
-    _TESTS = [{
+    _TEST = {
         'url': 'http://www.kuwo.cn/mv/6480076/',
         'info_dict': {
             'id': '6480076',
@@ -279,8 +282,8 @@ class KuwoMvIE(KuwoIE):
             'title': '我们家MV',
             'creator': '2PM',
         },
-    }]
-    _FORMATS = KuwoIE._FORMATS + [
+    }
+    _FORMATS = KuwoBaseIE._FORMATS + [
         {'format': 'mkv', 'ext': 'mkv', 'preference': 250},
         {'format': 'mp4', 'ext': 'mp4', 'preference': 200},
     ]

From cf2c5fda4f3e753cc64098e6a751cf1a220efae7 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 10 Jul 2015 23:48:48 +0800
Subject: [PATCH 1821/6219] [kuwo] Use single quotes

---
 youtube_dl/extractor/kuwo.py | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/kuwo.py b/youtube_dl/extractor/kuwo.py
index 928f7f62d..2b5321cc2 100644
--- a/youtube_dl/extractor/kuwo.py
+++ b/youtube_dl/extractor/kuwo.py
@@ -27,9 +27,9 @@ class KuwoBaseIE(InfoExtractor):
         formats = []
         for file_format in self._FORMATS:
             song_url = self._download_webpage(
-                "http://antiserver.kuwo.cn/anti.s?format=%s&br=%s&rid=MUSIC_%s&type=convert_url&response=url" %
+                'http://antiserver.kuwo.cn/anti.s?format=%s&br=%s&rid=MUSIC_%s&type=convert_url&response=url' %
                 (file_format['ext'], file_format.get('br', ''), song_id),
-                song_id, note="Download %s url info" % file_format["format"],
+                song_id, note='Download %s url info' % file_format['format'],
             )
             if song_url.startswith('http://') or song_url.startswith('https://'):
                 formats.append({
@@ -81,7 +81,7 @@ class KuwoIE(KuwoBaseIE):
         singer_name = self._html_search_regex(
             r'<div[^>]+class="s_img">\s*<a[^>]+title="([^>]+)"',
             webpage, 'singer name', default=None)
-        lrc_content = clean_html(get_element_by_id("lrcContent", webpage))
+        lrc_content = clean_html(get_element_by_id('lrcContent', webpage))
         if lrc_content == '暂无':     # indicates no lyrics
             lrc_content = None
 
@@ -94,7 +94,7 @@ class KuwoIE(KuwoBaseIE):
         publish_time = None
         if album_id is not None:
             album_info_page = self._download_webpage(
-                "http://www.kuwo.cn/album/%s/" % album_id, song_id,
+                'http://www.kuwo.cn/album/%s/' % album_id, song_id,
                 note='Download album detail info',
                 errnote='Unable to get album detail info')
 
@@ -138,11 +138,11 @@ class KuwoAlbumIE(InfoExtractor):
             r'<div[^>]+class="comm"[^<]+<h1[^>]+title="([^"]+)"', webpage,
             'album name')
         album_intro = remove_start(
-            clean_html(get_element_by_id("intro", webpage)),
+            clean_html(get_element_by_id('intro', webpage)),
             '%s简介：' % album_name)
 
         entries = [
-            self.url_result("http://www.kuwo.cn/yinyue/%s/" % song_id, 'Kuwo', song_id)
+            self.url_result('http://www.kuwo.cn/yinyue/%s/' % song_id, 'Kuwo', song_id)
             for song_id in re.findall(
                 r'<p[^>]+class="listen"><a[^>]+href="http://www\.kuwo\.cn/yinyue/(\d+)/"',
                 webpage)
@@ -176,7 +176,7 @@ class KuwoChartIE(InfoExtractor):
             r'<p[^>]+class="tabDef">(\d{4}第\d{2}期)</p>', webpage, 'chart desc')
 
         entries = [
-            self.url_result("http://www.kuwo.cn/yinyue/%s/" % song_id, 'Kuwo', song_id)
+            self.url_result('http://www.kuwo.cn/yinyue/%s/' % song_id, 'Kuwo', song_id)
             for song_id in re.findall(
                 r'<a[^>]+href="http://www\.kuwo\.cn/yinyue/(\d+)/"', webpage)
         ]
@@ -221,7 +221,7 @@ class KuwoSingerIE(InfoExtractor):
                 errnote='Unable to get song list page #%d' % page_num)
 
             entries.extend([
-                self.url_result("http://www.kuwo.cn/yinyue/%s/" % song_id, 'Kuwo', song_id)
+                self.url_result('http://www.kuwo.cn/yinyue/%s/' % song_id, 'Kuwo', song_id)
                 for song_id in re.findall(
                     r'<p[^>]+class="m_name"><a[^>]+href="http://www\.kuwo\.cn/yinyue/([0-9]+)/',
                     webpage)
@@ -256,7 +256,7 @@ class KuwoCategoryIE(InfoExtractor):
             r'<h1[^>]+title="([^<>]+?)">[^<>]+?</h1>', webpage, 'category name')
 
         category_desc = remove_start(
-            get_element_by_id("intro", webpage).strip(),
+            get_element_by_id('intro', webpage).strip(),
             '%s简介：' % category_name)
 
         jsonm = self._parse_json(self._html_search_regex(
@@ -264,7 +264,7 @@ class KuwoCategoryIE(InfoExtractor):
 
         entries = [
             self.url_result(
-                "http://www.kuwo.cn/yinyue/%s/" % song['musicrid'],
+                'http://www.kuwo.cn/yinyue/%s/' % song['musicrid'],
                 'Kuwo', song['musicrid'])
             for song in jsonm['musiclist']
         ]
@@ -301,7 +301,7 @@ class KuwoMvIE(KuwoBaseIE):
             song_name = mobj.group('song')
             singer_name = mobj.group('singer')
         else:
-            raise ExtractorError("Unable to find song or singer names")
+            raise ExtractorError('Unable to find song or singer names')
 
         formats = self._get_formats(song_id)
 

From 27713812a093ddfb329cd9cead878f3d0bf629cf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Jul 2015 21:49:09 +0600
Subject: [PATCH 1822/6219] [extractor/common] Add method for extracting form
 hidden input fields as dict

---
 youtube_dl/extractor/common.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index d859aea52..82f5de2d8 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -705,6 +705,12 @@ class InfoExtractor(object):
         return self._html_search_meta('twitter:player', html,
                                       'twitter card player')
 
+    @staticmethod
+    def _form_hidden_inputs(html):
+        return dict(re.findall(
+            r'<input\s+type="hidden"\s+name="([^"]+)"\s+(?:id="[^"]+"\s+)?value="([^"]*)"',
+            html))
+
     def _sort_formats(self, formats, field_preference=None):
         if not formats:
             raise ExtractorError('No video formats found')

From a0aab26a414e8424e852a361c1822739f6bc936c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Jul 2015 21:49:39 +0600
Subject: [PATCH 1823/6219] [twitch] Use `_form_hidden_inputs` when logging in

---
 youtube_dl/extractor/twitch.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 49535cd80..af2b798fb 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -59,9 +59,7 @@ class TwitchBaseIE(InfoExtractor):
         login_page = self._download_webpage(
             self._LOGIN_URL, None, 'Downloading login page')
 
-        login_form = dict(re.findall(
-            r'<input\s+type="hidden"\s+name="([^"]+)"\s+(?:id="[^"]+"\s+)?value="([^"]*)"',
-            login_page))
+        login_form = self._form_hidden_inputs(login_page)
 
         login_form.update({
             'login': username.encode('utf-8'),

From a5839317aa5b41e7bc99ecae7405459bbcbff8a7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Jul 2015 21:52:03 +0600
Subject: [PATCH 1824/6219] [vk] Use `_form_hidden_inputs` when logging in

---
 youtube_dl/extractor/vk.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 23d153031..c0292095b 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -154,9 +154,7 @@ class VKIE(InfoExtractor):
         login_page = self._download_webpage(
             'https://vk.com', None, 'Downloading login page')
 
-        login_form = dict(re.findall(
-            r'<input\s+type="hidden"\s+name="([^"]+)"\s+(?:id="[^"]+"\s+)?value="([^"]*)"',
-            login_page))
+        login_form = self._form_hidden_inputs(login_page)
 
         login_form.update({
             'email': username.encode('cp1251'),

From 2688176c776a2d80c6a4ba85f6e2d07dfd8dba24 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Jul 2015 21:53:38 +0600
Subject: [PATCH 1825/6219] [vodlocker] Use `_form_hidden_inputs`

---
 youtube_dl/extractor/vodlocker.py | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/vodlocker.py b/youtube_dl/extractor/vodlocker.py
index 1c0966a79..431f4e2e3 100644
--- a/youtube_dl/extractor/vodlocker.py
+++ b/youtube_dl/extractor/vodlocker.py
@@ -28,12 +28,7 @@ class VodlockerIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        fields = dict(re.findall(r'''(?x)<input\s+
-            type="hidden"\s+
-            name="([^"]+)"\s+
-            (?:id="[^"]+"\s+)?
-            value="([^"]*)"
-            ''', webpage))
+        fields = self._form_hidden_inputs(webpage)
 
         if fields['op'] == 'download1':
             self._sleep(3, video_id)  # they do detect when requests happen too fast!

From d3b8908886a78dd63441754f1ac20b70cb29de56 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 10 Jul 2015 23:53:48 +0800
Subject: [PATCH 1826/6219] [kuwo] Simpler calls to url_result()

---
 youtube_dl/extractor/kuwo.py | 19 +++++++------------
 1 file changed, 7 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/kuwo.py b/youtube_dl/extractor/kuwo.py
index 2b5321cc2..69afacac9 100644
--- a/youtube_dl/extractor/kuwo.py
+++ b/youtube_dl/extractor/kuwo.py
@@ -142,9 +142,8 @@ class KuwoAlbumIE(InfoExtractor):
             '%s简介：' % album_name)
 
         entries = [
-            self.url_result('http://www.kuwo.cn/yinyue/%s/' % song_id, 'Kuwo', song_id)
-            for song_id in re.findall(
-                r'<p[^>]+class="listen"><a[^>]+href="http://www\.kuwo\.cn/yinyue/(\d+)/"',
+            self.url_result(song_url, 'Kuwo') for song_url in re.findall(
+                r'<p[^>]+class="listen"><a[^>]+href="(http://www\.kuwo\.cn/yinyue/\d+/)"',
                 webpage)
         ]
         return self.playlist_result(entries, album_id, album_name, album_intro)
@@ -176,9 +175,8 @@ class KuwoChartIE(InfoExtractor):
             r'<p[^>]+class="tabDef">(\d{4}第\d{2}期)</p>', webpage, 'chart desc')
 
         entries = [
-            self.url_result('http://www.kuwo.cn/yinyue/%s/' % song_id, 'Kuwo', song_id)
-            for song_id in re.findall(
-                r'<a[^>]+href="http://www\.kuwo\.cn/yinyue/(\d+)/"', webpage)
+            self.url_result(song_url, 'Kuwo') for song_url in re.findall(
+                r'<a[^>]+href="(http://www\.kuwo\.cn/yinyue/\d+)/"', webpage)
         ]
         return self.playlist_result(entries, chart_id, chart_name, chart_desc)
 
@@ -221,9 +219,8 @@ class KuwoSingerIE(InfoExtractor):
                 errnote='Unable to get song list page #%d' % page_num)
 
             entries.extend([
-                self.url_result('http://www.kuwo.cn/yinyue/%s/' % song_id, 'Kuwo', song_id)
-                for song_id in re.findall(
-                    r'<p[^>]+class="m_name"><a[^>]+href="http://www\.kuwo\.cn/yinyue/([0-9]+)/',
+                self.url_result(song_url, 'Kuwo') for song_url in re.findall(
+                    r'<p[^>]+class="m_name"><a[^>]+href="(http://www\.kuwo\.cn/yinyue/\d+)/',
                     webpage)
             ][:10 if first_page_only else None])
 
@@ -263,9 +260,7 @@ class KuwoCategoryIE(InfoExtractor):
             r'var\s+jsonm\s*=\s*([^;]+);', webpage, 'category songs'), category_id)
 
         entries = [
-            self.url_result(
-                'http://www.kuwo.cn/yinyue/%s/' % song['musicrid'],
-                'Kuwo', song['musicrid'])
+            self.url_result('http://www.kuwo.cn/yinyue/%s/' % song['musicrid'], 'Kuwo')
             for song in jsonm['musiclist']
         ]
         return self.playlist_result(entries, category_id, category_name, category_desc)

From de195c23a6837d1526198320561feba74886b9df Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Jul 2015 21:54:39 +0600
Subject: [PATCH 1827/6219] [vimeo] Use `_form_hidden_inputs`

---
 youtube_dl/extractor/vimeo.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index cae90205d..d63c03183 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -452,11 +452,7 @@ class VimeoChannelIE(InfoExtractor):
         password = self._downloader.params.get('videopassword', None)
         if password is None:
             raise ExtractorError('This album is protected by a password, use the --video-password option', expected=True)
-        fields = dict(re.findall(r'''(?x)<input\s+
-            type="hidden"\s+
-            name="([^"]+)"\s+
-            value="([^"]*)"
-            ''', login_form))
+        fields = self._form_hidden_inputs(login_form)
         token = self._search_regex(r'xsrft[\s=:"\']+([^"\']+)', webpage, 'login token')
         fields['token'] = token
         fields['password'] = password

From 3b16d803c9633b7f3653d3d6d8022ed8642a09d5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Jul 2015 21:55:22 +0600
Subject: [PATCH 1828/6219] [shared] Use `_form_hidden_inputs`

---
 youtube_dl/extractor/shared.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/shared.py b/youtube_dl/extractor/shared.py
index 9f3e944e7..7fb68bc2d 100644
--- a/youtube_dl/extractor/shared.py
+++ b/youtube_dl/extractor/shared.py
@@ -35,8 +35,7 @@ class SharedIE(InfoExtractor):
             raise ExtractorError(
                 'Video %s does not exist' % video_id, expected=True)
 
-        download_form = dict(re.findall(
-            r'<input type="hidden" name="([^"]+)" value="([^"]*)"', webpage))
+        download_form = self._form_hidden_inputs(webpage)
         request = compat_urllib_request.Request(
             url, compat_urllib_parse.urlencode(download_form))
         request.add_header('Content-Type', 'application/x-www-form-urlencoded')

From b8070dbbd72fa24fb4c0b7973bf547825a0cfed3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Jul 2015 21:56:14 +0600
Subject: [PATCH 1829/6219] [promptfile] Use `_form_hidden_inputs`

---
 youtube_dl/extractor/promptfile.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/promptfile.py b/youtube_dl/extractor/promptfile.py
index f536e6e6c..81a63c7fc 100644
--- a/youtube_dl/extractor/promptfile.py
+++ b/youtube_dl/extractor/promptfile.py
@@ -35,10 +35,7 @@ class PromptFileIE(InfoExtractor):
             raise ExtractorError('Video %s does not exist' % video_id,
                                  expected=True)
 
-        fields = dict(re.findall(r'''(?x)type="hidden"\s+
-            name="(.+?)"\s+
-            value="(.*?)"
-            ''', webpage))
+        fields = self._form_hidden_inputs(webpage)
         post = compat_urllib_parse.urlencode(fields)
         req = compat_urllib_request.Request(url, post)
         req.add_header('Content-type', 'application/x-www-form-urlencoded')

From 46f0f500163f64ed8276466667d6733e036bfa2d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Jul 2015 21:56:41 +0600
Subject: [PATCH 1830/6219] [primesharetv] Use `_form_hidden_inputs`

---
 youtube_dl/extractor/primesharetv.py | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/primesharetv.py b/youtube_dl/extractor/primesharetv.py
index 01cc3d9ea..94c9fb2cb 100644
--- a/youtube_dl/extractor/primesharetv.py
+++ b/youtube_dl/extractor/primesharetv.py
@@ -31,12 +31,7 @@ class PrimeShareTVIE(InfoExtractor):
         if '>File not exist<' in webpage:
             raise ExtractorError('Video %s does not exist' % video_id, expected=True)
 
-        fields = dict(re.findall(r'''(?x)<input\s+
-            type="hidden"\s+
-            name="([^"]+)"\s+
-            (?:id="[^"]+"\s+)?
-            value="([^"]*)"
-            ''', webpage))
+        fields = self._form_hidden_inputs(webpage)
 
         headers = {
             'Referer': url,

From 9f01c1a803d06034d443882f57528eba342a9c94 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 10 Jul 2015 23:56:51 +0800
Subject: [PATCH 1831/6219] [kuwo] Use \d instead of [0-9]

---
 youtube_dl/extractor/kuwo.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/kuwo.py b/youtube_dl/extractor/kuwo.py
index 69afacac9..18bf66404 100644
--- a/youtube_dl/extractor/kuwo.py
+++ b/youtube_dl/extractor/kuwo.py
@@ -45,7 +45,7 @@ class KuwoBaseIE(InfoExtractor):
 
 class KuwoIE(KuwoBaseIE):
     IE_NAME = 'kuwo:song'
-    _VALID_URL = r'http://www\.kuwo\.cn/yinyue/(?P<id>[0-9]+?)/'
+    _VALID_URL = r'http://www\.kuwo\.cn/yinyue/(?P<id>\d+?)/'
     _TESTS = [{
         'url': 'http://www.kuwo.cn/yinyue/635632/',
         'info_dict': {
@@ -116,7 +116,7 @@ class KuwoIE(KuwoBaseIE):
 
 class KuwoAlbumIE(InfoExtractor):
     IE_NAME = 'kuwo:album'
-    _VALID_URL = r'http://www\.kuwo\.cn/album/(?P<id>[0-9]+?)/'
+    _VALID_URL = r'http://www\.kuwo\.cn/album/(?P<id>\d+?)/'
     _TEST = {
         'url': 'http://www.kuwo.cn/album/502294/',
         'info_dict': {
@@ -157,7 +157,7 @@ class KuwoChartIE(InfoExtractor):
         'info_dict': {
             'id': '香港中文龙虎榜',
             'title': '香港中文龙虎榜',
-            'description': 're:[0-9]{4}第[0-9]{2}期',
+            'description': 're:\d{4}第\d{2}期',
         },
         'playlist_mincount': 10,
     }
@@ -211,7 +211,7 @@ class KuwoSingerIE(InfoExtractor):
         )
 
         entries = []
-        first_page_only = False if re.search(r'/music(?:_[0-9]+)?\.htm', url) else True
+        first_page_only = False if re.search(r'/music(?:_\d+)?\.htm', url) else True
         for page_num in itertools.count(1):
             webpage = self._download_webpage(
                 'http://www.kuwo.cn/mingxing/%s/music_%d.htm' % (singer_id, page_num),
@@ -232,7 +232,7 @@ class KuwoSingerIE(InfoExtractor):
 
 class KuwoCategoryIE(InfoExtractor):
     IE_NAME = 'kuwo:category'
-    _VALID_URL = r'http://yinyue\.kuwo\.cn/yy/cinfo_(?P<id>[0-9]+?).htm'
+    _VALID_URL = r'http://yinyue\.kuwo\.cn/yy/cinfo_(?P<id>\d+?).htm'
     _TEST = {
         'url': 'http://yinyue.kuwo.cn/yy/cinfo_86375.htm',
         'info_dict': {
@@ -268,7 +268,7 @@ class KuwoCategoryIE(InfoExtractor):
 
 class KuwoMvIE(KuwoBaseIE):
     IE_NAME = 'kuwo:mv'
-    _VALID_URL = r'http://www\.kuwo\.cn/mv/(?P<id>[0-9]+?)/'
+    _VALID_URL = r'http://www\.kuwo\.cn/mv/(?P<id>\d+?)/'
     _TEST = {
         'url': 'http://www.kuwo.cn/mv/6480076/',
         'info_dict': {

From 01b89d5682adaecfd319897683b2043cfc7344c6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Jul 2015 21:57:29 +0600
Subject: [PATCH 1832/6219] [played] Use `_form_hidden_inputs`

---
 youtube_dl/extractor/played.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/played.py b/youtube_dl/extractor/played.py
index 45716c75d..9fe1524f2 100644
--- a/youtube_dl/extractor/played.py
+++ b/youtube_dl/extractor/played.py
@@ -38,9 +38,7 @@ class PlayedIE(InfoExtractor):
         if m_error:
             raise ExtractorError(m_error.group('msg'), expected=True)
 
-        fields = re.findall(
-            r'type="hidden" name="([^"]+)"\s+value="([^"]+)">', orig_webpage)
-        data = dict(fields)
+        data = self._form_hidden_inputs(orig_webpage)
 
         self._sleep(2, video_id)
 

From 8fa7e5817a37756e4135fcb9daba47fc543be36a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Jul 2015 21:58:24 +0600
Subject: [PATCH 1833/6219] [hostingbulk] Use `_form_hidden_inputs`

---
 youtube_dl/extractor/hostingbulk.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/hostingbulk.py b/youtube_dl/extractor/hostingbulk.py
index 704d0285d..63f579592 100644
--- a/youtube_dl/extractor/hostingbulk.py
+++ b/youtube_dl/extractor/hostingbulk.py
@@ -58,11 +58,7 @@ class HostingBulkIE(InfoExtractor):
             r'<img src="([^"]+)".+?class="pic"',
             webpage, 'thumbnail', fatal=False)
 
-        fields = dict(re.findall(r'''(?x)<input\s+
-            type="hidden"\s+
-            name="([^"]+)"\s+
-            value="([^"]*)"
-            ''', webpage))
+        fields = self._form_hidden_inputs(webpage)
 
         request = compat_urllib_request.Request(url, urlencode_postdata(fields))
         request.add_header('Content-type', 'application/x-www-form-urlencoded')

From 1c0163a5cc1d6b8fa5c7b4ee39b28e6e47c8981c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Jul 2015 21:58:49 +0600
Subject: [PATCH 1834/6219] [gorillavid] Use `_form_hidden_inputs`

---
 youtube_dl/extractor/gorillavid.py | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/gorillavid.py b/youtube_dl/extractor/gorillavid.py
index 6147596e4..aabf07a20 100644
--- a/youtube_dl/extractor/gorillavid.py
+++ b/youtube_dl/extractor/gorillavid.py
@@ -78,12 +78,7 @@ class GorillaVidIE(InfoExtractor):
         if re.search(self._FILE_NOT_FOUND_REGEX, webpage) is not None:
             raise ExtractorError('Video %s does not exist' % video_id, expected=True)
 
-        fields = dict(re.findall(r'''(?x)<input\s+
-            type="hidden"\s+
-            name="([^"]+)"\s+
-            (?:id="[^"]+"\s+)?
-            value="([^"]*)"
-            ''', webpage))
+        fields = self._form_hidden_inputs(webpage)
 
         if fields['op'] == 'download1':
             countdown = int_or_none(self._search_regex(

From 094790d2c963ef7ee9f5861e0458174ddd15ed87 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 11 Jul 2015 00:03:49 +0800
Subject: [PATCH 1835/6219] [kuwo:song] Give warnings for unavailable optional
 fields

---
 youtube_dl/extractor/kuwo.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/kuwo.py b/youtube_dl/extractor/kuwo.py
index 18bf66404..a021f3cdf 100644
--- a/youtube_dl/extractor/kuwo.py
+++ b/youtube_dl/extractor/kuwo.py
@@ -80,7 +80,7 @@ class KuwoIE(KuwoBaseIE):
             r'<h1[^>]+title="([^"]+)">', webpage, 'song name')
         singer_name = self._html_search_regex(
             r'<div[^>]+class="s_img">\s*<a[^>]+title="([^>]+)"',
-            webpage, 'singer name', default=None)
+            webpage, 'singer name', fatal=False)
         lrc_content = clean_html(get_element_by_id('lrcContent', webpage))
         if lrc_content == '暂无':     # indicates no lyrics
             lrc_content = None
@@ -89,7 +89,7 @@ class KuwoIE(KuwoBaseIE):
 
         album_id = self._html_search_regex(
             r'<p[^>]+class="album"[^<]+<a[^>]+href="http://www\.kuwo\.cn/album/(\d+)/"',
-            webpage, 'album id', default=None, fatal=False)
+            webpage, 'album id', fatal=False)
 
         publish_time = None
         if album_id is not None:
@@ -100,7 +100,7 @@ class KuwoIE(KuwoBaseIE):
 
             publish_time = self._html_search_regex(
                 r'发行时间：(\d{4}-\d{2}-\d{2})', album_info_page,
-                'publish time', default=None)
+                'publish time', fatal=False)
             if publish_time:
                 publish_time = publish_time.replace('-', '')
 

From 446e7645001d276e6036b9a81e7996edc93b360b Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 11 Jul 2015 00:49:59 +0800
Subject: [PATCH 1836/6219] [baidu] Add localized name

---
 youtube_dl/extractor/baidu.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/baidu.py b/youtube_dl/extractor/baidu.py
index 906895c1e..e37ee4440 100644
--- a/youtube_dl/extractor/baidu.py
+++ b/youtube_dl/extractor/baidu.py
@@ -8,6 +8,7 @@ from ..compat import compat_urlparse
 
 
 class BaiduVideoIE(InfoExtractor):
+    IE_DESC = '百度视频'
     _VALID_URL = r'http://v\.baidu\.com/(?P<type>[a-z]+)/(?P<id>\d+)\.htm'
     _TESTS = [{
         'url': 'http://v.baidu.com/comic/1069.htm?frp=bdbrand&q=%E4%B8%AD%E5%8D%8E%E5%B0%8F%E5%BD%93%E5%AE%B6',

From e1ba152352e95fb4cb6f3a5025166627d9df9787 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 11 Jul 2015 00:50:31 +0800
Subject: [PATCH 1837/6219] [ctsnews] Add localized name

---
 youtube_dl/extractor/ctsnews.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/ctsnews.py b/youtube_dl/extractor/ctsnews.py
index 0226f8036..45049bf37 100644
--- a/youtube_dl/extractor/ctsnews.py
+++ b/youtube_dl/extractor/ctsnews.py
@@ -6,6 +6,7 @@ from ..utils import parse_iso8601, ExtractorError
 
 
 class CtsNewsIE(InfoExtractor):
+    IE_DESC = '華視新聞'
     # https connection failed (Connection reset)
     _VALID_URL = r'http://news\.cts\.com\.tw/[a-z]+/[a-z]+/\d+/(?P<id>\d+)\.html'
     _TESTS = [{

From 513cbdda935b9f319cb61bffcdb0107249eb6b78 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 11 Jul 2015 00:50:45 +0800
Subject: [PATCH 1838/6219] [douyutv] Add localized name

---
 youtube_dl/extractor/douyutv.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/douyutv.py b/youtube_dl/extractor/douyutv.py
index 479430c51..373b3b4b4 100644
--- a/youtube_dl/extractor/douyutv.py
+++ b/youtube_dl/extractor/douyutv.py
@@ -9,6 +9,7 @@ from ..compat import (compat_str, compat_basestring)
 
 
 class DouyuTVIE(InfoExtractor):
+    IE_DESC = '斗鱼'
     _VALID_URL = r'http://(?:www\.)?douyutv\.com/(?P<id>[A-Za-z0-9]+)'
     _TESTS = [{
         'url': 'http://www.douyutv.com/iseven',

From 44c514eb9c03ddf495feeaf94495bee39cffd842 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 11 Jul 2015 00:51:03 +0800
Subject: [PATCH 1839/6219] [iqiyi] Add localized name

---
 youtube_dl/extractor/iqiyi.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index 9106dd074..0f6707d7c 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -20,6 +20,7 @@ from ..utils import (
 
 class IqiyiIE(InfoExtractor):
     IE_NAME = 'iqiyi'
+    IE_DESC = '爱奇艺'
 
     _VALID_URL = r'http://(?:www\.)iqiyi.com/v_.+?\.html'
 

From 0f08d7f851d8e41e09326d5e55dd93038d3ec752 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 11 Jul 2015 00:51:14 +0800
Subject: [PATCH 1840/6219] [kuwo] Add localized name

---
 youtube_dl/extractor/kuwo.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/kuwo.py b/youtube_dl/extractor/kuwo.py
index a021f3cdf..43f616a5d 100644
--- a/youtube_dl/extractor/kuwo.py
+++ b/youtube_dl/extractor/kuwo.py
@@ -45,6 +45,7 @@ class KuwoBaseIE(InfoExtractor):
 
 class KuwoIE(KuwoBaseIE):
     IE_NAME = 'kuwo:song'
+    IE_DESC = '酷我音乐'
     _VALID_URL = r'http://www\.kuwo\.cn/yinyue/(?P<id>\d+?)/'
     _TESTS = [{
         'url': 'http://www.kuwo.cn/yinyue/635632/',

From 963d0ce7e342841378945a8cb58d3c30bca02c60 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 11 Jul 2015 00:51:26 +0800
Subject: [PATCH 1841/6219] [letv] Add localized name

---
 youtube_dl/extractor/letv.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/letv.py b/youtube_dl/extractor/letv.py
index da896caf1..ba2ae8085 100644
--- a/youtube_dl/extractor/letv.py
+++ b/youtube_dl/extractor/letv.py
@@ -19,6 +19,7 @@ from ..utils import (
 
 
 class LetvIE(InfoExtractor):
+    IE_DESC = '乐视网'
     _VALID_URL = r'http://www\.letv\.com/ptv/vplay/(?P<id>\d+).html'
 
     _TESTS = [{

From 6ce89aecc330fe4b812526da4e119e7587548e27 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 11 Jul 2015 00:51:35 +0800
Subject: [PATCH 1842/6219] [neteasemusic] Add localized name

---
 youtube_dl/extractor/neteasemusic.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/neteasemusic.py b/youtube_dl/extractor/neteasemusic.py
index ee52efaee..8758a6f0b 100644
--- a/youtube_dl/extractor/neteasemusic.py
+++ b/youtube_dl/extractor/neteasemusic.py
@@ -63,6 +63,7 @@ class NetEaseMusicBaseIE(InfoExtractor):
 
 class NetEaseMusicIE(NetEaseMusicBaseIE):
     IE_NAME = 'netease:song'
+    IE_DESC = '网易云音乐'
     _VALID_URL = r'https?://music\.163\.com/(#/)?song\?id=(?P<id>[0-9]+)'
     _TESTS = [{
         'url': 'http://music.163.com/#/song?id=32102397',

From 9d16788ad928800f4b6c5682dd08d0814a98754b Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 11 Jul 2015 00:51:55 +0800
Subject: [PATCH 1843/6219] [nextmedia] Add localized name

---
 youtube_dl/extractor/nextmedia.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/nextmedia.py b/youtube_dl/extractor/nextmedia.py
index d1b7cff4c..c10784f6b 100644
--- a/youtube_dl/extractor/nextmedia.py
+++ b/youtube_dl/extractor/nextmedia.py
@@ -6,6 +6,7 @@ from ..utils import parse_iso8601
 
 
 class NextMediaIE(InfoExtractor):
+    IE_DESC = '蘋果日報'
     _VALID_URL = r'http://hk.apple.nextmedia.com/[^/]+/[^/]+/(?P<date>\d+)/(?P<id>\d+)'
     _TESTS = [{
         'url': 'http://hk.apple.nextmedia.com/realtime/news/20141108/53109199',
@@ -66,6 +67,7 @@ class NextMediaIE(InfoExtractor):
 
 
 class NextMediaActionNewsIE(NextMediaIE):
+    IE_DESC = '蘋果日報 - 動新聞'
     _VALID_URL = r'http://hk.dv.nextmedia.com/actionnews/[^/]+/(?P<date>\d+)/(?P<id>\d+)/\d+'
     _TESTS = [{
         'url': 'http://hk.dv.nextmedia.com/actionnews/hit/20150121/19009428/20061460',
@@ -90,6 +92,7 @@ class NextMediaActionNewsIE(NextMediaIE):
 
 
 class AppleDailyIE(NextMediaIE):
+    IE_DESC = '臺灣蘋果日報'
     _VALID_URL = r'http://(www|ent).appledaily.com.tw/(?:animation|appledaily|enews|realtimenews)/[^/]+/[^/]+/(?P<date>\d+)/(?P<id>\d+)(/.*)?'
     _TESTS = [{
         'url': 'http://ent.appledaily.com.tw/enews/article/entertainment/20150128/36354694',

From a7ada46bd90666b9c351bb54bd63b7225307afed Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 11 Jul 2015 00:52:07 +0800
Subject: [PATCH 1844/6219] [qqmusic] Add localized name

---
 youtube_dl/extractor/qqmusic.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index 476432330..4c05fded4 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -16,6 +16,7 @@ from ..compat import compat_urllib_request
 
 class QQMusicIE(InfoExtractor):
     IE_NAME = 'qqmusic'
+    IE_DESC = 'QQ音乐'
     _VALID_URL = r'http://y.qq.com/#type=song&mid=(?P<id>[0-9A-Za-z]+)'
     _TESTS = [{
         'url': 'http://y.qq.com/#type=song&mid=004295Et37taLD',

From 9b15be97aa0e094cccaf13e181b51b4399347d4d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 11 Jul 2015 00:52:19 +0800
Subject: [PATCH 1845/6219] [udn] Add localized name

---
 youtube_dl/extractor/udn.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/udn.py b/youtube_dl/extractor/udn.py
index c08428acf..2151f8338 100644
--- a/youtube_dl/extractor/udn.py
+++ b/youtube_dl/extractor/udn.py
@@ -11,6 +11,7 @@ from ..compat import compat_urlparse
 
 
 class UDNEmbedIE(InfoExtractor):
+    IE_DESC = '聯合影音'
     _VALID_URL = r'https?://video\.udn\.com/(?:embed|play)/news/(?P<id>\d+)'
     _TESTS = [{
         'url': 'http://video.udn.com/embed/news/300040',

From 4fa5f402321c71d6b2d8a281b60b8a23041c0e8f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 11 Jul 2015 00:52:56 +0800
Subject: [PATCH 1846/6219] [xuite] Add localized name

---
 youtube_dl/extractor/xuite.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/xuite.py b/youtube_dl/extractor/xuite.py
index 81d885fdc..5aac8adb3 100644
--- a/youtube_dl/extractor/xuite.py
+++ b/youtube_dl/extractor/xuite.py
@@ -13,6 +13,7 @@ from ..utils import (
 
 
 class XuiteIE(InfoExtractor):
+    IE_DESC = '隨意窩Xuite影音'
     _REGEX_BASE64 = r'(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?'
     _VALID_URL = r'https?://vlog\.xuite\.net/(?:play|embed)/(?P<id>%s)' % _REGEX_BASE64
     _TESTS = [{

From e014ff015d5240bac9371e5ffe5342ce025a574f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 11 Jul 2015 00:53:08 +0800
Subject: [PATCH 1847/6219] [yam] Add localized name

---
 youtube_dl/extractor/yam.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/yam.py b/youtube_dl/extractor/yam.py
index 9d851bae3..001ee17b6 100644
--- a/youtube_dl/extractor/yam.py
+++ b/youtube_dl/extractor/yam.py
@@ -14,6 +14,7 @@ from ..utils import (
 
 
 class YamIE(InfoExtractor):
+    IE_DESC = '蕃薯藤yam天空部落'
     _VALID_URL = r'http://mymedia.yam.com/m/(?P<id>\d+)'
 
     _TESTS = [{

From b931fbe5aba8e68a822f88d91d4ec424bc6509d3 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 11 Jul 2015 00:57:59 +0800
Subject: [PATCH 1848/6219] [yinyuetai] Add localized name

---
 youtube_dl/extractor/yinyuetai.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/yinyuetai.py b/youtube_dl/extractor/yinyuetai.py
index fa6b40816..834d860af 100644
--- a/youtube_dl/extractor/yinyuetai.py
+++ b/youtube_dl/extractor/yinyuetai.py
@@ -7,6 +7,7 @@ from ..utils import ExtractorError
 
 class YinYueTaiIE(InfoExtractor):
     IE_NAME = 'yinyuetai:video'
+    IE_DESC = '音悦Tai'
     _VALID_URL = r'https?://v\.yinyuetai\.com/video(?:/h5)?/(?P<id>[0-9]+)'
     _TESTS = [{
         'url': 'http://v.yinyuetai.com/video/2322376',

From 246995dbc8ac0eb61355966eba7c7559a9ab7b24 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 11 Jul 2015 00:58:13 +0800
Subject: [PATCH 1849/6219] [youku] Add localized name

---
 youtube_dl/extractor/youku.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index ced3a10cd..78caeb8b3 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -15,6 +15,7 @@ from ..compat import (
 
 class YoukuIE(InfoExtractor):
     IE_NAME = 'youku'
+    IE_DESC = '优酷'
     _VALID_URL = r'''(?x)
         (?:
             http://(?:v|player)\.youku\.com/(?:v_show/id_|player\.php/sid/)|

From edd66be5be584dfb04d740cdd663fbd6203e3730 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 11 Jul 2015 01:21:04 +0800
Subject: [PATCH 1850/6219] [kuwo] Add more localized names

---
 youtube_dl/extractor/kuwo.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/kuwo.py b/youtube_dl/extractor/kuwo.py
index 43f616a5d..1077846f2 100644
--- a/youtube_dl/extractor/kuwo.py
+++ b/youtube_dl/extractor/kuwo.py
@@ -117,6 +117,7 @@ class KuwoIE(KuwoBaseIE):
 
 class KuwoAlbumIE(InfoExtractor):
     IE_NAME = 'kuwo:album'
+    IE_DESC = '酷我音乐 - 专辑'
     _VALID_URL = r'http://www\.kuwo\.cn/album/(?P<id>\d+?)/'
     _TEST = {
         'url': 'http://www.kuwo.cn/album/502294/',
@@ -152,6 +153,7 @@ class KuwoAlbumIE(InfoExtractor):
 
 class KuwoChartIE(InfoExtractor):
     IE_NAME = 'kuwo:chart'
+    IE_DESC = '酷我音乐 - 排行榜'
     _VALID_URL = r'http://yinyue\.kuwo\.cn/billboard_(?P<id>[^.]+).htm'
     _TEST = {
         'url': 'http://yinyue.kuwo.cn/billboard_香港中文龙虎榜.htm',
@@ -184,6 +186,7 @@ class KuwoChartIE(InfoExtractor):
 
 class KuwoSingerIE(InfoExtractor):
     IE_NAME = 'kuwo:singer'
+    IE_DESC = '酷我音乐 - 歌手'
     _VALID_URL = r'http://www\.kuwo\.cn/mingxing/(?P<id>[^/]+)'
     _TESTS = [{
         'url': 'http://www.kuwo.cn/mingxing/bruno+mars/',
@@ -233,6 +236,7 @@ class KuwoSingerIE(InfoExtractor):
 
 class KuwoCategoryIE(InfoExtractor):
     IE_NAME = 'kuwo:category'
+    IE_DESC = '酷我音乐 - 分类'
     _VALID_URL = r'http://yinyue\.kuwo\.cn/yy/cinfo_(?P<id>\d+?).htm'
     _TEST = {
         'url': 'http://yinyue.kuwo.cn/yy/cinfo_86375.htm',
@@ -269,6 +273,7 @@ class KuwoCategoryIE(InfoExtractor):
 
 class KuwoMvIE(KuwoBaseIE):
     IE_NAME = 'kuwo:mv'
+    IE_DESC = '酷我音乐 - MV'
     _VALID_URL = r'http://www\.kuwo\.cn/mv/(?P<id>\d+?)/'
     _TEST = {
         'url': 'http://www.kuwo.cn/mv/6480076/',

From ed848087d53d09797eec932e84c8c36c010d04a6 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 11 Jul 2015 01:21:18 +0800
Subject: [PATCH 1851/6219] [neteasemusic] Add more localized names

---
 youtube_dl/extractor/neteasemusic.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/neteasemusic.py b/youtube_dl/extractor/neteasemusic.py
index 8758a6f0b..a8e0a64ed 100644
--- a/youtube_dl/extractor/neteasemusic.py
+++ b/youtube_dl/extractor/neteasemusic.py
@@ -171,6 +171,7 @@ class NetEaseMusicIE(NetEaseMusicBaseIE):
 
 class NetEaseMusicAlbumIE(NetEaseMusicBaseIE):
     IE_NAME = 'netease:album'
+    IE_DESC = '网易云音乐 - 专辑'
     _VALID_URL = r'https?://music\.163\.com/(#/)?album\?id=(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://music.163.com/#/album?id=220780',
@@ -200,6 +201,7 @@ class NetEaseMusicAlbumIE(NetEaseMusicBaseIE):
 
 class NetEaseMusicSingerIE(NetEaseMusicBaseIE):
     IE_NAME = 'netease:singer'
+    IE_DESC = '网易云音乐 - 歌手'
     _VALID_URL = r'https?://music\.163\.com/(#/)?artist\?id=(?P<id>[0-9]+)'
     _TESTS = [{
         'note': 'Singer has aliases.',
@@ -242,6 +244,7 @@ class NetEaseMusicSingerIE(NetEaseMusicBaseIE):
 
 class NetEaseMusicListIE(NetEaseMusicBaseIE):
     IE_NAME = 'netease:playlist'
+    IE_DESC = '网易云音乐 - 歌单'
     _VALID_URL = r'https?://music\.163\.com/(#/)?(playlist|discover/toplist)\?id=(?P<id>[0-9]+)'
     _TESTS = [{
         'url': 'http://music.163.com/#/playlist?id=79177352',
@@ -287,6 +290,7 @@ class NetEaseMusicListIE(NetEaseMusicBaseIE):
 
 class NetEaseMusicMvIE(NetEaseMusicBaseIE):
     IE_NAME = 'netease:mv'
+    IE_DESC = '网易云音乐 - MV'
     _VALID_URL = r'https?://music\.163\.com/(#/)?mv\?id=(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://music.163.com/#/mv?id=415350',
@@ -327,6 +331,7 @@ class NetEaseMusicMvIE(NetEaseMusicBaseIE):
 
 class NetEaseMusicProgramIE(NetEaseMusicBaseIE):
     IE_NAME = 'netease:program'
+    IE_DESC = '网易云音乐 - 电台节目'
     _VALID_URL = r'https?://music\.163\.com/(#/?)program\?id=(?P<id>[0-9]+)'
     _TESTS = [{
         'url': 'http://music.163.com/#/program?id=10109055',
@@ -411,6 +416,7 @@ class NetEaseMusicProgramIE(NetEaseMusicBaseIE):
 
 class NetEaseMusicDjRadioIE(NetEaseMusicBaseIE):
     IE_NAME = 'netease:djradio'
+    IE_DESC = '网易云音乐 - 电台'
     _VALID_URL = r'https?://music\.163\.com/(#/)?djradio\?id=(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://music.163.com/#/djradio?id=42',

From 181c4ccaaa186685d4a5fce6f338486182079b9e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 11 Jul 2015 01:21:36 +0800
Subject: [PATCH 1852/6219] [qqmusic] Add more localized names

---
 youtube_dl/extractor/qqmusic.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index 4c05fded4..1654a641f 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -143,6 +143,7 @@ class QQPlaylistBaseIE(InfoExtractor):
 
 class QQMusicSingerIE(QQPlaylistBaseIE):
     IE_NAME = 'qqmusic:singer'
+    IE_DESC = 'QQ音乐 - 歌手'
     _VALID_URL = r'http://y.qq.com/#type=singer&mid=(?P<id>[0-9A-Za-z]+)'
     _TEST = {
         'url': 'http://y.qq.com/#type=singer&mid=001BLpXF2DyJe2',
@@ -187,6 +188,7 @@ class QQMusicSingerIE(QQPlaylistBaseIE):
 
 class QQMusicAlbumIE(QQPlaylistBaseIE):
     IE_NAME = 'qqmusic:album'
+    IE_DESC = 'QQ音乐 - 专辑'
     _VALID_URL = r'http://y.qq.com/#type=album&mid=(?P<id>[0-9A-Za-z]+)'
 
     _TESTS = [{
@@ -229,6 +231,7 @@ class QQMusicAlbumIE(QQPlaylistBaseIE):
 
 class QQMusicToplistIE(QQPlaylistBaseIE):
     IE_NAME = 'qqmusic:toplist'
+    IE_DESC = 'QQ音乐 - 排行榜'
     _VALID_URL = r'http://y\.qq\.com/#type=toplist&p=(?P<id>(top|global)_[0-9]+)'
 
     _TESTS = [{
@@ -282,6 +285,7 @@ class QQMusicToplistIE(QQPlaylistBaseIE):
 
 class QQMusicPlaylistIE(QQPlaylistBaseIE):
     IE_NAME = 'qqmusic:playlist'
+    IE_DESC = 'QQ音乐 - 歌单'
     _VALID_URL = r'http://y\.qq\.com/#type=taoge&id=(?P<id>[0-9]+)'
 
     _TEST = {

From 86f2541695ba0280982e47e52c5cf26946d5d7c6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 10 Jul 2015 22:22:33 +0200
Subject: [PATCH 1853/6219] Remove unused 're' imports

---
 youtube_dl/extractor/primesharetv.py | 2 --
 youtube_dl/extractor/shared.py       | 1 -
 youtube_dl/extractor/vodlocker.py    | 2 --
 3 files changed, 5 deletions(-)

diff --git a/youtube_dl/extractor/primesharetv.py b/youtube_dl/extractor/primesharetv.py
index 94c9fb2cb..9aa0c862a 100644
--- a/youtube_dl/extractor/primesharetv.py
+++ b/youtube_dl/extractor/primesharetv.py
@@ -1,7 +1,5 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse,
diff --git a/youtube_dl/extractor/shared.py b/youtube_dl/extractor/shared.py
index 7fb68bc2d..6e2b94e7d 100644
--- a/youtube_dl/extractor/shared.py
+++ b/youtube_dl/extractor/shared.py
@@ -1,6 +1,5 @@
 from __future__ import unicode_literals
 
-import re
 import base64
 
 from .common import InfoExtractor
diff --git a/youtube_dl/extractor/vodlocker.py b/youtube_dl/extractor/vodlocker.py
index 431f4e2e3..4804692bf 100644
--- a/youtube_dl/extractor/vodlocker.py
+++ b/youtube_dl/extractor/vodlocker.py
@@ -1,8 +1,6 @@
 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse,

From 0a31a350981931ab9403d58258f5058ed98142a5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 10 Jul 2015 22:46:25 +0200
Subject: [PATCH 1854/6219] [YoutubeDL] format spec: add additional checks for
 invalid syntax

---
 test/test_YoutubeDL.py  | 10 ++++++++++
 youtube_dl/YoutubeDL.py |  4 ++++
 2 files changed, 14 insertions(+)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index bf2baae07..20f45f439 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -302,6 +302,16 @@ class TestFormatSelection(unittest.TestCase):
             downloaded = ydl.downloaded_info_dicts[0]
             self.assertEqual(downloaded['format_id'], f1['format_id'])
 
+    def test_invalid_format_specs(self):
+        def assert_syntax_error(format_spec):
+            ydl = YDL({'format': format_spec})
+            info_dict = _make_result([{'format_id': 'foo', 'url': TEST_URL}])
+            self.assertRaises(SyntaxError, ydl.process_ie_result, info_dict)
+
+        assert_syntax_error('bestvideo,,best')
+        assert_syntax_error('+bestaudio')
+        assert_syntax_error('bestvideo+')
+
     def test_format_filtering(self):
         formats = [
             {'format_id': 'A', 'filesize': 500, 'width': 1000},
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 6478d05dc..da7c51008 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -953,6 +953,8 @@ class YoutubeDL(object):
                         tokens.restore_last_token()
                         break
                     elif string == ',':
+                        if not current_selector:
+                            raise syntax_error('"," must follow a format selector', start)
                         selectors.append(current_selector)
                         current_selector = None
                     elif string == '/':
@@ -972,6 +974,8 @@ class YoutubeDL(object):
                     elif string == '+':
                         video_selector = current_selector
                         audio_selector = _parse_format_selection(tokens, inside_merge=True)
+                        if not video_selector or not audio_selector:
+                            raise syntax_error('"+" must be between two format selectors', start)
                         current_selector = FormatSelector(MERGE, (video_selector, audio_selector), [])
                     else:
                         raise syntax_error('Operator not recognized: "{0}"'.format(string), start)

From dc48695ab9c84cda56ee723aee11e5da44c2258a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 10 Jul 2015 22:59:45 +0200
Subject: [PATCH 1855/6219] Document how to group format selectors

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index e3452c9e1..4e6cb3fc7 100644
--- a/README.md
+++ b/README.md
@@ -268,7 +268,7 @@ youtube-dl_test_video_.mp4          # A simple file name
 By default youtube-dl tries to download the best quality, but sometimes you may want to download other format.
 The simplest case is requesting a specific format, for example `-f 22`. You can get the list of available formats using `--list-formats`, you can also use a file extension (currently it supports aac, m4a, mp3, mp4, ogg, wav, webm) or the special names `best`, `bestvideo`, `bestaudio` and `worst`.
 
-If you want to download multiple videos and they don't have the same formats available, you can specify the order of preference using slashes, as in `-f 22/17/18`. You can also filter the video results by putting a condition in brackets, as in `-f "best[height=720]"` (or `-f "[filesize>10M]"`).  This works for filesize, height, width, tbr, abr, vbr, asr, and fps and the comparisons <, <=, >, >=, =, != and for ext, acodec, vcodec, container, and protocol and the comparisons =, != . Formats for which the value is not known are excluded unless you put a question mark (?) after the operator. You can combine format filters, so  `-f "[height <=? 720][tbr>500]"` selects up to 720p videos (or videos where the height is not known) with a bitrate of at least 500 KBit/s. Use commas to download multiple formats, such as `-f 136/137/mp4/bestvideo,140/m4a/bestaudio`. You can merge the video and audio of two formats into a single file using `-f <video-format>+<audio-format>` (requires ffmpeg or avconv), for example `-f bestvideo+bestaudio`.
+If you want to download multiple videos and they don't have the same formats available, you can specify the order of preference using slashes, as in `-f 22/17/18`. You can also filter the video results by putting a condition in brackets, as in `-f "best[height=720]"` (or `-f "[filesize>10M]"`).  This works for filesize, height, width, tbr, abr, vbr, asr, and fps and the comparisons <, <=, >, >=, =, != and for ext, acodec, vcodec, container, and protocol and the comparisons =, != . Formats for which the value is not known are excluded unless you put a question mark (?) after the operator. You can combine format filters, so  `-f "[height <=? 720][tbr>500]"` selects up to 720p videos (or videos where the height is not known) with a bitrate of at least 500 KBit/s. Use commas to download multiple formats, such as `-f 136/137/mp4/bestvideo,140/m4a/bestaudio`. You can merge the video and audio of two formats into a single file using `-f <video-format>+<audio-format>` (requires ffmpeg or avconv), for example `-f bestvideo+bestaudio`. Format selectors can also be grouped using parentheses, for example if you want to download the best mp4 and webm formats with a height lower than 480 you can use `-f '(mp4,webm)[height<480]'`.
 
 Since the end of April 2015 and version 2015.04.26 youtube-dl uses `-f bestvideo+bestaudio/best` as default format selection (see #5447, #5456). If ffmpeg or avconv are installed this results in downloading `bestvideo` and `bestaudio` separately and muxing them together into a single file giving the best overall quality available. Otherwise it falls back to `best` and results in downloading best available quality served as a single file. `best` is also needed for videos that don't come from YouTube because they don't provide the audio and video in two different files. If you want to only download some dash formats (for example if you are not interested in getting videos with a resolution higher than 1080p), you can add `-f bestvideo[height<=?1080]+bestaudio/best` to your configuration file. Note that if you use youtube-dl to stream to `stdout` (and most likely to pipe it to your media player then), i.e. you explicitly specify output template as `-o -`, youtube-dl still uses `-f best` format selection in order to start content delivery immediately to your player and not to wait until `bestvideo` and `bestaudio` are downloaded and muxed.
 

From fac54cb426d85572c6928ce0d9e5bf1efb459548 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 11 Jul 2015 04:43:29 +0600
Subject: [PATCH 1856/6219] [webofstories:playlist] Improve and add test

---
 youtube_dl/extractor/webofstories.py | 28 ++++++++++++++++++++++------
 1 file changed, 22 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/webofstories.py b/youtube_dl/extractor/webofstories.py
index d70e30c00..2037d9b3d 100644
--- a/youtube_dl/extractor/webofstories.py
+++ b/youtube_dl/extractor/webofstories.py
@@ -104,7 +104,14 @@ class WebOfStoriesIE(InfoExtractor):
 
 class WebOfStoriesPlaylistIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?webofstories\.com/playAll/(?P<id>[^/]+)'
-    _TESTS = []
+    _TEST = {
+        'url': 'http://www.webofstories.com/playAll/donald.knuth',
+        'info_dict': {
+            'id': 'donald.knuth',
+            'title': 'Donald Knuth (Scientist)',
+        },
+        'playlist_mincount': 97,
+    }
 
     def _real_extract(self, url):
         playlist_id = self._match_id(url)
@@ -116,10 +123,19 @@ class WebOfStoriesPlaylistIE(InfoExtractor):
             for video_number in set(re.findall('href="/playAll/%s\?sId=(\d+)"' % playlist_id, webpage))
         ]
 
-        title = self._html_search_regex(
-            r'<title>([^<]+)\s*-\s*Web\sof\sStories</title>', webpage, 'title')
+        title = self._search_regex(
+            r'<div id="speakerName">\s*<span>([^<]+)</span>',
+            webpage, 'speaker', default=None)
+        if title:
+            field = self._search_regex(
+                r'<span id="primaryField">([^<]+)</span>',
+                webpage, 'field', default=None)
+            if field:
+                title += ' (%s)' % field
 
-        description = self._html_search_meta(
-            'description', webpage, 'description')
+        if not title:
+            title = self._search_regex(
+                r'<title>Play\s+all\s+stories\s*-\s*([^<]+)\s*-\s*Web\s+of\s+Stories</title>',
+                webpage, 'title')
 
-        return self.playlist_result(entries, playlist_id, title, description)
+        return self.playlist_result(entries, playlist_id, title)

From 79913fde35ead6247d7b764f3a95059fc4f17dbb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 11 Jul 2015 21:23:49 +0600
Subject: [PATCH 1857/6219] [vk] Add list id to info_url

---
 youtube_dl/extractor/vk.py | 22 +++++++++++++++++++++-
 1 file changed, 21 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index c0292095b..333c2a634 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -29,7 +29,7 @@ class VKIE(InfoExtractor):
                                 (?:m\.)?vk\.com/(?:.+?\?.*?z=)?video|
                                 (?:www\.)?biqle\.ru/watch/
                             )
-                            (?P<videoid>[^s].*?)(?:\?|%2F|$)
+                            (?P<videoid>[^s].*?)(?:\?(?:.*\blist=(?P<list_id>[\da-f]+))?|%2F|$)
                         )
                     '''
     _NETRC_MACHINE = 'vk'
@@ -119,6 +119,20 @@ class VKIE(InfoExtractor):
             },
             'skip': 'Only works from Russia',
         },
+        {
+            # video (removed?) only available with list id
+            'url': 'https://vk.com/video30481095_171201961?list=8764ae2d21f14088d4',
+            'md5': '091287af5402239a1051c37ec7b92913',
+            'info_dict': {
+                'id': '171201961',
+                'ext': 'mp4',
+                'title': 'ТюменцевВВ_09.07.2015',
+                'uploader': 'Anton Ivanov',
+                'duration': 109,
+                'upload_date': '20150709',
+                'view_count': int,
+            },
+        },
         {
             # youtube embed
             'url': 'https://vk.com/video276849682_170681728',
@@ -182,6 +196,12 @@ class VKIE(InfoExtractor):
             video_id = '%s_%s' % (mobj.group('oid'), mobj.group('id'))
 
         info_url = 'https://vk.com/al_video.php?act=show&al=1&module=video&video=%s' % video_id
+
+        # Some videos (removed?) can only be downloaded with list id specified
+        list_id = mobj.group('list_id')
+        if list_id:
+            info_url += '&list=%s' % list_id
+
         info_page = self._download_webpage(info_url, video_id)
 
         if re.search(r'<!>/login\.php\?.*\bact=security_check', info_page):

From d919fa3344463f94a152a10ba65981cd290d9ec8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 11 Jul 2015 21:26:03 +0600
Subject: [PATCH 1858/6219] [vk] Handle access denied error

---
 youtube_dl/extractor/vk.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 333c2a634..8ac3aeac0 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -222,6 +222,9 @@ class VKIE(InfoExtractor):
 
             r'<!>Видео временно недоступно':
             'Video %s is temporarily unavailable.',
+
+            r'<!>Access denied':
+            'Access denied to video %s.',
         }
 
         for error_re, error_msg in ERRORS.items():

From f72b0a603270dff0fdd72dd5218126790232199a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 11 Jul 2015 22:15:16 +0600
Subject: [PATCH 1859/6219] Revert xvid to avi and make docs to be similar to
 existing external downloader option

---
 README.md                          | 4 ++--
 youtube_dl/YoutubeDL.py            | 3 ++-
 youtube_dl/__init__.py             | 2 +-
 youtube_dl/options.py              | 6 +++---
 youtube_dl/postprocessor/ffmpeg.py | 8 +++-----
 5 files changed, 11 insertions(+), 12 deletions(-)

diff --git a/README.md b/README.md
index d8926d2b7..9779c2058 100644
--- a/README.md
+++ b/README.md
@@ -214,8 +214,8 @@ which means you can modify it, redistribute it or use it however you like.
     --audio-format FORMAT            Specify audio format: "best", "aac", "vorbis", "mp3", "m4a", "opus", or "wav"; "best" by default
     --audio-quality QUALITY          Specify ffmpeg/avconv audio quality, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K (default
                                      5)
-    --recode-video FORMAT            Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm|mkv|xvid)
-    --postprocessor-args             Extra parameters for video post-processor.
+    --recode-video FORMAT            Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm|mkv|avi)
+    --postprocessor-args ARGS        Give these arguments to the postprocessor
     -k, --keep-video                 Keep the video file on disk after the post-processing; the video is erased by default
     --no-post-overwrites             Do not overwrite post-processed files; the post-processed files are overwritten by default
     --embed-subs                     Embed subtitles in the video (only for mkv and mp4 videos)
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 8580f99a7..00af78e06 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -262,7 +262,8 @@ class YoutubeDL(object):
     The following options are used by the post processors:
     prefer_ffmpeg:     If True, use ffmpeg instead of avconv if both are available,
                        otherwise prefer avconv.
-    postprocessor_args: Extra parameters for external apps, like avconv.
+    postprocessor_args: A list of additional command-line arguments for the
+                        postprocessor.
     """
 
     params = None
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index fb31d1569..2d416943f 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -169,7 +169,7 @@ def _real_main(argv=None):
         if not opts.audioquality.isdigit():
             parser.error('invalid audio quality specified')
     if opts.recodevideo is not None:
-        if opts.recodevideo not in ['mp4', 'flv', 'webm', 'ogg', 'mkv', 'xvid']:
+        if opts.recodevideo not in ['mp4', 'flv', 'webm', 'ogg', 'mkv', 'avi']:
             parser.error('invalid video recode format specified')
     if opts.convertsubtitles is not None:
         if opts.convertsubtitles not in ['srt', 'vtt', 'ass']:
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index c15dadb21..85365d769 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -691,11 +691,11 @@ def parseOpts(overrideArguments=None):
     postproc.add_option(
         '--recode-video',
         metavar='FORMAT', dest='recodevideo', default=None,
-        help='Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm|mkv|xvid)')
+        help='Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm|mkv|avi)')
     postproc.add_option(
         '--postprocessor-args',
-        dest='postprocessor_args', default=None, metavar='ARGS',
-        help='Extra parameters for video post-processor.')
+        dest='postprocessor_args', metavar='ARGS',
+        help='Give these arguments to the postprocessor')
     postproc.add_option(
         '-k', '--keep-video',
         action='store_true', dest='keepvideo', default=False,
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index d4ba3572b..62d13a567 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -295,12 +295,10 @@ class FFmpegVideoConvertorPP(FFmpegPostProcessor):
     def run(self, information):
         path = information['filepath']
         prefix, sep, ext = path.rpartition('.')
-        ext = self._preferedformat
+        outpath = prefix + sep + self._preferedformat
         options = self._extra_cmd_args
-        if self._preferedformat == 'xvid':
-            ext = 'avi'
+        if self._preferedformat == 'avi':
             options.extend(['-c:v', 'libxvid', '-vtag', 'XVID'])
-        outpath = prefix + sep + ext
         if information['ext'] == self._preferedformat:
             self._downloader.to_screen('[ffmpeg] Not converting video file %s - already is in target format %s' % (path, self._preferedformat))
             return [], information
@@ -308,7 +306,7 @@ class FFmpegVideoConvertorPP(FFmpegPostProcessor):
         self.run_ffmpeg(path, outpath, options)
         information['filepath'] = outpath
         information['format'] = self._preferedformat
-        information['ext'] = ext
+        information['ext'] = self._preferedformat
         return [path], information
 
 

From e35b23f54df83886dd14428b618aa9d0221968ad Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 11 Jul 2015 22:41:33 +0600
Subject: [PATCH 1860/6219] [postprocessor/common] Improve postprocessor args
 fetching and clarify doc

---
 youtube_dl/postprocessor/common.py | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/postprocessor/common.py b/youtube_dl/postprocessor/common.py
index bee64c457..4191d040b 100644
--- a/youtube_dl/postprocessor/common.py
+++ b/youtube_dl/postprocessor/common.py
@@ -22,14 +22,15 @@ class PostProcessor(object):
     of the chain is reached.
 
     PostProcessor objects follow a "mutual registration" process similar
-    to InfoExtractor objects. And it can receive parameters from CLI trough
-    --postprocessor-args.
+    to InfoExtractor objects.
+
+    Optionally PostProcessor can use a list of additional command-line arguments
+    with self._configuration_args.
     """
 
     _downloader = None
 
     def __init__(self, downloader=None):
-        self._extra_cmd_args = downloader.params.get('postprocessor_args')
         self._downloader = downloader
 
     def set_downloader(self, downloader):
@@ -59,6 +60,13 @@ class PostProcessor(object):
         except Exception:
             self._downloader.report_warning(errnote)
 
+    def _configuration_args(self, default=[]):
+        pp_args = self._downloader.params.get('postprocessor_args')
+        if pp_args is None:
+            return default
+        assert isinstance(pp_args, list)
+        return pp_args
+
 
 class AudioConversionError(PostProcessingError):
     pass

From 15006fedb9511599c0e948a982fa282cde74f2ca Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 11 Jul 2015 22:42:03 +0600
Subject: [PATCH 1861/6219] [postprocessor/ffmpeg] Spread postprocessor args
 usage on all ffmpeg extractors

---
 youtube_dl/postprocessor/ffmpeg.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 62d13a567..1ecce22e7 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -131,6 +131,8 @@ class FFmpegPostProcessor(PostProcessor):
         oldest_mtime = min(
             os.stat(encodeFilename(path)).st_mtime for path in input_paths)
 
+        opts += self._configuration_args()
+
         files_cmd = []
         for path in input_paths:
             files_cmd.extend([encodeArgument('-i'), encodeFilename(path, True)])
@@ -294,14 +296,14 @@ class FFmpegVideoConvertorPP(FFmpegPostProcessor):
 
     def run(self, information):
         path = information['filepath']
-        prefix, sep, ext = path.rpartition('.')
-        outpath = prefix + sep + self._preferedformat
-        options = self._extra_cmd_args
-        if self._preferedformat == 'avi':
-            options.extend(['-c:v', 'libxvid', '-vtag', 'XVID'])
         if information['ext'] == self._preferedformat:
             self._downloader.to_screen('[ffmpeg] Not converting video file %s - already is in target format %s' % (path, self._preferedformat))
             return [], information
+        options = []
+        if self._preferedformat == 'avi':
+            options.extend(['-c:v', 'libxvid', '-vtag', 'XVID'])
+        prefix, sep, ext = path.rpartition('.')
+        outpath = prefix + sep + self._preferedformat
         self._downloader.to_screen('[' + 'ffmpeg' + '] Converting video from %s to %s, Destination: ' % (information['ext'], self._preferedformat) + outpath)
         self.run_ffmpeg(path, outpath, options)
         information['filepath'] = outpath

From 369e195a44530b43bfd0e95087c9e1a238d91184 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 11 Jul 2015 22:43:02 +0600
Subject: [PATCH 1862/6219] Handle postprocessor_args similarly to
 external_downloader_args

---
 youtube_dl/__init__.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 2d416943f..55b22c889 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -263,6 +263,9 @@ def _real_main(argv=None):
     external_downloader_args = None
     if opts.external_downloader_args:
         external_downloader_args = shlex.split(opts.external_downloader_args)
+    postprocessor_args = None
+    if opts.postprocessor_args:
+        postprocessor_args = shlex.split(opts.postprocessor_args)
     match_filter = (
         None if opts.match_filter is None
         else match_filter_func(opts.match_filter))
@@ -354,7 +357,6 @@ def _real_main(argv=None):
         'extract_flat': opts.extract_flat,
         'merge_output_format': opts.merge_output_format,
         'postprocessors': postprocessors,
-        'postprocessor_args': shlex.split(opts.postprocessor_args or ''),
         'fixup': opts.fixup,
         'source_address': opts.source_address,
         'call_home': opts.call_home,
@@ -368,6 +370,7 @@ def _real_main(argv=None):
         'ffmpeg_location': opts.ffmpeg_location,
         'hls_prefer_native': opts.hls_prefer_native,
         'external_downloader_args': external_downloader_args,
+        'postprocessor_args': postprocessor_args,
         'cn_verification_proxy': opts.cn_verification_proxy,
     }
 

From 41c0d2f8cb22fe34d957bc9b5f9032a9160685ff Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 11 Jul 2015 23:00:19 +0600
Subject: [PATCH 1863/6219] Credit @aurium for avi recode and postprocessort
 args (#5942)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index d5418dd37..c0201d875 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -130,3 +130,4 @@ Peter Ding
 jackyzy823
 George Brighton
 Remita Amine
+Aurélio A. Heckert

From ac8f97f2b37115bf81a653360819b5d30eebda18 Mon Sep 17 00:00:00 2001
From: felix <felix.von.s@posteo.de>
Date: Sat, 11 Jul 2015 18:56:07 +0200
Subject: [PATCH 1864/6219] [rdsca] New extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/rdsca.py    | 50 ++++++++++++++++++++++++++++++++
 2 files changed, 51 insertions(+)
 create mode 100644 youtube_dl/extractor/rdsca.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index cbaa07391..73fa0e4ed 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -460,6 +460,7 @@ from .radiobremen import RadioBremenIE
 from .radiofrance import RadioFranceIE
 from .rai import RaiIE
 from .rbmaradio import RBMARadioIE
+from .rdsca import RDScaIE
 from .redtube import RedTubeIE
 from .restudy import RestudyIE
 from .reverbnation import ReverbNationIE
diff --git a/youtube_dl/extractor/rdsca.py b/youtube_dl/extractor/rdsca.py
new file mode 100644
index 000000000..a8df927e4
--- /dev/null
+++ b/youtube_dl/extractor/rdsca.py
@@ -0,0 +1,50 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    parse_iso8601,
+    url_basename,
+)
+
+
+class RDScaIE(InfoExtractor):
+    IE_NAME = 'RDS.ca'
+    _VALID_URL = r'http://(?:www\.)?rds\.ca/videos/(?P<id>.*)'
+
+    _TESTS = [{
+        'url': 'http://www.rds.ca/videos/football/nfl/fowler-jr-prend-la-direction-de-jacksonville-3.1132799',
+        'info_dict': {
+            "ext": "mp4",
+            "title": "Fowler Jr. prend la direction de Jacksonville",
+            "description": "Dante Fowler Jr. est le troisième choix du repêchage 2015 de la NFL. ",
+            "timestamp": 1430397346,
+        }
+    }]
+
+    def _real_extract(self, url):
+        video_id = url_basename(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        title = self._search_regex(
+            r'<span itemprop="name"[^>]*>([^\n]*)</span>', webpage, 'video title', default=None)
+        video_url = self._search_regex(
+            r'<span itemprop="contentURL" content="([^"]+)"', webpage, 'video URL')
+        upload_date = parse_iso8601(self._search_regex(
+            r'<span itemprop="uploadDate" content="([^"]+)"', webpage, 'upload date', default=None))
+        description = self._search_regex(
+            r'<span itemprop="description"[^>]*>([^\n]*)</span>', webpage, 'description', default=None)
+        thumbnail = self._search_regex(
+            r'<span itemprop="thumbnailUrl" content="([^"]+)"', webpage, 'upload date', default=None)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'timestamp': upload_date,
+            'formats': [{
+                'url': video_url,
+            }],
+        }

From 53429e6551b80f78273f5245e11613fd39625113 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 12 Jul 2015 12:53:15 +0600
Subject: [PATCH 1865/6219] [onionstudios] Fix extraction

---
 youtube_dl/extractor/onionstudios.py | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/onionstudios.py b/youtube_dl/extractor/onionstudios.py
index 8fa507dec..0f1f448fe 100644
--- a/youtube_dl/extractor/onionstudios.py
+++ b/youtube_dl/extractor/onionstudios.py
@@ -49,19 +49,21 @@ class OnionStudiosIE(InfoExtractor):
         self._sort_formats(formats)
 
         title = self._search_regex(
-            r'share_title\s*=\s*"([^"]+)"', webpage, 'title')
+            r'share_title\s*=\s*(["\'])(?P<title>[^\1]+?)\1',
+            webpage, 'title', group='title')
         description = self._search_regex(
-            r'share_description\s*=\s*"([^"]+)"', webpage,
-            'description', default=None)
+            r'share_description\s*=\s*(["\'])(?P<description>[^\1]+?)\1',
+            webpage, 'description', default=None, group='description')
         thumbnail = self._search_regex(
-            r'poster="([^"]+)"', webpage, 'thumbnail', default=False)
+            r'poster\s*=\s*(["\'])(?P<thumbnail>[^\1]+?)\1',
+            webpage, 'thumbnail', default=False, group='thumbnail')
 
         uploader_id = self._search_regex(
-            r'twitter_handle\s*=\s*"([^"]+)"',
-            webpage, 'uploader id', fatal=False)
+            r'twitter_handle\s*=\s*(["\'])(?P<uploader_id>[^\1]+?)\1',
+            webpage, 'uploader id', fatal=False, group='uploader_id')
         uploader = self._search_regex(
-            r'window\.channelName\s*=\s*"Embedded:([^"]+)"',
-            webpage, 'uploader', default=False)
+            r'window\.channelName\s*=\s*(["\'])Embedded:(?P<uploader>[^\1]+?)\1',
+            webpage, 'uploader', default=False, group='uploader')
 
         return {
             'id': video_id,

From 667170e2c7dee6bcf5e357dd53d461807434c5b3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 12 Jul 2015 20:40:00 +0600
Subject: [PATCH 1866/6219] [nrk:tv] Add support for radio URLs (Closes #6200)

---
 youtube_dl/extractor/nrk.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index 9e4581cf9..eb49f292e 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -116,7 +116,8 @@ class NRKPlaylistIE(InfoExtractor):
 
 
 class NRKTVIE(InfoExtractor):
-    _VALID_URL = r'(?P<baseurl>https?://tv\.nrk(?:super)?\.no/)(?:serie/[^/]+|program)/(?P<id>[a-zA-Z]{4}\d{8})(?:/\d{2}-\d{2}-\d{4})?(?:#del=(?P<part_id>\d+))?'
+    IE_DESC = 'NRK TV and NRK Radio'
+    _VALID_URL = r'(?P<baseurl>https?://(?:tv|radio)\.nrk(?:super)?\.no/)(?:serie/[^/]+|program)/(?P<id>[a-zA-Z]{4}\d{8})(?:/\d{2}-\d{2}-\d{4})?(?:#del=(?P<part_id>\d+))?'
 
     _TESTS = [
         {
@@ -188,6 +189,10 @@ class NRKTVIE(InfoExtractor):
                 'duration': 6947.5199999999995,
             },
             'skip': 'Only works from Norway',
+        },
+        {
+            'url': 'https://radio.nrk.no/serie/dagsnytt/NPUB21019315/12-07-2015#',
+            'only_matching': True,
         }
     ]
 

From c4f1fde75bf69e05be29283c9211089e6ab269ad Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 12 Jul 2015 20:42:18 +0600
Subject: [PATCH 1867/6219] [nrk:tv] Add format id prefixes

---
 youtube_dl/extractor/nrk.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index eb49f292e..d066a96db 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -211,7 +211,8 @@ class NRKTVIE(InfoExtractor):
         ]}
 
     def _extract_f4m(self, manifest_url, video_id):
-        return self._extract_f4m_formats(manifest_url + '?hdcore=3.1.1&plugin=aasp-3.1.1.69.124', video_id)
+        return self._extract_f4m_formats(
+            manifest_url + '?hdcore=3.1.1&plugin=aasp-3.1.1.69.124', video_id, f4m_id='hds')
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
@@ -273,7 +274,7 @@ class NRKTVIE(InfoExtractor):
 
         m3u8_url = re.search(r'data-hls-media="([^"]+)"', webpage)
         if m3u8_url:
-            formats.extend(self._extract_m3u8_formats(m3u8_url.group(1), video_id, 'mp4'))
+            formats.extend(self._extract_m3u8_formats(m3u8_url.group(1), video_id, 'mp4', m3u8_id='hls'))
         self._sort_formats(formats)
 
         subtitles_url = self._html_search_regex(

From b6ea9ef21ad3a43bddf24f5769bdf21a372381fa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 12 Jul 2015 23:35:56 +0600
Subject: [PATCH 1868/6219] [rds] Improve

---
 youtube_dl/extractor/__init__.py |  2 +-
 youtube_dl/extractor/rds.py      | 70 ++++++++++++++++++++++++++++++++
 youtube_dl/extractor/rdsca.py    | 50 -----------------------
 3 files changed, 71 insertions(+), 51 deletions(-)
 create mode 100644 youtube_dl/extractor/rds.py
 delete mode 100644 youtube_dl/extractor/rdsca.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 73fa0e4ed..3f4f23521 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -460,7 +460,7 @@ from .radiobremen import RadioBremenIE
 from .radiofrance import RadioFranceIE
 from .rai import RaiIE
 from .rbmaradio import RBMARadioIE
-from .rdsca import RDScaIE
+from .rds import RDSIE
 from .redtube import RedTubeIE
 from .restudy import RestudyIE
 from .reverbnation import ReverbNationIE
diff --git a/youtube_dl/extractor/rds.py b/youtube_dl/extractor/rds.py
new file mode 100644
index 000000000..fdd3f36a6
--- /dev/null
+++ b/youtube_dl/extractor/rds.py
@@ -0,0 +1,70 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    parse_duration,
+    parse_iso8601,
+)
+
+
+class RDSIE(InfoExtractor):
+    IE_DESC = 'RDS.ca'
+    _VALID_URL = r'https?://(?:www\.)?rds\.ca/videos/(?:[^/]+/)+(?P<display_id>[^/]+)-(?P<id>\d+\.\d+)'
+
+    _TEST = {
+        'url': 'http://www.rds.ca/videos/football/nfl/fowler-jr-prend-la-direction-de-jacksonville-3.1132799',
+        'info_dict': {
+            'id': '3.1132799',
+            'display_id': 'fowler-jr-prend-la-direction-de-jacksonville',
+            'ext': 'mp4',
+            'title': 'Fowler Jr. prend la direction de Jacksonville',
+            'description': 'Dante Fowler Jr. est le troisième choix du repêchage 2015 de la NFL. ',
+            'timestamp': 1430397346,
+            'upload_date': '20150430',
+            'duration': 154.354,
+            'age_limit': 0,
+        }
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        display_id = mobj.group('display_id')
+
+        webpage = self._download_webpage(url, display_id)
+
+        # TODO: extract f4m from 9c9media.com
+        video_url = self._search_regex(
+            r'<span[^>]+itemprop="contentURL"[^>]+content="([^"]+)"',
+            webpage, 'video url')
+
+        title = self._og_search_title(webpage) or self._html_search_meta(
+            'title', webpage, 'title', fatal=True)
+        description = self._og_search_description(webpage) or self._html_search_meta(
+            'description', webpage, 'description')
+        thumbnail = self._og_search_thumbnail(webpage) or self._search_regex(
+            [r'<link[^>]+itemprop="thumbnailUrl"[^>]+href="([^"]+)"',
+             r'<span[^>]+itemprop="thumbnailUrl"[^>]+content="([^"]+)"'],
+            webpage, 'thumbnail', fatal=False)
+        timestamp = parse_iso8601(self._search_regex(
+            r'<span[^>]+itemprop="uploadDate"[^>]+content="([^"]+)"',
+            webpage, 'upload date', fatal=False))
+        duration = parse_duration(self._search_regex(
+            r'<span[^>]+itemprop="duration"[^>]+content="([^"]+)"',
+            webpage, 'duration', fatal=False))
+        age_limit = self._family_friendly_search(webpage)
+
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            'url': video_url,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'timestamp': timestamp,
+            'duration': duration,
+            'age_limit': age_limit,
+        }
diff --git a/youtube_dl/extractor/rdsca.py b/youtube_dl/extractor/rdsca.py
deleted file mode 100644
index a8df927e4..000000000
--- a/youtube_dl/extractor/rdsca.py
+++ /dev/null
@@ -1,50 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-from .common import InfoExtractor
-from ..utils import (
-    parse_iso8601,
-    url_basename,
-)
-
-
-class RDScaIE(InfoExtractor):
-    IE_NAME = 'RDS.ca'
-    _VALID_URL = r'http://(?:www\.)?rds\.ca/videos/(?P<id>.*)'
-
-    _TESTS = [{
-        'url': 'http://www.rds.ca/videos/football/nfl/fowler-jr-prend-la-direction-de-jacksonville-3.1132799',
-        'info_dict': {
-            "ext": "mp4",
-            "title": "Fowler Jr. prend la direction de Jacksonville",
-            "description": "Dante Fowler Jr. est le troisième choix du repêchage 2015 de la NFL. ",
-            "timestamp": 1430397346,
-        }
-    }]
-
-    def _real_extract(self, url):
-        video_id = url_basename(url)
-
-        webpage = self._download_webpage(url, video_id)
-
-        title = self._search_regex(
-            r'<span itemprop="name"[^>]*>([^\n]*)</span>', webpage, 'video title', default=None)
-        video_url = self._search_regex(
-            r'<span itemprop="contentURL" content="([^"]+)"', webpage, 'video URL')
-        upload_date = parse_iso8601(self._search_regex(
-            r'<span itemprop="uploadDate" content="([^"]+)"', webpage, 'upload date', default=None))
-        description = self._search_regex(
-            r'<span itemprop="description"[^>]*>([^\n]*)</span>', webpage, 'description', default=None)
-        thumbnail = self._search_regex(
-            r'<span itemprop="thumbnailUrl" content="([^"]+)"', webpage, 'upload date', default=None)
-
-        return {
-            'id': video_id,
-            'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
-            'timestamp': upload_date,
-            'formats': [{
-                'url': video_url,
-            }],
-        }

From 28fb109ed0276e85207966ac925e9885c2ed0e7b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 12 Jul 2015 23:45:47 +0600
Subject: [PATCH 1869/6219] [rds] Improve _VALID_URL

---
 youtube_dl/extractor/rds.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/rds.py b/youtube_dl/extractor/rds.py
index fdd3f36a6..796adfdf9 100644
--- a/youtube_dl/extractor/rds.py
+++ b/youtube_dl/extractor/rds.py
@@ -12,9 +12,9 @@ from ..utils import (
 
 class RDSIE(InfoExtractor):
     IE_DESC = 'RDS.ca'
-    _VALID_URL = r'https?://(?:www\.)?rds\.ca/videos/(?:[^/]+/)+(?P<display_id>[^/]+)-(?P<id>\d+\.\d+)'
+    _VALID_URL = r'https?://(?:www\.)?rds\.ca/vid(?:[eé]|%C3%A9)os/(?:[^/]+/)*(?P<display_id>[^/]+)-(?P<id>\d+\.\d+)'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.rds.ca/videos/football/nfl/fowler-jr-prend-la-direction-de-jacksonville-3.1132799',
         'info_dict': {
             'id': '3.1132799',
@@ -27,7 +27,10 @@ class RDSIE(InfoExtractor):
             'duration': 154.354,
             'age_limit': 0,
         }
-    }
+    }, {
+        'url': 'http://www.rds.ca/vid%C3%A9os/un-voyage-positif-3.877934',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)

From 75a40b22511b4d4344cd6903e28429fa6c3991f1 Mon Sep 17 00:00:00 2001
From: Seamus Phelan <SeamusPhelan@gmail.com>
Date: Mon, 13 Jul 2015 22:35:45 +1000
Subject: [PATCH 1870/6219] [SBS] fixes due to website changes

---
 youtube_dl/extractor/sbs.py | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/sbs.py b/youtube_dl/extractor/sbs.py
index d4bd1a0d7..ab4d1c884 100644
--- a/youtube_dl/extractor/sbs.py
+++ b/youtube_dl/extractor/sbs.py
@@ -1,12 +1,8 @@
 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
-import re
 from .common import InfoExtractor
-from ..utils import (
-    js_to_json,
-    remove_end,
-)
+from ..utils import remove_end
 
 
 class SBSIE(InfoExtractor):
@@ -34,18 +30,22 @@ class SBSIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        webpage = self._download_webpage(url, video_id)
+        # the video is in the following iframe
+        iframe_url = 'http://www.sbs.com.au/ondemand/video/single/' + video_id + '?context=web'
+        webpage = self._download_webpage(iframe_url, video_id)
 
-        player = self._search_regex(
-            r'(?s)playerParams\.releaseUrls\s*=\s*(\{.*?\n\});\n',
-            webpage, 'player')
-        player = re.sub(r"'\s*\+\s*[\da-zA-Z_]+\s*\+\s*'", '', player)
+        player_params = self._search_regex(
+            r'(?s)(playerParams.+?releaseUrls.+?\n)',
+            webpage, 'playerParams')
+        player_params_js = self._search_regex(
+            r'({.*})',
+            player_params, 'player_param_js')
 
-        release_urls = self._parse_json(js_to_json(player), video_id)
+        player_params_json = self._parse_json(player_params_js, video_id)
 
-        theplatform_url = release_urls.get('progressive') or release_urls['standard']
+        theplatform_url = player_params_json.get('releaseUrls')['progressive'] or player_params_json.get('releaseUrls')['standard']
 
-        title = remove_end(self._og_search_title(webpage), ' (The Feed)')
+        title = remove_end(self._og_search_title(webpage, default=video_id, fatal=False), ' (The Feed)')
         description = self._html_search_meta('description', webpage)
         thumbnail = self._og_search_thumbnail(webpage)
 

From f870544302f75bee0d96f6a8623c8ff270beca89 Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Mon, 13 Jul 2015 07:41:38 -0500
Subject: [PATCH 1871/6219] Add support for democracynow.org

Supports downloading clips or entire shows. Subtitle support
---
 youtube_dl/extractor/__init__.py     |   1 +
 youtube_dl/extractor/democracynow.py | 100 +++++++++++++++++++++++++++
 2 files changed, 101 insertions(+)
 create mode 100644 youtube_dl/extractor/democracynow.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index cbaa07391..5cc03b875 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -112,6 +112,7 @@ from .daum import DaumIE
 from .dbtv import DBTVIE
 from .dctp import DctpTvIE
 from .deezer import DeezerPlaylistIE
+from .democracynow import DemocracynowIE
 from .dfb import DFBIE
 from .dhm import DHMIE
 from .dotsub import DotsubIE
diff --git a/youtube_dl/extractor/democracynow.py b/youtube_dl/extractor/democracynow.py
new file mode 100644
index 000000000..1c9b36052
--- /dev/null
+++ b/youtube_dl/extractor/democracynow.py
@@ -0,0 +1,100 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import json
+import time
+import hmac
+import hashlib
+import itertools
+import re
+from ..utils import (
+    ExtractorError,
+    int_or_none,
+    parse_age_limit,
+    parse_iso8601,
+)
+from ..compat import compat_urllib_request
+from .common import InfoExtractor
+
+
+class DemocracynowIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?democracynow.org/?(?P<id>[^\?]*)'
+    IE_NAME = 'democracynow'
+    _TESTS = [{
+        'url': 'http://www.democracynow.org/shows/2015/7/3',
+        'info_dict': {
+            'id': '2015-0703-001',
+            'ext': 'mp4',
+            'title': 'July 03, 2015 - Democracy Now!',
+            'description': 'A daily independent global news hour with Amy Goodman & Juan Gonz\xe1lez "What to the Slave is 4th of July?": James Earl Jones Reads Frederick Douglass\u2019 Historic Speech : "This Flag Comes Down Today": Bree Newsome Scales SC Capitol Flagpole, Takes Down Confederate Flag : "We Shall Overcome": Remembering Folk Icon, Activist Pete Seeger in His Own Words & Songs',
+            'uploader': 'Democracy Now',
+            'upload_date': None,
+        },
+     },{
+        'url': 'http://www.democracynow.org/2015/7/3/this_flag_comes_down_today_bree',
+        'info_dict': {
+            'id': '2015-0703-001',
+            'ext': 'mp4',
+            'title': '"This Flag Comes Down Today": Bree Newsome Scales SC Capitol Flagpole, Takes Down Confederate Flag',
+            'description': 'md5:4d2bc4f0d29f5553c2210a4bc7761a21',
+            'uploader': 'Democracy Now',
+            'upload_date': None,
+        },
+
+    }]
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        base_host = re.search(r'^(.+?://[^/]+)', url).group(1)
+        if display_id == '':
+            display_id = 'home'
+        webpage = self._download_webpage(url, display_id)
+        re_desc = re.search(r'<meta property=.og:description. content=(["\'])(.+?)\1',webpage,re.DOTALL)
+        description = re_desc.group(2) if re_desc else ''
+
+        jstr = self._search_regex(r'({.+?"related_video_xml".+?})', webpage, 'json', default=None)
+        js = self._parse_json(jstr, display_id)
+        video_id = None
+        formats = []
+        subtitles = {}
+        for key in ('caption_file','.......'):
+            # ....... = pending vtt support that doesn't clobber srt 'chapter_file':
+            url = js.get(key,'')
+            if url == '' or url == None:
+                continue
+            if not re.match(r'^https?://',url):
+                url = base_host + url
+            ext = re.search(r'\.([^\.]+)$',url).group(1)
+            subtitles['eng'] = [{
+                'ext': ext,
+                'url': url,
+            }]
+        for key in ('file', 'audio'):
+            url = js.get(key,'')
+            if url == '' or url == None:
+                continue
+            if not re.match(r'^https?://',url):
+                url = base_host + url
+            purl = re.search(r'/(?P<dir>[^/]+)/(?:dn)?(?P<fn>[^/]+?)\.(?P<ext>[^\.\?]+)(?P<hasparams>\?|$)',url)
+            if video_id == None:
+                video_id = purl.group('fn')
+            if js.get('start') != None:
+                url += '&' if purl.group('hasparams') == '?' else '?'
+                url = url + 'start='+str(js.get('start'))
+            formats.append({
+                'format_id': purl.group('dir'),
+                'ext': purl.group('ext'),
+                'url': url,
+            })
+        self._sort_formats(formats)
+        ret = {
+            'id': video_id,
+            'title': js.get('title'),
+            'description': description,
+            'uploader': 'Democracy Now',
+#            'thumbnails': thumbnails,
+            'subtitles': subtitles,
+            'formats': formats,
+        }
+        return ret
+#
\ No newline at end of file

From 6dfa0602f059397d9a41ef382d9dfa8b432ff0fa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 13 Jul 2015 22:11:05 +0600
Subject: [PATCH 1872/6219] [nowtv] Fix extraction (Closes #6169)

---
 youtube_dl/extractor/nowtv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nowtv.py b/youtube_dl/extractor/nowtv.py
index 173e46cd8..0b5ff4760 100644
--- a/youtube_dl/extractor/nowtv.py
+++ b/youtube_dl/extractor/nowtv.py
@@ -133,7 +133,7 @@ class NowTVIE(InfoExtractor):
         station = mobj.group('station')
 
         info = self._download_json(
-            'https://api.nowtv.de/v3/movies/%s?fields=*,format,files' % display_id,
+            'https://api.nowtv.de/v3/movies/%s?fields=id,title,free,geoblocked,articleLong,articleShort,broadcastStartDate,seoUrl,duration,format,files' % display_id,
             display_id)
 
         video_id = compat_str(info['id'])

From 7869eb3fc44826533d859d3e7e32df5cd4be7ef4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 13 Jul 2015 22:21:35 +0600
Subject: [PATCH 1873/6219] Credit Bernhard Minks for nowtv patch

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index c0201d875..531ec5767 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -131,3 +131,4 @@ jackyzy823
 George Brighton
 Remita Amine
 Aurélio A. Heckert
+Bernhard Minks

From cfe5537ee5ca5156f3969f513a1257be83015b1d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 14 Jul 2015 00:21:11 +0800
Subject: [PATCH 1874/6219] [myspass] Fix extraction (closes #6206)

---
 youtube_dl/extractor/myspass.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/myspass.py b/youtube_dl/extractor/myspass.py
index 5b9b9fbcd..4557a2b13 100644
--- a/youtube_dl/extractor/myspass.py
+++ b/youtube_dl/extractor/myspass.py
@@ -35,7 +35,8 @@ class MySpassIE(InfoExtractor):
 
         # get metadata
         metadata_url = META_DATA_URL_TEMPLATE % video_id
-        metadata = self._download_xml(metadata_url, video_id)
+        metadata = self._download_xml(
+            metadata_url, video_id, transform_source=lambda s: s.strip())
 
         # extract values from metadata
         url_flv_el = metadata.find('url_flv')

From 1d1dd597ed96790b3d041274044742baec13132e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 14 Jul 2015 01:35:44 +0600
Subject: [PATCH 1875/6219] [dramafever] Extract srt (Closes #6207)

---
 youtube_dl/extractor/dramafever.py | 49 +++++++++++++++++++++---------
 1 file changed, 34 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/dramafever.py b/youtube_dl/extractor/dramafever.py
index ca41a3abf..38e6597c8 100644
--- a/youtube_dl/extractor/dramafever.py
+++ b/youtube_dl/extractor/dramafever.py
@@ -23,8 +23,23 @@ class DramaFeverBaseIE(InfoExtractor):
     _LOGIN_URL = 'https://www.dramafever.com/accounts/login/'
     _NETRC_MACHINE = 'dramafever'
 
+    _CONSUMER_SECRET = 'DA59dtVXYLxajktV'
+
+    _consumer_secret = None
+
+    def _get_consumer_secret(self):
+        mainjs = self._download_webpage(
+            'http://www.dramafever.com/static/51afe95/df2014/scripts/main.js',
+            None, 'Downloading main.js', fatal=False)
+        if not mainjs:
+            return self._CONSUMER_SECRET
+        return self._search_regex(
+            r"var\s+cs\s*=\s*'([^']+)'", mainjs,
+            'consumer secret', default=self._CONSUMER_SECRET)
+
     def _real_initialize(self):
         self._login()
+        self._consumer_secret = self._get_consumer_secret()
 
     def _login(self):
         (username, password) = self._get_login_info()
@@ -119,6 +134,23 @@ class DramaFeverIE(DramaFeverBaseIE):
                 'url': href,
             }]
 
+        series_id, episode_number = video_id.split('.')
+        episode_info = self._download_json(
+            # We only need a single episode info, so restricting page size to one episode
+            # and dealing with page number as with episode number
+            r'http://www.dramafever.com/api/4/episode/series/?cs=%s&series_id=%s&page_number=%s&page_size=1'
+            % (self._consumer_secret, series_id, episode_number),
+            video_id, 'Downloading episode info JSON', fatal=False)
+        if episode_info:
+            value = episode_info.get('value')
+            if value:
+                subfile = value[0].get('subfile') or value[0].get('new_subfile')
+                if subfile and subfile != 'http://www.dramafever.com/st/':
+                    subtitles.setdefault('English', []).append({
+                        'ext': 'srt',
+                        'url': subfile,
+                    })
+
         return {
             'id': video_id,
             'title': title,
@@ -152,27 +184,14 @@ class DramaFeverSeriesIE(DramaFeverBaseIE):
         'playlist_count': 20,
     }]
 
-    _CONSUMER_SECRET = 'DA59dtVXYLxajktV'
     _PAGE_SIZE = 60  # max is 60 (see http://api.drama9.com/#get--api-4-episode-series-)
 
-    def _get_consumer_secret(self, video_id):
-        mainjs = self._download_webpage(
-            'http://www.dramafever.com/static/51afe95/df2014/scripts/main.js',
-            video_id, 'Downloading main.js', fatal=False)
-        if not mainjs:
-            return self._CONSUMER_SECRET
-        return self._search_regex(
-            r"var\s+cs\s*=\s*'([^']+)'", mainjs,
-            'consumer secret', default=self._CONSUMER_SECRET)
-
     def _real_extract(self, url):
         series_id = self._match_id(url)
 
-        consumer_secret = self._get_consumer_secret(series_id)
-
         series = self._download_json(
             'http://www.dramafever.com/api/4/series/query/?cs=%s&series_id=%s'
-            % (consumer_secret, series_id),
+            % (self._consumer_secret, series_id),
             series_id, 'Downloading series JSON')['series'][series_id]
 
         title = clean_html(series['name'])
@@ -182,7 +201,7 @@ class DramaFeverSeriesIE(DramaFeverBaseIE):
         for page_num in itertools.count(1):
             episodes = self._download_json(
                 'http://www.dramafever.com/api/4/episode/series/?cs=%s&series_id=%s&page_size=%d&page_number=%d'
-                % (consumer_secret, series_id, self._PAGE_SIZE, page_num),
+                % (self._consumer_secret, series_id, self._PAGE_SIZE, page_num),
                 series_id, 'Downloading episodes JSON page #%d' % page_num)
             for episode in episodes.get('value', []):
                 episode_url = episode.get('episode_url')

From 50aa2bb6b94805d901f00d4da7a1b4ff1fcd5169 Mon Sep 17 00:00:00 2001
From: cazulu <jvlarapeinado@gmail.com>
Date: Tue, 14 Jul 2015 15:33:55 +0900
Subject: [PATCH 1876/6219] [dailymotion] Extract duration (closes #6221)

---
 youtube_dl/extractor/dailymotion.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index 8852f0add..1a41c0db1 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -53,6 +53,7 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
                 'uploader': 'IGN',
                 'title': 'Steam Machine Models, Pricing Listed on Steam Store - IGN News',
                 'upload_date': '20150306',
+                'duration': 74,
             }
         },
         # Vevo video
@@ -164,6 +165,7 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
             'thumbnail': info['thumbnail_url'],
             'age_limit': age_limit,
             'view_count': view_count,
+            'duration': info['duration']
         }
 
     def _get_subtitles(self, video_id, webpage):

From 9750e7d70eed92a6b05637465698cdd30e87a44c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 14 Jul 2015 12:56:32 +0200
Subject: [PATCH 1877/6219] [postprocessor/ffmpeg] Don't use '[youtube] ...' in
 messages

Because it can be used for other extractors.
---
 youtube_dl/postprocessor/ffmpeg.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 1ecce22e7..1f723908b 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -265,7 +265,7 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
         # If we download foo.mp3 and convert it to... foo.mp3, then don't delete foo.mp3, silly.
         if (new_path == path or
                 (self._nopostoverwrites and os.path.exists(encodeFilename(new_path)))):
-            self._downloader.to_screen('[youtube] Post-process file %s exists, skipping' % new_path)
+            self._downloader.to_screen('[ffmpeg] Post-process file %s exists, skipping' % new_path)
             return [], information
 
         try:

From f8da79f828637757889f3f35d7adfa9aabbfc721 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 14 Jul 2015 22:36:30 +0600
Subject: [PATCH 1878/6219] [extractor/common] Improve _form_hidden_inputs and
 rename to _hidden_inputs

---
 youtube_dl/extractor/common.py       | 15 +++++++++++----
 youtube_dl/extractor/gorillavid.py   |  2 +-
 youtube_dl/extractor/hostingbulk.py  |  2 +-
 youtube_dl/extractor/played.py       |  2 +-
 youtube_dl/extractor/primesharetv.py |  2 +-
 youtube_dl/extractor/promptfile.py   |  2 +-
 youtube_dl/extractor/shared.py       |  2 +-
 youtube_dl/extractor/twitch.py       |  2 +-
 youtube_dl/extractor/vimeo.py        |  2 +-
 youtube_dl/extractor/vk.py           |  2 +-
 youtube_dl/extractor/vodlocker.py    |  2 +-
 11 files changed, 21 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 82f5de2d8..315fe4a72 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -706,10 +706,17 @@ class InfoExtractor(object):
                                       'twitter card player')
 
     @staticmethod
-    def _form_hidden_inputs(html):
-        return dict(re.findall(
-            r'<input\s+type="hidden"\s+name="([^"]+)"\s+(?:id="[^"]+"\s+)?value="([^"]*)"',
-            html))
+    def _hidden_inputs(html):
+        return dict([
+            (input.group('name'), input.group('value')) for input in re.finditer(
+                r'''(?x)
+                    <input\s+
+                        type=(?P<q_hidden>["\'])hidden(?P=q_hidden)\s+
+                        name=(?P<q_name>["\'])(?P<name>.+?)(?P=q_name)\s+
+                        (?:id=(?P<q_id>["\']).+?(?P=q_id)\s+)?
+                        value=(?P<q_value>["\'])(?P<value>.*?)(?P=q_value)
+                ''', html)
+        ])
 
     def _sort_formats(self, formats, field_preference=None):
         if not formats:
diff --git a/youtube_dl/extractor/gorillavid.py b/youtube_dl/extractor/gorillavid.py
index aabf07a20..f006f0cb1 100644
--- a/youtube_dl/extractor/gorillavid.py
+++ b/youtube_dl/extractor/gorillavid.py
@@ -78,7 +78,7 @@ class GorillaVidIE(InfoExtractor):
         if re.search(self._FILE_NOT_FOUND_REGEX, webpage) is not None:
             raise ExtractorError('Video %s does not exist' % video_id, expected=True)
 
-        fields = self._form_hidden_inputs(webpage)
+        fields = self._hidden_inputs(webpage)
 
         if fields['op'] == 'download1':
             countdown = int_or_none(self._search_regex(
diff --git a/youtube_dl/extractor/hostingbulk.py b/youtube_dl/extractor/hostingbulk.py
index 63f579592..a3154cfde 100644
--- a/youtube_dl/extractor/hostingbulk.py
+++ b/youtube_dl/extractor/hostingbulk.py
@@ -58,7 +58,7 @@ class HostingBulkIE(InfoExtractor):
             r'<img src="([^"]+)".+?class="pic"',
             webpage, 'thumbnail', fatal=False)
 
-        fields = self._form_hidden_inputs(webpage)
+        fields = self._hidden_inputs(webpage)
 
         request = compat_urllib_request.Request(url, urlencode_postdata(fields))
         request.add_header('Content-type', 'application/x-www-form-urlencoded')
diff --git a/youtube_dl/extractor/played.py b/youtube_dl/extractor/played.py
index 9fe1524f2..8a1c296dd 100644
--- a/youtube_dl/extractor/played.py
+++ b/youtube_dl/extractor/played.py
@@ -38,7 +38,7 @@ class PlayedIE(InfoExtractor):
         if m_error:
             raise ExtractorError(m_error.group('msg'), expected=True)
 
-        data = self._form_hidden_inputs(orig_webpage)
+        data = self._hidden_inputs(orig_webpage)
 
         self._sleep(2, video_id)
 
diff --git a/youtube_dl/extractor/primesharetv.py b/youtube_dl/extractor/primesharetv.py
index 9aa0c862a..304359dc5 100644
--- a/youtube_dl/extractor/primesharetv.py
+++ b/youtube_dl/extractor/primesharetv.py
@@ -29,7 +29,7 @@ class PrimeShareTVIE(InfoExtractor):
         if '>File not exist<' in webpage:
             raise ExtractorError('Video %s does not exist' % video_id, expected=True)
 
-        fields = self._form_hidden_inputs(webpage)
+        fields = self._hidden_inputs(webpage)
 
         headers = {
             'Referer': url,
diff --git a/youtube_dl/extractor/promptfile.py b/youtube_dl/extractor/promptfile.py
index 81a63c7fc..8190ed676 100644
--- a/youtube_dl/extractor/promptfile.py
+++ b/youtube_dl/extractor/promptfile.py
@@ -35,7 +35,7 @@ class PromptFileIE(InfoExtractor):
             raise ExtractorError('Video %s does not exist' % video_id,
                                  expected=True)
 
-        fields = self._form_hidden_inputs(webpage)
+        fields = self._hidden_inputs(webpage)
         post = compat_urllib_parse.urlencode(fields)
         req = compat_urllib_request.Request(url, post)
         req.add_header('Content-type', 'application/x-www-form-urlencoded')
diff --git a/youtube_dl/extractor/shared.py b/youtube_dl/extractor/shared.py
index 6e2b94e7d..a07677686 100644
--- a/youtube_dl/extractor/shared.py
+++ b/youtube_dl/extractor/shared.py
@@ -34,7 +34,7 @@ class SharedIE(InfoExtractor):
             raise ExtractorError(
                 'Video %s does not exist' % video_id, expected=True)
 
-        download_form = self._form_hidden_inputs(webpage)
+        download_form = self._hidden_inputs(webpage)
         request = compat_urllib_request.Request(
             url, compat_urllib_parse.urlencode(download_form))
         request.add_header('Content-Type', 'application/x-www-form-urlencoded')
diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index af2b798fb..92b6dc1b8 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -59,7 +59,7 @@ class TwitchBaseIE(InfoExtractor):
         login_page = self._download_webpage(
             self._LOGIN_URL, None, 'Downloading login page')
 
-        login_form = self._form_hidden_inputs(login_page)
+        login_form = self._hidden_inputs(login_page)
 
         login_form.update({
             'login': username.encode('utf-8'),
diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index d63c03183..10d6745af 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -452,7 +452,7 @@ class VimeoChannelIE(InfoExtractor):
         password = self._downloader.params.get('videopassword', None)
         if password is None:
             raise ExtractorError('This album is protected by a password, use the --video-password option', expected=True)
-        fields = self._form_hidden_inputs(login_form)
+        fields = self._hidden_inputs(login_form)
         token = self._search_regex(r'xsrft[\s=:"\']+([^"\']+)', webpage, 'login token')
         fields['token'] = token
         fields['password'] = password
diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 8ac3aeac0..8f677cae3 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -168,7 +168,7 @@ class VKIE(InfoExtractor):
         login_page = self._download_webpage(
             'https://vk.com', None, 'Downloading login page')
 
-        login_form = self._form_hidden_inputs(login_page)
+        login_form = self._hidden_inputs(login_page)
 
         login_form.update({
             'email': username.encode('cp1251'),
diff --git a/youtube_dl/extractor/vodlocker.py b/youtube_dl/extractor/vodlocker.py
index 4804692bf..ccf1928b5 100644
--- a/youtube_dl/extractor/vodlocker.py
+++ b/youtube_dl/extractor/vodlocker.py
@@ -26,7 +26,7 @@ class VodlockerIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        fields = self._form_hidden_inputs(webpage)
+        fields = self._hidden_inputs(webpage)
 
         if fields['op'] == 'download1':
             self._sleep(3, video_id)  # they do detect when requests happen too fast!

From cf61d96df0984e28d8c34328177504a2d1424bd2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 14 Jul 2015 22:38:10 +0600
Subject: [PATCH 1879/6219] [extractor/common] Add _form_hidden_inputs

---
 youtube_dl/extractor/common.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 315fe4a72..3a396c0b0 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -718,6 +718,12 @@ class InfoExtractor(object):
                 ''', html)
         ])
 
+    def _form_hidden_inputs(self, form_id, html):
+        form = self._search_regex(
+            r'(?s)<form[^>]+?id=(["\'])%s\1[^>]*>(?P<form>.+?)</form>' % form_id,
+            html, '%s form' % form_id, group='form')
+        return self._hidden_inputs(form)
+
     def _sort_formats(self, formats, field_preference=None):
         if not formats:
             raise ExtractorError('No video formats found')

From dcd4d95c8e9acec6dde0706de75bf866a43c9aa6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 14 Jul 2015 22:39:41 +0600
Subject: [PATCH 1880/6219] [udemy] Fix authentication (Closes #6224)

---
 youtube_dl/extractor/udemy.py | 34 ++++++++++++++++++++--------------
 1 file changed, 20 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index 4667ed83b..192606077 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -15,7 +15,8 @@ from ..utils import (
 class UdemyIE(InfoExtractor):
     IE_NAME = 'udemy'
     _VALID_URL = r'https?://www\.udemy\.com/(?:[^#]+#/lecture/|lecture/view/?\?lectureId=)(?P<id>\d+)'
-    _LOGIN_URL = 'https://www.udemy.com/join/login-submit/'
+    _LOGIN_URL = 'https://www.udemy.com/join/login-popup/?displayType=ajax&showSkipButton=1'
+    _ORIGIN_URL = 'https://www.udemy.com'
     _NETRC_MACHINE = 'udemy'
 
     _TESTS = [{
@@ -74,29 +75,34 @@ class UdemyIE(InfoExtractor):
                 expected=True)
 
         login_popup = self._download_webpage(
-            'https://www.udemy.com/join/login-popup?displayType=ajax&showSkipButton=1', None,
-            'Downloading login popup')
+            self._LOGIN_URL, None, 'Downloading login popup')
 
         if login_popup == '<div class="run-command close-popup redirect" data-url="https://www.udemy.com/"></div>':
             return
 
-        csrf = self._html_search_regex(
-            r'<input type="hidden" name="csrf" value="(.+?)"',
-            login_popup, 'csrf token')
+        login_form = self._form_hidden_inputs('login-form', login_popup)
 
-        login_form = {
-            'email': username,
-            'password': password,
-            'csrf': csrf,
+        login_form.update({
             'displayType': 'json',
-            'isSubmitted': '1',
-        }
+            'email': username.encode('utf-8'),
+            'password': password.encode('utf-8'),
+        })
+
         request = compat_urllib_request.Request(
             self._LOGIN_URL, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
-        response = self._download_json(
+        request.add_header('Referer', self._ORIGIN_URL)
+        request.add_header('Origin', self._ORIGIN_URL)
+
+        response = self._download_webpage(
             request, None, 'Logging in as %s' % username)
 
-        if 'returnUrl' not in response:
+        if all(logout_pattern not in response
+               for logout_pattern in ['href="https://www.udemy.com/user/logout/', '>Logout<']):
+            error = self._html_search_regex(
+                r'(?s)<div[^>]+class="form-errors[^"]*">(.+?)</div>',
+                response, 'error message', default=None)
+            if error:
+                raise ExtractorError('Unable to login: %s' % error, expected=True)
             raise ExtractorError('Unable to log in')
 
     def _real_extract(self, url):

From 79057965a87a9179b3c3c2249df5495684c5e4ac Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 14 Jul 2015 22:46:26 +0600
Subject: [PATCH 1881/6219] [udemy] Remove superfluous field

---
 youtube_dl/extractor/udemy.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index 192606077..e2bab52fe 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -83,7 +83,6 @@ class UdemyIE(InfoExtractor):
         login_form = self._form_hidden_inputs('login-form', login_popup)
 
         login_form.update({
-            'displayType': 'json',
             'email': username.encode('utf-8'),
             'password': password.encode('utf-8'),
         })

From 01d115b06b5b752324a9346f128d659a98b58f31 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 14 Jul 2015 19:07:40 +0200
Subject: [PATCH 1882/6219] [jeuxvideo] Relax _VALID_URL (fixes #6230)

---
 youtube_dl/extractor/jeuxvideo.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/jeuxvideo.py b/youtube_dl/extractor/jeuxvideo.py
index d0720ff56..1df084d87 100644
--- a/youtube_dl/extractor/jeuxvideo.py
+++ b/youtube_dl/extractor/jeuxvideo.py
@@ -8,9 +8,9 @@ from .common import InfoExtractor
 
 
 class JeuxVideoIE(InfoExtractor):
-    _VALID_URL = r'http://.*?\.jeuxvideo\.com/.*/(.*?)-\d+\.htm'
+    _VALID_URL = r'http://.*?\.jeuxvideo\.com/.*/(.*?)\.htm'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.jeuxvideo.com/reportages-videos-jeux/0004/00046170/tearaway-playstation-vita-gc-2013-tearaway-nous-presente-ses-papiers-d-identite-00115182.htm',
         'md5': '046e491afb32a8aaac1f44dd4ddd54ee',
         'info_dict': {
@@ -19,7 +19,10 @@ class JeuxVideoIE(InfoExtractor):
             'title': 'Tearaway : GC 2013 : Tearaway nous présente ses papiers d\'identité',
             'description': 'Lorsque les développeurs de LittleBigPlanet proposent un nouveau titre, on ne peut que s\'attendre à un résultat original et fort attrayant.',
         },
-    }
+    }, {
+        'url': 'http://www.jeuxvideo.com/videos/chroniques/434220/l-histoire-du-jeu-video-la-saturn.htm',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)

From eae89f92e61d8227a6463c50750310a56c68cf09 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 14 Jul 2015 23:54:13 +0600
Subject: [PATCH 1883/6219] [dfb] Update test

---
 youtube_dl/extractor/dfb.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/dfb.py b/youtube_dl/extractor/dfb.py
index 8049779b0..6732e1751 100644
--- a/youtube_dl/extractor/dfb.py
+++ b/youtube_dl/extractor/dfb.py
@@ -10,13 +10,13 @@ class DFBIE(InfoExtractor):
     _VALID_URL = r'https?://tv\.dfb\.de/video/[^/]+/(?P<id>\d+)'
 
     _TEST = {
-        'url': 'http://tv.dfb.de/video/highlights-des-empfangs-in-berlin/9070/',
+        'url': 'http://tv.dfb.de/video/u-19-em-stimmen-zum-spiel-gegen-russland/11633/',
         # The md5 is different each time
         'info_dict': {
-            'id': '9070',
+            'id': '11633',
             'ext': 'flv',
-            'title': 'Highlights des Empfangs in Berlin',
-            'upload_date': '20140716',
+            'title': 'U 19-EM: Stimmen zum Spiel gegen Russland',
+            'upload_date': '20150714',
         },
     }
 

From ddcdc684e20af96a4fc57da49c27b31c75462aae Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 14 Jul 2015 23:59:21 +0600
Subject: [PATCH 1884/6219] [dfb] Extract display_id and modernize

---
 youtube_dl/extractor/dfb.py | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/dfb.py b/youtube_dl/extractor/dfb.py
index 6732e1751..f1df9b9a0 100644
--- a/youtube_dl/extractor/dfb.py
+++ b/youtube_dl/extractor/dfb.py
@@ -3,17 +3,19 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..utils import unified_strdate
 
 
 class DFBIE(InfoExtractor):
     IE_NAME = 'tv.dfb.de'
-    _VALID_URL = r'https?://tv\.dfb\.de/video/[^/]+/(?P<id>\d+)'
+    _VALID_URL = r'https?://tv\.dfb\.de/video/(?P<display_id>[^/]+)/(?P<id>\d+)'
 
     _TEST = {
         'url': 'http://tv.dfb.de/video/u-19-em-stimmen-zum-spiel-gegen-russland/11633/',
         # The md5 is different each time
         'info_dict': {
             'id': '11633',
+            'display_id': 'u-19-em-stimmen-zum-spiel-gegen-russland',
             'ext': 'flv',
             'title': 'U 19-EM: Stimmen zum Spiel gegen Russland',
             'upload_date': '20150714',
@@ -23,22 +25,25 @@ class DFBIE(InfoExtractor):
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
+        display_id = mobj.group('display_id')
 
-        webpage = self._download_webpage(url, video_id)
+        webpage = self._download_webpage(url, display_id)
         player_info = self._download_xml(
             'http://tv.dfb.de/server/hd_video.php?play=%s' % video_id,
-            video_id)
+            display_id)
         video_info = player_info.find('video')
 
-        f4m_info = self._download_xml(self._proto_relative_url(video_info.find('url').text.strip()), video_id)
+        f4m_info = self._download_xml(
+            self._proto_relative_url(video_info.find('url').text.strip()), display_id)
         token_el = f4m_info.find('token')
         manifest_url = token_el.attrib['url'] + '?' + 'hdnea=' + token_el.attrib['auth'] + '&hdcore=3.2.0'
 
         return {
             'id': video_id,
+            'display_id': display_id,
             'title': video_info.find('title').text,
             'url': manifest_url,
             'ext': 'flv',
             'thumbnail': self._og_search_thumbnail(webpage),
-            'upload_date': ''.join(video_info.find('time_date').text.split('.')[::-1]),
+            'upload_date': unified_strdate(video_info.find('time_date').text),
         }

From 6c1b0c0ed2678198e3196e2acf32f63806d4ccb6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 15 Jul 2015 00:01:41 +0600
Subject: [PATCH 1885/6219] [dfb] Extract formats

---
 youtube_dl/extractor/dfb.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/dfb.py b/youtube_dl/extractor/dfb.py
index f1df9b9a0..263532cc6 100644
--- a/youtube_dl/extractor/dfb.py
+++ b/youtube_dl/extractor/dfb.py
@@ -37,13 +37,13 @@ class DFBIE(InfoExtractor):
             self._proto_relative_url(video_info.find('url').text.strip()), display_id)
         token_el = f4m_info.find('token')
         manifest_url = token_el.attrib['url'] + '?' + 'hdnea=' + token_el.attrib['auth'] + '&hdcore=3.2.0'
+        formats = self._extract_f4m_formats(manifest_url, display_id)
 
         return {
             'id': video_id,
             'display_id': display_id,
             'title': video_info.find('title').text,
-            'url': manifest_url,
-            'ext': 'flv',
             'thumbnail': self._og_search_thumbnail(webpage),
             'upload_date': unified_strdate(video_info.find('time_date').text),
+            'formats': formats,
         }

From b062d94eefcc9327af5e60f3280cedaff9e40ac8 Mon Sep 17 00:00:00 2001
From: "Sergey M." <dstftw@gmail.com>
Date: Wed, 15 Jul 2015 03:08:36 +0600
Subject: [PATCH 1886/6219] [README.md] Clarify authentication with .netrc file

---
 README.md | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/README.md b/README.md
index 9779c2058..a2cc89cdb 100644
--- a/README.md
+++ b/README.md
@@ -238,6 +238,26 @@ which means you can modify it, redistribute it or use it however you like.
 
 You can configure youtube-dl by placing default arguments (such as `--extract-audio --no-mtime` to always extract the audio and not copy the mtime) into `/etc/youtube-dl.conf` and/or `~/.config/youtube-dl/config`. On Windows, the configuration file locations are `%APPDATA%\youtube-dl\config.txt` and `C:\Users\<user name>\youtube-dl.conf`.
 
+### Authentication with `.netrc` file ###
+
+You may also want to configure automatic credentials storage for extractors that support authentication (by providing login and password with `--username` and `--password`) in order not to pass credentials as command line arguments on every youtube-dl execution and prevent tracking plain text passwords in shell command history. You can achieve this using [`.netrc` file](http://stackoverflow.com/tags/.netrc/info) on per extractor basis. For that you will need to create `.netrc` file in your `$HOME` and restrict permissions to read/write by you only:
+```
+touch $HOME/.netrc
+chmod a-rwx,u+rw $HOME/.netrc
+```
+After that you can add credentials for extractor in the following format, where *extractor* is the name of extractor in lowercase:
+```
+machine <extractor> login <login> password <password>
+```
+For example:
+```
+machine youtube login myaccount@gmail.com password my_youtube_password
+machine twitch login my_twitch_account_name password my_twitch_password
+```
+To activate authentication with `.netrc` file you should pass `--netrc` to youtube-dl or to place it in [configuration file](#configuration).
+
+On Windows you may also need to setup `%HOME%` environment variable manually.
+
 # OUTPUT TEMPLATE
 
 The `-o` option allows users to indicate a template for the output file names. The basic usage is not to set any template arguments when downloading a single file, like in `youtube-dl -o funny_video.flv "http://some/video"`. However, it may contain special sequences that will be replaced when downloading each video. The special sequences have the format `%(NAME)s`. To clarify, that is a percent symbol followed by a name in parenthesis, followed by a lowercase S. Allowed names are:

From 2af0f87c8b56567e0254aae7a1ccbedb04413b1c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 15 Jul 2015 23:32:52 +0600
Subject: [PATCH 1887/6219] [prosiebensat1] Fix extraction (Closes #6215)

---
 youtube_dl/extractor/prosiebensat1.py | 19 +++++++++++++++----
 1 file changed, 15 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/prosiebensat1.py b/youtube_dl/extractor/prosiebensat1.py
index 536a42dc8..22efa903f 100644
--- a/youtube_dl/extractor/prosiebensat1.py
+++ b/youtube_dl/extractor/prosiebensat1.py
@@ -9,8 +9,9 @@ from ..compat import (
     compat_urllib_parse,
 )
 from ..utils import (
-    unified_strdate,
+    fix_xml_ampersands,
     int_or_none,
+    unified_strdate,
 )
 
 
@@ -208,7 +209,7 @@ class ProSiebenSat1IE(InfoExtractor):
         clip_id = self._html_search_regex(self._CLIPID_REGEXES, webpage, 'clip id')
 
         access_token = 'prosieben'
-        client_name = 'kolibri-1.12.6'
+        client_name = 'kolibri-2.0.19-splec4'
         client_location = url
 
         videos_api_url = 'http://vas.sim-technik.de/vas/live/v2/videos?%s' % compat_urllib_parse.urlencode({
@@ -275,8 +276,9 @@ class ProSiebenSat1IE(InfoExtractor):
 
         for source in urls_sources:
             protocol = source['protocol']
+            source_url = source['url']
             if protocol == 'rtmp' or protocol == 'rtmpe':
-                mobj = re.search(r'^(?P<url>rtmpe?://[^/]+)/(?P<path>.+)$', source['url'])
+                mobj = re.search(r'^(?P<url>rtmpe?://[^/]+)/(?P<path>.+)$', source_url)
                 if not mobj:
                     continue
                 path = mobj.group('path')
@@ -293,9 +295,18 @@ class ProSiebenSat1IE(InfoExtractor):
                     'ext': 'mp4',
                     'format_id': '%s_%s' % (source['cdn'], source['bitrate']),
                 })
+            elif 'f4mgenerator' in source_url:
+                manifest = self._download_xml(
+                    source_url, clip_id, 'Downloading generated f4m manifest',
+                    transform_source=lambda s: fix_xml_ampersands(s).strip())
+                for media in manifest.findall('./{http://ns.adobe.com/f4m/2.0}media'):
+                    manifest_url = media.get('href')
+                    if manifest_url:
+                        formats.extend(self._extract_f4m_formats(
+                            manifest_url, clip_id, f4m_id='hds'))
             else:
                 formats.append({
-                    'url': source['url'],
+                    'url': source_url,
                     'vbr': fix_bitrate(source['bitrate']),
                 })
 

From 97f4aecfc1c5fd446e1d3edd37e49aafe246fe0b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 16 Jul 2015 01:14:08 +0600
Subject: [PATCH 1888/6219] [extractor/common] Handle malformed f4m manifests

---
 youtube_dl/extractor/common.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 3a396c0b0..f8a5ecced 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -28,6 +28,7 @@ from ..utils import (
     clean_html,
     compiled_regex_type,
     ExtractorError,
+    fix_xml_ampersands,
     float_or_none,
     int_or_none,
     RegexNotFoundError,
@@ -837,7 +838,10 @@ class InfoExtractor(object):
     def _extract_f4m_formats(self, manifest_url, video_id, preference=None, f4m_id=None):
         manifest = self._download_xml(
             manifest_url, video_id, 'Downloading f4m manifest',
-            'Unable to download f4m manifest')
+            'Unable to download f4m manifest',
+            # Some manifests may be malformed, e.g. prosiebensat1 generated manifests
+            # (see https://github.com/rg3/youtube-dl/issues/6215#issuecomment-121704244)
+            transform_source=lambda s: fix_xml_ampersands(s).strip())
 
         formats = []
         manifest_version = '1.0'

From cc357c4db8112ff6736a227b47fb9527d327797f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 16 Jul 2015 01:14:52 +0600
Subject: [PATCH 1889/6219] [extractor/common] Properly handle full URLs

---
 youtube_dl/extractor/common.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index f8a5ecced..78e5cf8d0 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -851,8 +851,10 @@ class InfoExtractor(object):
             media_nodes = manifest.findall('{http://ns.adobe.com/f4m/2.0}media')
         for i, media_el in enumerate(media_nodes):
             if manifest_version == '2.0':
-                manifest_url = ('/'.join(manifest_url.split('/')[:-1]) + '/' +
-                                (media_el.attrib.get('href') or media_el.attrib.get('url')))
+                media_url = media_el.attrib.get('href') or media_el.attrib['url']
+                manifest_url = (
+                    media_url if media_url.startswith('http://') or media_url.startswith('https://')
+                    else ('/'.join(manifest_url.split('/')[:-1]) + '/' + media_url))
             tbr = int_or_none(media_el.attrib.get('bitrate'))
             formats.append({
                 'format_id': '-'.join(filter(None, [f4m_id, compat_str(i if tbr is None else tbr)])),

From 70f0f5a8ca53d4426fc079b3ab46e9d4a8e81ea4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 16 Jul 2015 01:15:15 +0600
Subject: [PATCH 1890/6219] [extractor/common] Recursively extract child f4m
 manifests

---
 youtube_dl/extractor/common.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 78e5cf8d0..e3c610aa4 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -27,6 +27,7 @@ from ..utils import (
     bug_reports_message,
     clean_html,
     compiled_regex_type,
+    determine_ext,
     ExtractorError,
     fix_xml_ampersands,
     float_or_none,
@@ -855,6 +856,13 @@ class InfoExtractor(object):
                 manifest_url = (
                     media_url if media_url.startswith('http://') or media_url.startswith('https://')
                     else ('/'.join(manifest_url.split('/')[:-1]) + '/' + media_url))
+                # If media_url is itself a f4m manifest do the recursive extraction
+                # since bitrates in parent manifest (this one) and media_url manifest
+                # may differ leading to inability to resolve the format by requested
+                # bitrate in f4m downloader
+                if determine_ext(manifest_url) == 'f4m':
+                    formats.extend(self._extract_f4m_formats(manifest_url, video_id, preference, f4m_id))
+                    continue
             tbr = int_or_none(media_el.attrib.get('bitrate'))
             formats.append({
                 'format_id': '-'.join(filter(None, [f4m_id, compat_str(i if tbr is None else tbr)])),

From f01f731107010e0c10fc94782daa7a3ba543e92a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 16 Jul 2015 01:15:47 +0600
Subject: [PATCH 1891/6219] [prosiebensat1] Use generic f4m manifest extraction

---
 youtube_dl/extractor/prosiebensat1.py | 13 +++----------
 1 file changed, 3 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/prosiebensat1.py b/youtube_dl/extractor/prosiebensat1.py
index 22efa903f..2f9d95800 100644
--- a/youtube_dl/extractor/prosiebensat1.py
+++ b/youtube_dl/extractor/prosiebensat1.py
@@ -9,7 +9,7 @@ from ..compat import (
     compat_urllib_parse,
 )
 from ..utils import (
-    fix_xml_ampersands,
+    determine_ext,
     int_or_none,
     unified_strdate,
 )
@@ -295,15 +295,8 @@ class ProSiebenSat1IE(InfoExtractor):
                     'ext': 'mp4',
                     'format_id': '%s_%s' % (source['cdn'], source['bitrate']),
                 })
-            elif 'f4mgenerator' in source_url:
-                manifest = self._download_xml(
-                    source_url, clip_id, 'Downloading generated f4m manifest',
-                    transform_source=lambda s: fix_xml_ampersands(s).strip())
-                for media in manifest.findall('./{http://ns.adobe.com/f4m/2.0}media'):
-                    manifest_url = media.get('href')
-                    if manifest_url:
-                        formats.extend(self._extract_f4m_formats(
-                            manifest_url, clip_id, f4m_id='hds'))
+            elif 'f4mgenerator' in source_url or determine_ext(source_url) == 'f4m':
+                formats.extend(self._extract_f4m_formats(source_url, clip_id))
             else:
                 formats.append({
                     'url': source_url,

From 31c746e5dc46491f997eca757c5e35842f04cb59 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 16 Jul 2015 01:25:33 +0600
Subject: [PATCH 1892/6219] [extractor/common] Keep going in some media_url is
 missing

---
 youtube_dl/extractor/common.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index e3c610aa4..271bf8596 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -852,7 +852,9 @@ class InfoExtractor(object):
             media_nodes = manifest.findall('{http://ns.adobe.com/f4m/2.0}media')
         for i, media_el in enumerate(media_nodes):
             if manifest_version == '2.0':
-                media_url = media_el.attrib.get('href') or media_el.attrib['url']
+                media_url = media_el.attrib.get('href') or media_el.attrib.get('url')
+                if not media_url:
+                    continue
                 manifest_url = (
                     media_url if media_url.startswith('http://') or media_url.startswith('https://')
                     else ('/'.join(manifest_url.split('/')[:-1]) + '/' + media_url))

From ab9b890b524a49a9ffa4c8ac7243cd8afc15d270 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 16 Jul 2015 02:23:07 +0600
Subject: [PATCH 1893/6219] [prosiebensat1] Clarify test purpose

---
 youtube_dl/extractor/prosiebensat1.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/prosiebensat1.py b/youtube_dl/extractor/prosiebensat1.py
index 2f9d95800..fec008ce7 100644
--- a/youtube_dl/extractor/prosiebensat1.py
+++ b/youtube_dl/extractor/prosiebensat1.py
@@ -22,6 +22,11 @@ class ProSiebenSat1IE(InfoExtractor):
 
     _TESTS = [
         {
+            # Tests changes introduced in https://github.com/rg3/youtube-dl/pull/6242
+            # in response to fixing https://github.com/rg3/youtube-dl/issues/6215:
+            # - malformed f4m manifest support
+            # - proper handling of URLs starting with `https?://` in 2.0 manifests
+            # - recursive child f4m manifests extraction
             'url': 'http://www.prosieben.de/tv/circus-halligalli/videos/218-staffel-2-episode-18-jahresrueckblick-ganze-folge',
             'info_dict': {
                 'id': '2104602',

From e37c932fca29d93af77b7a47cccc9bb8578e3163 Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Wed, 15 Jul 2015 15:13:56 -0500
Subject: [PATCH 1894/6219] compat_urllib_parse_unquote: crash fix: only decode
 valid hex

on python 2 the following has a { "crash_rate": "100%" } of the time
as it tries to parse '" ' as hex.
---
 youtube_dl/compat.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index c3783337a..1f4ccf443 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -94,6 +94,8 @@ except ImportError:
             try:
                 if not item:
                     raise ValueError
+                if not re.match('[0-9a-fA-F][0-9a-fA-F]',item[:2]):
+                    raise ValueError
                 pct_sequence += item[:2].decode('hex')
                 rest = item[2:]
                 if not rest:

From 45eedbe58c8ab6344f11f1e1376d01648c1967ee Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Wed, 15 Jul 2015 15:30:47 -0500
Subject: [PATCH 1895/6219] Generic: use compat_urllib_parse_unquote to prevent
 utf8 mangling of the entire page in python 2.

-requires- fixed compat_urllib_parse_unquote

example - the following will save with a mangled playlist title,
 instead of the kanji for 'tsunami'. This affects all utf8encoded
 urls as well

youtube-dl -f18 -o '%(playlist_title)s-%(title)s.%(ext)s' \
  https://gist.githubusercontent.com/atomicdryad/fcb97465e6060fc519e1/raw/61c14c1e3a4985471dcf56c281d24d7e781a4e0e/tsunami.html
---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 392ad3648..fc1bf2b6e 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1115,7 +1115,7 @@ class GenericIE(InfoExtractor):
         # Sometimes embedded video player is hidden behind percent encoding
         # (e.g. https://github.com/rg3/youtube-dl/issues/2448)
         # Unescaping the whole page allows to handle those cases in a generic way
-        webpage = compat_urllib_parse.unquote(webpage)
+        webpage = compat_urllib_parse_unquote(webpage)
 
         # it's tempting to parse this further, but you would
         # have to take into account all the variations like

From e118031ef827e851e537daa5b439cf5c249ca88d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 16 Jul 2015 23:21:04 +0600
Subject: [PATCH 1896/6219] [npo] Extend _VALID_URL to support ntr.nl (Closes
 #6248)

---
 youtube_dl/extractor/npo.py | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index 62d12b7a6..1c823ec7f 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -37,8 +37,9 @@ class NPOBaseIE(InfoExtractor):
 
 
 class NPOIE(NPOBaseIE):
-    IE_NAME = 'npo.nl'
-    _VALID_URL = r'https?://(?:www\.)?npo\.nl/(?!live|radio)[^/]+/[^/]+/(?P<id>[^/?]+)'
+    IE_NAME = 'npo'
+    IE_DESC = 'npo.nl and ntr.nl'
+    _VALID_URL = r'https?://(?:www\.)?(?:npo|ntr)\.nl/(?!live|radio)(?:[^/]+/){2,}(?P<id>[^/?#]+)'
 
     _TESTS = [
         {
@@ -100,6 +101,18 @@ class NPOIE(NPOBaseIE):
                 'title': 'Hoe gaat Europa verder na Parijs?',
             },
         },
+        {
+            'url': 'http://www.ntr.nl/Aap-Poot-Pies/27/detail/Aap-poot-pies/VPWON_1233944#content',
+            'md5': '01c6a2841675995da1f0cf776f03a9c3',
+            'info_dict': {
+                'id': 'VPWON_1233944',
+                'ext': 'm4v',
+                'title': 'Aap, poot, pies',
+                'description': 'md5:c9c8005d1869ae65b858e82c01a91fde',
+                'upload_date': '20150508',
+                'duration': 599,
+            },
+        }
     ]
 
     def _real_extract(self, url):

From 525daedd5a092b0f5329952eee99a7dac5537433 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 16 Jul 2015 23:54:43 +0600
Subject: [PATCH 1897/6219] [npo] Add support for omroepwnl fragments

---
 youtube_dl/extractor/npo.py | 34 +++++++++++++++++++++++++++++++++-
 1 file changed, 33 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index 1c823ec7f..a5162c0c6 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -1,6 +1,12 @@
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
+from ..compat import (
+    compat_urllib_request,
+    compat_urllib_parse,
+)
 from ..utils import (
     fix_xml_ampersands,
     parse_duration,
@@ -39,7 +45,16 @@ class NPOBaseIE(InfoExtractor):
 class NPOIE(NPOBaseIE):
     IE_NAME = 'npo'
     IE_DESC = 'npo.nl and ntr.nl'
-    _VALID_URL = r'https?://(?:www\.)?(?:npo|ntr)\.nl/(?!live|radio)(?:[^/]+/){2,}(?P<id>[^/?#]+)'
+    _VALID_URL = r'''(?x)
+                    https?://
+                        (?:www\.)?
+                        (?:
+                            npo\.nl/(?!live|radio)(?:[^/]+/){2}|
+                            ntr\.nl/(?:[^/]+/){2,}|
+                            omroepwnl\.nl/video/fragment/[^/]+__
+                        )
+                        (?P<id>[^/?#]+)
+                '''
 
     _TESTS = [
         {
@@ -112,6 +127,18 @@ class NPOIE(NPOBaseIE):
                 'upload_date': '20150508',
                 'duration': 599,
             },
+        },
+        {
+            'url': 'http://www.omroepwnl.nl/video/fragment/vandaag-de-dag-verkiezingen__POMS_WNL_853698',
+            'md5': 'd30cd8417b8b9bca1fdff27428860d08',
+            'info_dict': {
+                'id': 'POW_00996502',
+                'ext': 'm4v',
+                'title': '''"Dit is wel een 'landslide'..."''',
+                'description': 'md5:f8d66d537dfb641380226e31ca57b8e8',
+                'upload_date': '20150508',
+                'duration': 462,
+            },
         }
     ]
 
@@ -127,6 +154,11 @@ class NPOIE(NPOBaseIE):
             transform_source=strip_jsonp,
         )
 
+        # For some videos actual video id (prid) is different (e.g. for
+        # http://www.omroepwnl.nl/video/fragment/vandaag-de-dag-verkiezingen__POMS_WNL_853698
+        # video id is POMS_WNL_853698 but prid is POW_00996502)
+        video_id = metadata.get('prid') or video_id
+
         token = self._get_token(video_id)
 
         formats = []

From 50ea2bb20d3a3e219910e87b8b30fc79ce534595 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 16 Jul 2015 23:56:57 +0600
Subject: [PATCH 1898/6219] [npo] Update test

---
 youtube_dl/extractor/npo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index a5162c0c6..cf6a388e5 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -87,7 +87,7 @@ class NPOIE(NPOBaseIE):
                 'id': 'VPWON_1169289',
                 'ext': 'm4v',
                 'title': 'Tegenlicht',
-                'description': 'md5:d6476bceb17a8c103c76c3b708f05dd1',
+                'description': 'md5:52cf4eefbc96fffcbdc06d024147abea',
                 'upload_date': '20130225',
                 'duration': 3000,
             },

From 03f32a7eadf9d832aef55673edf38023a8daff95 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 00:14:38 +0600
Subject: [PATCH 1899/6219] [wnl] Add extractor for omroepwnl playlists

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/npo.py      | 45 +++++++++++++++++++++++++++-----
 2 files changed, 40 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 3f4f23521..1d55275dc 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -390,6 +390,7 @@ from .npo import (
     NPORadioIE,
     NPORadioFragmentIE,
     TegenlichtVproIE,
+    WNLIE
 )
 from .nrk import (
     NRKIE,
diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index cf6a388e5..c6bf7619d 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -46,12 +46,15 @@ class NPOIE(NPOBaseIE):
     IE_NAME = 'npo'
     IE_DESC = 'npo.nl and ntr.nl'
     _VALID_URL = r'''(?x)
-                    https?://
-                        (?:www\.)?
-                        (?:
-                            npo\.nl/(?!live|radio)(?:[^/]+/){2}|
-                            ntr\.nl/(?:[^/]+/){2,}|
-                            omroepwnl\.nl/video/fragment/[^/]+__
+                    (?:
+                        npo:|
+                        https?://
+                            (?:www\.)?
+                            (?:
+                                npo\.nl/(?!live|radio)(?:[^/]+/){2}|
+                                ntr\.nl/(?:[^/]+/){2,}|
+                                omroepwnl\.nl/video/fragment/[^/]+__
+                            )
                         )
                         (?P<id>[^/?#]+)
                 '''
@@ -426,3 +429,33 @@ class TegenlichtVproIE(NPOIE):
         info_page = self._download_json(
             'http://rs.vpro.nl/v2/api/media/%s.json' % urn, name)
         return self._get_info(info_page['mid'])
+
+
+class WNLIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?omroepwnl\.nl/video/detail/(?P<id>[^/]+)__\d+'
+
+    _TEST = {
+        'url': 'http://www.omroepwnl.nl/video/detail/vandaag-de-dag-6-mei__060515',
+        'info_dict': {
+            'id': 'vandaag-de-dag-6-mei',
+            'title': 'Vandaag de Dag 6 mei',
+        },
+        'playlist_count': 4,
+    }
+
+    def _real_extract(self, url):
+        playlist_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, playlist_id)
+
+        entries = [
+            self.url_result('npo:%s' % video_id, 'NPO')
+            for video_id, part in re.findall(
+                r'<a[^>]+href="([^"]+)"[^>]+class="js-mid"[^>]*>(Deel \d+)', webpage)
+        ]
+
+        playlist_title = self._html_search_regex(
+            r'(?s)<h1[^>]+class="subject"[^>]*>(.+?)</h1>',
+            webpage, 'playlist title')
+
+        return self.playlist_result(entries, playlist_id, playlist_title)

From 611ac379bb466267aded6726f9c85e79b08168c4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 00:34:24 +0600
Subject: [PATCH 1900/6219] [vpro] Fix extraction and add support for vpro
 playlists

---
 youtube_dl/extractor/__init__.py |  2 +-
 youtube_dl/extractor/npo.py      | 35 +++++++++++++++++++++++---------
 2 files changed, 26 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 1d55275dc..06f21064b 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -389,7 +389,7 @@ from .npo import (
     NPOLiveIE,
     NPORadioIE,
     NPORadioFragmentIE,
-    TegenlichtVproIE,
+    VPROIE,
     WNLIE
 )
 from .nrk import (
diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index c6bf7619d..28d5c90b3 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -404,9 +404,8 @@ class NPORadioFragmentIE(InfoExtractor):
         }
 
 
-class TegenlichtVproIE(NPOIE):
-    IE_NAME = 'tegenlicht.vpro.nl'
-    _VALID_URL = r'https?://tegenlicht\.vpro\.nl/afleveringen/.*?'
+class VPROIE(NPOIE):
+    _VALID_URL = r'https?://(?:www\.)?(?:tegenlicht\.)?vpro\.nl/(?:[^/]+/){2,}(?P<id>[^/]+)\.html'
 
     _TESTS = [
         {
@@ -416,19 +415,35 @@ class TegenlichtVproIE(NPOIE):
                 'id': 'VPWON_1169289',
                 'ext': 'm4v',
                 'title': 'Tegenlicht',
-                'description': 'md5:d6476bceb17a8c103c76c3b708f05dd1',
+                'description': 'md5:52cf4eefbc96fffcbdc06d024147abea',
                 'upload_date': '20130225',
             },
         },
+        {
+            'url': 'http://www.vpro.nl/programmas/2doc/2015/sergio-herman.html',
+            'info_dict': {
+                'id': 'sergio-herman',
+                'title': 'Sergio Herman: Fucking perfect',
+            },
+            'playlist_count': 2,
+        }
     ]
 
     def _real_extract(self, url):
-        name = url_basename(url)
-        webpage = self._download_webpage(url, name)
-        urn = self._html_search_meta('mediaurn', webpage)
-        info_page = self._download_json(
-            'http://rs.vpro.nl/v2/api/media/%s.json' % urn, name)
-        return self._get_info(info_page['mid'])
+        playlist_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, playlist_id)
+
+        entries = [
+            self.url_result('npo:%s' % video_id, 'NPO')
+            for video_id in re.findall(r'data-media-id="([^"]+)"', webpage)
+        ]
+
+        playlist_title = self._search_regex(
+            r'<title>\s*([^>]+?)\s*-\s*Teledoc\s*-\s*VPRO\s*</title>',
+            webpage, 'playlist title', default=None) or self._og_search_title(webpage)
+
+        return self.playlist_result(entries, playlist_id, playlist_title)
 
 
 class WNLIE(InfoExtractor):

From 5ba761eb854d6e415b3ab542293cb31c073dc0f3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 00:39:22 +0600
Subject: [PATCH 1901/6219] [npo] Prefer aflevering_titel over titel

---
 youtube_dl/extractor/npo.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index 28d5c90b3..91adb23f0 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -234,7 +234,9 @@ class NPOIE(NPOBaseIE):
 
         return {
             'id': video_id,
-            'title': metadata['titel'],
+            # prefer aflevering_titel if any since titel may be too generic, e.g.
+            # http://tegenlicht.vpro.nl/afleveringen/2014-2015/access-to-africa.html
+            'title': metadata.get('aflevering_titel') or metadata['titel'],
             'description': metadata['info'],
             'thumbnail': metadata.get('images', [{'url': None}])[-1]['url'],
             'upload_date': unified_strdate(metadata.get('gidsdatum')),
@@ -414,7 +416,7 @@ class VPROIE(NPOIE):
             'info_dict': {
                 'id': 'VPWON_1169289',
                 'ext': 'm4v',
-                'title': 'Tegenlicht',
+                'title': 'De toekomst komt uit Afrika',
                 'description': 'md5:52cf4eefbc96fffcbdc06d024147abea',
                 'upload_date': '20130225',
             },

From 536b0700b03f0b29a1025be0b7753253bd627d6b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 00:40:04 +0600
Subject: [PATCH 1902/6219] [npo] Allow missing description

---
 youtube_dl/extractor/npo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index 91adb23f0..e733d96f6 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -237,7 +237,7 @@ class NPOIE(NPOBaseIE):
             # prefer aflevering_titel if any since titel may be too generic, e.g.
             # http://tegenlicht.vpro.nl/afleveringen/2014-2015/access-to-africa.html
             'title': metadata.get('aflevering_titel') or metadata['titel'],
-            'description': metadata['info'],
+            'description': metadata.get('info'),
             'thumbnail': metadata.get('images', [{'url': None}])[-1]['url'],
             'upload_date': unified_strdate(metadata.get('gidsdatum')),
             'duration': parse_duration(metadata.get('tijdsduur')),

From 574f42d79a8596ceda681b205e19e766e7bab046 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 00:53:12 +0600
Subject: [PATCH 1903/6219] [vpro] Improve playlist extraction

---
 youtube_dl/extractor/npo.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index e733d96f6..583ed3e14 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -428,6 +428,15 @@ class VPROIE(NPOIE):
                 'title': 'Sergio Herman: Fucking perfect',
             },
             'playlist_count': 2,
+        },
+        {
+            # playlist with youtube embed
+            'url': 'http://www.vpro.nl/programmas/2doc/2015/education-education.html',
+            'info_dict': {
+                'id': 'education-education',
+                'title': '2Doc',
+            },
+            'playlist_count': 2,
         }
     ]
 
@@ -437,7 +446,7 @@ class VPROIE(NPOIE):
         webpage = self._download_webpage(url, playlist_id)
 
         entries = [
-            self.url_result('npo:%s' % video_id, 'NPO')
+            self.url_result('npo:%s' % video_id if not video_id.startswith('http') else video_id)
             for video_id in re.findall(r'data-media-id="([^"]+)"', webpage)
         ]
 

From 1540119723e7e2195a47d659993a6a3bcc02d3e5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 00:54:05 +0600
Subject: [PATCH 1904/6219] [npo] Remove unused imports

---
 youtube_dl/extractor/npo.py | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index 583ed3e14..f5ffe1231 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -3,17 +3,12 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_request,
-    compat_urllib_parse,
-)
 from ..utils import (
     fix_xml_ampersands,
     parse_duration,
     qualities,
     strip_jsonp,
     unified_strdate,
-    url_basename,
 )
 
 

From 23fc384f2c3cf9afd41fd7e033fe0823d0fc5fa5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 01:28:52 +0600
Subject: [PATCH 1905/6219] [npo] Compound title

---
 youtube_dl/extractor/npo.py | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index f5ffe1231..0c2d02c10 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -72,7 +72,7 @@ class NPOIE(NPOBaseIE):
             'info_dict': {
                 'id': 'VARA_101191800',
                 'ext': 'm4v',
-                'title': 'De Mega Mike & Mega Thomas show',
+                'title': 'De Mega Mike & Mega Thomas show: The best of.',
                 'description': 'md5:3b74c97fc9d6901d5a665aac0e5400f4',
                 'upload_date': '20090227',
                 'duration': 2400,
@@ -84,7 +84,7 @@ class NPOIE(NPOBaseIE):
             'info_dict': {
                 'id': 'VPWON_1169289',
                 'ext': 'm4v',
-                'title': 'Tegenlicht',
+                'title': 'Tegenlicht: De toekomst komt uit Afrika',
                 'description': 'md5:52cf4eefbc96fffcbdc06d024147abea',
                 'upload_date': '20130225',
                 'duration': 3000,
@@ -157,6 +157,13 @@ class NPOIE(NPOBaseIE):
         # video id is POMS_WNL_853698 but prid is POW_00996502)
         video_id = metadata.get('prid') or video_id
 
+        # titel is too generic in some cases so utilize aflevering_titel as well
+        # when available (e.g. http://tegenlicht.vpro.nl/afleveringen/2014-2015/access-to-africa.html)
+        title = metadata['titel']
+        sub_title = metadata.get('aflevering_titel')
+        if sub_title and sub_title != title:
+            title += ': %s' % sub_title
+
         token = self._get_token(video_id)
 
         formats = []
@@ -229,9 +236,7 @@ class NPOIE(NPOBaseIE):
 
         return {
             'id': video_id,
-            # prefer aflevering_titel if any since titel may be too generic, e.g.
-            # http://tegenlicht.vpro.nl/afleveringen/2014-2015/access-to-africa.html
-            'title': metadata.get('aflevering_titel') or metadata['titel'],
+            'title': title,
             'description': metadata.get('info'),
             'thumbnail': metadata.get('images', [{'url': None}])[-1]['url'],
             'upload_date': unified_strdate(metadata.get('gidsdatum')),

From a38436e8898b6eca29d9279346e6e2136ec0bc8f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 17 Jul 2015 12:02:49 +0800
Subject: [PATCH 1906/6219] [extractor/common] Add 'transform_source' parameter
 to _extract_f4m_formats()

---
 youtube_dl/extractor/common.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 271bf8596..5a2d0d995 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -836,13 +836,14 @@ class InfoExtractor(object):
         self.to_screen(msg)
         time.sleep(timeout)
 
-    def _extract_f4m_formats(self, manifest_url, video_id, preference=None, f4m_id=None):
+    def _extract_f4m_formats(self, manifest_url, video_id, preference=None, f4m_id=None,
+                             transform_source=lambda s: fix_xml_ampersands(s).strip()):
         manifest = self._download_xml(
             manifest_url, video_id, 'Downloading f4m manifest',
             'Unable to download f4m manifest',
             # Some manifests may be malformed, e.g. prosiebensat1 generated manifests
             # (see https://github.com/rg3/youtube-dl/issues/6215#issuecomment-121704244)
-            transform_source=lambda s: fix_xml_ampersands(s).strip())
+            transform_source=transform_source)
 
         formats = []
         manifest_version = '1.0'

From c9c854cea7fa5992356dee5eab0d3615b4d40dc6 Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Fri, 17 Jul 2015 01:31:29 -0500
Subject: [PATCH 1907/6219] replace old compat_urllib_parse_unquote with
 backport from python3's function

  * required unquote_to_bytes function ported as well
    (uses .decode('hex') instead of dynamically populated _hextobyte global)
  * required implicit conversion to bytes and/or unicode in places due to
    differing type assumptions in p3
---
 youtube_dl/compat.py | 75 +++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 74 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 1f4ccf443..2fd2278aa 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -74,10 +74,81 @@ try:
 except ImportError:
     import BaseHTTPServer as compat_http_server
 
+from pprint import (pprint, pformat)
+
+
+def dprint(fmt):
+    sys.stderr.write(pformat(fmt) + "\n")
+
 try:
     from urllib.parse import unquote as compat_urllib_parse_unquote
 except ImportError:
-    def compat_urllib_parse_unquote(string, encoding='utf-8', errors='replace'):
+    def compat_urllib_parse_unquote_to_bytes(string):
+        """unquote_to_bytes('abc%20def') -> b'abc def'."""
+        # Note: strings are encoded as UTF-8. This is only an issue if it contains
+        # unescaped non-ASCII characters, which URIs should not.
+        if not string:
+            # Is it a string-like object?
+            string.split
+            return b''
+        if isinstance(string, str):
+            string = string.encode('utf-8')
+            # string = encode('utf-8')
+
+        # python3 -> 2: must implicitly convert to bits
+        bits = bytes(string).split(b'%')
+
+        if len(bits) == 1:
+            return string
+        res = [bits[0]]
+        append = res.append
+
+        for item in bits[1:]:
+            try:
+                append(item[:2].decode('hex'))
+                append(item[2:])
+            except:
+                append(b'%')
+                append(item)
+        return b''.join(res)
+
+    compat_urllib_parse_asciire = re.compile('([\x00-\x7f]+)')
+
+    def new_compat_urllib_parse_unquote(string, encoding='utf-8', errors='replace'):
+        """Replace %xx escapes by their single-character equivalent. The optional
+        encoding and errors parameters specify how to decode percent-encoded
+        sequences into Unicode characters, as accepted by the bytes.decode()
+        method.
+        By default, percent-encoded sequences are decoded with UTF-8, and invalid
+        sequences are replaced by a placeholder character.
+
+        unquote('abc%20def') -> 'abc def'.
+        """
+
+        if '%' not in string:
+            string.split
+            return string
+        if encoding is None:
+            encoding = 'utf-8'
+        if errors is None:
+            errors = 'replace'
+
+        bits = compat_urllib_parse_asciire.split(string)
+        res = [bits[0]]
+        append = res.append
+        for i in range(1, len(bits), 2):
+            foo = compat_urllib_parse_unquote_to_bytes(bits[i])
+            foo = foo.decode(encoding, errors)
+            append(foo)
+
+            if bits[i + 1]:
+                bar = bits[i + 1]
+                if not isinstance(bar, unicode):
+                    bar = bar.decode('utf-8')
+                append(bar)
+        return ''.join(res)
+
+    def old_compat_urllib_parse_unquote(string, encoding='utf-8', errors='replace'):
         if string == '':
             return string
         res = string.split('%')
@@ -114,6 +185,8 @@ except ImportError:
             string += pct_sequence.decode(encoding, errors)
         return string
 
+    compat_urllib_parse_unquote = new_compat_urllib_parse_unquote
+
 try:
     compat_str = unicode  # Python 2
 except NameError:

From 851229a01f34129286a57d46f8a27b9bb5fd9a6b Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Fri, 17 Jul 2015 01:49:55 -0500
Subject: [PATCH 1908/6219] remove debugprint

---
 youtube_dl/compat.py | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 2fd2278aa..554e3d5db 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -74,12 +74,6 @@ try:
 except ImportError:
     import BaseHTTPServer as compat_http_server
 
-from pprint import (pprint, pformat)
-
-
-def dprint(fmt):
-    sys.stderr.write(pformat(fmt) + "\n")
-
 try:
     from urllib.parse import unquote as compat_urllib_parse_unquote
 except ImportError:

From a0f28f90fa277d9c00f0305624dea36a20b8066e Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Fri, 17 Jul 2015 01:50:43 -0500
Subject: [PATCH 1909/6219] remove kebab

---
 youtube_dl/compat.py | 41 +----------------------------------------
 1 file changed, 1 insertion(+), 40 deletions(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 554e3d5db..de9ba2c14 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -108,7 +108,7 @@ except ImportError:
 
     compat_urllib_parse_asciire = re.compile('([\x00-\x7f]+)')
 
-    def new_compat_urllib_parse_unquote(string, encoding='utf-8', errors='replace'):
+    def compat_urllib_parse_unquote(string, encoding='utf-8', errors='replace'):
         """Replace %xx escapes by their single-character equivalent. The optional
         encoding and errors parameters specify how to decode percent-encoded
         sequences into Unicode characters, as accepted by the bytes.decode()
@@ -142,45 +142,6 @@ except ImportError:
                 append(bar)
         return ''.join(res)
 
-    def old_compat_urllib_parse_unquote(string, encoding='utf-8', errors='replace'):
-        if string == '':
-            return string
-        res = string.split('%')
-        if len(res) == 1:
-            return string
-        if encoding is None:
-            encoding = 'utf-8'
-        if errors is None:
-            errors = 'replace'
-        # pct_sequence: contiguous sequence of percent-encoded bytes, decoded
-        pct_sequence = b''
-        string = res[0]
-        for item in res[1:]:
-            try:
-                if not item:
-                    raise ValueError
-                if not re.match('[0-9a-fA-F][0-9a-fA-F]',item[:2]):
-                    raise ValueError
-                pct_sequence += item[:2].decode('hex')
-                rest = item[2:]
-                if not rest:
-                    # This segment was just a single percent-encoded character.
-                    # May be part of a sequence of code units, so delay decoding.
-                    # (Stored in pct_sequence).
-                    continue
-            except ValueError:
-                rest = '%' + item
-            # Encountered non-percent-encoded characters. Flush the current
-            # pct_sequence.
-            string += pct_sequence.decode(encoding, errors) + rest
-            pct_sequence = b''
-        if pct_sequence:
-            # Flush the final pct_sequence
-            string += pct_sequence.decode(encoding, errors)
-        return string
-
-    compat_urllib_parse_unquote = new_compat_urllib_parse_unquote
-
 try:
     compat_str = unicode  # Python 2
 except NameError:

From 36da48798a28b8261d2f39f73f2522651d58a364 Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Fri, 17 Jul 2015 02:27:50 -0500
Subject: [PATCH 1910/6219] handle titles and captions set to ''

---
 youtube_dl/extractor/bbc.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 471d865d2..c910eb55a 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -497,11 +497,13 @@ class BBCNewsIE(BBCCoUkIE):
             programme_id = jent.get('externalId')
             xml_url = jent.get('href')
 
-            title = jent.get('caption',list_title)
+            title = jent.get('caption','')
+            if title == '':
+               title = list_title
 
             duration = parse_duration(jent.get('duration'))
             description = list_title
-            if jent.get('caption'):
+            if jent.get('caption', '') != '':
                description += ' - ' + jent.get('caption')
             thumbnail = None
             if jent.has_key('image'):

From a3bfddfa5ee33cf085b959536f1025c0aa53cc77 Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Fri, 17 Jul 2015 02:47:02 -0500
Subject: [PATCH 1911/6219] bbc.py: correct syntax

---
 youtube_dl/extractor/bbc.py | 106 ++++++++++++++++++------------------
 1 file changed, 53 insertions(+), 53 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index c910eb55a..c8f285165 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -397,14 +397,14 @@ class BBCNewsIE(BBCCoUkIE):
             'title': 'Russia stages massive WW2 parade despite Western boycott',
         },
         'playlist_count': 2,
-    },{
+    }, {
         'url': 'http://www.bbc.com/news/business-28299555',
         'info_dict': {
             'id': 'business-28299555',
             'title': 'Farnborough Airshow: Video highlights',
         },
         'playlist_count': 9,
-    },{
+    }, {
         'url': 'http://www.bbc.com/news/world-europe-32041533',
         'note': 'Video',
         'info_dict': {
@@ -419,7 +419,7 @@ class BBCNewsIE(BBCCoUkIE):
         'params': {
             'skip_download': True,
         }
-    },{
+    }, {
         'url': 'http://www.bbc.com/turkce/haberler/2015/06/150615_telabyad_kentin_cogu',
         'note': 'Video',
         'info_dict': {
@@ -434,7 +434,7 @@ class BBCNewsIE(BBCCoUkIE):
         'params': {
             'skip_download': True,
         }
-    },{
+    }, {
         'url': 'http://www.bbc.com/mundo/video_fotos/2015/06/150619_video_honduras_militares_hospitales_corrupcion_aw',
         'note': 'Video',
         'info_dict': {
@@ -459,88 +459,88 @@ class BBCNewsIE(BBCCoUkIE):
 
         pubdate = self._html_search_regex(r'"datePublished":\s*"(\d+-\d+-\d+)', webpage, 'date', default=None)
         if pubdate:
-           pubdate = pubdate.replace('-','')
+            pubdate = pubdate.replace('-', '')
 
         ret = []
         jsent = []
 
         # works with bbc.com/news/something-something-123456 articles
         jsent = map(
-           lambda m: self._parse_json(m,list_id),
-           re.findall(r"data-media-meta='({[^']+})'", webpage)
+            lambda m: self._parse_json(m, list_id),
+            re.findall(r"data-media-meta='({[^']+})'", webpage)
         )
 
         if len(jsent) == 0:
-           # http://www.bbc.com/news/video_and_audio/international
-           # and single-video articles
-           masset = self._html_search_regex(r'mediaAssetPage\.init\(\s*({.+?}), "/', webpage, 'mediaassets', default=None)
-           if masset:
-              jmasset = self._parse_json(masset,list_id)
-              for key, val in jmasset.get('videos',{}).items():
-                  for skey, sval in val.items():
-                      sval['id'] = skey
-                      jsent.append(sval)
+            # http://www.bbc.com/news/video_and_audio/international
+            # and single-video articles
+            masset = self._html_search_regex(r'mediaAssetPage\.init\(\s*({.+?}), "/', webpage, 'mediaassets', default=None)
+            if masset:
+                jmasset = self._parse_json(masset, list_id)
+                for key, val in jmasset.get('videos', {}).items():
+                    for skey, sval in val.items():
+                        sval['id'] = skey
+                        jsent.append(sval)
 
         if len(jsent) == 0:
-           # stubbornly generic extractor for {json with "image":{allvideoshavethis},etc}
-           # in http://www.bbc.com/news/video_and_audio/international
-           # prone to breaking if entries have sourceFiles list
-           jsent = map(
-               lambda m: self._parse_json(m,list_id),
-               re.findall(r"({[^{}]+image\":{[^}]+}[^}]+})", webpage)
-           )          
+            # stubbornly generic extractor for {json with "image":{allvideoshavethis},etc}
+            # in http://www.bbc.com/news/video_and_audio/international
+            # prone to breaking if entries have sourceFiles list
+            jsent = map(
+                lambda m: self._parse_json(m, list_id),
+                re.findall(r"({[^{}]+image\":{[^}]+}[^}]+})", webpage)
+            )
 
         if len(jsent) == 0:
-           raise ExtractorError('No video found', expected=True)
+            raise ExtractorError('No video found', expected=True)
 
         for jent in jsent:
             programme_id = jent.get('externalId')
             xml_url = jent.get('href')
 
-            title = jent.get('caption','')
+            title = jent.get('caption', '')
             if title == '':
-               title = list_title
+                title = list_title
 
             duration = parse_duration(jent.get('duration'))
             description = list_title
             if jent.get('caption', '') != '':
-               description += ' - ' + jent.get('caption')
+                description += ' - ' + jent.get('caption')
             thumbnail = None
-            if jent.has_key('image'):
-               thumbnail=jent['image'].get('href')
+            if jent.get('image') is not None:
+                thumbnail = jent['image'].get('href')
 
             formats = []
             subtitles = []
 
             if programme_id:
-               formats, subtitles = self._download_media_selector(programme_id)
-            elif jent.has_key('sourceFiles'):
-               # mediaselector not used at
-               # http://www.bbc.com/turkce/haberler/2015/06/150615_telabyad_kentin_cogu
-               for key, val in jent['sourceFiles'].items():
-                  formats.append( {
-                     'ext': val.get('encoding'),
-                     'url': val.get('url'),
-                     'filesize': int(val.get('filesize')),
-                     'format_id': key
-                  } )
+                formats, subtitles = self._download_media_selector(programme_id)
+            elif jent.get('sourceFiles') is not None:
+                # mediaselector not used at
+                # http://www.bbc.com/turkce/haberler/2015/06/150615_telabyad_kentin_cogu
+                for key, val in jent['sourceFiles'].items():
+                    formats.append({
+                        'ext': val.get('encoding'),
+                        'url': val.get('url'),
+                        'filesize': int(val.get('filesize')),
+                        'format_id': key
+                    })
             elif xml_url:
-               # Cheap fallback
-               # http://playlists.bbc.co.uk/news/(list_id)[ABC..]/playlist.sxml
-               xml = self._download_webpage(xml_url, programme_id, 'Downloading playlist.sxml for externalId (fallback)')
-               programme_id = self._search_regex(r'<mediator [^>]*identifier="(.+?)"', xml, 'playlist.sxml (externalId fallback)')
-               formats, subtitles = self._download_media_selector(programme_id)
+                # Cheap fallback
+                # http://playlists.bbc.co.uk/news/(list_id)[ABC..]/playlist.sxml
+                xml = self._download_webpage(xml_url, programme_id, 'Downloading playlist.sxml for externalId (fallback)')
+                programme_id = self._search_regex(r'<mediator [^>]*identifier="(.+?)"', xml, 'playlist.sxml (externalId fallback)')
+                formats, subtitles = self._download_media_selector(programme_id)
 
             if len(formats) == 0:
-               raise ExtractorError('unsupported json media entry.\n    '+str(jent)+'\n')
-               
+                raise ExtractorError('unsupported json media entry.\n    ' + str(jent) + '\n')
+
             self._sort_formats(formats)
 
-            id = jent.get('id') if programme_id == None else programme_id
-            if id == None:
-               id = 'NA'
+            id = jent.get('id') if programme_id is None else programme_id
+            if id is None:
+                id = 'NA'
 
-            ret.append( {
+            ret.append({
                 'id': id,
                 'uploader': 'BBC News',
                 'upload_date': pubdate,
@@ -550,8 +550,8 @@ class BBCNewsIE(BBCCoUkIE):
                 'duration': duration,
                 'formats': formats,
                 'subtitles': subtitles,
-            } )
+            })
 
         if len(ret) > 0:
-           return self.playlist_result(ret, list_id, list_title)
+            return self.playlist_result(ret, list_id, list_title)
         raise ExtractorError('No video found', expected=True)

From eb08081330f5ef52d66140589137ae1bb05eee5f Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Fri, 17 Jul 2015 02:57:08 -0500
Subject: [PATCH 1912/6219] democracynow: correct syntax

---
 youtube_dl/extractor/democracynow.py | 43 +++++++++-------------------
 1 file changed, 14 insertions(+), 29 deletions(-)

diff --git a/youtube_dl/extractor/democracynow.py b/youtube_dl/extractor/democracynow.py
index 1c9b36052..973bb437b 100644
--- a/youtube_dl/extractor/democracynow.py
+++ b/youtube_dl/extractor/democracynow.py
@@ -1,19 +1,7 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import json
-import time
-import hmac
-import hashlib
-import itertools
 import re
-from ..utils import (
-    ExtractorError,
-    int_or_none,
-    parse_age_limit,
-    parse_iso8601,
-)
-from ..compat import compat_urllib_request
 from .common import InfoExtractor
 
 
@@ -30,7 +18,7 @@ class DemocracynowIE(InfoExtractor):
             'uploader': 'Democracy Now',
             'upload_date': None,
         },
-     },{
+    }, {
         'url': 'http://www.democracynow.org/2015/7/3/this_flag_comes_down_today_bree',
         'info_dict': {
             'id': '2015-0703-001',
@@ -40,7 +28,6 @@ class DemocracynowIE(InfoExtractor):
             'uploader': 'Democracy Now',
             'upload_date': None,
         },
-
     }]
 
     def _real_extract(self, url):
@@ -49,7 +36,7 @@ class DemocracynowIE(InfoExtractor):
         if display_id == '':
             display_id = 'home'
         webpage = self._download_webpage(url, display_id)
-        re_desc = re.search(r'<meta property=.og:description. content=(["\'])(.+?)\1',webpage,re.DOTALL)
+        re_desc = re.search(r'<meta property=.og:description. content=(["\'])(.+?)\1', webpage, re.DOTALL)
         description = re_desc.group(2) if re_desc else ''
 
         jstr = self._search_regex(r'({.+?"related_video_xml".+?})', webpage, 'json', default=None)
@@ -57,30 +44,30 @@ class DemocracynowIE(InfoExtractor):
         video_id = None
         formats = []
         subtitles = {}
-        for key in ('caption_file','.......'):
+        for key in ('caption_file', '.......'):
             # ....... = pending vtt support that doesn't clobber srt 'chapter_file':
-            url = js.get(key,'')
-            if url == '' or url == None:
+            url = js.get(key, '')
+            if url == '' or url is None:
                 continue
-            if not re.match(r'^https?://',url):
+            if not re.match(r'^https?://', url):
                 url = base_host + url
-            ext = re.search(r'\.([^\.]+)$',url).group(1)
+            ext = re.search(r'\.([^\.]+)$', url).group(1)
             subtitles['eng'] = [{
                 'ext': ext,
                 'url': url,
             }]
         for key in ('file', 'audio'):
-            url = js.get(key,'')
-            if url == '' or url == None:
+            url = js.get(key, '')
+            if url == '' or url is None:
                 continue
-            if not re.match(r'^https?://',url):
+            if not re.match(r'^https?://', url):
                 url = base_host + url
-            purl = re.search(r'/(?P<dir>[^/]+)/(?:dn)?(?P<fn>[^/]+?)\.(?P<ext>[^\.\?]+)(?P<hasparams>\?|$)',url)
-            if video_id == None:
+            purl = re.search(r'/(?P<dir>[^/]+)/(?:dn)?(?P<fn>[^/]+?)\.(?P<ext>[^\.\?]+)(?P<hasparams>\?|$)', url)
+            if video_id is None:
                 video_id = purl.group('fn')
-            if js.get('start') != None:
+            if js.get('start') is not None:
                 url += '&' if purl.group('hasparams') == '?' else '?'
-                url = url + 'start='+str(js.get('start'))
+                url = url + 'start=' + str(js.get('start'))
             formats.append({
                 'format_id': purl.group('dir'),
                 'ext': purl.group('ext'),
@@ -92,9 +79,7 @@ class DemocracynowIE(InfoExtractor):
             'title': js.get('title'),
             'description': description,
             'uploader': 'Democracy Now',
-#            'thumbnails': thumbnails,
             'subtitles': subtitles,
             'formats': formats,
         }
         return ret
-#
\ No newline at end of file

From 9fefc88656eceac13604fd86dfb25dc736ed239a Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Fri, 17 Jul 2015 07:24:07 -0500
Subject: [PATCH 1913/6219] fix TestCompat test_all_present

---
 youtube_dl/compat.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index de9ba2c14..8b4d0287c 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -451,7 +451,9 @@ __all__ = [
     'compat_subprocess_get_DEVNULL',
     'compat_urllib_error',
     'compat_urllib_parse',
+    'compat_urllib_parse_asciire',
     'compat_urllib_parse_unquote',
+    'compat_urllib_parse_unquote_to_bytes',
     'compat_urllib_parse_urlparse',
     'compat_urllib_request',
     'compat_urlparse',

From 593b77064c51c411071e310578b542017b9b2ec8 Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Fri, 17 Jul 2015 09:45:49 -0500
Subject: [PATCH 1914/6219] Don't forget trailing '%'

---
 youtube_dl/compat.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 8b4d0287c..9e506352f 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -98,6 +98,9 @@ except ImportError:
         append = res.append
 
         for item in bits[1:]:
+            if item == '':
+                append(b'%')
+                continue
             try:
                 append(item[:2].decode('hex'))
                 append(item[2:])

From 4a632911443f0dbc2384fb82ade85382aeecc8dc Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Fri, 17 Jul 2015 09:46:08 -0500
Subject: [PATCH 1915/6219] Add tests for compat_urllib_parse_unquote

---
 test/test_compat.py | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/test/test_compat.py b/test/test_compat.py
index 1eb454e06..431e6bdf1 100644
--- a/test/test_compat.py
+++ b/test/test_compat.py
@@ -14,6 +14,7 @@ from youtube_dl.utils import get_filesystem_encoding
 from youtube_dl.compat import (
     compat_getenv,
     compat_expanduser,
+    compat_urllib_parse_unquote,
 )
 
 
@@ -42,5 +43,24 @@ class TestCompat(unittest.TestCase):
             dir(youtube_dl.compat))) - set(['unicode_literals'])
         self.assertEqual(all_names, sorted(present_names))
 
+    def test_compat_urllib_parse_unquote(self):
+        test_strings = [
+            ['''''', ''''''],
+            ['''津波''', '''%E6%B4%A5%E6%B3%A2'''],
+            ['''津波''', str('%E6%B4%A5%E6%B3%A2')],
+            ['''<meta property="og:description" content="▁▂▃▄%▅▆▇█" />
+%<a href="https://ar.wikipedia.org/wiki/تسونامي">%a''',
+             '''<meta property="og:description" content="%E2%96%81%E2%96%82%E2%96%83%E2%96%84%25%E2%96%85%E2%96%86%E2%96%87%E2%96%88" />
+%<a href="https://ar.wikipedia.org/wiki/%D8%AA%D8%B3%D9%88%D9%86%D8%A7%D9%85%D9%8A">%a'''],
+            ['''(^◣_◢^)っ︻デ═一    ⇀    ⇀    ⇀    ⇀    ⇀    ↶%I%Break%Things%''',
+             '''%28%5E%E2%97%A3_%E2%97%A2%5E%29%E3%81%A3%EF%B8%BB%E3%83%87%E2%95%90%E4%B8%80    %E2%87%80    %E2%87%80    %E2%87%80    %E2%87%80    %E2%87%80    %E2%86%B6%I%Break%25Things%''']
+        ]
+        for test in test_strings:
+            strutf = test[0]
+            strurlenc = test[1]
+            strurldec = compat_urllib_parse_unquote(strurlenc)
+            self.assertEqual(strutf, strurldec)
+            self.assertEqual(strutf, compat_urllib_parse_unquote(strurlenc))
+
 if __name__ == '__main__':
     unittest.main()

From cabe001590914338524dff0c1edffbde4a1447ab Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 17 Jul 2015 16:18:33 +0100
Subject: [PATCH 1916/6219] [howstuffwoks] fix _VALID_URL regex

---
 youtube_dl/extractor/howstuffworks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/howstuffworks.py b/youtube_dl/extractor/howstuffworks.py
index e97339121..f59393150 100644
--- a/youtube_dl/extractor/howstuffworks.py
+++ b/youtube_dl/extractor/howstuffworks.py
@@ -10,7 +10,7 @@ from ..utils import (
 
 
 class HowStuffWorksIE(InfoExtractor):
-    _VALID_URL = r'https?://[\da-z-]+\.howstuffworks\.com/(?:[^/]+/)*\d+-(?P<id>.+?)-video\.htm'
+    _VALID_URL = r'https?://[\da-z-]+\.howstuffworks\.com/(?:[^/]+/)*(\d+-)*(?P<id>.+?)-video\.htm'
     _TESTS = [
         {
             'url': 'http://adventure.howstuffworks.com/5266-cool-jobs-iditarod-musher-video.htm',

From f354385bf50e04b0e7e1defcc14264d1a66e0a07 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 17 Jul 2015 16:43:27 +0100
Subject: [PATCH 1917/6219] Add test for urls without a number

---
 youtube_dl/extractor/howstuffworks.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/youtube_dl/extractor/howstuffworks.py b/youtube_dl/extractor/howstuffworks.py
index f59393150..c9b6579a6 100644
--- a/youtube_dl/extractor/howstuffworks.py
+++ b/youtube_dl/extractor/howstuffworks.py
@@ -46,6 +46,17 @@ class HowStuffWorksIE(InfoExtractor):
                 'thumbnail': 're:^https?://.*\.jpg$',
             },
         },
+        {
+            'url': 'http://shows.howstuffworks.com/stuff-to-blow-your-mind/optical-illusions-video.htm',
+            'info_dict': {
+                'id': '855410',
+                'ext': 'mp4',
+                'title': 'Stuff to Blow Your Mind',
+                'description': 'When it comes to optical illusions it’s pretty easy to jailbreak the operating system of your mind and load a new program onto it. Find out why your visual system willingly makes errors in interpretation.',
+                'display_id': 'optical-illusions',
+                'thumbnail': 're:^https?://.*\.jpg$',
+            },
+        }
     ]
 
     def _real_extract(self, url):

From 1186e3f91ab940017be263c65c27b821992e58cc Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 17 Jul 2015 16:45:53 +0100
Subject: [PATCH 1918/6219] do not capture a group

---
 youtube_dl/extractor/howstuffworks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/howstuffworks.py b/youtube_dl/extractor/howstuffworks.py
index c9b6579a6..cc931d571 100644
--- a/youtube_dl/extractor/howstuffworks.py
+++ b/youtube_dl/extractor/howstuffworks.py
@@ -10,7 +10,7 @@ from ..utils import (
 
 
 class HowStuffWorksIE(InfoExtractor):
-    _VALID_URL = r'https?://[\da-z-]+\.howstuffworks\.com/(?:[^/]+/)*(\d+-)*(?P<id>.+?)-video\.htm'
+    _VALID_URL = r'https?://[\da-z-]+\.howstuffworks\.com/(?:[^/]+/)*(?:\d+-)?(?P<id>.+?)-video\.htm'
     _TESTS = [
         {
             'url': 'http://adventure.howstuffworks.com/5266-cool-jobs-iditarod-musher-video.htm',

From c4fe07c7afcc59dfcd5cfa4626ad37deb19e1033 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 17 Jul 2015 17:07:55 +0100
Subject: [PATCH 1919/6219] match only the test url

---
 youtube_dl/extractor/howstuffworks.py | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/howstuffworks.py b/youtube_dl/extractor/howstuffworks.py
index cc931d571..663e6632a 100644
--- a/youtube_dl/extractor/howstuffworks.py
+++ b/youtube_dl/extractor/howstuffworks.py
@@ -48,14 +48,7 @@ class HowStuffWorksIE(InfoExtractor):
         },
         {
             'url': 'http://shows.howstuffworks.com/stuff-to-blow-your-mind/optical-illusions-video.htm',
-            'info_dict': {
-                'id': '855410',
-                'ext': 'mp4',
-                'title': 'Stuff to Blow Your Mind',
-                'description': 'When it comes to optical illusions it’s pretty easy to jailbreak the operating system of your mind and load a new program onto it. Find out why your visual system willingly makes errors in interpretation.',
-                'display_id': 'optical-illusions',
-                'thumbnail': 're:^https?://.*\.jpg$',
-            },
+            'only_matching': True,
         }
     ]
 

From 55139679261f8c2409ca150906a2693731452a13 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 22:58:13 +0600
Subject: [PATCH 1920/6219] [compat] Simplify and use latest cpython 3 code

---
 youtube_dl/compat.py | 41 +++++++++++++----------------------------
 1 file changed, 13 insertions(+), 28 deletions(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 9e506352f..54ccf1d28 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -75,8 +75,13 @@ except ImportError:
     import BaseHTTPServer as compat_http_server
 
 try:
+    from urllib.parse import unquote_to_bytes as compat_urllib_parse_unquote_to_bytes
     from urllib.parse import unquote as compat_urllib_parse_unquote
-except ImportError:
+except ImportError:  # Python 2
+    # HACK: The following are the correct unquote_to_bytes and unquote
+    # implementations from cpython 3.4.3's stdlib. Python 2's version
+    # is apparently broken (see https://github.com/rg3/youtube-dl/pull/6244)
+
     def compat_urllib_parse_unquote_to_bytes(string):
         """unquote_to_bytes('abc%20def') -> b'abc def'."""
         # Note: strings are encoded as UTF-8. This is only an issue if it contains
@@ -85,32 +90,22 @@ except ImportError:
             # Is it a string-like object?
             string.split
             return b''
-        if isinstance(string, str):
+        if isinstance(string, unicode):
             string = string.encode('utf-8')
-            # string = encode('utf-8')
-
-        # python3 -> 2: must implicitly convert to bits
-        bits = bytes(string).split(b'%')
-
+        bits = string.split(b'%')
         if len(bits) == 1:
             return string
         res = [bits[0]]
         append = res.append
-
         for item in bits[1:]:
-            if item == '':
-                append(b'%')
-                continue
             try:
-                append(item[:2].decode('hex'))
+                append(compat_urllib_parse._hextochr[item[:2]])
                 append(item[2:])
-            except:
+            except KeyError:
                 append(b'%')
                 append(item)
         return b''.join(res)
 
-    compat_urllib_parse_asciire = re.compile('([\x00-\x7f]+)')
-
     def compat_urllib_parse_unquote(string, encoding='utf-8', errors='replace'):
         """Replace %xx escapes by their single-character equivalent. The optional
         encoding and errors parameters specify how to decode percent-encoded
@@ -121,7 +116,6 @@ except ImportError:
 
         unquote('abc%20def') -> 'abc def'.
         """
-
         if '%' not in string:
             string.split
             return string
@@ -129,20 +123,12 @@ except ImportError:
             encoding = 'utf-8'
         if errors is None:
             errors = 'replace'
-
-        bits = compat_urllib_parse_asciire.split(string)
+        bits = compat_urllib_parse._asciire.split(string)
         res = [bits[0]]
         append = res.append
         for i in range(1, len(bits), 2):
-            foo = compat_urllib_parse_unquote_to_bytes(bits[i])
-            foo = foo.decode(encoding, errors)
-            append(foo)
-
-            if bits[i + 1]:
-                bar = bits[i + 1]
-                if not isinstance(bar, unicode):
-                    bar = bar.decode('utf-8')
-                append(bar)
+            append(compat_urllib_parse_unquote_to_bytes(bits[i]).decode(encoding, errors))
+            append(bits[i + 1])
         return ''.join(res)
 
 try:
@@ -454,7 +440,6 @@ __all__ = [
     'compat_subprocess_get_DEVNULL',
     'compat_urllib_error',
     'compat_urllib_parse',
-    'compat_urllib_parse_asciire',
     'compat_urllib_parse_unquote',
     'compat_urllib_parse_unquote_to_bytes',
     'compat_urllib_parse_urlparse',

From 14309e1ddc476a7e2fc444a0443b2fc23186a385 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 22:58:39 +0600
Subject: [PATCH 1921/6219] [test_compat] Make tests more idiomatic

---
 test/test_compat.py | 33 ++++++++++++++++-----------------
 1 file changed, 16 insertions(+), 17 deletions(-)

diff --git a/test/test_compat.py b/test/test_compat.py
index 431e6bdf1..2ffbc2c48 100644
--- a/test/test_compat.py
+++ b/test/test_compat.py
@@ -44,23 +44,22 @@ class TestCompat(unittest.TestCase):
         self.assertEqual(all_names, sorted(present_names))
 
     def test_compat_urllib_parse_unquote(self):
-        test_strings = [
-            ['''''', ''''''],
-            ['''津波''', '''%E6%B4%A5%E6%B3%A2'''],
-            ['''津波''', str('%E6%B4%A5%E6%B3%A2')],
-            ['''<meta property="og:description" content="▁▂▃▄%▅▆▇█" />
-%<a href="https://ar.wikipedia.org/wiki/تسونامي">%a''',
-             '''<meta property="og:description" content="%E2%96%81%E2%96%82%E2%96%83%E2%96%84%25%E2%96%85%E2%96%86%E2%96%87%E2%96%88" />
-%<a href="https://ar.wikipedia.org/wiki/%D8%AA%D8%B3%D9%88%D9%86%D8%A7%D9%85%D9%8A">%a'''],
-            ['''(^◣_◢^)っ︻デ═一    ⇀    ⇀    ⇀    ⇀    ⇀    ↶%I%Break%Things%''',
-             '''%28%5E%E2%97%A3_%E2%97%A2%5E%29%E3%81%A3%EF%B8%BB%E3%83%87%E2%95%90%E4%B8%80    %E2%87%80    %E2%87%80    %E2%87%80    %E2%87%80    %E2%87%80    %E2%86%B6%I%Break%25Things%''']
-        ]
-        for test in test_strings:
-            strutf = test[0]
-            strurlenc = test[1]
-            strurldec = compat_urllib_parse_unquote(strurlenc)
-            self.assertEqual(strutf, strurldec)
-            self.assertEqual(strutf, compat_urllib_parse_unquote(strurlenc))
+        self.assertEqual(compat_urllib_parse_unquote(''), '')
+        self.assertEqual(compat_urllib_parse_unquote('%'), '%')
+        self.assertEqual(compat_urllib_parse_unquote('%%'), '%%')
+        self.assertEqual(compat_urllib_parse_unquote('%%%'), '%%%')
+        self.assertEqual(compat_urllib_parse_unquote('%2F'), '/')
+        self.assertEqual(compat_urllib_parse_unquote('%2f'), '/')
+        self.assertEqual(compat_urllib_parse_unquote('%E6%B4%A5%E6%B3%A2'), '津波')
+        self.assertEqual(compat_urllib_parse_unquote(str('%E6%B4%A5%E6%B3%A2')), '津波')
+        self.assertEqual(
+            compat_urllib_parse_unquote('''<meta property="og:description" content="%E2%96%81%E2%96%82%E2%96%83%E2%96%84%25%E2%96%85%E2%96%86%E2%96%87%E2%96%88" />
+%<a href="https://ar.wikipedia.org/wiki/%D8%AA%D8%B3%D9%88%D9%86%D8%A7%D9%85%D9%8A">%a'''),
+            '''<meta property="og:description" content="▁▂▃▄%▅▆▇█" />
+%<a href="https://ar.wikipedia.org/wiki/تسونامي">%a''')
+        self.assertEqual(
+            compat_urllib_parse_unquote('''%28%5E%E2%97%A3_%E2%97%A2%5E%29%E3%81%A3%EF%B8%BB%E3%83%87%E2%95%90%E4%B8%80    %E2%87%80    %E2%87%80    %E2%87%80    %E2%87%80    %E2%87%80    %E2%86%B6%I%Break%25Things%'''),
+            '''(^◣_◢^)っ︻デ═一    ⇀    ⇀    ⇀    ⇀    ⇀    ↶%I%Break%Things%''')
 
 if __name__ == '__main__':
     unittest.main()

From d79febcd0619608aa100c17b0a2aefe9f4836d9b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:09:56 +0600
Subject: [PATCH 1922/6219] [test_compat] Remove redundant test

---
 test/test_compat.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/test/test_compat.py b/test/test_compat.py
index 2ffbc2c48..d816a9236 100644
--- a/test/test_compat.py
+++ b/test/test_compat.py
@@ -51,7 +51,6 @@ class TestCompat(unittest.TestCase):
         self.assertEqual(compat_urllib_parse_unquote('%2F'), '/')
         self.assertEqual(compat_urllib_parse_unquote('%2f'), '/')
         self.assertEqual(compat_urllib_parse_unquote('%E6%B4%A5%E6%B3%A2'), '津波')
-        self.assertEqual(compat_urllib_parse_unquote(str('%E6%B4%A5%E6%B3%A2')), '津波')
         self.assertEqual(
             compat_urllib_parse_unquote('''<meta property="og:description" content="%E2%96%81%E2%96%82%E2%96%83%E2%96%84%25%E2%96%85%E2%96%86%E2%96%87%E2%96%88" />
 %<a href="https://ar.wikipedia.org/wiki/%D8%AA%D8%B3%D9%88%D9%86%D8%A7%D9%85%D9%8A">%a'''),

From aa99aa4e853966d4f01e280313e3aadf2ddf7ad6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:28:34 +0600
Subject: [PATCH 1923/6219] [compat] Add compat_urllib_parse_unquote_plus

---
 youtube_dl/compat.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 54ccf1d28..a3a2aef53 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -77,6 +77,7 @@ except ImportError:
 try:
     from urllib.parse import unquote_to_bytes as compat_urllib_parse_unquote_to_bytes
     from urllib.parse import unquote as compat_urllib_parse_unquote
+    from urllib.parse import unquote_plus as compat_urllib_parse_unquote_plus
 except ImportError:  # Python 2
     # HACK: The following are the correct unquote_to_bytes and unquote
     # implementations from cpython 3.4.3's stdlib. Python 2's version
@@ -131,6 +132,15 @@ except ImportError:  # Python 2
             append(bits[i + 1])
         return ''.join(res)
 
+    def compat_urllib_parse_unquote_plus(string, encoding='utf-8', errors='replace'):
+        """Like unquote(), but also replace plus signs by spaces, as required for
+        unquoting HTML form values.
+
+        unquote_plus('%7e/abc+def') -> '~/abc def'
+        """
+        string = string.replace('+', ' ')
+        return compat_urllib_parse_unquote(string, encoding, errors)
+
 try:
     compat_str = unicode  # Python 2
 except NameError:
@@ -441,6 +451,7 @@ __all__ = [
     'compat_urllib_error',
     'compat_urllib_parse',
     'compat_urllib_parse_unquote',
+    'compat_urllib_parse_unquote_plus',
     'compat_urllib_parse_unquote_to_bytes',
     'compat_urllib_parse_urlparse',
     'compat_urllib_request',

From 8954e481404746c65ae1e82a6c5edec69114bfb1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:31:23 +0600
Subject: [PATCH 1924/6219] [test_compat] Add tests for
 compat_urllib_parse_unquote_plus

---
 test/test_compat.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/test/test_compat.py b/test/test_compat.py
index d816a9236..c3ba8ad2e 100644
--- a/test/test_compat.py
+++ b/test/test_compat.py
@@ -15,6 +15,7 @@ from youtube_dl.compat import (
     compat_getenv,
     compat_expanduser,
     compat_urllib_parse_unquote,
+    compat_urllib_parse_unquote_plus,
 )
 
 
@@ -44,6 +45,8 @@ class TestCompat(unittest.TestCase):
         self.assertEqual(all_names, sorted(present_names))
 
     def test_compat_urllib_parse_unquote(self):
+        self.assertEqual(compat_urllib_parse_unquote('abc%20def'), 'abc def')
+        self.assertEqual(compat_urllib_parse_unquote('%7e/abc+def'), '~/abc+def')
         self.assertEqual(compat_urllib_parse_unquote(''), '')
         self.assertEqual(compat_urllib_parse_unquote('%'), '%')
         self.assertEqual(compat_urllib_parse_unquote('%%'), '%%')
@@ -60,5 +63,9 @@ class TestCompat(unittest.TestCase):
             compat_urllib_parse_unquote('''%28%5E%E2%97%A3_%E2%97%A2%5E%29%E3%81%A3%EF%B8%BB%E3%83%87%E2%95%90%E4%B8%80    %E2%87%80    %E2%87%80    %E2%87%80    %E2%87%80    %E2%87%80    %E2%86%B6%I%Break%25Things%'''),
             '''(^◣_◢^)っ︻デ═一    ⇀    ⇀    ⇀    ⇀    ⇀    ↶%I%Break%Things%''')
 
+    def test_compat_urllib_parse_unquote_plus(self):
+        self.assertEqual(compat_urllib_parse_unquote_plus('abc%20def'), 'abc def')
+        self.assertEqual(compat_urllib_parse_unquote_plus('%7e/abc+def'), '~/abc def')
+
 if __name__ == '__main__':
     unittest.main()

From 4d08161ac23cfdf563b5d79b9f75cde6e6e57178 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:32:43 +0600
Subject: [PATCH 1925/6219] [compat] Mention unquote_plus

---
 youtube_dl/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index a3a2aef53..e950a4688 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -79,7 +79,7 @@ try:
     from urllib.parse import unquote as compat_urllib_parse_unquote
     from urllib.parse import unquote_plus as compat_urllib_parse_unquote_plus
 except ImportError:  # Python 2
-    # HACK: The following are the correct unquote_to_bytes and unquote
+    # HACK: The following are the correct unquote_to_bytes, unquote and unquote_plus
     # implementations from cpython 3.4.3's stdlib. Python 2's version
     # is apparently broken (see https://github.com/rg3/youtube-dl/pull/6244)
 

From b94b78971ce70a999584acab57d0a620a91bc2aa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:35:14 +0600
Subject: [PATCH 1926/6219] [bet] Use compat_urllib_parse_unquote

---
 youtube_dl/extractor/bet.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/bet.py b/youtube_dl/extractor/bet.py
index 26b934543..03dad4636 100644
--- a/youtube_dl/extractor/bet.py
+++ b/youtube_dl/extractor/bet.py
@@ -1,7 +1,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse
+from ..compat import compat_urllib_parse_unquote
 from ..utils import (
     xpath_text,
     xpath_with_ns,
@@ -57,7 +57,7 @@ class BetIE(InfoExtractor):
         display_id = self._match_id(url)
         webpage = self._download_webpage(url, display_id)
 
-        media_url = compat_urllib_parse.unquote(self._search_regex(
+        media_url = compat_urllib_parse_unquote(self._search_regex(
             [r'mediaURL\s*:\s*"([^"]+)"', r"var\s+mrssMediaUrl\s*=\s*'([^']+)'"],
             webpage, 'media URL'))
 

From 3e72f5f10e8dbac71e267ce10e791251a54e61ab Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:36:11 +0600
Subject: [PATCH 1927/6219] [ceskatelevize] Use compat_urllib_parse_unquote

---
 youtube_dl/extractor/ceskatelevize.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ceskatelevize.py b/youtube_dl/extractor/ceskatelevize.py
index 65f6be623..dda583680 100644
--- a/youtube_dl/extractor/ceskatelevize.py
+++ b/youtube_dl/extractor/ceskatelevize.py
@@ -7,6 +7,7 @@ from .common import InfoExtractor
 from ..compat import (
     compat_urllib_request,
     compat_urllib_parse,
+    compat_urllib_parse_unquote,
     compat_urllib_parse_urlparse,
 )
 from ..utils import (
@@ -88,7 +89,7 @@ class CeskaTelevizeIE(InfoExtractor):
         if playlist_url == 'error_region':
             raise ExtractorError(NOT_AVAILABLE_STRING, expected=True)
 
-        req = compat_urllib_request.Request(compat_urllib_parse.unquote(playlist_url))
+        req = compat_urllib_request.Request(compat_urllib_parse_unquote(playlist_url))
         req.add_header('Referer', url)
 
         playlist = self._download_json(req, video_id)

From a60cccbf9f76e7e42509bf1ed4e054643ecc348f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:36:58 +0600
Subject: [PATCH 1928/6219] [crunchyroll] Use compat_urllib_parse_unquote

---
 youtube_dl/extractor/crunchyroll.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 73f1e22ef..d1b6d7366 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -12,6 +12,7 @@ from math import pow, sqrt, floor
 from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse,
+    compat_urllib_parse_unquote,
     compat_urllib_request,
 )
 from ..utils import (
@@ -254,7 +255,7 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
             video_upload_date = unified_strdate(video_upload_date)
         video_uploader = self._html_search_regex(r'<div>\s*Publisher:(.+?)</div>', webpage, 'video_uploader', fatal=False, flags=re.DOTALL)
 
-        playerdata_url = compat_urllib_parse.unquote(self._html_search_regex(r'"config_url":"([^"]+)', webpage, 'playerdata_url'))
+        playerdata_url = compat_urllib_parse_unquote(self._html_search_regex(r'"config_url":"([^"]+)', webpage, 'playerdata_url'))
         playerdata_req = compat_urllib_request.Request(playerdata_url)
         playerdata_req.data = compat_urllib_parse.urlencode({'current_page': webpage_url})
         playerdata_req.add_header('Content-Type', 'application/x-www-form-urlencoded')

From d3671b344f663c3aa82aea86de51a70f24511792 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:37:23 +0600
Subject: [PATCH 1929/6219] [ehow] Use compat_urllib_parse_unquote

---
 youtube_dl/extractor/ehow.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/ehow.py b/youtube_dl/extractor/ehow.py
index 9cb1bf301..ba9685991 100644
--- a/youtube_dl/extractor/ehow.py
+++ b/youtube_dl/extractor/ehow.py
@@ -1,7 +1,7 @@
 from __future__ import unicode_literals
 
 from ..compat import (
-    compat_urllib_parse,
+    compat_urllib_parse_unquote,
 )
 from .common import InfoExtractor
 
@@ -26,7 +26,7 @@ class EHowIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
         video_url = self._search_regex(
             r'(?:file|source)=(http[^\'"&]*)', webpage, 'video URL')
-        final_url = compat_urllib_parse.unquote(video_url)
+        final_url = compat_urllib_parse_unquote(video_url)
         uploader = self._html_search_meta('uploader', webpage)
         title = self._og_search_title(webpage).replace(' | eHow', '')
 

From d7011316d0bce88796a6d33e895fdafa231fd8ec Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:37:56 +0600
Subject: [PATCH 1930/6219] [facebook] Use compat_urllib_parse_unquote

---
 youtube_dl/extractor/facebook.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index 82dc27bc6..e17bb9aea 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -9,7 +9,7 @@ from ..compat import (
     compat_http_client,
     compat_str,
     compat_urllib_error,
-    compat_urllib_parse,
+    compat_urllib_parse_unquote,
     compat_urllib_request,
 )
 from ..utils import (
@@ -136,7 +136,7 @@ class FacebookIE(InfoExtractor):
             else:
                 raise ExtractorError('Cannot parse data')
         data = dict(json.loads(m.group(1)))
-        params_raw = compat_urllib_parse.unquote(data['params'])
+        params_raw = compat_urllib_parse_unquote(data['params'])
         params = json.loads(params_raw)
         video_data = params['video_data'][0]
 

From 1f80e360fce9fc057847379ec06d0e635f697198 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:38:30 +0600
Subject: [PATCH 1931/6219] [gamespot] Use compat_urllib_parse_unquote

---
 youtube_dl/extractor/gamespot.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/gamespot.py b/youtube_dl/extractor/gamespot.py
index 2d33fa7f5..b3f1bafcc 100644
--- a/youtube_dl/extractor/gamespot.py
+++ b/youtube_dl/extractor/gamespot.py
@@ -5,7 +5,7 @@ import json
 
 from .common import InfoExtractor
 from ..compat import (
-    compat_urllib_parse,
+    compat_urllib_parse_unquote,
     compat_urlparse,
 )
 from ..utils import (
@@ -75,7 +75,7 @@ class GameSpotIE(InfoExtractor):
         return {
             'id': data_video['guid'],
             'display_id': page_id,
-            'title': compat_urllib_parse.unquote(data_video['title']),
+            'title': compat_urllib_parse_unquote(data_video['title']),
             'formats': formats,
             'description': self._html_search_meta('description', webpage),
             'thumbnail': self._og_search_thumbnail(webpage),

From f7e6f7fa2374864ba28fbe6840a0462b7c1033db Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:39:32 +0600
Subject: [PATCH 1932/6219] [extractor/generic] Use compat_urllib_parse_unquote

---
 youtube_dl/extractor/generic.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index fc1bf2b6e..a62287e50 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -8,7 +8,6 @@ import re
 from .common import InfoExtractor
 from .youtube import YoutubeIE
 from ..compat import (
-    compat_urllib_parse,
     compat_urllib_parse_unquote,
     compat_urllib_request,
     compat_urlparse,
@@ -1369,7 +1368,7 @@ class GenericIE(InfoExtractor):
             return self.url_result(mobj.group('url'))
         mobj = re.search(r'class=["\']embedly-embed["\'][^>]src=["\'][^"\']*url=(?P<url>[^&]+)', webpage)
         if mobj is not None:
-            return self.url_result(compat_urllib_parse.unquote(mobj.group('url')))
+            return self.url_result(compat_urllib_parse_unquote(mobj.group('url')))
 
         # Look for funnyordie embed
         matches = re.findall(r'<iframe[^>]+?src="(https?://(?:www\.)?funnyordie\.com/embed/[^"]+)"', webpage)
@@ -1682,7 +1681,7 @@ class GenericIE(InfoExtractor):
         entries = []
         for video_url in found:
             video_url = compat_urlparse.urljoin(url, video_url)
-            video_id = compat_urllib_parse.unquote(os.path.basename(video_url))
+            video_id = compat_urllib_parse_unquote(os.path.basename(video_url))
 
             # Sometimes, jwplayer extraction will result in a YouTube URL
             if YoutubeIE.suitable(video_url):

From 8ee4ecb48dd7824f451c11c4cebb05b603a2da27 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:39:54 +0600
Subject: [PATCH 1933/6219] [infoq] Use compat_urllib_parse_unquote

---
 youtube_dl/extractor/infoq.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/infoq.py b/youtube_dl/extractor/infoq.py
index 91a1b3ccb..71cfd12c5 100644
--- a/youtube_dl/extractor/infoq.py
+++ b/youtube_dl/extractor/infoq.py
@@ -4,7 +4,7 @@ import base64
 
 from .common import InfoExtractor
 from ..compat import (
-    compat_urllib_parse,
+    compat_urllib_parse_unquote,
     compat_urlparse,
 )
 
@@ -39,7 +39,7 @@ class InfoQIE(InfoExtractor):
         # Extract video URL
         encoded_id = self._search_regex(
             r"jsclassref\s*=\s*'([^']*)'", webpage, 'encoded id')
-        real_id = compat_urllib_parse.unquote(base64.b64decode(encoded_id.encode('ascii')).decode('utf-8'))
+        real_id = compat_urllib_parse_unquote(base64.b64decode(encoded_id.encode('ascii')).decode('utf-8'))
         playpath = 'mp4:' + real_id
 
         video_filename = playpath.split('/')[-1]

From 899a3e2f139a7b774d1a9483bb975e480e642c75 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:40:35 +0600
Subject: [PATCH 1934/6219] [karaoketv] Use compat_urllib_parse_unquote_plus

---
 youtube_dl/extractor/karaoketv.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/karaoketv.py b/youtube_dl/extractor/karaoketv.py
index e3b43ff8d..06daf5a89 100644
--- a/youtube_dl/extractor/karaoketv.py
+++ b/youtube_dl/extractor/karaoketv.py
@@ -2,7 +2,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse
+from ..compat import compat_urllib_parse_unquote_plus
 from ..utils import (
     js_to_json,
 )
@@ -24,7 +24,7 @@ class KaraoketvIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         page_video_url = self._og_search_video_url(webpage, video_id)
-        config_json = compat_urllib_parse.unquote_plus(self._search_regex(
+        config_json = compat_urllib_parse_unquote_plus(self._search_regex(
             r'config=(.*)', page_video_url, 'configuration'))
 
         urls_info_json = self._download_json(

From 977a247a0639728146dd5c0d369b542e0a1af69b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:41:14 +0600
Subject: [PATCH 1935/6219] [malemotion] Use compat_urllib_parse_unquote

---
 youtube_dl/extractor/malemotion.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/malemotion.py b/youtube_dl/extractor/malemotion.py
index 0b85a59d1..92511a671 100644
--- a/youtube_dl/extractor/malemotion.py
+++ b/youtube_dl/extractor/malemotion.py
@@ -2,9 +2,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse,
-)
+from ..compat import compat_urllib_parse_unquote
 
 
 class MalemotionIE(InfoExtractor):
@@ -24,7 +22,7 @@ class MalemotionIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        video_url = compat_urllib_parse.unquote(self._search_regex(
+        video_url = compat_urllib_parse_unquote(self._search_regex(
             r'<source type="video/mp4" src="(.+?)"', webpage, 'video URL'))
         video_title = self._html_search_regex(
             r'<title>(.*?)</title', webpage, 'title')

From c177bb3a50cde6035976603ba8094dd72cc310fa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:41:47 +0600
Subject: [PATCH 1936/6219] [metacafe] Use compat_urllib_parse_unquote

---
 youtube_dl/extractor/metacafe.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/metacafe.py b/youtube_dl/extractor/metacafe.py
index 8bc333b02..6e2e73a51 100644
--- a/youtube_dl/extractor/metacafe.py
+++ b/youtube_dl/extractor/metacafe.py
@@ -6,6 +6,7 @@ from .common import InfoExtractor
 from ..compat import (
     compat_parse_qs,
     compat_urllib_parse,
+    compat_urllib_parse_unquote,
     compat_urllib_request,
 )
 from ..utils import (
@@ -155,7 +156,7 @@ class MetacafeIE(InfoExtractor):
         video_url = None
         mobj = re.search(r'(?m)&mediaURL=([^&]+)', webpage)
         if mobj is not None:
-            mediaURL = compat_urllib_parse.unquote(mobj.group(1))
+            mediaURL = compat_urllib_parse_unquote(mobj.group(1))
             video_ext = mediaURL[-3:]
 
             # Extract gdaKey if available

From 09b718c4398b2ce9964ec3ae5e329278d6211668 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:42:11 +0600
Subject: [PATCH 1937/6219] [mitele] Use compat_urllib_parse_unquote

---
 youtube_dl/extractor/mitele.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mitele.py b/youtube_dl/extractor/mitele.py
index 7091f3335..852d72266 100644
--- a/youtube_dl/extractor/mitele.py
+++ b/youtube_dl/extractor/mitele.py
@@ -5,6 +5,7 @@ import json
 from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse,
+    compat_urllib_parse_unquote,
     compat_urlparse,
 )
 from ..utils import (
@@ -48,7 +49,7 @@ class MiTeleIE(InfoExtractor):
             domain = 'http://' + domain
         info_url = compat_urlparse.urljoin(
             domain,
-            compat_urllib_parse.unquote(embed_data['flashvars']['host'])
+            compat_urllib_parse_unquote(embed_data['flashvars']['host'])
         )
         info_el = self._download_xml(info_url, episode).find('./video/info')
 

From c2daf8dfa44b07179b272874c336d2435a2ff653 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:42:43 +0600
Subject: [PATCH 1938/6219] [mixcloud] Use compat_urllib_parse_unquote

---
 youtube_dl/extractor/mixcloud.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dl/extractor/mixcloud.py
index 425a4ccf1..d47aeceda 100644
--- a/youtube_dl/extractor/mixcloud.py
+++ b/youtube_dl/extractor/mixcloud.py
@@ -3,9 +3,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse,
-)
+from ..compat import compat_urllib_parse_unquote
 from ..utils import (
     ExtractorError,
     HEADRequest,
@@ -60,7 +58,7 @@ class MixcloudIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         uploader = mobj.group(1)
         cloudcast_name = mobj.group(2)
-        track_id = compat_urllib_parse.unquote('-'.join((uploader, cloudcast_name)))
+        track_id = compat_urllib_parse_unquote('-'.join((uploader, cloudcast_name)))
 
         webpage = self._download_webpage(url, track_id)
 

From e97bb3de838219a8c2800f3eb646313977503af2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:43:36 +0600
Subject: [PATCH 1939/6219] [mofosex] Use compat_urllib_parse_unquote

---
 youtube_dl/extractor/mofosex.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/mofosex.py b/youtube_dl/extractor/mofosex.py
index 2cec12d35..9bf99a54a 100644
--- a/youtube_dl/extractor/mofosex.py
+++ b/youtube_dl/extractor/mofosex.py
@@ -5,9 +5,9 @@ import re
 
 from .common import InfoExtractor
 from ..compat import (
+    compat_urllib_parse_unquote,
     compat_urllib_parse_urlparse,
     compat_urllib_request,
-    compat_urllib_parse,
 )
 
 
@@ -34,7 +34,7 @@ class MofosexIE(InfoExtractor):
         webpage = self._download_webpage(req, video_id)
 
         video_title = self._html_search_regex(r'<h1>(.+?)<', webpage, 'title')
-        video_url = compat_urllib_parse.unquote(self._html_search_regex(r'flashvars.video_url = \'([^\']+)', webpage, 'video_url'))
+        video_url = compat_urllib_parse_unquote(self._html_search_regex(r'flashvars.video_url = \'([^\']+)', webpage, 'video_url'))
         path = compat_urllib_parse_urlparse(video_url).path
         extension = os.path.splitext(path)[1][1:]
         format = path.split('/')[5].split('_')[:2]

From 9fd3bf04b7b160c1f24f9818689075f2570b8a2e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:44:38 +0600
Subject: [PATCH 1940/6219] [myvideo] Use compat_urllib_parse_unquote

---
 youtube_dl/extractor/myvideo.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/myvideo.py b/youtube_dl/extractor/myvideo.py
index 5e754fcff..c96f472a3 100644
--- a/youtube_dl/extractor/myvideo.py
+++ b/youtube_dl/extractor/myvideo.py
@@ -10,6 +10,7 @@ from .common import InfoExtractor
 from ..compat import (
     compat_ord,
     compat_urllib_parse,
+    compat_urllib_parse_unquote,
     compat_urllib_request,
 )
 from ..utils import (
@@ -107,7 +108,7 @@ class MyVideoIE(InfoExtractor):
             if not a == '_encxml':
                 params[a] = b
             else:
-                encxml = compat_urllib_parse.unquote(b)
+                encxml = compat_urllib_parse_unquote(b)
         if not params.get('domain'):
             params['domain'] = 'www.myvideo.de'
         xmldata_url = '%s?%s' % (encxml, compat_urllib_parse.urlencode(params))
@@ -135,7 +136,7 @@ class MyVideoIE(InfoExtractor):
         video_url = None
         mobj = re.search('connectionurl=\'(.*?)\'', dec_data)
         if mobj:
-            video_url = compat_urllib_parse.unquote(mobj.group(1))
+            video_url = compat_urllib_parse_unquote(mobj.group(1))
             if 'myvideo2flash' in video_url:
                 self.report_warning(
                     'Rewriting URL to use unencrypted rtmp:// ...',
@@ -147,10 +148,10 @@ class MyVideoIE(InfoExtractor):
             mobj = re.search('path=\'(http.*?)\' source=\'(.*?)\'', dec_data)
             if mobj is None:
                 raise ExtractorError('unable to extract url')
-            video_url = compat_urllib_parse.unquote(mobj.group(1)) + compat_urllib_parse.unquote(mobj.group(2))
+            video_url = compat_urllib_parse_unquote(mobj.group(1)) + compat_urllib_parse_unquote(mobj.group(2))
 
         video_file = self._search_regex('source=\'(.*?)\'', dec_data, 'video file')
-        video_file = compat_urllib_parse.unquote(video_file)
+        video_file = compat_urllib_parse_unquote(video_file)
 
         if not video_file.endswith('f4m'):
             ppath, prefix = video_file.split('.')
@@ -159,7 +160,7 @@ class MyVideoIE(InfoExtractor):
             video_playpath = ''
 
         video_swfobj = self._search_regex('swfobject.embedSWF\(\'(.+?)\'', webpage, 'swfobj')
-        video_swfobj = compat_urllib_parse.unquote(video_swfobj)
+        video_swfobj = compat_urllib_parse_unquote(video_swfobj)
 
         video_title = self._html_search_regex("<h1(?: class='globalHd')?>(.*?)</h1>",
                                               webpage, 'title')

From b78f5ec4c338e5e9379c1d7a2b0c99451e06897a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:45:00 +0600
Subject: [PATCH 1941/6219] [odnoklassniki] Use compat_urllib_parse_unquote

---
 youtube_dl/extractor/odnoklassniki.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/odnoklassniki.py b/youtube_dl/extractor/odnoklassniki.py
index 6c7149fe3..215ffe87b 100644
--- a/youtube_dl/extractor/odnoklassniki.py
+++ b/youtube_dl/extractor/odnoklassniki.py
@@ -2,7 +2,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse
+from ..compat import compat_urllib_parse_unquote
 from ..utils import (
     unified_strdate,
     int_or_none,
@@ -62,7 +62,7 @@ class OdnoklassnikiIE(InfoExtractor):
             metadata = self._parse_json(metadata, video_id)
         else:
             metadata = self._download_json(
-                compat_urllib_parse.unquote(flashvars['metadataUrl']),
+                compat_urllib_parse_unquote(flashvars['metadataUrl']),
                 video_id, 'Downloading metadata JSON')
 
         movie = metadata['movie']

From d54f1c74776a8d6c7cf07ae16ae67bd39ee51f93 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:45:26 +0600
Subject: [PATCH 1942/6219] [openfilm] Use compat_urllib_parse_unquote_plus

---
 youtube_dl/extractor/openfilm.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/openfilm.py b/youtube_dl/extractor/openfilm.py
index 2249657eb..7e10bdf2c 100644
--- a/youtube_dl/extractor/openfilm.py
+++ b/youtube_dl/extractor/openfilm.py
@@ -5,7 +5,7 @@ import json
 from .common import InfoExtractor
 from ..utils import (
     parse_iso8601,
-    compat_urllib_parse,
+    compat_urllib_parse_unquote_plus,
     parse_age_limit,
     int_or_none,
 )
@@ -37,7 +37,7 @@ class OpenFilmIE(InfoExtractor):
 
         webpage = self._download_webpage(url, display_id)
 
-        player = compat_urllib_parse.unquote_plus(
+        player = compat_urllib_parse_unquote_plus(
             self._og_search_video_url(webpage))
 
         video = json.loads(self._search_regex(

From 2ebbb6f1f771e5c81b103b6216073341e85cc1aa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:45:55 +0600
Subject: [PATCH 1943/6219] [photobucket] Use compat_urllib_parse_unquote

---
 youtube_dl/extractor/photobucket.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/photobucket.py b/youtube_dl/extractor/photobucket.py
index c66db3cdc..788411ccc 100644
--- a/youtube_dl/extractor/photobucket.py
+++ b/youtube_dl/extractor/photobucket.py
@@ -4,7 +4,7 @@ import json
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse
+from ..compat import compat_urllib_parse_unquote
 
 
 class PhotobucketIE(InfoExtractor):
@@ -34,7 +34,7 @@ class PhotobucketIE(InfoExtractor):
         info_json = self._search_regex(r'Pb\.Data\.Shared\.put\(Pb\.Data\.Shared\.MEDIA, (.*?)\);',
                                        webpage, 'info json')
         info = json.loads(info_json)
-        url = compat_urllib_parse.unquote(self._html_search_regex(r'file=(.+\.mp4)', info['linkcodes']['html'], 'url'))
+        url = compat_urllib_parse_unquote(self._html_search_regex(r'file=(.+\.mp4)', info['linkcodes']['html'], 'url'))
         return {
             'id': video_id,
             'url': url,

From 388ad0c05c976a769ae9cf9d1f0e59526921b68c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:46:33 +0600
Subject: [PATCH 1944/6219] [playvid] Use compat_urllib_parse_unquote and
 compat_urllib_parse_unquote_plus

---
 youtube_dl/extractor/playvid.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/playvid.py b/youtube_dl/extractor/playvid.py
index c3e667e9e..2eb4fd96d 100644
--- a/youtube_dl/extractor/playvid.py
+++ b/youtube_dl/extractor/playvid.py
@@ -4,7 +4,8 @@ import re
 
 from .common import InfoExtractor
 from ..compat import (
-    compat_urllib_parse,
+    compat_urllib_parse_unquote,
+    compat_urllib_parse_unquote_plus,
 )
 from ..utils import (
     clean_html,
@@ -44,7 +45,7 @@ class PlayvidIE(InfoExtractor):
         flashvars = self._html_search_regex(
             r'flashvars="(.+?)"', webpage, 'flashvars')
 
-        infos = compat_urllib_parse.unquote(flashvars).split(r'&')
+        infos = compat_urllib_parse_unquote(flashvars).split(r'&')
         for info in infos:
             videovars_match = re.match(r'^video_vars\[(.+?)\]=(.+?)$', info)
             if videovars_match:
@@ -52,7 +53,7 @@ class PlayvidIE(InfoExtractor):
                 val = videovars_match.group(2)
 
                 if key == 'title':
-                    video_title = compat_urllib_parse.unquote_plus(val)
+                    video_title = compat_urllib_parse_unquote_plus(val)
                 if key == 'duration':
                     try:
                         duration = int(val)

From 605cbef65326b52ba9fdb994fc06798f15a3a1ee Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:47:23 +0600
Subject: [PATCH 1945/6219] [pornhub] Use compat_urllib_parse_unquote and
 compat_urllib_parse_unquote_plus

---
 youtube_dl/extractor/pornhub.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index 8172bc997..0b7886840 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -5,7 +5,8 @@ import re
 
 from .common import InfoExtractor
 from ..compat import (
-    compat_urllib_parse,
+    compat_urllib_parse_unquote,
+    compat_urllib_parse_unquote_plus,
     compat_urllib_parse_urlparse,
     compat_urllib_request,
 )
@@ -69,7 +70,7 @@ class PornHubIE(InfoExtractor):
             webpage, 'uploader', fatal=False)
         thumbnail = self._html_search_regex(r'"image_url":"([^"]+)', webpage, 'thumbnail', fatal=False)
         if thumbnail:
-            thumbnail = compat_urllib_parse.unquote(thumbnail)
+            thumbnail = compat_urllib_parse_unquote(thumbnail)
 
         view_count = self._extract_count(
             r'<span class="count">([\d,\.]+)</span> views', webpage, 'view')
@@ -80,9 +81,9 @@ class PornHubIE(InfoExtractor):
         comment_count = self._extract_count(
             r'All Comments\s*<span>\(([\d,.]+)\)', webpage, 'comment')
 
-        video_urls = list(map(compat_urllib_parse.unquote, re.findall(r'"quality_[0-9]{3}p":"([^"]+)', webpage)))
+        video_urls = list(map(compat_urllib_parse_unquote, re.findall(r'"quality_[0-9]{3}p":"([^"]+)', webpage)))
         if webpage.find('"encrypted":true') != -1:
-            password = compat_urllib_parse.unquote_plus(
+            password = compat_urllib_parse_unquote_plus(
                 self._search_regex(r'"video_title":"([^"]+)', webpage, 'password'))
             video_urls = list(map(lambda s: aes_decrypt_text(s, password, 32).decode('utf-8'), video_urls))
 

From 47af21e8f15be1ddb6195327719569d8723a669e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:47:51 +0600
Subject: [PATCH 1946/6219] [spankwire] Use compat_urllib_parse_unquote

---
 youtube_dl/extractor/spankwire.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/spankwire.py b/youtube_dl/extractor/spankwire.py
index bff75d6b2..5fa6faf18 100644
--- a/youtube_dl/extractor/spankwire.py
+++ b/youtube_dl/extractor/spankwire.py
@@ -4,7 +4,7 @@ import re
 
 from .common import InfoExtractor
 from ..compat import (
-    compat_urllib_parse,
+    compat_urllib_parse_unquote,
     compat_urllib_parse_urlparse,
     compat_urllib_request,
 )
@@ -68,7 +68,7 @@ class SpankwireIE(InfoExtractor):
             webpage, 'comment count', fatal=False))
 
         video_urls = list(map(
-            compat_urllib_parse.unquote,
+            compat_urllib_parse_unquote,
             re.findall(r'playerData\.cdnPath[0-9]{3,}\s*=\s*(?:encodeURIComponent\()?["\']([^"\']+)["\']', webpage)))
         if webpage.find('flashvars\.encrypted = "true"') != -1:
             password = self._search_regex(

From 736f003f2e96b37a9554d77a26fdef6c0a38fd5d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:49:14 +0600
Subject: [PATCH 1947/6219] [xbef] Use compat_urllib_parse_unquote

---
 youtube_dl/extractor/xbef.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/xbef.py b/youtube_dl/extractor/xbef.py
index 80c48c37d..4ff99e5ca 100644
--- a/youtube_dl/extractor/xbef.py
+++ b/youtube_dl/extractor/xbef.py
@@ -1,9 +1,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse,
-)
+from ..compat import compat_urllib_parse_unquote
 
 
 class XBefIE(InfoExtractor):
@@ -30,7 +28,7 @@ class XBefIE(InfoExtractor):
         config_url_enc = self._download_webpage(
             'http://xbef.com/Main/GetVideoURLEncoded/%s' % video_id, video_id,
             note='Retrieving config URL')
-        config_url = compat_urllib_parse.unquote(config_url_enc)
+        config_url = compat_urllib_parse_unquote(config_url_enc)
         config = self._download_xml(
             config_url, video_id, note='Retrieving config')
 

From 7dde5f6a8d10ba98b43f920c73248a8a7e73dc38 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:49:38 +0600
Subject: [PATCH 1948/6219] [xnxx] Use compat_urllib_parse_unquote

---
 youtube_dl/extractor/xnxx.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/xnxx.py b/youtube_dl/extractor/xnxx.py
index 79ed6c744..5a41f8ffa 100644
--- a/youtube_dl/extractor/xnxx.py
+++ b/youtube_dl/extractor/xnxx.py
@@ -2,9 +2,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse,
-)
+from ..compat import compat_urllib_parse_unquote
 
 
 class XNXXIE(InfoExtractor):
@@ -26,7 +24,7 @@ class XNXXIE(InfoExtractor):
 
         video_url = self._search_regex(r'flv_url=(.*?)&amp;',
                                        webpage, 'video URL')
-        video_url = compat_urllib_parse.unquote(video_url)
+        video_url = compat_urllib_parse_unquote(video_url)
 
         video_title = self._html_search_regex(r'<title>(.*?)\s+-\s+XNXX.COM',
                                               webpage, 'title')

From ee8de13e14f0dee555c340ef8b9f9c15a6bac6dc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:50:02 +0600
Subject: [PATCH 1949/6219] [xtube] Use compat_urllib_parse_unquote

---
 youtube_dl/extractor/xtube.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/xtube.py b/youtube_dl/extractor/xtube.py
index 1644f53c8..779e4f46a 100644
--- a/youtube_dl/extractor/xtube.py
+++ b/youtube_dl/extractor/xtube.py
@@ -5,7 +5,7 @@ import re
 from .common import InfoExtractor
 from ..compat import (
     compat_urllib_request,
-    compat_urllib_parse,
+    compat_urllib_parse_unquote,
 )
 from ..utils import (
     parse_duration,
@@ -59,7 +59,7 @@ class XTubeIE(InfoExtractor):
         for format_id, video_url in re.findall(
                 r'flashvars\.quality_(.+?)\s*=\s*"([^"]+)"', webpage):
             fmt = {
-                'url': compat_urllib_parse.unquote(video_url),
+                'url': compat_urllib_parse_unquote(video_url),
                 'format_id': format_id,
             }
             m = re.search(r'^(?P<height>\d+)[pP]', format_id)
@@ -68,7 +68,7 @@ class XTubeIE(InfoExtractor):
             formats.append(fmt)
 
         if not formats:
-            video_url = compat_urllib_parse.unquote(self._search_regex(
+            video_url = compat_urllib_parse_unquote(self._search_regex(
                 r'flashvars\.video_url\s*=\s*"([^"]+)"',
                 webpage, 'video URL'))
             formats.append({'url': video_url})

From aa4789d632b71abb79561ecf6769258a66e158f2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:50:25 +0600
Subject: [PATCH 1950/6219] [xvideos] Use compat_urllib_parse_unquote

---
 youtube_dl/extractor/xvideos.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/xvideos.py b/youtube_dl/extractor/xvideos.py
index d8415bed4..5dcf2fdd1 100644
--- a/youtube_dl/extractor/xvideos.py
+++ b/youtube_dl/extractor/xvideos.py
@@ -4,7 +4,7 @@ import re
 
 from .common import InfoExtractor
 from ..compat import (
-    compat_urllib_parse,
+    compat_urllib_parse_unquote,
     compat_urllib_request,
 )
 from ..utils import (
@@ -37,7 +37,7 @@ class XVideosIE(InfoExtractor):
         if mobj:
             raise ExtractorError('%s said: %s' % (self.IE_NAME, clean_html(mobj.group(1))), expected=True)
 
-        video_url = compat_urllib_parse.unquote(
+        video_url = compat_urllib_parse_unquote(
             self._search_regex(r'flv_url=(.+?)&', webpage, 'video URL'))
         video_title = self._html_search_regex(
             r'<title>(.*?)\s+-\s+XVID', webpage, 'title')

From db6c50f109e732f287cc0c61216cdd865a031ea3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:50:52 +0600
Subject: [PATCH 1951/6219] [ynet] Use compat_urllib_parse_unquote_plus

---
 youtube_dl/extractor/ynet.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/ynet.py b/youtube_dl/extractor/ynet.py
index 894678a23..869f3e819 100644
--- a/youtube_dl/extractor/ynet.py
+++ b/youtube_dl/extractor/ynet.py
@@ -5,7 +5,7 @@ import re
 import json
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse
+from ..compat import compat_urllib_parse_unquote_plus
 
 
 class YnetIE(InfoExtractor):
@@ -34,7 +34,7 @@ class YnetIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        content = compat_urllib_parse.unquote_plus(self._og_search_video_url(webpage))
+        content = compat_urllib_parse_unquote_plus(self._og_search_video_url(webpage))
         config = json.loads(self._search_regex(r'config=({.+?})$', content, 'video config'))
         f4m_url = config['clip']['url']
         title = self._og_search_title(webpage)

From 7fd002c0061f9f96eca2224553a2551748970e7d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:51:57 +0600
Subject: [PATCH 1952/6219] [youtube] Use compat_urllib_parse_unquote and
 compat_urllib_parse_unquote_plus

---
 youtube_dl/extractor/youtube.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 3c629d38a..e7f5c7861 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -17,6 +17,8 @@ from ..compat import (
     compat_chr,
     compat_parse_qs,
     compat_urllib_parse,
+    compat_urllib_parse_unquote,
+    compat_urllib_parse_unquote_plus,
     compat_urllib_request,
     compat_urlparse,
     compat_str,
@@ -865,7 +867,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         # Extract original video URL from URL with redirection, like age verification, using next_url parameter
         mobj = re.search(self._NEXT_URL_RE, url)
         if mobj:
-            url = proto + '://www.youtube.com/' + compat_urllib_parse.unquote(mobj.group(1)).lstrip('/')
+            url = proto + '://www.youtube.com/' + compat_urllib_parse_unquote(mobj.group(1)).lstrip('/')
         video_id = self.extract_id(url)
 
         # Get video webpage
@@ -973,7 +975,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         # uploader
         if 'author' not in video_info:
             raise ExtractorError('Unable to extract uploader name')
-        video_uploader = compat_urllib_parse.unquote_plus(video_info['author'][0])
+        video_uploader = compat_urllib_parse_unquote_plus(video_info['author'][0])
 
         # uploader_id
         video_uploader_id = None
@@ -1000,7 +1002,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             self._downloader.report_warning('unable to extract video thumbnail')
             video_thumbnail = None
         else:   # don't panic if we can't find it
-            video_thumbnail = compat_urllib_parse.unquote_plus(video_info['thumbnail_url'][0])
+            video_thumbnail = compat_urllib_parse_unquote_plus(video_info['thumbnail_url'][0])
 
         # upload date
         upload_date = self._html_search_meta(
@@ -1062,7 +1064,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             self._downloader.report_warning('unable to extract video duration')
             video_duration = None
         else:
-            video_duration = int(compat_urllib_parse.unquote_plus(video_info['length_seconds'][0]))
+            video_duration = int(compat_urllib_parse_unquote_plus(video_info['length_seconds'][0]))
 
         # annotations
         video_annotations = None
@@ -1609,7 +1611,7 @@ class YoutubeSearchURLIE(InfoExtractor):
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        query = compat_urllib_parse.unquote_plus(mobj.group('query'))
+        query = compat_urllib_parse_unquote_plus(mobj.group('query'))
 
         webpage = self._download_webpage(url, query)
         result_code = self._search_regex(

From c60e8cfaf7405e8b8075f06a62e7547adad8beba Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:54:38 +0600
Subject: [PATCH 1953/6219] [ehow] Simplify

---
 youtube_dl/extractor/ehow.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/ehow.py b/youtube_dl/extractor/ehow.py
index ba9685991..b1cd4f5d4 100644
--- a/youtube_dl/extractor/ehow.py
+++ b/youtube_dl/extractor/ehow.py
@@ -1,9 +1,7 @@
 from __future__ import unicode_literals
 
-from ..compat import (
-    compat_urllib_parse_unquote,
-)
 from .common import InfoExtractor
+from ..compat import compat_urllib_parse_unquote
 
 
 class EHowIE(InfoExtractor):

From 7bd42d0d963b543aae1f4b4e8c157dc3421189fd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jul 2015 23:56:27 +0600
Subject: [PATCH 1954/6219] [openfilm] Fix compat_urllib_parse_unquote_plus
 import

---
 youtube_dl/extractor/openfilm.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/openfilm.py b/youtube_dl/extractor/openfilm.py
index 7e10bdf2c..d2ceedd01 100644
--- a/youtube_dl/extractor/openfilm.py
+++ b/youtube_dl/extractor/openfilm.py
@@ -3,9 +3,9 @@ from __future__ import unicode_literals
 import json
 
 from .common import InfoExtractor
+from ..compat import compat_urllib_parse_unquote_plus
 from ..utils import (
     parse_iso8601,
-    compat_urllib_parse_unquote_plus,
     parse_age_limit,
     int_or_none,
 )

From 6b19647d5706b75b5e2d7da836495fe5174919f4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jul 2015 00:04:25 +0600
Subject: [PATCH 1955/6219] [veehd] Use compat_urllib_parse_unquote

---
 youtube_dl/extractor/veehd.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/veehd.py b/youtube_dl/extractor/veehd.py
index 346edf485..0d8d832cc 100644
--- a/youtube_dl/extractor/veehd.py
+++ b/youtube_dl/extractor/veehd.py
@@ -5,6 +5,7 @@ import json
 
 from .common import InfoExtractor
 from ..compat import (
+    compat_urllib_parse_unquote,
     compat_urlparse,
 )
 from ..utils import (
@@ -76,7 +77,7 @@ class VeeHDIE(InfoExtractor):
 
         if config_json:
             config = json.loads(config_json)
-            video_url = compat_urlparse.unquote(config['clip']['url'])
+            video_url = compat_urllib_parse_unquote(config['clip']['url'])
 
         if not video_url:
             video_url = self._html_search_regex(

From 3cc8b4c327585ff4dbb045d17d6f5c160df6fdb8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jul 2015 00:24:39 +0600
Subject: [PATCH 1956/6219] [compat] Fix missing _asciire on python 2.6

---
 youtube_dl/compat.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index e950a4688..db0da5828 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -79,6 +79,8 @@ try:
     from urllib.parse import unquote as compat_urllib_parse_unquote
     from urllib.parse import unquote_plus as compat_urllib_parse_unquote_plus
 except ImportError:  # Python 2
+    _asciire = re.compile('([\x00-\x7f]+)') if sys.version_info < (2, 7) else compat_urllib_parse._asciire
+
     # HACK: The following are the correct unquote_to_bytes, unquote and unquote_plus
     # implementations from cpython 3.4.3's stdlib. Python 2's version
     # is apparently broken (see https://github.com/rg3/youtube-dl/pull/6244)
@@ -124,7 +126,7 @@ except ImportError:  # Python 2
             encoding = 'utf-8'
         if errors is None:
             errors = 'replace'
-        bits = compat_urllib_parse._asciire.split(string)
+        bits = _asciire.split(string)
         res = [bits[0]]
         append = res.append
         for i in range(1, len(bits), 2):

From 9441f77faaf078aca2947b7613f61f46b6d0aea2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jul 2015 01:45:36 +0600
Subject: [PATCH 1957/6219] [bliptv] Use xpath for extraction and fix test

---
 youtube_dl/extractor/bliptv.py | 51 +++++++++++++++++-----------------
 1 file changed, 26 insertions(+), 25 deletions(-)

diff --git a/youtube_dl/extractor/bliptv.py b/youtube_dl/extractor/bliptv.py
index fb56cd78d..cd06dbcea 100644
--- a/youtube_dl/extractor/bliptv.py
+++ b/youtube_dl/extractor/bliptv.py
@@ -14,6 +14,8 @@ from ..utils import (
     int_or_none,
     parse_iso8601,
     unescapeHTML,
+    xpath_text,
+    xpath_with_ns,
 )
 
 
@@ -23,10 +25,10 @@ class BlipTVIE(InfoExtractor):
     _TESTS = [
         {
             'url': 'http://blip.tv/cbr/cbr-exclusive-gotham-city-imposters-bats-vs-jokerz-short-3-5796352',
-            'md5': 'c6934ad0b6acf2bd920720ec888eb812',
+            'md5': '80baf1ec5c3d2019037c1c707d676b9f',
             'info_dict': {
                 'id': '5779306',
-                'ext': 'mov',
+                'ext': 'm4v',
                 'title': 'CBR EXCLUSIVE: "Gotham City Imposters" Bats VS Jokerz Short 3',
                 'description': 'md5:9bc31f227219cde65e47eeec8d2dc596',
                 'timestamp': 1323138843,
@@ -128,35 +130,34 @@ class BlipTVIE(InfoExtractor):
 
         rss = self._download_xml('http://blip.tv/rss/flash/%s' % video_id, video_id, 'Downloading video RSS')
 
-        def blip(s):
-            return '{http://blip.tv/dtd/blip/1.0}%s' % s
-
-        def media(s):
-            return '{http://search.yahoo.com/mrss/}%s' % s
-
-        def itunes(s):
-            return '{http://www.itunes.com/dtds/podcast-1.0.dtd}%s' % s
+        def _x(p):
+            return xpath_with_ns(p, {
+                'blip': 'http://blip.tv/dtd/blip/1.0',
+                'media': 'http://search.yahoo.com/mrss/',
+                'itunes': 'http://www.itunes.com/dtds/podcast-1.0.dtd',
+            })
 
         item = rss.find('channel/item')
 
-        video_id = item.find(blip('item_id')).text
-        title = item.find('./title').text
-        description = clean_html(compat_str(item.find(blip('puredescription')).text))
-        timestamp = parse_iso8601(item.find(blip('datestamp')).text)
-        uploader = item.find(blip('user')).text
-        uploader_id = item.find(blip('userid')).text
-        duration = int(item.find(blip('runtime')).text)
-        media_thumbnail = item.find(media('thumbnail'))
-        thumbnail = media_thumbnail.get('url') if media_thumbnail is not None else item.find(itunes('image')).text
-        categories = [category.text for category in item.findall('category')]
+        video_id = xpath_text(item, _x('blip:item_id'), 'video id') or lookup_id
+        title = xpath_text(item, 'title', 'title', fatal=True)
+        description = clean_html(xpath_text(item, _x('blip:puredescription'), 'description'))
+        timestamp = parse_iso8601(xpath_text(item, _x('blip:datestamp'), 'timestamp'))
+        uploader = xpath_text(item, _x('blip:user'), 'uploader')
+        uploader_id = xpath_text(item, _x('blip:userid'), 'uploader id')
+        duration = int_or_none(xpath_text(item, _x('blip:runtime'), 'duration'))
+        media_thumbnail = item.find(_x('media:thumbnail'))
+        thumbnail = (media_thumbnail.get('url') if media_thumbnail is not None
+                     else xpath_text(item, 'image', 'thumbnail'))
+        categories = [category.text for category in item.findall('category') if category is not None]
 
         formats = []
         subtitles_urls = {}
 
-        media_group = item.find(media('group'))
-        for media_content in media_group.findall(media('content')):
+        media_group = item.find(_x('media:group'))
+        for media_content in media_group.findall(_x('media:content')):
             url = media_content.get('url')
-            role = media_content.get(blip('role'))
+            role = media_content.get(_x('blip:role'))
             msg = self._download_webpage(
                 url + '?showplayer=20140425131715&referrer=http://blip.tv&mask=7&skin=flashvars&view=url',
                 video_id, 'Resolving URL for %s' % role)
@@ -175,8 +176,8 @@ class BlipTVIE(InfoExtractor):
                     'url': real_url,
                     'format_id': role,
                     'format_note': media_type,
-                    'vcodec': media_content.get(blip('vcodec')) or 'none',
-                    'acodec': media_content.get(blip('acodec')),
+                    'vcodec': media_content.get(_x('blip:vcodec')) or 'none',
+                    'acodec': media_content.get(_x('blip:acodec')),
                     'filesize': media_content.get('filesize'),
                     'width': int_or_none(media_content.get('width')),
                     'height': int_or_none(media_content.get('height')),

From 530857182df72db028f959c0a58a9daf219d6b22 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jul 2015 01:45:54 +0600
Subject: [PATCH 1958/6219] [bliptv] Add test with missing duration

---
 youtube_dl/extractor/bliptv.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/youtube_dl/extractor/bliptv.py b/youtube_dl/extractor/bliptv.py
index cd06dbcea..a69ee482b 100644
--- a/youtube_dl/extractor/bliptv.py
+++ b/youtube_dl/extractor/bliptv.py
@@ -102,6 +102,20 @@ class BlipTVIE(InfoExtractor):
                 'vcodec': 'none',
             }
         },
+        {
+            # missing duration
+            'url': 'http://blip.tv/rss/flash/6700880',
+            'info_dict': {
+                'id': '6684191',
+                'ext': 'm4v',
+                'title': 'Cowboy Bebop: Gateway Shuffle Review',
+                'description': 'md5:3acc480c0f9ae157f5fe88547ecaf3f8',
+                'timestamp': 1386639757,
+                'upload_date': '20131210',
+                'uploader': 'sfdebris',
+                'uploader_id': '706520',
+            }
+        }
     ]
 
     @staticmethod

From 3c283a381e4f7a69bf57c3ea85aab3c85ce0e309 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jul 2015 02:43:18 +0600
Subject: [PATCH 1959/6219] [sbs] Simplify

---
 youtube_dl/extractor/sbs.py | 40 +++++++++++++++----------------------
 1 file changed, 16 insertions(+), 24 deletions(-)

diff --git a/youtube_dl/extractor/sbs.py b/youtube_dl/extractor/sbs.py
index ab4d1c884..d6ee2d9e2 100644
--- a/youtube_dl/extractor/sbs.py
+++ b/youtube_dl/extractor/sbs.py
@@ -2,12 +2,11 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..utils import remove_end
 
 
 class SBSIE(InfoExtractor):
     IE_DESC = 'sbs.com.au'
-    _VALID_URL = r'https?://(?:www\.)?sbs\.com\.au/ondemand/video/(?:single/)?(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?sbs\.com\.au/(?:ondemand|news)/video/(?:single/)?(?P<id>[0-9]+)'
 
     _TESTS = [{
         # Original URL is handled by the generic IE which finds the iframe:
@@ -17,43 +16,36 @@ class SBSIE(InfoExtractor):
         'info_dict': {
             'id': '320403011771',
             'ext': 'mp4',
-            'title': 'Dingo Conservation',
-            'description': 'Dingoes are on the brink of extinction; most of the animals we think are dingoes are in fact crossbred with wild dogs. This family run a dingo conservation park to prevent their extinction',
+            'title': 'Dingo Conservation (The Feed)',
+            'description': 'md5:f250a9856fca50d22dec0b5b8015f8a5',
             'thumbnail': 're:http://.*\.jpg',
+            'duration': 308,
         },
-        'add_ies': ['generic'],
     }, {
         'url': 'http://www.sbs.com.au/ondemand/video/320403011771/Dingo-Conservation-The-Feed',
         'only_matching': True,
+    }, {
+        'url': 'http://www.sbs.com.au/news/video/471395907773/The-Feed-July-9',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        # the video is in the following iframe
-        iframe_url = 'http://www.sbs.com.au/ondemand/video/single/' + video_id + '?context=web'
-        webpage = self._download_webpage(iframe_url, video_id)
+        webpage = self._download_webpage(
+            'http://www.sbs.com.au/ondemand/video/single/%s?context=web' % video_id, video_id)
 
-        player_params = self._search_regex(
-            r'(?s)(playerParams.+?releaseUrls.+?\n)',
-            webpage, 'playerParams')
-        player_params_js = self._search_regex(
-            r'({.*})',
-            player_params, 'player_param_js')
+        player_params = self._parse_json(
+            self._search_regex(
+                r'(?s)var\s+playerParams\s*=\s*({.+?});', webpage, 'playerParams'),
+            video_id)
 
-        player_params_json = self._parse_json(player_params_js, video_id)
-
-        theplatform_url = player_params_json.get('releaseUrls')['progressive'] or player_params_json.get('releaseUrls')['standard']
-
-        title = remove_end(self._og_search_title(webpage, default=video_id, fatal=False), ' (The Feed)')
-        description = self._html_search_meta('description', webpage)
-        thumbnail = self._og_search_thumbnail(webpage)
+        urls = player_params['releaseUrls']
+        theplatform_url = (urls.get('progressive') or urls.get('standard') or
+                           urls.get('html') or player_params['relatedItemsURL'])
 
         return {
             '_type': 'url_transparent',
             'id': video_id,
             'url': theplatform_url,
-            'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
         }

From 4e1ad6e9a86e03a4309ed63914a08484aaff2784 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jul 2015 04:13:45 +0600
Subject: [PATCH 1960/6219] [videomega] Extend _VALID_URL and improve
 extraction (Closes #6260)

---
 youtube_dl/extractor/videomega.py | 23 ++++++++++-------------
 1 file changed, 10 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/videomega.py b/youtube_dl/extractor/videomega.py
index eb309a7cd..d45f88646 100644
--- a/youtube_dl/extractor/videomega.py
+++ b/youtube_dl/extractor/videomega.py
@@ -8,17 +8,14 @@ from ..compat import compat_urllib_request
 
 
 class VideoMegaIE(InfoExtractor):
-    _VALID_URL = r'''(?x)https?://
-        (?:www\.)?videomega\.tv/
-        (?:iframe\.php|cdn\.php)?\?ref=(?P<id>[A-Za-z0-9]+)
-        '''
+    _VALID_URL = r'https?://(?:www\.)?videomega\.tv/(?:(?:view|iframe|cdn)\.php)?\?ref=(?P<id>[A-Za-z0-9]+)'
     _TEST = {
-        'url': 'http://videomega.tv/?ref=4GNA688SU99US886ANG4',
-        'md5': 'bf5c2f95c4c917536e80936af7bc51e1',
+        'url': 'http://videomega.tv/cdn.php?ref=AOSQBJYKIDDIKYJBQSOA&width=1070&height=600',
+        'md5': 'cc1920a58add3f05c6a93285b84fb3aa',
         'info_dict': {
-            'id': '4GNA688SU99US886ANG4',
+            'id': 'AOSQBJYKIDDIKYJBQSOA',
             'ext': 'mp4',
-            'title': 'BigBuckBunny_320x180',
+            'title': '1254207',
             'thumbnail': 're:^https?://.*\.jpg$',
         }
     }
@@ -26,15 +23,15 @@ class VideoMegaIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        iframe_url = 'http://videomega.tv/cdn.php?ref=%s' % video_id
-        req = compat_urllib_request.Request(iframe_url)
+        req = compat_urllib_request.Request(url)
         req.add_header('Referer', url)
+        req.add_header('Cookie', 'noadvtday=0')
         webpage = self._download_webpage(req, video_id)
 
         title = self._html_search_regex(
-            r'<title>(.*?)</title>', webpage, 'title')
+            r'<title>(.+?)</title>', webpage, 'title')
         title = re.sub(
-            r'(?:^[Vv]ideo[Mm]ega\.tv\s-\s?|\s?-\svideomega\.tv$)', '', title)
+            r'(?:^[Vv]ideo[Mm]ega\.tv\s-\s*|\s*-\svideomega\.tv$)', '', title)
         thumbnail = self._search_regex(
             r'<video[^>]+?poster="([^"]+)"', webpage, 'thumbnail', fatal=False)
         video_url = self._search_regex(
@@ -46,6 +43,6 @@ class VideoMegaIE(InfoExtractor):
             'url': video_url,
             'thumbnail': thumbnail,
             'http_headers': {
-                'Referer': iframe_url,
+                'Referer': url,
             },
         }

From 8a37f53685c3a9227aad139c9faf13df8576abb3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jul 2015 04:25:10 +0600
Subject: [PATCH 1961/6219] [videomega] Revert iframe URL

---
 youtube_dl/extractor/videomega.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/videomega.py b/youtube_dl/extractor/videomega.py
index d45f88646..42d42adb0 100644
--- a/youtube_dl/extractor/videomega.py
+++ b/youtube_dl/extractor/videomega.py
@@ -10,7 +10,7 @@ from ..compat import compat_urllib_request
 class VideoMegaIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?videomega\.tv/(?:(?:view|iframe|cdn)\.php)?\?ref=(?P<id>[A-Za-z0-9]+)'
     _TEST = {
-        'url': 'http://videomega.tv/cdn.php?ref=AOSQBJYKIDDIKYJBQSOA&width=1070&height=600',
+        'url': 'http://videomega.tv/cdn.php?ref=AOSQBJYKIDDIKYJBQSOA',
         'md5': 'cc1920a58add3f05c6a93285b84fb3aa',
         'info_dict': {
             'id': 'AOSQBJYKIDDIKYJBQSOA',
@@ -23,7 +23,8 @@ class VideoMegaIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        req = compat_urllib_request.Request(url)
+        iframe_url = 'http://videomega.tv/cdn.php?ref=%s' % video_id
+        req = compat_urllib_request.Request(iframe_url)
         req.add_header('Referer', url)
         req.add_header('Cookie', 'noadvtday=0')
         webpage = self._download_webpage(req, video_id)
@@ -43,6 +44,6 @@ class VideoMegaIE(InfoExtractor):
             'url': video_url,
             'thumbnail': thumbnail,
             'http_headers': {
-                'Referer': url,
+                'Referer': iframe_url,
             },
         }

From 4bdfef5a1874d99457cf5758c5d8e1cc0c487f08 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jul 2015 04:25:30 +0600
Subject: [PATCH 1962/6219] [videomega] Add tests

---
 youtube_dl/extractor/videomega.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/videomega.py b/youtube_dl/extractor/videomega.py
index 42d42adb0..b2ef637fa 100644
--- a/youtube_dl/extractor/videomega.py
+++ b/youtube_dl/extractor/videomega.py
@@ -9,7 +9,7 @@ from ..compat import compat_urllib_request
 
 class VideoMegaIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?videomega\.tv/(?:(?:view|iframe|cdn)\.php)?\?ref=(?P<id>[A-Za-z0-9]+)'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://videomega.tv/cdn.php?ref=AOSQBJYKIDDIKYJBQSOA',
         'md5': 'cc1920a58add3f05c6a93285b84fb3aa',
         'info_dict': {
@@ -18,7 +18,13 @@ class VideoMegaIE(InfoExtractor):
             'title': '1254207',
             'thumbnail': 're:^https?://.*\.jpg$',
         }
-    }
+    }, {
+        'url': 'http://videomega.tv/cdn.php?ref=AOSQBJYKIDDIKYJBQSOA&width=1070&height=600',
+        'only_matching': True,
+    }, {
+        'url': 'http://videomega.tv/view.php?ref=090051111052065112106089103052052103089106112065052111051090',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From 4211e1941b39cb153505c64ab8fcb31a762674d1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jul 2015 04:27:09 +0600
Subject: [PATCH 1963/6219] [videomega] Add shortcut to _VALID_URL

---
 youtube_dl/extractor/videomega.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/videomega.py b/youtube_dl/extractor/videomega.py
index b2ef637fa..78ff6310a 100644
--- a/youtube_dl/extractor/videomega.py
+++ b/youtube_dl/extractor/videomega.py
@@ -8,7 +8,7 @@ from ..compat import compat_urllib_request
 
 
 class VideoMegaIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?videomega\.tv/(?:(?:view|iframe|cdn)\.php)?\?ref=(?P<id>[A-Za-z0-9]+)'
+    _VALID_URL = r'(?:videomega:|https?://(?:www\.)?videomega\.tv/(?:(?:view|iframe|cdn)\.php)?\?ref=)(?P<id>[A-Za-z0-9]+)'
     _TESTS = [{
         'url': 'http://videomega.tv/cdn.php?ref=AOSQBJYKIDDIKYJBQSOA',
         'md5': 'cc1920a58add3f05c6a93285b84fb3aa',

From 1793d71db61acc9f6cceefbc63d7de2b558b7582 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jul 2015 06:18:03 +0600
Subject: [PATCH 1964/6219] [twitch:stream] Fix channel_id in different case
 (Closes #6263)

---
 youtube_dl/extractor/twitch.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 92b6dc1b8..ab557a1a5 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -346,6 +346,8 @@ class TwitchStreamIE(TwitchBaseIE):
                 'http://www.twitch.tv/%s/profile' % channel_id,
                 'TwitchProfile', channel_id)
 
+        channel_id = stream.get('channel', {}).get('name') or channel_id.lower()
+
         access_token = self._download_json(
             '%s/api/channels/%s/access_token' % (self._API_BASE, channel_id), channel_id,
             'Downloading channel access token')

From 06966677343c4bbb51e8da3f0ed4d4985bdf54f2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jul 2015 06:27:45 +0600
Subject: [PATCH 1965/6219] [twitch:stream] Clarify channel_id reassignment
 rationale (#6263)

---
 youtube_dl/extractor/twitch.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index ab557a1a5..948c8ce39 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -346,6 +346,10 @@ class TwitchStreamIE(TwitchBaseIE):
                 'http://www.twitch.tv/%s/profile' % channel_id,
                 'TwitchProfile', channel_id)
 
+        # Channel name may be typed if different case than the original channel name
+        # (e.g. http://www.twitch.tv/TWITCHPLAYSPOKEMON) that will lead to constructing
+        # an invalid m3u8 URL. Working around by use of original channel name from stream
+        # JSON and fallback to lowercase if it's not available.
         channel_id = stream.get('channel', {}).get('name') or channel_id.lower()
 
         access_token = self._download_json(

From f29ac588ddb5b36ebfe38e3a809a70cf0cb543a6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jul 2015 06:33:50 +0600
Subject: [PATCH 1966/6219] [nationalgeographics] Fix extraction (Closes #6262)

---
 youtube_dl/extractor/nationalgeographic.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/nationalgeographic.py b/youtube_dl/extractor/nationalgeographic.py
index c18640c5a..f793b72f5 100644
--- a/youtube_dl/extractor/nationalgeographic.py
+++ b/youtube_dl/extractor/nationalgeographic.py
@@ -25,8 +25,11 @@ class NationalGeographicIE(InfoExtractor):
         name = url_basename(url)
 
         webpage = self._download_webpage(url, name)
-        feed_url = self._search_regex(r'data-feed-url="([^"]+)"', webpage, 'feed url')
-        guid = self._search_regex(r'data-video-guid="([^"]+)"', webpage, 'guid')
+        feed_url = self._search_regex(
+            r'data-feed-url="([^"]+)"', webpage, 'feed url')
+        guid = self._search_regex(
+            r'id="(?:videoPlayer|player-container)"[^>]+data-guid="([^"]+)"',
+            webpage, 'guid')
 
         feed = self._download_xml('%s?byGuid=%s' % (feed_url, guid), name)
         content = feed.find('.//{http://search.yahoo.com/mrss/}content')

From 1b541d8d6ee5e3543253501d725af7ce4a55cad8 Mon Sep 17 00:00:00 2001
From: sceext <sceext@foxmail.com>
Date: Sat, 18 Jul 2015 12:29:35 +0800
Subject: [PATCH 1967/6219] [iqiyi] fix iqiyi (2015-07-17), update the md5 salt
 (enc_key) to iqiyi latest (2015-07-17) flash player.

---
 youtube_dl/extractor/iqiyi.py | 15 +--------------
 1 file changed, 1 insertion(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index 0f6707d7c..ac5416953 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -212,20 +212,7 @@ class IqiyiIE(InfoExtractor):
         return raw_data
 
     def get_enc_key(self, swf_url, video_id):
-        filename, _ = os.path.splitext(url_basename(swf_url))
-        enc_key_json = self._downloader.cache.load('iqiyi-enc-key', filename)
-        if enc_key_json is not None:
-            return enc_key_json[0]
-
-        req = self._request_webpage(
-            swf_url, video_id, note='download swf content')
-        cn = req.read()
-        cn = zlib.decompress(cn[8:])
-        pt = re.compile(b'MixerRemote\x08(?P<enc_key>.+?)\$&vv')
-        enc_key = self._search_regex(pt, cn, 'enc_key').decode('utf8')
-
-        self._downloader.cache.store('iqiyi-enc-key', filename, [enc_key])
-
+        enc_key = '8e29ab5666d041c3a1ea76e06dabdffb'
         return enc_key
 
     def _real_extract(self, url):

From b0bff54b08a44966d321e8de0ed08dc1dde6b96e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 18 Jul 2015 11:59:45 +0200
Subject: [PATCH 1968/6219] release 2015.07.18

---
 docs/supportedsites.md | 58 +++++++++++++++++++++++++++---------------
 youtube_dl/version.py  |  2 +-
 2 files changed, 39 insertions(+), 21 deletions(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 0ca06c71d..a84878026 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -28,7 +28,7 @@
  - **anitube.se**
  - **AnySex**
  - **Aparat**
- - **AppleDaily**
+ - **AppleDaily**: 臺灣蘋果日報
  - **AppleTrailers**
  - **archive.org**: archive.org videos
  - **ARD**
@@ -45,7 +45,7 @@
  - **audiomack**
  - **audiomack:album**
  - **Azubu**
- - **BaiduVideo**
+ - **BaiduVideo**: 百度视频
  - **bambuser**
  - **bambuser:channel**
  - **Bandcamp**
@@ -106,7 +106,7 @@
  - **Crunchyroll**
  - **crunchyroll:playlist**
  - **CSpan**: C-SPAN
- - **CtsNews**
+ - **CtsNews**: 華視新聞
  - **culturebox.francetvinfo.fr**
  - **dailymotion**
  - **dailymotion:playlist**
@@ -121,7 +121,7 @@
  - **Discovery**
  - **divxstage**: DivxStage
  - **Dotsub**
- - **DouyuTV**
+ - **DouyuTV**: 斗鱼
  - **dramafever**
  - **dramafever:series**
  - **DRBonanza**
@@ -222,7 +222,7 @@
  - **instagram:user**: Instagram user profile
  - **InternetVideoArchive**
  - **IPrima**
- - **iqiyi**
+ - **iqiyi**: 爱奇艺
  - **ivi**: ivi.ru
  - **ivi:compilation**: ivi.ru compilations
  - **Izlesene**
@@ -243,9 +243,15 @@
  - **kontrtube**: KontrTube.ru - Труба зовёт
  - **KrasView**: Красвью
  - **Ku6**
+ - **kuwo:album**: 酷我音乐 - 专辑
+ - **kuwo:category**: 酷我音乐 - 分类
+ - **kuwo:chart**: 酷我音乐 - 排行榜
+ - **kuwo:mv**: 酷我音乐 - MV
+ - **kuwo:singer**: 酷我音乐 - 歌手
+ - **kuwo:song**: 酷我音乐
  - **la7.tv**
  - **Laola1Tv**
- - **Letv**
+ - **Letv**: 乐视网
  - **LetvPlaylist**
  - **LetvTv**
  - **Libsyn**
@@ -297,6 +303,7 @@
  - **MySpace**
  - **MySpace:album**
  - **MySpass**
+ - **Myvi**
  - **myvideo**
  - **MyVidster**
  - **N-JOY**
@@ -312,11 +319,18 @@
  - **NDTV**
  - **NerdCubedFeed**
  - **Nerdist**
+ - **netease:album**: 网易云音乐 - 专辑
+ - **netease:djradio**: 网易云音乐 - 电台
+ - **netease:mv**: 网易云音乐 - MV
+ - **netease:playlist**: 网易云音乐 - 歌单
+ - **netease:program**: 网易云音乐 - 电台节目
+ - **netease:singer**: 网易云音乐 - 歌手
+ - **netease:song**: 网易云音乐
  - **Netzkino**
  - **Newgrounds**
  - **Newstube**
- - **NextMedia**
- - **NextMediaActionNews**
+ - **NextMedia**: 蘋果日報
+ - **NextMediaActionNews**: 蘋果日報 - 動新聞
  - **nfb**: National Film Board of Canada
  - **nfl.com**
  - **nhl.com**
@@ -332,13 +346,14 @@
  - **Nowness**
  - **NowTV**
  - **nowvideo**: NowVideo
- - **npo.nl**
+ - **npo**: npo.nl and ntr.nl
+ - **npo**: npo.nl and ntr.nl
  - **npo.nl:live**
  - **npo.nl:radio**
  - **npo.nl:radio:fragment**
  - **NRK**
  - **NRKPlaylist**
- - **NRKTV**
+ - **NRKTV**: NRK TV and NRK Radio
  - **ntv.ru**
  - **Nuvid**
  - **NYTimes**
@@ -382,11 +397,11 @@
  - **prosiebensat1**: ProSiebenSat.1 Digital
  - **Puls4**
  - **Pyvideo**
- - **qqmusic**
- - **qqmusic:album**
- - **qqmusic:playlist**
- - **qqmusic:singer**
- - **qqmusic:toplist**
+ - **qqmusic**: QQ音乐
+ - **qqmusic:album**: QQ音乐 - 专辑
+ - **qqmusic:playlist**: QQ音乐 - 歌单
+ - **qqmusic:singer**: QQ音乐 - 歌手
+ - **qqmusic:toplist**: QQ音乐 - 排行榜
  - **QuickVid**
  - **R7**
  - **radio.de**
@@ -395,6 +410,7 @@
  - **RadioJavan**
  - **Rai**
  - **RBMARadio**
+ - **RDS**: RDS.ca
  - **RedTube**
  - **Restudy**
  - **ReverbNation**
@@ -495,7 +511,6 @@
  - **TechTalks**
  - **techtv.mit.edu**
  - **ted**
- - **tegenlicht.vpro.nl**
  - **TeleBruxelles**
  - **telecinco.es**
  - **TeleMB**
@@ -551,7 +566,7 @@
  - **Ubu**
  - **udemy**
  - **udemy:course**
- - **UDNEmbed**
+ - **UDNEmbed**: 聯合影音
  - **Ultimedia**
  - **Unistra**
  - **Urort**: NRK P3 Urørt
@@ -613,9 +628,11 @@
  - **wdr:mobile**
  - **WDRMaus**: Sendung mit der Maus
  - **WebOfStories**
+ - **WebOfStoriesPlaylist**
  - **Weibo**
  - **Wimp**
  - **Wistia**
+ - **WNL**
  - **WorldStarHipHop**
  - **wrzuta.pl**
  - **WSJ**: Wall Street Journal
@@ -628,18 +645,19 @@
  - **Xstream**
  - **XTube**
  - **XTubeUser**: XTube user profile
- - **Xuite**
+ - **Xuite**: 隨意窩Xuite影音
  - **XVideos**
  - **XXXYMovies**
  - **Yahoo**: Yahoo screen and movies
- - **Yam**
+ - **Yam**: 蕃薯藤yam天空部落
  - **yandexmusic:album**: Яндекс.Музыка - Альбом
  - **yandexmusic:playlist**: Яндекс.Музыка - Плейлист
  - **yandexmusic:track**: Яндекс.Музыка - Трек
  - **YesJapan**
+ - **yinyuetai:video**: 音悦Tai
  - **Ynet**
  - **YouJizz**
- - **youku**
+ - **youku**: 优酷
  - **YouPorn**
  - **YourUpload**
  - **youtube**: YouTube.com
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 3364647ed..3ad7a2bc0 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.07.07'
+__version__ = '2015.07.18'

From 74fe23ec35483c552f9c253be4c565546f78f001 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jul 2015 16:35:28 +0600
Subject: [PATCH 1969/6219] [extractor/common] Style

---
 youtube_dl/extractor/common.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 5a2d0d995..b9014fc23 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -996,7 +996,7 @@ class InfoExtractor(object):
     def _parse_smil_video(self, video, video_id, base, rtmp_count):
         src = video.get('src')
         if not src:
-            return ([], rtmp_count)
+            return [], rtmp_count
         bitrate = int_or_none(video.get('system-bitrate') or video.get('systemBitrate'), 1000)
         width = int_or_none(video.get('width'))
         height = int_or_none(video.get('height'))
@@ -1009,7 +1009,7 @@ class InfoExtractor(object):
                     proto = 'http'
         ext = video.get('ext')
         if proto == 'm3u8':
-            return (self._extract_m3u8_formats(src, video_id, ext), rtmp_count)
+            return self._extract_m3u8_formats(src, video_id, ext), rtmp_count
         elif proto == 'rtmp':
             rtmp_count += 1
             streamer = video.get('streamer') or base

From dc786d3db59a611cbcb716a476602719e74a34e5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jul 2015 17:22:25 +0600
Subject: [PATCH 1970/6219] [vk:uservideos] Improve extraction

---
 youtube_dl/extractor/vk.py | 27 ++++++++++++++++++---------
 1 file changed, 18 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 8f677cae3..93c5b18c9 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -291,23 +291,32 @@ class VKIE(InfoExtractor):
 class VKUserVideosIE(InfoExtractor):
     IE_NAME = 'vk.com:user-videos'
     IE_DESC = 'vk.com:All of a user\'s videos'
-    _VALID_URL = r'https?://vk\.com/videos(?P<id>[0-9]+)(?:m\?.*)?'
+    _VALID_URL = r'https?://vk\.com/videos(?P<id>-?[0-9]+)$'
     _TEMPLATE_URL = 'https://vk.com/videos'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://vk.com/videos205387401',
         'info_dict': {
             'id': '205387401',
+            'title': "Tom Cruise's Videos",
         },
         'playlist_mincount': 4,
-    }
+    }, {
+        'url': 'http://vk.com/videos-77521',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         page_id = self._match_id(url)
-        page = self._download_webpage(url, page_id)
-        video_ids = orderedSet(
-            m.group(1) for m in re.finditer(r'href="/video([0-9_]+)"', page))
-        url_entries = [
+
+        webpage = self._download_webpage(url, page_id)
+
+        entries = [
             self.url_result(
                 'http://vk.com/video' + video_id, 'VK', video_id=video_id)
-            for video_id in video_ids]
-        return self.playlist_result(url_entries, page_id)
+            for video_id in set(re.findall(r'href="/video(-?[0-9_]+)"', webpage))]
+
+        title = unescapeHTML(self._search_regex(
+            r'<title>\s*([^<]+?)\s+\|\s+\d+\s+videos',
+            webpage, 'title', default=page_id))
+
+        return self.playlist_result(entries, page_id, title)

From 1ecb5d1d83970d6295c655a37c99e390fb5d7dee Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jul 2015 17:23:33 +0600
Subject: [PATCH 1971/6219] [vk] Clarify extractor names

---
 youtube_dl/extractor/vk.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 93c5b18c9..ef94b6894 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -20,7 +20,8 @@ from ..utils import (
 
 
 class VKIE(InfoExtractor):
-    IE_NAME = 'vk.com'
+    IE_NAME = 'vk'
+    IE_DESC = 'VK'
     _VALID_URL = r'''(?x)
                     https?://
                         (?:
@@ -289,8 +290,8 @@ class VKIE(InfoExtractor):
 
 
 class VKUserVideosIE(InfoExtractor):
-    IE_NAME = 'vk.com:user-videos'
-    IE_DESC = 'vk.com:All of a user\'s videos'
+    IE_NAME = 'vk:uservideos'
+    IE_DESC = "VK - User's Videos"
     _VALID_URL = r'https?://vk\.com/videos(?P<id>-?[0-9]+)$'
     _TEMPLATE_URL = 'https://vk.com/videos'
     _TESTS = [{

From c6b68648f45498bcacc71cc8f29696fa93259a7a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jul 2015 17:36:46 +0600
Subject: [PATCH 1972/6219] [bilibili] Show georestriction error

---
 youtube_dl/extractor/bilibili.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/bilibili.py b/youtube_dl/extractor/bilibili.py
index bf60450c2..ecc17ebeb 100644
--- a/youtube_dl/extractor/bilibili.py
+++ b/youtube_dl/extractor/bilibili.py
@@ -41,8 +41,15 @@ class BiliBiliIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        if self._search_regex(r'(此视频不存在或被删除)', webpage, 'error message', default=None):
-            raise ExtractorError('The video does not exist or was deleted', expected=True)
+        if '(此视频不存在或被删除)' in webpage:
+            raise ExtractorError(
+                'The video does not exist or was deleted', expected=True)
+
+        if '>你没有权限浏览！ 由于版权相关问题 我们不对您所在的地区提供服务<' in webpage:
+            raise ExtractorError(
+                'The video is not available in your region due to copyright reasons',
+                expected=True)
+
         video_code = self._search_regex(
             r'(?s)<div itemprop="video".*?>(.*?)</div>', webpage, 'video code')
 

From fec73daaa30d006bc6c5d0483d255b7bc1c256b1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jul 2015 18:05:08 +0600
Subject: [PATCH 1973/6219] [vk:uservideos] Revert orderedSet

---
 youtube_dl/extractor/vk.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index ef94b6894..98bd3a141 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -314,7 +314,7 @@ class VKUserVideosIE(InfoExtractor):
         entries = [
             self.url_result(
                 'http://vk.com/video' + video_id, 'VK', video_id=video_id)
-            for video_id in set(re.findall(r'href="/video(-?[0-9_]+)"', webpage))]
+            for video_id in orderedSet(re.findall(r'href="/video(-?[0-9_]+)"', webpage))]
 
         title = unescapeHTML(self._search_regex(
             r'<title>\s*([^<]+?)\s+\|\s+\d+\s+videos',

From 22603348aa0b3e02c520589dea092507a04ab06a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jul 2015 18:32:52 +0600
Subject: [PATCH 1974/6219] [compat] Fix _asciire

---
 youtube_dl/compat.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index db0da5828..0c57c7aeb 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -79,7 +79,8 @@ try:
     from urllib.parse import unquote as compat_urllib_parse_unquote
     from urllib.parse import unquote_plus as compat_urllib_parse_unquote_plus
 except ImportError:  # Python 2
-    _asciire = re.compile('([\x00-\x7f]+)') if sys.version_info < (2, 7) else compat_urllib_parse._asciire
+    _asciire = (compat_urllib_parse._asciire if hasattr(compat_urllib_parse, '_asciire')
+                else re.compile('([\x00-\x7f]+)'))
 
     # HACK: The following are the correct unquote_to_bytes, unquote and unquote_plus
     # implementations from cpython 3.4.3's stdlib. Python 2's version

From 9ac09ed4deafc77cfd94e6a8827e4b0c47517b5c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jul 2015 19:13:10 +0600
Subject: [PATCH 1975/6219] [bliptv] Remove unused import

---
 youtube_dl/extractor/bliptv.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/bliptv.py b/youtube_dl/extractor/bliptv.py
index a69ee482b..c3296283d 100644
--- a/youtube_dl/extractor/bliptv.py
+++ b/youtube_dl/extractor/bliptv.py
@@ -5,7 +5,6 @@ import re
 from .common import InfoExtractor
 
 from ..compat import (
-    compat_str,
     compat_urllib_request,
     compat_urlparse,
 )

From ee48b6a88f4b8f076699f539ee4c92389fa37643 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jul 2015 19:15:20 +0600
Subject: [PATCH 1976/6219] [vk] Capture error message

---
 youtube_dl/extractor/vk.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 98bd3a141..9487fc9f5 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -205,6 +205,12 @@ class VKIE(InfoExtractor):
 
         info_page = self._download_webpage(info_url, video_id)
 
+        error_message = self._html_search_regex(
+            r'(?s)<!><div[^>]+class="video_layer_message"[^>]*>(.+?)</div>',
+            info_page, 'error message', default=None)
+        if error_message:
+            raise ExtractorError(error_message, expected=True)
+
         if re.search(r'<!>/login\.php\?.*\bact=security_check', info_page):
             raise ExtractorError(
                 'You are trying to log in from an unusual location. You should confirm ownership at vk.com to log in with this IP.',

From e58066e244bf9a04c2853e8829f7939bdefc9084 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jul 2015 19:25:06 +0600
Subject: [PATCH 1977/6219] [vk] Add age restricted video test for reference

---
 youtube_dl/extractor/vk.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 9487fc9f5..c30c5a8e5 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -154,6 +154,11 @@ class VKIE(InfoExtractor):
             'url': 'http://vk.com/feed?z=video-43215063_166094326%2Fbb50cacd3177146d7a',
             'only_matching': True,
         },
+        {
+            # age restricted video, requires vk account credentials
+            'url': 'https://vk.com/video205387401_164765225',
+            'only_matching': True,
+        },
         {
             # vk wrapper
             'url': 'http://www.biqle.ru/watch/847655_160197695',

From bfed4813b224f720e716de4d4f27471ca9053a0b Mon Sep 17 00:00:00 2001
From: linhua55 <5linhua5@gmail.com>
Date: Sat, 18 Jul 2015 23:33:42 +0800
Subject: [PATCH 1978/6219] fix extraction

 for http://www.tudou.com/albumplay/cJAHGih4yYg.html
---
 youtube_dl/extractor/tudou.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tudou.py b/youtube_dl/extractor/tudou.py
index c89de5ba4..8095e18d2 100644
--- a/youtube_dl/extractor/tudou.py
+++ b/youtube_dl/extractor/tudou.py
@@ -9,7 +9,7 @@ from .common import InfoExtractor
 
 
 class TudouIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?tudou\.com/(?:listplay|programs(?:/view)?|albumplay)/.*?/(?P<id>[^/?#]+?)(?:\.html)?/?(?:$|[?#])'
+    _VALID_URL = r'https?://(?:www\.)?tudou\.com/(?:listplay|programs(?:/view)?|albumplay)/?.*/(?P<id>[^/?#]+?)(?:\.html)?/?(?:$|[?#])'
     _TESTS = [{
         'url': 'http://www.tudou.com/listplay/zzdE77v6Mmo/2xN2duXMxmw.html',
         'md5': '140a49ed444bd22f93330985d8475fcb',

From 3f5c6d0c1b955f9312b83fb0e1f3ae59d1230c82 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jul 2015 22:41:34 +0600
Subject: [PATCH 1979/6219] [francetv] Add support for embeds and clean up
 _VALID_URL

---
 youtube_dl/extractor/francetv.py | 44 ++++++++++++++++++++++++++------
 1 file changed, 36 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index b2c984bf2..827400172 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -160,11 +160,19 @@ class FranceTvInfoIE(FranceTVBaseInfoExtractor):
 class FranceTVIE(FranceTVBaseInfoExtractor):
     IE_NAME = 'francetv'
     IE_DESC = 'France 2, 3, 4, 5 and Ô'
-    _VALID_URL = r'''(?x)https?://www\.france[2345o]\.fr/
-        (?:
-            emissions/.*?/(videos|emissions)/(?P<id>[^/?]+)
-        |   (emissions?|jt)/(?P<key>[^/?]+)
-        )'''
+    _VALID_URL = r'''(?x)
+                    https?://
+                        (?:
+                            (?:www\.)?france[2345o]\.fr/
+                                (?:
+                                    emissions/[^/]+/(?:videos|diffusions)?|
+                                    videos
+                                )
+                            /|
+                            embed\.francetv\.fr/\?ue=
+                        )
+                        (?P<id>[^/?]+)
+                    '''
 
     _TESTS = [
         # france2
@@ -232,13 +240,33 @@ class FranceTVIE(FranceTVBaseInfoExtractor):
                 'timestamp': 1410822000,
             },
         },
+        {
+            # francetv embed
+            'url': 'http://embed.francetv.fr/?ue=8d7d3da1e3047c42ade5a5d7dfd3fc87',
+            'info_dict': {
+                'id': 'EV_30231',
+                'ext': 'flv',
+                'title': 'Alcaline, le concert avec Calogero',
+                'description': 'md5:',
+                'upload_date': '20150226',
+                'timestamp': 1424989860,
+            },
+        },
+        {
+            'url': 'http://www.france4.fr/emission/highlander/diffusion-du-17-07-2015-04h05',
+            'only_matching': True,
+        },
+        {
+            'url': 'http://www.franceo.fr/videos/125377617',
+            'only_matching': True,
+        }
     ]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        webpage = self._download_webpage(url, mobj.group('key') or mobj.group('id'))
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
         video_id, catalogue = self._html_search_regex(
-            r'href="http://videos\.francetv\.fr/video/([^@]+@[^"]+)"',
+            r'href="http://videos?\.francetv\.fr/video/([^@]+@[^"]+)"',
             webpage, 'video ID').split('@')
         return self._extract_video(video_id, catalogue)
 

From 5705ee6ef8d49175bf4afac36af4b3b9a4e72596 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jul 2015 22:43:42 +0600
Subject: [PATCH 1980/6219] [francetv] Fix duration extraction

---
 youtube_dl/extractor/francetv.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index 827400172..d4f98ca16 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -87,7 +87,7 @@ class FranceTVBaseInfoExtractor(InfoExtractor):
             'title': info['titre'],
             'description': clean_html(info['synopsis']),
             'thumbnail': compat_urlparse.urljoin('http://pluzz.francetv.fr', info['image']),
-            'duration': float_or_none(info.get('real_duration'), 1000) or parse_duration(info['duree']),
+            'duration': int_or_none(info.get('real_duration')) or parse_duration(info['duree']),
             'timestamp': int_or_none(info['diffusion']['timestamp']),
             'formats': formats,
         }
@@ -250,6 +250,7 @@ class FranceTVIE(FranceTVBaseInfoExtractor):
                 'description': 'md5:',
                 'upload_date': '20150226',
                 'timestamp': 1424989860,
+                'duration': 5400,
             },
         },
         {

From cbd55ade680c5a3ffe5f7b5d36dc01af2c6dc48d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jul 2015 22:56:00 +0600
Subject: [PATCH 1981/6219] [extractor/generic] Add support for francetv embeds

---
 youtube_dl/extractor/generic.py | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index a62287e50..dc24a8a8b 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -407,6 +407,26 @@ class GenericIE(InfoExtractor):
                 'skip_download': 'Requires rtmpdump'
             }
         },
+        # francetv embed
+        {
+            'url': 'http://www.tsprod.com/replay-du-concert-alcaline-de-calogero',
+            'info_dict': {
+                'id': 'EV_30231',
+                'ext': 'mp4',
+                'title': 'Alcaline, le concert avec Calogero',
+                'description': 'md5:61f08036dcc8f47e9cfc33aed08ffaff',
+                'upload_date': '20150226',
+                'timestamp': 1424989860,
+                'duration': 5400,
+            },
+            'params': {
+                # m3u8 downloads
+                'skip_download': True,
+            },
+            'expected_warnings': [
+                'Forbidden'
+            ]
+        },
         # Condé Nast embed
         {
             'url': 'http://www.wired.com/2014/04/honda-asimo/',
@@ -1431,6 +1451,13 @@ class GenericIE(InfoExtractor):
         if mobj is not None:
             return self.url_result(mobj.group('url'), 'ArteTVEmbed')
 
+        # Look for embedded francetv player
+        mobj = re.search(
+            r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?://)?embed\.francetv\.fr/\?ue=.+?)\1',
+            webpage)
+        if mobj is not None:
+            return self.url_result(mobj.group('url'))
+
         # Look for embedded smotri.com player
         smotri_url = SmotriIE._extract_url(webpage)
         if smotri_url:

From 826a7da808448932d17c09a13a106d75ba0c10d7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 19 Jul 2015 00:27:23 +0600
Subject: [PATCH 1982/6219] [rtlnl] Avoid episodes completely (Closes #6275)

---
 youtube_dl/extractor/rtlnl.py | 25 +++++++++++++++++++++----
 1 file changed, 21 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/rtlnl.py b/youtube_dl/extractor/rtlnl.py
index a4d3d73ff..e0c530d64 100644
--- a/youtube_dl/extractor/rtlnl.py
+++ b/youtube_dl/extractor/rtlnl.py
@@ -43,6 +43,21 @@ class RtlNlIE(InfoExtractor):
             'upload_date': '20150215',
             'description': 'Er zijn nieuwe beelden vrijgegeven die vlak na de aanslag in Kopenhagen zijn gemaakt. Op de video is goed te zien hoe omstanders zich bekommeren om één van de slachtoffers, terwijl de eerste agenten ter plaatse komen.',
         }
+    }, {
+        # empty synopsis and missing episodes (see https://github.com/rg3/youtube-dl/issues/6275)
+        'url': 'http://www.rtl.nl/system/videoplayer/derden/rtlnieuws/video_embed.html#uuid=f536aac0-1dc3-4314-920e-3bd1c5b3811a/autoplay=false',
+        'info_dict': {
+            'id': 'f536aac0-1dc3-4314-920e-3bd1c5b3811a',
+            'ext': 'mp4',
+            'title': 'RTL Nieuws - Meer beelden van overval juwelier',
+            'thumbnail': 're:^https?://screenshots\.rtl\.nl/system/thumb/sz=[0-9]+x[0-9]+/uuid=f536aac0-1dc3-4314-920e-3bd1c5b3811a$',
+            'timestamp': 1437233400,
+            'upload_date': '20150718',
+            'duration': 30.474,
+        },
+        'params': {
+            'skip_download': True,
+        },
     }, {
         # encrypted m3u8 streams, georestricted
         'url': 'http://www.rtlxl.nl/#!/afl-2-257632/52a74543-c504-4cde-8aa8-ec66fe8d68a7',
@@ -59,9 +74,11 @@ class RtlNlIE(InfoExtractor):
             uuid)
 
         material = info['material'][0]
-        progname = info['abstracts'][0]['name']
-        subtitle = material['title'] or info['episodes'][0]['name']
-        description = material.get('synopsis') or info['episodes'][0]['synopsis']
+        title = info['abstracts'][0]['name']
+        subtitle = material.get('title')
+        if subtitle:
+            title += ' - %s' % subtitle
+        description = material.get('synopsis')
 
         meta = info.get('meta', {})
 
@@ -107,7 +124,7 @@ class RtlNlIE(InfoExtractor):
 
         return {
             'id': uuid,
-            'title': '%s - %s' % (progname, subtitle),
+            'title': title,
             'formats': formats,
             'timestamp': material['original_date'],
             'description': description,

From 761ee0d827d17eefff0538a99c453e050b6e933e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 19 Jul 2015 02:28:43 +0800
Subject: [PATCH 1983/6219] [iqiyi] Remove unused imports

---
 youtube_dl/extractor/iqiyi.py | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index ac5416953..7caaa1183 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -3,19 +3,13 @@ from __future__ import unicode_literals
 
 import hashlib
 import math
-import os.path
 import random
-import re
 import time
 import uuid
-import zlib
 
 from .common import InfoExtractor
 from ..compat import compat_urllib_parse
-from ..utils import (
-    ExtractorError,
-    url_basename,
-)
+from ..utils import ExtractorError
 
 
 class IqiyiIE(InfoExtractor):

From 3af1fac7b0f43778e44b3b86e0c74bf25fb6f489 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 18 Jul 2015 09:51:59 +0100
Subject: [PATCH 1984/6219] [dcn] Add new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/dcn.py      | 46 ++++++++++++++++++++++++++++++++
 2 files changed, 47 insertions(+)
 create mode 100644 youtube_dl/extractor/dcn.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 06f21064b..cc0da81d1 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -110,6 +110,7 @@ from .dailymotion import (
 )
 from .daum import DaumIE
 from .dbtv import DBTVIE
+from .dcn import DcnIE
 from .dctp import DctpTvIE
 from .deezer import DeezerPlaylistIE
 from .dfb import DFBIE
diff --git a/youtube_dl/extractor/dcn.py b/youtube_dl/extractor/dcn.py
new file mode 100644
index 000000000..5263def4c
--- /dev/null
+++ b/youtube_dl/extractor/dcn.py
@@ -0,0 +1,46 @@
+from .common import InfoExtractor
+
+class DcnIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?dcndigital\.ae/(?:#/)?(?:video/.+|show/\d+/.+?)/(?P<id>\d+)/?'
+    _TEST = {
+        'url': 'http://www.dcndigital.ae/#/show/199074/%D8%B1%D8%AD%D9%84%D8%A9-%D8%A7%D9%84%D8%B9%D9%85%D8%B1-%D8%A7%D9%84%D8%AD%D9%84%D9%82%D8%A9-1/17375/6887',
+        'info_dict':
+        {
+            'id': '17375',
+            'ext': 'm3u8',
+            'title': 'رحلة العمر : الحلقة 1',
+            'description': '"في هذه الحلقة من برنامج رحلة العمر يقدّم الدكتور عمر عبد الكافي تبسيطاً لمناسك الحج والعمرة ويجيب مباشرة على استفسارات حجاج بيت الله الحرام بخصوص مناسك الحج والعمرة1"',
+            'thumbnail': 'http://admin.mangomolo.com/analytics/uploads/71/images/media/2/2cefc09d7bec80afa754682f40e49503.jpg',
+            'duration': '2041'
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        json_data = self._download_json(
+            'http://admin.mangomolo.com/analytics/index.php/plus/video?id='+video_id,
+            video_id
+        )
+        title = json_data['title_ar'];
+        thumbnail = 'http://admin.mangomolo.com/analytics/'+json_data['img'];
+        duration = json_data['duration'];
+        description = json_data['description_ar'];
+        webpage = self._download_webpage(
+            'http://admin.mangomolo.com/analytics/index.php/customers/embed/video?id='+json_data['id']+'&user_id='+json_data['user_id']+'&countries=Q0M=&w=100%&h=100%&filter=DENY&signature='+json_data['signature'],
+            video_id
+        )
+        m3u8_url = self._html_search_regex(
+            r'file: "(?P<m3u8_url>.*?)"',
+            webpage,
+            'm3u8_url',
+            group='m3u8_url'
+        )
+        formats = self._extract_m3u8_formats(m3u8_url, video_id)
+        return {
+            'id': video_id,
+            'title': title,
+            'thumbnail': thumbnail,
+            'duration': duration,
+            'description': description,
+            'formats': formats,
+        }

From 9d681c2bb3b75a666b76d8e346ffab66b65f9132 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 18 Jul 2015 10:00:24 +0100
Subject: [PATCH 1985/6219] remove unnecessary group name

---
 youtube_dl/extractor/dcn.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/dcn.py b/youtube_dl/extractor/dcn.py
index 5263def4c..f76ebda9e 100644
--- a/youtube_dl/extractor/dcn.py
+++ b/youtube_dl/extractor/dcn.py
@@ -30,10 +30,9 @@ class DcnIE(InfoExtractor):
             video_id
         )
         m3u8_url = self._html_search_regex(
-            r'file: "(?P<m3u8_url>.*?)"',
+            r'file:\s*"([^"]+)',
             webpage,
-            'm3u8_url',
-            group='m3u8_url'
+            'm3u8_url'
         )
         formats = self._extract_m3u8_formats(m3u8_url, video_id)
         return {

From 36068ae019138710a3023334db2241ec815e2fe6 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 19 Jul 2015 03:54:59 +0800
Subject: [PATCH 1986/6219] [iqiyi] Comment out some MD5 sums

The value is different on Travis CI server
---
 youtube_dl/extractor/iqiyi.py | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index 7caaa1183..d93b0867d 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -33,56 +33,48 @@ class IqiyiIE(InfoExtractor):
             'title': '名侦探柯南第752集',
         },
         'playlist': [{
-            'md5': '7e49376fecaffa115d951634917fe105',
             'info_dict': {
                 'id': 'e3f585b550a280af23c98b6cb2be19fb_part1',
                 'ext': 'f4v',
                 'title': '名侦探柯南第752集',
             },
         }, {
-            'md5': '41b75ba13bb7ac0e411131f92bc4f6ca',
             'info_dict': {
                 'id': 'e3f585b550a280af23c98b6cb2be19fb_part2',
                 'ext': 'f4v',
                 'title': '名侦探柯南第752集',
             },
         }, {
-            'md5': '0cee1dd0a3d46a83e71e2badeae2aab0',
             'info_dict': {
                 'id': 'e3f585b550a280af23c98b6cb2be19fb_part3',
                 'ext': 'f4v',
                 'title': '名侦探柯南第752集',
             },
         }, {
-            'md5': '4f8ad72373b0c491b582e7c196b0b1f9',
             'info_dict': {
                 'id': 'e3f585b550a280af23c98b6cb2be19fb_part4',
                 'ext': 'f4v',
                 'title': '名侦探柯南第752集',
             },
         }, {
-            'md5': 'd89ad028bcfad282918e8098e811711d',
             'info_dict': {
                 'id': 'e3f585b550a280af23c98b6cb2be19fb_part5',
                 'ext': 'f4v',
                 'title': '名侦探柯南第752集',
             },
         }, {
-            'md5': '9cb1e5c95da25dff0660c32ae50903b7',
             'info_dict': {
                 'id': 'e3f585b550a280af23c98b6cb2be19fb_part6',
                 'ext': 'f4v',
                 'title': '名侦探柯南第752集',
             },
         }, {
-            'md5': '155116e0ff1867bbc9b98df294faabc9',
             'info_dict': {
                 'id': 'e3f585b550a280af23c98b6cb2be19fb_part7',
                 'ext': 'f4v',
                 'title': '名侦探柯南第752集',
             },
         }, {
-            'md5': '53f5db77622ae14fa493ed2a278a082b',
             'info_dict': {
                 'id': 'e3f585b550a280af23c98b6cb2be19fb_part8',
                 'ext': 'f4v',

From 114ed20e6435f9a68eb757f2d6e26e7ea941afbf Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 18 Jul 2015 22:10:57 +0100
Subject: [PATCH 1987/6219] [shahid] Add new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/shahid.py   | 56 ++++++++++++++++++++++++++++++++
 2 files changed, 57 insertions(+)
 create mode 100644 youtube_dl/extractor/shahid.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 06f21064b..6c1d4f08a 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -502,6 +502,7 @@ from .senateisvp import SenateISVPIE
 from .servingsys import ServingSysIE
 from .sexu import SexuIE
 from .sexykarma import SexyKarmaIE
+from .shahid import ShahidIE
 from .shared import SharedIE
 from .sharesix import ShareSixIE
 from .sina import SinaIE
diff --git a/youtube_dl/extractor/shahid.py b/youtube_dl/extractor/shahid.py
new file mode 100644
index 000000000..d492070da
--- /dev/null
+++ b/youtube_dl/extractor/shahid.py
@@ -0,0 +1,56 @@
+from .common import InfoExtractor
+from ..utils import get_element_by_id
+
+class ShahidIE(InfoExtractor):
+    _VALID_URL = r'https?://shahid\.mbc\.net/ar/episode/(?P<id>\d+)/?'
+    _TESTS = [
+        {
+            'url': 'https://shahid.mbc.net/ar/episode/108084/%D8%AE%D9%88%D8%A7%D8%B7%D8%B1-%D8%A7%D9%84%D9%85%D9%88%D8%B3%D9%85-11-%D8%A7%D9%84%D8%AD%D9%84%D9%82%D8%A9-1.html',
+            'info_dict': {
+                'id': '108084',
+                'ext': 'm3u8',
+                'title': 'بسم الله',
+                'description': 'بسم الله'
+            },
+            'params': {
+                # m3u8 download
+                'skip_download': True,
+            }
+        },
+        {
+            #shahid plus subscriber only
+            'url': 'https://shahid.mbc.net/ar/series/90497/%D9%85%D8%B1%D8%A7%D9%8A%D8%A7-2011.html',
+            'only_matching': True
+        }
+    ]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        json_data = self._parse_json(
+            get_element_by_id('jsonld', webpage),
+            video_id
+        )
+        title = json_data['name']
+        thumbnail = json_data['image']
+        categories = json_data['genre']
+        description = json_data['description']
+        player_json_data = self._download_json(
+            'https://shahid.mbc.net/arContent/getPlayerContent-param-.id-'+video_id+'.type-player.html',
+            video_id
+        )['data']
+        if 'url' in player_json_data:
+            m3u8_url = player_json_data['url']
+        else:
+            for error in json_data['error'].values():
+                self.report_warning(error)
+            return
+        formats = self._extract_m3u8_formats(m3u8_url, video_id)
+        return {
+            'id': video_id,
+            'title': title,
+            'thumbnail': thumbnail,
+            'categories': categories,
+            'description': description,
+            'formats': formats,
+        }

From 02c126a7c2453b101505e3c7b8209e49e36fcd71 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 18 Jul 2015 22:55:40 +0100
Subject: [PATCH 1988/6219] [shahid] raise ExtractorError instead of warning

---
 youtube_dl/extractor/shahid.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/shahid.py b/youtube_dl/extractor/shahid.py
index d492070da..6d76ef590 100644
--- a/youtube_dl/extractor/shahid.py
+++ b/youtube_dl/extractor/shahid.py
@@ -1,5 +1,8 @@
 from .common import InfoExtractor
-from ..utils import get_element_by_id
+from ..utils import (
+    get_element_by_id,
+    ExtractorError,
+}
 
 class ShahidIE(InfoExtractor):
     _VALID_URL = r'https?://shahid\.mbc\.net/ar/episode/(?P<id>\d+)/?'
@@ -43,7 +46,7 @@ class ShahidIE(InfoExtractor):
             m3u8_url = player_json_data['url']
         else:
             for error in json_data['error'].values():
-                self.report_warning(error)
+                raise ExtractorError(error)
             return
         formats = self._extract_m3u8_formats(m3u8_url, video_id)
         return {

From 84c0ed50a5491146a8a6b489c530c2335c023f8f Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 18 Jul 2015 23:10:20 +0100
Subject: [PATCH 1989/6219] [shahid] fix backets closing

---
 youtube_dl/extractor/shahid.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/shahid.py b/youtube_dl/extractor/shahid.py
index 6d76ef590..9846c052c 100644
--- a/youtube_dl/extractor/shahid.py
+++ b/youtube_dl/extractor/shahid.py
@@ -2,7 +2,7 @@ from .common import InfoExtractor
 from ..utils import (
     get_element_by_id,
     ExtractorError,
-}
+)
 
 class ShahidIE(InfoExtractor):
     _VALID_URL = r'https?://shahid\.mbc\.net/ar/episode/(?P<id>\d+)/?'

From 3c07a729a652a25caf9aac6143a19d040ecf9a15 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 18 Jul 2015 23:16:57 +0100
Subject: [PATCH 1990/6219] [shahid] don't fail on missing info

---
 youtube_dl/extractor/shahid.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/shahid.py b/youtube_dl/extractor/shahid.py
index 9846c052c..04de866f4 100644
--- a/youtube_dl/extractor/shahid.py
+++ b/youtube_dl/extractor/shahid.py
@@ -35,9 +35,9 @@ class ShahidIE(InfoExtractor):
             video_id
         )
         title = json_data['name']
-        thumbnail = json_data['image']
-        categories = json_data['genre']
-        description = json_data['description']
+        thumbnail = json_data.get('image')
+        categories = json_data.get('genre')
+        description = json_data.get('description')
         player_json_data = self._download_json(
             'https://shahid.mbc.net/arContent/getPlayerContent-param-.id-'+video_id+'.type-player.html',
             video_id

From e89d7e30294d3c5a36e6af5dd730ed543934d40e Mon Sep 17 00:00:00 2001
From: Roland Hieber <rohieb@rohieb.name>
Date: Sun, 19 Jul 2015 03:34:22 +0200
Subject: [PATCH 1991/6219] [tagesschau] add support for more video types
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

I found that currently only tagesschau videos are played. There are some
more shows hosted on tagesschau.de (see [0] for example) which are
easily playable by adjusting the regex. So this patch adds support for:
* tagesthemen
* tagesschau vor 20 Jahren
* tagesschau (mit Gebärdensprache)
* nachtmagazin

Note that some videos don't provide a description, so in order for the
tests to succeed, an ExtractorError needs to get caught.

[0]: http://www.tagesschau.de/multimedia/video/videoarchiv2~_date-20150714.html
---
 youtube_dl/extractor/tagesschau.py | 55 +++++++++++++++++++++++++++---
 1 file changed, 50 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/tagesschau.py b/youtube_dl/extractor/tagesschau.py
index bfe07b024..2a2aafca0 100644
--- a/youtube_dl/extractor/tagesschau.py
+++ b/youtube_dl/extractor/tagesschau.py
@@ -4,11 +4,11 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import parse_filesize
+from ..utils import parse_filesize, ExtractorError
 
 
 class TagesschauIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?tagesschau\.de/multimedia/(?:sendung/ts|video/video)(?P<id>-?[0-9]+)\.html'
+    _VALID_URL = r'https?://(?:www\.)?tagesschau\.de/multimedia/(?:sendung/(ts|tsg|tt|nm)|video/video|tsvorzwanzig)(?P<id>-?[0-9]+)\.html'
 
     _TESTS = [{
         'url': 'http://www.tagesschau.de/multimedia/video/video1399128.html',
@@ -30,6 +30,46 @@ class TagesschauIE(InfoExtractor):
             'title': 'Sendung: tagesschau \t04.12.2014 20:00 Uhr',
             'thumbnail': 're:^http:.*\.jpg$',
         }
+    }, {
+        'url': 'http://www.tagesschau.de/multimedia/sendung/tsg-3771.html',
+        'md5': '90757268b49ef56deae90c7b48928d58',
+        'info_dict': {
+            'id': '3771',
+            'ext': 'mp4',
+            'description': '',
+            'title': 'Sendung: tagesschau (mit Gebärdensprache) \t14.07.2015 20:00 Uhr',
+            'thumbnail': 're:^http:.*\.jpg$',
+        }
+    }, {
+        'url': 'http://www.tagesschau.de/multimedia/sendung/tt-3827.html',
+        'md5': '6e3ebdc75e8d67da966a8d06721eda71',
+        'info_dict': {
+            'id': '3827',
+            'ext': 'mp4',
+            'description': 'md5:d511d0e278b0ad341a95ad9ab992ce66',
+            'title': 'Sendung: tagesthemen \t14.07.2015 22:15 Uhr',
+            'thumbnail': 're:^http:.*\.jpg$',
+        }
+    }, {
+        'url': 'http://www.tagesschau.de/multimedia/sendung/nm-3475.html',
+        'md5': '8a8875a568f0a5ae5ceef93c501a225f',
+        'info_dict': {
+            'id': '3475',
+            'ext': 'mp4',
+            'description': 'md5:ed149f5649cda3dac86813a9d777e131',
+            'title': 'Sendung: nachtmagazin \t15.07.2015 00:15 Uhr',
+            'thumbnail': 're:^http:.*\.jpg$',
+        }
+    }, {
+        'url': 'http://www.tagesschau.de/multimedia/tsvorzwanzig-959.html',
+        'md5': 'be4d6f0421f2acd8abe25ea29f6f015b',
+        'info_dict': {
+            'id': '959',
+            'ext': 'mp4',
+            'description': '',
+            'title': 'Sendung: tagesschau vor 20 Jahren \t14.07.2015 22:45 Uhr',
+            'thumbnail': 're:^http:.*\.jpg$',
+        }
     }]
 
     _FORMATS = {
@@ -102,9 +142,14 @@ class TagesschauIE(InfoExtractor):
             thumbnail_fn = self._search_regex(
                 r'(?s)<img alt="Sendungsbild".*?src="([^"]+)"',
                 webpage, 'thumbnail', fatal=False)
-            description = self._html_search_regex(
-                r'(?s)<p class="teasertext">(.*?)</p>',
-                webpage, 'description', fatal=False)
+            # there are some videos without description
+            description = ""
+            try:
+                description = self._html_search_regex(
+                    r'(?s)<p class="teasertext">(.*?)</p>',
+                    webpage, 'description', fatal=False)
+            except ExtractorError:
+                pass
             title = self._html_search_regex(
                 r'<span class="headline".*?>(.*?)</span>', webpage, 'title')
 

From c51bc70e0feec42664c57228cc4b395a3a629420 Mon Sep 17 00:00:00 2001
From: Roland Hieber <rohieb@rohieb.name>
Date: Sun, 19 Jul 2015 03:52:54 +0200
Subject: [PATCH 1992/6219] [tagesschau] fix test which links to nonexisting
 video

---
 youtube_dl/extractor/tagesschau.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/tagesschau.py b/youtube_dl/extractor/tagesschau.py
index bfe07b024..cf1b37a75 100644
--- a/youtube_dl/extractor/tagesschau.py
+++ b/youtube_dl/extractor/tagesschau.py
@@ -11,13 +11,13 @@ class TagesschauIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?tagesschau\.de/multimedia/(?:sendung/ts|video/video)(?P<id>-?[0-9]+)\.html'
 
     _TESTS = [{
-        'url': 'http://www.tagesschau.de/multimedia/video/video1399128.html',
-        'md5': 'bcdeac2194fb296d599ce7929dfa4009',
+        'url': 'http://www.tagesschau.de/multimedia/video/video-102143.html',
+        'md5': '917a228bc7df7850783bc47979673a09',
         'info_dict': {
-            'id': '1399128',
+            'id': '102143',
             'ext': 'mp4',
-            'title': 'Harald Range, Generalbundesanwalt, zu den Ermittlungen',
-            'description': 'md5:69da3c61275b426426d711bde96463ab',
+            'title': 'Regierungsumbildung in Athen: Neue Minister in Griechenland vereidigt',
+            'description': 'md5:171feccd9d9b3dd54d05d501568f6359',
             'thumbnail': 're:^http:.*\.jpg$',
         },
     }, {

From 3c6ae8b59ed2517891ff19507e4cef2322ef9eb4 Mon Sep 17 00:00:00 2001
From: Roland Hieber <rohieb@rohieb.name>
Date: Sun, 19 Jul 2015 04:27:58 +0200
Subject: [PATCH 1993/6219] [tagesschau] add support for Bericht aus Berlin
 videos

---
 youtube_dl/extractor/tagesschau.py | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tagesschau.py b/youtube_dl/extractor/tagesschau.py
index 2a2aafca0..682f8df8f 100644
--- a/youtube_dl/extractor/tagesschau.py
+++ b/youtube_dl/extractor/tagesschau.py
@@ -8,7 +8,7 @@ from ..utils import parse_filesize, ExtractorError
 
 
 class TagesschauIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?tagesschau\.de/multimedia/(?:sendung/(ts|tsg|tt|nm)|video/video|tsvorzwanzig)(?P<id>-?[0-9]+)\.html'
+    _VALID_URL = r'https?://(?:www\.)?tagesschau\.de/multimedia/(?:sendung/(ts|tsg|tt|nm|bab/bab)|video/video|tsvorzwanzig)(?P<id>-?[0-9]+)(?:~[-_a-zA-Z0-9]*)?\.html'
 
     _TESTS = [{
         'url': 'http://www.tagesschau.de/multimedia/video/video1399128.html',
@@ -70,6 +70,16 @@ class TagesschauIE(InfoExtractor):
             'title': 'Sendung: tagesschau vor 20 Jahren \t14.07.2015 22:45 Uhr',
             'thumbnail': 're:^http:.*\.jpg$',
         }
+    }, {
+        'url': 'http://www.tagesschau.de/multimedia/sendung/bab/bab-3299~_bab-sendung-209.html',
+        'md5': '42e3757018d9908581481a80cc1806da',
+        'info_dict': {
+            'id': '3299',
+            'ext': 'mp4',
+            'description': '',
+            'title': 'Nach dem Referendum: Schaltgespräch nach Athen',
+            'thumbnail': 're:^http:.*\.jpg$',
+        }
     }]
 
     _FORMATS = {

From 726adc43ec184b6d7e05cdbc197aa02c53b32347 Mon Sep 17 00:00:00 2001
From: Roland Hieber <rohieb@rohieb.name>
Date: Sun, 19 Jul 2015 05:09:29 +0200
Subject: [PATCH 1994/6219] [tagesschau] set description=None for empty
 descriptions

---
 youtube_dl/extractor/tagesschau.py | 15 ++++++---------
 1 file changed, 6 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/tagesschau.py b/youtube_dl/extractor/tagesschau.py
index 682f8df8f..4a755c657 100644
--- a/youtube_dl/extractor/tagesschau.py
+++ b/youtube_dl/extractor/tagesschau.py
@@ -36,7 +36,7 @@ class TagesschauIE(InfoExtractor):
         'info_dict': {
             'id': '3771',
             'ext': 'mp4',
-            'description': '',
+            'description': None,
             'title': 'Sendung: tagesschau (mit Gebärdensprache) \t14.07.2015 20:00 Uhr',
             'thumbnail': 're:^http:.*\.jpg$',
         }
@@ -66,7 +66,7 @@ class TagesschauIE(InfoExtractor):
         'info_dict': {
             'id': '959',
             'ext': 'mp4',
-            'description': '',
+            'description': None,
             'title': 'Sendung: tagesschau vor 20 Jahren \t14.07.2015 22:45 Uhr',
             'thumbnail': 're:^http:.*\.jpg$',
         }
@@ -76,7 +76,7 @@ class TagesschauIE(InfoExtractor):
         'info_dict': {
             'id': '3299',
             'ext': 'mp4',
-            'description': '',
+            'description': None,
             'title': 'Nach dem Referendum: Schaltgespräch nach Athen',
             'thumbnail': 're:^http:.*\.jpg$',
         }
@@ -154,12 +154,9 @@ class TagesschauIE(InfoExtractor):
                 webpage, 'thumbnail', fatal=False)
             # there are some videos without description
             description = ""
-            try:
-                description = self._html_search_regex(
-                    r'(?s)<p class="teasertext">(.*?)</p>',
-                    webpage, 'description', fatal=False)
-            except ExtractorError:
-                pass
+            description = self._html_search_regex(
+                r'(?s)<p class="teasertext">(.*?)</p>',
+                webpage, 'description', fatal=False, default=None)
             title = self._html_search_regex(
                 r'<span class="headline".*?>(.*?)</span>', webpage, 'title')
 

From 4951c9f8214153d84033a77a4ef798ae29e1466a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 19 Jul 2015 11:16:29 +0800
Subject: [PATCH 1995/6219] Credit @sceext2 for fixing iQiyi extractor (#6266)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 531ec5767..4fd65f46f 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -132,3 +132,4 @@ George Brighton
 Remita Amine
 Aurélio A. Heckert
 Bernhard Minks
+sceext

From c2d1be8981c0f6b91635c32a3163e01b91d5db54 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 19 Jul 2015 11:20:05 +0800
Subject: [PATCH 1996/6219] [iqiyi] Add skip_download to reduce network traffic

MD5 checksums are commented out in
36068ae019138710a3023334db2241ec815e2fe6, and actual downloads are not
necessary anymore.
---
 youtube_dl/extractor/iqiyi.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index d93b0867d..afb7f4e61 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -81,6 +81,9 @@ class IqiyiIE(InfoExtractor):
                 'title': '名侦探柯南第752集',
             },
         }],
+        'params': {
+            'skip_download': True,
+        },
     }]
 
     _FORMATS_MAP = [

From 0215103e92c7a40aa0cea319e72dcf49f02053fe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 19 Jul 2015 11:13:27 +0200
Subject: [PATCH 1997/6219] [francetv] Remove unused import

---
 youtube_dl/extractor/francetv.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index d4f98ca16..477e22793 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -14,7 +14,6 @@ from ..utils import (
     clean_html,
     ExtractorError,
     int_or_none,
-    float_or_none,
     parse_duration,
     determine_ext,
 )

From 8a7a2089050957c942bde0b9ab70963b2dea8f35 Mon Sep 17 00:00:00 2001
From: slangangular <slangangular@infrablue.org>
Date: Sun, 12 Jul 2015 10:30:51 +0200
Subject: [PATCH 1998/6219] [sportschau] Add support for sportschau.de

Closes #6199.
---
 youtube_dl/extractor/__init__.py   |  1 +
 youtube_dl/extractor/sportschau.py | 43 ++++++++++++++++++++++++++++++
 2 files changed, 44 insertions(+)
 create mode 100644 youtube_dl/extractor/sportschau.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 06f21064b..d70de7690 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -552,6 +552,7 @@ from .sportbox import (
     SportBoxEmbedIE,
 )
 from .sportdeutschland import SportDeutschlandIE
+from .sportschau import SportschauIE
 from .srf import SrfIE
 from .srmediathek import SRMediathekIE
 from .ssa import SSAIE
diff --git a/youtube_dl/extractor/sportschau.py b/youtube_dl/extractor/sportschau.py
new file mode 100644
index 000000000..ff30e1f0e
--- /dev/null
+++ b/youtube_dl/extractor/sportschau.py
@@ -0,0 +1,43 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class SportschauIE(InfoExtractor):
+    IE_NAME = 'Sportschau'
+    _VALID_URL = r'https?://(?:www\.)?sportschau\.de/\w+(?:/\w+)?/video(?P<id>\w+)\.html'
+    _TEST = {
+        'url': 'http://www.sportschau.de/tourdefrance/videoseppeltkokainhatnichtsmitklassischemdopingzutun100.html',
+        'md5': 'a6ef460ab9f4089b079832e06d554cec',
+        'info_dict': {
+            'id': 'seppeltkokainhatnichtsmitklassischemdopingzutun100',
+            'ext': 'mp4',
+            'title': 'Seppelt: "Kokain hat nichts mit klassischem Doping zu tun" - Tour de France - sportschau.de',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'description': 'Der ARD-Doping Experte Hajo Seppelt gibt seine Einschätzung zum ersten Dopingfall der diesjährigen Tour de France um den Italiener Luca Paolini ab.',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        ext = '-mc_defaultQuality-h.json'
+        json_url = url[:-5] + ext
+
+        json = self._download_json(json_url, video_id)
+        thumb_url = json['_previewImage']
+
+        m3u8_url = json['_mediaArray'][1]['_mediaStreamArray'][0]['_stream'][0]
+        m3u8_formats = self._extract_m3u8_formats(m3u8_url, video_id, ext="mp4")
+
+        webpage = self._download_webpage(url, video_id)
+        title = self._html_search_regex(r'<title>(.*?)</title>', webpage, 'title')
+        desc = self._html_search_meta('description', webpage)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': m3u8_formats,
+            'description': desc,
+            'thumbnail': thumb_url,
+        }

From 8b61bfd6389b62f054cdf9dcb3436395c82a8e28 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 19 Jul 2015 11:21:18 +0200
Subject: [PATCH 1999/6219] [sportschau] skip download in test

---
 youtube_dl/extractor/sportschau.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/sportschau.py b/youtube_dl/extractor/sportschau.py
index ff30e1f0e..28797266f 100644
--- a/youtube_dl/extractor/sportschau.py
+++ b/youtube_dl/extractor/sportschau.py
@@ -9,14 +9,17 @@ class SportschauIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?sportschau\.de/\w+(?:/\w+)?/video(?P<id>\w+)\.html'
     _TEST = {
         'url': 'http://www.sportschau.de/tourdefrance/videoseppeltkokainhatnichtsmitklassischemdopingzutun100.html',
-        'md5': 'a6ef460ab9f4089b079832e06d554cec',
         'info_dict': {
             'id': 'seppeltkokainhatnichtsmitklassischemdopingzutun100',
             'ext': 'mp4',
             'title': 'Seppelt: "Kokain hat nichts mit klassischem Doping zu tun" - Tour de France - sportschau.de',
             'thumbnail': 're:^https?://.*\.jpg$',
             'description': 'Der ARD-Doping Experte Hajo Seppelt gibt seine Einschätzung zum ersten Dopingfall der diesjährigen Tour de France um den Italiener Luca Paolini ab.',
-        }
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
     }
 
     def _real_extract(self, url):

From 32470bf619d31605dc9c51ad107839a097f829f4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 19 Jul 2015 11:24:19 +0200
Subject: [PATCH 2000/6219] [sportschau] Improve title extraction

The html '<title>' ends with '- sportschau.de', which shouldn't be part of the title.
---
 youtube_dl/extractor/sportschau.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/sportschau.py b/youtube_dl/extractor/sportschau.py
index 28797266f..bf9b075db 100644
--- a/youtube_dl/extractor/sportschau.py
+++ b/youtube_dl/extractor/sportschau.py
@@ -2,6 +2,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..utils import get_element_by_attribute
 
 
 class SportschauIE(InfoExtractor):
@@ -12,7 +13,7 @@ class SportschauIE(InfoExtractor):
         'info_dict': {
             'id': 'seppeltkokainhatnichtsmitklassischemdopingzutun100',
             'ext': 'mp4',
-            'title': 'Seppelt: "Kokain hat nichts mit klassischem Doping zu tun" - Tour de France - sportschau.de',
+            'title': 'Seppelt: "Kokain hat nichts mit klassischem Doping zu tun"',
             'thumbnail': 're:^https?://.*\.jpg$',
             'description': 'Der ARD-Doping Experte Hajo Seppelt gibt seine Einschätzung zum ersten Dopingfall der diesjährigen Tour de France um den Italiener Luca Paolini ab.',
         },
@@ -34,7 +35,7 @@ class SportschauIE(InfoExtractor):
         m3u8_formats = self._extract_m3u8_formats(m3u8_url, video_id, ext="mp4")
 
         webpage = self._download_webpage(url, video_id)
-        title = self._html_search_regex(r'<title>(.*?)</title>', webpage, 'title')
+        title = get_element_by_attribute('class', 'headline', webpage)
         desc = self._html_search_meta('description', webpage)
 
         return {

From 1dc31c2786b34f833acc5fc646afcf992f71444d Mon Sep 17 00:00:00 2001
From: Zach Bruggeman <mail@bruggie.com>
Date: Fri, 10 Jul 2015 19:27:48 -0700
Subject: [PATCH 2001/6219] [appleconnect] Add new extractor (fixes #6189)

Closes #6190.
---
 youtube_dl/extractor/__init__.py     |  1 +
 youtube_dl/extractor/appleconnect.py | 50 ++++++++++++++++++++++++++++
 2 files changed, 51 insertions(+)
 create mode 100644 youtube_dl/extractor/appleconnect.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index d70de7690..5033d67ed 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -19,6 +19,7 @@ from .anysex import AnySexIE
 from .aol import AolIE
 from .allocine import AllocineIE
 from .aparat import AparatIE
+from .appleconnect import AppleConnectIE
 from .appletrailers import AppleTrailersIE
 from .archiveorg import ArchiveOrgIE
 from .ard import ARDIE, ARDMediathekIE
diff --git a/youtube_dl/extractor/appleconnect.py b/youtube_dl/extractor/appleconnect.py
new file mode 100644
index 000000000..ea7a70393
--- /dev/null
+++ b/youtube_dl/extractor/appleconnect.py
@@ -0,0 +1,50 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    str_to_int,
+    ExtractorError
+)
+
+
+class AppleConnectIE(InfoExtractor):
+    _VALID_URL = r'https?://itunes\.apple\.com/\w{0,2}/?post/idsa\.(?P<id>[\w-]+)'
+    _TEST = {
+        'url': 'https://itunes.apple.com/us/post/idsa.4ab17a39-2720-11e5-96c5-a5b38f6c42d3',
+        'md5': '10d0f2799111df4cb1c924520ca78f98',
+        'info_dict': {
+            'id': '4ab17a39-2720-11e5-96c5-a5b38f6c42d3',
+            'ext': 'm4v',
+            'title': 'Energy',
+            'uploader': 'Drake',
+            'thumbnail': 'http://is5.mzstatic.com/image/thumb/Video5/v4/78/61/c5/7861c5fa-ad6d-294b-1464-cf7605b911d6/source/1920x1080sr.jpg',
+            'upload_date': '20150710',
+            'timestamp': 1436545535,
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        try:
+            video_json = self._html_search_regex(
+                r'class="auc-video-data">(\{.*?\})', webpage, 'json')
+        except ExtractorError:
+            raise ExtractorError('This post doesn\'t contain a video', expected=True)
+
+        video_data = self._parse_json(video_json, video_id)
+        timestamp = str_to_int(self._html_search_regex(r'data-timestamp="(\d+)"', webpage, 'timestamp'))
+        like_count = str_to_int(self._html_search_regex(r'(\d+) Loves', webpage, 'like count'))
+
+        return {
+            'id': video_id,
+            'url': video_data['sslSrc'],
+            'title': video_data['title'],
+            'description': video_data['description'],
+            'uploader': video_data['artistName'],
+            'thumbnail': video_data['artworkUrl'],
+            'timestamp': timestamp,
+            'like_count': like_count,
+        }

From ecdbe09e1017fdd771720f9954f8427b9084bbf7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 19 Jul 2015 21:45:49 +0600
Subject: [PATCH 2002/6219] [francetv] Fix f4m extraction

---
 youtube_dl/extractor/francetv.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index 477e22793..624895ff9 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -58,9 +58,8 @@ class FranceTVBaseInfoExtractor(InfoExtractor):
                     # See https://github.com/rg3/youtube-dl/issues/3963
                     # m3u8 urls work fine
                     continue
-                video_url_parsed = compat_urllib_parse_urlparse(video_url)
                 f4m_url = self._download_webpage(
-                    'http://hdfauth.francetv.fr/esi/TA?url=%s' % video_url_parsed.path,
+                    'http://hdfauth.francetv.fr/esi/TA?url=%s' % video_url,
                     video_id, 'Downloading f4m manifest token', fatal=False)
                 if f4m_url:
                     formats.extend(self._extract_f4m_formats(f4m_url, video_id, 1, format_id))

From 789a12aaafe7036a155435d9c93d062309f547b7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 19 Jul 2015 21:50:25 +0600
Subject: [PATCH 2003/6219] [francetv] Restore support for jt videos

---
 youtube_dl/extractor/francetv.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index 624895ff9..11db6d921 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -164,7 +164,8 @@ class FranceTVIE(FranceTVBaseInfoExtractor):
                             (?:www\.)?france[2345o]\.fr/
                                 (?:
                                     emissions/[^/]+/(?:videos|diffusions)?|
-                                    videos
+                                    videos|
+                                    jt
                                 )
                             /|
                             embed\.francetv\.fr/\?ue=

From 632cbb8efa32ef704219a769972fdeae04577ab8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 19 Jul 2015 21:51:06 +0600
Subject: [PATCH 2004/6219] [francetv] Fix f4m extraction completely

---
 youtube_dl/extractor/francetv.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index 11db6d921..746ab9d84 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -62,7 +62,8 @@ class FranceTVBaseInfoExtractor(InfoExtractor):
                     'http://hdfauth.francetv.fr/esi/TA?url=%s' % video_url,
                     video_id, 'Downloading f4m manifest token', fatal=False)
                 if f4m_url:
-                    formats.extend(self._extract_f4m_formats(f4m_url, video_id, 1, format_id))
+                    formats.extend(self._extract_f4m_formats(
+                        f4m_url + '&hdcore=3.7.0&plugin=aasp-3.7.0.39.44', video_id, 1, format_id))
             elif ext == 'm3u8':
                 formats.extend(self._extract_m3u8_formats(video_url, video_id, 'mp4', m3u8_id=format_id))
             elif video_url.startswith('rtmp'):

From 3bc9fb58892d0ca3dfc044b6e0ef65426ecd8c1a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 19 Jul 2015 21:53:09 +0600
Subject: [PATCH 2005/6219] [francetv] Update jt test

---
 youtube_dl/extractor/francetv.py | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index 746ab9d84..f5aeab110 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -229,15 +229,16 @@ class FranceTVIE(FranceTVBaseInfoExtractor):
         },
         # franceo
         {
-            'url': 'http://www.franceo.fr/jt/info-afrique/04-12-2013',
-            'md5': '52f0bfe202848b15915a2f39aaa8981b',
+            'url': 'http://www.franceo.fr/jt/info-soir/18-07-2015',
+            'md5': '47d5816d3b24351cdce512ad7ab31da8',
             'info_dict': {
-                'id': '108634970',
+                'id': '125377621',
                 'ext': 'flv',
-                'title': 'Infô Afrique',
-                'description': 'md5:ebf346da789428841bee0fd2a935ea55',
-                'upload_date': '20140915',
-                'timestamp': 1410822000,
+                'title': 'Infô soir',
+                'description': 'md5:01b8c6915a3d93d8bbbd692651714309',
+                'upload_date': '20150718',
+                'timestamp': 1437241200,
+                'duration': 414,
             },
         },
         {

From ac4b8df5e47a113da5bb66fc975f938bc176b58c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 19 Jul 2015 21:53:54 +0600
Subject: [PATCH 2006/6219] [francetv] Fix embed test

---
 youtube_dl/extractor/francetv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index f5aeab110..08b124805 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -248,7 +248,7 @@ class FranceTVIE(FranceTVBaseInfoExtractor):
                 'id': 'EV_30231',
                 'ext': 'flv',
                 'title': 'Alcaline, le concert avec Calogero',
-                'description': 'md5:',
+                'description': 'md5:61f08036dcc8f47e9cfc33aed08ffaff',
                 'upload_date': '20150226',
                 'timestamp': 1424989860,
                 'duration': 5400,

From 1d18e26ecad636cb88ee9710807e9583eec98cd4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 19 Jul 2015 21:54:12 +0600
Subject: [PATCH 2007/6219] [francetv] Remove unused import

---
 youtube_dl/extractor/francetv.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index 08b124805..706ed9c99 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -6,10 +6,7 @@ import re
 import json
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse_urlparse,
-    compat_urlparse,
-)
+from ..compat import compat_urlparse
 from ..utils import (
     clean_html,
     ExtractorError,

From 7a4a945f138078b95f02e8cec1e38123d66d7f09 Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Sun, 19 Jul 2015 11:31:38 -0500
Subject: [PATCH 2008/6219] fix/support news.vice.com

---
 youtube_dl/extractor/vice.py | 40 ++++++++++++++++++++++++------------
 1 file changed, 27 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/vice.py b/youtube_dl/extractor/vice.py
index 04e2b0ba7..dfb621468 100644
--- a/youtube_dl/extractor/vice.py
+++ b/youtube_dl/extractor/vice.py
@@ -7,20 +7,34 @@ from ..utils import ExtractorError
 
 
 class ViceIE(InfoExtractor):
-    _VALID_URL = r'http://www\.vice\.com/.*?/(?P<name>.+)'
+    _VALID_URL = r'https?://(.+?\.)?vice\.com/.*?/(?P<name>.+)'
 
-    _TEST = {
-        'url': 'http://www.vice.com/Fringes/cowboy-capitalists-part-1',
-        'info_dict': {
-            'id': '43cW1mYzpia9IlestBjVpd23Yu3afAfp',
-            'ext': 'mp4',
-            'title': 'VICE_COWBOYCAPITALISTS_PART01_v1_VICE_WM_1080p.mov',
-        },
-        'params': {
-            # Requires ffmpeg (m3u8 manifest)
-            'skip_download': True,
-        },
-    }
+    _TESTS = [
+        {
+            'url': 'http://www.vice.com/Fringes/cowboy-capitalists-part-1',
+            'info_dict': {
+                'id': '43cW1mYzpia9IlestBjVpd23Yu3afAfp',
+                'ext': 'mp4',
+                'title': 'VICE_COWBOYCAPITALISTS_PART01_v1_VICE_WM_1080p.mov',
+            },
+            'params': {
+                # Requires ffmpeg (m3u8 manifest)
+                'skip_download': True,
+            },
+        }, {
+            'url': 'https://news.vice.com/video/experimenting-on-animals-inside-the-monkey-lab',
+            'info_dict': {
+                'id': 'N2bzkydjraWDGwnt8jAttCF6Y0PDv4Zj',
+                'ext': 'mp4',
+                'title': 'VICE News - Inside the Monkey Lab',
+                'description': 'md5:1f660d467d3515f29d11e5ef742a4b82',
+            },
+            'params': {
+                # Requires ffmpeg (m3u8 manifest)
+                'skip_download': True,
+            },
+        }
+    ]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)

From a5dd9a0c5dcf6aace1b7af87acb982480d606753 Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Sun, 19 Jul 2015 11:47:58 -0500
Subject: [PATCH 2009/6219] pbs: fix vague 'Full Episode' titles; prepend name
 of show

Many videos are given the title 'Full Episode' or 'Episode x', etc;
in the info json 'title' is the episode title (for oneshots this is
'Full Episode') while the program name is 'program':{'title'. This
changes the title from '(episode title)' to
 '(program name) - (episode title)'

The following urls demonstrate useless titles

    http://video.pbs.org/video/2365367186/
    http://video.pbs.org/video/2365519307/
    http://video.pbs.org/video/2365527039/
    http://video.pbs.org/video/2365530605/

Before change

    Full Episode
    Episode 5 | Preview
    Season 3 | Episode 4 Preview
    July 17, 2015

After change

    To Catch A Comet - Full Episode
    The Crimson Field - Episode 5 | Preview
    Last Tango in Halifax - Season 3 | Episode 4 Preview
    Charlie Rose The Week - July 17, 2015
---
 youtube_dl/extractor/pbs.py | 33 +++++++++++++++++++++++++++------
 1 file changed, 27 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index fec5d65ad..cccff270b 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -32,7 +32,7 @@ class PBSIE(InfoExtractor):
             'info_dict': {
                 'id': '2365006249',
                 'ext': 'mp4',
-                'title': 'A More Perfect Union',
+                'title': 'Constitution USA with Peter Sagal - A More Perfect Union',
                 'description': 'md5:ba0c207295339c8d6eced00b7c363c6a',
                 'duration': 3190,
             },
@@ -46,7 +46,7 @@ class PBSIE(InfoExtractor):
             'info_dict': {
                 'id': '2365297690',
                 'ext': 'mp4',
-                'title': 'Losing Iraq',
+                'title': 'FRONTLINE - Losing Iraq',
                 'description': 'md5:f5bfbefadf421e8bb8647602011caf8e',
                 'duration': 5050,
             },
@@ -60,7 +60,7 @@ class PBSIE(InfoExtractor):
             'info_dict': {
                 'id': '2201174722',
                 'ext': 'mp4',
-                'title': 'Cyber Schools Gain Popularity, but Quality Questions Persist',
+                'title': 'PBS NewsHour - Cyber Schools Gain Popularity, but Quality Questions Persist',
                 'description': 'md5:5871c15cba347c1b3d28ac47a73c7c28',
                 'duration': 801,
             },
@@ -72,7 +72,7 @@ class PBSIE(InfoExtractor):
                 'id': '2365297708',
                 'ext': 'mp4',
                 'description': 'md5:68d87ef760660eb564455eb30ca464fe',
-                'title': 'Dudamel Conducts Verdi Requiem at the Hollywood Bowl - Full',
+                'title': 'Great Performances - Dudamel Conducts Verdi Requiem at the Hollywood Bowl - Full',
                 'duration': 6559,
                 'thumbnail': 're:^https?://.*\.jpg$',
             },
@@ -88,7 +88,7 @@ class PBSIE(InfoExtractor):
                 'display_id': 'killer-typhoon',
                 'ext': 'mp4',
                 'description': 'md5:c741d14e979fc53228c575894094f157',
-                'title': 'Killer Typhoon',
+                'title': 'NOVA - Killer Typhoon',
                 'duration': 3172,
                 'thumbnail': 're:^https?://.*\.jpg$',
                 'upload_date': '20140122',
@@ -110,7 +110,7 @@ class PBSIE(InfoExtractor):
                 'id': '2280706814',
                 'display_id': 'player',
                 'ext': 'mp4',
-                'title': 'Death and the Civil War',
+                'title': 'American Experience - Death and the Civil War',
                 'description': 'American Experience, TV’s most-watched history series, brings to life the compelling stories from our past that inform our understanding of the world today.',
                 'duration': 6705,
                 'thumbnail': 're:^https?://.*\.jpg$',
@@ -118,6 +118,21 @@ class PBSIE(InfoExtractor):
             'params': {
                 'skip_download': True,  # requires ffmpeg
             },
+        },
+        {
+            'url': 'http://video.pbs.org/video/2365367186/',
+            'info_dict': {
+                'id': '2365367186',
+                'display_id': '2365367186',
+                'ext': 'mp4',
+                'title': 'To Catch A Comet - Full Episode',
+                'description': 'On November 12, 2014, billions of kilometers from Earth, spacecraft orbiter Rosetta and lander Philae did what no other had dared to attempt \u2014 land on the volatile surface of a comet as it zooms around the sun at 67,000 km/hr. The European Space Agency hopes this mission can help peer into our past and unlock secrets of our origins.',
+                'duration': 3342,
+                'thumbnail': 're:^https?://.*\.jpg$',
+            },
+            'params': {
+                'skip_download': True,  # requires ffmpeg
+            },
         }
     ]
 
@@ -232,6 +247,12 @@ class PBSIE(InfoExtractor):
                 'url': closed_captions_url,
             }]
 
+        # video.pbs.org video.pbs.org/videoInfo/... frequently provides an obscure 'title' value, like
+        # 'Full Episode', 'Episode 5', etc. prepend program->title
+        alt_title = info.get('program', {}).get('title', '')
+        if alt_title != '':
+            info['title'] = alt_title + ' - ' + re.sub(r'^' + alt_title + '[\s\-\:]+', '', info['title'])
+
         return {
             'id': video_id,
             'display_id': display_id,

From e37c92ec6d137cd2d09c5f13e9a261c4fae552dd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 19 Jul 2015 23:59:50 +0600
Subject: [PATCH 2010/6219] [ard] Extract all formats

---
 youtube_dl/extractor/ard.py | 125 ++++++++++++++++++++++++++----------
 1 file changed, 91 insertions(+), 34 deletions(-)

diff --git a/youtube_dl/extractor/ard.py b/youtube_dl/extractor/ard.py
index 6a35ea463..55f940d57 100644
--- a/youtube_dl/extractor/ard.py
+++ b/youtube_dl/extractor/ard.py
@@ -35,6 +35,87 @@ class ARDMediathekIE(InfoExtractor):
         'skip': 'Blocked outside of Germany',
     }]
 
+    def _extract_media_info(self, media_info_url, webpage, video_id):
+        media_info = self._download_json(
+            media_info_url, video_id, 'Downloading media JSON')
+
+        formats = self._extract_formats(media_info, video_id)
+
+        if not formats:
+            if '"fsk"' in webpage:
+                raise ExtractorError(
+                    'This video is only available after 20:00', expected=True)
+            elif media_info.get('_geoblocked'):
+                raise ExtractorError('This video is not available due to geo restriction', expected=True)
+
+        self._sort_formats(formats)
+
+        duration = int_or_none(media_info.get('_duration'))
+        thumbnail = media_info.get('_previewImage')
+
+        subtitles = {}
+        subtitle_url = media_info.get('_subtitleUrl')
+        if subtitle_url:
+            subtitles['de'] = [{
+                'ext': 'srt',
+                'url': subtitle_url,
+            }]
+
+        return {
+            'id': video_id,
+            'duration': duration,
+            'thumbnail': thumbnail,
+            'formats': formats,
+            'subtitles': subtitles,
+        }
+
+    def _extract_formats(self, media_info, video_id):
+        type_ = media_info.get('_type')
+        media_array = media_info.get('_mediaArray', [])
+        formats = []
+        for num, media in enumerate(media_array):
+            for stream in media.get('_mediaStreamArray', []):
+                stream_urls = stream.get('_stream')
+                if not stream_urls:
+                    continue
+                if not isinstance(stream_urls, list):
+                    stream_urls = [stream_urls]
+                quality = stream.get('_quality')
+                server = stream.get('_server')
+                for stream_url in stream_urls:
+                    ext = determine_ext(stream_url)
+                    if ext == 'f4m':
+                        formats.extend(self._extract_f4m_formats(
+                            stream_url + '?hdcore=3.1.1&plugin=aasp-3.1.1.69.124',
+                            video_id, preference=-1, f4m_id='hds'))
+                    elif ext == 'm3u8':
+                        formats.extend(self._extract_m3u8_formats(
+                            stream_url, video_id, 'mp4', preference=1, m3u8_id='hls'))
+                    else:
+                        if server and server.startswith('rtmp'):
+                            f = {
+                                'url': server,
+                                'play_path': stream_url,
+                                'format_id': 'a%s-rtmp-%s' % (num, quality),
+                            }
+                        elif stream_url.startswith('http'):
+                            f = {
+                                'url': stream_url,
+                                'format_id': 'a%s-%s-%s' % (num, ext, quality)
+                            }
+                        else:
+                            continue
+                        m = re.search(r'_(?P<width>\d+)x(?P<height>\d+)\.mp4$', stream_url)
+                        if m:
+                            f.update({
+                                'width': int(m.group('width')),
+                                'height': int(m.group('height')),
+                            })
+                        if type_ == 'audio':
+                            f['vcodec'] = 'none'
+                        formats.append(f)
+        return formats
+
     def _real_extract(self, url):
         # determine video id from url
         m = re.match(self._VALID_URL, url)
@@ -92,46 +173,22 @@ class ARDMediathekIE(InfoExtractor):
                     'format_id': fid,
                     'url': furl,
                 })
+            self._sort_formats(formats)
+            info = {
+                'formats': formats,
+            }
         else:  # request JSON file
-            media_info = self._download_json(
-                'http://www.ardmediathek.de/play/media/%s' % video_id, video_id)
-            # The second element of the _mediaArray contains the standard http urls
-            streams = media_info['_mediaArray'][1]['_mediaStreamArray']
-            if not streams:
-                if '"fsk"' in webpage:
-                    raise ExtractorError('This video is only available after 20:00')
+            info = self._extract_media_info(
+                'http://www.ardmediathek.de/play/media/%s' % video_id, webpage, video_id)
 
-            formats = []
-            for s in streams:
-                if type(s['_stream']) == list:
-                    for index, url in enumerate(s['_stream'][::-1]):
-                        quality = s['_quality'] + index
-                        formats.append({
-                            'quality': quality,
-                            'url': url,
-                            'format_id': '%s-%s' % (determine_ext(url), quality)
-                        })
-                    continue
-
-                format = {
-                    'quality': s['_quality'],
-                    'url': s['_stream'],
-                }
-
-                format['format_id'] = '%s-%s' % (
-                    determine_ext(format['url']), format['quality'])
-
-                formats.append(format)
-
-        self._sort_formats(formats)
-
-        return {
+        info.update({
             'id': video_id,
             'title': title,
             'description': description,
-            'formats': formats,
             'thumbnail': thumbnail,
-        }
+        })
+
+        return info
 
 
 class ARDIE(InfoExtractor):

From 3bf8c316a663741ab806f48cf9121209aba88b72 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 20 Jul 2015 00:01:22 +0600
Subject: [PATCH 2011/6219] [sportschau] Reimplement in terms of ard extractor

---
 youtube_dl/extractor/__init__.py   |  7 +++--
 youtube_dl/extractor/ard.py        | 39 +++++++++++++++++++++++++
 youtube_dl/extractor/sportschau.py | 47 ------------------------------
 3 files changed, 44 insertions(+), 49 deletions(-)
 delete mode 100644 youtube_dl/extractor/sportschau.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 5033d67ed..50da08830 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -22,7 +22,11 @@ from .aparat import AparatIE
 from .appleconnect import AppleConnectIE
 from .appletrailers import AppleTrailersIE
 from .archiveorg import ArchiveOrgIE
-from .ard import ARDIE, ARDMediathekIE
+from .ard import (
+    ARDIE,
+    ARDMediathekIE,
+    SportschauIE,
+)
 from .arte import (
     ArteTvIE,
     ArteTVPlus7IE,
@@ -553,7 +557,6 @@ from .sportbox import (
     SportBoxEmbedIE,
 )
 from .sportdeutschland import SportDeutschlandIE
-from .sportschau import SportschauIE
 from .srf import SrfIE
 from .srmediathek import SRMediathekIE
 from .ssa import SSAIE
diff --git a/youtube_dl/extractor/ard.py b/youtube_dl/extractor/ard.py
index 55f940d57..2c368d833 100644
--- a/youtube_dl/extractor/ard.py
+++ b/youtube_dl/extractor/ard.py
@@ -8,6 +8,7 @@ from .generic import GenericIE
 from ..utils import (
     determine_ext,
     ExtractorError,
+    get_element_by_attribute,
     qualities,
     int_or_none,
     parse_duration,
@@ -246,3 +247,41 @@ class ARDIE(InfoExtractor):
             'upload_date': upload_date,
             'thumbnail': thumbnail,
         }
+
+
+class SportschauIE(ARDMediathekIE):
+    IE_NAME = 'Sportschau'
+    _VALID_URL = r'(?P<baseurl>https?://(?:www\.)?sportschau\.de/(?:[^/]+/)+video(?P<id>[^/#?]+))\.html'
+    _TEST = {
+        'url': 'http://www.sportschau.de/tourdefrance/videoseppeltkokainhatnichtsmitklassischemdopingzutun100.html',
+        'info_dict': {
+            'id': 'seppeltkokainhatnichtsmitklassischemdopingzutun100',
+            'ext': 'mp4',
+            'title': 'Seppelt: "Kokain hat nichts mit klassischem Doping zu tun"',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'description': 'Der ARD-Doping Experte Hajo Seppelt gibt seine Einschätzung zum ersten Dopingfall der diesjährigen Tour de France um den Italiener Luca Paolini ab.',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        base_url = mobj.group('baseurl')
+
+        webpage = self._download_webpage(url, video_id)
+        title = get_element_by_attribute('class', 'headline', webpage)
+        description = self._html_search_meta('description', webpage, 'description')
+
+        info = self._extract_media_info(
+            base_url + '-mc_defaultQuality-h.json', webpage, video_id)
+
+        info.update({
+            'title': title,
+            'description': description,
+        })
+
+        return info
diff --git a/youtube_dl/extractor/sportschau.py b/youtube_dl/extractor/sportschau.py
deleted file mode 100644
index bf9b075db..000000000
--- a/youtube_dl/extractor/sportschau.py
+++ /dev/null
@@ -1,47 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-from .common import InfoExtractor
-from ..utils import get_element_by_attribute
-
-
-class SportschauIE(InfoExtractor):
-    IE_NAME = 'Sportschau'
-    _VALID_URL = r'https?://(?:www\.)?sportschau\.de/\w+(?:/\w+)?/video(?P<id>\w+)\.html'
-    _TEST = {
-        'url': 'http://www.sportschau.de/tourdefrance/videoseppeltkokainhatnichtsmitklassischemdopingzutun100.html',
-        'info_dict': {
-            'id': 'seppeltkokainhatnichtsmitklassischemdopingzutun100',
-            'ext': 'mp4',
-            'title': 'Seppelt: "Kokain hat nichts mit klassischem Doping zu tun"',
-            'thumbnail': 're:^https?://.*\.jpg$',
-            'description': 'Der ARD-Doping Experte Hajo Seppelt gibt seine Einschätzung zum ersten Dopingfall der diesjährigen Tour de France um den Italiener Luca Paolini ab.',
-        },
-        'params': {
-            # m3u8 download
-            'skip_download': True,
-        },
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        ext = '-mc_defaultQuality-h.json'
-        json_url = url[:-5] + ext
-
-        json = self._download_json(json_url, video_id)
-        thumb_url = json['_previewImage']
-
-        m3u8_url = json['_mediaArray'][1]['_mediaStreamArray'][0]['_stream'][0]
-        m3u8_formats = self._extract_m3u8_formats(m3u8_url, video_id, ext="mp4")
-
-        webpage = self._download_webpage(url, video_id)
-        title = get_element_by_attribute('class', 'headline', webpage)
-        desc = self._html_search_meta('description', webpage)
-
-        return {
-            'id': video_id,
-            'title': title,
-            'formats': m3u8_formats,
-            'description': desc,
-            'thumbnail': thumb_url,
-        }

From 86b4e98ac6f57c3703cd2082a69351ece797111c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 20 Jul 2015 00:05:10 +0600
Subject: [PATCH 2012/6219] [ard:mediathek] Add audio test

---
 youtube_dl/extractor/ard.py | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/ard.py b/youtube_dl/extractor/ard.py
index 2c368d833..fdb94af6b 100644
--- a/youtube_dl/extractor/ard.py
+++ b/youtube_dl/extractor/ard.py
@@ -34,6 +34,17 @@ class ARDMediathekIE(InfoExtractor):
             'description': 'Auf einem restaurierten Hof bei Wolbeck wird der Heilpraktiker Raffael Lembeck eines morgens von seiner Frau Stella tot aufgefunden. Das Opfer war offensichtlich in seiner Praxis zu Fall gekommen und ist dann verblutet, erklärt Prof. Boerne am Tatort.',
         },
         'skip': 'Blocked outside of Germany',
+    }, {
+        # audio
+        'url': 'http://www.ardmediathek.de/tv/WDR-H%C3%B6rspiel-Speicher/Tod-eines-Fu%C3%9Fballers/WDR-3/Audio-Podcast?documentId=28488308&bcastId=23074086',
+        'md5': '219d94d8980b4f538c7fcb0865eb7f2c',
+        'info_dict': {
+            'id': '28488308',
+            'ext': 'mp3',
+            'title': 'Tod eines Fußballers',
+            'description': 'md5:f6e39f3461f0e1f54bfa48c8875c86ef',
+            'duration': 3240,
+        },
     }]
 
     def _extract_media_info(self, media_info_url, webpage, video_id):
@@ -252,7 +263,7 @@ class ARDIE(InfoExtractor):
 class SportschauIE(ARDMediathekIE):
     IE_NAME = 'Sportschau'
     _VALID_URL = r'(?P<baseurl>https?://(?:www\.)?sportschau\.de/(?:[^/]+/)+video(?P<id>[^/#?]+))\.html'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.sportschau.de/tourdefrance/videoseppeltkokainhatnichtsmitklassischemdopingzutun100.html',
         'info_dict': {
             'id': 'seppeltkokainhatnichtsmitklassischemdopingzutun100',
@@ -265,7 +276,7 @@ class SportschauIE(ARDMediathekIE):
             # m3u8 download
             'skip_download': True,
         },
-    }
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)

From 769efa16afc1c9a2871543f69081f1bced361dc6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 20 Jul 2015 00:08:21 +0600
Subject: [PATCH 2013/6219] [ard:mediathek] Remove dead test

---
 youtube_dl/extractor/ard.py | 20 ++++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/ard.py b/youtube_dl/extractor/ard.py
index fdb94af6b..c78475748 100644
--- a/youtube_dl/extractor/ard.py
+++ b/youtube_dl/extractor/ard.py
@@ -23,17 +23,18 @@ class ARDMediathekIE(InfoExtractor):
     _VALID_URL = r'^https?://(?:(?:www\.)?ardmediathek\.de|mediathek\.daserste\.de)/(?:.*/)(?P<video_id>[0-9]+|[^0-9][^/\?]+)[^/\?]*(?:\?.*)?'
 
     _TESTS = [{
-        'url': 'http://mediathek.daserste.de/sendungen_a-z/328454_anne-will/22429276_vertrauen-ist-gut-spionieren-ist-besser-geht',
-        'only_matching': True,
-    }, {
-        'url': 'http://www.ardmediathek.de/tv/Tatort/Das-Wunder-von-Wolbeck-Video-tgl-ab-20/Das-Erste/Video?documentId=22490580&bcastId=602916',
+        'url': 'http://www.ardmediathek.de/tv/Dokumentation-und-Reportage/Ich-liebe-das-Leben-trotzdem/rbb-Fernsehen/Video?documentId=29582122&bcastId=3822114',
         'info_dict': {
-            'id': '22490580',
+            'id': '29582122',
             'ext': 'mp4',
-            'title': 'Das Wunder von Wolbeck (Video tgl. ab 20 Uhr)',
-            'description': 'Auf einem restaurierten Hof bei Wolbeck wird der Heilpraktiker Raffael Lembeck eines morgens von seiner Frau Stella tot aufgefunden. Das Opfer war offensichtlich in seiner Praxis zu Fall gekommen und ist dann verblutet, erklärt Prof. Boerne am Tatort.',
+            'title': 'Ich liebe das Leben trotzdem',
+            'description': 'md5:45e4c225c72b27993314b31a84a5261c',
+            'duration': 4557,
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
         },
-        'skip': 'Blocked outside of Germany',
     }, {
         # audio
         'url': 'http://www.ardmediathek.de/tv/WDR-H%C3%B6rspiel-Speicher/Tod-eines-Fu%C3%9Fballers/WDR-3/Audio-Podcast?documentId=28488308&bcastId=23074086',
@@ -45,6 +46,9 @@ class ARDMediathekIE(InfoExtractor):
             'description': 'md5:f6e39f3461f0e1f54bfa48c8875c86ef',
             'duration': 3240,
         },
+    }, {
+        'url': 'http://mediathek.daserste.de/sendungen_a-z/328454_anne-will/22429276_vertrauen-ist-gut-spionieren-ist-besser-geht',
+        'only_matching': True,
     }]
 
     def _extract_media_info(self, media_info_url, webpage, video_id):

From d719c6a5abf746c31f301ed1a1d8934715eaeb05 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 20 Jul 2015 00:13:39 +0600
Subject: [PATCH 2014/6219] [ard:mediathek] Add test for direct mp4

---
 youtube_dl/extractor/ard.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/youtube_dl/extractor/ard.py b/youtube_dl/extractor/ard.py
index c78475748..6f465789b 100644
--- a/youtube_dl/extractor/ard.py
+++ b/youtube_dl/extractor/ard.py
@@ -35,6 +35,16 @@ class ARDMediathekIE(InfoExtractor):
             # m3u8 download
             'skip_download': True,
         },
+    }, {
+        'url': 'http://www.ardmediathek.de/tv/Tatort/Tatort-Scheinwelten-H%C3%B6rfassung-Video/Das-Erste/Video?documentId=29522730&bcastId=602916',
+        'md5': 'f4d98b10759ac06c0072bbcd1f0b9e3e',
+        'info_dict': {
+            'id': '29522730',
+            'ext': 'mp4',
+            'title': 'Tatort: Scheinwelten - Hörfassung (Video tgl. ab 20 Uhr)',
+            'description': 'md5:196392e79876d0ac94c94e8cdb2875f1',
+            'duration': 5252,
+        },
     }, {
         # audio
         'url': 'http://www.ardmediathek.de/tv/WDR-H%C3%B6rspiel-Speicher/Tod-eines-Fu%C3%9Fballers/WDR-3/Audio-Podcast?documentId=28488308&bcastId=23074086',

From 772acaf31f4614d390a445fe4c0d4e533c165774 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 20 Jul 2015 00:39:55 +0600
Subject: [PATCH 2015/6219] [vice] Do not capture unused groups in _VALID_URL

---
 youtube_dl/extractor/vice.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vice.py b/youtube_dl/extractor/vice.py
index dfb621468..e3ad4ef54 100644
--- a/youtube_dl/extractor/vice.py
+++ b/youtube_dl/extractor/vice.py
@@ -7,7 +7,7 @@ from ..utils import ExtractorError
 
 
 class ViceIE(InfoExtractor):
-    _VALID_URL = r'https?://(.+?\.)?vice\.com/.*?/(?P<name>.+)'
+    _VALID_URL = r'https?://(?:.+?\.)?vice\.com/.*?/(?P<name>.+)'
 
     _TESTS = [
         {

From 65c2b21df15053581cac21690605d1f7ac834083 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 20 Jul 2015 00:40:43 +0600
Subject: [PATCH 2016/6219] [vice] Make test only matching

---
 youtube_dl/extractor/vice.py | 11 +----------
 1 file changed, 1 insertion(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/vice.py b/youtube_dl/extractor/vice.py
index e3ad4ef54..ae1faca1c 100644
--- a/youtube_dl/extractor/vice.py
+++ b/youtube_dl/extractor/vice.py
@@ -23,16 +23,7 @@ class ViceIE(InfoExtractor):
             },
         }, {
             'url': 'https://news.vice.com/video/experimenting-on-animals-inside-the-monkey-lab',
-            'info_dict': {
-                'id': 'N2bzkydjraWDGwnt8jAttCF6Y0PDv4Zj',
-                'ext': 'mp4',
-                'title': 'VICE News - Inside the Monkey Lab',
-                'description': 'md5:1f660d467d3515f29d11e5ef742a4b82',
-            },
-            'params': {
-                # Requires ffmpeg (m3u8 manifest)
-                'skip_download': True,
-            },
+            'only_matching': True,
         }
     ]
 

From 18ae46ad4bcf4d427984e85a9feaf647c3f80737 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 20 Jul 2015 00:42:25 +0600
Subject: [PATCH 2017/6219] [vice] Modernize

---
 youtube_dl/extractor/vice.py | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/vice.py b/youtube_dl/extractor/vice.py
index ae1faca1c..01af7a995 100644
--- a/youtube_dl/extractor/vice.py
+++ b/youtube_dl/extractor/vice.py
@@ -1,5 +1,4 @@
 from __future__ import unicode_literals
-import re
 
 from .common import InfoExtractor
 from .ooyala import OoyalaIE
@@ -7,7 +6,7 @@ from ..utils import ExtractorError
 
 
 class ViceIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:.+?\.)?vice\.com/.*?/(?P<name>.+)'
+    _VALID_URL = r'https?://(?:.+?\.)?vice\.com/(?:[^/]+/)+(?P<id>.+)'
 
     _TESTS = [
         {
@@ -28,9 +27,8 @@ class ViceIE(InfoExtractor):
     ]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        name = mobj.group('name')
-        webpage = self._download_webpage(url, name)
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
         try:
             embed_code = self._search_regex(
                 r'embedCode=([^&\'"]+)', webpage,

From 0eacd2aaae7d4150d9cf4e1dd8ffc2ebc7ab030b Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Sun, 19 Jul 2015 13:59:12 -0500
Subject: [PATCH 2018/6219] less clunky if statement

---
 youtube_dl/extractor/pbs.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index cccff270b..ccbe2a9f3 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -249,8 +249,8 @@ class PBSIE(InfoExtractor):
 
         # video.pbs.org video.pbs.org/videoInfo/... frequently provides an obscure 'title' value, like
         # 'Full Episode', 'Episode 5', etc. prepend program->title
-        alt_title = info.get('program', {}).get('title', '')
-        if alt_title != '':
+        alt_title = info.get('program', {}).get('title')
+        if alt_title:
             info['title'] = alt_title + ' - ' + re.sub(r'^' + alt_title + '[\s\-\:]+', '', info['title'])
 
         return {

From 308c505c3d66b2a11df8c1a5cd7e0fdaa694b585 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 20 Jul 2015 01:03:43 +0600
Subject: [PATCH 2019/6219] [francetv] Improve _VALID_URL

---
 youtube_dl/extractor/francetv.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index 706ed9c99..75723c00d 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -161,7 +161,8 @@ class FranceTVIE(FranceTVBaseInfoExtractor):
                         (?:
                             (?:www\.)?france[2345o]\.fr/
                                 (?:
-                                    emissions/[^/]+/(?:videos|diffusions)?|
+                                    emissions/[^/]+/(?:videos|diffusions)|
+                                    emission/[^/]+|
                                     videos|
                                     jt
                                 )

From faa1f83ab481a1f456db13d3ead7f0f22908aab1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 20 Jul 2015 02:16:07 +0600
Subject: [PATCH 2020/6219] [twitch:stream] Improve _VALID_URL (Closes #6294)

---
 youtube_dl/extractor/twitch.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 948c8ce39..73ce335b7 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -310,9 +310,9 @@ class TwitchBookmarksIE(TwitchPlaylistBaseIE):
 
 class TwitchStreamIE(TwitchBaseIE):
     IE_NAME = 'twitch:stream'
-    _VALID_URL = r'%s/(?P<id>[^/]+)/?(?:\#.*)?$' % TwitchBaseIE._VALID_URL_BASE
+    _VALID_URL = r'%s/(?P<id>[^/#?]+)/?(?:\#.*)?$' % TwitchBaseIE._VALID_URL_BASE
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.twitch.tv/shroomztv',
         'info_dict': {
             'id': '12772022048',
@@ -331,7 +331,10 @@ class TwitchStreamIE(TwitchBaseIE):
             # m3u8 download
             'skip_download': True,
         },
-    }
+    }, {
+        'url': 'http://www.twitch.tv/miracle_doto#profile-0',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         channel_id = self._match_id(url)

From a1b85269a49404cab1f549d7c59352fe48dfbf19 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 20 Jul 2015 18:49:53 +0800
Subject: [PATCH 2021/6219] [extractor/generic] Support vid.me embeds

---
 youtube_dl/extractor/generic.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index dc24a8a8b..cd133a10c 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1196,6 +1196,12 @@ class GenericIE(InfoExtractor):
         if vimeo_url is not None:
             return self.url_result(vimeo_url)
 
+        vid_me_embed_url = self._search_regex(
+            r'src=[\'"](https?://vid\.me/[^\'"]+)[\'"]',
+            webpage, 'vid.me embed', default=None)
+        if vid_me_embed_url is not None:
+            return self.url_result(vid_me_embed_url, 'Vidme')
+
         # Look for embedded YouTube player
         matches = re.findall(r'''(?x)
             (?:

From c909e5820e94ecfefd9a2eb41b1beaa684c530c4 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 20 Jul 2015 18:51:25 +0800
Subject: [PATCH 2022/6219] [tumblr] Delegate to GenericIE for non-tumblr
 videos

Fixes #6162
---
 youtube_dl/extractor/tumblr.py | 23 +++++------------------
 1 file changed, 5 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/tumblr.py b/youtube_dl/extractor/tumblr.py
index 9ead13a91..772f94f7d 100644
--- a/youtube_dl/extractor/tumblr.py
+++ b/youtube_dl/extractor/tumblr.py
@@ -4,8 +4,6 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from .pornhub import PornHubIE
-from .vimeo import VimeoIE
 
 
 class TumblrIE(InfoExtractor):
@@ -60,25 +58,14 @@ class TumblrIE(InfoExtractor):
         blog = m_url.group('blog_name')
 
         url = 'http://%s.tumblr.com/post/%s/' % (blog, video_id)
-        webpage = self._download_webpage(url, video_id)
-
-        vid_me_embed_url = self._search_regex(
-            r'src=[\'"](https?://vid\.me/[^\'"]+)[\'"]',
-            webpage, 'vid.me embed', default=None)
-        if vid_me_embed_url is not None:
-            return self.url_result(vid_me_embed_url, 'Vidme')
-
-        pornhub_url = PornHubIE._extract_url(webpage)
-        if pornhub_url:
-            return self.url_result(pornhub_url, 'PornHub')
-
-        vimeo_url = VimeoIE._extract_vimeo_url(url, webpage)
-        if vimeo_url:
-            return self.url_result(vimeo_url, 'Vimeo')
+        webpage, urlh = self._download_webpage_handle(url, video_id)
 
         iframe_url = self._search_regex(
             r'src=\'(https?://www\.tumblr\.com/video/[^\']+)\'',
-            webpage, 'iframe url')
+            webpage, 'iframe url', default=None)
+        if iframe_url is None:
+            return self.url_result(urlh.geturl(), 'Generic')
+
         iframe = self._download_webpage(iframe_url, video_id)
         video_url = self._search_regex(r'<source src="([^"]+)"',
                                        iframe, 'video url')

From 3f302bca8cf29f618d1f1c2fb7bc4ea9538af805 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 20 Jul 2015 18:54:47 +0800
Subject: [PATCH 2023/6219] [tumblr] Improve downloading notes

---
 youtube_dl/extractor/tumblr.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tumblr.py b/youtube_dl/extractor/tumblr.py
index 772f94f7d..3d3b635e4 100644
--- a/youtube_dl/extractor/tumblr.py
+++ b/youtube_dl/extractor/tumblr.py
@@ -66,7 +66,8 @@ class TumblrIE(InfoExtractor):
         if iframe_url is None:
             return self.url_result(urlh.geturl(), 'Generic')
 
-        iframe = self._download_webpage(iframe_url, video_id)
+        iframe = self._download_webpage(iframe_url, video_id,
+                                        'Downloading iframe page')
         video_url = self._search_regex(r'<source src="([^"]+)"',
                                        iframe, 'video url')
 

From 17ee98e1a5a6897ed0d9c52d9257b45ed9c3f20a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 20 Jul 2015 19:48:50 +0200
Subject: [PATCH 2024/6219] [youtube] Fix upload_date in test

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 9a08924ef..f73032abd 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -543,7 +543,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             'info_dict': {
                 'id': 'CsmdDsKjzN8',
                 'ext': 'mp4',
-                'upload_date': '20150510',
+                'upload_date': '20150501',  # According to '<meta itemprop="datePublished"', but in other places it's 20150510
                 'uploader': 'Airtek',
                 'description': 'Retransmisión en directo de la XVIII media maratón de Zaragoza.',
                 'uploader_id': 'UCzTzUmjXxxacNnL8I3m4LnQ',

From 2fe1ff85829244f6498dde8c85ecdc1415f5ad7a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 20 Jul 2015 20:14:20 +0200
Subject: [PATCH 2025/6219] [youtube] Set 'is_live'

---
 youtube_dl/extractor/youtube.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index f73032abd..3d8b31f98 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -915,6 +915,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
 
         # Get video info
         embed_webpage = None
+        is_live = None
         if re.search(r'player-age-gate-content">', video_webpage) is not None:
             age_gate = True
             # We simulate the access to the video from www.youtube.com/v/{video_id}
@@ -947,6 +948,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     # Convert to the same format returned by compat_parse_qs
                     video_info = dict((k, [v]) for k, v in args.items())
                     add_dash_mpd(video_info)
+                if args.get('livestream') == '1' or args.get('live_playback') == 1:
+                    is_live = True
             if not video_info or self._downloader.params.get('youtube_include_dash_manifest', True):
                 # We also try looking in get_video_info since it may contain different dashmpd
                 # URL that points to a DASH manifest with possibly different itag set (some itags
@@ -1251,6 +1254,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             'dislike_count': dislike_count,
             'average_rating': float_or_none(video_info.get('avg_rating', [None])[0]),
             'formats': formats,
+            'is_live': is_live,
         }
 
 

From 8250c32f49fed0986bd514d1435e00a0182e8314 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 20 Jul 2015 20:25:53 +0200
Subject: [PATCH 2026/6219] [YoutubeDL] don't default to
 'bestvideo+bestaudio/best' for live videos

Doesn't work currently.
---
 youtube_dl/YoutubeDL.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 00af78e06..702a6ad50 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1104,7 +1104,8 @@ class YoutubeDL(object):
         if req_format is None:
             req_format_list = []
             if (self.params.get('outtmpl', DEFAULT_OUTTMPL) != '-' and
-                    info_dict['extractor'] in ['youtube', 'ted']):
+                    info_dict['extractor'] in ['youtube', 'ted'] and
+                    not info_dict.get('is_live')):
                 merger = FFmpegMergerPP(self)
                 if merger.available and merger.can_merge():
                     req_format_list.append('bestvideo+bestaudio')

From 7c80519cbf7e3daa029239fbbd147652817877f8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 20 Jul 2015 21:10:28 +0200
Subject: [PATCH 2027/6219] [youtube] Extract start_time

From the 't=*' in the url.
Currently youtube-dl doesn't use the value, but it was requested for the mpv plugin.
---
 youtube_dl/extractor/common.py  |  2 ++
 youtube_dl/extractor/youtube.py | 14 +++++++++++++-
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index b9014fc23..9e8751877 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -183,6 +183,8 @@ class InfoExtractor(object):
                     ["Sports", "Berlin"]
     is_live:        True, False, or None (=unknown). Whether this video is a
                     live stream that goes on instead of a fixed-length video.
+    start_time:     Time in seconds where the reproduction should start, as
+                    specified in the url.
 
     Unless mentioned otherwise, the fields should be Unicode strings.
 
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 3d8b31f98..afbd34f4a 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -19,6 +19,7 @@ from ..compat import (
     compat_urllib_parse,
     compat_urllib_parse_unquote,
     compat_urllib_parse_unquote_plus,
+    compat_urllib_parse_urlparse,
     compat_urllib_request,
     compat_urlparse,
     compat_str,
@@ -31,6 +32,7 @@ from ..utils import (
     get_element_by_id,
     int_or_none,
     orderedSet,
+    parse_duration,
     str_to_int,
     unescapeHTML,
     unified_strdate,
@@ -317,7 +319,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
     IE_NAME = 'youtube'
     _TESTS = [
         {
-            'url': 'http://www.youtube.com/watch?v=BaW_jenozKc',
+            'url': 'http://www.youtube.com/watch?v=BaW_jenozKcj&t=1s',
             'info_dict': {
                 'id': 'BaW_jenozKc',
                 'ext': 'mp4',
@@ -329,6 +331,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'categories': ['Science & Technology'],
                 'like_count': int,
                 'dislike_count': int,
+                'start_time': 1,
             }
         },
         {
@@ -889,6 +892,14 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             'http' if self._downloader.params.get('prefer_insecure', False)
             else 'https')
 
+        start_time = None
+        parsed_url = compat_urllib_parse_urlparse(url)
+        for component in [parsed_url.fragment, parsed_url.query]:
+            query = compat_parse_qs(component)
+            if 't' in query:
+                start_time = parse_duration(query['t'][0])
+                break
+
         # Extract original video URL from URL with redirection, like age verification, using next_url parameter
         mobj = re.search(self._NEXT_URL_RE, url)
         if mobj:
@@ -1255,6 +1266,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             'average_rating': float_or_none(video_info.get('avg_rating', [None])[0]),
             'formats': formats,
             'is_live': is_live,
+            'start_time': start_time,
         }
 
 

From f993afb26d20ac79891621494922887b3c8df02b Mon Sep 17 00:00:00 2001
From: Jakub Wilk <jwilk@jwilk.net>
Date: Mon, 20 Jul 2015 23:10:29 +0200
Subject: [PATCH 2028/6219] README: fix a typo

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index a2cc89cdb..ac54d7b67 100644
--- a/README.md
+++ b/README.md
@@ -75,7 +75,7 @@ which means you can modify it, redistribute it or use it however you like.
 ## Video Selection:
     --playlist-start NUMBER          Playlist video to start at (default is 1)
     --playlist-end NUMBER            Playlist video to end at (default is last)
-    --playlist-items ITEM_SPEC       Playlist video items to download. Specify indices of the videos in the playlist seperated by commas like: "--playlist-items 1,2,5,8"
+    --playlist-items ITEM_SPEC       Playlist video items to download. Specify indices of the videos in the playlist separated by commas like: "--playlist-items 1,2,5,8"
                                      if you want to download videos indexed 1, 2, 5, 8 in the playlist. You can specify range: "--playlist-items 1-3,7,10-13", it will
                                      download the videos at index 1, 2, 3, 7, 10, 11, 12 and 13.
     --match-title REGEX              Download only matching titles (regex or caseless sub-string)

From 4eb59a6b1cd1c4ca5f7dd284f0c26a329bb8e973 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 21 Jul 2015 09:11:23 +0600
Subject: [PATCH 2029/6219] [options] Fix a typo (#6307)

---
 youtube_dl/options.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 85365d769..9016e3498 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -219,7 +219,7 @@ def parseOpts(overrideArguments=None):
     selection.add_option(
         '--playlist-items',
         dest='playlist_items', metavar='ITEM_SPEC', default=None,
-        help='Playlist video items to download. Specify indices of the videos in the playlist seperated by commas like: "--playlist-items 1,2,5,8" if you want to download videos indexed 1, 2, 5, 8 in the playlist. You can specify range: "--playlist-items 1-3,7,10-13", it will download the videos at index 1, 2, 3, 7, 10, 11, 12 and 13.')
+        help='Playlist video items to download. Specify indices of the videos in the playlist separated by commas like: "--playlist-items 1,2,5,8" if you want to download videos indexed 1, 2, 5, 8 in the playlist. You can specify range: "--playlist-items 1-3,7,10-13", it will download the videos at index 1, 2, 3, 7, 10, 11, 12 and 13.')
     selection.add_option(
         '--match-title',
         dest='matchtitle', metavar='REGEX',

From ce9512b78bbd5f180f44670f6b78784a0be3b815 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 21 Jul 2015 17:20:54 +0200
Subject: [PATCH 2030/6219] release 2015.07.21

---
 docs/supportedsites.md | 6 ++++--
 youtube_dl/version.py  | 2 +-
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index a84878026..73445137f 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -28,6 +28,7 @@
  - **anitube.se**
  - **AnySex**
  - **Aparat**
+ - **AppleConnect**
  - **AppleDaily**: 臺灣蘋果日報
  - **AppleTrailers**
  - **archive.org**: archive.org videos
@@ -486,6 +487,7 @@
  - **SportBox**
  - **SportBoxEmbed**
  - **SportDeutschland**
+ - **Sportschau**
  - **Srf**
  - **SRMediathek**: Saarländischer Rundfunk
  - **SSA**
@@ -611,8 +613,8 @@
  - **Vimple**: Vimple - one-click video hosting
  - **Vine**
  - **vine:user**
- - **vk.com**
- - **vk.com:user-videos**: vk.com:All of a user's videos
+ - **vk**: VK
+ - **vk:uservideos**: VK - User's Videos
  - **Vodlocker**
  - **VoiceRepublic**
  - **Vporn**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 3ad7a2bc0..280afdd7f 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.07.18'
+__version__ = '2015.07.21'

From 496ce6b3493757e806d857363955a2289410a4e5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 21 Jul 2015 23:54:31 +0600
Subject: [PATCH 2031/6219] [snagfilms] Improve m3u8 extraction (Closes #6309)

---
 youtube_dl/extractor/snagfilms.py | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/snagfilms.py b/youtube_dl/extractor/snagfilms.py
index cf495f310..0e820d250 100644
--- a/youtube_dl/extractor/snagfilms.py
+++ b/youtube_dl/extractor/snagfilms.py
@@ -23,6 +23,15 @@ class SnagFilmsEmbedIE(InfoExtractor):
             'ext': 'mp4',
             'title': '#whilewewatch',
         }
+    }, {
+        # invalid labels, 360p is better that 480p
+        'url': 'http://www.snagfilms.com/embed/player?filmId=17ca0950-a74a-11e0-a92a-0026bb61d036',
+        'md5': '882fca19b9eb27ef865efeeaed376a48',
+        'info_dict': {
+            'id': '17ca0950-a74a-11e0-a92a-0026bb61d036',
+            'ext': 'mp4',
+            'title': 'Life in Limbo',
+        }
     }, {
         'url': 'http://www.snagfilms.com/embed/player?filmId=0000014c-de2f-d5d6-abcf-ffef58af0017',
         'only_matching': True,
@@ -52,14 +61,15 @@ class SnagFilmsEmbedIE(InfoExtractor):
             if not file_:
                 continue
             type_ = source.get('type')
-            format_id = source.get('label')
             ext = determine_ext(file_)
-            if any(_ == 'm3u8' for _ in (type_, ext)):
+            format_id = source.get('label') or ext
+            if all(_ == 'm3u8' for _ in (type_, ext)):
                 formats.extend(self._extract_m3u8_formats(
                     file_, video_id, 'mp4', m3u8_id='hls'))
             else:
                 bitrate = int_or_none(self._search_regex(
-                    r'(\d+)kbps', file_, 'bitrate', default=None))
+                    [r'(\d+)kbps', r'_\d{1,2}x\d{1,2}_(\d{3,})\.%s' % ext],
+                    file_, 'bitrate', default=None))
                 height = int_or_none(self._search_regex(
                     r'^(\d+)[pP]$', format_id, 'height', default=None))
                 formats.append({

From 28afa6e77a8fb95541ae9bead0ec0e57958cbfc5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 21 Jul 2015 20:50:02 +0200
Subject: [PATCH 2032/6219] [snagfilms] Don't use '_' as a variable that is
 used

---
 youtube_dl/extractor/snagfilms.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/snagfilms.py b/youtube_dl/extractor/snagfilms.py
index 0e820d250..6977afb27 100644
--- a/youtube_dl/extractor/snagfilms.py
+++ b/youtube_dl/extractor/snagfilms.py
@@ -63,7 +63,7 @@ class SnagFilmsEmbedIE(InfoExtractor):
             type_ = source.get('type')
             ext = determine_ext(file_)
             format_id = source.get('label') or ext
-            if all(_ == 'm3u8' for _ in (type_, ext)):
+            if all(v == 'm3u8' for v in (type_, ext)):
                 formats.extend(self._extract_m3u8_formats(
                     file_, video_id, 'mp4', m3u8_id='hls'))
             else:

From b68a2613f856fbbf943abf8d6de6ffc6aff2e930 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 22 Jul 2015 02:00:21 +0600
Subject: [PATCH 2033/6219] [viewster] Rewrite for new API (Closes #6317)

---
 youtube_dl/extractor/viewster.py | 218 ++++++++++++++++---------------
 1 file changed, 113 insertions(+), 105 deletions(-)

diff --git a/youtube_dl/extractor/viewster.py b/youtube_dl/extractor/viewster.py
index 1742e66f4..03c873608 100644
--- a/youtube_dl/extractor/viewster.py
+++ b/youtube_dl/extractor/viewster.py
@@ -1,129 +1,137 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_request
+from ..compat import (
+    compat_urllib_request,
+    compat_urllib_parse,
+)
+from ..utils import (
+    determine_ext,
+    int_or_none,
+    parse_iso8601,
+)
 
 
 class ViewsterIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?viewster\.com/movie/(?P<id>\d+-\d+-\d+)'
+    _VALID_URL = r'http://(?:www\.)?viewster\.com/(?:serie|movie)/(?P<id>\d+-\d+-\d+)'
     _TESTS = [{
-        # movielink, paymethod=fre
-        'url': 'http://www.viewster.com/movie/1293-19341-000/hout-wood/',
-        'playlist': [{
-            'md5': '8f9d94b282d80c42b378dffdbb11caf3',
-            'info_dict': {
-                'id': '1293-19341-000-movie',
-                'ext': 'flv',
-                'title': "'Hout' (Wood) - Movie",
-            },
-        }],
-        'info_dict': {
-            'id': '1293-19341-000',
-            'title': "'Hout' (Wood)",
-            'description': 'md5:925733185a9242ef96f436937683f33b',
-        }
-    }, {
-        # movielink, paymethod=adv
+        # movie, Type=Movie
         'url': 'http://www.viewster.com/movie/1140-11855-000/the-listening-project/',
-        'playlist': [{
-            'md5': '77a005453ca7396cbe3d35c9bea30aef',
-            'info_dict': {
-                'id': '1140-11855-000-movie',
-                'ext': 'flv',
-                'title': "THE LISTENING PROJECT - Movie",
-            },
-        }],
+        'md5': '14d3cfffe66d57b41ae2d9c873416f01',
         'info_dict': {
             'id': '1140-11855-000',
-            'title': "THE LISTENING PROJECT",
-            'description': 'md5:714421ae9957e112e672551094bf3b08',
-        }
+            'ext': 'flv',
+            'title': 'The listening Project',
+            'description': 'md5:bac720244afd1a8ea279864e67baa071',
+            'timestamp': 1214870400,
+            'upload_date': '20080701',
+            'duration': 4680,
+        },
     }, {
-        # direct links, no movielink
-        'url': 'http://www.viewster.com/movie/1198-56411-000/sinister/',
-        'playlist': [{
-            'md5': '0307b7eac6bfb21ab0577a71f6eebd8f',
-            'info_dict': {
-                'id': '1198-56411-000-trailer',
-                'ext': 'mp4',
-                'title': "Sinister - Trailer",
-            },
-        }, {
-            'md5': '80b9ee3ad69fb368f104cb5d9732ae95',
-            'info_dict': {
-                'id': '1198-56411-000-behind-scenes',
-                'ext': 'mp4',
-                'title': "Sinister - Behind Scenes",
-            },
-        }, {
-            'md5': '3b3ea897ecaa91fca57a8a94ac1b15c5',
-            'info_dict': {
-                'id': '1198-56411-000-scene-from-movie',
-                'ext': 'mp4',
-                'title': "Sinister - Scene from movie",
-            },
-        }],
+        # series episode, Type=Episode
+        'url': 'http://www.viewster.com/serie/1284-19427-001/the-world-and-a-wall/',
+        'md5': 'd5434c80fcfdb61651cc2199a88d6ba3',
         'info_dict': {
-            'id': '1198-56411-000',
-            'title': "Sinister",
-            'description': 'md5:014c40b0488848de9683566a42e33372',
-        }
+            'id': '1284-19427-001',
+            'ext': 'flv',
+            'title': 'The World and a Wall',
+            'description': 'md5:24814cf74d3453fdf5bfef9716d073e3',
+            'timestamp': 1428192000,
+            'upload_date': '20150405',
+            'duration': 1500,
+        },
+    }, {
+        # serie, Type=Serie
+        'url': 'http://www.viewster.com/serie/1303-19426-000/',
+        'info_dict': {
+            'id': '1303-19426-000',
+            'title': 'Is It Wrong to Try to Pick up Girls in a Dungeon?',
+            'description': 'md5:eeda9bef25b0d524b3a29a97804c2f11',
+        },
+        'playlist_count': 13,
+    }, {
+        # unfinished serie, no Type
+        'url': 'http://www.viewster.com/serie/1284-19427-000/baby-steps-season-2/',
+        'info_dict': {
+            'id': '1284-19427-000',
+            'title': 'Baby Steps—Season 2',
+            'description': 'md5:e7097a8fc97151e25f085c9eb7a1cdb1',
+        },
+        'playlist_mincount': 16,
     }]
 
     _ACCEPT_HEADER = 'application/json, text/javascript, */*; q=0.01'
+    _AUTH_TOKEN = '/YqhSYsx8EaU9Bsta3ojlA=='
+
+    def _download_json(self, url, video_id, note='Downloading JSON metadata', fatal=True):
+        request = compat_urllib_request.Request(url)
+        request.add_header('Accept', self._ACCEPT_HEADER)
+        request.add_header('Auth-token', self._AUTH_TOKEN)
+        return super(ViewsterIE, self)._download_json(request, video_id, note, fatal=fatal)
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        request = compat_urllib_request.Request(
-            'http://api.live.viewster.com/api/v1/movie/%s' % video_id)
-        request.add_header('Accept', self._ACCEPT_HEADER)
+        info = self._download_json(
+            'https://public-api.viewster.com/search/%s' % video_id,
+            video_id, 'Downloading entry JSON')
 
-        movie = self._download_json(
-            request, video_id, 'Downloading movie metadata JSON')
+        entry_id = info.get('Id') or info['id']
 
-        title = movie.get('title') or movie['original_title']
-        description = movie.get('synopsis')
-        thumbnail = movie.get('large_artwork') or movie.get('artwork')
+        # unfinished serie has no Type
+        if info.get('Type') in ['Serie', None]:
+            episodes = self._download_json(
+                'https://public-api.viewster.com/series/%s/episodes' % entry_id,
+                video_id, 'Downloading series JSON')
+            entries = [
+                self.url_result(
+                    'http://www.viewster.com/movie/%s' % episode['OriginId'], 'Viewster')
+                for episode in episodes]
+            title = info.get('Title') or info['Synopsis']['Title']
+            description = info.get('Synopsis', {}).get('Detailed')
+            return self.playlist_result(entries, video_id, title, description)
 
-        entries = []
-        for clip in movie['play_list']:
-            entry = None
-
-            # movielink api
-            link_request = clip.get('link_request')
-            if link_request:
-                request = compat_urllib_request.Request(
-                    'http://api.live.viewster.com/api/v1/movielink?movieid=%(movieid)s&action=%(action)s&paymethod=%(paymethod)s&price=%(price)s&currency=%(currency)s&language=%(language)s&subtitlelanguage=%(subtitlelanguage)s&ischromecast=%(ischromecast)s'
-                    % link_request)
-                request.add_header('Accept', self._ACCEPT_HEADER)
-
-                movie_link = self._download_json(
-                    request, video_id, 'Downloading movie link JSON', fatal=False)
-
-                if movie_link:
-                    formats = self._extract_f4m_formats(
-                        movie_link['url'] + '&hdcore=3.2.0&plugin=flowplayer-3.2.0.1', video_id)
-                    self._sort_formats(formats)
-                    entry = {
-                        'formats': formats,
-                    }
-
-            # direct link
-            clip_url = clip.get('clip_data', {}).get('url')
-            if clip_url:
-                entry = {
-                    'url': clip_url,
-                    'ext': 'mp4',
-                }
-
-            if entry:
-                entry.update({
-                    'id': '%s-%s' % (video_id, clip['canonical_title']),
-                    'title': '%s - %s' % (title, clip['title']),
+        formats = []
+        for media_type in ('application/f4m+xml', 'application/x-mpegURL'):
+            media = self._download_json(
+                'https://public-api.viewster.com/movies/%s/video?mediaType=%s'
+                % (entry_id, compat_urllib_parse.quote(media_type)),
+                video_id, 'Downloading %s JSON' % media_type, fatal=False)
+            if not media:
+                continue
+            video_url = media.get('Uri')
+            if not video_url:
+                continue
+            ext = determine_ext(video_url)
+            if ext == 'f4m':
+                video_url += '&' if '?' in video_url else '?'
+                video_url += 'hdcore=3.2.0&plugin=flowplayer-3.2.0.1'
+                formats.extend(self._extract_f4m_formats(
+                    video_url, video_id, f4m_id='hds'))
+            elif ext == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    video_url, video_id, 'mp4', m3u8_id='hls',
+                    fatal=False  # m3u8 sometimes fail
+                ))
+            else:
+                formats.append({
+                    'url': video_url,
                 })
-                entries.append(entry)
+        self._sort_formats(formats)
 
-        playlist = self.playlist_result(entries, video_id, title, description)
-        playlist['thumbnail'] = thumbnail
-        return playlist
+        synopsis = info.get('Synopsis', {})
+        # Prefer title outside synopsis since it's less messy
+        title = info.get('Title') or synopsis['Title'].strip()
+        description = synopsis.get('Detailed') or info.get('Synopsis', {}).get('Short')
+        duration = int_or_none(info.get('Duration'))
+        timestamp = parse_iso8601(info.get('ReleaseDate'))
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'timestamp': timestamp,
+            'duration': duration,
+            'formats': formats,
+        }

From c84683c88bbcb3cfd8d27af54a418035f431371d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 22 Jul 2015 02:08:25 +0600
Subject: [PATCH 2034/6219] [viewster] Strip titles

---
 youtube_dl/extractor/viewster.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/viewster.py b/youtube_dl/extractor/viewster.py
index 03c873608..6ef36290b 100644
--- a/youtube_dl/extractor/viewster.py
+++ b/youtube_dl/extractor/viewster.py
@@ -88,7 +88,7 @@ class ViewsterIE(InfoExtractor):
                 self.url_result(
                     'http://www.viewster.com/movie/%s' % episode['OriginId'], 'Viewster')
                 for episode in episodes]
-            title = info.get('Title') or info['Synopsis']['Title']
+            title = (info.get('Title') or info['Synopsis']['Title']).strip()
             description = info.get('Synopsis', {}).get('Detailed')
             return self.playlist_result(entries, video_id, title, description)
 
@@ -122,7 +122,7 @@ class ViewsterIE(InfoExtractor):
 
         synopsis = info.get('Synopsis', {})
         # Prefer title outside synopsis since it's less messy
-        title = info.get('Title') or synopsis['Title'].strip()
+        title = (info.get('Title') or synopsis['Title']).strip()
         description = synopsis.get('Detailed') or info.get('Synopsis', {}).get('Short')
         duration = int_or_none(info.get('Duration'))
         timestamp = parse_iso8601(info.get('ReleaseDate'))

From f57f84f606b246db4f102fc5bc55e64e4f7a3d60 Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Tue, 21 Jul 2015 16:38:40 -0500
Subject: [PATCH 2035/6219] Twitter: get and describe video from status urls

---
 youtube_dl/extractor/twitter.py | 44 +++++++++++++++++++++++++++++++++
 1 file changed, 44 insertions(+)

diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 1aaa06305..a65252cc6 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -70,3 +70,47 @@ class TwitterCardIE(InfoExtractor):
             'duration': duration,
             'formats': formats,
         }
+
+
+class TwitterIE(TwitterCardIE):
+    _VALID_URL = r'https?://(?:www|m|mobile)?\.?twitter\.com/(?P<id>[^/]+/status/\d+)'
+
+    _TESTS = [{
+        'url': 'https://m.twitter.com/thereaIbanksy/status/614301758345490432',
+        'md5': '8bbccb487bd7a31349b775915fcd412f',
+        'info_dict': {
+            'id': '614301758345490432',
+            'ext': 'mp4',
+            'title': 'thereaIbanksy - This time lapse is so pretty \U0001f60d\U0001f60d',
+            'thumbnail': 're:^https?://.*\.jpg',
+            'duration': 29.5,
+            'description': 'banksy on Twitter: "This time lapse is so pretty \U0001f60d\U0001f60d http://t.co/QB8DDbqiR1"',
+            'uploader': 'banksy',
+            'uploader_id': 'thereaIbanksy',
+        },
+    }]
+
+    def _real_extract(self, url):
+        id = self._match_id(url)
+        username, twid = re.match(r'([^/]+)/status/(\d+)', id).groups()
+        name = username
+        url = re.sub(r'https?://(m|mobile)\.', 'https://', url)
+        webpage = self._download_webpage(url, 'tweet: ' + url)
+        description = unescapeHTML(self._search_regex('<title>\s*(.+?)\s*</title>', webpage, 'title'))
+        title = description.replace('\n', ' ')
+        splitdesc = re.match(r'^(.+?)\s*on Twitter:\s* "(.+?)"$', title)
+        if splitdesc:
+            name, title = splitdesc.groups()
+        title = re.sub(r'\s*https?://[^ ]+', '', title)  # strip  'https -_t.co_BJYgOjSeGA' junk from filenames
+        card_id = self._search_regex(r'["\']/i/cards/tfw/v1/(\d+)', webpage, '/i/card/...')
+        card_url = 'https://twitter.com/i/cards/tfw/v1/' + card_id
+        return {
+            '_type': 'url_transparent',
+            'ie_key': 'TwitterCard',
+            'uploader_id': username,
+            'uploader': name,
+            'url': card_url,
+            'webpage_url': url,
+            'description': description,
+            'title': username + ' - ' + title,
+        }

From c3dea3f878133f3cbdad9e548609d3077572af66 Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Tue, 21 Jul 2015 16:45:36 -0500
Subject: [PATCH 2036/6219] Twittercard: support vmapurl method

---
 youtube_dl/extractor/twitter.py | 47 ++++++++++++++++++++++++++-------
 1 file changed, 37 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index a65252cc6..1dd43ff3c 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -12,17 +12,30 @@ from ..utils import (
 
 class TwitterCardIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?twitter\.com/i/cards/tfw/v1/(?P<id>\d+)'
-    _TEST = {
-        'url': 'https://twitter.com/i/cards/tfw/v1/560070183650213889',
-        'md5': 'a74f50b310c83170319ba16de6955192',
-        'info_dict': {
-            'id': '560070183650213889',
-            'ext': 'mp4',
-            'title': 'TwitterCard',
-            'thumbnail': 're:^https?://.*\.jpg$',
-            'duration': 30.033,
+    _TESTS = [
+        {
+            'url': 'https://twitter.com/i/cards/tfw/v1/560070183650213889',
+            'md5': 'a74f50b310c83170319ba16de6955192',
+            'info_dict': {
+                'id': '560070183650213889',
+                'ext': 'mp4',
+                'title': 'TwitterCard',
+                'thumbnail': 're:^https?://.*\.jpg$',
+                'duration': 30.033,
+            }
         },
-    }
+        {
+            'url': 'https://twitter.com/i/cards/tfw/v1/623160978427936768',
+            'md5': '7ee2a553b63d1bccba97fbed97d9e1c8',
+            'info_dict': {
+                'id': '623160978427936768',
+                'ext': 'mp4',
+                'title': 'TwitterCard',
+                'thumbnail': 're:^https?://.*\.jpg',
+                'duration': 80.155,
+            },
+        }
+    ]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -44,6 +57,20 @@ class TwitterCardIE(InfoExtractor):
                 unescapeHTML(self._search_regex(
                     r'data-player-config="([^"]+)"', webpage, 'data player config')),
                 video_id)
+            if 'playlist' not in config:
+                if 'vmapUrl' in config:
+                    webpage = self._download_webpage(config['vmapUrl'], video_id + ' (xml)')
+                    video_url = self._search_regex(
+                        r'<MediaFile>\s*<!\[CDATA\[(https?://.+?)\]\]>', webpage, 'data player config (xml)')
+                    f = {
+                        'url': video_url,
+                    }
+                    ext = re.search(r'\.([a-z0-9]{2,4})(\?.+)?$', video_url)
+                    if ext:
+                        f['ext'] = ext.group(1)
+                    formats.append(f)
+                    break   # same video regardless of UA
+                continue
 
             video_url = config['playlist'][0]['source']
 

From 9e7e0dffd5e3e3c959e8d99a5e236b9099886fe9 Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Tue, 21 Jul 2015 16:56:35 -0500
Subject: [PATCH 2037/6219] Actually add the extractor

---
 youtube_dl/extractor/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 50da08830..5c03bf8e8 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -651,7 +651,7 @@ from .twitch import (
     TwitchBookmarksIE,
     TwitchStreamIE,
 )
-from .twitter import TwitterCardIE
+from .twitter import TwitterCardIE, TwitterIE
 from .ubu import UbuIE
 from .udemy import (
     UdemyIE,

From 70c857b7283270146864f92d159445e038f40781 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 22 Jul 2015 11:49:54 +0200
Subject: [PATCH 2038/6219] Credit Zach Bruggeman for the appleconnect
 extractor (#6190)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 4fd65f46f..373e05c9f 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -133,3 +133,4 @@ Remita Amine
 Aurélio A. Heckert
 Bernhard Minks
 sceext
+Zach Bruggeman

From 981b9cdc8c12d817eaf3ec6b030538c252efe48e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 22 Jul 2015 22:09:01 +0800
Subject: [PATCH 2039/6219] [lecture2go] Improve some regular expressions

---
 youtube_dl/extractor/lecture2go.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/lecture2go.py b/youtube_dl/extractor/lecture2go.py
index 9cf28e31c..fd115ff54 100644
--- a/youtube_dl/extractor/lecture2go.py
+++ b/youtube_dl/extractor/lecture2go.py
@@ -5,7 +5,7 @@ from .common import InfoExtractor
 
 
 class Lecture2GoIE(InfoExtractor):
-    _VALID_URL = r'https?://lecture2go.uni-hamburg.de/veranstaltungen/-/v/(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://lecture2go\.uni-hamburg\.de/veranstaltungen/-/v/(?P<id>\d+)'
     _TEST = {
         'url': 'https://lecture2go.uni-hamburg.de/veranstaltungen/-/v/17473',
         'md5': 'a9e76f83b3ef58019c4b7dbc35f406c1',
@@ -21,9 +21,9 @@ class Lecture2GoIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        title = self._html_search_regex(r'<em class="title">(.*?)</em>', webpage, 'title')
+        title = self._html_search_regex(r'<em[^>]+class="title">(.+)</em>', webpage, 'title')
         video_url = self._search_regex(r'b.isFirefox..a.useHTML5\).b.setOption.a,"src","(.*.mp4)"\).else', webpage, 'video_url')
-        creator = self._html_search_regex(r'<div id="description">(.*)</div>', webpage, 'creator')
+        creator = self._html_search_regex(r'<div[^>]+id="description">([^<]+)</div>', webpage, 'creator')
 
         return {
             'id': video_id,

From 795704f0f1f963d3f61a7e20074ce41eeb3cdf95 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 22 Jul 2015 22:39:46 +0800
Subject: [PATCH 2040/6219] [lecture2go] Support more formats

---
 youtube_dl/extractor/lecture2go.py | 21 +++++++++++++++++++--
 1 file changed, 19 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/lecture2go.py b/youtube_dl/extractor/lecture2go.py
index fd115ff54..0075b8a2e 100644
--- a/youtube_dl/extractor/lecture2go.py
+++ b/youtube_dl/extractor/lecture2go.py
@@ -1,7 +1,10 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
+from ..utils import determine_ext
 
 
 class Lecture2GoIE(InfoExtractor):
@@ -22,12 +25,26 @@ class Lecture2GoIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         title = self._html_search_regex(r'<em[^>]+class="title">(.+)</em>', webpage, 'title')
-        video_url = self._search_regex(r'b.isFirefox..a.useHTML5\).b.setOption.a,"src","(.*.mp4)"\).else', webpage, 'video_url')
+
+        formats = []
+        for url in set(re.findall(r'"src","([^"]+)"', webpage)):
+            ext = determine_ext(url)
+            if ext == 'f4m':
+                formats.extend(self._extract_f4m_formats(url, video_id))
+            elif ext == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(url, video_id))
+            else:
+                formats.append({
+                    'url': url,
+                })
+
+        self._sort_formats(formats)
+
         creator = self._html_search_regex(r'<div[^>]+id="description">([^<]+)</div>', webpage, 'creator')
 
         return {
             'id': video_id,
             'title': title,
-            'url': video_url,
+            'formats': formats,
             'creator': creator
         }

From 1e124295644e3760cd457bb1a6ae717e1cb2c0fc Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 22 Jul 2015 23:05:14 +0800
Subject: [PATCH 2041/6219] [lecture2go] Update _TEST

---
 youtube_dl/extractor/lecture2go.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/lecture2go.py b/youtube_dl/extractor/lecture2go.py
index 0075b8a2e..d0e9416f5 100644
--- a/youtube_dl/extractor/lecture2go.py
+++ b/youtube_dl/extractor/lecture2go.py
@@ -11,12 +11,12 @@ class Lecture2GoIE(InfoExtractor):
     _VALID_URL = r'https?://lecture2go\.uni-hamburg\.de/veranstaltungen/-/v/(?P<id>\d+)'
     _TEST = {
         'url': 'https://lecture2go.uni-hamburg.de/veranstaltungen/-/v/17473',
-        'md5': 'a9e76f83b3ef58019c4b7dbc35f406c1',
+        'md5': 'ac02b570883020d208d405d5a3fd2f7f',
         'info_dict': {
             'id': '17473',
-            'ext': 'mp4',
-            'url': 'https://fms1.rrz.uni-hamburg.de/abo/64.050_FrankHeitmann_2015-04-13_14-35.mp4',
-            'title': '2 - Endliche Automaten und reguläre Sprachen'
+            'ext': 'flv',
+            'title': '2 - Endliche Automaten und reguläre Sprachen',
+            'creator': 'Frank Heitmann',
         }
     }
 

From 9c29bc69f7d6365835f495dff10f3c5f49671a55 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 22 Jul 2015 23:15:22 +0800
Subject: [PATCH 2042/6219] [utils] Improve parse_duration

Now dots are parsed. For example '87 Min.'
---
 test/test_utils.py  | 1 +
 youtube_dl/utils.py | 4 ++--
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index e13e11b59..65692a9fb 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -324,6 +324,7 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(parse_duration('02:03:04'), 7384)
         self.assertEqual(parse_duration('01:02:03:04'), 93784)
         self.assertEqual(parse_duration('1 hour 3 minutes'), 3780)
+        self.assertEqual(parse_duration('87 Min.'), 5220)
 
     def test_fix_xml_ampersands(self):
         self.assertEqual(
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 942f76d24..ae813099d 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1309,10 +1309,10 @@ def parse_duration(s):
     m = re.match(
         r'''(?ix)(?:P?T)?
         (?:
-            (?P<only_mins>[0-9.]+)\s*(?:mins?|minutes?)\s*|
+            (?P<only_mins>[0-9.]+)\s*(?:mins?\.?|minutes?)\s*|
             (?P<only_hours>[0-9.]+)\s*(?:hours?)|
 
-            \s*(?P<hours_reversed>[0-9]+)\s*(?:[:h]|hours?)\s*(?P<mins_reversed>[0-9]+)\s*(?:[:m]|mins?|minutes?)\s*|
+            \s*(?P<hours_reversed>[0-9]+)\s*(?:[:h]|hours?)\s*(?P<mins_reversed>[0-9]+)\s*(?:[:m]|mins?\.?|minutes?)\s*|
             (?:
                 (?:
                     (?:(?P<days>[0-9]+)\s*(?:[:d]|days?)\s*)?

From e9c6deffee26db40992293b3055df31804ca7e12 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 22 Jul 2015 23:22:19 +0800
Subject: [PATCH 2043/6219] [lecture2go] Add more metadata fields

---
 youtube_dl/extractor/lecture2go.py | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/lecture2go.py b/youtube_dl/extractor/lecture2go.py
index d0e9416f5..a2f9d5c54 100644
--- a/youtube_dl/extractor/lecture2go.py
+++ b/youtube_dl/extractor/lecture2go.py
@@ -4,7 +4,11 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import determine_ext
+from ..utils import (
+    determine_ext,
+    parse_duration,
+    int_or_none,
+)
 
 
 class Lecture2GoIE(InfoExtractor):
@@ -17,6 +21,7 @@ class Lecture2GoIE(InfoExtractor):
             'ext': 'flv',
             'title': '2 - Endliche Automaten und reguläre Sprachen',
             'creator': 'Frank Heitmann',
+            'duration': 5220,
         }
     }
 
@@ -41,10 +46,16 @@ class Lecture2GoIE(InfoExtractor):
         self._sort_formats(formats)
 
         creator = self._html_search_regex(r'<div[^>]+id="description">([^<]+)</div>', webpage, 'creator')
+        duration = parse_duration(self._html_search_regex(
+            r'Duration:\s*</em>\s*<em[^>]*>([^<]+)</em>', webpage, 'duration', fatal=False))
+        view_count = int_or_none(self._html_search_regex(
+            r'Views:\s*</em>\s*<em[^>]+>(\d+)</em>', webpage, 'view count', fatal=False))
 
         return {
             'id': video_id,
             'title': title,
             'formats': formats,
-            'creator': creator
+            'creator': creator,
+            'duration': duration,
+            'view_count': view_count,
         }

From 40101dc311909523852a88ba69df76be9b6bc920 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 22 Jul 2015 23:25:32 +0800
Subject: [PATCH 2044/6219] [lecture2go] Make optional fields non-fatal

---
 youtube_dl/extractor/lecture2go.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/lecture2go.py b/youtube_dl/extractor/lecture2go.py
index a2f9d5c54..40a3d2346 100644
--- a/youtube_dl/extractor/lecture2go.py
+++ b/youtube_dl/extractor/lecture2go.py
@@ -45,7 +45,8 @@ class Lecture2GoIE(InfoExtractor):
 
         self._sort_formats(formats)
 
-        creator = self._html_search_regex(r'<div[^>]+id="description">([^<]+)</div>', webpage, 'creator')
+        creator = self._html_search_regex(
+            r'<div[^>]+id="description">([^<]+)</div>', webpage, 'creator', fatal=False)
         duration = parse_duration(self._html_search_regex(
             r'Duration:\s*</em>\s*<em[^>]*>([^<]+)</em>', webpage, 'duration', fatal=False))
         view_count = int_or_none(self._html_search_regex(

From d609edf4f14e56e3dbb3ffc0be057585c0533666 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 22 Jul 2015 22:49:00 +0600
Subject: [PATCH 2045/6219] [udemy] Handle already-logged-in scenario (Closes
 #6327)

---
 youtube_dl/extractor/udemy.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index e2bab52fe..4a0eaf65f 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -77,7 +77,11 @@ class UdemyIE(InfoExtractor):
         login_popup = self._download_webpage(
             self._LOGIN_URL, None, 'Downloading login popup')
 
-        if login_popup == '<div class="run-command close-popup redirect" data-url="https://www.udemy.com/"></div>':
+        def is_logged(webpage):
+            return any(p in webpage for p in ['href="https://www.udemy.com/user/logout/', '>Logout<'])
+
+        # already logged in
+        if is_logged(login_popup):
             return
 
         login_form = self._form_hidden_inputs('login-form', login_popup)
@@ -95,8 +99,7 @@ class UdemyIE(InfoExtractor):
         response = self._download_webpage(
             request, None, 'Logging in as %s' % username)
 
-        if all(logout_pattern not in response
-               for logout_pattern in ['href="https://www.udemy.com/user/logout/', '>Logout<']):
+        if not is_logged(response):
             error = self._html_search_regex(
                 r'(?s)<div[^>]+class="form-errors[^"]*">(.+?)</div>',
                 response, 'error message', default=None)

From ce1bafdce9f7f9e19c172b319a5ae75d1c85759b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 22 Jul 2015 23:49:08 +0600
Subject: [PATCH 2046/6219] [pbs] Clean up title construction rationale

---
 youtube_dl/extractor/pbs.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index ccbe2a9f3..ba6527b80 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -247,8 +247,8 @@ class PBSIE(InfoExtractor):
                 'url': closed_captions_url,
             }]
 
-        # video.pbs.org video.pbs.org/videoInfo/... frequently provides an obscure 'title' value, like
-        # 'Full Episode', 'Episode 5', etc. prepend program->title
+        # info['title'] is often incomplete (e.g. 'Full Episode', 'Episode 5', etc)
+        # Try turning it to 'program - title' naming scheme if possible
         alt_title = info.get('program', {}).get('title')
         if alt_title:
             info['title'] = alt_title + ' - ' + re.sub(r'^' + alt_title + '[\s\-\:]+', '', info['title'])

From c7620992d20973320fddb85f936f5dc602d1d708 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 22 Jul 2015 23:49:55 +0600
Subject: [PATCH 2047/6219] [pbs] No need to escape colon

---
 youtube_dl/extractor/pbs.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index ba6527b80..a53479aad 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -251,7 +251,7 @@ class PBSIE(InfoExtractor):
         # Try turning it to 'program - title' naming scheme if possible
         alt_title = info.get('program', {}).get('title')
         if alt_title:
-            info['title'] = alt_title + ' - ' + re.sub(r'^' + alt_title + '[\s\-\:]+', '', info['title'])
+            info['title'] = alt_title + ' - ' + re.sub(r'^' + alt_title + '[\s\-:]+', '', info['title'])
 
         return {
             'id': video_id,

From f79ebf09a24f9bf0e91d0dc26782eeea02f10ad1 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 23 Jul 2015 01:56:55 +0800
Subject: [PATCH 2048/6219] Credit @nichdu for Lecture2Go extractor

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 373e05c9f..e75e9885d 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -134,3 +134,4 @@ Aurélio A. Heckert
 Bernhard Minks
 sceext
 Zach Bruggeman
+Tjark Saul

From 948199deac4d9ec6ba6bc4359ad43db7951a1c53 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 23 Jul 2015 00:30:48 +0600
Subject: [PATCH 2049/6219] [tagesschau] Relax _VALID_URL and simplify

---
 youtube_dl/extractor/tagesschau.py | 62 ++++++++----------------------
 1 file changed, 17 insertions(+), 45 deletions(-)

diff --git a/youtube_dl/extractor/tagesschau.py b/youtube_dl/extractor/tagesschau.py
index 636607db5..b84892364 100644
--- a/youtube_dl/extractor/tagesschau.py
+++ b/youtube_dl/extractor/tagesschau.py
@@ -8,7 +8,7 @@ from ..utils import parse_filesize, ExtractorError
 
 
 class TagesschauIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?tagesschau\.de/multimedia/(?:sendung/(ts|tsg|tt|nm|bab/bab)|video/video|tsvorzwanzig)(?P<id>-?[0-9]+)(?:~[-_a-zA-Z0-9]*)?\.html'
+    _VALID_URL = r'https?://(?:www\.)?tagesschau\.de/multimedia/(?:[^/]+/)*?[^/#?]+?(?P<id>-?[0-9]+)(?:~_[^/#?]+?)?\.html'
 
     _TESTS = [{
         'url': 'http://www.tagesschau.de/multimedia/video/video-102143.html',
@@ -29,57 +29,31 @@ class TagesschauIE(InfoExtractor):
             'description': 'md5:695c01bfd98b7e313c501386327aea59',
             'title': 'Sendung: tagesschau \t04.12.2014 20:00 Uhr',
             'thumbnail': 're:^http:.*\.jpg$',
-        }
+        },
     }, {
         'url': 'http://www.tagesschau.de/multimedia/sendung/tsg-3771.html',
-        'md5': '90757268b49ef56deae90c7b48928d58',
-        'info_dict': {
-            'id': '3771',
-            'ext': 'mp4',
-            'description': None,
-            'title': 'Sendung: tagesschau (mit Gebärdensprache) \t14.07.2015 20:00 Uhr',
-            'thumbnail': 're:^http:.*\.jpg$',
-        }
+        'only_matching': True,
     }, {
         'url': 'http://www.tagesschau.de/multimedia/sendung/tt-3827.html',
-        'md5': '6e3ebdc75e8d67da966a8d06721eda71',
-        'info_dict': {
-            'id': '3827',
-            'ext': 'mp4',
-            'description': 'md5:d511d0e278b0ad341a95ad9ab992ce66',
-            'title': 'Sendung: tagesthemen \t14.07.2015 22:15 Uhr',
-            'thumbnail': 're:^http:.*\.jpg$',
-        }
+        'only_matching': True,
     }, {
         'url': 'http://www.tagesschau.de/multimedia/sendung/nm-3475.html',
-        'md5': '8a8875a568f0a5ae5ceef93c501a225f',
-        'info_dict': {
-            'id': '3475',
-            'ext': 'mp4',
-            'description': 'md5:ed149f5649cda3dac86813a9d777e131',
-            'title': 'Sendung: nachtmagazin \t15.07.2015 00:15 Uhr',
-            'thumbnail': 're:^http:.*\.jpg$',
-        }
+        'only_matching': True,
+    }, {
+        'url': 'http://www.tagesschau.de/multimedia/sendung/weltspiegel-3167.html',
+        'only_matching': True,
     }, {
         'url': 'http://www.tagesschau.de/multimedia/tsvorzwanzig-959.html',
-        'md5': 'be4d6f0421f2acd8abe25ea29f6f015b',
-        'info_dict': {
-            'id': '959',
-            'ext': 'mp4',
-            'description': None,
-            'title': 'Sendung: tagesschau vor 20 Jahren \t14.07.2015 22:45 Uhr',
-            'thumbnail': 're:^http:.*\.jpg$',
-        }
+        'only_matching': True,
     }, {
         'url': 'http://www.tagesschau.de/multimedia/sendung/bab/bab-3299~_bab-sendung-209.html',
-        'md5': '42e3757018d9908581481a80cc1806da',
-        'info_dict': {
-            'id': '3299',
-            'ext': 'mp4',
-            'description': None,
-            'title': 'Nach dem Referendum: Schaltgespräch nach Athen',
-            'thumbnail': 're:^http:.*\.jpg$',
-        }
+        'only_matching': True,
+    }, {
+        'url': 'http://www.tagesschau.de/multimedia/video/video-102303~_bab-sendung-211.html',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.tagesschau.de/multimedia/politikimradio/audio-18407.html',
+        'only_matching': True,
     }]
 
     _FORMATS = {
@@ -152,11 +126,9 @@ class TagesschauIE(InfoExtractor):
             thumbnail_fn = self._search_regex(
                 r'(?s)<img alt="Sendungsbild".*?src="([^"]+)"',
                 webpage, 'thumbnail', fatal=False)
-            # there are some videos without description
-            description = ""
             description = self._html_search_regex(
                 r'(?s)<p class="teasertext">(.*?)</p>',
-                webpage, 'description', fatal=False, default=None)
+                webpage, 'description', default=None)
             title = self._html_search_regex(
                 r'<span class="headline".*?>(.*?)</span>', webpage, 'title')
 

From a47b602b0877dcde1b795bf53bfe3629c6595870 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 23 Jul 2015 01:01:04 +0600
Subject: [PATCH 2050/6219] [tagesschau] Add support for audio

---
 youtube_dl/extractor/tagesschau.py | 37 +++++++++++++++++++-----------
 1 file changed, 24 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/tagesschau.py b/youtube_dl/extractor/tagesschau.py
index b84892364..7fd0ba987 100644
--- a/youtube_dl/extractor/tagesschau.py
+++ b/youtube_dl/extractor/tagesschau.py
@@ -30,6 +30,16 @@ class TagesschauIE(InfoExtractor):
             'title': 'Sendung: tagesschau \t04.12.2014 20:00 Uhr',
             'thumbnail': 're:^http:.*\.jpg$',
         },
+    }, {
+        'url': 'http://www.tagesschau.de/multimedia/politikimradio/audio-18407.html',
+        'md5': 'aef45de271c4bf0a5db834aa40bf774c',
+        'info_dict': {
+            'id': '18407',
+            'ext': 'mp3',
+            'title': 'Flüchtlingsdebatte: Hitzig, aber wenig hilfreich',
+            'description': 'Flüchtlingsdebatte: Hitzig, aber wenig hilfreich',
+            'thumbnail': 're:^https?:.*\.jpg$',
+        },
     }, {
         'url': 'http://www.tagesschau.de/multimedia/sendung/tsg-3771.html',
         'only_matching': True,
@@ -51,9 +61,6 @@ class TagesschauIE(InfoExtractor):
     }, {
         'url': 'http://www.tagesschau.de/multimedia/video/video-102303~_bab-sendung-211.html',
         'only_matching': True,
-    }, {
-        'url': 'http://www.tagesschau.de/multimedia/politikimradio/audio-18407.html',
-        'only_matching': True,
     }]
 
     _FORMATS = {
@@ -73,19 +80,26 @@ class TagesschauIE(InfoExtractor):
             playerpage = self._download_webpage(
                 player_url, display_id, 'Downloading player page')
 
-            medias = re.findall(
-                r'"(http://media.+?)", type:"video/(.+?)", quality:"(.+?)"',
-                playerpage)
             formats = []
-            for url, ext, res in medias:
+            for media in re.finditer(
+                    r'''(?x)
+                        (?P<q_url>["\'])(?P<url>http://media.+?)(?P=q_url)
+                        ,\s*type:(?P<q_type>["\'])(?P<type>video|audio)/(?P<ext>.+?)(?P=q_type)
+                        (?:,\s*quality:(?P<q_quality>["\'])(?P<quality>.+?)(?P=q_quality))?
+                    ''', playerpage):
+                url = media.group('url')
+                type_ = media.group('type')
+                ext = media.group('ext')
+                res = media.group('quality')
                 f = {
-                    'format_id': res + '_' + ext,
+                    'format_id': '%s_%s' % (res, ext) if res else ext,
                     'url': url,
                     'ext': ext,
+                    'vcodec': 'none' if type_ == 'audio' else None,
                 }
                 f.update(self._FORMATS.get(res, {}))
                 formats.append(f)
-            thumbnail_fn = re.findall(r'"(/multimedia/.+?\.jpg)"', playerpage)[-1]
+            thumbnail = self._og_search_thumbnail(playerpage)
             title = self._og_search_title(webpage).strip()
             description = self._og_search_description(webpage).strip()
         else:
@@ -123,9 +137,7 @@ class TagesschauIE(InfoExtractor):
                         'filesize_approx': parse_filesize(m.group('filesize_approx')),
                     })
                 formats.append(format)
-            thumbnail_fn = self._search_regex(
-                r'(?s)<img alt="Sendungsbild".*?src="([^"]+)"',
-                webpage, 'thumbnail', fatal=False)
+            thumbnail = self._og_search_thumbnail(webpage)
             description = self._html_search_regex(
                 r'(?s)<p class="teasertext">(.*?)</p>',
                 webpage, 'description', default=None)
@@ -133,7 +145,6 @@ class TagesschauIE(InfoExtractor):
                 r'<span class="headline".*?>(.*?)</span>', webpage, 'title')
 
         self._sort_formats(formats)
-        thumbnail = 'http://www.tagesschau.de' + thumbnail_fn
 
         return {
             'id': display_id,

From 3e214851a49992b010ae90ef6dbaed11f70ceb3a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 23 Jul 2015 01:01:35 +0600
Subject: [PATCH 2051/6219] [tagesschau] Improve tests

---
 youtube_dl/extractor/tagesschau.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/tagesschau.py b/youtube_dl/extractor/tagesschau.py
index 7fd0ba987..1a7ea98d2 100644
--- a/youtube_dl/extractor/tagesschau.py
+++ b/youtube_dl/extractor/tagesschau.py
@@ -18,7 +18,7 @@ class TagesschauIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Regierungsumbildung in Athen: Neue Minister in Griechenland vereidigt',
             'description': 'md5:171feccd9d9b3dd54d05d501568f6359',
-            'thumbnail': 're:^http:.*\.jpg$',
+            'thumbnail': 're:^https?:.*\.jpg$',
         },
     }, {
         'url': 'http://www.tagesschau.de/multimedia/sendung/ts-5727.html',
@@ -28,7 +28,7 @@ class TagesschauIE(InfoExtractor):
             'ext': 'mp4',
             'description': 'md5:695c01bfd98b7e313c501386327aea59',
             'title': 'Sendung: tagesschau \t04.12.2014 20:00 Uhr',
-            'thumbnail': 're:^http:.*\.jpg$',
+            'thumbnail': 're:^https?:.*\.jpg$',
         },
     }, {
         'url': 'http://www.tagesschau.de/multimedia/politikimradio/audio-18407.html',

From c59b61c0da8fc0fb73d745de59122e290a33b122 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 23 Jul 2015 02:13:51 +0600
Subject: [PATCH 2052/6219] [viki] Fix height (Closes #6333)

---
 youtube_dl/extractor/viki.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index 51cdc6b65..3a7e9a0f2 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -242,8 +242,8 @@ class VikiIE(VikiBaseIE):
 
         formats = []
         for format_id, stream_dict in streams.items():
-            height = self._search_regex(
-                r'^(\d+)[pP]$', format_id, 'height', default=None)
+            height = int_or_none(self._search_regex(
+                r'^(\d+)[pP]$', format_id, 'height', default=None))
             for protocol, format_dict in stream_dict.items():
                 if format_id == 'm3u8':
                     formats = self._extract_m3u8_formats(

From 61be92e26a16e24a702c9c65b46f15c375726345 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 23 Jul 2015 03:04:23 +0600
Subject: [PATCH 2053/6219] [prosiebensat1] Recognize DRM protected videos
 (#6334)

---
 youtube_dl/extractor/prosiebensat1.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/prosiebensat1.py b/youtube_dl/extractor/prosiebensat1.py
index fec008ce7..0739234c6 100644
--- a/youtube_dl/extractor/prosiebensat1.py
+++ b/youtube_dl/extractor/prosiebensat1.py
@@ -9,6 +9,7 @@ from ..compat import (
     compat_urllib_parse,
 )
 from ..utils import (
+    ExtractorError,
     determine_ext,
     int_or_none,
     unified_strdate,
@@ -224,10 +225,13 @@ class ProSiebenSat1IE(InfoExtractor):
             'ids': clip_id,
         })
 
-        videos = self._download_json(videos_api_url, clip_id, 'Downloading videos JSON')
+        video = self._download_json(videos_api_url, clip_id, 'Downloading videos JSON')[0]
 
-        duration = float(videos[0]['duration'])
-        source_ids = [source['id'] for source in videos[0]['sources']]
+        if video.get('is_protected') is True:
+            raise ExtractorError('This video is DRM protected.', expected=True)
+
+        duration = float(video['duration'])
+        source_ids = [source['id'] for source in video['sources']]
         source_ids_str = ','.join(map(str, source_ids))
 
         g = '01!8d8F_)r9]4s[qeuXfP%'

From 993df6bc228ca540bba45e0f5b6041ac9440d695 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 23 Jul 2015 03:05:16 +0600
Subject: [PATCH 2054/6219] [prosiebensat1] Modernize

---
 youtube_dl/extractor/prosiebensat1.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/prosiebensat1.py b/youtube_dl/extractor/prosiebensat1.py
index 0739234c6..effcf1db3 100644
--- a/youtube_dl/extractor/prosiebensat1.py
+++ b/youtube_dl/extractor/prosiebensat1.py
@@ -11,6 +11,7 @@ from ..compat import (
 from ..utils import (
     ExtractorError,
     determine_ext,
+    float_or_none,
     int_or_none,
     unified_strdate,
 )
@@ -230,7 +231,7 @@ class ProSiebenSat1IE(InfoExtractor):
         if video.get('is_protected') is True:
             raise ExtractorError('This video is DRM protected.', expected=True)
 
-        duration = float(video['duration'])
+        duration = float_or_none(video.get('duration'))
         source_ids = [source['id'] for source in video['sources']]
         source_ids_str = ','.join(map(str, source_ids))
 

From ca4456eda8f49ed2f939c7ae5cfe55b45777217c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 22 Jul 2015 23:23:38 +0200
Subject: [PATCH 2055/6219] [tagesschau] Remove unused import

---
 youtube_dl/extractor/tagesschau.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tagesschau.py b/youtube_dl/extractor/tagesschau.py
index 1a7ea98d2..73e7657d4 100644
--- a/youtube_dl/extractor/tagesschau.py
+++ b/youtube_dl/extractor/tagesschau.py
@@ -4,7 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import parse_filesize, ExtractorError
+from ..utils import parse_filesize
 
 
 class TagesschauIE(InfoExtractor):

From 87dc451108b278eb5e5600a0159bdbd337272392 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 20 Jul 2015 20:35:26 +0200
Subject: [PATCH 2056/6219] [youtube] Don't use the DASH manifest from
 'get_video_info' if 'use_cipher_signature' is True (#5118)

Currently they give a 403 Forbidden error.
---
 youtube_dl/extractor/youtube.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 3d8b31f98..323681960 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -967,7 +967,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                         video_id, note=False,
                         errnote='unable to download video info webpage')
                     get_video_info = compat_parse_qs(video_info_webpage)
-                    add_dash_mpd(get_video_info)
+                    if get_video_info.get('use_cipher_signature') != ['True']:
+                        add_dash_mpd(get_video_info)
                     if not video_info:
                         video_info = get_video_info
                     if 'token' in get_video_info:

From b37317d8b0391651b396fc2dc4e35b59e52fddf3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 22 Jul 2015 23:33:49 +0200
Subject: [PATCH 2057/6219] [generic] Unescape HTML escape sequences in
 redirect urls (fixes #6311)

---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index cd133a10c..6d2efb22e 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1702,7 +1702,7 @@ class GenericIE(InfoExtractor):
                 if refresh_header:
                     found = re.search(REDIRECT_REGEX, refresh_header)
             if found:
-                new_url = compat_urlparse.urljoin(url, found.group(1))
+                new_url = compat_urlparse.urljoin(url, unescapeHTML(found.group(1)))
                 self.report_following_redirect(new_url)
                 return {
                     '_type': 'url',

From 41597d9bed9eaa5e55d5bb572f2ec3f5a312d392 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 23 Jul 2015 13:39:19 +0800
Subject: [PATCH 2058/6219] [viki] Fix description extraction (closes #6339)

---
 youtube_dl/extractor/viki.py | 21 +++++++++++++++++++--
 1 file changed, 19 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index 3a7e9a0f2..e987badbd 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -173,6 +173,19 @@ class VikiIE(VikiBaseIE):
     }, {
         'url': 'http://www.viki.com/player/44699v',
         'only_matching': True,
+    }, {
+        # non-English description
+        'url': 'http://www.viki.com/videos/158036v-love-in-magic',
+        'md5': '1713ae35df5a521b31f6dc40730e7c9c',
+        'info_dict': {
+            'id': '158036v',
+            'ext': 'mp4',
+            'uploader': 'I Planet Entertainment',
+            'upload_date': '20111122',
+            'timestamp': 1321985454,
+            'description': 'md5:44b1e46619df3a072294645c770cef36',
+            'title': 'Love In Magic',
+        },
     }]
 
     def _real_extract(self, url):
@@ -192,8 +205,12 @@ class VikiIE(VikiBaseIE):
                 container_title = container_titles.get('en') or container_titles[container_titles.keys()[0]]
                 title = '%s - %s' % (container_title, title)
 
-        descriptions = video.get('descriptions')
-        description = descriptions.get('en') or descriptions[titles.keys()[0]] if descriptions else None
+        descriptions = video.get('descriptions', {})
+        description = descriptions.get('en')
+        if description is None:
+            filtered_descriptions = list(filter(None, [descriptions.get(k) for k in titles.keys()]))
+            if filtered_descriptions:
+                description = filtered_descriptions[0]
 
         duration = int_or_none(video.get('duration'))
         timestamp = parse_iso8601(video.get('created_at'))

From b73b14f72c8240466b5bded9fb891697549b89ec Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 23 Jul 2015 14:02:19 +0800
Subject: [PATCH 2059/6219] [viki] Rewrite dict selection codes

---
 youtube_dl/extractor/viki.py | 33 +++++++++++++++------------------
 1 file changed, 15 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index e987badbd..ddbd395c8 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -88,6 +88,14 @@ class VikiBaseIE(InfoExtractor):
         if not self._token:
             self.report_warning('Unable to get session token, login has probably failed')
 
+    @staticmethod
+    def dict_selection(dict_obj, preferred_key):
+        if preferred_key in dict_obj:
+            return dict_obj.get(preferred_key)
+
+        filtered_dict = list(filter(None, [dict_obj.get(k) for k in dict_obj.keys()]))
+        return filtered_dict[0] if filtered_dict else None
+
 
 class VikiIE(VikiBaseIE):
     IE_NAME = 'viki'
@@ -194,23 +202,14 @@ class VikiIE(VikiBaseIE):
         video = self._call_api(
             'videos/%s.json' % video_id, video_id, 'Downloading video JSON')
 
-        title = None
-        titles = video.get('titles')
-        if titles:
-            title = titles.get('en') or titles[titles.keys()[0]]
+        title = self.dict_selection(video.get('titles', {}), 'en')
         if not title:
             title = 'Episode %d' % video.get('number') if video.get('type') == 'episode' else video.get('id') or video_id
-            container_titles = video.get('container', {}).get('titles')
-            if container_titles:
-                container_title = container_titles.get('en') or container_titles[container_titles.keys()[0]]
-                title = '%s - %s' % (container_title, title)
+            container_titles = video.get('container', {}).get('titles', {})
+            container_title = self.dict_selection(container_titles, 'en')
+            title = '%s - %s' % (container_title, title)
 
-        descriptions = video.get('descriptions', {})
-        description = descriptions.get('en')
-        if description is None:
-            filtered_descriptions = list(filter(None, [descriptions.get(k) for k in titles.keys()]))
-            if filtered_descriptions:
-                description = filtered_descriptions[0]
+        description = self.dict_selection(video.get('descriptions', {}), 'en')
 
         duration = int_or_none(video.get('duration'))
         timestamp = parse_iso8601(video.get('created_at'))
@@ -316,11 +315,9 @@ class VikiChannelIE(VikiBaseIE):
             'containers/%s.json' % channel_id, channel_id,
             'Downloading channel JSON')
 
-        titles = channel['titles']
-        title = titles.get('en') or titles[titles.keys()[0]]
+        title = self.dict_selection(channel['titles'], 'en')
 
-        descriptions = channel['descriptions']
-        description = descriptions.get('en') or descriptions[descriptions.keys()[0]]
+        description = self.dict_selection(channel['descriptions'], 'en')
 
         entries = []
         for video_type in ('episodes', 'clips', 'movies'):

From 59db9f80187c4dc3768f8656ede281347a805e1d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 23 Jul 2015 12:09:30 +0200
Subject: [PATCH 2060/6219] [downloader/dash] Improve 'combine_url' (fixes
 #6341)

In some videos the base_url already ends with '/'.
---
 youtube_dl/downloader/dash.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/dash.py b/youtube_dl/downloader/dash.py
index a4685d307..8b6fa2753 100644
--- a/youtube_dl/downloader/dash.py
+++ b/youtube_dl/downloader/dash.py
@@ -37,7 +37,7 @@ class DashSegmentsFD(FileDownloader):
         def combine_url(base_url, target_url):
             if re.match(r'^https?://', target_url):
                 return target_url
-            return '%s/%s' % (base_url, target_url)
+            return '%s%s%s' % (base_url, '' if base_url.endswith('/') else '/', target_url)
 
         with open(tmpfilename, 'wb') as outf:
             append_url_to_file(

From 53b8247cb5b5ac4a7822c82b94ec2f1221a40625 Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Thu, 23 Jul 2015 01:38:55 -0500
Subject: [PATCH 2061/6219] NationalGeographic._VALID_URL: work site-wide

Closes #6343.
---
 youtube_dl/extractor/nationalgeographic.py | 35 +++++++++++++++-------
 1 file changed, 24 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/nationalgeographic.py b/youtube_dl/extractor/nationalgeographic.py
index f793b72f5..6fc9e7b05 100644
--- a/youtube_dl/extractor/nationalgeographic.py
+++ b/youtube_dl/extractor/nationalgeographic.py
@@ -8,18 +8,30 @@ from ..utils import (
 
 
 class NationalGeographicIE(InfoExtractor):
-    _VALID_URL = r'http://video\.nationalgeographic\.com/video/.*?'
+    _VALID_URL = r'http://video\.nationalgeographic\.com/.*?'
 
-    _TEST = {
-        'url': 'http://video.nationalgeographic.com/video/news/150210-news-crab-mating-vin?source=featuredvideo',
-        'info_dict': {
-            'id': '4DmDACA6Qtk_',
-            'ext': 'flv',
-            'title': 'Mating Crabs Busted by Sharks',
-            'description': 'md5:16f25aeffdeba55aaa8ec37e093ad8b3',
+    _TESTS = [
+        {
+            'url': 'http://video.nationalgeographic.com/video/news/150210-news-crab-mating-vin?source=featuredvideo',
+            'info_dict': {
+                'id': '4DmDACA6Qtk_',
+                'ext': 'flv',
+                'title': 'Mating Crabs Busted by Sharks',
+                'description': 'md5:16f25aeffdeba55aaa8ec37e093ad8b3',
+            },
+            'add_ie': ['ThePlatform'],
         },
-        'add_ie': ['ThePlatform'],
-    }
+        {
+            'url': 'http://video.nationalgeographic.com/wild/when-sharks-attack/the-real-jaws',
+            'info_dict': {
+                'id': '_JeBD_D7PlS5',
+                'ext': 'flv',
+                'title': 'The Real Jaws',
+                'description': 'md5:8d3e09d9d53a85cd397b4b21b2c77be6',
+            },
+            'add_ie': ['ThePlatform'],
+        },
+    ]
 
     def _real_extract(self, url):
         name = url_basename(url)
@@ -37,5 +49,6 @@ class NationalGeographicIE(InfoExtractor):
 
         return self.url_result(smuggle_url(
             'http://link.theplatform.com/s/ngs/%s?format=SMIL&formats=MPEG4&manifest=f4m' % theplatform_id,
-            # For some reason, the normal links don't work and we must force the use of f4m
+            # For some reason, the normal links don't work and we must force
+            # the use of f4m
             {'force_smil_url': True}))

From 984e4d487520bd2a860b31b3165416c879b28096 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 24 Jun 2015 01:13:23 +0100
Subject: [PATCH 2062/6219] [googledrive] Add new extractor

---
 youtube_dl/extractor/__init__.py    |   1 +
 youtube_dl/extractor/googledrive.py | 106 ++++++++++++++++++++++++++++
 2 files changed, 107 insertions(+)
 create mode 100644 youtube_dl/extractor/googledrive.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 3cfa804ec..6655d7eb5 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -209,6 +209,7 @@ from .globo import GloboIE
 from .godtube import GodTubeIE
 from .goldenmoustache import GoldenMoustacheIE
 from .golem import GolemIE
+from .googledrive import GoogleDriveIE
 from .googleplus import GooglePlusIE
 from .googlesearch import GoogleSearchIE
 from .gorillavid import GorillaVidIE
diff --git a/youtube_dl/extractor/googledrive.py b/youtube_dl/extractor/googledrive.py
new file mode 100644
index 000000000..8c611fa47
--- /dev/null
+++ b/youtube_dl/extractor/googledrive.py
@@ -0,0 +1,106 @@
+from .common import InfoExtractor
+from ..utils import RegexNotFoundError
+
+class GoogleDriveIE(InfoExtractor):
+    _VALID_URL = r'(?:https?://)?(?:video\.google\.com/get_player\?.*?docid=|(?:docs|drive)\.google\.com/(?:uc\?.*?id=|file/d/))(?P<id>.+?)(?:&|/|$)'
+    _TEST = {
+        'url': 'https://drive.google.com/file/d/0BzpExh0WzJF0NlR5WUlxdEVsY0U/edit?pli=1',
+        'info_dict': {
+            'id': '0BzpExh0WzJF0NlR5WUlxdEVsY0U',
+            'ext': 'mp4',
+            'title': '[AHSH] Fairy Tail S2 - 01 [720p].mp4',
+        }
+    }
+    _formats = {
+        '5': {'ext': 'flv'},
+        '6': {'ext': 'flv'},
+        '13': {'ext': '3gp'},
+        '17': {'ext': '3gp'},
+        '18': {'ext': 'mp4'},
+        '22': {'ext': 'mp4'},
+        '34': {'ext': 'flv'},
+        '35': {'ext': 'flv'},
+        '36': {'ext': '3gp'},
+        '37': {'ext': 'mp4'},
+        '38': {'ext': 'mp4'},
+        '43': {'ext': 'webm'},
+        '44': {'ext': 'webm'},
+        '45': {'ext': 'webm'},
+        '46': {'ext': 'webm'},
+        '59': {'ext': 'mp4'}
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(
+            'http://docs.google.com/file/d/'+video_id, video_id, encoding='unicode_escape'
+        )
+        try:
+            title = self._html_search_regex(
+                r'"title","(?P<title>.*?)"',
+                webpage,
+                'title',
+                group='title'
+            )
+            fmt_stream_map = self._html_search_regex(
+                r'"fmt_stream_map","(?P<fmt_stream_map>.*?)"',
+                webpage,
+                'fmt_stream_map',
+                group='fmt_stream_map'
+            )
+            fmt_list = self._html_search_regex(
+                r'"fmt_list","(?P<fmt_list>.*?)"',
+                webpage,
+                'fmt_list',
+                group='fmt_list'
+            )
+#			timestamp = self._html_search_regex(
+#				r'"timestamp","(?P<timestamp>.*?)"',
+#				webpage,
+#				'timestamp',
+#				group='timestamp'
+#			)
+            length_seconds = self._html_search_regex(
+                r'"length_seconds","(?P<length_seconds>.*?)"',
+                webpage,
+                'length_seconds',
+                group='length_seconds'
+            )
+        except RegexNotFoundError:
+            try:
+                reason = self._html_search_regex(
+                    r'"reason","(?P<reason>.*?)"',
+                    webpage,
+                    'reason',
+                    group='reason'
+                )
+                self.report_warning(reason)
+                return
+            except RegexNotFoundError:
+                self.report_warning('not a video')
+                return
+
+        fmt_stream_map = fmt_stream_map.split(',')
+        fmt_list = fmt_list.split(',')
+        formats = []
+        for i in range(len(fmt_stream_map)):
+            fmt_id, fmt_url = fmt_stream_map[i].split('|')
+            resolution = fmt_list[i].split('/')[1]
+            width, height = resolution.split('x')
+            formats.append({
+                'url': fmt_url,
+                'format_id': fmt_id,
+                'resolution': resolution,
+                'width': int(width),
+                'height': int(height),
+                'ext': self._formats[fmt_id]['ext']
+            })
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': title,
+#           'timestamp': int(timestamp),
+            'duration': int(length_seconds),
+            'formats': formats
+        }

From f120a7ab5e9c560a8114f9662e2f213243a945b0 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 24 Jun 2015 14:56:19 +0100
Subject: [PATCH 2063/6219] change the _TEST info

---
 youtube_dl/extractor/googledrive.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/googledrive.py b/youtube_dl/extractor/googledrive.py
index 8c611fa47..e3d5c3418 100644
--- a/youtube_dl/extractor/googledrive.py
+++ b/youtube_dl/extractor/googledrive.py
@@ -4,11 +4,11 @@ from ..utils import RegexNotFoundError
 class GoogleDriveIE(InfoExtractor):
     _VALID_URL = r'(?:https?://)?(?:video\.google\.com/get_player\?.*?docid=|(?:docs|drive)\.google\.com/(?:uc\?.*?id=|file/d/))(?P<id>.+?)(?:&|/|$)'
     _TEST = {
-        'url': 'https://drive.google.com/file/d/0BzpExh0WzJF0NlR5WUlxdEVsY0U/edit?pli=1',
+        'url': 'https://drive.google.com/file/d/0ByeS4oOUV-49Zzh4R1J6R09zazQ/edit?pli=1',
         'info_dict': {
-            'id': '0BzpExh0WzJF0NlR5WUlxdEVsY0U',
+            'id': '0ByeS4oOUV-49Zzh4R1J6R09zazQ',
             'ext': 'mp4',
-            'title': '[AHSH] Fairy Tail S2 - 01 [720p].mp4',
+            'title': 'Big Buck Bunny.mp4',
         }
     }
     _formats = {

From 3e5f3df1729846a33631dd38a887cd1d81a727c1 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 29 Jun 2015 07:53:21 +0100
Subject: [PATCH 2064/6219] move the embed to a separate class

---
 youtube_dl/extractor/googledrive.py | 31 ++++++++++++++++++++++++++++-
 1 file changed, 30 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/googledrive.py b/youtube_dl/extractor/googledrive.py
index e3d5c3418..ac891b275 100644
--- a/youtube_dl/extractor/googledrive.py
+++ b/youtube_dl/extractor/googledrive.py
@@ -1,8 +1,37 @@
+import re
+
 from .common import InfoExtractor
 from ..utils import RegexNotFoundError
 
+class GoogleDriveEmbedIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:video\.google\.com/get_player\?.*?docid=|(?:docs|drive)\.google\.com/file/d/)(?P<id>[a-zA-Z0-9-]{28})(?:/preview)'
+    _TEST = {
+        'url': 'https://docs.google.com/file/d/0B8KB9DRosYGKMXNoeWxqa3JYclE/preview',
+        'info_dict': {
+            'id': '0B8KB9DRosYGKMXNoeWxqa3JYclE',
+            'ext': 'mp4',
+            'title': 'Jimmy Fallon Sings Since You\'ve Been Gone.wmv',
+        }
+    }
+
+    @staticmethod
+    def _extract_url(webpage):
+        mobj = re.search(
+            r'<iframe src="https?://(?:video\.google\.com/get_player\?.*?docid=|(?:docs|drive)\.google\.com/file/d/)(?P<id>[a-zA-Z0-9-]{28})(?:/preview)',
+            webpage)
+        if mobj:
+            return 'https://drive.google.com/file/d/%s' % mobj.group('id')
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        return {
+            '_type': 'url',
+            'ie-key': 'GoogleDrive',
+            'url': 'https://drive.google.com/file/d/%s' % video_id
+        }
+
 class GoogleDriveIE(InfoExtractor):
-    _VALID_URL = r'(?:https?://)?(?:video\.google\.com/get_player\?.*?docid=|(?:docs|drive)\.google\.com/(?:uc\?.*?id=|file/d/))(?P<id>.+?)(?:&|/|$)'
+    _VALID_URL = r'https?://(?:docs|drive)\.google\.com/(?:uc\?.*?id=|file/d/)(?P<id>[a-zA-Z0-9-]{28})'
     _TEST = {
         'url': 'https://drive.google.com/file/d/0ByeS4oOUV-49Zzh4R1J6R09zazQ/edit?pli=1',
         'info_dict': {

From 2d651a2d02885cddf1752b45497e9113d3a3d403 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 29 Jun 2015 07:55:44 +0100
Subject: [PATCH 2065/6219] import google drive embed class

---
 youtube_dl/extractor/__init__.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 6655d7eb5..02e18a0da 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -209,7 +209,10 @@ from .globo import GloboIE
 from .godtube import GodTubeIE
 from .goldenmoustache import GoldenMoustacheIE
 from .golem import GolemIE
-from .googledrive import GoogleDriveIE
+from .googledrive import (
+    GoogleDriveEmbedIE,
+    GoogleDriveIE,
+)
 from .googleplus import GooglePlusIE
 from .googlesearch import GoogleSearchIE
 from .gorillavid import GorillaVidIE

From 653789afc72d1a225b971541fb633dd768d58942 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 29 Jun 2015 08:01:30 +0100
Subject: [PATCH 2066/6219] add google drive embeds

---
 youtube_dl/extractor/generic.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 6d2efb22e..3f7b094db 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -48,6 +48,7 @@ from .vimeo import VimeoIE
 from .dailymotion import DailymotionCloudIE
 from .onionstudios import OnionStudiosIE
 from .snagfilms import SnagFilmsEmbedIE
+from .googledrive import GoogleDriveEmbedIE
 
 
 class GenericIE(InfoExtractor):
@@ -1599,6 +1600,11 @@ class GenericIE(InfoExtractor):
         if nbc_sports_url:
             return self.url_result(nbc_sports_url, 'NBCSportsVPlayer')
 
+        # Look for Google Drive embeds
+        google_drive_url = GoogleDriveEmbedIE._extract_url(webpage)
+        if google_drive_url:
+            return self.url_result(google_drive_url, 'GoogleDrive')
+
         # Look for UDN embeds
         mobj = re.search(
             r'<iframe[^>]+src="(?P<url>%s)"' % UDNEmbedIE._VALID_URL, webpage)

From 3b3d531965f0f36c20f5fa8557481c144170653f Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 17 Jul 2015 14:17:19 +0100
Subject: [PATCH 2067/6219] fix embed regex

---
 youtube_dl/extractor/googledrive.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/googledrive.py b/youtube_dl/extractor/googledrive.py
index ac891b275..c82c9037f 100644
--- a/youtube_dl/extractor/googledrive.py
+++ b/youtube_dl/extractor/googledrive.py
@@ -4,7 +4,7 @@ from .common import InfoExtractor
 from ..utils import RegexNotFoundError
 
 class GoogleDriveEmbedIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:video\.google\.com/get_player\?.*?docid=|(?:docs|drive)\.google\.com/file/d/)(?P<id>[a-zA-Z0-9-]{28})(?:/preview)'
+    _VALID_URL = r'https?://(?:video\.google\.com/get_player\?.*?docid=|(?:docs|drive)\.google\.com/file/d/)(?P<id>[a-zA-Z0-9_-]{28})'
     _TEST = {
         'url': 'https://docs.google.com/file/d/0B8KB9DRosYGKMXNoeWxqa3JYclE/preview',
         'info_dict': {
@@ -17,7 +17,7 @@ class GoogleDriveEmbedIE(InfoExtractor):
     @staticmethod
     def _extract_url(webpage):
         mobj = re.search(
-            r'<iframe src="https?://(?:video\.google\.com/get_player\?.*?docid=|(?:docs|drive)\.google\.com/file/d/)(?P<id>[a-zA-Z0-9-]{28})(?:/preview)',
+            r'<iframe src="https?://(?:video\.google\.com/get_player\?.*?docid=|(?:docs|drive)\.google\.com/file/d/)(?P<id>[a-zA-Z0-9_-]{28})',
             webpage)
         if mobj:
             return 'https://drive.google.com/file/d/%s' % mobj.group('id')
@@ -31,7 +31,7 @@ class GoogleDriveEmbedIE(InfoExtractor):
         }
 
 class GoogleDriveIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:docs|drive)\.google\.com/(?:uc\?.*?id=|file/d/)(?P<id>[a-zA-Z0-9-]{28})'
+    _VALID_URL = r'https?://(?:docs|drive)\.google\.com/(?:uc\?.*?id=|file/d/)(?P<id>[a-zA-Z0-9_-]{28})'
     _TEST = {
         'url': 'https://drive.google.com/file/d/0ByeS4oOUV-49Zzh4R1J6R09zazQ/edit?pli=1',
         'info_dict': {

From d1cc05e17eccccb7ee6473574c6a4f887104baeb Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 17 Jul 2015 14:37:21 +0100
Subject: [PATCH 2068/6219] remove unnecessary regex group names

---
 youtube_dl/extractor/googledrive.py | 32 ++++++++++++-----------------
 1 file changed, 13 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/extractor/googledrive.py b/youtube_dl/extractor/googledrive.py
index c82c9037f..6d9bcfefd 100644
--- a/youtube_dl/extractor/googledrive.py
+++ b/youtube_dl/extractor/googledrive.py
@@ -62,46 +62,40 @@ class GoogleDriveIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(
-            'http://docs.google.com/file/d/'+video_id, video_id, encoding='unicode_escape'
+            'http://docs.google.com/file/d/' + video_id, video_id, encoding='unicode_escape'
         )
         try:
             title = self._html_search_regex(
-                r'"title","(?P<title>.*?)"',
+                r'"title"\s+,\s+"[^"]+',
                 webpage,
-                'title',
-                group='title'
+                'title'
             )
             fmt_stream_map = self._html_search_regex(
-                r'"fmt_stream_map","(?P<fmt_stream_map>.*?)"',
+                r'"fmt_stream_map"\s+,\s+"[^"]+',
                 webpage,
-                'fmt_stream_map',
-                group='fmt_stream_map'
+                'fmt_stream_map'
             )
             fmt_list = self._html_search_regex(
-                r'"fmt_list","(?P<fmt_list>.*?)"',
+                r'"fmt_list"\s+,\s+"[^"]+',
                 webpage,
-                'fmt_list',
-                group='fmt_list'
+                'fmt_list'
             )
 #			timestamp = self._html_search_regex(
-#				r'"timestamp","(?P<timestamp>.*?)"',
+#				r'"timestamp"\s+,\s+"[^"]+',
 #				webpage,
-#				'timestamp',
-#				group='timestamp'
+#				'timestamp'
 #			)
             length_seconds = self._html_search_regex(
-                r'"length_seconds","(?P<length_seconds>.*?)"',
+                r'"length_seconds"\s+,\s+"[^"]+',
                 webpage,
-                'length_seconds',
-                group='length_seconds'
+                'length_seconds'
             )
         except RegexNotFoundError:
             try:
                 reason = self._html_search_regex(
-                    r'"reason","(?P<reason>.*?)"',
+                    r'"reason","[^"]+',
                     webpage,
-                    'reason',
-                    group='reason'
+                    'reason'
                 )
                 self.report_warning(reason)
                 return

From 36dbca87848fc5698d3e0b89380c7bcec741ceaf Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 17 Jul 2015 14:52:01 +0100
Subject: [PATCH 2069/6219] fix recursive error

---
 youtube_dl/extractor/googledrive.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/googledrive.py b/youtube_dl/extractor/googledrive.py
index 6d9bcfefd..a3d9b4450 100644
--- a/youtube_dl/extractor/googledrive.py
+++ b/youtube_dl/extractor/googledrive.py
@@ -26,7 +26,7 @@ class GoogleDriveEmbedIE(InfoExtractor):
         video_id = self._match_id(url)
         return {
             '_type': 'url',
-            'ie-key': 'GoogleDrive',
+            'ie_key': 'GoogleDrive',
             'url': 'https://drive.google.com/file/d/%s' % video_id
         }
 
@@ -66,34 +66,34 @@ class GoogleDriveIE(InfoExtractor):
         )
         try:
             title = self._html_search_regex(
-                r'"title"\s+,\s+"[^"]+',
+                r'"title"\s*,\s*"([^"]+)',
                 webpage,
                 'title'
             )
             fmt_stream_map = self._html_search_regex(
-                r'"fmt_stream_map"\s+,\s+"[^"]+',
+                r'"fmt_stream_map"\s*,\s*"([^"]+)',
                 webpage,
                 'fmt_stream_map'
             )
             fmt_list = self._html_search_regex(
-                r'"fmt_list"\s+,\s+"[^"]+',
+                r'"fmt_list"\s*,\s*"([^"]+)',
                 webpage,
                 'fmt_list'
             )
 #			timestamp = self._html_search_regex(
-#				r'"timestamp"\s+,\s+"[^"]+',
+#				r'"timestamp"\s*,\s*"([^"]+)',
 #				webpage,
 #				'timestamp'
 #			)
             length_seconds = self._html_search_regex(
-                r'"length_seconds"\s+,\s+"[^"]+',
+                r'"length_seconds"\s*,\s*"([^"]+)',
                 webpage,
                 'length_seconds'
             )
         except RegexNotFoundError:
             try:
                 reason = self._html_search_regex(
-                    r'"reason","[^"]+',
+                    r'"reason","([^"]+)',
                     webpage,
                     'reason'
                 )

From 8e92d21ebf6f17e14c9e916f22e49f27529556af Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 18 Jul 2015 23:31:14 +0100
Subject: [PATCH 2070/6219] [googledrive] raise ExtractorError instead of
 warning

---
 youtube_dl/extractor/googledrive.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/googledrive.py b/youtube_dl/extractor/googledrive.py
index a3d9b4450..7bc7b7a0d 100644
--- a/youtube_dl/extractor/googledrive.py
+++ b/youtube_dl/extractor/googledrive.py
@@ -1,7 +1,10 @@
 import re
 
 from .common import InfoExtractor
-from ..utils import RegexNotFoundError
+from ..utils import (
+    RegexNotFoundError,
+    ExtractorError,
+)
 
 class GoogleDriveEmbedIE(InfoExtractor):
     _VALID_URL = r'https?://(?:video\.google\.com/get_player\?.*?docid=|(?:docs|drive)\.google\.com/file/d/)(?P<id>[a-zA-Z0-9_-]{28})'
@@ -97,10 +100,10 @@ class GoogleDriveIE(InfoExtractor):
                     webpage,
                     'reason'
                 )
-                self.report_warning(reason)
+                raise ExtractorError(reason)
                 return
             except RegexNotFoundError:
-                self.report_warning('not a video')
+                raise ExtractorError('not a video')
                 return
 
         fmt_stream_map = fmt_stream_map.split(',')

From 297a564beeb20ca8b00d94f5707532110631f409 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 23 Jul 2015 13:20:21 +0200
Subject: [PATCH 2071/6219] [youtube] Extract end_time

---
 youtube_dl/extractor/common.py  |  2 ++
 youtube_dl/extractor/youtube.py | 10 +++++++---
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 9e8751877..1272834c5 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -185,6 +185,8 @@ class InfoExtractor(object):
                     live stream that goes on instead of a fixed-length video.
     start_time:     Time in seconds where the reproduction should start, as
                     specified in the url.
+    end_time:       Time in seconds where the reproduction should end, as
+                    specified in the url.
 
     Unless mentioned otherwise, the fields should be Unicode strings.
 
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index afbd34f4a..117ef2e77 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -319,7 +319,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
     IE_NAME = 'youtube'
     _TESTS = [
         {
-            'url': 'http://www.youtube.com/watch?v=BaW_jenozKcj&t=1s',
+            'url': 'http://www.youtube.com/watch?v=BaW_jenozKcj&t=1s&end=9',
             'info_dict': {
                 'id': 'BaW_jenozKc',
                 'ext': 'mp4',
@@ -332,6 +332,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'like_count': int,
                 'dislike_count': int,
                 'start_time': 1,
+                'end_time': 9,
             }
         },
         {
@@ -893,12 +894,14 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             else 'https')
 
         start_time = None
+        end_time = None
         parsed_url = compat_urllib_parse_urlparse(url)
         for component in [parsed_url.fragment, parsed_url.query]:
             query = compat_parse_qs(component)
-            if 't' in query:
+            if start_time is None and 't' in query:
                 start_time = parse_duration(query['t'][0])
-                break
+            if end_time is None and 'end' in query:
+                end_time = parse_duration(query['end'][0])
 
         # Extract original video URL from URL with redirection, like age verification, using next_url parameter
         mobj = re.search(self._NEXT_URL_RE, url)
@@ -1267,6 +1270,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             'formats': formats,
             'is_live': is_live,
             'start_time': start_time,
+            'end_time': end_time,
         }
 
 

From 2929fa0e79dfd3a1366e7e23eb4344bc93dd3a10 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 23 Jul 2015 13:21:18 +0200
Subject: [PATCH 2072/6219] [youtube] Also look into the 'start' field for
 start_time

---
 youtube_dl/extractor/youtube.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 117ef2e77..462d244d8 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -900,6 +900,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             query = compat_parse_qs(component)
             if start_time is None and 't' in query:
                 start_time = parse_duration(query['t'][0])
+            if start_time is None and 'start' in query:
+                start_time = parse_duration(query['start'][0])
             if end_time is None and 'end' in query:
                 end_time = parse_duration(query['end'][0])
 

From d3f007af183a0f5ed278602128e6bba3cc1350b3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 23 Jul 2015 22:04:47 +0600
Subject: [PATCH 2073/6219] [daylimotion] Adapt to player v5 and modernize
 (Closes #6151, closes #6250)

---
 youtube_dl/extractor/dailymotion.py | 160 ++++++++++++++++++++--------
 1 file changed, 117 insertions(+), 43 deletions(-)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index 1a41c0db1..8fcae7402 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -13,8 +13,10 @@ from ..compat import (
 )
 from ..utils import (
     ExtractorError,
+    determine_ext,
     int_or_none,
     orderedSet,
+    parse_iso8601,
     str_to_int,
     unescapeHTML,
 )
@@ -28,10 +30,12 @@ class DailymotionBaseInfoExtractor(InfoExtractor):
         request.add_header('Cookie', 'family_filter=off; ff=off')
         return request
 
+    def _download_webpage_no_ff(self, url, *args, **kwargs):
+        request = self._build_request(url)
+        return self._download_webpage(request, *args, **kwargs)
+
 
 class DailymotionIE(DailymotionBaseInfoExtractor):
-    """Information Extractor for Dailymotion"""
-
     _VALID_URL = r'(?i)(?:https?://)?(?:(www|touch)\.)?dailymotion\.[a-z]{2,3}/(?:(embed|#)/)?video/(?P<id>[^/?_]+)'
     IE_NAME = 'dailymotion'
 
@@ -50,10 +54,17 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
             'info_dict': {
                 'id': 'x2iuewm',
                 'ext': 'mp4',
-                'uploader': 'IGN',
                 'title': 'Steam Machine Models, Pricing Listed on Steam Store - IGN News',
-                'upload_date': '20150306',
+                'description': 'Several come bundled with the Steam Controller.',
+                'thumbnail': 're:^https?:.*\.(?:jpg|png)$',
                 'duration': 74,
+                'timestamp': 1425657362,
+                'upload_date': '20150306',
+                'uploader': 'IGN',
+                'uploader_id': 'xijv66',
+                'age_limit': 0,
+                'view_count': int,
+                'comment_count': int,
             }
         },
         # Vevo video
@@ -87,38 +98,106 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        url = 'https://www.dailymotion.com/video/%s' % video_id
 
-        # Retrieve video webpage to extract further information
-        request = self._build_request(url)
-        webpage = self._download_webpage(request, video_id)
-
-        # Extract URL, uploader and title from webpage
-        self.report_extraction(video_id)
-
-        # It may just embed a vevo video:
-        m_vevo = re.search(
-            r'<link rel="video_src" href="[^"]*?vevo.com[^"]*?video=(?P<id>[\w]*)',
-            webpage)
-        if m_vevo is not None:
-            vevo_id = m_vevo.group('id')
-            self.to_screen('Vevo video detected: %s' % vevo_id)
-            return self.url_result('vevo:%s' % vevo_id, ie='Vevo')
+        webpage = self._download_webpage_no_ff(
+            'https://www.dailymotion.com/video/%s' % video_id, video_id)
 
         age_limit = self._rta_search(webpage)
 
-        video_upload_date = None
-        mobj = re.search(r'<meta property="video:release_date" content="([0-9]{4})-([0-9]{2})-([0-9]{2}).+?"/>', webpage)
-        if mobj is not None:
-            video_upload_date = mobj.group(1) + mobj.group(2) + mobj.group(3)
+        description = self._og_search_description(webpage) or self._html_search_meta(
+            'description', webpage, 'description')
+
+        view_count = str_to_int(self._search_regex(
+            [r'<meta[^>]+itemprop="interactionCount"[^>]+content="UserPlays:(\d+)"',
+             r'video_views_count[^>]+>\s+([\d\.,]+)'],
+            webpage, 'view count', fatal=False))
+        comment_count = int_or_none(self._search_regex(
+            r'<meta[^>]+itemprop="interactionCount"[^>]+content="UserComments:(\d+)"',
+            webpage, 'comment count', fatal=False))
+
+        player_v5 = self._search_regex(
+            r'playerV5\s*=\s*dmp\.create\([^,]+?,\s*({.+?})\);',
+            webpage, 'player v5', default=None)
+        if player_v5:
+            player = self._parse_json(player_v5, video_id)
+            metadata = player['metadata']
+            formats = []
+            for quality, media_list in metadata['qualities'].items():
+                for media in media_list:
+                    media_url = media.get('url')
+                    if not media_url:
+                        continue
+                    type_ = media.get('type')
+                    if type_ == 'application/vnd.lumberjack.manifest':
+                        continue
+                    if type_ == 'application/x-mpegURL' or determine_ext(media_url) == 'm3u8':
+                        formats.extend(self._extract_m3u8_formats(
+                            media_url, video_id, 'mp4', m3u8_id='hls'))
+                    else:
+                        f = {
+                            'url': media_url,
+                            'format_id': quality,
+                        }
+                        m = re.search(r'H264-(?P<width>\d+)x(?P<height>\d+)', media_url)
+                        if m:
+                            f.update({
+                                'width': int(m.group('width')),
+                                'height': int(m.group('height')),
+                            })
+                        formats.append(f)
+            self._sort_formats(formats)
+
+            title = metadata['title']
+            duration = int_or_none(metadata.get('duration'))
+            timestamp = int_or_none(metadata.get('created_time'))
+            thumbnail = metadata.get('poster_url')
+            uploader = metadata.get('owner', {}).get('screenname')
+            uploader_id = metadata.get('owner', {}).get('id')
+
+            subtitles = {}
+            for subtitle_lang, subtitle in metadata.get('subtitles', {}).get('data', {}).items():
+                subtitles[subtitle_lang] = [{
+                    'ext': determine_ext(subtitle_url),
+                    'url': subtitle_url,
+                } for subtitle_url in subtitle.get('urls', [])]
+
+            return {
+                'id': video_id,
+                'title': title,
+                'description': description,
+                'thumbnail': thumbnail,
+                'duration': duration,
+                'timestamp': timestamp,
+                'uploader': uploader,
+                'uploader_id': uploader_id,
+                'age_limit': age_limit,
+                'view_count': view_count,
+                'comment_count': comment_count,
+                'formats': formats,
+                'subtitles': subtitles,
+            }
+
+        # vevo embed
+        vevo_id = self._search_regex(
+            r'<link rel="video_src" href="[^"]*?vevo.com[^"]*?video=(?P<id>[\w]*)',
+            webpage, 'vevo embed', default=None)
+        if vevo_id:
+            return self.url_result('vevo:%s' % vevo_id, 'Vevo')
+
+        # fallback old player
+        embed_page = self._download_webpage_no_ff(
+            'https://www.dailymotion.com/embed/video/%s' % video_id,
+            video_id, 'Downloading embed page')
+
+        timestamp = parse_iso8601(self._html_search_meta(
+            'video:release_date', webpage, 'upload date'))
+
+        info = self._parse_json(
+            self._search_regex(
+                r'var info = ({.*?}),$', embed_page,
+                'video info', flags=re.MULTILINE),
+            video_id)
 
-        embed_url = 'https://www.dailymotion.com/embed/video/%s' % video_id
-        embed_request = self._build_request(embed_url)
-        embed_page = self._download_webpage(
-            embed_request, video_id, 'Downloading embed page')
-        info = self._search_regex(r'var info = ({.*?}),$', embed_page,
-                                  'video info', flags=re.MULTILINE)
-        info = json.loads(info)
         if info.get('error') is not None:
             msg = 'Couldn\'t get video, Dailymotion says: %s' % info['error']['title']
             raise ExtractorError(msg, expected=True)
@@ -139,16 +218,11 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
                     'width': width,
                     'height': height,
                 })
-        if not formats:
-            raise ExtractorError('Unable to extract video URL')
+        self._sort_formats(formats)
 
         # subtitles
         video_subtitles = self.extract_subtitles(video_id, webpage)
 
-        view_count = str_to_int(self._search_regex(
-            r'video_views_count[^>]+>\s+([\d\.,]+)',
-            webpage, 'view count', fatal=False))
-
         title = self._og_search_title(webpage, default=None)
         if title is None:
             title = self._html_search_regex(
@@ -159,8 +233,9 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
             'id': video_id,
             'formats': formats,
             'uploader': info['owner.screenname'],
-            'upload_date': video_upload_date,
+            'timestamp': timestamp,
             'title': title,
+            'description': description,
             'subtitles': video_subtitles,
             'thumbnail': info['thumbnail_url'],
             'age_limit': age_limit,
@@ -201,9 +276,9 @@ class DailymotionPlaylistIE(DailymotionBaseInfoExtractor):
     def _extract_entries(self, id):
         video_ids = []
         for pagenum in itertools.count(1):
-            request = self._build_request(self._PAGE_TEMPLATE % (id, pagenum))
-            webpage = self._download_webpage(request,
-                                             id, 'Downloading page %s' % pagenum)
+            webpage = self._download_webpage_no_ff(
+                self._PAGE_TEMPLATE % (id, pagenum),
+                id, 'Downloading page %s' % pagenum)
 
             video_ids.extend(re.findall(r'data-xid="(.+?)"', webpage))
 
@@ -286,8 +361,7 @@ class DailymotionCloudIE(DailymotionBaseInfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        request = self._build_request(url)
-        webpage = self._download_webpage(request, video_id)
+        webpage = self._download_webpage_no_ff(url, video_id)
 
         title = self._html_search_regex(r'<title>([^>]+)</title>', webpage, 'title')
 

From ba911137fa53737b663cb2879d357bc8a35ab558 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 23 Jul 2015 22:50:59 +0600
Subject: [PATCH 2074/6219] [rts] Add support for articles with videos on rhs
 (Closes #6332)

---
 youtube_dl/extractor/rts.py | 31 +++++++++++++++++++++++++++++--
 1 file changed, 29 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/rts.py b/youtube_dl/extractor/rts.py
index 9fbe239d8..12639f08b 100644
--- a/youtube_dl/extractor/rts.py
+++ b/youtube_dl/extractor/rts.py
@@ -19,7 +19,16 @@ from ..utils import (
 
 class RTSIE(InfoExtractor):
     IE_DESC = 'RTS.ch'
-    _VALID_URL = r'https?://(?:www\.)?rts\.ch/(?:(?:[^/]+/){2,}(?P<id>[0-9]+)-(?P<display_id>.+?)\.html|play/tv/[^/]+/video/(?P<display_id_new>.+?)\?id=(?P<id_new>[0-9]+))'
+    _VALID_URL = r'''(?x)
+                    (?:
+                        rts:(?P<rts_id>\d+)|
+                        https?://
+                            (?:www\.)?rts\.ch/
+                            (?:
+                                (?:[^/]+/){2,}(?P<id>[0-9]+)-(?P<display_id>.+?)\.html|
+                                play/tv/[^/]+/video/(?P<display_id_new>.+?)\?id=(?P<id_new>[0-9]+)
+                            )
+                    )'''
 
     _TESTS = [
         {
@@ -122,6 +131,15 @@ class RTSIE(InfoExtractor):
                 'view_count': int,
             },
         },
+        {
+            # article with videos on rhs
+            'url': 'http://www.rts.ch/sport/hockey/6693917-hockey-davos-decroche-son-31e-titre-de-champion-de-suisse.html',
+            'info_dict': {
+                'id': '6693917',
+                'title': 'Hockey: Davos décroche son 31e titre de champion de Suisse',
+            },
+            'playlist_mincount': 5,
+        },
         {
             'url': 'http://www.rts.ch/play/tv/le-19h30/video/le-chantier-du-nouveau-parlement-vaudois-a-permis-une-trouvaille-historique?id=6348280',
             'only_matching': True,
@@ -130,7 +148,7 @@ class RTSIE(InfoExtractor):
 
     def _real_extract(self, url):
         m = re.match(self._VALID_URL, url)
-        video_id = m.group('id') or m.group('id_new')
+        video_id = m.group('rts_id') or m.group('id') or m.group('id_new')
         display_id = m.group('display_id') or m.group('display_id_new')
 
         def download_json(internal_id):
@@ -143,6 +161,15 @@ class RTSIE(InfoExtractor):
         # video_id extracted out of URL is not always a real id
         if 'video' not in all_info and 'audio' not in all_info:
             page = self._download_webpage(url, display_id)
+
+            # article with videos on rhs
+            videos = re.findall(
+                r'<article[^>]+class="content-item"[^>]*>\s*<a[^>]+data-video-urn="urn:rts:video:(\d+)"',
+                page)
+            if videos:
+                entries = [self.url_result('rts:%s' % video_urn, 'RTS') for video_urn in videos]
+                return self.playlist_result(entries, video_id, self._og_search_title(page))
+
             internal_id = self._html_search_regex(
                 r'<(?:video|audio) data-id="([0-9]+)"', page,
                 'internal video id')

From a8b7b26068fa8de9983ffef995ea6cd4fd3ce90a Mon Sep 17 00:00:00 2001
From: corone17 <reino@degeelebosch.nl>
Date: Wed, 22 Jul 2015 22:38:04 +0200
Subject: [PATCH 2075/6219] Undo adaptive -> flash workaround

For a couple of days now rtlXL's non-DRM adaptive (m3u8) streams don't work anymore. By undoing the adaptive -> flash workaround youtube-dl returns the DRM streams again. Too bad for foreigners (geoblock).
The progressive streams still work fine.
---
 youtube_dl/extractor/rtlnl.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/rtlnl.py b/youtube_dl/extractor/rtlnl.py
index e0c530d64..518f7a7b4 100644
--- a/youtube_dl/extractor/rtlnl.py
+++ b/youtube_dl/extractor/rtlnl.py
@@ -86,12 +86,12 @@ class RtlNlIE(InfoExtractor):
         # NB: nowadays, recent ffmpeg and avconv can handle these encrypted streams, so
         # this adaptive -> flash workaround is not required in general, but it also
         # allows bypassing georestriction therefore is retained for now.
-        videopath = material['videopath'].replace('/adaptive/', '/flash/')
+        videopath = material['videopath']
         m3u8_url = meta.get('videohost', 'http://manifest.us.rtl.nl') + videopath
 
         formats = self._extract_m3u8_formats(m3u8_url, uuid, ext='mp4')
 
-        video_urlpart = videopath.split('/flash/')[1][:-5]
+        video_urlpart = videopath.split('/adaptive/')[1][:-5]
         PG_URL_TEMPLATE = 'http://pg.us.rtl.nl/rtlxl/network/%s/progressive/%s.mp4'
 
         formats.extend([

From a9e8f60ef686eeeb3d31f2e5211ce176dc8f5298 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 23 Jul 2015 23:20:24 +0600
Subject: [PATCH 2076/6219] [rtlnl] Update unencrypted streams comment (#6337)

---
 youtube_dl/extractor/rtlnl.py | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/rtlnl.py b/youtube_dl/extractor/rtlnl.py
index 518f7a7b4..543d94417 100644
--- a/youtube_dl/extractor/rtlnl.py
+++ b/youtube_dl/extractor/rtlnl.py
@@ -82,10 +82,15 @@ class RtlNlIE(InfoExtractor):
 
         meta = info.get('meta', {})
 
-        # Use unencrypted m3u8 streams (See https://github.com/rg3/youtube-dl/issues/4118)
-        # NB: nowadays, recent ffmpeg and avconv can handle these encrypted streams, so
-        # this adaptive -> flash workaround is not required in general, but it also
-        # allows bypassing georestriction therefore is retained for now.
+        # m3u8 streams are encrypted and may not be handled properly by older ffmpeg/avconv.
+        # To workaround this previously adaptive -> flash trick was used to obtain
+        # unencrypted m3u8 streams (see https://github.com/rg3/youtube-dl/issues/4118)
+        # and bypass georestrictions as well.
+        # Currently, unencrypted m3u8 playlists are (intentionally?) invalid and therefore
+        # unusable albeit can be fixed by simple string replacement (see
+        # https://github.com/rg3/youtube-dl/pull/6337)
+        # Since recent ffmpeg and avconv handle encrypted streams just fine encrypted
+        # streams are used now.
         videopath = material['videopath']
         m3u8_url = meta.get('videohost', 'http://manifest.us.rtl.nl') + videopath
 

From 10952eb2cf2c86c841e46690a3bfa6a39c82d67d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 23 Jul 2015 23:37:45 +0600
Subject: [PATCH 2077/6219] [extractor/common] Consistent URL spelling

---
 youtube_dl/extractor/common.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 1272834c5..14b9b4fe2 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -65,7 +65,7 @@ class InfoExtractor(object):
 
                     Potential fields:
                     * url        Mandatory. The URL of the video file
-                    * ext        Will be calculated from url if missing
+                    * ext        Will be calculated from URL if missing
                     * format     A human-readable description of the format
                                  ("mp4 container with h264/opus").
                                  Calculated from the format_id, width, height.
@@ -155,7 +155,7 @@ class InfoExtractor(object):
                     lower to higher preference, each element is a dictionary
                     with the "ext" entry and one of:
                         * "data": The subtitles file contents
-                        * "url": A url pointing to the subtitles file
+                        * "url": A URL pointing to the subtitles file
     automatic_captions: Like 'subtitles', used by the YoutubeIE for
                     automatically generated captions
     duration:       Length of the video in seconds, as an integer.
@@ -176,7 +176,7 @@ class InfoExtractor(object):
                                      Set to "root" to indicate that this is a
                                      comment to the original video.
     age_limit:      Age restriction for the video, as an integer (years)
-    webpage_url:    The url to the video webpage, if given to youtube-dl it
+    webpage_url:    The URL to the video webpage, if given to youtube-dl it
                     should allow to get the same result again. (It will be set
                     by YoutubeDL if it's missing)
     categories:     A list of categories that the video falls in, for example
@@ -184,9 +184,9 @@ class InfoExtractor(object):
     is_live:        True, False, or None (=unknown). Whether this video is a
                     live stream that goes on instead of a fixed-length video.
     start_time:     Time in seconds where the reproduction should start, as
-                    specified in the url.
+                    specified in the URL.
     end_time:       Time in seconds where the reproduction should end, as
-                    specified in the url.
+                    specified in the URL.
 
     Unless mentioned otherwise, the fields should be Unicode strings.
 
@@ -505,7 +505,7 @@ class InfoExtractor(object):
     # Methods for following #608
     @staticmethod
     def url_result(url, ie=None, video_id=None, video_title=None):
-        """Returns a url that points to a page that should be processed"""
+        """Returns a URL that points to a page that should be processed"""
         # TODO: ie should be the class used for getting the info
         video_info = {'_type': 'url',
                       'url': url,
@@ -639,7 +639,7 @@ class InfoExtractor(object):
         return unescapeHTML(escaped)
 
     def _og_search_thumbnail(self, html, **kargs):
-        return self._og_search_property('image', html, 'thumbnail url', fatal=False, **kargs)
+        return self._og_search_property('image', html, 'thumbnail URL', fatal=False, **kargs)
 
     def _og_search_description(self, html, **kargs):
         return self._og_search_property('description', html, fatal=False, **kargs)
@@ -1120,7 +1120,7 @@ class InfoExtractor(object):
 class SearchInfoExtractor(InfoExtractor):
     """
     Base class for paged search queries extractors.
-    They accept urls in the format _SEARCH_KEY(|all|[0-9]):{query}
+    They accept URLs in the format _SEARCH_KEY(|all|[0-9]):{query}
     Instances should define _SEARCH_KEY and _MAX_RESULTS.
     """
 

From 660f9459dabfc01c7235b4cc57f9c561de522e07 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 24 Jul 2015 00:17:36 +0600
Subject: [PATCH 2078/6219] [canalplus] Fix m3u8 videos extension

---
 youtube_dl/extractor/canalplus.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/canalplus.py b/youtube_dl/extractor/canalplus.py
index 699b4f7d0..8671236dd 100644
--- a/youtube_dl/extractor/canalplus.py
+++ b/youtube_dl/extractor/canalplus.py
@@ -106,7 +106,7 @@ class CanalplusIE(InfoExtractor):
                 continue
             format_id = fmt.tag
             if format_id == 'HLS':
-                hls_formats = self._extract_m3u8_formats(format_url, video_id, 'flv')
+                hls_formats = self._extract_m3u8_formats(format_url, video_id, 'mp4')
                 for fmt in hls_formats:
                     fmt['preference'] = preference(format_id)
                 formats.extend(hls_formats)

From f3f0b8e4030a07e07385afe6ebb4485ac33e5357 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 24 Jul 2015 00:21:13 +0600
Subject: [PATCH 2079/6219] [canalplus] Modernize

---
 youtube_dl/extractor/canalplus.py | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/canalplus.py b/youtube_dl/extractor/canalplus.py
index 8671236dd..57e0cda2c 100644
--- a/youtube_dl/extractor/canalplus.py
+++ b/youtube_dl/extractor/canalplus.py
@@ -106,15 +106,11 @@ class CanalplusIE(InfoExtractor):
                 continue
             format_id = fmt.tag
             if format_id == 'HLS':
-                hls_formats = self._extract_m3u8_formats(format_url, video_id, 'mp4')
-                for fmt in hls_formats:
-                    fmt['preference'] = preference(format_id)
-                formats.extend(hls_formats)
+                formats.extend(self._extract_m3u8_formats(
+                    format_url, video_id, 'mp4', preference=preference(format_id)))
             elif format_id == 'HDS':
-                hds_formats = self._extract_f4m_formats(format_url + '?hdcore=2.11.3', video_id)
-                for fmt in hds_formats:
-                    fmt['preference'] = preference(format_id)
-                formats.extend(hds_formats)
+                formats.extend(self._extract_f4m_formats(
+                    format_url + '?hdcore=2.11.3', video_id, preference=preference(format_id)))
             else:
                 formats.append({
                     'url': format_url,

From d97f5cd795b0bd8b6bd636b899bdbc8b10d5907b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 24 Jul 2015 02:56:54 +0600
Subject: [PATCH 2080/6219] [bbccouk] Make more robust (Closes #6345)

---
 youtube_dl/extractor/bbccouk.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/bbccouk.py b/youtube_dl/extractor/bbccouk.py
index 5825d2867..b2e5f7418 100644
--- a/youtube_dl/extractor/bbccouk.py
+++ b/youtube_dl/extractor/bbccouk.py
@@ -210,12 +210,12 @@ class BBCCoUkIE(InfoExtractor):
 
     def _extract_video(self, media, programme_id):
         formats = []
-        vbr = int(media.get('bitrate'))
+        vbr = int_or_none(media.get('bitrate'))
         vcodec = media.get('encoding')
         service = media.get('service')
-        width = int(media.get('width'))
-        height = int(media.get('height'))
-        file_size = int(media.get('media_file_size'))
+        width = int_or_none(media.get('width'))
+        height = int_or_none(media.get('height'))
+        file_size = int_or_none(media.get('media_file_size'))
         for connection in self._extract_connections(media):
             conn_formats = self._extract_connection(connection, programme_id)
             for format in conn_formats:
@@ -232,7 +232,7 @@ class BBCCoUkIE(InfoExtractor):
 
     def _extract_audio(self, media, programme_id):
         formats = []
-        abr = int(media.get('bitrate'))
+        abr = int_or_none(media.get('bitrate'))
         acodec = media.get('encoding')
         service = media.get('service')
         for connection in self._extract_connections(media):
@@ -300,7 +300,7 @@ class BBCCoUkIE(InfoExtractor):
                     if kind != 'programme' and kind != 'radioProgramme':
                         continue
                     programme_id = item.get('vpid')
-                    duration = int(item.get('duration'))
+                    duration = int_or_none(item.get('duration'))
                     formats, subtitles = self._download_media_selector(programme_id)
                 return programme_id, title, description, duration, formats, subtitles
         except ExtractorError as ee:
@@ -332,7 +332,7 @@ class BBCCoUkIE(InfoExtractor):
             title = playlist.find('./{http://bbc.co.uk/2008/emp/playlist}title').text
             description = playlist.find('./{http://bbc.co.uk/2008/emp/playlist}summary').text
             programme_id = item.get('identifier')
-            duration = int(item.get('duration'))
+            duration = int_or_none(item.get('duration'))
             formats, subtitles = self._download_media_selector(programme_id)
 
         return programme_id, title, description, duration, formats, subtitles

From 2b2ee140c3c6f08b4078cc6a5a289e5e74bec2b4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 24 Jul 2015 03:40:24 +0600
Subject: [PATCH 2081/6219] [dailymotion:user] Fix _VALID_URL (Closes #6346)

---
 youtube_dl/extractor/dailymotion.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index 8fcae7402..5a4987772 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -302,7 +302,7 @@ class DailymotionPlaylistIE(DailymotionBaseInfoExtractor):
 
 class DailymotionUserIE(DailymotionPlaylistIE):
     IE_NAME = 'dailymotion:user'
-    _VALID_URL = r'https?://(?:www\.)?dailymotion\.[a-z]{2,3}/(?:(?:old/)?user/)?(?P<user>[^/]+)$'
+    _VALID_URL = r'https?://(?:www\.)?dailymotion\.[a-z]{2,3}/(?!(?:embed|#|video|playlist)/)(?:(?:old/)?user/)?(?P<user>[^/]+)'
     _PAGE_TEMPLATE = 'http://www.dailymotion.com/user/%s/%s'
     _TESTS = [{
         'url': 'https://www.dailymotion.com/user/nqtv',

From 1a117a77287e7dbd4d92f29062dabcf4efb86cb5 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 24 Jul 2015 12:00:20 +0100
Subject: [PATCH 2082/6219] [clipfish] extract mp4 video link

---
 youtube_dl/extractor/clipfish.py | 37 ++++++++++++++++----------------
 1 file changed, 19 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/clipfish.py b/youtube_dl/extractor/clipfish.py
index a5c3cb7c6..09dfaac60 100644
--- a/youtube_dl/extractor/clipfish.py
+++ b/youtube_dl/extractor/clipfish.py
@@ -1,13 +1,11 @@
 from __future__ import unicode_literals
 
-import re
-import time
-import xml.etree.ElementTree
-
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
-    parse_duration,
+    int_or_none,
+    js_to_json,
+    determine_ext,
 )
 
 
@@ -17,37 +15,40 @@ class ClipfishIE(InfoExtractor):
     _VALID_URL = r'^https?://(?:www\.)?clipfish\.de/.*?/video/(?P<id>[0-9]+)/'
     _TEST = {
         'url': 'http://www.clipfish.de/special/game-trailer/video/3966754/fifa-14-e3-2013-trailer/',
-        'md5': '2521cd644e862936cf2e698206e47385',
+        'md5': '79bc922f3e8a9097b3d68a93780fd475',
         'info_dict': {
             'id': '3966754',
             'ext': 'mp4',
             'title': 'FIFA 14 - E3 2013 Trailer',
             'duration': 82,
-        },
-        'skip': 'Blocked in the US'
+        }
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group(1)
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        video_info = self._parse_json(
+            js_to_json(self._html_search_regex('var videoObject = ({[^}]+?})', webpage, 'videoObject')),
+            video_id
+        )
+        info_url = self._parse_json(
+            js_to_json(self._html_search_regex('var globalFlashvars = ({[^}]+?})', webpage, 'globalFlashvars')),
+            video_id
+        )['data']
 
-        info_url = ('http://www.clipfish.de/devxml/videoinfo/%s?ts=%d' %
-                    (video_id, int(time.time())))
         doc = self._download_xml(
             info_url, video_id, note='Downloading info page')
         title = doc.find('title').text
         video_url = doc.find('filename').text
-        if video_url is None:
-            xml_bytes = xml.etree.ElementTree.tostring(doc)
-            raise ExtractorError('Cannot find video URL in document %r' %
-                                 xml_bytes)
         thumbnail = doc.find('imageurl').text
-        duration = parse_duration(doc.find('duration').text)
+        duration = int_or_none(video_info['length'])
+        formats = [{'url': video_info['videourl']},{'url': video_url}]
+        self._sort_formats(formats)
 
         return {
             'id': video_id,
             'title': title,
-            'url': video_url,
+            'formats': formats,
             'thumbnail': thumbnail,
             'duration': duration,
         }

From 12434026574bcaaaa705c31ef14428cc91a5efad Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 24 Jul 2015 21:29:44 +0800
Subject: [PATCH 2083/6219] [dailymotion:playlist] Detect problematic
 redirection (fixes #6347)

---
 youtube_dl/extractor/dailymotion.py | 28 +++++++++++++++++++++++++---
 1 file changed, 25 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index 5a4987772..85d945509 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -30,6 +30,10 @@ class DailymotionBaseInfoExtractor(InfoExtractor):
         request.add_header('Cookie', 'family_filter=off; ff=off')
         return request
 
+    def _download_webpage_handle_no_ff(self, url, *args, **kwargs):
+        request = self._build_request(url)
+        return self._download_webpage_handle(request, *args, **kwargs)
+
     def _download_webpage_no_ff(self, url, *args, **kwargs):
         request = self._build_request(url)
         return self._download_webpage(request, *args, **kwargs)
@@ -275,10 +279,17 @@ class DailymotionPlaylistIE(DailymotionBaseInfoExtractor):
 
     def _extract_entries(self, id):
         video_ids = []
+        processed_urls = set()
         for pagenum in itertools.count(1):
-            webpage = self._download_webpage_no_ff(
-                self._PAGE_TEMPLATE % (id, pagenum),
-                id, 'Downloading page %s' % pagenum)
+            page_url = self._PAGE_TEMPLATE % (id, pagenum)
+            webpage, urlh = self._download_webpage_handle_no_ff(
+                page_url, id, 'Downloading page %s' % pagenum)
+            if urlh.geturl() in processed_urls:
+                self.report_warning('Stopped at duplicated page %s, which is the same as %s' % (
+                    page_url, urlh.geturl()), id)
+                break
+
+            processed_urls.add(urlh.geturl())
 
             video_ids.extend(re.findall(r'data-xid="(.+?)"', webpage))
 
@@ -311,6 +322,17 @@ class DailymotionUserIE(DailymotionPlaylistIE):
             'title': 'Rémi Gaillard',
         },
         'playlist_mincount': 100,
+    }, {
+        'url': 'http://www.dailymotion.com/user/UnderProject',
+        'info_dict': {
+            'id': 'UnderProject',
+            'title': 'UnderProject',
+        },
+        'playlist_mincount': 1800,
+        'expected_warnings': [
+            'Stopped at duplicated page',
+        ],
+        'skip': 'Takes too long time',
     }]
 
     def _real_extract(self, url):

From 593ddd851b87eb7ac327a9216800162f07b2acca Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 24 Jul 2015 14:46:45 +0100
Subject: [PATCH 2084/6219] [letv] fix height

---
 youtube_dl/extractor/letv.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/letv.py b/youtube_dl/extractor/letv.py
index ba2ae8085..a28abb0f0 100644
--- a/youtube_dl/extractor/letv.py
+++ b/youtube_dl/extractor/letv.py
@@ -15,6 +15,7 @@ from ..utils import (
     determine_ext,
     ExtractorError,
     parse_iso8601,
+    int_or_none,
 )
 
 
@@ -134,7 +135,7 @@ class LetvIE(InfoExtractor):
                 }
 
                 if format_id[-1:] == 'p':
-                    url_info_dict['height'] = format_id[:-1]
+                    url_info_dict['height'] = int_or_none(format_id[:-1])
 
                 urls.append(url_info_dict)
 

From 678e436f2e77f1ae3a57c4b5d1fc3d74342ab412 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 25 Jul 2015 02:09:34 +0600
Subject: [PATCH 2085/6219] [youtube] Handle empty allowed regions (Closes
 #6351)

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 229fc3a0f..4023a6e50 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -993,7 +993,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             if 'reason' in video_info:
                 if 'The uploader has not made this video available in your country.' in video_info['reason']:
                     regions_allowed = self._html_search_meta('regionsAllowed', video_webpage, default=None)
-                    if regions_allowed is not None:
+                    if regions_allowed:
                         raise ExtractorError('YouTube said: This video is available in %s only' % (
                             ', '.join(map(ISO3166Utils.short2full, regions_allowed.split(',')))),
                             expected=True)

From b14fa8e6874818a3f210b2a67cf53345000defdd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 25 Jul 2015 15:47:53 +0600
Subject: [PATCH 2086/6219] [soundcloud:set] Defer download link resolve
 (Closes #6354)

---
 youtube_dl/extractor/soundcloud.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 118ca4832..cdee8e2a3 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -282,9 +282,11 @@ class SoundcloudSetIE(SoundcloudIE):
             msgs = (compat_str(err['error_message']) for err in info['errors'])
             raise ExtractorError('unable to download video webpage: %s' % ','.join(msgs))
 
+        entries = [self.url_result(track['permalink_url'], 'Soundcloud') for track in info['tracks']]
+
         return {
             '_type': 'playlist',
-            'entries': [self._extract_info_dict(track, secret_token=token) for track in info['tracks']],
+            'entries': entries,
             'id': '%s' % info['id'],
             'title': info['title'],
         }

From 40a2d17052e9b542eb3c360a0ce067d244e07fb4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 25 Jul 2015 15:48:44 +0600
Subject: [PATCH 2087/6219] [soundcloud:playlist] Defer download link resolve

---
 youtube_dl/extractor/soundcloud.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index cdee8e2a3..0a6c9fe72 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -381,9 +381,7 @@ class SoundcloudPlaylistIE(SoundcloudIE):
         data = self._download_json(
             base_url + data, playlist_id, 'Downloading playlist')
 
-        entries = [
-            self._extract_info_dict(t, quiet=True, secret_token=token)
-            for t in data['tracks']]
+        entries = [self.url_result(track['permalink_url'], 'Soundcloud') for track in data['tracks']]
 
         return {
             '_type': 'playlist',

From eab7faa0c1e8511bc91c64347d0dffc28c94f101 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 25 Jul 2015 18:39:01 +0800
Subject: [PATCH 2088/6219] [ir90tv] Test (?:www\.)? part in _VALID_URL

---
 youtube_dl/extractor/ir90tv.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/ir90tv.py b/youtube_dl/extractor/ir90tv.py
index b79529b1b..880a6e32f 100644
--- a/youtube_dl/extractor/ir90tv.py
+++ b/youtube_dl/extractor/ir90tv.py
@@ -6,7 +6,7 @@ from .common import InfoExtractor
 
 class Ir90TvIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?90tv\.ir/video/(?P<id>[0-9]+)/.*'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://90tv.ir/video/95719/%D8%B4%D8%A7%DB%8C%D8%B9%D8%A7%D8%AA-%D9%86%D9%82%D9%84-%D9%88-%D8%A7%D9%86%D8%AA%D9%82%D8%A7%D9%84%D8%A7%D8%AA-%D9%85%D9%87%D9%85-%D9%81%D9%88%D8%AA%D8%A8%D8%A7%D9%84-%D8%A7%D8%B1%D9%88%D9%BE%D8%A7-940218',
         'md5': '411dbd94891381960cb9e13daa47a869',
         'info_dict': {
@@ -15,7 +15,10 @@ class Ir90TvIE(InfoExtractor):
             'title': 'شایعات نقل و انتقالات مهم فوتبال اروپا 94/02/18',
             'thumbnail': 're:^https?://.*\.jpg$',
         }
-    }
+    }, {
+        'url': 'http://www.90tv.ir/video/95719/%D8%B4%D8%A7%DB%8C%D8%B9%D8%A7%D8%AA-%D9%86%D9%82%D9%84-%D9%88-%D8%A7%D9%86%D8%AA%D9%82%D8%A7%D9%84%D8%A7%D8%AA-%D9%85%D9%87%D9%85-%D9%81%D9%88%D8%AA%D8%A8%D8%A7%D9%84-%D8%A7%D8%B1%D9%88%D9%BE%D8%A7-940218',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From 9700cd9097445d598515dc41fb3cb9421403b9b9 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 25 Jul 2015 18:42:40 +0800
Subject: [PATCH 2089/6219] [ir90tv] Improve title extraction

---
 youtube_dl/extractor/ir90tv.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/ir90tv.py b/youtube_dl/extractor/ir90tv.py
index 880a6e32f..92333c3ea 100644
--- a/youtube_dl/extractor/ir90tv.py
+++ b/youtube_dl/extractor/ir90tv.py
@@ -2,6 +2,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..utils import remove_start
 
 
 class Ir90TvIE(InfoExtractor):
@@ -24,8 +25,8 @@ class Ir90TvIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        title = self._html_search_regex(
-            r'<title>\n90tv.ir :: (.*?)</title>', webpage, 'title')
+        title = remove_start(self._html_search_regex(
+            r'<title>([^<]+)</title>', webpage, 'title'), '90tv.ir :: ')
 
         video_url = self._search_regex(
             r'<source[^>]+src="([^"]+)"', webpage, 'video url')

From 7523647391969f8d747ba0fc178592f7f3d5e453 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 25 Jul 2015 18:43:07 +0800
Subject: [PATCH 2090/6219] [ir90tv] PEP8

---
 youtube_dl/extractor/ir90tv.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/ir90tv.py b/youtube_dl/extractor/ir90tv.py
index 92333c3ea..6418d6178 100644
--- a/youtube_dl/extractor/ir90tv.py
+++ b/youtube_dl/extractor/ir90tv.py
@@ -37,6 +37,6 @@ class Ir90TvIE(InfoExtractor):
             'url': video_url,
             'id': video_id,
             'title': title,
-            'video_url' : video_url,
-            'thumbnail' : thumbnail,
+            'video_url': video_url,
+            'thumbnail': thumbnail,
         }

From 2c7c721933e53ece49bee0140d2dad9a8219d6e4 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 25 Jul 2015 18:48:00 +0800
Subject: [PATCH 2091/6219] [ir90tv] Optional fields should be non-fatal

---
 youtube_dl/extractor/ir90tv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ir90tv.py b/youtube_dl/extractor/ir90tv.py
index 6418d6178..214bcd5b5 100644
--- a/youtube_dl/extractor/ir90tv.py
+++ b/youtube_dl/extractor/ir90tv.py
@@ -31,7 +31,7 @@ class Ir90TvIE(InfoExtractor):
         video_url = self._search_regex(
             r'<source[^>]+src="([^"]+)"', webpage, 'video url')
 
-        thumbnail = self._search_regex(r'poster="([^"]+)"', webpage, 'thumbnail url')
+        thumbnail = self._search_regex(r'poster="([^"]+)"', webpage, 'thumbnail url', fatal=False)
 
         return {
             'url': video_url,

From 9afa1770d1a6835bc8fee48dc86cd1a702d1f67a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 25 Jul 2015 20:21:42 +0600
Subject: [PATCH 2092/6219] [bbc] Improve playlist extraction, refactor, expand
 support and document

---
 youtube_dl/extractor/__init__.py |   5 +-
 youtube_dl/extractor/bbc.py      | 375 +++++++++++++++++++++----------
 2 files changed, 259 insertions(+), 121 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index bc61cbdc5..d77ed3ba2 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -43,7 +43,10 @@ from .azubu import AzubuIE
 from .baidu import BaiduVideoIE
 from .bambuser import BambuserIE, BambuserChannelIE
 from .bandcamp import BandcampIE, BandcampAlbumIE
-from .bbc import BBCCoUkIE, BBCNewsIE
+from .bbc import (
+    BBCCoUkIE,
+    BBCIE,
+)
 from .beeg import BeegIE
 from .behindkink import BehindKinkIE
 from .beatportpro import BeatportProIE
diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 86327d8ed..2a0901ee4 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -1,15 +1,18 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
+import re
 import xml.etree.ElementTree
 
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
-    parse_duration,
+    float_or_none,
     int_or_none,
+    parse_duration,
+    parse_iso8601,
 )
 from ..compat import compat_HTTPError
-import re
 
 
 class BBCCoUkIE(InfoExtractor):
@@ -17,7 +20,7 @@ class BBCCoUkIE(InfoExtractor):
     IE_DESC = 'BBC iPlayer'
     _VALID_URL = r'https?://(?:www\.)?bbc\.co\.uk/(?:(?:(?:programmes|iplayer(?:/[^/]+)?/(?:episode|playlist))/)|music/clips[/#])(?P<id>[\da-z]{8})'
 
-    mediaselector_url = 'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/pc/vpid/%s'
+    _MEDIASELECTOR_URL = 'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/pc/vpid/%s'
 
     _TESTS = [
         {
@@ -264,16 +267,21 @@ class BBCCoUkIE(InfoExtractor):
         return subtitles
 
     def _download_media_selector(self, programme_id):
+        return self._download_media_selector_url(
+            self._MEDIASELECTOR_URL % programme_id, programme_id)
+
+    def _download_media_selector_url(self, url, programme_id=None):
         try:
             media_selection = self._download_xml(
-                self.mediaselector_url % programme_id,
-                programme_id, 'Downloading media selection XML')
+                url, programme_id, 'Downloading media selection XML')
         except ExtractorError as ee:
             if isinstance(ee.cause, compat_HTTPError) and ee.cause.code == 403:
                 media_selection = xml.etree.ElementTree.fromstring(ee.cause.read().decode('utf-8'))
             else:
                 raise
+        return self._process_media_selector(media_selection, programme_id)
 
+    def _process_media_selector(self, media_selection, programme_id):
         formats = []
         subtitles = None
 
@@ -312,10 +320,21 @@ class BBCCoUkIE(InfoExtractor):
                 raise
 
         # fallback to legacy playlist
-        playlist = self._download_xml(
-            'http://www.bbc.co.uk/iplayer/playlist/%s' % playlist_id,
-            playlist_id, 'Downloading legacy playlist XML')
+        return self._process_legacy_playlist(playlist_id)
 
+    def _process_legacy_playlist_url(self, url, display_id):
+        playlist = self._download_legacy_playlist_url(url, display_id)
+        return self._extract_from_legacy_playlist(playlist, display_id)
+
+    def _process_legacy_playlist(self, playlist_id):
+        return self._process_legacy_playlist_url(
+            'http://www.bbc.co.uk/iplayer/playlist/%s' % playlist_id, playlist_id)
+
+    def _download_legacy_playlist_url(self, url, playlist_id=None):
+        return self._download_xml(
+            url, playlist_id, 'Downloading legacy playlist XML')
+
+    def _extract_from_legacy_playlist(self, playlist, playlist_id):
         no_items = playlist.find('./{http://bbc.co.uk/2008/emp/playlist}noItems')
         if no_items is not None:
             reason = no_items.get('reason')
@@ -335,8 +354,23 @@ class BBCCoUkIE(InfoExtractor):
                 continue
             title = playlist.find('./{http://bbc.co.uk/2008/emp/playlist}title').text
             description = playlist.find('./{http://bbc.co.uk/2008/emp/playlist}summary').text
-            programme_id = item.get('identifier')
+
+            def get_programme_id(item):
+                def get_from_attributes(item):
+                    for p in('identifier', 'group'):
+                        value = item.get(p)
+                        if value and re.match(r'^[pb][\da-z]{7}$', value):
+                            return value
+                get_from_attributes(item)
+                mediator = item.find('./{http://bbc.co.uk/2008/emp/playlist}mediator')
+                if mediator is not None:
+                    return get_from_attributes(mediator)
+
+            programme_id = get_programme_id(item)
             duration = int_or_none(item.get('duration'))
+            # TODO: programme_id can be None and media items can be incorporated right inside
+            # playlist's item (e.g. http://www.bbc.com/turkce/haberler/2015/06/150615_telabyad_kentin_cogu)
+            # as f4m and m3u8
             formats, subtitles = self._download_media_selector(programme_id)
 
         return programme_id, title, description, duration, formats, subtitles
@@ -383,175 +417,276 @@ class BBCCoUkIE(InfoExtractor):
         }
 
 
-class BBCNewsIE(BBCCoUkIE):
-    IE_NAME = 'bbc.com'
-    IE_DESC = 'BBC news'
-    _VALID_URL = r'https?://(?:www\.)?bbc\.com/.+?/(?P<id>[^/]+)$'
+class BBCIE(BBCCoUkIE):
+    IE_NAME = 'bbc'
+    IE_DESC = 'BBC'
+    _VALID_URL = r'https?://(?:www\.)?bbc\.(?:com|co\.uk)/(?:[^/]+/)+(?P<id>[^/#?]+)'
 
-    mediaselector_url = 'http://open.live.bbc.co.uk/mediaselector/4/mtis/stream/%s'
+    # fails with notukerror for some videos
+    #_MEDIASELECTOR_URL = 'http://open.live.bbc.co.uk/mediaselector/4/mtis/stream/%s'
+    _MEDIASELECTOR_URL = 'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/journalism-pc/vpid/%s'
 
     _TESTS = [{
+        # article with multiple videos embedded with data-media-meta containing
+        # playlist.sxml, externalId and no direct video links
         'url': 'http://www.bbc.com/news/world-europe-32668511',
         'info_dict': {
             'id': 'world-europe-32668511',
             'title': 'Russia stages massive WW2 parade despite Western boycott',
+            'description': 'md5:00ff61976f6081841f759a08bf78cc9c',
         },
         'playlist_count': 2,
     }, {
+        # article with multiple videos embedded with data-media-meta (more videos)
         'url': 'http://www.bbc.com/news/business-28299555',
         'info_dict': {
             'id': 'business-28299555',
             'title': 'Farnborough Airshow: Video highlights',
+            'description': 'BBC reports and video highlights at the Farnborough Airshow.',
         },
         'playlist_count': 9,
+        'skip': 'Save time',
     }, {
+        # single video embedded with mediaAssetPage.init()
         'url': 'http://www.bbc.com/news/world-europe-32041533',
-        'note': 'Video',
         'info_dict': {
             'id': 'p02mprgb',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': 'Aerial footage showed the site of the crash in the Alps - courtesy BFM TV',
-            'description': 'Germanwings plane crash site in aerial video - Aerial footage showed the site of the crash in the Alps - courtesy BFM TV',
             'duration': 47,
+            'timestamp': 1427219242,
             'upload_date': '20150324',
-            'uploader': 'BBC News',
         },
         'params': {
+            # rtmp download
             'skip_download': True,
         }
     }, {
+        # article with single video embedded with data-media-meta containing
+        # direct video links (for now these are extracted) and playlist.xml (with
+        # media items as f4m and m3u8 - currently unsupported)
         'url': 'http://www.bbc.com/turkce/haberler/2015/06/150615_telabyad_kentin_cogu',
-        'note': 'Video',
         'info_dict': {
-            'id': 'NA',
+            'id': '150615_telabyad_kentin_cogu',
             'ext': 'mp4',
-            'title': 'YPG: Tel Abyad\'\u0131n tamam\u0131 kontrol\xfcm\xfczde',
-            'description': 'YPG: Tel Abyad\'\u0131n tamam\u0131 kontrol\xfcm\xfczde',
+            'title': "YPG: Tel Abyad'ın tamamı kontrolümüzde",
             'duration': 47,
+            'timestamp': 1434397334,
             'upload_date': '20150615',
-            'uploader': 'BBC News',
         },
         'params': {
             'skip_download': True,
         }
     }, {
+        # single video embedded with mediaAssetPage.init() (regional section)
         'url': 'http://www.bbc.com/mundo/video_fotos/2015/06/150619_video_honduras_militares_hospitales_corrupcion_aw',
-        'note': 'Video',
         'info_dict': {
-            'id': '39275083',
+            'id': '150619_video_honduras_militares_hospitales_corrupcion_aw',
             'ext': 'mp4',
-            'title': 'Honduras militariza sus hospitales por nuevo esc\xe1ndalo de corrupci\xf3n',
-            'description': 'Honduras militariza sus hospitales por nuevo esc\xe1ndalo de corrupci\xf3n',
+            'title': 'Honduras militariza sus hospitales por nuevo escándalo de corrupción',
             'duration': 87,
+            'timestamp': 1434713142,
             'upload_date': '20150619',
-            'uploader': 'BBC News',
         },
         'params': {
             'skip_download': True,
         }
+    }, {
+        # single video story with digitalData
+        'url': 'http://www.bbc.com/travel/story/20150625-sri-lankas-spicy-secret',
+        'info_dict': {
+            'id': 'p02q6gc4',
+            'ext': 'flv',
+            'title': 'Sri Lanka’s spicy secret',
+            'description': 'As a new train line to Jaffna opens up the country’s north, travellers can experience a truly distinct slice of Tamil culture.',
+            'timestamp': 1437674293,
+            'upload_date': '20150723',
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        }
+    }, {
+        # single video story without digitalData
+        'url': 'http://www.bbc.com/autos/story/20130513-hyundais-rock-star',
+        'info_dict': {
+            'id': 'p018zqqg',
+            'ext': 'flv',
+            'title': 'Hyundai Santa Fe Sport: Rock star',
+            'description': 'md5:b042a26142c4154a6e472933cf20793d',
+            'timestamp': 1368473503,
+            'upload_date': '20130513',
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        }
+    }, {
+        # single video with playlist.sxml URL
+        'url': 'http://www.bbc.com/sport/0/football/33653409',
+        'info_dict': {
+            'id': 'p02xycnp',
+            'ext': 'flv',
+            'title': 'Transfers: Cristiano Ronaldo to Man Utd, Arsenal to spend?',
+            'description': 'md5:398fca0e2e701c609d726e034fa1fc89',
+            'duration': 140,
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        }
+    }, {
+        # single video with playlist URL from weather section
+        'url': 'http://www.bbc.com/weather/features/33601775',
+        'only_matching': True,
+    }, {
+        # custom redirection to www.bbc.com
+        'url': 'http://www.bbc.co.uk/news/science-environment-33661876',
+        'only_matching': True,
     }]
 
+    @classmethod
+    def suitable(cls, url):
+        return False if BBCCoUkIE.suitable(url) else super(BBCIE, cls).suitable(url)
+
+    def _extract_from_media_meta(self, media_meta, video_id):
+        # Direct links to media in media metadata (e.g.
+        # http://www.bbc.com/turkce/haberler/2015/06/150615_telabyad_kentin_cogu)
+        # TODO: there are also f4m and m3u8 streams incorporated in playlist.sxml
+        source_files = media_meta.get('sourceFiles')
+        if source_files:
+            return [{
+                'url': f['url'],
+                'format_id': format_id,
+                'ext': f.get('encoding'),
+                'tbr': float_or_none(f.get('bitrate'), 1000),
+                'filesize': int_or_none(f.get('filesize')),
+            } for format_id, f in source_files.items() if f.get('url')], []
+
+        programme_id = media_meta.get('externalId')
+        if programme_id:
+            return self._download_media_selector(programme_id)
+
+        # Process playlist.sxml as legacy playlist
+        href = media_meta.get('href')
+        if href:
+            playlist = self._download_legacy_playlist_url(href)
+            _, _, _, _, formats, subtitles = self._extract_from_legacy_playlist(playlist, video_id)
+            return formats, subtitles
+
+        return [], []
+
     def _real_extract(self, url):
-        list_id = self._match_id(url)
-        webpage = self._download_webpage(url, list_id)
+        playlist_id = self._match_id(url)
 
-        list_title = self._html_search_regex(r'<title>(.*?)(?:\s*-\s*BBC [^ ]+)?</title>', webpage, 'list title')
+        webpage = self._download_webpage(url, playlist_id)
 
-        pubdate = self._html_search_regex(r'"datePublished":\s*"(\d+-\d+-\d+)', webpage, 'date', default=None)
-        if pubdate:
-            pubdate = pubdate.replace('-', '')
-
-        ret = []
-        jsent = []
-
-        # works with bbc.com/news/something-something-123456 articles
-        jsent = map(
-            lambda m: self._parse_json(m, list_id),
-            re.findall(r"data-media-meta='({[^']+})'", webpage)
-        )
-
-        if len(jsent) == 0:
-            # http://www.bbc.com/news/video_and_audio/international
-            # and single-video articles
-            masset = self._html_search_regex(r'mediaAssetPage\.init\(\s*({.+?}), "/', webpage, 'mediaassets', default=None)
-            if masset:
-                jmasset = self._parse_json(masset, list_id)
-                for key, val in jmasset.get('videos', {}).items():
-                    for skey, sval in val.items():
-                        sval['id'] = skey
-                        jsent.append(sval)
-
-        if len(jsent) == 0:
-            # stubbornly generic extractor for {json with "image":{allvideoshavethis},etc}
-            # in http://www.bbc.com/news/video_and_audio/international
-            # prone to breaking if entries have sourceFiles list
-            jsent = map(
-                lambda m: self._parse_json(m, list_id),
-                re.findall(r"({[^{}]+image\":{[^}]+}[^}]+})", webpage)
-            )
-
-        if len(jsent) == 0:
-            raise ExtractorError('No video found', expected=True)
-
-        for jent in jsent:
-            programme_id = jent.get('externalId')
-            xml_url = jent.get('href')
-
-            title = jent.get('caption', '')
-            if title == '':
-                title = list_title
-
-            duration = parse_duration(jent.get('duration'))
-            description = list_title
-            if jent.get('caption', '') != '':
-                description += ' - ' + jent.get('caption')
-            thumbnail = None
-            if jent.get('image') is not None:
-                thumbnail = jent['image'].get('href')
-
-            formats = []
-            subtitles = []
-
-            if programme_id:
-                formats, subtitles = self._download_media_selector(programme_id)
-            elif jent.get('sourceFiles') is not None:
-                # mediaselector not used at
-                # http://www.bbc.com/turkce/haberler/2015/06/150615_telabyad_kentin_cogu
-                for key, val in jent['sourceFiles'].items():
-                    formats.append({
-                        'ext': val.get('encoding'),
-                        'url': val.get('url'),
-                        'filesize': int(val.get('filesize')),
-                        'format_id': key
-                    })
-            elif xml_url:
-                # Cheap fallback
-                # http://playlists.bbc.co.uk/news/(list_id)[ABC..]/playlist.sxml
-                xml = self._download_webpage(xml_url, programme_id, 'Downloading playlist.sxml for externalId (fallback)')
-                programme_id = self._search_regex(r'<mediator [^>]*identifier="(.+?)"', xml, 'playlist.sxml (externalId fallback)')
-                formats, subtitles = self._download_media_selector(programme_id)
-
-            if len(formats) == 0:
-                raise ExtractorError('unsupported json media entry.\n    ' + str(jent) + '\n')
+        timestamp = parse_iso8601(self._search_regex(
+            [r'"datePublished":\s*"([^"]+)',
+             r'<meta[^>]+property="article:published_time"[^>]+content="([^"]+)"',
+             r'itemprop="datePublished"[^>]+datetime="([^"]+)"'],
+            webpage, 'date', default=None))
 
+        # single video with playlist.sxml URL (e.g. http://www.bbc.com/sport/0/football/3365340ng)
+        playlist = self._search_regex(
+            r'<param[^>]+name="playlist"[^>]+value="([^"]+)"',
+            webpage, 'playlist', default=None)
+        if playlist:
+            programme_id, title, description, duration, formats, subtitles = \
+                self._process_legacy_playlist_url(playlist, playlist_id)
             self._sort_formats(formats)
-
-            id = jent.get('id') if programme_id is None else programme_id
-            if id is None:
-                id = 'NA'
-
-            ret.append({
-                'id': id,
-                'uploader': 'BBC News',
-                'upload_date': pubdate,
+            return {
+                'id': programme_id,
                 'title': title,
                 'description': description,
-                'thumbnail': thumbnail,
                 'duration': duration,
+                'timestamp': timestamp,
+                'formats': formats,
+                'subtitles': subtitles,
+            }
+
+        # single video story (e.g. http://www.bbc.com/travel/story/20150625-sri-lankas-spicy-secret)
+        programme_id = self._search_regex(
+            [r'data-video-player-vpid="([\da-z]{8})"',
+             r'<param[^>]+name="externalIdentifier"[^>]+value="([\da-z]{8})"'],
+            webpage, 'vpid', default=None)
+        if programme_id:
+            formats, subtitles = self._download_media_selector(programme_id)
+            self._sort_formats(formats)
+            # digitalData may be missing (e.g. http://www.bbc.com/autos/story/20130513-hyundais-rock-star)
+            digital_data = self._parse_json(
+                self._search_regex(
+                    r'var\s+digitalData\s*=\s*({.+?});?\n', webpage, 'digital data', default='{}'),
+                programme_id, fatal=False)
+            page_info = digital_data.get('page', {}).get('pageInfo', {})
+            title = page_info.get('pageName') or self._og_search_title(webpage)
+            description = page_info.get('description') or self._og_search_description(webpage)
+            timestamp = parse_iso8601(page_info.get('publicationDate')) or timestamp
+            return {
+                'id': programme_id,
+                'title': title,
+                'description': description,
+                'timestamp': timestamp,
+                'formats': formats,
+                'subtitles': subtitles,
+            }
+
+        playlist_title = self._html_search_regex(
+            r'<title>(.*?)(?:\s*-\s*BBC [^ ]+)?</title>', webpage, 'playlist title')
+        playlist_description = self._og_search_description(webpage)
+
+        # Multiple video article (e.g. http://www.bbc.com/news/world-europe-32668511)
+        medias = list(filter(None, map(
+            lambda s: self._parse_json(s, playlist_id, fatal=False),
+            re.findall(r"data-media-meta='({[^']+})'", webpage))))
+
+        if not medias:
+            # Single video article (e.g. http://www.bbc.com/news/video_and_audio/international)
+            media_asset_page = self._parse_json(
+                self._search_regex(
+                    r'mediaAssetPage\.init\(\s*({.+?}), "/', webpage, 'media asset'),
+                playlist_id)
+            medias = []
+            for video in media_asset_page.get('videos', {}).values():
+                medias.extend(video.values())
+
+        entries = []
+        for num, media_meta in enumerate(medias, start=1):
+            formats, subtitles = self._extract_from_media_meta(media_meta, playlist_id)
+            if not formats:
+                continue
+            self._sort_formats(formats)
+
+            video_id = media_meta.get('externalId')
+            if not video_id:
+                video_id = playlist_id if len(medias) == 1 else '%s-%s' % (playlist_id, num)
+
+            title = media_meta.get('caption')
+            if not title:
+                title = playlist_title if len(medias) == 1 else '%s - Video %s' % (playlist_title, num)
+
+            duration = int_or_none(media_meta.get('durationInSeconds')) or parse_duration(media_meta.get('duration'))
+
+            images = []
+            for image in media_meta.get('images', {}).values():
+                images.extend(image.values())
+            if 'image' in media_meta:
+                images.append(media_meta['image'])
+
+            thumbnails = [{
+                'url': image.get('href'),
+                'width': int_or_none(image.get('width')),
+                'height': int_or_none(image.get('height')),
+            } for image in images]
+
+            entries.append({
+                'id': video_id,
+                'title': title,
+                'thumbnails': thumbnails,
+                'duration': duration,
+                'timestamp': timestamp,
                 'formats': formats,
                 'subtitles': subtitles,
             })
 
-        if len(ret) > 0:
-            return self.playlist_result(ret, list_id, list_title)
-        raise ExtractorError('No video found', expected=True)
+        return self.playlist_result(entries, playlist_id, playlist_title, playlist_description)

From cf7e015f250d806916bed1ed2a1dfd8d943c05c2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 25 Jul 2015 21:30:34 +0600
Subject: [PATCH 2093/6219] [youtube] Add support for multifeed videos

---
 youtube_dl/extractor/youtube.py | 126 +++++++++++++++++++++++++-------
 1 file changed, 99 insertions(+), 27 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 4023a6e50..afe0a781b 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -33,9 +33,11 @@ from ..utils import (
     int_or_none,
     orderedSet,
     parse_duration,
+    smuggle_url,
     str_to_int,
     unescapeHTML,
     unified_strdate,
+    unsmuggle_url,
     uppercase_escape,
     ISO3166Utils,
 )
@@ -558,6 +560,59 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'format': '135',  # bestvideo
             }
         },
+        {
+            # Multifeed videos (multiple cameras), URL is for Main Camera
+            'url': 'https://www.youtube.com/watch?v=jqWvoWXjCVs',
+            'info_dict': {
+                'id': 'jqWvoWXjCVs',
+                'title': 'teamPGP: Rocket League Noob Stream',
+                'description': 'md5:dc7872fb300e143831327f1bae3af010',
+            },
+            'playlist': [{
+                'info_dict': {
+                    'id': 'jqWvoWXjCVs',
+                    'ext': 'mp4',
+                    'title': 'teamPGP: Rocket League Noob Stream (Main Camera)',
+                    'description': 'md5:dc7872fb300e143831327f1bae3af010',
+                    'upload_date': '20150721',
+                    'uploader': 'Beer Games Beer',
+                    'uploader_id': 'beergamesbeer',
+                },
+            }, {
+                'info_dict': {
+                    'id': '6h8e8xoXJzg',
+                    'ext': 'mp4',
+                    'title': 'teamPGP: Rocket League Noob Stream (kreestuh)',
+                    'description': 'md5:dc7872fb300e143831327f1bae3af010',
+                    'upload_date': '20150721',
+                    'uploader': 'Beer Games Beer',
+                    'uploader_id': 'beergamesbeer',
+                },
+            }, {
+                'info_dict': {
+                    'id': 'PUOgX5z9xZw',
+                    'ext': 'mp4',
+                    'title': 'teamPGP: Rocket League Noob Stream (grizzle)',
+                    'description': 'md5:dc7872fb300e143831327f1bae3af010',
+                    'upload_date': '20150721',
+                    'uploader': 'Beer Games Beer',
+                    'uploader_id': 'beergamesbeer',
+                },
+            }, {
+                'info_dict': {
+                    'id': 'teuwxikvS5k',
+                    'ext': 'mp4',
+                    'title': 'teamPGP: Rocket League Noob Stream (zim)',
+                    'description': 'md5:dc7872fb300e143831327f1bae3af010',
+                    'upload_date': '20150721',
+                    'uploader': 'Beer Games Beer',
+                    'uploader_id': 'beergamesbeer',
+                },
+            }],
+            'params': {
+                'skip_download': True,
+            },
+        }
     ]
 
     def __init__(self, *args, **kwargs):
@@ -889,6 +944,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         return formats
 
     def _real_extract(self, url):
+        url, smuggled_data = unsmuggle_url(url, {})
+
         proto = (
             'http' if self._downloader.params.get('prefer_insecure', False)
             else 'https')
@@ -1005,6 +1062,48 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     '"token" parameter not in video info for unknown reason',
                     video_id=video_id)
 
+        # title
+        if 'title' in video_info:
+            video_title = video_info['title'][0]
+        else:
+            self._downloader.report_warning('Unable to extract video title')
+            video_title = '_'
+
+        # description
+        video_description = get_element_by_id("eow-description", video_webpage)
+        if video_description:
+            video_description = re.sub(r'''(?x)
+                <a\s+
+                    (?:[a-zA-Z-]+="[^"]+"\s+)*?
+                    title="([^"]+)"\s+
+                    (?:[a-zA-Z-]+="[^"]+"\s+)*?
+                    class="yt-uix-redirect-link"\s*>
+                [^<]+
+                </a>
+            ''', r'\1', video_description)
+            video_description = clean_html(video_description)
+        else:
+            fd_mobj = re.search(r'<meta name="description" content="([^"]+)"', video_webpage)
+            if fd_mobj:
+                video_description = unescapeHTML(fd_mobj.group(1))
+            else:
+                video_description = ''
+
+        if not smuggled_data.get('force_singlefeed', False) and 'multifeed_metadata_list' in video_info:
+            entries = []
+            multifeed_metadata_list = compat_urllib_parse_unquote_plus(video_info['multifeed_metadata_list'][0])
+            for feed in multifeed_metadata_list.split(','):
+                feed_data = compat_parse_qs(feed)
+                entries.append({
+                    '_type': 'url_transparent',
+                    'ie_key': 'Youtube',
+                    'url': smuggle_url(
+                        '%s://www.youtube.com/watch?v=%s' % (proto, feed_data['id'][0]),
+                        {'force_singlefeed': True}),
+                    'title': '%s (%s)' % (video_title, feed_data['title'][0]),
+                })
+            return self.playlist_result(entries, video_id, video_title, video_description)
+
         if 'view_count' in video_info:
             view_count = int(video_info['view_count'][0])
         else:
@@ -1030,13 +1129,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         else:
             self._downloader.report_warning('unable to extract uploader nickname')
 
-        # title
-        if 'title' in video_info:
-            video_title = video_info['title'][0]
-        else:
-            self._downloader.report_warning('Unable to extract video title')
-            video_title = '_'
-
         # thumbnail image
         # We try first to get a high quality image:
         m_thumb = re.search(r'<span itemprop="thumbnail".*?href="(.*?)">',
@@ -1072,26 +1164,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         else:
             video_categories = None
 
-        # description
-        video_description = get_element_by_id("eow-description", video_webpage)
-        if video_description:
-            video_description = re.sub(r'''(?x)
-                <a\s+
-                    (?:[a-zA-Z-]+="[^"]+"\s+)*?
-                    title="([^"]+)"\s+
-                    (?:[a-zA-Z-]+="[^"]+"\s+)*?
-                    class="yt-uix-redirect-link"\s*>
-                [^<]+
-                </a>
-            ''', r'\1', video_description)
-            video_description = clean_html(video_description)
-        else:
-            fd_mobj = re.search(r'<meta name="description" content="([^"]+)"', video_webpage)
-            if fd_mobj:
-                video_description = unescapeHTML(fd_mobj.group(1))
-            else:
-                video_description = ''
-
         def _extract_count(count_name):
             return str_to_int(self._search_regex(
                 r'-%s-button[^>]+><span[^>]+class="yt-uix-button-content"[^>]*>([\d,]+)</span>'

From d8f0a9ecea6aab5b148d06904934c6a504ba3b03 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 25 Jul 2015 22:17:06 +0600
Subject: [PATCH 2094/6219] [youtube] Respect noplaylist for multifeed videos

---
 youtube_dl/extractor/youtube.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index afe0a781b..bcd27408d 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1089,7 +1089,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             else:
                 video_description = ''
 
-        if not smuggled_data.get('force_singlefeed', False) and 'multifeed_metadata_list' in video_info:
+        if (not self._downloader.params.get('noplaylist') and
+                not smuggled_data.get('force_singlefeed', False) and
+                'multifeed_metadata_list' in video_info):
             entries = []
             multifeed_metadata_list = compat_urllib_parse_unquote_plus(video_info['multifeed_metadata_list'][0])
             for feed in multifeed_metadata_list.split(','):

From 51da40e6218f1dda2fc61650c308194e9b4acbc2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 25 Jul 2015 22:19:54 +0600
Subject: [PATCH 2095/6219] [bbc] PEP8

---
 youtube_dl/extractor/bbc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 2a0901ee4..4b23f82ca 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -423,7 +423,7 @@ class BBCIE(BBCCoUkIE):
     _VALID_URL = r'https?://(?:www\.)?bbc\.(?:com|co\.uk)/(?:[^/]+/)+(?P<id>[^/#?]+)'
 
     # fails with notukerror for some videos
-    #_MEDIASELECTOR_URL = 'http://open.live.bbc.co.uk/mediaselector/4/mtis/stream/%s'
+    # _MEDIASELECTOR_URL = 'http://open.live.bbc.co.uk/mediaselector/4/mtis/stream/%s'
     _MEDIASELECTOR_URL = 'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/journalism-pc/vpid/%s'
 
     _TESTS = [{

From 7a896817226405a772baa3808d63062d4ad11c94 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 25 Jul 2015 22:32:54 +0600
Subject: [PATCH 2096/6219] [bbc] Skip DASH until supported

---
 youtube_dl/extractor/bbc.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 4b23f82ca..66e52641b 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -172,6 +172,7 @@ class BBCCoUkIE(InfoExtractor):
         supplier = connection.get('supplier')
         if protocol == 'http':
             href = connection.get('href')
+            transfer_format = connection.get('transferFormat')
             # ASX playlist
             if supplier == 'asx':
                 for i, ref in enumerate(self._extract_asx_playlist(connection, programme_id)):
@@ -179,6 +180,9 @@ class BBCCoUkIE(InfoExtractor):
                         'url': ref,
                         'format_id': 'ref%s_%s' % (i, supplier),
                     })
+            # Skip DASH until supported
+            elif transfer_format == 'dash':
+                pass
             # Direct link
             else:
                 formats.append({

From a62fd1af275a125b77d782e821724d6c6c896b69 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 25 Jul 2015 18:13:44 +0100
Subject: [PATCH 2097/6219] [shahid] improve info extraction

---
 youtube_dl/extractor/shahid.py | 42 +++++++++++++++++++++++++---------
 1 file changed, 31 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/shahid.py b/youtube_dl/extractor/shahid.py
index 04de866f4..da3141f11 100644
--- a/youtube_dl/extractor/shahid.py
+++ b/youtube_dl/extractor/shahid.py
@@ -1,7 +1,8 @@
 from .common import InfoExtractor
 from ..utils import (
-    get_element_by_id,
+    js_to_json,
     ExtractorError,
+    int_or_none
 )
 
 class ShahidIE(InfoExtractor):
@@ -12,8 +13,9 @@ class ShahidIE(InfoExtractor):
             'info_dict': {
                 'id': '108084',
                 'ext': 'm3u8',
-                'title': 'بسم الله',
-                'description': 'بسم الله'
+                'title': 'خواطر الموسم 11 الحلقة 1',
+                'description': 'بسم الله',
+                'duration': 1166,
             },
             'params': {
                 # m3u8 download
@@ -30,16 +32,33 @@ class ShahidIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
-        json_data = self._parse_json(
-            get_element_by_id('jsonld', webpage),
+        player_info = ''
+        for line in self._search_regex( 'var flashvars = ({[^}]+})', webpage, 'flashvars').splitlines():
+            if '+' not in line and '(' not in line and ')' not in line:
+                player_info += line
+        player_info = self._parse_json(js_to_json(player_info), video_id)
+        video_id = player_info['id']
+        player_type = player_info['playerType']
+
+        video_info = self._download_json(
+            player_info['url'] + '/' + player_type + '/' + video_id + 
+                '?apiKey=sh%40hid0nlin3&hash=b2wMCTHpSmyxGqQjJFOycRmLSex%2BBpTK%2Fooxy6vHaqs%3D',
             video_id
-        )
-        title = json_data['name']
-        thumbnail = json_data.get('image')
-        categories = json_data.get('genre')
-        description = json_data.get('description')
+        )['data']
+        if video_info['error']:
+            for error in video_info['error']:
+                raise ExtractorError(error)
+        video_info = video_info[player_type]
+        if video_info.get('availabilities').get('plus'):
+            raise ExtractorError('plus members only')
+        title = video_info['title']
+        thumbnail = video_info.get('thumbnailUrl')
+        categories = [category['name'] for category in video_info.get('genres')]
+        description = video_info.get('description')
+        duration = int_or_none(video_info.get('duration'))
+
         player_json_data = self._download_json(
-            'https://shahid.mbc.net/arContent/getPlayerContent-param-.id-'+video_id+'.type-player.html',
+            'https://shahid.mbc.net/arContent/getPlayerContent-param-.id-' + video_id + '.type-' + player_info['type'] + '.html',
             video_id
         )['data']
         if 'url' in player_json_data:
@@ -55,5 +74,6 @@ class ShahidIE(InfoExtractor):
             'thumbnail': thumbnail,
             'categories': categories,
             'description': description,
+            'duration': duration,
             'formats': formats,
         }

From dfaba1ab9586a7c47017bd897ffda0b795a1d27e Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 25 Jul 2015 19:14:18 +0100
Subject: [PATCH 2098/6219] [shahid] fix variable name

---
 youtube_dl/extractor/shahid.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/shahid.py b/youtube_dl/extractor/shahid.py
index da3141f11..b3b45da24 100644
--- a/youtube_dl/extractor/shahid.py
+++ b/youtube_dl/extractor/shahid.py
@@ -64,7 +64,7 @@ class ShahidIE(InfoExtractor):
         if 'url' in player_json_data:
             m3u8_url = player_json_data['url']
         else:
-            for error in json_data['error'].values():
+            for error in player_json_data['error'].values():
                 raise ExtractorError(error)
             return
         formats = self._extract_m3u8_formats(m3u8_url, video_id)

From 5bdec59de15b9bde73a3077a6b9ce517c10b9906 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 26 Jul 2015 09:51:54 +0600
Subject: [PATCH 2099/6219] [comcarcoff] Add support for singleshots (Closes
 #6366)

---
 youtube_dl/extractor/comcarcoff.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/comcarcoff.py b/youtube_dl/extractor/comcarcoff.py
index 9c25b2223..81f3d7697 100644
--- a/youtube_dl/extractor/comcarcoff.py
+++ b/youtube_dl/extractor/comcarcoff.py
@@ -36,7 +36,7 @@ class ComCarCoffIE(InfoExtractor):
             webpage, 'full data json'))
 
         video_id = full_data['activeVideo']['video']
-        video_data = full_data['videos'][video_id]
+        video_data = full_data.get('videos', {}).get(video_id) or full_data['singleshots'][video_id]
         thumbnails = [{
             'url': video_data['images']['thumb'],
         }, {

From aeb7b41d44313b6bb007b2f1cd0fc8cda84e59d5 Mon Sep 17 00:00:00 2001
From: tippfeler <tippfeler@gmx.de>
Date: Sun, 26 Jul 2015 12:57:06 +0200
Subject: [PATCH 2100/6219] [spiegel] Accept iframe urls

Closes #6370.
---
 youtube_dl/extractor/spiegel.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/spiegel.py b/youtube_dl/extractor/spiegel.py
index b868241d5..5bd3c0087 100644
--- a/youtube_dl/extractor/spiegel.py
+++ b/youtube_dl/extractor/spiegel.py
@@ -9,7 +9,7 @@ from .spiegeltv import SpiegeltvIE
 
 
 class SpiegelIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?spiegel\.de/video/[^/]*-(?P<id>[0-9]+)(?:-embed)?(?:\.html)?(?:#.*)?$'
+    _VALID_URL = r'https?://(?:www\.)?spiegel\.de/video/[^/]*-(?P<id>[0-9]+)(?:-embed|-iframe)?(?:\.html)?(?:#.*)?$'
     _TESTS = [{
         'url': 'http://www.spiegel.de/video/vulkan-tungurahua-in-ecuador-ist-wieder-aktiv-video-1259285.html',
         'md5': '2c2754212136f35fb4b19767d242f66e',
@@ -39,6 +39,9 @@ class SpiegelIE(InfoExtractor):
             'description': 'SPIEGEL ONLINE-Nutzer durften den deutschen Astronauten Alexander Gerst über sein Leben auf der ISS-Station befragen. Hier kommen seine Antworten auf die besten sechs Fragen.',
             'title': 'Fragen an Astronaut Alexander Gerst: "Bekommen Sie die Tageszeiten mit?"',
         }
+    }, {
+        'url': 'http://www.spiegel.de/video/astronaut-alexander-gerst-von-der-iss-station-beantwortet-fragen-video-1519126-iframe.html',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 4c6bd5b5b61adfd912e14f8d704fde47628d164e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 26 Jul 2015 14:14:28 +0200
Subject: [PATCH 2101/6219] [youtube] Use 'vp8' and 'vp9' in lowercase (fixes
 #6358)

That's how YouTube reports them in their DASH manifest.
---
 youtube_dl/extractor/youtube.py | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 4023a6e50..0e411bfb6 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -281,13 +281,13 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'vcodec': 'none', 'abr': 256, 'preference': -50, 'container': 'm4a_dash'},
 
         # Dash webm
-        '167': {'ext': 'webm', 'height': 360, 'width': 640, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'VP8', 'preference': -40},
-        '168': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'VP8', 'preference': -40},
-        '169': {'ext': 'webm', 'height': 720, 'width': 1280, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'VP8', 'preference': -40},
-        '170': {'ext': 'webm', 'height': 1080, 'width': 1920, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'VP8', 'preference': -40},
-        '218': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'VP8', 'preference': -40},
-        '219': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'VP8', 'preference': -40},
-        '278': {'ext': 'webm', 'height': 144, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'container': 'webm', 'vcodec': 'VP9'},
+        '167': {'ext': 'webm', 'height': 360, 'width': 640, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
+        '168': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
+        '169': {'ext': 'webm', 'height': 720, 'width': 1280, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
+        '170': {'ext': 'webm', 'height': 1080, 'width': 1920, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
+        '218': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
+        '219': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
+        '278': {'ext': 'webm', 'height': 144, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'container': 'webm', 'vcodec': 'vp9'},
         '242': {'ext': 'webm', 'height': 240, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
         '243': {'ext': 'webm', 'height': 360, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
         '244': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
@@ -297,11 +297,11 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         '248': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
         '271': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
         '272': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
-        '302': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'VP9'},
-        '303': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'VP9'},
-        '308': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'VP9'},
-        '313': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'vcodec': 'VP9'},
-        '315': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'VP9'},
+        '302': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'vp9'},
+        '303': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'vp9'},
+        '308': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'vp9'},
+        '313': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'vcodec': 'vp9'},
+        '315': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'vp9'},
 
         # Dash webm audio
         '171': {'ext': 'webm', 'vcodec': 'none', 'format_note': 'DASH audio', 'abr': 128, 'preference': -50},

From 2c7ed247967f3563d26879f2206e0f54dfdf9b23 Mon Sep 17 00:00:00 2001
From: Raphael Michel <mail@raphaelmichel.de>
Date: Sun, 26 Jul 2015 16:37:51 +0200
Subject: [PATCH 2102/6219] Remove redundant (and wrong) class parameters

---
 youtube_dl/utils.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index ae813099d..88f9f9070 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -576,11 +576,9 @@ class ContentTooShortError(Exception):
     download is too small for what the server announced first, indicating
     the connection was probably interrupted.
     """
-    # Both in bytes
-    downloaded = None
-    expected = None
 
     def __init__(self, downloaded, expected):
+        # Both in bytes
         self.downloaded = downloaded
         self.expected = expected
 

From cb23bcba294563857561914a19e7d06990c71829 Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Mon, 27 Jul 2015 04:24:04 -0500
Subject: [PATCH 2103/6219] BBCIE: Fix missing .mp4 formats on news sites

Prior to merge BBCNewsIE from pr #6026 was rewritten into BBCIE in 9afa177. Support was added for non-news
sites (/travel, /sports), however support for the news mediaselector was removed to support these sites.
This removed support for .mp4.

    pr #6026 ( news site, news mediaseletor : http://open.live.bbc.co.uk/mediaselector/4/mtis/stream/ )
    format code                                          extension  resolution note
    journalism_nonuk_stream_h264_flv_200_akamai          flv        400x224    h264@ 176k, 1001.77KiB
    journalism_nonuk_stream_h264_flv_400_akamai          flv        400x224    h264@ 512k, 2.85MiB
    journalism_nonuk_stream_h264_flv_med_akamai          flv        640x360    h264@ 800k, 4.43MiB
    journalism_nonuk_stream_h264_flv_hi_akamai           flv        688x384    h264@1500k, 8.34MiB
    journalism_world_stream_h264_http_200_sis_news_http  mp4        400x224    h264@ 176k, 1001.77KiB
    journalism_world_stream_h264_http_400_sis_news_http  mp4        400x224    h264@ 512k, 2.85MiB
    journalism_world_stream_h264_http_hi_sis_news_http   mp4        688x384    h264@1500k, 8.34MiB (best)

    # 9afa177 ( same url, non-news mediaselector: http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/journalism-pc/vpid/ )
    format code                                  extension  resolution note
    journalism_nonuk_stream_h264_flv_lo_akamai   flv        384x216    h264@ 496k, 2.76MiB
    journalism_nonuk_stream_h264_flv_med_akamai  flv        640x360    h264@ 800k, 4.43MiB
    journalism_nonuk_stream_h264_flv_hi_akamai   flv        688x384    h264@1500k, 8.34MiB (best)

This change corrects the above, by trying /mediaselector/5 if /mediaselector/4 fails.
---
 youtube_dl/extractor/bbc.py | 27 +++++++++++++++++----------
 1 file changed, 17 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 66e52641b..0f0ea7cfd 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -14,7 +14,6 @@ from ..utils import (
 )
 from ..compat import compat_HTTPError
 
-
 class BBCCoUkIE(InfoExtractor):
     IE_NAME = 'bbc.co.uk'
     IE_DESC = 'BBC iPlayer'
@@ -271,8 +270,16 @@ class BBCCoUkIE(InfoExtractor):
         return subtitles
 
     def _download_media_selector(self, programme_id):
-        return self._download_media_selector_url(
-            self._MEDIASELECTOR_URL % programme_id, programme_id)
+        try:
+            return self._download_media_selector_url(
+                self._MEDIASELECTOR_URL % programme_id, programme_id)
+        except ExtractorError as e:
+            if hasattr(self, '_MEDIASELECTOR_ALT_URL') and str(e) == 'bbc returned error: notukerror':
+                 # notukerror on bbc.com/travel using bbc news mediaselector: fallback to /mediaselector/5/
+                 return self._download_media_selector_url(
+                     self._MEDIASELECTOR_ALT_URL % programme_id, programme_id)
+            else:
+                 raise
 
     def _download_media_selector_url(self, url, programme_id=None):
         try:
@@ -297,7 +304,6 @@ class BBCCoUkIE(InfoExtractor):
                 formats.extend(self._extract_video(media, programme_id))
             elif kind == 'captions':
                 subtitles = self.extract_subtitles(media, programme_id)
-
         return formats, subtitles
 
     def _download_playlist(self, playlist_id):
@@ -426,9 +432,10 @@ class BBCIE(BBCCoUkIE):
     IE_DESC = 'BBC'
     _VALID_URL = r'https?://(?:www\.)?bbc\.(?:com|co\.uk)/(?:[^/]+/)+(?P<id>[^/#?]+)'
 
-    # fails with notukerror for some videos
-    # _MEDIASELECTOR_URL = 'http://open.live.bbc.co.uk/mediaselector/4/mtis/stream/%s'
-    _MEDIASELECTOR_URL = 'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/journalism-pc/vpid/%s'
+    # fails with notukerror for some videos ( non news sites such as bbc.com/travel )
+    _MEDIASELECTOR_URL = 'http://open.live.bbc.co.uk/mediaselector/4/mtis/stream/%s'
+    # limited selection of formats but may work where the above does not
+    _MEDIASELECTOR_ALT_URL = 'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/journalism-pc/vpid/%s'
 
     _TESTS = [{
         # article with multiple videos embedded with data-media-meta containing
@@ -455,7 +462,7 @@ class BBCIE(BBCCoUkIE):
         'url': 'http://www.bbc.com/news/world-europe-32041533',
         'info_dict': {
             'id': 'p02mprgb',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Aerial footage showed the site of the crash in the Alps - courtesy BFM TV',
             'duration': 47,
             'timestamp': 1427219242,
@@ -515,7 +522,7 @@ class BBCIE(BBCCoUkIE):
         'url': 'http://www.bbc.com/autos/story/20130513-hyundais-rock-star',
         'info_dict': {
             'id': 'p018zqqg',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Hyundai Santa Fe Sport: Rock star',
             'description': 'md5:b042a26142c4154a6e472933cf20793d',
             'timestamp': 1368473503,
@@ -530,7 +537,7 @@ class BBCIE(BBCCoUkIE):
         'url': 'http://www.bbc.com/sport/0/football/33653409',
         'info_dict': {
             'id': 'p02xycnp',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Transfers: Cristiano Ronaldo to Man Utd, Arsenal to spend?',
             'description': 'md5:398fca0e2e701c609d726e034fa1fc89',
             'duration': 140,

From 88ed52aec9f2e622188f304d74f2f5568b0caa1e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 27 Jul 2015 22:05:51 +0600
Subject: [PATCH 2104/6219] [bbc] Add support for direct bbc.co.uk embeds

---
 youtube_dl/extractor/bbc.py     | 34 +++++++++++++++++++++++++++++----
 youtube_dl/extractor/generic.py |  8 --------
 2 files changed, 30 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 66e52641b..c0433eabd 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -450,6 +450,14 @@ class BBCIE(BBCCoUkIE):
         },
         'playlist_count': 9,
         'skip': 'Save time',
+    }, {
+        # article with multiple videos embedded with `new SMP()`
+        'url': 'http://www.bbc.co.uk/blogs/adamcurtis/entries/3662a707-0af9-3149-963f-47bea720b460',
+        'info_dict': {
+            'id': '3662a707-0af9-3149-963f-47bea720b460',
+            'title': 'BBC Blogs - Adam Curtis - BUGGER',
+        },
+        'playlist_count': 18,
     }, {
         # single video embedded with mediaAssetPage.init()
         'url': 'http://www.bbc.com/news/world-europe-32041533',
@@ -637,12 +645,30 @@ class BBCIE(BBCCoUkIE):
 
         playlist_title = self._html_search_regex(
             r'<title>(.*?)(?:\s*-\s*BBC [^ ]+)?</title>', webpage, 'playlist title')
-        playlist_description = self._og_search_description(webpage)
+        playlist_description = self._og_search_description(webpage, default=None)
+
+        def extract_all(pattern):
+            return list(filter(None, map(
+                lambda s: self._parse_json(s, playlist_id, fatal=False),
+                re.findall(pattern, webpage))))
+
+        # Multiple video article (e.g.
+        # http://www.bbc.co.uk/blogs/adamcurtis/entries/3662a707-0af9-3149-963f-47bea720b460)
+        EMBED_URL = r'https?://(?:www\.)?bbc\.co\.uk/(?:[^/]+/)+[\da-z]{8}(?:\b[^"]*)?'
+        entries = []
+        for match in extract_all(r'new\s+SMP\(({.+?})\)'):
+            embed_url = match.get('playerSettings', {}).get('externalEmbedUrl')
+            if embed_url and re.match(EMBED_URL, embed_url):
+                entries.append(embed_url)
+        entries.extend(re.findall(
+            r'setPlaylist\("(%s)"\)' % EMBED_URL, webpage))
+        if entries:
+            return self.playlist_result(
+                [self.url_result(entry, 'BBCCoUk') for entry in entries],
+                playlist_id, playlist_title, playlist_description)
 
         # Multiple video article (e.g. http://www.bbc.com/news/world-europe-32668511)
-        medias = list(filter(None, map(
-            lambda s: self._parse_json(s, playlist_id, fatal=False),
-            re.findall(r"data-media-meta='({[^']+})'", webpage))))
+        medias = extract_all(r"data-media-meta='({[^']+})'")
 
         if not medias:
             # Single video article (e.g. http://www.bbc.com/news/video_and_audio/international)
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 6d2efb22e..8cef61c3c 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -276,14 +276,6 @@ class GenericIE(InfoExtractor):
                 'description': 'Episode 18: President Barack Obama sits down with Zach Galifianakis for his most memorable interview yet.',
             },
         },
-        # BBC iPlayer embeds
-        {
-            'url': 'http://www.bbc.co.uk/blogs/adamcurtis/posts/BUGGER',
-            'info_dict': {
-                'title': 'BBC - Blogs -  Adam Curtis - BUGGER',
-            },
-            'playlist_mincount': 18,
-        },
         # RUTV embed
         {
             'url': 'http://www.rg.ru/2014/03/15/reg-dfo/anklav-anons.html',

From d247a2c8bf2fe4e071cb35a2e2777fe16acecab3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 28 Jul 2015 02:06:27 +0600
Subject: [PATCH 2105/6219] [bbc] Fix regex

---
 youtube_dl/extractor/bbc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index c0433eabd..01d07c9c0 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -654,7 +654,7 @@ class BBCIE(BBCCoUkIE):
 
         # Multiple video article (e.g.
         # http://www.bbc.co.uk/blogs/adamcurtis/entries/3662a707-0af9-3149-963f-47bea720b460)
-        EMBED_URL = r'https?://(?:www\.)?bbc\.co\.uk/(?:[^/]+/)+[\da-z]{8}(?:\b[^"]*)?'
+        EMBED_URL = r'https?://(?:www\.)?bbc\.co\.uk/(?:[^/]+/)+[\da-z]{8}(?:\b[^"]+)?'
         entries = []
         for match in extract_all(r'new\s+SMP\(({.+?})\)'):
             embed_url = match.get('playerSettings', {}).get('externalEmbedUrl')

From 289bbb350e11ae1460db515560345088d7c58663 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 28 Jul 2015 11:28:33 +0200
Subject: [PATCH 2106/6219] release 2015.07.28

---
 docs/supportedsites.md | 3 +++
 youtube_dl/version.py  | 2 +-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 73445137f..657935dc6 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -51,6 +51,7 @@
  - **bambuser:channel**
  - **Bandcamp**
  - **Bandcamp:album**
+ - **bbc**: BBC
  - **bbc.co.uk**: BBC iPlayer
  - **BeatportPro**
  - **Beeg**
@@ -224,6 +225,7 @@
  - **InternetVideoArchive**
  - **IPrima**
  - **iqiyi**: 爱奇艺
+ - **Ir90Tv**
  - **ivi**: ivi.ru
  - **ivi:compilation**: ivi.ru compilations
  - **Izlesene**
@@ -252,6 +254,7 @@
  - **kuwo:song**: 酷我音乐
  - **la7.tv**
  - **Laola1Tv**
+ - **Lecture2Go**
  - **Letv**: 乐视网
  - **LetvPlaylist**
  - **LetvTv**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 280afdd7f..fa157cadb 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.07.21'
+__version__ = '2015.07.28'

From f171bc8b59ecf4560dd4076be56570a4f090d519 Mon Sep 17 00:00:00 2001
From: Purdea Andrei <andrei@purdea.ro>
Date: Tue, 28 Jul 2015 18:14:06 +0300
Subject: [PATCH 2107/6219] [youtube] save keywords in info jason when
 --write-info-json is used

---
 youtube_dl/extractor/youtube.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 0e411bfb6..15e327ec8 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1072,6 +1072,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         else:
             video_categories = None
 
+        m = re.findall(r'''<meta(?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]+|="[^"]+"|='[^']+'))*?\s+property=['"]?og:video:tag['"]?(?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]+|="[^"]+"|='[^']+'))*?\s+content=['"]?([^>'"]+?)['"]?\s*>'''
+        , video_webpage, re.DOTALL | re.IGNORECASE);
+        video_tags = ", ".join(m)
         # description
         video_description = get_element_by_id("eow-description", video_webpage)
         if video_description:
@@ -1259,6 +1262,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             'title': video_title,
             'thumbnail': video_thumbnail,
             'description': video_description,
+            'tags' : video_tags,
             'categories': video_categories,
             'subtitles': video_subtitles,
             'automatic_captions': automatic_captions,

From a6f774e9015995393a086273df8db1d7b0c098c4 Mon Sep 17 00:00:00 2001
From: Purdea Andrei <andrei@purdea.ro>
Date: Tue, 28 Jul 2015 18:29:13 +0300
Subject: [PATCH 2108/6219] [youtube]: tags key in info jason is now a list

---
 youtube_dl/extractor/youtube.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 15e327ec8..c0fafbfd5 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1072,9 +1072,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         else:
             video_categories = None
 
-        m = re.findall(r'''<meta(?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]+|="[^"]+"|='[^']+'))*?\s+property=['"]?og:video:tag['"]?(?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]+|="[^"]+"|='[^']+'))*?\s+content=['"]?([^>'"]+?)['"]?\s*>'''
+        video_tags = re.findall(r'''<meta(?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]+|="[^"]+"|='[^']+'))*?\s+property=['"]?og:video:tag['"]?(?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]+|="[^"]+"|='[^']+'))*?\s+content=['"]?([^>'"]+?)['"]?\s*>'''
         , video_webpage, re.DOTALL | re.IGNORECASE);
-        video_tags = ", ".join(m)
         # description
         video_description = get_element_by_id("eow-description", video_webpage)
         if video_description:

From 5316bf7487b608b7c085950ff2fb0444f2c36dc0 Mon Sep 17 00:00:00 2001
From: Purdea Andrei <andrei@purdea.ro>
Date: Tue, 28 Jul 2015 18:30:42 +0300
Subject: [PATCH 2109/6219] Documented tags as a possible dict key

---
 youtube_dl/extractor/common.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 14b9b4fe2..a227aeb9c 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -187,6 +187,7 @@ class InfoExtractor(object):
                     specified in the URL.
     end_time:       Time in seconds where the reproduction should end, as
                     specified in the URL.
+    tags:           A list of keywords attached to the video.
 
     Unless mentioned otherwise, the fields should be Unicode strings.
 

From 95d8f7ea12f40a986b541ee17cdf384dbedcfa55 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 29 Jul 2015 02:26:16 +0600
Subject: [PATCH 2110/6219] [fragment] Generalize fragmented media file
 downloader

---
 youtube_dl/downloader/fragment.py | 110 ++++++++++++++++++++++++++++++
 1 file changed, 110 insertions(+)
 create mode 100644 youtube_dl/downloader/fragment.py

diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
new file mode 100644
index 000000000..5f9d6796d
--- /dev/null
+++ b/youtube_dl/downloader/fragment.py
@@ -0,0 +1,110 @@
+from __future__ import division, unicode_literals
+
+import os
+import time
+
+from .common import FileDownloader
+from .http import HttpFD
+from ..utils import (
+    encodeFilename,
+    sanitize_open,
+)
+
+
+class HttpQuietDownloader(HttpFD):
+    def to_screen(self, *args, **kargs):
+        pass
+
+
+class FragmentFD(FileDownloader):
+    """
+    A base file downloader class for fragmented media (e.g. f4m/m3u8 manifests).
+    """
+
+    def _prepare_and_start_frag_download(self, ctx):
+        self._prepare_frag_download(ctx)
+        self._start_frag_download(ctx)
+
+    def _prepare_frag_download(self, ctx):
+        self.to_screen('[%s] Total fragments: %d' % (self.FD_NAME, ctx['total_frags']))
+        self.report_destination(ctx['filename'])
+        dl = HttpQuietDownloader(
+            self.ydl,
+            {
+                'continuedl': True,
+                'quiet': True,
+                'noprogress': True,
+                'ratelimit': self.params.get('ratelimit', None),
+                'test': self.params.get('test', False),
+            }
+        )
+        tmpfilename = self.temp_name(ctx['filename'])
+        dest_stream, tmpfilename = sanitize_open(tmpfilename, 'wb')
+        ctx.update({
+            'dl': dl,
+            'dest_stream': dest_stream,
+            'tmpfilename': tmpfilename,
+        })
+
+    def _start_frag_download(self, ctx):
+        total_frags = ctx['total_frags']
+        # This dict stores the download progress, it's updated by the progress
+        # hook
+        state = {
+            'status': 'downloading',
+            'downloaded_bytes': 0,
+            'frag_index': 0,
+            'frag_count': total_frags,
+            'filename': ctx['filename'],
+            'tmpfilename': ctx['tmpfilename'],
+        }
+        start = time.time()
+        ctx['started'] = start
+
+        def frag_progress_hook(s):
+            if s['status'] not in ('downloading', 'finished'):
+                return
+
+            frag_total_bytes = s.get('total_bytes', 0)
+            if s['status'] == 'finished':
+                state['downloaded_bytes'] += frag_total_bytes
+                state['frag_index'] += 1
+
+            estimated_size = (
+                (state['downloaded_bytes'] + frag_total_bytes) /
+                (state['frag_index'] + 1) * total_frags)
+            time_now = time.time()
+            state['total_bytes_estimate'] = estimated_size
+            state['elapsed'] = time_now - start
+
+            if s['status'] == 'finished':
+                progress = self.calc_percent(state['frag_index'], total_frags)
+            else:
+                frag_downloaded_bytes = s['downloaded_bytes']
+                frag_progress = self.calc_percent(frag_downloaded_bytes,
+                                                  frag_total_bytes)
+                progress = self.calc_percent(state['frag_index'], total_frags)
+                progress += frag_progress / float(total_frags)
+
+                state['eta'] = self.calc_eta(
+                    start, time_now, estimated_size, state['downloaded_bytes'] + frag_downloaded_bytes)
+                state['speed'] = s.get('speed')
+            self._hook_progress(state)
+
+        ctx['dl'].add_progress_hook(frag_progress_hook)
+
+        return start
+
+    def _finish_frag_download(self, ctx):
+        ctx['dest_stream'].close()
+        elapsed = time.time() - ctx['started']
+        self.try_rename(ctx['tmpfilename'], ctx['filename'])
+        fsize = os.path.getsize(encodeFilename(ctx['filename']))
+
+        self._hook_progress({
+            'downloaded_bytes': fsize,
+            'total_bytes': fsize,
+            'filename': ctx['filename'],
+            'status': 'finished',
+            'elapsed': elapsed,
+        })

From ab81ef8fa7d0f9e814890024140c7ed9587d3151 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 29 Jul 2015 02:27:50 +0600
Subject: [PATCH 2111/6219] [f4m] Implement f4m fd in terms of fragment fd

---
 youtube_dl/downloader/f4m.py | 98 +++++++-----------------------------
 1 file changed, 17 insertions(+), 81 deletions(-)

diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index b1a858c45..275564b59 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -7,8 +7,7 @@ import os
 import time
 import xml.etree.ElementTree as etree
 
-from .common import FileDownloader
-from .http import HttpFD
+from .fragment import FragmentFD
 from ..compat import (
     compat_urlparse,
     compat_urllib_error,
@@ -16,8 +15,6 @@ from ..compat import (
 from ..utils import (
     struct_pack,
     struct_unpack,
-    encodeFilename,
-    sanitize_open,
     xpath_text,
 )
 
@@ -226,16 +223,13 @@ def _add_ns(prop):
     return '{http://ns.adobe.com/f4m/1.0}%s' % prop
 
 
-class HttpQuietDownloader(HttpFD):
-    def to_screen(self, *args, **kargs):
-        pass
-
-
-class F4mFD(FileDownloader):
+class F4mFD(FragmentFD):
     """
     A downloader for f4m manifests or AdobeHDS.
     """
 
+    FD_NAME = 'f4m'
+
     def _get_unencrypted_media(self, doc):
         media = doc.findall(_add_ns('media'))
         if not media:
@@ -288,7 +282,7 @@ class F4mFD(FileDownloader):
     def real_download(self, filename, info_dict):
         man_url = info_dict['url']
         requested_bitrate = info_dict.get('tbr')
-        self.to_screen('[download] Downloading f4m manifest')
+        self.to_screen('[%s] Downloading f4m manifest' % self.FD_NAME)
         manifest = self.ydl.urlopen(man_url).read()
 
         doc = etree.fromstring(manifest)
@@ -320,67 +314,20 @@ class F4mFD(FileDownloader):
         # For some akamai manifests we'll need to add a query to the fragment url
         akamai_pv = xpath_text(doc, _add_ns('pv-2.0'))
 
-        self.report_destination(filename)
-        http_dl = HttpQuietDownloader(
-            self.ydl,
-            {
-                'continuedl': True,
-                'quiet': True,
-                'noprogress': True,
-                'ratelimit': self.params.get('ratelimit', None),
-                'test': self.params.get('test', False),
-            }
-        )
-        tmpfilename = self.temp_name(filename)
-        (dest_stream, tmpfilename) = sanitize_open(tmpfilename, 'wb')
+        ctx = {
+            'filename': filename,
+            'total_frags': total_frags,
+        }
+
+        self._prepare_frag_download(ctx)
+
+        dest_stream = ctx['dest_stream']
 
         write_flv_header(dest_stream)
         if not live:
             write_metadata_tag(dest_stream, metadata)
 
-        # This dict stores the download progress, it's updated by the progress
-        # hook
-        state = {
-            'status': 'downloading',
-            'downloaded_bytes': 0,
-            'frag_index': 0,
-            'frag_count': total_frags,
-            'filename': filename,
-            'tmpfilename': tmpfilename,
-        }
-        start = time.time()
-
-        def frag_progress_hook(s):
-            if s['status'] not in ('downloading', 'finished'):
-                return
-
-            frag_total_bytes = s.get('total_bytes', 0)
-            if s['status'] == 'finished':
-                state['downloaded_bytes'] += frag_total_bytes
-                state['frag_index'] += 1
-
-            estimated_size = (
-                (state['downloaded_bytes'] + frag_total_bytes) /
-                (state['frag_index'] + 1) * total_frags)
-            time_now = time.time()
-            state['total_bytes_estimate'] = estimated_size
-            state['elapsed'] = time_now - start
-
-            if s['status'] == 'finished':
-                progress = self.calc_percent(state['frag_index'], total_frags)
-            else:
-                frag_downloaded_bytes = s['downloaded_bytes']
-                frag_progress = self.calc_percent(frag_downloaded_bytes,
-                                                  frag_total_bytes)
-                progress = self.calc_percent(state['frag_index'], total_frags)
-                progress += frag_progress / float(total_frags)
-
-                state['eta'] = self.calc_eta(
-                    start, time_now, estimated_size, state['downloaded_bytes'] + frag_downloaded_bytes)
-                state['speed'] = s.get('speed')
-            self._hook_progress(state)
-
-        http_dl.add_progress_hook(frag_progress_hook)
+        self._start_frag_download(ctx)
 
         frags_filenames = []
         while fragments_list:
@@ -391,9 +338,9 @@ class F4mFD(FileDownloader):
                 url += '?' + akamai_pv.strip(';')
             if info_dict.get('extra_param_to_segment_url'):
                 url += info_dict.get('extra_param_to_segment_url')
-            frag_filename = '%s-%s' % (tmpfilename, name)
+            frag_filename = '%s-%s' % (ctx['tmpfilename'], name)
             try:
-                success = http_dl.download(frag_filename, {'url': url})
+                success = ctx['dl'].download(frag_filename, {'url': url})
                 if not success:
                     return False
                 with open(frag_filename, 'rb') as down:
@@ -425,20 +372,9 @@ class F4mFD(FileDownloader):
                     msg = 'Missed %d fragments' % (fragments_list[0][1] - (frag_i + 1))
                     self.report_warning(msg)
 
-        dest_stream.close()
+        self._finish_frag_download(ctx)
 
-        elapsed = time.time() - start
-        self.try_rename(tmpfilename, filename)
         for frag_file in frags_filenames:
             os.remove(frag_file)
 
-        fsize = os.path.getsize(encodeFilename(filename))
-        self._hook_progress({
-            'downloaded_bytes': fsize,
-            'total_bytes': fsize,
-            'filename': filename,
-            'status': 'finished',
-            'elapsed': elapsed,
-        })
-
         return True

From f9a5affad968c6629c022f489dc3c1561a4c57de Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 29 Jul 2015 02:28:30 +0600
Subject: [PATCH 2112/6219] [hls] Implement hlsnative fd in terms of fragment
 fd

---
 youtube_dl/downloader/hls.py | 81 +++++++++++++++++-------------------
 1 file changed, 38 insertions(+), 43 deletions(-)

diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index 8be4f4249..60dca0ab1 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -4,12 +4,11 @@ import os
 import re
 import subprocess
 
-from ..postprocessor.ffmpeg import FFmpegPostProcessor
 from .common import FileDownloader
-from ..compat import (
-    compat_urlparse,
-    compat_urllib_request,
-)
+from .fragment import FragmentFD
+
+from ..compat import compat_urlparse
+from ..postprocessor.ffmpeg import FFmpegPostProcessor
 from ..utils import (
     encodeArgument,
     encodeFilename,
@@ -51,54 +50,50 @@ class HlsFD(FileDownloader):
             return False
 
 
-class NativeHlsFD(FileDownloader):
+class NativeHlsFD(FragmentFD):
     """ A more limited implementation that does not require ffmpeg """
 
-    def real_download(self, filename, info_dict):
-        url = info_dict['url']
-        self.report_destination(filename)
-        tmpfilename = self.temp_name(filename)
+    FD_NAME = 'hlsnative'
 
-        self.to_screen(
-            '[hlsnative] %s: Downloading m3u8 manifest' % info_dict['id'])
-        data = self.ydl.urlopen(url).read()
-        s = data.decode('utf-8', 'ignore')
-        segment_urls = []
+    def real_download(self, filename, info_dict):
+        man_url = info_dict['url']
+        self.to_screen('[%s] Downloading m3u8 manifest' % self.FD_NAME)
+        manifest = self.ydl.urlopen(man_url).read()
+
+        s = manifest.decode('utf-8', 'ignore')
+        fragment_urls = []
         for line in s.splitlines():
             line = line.strip()
             if line and not line.startswith('#'):
                 segment_url = (
                     line
                     if re.match(r'^https?://', line)
-                    else compat_urlparse.urljoin(url, line))
-                segment_urls.append(segment_url)
-
-        is_test = self.params.get('test', False)
-        remaining_bytes = self._TEST_FILE_SIZE if is_test else None
-        byte_counter = 0
-        with open(tmpfilename, 'wb') as outf:
-            for i, segurl in enumerate(segment_urls):
-                self.to_screen(
-                    '[hlsnative] %s: Downloading segment %d / %d' %
-                    (info_dict['id'], i + 1, len(segment_urls)))
-                seg_req = compat_urllib_request.Request(segurl)
-                if remaining_bytes is not None:
-                    seg_req.add_header('Range', 'bytes=0-%d' % (remaining_bytes - 1))
-
-                segment = self.ydl.urlopen(seg_req).read()
-                if remaining_bytes is not None:
-                    segment = segment[:remaining_bytes]
-                    remaining_bytes -= len(segment)
-                outf.write(segment)
-                byte_counter += len(segment)
-                if remaining_bytes is not None and remaining_bytes <= 0:
+                    else compat_urlparse.urljoin(man_url, line))
+                fragment_urls.append(segment_url)
+                # We only download the first fragment during the test
+                if self.params.get('test', False):
                     break
 
-        self._hook_progress({
-            'downloaded_bytes': byte_counter,
-            'total_bytes': byte_counter,
+        ctx = {
             'filename': filename,
-            'status': 'finished',
-        })
-        self.try_rename(tmpfilename, filename)
+            'total_frags': len(fragment_urls),
+        }
+
+        self._prepare_and_start_frag_download(ctx)
+
+        frags_filenames = []
+        for i, frag_url in enumerate(fragment_urls):
+            frag_filename = '%s-Frag%d' % (ctx['tmpfilename'], i)
+            success = ctx['dl'].download(frag_filename, {'url': frag_url})
+            if not success:
+                return False
+            with open(frag_filename, 'rb') as down:
+                ctx['dest_stream'].write(down.read())
+            frags_filenames.append(frag_filename)
+
+        self._finish_frag_download(ctx)
+
+        for frag_file in frags_filenames:
+            os.remove(frag_file)
+
         return True

From 864f24bd2c0cf9bde034812a2049c3750c1bb05c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 29 Jul 2015 03:43:03 +0600
Subject: [PATCH 2113/6219] [extractor/common] Add _meta_regex and clarify tags
 field

---
 youtube_dl/extractor/common.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index a227aeb9c..d54866d1f 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -181,13 +181,13 @@ class InfoExtractor(object):
                     by YoutubeDL if it's missing)
     categories:     A list of categories that the video falls in, for example
                     ["Sports", "Berlin"]
+    tags:           A list of tags assigned to the video, e.g. ["sweden", "pop music"]
     is_live:        True, False, or None (=unknown). Whether this video is a
                     live stream that goes on instead of a fixed-length video.
     start_time:     Time in seconds where the reproduction should start, as
                     specified in the URL.
     end_time:       Time in seconds where the reproduction should end, as
                     specified in the URL.
-    tags:           A list of keywords attached to the video.
 
     Unless mentioned otherwise, the fields should be Unicode strings.
 
@@ -631,6 +631,12 @@ class InfoExtractor(object):
             template % (content_re, property_re),
         ]
 
+    @staticmethod
+    def _meta_regex(prop):
+        return r'''(?isx)<meta
+                    (?=[^>]+(?:itemprop|name|property)=(["\']?)%s\1)
+                    [^>]+?content=(["\'])(?P<content>.*?)\2''' % re.escape(prop)
+
     def _og_search_property(self, prop, html, name=None, **kargs):
         if name is None:
             name = 'OpenGraph %s' % prop
@@ -661,9 +667,7 @@ class InfoExtractor(object):
         if display_name is None:
             display_name = name
         return self._html_search_regex(
-            r'''(?isx)<meta
-                    (?=[^>]+(?:itemprop|name|property)=(["\']?)%s\1)
-                    [^>]+?content=(["\'])(?P<content>.*?)\2''' % re.escape(name),
+            self._meta_regex(name),
             html, display_name, fatal=fatal, group='content', **kwargs)
 
     def _dc_search_uploader(self, html):

From 000b6b5ae5cc214906effe4ac5b78b579bc7db70 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 29 Jul 2015 03:43:32 +0600
Subject: [PATCH 2114/6219] [youtube] Improve tags extraction and add test

---
 youtube_dl/extractor/youtube.py | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index c0fafbfd5..4c449fd74 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -329,6 +329,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'upload_date': '20121002',
                 'description': 'test chars:  "\'/\\ä↭𝕐\ntest URL: https://github.com/rg3/youtube-dl/issues/1892\n\nThis is a test video for youtube-dl.\n\nFor more information, contact phihag@phihag.de .',
                 'categories': ['Science & Technology'],
+                'tags': ['youtube-dl'],
                 'like_count': int,
                 'dislike_count': int,
                 'start_time': 1,
@@ -343,7 +344,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'ext': 'mp4',
                 'upload_date': '20120506',
                 'title': 'Icona Pop - I Love It (feat. Charli XCX) [OFFICIAL VIDEO]',
-                'description': 'md5:fea86fda2d5a5784273df5c7cc994d9f',
+                'description': 'md5:782e8651347686cba06e58f71ab51773',
+                'tags': ['Icona Pop i love it', 'sweden', 'pop music', 'big beat records', 'big beat', 'charli',
+                         'xcx', 'charli xcx', 'girls', 'hbo', 'i love it', "i don't care", 'icona', 'pop',
+                         'iconic ep', 'iconic', 'love', 'it'],
                 'uploader': 'Icona Pop',
                 'uploader_id': 'IconaPop',
             }
@@ -1072,8 +1076,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         else:
             video_categories = None
 
-        video_tags = re.findall(r'''<meta(?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]+|="[^"]+"|='[^']+'))*?\s+property=['"]?og:video:tag['"]?(?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]+|="[^"]+"|='[^']+'))*?\s+content=['"]?([^>'"]+?)['"]?\s*>'''
-        , video_webpage, re.DOTALL | re.IGNORECASE);
+        video_tags = [
+            unescapeHTML(m.group('content'))
+            for m in re.finditer(self._meta_regex('og:video:tag'), video_webpage)]
+
         # description
         video_description = get_element_by_id("eow-description", video_webpage)
         if video_description:
@@ -1261,8 +1267,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             'title': video_title,
             'thumbnail': video_thumbnail,
             'description': video_description,
-            'tags' : video_tags,
             'categories': video_categories,
+            'tags': video_tags,
             'subtitles': video_subtitles,
             'automatic_captions': automatic_captions,
             'duration': video_duration,

From 5e1eddb939a4fbc2a5ef10111c8141c842cf01d6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 29 Jul 2015 21:18:16 +0600
Subject: [PATCH 2115/6219] [youtube] Show info message for multifeed videos
 according to noplaylist option

---
 youtube_dl/extractor/youtube.py | 37 +++++++++++++++++++--------------
 1 file changed, 21 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index bcd27408d..8a5ef2e70 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1089,22 +1089,27 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             else:
                 video_description = ''
 
-        if (not self._downloader.params.get('noplaylist') and
-                not smuggled_data.get('force_singlefeed', False) and
-                'multifeed_metadata_list' in video_info):
-            entries = []
-            multifeed_metadata_list = compat_urllib_parse_unquote_plus(video_info['multifeed_metadata_list'][0])
-            for feed in multifeed_metadata_list.split(','):
-                feed_data = compat_parse_qs(feed)
-                entries.append({
-                    '_type': 'url_transparent',
-                    'ie_key': 'Youtube',
-                    'url': smuggle_url(
-                        '%s://www.youtube.com/watch?v=%s' % (proto, feed_data['id'][0]),
-                        {'force_singlefeed': True}),
-                    'title': '%s (%s)' % (video_title, feed_data['title'][0]),
-                })
-            return self.playlist_result(entries, video_id, video_title, video_description)
+        if 'multifeed_metadata_list' in video_info and not smuggled_data.get('force_singlefeed', False):
+            if not self._downloader.params.get('noplaylist'):
+                entries = []
+                feed_ids = []
+                multifeed_metadata_list = compat_urllib_parse_unquote_plus(video_info['multifeed_metadata_list'][0])
+                for feed in multifeed_metadata_list.split(','):
+                    feed_data = compat_parse_qs(feed)
+                    entries.append({
+                        '_type': 'url_transparent',
+                        'ie_key': 'Youtube',
+                        'url': smuggle_url(
+                            '%s://www.youtube.com/watch?v=%s' % (proto, feed_data['id'][0]),
+                            {'force_singlefeed': True}),
+                        'title': '%s (%s)' % (video_title, feed_data['title'][0]),
+                    })
+                    feed_ids.append(feed_data['id'][0])
+                self.to_screen(
+                    'Downloading multifeed video (%s) - add --no-playlist to just download video %s'
+                    % (', '.join(feed_ids), video_id))
+                return self.playlist_result(entries, video_id, video_title, video_description)
+            self.to_screen('Downloading just video %s because of --no-playlist' % video_id)
 
         if 'view_count' in video_info:
             view_count = int(video_info['view_count'][0])

From 2711e41bcdaf8e8234dd32e5df6fb76f23179f46 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 29 Jul 2015 18:47:20 +0200
Subject: [PATCH 2116/6219] Credit slangangular for the sportschau extractor
 (#6199)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index e75e9885d..aa6b88cc0 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -135,3 +135,4 @@ Bernhard Minks
 sceext
 Zach Bruggeman
 Tjark Saul
+slangangular

From d12a1a47d5ff0833a1ecd7fe47f4ffa67a9b4a50 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 30 Jul 2015 00:55:06 +0600
Subject: [PATCH 2117/6219] [bbc] Improve work with mediaselection URLs

---
 youtube_dl/extractor/bbc.py | 50 ++++++++++++++++++++++++-------------
 1 file changed, 33 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 3d9366644..9a1b6e3dc 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -14,12 +14,15 @@ from ..utils import (
 )
 from ..compat import compat_HTTPError
 
+
 class BBCCoUkIE(InfoExtractor):
     IE_NAME = 'bbc.co.uk'
     IE_DESC = 'BBC iPlayer'
     _VALID_URL = r'https?://(?:www\.)?bbc\.co\.uk/(?:(?:(?:programmes|iplayer(?:/[^/]+)?/(?:episode|playlist))/)|music/clips[/#])(?P<id>[\da-z]{8})'
 
-    _MEDIASELECTOR_URL = 'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/pc/vpid/%s'
+    _MEDIASELECTOR_URLS = [
+        'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/pc/vpid/%s',
+    ]
 
     _TESTS = [
         {
@@ -161,6 +164,10 @@ class BBCCoUkIE(InfoExtractor):
         }
     ]
 
+    class MediaSelectionError(Exception):
+        def __init__(self, id):
+            self.id = id
+
     def _extract_asx_playlist(self, connection, programme_id):
         asx = self._download_xml(connection.get('href'), programme_id, 'Downloading ASX playlist')
         return [ref.get('href') for ref in asx.findall('./Entry/ref')]
@@ -211,8 +218,7 @@ class BBCCoUkIE(InfoExtractor):
     def _extract_medias(self, media_selection):
         error = media_selection.find('./{http://bbc.co.uk/2008/mp/mediaselection}error')
         if error is not None:
-            raise ExtractorError(
-                '%s returned error: %s' % (self.IE_NAME, error.get('id')), expected=True)
+            raise BBCCoUkIE.MediaSelectionError(error.get('id'))
         return media_selection.findall('./{http://bbc.co.uk/2008/mp/mediaselection}media')
 
     def _extract_connections(self, media):
@@ -269,17 +275,23 @@ class BBCCoUkIE(InfoExtractor):
             ]
         return subtitles
 
+    def _raise_extractor_error(self, media_selection_error):
+        raise ExtractorError(
+            '%s returned error: %s' % (self.IE_NAME, media_selection_error.id),
+            expected=True)
+
     def _download_media_selector(self, programme_id):
-        try:
-            return self._download_media_selector_url(
-                self._MEDIASELECTOR_URL % programme_id, programme_id)
-        except ExtractorError as e:
-            if hasattr(self, '_MEDIASELECTOR_ALT_URL') and str(e) == 'bbc returned error: notukerror':
-                 # notukerror on bbc.com/travel using bbc news mediaselector: fallback to /mediaselector/5/
-                 return self._download_media_selector_url(
-                     self._MEDIASELECTOR_ALT_URL % programme_id, programme_id)
-            else:
-                 raise
+        last_exception = None
+        for mediaselector_url in self._MEDIASELECTOR_URLS:
+            try:
+                return self._download_media_selector_url(
+                    mediaselector_url % programme_id, programme_id)
+            except BBCCoUkIE.MediaSelectionError as e:
+                if e.id == 'notukerror':
+                    last_exception = e
+                    continue
+                self._raise_extractor_error(e)
+        self._raise_extractor_error(last_exception)
 
     def _download_media_selector_url(self, url, programme_id=None):
         try:
@@ -432,10 +444,14 @@ class BBCIE(BBCCoUkIE):
     IE_DESC = 'BBC'
     _VALID_URL = r'https?://(?:www\.)?bbc\.(?:com|co\.uk)/(?:[^/]+/)+(?P<id>[^/#?]+)'
 
-    # fails with notukerror for some videos ( non news sites such as bbc.com/travel )
-    _MEDIASELECTOR_URL = 'http://open.live.bbc.co.uk/mediaselector/4/mtis/stream/%s'
-    # limited selection of formats but may work where the above does not
-    _MEDIASELECTOR_ALT_URL = 'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/journalism-pc/vpid/%s'
+    _MEDIASELECTOR_URLS = [
+        # Provides more formats, namely direct mp4 links, but fails on some videos with
+        # notukerror for non UK (?) users (e.g.
+        # http://www.bbc.com/travel/story/20150625-sri-lankas-spicy-secret)
+        'http://open.live.bbc.co.uk/mediaselector/4/mtis/stream/%s',
+        # Provides fewer formats, but works everywhere for everybody (hopefully)
+        'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/journalism-pc/vpid/%s',
+    ]
 
     _TESTS = [{
         # article with multiple videos embedded with data-media-meta containing

From 799207e838e0404aaa5cb6658e41bef108aced16 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 29 Jul 2015 23:20:37 +0100
Subject: [PATCH 2118/6219] [viewster] extract the api auth token

Closes #6406.
---
 youtube_dl/compat.py             | 6 ++++++
 youtube_dl/extractor/common.py   | 8 ++++++++
 youtube_dl/extractor/viewster.py | 5 ++++-
 3 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 0c57c7aeb..e4b9286c0 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -42,6 +42,11 @@ try:
 except ImportError:  # Python 2
     import cookielib as compat_cookiejar
 
+try:
+    import http.cookies as compat_cookies
+except ImportError:  # Python 2
+    import Cookie as compat_cookies
+
 try:
     import html.entities as compat_html_entities
 except ImportError:  # Python 2
@@ -436,6 +441,7 @@ __all__ = [
     'compat_basestring',
     'compat_chr',
     'compat_cookiejar',
+    'compat_cookies',
     'compat_expanduser',
     'compat_get_terminal_size',
     'compat_getenv',
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index d54866d1f..dc5080504 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -14,10 +14,12 @@ import xml.etree.ElementTree
 
 from ..compat import (
     compat_cookiejar,
+    compat_cookies,
     compat_HTTPError,
     compat_http_client,
     compat_urllib_error,
     compat_urllib_parse_urlparse,
+    compat_urllib_request,
     compat_urlparse,
     compat_str,
 )
@@ -1074,6 +1076,12 @@ class InfoExtractor(object):
             None, '/', True, False, expire_time, '', None, None, None)
         self._downloader.cookiejar.set_cookie(cookie)
 
+    def _get_cookies(self, url):
+        """ Return a compat_cookies.SimpleCookie with the cookies for the url """
+        req = compat_urllib_request.Request(url)
+        self._downloader.cookiejar.add_cookie_header(req)
+        return compat_cookies.SimpleCookie(req.get_header('Cookie'))
+
     def get_testcases(self, include_onlymatching=False):
         t = getattr(self, '_TEST', None)
         if t:
diff --git a/youtube_dl/extractor/viewster.py b/youtube_dl/extractor/viewster.py
index 6ef36290b..393b63618 100644
--- a/youtube_dl/extractor/viewster.py
+++ b/youtube_dl/extractor/viewster.py
@@ -62,7 +62,6 @@ class ViewsterIE(InfoExtractor):
     }]
 
     _ACCEPT_HEADER = 'application/json, text/javascript, */*; q=0.01'
-    _AUTH_TOKEN = '/YqhSYsx8EaU9Bsta3ojlA=='
 
     def _download_json(self, url, video_id, note='Downloading JSON metadata', fatal=True):
         request = compat_urllib_request.Request(url)
@@ -72,6 +71,10 @@ class ViewsterIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
+        # Get 'api_token' cookie
+        self._request_webpage(url, video_id)
+        cookies = self._get_cookies(url)
+        self._AUTH_TOKEN = compat_urllib_parse.unquote(cookies['api_token'].value)
 
         info = self._download_json(
             'https://public-api.viewster.com/search/%s' % video_id,

From 1f04873517e41d2f54fc4e65c46c08ba85b23010 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 30 Jul 2015 19:12:37 +0200
Subject: [PATCH 2119/6219] [viewster] Use 'compat_urllib_parse_unquote'

---
 youtube_dl/extractor/viewster.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/viewster.py b/youtube_dl/extractor/viewster.py
index 393b63618..65324d903 100644
--- a/youtube_dl/extractor/viewster.py
+++ b/youtube_dl/extractor/viewster.py
@@ -5,6 +5,7 @@ from .common import InfoExtractor
 from ..compat import (
     compat_urllib_request,
     compat_urllib_parse,
+    compat_urllib_parse_unquote,
 )
 from ..utils import (
     determine_ext,
@@ -74,7 +75,7 @@ class ViewsterIE(InfoExtractor):
         # Get 'api_token' cookie
         self._request_webpage(url, video_id)
         cookies = self._get_cookies(url)
-        self._AUTH_TOKEN = compat_urllib_parse.unquote(cookies['api_token'].value)
+        self._AUTH_TOKEN = compat_urllib_parse_unquote(cookies['api_token'].value)
 
         info = self._download_json(
             'https://public-api.viewster.com/search/%s' % video_id,

From 80fb6d4aa47154a1e963b28a17a065dc40a436b8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 31 Jul 2015 00:54:26 +0600
Subject: [PATCH 2120/6219] [soundcloud:user] Rework extractor (Closes #6399)

---
 youtube_dl/extractor/soundcloud.py | 131 ++++++++++++++++++++++-------
 1 file changed, 101 insertions(+), 30 deletions(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 0a6c9fe72..f1307dc83 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -29,7 +29,7 @@ class SoundcloudIE(InfoExtractor):
     _VALID_URL = r'''(?x)^(?:https?://)?
                     (?:(?:(?:www\.|m\.)?soundcloud\.com/
                             (?P<uploader>[\w\d-]+)/
-                            (?!sets/|(?:likes|tracks)/?(?:$|[?#]))
+                            (?!(?:tracks|sets|reposts|likes|spotlight)/?(?:$|[?#]))
                             (?P<title>[\w\d-]+)/?
                             (?P<token>[^?]+?)?(?:[?].*)?$)
                        |(?:api\.soundcloud\.com/tracks/(?P<track_id>\d+)
@@ -293,60 +293,131 @@ class SoundcloudSetIE(SoundcloudIE):
 
 
 class SoundcloudUserIE(SoundcloudIE):
-    _VALID_URL = r'https?://(?:(?:www|m)\.)?soundcloud\.com/(?P<user>[^/]+)/?((?P<rsrc>tracks|likes)/?)?(\?.*)?$'
+    _VALID_URL = r'https?://(?:(?:www|m)\.)?soundcloud\.com/(?P<user>[^/]+)/?((?P<rsrc>tracks|sets|reposts|likes|spotlight)/?)?(\?.*)?$'
     IE_NAME = 'soundcloud:user'
     _TESTS = [{
-        'url': 'https://soundcloud.com/the-concept-band',
+        'url': 'https://soundcloud.com/the-akashic-chronicler',
         'info_dict': {
-            'id': '9615865',
-            'title': 'The Royal Concept',
+            'id': '114582580',
+            'title': 'The Akashic Chronicler (All)',
         },
-        'playlist_mincount': 12
-    }, {
-        'url': 'https://soundcloud.com/the-concept-band/likes',
-        'info_dict': {
-            'id': '9615865',
-            'title': 'The Royal Concept',
-        },
-        'playlist_mincount': 1,
+        'playlist_mincount': 112,
     }, {
         'url': 'https://soundcloud.com/the-akashic-chronicler/tracks',
-        'only_matching': True,
+        'info_dict': {
+            'id': '114582580',
+            'title': 'The Akashic Chronicler (Tracks)',
+        },
+        'playlist_mincount': 50,
+    }, {
+        'url': 'https://soundcloud.com/the-akashic-chronicler/sets',
+        'info_dict': {
+            'id': '114582580',
+            'title': 'The Akashic Chronicler (Playlists)',
+        },
+        'playlist_mincount': 3,
+    }, {
+        'url': 'https://soundcloud.com/the-akashic-chronicler/reposts',
+        'info_dict': {
+            'id': '114582580',
+            'title': 'The Akashic Chronicler (Reposts)',
+        },
+        'playlist_mincount': 9,
+    }, {
+        'url': 'https://soundcloud.com/the-akashic-chronicler/likes',
+        'info_dict': {
+            'id': '114582580',
+            'title': 'The Akashic Chronicler (Likes)',
+        },
+        'playlist_mincount': 333,
+    }, {
+        'url': 'https://soundcloud.com/grynpyret/spotlight',
+        'info_dict': {
+            'id': '7098329',
+            'title': 'Grynpyret (Spotlight)',
+        },
+        'playlist_mincount': 1,
     }]
 
+    _API_BASE = 'https://api.soundcloud.com'
+    _API_V2_BASE = 'https://api-v2.soundcloud.com'
+
+    _BASE_URL_MAP = {
+        'all': '%s/profile/soundcloud:users:%%s' % _API_V2_BASE,
+        'tracks': '%s/users/%%s/tracks' % _API_BASE,
+        'sets': '%s/users/%%s/playlists' % _API_V2_BASE,
+        'reposts': '%s/profile/soundcloud:users:%%s/reposts' % _API_V2_BASE,
+        'likes': '%s/users/%%s/likes' % _API_V2_BASE,
+        'spotlight': '%s/users/%%s/spotlight' % _API_V2_BASE,
+    }
+
+    _TITLE_MAP = {
+        'all': 'All',
+        'tracks': 'Tracks',
+        'sets': 'Playlists',
+        'reposts': 'Reposts',
+        'likes': 'Likes',
+        'spotlight': 'Spotlight',
+    }
+
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         uploader = mobj.group('user')
-        resource = mobj.group('rsrc')
-        if resource is None:
-            resource = 'tracks'
-        elif resource == 'likes':
-            resource = 'favorites'
 
         url = 'http://soundcloud.com/%s/' % uploader
         resolv_url = self._resolv_url(url)
         user = self._download_json(
             resolv_url, uploader, 'Downloading user info')
-        base_url = 'http://api.soundcloud.com/users/%s/%s.json?' % (uploader, resource)
+
+        resource = mobj.group('rsrc') or 'all'
+        base_url = self._BASE_URL_MAP[resource] % user['id']
+
+        next_href = None
 
         entries = []
         for i in itertools.count():
-            data = compat_urllib_parse.urlencode({
-                'offset': i * 50,
-                'limit': 50,
-                'client_id': self._CLIENT_ID,
-            })
-            new_entries = self._download_json(
-                base_url + data, uploader, 'Downloading track page %s' % (i + 1))
-            if len(new_entries) == 0:
+            if not next_href:
+                data = compat_urllib_parse.urlencode({
+                    'offset': i * 50,
+                    'limit': 50,
+                    'client_id': self._CLIENT_ID,
+                    'linked_partitioning': '1',
+                    'representation': 'speedy',
+                })
+                next_href = base_url + '?' + data
+
+            response = self._download_json(
+                next_href, uploader, 'Downloading track page %s' % (i + 1))
+
+            collection = response['collection']
+
+            if not collection:
                 self.to_screen('%s: End page received' % uploader)
                 break
-            entries.extend(self.url_result(e['permalink_url'], 'Soundcloud') for e in new_entries)
+
+            def resolve_permalink_url(candidates):
+                for cand in candidates:
+                    if isinstance(cand, dict):
+                        permalink_url = cand.get('permalink_url')
+                        if permalink_url and permalink_url.startswith('http'):
+                            return permalink_url
+
+            for e in collection:
+                permalink_url = resolve_permalink_url((e, e.get('track'), e.get('playlist')))
+                if permalink_url:
+                    entries.append(self.url_result(permalink_url))
+
+            if 'next_href' in response:
+                next_href = response['next_href']
+                if not next_href:
+                    break
+            else:
+                next_href = None
 
         return {
             '_type': 'playlist',
             'id': compat_str(user['id']),
-            'title': user['username'],
+            'title': '%s (%s)' % (user['username'], self._TITLE_MAP[resource]),
             'entries': entries,
         }
 

From 09b6468d3027bbc0180cd8b10aa848180cae4005 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 31 Jul 2015 21:28:59 +0100
Subject: [PATCH 2121/6219] [vidme] extract the uploader

---
 youtube_dl/extractor/vidme.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/vidme.py b/youtube_dl/extractor/vidme.py
index e0b55078b..575a62f3a 100644
--- a/youtube_dl/extractor/vidme.py
+++ b/youtube_dl/extractor/vidme.py
@@ -50,6 +50,7 @@ class VidmeIE(InfoExtractor):
         like_count = str_to_int(self._html_search_regex(
             r'class="score js-video-vote-score"[^>]+data-score="([\d,\.\s]+)">',
             webpage, 'like count', fatal=False))
+        uploader = self._html_search_regex('class="video_author_username">([^<]+)', webpage, 'uploader', fatal=False)
 
         return {
             'id': video_id,
@@ -63,4 +64,5 @@ class VidmeIE(InfoExtractor):
             'duration': duration,
             'view_count': view_count,
             'like_count': like_count,
+            'uploader': uploader,
         }

From 16a089780e87d57b5295c4dfb8da1217ef6eda70 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 1 Aug 2015 03:50:03 +0600
Subject: [PATCH 2122/6219] [soundcloud] Clean up _VALID_URLs

---
 youtube_dl/extractor/soundcloud.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index f1307dc83..6ce86cbcd 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -29,7 +29,7 @@ class SoundcloudIE(InfoExtractor):
     _VALID_URL = r'''(?x)^(?:https?://)?
                     (?:(?:(?:www\.|m\.)?soundcloud\.com/
                             (?P<uploader>[\w\d-]+)/
-                            (?!(?:tracks|sets|reposts|likes|spotlight)/?(?:$|[?#]))
+                            (?!(?:tracks|sets(?:/[^/?#]+)?|reposts|likes|spotlight)/?(?:$|[?#]))
                             (?P<title>[\w\d-]+)/?
                             (?P<token>[^?]+?)?(?:[?].*)?$)
                        |(?:api\.soundcloud\.com/tracks/(?P<track_id>\d+)
@@ -293,7 +293,15 @@ class SoundcloudSetIE(SoundcloudIE):
 
 
 class SoundcloudUserIE(SoundcloudIE):
-    _VALID_URL = r'https?://(?:(?:www|m)\.)?soundcloud\.com/(?P<user>[^/]+)/?((?P<rsrc>tracks|sets|reposts|likes|spotlight)/?)?(\?.*)?$'
+    _VALID_URL = r'''(?x)
+                        https?://
+                            (?:(?:www|m)\.)?soundcloud\.com/
+                            (?P<user>[^/]+)
+                            (?:/
+                                (?P<rsrc>tracks|sets|reposts|likes|spotlight)
+                            )?
+                            /?(?:[?#].*)?$
+                    '''
     IE_NAME = 'soundcloud:user'
     _TESTS = [{
         'url': 'https://soundcloud.com/the-akashic-chronicler',

From 3b58d94f71b0e7b3ace6f5965f7335aa95fd0c1e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 1 Aug 2015 04:03:52 +0600
Subject: [PATCH 2123/6219] [vidme] Make uploader optional and add test

---
 youtube_dl/extractor/vidme.py | 37 ++++++++++++++++++++++++++++++-----
 1 file changed, 32 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/vidme.py b/youtube_dl/extractor/vidme.py
index 575a62f3a..157bb74fe 100644
--- a/youtube_dl/extractor/vidme.py
+++ b/youtube_dl/extractor/vidme.py
@@ -22,6 +22,27 @@ class VidmeIE(InfoExtractor):
             'timestamp': 1406313244,
             'upload_date': '20140725',
             'thumbnail': 're:^https?://.*\.jpg',
+            'view_count': int,
+            'like_count': int,
+        },
+    }, {
+        # tests uploader field
+        'url': 'https://vid.me/4Iib',
+        'info_dict': {
+            'id': '4Iib',
+            'ext': 'mp4',
+            'title': 'The Carver',
+            'description': 'md5:e9c24870018ae8113be936645b93ba3c',
+            'duration': 97.859999999999999,
+            'timestamp': 1433203629,
+            'upload_date': '20150602',
+            'uploader': 'Thomas',
+            'thumbnail': 're:^https?://.*\.jpg',
+            'view_count': int,
+            'like_count': int,
+        },
+        'params': {
+            'skip_download': True,
         },
     }, {
         # From http://naked-yogi.tumblr.com/post/118312946248/naked-smoking-stretching
@@ -40,17 +61,23 @@ class VidmeIE(InfoExtractor):
         title = self._og_search_title(webpage)
         description = self._og_search_description(webpage, default='')
         thumbnail = self._og_search_thumbnail(webpage)
-        timestamp = int_or_none(self._og_search_property('updated_time', webpage, fatal=False))
-        width = int_or_none(self._og_search_property('video:width', webpage, fatal=False))
-        height = int_or_none(self._og_search_property('video:height', webpage, fatal=False))
+        timestamp = int_or_none(self._og_search_property(
+            'updated_time', webpage, fatal=False))
+        width = int_or_none(self._og_search_property(
+            'video:width', webpage, fatal=False))
+        height = int_or_none(self._og_search_property(
+            'video:height', webpage, fatal=False))
         duration = float_or_none(self._html_search_regex(
             r'data-duration="([^"]+)"', webpage, 'duration', fatal=False))
         view_count = str_to_int(self._html_search_regex(
-            r'<(?:li|span) class="video_views">\s*([\d,\.]+)\s*plays?', webpage, 'view count', fatal=False))
+            r'<(?:li|span) class="video_views">\s*([\d,\.]+)\s*plays?',
+            webpage, 'view count', fatal=False))
         like_count = str_to_int(self._html_search_regex(
             r'class="score js-video-vote-score"[^>]+data-score="([\d,\.\s]+)">',
             webpage, 'like count', fatal=False))
-        uploader = self._html_search_regex('class="video_author_username">([^<]+)', webpage, 'uploader', fatal=False)
+        uploader = self._html_search_regex(
+            'class="video_author_username"[^>]*>([^<]+)',
+            webpage, 'uploader', default=None)
 
         return {
             'id': video_id,

From 30a453884e6a4d228ced4cdddcfa8e79a768755a Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 31 Jul 2015 14:41:30 +0100
Subject: [PATCH 2124/6219] [viewster] use head request to extract api token

Closes #6419.
---
 youtube_dl/extractor/viewster.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/viewster.py b/youtube_dl/extractor/viewster.py
index 65324d903..cda02ba24 100644
--- a/youtube_dl/extractor/viewster.py
+++ b/youtube_dl/extractor/viewster.py
@@ -11,6 +11,7 @@ from ..utils import (
     determine_ext,
     int_or_none,
     parse_iso8601,
+    HEADRequest,
 )
 
 
@@ -73,7 +74,7 @@ class ViewsterIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
         # Get 'api_token' cookie
-        self._request_webpage(url, video_id)
+        self._request_webpage(HEADRequest(url), video_id)
         cookies = self._get_cookies(url)
         self._AUTH_TOKEN = compat_urllib_parse_unquote(cookies['api_token'].value)
 

From 44cae2fb2e7b8cffcdfdd57b1224115345009cc5 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 30 Jul 2015 23:55:57 +0100
Subject: [PATCH 2125/6219] [gdcvault] Add support for audio extraction (fixes
 #5784)

Closes #6412.
---
 youtube_dl/extractor/gdcvault.py | 29 ++++++++++++++++++++++-------
 1 file changed, 22 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/gdcvault.py b/youtube_dl/extractor/gdcvault.py
index 43f916412..c3017d451 100644
--- a/youtube_dl/extractor/gdcvault.py
+++ b/youtube_dl/extractor/gdcvault.py
@@ -7,7 +7,10 @@ from ..compat import (
     compat_urllib_parse,
     compat_urllib_request,
 )
-from ..utils import remove_end
+from ..utils import (
+    remove_end,
+    HEADRequest,
+)
 
 
 class GDCVaultIE(InfoExtractor):
@@ -73,10 +76,20 @@ class GDCVaultIE(InfoExtractor):
         return video_formats
 
     def _parse_flv(self, xml_description):
-        video_formats = []
+        formats = []
         akamai_url = xml_description.find('./metadata/akamaiHost').text
+        audios = xml_description.find('./metadata/audios')
+        if audios is not None:
+            for audio in audios:
+                formats.append({
+                    'url': 'rtmp://%s/ondemand?ovpfv=1.1' % akamai_url,
+                    'play_path': remove_end(audio.get('url'), '.flv'),
+                    'ext': 'flv',
+                    'vcodec': 'none',
+                    'format_id': audio.get('code'),
+                })
         slide_video_path = xml_description.find('./metadata/slideVideo').text
-        video_formats.append({
+        formats.append({
             'url': 'rtmp://%s/ondemand?ovpfv=1.1' % akamai_url,
             'play_path': remove_end(slide_video_path, '.flv'),
             'ext': 'flv',
@@ -86,7 +99,7 @@ class GDCVaultIE(InfoExtractor):
             'format_id': 'slides',
         })
         speaker_video_path = xml_description.find('./metadata/speakerVideo').text
-        video_formats.append({
+        formats.append({
             'url': 'rtmp://%s/ondemand?ovpfv=1.1' % akamai_url,
             'play_path': remove_end(speaker_video_path, '.flv'),
             'ext': 'flv',
@@ -95,7 +108,7 @@ class GDCVaultIE(InfoExtractor):
             'preference': -1,
             'format_id': 'speaker',
         })
-        return video_formats
+        return formats
 
     def _login(self, webpage_url, display_id):
         (username, password) = self._get_login_info()
@@ -133,16 +146,18 @@ class GDCVaultIE(InfoExtractor):
             r's1\.addVariable\("file",\s*encodeURIComponent\("(/[^"]+)"\)\);',
             start_page, 'url', default=None)
         if direct_url:
-            video_url = 'http://www.gdcvault.com/' + direct_url
             title = self._html_search_regex(
                 r'<td><strong>Session Name</strong></td>\s*<td>(.*?)</td>',
                 start_page, 'title')
+            video_url = 'http://www.gdcvault.com' + direct_url
+            # resolve the url so that we can detect the correct extension
+            head = self._request_webpage(HEADRequest(video_url), video_id)
+            video_url = head.geturl()
 
             return {
                 'id': video_id,
                 'display_id': display_id,
                 'url': video_url,
-                'ext': 'flv',
                 'title': title,
             }
 

From 525a87f58ee9c4ee91b5b0384184dabf6d87eef3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 1 Aug 2015 11:40:34 +0200
Subject: [PATCH 2126/6219] [gdcvault] Fix typo: xml_decription_url ->
 xml_description_url

---
 youtube_dl/extractor/gdcvault.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/gdcvault.py b/youtube_dl/extractor/gdcvault.py
index c3017d451..a6834db43 100644
--- a/youtube_dl/extractor/gdcvault.py
+++ b/youtube_dl/extractor/gdcvault.py
@@ -183,8 +183,8 @@ class GDCVaultIE(InfoExtractor):
             # Fallback to the older format
             xml_name = self._html_search_regex(r'<iframe src=".*?\?xmlURL=xml/(?P<xml_file>.+?\.xml).*?".*?</iframe>', start_page, 'xml filename')
 
-        xml_decription_url = xml_root + 'xml/' + xml_name
-        xml_description = self._download_xml(xml_decription_url, display_id)
+        xml_description_url = xml_root + 'xml/' + xml_name
+        xml_description = self._download_xml(xml_description_url, display_id)
 
         video_title = xml_description.find('./metadata/title').text
         video_formats = self._parse_mp4(xml_description)

From ee114368ad0bb9822449295910263a99f9de4e1e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 1 Aug 2015 20:22:13 +0600
Subject: [PATCH 2127/6219] [utils] Make value optional for find_xpath_attr

This allows selecting particular attributes by name but without specifying the value and similar to xpath syntax `[@attrib]`
---
 test/test_utils.py  |  9 +++++++++
 youtube_dl/utils.py | 13 ++++++++-----
 2 files changed, 17 insertions(+), 5 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 65692a9fb..a759b2da9 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -235,12 +235,21 @@ class TestUtil(unittest.TestCase):
             <node x="a"/>
             <node x="a" y="c" />
             <node x="b" y="d" />
+            <node x="" />
         </root>'''
         doc = xml.etree.ElementTree.fromstring(testxml)
 
+        self.assertEqual(find_xpath_attr(doc, './/fourohfour', 'n'), None)
         self.assertEqual(find_xpath_attr(doc, './/fourohfour', 'n', 'v'), None)
+        self.assertEqual(find_xpath_attr(doc, './/node', 'n'), None)
+        self.assertEqual(find_xpath_attr(doc, './/node', 'n', 'v'), None)
+        self.assertEqual(find_xpath_attr(doc, './/node', 'x'), doc[1])
         self.assertEqual(find_xpath_attr(doc, './/node', 'x', 'a'), doc[1])
+        self.assertEqual(find_xpath_attr(doc, './/node', 'x', 'b'), doc[3])
+        self.assertEqual(find_xpath_attr(doc, './/node', 'y'), doc[2])
         self.assertEqual(find_xpath_attr(doc, './/node', 'y', 'c'), doc[2])
+        self.assertEqual(find_xpath_attr(doc, './/node', 'y', 'd'), doc[3])
+        self.assertEqual(find_xpath_attr(doc, './/node', 'x', ''), doc[4])
 
     def test_xpath_with_ns(self):
         testxml = '''<root xmlns:media="http://example.com/">
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 88f9f9070..78dc2b449 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -139,21 +139,24 @@ def write_json_file(obj, fn):
 
 
 if sys.version_info >= (2, 7):
-    def find_xpath_attr(node, xpath, key, val):
+    def find_xpath_attr(node, xpath, key, val=None):
         """ Find the xpath xpath[@key=val] """
         assert re.match(r'^[a-zA-Z-]+$', key)
-        assert re.match(r'^[a-zA-Z0-9@\s:._-]*$', val)
-        expr = xpath + "[@%s='%s']" % (key, val)
+        if val:
+            assert re.match(r'^[a-zA-Z0-9@\s:._-]*$', val)
+        expr = xpath + ('[@%s]' % key if val is None else "[@%s='%s']" % (key, val))
         return node.find(expr)
 else:
-    def find_xpath_attr(node, xpath, key, val):
+    def find_xpath_attr(node, xpath, key, val=None):
         # Here comes the crazy part: In 2.6, if the xpath is a unicode,
         # .//node does not match if a node is a direct child of . !
         if isinstance(xpath, compat_str):
             xpath = xpath.encode('ascii')
 
         for f in node.findall(xpath):
-            if f.attrib.get(key) == val:
+            if key not in f.attrib:
+                continue
+            if val is None or f.attrib.get(key) == val:
                 return f
         return None
 

From 3f125c8c70e8109bc90d4446b40740133e343b85 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 1 Aug 2015 21:43:33 +0600
Subject: [PATCH 2128/6219] [nbcnews] Extend _VALID_URL

---
 youtube_dl/extractor/nbc.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index dc2091be0..ccdbfb6c9 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -124,7 +124,7 @@ class NBCSportsIE(InfoExtractor):
 class NBCNewsIE(InfoExtractor):
     _VALID_URL = r'''(?x)https?://(?:www\.)?nbcnews\.com/
         (?:video/.+?/(?P<id>\d+)|
-        (?:feature|nightly-news)/[^/]+/(?P<title>.+))
+        (?:watch|feature|nightly-news)/[^/]+/(?P<title>.+))
         '''
 
     _TESTS = [
@@ -169,6 +169,10 @@ class NBCNewsIE(InfoExtractor):
                 'description': 'md5:1c10c1eccbe84a26e5debb4381e2d3c5',
             },
         },
+        {
+            'url': 'http://www.nbcnews.com/watch/dateline/full-episode--deadly-betrayal-386250819952',
+            'only_matching': True,
+        },
     ]
 
     def _real_extract(self, url):

From 55eae65b39d754d699ad9de3f9c99fcdf62e0176 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 2 Aug 2015 00:42:23 +0800
Subject: [PATCH 2129/6219] Credit @cyb3r for the ir90tv extractor

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index aa6b88cc0..d16d34272 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -136,3 +136,4 @@ sceext
 Zach Bruggeman
 Tjark Saul
 slangangular
+Behrouz Abbasi

From a107193e4b7a3d5414dd7422263c34ac0e309ec4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 2 Aug 2015 01:13:21 +0600
Subject: [PATCH 2130/6219] [extractor/common] Extract f4m and m3u8 formats,
 subtitles and info

---
 youtube_dl/extractor/common.py | 200 ++++++++++++++++++++++++---------
 1 file changed, 149 insertions(+), 51 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index dc5080504..f9578b838 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -18,6 +18,7 @@ from ..compat import (
     compat_HTTPError,
     compat_http_client,
     compat_urllib_error,
+    compat_urllib_parse,
     compat_urllib_parse_urlparse,
     compat_urllib_request,
     compat_urlparse,
@@ -37,6 +38,7 @@ from ..utils import (
     RegexNotFoundError,
     sanitize_filename,
     unescapeHTML,
+    url_basename,
 )
 
 
@@ -978,69 +980,165 @@ class InfoExtractor(object):
         self._sort_formats(formats)
         return formats
 
-    # TODO: improve extraction
-    def _extract_smil_formats(self, smil_url, video_id, fatal=True):
-        smil = self._download_xml(
-            smil_url, video_id, 'Downloading SMIL file',
-            'Unable to download SMIL file', fatal=fatal)
+    @staticmethod
+    def _xpath_ns(path, namespace=None):
+        if not namespace:
+            return path
+        out = []
+        for c in path.split('/'):
+            if not c or c == '.':
+                out.append(c)
+            else:
+                out.append('{%s}%s' % (namespace, c))
+        return '/'.join(out)
+
+    def _extract_smil_formats(self, smil_url, video_id, fatal=True, f4m_params=None):
+        smil = self._download_smil(smil_url, video_id, fatal=fatal)
+
         if smil is False:
             assert not fatal
             return []
 
-        base = smil.find('./head/meta').get('base')
+        namespace = self._search_regex(
+            r'{([^}]+)?}smil', smil.tag, 'namespace', default=None)
+
+        return self._parse_smil_formats(
+            smil, smil_url, video_id, namespace=namespace, f4m_params=f4m_params)
+
+    def _extract_smil_info(self, smil_url, video_id, fatal=True, f4m_params=None):
+        smil = self._download_smil(smil_url, video_id, fatal=fatal)
+        if smil is False:
+            return {}
+        return self._parse_smil(smil, smil_url, video_id, f4m_params=f4m_params)
+
+    def _download_smil(self, smil_url, video_id, fatal=True):
+        return self._download_xml(
+            smil_url, video_id, 'Downloading SMIL file',
+            'Unable to download SMIL file', fatal=fatal)
+
+    def _parse_smil(self, smil, smil_url, video_id, f4m_params=None):
+        namespace = self._search_regex(
+            r'{([^}]+)?}smil', smil.tag, 'namespace', default=None)
+
+        formats = self._parse_smil_formats(
+            smil, smil_url, video_id, namespace=namespace, f4m_params=f4m_params)
+        subtitles = self._parse_smil_subtitles(smil, namespace=namespace)
+
+        video_id = os.path.splitext(url_basename(smil_url))[0]
+        title = None
+        description = None
+        for meta in smil.findall(self._xpath_ns('./head/meta', namespace)):
+            name = meta.attrib.get('name')
+            content = meta.attrib.get('content')
+            if not name or not content:
+                continue
+            if not title and name == 'title':
+                title = content
+            elif not description and name in ('description', 'abstract'):
+                description = content
+
+        return {
+            'id': video_id,
+            'title': title or video_id,
+            'description': description,
+            'formats': formats,
+            'subtitles': subtitles,
+        }
+
+    def _parse_smil_formats(self, smil, smil_url, video_id, namespace=None, f4m_params=None):
+        base = smil_url
+        for meta in smil.findall(self._xpath_ns('./head/meta', namespace)):
+            b = meta.get('base') or meta.get('httpBase')
+            if b:
+                base = b
+                break
 
         formats = []
         rtmp_count = 0
-        if smil.findall('./body/seq/video'):
-            video = smil.findall('./body/seq/video')[0]
-            fmts, rtmp_count = self._parse_smil_video(video, video_id, base, rtmp_count)
-            formats.extend(fmts)
-        else:
-            for video in smil.findall('./body/switch/video'):
-                fmts, rtmp_count = self._parse_smil_video(video, video_id, base, rtmp_count)
-                formats.extend(fmts)
+        http_count = 0
+
+        videos = smil.findall(self._xpath_ns('.//video', namespace))
+        for video in videos:
+            src = video.get('src')
+            if not src:
+                continue
+
+            bitrate = int_or_none(video.get('system-bitrate') or video.get('systemBitrate'), 1000)
+            filesize = int_or_none(video.get('size') or video.get('fileSize'))
+            width = int_or_none(video.get('width'))
+            height = int_or_none(video.get('height'))
+            proto = video.get('proto')
+            ext = video.get('ext')
+            src_ext = determine_ext(src)
+            streamer = video.get('streamer') or base
+
+            if proto == 'rtmp' or streamer.startswith('rtmp'):
+                rtmp_count += 1
+                formats.append({
+                    'url': streamer,
+                    'play_path': src,
+                    'ext': 'flv',
+                    'format_id': 'rtmp-%d' % (rtmp_count if bitrate is None else bitrate),
+                    'tbr': bitrate,
+                    'filesize': filesize,
+                    'width': width,
+                    'height': height,
+                })
+                continue
+
+            src_url = src if src.startswith('http') else compat_urlparse.urljoin(base, src)
+
+            if proto == 'm3u8' or src_ext == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    src_url, video_id, ext or 'mp4', m3u8_id='hls'))
+                continue
+
+            if src_ext == 'f4m':
+                f4m_url = src_url
+                if not f4m_params:
+                    f4m_params = {
+                        'hdcore': '3.2.0',
+                        'plugin': 'flowplayer-3.2.0.1',
+                    }
+                f4m_url += '&' if '?' in f4m_url else '?'
+                f4m_url += compat_urllib_parse.urlencode(f4m_params).encode('utf-8')
+                formats.extend(self._extract_f4m_formats(f4m_url, video_id, f4m_id='hds'))
+                continue
+
+            if src_url.startswith('http'):
+                http_count += 1
+                formats.append({
+                    'url': src_url,
+                    'ext': ext or src_ext or 'flv',
+                    'format_id': 'http-%d' % (bitrate or http_count),
+                    'tbr': bitrate,
+                    'filesize': filesize,
+                    'width': width,
+                    'height': height,
+                })
+                continue
 
         self._sort_formats(formats)
 
         return formats
 
-    def _parse_smil_video(self, video, video_id, base, rtmp_count):
-        src = video.get('src')
-        if not src:
-            return [], rtmp_count
-        bitrate = int_or_none(video.get('system-bitrate') or video.get('systemBitrate'), 1000)
-        width = int_or_none(video.get('width'))
-        height = int_or_none(video.get('height'))
-        proto = video.get('proto')
-        if not proto:
-            if base:
-                if base.startswith('rtmp'):
-                    proto = 'rtmp'
-                elif base.startswith('http'):
-                    proto = 'http'
-        ext = video.get('ext')
-        if proto == 'm3u8':
-            return self._extract_m3u8_formats(src, video_id, ext), rtmp_count
-        elif proto == 'rtmp':
-            rtmp_count += 1
-            streamer = video.get('streamer') or base
-            return ([{
-                'url': streamer,
-                'play_path': src,
-                'ext': 'flv',
-                'format_id': 'rtmp-%d' % (rtmp_count if bitrate is None else bitrate),
-                'tbr': bitrate,
-                'width': width,
-                'height': height,
-            }], rtmp_count)
-        elif proto.startswith('http'):
-            return ([{
-                'url': base + src,
-                'ext': ext or 'flv',
-                'tbr': bitrate,
-                'width': width,
-                'height': height,
-            }], rtmp_count)
+    def _parse_smil_subtitles(self, smil, namespace=None):
+        subtitles = {}
+        for num, textstream in enumerate(smil.findall(self._xpath_ns('.//textstream', namespace))):
+            src = textstream.get('src')
+            if not src:
+                continue
+            ext = textstream.get('ext') or determine_ext(src)
+            if not ext:
+                type_ = textstream.get('type')
+                if type_ == 'text/srt':
+                    ext = 'srt'
+            lang = textstream.get('systemLanguage') or textstream.get('systemLanguageName')
+            subtitles.setdefault(lang, []).append({
+                'url': src,
+                'ext': ext,
+            })
+        return subtitles
 
     def _live_title(self, name):
         """ Generate the title for a live video """

From e5e8d20a3a65832c74b002f247866fcbb92e9246 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 2 Aug 2015 01:13:59 +0600
Subject: [PATCH 2131/6219] [extractor/generic] Improve generic SMIL detection

---
 youtube_dl/extractor/generic.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 8cef61c3c..6900ed96f 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1110,11 +1110,13 @@ class GenericIE(InfoExtractor):
 
         self.report_extraction(video_id)
 
-        # Is it an RSS feed?
+        # Is it an RSS feed or a SMIL file?
         try:
             doc = parse_xml(webpage)
             if doc.tag == 'rss':
                 return self._extract_rss(url, video_id, doc)
+            elif re.match(r'^(?:{[^}]+})?smil$', doc.tag):
+                return self._parse_smil(doc, url, video_id)
         except compat_xml_parse_error:
             pass
 

From 308cfe0ab3ec7122602ba2d6a4e3acd2caa7a757 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 2 Aug 2015 01:14:41 +0600
Subject: [PATCH 2132/6219] [test_downloader] Respect --force-generic-extractor

---
 test/test_download.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/test/test_download.py b/test/test_download.py
index 1110357a7..284418834 100644
--- a/test/test_download.py
+++ b/test/test_download.py
@@ -136,7 +136,9 @@ def generator(test_case):
                     # We're not using .download here sine that is just a shim
                     # for outside error handling, and returns the exit code
                     # instead of the result dict.
-                    res_dict = ydl.extract_info(test_case['url'])
+                    res_dict = ydl.extract_info(
+                        test_case['url'],
+                        force_generic_extractor=params.get('force_generic_extractor', False))
                 except (DownloadError, ExtractorError) as err:
                     # Check if the exception is not a network related one
                     if not err.exc_info[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError, compat_http_client.BadStatusLine) or (err.exc_info[0] == compat_HTTPError and err.exc_info[1].code == 503):

From 645f814544f9d40386e504a1eb8cf3558f2c109e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 2 Aug 2015 01:15:33 +0600
Subject: [PATCH 2133/6219] [test/helper] Allow dicts for mincount

---
 test/helper.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/helper.py b/test/helper.py
index e1129e58f..c8b34654d 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -133,8 +133,8 @@ def expect_info_dict(self, got_dict, expected_dict):
             elif isinstance(expected, compat_str) and expected.startswith('mincount:'):
                 got = got_dict.get(info_field)
                 self.assertTrue(
-                    isinstance(got, list),
-                    'Expected field %s to be a list, but it is of type %s' % (
+                    isinstance(got, (list, dict)),
+                    'Expected field %s to be a list or a dict, but it is of type %s' % (
                         info_field, type(got).__name__))
                 expected_num = int(expected.partition(':')[2])
                 assertGreaterEqual(

From 8765222d2211cd6f2a40611249181af0bbb2d531 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 2 Aug 2015 01:16:21 +0600
Subject: [PATCH 2134/6219] [extractor/generic] Add generic SMIL tests

---
 youtube_dl/extractor/generic.py | 68 +++++++++++++++++++++++++++++++++
 1 file changed, 68 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 6900ed96f..27584c44c 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -130,6 +130,74 @@ class GenericIE(InfoExtractor):
                 'title': 'pdv_maddow_netcast_m4v-02-27-2015-201624',
             }
         },
+        # SMIL from http://videolectures.net/promogram_igor_mekjavic_eng
+        {
+            'url': 'http://videolectures.net/promogram_igor_mekjavic_eng/video/1/smil.xml',
+            'info_dict': {
+                'id': 'smil',
+                'ext': 'mp4',
+                'title': 'Automatics, robotics and biocybernetics',
+                'description': 'md5:815fc1deb6b3a2bff99de2d5325be482',
+                'formats': 'mincount:16',
+                'subtitles': 'mincount:1',
+            },
+            'params': {
+                'force_generic_extractor': True,
+                'skip_download': True,
+            },
+        },
+        # SMIL from http://www1.wdr.de/mediathek/video/livestream/index.html
+        {
+            'url': 'http://metafilegenerator.de/WDR/WDR_FS/hds/hds.smil',
+            'info_dict': {
+                'id': 'hds',
+                'ext': 'flv',
+                'title': 'hds',
+                'formats': 'mincount:1',
+            },
+            'params': {
+                'skip_download': True,
+            },
+        },
+        # SMIL from https://www.restudy.dk/video/play/id/1637
+        {
+            'url': 'https://www.restudy.dk/awsmedia/SmilDirectory/video_1637.xml',
+            'info_dict': {
+                'id': 'video_1637',
+                'ext': 'flv',
+                'title': 'video_1637',
+                'formats': 'mincount:3',
+            },
+            'params': {
+                'skip_download': True,
+            },
+        },
+        # SMIL from http://adventure.howstuffworks.com/5266-cool-jobs-iditarod-musher-video.htm
+        {
+            'url': 'http://services.media.howstuffworks.com/videos/450221/smil-service.smil',
+            'info_dict': {
+                'id': 'smil-service',
+                'ext': 'flv',
+                'title': 'smil-service',
+                'formats': 'mincount:1',
+            },
+            'params': {
+                'skip_download': True,
+            },
+        },
+        # SMIL from http://new.livestream.com/CoheedandCambria/WebsterHall/videos/4719370
+        {
+            'url': 'http://api.new.livestream.com/accounts/1570303/events/1585861/videos/4719370.smil',
+            'info_dict': {
+                'id': '4719370',
+                'ext': 'mp4',
+                'title': '571de1fd-47bc-48db-abf9-238872a58d1f',
+                'formats': 'mincount:3',
+            },
+            'params': {
+                'skip_download': True,
+            },
+        },
         # google redirect
         {
             'url': 'http://www.google.com/url?sa=t&rct=j&q=&esrc=s&source=web&cd=1&cad=rja&ved=0CCUQtwIwAA&url=http%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DcmQHVoWB5FY&ei=F-sNU-LLCaXk4QT52ICQBQ&usg=AFQjCNEw4hL29zgOohLXvpJ-Bdh2bils1Q&bvm=bv.61965928,d.bGE',

From 41c3a5a7beebbf5f60c5edb5093d564f0829c5c1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 2 Aug 2015 01:20:49 +0600
Subject: [PATCH 2135/6219] [extractor/common] Fix python 3

---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index f9578b838..c123d9fca 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1101,7 +1101,7 @@ class InfoExtractor(object):
                         'plugin': 'flowplayer-3.2.0.1',
                     }
                 f4m_url += '&' if '?' in f4m_url else '?'
-                f4m_url += compat_urllib_parse.urlencode(f4m_params).encode('utf-8')
+                f4m_url += compat_urllib_parse.urlencode(f4m_params)
                 formats.extend(self._extract_f4m_formats(f4m_url, video_id, f4m_id='hds'))
                 continue
 

From 17712eeb1933f53696c1fc53606174e988a96472 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 2 Aug 2015 01:31:17 +0600
Subject: [PATCH 2136/6219] [extractor/common] Extract namespace parse routine

---
 youtube_dl/extractor/common.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index c123d9fca..717dcec7b 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -999,8 +999,7 @@ class InfoExtractor(object):
             assert not fatal
             return []
 
-        namespace = self._search_regex(
-            r'{([^}]+)?}smil', smil.tag, 'namespace', default=None)
+        namespace = self._parse_smil_namespace(smil)
 
         return self._parse_smil_formats(
             smil, smil_url, video_id, namespace=namespace, f4m_params=f4m_params)
@@ -1017,8 +1016,7 @@ class InfoExtractor(object):
             'Unable to download SMIL file', fatal=fatal)
 
     def _parse_smil(self, smil, smil_url, video_id, f4m_params=None):
-        namespace = self._search_regex(
-            r'{([^}]+)?}smil', smil.tag, 'namespace', default=None)
+        namespace = self._parse_smil_namespace(smil)
 
         formats = self._parse_smil_formats(
             smil, smil_url, video_id, namespace=namespace, f4m_params=f4m_params)
@@ -1045,6 +1043,10 @@ class InfoExtractor(object):
             'subtitles': subtitles,
         }
 
+    def _parse_smil_namespace(self, smil):
+        return self._search_regex(
+            r'(?i)^{([^}]+)?}smil$', smil.tag, 'namespace', default=None)
+
     def _parse_smil_formats(self, smil, smil_url, video_id, namespace=None, f4m_params=None):
         base = smil_url
         for meta in smil.findall(self._xpath_ns('./head/meta', namespace)):

From fa7a1cc5ef52a8dd9a355ab37a74be55ac2ddc1f Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 22 Jul 2015 12:34:42 +0100
Subject: [PATCH 2137/6219] [screenwavemedia] fix info extraction (fixes #6270)

Closes #6330.
---
 youtube_dl/extractor/screenwavemedia.py | 84 +++++++++++--------------
 1 file changed, 36 insertions(+), 48 deletions(-)

diff --git a/youtube_dl/extractor/screenwavemedia.py b/youtube_dl/extractor/screenwavemedia.py
index d1ab66b32..09c085dcf 100644
--- a/youtube_dl/extractor/screenwavemedia.py
+++ b/youtube_dl/extractor/screenwavemedia.py
@@ -1,12 +1,11 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..utils import (
     int_or_none,
     unified_strdate,
+    js_to_json,
 )
 
 
@@ -22,59 +21,48 @@ class ScreenwaveMediaIE(InfoExtractor):
         video_id = self._match_id(url)
 
         playerdata = self._download_webpage(
-            'http://player.screenwavemedia.com/play/player.php?id=%s' % video_id,
+            'http://player.screenwavemedia.com/player.php?id=%s' % video_id,
             video_id, 'Downloading player webpage')
 
         vidtitle = self._search_regex(
             r'\'vidtitle\'\s*:\s*"([^"]+)"', playerdata, 'vidtitle').replace('\\/', '/')
-        vidurl = self._search_regex(
-            r'\'vidurl\'\s*:\s*"([^"]+)"', playerdata, 'vidurl').replace('\\/', '/')
 
-        videolist_url = None
+        playerconfig = self._download_webpage(
+            'http://player.screenwavemedia.com/player.js',
+            video_id, 'Downloading playerconfig webpage')
 
-        mobj = re.search(r"'videoserver'\s*:\s*'(?P<videoserver>[^']+)'", playerdata)
-        if mobj:
-            videoserver = mobj.group('videoserver')
-            mobj = re.search(r'\'vidid\'\s*:\s*"(?P<vidid>[^\']+)"', playerdata)
-            vidid = mobj.group('vidid') if mobj else video_id
-            videolist_url = 'http://%s/vod/smil:%s.smil/jwplayer.smil' % (videoserver, vidid)
-        else:
-            mobj = re.search(r"file\s*:\s*'(?P<smil>http.+?/jwplayer\.smil)'", playerdata)
-            if mobj:
-                videolist_url = mobj.group('smil')
+        videoserver = self._search_regex(r"'videoserver'\s*:\s*'([^']+)", playerconfig, 'videoserver')
 
-        if videolist_url:
-            videolist = self._download_xml(videolist_url, video_id, 'Downloading videolist XML')
-            formats = []
-            baseurl = vidurl[:vidurl.rfind('/') + 1]
-            for video in videolist.findall('.//video'):
-                src = video.get('src')
-                if not src:
-                    continue
-                file_ = src.partition(':')[-1]
-                width = int_or_none(video.get('width'))
-                height = int_or_none(video.get('height'))
-                bitrate = int_or_none(video.get('system-bitrate'), scale=1000)
-                format = {
-                    'url': baseurl + file_,
-                    'format_id': src.rpartition('.')[0].rpartition('_')[-1],
-                }
-                if width or height:
-                    format.update({
-                        'tbr': bitrate,
-                        'width': width,
-                        'height': height,
-                    })
-                else:
-                    format.update({
-                        'abr': bitrate,
-                        'vcodec': 'none',
-                    })
-                formats.append(format)
-        else:
-            formats = [{
-                'url': vidurl,
-            }]
+        sources = self._parse_json(
+            js_to_json(
+                self._search_regex(
+                    r"sources\s*:\s*(\[[^\]]+?\])", playerconfig,
+                    'sources',
+                ).replace(
+                    "' + thisObj.options.videoserver + '",
+                    videoserver
+                ).replace(
+                    "' + playerVidId + '",
+                    video_id
+                )
+            ),
+            video_id
+        )
+
+        formats = []
+        for source in sources:
+            if source['type'] == 'hls':
+                formats.extend(self._extract_m3u8_formats(source['file'], video_id))
+            else:
+                format_label = source.get('label')
+                height = int_or_none(self._search_regex(
+                    r'^(\d+)[pP]', format_label, 'height', default=None))
+                formats.append({
+                    'url': source['file'],
+                    'format': format_label,
+                    'ext': source.get('type'),
+                    'height': height,
+                })
         self._sort_formats(formats)
 
         return {

From 9cc93c64aa321260475a2bdf7d8626cdd16bf8ad Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 1 Aug 2015 22:15:43 +0200
Subject: [PATCH 2138/6219] [screenwavemedia] Use the IP for the videoserver
 (fixes #6397)

For http://cinemassacre.com/2015/07/28/avgn-seaman-for-dreamcast/ the other server returns a 403 error.
---
 youtube_dl/extractor/screenwavemedia.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/screenwavemedia.py b/youtube_dl/extractor/screenwavemedia.py
index 09c085dcf..3bc84989e 100644
--- a/youtube_dl/extractor/screenwavemedia.py
+++ b/youtube_dl/extractor/screenwavemedia.py
@@ -31,7 +31,7 @@ class ScreenwaveMediaIE(InfoExtractor):
             'http://player.screenwavemedia.com/player.js',
             video_id, 'Downloading playerconfig webpage')
 
-        videoserver = self._search_regex(r"'videoserver'\s*:\s*'([^']+)", playerconfig, 'videoserver')
+        videoserver = self._search_regex(r"\[ipaddress\]\s*=>\s*([\d\.]+)", playerdata, 'videoserver')
 
         sources = self._parse_json(
             js_to_json(

From cdc682d5a467b7188eb13b5eeb76eb5dd544d1f5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 2 Aug 2015 04:21:16 +0600
Subject: [PATCH 2139/6219] [nowtv] Fix extraction (Closes #6357)

---
 youtube_dl/extractor/nowtv.py | 63 +++++++++++++++--------------------
 1 file changed, 27 insertions(+), 36 deletions(-)

diff --git a/youtube_dl/extractor/nowtv.py b/youtube_dl/extractor/nowtv.py
index 0b5ff4760..de6bc6d96 100644
--- a/youtube_dl/extractor/nowtv.py
+++ b/youtube_dl/extractor/nowtv.py
@@ -7,6 +7,7 @@ from .common import InfoExtractor
 from ..compat import compat_str
 from ..utils import (
     ExtractorError,
+    determine_ext,
     int_or_none,
     parse_iso8601,
     parse_duration,
@@ -15,7 +16,7 @@ from ..utils import (
 
 
 class NowTVIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?nowtv\.de/(?P<station>rtl|rtl2|rtlnitro|superrtl|ntv|vox)/(?P<id>.+?)/player'
+    _VALID_URL = r'https?://(?:www\.)?nowtv\.de/(?:rtl|rtl2|rtlnitro|superrtl|ntv|vox)/(?P<id>.+?)/player'
 
     _TESTS = [{
         # rtl
@@ -23,7 +24,7 @@ class NowTVIE(InfoExtractor):
         'info_dict': {
             'id': '203519',
             'display_id': 'bauer-sucht-frau/die-neuen-bauern-und-eine-hochzeit',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': 'Die neuen Bauern und eine Hochzeit',
             'description': 'md5:e234e1ed6d63cf06be5c070442612e7e',
             'thumbnail': 're:^https?://.*\.jpg$',
@@ -32,7 +33,7 @@ class NowTVIE(InfoExtractor):
             'duration': 2786,
         },
         'params': {
-            # m3u8 download
+            # rtmp download
             'skip_download': True,
         },
     }, {
@@ -41,7 +42,7 @@ class NowTVIE(InfoExtractor):
         'info_dict': {
             'id': '203481',
             'display_id': 'berlin-tag-nacht/berlin-tag-nacht-folge-934',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': 'Berlin - Tag & Nacht (Folge 934)',
             'description': 'md5:c85e88c2e36c552dfe63433bc9506dd0',
             'thumbnail': 're:^https?://.*\.jpg$',
@@ -50,7 +51,7 @@ class NowTVIE(InfoExtractor):
             'duration': 2641,
         },
         'params': {
-            # m3u8 download
+            # rtmp download
             'skip_download': True,
         },
     }, {
@@ -59,7 +60,7 @@ class NowTVIE(InfoExtractor):
         'info_dict': {
             'id': '165780',
             'display_id': 'alarm-fuer-cobra-11-die-autobahnpolizei/hals-und-beinbruch-2014-08-23-21-10-00',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': 'Hals- und Beinbruch',
             'description': 'md5:b50d248efffe244e6f56737f0911ca57',
             'thumbnail': 're:^https?://.*\.jpg$',
@@ -68,7 +69,7 @@ class NowTVIE(InfoExtractor):
             'duration': 2742,
         },
         'params': {
-            # m3u8 download
+            # rtmp download
             'skip_download': True,
         },
     }, {
@@ -77,7 +78,7 @@ class NowTVIE(InfoExtractor):
         'info_dict': {
             'id': '99205',
             'display_id': 'medicopter-117/angst',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': 'Angst!',
             'description': 'md5:30cbc4c0b73ec98bcd73c9f2a8c17c4e',
             'thumbnail': 're:^https?://.*\.jpg$',
@@ -86,7 +87,7 @@ class NowTVIE(InfoExtractor):
             'duration': 3025,
         },
         'params': {
-            # m3u8 download
+            # rtmp download
             'skip_download': True,
         },
     }, {
@@ -95,7 +96,7 @@ class NowTVIE(InfoExtractor):
         'info_dict': {
             'id': '203521',
             'display_id': 'ratgeber-geld/thema-ua-der-erste-blick-die-apple-watch',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': 'Thema u.a.: Der erste Blick: Die Apple Watch',
             'description': 'md5:4312b6c9d839ffe7d8caf03865a531af',
             'thumbnail': 're:^https?://.*\.jpg$',
@@ -104,7 +105,7 @@ class NowTVIE(InfoExtractor):
             'duration': 1083,
         },
         'params': {
-            # m3u8 download
+            # rtmp download
             'skip_download': True,
         },
     }, {
@@ -113,7 +114,7 @@ class NowTVIE(InfoExtractor):
         'info_dict': {
             'id': '128953',
             'display_id': 'der-hundeprofi/buero-fall-chihuahua-joel',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': "Büro-Fall / Chihuahua 'Joel'",
             'description': 'md5:e62cb6bf7c3cc669179d4f1eb279ad8d',
             'thumbnail': 're:^https?://.*\.jpg$',
@@ -122,15 +123,13 @@ class NowTVIE(InfoExtractor):
             'duration': 3092,
         },
         'params': {
-            # m3u8 download
+            # rtmp download
             'skip_download': True,
         },
     }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        display_id = mobj.group('id')
-        station = mobj.group('station')
+        display_id = self._match_id(url)
 
         info = self._download_json(
             'https://api.nowtv.de/v3/movies/%s?fields=id,title,free,geoblocked,articleLong,articleShort,broadcastStartDate,seoUrl,duration,format,files' % display_id,
@@ -148,29 +147,19 @@ class NowTVIE(InfoExtractor):
                 raise ExtractorError(
                     'Video %s is not available for free' % video_id, expected=True)
 
-        f = info.get('format', {})
-        station = f.get('station') or station
-
-        STATIONS = {
-            'rtl': 'rtlnow',
-            'rtl2': 'rtl2now',
-            'vox': 'voxnow',
-            'nitro': 'rtlnitronow',
-            'ntv': 'n-tvnow',
-            'superrtl': 'superrtlnow'
-        }
-
         formats = []
         for item in files['items']:
-            item_path = remove_start(item['path'], '/')
-            tbr = int_or_none(item['bitrate'])
-            m3u8_url = 'http://hls.fra.%s.de/hls-vod-enc/%s.m3u8' % (STATIONS[station], item_path)
-            m3u8_url = m3u8_url.replace('now/', 'now/videos/')
+            if determine_ext(item['path']) != 'f4v':
+                continue
+            app, play_path = remove_start(item['path'], '/').split('/', 1)
             formats.append({
-                'url': m3u8_url,
-                'format_id': '%s-%sk' % (item['id'], tbr),
-                'ext': 'mp4',
-                'tbr': tbr,
+                'url': 'rtmpe://fms.rtl.de',
+                'app': app,
+                'play_path': 'mp4:%s' % play_path,
+                'ext': 'flv',
+                'page_url': url,
+                'player_url': 'http://rtl-now.rtl.de/includes/nc_player.swf',
+                'tbr': int_or_none(item.get('bitrate')),
             })
         self._sort_formats(formats)
 
@@ -178,6 +167,8 @@ class NowTVIE(InfoExtractor):
         description = info.get('articleLong') or info.get('articleShort')
         timestamp = parse_iso8601(info.get('broadcastStartDate'), ' ')
         duration = parse_duration(info.get('duration'))
+
+        f = info.get('format', {})
         thumbnail = f.get('defaultImage169Format') or f.get('defaultImage169Logo')
 
         return {

From e422d7f4f78994de8483d2207ab4e00174a2408c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 2 Aug 2015 04:26:59 +0600
Subject: [PATCH 2140/6219] [nowtv] Expand _VALID_URL

---
 youtube_dl/extractor/nowtv.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nowtv.py b/youtube_dl/extractor/nowtv.py
index de6bc6d96..11ce37168 100644
--- a/youtube_dl/extractor/nowtv.py
+++ b/youtube_dl/extractor/nowtv.py
@@ -16,7 +16,7 @@ from ..utils import (
 
 
 class NowTVIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?nowtv\.de/(?:rtl|rtl2|rtlnitro|superrtl|ntv|vox)/(?P<id>.+?)/player'
+    _VALID_URL = r'https?://(?:www\.)?nowtv\.de/(?:rtl|rtl2|rtlnitro|superrtl|ntv|vox)/(?P<id>.+?)/(?:player|preview)'
 
     _TESTS = [{
         # rtl
@@ -126,6 +126,9 @@ class NowTVIE(InfoExtractor):
             # rtmp download
             'skip_download': True,
         },
+    }, {
+        'url': 'http://www.nowtv.de/rtl/bauer-sucht-frau/die-neuen-bauern-und-eine-hochzeit/preview',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From d41d04c0f513ad3b83ab6aee60cf2201710b6063 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 2 Aug 2015 06:35:35 +0600
Subject: [PATCH 2141/6219] [videolectures] Fix _VALID_URL

---
 youtube_dl/extractor/videolecturesnet.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/videolecturesnet.py b/youtube_dl/extractor/videolecturesnet.py
index d6a7eb203..24584dc80 100644
--- a/youtube_dl/extractor/videolecturesnet.py
+++ b/youtube_dl/extractor/videolecturesnet.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 
 class VideoLecturesNetIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?videolectures\.net/(?P<id>[^/#?]+)/'
+    _VALID_URL = r'http://(?:www\.)?videolectures\.net/(?P<id>[^/#?]+)(?:/?[#?].*)?$'
     IE_NAME = 'videolectures.net'
 
     _TEST = {

From 5c45bbe57bd791debfd64052ab030298a7c6b718 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 2 Aug 2015 15:19:30 +0200
Subject: [PATCH 2142/6219] [nowtv] Remove unused import

---
 youtube_dl/extractor/nowtv.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/youtube_dl/extractor/nowtv.py b/youtube_dl/extractor/nowtv.py
index 11ce37168..ad938fb62 100644
--- a/youtube_dl/extractor/nowtv.py
+++ b/youtube_dl/extractor/nowtv.py
@@ -1,8 +1,6 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..compat import compat_str
 from ..utils import (

From 25a4c5a9ed59eca0241922363e83e61172527658 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 2 Aug 2015 15:19:57 +0200
Subject: [PATCH 2143/6219] [dailymotion:playlist] Use an iterator for the
 entries

So that using '--playlist-end' only downloads the required pages (reported in #2175).
---
 youtube_dl/extractor/dailymotion.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index 85d945509..2d90b2224 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -15,7 +15,6 @@ from ..utils import (
     ExtractorError,
     determine_ext,
     int_or_none,
-    orderedSet,
     parse_iso8601,
     str_to_int,
     unescapeHTML,
@@ -278,7 +277,7 @@ class DailymotionPlaylistIE(DailymotionBaseInfoExtractor):
     }]
 
     def _extract_entries(self, id):
-        video_ids = []
+        video_ids = set()
         processed_urls = set()
         for pagenum in itertools.count(1):
             page_url = self._PAGE_TEMPLATE % (id, pagenum)
@@ -291,12 +290,13 @@ class DailymotionPlaylistIE(DailymotionBaseInfoExtractor):
 
             processed_urls.add(urlh.geturl())
 
-            video_ids.extend(re.findall(r'data-xid="(.+?)"', webpage))
+            for video_id in re.findall(r'data-xid="(.+?)"', webpage):
+                if video_id not in video_ids:
+                    yield self.url_result('http://www.dailymotion.com/video/%s' % video_id, 'Dailymotion')
+                    video_ids.add(video_id)
 
             if re.search(self._MORE_PAGES_INDICATOR, webpage) is None:
                 break
-        return [self.url_result('http://www.dailymotion.com/video/%s' % video_id, 'Dailymotion')
-                for video_id in orderedSet(video_ids)]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)

From d7d2a9a3dbf1cef78c5085a4aab5d2f336c64cff Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 2 Aug 2015 03:28:04 +0100
Subject: [PATCH 2144/6219] [utils] restart download if server does not support
 byte ranges

---
 youtube_dl/downloader/http.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index b7f144af9..b2e82cfde 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -57,6 +57,20 @@ class HttpFD(FileDownloader):
             # Establish connection
             try:
                 data = self.ydl.urlopen(request)
+
+                if resume_len > 0:
+                    content_range = data.headers.get('Content-Range')
+                    if content_range:
+                        content_range_m = re.search(r'bytes (\d+)-', content_range)
+                        if content_range_m:
+                            # Content-Range is correct - go on
+                            if resume_len == int(content_range_m.group(1)):
+                                break
+
+                    # Content-Range is invalid - wipe the file and do entire redownload
+                    resume_len = 0
+                    open_mode = 'wb'
+
                 break
             except (compat_urllib_error.HTTPError, ) as err:
                 if (err.code < 500 or err.code >= 600) and err.code != 416:

From 8d5b8b477e4b1051482b21ea451f0de1ce23bce7 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 2 Aug 2015 03:58:02 +0100
Subject: [PATCH 2145/6219] [utils] import re

---
 youtube_dl/downloader/http.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index b2e82cfde..f796ee113 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -4,6 +4,7 @@ import errno
 import os
 import socket
 import time
+import re
 
 from .common import FileDownloader
 from ..compat import (

From c3124c3085e6a9a83ee31ace3a7d528a324c42da Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 3 Aug 2015 02:25:08 +0600
Subject: [PATCH 2146/6219] [downloader/http] Simplify

---
 youtube_dl/downloader/http.py | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index f796ee113..0862e90bb 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -58,20 +58,16 @@ class HttpFD(FileDownloader):
             # Establish connection
             try:
                 data = self.ydl.urlopen(request)
-
                 if resume_len > 0:
                     content_range = data.headers.get('Content-Range')
                     if content_range:
                         content_range_m = re.search(r'bytes (\d+)-', content_range)
-                        if content_range_m:
-                            # Content-Range is correct - go on
-                            if resume_len == int(content_range_m.group(1)):
-                                break
-
+                        # Content-Range is correct - go on
+                    if content_range_m and resume_len == int(content_range_m.group(1)):
+                        break
                     # Content-Range is invalid - wipe the file and do entire redownload
                     resume_len = 0
                     open_mode = 'wb'
-
                 break
             except (compat_urllib_error.HTTPError, ) as err:
                 if (err.code < 500 or err.code >= 600) and err.code != 416:

From 10eaa8ef1d2a9699052af9262aa472456548e99b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 3 Aug 2015 02:25:40 +0600
Subject: [PATCH 2147/6219] [downloader/http] Report unable to resume

---
 youtube_dl/downloader/http.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index 0862e90bb..2f8490f02 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -66,6 +66,7 @@ class HttpFD(FileDownloader):
                     if content_range_m and resume_len == int(content_range_m.group(1)):
                         break
                     # Content-Range is invalid - wipe the file and do entire redownload
+                    self.report_unable_to_resume()
                     resume_len = 0
                     open_mode = 'wb'
                 break

From 84bc4dcb0f678f0a8c9f993e101b9769e3959f76 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 3 Aug 2015 02:27:47 +0600
Subject: [PATCH 2148/6219] [downloader/http] Clarify rationale for
 Content-Range check (#6426)

---
 youtube_dl/downloader/http.py | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index 2f8490f02..a29f5cf31 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -58,14 +58,21 @@ class HttpFD(FileDownloader):
             # Establish connection
             try:
                 data = self.ydl.urlopen(request)
+                # When trying to resume, Content-Range HTTP header of response has to be checked
+                # to match the value of requested Range HTTP header. This is due to a webservers
+                # that don't support resuming and serve a whole file with no Content-Range
+                # set in response despite of requested Range (see
+                # https://github.com/rg3/youtube-dl/issues/6057#issuecomment-126129799)
                 if resume_len > 0:
                     content_range = data.headers.get('Content-Range')
                     if content_range:
                         content_range_m = re.search(r'bytes (\d+)-', content_range)
-                        # Content-Range is correct - go on
-                    if content_range_m and resume_len == int(content_range_m.group(1)):
-                        break
-                    # Content-Range is invalid - wipe the file and do entire redownload
+                        # Content-Range is present and matches requested Range, resume is possible
+                        if content_range_m and resume_len == int(content_range_m.group(1)):
+                            break
+                    # Content-Range is either not present or invalid. Assuming remote webserver is
+                    # trying to send the whole file, resume is not possible, so wiping the local file
+                    # and performing entire redownload
                     self.report_unable_to_resume()
                     resume_len = 0
                     open_mode = 'wb'

From 754e70cf3e74218ae5d840985fbf07bbe274332a Mon Sep 17 00:00:00 2001
From: George Brighton <george@gebn.co.uk>
Date: Sun, 2 Aug 2015 19:21:10 +0100
Subject: [PATCH 2149/6219] [pornhub] Fix video url regular expression.

PornHub seems to have subtly changed their JavaScript. Before, video URL strings were embedded directly in the video's `flashvars_*` object, but they are now assigned to variables of the form `player_quality_*`, which are then added to this object later under the relevant quality key.
---
 youtube_dl/extractor/pornhub.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index 0b7886840..fbaa830d6 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -81,7 +81,7 @@ class PornHubIE(InfoExtractor):
         comment_count = self._extract_count(
             r'All Comments\s*<span>\(([\d,.]+)\)', webpage, 'comment')
 
-        video_urls = list(map(compat_urllib_parse_unquote, re.findall(r'"quality_[0-9]{3}p":"([^"]+)', webpage)))
+        video_urls = list(map(compat_urllib_parse_unquote, re.findall(r"var player_quality_[0-9]{3}p = '([^']+)'", webpage)))
         if webpage.find('"encrypted":true') != -1:
             password = compat_urllib_parse_unquote_plus(
                 self._search_regex(r'"video_title":"([^"]+)', webpage, 'password'))

From 524229a2975c20887a9a71cae77132e775003537 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 3 Aug 2015 02:41:17 +0600
Subject: [PATCH 2150/6219] [pornhub] Improve

---
 youtube_dl/extractor/pornhub.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index fbaa830d6..fec493046 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -81,7 +81,7 @@ class PornHubIE(InfoExtractor):
         comment_count = self._extract_count(
             r'All Comments\s*<span>\(([\d,.]+)\)', webpage, 'comment')
 
-        video_urls = list(map(compat_urllib_parse_unquote, re.findall(r"var player_quality_[0-9]{3}p = '([^']+)'", webpage)))
+        video_urls = list(map(compat_urllib_parse_unquote, re.findall(r"player_quality_[0-9]{3}p\s*=\s*'([^']+)'", webpage)))
         if webpage.find('"encrypted":true') != -1:
             password = compat_urllib_parse_unquote_plus(
                 self._search_regex(r'"video_title":"([^"]+)', webpage, 'password'))

From 51a575159a5a83e4477b03544f419dcf2e9ff0fa Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 2 Aug 2015 22:52:12 +0100
Subject: [PATCH 2151/6219] [facebook] extract uploader

---
 youtube_dl/extractor/facebook.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index e17bb9aea..734de4da2 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -17,6 +17,8 @@ from ..utils import (
     int_or_none,
     limit_length,
     urlencode_postdata,
+    get_element_by_id,
+    clean_html,
 )
 
 
@@ -161,6 +163,7 @@ class FacebookIE(InfoExtractor):
             video_title = limit_length(video_title, 80)
         if not video_title:
             video_title = 'Facebook video #%s' % video_id
+        uploader = clean_html(get_element_by_id('fbPhotoPageAuthorName', webpage))
 
         return {
             'id': video_id,
@@ -168,4 +171,5 @@ class FacebookIE(InfoExtractor):
             'formats': formats,
             'duration': int_or_none(video_data.get('video_duration')),
             'thumbnail': video_data.get('thumbnail_src'),
+            'uploader': uploader,
         }

From 67b8a28a2f69764259cf2e90c0a3785c05c55551 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 3 Aug 2015 00:09:21 +0100
Subject: [PATCH 2152/6219] [facebook] add uploader value to the tests

---
 youtube_dl/extractor/facebook.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index 734de4da2..178a7ca4c 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -44,6 +44,7 @@ class FacebookIE(InfoExtractor):
             'id': '637842556329505',
             'ext': 'mp4',
             'title': 're:Did you know Kei Nishikori is the first Asian man to ever reach a Grand Slam',
+            'uploader': 'Tennis on Facebook',
         }
     }, {
         'note': 'Video without discernible title',
@@ -52,6 +53,7 @@ class FacebookIE(InfoExtractor):
             'id': '274175099429670',
             'ext': 'mp4',
             'title': 'Facebook video #274175099429670',
+            'uploader': 'Asif Nawab Butt',
         },
         'expected_warnings': [
             'title'

From 8de922724b8f3ad31ff7249799de371ff8a5c3ad Mon Sep 17 00:00:00 2001
From: "Sergey M." <dstftw@gmail.com>
Date: Mon, 3 Aug 2015 05:36:17 +0600
Subject: [PATCH 2153/6219] [README.md] Clarify using cookies

---
 README.md | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/README.md b/README.md
index ac54d7b67..2db3139ee 100644
--- a/README.md
+++ b/README.md
@@ -439,6 +439,12 @@ Either prepend `http://www.youtube.com/watch?v=` or separate the ID from the opt
     youtube-dl -- -wNyEUrxzFU
     youtube-dl "http://www.youtube.com/watch?v=-wNyEUrxzFU"
 
+### How do I pass cookies to youtube-dl?
+
+Use the `--cookies` option, for example `--cookies /path/to/cookies/file.txt`. Note that cookies file must be in Mozilla/Netscape format and the first line of cookies file must be either `# HTTP Cookie File` or `# Netscape HTTP Cookie File`. Make sure you have correct [newline format](https://en.wikipedia.org/wiki/Newline) in cookies file and convert newlines if necessary to correspond your OS, namely `CRLF` (`\r\n`) for Windows, `LF` (`\n`) for Linux and `CR` (`\r`) for Mac OS. `HTTP Error 400: Bad Request` when using `--cookies` is a good sign of invalid newline format.
+
+Passing cookies to youtube-dl is a good way to workaround login when particular extractor does not implement it explicitly.
+
 ### Can you add support for this anime video site, or site which shows current movies for free?
 
 As a matter of policy (as well as legality), youtube-dl does not include support for services that specialize in infringing copyright. As a rule of thumb, if you cannot easily find a video that the service is quite obviously allowed to distribute (i.e. that has been uploaded by the creator, the creator's distributor, or is published under a free license), the service is probably unfit for inclusion to youtube-dl.

From 47a8b7c14a085ce558db3b5a85ded850cd5df642 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 3 Aug 2015 12:00:08 +0200
Subject: [PATCH 2154/6219] [mdr] Change XPath to make it work in python 2.6
 (fixes #6443)

The 'progressiveDownloadUrl' element is a direct child, so they should be equivalent.
---
 youtube_dl/extractor/mdr.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mdr.py b/youtube_dl/extractor/mdr.py
index 5fdd19027..fc7499958 100644
--- a/youtube_dl/extractor/mdr.py
+++ b/youtube_dl/extractor/mdr.py
@@ -29,7 +29,7 @@ class MDRIE(InfoExtractor):
         doc = self._download_xml(domain + xmlurl, video_id)
         formats = []
         for a in doc.findall('./assets/asset'):
-            url_el = a.find('.//progressiveDownloadUrl')
+            url_el = a.find('./progressiveDownloadUrl')
             if url_el is None:
                 continue
             abr = int(a.find('bitrateAudio').text) // 1000

From 8f5639afcbb967f276fb8b35a24559cdcc3b6d32 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 3 Aug 2015 19:37:48 +0600
Subject: [PATCH 2155/6219] [pornhub] Improve video quality regex

---
 youtube_dl/extractor/pornhub.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index fec493046..7b0cdc41a 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -94,7 +94,7 @@ class PornHubIE(InfoExtractor):
             format = path.split('/')[5].split('_')[:2]
             format = "-".join(format)
 
-            m = re.match(r'^(?P<height>[0-9]+)P-(?P<tbr>[0-9]+)K$', format)
+            m = re.match(r'^(?P<height>[0-9]+)[pP]-(?P<tbr>[0-9]+)[kK]$', format)
             if m is None:
                 height = None
                 tbr = None

From e704f87f869b98bbed56d7dd0fe27710306c8272 Mon Sep 17 00:00:00 2001
From: Niklas Haas <git@nand.wakku.to>
Date: Mon, 3 Aug 2015 01:54:21 +0200
Subject: [PATCH 2156/6219] [twitch] Parse start_time from 't' (closes #6441)

Eg. for VOD links like http://www.twitch.tv/gamesdonequick/v/9136645?t=14h29m15s
---
 youtube_dl/extractor/twitch.py | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 73ce335b7..a2b6a35aa 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -7,12 +7,15 @@ import random
 
 from .common import InfoExtractor
 from ..compat import (
+    compat_parse_qs,
     compat_str,
     compat_urllib_parse,
+    compat_urllib_parse_urlparse,
     compat_urllib_request,
 )
 from ..utils import (
     ExtractorError,
+    parse_duration,
     parse_iso8601,
 )
 
@@ -185,7 +188,7 @@ class TwitchVodIE(TwitchItemBaseIE):
     _ITEM_SHORTCUT = 'v'
 
     _TEST = {
-        'url': 'http://www.twitch.tv/riotgames/v/6528877',
+        'url': 'http://www.twitch.tv/riotgames/v/6528877?t=5m10s',
         'info_dict': {
             'id': 'v6528877',
             'ext': 'mp4',
@@ -197,6 +200,7 @@ class TwitchVodIE(TwitchItemBaseIE):
             'uploader': 'Riot Games',
             'uploader_id': 'riotgames',
             'view_count': int,
+            'start_time': 310,
         },
         'params': {
             # m3u8 download
@@ -216,6 +220,12 @@ class TwitchVodIE(TwitchItemBaseIE):
             item_id, 'mp4')
         self._prefer_source(formats)
         info['formats'] = formats
+
+        parsed_url = compat_urllib_parse_urlparse(url)
+        query = compat_parse_qs(parsed_url.query)
+        if 't' in query:
+            info['start_time'] = parse_duration(query['t'][0])
+
         return info
 
 

From d96d604e5311628ece0234733dbbfe73a58c8d18 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 3 Aug 2015 23:04:11 +0200
Subject: [PATCH 2157/6219] YoutubeDL: format spec: don't accept a bare '/'
 (#6124)

---
 test/test_YoutubeDL.py  | 1 +
 youtube_dl/YoutubeDL.py | 2 ++
 2 files changed, 3 insertions(+)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 20f45f439..9a3c28f8c 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -311,6 +311,7 @@ class TestFormatSelection(unittest.TestCase):
         assert_syntax_error('bestvideo,,best')
         assert_syntax_error('+bestaudio')
         assert_syntax_error('bestvideo+')
+        assert_syntax_error('/')
 
     def test_format_filtering(self):
         formats = [
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index efa3254ce..c608ff91a 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -960,6 +960,8 @@ class YoutubeDL(object):
                         selectors.append(current_selector)
                         current_selector = None
                     elif string == '/':
+                        if not current_selector:
+                            raise syntax_error('"/" must follow a format selector', start)
                         first_choice = current_selector
                         second_choice = _parse_format_selection(tokens, inside_choice=True)
                         current_selector = FormatSelector(PICKFIRST, (first_choice, second_choice), [])

From a346b1ff57a94382e80fd4edd5a6d4b91a7cb45e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 4 Aug 2015 20:44:22 +0600
Subject: [PATCH 2158/6219] [bbc] Add support for vxp-playlist-data embeds
 (Closes #6453)

---
 youtube_dl/extractor/bbc.py | 45 ++++++++++++++++++++++++++++++++-----
 1 file changed, 40 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 9a1b6e3dc..abc5a44a1 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -526,6 +526,18 @@ class BBCIE(BBCCoUkIE):
         'params': {
             'skip_download': True,
         }
+    }, {
+        # single video from video playlist embedded with vxp-playlist-data JSON
+        'url': 'http://www.bbc.com/news/video_and_audio/must_see/33376376',
+        'info_dict': {
+            'id': 'p02w6qjc',
+            'ext': 'mp4',
+            'title': '''Judge Mindy Glazer: "I'm sorry to see you here... I always wondered what happened to you"''',
+            'duration': 56,
+        },
+        'params': {
+            'skip_download': True,
+        }
     }, {
         # single video story with digitalData
         'url': 'http://www.bbc.com/travel/story/20150625-sri-lankas-spicy-secret',
@@ -695,13 +707,36 @@ class BBCIE(BBCCoUkIE):
 
         if not medias:
             # Single video article (e.g. http://www.bbc.com/news/video_and_audio/international)
-            media_asset_page = self._parse_json(
+            media_asset = self._search_regex(
+                r'mediaAssetPage\.init\(\s*({.+?}), "/',
+                webpage, 'media asset', default=None)
+            if media_asset:
+                media_asset_page = self._parse_json(media_asset, playlist_id, fatal=False)
+                medias = []
+                for video in media_asset_page.get('videos', {}).values():
+                    medias.extend(video.values())
+
+        if not medias:
+            # Multiple video playlist with single `now playing` entry (e.g.
+            # http://www.bbc.com/news/video_and_audio/must_see/33767813)
+            vxp_playlist = self._parse_json(
                 self._search_regex(
-                    r'mediaAssetPage\.init\(\s*({.+?}), "/', webpage, 'media asset'),
+                    r'<script[^>]+class="vxp-playlist-data"[^>]+type="application/json"[^>]*>([^<]+)</script>',
+                    webpage, 'playlist data'),
                 playlist_id)
-            medias = []
-            for video in media_asset_page.get('videos', {}).values():
-                medias.extend(video.values())
+            playlist_medias = []
+            for item in vxp_playlist:
+                media = item.get('media')
+                if not media:
+                    continue
+                playlist_medias.append(media)
+                # Download single video if found media with asset id matching the video id from URL
+                if item.get('advert', {}).get('assetId') == playlist_id:
+                    medias = [media]
+                    break
+            # Fallback to the whole playlist
+            if not medias:
+                medias = playlist_medias
 
         entries = []
         for num, media_meta in enumerate(medias, start=1):

From 232541df441741d3d55605f03e28ec3c34249a5b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 4 Aug 2015 22:29:23 +0200
Subject: [PATCH 2159/6219] [YoutubeDL] format spec: correctly handle dashes
 and other unused operators

'mp4-baseline-16x9' must be handled as a single string, but the '-' was treated as an operator.
---
 test/test_YoutubeDL.py  |  6 ++++++
 youtube_dl/YoutubeDL.py | 33 ++++++++++++++++++++++++++++++++-
 2 files changed, 38 insertions(+), 1 deletion(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 9a3c28f8c..0388c0bf3 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -105,6 +105,7 @@ class TestFormatSelection(unittest.TestCase):
     def test_format_selection(self):
         formats = [
             {'format_id': '35', 'ext': 'mp4', 'preference': 1, 'url': TEST_URL},
+            {'format_id': 'example-with-dashes', 'ext': 'webm', 'preference': 1, 'url': TEST_URL},
             {'format_id': '45', 'ext': 'webm', 'preference': 2, 'url': TEST_URL},
             {'format_id': '47', 'ext': 'webm', 'preference': 3, 'url': TEST_URL},
             {'format_id': '2', 'ext': 'flv', 'preference': 4, 'url': TEST_URL},
@@ -136,6 +137,11 @@ class TestFormatSelection(unittest.TestCase):
         downloaded = ydl.downloaded_info_dicts[0]
         self.assertEqual(downloaded['format_id'], '35')
 
+        ydl = YDL({'format': 'example-with-dashes'})
+        ydl.process_ie_result(info_dict.copy())
+        downloaded = ydl.downloaded_info_dicts[0]
+        self.assertEqual(downloaded['format_id'], 'example-with-dashes')
+
     def test_format_selection_audio(self):
         formats = [
             {'format_id': 'audio-low', 'ext': 'webm', 'preference': 1, 'vcodec': 'none', 'url': TEST_URL},
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index c608ff91a..1446b3254 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -933,6 +933,37 @@ class YoutubeDL(object):
                 else:
                     filter_parts.append(string)
 
+        def _remove_unused_ops(tokens):
+            # Remove operators that we don't use and join them with the sourrounding strings
+            # for example: 'mp4' '-' 'baseline' '-' '16x9' is converted to 'mp4-baseline-16x9'
+            ALLOWED_OPS = ('/', '+', ',', '(', ')')
+            last_string, last_start, last_end, last_line = None, None, None, None
+            for type, string, start, end, line in tokens:
+                if type == tokenize.OP and string == '[':
+                    if last_string:
+                        yield tokenize.NAME, last_string, last_start, last_end, last_line
+                        last_string = None
+                    yield type, string, start, end, line
+                    # everything inside brackets will be handled by _parse_filter
+                    for type, string, start, end, line in tokens:
+                        yield type, string, start, end, line
+                        if type == tokenize.OP and string == ']':
+                            break
+                elif type == tokenize.OP and string in ALLOWED_OPS:
+                    if last_string:
+                        yield tokenize.NAME, last_string, last_start, last_end, last_line
+                        last_string = None
+                    yield type, string, start, end, line
+                elif type in [tokenize.NAME, tokenize.NUMBER, tokenize.OP]:
+                    if not last_string:
+                        last_string = string
+                        last_start = start
+                        last_end = end
+                    else:
+                        last_string += string
+            if last_string:
+                yield tokenize.NAME, last_string, last_start, last_end, last_line
+
         def _parse_format_selection(tokens, inside_merge=False, inside_choice=False, inside_group=False):
             selectors = []
             current_selector = None
@@ -1111,7 +1142,7 @@ class YoutubeDL(object):
 
         stream = io.BytesIO(format_spec.encode('utf-8'))
         try:
-            tokens = list(compat_tokenize_tokenize(stream.readline))
+            tokens = list(_remove_unused_ops(compat_tokenize_tokenize(stream.readline)))
         except tokenize.TokenError:
             raise syntax_error('Missing closing/opening brackets or parenthesis', (0, len(format_spec)))
 

From 8a5601e42f6974e6694f01089b4c7e014b6a1b7a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 5 Aug 2015 19:52:04 +0600
Subject: [PATCH 2160/6219] [lynda] Fix login (Closes #6462)

---
 youtube_dl/extractor/lynda.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index a00f6e5e5..39214de2f 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -17,7 +17,7 @@ from ..utils import (
 
 class LyndaBaseIE(InfoExtractor):
     _LOGIN_URL = 'https://www.lynda.com/login/login.aspx'
-    _SUCCESSFUL_LOGIN_REGEX = r'isLoggedIn: true'
+    _SUCCESSFUL_LOGIN_REGEX = r'isLoggedIn\s*:\s*true'
     _ACCOUNT_CREDENTIALS_HINT = 'Use --username and --password options to provide lynda.com account credentials.'
     _NETRC_MACHINE = 'lynda'
 

From 5b7dab2dd640c93ec0f63ca8b901e701679a4c7b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 5 Aug 2015 20:06:48 +0600
Subject: [PATCH 2161/6219] [lynda] Make login more robust

---
 youtube_dl/extractor/lynda.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index 39214de2f..deead220a 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -17,7 +17,6 @@ from ..utils import (
 
 class LyndaBaseIE(InfoExtractor):
     _LOGIN_URL = 'https://www.lynda.com/login/login.aspx'
-    _SUCCESSFUL_LOGIN_REGEX = r'isLoggedIn\s*:\s*true'
     _ACCOUNT_CREDENTIALS_HINT = 'Use --username and --password options to provide lynda.com account credentials.'
     _NETRC_MACHINE = 'lynda'
 
@@ -41,7 +40,7 @@ class LyndaBaseIE(InfoExtractor):
             request, None, 'Logging in as %s' % username)
 
         # Not (yet) logged in
-        m = re.search(r'loginResultJson = \'(?P<json>[^\']+)\';', login_page)
+        m = re.search(r'loginResultJson\s*=\s*\'(?P<json>[^\']+)\';', login_page)
         if m is not None:
             response = m.group('json')
             response_json = json.loads(response)
@@ -70,7 +69,7 @@ class LyndaBaseIE(InfoExtractor):
                     request, None,
                     'Confirming log in and log out from another device')
 
-        if re.search(self._SUCCESSFUL_LOGIN_REGEX, login_page) is None:
+        if all(not re.search(p, login_page) for p in ('isLoggedIn\s*:\s*true', r'logout\.aspx', r'>Log out<')):
             raise ExtractorError('Unable to log in')
 
 

From 354b4b8604ec13ccf4bd89b9d1b77cb7246fe379 Mon Sep 17 00:00:00 2001
From: vijayanand nandam <vijay@cybrilla.com>
Date: Wed, 5 Aug 2015 19:37:59 +0530
Subject: [PATCH 2162/6219] fixing xhamster file extraction

---
 youtube_dl/extractor/xhamster.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index b4ad513a0..9d025530f 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -47,7 +47,7 @@ class XHamsterIE(InfoExtractor):
 
     def _real_extract(self, url):
         def extract_video_url(webpage):
-            mp4 = re.search(r'<video\s+.*?file="([^"]+)".*?>', webpage)
+            mp4 = re.search(r'file:\s+\'([^\']+)\'', webpage)
             if mp4 is None:
                 raise ExtractorError('Unable to extract media URL')
             else:

From be7a8379b47c35afe66abcc02aee597e5143b1d5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 5 Aug 2015 20:32:44 +0600
Subject: [PATCH 2163/6219] [xhamster] Make more robust

---
 youtube_dl/extractor/xhamster.py | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index 9d025530f..481d79b89 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -46,12 +46,12 @@ class XHamsterIE(InfoExtractor):
     ]
 
     def _real_extract(self, url):
-        def extract_video_url(webpage):
-            mp4 = re.search(r'file:\s+\'([^\']+)\'', webpage)
-            if mp4 is None:
-                raise ExtractorError('Unable to extract media URL')
-            else:
-                return mp4.group(1)
+        def extract_video_url(webpage, name):
+            return self._search_regex(
+                [r'''file\s*:\s*(?P<q>["'])(?P<mp4>.+?)(?P=q)''',
+                 r'''<a\s+href=(?P<q>["'])(?P<mp4>.+?)(?P=q)\s+class=["']mp4Thumb''',
+                 r'''<video[^>]+file=(?P<q>["'])(?P<mp4>.+?)(?P=q)[^>]*>'''],
+                webpage, name, group='mp4')
 
         def is_hd(webpage):
             return '<div class=\'icon iconHD\'' in webpage
@@ -97,7 +97,9 @@ class XHamsterIE(InfoExtractor):
 
         hd = is_hd(webpage)
 
-        video_url = extract_video_url(webpage)
+        format_id = 'hd' if hd else 'sd'
+
+        video_url = extract_video_url(webpage, format_id)
         formats = [{
             'url': video_url,
             'format_id': 'hd' if hd else 'sd',
@@ -108,7 +110,7 @@ class XHamsterIE(InfoExtractor):
             mrss_url = self._search_regex(r'<link rel="canonical" href="([^"]+)', webpage, 'mrss_url')
             webpage = self._download_webpage(mrss_url + '?hd', video_id, note='Downloading HD webpage')
             if is_hd(webpage):
-                video_url = extract_video_url(webpage)
+                video_url = extract_video_url(webpage, 'hd')
                 formats.append({
                     'url': video_url,
                     'format_id': 'hd',

From 251a44b776264c17d7799e017b856143c6cacd9a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 5 Aug 2015 20:36:37 +0600
Subject: [PATCH 2164/6219] [xhamster] Fix thumbnail extraction

---
 youtube_dl/extractor/xhamster.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index 481d79b89..b57e7c813 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -78,7 +78,10 @@ class XHamsterIE(InfoExtractor):
         uploader_id = self._html_search_regex(r'<a href=\'/user/[^>]+>(?P<uploader_id>[^<]+)',
                                               webpage, 'uploader id', default='anonymous')
 
-        thumbnail = self._html_search_regex(r'<video\s+.*?poster="([^"]+)".*?>', webpage, 'thumbnail', fatal=False)
+        thumbnail = self._search_regex(
+            [r'''thumb\s*:\s*(?P<q>["'])(?P<thumbnail>.+?)(?P=q)''',
+             r'''<video[^>]+poster=(?P<q>["'])(?P<thumbnail>.+?)(?P=q)[^>]*>'''],
+             webpage, 'thumbnail', fatal=False, group='thumbnail')
 
         duration = parse_duration(self._html_search_regex(r'<span>Runtime:</span> (\d+:\d+)</div>',
                                                           webpage, 'duration', fatal=False))

From 3e4852247744b131600ba43275ab321eb1b32bb1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 5 Aug 2015 20:41:40 +0600
Subject: [PATCH 2165/6219] [xhamster] Fix uploader extraction

---
 youtube_dl/extractor/xhamster.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index b57e7c813..06fedf840 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -22,7 +22,7 @@ class XHamsterIE(InfoExtractor):
                 'ext': 'mp4',
                 'title': 'FemaleAgent Shy beauty takes the bait',
                 'upload_date': '20121014',
-                'uploader_id': 'Ruseful2011',
+                'uploader': 'Ruseful2011',
                 'duration': 893,
                 'age_limit': 18,
             }
@@ -34,7 +34,7 @@ class XHamsterIE(InfoExtractor):
                 'ext': 'mp4',
                 'title': 'Britney Spears  Sexy Booty',
                 'upload_date': '20130914',
-                'uploader_id': 'jojo747400',
+                'uploader': 'jojo747400',
                 'duration': 200,
                 'age_limit': 18,
             }
@@ -75,8 +75,9 @@ class XHamsterIE(InfoExtractor):
         if upload_date:
             upload_date = unified_strdate(upload_date)
 
-        uploader_id = self._html_search_regex(r'<a href=\'/user/[^>]+>(?P<uploader_id>[^<]+)',
-                                              webpage, 'uploader id', default='anonymous')
+        uploader = self._html_search_regex(
+            r"<a href='[^']+xhamster\.com/user/[^>]+>(?P<uploader>[^<]+)",
+            webpage, 'uploader', default='anonymous')
 
         thumbnail = self._search_regex(
             [r'''thumb\s*:\s*(?P<q>["'])(?P<thumbnail>.+?)(?P=q)''',
@@ -127,7 +128,7 @@ class XHamsterIE(InfoExtractor):
             'title': title,
             'description': description,
             'upload_date': upload_date,
-            'uploader_id': uploader_id,
+            'uploader': uploader,
             'thumbnail': thumbnail,
             'duration': duration,
             'view_count': view_count,

From 54a9328b205e8a2c916d59fd81bdb1ede25cf87a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 5 Aug 2015 21:19:52 +0600
Subject: [PATCH 2166/6219] [generic] Expand jwplayer support

---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 8cef61c3c..6df89f814 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1655,7 +1655,7 @@ class GenericIE(InfoExtractor):
         if not found:
             # Broaden the findall a little bit: JWPlayer JS loader
             found = filter_video(re.findall(
-                r'[^A-Za-z0-9]?file["\']?:\s*["\'](http(?![^\'"]+\.[0-9]+[\'"])[^\'"]+)["\']', webpage))
+                r'[^A-Za-z0-9]?(?:file|video_url)["\']?:\s*["\'](http(?![^\'"]+\.[0-9]+[\'"])[^\'"]+)["\']', webpage))
         if not found:
             # Flow player
             found = filter_video(re.findall(r'''(?xs)

From c71a3195afa8c2a9ed5fe0ffa56ff6c969147d91 Mon Sep 17 00:00:00 2001
From: Delon <liuxi326@qq.com>
Date: Wed, 5 Aug 2015 18:22:25 +0800
Subject: [PATCH 2167/6219] [tudou] Fix extracion

---
 youtube_dl/extractor/tudou.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/tudou.py b/youtube_dl/extractor/tudou.py
index c89de5ba4..9b934cb57 100644
--- a/youtube_dl/extractor/tudou.py
+++ b/youtube_dl/extractor/tudou.py
@@ -29,6 +29,8 @@ class TudouIE(InfoExtractor):
         }
     }]
 
+    _PLAYER_URL = 'http://js.tudouui.com/bin/lingtong/PortalPlayer_177.swf'
+	
     def _url_for_id(self, id, quality=None):
         info_url = "http://v2.tudou.com/f?id=" + str(id)
         if quality:
@@ -76,6 +78,9 @@ class TudouIE(InfoExtractor):
                 'ext': ext,
                 'title': title,
                 'thumbnail': thumbnail_url,
+                'http_headers': {
+                    'Referer': self._PLAYER_URL,
+                },
             }
             result.append(part_info)
 

From 238755752f4f9169a1edda91067c8627afe19cce Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 5 Aug 2015 23:07:52 +0600
Subject: [PATCH 2168/6219] [tudou] Extract player URL from the webpage

---
 youtube_dl/extractor/tudou.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/tudou.py b/youtube_dl/extractor/tudou.py
index 9b934cb57..84fe71aef 100644
--- a/youtube_dl/extractor/tudou.py
+++ b/youtube_dl/extractor/tudou.py
@@ -30,7 +30,7 @@ class TudouIE(InfoExtractor):
     }]
 
     _PLAYER_URL = 'http://js.tudouui.com/bin/lingtong/PortalPlayer_177.swf'
-	
+
     def _url_for_id(self, id, quality=None):
         info_url = "http://v2.tudou.com/f?id=" + str(id)
         if quality:
@@ -56,6 +56,10 @@ class TudouIE(InfoExtractor):
         thumbnail_url = self._search_regex(
             r",pic:\s*[\"'](.+?)[\"']", webpage, 'thumbnail URL', fatal=False)
 
+        player_url = self._search_regex(
+            r"playerUrl\s*:\s*['\"](.+?\.swf)[\"']",
+            webpage, 'player URL', default=self._PLAYER_URL)
+
         segs_json = self._search_regex(r'segs: \'(.*)\'', webpage, 'segments')
         segments = json.loads(segs_json)
         # It looks like the keys are the arguments that have to be passed as
@@ -79,7 +83,7 @@ class TudouIE(InfoExtractor):
                 'title': title,
                 'thumbnail': thumbnail_url,
                 'http_headers': {
-                    'Referer': self._PLAYER_URL,
+                    'Referer': player_url,
                 },
             }
             result.append(part_info)

From f535ec8278c8f465b47919d3f451571ae8ccfc7b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 5 Aug 2015 23:08:26 +0600
Subject: [PATCH 2169/6219] [xhamster] Remove unused import

---
 youtube_dl/extractor/xhamster.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index 06fedf840..f76ee8fd4 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -4,7 +4,6 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
-    ExtractorError,
     unified_strdate,
     str_to_int,
     int_or_none,

From c73cdd800f0dc7b465ac0b36d338875bb80c23aa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 5 Aug 2015 23:08:55 +0600
Subject: [PATCH 2170/6219] [xhamster] flake8

---
 youtube_dl/extractor/xhamster.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index f76ee8fd4..97315750f 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -81,7 +81,7 @@ class XHamsterIE(InfoExtractor):
         thumbnail = self._search_regex(
             [r'''thumb\s*:\s*(?P<q>["'])(?P<thumbnail>.+?)(?P=q)''',
              r'''<video[^>]+poster=(?P<q>["'])(?P<thumbnail>.+?)(?P=q)[^>]*>'''],
-             webpage, 'thumbnail', fatal=False, group='thumbnail')
+            webpage, 'thumbnail', fatal=False, group='thumbnail')
 
         duration = parse_duration(self._html_search_regex(r'<span>Runtime:</span> (\d+:\d+)</div>',
                                                           webpage, 'duration', fatal=False))

From 51f267d9d4d26c3cd67f318a2040513946f2b4d3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 6 Aug 2015 22:01:01 +0600
Subject: [PATCH 2171/6219] [YoutubeDL:utils] Move percent encode non-ASCII
 URLs workaround to http_request and simplify (Closes #6457)

---
 youtube_dl/YoutubeDL.py | 21 ---------------------
 youtube_dl/utils.py     | 20 ++++++++++++++++++++
 2 files changed, 20 insertions(+), 21 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 1446b3254..079d42ce8 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1860,27 +1860,6 @@ class YoutubeDL(object):
 
     def urlopen(self, req):
         """ Start an HTTP download """
-
-        # According to RFC 3986, URLs can not contain non-ASCII characters, however this is not
-        # always respected by websites, some tend to give out URLs with non percent-encoded
-        # non-ASCII characters (see telemb.py, ard.py [#3412])
-        # urllib chokes on URLs with non-ASCII characters (see http://bugs.python.org/issue3991)
-        # To work around aforementioned issue we will replace request's original URL with
-        # percent-encoded one
-        req_is_string = isinstance(req, compat_basestring)
-        url = req if req_is_string else req.get_full_url()
-        url_escaped = escape_url(url)
-
-        # Substitute URL if any change after escaping
-        if url != url_escaped:
-            if req_is_string:
-                req = url_escaped
-            else:
-                req_type = HEADRequest if req.get_method() == 'HEAD' else compat_urllib_request.Request
-                req = req_type(
-                    url_escaped, data=req.data, headers=req.headers,
-                    origin_req_host=req.origin_req_host, unverifiable=req.unverifiable)
-
         return self._opener.open(req, timeout=self._socket_timeout)
 
     def print_debug_header(self):
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 78dc2b449..c7db75f80 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -651,6 +651,26 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
         return ret
 
     def http_request(self, req):
+        # According to RFC 3986, URLs can not contain non-ASCII characters, however this is not
+        # always respected by websites, some tend to give out URLs with non percent-encoded
+        # non-ASCII characters (see telemb.py, ard.py [#3412])
+        # urllib chokes on URLs with non-ASCII characters (see http://bugs.python.org/issue3991)
+        # To work around aforementioned issue we will replace request's original URL with
+        # percent-encoded one
+        # Since redirects are also affected (e.g. http://www.southpark.de/alle-episoden/s18e09)
+        # the code of this workaround has been moved here from YoutubeDL.urlopen()
+        url = req.get_full_url()
+        url_escaped = escape_url(url)
+
+        # Substitute URL if any change after escaping
+        if url != url_escaped:
+            req_type = HEADRequest if req.get_method() == 'HEAD' else compat_urllib_request.Request
+            new_req = req_type(
+                url_escaped, data=req.data, headers=req.headers,
+                origin_req_host=req.origin_req_host, unverifiable=req.unverifiable)
+            new_req.timeout = req.timeout
+            req = new_req
+
         for h, v in std_headers.items():
             # Capitalize is needed because of Python bug 2275: http://bugs.python.org/issue2275
             # The dict keys are capitalized because of this bug by urllib

From bd690a9f9368095f561184778fb2f3ef12c66342 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 6 Aug 2015 22:01:31 +0600
Subject: [PATCH 2172/6219] [southpark:de] Add test for non-ASCII in URLs

---
 youtube_dl/extractor/southpark.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/youtube_dl/extractor/southpark.py b/youtube_dl/extractor/southpark.py
index 7fb165a87..87b650468 100644
--- a/youtube_dl/extractor/southpark.py
+++ b/youtube_dl/extractor/southpark.py
@@ -45,6 +45,14 @@ class SouthParkDeIE(SouthParkIE):
             'title': 'The Government Won\'t Respect My Privacy',
             'description': 'Cartman explains the benefits of "Shitter" to Stan, Kyle and Craig.',
         },
+    }, {
+        # non-ASCII characters in initial URL
+        'url': 'http://www.southpark.de/alle-episoden/s18e09-hashtag-aufwärmen',
+        'playlist_count': 4,
+    }, {
+        # non-ASCII characters in redirect URL
+        'url': 'http://www.southpark.de/alle-episoden/s18e09',
+        'playlist_count': 4,
     }]
 
 

From 4f34cdb0a87a506d25a352ff265678c86cb9b979 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 6 Aug 2015 23:56:44 +0600
Subject: [PATCH 2173/6219] [southpark:de] Skip test

---
 youtube_dl/extractor/southpark.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/southpark.py b/youtube_dl/extractor/southpark.py
index 87b650468..ad63a8785 100644
--- a/youtube_dl/extractor/southpark.py
+++ b/youtube_dl/extractor/southpark.py
@@ -53,6 +53,7 @@ class SouthParkDeIE(SouthParkIE):
         # non-ASCII characters in redirect URL
         'url': 'http://www.southpark.de/alle-episoden/s18e09',
         'playlist_count': 4,
+        'skip': 'Broken python 3',
     }]
 
 

From 671302b5c0ff8cefa5f26e599423ef7799b19631 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 7 Aug 2015 00:08:11 +0600
Subject: [PATCH 2174/6219] [YoutubeDL] Remove unused imports

---
 youtube_dl/YoutubeDL.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 079d42ce8..cad6b026e 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -28,7 +28,6 @@ if os.name == 'nt':
     import ctypes
 
 from .compat import (
-    compat_basestring,
     compat_cookiejar,
     compat_expanduser,
     compat_get_terminal_size,
@@ -40,7 +39,6 @@ from .compat import (
     compat_urllib_request,
 )
 from .utils import (
-    escape_url,
     ContentTooShortError,
     date_from_str,
     DateRange,
@@ -51,7 +49,6 @@ from .utils import (
     ExtractorError,
     format_bytes,
     formatSeconds,
-    HEADRequest,
     locked_file,
     make_HTTPS_handler,
     MaxDownloadsReached,

From cd6b555e19c601d575679dd29da0080eda7f8890 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 6 Aug 2015 19:17:50 +0100
Subject: [PATCH 2175/6219] [dcn] add origin to api request and fix the test
 and check with flake8

---
 youtube_dl/extractor/dcn.py | 30 ++++++++++++++++++++----------
 1 file changed, 20 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/dcn.py b/youtube_dl/extractor/dcn.py
index f76ebda9e..d44e8cef0 100644
--- a/youtube_dl/extractor/dcn.py
+++ b/youtube_dl/extractor/dcn.py
@@ -1,4 +1,9 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
 from .common import InfoExtractor
+from ..compat import compat_urllib_request
+
 
 class DcnIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?dcndigital\.ae/(?:#/)?(?:video/.+|show/\d+/.+?)/(?P<id>\d+)/?'
@@ -9,24 +14,29 @@ class DcnIE(InfoExtractor):
             'id': '17375',
             'ext': 'm3u8',
             'title': 'رحلة العمر : الحلقة 1',
-            'description': '"في هذه الحلقة من برنامج رحلة العمر يقدّم الدكتور عمر عبد الكافي تبسيطاً لمناسك الحج والعمرة ويجيب مباشرة على استفسارات حجاج بيت الله الحرام بخصوص مناسك الحج والعمرة1"',
+            'description': 'في هذه الحلقة من برنامج رحلة العمر يقدّم الدكتور عمر عبد الكافي تبسيطاً لمناسك الحج والعمرة ويجيب مباشرة على استفسارات حجاج بيت الله الحرام بخصوص مناسك الحج والعمرة\n1',
             'thumbnail': 'http://admin.mangomolo.com/analytics/uploads/71/images/media/2/2cefc09d7bec80afa754682f40e49503.jpg',
             'duration': '2041'
-        }
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
     }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        json_data = self._download_json(
-            'http://admin.mangomolo.com/analytics/index.php/plus/video?id='+video_id,
-            video_id
+        request = compat_urllib_request.Request(
+            'http://admin.mangomolo.com/analytics/index.php/plus/video?id=' + video_id,
+            headers={'Origin': 'http://www.dcndigital.ae'}
         )
-        title = json_data['title_ar'];
-        thumbnail = 'http://admin.mangomolo.com/analytics/'+json_data['img'];
-        duration = json_data['duration'];
-        description = json_data['description_ar'];
+        json_data = self._download_json(request, video_id)
+        title = json_data['title_ar']
+        thumbnail = 'http://admin.mangomolo.com/analytics/' + json_data['img']
+        duration = json_data['duration']
+        description = json_data['description_ar']
         webpage = self._download_webpage(
-            'http://admin.mangomolo.com/analytics/index.php/customers/embed/video?id='+json_data['id']+'&user_id='+json_data['user_id']+'&countries=Q0M=&w=100%&h=100%&filter=DENY&signature='+json_data['signature'],
+            'http://admin.mangomolo.com/analytics/index.php/customers/embed/video?id=' + json_data['id'] + '&user_id=' + json_data['user_id'] + '&countries=Q0M=&w=100%&h=100%&filter=DENY&signature=' + json_data['signature'],
             video_id
         )
         m3u8_url = self._html_search_regex(

From 3be3c622dc1d3d7b92c5268a079d202a9f2b0a5a Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 6 Aug 2015 19:37:45 +0100
Subject: [PATCH 2176/6219] [shahid] generic errors handling and check with
 flake8

---
 youtube_dl/extractor/shahid.py | 42 ++++++++++++++++++----------------
 1 file changed, 22 insertions(+), 20 deletions(-)

diff --git a/youtube_dl/extractor/shahid.py b/youtube_dl/extractor/shahid.py
index b3b45da24..57c159833 100644
--- a/youtube_dl/extractor/shahid.py
+++ b/youtube_dl/extractor/shahid.py
@@ -1,3 +1,6 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
 from .common import InfoExtractor
 from ..utils import (
     js_to_json,
@@ -5,6 +8,7 @@ from ..utils import (
     int_or_none
 )
 
+
 class ShahidIE(InfoExtractor):
     _VALID_URL = r'https?://shahid\.mbc\.net/ar/episode/(?P<id>\d+)/?'
     _TESTS = [
@@ -23,7 +27,7 @@ class ShahidIE(InfoExtractor):
             }
         },
         {
-            #shahid plus subscriber only
+            # shahid plus subscriber only
             'url': 'https://shahid.mbc.net/ar/series/90497/%D9%85%D8%B1%D8%A7%D9%8A%D8%A7-2011.html',
             'only_matching': True
         }
@@ -32,31 +36,15 @@ class ShahidIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
+
         player_info = ''
-        for line in self._search_regex( 'var flashvars = ({[^}]+})', webpage, 'flashvars').splitlines():
+        for line in self._search_regex('var flashvars = ({[^}]+})', webpage, 'flashvars').splitlines():
             if '+' not in line and '(' not in line and ')' not in line:
                 player_info += line
         player_info = self._parse_json(js_to_json(player_info), video_id)
         video_id = player_info['id']
         player_type = player_info['playerType']
 
-        video_info = self._download_json(
-            player_info['url'] + '/' + player_type + '/' + video_id + 
-                '?apiKey=sh%40hid0nlin3&hash=b2wMCTHpSmyxGqQjJFOycRmLSex%2BBpTK%2Fooxy6vHaqs%3D',
-            video_id
-        )['data']
-        if video_info['error']:
-            for error in video_info['error']:
-                raise ExtractorError(error)
-        video_info = video_info[player_type]
-        if video_info.get('availabilities').get('plus'):
-            raise ExtractorError('plus members only')
-        title = video_info['title']
-        thumbnail = video_info.get('thumbnailUrl')
-        categories = [category['name'] for category in video_info.get('genres')]
-        description = video_info.get('description')
-        duration = int_or_none(video_info.get('duration'))
-
         player_json_data = self._download_json(
             'https://shahid.mbc.net/arContent/getPlayerContent-param-.id-' + video_id + '.type-' + player_info['type'] + '.html',
             video_id
@@ -66,8 +54,22 @@ class ShahidIE(InfoExtractor):
         else:
             for error in player_json_data['error'].values():
                 raise ExtractorError(error)
-            return
         formats = self._extract_m3u8_formats(m3u8_url, video_id)
+
+        video_info = self._download_json(
+            player_info['url'] + '/' + player_type + '/' + video_id + '?apiKey=sh%40hid0nlin3&hash=b2wMCTHpSmyxGqQjJFOycRmLSex%2BBpTK%2Fooxy6vHaqs%3D',
+            video_id
+        )['data']
+        if video_info.get('error'):
+            for error in video_info['error']:
+                raise ExtractorError(error)
+        video_info = video_info[player_type]
+        title = video_info['title']
+        thumbnail = video_info.get('thumbnailUrl')
+        categories = [category['name'] for category in video_info.get('genres')]
+        description = video_info.get('description')
+        duration = int_or_none(video_info.get('duration'))
+
         return {
             'id': video_id,
             'title': title,

From 5a4d9ddb218e761fe7ab15d197690e0cb132a536 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 7 Aug 2015 01:26:40 +0600
Subject: [PATCH 2177/6219] [utils] Percent-encode redirect URL of Location
 header (Closes #6457)

---
 youtube_dl/utils.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index c7db75f80..e265c7574 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -715,6 +715,17 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
             gz = io.BytesIO(self.deflate(resp.read()))
             resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
             resp.msg = old_resp.msg
+        # Percent-encode redirect URL of Location HTTP header to satisfy RFC 3986
+        if 300 <= resp.code < 400:
+            location = resp.headers.get('Location')
+            if location:
+                # As of RFC 2616 default charset is iso-8859-1 that is respected by python 3
+                if sys.version_info >= (3, 0):
+                    location = location.encode('iso-8859-1').decode('utf-8')
+                location_escaped = escape_url(location)
+                if location != location_escaped:
+                    del resp.headers['Location']
+                    resp.headers['Location'] = location_escaped
         return resp
 
     https_request = http_request

From 9663bd3abb78911bddad75742bd41006677d628e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 7 Aug 2015 01:27:07 +0600
Subject: [PATCH 2178/6219] [southpark:de] Enable non-ASCII redirect URL test

---
 youtube_dl/extractor/southpark.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/southpark.py b/youtube_dl/extractor/southpark.py
index ad63a8785..87b650468 100644
--- a/youtube_dl/extractor/southpark.py
+++ b/youtube_dl/extractor/southpark.py
@@ -53,7 +53,6 @@ class SouthParkDeIE(SouthParkIE):
         # non-ASCII characters in redirect URL
         'url': 'http://www.southpark.de/alle-episoden/s18e09',
         'playlist_count': 4,
-        'skip': 'Broken python 3',
     }]
 
 

From 3eb5fdb58112032a9831eda1d2e3b8a151ea217f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 6 Aug 2015 22:55:43 +0200
Subject: [PATCH 2179/6219] release 2015.08.06

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index fa157cadb..b81d5e658 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.07.28'
+__version__ = '2015.08.06'

From 430b092a5f59fbe407b92ebcb0c42b9f7062a334 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 6 Aug 2015 23:06:21 +0200
Subject: [PATCH 2180/6219] release 2015.08.06.1

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index b81d5e658..9f209499c 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.08.06'
+__version__ = '2015.08.06.1'

From 6d30cf04db9c9662dbb30c2490e24eb5c6dca4c3 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 7 Aug 2015 10:01:18 +0100
Subject: [PATCH 2181/6219] [dcn] fix type and key errors

---
 youtube_dl/extractor/dcn.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/dcn.py b/youtube_dl/extractor/dcn.py
index d44e8cef0..22ff35b56 100644
--- a/youtube_dl/extractor/dcn.py
+++ b/youtube_dl/extractor/dcn.py
@@ -3,6 +3,7 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..compat import compat_urllib_request
+from ..utils import int_or_none
 
 
 class DcnIE(InfoExtractor):
@@ -16,7 +17,7 @@ class DcnIE(InfoExtractor):
             'title': 'رحلة العمر : الحلقة 1',
             'description': 'في هذه الحلقة من برنامج رحلة العمر يقدّم الدكتور عمر عبد الكافي تبسيطاً لمناسك الحج والعمرة ويجيب مباشرة على استفسارات حجاج بيت الله الحرام بخصوص مناسك الحج والعمرة\n1',
             'thumbnail': 'http://admin.mangomolo.com/analytics/uploads/71/images/media/2/2cefc09d7bec80afa754682f40e49503.jpg',
-            'duration': '2041'
+            'duration': 2041
         },
         'params': {
             # m3u8 download
@@ -32,9 +33,9 @@ class DcnIE(InfoExtractor):
         )
         json_data = self._download_json(request, video_id)
         title = json_data['title_ar']
-        thumbnail = 'http://admin.mangomolo.com/analytics/' + json_data['img']
-        duration = json_data['duration']
-        description = json_data['description_ar']
+        thumbnail = 'http://admin.mangomolo.com/analytics/' + json_data.get('img')
+        duration = int_or_none(json_data.get('duration'))
+        description = json_data.get('description_ar')
         webpage = self._download_webpage(
             'http://admin.mangomolo.com/analytics/index.php/customers/embed/video?id=' + json_data['id'] + '&user_id=' + json_data['user_id'] + '&countries=Q0M=&w=100%&h=100%&filter=DENY&signature=' + json_data['signature'],
             video_id

From 8002ac9e0a88d918735c06599dbf8f2005f79666 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 7 Aug 2015 19:04:44 +0600
Subject: [PATCH 2182/6219] [nowtv] Add support for .at TLD

---
 youtube_dl/extractor/nowtv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nowtv.py b/youtube_dl/extractor/nowtv.py
index ad938fb62..78e8851c0 100644
--- a/youtube_dl/extractor/nowtv.py
+++ b/youtube_dl/extractor/nowtv.py
@@ -14,7 +14,7 @@ from ..utils import (
 
 
 class NowTVIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?nowtv\.de/(?:rtl|rtl2|rtlnitro|superrtl|ntv|vox)/(?P<id>.+?)/(?:player|preview)'
+    _VALID_URL = r'https?://(?:www\.)?nowtv\.(?:de|at)/(?:rtl|rtl2|rtlnitro|superrtl|ntv|vox)/(?P<id>.+?)/(?:player|preview)'
 
     _TESTS = [{
         # rtl

From acc1adbe7ab93657cd4d303cee1fba4464931a50 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 7 Aug 2015 19:50:54 +0600
Subject: [PATCH 2183/6219] [nowtv] Add support for .ch TLD

---
 youtube_dl/extractor/nowtv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nowtv.py b/youtube_dl/extractor/nowtv.py
index 78e8851c0..fc21d8e3f 100644
--- a/youtube_dl/extractor/nowtv.py
+++ b/youtube_dl/extractor/nowtv.py
@@ -14,7 +14,7 @@ from ..utils import (
 
 
 class NowTVIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?nowtv\.(?:de|at)/(?:rtl|rtl2|rtlnitro|superrtl|ntv|vox)/(?P<id>.+?)/(?:player|preview)'
+    _VALID_URL = r'https?://(?:www\.)?nowtv\.(?:de|at|ch)/(?:rtl|rtl2|rtlnitro|superrtl|ntv|vox)/(?P<id>.+?)/(?:player|preview)'
 
     _TESTS = [{
         # rtl

From 0f422256d6eea5aff062a4c35d7434cd118c7a0b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 7 Aug 2015 19:51:09 +0600
Subject: [PATCH 2184/6219] [nowtv] Add .at test

---
 youtube_dl/extractor/nowtv.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/nowtv.py b/youtube_dl/extractor/nowtv.py
index fc21d8e3f..66c627bec 100644
--- a/youtube_dl/extractor/nowtv.py
+++ b/youtube_dl/extractor/nowtv.py
@@ -127,6 +127,9 @@ class NowTVIE(InfoExtractor):
     }, {
         'url': 'http://www.nowtv.de/rtl/bauer-sucht-frau/die-neuen-bauern-und-eine-hochzeit/preview',
         'only_matching': True,
+    }, {
+        'url': 'http://www.nowtv.at/rtl/bauer-sucht-frau/die-neuen-bauern-und-eine-hochzeit/preview?return=/rtl/bauer-sucht-frau/die-neuen-bauern-und-eine-hochzeit',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From f94639fadf91312bf3365802981f506ecba698dc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Aug 2015 00:06:03 +0600
Subject: [PATCH 2185/6219] [dcn] Improve

---
 youtube_dl/extractor/__init__.py |  2 +-
 youtube_dl/extractor/dcn.py      | 78 ++++++++++++++++++++++----------
 2 files changed, 54 insertions(+), 26 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index eb8ef1fe3..922d9b3d8 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -118,7 +118,7 @@ from .dailymotion import (
 )
 from .daum import DaumIE
 from .dbtv import DBTVIE
-from .dcn import DcnIE
+from .dcn import DCNIE
 from .dctp import DctpTvIE
 from .deezer import DeezerPlaylistIE
 from .dfb import DFBIE
diff --git a/youtube_dl/extractor/dcn.py b/youtube_dl/extractor/dcn.py
index 22ff35b56..b98a6c032 100644
--- a/youtube_dl/extractor/dcn.py
+++ b/youtube_dl/extractor/dcn.py
@@ -2,22 +2,30 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_request
-from ..utils import int_or_none
+from ..compat import (
+    compat_urllib_parse,
+    compat_urllib_request,
+)
+from ..utils import (
+    int_or_none,
+    parse_iso8601,
+)
 
 
-class DcnIE(InfoExtractor):
+class DCNIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?dcndigital\.ae/(?:#/)?(?:video/.+|show/\d+/.+?)/(?P<id>\d+)/?'
     _TEST = {
         'url': 'http://www.dcndigital.ae/#/show/199074/%D8%B1%D8%AD%D9%84%D8%A9-%D8%A7%D9%84%D8%B9%D9%85%D8%B1-%D8%A7%D9%84%D8%AD%D9%84%D9%82%D8%A9-1/17375/6887',
         'info_dict':
         {
             'id': '17375',
-            'ext': 'm3u8',
+            'ext': 'mp4',
             'title': 'رحلة العمر : الحلقة 1',
-            'description': 'في هذه الحلقة من برنامج رحلة العمر يقدّم الدكتور عمر عبد الكافي تبسيطاً لمناسك الحج والعمرة ويجيب مباشرة على استفسارات حجاج بيت الله الحرام بخصوص مناسك الحج والعمرة\n1',
-            'thumbnail': 'http://admin.mangomolo.com/analytics/uploads/71/images/media/2/2cefc09d7bec80afa754682f40e49503.jpg',
-            'duration': 2041
+            'description': 'md5:0156e935d870acb8ef0a66d24070c6d6',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 2041,
+            'timestamp': 1227504126,
+            'upload_date': '20081124',
         },
         'params': {
             # m3u8 download
@@ -27,30 +35,50 @@ class DcnIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
+
         request = compat_urllib_request.Request(
-            'http://admin.mangomolo.com/analytics/index.php/plus/video?id=' + video_id,
-            headers={'Origin': 'http://www.dcndigital.ae'}
-        )
-        json_data = self._download_json(request, video_id)
-        title = json_data['title_ar']
-        thumbnail = 'http://admin.mangomolo.com/analytics/' + json_data.get('img')
-        duration = int_or_none(json_data.get('duration'))
-        description = json_data.get('description_ar')
+            'http://admin.mangomolo.com/analytics/index.php/plus/video?id=%s' % video_id,
+            headers={'Origin': 'http://www.dcndigital.ae'})
+
+        video = self._download_json(request, video_id)
+        title = video.get('title_en') or video['title_ar']
+
         webpage = self._download_webpage(
-            'http://admin.mangomolo.com/analytics/index.php/customers/embed/video?id=' + json_data['id'] + '&user_id=' + json_data['user_id'] + '&countries=Q0M=&w=100%&h=100%&filter=DENY&signature=' + json_data['signature'],
-            video_id
-        )
-        m3u8_url = self._html_search_regex(
-            r'file:\s*"([^"]+)',
-            webpage,
-            'm3u8_url'
-        )
-        formats = self._extract_m3u8_formats(m3u8_url, video_id)
+            'http://admin.mangomolo.com/analytics/index.php/customers/embed/video?'
+            + compat_urllib_parse.urlencode({
+                'id': video['id'],
+                'user_id': video['user_id'],
+                'signature': video['signature'],
+                'countries': 'Q0M=',
+                'filter': 'DENY',
+            }), video_id)
+
+        m3u8_url = self._html_search_regex(r'file:\s*"([^"]+)', webpage, 'm3u8 url')
+        formats = self._extract_m3u8_formats(
+            m3u8_url, video_id, 'mp4', entry_protocol='m3u8_native', m3u8_id='hls')
+
+        rtsp_url = self._search_regex(
+            r'<a[^>]+href="(rtsp://[^"]+)"', webpage, 'rtsp url', fatal=False)
+        if rtsp_url:
+            formats.append({
+                'url': rtsp_url,
+                'format_id': 'rtsp',
+            })
+
+        self._sort_formats(formats)
+
+        img = video.get('img')
+        thumbnail = 'http://admin.mangomolo.com/analytics/%s' % img if img else None
+        duration = int_or_none(video.get('duration'))
+        description = video.get('description_en') or video.get('description_ar')
+        timestamp = parse_iso8601(video.get('create_time') or video.get('update_time'), ' ')
+
         return {
             'id': video_id,
             'title': title,
+            'description': description,
             'thumbnail': thumbnail,
             'duration': duration,
-            'description': description,
+            'timestamp': timestamp,
             'formats': formats,
         }

From 4a7434d0b09e14b773c2d278c8299efa6225b84e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Aug 2015 00:19:40 +0600
Subject: [PATCH 2186/6219] [dcn] Simplify _VALID_URL

---
 youtube_dl/extractor/dcn.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/dcn.py b/youtube_dl/extractor/dcn.py
index b98a6c032..82261e25c 100644
--- a/youtube_dl/extractor/dcn.py
+++ b/youtube_dl/extractor/dcn.py
@@ -13,7 +13,7 @@ from ..utils import (
 
 
 class DCNIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?dcndigital\.ae/(?:#/)?(?:video/.+|show/\d+/.+?)/(?P<id>\d+)/?'
+    _VALID_URL = r'https?://(?:www\.)?dcndigital\.ae/(?:#/)?(?:video/.+|show/\d+/.+?)/(?P<id>\d+)'
     _TEST = {
         'url': 'http://www.dcndigital.ae/#/show/199074/%D8%B1%D8%AD%D9%84%D8%A9-%D8%A7%D9%84%D8%B9%D9%85%D8%B1-%D8%A7%D9%84%D8%AD%D9%84%D9%82%D8%A9-1/17375/6887',
         'info_dict':

From fd5d8270dcd6d8baada3390a4a1cae5bdbcb6da4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Aug 2015 01:10:41 +0600
Subject: [PATCH 2187/6219] [clipfish] Fix extraction, minimize requests, get
 rid of drm hds, extract m3u8 and more metadata

---
 youtube_dl/extractor/clipfish.py | 56 ++++++++++++++++++++------------
 1 file changed, 35 insertions(+), 21 deletions(-)

diff --git a/youtube_dl/extractor/clipfish.py b/youtube_dl/extractor/clipfish.py
index 09dfaac60..7af903571 100644
--- a/youtube_dl/extractor/clipfish.py
+++ b/youtube_dl/extractor/clipfish.py
@@ -1,18 +1,19 @@
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..utils import (
-    ExtractorError,
+    determine_ext,
     int_or_none,
     js_to_json,
-    determine_ext,
+    parse_iso8601,
+    remove_end,
 )
 
 
 class ClipfishIE(InfoExtractor):
-    IE_NAME = 'clipfish'
-
-    _VALID_URL = r'^https?://(?:www\.)?clipfish\.de/.*?/video/(?P<id>[0-9]+)/'
+    _VALID_URL = r'https?://(?:www\.)?clipfish\.de/(?:[^/]+/)+video/(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://www.clipfish.de/special/game-trailer/video/3966754/fifa-14-e3-2013-trailer/',
         'md5': '79bc922f3e8a9097b3d68a93780fd475',
@@ -20,35 +21,48 @@ class ClipfishIE(InfoExtractor):
             'id': '3966754',
             'ext': 'mp4',
             'title': 'FIFA 14 - E3 2013 Trailer',
+            'timestamp': 1370938118,
+            'upload_date': '20130611',
             'duration': 82,
         }
     }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-        video_info = self._parse_json(
-            js_to_json(self._html_search_regex('var videoObject = ({[^}]+?})', webpage, 'videoObject')),
-            video_id
-        )
-        info_url = self._parse_json(
-            js_to_json(self._html_search_regex('var globalFlashvars = ({[^}]+?})', webpage, 'globalFlashvars')),
-            video_id
-        )['data']
 
-        doc = self._download_xml(
-            info_url, video_id, note='Downloading info page')
-        title = doc.find('title').text
-        video_url = doc.find('filename').text
-        thumbnail = doc.find('imageurl').text
-        duration = int_or_none(video_info['length'])
-        formats = [{'url': video_info['videourl']},{'url': video_url}]
+        webpage = self._download_webpage(url, video_id)
+
+        video_info = self._parse_json(
+            js_to_json(self._html_search_regex(
+                '(?s)videoObject\s*=\s*({.+?});', webpage, 'video object')),
+            video_id)
+
+        formats = []
+        for video_url in re.findall(r'var\s+videourl\s*=\s*"([^"]+)"', webpage):
+            ext = determine_ext(video_url)
+            if ext == 'm3u8':
+                formats.append({
+                    'url': video_url.replace('de.hls.fra.clipfish.de', 'hls.fra.clipfish.de'),
+                    'ext': 'mp4',
+                    'format_id': 'hls',
+                })
+            else:
+                formats.append({
+                    'url': video_url,
+                    'format_id': ext,
+                })
         self._sort_formats(formats)
 
+        title = remove_end(self._og_search_title(webpage), ' - Video')
+        thumbnail = self._og_search_thumbnail(webpage)
+        duration = int_or_none(video_info.get('length'))
+        timestamp = parse_iso8601(self._html_search_meta('uploadDate', webpage, 'upload date'))
+
         return {
             'id': video_id,
             'title': title,
             'formats': formats,
             'thumbnail': thumbnail,
             'duration': duration,
+            'timestamp': timestamp,
         }

From 8a37aa1517ccc474b3e2831b77e48534cb8ed47c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Aug 2015 01:55:59 +0600
Subject: [PATCH 2188/6219] [extractor/generic] Expand ooyala regex (Closes
 #6485)

---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 6df89f814..649c0bce6 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1320,7 +1320,7 @@ class GenericIE(InfoExtractor):
             return self.url_result(mobj.group('url'))
 
         # Look for Ooyala videos
-        mobj = (re.search(r'player\.ooyala\.com/[^"?]+\?[^"]*?(?:embedCode|ec)=(?P<ec>[^"&]+)', webpage) or
+        mobj = (re.search(r'player\.ooyala\.com/[^"?]+[?#][^"]*?(?:embedCode|ec)=(?P<ec>[^"&]+)', webpage) or
                 re.search(r'OO\.Player\.create\([\'"].*?[\'"],\s*[\'"](?P<ec>.{32})[\'"]', webpage) or
                 re.search(r'SBN\.VideoLinkset\.ooyala\([\'"](?P<ec>.{32})[\'"]\)', webpage) or
                 re.search(r'data-ooyala-video-id\s*=\s*[\'"](?P<ec>.{32})[\'"]', webpage))

From bf94d763ba73e09fd77d25110c7219254b63c786 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Aug 2015 02:00:49 +0600
Subject: [PATCH 2189/6219] [extractor/generic] Add test for #6485

---
 youtube_dl/extractor/generic.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 649c0bce6..469909a51 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -236,6 +236,19 @@ class GenericIE(InfoExtractor):
             },
             'add_ie': ['Ooyala'],
         },
+        {
+            # ooyala video embedded with http://player.ooyala.com/iframe.js
+            'url': 'http://www.macrumors.com/2015/07/24/steve-jobs-the-man-in-the-machine-first-trailer/',
+            'info_dict': {
+                'id': 'p0MGJndjoG5SOKqO_hZJuZFPB-Tr5VgB',
+                'ext': 'mp4',
+                'title': '"Steve Jobs: Man in the Machine" trailer',
+                'description': 'The first trailer for the Alex Gibney documentary "Steve Jobs: Man in the Machine."',
+            },
+            'params': {
+                'skip_download': True,
+            },
+        },
         # multiple ooyala embeds on SBN network websites
         {
             'url': 'http://www.sbnation.com/college-football-recruiting/2015/2/3/7970291/national-signing-day-rationalizations-itll-be-ok-itll-be-ok',

From c29458f3ec77072e9c17169b78871bf4473134d6 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 7 Aug 2015 21:38:50 +0100
Subject: [PATCH 2190/6219] [shahid] change the tests

---
 youtube_dl/extractor/shahid.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/shahid.py b/youtube_dl/extractor/shahid.py
index 57c159833..b2050525e 100644
--- a/youtube_dl/extractor/shahid.py
+++ b/youtube_dl/extractor/shahid.py
@@ -13,13 +13,13 @@ class ShahidIE(InfoExtractor):
     _VALID_URL = r'https?://shahid\.mbc\.net/ar/episode/(?P<id>\d+)/?'
     _TESTS = [
         {
-            'url': 'https://shahid.mbc.net/ar/episode/108084/%D8%AE%D9%88%D8%A7%D8%B7%D8%B1-%D8%A7%D9%84%D9%85%D9%88%D8%B3%D9%85-11-%D8%A7%D9%84%D8%AD%D9%84%D9%82%D8%A9-1.html',
+            'url': 'https://shahid.mbc.net/ar/episode/90574/%D8%A7%D9%84%D9%85%D9%84%D9%83-%D8%B9%D8%A8%D8%AF%D8%A7%D9%84%D9%84%D9%87-%D8%A7%D9%84%D8%A5%D9%86%D8%B3%D8%A7%D9%86-%D8%A7%D9%84%D9%85%D9%88%D8%B3%D9%85-1-%D9%83%D9%84%D9%8A%D8%A8-3.html',
             'info_dict': {
-                'id': '108084',
+                'id': '90574',
                 'ext': 'm3u8',
-                'title': 'خواطر الموسم 11 الحلقة 1',
-                'description': 'بسم الله',
-                'duration': 1166,
+                'title': 'الملك عبدالله الإنسان الموسم 1 كليب 3',
+                'description': 'الفيلم الوثائقي - الملك عبد الله الإنسان',
+                'duration': 2972,
             },
             'params': {
                 # m3u8 download
@@ -28,7 +28,7 @@ class ShahidIE(InfoExtractor):
         },
         {
             # shahid plus subscriber only
-            'url': 'https://shahid.mbc.net/ar/series/90497/%D9%85%D8%B1%D8%A7%D9%8A%D8%A7-2011.html',
+            'url': 'https://shahid.mbc.net/ar/episode/90511/%D9%85%D8%B1%D8%A7%D9%8A%D8%A7-2011-%D8%A7%D9%84%D9%85%D9%88%D8%B3%D9%85-1-%D8%A7%D9%84%D8%AD%D9%84%D9%82%D8%A9-1.html',
             'only_matching': True
         }
     ]

From e0ac521438218e978b9c4bbcd92cfc2d5fef79cb Mon Sep 17 00:00:00 2001
From: vijayanand nandam <vijay@cybrilla.com>
Date: Thu, 6 Aug 2015 22:42:58 +0530
Subject: [PATCH 2191/6219] adding support for axel download manager

---
 youtube_dl/downloader/external.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index 1d5cc9904..30699934b 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -83,6 +83,16 @@ class CurlFD(ExternalFD):
         return cmd
 
 
+class AxelFD(ExternalFD):
+    def _make_cmd(self, tmpfilename, info_dict):
+        cmd = [self.exe, '-o', tmpfilename]
+        for key, val in info_dict['http_headers'].items():
+            cmd += ['-H', '%s: %s' % (key, val)]
+        cmd += self._configuration_args()
+        cmd += ['--', info_dict['url']]
+        return cmd
+
+
 class WgetFD(ExternalFD):
     def _make_cmd(self, tmpfilename, info_dict):
         cmd = [self.exe, '-O', tmpfilename, '-nv', '--no-cookies']

From 5b0c40da24b5ddb789428de731e02ac8759a363c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Aug 2015 03:36:29 +0600
Subject: [PATCH 2192/6219] [extractor/common] Expand meta regex

---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index dc5080504..507ea5ec0 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -636,7 +636,7 @@ class InfoExtractor(object):
     @staticmethod
     def _meta_regex(prop):
         return r'''(?isx)<meta
-                    (?=[^>]+(?:itemprop|name|property)=(["\']?)%s\1)
+                    (?=[^>]+(?:itemprop|name|property|id)=(["\']?)%s\1)
                     [^>]+?content=(["\'])(?P<content>.*?)\2''' % re.escape(prop)
 
     def _og_search_property(self, prop, html, name=None, **kargs):

From 3550821fb4ca2f0e47542a7fa16b6543b06df724 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Aug 2015 03:38:55 +0600
Subject: [PATCH 2193/6219] [periscope] Add extractor (Closes #5850, closes
 #6459)

---
 youtube_dl/extractor/__init__.py  |  1 +
 youtube_dl/extractor/periscope.py | 66 +++++++++++++++++++++++++++++++
 2 files changed, 67 insertions(+)
 create mode 100644 youtube_dl/extractor/periscope.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 922d9b3d8..bd86a5be2 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -432,6 +432,7 @@ from .orf import (
 from .parliamentliveuk import ParliamentLiveUKIE
 from .patreon import PatreonIE
 from .pbs import PBSIE
+from .periscope import PeriscopeIE
 from .philharmoniedeparis import PhilharmonieDeParisIE
 from .phoenix import PhoenixIE
 from .photobucket import PhotobucketIE
diff --git a/youtube_dl/extractor/periscope.py b/youtube_dl/extractor/periscope.py
new file mode 100644
index 000000000..5219e1a75
--- /dev/null
+++ b/youtube_dl/extractor/periscope.py
@@ -0,0 +1,66 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    parse_iso8601,
+    unescapeHTML,
+)
+
+
+class PeriscopeIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?periscope\.tv/w/(?P<id>[^/?#]+)'
+    _TEST = {
+        'url': 'https://www.periscope.tv/w/aJUQnjY3MjA3ODF8NTYxMDIyMDl2zCg2pECBgwTqRpQuQD352EMPTKQjT4uqlM3cgWFA-g==',
+        'md5': '65b57957972e503fcbbaeed8f4fa04ca',
+        'info_dict': {
+            'id': '56102209',
+            'ext': 'mp4',
+            'title': 'Bec Boop - 🚠✈️🇬🇧 Fly above #London in Emirates Air Line cable car at night 🇬🇧✈️🚠 #BoopScope 🎀💗',
+            'timestamp': 1438978559,
+            'upload_date': '20150807',
+            'uploader': 'Bec Boop',
+            'uploader_id': '1465763',
+        },
+        'skip': 'Expires in 24 hours',
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        replay = self._download_json(
+            'https://api.periscope.tv/api/v2/getAccessPublic?token=%s' % video_id, video_id)
+
+        video_url = replay['replay_url']
+
+        webpage = self._download_webpage(url, video_id)
+
+        broadcast_data = self._parse_json(
+            unescapeHTML(self._html_search_meta(
+                'broadcast-data', webpage, 'broadcast data', fatal=True)),
+            video_id)
+
+        broadcast = broadcast_data['broadcast']
+        status = broadcast['status']
+
+        uploader = broadcast.get('user_display_name') or broadcast_data.get('user', {}).get('display_name')
+        uploader_id = broadcast.get('user_id') or broadcast_data.get('user', {}).get('id')
+
+        title = '%s - %s' % (uploader, status) if uploader else status
+        timestamp = parse_iso8601(broadcast.get('created_at'))
+
+        thumbnails = [{
+            'url': broadcast[image],
+        } for image in ('image_url', 'image_url_small') if broadcast.get(image)]
+
+        return {
+            'id': broadcast.get('id') or video_id,
+            'url': video_url,
+            'ext': 'mp4',
+            'protocol': 'm3u8_native',
+            'title': title,
+            'timestamp': timestamp,
+            'uploader': uploader,
+            'uploader_id': uploader_id,
+            'thumbnails': thumbnails,
+        }

From 621d6a9516e0f9cd8c45e12904f4d4b7615e7fb8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Aug 2015 04:00:52 +0600
Subject: [PATCH 2194/6219] [periscope] Switch to API for broadcast data

---
 youtube_dl/extractor/periscope.py | 20 ++++++++------------
 1 file changed, 8 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/periscope.py b/youtube_dl/extractor/periscope.py
index 5219e1a75..11648a511 100644
--- a/youtube_dl/extractor/periscope.py
+++ b/youtube_dl/extractor/periscope.py
@@ -25,21 +25,17 @@ class PeriscopeIE(InfoExtractor):
         'skip': 'Expires in 24 hours',
     }
 
+    def _call_api(self, method, token):
+        return self._download_json(
+            'https://api.periscope.tv/api/v2/%s?token=%s' % (method, token), token)
+
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        replay = self._download_json(
-            'https://api.periscope.tv/api/v2/getAccessPublic?token=%s' % video_id, video_id)
+        token = self._match_id(url)
 
+        replay = self._call_api('getAccessPublic', token)
         video_url = replay['replay_url']
 
-        webpage = self._download_webpage(url, video_id)
-
-        broadcast_data = self._parse_json(
-            unescapeHTML(self._html_search_meta(
-                'broadcast-data', webpage, 'broadcast data', fatal=True)),
-            video_id)
-
+        broadcast_data = self._call_api('getBroadcastPublic', token)
         broadcast = broadcast_data['broadcast']
         status = broadcast['status']
 
@@ -54,7 +50,7 @@ class PeriscopeIE(InfoExtractor):
         } for image in ('image_url', 'image_url_small') if broadcast.get(image)]
 
         return {
-            'id': broadcast.get('id') or video_id,
+            'id': broadcast.get('id') or token,
             'url': video_url,
             'ext': 'mp4',
             'protocol': 'm3u8_native',

From 1e83741c9a5d67e8bbe65510d41b558361496fe8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Aug 2015 05:33:53 +0600
Subject: [PATCH 2195/6219] [periscope] Add support for running streams

---
 youtube_dl/extractor/periscope.py | 34 +++++++++++++++++++++++--------
 1 file changed, 25 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/periscope.py b/youtube_dl/extractor/periscope.py
index 11648a511..de53b752d 100644
--- a/youtube_dl/extractor/periscope.py
+++ b/youtube_dl/extractor/periscope.py
@@ -2,13 +2,15 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..utils import (
-    parse_iso8601,
-    unescapeHTML,
+from ..compat import (
+    compat_urllib_parse,
+    compat_urllib_request,
 )
+from ..utils import parse_iso8601
 
 
 class PeriscopeIE(InfoExtractor):
+    IE_DESC = 'Periscope'
     _VALID_URL = r'https?://(?:www\.)?periscope\.tv/w/(?P<id>[^/?#]+)'
     _TEST = {
         'url': 'https://www.periscope.tv/w/aJUQnjY3MjA3ODF8NTYxMDIyMDl2zCg2pECBgwTqRpQuQD352EMPTKQjT4uqlM3cgWFA-g==',
@@ -32,9 +34,6 @@ class PeriscopeIE(InfoExtractor):
     def _real_extract(self, url):
         token = self._match_id(url)
 
-        replay = self._call_api('getAccessPublic', token)
-        video_url = replay['replay_url']
-
         broadcast_data = self._call_api('getBroadcastPublic', token)
         broadcast = broadcast_data['broadcast']
         status = broadcast['status']
@@ -43,20 +42,37 @@ class PeriscopeIE(InfoExtractor):
         uploader_id = broadcast.get('user_id') or broadcast_data.get('user', {}).get('id')
 
         title = '%s - %s' % (uploader, status) if uploader else status
+        state = broadcast.get('state').lower()
+        if state == 'running':
+            title = self._live_title(title)
         timestamp = parse_iso8601(broadcast.get('created_at'))
 
         thumbnails = [{
             'url': broadcast[image],
         } for image in ('image_url', 'image_url_small') if broadcast.get(image)]
 
+        stream = self._call_api('getAccessPublic', token)
+
+        formats = []
+        for format_id in ('replay', 'rtmp', 'hls', 'https_hls'):
+            video_url = stream.get(format_id + '_url')
+            if not video_url:
+                continue
+            f = {
+                'url': video_url,
+                'ext': 'flv' if format_id == 'rtmp' else 'mp4',
+            }
+            if format_id != 'rtmp':
+                f['protocol'] = 'm3u8_native' if state == 'ended' else 'm3u8'
+            formats.append(f)
+        self._sort_formats(formats)
+
         return {
             'id': broadcast.get('id') or token,
-            'url': video_url,
-            'ext': 'mp4',
-            'protocol': 'm3u8_native',
             'title': title,
             'timestamp': timestamp,
             'uploader': uploader,
             'uploader_id': uploader_id,
             'thumbnails': thumbnails,
+            'formats': formats,
         }

From 428e4e4a850df81031e8267dddf759da605639e3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Aug 2015 05:37:38 +0600
Subject: [PATCH 2196/6219] [quickscope] Add extractor

---
 youtube_dl/extractor/__init__.py  |  5 ++++-
 youtube_dl/extractor/periscope.py | 21 +++++++++++++++++++++
 2 files changed, 25 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index bd86a5be2..e38e77a27 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -432,7 +432,10 @@ from .orf import (
 from .parliamentliveuk import ParliamentLiveUKIE
 from .patreon import PatreonIE
 from .pbs import PBSIE
-from .periscope import PeriscopeIE
+from .periscope import (
+    PeriscopeIE,
+    QuickscopeIE,
+)
 from .philharmoniedeparis import PhilharmonieDeParisIE
 from .phoenix import PhoenixIE
 from .photobucket import PhotobucketIE
diff --git a/youtube_dl/extractor/periscope.py b/youtube_dl/extractor/periscope.py
index de53b752d..578b53a24 100644
--- a/youtube_dl/extractor/periscope.py
+++ b/youtube_dl/extractor/periscope.py
@@ -76,3 +76,24 @@ class PeriscopeIE(InfoExtractor):
             'thumbnails': thumbnails,
             'formats': formats,
         }
+
+
+class QuickscopeIE(InfoExtractor):
+    IE_DESC = 'Quisck Scope'
+    _VALID_URL = r'https?://watchonperiscope\.com/broadcast/(?P<id>\d+)'
+    _TEST = {
+        'url': 'https://watchonperiscope.com/broadcast/56180087',
+        'only_matching': True,
+    }
+
+    def _real_extract(self, url):
+        broadcast_id = self._match_id(url)
+        request = compat_urllib_request.Request(
+            'https://watchonperiscope.com/api/accessChannel', compat_urllib_parse.urlencode({
+                'broadcast_id': broadcast_id,
+                'entry_ticket': '',
+                'from_push': 'false',
+                'uses_sessions': 'true',
+            }).encode('utf-8'))
+        return self.url_result(
+            self._download_json(request, broadcast_id)['share_url'], 'Periscope')

From b2f82948ee5eadc483c01dc589b82426bb32ba68 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Aug 2015 05:40:41 +0600
Subject: [PATCH 2197/6219] [quickscope] Fix typo

---
 youtube_dl/extractor/periscope.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/periscope.py b/youtube_dl/extractor/periscope.py
index 578b53a24..8ad936758 100644
--- a/youtube_dl/extractor/periscope.py
+++ b/youtube_dl/extractor/periscope.py
@@ -79,7 +79,7 @@ class PeriscopeIE(InfoExtractor):
 
 
 class QuickscopeIE(InfoExtractor):
-    IE_DESC = 'Quisck Scope'
+    IE_DESC = 'Quick Scope'
     _VALID_URL = r'https?://watchonperiscope\.com/broadcast/(?P<id>\d+)'
     _TEST = {
         'url': 'https://watchonperiscope.com/broadcast/56180087',

From 59e89e62d7b45554cef502dc4986f35618110679 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 8 Aug 2015 12:59:10 +0100
Subject: [PATCH 2198/6219] [shahid] add default fallbacks for extracting api
 vars

---
 youtube_dl/extractor/shahid.py | 28 +++++++++++++++++++---------
 1 file changed, 19 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/shahid.py b/youtube_dl/extractor/shahid.py
index b2050525e..399140189 100644
--- a/youtube_dl/extractor/shahid.py
+++ b/youtube_dl/extractor/shahid.py
@@ -33,20 +33,30 @@ class ShahidIE(InfoExtractor):
         }
     ]
 
+    _api_vars = {
+        'type': 'player',
+        'url': 'http://api.shahid.net/api/v1_1',
+        'playerType': 'episode',
+    }
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
         player_info = ''
-        for line in self._search_regex('var flashvars = ({[^}]+})', webpage, 'flashvars').splitlines():
-            if '+' not in line and '(' not in line and ')' not in line:
-                player_info += line
-        player_info = self._parse_json(js_to_json(player_info), video_id)
-        video_id = player_info['id']
-        player_type = player_info['playerType']
+        flash_vars = self._search_regex('var flashvars = ({[^}]+})', webpage, 'flashvars', None)
+        if flash_vars is not None:
+            for line in flash_vars.splitlines():
+                if '+' not in line and '(' not in line and ')' not in line:
+                    player_info += line
+            player_info = self._parse_json(player_info, video_id, js_to_json, False)
+            if player_info is not None:
+                for key in self._api_vars:
+                    if key in player_info:
+                        self._api_vars[key] = player_info[key]
 
         player_json_data = self._download_json(
-            'https://shahid.mbc.net/arContent/getPlayerContent-param-.id-' + video_id + '.type-' + player_info['type'] + '.html',
+            'https://shahid.mbc.net/arContent/getPlayerContent-param-.id-' + video_id + '.type-' + self._api_vars['type'] + '.html',
             video_id
         )['data']
         if 'url' in player_json_data:
@@ -57,13 +67,13 @@ class ShahidIE(InfoExtractor):
         formats = self._extract_m3u8_formats(m3u8_url, video_id)
 
         video_info = self._download_json(
-            player_info['url'] + '/' + player_type + '/' + video_id + '?apiKey=sh%40hid0nlin3&hash=b2wMCTHpSmyxGqQjJFOycRmLSex%2BBpTK%2Fooxy6vHaqs%3D',
+            self._api_vars['url'] + '/' + self._api_vars['playerType'] + '/' + video_id + '?apiKey=sh%40hid0nlin3&hash=b2wMCTHpSmyxGqQjJFOycRmLSex%2BBpTK%2Fooxy6vHaqs%3D',
             video_id
         )['data']
         if video_info.get('error'):
             for error in video_info['error']:
                 raise ExtractorError(error)
-        video_info = video_info[player_type]
+        video_info = video_info[self._api_vars['playerType']]
         title = video_info['title']
         thumbnail = video_info.get('thumbnailUrl')
         categories = [category['name'] for category in video_info.get('genres')]

From 154655a85ae8b7740aa9fe7821544050fd65641b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Aug 2015 19:21:05 +0600
Subject: [PATCH 2199/6219] [downloader/external] Respect
 --no-check-certificate for wget

---
 youtube_dl/downloader/external.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index 30699934b..07ce59f7d 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -51,6 +51,9 @@ class ExternalFD(FileDownloader):
             return []
         return [command_option, source_address]
 
+    def _no_check_certificate(self, command_option):
+        return [command_option] if self.params.get('nocheckcertificate', False) else []
+
     def _configuration_args(self, default=[]):
         ex_args = self.params.get('external_downloader_args')
         if ex_args is None:
@@ -99,6 +102,7 @@ class WgetFD(ExternalFD):
         for key, val in info_dict['http_headers'].items():
             cmd += ['--header', '%s: %s' % (key, val)]
         cmd += self._source_address('--bind-address')
+        cmd += self._no_check_certificate('--no-check-certificate')
         cmd += self._configuration_args()
         cmd += ['--', info_dict['url']]
         return cmd

From b465083f45e63fe8aeb0255b5cea7dfbf0770a2b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Aug 2015 21:27:10 +0600
Subject: [PATCH 2200/6219] [sexykarma] Fix test

---
 youtube_dl/extractor/sexykarma.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/sexykarma.py b/youtube_dl/extractor/sexykarma.py
index 6446d26dc..e33483674 100644
--- a/youtube_dl/extractor/sexykarma.py
+++ b/youtube_dl/extractor/sexykarma.py
@@ -29,6 +29,7 @@ class SexyKarmaIE(InfoExtractor):
             'view_count': int,
             'comment_count': int,
             'categories': list,
+            'age_limit': 18,
         }
     }, {
         'url': 'http://www.sexykarma.com/gonewild/video/pot-pixie-tribute-8Id6EZPbuHf.html',

From b61b7787cbef408154695bbb9f5c3d29a70fdd38 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Aug 2015 21:30:57 +0600
Subject: [PATCH 2201/6219] [91porn] Extract age limit

---
 youtube_dl/extractor/porn91.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/porn91.py b/youtube_dl/extractor/porn91.py
index 72d1b2718..3e15533e9 100644
--- a/youtube_dl/extractor/porn91.py
+++ b/youtube_dl/extractor/porn91.py
@@ -22,6 +22,7 @@ class Porn91IE(InfoExtractor):
             'title': '18岁大一漂亮学妹，水嫩性感，再爽一次！',
             'ext': 'mp4',
             'duration': 431,
+            'age_limit': 18,
         }
     }
 
@@ -68,4 +69,5 @@ class Porn91IE(InfoExtractor):
             'url': video_url,
             'duration': duration,
             'comment_count': comment_count,
+            'age_limit': self._rta_search(webpage),
         }

From 8e2b1be12791b4e62c463562b570661e7b2c5852 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Aug 2015 21:42:50 +0600
Subject: [PATCH 2202/6219] [test/helper] Make age_limit checkable field

---
 test/helper.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/helper.py b/test/helper.py
index c8b34654d..cb6eec8d9 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -160,7 +160,7 @@ def expect_info_dict(self, got_dict, expected_dict):
     # Are checkable fields missing from the test case definition?
     test_info_dict = dict((key, value if not isinstance(value, compat_str) or len(value) < 250 else 'md5:' + md5(value))
                           for key, value in got_dict.items()
-                          if value and key in ('id', 'title', 'description', 'uploader', 'upload_date', 'timestamp', 'uploader_id', 'location'))
+                          if value and key in ('id', 'title', 'description', 'uploader', 'upload_date', 'timestamp', 'uploader_id', 'location', 'age_limit'))
     missing_keys = set(test_info_dict.keys()) - set(expected_dict.keys())
     if missing_keys:
         def _repr(v):

From 18c3281f9e1e32e00c778b149137fc91accb3b1e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Aug 2015 21:43:20 +0600
Subject: [PATCH 2203/6219] [24video] Fix test

---
 youtube_dl/extractor/fourtube.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/fourtube.py b/youtube_dl/extractor/fourtube.py
index b2284ab01..3bb4f6239 100644
--- a/youtube_dl/extractor/fourtube.py
+++ b/youtube_dl/extractor/fourtube.py
@@ -32,6 +32,7 @@ class FourTubeIE(InfoExtractor):
             'view_count': int,
             'like_count': int,
             'categories': list,
+            'age_limit': 18,
         }
     }
 

From 464e792496665b2e3dcabf5df43a45604673730a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Aug 2015 21:51:21 +0600
Subject: [PATCH 2204/6219] [vpro] Override npo IE_NAME

---
 youtube_dl/extractor/npo.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index 0c2d02c10..eb12fb810 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -407,6 +407,7 @@ class NPORadioFragmentIE(InfoExtractor):
 
 
 class VPROIE(NPOIE):
+    IE_NAME = 'vpro'
     _VALID_URL = r'https?://(?:www\.)?(?:tegenlicht\.)?vpro\.nl/(?:[^/]+/){2,}(?P<id>[^/]+)\.html'
 
     _TESTS = [

From d7bb8884afc8651b0ad86046dcd56a5330c98dd5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Aug 2015 21:58:24 +0600
Subject: [PATCH 2205/6219] [break] Add age_limit to test

---
 youtube_dl/extractor/breakcom.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/breakcom.py b/youtube_dl/extractor/breakcom.py
index 809287d14..aa08051b1 100644
--- a/youtube_dl/extractor/breakcom.py
+++ b/youtube_dl/extractor/breakcom.py
@@ -18,6 +18,7 @@ class BreakIE(InfoExtractor):
             'id': '2468056',
             'ext': 'mp4',
             'title': 'When Girls Act Like D-Bags',
+            'age_limit': 13,
         }
     }, {
         'url': 'http://www.break.com/video/ugc/baby-flex-2773063',

From 9f2e7c2f34c48942a2a3e55532dd0d0ef8ed4d98 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Aug 2015 22:04:48 +0600
Subject: [PATCH 2206/6219] [ok] Add age_limit to tests

---
 youtube_dl/extractor/odnoklassniki.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/odnoklassniki.py b/youtube_dl/extractor/odnoklassniki.py
index 215ffe87b..e5fd1ba04 100644
--- a/youtube_dl/extractor/odnoklassniki.py
+++ b/youtube_dl/extractor/odnoklassniki.py
@@ -25,6 +25,7 @@ class OdnoklassnikiIE(InfoExtractor):
             'uploader_id': '330537914540',
             'uploader': 'Виталий Добровольский',
             'like_count': int,
+            'age_limit': 0,
         },
     }, {
         # metadataUrl
@@ -38,6 +39,7 @@ class OdnoklassnikiIE(InfoExtractor):
             'uploader_id': '534380003155',
             'uploader': 'Андрей Мещанинов',
             'like_count': int,
+            'age_limit': 0,
         },
     }, {
         'url': 'http://ok.ru/web-api/video/moviePlayer/20079905452',

From 887e9bc7b561f9b2b97dec8f99f9c04392d95d40 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Aug 2015 22:08:54 +0600
Subject: [PATCH 2207/6219] [ok] Update tests

---
 youtube_dl/extractor/odnoklassniki.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/odnoklassniki.py b/youtube_dl/extractor/odnoklassniki.py
index e5fd1ba04..003d27de7 100644
--- a/youtube_dl/extractor/odnoklassniki.py
+++ b/youtube_dl/extractor/odnoklassniki.py
@@ -16,12 +16,13 @@ class OdnoklassnikiIE(InfoExtractor):
     _TESTS = [{
         # metadata in JSON
         'url': 'http://ok.ru/video/20079905452',
-        'md5': '8e24ad2da6f387948e7a7d44eb8668fe',
+        'md5': '6ba728d85d60aa2e6dd37c9e70fdc6bc',
         'info_dict': {
             'id': '20079905452',
             'ext': 'mp4',
             'title': 'Культура меняет нас (прекрасный ролик!))',
             'duration': 100,
+            'upload_date': '20141207',
             'uploader_id': '330537914540',
             'uploader': 'Виталий Добровольский',
             'like_count': int,
@@ -36,8 +37,9 @@ class OdnoklassnikiIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Девушка без комплексов ...',
             'duration': 191,
+            'upload_date': '20150518',
             'uploader_id': '534380003155',
-            'uploader': 'Андрей Мещанинов',
+            'uploader': '☭ Андрей Мещанинов ☭',
             'like_count': int,
             'age_limit': 0,
         },

From c8d1be772daa496759bd85cb95c4ec799294c7f8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 8 Aug 2015 22:11:06 +0600
Subject: [PATCH 2208/6219] [rutube] Add age_limit to test

---
 youtube_dl/extractor/rutube.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/rutube.py b/youtube_dl/extractor/rutube.py
index 5b1c3577a..d94dc7399 100644
--- a/youtube_dl/extractor/rutube.py
+++ b/youtube_dl/extractor/rutube.py
@@ -30,6 +30,7 @@ class RutubeIE(InfoExtractor):
             'uploader': 'NTDRussian',
             'uploader_id': '29790',
             'upload_date': '20131016',
+            'age_limit': 0,
         },
         'params': {
             # It requires ffmpeg (m3u8 download)

From 08df685fe7764ef9f7dc271075340e4effc5e621 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 9 Aug 2015 08:51:37 +0600
Subject: [PATCH 2209/6219] [videolectures] Fix _VALID_URL for
 test_no_duplicates to pass

---
 youtube_dl/extractor/videolecturesnet.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/videolecturesnet.py b/youtube_dl/extractor/videolecturesnet.py
index 24584dc80..ef2da5632 100644
--- a/youtube_dl/extractor/videolecturesnet.py
+++ b/youtube_dl/extractor/videolecturesnet.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 
 class VideoLecturesNetIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?videolectures\.net/(?P<id>[^/#?]+)(?:/?[#?].*)?$'
+    _VALID_URL = r'http://(?:www\.)?videolectures\.net/(?P<id>[^/#?]+)/*(?:[#?].*)?$'
     IE_NAME = 'videolectures.net'
 
     _TEST = {

From 12bb392a0ff8adbde2ced75b0c4976d0aabc7f4f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 9 Aug 2015 17:10:40 +0600
Subject: [PATCH 2210/6219] [vimeo] Fix password protected videos (Closes
 #6507)

---
 youtube_dl/extractor/vimeo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 10d6745af..4c4e3c72a 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -203,7 +203,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
             url = url.replace('http://', 'https://')
         password_request = compat_urllib_request.Request(url + '/password', data)
         password_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
-        password_request.add_header('Cookie', 'xsrft=%s' % token)
+        password_request.add_header('Referer', url)
         return self._download_webpage(
             password_request, video_id,
             'Verifying the password', 'Wrong password')

From 8d6765cf48138cc44fdbaee4e8c7a199ae348bb5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 9 Aug 2015 19:07:18 +0600
Subject: [PATCH 2211/6219] [extractor/generic] Add generic support for xspf
 playist extraction

---
 youtube_dl/extractor/common.py | 41 ++++++++++++++++++++++++++++++++++
 1 file changed, 41 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index def6caa0d..e201ea6db 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -39,6 +39,8 @@ from ..utils import (
     sanitize_filename,
     unescapeHTML,
     url_basename,
+    xpath_text,
+    xpath_with_ns,
 )
 
 
@@ -1142,6 +1144,45 @@ class InfoExtractor(object):
             })
         return subtitles
 
+    def _extract_xspf_playlist(self, playlist_url, playlist_id):
+        playlist = self._download_xml(
+            playlist_url, playlist_id, 'Downloading xpsf playlist',
+            'Unable to download xspf manifest')
+
+        NS_MAP = {
+            'xspf': 'http://xspf.org/ns/0/',
+            's1': 'http://static.streamone.nl/player/ns/0',
+        }
+
+        entries = []
+        for track in playlist.findall(xpath_with_ns('./xspf:trackList/xspf:track', NS_MAP)):
+            title = xpath_text(
+                track, xpath_with_ns('./xspf:title', NS_MAP), 'title')
+            description = xpath_text(
+                track, xpath_with_ns('./xspf:annotation', NS_MAP), 'description')
+            thumbnail = xpath_text(
+                track, xpath_with_ns('./xspf:image', NS_MAP), 'thumbnail')
+            duration = float_or_none(
+                xpath_text(track, xpath_with_ns('./xspf:duration', NS_MAP), 'duration'), 1000)
+
+            formats = [{
+                'url': location.text,
+                'format_id': location.get(xpath_with_ns('s1:label', NS_MAP)),
+                'width': int_or_none(location.get(xpath_with_ns('s1:width', NS_MAP))),
+                'height': int_or_none(location.get(xpath_with_ns('s1:height', NS_MAP))),
+            } for location in track.findall(xpath_with_ns('./xspf:location', NS_MAP))]
+            self._sort_formats(formats)
+
+            entries.append({
+                'id': playlist_id,
+                'title': title,
+                'description': description,
+                'thumbnail': thumbnail,
+                'duration': duration,
+                'formats': formats,
+            })
+        return entries
+
     def _live_title(self, name):
         """ Generate the title for a live video """
         now = datetime.datetime.now()

From e0b9d78fab76e2c2819c8a9a7512ad4533319b72 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 9 Aug 2015 19:09:50 +0600
Subject: [PATCH 2212/6219] [extractor/common] Clarify playlists can have
 description field

---
 youtube_dl/extractor/common.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index e201ea6db..9b4775e0a 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -204,8 +204,8 @@ class InfoExtractor(object):
     There must be a key "entries", which is a list, an iterable, or a PagedList
     object, each element of which is a valid dictionary by this specification.
 
-    Additionally, playlists can have "title" and "id" attributes with the same
-    semantics as videos (see above).
+    Additionally, playlists can have "title", "description" and "id" attributes
+    with the same semantics as videos (see above).
 
 
     _type "multi_video" indicates that there are multiple videos that

From 3a30508b943c044e5f684b703ff58ac352686f63 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 9 Aug 2015 19:11:23 +0600
Subject: [PATCH 2213/6219] [telegraaf] Add extractor (Closes #6492)

---
 youtube_dl/extractor/__init__.py  |  1 +
 youtube_dl/extractor/telegraaf.py | 35 +++++++++++++++++++++++++++++++
 2 files changed, 36 insertions(+)
 create mode 100644 youtube_dl/extractor/telegraaf.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index e38e77a27..dad3ec87f 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -596,6 +596,7 @@ from .techtalks import TechTalksIE
 from .ted import TEDIE
 from .telebruxelles import TeleBruxellesIE
 from .telecinco import TelecincoIE
+from .telegraaf import TelegraafIE
 from .telemb import TeleMBIE
 from .teletask import TeleTaskIE
 from .tenplay import TenPlayIE
diff --git a/youtube_dl/extractor/telegraaf.py b/youtube_dl/extractor/telegraaf.py
new file mode 100644
index 000000000..6f8333cfc
--- /dev/null
+++ b/youtube_dl/extractor/telegraaf.py
@@ -0,0 +1,35 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import remove_end
+
+
+class TelegraafIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?telegraaf\.nl/tv/(?:[^/]+/)+(?P<id>\d+)/[^/]+\.html'
+    _TEST = {
+        'url': 'http://www.telegraaf.nl/tv/nieuws/binnenland/24353229/__Tikibad_ontruimd_wegens_brand__.html',
+        'md5': '83245a9779bcc4a24454bfd53c65b6dc',
+        'info_dict': {
+            'id': '24353229',
+            'ext': 'mp4',
+            'title': 'Tikibad ontruimd wegens brand',
+            'description': 'md5:05ca046ff47b931f9b04855015e163a4',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 33,
+        },
+    }
+
+    def _real_extract(self, url):
+        playlist_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, playlist_id)
+
+        playlist_url = self._search_regex(
+            r"iframe\.loadPlayer\('([^']+)'", webpage, 'player')
+
+        entries = self._extract_xspf_playlist(playlist_url, playlist_id)
+        title = remove_end(self._og_search_title(webpage), ' - VIDEO')
+        description = self._og_search_description(webpage)
+
+        return self.playlist_result(entries, playlist_id, title, description)

From f32143469fd0a2720bd40908ea8360490983b97d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 9 Aug 2015 19:15:00 +0600
Subject: [PATCH 2214/6219] [tweakers] Use _extract_xspf_playlist

---
 youtube_dl/extractor/tweakers.py | 42 +++-----------------------------
 1 file changed, 4 insertions(+), 38 deletions(-)

diff --git a/youtube_dl/extractor/tweakers.py b/youtube_dl/extractor/tweakers.py
index c80ec15cf..4bbe76e96 100644
--- a/youtube_dl/extractor/tweakers.py
+++ b/youtube_dl/extractor/tweakers.py
@@ -25,41 +25,7 @@ class TweakersIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        playlist = self._download_xml(
-            'https://tweakers.net/video/s1playlist/%s/playlist.xspf' % video_id,
-            video_id)
-
-        NS_MAP = {
-            'xspf': 'http://xspf.org/ns/0/',
-            's1': 'http://static.streamone.nl/player/ns/0',
-        }
-
-        track = playlist.find(xpath_with_ns('./xspf:trackList/xspf:track', NS_MAP))
-
-        title = xpath_text(
-            track, xpath_with_ns('./xspf:title', NS_MAP), 'title')
-        description = xpath_text(
-            track, xpath_with_ns('./xspf:annotation', NS_MAP), 'description')
-        thumbnail = xpath_text(
-            track, xpath_with_ns('./xspf:image', NS_MAP), 'thumbnail')
-        duration = float_or_none(
-            xpath_text(track, xpath_with_ns('./xspf:duration', NS_MAP), 'duration'),
-            1000)
-
-        formats = [{
-            'url': location.text,
-            'format_id': location.get(xpath_with_ns('s1:label', NS_MAP)),
-            'width': int_or_none(location.get(xpath_with_ns('s1:width', NS_MAP))),
-            'height': int_or_none(location.get(xpath_with_ns('s1:height', NS_MAP))),
-        } for location in track.findall(xpath_with_ns('./xspf:location', NS_MAP))]
-
-        return {
-            'id': video_id,
-            'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
-            'duration': duration,
-            'formats': formats,
-        }
+        playlist_id = self._match_id(url)
+        entries = self._extract_xspf_playlist(
+            'https://tweakers.net/video/s1playlist/%s/playlist.xspf' % playlist_id, playlist_id)
+        return self.playlist_result(entries, playlist_id)

From 0dcb318f622d944ad0f5c23c32c9bc9b00e76aaa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 9 Aug 2015 19:15:20 +0600
Subject: [PATCH 2215/6219] [tweakers] Fix test

---
 youtube_dl/extractor/tweakers.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tweakers.py b/youtube_dl/extractor/tweakers.py
index 4bbe76e96..6eeffb1cc 100644
--- a/youtube_dl/extractor/tweakers.py
+++ b/youtube_dl/extractor/tweakers.py
@@ -13,7 +13,7 @@ class TweakersIE(InfoExtractor):
     _VALID_URL = r'https?://tweakers\.net/video/(?P<id>\d+)'
     _TEST = {
         'url': 'https://tweakers.net/video/9926/new-nintendo-3ds-xl-op-alle-fronten-beter.html',
-        'md5': '1b5afa817403bb5baa08359dca31e6df',
+        'md5': '3147e4ddad366f97476a93863e4557c8',
         'info_dict': {
             'id': '9926',
             'ext': 'mp4',

From 98044462b1035000a44b35a41f4f780b2e844f2e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 9 Aug 2015 19:18:50 +0600
Subject: [PATCH 2216/6219] [extractor/common] Use playlist id as default title

---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 9b4775e0a..be91e03e9 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1157,7 +1157,7 @@ class InfoExtractor(object):
         entries = []
         for track in playlist.findall(xpath_with_ns('./xspf:trackList/xspf:track', NS_MAP)):
             title = xpath_text(
-                track, xpath_with_ns('./xspf:title', NS_MAP), 'title')
+                track, xpath_with_ns('./xspf:title', NS_MAP), 'title', default=playlist_id)
             description = xpath_text(
                 track, xpath_with_ns('./xspf:annotation', NS_MAP), 'description')
             thumbnail = xpath_text(

From fb2f339fec20c35cb62c1da682e0dfd418faef81 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 9 Aug 2015 19:21:25 +0600
Subject: [PATCH 2217/6219] [dhm] Use _extract_xspf_playlist

---
 youtube_dl/extractor/dhm.py | 25 +++++++------------------
 1 file changed, 7 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/dhm.py b/youtube_dl/extractor/dhm.py
index 3ed1f1663..127eb0439 100644
--- a/youtube_dl/extractor/dhm.py
+++ b/youtube_dl/extractor/dhm.py
@@ -34,24 +34,14 @@ class DHMIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        playlist_id = self._match_id(url)
 
-        webpage = self._download_webpage(url, video_id)
+        webpage = self._download_webpage(url, playlist_id)
 
         playlist_url = self._search_regex(
             r"file\s*:\s*'([^']+)'", webpage, 'playlist url')
 
-        playlist = self._download_xml(playlist_url, video_id)
-
-        track = playlist.find(
-            './{http://xspf.org/ns/0/}trackList/{http://xspf.org/ns/0/}track')
-
-        video_url = xpath_text(
-            track, './{http://xspf.org/ns/0/}location',
-            'video url', fatal=True)
-        thumbnail = xpath_text(
-            track, './{http://xspf.org/ns/0/}image',
-            'thumbnail')
+        entries = self._extract_xspf_playlist(playlist_url, playlist_id)
 
         title = self._search_regex(
             [r'dc:title="([^"]+)"', r'<title> &raquo;([^<]+)</title>'],
@@ -63,11 +53,10 @@ class DHMIE(InfoExtractor):
             r'<em>Length\s*</em>\s*:\s*</strong>([^<]+)',
             webpage, 'duration', default=None))
 
-        return {
-            'id': video_id,
-            'url': video_url,
+        entries[0].update({
             'title': title,
             'description': description,
             'duration': duration,
-            'thumbnail': thumbnail,
-        }
+        })
+
+        return self.playlist_result(entries, playlist_id)

From 942acef594428b5f5c7e0ed7860cb6d725d8f1e5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 9 Aug 2015 19:41:55 +0600
Subject: [PATCH 2218/6219] [extractor/common] Extract _parse_xspf

---
 youtube_dl/extractor/common.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index be91e03e9..5982055be 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1144,11 +1144,15 @@ class InfoExtractor(object):
             })
         return subtitles
 
-    def _extract_xspf_playlist(self, playlist_url, playlist_id):
-        playlist = self._download_xml(
+    def _extract_xspf_playlist(self, playlist_url, playlist_id, fatal=True):
+        xspf = self._download_xml(
             playlist_url, playlist_id, 'Downloading xpsf playlist',
-            'Unable to download xspf manifest')
+            'Unable to download xspf manifest', fatal=fatal)
+        if xspf is False:
+            return []
+        return self._parse_xspf(xspf, playlist_id)
 
+    def _parse_xspf(self, playlist, playlist_id):
         NS_MAP = {
             'xspf': 'http://xspf.org/ns/0/',
             's1': 'http://static.streamone.nl/player/ns/0',

From 729accb48221bd72e40076939616792c1c6fc15f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 9 Aug 2015 19:43:42 +0600
Subject: [PATCH 2219/6219] [extractor/generic] Add support for xspf playlists

---
 youtube_dl/extractor/generic.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 901f77304..a382d6be4 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1198,6 +1198,8 @@ class GenericIE(InfoExtractor):
                 return self._extract_rss(url, video_id, doc)
             elif re.match(r'^(?:{[^}]+})?smil$', doc.tag):
                 return self._parse_smil(doc, url, video_id)
+            elif doc.tag == '{http://xspf.org/ns/0/}playlist':
+                return self.playlist_result(self._parse_xspf(doc, video_id), video_id)
         except compat_xml_parse_error:
             pass
 
@@ -1799,7 +1801,8 @@ class GenericIE(InfoExtractor):
             # here's a fun little line of code for you:
             video_id = os.path.splitext(video_id)[0]
 
-            if determine_ext(video_url) == 'smil':
+            ext = determine_ext(video_url)
+            if ext == 'smil':
                 entries.append({
                     'id': video_id,
                     'formats': self._extract_smil_formats(video_url, video_id),
@@ -1807,6 +1810,8 @@ class GenericIE(InfoExtractor):
                     'title': video_title,
                     'age_limit': age_limit,
                 })
+            elif ext == 'xspf':
+                return self.playlist_result(self._extract_xspf_playlist(video_url, video_id), video_id)
             else:
                 entries.append({
                     'id': video_id,

From 1de5cd3ba51ce67d9a1cd3b40157058e78e46692 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 9 Aug 2015 19:47:08 +0600
Subject: [PATCH 2220/6219] [extractor/generic] Add test for xspf playlist

---
 youtube_dl/extractor/generic.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index a382d6be4..4756a658f 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -198,6 +198,21 @@ class GenericIE(InfoExtractor):
                 'skip_download': True,
             },
         },
+        # XSPF playlist from http://www.telegraaf.nl/tv/nieuws/binnenland/24353229/__Tikibad_ontruimd_wegens_brand__.html
+        {
+            'url': 'http://www.telegraaf.nl/xml/playlist/2015/8/7/mZlp2ctYIUEB.xspf',
+            'info_dict': {
+                'id': 'mZlp2ctYIUEB',
+                'ext': 'mp4',
+                'title': 'Tikibad ontruimd wegens brand',
+                'description': 'md5:05ca046ff47b931f9b04855015e163a4',
+                'thumbnail': 're:^https?://.*\.jpg$',
+                'duration': 33,
+            },
+            'params': {
+                'skip_download': True,
+            },
+        },
         # google redirect
         {
             'url': 'http://www.google.com/url?sa=t&rct=j&q=&esrc=s&source=web&cd=1&cad=rja&ved=0CCUQtwIwAA&url=http%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DcmQHVoWB5FY&ei=F-sNU-LLCaXk4QT52ICQBQ&usg=AFQjCNEw4hL29zgOohLXvpJ-Bdh2bils1Q&bvm=bv.61965928,d.bGE',

From 0791ac1b4415601f464f9656a4485b3ae6b67f4e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 9 Aug 2015 19:47:58 +0600
Subject: [PATCH 2221/6219] [extractor/generic] Clarify comment

---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 4756a658f..376feecae 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1206,7 +1206,7 @@ class GenericIE(InfoExtractor):
 
         self.report_extraction(video_id)
 
-        # Is it an RSS feed or a SMIL file?
+        # Is it an RSS feed, a SMIL file or a XSPF playlist?
         try:
             doc = parse_xml(webpage)
             if doc.tag == 'rss':

From 27c7114af6b82bfe8be6b8e4dfa6e11dd1356044 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 9 Aug 2015 20:13:02 +0200
Subject: [PATCH 2222/6219] release 2015.08.09

---
 README.md              | 2 +-
 docs/supportedsites.md | 8 ++++++--
 youtube_dl/version.py  | 2 +-
 3 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index 02b9775f9..15baf75ce 100644
--- a/README.md
+++ b/README.md
@@ -108,7 +108,7 @@ which means you can modify it, redistribute it or use it however you like.
     --playlist-reverse               Download playlist videos in reverse order
     --xattr-set-filesize             Set file xattribute ytdl.filesize with expected filesize (experimental)
     --hls-prefer-native              Use the native HLS downloader instead of ffmpeg (experimental)
-    --external-downloader COMMAND    Use the specified external downloader. Currently supports aria2c,curl,httpie,wget
+    --external-downloader COMMAND    Use the specified external downloader. Currently supports aria2c,axel,curl,httpie,wget
     --external-downloader-args ARGS  Give these arguments to the external downloader
 
 ## Filesystem Options:
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 657935dc6..e21471102 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -86,7 +86,7 @@
  - **chirbit:profile**
  - **Cinchcast**
  - **Cinemassacre**
- - **clipfish**
+ - **Clipfish**
  - **cliphunter**
  - **Clipsyndicate**
  - **Cloudy**
@@ -116,6 +116,7 @@
  - **DailymotionCloud**
  - **daum.net**
  - **DBTV**
+ - **DCN**
  - **DctpTv**
  - **DeezerPlaylist**
  - **defense.gouv.fr**
@@ -351,7 +352,6 @@
  - **NowTV**
  - **nowvideo**: NowVideo
  - **npo**: npo.nl and ntr.nl
- - **npo**: npo.nl and ntr.nl
  - **npo.nl:live**
  - **npo.nl:radio**
  - **npo.nl:radio:fragment**
@@ -377,6 +377,7 @@
  - **parliamentlive.tv**: UK parliament videos
  - **Patreon**
  - **PBS**
+ - **Periscope**: Periscope
  - **PhilharmonieDeParis**: Philharmonie de Paris
  - **Phoenix**
  - **Photobucket**
@@ -406,6 +407,7 @@
  - **qqmusic:playlist**: QQ音乐 - 歌单
  - **qqmusic:singer**: QQ音乐 - 歌手
  - **qqmusic:toplist**: QQ音乐 - 排行榜
+ - **Quickscope**: Quick Scope
  - **QuickVid**
  - **R7**
  - **radio.de**
@@ -518,6 +520,7 @@
  - **ted**
  - **TeleBruxelles**
  - **telecinco.es**
+ - **Telegraaf**
  - **TeleMB**
  - **TeleTask**
  - **TenPlay**
@@ -621,6 +624,7 @@
  - **Vodlocker**
  - **VoiceRepublic**
  - **Vporn**
+ - **vpro**: npo.nl and ntr.nl
  - **VRT**
  - **vube**: Vube.com
  - **VuClip**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 9f209499c..6462d4477 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.08.06.1'
+__version__ = '2015.08.09'

From c5864a8ce6379dca300f447cca12a5a946d67d3d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 10 Aug 2015 21:38:58 +0600
Subject: [PATCH 2223/6219] [fc2] Fix python 2.6 (Closes #6512)

---
 youtube_dl/extractor/fc2.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/fc2.py b/youtube_dl/extractor/fc2.py
index 1ccc1a964..e4f7195a8 100644
--- a/youtube_dl/extractor/fc2.py
+++ b/youtube_dl/extractor/fc2.py
@@ -86,7 +86,7 @@ class FC2IE(InfoExtractor):
 
         info_url = (
             "http://video.fc2.com/ginfo.php?mimi={1:s}&href={2:s}&v={0:s}&fversion=WIN%2011%2C6%2C602%2C180&from=2&otag=0&upid={0:s}&tk=null&".
-            format(video_id, mimi, compat_urllib_request.quote(refer, safe='').replace('.', '%2E')))
+            format(video_id, mimi, compat_urllib_request.quote(refer, safe=b'').replace('.', '%2E')))
 
         info_webpage = self._download_webpage(
             info_url, video_id, note='Downloading info page')

From f6c3664d717857a7994f189a01a00402df2b4168 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 10 Aug 2015 23:35:08 +0600
Subject: [PATCH 2224/6219] [vimeo] Fix login (Closes #6488)

---
 youtube_dl/extractor/vimeo.py | 25 +++++++++++++++----------
 1 file changed, 15 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 4c4e3c72a..5bce78ac0 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -29,6 +29,7 @@ from ..utils import (
 class VimeoBaseInfoExtractor(InfoExtractor):
     _NETRC_MACHINE = 'vimeo'
     _LOGIN_REQUIRED = False
+    _LOGIN_URL = 'https://vimeo.com/log_in'
 
     def _login(self):
         (username, password) = self._get_login_info()
@@ -37,21 +38,25 @@ class VimeoBaseInfoExtractor(InfoExtractor):
                 raise ExtractorError('No login info available, needed for using %s.' % self.IE_NAME, expected=True)
             return
         self.report_login()
-        login_url = 'https://vimeo.com/log_in'
-        webpage = self._download_webpage(login_url, None, False)
-        token = self._search_regex(r'xsrft":"(.*?)"', webpage, 'login token')
+        webpage = self._download_webpage(self._LOGIN_URL, None, False)
+        token = self._extract_xsrft(webpage)
         data = urlencode_postdata({
+            'action': 'login',
             'email': username,
             'password': password,
-            'action': 'login',
             'service': 'vimeo',
             'token': token,
         })
-        login_request = compat_urllib_request.Request(login_url, data)
+        login_request = compat_urllib_request.Request(self._LOGIN_URL, data)
         login_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
-        login_request.add_header('Cookie', 'xsrft=%s' % token)
+        login_request.add_header('Referer', self._LOGIN_URL)
         self._download_webpage(login_request, None, False, 'Wrong login info')
 
+    def _extract_xsrft(self, webpage):
+        return self._search_regex(
+            r'xsrft\s*[=:]\s*(?P<q>["\'])(?P<xsrft>.+?)(?P=q)',
+            webpage, 'login token', group='xsrft')
+
 
 class VimeoIE(VimeoBaseInfoExtractor):
     """Information extractor for vimeo.com."""
@@ -193,7 +198,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
         password = self._downloader.params.get('videopassword', None)
         if password is None:
             raise ExtractorError('This video is protected by a password, use the --video-password option', expected=True)
-        token = self._search_regex(r'xsrft[\s=:"\']+([^"\']+)', webpage, 'login token')
+        token = self._extract_xsrft(webpage)
         data = urlencode_postdata({
             'password': password,
             'token': token,
@@ -422,7 +427,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
         }
 
 
-class VimeoChannelIE(InfoExtractor):
+class VimeoChannelIE(VimeoBaseInfoExtractor):
     IE_NAME = 'vimeo:channel'
     _VALID_URL = r'https://vimeo\.com/channels/(?P<id>[^/?#]+)/?(?:$|[?#])'
     _MORE_PAGES_INDICATOR = r'<a.+?rel="next"'
@@ -453,7 +458,7 @@ class VimeoChannelIE(InfoExtractor):
         if password is None:
             raise ExtractorError('This album is protected by a password, use the --video-password option', expected=True)
         fields = self._hidden_inputs(login_form)
-        token = self._search_regex(r'xsrft[\s=:"\']+([^"\']+)', webpage, 'login token')
+        token = self._extract_xsrft(webpage)
         fields['token'] = token
         fields['password'] = password
         post = urlencode_postdata(fields)
@@ -603,7 +608,7 @@ class VimeoReviewIE(InfoExtractor):
         return self.url_result(player_url, 'Vimeo', video_id)
 
 
-class VimeoWatchLaterIE(VimeoBaseInfoExtractor, VimeoChannelIE):
+class VimeoWatchLaterIE(VimeoChannelIE):
     IE_NAME = 'vimeo:watchlater'
     IE_DESC = 'Vimeo watch later list, "vimeowatchlater" keyword (requires authentication)'
     _VALID_URL = r'https://vimeo\.com/home/watchlater|:vimeowatchlater'

From 844587669eeffaf139b9c8888550c9d3b88335e7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 10 Aug 2015 23:58:01 +0600
Subject: [PATCH 2225/6219] [vimeo:watchlater] Fix extraction (Closes #3886)

---
 youtube_dl/extractor/vimeo.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 5bce78ac0..1eeb4618e 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -431,6 +431,7 @@ class VimeoChannelIE(VimeoBaseInfoExtractor):
     IE_NAME = 'vimeo:channel'
     _VALID_URL = r'https://vimeo\.com/channels/(?P<id>[^/?#]+)/?(?:$|[?#])'
     _MORE_PAGES_INDICATOR = r'<a.+?rel="next"'
+    _TITLE = None
     _TITLE_RE = r'<link rel="alternate"[^>]+?title="(.*?)"'
     _TESTS = [{
         'url': 'https://vimeo.com/channels/tributes',
@@ -445,7 +446,7 @@ class VimeoChannelIE(VimeoBaseInfoExtractor):
         return '%s/videos/page:%d/' % (base_url, pagenum)
 
     def _extract_list_title(self, webpage):
-        return self._html_search_regex(self._TITLE_RE, webpage, 'list title')
+        return self._TITLE or self._html_search_regex(self._TITLE_RE, webpage, 'list title')
 
     def _login_list_password(self, page_url, list_id, webpage):
         login_form = self._search_regex(
@@ -611,11 +612,11 @@ class VimeoReviewIE(InfoExtractor):
 class VimeoWatchLaterIE(VimeoChannelIE):
     IE_NAME = 'vimeo:watchlater'
     IE_DESC = 'Vimeo watch later list, "vimeowatchlater" keyword (requires authentication)'
-    _VALID_URL = r'https://vimeo\.com/home/watchlater|:vimeowatchlater'
+    _VALID_URL = r'https://vimeo\.com/(?:home/)?watchlater|:vimeowatchlater'
+    _TITLE = 'Watch Later'
     _LOGIN_REQUIRED = True
-    _TITLE_RE = r'href="/home/watchlater".*?>(.*?)<'
     _TESTS = [{
-        'url': 'https://vimeo.com/home/watchlater',
+        'url': 'https://vimeo.com/watchlater',
         'only_matching': True,
     }]
 
@@ -631,7 +632,7 @@ class VimeoWatchLaterIE(VimeoChannelIE):
         return request
 
     def _real_extract(self, url):
-        return self._extract_videos('watchlater', 'https://vimeo.com/home/watchlater')
+        return self._extract_videos('watchlater', 'https://vimeo.com/watchlater')
 
 
 class VimeoLikesIE(InfoExtractor):

From 11b5605815d685263b271b4e061c43f9cb55a08c Mon Sep 17 00:00:00 2001
From: Puck Meerburg <puck@puckipedia.nl>
Date: Mon, 10 Aug 2015 20:52:38 +0200
Subject: [PATCH 2226/6219] [youtube] Use the first v= argument in the URL

This is according to how youtube handles multiple v= values in one URL.
Before this, it was possible to make a single URL show up differently
 on youtube itself, and if you downloaded/viewed it with youtube-dl/mpv
---
 youtube_dl/extractor/youtube.py | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 67a1df9a0..eaf058cfb 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -213,7 +213,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                              |(?:                                             # or the v= param in all its forms
                                  (?:(?:watch|movie)(?:_popup)?(?:\.php)?/?)?  # preceding watch(_popup|.php) or nothing (like /?v=xxxx)
                                  (?:\?|\#!?)                                  # the params delimiter ? or # or #!
-                                 (?:.*?&)?                                    # any other preceding param (like /?s=tuff&v=xxxx)
+                                 (?:.*?&)??                                   # any other preceding param (like /?s=tuff&v=xxxx)
                                  v=
                              )
                          ))
@@ -380,6 +380,23 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader_id': 'setindia'
             }
         },
+        {
+            'url': 'http://www.youtube.com/watch?v=BaW_jenozKcj&v=UxxajLWwzqY',
+            'note': 'Use the first video ID in the URL',
+            'info_dict': {
+                'id': 'BaW_jenozKc',
+                'ext': 'mp4',
+                'title': 'youtube-dl test video "\'/\\ä↭𝕐',
+                'uploader': 'Philipp Hagemeister',
+                'uploader_id': 'phihag',
+                'upload_date': '20121002',
+                'description': 'test chars:  "\'/\\ä↭𝕐\ntest URL: https://github.com/rg3/youtube-dl/issues/1892\n\nThis is a test video for youtube-dl.\n\nFor more information, contact phihag@phihag.de .',
+                'categories': ['Science & Technology'],
+                'tags': ['youtube-dl'],
+                'like_count': int,
+                'dislike_count': int,
+            }
+        },
         {
             'url': 'http://www.youtube.com/watch?v=a9LDPn-MO4I',
             'note': '256k DASH audio (format 141) via DASH manifest',

From b29440aee64027b3e4145070b0235193752b4d9f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 11 Aug 2015 01:17:41 +0600
Subject: [PATCH 2227/6219] [vimeo:user] Do not match watchlater

---
 youtube_dl/extractor/vimeo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 1eeb4618e..50df79ca1 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -505,7 +505,7 @@ class VimeoChannelIE(VimeoBaseInfoExtractor):
 
 class VimeoUserIE(VimeoChannelIE):
     IE_NAME = 'vimeo:user'
-    _VALID_URL = r'https://vimeo\.com/(?![0-9]+(?:$|[?#/]))(?P<name>[^/]+)(?:/videos|[#?]|$)'
+    _VALID_URL = r'https://vimeo\.com/(?!(?:[0-9]+|watchlater)(?:$|[?#/]))(?P<name>[^/]+)(?:/videos|[#?]|$)'
     _TITLE_RE = r'<a[^>]+?class="user">([^<>]+?)</a>'
     _TESTS = [{
         'url': 'https://vimeo.com/nkistudio/videos',

From 34a7de2970d8bbceeb3f485d64a57f67489a44d3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 11 Aug 2015 01:22:06 +0600
Subject: [PATCH 2228/6219] [youtube] Skip download for multiple v= test

---
 youtube_dl/extractor/youtube.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index eaf058cfb..01dbbfa3c 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -395,7 +395,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'tags': ['youtube-dl'],
                 'like_count': int,
                 'dislike_count': int,
-            }
+            },
+            'params': {
+                'skip_download': True,
+            },
         },
         {
             'url': 'http://www.youtube.com/watch?v=a9LDPn-MO4I',

From 34952f09e175e0b78c929fddf56f82ccf028dc5c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 11 Aug 2015 01:24:53 +0600
Subject: [PATCH 2229/6219] [youtube] Add age limit to tests

---
 youtube_dl/extractor/youtube.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 01dbbfa3c..e74a39095 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -365,6 +365,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'description': 'md5:64249768eec3bc4276236606ea996373',
                 'uploader': 'justintimberlakeVEVO',
                 'uploader_id': 'justintimberlakeVEVO',
+                'age_limit': 18,
             }
         },
         {
@@ -475,6 +476,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader': 'The Witcher',
                 'uploader_id': 'WitcherGame',
                 'upload_date': '20140605',
+                'age_limit': 18,
             },
         },
         # Age-gate video with encrypted signature
@@ -488,6 +490,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader': 'LloydVEVO',
                 'uploader_id': 'LloydVEVO',
                 'upload_date': '20110629',
+                'age_limit': 18,
             },
         },
         # video_info is None (https://github.com/rg3/youtube-dl/issues/4421)

From fb0d12c6cbcabd6f9e84d51c82dea6778d0bb863 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 11 Aug 2015 01:46:25 +0600
Subject: [PATCH 2230/6219] [pbs] Add age limit to tests

---
 youtube_dl/extractor/pbs.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index a53479aad..683c81de3 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -92,6 +92,7 @@ class PBSIE(InfoExtractor):
                 'duration': 3172,
                 'thumbnail': 're:^https?://.*\.jpg$',
                 'upload_date': '20140122',
+                'age_limit': 10,
             },
             'params': {
                 'skip_download': True,  # requires ffmpeg

From b1ac38fadc65049dc6f9611fa7e9649de1e7eb93 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 11 Aug 2015 01:49:23 +0600
Subject: [PATCH 2231/6219] [tvplay] Add age limit to tests

---
 youtube_dl/extractor/tvplay.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/tvplay.py b/youtube_dl/extractor/tvplay.py
index 79863e781..b4683de54 100644
--- a/youtube_dl/extractor/tvplay.py
+++ b/youtube_dl/extractor/tvplay.py
@@ -104,6 +104,7 @@ class TVPlayIE(InfoExtractor):
                 'duration': 1492,
                 'timestamp': 1330522854,
                 'upload_date': '20120229',
+                'age_limit': 18,
             },
             'params': {
                 # rtmp download

From bf812ef71438036c23640f29bd7ae955289720ed Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 11 Aug 2015 23:00:45 +0600
Subject: [PATCH 2232/6219] [downloader/external] Forward --proxy to wget and
 aria2c

---
 youtube_dl/downloader/external.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index 07ce59f7d..49d806ee4 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -51,6 +51,14 @@ class ExternalFD(FileDownloader):
             return []
         return [command_option, source_address]
 
+    def _option(self, command_option, param):
+        param = self.params.get(param)
+        if param is None:
+            return []
+        if isinstance(param, bool):
+            return [command_option]
+        return [command_option, param]
+
     def _no_check_certificate(self, command_option):
         return [command_option] if self.params.get('nocheckcertificate', False) else []
 
@@ -102,6 +110,7 @@ class WgetFD(ExternalFD):
         for key, val in info_dict['http_headers'].items():
             cmd += ['--header', '%s: %s' % (key, val)]
         cmd += self._source_address('--bind-address')
+        cmd += self._option('--proxy', 'proxy')
         cmd += self._no_check_certificate('--no-check-certificate')
         cmd += self._configuration_args()
         cmd += ['--', info_dict['url']]
@@ -120,6 +129,7 @@ class Aria2cFD(ExternalFD):
         for key, val in info_dict['http_headers'].items():
             cmd += ['--header', '%s: %s' % (key, val)]
         cmd += self._source_address('--interface')
+        cmd += self._option('--all-proxy', 'proxy')
         cmd += ['--', info_dict['url']]
         return cmd
 

From 9f3da138606773339de9accc2bc6522ea88185fe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 11 Aug 2015 23:05:04 +0600
Subject: [PATCH 2233/6219] [downloader/external] Use generic _option

---
 youtube_dl/downloader/external.py | 17 ++++-------------
 1 file changed, 4 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index 49d806ee4..6c310346c 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -45,12 +45,6 @@ class ExternalFD(FileDownloader):
     def supports(cls, info_dict):
         return info_dict['protocol'] in ('http', 'https', 'ftp', 'ftps')
 
-    def _source_address(self, command_option):
-        source_address = self.params.get('source_address')
-        if source_address is None:
-            return []
-        return [command_option, source_address]
-
     def _option(self, command_option, param):
         param = self.params.get(param)
         if param is None:
@@ -59,9 +53,6 @@ class ExternalFD(FileDownloader):
             return [command_option]
         return [command_option, param]
 
-    def _no_check_certificate(self, command_option):
-        return [command_option] if self.params.get('nocheckcertificate', False) else []
-
     def _configuration_args(self, default=[]):
         ex_args = self.params.get('external_downloader_args')
         if ex_args is None:
@@ -88,7 +79,7 @@ class CurlFD(ExternalFD):
         cmd = [self.exe, '--location', '-o', tmpfilename]
         for key, val in info_dict['http_headers'].items():
             cmd += ['--header', '%s: %s' % (key, val)]
-        cmd += self._source_address('--interface')
+        cmd += self._option('--interface', 'source_address')
         cmd += self._configuration_args()
         cmd += ['--', info_dict['url']]
         return cmd
@@ -109,9 +100,9 @@ class WgetFD(ExternalFD):
         cmd = [self.exe, '-O', tmpfilename, '-nv', '--no-cookies']
         for key, val in info_dict['http_headers'].items():
             cmd += ['--header', '%s: %s' % (key, val)]
-        cmd += self._source_address('--bind-address')
+        cmd += self._option('--bind-address', 'source_address')
         cmd += self._option('--proxy', 'proxy')
-        cmd += self._no_check_certificate('--no-check-certificate')
+        cmd += self._option('--no-check-certificate', 'nocheckcertificate')
         cmd += self._configuration_args()
         cmd += ['--', info_dict['url']]
         return cmd
@@ -128,7 +119,7 @@ class Aria2cFD(ExternalFD):
         cmd += ['--out', os.path.basename(tmpfilename)]
         for key, val in info_dict['http_headers'].items():
             cmd += ['--header', '%s: %s' % (key, val)]
-        cmd += self._source_address('--interface')
+        cmd += self._option('--interface', 'source_address')
         cmd += self._option('--all-proxy', 'proxy')
         cmd += ['--', info_dict['url']]
         return cmd

From 0a19d4ccd6914d8547fd3e42fd279c960d9f8fad Mon Sep 17 00:00:00 2001
From: sceext <sceext@foxmail.com>
Date: Wed, 12 Aug 2015 14:01:48 +0800
Subject: [PATCH 2234/6219]  [iqiyi] update md5 salt (2015-08-10 Zombie)

---
 youtube_dl/extractor/iqiyi.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index afb7f4e61..dfc6d58a0 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -201,7 +201,7 @@ class IqiyiIE(InfoExtractor):
         return raw_data
 
     def get_enc_key(self, swf_url, video_id):
-        enc_key = '8e29ab5666d041c3a1ea76e06dabdffb'
+        enc_key = '3601ba290e4f4662848c710e2122007e'	# last update at 2015-08-10 for Zombie
         return enc_key
 
     def _real_extract(self, url):

From 1df3186e0e2c49993f4230ec77a9de351177b271 Mon Sep 17 00:00:00 2001
From: ngld <ngld@tproxy.de>
Date: Wed, 12 Aug 2015 16:01:47 +0200
Subject: [PATCH 2235/6219] [funnyordie] Handle protocol-relative URLs (fixes
 #6490)

---
 youtube_dl/extractor/funnyordie.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/funnyordie.py b/youtube_dl/extractor/funnyordie.py
index dd87257c4..f5f13689c 100644
--- a/youtube_dl/extractor/funnyordie.py
+++ b/youtube_dl/extractor/funnyordie.py
@@ -53,7 +53,7 @@ class FunnyOrDieIE(InfoExtractor):
         for bitrate in bitrates:
             for link in links:
                 formats.append({
-                    'url': '%s%d.%s' % (link[0], bitrate, link[1]),
+                    'url': self._proto_relative_url('%s%d.%s' % (link[0], bitrate, link[1])),
                     'format_id': '%s-%d' % (link[1], bitrate),
                     'vbr': bitrate,
                 })

From f57b7835e21b00a1b2205b4bcfba50c630ff68b4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 12 Aug 2015 21:27:58 +0600
Subject: [PATCH 2236/6219] [youtube] Update tests

---
 youtube_dl/extractor/youtube.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index e74a39095..facd837ad 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -442,7 +442,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'id': 'nfWlot6h_JM',
                 'ext': 'm4a',
                 'title': 'Taylor Swift - Shake It Off',
-                'description': 'md5:2acfda1b285bdd478ccec22f9918199d',
+                'description': 'md5:95f66187cd7c8b2c13eb78e1223b63c3',
                 'uploader': 'TaylorSwiftVEVO',
                 'uploader_id': 'TaylorSwiftVEVO',
                 'upload_date': '20140818',
@@ -515,7 +515,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             'info_dict': {
                 'id': 'lqQg6PlCWgI',
                 'ext': 'mp4',
-                'upload_date': '20120731',
+                'upload_date': '20120724',
                 'uploader_id': 'olympic',
                 'description': 'HO09  - Women -  GER-AUS - Hockey - 31 July 2012 - London 2012 Olympic Games',
                 'uploader': 'Olympics',
@@ -544,7 +544,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             'url': 'qEJwOuvDf7I',
             'info_dict': {
                 'id': 'qEJwOuvDf7I',
-                'ext': 'mp4',
+                'ext': 'webm',
                 'title': 'Обсуждение судебной практики по выборам 14 сентября 2014 года в Санкт-Петербурге',
                 'description': '',
                 'upload_date': '20150404',

From f0f3a6c99d2834ca8af87be4978c0040c3744628 Mon Sep 17 00:00:00 2001
From: ngld <ngld@tproxy.de>
Date: Wed, 12 Aug 2015 18:07:27 +0200
Subject: [PATCH 2237/6219] [rtvnhnl] Added new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/rtvnhnl.py  | 40 ++++++++++++++++++++++++++++++++
 2 files changed, 41 insertions(+)
 create mode 100644 youtube_dl/extractor/rtvnhnl.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index dad3ec87f..f026a4171 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -491,6 +491,7 @@ from .rtl2 import RTL2IE
 from .rtp import RTPIE
 from .rts import RTSIE
 from .rtve import RTVEALaCartaIE, RTVELiveIE, RTVEInfantilIE
+from .rtvnhnl import RtvnhNlIE
 from .ruhd import RUHDIE
 from .rutube import (
     RutubeIE,
diff --git a/youtube_dl/extractor/rtvnhnl.py b/youtube_dl/extractor/rtvnhnl.py
new file mode 100644
index 000000000..ce84900a0
--- /dev/null
+++ b/youtube_dl/extractor/rtvnhnl.py
@@ -0,0 +1,40 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class RtvnhNlIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?rtvnh\.nl/video/(?P<id>[0-9]+)'
+    _TEST = {
+        'params': {
+            'hls_prefer_native': True
+        },
+
+        'url': 'http://www.rtvnh.nl/video/131946',
+        'md5': '6e1d0ab079e2a00b6161442d3ceacfc1',
+        'info_dict': {
+            'id': '131946',
+            'ext': 'mp4',
+            'title': 'Grote zoektocht in zee bij Zandvoort naar vermiste vrouw',
+            'thumbnail': 're:^https?://rtvnh-webfiles\.[^.]+\.amazonaws\.com/data/cache/[0-9]+/basedata/pf_image/[0-9.]+/[0-9\-a-f]+\.jpg$'
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        meta = self._parse_json(self._download_webpage('http://www.rtvnh.nl/video/json?m=' + video_id, video_id), video_id)
+        formats = self._extract_smil_formats('http://www.rtvnh.nl/video/smil?m=' + video_id, video_id)
+
+        for item in meta['source']['fb']:
+            if item.get('type') == 'hls':
+                formats.extend(self._extract_m3u8_formats(item['file'], video_id, ext='mp4'))
+            elif item.get('type') == '':
+                formats.append({'url': item['file']})
+        
+        return {
+            'id': video_id,
+            'title': meta['title'].strip(),
+            'thumbnail': meta['image'],
+            'formats': formats
+        }

From fb124e37419668c34b4056575614776b0c64b401 Mon Sep 17 00:00:00 2001
From: ngld <ngld@tproxy.de>
Date: Wed, 12 Aug 2015 20:21:32 +0200
Subject: [PATCH 2238/6219] [rtvnhnl] Relax the thumbnail check

---
 youtube_dl/extractor/rtvnhnl.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/rtvnhnl.py b/youtube_dl/extractor/rtvnhnl.py
index ce84900a0..0921e2648 100644
--- a/youtube_dl/extractor/rtvnhnl.py
+++ b/youtube_dl/extractor/rtvnhnl.py
@@ -17,7 +17,7 @@ class RtvnhNlIE(InfoExtractor):
             'id': '131946',
             'ext': 'mp4',
             'title': 'Grote zoektocht in zee bij Zandvoort naar vermiste vrouw',
-            'thumbnail': 're:^https?://rtvnh-webfiles\.[^.]+\.amazonaws\.com/data/cache/[0-9]+/basedata/pf_image/[0-9.]+/[0-9\-a-f]+\.jpg$'
+            'thumbnail': 're:^http:.*\.jpg$'
         }
     }
 

From d9ab5262b137962995af1b444f45f7f32dc33a77 Mon Sep 17 00:00:00 2001
From: ngld <ngld@tproxy.de>
Date: Wed, 12 Aug 2015 20:26:13 +0200
Subject: [PATCH 2239/6219] [rtvnh] Renamed rtvnhnl -> rtvnh

---
 youtube_dl/extractor/__init__.py              | 2 +-
 youtube_dl/extractor/{rtvnhnl.py => rtvnh.py} | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)
 rename youtube_dl/extractor/{rtvnhnl.py => rtvnh.py} (94%)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index f026a4171..9a6308723 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -491,7 +491,7 @@ from .rtl2 import RTL2IE
 from .rtp import RTPIE
 from .rts import RTSIE
 from .rtve import RTVEALaCartaIE, RTVELiveIE, RTVEInfantilIE
-from .rtvnhnl import RtvnhNlIE
+from .rtvnh import RTVNHIE
 from .ruhd import RUHDIE
 from .rutube import (
     RutubeIE,
diff --git a/youtube_dl/extractor/rtvnhnl.py b/youtube_dl/extractor/rtvnh.py
similarity index 94%
rename from youtube_dl/extractor/rtvnhnl.py
rename to youtube_dl/extractor/rtvnh.py
index 0921e2648..f5c0b94a8 100644
--- a/youtube_dl/extractor/rtvnhnl.py
+++ b/youtube_dl/extractor/rtvnh.py
@@ -4,7 +4,7 @@ from __future__ import unicode_literals
 from .common import InfoExtractor
 
 
-class RtvnhNlIE(InfoExtractor):
+class RTVNHIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?rtvnh\.nl/video/(?P<id>[0-9]+)'
     _TEST = {
         'params': {
@@ -17,7 +17,7 @@ class RtvnhNlIE(InfoExtractor):
             'id': '131946',
             'ext': 'mp4',
             'title': 'Grote zoektocht in zee bij Zandvoort naar vermiste vrouw',
-            'thumbnail': 're:^http:.*\.jpg$'
+            'thumbnail': 're:^https?:.*\.jpg$'
         }
     }
 

From d7dbfc7cc18c2d54d7e1752def6c4710c58b49fc Mon Sep 17 00:00:00 2001
From: ngld <ngld@tproxy.de>
Date: Wed, 12 Aug 2015 20:51:28 +0200
Subject: [PATCH 2240/6219] Use native HLS implementation by default.

---
 youtube_dl/extractor/rtvnh.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/rtvnh.py b/youtube_dl/extractor/rtvnh.py
index f5c0b94a8..2799f01a6 100644
--- a/youtube_dl/extractor/rtvnh.py
+++ b/youtube_dl/extractor/rtvnh.py
@@ -7,10 +7,6 @@ from .common import InfoExtractor
 class RTVNHIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?rtvnh\.nl/video/(?P<id>[0-9]+)'
     _TEST = {
-        'params': {
-            'hls_prefer_native': True
-        },
-
         'url': 'http://www.rtvnh.nl/video/131946',
         'md5': '6e1d0ab079e2a00b6161442d3ceacfc1',
         'info_dict': {
@@ -28,7 +24,7 @@ class RTVNHIE(InfoExtractor):
 
         for item in meta['source']['fb']:
             if item.get('type') == 'hls':
-                formats.extend(self._extract_m3u8_formats(item['file'], video_id, ext='mp4'))
+                formats.extend(self._extract_m3u8_formats(item['file'], video_id, ext='mp4', entry_protocol='m3u8_native'))
             elif item.get('type') == '':
                 formats.append({'url': item['file']})
         

From 240ca32e57a027ff8cec8617c154bb7100bead1a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 13 Aug 2015 01:00:05 +0600
Subject: [PATCH 2241/6219] [rtvnh] Carry long lines

---
 youtube_dl/extractor/rtvnh.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/rtvnh.py b/youtube_dl/extractor/rtvnh.py
index 2799f01a6..998a3c53d 100644
--- a/youtube_dl/extractor/rtvnh.py
+++ b/youtube_dl/extractor/rtvnh.py
@@ -19,12 +19,16 @@ class RTVNHIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        meta = self._parse_json(self._download_webpage('http://www.rtvnh.nl/video/json?m=' + video_id, video_id), video_id)
-        formats = self._extract_smil_formats('http://www.rtvnh.nl/video/smil?m=' + video_id, video_id)
+
+        meta = self._parse_json(self._download_webpage(
+            'http://www.rtvnh.nl/video/json?m=' + video_id, video_id), video_id)
+        formats = self._extract_smil_formats(
+            'http://www.rtvnh.nl/video/smil?m=' + video_id, video_id)
 
         for item in meta['source']['fb']:
             if item.get('type') == 'hls':
-                formats.extend(self._extract_m3u8_formats(item['file'], video_id, ext='mp4', entry_protocol='m3u8_native'))
+                formats.extend(self._extract_m3u8_formats(
+                    item['file'], video_id, ext='mp4', entry_protocol='m3u8_native'))
             elif item.get('type') == '':
                 formats.append({'url': item['file']})
         

From f196047832a2da74d5adf75759877b5d95ec5b5b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 13 Aug 2015 01:00:25 +0600
Subject: [PATCH 2242/6219] [rtvnh] Make thumbnail optional

---
 youtube_dl/extractor/rtvnh.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/rtvnh.py b/youtube_dl/extractor/rtvnh.py
index 998a3c53d..d576a3410 100644
--- a/youtube_dl/extractor/rtvnh.py
+++ b/youtube_dl/extractor/rtvnh.py
@@ -35,6 +35,6 @@ class RTVNHIE(InfoExtractor):
         return {
             'id': video_id,
             'title': meta['title'].strip(),
-            'thumbnail': meta['image'],
+            'thumbnail': meta.get('image'),
             'formats': formats
         }

From 60231c65b9a50e08967d748c3ed401488fed3587 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 13 Aug 2015 01:02:50 +0600
Subject: [PATCH 2243/6219] [rtvnh] Make SMIL not fatal

---
 youtube_dl/extractor/rtvnh.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/rtvnh.py b/youtube_dl/extractor/rtvnh.py
index d576a3410..202ea0181 100644
--- a/youtube_dl/extractor/rtvnh.py
+++ b/youtube_dl/extractor/rtvnh.py
@@ -23,7 +23,7 @@ class RTVNHIE(InfoExtractor):
         meta = self._parse_json(self._download_webpage(
             'http://www.rtvnh.nl/video/json?m=' + video_id, video_id), video_id)
         formats = self._extract_smil_formats(
-            'http://www.rtvnh.nl/video/smil?m=' + video_id, video_id)
+            'http://www.rtvnh.nl/video/smil?m=' + video_id, video_id, fatal=False)
 
         for item in meta['source']['fb']:
             if item.get('type') == 'hls':

From 2c919adb74893544ab6def1d56ff8ed37c282ecb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 13 Aug 2015 01:11:55 +0600
Subject: [PATCH 2244/6219] [rtvnh] Check status code

---
 youtube_dl/extractor/rtvnh.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/rtvnh.py b/youtube_dl/extractor/rtvnh.py
index 202ea0181..7c9d4b0cd 100644
--- a/youtube_dl/extractor/rtvnh.py
+++ b/youtube_dl/extractor/rtvnh.py
@@ -2,6 +2,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..utils import ExtractorError
 
 
 class RTVNHIE(InfoExtractor):
@@ -22,6 +23,12 @@ class RTVNHIE(InfoExtractor):
 
         meta = self._parse_json(self._download_webpage(
             'http://www.rtvnh.nl/video/json?m=' + video_id, video_id), video_id)
+
+        status = meta.get('status')
+        if status != 200:
+            raise ExtractorError(
+                '%s returned error code %d' % (self.IE_NAME, status), expected=True)
+
         formats = self._extract_smil_formats(
             'http://www.rtvnh.nl/video/smil?m=' + video_id, video_id, fatal=False)
 
@@ -31,7 +38,7 @@ class RTVNHIE(InfoExtractor):
                     item['file'], video_id, ext='mp4', entry_protocol='m3u8_native'))
             elif item.get('type') == '':
                 formats.append({'url': item['file']})
-        
+
         return {
             'id': video_id,
             'title': meta['title'].strip(),

From 3b7130439aade87b628fa6dd727df5860323a68f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 13 Aug 2015 01:15:58 +0600
Subject: [PATCH 2245/6219] Credit @ngld for RTVNH (#6537)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index d16d34272..71c420165 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -137,3 +137,4 @@ Zach Bruggeman
 Tjark Saul
 slangangular
 Behrouz Abbasi
+ngld

From b6b2711298f8d43414deac939f92c7c3477826b2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 13 Aug 2015 01:17:15 +0600
Subject: [PATCH 2246/6219] [tweakers] Remove unused imports

---
 youtube_dl/extractor/tweakers.py | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/youtube_dl/extractor/tweakers.py b/youtube_dl/extractor/tweakers.py
index 6eeffb1cc..f3198fb85 100644
--- a/youtube_dl/extractor/tweakers.py
+++ b/youtube_dl/extractor/tweakers.py
@@ -1,12 +1,6 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..utils import (
-    xpath_text,
-    xpath_with_ns,
-    int_or_none,
-    float_or_none,
-)
 
 
 class TweakersIE(InfoExtractor):

From e73c85cb23d278702357412479fd4b162a3abbb3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 13 Aug 2015 01:18:49 +0600
Subject: [PATCH 2247/6219] [iqiyi] PEP 8

---
 youtube_dl/extractor/iqiyi.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index dfc6d58a0..393e67e35 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -201,7 +201,7 @@ class IqiyiIE(InfoExtractor):
         return raw_data
 
     def get_enc_key(self, swf_url, video_id):
-        enc_key = '3601ba290e4f4662848c710e2122007e'	# last update at 2015-08-10 for Zombie
+        enc_key = '3601ba290e4f4662848c710e2122007e'  # last update at 2015-08-10 for Zombie
         return enc_key
 
     def _real_extract(self, url):

From 237c03c8eaa4da1713a635e87f98ac14430b35cb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 13 Aug 2015 01:19:23 +0600
Subject: [PATCH 2248/6219] [dhm] Remove unused import

---
 youtube_dl/extractor/dhm.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/dhm.py b/youtube_dl/extractor/dhm.py
index 127eb0439..44e0c5d4d 100644
--- a/youtube_dl/extractor/dhm.py
+++ b/youtube_dl/extractor/dhm.py
@@ -1,10 +1,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..utils import (
-    xpath_text,
-    parse_duration,
-)
+from ..utils import parse_duration
 
 
 class DHMIE(InfoExtractor):

From 28479149ccf3425e6a6e35d3a155f6802629728a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 13 Aug 2015 12:56:12 +0800
Subject: [PATCH 2249/6219] [theplatform] Fallback to hardcoded releaseUrl if
 not available

Fixes #6546.

Not adding a test case as test_NBC has the same problem.
---
 youtube_dl/extractor/theplatform.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index 83d833e30..0643eccaf 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -108,7 +108,11 @@ class ThePlatformIE(InfoExtractor):
             config_url = config_url.replace('swf/', 'config/')
             config_url = config_url.replace('onsite/', 'onsite/config/')
             config = self._download_json(config_url, video_id, 'Downloading config')
-            smil_url = config['releaseUrl'] + '&format=SMIL&formats=MPEG4&manifest=f4m'
+            if 'releaseUrl' in config:
+                release_url = config['releaseUrl']
+            else:
+                release_url = 'http://link.theplatform.com/s/%s?mbr=true' % path
+            smil_url = release_url + '&format=SMIL&formats=MPEG4&manifest=f4m'
         else:
             smil_url = 'http://link.theplatform.com/s/%s/meta.smil?format=smil&mbr=true' % path
 

From 6828c809e44fca7b19da3c62a11cea313a86b64e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 13 Aug 2015 21:07:14 +0600
Subject: [PATCH 2250/6219] [downloader/fragment] Respect --retries for
 fragment based downloaders (Closes #6549)

---
 youtube_dl/downloader/fragment.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
index 5f9d6796d..5a64b29ee 100644
--- a/youtube_dl/downloader/fragment.py
+++ b/youtube_dl/downloader/fragment.py
@@ -35,6 +35,7 @@ class FragmentFD(FileDownloader):
                 'quiet': True,
                 'noprogress': True,
                 'ratelimit': self.params.get('ratelimit', None),
+                'retries': self.params.get('retries', 0),
                 'test': self.params.get('test', False),
             }
         )

From 7393746da213bec686f8425165854e5e383b7eb9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 13 Aug 2015 21:10:11 +0600
Subject: [PATCH 2251/6219] [downloader/hls] Add _debug_cmd

---
 youtube_dl/downloader/hls.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index 60dca0ab1..2b6c3370f 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -32,6 +32,8 @@ class HlsFD(FileDownloader):
             for opt in (ffpp.executable, '-y', '-i', url, '-f', 'mp4', '-c', 'copy', '-bsf:a', 'aac_adtstoasc')]
         args.append(encodeFilename(tmpfilename, True))
 
+        self._debug_cmd(args)
+
         retval = subprocess.call(args)
         if retval == 0:
             fsize = os.path.getsize(encodeFilename(tmpfilename))

From cb28e0338665c96b2d5b35d203b1d54a57f3feb1 Mon Sep 17 00:00:00 2001
From: nyuszika7h <nyuszika7h@openmailbox.org>
Date: Mon, 10 Aug 2015 19:27:16 +0200
Subject: [PATCH 2252/6219] [indavideo] Add new extractor

Closes #2147.
---
 youtube_dl/extractor/__init__.py  |  1 +
 youtube_dl/extractor/indavideo.py | 79 +++++++++++++++++++++++++++++++
 2 files changed, 80 insertions(+)
 create mode 100644 youtube_dl/extractor/indavideo.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 9a6308723..3bcfa93bb 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -242,6 +242,7 @@ from .imdb import (
 )
 from .imgur import ImgurIE
 from .ina import InaIE
+from .indavideo import IndavideoIE
 from .infoq import InfoQIE
 from .instagram import InstagramIE, InstagramUserIE
 from .internetvideoarchive import InternetVideoArchiveIE
diff --git a/youtube_dl/extractor/indavideo.py b/youtube_dl/extractor/indavideo.py
new file mode 100644
index 000000000..2a2cf2bd3
--- /dev/null
+++ b/youtube_dl/extractor/indavideo.py
@@ -0,0 +1,79 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .. import utils
+from .common import InfoExtractor
+
+
+class IndavideoIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?indavideo\.hu/video/(?P<id>.+)'
+    _TESTS = [
+        {
+            'url': 'http://indavideo.hu/video/Cicatanc',
+            'md5': 'c8a507a1c7410685f83a06eaeeaafeab',
+            'info_dict': {
+                'id': '1837039',
+                'title': 'Cicatánc',
+                'ext': 'mp4',
+                'display_id': 'Cicatanc',
+                'thumbnail': 're:^https?://.*\.jpg$',
+                'description': '',
+                'uploader': 'cukiajanlo',
+                'uploader_id': '83729',
+                'duration': 72,
+                'age_limit': 0,
+                'tags': ['tánc', 'cica', 'cuki', 'cukiajanlo', 'newsroom']
+            },
+        },
+        {
+            'url': 'http://indavideo.hu/video/Vicces_cica_1',
+            'md5': '8c82244ba85d2a2310275b318eb51eac',
+            'info_dict': {
+                'id': '1335611',
+                'title': 'Vicces cica',
+                'ext': 'mp4',
+                'display_id': 'Vicces_cica_1',
+                'thumbnail': 're:^https?://.*\.jpg$',
+                'description': 'Játszik a tablettel. :D',
+                'uploader': 'Jet_Pack',
+                'uploader_id': '491217',
+                'duration': 7,
+                'age_limit': 0,
+                'tags': ['vicces', 'macska', 'cica', 'ügyes', 'nevetés', 'játszik', 'Cukiság', 'Jet_Pack'],
+            },
+        },
+    ]
+
+    def _real_extract(self, url):
+        video_disp_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_disp_id)
+
+        embed_url = self._html_search_regex(r'<link rel="video_src" href="(.+?)"/>', webpage, 'embed_url')
+        video_hash = embed_url.split('/')[-1]
+
+        payload = self._download_json('http://amfphp.indavideo.hu/SYm0json.php/player.playerHandler.getVideoData/' + video_hash, video_disp_id)
+        video_info = payload['data']
+
+        thumbnails = video_info.get('thumbnails')
+        if thumbnails:
+            thumbnails = [{'url': self._proto_relative_url(x)} for x in thumbnails]
+
+        tags = video_info.get('tags')
+        if tags:
+            tags = [x['title'] for x in tags]
+
+        return {
+            'id': video_info.get('id'),
+            'title': video_info['title'],
+            'url': video_info['video_file'],
+            'ext': 'mp4',
+            'display_id': video_disp_id,
+            'thumbnails': thumbnails,
+            'description': video_info.get('description'),
+            'uploader': video_info.get('user_name'),
+            # TODO: upload date (it's in CET/CEST)
+            'uploader_id': video_info.get('user_id'),
+            'duration': utils.int_or_none(video_info.get('length')),
+            'age_limit': utils.int_or_none(video_info.get('age_limit')),
+            'tags': tags,
+        }

From 3c12a027d48a2d6d1162ab515df0308237aef881 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 13 Aug 2015 23:25:47 +0600
Subject: [PATCH 2253/6219] [indavideo] Split in two extractors, extract all
 formats and fix timestamp

---
 youtube_dl/extractor/__init__.py  |   5 +-
 youtube_dl/extractor/indavideo.py | 178 +++++++++++++++++++-----------
 2 files changed, 118 insertions(+), 65 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 3bcfa93bb..83d21bd15 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -242,7 +242,10 @@ from .imdb import (
 )
 from .imgur import ImgurIE
 from .ina import InaIE
-from .indavideo import IndavideoIE
+from .indavideo import (
+    IndavideoIE,
+    IndavideoEmbedIE,
+)
 from .infoq import InfoQIE
 from .instagram import InstagramIE, InstagramUserIE
 from .internetvideoarchive import InternetVideoArchiveIE
diff --git a/youtube_dl/extractor/indavideo.py b/youtube_dl/extractor/indavideo.py
index 2a2cf2bd3..b75715244 100644
--- a/youtube_dl/extractor/indavideo.py
+++ b/youtube_dl/extractor/indavideo.py
@@ -3,77 +3,127 @@ from __future__ import unicode_literals
 
 from .. import utils
 from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    parse_age_limit,
+    parse_iso8601,
+)
+
+
+class IndavideoEmbedIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:(?:embed\.)?indavideo\.hu/player/video/|assets\.indavideo\.hu/swf/player\.swf\?.*\b(?:v(?:ID|id))=)(?P<id>[\da-f]+)'
+    _TESTS = [{
+        'url': 'http://indavideo.hu/player/video/1bdc3c6d80/',
+        'md5': 'f79b009c66194acacd40712a6778acfa',
+        'info_dict': {
+            'id': '1837039',
+            'ext': 'mp4',
+            'title': 'Cicatánc',
+            'description': '',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'uploader': 'cukiajanlo',
+            'uploader_id': '83729',
+            'timestamp': 1439193826,
+            'upload_date': '20150810',
+            'duration': 72,
+            'age_limit': 0,
+            'tags': ['tánc', 'cica', 'cuki', 'cukiajanlo', 'newsroom'],
+        },
+    }, {
+        'url': 'http://embed.indavideo.hu/player/video/1bdc3c6d80?autostart=1&hide=1',
+        'only_matching': True,
+    }, {
+        'url': 'http://assets.indavideo.hu/swf/player.swf?v=fe25e500&vID=1bdc3c6d80&autostart=1&hide=1&i=1',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        video = self._download_json(
+            'http://amfphp.indavideo.hu/SYm0json.php/player.playerHandler.getVideoData/%s' % video_id,
+            video_id)['data']
+
+        video_id = video['id']
+        title = video['title']
+
+        video_urls = video.get('video_files', [])
+        video_file = video.get('video_file')
+        if video:
+            video_urls.append(video_file)
+        video_urls = list(set(video_urls))
+
+        video_prefix = video_urls[0].rsplit('/', 1)[0]
+
+        for flv_file in video.get('flv_files', []):
+            flv_url = '%s/%s' % (video_prefix, flv_file)
+            if flv_url not in video_urls:
+                video_urls.append(flv_url)
+
+        formats = [{
+            'url': video_url,
+            'height': self._search_regex(r'\.(\d{3,4})\.mp4$', video_url, 'height', default=None),
+        } for video_url in video_urls]
+        self._sort_formats(formats)
+
+        timestamp = video.get('date')
+        if timestamp:
+            # upload date is in CEST
+            timestamp = parse_iso8601(timestamp + ' +0200', ' ')
+
+        thumbnails = [{
+            'url': self._proto_relative_url(thumbnail)
+        } for thumbnail in video.get('thumbnails', [])]
+
+        tags = [tag['title'] for tag in video.get('tags', [])]
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': video.get('description'),
+            'thumbnails': thumbnails,
+            'uploader': video.get('user_name'),
+            'uploader_id': video.get('user_id'),
+            'timestamp': timestamp,
+            'duration': int_or_none(video.get('length')),
+            'age_limit': parse_age_limit(video.get('age_limit')),
+            'tags': tags,
+            'formats': formats,
+        }
 
 
 class IndavideoIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?indavideo\.hu/video/(?P<id>.+)'
-    _TESTS = [
-        {
-            'url': 'http://indavideo.hu/video/Cicatanc',
-            'md5': 'c8a507a1c7410685f83a06eaeeaafeab',
-            'info_dict': {
-                'id': '1837039',
-                'title': 'Cicatánc',
-                'ext': 'mp4',
-                'display_id': 'Cicatanc',
-                'thumbnail': 're:^https?://.*\.jpg$',
-                'description': '',
-                'uploader': 'cukiajanlo',
-                'uploader_id': '83729',
-                'duration': 72,
-                'age_limit': 0,
-                'tags': ['tánc', 'cica', 'cuki', 'cukiajanlo', 'newsroom']
-            },
+    _VALID_URL = r'https?://(?:www\.)?indavideo\.hu/video/(?P<id>[^/#?]+)'
+    _TEST = {
+        'url': 'http://indavideo.hu/video/Vicces_cica_1',
+        'md5': '8c82244ba85d2a2310275b318eb51eac',
+        'info_dict': {
+            'id': '1335611',
+            'display_id': 'Vicces_cica_1',
+            'ext': 'mp4',
+            'title': 'Vicces cica',
+            'description': 'Játszik a tablettel. :D',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'uploader': 'Jet_Pack',
+            'uploader_id': '491217',
+            'timestamp': 1390821212,
+            'upload_date': '20140127',
+            'duration': 7,
+            'age_limit': 0,
+            'tags': ['vicces', 'macska', 'cica', 'ügyes', 'nevetés', 'játszik', 'Cukiság', 'Jet_Pack'],
         },
-        {
-            'url': 'http://indavideo.hu/video/Vicces_cica_1',
-            'md5': '8c82244ba85d2a2310275b318eb51eac',
-            'info_dict': {
-                'id': '1335611',
-                'title': 'Vicces cica',
-                'ext': 'mp4',
-                'display_id': 'Vicces_cica_1',
-                'thumbnail': 're:^https?://.*\.jpg$',
-                'description': 'Játszik a tablettel. :D',
-                'uploader': 'Jet_Pack',
-                'uploader_id': '491217',
-                'duration': 7,
-                'age_limit': 0,
-                'tags': ['vicces', 'macska', 'cica', 'ügyes', 'nevetés', 'játszik', 'Cukiság', 'Jet_Pack'],
-            },
-        },
-    ]
+    }
 
     def _real_extract(self, url):
-        video_disp_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_disp_id)
+        display_id = self._match_id(url)
 
-        embed_url = self._html_search_regex(r'<link rel="video_src" href="(.+?)"/>', webpage, 'embed_url')
-        video_hash = embed_url.split('/')[-1]
-
-        payload = self._download_json('http://amfphp.indavideo.hu/SYm0json.php/player.playerHandler.getVideoData/' + video_hash, video_disp_id)
-        video_info = payload['data']
-
-        thumbnails = video_info.get('thumbnails')
-        if thumbnails:
-            thumbnails = [{'url': self._proto_relative_url(x)} for x in thumbnails]
-
-        tags = video_info.get('tags')
-        if tags:
-            tags = [x['title'] for x in tags]
+        webpage = self._download_webpage(url, display_id)
+        embed_url = self._search_regex(
+            r'<link[^>]+rel="video_src"[^>]+href="(.+?)"', webpage, 'embed url')
 
         return {
-            'id': video_info.get('id'),
-            'title': video_info['title'],
-            'url': video_info['video_file'],
-            'ext': 'mp4',
-            'display_id': video_disp_id,
-            'thumbnails': thumbnails,
-            'description': video_info.get('description'),
-            'uploader': video_info.get('user_name'),
-            # TODO: upload date (it's in CET/CEST)
-            'uploader_id': video_info.get('user_id'),
-            'duration': utils.int_or_none(video_info.get('length')),
-            'age_limit': utils.int_or_none(video_info.get('age_limit')),
-            'tags': tags,
+            '_type': 'url_transparent',
+            'ie_key': 'IndavideoEmbed',
+            'url': embed_url,
+            'display_id': display_id,
         }

From a34e19629c407a08cd9065223f26f1f5468a4423 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 13 Aug 2015 23:40:20 +0600
Subject: [PATCH 2254/6219] [indavideo] Relax _VALID_URL to match subdomains
 and add tests

---
 youtube_dl/extractor/indavideo.py | 24 +++++++++++++++++++-----
 1 file changed, 19 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/indavideo.py b/youtube_dl/extractor/indavideo.py
index b75715244..550a7001b 100644
--- a/youtube_dl/extractor/indavideo.py
+++ b/youtube_dl/extractor/indavideo.py
@@ -44,7 +44,6 @@ class IndavideoEmbedIE(InfoExtractor):
             'http://amfphp.indavideo.hu/SYm0json.php/player.playerHandler.getVideoData/%s' % video_id,
             video_id)['data']
 
-        video_id = video['id']
         title = video['title']
 
         video_urls = video.get('video_files', [])
@@ -78,7 +77,7 @@ class IndavideoEmbedIE(InfoExtractor):
         tags = [tag['title'] for tag in video.get('tags', [])]
 
         return {
-            'id': video_id,
+            'id': video.get('id') or video_id,
             'title': title,
             'description': video.get('description'),
             'thumbnails': thumbnails,
@@ -93,8 +92,8 @@ class IndavideoEmbedIE(InfoExtractor):
 
 
 class IndavideoIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?indavideo\.hu/video/(?P<id>[^/#?]+)'
-    _TEST = {
+    _VALID_URL = r'https?://(?:.+?\.)?indavideo\.hu/video/(?P<id>[^/#?]+)'
+    _TESTS = [{
         'url': 'http://indavideo.hu/video/Vicces_cica_1',
         'md5': '8c82244ba85d2a2310275b318eb51eac',
         'info_dict': {
@@ -112,7 +111,22 @@ class IndavideoIE(InfoExtractor):
             'age_limit': 0,
             'tags': ['vicces', 'macska', 'cica', 'ügyes', 'nevetés', 'játszik', 'Cukiság', 'Jet_Pack'],
         },
-    }
+    }, {
+        'url': 'http://index.indavideo.hu/video/2015_0728_beregszasz',
+        'only_matching': True,
+    }, {
+        'url': 'http://auto.indavideo.hu/video/Sajat_utanfutoban_a_kis_tacsko',
+        'only_matching': True,
+    }, {
+        'url': 'http://erotika.indavideo.hu/video/Amator_tini_punci',
+        'only_matching': True,
+    }, {
+        'url': 'http://film.indavideo.hu/video/f_hrom_nagymamm_volt',
+        'only_matching': True,
+    }, {
+        'url': 'http://palyazat.indavideo.hu/video/Embertelen_dal_Dodgem_egyuttes',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         display_id = self._match_id(url)

From fb56131dd9cf3bfa31d7d6920a135281d151f803 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 13 Aug 2015 23:47:12 +0600
Subject: [PATCH 2255/6219] Credit @nyuszika7h for indavideo (#6517)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 71c420165..ded9e87d2 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -138,3 +138,4 @@ Tjark Saul
 slangangular
 Behrouz Abbasi
 ngld
+nyuszika7h

From 594f51b85934878ff20b608f312d7f564e3a3d71 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 13 Aug 2015 23:47:49 +0600
Subject: [PATCH 2256/6219] [indavideo] Remove unused import

---
 youtube_dl/extractor/indavideo.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/indavideo.py b/youtube_dl/extractor/indavideo.py
index 550a7001b..12fb5e8e1 100644
--- a/youtube_dl/extractor/indavideo.py
+++ b/youtube_dl/extractor/indavideo.py
@@ -1,7 +1,6 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-from .. import utils
 from .common import InfoExtractor
 from ..utils import (
     int_or_none,

From 3cafca04aaf2bfc4d31e8255b9cb75e8f1ad4b16 Mon Sep 17 00:00:00 2001
From: reddraggone9 <cljenkins9@gmail.com>
Date: Fri, 14 Aug 2015 00:35:35 -0500
Subject: [PATCH 2257/6219] Updated line numbers in the fragment portion of
 README links.

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 15baf75ce..e91119d84 100644
--- a/README.md
+++ b/README.md
@@ -544,7 +544,7 @@ If you want to add support for a new site, you can follow this quick list (assum
     ```
 5. Add an import in [`youtube_dl/extractor/__init__.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/__init__.py).
 6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will be then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc.
-7. Have a look at [`youtube_dl/common/extractor/common.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should return](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L38). Add tests and code for as many as you want.
+7. Have a look at [`youtube_dl/common/extractor/common.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should return](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L62). Add tests and code for as many as you want.
 8. If you can, check the code with [flake8](https://pypi.python.org/pypi/flake8).
 9. When the tests pass, [add](http://git-scm.com/docs/git-add) the new files and [commit](http://git-scm.com/docs/git-commit) them and [push](http://git-scm.com/docs/git-push) the result, like this:
 
@@ -572,7 +572,7 @@ with youtube_dl.YoutubeDL(ydl_opts) as ydl:
     ydl.download(['http://www.youtube.com/watch?v=BaW_jenozKc'])
 ```
 
-Most likely, you'll want to use various options. For a list of what can be done, have a look at [youtube_dl/YoutubeDL.py](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/YoutubeDL.py#L69). For a start, if you want to intercept youtube-dl's output, set a `logger` object.
+Most likely, you'll want to use various options. For a list of what can be done, have a look at [youtube_dl/YoutubeDL.py](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/YoutubeDL.py#L92). For a start, if you want to intercept youtube-dl's output, set a `logger` object.
 
 Here's a more complete example of a program that outputs only errors (and a short message after the download is finished), and downloads/converts the video to an mp3 file:
 

From 4d2ad866f347086d3a1cf4cb7e0a8cadd3c87748 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 14 Aug 2015 19:18:03 +0800
Subject: [PATCH 2258/6219] [README.md] Document format_id field in output
 template section (#6557)

---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index 15baf75ce..8fa402ee2 100644
--- a/README.md
+++ b/README.md
@@ -272,6 +272,7 @@ The `-o` option allows users to indicate a template for the output file names. T
  - `autonumber`: The sequence will be replaced by a five-digit number that will be increased with each download, starting at zero.
  - `playlist`: The name or the id of the playlist that contains the video.
  - `playlist_index`: The index of the video in the playlist, a five-digit number.
+ - `format_id`: The sequence will be replaced by the format code specified by `--format`.
 
 The current default template is `%(title)s-%(id)s.%(ext)s`.
 

From 41dbc50f9c7dfaad4084fbeac77192c7ac37daca Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 14 Aug 2015 22:07:02 +0600
Subject: [PATCH 2259/6219] [lynda] Capture and output login error (Closes
 #6556)

---
 youtube_dl/extractor/lynda.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index deead220a..5b9157ed4 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -11,6 +11,7 @@ from ..compat import (
 )
 from ..utils import (
     ExtractorError,
+    clean_html,
     int_or_none,
 )
 
@@ -70,6 +71,15 @@ class LyndaBaseIE(InfoExtractor):
                     'Confirming log in and log out from another device')
 
         if all(not re.search(p, login_page) for p in ('isLoggedIn\s*:\s*true', r'logout\.aspx', r'>Log out<')):
+            if 'login error' in login_page:
+                mobj = re.search(
+                    r'(?s)<h1[^>]+class="topmost">(?P<title>[^<]+)</h1>\s*<div>(?P<description>.+?)</div>',
+                    login_page)
+                if mobj:
+                    raise ExtractorError(
+                        'lynda returned error: %s - %s'
+                        % (mobj.group('title'), clean_html(mobj.group('description'))),
+                        expected=True)
             raise ExtractorError('Unable to log in')
 
 

From 6be5e46994ea5db76d7a2659260606898c265957 Mon Sep 17 00:00:00 2001
From: "Sergey M." <dstftw@gmail.com>
Date: Fri, 14 Aug 2015 22:22:39 +0600
Subject: [PATCH 2260/6219] [README.md] Clarify line ranges

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index e52cdb941..542a7c26a 100644
--- a/README.md
+++ b/README.md
@@ -545,7 +545,7 @@ If you want to add support for a new site, you can follow this quick list (assum
     ```
 5. Add an import in [`youtube_dl/extractor/__init__.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/__init__.py).
 6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will be then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc.
-7. Have a look at [`youtube_dl/common/extractor/common.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should return](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L62). Add tests and code for as many as you want.
+7. Have a look at [`youtube_dl/common/extractor/common.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L62-L200). Add tests and code for as many as you want.
 8. If you can, check the code with [flake8](https://pypi.python.org/pypi/flake8).
 9. When the tests pass, [add](http://git-scm.com/docs/git-add) the new files and [commit](http://git-scm.com/docs/git-commit) them and [push](http://git-scm.com/docs/git-push) the result, like this:
 
@@ -573,7 +573,7 @@ with youtube_dl.YoutubeDL(ydl_opts) as ydl:
     ydl.download(['http://www.youtube.com/watch?v=BaW_jenozKc'])
 ```
 
-Most likely, you'll want to use various options. For a list of what can be done, have a look at [youtube_dl/YoutubeDL.py](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/YoutubeDL.py#L92). For a start, if you want to intercept youtube-dl's output, set a `logger` object.
+Most likely, you'll want to use various options. For a list of what can be done, have a look at [youtube_dl/YoutubeDL.py](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/YoutubeDL.py#L117-L265). For a start, if you want to intercept youtube-dl's output, set a `logger` object.
 
 Here's a more complete example of a program that outputs only errors (and a short message after the download is finished), and downloads/converts the video to an mp3 file:
 

From d0d6c097fc7859180f16a445536029c600b1e57f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 15 Aug 2015 15:17:27 +0800
Subject: [PATCH 2261/6219] [moniker] Support embed- URLs (#6450)

---
 youtube_dl/extractor/moniker.py | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/moniker.py b/youtube_dl/extractor/moniker.py
index 88dcd4f73..69e4bcd1a 100644
--- a/youtube_dl/extractor/moniker.py
+++ b/youtube_dl/extractor/moniker.py
@@ -9,7 +9,10 @@ from ..compat import (
     compat_urllib_parse,
     compat_urllib_request,
 )
-from ..utils import ExtractorError
+from ..utils import (
+    ExtractorError,
+    remove_start,
+)
 
 
 class MonikerIE(InfoExtractor):
@@ -24,6 +27,14 @@ class MonikerIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'youtube-dl test video',
         },
+    }, {
+        'url': 'http://allmyvideos.net/embed-jih3nce3x6wn',
+        'md5': '710883dee1bfc370ecf9fa6a89307c88',
+        'info_dict': {
+            'id': 'jih3nce3x6wn',
+            'ext': 'mp4',
+            'title': 'youtube-dl test video',
+        },
     }, {
         'url': 'http://vidspot.net/l2ngsmhs8ci5',
         'md5': '710883dee1bfc370ecf9fa6a89307c88',
@@ -38,7 +49,10 @@ class MonikerIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        orig_video_id = self._match_id(url)
+        video_id = remove_start(orig_video_id, 'embed-')
+        url = url.replace(orig_video_id, video_id)
+        assert re.match(self._VALID_URL, url) is not None
         orig_webpage = self._download_webpage(url, video_id)
 
         if '>File Not Found<' in orig_webpage:

From 8b8c1093b65ee02aad859ed8d82217312ed0d9d8 Mon Sep 17 00:00:00 2001
From: Shaun Walbridge <shaun.walbridge@gmail.com>
Date: Sat, 18 Apr 2015 00:37:04 -0400
Subject: [PATCH 2262/6219] [EsriVideo] Add new extractor

Add extractor for [videos.esri.com](https://videos.esri.com), a collection
of videos relating to GIS.
---
 youtube_dl/extractor/__init__.py  |  1 +
 youtube_dl/extractor/videoesri.py | 90 +++++++++++++++++++++++++++++++
 2 files changed, 91 insertions(+)
 create mode 100644 youtube_dl/extractor/videoesri.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 83d21bd15..a4387636f 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -695,6 +695,7 @@ from .vice import ViceIE
 from .viddler import ViddlerIE
 from .videobam import VideoBamIE
 from .videodetective import VideoDetectiveIE
+from .videoesri import VideoEsriIE
 from .videolecturesnet import VideoLecturesNetIE
 from .videofyme import VideofyMeIE
 from .videomega import VideoMegaIE
diff --git a/youtube_dl/extractor/videoesri.py b/youtube_dl/extractor/videoesri.py
new file mode 100644
index 000000000..0f84323a4
--- /dev/null
+++ b/youtube_dl/extractor/videoesri.py
@@ -0,0 +1,90 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import os
+import re
+
+from .common import InfoExtractor
+
+from ..utils import (
+    unified_strdate
+)
+
+
+class VideoEsriIE(InfoExtractor):
+    _VALID_URL = r'https?://video\.esri\.com/watch/(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'https://video.esri.com/watch/4228',
+        'md5': '170b4d513c2466ed483c150a48384133',
+        'info_dict': {
+            'id': '4228',
+            'ext': 'mp4',
+            'title': 'AppStudio for ArcGIS',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'upload_date': '20150310',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        title = self._html_search_regex(r'<h1>(.*?)</h1>', webpage, 'title')
+
+        upload_date_raw = self._search_regex(
+            r'http-equiv="last-modified" content="(.*)"',
+            webpage, 'upload date')
+        upload_date = unified_strdate(upload_date_raw)
+
+        settings_info = self._search_regex(
+            r'evPlayerSettings = {(.*?);\s*$',
+            webpage, 'settings info', flags=re.MULTILINE | re.DOTALL)
+
+        # thumbnail includes '_x' for large, also has {_m,_t,_s} or
+        # without size suffix returns full image
+        thumbnail_path = re.findall(
+            r'image\': \'(\/thumbs.*)\'',
+            settings_info)[0]
+
+        if thumbnail_path:
+            thumbnail = '/'.join(['http://video.esri.com', thumbnail_path])
+
+        # note that this misses the (exceedly rare) webm files
+        video_paths = re.findall(r'mp4:(.*)\'', settings_info)
+
+        # find possible http servers of the mp4 files (also has rtsp)
+        base_url = re.findall(
+            r'netstreambasepath\':\s\'(h.*)\'', settings_info)[0]
+
+        # these are the numbers used internally, but really map
+        # to other resolutions, e.g. 960 is 720p.
+        heights = [480, 720, 960]
+        videos_by_res = {}
+        for video_path in video_paths:
+            url = "{base_url}{video_path}".format(
+                base_url=base_url,
+                video_path=video_path)
+            filename, ext = os.path.splitext(video_path)
+            height_label = int(filename.split('_')[1])
+            videos_by_res[height_label] = {
+                'url': url,
+                'ext': ext[1:],
+                'protocol': 'http',  # http-only supported currently
+            }
+
+        formats = []
+        for height in heights:
+            if height in videos_by_res:
+                formats.append(videos_by_res[height])
+
+        result = {
+            'id': video_id,
+            'title': title,
+            'upload_date': upload_date,
+            'formats': formats,
+        }
+
+        if thumbnail:
+            result['thumbnail'] = thumbnail
+
+        return result

From 8b9848ac5678356757f67a412f7ed89a0f559be7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 15 Aug 2015 15:58:30 +0600
Subject: [PATCH 2263/6219] [extractor/common] Expand meta regex

---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 5982055be..16ae4b98f 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -640,7 +640,7 @@ class InfoExtractor(object):
     @staticmethod
     def _meta_regex(prop):
         return r'''(?isx)<meta
-                    (?=[^>]+(?:itemprop|name|property|id)=(["\']?)%s\1)
+                    (?=[^>]+(?:itemprop|name|property|id|http-equiv)=(["\']?)%s\1)
                     [^>]+?content=(["\'])(?P<content>.*?)\2''' % re.escape(prop)
 
     def _og_search_property(self, prop, html, name=None, **kargs):

From 3aa697f993e3719cf032c5b1e192a034100b0534 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 15 Aug 2015 15:58:56 +0600
Subject: [PATCH 2264/6219] [esri:video] Extract all formats and simplify

---
 youtube_dl/extractor/videoesri.py | 106 +++++++++++++-----------------
 1 file changed, 45 insertions(+), 61 deletions(-)

diff --git a/youtube_dl/extractor/videoesri.py b/youtube_dl/extractor/videoesri.py
index 0f84323a4..84faba678 100644
--- a/youtube_dl/extractor/videoesri.py
+++ b/youtube_dl/extractor/videoesri.py
@@ -1,90 +1,74 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import os
 import re
 
 from .common import InfoExtractor
-
+from ..compat import compat_urlparse
 from ..utils import (
-    unified_strdate
+    int_or_none,
+    parse_filesize,
+    unified_strdate,
 )
 
 
 class VideoEsriIE(InfoExtractor):
     _VALID_URL = r'https?://video\.esri\.com/watch/(?P<id>[0-9]+)'
     _TEST = {
-        'url': 'https://video.esri.com/watch/4228',
-        'md5': '170b4d513c2466ed483c150a48384133',
+        'url': 'https://video.esri.com/watch/1124/arcgis-online-_dash_-developing-applications',
+        'md5': 'd4aaf1408b221f1b38227a9bbaeb95bc',
         'info_dict': {
-            'id': '4228',
+            'id': '1124',
             'ext': 'mp4',
-            'title': 'AppStudio for ArcGIS',
+            'title': 'ArcGIS Online - Developing Applications',
+            'description': 'Jeremy Bartley demonstrates how to develop applications with ArcGIS Online.',
             'thumbnail': 're:^https?://.*\.jpg$',
-            'upload_date': '20150310',
+            'duration': 185,
+            'upload_date': '20120419',
         }
     }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
+
         webpage = self._download_webpage(url, video_id)
 
-        title = self._html_search_regex(r'<h1>(.*?)</h1>', webpage, 'title')
-
-        upload_date_raw = self._search_regex(
-            r'http-equiv="last-modified" content="(.*)"',
-            webpage, 'upload date')
-        upload_date = unified_strdate(upload_date_raw)
-
-        settings_info = self._search_regex(
-            r'evPlayerSettings = {(.*?);\s*$',
-            webpage, 'settings info', flags=re.MULTILINE | re.DOTALL)
-
-        # thumbnail includes '_x' for large, also has {_m,_t,_s} or
-        # without size suffix returns full image
-        thumbnail_path = re.findall(
-            r'image\': \'(\/thumbs.*)\'',
-            settings_info)[0]
-
-        if thumbnail_path:
-            thumbnail = '/'.join(['http://video.esri.com', thumbnail_path])
-
-        # note that this misses the (exceedly rare) webm files
-        video_paths = re.findall(r'mp4:(.*)\'', settings_info)
-
-        # find possible http servers of the mp4 files (also has rtsp)
-        base_url = re.findall(
-            r'netstreambasepath\':\s\'(h.*)\'', settings_info)[0]
-
-        # these are the numbers used internally, but really map
-        # to other resolutions, e.g. 960 is 720p.
-        heights = [480, 720, 960]
-        videos_by_res = {}
-        for video_path in video_paths:
-            url = "{base_url}{video_path}".format(
-                base_url=base_url,
-                video_path=video_path)
-            filename, ext = os.path.splitext(video_path)
-            height_label = int(filename.split('_')[1])
-            videos_by_res[height_label] = {
-                'url': url,
-                'ext': ext[1:],
-                'protocol': 'http',  # http-only supported currently
-            }
-
         formats = []
-        for height in heights:
-            if height in videos_by_res:
-                formats.append(videos_by_res[height])
+        for width, height, content in re.findall(
+                r'(?s)<li><strong>(\d+)x(\d+):</strong>(.+?)</li>', webpage):
+            for video_url, ext, filesize in re.findall(
+                    r'<a[^>]+href="([^"]+)">([^<]+)&nbsp;\(([^<]+)\)</a>', content):
+                formats.append({
+                    'url': compat_urlparse.urljoin(url, video_url),
+                    'ext': ext.lower(),
+                    'format_id': '%s-%s' % (ext.lower(), height),
+                    'width': int(width),
+                    'height': int(height),
+                    'filesize_approx': parse_filesize(filesize),
+                })
+        self._sort_formats(formats)
 
-        result = {
+        title = self._html_search_meta('title', webpage, 'title')
+        description = self._html_search_meta(
+            'description', webpage, 'description', fatal=False)
+
+        thumbnail = self._html_search_meta('thumbnail', webpage, 'thumbnail', fatal=False)
+        if thumbnail:
+            thumbnail = re.sub(r'_[st]\.jpg$', '_x.jpg', thumbnail)
+
+        duration = int_or_none(self._search_regex(
+            [r'var\s+videoSeconds\s*=\s*(\d+)', r"'duration'\s*:\s*(\d+)"],
+            webpage, 'duration', fatal=False))
+
+        upload_date = unified_strdate(self._html_search_meta(
+            'last-modified', webpage, 'upload date', fatal=None))
+
+        return {
             'id': video_id,
             'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'duration': duration,
             'upload_date': upload_date,
-            'formats': formats,
+            'formats': formats
         }
-
-        if thumbnail:
-            result['thumbnail'] = thumbnail
-
-        return result

From 9c21f229236c77a8865c857b43c6cbd95dcc6f23 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 15 Aug 2015 15:59:35 +0600
Subject: [PATCH 2265/6219] [esri:video] Rename extractor

---
 youtube_dl/extractor/__init__.py               | 2 +-
 youtube_dl/extractor/{videoesri.py => esri.py} | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)
 rename youtube_dl/extractor/{videoesri.py => esri.py} (98%)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index a4387636f..760f73918 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -695,7 +695,7 @@ from .vice import ViceIE
 from .viddler import ViddlerIE
 from .videobam import VideoBamIE
 from .videodetective import VideoDetectiveIE
-from .videoesri import VideoEsriIE
+from .esri import EsriVideoIE
 from .videolecturesnet import VideoLecturesNetIE
 from .videofyme import VideofyMeIE
 from .videomega import VideoMegaIE
diff --git a/youtube_dl/extractor/videoesri.py b/youtube_dl/extractor/esri.py
similarity index 98%
rename from youtube_dl/extractor/videoesri.py
rename to youtube_dl/extractor/esri.py
index 84faba678..bf5d2019f 100644
--- a/youtube_dl/extractor/videoesri.py
+++ b/youtube_dl/extractor/esri.py
@@ -12,7 +12,7 @@ from ..utils import (
 )
 
 
-class VideoEsriIE(InfoExtractor):
+class EsriVideoIE(InfoExtractor):
     _VALID_URL = r'https?://video\.esri\.com/watch/(?P<id>[0-9]+)'
     _TEST = {
         'url': 'https://video.esri.com/watch/1124/arcgis-online-_dash_-developing-applications',

From 1d25e9d173931da0d2cb65b114f44bbf24184f6e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 15 Aug 2015 16:00:24 +0600
Subject: [PATCH 2266/6219] [extractor/__init__] Fix order

---
 youtube_dl/extractor/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 760f73918..a8be63624 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -158,6 +158,7 @@ from .eporner import EpornerIE
 from .eroprofile import EroProfileIE
 from .escapist import EscapistIE
 from .espn import ESPNIE
+from .esri import EsriVideoIE
 from .everyonesmixtape import EveryonesMixtapeIE
 from .exfm import ExfmIE
 from .expotv import ExpoTVIE
@@ -695,7 +696,6 @@ from .vice import ViceIE
 from .viddler import ViddlerIE
 from .videobam import VideoBamIE
 from .videodetective import VideoDetectiveIE
-from .esri import EsriVideoIE
 from .videolecturesnet import VideoLecturesNetIE
 from .videofyme import VideofyMeIE
 from .videomega import VideoMegaIE

From fab83e24567226fa70e7f5076d961b83239ccfbf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 15 Aug 2015 16:10:20 +0600
Subject: [PATCH 2267/6219] Credit @scw for video.esri.com (#5459)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index ded9e87d2..d1693224e 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -139,3 +139,4 @@ slangangular
 Behrouz Abbasi
 ngld
 nyuszika7h
+Shaun Walbridge

From c576ef1e7cfd31ca94ca6025c054b3ae4f611b21 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 15 Aug 2015 18:13:37 +0600
Subject: [PATCH 2268/6219] [shahid] Improve and simplify

---
 youtube_dl/extractor/shahid.py | 140 ++++++++++++++++++---------------
 1 file changed, 78 insertions(+), 62 deletions(-)

diff --git a/youtube_dl/extractor/shahid.py b/youtube_dl/extractor/shahid.py
index 399140189..6e9903d5e 100644
--- a/youtube_dl/extractor/shahid.py
+++ b/youtube_dl/extractor/shahid.py
@@ -2,90 +2,106 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..compat import compat_urllib_parse
 from ..utils import (
-    js_to_json,
     ExtractorError,
-    int_or_none
+    int_or_none,
+    parse_iso8601,
 )
 
 
 class ShahidIE(InfoExtractor):
     _VALID_URL = r'https?://shahid\.mbc\.net/ar/episode/(?P<id>\d+)/?'
-    _TESTS = [
-        {
-            'url': 'https://shahid.mbc.net/ar/episode/90574/%D8%A7%D9%84%D9%85%D9%84%D9%83-%D8%B9%D8%A8%D8%AF%D8%A7%D9%84%D9%84%D9%87-%D8%A7%D9%84%D8%A5%D9%86%D8%B3%D8%A7%D9%86-%D8%A7%D9%84%D9%85%D9%88%D8%B3%D9%85-1-%D9%83%D9%84%D9%8A%D8%A8-3.html',
-            'info_dict': {
-                'id': '90574',
-                'ext': 'm3u8',
-                'title': 'الملك عبدالله الإنسان الموسم 1 كليب 3',
-                'description': 'الفيلم الوثائقي - الملك عبد الله الإنسان',
-                'duration': 2972,
-            },
-            'params': {
-                # m3u8 download
-                'skip_download': True,
-            }
+    _TESTS = [{
+        'url': 'https://shahid.mbc.net/ar/episode/90574/%D8%A7%D9%84%D9%85%D9%84%D9%83-%D8%B9%D8%A8%D8%AF%D8%A7%D9%84%D9%84%D9%87-%D8%A7%D9%84%D8%A5%D9%86%D8%B3%D8%A7%D9%86-%D8%A7%D9%84%D9%85%D9%88%D8%B3%D9%85-1-%D9%83%D9%84%D9%8A%D8%A8-3.html',
+        'info_dict': {
+            'id': '90574',
+            'ext': 'm3u8',
+            'title': 'الملك عبدالله الإنسان الموسم 1 كليب 3',
+            'description': 'الفيلم الوثائقي - الملك عبد الله الإنسان',
+            'duration': 2972,
+            'timestamp': 1422057420,
+            'upload_date': '20150123',
         },
-        {
-            # shahid plus subscriber only
-            'url': 'https://shahid.mbc.net/ar/episode/90511/%D9%85%D8%B1%D8%A7%D9%8A%D8%A7-2011-%D8%A7%D9%84%D9%85%D9%88%D8%B3%D9%85-1-%D8%A7%D9%84%D8%AD%D9%84%D9%82%D8%A9-1.html',
-            'only_matching': True
+        'params': {
+            # m3u8 download
+            'skip_download': True,
         }
-    ]
+    }, {
+        # shahid plus subscriber only
+        'url': 'https://shahid.mbc.net/ar/episode/90511/%D9%85%D8%B1%D8%A7%D9%8A%D8%A7-2011-%D8%A7%D9%84%D9%85%D9%88%D8%B3%D9%85-1-%D8%A7%D9%84%D8%AD%D9%84%D9%82%D8%A9-1.html',
+        'only_matching': True
+    }]
 
-    _api_vars = {
-        'type': 'player',
-        'url': 'http://api.shahid.net/api/v1_1',
-        'playerType': 'episode',
-    }
+    def _handle_error(self, response):
+        if not isinstance(response, dict):
+            return
+        error = response.get('error')
+        if error:
+            raise ExtractorError(
+                '%s returned error: %s' % (self.IE_NAME, '\n'.join(error.values())),
+                expected=True)
+
+    def _download_json(self, url, video_id, note='Downloading JSON metadata'):
+        response = super(ShahidIE, self)._download_json(url, video_id, note)['data']
+        self._handle_error(response)
+        return response
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
+
         webpage = self._download_webpage(url, video_id)
 
-        player_info = ''
-        flash_vars = self._search_regex('var flashvars = ({[^}]+})', webpage, 'flashvars', None)
-        if flash_vars is not None:
-            for line in flash_vars.splitlines():
-                if '+' not in line and '(' not in line and ')' not in line:
-                    player_info += line
-            player_info = self._parse_json(player_info, video_id, js_to_json, False)
-            if player_info is not None:
-                for key in self._api_vars:
-                    if key in player_info:
-                        self._api_vars[key] = player_info[key]
+        api_vars = {
+            'id': video_id,
+            'type': 'player',
+            'url': 'http://api.shahid.net/api/v1_1',
+            'playerType': 'episode',
+        }
 
-        player_json_data = self._download_json(
-            'https://shahid.mbc.net/arContent/getPlayerContent-param-.id-' + video_id + '.type-' + self._api_vars['type'] + '.html',
-            video_id
-        )['data']
-        if 'url' in player_json_data:
-            m3u8_url = player_json_data['url']
-        else:
-            for error in player_json_data['error'].values():
-                raise ExtractorError(error)
-        formats = self._extract_m3u8_formats(m3u8_url, video_id)
+        flashvars = self._search_regex(
+            r'var\s+flashvars\s*=\s*({[^}]+})', webpage, 'flashvars', default=None)
+        if flashvars:
+            for key in api_vars.keys():
+                value = self._search_regex(
+                    r'\b%s\s*:\s*(?P<q>["\'])(?P<value>.+?)(?P=q)' % key,
+                    flashvars, 'type', default=None, group='value')
+                if value:
+                    api_vars[key] = value
 
-        video_info = self._download_json(
-            self._api_vars['url'] + '/' + self._api_vars['playerType'] + '/' + video_id + '?apiKey=sh%40hid0nlin3&hash=b2wMCTHpSmyxGqQjJFOycRmLSex%2BBpTK%2Fooxy6vHaqs%3D',
-            video_id
-        )['data']
-        if video_info.get('error'):
-            for error in video_info['error']:
-                raise ExtractorError(error)
-        video_info = video_info[self._api_vars['playerType']]
-        title = video_info['title']
-        thumbnail = video_info.get('thumbnailUrl')
-        categories = [category['name'] for category in video_info.get('genres')]
-        description = video_info.get('description')
-        duration = int_or_none(video_info.get('duration'))
+        player = self._download_json(
+            'https://shahid.mbc.net/arContent/getPlayerContent-param-.id-%s.type-%s.html'
+            % (video_id, api_vars['type']), video_id, 'Downloading player JSON')
+
+        formats = self._extract_m3u8_formats(player['url'], video_id, 'mp4')
+
+        video = self._download_json(
+            '%s/%s/%s?%s' % (
+                api_vars['url'], api_vars['playerType'], api_vars['id'],
+                compat_urllib_parse.urlencode({
+                    'apiKey': 'sh@hid0nlin3',
+                    'hash': 'b2wMCTHpSmyxGqQjJFOycRmLSex+BpTK/ooxy6vHaqs=',
+                }).encode('utf-8')),
+            video_id, 'Downloading video JSON')
+
+        video = video[api_vars['playerType']]
+
+        title = video['title']
+        description = video.get('description')
+        thumbnail = video.get('thumbnailUrl')
+        duration = int_or_none(video.get('duration'))
+        timestamp = parse_iso8601(video.get('referenceDate'))
+        categories = [
+            category['name']
+            for category in video.get('genres', []) if 'name' in category]
 
         return {
             'id': video_id,
             'title': title,
-            'thumbnail': thumbnail,
-            'categories': categories,
             'description': description,
+            'thumbnail': thumbnail,
             'duration': duration,
+            'timestamp': timestamp,
+            'categories': categories,
             'formats': formats,
         }

From 9303ce3e6969b5818982d6214a8d0ff4e3c95286 Mon Sep 17 00:00:00 2001
From: reddraggone9 <cljenkins9@gmail.com>
Date: Thu, 13 Aug 2015 22:11:11 -0500
Subject: [PATCH 2269/6219] [youtube] Fix two-factor authentication

---
 youtube_dl/extractor/youtube.py | 40 +++++++++++++++------------------
 1 file changed, 18 insertions(+), 22 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index facd837ad..bfa9a12a8 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -46,7 +46,7 @@ from ..utils import (
 class YoutubeBaseInfoExtractor(InfoExtractor):
     """Provide base functions for Youtube extractors"""
     _LOGIN_URL = 'https://accounts.google.com/ServiceLogin'
-    _TWOFACTOR_URL = 'https://accounts.google.com/SecondFactor'
+    _TWOFACTOR_URL = 'https://accounts.google.com/signin/challenge'
     _NETRC_MACHINE = 'youtube'
     # If True it will raise an error if no login info is provided
     _LOGIN_REQUIRED = False
@@ -128,7 +128,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
         # Two-Factor
         # TODO add SMS and phone call support - these require making a request and then prompting the user
 
-        if re.search(r'(?i)<form[^>]* id="gaia_secondfactorform"', login_results) is not None:
+        if re.search(r'(?i)<form[^>]* id="challenge"', login_results) is not None:
             tfa_code = self._get_tfa_info()
 
             if tfa_code is None:
@@ -136,31 +136,27 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
                 self._downloader.report_warning('(Note that only TOTP (Google Authenticator App) codes work at this time.)')
                 return False
 
-            # Unlike the first login form, secTok and timeStmp are both required for the TFA form
+            def find_value(element_id):
+                match = re.search(r'id="%s"\s+value="(.+?)">' % element_id, login_results, re.M | re.U)
+                if match is None:
+                    self._downloader.report_warning('Failed to get %s - did the page structure change?' % id)
+                return match.group(1)
 
-            match = re.search(r'id="secTok"\n\s+value=\'(.+)\'/>', login_results, re.M | re.U)
-            if match is None:
-                self._downloader.report_warning('Failed to get secTok - did the page structure change?')
-            secTok = match.group(1)
-            match = re.search(r'id="timeStmp"\n\s+value=\'(.+)\'/>', login_results, re.M | re.U)
-            if match is None:
-                self._downloader.report_warning('Failed to get timeStmp - did the page structure change?')
-            timeStmp = match.group(1)
+            challengeId = find_value('challengeId')
+            challengeType = find_value('challengeType')
+            gxf = find_value('gxf')
 
             tfa_form_strs = {
+                'challengeId': challengeId,
+                'challengeType': challengeType,  # This doesn't appear to change
                 'continue': 'https://www.youtube.com/signin?action_handle_signin=true&feature=sign_in_button&hl=en_US&nomobiletemp=1',
-                'smsToken': '',
-                'smsUserPin': tfa_code,
-                'smsVerifyPin': 'Verify',
-
-                'PersistentCookie': 'yes',
-                'checkConnection': '',
-                'checkedDomains': 'youtube',
-                'pstMsg': '1',
-                'secTok': secTok,
-                'timeStmp': timeStmp,
                 'service': 'youtube',
                 'hl': 'en_US',
+                'checkedDomains': 'youtube',
+                'pstMsg': '0',
+                'gxf': gxf,
+                'Pin': tfa_code,
+                'TrustDevice': 'on',
             }
             tfa_form = dict((k.encode('utf-8'), v.encode('utf-8')) for k, v in tfa_form_strs.items())
             tfa_data = compat_urllib_parse.urlencode(tfa_form).encode('ascii')
@@ -173,7 +169,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
             if tfa_results is False:
                 return False
 
-            if re.search(r'(?i)<form[^>]* id="gaia_secondfactorform"', tfa_results) is not None:
+            if re.search(r'(?i)<form[^>]* id="challenge"', tfa_results) is not None:
                 self._downloader.report_warning('Two-factor code expired. Please try again, or use a one-use backup code instead.')
                 return False
             if re.search(r'(?i)<form[^>]* id="gaia_loginform"', tfa_results) is not None:

From 201ea3ee8e392d6c82bb8137b80b4328db40a399 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 15 Aug 2015 21:52:22 +0600
Subject: [PATCH 2270/6219] [extractor/common] Improve _hidden_inputs

---
 youtube_dl/extractor/common.py | 22 ++++++++++++----------
 1 file changed, 12 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 16ae4b98f..e2ace827f 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -724,16 +724,18 @@ class InfoExtractor(object):
 
     @staticmethod
     def _hidden_inputs(html):
-        return dict([
-            (input.group('name'), input.group('value')) for input in re.finditer(
-                r'''(?x)
-                    <input\s+
-                        type=(?P<q_hidden>["\'])hidden(?P=q_hidden)\s+
-                        name=(?P<q_name>["\'])(?P<name>.+?)(?P=q_name)\s+
-                        (?:id=(?P<q_id>["\']).+?(?P=q_id)\s+)?
-                        value=(?P<q_value>["\'])(?P<value>.*?)(?P=q_value)
-                ''', html)
-        ])
+        hidden_inputs = {}
+        for input in re.findall(r'<input([^>]+)>', html):
+            if not re.search(r'type=(["\'])hidden\1', input):
+                continue
+            name = re.search(r'name=(["\'])(?P<value>.+?)\1', input)
+            if not name:
+                continue
+            value = re.search(r'value=(["\'])(?P<value>.*?)\1', input)
+            if not value:
+                continue
+            hidden_inputs[name.group('value')] = value.group('value')
+        return hidden_inputs
 
     def _form_hidden_inputs(self, form_id, html):
         form = self._search_regex(

From e64b756943440d602dc757f81787cad6aee8f412 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 15 Aug 2015 21:55:07 +0600
Subject: [PATCH 2271/6219] [extractor/common] Interactive TFA code input

---
 youtube_dl/extractor/common.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index e2ace827f..65835d257 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -15,6 +15,7 @@ import xml.etree.ElementTree
 from ..compat import (
     compat_cookiejar,
     compat_cookies,
+    compat_getpass,
     compat_HTTPError,
     compat_http_client,
     compat_urllib_error,
@@ -610,7 +611,7 @@ class InfoExtractor(object):
 
         return (username, password)
 
-    def _get_tfa_info(self):
+    def _get_tfa_info(self, note='two-factor verification code'):
         """
         Get the two-factor authentication info
         TODO - asking the user will be required for sms/phone verify
@@ -624,7 +625,7 @@ class InfoExtractor(object):
         if downloader_params.get('twofactor', None) is not None:
             return downloader_params['twofactor']
 
-        return None
+        return compat_getpass('Type %s and press [Return]: ' % note)
 
     # Helper functions for extracting OpenGraph info
     @staticmethod

From 041bc3adc55bbe81649a1c5d283302e5a120659e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 15 Aug 2015 22:03:43 +0600
Subject: [PATCH 2272/6219] [youtube] Simplify two-factor authentication

---
 youtube_dl/extractor/youtube.py | 35 +++++++++++----------------------
 1 file changed, 12 insertions(+), 23 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index bfa9a12a8..887c46d95 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -33,6 +33,7 @@ from ..utils import (
     int_or_none,
     orderedSet,
     parse_duration,
+    remove_start,
     smuggle_url,
     str_to_int,
     unescapeHTML,
@@ -129,35 +130,23 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
         # TODO add SMS and phone call support - these require making a request and then prompting the user
 
         if re.search(r'(?i)<form[^>]* id="challenge"', login_results) is not None:
-            tfa_code = self._get_tfa_info()
+            tfa_code = self._get_tfa_info('2-step verification code')
 
-            if tfa_code is None:
-                self._downloader.report_warning('Two-factor authentication required. Provide it with --twofactor <code>')
-                self._downloader.report_warning('(Note that only TOTP (Google Authenticator App) codes work at this time.)')
+            if not tfa_code:
+                self._downloader.report_warning(
+                    'Two-factor authentication required. Provide it either interactively or with --twofactor <code>'
+                    '(Note that only TOTP (Google Authenticator App) codes work at this time.)')
                 return False
 
-            def find_value(element_id):
-                match = re.search(r'id="%s"\s+value="(.+?)">' % element_id, login_results, re.M | re.U)
-                if match is None:
-                    self._downloader.report_warning('Failed to get %s - did the page structure change?' % id)
-                return match.group(1)
+            tfa_code = remove_start(tfa_code, 'G-')
 
-            challengeId = find_value('challengeId')
-            challengeType = find_value('challengeType')
-            gxf = find_value('gxf')
+            tfa_form_strs = self._form_hidden_inputs('challenge', login_results)
 
-            tfa_form_strs = {
-                'challengeId': challengeId,
-                'challengeType': challengeType,  # This doesn't appear to change
-                'continue': 'https://www.youtube.com/signin?action_handle_signin=true&feature=sign_in_button&hl=en_US&nomobiletemp=1',
-                'service': 'youtube',
-                'hl': 'en_US',
-                'checkedDomains': 'youtube',
-                'pstMsg': '0',
-                'gxf': gxf,
+            tfa_form_strs.update({
                 'Pin': tfa_code,
                 'TrustDevice': 'on',
-            }
+            })
+
             tfa_form = dict((k.encode('utf-8'), v.encode('utf-8')) for k, v in tfa_form_strs.items())
             tfa_data = compat_urllib_parse.urlencode(tfa_form).encode('ascii')
 
@@ -170,7 +159,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
                 return False
 
             if re.search(r'(?i)<form[^>]* id="challenge"', tfa_results) is not None:
-                self._downloader.report_warning('Two-factor code expired. Please try again, or use a one-use backup code instead.')
+                self._downloader.report_warning('Two-factor code expired or invalid. Please try again, or use a one-use backup code instead.')
                 return False
             if re.search(r'(?i)<form[^>]* id="gaia_loginform"', tfa_results) is not None:
                 self._downloader.report_warning('unable to log in - did the page structure change?')

From eaa5646483d22d3b658dcf63b61e6c3b67aa5bc5 Mon Sep 17 00:00:00 2001
From: "Sergey M." <dstftw@gmail.com>
Date: Sat, 15 Aug 2015 22:57:06 +0600
Subject: [PATCH 2273/6219] [README.md] Clarify configuration file usage
 (Closes #6530)

---
 README.md | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 542a7c26a..25844eb6d 100644
--- a/README.md
+++ b/README.md
@@ -236,7 +236,14 @@ which means you can modify it, redistribute it or use it however you like.
 
 # CONFIGURATION
 
-You can configure youtube-dl by placing default arguments (such as `--extract-audio --no-mtime` to always extract the audio and not copy the mtime) into `/etc/youtube-dl.conf` and/or `~/.config/youtube-dl/config`. On Windows, the configuration file locations are `%APPDATA%\youtube-dl\config.txt` and `C:\Users\<user name>\youtube-dl.conf`.
+You can configure youtube-dl by placing any supported command line option to a configuration file. On Linux, system wide configuration file is located at `/etc/youtube-dl.conf` and user wide configuration file at `~/.config/youtube-dl/config`. On Windows, the user wide configuration file locations are `%APPDATA%\youtube-dl\config.txt` or `C:\Users\<user name>\youtube-dl.conf`. For example, with the following configration file youtube-dl will always extract the audio, not copy the mtime and use proxy:
+```
+--extract-audio
+--no-mtime
+--proxy 127.0.0.1:3128
+```
+
+You can use `--ignore-config` if you want to disable configuration file for a particular youtube-dl run.
 
 ### Authentication with `.netrc` file ###
 

From 221a59fe6fe22ec286830319e72cbabdc83fd02f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ond=C5=99ej=20Caletka?= <ondrej@caletka.cz>
Date: Sun, 9 Aug 2015 12:27:31 +0200
Subject: [PATCH 2274/6219] [playtvak] Initial support for videos from Mafra
 group servers
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Support for videos in articles from idnes.cz, lidovky.cz, metro.cz, as
well as video and live streams from playtvak.cz

Signed-off-by: Ondřej Caletka <ondrej@caletka.cz>
---
 youtube_dl/extractor/__init__.py |   1 +
 youtube_dl/extractor/playtvak.py | 127 +++++++++++++++++++++++++++++++
 2 files changed, 128 insertions(+)
 create mode 100644 youtube_dl/extractor/playtvak.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 83d21bd15..5307240f8 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -448,6 +448,7 @@ from .planetaplay import PlanetaPlayIE
 from .pladform import PladformIE
 from .played import PlayedIE
 from .playfm import PlayFMIE
+from .playtvak import PlaytvakIE
 from .playvid import PlayvidIE
 from .playwire import PlaywireIE
 from .podomatic import PodomaticIE
diff --git a/youtube_dl/extractor/playtvak.py b/youtube_dl/extractor/playtvak.py
new file mode 100644
index 000000000..6dff6650c
--- /dev/null
+++ b/youtube_dl/extractor/playtvak.py
@@ -0,0 +1,127 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..compat import (
+    compat_urlparse,
+    compat_urllib_parse,
+)
+from ..utils import ExtractorError
+
+
+def _extract_json(code):
+    return re.sub(
+        r'(?s)^VideoPlayer.data\("", ({.*})\);?\s*?(?://[^\n]*)*$', r'\1', code)
+
+
+class PlaytvakIE(InfoExtractor):
+    _VALID_URL = r'https?://.*?(playtvak|idnes|lidovky|metro)\.cz/.*\?c=(?P<id>[A-Z][0-9]{6}_[0-9]{6}_.*)'
+    _TESTS = [{
+        'url': 'http://www.playtvak.cz/vyzente-vosy-a-srsne-ze-zahrady-dn5-/hodinovy-manzel.aspx?c=A150730_150323_hodinovy-manzel_kuko',
+        'md5': '4525ae312c324b4be2f4603cc78ceb4a',
+        'info_dict': {
+            'id': 'A150730_150323_hodinovy-manzel_kuko',
+            'ext': 'mp4',
+            'title': 'Vyžeňte vosy a sršně ze zahrady',
+            'thumbnail': 'http://oidnes.cz/15/074/mobil/KUK5cea00_010hodmanel58154.jpg',
+            'description': 'Málo co kazí atmosféru venkovního posezení tak jako neustálé bzučení kolem hlavy.  Vyzkoušejte náš lapač a odpuzovač vos a sršňů.',
+        }
+    }, {  # live video test
+        'url': 'http://slowtv.playtvak.cz/planespotting-0pr-/planespotting.aspx?c=A150624_164934_planespotting_cat',
+        'info_dict': {
+            'id': 'A150624_164934_planespotting_cat',
+            'ext': 'flv',
+            'title': 're:^Přímý přenos iDNES.cz [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
+            'thumbnail': 'http://data.idnes.cz/soubory/servisni-play-porady/89A150630_ACEK_026_VIDEOPLAYER-STREA.PNG',
+            'description': 'Sledujte provoz na ranveji Letiště Václava Havla v Praze',
+            'is_live': True,
+        },
+        'params': {
+            'skip_download': True,  # requires rtmpdump
+        },
+    }, {  # idnes.cz
+        'url': 'http://zpravy.idnes.cz/pes-zavreny-v-aute-rozbijeni-okynek-v-aute-fj5-/domaci.aspx?c=A150809_104116_domaci_pku',
+        'md5': '819832ba33cd7016e58a6658577fe289',
+        'info_dict': {
+            'id': 'A150809_104116_domaci_pku',
+            'ext': 'mp4',
+            'title': 'Zavřeli jsme mraženou pizzu do auta. Upekla se',
+            'thumbnail': 'http://i.idnes.cz/15/081/vidw/SHA5d1786_pizzaauto.jpg',
+            'description': 'Na sociálních sítích se objevila výzva, aby lidé, kteří v horkých letních dnech uvidí v zaparkovaném autě zavřeného psa, neváhali rozbít okénko. Zastánci tohoto postoje argumentují zdravím zvířete, které v dusnu může zkolabovat. Policie doporučuje nejprve volat tísňovou linku.',
+        }
+    }, {  # lidovky.cz
+        'url': 'http://www.lidovky.cz/dalsi-demonstrace-v-praze-o-migraci-duq-/video.aspx?c=A150808_214044_ln-video_ELE',
+        'md5': 'c7209ac4ba9d234d4ad5bab7485bcee8',
+        'info_dict': {
+            'id': 'A150808_214044_ln-video_ELE',
+            'ext': 'mp4',
+            'title': 'Táhni! Demonstrace proti imigrantům budila emoce',
+            'thumbnail': 'http://i.idnes.cz/15/081/vidw/PID5d1d52_vandas3.jpg',
+            'description': 'Desítky lidí se sešly v Praze na protest proti imigrantům. Současně probíhala i demonstrace na jejich podporu. Na Staroměstském náměstí vystoupil i předseda dělnické strany Tomáš Vandas a kontroverzní slovenský politik Marian Kotleba. Dalšího slovenského nacionalistu Mariána Magáta odvedla policie.',
+        }
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        infourl = self._html_search_regex(r'Misc.videoFLV\({ data: "([^"]+)"', webpage, 'xmlinfourl')
+        parsedurl = compat_urlparse.urlparse(infourl)
+        qs = compat_urlparse.parse_qs(parsedurl.query)
+        if 'reklama' in qs:  # Don't ask for ads
+            qs['reklama'] = ['0']
+        qs['type'] = ['js']  # Ask for JS-based info file
+        newquery = compat_urllib_parse.urlencode(qs, True)
+        infourl = compat_urlparse.urlunparse(parsedurl[:4] + (newquery, ''))
+        jsoninfo = self._download_json(infourl, video_id, transform_source=_extract_json)
+
+        item = None
+        for i in jsoninfo['items']:
+            if i['type'] == 'video' or i['type'] == 'stream':
+                item = i
+                break
+        if item is None:
+            raise ExtractorError('No suitable stream found')
+        title = item['title']
+        thumbnail = item['image']
+        is_live = item['type'] == 'stream'
+        if is_live:
+            title = self._live_title(title)
+
+        formats = []
+        for fmt in item['video']:
+            format_entry = {'url': fmt['file'],
+                            'format_id': ("%s_%s" % (fmt['format'], fmt['quality'])),
+                            }
+            if fmt['quality'] == 'middle':
+                format_entry['quality'] = -2
+            elif fmt['quality'] == 'low':
+                format_entry['quality'] = -3
+
+            if fmt['format'] == 'mp4':
+                format_entry['ext'] = 'mp4'
+            elif fmt['format'] == 'webm':
+                format_entry['ext'] = 'webm'
+            elif fmt['format'] == 'apple':
+                format_entry['ext'] = 'mp4'
+                format_entry['protocol'] = 'm3u8'
+                # Some streams have mp3 audio which does not play
+                # well with ffmpeg filter aac_adtstoasc
+                format_entry['preference'] = -1
+            elif fmt['format'] == 'rtmp':
+                format_entry['ext'] = 'flv'
+            else:  # Other formats not supported yet
+                continue
+
+            formats.append(format_entry)
+
+        self._sort_formats(formats)
+        return {
+            'id': video_id,
+            'title': title,
+            'thumbnail': thumbnail,
+            'description': self._og_search_description(webpage),
+            'is_live': is_live,
+            'formats': formats,
+        }

From 276c9897720fe087924aef4ac80cf528e621b832 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 16 Aug 2015 03:07:05 +0600
Subject: [PATCH 2275/6219] [playtvak] Improve and simplify

---
 youtube_dl/extractor/playtvak.py | 145 ++++++++++++++++++++-----------
 1 file changed, 92 insertions(+), 53 deletions(-)

diff --git a/youtube_dl/extractor/playtvak.py b/youtube_dl/extractor/playtvak.py
index 6dff6650c..4e5034dc6 100644
--- a/youtube_dl/extractor/playtvak.py
+++ b/youtube_dl/extractor/playtvak.py
@@ -1,23 +1,22 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..compat import (
     compat_urlparse,
     compat_urllib_parse,
 )
-from ..utils import ExtractorError
-
-
-def _extract_json(code):
-    return re.sub(
-        r'(?s)^VideoPlayer.data\("", ({.*})\);?\s*?(?://[^\n]*)*$', r'\1', code)
+from ..utils import (
+    ExtractorError,
+    int_or_none,
+    parse_iso8601,
+    qualities,
+)
 
 
 class PlaytvakIE(InfoExtractor):
-    _VALID_URL = r'https?://.*?(playtvak|idnes|lidovky|metro)\.cz/.*\?c=(?P<id>[A-Z][0-9]{6}_[0-9]{6}_.*)'
+    IE_DESC = 'Playtvak.cz, iDNES.cz and Lidovky.cz'
+    _VALID_URL = r'https?://(?:.+?\.)?(?:playtvak|idnes|lidovky|metro)\.cz/.*\?(?:c|idvideo)=(?P<id>[^&]+)'
     _TESTS = [{
         'url': 'http://www.playtvak.cz/vyzente-vosy-a-srsne-ze-zahrady-dn5-/hodinovy-manzel.aspx?c=A150730_150323_hodinovy-manzel_kuko',
         'md5': '4525ae312c324b4be2f4603cc78ceb4a',
@@ -25,8 +24,12 @@ class PlaytvakIE(InfoExtractor):
             'id': 'A150730_150323_hodinovy-manzel_kuko',
             'ext': 'mp4',
             'title': 'Vyžeňte vosy a sršně ze zahrady',
-            'thumbnail': 'http://oidnes.cz/15/074/mobil/KUK5cea00_010hodmanel58154.jpg',
-            'description': 'Málo co kazí atmosféru venkovního posezení tak jako neustálé bzučení kolem hlavy.  Vyzkoušejte náš lapač a odpuzovač vos a sršňů.',
+            'description': 'md5:f93d398691044d303bc4a3de62f3e976',
+            'thumbnail': 're:(?i)^https?://.*\.(?:jpg|png)$',
+            'duration': 279,
+            'timestamp': 1438732860,
+            'upload_date': '20150805',
+            'is_live': False,
         }
     }, {  # live video test
         'url': 'http://slowtv.playtvak.cz/planespotting-0pr-/planespotting.aspx?c=A150624_164934_planespotting_cat',
@@ -34,8 +37,8 @@ class PlaytvakIE(InfoExtractor):
             'id': 'A150624_164934_planespotting_cat',
             'ext': 'flv',
             'title': 're:^Přímý přenos iDNES.cz [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
-            'thumbnail': 'http://data.idnes.cz/soubory/servisni-play-porady/89A150630_ACEK_026_VIDEOPLAYER-STREA.PNG',
             'description': 'Sledujte provoz na ranveji Letiště Václava Havla v Praze',
+            'thumbnail': 're:(?i)^https?://.*\.(?:jpg|png)$',
             'is_live': True,
         },
         'params': {
@@ -48,8 +51,12 @@ class PlaytvakIE(InfoExtractor):
             'id': 'A150809_104116_domaci_pku',
             'ext': 'mp4',
             'title': 'Zavřeli jsme mraženou pizzu do auta. Upekla se',
-            'thumbnail': 'http://i.idnes.cz/15/081/vidw/SHA5d1786_pizzaauto.jpg',
-            'description': 'Na sociálních sítích se objevila výzva, aby lidé, kteří v horkých letních dnech uvidí v zaparkovaném autě zavřeného psa, neváhali rozbít okénko. Zastánci tohoto postoje argumentují zdravím zvířete, které v dusnu může zkolabovat. Policie doporučuje nejprve volat tísňovou linku.',
+            'description': 'md5:01e73f02329e2e5760bd5eed4d42e3c2',
+            'thumbnail': 're:(?i)^https?://.*\.(?:jpg|png)$',
+            'duration': 39,
+            'timestamp': 1438969140,
+            'upload_date': '20150807',
+            'is_live': False,
         }
     }, {  # lidovky.cz
         'url': 'http://www.lidovky.cz/dalsi-demonstrace-v-praze-o-migraci-duq-/video.aspx?c=A150808_214044_ln-video_ELE',
@@ -58,70 +65,102 @@ class PlaytvakIE(InfoExtractor):
             'id': 'A150808_214044_ln-video_ELE',
             'ext': 'mp4',
             'title': 'Táhni! Demonstrace proti imigrantům budila emoce',
-            'thumbnail': 'http://i.idnes.cz/15/081/vidw/PID5d1d52_vandas3.jpg',
-            'description': 'Desítky lidí se sešly v Praze na protest proti imigrantům. Současně probíhala i demonstrace na jejich podporu. Na Staroměstském náměstí vystoupil i předseda dělnické strany Tomáš Vandas a kontroverzní slovenský politik Marian Kotleba. Dalšího slovenského nacionalistu Mariána Magáta odvedla policie.',
+            'description': 'md5:97c81d589a9491fbfa323c9fa3cca72c',
+            'thumbnail': 're:(?i)^https?://.*\.(?:jpg|png)$',
+            'timestamp': 1439052180,
+            'upload_date': '20150808',
+            'is_live': False,
         }
+    }, {
+        'url': 'http://www.playtvak.cz/embed.aspx?idvideo=V150729_141549_play-porad_kuko',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
+
         webpage = self._download_webpage(url, video_id)
-        infourl = self._html_search_regex(r'Misc.videoFLV\({ data: "([^"]+)"', webpage, 'xmlinfourl')
-        parsedurl = compat_urlparse.urlparse(infourl)
-        qs = compat_urlparse.parse_qs(parsedurl.query)
-        if 'reklama' in qs:  # Don't ask for ads
-            qs['reklama'] = ['0']
-        qs['type'] = ['js']  # Ask for JS-based info file
-        newquery = compat_urllib_parse.urlencode(qs, True)
-        infourl = compat_urlparse.urlunparse(parsedurl[:4] + (newquery, ''))
-        jsoninfo = self._download_json(infourl, video_id, transform_source=_extract_json)
+
+        info_url = self._html_search_regex(
+            r'Misc\.videoFLV\(\s*{\s*data\s*:\s*"([^"]+)"', webpage, 'info url')
+
+        parsed_url = compat_urlparse.urlparse(info_url)
+
+        qs = compat_urlparse.parse_qs(parsed_url.query)
+        qs.update({
+            'reklama': ['0'],
+            'type': ['js'],
+        })
+
+        info_url = compat_urlparse.urlunparse(
+            parsed_url._replace(query = compat_urllib_parse.urlencode(qs, True)))
+
+        json_info = self._download_json(
+            info_url, video_id,
+            transform_source=lambda s: s[s.index('{'):s.rindex('}') + 1])
 
         item = None
-        for i in jsoninfo['items']:
-            if i['type'] == 'video' or i['type'] == 'stream':
+        for i in json_info['items']:
+            if i.get('type') == 'video' or i.get('type') == 'stream':
                 item = i
                 break
-        if item is None:
+        if not item:
             raise ExtractorError('No suitable stream found')
-        title = item['title']
-        thumbnail = item['image']
-        is_live = item['type'] == 'stream'
-        if is_live:
-            title = self._live_title(title)
+
+        quality = qualities(['low', 'middle', 'high'])
 
         formats = []
         for fmt in item['video']:
-            format_entry = {'url': fmt['file'],
-                            'format_id': ("%s_%s" % (fmt['format'], fmt['quality'])),
-                            }
-            if fmt['quality'] == 'middle':
-                format_entry['quality'] = -2
-            elif fmt['quality'] == 'low':
-                format_entry['quality'] = -3
+            video_url = fmt.get('file')
+            if not video_url:
+                continue
 
-            if fmt['format'] == 'mp4':
-                format_entry['ext'] = 'mp4'
-            elif fmt['format'] == 'webm':
-                format_entry['ext'] = 'webm'
-            elif fmt['format'] == 'apple':
-                format_entry['ext'] = 'mp4'
-                format_entry['protocol'] = 'm3u8'
+            format_ = fmt['format']
+            format_id = '%s_%s' % (format_, fmt['quality'])
+            preference = None
+
+            if format_ in ['mp4', 'webm']:
+                ext = format_
+            elif format_ == 'rtmp':
+                ext = 'flv'
+            elif format_ == 'apple':
+                ext = 'mp4'
                 # Some streams have mp3 audio which does not play
                 # well with ffmpeg filter aac_adtstoasc
-                format_entry['preference'] = -1
-            elif fmt['format'] == 'rtmp':
-                format_entry['ext'] = 'flv'
+                preference = -1
+            elif format_ == 'adobe':  # f4m manifest fails with 404 in 80% of requests
+                continue
             else:  # Other formats not supported yet
                 continue
 
-            formats.append(format_entry)
-
+            formats.append({
+                'url': video_url,
+                'ext': ext,
+                'format_id': format_id,
+                'quality': quality(fmt.get('quality')),
+                'preference': preference,
+            })
         self._sort_formats(formats)
+
+        title = item['title']
+        is_live = item['type'] == 'stream'
+        if is_live:
+            title = self._live_title(title)
+        timestamp = None
+        duration = None
+        if not is_live:
+            duration = int_or_none(item.get('length'))
+            timestamp = item.get('published')
+            if timestamp:
+                timestamp = parse_iso8601(timestamp[:-5])
+
         return {
             'id': video_id,
             'title': title,
-            'thumbnail': thumbnail,
             'description': self._og_search_description(webpage),
+            'thumbnail': item.get('image'),
+            'duration': duration,
+            'timestamp': timestamp,
             'is_live': is_live,
             'formats': formats,
         }

From 6900b4f6f52ba783bb7a6028fd174250c8832a38 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 16 Aug 2015 01:05:04 +0200
Subject: [PATCH 2276/6219] release 2015.08.16

---
 CONTRIBUTING.md        | 2 +-
 docs/supportedsites.md | 6 ++++++
 youtube_dl/version.py  | 2 +-
 3 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 588b15bde..42333c450 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -125,7 +125,7 @@ If you want to add support for a new site, you can follow this quick list (assum
     ```
 5. Add an import in [`youtube_dl/extractor/__init__.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/__init__.py).
 6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will be then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc.
-7. Have a look at [`youtube_dl/common/extractor/common.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should return](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L38). Add tests and code for as many as you want.
+7. Have a look at [`youtube_dl/common/extractor/common.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L62-L200). Add tests and code for as many as you want.
 8. If you can, check the code with [flake8](https://pypi.python.org/pypi/flake8).
 9. When the tests pass, [add](http://git-scm.com/docs/git-add) the new files and [commit](http://git-scm.com/docs/git-commit) them and [push](http://git-scm.com/docs/git-push) the result, like this:
 
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index e21471102..9099e2da4 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -150,6 +150,7 @@
  - **EroProfile**
  - **Escapist**
  - **ESPN** (Currently broken)
+ - **EsriVideo**
  - **EveryonesMixtape**
  - **exfm**: ex.fm
  - **ExpoTV**
@@ -220,6 +221,8 @@
  - **imdb:list**: Internet Movie Database lists
  - **Imgur**
  - **Ina**
+ - **Indavideo**
+ - **IndavideoEmbed**
  - **InfoQ**
  - **Instagram**
  - **instagram:user**: Instagram user profile
@@ -386,6 +389,7 @@
  - **PlanetaPlay**
  - **play.fm**
  - **played.to**
+ - **Playtvak**: Playtvak.cz, iDNES.cz and Lidovky.cz
  - **Playvid**
  - **Playwire**
  - **plus.google**: Google Plus
@@ -432,6 +436,7 @@
  - **rtve.es:alacarta**: RTVE a la carta
  - **rtve.es:infantil**: RTVE infantil
  - **rtve.es:live**: RTVE.es live streams
+ - **RTVNH**
  - **RUHD**
  - **rutube**: Rutube videos
  - **rutube:channel**: Rutube channels
@@ -455,6 +460,7 @@
  - **ServingSys**
  - **Sexu**
  - **SexyKarma**: Sexy Karma and Watch Indian Porn
+ - **Shahid**
  - **Shared**
  - **ShareSix**
  - **Sina**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 6462d4477..689d6fca7 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.08.09'
+__version__ = '2015.08.16'

From 974f1a385a452b1c86d6f3ff16035b30baaeeb64 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 16 Aug 2015 17:22:13 +0600
Subject: [PATCH 2277/6219] [playtvak] Improve description extraction and add
 test for metro

---
 youtube_dl/extractor/playtvak.py | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/playtvak.py b/youtube_dl/extractor/playtvak.py
index 4e5034dc6..2b338966f 100644
--- a/youtube_dl/extractor/playtvak.py
+++ b/youtube_dl/extractor/playtvak.py
@@ -71,6 +71,19 @@ class PlaytvakIE(InfoExtractor):
             'upload_date': '20150808',
             'is_live': False,
         }
+    }, {  # metro.cz
+        'url': 'http://www.metro.cz/video-pod-billboardem-se-na-vltavske-roztocil-kolotoc-deti-vozil-jen-par-hodin-1hx-/metro-extra.aspx?c=A141111_173251_metro-extra_row',
+        'md5': '84fc1deedcac37b7d4a6ccae7c716668',
+        'info_dict': {
+            'id': 'A141111_173251_metro-extra_row',
+            'ext': 'mp4',
+            'title': 'Recesisté udělali z billboardu kolotoč',
+            'description': 'md5:7369926049588c3989a66c9c1a043c4c',
+            'thumbnail': 're:(?i)^https?://.*\.(?:jpg|png)$',
+            'timestamp': 1415725500,
+            'upload_date': '20141111',
+            'is_live': False,
+        }
     }, {
         'url': 'http://www.playtvak.cz/embed.aspx?idvideo=V150729_141549_play-porad_kuko',
         'only_matching': True,
@@ -146,6 +159,8 @@ class PlaytvakIE(InfoExtractor):
         is_live = item['type'] == 'stream'
         if is_live:
             title = self._live_title(title)
+        description = self._og_search_description(webpage, default=None) or self._html_search_meta(
+            'description', webpage, 'description')
         timestamp = None
         duration = None
         if not is_live:
@@ -157,7 +172,7 @@ class PlaytvakIE(InfoExtractor):
         return {
             'id': video_id,
             'title': title,
-            'description': self._og_search_description(webpage),
+            'description': description,
             'thumbnail': item.get('image'),
             'duration': duration,
             'timestamp': timestamp,

From 7fc18d930917ad407c78bb9b0465dc4fae2fb335 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 16 Aug 2015 19:53:14 +0600
Subject: [PATCH 2278/6219] [screenwavemedia] Fix extraction (Closes #6575)

---
 youtube_dl/extractor/screenwavemedia.py | 23 ++++++++++++++---------
 1 file changed, 14 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/screenwavemedia.py b/youtube_dl/extractor/screenwavemedia.py
index 3bc84989e..78b068be2 100644
--- a/youtube_dl/extractor/screenwavemedia.py
+++ b/youtube_dl/extractor/screenwavemedia.py
@@ -1,6 +1,8 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..utils import (
     int_or_none,
@@ -35,15 +37,18 @@ class ScreenwaveMediaIE(InfoExtractor):
 
         sources = self._parse_json(
             js_to_json(
-                self._search_regex(
-                    r"sources\s*:\s*(\[[^\]]+?\])", playerconfig,
-                    'sources',
-                ).replace(
-                    "' + thisObj.options.videoserver + '",
-                    videoserver
-                ).replace(
-                    "' + playerVidId + '",
-                    video_id
+                re.sub(
+                    r'(?s)/\*.*?\*/', '',
+                    self._search_regex(
+                        r"sources\s*:\s*(\[[^\]]+?\])", playerconfig,
+                        'sources',
+                    ).replace(
+                        "' + thisObj.options.videoserver + '",
+                        videoserver
+                    ).replace(
+                        "' + playerVidId + '",
+                        video_id
+                    )
                 )
             ),
             video_id

From 008687427725b8d857c44d75f358059c2533539a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 16 Aug 2015 19:59:03 +0600
Subject: [PATCH 2279/6219] [playtvak] Use tuples

---
 youtube_dl/extractor/playtvak.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/playtvak.py b/youtube_dl/extractor/playtvak.py
index 2b338966f..278fdc1aa 100644
--- a/youtube_dl/extractor/playtvak.py
+++ b/youtube_dl/extractor/playtvak.py
@@ -120,7 +120,7 @@ class PlaytvakIE(InfoExtractor):
         if not item:
             raise ExtractorError('No suitable stream found')
 
-        quality = qualities(['low', 'middle', 'high'])
+        quality = qualities(('low', 'middle', 'high'))
 
         formats = []
         for fmt in item['video']:
@@ -132,7 +132,7 @@ class PlaytvakIE(InfoExtractor):
             format_id = '%s_%s' % (format_, fmt['quality'])
             preference = None
 
-            if format_ in ['mp4', 'webm']:
+            if format_ in ('mp4', 'webm'):
                 ext = format_
             elif format_ == 'rtmp':
                 ext = 'flv'

From 8626b23e4ea091c4093c25626ca9fc12293b2830 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 16 Aug 2015 20:18:15 +0600
Subject: [PATCH 2280/6219] [screenwavemedia] Make more robust

---
 youtube_dl/extractor/screenwavemedia.py | 21 ++++++++++++++++++++-
 1 file changed, 20 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/screenwavemedia.py b/youtube_dl/extractor/screenwavemedia.py
index 78b068be2..220d39078 100644
--- a/youtube_dl/extractor/screenwavemedia.py
+++ b/youtube_dl/extractor/screenwavemedia.py
@@ -51,19 +51,38 @@ class ScreenwaveMediaIE(InfoExtractor):
                     )
                 )
             ),
-            video_id
+            video_id, fatal=False
         )
 
+        # Fallback to hardcoded sources if JS changes again
+        if not sources:
+            sources = [{
+                'file': 'http://%s/vod/%s_%s.mp4' % (videoserver, video_id, format_id),
+                'type': 'mp4',
+                'label': format_label,
+            } for format_id, format_label in (
+                ('low', '144p Low'), ('med', '160p Med'), ('high', '360p High'), ('hd1', '720p HD1'))]
+            sources.append({
+                'file': 'http://%s/vod/smil:%s.smil/playlist.m3u8' % (videoserver, video_id),
+                'type': 'hls',
+            })
+
         formats = []
         for source in sources:
             if source['type'] == 'hls':
                 formats.extend(self._extract_m3u8_formats(source['file'], video_id))
             else:
+                file_ = source.get('file')
+                if not file_:
+                    continue
                 format_label = source.get('label')
+                format_id = self._search_regex(
+                    r'_(.+?)\.[^.]+$', file_, 'format id', default=None)
                 height = int_or_none(self._search_regex(
                     r'^(\d+)[pP]', format_label, 'height', default=None))
                 formats.append({
                     'url': source['file'],
+                    'format_id': format_id,
                     'format': format_label,
                     'ext': source.get('type'),
                     'height': height,

From f74a7348f6ac52259ea66b74a40165b448fbd702 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 16 Aug 2015 23:33:17 +0600
Subject: [PATCH 2281/6219] [youtube:search_url] Fix extraction (Closes #6578)

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 887c46d95..4d1ca9298 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1762,7 +1762,7 @@ class YoutubeSearchURLIE(InfoExtractor):
             r'(?s)<ol[^>]+class="item-section"(.*?)</ol>', webpage, 'result HTML')
 
         part_codes = re.findall(
-            r'(?s)<h3 class="yt-lockup-title">(.*?)</h3>', result_code)
+            r'(?s)<h3[^>]+class="[^"]*yt-lockup-title[^"]*"[^>]*>(.*?)</h3>', result_code)
         entries = []
         for part_code in part_codes:
             part_title = self._html_search_regex(

From cbaed4bb5e5e90103a1164d9326043a3abd0bf83 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 17 Aug 2015 02:04:13 +0600
Subject: [PATCH 2282/6219] [youtube] Expand _VALID_URL to support vid.plus

---
 youtube_dl/extractor/youtube.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 4d1ca9298..8e2da46e3 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -202,7 +202,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                                  v=
                              )
                          ))
-                         |youtu\.be/                                          # just youtu.be/xxxx
+                         |(?:
+                            youtu\.be|                                        # just youtu.be/xxxx
+                            vid\.plus                                         # or vid.plus/xxxx
+                         )/
                          |(?:www\.)?cleanvideosearch\.com/media/action/yt/watch\?videoId=
                          )
                      )?                                                       # all until now is optional -> you can pass the naked ID
@@ -624,6 +627,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             'params': {
                 'skip_download': True,
             },
+        },
+        {
+            'url': 'http://vid.plus/FlRa-iH7PGw',
+            'only_matching': True,
         }
     ]
 

From c00c7c0af0fdcb380aef0ea9e072a61979d17816 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 16 Aug 2015 23:39:50 +0200
Subject: [PATCH 2283/6219] [sportdeutschland] Fix extraction

---
 youtube_dl/extractor/sportdeutschland.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/sportdeutschland.py b/youtube_dl/extractor/sportdeutschland.py
index 1a57aebf1..7ec6c613f 100644
--- a/youtube_dl/extractor/sportdeutschland.py
+++ b/youtube_dl/extractor/sportdeutschland.py
@@ -38,10 +38,12 @@ class SportDeutschlandIE(InfoExtractor):
             'upload_date': '20140825',
             'description': 'md5:60a20536b57cee7d9a4ec005e8687504',
             'timestamp': 1408976060,
+            'duration': 2732,
             'title': 'Li-Ning Badminton Weltmeisterschaft 2014 Kopenhagen: Herren Einzel, Wei Lee vs. Keun Lee',
             'thumbnail': 're:^https?://.*\.jpg$',
             'view_count': int,
             'categories': ['Li-Ning Badminton WM 2014'],
+
         }
     }]
 
@@ -50,7 +52,7 @@ class SportDeutschlandIE(InfoExtractor):
         video_id = mobj.group('id')
         sport_id = mobj.group('sport')
 
-        api_url = 'http://splink.tv/api/permalinks/%s/%s' % (
+        api_url = 'http://proxy.vidibusdynamic.net/sportdeutschland.tv/api/permalinks/%s/%s?access_token=true' % (
             sport_id, video_id)
         req = compat_urllib_request.Request(api_url, headers={
             'Accept': 'application/vnd.vidibus.v2.html+json',
@@ -58,12 +60,11 @@ class SportDeutschlandIE(InfoExtractor):
         })
         data = self._download_json(req, video_id)
 
-        categories = list(data.get('section', {}).get('tags', {}).values())
         asset = data['asset']
-        assets_info = self._download_json(asset['url'], video_id)
+        categories = [data['section']['title']]
 
         formats = []
-        smil_url = assets_info['video']
+        smil_url = asset['video']
         if '.smil' in smil_url:
             m3u8_url = smil_url.replace('.smil', '.m3u8')
             formats.extend(
@@ -91,6 +92,7 @@ class SportDeutschlandIE(InfoExtractor):
             'title': asset['title'],
             'thumbnail': asset.get('image'),
             'description': asset.get('teaser'),
+            'duration': asset.get('duration'),
             'categories': categories,
             'view_count': asset.get('views'),
             'rtmp_live': asset.get('live'),

From 0fa5795b85f8d97bf67f10e39a79b49656be58db Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 16 Aug 2015 23:40:07 +0200
Subject: [PATCH 2284/6219] release 2015.08.16.1

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 689d6fca7..c090c6df7 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.08.16'
+__version__ = '2015.08.16.1'

From 369c12e038c3183a0e725a929dd9bed4ec35fa11 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 17 Aug 2015 20:16:43 +0600
Subject: [PATCH 2285/6219] [twitch] Allow untitled videos (Closes #6585)

---
 youtube_dl/extractor/twitch.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index a2b6a35aa..0521257e5 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -132,7 +132,7 @@ class TwitchItemBaseIE(TwitchBaseIE):
     def _extract_info(self, info):
         return {
             'id': info['_id'],
-            'title': info['title'],
+            'title': info.get('title') or 'Untitled Broadcast',
             'description': info['description'],
             'duration': info['length'],
             'thumbnail': info['preview'],

From 7a6e8a1b17a6a821d9200531ebf65562ccc2d428 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 17 Aug 2015 20:20:04 +0600
Subject: [PATCH 2286/6219] [twitch] Make more robust

---
 youtube_dl/extractor/twitch.py | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 0521257e5..8cba97bd4 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -15,6 +15,7 @@ from ..compat import (
 )
 from ..utils import (
     ExtractorError,
+    int_or_none,
     parse_duration,
     parse_iso8601,
 )
@@ -133,13 +134,13 @@ class TwitchItemBaseIE(TwitchBaseIE):
         return {
             'id': info['_id'],
             'title': info.get('title') or 'Untitled Broadcast',
-            'description': info['description'],
-            'duration': info['length'],
-            'thumbnail': info['preview'],
-            'uploader': info['channel']['display_name'],
-            'uploader_id': info['channel']['name'],
-            'timestamp': parse_iso8601(info['recorded_at']),
-            'view_count': info['views'],
+            'description': info.get('description'),
+            'duration': int_or_none(info.get('length')),
+            'thumbnail': info.get('preview'),
+            'uploader': info.get('channel', {}).get('display_name'),
+            'uploader_id': info.get('channel', {}).get('name'),
+            'timestamp': parse_iso8601(info.get('recorded_at')),
+            'view_count': int_or_none(info.get('views')),
         }
 
     def _real_extract(self, url):

From 9c724a980210ec6a7659fe869cce401dde6e189d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 17 Aug 2015 20:23:52 +0600
Subject: [PATCH 2287/6219] [twitch:vod] Add test for #6585

---
 youtube_dl/extractor/twitch.py | 23 +++++++++++++++++++++--
 1 file changed, 21 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 8cba97bd4..4f4eb6d72 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -188,7 +188,7 @@ class TwitchVodIE(TwitchItemBaseIE):
     _ITEM_TYPE = 'vod'
     _ITEM_SHORTCUT = 'v'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.twitch.tv/riotgames/v/6528877?t=5m10s',
         'info_dict': {
             'id': 'v6528877',
@@ -207,7 +207,26 @@ class TwitchVodIE(TwitchItemBaseIE):
             # m3u8 download
             'skip_download': True,
         },
-    }
+    }, {
+        # Untitled broadcast (title is None)
+        'url': 'http://www.twitch.tv/belkao_o/v/11230755',
+        'info_dict': {
+            'id': 'v11230755',
+            'ext': 'mp4',
+            'title': 'Untitled Broadcast',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 1638,
+            'timestamp': 1439746708,
+            'upload_date': '20150816',
+            'uploader': 'BelkAO_o',
+            'uploader_id': 'belkao_o',
+            'view_count': int,
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }]
 
     def _real_extract(self, url):
         item_id = self._match_id(url)

From 3b9b32f404ab09d9dc801dd8ec57d79711be5cb3 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 18 Aug 2015 13:02:41 +0200
Subject: [PATCH 2288/6219] [libsyn] Strip options from player URL

---
 youtube_dl/extractor/libsyn.py | 30 ++++++++++++++++++++----------
 1 file changed, 20 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/libsyn.py b/youtube_dl/extractor/libsyn.py
index 9ab1416f5..d375695f5 100644
--- a/youtube_dl/extractor/libsyn.py
+++ b/youtube_dl/extractor/libsyn.py
@@ -8,9 +8,9 @@ from ..utils import unified_strdate
 
 
 class LibsynIE(InfoExtractor):
-    _VALID_URL = r'https?://html5-player\.libsyn\.com/embed/episode/id/(?P<id>[0-9]+)'
+    _VALID_URL = r'(?P<mainurl>https?://html5-player\.libsyn\.com/embed/episode/id/(?P<id>[0-9]+))'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://html5-player.libsyn.com/embed/episode/id/3377616/',
         'md5': '443360ee1b58007bc3dcf09b41d093bb',
         'info_dict': {
@@ -19,12 +19,24 @@ class LibsynIE(InfoExtractor):
             'title': "The Daily Show Podcast without Jon Stewart - Episode 12: Bassem Youssef: Egypt's Jon Stewart",
             'description': 'md5:601cb790edd05908957dae8aaa866465',
             'upload_date': '20150220',
+            'thumbnail': 're:^https?://.*',
         },
-    }
+    }, {
+        'url': 'https://html5-player.libsyn.com/embed/episode/id/3727166/height/75/width/200/theme/standard/direction/no/autoplay/no/autonext/no/thumbnail/no/preload/no/no_addthis/no/',
+        'md5': '6c5cb21acd622d754d3b1a92b582ce42',
+        'info_dict': {
+            'id': '3727166',
+            'ext': 'mp3',
+            'title': 'Clients From Hell Podcast - How a Sex Toy Company Kickstarted my Freelance Career',
+            'upload_date': '20150818',
+            'thumbnail': 're:^https?://.*',
+        }
+    }]
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-
+        m = re.match(self._VALID_URL, url)
+        video_id = m.group('id')
+        url = m.group('mainurl')
         webpage = self._download_webpage(url, video_id)
 
         formats = [{
@@ -32,20 +44,18 @@ class LibsynIE(InfoExtractor):
         } for media_url in set(re.findall('var\s+mediaURL(?:Libsyn)?\s*=\s*"([^"]+)"', webpage))]
 
         podcast_title = self._search_regex(
-            r'<h2>([^<]+)</h2>', webpage, 'title')
+            r'<h2>([^<]+)</h2>', webpage, 'podcast title', default=None)
         episode_title = self._search_regex(
-            r'<h3>([^<]+)</h3>', webpage, 'title', default=None)
+            r'(?:<div class="episode-title">|<h3>)([^<]+)</', webpage, 'episode title')
 
         title = '%s - %s' % (podcast_title, episode_title) if podcast_title else episode_title
 
         description = self._html_search_regex(
             r'<div id="info_text_body">(.+?)</div>', webpage,
-            'description', fatal=False)
-
+            'description', default=None)
         thumbnail = self._search_regex(
             r'<img[^>]+class="info-show-icon"[^>]+src="([^"]+)"',
             webpage, 'thumbnail', fatal=False)
-
         release_date = unified_strdate(self._search_regex(
             r'<div class="release_date">Released: ([^<]+)<', webpage, 'release date', fatal=False))
 

From 34a4cd0a34bc9f07d865b02f6982fba60421ed0e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 18 Aug 2015 20:02:56 +0600
Subject: [PATCH 2289/6219] [telecinco] Relax _VALID_URL (Closes #6601)

---
 youtube_dl/extractor/telecinco.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/telecinco.py b/youtube_dl/extractor/telecinco.py
index a0c744fd1..ae94f055c 100644
--- a/youtube_dl/extractor/telecinco.py
+++ b/youtube_dl/extractor/telecinco.py
@@ -6,7 +6,7 @@ from .mitele import MiTeleIE
 
 class TelecincoIE(MiTeleIE):
     IE_NAME = 'telecinco.es'
-    _VALID_URL = r'https?://www\.telecinco\.es/[^/]+/[^/]+/(?:[^/]+/)?(?P<id>.*?)\.html'
+    _VALID_URL = r'https?://www\.telecinco\.es/(?:[^/]+/)+(?P<id>.+?)\.html'
 
     _TESTS = [{
         'url': 'http://www.telecinco.es/robinfood/temporada-01/t01xp14/Bacalao-cocochas-pil-pil_0_1876350223.html',
@@ -23,4 +23,7 @@ class TelecincoIE(MiTeleIE):
     }, {
         'url': 'http://www.telecinco.es/informativos/nacional/Pablo_Iglesias-Informativos_Telecinco-entrevista-Pedro_Piqueras_2_1945155182.html',
         'only_matching': True,
+    }, {
+        'url': 'http://www.telecinco.es/espanasinirmaslejos/Espana-gran-destino-turistico_2_1240605043.html',
+        'only_matching': True,
     }]

From 03c635a4b57e6ea4b874029d9fe3738508f6fc7d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 18 Aug 2015 20:26:45 +0600
Subject: [PATCH 2290/6219] [twitch] Fix login (Closes #6599)

---
 youtube_dl/extractor/twitch.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 4f4eb6d72..023911c41 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -12,6 +12,7 @@ from ..compat import (
     compat_urllib_parse,
     compat_urllib_parse_urlparse,
     compat_urllib_request,
+    compat_urlparse,
 )
 from ..utils import (
     ExtractorError,
@@ -27,7 +28,7 @@ class TwitchBaseIE(InfoExtractor):
     _API_BASE = 'https://api.twitch.tv'
     _USHER_BASE = 'http://usher.twitch.tv'
     _LOGIN_URL = 'https://secure.twitch.tv/login'
-    _LOGIN_POST_URL = 'https://passport.twitch.tv/authorize'
+    _LOGIN_POST_URL = 'https://passport.twitch.tv/authentications/new'
     _NETRC_MACHINE = 'twitch'
 
     def _handle_error(self, response):
@@ -70,8 +71,15 @@ class TwitchBaseIE(InfoExtractor):
             'password': password.encode('utf-8'),
         })
 
+        post_url = self._search_regex(
+            r'<form[^>]+action=(["\'])(?P<url>.+?)\1', login_page,
+            'post url', default=self._LOGIN_POST_URL, group='url')
+
+        if not post_url.startswith('http'):
+            post_url = compat_urlparse.urljoin(self._LOGIN_URL, post_url)
+
         request = compat_urllib_request.Request(
-            self._LOGIN_POST_URL, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
+            post_url, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
         request.add_header('Referer', self._LOGIN_URL)
         response = self._download_webpage(
             request, None, 'Logging in as %s' % username)

From 559f4c550f215a657ce386cab572bfc212128595 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 18 Aug 2015 20:27:58 +0600
Subject: [PATCH 2291/6219] [playtvak] PEP 8

---
 youtube_dl/extractor/playtvak.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/playtvak.py b/youtube_dl/extractor/playtvak.py
index 278fdc1aa..e360404f7 100644
--- a/youtube_dl/extractor/playtvak.py
+++ b/youtube_dl/extractor/playtvak.py
@@ -106,7 +106,7 @@ class PlaytvakIE(InfoExtractor):
         })
 
         info_url = compat_urlparse.urlunparse(
-            parsed_url._replace(query = compat_urllib_parse.urlencode(qs, True)))
+            parsed_url._replace(query=compat_urllib_parse.urlencode(qs, True)))
 
         json_info = self._download_json(
             info_url, video_id,

From f3a65d9636908ee49ff3d50c24efb8067caa32c6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 18 Aug 2015 21:10:52 +0600
Subject: [PATCH 2292/6219] [travis] Move to new infrastructure

We don't use rtmpdump in tests anyway
---
 .travis.yml | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 511bee64c..e78a2fa76 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -5,9 +5,7 @@ python:
   - "3.2"
   - "3.3"
   - "3.4"
-before_install:
-  - sudo apt-get update -qq
-  - sudo apt-get install -yqq rtmpdump
+sudo: false
 script: nosetests test --verbose
 notifications:
   email:

From a01da8bbf83dfd4f87e3fdd105b9f7c850e76cad Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 18 Aug 2015 23:02:57 +0600
Subject: [PATCH 2293/6219] [crunchyroll] Workaround fplive.net rtmp URLs
 (Closes #5881)

---
 youtube_dl/extractor/crunchyroll.py | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index d1b6d7366..33a033a7f 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -14,11 +14,13 @@ from ..compat import (
     compat_urllib_parse,
     compat_urllib_parse_unquote,
     compat_urllib_request,
+    compat_urlparse,
 )
 from ..utils import (
     ExtractorError,
     bytes_to_intlist,
     intlist_to_bytes,
+    remove_end,
     unified_strdate,
     urlencode_postdata,
 )
@@ -279,6 +281,20 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
             stream_info = streamdata.find('./{default}preload/stream_info')
             video_url = stream_info.find('./host').text
             video_play_path = stream_info.find('./file').text
+
+            if '.fplive.net/' in video_url:
+                video_url = re.sub(r'^rtmpe?://', 'http://', video_url.strip())
+                parsed_video_url = compat_urlparse.urlparse(video_url)
+                direct_video_url = compat_urlparse.urlunparse(parsed_video_url._replace(
+                    netloc='v.lvlt.crcdn.net',
+                    path='%s/%s' % (remove_end(parsed_video_url.path, '/'), video_play_path.split(':')[-1])))
+                if self._is_valid_url(direct_video_url, video_id, video_format):
+                    formats.append({
+                        'url': direct_video_url,
+                        'format_id': video_format,
+                    })
+                    continue
+
             formats.append({
                 'url': video_url,
                 'play_path': video_play_path,

From ca681f7041838fa215f8ab5266cd7b442f3f9445 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 19 Aug 2015 20:52:36 +0600
Subject: [PATCH 2294/6219] [videobam] Remove extractor

videobam.com redirects to sendvid.com now
---
 youtube_dl/extractor/__init__.py |  1 -
 youtube_dl/extractor/videobam.py | 81 --------------------------------
 2 files changed, 82 deletions(-)
 delete mode 100644 youtube_dl/extractor/videobam.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index fa9acc923..c8c9f1855 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -696,7 +696,6 @@ from .vgtv import (
 from .vh1 import VH1IE
 from .vice import ViceIE
 from .viddler import ViddlerIE
-from .videobam import VideoBamIE
 from .videodetective import VideoDetectiveIE
 from .videolecturesnet import VideoLecturesNetIE
 from .videofyme import VideofyMeIE
diff --git a/youtube_dl/extractor/videobam.py b/youtube_dl/extractor/videobam.py
deleted file mode 100644
index 0eb3d9414..000000000
--- a/youtube_dl/extractor/videobam.py
+++ /dev/null
@@ -1,81 +0,0 @@
-from __future__ import unicode_literals
-
-import re
-import json
-
-from .common import InfoExtractor
-from ..utils import int_or_none
-
-
-class VideoBamIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?videobam\.com/(?:videos/download/)?(?P<id>[a-zA-Z]+)'
-
-    _TESTS = [
-        {
-            'url': 'http://videobam.com/OiJQM',
-            'md5': 'db471f27763a531f10416a0c58b5a1e0',
-            'info_dict': {
-                'id': 'OiJQM',
-                'ext': 'mp4',
-                'title': 'Is Alcohol Worse Than Ecstasy?',
-                'description': 'md5:d25b96151515c91debc42bfbb3eb2683',
-                'uploader': 'frihetsvinge',
-            },
-        },
-        {
-            'url': 'http://videobam.com/pqLvq',
-            'md5': 'd9a565b5379a99126ef94e1d7f9a383e',
-            'note': 'HD video',
-            'info_dict': {
-                'id': 'pqLvq',
-                'ext': 'mp4',
-                'title': '_',
-            }
-        },
-    ]
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
-        page = self._download_webpage('http://videobam.com/%s' % video_id, video_id, 'Downloading page')
-
-        formats = []
-
-        for preference, format_id in enumerate(['low', 'high']):
-            mobj = re.search(r"%s: '(?P<url>[^']+)'" % format_id, page)
-            if not mobj:
-                continue
-            formats.append({
-                'url': mobj.group('url'),
-                'ext': 'mp4',
-                'format_id': format_id,
-                'preference': preference,
-            })
-
-        if not formats:
-            player_config = json.loads(self._html_search_regex(r'var player_config = ({.+?});', page, 'player config'))
-            formats = [{
-                'url': item['url'],
-                'ext': 'mp4',
-            } for item in player_config['playlist'] if 'autoPlay' in item]
-
-        self._sort_formats(formats)
-
-        title = self._og_search_title(page, default='_', fatal=False)
-        description = self._og_search_description(page, default=None)
-        thumbnail = self._og_search_thumbnail(page)
-        uploader = self._html_search_regex(r'Upload by ([^<]+)</a>', page, 'uploader', fatal=False, default=None)
-        view_count = int_or_none(
-            self._html_search_regex(r'<strong>Views:</strong> (\d+) ', page, 'view count', fatal=False))
-
-        return {
-            'id': video_id,
-            'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
-            'uploader': uploader,
-            'view_count': view_count,
-            'formats': formats,
-            'age_limit': 18,
-        }

From f877c6ae5a6e252d6904f90d597479451d2107aa Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 19 Aug 2015 23:11:25 +0800
Subject: [PATCH 2295/6219] [theplatform] Use
 InfoExtractor._parse_smil_formats()

---
 youtube_dl/extractor/common.py      |  8 +++-
 youtube_dl/extractor/theplatform.py | 72 ++++++-----------------------
 2 files changed, 20 insertions(+), 60 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 65835d257..ac12be933 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1052,7 +1052,7 @@ class InfoExtractor(object):
         return self._search_regex(
             r'(?i)^{([^}]+)?}smil$', smil.tag, 'namespace', default=None)
 
-    def _parse_smil_formats(self, smil, smil_url, video_id, namespace=None, f4m_params=None):
+    def _parse_smil_formats(self, smil, smil_url, video_id, namespace=None, f4m_params=None, transform_rtmp_url=None):
         base = smil_url
         for meta in smil.findall(self._xpath_ns('./head/meta', namespace)):
             b = meta.get('base') or meta.get('httpBase')
@@ -1091,6 +1091,12 @@ class InfoExtractor(object):
                     'width': width,
                     'height': height,
                 })
+                if transform_rtmp_url:
+                    streamer, src = transform_rtmp_url(streamer, src)
+                    formats[-1].update({
+                        'url': streamer,
+                        'play_path': src,
+                    })
                 continue
 
             src_url = src if src.startswith('http') else compat_urlparse.urljoin(base, src)
diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index 0643eccaf..29f938a76 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -9,9 +9,6 @@ import hashlib
 
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_str,
-)
 from ..utils import (
     determine_ext,
     ExtractorError,
@@ -20,7 +17,8 @@ from ..utils import (
     int_or_none,
 )
 
-_x = lambda p: xpath_with_ns(p, {'smil': 'http://www.w3.org/2005/SMIL21/Language'})
+default_ns = 'http://www.w3.org/2005/SMIL21/Language'
+_x = lambda p: xpath_with_ns(p, {'smil': default_ns})
 
 
 class ThePlatformIE(InfoExtractor):
@@ -145,63 +143,19 @@ class ThePlatformIE(InfoExtractor):
                     'url': src,
                 }]
 
-        head = meta.find(_x('smil:head'))
-        body = meta.find(_x('smil:body'))
-
-        f4m_node = body.find(_x('smil:seq//smil:video'))
-        if f4m_node is None:
-            f4m_node = body.find(_x('smil:seq/smil:video'))
-        if f4m_node is not None and '.f4m' in f4m_node.attrib['src']:
-            f4m_url = f4m_node.attrib['src']
-            if 'manifest.f4m?' not in f4m_url:
-                f4m_url += '?'
+        formats = self._parse_smil_formats(
+            meta, smil_url, video_id, namespace=default_ns,
             # the parameters are from syfy.com, other sites may use others,
             # they also work for nbc.com
-            f4m_url += '&g=UXWGVKRWHFSP&hdcore=3.0.3'
-            formats = self._extract_f4m_formats(f4m_url, video_id)
-        else:
-            formats = []
-            switch = body.find(_x('smil:switch'))
-            if switch is None:
-                switch = body.find(_x('smil:par//smil:switch'))
-            if switch is None:
-                switch = body.find(_x('smil:par/smil:switch'))
-            if switch is None:
-                switch = body.find(_x('smil:par'))
-            if switch is not None:
-                base_url = head.find(_x('smil:meta')).attrib['base']
-                for f in switch.findall(_x('smil:video')):
-                    attr = f.attrib
-                    width = int_or_none(attr.get('width'))
-                    height = int_or_none(attr.get('height'))
-                    vbr = int_or_none(attr.get('system-bitrate'), 1000)
-                    format_id = '%dx%d_%dk' % (width, height, vbr)
-                    formats.append({
-                        'format_id': format_id,
-                        'url': base_url,
-                        'play_path': 'mp4:' + attr['src'],
-                        'ext': 'flv',
-                        'width': width,
-                        'height': height,
-                        'vbr': vbr,
-                    })
-            else:
-                switch = body.find(_x('smil:seq//smil:switch'))
-                if switch is None:
-                    switch = body.find(_x('smil:seq/smil:switch'))
-                for f in switch.findall(_x('smil:video')):
-                    attr = f.attrib
-                    vbr = int_or_none(attr.get('system-bitrate'), 1000)
-                    ext = determine_ext(attr['src'])
-                    if ext == 'once':
-                        ext = 'mp4'
-                    formats.append({
-                        'format_id': compat_str(vbr),
-                        'url': attr['src'],
-                        'vbr': vbr,
-                        'ext': ext,
-                    })
-            self._sort_formats(formats)
+            f4m_params={'g': 'UXWGVKRWHFSP', 'hdcore': '3.0.3'},
+            transform_rtmp_url=lambda streamer, src: (streamer, 'mp4:' + src))
+
+        for _format in formats:
+            ext = determine_ext(_format['url'])
+            if ext == 'once':
+                _format['ext'] = 'mp4'
+
+        self._sort_formats(formats)
 
         return {
             'id': video_id,

From 7900aede14d2e2a46c8fd4430e48cde41f354859 Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Wed, 22 Jul 2015 14:31:29 +0800
Subject: [PATCH 2296/6219] [mwave] New extractor for mwave.interest.me

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/mwave.py    | 46 ++++++++++++++++++++++++++++++++
 2 files changed, 47 insertions(+)
 create mode 100644 youtube_dl/extractor/mwave.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index c8c9f1855..006ef3922 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -345,6 +345,7 @@ from .muenchentv import MuenchenTVIE
 from .musicplayon import MusicPlayOnIE
 from .musicvault import MusicVaultIE
 from .muzu import MuzuTVIE
+from .mwave import MwaveIE
 from .myspace import MySpaceIE, MySpaceAlbumIE
 from .myspass import MySpassIE
 from .myvi import MyviIE
diff --git a/youtube_dl/extractor/mwave.py b/youtube_dl/extractor/mwave.py
new file mode 100644
index 000000000..7f91aa269
--- /dev/null
+++ b/youtube_dl/extractor/mwave.py
@@ -0,0 +1,46 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class MwaveIE(InfoExtractor):
+    IE_NAME = 'mwave'
+    _VALID_URL = r'https?://mwave\.interest\.me/mnettv/videodetail\.m\?searchVideoDetailVO\.clip_id=(?P<id>[0-9]+)'
+    _TESTS = [{
+        'url': 'http://mwave.interest.me/mnettv/videodetail.m?searchVideoDetailVO.clip_id=168859',
+        'info_dict': {
+            'id': '168859',
+            'ext': 'flv',
+            'title': '[M COUNTDOWN] SISTAR - SHAKE IT',
+            'creator': 'M COUNTDOWN',
+        }
+    }, {
+        'url': 'http://mwave.interest.me/mnettv/videodetail.m?searchVideoDetailVO.clip_id=168860',
+        'info_dict': {
+            'id': '168860',
+            'ext': 'flv',
+            'title': '[Full Ver.] M GIGS Ep. 59 - IDIOTAPE Live Part 1',
+            'creator': 'M-GIGS',
+        }
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        stream_info = self._download_json(
+            'http://mwave.interest.me/onair/vod_info.m?vodtype=CL&sectorid=&endinfo=Y&id=%s' % video_id,
+            'Download stream info')
+
+        formats = []
+        for info in stream_info['cdn']:
+            f4m_stream = self._download_json(info['url'], video_id, 'Download f4m stream')
+            formats.extend(
+                self._extract_f4m_formats(f4m_stream['fileurl'] + '&g=PCROWKHLYUDY&hdcore=3.0.3', video_id))
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': stream_info['title'],
+            'creator': stream_info.get('program_title'),
+            'formats': formats,
+        }

From 22c83245c51faa53118a8f815b13b2e4c2df9923 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 19 Aug 2015 23:07:41 +0600
Subject: [PATCH 2297/6219] [mwave] Improve

---
 youtube_dl/extractor/mwave.py | 50 ++++++++++++++++++++++-------------
 1 file changed, 31 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/extractor/mwave.py b/youtube_dl/extractor/mwave.py
index 7f91aa269..66b523197 100644
--- a/youtube_dl/extractor/mwave.py
+++ b/youtube_dl/extractor/mwave.py
@@ -1,46 +1,58 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..compat import compat_str
+from ..utils import (
+    int_or_none,
+    parse_duration,
+)
 
 
 class MwaveIE(InfoExtractor):
-    IE_NAME = 'mwave'
     _VALID_URL = r'https?://mwave\.interest\.me/mnettv/videodetail\.m\?searchVideoDetailVO\.clip_id=(?P<id>[0-9]+)'
-    _TESTS = [{
+    _TEST = {
         'url': 'http://mwave.interest.me/mnettv/videodetail.m?searchVideoDetailVO.clip_id=168859',
+        'md5': 'c930e27b7720aaa3c9d0018dfc8ff6cc',
         'info_dict': {
             'id': '168859',
             'ext': 'flv',
             'title': '[M COUNTDOWN] SISTAR - SHAKE IT',
-            'creator': 'M COUNTDOWN',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'uploader': 'M COUNTDOWN',
+            'duration': 206,
+            'view_count': int,
         }
-    }, {
-        'url': 'http://mwave.interest.me/mnettv/videodetail.m?searchVideoDetailVO.clip_id=168860',
-        'info_dict': {
-            'id': '168860',
-            'ext': 'flv',
-            'title': '[Full Ver.] M GIGS Ep. 59 - IDIOTAPE Live Part 1',
-            'creator': 'M-GIGS',
-        }
-    }]
+    }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        stream_info = self._download_json(
+        vod_info = self._download_json(
             'http://mwave.interest.me/onair/vod_info.m?vodtype=CL&sectorid=&endinfo=Y&id=%s' % video_id,
-            'Download stream info')
+            video_id, 'Download vod JSON')
 
         formats = []
-        for info in stream_info['cdn']:
-            f4m_stream = self._download_json(info['url'], video_id, 'Download f4m stream')
+        for num, cdn_info in enumerate(vod_info['cdn']):
+            stream_url = cdn_info.get('url')
+            if not stream_url:
+                continue
+            stream_name = cdn_info.get('name') or compat_str(num)
+            f4m_stream = self._download_json(
+                stream_url, video_id,
+                'Download %s stream JSON' % stream_name)
+            f4m_url = f4m_stream.get('fileurl')
+            if not f4m_url:
+                continue
             formats.extend(
-                self._extract_f4m_formats(f4m_stream['fileurl'] + '&g=PCROWKHLYUDY&hdcore=3.0.3', video_id))
+                self._extract_f4m_formats(f4m_url + '&hdcore=3.0.3', video_id, f4m_id=stream_name))
         self._sort_formats(formats)
 
         return {
             'id': video_id,
-            'title': stream_info['title'],
-            'creator': stream_info.get('program_title'),
+            'title': vod_info['title'],
+            'thumbnail': vod_info.get('cover'),
+            'uploader': vod_info.get('program_title'),
+            'duration': parse_duration(vod_info.get('time')),
+            'view_count': int_or_none(vod_info.get('hit')),
             'formats': formats,
         }

From 26e1c3514f4af1ed60cd1114a653fe49e1fa8d11 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 20 Aug 2015 01:24:32 +0800
Subject: [PATCH 2298/6219] [theplatform] Add ThePlatformFeedIE

---
 youtube_dl/extractor/__init__.py    |   5 +-
 youtube_dl/extractor/theplatform.py | 162 +++++++++++++++++++++-------
 2 files changed, 126 insertions(+), 41 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index fa9acc923..b5f7ff9a9 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -612,7 +612,10 @@ from .testurl import TestURLIE
 from .testtube import TestTubeIE
 from .tf1 import TF1IE
 from .theonion import TheOnionIE
-from .theplatform import ThePlatformIE
+from .theplatform import (
+    ThePlatformIE,
+    ThePlatformFeedIE,
+)
 from .thesixtyone import TheSixtyOneIE
 from .thisamericanlife import ThisAmericanLifeIE
 from .thisav import ThisAVIE
diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index 29f938a76..f02e0f58d 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -15,13 +15,68 @@ from ..utils import (
     xpath_with_ns,
     unsmuggle_url,
     int_or_none,
+    url_basename,
+    float_or_none,
 )
 
 default_ns = 'http://www.w3.org/2005/SMIL21/Language'
 _x = lambda p: xpath_with_ns(p, {'smil': default_ns})
 
 
-class ThePlatformIE(InfoExtractor):
+class ThePlatformBaseIE(InfoExtractor):
+    def _extract_theplatform_smil_formats(self, smil_url, video_id, note='Downloading SMIL data'):
+        meta = self._download_xml(smil_url, video_id, note=note)
+        try:
+            error_msg = next(
+                n.attrib['abstract']
+                for n in meta.findall(_x('.//smil:ref'))
+                if n.attrib.get('title') == 'Geographic Restriction' or n.attrib.get('title') == 'Expired')
+        except StopIteration:
+            pass
+        else:
+            raise ExtractorError(error_msg, expected=True)
+
+        formats = self._parse_smil_formats(
+            meta, smil_url, video_id, namespace=default_ns,
+            # the parameters are from syfy.com, other sites may use others,
+            # they also work for nbc.com
+            f4m_params={'g': 'UXWGVKRWHFSP', 'hdcore': '3.0.3'},
+            transform_rtmp_url=lambda streamer, src: (streamer, 'mp4:' + src))
+
+        for _format in formats:
+            ext = determine_ext(_format['url'])
+            if ext == 'once':
+                _format['ext'] = 'mp4'
+
+        self._sort_formats(formats)
+
+        return formats
+
+    def get_metadata(self, path, video_id):
+        info_url = 'http://link.theplatform.com/s/%s?format=preview' % path
+        info_json = self._download_webpage(info_url, video_id)
+        info = json.loads(info_json)
+
+        subtitles = {}
+        captions = info.get('captions')
+        if isinstance(captions, list):
+            for caption in captions:
+                lang, src, mime = caption.get('lang', 'en'), caption.get('src'), caption.get('type')
+                subtitles[lang] = [{
+                    'ext': 'srt' if mime == 'text/srt' else 'ttml',
+                    'url': src,
+                }]
+
+        return {
+            'title': info['title'],
+            'subtitles': subtitles,
+            'description': info['description'],
+            'thumbnail': info['defaultThumbnailUrl'],
+            'duration': int_or_none(info.get('duration'), 1000),
+        }
+
+
+class ThePlatformIE(ThePlatformBaseIE):
     _VALID_URL = r'''(?x)
         (?:https?://(?:link|player)\.theplatform\.com/[sp]/(?P<provider_id>[^/]+)/
            (?:(?P<media>(?:[^/]+/)+select/media/)|(?P<config>(?:[^/\?]+/(?:swf|config)|onsite)/select/))?
@@ -118,51 +173,78 @@ class ThePlatformIE(InfoExtractor):
         if sig:
             smil_url = self._sign_url(smil_url, sig['key'], sig['secret'])
 
-        meta = self._download_xml(smil_url, video_id)
-        try:
-            error_msg = next(
-                n.attrib['abstract']
-                for n in meta.findall(_x('.//smil:ref'))
-                if n.attrib.get('title') == 'Geographic Restriction' or n.attrib.get('title') == 'Expired')
-        except StopIteration:
-            pass
-        else:
-            raise ExtractorError(error_msg, expected=True)
+        formats = self._extract_theplatform_smil_formats(smil_url, video_id)
 
-        info_url = 'http://link.theplatform.com/s/%s?format=preview' % path
-        info_json = self._download_webpage(info_url, video_id)
-        info = json.loads(info_json)
+        ret = self.get_metadata(path, video_id)
+        ret.update({
+            'id': video_id,
+            'formats': formats,
+        })
 
-        subtitles = {}
-        captions = info.get('captions')
-        if isinstance(captions, list):
-            for caption in captions:
-                lang, src, mime = caption.get('lang', 'en'), caption.get('src'), caption.get('type')
-                subtitles[lang] = [{
-                    'ext': 'srt' if mime == 'text/srt' else 'ttml',
-                    'url': src,
-                }]
+        return ret
 
-        formats = self._parse_smil_formats(
-            meta, smil_url, video_id, namespace=default_ns,
-            # the parameters are from syfy.com, other sites may use others,
-            # they also work for nbc.com
-            f4m_params={'g': 'UXWGVKRWHFSP', 'hdcore': '3.0.3'},
-            transform_rtmp_url=lambda streamer, src: (streamer, 'mp4:' + src))
 
-        for _format in formats:
-            ext = determine_ext(_format['url'])
-            if ext == 'once':
-                _format['ext'] = 'mp4'
+class ThePlatformFeedIE(ThePlatformBaseIE):
+    _URL_TEMPLATE = '%s//feed.theplatform.com/f/%s/%s?form=json&byGuid=%s'
+    _VALID_URL = r'https?://feed\.theplatform\.com/f/(?P<provider_id>[^/]+)/(?P<feed_id>[^?/]+)\?(?:[^&]+&)*byGuid=(?P<id>[a-zA-Z0-9_]+)'
+    _TEST = {
+        # From http://player.theplatform.com/p/7wvmTC/MSNBCEmbeddedOffSite?guid=n_hardball_5biden_140207
+        'url': 'http://feed.theplatform.com/f/7wvmTC/msnbc_video-p-test?form=json&pretty=true&range=-40&byGuid=n_hardball_5biden_140207',
+        'md5': '22d2b84f058d3586efcd99e57d59d314',
+        'info_dict': {
+            'id': 'n_hardball_5biden_140207',
+            'ext': 'mp4',
+            'title': 'The Biden factor: will Joe run in 2016?',
+            'description': 'Could Vice President Joe Biden be preparing a 2016 campaign? Mark Halperin and Sam Stein weigh in.',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'upload_date': '20140208',
+            'timestamp': 1391824260,
+            'duration': 467.0,
+            'categories': ['MSNBC/Issues/Democrats', 'MSNBC/Issues/Elections/Election 2016'],
+        },
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+
+        video_id = mobj.group('id')
+        provider_id = mobj.group('provider_id')
+        feed_id = mobj.group('feed_id')
+
+        real_url = self._URL_TEMPLATE % (self.http_scheme(), provider_id, feed_id, video_id)
+        feed = self._download_json(real_url, video_id)
+        entry = feed['entries'][0]
+
+        formats = []
+        first_video_id = None
+        duration = None
+        for item in entry['media$content']:
+            smil_url = item['plfile$url'] + '&format=SMIL&Tracking=true&Embedded=true&formats=MPEG4,F4M'
+            cur_video_id = url_basename(smil_url)
+            if first_video_id is None:
+                first_video_id = cur_video_id
+                duration = float_or_none(item.get('plfile$duration'))
+            formats.extend(self._extract_theplatform_smil_formats(smil_url, video_id, 'Downloading SMIL data for %s' % cur_video_id))
 
         self._sort_formats(formats)
 
-        return {
+        thumbnails = [{
+            'url': thumbnail['plfile$url'],
+            'width': int_or_none(thumbnail.get('plfile$width')),
+            'height': int_or_none(thumbnail.get('plfile$height')),
+        } for thumbnail in entry.get('media$thumbnails', [])]
+
+        timestamp = int_or_none(entry.get('media$availableDate'), scale=1000)
+        categories = [item['media$name'] for item in entry.get('media$categories', [])]
+
+        ret = self.get_metadata('%s/%s' % (provider_id, first_video_id), video_id)
+        ret.update({
             'id': video_id,
-            'title': info['title'],
-            'subtitles': subtitles,
             'formats': formats,
-            'description': info['description'],
-            'thumbnail': info['defaultThumbnailUrl'],
-            'duration': int_or_none(info.get('duration'), 1000),
-        }
+            'thumbnails': thumbnails,
+            'duration': duration,
+            'timestamp': timestamp,
+            'categories': categories,
+        })
+
+        return ret

From 05fe2594e4589b4e714a423550172eeec3949a70 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 20 Aug 2015 01:38:39 +0800
Subject: [PATCH 2299/6219] [theplatform] Support URLs with 'guid='

---
 youtube_dl/extractor/theplatform.py | 36 +++++++++++++++++++++++++++++
 1 file changed, 36 insertions(+)

diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index f02e0f58d..883bf491c 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -9,6 +9,10 @@ import hashlib
 
 
 from .common import InfoExtractor
+from ..compat import (
+    compat_parse_qs,
+    compat_urllib_parse_urlparse,
+)
 from ..utils import (
     determine_ext,
     ExtractorError,
@@ -120,6 +124,20 @@ class ThePlatformIE(ThePlatformBaseIE):
     }, {
         'url': 'http://player.theplatform.com/p/NnzsPC/widget/select/media/4Y0TlYUr_ZT7',
         'only_matching': True,
+    }, {
+        'url': 'http://player.theplatform.com/p/2E2eJC/nbcNewsOffsite?guid=tdy_or_siri_150701',
+        'md5': '734f3790fb5fc4903da391beeebc4836',
+        'info_dict': {
+            'id': 'tdy_or_siri_150701',
+            'ext': 'mp4',
+            'title': 'iPhone Siri’s sassy response to a math question has people talking',
+            'description': 'md5:a565d1deadd5086f3331d57298ec6333',
+            'duration': 83.0,
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'timestamp': 1435752600,
+            'upload_date': '20150701',
+            'categories': ['Today/Shows/Orange Room', 'Today/Sections/Money', 'Today/Topics/Tech', "Today/Topics/Editor's picks"],
+        },
     }]
 
     @staticmethod
@@ -154,6 +172,24 @@ class ThePlatformIE(ThePlatformBaseIE):
             path += '/media'
         path += '/' + video_id
 
+        qs_dict = compat_parse_qs(compat_urllib_parse_urlparse(url).query)
+        if 'guid' in qs_dict:
+            webpage = self._download_webpage(url, video_id)
+            scripts = re.findall(r'<script[^>]+src="([^"]+)"', webpage)
+            feed_id = None
+            # feed id usually locates in the last script.
+            # Seems there's no pattern for the interested script filename, so
+            # I try one by one
+            for script in reversed(scripts):
+                feed_script = self._download_webpage(script, video_id, 'Downloading feed script')
+                feed_id = self._search_regex(r'defaultFeedId\s*:\s*"([^"]+)"', feed_script, 'default feed id', default=None)
+                if feed_id is not None:
+                    break
+            if feed_id is None:
+                raise ExtractorError('Unable to find feed id')
+            return self.url_result('http://feed.theplatform.com/f/%s/%s?byGuid=%s' % (
+                provider_id, feed_id, qs_dict['guid'][0]))
+
         if smuggled_data.get('force_smil_url', False):
             smil_url = url
         elif mobj.group('config'):

From dac14bf311fd1b3c6af6c57b3b03878a11ef5aae Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 20 Aug 2015 01:41:18 +0800
Subject: [PATCH 2300/6219] [nbc] Add MSNBCIE

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/nbc.py      | 25 +++++++++++++++++++++++++
 2 files changed, 26 insertions(+)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index b5f7ff9a9..86ea0576a 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -358,6 +358,7 @@ from .nbc import (
     NBCNewsIE,
     NBCSportsIE,
     NBCSportsVPlayerIE,
+    MSNBCIE,
 )
 from .ndr import (
     NDRIE,
diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index ccdbfb6c9..e683d24c4 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -236,3 +236,28 @@ class NBCNewsIE(InfoExtractor):
                 'url': info['videoAssets'][-1]['publicUrl'],
                 'ie_key': 'ThePlatform',
             }
+
+
+class MSNBCIE(InfoExtractor):
+    # https URLs redirect to corresponding http ones
+    _VALID_URL = r'http://www\.msnbc\.com/[^/]+/watch/(?P<id>[^/]+)'
+    _TEST = {
+        'url': 'http://www.msnbc.com/all-in-with-chris-hayes/watch/the-chaotic-gop-immigration-vote-314487875924',
+        'md5': '6d236bf4f3dddc226633ce6e2c3f814d',
+        'info_dict': {
+            'id': 'n_hayes_Aimm_140801_272214',
+            'ext': 'mp4',
+            'title': 'The chaotic GOP immigration vote',
+            'description': 'The Republican House votes on a border bill that has no chance of getting through the Senate or signed by the President and is drawing criticism from all sides.',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'timestamp': 1406937606,
+            'upload_date': '20140802',
+            'categories': ['MSNBC/Topics/Franchise/Best of last night', 'MSNBC/Topics/General/Congress'],
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        embed_url = self._html_search_meta('embedURL', webpage)
+        return self.url_result(embed_url)

From aa6cd05ed82b14af0e3827b2ff43eed02087b574 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 20 Aug 2015 01:47:55 +0800
Subject: [PATCH 2301/6219] [theplatform] Fix Python 2: declare coding

---
 youtube_dl/extractor/theplatform.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index 883bf491c..adaec3375 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
 import re

From ce00af87670d47f4dff6ad80e46a29e49cbdfe4f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 20 Aug 2015 00:56:17 +0600
Subject: [PATCH 2302/6219] [extractor/common] Add default subtitles lang

---
 youtube_dl/extractor/common.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index ac12be933..b1af45870 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1135,7 +1135,7 @@ class InfoExtractor(object):
 
         return formats
 
-    def _parse_smil_subtitles(self, smil, namespace=None):
+    def _parse_smil_subtitles(self, smil, namespace=None, subtitles_lang='en'):
         subtitles = {}
         for num, textstream in enumerate(smil.findall(self._xpath_ns('.//textstream', namespace))):
             src = textstream.get('src')
@@ -1146,7 +1146,7 @@ class InfoExtractor(object):
                 type_ = textstream.get('type')
                 if type_ == 'text/srt':
                     ext = 'srt'
-            lang = textstream.get('systemLanguage') or textstream.get('systemLanguageName')
+            lang = textstream.get('systemLanguage') or textstream.get('systemLanguageName') or subtitles_lang
             subtitles.setdefault(lang, []).append({
                 'url': src,
                 'ext': ext,

From 5cdefc46257802708816e1d4ea7ff5cafe910ff6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 20 Aug 2015 01:02:50 +0600
Subject: [PATCH 2303/6219] [extractor/common] Add more subtitle mime types for
 guess when ext is missing

---
 youtube_dl/extractor/common.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index b1af45870..ce2030d28 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1144,8 +1144,13 @@ class InfoExtractor(object):
             ext = textstream.get('ext') or determine_ext(src)
             if not ext:
                 type_ = textstream.get('type')
-                if type_ == 'text/srt':
-                    ext = 'srt'
+                SUBTITLES_TYPES = {
+                    'text/vtt': 'vtt',
+                    'text/srt': 'srt',
+                    'application/smptett+xml': 'tt',
+                }
+                if type_ in SUBTITLES_TYPES:
+                    ext = SUBTITLES_TYPES[type_]
             lang = textstream.get('systemLanguage') or textstream.get('systemLanguageName') or subtitles_lang
             subtitles.setdefault(lang, []).append({
                 'url': src,

From dd565ac1ad22fe48f8e358d95ea912b1768b1e5a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 20 Aug 2015 03:07:04 +0800
Subject: [PATCH 2304/6219] [theplatform] Use _download_json

---
 youtube_dl/extractor/theplatform.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index adaec3375..ba05ada39 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -2,7 +2,6 @@
 from __future__ import unicode_literals
 
 import re
-import json
 import time
 import hmac
 import binascii
@@ -59,8 +58,7 @@ class ThePlatformBaseIE(InfoExtractor):
 
     def get_metadata(self, path, video_id):
         info_url = 'http://link.theplatform.com/s/%s?format=preview' % path
-        info_json = self._download_webpage(info_url, video_id)
-        info = json.loads(info_json)
+        info = self._download_json(info_url, video_id)
 
         subtitles = {}
         captions = info.get('captions')

From 061f62da54cb4184a039108e40dee8e9eb2611c1 Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Thu, 20 Aug 2015 12:56:11 +0800
Subject: [PATCH 2305/6219] [vlive] New extractor for vlive.tv

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/vlive.py    | 94 ++++++++++++++++++++++++++++++++
 2 files changed, 95 insertions(+)
 create mode 100644 youtube_dl/extractor/vlive.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 1c53a5632..6bee5b63c 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -735,6 +735,7 @@ from .vk import (
     VKIE,
     VKUserVideosIE,
 )
+from .vlive import VLiveIE
 from .vodlocker import VodlockerIE
 from .voicerepublic import VoiceRepublicIE
 from .vporn import VpornIE
diff --git a/youtube_dl/extractor/vlive.py b/youtube_dl/extractor/vlive.py
new file mode 100644
index 000000000..b3bbd80fb
--- /dev/null
+++ b/youtube_dl/extractor/vlive.py
@@ -0,0 +1,94 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import hmac
+from hashlib import sha1
+from base64 import b64encode
+from time import time
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    determine_ext
+)
+from ..compat import compat_urllib_parse
+
+
+class VLiveIE(InfoExtractor):
+    IE_NAME = 'vlive'
+    _VALID_URL = r'https?://(?:(www|m)\.)?vlive\.tv/video/(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'http://m.vlive.tv/video/1326',
+        'md5': 'cc7314812855ce56de70a06a27314983',
+        'info_dict': {
+            'id': '1326',
+            'ext': 'mp4',
+            'title': '[V] Girl\'s Day\'s Broadcast',
+            'creator': 'Girl\'s Day',
+            'upload_date': '20150817',
+        },
+    }
+    _SECRET = 'rFkwZet6pqk1vQt6SxxUkAHX7YL3lmqzUMrU4IDusTo4jEBdtOhNfT4BYYAdArwH'
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(
+            'http://m.vlive.tv/video/%s' % video_id,
+            video_id, note='Download video page')
+
+        title = self._og_search_title(webpage)
+        thumbnail = self._og_search_thumbnail(webpage)
+        creator = self._html_search_regex(
+            r'<span class="name">([^<>]+)</span>', webpage, 'creator')
+        upload_date = self._html_search_regex(
+            r'<span class="time">(\d{4}\.\d{2}\.\d{2})</span>', webpage,
+            'upload date', default=None, fatal=False)
+        if upload_date:
+            upload_date = upload_date.replace('.', '')
+
+        url = 'http://global.apis.naver.com/globalV/globalV/vod/%s/playinfo?' % video_id
+        msgpad = {'msgpad': '%.0f' % (time() * 1000)}
+        md = {
+            'md': b64encode(
+                hmac.new(self._SECRET.encode('ascii'),
+                         (url[:255] + msgpad['msgpad']).encode('ascii'), sha1).digest())
+        }
+        url += '&' + compat_urllib_parse.urlencode(msgpad) + '&' + compat_urllib_parse.urlencode(md)
+
+        playinfo = self._download_json(url, video_id, 'Downloading video json')
+
+        if playinfo.get('message', '') != 'success':
+            raise ExtractorError(playinfo['message'])
+
+        if not playinfo.get('result'):
+            raise ExtractorError('No videos found.')
+
+        formats = []
+        for vid in playinfo['result'].get('videos', {}).get('list', []):
+            formats.append({
+                'url': vid['source'],
+                'ext': 'mp4',
+                'abr': vid.get('bitrate', {}).get('audio'),
+                'vbr': vid.get('bitrate', {}).get('video'),
+                'format_id': vid['encodingOption']['name'],
+                'height': vid.get('height'),
+                'width': vid.get('width'),
+            })
+        self._sort_formats(formats)
+
+        subtitles = {}
+        for caption in playinfo['result'].get('captions', {}).get('list', []):
+            subtitles[caption['language']] = [
+                {'ext': determine_ext(caption['source'], default_ext='vtt'),
+                 'url': caption['source']}]
+
+        return {
+            'id': video_id,
+            'title': title,
+            'creator': creator,
+            'thumbnail': thumbnail,
+            'formats': formats,
+            'upload_date': upload_date,
+            'subtitles': subtitles,
+        }

From 03bc7237add1747de4c0c5d09e72e03639b4fd21 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 20 Aug 2015 23:18:58 +0800
Subject: [PATCH 2306/6219] [common] _parse_smil_subtitles: accept `lang` as
 the subtitle language

---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index ce2030d28..999afc110 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1151,7 +1151,7 @@ class InfoExtractor(object):
                 }
                 if type_ in SUBTITLES_TYPES:
                     ext = SUBTITLES_TYPES[type_]
-            lang = textstream.get('systemLanguage') or textstream.get('systemLanguageName') or subtitles_lang
+            lang = textstream.get('systemLanguage') or textstream.get('systemLanguageName') or textstream.get('lang') or subtitles_lang
             subtitles.setdefault(lang, []).append({
                 'url': src,
                 'ext': ext,

From 912e0b7e46d795df3ec1866f9b0ff071cca8d550 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 21 Aug 2015 01:37:07 +0800
Subject: [PATCH 2307/6219] [common] Add _merge_subtitles()

---
 youtube_dl/extractor/common.py | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 999afc110..b7437af5a 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1279,6 +1279,26 @@ class InfoExtractor(object):
     def _get_subtitles(self, *args, **kwargs):
         raise NotImplementedError("This method must be implemented by subclasses")
 
+    @staticmethod
+    def _merge_subtitle_items(subtitle_list1, subtitle_list2):
+        """ Merge subtitle items for one language. Items with duplicated URLs
+        will be dropped. """
+        list1_urls = set([item['url'] for item in subtitle_list1])
+        ret = list(subtitle_list1)
+        ret.extend([item for item in subtitle_list2 if item['url'] not in list1_urls])
+        return ret
+
+    @classmethod
+    def _merge_subtitles(kls, subtitle_dict1, subtitle_dict2):
+        """ Merge two subtitle dictionaries, language by language. """
+        print(subtitle_dict1)
+        print(subtitle_dict2)
+        ret = dict(subtitle_dict1)
+        for lang in subtitle_dict2:
+            ret[lang] = kls._merge_subtitle_items(subtitle_dict1.get(lang, []), subtitle_dict2[lang])
+        print(ret)
+        return ret
+
     def extract_automatic_captions(self, *args, **kwargs):
         if (self._downloader.params.get('writeautomaticsub', False) or
                 self._downloader.params.get('listsubtitles')):

From c687ac745b3c94b2fd246214e78c92a31bd9fc0f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 21 Aug 2015 01:37:43 +0800
Subject: [PATCH 2308/6219] [theplatform] Use subtitles from SMIL, too

---
 youtube_dl/extractor/theplatform.py | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index ba05ada39..25edc3100 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -28,7 +28,7 @@ _x = lambda p: xpath_with_ns(p, {'smil': default_ns})
 
 
 class ThePlatformBaseIE(InfoExtractor):
-    def _extract_theplatform_smil_formats(self, smil_url, video_id, note='Downloading SMIL data'):
+    def _extract_theplatform_smil(self, smil_url, video_id, note='Downloading SMIL data'):
         meta = self._download_xml(smil_url, video_id, note=note)
         try:
             error_msg = next(
@@ -54,7 +54,9 @@ class ThePlatformBaseIE(InfoExtractor):
 
         self._sort_formats(formats)
 
-        return formats
+        subtitles = self._parse_smil_subtitles(meta, default_ns)
+
+        return formats, subtitles
 
     def get_metadata(self, path, video_id):
         info_url = 'http://link.theplatform.com/s/%s?format=preview' % path
@@ -208,12 +210,14 @@ class ThePlatformIE(ThePlatformBaseIE):
         if sig:
             smil_url = self._sign_url(smil_url, sig['key'], sig['secret'])
 
-        formats = self._extract_theplatform_smil_formats(smil_url, video_id)
+        formats, subtitles = self._extract_theplatform_smil(smil_url, video_id)
 
         ret = self.get_metadata(path, video_id)
+        combined_subtitles = self._merge_subtitles(ret.get('subtitles', {}), subtitles)
         ret.update({
             'id': video_id,
             'formats': formats,
+            'subtitles': combined_subtitles,
         })
 
         return ret
@@ -251,6 +255,7 @@ class ThePlatformFeedIE(ThePlatformBaseIE):
         entry = feed['entries'][0]
 
         formats = []
+        subtitles = {}
         first_video_id = None
         duration = None
         for item in entry['media$content']:
@@ -259,7 +264,9 @@ class ThePlatformFeedIE(ThePlatformBaseIE):
             if first_video_id is None:
                 first_video_id = cur_video_id
                 duration = float_or_none(item.get('plfile$duration'))
-            formats.extend(self._extract_theplatform_smil_formats(smil_url, video_id, 'Downloading SMIL data for %s' % cur_video_id))
+            cur_formats, cur_subtitles = self._extract_theplatform_smil(smil_url, video_id, 'Downloading SMIL data for %s' % cur_video_id)
+            formats.extend(cur_formats)
+            subtitles = self._merge_subtitles(subtitles, cur_subtitles)
 
         self._sort_formats(formats)
 
@@ -273,9 +280,11 @@ class ThePlatformFeedIE(ThePlatformBaseIE):
         categories = [item['media$name'] for item in entry.get('media$categories', [])]
 
         ret = self.get_metadata('%s/%s' % (provider_id, first_video_id), video_id)
+        subtitles = self._merge_subtitles(subtitles, ret['subtitles'])
         ret.update({
             'id': video_id,
             'formats': formats,
+            'subtitles': subtitles,
             'thumbnails': thumbnails,
             'duration': duration,
             'timestamp': timestamp,

From f908b74fa38b2678e26aea128dbd934cd781a9b6 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 21 Aug 2015 01:38:57 +0800
Subject: [PATCH 2309/6219] [test/subtitles] Add test for ThePlatformFeedIE

---
 test/test_subtitles.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/test/test_subtitles.py b/test/test_subtitles.py
index c4e3adb67..0343967d9 100644
--- a/test/test_subtitles.py
+++ b/test/test_subtitles.py
@@ -25,6 +25,7 @@ from youtube_dl.extractor import (
     RaiIE,
     VikiIE,
     ThePlatformIE,
+    ThePlatformFeedIE,
     RTVEALaCartaIE,
     FunnyOrDieIE,
 )
@@ -307,6 +308,18 @@ class TestThePlatformSubtitles(BaseTestSubtitles):
         self.assertEqual(md5(subtitles['en']), '97e7670cbae3c4d26ae8bcc7fdd78d4b')
 
 
+class TestThePlatformFeedSubtitles(BaseTestSubtitles):
+    url = 'http://feed.theplatform.com/f/7wvmTC/msnbc_video-p-test?form=json&pretty=true&range=-40&byGuid=n_hardball_5biden_140207'
+    IE = ThePlatformFeedIE
+
+    def test_allsubtitles(self):
+        self.DL.params['writesubtitles'] = True
+        self.DL.params['allsubtitles'] = True
+        subtitles = self.getSubtitles()
+        self.assertEqual(set(subtitles.keys()), set(['en']))
+        self.assertEqual(md5(subtitles['en']), '48649a22e82b2da21c9a67a395eedade')
+
+
 class TestRtveSubtitles(BaseTestSubtitles):
     url = 'http://www.rtve.es/alacarta/videos/los-misterios-de-laura/misterios-laura-capitulo-32-misterio-del-numero-17-2-parte/2428621/'
     IE = RTVEALaCartaIE

From f738dd7b7c7aefe4d26a65905dee9567a691d262 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 21 Aug 2015 01:43:22 +0800
Subject: [PATCH 2310/6219] [common] Remove debugging codes

---
 youtube_dl/extractor/common.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index b7437af5a..f731703fb 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1291,12 +1291,9 @@ class InfoExtractor(object):
     @classmethod
     def _merge_subtitles(kls, subtitle_dict1, subtitle_dict2):
         """ Merge two subtitle dictionaries, language by language. """
-        print(subtitle_dict1)
-        print(subtitle_dict2)
         ret = dict(subtitle_dict1)
         for lang in subtitle_dict2:
             ret[lang] = kls._merge_subtitle_items(subtitle_dict1.get(lang, []), subtitle_dict2[lang])
-        print(ret)
         return ret
 
     def extract_automatic_captions(self, *args, **kwargs):

From dc95bd503e82d3eb04c347ac0cdbcbabd7e14552 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 21 Aug 2015 08:54:28 +0600
Subject: [PATCH 2311/6219] [folketinget] Add skip_download for test

---
 youtube_dl/extractor/folketinget.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/folketinget.py b/youtube_dl/extractor/folketinget.py
index 0fb29de75..75399fa7d 100644
--- a/youtube_dl/extractor/folketinget.py
+++ b/youtube_dl/extractor/folketinget.py
@@ -30,6 +30,10 @@ class FolketingetIE(InfoExtractor):
             'upload_date': '20141120',
             'duration': 3960,
         },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        },
     }
 
     def _real_extract(self, url):

From 5d003e29b188dd2f140fe1b9b93f1bb1ad8263a0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 21 Aug 2015 08:56:05 +0600
Subject: [PATCH 2312/6219] [rtp] Add skip_download for test

---
 youtube_dl/extractor/rtp.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/rtp.py b/youtube_dl/extractor/rtp.py
index ecf4939cd..82b323cdd 100644
--- a/youtube_dl/extractor/rtp.py
+++ b/youtube_dl/extractor/rtp.py
@@ -18,6 +18,10 @@ class RTPIE(InfoExtractor):
             'description': 'As paixões musicais de António Cartaxo e António Macedo',
             'thumbnail': 're:^https?://.*\.jpg',
         },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        },
     }, {
         'url': 'http://www.rtp.pt/play/p831/a-quimica-das-coisas',
         'only_matching': True,

From 4932a817a0c2375df14d66c9ac86cfa28988327d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 21 Aug 2015 13:00:08 +0800
Subject: [PATCH 2313/6219] [rtl2] Add skip_download for test

---
 youtube_dl/extractor/rtl2.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/rtl2.py b/youtube_dl/extractor/rtl2.py
index 72cd80498..e9589449e 100644
--- a/youtube_dl/extractor/rtl2.py
+++ b/youtube_dl/extractor/rtl2.py
@@ -15,6 +15,10 @@ class RTL2IE(InfoExtractor):
             'title': 'GRIP sucht den Sommerkönig',
             'description': 'Matthias, Det und Helge treten gegeneinander an.'
         },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        },
     }, {
         'url': 'http://www.rtl2.de/sendung/koeln-50667/video/5512-anna/21040-anna-erwischt-alex/',
         'md5': 'ffcd517d2805b57ce11a58a2980c2b02',

From 9eb4ab6ad915a777b6f7d7b39d03d05d7d31cd24 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 21 Aug 2015 13:04:25 +0800
Subject: [PATCH 2314/6219] [rtl2] Remove an unused line

---
 youtube_dl/extractor/rtl2.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/rtl2.py b/youtube_dl/extractor/rtl2.py
index e9589449e..9e0c6890e 100644
--- a/youtube_dl/extractor/rtl2.py
+++ b/youtube_dl/extractor/rtl2.py
@@ -43,7 +43,6 @@ class RTL2IE(InfoExtractor):
         vivi_id = self._html_search_regex(
             r'vivi_id\s*:\s*([0-9]+)', webpage, 'vivi_id')
         info_url = 'http://www.rtl2.de/video/php/get_video.php?vico_id=' + vico_id + '&vivi_id=' + vivi_id
-        webpage = self._download_webpage(info_url, '')
 
         info = self._download_json(info_url, video_id)
         video_info = info['video']

From 5e1a5ac8de12391cb22d2fa0dfb2119527bd7fc2 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 21 Aug 2015 13:20:32 +0800
Subject: [PATCH 2315/6219] [rtl2] Fix extraction for test_RTL2_1

---
 youtube_dl/extractor/rtl2.py | 20 ++++++++++++++++----
 1 file changed, 16 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/rtl2.py b/youtube_dl/extractor/rtl2.py
index 9e0c6890e..276612fc7 100644
--- a/youtube_dl/extractor/rtl2.py
+++ b/youtube_dl/extractor/rtl2.py
@@ -1,6 +1,7 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
+import re
 from .common import InfoExtractor
 
 
@@ -28,6 +29,10 @@ class RTL2IE(InfoExtractor):
             'title': 'Anna erwischt Alex!',
             'description': 'Anna ist Alex\' Tochter bei Köln 50667.'
         },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        },
     }]
 
     def _real_extract(self, url):
@@ -38,10 +43,17 @@ class RTL2IE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        vico_id = self._html_search_regex(
-            r'vico_id\s*:\s*([0-9]+)', webpage, 'vico_id')
-        vivi_id = self._html_search_regex(
-            r'vivi_id\s*:\s*([0-9]+)', webpage, 'vivi_id')
+        mobj = re.search(
+            r'<div[^>]+data-collection="(?P<vico_id>\d+)"[^>]+data-video="(?P<vivi_id>\d+)"',
+            webpage)
+        if mobj:
+            vico_id = mobj.group('vico_id')
+            vivi_id = mobj.group('vivi_id')
+        else:
+            vico_id = self._html_search_regex(
+                r'vico_id\s*:\s*([0-9]+)', webpage, 'vico_id')
+            vivi_id = self._html_search_regex(
+                r'vivi_id\s*:\s*([0-9]+)', webpage, 'vivi_id')
         info_url = 'http://www.rtl2.de/video/php/get_video.php?vico_id=' + vico_id + '&vivi_id=' + vivi_id
 
         info = self._download_json(info_url, video_id)

From d7c16305707f2af5c47d91b67cf0850b4dcada3a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 21 Aug 2015 13:21:21 +0800
Subject: [PATCH 2316/6219] [rtl2] Remove MD5 checksums

---
 youtube_dl/extractor/rtl2.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/youtube_dl/extractor/rtl2.py b/youtube_dl/extractor/rtl2.py
index 276612fc7..25f7faf76 100644
--- a/youtube_dl/extractor/rtl2.py
+++ b/youtube_dl/extractor/rtl2.py
@@ -9,7 +9,6 @@ class RTL2IE(InfoExtractor):
     _VALID_URL = r'http?://(?:www\.)?rtl2\.de/[^?#]*?/(?P<id>[^?#/]*?)(?:$|/(?:$|[?#]))'
     _TESTS = [{
         'url': 'http://www.rtl2.de/sendung/grip-das-motormagazin/folge/folge-203-0',
-        'md5': 'bfcc179030535b08dc2b36b469b5adc7',
         'info_dict': {
             'id': 'folge-203-0',
             'ext': 'f4v',
@@ -22,7 +21,6 @@ class RTL2IE(InfoExtractor):
         },
     }, {
         'url': 'http://www.rtl2.de/sendung/koeln-50667/video/5512-anna/21040-anna-erwischt-alex/',
-        'md5': 'ffcd517d2805b57ce11a58a2980c2b02',
         'info_dict': {
             'id': '21040-anna-erwischt-alex',
             'ext': 'mp4',

From 8c97f81943de1c2bf8d2f524ba5ca09b29579dc8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 21 Aug 2015 11:35:51 +0200
Subject: [PATCH 2317/6219] [common] Follow convention of using 'cls' in
 classmethods

---
 youtube_dl/extractor/common.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index f731703fb..5d24bcb6a 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1289,11 +1289,11 @@ class InfoExtractor(object):
         return ret
 
     @classmethod
-    def _merge_subtitles(kls, subtitle_dict1, subtitle_dict2):
+    def _merge_subtitles(cls, subtitle_dict1, subtitle_dict2):
         """ Merge two subtitle dictionaries, language by language. """
         ret = dict(subtitle_dict1)
         for lang in subtitle_dict2:
-            ret[lang] = kls._merge_subtitle_items(subtitle_dict1.get(lang, []), subtitle_dict2[lang])
+            ret[lang] = cls._merge_subtitle_items(subtitle_dict1.get(lang, []), subtitle_dict2[lang])
         return ret
 
     def extract_automatic_captions(self, *args, **kwargs):

From 66ce97024d0de7836777562a6eb60603796636d5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 22 Aug 2015 06:30:00 +0600
Subject: [PATCH 2318/6219] [soundcloud:user] Update tests

---
 youtube_dl/extractor/soundcloud.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 6ce86cbcd..ed5dcc0d3 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -309,7 +309,7 @@ class SoundcloudUserIE(SoundcloudIE):
             'id': '114582580',
             'title': 'The Akashic Chronicler (All)',
         },
-        'playlist_mincount': 112,
+        'playlist_mincount': 111,
     }, {
         'url': 'https://soundcloud.com/the-akashic-chronicler/tracks',
         'info_dict': {
@@ -330,14 +330,14 @@ class SoundcloudUserIE(SoundcloudIE):
             'id': '114582580',
             'title': 'The Akashic Chronicler (Reposts)',
         },
-        'playlist_mincount': 9,
+        'playlist_mincount': 7,
     }, {
         'url': 'https://soundcloud.com/the-akashic-chronicler/likes',
         'info_dict': {
             'id': '114582580',
             'title': 'The Akashic Chronicler (Likes)',
         },
-        'playlist_mincount': 333,
+        'playlist_mincount': 321,
     }, {
         'url': 'https://soundcloud.com/grynpyret/spotlight',
         'info_dict': {

From 483fc223bb1509d11ac1843a5852f75c0aec3475 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 23 Aug 2015 10:42:34 +0600
Subject: [PATCH 2319/6219] [pluralsight] Add extractor (Closes #6090)

---
 youtube_dl/extractor/__init__.py    |   4 +
 youtube_dl/extractor/pluralsight.py | 218 ++++++++++++++++++++++++++++
 2 files changed, 222 insertions(+)
 create mode 100644 youtube_dl/extractor/pluralsight.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 1c53a5632..d59882598 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -454,6 +454,10 @@ from .playfm import PlayFMIE
 from .playtvak import PlaytvakIE
 from .playvid import PlayvidIE
 from .playwire import PlaywireIE
+from .pluralsight import (
+    PluralsightIE,
+    PluralsightCourseIE,
+)
 from .podomatic import PodomaticIE
 from .porn91 import Porn91IE
 from .pornhd import PornHdIE
diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
new file mode 100644
index 000000000..1bdcacbaa
--- /dev/null
+++ b/youtube_dl/extractor/pluralsight.py
@@ -0,0 +1,218 @@
+from __future__ import unicode_literals
+
+import re
+import json
+
+from .common import InfoExtractor
+from ..compat import (
+    compat_str,
+    compat_urllib_parse,
+    compat_urllib_request,
+    compat_urlparse,
+)
+from ..utils import (
+    ExtractorError,
+    int_or_none,
+    parse_duration,
+)
+
+
+class PluralsightIE(InfoExtractor):
+    IE_NAME = 'pluralsight'
+    _VALID_URL = r'https?://(?:www\.)?pluralsight\.com/training/player\?author=(?P<author>[^&]+)&name=(?P<name>[^&]+)(?:&mode=live)?&clip=(?P<clip>\d+)&course=(?P<course>[^&]+)'
+    _LOGIN_URL = 'https://www.pluralsight.com/id/'
+    _ACCOUNT_CREDENTIALS_HINT = 'Use --username and --password options to provide lynda.com account credentials.'
+    _NETRC_MACHINE = 'pluralsight'
+
+    _TEST = {
+        'url': 'http://www.pluralsight.com/training/player?author=mike-mckeown&name=hosting-sql-server-windows-azure-iaas-m7-mgmt&mode=live&clip=3&course=hosting-sql-server-windows-azure-iaas',
+        'md5': '4d458cf5cf4c593788672419a8dd4cf8',
+        'info_dict': {
+            'id': 'hosting-sql-server-windows-azure-iaas-m7-mgmt-04',
+            'ext': 'mp4',
+            'title': 'Management of SQL Server - Demo Monitoring',
+            'duration': 338,
+        },
+        'skip': 'Requires pluralsight account credentials',
+    }
+
+    def _real_initialize(self):
+        self._login()
+
+    def _login(self):
+        (username, password) = self._get_login_info()
+        if username is None:
+            raise ExtractorError(
+                'Pluralsight account is required, use --username and --password options to provide account credentials.',
+                expected=True)
+
+        login_page = self._download_webpage(
+            self._LOGIN_URL, None, 'Downloading login page')
+
+        login_form = self._hidden_inputs(login_page)
+
+        login_form.update({
+            'Username': username.encode('utf-8'),
+            'Password': password.encode('utf-8'),
+        })
+
+        post_url = self._search_regex(
+            r'<form[^>]+action=(["\'])(?P<url>.+?)\1', login_page,
+            'post url', default=self._LOGIN_URL, group='url')
+
+        if not post_url.startswith('http'):
+            post_url = compat_urlparse.urljoin(self._LOGIN_URL, post_url)
+
+        request = compat_urllib_request.Request(
+            post_url, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
+        request.add_header('Content-Type', 'application/x-www-form-urlencoded')
+
+        response = self._download_webpage(
+            request, None, 'Logging in as %s' % username)
+
+        error = self._search_regex(
+            r'<span[^>]+class="field-validation-error"[^>]*>([^<]+)</span>',
+            response, 'error message', default=None)
+        if error:
+            raise ExtractorError('Unable to login: %s' % error, expected=True)
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        author = mobj.group('author')
+        name = mobj.group('name')
+        clip_id = mobj.group('clip')
+        course = mobj.group('course')
+
+        display_id = '%s-%s' % (name, clip_id)
+
+        webpage = self._download_webpage(url, display_id)
+
+        collection = self._parse_json(
+            self._search_regex(
+                r'moduleCollection\s*:\s*new\s+ModuleCollection\((\[.+?\])\s*,\s*\$rootScope\)',
+                webpage, 'modules'),
+            display_id)
+
+        module, clip = None, None
+
+        for module_ in collection:
+            if module_.get('moduleName') == name:
+                module = module_
+                for clip_ in module_.get('clips', []):
+                    clip_index = clip_.get('clipIndex')
+                    if clip_index is None:
+                        continue
+                    if compat_str(clip_index) == clip_id:
+                        clip = clip_
+                        break
+
+        if not clip:
+            raise ExtractorError('Unable to resolve clip')
+
+        QUALITIES = {
+            'low': {'width': 640, 'height': 480},
+            'medium': {'width': 848, 'height': 640},
+            'high': {'width': 1024, 'height': 768},
+        }
+
+        ALLOWED_QUALITIES = (
+            ('webm', ('high',)),
+            ('mp4', ('low', 'medium', 'high',)),
+        )
+
+        formats = []
+        for ext, qualities in ALLOWED_QUALITIES:
+            for quality in qualities:
+                f = QUALITIES[quality].copy()
+                clip_post = {
+                    'a': author,
+                    'cap': 'false',
+                    'cn': clip_id,
+                    'course': course,
+                    'lc': 'en',
+                    'm': name,
+                    'mt': ext,
+                    'q': '%dx%d' % (f['width'], f['height']),
+                }
+                request = compat_urllib_request.Request(
+                    'http://www.pluralsight.com/training/Player/ViewClip',
+                    json.dumps(clip_post).encode('utf-8'))
+                request.add_header('Content-Type', 'application/json;charset=utf-8')
+                format_id = '%s-%s' % (ext, quality)
+                clip_url = self._download_webpage(
+                    request, display_id, 'Downloading %s URL' % format_id, fatal=False)
+                if not clip_url:
+                    continue
+                f.update({
+                    'url': clip_url,
+                    'ext': ext,
+                    'format_id': format_id,
+                })
+                formats.append(f)
+        self._sort_formats(formats)
+
+        # TODO: captions
+        # http://www.pluralsight.com/training/Player/ViewClip + cap = true
+        # or
+        # http://www.pluralsight.com/training/Player/Captions
+        # { a = author, cn = clip_id, lc = end, m = name }
+
+        return {
+            'id': clip['clipName'],
+            'title': '%s - %s' % (module['title'], clip['title']),
+            'duration': int_or_none(clip.get('duration')) or parse_duration(clip.get('formattedDuration')),
+            'creator': author,
+            'formats': formats
+        }
+
+
+class PluralsightCourseIE(InfoExtractor):
+    IE_NAME = 'pluralsight:course'
+    _VALID_URL = r'https?://(?:www\.)?pluralsight\.com/courses/(?P<id>[^/]+)'
+    _TEST = {
+        # Free course from Pluralsight Starter Subscription for Microsoft TechNet
+        # https://offers.pluralsight.com/technet?loc=zTS3z&prod=zOTprodz&tech=zOttechz&prog=zOTprogz&type=zSOz&media=zOTmediaz&country=zUSz
+        'url': 'http://www.pluralsight.com/courses/hosting-sql-server-windows-azure-iaas',
+        'info_dict': {
+            'id': 'hosting-sql-server-windows-azure-iaas',
+            'title': 'Hosting SQL Server in Microsoft Azure IaaS Fundamentals',
+            'description': 'md5:61b37e60f21c4b2f91dc621a977d0986',
+        },
+        'playlist_count': 31,
+    }
+
+    def _real_extract(self, url):
+        course_id = self._match_id(url)
+
+        course = self._download_json(
+            'http://www.pluralsight.com/data/course/%s' % course_id,
+            course_id, 'Downloading course JSON')
+
+        title = course['title']
+        description = course.get('description') or course.get('shortDescription')
+
+        course_data = self._download_json(
+            'http://www.pluralsight.com/data/course/content/%s' % course_id,
+            course_id, 'Downloading course data JSON')
+
+        may_not_view = 0
+
+        entries = []
+        for module in course_data:
+            for clip in module.get('clips', []):
+                if clip.get('userMayViewClip') is False:
+                    may_not_view += 1
+                    continue
+                player_parameters = clip.get('playerParameters')
+                if not player_parameters:
+                    continue
+                entries.append(self.url_result(
+                    'http://www.pluralsight.com/training/player?%s' % player_parameters,
+                    'Pluralsight'))
+
+        if may_not_view > 0:
+            self._downloader.report_warning(
+                'There are %d videos in this course that are not available for you. '
+                'Upgrade your account to get access to these videos.' % may_not_view)
+
+        return self.playlist_result(entries, course_id, title, description)

From 468083d2f5596314a0813859f3afe7d2fce3cac7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 23 Aug 2015 10:44:10 +0600
Subject: [PATCH 2320/6219] [pluralsight] Remove unused const

---
 youtube_dl/extractor/pluralsight.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index 1bdcacbaa..7c7f762ff 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -21,7 +21,6 @@ class PluralsightIE(InfoExtractor):
     IE_NAME = 'pluralsight'
     _VALID_URL = r'https?://(?:www\.)?pluralsight\.com/training/player\?author=(?P<author>[^&]+)&name=(?P<name>[^&]+)(?:&mode=live)?&clip=(?P<clip>\d+)&course=(?P<course>[^&]+)'
     _LOGIN_URL = 'https://www.pluralsight.com/id/'
-    _ACCOUNT_CREDENTIALS_HINT = 'Use --username and --password options to provide lynda.com account credentials.'
     _NETRC_MACHINE = 'pluralsight'
 
     _TEST = {

From 2b6bda1ed86e1b64242b33c032286dc315d541ae Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 23 Aug 2015 11:21:56 +0600
Subject: [PATCH 2321/6219] [pluralsight] Do not yet rely on userMayViewClip

---
 youtube_dl/extractor/pluralsight.py | 12 ++----------
 1 file changed, 2 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index 7c7f762ff..7ba396aef 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -183,6 +183,8 @@ class PluralsightCourseIE(InfoExtractor):
     def _real_extract(self, url):
         course_id = self._match_id(url)
 
+        # TODO: PSM cookie
+
         course = self._download_json(
             'http://www.pluralsight.com/data/course/%s' % course_id,
             course_id, 'Downloading course JSON')
@@ -194,14 +196,9 @@ class PluralsightCourseIE(InfoExtractor):
             'http://www.pluralsight.com/data/course/content/%s' % course_id,
             course_id, 'Downloading course data JSON')
 
-        may_not_view = 0
-
         entries = []
         for module in course_data:
             for clip in module.get('clips', []):
-                if clip.get('userMayViewClip') is False:
-                    may_not_view += 1
-                    continue
                 player_parameters = clip.get('playerParameters')
                 if not player_parameters:
                     continue
@@ -209,9 +206,4 @@ class PluralsightCourseIE(InfoExtractor):
                     'http://www.pluralsight.com/training/player?%s' % player_parameters,
                     'Pluralsight'))
 
-        if may_not_view > 0:
-            self._downloader.report_warning(
-                'There are %d videos in this course that are not available for you. '
-                'Upgrade your account to get access to these videos.' % may_not_view)
-
         return self.playlist_result(entries, course_id, title, description)

From 2006a06eff606c5a996c315a3e597b9d2603db9e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 23 Aug 2015 21:43:28 +0600
Subject: [PATCH 2322/6219] [kontrtube] Fix extraction (Closes #6644)

---
 youtube_dl/extractor/kontrtube.py | 40 +++++++++++++++----------------
 1 file changed, 19 insertions(+), 21 deletions(-)

diff --git a/youtube_dl/extractor/kontrtube.py b/youtube_dl/extractor/kontrtube.py
index 720bc939b..a59c529f4 100644
--- a/youtube_dl/extractor/kontrtube.py
+++ b/youtube_dl/extractor/kontrtube.py
@@ -4,7 +4,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import int_or_none
+from ..utils import (
+    int_or_none,
+    parse_duration,
+)
 
 
 class KontrTubeIE(InfoExtractor):
@@ -34,33 +37,28 @@ class KontrTubeIE(InfoExtractor):
         webpage = self._download_webpage(
             url, display_id, 'Downloading page')
 
-        video_url = self._html_search_regex(
+        video_url = self._search_regex(
             r"video_url\s*:\s*'(.+?)/?',", webpage, 'video URL')
-        thumbnail = self._html_search_regex(
-            r"preview_url\s*:\s*'(.+?)/?',", webpage, 'video thumbnail', fatal=False)
+        thumbnail = self._search_regex(
+            r"preview_url\s*:\s*'(.+?)/?',", webpage, 'thumbnail', fatal=False)
         title = self._html_search_regex(
-            r'<title>(.+?)</title>', webpage, 'video title')
+            r'(?s)<h2>(.+?)</h2>', webpage, 'title')
         description = self._html_search_meta(
-            'description', webpage, 'video description')
+            'description', webpage, 'description')
 
-        mobj = re.search(
-            r'<div class="col_2">Длительность: <span>(?P<minutes>\d+)м:(?P<seconds>\d+)с</span></div>',
-            webpage)
-        duration = int(mobj.group('minutes')) * 60 + int(mobj.group('seconds')) if mobj else None
+        duration = self._search_regex(
+            r'Длительность: <em>([^<]+)</em>', webpage, 'duration', fatal=False)
+        if duration:
+            duration = parse_duration(duration.replace('мин', 'min').replace('сек', 'sec'))
 
-        view_count = self._html_search_regex(
-            r'<div class="col_2">Просмотров: <span>(\d+)</span></div>',
+        view_count = self._search_regex(
+            r'Просмотров: <em>([^<]+)</em>',
             webpage, 'view count', fatal=False)
+        if view_count:
+            view_count = int_or_none(view_count.replace(' ', ''))
 
-        comment_count = None
-        comment_str = self._html_search_regex(
-            r'Комментарии: <span>([^<]+)</span>', webpage, 'comment count', fatal=False)
-        if comment_str.startswith('комментариев нет'):
-            comment_count = 0
-        else:
-            mobj = re.search(r'\d+ из (?P<total>\d+) комментариев', comment_str)
-            if mobj:
-                comment_count = mobj.group('total')
+        comment_count = int_or_none(self._search_regex(
+            r'Комментарии \((\d+)\)<', webpage, ' comment count', fatal=False))
 
         return {
             'id': video_id,

From 9990c960f2d944cfbecb7d613062b98fe99464a7 Mon Sep 17 00:00:00 2001
From: clauderains <clauderains3319@gmail.com>
Date: Sun, 23 Aug 2015 02:46:29 -0700
Subject: [PATCH 2323/6219] [spankwire] Fixed uploader_id field extraction so
 that test case passes

---
 youtube_dl/extractor/spankwire.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/spankwire.py b/youtube_dl/extractor/spankwire.py
index 5fa6faf18..0a35c2b3b 100644
--- a/youtube_dl/extractor/spankwire.py
+++ b/youtube_dl/extractor/spankwire.py
@@ -54,7 +54,7 @@ class SpankwireIE(InfoExtractor):
             r'by:\s*<a [^>]*>(.+?)</a>',
             webpage, 'uploader', fatal=False)
         uploader_id = self._html_search_regex(
-            r'by:\s*<a href="/Profile\.aspx\?.*?UserId=(\d+).*?"',
+            r'by:\s*<a href="/user/viewProfile\?.*?UserId=(\d+).*?"',
             webpage, 'uploader id', fatal=False)
         upload_date = unified_strdate(self._html_search_regex(
             r'</a> on (.+?) at \d+:\d+',

From 59e6acc757a9df85ca78e519623e84072ffd9c01 Mon Sep 17 00:00:00 2001
From: clauderains <clauderains3319@gmail.com>
Date: Sun, 23 Aug 2015 02:47:20 -0700
Subject: [PATCH 2324/6219] [spankwire] Support new cdn video url format

---
 youtube_dl/extractor/spankwire.py | 84 +++++++++++++++++++++----------
 1 file changed, 58 insertions(+), 26 deletions(-)

diff --git a/youtube_dl/extractor/spankwire.py b/youtube_dl/extractor/spankwire.py
index 0a35c2b3b..0f2d8d0de 100644
--- a/youtube_dl/extractor/spankwire.py
+++ b/youtube_dl/extractor/spankwire.py
@@ -17,20 +17,34 @@ from ..aes import aes_decrypt_text
 
 class SpankwireIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?(?P<url>spankwire\.com/[^/]*/video(?P<videoid>[0-9]+)/?)'
-    _TEST = {
-        'url': 'http://www.spankwire.com/Buckcherry-s-X-Rated-Music-Video-Crazy-Bitch/video103545/',
-        'md5': '8bbfde12b101204b39e4b9fe7eb67095',
-        'info_dict': {
-            'id': '103545',
-            'ext': 'mp4',
-            'title': 'Buckcherry`s X Rated Music Video Crazy Bitch',
-            'description': 'Crazy Bitch X rated music video.',
-            'uploader': 'oreusz',
-            'uploader_id': '124697',
-            'upload_date': '20070507',
-            'age_limit': 18,
-        }
-    }
+    _TESTS = [{
+                'url': 'http://www.spankwire.com/Buckcherry-s-X-Rated-Music-Video-Crazy-Bitch/video103545/',
+                'md5': '8bbfde12b101204b39e4b9fe7eb67095',
+                    'info_dict': {
+                                    'id': '103545',
+                                    'ext': 'mp4',
+                                    'title': 'Buckcherry`s X Rated Music Video Crazy Bitch',
+                                    'description': 'Crazy Bitch X rated music video.',
+                                    'uploader': 'oreusz',
+                                    'uploader_id': '124697',
+                                    'upload_date': '20070507',
+                                    'age_limit': 18,
+                    }
+              },
+              {
+                'url': 'http://www.spankwire.com/Titcums-Compiloation-I/video1921551/',
+                'md5': '09b3c20833308b736ae8902db2f8d7e6',
+                    'info_dict': {
+                                    'id': '1921551',
+                                    'ext': 'mp4',
+                                    'title': 'Titcums Compiloation I',
+                                    'description': 'cum on tits',
+                                    'uploader': 'dannyh78999',
+                                    'uploader_id': '3056053',
+                                    'upload_date': '20150822',
+                                    'age_limit': 18,
+                    }
+              }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
@@ -82,18 +96,36 @@ class SpankwireIE(InfoExtractor):
         for video_url in video_urls:
             path = compat_urllib_parse_urlparse(video_url).path
             format = path.split('/')[4].split('_')[:2]
-            resolution, bitrate_str = format
-            format = "-".join(format)
-            height = int(resolution.rstrip('Pp'))
-            tbr = int(bitrate_str.rstrip('Kk'))
-            formats.append({
-                'url': video_url,
-                'resolution': resolution,
-                'format': format,
-                'tbr': tbr,
-                'height': height,
-                'format_id': format,
-            })
+            if format[0] == 'mp4':
+                format_id, quality = format
+                format = "-".join(format)
+                if quality == 'normal':
+                    height = 180
+                elif quality == 'high':
+                    height = 240
+                elif quality == 'ultra':
+                    height = 480
+                elif quality == '720p':
+                    height = 720
+                formats.append({
+                    'url': video_url,
+                    'format': format,
+                    'height': height,
+                    'format_id': format,
+                })
+            else:
+                resolution, bitrate_str = format
+                format = "-".join(format)
+                height = int(resolution.rstrip('Pp'))
+                tbr = int(bitrate_str.rstrip('Kk'))
+                formats.append({
+                    'url': video_url,
+                    'resolution': resolution,
+                    'format': format,
+                    'tbr': tbr,
+                    'height': height,
+                    'format_id': format,
+                })
         self._sort_formats(formats)
 
         age_limit = self._rta_search(webpage)

From 551c7837ace81190ce9141551dceec24dfdae1bc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 23 Aug 2015 22:32:20 +0600
Subject: [PATCH 2325/6219] [spankwire] Simplify and properly format

---
 youtube_dl/extractor/spankwire.py | 105 +++++++++++++-----------------
 1 file changed, 45 insertions(+), 60 deletions(-)

diff --git a/youtube_dl/extractor/spankwire.py b/youtube_dl/extractor/spankwire.py
index 0f2d8d0de..0a47441b1 100644
--- a/youtube_dl/extractor/spankwire.py
+++ b/youtube_dl/extractor/spankwire.py
@@ -18,33 +18,34 @@ from ..aes import aes_decrypt_text
 class SpankwireIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?(?P<url>spankwire\.com/[^/]*/video(?P<videoid>[0-9]+)/?)'
     _TESTS = [{
-                'url': 'http://www.spankwire.com/Buckcherry-s-X-Rated-Music-Video-Crazy-Bitch/video103545/',
-                'md5': '8bbfde12b101204b39e4b9fe7eb67095',
-                    'info_dict': {
-                                    'id': '103545',
-                                    'ext': 'mp4',
-                                    'title': 'Buckcherry`s X Rated Music Video Crazy Bitch',
-                                    'description': 'Crazy Bitch X rated music video.',
-                                    'uploader': 'oreusz',
-                                    'uploader_id': '124697',
-                                    'upload_date': '20070507',
-                                    'age_limit': 18,
-                    }
-              },
-              {
-                'url': 'http://www.spankwire.com/Titcums-Compiloation-I/video1921551/',
-                'md5': '09b3c20833308b736ae8902db2f8d7e6',
-                    'info_dict': {
-                                    'id': '1921551',
-                                    'ext': 'mp4',
-                                    'title': 'Titcums Compiloation I',
-                                    'description': 'cum on tits',
-                                    'uploader': 'dannyh78999',
-                                    'uploader_id': '3056053',
-                                    'upload_date': '20150822',
-                                    'age_limit': 18,
-                    }
-              }]
+        # download URL pattern: */<height>P_<tbr>K_<video_id>.mp4
+        'url': 'http://www.spankwire.com/Buckcherry-s-X-Rated-Music-Video-Crazy-Bitch/video103545/',
+        'md5': '8bbfde12b101204b39e4b9fe7eb67095',
+        'info_dict': {
+            'id': '103545',
+            'ext': 'mp4',
+            'title': 'Buckcherry`s X Rated Music Video Crazy Bitch',
+            'description': 'Crazy Bitch X rated music video.',
+            'uploader': 'oreusz',
+            'uploader_id': '124697',
+            'upload_date': '20070507',
+            'age_limit': 18,
+        }
+    }, {
+        # download URL pattern: */mp4_<format_id>_<video_id>.mp4
+        'url': 'http://www.spankwire.com/Titcums-Compiloation-I/video1921551/',
+        'md5': '09b3c20833308b736ae8902db2f8d7e6',
+        'info_dict': {
+            'id': '1921551',
+            'ext': 'mp4',
+            'title': 'Titcums Compiloation I',
+            'description': 'cum on tits',
+            'uploader': 'dannyh78999',
+            'uploader_id': '3056053',
+            'upload_date': '20150822',
+            'age_limit': 18,
+        },
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
@@ -81,9 +82,10 @@ class SpankwireIE(InfoExtractor):
             r'<span\s+id="spCommentCount"[^>]*>([\d,\.]+)</span>',
             webpage, 'comment count', fatal=False))
 
-        video_urls = list(map(
-            compat_urllib_parse_unquote,
-            re.findall(r'playerData\.cdnPath[0-9]{3,}\s*=\s*(?:encodeURIComponent\()?["\']([^"\']+)["\']', webpage)))
+        videos = re.findall(
+            r'playerData\.cdnPath([0-9]{3,})\s*=\s*(?:encodeURIComponent\()?["\']([^"\']+)["\']', webpage)
+        heights = [int(video[0]) for video in videos]
+        video_urls = list(map(compat_urllib_parse_unquote, [video[1] for video in videos]))
         if webpage.find('flashvars\.encrypted = "true"') != -1:
             password = self._search_regex(
                 r'flashvars\.video_title = "([^"]+)',
@@ -93,39 +95,22 @@ class SpankwireIE(InfoExtractor):
                 video_urls))
 
         formats = []
-        for video_url in video_urls:
+        for height, video_url in zip(heights, video_urls):
             path = compat_urllib_parse_urlparse(video_url).path
-            format = path.split('/')[4].split('_')[:2]
-            if format[0] == 'mp4':
-                format_id, quality = format
-                format = "-".join(format)
-                if quality == 'normal':
-                    height = 180
-                elif quality == 'high':
-                    height = 240
-                elif quality == 'ultra':
-                    height = 480
-                elif quality == '720p':
-                    height = 720
-                formats.append({
-                    'url': video_url,
-                    'format': format,
-                    'height': height,
-                    'format_id': format,
+            _, quality = path.split('/')[4].split('_')[:2]
+            f = {
+                'url': video_url,
+                'height': height,
+            }
+            tbr = self._search_regex(r'^(\d+)[Kk]$', quality, 'tbr', default=None)
+            if tbr:
+                f.update({
+                    'tbr': int(tbr),
+                    'format_id': '%dp' % height,
                 })
             else:
-                resolution, bitrate_str = format
-                format = "-".join(format)
-                height = int(resolution.rstrip('Pp'))
-                tbr = int(bitrate_str.rstrip('Kk'))
-                formats.append({
-                    'url': video_url,
-                    'resolution': resolution,
-                    'format': format,
-                    'tbr': tbr,
-                    'height': height,
-                    'format_id': format,
-                })
+                f['format_id'] = quality
+            formats.append(f)
         self._sort_formats(formats)
 
         age_limit = self._rta_search(webpage)

From 28b83495d898530e72d242874576f4d2d6d8ab3d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 23 Aug 2015 22:32:44 +0600
Subject: [PATCH 2326/6219] [spankwire] Simplify

---
 youtube_dl/extractor/spankwire.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/spankwire.py b/youtube_dl/extractor/spankwire.py
index 0a47441b1..609f78294 100644
--- a/youtube_dl/extractor/spankwire.py
+++ b/youtube_dl/extractor/spankwire.py
@@ -16,7 +16,7 @@ from ..aes import aes_decrypt_text
 
 
 class SpankwireIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?(?P<url>spankwire\.com/[^/]*/video(?P<videoid>[0-9]+)/?)'
+    _VALID_URL = r'https?://(?:www\.)?(?P<url>spankwire\.com/[^/]*/video(?P<id>[0-9]+)/?)'
     _TESTS = [{
         # download URL pattern: */<height>P_<tbr>K_<video_id>.mp4
         'url': 'http://www.spankwire.com/Buckcherry-s-X-Rated-Music-Video-Crazy-Bitch/video103545/',
@@ -49,10 +49,9 @@ class SpankwireIE(InfoExtractor):
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('videoid')
-        url = 'http://www.' + mobj.group('url')
+        video_id = mobj.group('id')
 
-        req = compat_urllib_request.Request(url)
+        req = compat_urllib_request.Request('http://www.' + mobj.group('url'))
         req.add_header('Cookie', 'age_verified=1')
         webpage = self._download_webpage(req, video_id)
 

From 90076b6172f25a36ca2a00c1b85cda169f2133c8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 23 Aug 2015 22:33:26 +0600
Subject: [PATCH 2327/6219] [spankwire] Preserve old uploader pattern

---
 youtube_dl/extractor/spankwire.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/spankwire.py b/youtube_dl/extractor/spankwire.py
index 609f78294..9e8fb35b2 100644
--- a/youtube_dl/extractor/spankwire.py
+++ b/youtube_dl/extractor/spankwire.py
@@ -68,7 +68,7 @@ class SpankwireIE(InfoExtractor):
             r'by:\s*<a [^>]*>(.+?)</a>',
             webpage, 'uploader', fatal=False)
         uploader_id = self._html_search_regex(
-            r'by:\s*<a href="/user/viewProfile\?.*?UserId=(\d+).*?"',
+            r'by:\s*<a href="/(?:user/viewProfile|Profile\.aspx)\?.*?UserId=(\d+).*?"',
             webpage, 'uploader id', fatal=False)
         upload_date = unified_strdate(self._html_search_regex(
             r'</a> on (.+?) at \d+:\d+',

From e7c14660d3aef3a5a303a82dee7e11dfe063048d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 24 Aug 2015 00:36:24 +0600
Subject: [PATCH 2328/6219] [yandexmusic] Defer link resolve till actual
 download time to prevent link expiry (Closes #6650)

---
 youtube_dl/extractor/yandexmusic.py | 21 ++++++++++++++-------
 1 file changed, 14 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/yandexmusic.py b/youtube_dl/extractor/yandexmusic.py
index f4c0f5702..85c495c11 100644
--- a/youtube_dl/extractor/yandexmusic.py
+++ b/youtube_dl/extractor/yandexmusic.py
@@ -1,4 +1,4 @@
-# coding=utf-8
+# coding: utf-8
 from __future__ import unicode_literals
 
 import re
@@ -64,7 +64,15 @@ class YandexMusicTrackIE(YandexMusicBaseIE):
         return self._get_track_info(track)
 
 
-class YandexMusicAlbumIE(YandexMusicBaseIE):
+class YandexMusicPlaylistBaseIE(InfoExtractor):
+    def _build_playlist(self, tracks):
+        return [
+            self.url_result(
+                'http://music.yandex.ru/album/%s/track/%s' % (track['albums'][0]['id'], track['id']))
+            for track in tracks]
+
+
+class YandexMusicAlbumIE(YandexMusicPlaylistBaseIE):
     IE_NAME = 'yandexmusic:album'
     IE_DESC = 'Яндекс.Музыка - Альбом'
     _VALID_URL = r'https?://music\.yandex\.(?:ru|kz|ua|by)/album/(?P<id>\d+)/?(\?|$)'
@@ -85,7 +93,7 @@ class YandexMusicAlbumIE(YandexMusicBaseIE):
             'http://music.yandex.ru/handlers/album.jsx?album=%s' % album_id,
             album_id, 'Downloading album JSON')
 
-        entries = [self._get_track_info(track) for track in album['volumes'][0]]
+        entries = self._build_playlist(album['volumes'][0])
 
         title = '%s - %s' % (album['artists'][0]['name'], album['title'])
         year = album.get('year')
@@ -95,7 +103,7 @@ class YandexMusicAlbumIE(YandexMusicBaseIE):
         return self.playlist_result(entries, compat_str(album['id']), title)
 
 
-class YandexMusicPlaylistIE(YandexMusicBaseIE):
+class YandexMusicPlaylistIE(YandexMusicPlaylistBaseIE):
     IE_NAME = 'yandexmusic:playlist'
     IE_DESC = 'Яндекс.Музыка - Плейлист'
     _VALID_URL = r'https?://music\.yandex\.(?:ru|kz|ua|by)/users/[^/]+/playlists/(?P<id>\d+)'
@@ -120,8 +128,7 @@ class YandexMusicPlaylistIE(YandexMusicBaseIE):
                 r'var\s+Mu\s*=\s*({.+?});\s*</script>', webpage, 'player'),
             playlist_id)['pageData']['playlist']
 
-        entries = [self._get_track_info(track) for track in playlist['tracks']]
-
         return self.playlist_result(
-            entries, compat_str(playlist_id),
+            self._build_playlist(playlist['tracks']),
+            compat_str(playlist_id),
             playlist['title'], playlist.get('description'))

From e4df2f98ccbe2e24785dd6883d7fd495193fd8e6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 24 Aug 2015 00:36:54 +0600
Subject: [PATCH 2329/6219] [yandexmusic:track] Eliminate base class

---
 youtube_dl/extractor/yandexmusic.py | 36 ++++++++++++++---------------
 1 file changed, 17 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/extractor/yandexmusic.py b/youtube_dl/extractor/yandexmusic.py
index 85c495c11..91829be1c 100644
--- a/youtube_dl/extractor/yandexmusic.py
+++ b/youtube_dl/extractor/yandexmusic.py
@@ -12,7 +12,23 @@ from ..utils import (
 )
 
 
-class YandexMusicBaseIE(InfoExtractor):
+class YandexMusicTrackIE(InfoExtractor):
+    IE_NAME = 'yandexmusic:track'
+    IE_DESC = 'Яндекс.Музыка - Трек'
+    _VALID_URL = r'https?://music\.yandex\.(?:ru|kz|ua|by)/album/(?P<album_id>\d+)/track/(?P<id>\d+)'
+
+    _TEST = {
+        'url': 'http://music.yandex.ru/album/540508/track/4878838',
+        'md5': 'f496818aa2f60b6c0062980d2e00dc20',
+        'info_dict': {
+            'id': '4878838',
+            'ext': 'mp3',
+            'title': 'Carlo Ambrosio - Gypsy Eyes 1',
+            'filesize': 4628061,
+            'duration': 193.04,
+        }
+    }
+
     def _get_track_url(self, storage_dir, track_id):
         data = self._download_json(
             'http://music.yandex.ru/api/v1.5/handlers/api-jsonp.jsx?action=getTrackSrc&p=download-info/%s'
@@ -35,24 +51,6 @@ class YandexMusicBaseIE(InfoExtractor):
             'duration': float_or_none(track.get('durationMs'), 1000),
         }
 
-
-class YandexMusicTrackIE(YandexMusicBaseIE):
-    IE_NAME = 'yandexmusic:track'
-    IE_DESC = 'Яндекс.Музыка - Трек'
-    _VALID_URL = r'https?://music\.yandex\.(?:ru|kz|ua|by)/album/(?P<album_id>\d+)/track/(?P<id>\d+)'
-
-    _TEST = {
-        'url': 'http://music.yandex.ru/album/540508/track/4878838',
-        'md5': 'f496818aa2f60b6c0062980d2e00dc20',
-        'info_dict': {
-            'id': '4878838',
-            'ext': 'mp3',
-            'title': 'Carlo Ambrosio - Gypsy Eyes 1',
-            'filesize': 4628061,
-            'duration': 193.04,
-        }
-    }
-
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         album_id, track_id = mobj.group('album_id'), mobj.group('id')

From 11addc50ffa9ce65ac3bef7af6b1c38d7eae1af6 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 23 Aug 2015 23:52:47 +0200
Subject: [PATCH 2330/6219] release 2015.08.23

---
 docs/supportedsites.md | 6 +++++-
 youtube_dl/version.py  | 2 +-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 9099e2da4..8d9db53a6 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -301,6 +301,7 @@
  - **Moviezine**
  - **movshare**: MovShare
  - **MPORA**
+ - **MSNBC**
  - **MTV**
  - **mtviggy.com**
  - **mtvservices:embedded**
@@ -308,6 +309,7 @@
  - **MusicPlayOn**
  - **MusicVault**
  - **muzu.tv**
+ - **Mwave**
  - **MySpace**
  - **MySpace:album**
  - **MySpass**
@@ -392,6 +394,8 @@
  - **Playtvak**: Playtvak.cz, iDNES.cz and Lidovky.cz
  - **Playvid**
  - **Playwire**
+ - **pluralsight**
+ - **pluralsight:course**
  - **plus.google**: Google Plus
  - **pluzz.francetv.fr**
  - **podomatic**
@@ -534,6 +538,7 @@
  - **TF1**
  - **TheOnion**
  - **ThePlatform**
+ - **ThePlatformFeed**
  - **TheSixtyOne**
  - **ThisAmericanLife**
  - **ThisAV**
@@ -599,7 +604,6 @@
  - **Viddler**
  - **video.google:search**: Google Video search
  - **video.mit.edu**
- - **VideoBam**
  - **VideoDetective**
  - **videofy.me**
  - **videolectures.net**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index c090c6df7..394951ca7 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.08.16.1'
+__version__ = '2015.08.23'

From eba470f2f22389ab32164e4eb39067ceecf900f5 Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Mon, 24 Aug 2015 16:30:00 +0800
Subject: [PATCH 2331/6219] [vlive] Remove upload_date extraction & cleanup

---
 youtube_dl/extractor/vlive.py | 23 +++++++----------------
 1 file changed, 7 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/extractor/vlive.py b/youtube_dl/extractor/vlive.py
index b3bbd80fb..6a403cc64 100644
--- a/youtube_dl/extractor/vlive.py
+++ b/youtube_dl/extractor/vlive.py
@@ -25,7 +25,6 @@ class VLiveIE(InfoExtractor):
             'ext': 'mp4',
             'title': '[V] Girl\'s Day\'s Broadcast',
             'creator': 'Girl\'s Day',
-            'upload_date': '20150817',
         },
     }
     _SECRET = 'rFkwZet6pqk1vQt6SxxUkAHX7YL3lmqzUMrU4IDusTo4jEBdtOhNfT4BYYAdArwH'
@@ -41,21 +40,14 @@ class VLiveIE(InfoExtractor):
         thumbnail = self._og_search_thumbnail(webpage)
         creator = self._html_search_regex(
             r'<span class="name">([^<>]+)</span>', webpage, 'creator')
-        upload_date = self._html_search_regex(
-            r'<span class="time">(\d{4}\.\d{2}\.\d{2})</span>', webpage,
-            'upload date', default=None, fatal=False)
-        if upload_date:
-            upload_date = upload_date.replace('.', '')
-
+        
         url = 'http://global.apis.naver.com/globalV/globalV/vod/%s/playinfo?' % video_id
-        msgpad = {'msgpad': '%.0f' % (time() * 1000)}
-        md = {
-            'md': b64encode(
-                hmac.new(self._SECRET.encode('ascii'),
-                         (url[:255] + msgpad['msgpad']).encode('ascii'), sha1).digest())
-        }
-        url += '&' + compat_urllib_parse.urlencode(msgpad) + '&' + compat_urllib_parse.urlencode(md)
-
+        msgpad = '%.0f' % (time() * 1000)
+        md = b64encode(
+            hmac.new(self._SECRET.encode('ascii'),
+                     (url[:255] + msgpad).encode('ascii'), sha1).digest()
+        )
+        url += '&' + compat_urllib_parse.urlencode({'msgpad': msgpad, 'md': md})
         playinfo = self._download_json(url, video_id, 'Downloading video json')
 
         if playinfo.get('message', '') != 'success':
@@ -89,6 +81,5 @@ class VLiveIE(InfoExtractor):
             'creator': creator,
             'thumbnail': thumbnail,
             'formats': formats,
-            'upload_date': upload_date,
             'subtitles': subtitles,
         }

From 95e431e9ec2477694d368a050222d6381a6f88ab Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 25 Aug 2015 21:08:38 +0600
Subject: [PATCH 2332/6219] [mailru] Skip tests

---
 youtube_dl/extractor/mailru.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/mailru.py b/youtube_dl/extractor/mailru.py
index 54a14cb94..ab1300185 100644
--- a/youtube_dl/extractor/mailru.py
+++ b/youtube_dl/extractor/mailru.py
@@ -25,6 +25,7 @@ class MailRuIE(InfoExtractor):
                 'uploader_id': 'sonypicturesrus@mail.ru',
                 'duration': 184,
             },
+            'skip': 'Not accessible from Travis CI server',
         },
         {
             'url': 'http://my.mail.ru/corp/hitech/video/news_hi-tech_mail_ru/1263.html',
@@ -39,6 +40,7 @@ class MailRuIE(InfoExtractor):
                 'uploader_id': 'hitech@corp.mail.ru',
                 'duration': 245,
             },
+            'skip': 'Not accessible from Travis CI server',
         },
     ]
 

From ebbf078c7df575903ceb1be53e53533508c79dad Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 25 Aug 2015 21:19:21 +0600
Subject: [PATCH 2333/6219] [krasview] Skip download for test

---
 youtube_dl/extractor/krasview.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/krasview.py b/youtube_dl/extractor/krasview.py
index 96f95979a..0ae8ebd68 100644
--- a/youtube_dl/extractor/krasview.py
+++ b/youtube_dl/extractor/krasview.py
@@ -25,6 +25,9 @@ class KrasViewIE(InfoExtractor):
             'duration': 27,
             'thumbnail': 're:^https?://.*\.jpg',
         },
+        'params': {
+            'skip_download': 'Not accessible from Travis CI server',
+        },
     }
 
     def _real_extract(self, url):

From 6d53cdd6ce441dd7bc1d93bf1445f0594cfdffef Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 25 Aug 2015 23:29:02 +0600
Subject: [PATCH 2334/6219] [yandexmusic] Skip removed tracks (#6666)

---
 youtube_dl/extractor/yandexmusic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/yandexmusic.py b/youtube_dl/extractor/yandexmusic.py
index 91829be1c..166cbf344 100644
--- a/youtube_dl/extractor/yandexmusic.py
+++ b/youtube_dl/extractor/yandexmusic.py
@@ -67,7 +67,7 @@ class YandexMusicPlaylistBaseIE(InfoExtractor):
         return [
             self.url_result(
                 'http://music.yandex.ru/album/%s/track/%s' % (track['albums'][0]['id'], track['id']))
-            for track in tracks]
+            for track in tracks if track.get('albums') and isinstance(track.get('albums'), list)]
 
 
 class YandexMusicAlbumIE(YandexMusicPlaylistBaseIE):

From baf510bf8cb296d2ed2a2f742ec9387d094623e6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 26 Aug 2015 00:11:15 +0600
Subject: [PATCH 2335/6219] [yandexmusic:playlist] Handle playlists with more
 than 150 tracks (Closes #6666)

---
 youtube_dl/extractor/yandexmusic.py | 51 +++++++++++++++++++++++++----
 1 file changed, 45 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/yandexmusic.py b/youtube_dl/extractor/yandexmusic.py
index 166cbf344..4098e4629 100644
--- a/youtube_dl/extractor/yandexmusic.py
+++ b/youtube_dl/extractor/yandexmusic.py
@@ -5,7 +5,11 @@ import re
 import hashlib
 
 from .common import InfoExtractor
-from ..compat import compat_str
+from ..compat import (
+    compat_str,
+    compat_urllib_parse,
+    compat_urllib_request,
+)
 from ..utils import (
     int_or_none,
     float_or_none,
@@ -106,7 +110,7 @@ class YandexMusicPlaylistIE(YandexMusicPlaylistBaseIE):
     IE_DESC = 'Яндекс.Музыка - Плейлист'
     _VALID_URL = r'https?://music\.yandex\.(?:ru|kz|ua|by)/users/[^/]+/playlists/(?P<id>\d+)'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://music.yandex.ru/users/music.partners/playlists/1245',
         'info_dict': {
             'id': '1245',
@@ -114,19 +118,54 @@ class YandexMusicPlaylistIE(YandexMusicPlaylistBaseIE):
             'description': 'md5:3b9f27b0efbe53f2ee1e844d07155cc9',
         },
         'playlist_count': 6,
-    }
+    }, {
+        # playlist exceeding the limit of 150 tracks shipped with webpage (see
+        # https://github.com/rg3/youtube-dl/issues/6666)
+        'url': 'https://music.yandex.ru/users/ya.playlist/playlists/1036',
+        'info_dict': {
+            'id': '1036',
+            'title': 'Музыка 90-х',
+        },
+        'playlist_count': 310,
+    }]
 
     def _real_extract(self, url):
         playlist_id = self._match_id(url)
 
         webpage = self._download_webpage(url, playlist_id)
 
-        playlist = self._parse_json(
+        mu = self._parse_json(
             self._search_regex(
                 r'var\s+Mu\s*=\s*({.+?});\s*</script>', webpage, 'player'),
-            playlist_id)['pageData']['playlist']
+            playlist_id)
+
+        playlist = mu['pageData']['playlist']
+        tracks, track_ids = playlist['tracks'], playlist['trackIds']
+
+        # tracks dictionary shipped with webpage is limited to 150 tracks,
+        # missing tracks should be retrieved manually.
+        if len(tracks) < len(track_ids):
+            present_track_ids = set([compat_str(track['id']) for track in tracks if track.get('id')])
+            missing_track_ids = set(map(compat_str, track_ids)) - set(present_track_ids)
+            request = compat_urllib_request.Request(
+                'https://music.yandex.ru/handlers/track-entries.jsx',
+                compat_urllib_parse.urlencode({
+                    'entries': ','.join(missing_track_ids),
+                    'lang': mu.get('settings', {}).get('lang', 'en'),
+                    'external-domain': 'music.yandex.ru',
+                    'overembed': 'false',
+                    'sign': mu.get('authData', {}).get('user', {}).get('sign'),
+                    'strict': 'true',
+                }).encode('utf-8'))
+            request.add_header('Referer', url)
+            request.add_header('X-Requested-With', 'XMLHttpRequest')
+
+            missing_tracks = self._download_json(
+                request, playlist_id, 'Downloading missing tracks JSON', fatal=False)
+            if missing_tracks:
+                tracks.extend(missing_tracks)
 
         return self.playlist_result(
-            self._build_playlist(playlist['tracks']),
+            self._build_playlist(tracks),
             compat_str(playlist_id),
             playlist['title'], playlist.get('description'))

From 4bc8eec4ebf5ffcca3b2e17c864be08df5215f44 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 26 Aug 2015 15:21:55 +0200
Subject: [PATCH 2336/6219] [youtube] Adapt player version regex to handle urls
 ending in '/html5player-new.js'

It was always extracting 'new' as the version, breaking the cache system.
---
 youtube_dl/extractor/youtube.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 8e2da46e3..ab6754154 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -660,7 +660,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
 
     def _extract_signature_function(self, video_id, player_url, example_sig):
         id_m = re.match(
-            r'.*?-(?P<id>[a-zA-Z0-9_-]+)(?:/watch_as3|/html5player)?\.(?P<ext>[a-z]+)$',
+            r'.*?-(?P<id>[a-zA-Z0-9_-]+)(?:/watch_as3|/html5player(?:-new)?)?\.(?P<ext>[a-z]+)$',
             player_url)
         if not id_m:
             raise ExtractorError('Cannot identify player %r' % player_url)
@@ -1289,7 +1289,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                                 player_desc = 'flash player %s' % player_version
                             else:
                                 player_version = self._search_regex(
-                                    r'html5player-([^/]+?)(?:/html5player)?\.js',
+                                    r'html5player-([^/]+?)(?:/html5player(?:-new)?)?\.js',
                                     player_url,
                                     'html5 player', fatal=False)
                                 player_desc = 'html5 player %s' % player_version

From 2f72e83bbd915054cac0e8f70df0c2cab4b9c116 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 26 Aug 2015 20:47:57 +0600
Subject: [PATCH 2337/6219] [crunchyroll] Detect required login (#6677)

---
 youtube_dl/extractor/crunchyroll.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 33a033a7f..98d1881ae 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -237,7 +237,9 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
             webpage_url = 'http://www.' + mobj.group('url')
 
         webpage = self._download_webpage(webpage_url, video_id, 'Downloading webpage')
-        note_m = self._html_search_regex(r'<div class="showmedia-trailer-notice">(.+?)</div>', webpage, 'trailer-notice', default='')
+        note_m = self._html_search_regex(
+            r'<div class="showmedia-trailer-notice">(.+?)</div>',
+            webpage, 'trailer-notice', default='')
         if note_m:
             raise ExtractorError(note_m)
 
@@ -247,6 +249,12 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
             if msg.get('type') == 'error':
                 raise ExtractorError('crunchyroll returned error: %s' % msg['message_body'], expected=True)
 
+        if 'To view this, please log in to verify you are 18 or older.' in webpage:
+            raise ExtractorError(
+                'This video is only available for registered users, '
+                'use --username and --password options to provide account credentials.',
+                expected=True)
+
         video_title = self._html_search_regex(r'<h1[^>]*>(.+?)</h1>', webpage, 'video_title', flags=re.DOTALL)
         video_title = re.sub(r' {2,}', ' ', video_title)
         video_description = self._html_search_regex(r'"description":"([^"]+)', webpage, 'video_description', default='')

From 43e7d3c9453338ae29552311b1447fe95be05db5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 26 Aug 2015 21:24:47 +0600
Subject: [PATCH 2338/6219] [extractor/common] Add raise_login_required

---
 youtube_dl/extractor/common.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 5d24bcb6a..39cef9c5b 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -510,6 +510,12 @@ class InfoExtractor(object):
         """Report attempt to log in."""
         self.to_screen('Logging in')
 
+    @staticmethod
+    def raise_login_required(msg='This video is only available for registered users'):
+        raise ExtractorError(
+            '%s. Use --username and --password or --netrc to provide account credentials.' % msg,
+            expected=True)
+
     # Methods for following #608
     @staticmethod
     def url_result(url, ie=None, video_id=None, video_title=None):

From bbb43a39fd11c2fdf28ae593eaa994f22ce663bc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 26 Aug 2015 21:25:04 +0600
Subject: [PATCH 2339/6219] [crunchyroll] Use raise_login_required

---
 youtube_dl/extractor/crunchyroll.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 98d1881ae..801b9b48e 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -250,10 +250,7 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
                 raise ExtractorError('crunchyroll returned error: %s' % msg['message_body'], expected=True)
 
         if 'To view this, please log in to verify you are 18 or older.' in webpage:
-            raise ExtractorError(
-                'This video is only available for registered users, '
-                'use --username and --password options to provide account credentials.',
-                expected=True)
+            self.raise_login_required(video_id)
 
         video_title = self._html_search_regex(r'<h1[^>]*>(.+?)</h1>', webpage, 'video_title', flags=re.DOTALL)
         video_title = re.sub(r' {2,}', ' ', video_title)

From 3c53455d15035a94bcd2bc915f565420e1a4279f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 26 Aug 2015 21:25:37 +0600
Subject: [PATCH 2340/6219] [eroprofile] Use raise_login_required

---
 youtube_dl/extractor/eroprofile.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/eroprofile.py b/youtube_dl/extractor/eroprofile.py
index 316033cf1..7fcd0151d 100644
--- a/youtube_dl/extractor/eroprofile.py
+++ b/youtube_dl/extractor/eroprofile.py
@@ -71,8 +71,7 @@ class EroProfileIE(InfoExtractor):
 
         m = re.search(r'You must be logged in to view this video\.', webpage)
         if m:
-            raise ExtractorError(
-                'This video requires login. Please specify a username and password and try again.', expected=True)
+            self.raise_login_required('This video requires login')
 
         video_id = self._search_regex(
             [r"glbUpdViews\s*\('\d*','(\d+)'", r'p/report/video/(\d+)'],

From 62984e4584c2962e622514c7d6a475636a8c21d8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 26 Aug 2015 21:25:53 +0600
Subject: [PATCH 2341/6219] [lynda] Use raise_login_required

---
 youtube_dl/extractor/lynda.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index 5b9157ed4..378117270 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -118,9 +118,7 @@ class LyndaIE(LyndaBaseIE):
                 'lynda returned error: %s' % video_json['Message'], expected=True)
 
         if video_json['HasAccess'] is False:
-            raise ExtractorError(
-                'Video %s is only available for members. '
-                % video_id + self._ACCOUNT_CREDENTIALS_HINT, expected=True)
+            self.raise_login_required('Video %s is only available for members' % video_id)
 
         video_id = compat_str(video_json['ID'])
         duration = video_json['DurationInSeconds']

From e7ddaef5bd209dd8d24b0025631cde1f5969e71d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 26 Aug 2015 21:26:09 +0600
Subject: [PATCH 2342/6219] [pluralsight] Use raise_login_required

---
 youtube_dl/extractor/pluralsight.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index 7ba396aef..fd32836cc 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -41,9 +41,7 @@ class PluralsightIE(InfoExtractor):
     def _login(self):
         (username, password) = self._get_login_info()
         if username is None:
-            raise ExtractorError(
-                'Pluralsight account is required, use --username and --password options to provide account credentials.',
-                expected=True)
+            self.raise_login_required('Pluralsight account is required')
 
         login_page = self._download_webpage(
             self._LOGIN_URL, None, 'Downloading login page')

From e269d3ae7dbebb22d5b51bd5e6d477a69ae4f3b4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 26 Aug 2015 21:26:24 +0600
Subject: [PATCH 2343/6219] [safari] Use raise_login_required

---
 youtube_dl/extractor/safari.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/safari.py b/youtube_dl/extractor/safari.py
index f3c80708c..a602af692 100644
--- a/youtube_dl/extractor/safari.py
+++ b/youtube_dl/extractor/safari.py
@@ -20,7 +20,6 @@ from ..utils import (
 class SafariBaseIE(InfoExtractor):
     _LOGIN_URL = 'https://www.safaribooksonline.com/accounts/login/'
     _SUCCESSFUL_LOGIN_REGEX = r'<a href="/accounts/logout/"[^>]*>Sign Out</a>'
-    _ACCOUNT_CREDENTIALS_HINT = 'Use --username and --password options to supply credentials for safaribooksonline.com'
     _NETRC_MACHINE = 'safari'
 
     _API_BASE = 'https://www.safaribooksonline.com/api/v1/book'
@@ -37,9 +36,7 @@ class SafariBaseIE(InfoExtractor):
     def _login(self):
         (username, password) = self._get_login_info()
         if username is None:
-            raise ExtractorError(
-                self._ACCOUNT_CREDENTIALS_HINT,
-                expected=True)
+            self.raise_login_required('safaribooksonline.com account is required')
 
         headers = std_headers
         if 'Referer' not in headers:

From 42e7373bd3c819ee7cebf5898e4bdd33730dde6f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 26 Aug 2015 21:26:35 +0600
Subject: [PATCH 2344/6219] [smotri] Use raise_login_required

---
 youtube_dl/extractor/smotri.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/smotri.py b/youtube_dl/extractor/smotri.py
index 93a7cfe15..35a81ee87 100644
--- a/youtube_dl/extractor/smotri.py
+++ b/youtube_dl/extractor/smotri.py
@@ -330,10 +330,7 @@ class SmotriBroadcastIE(InfoExtractor):
 
             (username, password) = self._get_login_info()
             if username is None:
-                raise ExtractorError(
-                    'Erotic broadcasts allowed only for registered users, '
-                    'use --username and --password options to provide account credentials.',
-                    expected=True)
+                self.raise_login_required('Erotic broadcasts allowed only for registered users')
 
             login_form = {
                 'login-hint53': '1',

From 61a7ff16222accdb259f771d0a6f0adb229b34dd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 26 Aug 2015 21:26:47 +0600
Subject: [PATCH 2345/6219] [tubitv] Use raise_login_required

---
 youtube_dl/extractor/tubitv.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/tubitv.py b/youtube_dl/extractor/tubitv.py
index 2c4b21807..4f86b3ee9 100644
--- a/youtube_dl/extractor/tubitv.py
+++ b/youtube_dl/extractor/tubitv.py
@@ -60,9 +60,7 @@ class TubiTvIE(InfoExtractor):
 
         webpage = self._download_webpage(url, video_id)
         if re.search(r"<(?:DIV|div) class='login-required-screen'>", webpage):
-            raise ExtractorError(
-                'This video requires login, use --username and --password '
-                'options to provide account credentials.', expected=True)
+            self.raise_login_required('This video requires login')
 
         title = self._og_search_title(webpage)
         description = self._og_search_description(webpage)

From a882c5f4747c527bb50d87828ea4cceae6d12533 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 26 Aug 2015 21:27:07 +0600
Subject: [PATCH 2346/6219] [udemy] Use raise_login_required

---
 youtube_dl/extractor/udemy.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index 4a0eaf65f..365d8b4bf 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -70,9 +70,7 @@ class UdemyIE(InfoExtractor):
     def _login(self):
         (username, password) = self._get_login_info()
         if username is None:
-            raise ExtractorError(
-                'Udemy account is required, use --username and --password options to provide account credentials.',
-                expected=True)
+            self.raise_login_required('Udemy account is required')
 
         login_popup = self._download_webpage(
             self._LOGIN_URL, None, 'Downloading login popup')

From 39affb5aa427a3a1830e97523470d11bfdbd067e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 26 Aug 2015 21:27:57 +0600
Subject: [PATCH 2347/6219] [crunchyroll] Fix typo

---
 youtube_dl/extractor/crunchyroll.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 801b9b48e..c2162aa68 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -250,7 +250,7 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
                 raise ExtractorError('crunchyroll returned error: %s' % msg['message_body'], expected=True)
 
         if 'To view this, please log in to verify you are 18 or older.' in webpage:
-            self.raise_login_required(video_id)
+            self.raise_login_required()
 
         video_title = self._html_search_regex(r'<h1[^>]*>(.+?)</h1>', webpage, 'video_title', flags=re.DOTALL)
         video_title = re.sub(r' {2,}', ' ', video_title)

From 3d8132f5e20b7cbdaa8f69aca482553b2c02bed8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 26 Aug 2015 22:03:58 +0600
Subject: [PATCH 2348/6219] [shared] Extend _VALID_URL to support vivo.sx
 (Closes #6681)

---
 youtube_dl/extractor/shared.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/shared.py b/youtube_dl/extractor/shared.py
index a07677686..000ef1a07 100644
--- a/youtube_dl/extractor/shared.py
+++ b/youtube_dl/extractor/shared.py
@@ -14,7 +14,7 @@ from ..utils import (
 
 
 class SharedIE(InfoExtractor):
-    _VALID_URL = r'http://shared\.sx/(?P<id>[\da-z]{10})'
+    _VALID_URL = r'http://(?:shared|vivo)\.sx/(?P<id>[\da-z]{10})'
 
     _TEST = {
         'url': 'http://shared.sx/0060718775',

From 70113c38c9e551d7d9ab2a4d1f7e76c81b68ae76 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 26 Aug 2015 22:04:39 +0600
Subject: [PATCH 2349/6219] [shared] Clarify IE_DESC

---
 youtube_dl/extractor/shared.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/shared.py b/youtube_dl/extractor/shared.py
index 000ef1a07..cf0a3bfef 100644
--- a/youtube_dl/extractor/shared.py
+++ b/youtube_dl/extractor/shared.py
@@ -14,6 +14,7 @@ from ..utils import (
 
 
 class SharedIE(InfoExtractor):
+    IE_DESC = 'shared.sx and vivo.sx'
     _VALID_URL = r'http://(?:shared|vivo)\.sx/(?P<id>[\da-z]{10})'
 
     _TEST = {

From f62e02c24f1f0e0488b40df178ddb9bb5fdf9fc6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 26 Aug 2015 22:05:45 +0600
Subject: [PATCH 2350/6219] [shared] Add test for vivo

---
 youtube_dl/extractor/shared.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/shared.py b/youtube_dl/extractor/shared.py
index cf0a3bfef..4fa991dff 100644
--- a/youtube_dl/extractor/shared.py
+++ b/youtube_dl/extractor/shared.py
@@ -17,7 +17,7 @@ class SharedIE(InfoExtractor):
     IE_DESC = 'shared.sx and vivo.sx'
     _VALID_URL = r'http://(?:shared|vivo)\.sx/(?P<id>[\da-z]{10})'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://shared.sx/0060718775',
         'md5': '106fefed92a8a2adb8c98e6a0652f49b',
         'info_dict': {
@@ -25,7 +25,16 @@ class SharedIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Bmp4',
         },
-    }
+    }, {
+        'url': 'http://vivo.sx/d7ddda0e78',
+        'md5': '15b3af41be0b4fe01f4df075c2678b2c',
+        'info_dict': {
+            'id': 'd7ddda0e78',
+            'ext': 'mp4',
+            'title': 'Chicken',
+            'filesize': 528031,
+        },
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From f11c316347bea41d9148d1c8d5d7738a594a06d0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 26 Aug 2015 22:06:10 +0600
Subject: [PATCH 2351/6219] [shared] Add filesize to test

---
 youtube_dl/extractor/shared.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/shared.py b/youtube_dl/extractor/shared.py
index 4fa991dff..c5636e8e9 100644
--- a/youtube_dl/extractor/shared.py
+++ b/youtube_dl/extractor/shared.py
@@ -24,6 +24,7 @@ class SharedIE(InfoExtractor):
             'id': '0060718775',
             'ext': 'mp4',
             'title': 'Bmp4',
+            'filesize': 1720110,
         },
     }, {
         'url': 'http://vivo.sx/d7ddda0e78',

From d7e8264517d29156697f82b7761dc99d13994c21 Mon Sep 17 00:00:00 2001
From: nmrugg <nmrugg@gmail.com>
Date: Thu, 27 Aug 2015 23:24:13 +0800
Subject: [PATCH 2352/6219] Make FoxBusiness work.

---
 youtube_dl/extractor/foxnews.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/foxnews.py b/youtube_dl/extractor/foxnews.py
index 917f76b1e..7de88ab66 100644
--- a/youtube_dl/extractor/foxnews.py
+++ b/youtube_dl/extractor/foxnews.py
@@ -1,5 +1,7 @@
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..utils import (
     parse_iso8601,
@@ -8,7 +10,7 @@ from ..utils import (
 
 
 class FoxNewsIE(InfoExtractor):
-    _VALID_URL = r'https?://video\.foxnews\.com/v/(?:video-embed\.html\?video_id=)?(?P<id>\d+)'
+    _VALID_URL = r'https?://video\.fox(?:news|business)\.com/v/(?:video-embed\.html\?video_id=)?(?P<id>\d+)'
     _TESTS = [
         {
             'url': 'http://video.foxnews.com/v/3937480/frozen-in-time/#sp=show-clips',
@@ -47,8 +49,10 @@ class FoxNewsIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
+        m = re.match(r'^https?://video\.fox(news|business)', url)
+
         video = self._download_json(
-            'http://video.foxnews.com/v/feed/video/%s.js?template=fox' % video_id, video_id)
+            'http://video.fox' + m.group(1) + '.com/v/feed/video/%s.js?template=fox' % video_id, video_id)
 
         item = video['channel']['item']
         title = item['title']

From 8df8c278b6d5e2b5a350446690873dc9f5f48aff Mon Sep 17 00:00:00 2001
From: nmrugg <nmrugg@gmail.com>
Date: Thu, 27 Aug 2015 23:24:28 +0800
Subject: [PATCH 2353/6219] Added matching test for FoxBusiness.

---
 youtube_dl/extractor/foxnews.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/foxnews.py b/youtube_dl/extractor/foxnews.py
index 7de88ab66..a8902c960 100644
--- a/youtube_dl/extractor/foxnews.py
+++ b/youtube_dl/extractor/foxnews.py
@@ -44,6 +44,10 @@ class FoxNewsIE(InfoExtractor):
             'url': 'http://video.foxnews.com/v/video-embed.html?video_id=3937480&d=video.foxnews.com',
             'only_matching': True,
         },
+        {
+            'url': 'http://video.foxbusiness.com/v/4442309889001',
+            'only_matching': True,
+        },
     ]
 
     def _real_extract(self, url):

From 1b660cce120c733f2bb195ef1cfe2ff2421b439f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 27 Aug 2015 21:48:03 +0600
Subject: [PATCH 2354/6219] [foxnews] Simplify (Closes #6694)

---
 youtube_dl/extractor/foxnews.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/foxnews.py b/youtube_dl/extractor/foxnews.py
index a8902c960..244c75f0b 100644
--- a/youtube_dl/extractor/foxnews.py
+++ b/youtube_dl/extractor/foxnews.py
@@ -10,7 +10,7 @@ from ..utils import (
 
 
 class FoxNewsIE(InfoExtractor):
-    _VALID_URL = r'https?://video\.fox(?:news|business)\.com/v/(?:video-embed\.html\?video_id=)?(?P<id>\d+)'
+    _VALID_URL = r'https?://(?P<host>video\.fox(?:news|business)\.com)/v/(?:video-embed\.html\?video_id=)?(?P<id>\d+)'
     _TESTS = [
         {
             'url': 'http://video.foxnews.com/v/3937480/frozen-in-time/#sp=show-clips',
@@ -51,12 +51,12 @@ class FoxNewsIE(InfoExtractor):
     ]
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        m = re.match(r'^https?://video\.fox(news|business)', url)
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        host = mobj.group('host')
 
         video = self._download_json(
-            'http://video.fox' + m.group(1) + '.com/v/feed/video/%s.js?template=fox' % video_id, video_id)
+            'http://%s/v/feed/video/%s.js?template=fox' % (host, video_id), video_id)
 
         item = video['channel']['item']
         title = item['title']

From 5307c332329d6a1f3eec240b66a4f11905889f23 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 27 Aug 2015 21:48:47 +0600
Subject: [PATCH 2355/6219] [foxnews] Clarify IE_DESC

---
 youtube_dl/extractor/foxnews.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/foxnews.py b/youtube_dl/extractor/foxnews.py
index 244c75f0b..3a4a59135 100644
--- a/youtube_dl/extractor/foxnews.py
+++ b/youtube_dl/extractor/foxnews.py
@@ -10,6 +10,7 @@ from ..utils import (
 
 
 class FoxNewsIE(InfoExtractor):
+    IE_DESC = 'Fox News and Fox Business Video'
     _VALID_URL = r'https?://(?P<host>video\.fox(?:news|business)\.com)/v/(?:video-embed\.html\?video_id=)?(?P<id>\d+)'
     _TESTS = [
         {

From a4962b80d668de704fc347d5e76587be0e95dfef Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 28 Aug 2015 05:04:39 +0200
Subject: [PATCH 2356/6219] release 2015.08.28

---
 docs/supportedsites.md | 4 ++--
 youtube_dl/version.py  | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 8d9db53a6..328a819b3 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -166,7 +166,7 @@
  - **Folketinget**: Folketinget (ft.dk; Danish parliament)
  - **FootyRoom**
  - **Foxgay**
- - **FoxNews**
+ - **FoxNews**: Fox News and Fox Business Video
  - **FoxSports**
  - **france2.fr:generation-quoi**
  - **FranceCulture**
@@ -465,7 +465,7 @@
  - **Sexu**
  - **SexyKarma**: Sexy Karma and Watch Indian Porn
  - **Shahid**
- - **Shared**
+ - **Shared**: shared.sx and vivo.sx
  - **ShareSix**
  - **Sina**
  - **Slideshare**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 394951ca7..a07bc9233 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.08.23'
+__version__ = '2015.08.28'

From 071c10137b6b17b79ecfc8676736d5cc243022f6 Mon Sep 17 00:00:00 2001
From: Paul Hartmann <phaaurlt@gmail.com>
Date: Wed, 26 Aug 2015 00:06:44 +0200
Subject: [PATCH 2357/6219] [MTV] move German mtv site to new class

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/mtv.py      | 37 ++++++++++++++++++++++++++++++++
 2 files changed, 38 insertions(+)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index d59882598..66422b005 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -340,6 +340,7 @@ from .mtv import (
     MTVIE,
     MTVServicesEmbeddedIE,
     MTVIggyIE,
+    MTVDEIE,
 )
 from .muenchentv import MuenchenTVIE
 from .musicplayon import MusicPlayOnIE
diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index b48fac5e3..15df62649 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -288,3 +288,40 @@ class MTVIggyIE(MTVServicesInfoExtractor):
         }
     }
     _FEED_URL = 'http://all.mtvworldverticals.com/feed-xml/'
+
+class MTVDEIE(MTVServicesInfoExtractor):
+    IE_NAME = 'mtv.de'
+    _VALID_URL = r'''(?x)^https?://(?:www\.)?mtv\.de(?P<video_path>/artists/.*)'''
+    _TESTS = [
+        {
+            'url': 'http://www.mtv.de/artists/10571-cro/videos/61131-traum',
+            'info_dict': {
+                'id': 'a50bc5f0b3aa4b3190aa',
+                'ext': 'mp4',
+                'title': 'cro-traum',
+                'description': 'Cro - Traum',
+            },
+        },
+    ]
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        return self._get_videos_info(url, mobj.group('video_path'))
+
+    def _get_videos_info(self, url, video_path):
+        webpage = self._download_webpage(url, video_path)
+        playlist_js = self._search_regex(r'<script>\s*window.pagePlaylist =(.*?\]);\s*window.trackingParams =', webpage, 'playlist', flags=re.DOTALL)
+        playlist = self._parse_json(playlist_js, video_path)
+        info = None
+        for item in playlist:
+            if item['video_path'] == video_path:
+                info = item
+                break
+        if info == None:
+            raise ExtractorError('video not in playlist')
+        mrss_url = info['mrss']
+        idoc = self._download_xml(
+            mrss_url, video_path,
+            'Downloading info', transform_source=fix_xml_ampersands)
+        return self.playlist_result(
+            [self._get_video_info(item) for item in idoc.findall('.//item')])

From 79fa9db0da4aa86aa101cc32a5da4b7e254c6a0d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 28 Aug 2015 21:24:54 +0600
Subject: [PATCH 2358/6219] [mtvde] Simplify (Closes #6673)

---
 youtube_dl/extractor/mtv.py | 59 ++++++++++++++++++-------------------
 1 file changed, 29 insertions(+), 30 deletions(-)

diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index 15df62649..f2d0638bf 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -174,8 +174,11 @@ class MTVServicesInfoExtractor(InfoExtractor):
         if self._LANG:
             info_url += 'lang=%s&' % self._LANG
         info_url += data
+        return self._get_videos_info_from_url(info_url, video_id)
+
+    def _get_videos_info_from_url(self, url, video_id):
         idoc = self._download_xml(
-            info_url, video_id,
+            url, video_id,
             'Downloading info', transform_source=fix_xml_ampersands)
         return self.playlist_result(
             [self._get_video_info(item) for item in idoc.findall('.//item')])
@@ -289,39 +292,35 @@ class MTVIggyIE(MTVServicesInfoExtractor):
     }
     _FEED_URL = 'http://all.mtvworldverticals.com/feed-xml/'
 
+
 class MTVDEIE(MTVServicesInfoExtractor):
     IE_NAME = 'mtv.de'
-    _VALID_URL = r'''(?x)^https?://(?:www\.)?mtv\.de(?P<video_path>/artists/.*)'''
-    _TESTS = [
-        {
-            'url': 'http://www.mtv.de/artists/10571-cro/videos/61131-traum',
-            'info_dict': {
-                'id': 'a50bc5f0b3aa4b3190aa',
-                'ext': 'mp4',
-                'title': 'cro-traum',
-                'description': 'Cro - Traum',
-            },
+    _VALID_URL = r'https?://(?:www\.)?mtv\.de/(?:artists|shows)/(?:[^/]+/)+(?P<id>\d+)-[^/#?]+/*(?:[#?].*)?$'
+    _TESTS = [{
+        'url': 'http://www.mtv.de/artists/10571-cro/videos/61131-traum',
+        'info_dict': {
+            'id': 'music_video-a50bc5f0b3aa4b3190aa',
+            'ext': 'mp4',
+            'title': 'MusicVideo_cro-traum',
+            'description': 'Cro - Traum',
         },
-    ]
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        },
+    }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        return self._get_videos_info(url, mobj.group('video_path'))
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        playlist = self._parse_json(
+            self._search_regex(
+                r'window\.pagePlaylist\s*=\s*(\[.+?\]);\n', webpage, 'page playlist'),
+            video_id)
 
-    def _get_videos_info(self, url, video_path):
-        webpage = self._download_webpage(url, video_path)
-        playlist_js = self._search_regex(r'<script>\s*window.pagePlaylist =(.*?\]);\s*window.trackingParams =', webpage, 'playlist', flags=re.DOTALL)
-        playlist = self._parse_json(playlist_js, video_path)
-        info = None
         for item in playlist:
-            if item['video_path'] == video_path:
-                info = item
-                break
-        if info == None:
-            raise ExtractorError('video not in playlist')
-        mrss_url = info['mrss']
-        idoc = self._download_xml(
-            mrss_url, video_path,
-            'Downloading info', transform_source=fix_xml_ampersands)
-        return self.playlist_result(
-            [self._get_video_info(item) for item in idoc.findall('.//item')])
+            item_id = item.get('id')
+            if item_id and compat_str(item_id) == video_id:
+                return self._get_videos_info_from_url(item['mrss'], video_id)

From 56f447be9fa196bceafbacb6526cad7c122a30ea Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 28 Aug 2015 21:27:36 +0600
Subject: [PATCH 2359/6219] [mtvservices] Fix mediagen URL generation

---
 youtube_dl/extractor/mtv.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index f2d0638bf..56b255599 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -114,7 +114,8 @@ class MTVServicesInfoExtractor(InfoExtractor):
         # Remove the templates, like &device={device}
         mediagen_url = re.sub(r'&[^=]*?={.*?}(?=(&|$))', '', mediagen_url)
         if 'acceptMethods' not in mediagen_url:
-            mediagen_url += '&acceptMethods=fms'
+            mediagen_url += '&' if '?' in mediagen_url else '?'
+            mediagen_url += 'acceptMethods=fms'
 
         mediagen_doc = self._download_xml(mediagen_url, video_id,
                                           'Downloading video urls')

From c3c9f879541b99a5456991d887ed03a2aea5dcff Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 28 Aug 2015 21:31:38 +0600
Subject: [PATCH 2360/6219] [mtvde] Add test for mediagen URL without query

---
 youtube_dl/extractor/mtv.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index 56b255599..28301d406 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -309,6 +309,18 @@ class MTVDEIE(MTVServicesInfoExtractor):
             # rtmp download
             'skip_download': True,
         },
+    }, {
+        # mediagen URL without query (e.g. http://videos.mtvnn.com/mediagen/e865da714c166d18d6f80893195fcb97)
+        'url': 'http://www.mtv.de/shows/933-teen-mom-2/staffeln/5353/folgen/63565-enthullungen',
+        'info_dict': {
+            'id': 'local_playlist-f5ae778b9832cc837189',
+            'ext': 'mp4',
+            'title': 'Episode_teen-mom-2_shows_season-5_episode-1_full-episode_part1',
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        },
     }]
 
     def _real_extract(self, url):

From 65488b820c6792f6c9f5e729c417985d501c8eb1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 28 Aug 2015 21:41:54 +0600
Subject: [PATCH 2361/6219] [mtvde] Add support for news

---
 youtube_dl/extractor/mtv.py | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index 28301d406..d160f4a22 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -296,7 +296,7 @@ class MTVIggyIE(MTVServicesInfoExtractor):
 
 class MTVDEIE(MTVServicesInfoExtractor):
     IE_NAME = 'mtv.de'
-    _VALID_URL = r'https?://(?:www\.)?mtv\.de/(?:artists|shows)/(?:[^/]+/)+(?P<id>\d+)-[^/#?]+/*(?:[#?].*)?$'
+    _VALID_URL = r'https?://(?:www\.)?mtv\.de/(?:artists|shows|news)/(?:[^/]+/)*(?P<id>\d+)-[^/#?]+/*(?:[#?].*)?$'
     _TESTS = [{
         'url': 'http://www.mtv.de/artists/10571-cro/videos/61131-traum',
         'info_dict': {
@@ -321,6 +321,19 @@ class MTVDEIE(MTVServicesInfoExtractor):
             # rtmp download
             'skip_download': True,
         },
+    }, {
+        # single video in pagePlaylist with different id
+        'url': 'http://www.mtv.de/news/77491-mtv-movies-spotlight-pixels-teil-3',
+        'info_dict': {
+            'id': 'local_playlist-4e760566473c4c8c5344',
+            'ext': 'mp4',
+            'title': 'Article_mtv-movies-spotlight-pixels-teil-3_short-clips_part1',
+            'description': 'MTV Movies Supercut',
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        },
     }]
 
     def _real_extract(self, url):
@@ -333,6 +346,10 @@ class MTVDEIE(MTVServicesInfoExtractor):
                 r'window\.pagePlaylist\s*=\s*(\[.+?\]);\n', webpage, 'page playlist'),
             video_id)
 
+        # news pages contain single video in playlist with different id
+        if len(playlist) == 1:
+            return self._get_videos_info_from_url(playlist[0]['mrss'], video_id)
+
         for item in playlist:
             item_id = item.get('id')
             if item_id and compat_str(item_id) == video_id:

From 054d43bb11590da234e7e8f46a595a34b19efb8c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 28 Aug 2015 21:45:29 +0600
Subject: [PATCH 2362/6219] [mtvservices] Catch more georestriction errors

---
 youtube_dl/extractor/mtv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index d160f4a22..4e4358151 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -67,7 +67,7 @@ class MTVServicesInfoExtractor(InfoExtractor):
         return [{'url': url, 'ext': 'mp4'}]
 
     def _extract_video_formats(self, mdoc, mtvn_id):
-        if re.match(r'.*/(error_country_block\.swf|geoblock\.mp4)$', mdoc.find('.//src').text) is not None:
+        if re.match(r'.*/(error_country_block\.swf|geoblock\.mp4|copyright_error\.flv(?:\?geo\b.+?)?)$', mdoc.find('.//src').text) is not None:
             if mtvn_id is not None and self._MOBILE_TEMPLATE is not None:
                 self.to_screen('The normal version is not available from your '
                                'country, trying with the mobile version')

From 20149a5da195b8ab5e50a4babfa0107ebf279b7f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 28 Aug 2015 22:10:49 +0600
Subject: [PATCH 2363/6219] [mtvservices] Fix title extraction under python 2

---
 youtube_dl/extractor/mtv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index 4e4358151..a597714e9 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -142,7 +142,7 @@ class MTVServicesInfoExtractor(InfoExtractor):
         if title_el is None:
             title_el = itemdoc.find('.//{http://search.yahoo.com/mrss/}title')
         if title_el is None:
-            title_el = itemdoc.find('.//title')
+            title_el = itemdoc.find('.//title') or itemdoc.find('./title')
             if title_el.text is None:
                 title_el = None
 

From 8ca31a0e056a7a7b71ea35753504ed0f870e834f Mon Sep 17 00:00:00 2001
From: felix <felix.von.s@posteo.de>
Date: Sat, 29 Aug 2015 07:55:20 +0200
Subject: [PATCH 2364/6219] [generic] Look for ScreenwaveMedia embeds

---
 youtube_dl/extractor/generic.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 376feecae..5d41b6319 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1718,6 +1718,15 @@ class GenericIE(InfoExtractor):
         if snagfilms_url:
             return self.url_result(snagfilms_url)
 
+        # Look for ScreenwaveMedia embeds
+        mobj = re.search(
+            r'<script[^>]+src=[\'"](http://player\d?\.screenwavemedia\.com/player\.php\?id=[A-Za-z0-9-]+)[\'"]',
+            webpage)
+        if mobj is not None:
+            return self.url_result(
+                unescapeHTML(mobj.group(1)),
+                'ScreenwaveMedia')
+
         # Look for AdobeTVVideo embeds
         mobj = re.search(
             r'<iframe[^>]+src=[\'"]((?:https?:)?//video\.tv\.adobe\.com/v/\d+[^"]+)[\'"]',

From 9b22cb10c486b827102093d05abeb21a4d4913b7 Mon Sep 17 00:00:00 2001
From: felix <felix.von.s@posteo.de>
Date: Sat, 29 Aug 2015 08:00:25 +0200
Subject: [PATCH 2365/6219] [screenwavemedia] use the true JS value instead of
 a comment

---
 youtube_dl/extractor/screenwavemedia.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/screenwavemedia.py b/youtube_dl/extractor/screenwavemedia.py
index 220d39078..e1604b5c0 100644
--- a/youtube_dl/extractor/screenwavemedia.py
+++ b/youtube_dl/extractor/screenwavemedia.py
@@ -33,7 +33,7 @@ class ScreenwaveMediaIE(InfoExtractor):
             'http://player.screenwavemedia.com/player.js',
             video_id, 'Downloading playerconfig webpage')
 
-        videoserver = self._search_regex(r"\[ipaddress\]\s*=>\s*([\d\.]+)", playerdata, 'videoserver')
+        videoserver = self._search_regex(r'SWMServer\s*=\s*"([\d\.]+)"', playerdata, 'videoserver')
 
         sources = self._parse_json(
             js_to_json(

From e276fd2cb39a94b1f40f47a5c52913aac8557976 Mon Sep 17 00:00:00 2001
From: felix <felix.von.s@posteo.de>
Date: Sat, 29 Aug 2015 08:01:10 +0200
Subject: [PATCH 2366/6219] [screenwavemedia] warn when falling back to a
 hardcoded list of streams

---
 youtube_dl/extractor/screenwavemedia.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/screenwavemedia.py b/youtube_dl/extractor/screenwavemedia.py
index e1604b5c0..671ea57d0 100644
--- a/youtube_dl/extractor/screenwavemedia.py
+++ b/youtube_dl/extractor/screenwavemedia.py
@@ -56,6 +56,7 @@ class ScreenwaveMediaIE(InfoExtractor):
 
         # Fallback to hardcoded sources if JS changes again
         if not sources:
+            self.report_warning('Falling back to a hardcoded list of streams')
             sources = [{
                 'file': 'http://%s/vod/%s_%s.mp4' % (videoserver, video_id, format_id),
                 'type': 'mp4',

From 5e58956d0a057e7294af15bef5a22c675812a18b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 29 Aug 2015 18:57:44 +0600
Subject: [PATCH 2367/6219] [screenwavemedia] Add EMBED_PATTERN

---
 youtube_dl/extractor/screenwavemedia.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/screenwavemedia.py b/youtube_dl/extractor/screenwavemedia.py
index 671ea57d0..4d1379615 100644
--- a/youtube_dl/extractor/screenwavemedia.py
+++ b/youtube_dl/extractor/screenwavemedia.py
@@ -13,7 +13,7 @@ from ..utils import (
 
 class ScreenwaveMediaIE(InfoExtractor):
     _VALID_URL = r'http://player\d?\.screenwavemedia\.com/(?:play/)?[a-zA-Z]+\.php\?[^"]*\bid=(?P<id>.+)'
-
+    EMBED_PATTERN = r'src=(["\'])(?P<url>(?:https?:)?//player\d?\.screenwavemedia\.com/(?:play/)?[a-zA-Z]+\.php\?.*\bid=.+?)\1'
     _TESTS = [{
         'url': 'http://player.screenwavemedia.com/play/play.php?playerdiv=videoarea&companiondiv=squareAd&id=Cinemassacre-19911',
         'only_matching': True,

From 678d33295ba45833cedc2abab9c906d12dae9bee Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 29 Aug 2015 19:05:11 +0600
Subject: [PATCH 2368/6219] [screenwavemedia] Improve _VALID_URL

---
 youtube_dl/extractor/screenwavemedia.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/screenwavemedia.py b/youtube_dl/extractor/screenwavemedia.py
index 4d1379615..05f93904c 100644
--- a/youtube_dl/extractor/screenwavemedia.py
+++ b/youtube_dl/extractor/screenwavemedia.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 
 class ScreenwaveMediaIE(InfoExtractor):
-    _VALID_URL = r'http://player\d?\.screenwavemedia\.com/(?:play/)?[a-zA-Z]+\.php\?[^"]*\bid=(?P<id>.+)'
+    _VALID_URL = r'https?://player\d?\.screenwavemedia\.com/(?:play/)?[a-zA-Z]+\.php\?.*\bid=(?P<id>[A-Za-z0-9-]+)'
     EMBED_PATTERN = r'src=(["\'])(?P<url>(?:https?:)?//player\d?\.screenwavemedia\.com/(?:play/)?[a-zA-Z]+\.php\?.*\bid=.+?)\1'
     _TESTS = [{
         'url': 'http://player.screenwavemedia.com/play/play.php?playerdiv=videoarea&companiondiv=squareAd&id=Cinemassacre-19911',

From 109a4156e17b2c0dee414151fa37db2d208df43d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 29 Aug 2015 19:06:55 +0600
Subject: [PATCH 2369/6219] [cinemassacre] Use screenwavemedia embed pattern

---
 youtube_dl/extractor/cinemassacre.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/cinemassacre.py b/youtube_dl/extractor/cinemassacre.py
index c949a4814..fd1770dac 100644
--- a/youtube_dl/extractor/cinemassacre.py
+++ b/youtube_dl/extractor/cinemassacre.py
@@ -6,6 +6,7 @@ import re
 from .common import InfoExtractor
 from ..utils import ExtractorError
 from .bliptv import BlipTVIE
+from .screenwavemedia import ScreenwaveMediaIE
 
 
 class CinemassacreIE(InfoExtractor):
@@ -83,10 +84,10 @@ class CinemassacreIE(InfoExtractor):
 
         playerdata_url = self._search_regex(
             [
-                r'src="(http://(?:player2\.screenwavemedia\.com|player\.screenwavemedia\.com/play)/[a-zA-Z]+\.php\?[^"]*\bid=.+?)"',
-                r'<iframe[^>]+src="((?:https?:)?//(?:[^.]+\.)?youtube\.com/.+?)"',
+                ScreenwaveMediaIE.EMBED_PATTERN,
+                r'<iframe[^>]+src="(?P<url>(?:https?:)?//(?:[^.]+\.)?youtube\.com/.+?)"',
             ],
-            webpage, 'player data URL', default=None)
+            webpage, 'player data URL', default=None, group='url')
         if not playerdata_url:
             playerdata_url = BlipTVIE._extract_url(webpage)
         if not playerdata_url:

From efd712c69b6c77d216c1ec3e6243e49254f5e37f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 29 Aug 2015 19:07:31 +0600
Subject: [PATCH 2370/6219] [generic] Use screenwavemedia embed pattern

---
 youtube_dl/extractor/generic.py | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 5d41b6319..c9b36221e 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -48,6 +48,7 @@ from .vimeo import VimeoIE
 from .dailymotion import DailymotionCloudIE
 from .onionstudios import OnionStudiosIE
 from .snagfilms import SnagFilmsEmbedIE
+from .screenwavemedia import ScreenwaveMediaIE
 
 
 class GenericIE(InfoExtractor):
@@ -1719,13 +1720,9 @@ class GenericIE(InfoExtractor):
             return self.url_result(snagfilms_url)
 
         # Look for ScreenwaveMedia embeds
-        mobj = re.search(
-            r'<script[^>]+src=[\'"](http://player\d?\.screenwavemedia\.com/player\.php\?id=[A-Za-z0-9-]+)[\'"]',
-            webpage)
+        mobj = re.search(ScreenwaveMediaIE.EMBED_PATTERN, webpage)
         if mobj is not None:
-            return self.url_result(
-                unescapeHTML(mobj.group(1)),
-                'ScreenwaveMedia')
+            return self.url_result(unescapeHTML(mobj.group('url')), 'ScreenwaveMedia')
 
         # Look for AdobeTVVideo embeds
         mobj = re.search(

From 1f8125805e7af127efd331a41775b7b5c90a77c6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 29 Aug 2015 19:12:38 +0600
Subject: [PATCH 2371/6219] [generic] Add test for screenwavemedia embed

---
 youtube_dl/extractor/generic.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index c9b36221e..953ec32c3 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1002,6 +1002,16 @@ class GenericIE(InfoExtractor):
                 'description': 'New experience with Acrobat DC',
                 'duration': 248.667,
             },
+        },
+        # ScreenwaveMedia embed
+        {
+            'url': 'http://www.thecinemasnob.com/the-cinema-snob/a-nightmare-on-elm-street-2-freddys-revenge1',
+            'md5': '24ace5baba0d35d55c6810b51f34e9e0',
+            'info_dict': {
+                'id': 'cinemasnob-55d26273809dd',
+                'ext': 'mp4',
+                'title': 'cinemasnob',
+            },
         }
     ]
 

From ef49b5905345255bc6913e279b9d76a0bb83e1ce Mon Sep 17 00:00:00 2001
From: sourcerect <hypershadsy@gmail.com>
Date: Sat, 29 Aug 2015 02:12:15 -0400
Subject: [PATCH 2372/6219] [kaltura] add html5 player urls

---
 youtube_dl/extractor/kaltura.py | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/kaltura.py b/youtube_dl/extractor/kaltura.py
index d28730492..1f64df4d5 100644
--- a/youtube_dl/extractor/kaltura.py
+++ b/youtube_dl/extractor/kaltura.py
@@ -14,11 +14,12 @@ from ..utils import (
 class KalturaIE(InfoExtractor):
     _VALID_URL = r'''(?x)
     (?:kaltura:|
-       https?://(:?(?:www|cdnapisec)\.)?kaltura\.com/index\.php/kwidget/(?:[^/]+/)*?wid/_
-    )(?P<partner_id>\d+)
-    (?::|
-       /(?:[^/]+/)*?entry_id/
-    )(?P<id>[0-9a-z_]+)'''
+        https?://(:?(?:www|cdnapisec)\.)?kaltura\.com/(?:
+            (?:index\.php/kwidget/(?:[^/]+/)*?wid/_)|
+            (?:html5/html5lib/v(?:[\d.]+)/mwEmbedFrame.php/p/\d+)
+        )
+    )(?P<partner_id>\d+)?(?::|/(?:[^/]+/)*?entry_id/)(?P<id>[0-9a-z_]+)
+    (?:\?wid=_(?P<partner_id_html5>\d+))?'''
     _API_BASE = 'http://cdnapi.kaltura.com/api_v3/index.php?'
     _TESTS = [
         {
@@ -43,6 +44,10 @@ class KalturaIE(InfoExtractor):
             'url': 'https://cdnapisec.kaltura.com/index.php/kwidget/wid/_557781/uiconf_id/22845202/entry_id/1_plr1syf3',
             'only_matching': True,
         },
+        {
+            'url': 'https://cdnapisec.kaltura.com/html5/html5lib/v2.30.2/mwEmbedFrame.php/p/1337/uiconf_id/20540612/entry_id/1_sf5ovm7u?wid=_243342',
+            'only_matching': True,
+        }
     ]
 
     def _kaltura_api_call(self, video_id, actions, *args, **kwargs):
@@ -107,7 +112,7 @@ class KalturaIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
         mobj = re.match(self._VALID_URL, url)
-        partner_id, entry_id = mobj.group('partner_id'), mobj.group('id')
+        partner_id, entry_id = mobj.group('partner_id') or mobj.group('partner_id_html5'), mobj.group('id')
 
         info, source_data = self._get_video_info(entry_id, partner_id)
 

From ee3ec091f46bd52979274d0aa42ff1f1c3344de6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 29 Aug 2015 20:21:59 +0600
Subject: [PATCH 2373/6219] [kaltura] Strictify _VALID_URL

---
 youtube_dl/extractor/kaltura.py | 32 ++++++++++++++++++++++----------
 1 file changed, 22 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/kaltura.py b/youtube_dl/extractor/kaltura.py
index 1f64df4d5..c57f33ece 100644
--- a/youtube_dl/extractor/kaltura.py
+++ b/youtube_dl/extractor/kaltura.py
@@ -13,13 +13,25 @@ from ..utils import (
 
 class KalturaIE(InfoExtractor):
     _VALID_URL = r'''(?x)
-    (?:kaltura:|
-        https?://(:?(?:www|cdnapisec)\.)?kaltura\.com/(?:
-            (?:index\.php/kwidget/(?:[^/]+/)*?wid/_)|
-            (?:html5/html5lib/v(?:[\d.]+)/mwEmbedFrame.php/p/\d+)
-        )
-    )(?P<partner_id>\d+)?(?::|/(?:[^/]+/)*?entry_id/)(?P<id>[0-9a-z_]+)
-    (?:\?wid=_(?P<partner_id_html5>\d+))?'''
+                (?:
+                    kaltura:(?P<partner_id_s>\d+):(?P<id_s>[0-9a-z_]+)|
+                    https?://
+                    (:?
+                        (?:www|cdnapisec)\.)?kaltura\.com/
+                        (?:
+                            (?:
+                                # flash player
+                                index\.php/kwidget/
+                                (?:[^/]+/)*?wid/_(?P<partner_id>\d+)/
+                                (?:[^/]+/)*?entry_id/(?P<id>[0-9a-z_]+)|
+                                # html player
+                                html5/html5lib/
+                                (?:[^/]+/)*?entry_id/(?P<id_html5>[0-9a-z_]+)
+                                .*\?.*\bwid=_(?P<partner_id_html5>\d+)
+                        )
+                    )
+                )
+                '''
     _API_BASE = 'http://cdnapi.kaltura.com/api_v3/index.php?'
     _TESTS = [
         {
@@ -110,9 +122,9 @@ class KalturaIE(InfoExtractor):
             video_id, actions, note='Downloading video info JSON')
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
         mobj = re.match(self._VALID_URL, url)
-        partner_id, entry_id = mobj.group('partner_id') or mobj.group('partner_id_html5'), mobj.group('id')
+        partner_id = mobj.group('partner_id_s') or mobj.group('partner_id') or mobj.group('partner_id_html5')
+        entry_id = mobj.group('id_s') or mobj.group('id') or mobj.group('id_html5')
 
         info, source_data = self._get_video_info(entry_id, partner_id)
 
@@ -131,7 +143,7 @@ class KalturaIE(InfoExtractor):
         self._sort_formats(formats)
 
         return {
-            'id': video_id,
+            'id': entry_id,
             'title': info['name'],
             'formats': formats,
             'description': info.get('description'),

From b184f94413ea2837a52816e3beb009f386351d70 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 29 Aug 2015 20:23:06 +0600
Subject: [PATCH 2374/6219] [kaltura] Clarify html5 player comment

---
 youtube_dl/extractor/kaltura.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/kaltura.py b/youtube_dl/extractor/kaltura.py
index c57f33ece..c5b981092 100644
--- a/youtube_dl/extractor/kaltura.py
+++ b/youtube_dl/extractor/kaltura.py
@@ -24,7 +24,7 @@ class KalturaIE(InfoExtractor):
                                 index\.php/kwidget/
                                 (?:[^/]+/)*?wid/_(?P<partner_id>\d+)/
                                 (?:[^/]+/)*?entry_id/(?P<id>[0-9a-z_]+)|
-                                # html player
+                                # html5 player
                                 html5/html5lib/
                                 (?:[^/]+/)*?entry_id/(?P<id_html5>[0-9a-z_]+)
                                 .*\?.*\bwid=_(?P<partner_id_html5>\d+)

From ae6a802106663a5bb0f94803dec21bf5feaa2527 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 29 Aug 2015 20:26:51 +0600
Subject: [PATCH 2375/6219] [kaltura] Fix _VALID_URL formatting

---
 youtube_dl/extractor/kaltura.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/kaltura.py b/youtube_dl/extractor/kaltura.py
index c5b981092..3dca0e566 100644
--- a/youtube_dl/extractor/kaltura.py
+++ b/youtube_dl/extractor/kaltura.py
@@ -16,8 +16,7 @@ class KalturaIE(InfoExtractor):
                 (?:
                     kaltura:(?P<partner_id_s>\d+):(?P<id_s>[0-9a-z_]+)|
                     https?://
-                    (:?
-                        (?:www|cdnapisec)\.)?kaltura\.com/
+                        (:?(?:www|cdnapisec)\.)?kaltura\.com/
                         (?:
                             (?:
                                 # flash player
@@ -28,8 +27,8 @@ class KalturaIE(InfoExtractor):
                                 html5/html5lib/
                                 (?:[^/]+/)*?entry_id/(?P<id_html5>[0-9a-z_]+)
                                 .*\?.*\bwid=_(?P<partner_id_html5>\d+)
+                            )
                         )
-                    )
                 )
                 '''
     _API_BASE = 'http://cdnapi.kaltura.com/api_v3/index.php?'

From c88e118b3cfb611bf2eae300a13885772481134f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 29 Aug 2015 22:41:49 +0600
Subject: [PATCH 2376/6219] [nowtv] Fix display_id (Closes #6702)

---
 youtube_dl/extractor/nowtv.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/nowtv.py b/youtube_dl/extractor/nowtv.py
index 66c627bec..c8257719f 100644
--- a/youtube_dl/extractor/nowtv.py
+++ b/youtube_dl/extractor/nowtv.py
@@ -130,10 +130,16 @@ class NowTVIE(InfoExtractor):
     }, {
         'url': 'http://www.nowtv.at/rtl/bauer-sucht-frau/die-neuen-bauern-und-eine-hochzeit/preview?return=/rtl/bauer-sucht-frau/die-neuen-bauern-und-eine-hochzeit',
         'only_matching': True,
+    }, {
+        'url': 'http://www.nowtv.de/rtl2/echtzeit/list/aktuell/schnelles-geld-am-ende-der-welt/player',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
+        display_id_split = display_id.split('/')
+        if len(display_id) > 2:
+            display_id = '/'.join((display_id_split[0], display_id_split[-1]))
 
         info = self._download_json(
             'https://api.nowtv.de/v3/movies/%s?fields=id,title,free,geoblocked,articleLong,articleShort,broadcastStartDate,seoUrl,duration,format,files' % display_id,

From 17a647630b504c730d77e71faf248628e45529c5 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 30 Aug 2015 01:08:55 +0800
Subject: [PATCH 2377/6219] [abc] Support YouTube embeds (fixes #6595)

---
 youtube_dl/extractor/abc.py | 42 ++++++++++++++++++++++++++++++-------
 1 file changed, 35 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/abc.py b/youtube_dl/extractor/abc.py
index dc0fb85d6..06acf50d8 100644
--- a/youtube_dl/extractor/abc.py
+++ b/youtube_dl/extractor/abc.py
@@ -1,16 +1,19 @@
 from __future__ import unicode_literals
 
 import re
-import json
 
 from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    js_to_json,
+)
 
 
 class ABCIE(InfoExtractor):
     IE_NAME = 'abc.net.au'
     _VALID_URL = r'http://www\.abc\.net\.au/news/[^/]+/[^/]+/(?P<id>\d+)'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.abc.net.au/news/2014-11-05/australia-to-staff-ebola-treatment-centre-in-sierra-leone/5868334',
         'md5': 'cb3dd03b18455a661071ee1e28344d9f',
         'info_dict': {
@@ -19,16 +22,41 @@ class ABCIE(InfoExtractor):
             'title': 'Australia to help staff Ebola treatment centre in Sierra Leone',
             'description': 'md5:809ad29c67a05f54eb41f2a105693a67',
         },
-    }
+    }, {
+        'url': 'http://www.abc.net.au/news/2015-08-17/warren-entsch-introduces-same-sex-marriage-bill/6702326',
+        'md5': 'db2a5369238b51f9811ad815b69dc086',
+        'info_dict': {
+            'id': 'NvqvPeNZsHU',
+            'ext': 'mp4',
+            'upload_date': '20150816',
+            'uploader': 'ABC News (Australia)',
+            'description': 'Government backbencher Warren Entsch introduces a cross-party sponsored bill to legalise same-sex marriage, saying the bill is designed to promote "an inclusive Australia, not a divided one.". Read more here: http://ab.co/1Mwc6ef',
+            'uploader_id': 'NewsOnABC',
+            'title': 'Marriage Equality: Warren Entsch introduces same sex marriage bill',
+        },
+        'add_ie': ['Youtube'],
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        urls_info_json = self._search_regex(
-            r'inlineVideoData\.push\((.*?)\);', webpage, 'video urls',
-            flags=re.DOTALL)
-        urls_info = json.loads(urls_info_json.replace('\'', '"'))
+        mobj = re.search(
+            r'inline(?P<type>Video|YouTube)Data\.push\((?P<json_data>[^)]+)\);',
+            webpage)
+        if mobj is None:
+            raise ExtractorError('Unable to extract video urls')
+
+        urls_info = self._parse_json(
+            mobj.group('json_data'), video_id, transform_source=js_to_json)
+
+        if not isinstance(urls_info, list):
+            urls_info = [urls_info]
+
+        if mobj.group('type') == 'YouTube':
+            return self.playlist_result([
+                self.url_result(url_info['url']) for url_info in urls_info])
+
         formats = [{
             'url': url_info['url'],
             'width': int(url_info['width']),

From c0a656876cf3831deab90d0937e7cd81e95f6184 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 30 Aug 2015 01:11:56 +0800
Subject: [PATCH 2378/6219] [abc] Use int_or_none

---
 youtube_dl/extractor/abc.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/abc.py b/youtube_dl/extractor/abc.py
index 06acf50d8..f9a389f67 100644
--- a/youtube_dl/extractor/abc.py
+++ b/youtube_dl/extractor/abc.py
@@ -6,6 +6,7 @@ from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
     js_to_json,
+    int_or_none,
 )
 
 
@@ -59,10 +60,10 @@ class ABCIE(InfoExtractor):
 
         formats = [{
             'url': url_info['url'],
-            'width': int(url_info['width']),
-            'height': int(url_info['height']),
-            'tbr': int(url_info['bitrate']),
-            'filesize': int(url_info['filesize']),
+            'width': int_or_none(url_info.get('width')),
+            'height': int_or_none(url_info.get('height')),
+            'tbr': int_or_none(url_info.get('bitrate')),
+            'filesize': int_or_none(url_info.get('filesize')),
         } for url_info in urls_info]
         self._sort_formats(formats)
 

From c9afb51ceaeae7cbd74502e0451eb9163a1ab7fb Mon Sep 17 00:00:00 2001
From: Anssi Hannula <anssi.hannula@iki.fi>
Date: Mon, 15 Jun 2015 20:06:43 +0300
Subject: [PATCH 2379/6219] [youtube] Fix missing format details for 60fps DASH
 formats

60fps DASH formats do not appear in the DASH manifest, but the non-DASH
video info page does contain additional parameters for DASH formats that
we can parse.

Use those when they exist.

This fixes "bestvideo" not selecting 60fps formats over similar 30fps
formats just because the file size is unknown.
---
 youtube_dl/extractor/youtube.py | 31 ++++++++++++++++++++++++++++---
 1 file changed, 28 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index ab6754154..3a2c7c562 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1243,7 +1243,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             encoded_url_map = video_info.get('url_encoded_fmt_stream_map', [''])[0] + ',' + video_info.get('adaptive_fmts', [''])[0]
             if 'rtmpe%3Dyes' in encoded_url_map:
                 raise ExtractorError('rtmpe downloads are not supported, see https://github.com/rg3/youtube-dl/issues/343 for more information.', expected=True)
-            url_map = {}
+            formats = []
             for url_data_str in encoded_url_map.split(','):
                 url_data = compat_parse_qs(url_data_str)
                 if 'itag' not in url_data or 'url' not in url_data:
@@ -1303,8 +1303,33 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     url += '&signature=' + signature
                 if 'ratebypass' not in url:
                     url += '&ratebypass=yes'
-                url_map[format_id] = url
-            formats = _map_to_format_list(url_map)
+
+                width = None
+                height = None
+                size_str = url_data.get('size', [''])[0]
+                if size_str.count('x') == 1:
+                    width, height = [int_or_none(x) for x in size_str.split('x')]
+
+                format_url = {
+                    'format_id': format_id,
+                    'url': url,
+                    'player_url': player_url,
+                    # As of this writing these are only defined for DASH formats:
+                    'filesize': int_or_none(url_data.get('clen', [None])[0]),
+                    'tbr': float_or_none(url_data.get('bitrate', [None])[0], scale=1024),
+                    'width': width,
+                    'height': height,
+                    'fps': int_or_none(url_data.get('fps', [None])[0]),
+                }
+
+                # drop Nones so they do not overwrite the defaults from self._formats
+                format_url = dict((k, v) for k, v in format_url.items() if v is not None)
+
+                format_full = self._formats.get(format_id, {}).copy()
+                format_full.update(format_url)
+
+                formats.append(format_full)
+
         elif video_info.get('hlsvp'):
             manifest_url = video_info['hlsvp'][0]
             url_map = self._extract_from_m3u8(manifest_url, video_id)

From aabc2be69316550cbec486bbde5695d1ae13ee9c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 30 Aug 2015 08:07:07 +0600
Subject: [PATCH 2380/6219] [youtube] Simplify and extract more metadata from
 url_encoded_fmt_stream_map (Closes #5993)

---
 youtube_dl/extractor/youtube.py | 53 ++++++++++++++++++++++-----------
 1 file changed, 35 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 3a2c7c562..030ec70ca 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1304,32 +1304,49 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 if 'ratebypass' not in url:
                     url += '&ratebypass=yes'
 
-                width = None
-                height = None
-                size_str = url_data.get('size', [''])[0]
-                if size_str.count('x') == 1:
-                    width, height = [int_or_none(x) for x in size_str.split('x')]
-
-                format_url = {
+                # Some itags are not included in DASH manifest thus corresponding formats will
+                # lack metadata (see https://github.com/rg3/youtube-dl/pull/5993).
+                # Trying to extract metadata from url_encoded_fmt_stream_map entry.
+                mobj = re.search(r'^(?P<width>\d+)[xX](?P<height>\d+)$', url_data.get('size', [''])[0])
+                width, height = (int(mobj.group('width')), int(mobj.group('height'))) if mobj else (None, None)
+                dct = {
                     'format_id': format_id,
                     'url': url,
                     'player_url': player_url,
-                    # As of this writing these are only defined for DASH formats:
                     'filesize': int_or_none(url_data.get('clen', [None])[0]),
-                    'tbr': float_or_none(url_data.get('bitrate', [None])[0], scale=1024),
+                    'tbr': float_or_none(url_data.get('bitrate', [None])[0], 1000),
                     'width': width,
                     'height': height,
                     'fps': int_or_none(url_data.get('fps', [None])[0]),
+                    'format_note': url_data.get('quality_label', [None])[0] or url_data.get('quality', [None])[0],
                 }
-
-                # drop Nones so they do not overwrite the defaults from self._formats
-                format_url = dict((k, v) for k, v in format_url.items() if v is not None)
-
-                format_full = self._formats.get(format_id, {}).copy()
-                format_full.update(format_url)
-
-                formats.append(format_full)
-
+                type_ = url_data.get('type', [None])[0]
+                if type_:
+                    type_split = type_.split(';')
+                    kind_ext = type_split[0].split('/')
+                    if len(kind_ext) == 2:
+                        kind, ext = kind_ext
+                        dct['ext'] = ext
+                        if kind in ('audio', 'video'):
+                            codecs = None
+                            for mobj in re.finditer(
+                                    r'(?P<key>[a-zA-Z_-]+)=(?P<quote>["\']?)(?P<val>.+?)(?P=quote)(?:;|$)', type_):
+                                if mobj.group('key') == 'codecs':
+                                    codecs = mobj.group('val')
+                                    break
+                            if codecs:
+                                codecs = codecs.split(',')
+                                if len(codecs) == 2:
+                                    acodec, vcodec = codecs[0], codecs[1]
+                                else:
+                                    acodec, vcodec = (codecs[0], 'none') if kind == 'audio' else ('none', codecs[0])
+                                dct.update({
+                                    'acodec': acodec,
+                                    'vcodec': vcodec,
+                                })
+                if format_id in self._formats:
+                    dct.update(self._formats[format_id])
+                formats.append(dct)
         elif video_info.get('hlsvp'):
             manifest_url = video_info['hlsvp'][0]
             url_map = self._extract_from_m3u8(manifest_url, video_id)

From 8875b3d572ba37185fa632fa909bb6f9ecda8218 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 30 Aug 2015 19:03:21 +0600
Subject: [PATCH 2381/6219] [imgut:album] Add extractor (Closes #6711)

---
 youtube_dl/extractor/__init__.py |  5 ++++-
 youtube_dl/extractor/imgur.py    | 27 ++++++++++++++++++++++++++-
 2 files changed, 30 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 66422b005..39b05ce8f 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -241,7 +241,10 @@ from .imdb import (
     ImdbIE,
     ImdbListIE
 )
-from .imgur import ImgurIE
+from .imgur import (
+    ImgurIE,
+    ImgurAlbumIE,
+)
 from .ina import InaIE
 from .indavideo import (
     IndavideoIE,
diff --git a/youtube_dl/extractor/imgur.py b/youtube_dl/extractor/imgur.py
index d692ea79a..70c8ca64e 100644
--- a/youtube_dl/extractor/imgur.py
+++ b/youtube_dl/extractor/imgur.py
@@ -13,7 +13,7 @@ from ..utils import (
 
 
 class ImgurIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:i\.)?imgur\.com/(?P<id>[a-zA-Z0-9]+)'
+    _VALID_URL = r'https?://(?:i\.)?imgur\.com/(?!gallery)(?P<id>[a-zA-Z0-9]+)'
 
     _TESTS = [{
         'url': 'https://i.imgur.com/A61SaA1.gifv',
@@ -97,3 +97,28 @@ class ImgurIE(InfoExtractor):
             'description': self._og_search_description(webpage),
             'title': self._og_search_title(webpage),
         }
+
+
+class ImgurAlbumIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:i\.)?imgur\.com/gallery/(?P<id>[a-zA-Z0-9]+)'
+
+    _TEST = {
+        'url': 'http://imgur.com/gallery/Q95ko',
+        'info_dict': {
+            'id': 'Q95ko',
+        },
+        'playlist_count': 25,
+    }
+
+    def _real_extract(self, url):
+        album_id = self._match_id(url)
+
+        album_images = self._download_json(
+            'http://imgur.com/gallery/%s/album_images/hit.json?all=true' % album_id,
+            album_id)['data']['images']
+
+        entries = [
+            self.url_result('http://imgur.com/%s' % image['hash'])
+            for image in album_images if image.get('hash')]
+
+        return self.playlist_result(entries, album_id)

From d3d89c325681b1f2d9dcc8d8dea3439aad570f1f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 31 Aug 2015 02:01:13 +0600
Subject: [PATCH 2382/6219] [YoutubeDL] Encode thumbnail filename (Closes
 #6714)

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index cad6b026e..982e658ce 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -2009,7 +2009,7 @@ class YoutubeDL(object):
                                (info_dict['extractor'], info_dict['id'], thumb_display_id))
                 try:
                     uf = self.urlopen(t['url'])
-                    with open(thumb_filename, 'wb') as thumbf:
+                    with open(encodeFilename(thumb_filename), 'wb') as thumbf:
                         shutil.copyfileobj(uf, thumbf)
                     self.to_screen('[%s] %s: Writing thumbnail %sto: %s' %
                                    (info_dict['extractor'], info_dict['id'], thumb_display_id, thumb_filename))

From db8f2bfd99fccd3ac4d96b49cf34056962a5652f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 31 Aug 2015 02:20:29 +0600
Subject: [PATCH 2383/6219] [f4m] Add missing imports

---
 youtube_dl/downloader/f4m.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index f478fc03c..174180db5 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -13,6 +13,8 @@ from ..compat import (
     compat_urllib_error,
 )
 from ..utils import (
+    encodeFilename,
+    sanitize_open,
     struct_pack,
     struct_unpack,
     xpath_text,

From fcd9e423ec15ac51844786541e1f5cb464eb429a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 31 Aug 2015 02:28:36 +0600
Subject: [PATCH 2384/6219] [hls] Properly encode fragment filename

---
 youtube_dl/downloader/hls.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index 2b6c3370f..71aafdc73 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -12,6 +12,7 @@ from ..postprocessor.ffmpeg import FFmpegPostProcessor
 from ..utils import (
     encodeArgument,
     encodeFilename,
+    sanitize_open,
 )
 
 
@@ -89,13 +90,13 @@ class NativeHlsFD(FragmentFD):
             success = ctx['dl'].download(frag_filename, {'url': frag_url})
             if not success:
                 return False
-            with open(frag_filename, 'rb') as down:
-                ctx['dest_stream'].write(down.read())
-            frags_filenames.append(frag_filename)
+            down, frag_sanitized = sanitize_open(frag_filename, 'rb')
+            ctx['dest_stream'].write(down.read())
+            frags_filenames.append(frag_sanitized)
 
         self._finish_frag_download(ctx)
 
         for frag_file in frags_filenames:
-            os.remove(frag_file)
+            os.remove(encodeFilename(frag_file))
 
         return True

From 615f155a3afd5c854fbf855c02f17ddb7f217b05 Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Mon, 31 Aug 2015 12:46:28 +0800
Subject: [PATCH 2385/6219] [vlive] Fixes for review

---
 youtube_dl/extractor/vlive.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vlive.py b/youtube_dl/extractor/vlive.py
index 6a403cc64..17ae3b964 100644
--- a/youtube_dl/extractor/vlive.py
+++ b/youtube_dl/extractor/vlive.py
@@ -39,7 +39,7 @@ class VLiveIE(InfoExtractor):
         title = self._og_search_title(webpage)
         thumbnail = self._og_search_thumbnail(webpage)
         creator = self._html_search_regex(
-            r'<span class="name">([^<>]+)</span>', webpage, 'creator')
+            r'<span[^>]+class="name">([^<>]+)</span>', webpage, 'creator')
         
         url = 'http://global.apis.naver.com/globalV/globalV/vod/%s/playinfo?' % video_id
         msgpad = '%.0f' % (time() * 1000)
@@ -51,7 +51,7 @@ class VLiveIE(InfoExtractor):
         playinfo = self._download_json(url, video_id, 'Downloading video json')
 
         if playinfo.get('message', '') != 'success':
-            raise ExtractorError(playinfo['message'])
+            raise ExtractorError(playinfo.get('message', 'JSON request unsuccessful'))
 
         if not playinfo.get('result'):
             raise ExtractorError('No videos found.')

From 93462856e107237f85295357f978f2207d15723c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 31 Aug 2015 21:05:23 +0600
Subject: [PATCH 2386/6219] [fc2] Relax _VALID_URL (Closes #3171)

---
 youtube_dl/extractor/fc2.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/fc2.py b/youtube_dl/extractor/fc2.py
index e4f7195a8..1c6dc5bd9 100644
--- a/youtube_dl/extractor/fc2.py
+++ b/youtube_dl/extractor/fc2.py
@@ -15,7 +15,7 @@ from ..utils import (
 
 
 class FC2IE(InfoExtractor):
-    _VALID_URL = r'^http://video\.fc2\.com/(?:[^/]+/)?content/(?P<id>[^/]+)'
+    _VALID_URL = r'^http://video\.fc2\.com/(?:[^/]+/)*content/(?P<id>[^/]+)'
     IE_NAME = 'fc2'
     _NETRC_MACHINE = 'fc2'
     _TESTS = [{

From 52dfb7ffe2efdd223872bfe020ddaa25b89a94f9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 31 Aug 2015 21:05:37 +0600
Subject: [PATCH 2387/6219] [fc2] Add test for #3171

---
 youtube_dl/extractor/fc2.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/fc2.py b/youtube_dl/extractor/fc2.py
index 1c6dc5bd9..2443846c8 100644
--- a/youtube_dl/extractor/fc2.py
+++ b/youtube_dl/extractor/fc2.py
@@ -37,6 +37,9 @@ class FC2IE(InfoExtractor):
             'password': '(snip)',
             'skip': 'requires actual password'
         }
+    }, {
+        'url': 'http://video.fc2.com/en/a/content/20130926eZpARwsF',
+        'only_matching': True,
     }]
 
     def _login(self):

From 39955b0451098c6a79fc7b777a0674ee093ae0ff Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 31 Aug 2015 21:12:21 +0600
Subject: [PATCH 2388/6219] [fc2] Fix refer

---
 youtube_dl/extractor/fc2.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/fc2.py b/youtube_dl/extractor/fc2.py
index 2443846c8..5c1137e94 100644
--- a/youtube_dl/extractor/fc2.py
+++ b/youtube_dl/extractor/fc2.py
@@ -83,7 +83,7 @@ class FC2IE(InfoExtractor):
 
         title = self._og_search_title(webpage)
         thumbnail = self._og_search_thumbnail(webpage)
-        refer = url.replace('/content/', '/a/content/')
+        refer = url.replace('/content/', '/a/content/') if '/a/content/' not in url else url
 
         mimi = hashlib.md5((video_id + '_gGddgPfeaf_gzyr').encode('utf-8')).hexdigest()
 

From d14f0c45fc8ab447d66caf7a3b9ce0888446eb7c Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Tue, 1 Sep 2015 11:11:14 +0800
Subject: [PATCH 2389/6219] [vlive] Add info about  www. to m. redirects for
 mobile

---
 youtube_dl/extractor/vlive.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/vlive.py b/youtube_dl/extractor/vlive.py
index 17ae3b964..a456f8217 100644
--- a/youtube_dl/extractor/vlive.py
+++ b/youtube_dl/extractor/vlive.py
@@ -16,6 +16,7 @@ from ..compat import compat_urllib_parse
 
 class VLiveIE(InfoExtractor):
     IE_NAME = 'vlive'
+    # www.vlive.tv/video/ links redirect to m.vlive.tv/video/ for mobile devices
     _VALID_URL = r'https?://(?:(www|m)\.)?vlive\.tv/video/(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://m.vlive.tv/video/1326',

From cdc8d0c37325e83da0c08ee4d97d81040e10be91 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 1 Sep 2015 16:34:05 +0800
Subject: [PATCH 2390/6219] [odnoklassniki] Fix extraction and extend
 _VALID_URL (closes #6726)

---
 youtube_dl/extractor/odnoklassniki.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/odnoklassniki.py b/youtube_dl/extractor/odnoklassniki.py
index 003d27de7..f552e6f7e 100644
--- a/youtube_dl/extractor/odnoklassniki.py
+++ b/youtube_dl/extractor/odnoklassniki.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 
 class OdnoklassnikiIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:odnoklassniki|ok)\.ru/(?:video|web-api/video/moviePlayer)/(?P<id>[\d-]+)'
+    _VALID_URL = r'https?://(?:www\.)?(?:odnoklassniki|ok)\.ru/(?:video|web-api/video/moviePlayer)/(?P<id>[\d-]+)'
     _TESTS = [{
         # metadata in JSON
         'url': 'http://ok.ru/video/20079905452',
@@ -46,6 +46,9 @@ class OdnoklassnikiIE(InfoExtractor):
     }, {
         'url': 'http://ok.ru/web-api/video/moviePlayer/20079905452',
         'only_matching': True,
+    }, {
+        'url': 'http://www.ok.ru/video/20648036891',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
@@ -56,7 +59,7 @@ class OdnoklassnikiIE(InfoExtractor):
 
         player = self._parse_json(
             unescapeHTML(self._search_regex(
-                r'data-attributes="([^"]+)"', webpage, 'player')),
+                r'data-options="([^"]+)"', webpage, 'player')),
             video_id)
 
         flashvars = player['flashvars']

From fffccaaf414800e24a5ebc9e90c324aaa37cd3a9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 1 Sep 2015 20:41:52 +0600
Subject: [PATCH 2391/6219] [globo] Fix extraction and make more robust (Closes
 #6728)

---
 youtube_dl/extractor/globo.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/globo.py b/youtube_dl/extractor/globo.py
index 8a95793ca..33d6432a6 100644
--- a/youtube_dl/extractor/globo.py
+++ b/youtube_dl/extractor/globo.py
@@ -13,6 +13,7 @@ from ..compat import (
 from ..utils import (
     ExtractorError,
     float_or_none,
+    int_or_none,
 )
 
 
@@ -359,13 +360,8 @@ class GloboIE(InfoExtractor):
             self._API_URL_TEMPLATE % video_id, video_id)['videos'][0]
 
         title = video['title']
-        duration = float_or_none(video['duration'], 1000)
-        like_count = video['likes']
-        uploader = video['channel']
-        uploader_id = video['channel_id']
 
         formats = []
-
         for resource in video['resources']:
             resource_id = resource.get('_id')
             if not resource_id:
@@ -407,6 +403,11 @@ class GloboIE(InfoExtractor):
 
         self._sort_formats(formats)
 
+        duration = float_or_none(video.get('duration'), 1000)
+        like_count = int_or_none(video.get('likes'))
+        uploader = video.get('channel')
+        uploader_id = video.get('channel_id')
+
         return {
             'id': video_id,
             'title': title,

From aa3f98677d0ca7f8feae63cf2a5c62dbdda6b586 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 1 Sep 2015 18:13:33 +0200
Subject: [PATCH 2392/6219] [dumpert] Support 'embed/' urls (fixes #6712)

---
 youtube_dl/extractor/dumpert.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/dumpert.py b/youtube_dl/extractor/dumpert.py
index 999fb5620..1f00386fe 100644
--- a/youtube_dl/extractor/dumpert.py
+++ b/youtube_dl/extractor/dumpert.py
@@ -9,8 +9,8 @@ from ..utils import qualities
 
 
 class DumpertIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?dumpert\.nl/mediabase/(?P<id>[0-9]+/[0-9a-zA-Z]+)'
-    _TEST = {
+    _VALID_URL = r'https?://(?:www\.)?dumpert\.nl/(?:mediabase|embed)/(?P<id>[0-9]+/[0-9a-zA-Z]+)'
+    _TESTS = [{
         'url': 'http://www.dumpert.nl/mediabase/6646981/951bc60f/',
         'md5': '1b9318d7d5054e7dcb9dc7654f21d643',
         'info_dict': {
@@ -20,11 +20,15 @@ class DumpertIE(InfoExtractor):
             'description': 'Niet schrikken hoor',
             'thumbnail': 're:^https?://.*\.jpg$',
         }
-    }
+    }, {
+        'url': 'http://www.dumpert.nl/embed/6675421/dc440fe7/',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
+        url = 'https://www.dumpert.nl/mediabase/' + video_id
         req = compat_urllib_request.Request(url)
         req.add_header('Cookie', 'nsfw=1; cpc=10')
         webpage = self._download_webpage(req, video_id)

From 33eae08f0422fb824cea2a4dffc98a347e916169 Mon Sep 17 00:00:00 2001
From: PSJay <psjay.peng@gmail.com>
Date: Tue, 1 Sep 2015 00:19:58 +0800
Subject: [PATCH 2393/6219] Support video-password on Youku.

---
 youtube_dl/extractor/youku.py | 24 +++++++++++++++++++++---
 1 file changed, 21 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index 78caeb8b3..220f84221 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -49,6 +49,17 @@ class YoukuIE(InfoExtractor):
         },
         'playlist_count': 13,
         'skip': 'Available in China only',
+    }, {
+        'url': 'http://v.youku.com/v_show/id_XNjA1NzA2Njgw.html',
+        'note': 'Video protected with password',
+        'info_dict': {
+            'id': 'XNjA1NzA2Njgw',
+            'ext': 'flv',
+            'title': u'邢義田复旦讲座之想象中的胡人—从“左衽孔子”说起',
+        },
+        'params': {
+            'videopassword': '100600',
+        },
     }]
 
     def construct_video_urls(self, data1, data2):
@@ -185,10 +196,17 @@ class YoukuIE(InfoExtractor):
             raw_data = self._download_json(req, video_id, note=note)
             return raw_data['data'][0]
 
+        video_password = self._downloader.params.get('videopassword', None)
+
         # request basic data
-        data1 = retrieve_data(
-            'http://v.youku.com/player/getPlayList/VideoIDS/%s' % video_id,
-            'Downloading JSON metadata 1')
+        if video_password:
+            data1 = retrieve_data(
+                'http://v.youku.com/player/getPlayList/VideoIDS/%s?password=%s' % (video_id, video_password),
+                'Downloading JSON metadata 1')
+        else:
+            data1 = retrieve_data(
+                'http://v.youku.com/player/getPlayList/VideoIDS/%s' % video_id,
+                'Downloading JSON metadata 1')
         data2 = retrieve_data(
             'http://v.youku.com/player/getPlayList/VideoIDS/%s/Pf/4/ctype/12/ev/1' % video_id,
             'Downloading JSON metadata 2')

From cb3d2eb9e99e1a3f05a5009fc15afc0212a1af1e Mon Sep 17 00:00:00 2001
From: PSJay <psjay.peng@gmail.com>
Date: Tue, 1 Sep 2015 11:07:47 +0800
Subject: [PATCH 2394/6219] Fix coding styple in YouKuIE.

---
 youtube_dl/extractor/youku.py | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index 220f84221..6219bc1bb 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -199,14 +199,13 @@ class YoukuIE(InfoExtractor):
         video_password = self._downloader.params.get('videopassword', None)
 
         # request basic data
+        basic_data_url = 'http://v.youku.com/player/getPlayList/VideoIDS/%s' % video_id
         if video_password:
-            data1 = retrieve_data(
-                'http://v.youku.com/player/getPlayList/VideoIDS/%s?password=%s' % (video_id, video_password),
-                'Downloading JSON metadata 1')
-        else:
-            data1 = retrieve_data(
-                'http://v.youku.com/player/getPlayList/VideoIDS/%s' % video_id,
-                'Downloading JSON metadata 1')
+            basic_data_url = '%s?password=%s' % (basic_data_url, video_password)
+
+        data1 = retrieve_data(
+            basic_data_url,
+            'Downloading JSON metadata 1')
         data2 = retrieve_data(
             'http://v.youku.com/player/getPlayList/VideoIDS/%s/Pf/4/ctype/12/ev/1' % video_id,
             'Downloading JSON metadata 2')

From f859695b497717d40e164d0d6a3e29b3c813dad1 Mon Sep 17 00:00:00 2001
From: PSJay <psjay.peng@gmail.com>
Date: Tue, 1 Sep 2015 12:50:01 +0800
Subject: [PATCH 2395/6219] Update README

Add Youku in video password description.
---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 25844eb6d..b6f0677f7 100644
--- a/README.md
+++ b/README.md
@@ -207,7 +207,7 @@ which means you can modify it, redistribute it or use it however you like.
     -p, --password PASSWORD          Account password. If this option is left out, youtube-dl will ask interactively.
     -2, --twofactor TWOFACTOR        Two-factor auth code
     -n, --netrc                      Use .netrc authentication data
-    --video-password PASSWORD        Video password (vimeo, smotri)
+    --video-password PASSWORD        Video password (vimeo, smotri, Youku)
 
 ## Post-processing Options:
     -x, --extract-audio              Convert video files to audio-only files (requires ffmpeg or avconv and ffprobe or avprobe)

From 5ddc127da6e2df440723e067b04bfb2ce86bb827 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 1 Sep 2015 22:26:17 +0600
Subject: [PATCH 2396/6219] [youku] Simplify (Closes #6717)

---
 youtube_dl/extractor/youku.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index 6219bc1bb..3447e7a6f 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -55,7 +55,7 @@ class YoukuIE(InfoExtractor):
         'info_dict': {
             'id': 'XNjA1NzA2Njgw',
             'ext': 'flv',
-            'title': u'邢義田复旦讲座之想象中的胡人—从“左衽孔子”说起',
+            'title': '邢義田复旦讲座之想象中的胡人—从“左衽孔子”说起',
         },
         'params': {
             'videopassword': '100600',
@@ -201,7 +201,7 @@ class YoukuIE(InfoExtractor):
         # request basic data
         basic_data_url = 'http://v.youku.com/player/getPlayList/VideoIDS/%s' % video_id
         if video_password:
-            basic_data_url = '%s?password=%s' % (basic_data_url, video_password)
+            basic_data_url += '?password=%s' % video_password
 
         data1 = retrieve_data(
             basic_data_url,

From cd5d75427e0a739018c018af49152e0052b7a1fd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 1 Sep 2015 22:28:03 +0600
Subject: [PATCH 2397/6219] [youku] Fix videopassword test

---
 youtube_dl/extractor/youku.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index 3447e7a6f..2e81d9223 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -54,9 +54,9 @@ class YoukuIE(InfoExtractor):
         'note': 'Video protected with password',
         'info_dict': {
             'id': 'XNjA1NzA2Njgw',
-            'ext': 'flv',
             'title': '邢義田复旦讲座之想象中的胡人—从“左衽孔子”说起',
         },
+        'playlist_count': 19,
         'params': {
             'videopassword': '100600',
         },

From 804c343a4ff252c46fcc4a36f73a913c8def9b93 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 1 Sep 2015 22:30:28 +0600
Subject: [PATCH 2398/6219] [options] Clarify extractors supporting
 videopassword

---
 README.md             | 2 +-
 youtube_dl/options.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index b6f0677f7..1d6c1151d 100644
--- a/README.md
+++ b/README.md
@@ -207,7 +207,7 @@ which means you can modify it, redistribute it or use it however you like.
     -p, --password PASSWORD          Account password. If this option is left out, youtube-dl will ask interactively.
     -2, --twofactor TWOFACTOR        Two-factor auth code
     -n, --netrc                      Use .netrc authentication data
-    --video-password PASSWORD        Video password (vimeo, smotri, Youku)
+    --video-password PASSWORD        Video password (vimeo, smotri, youku)
 
 ## Post-processing Options:
     -x, --extract-audio              Convert video files to audio-only files (requires ffmpeg or avconv and ffprobe or avprobe)
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 9016e3498..8c4ff12bd 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -320,7 +320,7 @@ def parseOpts(overrideArguments=None):
     authentication.add_option(
         '--video-password',
         dest='videopassword', metavar='PASSWORD',
-        help='Video password (vimeo, smotri)')
+        help='Video password (vimeo, smotri, youku)')
 
     video_format = optparse.OptionGroup(parser, 'Video Format Options')
     video_format.add_option(

From 36c15522c121c0ffeb5208847a4f5461adc4f3b5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 1 Sep 2015 22:37:42 +0600
Subject: [PATCH 2399/6219] [francetv] Use subtitle when present (Closes #6715)

---
 youtube_dl/extractor/francetv.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index 75723c00d..c053774b9 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -78,9 +78,14 @@ class FranceTVBaseInfoExtractor(InfoExtractor):
                 })
         self._sort_formats(formats)
 
+        title = info['titre']
+        subtitle = info.get('sous_titre')
+        if subtitle:
+            title += ' - %s' % subtitle
+
         return {
             'id': video_id,
-            'title': info['titre'],
+            'title': title,
             'description': clean_html(info['synopsis']),
             'thumbnail': compat_urlparse.urljoin('http://pluzz.francetv.fr', info['image']),
             'duration': int_or_none(info.get('real_duration')) or parse_duration(info['duree']),

From 6917d2a2f0396f7fb6373e6160c6f25b8664aafb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 1 Sep 2015 22:39:26 +0600
Subject: [PATCH 2400/6219] [francetv] Update tests

---
 youtube_dl/extractor/francetv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index c053774b9..f70a6fea0 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -224,7 +224,7 @@ class FranceTVIE(FranceTVBaseInfoExtractor):
             'info_dict': {
                 'id': '108961659',
                 'ext': 'flv',
-                'title': 'C à dire ?!',
+                'title': 'C à dire ?! - Quels sont les enjeux de cette rentrée politique ?',
                 'description': 'md5:1a4aeab476eb657bf57c4ff122129f81',
                 'upload_date': '20140915',
                 'timestamp': 1410795000,

From 77306e8b970f462d202db5914d0382b176172f05 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 1 Sep 2015 22:42:43 +0600
Subject: [PATCH 2401/6219] [francetv] Update tests (2)

---
 youtube_dl/extractor/francetv.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index f70a6fea0..129984a5f 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -219,15 +219,15 @@ class FranceTVIE(FranceTVBaseInfoExtractor):
         },
         # france5
         {
-            'url': 'http://www.france5.fr/emissions/c-a-dire/videos/92837968',
-            'md5': '78f0f4064f9074438e660785bbf2c5d9',
+            'url': 'http://www.france5.fr/emissions/c-a-dire/videos/quels_sont_les_enjeux_de_cette_rentree_politique__31-08-2015_908948?onglet=tous&page=1',
+            'md5': 'f6c577df3806e26471b3d21631241fd0',
             'info_dict': {
-                'id': '108961659',
+                'id': '123327454',
                 'ext': 'flv',
                 'title': 'C à dire ?! - Quels sont les enjeux de cette rentrée politique ?',
-                'description': 'md5:1a4aeab476eb657bf57c4ff122129f81',
-                'upload_date': '20140915',
-                'timestamp': 1410795000,
+                'description': 'md5:4a0d5cb5dce89d353522a84462bae5a4',
+                'upload_date': '20150831',
+                'timestamp': 1441035120,
             },
         },
         # franceo

From 080997b8083fed357061e301ab7f38484b24ded2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 1 Sep 2015 20:30:58 +0200
Subject: [PATCH 2402/6219] [rtve] Fix extraction (fixes #6723)

Adding the 'Referer' header to the png request is enough.
---
 youtube_dl/extractor/rtve.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/rtve.py b/youtube_dl/extractor/rtve.py
index 82cd98ac7..5b97d33ca 100644
--- a/youtube_dl/extractor/rtve.py
+++ b/youtube_dl/extractor/rtve.py
@@ -6,7 +6,7 @@ import re
 import time
 
 from .common import InfoExtractor
-from ..compat import compat_urlparse
+from ..compat import compat_urllib_request, compat_urlparse
 from ..utils import (
     ExtractorError,
     float_or_none,
@@ -102,7 +102,9 @@ class RTVEALaCartaIE(InfoExtractor):
         if info['state'] == 'DESPU':
             raise ExtractorError('The video is no longer available', expected=True)
         png_url = 'http://www.rtve.es/ztnr/movil/thumbnail/%s/videos/%s.png' % (self._manager, video_id)
-        png = self._download_webpage(png_url, video_id, 'Downloading url information')
+        png_request = compat_urllib_request.Request(png_url)
+        png_request.add_header('Referer', url)
+        png = self._download_webpage(png_request, video_id, 'Downloading url information')
         video_url = _decrypt_url(png)
         if not video_url.endswith('.f4m'):
             auth_url = video_url.replace(

From e5e78797e6e9873b93e550c32f454bbd5444a4ab Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 2 Sep 2015 02:16:04 +0600
Subject: [PATCH 2403/6219] [utils] Strict HTTP responses (Closes #6727)

---
 youtube_dl/utils.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index e265c7574..b7a423166 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -587,6 +587,11 @@ class ContentTooShortError(Exception):
 
 
 def _create_http_connection(ydl_handler, http_class, is_https, *args, **kwargs):
+    # Working around python 2 bug (see http://bugs.python.org/issue17849) by limiting
+    # expected HTTP responses to meet HTTP/1.0 or later (see also
+    # https://github.com/rg3/youtube-dl/issues/6727)
+    if sys.version_info < (3, 0):
+        kwargs['strict'] = True
     hc = http_class(*args, **kwargs)
     source_address = ydl_handler._params.get('source_address')
     if source_address is not None:

From ebf4ca39baa0ac2f38cbeebc1c76f02524002c91 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 2 Sep 2015 01:24:59 +0100
Subject: [PATCH 2404/6219] link to developer instructions and correct
 common.py path

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 1d6c1151d..722f11679 100644
--- a/README.md
+++ b/README.md
@@ -34,7 +34,7 @@ You can also use pip:
 
     sudo pip install youtube-dl
 
-Alternatively, refer to the developer instructions below for how to check out and work with the git repository. For further options, including PGP signatures, see https://rg3.github.io/youtube-dl/download.html .
+Alternatively, refer to the [developer instructions](#developer-instructions) below for how to check out and work with the git repository. For further options, including PGP signatures, see https://rg3.github.io/youtube-dl/download.html .
 
 # DESCRIPTION
 **youtube-dl** is a small command-line program to download videos from
@@ -552,7 +552,7 @@ If you want to add support for a new site, you can follow this quick list (assum
     ```
 5. Add an import in [`youtube_dl/extractor/__init__.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/__init__.py).
 6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will be then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc.
-7. Have a look at [`youtube_dl/common/extractor/common.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L62-L200). Add tests and code for as many as you want.
+7. Have a look at [`youtube_dl/extractor/common.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L62-L200). Add tests and code for as many as you want.
 8. If you can, check the code with [flake8](https://pypi.python.org/pypi/flake8).
 9. When the tests pass, [add](http://git-scm.com/docs/git-add) the new files and [commit](http://git-scm.com/docs/git-commit) them and [push](http://git-scm.com/docs/git-push) the result, like this:
 

From 8f9d522f6252f874d41045b2458256ca8cf1a312 Mon Sep 17 00:00:00 2001
From: "Sergey M." <dstftw@gmail.com>
Date: Wed, 2 Sep 2015 21:09:31 +0600
Subject: [PATCH 2405/6219] [README.md] Remove superfluous word

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 722f11679..24bfe38a2 100644
--- a/README.md
+++ b/README.md
@@ -34,7 +34,7 @@ You can also use pip:
 
     sudo pip install youtube-dl
 
-Alternatively, refer to the [developer instructions](#developer-instructions) below for how to check out and work with the git repository. For further options, including PGP signatures, see https://rg3.github.io/youtube-dl/download.html .
+Alternatively, refer to the [developer instructions](#developer-instructions) for how to check out and work with the git repository. For further options, including PGP signatures, see https://rg3.github.io/youtube-dl/download.html .
 
 # DESCRIPTION
 **youtube-dl** is a small command-line program to download videos from

From 198492bbf0e497079158bdf9ff94df3c91d6f3b7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 2 Sep 2015 21:36:22 +0600
Subject: [PATCH 2406/6219] [ok] Extract correct data-options (Closes #6726)

---
 youtube_dl/extractor/odnoklassniki.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/odnoklassniki.py b/youtube_dl/extractor/odnoklassniki.py
index f552e6f7e..97900a4c3 100644
--- a/youtube_dl/extractor/odnoklassniki.py
+++ b/youtube_dl/extractor/odnoklassniki.py
@@ -59,7 +59,7 @@ class OdnoklassnikiIE(InfoExtractor):
 
         player = self._parse_json(
             unescapeHTML(self._search_regex(
-                r'data-options="([^"]+)"', webpage, 'player')),
+                r'data-options="([^"]+%s[^"]+)"' % video_id, webpage, 'player')),
             video_id)
 
         flashvars = player['flashvars']

From 1e804244d062c7620bcc9498cba1c7d3d8bd8095 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 2 Sep 2015 21:38:56 +0600
Subject: [PATCH 2407/6219] [ok] Improve player regex

---
 youtube_dl/extractor/odnoklassniki.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/odnoklassniki.py b/youtube_dl/extractor/odnoklassniki.py
index 97900a4c3..4564fd0be 100644
--- a/youtube_dl/extractor/odnoklassniki.py
+++ b/youtube_dl/extractor/odnoklassniki.py
@@ -59,7 +59,8 @@ class OdnoklassnikiIE(InfoExtractor):
 
         player = self._parse_json(
             unescapeHTML(self._search_regex(
-                r'data-options="([^"]+%s[^"]+)"' % video_id, webpage, 'player')),
+                r'data-options=(?P<quote>["\'])(?P<player>{.+?%s.+?})(?P=quote)' % video_id,
+                webpage, 'player', group='player')),
             video_id)
 
         flashvars = player['flashvars']

From 88720ed09b7921c967432fc3f4aa7a994e8028a5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 2 Sep 2015 22:08:50 +0600
Subject: [PATCH 2408/6219] [ok] Add support for youtube embeds

---
 youtube_dl/extractor/odnoklassniki.py | 46 ++++++++++++++++++++-------
 1 file changed, 35 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/odnoklassniki.py b/youtube_dl/extractor/odnoklassniki.py
index 4564fd0be..a077b676f 100644
--- a/youtube_dl/extractor/odnoklassniki.py
+++ b/youtube_dl/extractor/odnoklassniki.py
@@ -43,6 +43,21 @@ class OdnoklassnikiIE(InfoExtractor):
             'like_count': int,
             'age_limit': 0,
         },
+    }, {
+        # YouTube embed (metadataUrl, provider == USER_YOUTUBE)
+        'url': 'http://ok.ru/video/64211978996595-1',
+        'md5': '5d7475d428845cd2e13bae6f1a992278',
+        'info_dict': {
+            'id': '64211978996595-1',
+            'ext': 'mp4',
+            'title': 'Космическая среда от 26 августа 2015',
+            'description': 'md5:848eb8b85e5e3471a3a803dae1343ed0',
+            'duration': 440,
+            'upload_date': '20150826',
+            'uploader_id': '750099571',
+            'uploader': 'Алина П',
+            'age_limit': 0,
+        },
     }, {
         'url': 'http://ok.ru/web-api/video/moviePlayer/20079905452',
         'only_matching': True,
@@ -93,16 +108,7 @@ class OdnoklassnikiIE(InfoExtractor):
 
         like_count = int_or_none(metadata.get('likeCount'))
 
-        quality = qualities(('mobile', 'lowest', 'low', 'sd', 'hd'))
-
-        formats = [{
-            'url': f['url'],
-            'ext': 'mp4',
-            'format_id': f['name'],
-            'quality': quality(f['name']),
-        } for f in metadata['videos']]
-
-        return {
+        info = {
             'id': video_id,
             'title': title,
             'thumbnail': thumbnail,
@@ -112,5 +118,23 @@ class OdnoklassnikiIE(InfoExtractor):
             'uploader_id': uploader_id,
             'like_count': like_count,
             'age_limit': age_limit,
-            'formats': formats,
         }
+
+        if metadata.get('provider') == 'USER_YOUTUBE':
+            info.update({
+                '_type': 'url_transparent',
+                'url': movie['contentId'],
+            })
+            return info
+
+        quality = qualities(('mobile', 'lowest', 'low', 'sd', 'hd'))
+
+        formats = [{
+            'url': f['url'],
+            'ext': 'mp4',
+            'format_id': f['name'],
+            'quality': quality(f['name']),
+        } for f in metadata['videos']]
+
+        info['formats'] = formats
+        return info

From e8dcfa3d698fd328760e40f6a76f99c45dfcdc85 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 2 Sep 2015 22:09:33 +0600
Subject: [PATCH 2409/6219] [ok] Sort formats

---
 youtube_dl/extractor/odnoklassniki.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/odnoklassniki.py b/youtube_dl/extractor/odnoklassniki.py
index a077b676f..66520c2c5 100644
--- a/youtube_dl/extractor/odnoklassniki.py
+++ b/youtube_dl/extractor/odnoklassniki.py
@@ -135,6 +135,7 @@ class OdnoklassnikiIE(InfoExtractor):
             'format_id': f['name'],
             'quality': quality(f['name']),
         } for f in metadata['videos']]
+        self._sort_formats(formats)
 
         info['formats'] = formats
         return info

From 9f4921bfa0ce3a48d2f93b4946f361116cfde5e9 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 3 Sep 2015 00:29:53 +0100
Subject: [PATCH 2410/6219] [dcn] add show extraction and support for other
 types of urls

---
 youtube_dl/extractor/__init__.py |  6 ++-
 youtube_dl/extractor/dcn.py      | 81 ++++++++++++++++++++++++++++++--
 2 files changed, 83 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 39b05ce8f..d4a3e8ab0 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -118,7 +118,11 @@ from .dailymotion import (
 )
 from .daum import DaumIE
 from .dbtv import DBTVIE
-from .dcn import DCNIE
+from .dcn import (
+    DCNGeneralIE,
+    DCNVideoIE,
+    DCNShowIE,
+)
 from .dctp import DctpTvIE
 from .deezer import DeezerPlaylistIE
 from .dfb import DFBIE
diff --git a/youtube_dl/extractor/dcn.py b/youtube_dl/extractor/dcn.py
index 82261e25c..352d35c7a 100644
--- a/youtube_dl/extractor/dcn.py
+++ b/youtube_dl/extractor/dcn.py
@@ -1,6 +1,8 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse,
@@ -12,10 +14,33 @@ from ..utils import (
 )
 
 
-class DCNIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?dcndigital\.ae/(?:#/)?(?:video/.+|show/\d+/.+?)/(?P<id>\d+)'
+class DCNGeneralIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?dcndigital\.ae/(?:#/)?show/(?P<show_id>\d+)/[^/]+(?:/(?P<video_id>\d+)/(?P<season_id>\d+))?'
+
+    def _real_extract(self, url):
+        show_id, video_id, season_id = re.match(self._VALID_URL, url).groups()
+        url = ''
+        ie_key = ''
+        if video_id and int(video_id) > 0:
+            url = 'http://www.dcndigital.ae/#/media/%s' % video_id
+            ie_key = 'DCNVideo'
+        else:
+            ie_key = 'DCNShow'
+            if season_id and int(season_id) > 0:
+                url = 'http://www.dcndigital.ae/#/program/season/%s' % season_id
+            else:
+                url = 'http://www.dcndigital.ae/#/program/%s' % show_id
+        return {
+            'url': url,
+            '_type': 'url',
+            'ie_key': ie_key
+        }
+
+
+class DCNVideoIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?dcndigital\.ae/(?:#/)?(?:video/[^/]+|media)/(?P<id>\d+)'
     _TEST = {
-        'url': 'http://www.dcndigital.ae/#/show/199074/%D8%B1%D8%AD%D9%84%D8%A9-%D8%A7%D9%84%D8%B9%D9%85%D8%B1-%D8%A7%D9%84%D8%AD%D9%84%D9%82%D8%A9-1/17375/6887',
+        'url': 'http://www.dcndigital.ae/#/video/%D8%B1%D8%AD%D9%84%D8%A9-%D8%A7%D9%84%D8%B9%D9%85%D8%B1-%D8%A7%D9%84%D8%AD%D9%84%D9%82%D8%A9-1/17375',
         'info_dict':
         {
             'id': '17375',
@@ -82,3 +107,53 @@ class DCNIE(InfoExtractor):
             'timestamp': timestamp,
             'formats': formats,
         }
+
+
+class DCNShowIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?dcndigital\.ae/(?:#/)?program/(?:(?P<show_id>\d+)|season/(?P<season_id>\d+))'
+    _TEST = {
+        'url': 'http://dcndigital.ae/#/program/205024/%D9%85%D8%AD%D8%A7%D8%B6%D8%B1%D8%A7%D8%AA-%D8%A7%D9%84%D8%B4%D9%8A%D8%AE-%D8%A7%D9%84%D8%B4%D8%B9%D8%B1%D8%A7%D9%88%D9%8A',
+        'info_dict':
+        {
+            'id': '205024',
+            'title': 'محاضرات الشيخ الشعراوي',
+            'description': '',
+        },
+        'playlist_mincount': 27,
+    }
+
+    def _real_extract(self, url):
+        show_id, season_id = re.match(self._VALID_URL, url).groups()
+        data = {}
+        if season_id:
+            request = compat_urllib_request.Request(
+                'http://admin.mangomolo.com/analytics/index.php/plus/season_info?id=%s' % season_id,
+                headers={'Origin': 'http://www.dcndigital.ae'})
+            season = self._download_json(request, season_id)
+            show_id = season['id']
+            data['season'] = season_id
+        data['show_id'] = show_id
+        request = compat_urllib_request.Request(
+            'http://admin.mangomolo.com/analytics/index.php/plus/show',
+            compat_urllib_parse.urlencode(data),
+            {
+                'Origin': 'http://www.dcndigital.ae',
+                'Content-Type': 'application/x-www-form-urlencoded'
+            })
+        show = self._download_json(request, show_id)
+        title = show['cat'].get('title_en') or show['cat']['title_ar']
+        description = show['cat'].get('description_en') or show['cat'].get('description_ar')
+        entries = []
+        for video in show['videos']:
+            entries.append({
+                'url': 'http://www.dcndigital.ae/#/media/%s' % video['id'],
+                '_type': 'url',
+                'ie_key': 'DCNVideo',
+            })
+        return {
+            'id': show_id,
+            'title': title,
+            'description': description,
+            'entries': entries,
+            '_type': 'playlist',
+        }

From bd6742137f532d2a3c09c8b89543618ae982abf5 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 3 Sep 2015 12:34:20 +0200
Subject: [PATCH 2411/6219] release 2015.09.03

---
 CONTRIBUTING.md        | 2 +-
 docs/supportedsites.md | 2 ++
 youtube_dl/version.py  | 2 +-
 3 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 42333c450..f8ab29631 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -125,7 +125,7 @@ If you want to add support for a new site, you can follow this quick list (assum
     ```
 5. Add an import in [`youtube_dl/extractor/__init__.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/__init__.py).
 6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will be then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc.
-7. Have a look at [`youtube_dl/common/extractor/common.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L62-L200). Add tests and code for as many as you want.
+7. Have a look at [`youtube_dl/extractor/common.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L62-L200). Add tests and code for as many as you want.
 8. If you can, check the code with [flake8](https://pypi.python.org/pypi/flake8).
 9. When the tests pass, [add](http://git-scm.com/docs/git-add) the new files and [commit](http://git-scm.com/docs/git-commit) them and [push](http://git-scm.com/docs/git-push) the result, like this:
 
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 328a819b3..04b9959ac 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -220,6 +220,7 @@
  - **imdb**: Internet Movie Database trailers
  - **imdb:list**: Internet Movie Database lists
  - **Imgur**
+ - **ImgurAlbum**
  - **Ina**
  - **Indavideo**
  - **IndavideoEmbed**
@@ -303,6 +304,7 @@
  - **MPORA**
  - **MSNBC**
  - **MTV**
+ - **mtv.de**
  - **mtviggy.com**
  - **mtvservices:embedded**
  - **MuenchenTV**: münchen.tv
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index a07bc9233..6bc689b75 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.08.28'
+__version__ = '2015.09.03'

From 725d1c58aa25da8640eef8d62b9451cbd6762169 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 3 Sep 2015 13:15:02 +0200
Subject: [PATCH 2412/6219] [crunchyroll] Extract width and height (closes
 #6749)

---
 youtube_dl/extractor/crunchyroll.py | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index c2162aa68..ce123482e 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -20,9 +20,11 @@ from ..utils import (
     ExtractorError,
     bytes_to_intlist,
     intlist_to_bytes,
+    int_or_none,
     remove_end,
     unified_strdate,
     urlencode_postdata,
+    xpath_text,
 )
 from ..aes import (
     aes_cbc_decrypt,
@@ -286,6 +288,13 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
             stream_info = streamdata.find('./{default}preload/stream_info')
             video_url = stream_info.find('./host').text
             video_play_path = stream_info.find('./file').text
+            metadata = stream_info.find('./metadata')
+            format_info = {
+                'format': video_format,
+                'format_id': video_format,
+                'height': int_or_none(xpath_text(metadata, './height')),
+                'width': int_or_none(xpath_text(metadata, './width')),
+            }
 
             if '.fplive.net/' in video_url:
                 video_url = re.sub(r'^rtmpe?://', 'http://', video_url.strip())
@@ -294,19 +303,18 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
                     netloc='v.lvlt.crcdn.net',
                     path='%s/%s' % (remove_end(parsed_video_url.path, '/'), video_play_path.split(':')[-1])))
                 if self._is_valid_url(direct_video_url, video_id, video_format):
-                    formats.append({
+                    format_info.update({
                         'url': direct_video_url,
-                        'format_id': video_format,
                     })
+                    formats.append(format_info)
                     continue
 
-            formats.append({
+            format_info.update({
                 'url': video_url,
                 'play_path': video_play_path,
                 'ext': 'flv',
-                'format': video_format,
-                'format_id': video_format,
             })
+            formats.append(format_info)
 
         subtitles = self.extract_subtitles(video_id, webpage)
 

From b477da2094db30a232f67edf3b342dc460aa14d4 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 3 Sep 2015 16:59:10 +0100
Subject: [PATCH 2413/6219] correct the extractor name and id and remove
 unnecessary request

---
 youtube_dl/extractor/__init__.py |  2 +-
 youtube_dl/extractor/dcn.py      | 28 ++++++++++++++++++----------
 2 files changed, 19 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index d4a3e8ab0..4e41d9bf9 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -121,7 +121,7 @@ from .dbtv import DBTVIE
 from .dcn import (
     DCNGeneralIE,
     DCNVideoIE,
-    DCNShowIE,
+    DCNSeasonIE,
 )
 from .dctp import DctpTvIE
 from .deezer import DeezerPlaylistIE
diff --git a/youtube_dl/extractor/dcn.py b/youtube_dl/extractor/dcn.py
index 352d35c7a..8a36c10f6 100644
--- a/youtube_dl/extractor/dcn.py
+++ b/youtube_dl/extractor/dcn.py
@@ -11,6 +11,8 @@ from ..compat import (
 from ..utils import (
     int_or_none,
     parse_iso8601,
+    smuggle_url,
+    unsmuggle_url,
 )
 
 
@@ -25,9 +27,9 @@ class DCNGeneralIE(InfoExtractor):
             url = 'http://www.dcndigital.ae/#/media/%s' % video_id
             ie_key = 'DCNVideo'
         else:
-            ie_key = 'DCNShow'
+            ie_key = 'DCNSeason'
             if season_id and int(season_id) > 0:
-                url = 'http://www.dcndigital.ae/#/program/season/%s' % season_id
+                url = smuggle_url('http://www.dcndigital.ae/#/program/season/%s' % season_id, {'show_id': show_id})
             else:
                 url = 'http://www.dcndigital.ae/#/program/%s' % show_id
         return {
@@ -38,6 +40,7 @@ class DCNGeneralIE(InfoExtractor):
 
 
 class DCNVideoIE(InfoExtractor):
+    IE_NAME = 'dcn:video'
     _VALID_URL = r'https?://(?:www\.)?dcndigital\.ae/(?:#/)?(?:video/[^/]+|media)/(?P<id>\d+)'
     _TEST = {
         'url': 'http://www.dcndigital.ae/#/video/%D8%B1%D8%AD%D9%84%D8%A9-%D8%A7%D9%84%D8%B9%D9%85%D8%B1-%D8%A7%D9%84%D8%AD%D9%84%D9%82%D8%A9-1/17375',
@@ -109,13 +112,14 @@ class DCNVideoIE(InfoExtractor):
         }
 
 
-class DCNShowIE(InfoExtractor):
+class DCNSeasonIE(InfoExtractor):
+    IE_NAME = 'dcn:season'
     _VALID_URL = r'https?://(?:www\.)?dcndigital\.ae/(?:#/)?program/(?:(?P<show_id>\d+)|season/(?P<season_id>\d+))'
     _TEST = {
         'url': 'http://dcndigital.ae/#/program/205024/%D9%85%D8%AD%D8%A7%D8%B6%D8%B1%D8%A7%D8%AA-%D8%A7%D9%84%D8%B4%D9%8A%D8%AE-%D8%A7%D9%84%D8%B4%D8%B9%D8%B1%D8%A7%D9%88%D9%8A',
         'info_dict':
         {
-            'id': '205024',
+            'id': '7910',
             'title': 'محاضرات الشيخ الشعراوي',
             'description': '',
         },
@@ -123,15 +127,18 @@ class DCNShowIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
+        url, smuggled_data = unsmuggle_url(url, {})
         show_id, season_id = re.match(self._VALID_URL, url).groups()
         data = {}
         if season_id:
-            request = compat_urllib_request.Request(
-                'http://admin.mangomolo.com/analytics/index.php/plus/season_info?id=%s' % season_id,
-                headers={'Origin': 'http://www.dcndigital.ae'})
-            season = self._download_json(request, season_id)
-            show_id = season['id']
             data['season'] = season_id
+            show_id = smuggled_data.get('show_id')
+            if show_id is None:
+                request = compat_urllib_request.Request(
+                    'http://admin.mangomolo.com/analytics/index.php/plus/season_info?id=%s' % season_id,
+                    headers={'Origin': 'http://www.dcndigital.ae'})
+                season = self._download_json(request, season_id)
+                show_id = season['id']
         data['show_id'] = show_id
         request = compat_urllib_request.Request(
             'http://admin.mangomolo.com/analytics/index.php/plus/show',
@@ -141,6 +148,7 @@ class DCNShowIE(InfoExtractor):
                 'Content-Type': 'application/x-www-form-urlencoded'
             })
         show = self._download_json(request, show_id)
+        season_id = season_id or show['default_season']
         title = show['cat'].get('title_en') or show['cat']['title_ar']
         description = show['cat'].get('description_en') or show['cat'].get('description_ar')
         entries = []
@@ -151,7 +159,7 @@ class DCNShowIE(InfoExtractor):
                 'ie_key': 'DCNVideo',
             })
         return {
-            'id': show_id,
+            'id': season_id,
             'title': title,
             'description': description,
             'entries': entries,

From e7a8c3032d8610bc429b6318678387433728b2a9 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 3 Sep 2015 22:25:33 +0100
Subject: [PATCH 2414/6219] [downloader/external] Respect
 --no-check-certificate for curl and aria2c and --proxy for curl

---
 youtube_dl/downloader/external.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index 6c310346c..072ac3a46 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -47,7 +47,7 @@ class ExternalFD(FileDownloader):
 
     def _option(self, command_option, param):
         param = self.params.get(param)
-        if param is None:
+        if param is None or param is False:
             return []
         if isinstance(param, bool):
             return [command_option]
@@ -80,6 +80,8 @@ class CurlFD(ExternalFD):
         for key, val in info_dict['http_headers'].items():
             cmd += ['--header', '%s: %s' % (key, val)]
         cmd += self._option('--interface', 'source_address')
+        cmd += self._option('--proxy', 'proxy')
+        cmd += self._option('--insecure', 'nocheckcertificate')
         cmd += self._configuration_args()
         cmd += ['--', info_dict['url']]
         return cmd
@@ -121,6 +123,7 @@ class Aria2cFD(ExternalFD):
             cmd += ['--header', '%s: %s' % (key, val)]
         cmd += self._option('--interface', 'source_address')
         cmd += self._option('--all-proxy', 'proxy')
+        cmd += self._option('--check-certificate=false', 'nocheckcertificate')
         cmd += ['--', info_dict['url']]
         return cmd
 

From 26c61e080908b20f3c6707789bb42933bbbb42a0 Mon Sep 17 00:00:00 2001
From: Marco Ferragina <marco.ferragina@gmail.com>
Date: Fri, 4 Sep 2015 12:59:13 +0200
Subject: [PATCH 2415/6219] Added ec extension to nowvideo extractor

---
 youtube_dl/extractor/nowvideo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nowvideo.py b/youtube_dl/extractor/nowvideo.py
index dec09cdfe..17baa9679 100644
--- a/youtube_dl/extractor/nowvideo.py
+++ b/youtube_dl/extractor/nowvideo.py
@@ -7,7 +7,7 @@ class NowVideoIE(NovaMovIE):
     IE_NAME = 'nowvideo'
     IE_DESC = 'NowVideo'
 
-    _VALID_URL = NovaMovIE._VALID_URL_TEMPLATE % {'host': 'nowvideo\.(?:ch|sx|eu|at|ag|co|li)'}
+    _VALID_URL = NovaMovIE._VALID_URL_TEMPLATE % {'host': 'nowvideo\.(?:ch|ec|sx|eu|at|ag|co|li)'}
 
     _HOST = 'www.nowvideo.ch'
 

From 8e2898edf930830260ab6b294c8866e7651a01a6 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 4 Sep 2015 15:42:09 +0100
Subject: [PATCH 2416/6219] [dcn] add support for live streams and catchup
 videos

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/dcn.py      | 62 +++++++++++++++++++++++++++++++-
 2 files changed, 62 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 4e41d9bf9..677c75564 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -121,6 +121,7 @@ from .dbtv import DBTVIE
 from .dcn import (
     DCNGeneralIE,
     DCNVideoIE,
+    DCNLiveIE,
     DCNSeasonIE,
 )
 from .dctp import DctpTvIE
diff --git a/youtube_dl/extractor/dcn.py b/youtube_dl/extractor/dcn.py
index 8a36c10f6..2e8fff660 100644
--- a/youtube_dl/extractor/dcn.py
+++ b/youtube_dl/extractor/dcn.py
@@ -2,6 +2,7 @@
 from __future__ import unicode_literals
 
 import re
+import base64
 
 from .common import InfoExtractor
 from ..compat import (
@@ -41,7 +42,7 @@ class DCNGeneralIE(InfoExtractor):
 
 class DCNVideoIE(InfoExtractor):
     IE_NAME = 'dcn:video'
-    _VALID_URL = r'https?://(?:www\.)?dcndigital\.ae/(?:#/)?(?:video/[^/]+|media)/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?dcndigital\.ae/(?:#/)?(?:video/[^/]+|media|catchup/[^/]+/[^/]+)/(?P<id>\d+)'
     _TEST = {
         'url': 'http://www.dcndigital.ae/#/video/%D8%B1%D8%AD%D9%84%D8%A9-%D8%A7%D9%84%D8%B9%D9%85%D8%B1-%D8%A7%D9%84%D8%AD%D9%84%D9%82%D8%A9-1/17375',
         'info_dict':
@@ -112,6 +113,65 @@ class DCNVideoIE(InfoExtractor):
         }
 
 
+class DCNLiveIE(InfoExtractor):
+    IE_NAME = 'dcn:live'
+    _VALID_URL = r'https?://(?:www\.)?dcndigital\.ae/(?:#/)?live/(?P<id>\d+)'
+    _TEST = {
+        'url': 'http://www.dcndigital.ae/#/live/6/dubai-tv',
+        'info_dict':
+        {
+            'id': '6',
+            'ext': 'mp4',
+            'title': 'Dubai Al Oula',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }
+
+    def _real_extract(self, url):
+        channel_id = self._match_id(url)
+
+        request = compat_urllib_request.Request(
+            'http://admin.mangomolo.com/analytics/index.php/plus/getchanneldetails?channel_id=%s' % channel_id,
+            headers={'Origin': 'http://www.dcndigital.ae'})
+
+        channel = self._download_json(request, channel_id)
+        title = channel.get('title_en') or channel['title_ar']
+
+        webpage = self._download_webpage(
+            'http://admin.mangomolo.com/analytics/index.php/customers/embed/index?'
+            + compat_urllib_parse.urlencode({
+                'id': base64.b64encode(channel['user_id'].encode()).decode(),
+                'channelid': base64.b64encode(channel['id'].encode()).decode(),
+                'signature': channel['signature'],
+                'countries': 'Q0M=',
+                'filter': 'DENY',
+            }), channel_id)
+
+        m3u8_url = self._html_search_regex(r'file:\s*"([^"]+)', webpage, 'm3u8 url')
+        formats = self._extract_m3u8_formats(
+            m3u8_url, channel_id, 'mp4', entry_protocol='m3u8_native', m3u8_id='hls')
+
+        rtsp_url = self._search_regex(
+            r'<a[^>]+href="(rtsp://[^"]+)"', webpage, 'rtsp url', fatal=False)
+        if rtsp_url:
+            formats.append({
+                'url': rtsp_url,
+                'format_id': 'rtsp',
+            })
+
+        self._sort_formats(formats)
+
+        return {
+            'id': channel_id,
+            'title': title,
+            'formats': formats,
+            'is_live': True,
+        }
+
+
 class DCNSeasonIE(InfoExtractor):
     IE_NAME = 'dcn:season'
     _VALID_URL = r'https?://(?:www\.)?dcndigital\.ae/(?:#/)?program/(?:(?P<show_id>\d+)|season/(?P<season_id>\d+))'

From a41fb80ce1e2fec48d7a4bc15d169061e76ca672 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 4 Sep 2015 23:56:45 +0600
Subject: [PATCH 2417/6219] [utils] Add xpath_element and xpath_attr

---
 youtube_dl/utils.py | 21 +++++++++++++++++++--
 1 file changed, 19 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index b7a423166..de5069c7b 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -176,7 +176,7 @@ def xpath_with_ns(path, ns_map):
     return '/'.join(replaced)
 
 
-def xpath_text(node, xpath, name=None, fatal=False, default=NO_DEFAULT):
+def xpath_element(node, xpath, name=None, fatal=False, default=NO_DEFAULT):
     if sys.version_info < (2, 7):  # Crazy 2.6
         xpath = xpath.encode('ascii')
 
@@ -189,7 +189,24 @@ def xpath_text(node, xpath, name=None, fatal=False, default=NO_DEFAULT):
             raise ExtractorError('Could not find XML element %s' % name)
         else:
             return None
-    return n.text
+    return n
+
+
+def xpath_text(node, xpath, name=None, fatal=False, default=NO_DEFAULT):
+    return xpath_element(node, xpath, name, fatal=fatal, default=default).text
+
+
+def xpath_attr(node, xpath, key, name=None, fatal=False, default=NO_DEFAULT):
+    n = find_xpath_attr(node, xpath, key)
+    if n is None:
+        if default is not NO_DEFAULT:
+            return default
+        elif fatal:
+            name = '%s[@%s]' % (xpath, key) if name is None else name
+            raise ExtractorError('Could not find XML attribute %s' % name)
+        else:
+            return None
+    return n.attrib[key]
 
 
 def get_element_by_id(id, html):

From 5d2354f1779182f260b4ef44ef6e92a3b5944183 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 4 Sep 2015 23:57:27 +0600
Subject: [PATCH 2418/6219] [utils] Relax attribute key assert

---
 youtube_dl/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index de5069c7b..86b757103 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -141,7 +141,7 @@ def write_json_file(obj, fn):
 if sys.version_info >= (2, 7):
     def find_xpath_attr(node, xpath, key, val=None):
         """ Find the xpath xpath[@key=val] """
-        assert re.match(r'^[a-zA-Z-]+$', key)
+        assert re.match(r'^[a-zA-Z_-]+$', key)
         if val:
             assert re.match(r'^[a-zA-Z0-9@\s:._-]*$', val)
         expr = xpath + ('[@%s]' % key if val is None else "[@%s='%s']" % (key, val))

From 22889ab175d990d50050603fb983b1db7d1615d6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 5 Sep 2015 00:01:40 +0600
Subject: [PATCH 2419/6219] [ruutu] Fix extraction (Closes #6759)

---
 youtube_dl/extractor/ruutu.py | 41 +++++++++--------------------------
 1 file changed, 10 insertions(+), 31 deletions(-)

diff --git a/youtube_dl/extractor/ruutu.py b/youtube_dl/extractor/ruutu.py
index 4e22628d0..8ca6e56bb 100644
--- a/youtube_dl/extractor/ruutu.py
+++ b/youtube_dl/extractor/ruutu.py
@@ -6,19 +6,19 @@ from ..compat import compat_urllib_parse_urlparse
 from ..utils import (
     determine_ext,
     int_or_none,
+    xpath_attr,
     xpath_text,
 )
 
 
 class RuutuIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?ruutu\.fi/ohjelmat/(?:[^/?#]+/)*(?P<id>[^/?#]+)'
+    _VALID_URL = r'https?://(?:www\.)?ruutu\.fi/video/(?P<id>\d+)'
     _TESTS = [
         {
-            'url': 'http://www.ruutu.fi/ohjelmat/oletko-aina-halunnut-tietaa-mita-tapahtuu-vain-hetki-ennen-lahetysta-nyt-se-selvisi',
+            'url': 'http://www.ruutu.fi/video/2058907',
             'md5': 'ab2093f39be1ca8581963451b3c0234f',
             'info_dict': {
                 'id': '2058907',
-                'display_id': 'oletko-aina-halunnut-tietaa-mita-tapahtuu-vain-hetki-ennen-lahetysta-nyt-se-selvisi',
                 'ext': 'mp4',
                 'title': 'Oletko aina halunnut tietää mitä tapahtuu vain hetki ennen lähetystä? - Nyt se selvisi!',
                 'description': 'md5:cfc6ccf0e57a814360df464a91ff67d6',
@@ -32,10 +32,9 @@ class RuutuIE(InfoExtractor):
             'md5': '065a10ae4d5b8cfd9d0c3d332465e3d9',
             'info_dict': {
                 'id': '2057306',
-                'display_id': 'superpesis-katso-koko-kausi-ruudussa',
                 'ext': 'mp4',
                 'title': 'Superpesis: katso koko kausi Ruudussa',
-                'description': 'md5:44c44a99fdbe5b380ab74ebd75f0af77',
+                'description': 'md5:da2736052fef3b2bd5e0005e63c25eac',
                 'thumbnail': 're:^https?://.*\.jpg$',
                 'duration': 40,
                 'age_limit': 0,
@@ -44,29 +43,10 @@ class RuutuIE(InfoExtractor):
     ]
 
     def _real_extract(self, url):
-        display_id = self._match_id(url)
+        video_id = self._match_id(url)
 
-        webpage = self._download_webpage(url, display_id)
-
-        video_id = self._search_regex(
-            r'data-media-id="(\d+)"', webpage, 'media id')
-
-        video_xml_url = None
-
-        media_data = self._search_regex(
-            r'jQuery\.extend\([^,]+,\s*(.+?)\);', webpage,
-            'media data', default=None)
-        if media_data:
-            media_json = self._parse_json(media_data, display_id, fatal=False)
-            if media_json:
-                xml_url = media_json.get('ruutuplayer', {}).get('xmlUrl')
-                if xml_url:
-                    video_xml_url = xml_url.replace('{ID}', video_id)
-
-        if not video_xml_url:
-            video_xml_url = 'http://gatling.ruutu.fi/media-xml-cache?id=%s' % video_id
-
-        video_xml = self._download_xml(video_xml_url, video_id)
+        video_xml = self._download_xml(
+            'http://gatling.ruutu.fi/media-xml-cache?id=%s' % video_id, video_id)
 
         formats = []
         processed_urls = []
@@ -109,10 +89,9 @@ class RuutuIE(InfoExtractor):
 
         return {
             'id': video_id,
-            'display_id': display_id,
-            'title': self._og_search_title(webpage),
-            'description': self._og_search_description(webpage),
-            'thumbnail': self._og_search_thumbnail(webpage),
+            'title': xpath_attr(video_xml, './/Behavior/Program', 'program_name', 'title', fatal=True),
+            'description': xpath_attr(video_xml, './/Behavior/Program', 'description', 'description'),
+            'thumbnail': xpath_attr(video_xml, './/Behavior/Startpicture', 'href', 'thumbnail'),
             'duration': int_or_none(xpath_text(video_xml, './/Runtime', 'duration')),
             'age_limit': int_or_none(xpath_text(video_xml, './/AgeLimit', 'age limit')),
             'formats': formats,

From 8e636da4990643084de17c03c6a0639e9aa4085e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 5 Sep 2015 00:34:49 +0600
Subject: [PATCH 2420/6219] [utils] Improve xpath_text

---
 youtube_dl/utils.py | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 86b757103..cc792be6a 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -181,7 +181,7 @@ def xpath_element(node, xpath, name=None, fatal=False, default=NO_DEFAULT):
         xpath = xpath.encode('ascii')
 
     n = node.find(xpath)
-    if n is None or n.text is None:
+    if n is None:
         if default is not NO_DEFAULT:
             return default
         elif fatal:
@@ -193,7 +193,18 @@ def xpath_element(node, xpath, name=None, fatal=False, default=NO_DEFAULT):
 
 
 def xpath_text(node, xpath, name=None, fatal=False, default=NO_DEFAULT):
-    return xpath_element(node, xpath, name, fatal=fatal, default=default).text
+    n = xpath_element(node, xpath, name, fatal=fatal, default=default)
+    if n is None or n == default:
+        return n
+    if n.text is None:
+        if default is not NO_DEFAULT:
+            return default
+        elif fatal:
+            name = xpath if name is None else name
+            raise ExtractorError('Could not find XML element\'s text %s' % name)
+        else:
+            return None
+    return n.text
 
 
 def xpath_attr(node, xpath, key, name=None, fatal=False, default=NO_DEFAULT):

From 87f70ab39d0f65e4462403c0afcad0969171c356 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 5 Sep 2015 00:36:16 +0600
Subject: [PATCH 2421/6219] [test_utils] Add more tests for xpath

---
 test/test_utils.py | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/test/test_utils.py b/test/test_utils.py
index a759b2da9..15a59692f 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -57,7 +57,9 @@ from youtube_dl.utils import (
     urlencode_postdata,
     version_tuple,
     xpath_with_ns,
+    xpath_element,
     xpath_text,
+    xpath_attr,
     render_table,
     match_str,
     parse_dfxp_time_expr,
@@ -264,6 +266,16 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(find('media:song/media:author').text, 'The Author')
         self.assertEqual(find('media:song/url').text, 'http://server.com/download.mp3')
 
+    def test_xpath_element(self):
+        doc = xml.etree.ElementTree.Element('root')
+        div = xml.etree.ElementTree.SubElement(doc, 'div')
+        p = xml.etree.ElementTree.SubElement(div, 'p')
+        p.text = 'Foo'
+        self.assertEqual(xpath_element(doc, 'div/p'), p)
+        self.assertEqual(xpath_element(doc, 'div/bar', default='default'), 'default')
+        self.assertTrue(xpath_element(doc, 'div/bar') is None)
+        self.assertRaises(ExtractorError, xpath_element, doc, 'div/bar', fatal=True)
+
     def test_xpath_text(self):
         testxml = '''<root>
             <div>
@@ -272,9 +284,25 @@ class TestUtil(unittest.TestCase):
         </root>'''
         doc = xml.etree.ElementTree.fromstring(testxml)
         self.assertEqual(xpath_text(doc, 'div/p'), 'Foo')
+        self.assertEqual(xpath_text(doc, 'div/bar', default='default'), 'default')
         self.assertTrue(xpath_text(doc, 'div/bar') is None)
         self.assertRaises(ExtractorError, xpath_text, doc, 'div/bar', fatal=True)
 
+    def test_xpath_attr(self):
+        testxml = '''<root>
+            <div>
+                <p x="a">Foo</p>
+            </div>
+        </root>'''
+        doc = xml.etree.ElementTree.fromstring(testxml)
+        self.assertEqual(xpath_attr(doc, 'div/p', 'x'), 'a')
+        self.assertEqual(xpath_attr(doc, 'div/bar', 'x'), None)
+        self.assertEqual(xpath_attr(doc, 'div/p', 'y'), None)
+        self.assertEqual(xpath_attr(doc, 'div/bar', 'x', default='default'), 'default')
+        self.assertEqual(xpath_attr(doc, 'div/p', 'y', default='default'), 'default')
+        self.assertRaises(ExtractorError, xpath_attr, doc, 'div/bar', 'x', fatal=True)
+        self.assertRaises(ExtractorError, xpath_attr, doc, 'div/p', 'y', fatal=True)
+
     def test_smuggle_url(self):
         data = {"ö": "ö", "abc": [3]}
         url = 'https://foo.bar/baz?x=y#a'

From 266b0ad6762eaa48156b021094126062cb9f44d6 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 4 Sep 2015 20:07:36 +0100
Subject: [PATCH 2422/6219] [downloader/external] add _bool_option to pass
 value to bool option

---
 youtube_dl/downloader/external.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index 072ac3a46..22dfd2833 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -53,6 +53,14 @@ class ExternalFD(FileDownloader):
             return [command_option]
         return [command_option, param]
 
+    def _bool_option(self, command_option, param, true_value='true', false_value='false', separator=None):
+        param = self.params.get(param)
+        if not isinstance(param, bool):
+            return []
+        if separator:
+            return [command_option + separator + (true_value if param else false_value)]
+        return [command_option, true_value if param else false_value]
+
     def _configuration_args(self, default=[]):
         ex_args = self.params.get('external_downloader_args')
         if ex_args is None:
@@ -123,7 +131,7 @@ class Aria2cFD(ExternalFD):
             cmd += ['--header', '%s: %s' % (key, val)]
         cmd += self._option('--interface', 'source_address')
         cmd += self._option('--all-proxy', 'proxy')
-        cmd += self._option('--check-certificate=false', 'nocheckcertificate')
+        cmd += self._bool_option('--check-certificate', 'nocheckcertificate', 'false', 'true', '=')
         cmd += ['--', info_dict['url']]
         return cmd
 

From c482b3c69aac9069a8c1acb1f2b60d6578331461 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 4 Sep 2015 21:11:29 +0200
Subject: [PATCH 2423/6219] [ruutu] Update test url

---
 youtube_dl/extractor/ruutu.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ruutu.py b/youtube_dl/extractor/ruutu.py
index 8ca6e56bb..c67ad25ce 100644
--- a/youtube_dl/extractor/ruutu.py
+++ b/youtube_dl/extractor/ruutu.py
@@ -28,7 +28,7 @@ class RuutuIE(InfoExtractor):
             },
         },
         {
-            'url': 'http://www.ruutu.fi/ohjelmat/superpesis/superpesis-katso-koko-kausi-ruudussa',
+            'url': 'http://www.ruutu.fi/video/2057306',
             'md5': '065a10ae4d5b8cfd9d0c3d332465e3d9',
             'info_dict': {
                 'id': '2057306',

From f30c2e8e9867373815aaaa2285f23524a67da730 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 4 Sep 2015 20:57:19 +0100
Subject: [PATCH 2424/6219] [downloader/external] add _argless_option for
 option without arguments

---
 youtube_dl/downloader/external.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index 22dfd2833..cae8f6f0f 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -49,8 +49,6 @@ class ExternalFD(FileDownloader):
         param = self.params.get(param)
         if param is None or param is False:
             return []
-        if isinstance(param, bool):
-            return [command_option]
         return [command_option, param]
 
     def _bool_option(self, command_option, param, true_value='true', false_value='false', separator=None):
@@ -61,6 +59,10 @@ class ExternalFD(FileDownloader):
             return [command_option + separator + (true_value if param else false_value)]
         return [command_option, true_value if param else false_value]
 
+    def _argless_option(self, command_option, param, expected_value=True):
+        param = self.params.get(param)
+        return [command_option] if param == expected_value else []
+
     def _configuration_args(self, default=[]):
         ex_args = self.params.get('external_downloader_args')
         if ex_args is None:
@@ -89,7 +91,7 @@ class CurlFD(ExternalFD):
             cmd += ['--header', '%s: %s' % (key, val)]
         cmd += self._option('--interface', 'source_address')
         cmd += self._option('--proxy', 'proxy')
-        cmd += self._option('--insecure', 'nocheckcertificate')
+        cmd += self._argless_option('--insecure', 'nocheckcertificate')
         cmd += self._configuration_args()
         cmd += ['--', info_dict['url']]
         return cmd
@@ -112,7 +114,7 @@ class WgetFD(ExternalFD):
             cmd += ['--header', '%s: %s' % (key, val)]
         cmd += self._option('--bind-address', 'source_address')
         cmd += self._option('--proxy', 'proxy')
-        cmd += self._option('--no-check-certificate', 'nocheckcertificate')
+        cmd += self._argless_option('--no-check-certificate', 'nocheckcertificate')
         cmd += self._configuration_args()
         cmd += ['--', info_dict['url']]
         return cmd

From dc534b674f9dfbe869d23f6a64ea48f39bde6655 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 4 Sep 2015 21:12:13 +0100
Subject: [PATCH 2425/6219] [downloader/external] change _argless_option
 function to _valueless_option

---
 youtube_dl/downloader/external.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index cae8f6f0f..056215f8c 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -47,7 +47,7 @@ class ExternalFD(FileDownloader):
 
     def _option(self, command_option, param):
         param = self.params.get(param)
-        if param is None or param is False:
+        if param is None:
             return []
         return [command_option, param]
 
@@ -59,7 +59,7 @@ class ExternalFD(FileDownloader):
             return [command_option + separator + (true_value if param else false_value)]
         return [command_option, true_value if param else false_value]
 
-    def _argless_option(self, command_option, param, expected_value=True):
+    def _valueless_option(self, command_option, param, expected_value=True):
         param = self.params.get(param)
         return [command_option] if param == expected_value else []
 
@@ -91,7 +91,7 @@ class CurlFD(ExternalFD):
             cmd += ['--header', '%s: %s' % (key, val)]
         cmd += self._option('--interface', 'source_address')
         cmd += self._option('--proxy', 'proxy')
-        cmd += self._argless_option('--insecure', 'nocheckcertificate')
+        cmd += self._valueless_option('--insecure', 'nocheckcertificate')
         cmd += self._configuration_args()
         cmd += ['--', info_dict['url']]
         return cmd
@@ -114,7 +114,7 @@ class WgetFD(ExternalFD):
             cmd += ['--header', '%s: %s' % (key, val)]
         cmd += self._option('--bind-address', 'source_address')
         cmd += self._option('--proxy', 'proxy')
-        cmd += self._argless_option('--no-check-certificate', 'nocheckcertificate')
+        cmd += self._valueless_option('--no-check-certificate', 'nocheckcertificate')
         cmd += self._configuration_args()
         cmd += ['--', info_dict['url']]
         return cmd

From 66e289bab466079558b7acf5cea1057ae35c9bfa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 5 Sep 2015 03:05:11 +0600
Subject: [PATCH 2426/6219] [utils] Generalize cli option converters

---
 youtube_dl/utils.py | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index cc792be6a..79381b380 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1951,6 +1951,32 @@ def dfxp2srt(dfxp_data):
     return ''.join(out)
 
 
+def cli_option(params, command_option, param):
+    param = params.get(param)
+    return [command_option, param] if param is not None else []
+
+
+def cli_bool_option(params, command_option, param, true_value='true', false_value='false', separator=None):
+    param = params.get(param)
+    assert isinstance(param, bool)
+    if separator:
+        return [command_option + separator + (true_value if param else false_value)]
+    return [command_option, true_value if param else false_value]
+
+
+def cli_valueless_option(params, command_option, param, expected_value=True):
+    param = params.get(param)
+    return [command_option] if param == expected_value else []
+
+
+def cli_configuration_args(params, param, default=[]):
+    ex_args = params.get(param)
+    if ex_args is None:
+        return default
+    assert isinstance(ex_args, list)
+    return ex_args
+
+
 class ISO639Utils(object):
     # See http://www.loc.gov/standards/iso639-2/ISO-639-2_utf-8.txt
     _lang_map = {

From 1195a38f460a6bf895ada6405284da103dc467de Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 5 Sep 2015 03:06:28 +0600
Subject: [PATCH 2427/6219] [downloader/external] Use generalized cli option
 converters

---
 youtube_dl/downloader/external.py | 25 ++++++++-----------------
 1 file changed, 8 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index 056215f8c..2bc011266 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -5,6 +5,10 @@ import subprocess
 
 from .common import FileDownloader
 from ..utils import (
+    cli_option,
+    cli_valueless_option,
+    cli_bool_option,
+    cli_configuration_args,
     encodeFilename,
     encodeArgument,
 )
@@ -46,29 +50,16 @@ class ExternalFD(FileDownloader):
         return info_dict['protocol'] in ('http', 'https', 'ftp', 'ftps')
 
     def _option(self, command_option, param):
-        param = self.params.get(param)
-        if param is None:
-            return []
-        return [command_option, param]
+        return cli_option(self.params, command_option, param)
 
     def _bool_option(self, command_option, param, true_value='true', false_value='false', separator=None):
-        param = self.params.get(param)
-        if not isinstance(param, bool):
-            return []
-        if separator:
-            return [command_option + separator + (true_value if param else false_value)]
-        return [command_option, true_value if param else false_value]
+        return cli_bool_option(self.params, command_option, param, true_value, false_value, separator)
 
     def _valueless_option(self, command_option, param, expected_value=True):
-        param = self.params.get(param)
-        return [command_option] if param == expected_value else []
+        return cli_valueless_option(self.params, command_option, param, expected_value)
 
     def _configuration_args(self, default=[]):
-        ex_args = self.params.get('external_downloader_args')
-        if ex_args is None:
-            return default
-        assert isinstance(ex_args, list)
-        return ex_args
+        return cli_configuration_args(self.params, 'external_downloader_args', default)
 
     def _call_downloader(self, tmpfilename, info_dict):
         """ Either overwrite this or implement _make_cmd """

From 6270239a6d75be02a13aec10bb7f229370ffea6f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 5 Sep 2015 03:06:53 +0600
Subject: [PATCH 2428/6219] [postprocessor/common] Use generalized cli option
 converters

---
 youtube_dl/postprocessor/common.py | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/postprocessor/common.py b/youtube_dl/postprocessor/common.py
index 4191d040b..150ef9173 100644
--- a/youtube_dl/postprocessor/common.py
+++ b/youtube_dl/postprocessor/common.py
@@ -4,6 +4,7 @@ import os
 
 from ..utils import (
     PostProcessingError,
+    cli_configuration_args,
     encodeFilename,
 )
 
@@ -61,11 +62,7 @@ class PostProcessor(object):
             self._downloader.report_warning(errnote)
 
     def _configuration_args(self, default=[]):
-        pp_args = self._downloader.params.get('postprocessor_args')
-        if pp_args is None:
-            return default
-        assert isinstance(pp_args, list)
-        return pp_args
+        return cli_configuration_args(self.params, 'postprocessor_args', default)
 
 
 class AudioConversionError(PostProcessingError):

From f71264490cbd1bb9d953aafdf0a37aae956c24d0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 5 Sep 2015 03:07:19 +0600
Subject: [PATCH 2429/6219] [test_utils] Add tests for cli option converters

---
 test/test_utils.py | 48 ++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 48 insertions(+)

diff --git a/test/test_utils.py b/test/test_utils.py
index 15a59692f..a5f164c49 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -64,6 +64,9 @@ from youtube_dl.utils import (
     match_str,
     parse_dfxp_time_expr,
     dfxp2srt,
+    cli_option,
+    cli_valueless_option,
+    cli_bool_option,
 )
 
 
@@ -674,6 +677,51 @@ The first line
 '''
         self.assertEqual(dfxp2srt(dfxp_data_no_default_namespace), srt_data)
 
+    def test_cli_option(self):
+        self.assertEqual(cli_option({'proxy': '127.0.0.1:3128'}, '--proxy', 'proxy'), ['--proxy', '127.0.0.1:3128'])
+        self.assertEqual(cli_option({'proxy': None}, '--proxy', 'proxy'), [])
+        self.assertEqual(cli_option({}, '--proxy', 'proxy'), [])
+
+    def test_cli_valueless_option(self):
+        self.assertEqual(cli_valueless_option(
+            {'downloader': 'external'}, '--external-downloader', 'downloader', 'external'), ['--external-downloader'])
+        self.assertEqual(cli_valueless_option(
+            {'downloader': 'internal'}, '--external-downloader', 'downloader', 'external'), [])
+        self.assertEqual(cli_valueless_option(
+            {'nocheckcertificate': True}, '--no-check-certificate', 'nocheckcertificate'), ['--no-check-certificate'])
+        self.assertEqual(cli_valueless_option(
+            {'nocheckcertificate': False}, '--no-check-certificate', 'nocheckcertificate'), [])
+        self.assertEqual(cli_valueless_option(
+            {'checkcertificate': True}, '--no-check-certificate', 'checkcertificate', False), [])
+        self.assertEqual(cli_valueless_option(
+            {'checkcertificate': False}, '--no-check-certificate', 'checkcertificate', False), ['--no-check-certificate'])
+
+    def test_cli_bool_option(self):
+        self.assertEqual(
+            cli_bool_option(
+                {'nocheckcertificate': True}, '--no-check-certificate', 'nocheckcertificate'),
+            ['--no-check-certificate', 'true'])
+        self.assertEqual(
+            cli_bool_option(
+                {'nocheckcertificate': True}, '--no-check-certificate', 'nocheckcertificate', separator='='),
+            ['--no-check-certificate=true'])
+        self.assertEqual(
+            cli_bool_option(
+                {'nocheckcertificate': True}, '--check-certificate', 'nocheckcertificate', 'false', 'true'),
+            ['--check-certificate', 'false'])
+        self.assertEqual(
+            cli_bool_option(
+                {'nocheckcertificate': True}, '--check-certificate', 'nocheckcertificate', 'false', 'true', '='),
+            ['--check-certificate=false'])
+        self.assertEqual(
+            cli_bool_option(
+                {'nocheckcertificate': False}, '--check-certificate', 'nocheckcertificate', 'false', 'true'),
+            ['--check-certificate', 'true'])
+        self.assertEqual(
+            cli_bool_option(
+                {'nocheckcertificate': False}, '--check-certificate', 'nocheckcertificate', 'false', 'true', '='),
+            ['--check-certificate=true'])
+
 
 if __name__ == '__main__':
     unittest.main()

From 08354db47bba1138798c164c6bab8024600fd057 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 5 Sep 2015 15:25:19 +0800
Subject: [PATCH 2430/6219] [vlive] PEP8

---
 youtube_dl/extractor/vlive.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vlive.py b/youtube_dl/extractor/vlive.py
index a456f8217..86c1cb5ef 100644
--- a/youtube_dl/extractor/vlive.py
+++ b/youtube_dl/extractor/vlive.py
@@ -41,7 +41,7 @@ class VLiveIE(InfoExtractor):
         thumbnail = self._og_search_thumbnail(webpage)
         creator = self._html_search_regex(
             r'<span[^>]+class="name">([^<>]+)</span>', webpage, 'creator')
-        
+
         url = 'http://global.apis.naver.com/globalV/globalV/vod/%s/playinfo?' % video_id
         msgpad = '%.0f' % (time() * 1000)
         md = b64encode(

From 9dd73ef4a49597e73388e7e97e32eae5d0f0f027 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 5 Sep 2015 15:38:05 +0800
Subject: [PATCH 2431/6219] [dcn] PEP8

---
 youtube_dl/extractor/dcn.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/dcn.py b/youtube_dl/extractor/dcn.py
index 82261e25c..6f2fea5ff 100644
--- a/youtube_dl/extractor/dcn.py
+++ b/youtube_dl/extractor/dcn.py
@@ -44,8 +44,8 @@ class DCNIE(InfoExtractor):
         title = video.get('title_en') or video['title_ar']
 
         webpage = self._download_webpage(
-            'http://admin.mangomolo.com/analytics/index.php/customers/embed/video?'
-            + compat_urllib_parse.urlencode({
+            'http://admin.mangomolo.com/analytics/index.php/customers/embed/video?' +
+            compat_urllib_parse.urlencode({
                 'id': video['id'],
                 'user_id': video['user_id'],
                 'signature': video['signature'],

From 3026164b1693ef57d0712cd6eeed6ae7a79b0916 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 5 Sep 2015 12:11:36 +0200
Subject: [PATCH 2432/6219] [postprocessor/common] Use
 'self._downloader.params' instead of 'self.params'

'self.params' is not defined
---
 youtube_dl/postprocessor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/postprocessor/common.py b/youtube_dl/postprocessor/common.py
index 150ef9173..599dd1df2 100644
--- a/youtube_dl/postprocessor/common.py
+++ b/youtube_dl/postprocessor/common.py
@@ -62,7 +62,7 @@ class PostProcessor(object):
             self._downloader.report_warning(errnote)
 
     def _configuration_args(self, default=[]):
-        return cli_configuration_args(self.params, 'postprocessor_args', default)
+        return cli_configuration_args(self._downloader.params, 'postprocessor_args', default)
 
 
 class AudioConversionError(PostProcessingError):

From 486375154cb7d79bd084879467bc70550104b555 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 5 Sep 2015 11:30:42 +0100
Subject: [PATCH 2433/6219] correct season info extraction and simplify

---
 youtube_dl/extractor/dcn.py | 64 ++++++++++++++++++-------------------
 1 file changed, 31 insertions(+), 33 deletions(-)

diff --git a/youtube_dl/extractor/dcn.py b/youtube_dl/extractor/dcn.py
index 2e8fff660..8b360a9d7 100644
--- a/youtube_dl/extractor/dcn.py
+++ b/youtube_dl/extractor/dcn.py
@@ -25,19 +25,13 @@ class DCNGeneralIE(InfoExtractor):
         url = ''
         ie_key = ''
         if video_id and int(video_id) > 0:
-            url = 'http://www.dcndigital.ae/#/media/%s' % video_id
-            ie_key = 'DCNVideo'
+            return self.url_result('http://www.dcndigital.ae/#/media/%s' % video_id, 'DCNVideo')
         else:
-            ie_key = 'DCNSeason'
             if season_id and int(season_id) > 0:
                 url = smuggle_url('http://www.dcndigital.ae/#/program/season/%s' % season_id, {'show_id': show_id})
             else:
                 url = 'http://www.dcndigital.ae/#/program/%s' % show_id
-        return {
-            'url': url,
-            '_type': 'url',
-            'ie_key': ie_key
-        }
+            return self.url_result(url, 'DCNSeason')
 
 
 class DCNVideoIE(InfoExtractor):
@@ -71,6 +65,11 @@ class DCNVideoIE(InfoExtractor):
 
         video = self._download_json(request, video_id)
         title = video.get('title_en') or video['title_ar']
+        img = video.get('img')
+        thumbnail = 'http://admin.mangomolo.com/analytics/%s' % img if img else None
+        duration = int_or_none(video.get('duration'))
+        description = video.get('description_en') or video.get('description_ar')
+        timestamp = parse_iso8601(video.get('create_time') or video.get('update_time'), ' ')
 
         webpage = self._download_webpage(
             'http://admin.mangomolo.com/analytics/index.php/customers/embed/video?'
@@ -96,12 +95,6 @@ class DCNVideoIE(InfoExtractor):
 
         self._sort_formats(formats)
 
-        img = video.get('img')
-        thumbnail = 'http://admin.mangomolo.com/analytics/%s' % img if img else None
-        duration = int_or_none(video.get('duration'))
-        description = video.get('description_en') or video.get('description_ar')
-        timestamp = parse_iso8601(video.get('create_time') or video.get('update_time'), ' ')
-
         return {
             'id': video_id,
             'title': title,
@@ -122,7 +115,9 @@ class DCNLiveIE(InfoExtractor):
         {
             'id': '6',
             'ext': 'mp4',
-            'title': 'Dubai Al Oula',
+            'title': 're:^Dubai Al Oula [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
+            'thumbnail': 're:^https?://.*\.png$',
+            'is_live': True,
         },
         'params': {
             # m3u8 download
@@ -139,10 +134,14 @@ class DCNLiveIE(InfoExtractor):
 
         channel = self._download_json(request, channel_id)
         title = channel.get('title_en') or channel['title_ar']
+        img = channel.get('thumbnail')
+        thumbnail = 'http://admin.mangomolo.com/analytics/%s' % img if img else None
+        description = channel.get('description_en') or channel.get('description_ar')
+        timestamp = parse_iso8601(channel.get('create_time') or channel.get('update_time'), ' ')
 
         webpage = self._download_webpage(
-            'http://admin.mangomolo.com/analytics/index.php/customers/embed/index?'
-            + compat_urllib_parse.urlencode({
+            'http://admin.mangomolo.com/analytics/index.php/customers/embed/index?' +
+            compat_urllib_parse.urlencode({
                 'id': base64.b64encode(channel['user_id'].encode()).decode(),
                 'channelid': base64.b64encode(channel['id'].encode()).decode(),
                 'signature': channel['signature'],
@@ -166,7 +165,9 @@ class DCNLiveIE(InfoExtractor):
 
         return {
             'id': channel_id,
-            'title': title,
+            'title': self._live_title(title),
+            'description': description,
+            'thumbnail': thumbnail,
             'formats': formats,
             'is_live': True,
         }
@@ -181,7 +182,6 @@ class DCNSeasonIE(InfoExtractor):
         {
             'id': '7910',
             'title': 'محاضرات الشيخ الشعراوي',
-            'description': '',
         },
         'playlist_mincount': 27,
     }
@@ -189,6 +189,7 @@ class DCNSeasonIE(InfoExtractor):
     def _real_extract(self, url):
         url, smuggled_data = unsmuggle_url(url, {})
         show_id, season_id = re.match(self._VALID_URL, url).groups()
+
         data = {}
         if season_id:
             data['season'] = season_id
@@ -207,21 +208,18 @@ class DCNSeasonIE(InfoExtractor):
                 'Origin': 'http://www.dcndigital.ae',
                 'Content-Type': 'application/x-www-form-urlencoded'
             })
+
         show = self._download_json(request, show_id)
         season_id = season_id or show['default_season']
-        title = show['cat'].get('title_en') or show['cat']['title_ar']
-        description = show['cat'].get('description_en') or show['cat'].get('description_ar')
+        season = {}
+        for _ in show['seasons']:
+            if _['id'] == season_id:
+                season = _
+                break
+        title = season.get('title_en') or season['title_ar']
+
         entries = []
         for video in show['videos']:
-            entries.append({
-                'url': 'http://www.dcndigital.ae/#/media/%s' % video['id'],
-                '_type': 'url',
-                'ie_key': 'DCNVideo',
-            })
-        return {
-            'id': season_id,
-            'title': title,
-            'description': description,
-            'entries': entries,
-            '_type': 'playlist',
-        }
+            entries.append(self.url_result('http://www.dcndigital.ae/#/media/%s' % video['id'], 'DCNVideo'))
+
+        return self.playlist_result(entries, season_id, title)

From 4abf617b9c662851889de4a3ceb5aa63040dc21e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 5 Sep 2015 19:17:30 +0600
Subject: [PATCH 2434/6219] [YoutubeDL] Ensure bool params always present in
 downloader

---
 youtube_dl/YoutubeDL.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 982e658ce..0afda5ecb 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -284,7 +284,11 @@ class YoutubeDL(object):
         self._num_downloads = 0
         self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
         self._err_file = sys.stderr
-        self.params = params
+        self.params = {
+            # Default parameters
+            'nocheckcertificate': False,
+        }
+        self.params.update(params)
         self.cache = Cache(self)
 
         if params.get('bidi_workaround', False):

From c23c3d7d7d3ca9368b6fca70b696fe64ca55ba07 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 5 Sep 2015 16:38:05 +0100
Subject: [PATCH 2435/6219] [nowness] fix video extraction and add support
 serie and playlist extraction (fixes #6720)

---
 youtube_dl/extractor/__init__.py |   6 +-
 youtube_dl/extractor/nowness.py  | 109 +++++++++++++++++++++++--------
 2 files changed, 88 insertions(+), 27 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 5d2ea39d0..7408f68d6 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -403,7 +403,11 @@ from .normalboots import NormalbootsIE
 from .nosvideo import NosVideoIE
 from .nova import NovaIE
 from .novamov import NovaMovIE
-from .nowness import NownessIE
+from .nowness import (
+    NownessIE,
+    NownessPlaylistIE,
+    NownessSerieIE,
+)
 from .nowtv import NowTVIE
 from .nowvideo import NowVideoIE
 from .npo import (
diff --git a/youtube_dl/extractor/nowness.py b/youtube_dl/extractor/nowness.py
index 6b2f3f55a..1acb9b92a 100644
--- a/youtube_dl/extractor/nowness.py
+++ b/youtube_dl/extractor/nowness.py
@@ -1,19 +1,42 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .brightcove import BrightcoveIE
 from .common import InfoExtractor
 from ..utils import ExtractorError
+from ..compat import compat_urllib_request
 
 
-class NownessIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:(?:www|cn)\.)?nowness\.com/[^?#]*?/(?P<id>[0-9]+)/(?P<slug>[^/]+?)(?:$|[?#])'
+class NownessBaseIE(InfoExtractor):
+    def extract_url_result(self, post):
+        if post['type'] == 'video':
+            for media in post['media']:
+                if media['type'] == 'video':
+                    video_id = media['content']
+                    source = media['source']
+                    if source == 'brightcove':
+                        player_code = self._download_webpage(
+                            'http://www.nowness.com/iframe?id=%s' % video_id, video_id,
+                            note='Downloading player JavaScript',
+                            errnote='Player download failed')
+                        bc_url = BrightcoveIE._extract_brightcove_url(player_code)
+                        if bc_url is None:
+                            raise ExtractorError('Could not find player definition')
+                        return self.url_result(bc_url, 'Brightcove')
+                    elif source == 'vimeo':
+                        return self.url_result('http://vimeo.com/%s' % video_id, 'Vimeo')
+                    elif source == 'youtube':
+                        return self.url_result(video_id, 'Youtube')
+                    elif source == 'cinematique':
+                        return self.url_result('http://cinematique.com/embed/%s' % video_id, 'Cinematique')
 
+
+class NownessIE(NownessBaseIE):
+    IE_NAME = 'nowness'
+    _VALID_URL = r'https?://(?:(?:www|cn)\.)?nowness\.com/(story|series/[^/])/(?P<id>[0-9a-z-]+)'
     _TESTS = [
         {
-            'url': 'http://www.nowness.com/day/2013/6/27/3131/candor--the-art-of-gesticulation',
+            'url': 'https://www.nowness.com/story/candor-the-art-of-gesticulation',
             'md5': '068bc0202558c2e391924cb8cc470676',
             'info_dict': {
                 'id': '2520295746001',
@@ -25,7 +48,7 @@ class NownessIE(InfoExtractor):
             }
         },
         {
-            'url': 'http://cn.nowness.com/day/2014/8/7/4069/kasper-bj-rke-ft-jaakko-eino-kalevi--tnr',
+            'url': 'https://cn.nowness.com/story/kasper-bjorke-ft-jaakko-eino-kalevi-tnr',
             'md5': 'e79cf125e387216f86b2e0a5b5c63aa3',
             'info_dict': {
                 'id': '3716354522001',
@@ -39,26 +62,60 @@ class NownessIE(InfoExtractor):
     ]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('slug')
+        display_id = self._match_id(url)
 
-        webpage = self._download_webpage(url, video_id)
-        player_url = self._search_regex(
-            r'"([^"]+/content/issue-[0-9.]+.js)"', webpage, 'player URL')
-        real_id = self._search_regex(
-            r'\sdata-videoId="([0-9]+)"', webpage, 'internal video ID')
+        lang = 'zh-cn' if 'cn.nowness.com' in url else 'en-us'
+        request = compat_urllib_request.Request('http://api.nowness.com/api/post/getBySlug/%s' % display_id, headers={
+            'X-Nowness-Language': lang,
+        })
+        post = self._download_json(request, display_id)
+        return self.extract_url_result(post)
 
-        player_code = self._download_webpage(
-            player_url, video_id,
-            note='Downloading player JavaScript',
-            errnote='Player download failed')
-        player_code = player_code.replace("'+d+'", real_id)
 
-        bc_url = BrightcoveIE._extract_brightcove_url(player_code)
-        if bc_url is None:
-            raise ExtractorError('Could not find player definition')
-        return {
-            '_type': 'url',
-            'url': bc_url,
-            'ie_key': 'Brightcove',
-        }
+class NownessPlaylistIE(NownessBaseIE):
+    IE_NAME = 'nowness:playlist'
+    _VALID_URL = r'https?://(?:(?:www|cn)\.)?nowness\.com/playlist/(?P<id>\d+)/[0-9a-z-]+'
+    _TEST = {
+        'url': 'https://www.nowness.com/playlist/3286/i-guess-thats-why-they-call-it-the-blues',
+        'info_dict':
+        {
+            'id': '3286',
+        },
+        'playlist_mincount': 8,
+    }
+
+    def _real_extract(self, url):
+        playlist_id = self._match_id(url)
+
+        lang = 'zh-cn' if 'cn.nowness.com' in url else 'en-us'
+        request = compat_urllib_request.Request('http://api.nowness.com/api/post?PlaylistId=%s' % playlist_id, headers={
+            'X-Nowness-Language': lang,
+        })
+        playlist = self._download_json(request, playlist_id)
+        entries = [self.extract_url_result(item) for item in playlist['items']]
+        return self.playlist_result(entries, playlist_id)
+
+
+class NownessSerieIE(NownessBaseIE):
+    IE_NAME = 'nowness:serie'
+    _VALID_URL = r'https?://(?:(?:www|cn)\.)?nowness\.com/series/(?P<id>[0-9a-z-]+)'
+    _TEST = {
+        'url': 'https://www.nowness.com/series/60-seconds',
+        'info_dict':
+        {
+            'id': '60',
+        },
+        'playlist_mincount': 4,
+    }
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+
+        lang = 'zh-cn' if 'cn.nowness.com' in url else 'en-us'
+        request = compat_urllib_request.Request('https://api.nowness.com/api/series/getBySlug/%s' % display_id, headers={
+            'X-Nowness-Language': lang,
+        })
+        serie = self._download_json(request, display_id)
+        serie_id = str(serie['id'])
+        entries = [self.extract_url_result(post) for post in serie['posts']]
+        return self.playlist_result(entries, serie_id)

From 51f579b6358ab56ff9d03ef72e5bfeb345632b4e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 5 Sep 2015 21:40:44 +0600
Subject: [PATCH 2436/6219] [compat] Add compat shlex.split

---
 youtube_dl/compat.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index ace5bd716..9b77316f9 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -5,6 +5,7 @@ import getpass
 import optparse
 import os
 import re
+import shlex
 import shutil
 import socket
 import subprocess
@@ -227,6 +228,17 @@ except ImportError:  # Python < 3.3
             return "'" + s.replace("'", "'\"'\"'") + "'"
 
 
+if sys.version_info > (2, 7, 2):
+    compat_shlex_split = shlex.split
+else:
+    # Working around shlex issue with unicode strings on some python 2
+    # versions (see http://bugs.python.org/issue1548891)
+    def compat_shlex_split(s, comments=False, posix=True):
+        if isinstance(s, unicode):
+            s = s.encode('utf-8')
+        return shlex.split(s, comments, posix)
+
+
 def compat_ord(c):
     if type(c) is int:
         return c
@@ -459,6 +471,7 @@ __all__ = [
     'compat_ord',
     'compat_parse_qs',
     'compat_print',
+    'compat_shlex_split',
     'compat_socket_create_connection',
     'compat_str',
     'compat_subprocess_get_DEVNULL',

From ee087c79adc0cd94458d6fa0191acb4592bb0e45 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 5 Sep 2015 21:41:34 +0600
Subject: [PATCH 2437/6219] [test_compat] Add test for compat_shlex_split

---
 test/test_compat.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/test/test_compat.py b/test/test_compat.py
index c3ba8ad2e..d41a25e8e 100644
--- a/test/test_compat.py
+++ b/test/test_compat.py
@@ -14,6 +14,7 @@ from youtube_dl.utils import get_filesystem_encoding
 from youtube_dl.compat import (
     compat_getenv,
     compat_expanduser,
+    compat_shlex_split,
     compat_urllib_parse_unquote,
     compat_urllib_parse_unquote_plus,
 )
@@ -67,5 +68,8 @@ class TestCompat(unittest.TestCase):
         self.assertEqual(compat_urllib_parse_unquote_plus('abc%20def'), 'abc def')
         self.assertEqual(compat_urllib_parse_unquote_plus('%7e/abc+def'), '~/abc def')
 
+    def test_compat_shlex(self):
+        self.assertEqual(compat_shlex_split('-option "one two"'), ['-option', 'one two'])
+
 if __name__ == '__main__':
     unittest.main()

From 8870358b1b1a13c936277081b668f94c5abf95e3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 5 Sep 2015 21:42:22 +0600
Subject: [PATCH 2438/6219] [__init__] Use compat_shlex_split

---
 youtube_dl/__init__.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 55b22c889..5e2ed4d4b 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -9,7 +9,6 @@ import codecs
 import io
 import os
 import random
-import shlex
 import sys
 
 
@@ -20,6 +19,7 @@ from .compat import (
     compat_expanduser,
     compat_getpass,
     compat_print,
+    compat_shlex_split,
     workaround_optparse_bug9161,
 )
 from .utils import (
@@ -262,10 +262,10 @@ def _real_main(argv=None):
             parser.error('setting filesize xattr requested but python-xattr is not available')
     external_downloader_args = None
     if opts.external_downloader_args:
-        external_downloader_args = shlex.split(opts.external_downloader_args)
+        external_downloader_args = compat_shlex_split(opts.external_downloader_args)
     postprocessor_args = None
     if opts.postprocessor_args:
-        postprocessor_args = shlex.split(opts.postprocessor_args)
+        postprocessor_args = compat_shlex_split(opts.postprocessor_args)
     match_filter = (
         None if opts.match_filter is None
         else match_filter_func(opts.match_filter))

From cab792abe521d36e1cf296fa8578a37a499a78fe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 5 Sep 2015 21:42:41 +0600
Subject: [PATCH 2439/6219] [options] Use compat_shlex_split

---
 youtube_dl/options.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 8c4ff12bd..5eccc0a70 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -2,7 +2,6 @@ from __future__ import unicode_literals
 
 import os.path
 import optparse
-import shlex
 import sys
 
 from .downloader.external import list_external_downloaders
@@ -11,6 +10,7 @@ from .compat import (
     compat_get_terminal_size,
     compat_getenv,
     compat_kwargs,
+    compat_shlex_split,
 )
 from .utils import (
     preferredencoding,
@@ -28,7 +28,7 @@ def parseOpts(overrideArguments=None):
         try:
             res = []
             for l in optionf:
-                res += shlex.split(l, comments=True)
+                res += compat_shlex_split(l, comments=True)
         finally:
             optionf.close()
         return res

From 3513d4143657c1bdc7da72f048a63d4b1ee794d3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 5 Sep 2015 21:45:52 +0600
Subject: [PATCH 2440/6219] [test_compat] Fix typo

---
 test/test_compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/test_compat.py b/test/test_compat.py
index d41a25e8e..4ee0dc99d 100644
--- a/test/test_compat.py
+++ b/test/test_compat.py
@@ -68,7 +68,7 @@ class TestCompat(unittest.TestCase):
         self.assertEqual(compat_urllib_parse_unquote_plus('abc%20def'), 'abc def')
         self.assertEqual(compat_urllib_parse_unquote_plus('%7e/abc+def'), '~/abc def')
 
-    def test_compat_shlex(self):
+    def test_compat_shlex_split(self):
         self.assertEqual(compat_shlex_split('-option "one two"'), ['-option', 'one two'])
 
 if __name__ == '__main__':

From 12439dd5ec46ccac1ef13db7280473ecef44a096 Mon Sep 17 00:00:00 2001
From: felix <felix.von.s@posteo.de>
Date: Thu, 3 Sep 2015 07:13:05 +0200
Subject: [PATCH 2441/6219] [generic] Capture <audio> tags in addition to
 <video> tags

---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 953ec32c3..ec748ed9f 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1797,7 +1797,7 @@ class GenericIE(InfoExtractor):
                 found = filter_video(re.findall(r'<meta.*?property="og:video".*?content="(.*?)"', webpage))
         if not found:
             # HTML5 video
-            found = re.findall(r'(?s)<video[^<]*(?:>.*?<source[^>]*)?\s+src=["\'](.*?)["\']', webpage)
+            found = re.findall(r'(?s)<(?:video|audio)[^<]*(?:>.*?<source[^>]*)?\s+src=["\'](.*?)["\']', webpage)
         if not found:
             REDIRECT_REGEX = r'[0-9]{,2};\s*(?:URL|url)=\'?([^\'"]+)'
             found = re.search(

From 75b399f45557468e602c3b0126a9b5ce4736218f Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 5 Sep 2015 16:53:17 +0100
Subject: [PATCH 2442/6219] [nowness] fix _VALID_URL regex

---
 youtube_dl/extractor/nowness.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/nowness.py b/youtube_dl/extractor/nowness.py
index 1acb9b92a..401d3b646 100644
--- a/youtube_dl/extractor/nowness.py
+++ b/youtube_dl/extractor/nowness.py
@@ -33,7 +33,7 @@ class NownessBaseIE(InfoExtractor):
 
 class NownessIE(NownessBaseIE):
     IE_NAME = 'nowness'
-    _VALID_URL = r'https?://(?:(?:www|cn)\.)?nowness\.com/(story|series/[^/])/(?P<id>[0-9a-z-]+)'
+    _VALID_URL = r'https?://(?:(?:www|cn)\.)?nowness\.com/(story|series/[^/]+)/(?P<id>[0-9a-z-]+)'
     _TESTS = [
         {
             'url': 'https://www.nowness.com/story/candor-the-art-of-gesticulation',
@@ -112,7 +112,7 @@ class NownessSerieIE(NownessBaseIE):
         display_id = self._match_id(url)
 
         lang = 'zh-cn' if 'cn.nowness.com' in url else 'en-us'
-        request = compat_urllib_request.Request('https://api.nowness.com/api/series/getBySlug/%s' % display_id, headers={
+        request = compat_urllib_request.Request('http://api.nowness.com/api/series/getBySlug/%s' % display_id, headers={
             'X-Nowness-Language': lang,
         })
         serie = self._download_json(request, display_id)

From 8df5ae15d1f010bd96a8c51087718c5f41d3d90f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 5 Sep 2015 22:21:06 +0600
Subject: [PATCH 2443/6219] [compat] Fix python version check for
 compat_shlex_split

---
 youtube_dl/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 9b77316f9..e32bef279 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -228,7 +228,7 @@ except ImportError:  # Python < 3.3
             return "'" + s.replace("'", "'\"'\"'") + "'"
 
 
-if sys.version_info > (2, 7, 2):
+if sys.version_info >= (2, 7, 3):
     compat_shlex_split = shlex.split
 else:
     # Working around shlex issue with unicode strings on some python 2

From f33f32f1591087743454cbaa6a6689299f1da1d4 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 5 Sep 2015 20:32:31 +0100
Subject: [PATCH 2444/6219] [nowness] add api abstration function adn extend
 _VALID_URL regex

---
 youtube_dl/extractor/nowness.py | 40 +++++++++++++--------------------
 1 file changed, 16 insertions(+), 24 deletions(-)

diff --git a/youtube_dl/extractor/nowness.py b/youtube_dl/extractor/nowness.py
index 401d3b646..04d614c9c 100644
--- a/youtube_dl/extractor/nowness.py
+++ b/youtube_dl/extractor/nowness.py
@@ -30,10 +30,20 @@ class NownessBaseIE(InfoExtractor):
                     elif source == 'cinematique':
                         return self.url_result('http://cinematique.com/embed/%s' % video_id, 'Cinematique')
 
+    def api_request(self, url, request_url):
+        id = self._match_id(url)
+
+        lang = 'zh-cn' if 'cn.nowness.com' in url else 'en-us'
+        request = compat_urllib_request.Request(request_url % id, headers={
+            'X-Nowness-Language': lang,
+        })
+        json_data = self._download_json(request, id)
+        return id, json_data
+
 
 class NownessIE(NownessBaseIE):
     IE_NAME = 'nowness'
-    _VALID_URL = r'https?://(?:(?:www|cn)\.)?nowness\.com/(story|series/[^/]+)/(?P<id>[0-9a-z-]+)'
+    _VALID_URL = r'https?://(?:(?:www|cn)\.)?nowness\.com/(?:story|(?:series|category)/[^/]+)/(?P<id>[^/]+?)(?:$|[?#])'
     _TESTS = [
         {
             'url': 'https://www.nowness.com/story/candor-the-art-of-gesticulation',
@@ -62,19 +72,13 @@ class NownessIE(NownessBaseIE):
     ]
 
     def _real_extract(self, url):
-        display_id = self._match_id(url)
-
-        lang = 'zh-cn' if 'cn.nowness.com' in url else 'en-us'
-        request = compat_urllib_request.Request('http://api.nowness.com/api/post/getBySlug/%s' % display_id, headers={
-            'X-Nowness-Language': lang,
-        })
-        post = self._download_json(request, display_id)
+        display_id, post = self.api_request(url, 'http://api.nowness.com/api/post/getBySlug/%s')
         return self.extract_url_result(post)
 
 
 class NownessPlaylistIE(NownessBaseIE):
     IE_NAME = 'nowness:playlist'
-    _VALID_URL = r'https?://(?:(?:www|cn)\.)?nowness\.com/playlist/(?P<id>\d+)/[0-9a-z-]+'
+    _VALID_URL = r'https?://(?:(?:www|cn)\.)?nowness\.com/playlist/(?P<id>\d+)'
     _TEST = {
         'url': 'https://www.nowness.com/playlist/3286/i-guess-thats-why-they-call-it-the-blues',
         'info_dict':
@@ -85,20 +89,14 @@ class NownessPlaylistIE(NownessBaseIE):
     }
 
     def _real_extract(self, url):
-        playlist_id = self._match_id(url)
-
-        lang = 'zh-cn' if 'cn.nowness.com' in url else 'en-us'
-        request = compat_urllib_request.Request('http://api.nowness.com/api/post?PlaylistId=%s' % playlist_id, headers={
-            'X-Nowness-Language': lang,
-        })
-        playlist = self._download_json(request, playlist_id)
+        playlist_id, playlist = self.api_request(url, 'http://api.nowness.com/api/post?PlaylistId=%s')
         entries = [self.extract_url_result(item) for item in playlist['items']]
         return self.playlist_result(entries, playlist_id)
 
 
 class NownessSerieIE(NownessBaseIE):
     IE_NAME = 'nowness:serie'
-    _VALID_URL = r'https?://(?:(?:www|cn)\.)?nowness\.com/series/(?P<id>[0-9a-z-]+)'
+    _VALID_URL = r'https?://(?:(?:www|cn)\.)?nowness\.com/series/(?P<id>[^/]+?)(?:$|[?#])'
     _TEST = {
         'url': 'https://www.nowness.com/series/60-seconds',
         'info_dict':
@@ -109,13 +107,7 @@ class NownessSerieIE(NownessBaseIE):
     }
 
     def _real_extract(self, url):
-        display_id = self._match_id(url)
-
-        lang = 'zh-cn' if 'cn.nowness.com' in url else 'en-us'
-        request = compat_urllib_request.Request('http://api.nowness.com/api/series/getBySlug/%s' % display_id, headers={
-            'X-Nowness-Language': lang,
-        })
-        serie = self._download_json(request, display_id)
+        display_id, serie = self.api_request(url, 'http://api.nowness.com/api/series/getBySlug/%s')
         serie_id = str(serie['id'])
         entries = [self.extract_url_result(post) for post in serie['posts']]
         return self.playlist_result(entries, serie_id)

From f95c5e1218aacc5cc4c01b19f5ee9f7baf9269d6 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 5 Sep 2015 20:51:27 +0100
Subject: [PATCH 2445/6219] [nowness] change id variable name

---
 youtube_dl/extractor/nowness.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/nowness.py b/youtube_dl/extractor/nowness.py
index 04d614c9c..a8ee4fe68 100644
--- a/youtube_dl/extractor/nowness.py
+++ b/youtube_dl/extractor/nowness.py
@@ -31,14 +31,14 @@ class NownessBaseIE(InfoExtractor):
                         return self.url_result('http://cinematique.com/embed/%s' % video_id, 'Cinematique')
 
     def api_request(self, url, request_url):
-        id = self._match_id(url)
+        display_id = self._match_id(url)
 
         lang = 'zh-cn' if 'cn.nowness.com' in url else 'en-us'
-        request = compat_urllib_request.Request(request_url % id, headers={
+        request = compat_urllib_request.Request(request_url % display_id, headers={
             'X-Nowness-Language': lang,
         })
-        json_data = self._download_json(request, id)
-        return id, json_data
+        json_data = self._download_json(request, display_id)
+        return display_id, json_data
 
 
 class NownessIE(NownessBaseIE):

From 673bf566fc1aa1351a40c7378893aff6cb0fc91b Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 5 Sep 2015 21:03:13 +0100
Subject: [PATCH 2446/6219] [nowness] more api abstraction

---
 youtube_dl/extractor/nowness.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/nowness.py b/youtube_dl/extractor/nowness.py
index a8ee4fe68..3371145c1 100644
--- a/youtube_dl/extractor/nowness.py
+++ b/youtube_dl/extractor/nowness.py
@@ -30,11 +30,11 @@ class NownessBaseIE(InfoExtractor):
                     elif source == 'cinematique':
                         return self.url_result('http://cinematique.com/embed/%s' % video_id, 'Cinematique')
 
-    def api_request(self, url, request_url):
+    def api_request(self, url, request_path):
         display_id = self._match_id(url)
 
         lang = 'zh-cn' if 'cn.nowness.com' in url else 'en-us'
-        request = compat_urllib_request.Request(request_url % display_id, headers={
+        request = compat_urllib_request.Request('http://api.nowness.com/api/' + request_path % display_id, headers={
             'X-Nowness-Language': lang,
         })
         json_data = self._download_json(request, display_id)
@@ -72,7 +72,7 @@ class NownessIE(NownessBaseIE):
     ]
 
     def _real_extract(self, url):
-        display_id, post = self.api_request(url, 'http://api.nowness.com/api/post/getBySlug/%s')
+        display_id, post = self.api_request(url, 'post/getBySlug/%s')
         return self.extract_url_result(post)
 
 
@@ -89,7 +89,7 @@ class NownessPlaylistIE(NownessBaseIE):
     }
 
     def _real_extract(self, url):
-        playlist_id, playlist = self.api_request(url, 'http://api.nowness.com/api/post?PlaylistId=%s')
+        playlist_id, playlist = self.api_request(url, 'post?PlaylistId=%s')
         entries = [self.extract_url_result(item) for item in playlist['items']]
         return self.playlist_result(entries, playlist_id)
 
@@ -107,7 +107,7 @@ class NownessSerieIE(NownessBaseIE):
     }
 
     def _real_extract(self, url):
-        display_id, serie = self.api_request(url, 'http://api.nowness.com/api/series/getBySlug/%s')
+        display_id, serie = self.api_request(url, 'series/getBySlug/%s')
         serie_id = str(serie['id'])
         entries = [self.extract_url_result(post) for post in serie['posts']]
         return self.playlist_result(entries, serie_id)

From f43c1631582267fba591d150a18bee764bb37708 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 5 Sep 2015 21:18:21 +0100
Subject: [PATCH 2447/6219] [nowness] pass cinematique

---
 youtube_dl/extractor/nowness.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nowness.py b/youtube_dl/extractor/nowness.py
index 3371145c1..9a0e46917 100644
--- a/youtube_dl/extractor/nowness.py
+++ b/youtube_dl/extractor/nowness.py
@@ -28,7 +28,9 @@ class NownessBaseIE(InfoExtractor):
                     elif source == 'youtube':
                         return self.url_result(video_id, 'Youtube')
                     elif source == 'cinematique':
-                        return self.url_result('http://cinematique.com/embed/%s' % video_id, 'Cinematique')
+                        # youtube-dl currently doesn't support cinematique
+                        # return self.url_result('http://cinematique.com/embed/%s' % video_id, 'Cinematique')
+                        pass
 
     def api_request(self, url, request_path):
         display_id = self._match_id(url)

From eb387896e90d3c0b9043169769783796c564f4a0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Sep 2015 02:24:14 +0600
Subject: [PATCH 2448/6219] [adultswim] Fix extraction on python 2.6

---
 youtube_dl/extractor/adultswim.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/adultswim.py b/youtube_dl/extractor/adultswim.py
index 39335b827..4327c2f61 100644
--- a/youtube_dl/extractor/adultswim.py
+++ b/youtube_dl/extractor/adultswim.py
@@ -156,7 +156,7 @@ class AdultSwimIE(InfoExtractor):
                 xpath_text(idoc, './/trt', 'segment duration').strip())
 
             formats = []
-            file_els = idoc.findall('.//files/file')
+            file_els = idoc.findall('.//files/file') or idoc.findall('./files/file')
 
             for file_el in file_els:
                 bitrate = file_el.attrib.get('bitrate')

From a6420bf50cc60aa9c8e37bfb2acb10eca361a4f0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Sep 2015 06:20:48 +0600
Subject: [PATCH 2449/6219] [utils] Add cookie processor for cookie correction
 (Closes #6769)

---
 youtube_dl/utils.py | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 79381b380..9bc292a39 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -782,6 +782,32 @@ class YoutubeDLHTTPSHandler(compat_urllib_request.HTTPSHandler):
             req, **kwargs)
 
 
+class YoutubeDLCookieProcessor(compat_urllib_request.HTTPCookieProcessor):
+    def __init__(self, cookiejar=None):
+        compat_urllib_request.HTTPCookieProcessor.__init__(self, cookiejar)
+
+    def http_response(self, request, response):
+        # Python 2 will choke on next HTTP request in row if there are non-ASCII
+        # characters in Set-Cookie HTTP header of last response (see
+        # https://github.com/rg3/youtube-dl/issues/6769).
+        # In order to at least prevent crashing we will percent encode Set-Cookie
+        # header before HTTPCookieProcessor starts processing it.
+        if sys.version_info < (3, 0) and response.headers:
+            for set_cookie_header in ('Set-Cookie', 'Set-Cookie2'):
+                set_cookie = response.headers.get(set_cookie_header)
+                if set_cookie:
+                    set_cookie_escaped = '; '.join([
+                        escape_rfc3986(cookie_attr.strip())
+                        for cookie_attr in set_cookie.decode('iso-8859-1').split(';')]).encode('iso-8859-1')
+                    if set_cookie != set_cookie_escaped:
+                        del response.headers[set_cookie_header]
+                        response.headers[set_cookie_header] = set_cookie_escaped
+        return compat_urllib_request.HTTPCookieProcessor.http_response(self, request, response)
+
+    https_request = compat_urllib_request.HTTPCookieProcessor.http_request
+    https_response = http_response
+
+
 def parse_iso8601(date_str, delimiter='T', timezone=None):
     """ Return a UNIX timestamp from the given date """
 

From 6a3f4c3f829eaff87caacf162ae4fc60045bd059 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Sep 2015 06:21:33 +0600
Subject: [PATCH 2450/6219] [YoutubeDL] Use custom cookie processor

---
 youtube_dl/YoutubeDL.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 0afda5ecb..d65253882 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -69,6 +69,7 @@ from .utils import (
     version_tuple,
     write_json_file,
     write_string,
+    YoutubeDLCookieProcessor,
     YoutubeDLHandler,
     prepend_extension,
     replace_extension,
@@ -1943,8 +1944,7 @@ class YoutubeDL(object):
             if os.access(opts_cookiefile, os.R_OK):
                 self.cookiejar.load()
 
-        cookie_processor = compat_urllib_request.HTTPCookieProcessor(
-            self.cookiejar)
+        cookie_processor = YoutubeDLCookieProcessor(self.cookiejar)
         if opts_proxy is not None:
             if opts_proxy == '':
                 proxies = {}

From ad7291727499590a9449b1e3ba3d7a33b733ddd0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Sep 2015 06:23:44 +0600
Subject: [PATCH 2451/6219] [utils] Add issue URL in comment for #6457

---
 youtube_dl/utils.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 9bc292a39..7cf91ac19 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -748,7 +748,8 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
             gz = io.BytesIO(self.deflate(resp.read()))
             resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
             resp.msg = old_resp.msg
-        # Percent-encode redirect URL of Location HTTP header to satisfy RFC 3986
+        # Percent-encode redirect URL of Location HTTP header to satisfy RFC 3986 (see
+        # https://github.com/rg3/youtube-dl/issues/6457).
         if 300 <= resp.code < 400:
             location = resp.headers.get('Location')
             if location:

From be0e5dbd8356e92f8032d4cf8be2e668f1b30539 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Sep 2015 07:20:47 +0600
Subject: [PATCH 2452/6219] [extractor/common] Extract submit inputs

---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 39cef9c5b..5eeeda08d 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -733,7 +733,7 @@ class InfoExtractor(object):
     def _hidden_inputs(html):
         hidden_inputs = {}
         for input in re.findall(r'<input([^>]+)>', html):
-            if not re.search(r'type=(["\'])hidden\1', input):
+            if not re.search(r'type=(["\'])(?:hidden|submit)\1', input):
                 continue
             name = re.search(r'name=(["\'])(?P<value>.+?)\1', input)
             if not name:

From 1639282434c77f4de005fb0a6b81a49425e64d08 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Sep 2015 07:22:20 +0600
Subject: [PATCH 2453/6219] [utils] Add encode_dict

---
 youtube_dl/utils.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 7cf91ac19..5529d5291 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1638,6 +1638,10 @@ def urlencode_postdata(*args, **kargs):
     return compat_urllib_parse.urlencode(*args, **kargs).encode('ascii')
 
 
+def encode_dict(d, encoding='utf-8'):
+    return dict((k.encode(encoding), v.encode(encoding)) for k, v in d.items())
+
+
 try:
     etree_iter = xml.etree.ElementTree.Element.iter
 except AttributeError:  # Python <=2.6

From e213c98df179270a33b67ef64fa53a73f93ab4c9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Sep 2015 07:23:53 +0600
Subject: [PATCH 2454/6219] [gorillavid] Build correct URL

---
 youtube_dl/extractor/gorillavid.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/gorillavid.py b/youtube_dl/extractor/gorillavid.py
index f006f0cb1..6226b90c8 100644
--- a/youtube_dl/extractor/gorillavid.py
+++ b/youtube_dl/extractor/gorillavid.py
@@ -73,7 +73,8 @@ class GorillaVidIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
 
-        webpage = self._download_webpage('http://%s/%s' % (mobj.group('host'), video_id), video_id)
+        url = 'http://%s/%s' % (mobj.group('host'), video_id)
+        webpage = self._download_webpage(url, video_id)
 
         if re.search(self._FILE_NOT_FOUND_REGEX, webpage) is not None:
             raise ExtractorError('Video %s does not exist' % video_id, expected=True)

From e94cb5ae7e61627b3772c57c85195d0ce5fcf5bf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Sep 2015 07:24:47 +0600
Subject: [PATCH 2455/6219] [gorillavid] Add support for filehoot.com

---
 youtube_dl/extractor/gorillavid.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/gorillavid.py b/youtube_dl/extractor/gorillavid.py
index 6226b90c8..1024a5c22 100644
--- a/youtube_dl/extractor/gorillavid.py
+++ b/youtube_dl/extractor/gorillavid.py
@@ -15,10 +15,10 @@ from ..utils import (
 
 
 class GorillaVidIE(InfoExtractor):
-    IE_DESC = 'GorillaVid.in, daclips.in, movpod.in, fastvideo.in and realvid.net'
+    IE_DESC = 'GorillaVid.in, daclips.in, movpod.in, fastvideo.in, realvid.net and filehoot.com'
     _VALID_URL = r'''(?x)
         https?://(?P<host>(?:www\.)?
-            (?:daclips\.in|gorillavid\.in|movpod\.in|fastvideo\.in|realvid\.net))/
+            (?:daclips\.in|gorillavid\.in|movpod\.in|fastvideo\.in|realvid\.net|filehoot\.com))/
         (?:embed-)?(?P<id>[0-9a-zA-Z]+)(?:-[0-9]+x[0-9]+\.html)?
     '''
 

From 4abe2144994dce59b823d2bc7ea3ddc90d68b2c8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Sep 2015 07:30:02 +0600
Subject: [PATCH 2456/6219] [gorillavid] Encode fields

---
 youtube_dl/extractor/gorillavid.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/gorillavid.py b/youtube_dl/extractor/gorillavid.py
index 1024a5c22..1a42de2d5 100644
--- a/youtube_dl/extractor/gorillavid.py
+++ b/youtube_dl/extractor/gorillavid.py
@@ -10,6 +10,7 @@ from ..compat import (
 )
 from ..utils import (
     ExtractorError,
+    encode_dict,
     int_or_none,
 )
 
@@ -88,7 +89,7 @@ class GorillaVidIE(InfoExtractor):
             if countdown:
                 self._sleep(countdown, video_id)
 
-            post = compat_urllib_parse.urlencode(fields)
+            post = compat_urllib_parse.urlencode(encode_dict(fields))
 
             req = compat_urllib_request.Request(url, post)
             req.add_header('Content-type', 'application/x-www-form-urlencoded')

From b3e64671cc7a33386c2e484da8eddcc694d85302 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Sep 2015 07:30:16 +0600
Subject: [PATCH 2457/6219] [gorillavid] Improve title

---
 youtube_dl/extractor/gorillavid.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/gorillavid.py b/youtube_dl/extractor/gorillavid.py
index 1a42de2d5..afdf4e4c4 100644
--- a/youtube_dl/extractor/gorillavid.py
+++ b/youtube_dl/extractor/gorillavid.py
@@ -97,7 +97,7 @@ class GorillaVidIE(InfoExtractor):
             webpage = self._download_webpage(req, video_id, 'Downloading video page')
 
         title = self._search_regex(
-            [r'style="z-index: [0-9]+;">([^<]+)</span>', r'>Watch (.+) '],
+            [r'style="z-index: [0-9]+;">([^<]+)</span>', r'<td nowrap>([^<]+)</td>', r'>Watch (.+) '],
             webpage, 'title', default=None) or self._og_search_title(webpage)
         video_url = self._search_regex(
             r'file\s*:\s*["\'](http[^"\']+)["\'],', webpage, 'file url')

From c7c0996d8c2136dc863e45169a19def3269562a0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Sep 2015 07:32:06 +0600
Subject: [PATCH 2458/6219] [gorillavid] Add test for filehoot.com (Closes
 #6771)

---
 youtube_dl/extractor/gorillavid.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/youtube_dl/extractor/gorillavid.py b/youtube_dl/extractor/gorillavid.py
index afdf4e4c4..d23e3eac1 100644
--- a/youtube_dl/extractor/gorillavid.py
+++ b/youtube_dl/extractor/gorillavid.py
@@ -68,6 +68,14 @@ class GorillaVidIE(InfoExtractor):
     }, {
         'url': 'http://movpod.in/0wguyyxi1yca',
         'only_matching': True,
+    }, {
+        'url': 'http://filehoot.com/3ivfabn7573c.html',
+        'info_dict': {
+            'id': '3ivfabn7573c',
+            'ext': 'mp4',
+            'title': 'youtube-dl test video \'äBaW_jenozKc.mp4.mp4',
+            'thumbnail': 're:http://.*\.jpg',
+        }
     }]
 
     def _real_extract(self, url):

From 515fc8776fa1fff6ab3b1e2445ddccf5a22c520b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Sep 2015 07:42:15 +0600
Subject: [PATCH 2459/6219] [youtube] Use encode_dict

---
 youtube_dl/extractor/youtube.py | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 030ec70ca..231c86c37 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -26,6 +26,7 @@ from ..compat import (
 )
 from ..utils import (
     clean_html,
+    encode_dict,
     ExtractorError,
     float_or_none,
     get_element_by_attribute,
@@ -111,10 +112,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
             'hl': 'en_US',
         }
 
-        # Convert to UTF-8 *before* urlencode because Python 2.x's urlencode
-        # chokes on unicode
-        login_form = dict((k.encode('utf-8'), v.encode('utf-8')) for k, v in login_form_strs.items())
-        login_data = compat_urllib_parse.urlencode(login_form).encode('ascii')
+        login_data = compat_urllib_parse.urlencode(encode_dict(login_form_strs)).encode('ascii')
 
         req = compat_urllib_request.Request(self._LOGIN_URL, login_data)
         login_results = self._download_webpage(
@@ -147,8 +145,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
                 'TrustDevice': 'on',
             })
 
-            tfa_form = dict((k.encode('utf-8'), v.encode('utf-8')) for k, v in tfa_form_strs.items())
-            tfa_data = compat_urllib_parse.urlencode(tfa_form).encode('ascii')
+            tfa_data = compat_urllib_parse.urlencode(encode_dict(tfa_form_strs)).encode('ascii')
 
             tfa_req = compat_urllib_request.Request(self._TWOFACTOR_URL, tfa_data)
             tfa_results = self._download_webpage(

From cd019668dc06b3fec1de9168a7414f3bcc3fe52b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Sep 2015 07:42:45 +0600
Subject: [PATCH 2460/6219] [niconico] Use encode_dict

---
 youtube_dl/extractor/niconico.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/niconico.py b/youtube_dl/extractor/niconico.py
index 0f8aa5ada..bda1cff05 100644
--- a/youtube_dl/extractor/niconico.py
+++ b/youtube_dl/extractor/niconico.py
@@ -12,6 +12,7 @@ from ..compat import (
     compat_urlparse,
 )
 from ..utils import (
+    encode_dict,
     ExtractorError,
     int_or_none,
     parse_duration,
@@ -100,10 +101,7 @@ class NiconicoIE(InfoExtractor):
             'mail': username,
             'password': password,
         }
-        # Convert to UTF-8 *before* urlencode because Python 2.x's urlencode
-        # chokes on unicode
-        login_form = dict((k.encode('utf-8'), v.encode('utf-8')) for k, v in login_form_strs.items())
-        login_data = compat_urllib_parse.urlencode(login_form).encode('utf-8')
+        login_data = compat_urllib_parse.urlencode(encode_dict(login_form_strs)).encode('utf-8')
         request = compat_urllib_request.Request(
             'https://secure.nicovideo.jp/secure/login', login_data)
         login_results = self._download_webpage(

From cf83f532ae8a21dd58e7506d1496f611c372f861 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Sep 2015 07:43:05 +0600
Subject: [PATCH 2461/6219] [niconico] Use encode_dict

---
 youtube_dl/extractor/fc2.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/fc2.py b/youtube_dl/extractor/fc2.py
index 5c1137e94..a406945e8 100644
--- a/youtube_dl/extractor/fc2.py
+++ b/youtube_dl/extractor/fc2.py
@@ -10,6 +10,7 @@ from ..compat import (
     compat_urlparse,
 )
 from ..utils import (
+    encode_dict,
     ExtractorError,
 )
 
@@ -55,10 +56,7 @@ class FC2IE(InfoExtractor):
             'Submit': ' Login ',
         }
 
-        # Convert to UTF-8 *before* urlencode because Python 2.x's urlencode
-        # chokes on unicode
-        login_form = dict((k.encode('utf-8'), v.encode('utf-8')) for k, v in login_form_strs.items())
-        login_data = compat_urllib_parse.urlencode(login_form).encode('utf-8')
+        login_data = compat_urllib_parse.urlencode(encode_dict(login_form_strs)).encode('utf-8')
         request = compat_urllib_request.Request(
             'https://secure.id.fc2.com/index.php?mode=login&switch_language=en', login_data)
 

From 266e466ee48f510226de9c7eb99f8727c99175bc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Sep 2015 07:53:11 +0600
Subject: [PATCH 2462/6219] [utils] Simplify cookie processor

---
 youtube_dl/utils.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 5529d5291..9c547729e 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -797,9 +797,7 @@ class YoutubeDLCookieProcessor(compat_urllib_request.HTTPCookieProcessor):
             for set_cookie_header in ('Set-Cookie', 'Set-Cookie2'):
                 set_cookie = response.headers.get(set_cookie_header)
                 if set_cookie:
-                    set_cookie_escaped = '; '.join([
-                        escape_rfc3986(cookie_attr.strip())
-                        for cookie_attr in set_cookie.decode('iso-8859-1').split(';')]).encode('iso-8859-1')
+                    set_cookie_escaped = compat_urllib_parse.quote(set_cookie, b"%/;:@&=+$,!~*'()?#[] ")
                     if set_cookie != set_cookie_escaped:
                         del response.headers[set_cookie_header]
                         response.headers[set_cookie_header] = set_cookie_escaped

From 12bbd32ad0993d512ef798d5a56891d82fb29d12 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Sep 2015 08:00:16 +0600
Subject: [PATCH 2463/6219] [kuwo:singer] Skip test

---
 youtube_dl/extractor/kuwo.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/kuwo.py b/youtube_dl/extractor/kuwo.py
index 1077846f2..fa233377d 100644
--- a/youtube_dl/extractor/kuwo.py
+++ b/youtube_dl/extractor/kuwo.py
@@ -202,6 +202,7 @@ class KuwoSingerIE(InfoExtractor):
             'title': 'Ali',
         },
         'playlist_mincount': 95,
+        'skip': 'Regularly stalls travis build',  # See https://travis-ci.org/rg3/youtube-dl/jobs/78878540
     }]
 
     def _real_extract(self, url):

From e28034c5ac4e08414ba63decae8bf7d956fa6d93 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Sep 2015 08:16:39 +0600
Subject: [PATCH 2464/6219] [utils] Comment cookie processing until result from
 travis and some more testing

---
 youtube_dl/utils.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 9c547729e..206dd56bc 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -793,14 +793,14 @@ class YoutubeDLCookieProcessor(compat_urllib_request.HTTPCookieProcessor):
         # https://github.com/rg3/youtube-dl/issues/6769).
         # In order to at least prevent crashing we will percent encode Set-Cookie
         # header before HTTPCookieProcessor starts processing it.
-        if sys.version_info < (3, 0) and response.headers:
-            for set_cookie_header in ('Set-Cookie', 'Set-Cookie2'):
-                set_cookie = response.headers.get(set_cookie_header)
-                if set_cookie:
-                    set_cookie_escaped = compat_urllib_parse.quote(set_cookie, b"%/;:@&=+$,!~*'()?#[] ")
-                    if set_cookie != set_cookie_escaped:
-                        del response.headers[set_cookie_header]
-                        response.headers[set_cookie_header] = set_cookie_escaped
+        # if sys.version_info < (3, 0) and response.headers:
+        #     for set_cookie_header in ('Set-Cookie', 'Set-Cookie2'):
+        #         set_cookie = response.headers.get(set_cookie_header)
+        #         if set_cookie:
+        #             set_cookie_escaped = compat_urllib_parse.quote(set_cookie, b"%/;:@&=+$,!~*'()?#[] ")
+        #             if set_cookie != set_cookie_escaped:
+        #                 del response.headers[set_cookie_header]
+        #                 response.headers[set_cookie_header] = set_cookie_escaped
         return compat_urllib_request.HTTPCookieProcessor.http_response(self, request, response)
 
     https_request = compat_urllib_request.HTTPCookieProcessor.http_request

From 7465222a9cbcc67f932ffc3468086317518becd7 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 6 Sep 2015 04:45:13 +0100
Subject: [PATCH 2465/6219] [airmozilla] fix test info

---
 youtube_dl/extractor/airmozilla.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/airmozilla.py b/youtube_dl/extractor/airmozilla.py
index 611ad1e9d..f8e70f4e5 100644
--- a/youtube_dl/extractor/airmozilla.py
+++ b/youtube_dl/extractor/airmozilla.py
@@ -20,14 +20,14 @@ class AirMozillaIE(InfoExtractor):
             'id': '6x4q2w',
             'ext': 'mp4',
             'title': 'Privacy Lab - a meetup for privacy minded people in San Francisco',
-            'thumbnail': 're:https://\w+\.cloudfront\.net/6x4q2w/poster\.jpg\?t=\d+',
+            'thumbnail': 're:https?://vid\.ly/(?P<id>[0-9a-z-]+)/poster',
             'description': 'Brings together privacy professionals and others interested in privacy at for-profits, non-profits, and NGOs in an effort to contribute to the state of the ecosystem...',
             'timestamp': 1422487800,
             'upload_date': '20150128',
             'location': 'SFO Commons',
             'duration': 3780,
             'view_count': int,
-            'categories': ['Main'],
+            'categories': ['Main', 'Privacy'],
         }
     }
 

From a2f42a3baf03f1b8ac7180337fe77ab1fd6fdd2c Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 6 Sep 2015 04:52:27 +0100
Subject: [PATCH 2466/6219] [academicearth] fix test info

---
 youtube_dl/extractor/academicearth.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/academicearth.py b/youtube_dl/extractor/academicearth.py
index 47313fba8..34095501c 100644
--- a/youtube_dl/extractor/academicearth.py
+++ b/youtube_dl/extractor/academicearth.py
@@ -15,7 +15,7 @@ class AcademicEarthCourseIE(InfoExtractor):
             'title': 'Laws of Nature',
             'description': 'Introduce yourself to the laws of nature with these free online college lectures from Yale, Harvard, and MIT.',
         },
-        'playlist_count': 4,
+        'playlist_count': 3,
     }
 
     def _real_extract(self, url):

From bca553caac52551e9222eb5bb19f3145e0c09768 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Sep 2015 10:21:15 +0600
Subject: [PATCH 2467/6219] [eagleplatform] Fix extraction

---
 youtube_dl/extractor/eagleplatform.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/eagleplatform.py b/youtube_dl/extractor/eagleplatform.py
index 688dfc2f7..a1ee51568 100644
--- a/youtube_dl/extractor/eagleplatform.py
+++ b/youtube_dl/extractor/eagleplatform.py
@@ -79,7 +79,7 @@ class EaglePlatformIE(InfoExtractor):
             age_limit = 0 if age_restriction == 'allow_all' else 18
 
         m3u8_data = self._download_json(
-            media['sources']['secure_m3u8']['auto'],
+            self._proto_relative_url(media['sources']['secure_m3u8']['auto'], 'http:'),
             video_id, 'Downloading m3u8 JSON')
 
         formats = self._extract_m3u8_formats(

From 9af461de351f6c9594e693969ec6b6a7938479bf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Sep 2015 10:28:40 +0600
Subject: [PATCH 2468/6219] [techtvmit] Fix extraction

---
 youtube_dl/extractor/mit.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/mit.py b/youtube_dl/extractor/mit.py
index d7ab6a9ae..6cef41159 100644
--- a/youtube_dl/extractor/mit.py
+++ b/youtube_dl/extractor/mit.py
@@ -33,8 +33,8 @@ class TechTVMITIE(InfoExtractor):
             'http://techtv.mit.edu/videos/%s' % video_id, video_id)
         clean_page = re.compile(r'<!--.*?-->', re.S).sub('', raw_page)
 
-        base_url = self._search_regex(
-            r'ipadUrl: \'(.+?cloudfront.net/)', raw_page, 'base url')
+        base_url = self._proto_relative_url(self._search_regex(
+            r'ipadUrl: \'(.+?cloudfront.net/)', raw_page, 'base url'), 'http:')
         formats_json = self._search_regex(
             r'bitrates: (\[.+?\])', raw_page, 'video formats')
         formats_mit = json.loads(formats_json)

From 9dbdb65abe11c8b7c56354b78431879d2ddee52a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Sep 2015 10:32:42 +0600
Subject: [PATCH 2469/6219] [techtvmit] Update test

---
 youtube_dl/extractor/mit.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/mit.py b/youtube_dl/extractor/mit.py
index 6cef41159..f088ab9e2 100644
--- a/youtube_dl/extractor/mit.py
+++ b/youtube_dl/extractor/mit.py
@@ -18,12 +18,12 @@ class TechTVMITIE(InfoExtractor):
 
     _TEST = {
         'url': 'http://techtv.mit.edu/videos/25418-mit-dna-learning-center-set',
-        'md5': '1f8cb3e170d41fd74add04d3c9330e5f',
+        'md5': '00a3a27ee20d44bcaa0933ccec4a2cf7',
         'info_dict': {
             'id': '25418',
             'ext': 'mp4',
-            'title': 'MIT DNA Learning Center Set',
-            'description': 'md5:82313335e8a8a3f243351ba55bc1b474',
+            'title': 'MIT DNA and Protein Sets',
+            'description': 'md5:46f5c69ce434f0a97e7c628cc142802d',
         },
     }
 

From 97243fe3957769cc91f6a4c936dcccfa356f82db Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 7 Sep 2015 04:52:26 +0600
Subject: [PATCH 2470/6219] [ceskatelevize] Extract playlists (Closes #6772)

---
 youtube_dl/extractor/ceskatelevize.py | 156 ++++++++++++++++----------
 1 file changed, 96 insertions(+), 60 deletions(-)

diff --git a/youtube_dl/extractor/ceskatelevize.py b/youtube_dl/extractor/ceskatelevize.py
index dda583680..e857e66f4 100644
--- a/youtube_dl/extractor/ceskatelevize.py
+++ b/youtube_dl/extractor/ceskatelevize.py
@@ -17,55 +17,81 @@ from ..utils import (
 
 
 class CeskaTelevizeIE(InfoExtractor):
-    _VALID_URL = r'https?://www\.ceskatelevize\.cz/(porady|ivysilani)/(.+/)?(?P<id>[^?#]+)'
-
-    _TESTS = [
-        {
-            'url': 'http://www.ceskatelevize.cz/ivysilani/ivysilani/10441294653-hyde-park-civilizace/214411058091220',
-            'info_dict': {
-                'id': '214411058091220',
-                'ext': 'mp4',
-                'title': 'Hyde Park Civilizace',
-                'description': 'Věda a současná civilizace. Interaktivní pořad - prostor pro vaše otázky a komentáře',
-                'thumbnail': 're:^https?://.*\.jpg',
-                'duration': 3350,
-            },
-            'params': {
-                # m3u8 download
-                'skip_download': True,
-            },
+    _VALID_URL = r'https?://www\.ceskatelevize\.cz/(porady|ivysilani)/(?:[^/]+/)*(?P<id>[^/#?]+)/*(?:[#?].*)?$'
+    _TESTS = [{
+        'url': 'http://www.ceskatelevize.cz/ivysilani/ivysilani/10441294653-hyde-park-civilizace/214411058091220',
+        'info_dict': {
+            'id': '61924494876951776',
+            'ext': 'mp4',
+            'title': 'Hyde Park Civilizace',
+            'description': 'md5:fe93f6eda372d150759d11644ebbfb4a',
+            'thumbnail': 're:^https?://.*\.jpg',
+            'duration': 3350,
         },
-        {
-            'url': 'http://www.ceskatelevize.cz/ivysilani/10532695142-prvni-republika/bonus/14716-zpevacka-z-duparny-bobina',
-            'info_dict': {
-                'id': '14716',
-                'ext': 'mp4',
-                'title': 'První republika: Zpěvačka z Dupárny Bobina',
-                'description': 'Sága mapující atmosféru první republiky od r. 1918 do r. 1945.',
-                'thumbnail': 're:^https?://.*\.jpg',
-                'duration': 88.4,
-            },
-            'params': {
-                # m3u8 download
-                'skip_download': True,
-            },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
         },
-    ]
+    }, {
+        'url': 'http://www.ceskatelevize.cz/ivysilani/10532695142-prvni-republika/bonus/14716-zpevacka-z-duparny-bobina',
+        'info_dict': {
+            'id': '61924494876844374',
+            'ext': 'mp4',
+            'title': 'První republika: Zpěvačka z Dupárny Bobina',
+            'description': 'Sága mapující atmosféru první republiky od r. 1918 do r. 1945.',
+            'thumbnail': 're:^https?://.*\.jpg',
+            'duration': 88.4,
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }, {
+        # video with 18+ caution trailer
+        'url': 'http://www.ceskatelevize.cz/porady/10520528904-queer/215562210900007-bogotart/',
+        'info_dict': {
+            'id': '215562210900007-bogotart',
+            'title': 'Queer: Bogotart',
+            'description': 'Alternativní průvodce současným queer světem',
+        },
+        'playlist': [{
+            'info_dict': {
+                'id': '61924494876844842',
+                'ext': 'mp4',
+                'title': 'Queer: Bogotart (Varování 18+)',
+                'duration': 10.2,
+            },
+        }, {
+            'info_dict': {
+                'id': '61924494877068022',
+                'ext': 'mp4',
+                'title': 'Queer: Bogotart (Queer)',
+                'thumbnail': 're:^https?://.*\.jpg',
+                'duration': 1558.3,
+            },
+        }],
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }]
 
     def _real_extract(self, url):
         url = url.replace('/porady/', '/ivysilani/').replace('/video/', '')
 
         mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        playlist_id = mobj.group('id')
 
-        webpage = self._download_webpage(url, video_id)
+        webpage = self._download_webpage(url, playlist_id)
 
         NOT_AVAILABLE_STRING = 'This content is not available at your territory due to limited copyright.'
         if '%s</p>' % NOT_AVAILABLE_STRING in webpage:
             raise ExtractorError(NOT_AVAILABLE_STRING, expected=True)
 
-        typ = self._html_search_regex(r'getPlaylistUrl\(\[\{"type":"(.+?)","id":".+?"\}\],', webpage, 'type')
-        episode_id = self._html_search_regex(r'getPlaylistUrl\(\[\{"type":".+?","id":"(.+?)"\}\],', webpage, 'episode_id')
+        typ = self._html_search_regex(
+            r'getPlaylistUrl\(\[\{"type":"(.+?)","id":".+?"\}\],', webpage, 'type')
+        episode_id = self._html_search_regex(
+            r'getPlaylistUrl\(\[\{"type":".+?","id":"(.+?)"\}\],', webpage, 'episode_id')
 
         data = {
             'playlist[0][type]': typ,
@@ -83,7 +109,7 @@ class CeskaTelevizeIE(InfoExtractor):
         req.add_header('X-Requested-With', 'XMLHttpRequest')
         req.add_header('Referer', url)
 
-        playlistpage = self._download_json(req, video_id)
+        playlistpage = self._download_json(req, playlist_id)
 
         playlist_url = playlistpage['url']
         if playlist_url == 'error_region':
@@ -92,33 +118,43 @@ class CeskaTelevizeIE(InfoExtractor):
         req = compat_urllib_request.Request(compat_urllib_parse_unquote(playlist_url))
         req.add_header('Referer', url)
 
-        playlist = self._download_json(req, video_id)
+        playlist_title = self._og_search_title(webpage)
+        playlist_description = self._og_search_description(webpage)
 
-        item = playlist['playlist'][0]
-        formats = []
-        for format_id, stream_url in item['streamUrls'].items():
-            formats.extend(self._extract_m3u8_formats(stream_url, video_id, 'mp4'))
-        self._sort_formats(formats)
+        playlist = self._download_json(req, playlist_id)['playlist']
+        playlist_len = len(playlist)
 
-        title = self._og_search_title(webpage)
-        description = self._og_search_description(webpage)
-        duration = float_or_none(item.get('duration'))
-        thumbnail = item.get('previewImageUrl')
+        entries = []
+        for item in playlist:
+            formats = []
+            for format_id, stream_url in item['streamUrls'].items():
+                formats.extend(self._extract_m3u8_formats(
+                    stream_url, playlist_id, 'mp4', entry_protocol='m3u8_native'))
+            self._sort_formats(formats)
 
-        subtitles = {}
-        subs = item.get('subtitles')
-        if subs:
-            subtitles = self.extract_subtitles(episode_id, subs)
+            item_id = item.get('id') or item['assetId']
+            title = item['title']
 
-        return {
-            'id': episode_id,
-            'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
-            'duration': duration,
-            'formats': formats,
-            'subtitles': subtitles,
-        }
+            duration = float_or_none(item.get('duration'))
+            thumbnail = item.get('previewImageUrl')
+
+            subtitles = {}
+            if item.get('type') == 'VOD':
+                subs = item.get('subtitles')
+                if subs:
+                    subtitles = self.extract_subtitles(episode_id, subs)
+
+            entries.append({
+                'id': item_id,
+                'title': playlist_title if playlist_len == 1 else '%s (%s)' % (playlist_title, title),
+                'description': playlist_description if playlist_len == 1 else None,
+                'thumbnail': thumbnail,
+                'duration': duration,
+                'formats': formats,
+                'subtitles': subtitles,
+            })
+
+        return self.playlist_result(entries, playlist_id, playlist_title, playlist_description)
 
     def _get_subtitles(self, episode_id, subs):
         original_subtitles = self._download_webpage(

From daeb0f04cdd70a72ab9919340c7340d734fc5068 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 7 Sep 2015 08:09:34 +0100
Subject: [PATCH 2471/6219] [yahoo] change test md5 checksum

---
 youtube_dl/extractor/yahoo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/yahoo.py b/youtube_dl/extractor/yahoo.py
index f9afbdbab..b8579b573 100644
--- a/youtube_dl/extractor/yahoo.py
+++ b/youtube_dl/extractor/yahoo.py
@@ -101,7 +101,7 @@ class YahooIE(InfoExtractor):
             }
         }, {
             'url': 'http://news.yahoo.com/video/china-moses-crazy-blues-104538833.html',
-            'md5': '67010fdf3a08d290e060a4dd96baa07b',
+            'md5': '88e209b417f173d86186bef6e4d1f160',
             'info_dict': {
                 'id': 'f885cf7f-43d4-3450-9fac-46ac30ece521',
                 'ext': 'mp4',

From cf33a47df0806a2df349b8e2629d7ffa2c62cbdc Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 7 Sep 2015 08:22:38 +0100
Subject: [PATCH 2472/6219] [ajazzera] skip test

---
 youtube_dl/extractor/aljazeera.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/aljazeera.py b/youtube_dl/extractor/aljazeera.py
index 612708e25..184a14a4f 100644
--- a/youtube_dl/extractor/aljazeera.py
+++ b/youtube_dl/extractor/aljazeera.py
@@ -16,6 +16,7 @@ class AlJazeeraIE(InfoExtractor):
             'uploader': 'Al Jazeera English',
         },
         'add_ie': ['Brightcove'],
+        'skip': 'Not accessible from Travis CI server',
     }
 
     def _real_extract(self, url):

From c67a055d16075fb59d365f4f259af6c68d4db76b Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 7 Sep 2015 07:09:55 +0100
Subject: [PATCH 2473/6219] [test/test_write_annotations] fix test filename

Closes #6781
---
 test/test_write_annotations.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/test_write_annotations.py b/test/test_write_annotations.py
index 780636c77..84b8f39e0 100644
--- a/test/test_write_annotations.py
+++ b/test/test_write_annotations.py
@@ -33,7 +33,7 @@ params = get_params({
 
 
 TEST_ID = 'gr51aVj-mLg'
-ANNOTATIONS_FILE = TEST_ID + '.flv.annotations.xml'
+ANNOTATIONS_FILE = TEST_ID + '.annotations.xml'
 EXPECTED_ANNOTATIONS = ['Speech bubble', 'Note', 'Title', 'Spotlight', 'Label']
 
 

From 3b18f539a7e122004f4010981b044738985cf871 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 7 Sep 2015 11:18:21 +0100
Subject: [PATCH 2474/6219] [ndr] fix info extraction

---
 youtube_dl/extractor/ndr.py | 104 +++++++++++++++++-------------------
 1 file changed, 49 insertions(+), 55 deletions(-)

diff --git a/youtube_dl/extractor/ndr.py b/youtube_dl/extractor/ndr.py
index 79a13958b..77b7e5eb8 100644
--- a/youtube_dl/extractor/ndr.py
+++ b/youtube_dl/extractor/ndr.py
@@ -1,68 +1,66 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
     int_or_none,
-    qualities,
-    parse_duration,
 )
 
 
+preferences = {'xl': 4, 'l': 3, 'm': 2, 's': 1, 'xs': 0,}
+
+
 class NDRBaseIE(InfoExtractor):
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
-        page = self._download_webpage(url, video_id, 'Downloading page')
-
-        title = self._og_search_title(page).strip()
-        description = self._og_search_description(page)
-        if description:
-            description = description.strip()
-
-        duration = int_or_none(self._html_search_regex(r'duration: (\d+),\n', page, 'duration', default=None))
-        if not duration:
-            duration = parse_duration(self._html_search_regex(
-                r'(<span class="min">\d+</span>:<span class="sec">\d+</span>)',
-                page, 'duration', default=None))
+        json_data = self._download_json('http://www.ndr.de/%s-ppjson.json' % video_id, video_id, 'Downloading page')
 
         formats = []
-
-        mp3_url = re.search(r'''\{src:'(?P<audio>[^']+)', type:"audio/mp3"},''', page)
-        if mp3_url:
-            formats.append({
-                'url': mp3_url.group('audio'),
-                'format_id': 'mp3',
-            })
-
-        thumbnail = None
-
-        video_url = re.search(r'''3: \{src:'(?P<video>.+?)\.(lo|hi|hq)\.mp4', type:"video/mp4"},''', page)
-        if video_url:
-            thumbnails = re.findall(r'''\d+: \{src: "([^"]+)"(?: \|\| '[^']+')?, quality: '([^']+)'}''', page)
-            if thumbnails:
-                quality_key = qualities(['xs', 's', 'm', 'l', 'xl'])
-                largest = max(thumbnails, key=lambda thumb: quality_key(thumb[1]))
-                thumbnail = 'http://www.ndr.de' + largest[0]
-
-            for format_id in 'lo', 'hi', 'hq':
-                formats.append({
-                    'url': '%s.%s.mp4' % (video_url.group('video'), format_id),
-                    'format_id': format_id,
-                })
-
-        if not formats:
+        objetType = json_data.get('config').get('objectType')
+        if objetType == 'video':
+            for key, f in json_data.get('playlist').items():
+                if key != 'config':
+                    src = f['src']
+                    if '.f4m' in src:
+                        formats.extend(self._extract_f4m_formats(src, video_id))
+                    elif '.m3u8' in src:
+                        formats.extend(self._extract_m3u8_formats(src, video_id))
+                    else:
+                        quality = f.get('quality')
+                        formats.append({
+                            'url': src,
+                            'format_id': quality,
+                            'preference': preferences.get(quality),
+                        })
+        elif objetType == 'audio':
+            for key, f in json_data.get('playlist').items():
+                if key != 'config':
+                    formats.append({
+                        'url': f['src'],
+                        'format_id': 'mp3',
+                        
+                    })
+        else:
             raise ExtractorError('No media links available for %s' % video_id)
 
+        self._sort_formats(formats)
+
+        config = json_data.get('playlist').get('config')
+
+        title = config['title']
+        duration = int_or_none(config.get('duration'))
+        thumbnails = [{
+            'id': thumbnail.get('quality'),
+            'url': thumbnail.get('src'),
+            'preference': preferences.get(thumbnail.get('quality'))
+        } for thumbnail in config.get('poster').values()]
+
         return {
             'id': video_id,
             'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
+            'thumbnails': thumbnails,
             'duration': duration,
             'formats': formats,
         }
@@ -71,7 +69,7 @@ class NDRBaseIE(InfoExtractor):
 class NDRIE(NDRBaseIE):
     IE_NAME = 'ndr'
     IE_DESC = 'NDR.de - Mediathek'
-    _VALID_URL = r'https?://www\.ndr\.de/.+?(?P<id>\d+)\.html'
+    _VALID_URL = r'https?://www\.ndr\.de/.+?,(?P<id>\w+)\.html'
 
     _TESTS = [
         {
@@ -79,10 +77,9 @@ class NDRIE(NDRBaseIE):
             'md5': '5bc5f5b92c82c0f8b26cddca34f8bb2c',
             'note': 'Video file',
             'info_dict': {
-                'id': '25866',
+                'id': 'nordmagazin25866',
                 'ext': 'mp4',
                 'title': 'Kartoffeltage in der Lewitz',
-                'description': 'md5:48c4c04dde604c8a9971b3d4e3b9eaa8',
                 'duration': 166,
             },
             'skip': '404 Not found',
@@ -91,22 +88,20 @@ class NDRIE(NDRBaseIE):
             'url': 'http://www.ndr.de/fernsehen/Party-Poette-und-Parade,hafengeburtstag988.html',
             'md5': 'dadc003c55ae12a5d2f6bd436cd73f59',
             'info_dict': {
-                'id': '988',
+                'id': 'hafengeburtstag988',
                 'ext': 'mp4',
                 'title': 'Party, Pötte und Parade',
-                'description': 'Hunderttausende feiern zwischen Speicherstadt und St. Pauli den 826. Hafengeburtstag. Die NDR Sondersendung zeigt die schönsten und spektakulärsten Bilder vom Auftakt.',
                 'duration': 3498,
             },
         },
         {
-            'url': 'http://www.ndr.de/info/audio51535.html',
+            'url': 'http://www.ndr.de/info/La-Valette-entgeht-der-Hinrichtung,audio51535.html',
             'md5': 'bb3cd38e24fbcc866d13b50ca59307b8',
             'note': 'Audio file',
             'info_dict': {
-                'id': '51535',
+                'id': 'audio51535',
                 'ext': 'mp3',
                 'title': 'La Valette entgeht der Hinrichtung',
-                'description': 'md5:22f9541913a40fe50091d5cdd7c9f536',
                 'duration': 884,
             }
         }
@@ -115,7 +110,7 @@ class NDRIE(NDRBaseIE):
 
 class NJoyIE(NDRBaseIE):
     IE_NAME = 'N-JOY'
-    _VALID_URL = r'https?://www\.n-joy\.de/.+?(?P<id>\d+)\.html'
+    _VALID_URL = r'https?://www\.n-joy\.de/.+?,(?P<id>\w+)\.html'
 
     _TEST = {
         'url': 'http://www.n-joy.de/entertainment/comedy/comedy_contest/Benaissa-beim-NDR-Comedy-Contest,comedycontest2480.html',
@@ -124,7 +119,6 @@ class NJoyIE(NDRBaseIE):
             'id': '2480',
             'ext': 'mp4',
             'title': 'Benaissa beim NDR Comedy Contest',
-            'description': 'Von seinem sehr "behaarten" Leben lässt sich Benaissa trotz aller Schwierigkeiten nicht unterkriegen.',
             'duration': 654,
         }
     }

From 8801255d7deb6b3cbbbae75d14d855b39b309090 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 7 Sep 2015 07:47:55 +0100
Subject: [PATCH 2475/6219] [youtube] fix show extraction (fixes #4841)

Closes #6782)
---
 youtube_dl/extractor/youtube.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 231c86c37..6d7b90016 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1836,7 +1836,7 @@ class YoutubeShowIE(InfoExtractor):
     IE_NAME = 'youtube:show'
     _TESTS = [{
         'url': 'http://www.youtube.com/show/airdisasters',
-        'playlist_mincount': 3,
+        'playlist_mincount': 5,
         'info_dict': {
             'id': 'airdisasters',
             'title': 'Air Disasters',
@@ -1847,7 +1847,7 @@ class YoutubeShowIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         playlist_id = mobj.group('id')
         webpage = self._download_webpage(
-            url, playlist_id, 'Downloading show webpage')
+            'http://www.youtube.com/show/%s/playlists' % playlist_id, playlist_id, 'Downloading show webpage')
         # There's one playlist for each season of the show
         m_seasons = list(re.finditer(r'href="(/playlist\?list=.*?)"', webpage))
         self.to_screen('%s: Found %s seasons' % (playlist_id, len(m_seasons)))

From 4003bd82b099872d602713824f0a36957a4f1266 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 7 Sep 2015 12:56:16 +0200
Subject: [PATCH 2476/6219] [youtube:shows] Use https

---
 youtube_dl/extractor/youtube.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 6d7b90016..97ce36550 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1835,7 +1835,7 @@ class YoutubeShowIE(InfoExtractor):
     _VALID_URL = r'https?://www\.youtube\.com/show/(?P<id>[^?#]*)'
     IE_NAME = 'youtube:show'
     _TESTS = [{
-        'url': 'http://www.youtube.com/show/airdisasters',
+        'url': 'https://www.youtube.com/show/airdisasters',
         'playlist_mincount': 5,
         'info_dict': {
             'id': 'airdisasters',
@@ -1847,7 +1847,7 @@ class YoutubeShowIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         playlist_id = mobj.group('id')
         webpage = self._download_webpage(
-            'http://www.youtube.com/show/%s/playlists' % playlist_id, playlist_id, 'Downloading show webpage')
+            'https://www.youtube.com/show/%s/playlists' % playlist_id, playlist_id, 'Downloading show webpage')
         # There's one playlist for each season of the show
         m_seasons = list(re.finditer(r'href="(/playlist\?list=.*?)"', webpage))
         self.to_screen('%s: Found %s seasons' % (playlist_id, len(m_seasons)))

From b1b7d1ffbac7150416e3cfcb3c2fb4ddddb9a6a4 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 7 Sep 2015 12:49:59 +0100
Subject: [PATCH 2477/6219] [wimp] fix tests info

---
 youtube_dl/extractor/wimp.py | 23 ++++++++---------------
 1 file changed, 8 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/wimp.py b/youtube_dl/extractor/wimp.py
index f69d46a28..e4f50e64c 100644
--- a/youtube_dl/extractor/wimp.py
+++ b/youtube_dl/extractor/wimp.py
@@ -1,40 +1,33 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from .youtube import YoutubeIE
 
 
 class WimpIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?wimp\.com/([^/]+)/'
+    _VALID_URL = r'http://(?:www\.)?wimp\.com/(?P<id>[^/]+)/'
     _TESTS = [{
         'url': 'http://www.wimp.com/maruexhausted/',
-        'md5': 'f1acced123ecb28d9bb79f2479f2b6a1',
+        'md5': 'ee21217ffd66d058e8b16be340b74883',
         'info_dict': {
             'id': 'maruexhausted',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Maru is exhausted.',
             'description': 'md5:57e099e857c0a4ea312542b684a869b8',
         }
     }, {
-        # youtube video
         'url': 'http://www.wimp.com/clowncar/',
+        'md5': '4e2986c793694b55b37cf92521d12bb4',
         'info_dict': {
-            'id': 'cG4CEr2aiSg',
+            'id': 'clowncar',
             'ext': 'mp4',
-            'title': 'Basset hound clown car...incredible!',
-            'description': 'md5:8d228485e0719898c017203f900b3a35',
-            'uploader': 'Gretchen Hoey',
-            'uploader_id': 'gretchenandjeff1',
-            'upload_date': '20140303',
+            'title': 'It\'s like a clown car.',
+            'description': 'md5:0e56db1370a6e49c5c1d19124c0d2fb2',
         },
-        'add_ie': ['Youtube'],
     }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group(1)
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
         video_url = self._search_regex(
             [r"[\"']file[\"']\s*[:,]\s*[\"'](.+?)[\"']", r"videoId\s*:\s*[\"']([^\"']+)[\"']"],

From 8b55cadc83f198e0fa6bac7158f9b05826f39257 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 7 Sep 2015 16:39:01 +0100
Subject: [PATCH 2478/6219] [canal13cl] fix info extraction

---
 youtube_dl/extractor/__init__.py  |  2 +-
 youtube_dl/extractor/canal13cl.py | 48 -------------------
 youtube_dl/extractor/tele13.py    | 77 +++++++++++++++++++++++++++++++
 3 files changed, 78 insertions(+), 49 deletions(-)
 delete mode 100644 youtube_dl/extractor/canal13cl.py
 create mode 100644 youtube_dl/extractor/tele13.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 5d2ea39d0..661b53e63 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -67,7 +67,6 @@ from .camdemy import (
     CamdemyIE,
     CamdemyFolderIE
 )
-from .canal13cl import Canal13clIE
 from .canalplus import CanalplusIE
 from .canalc2 import Canalc2IE
 from .cbs import CBSIE
@@ -612,6 +611,7 @@ from .teachingchannel import TeachingChannelIE
 from .teamcoco import TeamcocoIE
 from .techtalks import TechTalksIE
 from .ted import TEDIE
+from .tele13 import Tele13IE
 from .telebruxelles import TeleBruxellesIE
 from .telecinco import TelecincoIE
 from .telegraaf import TelegraafIE
diff --git a/youtube_dl/extractor/canal13cl.py b/youtube_dl/extractor/canal13cl.py
deleted file mode 100644
index 93241fefe..000000000
--- a/youtube_dl/extractor/canal13cl.py
+++ /dev/null
@@ -1,48 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-import re
-
-from .common import InfoExtractor
-
-
-class Canal13clIE(InfoExtractor):
-    _VALID_URL = r'^http://(?:www\.)?13\.cl/(?:[^/?#]+/)*(?P<id>[^/?#]+)'
-    _TEST = {
-        'url': 'http://www.13.cl/t13/nacional/el-circulo-de-hierro-de-michelle-bachelet-en-su-regreso-a-la-moneda',
-        'md5': '4cb1fa38adcad8fea88487a078831755',
-        'info_dict': {
-            'id': '1403022125',
-            'display_id': 'el-circulo-de-hierro-de-michelle-bachelet-en-su-regreso-a-la-moneda',
-            'ext': 'mp4',
-            'title': 'El "círculo de hierro" de Michelle Bachelet en su regreso a La Moneda',
-            'description': '(Foto: Agencia Uno) En nueve días más, Michelle Bachelet va a asumir por segunda vez como presidenta de la República. Entre aquellos que la acompañarán hay caras que se repiten y otras que se consolidan en su entorno de colaboradores más cercanos.',
-        }
-    }
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        display_id = mobj.group('id')
-
-        webpage = self._download_webpage(url, display_id)
-
-        title = self._html_search_meta(
-            'twitter:title', webpage, 'title', fatal=True)
-        description = self._html_search_meta(
-            'twitter:description', webpage, 'description')
-        url = self._html_search_regex(
-            r'articuloVideo = \"(.*?)\"', webpage, 'url')
-        real_id = self._search_regex(
-            r'[^0-9]([0-9]{7,})[^0-9]', url, 'id', default=display_id)
-        thumbnail = self._html_search_regex(
-            r'articuloImagen = \"(.*?)\"', webpage, 'thumbnail')
-
-        return {
-            'id': real_id,
-            'display_id': display_id,
-            'url': url,
-            'title': title,
-            'description': description,
-            'ext': 'mp4',
-            'thumbnail': thumbnail,
-        }
diff --git a/youtube_dl/extractor/tele13.py b/youtube_dl/extractor/tele13.py
new file mode 100644
index 000000000..5d89e757f
--- /dev/null
+++ b/youtube_dl/extractor/tele13.py
@@ -0,0 +1,77 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import js_to_json
+
+
+class Tele13IE(InfoExtractor):
+    _VALID_URL = r'^http://(?:www\.)?t13\.cl/videos(?:/[^/]+)+/(?P<id>[\w-]+)'
+    _TESTS = [
+        {
+            'url': 'http://www.t13.cl/videos/actualidad/el-circulo-de-hierro-de-michelle-bachelet-en-su-regreso-a-la-moneda',
+            'md5': '4cb1fa38adcad8fea88487a078831755',
+            'info_dict': {
+                'id': 'el-circulo-de-hierro-de-michelle-bachelet-en-su-regreso-a-la-moneda',
+                'ext': 'mp4',
+                'title': 'El c\u00edrculo de hierro de Michelle Bachelet en su regreso a La Moneda',
+            }
+        },
+        {
+            'url': 'http://www.t13.cl/videos/mundo/tendencias/video-captan-misteriosa-bola-fuego-cielos-bangkok',
+            'md5': '65d1ae54812c96f4b345dd21d3bb1adc',
+            'info_dict': {
+                'id': 'rOoKv2OMpOw',
+                'ext': 'mp4',
+                'title': 'Shooting star seen on 7-Sep-2015',
+                'description': 'md5:a1cd2e74f6ee6851552c9cf5851d6b06',
+                'uploader': 'Porjai Jaturongkhakun',
+                'upload_date': '20150906',
+                'uploader_id': 'UCnLY_3ezwNcDSC_Wc6suZxw',
+            },
+            'add_ie': ['Youtube'],
+        }
+    ]
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, display_id)
+
+        setup_js = self._parse_json(
+            js_to_json(
+                self._search_regex(
+                    r"jwplayer\('player-vivo'\).setup\((\{.*?\})\)",
+                    webpage,
+                    'setup code',
+                    flags=re.DOTALL
+                ).replace('\n//', '')
+            ),
+            display_id
+        )
+        title = setup_js['title']
+        thumbnail = setup_js.get('image') or setup_js['playlist'][0].get('image')
+        description = self._html_search_meta(
+            'description', webpage, 'description')
+
+        formats = []
+        for f in setup_js['playlist'][0]['sources']:
+            format_url = f['file']
+            if format_url != '':
+                if '.m3u8' in format_url:
+                    formats.extend(self._extract_m3u8_formats(format_url, display_id))
+                else:
+                    if 'youtube.com' in format_url:
+                        return self.url_result(format_url, 'Youtube')
+                    else:
+                        formats.append({'url': format_url, 'format_id': f.get('label')})
+
+        return {
+            'id': display_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'formats': formats,
+        }

From 64997815c49a79c3d91e65198cc79d13bed234e9 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 7 Sep 2015 20:53:37 +0100
Subject: [PATCH 2479/6219] [ndr] add support for embeds

---
 youtube_dl/extractor/__init__.py |  2 +
 youtube_dl/extractor/ndr.py      | 84 +++++++++++++++++++++++++++-----
 2 files changed, 73 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 5d2ea39d0..84830c885 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -368,6 +368,8 @@ from .nbc import (
 from .ndr import (
     NDRIE,
     NJoyIE,
+    NDREmbedIE,
+    NJoyEmbedIE,
 )
 from .ndtv import NDTVIE
 from .netzkino import NetzkinoIE
diff --git a/youtube_dl/extractor/ndr.py b/youtube_dl/extractor/ndr.py
index 77b7e5eb8..aca85825d 100644
--- a/youtube_dl/extractor/ndr.py
+++ b/youtube_dl/extractor/ndr.py
@@ -12,21 +12,18 @@ preferences = {'xl': 4, 'l': 3, 'm': 2, 's': 1, 'xs': 0,}
 
 
 class NDRBaseIE(InfoExtractor):
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        json_data = self._download_json('http://www.ndr.de/%s-ppjson.json' % video_id, video_id, 'Downloading page')
 
+    def extract_video_info(self, playlist, video_id):
         formats = []
-        objetType = json_data.get('config').get('objectType')
-        if objetType == 'video':
-            for key, f in json_data.get('playlist').items():
+        streamType = playlist.get('config').get('streamType')
+        if streamType == 'httpVideo':
+            for key, f in playlist.items():
                 if key != 'config':
                     src = f['src']
                     if '.f4m' in src:
                         formats.extend(self._extract_f4m_formats(src, video_id))
                     elif '.m3u8' in src:
-                        formats.extend(self._extract_m3u8_formats(src, video_id))
+                        formats.extend(self._extract_m3u8_formats(src, video_id, fatal=False))
                     else:
                         quality = f.get('quality')
                         formats.append({
@@ -34,20 +31,20 @@ class NDRBaseIE(InfoExtractor):
                             'format_id': quality,
                             'preference': preferences.get(quality),
                         })
-        elif objetType == 'audio':
-            for key, f in json_data.get('playlist').items():
+        elif streamType == 'httpAudio':
+            for key, f in playlist.items():
                 if key != 'config':
                     formats.append({
                         'url': f['src'],
                         'format_id': 'mp3',
-                        
+                        'vcodec': 'none',
                     })
         else:
             raise ExtractorError('No media links available for %s' % video_id)
 
         self._sort_formats(formats)
 
-        config = json_data.get('playlist').get('config')
+        config = playlist.get('config')
 
         title = config['title']
         duration = int_or_none(config.get('duration'))
@@ -65,6 +62,25 @@ class NDRBaseIE(InfoExtractor):
             'formats': formats,
         }
 
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        json_data = self._download_json('http://www.ndr.de/%s-ppjson.json' % video_id, video_id, fatal=False)
+
+        if not json_data:
+            webpage = self._download_webpage(url, video_id)
+            embed_url = self._html_search_regex(r'<iframe[^>]+id="pp_\w+"[^>]+src="(/.*)"', webpage, 'embed url', None, False)
+            if not embed_url:
+                embed_url = self._html_search_meta('embedURL', webpage, fatal=False)
+            if embed_url:
+                if embed_url.startswith('/'):
+                    return self.url_result('http://www.ndr.de%s' % embed_url, 'NDREmbed')
+                else:
+                    return self.url_result(embed_url, 'NDREmbed')
+            raise ExtractorError('No media links available for %s' % video_id)
+
+        return self.extract_video_info(json_data['playlist'], video_id)
+
 
 class NDRIE(NDRBaseIE):
     IE_NAME = 'ndr'
@@ -116,9 +132,51 @@ class NJoyIE(NDRBaseIE):
         'url': 'http://www.n-joy.de/entertainment/comedy/comedy_contest/Benaissa-beim-NDR-Comedy-Contest,comedycontest2480.html',
         'md5': 'cb63be60cd6f9dd75218803146d8dc67',
         'info_dict': {
-            'id': '2480',
+            'id': 'comedycontest2480',
             'ext': 'mp4',
             'title': 'Benaissa beim NDR Comedy Contest',
             'duration': 654,
         }
     }
+
+
+class NDREmbedBaseIE(NDRBaseIE):
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        json_data = self._download_json('http://www.ndr.de/%s-ppjson.json' % video_id, video_id, fatal=False)
+        if not json_data:
+            raise ExtractorError('No media links available for %s' % video_id)
+        return self.extract_video_info(json_data['playlist'], video_id)
+
+
+class NDREmbedIE(NDREmbedBaseIE):
+    IE_NAME = 'ndr:embed'
+    _VALID_URL = r'https?://www\.ndr\.de/(?:[^/]+/)+(?P<id>\w+)'
+
+    _TEST = {
+        'url': 'http://www.ndr.de/fernsehen/sendungen/ndr_aktuell/ndraktuell28488-player.html',
+        'md5': 'cb63be60cd6f9dd75218803146d8dc67',
+        'info_dict': {
+            'id': 'ndraktuell28488',
+            'ext': 'mp4',
+            'title': 'Norddeutschland begrüßt Flüchtlinge',
+            'duration': 132,
+        }
+    }
+
+
+class NJoyEmbedIE(NDREmbedBaseIE):
+    IE_NAME = 'N-JOY:embed'
+    _VALID_URL = r'https?://www\.n-joy\.de/(?:[^/]+/)(?P<id>\w+)'
+
+    _TEST = {
+        'url': 'http://www.n-joy.de/entertainment/film/portraet374-player_image-832d9b79-fa8a-4026-92e2-e0fd99deb2f9_theme-n-joy.html',
+        'md5': 'cb63be60cd6f9dd75218803146d8dc67',
+        'info_dict': {
+            'id': 'portraet374',
+            'ext': 'mp4',
+            'title': 'Viviane Andereggen - "Schuld um Schuld"',
+            'duration': 129,
+        }
+    }

From 436416afe2ea70dd6b55f8c9d699ddb0bdc1ec5f Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 7 Sep 2015 21:13:49 +0100
Subject: [PATCH 2480/6219] [tele13] skip test

---
 youtube_dl/extractor/tele13.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/tele13.py b/youtube_dl/extractor/tele13.py
index 5d89e757f..f1764eb2f 100644
--- a/youtube_dl/extractor/tele13.py
+++ b/youtube_dl/extractor/tele13.py
@@ -16,8 +16,12 @@ class Tele13IE(InfoExtractor):
             'info_dict': {
                 'id': 'el-circulo-de-hierro-de-michelle-bachelet-en-su-regreso-a-la-moneda',
                 'ext': 'mp4',
-                'title': 'El c\u00edrculo de hierro de Michelle Bachelet en su regreso a La Moneda',
-            }
+                'title': 'El círculo de hierro de Michelle Bachelet en su regreso a La Moneda',
+            },
+            'params': {
+                # HTTP Error 404: Not Found
+                'skip_download': True,
+            },
         },
         {
             'url': 'http://www.t13.cl/videos/mundo/tendencias/video-captan-misteriosa-bola-fuego-cielos-bangkok',

From b88ebd472e555189cc0809f6497b931266733e1b Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 7 Sep 2015 21:25:32 +0100
Subject: [PATCH 2481/6219] [musicvault] remove extractor

---
 youtube_dl/extractor/musicvault.py | 63 ------------------------------
 1 file changed, 63 deletions(-)
 delete mode 100644 youtube_dl/extractor/musicvault.py

diff --git a/youtube_dl/extractor/musicvault.py b/youtube_dl/extractor/musicvault.py
deleted file mode 100644
index 0e46ac7c1..000000000
--- a/youtube_dl/extractor/musicvault.py
+++ /dev/null
@@ -1,63 +0,0 @@
-from __future__ import unicode_literals
-
-import re
-
-from .common import InfoExtractor
-
-
-class MusicVaultIE(InfoExtractor):
-    _VALID_URL = r'https?://www\.musicvault\.com/(?P<uploader_id>[^/?#]*)/video/(?P<display_id>[^/?#]*)_(?P<id>[0-9]+)\.html'
-    _TEST = {
-        'url': 'http://www.musicvault.com/the-allman-brothers-band/video/straight-from-the-heart_1010863.html',
-        'md5': '3adcbdb3dcc02d647539e53f284ba171',
-        'info_dict': {
-            'id': '1010863',
-            'ext': 'mp4',
-            'uploader_id': 'the-allman-brothers-band',
-            'title': 'Straight from the Heart',
-            'duration': 244,
-            'uploader': 'The Allman Brothers Band',
-            'thumbnail': 're:^https?://.*/thumbnail/.*',
-            'upload_date': '20131219',
-            'location': 'Capitol Theatre (Passaic, NJ)',
-            'description': 'Listen to The Allman Brothers Band perform Straight from the Heart at Capitol Theatre (Passaic, NJ) on Dec 16, 1981',
-            'timestamp': int,
-        }
-    }
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        display_id = mobj.group('display_id')
-        webpage = self._download_webpage(url, display_id)
-
-        thumbnail = self._search_regex(
-            r'<meta itemprop="thumbnail" content="([^"]+)"',
-            webpage, 'thumbnail', fatal=False)
-
-        data_div = self._search_regex(
-            r'(?s)<div class="data">(.*?)</div>', webpage, 'data fields')
-        uploader = self._html_search_regex(
-            r'<h1.*?>(.*?)</h1>', data_div, 'uploader', fatal=False)
-        title = self._html_search_regex(
-            r'<h2.*?>(.*?)</h2>', data_div, 'title')
-        location = self._html_search_regex(
-            r'<h4.*?>(.*?)</h4>', data_div, 'location', fatal=False)
-
-        kaltura_id = self._search_regex(
-            r'<div id="video-detail-player" data-kaltura-id="([^"]+)"',
-            webpage, 'kaltura ID')
-        wid = self._search_regex(r'/wid/_([0-9]+)/', webpage, 'wid')
-
-        return {
-            'id': mobj.group('id'),
-            '_type': 'url_transparent',
-            'url': 'kaltura:%s:%s' % (wid, kaltura_id),
-            'ie_key': 'Kaltura',
-            'display_id': display_id,
-            'uploader_id': mobj.group('uploader_id'),
-            'thumbnail': thumbnail,
-            'description': self._html_search_meta('description', webpage),
-            'location': location,
-            'title': title,
-            'uploader': uploader,
-        }

From 7a459170fa476eb5b5dde790241b4e961e0d6a22 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 8 Sep 2015 05:40:00 +0600
Subject: [PATCH 2482/6219] [__init__] Remove musicvault

---
 youtube_dl/extractor/__init__.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 5d2ea39d0..57f55b479 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -347,7 +347,6 @@ from .mtv import (
 )
 from .muenchentv import MuenchenTVIE
 from .musicplayon import MusicPlayOnIE
-from .musicvault import MusicVaultIE
 from .muzu import MuzuTVIE
 from .mwave import MwaveIE
 from .myspace import MySpaceIE, MySpaceAlbumIE

From 12810c9cd3c2a5a50ca0ea309d27af2decc8b263 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 8 Sep 2015 14:11:20 +0600
Subject: [PATCH 2483/6219] [crunchyroll] Force Accept-Language to any for all
 requests (Closes #6797)

---
 youtube_dl/extractor/crunchyroll.py | 20 ++++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index ce123482e..c5dbc682d 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -31,7 +31,23 @@ from ..aes import (
 )
 
 
-class CrunchyrollIE(InfoExtractor):
+class CrunchyrollBaseIE(InfoExtractor):
+    def _download_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True, tries=1, timeout=5, encoding=None):
+        request = (url_or_request if isinstance(url_or_request, compat_urllib_request.Request)
+                   else compat_urllib_request.Request(url_or_request))
+        # Accept-Language must be set explicitly to accept any language to avoid issues
+        # similar to https://github.com/rg3/youtube-dl/issues/6797.
+        # Along with IP address Crunchyroll uses Accept-Language to guess whether georestriction
+        # should be imposed or not (from what I can see it just takes the first language
+        # ignoring the priority and requires it to correspond the IP). By the way this causes
+        # Crunchyroll to not work in georestriction cases in some browsers that don't place
+        # the locale lang first in header. However allowing any language seems to workaround the issue.
+        request.add_header('Accept-Language', '*')
+        return super(CrunchyrollBaseIE, self)._download_webpage(
+            request, video_id, note, errnote, fatal, tries, timeout, encoding)
+
+
+class CrunchyrollIE(CrunchyrollBaseIE):
     _VALID_URL = r'https?://(?:(?P<prefix>www|m)\.)?(?P<url>crunchyroll\.(?:com|fr)/(?:media(?:-|/\?id=)|[^/]*/[^/?&]*?)(?P<video_id>[0-9]+))(?:[/?&]|$)'
     _NETRC_MACHINE = 'crunchyroll'
     _TESTS = [{
@@ -330,7 +346,7 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
         }
 
 
-class CrunchyrollShowPlaylistIE(InfoExtractor):
+class CrunchyrollShowPlaylistIE(CrunchyrollBaseIE):
     IE_NAME = "crunchyroll:playlist"
     _VALID_URL = r'https?://(?:(?P<prefix>www|m)\.)?(?P<url>crunchyroll\.com/(?!(?:news|anime-news|library|forum|launchcalendar|lineup|store|comics|freetrial|login))(?P<id>[\w\-]+))/?$'
 

From 47004d9579839137bff17949f38db2844829bbd6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 8 Sep 2015 14:37:53 +0600
Subject: [PATCH 2484/6219] [crunchyroll] Fix uploader and upload date
 extraction

---
 youtube_dl/extractor/crunchyroll.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index c5dbc682d..95952bc29 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -275,10 +275,14 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
         video_description = self._html_search_regex(r'"description":"([^"]+)', webpage, 'video_description', default='')
         if not video_description:
             video_description = None
-        video_upload_date = self._html_search_regex(r'<div>Availability for free users:(.+?)</div>', webpage, 'video_upload_date', fatal=False, flags=re.DOTALL)
+        video_upload_date = self._html_search_regex(
+            [r'<div>Availability for free users:(.+?)</div>', r'<div>[^<>]+<span>\s*(.+?\d{4})\s*</span></div>'],
+            webpage, 'video_upload_date', fatal=False, flags=re.DOTALL)
         if video_upload_date:
             video_upload_date = unified_strdate(video_upload_date)
-        video_uploader = self._html_search_regex(r'<div>\s*Publisher:(.+?)</div>', webpage, 'video_uploader', fatal=False, flags=re.DOTALL)
+        video_uploader = self._html_search_regex(
+            r'<a[^>]+href="/publisher/[^"]+"[^>]*>([^<]+)</a>', webpage,
+            'video_uploader', fatal=False)
 
         playerdata_url = compat_urllib_parse_unquote(self._html_search_regex(r'"config_url":"([^"]+)', webpage, 'playerdata_url'))
         playerdata_req = compat_urllib_request.Request(playerdata_url)

From b58a22b9632a226fc453bb52f195e7577476f35e Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 8 Sep 2015 15:37:20 +0100
Subject: [PATCH 2485/6219] [ndr] make NDREmbed and NJoyEmbed _VALID_URL regex
 unique

---
 youtube_dl/extractor/ndr.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/ndr.py b/youtube_dl/extractor/ndr.py
index aca85825d..2a73650d4 100644
--- a/youtube_dl/extractor/ndr.py
+++ b/youtube_dl/extractor/ndr.py
@@ -152,7 +152,7 @@ class NDREmbedBaseIE(NDRBaseIE):
 
 class NDREmbedIE(NDREmbedBaseIE):
     IE_NAME = 'ndr:embed'
-    _VALID_URL = r'https?://www\.ndr\.de/(?:[^/]+/)+(?P<id>\w+)'
+    _VALID_URL = r'https?://www\.ndr\.de/(?:[^/]+/)+(?P<id>[a-z0-9]+)-(?:player|externalPlayer)\.html'
 
     _TEST = {
         'url': 'http://www.ndr.de/fernsehen/sendungen/ndr_aktuell/ndraktuell28488-player.html',
@@ -168,7 +168,7 @@ class NDREmbedIE(NDREmbedBaseIE):
 
 class NJoyEmbedIE(NDREmbedBaseIE):
     IE_NAME = 'N-JOY:embed'
-    _VALID_URL = r'https?://www\.n-joy\.de/(?:[^/]+/)(?P<id>\w+)'
+    _VALID_URL = r'https?://www\.n-joy\.de/(?:[^/]+/)+(?P<id>[a-z0-9]+)-(?:player|externalPlayer)\.html'
 
     _TEST = {
         'url': 'http://www.n-joy.de/entertainment/film/portraet374-player_image-832d9b79-fa8a-4026-92e2-e0fd99deb2f9_theme-n-joy.html',

From 376e1ad081abbeb234419c08be121fe81eb9b51d Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 8 Sep 2015 15:58:32 +0100
Subject: [PATCH 2486/6219] [ndr] use utils.qualites

---
 youtube_dl/extractor/ndr.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/ndr.py b/youtube_dl/extractor/ndr.py
index 2a73650d4..87f3edbbe 100644
--- a/youtube_dl/extractor/ndr.py
+++ b/youtube_dl/extractor/ndr.py
@@ -5,10 +5,11 @@ from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
     int_or_none,
+    qualities,
 )
 
 
-preferences = {'xl': 4, 'l': 3, 'm': 2, 's': 1, 'xs': 0,}
+preference = qualities(['xs', 's', 'm','l', 'xl'])
 
 
 class NDRBaseIE(InfoExtractor):
@@ -29,7 +30,7 @@ class NDRBaseIE(InfoExtractor):
                         formats.append({
                             'url': src,
                             'format_id': quality,
-                            'preference': preferences.get(quality),
+                            'preference': preference(quality),
                         })
         elif streamType == 'httpAudio':
             for key, f in playlist.items():
@@ -51,7 +52,7 @@ class NDRBaseIE(InfoExtractor):
         thumbnails = [{
             'id': thumbnail.get('quality'),
             'url': thumbnail.get('src'),
-            'preference': preferences.get(thumbnail.get('quality'))
+            'preference': preference(thumbnail.get('quality'))
         } for thumbnail in config.get('poster').values()]
 
         return {

From 75bb5c7028bd9072bd03371c2d337932ec832642 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 8 Sep 2015 20:12:30 +0100
Subject: [PATCH 2487/6219] [xuite] fix test md5

---
 youtube_dl/extractor/xuite.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/xuite.py b/youtube_dl/extractor/xuite.py
index 5aac8adb3..8bbac54e2 100644
--- a/youtube_dl/extractor/xuite.py
+++ b/youtube_dl/extractor/xuite.py
@@ -19,7 +19,7 @@ class XuiteIE(InfoExtractor):
     _TESTS = [{
         # Audio
         'url': 'http://vlog.xuite.net/play/RGkzc1ZULTM4NjA5MTQuZmx2',
-        'md5': '63a42c705772aa53fd4c1a0027f86adf',
+        'md5': 'e79284c87b371424885448d11f6398c8',
         'info_dict': {
             'id': '3860914',
             'ext': 'mp3',

From 10723362492845d25d4ad996dd209dafbce7eb04 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 8 Sep 2015 20:40:23 +0100
Subject: [PATCH 2488/6219] [washingtonpost] fix test info

---
 youtube_dl/extractor/washingtonpost.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/washingtonpost.py b/youtube_dl/extractor/washingtonpost.py
index 72eb010f8..ec8b99998 100644
--- a/youtube_dl/extractor/washingtonpost.py
+++ b/youtube_dl/extractor/washingtonpost.py
@@ -19,25 +19,25 @@ class WashingtonPostIE(InfoExtractor):
             'title': 'Sinkhole of bureaucracy',
         },
         'playlist': [{
-            'md5': '79132cc09ec5309fa590ae46e4cc31bc',
+            'md5': 'b9be794ceb56c7267d410a13f99d801a',
             'info_dict': {
                 'id': 'fc433c38-b146-11e3-b8b3-44b1d1cd4c1f',
                 'ext': 'mp4',
                 'title': 'Breaking Points: The Paper Mine',
-                'duration': 1287,
+                'duration': 1290,
                 'description': 'Overly complicated paper pushing is nothing new to government bureaucracy. But the way federal retirement applications are filed may be the most outdated. David Fahrenthold explains.',
                 'uploader': 'The Washington Post',
                 'timestamp': 1395527908,
                 'upload_date': '20140322',
             },
         }, {
-            'md5': 'e1d5734c06865cc504ad99dc2de0d443',
+            'md5': '1fff6a689d8770966df78c8cb6c8c17c',
             'info_dict': {
                 'id': '41255e28-b14a-11e3-b8b3-44b1d1cd4c1f',
                 'ext': 'mp4',
                 'title': 'The town bureaucracy sustains',
                 'description': 'Underneath the friendly town of Boyers is a sea of government paperwork. In a disused limestone mine, hundreds of locals now track, file and process retirement applications for the federal government. We set out to find out what it\'s like to do paperwork 230 feet underground.',
-                'duration': 2217,
+                'duration': 2220,
                 'timestamp': 1395528005,
                 'upload_date': '20140322',
                 'uploader': 'The Washington Post',

From 7b4137c351222a94f46f854bf490a299e4124acc Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 9 Sep 2015 10:42:47 +0100
Subject: [PATCH 2489/6219] [fktv] fix info extraction

---
 youtube_dl/extractor/__init__.py |  5 +-
 youtube_dl/extractor/fktv.py     | 78 ++++++++------------------------
 2 files changed, 20 insertions(+), 63 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 57f55b479..f8d4c8462 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -170,10 +170,7 @@ from .firstpost import FirstpostIE
 from .firsttv import FirstTVIE
 from .fivemin import FiveMinIE
 from .fivetv import FiveTVIE
-from .fktv import (
-    FKTVIE,
-    FKTVPosteckeIE,
-)
+from .fktv import FKTVIE
 from .flickr import FlickrIE
 from .folketinget import FolketingetIE
 from .footyroom import FootyRoomIE
diff --git a/youtube_dl/extractor/fktv.py b/youtube_dl/extractor/fktv.py
index 190d9f9ad..c2aa23aa2 100644
--- a/youtube_dl/extractor/fktv.py
+++ b/youtube_dl/extractor/fktv.py
@@ -1,13 +1,11 @@
 from __future__ import unicode_literals
 
 import re
-import random
-import json
 
 from .common import InfoExtractor
 from ..utils import (
-    get_element_by_id,
     clean_html,
+    determine_ext,
 )
 
 
@@ -17,66 +15,28 @@ class FKTVIE(InfoExtractor):
 
     _TEST = {
         'url': 'http://fernsehkritik.tv/folge-1',
+        'md5': '21f0b0c99bce7d5b524eb1b17b1c6d79',
         'info_dict': {
-            'id': '00011',
-            'ext': 'flv',
+            'id': '1',
+            'ext': 'mp4',
             'title': 'Folge 1 vom 10. April 2007',
-            'description': 'md5:fb4818139c7cfe6907d4b83412a6864f',
         },
     }
 
     def _real_extract(self, url):
-        episode = int(self._match_id(url))
+        episode = self._match_id(url)
 
-        video_thumbnail = 'http://fernsehkritik.tv/images/magazin/folge%s.jpg' % episode
-        start_webpage = self._download_webpage('http://fernsehkritik.tv/folge-%s/Start' % episode,
-                                               episode)
-        playlist = self._search_regex(r'playlist = (\[.*?\]);', start_webpage,
-                                      'playlist', flags=re.DOTALL)
-        files = json.loads(re.sub('{[^{}]*?}', '{}', playlist))
-
-        videos = []
-        for i, _ in enumerate(files, 1):
-            video_id = '%04d%d' % (episode, i)
-            video_url = 'http://fernsehkritik.tv/js/directme.php?file=%s%s.flv' % (episode, '' if i == 1 else '-%d' % i)
-            videos.append({
-                'ext': 'flv',
-                'id': video_id,
-                'url': video_url,
-                'title': clean_html(get_element_by_id('eptitle', start_webpage)),
-                'description': clean_html(get_element_by_id('contentlist', start_webpage)),
-                'thumbnail': video_thumbnail
-            })
-        return {
-            '_type': 'multi_video',
-            'entries': videos,
-            'id': 'folge-%s' % episode,
-        }
-
-
-class FKTVPosteckeIE(InfoExtractor):
-    IE_NAME = 'fernsehkritik.tv:postecke'
-    _VALID_URL = r'http://(?:www\.)?fernsehkritik\.tv/inline-video/postecke\.php\?(.*&)?ep=(?P<ep>[0-9]+)(&|$)'
-    _TEST = {
-        'url': 'http://fernsehkritik.tv/inline-video/postecke.php?iframe=true&width=625&height=440&ep=120',
-        'md5': '262f0adbac80317412f7e57b4808e5c4',
-        'info_dict': {
-            'id': '0120',
-            'ext': 'flv',
-            'title': 'Postecke 120',
-        }
-    }
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        episode = int(mobj.group('ep'))
-
-        server = random.randint(2, 4)
-        video_id = '%04d' % episode
-        video_url = 'http://dl%d.fernsehkritik.tv/postecke/postecke%d.flv' % (server, episode)
-        video_title = 'Postecke %d' % episode
-        return {
-            'id': video_id,
-            'url': video_url,
-            'title': video_title,
-        }
+        webpage = self._download_webpage('http://fernsehkritik.tv/folge-%s/play' % episode, episode)
+        title = clean_html(self._html_search_regex('<h3>([^<]+?)</h3>', webpage, 'title'))
+        matchs = re.search(r'(?s)<video[^>]*poster="([^"]+)"[^>]*>(.*?)</video>', webpage)
+        if matchs:
+            poster, sources = matchs.groups()
+            urls = re.findall(r'(?s)<source[^>]*src="([^"]+)"[^>]*>', sources)
+            if sources:
+                formats = [{'url': url, 'format_id': determine_ext(url)} for url in urls]
+                return {
+                    'id': episode,
+                    'title': title,
+                    'formats': formats,
+                    'thumbnail': poster,
+                }

From d9c19db340c568db1afcd6d85ab5b9c4681dd7df Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Lalinsk=C3=BD?= <lukas@oxygene.sk>
Date: Wed, 9 Sep 2015 17:02:28 +0200
Subject: [PATCH 2490/6219] Extract all vid.me formats

---
 youtube_dl/extractor/vidme.py | 82 ++++++++++++++++++++++-------------
 1 file changed, 52 insertions(+), 30 deletions(-)

diff --git a/youtube_dl/extractor/vidme.py b/youtube_dl/extractor/vidme.py
index 157bb74fe..7f44948e9 100644
--- a/youtube_dl/extractor/vidme.py
+++ b/youtube_dl/extractor/vidme.py
@@ -1,10 +1,10 @@
 from __future__ import unicode_literals
 
-from .common import InfoExtractor
+from .common import InfoExtractor, ExtractorError
 from ..utils import (
     int_or_none,
     float_or_none,
-    str_to_int,
+    parse_iso8601,
 )
 
 
@@ -12,7 +12,7 @@ class VidmeIE(InfoExtractor):
     _VALID_URL = r'https?://vid\.me/(?:e/)?(?P<id>[\da-zA-Z]+)'
     _TESTS = [{
         'url': 'https://vid.me/QNB',
-        'md5': 'f42d05e7149aeaec5c037b17e5d3dc82',
+        'md5': 'c62f1156138dc3323902188c5b5a8bd6',
         'info_dict': {
             'id': 'QNB',
             'ext': 'mp4',
@@ -24,6 +24,26 @@ class VidmeIE(InfoExtractor):
             'thumbnail': 're:^https?://.*\.jpg',
             'view_count': int,
             'like_count': int,
+            'comment_count': int,
+        },
+    }, {
+        'url': 'https://vid.me/Gc6M',
+        'md5': 'f42d05e7149aeaec5c037b17e5d3dc82',
+        'info_dict': {
+            'id': 'Gc6M',
+            'ext': 'mp4',
+            'title': 'O Mere Dil ke chain - Arnav and Khushi VM',
+            'duration': 223.72,
+            'timestamp': 1441211642,
+            'upload_date': '20150902',
+            'thumbnail': 're:^https?://.*\.jpg',
+            'view_count': int,
+            'like_count': int,
+            'comment_count': int,
+            'comment_count': int,
+        },
+        'params': {
+            'skip_download': True,
         },
     }, {
         # tests uploader field
@@ -40,6 +60,7 @@ class VidmeIE(InfoExtractor):
             'thumbnail': 're:^https?://.*\.jpg',
             'view_count': int,
             'like_count': int,
+            'comment_count': int,
         },
         'params': {
             'skip_download': True,
@@ -51,45 +72,46 @@ class VidmeIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        url = url.replace('vid.me/e/', 'vid.me/')
         video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
+        api_url = 'https://api.vid.me/videoByUrl/' + video_id
+        data = self._download_json(api_url, video_id)
 
-        video_url = self._html_search_regex(
-            r'<source src="([^"]+)"', webpage, 'video URL')
+        video_data = data.get('video')
+        if video_data is None:
+            raise ExtractorError('Could not extract the vid.me video data')
 
-        title = self._og_search_title(webpage)
-        description = self._og_search_description(webpage, default='')
-        thumbnail = self._og_search_thumbnail(webpage)
-        timestamp = int_or_none(self._og_search_property(
-            'updated_time', webpage, fatal=False))
-        width = int_or_none(self._og_search_property(
-            'video:width', webpage, fatal=False))
-        height = int_or_none(self._og_search_property(
-            'video:height', webpage, fatal=False))
-        duration = float_or_none(self._html_search_regex(
-            r'data-duration="([^"]+)"', webpage, 'duration', fatal=False))
-        view_count = str_to_int(self._html_search_regex(
-            r'<(?:li|span) class="video_views">\s*([\d,\.]+)\s*plays?',
-            webpage, 'view count', fatal=False))
-        like_count = str_to_int(self._html_search_regex(
-            r'class="score js-video-vote-score"[^>]+data-score="([\d,\.\s]+)">',
-            webpage, 'like count', fatal=False))
-        uploader = self._html_search_regex(
-            'class="video_author_username"[^>]*>([^<]+)',
-            webpage, 'uploader', default=None)
+        title = video_data.get('title')
+        description = video_data.get('description')
+        thumbnail = video_data.get('thumbnail_url')
+        timestamp = parse_iso8601(video_data.get('date_created'), ' ')
+        duration = float_or_none(video_data.get('duration'))
+        view_count = int_or_none(video_data.get('view_count'))
+        like_count = int_or_none(video_data.get('likes_count'))
+        comment_count = int_or_none(video_data.get('comment_count'))
+
+        uploader = None
+        user_data = video_data.get('user')
+        if user_data is not None:
+            uploader = user_data.get('username')
+
+        formats = [{
+            'format_id': format['type'],
+            'url': format['uri'],
+            'width': int_or_none(format['width']),
+            'height': int_or_none(format['height']),
+        } for format in video_data.get('formats', [])]
+        self._sort_formats(formats)
 
         return {
             'id': video_id,
-            'url': video_url,
             'title': title,
             'description': description,
             'thumbnail': thumbnail,
             'timestamp': timestamp,
-            'width': width,
-            'height': height,
             'duration': duration,
             'view_count': view_count,
             'like_count': like_count,
+            'comment_count': comment_count,
             'uploader': uploader,
+            'formats': formats,
         }

From 482aa3feccd70e773346bfd4598cba4ea6e80621 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 9 Sep 2015 22:04:05 +0600
Subject: [PATCH 2491/6219] [vidme] Simplify, make more robust, extract more
 metadata and capture errors (Closes #6812)

---
 youtube_dl/extractor/vidme.py | 100 +++++++++++++++++++++++-----------
 1 file changed, 67 insertions(+), 33 deletions(-)

diff --git a/youtube_dl/extractor/vidme.py b/youtube_dl/extractor/vidme.py
index 7f44948e9..9a794e609 100644
--- a/youtube_dl/extractor/vidme.py
+++ b/youtube_dl/extractor/vidme.py
@@ -1,7 +1,9 @@
 from __future__ import unicode_literals
 
-from .common import InfoExtractor, ExtractorError
+from .common import InfoExtractor
+from ..compat import compat_HTTPError
 from ..utils import (
+    ExtractorError,
     int_or_none,
     float_or_none,
     parse_iso8601,
@@ -18,10 +20,11 @@ class VidmeIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Fishing for piranha - the easy way',
             'description': 'source: https://www.facebook.com/photo.php?v=312276045600871',
-            'duration': 119.92,
+            'thumbnail': 're:^https?://.*\.jpg',
             'timestamp': 1406313244,
             'upload_date': '20140725',
-            'thumbnail': 're:^https?://.*\.jpg',
+            'age_limit': 0,
+            'duration': 119.92,
             'view_count': int,
             'like_count': int,
             'comment_count': int,
@@ -33,14 +36,16 @@ class VidmeIE(InfoExtractor):
             'id': 'Gc6M',
             'ext': 'mp4',
             'title': 'O Mere Dil ke chain - Arnav and Khushi VM',
-            'duration': 223.72,
+            'thumbnail': 're:^https?://.*\.jpg',
             'timestamp': 1441211642,
             'upload_date': '20150902',
-            'thumbnail': 're:^https?://.*\.jpg',
+            'uploader': 'SunshineM',
+            'uploader_id': '3552827',
+            'age_limit': 0,
+            'duration': 223.72,
             'view_count': int,
             'like_count': int,
             'comment_count': int,
-            'comment_count': int,
         },
         'params': {
             'skip_download': True,
@@ -53,11 +58,13 @@ class VidmeIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'The Carver',
             'description': 'md5:e9c24870018ae8113be936645b93ba3c',
-            'duration': 97.859999999999999,
+            'thumbnail': 're:^https?://.*\.jpg',
             'timestamp': 1433203629,
             'upload_date': '20150602',
             'uploader': 'Thomas',
-            'thumbnail': 're:^https?://.*\.jpg',
+            'uploader_id': '109747',
+            'age_limit': 0,
+            'duration': 97.859999999999999,
             'view_count': int,
             'like_count': int,
             'comment_count': int,
@@ -66,52 +73,79 @@ class VidmeIE(InfoExtractor):
             'skip_download': True,
         },
     }, {
-        # From http://naked-yogi.tumblr.com/post/118312946248/naked-smoking-stretching
+        # nsfw test from http://naked-yogi.tumblr.com/post/118312946248/naked-smoking-stretching
         'url': 'https://vid.me/e/Wmur',
-        'only_matching': True,
+        'info_dict': {
+            'id': 'Wmur',
+            'ext': 'mp4',
+            'title': 'naked smoking & stretching',
+            'thumbnail': 're:^https?://.*\.jpg',
+            'timestamp': 1430931613,
+            'upload_date': '20150506',
+            'uploader': 'naked-yogi',
+            'uploader_id': '1638622',
+            'age_limit': 18,
+            'duration': 653.26999999999998,
+            'view_count': int,
+            'like_count': int,
+            'comment_count': int,
+        },
+        'params': {
+            'skip_download': True,
+        },
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        api_url = 'https://api.vid.me/videoByUrl/' + video_id
-        data = self._download_json(api_url, video_id)
 
-        video_data = data.get('video')
-        if video_data is None:
-            raise ExtractorError('Could not extract the vid.me video data')
+        try:
+            response = self._download_json(
+                'https://api.vid.me/videoByUrl/%s' % video_id, video_id)
+        except ExtractorError as e:
+            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 400:
+                response = self._parse_json(e.cause.read(), video_id)
+            else:
+                raise
 
-        title = video_data.get('title')
-        description = video_data.get('description')
-        thumbnail = video_data.get('thumbnail_url')
-        timestamp = parse_iso8601(video_data.get('date_created'), ' ')
-        duration = float_or_none(video_data.get('duration'))
-        view_count = int_or_none(video_data.get('view_count'))
-        like_count = int_or_none(video_data.get('likes_count'))
-        comment_count = int_or_none(video_data.get('comment_count'))
+        error = response.get('error')
+        if error:
+            raise ExtractorError(
+                '%s returned error: %s' % (self.IE_NAME, error), expected=True)
 
-        uploader = None
-        user_data = video_data.get('user')
-        if user_data is not None:
-            uploader = user_data.get('username')
+        video = response['video']
 
         formats = [{
-            'format_id': format['type'],
-            'url': format['uri'],
-            'width': int_or_none(format['width']),
-            'height': int_or_none(format['height']),
-        } for format in video_data.get('formats', [])]
+            'format_id': f.get('type'),
+            'url': f['uri'],
+            'width': int_or_none(f.get('width')),
+            'height': int_or_none(f.get('height')),
+        } for f in video.get('formats', []) if f.get('uri')]
         self._sort_formats(formats)
 
+        title = video['title']
+        description = video.get('description')
+        thumbnail = video.get('thumbnail_url')
+        timestamp = parse_iso8601(video.get('date_created'), ' ')
+        uploader = video.get('user', {}).get('username')
+        uploader_id = video.get('user', {}).get('user_id')
+        age_limit = 18 if video.get('nsfw') is True else 0
+        duration = float_or_none(video.get('duration'))
+        view_count = int_or_none(video.get('view_count'))
+        like_count = int_or_none(video.get('likes_count'))
+        comment_count = int_or_none(video.get('comment_count'))
+
         return {
             'id': video_id,
             'title': title,
             'description': description,
             'thumbnail': thumbnail,
+            'uploader': uploader,
+            'uploader_id': uploader_id,
+            'age_limit': age_limit,
             'timestamp': timestamp,
             'duration': duration,
             'view_count': view_count,
             'like_count': like_count,
             'comment_count': comment_count,
-            'uploader': uploader,
             'formats': formats,
         }

From 2f29b758e0a37e084bc4a76bc920919bf9d6b181 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 9 Sep 2015 23:41:44 +0600
Subject: [PATCH 2492/6219] Credit @lalinsky for vidme improvement (#6812)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index d1693224e..8e42e1f5e 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -140,3 +140,4 @@ Behrouz Abbasi
 ngld
 nyuszika7h
 Shaun Walbridge
+Lukáš Lalinský

From b17e7d9a9bac82485a2d1241e1e185593f1ac93a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 9 Sep 2015 23:44:49 +0600
Subject: [PATCH 2493/6219] Credit @anssih for missing metadata extraction on
 youtube (#5993)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 8e42e1f5e..2087e5141 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -140,4 +140,5 @@ Behrouz Abbasi
 ngld
 nyuszika7h
 Shaun Walbridge
+Anssi Hannula
 Lukáš Lalinský

From 73f536439ee70145237c72127262052206097e20 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 9 Sep 2015 23:49:36 +0600
Subject: [PATCH 2494/6219] Credit @reddraggone9 for fixing youtube 2-factor
 authentication (#6553)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 2087e5141..901c1b263 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -140,5 +140,6 @@ Behrouz Abbasi
 ngld
 nyuszika7h
 Shaun Walbridge
+Lee Jenkins
 Anssi Hannula
 Lukáš Lalinský

From cc1ac110173f77a22c9becbee4cb6522087986a1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 9 Sep 2015 23:59:17 +0600
Subject: [PATCH 2495/6219] [vier:videos] Fix extraction with old approach
 (Closes #6806)

---
 youtube_dl/extractor/vier.py | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/vier.py b/youtube_dl/extractor/vier.py
index 15377097e..c76c20614 100644
--- a/youtube_dl/extractor/vier.py
+++ b/youtube_dl/extractor/vier.py
@@ -2,6 +2,7 @@
 from __future__ import unicode_literals
 
 import re
+import itertools
 
 from .common import InfoExtractor
 
@@ -91,31 +92,27 @@ class VierVideosIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         program = mobj.group('program')
 
-        webpage = self._download_webpage(url, program)
-
         page_id = mobj.group('page')
         if page_id:
             page_id = int(page_id)
             start_page = page_id
-            last_page = start_page + 1
             playlist_id = '%s-page%d' % (program, page_id)
         else:
             start_page = 0
-            last_page = int(self._search_regex(
-                r'videos\?page=(\d+)">laatste</a>',
-                webpage, 'last page', default=0)) + 1
             playlist_id = program
 
         entries = []
-        for current_page_id in range(start_page, last_page):
+        for current_page_id in itertools.count(start_page):
             current_page = self._download_webpage(
                 'http://www.vier.be/%s/videos?page=%d' % (program, current_page_id),
                 program,
-                'Downloading page %d' % (current_page_id + 1)) if current_page_id != page_id else webpage
+                'Downloading page %d' % (current_page_id + 1))
             page_entries = [
                 self.url_result('http://www.vier.be' + video_url, 'Vier')
                 for video_url in re.findall(
                     r'<h3><a href="(/[^/]+/videos/[^/]+(?:/\d+)?)">', current_page)]
             entries.extend(page_entries)
+            if page_id or '>Meer<' not in current_page:
+                break
 
         return self.playlist_result(entries, playlist_id)

From d85187eb747b9d32d44ce4f31985432288f4a4f2 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 9 Sep 2015 21:20:01 +0200
Subject: [PATCH 2496/6219] release 2015.09.09

---
 docs/supportedsites.md | 4 ++--
 youtube_dl/version.py  | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 04b9959ac..66091e6be 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -195,7 +195,7 @@
  - **GodTube**
  - **GoldenMoustache**
  - **Golem**
- - **GorillaVid**: GorillaVid.in, daclips.in, movpod.in, fastvideo.in and realvid.net
+ - **GorillaVid**: GorillaVid.in, daclips.in, movpod.in, fastvideo.in, realvid.net and filehoot.com
  - **Goshgay**
  - **Groupon**
  - **Hark**
@@ -309,7 +309,6 @@
  - **mtvservices:embedded**
  - **MuenchenTV**: münchen.tv
  - **MusicPlayOn**
- - **MusicVault**
  - **muzu.tv**
  - **Mwave**
  - **MySpace**
@@ -633,6 +632,7 @@
  - **vine:user**
  - **vk**: VK
  - **vk:uservideos**: VK - User's Videos
+ - **vlive**
  - **Vodlocker**
  - **VoiceRepublic**
  - **Vporn**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 6bc689b75..0cc7411f2 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.09.03'
+__version__ = '2015.09.09'

From 133a2b4ac21e041d7ed66c1d0fa00f7dc5f757af Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 10 Sep 2015 02:41:30 +0600
Subject: [PATCH 2497/6219] [downloader/hls] Properly close stream

---
 youtube_dl/downloader/hls.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index 71aafdc73..b2436e732 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -92,6 +92,7 @@ class NativeHlsFD(FragmentFD):
                 return False
             down, frag_sanitized = sanitize_open(frag_filename, 'rb')
             ctx['dest_stream'].write(down.read())
+            down.close()
             frags_filenames.append(frag_sanitized)
 
         self._finish_frag_download(ctx)

From 364ca0582eb1506af049c0e17595509c6bc0e00c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 10 Sep 2015 02:49:33 +0600
Subject: [PATCH 2498/6219] [yahoo] Process query result embedded in webpage

---
 youtube_dl/extractor/yahoo.py | 63 ++++++++++++++++++++++++++---------
 1 file changed, 48 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/yahoo.py b/youtube_dl/extractor/yahoo.py
index b8579b573..1d9b98750 100644
--- a/youtube_dl/extractor/yahoo.py
+++ b/youtube_dl/extractor/yahoo.py
@@ -144,6 +144,20 @@ class YahooIE(InfoExtractor):
         }, {
             'url': 'https://tw.news.yahoo.com/-100120367.html',
             'only_matching': True,
+        }, {
+            # Query result is embedded in webpage, but explicit request to video API fails with geo restriction
+            'url': 'https://screen.yahoo.com/community/communitary-community-episode-1-ladders-154501237.html',
+            'info_dict': {
+                'id': '1f32853c-a271-3eef-8cb6-f6d6872cb504',
+                'ext': 'mp4',
+                'title': 'Communitary - Community Episode 1: Ladders',
+                'description': 'md5:8fc39608213295748e1e289807838c97',
+                'duration': 1646,
+            },
+            'params': {
+                # m3u8 download
+                'skip_download': True,
+            },
         }
     ]
 
@@ -171,6 +185,19 @@ class YahooIE(InfoExtractor):
         if nbc_sports_url:
             return self.url_result(nbc_sports_url, 'NBCSportsVPlayer')
 
+        # Query result is often embedded in webpage as JSON. Sometimes explicit requests
+        # to video API results in a failure with geo restriction reason therefore using
+        # embedded query result when present sounds reasonable.
+        config_json = self._search_regex(
+            r'window\.Af\.bootstrap\[[^\]]+\]\s*=\s*({.*?"applet_type"\s*:\s*"td-applet-videoplayer".*?});(?:</script>|$)',
+            webpage, 'videoplayer applet', default=None)
+        if config_json:
+            config = self._parse_json(config_json, display_id, fatal=False)
+            if config:
+                sapi = config.get('models', {}).get('applet_model', {}).get('data', {}).get('sapi')
+                if sapi:
+                    return self._extract_info(display_id, sapi, webpage)
+
         items_json = self._search_regex(
             r'mediaItems: ({.*?})$', webpage, 'items', flags=re.MULTILINE,
             default=None)
@@ -190,22 +217,10 @@ class YahooIE(InfoExtractor):
             video_id = info['id']
         return self._get_info(video_id, display_id, webpage)
 
-    def _get_info(self, video_id, display_id, webpage):
-        region = self._search_regex(
-            r'\\?"region\\?"\s*:\s*\\?"([^"]+?)\\?"',
-            webpage, 'region', fatal=False, default='US')
-        data = compat_urllib_parse.urlencode({
-            'protocol': 'http',
-            'region': region,
-        })
-        query_url = (
-            'https://video.media.yql.yahoo.com/v1/video/sapi/streams/'
-            '{id}?{data}'.format(id=video_id, data=data))
-        query_result = self._download_json(
-            query_url, display_id, 'Downloading video info')
-
-        info = query_result['query']['results']['mediaObj'][0]
+    def _extract_info(self, display_id, query, webpage):
+        info = query['query']['results']['mediaObj'][0]
         meta = info.get('meta')
+        video_id = info.get('id')
 
         if not meta:
             msg = info['status'].get('msg')
@@ -231,6 +246,9 @@ class YahooIE(InfoExtractor):
                     'ext': 'flv',
                 })
             else:
+                if s.get('format') == 'm3u8_playlist':
+                    format_info['protocol'] = 'm3u8_native'
+                    format_info['ext'] = 'mp4'
                 format_url = compat_urlparse.urljoin(host, path)
                 format_info['url'] = format_url
             formats.append(format_info)
@@ -264,6 +282,21 @@ class YahooIE(InfoExtractor):
             'subtitles': subtitles,
         }
 
+    def _get_info(self, video_id, display_id, webpage):
+        region = self._search_regex(
+            r'\\?"region\\?"\s*:\s*\\?"([^"]+?)\\?"',
+            webpage, 'region', fatal=False, default='US')
+        data = compat_urllib_parse.urlencode({
+            'protocol': 'http',
+            'region': region,
+        })
+        query_url = (
+            'https://video.media.yql.yahoo.com/v1/video/sapi/streams/'
+            '{id}?{data}'.format(id=video_id, data=data))
+        query_result = self._download_json(
+            query_url, display_id, 'Downloading video info')
+        return self._extract_info(display_id, query_result, webpage)
+
 
 class YahooSearchIE(SearchInfoExtractor):
     IE_DESC = 'Yahoo screen search'

From 1721fef28b89ac4264db978ab7fee3b4dd154056 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 10 Sep 2015 02:58:40 +0600
Subject: [PATCH 2499/6219] [yahoo] Fix test

---
 youtube_dl/extractor/yahoo.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/yahoo.py b/youtube_dl/extractor/yahoo.py
index 1d9b98750..fca5ddc69 100644
--- a/youtube_dl/extractor/yahoo.py
+++ b/youtube_dl/extractor/yahoo.py
@@ -147,6 +147,7 @@ class YahooIE(InfoExtractor):
         }, {
             # Query result is embedded in webpage, but explicit request to video API fails with geo restriction
             'url': 'https://screen.yahoo.com/community/communitary-community-episode-1-ladders-154501237.html',
+            'md5': '4fbafb9c9b6f07aa8f870629f6671b35',
             'info_dict': {
                 'id': '1f32853c-a271-3eef-8cb6-f6d6872cb504',
                 'ext': 'mp4',
@@ -154,10 +155,6 @@ class YahooIE(InfoExtractor):
                 'description': 'md5:8fc39608213295748e1e289807838c97',
                 'duration': 1646,
             },
-            'params': {
-                # m3u8 download
-                'skip_download': True,
-            },
         }
     ]
 

From 689fb748ee1ba8e61f99d21a3bcb1bc83b708649 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 11 Sep 2015 04:44:17 +0100
Subject: [PATCH 2500/6219] [utlis] add extract_attributes for extracting html
 tags attributes

---
 youtube_dl/utils.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 206dd56bc..bcebf9cc5 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -248,6 +248,14 @@ def get_element_by_attribute(attribute, value, html):
     return unescapeHTML(res)
 
 
+def extract_attributes(attributes_str, attributes_regex=r'(?s)\s*([^\s=]+)\s*=\s*["\']([^"\']+)["\']'):
+    attributes = re.findall(attributes_regex, attributes_str)
+    attributes_dict = {}
+    if attributes:
+        attributes_dict = {attribute_name: attribute_value for (attribute_name, attribute_value) in attributes}
+    return attributes_dict
+
+
 def clean_html(html):
     """Clean an HTML snippet into a readable string"""
 

From ed1269000f24a6ddc683a295ff402ef3ded5c4fb Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 11 Sep 2015 04:46:21 +0100
Subject: [PATCH 2501/6219] [brightcove] add support for brightcove in page
 embed(fixes #6824)

---
 youtube_dl/extractor/__init__.py   |  5 +-
 youtube_dl/extractor/brightcove.py | 92 ++++++++++++++++++++++++++++++
 youtube_dl/extractor/generic.py    | 21 ++++++-
 3 files changed, 116 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 57f55b479..fcd9edec3 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -59,7 +59,10 @@ from .bloomberg import BloombergIE
 from .bpb import BpbIE
 from .br import BRIE
 from .breakcom import BreakIE
-from .brightcove import BrightcoveIE
+from .brightcove import (
+    BrightcoveIE,
+    BrightcoveInPageEmbedIE,
+)
 from .buzzfeed import BuzzFeedIE
 from .byutv import BYUtvIE
 from .c56 import C56IE
diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 4721c2293..a07c0888f 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -22,6 +22,10 @@ from ..utils import (
     fix_xml_ampersands,
     unescapeHTML,
     unsmuggle_url,
+    js_to_json,
+    int_or_none,
+    parse_iso8601,
+    extract_attributes,
 )
 
 
@@ -346,3 +350,91 @@ class BrightcoveIE(InfoExtractor):
         if 'url' not in info and not info.get('formats'):
             raise ExtractorError('Unable to extract video url for %s' % info['id'])
         return info
+
+
+class BrightcoveInPageEmbedIE(InfoExtractor):
+    _VALID_URL = r'https?://players\.brightcove\.net/(?P<account_id>\d+)/([a-z0-9-]+)_([a-z]+)/index.html?.*videoId=(?P<video_id>\d+)'
+    TEST = {
+        'url': 'http://players.brightcove.net/929656772001/e41d32dc-ec74-459e-a845-6c69f7b724ea_default/index.html?videoId=4463358922001',
+        'info_dict': {
+            'id': '4463358922001',
+            'ext': 'flv',
+            'title': 'Meet the man behind Popcorn Time',
+            'description': 'md5:a950cc4285c43e44d763d036710cd9cd',
+            'duration': 165768,
+        }
+    }
+
+    @staticmethod
+    def _extract_url(webpage):
+        video_attributes = re.search(r'(?s)<video([^>]*)>.*?</(?:video|audio)>', webpage)
+        if video_attributes:
+            video_attributes = extract_attributes(video_attributes.group(), r'(?s)\s*data-(account|video-id|playlist-id|policy-key|player|embed)\s*=\s*["\']([^"\']+)["\']')
+            account_id = video_attributes.get('account')
+            player_id = video_attributes.get('player')
+            embed = video_attributes.get('embed')
+            video_id = video_attributes.get('video-id')
+            if account_id and player_id and embed and video_id:
+                return 'http://players.brightcove.net/%s/%s_%s/index.html?videoId=%s' % (account_id, player_id, embed, video_id)
+        return None
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        account_id, player_id, embed, video_id = mobj.groups()
+
+        webpage = self._download_webpage('http://players.brightcove.net/%s/%s_%s/index.min.js' % (account_id, player_id, embed), video_id)
+
+        catalog = self._parse_json(
+            js_to_json(
+                self._search_regex(
+                    r'catalog\(({[^}]+})\);',
+                    webpage,
+                    'catalog'
+                )
+            ),
+            video_id
+        )
+        policy_key = catalog['policyKey']
+
+        req = compat_urllib_request.Request(
+            'https://edge.api.brightcove.com/playback/v1/accounts/%s/videos/%s' % (account_id, video_id),
+            headers={'Accept': 'application/json;pk=%s' % policy_key})
+        json_data = self._download_json(req, video_id)
+
+        title = json_data['name']
+        description = json_data.get('description')
+        thumbnail = json_data.get('name')
+        timestamp = parse_iso8601(json_data.get('published_at'))
+        duration = int_or_none(json_data.get('duration'))
+
+        formats = []
+        for source in json_data.get('sources'):
+            source_type = source.get('type')
+            if source_type == 'application/x-mpegURL':
+                formats.extend(self._extract_m3u8_formats(source.get('src'), video_id))
+            else:
+                src = source.get('src')
+                if src:
+                    formats.append({
+                        'url': src,
+                        'abr': source.get('avg_bitrate'),
+                        'width': int_or_none(source.get('width')),
+                        'height': int_or_none(source.get('height')),
+                        'filesize': source.get('size'),
+                        'container': source.get('container'),
+                        'vcodec': source.get('container'),
+                    })
+                else:
+                    formats.extend(self._extract_f4m_formats(source.get('streaming_src'), video_id))
+
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'timestamp': timestamp,
+            'duration': duration,
+            'formats': formats,
+        }
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index ec748ed9f..7a3a7f66b 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -29,7 +29,10 @@ from ..utils import (
     url_basename,
     xpath_text,
 )
-from .brightcove import BrightcoveIE
+from .brightcove import (
+    BrightcoveIE,
+    BrightcoveInPageEmbedIE,
+)
 from .nbc import NBCSportsVPlayerIE
 from .ooyala import OoyalaIE
 from .rutv import RUTVIE
@@ -1012,6 +1015,17 @@ class GenericIE(InfoExtractor):
                 'ext': 'mp4',
                 'title': 'cinemasnob',
             },
+        },
+        # BrightcoveInPageEmbed embed
+        {
+            'url': 'http://www.geekandsundry.com/tabletop-bonus-wils-final-thoughts-on-dread/',
+            'info_dict': {
+                'id': '4238694884001',
+                'ext': 'flv',
+                'title': 'Tabletop: Dread, Last Thoughts',
+                'description': 'Tabletop: Dread, Last Thoughts',
+                'duration': 51690,
+            },
         }
     ]
 
@@ -1288,6 +1302,11 @@ class GenericIE(InfoExtractor):
                 'entries': entries,
             }
 
+        # Look for Brightcove In Page Embed:
+        brightcove_in_page_embed_url = BrightcoveInPageEmbedIE._extract_url(webpage)
+        if brightcove_in_page_embed_url:
+            return self.url_result(brightcove_in_page_embed_url, 'BrightcoveInPageEmbed')
+
         # Look for embedded rtl.nl player
         matches = re.findall(
             r'<iframe[^>]+?src="((?:https?:)?//(?:www\.)?rtl\.nl/system/videoplayer/[^"]+(?:video_)?embed[^"]+)"',

From 73eb13dfc74132b8f0e5c1ac1ea75f66e0aca6bd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 11 Sep 2015 20:43:05 +0600
Subject: [PATCH 2502/6219] [extractor/common] Case insensitive inputs
 extraction

---
 youtube_dl/extractor/common.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 5eeeda08d..835f6f368 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -732,7 +732,7 @@ class InfoExtractor(object):
     @staticmethod
     def _hidden_inputs(html):
         hidden_inputs = {}
-        for input in re.findall(r'<input([^>]+)>', html):
+        for input in re.findall(r'(?i)<input([^>]+)>', html):
             if not re.search(r'type=(["\'])(?:hidden|submit)\1', input):
                 continue
             name = re.search(r'name=(["\'])(?P<value>.+?)\1', input)
@@ -746,7 +746,7 @@ class InfoExtractor(object):
 
     def _form_hidden_inputs(self, form_id, html):
         form = self._search_regex(
-            r'(?s)<form[^>]+?id=(["\'])%s\1[^>]*>(?P<form>.+?)</form>' % form_id,
+            r'(?is)<form[^>]+?id=(["\'])%s\1[^>]*>(?P<form>.+?)</form>' % form_id,
             html, '%s form' % form_id, group='form')
         return self._hidden_inputs(form)
 

From 586f1cc532d167c28e733779cbf132b94d8f76e6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 11 Sep 2015 21:07:32 +0600
Subject: [PATCH 2503/6219] [extractor/common] Skip html comment tags (Closes
 #6822)

---
 youtube_dl/extractor/common.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 835f6f368..d694e818e 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -731,6 +731,7 @@ class InfoExtractor(object):
 
     @staticmethod
     def _hidden_inputs(html):
+        html = re.sub(r'<!--(?:(?!<!--).)*-->', '', html)
         hidden_inputs = {}
         for input in re.findall(r'(?i)<input([^>]+)>', html):
             if not re.search(r'type=(["\'])(?:hidden|submit)\1', input):

From 60ed60353b9ca57e8181f0b14d525ce487e673ed Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 12 Sep 2015 20:34:48 +0600
Subject: [PATCH 2504/6219] [openfilm] Remove extractor

OpenFilm has been shut down
---
 youtube_dl/extractor/__init__.py |  1 -
 youtube_dl/extractor/openfilm.py | 70 --------------------------------
 2 files changed, 71 deletions(-)
 delete mode 100644 youtube_dl/extractor/openfilm.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 57f55b479..2e7272931 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -432,7 +432,6 @@ from .ooyala import (
     OoyalaIE,
     OoyalaExternalIE,
 )
-from .openfilm import OpenFilmIE
 from .orf import (
     ORFTVthekIE,
     ORFOE1IE,
diff --git a/youtube_dl/extractor/openfilm.py b/youtube_dl/extractor/openfilm.py
deleted file mode 100644
index d2ceedd01..000000000
--- a/youtube_dl/extractor/openfilm.py
+++ /dev/null
@@ -1,70 +0,0 @@
-from __future__ import unicode_literals
-
-import json
-
-from .common import InfoExtractor
-from ..compat import compat_urllib_parse_unquote_plus
-from ..utils import (
-    parse_iso8601,
-    parse_age_limit,
-    int_or_none,
-)
-
-
-class OpenFilmIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)openfilm\.com/videos/(?P<id>.+)'
-    _TEST = {
-        'url': 'http://www.openfilm.com/videos/human-resources-remastered',
-        'md5': '42bcd88c2f3ec13b65edf0f8ad1cac37',
-        'info_dict': {
-            'id': '32736',
-            'display_id': 'human-resources-remastered',
-            'ext': 'mp4',
-            'title': 'Human Resources (Remastered)',
-            'description': 'Social Engineering in the 20th Century.',
-            'thumbnail': 're:^https?://.*\.jpg$',
-            'duration': 7164,
-            'timestamp': 1334756988,
-            'upload_date': '20120418',
-            'uploader_id': '41117',
-            'view_count': int,
-            'age_limit': 0,
-        },
-    }
-
-    def _real_extract(self, url):
-        display_id = self._match_id(url)
-
-        webpage = self._download_webpage(url, display_id)
-
-        player = compat_urllib_parse_unquote_plus(
-            self._og_search_video_url(webpage))
-
-        video = json.loads(self._search_regex(
-            r'\bp=({.+?})(?:&|$)', player, 'video JSON'))
-
-        video_url = '%s1.mp4' % video['location']
-        video_id = video.get('video_id')
-        display_id = video.get('alias') or display_id
-        title = video.get('title')
-        description = video.get('description')
-        thumbnail = video.get('main_thumb')
-        duration = int_or_none(video.get('duration'))
-        timestamp = parse_iso8601(video.get('dt_published'), ' ')
-        uploader_id = video.get('user_id')
-        view_count = int_or_none(video.get('views_count'))
-        age_limit = parse_age_limit(video.get('age_limit'))
-
-        return {
-            'id': video_id,
-            'display_id': display_id,
-            'url': video_url,
-            'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
-            'duration': duration,
-            'timestamp': timestamp,
-            'uploader_id': uploader_id,
-            'view_count': view_count,
-            'age_limit': age_limit,
-        }

From 41ebd6530b124b9265a3df9d7d09aef02041b088 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 12 Sep 2015 22:42:57 +0800
Subject: [PATCH 2505/6219] [tudou] Add the test case (#6273)

---
 youtube_dl/extractor/tudou.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/tudou.py b/youtube_dl/extractor/tudou.py
index e800477e2..950c42afb 100644
--- a/youtube_dl/extractor/tudou.py
+++ b/youtube_dl/extractor/tudou.py
@@ -27,6 +27,9 @@ class TudouIE(InfoExtractor):
             'title': 'La Sylphide-Bolshoi-Ekaterina Krysanova & Vyacheslav Lopatin 2012',
             'thumbnail': 're:^https?://.*\.jpg$',
         }
+    }, {
+        'url': 'http://www.tudou.com/albumplay/cJAHGih4yYg.html',
+        'only_matching': True,
     }]
 
     _PLAYER_URL = 'http://js.tudouui.com/bin/lingtong/PortalPlayer_177.swf'

From 94e507aea798dac6974237cc44257dda45d5fa5a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 12 Sep 2015 22:45:09 +0800
Subject: [PATCH 2506/6219] [tudou] A more comprehensive _VALID_URL

---
 youtube_dl/extractor/tudou.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tudou.py b/youtube_dl/extractor/tudou.py
index 950c42afb..68712cb4a 100644
--- a/youtube_dl/extractor/tudou.py
+++ b/youtube_dl/extractor/tudou.py
@@ -9,7 +9,7 @@ from .common import InfoExtractor
 
 
 class TudouIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?tudou\.com/(?:listplay|programs(?:/view)?|albumplay)/?.*/(?P<id>[^/?#]+?)(?:\.html)?/?(?:$|[?#])'
+    _VALID_URL = r'https?://(?:www\.)?tudou\.com/(?:listplay|programs(?:/view)?|albumplay)/([^/]+/)*(?P<id>[^/?#]+?)(?:\.html)?/?(?:$|[?#])'
     _TESTS = [{
         'url': 'http://www.tudou.com/listplay/zzdE77v6Mmo/2xN2duXMxmw.html',
         'md5': '140a49ed444bd22f93330985d8475fcb',

From 141ba36996f77a420df69903a59792f6f93ae314 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 12 Sep 2015 22:51:49 +0800
Subject: [PATCH 2507/6219] [tudou] Modernize

---
 youtube_dl/extractor/tudou.py | 18 ++++++------------
 1 file changed, 6 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/tudou.py b/youtube_dl/extractor/tudou.py
index 68712cb4a..c9d80a7ef 100644
--- a/youtube_dl/extractor/tudou.py
+++ b/youtube_dl/extractor/tudou.py
@@ -2,9 +2,6 @@
 
 from __future__ import unicode_literals
 
-import re
-import json
-
 from .common import InfoExtractor
 
 
@@ -46,13 +43,10 @@ class TudouIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        m = re.search(r'vcode:\s*[\'"](.+?)[\'"]', webpage)
-        if m and m.group(1):
-            return {
-                '_type': 'url',
-                'url': 'youku:' + m.group(1),
-                'ie_key': 'Youku'
-            }
+        youku_vcode = self._search_regex(
+            r'vcode:\s*[\'"](.+?)[\'"]', webpage, 'youku vcode', default=None)
+        if youku_vcode:
+            return self.url_result('youku:' + youku_vcode, ie='Youku')
 
         title = self._search_regex(
             r",kw:\s*['\"](.+?)[\"']", webpage, 'title')
@@ -63,8 +57,8 @@ class TudouIE(InfoExtractor):
             r"playerUrl\s*:\s*['\"](.+?\.swf)[\"']",
             webpage, 'player URL', default=self._PLAYER_URL)
 
-        segs_json = self._search_regex(r'segs: \'(.*)\'', webpage, 'segments')
-        segments = json.loads(segs_json)
+        segments = self._parse_json(self._search_regex(
+            r'segs: \'(.*)\'', webpage, 'segments'), video_id)
         # It looks like the keys are the arguments that have to be passed as
         # the hd field in the request url, we pick the higher
         # Also, filter non-number qualities (see issue #3643).

From aab135516b288f24c55178b024976fd3e130c7b8 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 12 Sep 2015 22:52:51 +0800
Subject: [PATCH 2508/6219] [tudou] Avoid shadowing builtin names

---
 youtube_dl/extractor/tudou.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/tudou.py b/youtube_dl/extractor/tudou.py
index c9d80a7ef..6116b209d 100644
--- a/youtube_dl/extractor/tudou.py
+++ b/youtube_dl/extractor/tudou.py
@@ -31,11 +31,11 @@ class TudouIE(InfoExtractor):
 
     _PLAYER_URL = 'http://js.tudouui.com/bin/lingtong/PortalPlayer_177.swf'
 
-    def _url_for_id(self, id, quality=None):
-        info_url = "http://v2.tudou.com/f?id=" + str(id)
+    def _url_for_id(self, video_id, quality=None):
+        info_url = "http://v2.tudou.com/f?id=" + str(video_id)
         if quality:
             info_url += '&hd' + quality
-        webpage = self._download_webpage(info_url, id, "Opening the info webpage")
+        webpage = self._download_webpage(info_url, video_id, "Opening the info webpage")
         final_url = self._html_search_regex('>(.+?)</f>', webpage, 'video url')
         return final_url
 

From 87813a857009dc3c3dfcc421679e5e806d363863 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 13 Sep 2015 02:36:51 +0800
Subject: [PATCH 2509/6219] [tudou] Use _download_xml

---
 youtube_dl/extractor/tudou.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/tudou.py b/youtube_dl/extractor/tudou.py
index 6116b209d..3b993192c 100644
--- a/youtube_dl/extractor/tudou.py
+++ b/youtube_dl/extractor/tudou.py
@@ -35,8 +35,8 @@ class TudouIE(InfoExtractor):
         info_url = "http://v2.tudou.com/f?id=" + str(video_id)
         if quality:
             info_url += '&hd' + quality
-        webpage = self._download_webpage(info_url, video_id, "Opening the info webpage")
-        final_url = self._html_search_regex('>(.+?)</f>', webpage, 'video url')
+        xml_data = self._download_xml(info_url, video_id, "Opening the info XML page")
+        final_url = xml_data.text
         return final_url
 
     def _real_extract(self, url):

From 349b3a2ea0d6c264facacd92508516e8530108b2 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 13 Sep 2015 02:51:20 +0800
Subject: [PATCH 2510/6219] [tudou] Improve regexs

---
 youtube_dl/extractor/tudou.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/tudou.py b/youtube_dl/extractor/tudou.py
index 3b993192c..53ba8511f 100644
--- a/youtube_dl/extractor/tudou.py
+++ b/youtube_dl/extractor/tudou.py
@@ -44,21 +44,21 @@ class TudouIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         youku_vcode = self._search_regex(
-            r'vcode:\s*[\'"](.+?)[\'"]', webpage, 'youku vcode', default=None)
+            r'vcode\s*:\s*[\'"]([^\'"]*)[\'"]', webpage, 'youku vcode', default=None)
         if youku_vcode:
             return self.url_result('youku:' + youku_vcode, ie='Youku')
 
         title = self._search_regex(
-            r",kw:\s*['\"](.+?)[\"']", webpage, 'title')
+            r',kw\s*:\s*[\'"]([^\'"]+)[\'"]', webpage, 'title')
         thumbnail_url = self._search_regex(
-            r",pic:\s*[\"'](.+?)[\"']", webpage, 'thumbnail URL', fatal=False)
+            r',pic\s*:\s*[\'"]([^\'"]+)[\'"]', webpage, 'thumbnail URL', fatal=False)
 
         player_url = self._search_regex(
-            r"playerUrl\s*:\s*['\"](.+?\.swf)[\"']",
+            r'playerUrl\s*:\s*[\'"]([^\'"]+\.swf)[\'"]',
             webpage, 'player URL', default=self._PLAYER_URL)
 
         segments = self._parse_json(self._search_regex(
-            r'segs: \'(.*)\'', webpage, 'segments'), video_id)
+            r'segs: \'([^\']+)\'', webpage, 'segments'), video_id)
         # It looks like the keys are the arguments that have to be passed as
         # the hd field in the request url, we pick the higher
         # Also, filter non-number qualities (see issue #3643).

From b264c2130221912adfc7cc35d73c2a88d79eafeb Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 13 Sep 2015 02:57:14 +0800
Subject: [PATCH 2511/6219] [tudou] Use single quotes and compat_str

---
 youtube_dl/extractor/tudou.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tudou.py b/youtube_dl/extractor/tudou.py
index 53ba8511f..5f7ac4b35 100644
--- a/youtube_dl/extractor/tudou.py
+++ b/youtube_dl/extractor/tudou.py
@@ -3,6 +3,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..compat import compat_str
 
 
 class TudouIE(InfoExtractor):
@@ -32,7 +33,7 @@ class TudouIE(InfoExtractor):
     _PLAYER_URL = 'http://js.tudouui.com/bin/lingtong/PortalPlayer_177.swf'
 
     def _url_for_id(self, video_id, quality=None):
-        info_url = "http://v2.tudou.com/f?id=" + str(video_id)
+        info_url = 'http://v2.tudou.com/f?id=' + compat_str(video_id)
         if quality:
             info_url += '&hd' + quality
         xml_data = self._download_xml(info_url, video_id, "Opening the info XML page")

From 2ffe3bc14b5e65c902fe5ddd610143c791edaa52 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 13 Sep 2015 04:15:49 +0600
Subject: [PATCH 2512/6219] [ndr] Rework and cover with tests

---
 youtube_dl/extractor/__init__.py |   1 +
 youtube_dl/extractor/ndr.py      | 469 ++++++++++++++++++++++---------
 2 files changed, 334 insertions(+), 136 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 44ab7ce3c..fadba905d 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -367,6 +367,7 @@ from .nbc import (
 from .ndr import (
     NDRIE,
     NJoyIE,
+    NDREmbedBaseIE,
     NDREmbedIE,
     NJoyEmbedIE,
 )
diff --git a/youtube_dl/extractor/ndr.py b/youtube_dl/extractor/ndr.py
index 87f3edbbe..e3cc6fde8 100644
--- a/youtube_dl/extractor/ndr.py
+++ b/youtube_dl/extractor/ndr.py
@@ -1,183 +1,380 @@
-# encoding: utf-8
+# coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..utils import (
-    ExtractorError,
+    determine_ext,
     int_or_none,
+    parse_iso8601,
     qualities,
 )
 
 
-preference = qualities(['xs', 's', 'm','l', 'xl'])
-
-
 class NDRBaseIE(InfoExtractor):
-
-    def extract_video_info(self, playlist, video_id):
-        formats = []
-        streamType = playlist.get('config').get('streamType')
-        if streamType == 'httpVideo':
-            for key, f in playlist.items():
-                if key != 'config':
-                    src = f['src']
-                    if '.f4m' in src:
-                        formats.extend(self._extract_f4m_formats(src, video_id))
-                    elif '.m3u8' in src:
-                        formats.extend(self._extract_m3u8_formats(src, video_id, fatal=False))
-                    else:
-                        quality = f.get('quality')
-                        formats.append({
-                            'url': src,
-                            'format_id': quality,
-                            'preference': preference(quality),
-                        })
-        elif streamType == 'httpAudio':
-            for key, f in playlist.items():
-                if key != 'config':
-                    formats.append({
-                        'url': f['src'],
-                        'format_id': 'mp3',
-                        'vcodec': 'none',
-                    })
-        else:
-            raise ExtractorError('No media links available for %s' % video_id)
-
-        self._sort_formats(formats)
-
-        config = playlist.get('config')
-
-        title = config['title']
-        duration = int_or_none(config.get('duration'))
-        thumbnails = [{
-            'id': thumbnail.get('quality'),
-            'url': thumbnail.get('src'),
-            'preference': preference(thumbnail.get('quality'))
-        } for thumbnail in config.get('poster').values()]
-
-        return {
-            'id': video_id,
-            'title': title,
-            'thumbnails': thumbnails,
-            'duration': duration,
-            'formats': formats,
-        }
-
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        json_data = self._download_json('http://www.ndr.de/%s-ppjson.json' % video_id, video_id, fatal=False)
-
-        if not json_data:
-            webpage = self._download_webpage(url, video_id)
-            embed_url = self._html_search_regex(r'<iframe[^>]+id="pp_\w+"[^>]+src="(/.*)"', webpage, 'embed url', None, False)
-            if not embed_url:
-                embed_url = self._html_search_meta('embedURL', webpage, fatal=False)
-            if embed_url:
-                if embed_url.startswith('/'):
-                    return self.url_result('http://www.ndr.de%s' % embed_url, 'NDREmbed')
-                else:
-                    return self.url_result(embed_url, 'NDREmbed')
-            raise ExtractorError('No media links available for %s' % video_id)
-
-        return self.extract_video_info(json_data['playlist'], video_id)
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+        return self._extract_embed(webpage, display_id)
 
 
 class NDRIE(NDRBaseIE):
     IE_NAME = 'ndr'
-    IE_DESC = 'NDR.de - Mediathek'
-    _VALID_URL = r'https?://www\.ndr\.de/.+?,(?P<id>\w+)\.html'
+    IE_DESC = 'NDR.de - Norddeutscher Rundfunk'
+    _VALID_URL = r'https?://www\.ndr\.de/(?:[^/]+/)+(?P<id>[^/?#]+),[\da-z]+\.html'
+    _TESTS = [{
+        # httpVideo, same content id
+        'url': 'http://www.ndr.de/fernsehen/Party-Poette-und-Parade,hafengeburtstag988.html',
+        'md5': '6515bc255dc5c5f8c85bbc38e035a659',
+        'info_dict': {
+            'id': 'hafengeburtstag988',
+            'display_id': 'Party-Poette-und-Parade',
+            'ext': 'mp4',
+            'title': 'Party, Pötte und Parade',
+            'description': 'md5:ad14f9d2f91d3040b6930c697e5f6b4c',
+            'uploader': 'ndrtv',
+            'timestamp': 1431108900,
+            'upload_date': '20150510',
+            'duration': 3498,
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }, {
+        # httpVideo, different content id
+        'url': 'http://www.ndr.de/sport/fussball/40-Osnabrueck-spielt-sich-in-einen-Rausch,osna270.html',
+        'md5': '1043ff203eab307f0c51702ec49e9a71',
+        'info_dict': {
+            'id': 'osna272',
+            'display_id': '40-Osnabrueck-spielt-sich-in-einen-Rausch',
+            'ext': 'mp4',
+            'title': 'Osnabrück - Wehen Wiesbaden: Die Highlights',
+            'description': 'md5:32e9b800b3d2d4008103752682d5dc01',
+            'uploader': 'ndrtv',
+            'timestamp': 1442059200,
+            'upload_date': '20150912',
+            'duration': 510,
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }, {
+        # httpAudio, same content id
+        'url': 'http://www.ndr.de/info/La-Valette-entgeht-der-Hinrichtung,audio51535.html',
+        'md5': 'bb3cd38e24fbcc866d13b50ca59307b8',
+        'info_dict': {
+            'id': 'audio51535',
+            'display_id': 'La-Valette-entgeht-der-Hinrichtung',
+            'ext': 'mp3',
+            'title': 'La Valette entgeht der Hinrichtung',
+            'description': 'md5:22f9541913a40fe50091d5cdd7c9f536',
+            'uploader': 'ndrinfo',
+            'timestamp': 1290626100,
+            'upload_date': '20140729',
+            'duration': 884,
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }]
 
-    _TESTS = [
-        {
-            'url': 'http://www.ndr.de/fernsehen/sendungen/nordmagazin/Kartoffeltage-in-der-Lewitz,nordmagazin25866.html',
-            'md5': '5bc5f5b92c82c0f8b26cddca34f8bb2c',
-            'note': 'Video file',
-            'info_dict': {
-                'id': 'nordmagazin25866',
-                'ext': 'mp4',
-                'title': 'Kartoffeltage in der Lewitz',
-                'duration': 166,
-            },
-            'skip': '404 Not found',
-        },
-        {
-            'url': 'http://www.ndr.de/fernsehen/Party-Poette-und-Parade,hafengeburtstag988.html',
-            'md5': 'dadc003c55ae12a5d2f6bd436cd73f59',
-            'info_dict': {
-                'id': 'hafengeburtstag988',
-                'ext': 'mp4',
-                'title': 'Party, Pötte und Parade',
-                'duration': 3498,
-            },
-        },
-        {
-            'url': 'http://www.ndr.de/info/La-Valette-entgeht-der-Hinrichtung,audio51535.html',
-            'md5': 'bb3cd38e24fbcc866d13b50ca59307b8',
-            'note': 'Audio file',
-            'info_dict': {
-                'id': 'audio51535',
-                'ext': 'mp3',
-                'title': 'La Valette entgeht der Hinrichtung',
-                'duration': 884,
-            }
+    def _extract_embed(self, webpage, display_id):
+        embed_url = self._html_search_meta(
+            'embedURL', webpage, 'embed URL', fatal=True)
+        description = self._search_regex(
+            r'<p[^>]+itemprop="description">([^<]+)</p>',
+            webpage, 'description', fatal=False)
+        timestamp = parse_iso8601(
+            self._search_regex(
+                r'<span itemprop="datePublished" content="([^"]+)">',
+                webpage, 'upload date', fatal=False))
+        return {
+            '_type': 'url_transparent',
+            'url': embed_url,
+            'display_id': display_id,
+            'description': description,
+            'timestamp': timestamp,
         }
-    ]
 
 
 class NJoyIE(NDRBaseIE):
-    IE_NAME = 'N-JOY'
-    _VALID_URL = r'https?://www\.n-joy\.de/.+?,(?P<id>\w+)\.html'
-
-    _TEST = {
+    IE_NAME = 'njoy'
+    IE_DESC = 'N-JOY'
+    _VALID_URL = r'https?://www\.n-joy\.de/(?:[^/]+/)+(?P<id>[^/?#]+),[\da-z]+\.html'
+    _TESTS = [{
+        # httpVideo, same content id
         'url': 'http://www.n-joy.de/entertainment/comedy/comedy_contest/Benaissa-beim-NDR-Comedy-Contest,comedycontest2480.html',
         'md5': 'cb63be60cd6f9dd75218803146d8dc67',
         'info_dict': {
             'id': 'comedycontest2480',
+            'display_id': 'Benaissa-beim-NDR-Comedy-Contest',
             'ext': 'mp4',
             'title': 'Benaissa beim NDR Comedy Contest',
+            'description': 'md5:f057a6c4e1c728b10d33b5ffd36ddc39',
+            'uploader': 'ndrtv',
+            'upload_date': '20141129',
             'duration': 654,
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }, {
+        # httpVideo, different content id
+        'url': 'http://www.n-joy.de/musik/Das-frueheste-DJ-Set-des-Nordens-live-mit-Felix-Jaehn-,felixjaehn168.html',
+        'md5': '417660fffa90e6df2fda19f1b40a64d8',
+        'info_dict': {
+            'id': 'dockville882',
+            'display_id': 'Das-frueheste-DJ-Set-des-Nordens-live-mit-Felix-Jaehn-',
+            'ext': 'mp4',
+            'title': '"Ich hab noch nie" mit Felix Jaehn',
+            'description': 'md5:85dd312d53be1b99e1f998a16452a2f3',
+            'uploader': 'njoy',
+            'upload_date': '20150822',
+            'duration': 211,
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }]
+
+    def _extract_embed(self, webpage, display_id):
+        video_id = self._search_regex(
+            r'<iframe[^>]+id="pp_([\da-z]+)"', webpage, 'embed id')
+        description = self._search_regex(
+            r'<div[^>]+class="subline"[^>]*>[^<]+</div>\s*<p>([^<]+)</p>',
+            webpage, 'description', fatal=False)
+        return {
+            '_type': 'url_transparent',
+            'ie_key': 'NDREmbedBase',
+            'url': 'ndr:%s' % video_id,
+            'display_id': display_id,
+            'description': description,
         }
-    }
 
 
-class NDREmbedBaseIE(NDRBaseIE):
+class NDREmbedBaseIE(InfoExtractor):
+    IE_NAME = 'ndr:embed:base'
+    _VALID_URL = r'(?:ndr:(?P<id_s>[\da-z]+)|https?://www\.ndr\.de/(?P<id>[\da-z]+)-ppjson\.json)'
+    _TESTS = [{
+        'url': 'ndr:soundcheck3366',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.ndr.de/soundcheck3366-ppjson.json',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-        json_data = self._download_json('http://www.ndr.de/%s-ppjson.json' % video_id, video_id, fatal=False)
-        if not json_data:
-            raise ExtractorError('No media links available for %s' % video_id)
-        return self.extract_video_info(json_data['playlist'], video_id)
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id') or mobj.group('id_s')
+
+        ppjson = self._download_json(
+            'http://www.ndr.de/%s-ppjson.json' % video_id, video_id)
+
+        playlist = ppjson['playlist']
+
+        formats = []
+        quality_key = qualities(('xs', 's', 'm', 'l', 'xl'))
+
+        for format_id, f in playlist.items():
+            src = f.get('src')
+            if not src:
+                continue
+            ext = determine_ext(src, None)
+            if ext == 'f4m':
+                formats.extend(self._extract_f4m_formats(
+                    src + '?hdcore=3.7.0&plugin=aasp-3.7.0.39.44', video_id, f4m_id='hds'))
+            elif ext == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    src, video_id, m3u8_id='hls', entry_protocol='m3u8_native'))
+            else:
+                quality = f.get('quality')
+                ff = {
+                    'url': src,
+                    'format_id': quality or format_id,
+                    'quality': quality_key(quality),
+                }
+                type_ = f.get('type')
+                if type_ and type_.split('/')[0] == 'audio':
+                    ff['vcodec'] = 'none'
+                    ff['ext'] = ext or 'mp3'
+                formats.append(ff)
+        self._sort_formats(formats)
+
+        config = playlist['config']
+
+        live = playlist.get('config', {}).get('streamType') in ['httpVideoLive', 'httpAudioLive']
+        title = config['title']
+        if live:
+            title = self._live_title(title)
+        uploader = ppjson.get('config', {}).get('branding')
+        upload_date = ppjson.get('config', {}).get('publicationDate')
+        duration = int_or_none(config.get('duration'))
+
+        thumbnails = [{
+            'id': thumbnail.get('quality') or thumbnail_id,
+            'url': thumbnail['src'],
+            'preference': quality_key(thumbnail.get('quality')),
+        } for thumbnail_id, thumbnail in config.get('poster', {}).items() if thumbnail.get('src')]
+
+        return {
+            'id': video_id,
+            'title': title,
+            'is_live': live,
+            'uploader': uploader if uploader != '-' else None,
+            'upload_date': upload_date[0:8] if upload_date else None,
+            'duration': duration,
+            'thumbnails': thumbnails,
+            'formats': formats,
+        }
 
 
 class NDREmbedIE(NDREmbedBaseIE):
     IE_NAME = 'ndr:embed'
-    _VALID_URL = r'https?://www\.ndr\.de/(?:[^/]+/)+(?P<id>[a-z0-9]+)-(?:player|externalPlayer)\.html'
-
-    _TEST = {
+    _VALID_URL = r'https?://www\.ndr\.de/(?:[^/]+/)+(?P<id>[\da-z]+)-(?:player|externalPlayer)\.html'
+    _TESTS = [{
         'url': 'http://www.ndr.de/fernsehen/sendungen/ndr_aktuell/ndraktuell28488-player.html',
-        'md5': 'cb63be60cd6f9dd75218803146d8dc67',
+        'md5': '8b9306142fe65bbdefb5ce24edb6b0a9',
         'info_dict': {
             'id': 'ndraktuell28488',
             'ext': 'mp4',
             'title': 'Norddeutschland begrüßt Flüchtlinge',
+            'is_live': False,
+            'uploader': 'ndrtv',
+            'upload_date': '20150907',
             'duration': 132,
-        }
-    }
+        },
+    }, {
+        'url': 'http://www.ndr.de/ndr2/events/soundcheck/soundcheck3366-player.html',
+        'md5': '002085c44bae38802d94ae5802a36e78',
+        'info_dict': {
+            'id': 'soundcheck3366',
+            'ext': 'mp4',
+            'title': 'Ella Henderson braucht Vergleiche nicht zu scheuen',
+            'is_live': False,
+            'uploader': 'ndr2',
+            'upload_date': '20150912',
+            'duration': 3554,
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }, {
+        'url': 'http://www.ndr.de/info/audio51535-player.html',
+        'md5': 'bb3cd38e24fbcc866d13b50ca59307b8',
+        'info_dict': {
+            'id': 'audio51535',
+            'ext': 'mp3',
+            'title': 'La Valette entgeht der Hinrichtung',
+            'is_live': False,
+            'uploader': 'ndrinfo',
+            'upload_date': '20140729',
+            'duration': 884,
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }, {
+        'url': 'http://www.ndr.de/fernsehen/sendungen/visite/visite11010-externalPlayer.html',
+        'md5': 'ae57f80511c1e1f2fd0d0d3d31aeae7c',
+        'info_dict': {
+            'id': 'visite11010',
+            'ext': 'mp4',
+            'title': 'Visite - die ganze Sendung',
+            'is_live': False,
+            'uploader': 'ndrtv',
+            'upload_date': '20150902',
+            'duration': 3525,
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }, {
+        # httpVideoLive
+        'url': 'http://www.ndr.de/fernsehen/livestream/livestream217-externalPlayer.html',
+        'info_dict': {
+            'id': 'livestream217',
+            'ext': 'flv',
+            'title': 're:^NDR Fernsehen Niedersachsen \d{4}-\d{2}-\d{2} \d{2}:\d{2}$',
+            'is_live': True,
+            'upload_date': '20150910',
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }, {
+        'url': 'http://www.ndr.de/ndrkultur/audio255020-player.html',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.ndr.de/fernsehen/sendungen/nordtour/nordtour7124-player.html',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.ndr.de/kultur/film/videos/videoimport10424-player.html',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.ndr.de/fernsehen/sendungen/hamburg_journal/hamj43006-player.html',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.ndr.de/fernsehen/sendungen/weltbilder/weltbilder4518-player.html',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.ndr.de/fernsehen/doku952-player.html',
+        'only_matching': True,
+    }]
 
 
 class NJoyEmbedIE(NDREmbedBaseIE):
-    IE_NAME = 'N-JOY:embed'
-    _VALID_URL = r'https?://www\.n-joy\.de/(?:[^/]+/)+(?P<id>[a-z0-9]+)-(?:player|externalPlayer)\.html'
-
-    _TEST = {
-        'url': 'http://www.n-joy.de/entertainment/film/portraet374-player_image-832d9b79-fa8a-4026-92e2-e0fd99deb2f9_theme-n-joy.html',
-        'md5': 'cb63be60cd6f9dd75218803146d8dc67',
+    IE_NAME = 'njoy:embed'
+    _VALID_URL = r'https?://www\.n-joy\.de/(?:[^/]+/)+(?P<id>[\da-z]+)-(?:player|externalPlayer)_[^/]+\.html'
+    _TESTS = [{
+        # httpVideo
+        'url': 'http://www.n-joy.de/events/reeperbahnfestival/doku948-player_image-bc168e87-5263-4d6d-bd27-bb643005a6de_theme-n-joy.html',
+        'md5': '8483cbfe2320bd4d28a349d62d88bd74',
         'info_dict': {
-            'id': 'portraet374',
+            'id': 'doku948',
             'ext': 'mp4',
-            'title': 'Viviane Andereggen - "Schuld um Schuld"',
-            'duration': 129,
-        }
-    }
+            'title': 'Zehn Jahre Reeperbahn Festival - die Doku',
+            'is_live': False,
+            'upload_date': '20150807',
+            'duration': 1011,
+        },
+    }, {
+        # httpAudio
+        'url': 'http://www.n-joy.de/news_wissen/stefanrichter100-player_image-d5e938b1-f21a-4b9a-86b8-aaba8bca3a13_theme-n-joy.html',
+        'md5': 'd989f80f28ac954430f7b8a48197188a',
+        'info_dict': {
+            'id': 'stefanrichter100',
+            'ext': 'mp3',
+            'title': 'Interview mit einem Augenzeugen',
+            'is_live': False,
+            'uploader': 'njoy',
+            'upload_date': '20150909',
+            'duration': 140,
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }, {
+        # httpAudioLive, no explicit ext
+        'url': 'http://www.n-joy.de/news_wissen/webradioweltweit100-player_image-3fec0484-2244-4565-8fb8-ed25fd28b173_theme-n-joy.html',
+        'info_dict': {
+            'id': 'webradioweltweit100',
+            'ext': 'mp3',
+            'title': 're:^N-JOY Weltweit \d{4}-\d{2}-\d{2} \d{2}:\d{2}$',
+            'is_live': True,
+            'uploader': 'njoy',
+            'upload_date': '20150810',
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }, {
+        'url': 'http://www.n-joy.de/musik/dockville882-player_image-3905259e-0803-4764-ac72-8b7de077d80a_theme-n-joy.html',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.n-joy.de/radio/sendungen/morningshow/urlaubsfotos190-player_image-066a5df1-5c95-49ec-a323-941d848718db_theme-n-joy.html',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.n-joy.de/entertainment/comedy/krudetv290-player_image-ab261bfe-51bf-4bf3-87ba-c5122ee35b3d_theme-n-joy.html',
+        'only_matching': True,
+    }]

From 82c18e2a53a14cf88bc15a1d0b0f4342d2dc5db8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 13 Sep 2015 19:18:57 +0600
Subject: [PATCH 2513/6219] [nowness] Simplify

---
 youtube_dl/extractor/__init__.py |   2 +-
 youtube_dl/extractor/nowness.py  | 107 ++++++++++++++++---------------
 2 files changed, 57 insertions(+), 52 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 99da52d96..74b7df463 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -408,7 +408,7 @@ from .novamov import NovaMovIE
 from .nowness import (
     NownessIE,
     NownessPlaylistIE,
-    NownessSerieIE,
+    NownessSeriesIE,
 )
 from .nowtv import NowTVIE
 from .nowvideo import NowVideoIE
diff --git a/youtube_dl/extractor/nowness.py b/youtube_dl/extractor/nowness.py
index 9a0e46917..9fb916c18 100644
--- a/youtube_dl/extractor/nowness.py
+++ b/youtube_dl/extractor/nowness.py
@@ -4,11 +4,14 @@ from __future__ import unicode_literals
 from .brightcove import BrightcoveIE
 from .common import InfoExtractor
 from ..utils import ExtractorError
-from ..compat import compat_urllib_request
+from ..compat import (
+    compat_str,
+    compat_urllib_request,
+)
 
 
 class NownessBaseIE(InfoExtractor):
-    def extract_url_result(self, post):
+    def _extract_url_result(self, post):
         if post['type'] == 'video':
             for media in post['media']:
                 if media['type'] == 'video':
@@ -18,7 +21,7 @@ class NownessBaseIE(InfoExtractor):
                         player_code = self._download_webpage(
                             'http://www.nowness.com/iframe?id=%s' % video_id, video_id,
                             note='Downloading player JavaScript',
-                            errnote='Player download failed')
+                            errnote='Unable to download player JavaScript')
                         bc_url = BrightcoveIE._extract_brightcove_url(player_code)
                         if bc_url is None:
                             raise ExtractorError('Could not find player definition')
@@ -32,50 +35,46 @@ class NownessBaseIE(InfoExtractor):
                         # return self.url_result('http://cinematique.com/embed/%s' % video_id, 'Cinematique')
                         pass
 
-    def api_request(self, url, request_path):
+    def _api_request(self, url, request_path):
         display_id = self._match_id(url)
-
-        lang = 'zh-cn' if 'cn.nowness.com' in url else 'en-us'
-        request = compat_urllib_request.Request('http://api.nowness.com/api/' + request_path % display_id, headers={
-            'X-Nowness-Language': lang,
-        })
-        json_data = self._download_json(request, display_id)
-        return display_id, json_data
+        request = compat_urllib_request.Request(
+            'http://api.nowness.com/api/' + request_path % display_id,
+            headers={
+                'X-Nowness-Language': 'zh-cn' if 'cn.nowness.com' in url else 'en-us',
+            })
+        return display_id, self._download_json(request, display_id)
 
 
 class NownessIE(NownessBaseIE):
     IE_NAME = 'nowness'
     _VALID_URL = r'https?://(?:(?:www|cn)\.)?nowness\.com/(?:story|(?:series|category)/[^/]+)/(?P<id>[^/]+?)(?:$|[?#])'
-    _TESTS = [
-        {
-            'url': 'https://www.nowness.com/story/candor-the-art-of-gesticulation',
-            'md5': '068bc0202558c2e391924cb8cc470676',
-            'info_dict': {
-                'id': '2520295746001',
-                'ext': 'mp4',
-                'title': 'Candor: The Art of Gesticulation',
-                'description': 'Candor: The Art of Gesticulation',
-                'thumbnail': 're:^https?://.*\.jpg',
-                'uploader': 'Nowness',
-            }
-        },
-        {
-            'url': 'https://cn.nowness.com/story/kasper-bjorke-ft-jaakko-eino-kalevi-tnr',
-            'md5': 'e79cf125e387216f86b2e0a5b5c63aa3',
-            'info_dict': {
-                'id': '3716354522001',
-                'ext': 'mp4',
-                'title': 'Kasper Bjørke ft. Jaakko Eino Kalevi: TNR',
-                'description': 'Kasper Bjørke ft. Jaakko Eino Kalevi: TNR',
-                'thumbnail': 're:^https?://.*\.jpg',
-                'uploader': 'Nowness',
-            }
-        },
-    ]
+    _TESTS = [{
+        'url': 'https://www.nowness.com/story/candor-the-art-of-gesticulation',
+        'md5': '068bc0202558c2e391924cb8cc470676',
+        'info_dict': {
+            'id': '2520295746001',
+            'ext': 'mp4',
+            'title': 'Candor: The Art of Gesticulation',
+            'description': 'Candor: The Art of Gesticulation',
+            'thumbnail': 're:^https?://.*\.jpg',
+            'uploader': 'Nowness',
+        }
+    }, {
+        'url': 'https://cn.nowness.com/story/kasper-bjorke-ft-jaakko-eino-kalevi-tnr',
+        'md5': 'e79cf125e387216f86b2e0a5b5c63aa3',
+        'info_dict': {
+            'id': '3716354522001',
+            'ext': 'mp4',
+            'title': 'Kasper Bjørke ft. Jaakko Eino Kalevi: TNR',
+            'description': 'Kasper Bjørke ft. Jaakko Eino Kalevi: TNR',
+            'thumbnail': 're:^https?://.*\.jpg',
+            'uploader': 'Nowness',
+        }
+    }]
 
     def _real_extract(self, url):
-        display_id, post = self.api_request(url, 'post/getBySlug/%s')
-        return self.extract_url_result(post)
+        _, post = self._api_request(url, 'post/getBySlug/%s')
+        return self._extract_url_result(post)
 
 
 class NownessPlaylistIE(NownessBaseIE):
@@ -83,33 +82,39 @@ class NownessPlaylistIE(NownessBaseIE):
     _VALID_URL = r'https?://(?:(?:www|cn)\.)?nowness\.com/playlist/(?P<id>\d+)'
     _TEST = {
         'url': 'https://www.nowness.com/playlist/3286/i-guess-thats-why-they-call-it-the-blues',
-        'info_dict':
-        {
+        'info_dict': {
             'id': '3286',
         },
         'playlist_mincount': 8,
     }
 
     def _real_extract(self, url):
-        playlist_id, playlist = self.api_request(url, 'post?PlaylistId=%s')
-        entries = [self.extract_url_result(item) for item in playlist['items']]
+        playlist_id, playlist = self._api_request(url, 'post?PlaylistId=%s')
+        entries = [self._extract_url_result(item) for item in playlist['items']]
         return self.playlist_result(entries, playlist_id)
 
 
-class NownessSerieIE(NownessBaseIE):
-    IE_NAME = 'nowness:serie'
+class NownessSeriesIE(NownessBaseIE):
+    IE_NAME = 'nowness:series'
     _VALID_URL = r'https?://(?:(?:www|cn)\.)?nowness\.com/series/(?P<id>[^/]+?)(?:$|[?#])'
     _TEST = {
         'url': 'https://www.nowness.com/series/60-seconds',
-        'info_dict':
-        {
+        'info_dict': {
             'id': '60',
+            'title': '60 Seconds',
+            'description': 'One-minute wisdom in a new NOWNESS series',
         },
         'playlist_mincount': 4,
     }
 
     def _real_extract(self, url):
-        display_id, serie = self.api_request(url, 'series/getBySlug/%s')
-        serie_id = str(serie['id'])
-        entries = [self.extract_url_result(post) for post in serie['posts']]
-        return self.playlist_result(entries, serie_id)
+        display_id, series = self._api_request(url, 'series/getBySlug/%s')
+        entries = [self._extract_url_result(post) for post in series['posts']]
+        series_title = None
+        series_description = None
+        translations = series.get('translations', [])
+        if translations:
+            series_title = translations[0].get('title') or translations[0]['seoTitle']
+            series_description = translations[0].get('seoDescription')
+        return self.playlist_result(
+            entries, compat_str(series['id']), series_title, series_description)

From c246773599f6b541033ac926ac6ed360fb63f6ae Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 13 Sep 2015 19:27:22 +0600
Subject: [PATCH 2514/6219] [nowness] Add vimeo test

---
 youtube_dl/extractor/nowness.py | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/nowness.py b/youtube_dl/extractor/nowness.py
index 9fb916c18..b97f62fdb 100644
--- a/youtube_dl/extractor/nowness.py
+++ b/youtube_dl/extractor/nowness.py
@@ -58,7 +58,7 @@ class NownessIE(NownessBaseIE):
             'description': 'Candor: The Art of Gesticulation',
             'thumbnail': 're:^https?://.*\.jpg',
             'uploader': 'Nowness',
-        }
+        },
     }, {
         'url': 'https://cn.nowness.com/story/kasper-bjorke-ft-jaakko-eino-kalevi-tnr',
         'md5': 'e79cf125e387216f86b2e0a5b5c63aa3',
@@ -69,7 +69,21 @@ class NownessIE(NownessBaseIE):
             'description': 'Kasper Bjørke ft. Jaakko Eino Kalevi: TNR',
             'thumbnail': 're:^https?://.*\.jpg',
             'uploader': 'Nowness',
-        }
+        },
+    }, {
+        # vimeo
+        'url': 'https://www.nowness.com/series/nowness-picks/jean-luc-godard-supercut',
+        'md5': '9a5a6a8edf806407e411296ab6bc2a49',
+        'info_dict': {
+            'id': '130020913',
+            'ext': 'mp4',
+            'title': 'Bleu, Blanc, Rouge - A Godard Supercut',
+            'description': 'md5:f0ea5f1857dffca02dbd37875d742cec',
+            'thumbnail': 're:^https?://.*\.jpg',
+            'upload_date': '20150607',
+            'uploader': 'Cinema Sem Lei',
+            'uploader_id': 'cinemasemlei',
+        },
     }]
 
     def _real_extract(self, url):

From ac7a1b0dfb9e1e3ea384dd36a425ed9483ee17ef Mon Sep 17 00:00:00 2001
From: "Sergey M." <dstftw@gmail.com>
Date: Sun, 13 Sep 2015 19:31:20 +0600
Subject: [PATCH 2515/6219] [README.md] Clarify playlist_index description

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 24bfe38a2..27b20df04 100644
--- a/README.md
+++ b/README.md
@@ -278,7 +278,7 @@ The `-o` option allows users to indicate a template for the output file names. T
  - `epoch`: The sequence will be replaced by the Unix epoch when creating the file.
  - `autonumber`: The sequence will be replaced by a five-digit number that will be increased with each download, starting at zero.
  - `playlist`: The name or the id of the playlist that contains the video.
- - `playlist_index`: The index of the video in the playlist, a five-digit number.
+ - `playlist_index`: The sequence will be replaced by the index of the video in the playlist padded with leading zeros according to the total length of the playlist.
  - `format_id`: The sequence will be replaced by the format code specified by `--format`.
 
 The current default template is `%(title)s-%(id)s.%(ext)s`.

From 31208a07c247f5b8f1f0fc26da1d477f7440e513 Mon Sep 17 00:00:00 2001
From: "Sergey M." <dstftw@gmail.com>
Date: Sun, 13 Sep 2015 19:34:09 +0600
Subject: [PATCH 2516/6219] [README.md] Unify playlist description

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 27b20df04..1a00a02bf 100644
--- a/README.md
+++ b/README.md
@@ -277,7 +277,7 @@ The `-o` option allows users to indicate a template for the output file names. T
  - `ext`: The sequence will be replaced by the appropriate extension (like flv or mp4).
  - `epoch`: The sequence will be replaced by the Unix epoch when creating the file.
  - `autonumber`: The sequence will be replaced by a five-digit number that will be increased with each download, starting at zero.
- - `playlist`: The name or the id of the playlist that contains the video.
+ - `playlist`: The sequence will be replaced by the name or the id of the playlist that contains the video.
  - `playlist_index`: The sequence will be replaced by the index of the video in the playlist padded with leading zeros according to the total length of the playlist.
  - `format_id`: The sequence will be replaced by the format code specified by `--format`.
 

From e2ff3df314c7e710d429ca311b2a22839d4aa3f9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 14 Sep 2015 03:35:15 +0600
Subject: [PATCH 2517/6219] [clubic] Relax _VALID_URL (Closes #6854)

---
 youtube_dl/extractor/clubic.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/clubic.py b/youtube_dl/extractor/clubic.py
index 14f215c5c..1dfa7c12e 100644
--- a/youtube_dl/extractor/clubic.py
+++ b/youtube_dl/extractor/clubic.py
@@ -12,9 +12,9 @@ from ..utils import (
 
 
 class ClubicIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?clubic\.com/video/[^/]+/video.*-(?P<id>[0-9]+)\.html'
+    _VALID_URL = r'http://(?:www\.)?clubic\.com/video/(?:[^/]+/)*video.*-(?P<id>[0-9]+)\.html'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.clubic.com/video/clubic-week/video-clubic-week-2-0-le-fbi-se-lance-dans-la-photo-d-identite-448474.html',
         'md5': '1592b694ba586036efac1776b0b43cd3',
         'info_dict': {
@@ -24,7 +24,10 @@ class ClubicIE(InfoExtractor):
             'description': 're:Gueule de bois chez Nokia. Le constructeur a indiqué cette.*',
             'thumbnail': 're:^http://img\.clubic\.com/.*\.jpg$',
         }
-    }
+    }, {
+        'url': 'http://www.clubic.com/video/video-clubic-week-2-0-apple-iphone-6s-et-plus-mais-surtout-le-pencil-469792.html',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)

From 953fed280f1a35bc6e8a701a25b8450ec08efdd3 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 14 Sep 2015 00:25:08 +0200
Subject: [PATCH 2518/6219] [compat] Do not use unicode

If the code ever runs on 3.x, it would fail. Even if it never does, the unicode name confuses Python 3 code analysis tools.
---
 youtube_dl/compat.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index e32bef279..1ff42d94b 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -80,6 +80,11 @@ try:
 except ImportError:
     import BaseHTTPServer as compat_http_server
 
+try:
+    compat_str = unicode  # Python 2
+except NameError:
+    compat_str = str
+
 try:
     from urllib.parse import unquote_to_bytes as compat_urllib_parse_unquote_to_bytes
     from urllib.parse import unquote as compat_urllib_parse_unquote
@@ -100,7 +105,7 @@ except ImportError:  # Python 2
             # Is it a string-like object?
             string.split
             return b''
-        if isinstance(string, unicode):
+        if isinstance(string, compat_str):
             string = string.encode('utf-8')
         bits = string.split(b'%')
         if len(bits) == 1:
@@ -150,11 +155,6 @@ except ImportError:  # Python 2
         string = string.replace('+', ' ')
         return compat_urllib_parse_unquote(string, encoding, errors)
 
-try:
-    compat_str = unicode  # Python 2
-except NameError:
-    compat_str = str
-
 try:
     compat_basestring = basestring  # Python 2
 except NameError:
@@ -234,7 +234,7 @@ else:
     # Working around shlex issue with unicode strings on some python 2
     # versions (see http://bugs.python.org/issue1548891)
     def compat_shlex_split(s, comments=False, posix=True):
-        if isinstance(s, unicode):
+        if isinstance(s, compat_str):
             s = s.encode('utf-8')
         return shlex.split(s, comments, posix)
 

From 287be8c615574f90f4fac5adad0b32b3102f621e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 14 Sep 2015 00:26:12 +0200
Subject: [PATCH 2519/6219] [youtube:truncated_url] Add ?t=

Sometimes found in links - ?t=123&v=ABCD starts the video at position 123.
---
 youtube_dl/extractor/youtube.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 97ce36550..abacdfaed 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1970,6 +1970,7 @@ class YoutubeTruncatedURLIE(InfoExtractor):
             annotation_id=annotation_[^&]+|
             x-yt-cl=[0-9]+|
             hl=[^&]*|
+            t=[0-9]+
         )?
         |
             attribution_link\?a=[^&]+
@@ -1992,6 +1993,9 @@ class YoutubeTruncatedURLIE(InfoExtractor):
     }, {
         'url': 'https://www.youtube.com/watch?hl=en-GB',
         'only_matching': True,
+    }, {
+        'url': 'https://www.youtube.com/watch?t=2372',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 2b3c2546782f84e0b3fe42b55d2030e5350cbe92 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 14 Sep 2015 00:32:20 +0200
Subject: [PATCH 2520/6219] [youtube:channel] Correct 404 handling

Previously, when we encountered a 404 - such as youtube-dl https://www.youtube.com/ohJeiboh8oorehai - we crashed with a regexp error. Instead, make sure to go on and eventually report a 404.
---
 youtube_dl/extractor/youtube.py | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index abacdfaed..b252e36e1 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1654,12 +1654,15 @@ class YoutubeChannelIE(InfoExtractor):
         channel_page = self._download_webpage(
             url + '?view=57', channel_id,
             'Downloading channel page', fatal=False)
-        channel_playlist_id = self._html_search_meta(
-            'channelId', channel_page, 'channel id', default=None)
-        if not channel_playlist_id:
-            channel_playlist_id = self._search_regex(
-                r'data-channel-external-id="([^"]+)"',
-                channel_page, 'channel id', default=None)
+        if channel_page is False:
+            channel_playlist_id = False
+        else:
+            channel_playlist_id = self._html_search_meta(
+                'channelId', channel_page, 'channel id', default=None)
+            if not channel_playlist_id:
+                channel_playlist_id = self._search_regex(
+                    r'data-channel-external-id="([^"]+)"',
+                    channel_page, 'channel id', default=None)
         if channel_playlist_id and channel_playlist_id.startswith('UC'):
             playlist_id = 'UU' + channel_playlist_id[2:]
             return self.url_result(

From 3b9264a04902c4ce01d3fac67bf51799fece0e53 Mon Sep 17 00:00:00 2001
From: "Sergey M." <dstftw@gmail.com>
Date: Mon, 14 Sep 2015 23:07:03 +0600
Subject: [PATCH 2521/6219] [README.md] Fix typo

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 1a00a02bf..42656aae1 100644
--- a/README.md
+++ b/README.md
@@ -261,7 +261,7 @@ For example:
 machine youtube login myaccount@gmail.com password my_youtube_password
 machine twitch login my_twitch_account_name password my_twitch_password
 ```
-To activate authentication with `.netrc` file you should pass `--netrc` to youtube-dl or to place it in [configuration file](#configuration).
+To activate authentication with `.netrc` file you should pass `--netrc` to youtube-dl or place it in [configuration file](#configuration).
 
 On Windows you may also need to setup `%HOME%` environment variable manually.
 

From 0391bc8176c22e586f1c866145bc31e9cd2c50ed Mon Sep 17 00:00:00 2001
From: Alex Vong <alexvong1995@gmail.com>
Date: Mon, 10 Aug 2015 12:55:29 +0800
Subject: [PATCH 2522/6219] Add the missing tag `EMBEDDING YOUTUBE-DL` in the
 content table.

* README.md: Add missing tag.
---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index 42656aae1..2ed751791 100644
--- a/README.md
+++ b/README.md
@@ -9,6 +9,7 @@ youtube-dl - download videos from youtube.com or other video platforms
 - [VIDEO SELECTION](#video-selection)
 - [FAQ](#faq)
 - [DEVELOPER INSTRUCTIONS](#developer-instructions)
+- [EMBEDDING YOUTUBE-DL](#embedding-youtube-dl)
 - [BUGS](#bugs)
 - [COPYRIGHT](#copyright)
 

From 272e4db5c746ef4c94b7c9e7bac0fc008319b158 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 15 Sep 2015 21:24:01 +0600
Subject: [PATCH 2523/6219] [pornhub] Relax _VALID_URL (Closes #6868)

---
 youtube_dl/extractor/pornhub.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index 7b0cdc41a..a656ad85a 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -20,7 +20,7 @@ from ..aes import (
 
 
 class PornHubIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?pornhub\.com/(?:view_video\.php\?viewkey=|embed/)(?P<id>[0-9a-z]+)'
+    _VALID_URL = r'https?://(?:[a-z]+\.)?pornhub\.com/(?:view_video\.php\?viewkey=|embed/)(?P<id>[0-9a-z]+)'
     _TESTS = [{
         'url': 'http://www.pornhub.com/view_video.php?viewkey=648719015',
         'md5': '882f488fa1f0026f023f33576004a2ed',
@@ -34,6 +34,9 @@ class PornHubIE(InfoExtractor):
     }, {
         'url': 'http://www.pornhub.com/view_video.php?viewkey=ph557bbb6676d2d',
         'only_matching': True,
+    }, {
+        'url': 'http://fr.pornhub.com/view_video.php?viewkey=ph55ca2f9760862',
+        'only_matching': True,
     }]
 
     @classmethod

From 88060cce108f7436f8747186855ef0d63fa23338 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 15 Sep 2015 21:57:30 +0600
Subject: [PATCH 2524/6219] [rai] Add support for videos embedded with
 drawMediaRaiTV (Closes #6866)

---
 youtube_dl/extractor/rai.py | 22 +++++++++++++++++++---
 1 file changed, 19 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/rai.py b/youtube_dl/extractor/rai.py
index 1631faf29..7ff1d06c4 100644
--- a/youtube_dl/extractor/rai.py
+++ b/youtube_dl/extractor/rai.py
@@ -5,6 +5,7 @@ import re
 from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse,
+    compat_urlparse,
 )
 from ..utils import (
     parse_duration,
@@ -72,6 +73,18 @@ class RaiIE(InfoExtractor):
                 'description': 'Primo appuntamento con "Il candidato" con Filippo Timi, alias Piero Zucca presidente!',
                 'uploader': 'RaiTre',
             }
+        },
+        {
+            'url': 'http://www.report.rai.it/dl/Report/puntata/ContentItem-0c7a664b-d0f4-4b2c-8835-3f82e46f433e.html',
+            'md5': '037104d2c14132887e5e4cf114569214',
+            'info_dict': {
+                'id': '0c7a664b-d0f4-4b2c-8835-3f82e46f433e',
+                'ext': 'flv',
+                'title': 'Il pacco',
+                'description': 'md5:4b1afae1364115ce5d78ed83cd2e5b3a',
+                'uploader': 'RaiTre',
+                'upload_date': '20141221',
+            },
         }
     ]
 
@@ -90,11 +103,14 @@ class RaiIE(InfoExtractor):
         relinker_url = self._extract_relinker_url(webpage)
 
         if not relinker_url:
-            iframe_path = self._search_regex(
-                r'<iframe[^>]+src="/?(dl/[^"]+\?iframe\b[^"]*)"',
+            iframe_url = self._search_regex(
+                [r'<iframe[^>]+src="([^"]*/dl/[^"]+\?iframe\b[^"]*)"',
+                 r'drawMediaRaiTV\(["\'](.+?)["\']'],
                 webpage, 'iframe')
+            if not iframe_url.startswith('http'):
+                iframe_url = compat_urlparse.urljoin(url, iframe_url)
             webpage = self._download_webpage(
-                '%s/%s' % (host, iframe_path), video_id)
+                iframe_url, video_id)
             relinker_url = self._extract_relinker_url(webpage)
 
         relinker = self._download_json(

From 12bc2429444b9f89d1b7f4f631c25cc3af2a1d7c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 16 Sep 2015 22:25:25 +0600
Subject: [PATCH 2525/6219] [telecinco] Incorporate mitele code

---
 youtube_dl/extractor/telecinco.py | 54 +++++++++++++++++++++++++++++--
 1 file changed, 52 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/telecinco.py b/youtube_dl/extractor/telecinco.py
index ae94f055c..c0ad286c5 100644
--- a/youtube_dl/extractor/telecinco.py
+++ b/youtube_dl/extractor/telecinco.py
@@ -1,10 +1,22 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-from .mitele import MiTeleIE
+import json
+
+from .common import InfoExtractor
+from ..compat import (
+    compat_urllib_parse,
+    compat_urllib_parse_unquote,
+    compat_urlparse,
+)
+from ..utils import (
+    get_element_by_attribute,
+    parse_duration,
+    strip_jsonp,
+)
 
 
-class TelecincoIE(MiTeleIE):
+class TelecincoIE(InfoExtractor):
     IE_NAME = 'telecinco.es'
     _VALID_URL = r'https?://www\.telecinco\.es/(?:[^/]+/)+(?P<id>.+?)\.html'
 
@@ -27,3 +39,41 @@ class TelecincoIE(MiTeleIE):
         'url': 'http://www.telecinco.es/espanasinirmaslejos/Espana-gran-destino-turistico_2_1240605043.html',
         'only_matching': True,
     }]
+
+    def _real_extract(self, url):
+        episode = self._match_id(url)
+        webpage = self._download_webpage(url, episode)
+        embed_data_json = self._search_regex(
+            r'(?s)MSV\.embedData\[.*?\]\s*=\s*({.*?});', webpage, 'embed data',
+        ).replace('\'', '"')
+        embed_data = json.loads(embed_data_json)
+
+        domain = embed_data['mediaUrl']
+        if not domain.startswith('http'):
+            # only happens in telecinco.es videos
+            domain = 'http://' + domain
+        info_url = compat_urlparse.urljoin(
+            domain,
+            compat_urllib_parse_unquote(embed_data['flashvars']['host'])
+        )
+        info_el = self._download_xml(info_url, episode).find('./video/info')
+
+        video_link = info_el.find('videoUrl/link').text
+        token_query = compat_urllib_parse.urlencode({'id': video_link})
+        token_info = self._download_json(
+            embed_data['flashvars']['ov_tk'] + '?' + token_query,
+            episode,
+            transform_source=strip_jsonp
+        )
+        formats = self._extract_m3u8_formats(
+            token_info['tokenizedUrl'], episode, ext='mp4')
+
+        return {
+            'id': embed_data['videoId'],
+            'display_id': episode,
+            'title': info_el.find('title').text,
+            'formats': formats,
+            'description': get_element_by_attribute('class', 'text', webpage),
+            'thumbnail': info_el.find('thumb').text,
+            'duration': parse_duration(info_el.find('duration').text),
+        }

From f84ce1ebaf930f5fc422e4fab77fa9c7c86a999a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 16 Sep 2015 22:27:03 +0600
Subject: [PATCH 2526/6219] [mitele] Fix extraction (Closes #6414)

---
 youtube_dl/extractor/mitele.py | 105 ++++++++++++++++++---------------
 1 file changed, 58 insertions(+), 47 deletions(-)

diff --git a/youtube_dl/extractor/mitele.py b/youtube_dl/extractor/mitele.py
index 852d72266..9e8a8fc72 100644
--- a/youtube_dl/extractor/mitele.py
+++ b/youtube_dl/extractor/mitele.py
@@ -1,17 +1,11 @@
 from __future__ import unicode_literals
 
-import json
-
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse,
-    compat_urllib_parse_unquote,
-    compat_urlparse,
-)
+from ..compat import compat_urllib_parse
 from ..utils import (
+    encode_dict,
     get_element_by_attribute,
-    parse_duration,
-    strip_jsonp,
+    int_or_none,
 )
 
 
@@ -21,54 +15,71 @@ class MiTeleIE(InfoExtractor):
 
     _TESTS = [{
         'url': 'http://www.mitele.es/programas-tv/diario-de/la-redaccion/programa-144/',
+        'md5': 'ace7635b2a0b286aaa37d3ff192d2a8a',
         'info_dict': {
-            'id': '0fce117d',
-            'ext': 'mp4',
-            'title': 'Programa 144 - Tor, la web invisible',
-            'description': 'md5:3b6fce7eaa41b2d97358726378d9369f',
+            'id': '0NF1jJnxS1Wu3pHrmvFyw2',
             'display_id': 'programa-144',
+            'ext': 'flv',
+            'title': 'Tor, la web invisible',
+            'description': 'md5:3b6fce7eaa41b2d97358726378d9369f',
+            'thumbnail': 're:(?i)^https?://.*\.jpg$',
             'duration': 2913,
         },
-        'params': {
-            # m3u8 download
-            'skip_download': True,
-        },
     }]
 
     def _real_extract(self, url):
-        episode = self._match_id(url)
-        webpage = self._download_webpage(url, episode)
-        embed_data_json = self._search_regex(
-            r'(?s)MSV\.embedData\[.*?\]\s*=\s*({.*?});', webpage, 'embed data',
-        ).replace('\'', '"')
-        embed_data = json.loads(embed_data_json)
+        display_id = self._match_id(url)
 
-        domain = embed_data['mediaUrl']
-        if not domain.startswith('http'):
-            # only happens in telecinco.es videos
-            domain = 'http://' + domain
-        info_url = compat_urlparse.urljoin(
-            domain,
-            compat_urllib_parse_unquote(embed_data['flashvars']['host'])
-        )
-        info_el = self._download_xml(info_url, episode).find('./video/info')
+        webpage = self._download_webpage(url, display_id)
 
-        video_link = info_el.find('videoUrl/link').text
-        token_query = compat_urllib_parse.urlencode({'id': video_link})
-        token_info = self._download_json(
-            embed_data['flashvars']['ov_tk'] + '?' + token_query,
-            episode,
-            transform_source=strip_jsonp
-        )
-        formats = self._extract_m3u8_formats(
-            token_info['tokenizedUrl'], episode, ext='mp4')
+        config_url = self._search_regex(
+            r'data-config\s*=\s*"([^"]+)"', webpage, 'data config url')
+
+        config = self._download_json(
+            config_url, display_id, 'Downloading config JSON')
+
+        mmc = self._download_json(
+            config['services']['mmc'], display_id, 'Downloading mmc JSON')
+
+        formats = []
+        for location in mmc['locations']:
+            gat = self._proto_relative_url(location.get('gat'), 'http:')
+            bas = location.get('bas')
+            loc = location.get('loc')
+            ogn = location.get('ogn')
+            if None in (gat, bas, loc, ogn):
+                continue
+            token_data = {
+                'bas': bas,
+                'icd': loc,
+                'ogn': ogn,
+                'sta': '0',
+            }
+            media = self._download_json(
+                '%s/?%s' % (gat, compat_urllib_parse.urlencode(encode_dict(token_data)).encode('utf-8')),
+                display_id, 'Downloading %s JSON' % location['loc'])
+            file_ = media.get('file')
+            if not file_:
+                continue
+            formats.extend(self._extract_f4m_formats(
+                file_ + '&hdcore=3.2.0&plugin=aasp-3.2.0.77.18',
+                display_id, f4m_id=loc))
+
+        title = self._search_regex(
+            r'class="Destacado-text"[^>]*>\s*<strong>([^<]+)</strong>', webpage, 'title')
+
+        video_id = self._search_regex(
+            r'data-media-id\s*=\s*"([^"]+)"', webpage,
+            'data media id', default=None) or display_id
+        thumbnail = config.get('poster', {}).get('imageUrl')
+        duration = int_or_none(mmc.get('duration'))
 
         return {
-            'id': embed_data['videoId'],
-            'display_id': episode,
-            'title': info_el.find('title').text,
-            'formats': formats,
+            'id': video_id,
+            'display_id': display_id,
+            'title': title,
             'description': get_element_by_attribute('class', 'text', webpage),
-            'thumbnail': info_el.find('thumb').text,
-            'duration': parse_duration(info_el.find('duration').text),
+            'thumbnail': thumbnail,
+            'duration': duration,
+            'formats': formats,
         }

From d5e7657fe2c045545e3cdc089413661befad8b4c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 16 Sep 2015 22:37:01 +0600
Subject: [PATCH 2527/6219] [telecinco] Add support or cuatro.com

---
 youtube_dl/extractor/telecinco.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/telecinco.py b/youtube_dl/extractor/telecinco.py
index c0ad286c5..018fd7d34 100644
--- a/youtube_dl/extractor/telecinco.py
+++ b/youtube_dl/extractor/telecinco.py
@@ -18,7 +18,7 @@ from ..utils import (
 
 class TelecincoIE(InfoExtractor):
     IE_NAME = 'telecinco.es'
-    _VALID_URL = r'https?://www\.telecinco\.es/(?:[^/]+/)+(?P<id>.+?)\.html'
+    _VALID_URL = r'https?://www\.(?:telecinco\.es|cuatro\.com)/(?:[^/]+/)+(?P<id>.+?)\.html'
 
     _TESTS = [{
         'url': 'http://www.telecinco.es/robinfood/temporada-01/t01xp14/Bacalao-cocochas-pil-pil_0_1876350223.html',
@@ -32,6 +32,15 @@ class TelecincoIE(InfoExtractor):
             # m3u8 download
             'skip_download': True,
         },
+    }, {
+        'url': 'http://www.cuatro.com/deportes/futbol/barcelona/Leo_Messi-Champions-Roma_2_2052780128.html',
+        'md5': '0a5b9f3cc8b074f50a0578f823a12694',
+        'info_dict': {
+            'id': 'MDSVID20150916_0128',
+            'ext': 'mp4',
+            'title': '¿Quién es este ex futbolista con el que hablan ...',
+            'duration': 79,
+        },
     }, {
         'url': 'http://www.telecinco.es/informativos/nacional/Pablo_Iglesias-Informativos_Telecinco-entrevista-Pedro_Piqueras_2_1945155182.html',
         'only_matching': True,
@@ -66,7 +75,7 @@ class TelecincoIE(InfoExtractor):
             transform_source=strip_jsonp
         )
         formats = self._extract_m3u8_formats(
-            token_info['tokenizedUrl'], episode, ext='mp4')
+            token_info['tokenizedUrl'], episode, ext='mp4', entry_protocol='m3u8_native')
 
         return {
             'id': embed_data['videoId'],

From 369e60162e9cc99c89dac52a40297b68cbc82392 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 16 Sep 2015 22:37:47 +0600
Subject: [PATCH 2528/6219] [telecinco] Update test

---
 youtube_dl/extractor/telecinco.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/telecinco.py b/youtube_dl/extractor/telecinco.py
index 018fd7d34..685f1db40 100644
--- a/youtube_dl/extractor/telecinco.py
+++ b/youtube_dl/extractor/telecinco.py
@@ -22,16 +22,13 @@ class TelecincoIE(InfoExtractor):
 
     _TESTS = [{
         'url': 'http://www.telecinco.es/robinfood/temporada-01/t01xp14/Bacalao-cocochas-pil-pil_0_1876350223.html',
+        'md5': '5cbef3ad5ef17bf0d21570332d140729',
         'info_dict': {
             'id': 'MDSVID20141015_0058',
             'ext': 'mp4',
             'title': 'Con Martín Berasategui, hacer un bacalao al ...',
             'duration': 662,
         },
-        'params': {
-            # m3u8 download
-            'skip_download': True,
-        },
     }, {
         'url': 'http://www.cuatro.com/deportes/futbol/barcelona/Leo_Messi-Champions-Roma_2_2052780128.html',
         'md5': '0a5b9f3cc8b074f50a0578f823a12694',

From 0e1b2566fff795981d0f7cccbca6700a3fc260a6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 16 Sep 2015 22:39:24 +0600
Subject: [PATCH 2529/6219] [telecinco] Clarify IE_DESC

---
 youtube_dl/extractor/telecinco.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/telecinco.py b/youtube_dl/extractor/telecinco.py
index 685f1db40..9740c3452 100644
--- a/youtube_dl/extractor/telecinco.py
+++ b/youtube_dl/extractor/telecinco.py
@@ -17,7 +17,7 @@ from ..utils import (
 
 
 class TelecincoIE(InfoExtractor):
-    IE_NAME = 'telecinco.es'
+    IE_DESC = 'telecinco.es and cuatro.es'
     _VALID_URL = r'https?://www\.(?:telecinco\.es|cuatro\.com)/(?:[^/]+/)+(?P<id>.+?)\.html'
 
     _TESTS = [{

From 3368d70dceba93aa9acdb97ec2705471a3424c57 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 16 Sep 2015 22:39:48 +0600
Subject: [PATCH 2530/6219] [mitele] Clarify IE_DESC

---
 youtube_dl/extractor/mitele.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mitele.py b/youtube_dl/extractor/mitele.py
index 9e8a8fc72..54993e2c9 100644
--- a/youtube_dl/extractor/mitele.py
+++ b/youtube_dl/extractor/mitele.py
@@ -10,7 +10,7 @@ from ..utils import (
 
 
 class MiTeleIE(InfoExtractor):
-    IE_NAME = 'mitele.es'
+    IE_DESC = 'mitele.es'
     _VALID_URL = r'http://www\.mitele\.es/[^/]+/[^/]+/[^/]+/(?P<id>[^/]+)/'
 
     _TESTS = [{

From d492dad8f440c8d0d006a896de0a56b01cd46496 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 16 Sep 2015 22:45:39 +0600
Subject: [PATCH 2531/6219] [telecinco] Add support for mediaset.es

---
 youtube_dl/extractor/telecinco.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/telecinco.py b/youtube_dl/extractor/telecinco.py
index 9740c3452..b0bf9ab8d 100644
--- a/youtube_dl/extractor/telecinco.py
+++ b/youtube_dl/extractor/telecinco.py
@@ -17,8 +17,8 @@ from ..utils import (
 
 
 class TelecincoIE(InfoExtractor):
-    IE_DESC = 'telecinco.es and cuatro.es'
-    _VALID_URL = r'https?://www\.(?:telecinco\.es|cuatro\.com)/(?:[^/]+/)+(?P<id>.+?)\.html'
+    IE_DESC = 'telecinco.es, cuatro.es and mediaset.es'
+    _VALID_URL = r'https?://www\.(?:telecinco\.es|cuatro\.com|mediaset\.es)/(?:[^/]+/)+(?P<id>.+?)\.html'
 
     _TESTS = [{
         'url': 'http://www.telecinco.es/robinfood/temporada-01/t01xp14/Bacalao-cocochas-pil-pil_0_1876350223.html',
@@ -38,6 +38,15 @@ class TelecincoIE(InfoExtractor):
             'title': '¿Quién es este ex futbolista con el que hablan ...',
             'duration': 79,
         },
+    }, {
+        'url': 'http://www.mediaset.es/12meses/campanas/doylacara/conlatratanohaytrato/Ayudame-dar-cara-trata-trato_2_1986630220.html',
+        'md5': 'ad1bfaaba922dd4a295724b05b68f86a',
+        'info_dict': {
+            'id': 'MDSVID20150513_0220',
+            'ext': 'mp4',
+            'title': '#DOYLACARA. Con la trata no hay trato',
+            'duration': 50,
+        },
     }, {
         'url': 'http://www.telecinco.es/informativos/nacional/Pablo_Iglesias-Informativos_Telecinco-entrevista-Pedro_Piqueras_2_1945155182.html',
         'only_matching': True,

From 4647fd89103dc5ded16dc069828a2cf38216093c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 16 Sep 2015 22:49:26 +0600
Subject: [PATCH 2532/6219] [telecinco] Fix typo

---
 youtube_dl/extractor/telecinco.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/telecinco.py b/youtube_dl/extractor/telecinco.py
index b0bf9ab8d..2c8e9b941 100644
--- a/youtube_dl/extractor/telecinco.py
+++ b/youtube_dl/extractor/telecinco.py
@@ -17,7 +17,7 @@ from ..utils import (
 
 
 class TelecincoIE(InfoExtractor):
-    IE_DESC = 'telecinco.es, cuatro.es and mediaset.es'
+    IE_DESC = 'telecinco.es, cuatro.com and mediaset.es'
     _VALID_URL = r'https?://www\.(?:telecinco\.es|cuatro\.com|mediaset\.es)/(?:[^/]+/)+(?P<id>.+?)\.html'
 
     _TESTS = [{

From 659ffe204c946b977266aa373110fe50a2763066 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 17 Sep 2015 22:58:49 +0600
Subject: [PATCH 2533/6219] [divxstage] Remove extractor

DivxStage is offline for quite some time already
---
 youtube_dl/extractor/__init__.py  |  1 -
 youtube_dl/extractor/divxstage.py | 27 ---------------------------
 2 files changed, 28 deletions(-)
 delete mode 100644 youtube_dl/extractor/divxstage.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 74b7df463..c16e9e3d2 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -138,7 +138,6 @@ from .dump import DumpIE
 from .dumpert import DumpertIE
 from .defense import DefenseGouvFrIE
 from .discovery import DiscoveryIE
-from .divxstage import DivxStageIE
 from .dropbox import DropboxIE
 from .eagleplatform import EaglePlatformIE
 from .ebaumsworld import EbaumsWorldIE
diff --git a/youtube_dl/extractor/divxstage.py b/youtube_dl/extractor/divxstage.py
deleted file mode 100644
index b88379e06..000000000
--- a/youtube_dl/extractor/divxstage.py
+++ /dev/null
@@ -1,27 +0,0 @@
-from __future__ import unicode_literals
-
-from .novamov import NovaMovIE
-
-
-class DivxStageIE(NovaMovIE):
-    IE_NAME = 'divxstage'
-    IE_DESC = 'DivxStage'
-
-    _VALID_URL = NovaMovIE._VALID_URL_TEMPLATE % {'host': 'divxstage\.(?:eu|net|ch|co|at|ag|to)'}
-
-    _HOST = 'www.divxstage.eu'
-
-    _FILE_DELETED_REGEX = r'>This file no longer exists on our servers.<'
-    _TITLE_REGEX = r'<div class="video_det">\s*<strong>([^<]+)</strong>'
-    _DESCRIPTION_REGEX = r'<div class="video_det">\s*<strong>[^<]+</strong>\s*<p>([^<]+)</p>'
-
-    _TEST = {
-        'url': 'http://www.divxstage.eu/video/57f238e2e5e01',
-        'md5': '63969f6eb26533a1968c4d325be63e72',
-        'info_dict': {
-            'id': '57f238e2e5e01',
-            'ext': 'flv',
-            'title': 'youtubedl test video',
-            'description': 'This is a test video for youtubedl.',
-        }
-    }

From 1806a75415a3b62b07ab3fd282f915c6af14dfd3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 17 Sep 2015 22:59:15 +0600
Subject: [PATCH 2534/6219] [ok] Capture error message

---
 youtube_dl/extractor/odnoklassniki.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/youtube_dl/extractor/odnoklassniki.py b/youtube_dl/extractor/odnoklassniki.py
index 66520c2c5..0cd96c10a 100644
--- a/youtube_dl/extractor/odnoklassniki.py
+++ b/youtube_dl/extractor/odnoklassniki.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 from .common import InfoExtractor
 from ..compat import compat_urllib_parse_unquote
 from ..utils import (
+    ExtractorError,
     unified_strdate,
     int_or_none,
     qualities,
@@ -72,6 +73,12 @@ class OdnoklassnikiIE(InfoExtractor):
         webpage = self._download_webpage(
             'http://ok.ru/video/%s' % video_id, video_id)
 
+        error = self._search_regex(
+            r'[^>]+class="vp_video_stub_txt"[^>]*>([^<]+)<',
+            webpage, 'error', default=None)
+        if error:
+            raise ExtractorError(error, expected=True)
+
         player = self._parse_json(
             unescapeHTML(self._search_regex(
                 r'data-options=(?P<quote>["\'])(?P<player>{.+?%s.+?})(?P=quote)' % video_id,

From 131d05033bfe3f5a8ef3241489a44dc5c1e03830 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 17 Sep 2015 22:59:32 +0600
Subject: [PATCH 2535/6219] [ok] Skip test

---
 youtube_dl/extractor/odnoklassniki.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/odnoklassniki.py b/youtube_dl/extractor/odnoklassniki.py
index 0cd96c10a..ccc88cfb1 100644
--- a/youtube_dl/extractor/odnoklassniki.py
+++ b/youtube_dl/extractor/odnoklassniki.py
@@ -29,6 +29,7 @@ class OdnoklassnikiIE(InfoExtractor):
             'like_count': int,
             'age_limit': 0,
         },
+        'skip': 'Video has been blocked',
     }, {
         # metadataUrl
         'url': 'http://ok.ru/video/63567059965189-0',

From 8a7bbd160641dbbc13f68aaa26dfa49b71483a8e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 17 Sep 2015 15:22:19 +0200
Subject: [PATCH 2536/6219] [postprocessor/ffmpeg] Always use the 'file:'
 protocol for filenames (fixes #6874)

If the filename contains ':' it is interpreted as a protocol.
It also handles filenames starting with '-'.
---
 youtube_dl/downloader/hls.py       |  2 +-
 youtube_dl/postprocessor/ffmpeg.py | 13 ++++++++-----
 2 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index b2436e732..7743e176a 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -31,7 +31,7 @@ class HlsFD(FileDownloader):
         args = [
             encodeArgument(opt)
             for opt in (ffpp.executable, '-y', '-i', url, '-f', 'mp4', '-c', 'copy', '-bsf:a', 'aac_adtstoasc')]
-        args.append(encodeFilename(tmpfilename, True))
+        args.append(encodeFilename(ffpp._ffmpeg_filename_argument(tmpfilename), True))
 
         self._debug_cmd(args)
 
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 1f723908b..4f320e124 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -135,7 +135,10 @@ class FFmpegPostProcessor(PostProcessor):
 
         files_cmd = []
         for path in input_paths:
-            files_cmd.extend([encodeArgument('-i'), encodeFilename(path, True)])
+            files_cmd.extend([
+                encodeArgument('-i'),
+                encodeFilename(self._ffmpeg_filename_argument(path), True)
+            ])
         cmd = ([encodeFilename(self.executable, True), encodeArgument('-y')] +
                files_cmd +
                [encodeArgument(o) for o in opts] +
@@ -155,10 +158,10 @@ class FFmpegPostProcessor(PostProcessor):
         self.run_ffmpeg_multiple_files([path], out_path, opts)
 
     def _ffmpeg_filename_argument(self, fn):
-        # ffmpeg broke --, see https://ffmpeg.org/trac/ffmpeg/ticket/2127 for details
-        if fn.startswith('-'):
-            return './' + fn
-        return fn
+        # Always use 'file:' because the filename may contain ':' (ffmpeg
+        # interprets that as a protocol) or can start with '-' (-- is broken in
+        # ffmpeg, see https://ffmpeg.org/trac/ffmpeg/ticket/2127 for details)
+        return 'file:' + fn
 
 
 class FFmpegExtractAudioPP(FFmpegPostProcessor):

From 06368a232a9bc8b383db77b0087a2ab1af94a833 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 18 Sep 2015 21:01:55 +0600
Subject: [PATCH 2537/6219] [playwire] Fix test

---
 youtube_dl/extractor/playwire.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/playwire.py b/youtube_dl/extractor/playwire.py
index bdc71017b..6d138ef25 100644
--- a/youtube_dl/extractor/playwire.py
+++ b/youtube_dl/extractor/playwire.py
@@ -19,7 +19,7 @@ class PlaywireIE(InfoExtractor):
             'id': '3353705',
             'ext': 'mp4',
             'title': 'S04_RM_UCL_Rus',
-            'thumbnail': 're:^http://.*\.png$',
+            'thumbnail': 're:^https?://.*\.png$',
             'duration': 145.94,
         },
     }, {

From d1c694ea4ad75eb14e02e27ba6db5be95a598f21 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 19 Sep 2015 00:51:41 +0600
Subject: [PATCH 2538/6219] [bbc] Switch extraction to hq iptv hls (Closes
 #2732, closes #3023, closes #4828, closes #6803, closes #6855)

---
 youtube_dl/extractor/bbc.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index abc5a44a1..42526357a 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -21,6 +21,7 @@ class BBCCoUkIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?bbc\.co\.uk/(?:(?:(?:programmes|iplayer(?:/[^/]+)?/(?:episode|playlist))/)|music/clips[/#])(?P<id>[\da-z]{8})'
 
     _MEDIASELECTOR_URLS = [
+        'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/iptv-all/vpid/%s',
         'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/pc/vpid/%s',
     ]
 
@@ -189,6 +190,12 @@ class BBCCoUkIE(InfoExtractor):
             # Skip DASH until supported
             elif transfer_format == 'dash':
                 pass
+            elif transfer_format == 'hls':
+                m3u8_formats = self._extract_m3u8_formats(
+                    href, programme_id, ext='mp4', entry_protocol='m3u8_native',
+                    m3u8_id=supplier, fatal=False)
+                if m3u8_formats:
+                    formats.extend(m3u8_formats)
             # Direct link
             else:
                 formats.append({

From 1d67c9664099c261b7752cb56773e423d410fae6 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 18 Sep 2015 22:09:33 +0100
Subject: [PATCH 2539/6219] [shahid] fix api request url

---
 youtube_dl/extractor/shahid.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/shahid.py b/youtube_dl/extractor/shahid.py
index 6e9903d5e..3fd545373 100644
--- a/youtube_dl/extractor/shahid.py
+++ b/youtube_dl/extractor/shahid.py
@@ -81,7 +81,7 @@ class ShahidIE(InfoExtractor):
                 compat_urllib_parse.urlencode({
                     'apiKey': 'sh@hid0nlin3',
                     'hash': 'b2wMCTHpSmyxGqQjJFOycRmLSex+BpTK/ooxy6vHaqs=',
-                }).encode('utf-8')),
+                })),
             video_id, 'Downloading video JSON')
 
         video = video[api_vars['playerType']]

From 5de5ab89b4136e7f2191f50d8d69587b90e0c3bb Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 18 Sep 2015 22:10:36 +0100
Subject: [PATCH 2540/6219] [shahid] change test ext

---
 youtube_dl/extractor/shahid.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/shahid.py b/youtube_dl/extractor/shahid.py
index 3fd545373..f76fb12c0 100644
--- a/youtube_dl/extractor/shahid.py
+++ b/youtube_dl/extractor/shahid.py
@@ -16,7 +16,7 @@ class ShahidIE(InfoExtractor):
         'url': 'https://shahid.mbc.net/ar/episode/90574/%D8%A7%D9%84%D9%85%D9%84%D9%83-%D8%B9%D8%A8%D8%AF%D8%A7%D9%84%D9%84%D9%87-%D8%A7%D9%84%D8%A5%D9%86%D8%B3%D8%A7%D9%86-%D8%A7%D9%84%D9%85%D9%88%D8%B3%D9%85-1-%D9%83%D9%84%D9%8A%D8%A8-3.html',
         'info_dict': {
             'id': '90574',
-            'ext': 'm3u8',
+            'ext': 'mp4',
             'title': 'الملك عبدالله الإنسان الموسم 1 كليب 3',
             'description': 'الفيلم الوثائقي - الملك عبد الله الإنسان',
             'duration': 2972,

From 6c91a5a7f5408cf666f3ee40b53c0d9e42521b88 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 20 Sep 2015 11:16:12 +0600
Subject: [PATCH 2541/6219] [extractor/generic] Fix following redirect in
 Refresh HTTP header on python 2

---
 youtube_dl/extractor/generic.py | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index ec748ed9f..8881a8a23 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 
 import os
 import re
+import sys
 
 from .common import InfoExtractor
 from .youtube import YoutubeIE
@@ -230,6 +231,22 @@ class GenericIE(InfoExtractor):
                 'skip_download': False,
             }
         },
+        {
+            # redirect in Refresh HTTP header
+            'url': 'https://www.facebook.com/l.php?u=https%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DpO8h3EaFRdo&h=TAQHsoToz&enc=AZN16h-b6o4Zq9pZkCCdOLNKMN96BbGMNtcFwHSaazus4JHT_MFYkAA-WARTX2kvsCIdlAIyHZjl6d33ILIJU7Jzwk_K3mcenAXoAzBNoZDI_Q7EXGDJnIhrGkLXo_LJ_pAa2Jzbx17UHMd3jAs--6j2zaeto5w9RTn8T_1kKg3fdC5WPX9Dbb18vzH7YFX0eSJmoa6SP114rvlkw6pkS1-T&s=1',
+            'info_dict': {
+                'id': 'pO8h3EaFRdo',
+                'ext': 'mp4',
+                'title': 'Tripeo Boiler Room x Dekmantel Festival DJ Set',
+                'description': 'md5:6294cc1af09c4049e0652b51a2df10d5',
+                'upload_date': '20150917',
+                'uploader_id': 'brtvofficial',
+                'uploader': 'Boiler Room',
+            },
+            'params': {
+                'skip_download': False,
+            },
+        },
         {
             'url': 'http://www.hodiho.fr/2013/02/regis-plante-sa-jeep.html',
             'md5': '85b90ccc9d73b4acd9138d3af4c27f89',
@@ -1808,6 +1825,9 @@ class GenericIE(InfoExtractor):
                 # Look also in Refresh HTTP header
                 refresh_header = head_response.headers.get('Refresh')
                 if refresh_header:
+                    # In python 2 response HTTP headers are bytestrings
+                    if sys.version_info < (3, 0) and isinstance(refresh_header, str):
+                        refresh_header = refresh_header.decode('iso-8859-1')
                     found = re.search(REDIRECT_REGEX, refresh_header)
             if found:
                 new_url = compat_urlparse.urljoin(url, unescapeHTML(found.group(1)))

From f817adc4689a2064fcab733d6aebf83fd0e2cff7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 20 Sep 2015 11:31:23 +0600
Subject: [PATCH 2542/6219] [youtube:history] Disable extractor until #6893 is
 investigated

Wiped out part of my history as well
---
 youtube_dl/extractor/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index c16e9e3d2..b93156232 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -808,7 +808,7 @@ from .youtube import (
     YoutubeIE,
     YoutubeChannelIE,
     YoutubeFavouritesIE,
-    YoutubeHistoryIE,
+    #YoutubeHistoryIE,
     YoutubePlaylistIE,
     YoutubeRecommendedIE,
     YoutubeSearchDateIE,

From 393ca8c94d1adb1490b23265370ce69043b92546 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 20 Sep 2015 11:45:19 +0600
Subject: [PATCH 2543/6219] [arte:+7] Look for json vp url in iframe (Closes
 #6895)

---
 youtube_dl/extractor/arte.py | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index 76de24477..9ecb6786c 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -4,6 +4,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import (
+    compat_parse_qs,
+    compat_urllib_parse_urlparse,
+)
 from ..utils import (
     find_xpath_attr,
     unified_strdate,
@@ -77,7 +81,13 @@ class ArteTVPlus7IE(InfoExtractor):
     def _extract_from_webpage(self, webpage, video_id, lang):
         json_url = self._html_search_regex(
             [r'arte_vp_url=["\'](.*?)["\']', r'data-url=["\']([^"]+)["\']'],
-            webpage, 'json vp url')
+            webpage, 'json vp url', default=None)
+        if not json_url:
+            iframe_url = self._html_search_regex(
+                r'<iframe[^>]+src=(["\'])(?P<url>.+\bjson_url=.+?)\1',
+                webpage, 'iframe url', group='iframe url')
+            json_url = compat_parse_qs(
+                compat_urllib_parse_urlparse(iframe_url).query)['json_url'][0]
         return self._extract_from_json_url(json_url, video_id, lang)
 
     def _extract_from_json_url(self, json_url, video_id, lang):

From 5e39123b3b6fd740d133fd45824b49b4bc674fa7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 20 Sep 2015 11:47:03 +0600
Subject: [PATCH 2544/6219] [arte:+7] Fix typo

---
 youtube_dl/extractor/arte.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index 9ecb6786c..2a00da3ee 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -85,7 +85,7 @@ class ArteTVPlus7IE(InfoExtractor):
         if not json_url:
             iframe_url = self._html_search_regex(
                 r'<iframe[^>]+src=(["\'])(?P<url>.+\bjson_url=.+?)\1',
-                webpage, 'iframe url', group='iframe url')
+                webpage, 'iframe url', group='url')
             json_url = compat_parse_qs(
                 compat_urllib_parse_urlparse(iframe_url).query)['json_url'][0]
         return self._extract_from_json_url(json_url, video_id, lang)

From f005f96ea594a82969885ecd124b094c7e09949a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 20 Sep 2015 12:23:13 +0200
Subject: [PATCH 2545/6219] [youtube:history] Explain why it has disabled and
 skip test

---
 test/test_all_urls.py            | 2 +-
 youtube_dl/extractor/__init__.py | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/test/test_all_urls.py b/test/test_all_urls.py
index a9db42b30..a929afd15 100644
--- a/test/test_all_urls.py
+++ b/test/test_all_urls.py
@@ -99,7 +99,7 @@ class TestAllURLsMatching(unittest.TestCase):
     def test_keywords(self):
         self.assertMatch(':ytsubs', ['youtube:subscriptions'])
         self.assertMatch(':ytsubscriptions', ['youtube:subscriptions'])
-        self.assertMatch(':ythistory', ['youtube:history'])
+        # self.assertMatch(':ythistory', ['youtube:history'])
         self.assertMatch(':thedailyshow', ['ComedyCentralShows'])
         self.assertMatch(':tds', ['ComedyCentralShows'])
 
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index b93156232..2529d8657 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -808,6 +808,8 @@ from .youtube import (
     YoutubeIE,
     YoutubeChannelIE,
     YoutubeFavouritesIE,
+    # disabled because it can wipe the watch history (see #6893)
+    # remember to uncumment test in test/test_all_urls when it's fixed
     #YoutubeHistoryIE,
     YoutubePlaylistIE,
     YoutubeRecommendedIE,

From 5a1a2e94548f25b5fd540090af3f32a1d875f6b4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 20 Sep 2015 21:08:29 +0600
Subject: [PATCH 2546/6219] [utils] Fix kwargs on old python 2 (Closes #6905)

---
 youtube_dl/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 206dd56bc..1dc3153fd 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -619,7 +619,7 @@ def _create_http_connection(ydl_handler, http_class, is_https, *args, **kwargs):
     # expected HTTP responses to meet HTTP/1.0 or later (see also
     # https://github.com/rg3/youtube-dl/issues/6727)
     if sys.version_info < (3, 0):
-        kwargs['strict'] = True
+        kwargs[b'strict'] = True
     hc = http_class(*args, **kwargs)
     source_address = ydl_handler._params.get('source_address')
     if source_address is not None:

From 5b4c54631a8888d75fa766c0bd6ec1822e6caec5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 20 Sep 2015 23:12:40 +0600
Subject: [PATCH 2547/6219] [nfl] Add team domains (#6907)

---
 youtube_dl/extractor/nfl.py | 132 +++++++++++++++++++++++-------------
 1 file changed, 86 insertions(+), 46 deletions(-)

diff --git a/youtube_dl/extractor/nfl.py b/youtube_dl/extractor/nfl.py
index dc54634a5..fe143ef88 100644
--- a/youtube_dl/extractor/nfl.py
+++ b/youtube_dl/extractor/nfl.py
@@ -16,53 +16,93 @@ from ..utils import (
 
 class NFLIE(InfoExtractor):
     IE_NAME = 'nfl.com'
-    _VALID_URL = r'''(?x)https?://
-        (?P<host>(?:www\.)?(?:nfl\.com|.*?\.clubs\.nfl\.com))/
-        (?:.+?/)*
-        (?P<id>(?:[a-z0-9]{16}|\w{8}\-(?:\w{4}\-){3}\w{12}))'''
-    _TESTS = [
-        {
-            'url': 'http://www.nfl.com/videos/nfl-game-highlights/0ap3000000398478/Week-3-Redskins-vs-Eagles-highlights',
-            'md5': '394ef771ddcd1354f665b471d78ec4c6',
-            'info_dict': {
-                'id': '0ap3000000398478',
-                'ext': 'mp4',
-                'title': 'Week 3: Redskins vs. Eagles highlights',
-                'description': 'md5:56323bfb0ac4ee5ab24bd05fdf3bf478',
-                'upload_date': '20140921',
-                'timestamp': 1411337580,
-                'thumbnail': 're:^https?://.*\.jpg$',
-            }
-        },
-        {
-            'url': 'http://prod.www.steelers.clubs.nfl.com/video-and-audio/videos/LIVE_Post_Game_vs_Browns/9d72f26a-9e2b-4718-84d3-09fb4046c266',
-            'md5': 'cf85bdb4bc49f6e9d3816d130c78279c',
-            'info_dict': {
-                'id': '9d72f26a-9e2b-4718-84d3-09fb4046c266',
-                'ext': 'mp4',
-                'title': 'LIVE: Post Game vs. Browns',
-                'description': 'md5:6a97f7e5ebeb4c0e69a418a89e0636e8',
-                'upload_date': '20131229',
-                'timestamp': 1388354455,
-                'thumbnail': 're:^https?://.*\.jpg$',
-            }
-        },
-        {
-            'url': 'http://www.nfl.com/news/story/0ap3000000467586/article/patriots-seahawks-involved-in-lategame-skirmish',
-            'info_dict': {
-                'id': '0ap3000000467607',
-                'ext': 'mp4',
-                'title': 'Frustrations flare on the field',
-                'description': 'Emotions ran high at the end of the Super Bowl on both sides of the ball after a dramatic finish.',
-                'timestamp': 1422850320,
-                'upload_date': '20150202',
-            },
-        },
-        {
-            'url': 'http://www.nfl.com/videos/nfl-network-top-ten/09000d5d810a6bd4/Top-10-Gutsiest-Performances-Jack-Youngblood',
-            'only_matching': True,
+    _VALID_URL = r'''(?x)
+                    https?://
+                        (?P<host>
+                            (?:www\.)?
+                            (?:
+                                (?:
+                                    nfl|
+                                    buffalobills|
+                                    miamidolphins|
+                                    patriots|
+                                    newyorkjets|
+                                    baltimoreravens|
+                                    bengals|
+                                    clevelandbrowns|
+                                    steelers|
+                                    houstontexans|
+                                    colts|
+                                    jaguars|
+                                    titansonline|
+                                    denverbroncos|
+                                    kcchiefs|
+                                    raiders|
+                                    chargers|
+                                    dallascowboys|
+                                    giants|
+                                    philadelphiaeagles|
+                                    redskins|
+                                    chicagobears|
+                                    detroitlions|
+                                    packers|
+                                    vikings|
+                                    atlantafalcons|
+                                    panthers|
+                                    neworleanssaints|
+                                    buccaneers|
+                                    azcardinals|
+                                    stlouisrams|
+                                    49ers|
+                                    seahawks
+                                )\.com|
+                                .+?\.clubs\.nfl\.com
+                            )
+                        )/
+                        (?:.+?/)*
+                        (?P<id>(?:[a-z0-9]{16}|\w{8}\-(?:\w{4}\-){3}\w{12}))
+                    '''
+    _TESTS = [{
+        'url': 'http://www.nfl.com/videos/nfl-game-highlights/0ap3000000398478/Week-3-Redskins-vs-Eagles-highlights',
+        'md5': '394ef771ddcd1354f665b471d78ec4c6',
+        'info_dict': {
+            'id': '0ap3000000398478',
+            'ext': 'mp4',
+            'title': 'Week 3: Redskins vs. Eagles highlights',
+            'description': 'md5:56323bfb0ac4ee5ab24bd05fdf3bf478',
+            'upload_date': '20140921',
+            'timestamp': 1411337580,
+            'thumbnail': 're:^https?://.*\.jpg$',
         }
-    ]
+    }, {
+        'url': 'http://prod.www.steelers.clubs.nfl.com/video-and-audio/videos/LIVE_Post_Game_vs_Browns/9d72f26a-9e2b-4718-84d3-09fb4046c266',
+        'md5': 'cf85bdb4bc49f6e9d3816d130c78279c',
+        'info_dict': {
+            'id': '9d72f26a-9e2b-4718-84d3-09fb4046c266',
+            'ext': 'mp4',
+            'title': 'LIVE: Post Game vs. Browns',
+            'description': 'md5:6a97f7e5ebeb4c0e69a418a89e0636e8',
+            'upload_date': '20131229',
+            'timestamp': 1388354455,
+            'thumbnail': 're:^https?://.*\.jpg$',
+        }
+    }, {
+        'url': 'http://www.nfl.com/news/story/0ap3000000467586/article/patriots-seahawks-involved-in-lategame-skirmish',
+        'info_dict': {
+            'id': '0ap3000000467607',
+            'ext': 'mp4',
+            'title': 'Frustrations flare on the field',
+            'description': 'Emotions ran high at the end of the Super Bowl on both sides of the ball after a dramatic finish.',
+            'timestamp': 1422850320,
+            'upload_date': '20150202',
+        },
+    }, {
+        'url': 'http://www.nfl.com/videos/nfl-network-top-ten/09000d5d810a6bd4/Top-10-Gutsiest-Performances-Jack-Youngblood',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.buffalobills.com/video/videos/Rex_Ryan_Show_World_Wide_Rex/b1dcfab2-3190-4bb1-bfc0-d6e603d6601a',
+        'only_matching': True,
+    }]
 
     @staticmethod
     def prepend_host(host, url):

From 4423eba49b1ca5e9cfb78370ab0eee8f84fb72bd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 20 Sep 2015 23:45:01 +0600
Subject: [PATCH 2548/6219] [nfl] Add support for URLs without id (Closes
 #6907)

---
 youtube_dl/extractor/nfl.py | 24 ++++++++++++++++++------
 1 file changed, 18 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/nfl.py b/youtube_dl/extractor/nfl.py
index fe143ef88..55dc6107d 100644
--- a/youtube_dl/extractor/nfl.py
+++ b/youtube_dl/extractor/nfl.py
@@ -60,7 +60,7 @@ class NFLIE(InfoExtractor):
                             )
                         )/
                         (?:.+?/)*
-                        (?P<id>(?:[a-z0-9]{16}|\w{8}\-(?:\w{4}\-){3}\w{12}))
+                        (?P<id>[^/#?&]+)
                     '''
     _TESTS = [{
         'url': 'http://www.nfl.com/videos/nfl-game-highlights/0ap3000000398478/Week-3-Redskins-vs-Eagles-highlights',
@@ -96,6 +96,17 @@ class NFLIE(InfoExtractor):
             'timestamp': 1422850320,
             'upload_date': '20150202',
         },
+    }, {
+        'url': 'http://www.patriots.com/video/2015/09/18/10-days-gillette',
+        'md5': '4c319e2f625ffd0b481b4382c6fc124c',
+        'info_dict': {
+            'id': 'n-238346',
+            'ext': 'mp4',
+            'title': '10 Days at Gillette',
+            'description': 'md5:8cd9cd48fac16de596eadc0b24add951',
+            'timestamp': 1442618809,
+            'upload_date': '20150918',
+        },
     }, {
         'url': 'http://www.nfl.com/videos/nfl-network-top-ten/09000d5d810a6bd4/Top-10-Gutsiest-Performances-Jack-Youngblood',
         'only_matching': True,
@@ -135,13 +146,14 @@ class NFLIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         config_url = NFLIE.prepend_host(host, self._search_regex(
-            r'(?:config|configURL)\s*:\s*"([^"]+)"', webpage, 'config URL',
-            default='static/content/static/config/video/config.json'))
+            r'(?:(?:config|configURL)\s*:\s*|<nflcs:avplayer[^>]+data-config\s*=\s*)(["\'])(?P<config>.+?)\1',
+            webpage, 'config URL', default='static/content/static/config/video/config.json',
+            group='config'))
         # For articles, the id in the url is not the video id
         video_id = self._search_regex(
-            r'contentId\s*:\s*"([^"]+)"', webpage, 'video id', default=video_id)
-        config = self._download_json(config_url, video_id,
-                                     note='Downloading player config')
+            r'(?:<nflcs:avplayer[^>]+data-contentId\s*=\s*|contentId\s*:\s*)(["\'])(?P<id>.+?)\1',
+            webpage, 'video id', default=video_id, group='id')
+        config = self._download_json(config_url, video_id, 'Downloading player config')
         url_template = NFLIE.prepend_host(
             host, '{contentURLTemplate:}'.format(**config))
         video_data = self._download_json(

From 82c06a40acb41df77ee55acf8979eb5f0cfba4c3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 20 Sep 2015 23:54:05 +0600
Subject: [PATCH 2549/6219] Fix typo

---
 youtube_dl/extractor/__init__.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 2529d8657..35d7f0dd2 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -809,8 +809,8 @@ from .youtube import (
     YoutubeChannelIE,
     YoutubeFavouritesIE,
     # disabled because it can wipe the watch history (see #6893)
-    # remember to uncumment test in test/test_all_urls when it's fixed
-    #YoutubeHistoryIE,
+    # remember to uncomment test in test/test_all_urls when it's fixed
+    YoutubeHistoryIE,
     YoutubePlaylistIE,
     YoutubeRecommendedIE,
     YoutubeSearchDateIE,

From 9f5e8d16b3c7a1a2731cbab9661a920fbe04e09f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 21 Sep 2015 01:53:28 +0600
Subject: [PATCH 2550/6219] [youtube:history] Disable exractor

---
 youtube_dl/extractor/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 35d7f0dd2..030c00790 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -810,7 +810,7 @@ from .youtube import (
     YoutubeFavouritesIE,
     # disabled because it can wipe the watch history (see #6893)
     # remember to uncomment test in test/test_all_urls when it's fixed
-    YoutubeHistoryIE,
+    #YoutubeHistoryIE,
     YoutubePlaylistIE,
     YoutubeRecommendedIE,
     YoutubeSearchDateIE,

From c6aa838b51484ad3d036db21ba4e9e834720a693 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 21 Sep 2015 21:28:02 +0600
Subject: [PATCH 2551/6219] [youtube:history] Enable exractor

---
 test/test_all_urls.py            | 2 +-
 youtube_dl/extractor/__init__.py | 4 +---
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/test/test_all_urls.py b/test/test_all_urls.py
index a929afd15..a9db42b30 100644
--- a/test/test_all_urls.py
+++ b/test/test_all_urls.py
@@ -99,7 +99,7 @@ class TestAllURLsMatching(unittest.TestCase):
     def test_keywords(self):
         self.assertMatch(':ytsubs', ['youtube:subscriptions'])
         self.assertMatch(':ytsubscriptions', ['youtube:subscriptions'])
-        # self.assertMatch(':ythistory', ['youtube:history'])
+        self.assertMatch(':ythistory', ['youtube:history'])
         self.assertMatch(':thedailyshow', ['ComedyCentralShows'])
         self.assertMatch(':tds', ['ComedyCentralShows'])
 
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 030c00790..c16e9e3d2 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -808,9 +808,7 @@ from .youtube import (
     YoutubeIE,
     YoutubeChannelIE,
     YoutubeFavouritesIE,
-    # disabled because it can wipe the watch history (see #6893)
-    # remember to uncomment test in test/test_all_urls when it's fixed
-    #YoutubeHistoryIE,
+    YoutubeHistoryIE,
     YoutubePlaylistIE,
     YoutubeRecommendedIE,
     YoutubeSearchDateIE,

From 92085e7099d3607ff512f241454d6f6f4535b05a Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 20 Sep 2015 22:26:23 +0100
Subject: [PATCH 2552/6219] [viewster] accept https links and fix api_token
 extraction and extract mp4 video link(fixes #6787)

---
 youtube_dl/extractor/viewster.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/viewster.py b/youtube_dl/extractor/viewster.py
index cda02ba24..c68a38167 100644
--- a/youtube_dl/extractor/viewster.py
+++ b/youtube_dl/extractor/viewster.py
@@ -16,7 +16,7 @@ from ..utils import (
 
 
 class ViewsterIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?viewster\.com/(?:serie|movie)/(?P<id>\d+-\d+-\d+)'
+    _VALID_URL = r'https?://(?:www\.)?viewster\.com/(?:serie|movie)/(?P<id>\d+-\d+-\d+)'
     _TESTS = [{
         # movie, Type=Movie
         'url': 'http://www.viewster.com/movie/1140-11855-000/the-listening-project/',
@@ -74,8 +74,8 @@ class ViewsterIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
         # Get 'api_token' cookie
-        self._request_webpage(HEADRequest(url), video_id)
-        cookies = self._get_cookies(url)
+        self._request_webpage(HEADRequest('http://www.viewster.com/'), video_id)
+        cookies = self._get_cookies('http://www.viewster.com/')
         self._AUTH_TOKEN = compat_urllib_parse_unquote(cookies['api_token'].value)
 
         info = self._download_json(
@@ -98,7 +98,7 @@ class ViewsterIE(InfoExtractor):
             return self.playlist_result(entries, video_id, title, description)
 
         formats = []
-        for media_type in ('application/f4m+xml', 'application/x-mpegURL'):
+        for media_type in ('application/f4m+xml', 'application/x-mpegURL', 'video/mp4'):
             media = self._download_json(
                 'https://public-api.viewster.com/movies/%s/video?mediaType=%s'
                 % (entry_id, compat_urllib_parse.quote(media_type)),
@@ -122,6 +122,8 @@ class ViewsterIE(InfoExtractor):
             else:
                 formats.append({
                     'url': video_url,
+                    'height': int_or_none(media.get('Height')),
+                    'width': int_or_none(media.get('Width')),
                 })
         self._sort_formats(formats)
 

From 8e97596b7b62fb35ccfeb37c0ce5cb008b217d99 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 22 Sep 2015 21:47:56 +0600
Subject: [PATCH 2553/6219] [viewster] Extract height from bitrate and prefer
 mp4 videos

---
 youtube_dl/extractor/viewster.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/viewster.py b/youtube_dl/extractor/viewster.py
index c68a38167..c6aa73c0e 100644
--- a/youtube_dl/extractor/viewster.py
+++ b/youtube_dl/extractor/viewster.py
@@ -120,11 +120,18 @@ class ViewsterIE(InfoExtractor):
                     fatal=False  # m3u8 sometimes fail
                 ))
             else:
-                formats.append({
+                format_id = media.get('Bitrate')
+                f = {
                     'url': video_url,
+                    'format_id': 'mp4-%s' % format_id,
                     'height': int_or_none(media.get('Height')),
                     'width': int_or_none(media.get('Width')),
-                })
+                    'preference': 1,
+                }
+                if format_id and not f['height']:
+                    f['height'] = int_or_none(self._search_regex(
+                        r'^(\d+)[pP]$', format_id, 'height', default=None))
+                formats.append(f)
         self._sort_formats(formats)
 
         synopsis = info.get('Synopsis', {})

From cb4e421901ce5fe26b457c33c141267c5c18a237 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 22 Sep 2015 21:49:29 +0600
Subject: [PATCH 2554/6219] [voewster] Update tests

---
 youtube_dl/extractor/viewster.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/viewster.py b/youtube_dl/extractor/viewster.py
index c6aa73c0e..8defc1800 100644
--- a/youtube_dl/extractor/viewster.py
+++ b/youtube_dl/extractor/viewster.py
@@ -20,10 +20,10 @@ class ViewsterIE(InfoExtractor):
     _TESTS = [{
         # movie, Type=Movie
         'url': 'http://www.viewster.com/movie/1140-11855-000/the-listening-project/',
-        'md5': '14d3cfffe66d57b41ae2d9c873416f01',
+        'md5': 'e642d1b27fcf3a4ffa79f194f5adde36',
         'info_dict': {
             'id': '1140-11855-000',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'The listening Project',
             'description': 'md5:bac720244afd1a8ea279864e67baa071',
             'timestamp': 1214870400,
@@ -33,10 +33,10 @@ class ViewsterIE(InfoExtractor):
     }, {
         # series episode, Type=Episode
         'url': 'http://www.viewster.com/serie/1284-19427-001/the-world-and-a-wall/',
-        'md5': 'd5434c80fcfdb61651cc2199a88d6ba3',
+        'md5': '9243079a8531809efe1b089db102c069',
         'info_dict': {
             'id': '1284-19427-001',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'The World and a Wall',
             'description': 'md5:24814cf74d3453fdf5bfef9716d073e3',
             'timestamp': 1428192000,

From c430802e32932868e24a6e43d0845df963320829 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 22 Sep 2015 21:50:20 +0600
Subject: [PATCH 2555/6219] [extractor/common] Add raise_geo_restricted

---
 youtube_dl/extractor/common.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index d694e818e..1e7db8a9b 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -516,6 +516,12 @@ class InfoExtractor(object):
             '%s. Use --username and --password or --netrc to provide account credentials.' % msg,
             expected=True)
 
+    @staticmethod
+    def raise_geo_restricted(msg='This video is not available from your location due to geo restriction'):
+        raise ExtractorError(
+            '%s. You might want to use --proxy to workaround.' % msg,
+            expected=True)
+
     # Methods for following #608
     @staticmethod
     def url_result(url, ie=None, video_id=None, video_title=None):

From cccedc1aa43f94ce4f8fa6f807a5301250c2213c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 22 Sep 2015 21:52:41 +0600
Subject: [PATCH 2556/6219] [voewster] Detect series geo restriction

---
 youtube_dl/extractor/viewster.py | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/viewster.py b/youtube_dl/extractor/viewster.py
index 8defc1800..f8f4143c6 100644
--- a/youtube_dl/extractor/viewster.py
+++ b/youtube_dl/extractor/viewster.py
@@ -3,12 +3,14 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..compat import (
+    compat_HTTPError,
     compat_urllib_request,
     compat_urllib_parse,
     compat_urllib_parse_unquote,
 )
 from ..utils import (
     determine_ext,
+    ExtractorError,
     int_or_none,
     parse_iso8601,
     HEADRequest,
@@ -86,9 +88,15 @@ class ViewsterIE(InfoExtractor):
 
         # unfinished serie has no Type
         if info.get('Type') in ['Serie', None]:
-            episodes = self._download_json(
-                'https://public-api.viewster.com/series/%s/episodes' % entry_id,
-                video_id, 'Downloading series JSON')
+            try:
+                episodes = self._download_json(
+                    'https://public-api.viewster.com/series/%s/episodes' % entry_id,
+                    video_id, 'Downloading series JSON')
+            except ExtractorError as e:
+                if isinstance(e.cause, compat_HTTPError) and e.cause.code == 404:
+                    self.raise_geo_restricted()
+                else:
+                    raise
             entries = [
                 self.url_result(
                     'http://www.viewster.com/movie/%s' % episode['OriginId'], 'Viewster')

From 9612f2339909f958f0c7ee7ccc370151ec28358d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 22 Sep 2015 21:54:32 +0600
Subject: [PATCH 2557/6219] [viewster] Detect video geo restriction

---
 youtube_dl/extractor/viewster.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/viewster.py b/youtube_dl/extractor/viewster.py
index f8f4143c6..7c4328efb 100644
--- a/youtube_dl/extractor/viewster.py
+++ b/youtube_dl/extractor/viewster.py
@@ -140,6 +140,10 @@ class ViewsterIE(InfoExtractor):
                     f['height'] = int_or_none(self._search_regex(
                         r'^(\d+)[pP]$', format_id, 'height', default=None))
                 formats.append(f)
+
+        if not formats and not info.get('LanguageSets') and not info.get('VODSettings'):
+            self.raise_geo_restricted()
+
         self._sort_formats(formats)
 
         synopsis = info.get('Synopsis', {})

From 7ce50a355c3a4cbd61f18fb16d8175a97f14d510 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 22 Sep 2015 21:55:04 +0600
Subject: [PATCH 2558/6219] [viewster] Add geo restricted tests

---
 youtube_dl/extractor/viewster.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/youtube_dl/extractor/viewster.py b/youtube_dl/extractor/viewster.py
index 7c4328efb..487f2d744 100644
--- a/youtube_dl/extractor/viewster.py
+++ b/youtube_dl/extractor/viewster.py
@@ -63,6 +63,14 @@ class ViewsterIE(InfoExtractor):
             'description': 'md5:e7097a8fc97151e25f085c9eb7a1cdb1',
         },
         'playlist_mincount': 16,
+    }, {
+        # geo restricted series
+        'url': 'https://www.viewster.com/serie/1280-18794-002/',
+        'only_matching': True,
+    }, {
+        # geo restricted video
+        'url': 'https://www.viewster.com/serie/1280-18794-002/what-is-extraterritoriality-lawo/',
+        'only_matching': True,
     }]
 
     _ACCEPT_HEADER = 'application/json, text/javascript, */*; q=0.01'

From d0fed4ac028baa40b18082fd813ed93cd658ebe8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 22 Sep 2015 22:00:50 +0600
Subject: [PATCH 2559/6219] [viewster] Use tuple

---
 youtube_dl/extractor/viewster.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/viewster.py b/youtube_dl/extractor/viewster.py
index 487f2d744..632e57fb4 100644
--- a/youtube_dl/extractor/viewster.py
+++ b/youtube_dl/extractor/viewster.py
@@ -95,7 +95,7 @@ class ViewsterIE(InfoExtractor):
         entry_id = info.get('Id') or info['id']
 
         # unfinished serie has no Type
-        if info.get('Type') in ['Serie', None]:
+        if info.get('Type') in ('Serie', None):
             try:
                 episodes = self._download_json(
                     'https://public-api.viewster.com/series/%s/episodes' % entry_id,

From de3fc356e1597b31c8f0a55a7bf7e201ae436c66 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 21 Sep 2015 14:01:12 +0100
Subject: [PATCH 2560/6219] [ninegag] fix _VALID_URL regex and handle the use
 of other external providers

---
 youtube_dl/extractor/ninegag.py | 43 ++++++++++++++++++++++-----------
 1 file changed, 29 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/ninegag.py b/youtube_dl/extractor/ninegag.py
index 7f842b5c2..eee65873a 100644
--- a/youtube_dl/extractor/ninegag.py
+++ b/youtube_dl/extractor/ninegag.py
@@ -9,17 +9,12 @@ from ..utils import str_to_int
 
 class NineGagIE(InfoExtractor):
     IE_NAME = '9gag'
-    _VALID_URL = r'''(?x)^https?://(?:www\.)?9gag\.tv/
-        (?:
-            v/(?P<numid>[0-9]+)|
-            p/(?P<id>[a-zA-Z0-9]+)/(?P<display_id>[^?#/]+)
-        )
-    '''
+    _VALID_URL = r'https?://(?:www\.)?9gag\.com/tv/p/(?P<id>[a-zA-Z0-9]+)/(?P<display_id>[^?#/]+)'
 
     _TESTS = [{
-        "url": "http://9gag.tv/v/1912",
+        "url": "http://9gag.com/tv/p/Kk2X5/people-are-awesome-2013-is-absolutely-awesome",
         "info_dict": {
-            "id": "1912",
+            "id": "Kk2X5",
             "ext": "mp4",
             "description": "This 3-minute video will make you smile and then make you feel untalented and insignificant. Anyway, you should share this awesomeness. (Thanks, Dino!)",
             "title": "\"People Are Awesome 2013\" Is Absolutely Awesome",
@@ -31,7 +26,7 @@ class NineGagIE(InfoExtractor):
         },
         'add_ie': ['Youtube']
     }, {
-        'url': 'http://9gag.tv/p/KklwM/alternate-banned-opening-scene-of-gravity?ref=fsidebar',
+        'url': 'http://9gag.com/tv/p/KklwM/alternate-banned-opening-scene-of-gravity?ref=fsidebar',
         'info_dict': {
             'id': 'KklwM',
             'ext': 'mp4',
@@ -42,19 +37,39 @@ class NineGagIE(InfoExtractor):
             'upload_date': '20140401',
             'uploader_id': 'krishnashenoi93',
         },
+        'add_ie': ['Youtube']
     }]
+    _EXTERNAL_VIDEO_PROVIDER = {
+        '1': {
+            'url': '%s',
+            'ie_key': 'Youtube',
+        },
+        '2': {
+            'url': 'http://player.vimeo.com/video/%s',
+            'ie_key': 'Vimeo',
+        },
+        '3': {
+            'url': 'http://instagram.com/p/%s',
+            'ie_key': 'Instagram',
+        },
+        '4': {
+            'url': 'http://vine.co/v/%s',
+            'ie_key': 'Vine',
+        },
+    }
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('numid') or mobj.group('id')
-        display_id = mobj.group('display_id') or video_id
+        video_id = mobj.group('id')
+        display_id = mobj.group('display_id')
 
         webpage = self._download_webpage(url, display_id)
 
         post_view = json.loads(self._html_search_regex(
             r'var postView = new app\.PostView\({\s*post:\s*({.+?}),\s*posts:\s*prefetchedCurrentPost', webpage, 'post view'))
 
-        youtube_id = post_view['videoExternalId']
+        external_video_id = post_view['videoExternalId']
+        external_video_provider = post_view['videoExternalProvider']
         title = post_view['title']
         description = post_view['description']
         view_count = str_to_int(post_view['externalView'])
@@ -62,8 +77,8 @@ class NineGagIE(InfoExtractor):
 
         return {
             '_type': 'url_transparent',
-            'url': youtube_id,
-            'ie_key': 'Youtube',
+            'url': self._EXTERNAL_VIDEO_PROVIDER[external_video_provider]['url'] % external_video_id,
+            'ie_key': self._EXTERNAL_VIDEO_PROVIDER[external_video_provider]['ie_key'],
             'id': video_id,
             'display_id': display_id,
             'title': title,

From 6b8ce312e3cb7d81e949cd5c64ad2a824d27830b Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 22 Sep 2015 19:20:18 +0100
Subject: [PATCH 2561/6219] [ninegag] extract source url

---
 youtube_dl/extractor/ninegag.py | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/ninegag.py b/youtube_dl/extractor/ninegag.py
index eee65873a..00bf98ef5 100644
--- a/youtube_dl/extractor/ninegag.py
+++ b/youtube_dl/extractor/ninegag.py
@@ -68,8 +68,13 @@ class NineGagIE(InfoExtractor):
         post_view = json.loads(self._html_search_regex(
             r'var postView = new app\.PostView\({\s*post:\s*({.+?}),\s*posts:\s*prefetchedCurrentPost', webpage, 'post view'))
 
-        external_video_id = post_view['videoExternalId']
-        external_video_provider = post_view['videoExternalProvider']
+        ie_key = None
+        source_url = post_view.get('sourceUrl')
+        if not source_url or source_url == '':
+            external_video_id = post_view['videoExternalId']
+            external_video_provider = post_view['videoExternalProvider']
+            source_url = self._EXTERNAL_VIDEO_PROVIDER[external_video_provider]['url'] % external_video_id
+            ie_key = self._EXTERNAL_VIDEO_PROVIDER[external_video_provider]['ie_key']
         title = post_view['title']
         description = post_view['description']
         view_count = str_to_int(post_view['externalView'])
@@ -77,8 +82,8 @@ class NineGagIE(InfoExtractor):
 
         return {
             '_type': 'url_transparent',
-            'url': self._EXTERNAL_VIDEO_PROVIDER[external_video_provider]['url'] % external_video_id,
-            'ie_key': self._EXTERNAL_VIDEO_PROVIDER[external_video_provider]['ie_key'],
+            'url': source_url,
+            'ie_key': ie_key,
             'id': video_id,
             'display_id': display_id,
             'title': title,

From da9f18083596d0132d12652acd0bd8983c70c058 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 22 Sep 2015 20:28:00 +0100
Subject: [PATCH 2562/6219] [ninegag] remove unnecessary condition

---
 youtube_dl/extractor/ninegag.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ninegag.py b/youtube_dl/extractor/ninegag.py
index 00bf98ef5..0a2725c65 100644
--- a/youtube_dl/extractor/ninegag.py
+++ b/youtube_dl/extractor/ninegag.py
@@ -70,7 +70,7 @@ class NineGagIE(InfoExtractor):
 
         ie_key = None
         source_url = post_view.get('sourceUrl')
-        if not source_url or source_url == '':
+        if not source_url:
             external_video_id = post_view['videoExternalId']
             external_video_provider = post_view['videoExternalProvider']
             source_url = self._EXTERNAL_VIDEO_PROVIDER[external_video_provider]['url'] % external_video_id

From e28c794699596912092635014b041d0af888fd08 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 23 Sep 2015 01:40:06 +0600
Subject: [PATCH 2563/6219] [9gag] Make display_id optional

---
 youtube_dl/extractor/ninegag.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/ninegag.py b/youtube_dl/extractor/ninegag.py
index 0a2725c65..6103c7517 100644
--- a/youtube_dl/extractor/ninegag.py
+++ b/youtube_dl/extractor/ninegag.py
@@ -9,7 +9,7 @@ from ..utils import str_to_int
 
 class NineGagIE(InfoExtractor):
     IE_NAME = '9gag'
-    _VALID_URL = r'https?://(?:www\.)?9gag\.com/tv/p/(?P<id>[a-zA-Z0-9]+)/(?P<display_id>[^?#/]+)'
+    _VALID_URL = r'https?://(?:www\.)?9gag\.com/tv/p/(?P<id>[a-zA-Z0-9]+)(?:/(?P<display_id>[^?#/]+))?'
 
     _TESTS = [{
         "url": "http://9gag.com/tv/p/Kk2X5/people-are-awesome-2013-is-absolutely-awesome",
@@ -61,7 +61,7 @@ class NineGagIE(InfoExtractor):
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
-        display_id = mobj.group('display_id')
+        display_id = mobj.group('display_id') or video_id
 
         webpage = self._download_webpage(url, display_id)
 

From c3a4e2ec40f9a90d4ae8991e6764641c744ec8fb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 23 Sep 2015 01:41:44 +0600
Subject: [PATCH 2564/6219] [9gag] Remove redundant test

---
 youtube_dl/extractor/ninegag.py | 14 ++------------
 1 file changed, 2 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/ninegag.py b/youtube_dl/extractor/ninegag.py
index 6103c7517..34879d1c8 100644
--- a/youtube_dl/extractor/ninegag.py
+++ b/youtube_dl/extractor/ninegag.py
@@ -26,18 +26,8 @@ class NineGagIE(InfoExtractor):
         },
         'add_ie': ['Youtube']
     }, {
-        'url': 'http://9gag.com/tv/p/KklwM/alternate-banned-opening-scene-of-gravity?ref=fsidebar',
-        'info_dict': {
-            'id': 'KklwM',
-            'ext': 'mp4',
-            'display_id': 'alternate-banned-opening-scene-of-gravity',
-            "description": "While Gravity was a pretty awesome movie already, YouTuber Krishna Shenoi came up with a way to improve upon it, introducing a much better solution to Sandra Bullock's seemingly endless tumble in space. The ending is priceless.",
-            'title': "Banned Opening Scene Of \"Gravity\" That Changes The Whole Movie",
-            'uploader': 'Krishna Shenoi',
-            'upload_date': '20140401',
-            'uploader_id': 'krishnashenoi93',
-        },
-        'add_ie': ['Youtube']
+        'url': 'http://9gag.com/tv/p/KklwM',
+        'only_matching': True,
     }]
     _EXTERNAL_VIDEO_PROVIDER = {
         '1': {

From 6400f8ec0f20011a7e39da62e0a3e55e0fd2759a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 23 Sep 2015 01:43:26 +0600
Subject: [PATCH 2565/6219] [9gag] Allow old .tv domain

There are still references to it in webpage's source
---
 youtube_dl/extractor/ninegag.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ninegag.py b/youtube_dl/extractor/ninegag.py
index 34879d1c8..d157b0d10 100644
--- a/youtube_dl/extractor/ninegag.py
+++ b/youtube_dl/extractor/ninegag.py
@@ -9,7 +9,7 @@ from ..utils import str_to_int
 
 class NineGagIE(InfoExtractor):
     IE_NAME = '9gag'
-    _VALID_URL = r'https?://(?:www\.)?9gag\.com/tv/p/(?P<id>[a-zA-Z0-9]+)(?:/(?P<display_id>[^?#/]+))?'
+    _VALID_URL = r'https?://(?:www\.)?9gag(?:\.com/tv|\.tv)/p/(?P<id>[a-zA-Z0-9]+)(?:/(?P<display_id>[^?#/]+))?'
 
     _TESTS = [{
         "url": "http://9gag.com/tv/p/Kk2X5/people-are-awesome-2013-is-absolutely-awesome",
@@ -28,6 +28,9 @@ class NineGagIE(InfoExtractor):
     }, {
         'url': 'http://9gag.com/tv/p/KklwM',
         'only_matching': True,
+    }, {
+        'url': 'http://9gag.tv/p/Kk2X5',
+        'only_matching': True,
     }]
     _EXTERNAL_VIDEO_PROVIDER = {
         '1': {

From 5600e214c3b63e3e2a0862bea230026c02073d7a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 23 Sep 2015 01:44:38 +0600
Subject: [PATCH 2566/6219] [9gag] Make post view regex more robust

---
 youtube_dl/extractor/ninegag.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ninegag.py b/youtube_dl/extractor/ninegag.py
index d157b0d10..692cc3368 100644
--- a/youtube_dl/extractor/ninegag.py
+++ b/youtube_dl/extractor/ninegag.py
@@ -59,7 +59,8 @@ class NineGagIE(InfoExtractor):
         webpage = self._download_webpage(url, display_id)
 
         post_view = json.loads(self._html_search_regex(
-            r'var postView = new app\.PostView\({\s*post:\s*({.+?}),\s*posts:\s*prefetchedCurrentPost', webpage, 'post view'))
+            r'var\s+postView\s*=\s*new\s+app\.PostView\({\s*post:\s*({.+?})\s*,\s*posts:\s*prefetchedCurrentPost',
+            webpage, 'post view'))
 
         ie_key = None
         source_url = post_view.get('sourceUrl')

From 8ca2e93e1ab3485946df19d56a87e56699b2a712 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 23 Sep 2015 01:46:40 +0600
Subject: [PATCH 2567/6219] [9gag] Relax optional fields

---
 youtube_dl/extractor/ninegag.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/ninegag.py b/youtube_dl/extractor/ninegag.py
index 692cc3368..6daae7318 100644
--- a/youtube_dl/extractor/ninegag.py
+++ b/youtube_dl/extractor/ninegag.py
@@ -70,8 +70,8 @@ class NineGagIE(InfoExtractor):
             source_url = self._EXTERNAL_VIDEO_PROVIDER[external_video_provider]['url'] % external_video_id
             ie_key = self._EXTERNAL_VIDEO_PROVIDER[external_video_provider]['ie_key']
         title = post_view['title']
-        description = post_view['description']
-        view_count = str_to_int(post_view['externalView'])
+        description = post_view.get('description')
+        view_count = str_to_int(post_view.get('externalView'))
         thumbnail = post_view.get('thumbnail_700w') or post_view.get('ogImageUrl') or post_view.get('thumbnail_300w')
 
         return {

From c659022b5cabebaba3275df2d0f4ae97b468eac8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 23 Sep 2015 01:48:13 +0600
Subject: [PATCH 2568/6219] [9gag] Modernize

---
 youtube_dl/extractor/ninegag.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/ninegag.py b/youtube_dl/extractor/ninegag.py
index 6daae7318..d8e103189 100644
--- a/youtube_dl/extractor/ninegag.py
+++ b/youtube_dl/extractor/ninegag.py
@@ -58,9 +58,11 @@ class NineGagIE(InfoExtractor):
 
         webpage = self._download_webpage(url, display_id)
 
-        post_view = json.loads(self._html_search_regex(
-            r'var\s+postView\s*=\s*new\s+app\.PostView\({\s*post:\s*({.+?})\s*,\s*posts:\s*prefetchedCurrentPost',
-            webpage, 'post view'))
+        post_view = self._parse_json(
+            self._search_regex(
+                r'var\s+postView\s*=\s*new\s+app\.PostView\({\s*post:\s*({.+?})\s*,\s*posts:\s*prefetchedCurrentPost',
+                webpage, 'post view'),
+            display_id)
 
         ie_key = None
         source_url = post_view.get('sourceUrl')

From 8ea6bd2802a0dbb860187bb5aaf687585ff7c1c7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 23 Sep 2015 01:55:16 +0600
Subject: [PATCH 2569/6219] [9gag] Add vimeo test

---
 youtube_dl/extractor/ninegag.py | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/ninegag.py b/youtube_dl/extractor/ninegag.py
index d8e103189..0ef6a6c65 100644
--- a/youtube_dl/extractor/ninegag.py
+++ b/youtube_dl/extractor/ninegag.py
@@ -1,7 +1,6 @@
 from __future__ import unicode_literals
 
 import re
-import json
 
 from .common import InfoExtractor
 from ..utils import str_to_int
@@ -22,9 +21,21 @@ class NineGagIE(InfoExtractor):
             'uploader': 'CompilationChannel',
             'upload_date': '20131110',
             "view_count": int,
-            "thumbnail": "re:^https?://",
         },
-        'add_ie': ['Youtube']
+        'add_ie': ['Youtube'],
+    }, {
+        'url': 'http://9gag.com/tv/p/aKolP3',
+        'info_dict': {
+            'id': 'aKolP3',
+            'ext': 'mp4',
+            'title': 'This Guy Travelled 11 countries In 44 days Just To Make This Amazing Video',
+            'description': "I just saw more in 1 minute than I've seen in 1 year. This guy's video is epic!!",
+            'uploader_id': 'rickmereki',
+            'uploader': 'Rick Mereki',
+            'upload_date': '20110803',
+            'view_count': int,
+        },
+        'add_ie': ['Vimeo'],
     }, {
         'url': 'http://9gag.com/tv/p/KklwM',
         'only_matching': True,

From d8fef8faacad4f3b9d13e20df4ee9344ada3c68d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 23 Sep 2015 01:56:26 +0600
Subject: [PATCH 2570/6219] [9gag] Quotes consistency

---
 youtube_dl/extractor/ninegag.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/ninegag.py b/youtube_dl/extractor/ninegag.py
index 0ef6a6c65..4adff197e 100644
--- a/youtube_dl/extractor/ninegag.py
+++ b/youtube_dl/extractor/ninegag.py
@@ -11,16 +11,16 @@ class NineGagIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?9gag(?:\.com/tv|\.tv)/p/(?P<id>[a-zA-Z0-9]+)(?:/(?P<display_id>[^?#/]+))?'
 
     _TESTS = [{
-        "url": "http://9gag.com/tv/p/Kk2X5/people-are-awesome-2013-is-absolutely-awesome",
-        "info_dict": {
-            "id": "Kk2X5",
-            "ext": "mp4",
-            "description": "This 3-minute video will make you smile and then make you feel untalented and insignificant. Anyway, you should share this awesomeness. (Thanks, Dino!)",
-            "title": "\"People Are Awesome 2013\" Is Absolutely Awesome",
+        'url': 'http://9gag.com/tv/p/Kk2X5/people-are-awesome-2013-is-absolutely-awesome',
+        'info_dict': {
+            'id': 'Kk2X5',
+            'ext': 'mp4',
+            'description': 'This 3-minute video will make you smile and then make you feel untalented and insignificant. Anyway, you should share this awesomeness. (Thanks, Dino!)',
+            'title': '\"People Are Awesome 2013\" Is Absolutely Awesome',
             'uploader_id': 'UCdEH6EjDKwtTe-sO2f0_1XA',
             'uploader': 'CompilationChannel',
             'upload_date': '20131110',
-            "view_count": int,
+            'view_count': int,
         },
         'add_ie': ['Youtube'],
     }, {

From 78f9fb902b36c2a12dfe50d9724ac470b03c87d5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 23 Sep 2015 01:58:47 +0600
Subject: [PATCH 2571/6219] [9gag] Support embed URLs

---
 youtube_dl/extractor/ninegag.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ninegag.py b/youtube_dl/extractor/ninegag.py
index 4adff197e..a06d38afd 100644
--- a/youtube_dl/extractor/ninegag.py
+++ b/youtube_dl/extractor/ninegag.py
@@ -8,7 +8,7 @@ from ..utils import str_to_int
 
 class NineGagIE(InfoExtractor):
     IE_NAME = '9gag'
-    _VALID_URL = r'https?://(?:www\.)?9gag(?:\.com/tv|\.tv)/p/(?P<id>[a-zA-Z0-9]+)(?:/(?P<display_id>[^?#/]+))?'
+    _VALID_URL = r'https?://(?:www\.)?9gag(?:\.com/tv|\.tv)/(?:p|embed)/(?P<id>[a-zA-Z0-9]+)(?:/(?P<display_id>[^?#/]+))?'
 
     _TESTS = [{
         'url': 'http://9gag.com/tv/p/Kk2X5/people-are-awesome-2013-is-absolutely-awesome',
@@ -42,7 +42,11 @@ class NineGagIE(InfoExtractor):
     }, {
         'url': 'http://9gag.tv/p/Kk2X5',
         'only_matching': True,
+    }, {
+        'url': 'http://9gag.com/tv/embed/a5Dmvl',
+        'only_matching': True,
     }]
+
     _EXTERNAL_VIDEO_PROVIDER = {
         '1': {
             'url': '%s',

From b942db3dc3d51db0f24ac98ada861d8e2b3451db Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 22 Sep 2015 22:41:53 +0200
Subject: [PATCH 2572/6219] release 2015.09.22

---
 docs/supportedsites.md | 17 ++++++++++-------
 youtube_dl/version.py  |  2 +-
 2 files changed, 11 insertions(+), 8 deletions(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 66091e6be..ab153af6b 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -122,7 +122,6 @@
  - **defense.gouv.fr**
  - **DHM**: Filmarchiv - Deutsches Historisches Museum
  - **Discovery**
- - **divxstage**: DivxStage
  - **Dotsub**
  - **DouyuTV**: 斗鱼
  - **dramafever**
@@ -286,7 +285,7 @@
  - **Minhateca**
  - **MinistryGrid**
  - **miomio.tv**
- - **mitele.es**
+ - **MiTele**: mitele.es
  - **mixcloud**
  - **MLB**
  - **MoeVideo**: LetitBit video services: moevideo.net, playreplay.net and videochart.net
@@ -317,7 +316,6 @@
  - **Myvi**
  - **myvideo**
  - **MyVidster**
- - **N-JOY**
  - **n-tv.de**
  - **NationalGeographic**
  - **Naver**
@@ -326,7 +324,9 @@
  - **NBCNews**
  - **NBCSports**
  - **NBCSportsVPlayer**
- - **ndr**: NDR.de - Mediathek
+ - **ndr**: NDR.de - Norddeutscher Rundfunk
+ - **ndr:embed**
+ - **ndr:embed:base**
  - **NDTV**
  - **NerdCubedFeed**
  - **Nerdist**
@@ -349,12 +349,16 @@
  - **nhl.com:videocenter**: NHL videocenter category
  - **niconico**: ニコニコ動画
  - **NiconicoPlaylist**
+ - **njoy**: N-JOY
+ - **njoy:embed**
  - **Noco**
  - **Normalboots**
  - **NosVideo**
  - **Nova**: TN.cz, Prásk.tv, Nova.cz, Novaplus.cz, FANDA.tv, Krásná.cz and Doma.cz
  - **novamov**: NovaMov
- - **Nowness**
+ - **nowness**
+ - **nowness:playlist**
+ - **nowness:series**
  - **NowTV**
  - **nowvideo**: NowVideo
  - **npo**: npo.nl and ntr.nl
@@ -375,7 +379,6 @@
  - **OnionStudios**
  - **Ooyala**
  - **OoyalaExternal**
- - **OpenFilm**
  - **orf:fm4**: radio FM4
  - **orf:iptv**: iptv.ORF.at
  - **orf:oe1**: Radio Österreich 1
@@ -530,7 +533,7 @@
  - **techtv.mit.edu**
  - **ted**
  - **TeleBruxelles**
- - **telecinco.es**
+ - **Telecinco**: telecinco.es, cuatro.com and mediaset.es
  - **Telegraaf**
  - **TeleMB**
  - **TeleTask**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 0cc7411f2..7ef4f2755 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.09.09'
+__version__ = '2015.09.22'

From f1028194636e7acafb29fb38244cc7b1347d9313 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 23 Sep 2015 02:46:24 +0600
Subject: [PATCH 2573/6219] [downloader/hls] Pass http headers to downloader

---
 youtube_dl/downloader/hls.py | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index 7743e176a..a62d2047b 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -28,9 +28,18 @@ class HlsFD(FileDownloader):
             return False
         ffpp.check_version()
 
-        args = [
-            encodeArgument(opt)
-            for opt in (ffpp.executable, '-y', '-i', url, '-f', 'mp4', '-c', 'copy', '-bsf:a', 'aac_adtstoasc')]
+        args = [ffpp.executable, '-y']
+
+        if info_dict['http_headers']:
+            # Trailing \r\n after each HTTP header is important to prevent warning from ffmpeg/avconv:
+            # [http @ 00000000003d2fa0] No trailing CRLF found in HTTP header.
+            args += [
+                '-headers',
+                ''.join('%s: %s\r\n' % (key, val) for key, val in info_dict['http_headers'].items())]
+
+        args += ['-i', url, '-f', 'mp4', '-c', 'copy', '-bsf:a', 'aac_adtstoasc']
+
+        args = [encodeArgument(opt) for opt in args]
         args.append(encodeFilename(ffpp._ffmpeg_filename_argument(tmpfilename), True))
 
         self._debug_cmd(args)

From b306c439d7f2997ebf2a88385c73fe2d92227b76 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 23 Sep 2015 13:28:05 +0100
Subject: [PATCH 2574/6219] [cnet] fix extraction and extract more formats

---
 youtube_dl/extractor/cnet.py | 54 +++++++++++++++---------------------
 1 file changed, 23 insertions(+), 31 deletions(-)

diff --git a/youtube_dl/extractor/cnet.py b/youtube_dl/extractor/cnet.py
index 5dd69bff7..2fac0d79d 100644
--- a/youtube_dl/extractor/cnet.py
+++ b/youtube_dl/extractor/cnet.py
@@ -4,9 +4,7 @@ from __future__ import unicode_literals
 import json
 
 from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
-)
+from .theplatform import ThePlatformIE
 
 
 class CNETIE(InfoExtractor):
@@ -15,29 +13,22 @@ class CNETIE(InfoExtractor):
         'url': 'http://www.cnet.com/videos/hands-on-with-microsofts-windows-8-1-update/',
         'info_dict': {
             'id': '56f4ea68-bd21-4852-b08c-4de5b8354c60',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Hands-on with Microsoft Windows 8.1 Update',
             'description': 'The new update to the Windows 8 OS brings improved performance for mouse and keyboard users.',
-            'thumbnail': 're:^http://.*/flmswindows8.jpg$',
             'uploader_id': '6085384d-619e-11e3-b231-14feb5ca9861',
             'uploader': 'Sarah Mitroff',
         },
-        'params': {
-            'skip_download': 'requires rtmpdump',
-        }
     }, {
         'url': 'http://www.cnet.com/videos/whiny-pothole-tweets-at-local-government-when-hit-by-cars-tomorrow-daily-187/',
         'info_dict': {
             'id': '56527b93-d25d-44e3-b738-f989ce2e49ba',
-            'ext': 'flv',
+            'ext': 'mp4',
             'description': 'Khail and Ashley wonder what other civic woes can be solved by self-tweeting objects, investigate a new kind of VR camera and watch an origami robot self-assemble, walk, climb, dig and dissolve. #TDPothole',
             'uploader_id': 'b163284d-6b73-44fc-b3e6-3da66c392d40',
             'uploader': 'Ashley Esqueda',
             'title': 'Whiny potholes tweet at local government when hit by cars (Tomorrow Daily 187)',
         },
-        'params': {
-            'skip_download': True,  # requires rtmpdump
-        },
     }]
 
     def _real_extract(self, url):
@@ -45,26 +36,13 @@ class CNETIE(InfoExtractor):
         webpage = self._download_webpage(url, display_id)
 
         data_json = self._html_search_regex(
-            r"<div class=\"cnetVideoPlayer\"\s+.*?data-cnet-video-options='([^']+)'",
+            r"<div class=\"videoPlayer\"\s+.*?data-cnet-video-uvp-options='([^']+)'",
             webpage, 'data json')
         data = json.loads(data_json)
-        vdata = data['video']
-        if not vdata:
-            vdata = data['videos'][0]
-        if not vdata:
-            raise ExtractorError('Cannot find video data')
-
-        mpx_account = data['config']['players']['default']['mpx_account']
-        vid = vdata['files'].get('rtmp', vdata['files']['hds'])
-        tp_link = 'http://link.theplatform.com/s/%s/%s' % (mpx_account, vid)
+        vdata = data['videos'][0]
 
         video_id = vdata['id']
-        title = vdata.get('headline')
-        if title is None:
-            title = vdata.get('title')
-        if title is None:
-            raise ExtractorError('Cannot find title!')
-        thumbnail = vdata.get('image', {}).get('path')
+        title = vdata['title']
         author = vdata.get('author')
         if author:
             uploader = '%s %s' % (author['firstName'], author['lastName'])
@@ -73,13 +51,27 @@ class CNETIE(InfoExtractor):
             uploader = None
             uploader_id = None
 
+        mpx_account = data['config']['uvpConfig']['default']['mpx_account']
+        tp = ThePlatformIE(self._downloader)
+        formats = []
+        subtitles = {}
+        description = vdata.get('description')
+
+        for vid in vdata['files'].values():
+            result = tp.extract(('http://link.theplatform.com/s/%s/%s' % (mpx_account, vid)))
+            formats.extend(result['formats'])
+            subtitles = self._merge_subtitles(subtitles, result['subtitles'])
+            description = description or result.get('description')
+
+        self._sort_formats(formats)
+
         return {
-            '_type': 'url_transparent',
-            'url': tp_link,
             'id': video_id,
             'display_id': display_id,
             'title': title,
+            'description': description,
             'uploader': uploader,
             'uploader_id': uploader_id,
-            'thumbnail': thumbnail,
+            'subtitles': subtitles,
+            'formats': formats,
         }

From 53407e3f383ed80c67db9e06b8c3480257aa3184 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 23 Sep 2015 14:02:13 +0100
Subject: [PATCH 2575/6219] [brightcove] fix streaming_src extraction

---
 youtube_dl/extractor/brightcove.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index a07c0888f..e4a7befee 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -413,7 +413,7 @@ class BrightcoveInPageEmbedIE(InfoExtractor):
             if source_type == 'application/x-mpegURL':
                 formats.extend(self._extract_m3u8_formats(source.get('src'), video_id))
             else:
-                src = source.get('src')
+                src = source.get('src') or source.get('streaming_src')
                 if src:
                     formats.append({
                         'url': src,
@@ -424,8 +424,6 @@ class BrightcoveInPageEmbedIE(InfoExtractor):
                         'container': source.get('container'),
                         'vcodec': source.get('container'),
                     })
-                else:
-                    formats.extend(self._extract_f4m_formats(source.get('streaming_src'), video_id))
 
         self._sort_formats(formats)
 

From eb11cbe8674705647d6bd2947a44e08543663633 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 23 Sep 2015 19:54:40 +0600
Subject: [PATCH 2576/6219] [soundcloud] Update client id (Closes #6930)

---
 youtube_dl/extractor/soundcloud.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index ed5dcc0d3..2b60d354a 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -113,7 +113,7 @@ class SoundcloudIE(InfoExtractor):
         },
     ]
 
-    _CLIENT_ID = 'b45b1aa10f1ac2941910a7f0d10f8e28'
+    _CLIENT_ID = '02gUJC0hH2ct1EGOcYXQIzRFU91c72Ea'
     _IPHONE_CLIENT_ID = '376f225bf427445fc4bfb6b99b72e0bf'
 
     def report_resolve(self, video_id):

From 57565375c85ff6adb11beb961fd61d6bdd023ec1 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 23 Sep 2015 22:22:04 +0800
Subject: [PATCH 2577/6219] [iqiyi] Fix extraction (fixes #6878)

---
 youtube_dl/extractor/iqiyi.py | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index 393e67e35..2ce6627a4 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -95,6 +95,10 @@ class IqiyiIE(InfoExtractor):
         ('10', 'h1'),
     ]
 
+    @staticmethod
+    def md5_text(text):
+        return hashlib.md5(text.encode('utf-8')).hexdigest()
+
     def construct_video_urls(self, data, video_id, _uuid):
         def do_xor(x, y):
             a = y % 3
@@ -179,6 +183,7 @@ class IqiyiIE(InfoExtractor):
 
     def get_raw_data(self, tvid, video_id, enc_key, _uuid):
         tm = str(int(time.time()))
+        tail = tm + tvid
         param = {
             'key': 'fvip',
             'src': hashlib.md5(b'youtube-dl').hexdigest(),
@@ -186,13 +191,11 @@ class IqiyiIE(InfoExtractor):
             'vid': video_id,
             'vinfo': 1,
             'tm': tm,
-            'enc': hashlib.md5(
-                (enc_key + tm + tvid).encode('utf8')).hexdigest(),
+            'enc': self.md5_text((enc_key + tail)[1:64:2] + tail),
             'qyid': _uuid,
             'tn': random.random(),
             'um': 0,
-            'authkey': hashlib.md5(
-                (tm + tvid).encode('utf8')).hexdigest()
+            'authkey': self.md5_text(self.md5_text('') + tail),
         }
 
         api_url = 'http://cache.video.qiyi.com/vms' + '?' + \
@@ -201,7 +204,8 @@ class IqiyiIE(InfoExtractor):
         return raw_data
 
     def get_enc_key(self, swf_url, video_id):
-        enc_key = '3601ba290e4f4662848c710e2122007e'  # last update at 2015-08-10 for Zombie
+        # TODO: automatic key extraction
+        enc_key = 'eac64f22daf001da6ba9aa8da4d501508bbe90a4d4091fea3b0582a85b38c2cc'  # last update at 2015-09-23-23 for Zombie::bite
         return enc_key
 
     def _real_extract(self, url):

From 19f93d906e29e9a505d4bf5d286d75224c342c37 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 23 Sep 2015 22:25:16 +0800
Subject: [PATCH 2578/6219] [iqiyi] Use md5_text for all MD5 calls

---
 youtube_dl/extractor/iqiyi.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index 2ce6627a4..ce1ab3820 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -125,7 +125,7 @@ class IqiyiIE(InfoExtractor):
                 note='Download path key of segment %d for format %s' % (segment_index + 1, format_id)
             )['t']
             t = str(int(math.floor(int(tm) / (600.0))))
-            return hashlib.md5((t + mg + x).encode('utf8')).hexdigest()
+            return self.md5_text(t + mg + x)
 
         video_urls_dict = {}
         for format_item in data['vp']['tkl'][0]['vs']:
@@ -186,7 +186,7 @@ class IqiyiIE(InfoExtractor):
         tail = tm + tvid
         param = {
             'key': 'fvip',
-            'src': hashlib.md5(b'youtube-dl').hexdigest(),
+            'src': self.md5_text('youtube-dl'),
             'tvId': tvid,
             'vid': video_id,
             'vinfo': 1,

From 4395ca2e04f38c110259270fa4dfc4d9814aa926 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 24 Sep 2015 19:56:54 +0600
Subject: [PATCH 2579/6219] [xhamster] Fix title extraction (Closes #6944)

---
 youtube_dl/extractor/xhamster.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index 97315750f..f12fe13b1 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -63,7 +63,9 @@ class XHamsterIE(InfoExtractor):
         mrss_url = '%s://xhamster.com/movies/%s/%s.html' % (proto, video_id, seo)
         webpage = self._download_webpage(mrss_url, video_id)
 
-        title = self._html_search_regex(r'<title>(?P<title>.+?) - xHamster\.com</title>', webpage, 'title')
+        title = self._html_search_regex(
+            [r'<title>(?P<title>.+?)(?:, Free Porn: xHamster| - xHamster\.com)</title>',
+             r'<h1>([^<]+)</h1>'], webpage, 'title')
 
         # Only a few videos have an description
         mobj = re.search(r'<span>Description: </span>([^<]+)', webpage)

From 05b476a27087a45b8d418607123d8db62bf1770f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 24 Sep 2015 23:38:53 +0600
Subject: [PATCH 2580/6219] [vidme] Prefer non clip (Closes #6924)

---
 youtube_dl/extractor/vidme.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/vidme.py b/youtube_dl/extractor/vidme.py
index 9a794e609..078d283b2 100644
--- a/youtube_dl/extractor/vidme.py
+++ b/youtube_dl/extractor/vidme.py
@@ -119,6 +119,7 @@ class VidmeIE(InfoExtractor):
             'url': f['uri'],
             'width': int_or_none(f.get('width')),
             'height': int_or_none(f.get('height')),
+            'preference': 0 if f.get('type', '').endswith('clip') else 1,
         } for f in video.get('formats', []) if f.get('uri')]
         self._sort_formats(formats)
 

From 9fbd4b35a27a83055c3e170ab32f2b3e56f9616e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 24 Sep 2015 23:48:23 +0600
Subject: [PATCH 2581/6219] [nhl] Add support for embedded URLs

---
 youtube_dl/extractor/nhl.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nhl.py b/youtube_dl/extractor/nhl.py
index 279b18386..970a11f7c 100644
--- a/youtube_dl/extractor/nhl.py
+++ b/youtube_dl/extractor/nhl.py
@@ -72,7 +72,7 @@ class NHLBaseInfoExtractor(InfoExtractor):
 
 class NHLIE(NHLBaseInfoExtractor):
     IE_NAME = 'nhl.com'
-    _VALID_URL = r'https?://video(?P<team>\.[^.]*)?\.nhl\.com/videocenter/(?:console)?(?:\?(?:.*?[?&])?)(?:id|hlg)=(?P<id>[-0-9a-zA-Z,]+)'
+    _VALID_URL = r'https?://video(?P<team>\.[^.]*)?\.nhl\.com/videocenter/(?:console|embed)?(?:\?(?:.*?[?&])?)(?:id|hlg|playlist)=(?P<id>[-0-9a-zA-Z,]+)'
 
     _TESTS = [{
         'url': 'http://video.canucks.nhl.com/videocenter/console?catid=6?id=453614',
@@ -136,6 +136,9 @@ class NHLIE(NHLBaseInfoExtractor):
         'params': {
             'skip_download': True,  # Requires rtmpdump
         }
+    }, {
+        'url': 'http://video.nhl.com/videocenter/embed?playlist=836127',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 9c58885c70af75655288933220a99b4c4215ab4b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 24 Sep 2015 23:54:16 +0600
Subject: [PATCH 2582/6219] [nhl:news] Add support for iframe embeds (Closes
 #6941)

---
 youtube_dl/extractor/nhl.py | 21 +++++++++++++++++----
 1 file changed, 17 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/nhl.py b/youtube_dl/extractor/nhl.py
index 970a11f7c..e98a5ef89 100644
--- a/youtube_dl/extractor/nhl.py
+++ b/youtube_dl/extractor/nhl.py
@@ -149,9 +149,9 @@ class NHLIE(NHLBaseInfoExtractor):
 class NHLNewsIE(NHLBaseInfoExtractor):
     IE_NAME = 'nhl.com:news'
     IE_DESC = 'NHL news'
-    _VALID_URL = r'https?://(?:www\.)?nhl\.com/ice/news\.html?(?:\?(?:.*?[?&])?)id=(?P<id>[-0-9a-zA-Z]+)'
+    _VALID_URL = r'https?://(?:.+?\.)?nhl\.com/(?:ice|club)/news\.html?(?:\?(?:.*?[?&])?)id=(?P<id>[-0-9a-zA-Z]+)'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.nhl.com/ice/news.htm?id=750727',
         'md5': '4b3d1262e177687a3009937bd9ec0be8',
         'info_dict': {
@@ -162,13 +162,26 @@ class NHLNewsIE(NHLBaseInfoExtractor):
             'duration': 37,
             'upload_date': '20150128',
         },
-    }
+    }, {
+        # iframe embed
+        'url': 'http://sabres.nhl.com/club/news.htm?id=780189',
+        'md5': '9f663d1c006c90ac9fb82777d4294e12',
+        'info_dict': {
+            'id': '836127',
+            'ext': 'mp4',
+            'title': 'Morning Skate: OTT vs. BUF (9/23/15)',
+            'description': "Brian Duff chats with Tyler Ennis prior to Buffalo's first preseason home game.",
+            'duration': 93,
+            'upload_date': '20150923',
+        },
+    }]
 
     def _real_extract(self, url):
         news_id = self._match_id(url)
         webpage = self._download_webpage(url, news_id)
         video_id = self._search_regex(
-            [r'pVid(\d+)', r"nlid\s*:\s*'(\d+)'"],
+            [r'pVid(\d+)', r"nlid\s*:\s*'(\d+)'",
+             r'<iframe[^>]+src=["\']https?://video.*?\.nhl\.com/videocenter/embed\?.*\bplaylist=(\d+)'],
             webpage, 'video id')
         return self._real_extract_video(video_id)
 

From 47024eb564fd4047e362680f0a68304d1df79495 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 24 Sep 2015 19:49:10 +0100
Subject: [PATCH 2583/6219] [hostingbulk] remove extractor

---
 youtube_dl/extractor/__init__.py    |  1 -
 youtube_dl/extractor/hostingbulk.py | 80 -----------------------------
 2 files changed, 81 deletions(-)
 delete mode 100644 youtube_dl/extractor/hostingbulk.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index c16e9e3d2..7272859db 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -228,7 +228,6 @@ from .historicfilms import HistoricFilmsIE
 from .history import HistoryIE
 from .hitbox import HitboxIE, HitboxLiveIE
 from .hornbunny import HornBunnyIE
-from .hostingbulk import HostingBulkIE
 from .hotnewhiphop import HotNewHipHopIE
 from .howcast import HowcastIE
 from .howstuffworks import HowStuffWorksIE
diff --git a/youtube_dl/extractor/hostingbulk.py b/youtube_dl/extractor/hostingbulk.py
deleted file mode 100644
index a3154cfde..000000000
--- a/youtube_dl/extractor/hostingbulk.py
+++ /dev/null
@@ -1,80 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-import re
-
-from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_request,
-)
-from ..utils import (
-    ExtractorError,
-    int_or_none,
-    urlencode_postdata,
-)
-
-
-class HostingBulkIE(InfoExtractor):
-    _VALID_URL = r'''(?x)
-        https?://(?:www\.)?hostingbulk\.com/
-        (?:embed-)?(?P<id>[A-Za-z0-9]{12})(?:-\d+x\d+)?\.html'''
-    _FILE_DELETED_REGEX = r'<b>File Not Found</b>'
-    _TEST = {
-        'url': 'http://hostingbulk.com/n0ulw1hv20fm.html',
-        'md5': '6c8653c8ecf7ebfa83b76e24b7b2fe3f',
-        'info_dict': {
-            'id': 'n0ulw1hv20fm',
-            'ext': 'mp4',
-            'title': 'md5:5afeba33f48ec87219c269e054afd622',
-            'filesize': 6816081,
-            'thumbnail': 're:^http://.*\.jpg$',
-        }
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        url = 'http://hostingbulk.com/{0:}.html'.format(video_id)
-
-        # Custom request with cookie to set language to English, so our file
-        # deleted regex would work.
-        request = compat_urllib_request.Request(
-            url, headers={'Cookie': 'lang=english'})
-        webpage = self._download_webpage(request, video_id)
-
-        if re.search(self._FILE_DELETED_REGEX, webpage) is not None:
-            raise ExtractorError('Video %s does not exist' % video_id,
-                                 expected=True)
-
-        title = self._html_search_regex(r'<h3>(.*?)</h3>', webpage, 'title')
-        filesize = int_or_none(
-            self._search_regex(
-                r'<small>\((\d+)\sbytes?\)</small>',
-                webpage,
-                'filesize',
-                fatal=False
-            )
-        )
-        thumbnail = self._search_regex(
-            r'<img src="([^"]+)".+?class="pic"',
-            webpage, 'thumbnail', fatal=False)
-
-        fields = self._hidden_inputs(webpage)
-
-        request = compat_urllib_request.Request(url, urlencode_postdata(fields))
-        request.add_header('Content-type', 'application/x-www-form-urlencoded')
-        response = self._request_webpage(request, video_id,
-                                         'Submiting download request')
-        video_url = response.geturl()
-
-        formats = [{
-            'format_id': 'sd',
-            'filesize': filesize,
-            'url': video_url,
-        }]
-
-        return {
-            'id': video_id,
-            'title': title,
-            'thumbnail': thumbnail,
-            'formats': formats,
-        }

From 545a23f11bbdf27d6d204225c578512af7184e0b Mon Sep 17 00:00:00 2001
From: atomic83 <atomic83@riseup.net>
Date: Thu, 24 Sep 2015 23:05:32 +0200
Subject: [PATCH 2584/6219] More title extraction fixing.

---
 youtube_dl/extractor/xhamster.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index f12fe13b1..8938c0e45 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -64,7 +64,7 @@ class XHamsterIE(InfoExtractor):
         webpage = self._download_webpage(mrss_url, video_id)
 
         title = self._html_search_regex(
-            [r'<title>(?P<title>.+?)(?:, Free Porn: xHamster| - xHamster\.com)</title>',
+            [r'<title>(?P<title>.+?)(?:, (?:[^,]+? )?Porn: xHamster| - xHamster\.com)</title>',
              r'<h1>([^<]+)</h1>'], webpage, 'title')
 
         # Only a few videos have an description

From 857421024daf810e92036149cc02bcf1c337da5c Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 24 Sep 2015 21:55:44 +0100
Subject: [PATCH 2585/6219] [iconosquare] fix info extraction

---
 youtube_dl/extractor/iconosquare.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/iconosquare.py b/youtube_dl/extractor/iconosquare.py
index 70e4c0d41..4fff8c0b3 100644
--- a/youtube_dl/extractor/iconosquare.py
+++ b/youtube_dl/extractor/iconosquare.py
@@ -1,7 +1,10 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..utils import int_or_none
+from ..utils import (
+    int_or_none,
+    get_element_by_id,
+)
 
 
 class IconosquareIE(InfoExtractor):
@@ -12,7 +15,7 @@ class IconosquareIE(InfoExtractor):
         'info_dict': {
             'id': '522207370455279102_24101272',
             'ext': 'mp4',
-            'title': 'Instagram media by @aguynamedpatrick (Patrick Janelle)',
+            'title': 'A little over a year ago, I posted my first #dailycortado, a drink introduced to...',
             'description': 'md5:644406a9ec27457ed7aa7a9ebcd4ce3d',
             'timestamp': 1376471991,
             'upload_date': '20130814',
@@ -29,8 +32,7 @@ class IconosquareIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         media = self._parse_json(
-            self._search_regex(
-                r'window\.media\s*=\s*({.+?});\n', webpage, 'media'),
+            get_element_by_id('mediaJson', webpage),
             video_id)
 
         formats = [{
@@ -42,7 +44,7 @@ class IconosquareIE(InfoExtractor):
         self._sort_formats(formats)
 
         title = self._html_search_regex(
-            r'<title>(.+?)(?: *\(Videos?\))? \| (?:Iconosquare|Statigram)</title>',
+            r'<title>(.+?)</title>',
             webpage, 'title')
 
         timestamp = int_or_none(media.get('created_time') or media.get('caption', {}).get('created_time'))

From d4364f30bd39528b4da487799380737c330e88c5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 25 Sep 2015 04:44:52 +0600
Subject: [PATCH 2586/6219] [iconosquare] Revert title (Closes #6954)

---
 youtube_dl/extractor/iconosquare.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/iconosquare.py b/youtube_dl/extractor/iconosquare.py
index 4fff8c0b3..bb69c7a0f 100644
--- a/youtube_dl/extractor/iconosquare.py
+++ b/youtube_dl/extractor/iconosquare.py
@@ -4,6 +4,7 @@ from .common import InfoExtractor
 from ..utils import (
     int_or_none,
     get_element_by_id,
+    remove_end,
 )
 
 
@@ -15,7 +16,7 @@ class IconosquareIE(InfoExtractor):
         'info_dict': {
             'id': '522207370455279102_24101272',
             'ext': 'mp4',
-            'title': 'A little over a year ago, I posted my first #dailycortado, a drink introduced to...',
+            'title': 'Instagram photo by @aguynamedpatrick (Patrick Janelle)',
             'description': 'md5:644406a9ec27457ed7aa7a9ebcd4ce3d',
             'timestamp': 1376471991,
             'upload_date': '20130814',
@@ -43,9 +44,7 @@ class IconosquareIE(InfoExtractor):
         } for format_id, f in media['videos'].items()]
         self._sort_formats(formats)
 
-        title = self._html_search_regex(
-            r'<title>(.+?)</title>',
-            webpage, 'title')
+        title = remove_end(self._og_search_title(webpage), ' - via Iconosquare')
 
         timestamp = int_or_none(media.get('created_time') or media.get('caption', {}).get('created_time'))
         description = media.get('caption', {}).get('text')

From 9b166fc1f8039fe3f3632c40848ce590ade9f3fa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 25 Sep 2015 04:45:31 +0600
Subject: [PATCH 2587/6219] [iconosquare] Extract comments

---
 youtube_dl/extractor/iconosquare.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/youtube_dl/extractor/iconosquare.py b/youtube_dl/extractor/iconosquare.py
index bb69c7a0f..a39f422e9 100644
--- a/youtube_dl/extractor/iconosquare.py
+++ b/youtube_dl/extractor/iconosquare.py
@@ -62,6 +62,14 @@ class IconosquareIE(InfoExtractor):
             'height': int_or_none(t.get('height'))
         } for thumbnail_id, t in media.get('images', {}).items()]
 
+        comments = [{
+            'id': comment.get('id'),
+            'text': comment['text'],
+            'timestamp': int_or_none(comment.get('created_time')),
+            'author': comment.get('from', {}).get('full_name'),
+            'author_id': comment.get('from', {}).get('username'),
+        } for comment in media.get('comments', {}).get('data', []) if 'text' in comment]
+
         return {
             'id': video_id,
             'title': title,
@@ -73,4 +81,5 @@ class IconosquareIE(InfoExtractor):
             'comment_count': comment_count,
             'like_count': like_count,
             'formats': formats,
+            'comments': comments,
         }

From 882fc9052e310b5ac6675488bba767c43ca2185e Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 23 Sep 2015 17:48:39 +0100
Subject: [PATCH 2588/6219] [condenast] fix extraction and add support for
 other sites

---
 youtube_dl/extractor/condenast.py | 29 ++++++++++++++++++++---------
 1 file changed, 20 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/condenast.py b/youtube_dl/extractor/condenast.py
index 3db4db4e4..22c66da26 100644
--- a/youtube_dl/extractor/condenast.py
+++ b/youtube_dl/extractor/condenast.py
@@ -2,7 +2,6 @@
 from __future__ import unicode_literals
 
 import re
-import json
 
 from .common import InfoExtractor
 from ..compat import (
@@ -24,13 +23,25 @@ class CondeNastIE(InfoExtractor):
     # The keys are the supported sites and the values are the name to be shown
     # to the user and in the extractor description.
     _SITES = {
-        'wired': 'WIRED',
-        'gq': 'GQ',
-        'vogue': 'Vogue',
-        'glamour': 'Glamour',
-        'wmagazine': 'W Magazine',
-        'vanityfair': 'Vanity Fair',
+        'allure': 'Allure',
+        'architecturaldigest': 'Architectural Digest',
+        'arstechnica': 'Ars Technica',
+        'bonappetit': 'Bon Appetit',
+        'brides': 'Brides',
         'cnevids': 'Condé Nast',
+        'cntraveler': 'Condé Nast Traveler',
+        'details': 'Details',
+        'epicurious': 'Epicurious',
+        'glamour': 'Glamour',
+        'golfdigest': 'Golf Digest',
+        'gq': 'GQ',
+        'newyorker': 'The New Yorker',
+        'self': 'SELF',
+        'teenvogue': 'Teen Vogue',
+        'vanityfair': 'Vanity Fair',
+        'vogue': 'Vogue',
+        'wired': 'WIRED',
+        'wmagazine': 'W Magazine',
     }
 
     _VALID_URL = r'http://(video|www|player)\.(?P<site>%s)\.com/(?P<type>watch|series|video|embed)/(?P<id>[^/?#]+)' % '|'.join(_SITES.keys())
@@ -86,8 +97,8 @@ class CondeNastIE(InfoExtractor):
         info_url = base_info_url + data
         info_page = self._download_webpage(info_url, video_id,
                                            'Downloading video info')
-        video_info = self._search_regex(r'var video = ({.+?});', info_page, 'video info')
-        video_info = json.loads(video_info)
+        video_info = self._search_regex(r'var\s*video\s*=\s*({.+?});', info_page, 'video info')
+        video_info = self._parse_json(video_info, video_id)
 
         formats = [{
             'format_id': '%s-%s' % (fdata['type'].split('/')[-1], fdata['quality']),

From 2949a6cda9c39b3ff32891bdf0d6b48c46973f82 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 24 Sep 2015 15:54:23 +0100
Subject: [PATCH 2589/6219] [condenast] fix video info regex

---
 youtube_dl/extractor/condenast.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/condenast.py b/youtube_dl/extractor/condenast.py
index 22c66da26..d1380ac8d 100644
--- a/youtube_dl/extractor/condenast.py
+++ b/youtube_dl/extractor/condenast.py
@@ -97,7 +97,7 @@ class CondeNastIE(InfoExtractor):
         info_url = base_info_url + data
         info_page = self._download_webpage(info_url, video_id,
                                            'Downloading video info')
-        video_info = self._search_regex(r'var\s*video\s*=\s*({.+?});', info_page, 'video info')
+        video_info = self._search_regex(r'var\s+video\s*=\s*({.+?});', info_page, 'video info')
         video_info = self._parse_json(video_info, video_id)
 
         formats = [{

From 42ca72dff38c6cb23724dd91b39550e805bd8d25 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 25 Sep 2015 05:15:21 +0600
Subject: [PATCH 2590/6219] [condenast] Keep acute accent

---
 youtube_dl/extractor/condenast.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/condenast.py b/youtube_dl/extractor/condenast.py
index d1380ac8d..ef1d28091 100644
--- a/youtube_dl/extractor/condenast.py
+++ b/youtube_dl/extractor/condenast.py
@@ -26,7 +26,7 @@ class CondeNastIE(InfoExtractor):
         'allure': 'Allure',
         'architecturaldigest': 'Architectural Digest',
         'arstechnica': 'Ars Technica',
-        'bonappetit': 'Bon Appetit',
+        'bonappetit': 'Bon Appétit',
         'brides': 'Brides',
         'cnevids': 'Condé Nast',
         'cntraveler': 'Condé Nast Traveler',

From 0940c5b4c6a068d4919fd29a8db2a85ab3bbf703 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 25 Sep 2015 05:18:45 +0600
Subject: [PATCH 2591/6219] [condenast] Do not capture unused group in
 _VALID_URL

---
 youtube_dl/extractor/condenast.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/condenast.py b/youtube_dl/extractor/condenast.py
index ef1d28091..d6949ca28 100644
--- a/youtube_dl/extractor/condenast.py
+++ b/youtube_dl/extractor/condenast.py
@@ -44,7 +44,7 @@ class CondeNastIE(InfoExtractor):
         'wmagazine': 'W Magazine',
     }
 
-    _VALID_URL = r'http://(video|www|player)\.(?P<site>%s)\.com/(?P<type>watch|series|video|embed)/(?P<id>[^/?#]+)' % '|'.join(_SITES.keys())
+    _VALID_URL = r'http://(?:video|www|player)\.(?P<site>%s)\.com/(?P<type>watch|series|video|embed)/(?P<id>[^/?#]+)' % '|'.join(_SITES.keys())
     IE_DESC = 'Condé Nast media group: %s' % ', '.join(sorted(_SITES.values()))
 
     EMBED_URL = r'(?:https?:)?//player\.(?P<site>%s)\.com/(?P<type>embed)/.+?' % '|'.join(_SITES.keys())

From 1f9fb20fcda76f165ce39b01fe907fc74c8054d3 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 25 Sep 2015 07:39:22 +0100
Subject: [PATCH 2592/6219] [nextmedia] update AppleDailyIE tests

---
 youtube_dl/extractor/nextmedia.py | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/nextmedia.py b/youtube_dl/extractor/nextmedia.py
index c10784f6b..d1688457f 100644
--- a/youtube_dl/extractor/nextmedia.py
+++ b/youtube_dl/extractor/nextmedia.py
@@ -126,7 +126,8 @@ class AppleDailyIE(NextMediaIE):
             'thumbnail': 're:^https?://.*\.jpg$',
             'description': 'md5:23c0aac567dc08c9c16a3161a2c2e3cd',
             'upload_date': '20150128',
-        }
+        },
+        'skip': 'redirect to http://www.appledaily.com.tw/animation/',
     }, {
         # No thumbnail
         'url': 'http://www.appledaily.com.tw/animation/realtimenews/new/20150128/5003673/',
@@ -140,10 +141,19 @@ class AppleDailyIE(NextMediaIE):
         },
         'expected_warnings': [
             'video thumbnail',
-        ]
+        ],
+        'skip': 'redirect to http://www.appledaily.com.tw/animation/',
     }, {
         'url': 'http://www.appledaily.com.tw/appledaily/article/supplement/20140417/35770334/',
-        'only_matching': True,
+        'md5': 'eaa20e6b9df418c912d7f5dec2ba734d',
+        'info_dict': {
+            'id': '35770334',
+            'ext': 'mp4',
+            'title': '咖啡占卜測 XU裝熟指數',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'description': 'md5:7b859991a6a4fedbdf3dd3b66545c748',
+            'upload_date': '20140417',
+        },
     }]
 
     _URL_PATTERN = r'\{url: \'(.+)\'\}'

From 2e40a12225fbe64b84b3975b3063a676df0f4522 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 25 Sep 2015 17:24:35 +0800
Subject: [PATCH 2593/6219] [fktv] Correct spellings

---
 youtube_dl/extractor/fktv.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/fktv.py b/youtube_dl/extractor/fktv.py
index c2aa23aa2..93c4fd641 100644
--- a/youtube_dl/extractor/fktv.py
+++ b/youtube_dl/extractor/fktv.py
@@ -28,9 +28,9 @@ class FKTVIE(InfoExtractor):
 
         webpage = self._download_webpage('http://fernsehkritik.tv/folge-%s/play' % episode, episode)
         title = clean_html(self._html_search_regex('<h3>([^<]+?)</h3>', webpage, 'title'))
-        matchs = re.search(r'(?s)<video[^>]*poster="([^"]+)"[^>]*>(.*?)</video>', webpage)
-        if matchs:
-            poster, sources = matchs.groups()
+        matches = re.search(r'(?s)<video[^>]*poster="([^"]+)"[^>]*>(.*?)</video>', webpage)
+        if matches:
+            poster, sources = matches.groups()
             urls = re.findall(r'(?s)<source[^>]*src="([^"]+)"[^>]*>', sources)
             if sources:
                 formats = [{'url': url, 'format_id': determine_ext(url)} for url in urls]

From 8ddf48d59f2d04b7411202eb6bf02c6eaa387035 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 25 Sep 2015 17:48:51 +0800
Subject: [PATCH 2594/6219] [fktv] Raise an error is no videos found

---
 youtube_dl/extractor/fktv.py | 24 +++++++++++++-----------
 1 file changed, 13 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/fktv.py b/youtube_dl/extractor/fktv.py
index 93c4fd641..74c6cf866 100644
--- a/youtube_dl/extractor/fktv.py
+++ b/youtube_dl/extractor/fktv.py
@@ -6,6 +6,7 @@ from .common import InfoExtractor
 from ..utils import (
     clean_html,
     determine_ext,
+    ExtractorError,
 )
 
 
@@ -29,14 +30,15 @@ class FKTVIE(InfoExtractor):
         webpage = self._download_webpage('http://fernsehkritik.tv/folge-%s/play' % episode, episode)
         title = clean_html(self._html_search_regex('<h3>([^<]+?)</h3>', webpage, 'title'))
         matches = re.search(r'(?s)<video[^>]*poster="([^"]+)"[^>]*>(.*?)</video>', webpage)
-        if matches:
-            poster, sources = matches.groups()
-            urls = re.findall(r'(?s)<source[^>]*src="([^"]+)"[^>]*>', sources)
-            if sources:
-                formats = [{'url': url, 'format_id': determine_ext(url)} for url in urls]
-                return {
-                    'id': episode,
-                    'title': title,
-                    'formats': formats,
-                    'thumbnail': poster,
-                }
+        if matches is None:
+            raise ExtractorError('Unable to extract the video')
+
+        poster, sources = matches.groups()
+        urls = re.findall(r'(?s)<source[^>]*src="([^"]+)"[^>]*>', sources)
+        formats = [{'url': url, 'format_id': determine_ext(url)} for url in urls]
+        return {
+            'id': episode,
+            'title': title,
+            'formats': formats,
+            'thumbnail': poster,
+        }

From 140359fc2cfe7e9cbfecfd2fd625c6407232fe0f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 25 Sep 2015 17:51:48 +0800
Subject: [PATCH 2595/6219] [fktv] Correct and improve some regexs

---
 youtube_dl/extractor/fktv.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/fktv.py b/youtube_dl/extractor/fktv.py
index 74c6cf866..b081eb535 100644
--- a/youtube_dl/extractor/fktv.py
+++ b/youtube_dl/extractor/fktv.py
@@ -28,13 +28,13 @@ class FKTVIE(InfoExtractor):
         episode = self._match_id(url)
 
         webpage = self._download_webpage('http://fernsehkritik.tv/folge-%s/play' % episode, episode)
-        title = clean_html(self._html_search_regex('<h3>([^<]+?)</h3>', webpage, 'title'))
-        matches = re.search(r'(?s)<video[^>]*poster="([^"]+)"[^>]*>(.*?)</video>', webpage)
+        title = clean_html(self._html_search_regex('<h3>([^<]+)</h3>', webpage, 'title'))
+        matches = re.search(r'(?s)<video[^>]+poster="([^"]+)"[^>]*>(.*)</video>', webpage)
         if matches is None:
             raise ExtractorError('Unable to extract the video')
 
         poster, sources = matches.groups()
-        urls = re.findall(r'(?s)<source[^>]*src="([^"]+)"[^>]*>', sources)
+        urls = re.findall(r'<source[^>]+src="([^"]+)"', sources)
         formats = [{'url': url, 'format_id': determine_ext(url)} for url in urls]
         return {
             'id': episode,

From 6aeba407db84a636fc2522b4f2344eac9e0c1fdb Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 25 Sep 2015 10:52:48 +0100
Subject: [PATCH 2596/6219] [jukebox] remove extractor and handle it using
 generic extractor

---
 youtube_dl/extractor/__init__.py  |  1 -
 youtube_dl/extractor/generic.py   | 21 +++++++
 youtube_dl/extractor/jukebox.py   | 59 ------------------
 youtube_dl/extractor/ultimedia.py | 99 +++++++++++++------------------
 4 files changed, 61 insertions(+), 119 deletions(-)
 delete mode 100644 youtube_dl/extractor/jukebox.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 7272859db..1813c7e1b 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -262,7 +262,6 @@ from .izlesene import IzleseneIE
 from .jadorecettepub import JadoreCettePubIE
 from .jeuxvideo import JeuxVideoIE
 from .jove import JoveIE
-from .jukebox import JukeboxIE
 from .jpopsukitv import JpopsukiIE
 from .kaltura import KalturaIE
 from .kanalplay import KanalPlayIE
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 8881a8a23..4d1f75e63 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -50,6 +50,7 @@ from .dailymotion import DailymotionCloudIE
 from .onionstudios import OnionStudiosIE
 from .snagfilms import SnagFilmsEmbedIE
 from .screenwavemedia import ScreenwaveMediaIE
+from .ultimedia import UltimediaIE
 
 
 class GenericIE(InfoExtractor):
@@ -1029,6 +1030,21 @@ class GenericIE(InfoExtractor):
                 'ext': 'mp4',
                 'title': 'cinemasnob',
             },
+        },
+        # Ultimedia embed
+        {
+            'url': 'http://www.jukebox.es/kosheen/videoclip,pride,r303r.html',
+            'md5': '25551df6e7c7ab8096ceeeae048c5f64',
+            'info_dict': {
+                'id': 'r303r',
+                'ext': 'mp4',
+                'title': 'Kosheen - Pride (live)',
+                'thumbnail': 're:^https?://.*\.jpg',
+                'duration': 293,
+                'upload_date': '20081103',
+                'timestamp': 1225733392,
+                'uploader_id': '33m03',
+            },
         }
     ]
 
@@ -1751,6 +1767,11 @@ class GenericIE(InfoExtractor):
         if mobj is not None:
             return self.url_result(unescapeHTML(mobj.group('url')), 'ScreenwaveMedia')
 
+        # Look for Ulltimedia embeds
+        ultimedia_url = UltimediaIE._extract_url(webpage)
+        if ultimedia_url:
+            return self.url_result(self._proto_relative_url(ultimedia_url), 'Ultimedia')
+
         # Look for AdobeTVVideo embeds
         mobj = re.search(
             r'<iframe[^>]+src=[\'"]((?:https?:)?//video\.tv\.adobe\.com/v/\d+[^"]+)[\'"]',
diff --git a/youtube_dl/extractor/jukebox.py b/youtube_dl/extractor/jukebox.py
deleted file mode 100644
index da8068efc..000000000
--- a/youtube_dl/extractor/jukebox.py
+++ /dev/null
@@ -1,59 +0,0 @@
-from __future__ import unicode_literals
-
-import re
-
-from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
-    RegexNotFoundError,
-    unescapeHTML,
-)
-
-
-class JukeboxIE(InfoExtractor):
-    _VALID_URL = r'^http://www\.jukebox?\..+?\/.+[,](?P<id>[a-z0-9\-]+)\.html'
-    _TEST = {
-        'url': 'http://www.jukebox.es/kosheen/videoclip,pride,r303r.html',
-        'info_dict': {
-            'id': 'r303r',
-            'ext': 'flv',
-            'title': 'Kosheen-En Vivo Pride',
-            'uploader': 'Kosheen',
-        },
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        html = self._download_webpage(url, video_id)
-        iframe_url = unescapeHTML(self._search_regex(r'<iframe .*src="([^"]*)"', html, 'iframe url'))
-
-        iframe_html = self._download_webpage(iframe_url, video_id, 'Downloading iframe')
-        if re.search(r'class="jkb_waiting"', iframe_html) is not None:
-            raise ExtractorError('Video is not available(in your country?)!')
-
-        self.report_extraction(video_id)
-
-        try:
-            video_url = self._search_regex(r'"config":{"file":"(?P<video_url>http:[^"]+\?mdtk=[0-9]+)"',
-                                           iframe_html, 'video url')
-            video_url = unescapeHTML(video_url).replace('\/', '/')
-        except RegexNotFoundError:
-            youtube_url = self._search_regex(
-                r'config":{"file":"(http:\\/\\/www\.youtube\.com\\/watch\?v=[^"]+)"',
-                iframe_html, 'youtube url')
-            youtube_url = unescapeHTML(youtube_url).replace('\/', '/')
-            self.to_screen('Youtube video detected')
-            return self.url_result(youtube_url, ie='Youtube')
-
-        title = self._html_search_regex(r'<h1 class="inline">([^<]+)</h1>',
-                                        html, 'title')
-        artist = self._html_search_regex(r'<span id="infos_article_artist">([^<]+)</span>',
-                                         html, 'artist')
-
-        return {
-            'id': video_id,
-            'url': video_url,
-            'title': artist + '-' + title,
-            'uploader': artist,
-        }
diff --git a/youtube_dl/extractor/ultimedia.py b/youtube_dl/extractor/ultimedia.py
index c4751050e..45201332d 100644
--- a/youtube_dl/extractor/ultimedia.py
+++ b/youtube_dl/extractor/ultimedia.py
@@ -4,102 +4,83 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse_urlparse
-from ..utils import (
-    ExtractorError,
-    qualities,
-    unified_strdate,
-    clean_html,
-)
+from ..utils import int_or_none
 
 
 class UltimediaIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?ultimedia\.com/default/index/video[^/]+/id/(?P<id>[\d+a-z]+)'
+    _VALID_URL = r'https?://(?:www\.)?ultimedia\.com/deliver/(?P<type>generic|musique)(?:/[^/]+)*/(?:src|article)/(?P<id>[\d+a-z]+)'
     _TESTS = [{
         # news
-        'url': 'https://www.ultimedia.com/default/index/videogeneric/id/s8uk0r',
+        'url': 'https://www.ultimedia.com/deliver/generic/iframe/mdtk/01601930/zone/1/src/s8uk0r/autoplay/yes/ad/no/width/714/height/435',
         'md5': '276a0e49de58c7e85d32b057837952a2',
         'info_dict': {
             'id': 's8uk0r',
             'ext': 'mp4',
             'title': 'Loi sur la fin de vie: le texte prévoit un renforcement des directives anticipées',
-            'description': 'md5:3e5c8fd65791487333dda5db8aed32af',
             'thumbnail': 're:^https?://.*\.jpg',
+            'duration': 74,
             'upload_date': '20150317',
+            'timestamp': 1426604939,
+            'uploader_id': '3fszv',
         },
     }, {
         # music
-        'url': 'https://www.ultimedia.com/default/index/videomusic/id/xvpfp8',
+        'url': 'https://www.ultimedia.com/deliver/musique/iframe/mdtk/01601930/zone/1/article/xvpfp8/autoplay/yes/ad/no/width/714/height/435',
         'md5': '2ea3513813cf230605c7e2ffe7eca61c',
         'info_dict': {
             'id': 'xvpfp8',
             'ext': 'mp4',
-            'title': "Two - C'est la vie (Clip)",
-            'description': 'Two',
+            'title': 'Two - C\'est La Vie (clip)',
             'thumbnail': 're:^https?://.*\.jpg',
+            'duration': 233,
             'upload_date': '20150224',
+            'timestamp': 1424760500,
+            'uploader_id': '3rfzk',
         },
     }]
 
+    @staticmethod
+    def _extract_url(webpage):
+        mobj = re.search(
+            r'<(?:iframe|script)[^>]+src=["\'](?P<url>(?:https?:)?//(?:www\.)?ultimedia\.com/deliver/(?:generic|musique)(?:/[^/]+)*/(?:src|article)/[\d+a-z]+)',
+            webpage)
+        if mobj:
+            return mobj.group('url')
+
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
+        video_type, video_id = re.match(self._VALID_URL, url).groups()
 
-        deliver_url = self._proto_relative_url(self._search_regex(
-            r'<iframe[^>]+src="((?:https?:)?//(?:www\.)?ultimedia\.com/deliver/[^"]+)"',
-            webpage, 'deliver URL'), compat_urllib_parse_urlparse(url).scheme + ':')
-
-        deliver_page = self._download_webpage(
-            deliver_url, video_id, 'Downloading iframe page')
-
-        if '>This video is currently not available' in deliver_page:
-            raise ExtractorError(
-                'Video %s is currently not available' % video_id, expected=True)
-
-        player = self._parse_json(
-            self._search_regex(
-                r"jwplayer\('player(?:_temp)?'\)\.setup\(({.+?})\)\.on",
-                deliver_page, 'player'),
+        deliver_info = self._download_json(
+            'http://www.ultimedia.com/deliver/video?video=%s&topic=%s' % (video_id, video_type),
             video_id)
 
-        quality = qualities(['flash', 'html5'])
+        yt_id = deliver_info.get('yt_id')
+        if yt_id:
+            return self.url_result(yt_id, 'Youtube')
+
+        jwconf = deliver_info['jwconf']
+
         formats = []
-        for mode in player['modes']:
-            video_url = mode.get('config', {}).get('file')
-            if not video_url:
-                continue
-            if re.match(r'https?://www\.youtube\.com/.+?', video_url):
-                return self.url_result(video_url, 'Youtube')
+        for source in jwconf['playlist'][0]['sources']:
             formats.append({
-                'url': video_url,
-                'format_id': mode.get('type'),
-                'quality': quality(mode.get('type')),
+                'url': source['file'],
+                'format_id': source.get('label'),
             })
+
         self._sort_formats(formats)
 
-        thumbnail = player.get('image')
-
-        title = clean_html((
-            self._html_search_regex(
-                r'(?s)<div\s+id="catArticle">.+?</div>(.+?)</h1>',
-                webpage, 'title', default=None) or
-            self._search_regex(
-                r"var\s+nameVideo\s*=\s*'([^']+)'",
-                deliver_page, 'title')))
-
-        description = clean_html(self._html_search_regex(
-            r'(?s)<span>Description</span>(.+?)</p>', webpage,
-            'description', fatal=False))
-
-        upload_date = unified_strdate(self._search_regex(
-            r'Ajouté le\s*<span>([^<]+)', webpage,
-            'upload date', fatal=False))
+        title = deliver_info['title']
+        thumbnail = jwconf.get('image')
+        duration = int_or_none(deliver_info.get('duration'))
+        timestamp = int_or_none(deliver_info.get('release_time'))
+        uploader_id = deliver_info.get('owner_id')
 
         return {
             'id': video_id,
             'title': title,
-            'description': description,
             'thumbnail': thumbnail,
-            'upload_date': upload_date,
+            'duration': duration,
+            'timestamp': timestamp,
+            'uploader_id': uploader_id,
             'formats': formats,
         }

From 577380396171ba096240bfb3101f8151e32b587a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 25 Sep 2015 17:58:44 +0800
Subject: [PATCH 2597/6219] [fktv] Correct thumbnail extraction and add the
 test

---
 youtube_dl/extractor/fktv.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/fktv.py b/youtube_dl/extractor/fktv.py
index b081eb535..289cbb8c8 100644
--- a/youtube_dl/extractor/fktv.py
+++ b/youtube_dl/extractor/fktv.py
@@ -21,6 +21,7 @@ class FKTVIE(InfoExtractor):
             'id': '1',
             'ext': 'mp4',
             'title': 'Folge 1 vom 10. April 2007',
+            'thumbnail': 're:^https?://.*\.jpg$',
         },
     }
 
@@ -29,11 +30,14 @@ class FKTVIE(InfoExtractor):
 
         webpage = self._download_webpage('http://fernsehkritik.tv/folge-%s/play' % episode, episode)
         title = clean_html(self._html_search_regex('<h3>([^<]+)</h3>', webpage, 'title'))
-        matches = re.search(r'(?s)<video[^>]+poster="([^"]+)"[^>]*>(.*)</video>', webpage)
+        matches = re.search(r'(?s)<video[^>]+(?:poster="([^"]+)")?[^>]*>(.*)</video>', webpage)
         if matches is None:
             raise ExtractorError('Unable to extract the video')
 
         poster, sources = matches.groups()
+        if poster is None:
+            self.report_warning('unable to extract thumbnail')
+
         urls = re.findall(r'<source[^>]+src="([^"]+)"', sources)
         formats = [{'url': url, 'format_id': determine_ext(url)} for url in urls]
         return {

From 711762f0b70b65f2f28f2cd8023497f50fccd81a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 25 Sep 2015 18:01:08 +0800
Subject: [PATCH 2598/6219] [fktv] Coding style

---
 youtube_dl/extractor/fktv.py | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/fktv.py b/youtube_dl/extractor/fktv.py
index 289cbb8c8..0c14834f9 100644
--- a/youtube_dl/extractor/fktv.py
+++ b/youtube_dl/extractor/fktv.py
@@ -28,9 +28,12 @@ class FKTVIE(InfoExtractor):
     def _real_extract(self, url):
         episode = self._match_id(url)
 
-        webpage = self._download_webpage('http://fernsehkritik.tv/folge-%s/play' % episode, episode)
-        title = clean_html(self._html_search_regex('<h3>([^<]+)</h3>', webpage, 'title'))
-        matches = re.search(r'(?s)<video[^>]+(?:poster="([^"]+)")?[^>]*>(.*)</video>', webpage)
+        webpage = self._download_webpage(
+            'http://fernsehkritik.tv/folge-%s/play' % episode, episode)
+        title = clean_html(self._html_search_regex(
+            '<h3>([^<]+)</h3>', webpage, 'title'))
+        matches = re.search(
+            r'(?s)<video[^>]+(?:poster="([^"]+)")?[^>]*>(.*)</video>', webpage)
         if matches is None:
             raise ExtractorError('Unable to extract the video')
 
@@ -39,7 +42,10 @@ class FKTVIE(InfoExtractor):
             self.report_warning('unable to extract thumbnail')
 
         urls = re.findall(r'<source[^>]+src="([^"]+)"', sources)
-        formats = [{'url': url, 'format_id': determine_ext(url)} for url in urls]
+        formats = [{
+            'url': url,
+            'format_id': determine_ext(url),
+        } for url in urls]
         return {
             'id': episode,
             'title': title,

From 8de28761c41ca0451e3bbe4b9ee236d6651cca44 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 25 Sep 2015 18:17:48 +0800
Subject: [PATCH 2599/6219] [fktv] Fix a regex

---
 youtube_dl/extractor/fktv.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/fktv.py b/youtube_dl/extractor/fktv.py
index 0c14834f9..d9fc9952d 100644
--- a/youtube_dl/extractor/fktv.py
+++ b/youtube_dl/extractor/fktv.py
@@ -33,7 +33,8 @@ class FKTVIE(InfoExtractor):
         title = clean_html(self._html_search_regex(
             '<h3>([^<]+)</h3>', webpage, 'title'))
         matches = re.search(
-            r'(?s)<video[^>]+(?:poster="([^"]+)")?[^>]*>(.*)</video>', webpage)
+            r'(?s)<video(?:(?!poster)[^>])+(?:poster="([^"]+)")?[^>]*>(.*)</video>',
+            webpage)
         if matches is None:
             raise ExtractorError('Unable to extract the video')
 

From c44c7895b8774fb819b0b664bfcf64a7ebeea4e8 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 25 Sep 2015 11:28:26 +0100
Subject: [PATCH 2600/6219] [kuwo] fix title extraction and update test

---
 youtube_dl/extractor/kuwo.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/kuwo.py b/youtube_dl/extractor/kuwo.py
index fa233377d..51137a982 100644
--- a/youtube_dl/extractor/kuwo.py
+++ b/youtube_dl/extractor/kuwo.py
@@ -57,6 +57,7 @@ class KuwoIE(KuwoBaseIE):
             'upload_date': '20080122',
             'description': 'md5:ed13f58e3c3bf3f7fd9fbc4e5a7aa75c'
         },
+        'skip': 'this song has been offline because of copyright issues',
     }, {
         'url': 'http://www.kuwo.cn/yinyue/6446136/',
         'info_dict': {
@@ -78,7 +79,7 @@ class KuwoIE(KuwoBaseIE):
             errnote='Unable to get song detail info')
 
         song_name = self._html_search_regex(
-            r'<h1[^>]+title="([^"]+)">', webpage, 'song name')
+            r'(?s)class="[^"]*title[^"]*".*?<h1[^>]+title="([^"]+)"', webpage, 'song name')
         singer_name = self._html_search_regex(
             r'<div[^>]+class="s_img">\s*<a[^>]+title="([^>]+)"',
             webpage, 'singer name', fatal=False)

From 3d09aa4c82100649279d979f9910a8c84ba301ff Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 25 Sep 2015 11:40:32 +0100
Subject: [PATCH 2601/6219] [kuwo] extract title inside element with class
 title exactly

---
 youtube_dl/extractor/kuwo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/kuwo.py b/youtube_dl/extractor/kuwo.py
index 51137a982..a3c260838 100644
--- a/youtube_dl/extractor/kuwo.py
+++ b/youtube_dl/extractor/kuwo.py
@@ -79,7 +79,7 @@ class KuwoIE(KuwoBaseIE):
             errnote='Unable to get song detail info')
 
         song_name = self._html_search_regex(
-            r'(?s)class="[^"]*title[^"]*".*?<h1[^>]+title="([^"]+)"', webpage, 'song name')
+            r'(?s)class="(?:[^" ]+ +)*title(?: +[^" ]+)*".*?<h1[^>]+title="([^"]+)"', webpage, 'song name')
         singer_name = self._html_search_regex(
             r'<div[^>]+class="s_img">\s*<a[^>]+title="([^>]+)"',
             webpage, 'singer name', fatal=False)

From 4866b72eb2bf5747cde4654e75b1c1be0d456456 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 25 Sep 2015 21:58:45 +0200
Subject: [PATCH 2602/6219] [fktv] Don't redefine 'url' in list comprehension

Detected with flake8.
---
 youtube_dl/extractor/fktv.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/fktv.py b/youtube_dl/extractor/fktv.py
index d9fc9952d..fb9739b49 100644
--- a/youtube_dl/extractor/fktv.py
+++ b/youtube_dl/extractor/fktv.py
@@ -44,9 +44,9 @@ class FKTVIE(InfoExtractor):
 
         urls = re.findall(r'<source[^>]+src="([^"]+)"', sources)
         formats = [{
-            'url': url,
+            'url': furl,
             'format_id': determine_ext(url),
-        } for url in urls]
+        } for furl in urls]
         return {
             'id': episode,
             'title': title,

From 4c917d0314f0442fd17d6f8ec8e583252167512c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 25 Sep 2015 22:02:48 +0200
Subject: [PATCH 2603/6219] [README.md] Document the 'duration' field in the
 output template (#6929)

---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index 2ed751791..80152071d 100644
--- a/README.md
+++ b/README.md
@@ -281,6 +281,7 @@ The `-o` option allows users to indicate a template for the output file names. T
  - `playlist`: The sequence will be replaced by the name or the id of the playlist that contains the video.
  - `playlist_index`: The sequence will be replaced by the index of the video in the playlist padded with leading zeros according to the total length of the playlist.
  - `format_id`: The sequence will be replaced by the format code specified by `--format`.
+ - `duration`: The sequence will be replaced by the length of the video in seconds.
 
 The current default template is `%(title)s-%(id)s.%(ext)s`.
 

From 08bea4adde5d5beb5f2a5cc875216b4205b90d50 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 25 Sep 2015 22:34:02 +0200
Subject: [PATCH 2604/6219] Also run tests with python 3.5

---
 .travis.yml | 1 +
 tox.ini     | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index e78a2fa76..cc21fae8f 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -5,6 +5,7 @@ python:
   - "3.2"
   - "3.3"
   - "3.4"
+  - "3.5"
 sudo: false
 script: nosetests test --verbose
 notifications:
diff --git a/tox.ini b/tox.ini
index cd805fe8a..48504329f 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,5 +1,5 @@
 [tox]
-envlist = py26,py27,py33,py34
+envlist = py26,py27,py33,py34,py35
 [testenv]
 deps =
    nose

From 3706fb5dc8a38163689a29c7ed2b64f3b3d093e3 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 26 Sep 2015 07:51:11 +0100
Subject: [PATCH 2605/6219] [fktv] get format_id from video file ext

---
 youtube_dl/extractor/fktv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/fktv.py b/youtube_dl/extractor/fktv.py
index fb9739b49..40ea27895 100644
--- a/youtube_dl/extractor/fktv.py
+++ b/youtube_dl/extractor/fktv.py
@@ -45,7 +45,7 @@ class FKTVIE(InfoExtractor):
         urls = re.findall(r'<source[^>]+src="([^"]+)"', sources)
         formats = [{
             'url': furl,
-            'format_id': determine_ext(url),
+            'format_id': determine_ext(furl),
         } for furl in urls]
         return {
             'id': episode,

From 5db34f680f93917ef79ba59d501b9f82e6d44330 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 26 Sep 2015 10:31:32 +0100
Subject: [PATCH 2606/6219] [kuwo] check for the offline error page

---
 youtube_dl/extractor/kuwo.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/kuwo.py b/youtube_dl/extractor/kuwo.py
index a3c260838..36cb265ab 100644
--- a/youtube_dl/extractor/kuwo.py
+++ b/youtube_dl/extractor/kuwo.py
@@ -77,9 +77,11 @@ class KuwoIE(KuwoBaseIE):
         webpage = self._download_webpage(
             url, song_id, note='Download song detail info',
             errnote='Unable to get song detail info')
+        if '对不起，该歌曲由于版权问题已被下线，将返回网站首页' in webpage:
+            raise ExtractorError('this song has been offline because of copyright issues')
 
         song_name = self._html_search_regex(
-            r'(?s)class="(?:[^" ]+ +)*title(?: +[^" ]+)*".*?<h1[^>]+title="([^"]+)"', webpage, 'song name')
+            r'(?s)class="(?:[^"\s]+\s+)*title(?:\s+[^"\s]+)*".*?<h1[^>]+title="([^"]+)"', webpage, 'song name')
         singer_name = self._html_search_regex(
             r'<div[^>]+class="s_img">\s*<a[^>]+title="([^>]+)"',
             webpage, 'singer name', fatal=False)

From 7193650641f3fd84872b231e48263752d10b37b7 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 26 Sep 2015 11:44:35 +0100
Subject: [PATCH 2607/6219] [kuwo] treat the offline error as an expected
 ExtractorError

---
 youtube_dl/extractor/kuwo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/kuwo.py b/youtube_dl/extractor/kuwo.py
index 36cb265ab..0c8ed5d07 100644
--- a/youtube_dl/extractor/kuwo.py
+++ b/youtube_dl/extractor/kuwo.py
@@ -78,7 +78,7 @@ class KuwoIE(KuwoBaseIE):
             url, song_id, note='Download song detail info',
             errnote='Unable to get song detail info')
         if '对不起，该歌曲由于版权问题已被下线，将返回网站首页' in webpage:
-            raise ExtractorError('this song has been offline because of copyright issues')
+            raise ExtractorError('this song has been offline because of copyright issues', expected=True)
 
         song_name = self._html_search_regex(
             r'(?s)class="(?:[^"\s]+\s+)*title(?:\s+[^"\s]+)*".*?<h1[^>]+title="([^"]+)"', webpage, 'song name')

From fc42bc6ec9ab4858ec6bf1c8fe348ec47126375a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 26 Sep 2015 19:45:43 +0600
Subject: [PATCH 2608/6219] [mtv] Look for sm4:video:embed (Closes #6936,
 closes #6970)

---
 youtube_dl/extractor/mtv.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index a597714e9..4020cc2a9 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -200,7 +200,13 @@ class MTVServicesInfoExtractor(InfoExtractor):
         if mgid is None or ':' not in mgid:
             mgid = self._search_regex(
                 [r'data-mgid="(.*?)"', r'swfobject.embedSWF\(".*?(mgid:.*?)"'],
-                webpage, 'mgid')
+                webpage, 'mgid', default=None)
+
+        if not mgid:
+            sm4_embed = self._html_search_meta(
+                'sm4:video:embed', webpage, 'sm4 embed', default='')
+            mgid = self._search_regex(
+                r'embed/(mgid:.+?)["\'&?/]', sm4_embed, 'mgid')
 
         videos_info = self._get_videos_info(mgid)
         return videos_info

From fe1d858e35dde4c3068b9bec8a1bac7bd2cafadb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 26 Sep 2015 19:46:42 +0600
Subject: [PATCH 2609/6219] [mtvservices:embedded] Add _extract_url

---
 youtube_dl/extractor/mtv.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index 4020cc2a9..302c9bf35 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -228,6 +228,13 @@ class MTVServicesEmbeddedIE(MTVServicesInfoExtractor):
         },
     }
 
+    @staticmethod
+    def _extract_url(webpage):
+        mobj = re.search(
+            r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//media.mtvnservices.com/embed/.+?)\1', webpage)
+        if mobj:
+            return mobj.group('url')
+
     def _get_feed_url(self, uri):
         video_id = self._id_from_uri(uri)
         site_id = uri.replace(video_id, '')

From 46fde8a1a2e72bca3c22b6de4db06490fb90b59a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 26 Sep 2015 19:47:20 +0600
Subject: [PATCH 2610/6219] [extractor/generic] Use _extract_url for
 mtvservices

---
 youtube_dl/extractor/generic.py | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 8881a8a23..22ec78a02 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -50,6 +50,7 @@ from .dailymotion import DailymotionCloudIE
 from .onionstudios import OnionStudiosIE
 from .snagfilms import SnagFilmsEmbedIE
 from .screenwavemedia import ScreenwaveMediaIE
+from .mtv import MTVServicesEmbeddedIE
 
 
 class GenericIE(InfoExtractor):
@@ -1611,12 +1612,9 @@ class GenericIE(InfoExtractor):
             return self.url_result(url, ie='Vulture')
 
         # Look for embedded mtvservices player
-        mobj = re.search(
-            r'<iframe src="(?P<url>https?://media\.mtvnservices\.com/embed/[^"]+)"',
-            webpage)
-        if mobj is not None:
-            url = unescapeHTML(mobj.group('url'))
-            return self.url_result(url, ie='MTVServicesEmbedded')
+        mtvservices_url = MTVServicesEmbeddedIE._extract_url(webpage)
+        if mtvservices_url:
+            return self.url_result(mtvservices_url, ie='MTVServicesEmbedded')
 
         # Look for embedded yahoo player
         mobj = re.search(

From ee3d5a6d476ccca626bd4878e2cf871373fe5ca9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 26 Sep 2015 19:57:17 +0600
Subject: [PATCH 2611/6219] [bbc] Skip mediaselection on gelocation error
 (Closes #6983)

---
 youtube_dl/extractor/bbc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 42526357a..c1692b6c5 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -294,7 +294,7 @@ class BBCCoUkIE(InfoExtractor):
                 return self._download_media_selector_url(
                     mediaselector_url % programme_id, programme_id)
             except BBCCoUkIE.MediaSelectionError as e:
-                if e.id == 'notukerror':
+                if e.id in ('notukerror', 'geolocation'):
                     last_exception = e
                     continue
                 self._raise_extractor_error(e)

From 26ccc68bedb3f35fcbaad0196da0a6d93eef6e89 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 26 Sep 2015 20:06:21 +0600
Subject: [PATCH 2612/6219] [bbc] Clarify iptv-all mediaset rationale

---
 youtube_dl/extractor/bbc.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index c1692b6c5..074b56372 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -21,6 +21,9 @@ class BBCCoUkIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?bbc\.co\.uk/(?:(?:(?:programmes|iplayer(?:/[^/]+)?/(?:episode|playlist))/)|music/clips[/#])(?P<id>[\da-z]{8})'
 
     _MEDIASELECTOR_URLS = [
+        # Provides HQ HLS streams with even better quality that pc mediaset but fails
+        # with geolocation in some cases when it's even not geo restricted at all (e.g.
+        # http://www.bbc.co.uk/programmes/b06bp7lf)
         'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/iptv-all/vpid/%s',
         'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/pc/vpid/%s',
     ]

From 54914380c04ac7b52fa6dc69d49576fb7afe5f27 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 26 Sep 2015 20:07:12 +0600
Subject: [PATCH 2613/6219] [bbc] Add test for programme that fails with
 iptv-all mediaset

---
 youtube_dl/extractor/bbc.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 074b56372..cc2f6fed2 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -156,6 +156,21 @@ class BBCCoUkIE(InfoExtractor):
                 'skip_download': True,
             },
             'skip': 'geolocation',
+        }, {
+            # iptv-all mediaset fails with geolocation however there is no geo restriction
+            # for this programme at all
+            'url': 'http://www.bbc.co.uk/programmes/b06bp7lf',
+            'info_dict': {
+                'id': 'b06bp7kf',
+                'ext': 'flv',
+                'title': "Annie Mac's Friday Night, B.Traits sits in for Annie",
+                'description': 'B.Traits sits in for Annie Mac with a Mini-Mix from Disclosure.',
+                'duration': 10800,
+            },
+            'params': {
+                # rtmp download
+                'skip_download': True,
+            },
         }, {
             'url': 'http://www.bbc.co.uk/iplayer/playlist/p01dvks4',
             'only_matching': True,

From b65e5bb72f48c57316ac0166141810bd1e7b2de8 Mon Sep 17 00:00:00 2001
From: Qijiang Fan <fqj1994@gmail.com>
Date: Mon, 24 Aug 2015 00:31:30 +0800
Subject: [PATCH 2614/6219] [qqmusic] Add subtitles for QQMusic

Use .lrc lyrics as subtitles if lyrics in lrc format exist.
---
 youtube_dl/extractor/qqmusic.py | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index 1654a641f..8ead04606 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -58,6 +58,11 @@ class QQMusicIE(InfoExtractor):
         curMs = int(time.time() * 1000) % 1000
         return int(round(random.random() * 2147483647) * curMs % 1E10)
 
+    def _filter_lrc(self, data):
+        lyrics_expr = r'(\[[0-9]{2}:[0-9]{2}\.[0-9]{2,}\][^\n]*|\[[^\]]*\])'
+        texts = re.findall(lyrics_expr, data)
+        return ''.join(i + "\n" for i in texts)
+
     def _real_extract(self, url):
         mid = self._match_id(url)
 
@@ -112,15 +117,25 @@ class QQMusicIE(InfoExtractor):
         self._check_formats(formats, mid)
         self._sort_formats(formats)
 
-        return {
+        actual_lrc_lyrics = self._filter_lrc(lrc_content)
+
+        info_dict = {
             'id': mid,
             'formats': formats,
             'title': song_name,
             'upload_date': publish_time,
             'creator': singer,
             'description': lrc_content,
-            'thumbnail': thumbnail_url,
+            'thumbnail': thumbnail_url
         }
+        if actual_lrc_lyrics:
+            info_dict['subtitles'] = {
+                'origin': [{
+                    'ext': 'lrc',
+                    'data': actual_lrc_lyrics,
+                }]
+            }
+        return info_dict
 
 
 class QQPlaylistBaseIE(InfoExtractor):

From 961c5cbf172da49b26c09f806b0871719578e0e5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 26 Sep 2015 20:38:11 +0600
Subject: [PATCH 2615/6219] [qqmusic] Eliminate _filter_lrc and use single
 quotes

---
 youtube_dl/extractor/qqmusic.py | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index 8ead04606..bf6762034 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -58,11 +58,6 @@ class QQMusicIE(InfoExtractor):
         curMs = int(time.time() * 1000) % 1000
         return int(round(random.random() * 2147483647) * curMs % 1E10)
 
-    def _filter_lrc(self, data):
-        lyrics_expr = r'(\[[0-9]{2}:[0-9]{2}\.[0-9]{2,}\][^\n]*|\[[^\]]*\])'
-        texts = re.findall(lyrics_expr, data)
-        return ''.join(i + "\n" for i in texts)
-
     def _real_extract(self, url):
         mid = self._match_id(url)
 
@@ -117,7 +112,9 @@ class QQMusicIE(InfoExtractor):
         self._check_formats(formats, mid)
         self._sort_formats(formats)
 
-        actual_lrc_lyrics = self._filter_lrc(lrc_content)
+        actual_lrc_lyrics = ''.join(
+            line + '\n' for line in re.findall(
+                r'(\[[0-9]{2}:[0-9]{2}\.[0-9]{2,}\][^\n]*|\[[^\]]*\])', lrc_content))
 
         info_dict = {
             'id': mid,

From d4cd06138c70dd50d574c9a5d96bf585648373d5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 26 Sep 2015 20:54:41 +0600
Subject: [PATCH 2616/6219] [qqmusic] Do not capture braced text from the
 middle of the string

---
 youtube_dl/extractor/qqmusic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index bf6762034..21cf6f2d6 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -114,7 +114,7 @@ class QQMusicIE(InfoExtractor):
 
         actual_lrc_lyrics = ''.join(
             line + '\n' for line in re.findall(
-                r'(\[[0-9]{2}:[0-9]{2}\.[0-9]{2,}\][^\n]*|\[[^\]]*\])', lrc_content))
+                r'(?m)^(\[[0-9]{2}:[0-9]{2}\.[0-9]{2,}\][^\n]*|\[[^\]]*\])', lrc_content))
 
         info_dict = {
             'id': mid,

From cd1bb54990dbe97b74a184022403cc30b0dbfe05 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 26 Sep 2015 21:00:59 +0600
Subject: [PATCH 2617/6219] [qqmusic] Add test for a song with non .lrc lyrics

---
 youtube_dl/extractor/qqmusic.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index 21cf6f2d6..50b0d9f31 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -43,6 +43,21 @@ class QQMusicIE(InfoExtractor):
             'description': 'md5:46857d5ed62bc4ba84607a805dccf437',
             'thumbnail': 're:^https?://.*\.jpg$',
         }
+    }, {
+        'note': 'lyrics not in .lrc format',
+        'url': 'http://y.qq.com/#type=song&mid=001JyApY11tIp6',
+        'info_dict': {
+            'id': '001JyApY11tIp6',
+            'ext': 'mp3',
+            'title': 'Shadows Over Transylvania',
+            'upload_date': '19970225',
+            'creator': 'Dark Funeral',
+            'description': 'md5:ed14d5bd7ecec19609108052c25b2c11',
+            'thumbnail': 're:^https?://.*\.jpg$',
+        },
+        'params': {
+            'skip_download': True,
+        },
     }]
 
     _FORMATS = {

From 26c6d1922eaf70c980a054f0523c112f71863465 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 26 Sep 2015 21:01:28 +0600
Subject: [PATCH 2618/6219] Credit @fqj1994 for qqmusic .lrc support

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 901c1b263..cc552bcb2 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -143,3 +143,4 @@ Shaun Walbridge
 Lee Jenkins
 Anssi Hannula
 Lukáš Lalinský
+Qijiang Fan

From 8aab976bbd0a009a15b1737668882e44d84c7a71 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 26 Sep 2015 21:07:54 +0600
Subject: [PATCH 2619/6219] [extractor/common] Document release_date field

---
 youtube_dl/extractor/common.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 1e7db8a9b..2fe0d5d37 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -152,6 +152,7 @@ class InfoExtractor(object):
     description:    Full video description.
     uploader:       Full name of the video uploader.
     creator:        The main artist who created the video.
+    release_date:   The date (YYYYMMDD) when the video was released.
     timestamp:      UNIX timestamp of the moment the video became available.
     upload_date:    Video upload date (YYYYMMDD).
                     If not explicitly set, calculated from timestamp.

From a5d09d684eef68b42377a651706a61b0b3fca329 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 26 Sep 2015 21:08:23 +0600
Subject: [PATCH 2620/6219] [qqmusic] Use release_date

---
 youtube_dl/extractor/qqmusic.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index 50b0d9f31..c88823416 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -25,7 +25,7 @@ class QQMusicIE(InfoExtractor):
             'id': '004295Et37taLD',
             'ext': 'mp3',
             'title': '可惜没如果',
-            'upload_date': '20141227',
+            'release_date': '20141227',
             'creator': '林俊杰',
             'description': 'md5:d327722d0361576fde558f1ac68a7065',
             'thumbnail': 're:^https?://.*\.jpg$',
@@ -38,7 +38,7 @@ class QQMusicIE(InfoExtractor):
             'id': '004MsGEo3DdNxV',
             'ext': 'mp3',
             'title': '如果',
-            'upload_date': '20050626',
+            'release_date': '20050626',
             'creator': '李季美',
             'description': 'md5:46857d5ed62bc4ba84607a805dccf437',
             'thumbnail': 're:^https?://.*\.jpg$',
@@ -50,7 +50,7 @@ class QQMusicIE(InfoExtractor):
             'id': '001JyApY11tIp6',
             'ext': 'mp3',
             'title': 'Shadows Over Transylvania',
-            'upload_date': '19970225',
+            'release_date': '19970225',
             'creator': 'Dark Funeral',
             'description': 'md5:ed14d5bd7ecec19609108052c25b2c11',
             'thumbnail': 're:^https?://.*\.jpg$',
@@ -135,7 +135,7 @@ class QQMusicIE(InfoExtractor):
             'id': mid,
             'formats': formats,
             'title': song_name,
-            'upload_date': publish_time,
+            'release_date': publish_time,
             'creator': singer,
             'description': lrc_content,
             'thumbnail': thumbnail_url

From 95c5e101035eb3a77ebf62d75b2ebaac836373c3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 26 Sep 2015 21:15:34 +0600
Subject: [PATCH 2621/6219] [qqmusic] Allow [mm:ss] timestamps

---
 youtube_dl/extractor/qqmusic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index c88823416..c98539f6a 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -129,7 +129,7 @@ class QQMusicIE(InfoExtractor):
 
         actual_lrc_lyrics = ''.join(
             line + '\n' for line in re.findall(
-                r'(?m)^(\[[0-9]{2}:[0-9]{2}\.[0-9]{2,}\][^\n]*|\[[^\]]*\])', lrc_content))
+                r'(?m)^(\[[0-9]{2}:[0-9]{2}(?:\.[0-9]{2,})?\][^\n]*|\[[^\]]*\])', lrc_content))
 
         info_dict = {
             'id': mid,

From 9d632b1b279ca9bc5b73011b942c7fb452370788 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 26 Sep 2015 15:30:02 +0100
Subject: [PATCH 2622/6219] [eagleplatform] extract mp4 url and fix thumbnail
 url

---
 youtube_dl/extractor/eagleplatform.py | 25 +++++++++++++++----------
 1 file changed, 15 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/eagleplatform.py b/youtube_dl/extractor/eagleplatform.py
index a1ee51568..2987d5272 100644
--- a/youtube_dl/extractor/eagleplatform.py
+++ b/youtube_dl/extractor/eagleplatform.py
@@ -21,7 +21,7 @@ class EaglePlatformIE(InfoExtractor):
     _TESTS = [{
         # http://lenta.ru/news/2015/03/06/navalny/
         'url': 'http://lentaru.media.eagleplatform.com/index/player?player=new&record_id=227304&player_template_id=5201',
-        'md5': '0b7994faa2bd5c0f69a3db6db28d078d',
+        'md5': '70f5187fb620f2c1d503b3b22fd4efe3',
         'info_dict': {
             'id': '227304',
             'ext': 'mp4',
@@ -36,7 +36,7 @@ class EaglePlatformIE(InfoExtractor):
         # http://muz-tv.ru/play/7129/
         # http://media.clipyou.ru/index/player?record_id=12820&width=730&height=415&autoplay=true
         'url': 'eagleplatform:media.clipyou.ru:12820',
-        'md5': '6c2ebeab03b739597ce8d86339d5a905',
+        'md5': '90b26344ba442c8e44aa4cf8f301164a',
         'info_dict': {
             'id': '12820',
             'ext': 'mp4',
@@ -53,10 +53,10 @@ class EaglePlatformIE(InfoExtractor):
         if status != 200:
             raise ExtractorError(' '.join(response['errors']), expected=True)
 
-    def _download_json(self, url_or_request, video_id, note='Downloading JSON metadata'):
-        response = super(EaglePlatformIE, self)._download_json(url_or_request, video_id, note)
+    def _get_video_url(self, url_or_request, video_id, note='Downloading JSON metadata'):
+        response = self._download_json(url_or_request, video_id, note)
         self._handle_error(response)
-        return response
+        return response['data'][0]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
@@ -69,7 +69,7 @@ class EaglePlatformIE(InfoExtractor):
 
         title = media['title']
         description = media.get('description')
-        thumbnail = media.get('snapshot')
+        thumbnail = self._proto_relative_url(media.get('snapshot'))
         duration = int_or_none(media.get('duration'))
         view_count = int_or_none(media.get('views'))
 
@@ -78,13 +78,18 @@ class EaglePlatformIE(InfoExtractor):
         if age_restriction:
             age_limit = 0 if age_restriction == 'allow_all' else 18
 
-        m3u8_data = self._download_json(
-            self._proto_relative_url(media['sources']['secure_m3u8']['auto'], 'http:'),
-            video_id, 'Downloading m3u8 JSON')
+        secure_m3u8 = self._proto_relative_url(media['sources']['secure_m3u8']['auto'])
 
+        m3u8_url = self._get_video_url(secure_m3u8, video_id, 'Downloading m3u8 JSON')
         formats = self._extract_m3u8_formats(
-            m3u8_data['data'][0], video_id,
+            m3u8_url, video_id,
             'mp4', entry_protocol='m3u8_native')
+
+        mp4_url = self._get_video_url(
+            secure_m3u8.replace("m3u8", "mp4").replace("hlsvod", "mp4").replace("hls", "mp4"),
+            video_id, 'Downloading mp4 JSON')
+        formats.append({'url': mp4_url, 'format_id': 'mp4'})
+
         self._sort_formats(formats)
 
         return {

From 22becac4bd4338108b7949607fb7dccd6d173abf Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 26 Sep 2015 17:37:30 +0100
Subject: [PATCH 2623/6219] [eagleplatform] return the code to handle errors in
 all _download_json requests

---
 youtube_dl/extractor/eagleplatform.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/eagleplatform.py b/youtube_dl/extractor/eagleplatform.py
index 2987d5272..75cf73b25 100644
--- a/youtube_dl/extractor/eagleplatform.py
+++ b/youtube_dl/extractor/eagleplatform.py
@@ -53,10 +53,13 @@ class EaglePlatformIE(InfoExtractor):
         if status != 200:
             raise ExtractorError(' '.join(response['errors']), expected=True)
 
-    def _get_video_url(self, url_or_request, video_id, note='Downloading JSON metadata'):
-        response = self._download_json(url_or_request, video_id, note)
+    def _download_json(self, url_or_request, video_id, note='Downloading JSON metadata'):
+        response = super(EaglePlatformIE, self)._download_json(url_or_request, video_id, note)
         self._handle_error(response)
-        return response['data'][0]
+        return response
+
+    def _get_video_url(self, url_or_request, video_id, note='Downloading JSON metadata'):
+        return self._download_json(url_or_request, video_id, note)['data'][0]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)

From d045f0bdb72bc3f84cbcfa5fd940c46d3360b386 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 26 Sep 2015 18:53:57 +0100
Subject: [PATCH 2624/6219] [eagleplatform] use http urls explicitly

---
 youtube_dl/extractor/eagleplatform.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/eagleplatform.py b/youtube_dl/extractor/eagleplatform.py
index 75cf73b25..8bf88c77d 100644
--- a/youtube_dl/extractor/eagleplatform.py
+++ b/youtube_dl/extractor/eagleplatform.py
@@ -81,7 +81,7 @@ class EaglePlatformIE(InfoExtractor):
         if age_restriction:
             age_limit = 0 if age_restriction == 'allow_all' else 18
 
-        secure_m3u8 = self._proto_relative_url(media['sources']['secure_m3u8']['auto'])
+        secure_m3u8 = self._proto_relative_url(media['sources']['secure_m3u8']['auto'], 'http:')
 
         m3u8_url = self._get_video_url(secure_m3u8, video_id, 'Downloading m3u8 JSON')
         formats = self._extract_m3u8_formats(

From c471b34575eac87b0841616dd6acb90b221c6fe6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 27 Sep 2015 01:10:39 +0600
Subject: [PATCH 2625/6219] [eagleplatform] Simplify secure mp4 construction
 and clarify rationale

---
 youtube_dl/extractor/eagleplatform.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/eagleplatform.py b/youtube_dl/extractor/eagleplatform.py
index 8bf88c77d..43336e57f 100644
--- a/youtube_dl/extractor/eagleplatform.py
+++ b/youtube_dl/extractor/eagleplatform.py
@@ -89,7 +89,9 @@ class EaglePlatformIE(InfoExtractor):
             'mp4', entry_protocol='m3u8_native')
 
         mp4_url = self._get_video_url(
-            secure_m3u8.replace("m3u8", "mp4").replace("hlsvod", "mp4").replace("hls", "mp4"),
+            # Secure mp4 URL is constructed according to Player.prototype.mp4 from
+            # http://lentaru.media.eagleplatform.com/player/player.js
+            re.sub(r'm3u8|hlsvod|hls|f4m', 'mp4', secure_m3u8),
             video_id, 'Downloading mp4 JSON')
         formats.append({'url': mp4_url, 'format_id': 'mp4'})
 

From 3c63e1bb5740484dae483595a87325e8cf9c7e7b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 27 Sep 2015 01:12:46 +0600
Subject: [PATCH 2626/6219] [eagleplatform] Make _handle_error staticmethod

---
 youtube_dl/extractor/eagleplatform.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/eagleplatform.py b/youtube_dl/extractor/eagleplatform.py
index 43336e57f..ee24242a8 100644
--- a/youtube_dl/extractor/eagleplatform.py
+++ b/youtube_dl/extractor/eagleplatform.py
@@ -48,7 +48,8 @@ class EaglePlatformIE(InfoExtractor):
         'skip': 'Georestricted',
     }]
 
-    def _handle_error(self, response):
+    @staticmethod
+    def _handle_error(response):
         status = int_or_none(response.get('status', 200))
         if status != 200:
             raise ExtractorError(' '.join(response['errors']), expected=True)

From 2f962d0a91d22becd59489a9d879b1caafadefed Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 27 Sep 2015 01:17:44 +0600
Subject: [PATCH 2627/6219] [eagleplatform] Use http scheme for thumbnail

---
 youtube_dl/extractor/eagleplatform.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/eagleplatform.py b/youtube_dl/extractor/eagleplatform.py
index ee24242a8..e529b9b96 100644
--- a/youtube_dl/extractor/eagleplatform.py
+++ b/youtube_dl/extractor/eagleplatform.py
@@ -73,7 +73,7 @@ class EaglePlatformIE(InfoExtractor):
 
         title = media['title']
         description = media.get('description')
-        thumbnail = self._proto_relative_url(media.get('snapshot'))
+        thumbnail = self._proto_relative_url(media.get('snapshot'), 'http:')
         duration = int_or_none(media.get('duration'))
         view_count = int_or_none(media.get('views'))
 

From 95240b8093c6fe9007e33f0991ed8d54b42eaf3b Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 15 Sep 2015 21:30:24 +0800
Subject: [PATCH 2628/6219] Use `insert` for all sys.path manipulations

Closes #6867.
---
 devscripts/bash-completion.py       | 2 +-
 devscripts/fish-completion.py       | 2 +-
 devscripts/gh-pages/update-sites.py | 2 +-
 devscripts/make_supportedsites.py   | 2 +-
 devscripts/zsh-completion.py        | 2 +-
 youtube_dl/__main__.py              | 2 +-
 6 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/devscripts/bash-completion.py b/devscripts/bash-completion.py
index cd26cc089..ce68f26f9 100755
--- a/devscripts/bash-completion.py
+++ b/devscripts/bash-completion.py
@@ -5,7 +5,7 @@ import os
 from os.path import dirname as dirn
 import sys
 
-sys.path.append(dirn(dirn((os.path.abspath(__file__)))))
+sys.path.insert(0, dirn(dirn((os.path.abspath(__file__)))))
 import youtube_dl
 
 BASH_COMPLETION_FILE = "youtube-dl.bash-completion"
diff --git a/devscripts/fish-completion.py b/devscripts/fish-completion.py
index c2f238798..41629d87d 100755
--- a/devscripts/fish-completion.py
+++ b/devscripts/fish-completion.py
@@ -6,7 +6,7 @@ import os
 from os.path import dirname as dirn
 import sys
 
-sys.path.append(dirn(dirn((os.path.abspath(__file__)))))
+sys.path.insert(0, dirn(dirn((os.path.abspath(__file__)))))
 import youtube_dl
 from youtube_dl.utils import shell_quote
 
diff --git a/devscripts/gh-pages/update-sites.py b/devscripts/gh-pages/update-sites.py
index d3ef5f0b5..503c1372f 100755
--- a/devscripts/gh-pages/update-sites.py
+++ b/devscripts/gh-pages/update-sites.py
@@ -6,7 +6,7 @@ import os
 import textwrap
 
 # We must be able to import youtube_dl
-sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
 
 import youtube_dl
 
diff --git a/devscripts/make_supportedsites.py b/devscripts/make_supportedsites.py
index 3df4385a6..8cb4a4638 100644
--- a/devscripts/make_supportedsites.py
+++ b/devscripts/make_supportedsites.py
@@ -9,7 +9,7 @@ import sys
 
 # Import youtube_dl
 ROOT_DIR = os.path.join(os.path.dirname(__file__), '..')
-sys.path.append(ROOT_DIR)
+sys.path.insert(0, ROOT_DIR)
 import youtube_dl
 
 
diff --git a/devscripts/zsh-completion.py b/devscripts/zsh-completion.py
index f200f2c80..04728e8e2 100755
--- a/devscripts/zsh-completion.py
+++ b/devscripts/zsh-completion.py
@@ -5,7 +5,7 @@ import os
 from os.path import dirname as dirn
 import sys
 
-sys.path.append(dirn(dirn((os.path.abspath(__file__)))))
+sys.path.insert(0, dirn(dirn((os.path.abspath(__file__)))))
 import youtube_dl
 
 ZSH_COMPLETION_FILE = "youtube-dl.zsh"
diff --git a/youtube_dl/__main__.py b/youtube_dl/__main__.py
index 65a0f891c..42a0f8c6f 100755
--- a/youtube_dl/__main__.py
+++ b/youtube_dl/__main__.py
@@ -11,7 +11,7 @@ if __package__ is None and not hasattr(sys, "frozen"):
     # direct call of __main__.py
     import os.path
     path = os.path.realpath(os.path.abspath(__file__))
-    sys.path.append(os.path.dirname(os.path.dirname(path)))
+    sys.path.insert(0, os.path.dirname(os.path.dirname(path)))
 
 import youtube_dl
 

From c6b8f4d0c9413df9c33f3bd67859d1a9675ec8aa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 27 Sep 2015 05:53:21 +0600
Subject: [PATCH 2629/6219] [condenast] Add support for JS embeds

---
 youtube_dl/extractor/condenast.py | 25 +++++++++++++++++++++----
 1 file changed, 21 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/condenast.py b/youtube_dl/extractor/condenast.py
index d6949ca28..6f92ae2ed 100644
--- a/youtube_dl/extractor/condenast.py
+++ b/youtube_dl/extractor/condenast.py
@@ -11,6 +11,7 @@ from ..compat import (
 )
 from ..utils import (
     orderedSet,
+    remove_end,
 )
 
 
@@ -44,12 +45,12 @@ class CondeNastIE(InfoExtractor):
         'wmagazine': 'W Magazine',
     }
 
-    _VALID_URL = r'http://(?:video|www|player)\.(?P<site>%s)\.com/(?P<type>watch|series|video|embed)/(?P<id>[^/?#]+)' % '|'.join(_SITES.keys())
+    _VALID_URL = r'http://(?:video|www|player)\.(?P<site>%s)\.com/(?P<type>watch|series|video|embed(?:js)?)/(?P<id>[^/?#]+)' % '|'.join(_SITES.keys())
     IE_DESC = 'Condé Nast media group: %s' % ', '.join(sorted(_SITES.values()))
 
-    EMBED_URL = r'(?:https?:)?//player\.(?P<site>%s)\.com/(?P<type>embed)/.+?' % '|'.join(_SITES.keys())
+    EMBED_URL = r'(?:https?:)?//player\.(?P<site>%s)\.com/(?P<type>embed(?:js)?)/.+?' % '|'.join(_SITES.keys())
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://video.wired.com/watch/3d-printed-speakers-lit-with-led',
         'md5': '1921f713ed48aabd715691f774c451f7',
         'info_dict': {
@@ -58,7 +59,16 @@ class CondeNastIE(InfoExtractor):
             'title': '3D Printed Speakers Lit With LED',
             'description': 'Check out these beautiful 3D printed LED speakers.  You can\'t actually buy them, but LumiGeek is working on a board that will let you make you\'re own.',
         }
-    }
+    }, {
+        # JS embed
+        'url': 'http://player.cnevids.com/embedjs/55f9cf8b61646d1acf00000c/5511d76261646d5566020000.js',
+        'md5': 'f1a6f9cafb7083bab74a710f65d08999',
+        'info_dict': {
+            'id': '55f9cf8b61646d1acf00000c',
+            'ext': 'mp4',
+            'title': '3D printed TSA Travel Sentry keys really do open TSA locks',
+        }
+    }]
 
     def _extract_series(self, url, webpage):
         title = self._html_search_regex(r'<div class="cne-series-info">.*?<h1>(.+?)</h1>',
@@ -122,6 +132,13 @@ class CondeNastIE(InfoExtractor):
         url_type = mobj.group('type')
         item_id = mobj.group('id')
 
+        # Convert JS embed to regular embed
+        if url_type == 'embedjs':
+            parsed_url = compat_urlparse.urlparse(url)
+            url = compat_urlparse.urlunparse(parsed_url._replace(
+                path=remove_end(parsed_url.path, '.js').replace('/embedjs/', '/embed/')))
+            url_type = 'embed'
+
         self.to_screen('Extracting from %s with the Condé Nast extractor' % self._SITES[site])
         webpage = self._download_webpage(url, item_id)
 

From dd467d33d0883c0c0c94d7fc5ed041d5c12bb564 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 27 Sep 2015 05:55:48 +0600
Subject: [PATCH 2630/6219] [extractor/generic] Add support for condenast
 script embeds (Closes #6885, closes #6991)

---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 22ec78a02..ca5fbafb2 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1653,7 +1653,7 @@ class GenericIE(InfoExtractor):
             return self.url_result(mobj.group('url'), 'MLB')
 
         mobj = re.search(
-            r'<iframe[^>]+?src=(["\'])(?P<url>%s)\1' % CondeNastIE.EMBED_URL,
+            r'<(?:iframe|script)[^>]+?src=(["\'])(?P<url>%s)\1' % CondeNastIE.EMBED_URL,
             webpage)
         if mobj is not None:
             return self.url_result(self._proto_relative_url(mobj.group('url'), scheme='http:'), 'CondeNast')

From e13b9e7885d83f09d95882a494d5e32153e7baa6 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 24 Sep 2015 22:38:00 +0100
Subject: [PATCH 2631/6219] [keek] fix info extraction

---
 youtube_dl/extractor/keek.py | 28 ++++++----------------------
 1 file changed, 6 insertions(+), 22 deletions(-)

diff --git a/youtube_dl/extractor/keek.py b/youtube_dl/extractor/keek.py
index c0956ba09..4493a929d 100644
--- a/youtube_dl/extractor/keek.py
+++ b/youtube_dl/extractor/keek.py
@@ -4,43 +4,27 @@ from .common import InfoExtractor
 
 
 class KeekIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?keek\.com/(?:!|\w+/keeks/)(?P<id>\w+)'
+    _VALID_URL = r'https?://(?:www\.)?keek\.com/keek/(?P<id>\w+)'
     IE_NAME = 'keek'
     _TEST = {
-        'url': 'https://www.keek.com/ytdl/keeks/NODfbab',
-        'md5': '09c5c109067536c1cec8bac8c21fea05',
+        'url': 'https://www.keek.com/keek/NODfbab',
+        'md5': '9b0636f8c0f7614afa4ea5e4c6e57e83',
         'info_dict': {
             'id': 'NODfbab',
             'ext': 'mp4',
-            'uploader': 'youtube-dl project',
-            'uploader_id': 'ytdl',
-            'title': 'test chars: "\'/\\\u00e4<>This is a test video for youtube-dl.For more information, contact phihag@phihag.de .',
+            'title': 'test chars: "\'/\\ä<>This is a test video for youtube-dl.For more information, contact phihag@phihag.de . - Video - Videos on Keek',
         },
     }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        video_url = 'http://cdn.keek.com/keek/video/%s' % video_id
-        thumbnail = 'http://cdn.keek.com/keek/thumbnail/%s/w100/h75' % video_id
         webpage = self._download_webpage(url, video_id)
 
-        raw_desc = self._html_search_meta('description', webpage)
-        if raw_desc:
-            uploader = self._html_search_regex(
-                r'Watch (.*?)\s+\(', raw_desc, 'uploader', fatal=False)
-            uploader_id = self._html_search_regex(
-                r'Watch .*?\(@(.+?)\)', raw_desc, 'uploader_id', fatal=False)
-        else:
-            uploader = None
-            uploader_id = None
-
         return {
             'id': video_id,
-            'url': video_url,
+            'url': self._og_search_video_url(webpage),
             'ext': 'mp4',
             'title': self._og_search_title(webpage),
-            'thumbnail': thumbnail,
-            'uploader': uploader,
-            'uploader_id': uploader_id,
+            'thumbnail': self._og_search_thumbnail(webpage),
         }

From 6a24cb3d22876826477dc2857f9d46fe65a9fc28 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 26 Sep 2015 18:04:25 +0100
Subject: [PATCH 2632/6219] [keek] extract more info

---
 youtube_dl/extractor/keek.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/youtube_dl/extractor/keek.py b/youtube_dl/extractor/keek.py
index 4493a929d..24f5c7cb7 100644
--- a/youtube_dl/extractor/keek.py
+++ b/youtube_dl/extractor/keek.py
@@ -1,5 +1,7 @@
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 
 
@@ -13,6 +15,9 @@ class KeekIE(InfoExtractor):
             'id': 'NODfbab',
             'ext': 'mp4',
             'title': 'test chars: "\'/\\ä<>This is a test video for youtube-dl.For more information, contact phihag@phihag.de . - Video - Videos on Keek',
+            'description': 'test chars: "\'/\\ä<>This is a test video for youtube-dl.For more information, contact phihag@phihag.de .',
+            'uploader': 'ytdl',
+            'uploader_id': 'eGT5bab',
         },
     }
 
@@ -20,11 +25,18 @@ class KeekIE(InfoExtractor):
         video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id)
+        uploader = uploader_id = None
+        matches = re.search(r'data-username="(?P<uploader>[^"]+)"[^>]*data-user-id="(?P<uploader_id>[^"]+)"', webpage)
+        if matches:
+            uploader, uploader_id = matches.groups()
 
         return {
             'id': video_id,
             'url': self._og_search_video_url(webpage),
             'ext': 'mp4',
             'title': self._og_search_title(webpage),
+            'description': self._og_search_description(webpage),
             'thumbnail': self._og_search_thumbnail(webpage),
+            'uploader': uploader,
+            'uploader_id': uploader_id,
         }

From 80e98aed69102b0f300ac69001ab6ac64e2febec Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 26 Sep 2015 18:25:24 +0100
Subject: [PATCH 2633/6219] [keek] fix test title

---
 youtube_dl/extractor/keek.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/keek.py b/youtube_dl/extractor/keek.py
index 24f5c7cb7..280134a2a 100644
--- a/youtube_dl/extractor/keek.py
+++ b/youtube_dl/extractor/keek.py
@@ -14,8 +14,8 @@ class KeekIE(InfoExtractor):
         'info_dict': {
             'id': 'NODfbab',
             'ext': 'mp4',
-            'title': 'test chars: "\'/\\ä<>This is a test video for youtube-dl.For more information, contact phihag@phihag.de . - Video - Videos on Keek',
-            'description': 'test chars: "\'/\\ä<>This is a test video for youtube-dl.For more information, contact phihag@phihag.de .',
+            'title': 'test chars: "\'/\\\xe4<>This is a test video for youtube-dl.For more information, contact phihag@phihag.de . - Video - Videos on Keek',
+            'description': 'md5:35d42050a3ece241d5ddd7fdcc6fd896',
             'uploader': 'ytdl',
             'uploader_id': 'eGT5bab',
         },

From f30e9976d62c528bc7eda7b90d7c2fbc89d0b0be Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 26 Sep 2015 19:07:18 +0100
Subject: [PATCH 2634/6219] [keek] add utf-8 coding cookie

---
 youtube_dl/extractor/keek.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/keek.py b/youtube_dl/extractor/keek.py
index 280134a2a..d84774dca 100644
--- a/youtube_dl/extractor/keek.py
+++ b/youtube_dl/extractor/keek.py
@@ -1,3 +1,4 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
 import re
@@ -14,7 +15,7 @@ class KeekIE(InfoExtractor):
         'info_dict': {
             'id': 'NODfbab',
             'ext': 'mp4',
-            'title': 'test chars: "\'/\\\xe4<>This is a test video for youtube-dl.For more information, contact phihag@phihag.de . - Video - Videos on Keek',
+            'title': 'test chars: "\'/\\ä<>This is a test video for youtube-dl.For more information, contact phihag@phihag.de . - Video - Videos on Keek',
             'description': 'md5:35d42050a3ece241d5ddd7fdcc6fd896',
             'uploader': 'ytdl',
             'uploader_id': 'eGT5bab',

From 98e1c935a1ed676ca747c04c28d98370f740cadc Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 27 Sep 2015 19:09:48 +0100
Subject: [PATCH 2635/6219] [keek] extract uploader and uploader id with
 _search_regex

---
 youtube_dl/extractor/keek.py | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/keek.py b/youtube_dl/extractor/keek.py
index d84774dca..99154112b 100644
--- a/youtube_dl/extractor/keek.py
+++ b/youtube_dl/extractor/keek.py
@@ -1,8 +1,6 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 
 
@@ -26,10 +24,6 @@ class KeekIE(InfoExtractor):
         video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id)
-        uploader = uploader_id = None
-        matches = re.search(r'data-username="(?P<uploader>[^"]+)"[^>]*data-user-id="(?P<uploader_id>[^"]+)"', webpage)
-        if matches:
-            uploader, uploader_id = matches.groups()
 
         return {
             'id': video_id,
@@ -38,6 +32,6 @@ class KeekIE(InfoExtractor):
             'title': self._og_search_title(webpage),
             'description': self._og_search_description(webpage),
             'thumbnail': self._og_search_thumbnail(webpage),
-            'uploader': uploader,
-            'uploader_id': uploader_id,
+            'uploader': self._search_regex(r'data-username="([^"]+)"', webpage, 'uploader', None),
+            'uploader_id': self._search_regex(r'data-user-id="([^"]+)"', webpage, 'uploader id', None),
         }

From 817690ff736c276835e16860957d758d92afb2fd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 28 Sep 2015 01:05:24 +0600
Subject: [PATCH 2636/6219] [keek] Make uploader fields non fatal

---
 youtube_dl/extractor/keek.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/keek.py b/youtube_dl/extractor/keek.py
index 99154112b..7e1a3f67c 100644
--- a/youtube_dl/extractor/keek.py
+++ b/youtube_dl/extractor/keek.py
@@ -32,6 +32,6 @@ class KeekIE(InfoExtractor):
             'title': self._og_search_title(webpage),
             'description': self._og_search_description(webpage),
             'thumbnail': self._og_search_thumbnail(webpage),
-            'uploader': self._search_regex(r'data-username="([^"]+)"', webpage, 'uploader', None),
-            'uploader_id': self._search_regex(r'data-user-id="([^"]+)"', webpage, 'uploader id', None),
+            'uploader': self._search_regex(r'data-username="([^"]+)"', webpage, 'uploader', fatal=False),
+            'uploader_id': self._search_regex(r'data-user-id="([^"]+)"', webpage, 'uploader id', fatal=False),
         }

From 926fb62eec016db9ef23e9d8d210c8fb2e4de20e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 28 Sep 2015 01:14:14 +0600
Subject: [PATCH 2637/6219] [keek] Remove description

Since it equals title plus static suffix
---
 youtube_dl/extractor/keek.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/keek.py b/youtube_dl/extractor/keek.py
index 7e1a3f67c..dff4cb558 100644
--- a/youtube_dl/extractor/keek.py
+++ b/youtube_dl/extractor/keek.py
@@ -13,8 +13,7 @@ class KeekIE(InfoExtractor):
         'info_dict': {
             'id': 'NODfbab',
             'ext': 'mp4',
-            'title': 'test chars: "\'/\\ä<>This is a test video for youtube-dl.For more information, contact phihag@phihag.de . - Video - Videos on Keek',
-            'description': 'md5:35d42050a3ece241d5ddd7fdcc6fd896',
+            'title': 'md5:35d42050a3ece241d5ddd7fdcc6fd896',
             'uploader': 'ytdl',
             'uploader_id': 'eGT5bab',
         },
@@ -29,8 +28,7 @@ class KeekIE(InfoExtractor):
             'id': video_id,
             'url': self._og_search_video_url(webpage),
             'ext': 'mp4',
-            'title': self._og_search_title(webpage),
-            'description': self._og_search_description(webpage),
+            'title': self._og_search_description(webpage),
             'thumbnail': self._og_search_thumbnail(webpage),
             'uploader': self._search_regex(r'data-username="([^"]+)"', webpage, 'uploader', fatal=False),
             'uploader_id': self._search_regex(r'data-user-id="([^"]+)"', webpage, 'uploader id', fatal=False),

From 1ab1c4ef5708fc1db50c5b875dcbcfe9ce984035 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 28 Sep 2015 01:15:13 +0600
Subject: [PATCH 2638/6219] [keek] Improve uploader fields regexes

---
 youtube_dl/extractor/keek.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/keek.py b/youtube_dl/extractor/keek.py
index dff4cb558..dfd800a91 100644
--- a/youtube_dl/extractor/keek.py
+++ b/youtube_dl/extractor/keek.py
@@ -30,6 +30,10 @@ class KeekIE(InfoExtractor):
             'ext': 'mp4',
             'title': self._og_search_description(webpage),
             'thumbnail': self._og_search_thumbnail(webpage),
-            'uploader': self._search_regex(r'data-username="([^"]+)"', webpage, 'uploader', fatal=False),
-            'uploader_id': self._search_regex(r'data-user-id="([^"]+)"', webpage, 'uploader id', fatal=False),
+            'uploader': self._search_regex(
+                r'data-username=(["\'])(?P<uploader>.+?)\1', webpage,
+                'uploader', fatal=False, group='uploader'),
+            'uploader_id': self._search_regex(
+                r'data-user-id=(["\'])(?P<uploader_id>.+?)\1', webpage,
+                'uploader id', fatal=False, group='uploader_id'),
         }

From 86692c019cb2baeacfdf556ea6ca026dd4ac49e3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 28 Sep 2015 01:17:28 +0600
Subject: [PATCH 2639/6219] [keek] Strip title

---
 youtube_dl/extractor/keek.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/keek.py b/youtube_dl/extractor/keek.py
index dfd800a91..94a03d277 100644
--- a/youtube_dl/extractor/keek.py
+++ b/youtube_dl/extractor/keek.py
@@ -28,7 +28,7 @@ class KeekIE(InfoExtractor):
             'id': video_id,
             'url': self._og_search_video_url(webpage),
             'ext': 'mp4',
-            'title': self._og_search_description(webpage),
+            'title': self._og_search_description(webpage).strip(),
             'thumbnail': self._og_search_thumbnail(webpage),
             'uploader': self._search_regex(
                 r'data-username=(["\'])(?P<uploader>.+?)\1', webpage,

From c254f75bbb68c4d6fdb9e63bfa399d7eff7b2c23 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 28 Sep 2015 04:42:11 +0200
Subject: [PATCH 2640/6219] release 2015.09.28

---
 docs/supportedsites.md | 4 +---
 youtube_dl/version.py  | 2 +-
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index ab153af6b..fa83b68ad 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -101,7 +101,7 @@
  - **ComCarCoff**
  - **ComedyCentral**
  - **ComedyCentralShows**: The Daily Show / The Colbert Report
- - **CondeNast**: Condé Nast media group: Condé Nast, GQ, Glamour, Vanity Fair, Vogue, W Magazine, WIRED
+ - **CondeNast**: Condé Nast media group: Allure, Architectural Digest, Ars Technica, Bon Appétit, Brides, Condé Nast, Condé Nast Traveler, Details, Epicurious, GQ, Glamour, Golf Digest, SELF, Teen Vogue, The New Yorker, Vanity Fair, Vogue, W Magazine, WIRED
  - **Cracked**
  - **Criterion**
  - **CrooksAndLiars**
@@ -158,7 +158,6 @@
  - **faz.net**
  - **fc2**
  - **fernsehkritik.tv**
- - **fernsehkritik.tv:postecke**
  - **Firstpost**
  - **FiveTV**
  - **Flickr**
@@ -208,7 +207,6 @@
  - **hitbox**
  - **hitbox:live**
  - **HornBunny**
- - **HostingBulk**
  - **HotNewHipHop**
  - **Howcast**
  - **HowStuffWorks**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 7ef4f2755..8f0977849 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.09.22'
+__version__ = '2015.09.28'

From 8a64969404ecbbac9cb8b970f7ca36a9e6c03cf2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 29 Sep 2015 21:33:21 +0600
Subject: [PATCH 2641/6219] [adultswim] Prefer stream (Closes #7015)

---
 youtube_dl/extractor/adultswim.py | 37 ++++++++++++++++++++++---------
 1 file changed, 26 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/adultswim.py b/youtube_dl/extractor/adultswim.py
index 4327c2f61..27de07587 100644
--- a/youtube_dl/extractor/adultswim.py
+++ b/youtube_dl/extractor/adultswim.py
@@ -5,6 +5,7 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
+    determine_ext,
     ExtractorError,
     float_or_none,
     xpath_text,
@@ -123,7 +124,6 @@ class AdultSwimIE(InfoExtractor):
         else:
             collections = bootstrapped_data['show']['collections']
             collection, video_info = self.find_collection_containing_video(collections, episode_path)
-
             # Video wasn't found in the collections, let's try `slugged_video`.
             if video_info is None:
                 if bootstrapped_data.get('slugged_video', {}).get('slug') == episode_path:
@@ -133,7 +133,9 @@ class AdultSwimIE(InfoExtractor):
 
             show = bootstrapped_data['show']
             show_title = show['title']
-            segment_ids = [clip['videoPlaybackID'] for clip in video_info['clips']]
+            stream = video_info.get('stream')
+            clips = [stream] if stream else video_info['clips']
+            segment_ids = [clip['videoPlaybackID'] for clip in clips]
 
         episode_id = video_info['id']
         episode_title = video_info['title']
@@ -142,7 +144,7 @@ class AdultSwimIE(InfoExtractor):
 
         entries = []
         for part_num, segment_id in enumerate(segment_ids):
-            segment_url = 'http://www.adultswim.com/videos/api/v0/assets?id=%s&platform=mobile' % segment_id
+            segment_url = 'http://www.adultswim.com/videos/api/v0/assets?id=%s&platform=desktop' % segment_id
 
             segment_title = '%s - %s' % (show_title, episode_title)
             if len(segment_ids) > 1:
@@ -158,17 +160,30 @@ class AdultSwimIE(InfoExtractor):
             formats = []
             file_els = idoc.findall('.//files/file') or idoc.findall('./files/file')
 
+            unique_urls = []
+            unique_file_els = []
             for file_el in file_els:
+                media_url = file_el.text
+                if not media_url or determine_ext(media_url) == 'f4m':
+                    continue
+                if file_el.text not in unique_urls:
+                    unique_urls.append(file_el.text)
+                    unique_file_els.append(file_el)
+
+            for file_el in unique_file_els:
                 bitrate = file_el.attrib.get('bitrate')
                 ftype = file_el.attrib.get('type')
-
-                formats.append({
-                    'format_id': '%s_%s' % (bitrate, ftype),
-                    'url': file_el.text.strip(),
-                    # The bitrate may not be a number (for example: 'iphone')
-                    'tbr': int(bitrate) if bitrate.isdigit() else None,
-                    'quality': 1 if ftype == 'hd' else -1
-                })
+                media_url = file_el.text
+                if determine_ext(media_url) == 'm3u8':
+                    formats.extend(self._extract_m3u8_formats(
+                        media_url, segment_title, 'mp4', 'm3u8_native', preference=0, m3u8_id='hls'))
+                else:
+                    formats.append({
+                        'format_id': '%s_%s' % (bitrate, ftype),
+                        'url': file_el.text.strip(),
+                        # The bitrate may not be a number (for example: 'iphone')
+                        'tbr': int(bitrate) if bitrate.isdigit() else None,
+                    })
 
             self._sort_formats(formats)
 

From c01e1a96aa964ef6d5f0bf7675dbe34096b1d2c8 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 30 Sep 2015 11:20:43 +0100
Subject: [PATCH 2642/6219] [brightcove] fix test and fields extraction

---
 youtube_dl/extractor/brightcove.py | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index e4a7befee..b41cee91b 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -354,14 +354,18 @@ class BrightcoveIE(InfoExtractor):
 
 class BrightcoveInPageEmbedIE(InfoExtractor):
     _VALID_URL = r'https?://players\.brightcove\.net/(?P<account_id>\d+)/([a-z0-9-]+)_([a-z]+)/index.html?.*videoId=(?P<video_id>\d+)'
-    TEST = {
+    _TEST = {
         'url': 'http://players.brightcove.net/929656772001/e41d32dc-ec74-459e-a845-6c69f7b724ea_default/index.html?videoId=4463358922001',
+        'md5': 'c8100925723840d4b0d243f7025703be',
         'info_dict': {
             'id': '4463358922001',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Meet the man behind Popcorn Time',
-            'description': 'md5:a950cc4285c43e44d763d036710cd9cd',
+            'description': 'md5:eac376a4fe366edc70279bfb681aea16',
+            'timestamp': 1441391203,
+            'upload_date': '20150904',
             'duration': 165768,
+            'uploader_id': '929656772001',
         }
     }
 
@@ -403,7 +407,7 @@ class BrightcoveInPageEmbedIE(InfoExtractor):
 
         title = json_data['name']
         description = json_data.get('description')
-        thumbnail = json_data.get('name')
+        thumbnail = json_data.get('thumbnail')
         timestamp = parse_iso8601(json_data.get('published_at'))
         duration = int_or_none(json_data.get('duration'))
 
@@ -417,12 +421,13 @@ class BrightcoveInPageEmbedIE(InfoExtractor):
                 if src:
                     formats.append({
                         'url': src,
-                        'abr': source.get('avg_bitrate'),
+                        'tbr': source.get('avg_bitrate'),
                         'width': int_or_none(source.get('width')),
                         'height': int_or_none(source.get('height')),
                         'filesize': source.get('size'),
                         'container': source.get('container'),
-                        'vcodec': source.get('container'),
+                        'vcodec': source.get('codec'),
+                        'ext': source.get('container').lower(),
                     })
 
         self._sort_formats(formats)
@@ -435,4 +440,5 @@ class BrightcoveInPageEmbedIE(InfoExtractor):
             'timestamp': timestamp,
             'duration': duration,
             'formats': formats,
+            'uploader_id': account_id,
         }

From c596ce91cd22914e031b9ac94ec38d1137e50d60 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 30 Sep 2015 15:39:47 +0200
Subject: [PATCH 2643/6219] [comedycentral] Fix youtube-dl :thedailyshow

We'll let the generic IE follow the redirect and call back to us with the episode URL
---
 youtube_dl/extractor/comedycentral.py | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/comedycentral.py b/youtube_dl/extractor/comedycentral.py
index 91ebb0ce5..3e4bd10b6 100644
--- a/youtube_dl/extractor/comedycentral.py
+++ b/youtube_dl/extractor/comedycentral.py
@@ -151,12 +151,7 @@ class ComedyCentralShowsIE(MTVServicesInfoExtractor):
         mobj = re.match(self._VALID_URL, url)
 
         if mobj.group('shortname'):
-            if mobj.group('shortname') in ('tds', 'thedailyshow'):
-                url = 'http://thedailyshow.cc.com/full-episodes/'
-            else:
-                url = 'http://thecolbertreport.cc.com/full-episodes/'
-            mobj = re.match(self._VALID_URL, url, re.VERBOSE)
-            assert mobj is not None
+            return self.url_result('http://www.cc.com/shows/the-daily-show-with-trevor-noah/full-episodes')
 
         if mobj.group('clip'):
             if mobj.group('videotitle'):

From aedb930cfcd068600d3b5b0b3aef32df1a28f0c0 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 30 Sep 2015 13:30:48 +0100
Subject: [PATCH 2644/6219] [nfl] fix content id regex(fixes #7012)

---
 youtube_dl/extractor/nfl.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nfl.py b/youtube_dl/extractor/nfl.py
index 55dc6107d..4c35c561b 100644
--- a/youtube_dl/extractor/nfl.py
+++ b/youtube_dl/extractor/nfl.py
@@ -151,7 +151,7 @@ class NFLIE(InfoExtractor):
             group='config'))
         # For articles, the id in the url is not the video id
         video_id = self._search_regex(
-            r'(?:<nflcs:avplayer[^>]+data-contentId\s*=\s*|contentId\s*:\s*)(["\'])(?P<id>.+?)\1',
+            r'(?:<nflcs:avplayer[^>]+data-content[Ii]d\s*=\s*|content[Ii]d\s*:\s*)(["\'])(?P<id>.+?)\1',
             webpage, 'video id', default=video_id, group='id')
         config = self._download_json(config_url, video_id, 'Downloading player config')
         url_template = NFLIE.prepend_host(

From ee2d190253ae3e0a8254029dc6e8d7e42c194042 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 30 Sep 2015 20:06:21 +0600
Subject: [PATCH 2645/6219] [nfl] Add test for #7012

---
 youtube_dl/extractor/nfl.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/youtube_dl/extractor/nfl.py b/youtube_dl/extractor/nfl.py
index 4c35c561b..200874d68 100644
--- a/youtube_dl/extractor/nfl.py
+++ b/youtube_dl/extractor/nfl.py
@@ -107,6 +107,20 @@ class NFLIE(InfoExtractor):
             'timestamp': 1442618809,
             'upload_date': '20150918',
         },
+    }, {
+        # lowercase data-contentid
+        'url': 'http://www.steelers.com/news/article-1/Tomlin-on-Ben-getting-Vick-ready/56399c96-4160-48cf-a7ad-1d17d4a3aef7',
+        'info_dict': {
+            'id': '12693586-6ea9-4743-9c1c-02c59e4a5ef2',
+            'ext': 'mp4',
+            'title': 'Tomlin looks ahead to Ravens on a short week',
+            'description': 'md5:32f3f7b139f43913181d5cbb24ecad75',
+            'timestamp': 1443459651,
+            'upload_date': '20150928',
+        },
+        'params': {
+            'skip_download': True,
+        },
     }, {
         'url': 'http://www.nfl.com/videos/nfl-network-top-ten/09000d5d810a6bd4/Top-10-Gutsiest-Performances-Jack-Youngblood',
         'only_matching': True,

From 93bc7ef165731c729fd6fbbb3e95ecbbb16b4c21 Mon Sep 17 00:00:00 2001
From: Qijiang Fan <fqj1994@gmail.com>
Date: Sun, 30 Aug 2015 14:33:12 +0800
Subject: [PATCH 2646/6219] [test] recursively check dict and list in
 expect_info_dict

This allows to use md5:, re:, etc within the str inside a list
or dict.
---
 test/helper.py | 21 ++++++++++++++++++++-
 1 file changed, 20 insertions(+), 1 deletion(-)

diff --git a/test/helper.py b/test/helper.py
index cb6eec8d9..6612970ef 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -89,7 +89,7 @@ def gettestcases(include_onlymatching=False):
 md5 = lambda s: hashlib.md5(s.encode('utf-8')).hexdigest()
 
 
-def expect_info_dict(self, got_dict, expected_dict):
+def expect_dict(self, got_dict, expected_dict):
     for info_field, expected in expected_dict.items():
         if isinstance(expected, compat_str) and expected.startswith('re:'):
             got = got_dict.get(info_field)
@@ -127,6 +127,22 @@ def expect_info_dict(self, got_dict, expected_dict):
             got = got_dict.get(info_field)
             self.assertTrue(isinstance(got, expected),
                             'Expected type %r for field %s, but got value %r of type %r' % (expected, info_field, got, type(got)))
+        elif isinstance(expected, dict) and isinstance(got_dict.get(info_field, None), dict):
+            expect_dict(self, got_dict.get(info_field), expected)
+        elif isinstance(expected, list) and isinstance(got_dict.get(info_field, None), list):
+            got = got_dict.get(info_field, None)
+            self.assertEqual(len(expected), len(got),
+                             'Expect a list of length %d, but got a list of length %d' % (
+                             len(expected), len(got)))
+            _id = 0
+            for i, j in zip(got, expected):
+                _type_i = type(i)
+                _type_j = type(j)
+                self.assertEqual(_type_j, _type_i,
+                                 'Type doesn\'t match at element %d of the list in field %s, expect %s, got %s' % (
+                                 _id, info_field, _type_j, _type_i))
+                expect_dict(self, {'_': i}, {'_': j})
+                _id += 1
         else:
             if isinstance(expected, compat_str) and expected.startswith('md5:'):
                 got = 'md5:' + md5(got_dict.get(info_field))
@@ -149,6 +165,9 @@ def expect_info_dict(self, got_dict, expected_dict):
             self.assertEqual(expected, got,
                              'invalid value for field %s, expected %r, got %r' % (info_field, expected, got))
 
+
+def expect_info_dict(self, got_dict, expected_dict):
+    expect_dict(self, got_dict, expected_dict)
     # Check for the presence of mandatory fields
     if got_dict.get('_type') not in ('playlist', 'multi_video'):
         for key in ('id', 'url', 'title', 'ext'):

From 40c931de4b3c06c1959362c2380f4bf243ef5702 Mon Sep 17 00:00:00 2001
From: Qijiang Fan <fqj1994@gmail.com>
Date: Sat, 26 Sep 2015 23:10:38 +0800
Subject: [PATCH 2647/6219] [test] split expect_dict to two functions

---
 test/helper.py | 143 ++++++++++++++++++++++++-------------------------
 1 file changed, 70 insertions(+), 73 deletions(-)

diff --git a/test/helper.py b/test/helper.py
index 6612970ef..e50d7ff28 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -89,81 +89,78 @@ def gettestcases(include_onlymatching=False):
 md5 = lambda s: hashlib.md5(s.encode('utf-8')).hexdigest()
 
 
+def expect_value(self, got, expected, field):
+    if isinstance(expected, compat_str) and expected.startswith('re:'):
+        match_str = expected[len('re:'):]
+        match_rex = re.compile(match_str)
+
+        self.assertTrue(
+            isinstance(got, compat_str),
+            'Expected a %s object, but got %s for field %s' % (
+                compat_str.__name__, type(got).__name__, field))
+        self.assertTrue(
+            match_rex.match(got),
+            'field %s (value: %r) should match %r' % (field, got, match_str))
+    elif isinstance(expected, compat_str) and expected.startswith('startswith:'):
+        start_str = expected[len('startswith:'):]
+        self.assertTrue(
+            isinstance(got, compat_str),
+            'Expected a %s object, but got %s for field %s' % (
+                compat_str.__name__, type(got).__name__, field))
+        self.assertTrue(
+            got.startswith(start_str),
+            'field %s (value: %r) should start with %r' % (field, got, start_str))
+    elif isinstance(expected, compat_str) and expected.startswith('contains:'):
+        contains_str = expected[len('contains:'):]
+        self.assertTrue(
+            isinstance(got, compat_str),
+            'Expected a %s object, but got %s for field %s' % (
+                compat_str.__name__, type(got).__name__, field))
+        self.assertTrue(
+            contains_str in got,
+            'field %s (value: %r) should contain %r' % (field, got, contains_str))
+    elif isinstance(expected, type):
+        self.assertTrue(isinstance(got, expected),
+                        'Expected type %r for field %s, but got value %r of type %r' % (expected, field, got, type(got)))
+    elif isinstance(expected, dict) and isinstance(got, dict):
+        expect_dict(self, got, expected)
+    elif isinstance(expected, list) and isinstance(got, list):
+        self.assertEqual(len(expected), len(got),
+                         'Expect a list of length %d, but got a list of length %d' % (
+                         len(expected), len(got)))
+        _id = 0
+        for i, j in zip(got, expected):
+            _type_i = type(i)
+            _type_j = type(j)
+            self.assertEqual(_type_j, _type_i,
+                             'Type doesn\'t match at element %d of the list in field %s, expect %s, got %s' % (
+                             _id, field, _type_j, _type_i))
+            expect_value(self, i, j, field)
+            _id += 1
+    else:
+        if isinstance(expected, compat_str) and expected.startswith('md5:'):
+            got = 'md5:' + md5(got)
+        elif isinstance(expected, compat_str) and expected.startswith('mincount:'):
+            self.assertTrue(
+                isinstance(got, (list, dict)),
+                'Expected field %s to be a list or a dict, but it is of type %s' % (
+                    field, type(got).__name__))
+            expected_num = int(expected.partition(':')[2])
+            assertGreaterEqual(
+                self, len(got), expected_num,
+                'Expected %d items in field %s, but only got %d' % (
+                    expected_num, field, len(got)
+                )
+            )
+            return
+        self.assertEqual(expected, got,
+                         'invalid value for field %s, expected %r, got %r' % (field, expected, got))
+
+
 def expect_dict(self, got_dict, expected_dict):
     for info_field, expected in expected_dict.items():
-        if isinstance(expected, compat_str) and expected.startswith('re:'):
-            got = got_dict.get(info_field)
-            match_str = expected[len('re:'):]
-            match_rex = re.compile(match_str)
-
-            self.assertTrue(
-                isinstance(got, compat_str),
-                'Expected a %s object, but got %s for field %s' % (
-                    compat_str.__name__, type(got).__name__, info_field))
-            self.assertTrue(
-                match_rex.match(got),
-                'field %s (value: %r) should match %r' % (info_field, got, match_str))
-        elif isinstance(expected, compat_str) and expected.startswith('startswith:'):
-            got = got_dict.get(info_field)
-            start_str = expected[len('startswith:'):]
-            self.assertTrue(
-                isinstance(got, compat_str),
-                'Expected a %s object, but got %s for field %s' % (
-                    compat_str.__name__, type(got).__name__, info_field))
-            self.assertTrue(
-                got.startswith(start_str),
-                'field %s (value: %r) should start with %r' % (info_field, got, start_str))
-        elif isinstance(expected, compat_str) and expected.startswith('contains:'):
-            got = got_dict.get(info_field)
-            contains_str = expected[len('contains:'):]
-            self.assertTrue(
-                isinstance(got, compat_str),
-                'Expected a %s object, but got %s for field %s' % (
-                    compat_str.__name__, type(got).__name__, info_field))
-            self.assertTrue(
-                contains_str in got,
-                'field %s (value: %r) should contain %r' % (info_field, got, contains_str))
-        elif isinstance(expected, type):
-            got = got_dict.get(info_field)
-            self.assertTrue(isinstance(got, expected),
-                            'Expected type %r for field %s, but got value %r of type %r' % (expected, info_field, got, type(got)))
-        elif isinstance(expected, dict) and isinstance(got_dict.get(info_field, None), dict):
-            expect_dict(self, got_dict.get(info_field), expected)
-        elif isinstance(expected, list) and isinstance(got_dict.get(info_field, None), list):
-            got = got_dict.get(info_field, None)
-            self.assertEqual(len(expected), len(got),
-                             'Expect a list of length %d, but got a list of length %d' % (
-                             len(expected), len(got)))
-            _id = 0
-            for i, j in zip(got, expected):
-                _type_i = type(i)
-                _type_j = type(j)
-                self.assertEqual(_type_j, _type_i,
-                                 'Type doesn\'t match at element %d of the list in field %s, expect %s, got %s' % (
-                                 _id, info_field, _type_j, _type_i))
-                expect_dict(self, {'_': i}, {'_': j})
-                _id += 1
-        else:
-            if isinstance(expected, compat_str) and expected.startswith('md5:'):
-                got = 'md5:' + md5(got_dict.get(info_field))
-            elif isinstance(expected, compat_str) and expected.startswith('mincount:'):
-                got = got_dict.get(info_field)
-                self.assertTrue(
-                    isinstance(got, (list, dict)),
-                    'Expected field %s to be a list or a dict, but it is of type %s' % (
-                        info_field, type(got).__name__))
-                expected_num = int(expected.partition(':')[2])
-                assertGreaterEqual(
-                    self, len(got), expected_num,
-                    'Expected %d items in field %s, but only got %d' % (
-                        expected_num, info_field, len(got)
-                    )
-                )
-                continue
-            else:
-                got = got_dict.get(info_field)
-            self.assertEqual(expected, got,
-                             'invalid value for field %s, expected %r, got %r' % (info_field, expected, got))
+        got = got_dict.get(info_field)
+        expect_value(self, got, expected, info_field)
 
 
 def expect_info_dict(self, got_dict, expected_dict):

From 687c04cbb8d80f76b45caf9377b2d3b64462c203 Mon Sep 17 00:00:00 2001
From: Qijiang Fan <fqj1994@gmail.com>
Date: Wed, 30 Sep 2015 10:30:04 +0200
Subject: [PATCH 2648/6219] [test] use descriptive variable name

---
 test/helper.py | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/test/helper.py b/test/helper.py
index e50d7ff28..5c44105d4 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -128,15 +128,13 @@ def expect_value(self, got, expected, field):
         self.assertEqual(len(expected), len(got),
                          'Expect a list of length %d, but got a list of length %d' % (
                          len(expected), len(got)))
-        _id = 0
-        for i, j in zip(got, expected):
-            _type_i = type(i)
-            _type_j = type(j)
-            self.assertEqual(_type_j, _type_i,
+        for index, (item_got, item_expected) in enumerate(zip(got, expected)):
+            type_got = type(item_got)
+            type_expected = type(item_expected)
+            self.assertEqual(type_expected, type_got,
                              'Type doesn\'t match at element %d of the list in field %s, expect %s, got %s' % (
-                             _id, field, _type_j, _type_i))
-            expect_value(self, i, j, field)
-            _id += 1
+                             index, field, type_expected, type_got))
+            expect_value(self, item_got, item_expected, field)
     else:
         if isinstance(expected, compat_str) and expected.startswith('md5:'):
             got = 'md5:' + md5(got)

From 2e885de796e211300fd70709da17325b84c86dd6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 30 Sep 2015 20:21:01 +0600
Subject: [PATCH 2649/6219] [test/helper] Formatting

---
 test/helper.py | 28 ++++++++++++++--------------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/test/helper.py b/test/helper.py
index 5c44105d4..288ed237d 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -120,20 +120,22 @@ def expect_value(self, got, expected, field):
             contains_str in got,
             'field %s (value: %r) should contain %r' % (field, got, contains_str))
     elif isinstance(expected, type):
-        self.assertTrue(isinstance(got, expected),
-                        'Expected type %r for field %s, but got value %r of type %r' % (expected, field, got, type(got)))
+        self.assertTrue(
+            isinstance(got, expected),
+            'Expected type %r for field %s, but got value %r of type %r' % (expected, field, got, type(got)))
     elif isinstance(expected, dict) and isinstance(got, dict):
         expect_dict(self, got, expected)
     elif isinstance(expected, list) and isinstance(got, list):
-        self.assertEqual(len(expected), len(got),
-                         'Expect a list of length %d, but got a list of length %d' % (
-                         len(expected), len(got)))
+        self.assertEqual(
+            len(expected), len(got),
+            'Expect a list of length %d, but got a list of length %d' % (len(expected), len(got)))
         for index, (item_got, item_expected) in enumerate(zip(got, expected)):
             type_got = type(item_got)
             type_expected = type(item_expected)
-            self.assertEqual(type_expected, type_got,
-                             'Type doesn\'t match at element %d of the list in field %s, expect %s, got %s' % (
-                             index, field, type_expected, type_got))
+            self.assertEqual(
+                type_expected, type_got,
+                'Type doesn\'t match at element %d of the list in field %s, expect %s, got %s' % (
+                index, field, type_expected, type_got))
             expect_value(self, item_got, item_expected, field)
     else:
         if isinstance(expected, compat_str) and expected.startswith('md5:'):
@@ -146,13 +148,11 @@ def expect_value(self, got, expected, field):
             expected_num = int(expected.partition(':')[2])
             assertGreaterEqual(
                 self, len(got), expected_num,
-                'Expected %d items in field %s, but only got %d' % (
-                    expected_num, field, len(got)
-                )
-            )
+                'Expected %d items in field %s, but only got %d' % (expected_num, field, len(got)))
             return
-        self.assertEqual(expected, got,
-                         'invalid value for field %s, expected %r, got %r' % (field, expected, got))
+        self.assertEqual(
+            expected, got,
+            'invalid value for field %s, expected %r, got %r' % (field, expected, got))
 
 
 def expect_dict(self, got_dict, expected_dict):

From 386a7b52d548da0aaf11e5805d67d5ab30d9dfab Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 30 Sep 2015 20:26:42 +0600
Subject: [PATCH 2650/6219] [test/helper] Spelling

---
 test/helper.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/helper.py b/test/helper.py
index 288ed237d..0ce9a0fb3 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -134,7 +134,7 @@ def expect_value(self, got, expected, field):
             type_expected = type(item_expected)
             self.assertEqual(
                 type_expected, type_got,
-                'Type doesn\'t match at element %d of the list in field %s, expect %s, got %s' % (
+                'Type mismatch for list item at index %d for field %s, expected %r, got %r' % (
                 index, field, type_expected, type_got))
             expect_value(self, item_got, item_expected, field)
     else:
@@ -152,7 +152,7 @@ def expect_value(self, got, expected, field):
             return
         self.assertEqual(
             expected, got,
-            'invalid value for field %s, expected %r, got %r' % (field, expected, got))
+            'Invalid value for field %s, expected %r, got %r' % (field, expected, got))
 
 
 def expect_dict(self, got_dict, expected_dict):

From f88f1b40ce7f70a3a510259154364a25d7dceff6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 30 Sep 2015 20:31:29 +0600
Subject: [PATCH 2651/6219] [test/helper] Clarify field for list length
 mismatch

---
 test/helper.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/test/helper.py b/test/helper.py
index 0ce9a0fb3..28fd135b2 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -128,7 +128,8 @@ def expect_value(self, got, expected, field):
     elif isinstance(expected, list) and isinstance(got, list):
         self.assertEqual(
             len(expected), len(got),
-            'Expect a list of length %d, but got a list of length %d' % (len(expected), len(got)))
+            'Expect a list of length %d, but got a list of length %d for field %s' % (
+                len(expected), len(got), field))
         for index, (item_got, item_expected) in enumerate(zip(got, expected)):
             type_got = type(item_got)
             type_expected = type(item_expected)

From 8466336104164b37b3f32c32278e2426590db42f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 30 Sep 2015 22:12:52 +0600
Subject: [PATCH 2652/6219] [vk] Detect vimeo embeds (Closes #7021)

---
 youtube_dl/extractor/vk.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index c30c5a8e5..765e9e6fd 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -17,6 +17,7 @@ from ..utils import (
     unescapeHTML,
     unified_strdate,
 )
+from .vimeo import VimeoIE
 
 
 class VKIE(InfoExtractor):
@@ -249,6 +250,10 @@ class VKIE(InfoExtractor):
         if youtube_url:
             return self.url_result(youtube_url, 'Youtube')
 
+        vimeo_url = VimeoIE._extract_vimeo_url(url, info_page)
+        if vimeo_url is not None:
+            return self.url_result(vimeo_url)
+
         m_rutube = re.search(
             r'\ssrc="((?:https?:)?//rutube\.ru\\?/video\\?/embed(?:.*?))\\?"', info_page)
         if m_rutube is not None:

From f540b937062a4de2cd0a097245ec82dd9fd23277 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 1 Oct 2015 02:33:48 +0600
Subject: [PATCH 2653/6219] [naver] Improve error regex

---
 youtube_dl/extractor/naver.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/naver.py b/youtube_dl/extractor/naver.py
index 925967753..35cbb3e6d 100644
--- a/youtube_dl/extractor/naver.py
+++ b/youtube_dl/extractor/naver.py
@@ -46,11 +46,11 @@ class NaverIE(InfoExtractor):
         m_id = re.search(r'var rmcPlayer = new nhn.rmcnmv.RMCVideoPlayer\("(.+?)", "(.+?)"',
                          webpage)
         if m_id is None:
-            m_error = re.search(
-                r'(?s)<div class="(?:nation_error|nation_box)">\s*(?:<!--.*?-->)?\s*<p class="[^"]+">(?P<msg>.+?)</p>\s*</div>',
-                webpage)
-            if m_error:
-                raise ExtractorError(clean_html(m_error.group('msg')), expected=True)
+            error = self._html_search_regex(
+                r'(?s)<div class="(?:nation_error|nation_box|error_box)">\s*(?:<!--.*?-->)?\s*<p class="[^"]+">(?P<msg>.+?)</p>\s*</div>',
+                webpage, 'error', default=None)
+            if error:
+                raise ExtractorError(error, expected=True)
             raise ExtractorError('couldn\'t extract vid and key')
         vid = m_id.group(1)
         key = m_id.group(2)

From b83831df1fb0dd241210a2c992ac9753c5df59cd Mon Sep 17 00:00:00 2001
From: Joakim Fremstad <jfremstad@users.noreply.github.com>
Date: Thu, 1 Oct 2015 14:58:49 +0200
Subject: [PATCH 2654/6219] [nrk] Spelling

---
 youtube_dl/extractor/nrk.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index d066a96db..5ed235f1e 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -49,7 +49,7 @@ class NRKIE(InfoExtractor):
 
         if data['usageRights']['isGeoBlocked']:
             raise ExtractorError(
-                'NRK har ikke rettig-heter til å vise dette programmet utenfor Norge',
+                'NRK har ikke rettigheter til å vise dette programmet utenfor Norge',
                 expected=True)
 
         video_url = data['mediaUrl'] + '?hdcore=3.5.0&plugin=aasp-3.5.0.151.81'

From 2b3f951a2e2e01adfb068eac05b976ff445db073 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 1 Oct 2015 20:33:17 +0600
Subject: [PATCH 2655/6219] [nrktv] Rework subtitles and eliminate downloading
 twice

---
 youtube_dl/extractor/nrk.py | 28 +++++++++-------------------
 1 file changed, 9 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index 5ed235f1e..8ac38a174 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import compat_urlparse
 from ..utils import (
     ExtractorError,
     float_or_none,
@@ -196,20 +197,6 @@ class NRKTVIE(InfoExtractor):
         }
     ]
 
-    def _debug_print(self, txt):
-        if self._downloader.params.get('verbose', False):
-            self.to_screen('[debug] %s' % txt)
-
-    def _get_subtitles(self, subtitlesurl, video_id, baseurl):
-        url = "%s%s" % (baseurl, subtitlesurl)
-        self._debug_print('%s: Subtitle url: %s' % (video_id, url))
-        captions = self._download_xml(
-            url, video_id, 'Downloading subtitles')
-        lang = captions.get('lang', 'no')
-        return {lang: [
-            {'ext': 'ttml', 'url': url},
-        ]}
-
     def _extract_f4m(self, manifest_url, video_id):
         return self._extract_f4m_formats(
             manifest_url + '?hdcore=3.1.1&plugin=aasp-3.1.1.69.124', video_id, f4m_id='hds')
@@ -218,7 +205,7 @@ class NRKTVIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
         part_id = mobj.group('part_id')
-        baseurl = mobj.group('baseurl')
+        base_url = mobj.group('baseurl')
 
         webpage = self._download_webpage(url, video_id)
 
@@ -278,11 +265,14 @@ class NRKTVIE(InfoExtractor):
         self._sort_formats(formats)
 
         subtitles_url = self._html_search_regex(
-            r'data-subtitlesurl[ ]*=[ ]*"([^"]+)"',
-            webpage, 'subtitle URL', default=None)
-        subtitles = None
+            r'data-subtitlesurl\s*=\s*(["\'])(?P<url>.+?)\1',
+            webpage, 'subtitle URL', default=None, group='url')
+        subtitles = {}
         if subtitles_url:
-            subtitles = self.extract_subtitles(subtitles_url, video_id, baseurl)
+            subtitles['no'] = [{
+                'ext': 'ttml',
+                'url': compat_urlparse.urljoin(base_url, subtitles_url),
+            }]
 
         return {
             'id': video_id,

From e7d8e98a9ffdec2502bedb21a4f043df6da225a5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 1 Oct 2015 21:51:04 +0600
Subject: [PATCH 2656/6219] [extractor/common] Allow float bitrates

---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 2fe0d5d37..9c40d56a9 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1084,7 +1084,7 @@ class InfoExtractor(object):
             if not src:
                 continue
 
-            bitrate = int_or_none(video.get('system-bitrate') or video.get('systemBitrate'), 1000)
+            bitrate = float_or_none(video.get('system-bitrate') or video.get('systemBitrate'), 1000)
             filesize = int_or_none(video.get('size') or video.get('fileSize'))
             width = int_or_none(video.get('width'))
             height = int_or_none(video.get('height'))

From 1e5bcdec0264190ed2a05ee49c1f9f5b20ba3aa6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 1 Oct 2015 22:08:16 +0600
Subject: [PATCH 2657/6219] [extractor/common] Extract images from SMIL

---
 youtube_dl/extractor/common.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 9c40d56a9..5684227dc 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1054,10 +1054,18 @@ class InfoExtractor(object):
             elif not description and name in ('description', 'abstract'):
                 description = content
 
+        thumbnails = [{
+            'id': image.get('type'),
+            'url': image.get('src'),
+            'width': int_or_none(image.get('width')),
+            'height': int_or_none(image.get('height')),
+        } for image in smil.findall(self._xpath_ns('.//image', namespace)) if image.get('src')]
+
         return {
             'id': video_id,
             'title': title or video_id,
             'description': description,
+            'thumbnails': thumbnails,
             'formats': formats,
             'subtitles': subtitles,
         }

From 647eab4541d1f55dea6aa20af9b35c2726dda48a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 1 Oct 2015 22:18:59 +0600
Subject: [PATCH 2658/6219] [extractor/common] Extract upload date from SMIL

---
 youtube_dl/extractor/common.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 5684227dc..b86d06523 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -39,6 +39,7 @@ from ..utils import (
     RegexNotFoundError,
     sanitize_filename,
     unescapeHTML,
+    unified_strdate,
     url_basename,
     xpath_text,
     xpath_with_ns,
@@ -1044,6 +1045,7 @@ class InfoExtractor(object):
         video_id = os.path.splitext(url_basename(smil_url))[0]
         title = None
         description = None
+        upload_date = None
         for meta in smil.findall(self._xpath_ns('./head/meta', namespace)):
             name = meta.attrib.get('name')
             content = meta.attrib.get('content')
@@ -1053,6 +1055,8 @@ class InfoExtractor(object):
                 title = content
             elif not description and name in ('description', 'abstract'):
                 description = content
+            elif not upload_date and name == 'date':
+                upload_date = unified_strdate(content)
 
         thumbnails = [{
             'id': image.get('type'),
@@ -1065,6 +1069,7 @@ class InfoExtractor(object):
             'id': video_id,
             'title': title or video_id,
             'description': description,
+            'upload_date': upload_date,
             'thumbnails': thumbnails,
             'formats': formats,
             'subtitles': subtitles,

From acfb717a18cfa9c0f377372068e16862bba345b4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 1 Oct 2015 22:19:39 +0600
Subject: [PATCH 2659/6219] [videolecturesnet] Use generic SMIL extraction

---
 youtube_dl/extractor/videolecturesnet.py | 62 +++---------------------
 1 file changed, 8 insertions(+), 54 deletions(-)

diff --git a/youtube_dl/extractor/videolecturesnet.py b/youtube_dl/extractor/videolecturesnet.py
index ef2da5632..160dbb590 100644
--- a/youtube_dl/extractor/videolecturesnet.py
+++ b/youtube_dl/extractor/videolecturesnet.py
@@ -1,13 +1,8 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..utils import (
-    find_xpath_attr,
-    int_or_none,
     parse_duration,
-    unified_strdate,
 )
 
 
@@ -29,58 +24,17 @@ class VideoLecturesNetIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         smil_url = 'http://videolectures.net/%s/video/1/smil.xml' % video_id
-        smil = self._download_xml(smil_url, video_id)
+        smil = self._download_smil(smil_url, video_id)
 
-        title = find_xpath_attr(smil, './/meta', 'name', 'title').attrib['content']
-        description_el = find_xpath_attr(smil, './/meta', 'name', 'abstract')
-        description = (
-            None if description_el is None
-            else description_el.attrib['content'])
-        upload_date = unified_strdate(
-            find_xpath_attr(smil, './/meta', 'name', 'date').attrib['content'])
+        info = self._parse_smil(smil, smil_url, video_id)
+
+        info['id'] = video_id
 
         switch = smil.find('.//switch')
-        duration = parse_duration(switch.attrib.get('dur'))
-        thumbnail_el = find_xpath_attr(switch, './image', 'type', 'thumbnail')
-        thumbnail = (
-            None if thumbnail_el is None else thumbnail_el.attrib.get('src'))
+        if switch is not None:
+            info['duration'] = parse_duration(switch.attrib.get('dur'))
 
-        formats = []
-        for v in switch.findall('./video'):
-            proto = v.attrib.get('proto')
-            if proto not in ['http', 'rtmp']:
-                continue
-            f = {
-                'width': int_or_none(v.attrib.get('width')),
-                'height': int_or_none(v.attrib.get('height')),
-                'filesize': int_or_none(v.attrib.get('size')),
-                'tbr': int_or_none(v.attrib.get('systemBitrate')) / 1000.0,
-                'ext': v.attrib.get('ext'),
-            }
-            src = v.attrib['src']
-            if proto == 'http':
-                if self._is_valid_url(src, video_id):
-                    f['url'] = src
-                    formats.append(f)
-            elif proto == 'rtmp':
-                f.update({
-                    'url': v.attrib['streamer'],
-                    'play_path': src,
-                    'rtmp_real_time': True,
-                })
-                formats.append(f)
-        self._sort_formats(formats)
-
-        return {
-            'id': video_id,
-            'title': title,
-            'description': description,
-            'upload_date': upload_date,
-            'duration': duration,
-            'thumbnail': thumbnail,
-            'formats': formats,
-        }
+        return info

From 0c996b9f488bfaa74d79e94739af80a0be38e125 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 1 Oct 2015 22:39:38 +0600
Subject: [PATCH 2660/6219] [videolecturesnet] Add support for playlists
 (Closes #7031)

---
 youtube_dl/extractor/videolecturesnet.py | 19 +++++++++++++++----
 1 file changed, 15 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/videolecturesnet.py b/youtube_dl/extractor/videolecturesnet.py
index 160dbb590..eadff8d18 100644
--- a/youtube_dl/extractor/videolecturesnet.py
+++ b/youtube_dl/extractor/videolecturesnet.py
@@ -1,9 +1,10 @@
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
-from ..utils import (
-    parse_duration,
-)
+from ..compat import compat_urlparse
+from ..utils import parse_duration
 
 
 class VideoLecturesNetIE(InfoExtractor):
@@ -27,7 +28,17 @@ class VideoLecturesNetIE(InfoExtractor):
         video_id = self._match_id(url)
 
         smil_url = 'http://videolectures.net/%s/video/1/smil.xml' % video_id
-        smil = self._download_smil(smil_url, video_id)
+        smil = self._download_smil(smil_url, video_id, fatal=False)
+
+        # Probably a playlist
+        if smil is False:
+            webpage = self._download_webpage(url, video_id)
+            entries = [
+                self.url_result(compat_urlparse.urljoin(url, video_url), 'VideoLecturesNet')
+                for _, video_url in re.findall(r'<a[^>]+href=(["\'])(.+?)\1[^>]+id=["\']lec=\d+', webpage)]
+            playlist_title = self._html_search_meta('title', webpage, 'title', fatal=True)
+            playlist_description = self._html_search_meta('description', webpage, 'description')
+            return self.playlist_result(entries, video_id, playlist_title, playlist_description)
 
         info = self._parse_smil(smil, smil_url, video_id)
 

From fb97809e64a145768346345ff2b5e5539880c014 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 1 Oct 2015 22:44:51 +0600
Subject: [PATCH 2661/6219] [videolecturesnet] Improve playlist extraction

---
 youtube_dl/extractor/videolecturesnet.py | 32 +++++++++++++++---------
 1 file changed, 20 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/videolecturesnet.py b/youtube_dl/extractor/videolecturesnet.py
index eadff8d18..704165406 100644
--- a/youtube_dl/extractor/videolecturesnet.py
+++ b/youtube_dl/extractor/videolecturesnet.py
@@ -3,8 +3,14 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urlparse
-from ..utils import parse_duration
+from ..compat import (
+    compat_HTTPError,
+    compat_urlparse,
+)
+from ..utils import (
+    ExtractorError,
+    parse_duration,
+)
 
 
 class VideoLecturesNetIE(InfoExtractor):
@@ -28,17 +34,19 @@ class VideoLecturesNetIE(InfoExtractor):
         video_id = self._match_id(url)
 
         smil_url = 'http://videolectures.net/%s/video/1/smil.xml' % video_id
-        smil = self._download_smil(smil_url, video_id, fatal=False)
 
-        # Probably a playlist
-        if smil is False:
-            webpage = self._download_webpage(url, video_id)
-            entries = [
-                self.url_result(compat_urlparse.urljoin(url, video_url), 'VideoLecturesNet')
-                for _, video_url in re.findall(r'<a[^>]+href=(["\'])(.+?)\1[^>]+id=["\']lec=\d+', webpage)]
-            playlist_title = self._html_search_meta('title', webpage, 'title', fatal=True)
-            playlist_description = self._html_search_meta('description', webpage, 'description')
-            return self.playlist_result(entries, video_id, playlist_title, playlist_description)
+        try:
+            smil = self._download_smil(smil_url, video_id)
+        except ExtractorError as e:
+            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 404:
+                # Probably a playlist
+                webpage = self._download_webpage(url, video_id)
+                entries = [
+                    self.url_result(compat_urlparse.urljoin(url, video_url), 'VideoLecturesNet')
+                    for _, video_url in re.findall(r'<a[^>]+href=(["\'])(.+?)\1[^>]+id=["\']lec=\d+', webpage)]
+                playlist_title = self._html_search_meta('title', webpage, 'title', fatal=True)
+                playlist_description = self._html_search_meta('description', webpage, 'description')
+                return self.playlist_result(entries, video_id, playlist_title, playlist_description)
 
         info = self._parse_smil(smil, smil_url, video_id)
 

From 6edaa0e25b880be5d1ea513d9d907c91e18a8347 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 1 Oct 2015 22:45:10 +0600
Subject: [PATCH 2662/6219] [videolecturesnet] Add playlist test

---
 youtube_dl/extractor/videolecturesnet.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/videolecturesnet.py b/youtube_dl/extractor/videolecturesnet.py
index 704165406..113a2289b 100644
--- a/youtube_dl/extractor/videolecturesnet.py
+++ b/youtube_dl/extractor/videolecturesnet.py
@@ -17,7 +17,7 @@ class VideoLecturesNetIE(InfoExtractor):
     _VALID_URL = r'http://(?:www\.)?videolectures\.net/(?P<id>[^/#?]+)/*(?:[#?].*)?$'
     IE_NAME = 'videolectures.net'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://videolectures.net/promogram_igor_mekjavic_eng/',
         'info_dict': {
             'id': 'promogram_igor_mekjavic_eng',
@@ -28,7 +28,15 @@ class VideoLecturesNetIE(InfoExtractor):
             'duration': 565,
             'thumbnail': 're:http://.*\.jpg',
         },
-    }
+    }, {
+        'url': 'http://videolectures.net/deeplearning2015_montreal/',
+        'info_dict': {
+            'id': 'deeplearning2015_montreal',
+            'title': 'Deep Learning Summer School, Montreal 2015',
+            'description': 'md5:90121a40cc6926df1bf04dcd8563ed3b',
+        },
+        'playlist_count': 30,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From c78e48177c72baa0c985c4d47f95e6edc6ec33dc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 1 Oct 2015 22:54:54 +0600
Subject: [PATCH 2663/6219] [extractor/common] Check validity of direct URLs

---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index b86d06523..b928e24be 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1145,7 +1145,7 @@ class InfoExtractor(object):
                 formats.extend(self._extract_f4m_formats(f4m_url, video_id, f4m_id='hds'))
                 continue
 
-            if src_url.startswith('http'):
+            if src_url.startswith('http') and self._is_valid_url(src, video_id):
                 http_count += 1
                 formats.append({
                     'url': src_url,

From 3a1341a7bc9f11d972c7649a33e143fab72f8bc0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 1 Oct 2015 22:59:20 +0600
Subject: [PATCH 2664/6219] [extractor/common] Make m3u8 extraction for SMIL
 non fatal

---
 youtube_dl/extractor/common.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index b928e24be..9a5a7cc2c 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1129,8 +1129,10 @@ class InfoExtractor(object):
             src_url = src if src.startswith('http') else compat_urlparse.urljoin(base, src)
 
             if proto == 'm3u8' or src_ext == 'm3u8':
-                formats.extend(self._extract_m3u8_formats(
-                    src_url, video_id, ext or 'mp4', m3u8_id='hls'))
+                m3u8_formats = self._extract_m3u8_formats(
+                    src_url, video_id, ext or 'mp4', m3u8_id='hls', fatal=False)
+                if m3u8_formats:
+                    formats.extend(m3u8_formats)
                 continue
 
             if src_ext == 'f4m':

From 4de6131090e0232c7cc99bcaafe6a3e71269b7af Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 1 Oct 2015 23:03:31 +0600
Subject: [PATCH 2665/6219] [extractor/common] Add fatal to
 _extract_f4m_formats

---
 youtube_dl/extractor/common.py | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 9a5a7cc2c..c2aa3a749 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -870,13 +870,18 @@ class InfoExtractor(object):
         time.sleep(timeout)
 
     def _extract_f4m_formats(self, manifest_url, video_id, preference=None, f4m_id=None,
-                             transform_source=lambda s: fix_xml_ampersands(s).strip()):
+                             transform_source=lambda s: fix_xml_ampersands(s).strip(),
+                             fatal=True):
         manifest = self._download_xml(
             manifest_url, video_id, 'Downloading f4m manifest',
             'Unable to download f4m manifest',
             # Some manifests may be malformed, e.g. prosiebensat1 generated manifests
             # (see https://github.com/rg3/youtube-dl/issues/6215#issuecomment-121704244)
-            transform_source=transform_source)
+            transform_source=transform_source,
+            fatal=fatal)
+
+        if manifest is False:
+            return manifest
 
         formats = []
         manifest_version = '1.0'
@@ -897,7 +902,10 @@ class InfoExtractor(object):
                 # may differ leading to inability to resolve the format by requested
                 # bitrate in f4m downloader
                 if determine_ext(manifest_url) == 'f4m':
-                    formats.extend(self._extract_f4m_formats(manifest_url, video_id, preference, f4m_id))
+                    f4m_formats = self._extract_f4m_formats(
+                        manifest_url, video_id, preference, f4m_id, fatal=fatal)
+                    if f4m_formats:
+                        formats.extend(f4m_formats)
                     continue
             tbr = int_or_none(media_el.attrib.get('bitrate'))
             formats.append({

From e5851b963ab0872f27f2db1cff3c89548b6bf95e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 1 Oct 2015 23:04:56 +0600
Subject: [PATCH 2666/6219] [extractor/common] Make f4m extraction for SMIL non
 fatal

---
 youtube_dl/extractor/common.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index c2aa3a749..4fe2307cd 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1152,7 +1152,9 @@ class InfoExtractor(object):
                     }
                 f4m_url += '&' if '?' in f4m_url else '?'
                 f4m_url += compat_urllib_parse.urlencode(f4m_params)
-                formats.extend(self._extract_f4m_formats(f4m_url, video_id, f4m_id='hds'))
+                f4m_formats = self._extract_f4m_formats(f4m_url, video_id, f4m_id='hds', fatal=False)
+                if f4m_formats:
+                    formats.extend(f4m_formats)
                 continue
 
             if src_url.startswith('http') and self._is_valid_url(src, video_id):

From 06c6efa9701d0527147875adf0caf437ffbb0397 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 1 Oct 2015 23:10:36 +0600
Subject: [PATCH 2667/6219] [videolecturesnet] Add test video with broken
 direct format links

---
 youtube_dl/extractor/videolecturesnet.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/youtube_dl/extractor/videolecturesnet.py b/youtube_dl/extractor/videolecturesnet.py
index 113a2289b..649ac9433 100644
--- a/youtube_dl/extractor/videolecturesnet.py
+++ b/youtube_dl/extractor/videolecturesnet.py
@@ -28,6 +28,21 @@ class VideoLecturesNetIE(InfoExtractor):
             'duration': 565,
             'thumbnail': 're:http://.*\.jpg',
         },
+    }, {
+        # video with invalid direct format links (HTTP 403)
+        'url': 'http://videolectures.net/russir2010_filippova_nlp/',
+        'info_dict': {
+            'id': 'russir2010_filippova_nlp',
+            'ext': 'flv',
+            'title': 'NLP at Google',
+            'description': 'md5:fc7a6d9bf0302d7cc0e53f7ca23747b3',
+            'duration': 5352,
+            'thumbnail': 're:http://.*\.jpg',
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        },
     }, {
         'url': 'http://videolectures.net/deeplearning2015_montreal/',
         'info_dict': {

From 44451f22d5d1a5bed5f5851b27a963860813ecd6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 2 Oct 2015 13:41:52 +0200
Subject: [PATCH 2668/6219] [naver] Remove unused import

---
 youtube_dl/extractor/naver.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/naver.py b/youtube_dl/extractor/naver.py
index 35cbb3e6d..1f5fc2145 100644
--- a/youtube_dl/extractor/naver.py
+++ b/youtube_dl/extractor/naver.py
@@ -10,7 +10,6 @@ from ..compat import (
 )
 from ..utils import (
     ExtractorError,
-    clean_html,
 )
 
 

From 7d0ada5ff907824c66c466ee9b83008210250d5f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 2 Oct 2015 13:42:11 +0200
Subject: [PATCH 2669/6219] [test/helper] Fix style

Use the correct indentation to please flake8
---
 test/helper.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/helper.py b/test/helper.py
index 28fd135b2..bdd7acca4 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -136,7 +136,7 @@ def expect_value(self, got, expected, field):
             self.assertEqual(
                 type_expected, type_got,
                 'Type mismatch for list item at index %d for field %s, expected %r, got %r' % (
-                index, field, type_expected, type_got))
+                    index, field, type_expected, type_got))
             expect_value(self, item_got, item_expected, field)
     else:
         if isinstance(expected, compat_str) and expected.startswith('md5:'):

From 0facd2af3ebfda68b79c7e2e1c575d73f9680802 Mon Sep 17 00:00:00 2001
From: fluks <fluks.github@gmail.com>
Date: Fri, 2 Oct 2015 04:08:13 +0300
Subject: [PATCH 2670/6219] Fix ruutu extractor bug

If there's no resolution attribute in xml, only width gets a
value, height doesn't and ValueError is raised.
---
 youtube_dl/extractor/ruutu.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ruutu.py b/youtube_dl/extractor/ruutu.py
index c67ad25ce..7720f1383 100644
--- a/youtube_dl/extractor/ruutu.py
+++ b/youtube_dl/extractor/ruutu.py
@@ -74,7 +74,7 @@ class RuutuIE(InfoExtractor):
                         preference = -1 if proto == 'rtmp' else 1
                         label = child.get('label')
                         tbr = int_or_none(child.get('bitrate'))
-                        width, height = [int_or_none(x) for x in child.get('resolution', '').split('x')]
+                        width, height = [int_or_none(x) for x in child.get('resolution', 'x').split('x')]
                         formats.append({
                             'format_id': '%s-%s' % (proto, label if label else tbr),
                             'url': video_url,

From 59a9efe85b15e53c5928b7fdb810c150f5bf4b78 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 2 Oct 2015 20:48:39 +0600
Subject: [PATCH 2671/6219] [ruutu] Limit resolution split to 2 pieces (Closes
 #7037, closes #7042)

---
 youtube_dl/extractor/ruutu.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ruutu.py b/youtube_dl/extractor/ruutu.py
index 7720f1383..a16b73ff4 100644
--- a/youtube_dl/extractor/ruutu.py
+++ b/youtube_dl/extractor/ruutu.py
@@ -74,7 +74,7 @@ class RuutuIE(InfoExtractor):
                         preference = -1 if proto == 'rtmp' else 1
                         label = child.get('label')
                         tbr = int_or_none(child.get('bitrate'))
-                        width, height = [int_or_none(x) for x in child.get('resolution', 'x').split('x')]
+                        width, height = [int_or_none(x) for x in child.get('resolution', 'x').split('x')[:2]]
                         formats.append({
                             'format_id': '%s-%s' % (proto, label if label else tbr),
                             'url': video_url,

From 3bb3f0410822d3d21c6199bb8915b598990628e6 Mon Sep 17 00:00:00 2001
From: ngld <ngld@tproxy.de>
Date: Wed, 12 Aug 2015 16:59:04 +0200
Subject: [PATCH 2672/6219] [europa] Add new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/europa.py   | 60 ++++++++++++++++++++++++++++++++
 2 files changed, 61 insertions(+)
 create mode 100644 youtube_dl/extractor/europa.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index a73a1317e..495a18c17 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -158,6 +158,7 @@ from .eroprofile import EroProfileIE
 from .escapist import EscapistIE
 from .espn import ESPNIE
 from .esri import EsriVideoIE
+from .europa import EuropaIE
 from .everyonesmixtape import EveryonesMixtapeIE
 from .exfm import ExfmIE
 from .expotv import ExpoTVIE
diff --git a/youtube_dl/extractor/europa.py b/youtube_dl/extractor/europa.py
new file mode 100644
index 000000000..c437c4886
--- /dev/null
+++ b/youtube_dl/extractor/europa.py
@@ -0,0 +1,60 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    compat_urlparse,
+    xpath_text
+)
+
+
+class EuropaIE(InfoExtractor):
+    _VALID_URL = r'https?://ec\.europa\.eu/avservices/video/player\.cfm\?(?:[^&]|&(?!ref))*ref=(?P<id>[A-Za-z0-9]+)'
+    _TEST = {
+        'url': 'http://ec.europa.eu/avservices/video/player.cfm?ref=I107758',
+        'md5': '728cca2fd41d5aa7350cec1141fbe620',
+        'info_dict': {
+            'id': 'I107758',
+            'ext': 'mp4',
+            'title': 'TRADE - Wikileaks on TTIP',
+            'description': 'NEW  LIVE EC Midday press briefing of 11/08/2015',
+            'thumbnail': 're:^http://defiris\.ec\.streamcloud\.be/findmedia/18/107758/THUMB_[0-9A-Z]+\.jpg$'
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        query = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
+        lang = query.get('sitelang', ['en'])[0]
+
+        playlist = self._download_xml('http://ec.europa.eu/avservices/video/player/playlist.cfm?ID=' + video_id, video_id)
+        videos = {}
+        formats = []
+
+        for item in playlist.findall('info/title/item'):
+            videos[xpath_text(item, 'lg')] = {'title': xpath_text(item, 'label').strip()}
+
+        for item in playlist.findall('info/description/item'):
+            videos[xpath_text(item, 'lg')]['description'] = xpath_text(item, 'label').strip()
+
+        for item in playlist.findall('files/file'):
+            lg = xpath_text(item, 'lg')
+            vid = videos[lg]
+            vid['format_note'] = xpath_text(item, 'lglabel')
+            vid['url'] = xpath_text(item, 'url')
+
+            if lg == lang:
+                vid['language_preference'] = 10
+
+            formats.append(vid)
+
+        formats.reverse()
+        def_video = videos.get(lang, videos['int'])
+
+        return {
+            'id': video_id,
+            'title': def_video['title'],
+            'description': def_video['description'],
+            'thumbnail': xpath_text(playlist, 'info/thumburl', 'thumburl'),
+            'formats': formats
+        }

From 8fc226ef994a82f7b1050cdb72ec38922d3ab9cf Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 2 Oct 2015 17:24:30 +0100
Subject: [PATCH 2673/6219] [nba] extract all video formats and extract more
 info

---
 youtube_dl/extractor/__init__.py |   5 +-
 youtube_dl/extractor/nba.py      | 102 +++++++++++++++++++++----------
 2 files changed, 74 insertions(+), 33 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index a73a1317e..78478b38b 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -351,7 +351,10 @@ from .myvideo import MyVideoIE
 from .myvidster import MyVidsterIE
 from .nationalgeographic import NationalGeographicIE
 from .naver import NaverIE
-from .nba import NBAIE
+from .nba import (
+    NBAIE,
+    NBAWatchIE,
+)
 from .nbc import (
     NBCIE,
     NBCNewsIE,
diff --git a/youtube_dl/extractor/nba.py b/youtube_dl/extractor/nba.py
index 944096e1c..36ece5b64 100644
--- a/youtube_dl/extractor/nba.py
+++ b/youtube_dl/extractor/nba.py
@@ -2,62 +2,100 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..utils import (
-    remove_end,
     parse_duration,
+    parse_iso8601,
+    int_or_none,
 )
 
 
-class NBAIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:watch\.|www\.)?nba\.com/(?:nba/)?video(?P<id>/[^?]*?)/?(?:/index\.html)?(?:\?.*)?$'
+class NBABaseIE(InfoExtractor):
+    def _get_formats(self, video_id):
+        base_url = 'http://nba.cdn.turner.com/nba/big%s' % video_id
+        return [{
+            'url': base_url + '_nba_android_high.mp4',
+            'width': 480,
+            'height': 320,
+            'format_id': '320p',
+        },{
+            'url': base_url + '_640x360_664b.mp4',
+            'width': 640,
+            'height': 360,
+            'format_id': '360p',
+        },{
+            'url': base_url + '_768x432_1404.mp4',
+            'width': 768,
+            'height': 432,
+            'format_id': '432p',
+        },{
+            'url': base_url + '_1280x720.mp4',
+            'width': 1280,
+            'height': 720,
+            'format_id': '720p',
+        }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        ret = self._extract_metadata(webpage, video_id)
+        ret['id'] = video_id.rpartition('/')[2]
+        ret['formats'] = self._get_formats(video_id)
+        return ret
+
+
+class NBAIE(NBABaseIE):
+    IE_NAME = 'nba'
+    _VALID_URL = r'https?://(?:www\.)?nba\.com/(?:nba/)?video(?P<id>/[^?]*?)/?(?:/index\.html)?(?:\?.*)?$'
     _TESTS = [{
         'url': 'http://www.nba.com/video/games/nets/2012/12/04/0021200253-okc-bkn-recap.nba/index.html',
-        'md5': 'c0edcfc37607344e2ff8f13c378c88a4',
+        'md5': '9d902940d2a127af3f7f9d2f3dc79c96',
         'info_dict': {
             'id': '0021200253-okc-bkn-recap.nba',
             'ext': 'mp4',
             'title': 'Thunder vs. Nets',
             'description': 'Kevin Durant scores 32 points and dishes out six assists as the Thunder beat the Nets in Brooklyn.',
             'duration': 181,
+            'timestamp': 1354680189,
+            'upload_date': '20121205',
         },
     }, {
         'url': 'http://www.nba.com/video/games/hornets/2014/12/05/0021400276-nyk-cha-play5.nba/',
         'only_matching': True,
-    }, {
+    }]
+
+    def _extract_metadata(self, webpage, video_id):
+        return {
+            'title': self._html_search_meta('name', webpage),
+            'description': self._html_search_meta('description', webpage),
+            'duration': parse_duration(self._html_search_meta('duration', webpage)),
+            'thumbnail': self._html_search_meta('thumbnailUrl', webpage),
+            'timestamp': parse_iso8601(self._html_search_meta('uploadDate', webpage))
+        }
+
+class NBAWatchIE(NBABaseIE):
+    IE_NAME = 'nba:watch'
+    _VALID_URL = r'https?://watch.nba\.com/(?:nba/)?video(?P<id>/[^?]*?)/?(?:/index\.html)?(?:\?.*)?$'
+    _TESTS = [{
         'url': 'http://watch.nba.com/nba/video/channels/playoffs/2015/05/20/0041400301-cle-atl-recap.nba',
+        'md5': 'b2b39b81cf28615ae0c3360a3f9668c4',
         'info_dict': {
             'id': '0041400301-cle-atl-recap.nba',
             'ext': 'mp4',
-            'title': 'NBA GAME TIME | Video: Hawks vs. Cavaliers Game 1',
+            'title': 'Hawks vs. Cavaliers Game 1',
             'description': 'md5:8094c3498d35a9bd6b1a8c396a071b4d',
             'duration': 228,
-        },
-        'params': {
-            'skip_download': True,
+            'timestamp': 1432094400,
+            'upload_date': '20150520',
         }
     }]
 
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-
-        video_url = 'http://ht-mobile.cdn.turner.com/nba/big' + video_id + '_nba_1280x720.mp4'
-
-        shortened_video_id = video_id.rpartition('/')[2]
-        title = remove_end(
-            self._og_search_title(webpage, default=shortened_video_id), ' : NBA.com')
-
-        description = self._og_search_description(webpage)
-        duration_str = self._html_search_meta(
-            'duration', webpage, 'duration', default=None)
-        if not duration_str:
-            duration_str = self._html_search_regex(
-                r'Duration:</b>\s*(\d+:\d+)', webpage, 'duration', fatal=False)
-        duration = parse_duration(duration_str)
-
+    def _extract_metadata(self, webpage, video_id):
+        program_id = self._search_regex(r'var\s+programId\s*=\s*(\d+);', webpage, 'program id')
+        metadata = self._download_json(
+            'http://smbsolr.cdnak.neulion.com/solr_nbav6/nba/nba/mlt/?wt=json&fl=name,description,image,runtime,releaseDate&q=sequence%3A' + program_id, video_id)['match']['docs'][0]
         return {
-            'id': shortened_video_id,
-            'url': video_url,
-            'title': title,
-            'description': description,
-            'duration': duration,
+            'title': metadata['name'],
+            'description': metadata.get('description'),
+            'duration': int_or_none(metadata.get('runtime')),
+            'thumbnail': metadata.get('image'),
+            'timestamp': parse_iso8601(metadata.get('releaseDate'))
         }

From af17794c654bd24bbd5f47997596430b201ea08e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 2 Oct 2015 22:29:15 +0600
Subject: [PATCH 2674/6219] [europa] Improve extraction

---
 youtube_dl/extractor/europa.py | 94 ++++++++++++++++++++++------------
 1 file changed, 62 insertions(+), 32 deletions(-)

diff --git a/youtube_dl/extractor/europa.py b/youtube_dl/extractor/europa.py
index c437c4886..02ba8d63c 100644
--- a/youtube_dl/extractor/europa.py
+++ b/youtube_dl/extractor/europa.py
@@ -2,59 +2,89 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..compat import compat_urlparse
 from ..utils import (
-    compat_urlparse,
+    int_or_none,
+    orderedSet,
+    parse_duration,
+    qualities,
+    unified_strdate,
     xpath_text
 )
 
 
 class EuropaIE(InfoExtractor):
-    _VALID_URL = r'https?://ec\.europa\.eu/avservices/video/player\.cfm\?(?:[^&]|&(?!ref))*ref=(?P<id>[A-Za-z0-9]+)'
-    _TEST = {
+    _VALID_URL = r'https?://ec\.europa\.eu/avservices/video/player\.cfm\?.*?\bref=(?P<id>[A-Za-z0-9]+)'
+    _TESTS = [{
         'url': 'http://ec.europa.eu/avservices/video/player.cfm?ref=I107758',
-        'md5': '728cca2fd41d5aa7350cec1141fbe620',
+        'md5': '574f080699ddd1e19a675b0ddf010371',
         'info_dict': {
             'id': 'I107758',
             'ext': 'mp4',
             'title': 'TRADE - Wikileaks on TTIP',
             'description': 'NEW  LIVE EC Midday press briefing of 11/08/2015',
-            'thumbnail': 're:^http://defiris\.ec\.streamcloud\.be/findmedia/18/107758/THUMB_[0-9A-Z]+\.jpg$'
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'upload_date': '20150811',
+            'duration': 34,
+            'view_count': int,
+            'formats': 'mincount:3',
         }
-    }
+    }, {
+        'url': 'http://ec.europa.eu/avservices/video/player.cfm?sitelang=en&ref=I107786',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
+
+        playlist = self._download_xml(
+            'http://ec.europa.eu/avservices/video/player/playlist.cfm?ID=%s' % video_id, video_id)
+
+        def get_item(type_, preference):
+            items = {}
+            for item in playlist.findall('./info/%s/item' % type_):
+                lang, label = xpath_text(item, 'lg', default=None), xpath_text(item, 'label', default=None)
+                if lang and label:
+                    items[lang] = label.strip()
+            for p in preference:
+                if items.get(p):
+                    return items[p]
+
         query = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
-        lang = query.get('sitelang', ['en'])[0]
+        preferred_lang = query.get('sitelang', ('en', ))[0]
+
+        preferred_langs = orderedSet((preferred_lang, 'en', 'int'))
+
+        title = get_item('title', preferred_langs) or video_id
+        description = get_item('description', preferred_langs)
+        thumbnmail = xpath_text(playlist, './info/thumburl', 'thumbnail')
+        upload_date = unified_strdate(xpath_text(playlist, './info/date', 'upload date'))
+        duration = parse_duration(xpath_text(playlist, './info/duration', 'duration'))
+        view_count = int_or_none(xpath_text(playlist,'./info/views', 'views'))
+
+        language_preference = qualities(preferred_langs[::-1])
 
-        playlist = self._download_xml('http://ec.europa.eu/avservices/video/player/playlist.cfm?ID=' + video_id, video_id)
-        videos = {}
         formats = []
-
-        for item in playlist.findall('info/title/item'):
-            videos[xpath_text(item, 'lg')] = {'title': xpath_text(item, 'label').strip()}
-
-        for item in playlist.findall('info/description/item'):
-            videos[xpath_text(item, 'lg')]['description'] = xpath_text(item, 'label').strip()
-
-        for item in playlist.findall('files/file'):
-            lg = xpath_text(item, 'lg')
-            vid = videos[lg]
-            vid['format_note'] = xpath_text(item, 'lglabel')
-            vid['url'] = xpath_text(item, 'url')
-
-            if lg == lang:
-                vid['language_preference'] = 10
-
-            formats.append(vid)
-
-        formats.reverse()
-        def_video = videos.get(lang, videos['int'])
+        for file_ in playlist.findall('./files/file'):
+            video_url = xpath_text(file_, './url')
+            if not video_url:
+                continue
+            lang = xpath_text(file_, './lg')
+            formats.append({
+                'url': video_url,
+                'format_id': lang,
+                'format_note': xpath_text(file_, './lglabel'),
+                'language_preference': language_preference(lang)
+            })
+        self._sort_formats(formats)
 
         return {
             'id': video_id,
-            'title': def_video['title'],
-            'description': def_video['description'],
-            'thumbnail': xpath_text(playlist, 'info/thumburl', 'thumburl'),
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnmail,
+            'upload_date': upload_date,
+            'duration': duration,
+            'view_count': view_count,
             'formats': formats
         }

From f3b098fb90b985484d800fcdbfe18add2360e4df Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 2 Oct 2015 23:22:53 +0600
Subject: [PATCH 2675/6219] [europa] Add support for audio URLs

---
 youtube_dl/extractor/europa.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/europa.py b/youtube_dl/extractor/europa.py
index 02ba8d63c..9e33cacff 100644
--- a/youtube_dl/extractor/europa.py
+++ b/youtube_dl/extractor/europa.py
@@ -14,7 +14,7 @@ from ..utils import (
 
 
 class EuropaIE(InfoExtractor):
-    _VALID_URL = r'https?://ec\.europa\.eu/avservices/video/player\.cfm\?.*?\bref=(?P<id>[A-Za-z0-9]+)'
+    _VALID_URL = r'https?://ec\.europa\.eu/avservices/(?:video/player|audio/audioDetails)\.cfm\?.*?\bref=(?P<id>[A-Za-z0-9-]+)'
     _TESTS = [{
         'url': 'http://ec.europa.eu/avservices/video/player.cfm?ref=I107758',
         'md5': '574f080699ddd1e19a675b0ddf010371',
@@ -32,6 +32,9 @@ class EuropaIE(InfoExtractor):
     }, {
         'url': 'http://ec.europa.eu/avservices/video/player.cfm?sitelang=en&ref=I107786',
         'only_matching': True,
+    }, {
+        'url': 'http://ec.europa.eu/avservices/audio/audioDetails.cfm?ref=I-109295&sitelang=en',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From b203095d4c95d471bc2ac7045693c6938ee914d1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 2 Oct 2015 22:40:35 +0200
Subject: [PATCH 2676/6219] [europa] Style fix: add whitespace after comma

---
 youtube_dl/extractor/europa.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/europa.py b/youtube_dl/extractor/europa.py
index 9e33cacff..adc43919e 100644
--- a/youtube_dl/extractor/europa.py
+++ b/youtube_dl/extractor/europa.py
@@ -63,7 +63,7 @@ class EuropaIE(InfoExtractor):
         thumbnmail = xpath_text(playlist, './info/thumburl', 'thumbnail')
         upload_date = unified_strdate(xpath_text(playlist, './info/date', 'upload date'))
         duration = parse_duration(xpath_text(playlist, './info/duration', 'duration'))
-        view_count = int_or_none(xpath_text(playlist,'./info/views', 'views'))
+        view_count = int_or_none(xpath_text(playlist, './info/views', 'views'))
 
         language_preference = qualities(preferred_langs[::-1])
 

From adccf33632c51def397cdfb08c1271de6d6ec95e Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 2 Oct 2015 21:58:20 +0100
Subject: [PATCH 2677/6219] [ign] add support for pcmag and extract all formats
 and more metadata

---
 youtube_dl/extractor/__init__.py |   6 +-
 youtube_dl/extractor/ign.py      | 117 +++++++++++++++++++++++--------
 2 files changed, 93 insertions(+), 30 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index a73a1317e..191661390 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -231,7 +231,11 @@ from .howstuffworks import HowStuffWorksIE
 from .huffpost import HuffPostIE
 from .hypem import HypemIE
 from .iconosquare import IconosquareIE
-from .ign import IGNIE, OneUPIE
+from .ign import (
+    IGNIE,
+    OneUPIE,
+    PCMagIE,
+)
 from .imdb import (
     ImdbIE,
     ImdbListIE
diff --git a/youtube_dl/extractor/ign.py b/youtube_dl/extractor/ign.py
index bf2d2041b..fa4e67394 100644
--- a/youtube_dl/extractor/ign.py
+++ b/youtube_dl/extractor/ign.py
@@ -3,6 +3,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    parse_iso8601,
+)
 
 
 class IGNIE(InfoExtractor):
@@ -11,25 +15,23 @@ class IGNIE(InfoExtractor):
     Some videos of it.ign.com are also supported
     """
 
-    _VALID_URL = r'https?://.+?\.ign\.com/(?P<type>videos|show_videos|articles|(?:[^/]*/feature))(/.+)?/(?P<name_or_id>.+)'
+    _VALID_URL = r'https?://.+?\.ign\.com/(?:[^/]+/)?(?P<type>videos|show_videos|articles|feature|(?:[^/]+/\d+/video))(/.+)?/(?P<name_or_id>.+)'
     IE_NAME = 'ign.com'
 
-    _CONFIG_URL_TEMPLATE = 'http://www.ign.com/videos/configs/id/%s.config'
-    _DESCRIPTION_RE = [
-        r'<span class="page-object-description">(.+?)</span>',
-        r'id="my_show_video">.*?<p>(.*?)</p>',
-        r'<meta name="description" content="(.*?)"',
-    ]
+    _API_URL_TEMPLATE = 'http://apis.ign.com/video/v3/videos/%s'
+    _EMBED_RE = r'<iframe[^>]+?["\']((?:https?:)?//.+?\.ign\.com.+?/embed.+?)["\']'
 
     _TESTS = [
         {
             'url': 'http://www.ign.com/videos/2013/06/05/the-last-of-us-review',
-            'md5': 'eac8bdc1890980122c3b66f14bdd02e9',
+            'md5': 'febda82c4bafecd2d44b6e1a18a595f8',
             'info_dict': {
                 'id': '8f862beef863986b2785559b9e1aa599',
                 'ext': 'mp4',
                 'title': 'The Last of Us Review',
                 'description': 'md5:c8946d4260a4d43a00d5ae8ed998870c',
+                'timestamp': 1370440800,
+                'upload_date': '20130605',
             }
         },
         {
@@ -44,6 +46,8 @@ class IGNIE(InfoExtractor):
                         'ext': 'mp4',
                         'title': 'GTA 5 Video Review',
                         'description': 'Rockstar drops the mic on this generation of games. Watch our review of the masterly Grand Theft Auto V.',
+                        'timestamp': 1379339880,
+                        'upload_date': '20130916',
                     },
                 },
                 {
@@ -52,6 +56,8 @@ class IGNIE(InfoExtractor):
                         'ext': 'mp4',
                         'title': '26 Twisted Moments from GTA 5 in Slow Motion',
                         'description': 'The twisted beauty of GTA 5 in stunning slow motion.',
+                        'timestamp': 1386878820,
+                        'upload_date': '20131212',
                     },
                 },
             ],
@@ -66,10 +72,9 @@ class IGNIE(InfoExtractor):
                 'id': '078fdd005f6d3c02f63d795faa1b984f',
                 'ext': 'mp4',
                 'title': 'Rewind Theater - Wild Trailer Gamescom 2014',
-                'description': (
-                    'Giant skeletons, bloody hunts, and captivating'
-                    ' natural beauty take our breath away.'
-                ),
+                'description': 'Brian and Jared explore Michel Ancel\'s captivating new preview.',
+                'timestamp': 1408047180,
+                'upload_date': '20140814',
             },
         },
     ]
@@ -82,7 +87,7 @@ class IGNIE(InfoExtractor):
             r'<object id="vid_(.+?)"',
             r'<meta name="og:image" content=".*/(.+?)-(.+?)/.+.jpg"',
         ]
-        return self._search_regex(res_id, webpage, 'video id')
+        return self._search_regex(res_id, webpage, 'video id', default=None)
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
@@ -102,22 +107,45 @@ class IGNIE(InfoExtractor):
                 }
 
         video_id = self._find_video_id(webpage)
-        result = self._get_video_info(video_id)
-        description = self._html_search_regex(self._DESCRIPTION_RE,
-                                              webpage, 'video description', flags=re.DOTALL)
-        result['description'] = description
-        return result
+        if not video_id:
+            return self.url_result(self._search_regex(self._EMBED_RE, webpage, 'embed url'))
+        return self._get_video_info(video_id)
 
     def _get_video_info(self, video_id):
-        config_url = self._CONFIG_URL_TEMPLATE % video_id
-        config = self._download_json(config_url, video_id)
-        media = config['playlist']['media']
+        api_data = self._download_json(self._API_URL_TEMPLATE % video_id, video_id)
+
+        formats = []
+        m3u8_url = api_data['refs'].get('m3uUrl')
+        if m3u8_url:
+            formats.extend(self._extract_m3u8_formats(m3u8_url, video_id))
+        f4m_url = api_data['refs'].get('f4mUrl')
+        if f4m_url:
+            formats.extend(self._extract_f4m_formats(f4m_url, video_id))
+        for asset in api_data['assets']:
+            formats.append({
+                'url': asset['url'],
+                'tbr': asset.get('actual_bitrate_kbps'),
+                'fps': asset.get('frame_rate'),
+                'height': int_or_none(asset.get('height')),
+                'width': int_or_none(asset.get('width')),
+            })
+        self._sort_formats(formats)
+
+        thumbnails = []
+        for thumbnail in api_data['thumbnails']:
+            thumbnails.append({'url': thumbnail['url']})
+
+        metadata = api_data['metadata']
 
         return {
-            'id': media['metadata']['videoId'],
-            'url': media['url'],
-            'title': media['metadata']['title'],
-            'thumbnail': media['poster'][0]['url'].replace('{size}', 'grande'),
+            'id': api_data.get('videoId') or video_id,
+            'title': metadata.get('longTitle') or metadata.get('name') or metadata.get['title'],
+            'description': metadata.get('description'),
+            'timestamp': parse_iso8601(metadata.get('publishDate')),
+            'duration': int_or_none(metadata.get('duration')),
+            'display_id': metadata.get('slug') or video_id,
+            'thumbnails': thumbnails,
+            'formats': formats,
         }
 
 
@@ -125,16 +153,16 @@ class OneUPIE(IGNIE):
     _VALID_URL = r'https?://gamevideos\.1up\.com/(?P<type>video)/id/(?P<name_or_id>.+)\.html'
     IE_NAME = '1up.com'
 
-    _DESCRIPTION_RE = r'<div id="vid_summary">(.+?)</div>'
-
     _TESTS = [{
         'url': 'http://gamevideos.1up.com/video/id/34976.html',
-        'md5': '68a54ce4ebc772e4b71e3123d413163d',
+        'md5': 'c9cc69e07acb675c31a16719f909e347',
         'info_dict': {
             'id': '34976',
             'ext': 'mp4',
             'title': 'Sniper Elite V2 - Trailer',
-            'description': 'md5:5d289b722f5a6d940ca3136e9dae89cf',
+            'description': 'md5:bf0516c5ee32a3217aa703e9b1bc7826',
+            'timestamp': 1313099220,
+            'upload_date': '20110811',
         }
     }]
 
@@ -143,3 +171,34 @@ class OneUPIE(IGNIE):
         result = super(OneUPIE, self)._real_extract(url)
         result['id'] = mobj.group('name_or_id')
         return result
+
+
+class PCMagIE(IGNIE):
+    _VALID_URL = r'https?://(?:www\.)?pcmag\.com/(?P<type>videos|article2)(/.+)?/(?P<name_or_id>.+)'
+    IE_NAME = 'pcmag'
+
+    _EMBED_RE = r'iframe.setAttribute\("src",\s*__util.objToUrlString\("http://widgets\.ign\.com/video/embed/content.html?[^"]*url=([^"]+)["&]'
+
+    _TESTS = [{
+        'url': 'http://www.pcmag.com/videos/2015/01/06/010615-whats-new-now-is-gogo-snooping-on-your-data',
+        'md5': '212d6154fd0361a2781075f1febbe9ad',
+        'info_dict': {
+            'id': 'ee10d774b508c9b8ec07e763b9125b91',
+            'ext': 'mp4',
+            'title': '010615_What\'s New Now: Is GoGo Snooping on Your Data?',
+            'description': 'md5:a7071ae64d2f68cc821c729d4ded6bb3',
+            'timestamp': 1420571160,
+            'upload_date': '20150106',
+        }
+    },{
+        'url': 'http://www.pcmag.com/article2/0,2817,2470156,00.asp',
+        'md5': '94130c1ca07ba0adb6088350681f16c1',
+        'info_dict': {
+            'id': '042e560ba94823d43afcb12ddf7142ca',
+            'ext': 'mp4',
+            'title': 'HTC\'s Weird New Re Camera - What\'s New Now',
+            'description': 'md5:53433c45df96d2ea5d0fda18be2ca908',
+            'timestamp': 1412953920,
+            'upload_date': '20141010',
+        }
+    }]

From 5495937f461268a850a6a54d3fe19ed1f0f01eef Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 13 Sep 2015 20:00:10 +0800
Subject: [PATCH 2678/6219] [options] Cleanup double spaces in help texts

---
 youtube_dl/options.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 5eccc0a70..3dd6d290b 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -276,7 +276,7 @@ def parseOpts(overrideArguments=None):
             'For example, to only match videos that have been liked more than '
             '100 times and disliked less than 50 times (or the dislike '
             'functionality is not available at the given service), but who '
-            'also have a description, use  --match-filter '
+            'also have a description, use --match-filter '
             '"like_count > 100 & dislike_count <? 50 & description" .'
         ))
     selection.add_option(
@@ -602,7 +602,7 @@ def parseOpts(overrideArguments=None):
     filesystem.add_option(
         '-A', '--auto-number',
         action='store_true', dest='autonumber', default=False,
-        help='[deprecated; use  -o "%(autonumber)s-%(title)s.%(ext)s" ] Number downloaded files starting from 00000')
+        help='[deprecated; use -o "%(autonumber)s-%(title)s.%(ext)s" ] Number downloaded files starting from 00000')
     filesystem.add_option(
         '-t', '--title',
         action='store_true', dest='usetitle', default=False,

From 13118a50b88c26fe5d932eccf77e297249a78634 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 13 Sep 2015 20:04:27 +0800
Subject: [PATCH 2679/6219] [compat] Allow overriding by only COLUMNS or LINES
 in compat_get_terminal_size

Now the semantic of this function is identical to
shutil.get_terminal_size() in Python 3.3+. The new behavior also
corresponds to the old get_term_width(), which is removed in
003c69a84b68cadb46aeb8e03115848a722fd675
---
 youtube_dl/compat.py | 24 +++++++++++++++---------
 1 file changed, 15 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 1ff42d94b..c36c9c23f 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -416,7 +416,7 @@ if hasattr(shutil, 'get_terminal_size'):  # Python >= 3.3
 else:
     _terminal_size = collections.namedtuple('terminal_size', ['columns', 'lines'])
 
-    def compat_get_terminal_size():
+    def compat_get_terminal_size(fallback=(80, 24)):
         columns = compat_getenv('COLUMNS', None)
         if columns:
             columns = int(columns)
@@ -428,14 +428,20 @@ else:
         else:
             lines = None
 
-        try:
-            sp = subprocess.Popen(
-                ['stty', 'size'],
-                stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-            out, err = sp.communicate()
-            lines, columns = map(int, out.split())
-        except Exception:
-            pass
+        if columns <= 0 or lines <= 0:
+            try:
+                sp = subprocess.Popen(
+                    ['stty', 'size'],
+                    stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+                out, err = sp.communicate()
+                _columns, _lines = map(int, out.split())
+            except Exception:
+                _columns, _lines = _terminal_size(*fallback)
+
+            if columns <= 0:
+                columns = _columns
+            if lines <= 0:
+                lines = _lines
         return _terminal_size(columns, lines)
 
 try:

From bad84757eb135b85d5a1b29524a064d23ab4e1e9 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 13 Sep 2015 20:10:23 +0800
Subject: [PATCH 2680/6219] [doc] Better formatting of youtube-dl.1 (closes
 #6510)

---
 devscripts/prepare_manpage.py | 31 +++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/devscripts/prepare_manpage.py b/devscripts/prepare_manpage.py
index 7ece37754..776e6556e 100644
--- a/devscripts/prepare_manpage.py
+++ b/devscripts/prepare_manpage.py
@@ -8,6 +8,35 @@ import re
 ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 README_FILE = os.path.join(ROOT_DIR, 'README.md')
 
+
+def filter_options(readme):
+    ret = ''
+    in_options = False
+    for line in readme.split('\n'):
+        if line.startswith('# '):
+            if line[2:].startswith('OPTIONS'):
+                in_options = True
+            else:
+                in_options = False
+
+        if in_options:
+            if line.lstrip().startswith('-'):
+                option, description = re.split(r'\s{2,}', line.lstrip())
+                split_option = option.split(' ')
+
+                if not split_option[-1].startswith('-'):  # metavar
+                    option = ' '.join(split_option[:-1] + ['*%s*' % split_option[-1]])
+
+                # Pandoc's definition_lists. See http://pandoc.org/README.html
+                # for more information.
+                ret += '\n%s\n:   %s\n' % (option, description)
+            else:
+                ret += line.lstrip() + '\n'
+        else:
+            ret += line + '\n'
+
+    return ret
+
 with io.open(README_FILE, encoding='utf-8') as f:
     readme = f.read()
 
@@ -26,6 +55,8 @@ readme = re.sub(r'(?s)^.*?(?=# DESCRIPTION)', '', readme)
 readme = re.sub(r'\s+youtube-dl \[OPTIONS\] URL \[URL\.\.\.\]', '', readme)
 readme = PREFIX + readme
 
+readme = filter_options(readme)
+
 if sys.version_info < (3, 0):
     print(readme.encode('utf-8'))
 else:

From 28809ab07a8d10f9cafc3d712414c7b355c27166 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 3 Oct 2015 09:47:19 +0100
Subject: [PATCH 2681/6219] [nba] extract more formats

---
 youtube_dl/extractor/nba.py | 36 ++++++++++++++++++++++++++++++++++--
 1 file changed, 34 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/nba.py b/youtube_dl/extractor/nba.py
index 36ece5b64..8844b61a5 100644
--- a/youtube_dl/extractor/nba.py
+++ b/youtube_dl/extractor/nba.py
@@ -10,28 +10,60 @@ from ..utils import (
 
 class NBABaseIE(InfoExtractor):
     def _get_formats(self, video_id):
+        formats = self._extract_m3u8_formats(
+            'http://nbavod-f.akamaihd.net/i/nba/big%s_,640x360_664m,768x432_996,768x432_1404,960x540_2104,1280x720,.mp4.csmil/master.m3u8' % video_id,
+            video_id,
+            m3u8_id='hls')
+        formats.extend(self._extract_f4m_formats(
+            'http://nbavod-f.akamaihd.net/z/nba/big%s_,640x360_664m,768x432_996,768x432_1404,960x540_2104,1280x720,.mp4.csmil/manifest.f4m?hdcore=3.4.1.1' % video_id,
+            video_id,
+            f4m_id='hds'))
         base_url = 'http://nba.cdn.turner.com/nba/big%s' % video_id
-        return [{
+        formats.extend([{
+            'url': base_url + '_nba_ipad.mp4',
+            'width': 400,
+            'height': 224,
+            'format_id': '224p',
+            'preference': 1,
+        },{
             'url': base_url + '_nba_android_high.mp4',
             'width': 480,
             'height': 320,
             'format_id': '320p',
+            'preference': 2,
+        },{
+            'url': base_url + '_nba_576x324.mp4',
+            'width': 576,
+            'height': 324,
+            'format_id': '324p',
+            'preference': 3,
         },{
             'url': base_url + '_640x360_664b.mp4',
             'width': 640,
             'height': 360,
             'format_id': '360p',
+            'preference': 4,
         },{
             'url': base_url + '_768x432_1404.mp4',
             'width': 768,
             'height': 432,
             'format_id': '432p',
+            'preference': 5,
+        },{
+            'url': base_url + '_960x540_2104.mp4',
+            'width': 960,
+            'height': 540,
+            'format_id': '540p',
+            'preference': 6,
         },{
             'url': base_url + '_1280x720.mp4',
             'width': 1280,
             'height': 720,
             'format_id': '720p',
-        }]
+            'preference': 7,
+        }])
+        self._sort_formats(formats)
+        return formats
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From c233e6bcc398f9734d7138854978c1cb00fe757f Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 3 Oct 2015 12:30:05 +0100
Subject: [PATCH 2682/6219] [nba] extract video info from xml feed

---
 youtube_dl/extractor/__init__.py |   5 +-
 youtube_dl/extractor/nba.py      | 224 +++++++++++++++++--------------
 2 files changed, 126 insertions(+), 103 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 78478b38b..a73a1317e 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -351,10 +351,7 @@ from .myvideo import MyVideoIE
 from .myvidster import MyVidsterIE
 from .nationalgeographic import NationalGeographicIE
 from .naver import NaverIE
-from .nba import (
-    NBAIE,
-    NBAWatchIE,
-)
+from .nba import NBAIE
 from .nbc import (
     NBCIE,
     NBCNewsIE,
diff --git a/youtube_dl/extractor/nba.py b/youtube_dl/extractor/nba.py
index 8844b61a5..3d38d080e 100644
--- a/youtube_dl/extractor/nba.py
+++ b/youtube_dl/extractor/nba.py
@@ -3,131 +3,157 @@ from __future__ import unicode_literals
 from .common import InfoExtractor
 from ..utils import (
     parse_duration,
-    parse_iso8601,
     int_or_none,
 )
 
 
-class NBABaseIE(InfoExtractor):
-    def _get_formats(self, video_id):
-        formats = self._extract_m3u8_formats(
-            'http://nbavod-f.akamaihd.net/i/nba/big%s_,640x360_664m,768x432_996,768x432_1404,960x540_2104,1280x720,.mp4.csmil/master.m3u8' % video_id,
-            video_id,
-            m3u8_id='hls')
-        formats.extend(self._extract_f4m_formats(
-            'http://nbavod-f.akamaihd.net/z/nba/big%s_,640x360_664m,768x432_996,768x432_1404,960x540_2104,1280x720,.mp4.csmil/manifest.f4m?hdcore=3.4.1.1' % video_id,
-            video_id,
-            f4m_id='hds'))
-        base_url = 'http://nba.cdn.turner.com/nba/big%s' % video_id
-        formats.extend([{
-            'url': base_url + '_nba_ipad.mp4',
-            'width': 400,
-            'height': 224,
-            'format_id': '224p',
-            'preference': 1,
-        },{
-            'url': base_url + '_nba_android_high.mp4',
-            'width': 480,
-            'height': 320,
-            'format_id': '320p',
-            'preference': 2,
-        },{
-            'url': base_url + '_nba_576x324.mp4',
-            'width': 576,
-            'height': 324,
-            'format_id': '324p',
-            'preference': 3,
-        },{
-            'url': base_url + '_640x360_664b.mp4',
-            'width': 640,
-            'height': 360,
-            'format_id': '360p',
-            'preference': 4,
-        },{
-            'url': base_url + '_768x432_1404.mp4',
-            'width': 768,
-            'height': 432,
-            'format_id': '432p',
-            'preference': 5,
-        },{
-            'url': base_url + '_960x540_2104.mp4',
-            'width': 960,
-            'height': 540,
-            'format_id': '540p',
-            'preference': 6,
-        },{
-            'url': base_url + '_1280x720.mp4',
-            'width': 1280,
-            'height': 720,
-            'format_id': '720p',
-            'preference': 7,
-        }])
-        self._sort_formats(formats)
-        return formats
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-        ret = self._extract_metadata(webpage, video_id)
-        ret['id'] = video_id.rpartition('/')[2]
-        ret['formats'] = self._get_formats(video_id)
-        return ret
-
-
-class NBAIE(NBABaseIE):
-    IE_NAME = 'nba'
-    _VALID_URL = r'https?://(?:www\.)?nba\.com/(?:nba/)?video(?P<id>/[^?]*?)/?(?:/index\.html)?(?:\?.*)?$'
+class NBAIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:watch\.|www\.)?nba\.com/(?:nba/)?video/(?P<id>[^?]*?)/?(?:/index\.html)?(?:\?.*)?$'
     _TESTS = [{
         'url': 'http://www.nba.com/video/games/nets/2012/12/04/0021200253-okc-bkn-recap.nba/index.html',
         'md5': '9d902940d2a127af3f7f9d2f3dc79c96',
         'info_dict': {
-            'id': '0021200253-okc-bkn-recap.nba',
+            'id': '0021200253-okc-bkn-recap',
             'ext': 'mp4',
             'title': 'Thunder vs. Nets',
             'description': 'Kevin Durant scores 32 points and dishes out six assists as the Thunder beat the Nets in Brooklyn.',
             'duration': 181,
-            'timestamp': 1354680189,
-            'upload_date': '20121205',
+            'timestamp': 1354638466,
+            'upload_date': '20121204',
         },
     }, {
         'url': 'http://www.nba.com/video/games/hornets/2014/12/05/0021400276-nyk-cha-play5.nba/',
         'only_matching': True,
-    }]
-
-    def _extract_metadata(self, webpage, video_id):
-        return {
-            'title': self._html_search_meta('name', webpage),
-            'description': self._html_search_meta('description', webpage),
-            'duration': parse_duration(self._html_search_meta('duration', webpage)),
-            'thumbnail': self._html_search_meta('thumbnailUrl', webpage),
-            'timestamp': parse_iso8601(self._html_search_meta('uploadDate', webpage))
-        }
-
-class NBAWatchIE(NBABaseIE):
-    IE_NAME = 'nba:watch'
-    _VALID_URL = r'https?://watch.nba\.com/(?:nba/)?video(?P<id>/[^?]*?)/?(?:/index\.html)?(?:\?.*)?$'
-    _TESTS = [{
+    },{
         'url': 'http://watch.nba.com/nba/video/channels/playoffs/2015/05/20/0041400301-cle-atl-recap.nba',
         'md5': 'b2b39b81cf28615ae0c3360a3f9668c4',
         'info_dict': {
-            'id': '0041400301-cle-atl-recap.nba',
+            'id': '0041400301-cle-atl-recap',
             'ext': 'mp4',
             'title': 'Hawks vs. Cavaliers Game 1',
             'description': 'md5:8094c3498d35a9bd6b1a8c396a071b4d',
             'duration': 228,
-            'timestamp': 1432094400,
+            'timestamp': 1432134543,
             'upload_date': '20150520',
         }
     }]
 
-    def _extract_metadata(self, webpage, video_id):
-        program_id = self._search_regex(r'var\s+programId\s*=\s*(\d+);', webpage, 'program id')
-        metadata = self._download_json(
-            'http://smbsolr.cdnak.neulion.com/solr_nbav6/nba/nba/mlt/?wt=json&fl=name,description,image,runtime,releaseDate&q=sequence%3A' + program_id, video_id)['match']['docs'][0]
+    _BASE_PATHS = {
+        'turner': 'http://nba.cdn.turner.com/nba/big',
+        'akamai': 'http://nbavod-f.akamaihd.net',
+    }
+
+    _QUALITIES = {
+        '420mp4': {
+            'width': 400,
+            'height': 224,
+            'preference': 1,
+        },
+        '416x234': {
+            'width': 416,
+            'height': 234,
+            'preference': 2,
+        },
+        '556': {
+            'width': 416,
+            'height': 234,
+            'preference': 3,
+        },
+        '480x320_910': {
+            'width': 480,
+            'height': 320,
+            'preference': 4,
+        },
+        'nba_576x324': {
+            'width': 576,
+            'height': 324,
+            'preference': 5,
+        },
+        'nba_640x360': {
+            'width': 640,
+            'height': 360,
+            'preference': 6,
+        },
+        '640x360_664b': {
+            'width': 640,
+            'height': 360,
+            'preference': 7,
+        },
+        '640x360_664m': {
+            'width': 640,
+            'height': 360,
+            'preference': 8,
+        },
+        '768x432_996': {
+            'width': 768,
+            'height': 432,
+            'preference': 9,
+        },
+        '768x432_1404': {
+            'width': 768,
+            'height': 432,
+            'preference': 10,
+        },
+        '960x540_2104': {
+            'width': 960,
+            'height': 540,
+            'preference': 11,
+        },
+        '1280x720_3072': {
+            'width': 1280,
+            'height': 720,
+            'preference': 12,
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        video_info = self._download_xml('http://www.nba.com/video/%s.xml' % video_id, video_id)
+        video_id = video_info.find('slug').text
+        title = video_info.find('headline').text
+        description = video_info.find('description').text
+        duration = parse_duration(video_info.find('length').text)
+        timestamp = int_or_none(video_info.find('dateCreated').attrib.get('uts'))
+
+        thumbnails = []
+        for image in video_info.find('images'):
+            thumbnails.append({
+                'id': image.attrib.get('cut'),
+                'url': image.text,
+                'width': int_or_none(image.attrib.get('width')),
+                'height': int_or_none(image.attrib.get('height')),
+            })
+
+        formats = []
+        for video_file in video_info.find('files').iter('file'):
+            video_url = video_file.text
+            if not video_url.startswith('http://'):
+                if video_url.endswith('.m3u8') or video_url.endswith('.f4m'):
+                    video_url = self._BASE_PATHS['akamai'] + video_url
+                else:
+                    video_url = self._BASE_PATHS['turner'] + video_url
+            if video_url.endswith('.m3u8'):
+                formats.extend(self._extract_m3u8_formats(video_url, video_id))
+            elif video_url.endswith('.f4m'):
+                formats.extend(self._extract_f4m_formats(video_url + '?hdcore=3.4.1.1', video_id))
+            else:
+                key = video_file.attrib.get('bitrate')
+                quality = self._QUALITIES[key]
+                formats.append({
+                    'format_id': key,
+                    'url': video_url,
+                    'width': quality['width'],
+                    'height': quality['height'],
+                    'preference': quality['preference'],
+                })
+        self._sort_formats(formats)
+
         return {
-            'title': metadata['name'],
-            'description': metadata.get('description'),
-            'duration': int_or_none(metadata.get('runtime')),
-            'thumbnail': metadata.get('image'),
-            'timestamp': parse_iso8601(metadata.get('releaseDate'))
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'duration': duration,
+            'timestamp': timestamp,
+            'thumbnails': thumbnails,
+            'formats': formats,
         }

From 97d5bfcba65c8575ab06a34e91fae30a5fda3161 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 3 Oct 2015 14:17:17 +0100
Subject: [PATCH 2683/6219] [engadget] accept short video urls

---
 youtube_dl/extractor/engadget.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/engadget.py b/youtube_dl/extractor/engadget.py
index 4ea37ebd9..e4180701d 100644
--- a/youtube_dl/extractor/engadget.py
+++ b/youtube_dl/extractor/engadget.py
@@ -10,7 +10,7 @@ from ..utils import (
 
 class EngadgetIE(InfoExtractor):
     _VALID_URL = r'''(?x)https?://www.engadget.com/
-        (?:video/5min/(?P<id>\d+)|
+        (?:video(?:/5min)?/(?P<id>\d+)|
             [\d/]+/.*?)
         '''
 

From 60d23e5e592aebe4a77dfb4ab70e87337967721c Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Sat, 3 Oct 2015 16:25:33 +0300
Subject: [PATCH 2684/6219] [tapely] Improve _VALID_URL

---
 youtube_dl/extractor/tapely.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tapely.py b/youtube_dl/extractor/tapely.py
index f1f43d0a7..744f9db38 100644
--- a/youtube_dl/extractor/tapely.py
+++ b/youtube_dl/extractor/tapely.py
@@ -16,7 +16,7 @@ from ..utils import (
 
 
 class TapelyIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?tape\.ly/(?P<id>[A-Za-z0-9\-_]+)(?:/(?P<songnr>\d+))?'
+    _VALID_URL = r'https?://(?:www\.)?(?:tape\.ly|tapely\.com)/(?P<id>[A-Za-z0-9\-_]+)(?:/(?P<songnr>\d+))?'
     _API_URL = 'http://tape.ly/showtape?id={0:}'
     _S3_SONG_URL = 'http://mytape.s3.amazonaws.com/{0:}'
     _SOUNDCLOUD_SONG_URL = 'http://api.soundcloud.com{0:}'
@@ -42,6 +42,10 @@ class TapelyIE(InfoExtractor):
                 'ext': 'm4a',
             },
         },
+        {
+            'url': 'https://tapely.com/my-grief-as-told-by-water',
+            'only_matching': True,
+        },
     ]
 
     def _real_extract(self, url):

From 30787f7259c4e6a08f691cc691f14fa0c8fe4b87 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 3 Oct 2015 19:28:48 +0100
Subject: [PATCH 2685/6219] [cspan] correct the clip info extraction

---
 youtube_dl/extractor/cspan.py | 58 ++++++++++++++++-------------------
 1 file changed, 27 insertions(+), 31 deletions(-)

diff --git a/youtube_dl/extractor/cspan.py b/youtube_dl/extractor/cspan.py
index fbefd37d0..994e080d5 100644
--- a/youtube_dl/extractor/cspan.py
+++ b/youtube_dl/extractor/cspan.py
@@ -18,22 +18,21 @@ class CSpanIE(InfoExtractor):
     IE_DESC = 'C-SPAN'
     _TESTS = [{
         'url': 'http://www.c-span.org/video/?313572-1/HolderonV',
-        'md5': '8e44ce11f0f725527daccc453f553eb0',
+        'md5': '067803f994e049b455a58b16e5aab442',
         'info_dict': {
             'id': '315139',
             'ext': 'mp4',
             'title': 'Attorney General Eric Holder on Voting Rights Act Decision',
-            'description': 'Attorney General Eric Holder spoke to reporters following the Supreme Court decision in Shelby County v. Holder in which the court ruled that the preclearance provisions of the Voting Rights Act could not be enforced until Congress established new guidelines for review.',
+            'description': 'Attorney General Eric Holder speaks to reporters following the Supreme Court decision in [Shelby County v. Holder], in which the court ruled that the preclearance provisions of the Voting Rights Act could not be enforced.',
         },
         'skip': 'Regularly fails on travis, for unknown reasons',
     }, {
         'url': 'http://www.c-span.org/video/?c4486943/cspan-international-health-care-models',
-        # For whatever reason, the served video alternates between
-        # two different ones
+        'md5': '4eafd1e91a75d2b1e6a3cbd0995816a2',
         'info_dict': {
-            'id': '340723',
+            'id': 'c4486943',
             'ext': 'mp4',
-            'title': 'International Health Care Models',
+            'title': 'CSPAN - International Health Care Models',
             'description': 'md5:7a985a2d595dba00af3d9c9f0783c967',
         }
     }, {
@@ -44,7 +43,7 @@ class CSpanIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'General Motors Ignition Switch Recall',
             'duration': 14848,
-            'description': 'md5:70c7c3b8fa63fa60d42772440596034c'
+            'description': 'md5:118081aedd24bf1d3b68b3803344e7f3'
         },
     }, {
         # Video from senate.gov
@@ -57,36 +56,33 @@ class CSpanIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        page_id = mobj.group('id')
-        webpage = self._download_webpage(url, page_id)
-        video_id = self._search_regex(r'progid=\'?([0-9]+)\'?>', webpage, 'video id')
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        matches = re.search(r'data-(prog|clip)id=\'([0-9]+)\'', webpage)
+        if matches:
+            video_type, video_id = matches.groups()
+            if video_type == 'prog':
+                video_type = 'program'
+        else:
+            senate_isvp_url = SenateISVPIE._search_iframe_url(webpage)
+            if senate_isvp_url:
+                title = self._og_search_title(webpage)
+                surl = smuggle_url(senate_isvp_url, {'force_title': title})
+                return self.url_result(surl, 'SenateISVP', video_id, title)
 
-        description = self._html_search_regex(
-            [
-                # The full description
-                r'<div class=\'expandable\'>(.*?)<a href=\'#\'',
-                # If the description is small enough the other div is not
-                # present, otherwise this is a stripped version
-                r'<p class=\'initial\'>(.*?)</p>'
-            ],
-            webpage, 'description', flags=re.DOTALL, default=None)
-
-        info_url = 'http://c-spanvideo.org/videoLibrary/assets/player/ajax-player.php?os=android&html5=program&id=' + video_id
-        data = self._download_json(info_url, video_id)
+        data = self._download_json(
+            'http://c-spanvideo.org/videoLibrary/assets/player/ajax-player.php?os=android&html5=%s&id=%s' % (video_type, video_id),
+            video_id)
 
         doc = self._download_xml(
-            'http://www.c-span.org/common/services/flashXml.php?programid=' + video_id,
+            'http://www.c-span.org/common/services/flashXml.php?%sid=%s' % (video_type, video_id),
             video_id)
 
+        description = self._html_search_meta('description', webpage)
+
         title = find_xpath_attr(doc, './/string', 'name', 'title').text
         thumbnail = find_xpath_attr(doc, './/string', 'name', 'poster').text
 
-        senate_isvp_url = SenateISVPIE._search_iframe_url(webpage)
-        if senate_isvp_url:
-            surl = smuggle_url(senate_isvp_url, {'force_title': title})
-            return self.url_result(surl, 'SenateISVP', video_id, title)
-
         files = data['video']['files']
         try:
             capfile = data['video']['capfile']['#text']
@@ -112,12 +108,12 @@ class CSpanIE(InfoExtractor):
 
         if len(entries) == 1:
             entry = dict(entries[0])
-            entry['id'] = video_id
+            entry['id'] = 'c' + video_id if video_type == 'clip' else video_id
             return entry
         else:
             return {
                 '_type': 'playlist',
                 'entries': entries,
                 'title': title,
-                'id': video_id,
+                'id': 'c' + video_id if video_type == 'clip' else video_id,
             }

From ef5acfe32de4c995625f9800cfe0776237961436 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 1 Sep 2015 23:05:19 +0100
Subject: [PATCH 2686/6219] [limelight] Add new extractor

---
 youtube_dl/extractor/__init__.py  |   5 +
 youtube_dl/extractor/limelight.py | 176 ++++++++++++++++++++++++++++++
 2 files changed, 181 insertions(+)
 create mode 100644 youtube_dl/extractor/limelight.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 495a18c17..20cc3660c 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -295,6 +295,11 @@ from .lifenews import (
     LifeNewsIE,
     LifeEmbedIE,
 )
+from .limelight import (
+    LimeLightMediaIE,
+    LimeLightChannelIE,
+    LimeLightChannelListIE,
+)
 from .liveleak import LiveLeakIE
 from .livestream import (
     LivestreamIE,
diff --git a/youtube_dl/extractor/limelight.py b/youtube_dl/extractor/limelight.py
new file mode 100644
index 000000000..dcfc215c7
--- /dev/null
+++ b/youtube_dl/extractor/limelight.py
@@ -0,0 +1,176 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    determine_ext,
+)
+
+
+class LimeLightBaseIE(InfoExtractor):
+
+    def get_playlist_service(self, id, method):
+        return self._download_json(self.PLAYLIST_SERVICE_URL % (id, method), id)
+
+    def get_api(self, orgId, id, method):
+        return self._download_json(self.API_URL % (orgId, id, method), id)
+
+    def process_data(self, mobileUrls, streams, properties):
+        video_id = properties['media_id']
+        formats = []
+
+        for mobileUrl in mobileUrls:
+            if '.m3u8' in mobileUrl['mobileUrl']:
+                formats.extend(self._extract_m3u8_formats(mobileUrl['mobileUrl'], video_id))
+            else:
+                formats.append({'url': mobileUrl['mobileUrl']})
+
+        for stream in streams:
+            if '.f4m' in stream['url']:
+                formats.extend(self._extract_f4m_formats(stream['url'], video_id))
+            else:
+                fmt = {
+                    'url': stream.get('url'),
+                    'abr': stream.get('audioBitRate'),
+                    'vbr': stream.get('videoBitRate'),
+                    'fps': stream.get('videoFrameRate'),
+                    'width': stream.get('videoWidthInPixels'),
+                    'height': stream.get('videoHeightInPixels'),
+                    'ext': determine_ext(stream.get('url'))
+                }
+                rtmp = re.search(r'^(?P<url>rtmp://[^/]+/(?P<app>.+))/(?P<playpath>mp4:.+)$', stream['url'])
+                if rtmp:
+                    fmt.update({
+                        'url': rtmp.group('url'),
+                        'play_path': rtmp.group('playpath'),
+                        'app': rtmp.group('app'),
+                    })
+                formats.append(fmt)
+
+        self._sort_formats(formats)
+
+        title = properties['title']
+        description = properties.get('description')
+        timestamp = properties.get('create_date')
+        duration = int_or_none(properties.get('duration_in_milliseconds'))
+        filesize = properties.get('total_storage_in_bytes')
+        categories = [properties.get('category')]
+        thumbnails = [{
+            'url': thumbnail.get('url'),
+            'width': int_or_none(thumbnail.get('width')),
+            'height': int_or_none(thumbnail.get('height')),
+        } for thumbnail in properties.get('thumbnails')]
+        subtitles = {caption.get('language_code'): [{'url': caption.get('url')}] for caption in properties.get('captions')}
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'formats': formats,
+            'timestamp': timestamp,
+            'duration': duration,
+            'filesize': filesize,
+            'categories': categories,
+            'thumbnails': thumbnails,
+            'subtitles': subtitles,
+        }
+
+
+class LimeLightMediaIE(LimeLightBaseIE):
+    IE_NAME = 'limelight'
+    _VALID_URL = r'http://link\.videoplatform\.limelight\.com/media/?.*mediaId=(?P<id>[a-z0-9]{32})'
+    _TEST = {
+        'url': 'http://link.videoplatform.limelight.com/media/?mediaId=3ffd040b522b4485b6d84effc750cd86',
+        'md5': '3213605088be599705677ef785db6972',
+        'info_dict': {
+            'id': '3ffd040b522b4485b6d84effc750cd86',
+            'ext': 'mp4',
+            'title': 'HaP and the HB Prince Trailer',
+            'description': 'As Harry Potter begins his 6th year at Hogwarts School of Witchcraft and Wizardry, he discovers an old book marked mysteriously "This book is the property of the Half-Blood Prince" and begins to learn more about Lord Voldemort\'s dark past.',
+            'thumbnail': 're:^https?://.*\.jpeg$',
+            'duration': 144230,
+            'timestamp': 1244136834,
+            "upload_date": "20090604",
+        }
+    }
+    PLAYLIST_SERVICE_URL = 'http://production-ps.lvp.llnw.net/r/PlaylistService/media/%s/%s'
+    API_URL = 'http://api.video.limelight.com/rest/organizations/%s/media/%s/%s.json'
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        mobile_json_data = self.get_playlist_service(video_id, 'getMobilePlaylistByMediaId')
+        pc_json_data = self.get_playlist_service(video_id, 'getPlaylistByMediaId')
+        properties = self.get_api(pc_json_data['orgId'], video_id, 'properties')
+
+        return self.process_data(mobile_json_data['mediaList'][0]['mobileUrls'], pc_json_data['playlistItems'][0]['streams'], properties)
+
+
+class LimeLightChannelIE(LimeLightBaseIE):
+    IE_NAME = 'limelight:channel'
+    _VALID_URL = r'http://link\.videoplatform\.limelight\.com/media/?.*channelId=(?P<id>[a-z0-9]{32})'
+    _TEST = {
+        'url': 'http://link.videoplatform.limelight.com/media/?channelId=ab6a524c379342f9b23642917020c082',
+        'info_dict': {
+            'id': 'ab6a524c379342f9b23642917020c082',
+            'title': 'Javascript Sample Code',
+        },
+        'playlist_mincount': 3,
+    }
+    PLAYLIST_SERVICE_URL = 'http://production-ps.lvp.llnw.net/r/PlaylistService/channel/%s/%s'
+    API_URL = 'http://api.video.limelight.com/rest/organizations/%s/channels/%s/%s.json'
+
+    def _real_extract(self, url):
+        channel_id = self._match_id(url)
+
+        mobile_json_data = self.get_playlist_service(channel_id, 'getMobilePlaylistWithNItemsByChannelId?begin=0&count=-1')
+        pc_json_data = self.get_playlist_service(channel_id, 'getPlaylistByChannelId')
+        medias = self.get_api(pc_json_data['orgId'], channel_id, 'media')
+
+        entries = []
+        for i in range(len(medias['media_list'])):
+            entries.append(self.process_data(mobile_json_data['mediaList'][i]['mobileUrls'], pc_json_data['playlistItems'][i]['streams'], medias['media_list'][i]))
+
+        return {
+            'id': channel_id,
+            'title': pc_json_data['title'],
+            'entries': entries,
+            '_type': 'playlist',
+        }
+
+
+class LimeLightChannelListIE(LimeLightBaseIE):
+    IE_NAME = 'limelight:channel_list'
+    _VALID_URL = r'http://link\.videoplatform\.limelight\.com/media/?.*channelListId=(?P<id>[a-z0-9]{32})'
+    _TEST = {
+        'url': 'http://link.videoplatform.limelight.com/media/?channelListId=301b117890c4465c8179ede21fd92e2b',
+        'info_dict': {
+            'id': '301b117890c4465c8179ede21fd92e2b',
+            'title': 'Website - Hero Player',
+        },
+        'playlist_mincount': 2,
+    }
+    PLAYLIST_SERVICE_URL = 'http://production-ps.lvp.llnw.net/r/PlaylistService/channel_list/%s/%s'
+
+    def _real_extract(self, url):
+        channel_list_id = self._match_id(url)
+
+        json_data = self.get_playlist_service(channel_list_id, 'getMobileChannelListById')
+
+        entries = []
+        for channel in json_data['channelList']:
+            entries.append({
+                'url': 'http://link.videoplatform.limelight.com/media/?channelId=%s' % channel['id'],
+                '_type': 'url',
+                'ie_key': 'LimeLightChannel',
+            })
+
+        return {
+            'id': channel_list_id,
+            'title': json_data['title'],
+            'entries': entries,
+            '_type': 'playlist',
+        }

From 4bba371644818d79b5f8481b5b31c53ea8ecbcc5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 4 Oct 2015 20:33:42 +0600
Subject: [PATCH 2687/6219] [YoutubeDL] Autocalculate ext for subtitles when
 missing

---
 youtube_dl/YoutubeDL.py        | 11 +++++++++--
 youtube_dl/extractor/common.py |  1 +
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index d65253882..adf70d658 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1232,13 +1232,20 @@ class YoutubeDL(object):
             except (ValueError, OverflowError, OSError):
                 pass
 
+        subtitles = info_dict.get('subtitles')
+        if subtitles:
+            for _, subtitle in subtitles.items():
+                for subtitle_format in subtitle:
+                    if 'ext' not in subtitle_format:
+                        subtitle_format['ext'] = determine_ext(subtitle_format['url']).lower()
+
         if self.params.get('listsubtitles', False):
             if 'automatic_captions' in info_dict:
                 self.list_subtitles(info_dict['id'], info_dict.get('automatic_captions'), 'automatic captions')
-            self.list_subtitles(info_dict['id'], info_dict.get('subtitles'), 'subtitles')
+            self.list_subtitles(info_dict['id'], subtitles, 'subtitles')
             return
         info_dict['requested_subtitles'] = self.process_subtitles(
-            info_dict['id'], info_dict.get('subtitles'),
+            info_dict['id'], subtitles,
             info_dict.get('automatic_captions'))
 
         # We now pick which formats have to be downloaded
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 4fe2307cd..dbae75406 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -165,6 +165,7 @@ class InfoExtractor(object):
                     with the "ext" entry and one of:
                         * "data": The subtitles file contents
                         * "url": A URL pointing to the subtitles file
+                    "ext" will be calculated from URL if missing
     automatic_captions: Like 'subtitles', used by the YoutubeIE for
                     automatically generated captions
     duration:       Length of the video in seconds, as an integer.

From d7fc56318b72607758e7484c22076ec2999f10b0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 4 Oct 2015 20:41:57 +0600
Subject: [PATCH 2688/6219] [limelight] Fix python 2.6, simplify, make more
 robust (Closes #6734)

---
 youtube_dl/extractor/__init__.py  |   6 +-
 youtube_dl/extractor/limelight.py | 185 ++++++++++++++++++------------
 2 files changed, 113 insertions(+), 78 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 20cc3660c..3ace1cc2c 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -296,9 +296,9 @@ from .lifenews import (
     LifeEmbedIE,
 )
 from .limelight import (
-    LimeLightMediaIE,
-    LimeLightChannelIE,
-    LimeLightChannelListIE,
+    LimelightMediaIE,
+    LimelightChannelIE,
+    LimelightChannelListIE,
 )
 from .liveleak import LiveLeakIE
 from .livestream import (
diff --git a/youtube_dl/extractor/limelight.py b/youtube_dl/extractor/limelight.py
index dcfc215c7..599d8413d 100644
--- a/youtube_dl/extractor/limelight.py
+++ b/youtube_dl/extractor/limelight.py
@@ -5,65 +5,105 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
-    int_or_none,
     determine_ext,
+    float_or_none,
+    int_or_none,
 )
 
 
-class LimeLightBaseIE(InfoExtractor):
+class LimelightBaseIE(InfoExtractor):
+    _PLAYLIST_SERVICE_URL = 'http://production-ps.lvp.llnw.net/r/PlaylistService/%s/%s/%s'
+    _API_URL = 'http://api.video.limelight.com/rest/organizations/%s/%s/%s/%s.json'
 
-    def get_playlist_service(self, id, method):
-        return self._download_json(self.PLAYLIST_SERVICE_URL % (id, method), id)
+    def _call_playlist_service(self, item_id, method, fatal=True):
+        return self._download_json(
+            self._PLAYLIST_SERVICE_URL % (self._PLAYLIST_SERVICE_PATH, item_id, method),
+            item_id, 'Downloading PlaylistService %s JSON' % method, fatal=fatal)
 
-    def get_api(self, orgId, id, method):
-        return self._download_json(self.API_URL % (orgId, id, method), id)
+    def _call_api(self, organization_id, item_id, method):
+        return self._download_json(
+            self._API_URL % (organization_id, self._API_PATH, item_id, method),
+            item_id, 'Downloading API %s JSON' % method)
 
-    def process_data(self, mobileUrls, streams, properties):
+    def _extract(self, item_id, pc_method, mobile_method, meta_method):
+        pc = self._call_playlist_service(item_id, pc_method)
+        metadata = self._call_api(pc['orgId'], item_id, meta_method)
+        mobile = self._call_playlist_service(item_id, mobile_method, fatal=False)
+        return pc, mobile, metadata
+
+    def _extract_info(self, streams, mobile_urls, properties):
         video_id = properties['media_id']
         formats = []
 
-        for mobileUrl in mobileUrls:
-            if '.m3u8' in mobileUrl['mobileUrl']:
-                formats.extend(self._extract_m3u8_formats(mobileUrl['mobileUrl'], video_id))
-            else:
-                formats.append({'url': mobileUrl['mobileUrl']})
-
         for stream in streams:
-            if '.f4m' in stream['url']:
-                formats.extend(self._extract_f4m_formats(stream['url'], video_id))
+            stream_url = stream.get('url')
+            if not stream_url:
+                continue
+            if '.f4m' in stream_url:
+                formats.extend(self._extract_f4m_formats(stream_url, video_id))
             else:
                 fmt = {
-                    'url': stream.get('url'),
-                    'abr': stream.get('audioBitRate'),
-                    'vbr': stream.get('videoBitRate'),
-                    'fps': stream.get('videoFrameRate'),
-                    'width': stream.get('videoWidthInPixels'),
-                    'height': stream.get('videoHeightInPixels'),
-                    'ext': determine_ext(stream.get('url'))
+                    'url': stream_url,
+                    'abr': float_or_none(stream.get('audioBitRate')),
+                    'vbr': float_or_none(stream.get('videoBitRate')),
+                    'fps': float_or_none(stream.get('videoFrameRate')),
+                    'width': int_or_none(stream.get('videoWidthInPixels')),
+                    'height': int_or_none(stream.get('videoHeightInPixels')),
+                    'ext': determine_ext(stream_url)
                 }
-                rtmp = re.search(r'^(?P<url>rtmp://[^/]+/(?P<app>.+))/(?P<playpath>mp4:.+)$', stream['url'])
+                rtmp = re.search(r'^(?P<url>rtmpe?://[^/]+/(?P<app>.+))/(?P<playpath>mp4:.+)$', stream_url)
                 if rtmp:
+                    format_id = 'rtmp'
+                    if stream.get('videoBitRate'):
+                        format_id += '-%d' % int_or_none(stream['videoBitRate'])
                     fmt.update({
                         'url': rtmp.group('url'),
                         'play_path': rtmp.group('playpath'),
                         'app': rtmp.group('app'),
+                        'ext': 'flv',
+                        'format_id': format_id,
                     })
                 formats.append(fmt)
 
+        for mobile_url in mobile_urls:
+            media_url = mobile_url.get('mobileUrl')
+            if not media_url:
+                continue
+            format_id = mobile_url.get('targetMediaPlatform')
+            if determine_ext(media_url) == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    media_url, video_id, 'mp4', entry_protocol='m3u8_native',
+                    preference=-1, m3u8_id=format_id))
+            else:
+                formats.append({
+                    'url': media_url,
+                    'format_id': format_id,
+                    'preference': -1,
+                })
+
         self._sort_formats(formats)
 
         title = properties['title']
         description = properties.get('description')
-        timestamp = properties.get('create_date')
-        duration = int_or_none(properties.get('duration_in_milliseconds'))
-        filesize = properties.get('total_storage_in_bytes')
+        timestamp = int_or_none(properties.get('publish_date') or properties.get('create_date'))
+        duration = float_or_none(properties.get('duration_in_milliseconds'), 1000)
+        filesize = int_or_none(properties.get('total_storage_in_bytes'))
         categories = [properties.get('category')]
+        tags = properties.get('tags', [])
         thumbnails = [{
-            'url': thumbnail.get('url'),
+            'url': thumbnail['url'],
             'width': int_or_none(thumbnail.get('width')),
             'height': int_or_none(thumbnail.get('height')),
-        } for thumbnail in properties.get('thumbnails')]
-        subtitles = {caption.get('language_code'): [{'url': caption.get('url')}] for caption in properties.get('captions')}
+        } for thumbnail in properties.get('thumbnails', []) if thumbnail.get('url')]
+
+        subtitles = {}
+        for caption in properties.get('captions', {}):
+            lang = caption.get('language_code')
+            subtitles_url = caption.get('url')
+            if lang and subtitles_url:
+                subtitles[lang] = [{
+                    'url': subtitles_url,
+                }]
 
         return {
             'id': video_id,
@@ -74,44 +114,50 @@ class LimeLightBaseIE(InfoExtractor):
             'duration': duration,
             'filesize': filesize,
             'categories': categories,
+            'tags': tags,
             'thumbnails': thumbnails,
             'subtitles': subtitles,
         }
 
 
-class LimeLightMediaIE(LimeLightBaseIE):
+class LimelightMediaIE(LimelightBaseIE):
     IE_NAME = 'limelight'
-    _VALID_URL = r'http://link\.videoplatform\.limelight\.com/media/?.*mediaId=(?P<id>[a-z0-9]{32})'
+    _VALID_URL = r'(?:limelight:media:|http://link\.videoplatform\.limelight\.com/media/\??\bmediaId=)(?P<id>[a-z0-9]{32})'
     _TEST = {
         'url': 'http://link.videoplatform.limelight.com/media/?mediaId=3ffd040b522b4485b6d84effc750cd86',
-        'md5': '3213605088be599705677ef785db6972',
         'info_dict': {
             'id': '3ffd040b522b4485b6d84effc750cd86',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': 'HaP and the HB Prince Trailer',
             'description': 'As Harry Potter begins his 6th year at Hogwarts School of Witchcraft and Wizardry, he discovers an old book marked mysteriously "This book is the property of the Half-Blood Prince" and begins to learn more about Lord Voldemort\'s dark past.',
             'thumbnail': 're:^https?://.*\.jpeg$',
-            'duration': 144230,
+            'duration': 144.23,
             'timestamp': 1244136834,
-            "upload_date": "20090604",
-        }
+            'upload_date': '20090604',
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        },
     }
-    PLAYLIST_SERVICE_URL = 'http://production-ps.lvp.llnw.net/r/PlaylistService/media/%s/%s'
-    API_URL = 'http://api.video.limelight.com/rest/organizations/%s/media/%s/%s.json'
+    _PLAYLIST_SERVICE_PATH = 'media'
+    _API_PATH = 'media'
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        mobile_json_data = self.get_playlist_service(video_id, 'getMobilePlaylistByMediaId')
-        pc_json_data = self.get_playlist_service(video_id, 'getPlaylistByMediaId')
-        properties = self.get_api(pc_json_data['orgId'], video_id, 'properties')
+        pc, mobile, metadata = self._extract(
+            video_id, 'getPlaylistByMediaId', 'getMobilePlaylistByMediaId', 'properties')
 
-        return self.process_data(mobile_json_data['mediaList'][0]['mobileUrls'], pc_json_data['playlistItems'][0]['streams'], properties)
+        return self._extract_info(
+            pc['playlistItems'][0].get('streams', []),
+            mobile['mediaList'][0].get('mobileUrls', []) if mobile else [],
+            metadata)
 
 
-class LimeLightChannelIE(LimeLightBaseIE):
+class LimelightChannelIE(LimelightBaseIE):
     IE_NAME = 'limelight:channel'
-    _VALID_URL = r'http://link\.videoplatform\.limelight\.com/media/?.*channelId=(?P<id>[a-z0-9]{32})'
+    _VALID_URL = r'(?:limelight:channel:|http://link\.videoplatform\.limelight\.com/media/\??\bchannelId=)(?P<id>[a-z0-9]{32})'
     _TEST = {
         'url': 'http://link.videoplatform.limelight.com/media/?channelId=ab6a524c379342f9b23642917020c082',
         'info_dict': {
@@ -120,31 +166,29 @@ class LimeLightChannelIE(LimeLightBaseIE):
         },
         'playlist_mincount': 3,
     }
-    PLAYLIST_SERVICE_URL = 'http://production-ps.lvp.llnw.net/r/PlaylistService/channel/%s/%s'
-    API_URL = 'http://api.video.limelight.com/rest/organizations/%s/channels/%s/%s.json'
+    _PLAYLIST_SERVICE_PATH = 'channel'
+    _API_PATH = 'channels'
 
     def _real_extract(self, url):
         channel_id = self._match_id(url)
 
-        mobile_json_data = self.get_playlist_service(channel_id, 'getMobilePlaylistWithNItemsByChannelId?begin=0&count=-1')
-        pc_json_data = self.get_playlist_service(channel_id, 'getPlaylistByChannelId')
-        medias = self.get_api(pc_json_data['orgId'], channel_id, 'media')
+        pc, mobile, medias = self._extract(
+            channel_id, 'getPlaylistByChannelId',
+            'getMobilePlaylistWithNItemsByChannelId?begin=0&count=-1', 'media')
 
-        entries = []
-        for i in range(len(medias['media_list'])):
-            entries.append(self.process_data(mobile_json_data['mediaList'][i]['mobileUrls'], pc_json_data['playlistItems'][i]['streams'], medias['media_list'][i]))
+        entries = [
+            self._extract_info(
+                pc['playlistItems'][i].get('streams', []),
+                mobile['mediaList'][i].get('mobileUrls', []) if mobile else [],
+                medias['media_list'][i])
+            for i in range(len(medias['media_list']))]
 
-        return {
-            'id': channel_id,
-            'title': pc_json_data['title'],
-            'entries': entries,
-            '_type': 'playlist',
-        }
+        return self.playlist_result(entries, channel_id, pc['title'])
 
 
-class LimeLightChannelListIE(LimeLightBaseIE):
+class LimelightChannelListIE(LimelightBaseIE):
     IE_NAME = 'limelight:channel_list'
-    _VALID_URL = r'http://link\.videoplatform\.limelight\.com/media/?.*channelListId=(?P<id>[a-z0-9]{32})'
+    _VALID_URL = r'(?:limelight:channel_list:|http://link\.videoplatform\.limelight\.com/media/\?.*?\bchannelListId=)(?P<id>[a-z0-9]{32})'
     _TEST = {
         'url': 'http://link.videoplatform.limelight.com/media/?channelListId=301b117890c4465c8179ede21fd92e2b',
         'info_dict': {
@@ -153,24 +197,15 @@ class LimeLightChannelListIE(LimeLightBaseIE):
         },
         'playlist_mincount': 2,
     }
-    PLAYLIST_SERVICE_URL = 'http://production-ps.lvp.llnw.net/r/PlaylistService/channel_list/%s/%s'
+    _PLAYLIST_SERVICE_PATH = 'channel_list'
 
     def _real_extract(self, url):
         channel_list_id = self._match_id(url)
 
-        json_data = self.get_playlist_service(channel_list_id, 'getMobileChannelListById')
+        channel_list = self._call_playlist_service(channel_list_id, 'getMobileChannelListById')
 
-        entries = []
-        for channel in json_data['channelList']:
-            entries.append({
-                'url': 'http://link.videoplatform.limelight.com/media/?channelId=%s' % channel['id'],
-                '_type': 'url',
-                'ie_key': 'LimeLightChannel',
-            })
+        entries = [
+            self.url_result('limelight:channel:%s' % channel['id'], 'LimelightChannel')
+            for channel in channel_list['channelList']]
 
-        return {
-            'id': channel_list_id,
-            'title': json_data['title'],
-            'entries': entries,
-            '_type': 'playlist',
-        }
+        return self.playlist_result(entries, channel_list_id, channel_list['title'])

From 9c544e2537abda1d65e96f2b33a79984f3ab7c10 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 4 Oct 2015 20:48:44 +0600
Subject: [PATCH 2689/6219] [limelight] Add test video with subtitles

---
 youtube_dl/extractor/limelight.py | 24 +++++++++++++++++++++---
 1 file changed, 21 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/limelight.py b/youtube_dl/extractor/limelight.py
index 599d8413d..fb03dd527 100644
--- a/youtube_dl/extractor/limelight.py
+++ b/youtube_dl/extractor/limelight.py
@@ -123,13 +123,13 @@ class LimelightBaseIE(InfoExtractor):
 class LimelightMediaIE(LimelightBaseIE):
     IE_NAME = 'limelight'
     _VALID_URL = r'(?:limelight:media:|http://link\.videoplatform\.limelight\.com/media/\??\bmediaId=)(?P<id>[a-z0-9]{32})'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://link.videoplatform.limelight.com/media/?mediaId=3ffd040b522b4485b6d84effc750cd86',
         'info_dict': {
             'id': '3ffd040b522b4485b6d84effc750cd86',
             'ext': 'flv',
             'title': 'HaP and the HB Prince Trailer',
-            'description': 'As Harry Potter begins his 6th year at Hogwarts School of Witchcraft and Wizardry, he discovers an old book marked mysteriously "This book is the property of the Half-Blood Prince" and begins to learn more about Lord Voldemort\'s dark past.',
+            'description': 'md5:8005b944181778e313d95c1237ddb640',
             'thumbnail': 're:^https?://.*\.jpeg$',
             'duration': 144.23,
             'timestamp': 1244136834,
@@ -139,7 +139,25 @@ class LimelightMediaIE(LimelightBaseIE):
             # rtmp download
             'skip_download': True,
         },
-    }
+    }, {
+        # video with subtitles
+        'url': 'limelight:media:a3e00274d4564ec4a9b29b9466432335',
+        'info_dict': {
+            'id': 'a3e00274d4564ec4a9b29b9466432335',
+            'ext': 'flv',
+            'title': '3Play Media Overview Video',
+            'description': '',
+            'thumbnail': 're:^https?://.*\.jpeg$',
+            'duration': 78.101,
+            'timestamp': 1338929955,
+            'upload_date': '20120605',
+            'subtitles': 'mincount:9',
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        },
+    }]
     _PLAYLIST_SERVICE_PATH = 'media'
     _API_PATH = 'media'
 

From 0659dfccfea9df3206c476e83a2b090456c25a83 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 4 Oct 2015 21:13:13 +0600
Subject: [PATCH 2690/6219] [pbs] Improve player regex (Closes #7059)

---
 youtube_dl/extractor/pbs.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 683c81de3..0bca3152b 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -167,7 +167,7 @@ class PBSIE(InfoExtractor):
                 return media_id, presumptive_id, upload_date
 
             url = self._search_regex(
-                r'<iframe\s+[^>]*\s+src=["\']([^\'"]+partnerplayer[^\'"]+)["\']',
+                r'(?s)<iframe[^>]+?(?:[a-z-]+?=["\'].+?["\'][^>]+?)*?\bsrc=["\']([^\'"]+partnerplayer[^\'"]+)["\']',
                 webpage, 'player URL')
             mobj = re.match(self._VALID_URL, url)
 

From 96229998c29705c8ee4230915ec7ff050bcfecf8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 4 Oct 2015 21:19:47 +0600
Subject: [PATCH 2691/6219] [pbs] Allow empty attribute in player regex

---
 youtube_dl/extractor/pbs.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 0bca3152b..66b3dda47 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -167,7 +167,7 @@ class PBSIE(InfoExtractor):
                 return media_id, presumptive_id, upload_date
 
             url = self._search_regex(
-                r'(?s)<iframe[^>]+?(?:[a-z-]+?=["\'].+?["\'][^>]+?)*?\bsrc=["\']([^\'"]+partnerplayer[^\'"]+)["\']',
+                r'(?s)<iframe[^>]+?(?:[a-z-]+?=["\'].*?["\'][^>]+?)*?\bsrc=["\']([^\'"]+partnerplayer[^\'"]+)["\']',
                 webpage, 'player URL')
             mobj = re.match(self._VALID_URL, url)
 

From 90ab741e909c949039e31805da04f5e546a1a8c1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 4 Oct 2015 21:37:49 +0600
Subject: [PATCH 2692/6219] [pbs] Add test for #7059

---
 youtube_dl/extractor/pbs.py | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 66b3dda47..6923c6094 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -134,6 +134,24 @@ class PBSIE(InfoExtractor):
             'params': {
                 'skip_download': True,  # requires ffmpeg
             },
+        },
+        {
+            # Video embedded in iframe containing angle brackets as attribute's value (e.g.
+            # "<iframe style='position: absolute;<br />\ntop: 0; left: 0;' ...", see
+            # https://github.com/rg3/youtube-dl/issues/7059)
+            'url': 'http://www.pbs.org/food/features/a-chefs-life-season-3-episode-5-prickly-business/',
+            'info_dict': {
+                'id': '2365546844',
+                'display_id': 'a-chefs-life-season-3-episode-5-prickly-business',
+                'ext': 'mp4',
+                'title': "A Chef's Life - Season 3, Ep. 5: Prickly Business",
+                'description': 'md5:61db2ddf27c9912f09c241014b118ed1',
+                'duration': 1480,
+                'thumbnail': 're:^https?://.*\.jpg$',
+            },
+            'params': {
+                'skip_download': True,  # requires ffmpeg
+            },
         }
     ]
 

From e759a00119768862e63dbda33522f2399f1f43a4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 5 Oct 2015 20:21:53 +0600
Subject: [PATCH 2693/6219] [appletrailers] Quotes consistency

---
 youtube_dl/extractor/appletrailers.py | 68 +++++++++++++--------------
 1 file changed, 34 insertions(+), 34 deletions(-)

diff --git a/youtube_dl/extractor/appletrailers.py b/youtube_dl/extractor/appletrailers.py
index 576f03b5b..f68dc3236 100644
--- a/youtube_dl/extractor/appletrailers.py
+++ b/youtube_dl/extractor/appletrailers.py
@@ -13,53 +13,53 @@ from ..utils import (
 class AppleTrailersIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?trailers\.apple\.com/(?:trailers|ca)/(?P<company>[^/]+)/(?P<movie>[^/]+)'
     _TESTS = [{
-        "url": "http://trailers.apple.com/trailers/wb/manofsteel/",
+        'url': 'http://trailers.apple.com/trailers/wb/manofsteel/',
         'info_dict': {
             'id': 'manofsteel',
         },
-        "playlist": [
+        'playlist': [
             {
-                "md5": "d97a8e575432dbcb81b7c3acb741f8a8",
-                "info_dict": {
-                    "id": "manofsteel-trailer4",
-                    "ext": "mov",
-                    "duration": 111,
-                    "title": "Trailer 4",
-                    "upload_date": "20130523",
-                    "uploader_id": "wb",
+                'md5': 'd97a8e575432dbcb81b7c3acb741f8a8',
+                'info_dict': {
+                    'id': 'manofsteel-trailer4',
+                    'ext': 'mov',
+                    'duration': 111,
+                    'title': 'Trailer 4',
+                    'upload_date': '20130523',
+                    'uploader_id': 'wb',
                 },
             },
             {
-                "md5": "b8017b7131b721fb4e8d6f49e1df908c",
-                "info_dict": {
-                    "id": "manofsteel-trailer3",
-                    "ext": "mov",
-                    "duration": 182,
-                    "title": "Trailer 3",
-                    "upload_date": "20130417",
-                    "uploader_id": "wb",
+                'md5': 'b8017b7131b721fb4e8d6f49e1df908c',
+                'info_dict': {
+                    'id': 'manofsteel-trailer3',
+                    'ext': 'mov',
+                    'duration': 182,
+                    'title': 'Trailer 3',
+                    'upload_date': '20130417',
+                    'uploader_id': 'wb',
                 },
             },
             {
-                "md5": "d0f1e1150989b9924679b441f3404d48",
-                "info_dict": {
-                    "id": "manofsteel-trailer",
-                    "ext": "mov",
-                    "duration": 148,
-                    "title": "Trailer",
-                    "upload_date": "20121212",
-                    "uploader_id": "wb",
+                'md5': 'd0f1e1150989b9924679b441f3404d48',
+                'info_dict': {
+                    'id': 'manofsteel-trailer',
+                    'ext': 'mov',
+                    'duration': 148,
+                    'title': 'Trailer',
+                    'upload_date': '20121212',
+                    'uploader_id': 'wb',
                 },
             },
             {
-                "md5": "5fe08795b943eb2e757fa95cb6def1cb",
-                "info_dict": {
-                    "id": "manofsteel-teaser",
-                    "ext": "mov",
-                    "duration": 93,
-                    "title": "Teaser",
-                    "upload_date": "20120721",
-                    "uploader_id": "wb",
+                'md5': '5fe08795b943eb2e757fa95cb6def1cb',
+                'info_dict': {
+                    'id': 'manofsteel-teaser',
+                    'ext': 'mov',
+                    'duration': 93,
+                    'title': 'Teaser',
+                    'upload_date': '20120721',
+                    'uploader_id': 'wb',
                 },
             },
         ]

From 2a27e66234ec0030d2ca6f1ddf229db7f7eb8ded Mon Sep 17 00:00:00 2001
From: Oli Allen <oli@oliallen.com>
Date: Thu, 1 Oct 2015 22:50:12 +0100
Subject: [PATCH 2694/6219] [tumblr] Added support for HD video where available
 (#7036)

[tumblr] Replaced test URL for HD video as old one lead to 404

[tumblr] Don't make assumptions about video resolution, cleaner handling of no HD version available

[tumblr] Removed extraneous resolution key in HD video tests
---
 youtube_dl/extractor/tumblr.py | 58 ++++++++++++++++++++++++++++++++--
 1 file changed, 55 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/tumblr.py b/youtube_dl/extractor/tumblr.py
index 3d3b635e4..71cced562 100644
--- a/youtube_dl/extractor/tumblr.py
+++ b/youtube_dl/extractor/tumblr.py
@@ -28,6 +28,32 @@ class TumblrIE(InfoExtractor):
             'description': 'md5:dba62ac8639482759c8eb10ce474586a',
             'thumbnail': 're:http://.*\.jpg',
         }
+    }, {
+        'url': 'http://hdvideotest.tumblr.com/post/130323439814/test-description-for-my-hd-video',
+        'md5': '99a84522f60972bf064a0b80f87bcbb5',
+        'info_dict': {
+            'id': '130323439814',
+            'ext': 'mp4',
+            'title': 'HD Video Testing \u2014 Test description for my HD video',
+            'description': 'md5:97cc3ab5fcd27ee4af6356701541319c',
+            'thumbnail': 're:http://.*\.jpg',
+        },
+        'params': {
+            'format': 'sd',
+        },
+    }, {
+        'url': 'http://hdvideotest.tumblr.com/post/130323439814/test-description-for-my-hd-video',
+        'md5': '7ae503065ad150122dc3089f8cf1546c',
+        'info_dict': {
+            'id': '130323439814',
+            'ext': 'mp4',
+            'title': 'HD Video Testing \u2014 Test description for my HD video',
+            'description': 'md5:97cc3ab5fcd27ee4af6356701541319c',
+            'thumbnail': 're:http://.*\.jpg',
+        },
+        'params': {
+            'format': 'hd',
+        },
     }, {
         'url': 'http://naked-yogi.tumblr.com/post/118312946248/naked-smoking-stretching',
         'md5': 'de07e5211d60d4f3a2c3df757ea9f6ab',
@@ -57,6 +83,8 @@ class TumblrIE(InfoExtractor):
         video_id = m_url.group('id')
         blog = m_url.group('blog_name')
 
+        video_urls = []
+
         url = 'http://%s.tumblr.com/post/%s/' % (blog, video_id)
         webpage, urlh = self._download_webpage_handle(url, video_id)
 
@@ -68,8 +96,32 @@ class TumblrIE(InfoExtractor):
 
         iframe = self._download_webpage(iframe_url, video_id,
                                         'Downloading iframe page')
-        video_url = self._search_regex(r'<source src="([^"]+)"',
-                                       iframe, 'video url')
+
+        sd_video_url = self._search_regex(r'<source src="([^"]+)"',
+                                          iframe, 'sd video url')
+        resolution_id = sd_video_url.split("/")[-1] + 'p'
+        if len(resolution_id) != 4:
+            resolution_id = None
+        video_urls.append({
+            'ext': 'mp4',
+            'format_id': 'sd',
+            'url': sd_video_url,
+            'resolution': resolution_id,
+        })
+
+        hd_video_url = self._search_regex(r'hdUrl":"([^"]+)"', iframe,
+                                          'hd video url', default=None)
+        if hd_video_url:
+            hd_video_url = hd_video_url.replace("\\", "")
+            resolution_id = hd_video_url.split("/")[-1] + 'p'
+            if len(resolution_id) != 4:
+                resolution_id = None
+            video_urls.append({
+                'ext': 'mp4',
+                'format_id': 'hd',
+                'url': hd_video_url,
+                'resolution': resolution_id,
+            })
 
         # The only place where you can get a title, it's not complete,
         # but searching in other places doesn't work for all videos
@@ -79,7 +131,7 @@ class TumblrIE(InfoExtractor):
 
         return {
             'id': video_id,
-            'url': video_url,
+            'formats': video_urls,
             'ext': 'mp4',
             'title': video_title,
             'description': self._og_search_description(webpage, default=None),

From 140ac7396542d92a8ddd53be6c35c7a79db16180 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 5 Oct 2015 21:53:01 +0600
Subject: [PATCH 2695/6219] [tumblr] Simplify and extract duration

---
 youtube_dl/extractor/tumblr.py | 60 ++++++++++++++++++----------------
 1 file changed, 32 insertions(+), 28 deletions(-)

diff --git a/youtube_dl/extractor/tumblr.py b/youtube_dl/extractor/tumblr.py
index 71cced562..449ba29fa 100644
--- a/youtube_dl/extractor/tumblr.py
+++ b/youtube_dl/extractor/tumblr.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..utils import int_or_none
 
 
 class TumblrIE(InfoExtractor):
@@ -83,8 +84,6 @@ class TumblrIE(InfoExtractor):
         video_id = m_url.group('id')
         blog = m_url.group('blog_name')
 
-        video_urls = []
-
         url = 'http://%s.tumblr.com/post/%s/' % (blog, video_id)
         webpage, urlh = self._download_webpage_handle(url, video_id)
 
@@ -94,34 +93,38 @@ class TumblrIE(InfoExtractor):
         if iframe_url is None:
             return self.url_result(urlh.geturl(), 'Generic')
 
-        iframe = self._download_webpage(iframe_url, video_id,
-                                        'Downloading iframe page')
+        iframe = self._download_webpage(iframe_url, video_id, 'Downloading iframe page')
 
-        sd_video_url = self._search_regex(r'<source src="([^"]+)"',
-                                          iframe, 'sd video url')
-        resolution_id = sd_video_url.split("/")[-1] + 'p'
-        if len(resolution_id) != 4:
-            resolution_id = None
-        video_urls.append({
+        duration = None
+        sources = []
+
+        sd_url = self._search_regex(
+            r'<source[^>]+src=(["\'])(?P<url>.+?)\1', iframe,
+            'sd video url', default=None, group='url')
+        if sd_url:
+            sources.append((sd_url, 'sd'))
+
+        options = self._parse_json(
+            self._search_regex(
+                r'data-crt-options=(["\'])(?P<options>.+?)\1', iframe,
+                'hd video url', default='', group='options'),
+            video_id, fatal=False)
+        if options:
+            duration = int_or_none(options.get('duration'))
+            hd_url = options.get('hdUrl')
+            if hd_url:
+                sources.append((hd_url, 'hd'))
+
+        formats = [{
+            'url': video_url,
             'ext': 'mp4',
-            'format_id': 'sd',
-            'url': sd_video_url,
-            'resolution': resolution_id,
-        })
+            'format_id': format_id,
+            'height': int_or_none(self._search_regex(
+                r'/(\d{3,4})$', video_url, 'height', default=None)),
+            'quality': quality,
+        } for quality, (video_url, format_id) in enumerate(sources)]
 
-        hd_video_url = self._search_regex(r'hdUrl":"([^"]+)"', iframe,
-                                          'hd video url', default=None)
-        if hd_video_url:
-            hd_video_url = hd_video_url.replace("\\", "")
-            resolution_id = hd_video_url.split("/")[-1] + 'p'
-            if len(resolution_id) != 4:
-                resolution_id = None
-            video_urls.append({
-                'ext': 'mp4',
-                'format_id': 'hd',
-                'url': hd_video_url,
-                'resolution': resolution_id,
-            })
+        self._sort_formats(formats)
 
         # The only place where you can get a title, it's not complete,
         # but searching in other places doesn't work for all videos
@@ -131,9 +134,10 @@ class TumblrIE(InfoExtractor):
 
         return {
             'id': video_id,
-            'formats': video_urls,
             'ext': 'mp4',
             'title': video_title,
             'description': self._og_search_description(webpage, default=None),
             'thumbnail': self._og_search_thumbnail(webpage, default=None),
+            'duration': duration,
+            'formats': formats,
         }

From 5d84b79a3002925084690e17669738bf1ea711f7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 5 Oct 2015 21:53:59 +0600
Subject: [PATCH 2696/6219] [tumblr] Remove redundant test

---
 youtube_dl/extractor/tumblr.py | 13 -------------
 1 file changed, 13 deletions(-)

diff --git a/youtube_dl/extractor/tumblr.py b/youtube_dl/extractor/tumblr.py
index 449ba29fa..9f270318b 100644
--- a/youtube_dl/extractor/tumblr.py
+++ b/youtube_dl/extractor/tumblr.py
@@ -29,19 +29,6 @@ class TumblrIE(InfoExtractor):
             'description': 'md5:dba62ac8639482759c8eb10ce474586a',
             'thumbnail': 're:http://.*\.jpg',
         }
-    }, {
-        'url': 'http://hdvideotest.tumblr.com/post/130323439814/test-description-for-my-hd-video',
-        'md5': '99a84522f60972bf064a0b80f87bcbb5',
-        'info_dict': {
-            'id': '130323439814',
-            'ext': 'mp4',
-            'title': 'HD Video Testing \u2014 Test description for my HD video',
-            'description': 'md5:97cc3ab5fcd27ee4af6356701541319c',
-            'thumbnail': 're:http://.*\.jpg',
-        },
-        'params': {
-            'format': 'sd',
-        },
     }, {
         'url': 'http://hdvideotest.tumblr.com/post/130323439814/test-description-for-my-hd-video',
         'md5': '7ae503065ad150122dc3089f8cf1546c',

From 88c86d211bd2f542625d3b91ac4b15e62995345d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 5 Oct 2015 21:54:54 +0600
Subject: [PATCH 2697/6219] [tumblr] Add missing fields for vidme test

---
 youtube_dl/extractor/tumblr.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/tumblr.py b/youtube_dl/extractor/tumblr.py
index 9f270318b..cb91d08eb 100644
--- a/youtube_dl/extractor/tumblr.py
+++ b/youtube_dl/extractor/tumblr.py
@@ -51,6 +51,9 @@ class TumblrIE(InfoExtractor):
             'title': 'naked smoking & stretching',
             'upload_date': '20150506',
             'timestamp': 1430931613,
+            'age_limit': 18,
+            'uploader_id': '1638622',
+            'uploader': 'naked-yogi',
         },
         'add_ie': ['Vidme'],
     }, {

From 7fd4ed9939d7c467c61663cad7189ee8ad27c89b Mon Sep 17 00:00:00 2001
From: David Rabinowitz <drabinowitz@agtinternational.com>
Date: Thu, 1 Oct 2015 12:07:27 +0300
Subject: [PATCH 2698/6219] Fixed the ustream extractor to use the current
 ustream API

---
 youtube_dl/extractor/ustream.py | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/ustream.py b/youtube_dl/extractor/ustream.py
index c39c278ab..18add908e 100644
--- a/youtube_dl/extractor/ustream.py
+++ b/youtube_dl/extractor/ustream.py
@@ -52,17 +52,12 @@ class UstreamIE(InfoExtractor):
             desktop_url = 'http://www.ustream.tv/recorded/' + desktop_video_id
             return self.url_result(desktop_url, 'Ustream')
 
-        params = self._download_json(
-            'http://cdngw.ustream.tv/rgwjson/Viewer.getVideo/' + json.dumps({
-                'brandId': 1,
-                'videoId': int(video_id),
-                'autoplay': False,
-            }), video_id)
+        params = self._download_json('https://api.ustream.tv/videos/' + video_id + '.json', video_id)
 
         if 'error' in params:
             raise ExtractorError(params['error']['message'], expected=True)
 
-        video_url = params['flv']
+        video_url = params['video']['media_urls']['flv']
 
         webpage = self._download_webpage(url, video_id)
 

From 5820c4a29ea3781c47f6249c179a081e28eeeb64 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 5 Oct 2015 22:30:38 +0600
Subject: [PATCH 2699/6219] [ustream] Switch extraction to api

---
 youtube_dl/extractor/ustream.py | 69 ++++++++++++++++++---------------
 1 file changed, 37 insertions(+), 32 deletions(-)

diff --git a/youtube_dl/extractor/ustream.py b/youtube_dl/extractor/ustream.py
index 18add908e..3065c9f31 100644
--- a/youtube_dl/extractor/ustream.py
+++ b/youtube_dl/extractor/ustream.py
@@ -7,7 +7,11 @@ from .common import InfoExtractor
 from ..compat import (
     compat_urlparse,
 )
-from ..utils import ExtractorError
+from ..utils import (
+    ExtractorError,
+    int_or_none,
+    float_or_none,
+)
 
 
 class UstreamIE(InfoExtractor):
@@ -54,46 +58,47 @@ class UstreamIE(InfoExtractor):
 
         params = self._download_json('https://api.ustream.tv/videos/' + video_id + '.json', video_id)
 
-        if 'error' in params:
-            raise ExtractorError(params['error']['message'], expected=True)
+        error = params.get('error')
+        if error:
+            raise ExtractorError(
+                '%s returned error: %s' % (self.IE_NAME, error), expected=True)
 
-        video_url = params['video']['media_urls']['flv']
+        video = params['video']
 
-        webpage = self._download_webpage(url, video_id)
+        formats = [{
+            'id': format_id,
+            'url': video_url,
+            'ext': format_id,
+        } for format_id, video_url in video['media_urls'].items()]
+        self._sort_formats(formats)
 
-        self.report_extraction(video_id)
+        title = video['title']
+        description = video.get('description')
+        timestamp = int_or_none(video.get('created_at'))
+        duration = float_or_none(video.get('length'))
+        filesize = float_or_none(video.get('file_size'))
+        view_count = int_or_none(video.get('views'))
 
-        video_title = self._html_search_regex(r'data-title="(?P<title>.+)"',
-                                              webpage, 'title', default=None)
+        uploader = video.get('owner', {}).get('username')
+        uploader_id = video.get('owner', {}).get('id')
 
-        if not video_title:
-            try:
-                video_title = params['moduleConfig']['meta']['title']
-            except KeyError:
-                pass
-
-        if not video_title:
-            video_title = 'Ustream video ' + video_id
-
-        uploader = self._html_search_regex(r'data-content-type="channel".*?>(?P<uploader>.*?)</a>',
-                                           webpage, 'uploader', fatal=False, flags=re.DOTALL, default=None)
-
-        if not uploader:
-            try:
-                uploader = params['moduleConfig']['meta']['userName']
-            except KeyError:
-                uploader = None
-
-        thumbnail = self._html_search_regex(r'<link rel="image_src" href="(?P<thumb>.*?)"',
-                                            webpage, 'thumbnail', fatal=False)
+        thumbnails = [{
+            'id': thumbnail_id,
+            'url': thumbnail_url,
+        } for thumbnail_id, thumbnail_url in video.get('thumbnail', {}).items()]
 
         return {
             'id': video_id,
-            'url': video_url,
-            'ext': 'flv',
-            'title': video_title,
+            'title': title,
+            'description': description,
+            'thumbnails': thumbnails,
+            'timestamp': timestamp,
+            'duration': duration,
+            'filesize': filesize,
+            'view_count': view_count,
             'uploader': uploader,
-            'thumbnail': thumbnail,
+            'uploader_id': uploader_id,
+            'formats': formats,
         }
 
 

From 4853eb63fec766c7154e9353e91e531a3f3b1d1e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 5 Oct 2015 22:40:20 +0600
Subject: [PATCH 2700/6219] [ustream] Modernize

---
 youtube_dl/extractor/ustream.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/ustream.py b/youtube_dl/extractor/ustream.py
index 3065c9f31..9fdeb064d 100644
--- a/youtube_dl/extractor/ustream.py
+++ b/youtube_dl/extractor/ustream.py
@@ -15,7 +15,7 @@ from ..utils import (
 
 
 class UstreamIE(InfoExtractor):
-    _VALID_URL = r'https?://www\.ustream\.tv/(?P<type>recorded|embed|embed/recorded)/(?P<videoID>\d+)'
+    _VALID_URL = r'https?://www\.ustream\.tv/(?P<type>recorded|embed|embed/recorded)/(?P<id>\d+)'
     IE_NAME = 'ustream'
     _TESTS = [{
         'url': 'http://www.ustream.tv/recorded/20274954',
@@ -41,22 +41,23 @@ class UstreamIE(InfoExtractor):
 
     def _real_extract(self, url):
         m = re.match(self._VALID_URL, url)
-        video_id = m.group('videoID')
+        video_id = m.group('id')
 
         # some sites use this embed format (see: http://github.com/rg3/youtube-dl/issues/2990)
         if m.group('type') == 'embed/recorded':
-            video_id = m.group('videoID')
+            video_id = m.group('id')
             desktop_url = 'http://www.ustream.tv/recorded/' + video_id
             return self.url_result(desktop_url, 'Ustream')
         if m.group('type') == 'embed':
-            video_id = m.group('videoID')
+            video_id = m.group('id')
             webpage = self._download_webpage(url, video_id)
             desktop_video_id = self._html_search_regex(
                 r'ContentVideoIds=\["([^"]*?)"\]', webpage, 'desktop_video_id')
             desktop_url = 'http://www.ustream.tv/recorded/' + desktop_video_id
             return self.url_result(desktop_url, 'Ustream')
 
-        params = self._download_json('https://api.ustream.tv/videos/' + video_id + '.json', video_id)
+        params = self._download_json(
+            'https://api.ustream.tv/videos/%s.json' % video_id, video_id)
 
         error = params.get('error')
         if error:

From f2a7ed77ef76b8d564326f564afc880473fba7ea Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 5 Oct 2015 22:44:36 +0600
Subject: [PATCH 2701/6219] [tumblr] Remove redundant field

---
 youtube_dl/extractor/tumblr.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/tumblr.py b/youtube_dl/extractor/tumblr.py
index cb91d08eb..4f844706d 100644
--- a/youtube_dl/extractor/tumblr.py
+++ b/youtube_dl/extractor/tumblr.py
@@ -124,7 +124,6 @@ class TumblrIE(InfoExtractor):
 
         return {
             'id': video_id,
-            'ext': 'mp4',
             'title': video_title,
             'description': self._og_search_description(webpage, default=None),
             'thumbnail': self._og_search_thumbnail(webpage, default=None),

From 0bf219889e39d4d7e75fdb59d7452b8e09f4ddab Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 5 Oct 2015 22:44:59 +0600
Subject: [PATCH 2702/6219] [ustream] Remove unused import

---
 youtube_dl/extractor/ustream.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/ustream.py b/youtube_dl/extractor/ustream.py
index 9fdeb064d..7243d0eca 100644
--- a/youtube_dl/extractor/ustream.py
+++ b/youtube_dl/extractor/ustream.py
@@ -1,6 +1,5 @@
 from __future__ import unicode_literals
 
-import json
 import re
 
 from .common import InfoExtractor

From 41db73330835d6e8fe29ce18f869247a739467ce Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 5 Oct 2015 22:48:47 +0600
Subject: [PATCH 2703/6219] [ustream] Move filesize

---
 youtube_dl/extractor/ustream.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/ustream.py b/youtube_dl/extractor/ustream.py
index 7243d0eca..888f39f7a 100644
--- a/youtube_dl/extractor/ustream.py
+++ b/youtube_dl/extractor/ustream.py
@@ -65,18 +65,20 @@ class UstreamIE(InfoExtractor):
 
         video = params['video']
 
+        title = video['title']
+        filesize = float_or_none(video.get('file_size'))
+
         formats = [{
             'id': format_id,
             'url': video_url,
             'ext': format_id,
+            'filesize': filesize,
         } for format_id, video_url in video['media_urls'].items()]
         self._sort_formats(formats)
 
-        title = video['title']
         description = video.get('description')
         timestamp = int_or_none(video.get('created_at'))
         duration = float_or_none(video.get('length'))
-        filesize = float_or_none(video.get('file_size'))
         view_count = int_or_none(video.get('views'))
 
         uploader = video.get('owner', {}).get('username')
@@ -94,7 +96,6 @@ class UstreamIE(InfoExtractor):
             'thumbnails': thumbnails,
             'timestamp': timestamp,
             'duration': duration,
-            'filesize': filesize,
             'view_count': view_count,
             'uploader': uploader,
             'uploader_id': uploader_id,

From dc5756fd7729f616185508fba296b330f0892c24 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 5 Oct 2015 22:51:04 +0600
Subject: [PATCH 2704/6219] [ustream] Fix typo

---
 youtube_dl/extractor/ustream.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ustream.py b/youtube_dl/extractor/ustream.py
index 888f39f7a..a29d67e9f 100644
--- a/youtube_dl/extractor/ustream.py
+++ b/youtube_dl/extractor/ustream.py
@@ -69,7 +69,7 @@ class UstreamIE(InfoExtractor):
         filesize = float_or_none(video.get('file_size'))
 
         formats = [{
-            'id': format_id,
+            'id': video_id,
             'url': video_url,
             'ext': format_id,
             'filesize': filesize,

From 40fbb05e1c58625349160ec134343af789b803ff Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 5 Oct 2015 22:52:51 +0600
Subject: [PATCH 2705/6219] [ustream] Fix tests

---
 youtube_dl/extractor/ustream.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/ustream.py b/youtube_dl/extractor/ustream.py
index a29d67e9f..73b05ecab 100644
--- a/youtube_dl/extractor/ustream.py
+++ b/youtube_dl/extractor/ustream.py
@@ -22,8 +22,12 @@ class UstreamIE(InfoExtractor):
         'info_dict': {
             'id': '20274954',
             'ext': 'flv',
-            'uploader': 'Young Americans for Liberty',
             'title': 'Young Americans for Liberty February 7, 2012 2:28 AM',
+            'description': 'Young Americans for Liberty February 7, 2012 2:28 AM',
+            'timestamp': 1328577035,
+            'upload_date': '20120207',
+            'uploader': 'yaliberty',
+            'uploader_id': '6780869',
         },
     }, {
         # From http://sportscanada.tv/canadagames/index.php/week2/figure-skating/444
@@ -35,7 +39,8 @@ class UstreamIE(InfoExtractor):
             'ext': 'flv',
             'title': '-CG11- Canada Games Figure Skating',
             'uploader': 'sportscanadatv',
-        }
+        },
+        'skip': 'This Pro Broadcaster has chosen to remove this video from the ustream.tv site.',
     }]
 
     def _real_extract(self, url):

From fcc25462693fb0468d66b28664ba622f8f3bdb3d Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Tue, 6 Oct 2015 02:30:05 +0600
Subject: [PATCH 2706/6219] [README.md] Markdown improvements

---
 README.md | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/README.md b/README.md
index 80152071d..e7004c9fc 100644
--- a/README.md
+++ b/README.md
@@ -359,7 +359,7 @@ If you have installed youtube-dl with a package manager, pip, setup.py or a tarb
 
 By default, youtube-dl intends to have the best options (incidentally, if you have a convincing case that these should be different, [please file an issue where you explain that](https://yt-dl.org/bug)). Therefore, it is unnecessary and sometimes harmful to copy long option strings from webpages. In particular, the only option out of `-citw` that is regularly useful is `-i`.
 
-### Can you please put the -b option back?
+### Can you please put the `-b` option back?
 
 Most people asking this question are not aware that youtube-dl now defaults to downloading the highest available quality as reported by YouTube, which will be 1080p or 720p in some cases, so you no longer need the `-b` option. For some specific videos, maybe YouTube does not report them to be available in a specific high quality format you're interested in. In that case, simply request it with the `-f` option and youtube-dl will try to download it.
 
@@ -371,13 +371,13 @@ Apparently YouTube requires you to pass a CAPTCHA test if you download too much.
 
 Once the video is fully downloaded, use any video player, such as [vlc](http://www.videolan.org) or [mplayer](http://www.mplayerhq.hu/).
 
-### I extracted a video URL with -g, but it does not play on another machine / in my webbrowser.
+### I extracted a video URL with `-g`, but it does not play on another machine / in my webbrowser.
 
 It depends a lot on the service. In many cases, requests for the video (to download/play it) must come from the same IP address and with the same cookies.  Use the `--cookies` option to write the required cookies into a file, and advise your downloader to read cookies from that file. Some sites also require a common user agent to be used, use `--dump-user-agent` to see the one in use by youtube-dl.
 
 It may be beneficial to use IPv6; in some cases, the restrictions are only applied to IPv4. Some services (sometimes only for a subset of videos) do not restrict the video URL by IP address, cookie, or user-agent, but these are the exception rather than the rule.
 
-Please bear in mind that some URL protocols are **not** supported by browsers out of the box, including RTMP. If you are using -g, your own downloader must support these as well.
+Please bear in mind that some URL protocols are **not** supported by browsers out of the box, including RTMP. If you are using `-g`, your own downloader must support these as well.
 
 If you want to play the video on a machine that is not running youtube-dl, you can relay the video content from the machine that runs youtube-dl. You can use `-o -` to let youtube-dl stream a video to stdout, or simply allow the player to download the files written by youtube-dl in turn.
 
@@ -643,15 +643,15 @@ So please elaborate on what feature you are requesting, or what bug you want to
 
 If your report is shorter than two lines, it is almost certainly missing some of these, which makes it hard for us to respond to it. We're often too polite to close the issue outright, but the missing info makes misinterpretation likely. As a commiter myself, I often get frustrated by these issues, since the only possible way for me to move forward on them is to ask for clarification over and over.
 
-For bug reports, this means that your report should contain the *complete* output of youtube-dl when called with the -v flag. The error message you get for (most) bugs even says so, but you would not believe how many of our bug reports do not contain this information.
+For bug reports, this means that your report should contain the *complete* output of youtube-dl when called with the `-v` flag. The error message you get for (most) bugs even says so, but you would not believe how many of our bug reports do not contain this information.
 
-If your server has multiple IPs or you suspect censorship, adding --call-home may be a good idea to get more diagnostics. If the error is `ERROR: Unable to extract ...` and you cannot reproduce it from multiple countries, add `--dump-pages` (warning: this will yield a rather large output, redirect it to the file `log.txt` by adding `>log.txt 2>&1` to your command-line) or upload the `.dump` files you get when you add `--write-pages` [somewhere](https://gist.github.com/).
+If your server has multiple IPs or you suspect censorship, adding `--call-home` may be a good idea to get more diagnostics. If the error is `ERROR: Unable to extract ...` and you cannot reproduce it from multiple countries, add `--dump-pages` (warning: this will yield a rather large output, redirect it to the file `log.txt` by adding `>log.txt 2>&1` to your command-line) or upload the `.dump` files you get when you add `--write-pages` [somewhere](https://gist.github.com/).
 
 **Site support requests must contain an example URL**. An example URL is a URL you might want to download, like http://www.youtube.com/watch?v=BaW_jenozKc . There should be an obvious video present. Except under very special circumstances, the main page of a video service (e.g. http://www.youtube.com/ ) is *not* an example URL.
 
 ###  Are you using the latest version?
 
-Before reporting any issue, type youtube-dl -U. This should report that you're up-to-date. About 20% of the reports we receive are already fixed, but people are using outdated versions. This goes for feature requests as well.
+Before reporting any issue, type `youtube-dl -U`. This should report that you're up-to-date. About 20% of the reports we receive are already fixed, but people are using outdated versions. This goes for feature requests as well.
 
 ###  Is the issue already documented?
 

From c4af7684d85a17441c5f6f0b6e9f8c470644fec0 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 6 Oct 2015 09:08:10 +0200
Subject: [PATCH 2707/6219] release 2015.10.06

---
 CONTRIBUTING.md        |   6 +-
 README.md              | 395 +++++++++++++++++++++++++++++------------
 docs/supportedsites.md |   4 +
 youtube_dl/version.py  |   2 +-
 4 files changed, 289 insertions(+), 118 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index f8ab29631..57a94231d 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -16,15 +16,15 @@ So please elaborate on what feature you are requesting, or what bug you want to
 
 If your report is shorter than two lines, it is almost certainly missing some of these, which makes it hard for us to respond to it. We're often too polite to close the issue outright, but the missing info makes misinterpretation likely. As a commiter myself, I often get frustrated by these issues, since the only possible way for me to move forward on them is to ask for clarification over and over.
 
-For bug reports, this means that your report should contain the *complete* output of youtube-dl when called with the -v flag. The error message you get for (most) bugs even says so, but you would not believe how many of our bug reports do not contain this information.
+For bug reports, this means that your report should contain the *complete* output of youtube-dl when called with the `-v` flag. The error message you get for (most) bugs even says so, but you would not believe how many of our bug reports do not contain this information.
 
-If your server has multiple IPs or you suspect censorship, adding --call-home may be a good idea to get more diagnostics. If the error is `ERROR: Unable to extract ...` and you cannot reproduce it from multiple countries, add `--dump-pages` (warning: this will yield a rather large output, redirect it to the file `log.txt` by adding `>log.txt 2>&1` to your command-line) or upload the `.dump` files you get when you add `--write-pages` [somewhere](https://gist.github.com/).
+If your server has multiple IPs or you suspect censorship, adding `--call-home` may be a good idea to get more diagnostics. If the error is `ERROR: Unable to extract ...` and you cannot reproduce it from multiple countries, add `--dump-pages` (warning: this will yield a rather large output, redirect it to the file `log.txt` by adding `>log.txt 2>&1` to your command-line) or upload the `.dump` files you get when you add `--write-pages` [somewhere](https://gist.github.com/).
 
 **Site support requests must contain an example URL**. An example URL is a URL you might want to download, like http://www.youtube.com/watch?v=BaW_jenozKc . There should be an obvious video present. Except under very special circumstances, the main page of a video service (e.g. http://www.youtube.com/ ) is *not* an example URL.
 
 ###  Are you using the latest version?
 
-Before reporting any issue, type youtube-dl -U. This should report that you're up-to-date. About 20% of the reports we receive are already fixed, but people are using outdated versions. This goes for feature requests as well.
+Before reporting any issue, type `youtube-dl -U`. This should report that you're up-to-date. About 20% of the reports we receive are already fixed, but people are using outdated versions. This goes for feature requests as well.
 
 ###  Is the issue already documented?
 
diff --git a/README.md b/README.md
index e7004c9fc..5ff8ca85e 100644
--- a/README.md
+++ b/README.md
@@ -49,110 +49,220 @@ which means you can modify it, redistribute it or use it however you like.
 # OPTIONS
     -h, --help                       Print this help text and exit
     --version                        Print program version and exit
-    -U, --update                     Update this program to latest version. Make sure that you have sufficient permissions (run with sudo if needed)
-    -i, --ignore-errors              Continue on download errors, for example to skip unavailable videos in a playlist
-    --abort-on-error                 Abort downloading of further videos (in the playlist or the command line) if an error occurs
+    -U, --update                     Update this program to latest version. Make
+                                     sure that you have sufficient permissions
+                                     (run with sudo if needed)
+    -i, --ignore-errors              Continue on download errors, for example to
+                                     skip unavailable videos in a playlist
+    --abort-on-error                 Abort downloading of further videos (in the
+                                     playlist or the command line) if an error
+                                     occurs
     --dump-user-agent                Display the current browser identification
     --list-extractors                List all supported extractors
-    --extractor-descriptions         Output descriptions of all supported extractors
-    --force-generic-extractor        Force extraction to use the generic extractor
-    --default-search PREFIX          Use this prefix for unqualified URLs. For example "gvsearch2:" downloads two videos from google videos for youtube-dl "large apple".
-                                     Use the value "auto" to let youtube-dl guess ("auto_warning" to emit a warning when guessing). "error" just throws an error. The
-                                     default value "fixup_error" repairs broken URLs, but emits an error if this is not possible instead of searching.
-    --ignore-config                  Do not read configuration files. When given in the global configuration file /etc/youtube-dl.conf: Do not read the user configuration
-                                     in ~/.config/youtube-dl/config (%APPDATA%/youtube-dl/config.txt on Windows)
-    --flat-playlist                  Do not extract the videos of a playlist, only list them.
+    --extractor-descriptions         Output descriptions of all supported
+                                     extractors
+    --force-generic-extractor        Force extraction to use the generic
+                                     extractor
+    --default-search PREFIX          Use this prefix for unqualified URLs. For
+                                     example "gvsearch2:" downloads two videos
+                                     from google videos for youtube-dl "large
+                                     apple". Use the value "auto" to let
+                                     youtube-dl guess ("auto_warning" to emit a
+                                     warning when guessing). "error" just throws
+                                     an error. The default value "fixup_error"
+                                     repairs broken URLs, but emits an error if
+                                     this is not possible instead of searching.
+    --ignore-config                  Do not read configuration files. When given
+                                     in the global configuration file /etc
+                                     /youtube-dl.conf: Do not read the user
+                                     configuration in ~/.config/youtube-
+                                     dl/config (%APPDATA%/youtube-dl/config.txt
+                                     on Windows)
+    --flat-playlist                  Do not extract the videos of a playlist,
+                                     only list them.
     --no-color                       Do not emit color codes in output
 
 ## Network Options:
-    --proxy URL                      Use the specified HTTP/HTTPS proxy. Pass in an empty string (--proxy "") for direct connection
+    --proxy URL                      Use the specified HTTP/HTTPS proxy. Pass in
+                                     an empty string (--proxy "") for direct
+                                     connection
     --socket-timeout SECONDS         Time to wait before giving up, in seconds
-    --source-address IP              Client-side IP address to bind to (experimental)
-    -4, --force-ipv4                 Make all connections via IPv4 (experimental)
-    -6, --force-ipv6                 Make all connections via IPv6 (experimental)
-    --cn-verification-proxy URL      Use this proxy to verify the IP address for some Chinese sites. The default proxy specified by --proxy (or none, if the options is
-                                     not present) is used for the actual downloading. (experimental)
+    --source-address IP              Client-side IP address to bind to
+                                     (experimental)
+    -4, --force-ipv4                 Make all connections via IPv4
+                                     (experimental)
+    -6, --force-ipv6                 Make all connections via IPv6
+                                     (experimental)
+    --cn-verification-proxy URL      Use this proxy to verify the IP address for
+                                     some Chinese sites. The default proxy
+                                     specified by --proxy (or none, if the
+                                     options is not present) is used for the
+                                     actual downloading. (experimental)
 
 ## Video Selection:
     --playlist-start NUMBER          Playlist video to start at (default is 1)
     --playlist-end NUMBER            Playlist video to end at (default is last)
-    --playlist-items ITEM_SPEC       Playlist video items to download. Specify indices of the videos in the playlist separated by commas like: "--playlist-items 1,2,5,8"
-                                     if you want to download videos indexed 1, 2, 5, 8 in the playlist. You can specify range: "--playlist-items 1-3,7,10-13", it will
-                                     download the videos at index 1, 2, 3, 7, 10, 11, 12 and 13.
-    --match-title REGEX              Download only matching titles (regex or caseless sub-string)
-    --reject-title REGEX             Skip download for matching titles (regex or caseless sub-string)
+    --playlist-items ITEM_SPEC       Playlist video items to download. Specify
+                                     indices of the videos in the playlist
+                                     separated by commas like: "--playlist-items
+                                     1,2,5,8" if you want to download videos
+                                     indexed 1, 2, 5, 8 in the playlist. You can
+                                     specify range: "--playlist-items
+                                     1-3,7,10-13", it will download the videos
+                                     at index 1, 2, 3, 7, 10, 11, 12 and 13.
+    --match-title REGEX              Download only matching titles (regex or
+                                     caseless sub-string)
+    --reject-title REGEX             Skip download for matching titles (regex or
+                                     caseless sub-string)
     --max-downloads NUMBER           Abort after downloading NUMBER files
-    --min-filesize SIZE              Do not download any videos smaller than SIZE (e.g. 50k or 44.6m)
-    --max-filesize SIZE              Do not download any videos larger than SIZE (e.g. 50k or 44.6m)
+    --min-filesize SIZE              Do not download any videos smaller than
+                                     SIZE (e.g. 50k or 44.6m)
+    --max-filesize SIZE              Do not download any videos larger than SIZE
+                                     (e.g. 50k or 44.6m)
     --date DATE                      Download only videos uploaded in this date
-    --datebefore DATE                Download only videos uploaded on or before this date (i.e. inclusive)
-    --dateafter DATE                 Download only videos uploaded on or after this date (i.e. inclusive)
-    --min-views COUNT                Do not download any videos with less than COUNT views
-    --max-views COUNT                Do not download any videos with more than COUNT views
-    --match-filter FILTER            Generic video filter (experimental). Specify any key (see help for -o for a list of available keys) to match if the key is present,
-                                     !key to check if the key is not present,key > NUMBER (like "comment_count > 12", also works with >=, <, <=, !=, =) to compare against
-                                     a number, and & to require multiple matches. Values which are not known are excluded unless you put a question mark (?) after the
-                                     operator.For example, to only match videos that have been liked more than 100 times and disliked less than 50 times (or the dislike
-                                     functionality is not available at the given service), but who also have a description, use  --match-filter "like_count > 100 &
+    --datebefore DATE                Download only videos uploaded on or before
+                                     this date (i.e. inclusive)
+    --dateafter DATE                 Download only videos uploaded on or after
+                                     this date (i.e. inclusive)
+    --min-views COUNT                Do not download any videos with less than
+                                     COUNT views
+    --max-views COUNT                Do not download any videos with more than
+                                     COUNT views
+    --match-filter FILTER            Generic video filter (experimental).
+                                     Specify any key (see help for -o for a list
+                                     of available keys) to match if the key is
+                                     present, !key to check if the key is not
+                                     present,key > NUMBER (like "comment_count >
+                                     12", also works with >=, <, <=, !=, =) to
+                                     compare against a number, and & to require
+                                     multiple matches. Values which are not
+                                     known are excluded unless you put a
+                                     question mark (?) after the operator.For
+                                     example, to only match videos that have
+                                     been liked more than 100 times and disliked
+                                     less than 50 times (or the dislike
+                                     functionality is not available at the given
+                                     service), but who also have a description,
+                                     use --match-filter "like_count > 100 &
                                      dislike_count <? 50 & description" .
-    --no-playlist                    Download only the video, if the URL refers to a video and a playlist.
-    --yes-playlist                   Download the playlist, if the URL refers to a video and a playlist.
-    --age-limit YEARS                Download only videos suitable for the given age
-    --download-archive FILE          Download only videos not listed in the archive file. Record the IDs of all downloaded videos in it.
-    --include-ads                    Download advertisements as well (experimental)
+    --no-playlist                    Download only the video, if the URL refers
+                                     to a video and a playlist.
+    --yes-playlist                   Download the playlist, if the URL refers to
+                                     a video and a playlist.
+    --age-limit YEARS                Download only videos suitable for the given
+                                     age
+    --download-archive FILE          Download only videos not listed in the
+                                     archive file. Record the IDs of all
+                                     downloaded videos in it.
+    --include-ads                    Download advertisements as well
+                                     (experimental)
 
 ## Download Options:
-    -r, --rate-limit LIMIT           Maximum download rate in bytes per second (e.g. 50K or 4.2M)
-    -R, --retries RETRIES            Number of retries (default is 10), or "infinite".
-    --buffer-size SIZE               Size of download buffer (e.g. 1024 or 16K) (default is 1024)
-    --no-resize-buffer               Do not automatically adjust the buffer size. By default, the buffer size is automatically resized from an initial value of SIZE.
+    -r, --rate-limit LIMIT           Maximum download rate in bytes per second
+                                     (e.g. 50K or 4.2M)
+    -R, --retries RETRIES            Number of retries (default is 10), or
+                                     "infinite".
+    --buffer-size SIZE               Size of download buffer (e.g. 1024 or 16K)
+                                     (default is 1024)
+    --no-resize-buffer               Do not automatically adjust the buffer
+                                     size. By default, the buffer size is
+                                     automatically resized from an initial value
+                                     of SIZE.
     --playlist-reverse               Download playlist videos in reverse order
-    --xattr-set-filesize             Set file xattribute ytdl.filesize with expected filesize (experimental)
-    --hls-prefer-native              Use the native HLS downloader instead of ffmpeg (experimental)
-    --external-downloader COMMAND    Use the specified external downloader. Currently supports aria2c,axel,curl,httpie,wget
-    --external-downloader-args ARGS  Give these arguments to the external downloader
+    --xattr-set-filesize             Set file xattribute ytdl.filesize with
+                                     expected filesize (experimental)
+    --hls-prefer-native              Use the native HLS downloader instead of
+                                     ffmpeg (experimental)
+    --external-downloader COMMAND    Use the specified external downloader.
+                                     Currently supports
+                                     aria2c,axel,curl,httpie,wget
+    --external-downloader-args ARGS  Give these arguments to the external
+                                     downloader
 
 ## Filesystem Options:
-    -a, --batch-file FILE            File containing URLs to download ('-' for stdin)
+    -a, --batch-file FILE            File containing URLs to download ('-' for
+                                     stdin)
     --id                             Use only video ID in file name
-    -o, --output TEMPLATE            Output filename template. Use %(title)s to get the title, %(uploader)s for the uploader name, %(uploader_id)s for the uploader
-                                     nickname if different, %(autonumber)s to get an automatically incremented number, %(ext)s for the filename extension, %(format)s for
-                                     the format description (like "22 - 1280x720" or "HD"), %(format_id)s for the unique id of the format (like YouTube's itags: "137"),
-                                     %(upload_date)s for the upload date (YYYYMMDD), %(extractor)s for the provider (youtube, metacafe, etc), %(id)s for the video id,
-                                     %(playlist_title)s, %(playlist_id)s, or %(playlist)s (=title if present, ID otherwise) for the playlist the video is in,
-                                     %(playlist_index)s for the position in the playlist. %(height)s and %(width)s for the width and height of the video format.
-                                     %(resolution)s for a textual description of the resolution of the video format. %% for a literal percent. Use - to output to stdout.
-                                     Can also be used to download to a different directory, for example with -o '/my/downloads/%(uploader)s/%(title)s-%(id)s.%(ext)s' .
-    --autonumber-size NUMBER         Specify the number of digits in %(autonumber)s when it is present in output filename template or --auto-number option is given
-    --restrict-filenames             Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames
-    -A, --auto-number                [deprecated; use  -o "%(autonumber)s-%(title)s.%(ext)s" ] Number downloaded files starting from 00000
-    -t, --title                      [deprecated] Use title in file name (default)
+    -o, --output TEMPLATE            Output filename template. Use %(title)s to
+                                     get the title, %(uploader)s for the
+                                     uploader name, %(uploader_id)s for the
+                                     uploader nickname if different,
+                                     %(autonumber)s to get an automatically
+                                     incremented number, %(ext)s for the
+                                     filename extension, %(format)s for the
+                                     format description (like "22 - 1280x720" or
+                                     "HD"), %(format_id)s for the unique id of
+                                     the format (like YouTube's itags: "137"),
+                                     %(upload_date)s for the upload date
+                                     (YYYYMMDD), %(extractor)s for the provider
+                                     (youtube, metacafe, etc), %(id)s for the
+                                     video id, %(playlist_title)s,
+                                     %(playlist_id)s, or %(playlist)s (=title if
+                                     present, ID otherwise) for the playlist the
+                                     video is in, %(playlist_index)s for the
+                                     position in the playlist. %(height)s and
+                                     %(width)s for the width and height of the
+                                     video format. %(resolution)s for a textual
+                                     description of the resolution of the video
+                                     format. %% for a literal percent. Use - to
+                                     output to stdout. Can also be used to
+                                     download to a different directory, for
+                                     example with -o '/my/downloads/%(uploader)s
+                                     /%(title)s-%(id)s.%(ext)s' .
+    --autonumber-size NUMBER         Specify the number of digits in
+                                     %(autonumber)s when it is present in output
+                                     filename template or --auto-number option
+                                     is given
+    --restrict-filenames             Restrict filenames to only ASCII
+                                     characters, and avoid "&" and spaces in
+                                     filenames
+    -A, --auto-number                [deprecated; use -o
+                                     "%(autonumber)s-%(title)s.%(ext)s" ] Number
+                                     downloaded files starting from 00000
+    -t, --title                      [deprecated] Use title in file name
+                                     (default)
     -l, --literal                    [deprecated] Alias of --title
     -w, --no-overwrites              Do not overwrite files
-    -c, --continue                   Force resume of partially downloaded files. By default, youtube-dl will resume downloads if possible.
-    --no-continue                    Do not resume partially downloaded files (restart from beginning)
-    --no-part                        Do not use .part files - write directly into output file
-    --no-mtime                       Do not use the Last-modified header to set the file modification time
-    --write-description              Write video description to a .description file
+    -c, --continue                   Force resume of partially downloaded files.
+                                     By default, youtube-dl will resume
+                                     downloads if possible.
+    --no-continue                    Do not resume partially downloaded files
+                                     (restart from beginning)
+    --no-part                        Do not use .part files - write directly
+                                     into output file
+    --no-mtime                       Do not use the Last-modified header to set
+                                     the file modification time
+    --write-description              Write video description to a .description
+                                     file
     --write-info-json                Write video metadata to a .info.json file
-    --write-annotations              Write video annotations to a .annotations.xml file
-    --load-info FILE                 JSON file containing the video information (created with the "--write-info-json" option)
-    --cookies FILE                   File to read cookies from and dump cookie jar in
-    --cache-dir DIR                  Location in the filesystem where youtube-dl can store some downloaded information permanently. By default $XDG_CACHE_HOME/youtube-dl
-                                     or ~/.cache/youtube-dl . At the moment, only YouTube player files (for videos with obfuscated signatures) are cached, but that may
-                                     change.
+    --write-annotations              Write video annotations to a
+                                     .annotations.xml file
+    --load-info FILE                 JSON file containing the video information
+                                     (created with the "--write-info-json"
+                                     option)
+    --cookies FILE                   File to read cookies from and dump cookie
+                                     jar in
+    --cache-dir DIR                  Location in the filesystem where youtube-dl
+                                     can store some downloaded information
+                                     permanently. By default $XDG_CACHE_HOME
+                                     /youtube-dl or ~/.cache/youtube-dl . At the
+                                     moment, only YouTube player files (for
+                                     videos with obfuscated signatures) are
+                                     cached, but that may change.
     --no-cache-dir                   Disable filesystem caching
     --rm-cache-dir                   Delete all filesystem cache files
 
 ## Thumbnail images:
     --write-thumbnail                Write thumbnail image to disk
     --write-all-thumbnails           Write all thumbnail image formats to disk
-    --list-thumbnails                Simulate and list all available thumbnail formats
+    --list-thumbnails                Simulate and list all available thumbnail
+                                     formats
 
 ## Verbosity / Simulation Options:
     -q, --quiet                      Activate quiet mode
     --no-warnings                    Ignore warnings
-    -s, --simulate                   Do not download the video and do not write anything to disk
+    -s, --simulate                   Do not download the video and do not write
+                                     anything to disk
     --skip-download                  Do not download the video
     -g, --get-url                    Simulate, quiet but print URL
     -e, --get-title                  Simulate, quiet but print title
@@ -162,78 +272,135 @@ which means you can modify it, redistribute it or use it however you like.
     --get-duration                   Simulate, quiet but print video length
     --get-filename                   Simulate, quiet but print output filename
     --get-format                     Simulate, quiet but print output format
-    -j, --dump-json                  Simulate, quiet but print JSON information. See --output for a description of available keys.
-    -J, --dump-single-json           Simulate, quiet but print JSON information for each command-line argument. If the URL refers to a playlist, dump the whole playlist
-                                     information in a single line.
-    --print-json                     Be quiet and print the video information as JSON (video is still being downloaded).
+    -j, --dump-json                  Simulate, quiet but print JSON information.
+                                     See --output for a description of available
+                                     keys.
+    -J, --dump-single-json           Simulate, quiet but print JSON information
+                                     for each command-line argument. If the URL
+                                     refers to a playlist, dump the whole
+                                     playlist information in a single line.
+    --print-json                     Be quiet and print the video information as
+                                     JSON (video is still being downloaded).
     --newline                        Output progress bar as new lines
     --no-progress                    Do not print progress bar
     --console-title                  Display progress in console titlebar
     -v, --verbose                    Print various debugging information
-    --dump-pages                     Print downloaded pages encoded using base64 to debug problems (very verbose)
-    --write-pages                    Write downloaded intermediary pages to files in the current directory to debug problems
+    --dump-pages                     Print downloaded pages encoded using base64
+                                     to debug problems (very verbose)
+    --write-pages                    Write downloaded intermediary pages to
+                                     files in the current directory to debug
+                                     problems
     --print-traffic                  Display sent and read HTTP traffic
     -C, --call-home                  Contact the youtube-dl server for debugging
-    --no-call-home                   Do NOT contact the youtube-dl server for debugging
+    --no-call-home                   Do NOT contact the youtube-dl server for
+                                     debugging
 
 ## Workarounds:
     --encoding ENCODING              Force the specified encoding (experimental)
     --no-check-certificate           Suppress HTTPS certificate validation
-    --prefer-insecure                Use an unencrypted connection to retrieve information about the video. (Currently supported only for YouTube)
+    --prefer-insecure                Use an unencrypted connection to retrieve
+                                     information about the video. (Currently
+                                     supported only for YouTube)
     --user-agent UA                  Specify a custom user agent
-    --referer URL                    Specify a custom referer, use if the video access is restricted to one domain
-    --add-header FIELD:VALUE         Specify a custom HTTP header and its value, separated by a colon ':'. You can use this option multiple times
-    --bidi-workaround                Work around terminals that lack bidirectional text support. Requires bidiv or fribidi executable in PATH
-    --sleep-interval SECONDS         Number of seconds to sleep before each download.
+    --referer URL                    Specify a custom referer, use if the video
+                                     access is restricted to one domain
+    --add-header FIELD:VALUE         Specify a custom HTTP header and its value,
+                                     separated by a colon ':'. You can use this
+                                     option multiple times
+    --bidi-workaround                Work around terminals that lack
+                                     bidirectional text support. Requires bidiv
+                                     or fribidi executable in PATH
+    --sleep-interval SECONDS         Number of seconds to sleep before each
+                                     download.
 
 ## Video Format Options:
-    -f, --format FORMAT              Video format code, see the "FORMAT SELECTION" for all the info
+    -f, --format FORMAT              Video format code, see the "FORMAT
+                                     SELECTION" for all the info
     --all-formats                    Download all available video formats
-    --prefer-free-formats            Prefer free video formats unless a specific one is requested
+    --prefer-free-formats            Prefer free video formats unless a specific
+                                     one is requested
     -F, --list-formats               List all available formats
-    --youtube-skip-dash-manifest     Do not download the DASH manifests and related data on YouTube videos
-    --merge-output-format FORMAT     If a merge is required (e.g. bestvideo+bestaudio), output to given container format. One of mkv, mp4, ogg, webm, flv. Ignored if no
-                                     merge is required
+    --youtube-skip-dash-manifest     Do not download the DASH manifests and
+                                     related data on YouTube videos
+    --merge-output-format FORMAT     If a merge is required (e.g.
+                                     bestvideo+bestaudio), output to given
+                                     container format. One of mkv, mp4, ogg,
+                                     webm, flv. Ignored if no merge is required
 
 ## Subtitle Options:
     --write-sub                      Write subtitle file
-    --write-auto-sub                 Write automatic subtitle file (YouTube only)
-    --all-subs                       Download all the available subtitles of the video
+    --write-auto-sub                 Write automatic subtitle file (YouTube
+                                     only)
+    --all-subs                       Download all the available subtitles of the
+                                     video
     --list-subs                      List all available subtitles for the video
-    --sub-format FORMAT              Subtitle format, accepts formats preference, for example: "srt" or "ass/srt/best"
-    --sub-lang LANGS                 Languages of the subtitles to download (optional) separated by commas, use IETF language tags like 'en,pt'
+    --sub-format FORMAT              Subtitle format, accepts formats
+                                     preference, for example: "srt" or
+                                     "ass/srt/best"
+    --sub-lang LANGS                 Languages of the subtitles to download
+                                     (optional) separated by commas, use IETF
+                                     language tags like 'en,pt'
 
 ## Authentication Options:
     -u, --username USERNAME          Login with this account ID
-    -p, --password PASSWORD          Account password. If this option is left out, youtube-dl will ask interactively.
+    -p, --password PASSWORD          Account password. If this option is left
+                                     out, youtube-dl will ask interactively.
     -2, --twofactor TWOFACTOR        Two-factor auth code
     -n, --netrc                      Use .netrc authentication data
     --video-password PASSWORD        Video password (vimeo, smotri, youku)
 
 ## Post-processing Options:
-    -x, --extract-audio              Convert video files to audio-only files (requires ffmpeg or avconv and ffprobe or avprobe)
-    --audio-format FORMAT            Specify audio format: "best", "aac", "vorbis", "mp3", "m4a", "opus", or "wav"; "best" by default
-    --audio-quality QUALITY          Specify ffmpeg/avconv audio quality, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K (default
-                                     5)
-    --recode-video FORMAT            Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm|mkv|avi)
+    -x, --extract-audio              Convert video files to audio-only files
+                                     (requires ffmpeg or avconv and ffprobe or
+                                     avprobe)
+    --audio-format FORMAT            Specify audio format: "best", "aac",
+                                     "vorbis", "mp3", "m4a", "opus", or "wav";
+                                     "best" by default
+    --audio-quality QUALITY          Specify ffmpeg/avconv audio quality, insert
+                                     a value between 0 (better) and 9 (worse)
+                                     for VBR or a specific bitrate like 128K
+                                     (default 5)
+    --recode-video FORMAT            Encode the video to another format if
+                                     necessary (currently supported:
+                                     mp4|flv|ogg|webm|mkv|avi)
     --postprocessor-args ARGS        Give these arguments to the postprocessor
-    -k, --keep-video                 Keep the video file on disk after the post-processing; the video is erased by default
-    --no-post-overwrites             Do not overwrite post-processed files; the post-processed files are overwritten by default
-    --embed-subs                     Embed subtitles in the video (only for mkv and mp4 videos)
+    -k, --keep-video                 Keep the video file on disk after the post-
+                                     processing; the video is erased by default
+    --no-post-overwrites             Do not overwrite post-processed files; the
+                                     post-processed files are overwritten by
+                                     default
+    --embed-subs                     Embed subtitles in the video (only for mkv
+                                     and mp4 videos)
     --embed-thumbnail                Embed thumbnail in the audio as cover art
     --add-metadata                   Write metadata to the video file
-    --metadata-from-title FORMAT     Parse additional metadata like song title / artist from the video title. The format syntax is the same as --output, the parsed
-                                     parameters replace existing values. Additional templates: %(album)s, %(artist)s. Example: --metadata-from-title "%(artist)s -
-                                     %(title)s" matches a title like "Coldplay - Paradise"
-    --xattrs                         Write metadata to the video file's xattrs (using dublin core and xdg standards)
-    --fixup POLICY                   Automatically correct known faults of the file. One of never (do nothing), warn (only emit a warning), detect_or_warn (the default;
-                                     fix file if we can, warn otherwise)
-    --prefer-avconv                  Prefer avconv over ffmpeg for running the postprocessors (default)
-    --prefer-ffmpeg                  Prefer ffmpeg over avconv for running the postprocessors
-    --ffmpeg-location PATH           Location of the ffmpeg/avconv binary; either the path to the binary or its containing directory.
-    --exec CMD                       Execute a command on the file after downloading, similar to find's -exec syntax. Example: --exec 'adb push {} /sdcard/Music/ && rm
-                                     {}'
-    --convert-subtitles FORMAT       Convert the subtitles to other format (currently supported: srt|ass|vtt)
+    --metadata-from-title FORMAT     Parse additional metadata like song title /
+                                     artist from the video title. The format
+                                     syntax is the same as --output, the parsed
+                                     parameters replace existing values.
+                                     Additional templates: %(album)s,
+                                     %(artist)s. Example: --metadata-from-title
+                                     "%(artist)s - %(title)s" matches a title
+                                     like "Coldplay - Paradise"
+    --xattrs                         Write metadata to the video file's xattrs
+                                     (using dublin core and xdg standards)
+    --fixup POLICY                   Automatically correct known faults of the
+                                     file. One of never (do nothing), warn (only
+                                     emit a warning), detect_or_warn (the
+                                     default; fix file if we can, warn
+                                     otherwise)
+    --prefer-avconv                  Prefer avconv over ffmpeg for running the
+                                     postprocessors (default)
+    --prefer-ffmpeg                  Prefer ffmpeg over avconv for running the
+                                     postprocessors
+    --ffmpeg-location PATH           Location of the ffmpeg/avconv binary;
+                                     either the path to the binary or its
+                                     containing directory.
+    --exec CMD                       Execute a command on the file after
+                                     downloading, similar to find's -exec
+                                     syntax. Example: --exec 'adb push {}
+                                     /sdcard/Music/ && rm {}'
+    --convert-subtitles FORMAT       Convert the subtitles to other format
+                                     (currently supported: srt|ass|vtt)
 
 # CONFIGURATION
 
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index fa83b68ad..5beb2ecd4 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -150,6 +150,7 @@
  - **Escapist**
  - **ESPN** (Currently broken)
  - **EsriVideo**
+ - **Europa**
  - **EveryonesMixtape**
  - **exfm**: ex.fm
  - **ExpoTV**
@@ -263,6 +264,9 @@
  - **Libsyn**
  - **life:embed**
  - **lifenews**: LIFE | NEWS
+ - **limelight**
+ - **limelight:channel**
+ - **limelight:channel_list**
  - **LiveLeak**
  - **livestream**
  - **livestream:original**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 8f0977849..46c3356e9 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.09.28'
+__version__ = '2015.10.06'

From 4810c48d6d7e950c2cb1203f4d07bea1ba02c1e1 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 6 Oct 2015 14:28:14 +0200
Subject: [PATCH 2708/6219] [compat] Do not compare None <= 0

The result is meaningless (and it emits a warning in cpython2 when called with -3), so handle None before making integer comparisons.
---
 youtube_dl/compat.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index c36c9c23f..1ba4ab78c 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -417,18 +417,18 @@ else:
     _terminal_size = collections.namedtuple('terminal_size', ['columns', 'lines'])
 
     def compat_get_terminal_size(fallback=(80, 24)):
-        columns = compat_getenv('COLUMNS', None)
+        columns = compat_getenv('COLUMNS')
         if columns:
             columns = int(columns)
         else:
             columns = None
-        lines = compat_getenv('LINES', None)
+        lines = compat_getenv('LINES')
         if lines:
             lines = int(lines)
         else:
             lines = None
 
-        if columns <= 0 or lines <= 0:
+        if columns is None or lines is None or columns <= 0 or lines <= 0:
             try:
                 sp = subprocess.Popen(
                     ['stty', 'size'],
@@ -438,9 +438,9 @@ else:
             except Exception:
                 _columns, _lines = _terminal_size(*fallback)
 
-            if columns <= 0:
+            if columns is None or columns <= 0:
                 columns = _columns
-            if lines <= 0:
+            if lines is None or lines <= 0:
                 lines = _lines
         return _terminal_size(columns, lines)
 

From 86be82610c35a684bee97b22c8d9a2a83bab1bba Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 6 Oct 2015 17:43:50 +0200
Subject: [PATCH 2709/6219] release 2015.10.06.1

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 46c3356e9..4fce70fa6 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.10.06'
+__version__ = '2015.10.06.1'

From f2dbc54066f56a98c689099b920e6a596d4ffdfc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 6 Oct 2015 22:02:28 +0600
Subject: [PATCH 2710/6219] [compat] Fix wrong lines/columns order

stty size is rows x columns
---
 youtube_dl/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 1ba4ab78c..192e1c515 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -434,7 +434,7 @@ else:
                     ['stty', 'size'],
                     stdout=subprocess.PIPE, stderr=subprocess.PIPE)
                 out, err = sp.communicate()
-                _columns, _lines = map(int, out.split())
+                _lines, _columns = map(int, out.split())
             except Exception:
                 _columns, _lines = _terminal_size(*fallback)
 

From f648e682a7a82f71a278ec92159ac5d343a4b3eb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 6 Oct 2015 22:58:18 +0600
Subject: [PATCH 2711/6219] [bandcamp] Prepend download URL with scheme when
 necessary (Closes #7077)

---
 youtube_dl/extractor/bandcamp.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/bandcamp.py b/youtube_dl/extractor/bandcamp.py
index 505877b77..a27f3e748 100644
--- a/youtube_dl/extractor/bandcamp.py
+++ b/youtube_dl/extractor/bandcamp.py
@@ -93,8 +93,8 @@ class BandcampIE(InfoExtractor):
         final_url_webpage = self._download_webpage(request_url, video_id, 'Requesting download url')
         # If we could correctly generate the .rand field the url would be
         # in the "download_url" key
-        final_url = self._search_regex(
-            r'"retry_url":"(.*?)"', final_url_webpage, 'final video URL')
+        final_url = self._proto_relative_url(self._search_regex(
+            r'"retry_url":"(.+?)"', final_url_webpage, 'final video URL'), 'http:')
 
         return {
             'id': video_id,

From 83a56686944225137ce646748b8e940a21584941 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 6 Oct 2015 23:08:28 +0600
Subject: [PATCH 2712/6219] [canalplus] Extend video id regex (Closes #7076)

---
 youtube_dl/extractor/canalplus.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/canalplus.py b/youtube_dl/extractor/canalplus.py
index 57e0cda2c..c0ca3a0fa 100644
--- a/youtube_dl/extractor/canalplus.py
+++ b/youtube_dl/extractor/canalplus.py
@@ -78,7 +78,8 @@ class CanalplusIE(InfoExtractor):
         if video_id is None:
             webpage = self._download_webpage(url, display_id)
             video_id = self._search_regex(
-                r'<canal:player[^>]+?videoId="(\d+)"', webpage, 'video id')
+                [r'<canal:player[^>]+?videoId=(["\'])(?P<id>\d+)', r'id=["\']canal_video_player(?P<id>\d+)'],
+                 webpage, 'video id', group='id')
 
         info_url = self._VIDEO_INFO_TEMPLATE % (site_id, video_id)
         doc = self._download_xml(info_url, video_id, 'Downloading video XML')

From fc10824cb67feb837ea57d60decc293b7b719cfb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 7 Oct 2015 02:43:12 +0600
Subject: [PATCH 2713/6219] [canalplus] PEP 8

---
 youtube_dl/extractor/canalplus.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/canalplus.py b/youtube_dl/extractor/canalplus.py
index c0ca3a0fa..004372f8d 100644
--- a/youtube_dl/extractor/canalplus.py
+++ b/youtube_dl/extractor/canalplus.py
@@ -79,7 +79,7 @@ class CanalplusIE(InfoExtractor):
             webpage = self._download_webpage(url, display_id)
             video_id = self._search_regex(
                 [r'<canal:player[^>]+?videoId=(["\'])(?P<id>\d+)', r'id=["\']canal_video_player(?P<id>\d+)'],
-                 webpage, 'video id', group='id')
+                webpage, 'video id', group='id')
 
         info_url = self._VIDEO_INFO_TEMPLATE % (site_id, video_id)
         doc = self._download_xml(info_url, video_id, 'Downloading video XML')

From 945e5c56e3ccd45cdcc3ee45b9ffcbad7e614f90 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 6 Oct 2015 23:46:16 +0200
Subject: [PATCH 2714/6219] release 2015.10.06.2

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 4fce70fa6..112c78835 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.10.06.1'
+__version__ = '2015.10.06.2'

From 139f27827e1d771aba5cf7f1473129073686f5ab Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 7 Oct 2015 06:53:19 +0100
Subject: [PATCH 2715/6219] [nba] skip Legacy Video Files

---
 youtube_dl/extractor/nba.py | 35 +++++++++++------------------------
 1 file changed, 11 insertions(+), 24 deletions(-)

diff --git a/youtube_dl/extractor/nba.py b/youtube_dl/extractor/nba.py
index 3d38d080e..73116c7c6 100644
--- a/youtube_dl/extractor/nba.py
+++ b/youtube_dl/extractor/nba.py
@@ -38,11 +38,6 @@ class NBAIE(InfoExtractor):
         }
     }]
 
-    _BASE_PATHS = {
-        'turner': 'http://nba.cdn.turner.com/nba/big',
-        'akamai': 'http://nbavod-f.akamaihd.net',
-    }
-
     _QUALITIES = {
         '420mp4': {
             'width': 400,
@@ -54,55 +49,50 @@ class NBAIE(InfoExtractor):
             'height': 234,
             'preference': 2,
         },
-        '556': {
-            'width': 416,
-            'height': 234,
-            'preference': 3,
-        },
         '480x320_910': {
             'width': 480,
             'height': 320,
-            'preference': 4,
+            'preference': 3,
         },
         'nba_576x324': {
             'width': 576,
             'height': 324,
-            'preference': 5,
+            'preference': 4,
         },
         'nba_640x360': {
             'width': 640,
             'height': 360,
-            'preference': 6,
+            'preference': 5,
         },
         '640x360_664b': {
             'width': 640,
             'height': 360,
-            'preference': 7,
+            'preference': 6,
         },
         '640x360_664m': {
             'width': 640,
             'height': 360,
-            'preference': 8,
+            'preference': 7,
         },
         '768x432_996': {
             'width': 768,
             'height': 432,
-            'preference': 9,
+            'preference': 8,
         },
         '768x432_1404': {
             'width': 768,
             'height': 432,
-            'preference': 10,
+            'preference': 9,
         },
         '960x540_2104': {
             'width': 960,
             'height': 540,
-            'preference': 11,
+            'preference': 10,
         },
         '1280x720_3072': {
             'width': 1280,
             'height': 720,
-            'preference': 12,
+            'preference': 11,
         },
     }
 
@@ -127,11 +117,8 @@ class NBAIE(InfoExtractor):
         formats = []
         for video_file in video_info.find('files').iter('file'):
             video_url = video_file.text
-            if not video_url.startswith('http://'):
-                if video_url.endswith('.m3u8') or video_url.endswith('.f4m'):
-                    video_url = self._BASE_PATHS['akamai'] + video_url
-                else:
-                    video_url = self._BASE_PATHS['turner'] + video_url
+            if video_url.startswith('/'):
+                continue
             if video_url.endswith('.m3u8'):
                 formats.extend(self._extract_m3u8_formats(video_url, video_id))
             elif video_url.endswith('.f4m'):

From ecf6de5b02ad3996f770efd33f9b400d04ac8a85 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 7 Oct 2015 07:09:45 +0100
Subject: [PATCH 2716/6219] [nba] extract width,height and bitrate from format
 key

---
 youtube_dl/extractor/nba.py | 68 ++++---------------------------------
 1 file changed, 6 insertions(+), 62 deletions(-)

diff --git a/youtube_dl/extractor/nba.py b/youtube_dl/extractor/nba.py
index 73116c7c6..ea1482fc8 100644
--- a/youtube_dl/extractor/nba.py
+++ b/youtube_dl/extractor/nba.py
@@ -1,5 +1,7 @@
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..utils import (
     parse_duration,
@@ -38,64 +40,6 @@ class NBAIE(InfoExtractor):
         }
     }]
 
-    _QUALITIES = {
-        '420mp4': {
-            'width': 400,
-            'height': 224,
-            'preference': 1,
-        },
-        '416x234': {
-            'width': 416,
-            'height': 234,
-            'preference': 2,
-        },
-        '480x320_910': {
-            'width': 480,
-            'height': 320,
-            'preference': 3,
-        },
-        'nba_576x324': {
-            'width': 576,
-            'height': 324,
-            'preference': 4,
-        },
-        'nba_640x360': {
-            'width': 640,
-            'height': 360,
-            'preference': 5,
-        },
-        '640x360_664b': {
-            'width': 640,
-            'height': 360,
-            'preference': 6,
-        },
-        '640x360_664m': {
-            'width': 640,
-            'height': 360,
-            'preference': 7,
-        },
-        '768x432_996': {
-            'width': 768,
-            'height': 432,
-            'preference': 8,
-        },
-        '768x432_1404': {
-            'width': 768,
-            'height': 432,
-            'preference': 9,
-        },
-        '960x540_2104': {
-            'width': 960,
-            'height': 540,
-            'preference': 10,
-        },
-        '1280x720_3072': {
-            'width': 1280,
-            'height': 720,
-            'preference': 11,
-        },
-    }
-
     def _real_extract(self, url):
         video_id = self._match_id(url)
         video_info = self._download_xml('http://www.nba.com/video/%s.xml' % video_id, video_id)
@@ -125,13 +69,13 @@ class NBAIE(InfoExtractor):
                 formats.extend(self._extract_f4m_formats(video_url + '?hdcore=3.4.1.1', video_id))
             else:
                 key = video_file.attrib.get('bitrate')
-                quality = self._QUALITIES[key]
+                width, height, bitrate = re.search(r'(\d+)x(\d+)(?:_(\d+))?', key).groups()
                 formats.append({
                     'format_id': key,
                     'url': video_url,
-                    'width': quality['width'],
-                    'height': quality['height'],
-                    'preference': quality['preference'],
+                    'width': int_or_none(width),
+                    'height': int_or_none(height),
+                    'tbr': int_or_none(bitrate),
                 })
         self._sort_formats(formats)
 

From bd5376c182bd5ee2103aec21144286dbefbeb797 Mon Sep 17 00:00:00 2001
From: Tom Gijselinck <tomgijselinck@gmail.com>
Date: Wed, 7 Oct 2015 10:32:44 +0200
Subject: [PATCH 2717/6219] Fix typos and improve grammar and spelling

---
 README.md | 32 ++++++++++++++++----------------
 1 file changed, 16 insertions(+), 16 deletions(-)

diff --git a/README.md b/README.md
index 5ff8ca85e..cf4aebf3d 100644
--- a/README.md
+++ b/README.md
@@ -404,18 +404,18 @@ which means you can modify it, redistribute it or use it however you like.
 
 # CONFIGURATION
 
-You can configure youtube-dl by placing any supported command line option to a configuration file. On Linux, system wide configuration file is located at `/etc/youtube-dl.conf` and user wide configuration file at `~/.config/youtube-dl/config`. On Windows, the user wide configuration file locations are `%APPDATA%\youtube-dl\config.txt` or `C:\Users\<user name>\youtube-dl.conf`. For example, with the following configration file youtube-dl will always extract the audio, not copy the mtime and use proxy:
+You can configure youtube-dl by placing any supported command line option to a configuration file. On Linux, the system wide configuration file is located at `/etc/youtube-dl.conf` and the user wide configuration file at `~/.config/youtube-dl/config`. On Windows, the user wide configuration file locations are `%APPDATA%\youtube-dl\config.txt` or `C:\Users\<user name>\youtube-dl.conf`. For example, with the following configuration file youtube-dl will always extract the audio, not copy the mtime and use a proxy:
 ```
 --extract-audio
 --no-mtime
 --proxy 127.0.0.1:3128
 ```
 
-You can use `--ignore-config` if you want to disable configuration file for a particular youtube-dl run.
+You can use `--ignore-config` if you want to disable the configuration file for a particular youtube-dl run.
 
 ### Authentication with `.netrc` file ###
 
-You may also want to configure automatic credentials storage for extractors that support authentication (by providing login and password with `--username` and `--password`) in order not to pass credentials as command line arguments on every youtube-dl execution and prevent tracking plain text passwords in shell command history. You can achieve this using [`.netrc` file](http://stackoverflow.com/tags/.netrc/info) on per extractor basis. For that you will need to create `.netrc` file in your `$HOME` and restrict permissions to read/write by you only:
+You may also want to configure automatic credentials storage for extractors that support authentication (by providing login and password with `--username` and `--password`) in order not to pass credentials as command line arguments on every youtube-dl execution and prevent tracking plain text passwords in the shell command history. You can achieve this using a [`.netrc` file](http://stackoverflow.com/tags/.netrc/info) on per extractor basis. For that you will need to create a`.netrc` file in your `$HOME` and restrict permissions to read/write by you only:
 ```
 touch $HOME/.netrc
 chmod a-rwx,u+rw $HOME/.netrc
@@ -429,13 +429,13 @@ For example:
 machine youtube login myaccount@gmail.com password my_youtube_password
 machine twitch login my_twitch_account_name password my_twitch_password
 ```
-To activate authentication with `.netrc` file you should pass `--netrc` to youtube-dl or place it in [configuration file](#configuration).
+To activate authentication with the `.netrc` file you should pass `--netrc` to youtube-dl or place it in the [configuration file](#configuration).
 
-On Windows you may also need to setup `%HOME%` environment variable manually.
+On Windows you may also need to setup the `%HOME%` environment variable manually.
 
 # OUTPUT TEMPLATE
 
-The `-o` option allows users to indicate a template for the output file names. The basic usage is not to set any template arguments when downloading a single file, like in `youtube-dl -o funny_video.flv "http://some/video"`. However, it may contain special sequences that will be replaced when downloading each video. The special sequences have the format `%(NAME)s`. To clarify, that is a percent symbol followed by a name in parenthesis, followed by a lowercase S. Allowed names are:
+The `-o` option allows users to indicate a template for the output file names. The basic usage is not to set any template arguments when downloading a single file, like in `youtube-dl -o funny_video.flv "http://some/video"`. However, it may contain special sequences that will be replaced when downloading each video. The special sequences have the format `%(NAME)s`. To clarify, that is a percent symbol followed by a name in parentheses, followed by a lowercase S. Allowed names are:
 
  - `id`: The sequence will be replaced by the video identifier.
  - `url`: The sequence will be replaced by the video URL.
@@ -463,18 +463,18 @@ youtube-dl_test_video_.mp4          # A simple file name
 
 # FORMAT SELECTION
 
-By default youtube-dl tries to download the best quality, but sometimes you may want to download other format.
+By default youtube-dl tries to download the best quality, but sometimes you may want to download in a different format.
 The simplest case is requesting a specific format, for example `-f 22`. You can get the list of available formats using `--list-formats`, you can also use a file extension (currently it supports aac, m4a, mp3, mp4, ogg, wav, webm) or the special names `best`, `bestvideo`, `bestaudio` and `worst`.
 
 If you want to download multiple videos and they don't have the same formats available, you can specify the order of preference using slashes, as in `-f 22/17/18`. You can also filter the video results by putting a condition in brackets, as in `-f "best[height=720]"` (or `-f "[filesize>10M]"`).  This works for filesize, height, width, tbr, abr, vbr, asr, and fps and the comparisons <, <=, >, >=, =, != and for ext, acodec, vcodec, container, and protocol and the comparisons =, != . Formats for which the value is not known are excluded unless you put a question mark (?) after the operator. You can combine format filters, so  `-f "[height <=? 720][tbr>500]"` selects up to 720p videos (or videos where the height is not known) with a bitrate of at least 500 KBit/s. Use commas to download multiple formats, such as `-f 136/137/mp4/bestvideo,140/m4a/bestaudio`. You can merge the video and audio of two formats into a single file using `-f <video-format>+<audio-format>` (requires ffmpeg or avconv), for example `-f bestvideo+bestaudio`. Format selectors can also be grouped using parentheses, for example if you want to download the best mp4 and webm formats with a height lower than 480 you can use `-f '(mp4,webm)[height<480]'`.
 
-Since the end of April 2015 and version 2015.04.26 youtube-dl uses `-f bestvideo+bestaudio/best` as default format selection (see #5447, #5456). If ffmpeg or avconv are installed this results in downloading `bestvideo` and `bestaudio` separately and muxing them together into a single file giving the best overall quality available. Otherwise it falls back to `best` and results in downloading best available quality served as a single file. `best` is also needed for videos that don't come from YouTube because they don't provide the audio and video in two different files. If you want to only download some dash formats (for example if you are not interested in getting videos with a resolution higher than 1080p), you can add `-f bestvideo[height<=?1080]+bestaudio/best` to your configuration file. Note that if you use youtube-dl to stream to `stdout` (and most likely to pipe it to your media player then), i.e. you explicitly specify output template as `-o -`, youtube-dl still uses `-f best` format selection in order to start content delivery immediately to your player and not to wait until `bestvideo` and `bestaudio` are downloaded and muxed.
+Since the end of April 2015 and version 2015.04.26 youtube-dl uses `-f bestvideo+bestaudio/best` as default format selection (see #5447, #5456). If ffmpeg or avconv are installed this results in downloading `bestvideo` and `bestaudio` separately and muxing them together into a single file giving the best overall quality available. Otherwise it falls back to `best` and results in downloading the best available quality served as a single file. `best` is also needed for videos that don't come from YouTube because they don't provide the audio and video in two different files. If you want to only download some dash formats (for example if you are not interested in getting videos with a resolution higher than 1080p), you can add `-f bestvideo[height<=?1080]+bestaudio/best` to your configuration file. Note that if you use youtube-dl to stream to `stdout` (and most likely to pipe it to your media player then), i.e. you explicitly specify output template as `-o -`, youtube-dl still uses `-f best` format selection in order to start content delivery immediately to your player and not to wait until `bestvideo` and `bestaudio` are downloaded and muxed.
 
-If you want to preserve the old format selection behavior (prior to youtube-dl 2015.04.26), i.e. you want to download best available quality media served as a single file, you should explicitly specify your choice with `-f best`. You may want to add it to the [configuration file](#configuration) in order not to type it every time you run youtube-dl.
+If you want to preserve the old format selection behavior (prior to youtube-dl 2015.04.26), i.e. you want to download the best available quality media served as a single file, you should explicitly specify your choice with `-f best`. You may want to add it to the [configuration file](#configuration) in order not to type it every time you run youtube-dl.
 
 # VIDEO SELECTION
 
-Videos can be filtered by their upload date using the options `--date`, `--datebefore` or `--dateafter`, they accept dates in two formats:
+Videos can be filtered by their upload date using the options `--date`, `--datebefore` or `--dateafter`. They accept dates in two formats:
 
  - Absolute dates: Dates in the format `YYYYMMDD`.
  - Relative dates: Dates in the format `(now|today)[+-][0-9](day|week|month|year)(s)?`
@@ -488,7 +488,7 @@ $ youtube-dl --dateafter now-6months
 # Download only the videos uploaded on January 1, 1970
 $ youtube-dl --date 19700101
 
-$ # will only download the videos uploaded in the 200x decade
+$ # Download only the videos uploaded in the 200x decade
 $ youtube-dl --dateafter 20000101 --datebefore 20091231
 ```
 
@@ -500,7 +500,7 @@ If you've followed [our manual installation instructions](http://rg3.github.io/y
 
 If you have used pip, a simple `sudo pip install -U youtube-dl` is sufficient to update.
 
-If you have installed youtube-dl using a package manager like *apt-get* or *yum*, use the standard system update mechanism to update. Note that distribution packages are often outdated. As a rule of thumb, youtube-dl releases at least once a month, and often weekly or even daily. Simply go to http://yt-dl.org/ to find out the current version. Unfortunately, there is nothing we youtube-dl developers can do if your distributions serves a really outdated version. You can (and should) complain to your distribution in their bugtracker or support forum.
+If you have installed youtube-dl using a package manager like *apt-get* or *yum*, use the standard system update mechanism to update. Note that distribution packages are often outdated. As a rule of thumb, youtube-dl releases at least once a month, and often weekly or even daily. Simply go to http://yt-dl.org/ to find out the current version. Unfortunately, there is nothing we youtube-dl developers can do if your distribution serves a really outdated version. You can (and should) complain to your distribution in their bugtracker or support forum.
 
 As a last resort, you can also uninstall the version installed by your package manager and follow our manual installation instructions. For that, remove the distribution's package, with a line like
 
@@ -558,7 +558,7 @@ YouTube requires an additional signature since September 2012 which is not suppo
 
 ### Video URL contains an ampersand and I'm getting some strange output `[1] 2839` or `'v' is not recognized as an internal or external command` ###
 
-That's actually the output from your shell. Since ampersand is one of the special shell characters it's interpreted by shell preventing you from passing the whole URL to youtube-dl. To disable your shell from interpreting the ampersands (or any other special characters) you have to either put the whole URL in quotes or escape them with a backslash (which approach will work depends on your shell).
+That's actually the output from your shell. Since ampersand is one of the special shell characters it's interpreted by the shell preventing you from passing the whole URL to youtube-dl. To disable your shell from interpreting the ampersands (or any other special characters) you have to either put the whole URL in quotes or escape them with a backslash (which approach will work depends on your shell).
 
 For example if your URL is https://www.youtube.com/watch?t=4&v=BaW_jenozKc you should end up with following command:
 
@@ -618,9 +618,9 @@ Either prepend `http://www.youtube.com/watch?v=` or separate the ID from the opt
 
 ### How do I pass cookies to youtube-dl?
 
-Use the `--cookies` option, for example `--cookies /path/to/cookies/file.txt`. Note that cookies file must be in Mozilla/Netscape format and the first line of cookies file must be either `# HTTP Cookie File` or `# Netscape HTTP Cookie File`. Make sure you have correct [newline format](https://en.wikipedia.org/wiki/Newline) in cookies file and convert newlines if necessary to correspond your OS, namely `CRLF` (`\r\n`) for Windows, `LF` (`\n`) for Linux and `CR` (`\r`) for Mac OS. `HTTP Error 400: Bad Request` when using `--cookies` is a good sign of invalid newline format.
+Use the `--cookies` option, for example `--cookies /path/to/cookies/file.txt`. Note that the cookies file must be in Mozilla/Netscape format and the first line of the cookies file must be either `# HTTP Cookie File` or `# Netscape HTTP Cookie File`. Make sure you have correct [newline format](https://en.wikipedia.org/wiki/Newline) in the cookies file and convert newlines if necessary to correspond with your OS, namely `CRLF` (`\r\n`) for Windows, `LF` (`\n`) for Linux and `CR` (`\r`) for Mac OS. `HTTP Error 400: Bad Request` when using `--cookies` is a good sign of invalid newline format.
 
-Passing cookies to youtube-dl is a good way to workaround login when particular extractor does not implement it explicitly.
+Passing cookies to youtube-dl is a good way to workaround login when a particular extractor does not implement it explicitly.
 
 ### Can you add support for this anime video site, or site which shows current movies for free?
 
@@ -720,7 +720,7 @@ If you want to add support for a new site, you can follow this quick list (assum
             }
     ```
 5. Add an import in [`youtube_dl/extractor/__init__.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/__init__.py).
-6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will be then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc.
+6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc.
 7. Have a look at [`youtube_dl/extractor/common.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L62-L200). Add tests and code for as many as you want.
 8. If you can, check the code with [flake8](https://pypi.python.org/pypi/flake8).
 9. When the tests pass, [add](http://git-scm.com/docs/git-add) the new files and [commit](http://git-scm.com/docs/git-commit) them and [push](http://git-scm.com/docs/git-push) the result, like this:

From 6a11bb77baf9f70da76f2595b74061b31223d4ff Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 7 Oct 2015 12:17:32 +0100
Subject: [PATCH 2718/6219] [nba] add support for team subsites

---
 youtube_dl/extractor/nba.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/nba.py b/youtube_dl/extractor/nba.py
index ea1482fc8..a0cc58c12 100644
--- a/youtube_dl/extractor/nba.py
+++ b/youtube_dl/extractor/nba.py
@@ -10,13 +10,13 @@ from ..utils import (
 
 
 class NBAIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:watch\.|www\.)?nba\.com/(?:nba/)?video/(?P<id>[^?]*?)/?(?:/index\.html)?(?:\?.*)?$'
+    _VALID_URL = r'https?://(?:watch\.|www\.)?nba\.com/(?P<path>(?:[^/]+/)?video/(?P<id>[^?]*?))/?(?:/index\.html)?(?:\?.*)?$'
     _TESTS = [{
         'url': 'http://www.nba.com/video/games/nets/2012/12/04/0021200253-okc-bkn-recap.nba/index.html',
-        'md5': '9d902940d2a127af3f7f9d2f3dc79c96',
+        'md5': '9e7729d3010a9c71506fd1248f74e4f4',
         'info_dict': {
             'id': '0021200253-okc-bkn-recap',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': 'Thunder vs. Nets',
             'description': 'Kevin Durant scores 32 points and dishes out six assists as the Thunder beat the Nets in Brooklyn.',
             'duration': 181,
@@ -27,7 +27,7 @@ class NBAIE(InfoExtractor):
         'url': 'http://www.nba.com/video/games/hornets/2014/12/05/0021400276-nyk-cha-play5.nba/',
         'only_matching': True,
     },{
-        'url': 'http://watch.nba.com/nba/video/channels/playoffs/2015/05/20/0041400301-cle-atl-recap.nba',
+        'url': 'http://watch.nba.com/video/channels/playoffs/2015/05/20/0041400301-cle-atl-recap.nba',
         'md5': 'b2b39b81cf28615ae0c3360a3f9668c4',
         'info_dict': {
             'id': '0041400301-cle-atl-recap',
@@ -41,8 +41,8 @@ class NBAIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-        video_info = self._download_xml('http://www.nba.com/video/%s.xml' % video_id, video_id)
+        path, video_id = re.match(self._VALID_URL, url).groups()
+        video_info = self._download_xml('http://www.nba.com/%s.xml' % path, video_id)
         video_id = video_info.find('slug').text
         title = video_info.find('headline').text
         description = video_info.find('description').text
@@ -64,9 +64,9 @@ class NBAIE(InfoExtractor):
             if video_url.startswith('/'):
                 continue
             if video_url.endswith('.m3u8'):
-                formats.extend(self._extract_m3u8_formats(video_url, video_id))
+                formats.extend(self._extract_m3u8_formats(video_url, video_id, m3u8_id='hls'))
             elif video_url.endswith('.f4m'):
-                formats.extend(self._extract_f4m_formats(video_url + '?hdcore=3.4.1.1', video_id))
+                formats.extend(self._extract_f4m_formats(video_url + '?hdcore=3.4.1.1', video_id, f4m_id='hds'))
             else:
                 key = video_file.attrib.get('bitrate')
                 width, height, bitrate = re.search(r'(\d+)x(\d+)(?:_(\d+))?', key).groups()

From db7a28eccb743db4e546a12fb78ae5dc40ef44d9 Mon Sep 17 00:00:00 2001
From: AndroKev <AndroKev@users.noreply.github.com>
Date: Thu, 8 Oct 2015 09:56:39 +0200
Subject: [PATCH 2719/6219] FIX: nowtv

now the download works for me
thx to http://board.gulli.com/thread/1251646-videos-von-webseiten-runterladen-sammelthread-alle-fragen-hier-rein-/?p=14980081#post14980081
---
 youtube_dl/extractor/nowtv.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/nowtv.py b/youtube_dl/extractor/nowtv.py
index c8257719f..b0bdffc4e 100644
--- a/youtube_dl/extractor/nowtv.py
+++ b/youtube_dl/extractor/nowtv.py
@@ -167,8 +167,8 @@ class NowTVIE(InfoExtractor):
                 'app': app,
                 'play_path': 'mp4:%s' % play_path,
                 'ext': 'flv',
-                'page_url': url,
-                'player_url': 'http://rtl-now.rtl.de/includes/nc_player.swf',
+                'page_url': 'http://rtlnow.rtl.de',
+                'player_url': 'http://cdn.static-fra.de/now/vodplayer.swf',
                 'tbr': int_or_none(item.get('bitrate')),
             })
         self._sort_formats(formats)

From b90b0c4ffa6f84a2ea5556d4df99de5f8ef2c7dd Mon Sep 17 00:00:00 2001
From: kitty <magicvidyakitty@gmail.com>
Date: Thu, 8 Oct 2015 06:31:23 -0700
Subject: [PATCH 2720/6219] Fixed 4tube.com extractor to pull metadata from
 associated Javascript and not the HTML of the desired page.

---
 youtube_dl/extractor/fourtube.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/fourtube.py b/youtube_dl/extractor/fourtube.py
index 3bb4f6239..226ee67f0 100644
--- a/youtube_dl/extractor/fourtube.py
+++ b/youtube_dl/extractor/fourtube.py
@@ -45,11 +45,9 @@ class FourTubeIE(InfoExtractor):
             'uploadDate', webpage))
         thumbnail = self._html_search_meta('thumbnailUrl', webpage)
         uploader_id = self._html_search_regex(
-            r'<a class="img-avatar" href="[^"]+/channels/([^/"]+)" title="Go to [^"]+ page">',
-            webpage, 'uploader id')
+            r'<a class="img-avatar" href="[^"]+/users/([^/"]+)" title="Go to [^"]+ page">', webpage, 'uploader id')
         uploader = self._html_search_regex(
-            r'<a class="img-avatar" href="[^"]+/channels/[^/"]+" title="Go to ([^"]+) page">',
-            webpage, 'uploader')
+            r'<a class="img-avatar" href="[^"]+/users/[^/"]+" title="Go to ([^"]+) page">', webpage, 'uploader')
 
         categories_html = self._search_regex(
             r'(?s)><i class="icon icon-tag"></i>\s*Categories / Tags\s*.*?<ul class="list">(.*?)</ul>',
@@ -68,9 +66,12 @@ class FourTubeIE(InfoExtractor):
             webpage, 'like count', fatal=False))
         duration = parse_duration(self._html_search_meta('duration', webpage))
 
+        player_url = self._search_regex(r'<script id="playerembed" src="([^"]+)">',webpage,'player javascript')
+        player_js = self._download_webpage(player_url,video_id,'Downloading player Javascript')
+
         params_js = self._search_regex(
             r'\$\.ajax\(url,\ opts\);\s*\}\s*\}\)\(([0-9,\[\] ]+)\)',
-            webpage, 'initialization parameters'
+            player_js, 'initialization parameters'
         )
         params = self._parse_json('[%s]' % params_js, video_id)
         media_id = params[0]

From 96c48553317cf28ea92d7feaa5701e3ebd93ea25 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 8 Oct 2015 20:21:07 +0600
Subject: [PATCH 2721/6219] [4tube] Style and make more robust

---
 youtube_dl/extractor/fourtube.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/fourtube.py b/youtube_dl/extractor/fourtube.py
index 226ee67f0..406387e57 100644
--- a/youtube_dl/extractor/fourtube.py
+++ b/youtube_dl/extractor/fourtube.py
@@ -66,8 +66,11 @@ class FourTubeIE(InfoExtractor):
             webpage, 'like count', fatal=False))
         duration = parse_duration(self._html_search_meta('duration', webpage))
 
-        player_url = self._search_regex(r'<script id="playerembed" src="([^"]+)">',webpage,'player javascript')
-        player_js = self._download_webpage(player_url,video_id,'Downloading player Javascript')
+        player_js = self._download_webpage(
+            self._search_regex(
+                r'<script[^>]id=(["\'])playerembed\1[^>]+src=(["\'])(?P<url>.+?)\2',
+                webpage, 'player JS', group='url'),
+            video_id, 'Downloading player JS')
 
         params_js = self._search_regex(
             r'\$\.ajax\(url,\ opts\);\s*\}\s*\}\)\(([0-9,\[\] ]+)\)',

From c677e49bd11c3dfada05de9fc1d4de04a577d70e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 8 Oct 2015 20:22:08 +0600
Subject: [PATCH 2722/6219] [4tube] Revert uploader regexes and make non fatal

---
 youtube_dl/extractor/fourtube.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/fourtube.py b/youtube_dl/extractor/fourtube.py
index 406387e57..cd7668f18 100644
--- a/youtube_dl/extractor/fourtube.py
+++ b/youtube_dl/extractor/fourtube.py
@@ -45,9 +45,11 @@ class FourTubeIE(InfoExtractor):
             'uploadDate', webpage))
         thumbnail = self._html_search_meta('thumbnailUrl', webpage)
         uploader_id = self._html_search_regex(
-            r'<a class="img-avatar" href="[^"]+/users/([^/"]+)" title="Go to [^"]+ page">', webpage, 'uploader id')
+            r'<a class="img-avatar" href="[^"]+/channels/([^/"]+)" title="Go to [^"]+ page">',
+            webpage, 'uploader id', fatal=False)
         uploader = self._html_search_regex(
-            r'<a class="img-avatar" href="[^"]+/users/[^/"]+" title="Go to ([^"]+) page">', webpage, 'uploader')
+            r'<a class="img-avatar" href="[^"]+/channels/[^/"]+" title="Go to ([^"]+) page">',
+            webpage, 'uploader', fatal=False)
 
         categories_html = self._search_regex(
             r'(?s)><i class="icon icon-tag"></i>\s*Categories / Tags\s*.*?<ul class="list">(.*?)</ul>',

From 4356d907c1afae09ff67a17c90b53be466b13623 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 8 Oct 2015 20:49:56 +0600
Subject: [PATCH 2723/6219] [4tube] Try extracting sources from the webpage
 before fetching player.js (Closes #7103)

---
 youtube_dl/extractor/fourtube.py | 31 ++++++++++++++++++-------------
 1 file changed, 18 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/fourtube.py b/youtube_dl/extractor/fourtube.py
index cd7668f18..fb6d108c0 100644
--- a/youtube_dl/extractor/fourtube.py
+++ b/youtube_dl/extractor/fourtube.py
@@ -68,19 +68,24 @@ class FourTubeIE(InfoExtractor):
             webpage, 'like count', fatal=False))
         duration = parse_duration(self._html_search_meta('duration', webpage))
 
-        player_js = self._download_webpage(
-            self._search_regex(
-                r'<script[^>]id=(["\'])playerembed\1[^>]+src=(["\'])(?P<url>.+?)\2',
-                webpage, 'player JS', group='url'),
-            video_id, 'Downloading player JS')
-
-        params_js = self._search_regex(
-            r'\$\.ajax\(url,\ opts\);\s*\}\s*\}\)\(([0-9,\[\] ]+)\)',
-            player_js, 'initialization parameters'
-        )
-        params = self._parse_json('[%s]' % params_js, video_id)
-        media_id = params[0]
-        sources = ['%s' % p for p in params[2]]
+        media_id = self._search_regex(
+            r'<button[^>]+data-id=(["\'])(?P<id>\d+)\1[^>]+data-quality=', webpage,
+            'media id', default=None, group='id')
+        sources = [
+            quality
+            for _, quality in re.findall(r'<button[^>]+data-quality=(["\'])(.+?)\1', webpage)]
+        if not (media_id and sources):
+            player_js = self._download_webpage(
+                self._search_regex(
+                    r'<script[^>]id=(["\'])playerembed\1[^>]+src=(["\'])(?P<url>.+?)\2',
+                    webpage, 'player JS', group='url'),
+                video_id, 'Downloading player JS')
+            params_js = self._search_regex(
+                r'\$\.ajax\(url,\ opts\);\s*\}\s*\}\)\(([0-9,\[\] ]+)\)',
+                player_js, 'initialization parameters')
+            params = self._parse_json('[%s]' % params_js, video_id)
+            media_id = params[0]
+            sources = ['%s' % p for p in params[2]]
 
         token_url = 'http://tkn.4tube.com/{0}/desktop/{1}'.format(
             media_id, '+'.join(sources))

From 7faf7e752342b21772c8647902bbf4f13cc57956 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 8 Oct 2015 18:39:24 +0100
Subject: [PATCH 2724/6219] [pbs] detect errors from http error code

---
 youtube_dl/extractor/pbs.py | 23 +++++++++++++++--------
 1 file changed, 15 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 6923c6094..82218b933 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -39,6 +39,7 @@ class PBSIE(InfoExtractor):
             'params': {
                 'skip_download': True,  # requires ffmpeg
             },
+            'skip': 'Expired',
         },
         {
             'url': 'http://www.pbs.org/wgbh/pages/frontline/losing-iraq/',
@@ -108,12 +109,12 @@ class PBSIE(InfoExtractor):
         {
             'url': 'http://www.pbs.org/wgbh/americanexperience/films/death/player/',
             'info_dict': {
-                'id': '2280706814',
+                'id': '2276541483',
                 'display_id': 'player',
                 'ext': 'mp4',
-                'title': 'American Experience - Death and the Civil War',
+                'title': 'American Experience - Death and the Civil War, Chapter 1',
                 'description': 'American Experience, TV’s most-watched history series, brings to life the compelling stories from our past that inform our understanding of the world today.',
-                'duration': 6705,
+                'duration': 682,
                 'thumbnail': 're:^https?://.*\.jpg$',
             },
             'params': {
@@ -134,6 +135,7 @@ class PBSIE(InfoExtractor):
             'params': {
                 'skip_download': True,  # requires ffmpeg
             },
+            'skip': 'Expired',
         },
         {
             # Video embedded in iframe containing angle brackets as attribute's value (e.g.
@@ -231,13 +233,18 @@ class PBSIE(InfoExtractor):
                 'Downloading %s video url info' % encoding_name)
 
             if redirect_info['status'] == 'error':
-                if redirect_info['http_code'] == 403:
-                    message = (
-                        'The video is not available in your region due to '
-                        'right restrictions')
+                http_code = redirect_info['http_code']
+                if http_code == 403:
+                    message = 'We\'re sorry, but this video is not available in your region due to right restrictions.'
+                elif http_code == 101:
+                    message = 'We\'re sorry, but this video is not yet available.'
+                elif http_code == 404:
+                    message = 'We are experiencing technical difficulties that are preventing us from playing the video at this time. Please check back again soon.'
+                elif http_code == 410:
+                    message = 'This video has expired and is no longer available for online streaming.'
                 else:
                     message = redirect_info['message']
-                raise ExtractorError(message, expected=True)
+                raise ExtractorError('PBS said: %s' % message, expected=True)
 
             format_url = redirect_info.get('url')
             if not format_url:

From 0553d0ee40b6ecd62ed25ba8999aa2e4f7641590 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 8 Oct 2015 18:57:57 +0100
Subject: [PATCH 2725/6219] [pbs] place errors into a dict

---
 youtube_dl/extractor/pbs.py | 20 +++++++-------------
 1 file changed, 7 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 82218b933..814e97f48 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -39,7 +39,6 @@ class PBSIE(InfoExtractor):
             'params': {
                 'skip_download': True,  # requires ffmpeg
             },
-            'skip': 'Expired',
         },
         {
             'url': 'http://www.pbs.org/wgbh/pages/frontline/losing-iraq/',
@@ -156,6 +155,12 @@ class PBSIE(InfoExtractor):
             },
         }
     ]
+    _ERRORS = {
+        101: 'We\'re sorry, but this video is not yet available.',
+        403: 'We\'re sorry, but this video is not available in your region due to right restrictions.',
+        404: 'We are experiencing technical difficulties that are preventing us from playing the video at this time. Please check back again soon.',
+        410: 'This video has expired and is no longer available for online streaming.',
+    }
 
     def _extract_webpage(self, url):
         mobj = re.match(self._VALID_URL, url)
@@ -233,18 +238,7 @@ class PBSIE(InfoExtractor):
                 'Downloading %s video url info' % encoding_name)
 
             if redirect_info['status'] == 'error':
-                http_code = redirect_info['http_code']
-                if http_code == 403:
-                    message = 'We\'re sorry, but this video is not available in your region due to right restrictions.'
-                elif http_code == 101:
-                    message = 'We\'re sorry, but this video is not yet available.'
-                elif http_code == 404:
-                    message = 'We are experiencing technical difficulties that are preventing us from playing the video at this time. Please check back again soon.'
-                elif http_code == 410:
-                    message = 'This video has expired and is no longer available for online streaming.'
-                else:
-                    message = redirect_info['message']
-                raise ExtractorError('PBS said: %s' % message, expected=True)
+                raise ExtractorError('PBS said: %s' % self._ERRORS.get(redirect_info['http_code'], redirect_info['message']), expected=True)
 
             format_url = redirect_info.get('url')
             if not format_url:

From 9d5fb3b58d95fbd8c28d9c0eaf4a652660324d9e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 9 Oct 2015 00:09:10 +0600
Subject: [PATCH 2726/6219] [pbs] Carry long line

---
 youtube_dl/extractor/pbs.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 814e97f48..3448736a2 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -238,7 +238,11 @@ class PBSIE(InfoExtractor):
                 'Downloading %s video url info' % encoding_name)
 
             if redirect_info['status'] == 'error':
-                raise ExtractorError('PBS said: %s' % self._ERRORS.get(redirect_info['http_code'], redirect_info['message']), expected=True)
+                raise ExtractorError(
+                    '%s said: %s' % (
+                        self.IE_NAME,
+                        self._ERRORS.get(redirect_info['http_code'], redirect_info['message'])),
+                    expected=True)
 
             format_url = redirect_info.get('url')
             if not format_url:

From 54a5428518adc4ddca085bdac471bc4f286024e4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 9 Oct 2015 03:54:49 +0600
Subject: [PATCH 2727/6219] [dailymotion] Update player v5 regex (Closes #7107)

---
 youtube_dl/extractor/dailymotion.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index 2d90b2224..80a05cfee 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -119,7 +119,7 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
             webpage, 'comment count', fatal=False))
 
         player_v5 = self._search_regex(
-            r'playerV5\s*=\s*dmp\.create\([^,]+?,\s*({.+?})\);',
+            [r'buildPlayer\(({.+?})\);', r'playerV5\s*=\s*dmp\.create\([^,]+?,\s*({.+?})\);'],
             webpage, 'player v5', default=None)
         if player_v5:
             player = self._parse_json(player_v5, video_id)

From 57935b2564c082b90a60468d4c844b219118886a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 9 Oct 2015 14:11:00 +0800
Subject: [PATCH 2728/6219] [extractor/common] Allow HTML5 unquoted attribute
 values

Fixes #7108

HTML5 allows unquoted attribute values. See the "Unquoted attribute value
syntax" section [1] for more information

[1] http://www.w3.org/TR/html5/syntax.html
---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index dbae75406..242618c58 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -646,7 +646,7 @@ class InfoExtractor(object):
     @staticmethod
     def _og_regexes(prop):
         content_re = r'content=(?:"([^>]+?)"|\'([^>]+?)\')'
-        property_re = r'(?:name|property)=[\'"]og:%s[\'"]' % re.escape(prop)
+        property_re = r'(?:name|property)=[\'"]?og:%s[\'"]?' % re.escape(prop)
         template = r'<meta[^>]+?%s[^>]+?%s'
         return [
             template % (property_re, content_re),

From ef47b2c15f3fa9d9d491090f2ea46d2bd9967d21 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 9 Oct 2015 09:09:22 +0200
Subject: [PATCH 2729/6219] release 2015.10.09

---
 CONTRIBUTING.md       | 2 +-
 youtube_dl/version.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 57a94231d..32c2fd84c 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -124,7 +124,7 @@ If you want to add support for a new site, you can follow this quick list (assum
             }
     ```
 5. Add an import in [`youtube_dl/extractor/__init__.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/__init__.py).
-6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will be then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc.
+6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc.
 7. Have a look at [`youtube_dl/extractor/common.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L62-L200). Add tests and code for as many as you want.
 8. If you can, check the code with [flake8](https://pypi.python.org/pypi/flake8).
 9. When the tests pass, [add](http://git-scm.com/docs/git-add) the new files and [commit](http://git-scm.com/docs/git-commit) them and [push](http://git-scm.com/docs/git-push) the result, like this:
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 112c78835..faae8a2d7 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.10.06.2'
+__version__ = '2015.10.09'

From 1ef1563649374568870e9334cce7055f7c83a817 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 9 Oct 2015 20:08:37 +0100
Subject: [PATCH 2730/6219] [srgssr] Add generic extractor for SRGSSR Group
 sites

---
 youtube_dl/extractor/__init__.py |   5 +-
 youtube_dl/extractor/srf.py      | 104 -------------------------
 youtube_dl/extractor/srgssr.py   | 130 +++++++++++++++++++++++++++++++
 3 files changed, 134 insertions(+), 105 deletions(-)
 delete mode 100644 youtube_dl/extractor/srf.py
 create mode 100644 youtube_dl/extractor/srgssr.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 3ace1cc2c..042ad3678 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -591,7 +591,10 @@ from .sportbox import (
     SportBoxEmbedIE,
 )
 from .sportdeutschland import SportDeutschlandIE
-from .srf import SrfIE
+from .srgssr import (
+    SRGSSRIE,
+    SRGSSRPlayIE,
+)
 from .srmediathek import SRMediathekIE
 from .ssa import SSAIE
 from .stanfordoc import StanfordOpenClassroomIE
diff --git a/youtube_dl/extractor/srf.py b/youtube_dl/extractor/srf.py
deleted file mode 100644
index 77eec0bc7..000000000
--- a/youtube_dl/extractor/srf.py
+++ /dev/null
@@ -1,104 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-import re
-from .common import InfoExtractor
-from ..utils import (
-    determine_ext,
-    parse_iso8601,
-    xpath_text,
-)
-
-
-class SrfIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.srf\.ch/play(?:er)?/tv/[^/]+/video/(?P<display_id>[^?]+)\?id=|tp\.srgssr\.ch/p/flash\?urn=urn:srf:ais:video:)(?P<id>[0-9a-f\-]{36})'
-    _TESTS = [{
-        'url': 'http://www.srf.ch/play/tv/10vor10/video/snowden-beantragt-asyl-in-russland?id=28e1a57d-5b76-4399-8ab3-9097f071e6c5',
-        'md5': '4cd93523723beff51bb4bee974ee238d',
-        'info_dict': {
-            'id': '28e1a57d-5b76-4399-8ab3-9097f071e6c5',
-            'display_id': 'snowden-beantragt-asyl-in-russland',
-            'ext': 'm4v',
-            'upload_date': '20130701',
-            'title': 'Snowden beantragt Asyl in Russland',
-            'timestamp': 1372713995,
-        }
-    }, {
-        # No Speichern (Save) button
-        'url': 'http://www.srf.ch/play/tv/top-gear/video/jaguar-xk120-shadow-und-tornado-dampflokomotive?id=677f5829-e473-4823-ac83-a1087fe97faa',
-        'md5': 'd97e236e80d1d24729e5d0953d276a4f',
-        'info_dict': {
-            'id': '677f5829-e473-4823-ac83-a1087fe97faa',
-            'display_id': 'jaguar-xk120-shadow-und-tornado-dampflokomotive',
-            'ext': 'flv',
-            'upload_date': '20130710',
-            'title': 'Jaguar XK120, Shadow und Tornado-Dampflokomotive',
-            'timestamp': 1373493600,
-        },
-    }, {
-        'url': 'http://www.srf.ch/player/tv/10vor10/video/snowden-beantragt-asyl-in-russland?id=28e1a57d-5b76-4399-8ab3-9097f071e6c5',
-        'only_matching': True,
-    }, {
-        'url': 'https://tp.srgssr.ch/p/flash?urn=urn:srf:ais:video:28e1a57d-5b76-4399-8ab3-9097f071e6c5',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        display_id = re.match(self._VALID_URL, url).group('display_id') or video_id
-
-        video_data = self._download_xml(
-            'http://il.srgssr.ch/integrationlayer/1.0/ue/srf/video/play/%s.xml' % video_id,
-            display_id)
-
-        title = xpath_text(
-            video_data, './AssetMetadatas/AssetMetadata/title', fatal=True)
-        thumbnails = [{
-            'url': s.text
-        } for s in video_data.findall('.//ImageRepresentation/url')]
-        timestamp = parse_iso8601(xpath_text(video_data, './createdDate'))
-        # The <duration> field in XML is different from the exact duration, skipping
-
-        formats = []
-        for item in video_data.findall('./Playlists/Playlist') + video_data.findall('./Downloads/Download'):
-            for url_node in item.findall('url'):
-                quality = url_node.attrib['quality']
-                full_url = url_node.text
-                original_ext = determine_ext(full_url)
-                format_id = '%s-%s' % (quality, item.attrib['protocol'])
-                if original_ext == 'f4m':
-                    formats.extend(self._extract_f4m_formats(
-                        full_url + '?hdcore=3.4.0', display_id, f4m_id=format_id))
-                elif original_ext == 'm3u8':
-                    formats.extend(self._extract_m3u8_formats(
-                        full_url, display_id, 'mp4', m3u8_id=format_id))
-                else:
-                    formats.append({
-                        'url': full_url,
-                        'ext': original_ext,
-                        'format_id': format_id,
-                        'quality': 0 if 'HD' in quality else -1,
-                        'preference': 1,
-                    })
-
-        self._sort_formats(formats)
-
-        subtitles = {}
-        subtitles_data = video_data.find('Subtitles')
-        if subtitles_data is not None:
-            subtitles_list = [{
-                'url': sub.text,
-                'ext': determine_ext(sub.text),
-            } for sub in subtitles_data]
-            if subtitles_list:
-                subtitles['de'] = subtitles_list
-
-        return {
-            'id': video_id,
-            'display_id': display_id,
-            'formats': formats,
-            'title': title,
-            'thumbnails': thumbnails,
-            'timestamp': timestamp,
-            'subtitles': subtitles,
-        }
diff --git a/youtube_dl/extractor/srgssr.py b/youtube_dl/extractor/srgssr.py
new file mode 100644
index 000000000..addf4d26e
--- /dev/null
+++ b/youtube_dl/extractor/srgssr.py
@@ -0,0 +1,130 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    parse_iso8601,
+    qualities,
+)
+
+
+class SRGSSRIE(InfoExtractor):
+    _VALID_URL = r'(?:https?://tp\.srgssr\.ch/p(?:/[^/]+)+\?urn=)?urn:(?P<bu>srf|rts|rsi|rtr|swi):(?:[^:]+:)?(?P<type>video|audio):(?P<id>[0-9a-f\-]{36}|\d+)'
+
+    _ERRORS = {
+        'AGERATING12': 'To protect children under the age of 12, this video is only available between 8 p.m. and 6 a.m.',
+        'AGERATING18': 'To protect children under the age of 18, this video is only available between 11 p.m. and 5 a.m.',
+#        'ENDDATE': 'For legal reasons, this video was only available for a specified period of time.',
+        'GEOBLOCK': 'For legal reasons, this video is only available in Switzerland.',
+        'LEGAL': 'The video cannot be transmitted for legal reasons.',
+        'STARTDATE': 'This video is not yet available. Please try again later.',
+    }
+
+    def _real_extract(self, url):
+        bu, media_type, media_id = re.match(self._VALID_URL, url).groups()
+
+        media_data = self._download_json(
+            'http://il.srgssr.ch/integrationlayer/1.0/ue/%s/%s/play/%s.json' % (bu, media_type, media_id),
+            media_id)[media_type.capitalize()]
+
+        if media_data.get('block') and media_data['block'] in self._ERRORS:
+            raise ExtractorError(
+                    '%s said: %s' % (
+                        self.IE_NAME,
+                        self._ERRORS[media_data['block']]),
+                    expected=True)
+
+        metadata = media_data['AssetMetadatas']['AssetMetadata'][0]
+        title = metadata['title']
+        description = metadata.get('description')
+        created_date = media_data.get('createdDate') or metadata.get('createdDate')
+        timestamp = parse_iso8601(created_date)
+
+        thumbnails = []
+        for image in media_data['Image']['ImageRepresentations']['ImageRepresentation']:
+            thumbnails.append({
+                'id': image.get('id'),
+                'url': image['url'],
+            })
+
+        preference = qualities(['LQ', 'MQ', 'SD', 'HQ', 'HD'])
+        formats = []
+        for source in media_data['Playlists']['Playlist']:
+            assets = {}
+            protocol = source.get('@protocol')
+            if protocol in ('HTTP-HDS', 'HTTP-HLS'):
+                for quality in source['url']:
+                    assets[quality['@quality']] = quality['text']
+                asset_url = assets.get('HD') or assets.get('HQ') or assets.get('SD') or assets.get('MQ') or assets.get('LQ')
+                if '.f4m' in asset_url:
+                    formats.extend(self._extract_f4m_formats(asset_url + '?hdcore=3.4.0', media_id, f4m_id='hds'))
+                elif '.m3u8' in asset_url:
+                    formats.extend(self._extract_m3u8_formats(asset_url, media_id, m3u8_id='hls'))
+            else:
+                for asset in source['url']:
+                    asset_url = asset['text']
+                    ext = None
+                    if asset_url.startswith('rtmp'):
+                        ext = self._search_regex(r'([a-z0-9]+):[^/]+', asset_url, 'ext')
+                    formats.append({
+                        'url': asset_url,
+                        'preference': preference(asset['@quality']),
+                        'ext': ext,
+                    })
+
+        downloads = media_data.get('Downloads')
+        if downloads:
+            for source in downloads['Download']:
+                for asset in source['url']:
+                    formats.append({
+                        'url': asset['text'],
+                        'preference': preference(asset['@quality'])
+                    })
+        self._sort_formats(formats)
+
+        return {
+            'id': media_id,
+            'title': title,
+            'description': description,
+            'timestamp': timestamp,
+            'thumbnails': thumbnails,
+            'formats': formats,
+        }
+
+
+class SRGSSRPlayIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:(?:www|play)\.)?(?P<bu>srf|rts|rsi|rtr|swi)\.ch/play/(?:tv|radio)/[^/]+/(?P<type>video|audio)/[^?]+\?id=(?P<id>[0-9a-f\-]{36}|\d+)'
+
+    _TESTS = [{
+        'url': 'http://www.srf.ch/play/tv/10vor10/video/snowden-beantragt-asyl-in-russland?id=28e1a57d-5b76-4399-8ab3-9097f071e6c5',
+        'md5': '4cd93523723beff51bb4bee974ee238d',
+        'info_dict': {
+            'id': '28e1a57d-5b76-4399-8ab3-9097f071e6c5',
+            'ext': 'm4v',
+            'upload_date': '20130701',
+            'title': 'Snowden beantragt Asyl in Russland',
+            'timestamp': 1372713995,
+        }
+    }, {
+        # No Speichern (Save) button
+        'url': 'http://www.srf.ch/play/tv/top-gear/video/jaguar-xk120-shadow-und-tornado-dampflokomotive?id=677f5829-e473-4823-ac83-a1087fe97faa',
+        'md5': '0a274ce38fda48c53c01890651985bc6',
+        'info_dict': {
+            'id': '677f5829-e473-4823-ac83-a1087fe97faa',
+            'ext': 'flv',
+            'upload_date': '20130710',
+            'title': 'Jaguar XK120, Shadow und Tornado-Dampflokomotive',
+            'description': 'md5:88604432b60d5a38787f152dec89cd56',
+            'timestamp': 1373493600,
+        },
+    }, {
+        'url': 'http://www.srf.ch/player/tv/10vor10/video/snowden-beantragt-asyl-in-russland?id=28e1a57d-5b76-4399-8ab3-9097f071e6c5',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        bu, media_type, media_id = re.match(self._VALID_URL, url).groups()
+        return self.url_result('urn:%s:%s:%s' % (bu, media_type, media_id), 'SRGSSR')

From 05ad5409b4fd044169ea0f67b9ae92d555564c4e Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 9 Oct 2015 20:34:03 +0100
Subject: [PATCH 2731/6219] [srgssr] fix regex for swissinfo.ch

---
 youtube_dl/extractor/srgssr.py | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/srgssr.py b/youtube_dl/extractor/srgssr.py
index addf4d26e..3b5dcc503 100644
--- a/youtube_dl/extractor/srgssr.py
+++ b/youtube_dl/extractor/srgssr.py
@@ -96,7 +96,7 @@ class SRGSSRIE(InfoExtractor):
 
 
 class SRGSSRPlayIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:(?:www|play)\.)?(?P<bu>srf|rts|rsi|rtr|swi)\.ch/play/(?:tv|radio)/[^/]+/(?P<type>video|audio)/[^?]+\?id=(?P<id>[0-9a-f\-]{36}|\d+)'
+    _VALID_URL = r'https?://(?:(?:www|play)\.)?(?P<bu>srf|rts|rsi|rtr|swissinfo)\.ch/play/(?:tv|radio)/[^/]+/(?P<type>video|audio)/[^?]+\?id=(?P<id>[0-9a-f\-]{36}|\d+)'
 
     _TESTS = [{
         'url': 'http://www.srf.ch/play/tv/10vor10/video/snowden-beantragt-asyl-in-russland?id=28e1a57d-5b76-4399-8ab3-9097f071e6c5',
@@ -120,11 +120,8 @@ class SRGSSRPlayIE(InfoExtractor):
             'description': 'md5:88604432b60d5a38787f152dec89cd56',
             'timestamp': 1373493600,
         },
-    }, {
-        'url': 'http://www.srf.ch/player/tv/10vor10/video/snowden-beantragt-asyl-in-russland?id=28e1a57d-5b76-4399-8ab3-9097f071e6c5',
-        'only_matching': True,
     }]
 
     def _real_extract(self, url):
         bu, media_type, media_id = re.match(self._VALID_URL, url).groups()
-        return self.url_result('urn:%s:%s:%s' % (bu, media_type, media_id), 'SRGSSR')
+        return self.url_result('urn:%s:%s:%s' % (bu[:3], media_type, media_id), 'SRGSSR')

From 4180a3d8b7c40792b5371ca8804c1dad8fabd56d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 10 Oct 2015 01:44:33 +0600
Subject: [PATCH 2732/6219] [extractor/common] Allow quoteless content
 attribute in og regexes (Closes #7115)

---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 242618c58..0082a4c84 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -645,7 +645,7 @@ class InfoExtractor(object):
     # Helper functions for extracting OpenGraph info
     @staticmethod
     def _og_regexes(prop):
-        content_re = r'content=(?:"([^>]+?)"|\'([^>]+?)\')'
+        content_re = r'content=(?:"([^>]+?)"|\'([^>]+?)\'|\s*([^\s"\'=<>`]+?))'
         property_re = r'(?:name|property)=[\'"]?og:%s[\'"]?' % re.escape(prop)
         template = r'<meta[^>]+?%s[^>]+?%s'
         return [

From 47c165c3a9f40dd9a175b11f29f9e2002fdda8a9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 10 Oct 2015 05:56:01 +0600
Subject: [PATCH 2733/6219] [vimeo] Fix authentication (Closes #7110)

---
 youtube_dl/extractor/vimeo.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 50df79ca1..7dd52627d 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -40,6 +40,9 @@ class VimeoBaseInfoExtractor(InfoExtractor):
         self.report_login()
         webpage = self._download_webpage(self._LOGIN_URL, None, False)
         token = self._extract_xsrft(webpage)
+        vuid = self._search_regex(
+            r'["\']vuid["\']\s*:\s*(["\'])(?P<vuid>.+?)\1',
+            webpage, 'vuid', group='vuid')
         data = urlencode_postdata({
             'action': 'login',
             'email': username,
@@ -49,6 +52,7 @@ class VimeoBaseInfoExtractor(InfoExtractor):
         })
         login_request = compat_urllib_request.Request(self._LOGIN_URL, data)
         login_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
+        login_request.add_header('Cookie', 'vuid=%s' % vuid)
         login_request.add_header('Referer', self._LOGIN_URL)
         self._download_webpage(login_request, None, False, 'Wrong login info')
 

From 6a959f2e5266dcc6037a33589eb34fad9190c63e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 10 Oct 2015 15:03:01 +0800
Subject: [PATCH 2734/6219] [iqiyi] Update enc_key

---
 youtube_dl/extractor/iqiyi.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index ce1ab3820..0e53cb154 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -191,7 +191,7 @@ class IqiyiIE(InfoExtractor):
             'vid': video_id,
             'vinfo': 1,
             'tm': tm,
-            'enc': self.md5_text((enc_key + tail)[1:64:2] + tail),
+            'enc': self.md5_text(enc_key + tail),
             'qyid': _uuid,
             'tn': random.random(),
             'um': 0,
@@ -205,7 +205,9 @@ class IqiyiIE(InfoExtractor):
 
     def get_enc_key(self, swf_url, video_id):
         # TODO: automatic key extraction
-        enc_key = 'eac64f22daf001da6ba9aa8da4d501508bbe90a4d4091fea3b0582a85b38c2cc'  # last update at 2015-09-23-23 for Zombie::bite
+        # last update at 2015-10-10 for Zombie::bite
+        # '7239670519b6ac209a0bee4ef0446a6b24894b8ac2751506e42116212a0d0272e505'[2:66][1::2]
+        enc_key = '97596c0abee04ab49ba25564161ad225'
         return enc_key
 
     def _real_extract(self, url):

From dab062fb6ecd48e0c243a6d030d89b44cd44bd84 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 10 Oct 2015 20:34:06 +0600
Subject: [PATCH 2735/6219] [bbc] Add support for videos in news articles
 embedded with data-playable

---
 youtube_dl/extractor/bbc.py | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index cc2f6fed2..b2b39ff21 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -11,6 +11,7 @@ from ..utils import (
     int_or_none,
     parse_duration,
     parse_iso8601,
+    unescapeHTML,
 )
 from ..compat import compat_HTTPError
 
@@ -682,6 +683,21 @@ class BBCIE(BBCCoUkIE):
             [r'data-video-player-vpid="([\da-z]{8})"',
              r'<param[^>]+name="externalIdentifier"[^>]+value="([\da-z]{8})"'],
             webpage, 'vpid', default=None)
+
+        duration = None
+        if not programme_id:
+            # single video in news article embedded with data-playable (e.g.
+            # http://www.bbc.com/news/world-us-canada-34473351)
+            data_playable = self._parse_json(
+                unescapeHTML(self._search_regex(
+                    r'data-playable="({.+?})"', webpage, 'data playable', default='{}')),
+                programme_id, fatal=False)
+            if data_playable:
+                items = data_playable.get('settings', {}).get('playlistObject', {}).get('items')
+                if items and isinstance(items, list):
+                    duration = int_or_none(items[0].get('duration'))
+                    programme_id = items[0].get('vpid')
+
         if programme_id:
             formats, subtitles = self._download_media_selector(programme_id)
             self._sort_formats(formats)
@@ -699,6 +715,7 @@ class BBCIE(BBCCoUkIE):
                 'title': title,
                 'description': description,
                 'timestamp': timestamp,
+                'duration': duration,
                 'formats': formats,
                 'subtitles': subtitles,
             }

From de66571371e2a9705ecd3aed903880c4846d04ff Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 10 Oct 2015 20:40:56 +0600
Subject: [PATCH 2736/6219] [bbc] Support multiple videos in articles embedded
 with playlist.sxml

---
 youtube_dl/extractor/bbc.py | 38 ++++++++++++++++++++-----------------
 1 file changed, 21 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index b2b39ff21..930637cd7 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -660,23 +660,27 @@ class BBCIE(BBCCoUkIE):
              r'itemprop="datePublished"[^>]+datetime="([^"]+)"'],
             webpage, 'date', default=None))
 
-        # single video with playlist.sxml URL (e.g. http://www.bbc.com/sport/0/football/3365340ng)
-        playlist = self._search_regex(
-            r'<param[^>]+name="playlist"[^>]+value="([^"]+)"',
-            webpage, 'playlist', default=None)
-        if playlist:
-            programme_id, title, description, duration, formats, subtitles = \
-                self._process_legacy_playlist_url(playlist, playlist_id)
-            self._sort_formats(formats)
-            return {
-                'id': programme_id,
-                'title': title,
-                'description': description,
-                'duration': duration,
-                'timestamp': timestamp,
-                'formats': formats,
-                'subtitles': subtitles,
-            }
+        # article with multiple videos embedded with playlist.sxml (e.g.
+        # http://www.bbc.com/sport/0/football/34475836)
+        playlists = re.findall(r'<param[^>]+name="playlist"[^>]+value="([^"]+)"', webpage)
+        if playlists:
+            entries = []
+            for playlist in playlists:
+                programme_id, title, description, duration, formats, subtitles = \
+                    self._process_legacy_playlist_url(playlist, playlist_id)
+                self._sort_formats(formats)
+                entries.append({
+                    'id': programme_id,
+                    'title': title,
+                    'description': description,
+                    'duration': duration,
+                    'timestamp': timestamp,
+                    'formats': formats,
+                    'subtitles': subtitles,
+                })
+            playlist_title = self._og_search_title(webpage)
+            playlist_description = self._og_search_description(webpage)
+            return self.playlist_result(entries, playlist_id, playlist_title, playlist_description)
 
         # single video story (e.g. http://www.bbc.com/travel/story/20150625-sri-lankas-spicy-secret)
         programme_id = self._search_regex(

From b5d48cb1ef741ef999d038a3bc80943b1ae37a50 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 10 Oct 2015 20:55:46 +0600
Subject: [PATCH 2737/6219] [bbc] Add test for atricle with multiple videos
 embedded with playlist.sxml

---
 youtube_dl/extractor/bbc.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 930637cd7..ffe2afa96 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -608,6 +608,14 @@ class BBCIE(BBCCoUkIE):
             # rtmp download
             'skip_download': True,
         }
+    }, {
+        # article with multiple videos embedded with playlist.sxml
+        'url': 'http://www.bbc.com/sport/0/football/34475836',
+        'info_dict': {
+            'id': '34475836',
+            'title': 'What Liverpool can expect from Klopp',
+        },
+        'playlist_count': 3,
     }, {
         # single video with playlist URL from weather section
         'url': 'http://www.bbc.com/weather/features/33601775',

From 975977860d1880bbf5e9996ad77fb7f67622ccbe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 10 Oct 2015 20:56:07 +0600
Subject: [PATCH 2738/6219] [bbc] Make playlist title optional

---
 youtube_dl/extractor/bbc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index ffe2afa96..f943b5fd1 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -687,7 +687,7 @@ class BBCIE(BBCCoUkIE):
                     'subtitles': subtitles,
                 })
             playlist_title = self._og_search_title(webpage)
-            playlist_description = self._og_search_description(webpage)
+            playlist_description = self._og_search_description(webpage, default=None)
             return self.playlist_result(entries, playlist_id, playlist_title, playlist_description)
 
         # single video story (e.g. http://www.bbc.com/travel/story/20150625-sri-lankas-spicy-secret)

From baf39a1aa8af38ca295e5efdad277ddee5b11eb5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 10 Oct 2015 21:32:27 +0600
Subject: [PATCH 2739/6219] [bbc] Add one more scenario for data-playable
 embeds

---
 youtube_dl/extractor/bbc.py | 53 ++++++++++++++++++++++++-------------
 1 file changed, 35 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index f943b5fd1..df61f0157 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -657,6 +657,20 @@ class BBCIE(BBCCoUkIE):
 
         return [], []
 
+    def _extract_from_playlist_sxml(self, url, playlist_id, timestamp):
+        programme_id, title, description, duration, formats, subtitles = \
+            self._process_legacy_playlist_url(url, playlist_id)
+        self._sort_formats(formats)
+        return {
+            'id': programme_id,
+            'title': title,
+            'description': description,
+            'duration': duration,
+            'timestamp': timestamp,
+            'formats': formats,
+            'subtitles': subtitles,
+        }
+
     def _real_extract(self, url):
         playlist_id = self._match_id(url)
 
@@ -672,20 +686,9 @@ class BBCIE(BBCCoUkIE):
         # http://www.bbc.com/sport/0/football/34475836)
         playlists = re.findall(r'<param[^>]+name="playlist"[^>]+value="([^"]+)"', webpage)
         if playlists:
-            entries = []
-            for playlist in playlists:
-                programme_id, title, description, duration, formats, subtitles = \
-                    self._process_legacy_playlist_url(playlist, playlist_id)
-                self._sort_formats(formats)
-                entries.append({
-                    'id': programme_id,
-                    'title': title,
-                    'description': description,
-                    'duration': duration,
-                    'timestamp': timestamp,
-                    'formats': formats,
-                    'subtitles': subtitles,
-                })
+            entries = [
+                self._extract_from_playlist_sxml(playlist_url, playlist_id, timestamp)
+                for playlist_url in playlists]
             playlist_title = self._og_search_title(webpage)
             playlist_description = self._og_search_description(webpage, default=None)
             return self.playlist_result(entries, playlist_id, playlist_title, playlist_description)
@@ -705,10 +708,24 @@ class BBCIE(BBCCoUkIE):
                     r'data-playable="({.+?})"', webpage, 'data playable', default='{}')),
                 programme_id, fatal=False)
             if data_playable:
-                items = data_playable.get('settings', {}).get('playlistObject', {}).get('items')
-                if items and isinstance(items, list):
-                    duration = int_or_none(items[0].get('duration'))
-                    programme_id = items[0].get('vpid')
+                # data-playable has video vpid in settings.playlistObject.items (e.g.
+                # http://www.bbc.com/news/world-us-canada-34473351)
+                settings = data_playable.get('settings', {})
+                if settings:
+                    playlist_object = settings.get('playlistObject', {})
+                    if playlist_object:
+                        items = playlist_object.get('items')
+                        if items and isinstance(items, list):
+                            duration = int_or_none(items[0].get('duration'))
+                            programme_id = items[0].get('vpid')
+                if not programme_id:
+                    # data-playable has no vpid but has a playlist.sxml URLs
+                    # in otherSettings.playlist (e.g.
+                    # http://www.bbc.com/turkce/multimedya/2015/10/151010_vid_ankara_patlama_ani)
+                    playlist = data_playable.get('otherSettings', {}).get('playlist', {})
+                    if playlist:
+                        return self._extract_from_playlist_sxml(
+                            playlist.get('progressiveDownloadUrl'), playlist_id, timestamp)
 
         if programme_id:
             formats, subtitles = self._download_media_selector(programme_id)

From 87cc0fbd18345117926d27db7312cba663d0f5ad Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 10 Oct 2015 21:35:20 +0600
Subject: [PATCH 2740/6219] [bbc] Make summary optional in legacy playlist

---
 youtube_dl/extractor/bbc.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index df61f0157..911ccd757 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -401,7 +401,8 @@ class BBCCoUkIE(InfoExtractor):
             if kind != 'programme' and kind != 'radioProgramme':
                 continue
             title = playlist.find('./{http://bbc.co.uk/2008/emp/playlist}title').text
-            description = playlist.find('./{http://bbc.co.uk/2008/emp/playlist}summary').text
+            description_el = playlist.find('./{http://bbc.co.uk/2008/emp/playlist}summary')
+            description = description_el.text if description_el else None
 
             def get_programme_id(item):
                 def get_from_attributes(item):

From 97ae4d166c43b7e5068a7563bc5fbce385d86c2b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ph=E1=BA=A1m=20Ng=E1=BB=8Dc=20Quang=20Nam?=
 <nampnq@gmail.com>
Date: Sat, 10 Oct 2015 22:07:06 +0700
Subject: [PATCH 2741/6219] [zingmp3:album] Add support for playlists

Update for work with playlist, it same album but different url
Ex:
http://mp3.zing.vn/playlist/Duong-Hong-Loan-apollobee/IWCAACCB.html
http://mp3.zing.vn/album/Duong-Hong-Loan-apollobee/IWCAACCB.html
---
 youtube_dl/extractor/zingmp3.py | 24 +++++++++++++++---------
 1 file changed, 15 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/zingmp3.py b/youtube_dl/extractor/zingmp3.py
index 7dc1e2f2b..42ac124f0 100644
--- a/youtube_dl/extractor/zingmp3.py
+++ b/youtube_dl/extractor/zingmp3.py
@@ -85,16 +85,22 @@ class ZingMp3SongIE(ZingMp3BaseInfoExtractor):
 
 
 class ZingMp3AlbumIE(ZingMp3BaseInfoExtractor):
-    _VALID_URL = r'https?://mp3\.zing\.vn/album/(?P<slug>[^/]+)/(?P<album_id>\w+)\.html'
-    _TESTS = [{
-        'url': 'http://mp3.zing.vn/album/Lau-Dai-Tinh-Ai-Bang-Kieu-Minh-Tuyet/ZWZBWDAF.html',
-        'info_dict': {
-            '_type': 'playlist',
-            'id': 'ZWZBWDAF',
-            'title': 'Lâu Đài Tình Ái - Bằng Kiều ft. Minh Tuyết | Album 320 lossless',
+    _VALID_URL = r'https?://mp3\.zing\.vn/(?:album|playlist)/(?P<slug>[^/]+)/(?P<album_id>\w+)\.html'
+    _TESTS = [
+        {
+            'url': 'http://mp3.zing.vn/album/Lau-Dai-Tinh-Ai-Bang-Kieu-Minh-Tuyet/ZWZBWDAF.html',
+            'info_dict': {
+                '_type': 'playlist',
+                'id': 'ZWZBWDAF',
+                'title': 'Lâu Đài Tình Ái - Bằng Kiều ft. Minh Tuyết | Album 320 lossless',
+            },
+            'playlist_count': 10,
         },
-        'playlist_count': 10,
-    }]
+        {
+            'url': 'http://mp3.zing.vn/playlist/Duong-Hong-Loan-apollobee/IWCAACCB.html',
+            'only_matching': True,
+        }
+    ]
     IE_NAME = 'zingmp3:album'
     IE_DESC = 'mp3.zing.vn albums'
 

From 43abd79950b7ba0c3b6e8a26b624e439036b1c7a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 10 Oct 2015 22:05:01 +0600
Subject: [PATCH 2742/6219] [zingmp3:album] Style

---
 youtube_dl/extractor/zingmp3.py | 25 +++++++++++--------------
 1 file changed, 11 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/zingmp3.py b/youtube_dl/extractor/zingmp3.py
index 42ac124f0..1059dd75f 100644
--- a/youtube_dl/extractor/zingmp3.py
+++ b/youtube_dl/extractor/zingmp3.py
@@ -86,21 +86,18 @@ class ZingMp3SongIE(ZingMp3BaseInfoExtractor):
 
 class ZingMp3AlbumIE(ZingMp3BaseInfoExtractor):
     _VALID_URL = r'https?://mp3\.zing\.vn/(?:album|playlist)/(?P<slug>[^/]+)/(?P<album_id>\w+)\.html'
-    _TESTS = [
-        {
-            'url': 'http://mp3.zing.vn/album/Lau-Dai-Tinh-Ai-Bang-Kieu-Minh-Tuyet/ZWZBWDAF.html',
-            'info_dict': {
-                '_type': 'playlist',
-                'id': 'ZWZBWDAF',
-                'title': 'Lâu Đài Tình Ái - Bằng Kiều ft. Minh Tuyết | Album 320 lossless',
-            },
-            'playlist_count': 10,
+    _TESTS = [{
+        'url': 'http://mp3.zing.vn/album/Lau-Dai-Tinh-Ai-Bang-Kieu-Minh-Tuyet/ZWZBWDAF.html',
+        'info_dict': {
+            '_type': 'playlist',
+            'id': 'ZWZBWDAF',
+            'title': 'Lâu Đài Tình Ái - Bằng Kiều ft. Minh Tuyết | Album 320 lossless',
         },
-        {
-            'url': 'http://mp3.zing.vn/playlist/Duong-Hong-Loan-apollobee/IWCAACCB.html',
-            'only_matching': True,
-        }
-    ]
+        'playlist_count': 10,
+    }, {
+        'url': 'http://mp3.zing.vn/playlist/Duong-Hong-Loan-apollobee/IWCAACCB.html',
+        'only_matching': True,
+    }]
     IE_NAME = 'zingmp3:album'
     IE_DESC = 'mp3.zing.vn albums'
 

From 8119597d6ff52a671fa359378dfbdee13f24f882 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 10 Oct 2015 22:08:38 +0600
Subject: [PATCH 2743/6219] [zingmp3] Add fatal flag

---
 youtube_dl/extractor/zingmp3.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/zingmp3.py b/youtube_dl/extractor/zingmp3.py
index 1059dd75f..3f46f9049 100644
--- a/youtube_dl/extractor/zingmp3.py
+++ b/youtube_dl/extractor/zingmp3.py
@@ -9,9 +9,11 @@ from ..utils import ExtractorError
 
 class ZingMp3BaseInfoExtractor(InfoExtractor):
 
-    def _extract_item(self, item):
+    def _extract_item(self, item, fatal=True):
         error_message = item.find('./errormessage').text
         if error_message:
+            if not fatal:
+                return
             raise ExtractorError(
                 '%s returned error: %s' % (self.IE_NAME, error_message),
                 expected=True)

From f790c43f6e33782c6a6eaaff68bbd30aad875efe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 10 Oct 2015 22:09:21 +0600
Subject: [PATCH 2744/6219] [zingmp3:album] Skip broken items

---
 youtube_dl/extractor/zingmp3.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/zingmp3.py b/youtube_dl/extractor/zingmp3.py
index 3f46f9049..437eecb67 100644
--- a/youtube_dl/extractor/zingmp3.py
+++ b/youtube_dl/extractor/zingmp3.py
@@ -45,7 +45,9 @@ class ZingMp3BaseInfoExtractor(InfoExtractor):
             entries = []
 
             for i, item in enumerate(items, 1):
-                entry = self._extract_item(item)
+                entry = self._extract_item(item, fatal=False)
+                if not entry:
+                    continue
                 entry['id'] = '%s-%d' % (id, i)
                 entries.append(entry)
 

From e6174ee9753e044bdeac22c653f5762376cadbac Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 10 Oct 2015 23:01:20 +0600
Subject: [PATCH 2745/6219] [bbc] Extract legacy playlist embedded media

---
 youtube_dl/extractor/bbc.py | 48 ++++++++++++++++++++++++++-----------
 1 file changed, 34 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 911ccd757..7bbbc0b90 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -29,6 +29,14 @@ class BBCCoUkIE(InfoExtractor):
         'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/pc/vpid/%s',
     ]
 
+    _MEDIASELECTION_NS = 'http://bbc.co.uk/2008/mp/mediaselection'
+    _EMP_PLAYLIST_NS = 'http://bbc.co.uk/2008/emp/playlist'
+
+    _NAMESPACES = (
+        _MEDIASELECTION_NS,
+        _EMP_PLAYLIST_NS,
+    )
+
     _TESTS = [
         {
             'url': 'http://www.bbc.co.uk/programmes/b039g8p7',
@@ -194,6 +202,7 @@ class BBCCoUkIE(InfoExtractor):
 
     def _extract_connection(self, connection, programme_id):
         formats = []
+        kind = connection.get('kind')
         protocol = connection.get('protocol')
         supplier = connection.get('supplier')
         if protocol == 'http':
@@ -219,7 +228,7 @@ class BBCCoUkIE(InfoExtractor):
             else:
                 formats.append({
                     'url': href,
-                    'format_id': supplier,
+                    'format_id': supplier or kind or protocol,
                 })
         elif protocol == 'rtmp':
             application = connection.get('application', 'ondemand')
@@ -239,16 +248,24 @@ class BBCCoUkIE(InfoExtractor):
         return formats
 
     def _extract_items(self, playlist):
-        return playlist.findall('./{http://bbc.co.uk/2008/emp/playlist}item')
+        return playlist.findall('./{%s}item' % self._EMP_PLAYLIST_NS)
+
+    def _findall_ns(self, element, xpath):
+        elements = []
+        for ns in self._NAMESPACES:
+            elements.extend(element.findall(xpath % ns))
+        return elements
 
     def _extract_medias(self, media_selection):
-        error = media_selection.find('./{http://bbc.co.uk/2008/mp/mediaselection}error')
+        error = media_selection.find('./{%s}error' % self._MEDIASELECTION_NS)
+        if error is None:
+            media_selection.find('./{%s}error' % self._EMP_PLAYLIST_NS)
         if error is not None:
             raise BBCCoUkIE.MediaSelectionError(error.get('id'))
-        return media_selection.findall('./{http://bbc.co.uk/2008/mp/mediaselection}media')
+        return self._findall_ns(media_selection, './{%s}media')
 
     def _extract_connections(self, media):
-        return media.findall('./{http://bbc.co.uk/2008/mp/mediaselection}connection')
+        return self._findall_ns(media, './{%s}connection')
 
     def _extract_video(self, media, programme_id):
         formats = []
@@ -262,13 +279,14 @@ class BBCCoUkIE(InfoExtractor):
             conn_formats = self._extract_connection(connection, programme_id)
             for format in conn_formats:
                 format.update({
-                    'format_id': '%s_%s' % (service, format['format_id']),
                     'width': width,
                     'height': height,
                     'vbr': vbr,
                     'vcodec': vcodec,
                     'filesize': file_size,
                 })
+                if service:
+                    format['format_id'] = '%s_%s' % (service, format['format_id'])
             formats.extend(conn_formats)
         return formats
 
@@ -383,7 +401,7 @@ class BBCCoUkIE(InfoExtractor):
             url, playlist_id, 'Downloading legacy playlist XML')
 
     def _extract_from_legacy_playlist(self, playlist, playlist_id):
-        no_items = playlist.find('./{http://bbc.co.uk/2008/emp/playlist}noItems')
+        no_items = playlist.find('./{%s}noItems' % self._EMP_PLAYLIST_NS)
         if no_items is not None:
             reason = no_items.get('reason')
             if reason == 'preAvailability':
@@ -400,8 +418,8 @@ class BBCCoUkIE(InfoExtractor):
             kind = item.get('kind')
             if kind != 'programme' and kind != 'radioProgramme':
                 continue
-            title = playlist.find('./{http://bbc.co.uk/2008/emp/playlist}title').text
-            description_el = playlist.find('./{http://bbc.co.uk/2008/emp/playlist}summary')
+            title = playlist.find('./{%s}title' % self._EMP_PLAYLIST_NS).text
+            description_el = playlist.find('./{%s}summary' % self._EMP_PLAYLIST_NS)
             description = description_el.text if description_el else None
 
             def get_programme_id(item):
@@ -411,16 +429,18 @@ class BBCCoUkIE(InfoExtractor):
                         if value and re.match(r'^[pb][\da-z]{7}$', value):
                             return value
                 get_from_attributes(item)
-                mediator = item.find('./{http://bbc.co.uk/2008/emp/playlist}mediator')
+                mediator = item.find('./{%s}mediator' % self._EMP_PLAYLIST_NS)
                 if mediator is not None:
                     return get_from_attributes(mediator)
 
             programme_id = get_programme_id(item)
             duration = int_or_none(item.get('duration'))
-            # TODO: programme_id can be None and media items can be incorporated right inside
-            # playlist's item (e.g. http://www.bbc.com/turkce/haberler/2015/06/150615_telabyad_kentin_cogu)
-            # as f4m and m3u8
-            formats, subtitles = self._download_media_selector(programme_id)
+
+            if programme_id:
+                formats, subtitles = self._download_media_selector(programme_id)
+            else:
+                formats, subtitles = self._process_media_selector(item, playlist_id)
+                programme_id = playlist_id
 
         return programme_id, title, description, duration, formats, subtitles
 

From c936d8cc7b247f79dce2f5749dea5c18a08858fb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 10 Oct 2015 23:14:25 +0600
Subject: [PATCH 2746/6219] [bbc] Add another test

---
 youtube_dl/extractor/bbc.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 7bbbc0b90..0a08a57bd 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -559,6 +559,19 @@ class BBCIE(BBCCoUkIE):
         'params': {
             'skip_download': True,
         }
+    }, {
+        # single video embedded with playlist.sxml in data-playable
+        'url': 'http://www.bbc.com/turkce/multimedya/2015/10/151010_vid_ankara_patlama_ani',
+        'info_dict': {
+            'id': '151010_vid_ankara_patlama_ani',
+            'ext': 'mp4',
+            'title': "Ankara'da patlama anı",
+            'timestamp': 1444480325,
+            'upload_date': '20151010',
+        },
+        'params': {
+            'skip_download': True,
+        }
     }, {
         # single video embedded with mediaAssetPage.init() (regional section)
         'url': 'http://www.bbc.com/mundo/video_fotos/2015/06/150619_video_honduras_militares_hospitales_corrupcion_aw',

From b1ec70e4a9fc53b0ec583f48a5262c9f864db40b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 10 Oct 2015 23:20:30 +0600
Subject: [PATCH 2747/6219] [bbc] Improve data-playable regex

---
 youtube_dl/extractor/bbc.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 0a08a57bd..972abd0d4 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -739,7 +739,8 @@ class BBCIE(BBCCoUkIE):
             # http://www.bbc.com/news/world-us-canada-34473351)
             data_playable = self._parse_json(
                 unescapeHTML(self._search_regex(
-                    r'data-playable="({.+?})"', webpage, 'data playable', default='{}')),
+                    r'data-playable=(["\'])(?P<json>{.+?})\1', webpage,
+                    'data playable', default='{}', group='json')),
                 programme_id, fatal=False)
             if data_playable:
                 # data-playable has video vpid in settings.playlistObject.items (e.g.

From 78f9d843186977c614c5a0f6004732f5d410cd0c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 10 Oct 2015 23:40:20 +0600
Subject: [PATCH 2748/6219] [bbc] Support playlists of data-playable

---
 youtube_dl/extractor/bbc.py | 73 ++++++++++++++++++++++---------------
 1 file changed, 44 insertions(+), 29 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 972abd0d4..a15e67114 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -716,6 +716,8 @@ class BBCIE(BBCCoUkIE):
              r'itemprop="datePublished"[^>]+datetime="([^"]+)"'],
             webpage, 'date', default=None))
 
+        entries = []
+
         # article with multiple videos embedded with playlist.sxml (e.g.
         # http://www.bbc.com/sport/0/football/34475836)
         playlists = re.findall(r'<param[^>]+name="playlist"[^>]+value="([^"]+)"', webpage)
@@ -723,6 +725,48 @@ class BBCIE(BBCCoUkIE):
             entries = [
                 self._extract_from_playlist_sxml(playlist_url, playlist_id, timestamp)
                 for playlist_url in playlists]
+
+        # news article with multiple videos embedded with data-playable
+        data_playables = re.findall(r'data-playable=(["\'])({.+?})\1', webpage)
+        if data_playables:
+            for _, data_playable_json in data_playables:
+                data_playable = self._parse_json(
+                    unescapeHTML(data_playable_json), playlist_id, fatal=False)
+                if not data_playable:
+                    continue
+                settings = data_playable.get('settings', {})
+                if settings:
+                    # data-playable with video vpid in settings.playlistObject.items (e.g.
+                    # http://www.bbc.com/news/world-us-canada-34473351)
+                    playlist_object = settings.get('playlistObject', {})
+                    if playlist_object:
+                        items = playlist_object.get('items')
+                        if items and isinstance(items, list):
+                            title = playlist_object['title']
+                            description = playlist_object.get('summary')
+                            duration = int_or_none(items[0].get('duration'))
+                            programme_id = items[0].get('vpid')
+                            formats, subtitles = self._download_media_selector(programme_id)
+                            self._sort_formats(formats)
+                            entries.append({
+                                'id': programme_id,
+                                'title': title,
+                                'description': description,
+                                'timestamp': timestamp,
+                                'duration': duration,
+                                'formats': formats,
+                                'subtitles': subtitles,
+                            })
+                    else:
+                        # data-playable without vpid but with a playlist.sxml URLs
+                        # in otherSettings.playlist (e.g.
+                        # http://www.bbc.com/turkce/multimedya/2015/10/151010_vid_ankara_patlama_ani)
+                        playlist = data_playable.get('otherSettings', {}).get('playlist', {})
+                        if playlist:
+                            entries.append(self._extract_from_playlist_sxml(
+                                playlist.get('progressiveDownloadUrl'), playlist_id, timestamp))
+
+        if entries:
             playlist_title = self._og_search_title(webpage)
             playlist_description = self._og_search_description(webpage, default=None)
             return self.playlist_result(entries, playlist_id, playlist_title, playlist_description)
@@ -733,35 +777,6 @@ class BBCIE(BBCCoUkIE):
              r'<param[^>]+name="externalIdentifier"[^>]+value="([\da-z]{8})"'],
             webpage, 'vpid', default=None)
 
-        duration = None
-        if not programme_id:
-            # single video in news article embedded with data-playable (e.g.
-            # http://www.bbc.com/news/world-us-canada-34473351)
-            data_playable = self._parse_json(
-                unescapeHTML(self._search_regex(
-                    r'data-playable=(["\'])(?P<json>{.+?})\1', webpage,
-                    'data playable', default='{}', group='json')),
-                programme_id, fatal=False)
-            if data_playable:
-                # data-playable has video vpid in settings.playlistObject.items (e.g.
-                # http://www.bbc.com/news/world-us-canada-34473351)
-                settings = data_playable.get('settings', {})
-                if settings:
-                    playlist_object = settings.get('playlistObject', {})
-                    if playlist_object:
-                        items = playlist_object.get('items')
-                        if items and isinstance(items, list):
-                            duration = int_or_none(items[0].get('duration'))
-                            programme_id = items[0].get('vpid')
-                if not programme_id:
-                    # data-playable has no vpid but has a playlist.sxml URLs
-                    # in otherSettings.playlist (e.g.
-                    # http://www.bbc.com/turkce/multimedya/2015/10/151010_vid_ankara_patlama_ani)
-                    playlist = data_playable.get('otherSettings', {}).get('playlist', {})
-                    if playlist:
-                        return self._extract_from_playlist_sxml(
-                            playlist.get('progressiveDownloadUrl'), playlist_id, timestamp)
-
         if programme_id:
             formats, subtitles = self._download_media_selector(programme_id)
             self._sort_formats(formats)

From f6295bcb04c0e518369c31a44b691ad276fcc22f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 10 Oct 2015 23:45:15 +0600
Subject: [PATCH 2749/6219] [bbc] Remove duration

---
 youtube_dl/extractor/bbc.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index a15e67114..fad1f9ca7 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -794,7 +794,6 @@ class BBCIE(BBCCoUkIE):
                 'title': title,
                 'description': description,
                 'timestamp': timestamp,
-                'duration': duration,
                 'formats': formats,
                 'subtitles': subtitles,
             }

From 6a747190605229e9cfba5450cf0ecaf435b7a85e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 10 Oct 2015 23:56:55 +0600
Subject: [PATCH 2750/6219] [bbc] Update tests

---
 youtube_dl/extractor/bbc.py | 33 ++++++++++-----------------------
 1 file changed, 10 insertions(+), 23 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index fad1f9ca7..4eae4f52e 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -501,8 +501,7 @@ class BBCIE(BBCCoUkIE):
     ]
 
     _TESTS = [{
-        # article with multiple videos embedded with data-media-meta containing
-        # playlist.sxml, externalId and no direct video links
+        # article with multiple videos embedded with data-playable containing vpids
         'url': 'http://www.bbc.com/news/world-europe-32668511',
         'info_dict': {
             'id': 'world-europe-32668511',
@@ -511,7 +510,7 @@ class BBCIE(BBCCoUkIE):
         },
         'playlist_count': 2,
     }, {
-        # article with multiple videos embedded with data-media-meta (more videos)
+        # article with multiple videos embedded with data-playable (more videos)
         'url': 'http://www.bbc.com/news/business-28299555',
         'info_dict': {
             'id': 'business-28299555',
@@ -522,6 +521,7 @@ class BBCIE(BBCCoUkIE):
         'skip': 'Save time',
     }, {
         # article with multiple videos embedded with `new SMP()`
+        # broken
         'url': 'http://www.bbc.co.uk/blogs/adamcurtis/entries/3662a707-0af9-3149-963f-47bea720b460',
         'info_dict': {
             'id': '3662a707-0af9-3149-963f-47bea720b460',
@@ -529,7 +529,7 @@ class BBCIE(BBCCoUkIE):
         },
         'playlist_count': 18,
     }, {
-        # single video embedded with mediaAssetPage.init()
+        # single video embedded with data-playable containing vpid
         'url': 'http://www.bbc.com/news/world-europe-32041533',
         'info_dict': {
             'id': 'p02mprgb',
@@ -544,9 +544,9 @@ class BBCIE(BBCCoUkIE):
             'skip_download': True,
         }
     }, {
-        # article with single video embedded with data-media-meta containing
-        # direct video links (for now these are extracted) and playlist.xml (with
-        # media items as f4m and m3u8 - currently unsupported)
+        # article with single video embedded with data-playable containing XML playlist
+        # with direct video links as progressiveDownloadUrl (for now these are extracted)
+        # and playlist with f4m and m3u8 as streamingUrl
         'url': 'http://www.bbc.com/turkce/haberler/2015/06/150615_telabyad_kentin_cogu',
         'info_dict': {
             'id': '150615_telabyad_kentin_cogu',
@@ -560,20 +560,7 @@ class BBCIE(BBCCoUkIE):
             'skip_download': True,
         }
     }, {
-        # single video embedded with playlist.sxml in data-playable
-        'url': 'http://www.bbc.com/turkce/multimedya/2015/10/151010_vid_ankara_patlama_ani',
-        'info_dict': {
-            'id': '151010_vid_ankara_patlama_ani',
-            'ext': 'mp4',
-            'title': "Ankara'da patlama anı",
-            'timestamp': 1444480325,
-            'upload_date': '20151010',
-        },
-        'params': {
-            'skip_download': True,
-        }
-    }, {
-        # single video embedded with mediaAssetPage.init() (regional section)
+        # single video embedded with data-playable containing XML playlists (regional section)
         'url': 'http://www.bbc.com/mundo/video_fotos/2015/06/150619_video_honduras_militares_hospitales_corrupcion_aw',
         'info_dict': {
             'id': '150619_video_honduras_militares_hospitales_corrupcion_aw',
@@ -629,7 +616,7 @@ class BBCIE(BBCCoUkIE):
             'skip_download': True,
         }
     }, {
-        # single video with playlist.sxml URL
+        # single video with playlist.sxml URL in playlist param
         'url': 'http://www.bbc.com/sport/0/football/33653409',
         'info_dict': {
             'id': 'p02xycnp',
@@ -643,7 +630,7 @@ class BBCIE(BBCCoUkIE):
             'skip_download': True,
         }
     }, {
-        # article with multiple videos embedded with playlist.sxml
+        # article with multiple videos embedded with playlist.sxml in playlist param
         'url': 'http://www.bbc.com/sport/0/football/34475836',
         'info_dict': {
             'id': '34475836',

From ae8bdfd1a1548c83ab7df378096da927b5374a29 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 11 Oct 2015 00:25:09 +0600
Subject: [PATCH 2751/6219] [bbc] Extract article JSON and actualize tests

---
 youtube_dl/extractor/bbc.py | 45 ++++++++++++++++++++++++-------------
 1 file changed, 29 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 4eae4f52e..b98db95b9 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -11,6 +11,7 @@ from ..utils import (
     int_or_none,
     parse_duration,
     parse_iso8601,
+    remove_end,
     unescapeHTML,
 )
 from ..compat import compat_HTTPError
@@ -533,7 +534,7 @@ class BBCIE(BBCCoUkIE):
         'url': 'http://www.bbc.com/news/world-europe-32041533',
         'info_dict': {
             'id': 'p02mprgb',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': 'Aerial footage showed the site of the crash in the Alps - courtesy BFM TV',
             'duration': 47,
             'timestamp': 1427219242,
@@ -552,7 +553,6 @@ class BBCIE(BBCCoUkIE):
             'id': '150615_telabyad_kentin_cogu',
             'ext': 'mp4',
             'title': "YPG: Tel Abyad'ın tamamı kontrolümüzde",
-            'duration': 47,
             'timestamp': 1434397334,
             'upload_date': '20150615',
         },
@@ -566,7 +566,6 @@ class BBCIE(BBCCoUkIE):
             'id': '150619_video_honduras_militares_hospitales_corrupcion_aw',
             'ext': 'mp4',
             'title': 'Honduras militariza sus hospitales por nuevo escándalo de corrupción',
-            'duration': 87,
             'timestamp': 1434713142,
             'upload_date': '20150619',
         },
@@ -578,7 +577,7 @@ class BBCIE(BBCCoUkIE):
         'url': 'http://www.bbc.com/news/video_and_audio/must_see/33376376',
         'info_dict': {
             'id': 'p02w6qjc',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': '''Judge Mindy Glazer: "I'm sorry to see you here... I always wondered what happened to you"''',
             'duration': 56,
         },
@@ -605,11 +604,11 @@ class BBCIE(BBCCoUkIE):
         'url': 'http://www.bbc.com/autos/story/20130513-hyundais-rock-star',
         'info_dict': {
             'id': 'p018zqqg',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': 'Hyundai Santa Fe Sport: Rock star',
             'description': 'md5:b042a26142c4154a6e472933cf20793d',
-            'timestamp': 1368473503,
-            'upload_date': '20130513',
+            'timestamp': 1415867444,
+            'upload_date': '20141113',
         },
         'params': {
             # rtmp download
@@ -620,9 +619,8 @@ class BBCIE(BBCCoUkIE):
         'url': 'http://www.bbc.com/sport/0/football/33653409',
         'info_dict': {
             'id': 'p02xycnp',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': 'Transfers: Cristiano Ronaldo to Man Utd, Arsenal to spend?',
-            'description': 'md5:398fca0e2e701c609d726e034fa1fc89',
             'duration': 140,
         },
         'params': {
@@ -697,11 +695,26 @@ class BBCIE(BBCCoUkIE):
 
         webpage = self._download_webpage(url, playlist_id)
 
-        timestamp = parse_iso8601(self._search_regex(
-            [r'"datePublished":\s*"([^"]+)',
-             r'<meta[^>]+property="article:published_time"[^>]+content="([^"]+)"',
-             r'itemprop="datePublished"[^>]+datetime="([^"]+)"'],
-            webpage, 'date', default=None))
+        timestamp = None
+        playlist_title = None
+        playlist_description = None
+
+        ld = self._parse_json(
+            self._search_regex(
+                r'(?s)<script type="application/ld\+json">(.+?)</script>',
+                webpage, 'ld json', default='{}'),
+            playlist_id, fatal=False)
+        if ld:
+            timestamp = parse_iso8601(ld.get('datePublished'))
+            playlist_title = ld.get('headline')
+            playlist_description = ld.get('articleBody')
+
+        if not timestamp:
+            timestamp = parse_iso8601(self._search_regex(
+                [r'<meta[^>]+property="article:published_time"[^>]+content="([^"]+)"',
+                 r'itemprop="datePublished"[^>]+datetime="([^"]+)"',
+                 r'"datePublished":\s*"([^"]+)',],
+                webpage, 'date', default=None))
 
         entries = []
 
@@ -754,8 +767,8 @@ class BBCIE(BBCCoUkIE):
                                 playlist.get('progressiveDownloadUrl'), playlist_id, timestamp))
 
         if entries:
-            playlist_title = self._og_search_title(webpage)
-            playlist_description = self._og_search_description(webpage, default=None)
+            playlist_title = playlist_title or remove_end(self._og_search_title(webpage), ' - BBC News')
+            playlist_description = playlist_description or self._og_search_description(webpage, default=None)
             return self.playlist_result(entries, playlist_id, playlist_title, playlist_description)
 
         # single video story (e.g. http://www.bbc.com/travel/story/20150625-sri-lankas-spicy-secret)

From 55ebae26f937a6a3a1fcb78d2f797972080f8b88 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 11 Oct 2015 00:37:39 +0600
Subject: [PATCH 2752/6219] [bbc] Prefer iptv-all mediaset

---
 youtube_dl/extractor/bbc.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index b98db95b9..f3ded3f9c 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -493,6 +493,9 @@ class BBCIE(BBCCoUkIE):
     _VALID_URL = r'https?://(?:www\.)?bbc\.(?:com|co\.uk)/(?:[^/]+/)+(?P<id>[^/#?]+)'
 
     _MEDIASELECTOR_URLS = [
+        # Provides HQ HLS streams but fails with geolocation in some cases when it's
+        # even not geo restricted at all
+        'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/iptv-all/vpid/%s',
         # Provides more formats, namely direct mp4 links, but fails on some videos with
         # notukerror for non UK (?) users (e.g.
         # http://www.bbc.com/travel/story/20150625-sri-lankas-spicy-secret)
@@ -534,8 +537,9 @@ class BBCIE(BBCCoUkIE):
         'url': 'http://www.bbc.com/news/world-europe-32041533',
         'info_dict': {
             'id': 'p02mprgb',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Aerial footage showed the site of the crash in the Alps - courtesy BFM TV',
+            'description': 'md5:2868290467291b37feda7863f7a83f54',
             'duration': 47,
             'timestamp': 1427219242,
             'upload_date': '20150324',
@@ -577,7 +581,7 @@ class BBCIE(BBCCoUkIE):
         'url': 'http://www.bbc.com/news/video_and_audio/must_see/33376376',
         'info_dict': {
             'id': 'p02w6qjc',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': '''Judge Mindy Glazer: "I'm sorry to see you here... I always wondered what happened to you"''',
             'duration': 56,
         },
@@ -604,7 +608,7 @@ class BBCIE(BBCCoUkIE):
         'url': 'http://www.bbc.com/autos/story/20130513-hyundais-rock-star',
         'info_dict': {
             'id': 'p018zqqg',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Hyundai Santa Fe Sport: Rock star',
             'description': 'md5:b042a26142c4154a6e472933cf20793d',
             'timestamp': 1415867444,
@@ -619,7 +623,7 @@ class BBCIE(BBCCoUkIE):
         'url': 'http://www.bbc.com/sport/0/football/33653409',
         'info_dict': {
             'id': 'p02xycnp',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Transfers: Cristiano Ronaldo to Man Utd, Arsenal to spend?',
             'duration': 140,
         },

From 6f7893653c86c620099d7bf0e3bd4951be8b4ad1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 11 Oct 2015 00:39:28 +0600
Subject: [PATCH 2753/6219] [bbc] PEP 8

---
 youtube_dl/extractor/bbc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index f3ded3f9c..68995f81e 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -717,7 +717,7 @@ class BBCIE(BBCCoUkIE):
             timestamp = parse_iso8601(self._search_regex(
                 [r'<meta[^>]+property="article:published_time"[^>]+content="([^"]+)"',
                  r'itemprop="datePublished"[^>]+datetime="([^"]+)"',
-                 r'"datePublished":\s*"([^"]+)',],
+                 r'"datePublished":\s*"([^"]+)'],
                 webpage, 'date', default=None))
 
         entries = []

From f4076bb73686484310a3a2b7c0c4c20db6e186bc Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 10 Oct 2015 19:56:55 +0100
Subject: [PATCH 2754/6219] [vimeo] extract m3u8 manifest and bitrate

---
 youtube_dl/extractor/vimeo.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 7dd52627d..93638d6b2 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -397,8 +397,12 @@ class VimeoIE(VimeoBaseInfoExtractor):
                     'format_id': format_id,
                     'width': file_info.get('width'),
                     'height': file_info.get('height'),
+                    'tbr': file_info.get('bitrate'),
                 })
         formats = []
+        hls = config_files.get("hls")
+        if hls:
+            formats = self._extract_m3u8_formats(hls['all'], video_id, m3u8_id='hls')
         for key in ('other', 'sd', 'hd'):
             formats += files[key]
         if len(formats) == 0:

From 58cd7e173e70ae40a79bb10e08b2c2ea02bc8248 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 10 Oct 2015 12:28:12 +0100
Subject: [PATCH 2755/6219] [adultswim] detect when video needs authentication

---
 youtube_dl/extractor/adultswim.py | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/adultswim.py b/youtube_dl/extractor/adultswim.py
index 27de07587..0eb21b16d 100644
--- a/youtube_dl/extractor/adultswim.py
+++ b/youtube_dl/extractor/adultswim.py
@@ -41,7 +41,8 @@ class AdultSwimIE(InfoExtractor):
             'id': 'rQxZvXQ4ROaSOqq-or2Mow',
             'title': 'Rick and Morty - Pilot',
             'description': "Rick moves in with his daughter's family and establishes himself as a bad influence on his grandson, Morty. "
-        }
+        },
+        'skip': 'This video is only available for registered users',
     }, {
         'url': 'http://www.adultswim.com/videos/playlists/american-parenting/putting-francine-out-of-business/',
         'playlist': [
@@ -84,7 +85,10 @@ class AdultSwimIE(InfoExtractor):
     def find_video_info(collection, slug):
         for video in collection.get('videos'):
             if video.get('slug') == slug:
-                return video
+                if video.get('auth'):
+                    raise ExtractorError('This video is only available for registered users', expected=True)
+                else:
+                    return video
 
     @staticmethod
     def find_collection_by_linkURL(collections, linkURL):
@@ -97,7 +101,10 @@ class AdultSwimIE(InfoExtractor):
         for collection in collections:
             for video in collection.get('videos'):
                 if video.get('slug') == slug:
-                    return collection, video
+                    if video.get('auth'):
+                        raise ExtractorError('This video is only available for registered users', expected=True)
+                    else:
+                        return collection, video
         return None, None
 
     def _real_extract(self, url):
@@ -128,6 +135,8 @@ class AdultSwimIE(InfoExtractor):
             if video_info is None:
                 if bootstrapped_data.get('slugged_video', {}).get('slug') == episode_path:
                     video_info = bootstrapped_data['slugged_video']
+                    if video_info.get('auth'):
+                        raise ExtractorError('This video is only available for registered users', expected=True)
                 else:
                     raise ExtractorError('Unable to find video info')
 

From 00cde0b8dc3a534a929b88da086cc540dc212ea4 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 10 Oct 2015 17:57:05 +0100
Subject: [PATCH 2756/6219] [adultswim] raise ExtractorError if no clips in
 video_info

---
 youtube_dl/extractor/adultswim.py | 19 ++++++++-----------
 1 file changed, 8 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/adultswim.py b/youtube_dl/extractor/adultswim.py
index 0eb21b16d..35606878d 100644
--- a/youtube_dl/extractor/adultswim.py
+++ b/youtube_dl/extractor/adultswim.py
@@ -85,10 +85,7 @@ class AdultSwimIE(InfoExtractor):
     def find_video_info(collection, slug):
         for video in collection.get('videos'):
             if video.get('slug') == slug:
-                if video.get('auth'):
-                    raise ExtractorError('This video is only available for registered users', expected=True)
-                else:
-                    return video
+                return video
 
     @staticmethod
     def find_collection_by_linkURL(collections, linkURL):
@@ -101,10 +98,7 @@ class AdultSwimIE(InfoExtractor):
         for collection in collections:
             for video in collection.get('videos'):
                 if video.get('slug') == slug:
-                    if video.get('auth'):
-                        raise ExtractorError('This video is only available for registered users', expected=True)
-                    else:
-                        return collection, video
+                    return collection, video
         return None, None
 
     def _real_extract(self, url):
@@ -135,15 +129,18 @@ class AdultSwimIE(InfoExtractor):
             if video_info is None:
                 if bootstrapped_data.get('slugged_video', {}).get('slug') == episode_path:
                     video_info = bootstrapped_data['slugged_video']
-                    if video_info.get('auth'):
-                        raise ExtractorError('This video is only available for registered users', expected=True)
                 else:
                     raise ExtractorError('Unable to find video info')
 
             show = bootstrapped_data['show']
             show_title = show['title']
             stream = video_info.get('stream')
-            clips = [stream] if stream else video_info['clips']
+            clips = [stream] if stream else video_info.get('clips')
+            if not clips:
+                if video_info.get('auth'):
+                    raise ExtractorError('This video is only available for registered users', expected=True)
+                else:
+                    raise ExtractorError('Unable to find clips')
             segment_ids = [clip['videoPlaybackID'] for clip in clips]
 
         episode_id = video_info['id']

From 75f105d455bd0a2226ce4fa78b56c7a344ad6bd8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 11 Oct 2015 01:04:41 +0600
Subject: [PATCH 2757/6219] [adultswim] Clarify no media message

---
 youtube_dl/extractor/adultswim.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/adultswim.py b/youtube_dl/extractor/adultswim.py
index 35606878d..130afe791 100644
--- a/youtube_dl/extractor/adultswim.py
+++ b/youtube_dl/extractor/adultswim.py
@@ -137,10 +137,11 @@ class AdultSwimIE(InfoExtractor):
             stream = video_info.get('stream')
             clips = [stream] if stream else video_info.get('clips')
             if not clips:
-                if video_info.get('auth'):
-                    raise ExtractorError('This video is only available for registered users', expected=True)
-                else:
-                    raise ExtractorError('Unable to find clips')
+                raise ExtractorError(
+                    'This video is only available via cable service provider subscription that'
+                    ' is not currently supported. You may want to use --cookies.'
+                    if video_info.get('auth') is True else 'Unable to find stream or clips',
+                    expected=True)
             segment_ids = [clip['videoPlaybackID'] for clip in clips]
 
         episode_id = video_info['id']

From e5c209a1bcea206bee684914599c84acf886487c Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 10 Oct 2015 20:34:10 +0100
Subject: [PATCH 2758/6219] [vimeo] add parameters to _extract_m3u8_formats and
 sort formats

---
 youtube_dl/extractor/vimeo.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 93638d6b2..2ea5f0b79 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -402,11 +402,12 @@ class VimeoIE(VimeoBaseInfoExtractor):
         formats = []
         hls = config_files.get("hls")
         if hls:
-            formats = self._extract_m3u8_formats(hls['all'], video_id, m3u8_id='hls')
+            formats = self._extract_m3u8_formats(hls['all'], video_id, 'mp4', 'm3u8_native', 0, 'hls', fatal=False)
         for key in ('other', 'sd', 'hd'):
             formats += files[key]
         if len(formats) == 0:
             raise ExtractorError('No known codec found')
+        self._sort_formats(formats)
 
         subtitles = {}
         text_tracks = config['request'].get('text_tracks')

From fff496c689e44ac96909cf55c9ae746fb6b14e07 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 10 Oct 2015 20:45:34 +0100
Subject: [PATCH 2759/6219] [vimeo] remove check for empty formats

---
 youtube_dl/extractor/vimeo.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 2ea5f0b79..2051ac9de 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -405,8 +405,6 @@ class VimeoIE(VimeoBaseInfoExtractor):
             formats = self._extract_m3u8_formats(hls['all'], video_id, 'mp4', 'm3u8_native', 0, 'hls', fatal=False)
         for key in ('other', 'sd', 'hd'):
             formats += files[key]
-        if len(formats) == 0:
-            raise ExtractorError('No known codec found')
         self._sort_formats(formats)
 
         subtitles = {}

From 35a3ff1d337edd527c73db133d87ed23ca4469f5 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 10 Oct 2015 21:05:29 +0100
Subject: [PATCH 2760/6219] [vimeo] always convert width, height and bitrate to
 int

---
 youtube_dl/extractor/vimeo.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 2051ac9de..97590d220 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -395,14 +395,14 @@ class VimeoIE(VimeoBaseInfoExtractor):
                     'ext': codec_extension,
                     'url': video_url,
                     'format_id': format_id,
-                    'width': file_info.get('width'),
-                    'height': file_info.get('height'),
-                    'tbr': file_info.get('bitrate'),
+                    'width': int_or_none(file_info.get('width')),
+                    'height': int_or_none(file_info.get('height')),
+                    'tbr': int_or_none(file_info.get('bitrate')),
                 })
         formats = []
-        hls = config_files.get("hls")
-        if hls:
-            formats = self._extract_m3u8_formats(hls['all'], video_id, 'mp4', 'm3u8_native', 0, 'hls', fatal=False)
+        m3u8_url = config_files.get('hls', {}).get('all')
+        if m3u8_url:
+            formats = self._extract_m3u8_formats(m3u8_url, video_id, 'mp4', 'm3u8_native', 0, 'hls', fatal=False)
         for key in ('other', 'sd', 'hd'):
             formats += files[key]
         self._sort_formats(formats)

From 68f3b61f0e278891b5a0d6557d297dcfd3fb53cf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 11 Oct 2015 02:22:42 +0600
Subject: [PATCH 2761/6219] [vimeo] Update tests

---
 youtube_dl/extractor/vimeo.py | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 97590d220..9c173ce07 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -84,12 +84,12 @@ class VimeoIE(VimeoBaseInfoExtractor):
             'info_dict': {
                 'id': '56015672',
                 'ext': 'mp4',
-                "upload_date": "20121220",
-                "description": "This is a test case for youtube-dl.\nFor more information, see github.com/rg3/youtube-dl\nTest chars: \u2605 \" ' \u5e78 / \\ \u00e4 \u21ad \U0001d550",
-                "uploader_id": "user7108434",
-                "uploader": "Filippo Valsorda",
-                "title": "youtube-dl test video - \u2605 \" ' \u5e78 / \\ \u00e4 \u21ad \U0001d550",
-                "duration": 10,
+                'title': "youtube-dl test video - \u2605 \" ' \u5e78 / \\ \u00e4 \u21ad \U0001d550",
+                'description': 'md5:2d3305bad981a06ff79f027f19865021',
+                'upload_date': '20121220',
+                'uploader_id': 'user7108434',
+                'uploader': 'Filippo Valsorda',
+                'duration': 10,
             },
         },
         {
@@ -102,7 +102,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
                 'uploader_id': 'openstreetmapus',
                 'uploader': 'OpenStreetMap US',
                 'title': 'Andy Allan - Putting the Carto into OpenStreetMap Cartography',
-                'description': 'md5:380943ec71b89736ff4bf27183233d09',
+                'description': 'md5:fd69a7b8d8c34a4e1d2ec2e4afd6ec30',
                 'duration': 1595,
             },
         },
@@ -156,7 +156,6 @@ class VimeoIE(VimeoBaseInfoExtractor):
         },
         {
             'url': 'http://vimeo.com/76979871',
-            'md5': '3363dd6ffebe3784d56f4132317fd446',
             'note': 'Video with subtitles',
             'info_dict': {
                 'id': '76979871',

From 7c84562945cc5c37979b0d8d786118b1fe31f731 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 11 Oct 2015 02:31:37 +0600
Subject: [PATCH 2762/6219] [vimeo] Fix password protected videos

---
 youtube_dl/extractor/vimeo.py | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 9c173ce07..346eb60b4 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -39,10 +39,7 @@ class VimeoBaseInfoExtractor(InfoExtractor):
             return
         self.report_login()
         webpage = self._download_webpage(self._LOGIN_URL, None, False)
-        token = self._extract_xsrft(webpage)
-        vuid = self._search_regex(
-            r'["\']vuid["\']\s*:\s*(["\'])(?P<vuid>.+?)\1',
-            webpage, 'vuid', group='vuid')
+        token, vuid = self._extract_xsrft_and_vuid(webpage)
         data = urlencode_postdata({
             'action': 'login',
             'email': username,
@@ -56,10 +53,14 @@ class VimeoBaseInfoExtractor(InfoExtractor):
         login_request.add_header('Referer', self._LOGIN_URL)
         self._download_webpage(login_request, None, False, 'Wrong login info')
 
-    def _extract_xsrft(self, webpage):
-        return self._search_regex(
+    def _extract_xsrft_and_vuid(self, webpage):
+        xsrft = self._search_regex(
             r'xsrft\s*[=:]\s*(?P<q>["\'])(?P<xsrft>.+?)(?P=q)',
             webpage, 'login token', group='xsrft')
+        vuid = self._search_regex(
+            r'["\']vuid["\']\s*:\s*(["\'])(?P<vuid>.+?)\1',
+            webpage, 'vuid', group='vuid')
+        return xsrft, vuid
 
 
 class VimeoIE(VimeoBaseInfoExtractor):
@@ -201,7 +202,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
         password = self._downloader.params.get('videopassword', None)
         if password is None:
             raise ExtractorError('This video is protected by a password, use the --video-password option', expected=True)
-        token = self._extract_xsrft(webpage)
+        token, vuid = self._extract_xsrft_and_vuid(webpage)
         data = urlencode_postdata({
             'password': password,
             'token': token,
@@ -211,6 +212,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
             url = url.replace('http://', 'https://')
         password_request = compat_urllib_request.Request(url + '/password', data)
         password_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
+        password_request.add_header('Cookie', 'clip_v=1; vuid=%s' % vuid)
         password_request.add_header('Referer', url)
         return self._download_webpage(
             password_request, video_id,
@@ -465,7 +467,7 @@ class VimeoChannelIE(VimeoBaseInfoExtractor):
         if password is None:
             raise ExtractorError('This album is protected by a password, use the --video-password option', expected=True)
         fields = self._hidden_inputs(login_form)
-        token = self._extract_xsrft(webpage)
+        token, vuid = self._extract_xsrft_and_vuid(webpage)
         fields['token'] = token
         fields['password'] = password
         post = urlencode_postdata(fields)
@@ -474,6 +476,7 @@ class VimeoChannelIE(VimeoBaseInfoExtractor):
         password_url = compat_urlparse.urljoin(page_url, password_path)
         password_request = compat_urllib_request.Request(password_url, post)
         password_request.add_header('Content-type', 'application/x-www-form-urlencoded')
+        password_request.add_header('Cookie', 'vuid=%s' % vuid)
         self._set_cookie('vimeo.com', 'xsrft', token)
 
         return self._download_webpage(

From 70cb4d51c91c5b6c72b8564258cb161bcc68626c Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 10 Oct 2015 00:45:23 +0100
Subject: [PATCH 2763/6219] [bild] extract info from json request

---
 youtube_dl/extractor/bild.py | 19 ++++++++-----------
 1 file changed, 8 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/bild.py b/youtube_dl/extractor/bild.py
index 4d8cce1ef..ea84f20f2 100644
--- a/youtube_dl/extractor/bild.py
+++ b/youtube_dl/extractor/bild.py
@@ -4,7 +4,7 @@ from __future__ import unicode_literals
 from .common import InfoExtractor
 from ..utils import (
     int_or_none,
-    fix_xml_ampersands,
+    unescapeHTML,
 )
 
 
@@ -17,7 +17,7 @@ class BildIE(InfoExtractor):
         'info_dict': {
             'id': '38184146',
             'ext': 'mp4',
-            'title': 'BILD hat sie getestet',
+            'title': 'Das können die  neuen iPads ',
             'thumbnail': 're:^https?://.*\.jpg$',
             'duration': 196,
             'description': 'Mit dem iPad Air 2 und dem iPad Mini 3 hat Apple zwei neue Tablet-Modelle präsentiert. BILD-Reporter Sven Stein durfte die Geräte bereits testen. ',
@@ -27,16 +27,13 @@ class BildIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        xml_url = url.split(".bild.html")[0] + ",view=xml.bild.xml"
-        doc = self._download_xml(xml_url, video_id, transform_source=fix_xml_ampersands)
-
-        duration = int_or_none(doc.attrib.get('duration'), scale=1000)
+        video_data = self._download_json(url.split(".bild.html")[0] + ",view=json.bild.html", video_id)
 
         return {
             'id': video_id,
-            'title': doc.attrib['ueberschrift'],
-            'description': doc.attrib.get('text'),
-            'url': doc.attrib['src'],
-            'thumbnail': doc.attrib.get('img'),
-            'duration': duration,
+            'title': unescapeHTML(video_data['title']),
+            'description': unescapeHTML(video_data.get('description')),
+            'url': video_data['clipList'][0]['srces'][0]['src'],
+            'thumbnail': video_data.get('poster'),
+            'duration': int_or_none(video_data.get('durationSec')),
         }

From d8348c351d5ed4bb820b190e42b9a02e2dfc27a7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 11 Oct 2015 19:16:51 +0600
Subject: [PATCH 2764/6219] [bild] Strip title

---
 youtube_dl/extractor/bild.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/bild.py b/youtube_dl/extractor/bild.py
index ea84f20f2..1a0184861 100644
--- a/youtube_dl/extractor/bild.py
+++ b/youtube_dl/extractor/bild.py
@@ -17,21 +17,22 @@ class BildIE(InfoExtractor):
         'info_dict': {
             'id': '38184146',
             'ext': 'mp4',
-            'title': 'Das können die  neuen iPads ',
+            'title': 'Das können die  neuen iPads',
+            'description': 'md5:a4058c4fa2a804ab59c00d7244bbf62f',
             'thumbnail': 're:^https?://.*\.jpg$',
             'duration': 196,
-            'description': 'Mit dem iPad Air 2 und dem iPad Mini 3 hat Apple zwei neue Tablet-Modelle präsentiert. BILD-Reporter Sven Stein durfte die Geräte bereits testen. ',
         }
     }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        video_data = self._download_json(url.split(".bild.html")[0] + ",view=json.bild.html", video_id)
+        video_data = self._download_json(
+            url.split('.bild.html')[0] + ',view=json.bild.html', video_id)
 
         return {
             'id': video_id,
-            'title': unescapeHTML(video_data['title']),
+            'title': unescapeHTML(video_data['title']).strip(),
             'description': unescapeHTML(video_data.get('description')),
             'url': video_data['clipList'][0]['srces'][0]['src'],
             'thumbnail': video_data.get('poster'),

From 03e3b4e1198631b30914e8669b01bf1825a4385c Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 9 Oct 2015 23:36:31 +0100
Subject: [PATCH 2765/6219] [expotv] parse m3u8 manifest

---
 youtube_dl/extractor/expotv.py | 28 ++++++++++++++++------------
 1 file changed, 16 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/expotv.py b/youtube_dl/extractor/expotv.py
index a38b773e8..23a38c7c1 100644
--- a/youtube_dl/extractor/expotv.py
+++ b/youtube_dl/extractor/expotv.py
@@ -33,20 +33,24 @@ class ExpoTVIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
         player_key = self._search_regex(
             r'<param name="playerKey" value="([^"]+)"', webpage, 'player key')
-        config_url = 'http://client.expotv.com/video/config/%s/%s' % (
-            video_id, player_key)
         config = self._download_json(
-            config_url, video_id,
-            note='Downloading video configuration')
+                'http://client.expotv.com/video/config/%s/%s' % (video_id, player_key),
+                video_id,
+                note='Downloading video configuration')
 
-        formats = [{
-            'url': fcfg['file'],
-            'height': int_or_none(fcfg.get('height')),
-            'format_note': fcfg.get('label'),
-            'ext': self._search_regex(
-                r'filename=.*\.([a-z0-9_A-Z]+)&', fcfg['file'],
-                'file extension', default=None),
-        } for fcfg in config['sources']]
+        formats = []
+        for fcfg in config['sources']:
+            if fcfg['type'] == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(fcfg['file'], video_id))
+            else:
+                formats.append({
+                    'url': fcfg['file'],
+                    'height': int_or_none(fcfg.get('height')),
+                    'format_id': fcfg.get('label'),
+                    'ext': self._search_regex(
+                        r'filename=.*\.([a-z0-9_A-Z]+)&', fcfg['file'],
+                        'file extension', default=None),
+                })
         self._sort_formats(formats)
 
         title = self._og_search_title(webpage)

From 7d49502ab0990972bdb479e6bf32dbc8ffdb3e1c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 11 Oct 2015 19:28:00 +0600
Subject: [PATCH 2766/6219] [bild] Make more robust and improve hls extraction

---
 youtube_dl/extractor/expotv.py | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/expotv.py b/youtube_dl/extractor/expotv.py
index 23a38c7c1..1585a03bb 100644
--- a/youtube_dl/extractor/expotv.py
+++ b/youtube_dl/extractor/expotv.py
@@ -34,22 +34,25 @@ class ExpoTVIE(InfoExtractor):
         player_key = self._search_regex(
             r'<param name="playerKey" value="([^"]+)"', webpage, 'player key')
         config = self._download_json(
-                'http://client.expotv.com/video/config/%s/%s' % (video_id, player_key),
-                video_id,
-                note='Downloading video configuration')
+            'http://client.expotv.com/video/config/%s/%s' % (video_id, player_key),
+            video_id, 'Downloading video configuration')
 
         formats = []
         for fcfg in config['sources']:
-            if fcfg['type'] == 'm3u8':
-                formats.extend(self._extract_m3u8_formats(fcfg['file'], video_id))
+            media_url = fcfg.get('file')
+            if not media_url:
+                continue
+            if fcfg.get('type') == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    media_url, video_id, 'mp4', entry_protocol='m3u8_native', m3u8_id='hls'))
             else:
                 formats.append({
-                    'url': fcfg['file'],
+                    'url': media_url,
                     'height': int_or_none(fcfg.get('height')),
                     'format_id': fcfg.get('label'),
                     'ext': self._search_regex(
-                        r'filename=.*\.([a-z0-9_A-Z]+)&', fcfg['file'],
-                        'file extension', default=None),
+                        r'filename=.*\.([a-z0-9_A-Z]+)&', media_url,
+                        'file extension', default=None) or fcfg.get('type'),
                 })
         self._sort_formats(formats)
 

From da4daed5ef77b4a7219b1786978065a3606b85bc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 11 Oct 2015 19:36:23 +0600
Subject: [PATCH 2767/6219] [vimeo] Do not fail when no hls formats

---
 youtube_dl/extractor/vimeo.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 346eb60b4..6ee3069a8 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -403,7 +403,10 @@ class VimeoIE(VimeoBaseInfoExtractor):
         formats = []
         m3u8_url = config_files.get('hls', {}).get('all')
         if m3u8_url:
-            formats = self._extract_m3u8_formats(m3u8_url, video_id, 'mp4', 'm3u8_native', 0, 'hls', fatal=False)
+            m3u8_formats = self._extract_m3u8_formats(
+                m3u8_url, video_id, 'mp4', 'm3u8_native', 0, 'hls', fatal=False)
+            if m3u8_formats:
+                formats.append(m3u8_formats)
         for key in ('other', 'sd', 'hd'):
             formats += files[key]
         self._sort_formats(formats)

From 1bd390358205728823ff38a51b12ae35f9468929 Mon Sep 17 00:00:00 2001
From: PC <tioocnt@yandex.com>
Date: Tue, 6 Oct 2015 22:28:58 +0100
Subject: [PATCH 2768/6219] chaturbate streams

---
 youtube_dl/extractor/__init__.py   |  1 +
 youtube_dl/extractor/chaturbate.py | 24 ++++++++++++++++++++++++
 2 files changed, 25 insertions(+)
 create mode 100644 youtube_dl/extractor/chaturbate.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 3ace1cc2c..75720843c 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -76,6 +76,7 @@ from .cbssports import CBSSportsIE
 from .ccc import CCCIE
 from .ceskatelevize import CeskaTelevizeIE
 from .channel9 import Channel9IE
+from .chaturbate import ChaturbateIE
 from .chilloutzone import ChilloutzoneIE
 from .chirbit import (
     ChirbitIE,
diff --git a/youtube_dl/extractor/chaturbate.py b/youtube_dl/extractor/chaturbate.py
new file mode 100644
index 000000000..5e24e1e4f
--- /dev/null
+++ b/youtube_dl/extractor/chaturbate.py
@@ -0,0 +1,24 @@
+# encoding: utf-8
+
+from .common import InfoExtractor
+
+
+class ChaturbateIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?chaturbate\.com/(?P<id>[^/]+)/?$'
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        m3u8_url = self._search_regex(r"'(https?://.*?\.m3u8)'", webpage, 'playlist')
+
+        formats = self._extract_m3u8_formats(m3u8_url, video_id, ext='mp4')
+
+        return {
+            'id': video_id,
+            'title': self._live_title(video_id),
+            'description': self._html_search_meta('description', webpage, 'description'),
+            'is_live': True,
+            'thumbnail': 'https://cdn-s.highwebmedia.com/uHK3McUtGCG3SMFcd4ZJsRv8/roomimage/%s.jpg' % (video_id,),
+            'formats': formats,
+        }

From 0f61db4469db4c17712b8a3f9e527e845ec96ed4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 11 Oct 2015 20:35:22 +0600
Subject: [PATCH 2769/6219] [chaturbate] Improve and capture error message

---
 youtube_dl/extractor/chaturbate.py | 36 +++++++++++++++++++++++++-----
 1 file changed, 31 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/chaturbate.py b/youtube_dl/extractor/chaturbate.py
index 5e24e1e4f..0b67ba67d 100644
--- a/youtube_dl/extractor/chaturbate.py
+++ b/youtube_dl/extractor/chaturbate.py
@@ -1,24 +1,50 @@
-# encoding: utf-8
+from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..utils import ExtractorError
 
 
 class ChaturbateIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?chaturbate\.com/(?P<id>[^/]+)/?$'
+    _VALID_URL = r'https?://(?:[^/]+\.)?chaturbate\.com/(?P<id>[^/?#]+)'
+    _TESTS = [{
+        'url': 'https://www.chaturbate.com/siswet19/',
+        'info_dict': {
+            'id': 'siswet19',
+            'ext': 'mp4',
+            'title': 're:^siswet19 [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
+            'age_limit': 18,
+            'is_live': True,
+        },
+        'params': {
+            'skip_download': True,
+        }
+    }, {
+        'url': 'https://en.chaturbate.com/siswet19/',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
+
         webpage = self._download_webpage(url, video_id)
 
-        m3u8_url = self._search_regex(r"'(https?://.*?\.m3u8)'", webpage, 'playlist')
+        m3u8_url = self._search_regex(
+            r'src=(["\'])(?P<url>http.+?\.m3u8.*?)\1', webpage,
+            'playlist', default=None, group='url')
+
+        if not m3u8_url:
+            error = self._search_regex(
+                r'<span[^>]+class=(["\'])desc_span\1[^>]*>(?P<error>[^<]+)</span>',
+                webpage, 'error', group='error')
+            raise ExtractorError(error, expected=True)
 
         formats = self._extract_m3u8_formats(m3u8_url, video_id, ext='mp4')
 
         return {
             'id': video_id,
             'title': self._live_title(video_id),
-            'description': self._html_search_meta('description', webpage, 'description'),
+            'thumbnail': 'https://cdn-s.highwebmedia.com/uHK3McUtGCG3SMFcd4ZJsRv8/roomimage/%s.jpg' % video_id,
+            'age_limit': self._rta_search(webpage),
             'is_live': True,
-            'thumbnail': 'https://cdn-s.highwebmedia.com/uHK3McUtGCG3SMFcd4ZJsRv8/roomimage/%s.jpg' % (video_id,),
             'formats': formats,
         }

From f101079ae02560fee2df1ea0cacb1989eefc8e6c Mon Sep 17 00:00:00 2001
From: AndroKev <AndroKev@users.noreply.github.com>
Date: Sun, 11 Oct 2015 19:17:24 +0200
Subject: [PATCH 2770/6219] [downloader/rtmp] Respect --no-continue option

now when downloading a "rtmp-file" the --no-continue option works!
---
 youtube_dl/downloader/rtmp.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/rtmp.py b/youtube_dl/downloader/rtmp.py
index 7d19bb808..f1d219ba9 100644
--- a/youtube_dl/downloader/rtmp.py
+++ b/youtube_dl/downloader/rtmp.py
@@ -105,7 +105,7 @@ class RtmpFD(FileDownloader):
         protocol = info_dict.get('rtmp_protocol', None)
         real_time = info_dict.get('rtmp_real_time', False)
         no_resume = info_dict.get('no_resume', False)
-        continue_dl = info_dict.get('continuedl', True)
+        continue_dl = self.params.get('continuedl', True)
 
         self.report_destination(filename)
         tmpfilename = self.temp_name(filename)

From 964e7b2dd09099bd021a770522684716f341db43 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 12 Oct 2015 00:43:54 +0600
Subject: [PATCH 2771/6219] [downloader/common] Always skip "already
 downloaded" check when outputting to stdout

---
 youtube_dl/downloader/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index 97e755d4b..29a4500d3 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -325,7 +325,7 @@ class FileDownloader(object):
         )
 
         # Check file already present
-        if filename != '-' and nooverwrites_and_exists or continuedl_and_exists:
+        if filename != '-' and (nooverwrites_and_exists or continuedl_and_exists):
             self.report_file_already_downloaded(filename)
             self._hook_progress({
                 'filename': filename,

From 57d1db8dd00b833c08b3ac74fbd589a0216775f6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 11 Oct 2015 22:45:13 +0200
Subject: [PATCH 2772/6219] [rtbf] Fix extraction (closes #7133)

---
 youtube_dl/extractor/rtbf.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/rtbf.py b/youtube_dl/extractor/rtbf.py
index e4215d546..04a66df90 100644
--- a/youtube_dl/extractor/rtbf.py
+++ b/youtube_dl/extractor/rtbf.py
@@ -36,7 +36,7 @@ class RTBFIE(InfoExtractor):
 
         data = self._parse_json(
             unescapeHTML(self._search_regex(
-                r'data-video="([^"]+)"', webpage, 'data video')),
+                r'data-media="([^"]+)"', webpage, 'data video')),
             video_id)
 
         if data.get('provider').lower() == 'youtube':

From cd7364a89c69c4c19e4b9d4e54d08fd1eff9c51d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 12 Oct 2015 06:37:20 +0200
Subject: [PATCH 2773/6219] release 2015.10.12

---
 docs/supportedsites.md | 1 +
 youtube_dl/version.py  | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 5beb2ecd4..dc0354095 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -81,6 +81,7 @@
  - **CBSSports**
  - **CeskaTelevize**
  - **channel9**: Channel 9
+ - **Chaturbate**
  - **Chilloutzone**
  - **chirbit**
  - **chirbit:profile**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index faae8a2d7..0908e963d 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.10.09'
+__version__ = '2015.10.12'

From 73e732eb6b5fb2edd61701ae110cc52fe6836364 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 12 Oct 2015 16:34:55 +0800
Subject: [PATCH 2774/6219] [vimeo] Fix m3u8 formats

Reported at
https://github.com/rg3/youtube-dl/pull/7126#issuecomment-147327584
---
 youtube_dl/extractor/vimeo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 6ee3069a8..027f47ee3 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -406,7 +406,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
             m3u8_formats = self._extract_m3u8_formats(
                 m3u8_url, video_id, 'mp4', 'm3u8_native', 0, 'hls', fatal=False)
             if m3u8_formats:
-                formats.append(m3u8_formats)
+                formats.extend(m3u8_formats)
         for key in ('other', 'sd', 'hd'):
             formats += files[key]
         self._sort_formats(formats)

From ee2edd838a1e8770488e695c380943ded44d0983 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 13 Oct 2015 00:53:05 +0200
Subject: [PATCH 2775/6219] release 2015.10.13

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 0908e963d..aaa43d315 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.10.12'
+__version__ = '2015.10.13'

From 5946cda7c6f2e4a7eb90fff6f10c66af0ff2a0d7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 13 Oct 2015 21:04:39 +0600
Subject: [PATCH 2776/6219] [beeg] Fix extraction (Closes #7155)

---
 youtube_dl/extractor/beeg.py | 68 +++++++++++++++++++-----------------
 1 file changed, 35 insertions(+), 33 deletions(-)

diff --git a/youtube_dl/extractor/beeg.py b/youtube_dl/extractor/beeg.py
index b38057f2f..e6c928699 100644
--- a/youtube_dl/extractor/beeg.py
+++ b/youtube_dl/extractor/beeg.py
@@ -1,65 +1,67 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    parse_iso8601,
+)
 
 
 class BeegIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?beeg\.com/(?P<id>\d+)'
     _TEST = {
         'url': 'http://beeg.com/5416503',
-        'md5': '1bff67111adb785c51d1b42959ec10e5',
+        'md5': '46c384def73b33dbc581262e5ee67cef',
         'info_dict': {
             'id': '5416503',
             'ext': 'mp4',
             'title': 'Sultry Striptease',
-            'description': 'md5:6db3c6177972822aaba18652ff59c773',
-            'categories': list,  # NSFW
-            'thumbnail': 're:https?://.*\.jpg$',
+            'description': 'md5:d22219c09da287c14bed3d6c37ce4bc2',
+            'timestamp': 1391813355,
+            'upload_date': '20140207',
+            'duration': 383,
+            'tags': list,
             'age_limit': 18,
         }
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
-        webpage = self._download_webpage(url, video_id)
-
-        quality_arr = self._search_regex(
-            r'(?s)var\s+qualityArr\s*=\s*{\s*(.+?)\s*}', webpage, 'quality formats')
-
-        formats = [{
-            'url': fmt[1],
-            'format_id': fmt[0],
-            'height': int(fmt[0][:-1]),
-        } for fmt in re.findall(r"'([^']+)'\s*:\s*'([^']+)'", quality_arr)]
+        video = self._download_json(
+            'http://beeg.com/api/v1/video/%s' % video_id, video_id)
 
+        formats = []
+        for format_id, video_url in video.items():
+            height = self._search_regex(
+                r'^(\d+)[pP]$', format_id, 'height', default=None)
+            if not height:
+                continue
+            formats.append({
+                'url': self._proto_relative_url(video_url.replace('{DATA_MARKERS}', ''), 'http:'),
+                'format_id': format_id,
+                'height': int(height),
+            })
         self._sort_formats(formats)
 
-        title = self._html_search_regex(
-            r'<title>([^<]+)\s*-\s*beeg\.?</title>', webpage, 'title')
+        title = video['title']
+        video_id = video.get('id') or video_id
+        display_id = video.get('code')
+        description = video.get('desc')
 
-        description = self._html_search_regex(
-            r'<meta name="description" content="([^"]*)"',
-            webpage, 'description', fatal=False)
-        thumbnail = self._html_search_regex(
-            r'\'previewer.url\'\s*:\s*"([^"]*)"',
-            webpage, 'thumbnail', fatal=False)
+        timestamp = parse_iso8601(video.get('date'), ' ')
+        duration = int_or_none(video.get('duration'))
 
-        categories_str = self._html_search_regex(
-            r'<meta name="keywords" content="([^"]+)"', webpage, 'categories', fatal=False)
-        categories = (
-            None if categories_str is None
-            else categories_str.split(','))
+        tags = [tag.strip() for tag in video['tags'].split(',')] if video.get('tags') else None
 
         return {
             'id': video_id,
+            'display_id': display_id,
             'title': title,
             'description': description,
-            'thumbnail': thumbnail,
-            'categories': categories,
+            'timestamp': timestamp,
+            'duration': duration,
+            'tags': tags,
             'formats': formats,
             'age_limit': 18,
         }

From 3eeff489e80838cd2d1b3f55fc96db747386fd50 Mon Sep 17 00:00:00 2001
From: DesweR <egomail@inbox.ru>
Date: Tue, 13 Oct 2015 16:29:16 +0700
Subject: [PATCH 2777/6219] Extract thumbnail url

---
 youtube_dl/extractor/yandexmusic.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/youtube_dl/extractor/yandexmusic.py b/youtube_dl/extractor/yandexmusic.py
index 4098e4629..6842f834f 100644
--- a/youtube_dl/extractor/yandexmusic.py
+++ b/youtube_dl/extractor/yandexmusic.py
@@ -46,6 +46,14 @@ class YandexMusicTrackIE(InfoExtractor):
                 % (data['host'], key, data['ts'] + data['path'], storage[1]))
 
     def _get_track_info(self, track):
+        album = track['albums'][0]
+        a_thumb = None
+
+        if 'coverUri' in album:
+            a_thumb = album['coverUri']
+            if a_thumb:
+                a_thumb = 'http://' + a_thumb.replace('%%', '1000x1000')
+
         return {
             'id': track['id'],
             'ext': 'mp3',
@@ -53,6 +61,7 @@ class YandexMusicTrackIE(InfoExtractor):
             'title': '%s - %s' % (track['artists'][0]['name'], track['title']),
             'filesize': int_or_none(track.get('fileSize')),
             'duration': float_or_none(track.get('durationMs'), 1000),
+            'thumbnail': a_thumb,
         }
 
     def _real_extract(self, url):

From b30c4992a93d411f4f89faf2af153fc580138a90 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 13 Oct 2015 21:14:33 +0200
Subject: [PATCH 2778/6219] [channel9] Return a single dictionary for single
 videos (closes #7086)

Returning a list is deprecated.
---
 youtube_dl/extractor/channel9.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/channel9.py b/youtube_dl/extractor/channel9.py
index 3dfc24f5b..79fd0a30e 100644
--- a/youtube_dl/extractor/channel9.py
+++ b/youtube_dl/extractor/channel9.py
@@ -224,12 +224,12 @@ class Channel9IE(InfoExtractor):
         if contents is None:
             return contents
 
-        authors = self._extract_authors(html)
+        if len(contents) > 1:
+            raise ExtractorError('Got more than one entry')
+        result = contents[0]
+        result['authors'] = self._extract_authors(html)
 
-        for content in contents:
-            content['authors'] = authors
-
-        return contents
+        return result
 
     def _extract_session(self, html, content_path):
         contents = self._extract_content(html, content_path)

From 506e261d2073d8c00d5b43d272e8173cb0d63728 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 13 Oct 2015 21:18:30 +0200
Subject: [PATCH 2779/6219] [channel9] strip 'session_day'

---
 youtube_dl/extractor/channel9.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/channel9.py b/youtube_dl/extractor/channel9.py
index 79fd0a30e..1ce004932 100644
--- a/youtube_dl/extractor/channel9.py
+++ b/youtube_dl/extractor/channel9.py
@@ -158,7 +158,7 @@ class Channel9IE(InfoExtractor):
 
     def _extract_session_day(self, html):
         m = re.search(r'<li class="day">\s*<a href="/Events/[^"]+">(?P<day>[^<]+)</a>\s*</li>', html)
-        return m.group('day') if m is not None else None
+        return m.group('day').strip() if m is not None else None
 
     def _extract_session_room(self, html):
         m = re.search(r'<li class="room">\s*(?P<room>.+?)\s*</li>', html)

From 3dc582e5ea69af4ad7f51d30c1d87cf93aa6b72b Mon Sep 17 00:00:00 2001
From: kaspi <je326@hotmail.com>
Date: Mon, 12 Oct 2015 01:25:57 -0400
Subject: [PATCH 2780/6219] [fczenit] Add extractor

Closes #7143.
---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/fczenit.py  | 41 ++++++++++++++++++++++++++++++++
 2 files changed, 42 insertions(+)
 create mode 100644 youtube_dl/extractor/fczenit.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 75720843c..f6d185818 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -167,6 +167,7 @@ from .extremetube import ExtremeTubeIE
 from .facebook import FacebookIE
 from .faz import FazIE
 from .fc2 import FC2IE
+from .fczenit import FczenitIE
 from .firstpost import FirstpostIE
 from .firsttv import FirstTVIE
 from .fivemin import FiveMinIE
diff --git a/youtube_dl/extractor/fczenit.py b/youtube_dl/extractor/fczenit.py
new file mode 100644
index 000000000..f1f150ef2
--- /dev/null
+++ b/youtube_dl/extractor/fczenit.py
@@ -0,0 +1,41 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+
+class FczenitIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?fc-zenit\.ru/video/gl(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'http://fc-zenit.ru/video/gl6785/',
+        'md5': '458bacc24549173fe5a5aa29174a5606',
+        'info_dict': {
+            'id': '6785',
+            'ext': 'mp4',
+            'title': '«Зенит-ТВ»: как Олег Шатов играл против «Урала»',
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        video_title = self._html_search_regex(r'<div class=\"photoalbum__title\">([^<]+)', webpage, 'title')
+
+        bitrates_raw = self._html_search_regex(r'bitrates:.*\n(.*)\]', webpage, 'video URL')
+        bitrates = re.findall(r'url:.?\'(.+?)\'.*?bitrate:.?([0-9]{3}?)', bitrates_raw)
+
+        formats = [{
+            'url': furl,
+            'tbr': tbr,
+        } for furl, tbr in bitrates]
+
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': video_title,
+            'formats': formats,
+        }

From e09f58b3bc3af6ce1e541fb7d034fe869fba6e82 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 14 Oct 2015 10:40:54 +0100
Subject: [PATCH 2781/6219] [srgssr] change the url chortcut, fix image
 extraction ,add a test and extract format id

---
 youtube_dl/extractor/srgssr.py | 35 ++++++++++++++++++++++++----------
 1 file changed, 25 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/srgssr.py b/youtube_dl/extractor/srgssr.py
index 3b5dcc503..f759e5600 100644
--- a/youtube_dl/extractor/srgssr.py
+++ b/youtube_dl/extractor/srgssr.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 
 class SRGSSRIE(InfoExtractor):
-    _VALID_URL = r'(?:https?://tp\.srgssr\.ch/p(?:/[^/]+)+\?urn=)?urn:(?P<bu>srf|rts|rsi|rtr|swi):(?:[^:]+:)?(?P<type>video|audio):(?P<id>[0-9a-f\-]{36}|\d+)'
+    _VALID_URL = r'(?:https?://tp\.srgssr\.ch/p(?:/[^/]+)+\?urn=urn|srgssr):(?P<bu>srf|rts|rsi|rtr|swi):(?:[^:]+:)?(?P<type>video|audio):(?P<id>[0-9a-f\-]{36}|\d+)'
 
     _ERRORS = {
         'AGERATING12': 'To protect children under the age of 12, this video is only available between 8 p.m. and 6 a.m.',
@@ -44,11 +44,12 @@ class SRGSSRIE(InfoExtractor):
         timestamp = parse_iso8601(created_date)
 
         thumbnails = []
-        for image in media_data['Image']['ImageRepresentations']['ImageRepresentation']:
-            thumbnails.append({
-                'id': image.get('id'),
-                'url': image['url'],
-            })
+        if 'Image' in media_data:
+            for image in media_data['Image']['ImageRepresentations']['ImageRepresentation']:
+                thumbnails.append({
+                    'id': image.get('id'),
+                    'url': image['url'],
+                })
 
         preference = qualities(['LQ', 'MQ', 'SD', 'HQ', 'HD'])
         formats = []
@@ -70,16 +71,17 @@ class SRGSSRIE(InfoExtractor):
                     if asset_url.startswith('rtmp'):
                         ext = self._search_regex(r'([a-z0-9]+):[^/]+', asset_url, 'ext')
                     formats.append({
+                        'format_id': asset['@quality'],
                         'url': asset_url,
                         'preference': preference(asset['@quality']),
                         'ext': ext,
                     })
 
-        downloads = media_data.get('Downloads')
-        if downloads:
-            for source in downloads['Download']:
+        if 'Downloads' in media_data:
+            for source in media_data['Downloads']['Download']:
                 for asset in source['url']:
                     formats.append({
+                        'format_id': asset['@quality'],
                         'url': asset['text'],
                         'preference': preference(asset['@quality'])
                     })
@@ -120,8 +122,21 @@ class SRGSSRPlayIE(InfoExtractor):
             'description': 'md5:88604432b60d5a38787f152dec89cd56',
             'timestamp': 1373493600,
         },
+    },{
+        'url': 'http://www.rtr.ch/play/radio/actualitad/audio/saira-tujetsch-tuttina-cuntinuar-cun-sedrun-muster-turissem?id=63cb0778-27f8-49af-9284-8c7a8c6d15fc',
+        'info_dict': {
+            'id': '63cb0778-27f8-49af-9284-8c7a8c6d15fc',
+            'ext': 'mp3',
+            'upload_date': '20151013',
+            'title': 'Saira: Tujetsch - tuttina cuntinuar cun Sedrun Mustér Turissem',
+            'timestamp': 1444750398,
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        },
     }]
 
     def _real_extract(self, url):
         bu, media_type, media_id = re.match(self._VALID_URL, url).groups()
-        return self.url_result('urn:%s:%s:%s' % (bu[:3], media_type, media_id), 'SRGSSR')
+        return self.url_result('srgssr:%s:%s:%s' % (bu[:3], media_type, media_id), 'SRGSSR')

From db8e38b8cff2e67a9ff51104c4a7b33c20650204 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 14 Oct 2015 11:55:03 +0100
Subject: [PATCH 2782/6219] [ign] add tests for me.ign specific language urls

---
 youtube_dl/extractor/ign.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/youtube_dl/extractor/ign.py b/youtube_dl/extractor/ign.py
index fa4e67394..fb2753738 100644
--- a/youtube_dl/extractor/ign.py
+++ b/youtube_dl/extractor/ign.py
@@ -77,6 +77,14 @@ class IGNIE(InfoExtractor):
                 'upload_date': '20140814',
             },
         },
+        {
+            'url': 'http://me.ign.com/en/videos/112203/video/how-hitman-aims-to-be-different-than-every-other-s',
+            'only_matching': True,
+        },
+        {
+            'url': 'http://me.ign.com/ar/angry-birds-2/106533/video/lrd-ldyy-lwl-lfylm-angry-birds',
+            'only_matching': True,
+        },
     ]
 
     def _find_video_id(self, webpage):

From 26669ea3cf596f2ea4bce9e21ce73c1d8fc3ff72 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 10 Oct 2015 16:51:37 +0100
Subject: [PATCH 2783/6219] [5min] extract more video info and formats

Closes #7124.
---
 youtube_dl/extractor/fivemin.py | 84 ++++++++++++++++++++++++++++-----
 1 file changed, 71 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/fivemin.py b/youtube_dl/extractor/fivemin.py
index 157094e8c..2955965d9 100644
--- a/youtube_dl/extractor/fivemin.py
+++ b/youtube_dl/extractor/fivemin.py
@@ -2,11 +2,15 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..compat import (
-    compat_str,
     compat_urllib_parse,
+    compat_parse_qs,
+    compat_urllib_parse_urlparse,
+    compat_urlparse,
 )
 from ..utils import (
     ExtractorError,
+    parse_duration,
+    replace_extension,
 )
 
 
@@ -28,6 +32,7 @@ class FiveMinIE(InfoExtractor):
                 'id': '518013791',
                 'ext': 'mp4',
                 'title': 'iPad Mini with Retina Display Review',
+                'duration': 177,
             },
         },
         {
@@ -38,9 +43,52 @@ class FiveMinIE(InfoExtractor):
                 'id': '518086247',
                 'ext': 'mp4',
                 'title': 'How to Make a Next-Level Fruit Salad',
+                'duration': 184,
             },
         },
     ]
+    _ERRORS = {
+        'ErrorVideoNotExist': 'We\'re sorry, but the video you are trying to watch does not exist.',
+        'ErrorVideoNoLongerAvailable': 'We\'re sorry, but the video you are trying to watch is no longer available.',
+        'ErrorVideoRejected': 'We\'re sorry, but the video you are trying to watch has been removed.',
+        'ErrorVideoUserNotGeo': 'We\'re sorry, but the video you are trying to watch cannot be viewed from your current location.',
+        'ErrorVideoLibraryRestriction': 'We\'re sorry, but the video you are trying to watch is currently unavailable for viewing at this domain.',
+        'ErrorExposurePermission': 'We\'re sorry, but the video you are trying to watch is currently unavailable for viewing at this domain.',
+    }
+    _QUALITIES = {
+        1: {
+            'width': 640,
+            'height': 360,
+        },
+        2: {
+            'width': 854,
+            'height': 480,
+        },
+        4: {
+            'width': 1280,
+            'height': 720,
+        },
+        8: {
+            'width': 1920,
+            'height': 1080,
+        },
+        16: {
+            'width': 640,
+            'height': 360,
+        },
+        32: {
+            'width': 854,
+            'height': 480,
+        },
+        64: {
+            'width': 1280,
+            'height': 720,
+        },
+        128: {
+            'width': 640,
+            'height': 360,
+        },
+    }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -59,26 +107,36 @@ class FiveMinIE(InfoExtractor):
             'https://syn.5min.com/handlers/SenseHandler.ashx?' + query,
             video_id)
         if not response['success']:
-            err_msg = response['errorMessage']
-            if err_msg == 'ErrorVideoUserNotGeo':
-                msg = 'Video not available from your location'
-            else:
-                msg = 'Aol said: %s' % err_msg
-            raise ExtractorError(msg, expected=True, video_id=video_id)
+            raise ExtractorError(
+                '%s said: %s' % (
+                    self.IE_NAME,
+                    self._ERRORS.get(response['errorMessage'], response['errorMessage'])),
+                expected=True)
         info = response['binding'][0]
 
-        second_id = compat_str(int(video_id[:-2]) + 1)
         formats = []
-        for quality, height in [(1, 320), (2, 480), (4, 720), (8, 1080)]:
-            if any(r['ID'] == quality for r in info['Renditions']):
+        parsed_video_url = compat_urllib_parse_urlparse(compat_parse_qs(
+            compat_urllib_parse_urlparse(info['EmbededURL']).query)['videoUrl'][0])
+        for rendition in info['Renditions']:
+            if rendition['RenditionType'] == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(rendition['Url'], video_id, m3u8_id='hls'))
+            elif rendition['RenditionType'] == 'aac':
+                continue
+            else:
+                rendition_url = compat_urlparse.urlunparse(parsed_video_url._replace(path=replace_extension(parsed_video_url.path.replace('//', '/%s/' % rendition['ID']), rendition['RenditionType'])))
+                quality = self._QUALITIES.get(rendition['ID'], {})
                 formats.append({
-                    'format_id': compat_str(quality),
-                    'url': 'http://avideos.5min.com/%s/%s/%s_%s.mp4' % (second_id[-3:], second_id, video_id, quality),
-                    'height': height,
+                    'format_id': '%s-%d' % (rendition['RenditionType'], rendition['ID']),
+                    'url': rendition_url,
+                    'width': quality.get('width'),
+                    'height': quality.get('height'),
                 })
+        self._sort_formats(formats)
 
         return {
             'id': video_id,
             'title': info['Title'],
+            'thumbnail': info.get('ThumbURL'),
+            'duration': parse_duration(info.get('Duration')),
             'formats': formats,
         }

From 1f36085df94c2addd1175e7e299f6235aca3ac68 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 14 Oct 2015 13:41:39 +0200
Subject: [PATCH 2784/6219] [vimeo] Fix extraction of password protected videos
 (fixes #7169)

---
 youtube_dl/extractor/vimeo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 027f47ee3..fa1b22049 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -212,7 +212,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
             url = url.replace('http://', 'https://')
         password_request = compat_urllib_request.Request(url + '/password', data)
         password_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
-        password_request.add_header('Cookie', 'clip_v=1; vuid=%s' % vuid)
+        password_request.add_header('Cookie', 'clip_test2=1; vuid=%s' % vuid)
         password_request.add_header('Referer', url)
         return self._download_webpage(
             password_request, video_id,

From 36bb63fad19df5ee419979f875e2265936511644 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 14 Oct 2015 14:13:53 +0100
Subject: [PATCH 2785/6219] [criterion] fix description extraction

---
 youtube_dl/extractor/criterion.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/criterion.py b/youtube_dl/extractor/criterion.py
index 4fb178165..dedb810a0 100644
--- a/youtube_dl/extractor/criterion.py
+++ b/youtube_dl/extractor/criterion.py
@@ -27,9 +27,7 @@ class CriterionIE(InfoExtractor):
         final_url = self._search_regex(
             r'so.addVariable\("videoURL", "(.+?)"\)\;', webpage, 'video url')
         title = self._og_search_title(webpage)
-        description = self._html_search_regex(
-            r'<meta name="description" content="(.+?)" />',
-            webpage, 'video description')
+        description = self._html_search_meta('description', webpage)
         thumbnail = self._search_regex(
             r'so.addVariable\("thumbnailURL", "(.+?)"\)\;',
             webpage, 'thumbnail url')

From 7a6d76a64d8a89a08bb79791506fc18b993c4580 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 14 Oct 2015 20:49:39 +0600
Subject: [PATCH 2786/6219] [extractor/common] Require closing quote in
 _og_regexes (Closes #7174)

E.g. do not match `property='og:video:type'` when `og:video` is requested.
---
 youtube_dl/extractor/common.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 0082a4c84..a0c4af92f 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -646,7 +646,8 @@ class InfoExtractor(object):
     @staticmethod
     def _og_regexes(prop):
         content_re = r'content=(?:"([^>]+?)"|\'([^>]+?)\'|\s*([^\s"\'=<>`]+?))'
-        property_re = r'(?:name|property)=[\'"]?og:%s[\'"]?' % re.escape(prop)
+        property_re = (r'(?:name|property)=(?:\'og:%(prop)s\'|"og:%(prop)s"|\s*og:%(prop)s\b)'
+                       % {'prop': re.escape(prop)})
         template = r'<meta[^>]+?%s[^>]+?%s'
         return [
             template % (property_re, content_re),

From 1c29e81e620241b9013b23e7acd9d6ab06587fb1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 14 Oct 2015 20:58:52 +0600
Subject: [PATCH 2787/6219] [test_InfoExtractor] Add test for
 7a6d76a64d8a89a08bb79791506fc18b993c4580

---
 test/test_InfoExtractor.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/test/test_InfoExtractor.py b/test/test_InfoExtractor.py
index be8d12997..4ce5b5a35 100644
--- a/test/test_InfoExtractor.py
+++ b/test/test_InfoExtractor.py
@@ -35,10 +35,12 @@ class TestInfoExtractor(unittest.TestCase):
             <meta name="og:title" content='Foo'/>
             <meta content="Some video's description " name="og:description"/>
             <meta property='og:image' content='http://domain.com/pic.jpg?key1=val1&amp;key2=val2'/>
+            <meta content='application/x-shockwave-flash' property='og:video:type'>
             '''
         self.assertEqual(ie._og_search_title(html), 'Foo')
         self.assertEqual(ie._og_search_description(html), 'Some video\'s description ')
         self.assertEqual(ie._og_search_thumbnail(html), 'http://domain.com/pic.jpg?key1=val1&key2=val2')
+        self.assertEqual(ie._og_search_video_url(html, default=None), None)
 
     def test_html_search_meta(self):
         ie = self.ie

From db0a8ad97993cb3f0c398d3a5dc55389565e0ffd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 14 Oct 2015 21:11:06 +0600
Subject: [PATCH 2788/6219] [test_InfoExtractor] Add test for unquoted
 attribute

---
 test/test_InfoExtractor.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/test/test_InfoExtractor.py b/test/test_InfoExtractor.py
index 4ce5b5a35..2a00d09a5 100644
--- a/test/test_InfoExtractor.py
+++ b/test/test_InfoExtractor.py
@@ -36,11 +36,13 @@ class TestInfoExtractor(unittest.TestCase):
             <meta content="Some video's description " name="og:description"/>
             <meta property='og:image' content='http://domain.com/pic.jpg?key1=val1&amp;key2=val2'/>
             <meta content='application/x-shockwave-flash' property='og:video:type'>
+            <meta content='Foo' property=og:foobar>
             '''
         self.assertEqual(ie._og_search_title(html), 'Foo')
         self.assertEqual(ie._og_search_description(html), 'Some video\'s description ')
         self.assertEqual(ie._og_search_thumbnail(html), 'http://domain.com/pic.jpg?key1=val1&key2=val2')
         self.assertEqual(ie._og_search_video_url(html, default=None), None)
+        self.assertEqual(ie._og_search_property('foobar', html), 'Foo')
 
     def test_html_search_meta(self):
         ie = self.ie

From ab953c64a0e8b8558e95d0318110c0885a4eec3d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 14 Oct 2015 21:15:29 +0600
Subject: [PATCH 2789/6219] [yandexmusic:track] Extract original size thumbnail
 (Closes #7160)

---
 youtube_dl/extractor/yandexmusic.py | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/yandexmusic.py b/youtube_dl/extractor/yandexmusic.py
index 6842f834f..08dc81f3a 100644
--- a/youtube_dl/extractor/yandexmusic.py
+++ b/youtube_dl/extractor/yandexmusic.py
@@ -46,14 +46,12 @@ class YandexMusicTrackIE(InfoExtractor):
                 % (data['host'], key, data['ts'] + data['path'], storage[1]))
 
     def _get_track_info(self, track):
-        album = track['albums'][0]
-        a_thumb = None
-
-        if 'coverUri' in album:
-            a_thumb = album['coverUri']
-            if a_thumb:
-                a_thumb = 'http://' + a_thumb.replace('%%', '1000x1000')
-
+        thumbnail = None
+        cover_uri = track.get('albums', [{}])[0].get('coverUri')
+        if cover_uri:
+            thumbnail = cover_uri.replace('%%', 'orig')
+            if not thumbnail.startswith('http'):
+                thumbnail = 'http://' + thumbnail
         return {
             'id': track['id'],
             'ext': 'mp3',
@@ -61,7 +59,7 @@ class YandexMusicTrackIE(InfoExtractor):
             'title': '%s - %s' % (track['artists'][0]['name'], track['title']),
             'filesize': int_or_none(track.get('fileSize')),
             'duration': float_or_none(track.get('durationMs'), 1000),
-            'thumbnail': a_thumb,
+            'thumbnail': thumbnail,
         }
 
     def _real_extract(self, url):

From 9fb66c780cee8668b1bb07f70e70ae1161e13320 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 14 Oct 2015 17:25:07 +0100
Subject: [PATCH 2790/6219] [megavideoz] remove extractor

---
 youtube_dl/extractor/__init__.py   |  1 -
 youtube_dl/extractor/megavideoz.py | 56 ------------------------------
 2 files changed, 57 deletions(-)
 delete mode 100644 youtube_dl/extractor/megavideoz.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index f6d185818..462717b1e 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -319,7 +319,6 @@ from .macgamestore import MacGameStoreIE
 from .mailru import MailRuIE
 from .malemotion import MalemotionIE
 from .mdr import MDRIE
-from .megavideoz import MegaVideozIE
 from .metacafe import MetacafeIE
 from .metacritic import MetacriticIE
 from .mgoon import MgoonIE
diff --git a/youtube_dl/extractor/megavideoz.py b/youtube_dl/extractor/megavideoz.py
deleted file mode 100644
index af7ff07ea..000000000
--- a/youtube_dl/extractor/megavideoz.py
+++ /dev/null
@@ -1,56 +0,0 @@
-# encoding: utf-8
-from __future__ import unicode_literals
-
-import re
-
-from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
-    float_or_none,
-    xpath_text,
-)
-
-
-class MegaVideozIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?megavideoz\.eu/video/(?P<id>[^/]+)(?:/(?P<display_id>[^/]+))?'
-    _TEST = {
-        'url': 'http://megavideoz.eu/video/WM6UB919XMXH/SMPTE-Universal-Film-Leader',
-        'info_dict': {
-            'id': '48723',
-            'display_id': 'SMPTE-Universal-Film-Leader',
-            'ext': 'mp4',
-            'title': 'SMPTE Universal Film Leader',
-            'thumbnail': 're:https?://.*?\.jpg',
-            'duration': 10.93,
-        }
-    }
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        display_id = mobj.group('display_id') or video_id
-
-        webpage = self._download_webpage(url, display_id)
-
-        if any(p in webpage for p in ('>Video Not Found<', '>404 Error<')):
-            raise ExtractorError('Video %s does not exist' % video_id, expected=True)
-
-        config = self._download_xml(
-            self._search_regex(
-                r"var\s+cnf\s*=\s*'([^']+)'", webpage, 'cnf url'),
-            display_id)
-
-        video_url = xpath_text(config, './file', 'video url', fatal=True)
-        title = xpath_text(config, './title', 'title', fatal=True)
-        thumbnail = xpath_text(config, './image', 'thumbnail')
-        duration = float_or_none(xpath_text(config, './duration', 'duration'))
-        video_id = xpath_text(config, './mediaid', 'video id') or video_id
-
-        return {
-            'id': video_id,
-            'display_id': display_id,
-            'url': video_url,
-            'title': title,
-            'thumbnail': thumbnail,
-            'duration': duration
-        }

From 1812afb7b396f4954d5d1ca1cec1c3f2d67550c6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 14 Oct 2015 22:35:01 +0600
Subject: [PATCH 2791/6219] [utils] Do not fail in int_or_none on non-numeric
 data (Closes #7175)

---
 youtube_dl/utils.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 1dc3153fd..86c693358 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1371,7 +1371,12 @@ def int_or_none(v, scale=1, default=None, get_attr=None, invscale=1):
             v = getattr(v, get_attr, None)
     if v == '':
         v = None
-    return default if v is None else (int(v) * invscale // scale)
+    if v is None:
+        return default
+    try:
+        return int(v) * invscale // scale
+    except ValueError:
+        pass
 
 
 def str_or_none(v, default=None):

From caf80631f0c57b29187e2aa909fa1a3a6325d6e6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 14 Oct 2015 22:36:37 +0600
Subject: [PATCH 2792/6219] [utils] Do not fail in float_or_none on non-numeric
 data

---
 youtube_dl/utils.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 86c693358..83b44caaa 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1392,7 +1392,12 @@ def str_to_int(int_str):
 
 
 def float_or_none(v, scale=1, invscale=1, default=None):
-    return default if v is None else (float(v) * invscale / scale)
+    if v is None:
+        return default
+    try:
+        return float(v) * invscale / scale
+    except ValueError:
+        return default
 
 
 def parse_duration(s):

From af98f8ff37b3a0d9d1f743f4fc6c646333501eb6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 14 Oct 2015 22:37:03 +0600
Subject: [PATCH 2793/6219] [utils] Return default on fail in int_or_none

---
 youtube_dl/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 83b44caaa..7dbe25661 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1376,7 +1376,7 @@ def int_or_none(v, scale=1, default=None, get_attr=None, invscale=1):
     try:
         return int(v) * invscale // scale
     except ValueError:
-        pass
+        return default
 
 
 def str_or_none(v, default=None):

From 1db82381e38181aafbd78c65c58f005ad84cc08a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 15 Oct 2015 01:52:25 +0600
Subject: [PATCH 2794/6219] [channel9] Add low quality formats and modernize

---
 youtube_dl/extractor/channel9.py | 35 ++++++++++++--------------------
 1 file changed, 13 insertions(+), 22 deletions(-)

diff --git a/youtube_dl/extractor/channel9.py b/youtube_dl/extractor/channel9.py
index 1ce004932..3a88181d8 100644
--- a/youtube_dl/extractor/channel9.py
+++ b/youtube_dl/extractor/channel9.py
@@ -3,7 +3,11 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import ExtractorError
+from ..utils import (
+    ExtractorError,
+    parse_filesize,
+    qualities,
+)
 
 
 class Channel9IE(InfoExtractor):
@@ -52,23 +56,6 @@ class Channel9IE(InfoExtractor):
 
     _RSS_URL = 'http://channel9.msdn.com/%s/RSS'
 
-    # Sorted by quality
-    _known_formats = ['MP3', 'MP4', 'Mid Quality WMV', 'Mid Quality MP4', 'High Quality WMV', 'High Quality MP4']
-
-    def _restore_bytes(self, formatted_size):
-        if not formatted_size:
-            return 0
-        m = re.match(r'^(?P<size>\d+(?:\.\d+)?)\s+(?P<units>[a-zA-Z]+)', formatted_size)
-        if not m:
-            return 0
-        units = m.group('units')
-        try:
-            exponent = ['B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'].index(units.upper())
-        except ValueError:
-            return 0
-        size = float(m.group('size'))
-        return int(size * (1024 ** exponent))
-
     def _formats_from_html(self, html):
         FORMAT_REGEX = r'''
             (?x)
@@ -78,16 +65,20 @@ class Channel9IE(InfoExtractor):
             <h3>File\s+size</h3>\s*(?P<filesize>.*?)\s*
             </div>)?                                                # File size part may be missing
         '''
-        # Extract known formats
+        quality = qualities((
+            'MP3', 'MP4',
+            'Low Quality WMV', 'Low Quality MP4',
+            'Mid Quality WMV', 'Mid Quality MP4',
+            'High Quality WMV', 'High Quality MP4'))
         formats = [{
             'url': x.group('url'),
             'format_id': x.group('quality'),
             'format_note': x.group('note'),
             'format': '%s (%s)' % (x.group('quality'), x.group('note')),
-            'filesize': self._restore_bytes(x.group('filesize')),  # File size is approximate
-            'preference': self._known_formats.index(x.group('quality')),
+            'filesize_approx': parse_filesize(x.group('filesize')),
+            'quality': quality(x.group('quality')),
             'vcodec': 'none' if x.group('note') == 'Audio only' else None,
-        } for x in list(re.finditer(FORMAT_REGEX, html)) if x.group('quality') in self._known_formats]
+        } for x in list(re.finditer(FORMAT_REGEX, html))]
 
         self._sort_formats(formats)
 

From a13d06de420f6968425d48030c37e1150ff9ed6a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 15 Oct 2015 01:57:59 +0600
Subject: [PATCH 2795/6219] [channel9] Add test for low quality mp4

---
 youtube_dl/extractor/channel9.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/youtube_dl/extractor/channel9.py b/youtube_dl/extractor/channel9.py
index 3a88181d8..554399787 100644
--- a/youtube_dl/extractor/channel9.py
+++ b/youtube_dl/extractor/channel9.py
@@ -51,6 +51,21 @@ class Channel9IE(InfoExtractor):
                 'thumbnail': 'http://video.ch9.ms/ch9/87e1/0300391f-a455-4c72-bec3-4422f19287e1/selfservicenuk_512.jpg',
                 'authors': ['Mike Wilmot'],
             },
+        },
+        {
+            # low quality mp4 is best
+            'url': 'https://channel9.msdn.com/Events/CPP/CppCon-2015/Ranges-for-the-Standard-Library',
+            'info_dict': {
+                'id': 'Events/CPP/CppCon-2015/Ranges-for-the-Standard-Library',
+                'ext': 'mp4',
+                'title': 'Ranges for the Standard Library',
+                'description': 'md5:2e6b4917677af3728c5f6d63784c4c5d',
+                'duration': 5646,
+                'thumbnail': 're:http://.*\.jpg',
+            },
+            'params': {
+                'skip_download': True,
+            },
         }
     ]
 

From fafc7950e2230bf25ac7c7563f1704cf8f134f64 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 15 Oct 2015 01:59:11 +0600
Subject: [PATCH 2796/6219] [channel9] Update tests' thumbnails

---
 youtube_dl/extractor/channel9.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/channel9.py b/youtube_dl/extractor/channel9.py
index 554399787..c74553dcf 100644
--- a/youtube_dl/extractor/channel9.py
+++ b/youtube_dl/extractor/channel9.py
@@ -32,7 +32,7 @@ class Channel9IE(InfoExtractor):
                 'title': 'Developer Kick-Off Session: Stuff We Love',
                 'description': 'md5:c08d72240b7c87fcecafe2692f80e35f',
                 'duration': 4576,
-                'thumbnail': 'http://video.ch9.ms/ch9/9d51/03902f2d-fc97-4d3c-b195-0bfe15a19d51/KOS002_220.jpg',
+                'thumbnail': 're:http://.*\.jpg',
                 'session_code': 'KOS002',
                 'session_day': 'Day 1',
                 'session_room': 'Arena 1A',
@@ -48,7 +48,7 @@ class Channel9IE(InfoExtractor):
                 'title': 'Self-service BI with Power BI - nuclear testing',
                 'description': 'md5:d1e6ecaafa7fb52a2cacdf9599829f5b',
                 'duration': 1540,
-                'thumbnail': 'http://video.ch9.ms/ch9/87e1/0300391f-a455-4c72-bec3-4422f19287e1/selfservicenuk_512.jpg',
+                'thumbnail': 're:http://.*\.jpg',
                 'authors': ['Mike Wilmot'],
             },
         },

From 90bddb6cdd59107d137c13970dc50a6193d204a7 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 15 Oct 2015 14:28:56 +0100
Subject: [PATCH 2797/6219] [ooyala] extract more formats and metadata

---
 youtube_dl/extractor/ooyala.py | 151 ++++++++++++---------------------
 1 file changed, 53 insertions(+), 98 deletions(-)

diff --git a/youtube_dl/extractor/ooyala.py b/youtube_dl/extractor/ooyala.py
index a262a9f6d..592cdc564 100644
--- a/youtube_dl/extractor/ooyala.py
+++ b/youtube_dl/extractor/ooyala.py
@@ -1,108 +1,64 @@
 from __future__ import unicode_literals
 import re
-import json
 import base64
 
 from .common import InfoExtractor
 from ..utils import (
-    unescapeHTML,
-    ExtractorError,
-    determine_ext,
     int_or_none,
+    float_or_none,
 )
 
 
 class OoyalaBaseIE(InfoExtractor):
 
-    def _extract_result(self, info, more_info):
-        embedCode = info['embedCode']
-        video_url = info.get('ipad_url') or info['url']
-
-        if determine_ext(video_url) == 'm3u8':
-            formats = self._extract_m3u8_formats(video_url, embedCode, ext='mp4')
-        else:
-            formats = [{
-                'url': video_url,
-                'ext': 'mp4',
-            }]
-
-        return {
-            'id': embedCode,
-            'title': unescapeHTML(info['title']),
-            'formats': formats,
-            'description': unescapeHTML(more_info['description']),
-            'thumbnail': more_info['promo'],
+    def _extract(self, player_url, video_id):
+        print(player_url)
+        content_tree = self._download_json(player_url, video_id)['content_tree']
+        metadata = content_tree[list(content_tree)[0]]
+        embed_code = metadata['embed_code']
+        pcode = metadata.get('asset_pcode') or embed_code
+        video_info = {
+            'id': embed_code,
+            'title': metadata['title'],
+            'description': metadata.get('description'),
+            'thumbnail': metadata.get('thumbnail_image') or metadata.get('promo_image'),
+            'duration': int_or_none(metadata.get('duration')),
         }
 
-    def _extract(self, player_url, video_id):
-        player = self._download_webpage(player_url, video_id)
-        mobile_url = self._search_regex(r'mobile_player_url="(.+?)&device="',
-                                        player, 'mobile player url')
-        # Looks like some videos are only available for particular devices
-        # (e.g. http://player.ooyala.com/player.js?embedCode=x1b3lqZDq9y_7kMyC2Op5qo-p077tXD0
-        # is only available for ipad)
-        # Working around with fetching URLs for all the devices found starting with 'unknown'
-        # until we succeed or eventually fail for each device.
-        devices = re.findall(r'device\s*=\s*"([^"]+)";', player)
-        devices.remove('unknown')
-        devices.insert(0, 'unknown')
-        for device in devices:
-            mobile_player = self._download_webpage(
-                '%s&device=%s' % (mobile_url, device), video_id,
-                'Downloading mobile player JS for %s device' % device)
-            videos_info = self._search_regex(
-                r'var streams=window.oo_testEnv\?\[\]:eval\("\((\[{.*?}\])\)"\);',
-                mobile_player, 'info', fatal=False, default=None)
-            if videos_info:
-                break
-
-        if not videos_info:
-            formats = []
+        formats = []
+        for supported_format in ('mp4', 'm3u8', 'hds', 'rtmp'):
             auth_data = self._download_json(
-                'http://player.ooyala.com/sas/player_api/v1/authorization/embed_code/%s/%s?domain=www.example.org&supportedFormats=mp4,webm' % (video_id, video_id),
-                video_id)
+                'http://player.ooyala.com/sas/player_api/v1/authorization/embed_code/%s/%s?domain=www.example.org&supportedFormats=%s' % (pcode, embed_code, supported_format),
+                video_id, 'Downloading %s JSON' % supported_format)
 
-            cur_auth_data = auth_data['authorization_data'][video_id]
+            cur_auth_data = auth_data['authorization_data'][embed_code]
 
             for stream in cur_auth_data['streams']:
-                formats.append({
-                    'url': base64.b64decode(stream['url']['data'].encode('ascii')).decode('utf-8'),
-                    'ext': stream.get('delivery_type'),
-                    'format': stream.get('video_codec'),
-                    'format_id': stream.get('profile'),
-                    'width': int_or_none(stream.get('width')),
-                    'height': int_or_none(stream.get('height')),
-                    'abr': int_or_none(stream.get('audio_bitrate')),
-                    'vbr': int_or_none(stream.get('video_bitrate')),
-                })
-            if formats:
-                return {
-                    'id': video_id,
-                    'formats': formats,
-                    'title': 'Ooyala video',
-                }
+                url = base64.b64decode(stream['url']['data'].encode('ascii')).decode('utf-8')
+                delivery_type = stream['delivery_type']
+                if delivery_type == 'remote_asset':
+                    video_info['url'] = url
+                    return video_info
+                if delivery_type == 'hls':
+                    formats.extend(self._extract_m3u8_formats(url, embed_code, 'mp4', 'm3u8_native', 0, m3u8_id='hls', fatal=False))
+                elif delivery_type == 'hds':
+                    formats.extend(self._extract_f4m_formats(url, embed_code, f4m_id='hds', fatal=False))
+                else:
+                    formats.append({
+                        'url': url,
+                        'ext': stream.get('delivery_type'),
+                        'vcodec': stream.get('video_codec'),
+                        'format_id': stream.get('profile'),
+                        'width': int_or_none(stream.get('width')),
+                        'height': int_or_none(stream.get('height')),
+                        'abr': int_or_none(stream.get('audio_bitrate')),
+                        'vbr': int_or_none(stream.get('video_bitrate')),
+                        'fps': float_or_none(stream.get('framerate')),
+                    })
+        self._sort_formats(formats)
 
-            if not cur_auth_data['authorized']:
-                raise ExtractorError(cur_auth_data['message'], expected=True)
-
-        if not videos_info:
-            raise ExtractorError('Unable to extract info')
-        videos_info = videos_info.replace('\\"', '"')
-        videos_more_info = self._search_regex(
-            r'eval\("\(({.*?\\"promo\\".*?})\)"', mobile_player, 'more info').replace('\\"', '"')
-        videos_info = json.loads(videos_info)
-        videos_more_info = json.loads(videos_more_info)
-
-        if videos_more_info.get('lineup'):
-            videos = [self._extract_result(info, more_info) for (info, more_info) in zip(videos_info, videos_more_info['lineup'])]
-            return {
-                '_type': 'playlist',
-                'id': video_id,
-                'title': unescapeHTML(videos_more_info['title']),
-                'entries': videos,
-            }
-        else:
-            return self._extract_result(videos_info[0], videos_more_info)
+        video_info['formats'] = formats
+        return video_info
 
 
 class OoyalaIE(OoyalaBaseIE):
@@ -117,6 +73,7 @@ class OoyalaIE(OoyalaBaseIE):
                 'ext': 'mp4',
                 'title': 'Explaining Data Recovery from Hard Drives and SSDs',
                 'description': 'How badly damaged does a drive have to be to defeat Russell and his crew? Apparently, smashed to bits.',
+                'duration': 853386,
             },
         }, {
             # Only available for ipad
@@ -125,7 +82,7 @@ class OoyalaIE(OoyalaBaseIE):
                 'id': 'x1b3lqZDq9y_7kMyC2Op5qo-p077tXD0',
                 'ext': 'mp4',
                 'title': 'Simulation Overview - Levels of Simulation',
-                'description': '',
+                'duration': 194948,
             },
         },
         {
@@ -136,7 +93,8 @@ class OoyalaIE(OoyalaBaseIE):
             'info_dict': {
                 'id': 'FiOG81ZTrvckcchQxmalf4aQj590qTEx',
                 'ext': 'mp4',
-                'title': 'Ooyala video',
+                'title': 'Divide Tool Path.mp4',
+                'duration': 204405,
             }
         }
     ]
@@ -152,8 +110,8 @@ class OoyalaIE(OoyalaBaseIE):
 
     def _real_extract(self, url):
         embed_code = self._match_id(url)
-        player_url = 'http://player.ooyala.com/player.js?embedCode=%s' % embed_code
-        return self._extract(player_url, embed_code)
+        content_tree_url = 'http://player.ooyala.com/player_api/v1/content_tree/embed_code/%s/%s' % (embed_code, embed_code)
+        return self._extract(content_tree_url, embed_code)
 
 
 class OoyalaExternalIE(OoyalaBaseIE):
@@ -170,7 +128,7 @@ class OoyalaExternalIE(OoyalaBaseIE):
                         .*?&pcode=
                     )
                     (?P<pcode>.+?)
-                    (&|$)
+                    (?:&|$)
                     '''
 
     _TEST = {
@@ -179,7 +137,7 @@ class OoyalaExternalIE(OoyalaBaseIE):
             'id': 'FkYWtmazr6Ed8xmvILvKLWjd4QvYZpzG',
             'ext': 'mp4',
             'title': 'dm_140128_30for30Shorts___JudgingJewellv2',
-            'description': '',
+            'duration': 1302000,
         },
         'params': {
             # m3u8 download
@@ -188,9 +146,6 @@ class OoyalaExternalIE(OoyalaBaseIE):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        partner_id = mobj.group('partner_id')
-        video_id = mobj.group('id')
-        pcode = mobj.group('pcode')
-        player_url = 'http://player.ooyala.com/player.js?externalId=%s:%s&pcode=%s' % (partner_id, video_id, pcode)
-        return self._extract(player_url, video_id)
+        partner_id, video_id, pcode = re.match(self._VALID_URL, url).groups()
+        content_tree_url = 'http://player.ooyala.com/player_api/v1/content_tree/external_id/%s/%s:%s' % (pcode, partner_id, video_id)
+        return self._extract(content_tree_url, video_id)

From 497ca088a60fdd0a98f16e22a9d4fec135a26ab0 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 15 Oct 2015 14:37:05 +0100
Subject: [PATCH 2798/6219] [ooyala] remove print statment

---
 youtube_dl/extractor/ooyala.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/ooyala.py b/youtube_dl/extractor/ooyala.py
index 592cdc564..df99a39f4 100644
--- a/youtube_dl/extractor/ooyala.py
+++ b/youtube_dl/extractor/ooyala.py
@@ -12,7 +12,6 @@ from ..utils import (
 class OoyalaBaseIE(InfoExtractor):
 
     def _extract(self, player_url, video_id):
-        print(player_url)
         content_tree = self._download_json(player_url, video_id)['content_tree']
         metadata = content_tree[list(content_tree)[0]]
         embed_code = metadata['embed_code']

From 77302fe5c989b9cafcb675c0a03642b80fa557ff Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 15 Oct 2015 23:27:46 +0100
Subject: [PATCH 2799/6219] [bliptv] remove extractor and add support for site
 replacement(makertv)

---
 youtube_dl/extractor/__init__.py     |   3 +-
 youtube_dl/extractor/bliptv.py       | 292 ---------------------------
 youtube_dl/extractor/cinemassacre.py |  18 +-
 youtube_dl/extractor/generic.py      |   6 -
 youtube_dl/extractor/jwplatform.py   |  67 ++++++
 youtube_dl/extractor/makertv.py      |  27 +++
 6 files changed, 103 insertions(+), 310 deletions(-)
 delete mode 100644 youtube_dl/extractor/bliptv.py
 create mode 100644 youtube_dl/extractor/jwplatform.py
 create mode 100644 youtube_dl/extractor/makertv.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 462717b1e..f9c40e6cd 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -54,7 +54,6 @@ from .bet import BetIE
 from .bild import BildIE
 from .bilibili import BiliBiliIE
 from .blinkx import BlinkxIE
-from .bliptv import BlipTVIE, BlipTVUserIE
 from .bloomberg import BloombergIE
 from .bpb import BpbIE
 from .br import BRIE
@@ -263,6 +262,7 @@ from .jadorecettepub import JadoreCettePubIE
 from .jeuxvideo import JeuxVideoIE
 from .jove import JoveIE
 from .jukebox import JukeboxIE
+from .jwplatform import JWPlatformIE
 from .jpopsukitv import JpopsukiIE
 from .kaltura import KalturaIE
 from .kanalplay import KanalPlayIE
@@ -317,6 +317,7 @@ from .lynda import (
 from .m6 import M6IE
 from .macgamestore import MacGameStoreIE
 from .mailru import MailRuIE
+from .makertv import MakerTVIE
 from .malemotion import MalemotionIE
 from .mdr import MDRIE
 from .metacafe import MetacafeIE
diff --git a/youtube_dl/extractor/bliptv.py b/youtube_dl/extractor/bliptv.py
deleted file mode 100644
index c3296283d..000000000
--- a/youtube_dl/extractor/bliptv.py
+++ /dev/null
@@ -1,292 +0,0 @@
-from __future__ import unicode_literals
-
-import re
-
-from .common import InfoExtractor
-
-from ..compat import (
-    compat_urllib_request,
-    compat_urlparse,
-)
-from ..utils import (
-    clean_html,
-    int_or_none,
-    parse_iso8601,
-    unescapeHTML,
-    xpath_text,
-    xpath_with_ns,
-)
-
-
-class BlipTVIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:\w+\.)?blip\.tv/(?:(?:.+-|rss/flash/)(?P<id>\d+)|((?:play/|api\.swf#)(?P<lookup_id>[\da-zA-Z+_]+)))'
-
-    _TESTS = [
-        {
-            'url': 'http://blip.tv/cbr/cbr-exclusive-gotham-city-imposters-bats-vs-jokerz-short-3-5796352',
-            'md5': '80baf1ec5c3d2019037c1c707d676b9f',
-            'info_dict': {
-                'id': '5779306',
-                'ext': 'm4v',
-                'title': 'CBR EXCLUSIVE: "Gotham City Imposters" Bats VS Jokerz Short 3',
-                'description': 'md5:9bc31f227219cde65e47eeec8d2dc596',
-                'timestamp': 1323138843,
-                'upload_date': '20111206',
-                'uploader': 'cbr',
-                'uploader_id': '679425',
-                'duration': 81,
-            }
-        },
-        {
-            # https://github.com/rg3/youtube-dl/pull/2274
-            'note': 'Video with subtitles',
-            'url': 'http://blip.tv/play/h6Uag5OEVgI.html',
-            'md5': '309f9d25b820b086ca163ffac8031806',
-            'info_dict': {
-                'id': '6586561',
-                'ext': 'mp4',
-                'title': 'Red vs. Blue Season 11 Episode 1',
-                'description': 'One-Zero-One',
-                'timestamp': 1371261608,
-                'upload_date': '20130615',
-                'uploader': 'redvsblue',
-                'uploader_id': '792887',
-                'duration': 279,
-            }
-        },
-        {
-            # https://bugzilla.redhat.com/show_bug.cgi?id=967465
-            'url': 'http://a.blip.tv/api.swf#h6Uag5KbVwI',
-            'md5': '314e87b1ebe7a48fcbfdd51b791ce5a6',
-            'info_dict': {
-                'id': '6573122',
-                'ext': 'mov',
-                'upload_date': '20130520',
-                'description': 'Two hapless space marines argue over what to do when they realize they have an astronomically huge problem on their hands.',
-                'title': 'Red vs. Blue Season 11 Trailer',
-                'timestamp': 1369029609,
-                'uploader': 'redvsblue',
-                'uploader_id': '792887',
-            }
-        },
-        {
-            'url': 'http://blip.tv/play/gbk766dkj4Yn',
-            'md5': 'fe0a33f022d49399a241e84a8ea8b8e3',
-            'info_dict': {
-                'id': '1749452',
-                'ext': 'mp4',
-                'upload_date': '20090208',
-                'description': 'Witness the first appearance of the Nostalgia Critic character, as Doug reviews the movie Transformers.',
-                'title': 'Nostalgia Critic: Transformers',
-                'timestamp': 1234068723,
-                'uploader': 'NostalgiaCritic',
-                'uploader_id': '246467',
-            }
-        },
-        {
-            # https://github.com/rg3/youtube-dl/pull/4404
-            'note': 'Audio only',
-            'url': 'http://blip.tv/hilarios-productions/weekly-manga-recap-kingdom-7119982',
-            'md5': '76c0a56f24e769ceaab21fbb6416a351',
-            'info_dict': {
-                'id': '7103299',
-                'ext': 'flv',
-                'title': 'Weekly Manga Recap: Kingdom',
-                'description': 'And then Shin breaks the enemy line, and he&apos;s all like HWAH! And then he slices a guy and it&apos;s all like FWASHING! And... it&apos;s really hard to describe the best parts of this series without breaking down into sound effects, okay?',
-                'timestamp': 1417660321,
-                'upload_date': '20141204',
-                'uploader': 'The Rollo T',
-                'uploader_id': '407429',
-                'duration': 7251,
-                'vcodec': 'none',
-            }
-        },
-        {
-            # missing duration
-            'url': 'http://blip.tv/rss/flash/6700880',
-            'info_dict': {
-                'id': '6684191',
-                'ext': 'm4v',
-                'title': 'Cowboy Bebop: Gateway Shuffle Review',
-                'description': 'md5:3acc480c0f9ae157f5fe88547ecaf3f8',
-                'timestamp': 1386639757,
-                'upload_date': '20131210',
-                'uploader': 'sfdebris',
-                'uploader_id': '706520',
-            }
-        }
-    ]
-
-    @staticmethod
-    def _extract_url(webpage):
-        mobj = re.search(r'<meta\s[^>]*https?://api\.blip\.tv/\w+/redirect/\w+/(\d+)', webpage)
-        if mobj:
-            return 'http://blip.tv/a/a-' + mobj.group(1)
-        mobj = re.search(r'<(?:iframe|embed|object)\s[^>]*(https?://(?:\w+\.)?blip\.tv/(?:play/|api\.swf#)[a-zA-Z0-9_]+)', webpage)
-        if mobj:
-            return mobj.group(1)
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        lookup_id = mobj.group('lookup_id')
-
-        # See https://github.com/rg3/youtube-dl/issues/857 and
-        # https://github.com/rg3/youtube-dl/issues/4197
-        if lookup_id:
-            urlh = self._request_webpage(
-                'http://blip.tv/play/%s' % lookup_id, lookup_id, 'Resolving lookup id')
-            url = compat_urlparse.urlparse(urlh.geturl())
-            qs = compat_urlparse.parse_qs(url.query)
-            mobj = re.match(self._VALID_URL, qs['file'][0])
-
-        video_id = mobj.group('id')
-
-        rss = self._download_xml('http://blip.tv/rss/flash/%s' % video_id, video_id, 'Downloading video RSS')
-
-        def _x(p):
-            return xpath_with_ns(p, {
-                'blip': 'http://blip.tv/dtd/blip/1.0',
-                'media': 'http://search.yahoo.com/mrss/',
-                'itunes': 'http://www.itunes.com/dtds/podcast-1.0.dtd',
-            })
-
-        item = rss.find('channel/item')
-
-        video_id = xpath_text(item, _x('blip:item_id'), 'video id') or lookup_id
-        title = xpath_text(item, 'title', 'title', fatal=True)
-        description = clean_html(xpath_text(item, _x('blip:puredescription'), 'description'))
-        timestamp = parse_iso8601(xpath_text(item, _x('blip:datestamp'), 'timestamp'))
-        uploader = xpath_text(item, _x('blip:user'), 'uploader')
-        uploader_id = xpath_text(item, _x('blip:userid'), 'uploader id')
-        duration = int_or_none(xpath_text(item, _x('blip:runtime'), 'duration'))
-        media_thumbnail = item.find(_x('media:thumbnail'))
-        thumbnail = (media_thumbnail.get('url') if media_thumbnail is not None
-                     else xpath_text(item, 'image', 'thumbnail'))
-        categories = [category.text for category in item.findall('category') if category is not None]
-
-        formats = []
-        subtitles_urls = {}
-
-        media_group = item.find(_x('media:group'))
-        for media_content in media_group.findall(_x('media:content')):
-            url = media_content.get('url')
-            role = media_content.get(_x('blip:role'))
-            msg = self._download_webpage(
-                url + '?showplayer=20140425131715&referrer=http://blip.tv&mask=7&skin=flashvars&view=url',
-                video_id, 'Resolving URL for %s' % role)
-            real_url = compat_urlparse.parse_qs(msg.strip())['message'][0]
-
-            media_type = media_content.get('type')
-            if media_type == 'text/srt' or url.endswith('.srt'):
-                LANGS = {
-                    'english': 'en',
-                }
-                lang = role.rpartition('-')[-1].strip().lower()
-                langcode = LANGS.get(lang, lang)
-                subtitles_urls[langcode] = url
-            elif media_type.startswith('video/'):
-                formats.append({
-                    'url': real_url,
-                    'format_id': role,
-                    'format_note': media_type,
-                    'vcodec': media_content.get(_x('blip:vcodec')) or 'none',
-                    'acodec': media_content.get(_x('blip:acodec')),
-                    'filesize': media_content.get('filesize'),
-                    'width': int_or_none(media_content.get('width')),
-                    'height': int_or_none(media_content.get('height')),
-                })
-        self._check_formats(formats, video_id)
-        self._sort_formats(formats)
-
-        subtitles = self.extract_subtitles(video_id, subtitles_urls)
-
-        return {
-            'id': video_id,
-            'title': title,
-            'description': description,
-            'timestamp': timestamp,
-            'uploader': uploader,
-            'uploader_id': uploader_id,
-            'duration': duration,
-            'thumbnail': thumbnail,
-            'categories': categories,
-            'formats': formats,
-            'subtitles': subtitles,
-        }
-
-    def _get_subtitles(self, video_id, subtitles_urls):
-        subtitles = {}
-        for lang, url in subtitles_urls.items():
-            # For some weird reason, blip.tv serves a video instead of subtitles
-            # when we request with a common UA
-            req = compat_urllib_request.Request(url)
-            req.add_header('User-Agent', 'youtube-dl')
-            subtitles[lang] = [{
-                # The extension is 'srt' but it's actually an 'ass' file
-                'ext': 'ass',
-                'data': self._download_webpage(req, None, note=False),
-            }]
-        return subtitles
-
-
-class BlipTVUserIE(InfoExtractor):
-    _VALID_URL = r'(?:(?:https?://(?:\w+\.)?blip\.tv/)|bliptvuser:)(?!api\.swf)([^/]+)/*$'
-    _PAGE_SIZE = 12
-    IE_NAME = 'blip.tv:user'
-    _TEST = {
-        'url': 'http://blip.tv/actone',
-        'info_dict': {
-            'id': 'actone',
-            'title': 'Act One: The Series',
-        },
-        'playlist_count': 5,
-    }
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        username = mobj.group(1)
-
-        page_base = 'http://m.blip.tv/pr/show_get_full_episode_list?users_id=%s&lite=0&esi=1'
-
-        page = self._download_webpage(url, username, 'Downloading user page')
-        mobj = re.search(r'data-users-id="([^"]+)"', page)
-        page_base = page_base % mobj.group(1)
-        title = self._og_search_title(page)
-
-        # Download video ids using BlipTV Ajax calls. Result size per
-        # query is limited (currently to 12 videos) so we need to query
-        # page by page until there are no video ids - it means we got
-        # all of them.
-
-        video_ids = []
-        pagenum = 1
-
-        while True:
-            url = page_base + "&page=" + str(pagenum)
-            page = self._download_webpage(
-                url, username, 'Downloading video ids from page %d' % pagenum)
-
-            # Extract video identifiers
-            ids_in_page = []
-
-            for mobj in re.finditer(r'href="/([^"]+)"', page):
-                if mobj.group(1) not in ids_in_page:
-                    ids_in_page.append(unescapeHTML(mobj.group(1)))
-
-            video_ids.extend(ids_in_page)
-
-            # A little optimization - if current page is not
-            # "full", ie. does not contain PAGE_SIZE video ids then
-            # we can assume that this page is the last one - there
-            # are no more ids on further pages - no need to query
-            # again.
-
-            if len(ids_in_page) < self._PAGE_SIZE:
-                break
-
-            pagenum += 1
-
-        urls = ['http://blip.tv/%s' % video_id for video_id in video_ids]
-        url_entries = [self.url_result(vurl, 'BlipTV') for vurl in urls]
-        return self.playlist_result(
-            url_entries, playlist_title=title, playlist_id=username)
diff --git a/youtube_dl/extractor/cinemassacre.py b/youtube_dl/extractor/cinemassacre.py
index fd1770dac..6d9cd8abd 100644
--- a/youtube_dl/extractor/cinemassacre.py
+++ b/youtube_dl/extractor/cinemassacre.py
@@ -5,7 +5,6 @@ import re
 
 from .common import InfoExtractor
 from ..utils import ExtractorError
-from .bliptv import BlipTVIE
 from .screenwavemedia import ScreenwaveMediaIE
 
 
@@ -34,18 +33,17 @@ class CinemassacreIE(InfoExtractor):
             },
         },
         {
-            # blip.tv embedded video
+            # Youtube embedded video
             'url': 'http://cinemassacre.com/2006/12/07/chronologically-confused-about-bad-movie-and-video-game-sequel-titles/',
-            'md5': 'ca9b3c8dd5a66f9375daeb5135f5a3de',
+            'md5': 'df4cf8a1dcedaec79a73d96d83b99023',
             'info_dict': {
-                'id': '4065369',
-                'ext': 'flv',
+                'id': 'OEVzPCY2T-g',
+                'ext': 'mp4',
                 'title': 'AVGN: Chronologically Confused about Bad Movie and Video Game Sequel Titles',
                 'upload_date': '20061207',
-                'uploader': 'cinemassacre',
-                'uploader_id': '250778',
-                'timestamp': 1283233867,
-                'description': 'md5:0a108c78d130676b207d0f6d029ecffd',
+                'uploader': 'Cinemassacre',
+                'uploader_id': 'JamesNintendoNerd',
+                'description': 'md5:784734696c2b8b7f4b8625cc799e07f6',
             }
         },
         {
@@ -88,8 +86,6 @@ class CinemassacreIE(InfoExtractor):
                 r'<iframe[^>]+src="(?P<url>(?:https?:)?//(?:[^.]+\.)?youtube\.com/.+?)"',
             ],
             webpage, 'player data URL', default=None, group='url')
-        if not playerdata_url:
-            playerdata_url = BlipTVIE._extract_url(webpage)
         if not playerdata_url:
             raise ExtractorError('Unable to find player data')
 
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index ca5fbafb2..285c0ff66 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -41,7 +41,6 @@ from .myvi import MyviIE
 from .condenast import CondeNastIE
 from .udn import UDNEmbedIE
 from .senateisvp import SenateISVPIE
-from .bliptv import BlipTVIE
 from .svt import SVTIE
 from .pornhub import PornHubIE
 from .xhamster import XHamsterEmbedIE
@@ -1389,11 +1388,6 @@ class GenericIE(InfoExtractor):
                 'id': match.group('id')
             }
 
-        # Look for embedded blip.tv player
-        bliptv_url = BlipTVIE._extract_url(webpage)
-        if bliptv_url:
-            return self.url_result(bliptv_url, 'BlipTV')
-
         # Look for SVT player
         svt_url = SVTIE._extract_url(webpage)
         if svt_url:
diff --git a/youtube_dl/extractor/jwplatform.py b/youtube_dl/extractor/jwplatform.py
new file mode 100644
index 000000000..3a3dc439a
--- /dev/null
+++ b/youtube_dl/extractor/jwplatform.py
@@ -0,0 +1,67 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import int_or_none
+
+
+class JWPlatformIE(InfoExtractor):
+    _VALID_URL = r'(?:https?://content\.jwplatform\.com/(?:feeds|players|jw6)/|jwplatform:)(?P<id>[a-zA-Z0-9]{8})'
+    _TEST = {
+        'url': 'http://content.jwplatform.com/players/nPripu9l-ALJ3XQCI.js',
+        'md5': 'fa8899fa601eb7c83a64e9d568bdf325',
+        'info_dict': {
+            'id': 'nPripu9l',
+            'ext': 'mov',
+            'title': 'Big Buck Bunny Trailer',
+            'description': 'Big Buck Bunny is a short animated film by the Blender Institute. It is made using free and open source software.',
+            'upload_date': '20081127',
+            'timestamp': 1227796140,
+        }
+    }
+
+    @staticmethod
+    def _extract_url(webpage):
+        mobj = re.search(
+            r'<script[^>]+?src=["\'](?P<url>(?:https?:)?//content.jwplatform.com/players/[a-zA-Z0-9]{8}',
+            webpage)
+        if mobj:
+            return mobj.group('url')
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        json_data = self._download_json('http://content.jwplatform.com/feeds/%s.json' % video_id, video_id)
+        video_data = json_data['playlist'][0]
+        subtitles = {}
+        for track in video_data['tracks']:
+            if track['kind'] == 'captions':
+                subtitles[track['label']] = [{'url': self._proto_relative_url(track['file'])}]
+
+        formats = []
+        for source in video_data['sources']:
+            source_url = self._proto_relative_url(source['file'])
+            source_type = source.get('type') or ''
+            if source_type == 'application/vnd.apple.mpegurl':
+                formats.extend(self._extract_m3u8_formats(source_url, video_id, 'mp4', 'm3u8_native', fatal=None))
+            elif source_type.startswith('audio'):
+                formats.append({
+                    'url': source_url,
+                    'vcodec': 'none',
+                })
+            else:
+                formats.append({
+                    'url': source_url,
+                    'width': int_or_none(source.get('width')),
+                    'height': int_or_none(source.get('height')),
+                })
+        self._sort_formats(formats)
+
+        return {
+            'id': video_data['mediaid'],
+            'title': video_data['title'],
+            'description': video_data.get('description'),
+            'thumbnail': self._proto_relative_url(video_data.get('image')),
+            'timestamp': int_or_none(video_data.get('pubdate')),
+            'subtitles': subtitles,
+            'formats': formats,
+        }
diff --git a/youtube_dl/extractor/makertv.py b/youtube_dl/extractor/makertv.py
new file mode 100644
index 000000000..0256e4e24
--- /dev/null
+++ b/youtube_dl/extractor/makertv.py
@@ -0,0 +1,27 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class MakerTVIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:(?:www\.)?maker\.tv/(?:[^/]+/)?video|http://makerplayer.com/embed/maker)/(?P<id>[a-zA-Z0-9]{12})'
+    _TEST = {
+        'url': 'http://www.maker.tv/video/Fh3QgymL9gsc',
+        'md5': 'ca237a53a8eb20b6dc5bd60564d4ab3e',
+        'info_dict': {
+            'id': 'brOEcGut',
+            'ext': 'mp4',
+            'title': 'Maze Runner: The Scorch Trials Official Movie Review',
+            'description': 'md5:11ff3362d7ef1d679fdb649f6413975a',
+            'upload_date': '20150918',
+            'timestamp': 1442549540,
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        jwplatform_id = self._search_regex([r'jwid="([^"]+)"', r'Maker.jw_id\s*=\s*"([^"]+)";'], webpage, 'jwplatform id')
+
+        return self.url_result('jwplatform:%s' % jwplatform_id, 'JWPlatform')

From 6744f36db710eebe2ccc633e7f4f6132b968b0ec Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 16 Oct 2015 08:44:19 +0600
Subject: [PATCH 2800/6219] [jeuxvideo] Fallback on og:title (Closes #7186,
 closes #7190)

---
 youtube_dl/extractor/jeuxvideo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/jeuxvideo.py b/youtube_dl/extractor/jeuxvideo.py
index 1df084d87..eef7daa29 100644
--- a/youtube_dl/extractor/jeuxvideo.py
+++ b/youtube_dl/extractor/jeuxvideo.py
@@ -28,7 +28,7 @@ class JeuxVideoIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         title = mobj.group(1)
         webpage = self._download_webpage(url, title)
-        title = self._html_search_meta('name', webpage)
+        title = self._html_search_meta('name', webpage) or self._og_search_title(webpage)
         config_url = self._html_search_regex(
             r'data-src="(/contenu/medias/video.php.*?)"',
             webpage, 'config URL')

From dd414c970bcc493358ff6a76f6544a0417125594 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 16 Oct 2015 10:12:42 +0100
Subject: [PATCH 2801/6219] [ooyala] fix sorting and format id

---
 youtube_dl/extractor/ooyala.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/ooyala.py b/youtube_dl/extractor/ooyala.py
index df99a39f4..075b594ce 100644
--- a/youtube_dl/extractor/ooyala.py
+++ b/youtube_dl/extractor/ooyala.py
@@ -39,15 +39,15 @@ class OoyalaBaseIE(InfoExtractor):
                     video_info['url'] = url
                     return video_info
                 if delivery_type == 'hls':
-                    formats.extend(self._extract_m3u8_formats(url, embed_code, 'mp4', 'm3u8_native', 0, m3u8_id='hls', fatal=False))
+                    formats.extend(self._extract_m3u8_formats(url, embed_code, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
                 elif delivery_type == 'hds':
-                    formats.extend(self._extract_f4m_formats(url, embed_code, f4m_id='hds', fatal=False))
+                    formats.extend(self._extract_f4m_formats(url, embed_code, -1, 'hds', fatal=False))
                 else:
                     formats.append({
                         'url': url,
                         'ext': stream.get('delivery_type'),
                         'vcodec': stream.get('video_codec'),
-                        'format_id': stream.get('profile'),
+                        'format_id': '%s-%s-%sp' % (stream.get('profile'), delivery_type, stream.get('height')),
                         'width': int_or_none(stream.get('width')),
                         'height': int_or_none(stream.get('height')),
                         'abr': int_or_none(stream.get('audio_bitrate')),

From cce9d15d0115e8b4cd1f6e2a327b5e9dbdf0ee54 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 16 Oct 2015 16:02:40 +0100
Subject: [PATCH 2802/6219] [ooyala] extract domain,handle errors and change
 related tests

---
 youtube_dl/extractor/byutv.py           |  5 ++-
 youtube_dl/extractor/generic.py         |  9 ++--
 youtube_dl/extractor/groupon.py         |  2 +
 youtube_dl/extractor/howcast.py         |  1 +
 youtube_dl/extractor/ooyala.py          | 60 ++++++++++++++-----------
 youtube_dl/extractor/teachingchannel.py |  1 +
 youtube_dl/extractor/vice.py            |  1 +
 7 files changed, 48 insertions(+), 31 deletions(-)

diff --git a/youtube_dl/extractor/byutv.py b/youtube_dl/extractor/byutv.py
index 3b2de517e..ce25816f0 100644
--- a/youtube_dl/extractor/byutv.py
+++ b/youtube_dl/extractor/byutv.py
@@ -14,9 +14,10 @@ class BYUtvIE(InfoExtractor):
         'info_dict': {
             'id': 'studio-c-season-5-episode-5',
             'ext': 'mp4',
-            'description': 'md5:5438d33774b6bdc662f9485a340401cc',
+            'description': 'md5:e07269172baff037f8e8bf9956bc9747',
             'title': 'Season 5 Episode 5',
-            'thumbnail': 're:^https?://.*\.jpg$'
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 1486486,
         },
         'params': {
             'skip_download': True,
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index ca5fbafb2..805677364 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -335,6 +335,7 @@ class GenericIE(InfoExtractor):
                 'id': 'BwY2RxaTrTkslxOfcan0UCf0YqyvWysJ',
                 'ext': 'mp4',
                 'title': '2cc213299525360.mov',  # that's what we get
+                'duration': 238231,
             },
             'add_ie': ['Ooyala'],
         },
@@ -346,6 +347,7 @@ class GenericIE(InfoExtractor):
                 'ext': 'mp4',
                 'title': '"Steve Jobs: Man in the Machine" trailer',
                 'description': 'The first trailer for the Alex Gibney documentary "Steve Jobs: Man in the Machine."',
+                'duration': 135427,
             },
             'params': {
                 'skip_download': True,
@@ -943,8 +945,9 @@ class GenericIE(InfoExtractor):
             'info_dict': {
                 'id': '50YnY4czr4ms1vJ7yz3xzq0excz_pUMs',
                 'ext': 'mp4',
-                'description': 'VIDEO: Index/Match versus VLOOKUP.',
+                'description': 'VIDEO: INDEX/MATCH versus VLOOKUP.',
                 'title': 'This is what separates the Excel masters from the wannabes',
+                'duration': 191933,
             },
             'params': {
                 # m3u8 downloads
@@ -1454,7 +1457,7 @@ class GenericIE(InfoExtractor):
                 re.search(r'SBN\.VideoLinkset\.ooyala\([\'"](?P<ec>.{32})[\'"]\)', webpage) or
                 re.search(r'data-ooyala-video-id\s*=\s*[\'"](?P<ec>.{32})[\'"]', webpage))
         if mobj is not None:
-            return OoyalaIE._build_url_result(mobj.group('ec'))
+            return OoyalaIE._build_url_result(smuggle_url(mobj.group('ec'), {'domain': url}))
 
         # Look for multiple Ooyala embeds on SBN network websites
         mobj = re.search(r'SBN\.VideoLinkset\.entryGroup\((\[.*?\])', webpage)
@@ -1462,7 +1465,7 @@ class GenericIE(InfoExtractor):
             embeds = self._parse_json(mobj.group(1), video_id, fatal=False)
             if embeds:
                 return _playlist_from_matches(
-                    embeds, getter=lambda v: OoyalaIE._url_for_embed_code(v['provider_video_id']), ie='Ooyala')
+                    embeds, getter=lambda v: OoyalaIE._url_for_embed_code(smuggle_url(v['provider_video_id'], {'domain': url})), ie='Ooyala')
 
         # Look for Aparat videos
         mobj = re.search(r'<iframe .*?src="(http://www\.aparat\.com/video/[^"]+)"', webpage)
diff --git a/youtube_dl/extractor/groupon.py b/youtube_dl/extractor/groupon.py
index 8b9e0e2f8..22ff7182f 100644
--- a/youtube_dl/extractor/groupon.py
+++ b/youtube_dl/extractor/groupon.py
@@ -18,6 +18,8 @@ class GrouponIE(InfoExtractor):
                 'id': 'tubGNycTo_9Uxg82uESj4i61EYX8nyuf',
                 'ext': 'mp4',
                 'title': 'Bikram Yoga Huntington Beach | Orange County',
+                'description': 'md5:d41d8cd98f00b204e9800998ecf8427e',
+                'duration': 44961,
             },
         }],
         'params': {
diff --git a/youtube_dl/extractor/howcast.py b/youtube_dl/extractor/howcast.py
index 16677f179..165b9f39e 100644
--- a/youtube_dl/extractor/howcast.py
+++ b/youtube_dl/extractor/howcast.py
@@ -16,6 +16,7 @@ class HowcastIE(InfoExtractor):
             'description': 'md5:dbe792e5f6f1489027027bf2eba188a3',
             'timestamp': 1276081287,
             'upload_date': '20100609',
+            'duration': 56823,
         },
         'params': {
             # m3u8 download
diff --git a/youtube_dl/extractor/ooyala.py b/youtube_dl/extractor/ooyala.py
index 075b594ce..3b692e903 100644
--- a/youtube_dl/extractor/ooyala.py
+++ b/youtube_dl/extractor/ooyala.py
@@ -6,13 +6,16 @@ from .common import InfoExtractor
 from ..utils import (
     int_or_none,
     float_or_none,
+    ExtractorError,
+    unsmuggle_url,
 )
+from ..compat import compat_urllib_parse
 
 
 class OoyalaBaseIE(InfoExtractor):
 
-    def _extract(self, player_url, video_id):
-        content_tree = self._download_json(player_url, video_id)['content_tree']
+    def _extract(self, content_tree_url, video_id, domain='example.org'):
+        content_tree = self._download_json(content_tree_url, video_id)['content_tree']
         metadata = content_tree[list(content_tree)[0]]
         embed_code = metadata['embed_code']
         pcode = metadata.get('asset_pcode') or embed_code
@@ -27,33 +30,36 @@ class OoyalaBaseIE(InfoExtractor):
         formats = []
         for supported_format in ('mp4', 'm3u8', 'hds', 'rtmp'):
             auth_data = self._download_json(
-                'http://player.ooyala.com/sas/player_api/v1/authorization/embed_code/%s/%s?domain=www.example.org&supportedFormats=%s' % (pcode, embed_code, supported_format),
+                'http://player.ooyala.com/sas/player_api/v1/authorization/embed_code/%s/%s?' % (pcode, embed_code) + compat_urllib_parse.urlencode({'domain': domain, 'supportedFormats': supported_format}),
                 video_id, 'Downloading %s JSON' % supported_format)
 
             cur_auth_data = auth_data['authorization_data'][embed_code]
 
-            for stream in cur_auth_data['streams']:
-                url = base64.b64decode(stream['url']['data'].encode('ascii')).decode('utf-8')
-                delivery_type = stream['delivery_type']
-                if delivery_type == 'remote_asset':
-                    video_info['url'] = url
-                    return video_info
-                if delivery_type == 'hls':
-                    formats.extend(self._extract_m3u8_formats(url, embed_code, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
-                elif delivery_type == 'hds':
-                    formats.extend(self._extract_f4m_formats(url, embed_code, -1, 'hds', fatal=False))
-                else:
-                    formats.append({
-                        'url': url,
-                        'ext': stream.get('delivery_type'),
-                        'vcodec': stream.get('video_codec'),
-                        'format_id': '%s-%s-%sp' % (stream.get('profile'), delivery_type, stream.get('height')),
-                        'width': int_or_none(stream.get('width')),
-                        'height': int_or_none(stream.get('height')),
-                        'abr': int_or_none(stream.get('audio_bitrate')),
-                        'vbr': int_or_none(stream.get('video_bitrate')),
-                        'fps': float_or_none(stream.get('framerate')),
-                    })
+            if cur_auth_data['authorized']:
+                for stream in cur_auth_data['streams']:
+                    url = base64.b64decode(stream['url']['data'].encode('ascii')).decode('utf-8')
+                    delivery_type = stream['delivery_type']
+                    if delivery_type == 'remote_asset':
+                        video_info['url'] = url
+                        return video_info
+                    if delivery_type == 'hls':
+                        formats.extend(self._extract_m3u8_formats(url, embed_code, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
+                    elif delivery_type == 'hds':
+                        formats.extend(self._extract_f4m_formats(url, embed_code, -1, 'hds', fatal=False))
+                    else:
+                        formats.append({
+                            'url': url,
+                            'ext': stream.get('delivery_type'),
+                            'vcodec': stream.get('video_codec'),
+                            'format_id': '%s-%s-%sp' % (stream.get('profile'), delivery_type, stream.get('height')),
+                            'width': int_or_none(stream.get('width')),
+                            'height': int_or_none(stream.get('height')),
+                            'abr': int_or_none(stream.get('audio_bitrate')),
+                            'vbr': int_or_none(stream.get('video_bitrate')),
+                            'fps': float_or_none(stream.get('framerate')),
+                        })
+            else:
+                raise ExtractorError('%s said: %s' % (self.IE_NAME, cur_auth_data['message']), expected=True)
         self._sort_formats(formats)
 
         video_info['formats'] = formats
@@ -108,9 +114,11 @@ class OoyalaIE(OoyalaBaseIE):
                               ie=cls.ie_key())
 
     def _real_extract(self, url):
+        url, smuggled_data = unsmuggle_url(url, {})
         embed_code = self._match_id(url)
+        domain = smuggled_data.get('domain')
         content_tree_url = 'http://player.ooyala.com/player_api/v1/content_tree/embed_code/%s/%s' % (embed_code, embed_code)
-        return self._extract(content_tree_url, embed_code)
+        return self._extract(content_tree_url, embed_code, domain)
 
 
 class OoyalaExternalIE(OoyalaBaseIE):
diff --git a/youtube_dl/extractor/teachingchannel.py b/youtube_dl/extractor/teachingchannel.py
index 117afa9bf..36a6fc679 100644
--- a/youtube_dl/extractor/teachingchannel.py
+++ b/youtube_dl/extractor/teachingchannel.py
@@ -16,6 +16,7 @@ class TeachingChannelIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'A History of Teaming',
             'description': 'md5:2a9033db8da81f2edffa4c99888140b3',
+            'duration': 422255,
         },
         'params': {
             # m3u8 download
diff --git a/youtube_dl/extractor/vice.py b/youtube_dl/extractor/vice.py
index 01af7a995..7df87c31c 100644
--- a/youtube_dl/extractor/vice.py
+++ b/youtube_dl/extractor/vice.py
@@ -15,6 +15,7 @@ class ViceIE(InfoExtractor):
                 'id': '43cW1mYzpia9IlestBjVpd23Yu3afAfp',
                 'ext': 'mp4',
                 'title': 'VICE_COWBOYCAPITALISTS_PART01_v1_VICE_WM_1080p.mov',
+                'duration': 725983,
             },
             'params': {
                 # Requires ffmpeg (m3u8 manifest)

From 8daeeedc06f420e2a87ba4755b56e721391cedba Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 17 Oct 2015 00:26:45 +0600
Subject: [PATCH 2803/6219] [bbc] Fix FutureWarning

---
 youtube_dl/extractor/bbc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 68995f81e..1b3a33e4e 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -421,7 +421,7 @@ class BBCCoUkIE(InfoExtractor):
                 continue
             title = playlist.find('./{%s}title' % self._EMP_PLAYLIST_NS).text
             description_el = playlist.find('./{%s}summary' % self._EMP_PLAYLIST_NS)
-            description = description_el.text if description_el else None
+            description = description_el.text if description_el is not None else None
 
             def get_programme_id(item):
                 def get_from_attributes(item):

From 1e52776ac3ebbafc2ec4697f3bc6ba05b7e5a9f8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 17 Oct 2015 00:46:38 +0600
Subject: [PATCH 2804/6219] [bandcamp] Prepend download URL with scheme when
 necessary (2) (#7077)

---
 youtube_dl/extractor/bandcamp.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bandcamp.py b/youtube_dl/extractor/bandcamp.py
index a27f3e748..f19e19001 100644
--- a/youtube_dl/extractor/bandcamp.py
+++ b/youtube_dl/extractor/bandcamp.py
@@ -52,7 +52,7 @@ class BandcampIE(InfoExtractor):
                     ext, abr_str = format_id.split('-', 1)
                     formats.append({
                         'format_id': format_id,
-                        'url': format_url,
+                        'url': self._proto_relative_url(format_url, 'http:'),
                         'ext': ext,
                         'vcodec': 'none',
                         'acodec': ext,

From ba717dca97925a21870fedcb46358d06cd5485ec Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 17 Oct 2015 00:51:35 +0600
Subject: [PATCH 2805/6219] [bandcamp] Modernize

---
 youtube_dl/extractor/bandcamp.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/bandcamp.py b/youtube_dl/extractor/bandcamp.py
index f19e19001..c1ef8051d 100644
--- a/youtube_dl/extractor/bandcamp.py
+++ b/youtube_dl/extractor/bandcamp.py
@@ -10,6 +10,8 @@ from ..compat import (
 )
 from ..utils import (
     ExtractorError,
+    float_or_none,
+    int_or_none,
 )
 
 
@@ -56,7 +58,7 @@ class BandcampIE(InfoExtractor):
                         'ext': ext,
                         'vcodec': 'none',
                         'acodec': ext,
-                        'abr': int(abr_str),
+                        'abr': int_or_none(abr_str),
                     })
 
                 self._sort_formats(formats)
@@ -65,7 +67,7 @@ class BandcampIE(InfoExtractor):
                     'id': compat_str(data['id']),
                     'title': data['title'],
                     'formats': formats,
-                    'duration': float(data['duration']),
+                    'duration': float_or_none(data.get('duration')),
                 }
             else:
                 raise ExtractorError('No free songs found')

From 246ce1085804ead9126328cb1ec761d308f561c0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 17 Oct 2015 01:08:23 +0600
Subject: [PATCH 2806/6219] [rte] Update _VALID_URL (Closes #7198)

---
 youtube_dl/extractor/rte.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/rte.py b/youtube_dl/extractor/rte.py
index 04158b993..427c70866 100644
--- a/youtube_dl/extractor/rte.py
+++ b/youtube_dl/extractor/rte.py
@@ -9,7 +9,7 @@ from ..utils import (
 
 
 class RteIE(InfoExtractor):
-    _VALID_URL = r'http?://(?:www\.)?rte\.ie/player/[^/]{2,3}/show/(?P<id>[0-9]+)/'
+    _VALID_URL = r'http?://(?:www\.)?rte\.ie/player/[^/]{2,3}/show/[^/]+/(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://www.rte.ie/player/de/show/10363114/',
         'info_dict': {

From 2ccb37beb9e35ebbf2cdf65a4c1641e5286de1e8 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 16 Oct 2015 21:40:38 +0200
Subject: [PATCH 2807/6219] release 2015.10.16

---
 docs/supportedsites.md | 2 +-
 youtube_dl/version.py  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index dc0354095..47f7da86d 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -159,6 +159,7 @@
  - **facebook**
  - **faz.net**
  - **fc2**
+ - **Fczenit**
  - **fernsehkritik.tv**
  - **Firstpost**
  - **FiveTV**
@@ -281,7 +282,6 @@
  - **Malemotion**
  - **MDR**
  - **media.ccc.de**
- - **MegaVideoz**
  - **metacafe**
  - **Metacritic**
  - **Mgoon**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index aaa43d315..31d2a9dc0 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.10.13'
+__version__ = '2015.10.16'

From 36eb802baffda9930e8c821e1adf94b0b53b5ac6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 17 Oct 2015 11:49:51 +0200
Subject: [PATCH 2808/6219] [rte] Replace expired test

According to their webpage it should be available until October 2035.
---
 youtube_dl/extractor/rte.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/rte.py b/youtube_dl/extractor/rte.py
index 427c70866..2811767b9 100644
--- a/youtube_dl/extractor/rte.py
+++ b/youtube_dl/extractor/rte.py
@@ -11,14 +11,14 @@ from ..utils import (
 class RteIE(InfoExtractor):
     _VALID_URL = r'http?://(?:www\.)?rte\.ie/player/[^/]{2,3}/show/[^/]+/(?P<id>[0-9]+)'
     _TEST = {
-        'url': 'http://www.rte.ie/player/de/show/10363114/',
+        'url': 'http://www.rte.ie/player/ie/show/iwitness-862/10478715/',
         'info_dict': {
-            'id': '10363114',
+            'id': '10478715',
             'ext': 'mp4',
-            'title': 'One News',
+            'title': 'Watch iWitness  online',
             'thumbnail': 're:^https?://.*\.jpg$',
-            'description': 'The One O\'Clock News followed by Weather.',
-            'duration': 436.844,
+            'description': 'iWitness : The spirit of Ireland, one voice and one minute at a time.',
+            'duration': 60.046,
         },
         'params': {
             'skip_download': 'f4m fails with --test atm'

From 6df7179e6c3c7df165b5788ecb5e712da122356f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 17 Oct 2015 11:53:59 +0200
Subject: [PATCH 2809/6219] [rte] Actually recognize https urls

There was a missing 's' before the '?'.
---
 youtube_dl/extractor/rte.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/rte.py b/youtube_dl/extractor/rte.py
index 2811767b9..d9cfbf180 100644
--- a/youtube_dl/extractor/rte.py
+++ b/youtube_dl/extractor/rte.py
@@ -9,7 +9,7 @@ from ..utils import (
 
 
 class RteIE(InfoExtractor):
-    _VALID_URL = r'http?://(?:www\.)?rte\.ie/player/[^/]{2,3}/show/[^/]+/(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?rte\.ie/player/[^/]{2,3}/show/[^/]+/(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://www.rte.ie/player/ie/show/iwitness-862/10478715/',
         'info_dict': {

From fbd9f6ea804328d536aafd2b20a8afb72968e351 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 17 Oct 2015 18:28:21 +0600
Subject: [PATCH 2810/6219] [twitch] Improve authentication

---
 youtube_dl/extractor/twitch.py | 20 +++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 023911c41..891499a1f 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -15,6 +15,7 @@ from ..compat import (
     compat_urlparse,
 )
 from ..utils import (
+    encode_dict,
     ExtractorError,
     int_or_none,
     parse_duration,
@@ -27,8 +28,7 @@ class TwitchBaseIE(InfoExtractor):
 
     _API_BASE = 'https://api.twitch.tv'
     _USHER_BASE = 'http://usher.twitch.tv'
-    _LOGIN_URL = 'https://secure.twitch.tv/login'
-    _LOGIN_POST_URL = 'https://passport.twitch.tv/authentications/new'
+    _LOGIN_URL = 'http://www.twitch.tv/login'
     _NETRC_MACHINE = 'twitch'
 
     def _handle_error(self, response):
@@ -61,26 +61,28 @@ class TwitchBaseIE(InfoExtractor):
         if username is None:
             return
 
-        login_page = self._download_webpage(
+        login_page, handle = self._download_webpage_handle(
             self._LOGIN_URL, None, 'Downloading login page')
 
         login_form = self._hidden_inputs(login_page)
 
         login_form.update({
-            'login': username.encode('utf-8'),
-            'password': password.encode('utf-8'),
+            'username': username,
+            'password': password,
         })
 
+        redirect_url = handle.geturl()
+
         post_url = self._search_regex(
             r'<form[^>]+action=(["\'])(?P<url>.+?)\1', login_page,
-            'post url', default=self._LOGIN_POST_URL, group='url')
+            'post url', default=redirect_url, group='url')
 
         if not post_url.startswith('http'):
-            post_url = compat_urlparse.urljoin(self._LOGIN_URL, post_url)
+            post_url = compat_urlparse.urljoin(redirect_url, post_url)
 
         request = compat_urllib_request.Request(
-            post_url, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
-        request.add_header('Referer', self._LOGIN_URL)
+            post_url, compat_urllib_parse.urlencode(encode_dict(login_form)).encode('utf-8'))
+        request.add_header('Referer', redirect_url)
         response = self._download_webpage(
             request, None, 'Logging in as %s' % username)
 

From e5e9966199c00a6b89f1f25e1c7b85effb032537 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 17 Oct 2015 18:29:54 +0600
Subject: [PATCH 2811/6219] [twitch:vod] Improve extraction

---
 youtube_dl/extractor/twitch.py | 20 +++++++++++++++-----
 1 file changed, 15 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 891499a1f..21ea836ea 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -240,14 +240,24 @@ class TwitchVodIE(TwitchItemBaseIE):
 
     def _real_extract(self, url):
         item_id = self._match_id(url)
+
         info = self._download_info(self._ITEM_SHORTCUT, item_id)
         access_token = self._download_json(
-            '%s/api/vods/%s/access_token' % (self._API_BASE, item_id), item_id,
-            'Downloading %s access token' % self._ITEM_TYPE)
+           '%s/api/vods/%s/access_token' % (self._API_BASE, item_id), item_id,
+           'Downloading %s access token' % self._ITEM_TYPE)
+
         formats = self._extract_m3u8_formats(
-            '%s/vod/%s?nauth=%s&nauthsig=%s&allow_source=true'
-            % (self._USHER_BASE, item_id, access_token['token'], access_token['sig']),
-            item_id, 'mp4')
+           '%s/vod/%s?%s' % (
+               self._USHER_BASE, item_id,
+               compat_urllib_parse.urlencode({
+                    'allow_source': 'true',
+                    'allow_spectre': 'true',
+                    'player': 'twitchweb',
+                    'nauth': access_token['token'],
+                    'nauthsig': access_token['sig'],
+                })),
+           item_id, 'mp4')
+
         self._prefer_source(formats)
         info['formats'] = formats
 

From 350c9481336ac981eadc982b67ccdbc7e28ca0e3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 17 Oct 2015 18:43:12 +0600
Subject: [PATCH 2812/6219] [twitch:vod] Formatting

---
 youtube_dl/extractor/twitch.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 21ea836ea..3ec08b674 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -243,20 +243,20 @@ class TwitchVodIE(TwitchItemBaseIE):
 
         info = self._download_info(self._ITEM_SHORTCUT, item_id)
         access_token = self._download_json(
-           '%s/api/vods/%s/access_token' % (self._API_BASE, item_id), item_id,
-           'Downloading %s access token' % self._ITEM_TYPE)
+            '%s/api/vods/%s/access_token' % (self._API_BASE, item_id), item_id,
+            'Downloading %s access token' % self._ITEM_TYPE)
 
         formats = self._extract_m3u8_formats(
-           '%s/vod/%s?%s' % (
-               self._USHER_BASE, item_id,
-               compat_urllib_parse.urlencode({
+            '%s/vod/%s?%s' % (
+                self._USHER_BASE, item_id,
+                compat_urllib_parse.urlencode({
                     'allow_source': 'true',
                     'allow_spectre': 'true',
                     'player': 'twitchweb',
                     'nauth': access_token['token'],
                     'nauthsig': access_token['sig'],
                 })),
-           item_id, 'mp4')
+            item_id, 'mp4')
 
         self._prefer_source(formats)
         info['formats'] = formats

From d90e40305bee84f5e3cd4927c729e1d16bbd3dc6 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 17 Oct 2015 17:28:09 +0100
Subject: [PATCH 2813/6219] [bilibili] fix info extraction

---
 youtube_dl/extractor/bilibili.py | 151 ++++++++++++-------------------
 1 file changed, 57 insertions(+), 94 deletions(-)

diff --git a/youtube_dl/extractor/bilibili.py b/youtube_dl/extractor/bilibili.py
index ecc17ebeb..8f23a30ab 100644
--- a/youtube_dl/extractor/bilibili.py
+++ b/youtube_dl/extractor/bilibili.py
@@ -1,15 +1,12 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
-import itertools
 import json
 import xml.etree.ElementTree as ET
 
 from .common import InfoExtractor
 from ..utils import (
     int_or_none,
-    unified_strdate,
     ExtractorError,
 )
 
@@ -21,12 +18,15 @@ class BiliBiliIE(InfoExtractor):
         'url': 'http://www.bilibili.tv/video/av1074402/',
         'md5': '2c301e4dab317596e837c3e7633e7d86',
         'info_dict': {
-            'id': '1074402_part1',
+            'id': '1554319',
             'ext': 'flv',
             'title': '【金坷垃】金泡沫',
-            'duration': 308,
+            'duration': 308313,
             'upload_date': '20140420',
             'thumbnail': 're:^https?://.+\.jpg',
+            'description': 'md5:ce18c2a2d2193f0df2917d270f2e5923',
+            'timestamp': 1397983878,
+            'uploader': '菊子桑',
         },
     }, {
         'url': 'http://www.bilibili.com/video/av1041170/',
@@ -34,109 +34,72 @@ class BiliBiliIE(InfoExtractor):
             'id': '1041170',
             'title': '【BD1080P】刀语【诸神&异域】',
         },
-        'playlist_count': 9,
+        'playlist_count': 12,
     }]
 
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
+    def _extract_video_info(self, cid, view_data, page_num=1, num_pages=1):
+        title = view_data['title']
 
-        if '(此视频不存在或被删除)' in webpage:
-            raise ExtractorError(
-                'The video does not exist or was deleted', expected=True)
-
-        if '>你没有权限浏览！ 由于版权相关问题 我们不对您所在的地区提供服务<' in webpage:
-            raise ExtractorError(
-                'The video is not available in your region due to copyright reasons',
-                expected=True)
-
-        video_code = self._search_regex(
-            r'(?s)<div itemprop="video".*?>(.*?)</div>', webpage, 'video code')
-
-        title = self._html_search_meta(
-            'media:title', video_code, 'title', fatal=True)
-        duration_str = self._html_search_meta(
-            'duration', video_code, 'duration')
-        if duration_str is None:
-            duration = None
-        else:
-            duration_mobj = re.match(
-                r'^T(?:(?P<hours>[0-9]+)H)?(?P<minutes>[0-9]+)M(?P<seconds>[0-9]+)S$',
-                duration_str)
-            duration = (
-                int_or_none(duration_mobj.group('hours'), default=0) * 3600 +
-                int(duration_mobj.group('minutes')) * 60 +
-                int(duration_mobj.group('seconds')))
-        upload_date = unified_strdate(self._html_search_meta(
-            'uploadDate', video_code, fatal=False))
-        thumbnail = self._html_search_meta(
-            'thumbnailUrl', video_code, 'thumbnail', fatal=False)
-
-        cid = self._search_regex(r'cid=(\d+)', webpage, 'cid')
-
-        entries = []
-
-        lq_page = self._download_webpage(
-            'http://interface.bilibili.com/v_cdn_play?appkey=1&cid=%s' % cid,
-            video_id,
-            note='Downloading LQ video info'
+        page = self._download_webpage(
+            'http://interface.bilibili.com/v_cdn_play?appkey=8e9fc618fbd41e28&cid=%s' % cid,
+            cid,
+            'Downloading page %d/%d' % (page_num, num_pages)
         )
         try:
-            err_info = json.loads(lq_page)
+            err_info = json.loads(page)
             raise ExtractorError(
                 'BiliBili said: ' + err_info['error_text'], expected=True)
         except ValueError:
             pass
 
-        lq_doc = ET.fromstring(lq_page)
-        lq_durls = lq_doc.findall('./durl')
+        doc = ET.fromstring(page)
+        durls = doc.findall('./durl')
 
-        hq_doc = self._download_xml(
-            'http://interface.bilibili.com/playurl?appkey=1&cid=%s' % cid,
-            video_id,
-            note='Downloading HQ video info',
-            fatal=False,
-        )
-        if hq_doc is not False:
-            hq_durls = hq_doc.findall('./durl')
-            assert len(lq_durls) == len(hq_durls)
-        else:
-            hq_durls = itertools.repeat(None)
-
-        i = 1
-        for lq_durl, hq_durl in zip(lq_durls, hq_durls):
-            formats = [{
-                'format_id': 'lq',
-                'quality': 1,
-                'url': lq_durl.find('./url').text,
-                'filesize': int_or_none(
-                    lq_durl.find('./size'), get_attr='text'),
-            }]
-            if hq_durl is not None:
-                formats.append({
-                    'format_id': 'hq',
-                    'quality': 2,
-                    'ext': 'flv',
-                    'url': hq_durl.find('./url').text,
-                    'filesize': int_or_none(
-                        hq_durl.find('./size'), get_attr='text'),
-                })
-            self._sort_formats(formats)
+        entries = []
 
+        for durl in durls:
             entries.append({
-                'id': '%s_part%d' % (video_id, i),
+                'id': '%s_part%s' % (cid, durl.find('./order').text),
                 'title': title,
-                'formats': formats,
-                'duration': duration,
-                'upload_date': upload_date,
-                'thumbnail': thumbnail,
+                'url': durl.find('./url').text,
+                'filesize': int_or_none(durl.find('./filesize').text),
+                'ext': 'flv',
+                'duration': int_or_none(durl.find('./length').text) // 1000,
             })
 
-            i += 1
-
-        return {
-            '_type': 'multi_video',
-            'entries': entries,
-            'id': video_id,
-            'title': title
+        info = {
+            'id': cid,
+            'title': title,
+            'description': view_data.get('description'),
+            'thumbnail': view_data.get('pic'),
+            'uploader': view_data.get('author'),
+            'timestamp': int_or_none(view_data.get('created')),
+            'view_count': view_data.get('play'),
+            'duration': int_or_none(doc.find('./timelength').text),
         }
+
+        if len(entries) == 1:
+            entries[0].update(info)
+            return entries[0]
+        else:
+            info.update({
+                '_type': 'multi_video',
+                'entries': entries,
+            })
+            return info
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        view_data = self._download_json('http://api.bilibili.com/view?type=json&appkey=8e9fc618fbd41e28&id=%s' % video_id, video_id)
+
+        num_pages = int_or_none(view_data['pages'])
+        if num_pages > 1:
+            play_list_title = view_data['title']
+            page_list = self._download_json('http://www.bilibili.com/widget/getPageList?aid=%s' % video_id, video_id, 'Downloading page list metadata')
+            entries = []
+            for page in page_list:
+                view_data['title'] = page['pagename']
+                entries.append(self._extract_video_info(str(page['cid']), view_data, page['page'], num_pages))
+            return self.playlist_result(entries, video_id, play_list_title, view_data.get('description'))
+        else:
+            return self._extract_video_info(str(view_data['cid']), view_data)

From 41a7b00f183844e93ae2ba46fb4021f257f3ce79 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Lalinsk=C3=BD?= <lukas@oxygene.sk>
Date: Sat, 17 Oct 2015 18:18:40 +0200
Subject: [PATCH 2814/6219] [vimeo] Extract config URL from (new?) React-based
 Vimeo's page

---
 youtube_dl/extractor/vimeo.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index fa1b22049..88e462a4d 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -286,7 +286,14 @@ class VimeoIE(VimeoBaseInfoExtractor):
         try:
             try:
                 config_url = self._html_search_regex(
-                    r' data-config-url="(.+?)"', webpage, 'config URL')
+                    r' data-config-url="(.+?)"', webpage,
+                    'config URL', default=None)
+                if not config_url:
+                    # New react-based page
+                    vimeo_clip_page_config = self._search_regex(
+                        r'vimeo\.clip_page_config\s*=\s*({.+?});', webpage,
+                        'vimeo clip page config')
+                    config_url = self._parse_json(vimeo_clip_page_config, video_id)['player']['config_url']
                 config_json = self._download_webpage(config_url, video_id)
                 config = json.loads(config_json)
             except RegexNotFoundError:

From dd8417526b13c541e6db8f4200e717b8922a1620 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 17 Oct 2015 22:48:14 +0600
Subject: [PATCH 2815/6219] [vimeo] Clarify new react+flux website fallback

---
 youtube_dl/extractor/vimeo.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 88e462a4d..0f84656c0 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -289,11 +289,14 @@ class VimeoIE(VimeoBaseInfoExtractor):
                     r' data-config-url="(.+?)"', webpage,
                     'config URL', default=None)
                 if not config_url:
-                    # New react-based page
+                    # Sometimes new react-based page is served instead of old one that require
+                    # different config URL extraction approach (see
+                    # https://github.com/rg3/youtube-dl/pull/7209)
                     vimeo_clip_page_config = self._search_regex(
                         r'vimeo\.clip_page_config\s*=\s*({.+?});', webpage,
                         'vimeo clip page config')
-                    config_url = self._parse_json(vimeo_clip_page_config, video_id)['player']['config_url']
+                    config_url = self._parse_json(
+                        vimeo_clip_page_config, video_id)['player']['config_url']
                 config_json = self._download_webpage(config_url, video_id)
                 config = json.loads(config_json)
             except RegexNotFoundError:

From 59fe4824f80b7e266ea9918ae1b2e49a456b869f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Lalinsk=C3=BD?= <lukas@oxygene.sk>
Date: Sat, 17 Oct 2015 18:52:25 +0200
Subject: [PATCH 2816/6219] [vidme] Better error message for suspended vidme
 videos

---
 youtube_dl/extractor/vidme.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/vidme.py b/youtube_dl/extractor/vidme.py
index 078d283b2..81dcaa231 100644
--- a/youtube_dl/extractor/vidme.py
+++ b/youtube_dl/extractor/vidme.py
@@ -114,6 +114,12 @@ class VidmeIE(InfoExtractor):
 
         video = response['video']
 
+        if video.get('state') == 'user-disabled':
+            raise ExtractorError(
+                'Vidme said: This video has been suspended either due to a copyright claim, '
+                'or for violating the terms of use.',
+                expected=True)
+
         formats = [{
             'format_id': f.get('type'),
             'url': f['uri'],

From 9eb31b265f65ec6b04a508702af1a6feddafb8fe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 17 Oct 2015 23:01:24 +0600
Subject: [PATCH 2817/6219] [vidme] Add user-disabled test

---
 youtube_dl/extractor/vidme.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/vidme.py b/youtube_dl/extractor/vidme.py
index 81dcaa231..382517a4a 100644
--- a/youtube_dl/extractor/vidme.py
+++ b/youtube_dl/extractor/vidme.py
@@ -93,6 +93,10 @@ class VidmeIE(InfoExtractor):
         'params': {
             'skip_download': True,
         },
+    }, {
+        # nsfw, user-disabled
+        'url': 'https://vid.me/dzGJ',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 583882fdce19f8c565402f42523b275f96c91575 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Lalinsk=C3=BD?= <lukas@oxygene.sk>
Date: Sat, 17 Oct 2015 19:26:30 +0200
Subject: [PATCH 2818/6219] [dailymotion] Report errors from player v5

---
 youtube_dl/extractor/dailymotion.py | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index 80a05cfee..ea1edceb1 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -96,6 +96,11 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
                 'uploader': 'HotWaves1012',
                 'age_limit': 18,
             }
+        },
+        # geo-restricted, player v5
+        {
+            'url': 'http://www.dailymotion.com/video/xhza0o',
+            'only_matching': True,
         }
     ]
 
@@ -124,6 +129,9 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
         if player_v5:
             player = self._parse_json(player_v5, video_id)
             metadata = player['metadata']
+
+            self._check_error(metadata)
+
             formats = []
             for quality, media_list in metadata['qualities'].items():
                 for media in media_list:
@@ -201,9 +209,7 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
                 'video info', flags=re.MULTILINE),
             video_id)
 
-        if info.get('error') is not None:
-            msg = 'Couldn\'t get video, Dailymotion says: %s' % info['error']['title']
-            raise ExtractorError(msg, expected=True)
+        self._check_error(info)
 
         formats = []
         for (key, format_id) in self._FORMATS:
@@ -246,6 +252,11 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
             'duration': info['duration']
         }
 
+    def _check_error(self, info):
+        if info.get('error') is not None:
+            msg = 'Couldn\'t get video, Dailymotion says: %s' % info['error']['title']
+            raise ExtractorError(msg, expected=True)
+
     def _get_subtitles(self, video_id, webpage):
         try:
             sub_list = self._download_webpage(

From 55af2b26e0f169bef2f10a7b5f6ec8e34c6dbb6d Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 17 Oct 2015 18:30:51 +0100
Subject: [PATCH 2819/6219] [bilibili] extract backup url

---
 youtube_dl/extractor/bilibili.py | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/bilibili.py b/youtube_dl/extractor/bilibili.py
index 8f23a30ab..85156ce49 100644
--- a/youtube_dl/extractor/bilibili.py
+++ b/youtube_dl/extractor/bilibili.py
@@ -58,13 +58,21 @@ class BiliBiliIE(InfoExtractor):
         entries = []
 
         for durl in durls:
+            formats = []
+            backup_url = durl.find('./backup_url')
+            if backup_url is not None:
+                formats.append({'url': backup_url.find('./url').text})
+            size = durl.find('./filesize|./size')
+            formats.append({
+                'url': durl.find('./url').text,
+                'filesize': int_or_none(size.text) if size else None,
+                'ext': 'flv',
+            })
             entries.append({
                 'id': '%s_part%s' % (cid, durl.find('./order').text),
                 'title': title,
-                'url': durl.find('./url').text,
-                'filesize': int_or_none(durl.find('./filesize').text),
-                'ext': 'flv',
                 'duration': int_or_none(durl.find('./length').text) // 1000,
+                'formats': formats,
             })
 
         info = {

From 648e6a1ffe45ceae2995c3f9ec6a9413aad55640 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 18 Oct 2015 00:11:34 +0600
Subject: [PATCH 2820/6219] [youtube] Generalize playlist entries extraction
 (Closes #6699, closes #6992)

---
 youtube_dl/extractor/youtube.py | 121 ++++++++++++++------------------
 1 file changed, 52 insertions(+), 69 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index b252e36e1..08e821362 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -178,6 +178,52 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
             return
 
 
+class YoutubePlaylistBaseInfoExtractor(InfoExtractor):
+    # Extract the video ids from the playlist pages
+    def _entries(self, page, playlist_id):
+        more_widget_html = content_html = page
+        for page_num in itertools.count(1):
+            for video_id, video_title in self.extract_videos_from_page(content_html):
+                yield self.url_result(
+                    video_id, 'Youtube', video_id=video_id,
+                    video_title=video_title)
+
+            mobj = re.search(r'data-uix-load-more-href="/?(?P<more>[^"]+)"', more_widget_html)
+            if not mobj:
+                break
+
+            more = self._download_json(
+                'https://youtube.com/%s' % mobj.group('more'), playlist_id,
+                'Downloading page #%s' % page_num,
+                transform_source=uppercase_escape)
+            content_html = more['content_html']
+            if not content_html.strip():
+                # Some webpages show a "Load more" button but they don't
+                # have more videos
+                break
+            more_widget_html = more['load_more_widget_html']
+
+    def extract_videos_from_page(self, page):
+        ids_in_page = []
+        titles_in_page = []
+        for mobj in re.finditer(self._VIDEO_RE, page):
+            # The link with index 0 is not the first video of the playlist (not sure if still actual)
+            if 'index' in mobj.groupdict() and mobj.group('id') == '0':
+                continue
+            video_id = mobj.group('id')
+            video_title = unescapeHTML(mobj.group('title'))
+            if video_title:
+                video_title = video_title.strip()
+            try:
+                idx = ids_in_page.index(video_id)
+                if video_title and not titles_in_page[idx]:
+                    titles_in_page[idx] = video_title
+            except ValueError:
+                ids_in_page.append(video_id)
+                titles_in_page.append(video_title)
+        return zip(ids_in_page, titles_in_page)
+
+
 class YoutubeIE(YoutubeBaseInfoExtractor):
     IE_DESC = 'YouTube.com'
     _VALID_URL = r"""(?x)^
@@ -1419,7 +1465,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         }
 
 
-class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
+class YoutubePlaylistIE(YoutubeBaseInfoExtractor, YoutubePlaylistBaseInfoExtractor):
     IE_DESC = 'YouTube.com playlists'
     _VALID_URL = r"""(?x)(?:
                         (?:https?://)?
@@ -1440,7 +1486,7 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
                         ((?:PL|LL|EC|UU|FL|RD|UL)[0-9A-Za-z-_]{10,})
                      )"""
     _TEMPLATE_URL = 'https://www.youtube.com/playlist?list=%s'
-    _VIDEO_RE = r'href="\s*/watch\?v=(?P<id>[0-9A-Za-z_-]{11})&amp;[^"]*?index=(?P<index>\d+)'
+    _VIDEO_RE = r'href="\s*/watch\?v=(?P<id>[0-9A-Za-z_-]{11})&amp;[^"]*?index=(?P<index>\d+)(?:[^>]+>(?P<title>[^<]+))?'
     IE_NAME = 'youtube:playlist'
     _TESTS = [{
         'url': 'https://www.youtube.com/playlist?list=PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re',
@@ -1557,37 +1603,11 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
             else:
                 self.report_warning('Youtube gives an alert message: ' + match)
 
-        # Extract the video ids from the playlist pages
-        def _entries():
-            more_widget_html = content_html = page
-            for page_num in itertools.count(1):
-                matches = re.finditer(self._VIDEO_RE, content_html)
-                # We remove the duplicates and the link with index 0
-                # (it's not the first video of the playlist)
-                new_ids = orderedSet(m.group('id') for m in matches if m.group('index') != '0')
-                for vid_id in new_ids:
-                    yield self.url_result(vid_id, 'Youtube', video_id=vid_id)
-
-                mobj = re.search(r'data-uix-load-more-href="/?(?P<more>[^"]+)"', more_widget_html)
-                if not mobj:
-                    break
-
-                more = self._download_json(
-                    'https://youtube.com/%s' % mobj.group('more'), playlist_id,
-                    'Downloading page #%s' % page_num,
-                    transform_source=uppercase_escape)
-                content_html = more['content_html']
-                if not content_html.strip():
-                    # Some webpages show a "Load more" button but they don't
-                    # have more videos
-                    break
-                more_widget_html = more['load_more_widget_html']
-
         playlist_title = self._html_search_regex(
             r'(?s)<h1 class="pl-header-title[^"]*">\s*(.*?)\s*</h1>',
             page, 'title')
 
-        return self.playlist_result(_entries(), playlist_id, playlist_title)
+        return self.playlist_result(self._entries(page, playlist_id), playlist_id, playlist_title)
 
     def _real_extract(self, url):
         # Extract playlist id
@@ -1613,10 +1633,11 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
         return self._extract_playlist(playlist_id)
 
 
-class YoutubeChannelIE(InfoExtractor):
+class YoutubeChannelIE(YoutubePlaylistBaseInfoExtractor):
     IE_DESC = 'YouTube.com channels'
     _VALID_URL = r'https?://(?:youtu\.be|(?:\w+\.)?youtube(?:-nocookie)?\.com)/channel/(?P<id>[0-9A-Za-z_-]+)'
     _TEMPLATE_URL = 'https://www.youtube.com/channel/%s/videos'
+    _VIDEO_RE = r'(?:title="(?P<title>[^"]+)"[^>]+)?href="/watch\?v=(?P<id>[0-9A-Za-z_-]+)&?'
     IE_NAME = 'youtube:channel'
     _TESTS = [{
         'note': 'paginated channel',
@@ -1627,22 +1648,6 @@ class YoutubeChannelIE(InfoExtractor):
         }
     }]
 
-    @staticmethod
-    def extract_videos_from_page(page):
-        ids_in_page = []
-        titles_in_page = []
-        for mobj in re.finditer(r'(?:title="(?P<title>[^"]+)"[^>]+)?href="/watch\?v=(?P<id>[0-9A-Za-z_-]+)&?', page):
-            video_id = mobj.group('id')
-            video_title = unescapeHTML(mobj.group('title'))
-            try:
-                idx = ids_in_page.index(video_id)
-                if video_title and not titles_in_page[idx]:
-                    titles_in_page[idx] = video_title
-            except ValueError:
-                ids_in_page.append(video_id)
-                titles_in_page.append(video_title)
-        return zip(ids_in_page, titles_in_page)
-
     def _real_extract(self, url):
         channel_id = self._match_id(url)
 
@@ -1685,29 +1690,7 @@ class YoutubeChannelIE(InfoExtractor):
                 for video_id, video_title in self.extract_videos_from_page(channel_page)]
             return self.playlist_result(entries, channel_id)
 
-        def _entries():
-            more_widget_html = content_html = channel_page
-            for pagenum in itertools.count(1):
-
-                for video_id, video_title in self.extract_videos_from_page(content_html):
-                    yield self.url_result(
-                        video_id, 'Youtube', video_id=video_id,
-                        video_title=video_title)
-
-                mobj = re.search(
-                    r'data-uix-load-more-href="/?(?P<more>[^"]+)"',
-                    more_widget_html)
-                if not mobj:
-                    break
-
-                more = self._download_json(
-                    'https://youtube.com/%s' % mobj.group('more'), channel_id,
-                    'Downloading page #%s' % (pagenum + 1),
-                    transform_source=uppercase_escape)
-                content_html = more['content_html']
-                more_widget_html = more['load_more_widget_html']
-
-        return self.playlist_result(_entries(), channel_id)
+        return self.playlist_result(self._entries(channel_page, channel_id), channel_id)
 
 
 class YoutubeUserIE(YoutubeChannelIE):

From 8e5b1219489be399de55566090e145c89007fa48 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 18 Oct 2015 00:27:06 +0600
Subject: [PATCH 2821/6219] [test_youtube_lists] Add test flat playlist
 entries' titles

---
 test/test_youtube_lists.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/test/test_youtube_lists.py b/test/test_youtube_lists.py
index c889b6f15..26aadb34f 100644
--- a/test/test_youtube_lists.py
+++ b/test/test_youtube_lists.py
@@ -57,5 +57,14 @@ class TestYoutubeLists(unittest.TestCase):
         entries = result['entries']
         self.assertEqual(len(entries), 100)
 
+    def test_youtube_flat_playlist_titles(self):
+        dl = FakeYDL()
+        dl.params['extract_flat'] = True
+        ie = YoutubePlaylistIE(dl)
+        result = ie.extract('https://www.youtube.com/playlist?list=PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re')
+        self.assertIsPlaylist(result)
+        for entry in result['entries']:
+            self.assertTrue(entry.get('title'))
+
 if __name__ == '__main__':
     unittest.main()

From 7593fbaa126f8bf14eecff7f103cb497e3d31de5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 18 Oct 2015 01:00:37 +0600
Subject: [PATCH 2822/6219] [dailymotion] Error spelling

---
 youtube_dl/extractor/dailymotion.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index ea1edceb1..9cd9ff17d 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -254,8 +254,8 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
 
     def _check_error(self, info):
         if info.get('error') is not None:
-            msg = 'Couldn\'t get video, Dailymotion says: %s' % info['error']['title']
-            raise ExtractorError(msg, expected=True)
+            raise ExtractorError(
+                '%s said: %s' % (self.IE_NAME, info['error']['title']), expected=True)
 
     def _get_subtitles(self, video_id, webpage):
         try:

From 5a11b793fe70beb6b0c7a74a489db9e52c4a742b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 18 Oct 2015 01:36:03 +0600
Subject: [PATCH 2823/6219] [lynda] Extract all prioritized streams

---
 youtube_dl/extractor/lynda.py | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index 378117270..5c973e75c 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -140,13 +140,14 @@ class LyndaIE(LyndaBaseIE):
 
         prioritized_streams = video_json.get('PrioritizedStreams')
         if prioritized_streams:
-            formats.extend([
-                {
-                    'url': video_url,
-                    'width': int_or_none(format_id),
-                    'format_id': format_id,
-                } for format_id, video_url in prioritized_streams['0'].items()
-            ])
+            for prioritized_stream_id, prioritized_stream in prioritized_streams.items():
+                formats.extend([
+                    {
+                        'url': video_url,
+                        'width': int_or_none(format_id),
+                        'format_id': '%s-%s' % (prioritized_stream_id, format_id),
+                    } for format_id, video_url in prioritized_stream.items()
+                ])
 
         self._check_formats(formats, video_id)
         self._sort_formats(formats)

From 355c7ad361aa3c8a57ff83e3f702a496dce59e65 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 17 Oct 2015 21:30:38 +0100
Subject: [PATCH 2824/6219] [cspan] handle error massages and extract qualities

---
 youtube_dl/extractor/cspan.py | 67 +++++++++++++++++++++--------------
 1 file changed, 41 insertions(+), 26 deletions(-)

diff --git a/youtube_dl/extractor/cspan.py b/youtube_dl/extractor/cspan.py
index 994e080d5..c74b35fd9 100644
--- a/youtube_dl/extractor/cspan.py
+++ b/youtube_dl/extractor/cspan.py
@@ -9,16 +9,21 @@ from ..utils import (
     find_xpath_attr,
     smuggle_url,
     determine_ext,
+    ExtractorError,
 )
 from .senateisvp import SenateISVPIE
 
 
+def get_text_attr(d, attr):
+    return d.get(attr, {}).get('#text')
+
+
 class CSpanIE(InfoExtractor):
     _VALID_URL = r'http://(?:www\.)?c-span\.org/video/\?(?P<id>[0-9a-f]+)'
     IE_DESC = 'C-SPAN'
     _TESTS = [{
         'url': 'http://www.c-span.org/video/?313572-1/HolderonV',
-        'md5': '067803f994e049b455a58b16e5aab442',
+        'md5': '94b29a4f131ff03d23471dd6f60b6a1d',
         'info_dict': {
             'id': '315139',
             'ext': 'mp4',
@@ -28,7 +33,7 @@ class CSpanIE(InfoExtractor):
         'skip': 'Regularly fails on travis, for unknown reasons',
     }, {
         'url': 'http://www.c-span.org/video/?c4486943/cspan-international-health-care-models',
-        'md5': '4eafd1e91a75d2b1e6a3cbd0995816a2',
+        'md5': '8e5fbfabe6ad0f89f3012a7943c1287b',
         'info_dict': {
             'id': 'c4486943',
             'ext': 'mp4',
@@ -37,7 +42,7 @@ class CSpanIE(InfoExtractor):
         }
     }, {
         'url': 'http://www.c-span.org/video/?318608-1/gm-ignition-switch-recall',
-        'md5': '446562a736c6bf97118e389433ed88d4',
+        'md5': '2ae5051559169baadba13fc35345ae74',
         'info_dict': {
             'id': '342759',
             'ext': 'mp4',
@@ -71,8 +76,10 @@ class CSpanIE(InfoExtractor):
                 return self.url_result(surl, 'SenateISVP', video_id, title)
 
         data = self._download_json(
-            'http://c-spanvideo.org/videoLibrary/assets/player/ajax-player.php?os=android&html5=%s&id=%s' % (video_type, video_id),
-            video_id)
+            'http://www.c-span.org/assets/player/ajax-player.php?os=android&html5=%s&id=%s' % (video_type, video_id),
+            video_id)['video']
+        if data['@status'] != 'Success':
+            raise ExtractorError('%s said: %s' % (self.IE_NAME, get_text_attr(data, 'error')), expected=True)
 
         doc = self._download_xml(
             'http://www.c-span.org/common/services/flashXml.php?%sid=%s' % (video_type, video_id),
@@ -83,28 +90,36 @@ class CSpanIE(InfoExtractor):
         title = find_xpath_attr(doc, './/string', 'name', 'title').text
         thumbnail = find_xpath_attr(doc, './/string', 'name', 'poster').text
 
-        files = data['video']['files']
-        try:
-            capfile = data['video']['capfile']['#text']
-        except KeyError:
-            capfile = None
+        files = data['files']
+        capfile = get_text_attr(data, 'capfile')
 
-        entries = [{
-            'id': '%s_%d' % (video_id, partnum + 1),
-            'title': (
-                title if len(files) == 1 else
-                '%s part %d' % (title, partnum + 1)),
-            'url': unescapeHTML(f['path']['#text']),
-            'description': description,
-            'thumbnail': thumbnail,
-            'duration': int_or_none(f.get('length', {}).get('#text')),
-            'subtitles': {
-                'en': [{
-                    'url': capfile,
-                    'ext': determine_ext(capfile, 'dfxp')
-                }],
-            } if capfile else None,
-        } for partnum, f in enumerate(files)]
+        entries = []
+        for partnum, f in enumerate(files):
+            formats = []
+            for quality in f['qualities']:
+                formats.append({
+                    'format_id': '%s-%sp' % (get_text_attr(quality, 'bitrate'), get_text_attr(quality, 'height')),
+                    'url': unescapeHTML(get_text_attr(quality, 'file')),
+                    'height': int_or_none(get_text_attr(quality, 'height')),
+                    'tbr': int_or_none(get_text_attr(quality, 'bitrate')),
+                })
+            self._sort_formats(formats)
+            entries.append({
+                'id': '%s_%d' % (video_id, partnum + 1),
+                'title': (
+                    title if len(files) == 1 else
+                    '%s part %d' % (title, partnum + 1)),
+                'formats': formats,
+                'description': description,
+                'thumbnail': thumbnail,
+                'duration': int_or_none(get_text_attr(f, 'length')),
+                'subtitles': {
+                    'en': [{
+                        'url': capfile,
+                        'ext': determine_ext(capfile, 'dfxp')
+                    }],
+                } if capfile else None,
+            })
 
         if len(entries) == 1:
             entry = dict(entries[0])

From 520e753390aab5845bb257b964ebcd6f818455df Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 17 Oct 2015 23:12:58 +0100
Subject: [PATCH 2825/6219] [bilibili] add support for specefic page extraction

---
 youtube_dl/extractor/bilibili.py | 45 ++++++++++++++++----------------
 1 file changed, 23 insertions(+), 22 deletions(-)

diff --git a/youtube_dl/extractor/bilibili.py b/youtube_dl/extractor/bilibili.py
index 85156ce49..e00efb6eb 100644
--- a/youtube_dl/extractor/bilibili.py
+++ b/youtube_dl/extractor/bilibili.py
@@ -1,18 +1,20 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
 import json
 import xml.etree.ElementTree as ET
 
 from .common import InfoExtractor
 from ..utils import (
     int_or_none,
+    unescapeHTML,
     ExtractorError,
 )
 
 
 class BiliBiliIE(InfoExtractor):
-    _VALID_URL = r'http://www\.bilibili\.(?:tv|com)/video/av(?P<id>[0-9]+)/'
+    _VALID_URL = r'http://www\.bilibili\.(?:tv|com)/video/av(?P<id>\d+)(?:/index_(?P<page_num>\d+).html)?'
 
     _TESTS = [{
         'url': 'http://www.bilibili.tv/video/av1074402/',
@@ -33,17 +35,31 @@ class BiliBiliIE(InfoExtractor):
         'info_dict': {
             'id': '1041170',
             'title': '【BD1080P】刀语【诸神&异域】',
+            'description': '这是个神奇的故事~每个人不留弹幕不给走哦~切利哦！~',
+            'uploader': '枫叶逝去',
+            'timestamp': 1396501299,
         },
-        'playlist_count': 12,
+        'playlist_count': 9,
     }]
 
-    def _extract_video_info(self, cid, view_data, page_num=1, num_pages=1):
-        title = view_data['title']
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        page_num = mobj.group('page_num') or '1'
+
+        view_data = self._download_json(
+            'http://api.bilibili.com/view?type=json&appkey=8e9fc618fbd41e28&id=%s&page=%s' % (video_id, page_num),
+            video_id)
+        if 'error' in view_data:
+            raise ExtractorError('%s said: %s' % (self.IE_NAME, view_data['error']), expected=True)
+
+        cid = view_data['cid']
+        title = unescapeHTML(view_data['title'])
 
         page = self._download_webpage(
             'http://interface.bilibili.com/v_cdn_play?appkey=8e9fc618fbd41e28&cid=%s' % cid,
             cid,
-            'Downloading page %d/%d' % (page_num, num_pages)
+            'Downloading page %s/%s' % (page_num, view_data['pages'])
         )
         try:
             err_info = json.loads(page)
@@ -76,7 +92,7 @@ class BiliBiliIE(InfoExtractor):
             })
 
         info = {
-            'id': cid,
+            'id': str(cid),
             'title': title,
             'description': view_data.get('description'),
             'thumbnail': view_data.get('pic'),
@@ -92,22 +108,7 @@ class BiliBiliIE(InfoExtractor):
         else:
             info.update({
                 '_type': 'multi_video',
+                'id': video_id,
                 'entries': entries,
             })
             return info
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        view_data = self._download_json('http://api.bilibili.com/view?type=json&appkey=8e9fc618fbd41e28&id=%s' % video_id, video_id)
-
-        num_pages = int_or_none(view_data['pages'])
-        if num_pages > 1:
-            play_list_title = view_data['title']
-            page_list = self._download_json('http://www.bilibili.com/widget/getPageList?aid=%s' % video_id, video_id, 'Downloading page list metadata')
-            entries = []
-            for page in page_list:
-                view_data['title'] = page['pagename']
-                entries.append(self._extract_video_info(str(page['cid']), view_data, page['page'], num_pages))
-            return self.playlist_result(entries, video_id, play_list_title, view_data.get('description'))
-        else:
-            return self._extract_video_info(str(view_data['cid']), view_data)

From 80f48920c8a909ba55d13932524e55ed970f1c6a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 18 Oct 2015 06:57:57 +0600
Subject: [PATCH 2826/6219] [crunchyroll] Bypass maturity wall (Closes #7202)

---
 youtube_dl/extractor/crunchyroll.py | 59 ++++++++++++++++++-----------
 1 file changed, 36 insertions(+), 23 deletions(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 95952bc29..aa258bbc2 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -32,6 +32,26 @@ from ..aes import (
 
 
 class CrunchyrollBaseIE(InfoExtractor):
+    _NETRC_MACHINE = 'crunchyroll'
+
+    def _login(self):
+        (username, password) = self._get_login_info()
+        if username is None:
+            return
+        self.report_login()
+        login_url = 'https://www.crunchyroll.com/?a=formhandler'
+        data = urlencode_postdata({
+            'formname': 'RpcApiUser_Login',
+            'name': username,
+            'password': password,
+        })
+        login_request = compat_urllib_request.Request(login_url, data)
+        login_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
+        self._download_webpage(login_request, None, False, 'Wrong login info')
+
+    def _real_initialize(self):
+        self._login()
+
     def _download_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True, tries=1, timeout=5, encoding=None):
         request = (url_or_request if isinstance(url_or_request, compat_urllib_request.Request)
                    else compat_urllib_request.Request(url_or_request))
@@ -46,10 +66,22 @@ class CrunchyrollBaseIE(InfoExtractor):
         return super(CrunchyrollBaseIE, self)._download_webpage(
             request, video_id, note, errnote, fatal, tries, timeout, encoding)
 
+    @staticmethod
+    def _add_skip_wall(url):
+        parsed_url = compat_urlparse.urlparse(url)
+        qs = compat_urlparse.parse_qs(parsed_url.query)
+        # Always force skip_wall to bypass maturity wall, namely 18+ confirmation message:
+        # > This content may be inappropriate for some people.
+        # > Are you sure you want to continue?
+        # since it's not disabled by default in crunchyroll account's settings.
+        # See https://github.com/rg3/youtube-dl/issues/7202.
+        qs['skip_wall'] = ['1']
+        return compat_urlparse.urlunparse(
+            parsed_url._replace(query=compat_urllib_parse.urlencode(qs, True)))
+
 
 class CrunchyrollIE(CrunchyrollBaseIE):
     _VALID_URL = r'https?://(?:(?P<prefix>www|m)\.)?(?P<url>crunchyroll\.(?:com|fr)/(?:media(?:-|/\?id=)|[^/]*/[^/?&]*?)(?P<video_id>[0-9]+))(?:[/?&]|$)'
-    _NETRC_MACHINE = 'crunchyroll'
     _TESTS = [{
         'url': 'http://www.crunchyroll.com/wanna-be-the-strongest-in-the-world/episode-1-an-idol-wrestler-is-born-645513',
         'info_dict': {
@@ -81,7 +113,6 @@ class CrunchyrollIE(CrunchyrollBaseIE):
             # rtmp
             'skip_download': True,
         },
-
     }, {
         'url': 'http://www.crunchyroll.fr/girl-friend-beta/episode-11-goodbye-la-mode-661697',
         'only_matching': True,
@@ -94,24 +125,6 @@ class CrunchyrollIE(CrunchyrollBaseIE):
         '1080': ('80', '108'),
     }
 
-    def _login(self):
-        (username, password) = self._get_login_info()
-        if username is None:
-            return
-        self.report_login()
-        login_url = 'https://www.crunchyroll.com/?a=formhandler'
-        data = urlencode_postdata({
-            'formname': 'RpcApiUser_Login',
-            'name': username,
-            'password': password,
-        })
-        login_request = compat_urllib_request.Request(login_url, data)
-        login_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
-        self._download_webpage(login_request, None, False, 'Wrong login info')
-
-    def _real_initialize(self):
-        self._login()
-
     def _decrypt_subtitles(self, data, iv, id):
         data = bytes_to_intlist(base64.b64decode(data.encode('utf-8')))
         iv = bytes_to_intlist(base64.b64decode(iv.encode('utf-8')))
@@ -254,7 +267,7 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
         else:
             webpage_url = 'http://www.' + mobj.group('url')
 
-        webpage = self._download_webpage(webpage_url, video_id, 'Downloading webpage')
+        webpage = self._download_webpage(self._add_skip_wall(webpage_url), video_id, 'Downloading webpage')
         note_m = self._html_search_regex(
             r'<div class="showmedia-trailer-notice">(.+?)</div>',
             webpage, 'trailer-notice', default='')
@@ -352,7 +365,7 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
 
 class CrunchyrollShowPlaylistIE(CrunchyrollBaseIE):
     IE_NAME = "crunchyroll:playlist"
-    _VALID_URL = r'https?://(?:(?P<prefix>www|m)\.)?(?P<url>crunchyroll\.com/(?!(?:news|anime-news|library|forum|launchcalendar|lineup|store|comics|freetrial|login))(?P<id>[\w\-]+))/?$'
+    _VALID_URL = r'https?://(?:(?P<prefix>www|m)\.)?(?P<url>crunchyroll\.com/(?!(?:news|anime-news|library|forum|launchcalendar|lineup|store|comics|freetrial|login))(?P<id>[\w\-]+))/?(?:\?|$)'
 
     _TESTS = [{
         'url': 'http://www.crunchyroll.com/a-bridge-to-the-starry-skies-hoshizora-e-kakaru-hashi',
@@ -366,7 +379,7 @@ class CrunchyrollShowPlaylistIE(CrunchyrollBaseIE):
     def _real_extract(self, url):
         show_id = self._match_id(url)
 
-        webpage = self._download_webpage(url, show_id)
+        webpage = self._download_webpage(self._add_skip_wall(url), show_id)
         title = self._html_search_regex(
             r'(?s)<h1[^>]*>\s*<span itemprop="name">(.*?)</span>',
             webpage, 'title')

From 49941c4e4f6e33785a3be1e0d103bd81657d8a0d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 18 Oct 2015 07:06:47 +0600
Subject: [PATCH 2827/6219] [crunchyroll] Add maturity wall reference tests
 (#7202)

---
 youtube_dl/extractor/crunchyroll.py | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index aa258bbc2..cecd0c784 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -116,6 +116,10 @@ class CrunchyrollIE(CrunchyrollBaseIE):
     }, {
         'url': 'http://www.crunchyroll.fr/girl-friend-beta/episode-11-goodbye-la-mode-661697',
         'only_matching': True,
+    }, {
+        # geo-restricted (US), 18+ maturity wall, non-premium available
+        'url': 'http://www.crunchyroll.com/cosplay-complex-ova/episode-1-the-birth-of-the-cosplay-club-565617',
+        'only_matching': True,
     }]
 
     _FORMAT_IDS = {
@@ -374,6 +378,19 @@ class CrunchyrollShowPlaylistIE(CrunchyrollBaseIE):
             'title': 'A Bridge to the Starry Skies - Hoshizora e Kakaru Hashi'
         },
         'playlist_count': 13,
+    }, {
+        # geo-restricted (US), 18+ maturity wall, non-premium available
+        'url': 'http://www.crunchyroll.com/cosplay-complex-ova',
+        'info_dict': {
+            'id': 'cosplay-complex-ova',
+            'title': 'Cosplay Complex OVA'
+        },
+        'playlist_count': 3,
+        'skip': 'Georestricted',
+    }, {
+        # geo-restricted (US), 18+ maturity wall, non-premium will be available since 2015.11.14
+        'url': 'http://www.crunchyroll.com/ladies-versus-butlers?skip_wall=1',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 448ef1f31c8bcc1550cf907fd46e31026ec981b2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 18 Oct 2015 09:11:02 +0600
Subject: [PATCH 2828/6219] [extractor/common] Allow angle brackets in
 attributes in _og_regexes (#7215)

---
 test/test_InfoExtractor.py     | 4 ++++
 youtube_dl/extractor/common.py | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/test/test_InfoExtractor.py b/test/test_InfoExtractor.py
index 2a00d09a5..938466a80 100644
--- a/test/test_InfoExtractor.py
+++ b/test/test_InfoExtractor.py
@@ -37,12 +37,16 @@ class TestInfoExtractor(unittest.TestCase):
             <meta property='og:image' content='http://domain.com/pic.jpg?key1=val1&amp;key2=val2'/>
             <meta content='application/x-shockwave-flash' property='og:video:type'>
             <meta content='Foo' property=og:foobar>
+            <meta name="og:test1" content='foo > < bar'/>
+            <meta name="og:test2" content="foo >//< bar"/>
             '''
         self.assertEqual(ie._og_search_title(html), 'Foo')
         self.assertEqual(ie._og_search_description(html), 'Some video\'s description ')
         self.assertEqual(ie._og_search_thumbnail(html), 'http://domain.com/pic.jpg?key1=val1&key2=val2')
         self.assertEqual(ie._og_search_video_url(html, default=None), None)
         self.assertEqual(ie._og_search_property('foobar', html), 'Foo')
+        self.assertEqual(ie._og_search_property('test1', html), 'foo > < bar')
+        self.assertEqual(ie._og_search_property('test2', html), 'foo >//< bar')
 
     def test_html_search_meta(self):
         ie = self.ie
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index a0c4af92f..4365077f1 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -645,7 +645,7 @@ class InfoExtractor(object):
     # Helper functions for extracting OpenGraph info
     @staticmethod
     def _og_regexes(prop):
-        content_re = r'content=(?:"([^>]+?)"|\'([^>]+?)\'|\s*([^\s"\'=<>`]+?))'
+        content_re = r'content=(?:"([^"]+?)"|\'([^\']+?)\'|\s*([^\s"\'=<>`]+?))'
         property_re = (r'(?:name|property)=(?:\'og:%(prop)s\'|"og:%(prop)s"|\s*og:%(prop)s\b)'
                        % {'prop': re.escape(prop)})
         template = r'<meta[^>]+?%s[^>]+?%s'

From 94a773feb94a20be66526348a57ebe20495eba3f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Lalinsk=C3=BD?= <lukas@oxygene.sk>
Date: Sat, 17 Oct 2015 22:25:08 +0200
Subject: [PATCH 2829/6219] [vine] Use JS data to get title/alt_title

---
 youtube_dl/extractor/vine.py | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vine.py b/youtube_dl/extractor/vine.py
index c733a48fa..d80b580a0 100644
--- a/youtube_dl/extractor/vine.py
+++ b/youtube_dl/extractor/vine.py
@@ -51,6 +51,21 @@ class VineIE(InfoExtractor):
     }, {
         'url': 'https://vine.co/oembed/MYxVapFvz2z.json',
         'only_matching': True,
+    }, {
+        'url': 'https://vine.co/v/e192BnZnZ9V',
+        'info_dict': {
+            'id': 'e192BnZnZ9V',
+            'ext': 'mp4',
+            'title': u'\u0e22\u0e34\u0e49\u0e21~ \u0e40\u0e02\u0e34\u0e19~ \u0e2d\u0e32\u0e22~ \u0e19\u0e48\u0e32\u0e23\u0e49\u0e32\u0e01\u0e2d\u0e49\u0e30 >//< @n_whitewo @orlameena #lovesicktheseries  #lovesickseason2',
+            'alt_title': 'Vine by Pimry_zaa',
+            'description': u'\u0e22\u0e34\u0e49\u0e21~ \u0e40\u0e02\u0e34\u0e19~ \u0e2d\u0e32\u0e22~ \u0e19\u0e48\u0e32\u0e23\u0e49\u0e32\u0e01\u0e2d\u0e49\u0e30 >//< @n_whitewo @orlameena #lovesicktheseries  #lovesickseason2',
+            'upload_date': '20150705',
+            'uploader': 'Pimry_zaa',
+            'uploader_id': '1135760698325307392',
+        },
+        'params': {
+            'skip_download': True,
+        },
     }]
 
     def _real_extract(self, url):
@@ -74,8 +89,8 @@ class VineIE(InfoExtractor):
 
         return {
             'id': video_id,
-            'title': self._og_search_title(webpage),
-            'alt_title': self._og_search_description(webpage, default=None),
+            'title': data['description'],
+            'alt_title': 'Vine by %s' % data['username'],
             'description': data['description'],
             'thumbnail': data['thumbnailUrl'],
             'upload_date': unified_strdate(data['created']),

From 10c38c7ca248d06c2c0f069c5a810e27e207c61e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Lalinsk=C3=BD?= <lukas@oxygene.sk>
Date: Sat, 17 Oct 2015 22:29:49 +0200
Subject: [PATCH 2830/6219] [vine] Fix download tests

---
 youtube_dl/extractor/vine.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vine.py b/youtube_dl/extractor/vine.py
index d80b580a0..d1dbec893 100644
--- a/youtube_dl/extractor/vine.py
+++ b/youtube_dl/extractor/vine.py
@@ -29,10 +29,10 @@ class VineIE(InfoExtractor):
             'id': 'MYxVapFvz2z',
             'ext': 'mp4',
             'title': 'Fuck Da Police #Mikebrown #justice #ferguson #prayforferguson #protesting #NMOS14',
-            'alt_title': 'Vine by Luna',
+            'alt_title': 'Vine by Mars Ruiz',
             'description': 'Fuck Da Police #Mikebrown #justice #ferguson #prayforferguson #protesting #NMOS14',
             'upload_date': '20140815',
-            'uploader': 'Luna',
+            'uploader': 'Mars Ruiz',
             'uploader_id': '1102363502380728320',
         },
     }, {

From 91816e8f16408a3a2753fb254a9e963ad9429ced Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 18 Oct 2015 09:32:08 +0600
Subject: [PATCH 2831/6219] [vine] Remove duplicate metadata, make more robust
 and modernize (Closes #7215)

---
 youtube_dl/extractor/vine.py | 39 ++++++++++++++++++------------------
 1 file changed, 20 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/extractor/vine.py b/youtube_dl/extractor/vine.py
index d1dbec893..6e72cc253 100644
--- a/youtube_dl/extractor/vine.py
+++ b/youtube_dl/extractor/vine.py
@@ -1,10 +1,14 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
 import re
 import itertools
 
 from .common import InfoExtractor
-from ..utils import unified_strdate
+from ..utils import (
+    int_or_none,
+    unified_strdate,
+)
 
 
 class VineIE(InfoExtractor):
@@ -17,7 +21,6 @@ class VineIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Chicken.',
             'alt_title': 'Vine by Jack Dorsey',
-            'description': 'Chicken.',
             'upload_date': '20130519',
             'uploader': 'Jack Dorsey',
             'uploader_id': '76',
@@ -30,7 +33,6 @@ class VineIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Fuck Da Police #Mikebrown #justice #ferguson #prayforferguson #protesting #NMOS14',
             'alt_title': 'Vine by Mars Ruiz',
-            'description': 'Fuck Da Police #Mikebrown #justice #ferguson #prayforferguson #protesting #NMOS14',
             'upload_date': '20140815',
             'uploader': 'Mars Ruiz',
             'uploader_id': '1102363502380728320',
@@ -43,7 +45,6 @@ class VineIE(InfoExtractor):
             'ext': 'mp4',
             'title': '#mw3 #ac130 #killcam #angelofdeath',
             'alt_title': 'Vine by Z3k3',
-            'description': '#mw3 #ac130 #killcam #angelofdeath',
             'upload_date': '20130430',
             'uploader': 'Z3k3',
             'uploader_id': '936470460173008896',
@@ -56,9 +57,8 @@ class VineIE(InfoExtractor):
         'info_dict': {
             'id': 'e192BnZnZ9V',
             'ext': 'mp4',
-            'title': u'\u0e22\u0e34\u0e49\u0e21~ \u0e40\u0e02\u0e34\u0e19~ \u0e2d\u0e32\u0e22~ \u0e19\u0e48\u0e32\u0e23\u0e49\u0e32\u0e01\u0e2d\u0e49\u0e30 >//< @n_whitewo @orlameena #lovesicktheseries  #lovesickseason2',
+            'title': 'ยิ้ม~ เขิน~ อาย~ น่าร้ากอ้ะ >//< @n_whitewo @orlameena #lovesicktheseries  #lovesickseason2',
             'alt_title': 'Vine by Pimry_zaa',
-            'description': u'\u0e22\u0e34\u0e49\u0e21~ \u0e40\u0e02\u0e34\u0e19~ \u0e2d\u0e32\u0e22~ \u0e19\u0e48\u0e32\u0e23\u0e49\u0e32\u0e01\u0e2d\u0e49\u0e30 >//< @n_whitewo @orlameena #lovesicktheseries  #lovesickseason2',
             'upload_date': '20150705',
             'uploader': 'Pimry_zaa',
             'uploader_id': '1135760698325307392',
@@ -80,25 +80,26 @@ class VineIE(InfoExtractor):
 
         formats = [{
             'format_id': '%(format)s-%(rate)s' % f,
-            'vcodec': f['format'],
-            'quality': f['rate'],
+            'vcodec': f.get('format'),
+            'quality': f.get('rate'),
             'url': f['videoUrl'],
-        } for f in data['videoUrls']]
+        } for f in data['videoUrls'] if f.get('videoUrl')]
 
         self._sort_formats(formats)
 
+        username = data.get('username')
+
         return {
             'id': video_id,
-            'title': data['description'],
-            'alt_title': 'Vine by %s' % data['username'],
-            'description': data['description'],
-            'thumbnail': data['thumbnailUrl'],
-            'upload_date': unified_strdate(data['created']),
-            'uploader': data['username'],
-            'uploader_id': data['userIdStr'],
-            'like_count': data['likes']['count'],
-            'comment_count': data['comments']['count'],
-            'repost_count': data['reposts']['count'],
+            'title': data.get('description') or self._og_search_title(webpage),
+            'alt_title': 'Vine by %s' % username if username else self._og_search_description(webpage, default=None),
+            'thumbnail': data.get('thumbnailUrl'),
+            'upload_date': unified_strdate(data.get('created')),
+            'uploader': username,
+            'uploader_id': data.get('userIdStr'),
+            'like_count': int_or_none(data.get('likes', {}).get('count')),
+            'comment_count': int_or_none(data.get('comments', {}).get('count')),
+            'repost_count': int_or_none(data.get('reposts', {}).get('count')),
             'formats': formats,
         }
 

From 02835c6bf4403a907c058d43220a83b3b427e181 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 18 Oct 2015 09:34:54 +0600
Subject: [PATCH 2832/6219] [extractor/common] Document repost_count

---
 youtube_dl/extractor/common.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 4365077f1..6169fbbeb 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -172,6 +172,7 @@ class InfoExtractor(object):
     view_count:     How many users have watched the video on the platform.
     like_count:     Number of positive ratings of the video
     dislike_count:  Number of negative ratings of the video
+    repost_count:   Number of reposts of the video
     average_rating: Average rating give by users, the scale used depends on the webpage
     comment_count:  Number of comments on the video
     comments:       A list of comments, each with one or more of the following

From 2e022397c45fbcfd2ef6da43d14b0770221aabd5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 18 Oct 2015 09:36:19 +0600
Subject: [PATCH 2833/6219] [vine] Add counters to tests

---
 youtube_dl/extractor/vine.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/youtube_dl/extractor/vine.py b/youtube_dl/extractor/vine.py
index 6e72cc253..be72f3147 100644
--- a/youtube_dl/extractor/vine.py
+++ b/youtube_dl/extractor/vine.py
@@ -24,6 +24,9 @@ class VineIE(InfoExtractor):
             'upload_date': '20130519',
             'uploader': 'Jack Dorsey',
             'uploader_id': '76',
+            'like_count': int,
+            'comment_count': int,
+            'repost_count': int,
         },
     }, {
         'url': 'https://vine.co/v/MYxVapFvz2z',
@@ -36,6 +39,9 @@ class VineIE(InfoExtractor):
             'upload_date': '20140815',
             'uploader': 'Mars Ruiz',
             'uploader_id': '1102363502380728320',
+            'like_count': int,
+            'comment_count': int,
+            'repost_count': int,
         },
     }, {
         'url': 'https://vine.co/v/bxVjBbZlPUH',
@@ -48,6 +54,9 @@ class VineIE(InfoExtractor):
             'upload_date': '20130430',
             'uploader': 'Z3k3',
             'uploader_id': '936470460173008896',
+            'like_count': int,
+            'comment_count': int,
+            'repost_count': int,
         },
     }, {
         'url': 'https://vine.co/oembed/MYxVapFvz2z.json',
@@ -62,6 +71,9 @@ class VineIE(InfoExtractor):
             'upload_date': '20150705',
             'uploader': 'Pimry_zaa',
             'uploader_id': '1135760698325307392',
+            'like_count': int,
+            'comment_count': int,
+            'repost_count': int,
         },
         'params': {
             'skip_download': True,

From 1e399778ee870ee583135e65458268cd7c0fb923 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 22 Jul 2015 20:03:05 +0800
Subject: [PATCH 2834/6219] [letv] Fix extraction

Using data URIs for passing the decrypted M3U8 manifest, which is
supported by ffmpeg only.
---
 youtube_dl/extractor/letv.py | 70 ++++++++++++++++++++++++++----------
 youtube_dl/utils.py          |  5 +++
 2 files changed, 57 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/letv.py b/youtube_dl/extractor/letv.py
index a28abb0f0..9ebbc8089 100644
--- a/youtube_dl/extractor/letv.py
+++ b/youtube_dl/extractor/letv.py
@@ -9,13 +9,14 @@ from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse,
     compat_urllib_request,
-    compat_urlparse,
+    compat_ord,
 )
 from ..utils import (
     determine_ext,
     ExtractorError,
     parse_iso8601,
     int_or_none,
+    encode_data_uri,
 )
 
 
@@ -25,15 +26,16 @@ class LetvIE(InfoExtractor):
 
     _TESTS = [{
         'url': 'http://www.letv.com/ptv/vplay/22005890.html',
-        'md5': 'cab23bd68d5a8db9be31c9a222c1e8df',
+        'md5': 'edadcfe5406976f42f9f266057ee5e40',
         'info_dict': {
             'id': '22005890',
             'ext': 'mp4',
             'title': '第87届奥斯卡颁奖礼完美落幕 《鸟人》成最大赢家',
-            'timestamp': 1424747397,
-            'upload_date': '20150224',
             'description': 'md5:a9cb175fd753e2962176b7beca21a47c',
-        }
+        },
+        'params': {
+            'hls_prefer_native': True,
+        },
     }, {
         'url': 'http://www.letv.com/ptv/vplay/1415246.html',
         'info_dict': {
@@ -42,16 +44,22 @@ class LetvIE(InfoExtractor):
             'title': '美人天下01',
             'description': 'md5:f88573d9d7225ada1359eaf0dbf8bcda',
         },
+        'params': {
+            'hls_prefer_native': True,
+        },
     }, {
         'note': 'This video is available only in Mainland China, thus a proxy is needed',
         'url': 'http://www.letv.com/ptv/vplay/1118082.html',
-        'md5': 'f80936fbe20fb2f58648e81386ff7927',
+        'md5': '2424c74948a62e5f31988438979c5ad1',
         'info_dict': {
             'id': '1118082',
             'ext': 'mp4',
             'title': '与龙共舞 完整版',
             'description': 'md5:7506a5eeb1722bb9d4068f85024e3986',
         },
+        'params': {
+            'hls_prefer_native': True,
+        },
         'skip': 'Only available in China',
     }]
 
@@ -74,6 +82,27 @@ class LetvIE(InfoExtractor):
         _loc3_ = self.ror(_loc3_, _loc2_ % 17)
         return _loc3_
 
+    # see M3U8Encryption class in KLetvPlayer.swf
+    @staticmethod
+    def decrypt_m3u8(encrypted_data):
+        if encrypted_data[:5].decode('utf-8').lower() != 'vc_01':
+            return encrypted_data
+        encrypted_data = encrypted_data[5:]
+
+        _loc4_ = bytearray()
+        while encrypted_data:
+            b = compat_ord(encrypted_data[0])
+            _loc4_.extend([b // 16, b & 0x0f])
+            encrypted_data = encrypted_data[1:]
+        idx = len(_loc4_) - 11
+        _loc4_ = _loc4_[idx:] + _loc4_[:idx]
+        _loc7_ = bytearray()
+        while _loc4_:
+            _loc7_.append(_loc4_[0] * 16 + _loc4_[1])
+            _loc4_ = _loc4_[2:]
+
+        return bytes(_loc7_)
+
     def _real_extract(self, url):
         media_id = self._match_id(url)
         page = self._download_webpage(url, media_id)
@@ -115,23 +144,28 @@ class LetvIE(InfoExtractor):
         for format_id in formats:
             if format_id in dispatch:
                 media_url = playurl['domain'][0] + dispatch[format_id][0]
-
-                # Mimic what flvxz.com do
-                url_parts = list(compat_urlparse.urlparse(media_url))
-                qs = dict(compat_urlparse.parse_qs(url_parts[4]))
-                qs.update({
-                    'platid': '14',
-                    'splatid': '1401',
-                    'tss': 'no',
-                    'retry': 1
+                media_url += '&' + compat_urllib_parse.urlencode({
+                    'm3v': 1,
+                    'format': 1,
+                    'expect': 3,
+                    'rateid': format_id,
                 })
-                url_parts[4] = compat_urllib_parse.urlencode(qs)
-                media_url = compat_urlparse.urlunparse(url_parts)
+
+                nodes_data = self._download_json(
+                    media_url, media_id,
+                    'Download JSON metadata for format %s' % format_id)
+
+                req = self._request_webpage(
+                    nodes_data['nodelist'][0]['location'], media_id,
+                    note='Downloading m3u8 information for format %s' % format_id)
+
+                m3u8_data = self.decrypt_m3u8(req.read())
 
                 url_info_dict = {
-                    'url': media_url,
+                    'url': encode_data_uri(m3u8_data, 'application/x-mpegURL'),
                     'ext': determine_ext(dispatch[format_id][1]),
                     'format_id': format_id,
+                    'protocol': 'm3u8',
                 }
 
                 if format_id[-1:] == 'p':
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 7dbe25661..db5b3698e 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -3,6 +3,7 @@
 
 from __future__ import unicode_literals
 
+import base64
 import calendar
 import codecs
 import contextlib
@@ -1795,6 +1796,10 @@ def urlhandle_detect_ext(url_handle):
     return mimetype2ext(getheader('Content-Type'))
 
 
+def encode_data_uri(data, mime_type):
+    return 'data:%s;base64,%s' % (mime_type, base64.b64encode(data).decode('ascii'))
+
+
 def age_restricted(content_limit, age_limit):
     """ Returns True iff the content should be blocked """
 

From 985e4fdc07f00a3fdc8e7b7b4119471ee97f3890 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 17 Oct 2015 22:49:05 +0800
Subject: [PATCH 2835/6219] [downloader/hls] Add headers only for http(s) URLs

ffmpeg 2.8.1 raises an error with -headers and non-http input files.
---
 youtube_dl/downloader/hls.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index a62d2047b..9a83a73dd 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -30,7 +30,7 @@ class HlsFD(FileDownloader):
 
         args = [ffpp.executable, '-y']
 
-        if info_dict['http_headers']:
+        if info_dict['http_headers'] and re.match(r'^https?://', url):
             # Trailing \r\n after each HTTP header is important to prevent warning from ffmpeg/avconv:
             # [http @ 00000000003d2fa0] No trailing CRLF found in HTTP header.
             args += [

From 0a67a3632bb9cf76f64658986defc1947090ef50 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 17 Oct 2015 23:15:01 +0800
Subject: [PATCH 2836/6219] [compat] Add compat_urllib_request_DataHandler

---
 youtube_dl/compat.py | 44 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 44 insertions(+)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 192e1c515..d103ab9ad 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -1,7 +1,10 @@
 from __future__ import unicode_literals
 
+import binascii
 import collections
+import email
 import getpass
+import io
 import optparse
 import os
 import re
@@ -38,6 +41,11 @@ try:
 except ImportError:  # Python 2
     import urlparse as compat_urlparse
 
+try:
+    import urllib.response as compat_urllib_response
+except ImportError:  # Python 2
+    import urllib as compat_urllib_response
+
 try:
     import http.cookiejar as compat_cookiejar
 except ImportError:  # Python 2
@@ -155,6 +163,40 @@ except ImportError:  # Python 2
         string = string.replace('+', ' ')
         return compat_urllib_parse_unquote(string, encoding, errors)
 
+try:
+    from urllib.request import DataHandler as compat_urllib_request_DataHandler
+except ImportError:  # Python < 3.4
+    # Ported from CPython 98774:1733b3bd46db, Lib/urllib/request.py
+    class compat_urllib_request_DataHandler(compat_urllib_request.BaseHandler):
+        def data_open(self, req):
+            # data URLs as specified in RFC 2397.
+            #
+            # ignores POSTed data
+            #
+            # syntax:
+            # dataurl   := "data:" [ mediatype ] [ ";base64" ] "," data
+            # mediatype := [ type "/" subtype ] *( ";" parameter )
+            # data      := *urlchar
+            # parameter := attribute "=" value
+            url = req.get_full_url()
+
+            scheme, data = url.split(":", 1)
+            mediatype, data = data.split(",", 1)
+
+            # even base64 encoded data URLs might be quoted so unquote in any case:
+            data = compat_urllib_parse_unquote_to_bytes(data)
+            if mediatype.endswith(";base64"):
+                data = binascii.a2b_base64(data)
+                mediatype = mediatype[:-7]
+
+            if not mediatype:
+                mediatype = "text/plain;charset=US-ASCII"
+
+            headers = email.message_from_string(
+                "Content-type: %s\nContent-length: %d\n" % (mediatype, len(data)))
+
+            return compat_urllib_response.addinfourl(io.BytesIO(data), headers, url)
+
 try:
     compat_basestring = basestring  # Python 2
 except NameError:
@@ -489,6 +531,8 @@ __all__ = [
     'compat_urllib_parse_unquote_to_bytes',
     'compat_urllib_parse_urlparse',
     'compat_urllib_request',
+    'compat_urllib_request_DataHandler',
+    'compat_urllib_response',
     'compat_urlparse',
     'compat_urlretrieve',
     'compat_xml_parse_error',

From 8b172c2e10fb38c62c213673304c7e8dcd17b768 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 17 Oct 2015 23:16:40 +0800
Subject: [PATCH 2837/6219] [YoutubeDL] Use DataHandler

---
 youtube_dl/YoutubeDL.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index adf70d658..12977bf80 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -37,6 +37,7 @@ from .compat import (
     compat_tokenize_tokenize,
     compat_urllib_error,
     compat_urllib_request,
+    compat_urllib_request_DataHandler,
 )
 from .utils import (
     ContentTooShortError,
@@ -1967,8 +1968,9 @@ class YoutubeDL(object):
         debuglevel = 1 if self.params.get('debug_printtraffic') else 0
         https_handler = make_HTTPS_handler(self.params, debuglevel=debuglevel)
         ydlh = YoutubeDLHandler(self.params, debuglevel=debuglevel)
+        data_handler = compat_urllib_request_DataHandler()
         opener = compat_urllib_request.build_opener(
-            proxy_handler, https_handler, cookie_processor, ydlh)
+            proxy_handler, https_handler, cookie_processor, ydlh, data_handler)
 
         # Delete the default user-agent header, which would otherwise apply in
         # cases where our custom HTTP handler doesn't come into play

From 48aae2d2cf49843d0efa227fa393a0c783fc3c1e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 18 Oct 2015 17:07:48 +0800
Subject: [PATCH 2838/6219] [twitter] Update tests

---
 youtube_dl/extractor/twitter.py | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 1dd43ff3c..b2fff73b9 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -1,3 +1,4 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
 import re
@@ -15,7 +16,7 @@ class TwitterCardIE(InfoExtractor):
     _TESTS = [
         {
             'url': 'https://twitter.com/i/cards/tfw/v1/560070183650213889',
-            'md5': 'a74f50b310c83170319ba16de6955192',
+            'md5': '7d2f6b4d2eb841a7ccc893d479bfceb4',
             'info_dict': {
                 'id': '560070183650213889',
                 'ext': 'mp4',
@@ -103,17 +104,17 @@ class TwitterIE(TwitterCardIE):
     _VALID_URL = r'https?://(?:www|m|mobile)?\.?twitter\.com/(?P<id>[^/]+/status/\d+)'
 
     _TESTS = [{
-        'url': 'https://m.twitter.com/thereaIbanksy/status/614301758345490432',
-        'md5': '8bbccb487bd7a31349b775915fcd412f',
+        'url': 'https://twitter.com/freethenipple/status/643211948184596480',
+        'md5': '31cd83a116fc41f99ae3d909d4caf6a0',
         'info_dict': {
-            'id': '614301758345490432',
+            'id': '643211948184596480',
             'ext': 'mp4',
-            'title': 'thereaIbanksy - This time lapse is so pretty \U0001f60d\U0001f60d',
+            'title': 'freethenipple - FTN supporters on Hollywood Blvd today!',
             'thumbnail': 're:^https?://.*\.jpg',
-            'duration': 29.5,
-            'description': 'banksy on Twitter: "This time lapse is so pretty \U0001f60d\U0001f60d http://t.co/QB8DDbqiR1"',
-            'uploader': 'banksy',
-            'uploader_id': 'thereaIbanksy',
+            'duration': 12.922,
+            'description': 'FREE THE NIPPLE on Twitter: "FTN supporters on Hollywood Blvd today! http://t.co/c7jHH749xJ"',
+            'uploader': 'FREE THE NIPPLE',
+            'uploader_id': 'freethenipple',
         },
     }]
 

From 01d22d47039dedace1c5414c83e9fecfca41b5a5 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 18 Oct 2015 17:11:55 +0800
Subject: [PATCH 2839/6219] [twitter] Use _download_xml

---
 youtube_dl/extractor/twitter.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index b2fff73b9..37a9fd5fd 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -8,6 +8,7 @@ from ..compat import compat_urllib_request
 from ..utils import (
     float_or_none,
     unescapeHTML,
+    xpath_text,
 )
 
 
@@ -60,9 +61,8 @@ class TwitterCardIE(InfoExtractor):
                 video_id)
             if 'playlist' not in config:
                 if 'vmapUrl' in config:
-                    webpage = self._download_webpage(config['vmapUrl'], video_id + ' (xml)')
-                    video_url = self._search_regex(
-                        r'<MediaFile>\s*<!\[CDATA\[(https?://.+?)\]\]>', webpage, 'data player config (xml)')
+                    vmap_data = self._download_xml(config['vmapUrl'], video_id)
+                    video_url = xpath_text(vmap_data, './/MediaFile').strip()
                     f = {
                         'url': video_url,
                     }

From 014e880372e896cdd63f9075864d2a3bba60e706 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 18 Oct 2015 17:13:58 +0800
Subject: [PATCH 2840/6219] [twitter] Add IE_NAMEs

---
 youtube_dl/extractor/twitter.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 37a9fd5fd..5f697782e 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -13,6 +13,7 @@ from ..utils import (
 
 
 class TwitterCardIE(InfoExtractor):
+    IE_NAME = 'twitter:card'
     _VALID_URL = r'https?://(?:www\.)?twitter\.com/i/cards/tfw/v1/(?P<id>\d+)'
     _TESTS = [
         {
@@ -101,6 +102,7 @@ class TwitterCardIE(InfoExtractor):
 
 
 class TwitterIE(TwitterCardIE):
+    IE_NAME = 'twitter'
     _VALID_URL = r'https?://(?:www|m|mobile)?\.?twitter\.com/(?P<id>[^/]+/status/\d+)'
 
     _TESTS = [{

From f322bfb0638aeeb527459ebcf00f8a3dde26280c Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 18 Oct 2015 17:15:47 +0800
Subject: [PATCH 2841/6219] [twitter:card] Remove unneeded 'ext'

---
 youtube_dl/extractor/twitter.py | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 5f697782e..48bef5d80 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -64,13 +64,9 @@ class TwitterCardIE(InfoExtractor):
                 if 'vmapUrl' in config:
                     vmap_data = self._download_xml(config['vmapUrl'], video_id)
                     video_url = xpath_text(vmap_data, './/MediaFile').strip()
-                    f = {
+                    formats.append({
                         'url': video_url,
-                    }
-                    ext = re.search(r'\.([a-z0-9]{2,4})(\?.+)?$', video_url)
-                    if ext:
-                        f['ext'] = ext.group(1)
-                    formats.append(f)
+                    })
                     break   # same video regardless of UA
                 continue
 

From e04edad621efe56347e155b6dc59a0c3d589b3bd Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 18 Oct 2015 17:16:57 +0800
Subject: [PATCH 2842/6219] [twitter] Inherit from InfoExtractor directly

---
 youtube_dl/extractor/twitter.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 48bef5d80..c9b783745 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -97,11 +97,11 @@ class TwitterCardIE(InfoExtractor):
         }
 
 
-class TwitterIE(TwitterCardIE):
+class TwitterIE(InfoExtractor):
     IE_NAME = 'twitter'
     _VALID_URL = r'https?://(?:www|m|mobile)?\.?twitter\.com/(?P<id>[^/]+/status/\d+)'
 
-    _TESTS = [{
+    _TEST = {
         'url': 'https://twitter.com/freethenipple/status/643211948184596480',
         'md5': '31cd83a116fc41f99ae3d909d4caf6a0',
         'info_dict': {
@@ -114,7 +114,7 @@ class TwitterIE(TwitterCardIE):
             'uploader': 'FREE THE NIPPLE',
             'uploader_id': 'freethenipple',
         },
-    }]
+    }
 
     def _real_extract(self, url):
         id = self._match_id(url)

From f6dfd6603a9e9bb88ebcdcd52490974a34d1bd11 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 18 Oct 2015 17:18:01 +0800
Subject: [PATCH 2843/6219] [twitter] Use _html_search_regex

---
 youtube_dl/extractor/twitter.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index c9b783745..6ff15369c 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -122,7 +122,7 @@ class TwitterIE(InfoExtractor):
         name = username
         url = re.sub(r'https?://(m|mobile)\.', 'https://', url)
         webpage = self._download_webpage(url, 'tweet: ' + url)
-        description = unescapeHTML(self._search_regex('<title>\s*(.+?)\s*</title>', webpage, 'title'))
+        description = self._html_search_regex('<title>\s*(.+?)\s*</title>', webpage, 'title')
         title = description.replace('\n', ' ')
         splitdesc = re.match(r'^(.+?)\s*on Twitter:\s* "(.+?)"$', title)
         if splitdesc:

From 575036b40504bc921b18f05bde64e0e7dceacec6 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 18 Oct 2015 18:04:13 +0800
Subject: [PATCH 2844/6219] [twitter] Simplify and improve

---
 youtube_dl/extractor/twitter.py | 41 +++++++++++++++++++--------------
 1 file changed, 24 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 6ff15369c..6b3b39aee 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -9,6 +9,7 @@ from ..utils import (
     float_or_none,
     unescapeHTML,
     xpath_text,
+    remove_end,
 )
 
 
@@ -99,7 +100,8 @@ class TwitterCardIE(InfoExtractor):
 
 class TwitterIE(InfoExtractor):
     IE_NAME = 'twitter'
-    _VALID_URL = r'https?://(?:www|m|mobile)?\.?twitter\.com/(?P<id>[^/]+/status/\d+)'
+    _VALID_URL = r'https?://(?:www\.|m\.|mobile\.)?twitter\.com/(?P<user_id>[^/]+)/status/(?P<id>\d+)'
+    _TEMPLATE_URL = 'https://twitter.com/%s/status/%s'
 
     _TEST = {
         'url': 'https://twitter.com/freethenipple/status/643211948184596480',
@@ -107,7 +109,7 @@ class TwitterIE(InfoExtractor):
         'info_dict': {
             'id': '643211948184596480',
             'ext': 'mp4',
-            'title': 'freethenipple - FTN supporters on Hollywood Blvd today!',
+            'title': 'FREE THE NIPPLE - FTN supporters on Hollywood Blvd today!',
             'thumbnail': 're:^https?://.*\.jpg',
             'duration': 12.922,
             'description': 'FREE THE NIPPLE on Twitter: "FTN supporters on Hollywood Blvd today! http://t.co/c7jHH749xJ"',
@@ -117,26 +119,31 @@ class TwitterIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        id = self._match_id(url)
-        username, twid = re.match(r'([^/]+)/status/(\d+)', id).groups()
-        name = username
-        url = re.sub(r'https?://(m|mobile)\.', 'https://', url)
-        webpage = self._download_webpage(url, 'tweet: ' + url)
-        description = self._html_search_regex('<title>\s*(.+?)\s*</title>', webpage, 'title')
-        title = description.replace('\n', ' ')
-        splitdesc = re.match(r'^(.+?)\s*on Twitter:\s* "(.+?)"$', title)
-        if splitdesc:
-            name, title = splitdesc.groups()
-        title = re.sub(r'\s*https?://[^ ]+', '', title)  # strip  'https -_t.co_BJYgOjSeGA' junk from filenames
-        card_id = self._search_regex(r'["\']/i/cards/tfw/v1/(\d+)', webpage, '/i/card/...')
+        mobj = re.match(self._VALID_URL, url)
+        user_id = mobj.group('user_id')
+        twid = mobj.group('id')
+
+        webpage = self._download_webpage(self._TEMPLATE_URL % (user_id, twid), twid)
+
+        username = remove_end(self._og_search_title(webpage), ' on Twitter')
+
+        title = self._og_search_description(webpage).strip('').replace('\n', ' ')
+
+        # strip  'https -_t.co_BJYgOjSeGA' junk from filenames
+        mobj = re.match(r'“(.*)\s+(http://[^ ]+)”', title)
+        title, short_url = mobj.groups()
+
+        card_id = self._search_regex(
+            r'["\']/i/cards/tfw/v1/(\d+)', webpage, 'twitter card url')
         card_url = 'https://twitter.com/i/cards/tfw/v1/' + card_id
+
         return {
             '_type': 'url_transparent',
             'ie_key': 'TwitterCard',
-            'uploader_id': username,
-            'uploader': name,
+            'uploader_id': user_id,
+            'uploader': username,
             'url': card_url,
             'webpage_url': url,
-            'description': description,
+            'description': '%s on Twitter: "%s %s"' % (username, title, short_url),
             'title': username + ' - ' + title,
         }

From 77a54b6a658059a11de415d793588fdbfec14194 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 18 Oct 2015 18:08:24 +0800
Subject: [PATCH 2845/6219] [twitter:card] Use _html_search_regex

---
 youtube_dl/extractor/twitter.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 6b3b39aee..1cdca544c 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -7,7 +7,6 @@ from .common import InfoExtractor
 from ..compat import compat_urllib_request
 from ..utils import (
     float_or_none,
-    unescapeHTML,
     xpath_text,
     remove_end,
 )
@@ -57,9 +56,8 @@ class TwitterCardIE(InfoExtractor):
             request.add_header('User-Agent', user_agent)
             webpage = self._download_webpage(request, video_id)
 
-            config = self._parse_json(
-                unescapeHTML(self._search_regex(
-                    r'data-player-config="([^"]+)"', webpage, 'data player config')),
+            config = self._parse_json(self._html_search_regex(
+                r'data-player-config="([^"]+)"', webpage, 'data player config'),
                 video_id)
             if 'playlist' not in config:
                 if 'vmapUrl' in config:

From c88aec845a680ef9404b637b3dbcf706dcf00b68 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 18 Oct 2015 18:23:56 +0800
Subject: [PATCH 2846/6219] [twitter] Fix short URL extraction

---
 youtube_dl/extractor/twitter.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 1cdca544c..1472f22a7 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -128,7 +128,7 @@ class TwitterIE(InfoExtractor):
         title = self._og_search_description(webpage).strip('').replace('\n', ' ')
 
         # strip  'https -_t.co_BJYgOjSeGA' junk from filenames
-        mobj = re.match(r'“(.*)\s+(http://[^ ]+)”', title)
+        mobj = re.match(r'“(.*)\s+(https?://[^ ]+)”', title)
         title, short_url = mobj.groups()
 
         card_id = self._search_regex(

From 4a7b79038425f614af49116edab7897f0db13e5a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 18 Oct 2015 19:07:37 +0800
Subject: [PATCH 2847/6219] [twitter:card] Support YouTube embeds

---
 youtube_dl/extractor/twitter.py | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 1472f22a7..9d3e46b94 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -37,6 +37,19 @@ class TwitterCardIE(InfoExtractor):
                 'thumbnail': 're:^https?://.*\.jpg',
                 'duration': 80.155,
             },
+        },
+        {
+            'url': 'https://twitter.com/i/cards/tfw/v1/654001591733886977',
+            'md5': 'b6f35e8b08a0bec6c8af77a2f4b3a814',
+            'info_dict': {
+                'id': 'dq4Oj5quskI',
+                'ext': 'mp4',
+                'title': 'Ubuntu 11.10 Overview',
+                'description': 'Take a quick peek at what\'s new and improved in Ubuntu 11.10.\n\nOnce installed take a look at 10 Things to Do After Installing: http://www.omgubuntu.co.uk/2011/10/10-things-to-do-after-installing-ubuntu-11-10/',
+                'upload_date': '20111013',
+                'uploader': 'OMG! Ubuntu!',
+                'uploader_id': 'omgubuntu',
+            },
         }
     ]
 
@@ -56,6 +69,12 @@ class TwitterCardIE(InfoExtractor):
             request.add_header('User-Agent', user_agent)
             webpage = self._download_webpage(request, video_id)
 
+            youtube_url = self._html_search_regex(
+                r'<iframe[^>]+src="((?:https?:)?//www.youtube.com/embed/[^"]+)"',
+                webpage, 'youtube iframe', default=None)
+            if youtube_url:
+                return self.url_result(youtube_url, 'Youtube')
+
             config = self._parse_json(self._html_search_regex(
                 r'data-player-config="([^"]+)"', webpage, 'data player config'),
                 video_id)

From 05a3879f1c142cc2bf0287cde4690d8ccadcdc8f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 18 Oct 2015 19:19:46 +0800
Subject: [PATCH 2848/6219] [letv] Update M3U8's MIME type

The new MIME type appears in the following places:
https://www.iana.org/assignments/media-types/media-types.xhtml#application
https://hg.python.org/cpython/file/tip/Lib/mimetypes.py
---
 youtube_dl/extractor/letv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/letv.py b/youtube_dl/extractor/letv.py
index 9ebbc8089..effd9eb92 100644
--- a/youtube_dl/extractor/letv.py
+++ b/youtube_dl/extractor/letv.py
@@ -162,7 +162,7 @@ class LetvIE(InfoExtractor):
                 m3u8_data = self.decrypt_m3u8(req.read())
 
                 url_info_dict = {
-                    'url': encode_data_uri(m3u8_data, 'application/x-mpegURL'),
+                    'url': encode_data_uri(m3u8_data, 'application/vnd.apple.mpegurl'),
                     'ext': determine_ext(dispatch[format_id][1]),
                     'format_id': format_id,
                     'protocol': 'm3u8',

From dd67702a3ea007369109ee8e4b67043064e1f759 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 18 Oct 2015 14:13:06 +0200
Subject: [PATCH 2849/6219] [imdb] Fix extraction (fixes #7220)

---
 youtube_dl/extractor/imdb.py | 29 +++++++++++++++++++----------
 1 file changed, 19 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/imdb.py b/youtube_dl/extractor/imdb.py
index 4bb574cf3..02e1e428e 100644
--- a/youtube_dl/extractor/imdb.py
+++ b/youtube_dl/extractor/imdb.py
@@ -4,8 +4,8 @@ import re
 import json
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urlparse,
+from ..utils import (
+    qualities,
 )
 
 
@@ -30,24 +30,33 @@ class ImdbIE(InfoExtractor):
         descr = self._html_search_regex(
             r'(?s)<span itemprop="description">(.*?)</span>',
             webpage, 'description', fatal=False)
-        available_formats = re.findall(
-            r'case \'(?P<f_id>.*?)\' :$\s+url = \'(?P<path>.*?)\'', webpage,
-            flags=re.MULTILINE)
+        player_url = 'http://www.imdb.com/video/imdb/vi%s/imdb/single' % video_id
+        player_page = self._download_webpage(
+            player_url, video_id, 'Downloading player page')
+        # the player page contains the info for the default format, we have to
+        # fetch other pages for the rest of the formats
+        extra_formats = re.findall(r'href="(?P<url>%s.*?)".*?>(?P<name>.*?)<' % re.escape(player_url), player_page)
+        format_pages = [
+            self._download_webpage(
+                f_url, video_id, 'Downloading info for %s format' % f_name)
+            for f_url, f_name in extra_formats]
+        format_pages.append(player_page)
+
+        quality = qualities(['SD', '480p', '720p'])
         formats = []
-        for f_id, f_path in available_formats:
-            f_path = f_path.strip()
-            format_page = self._download_webpage(
-                compat_urlparse.urljoin(url, f_path),
-                'Downloading info for %s format' % f_id)
+        for format_page in format_pages:
             json_data = self._search_regex(
                 r'<script[^>]+class="imdb-player-data"[^>]*?>(.*?)</script>',
                 format_page, 'json data', flags=re.DOTALL)
             info = json.loads(json_data)
             format_info = info['videoPlayerObject']['video']
+            f_id = format_info['ffname']
             formats.append({
                 'format_id': f_id,
                 'url': format_info['videoInfoList'][0]['videoUrl'],
+                'quality': quality(f_id),
             })
+        self._sort_formats(formats)
 
         return {
             'id': video_id,

From b0f001a6cbd220c8b10c0ce359f17072d6347a8f Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 21 Sep 2015 15:52:36 +0100
Subject: [PATCH 2850/6219] [canalc2] fix info extraction

---
 youtube_dl/extractor/canalc2.py | 30 ++++++++++++++++++------------
 1 file changed, 18 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/canalc2.py b/youtube_dl/extractor/canalc2.py
index c4fefefe4..66a9ff093 100644
--- a/youtube_dl/extractor/canalc2.py
+++ b/youtube_dl/extractor/canalc2.py
@@ -8,34 +8,40 @@ from .common import InfoExtractor
 
 class Canalc2IE(InfoExtractor):
     IE_NAME = 'canalc2.tv'
-    _VALID_URL = r'http://.*?\.canalc2\.tv/video\.asp\?.*?idVideo=(?P<id>\d+)'
+    _VALID_URL = r'https?://(www\.)?canalc2\.tv/video/(?P<id>\d+)'
 
     _TEST = {
-        'url': 'http://www.canalc2.tv/video.asp?idVideo=12163&voir=oui',
+        'url': 'http://www.canalc2.tv/video/12163',
         'md5': '060158428b650f896c542dfbb3d6487f',
         'info_dict': {
             'id': '12163',
             'ext': 'mp4',
             'title': 'Terrasses du Numérique'
+        },
+        'params': {
+            'skip_download': True,  # Requires rtmpdump
         }
     }
 
     def _real_extract(self, url):
-        video_id = re.match(self._VALID_URL, url).group('id')
-        # We need to set the voir field for getting the file name
-        url = 'http://www.canalc2.tv/video.asp?idVideo=%s&voir=oui' % video_id
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
-        file_name = self._search_regex(
-            r"so\.addVariable\('file','(.*?)'\);",
-            webpage, 'file name')
-        video_url = 'http://vod-flash.u-strasbg.fr:8080/' + file_name
+        video_url = self._search_regex(
+            r'jwplayer\("Player"\).setup\({[^}]*file: "([^"]+)"',
+            webpage, 'video_url')
+        formats = [{'url': video_url}]
+        if video_url.startswith('rtmp://'):
+            rtmp = re.search(r'^(?P<url>rtmp://[^/]+/(?P<app>.+))/(?P<play_path>mp4:.+)$', video_url)
+            formats[0].update({
+                'app': rtmp.group('app'),
+                'play_path': rtmp.group('play_path'),
+            })
 
         title = self._html_search_regex(
-            r'class="evenement8">(.*?)</a>', webpage, 'title')
+            r'(?s)class="[^"]*col_description[^"]*">.*?<h3>(.*?)</h3>', webpage, 'title')
 
         return {
             'id': video_id,
-            'ext': 'mp4',
-            'url': video_url,
+            'formats': formats,
             'title': title,
         }

From 6682049dee5e73b98e99e1359b959240d0920d6b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 18 Oct 2015 19:19:43 +0600
Subject: [PATCH 2851/6219] [canalc2] Improve rtmp extraction

---
 youtube_dl/extractor/canalc2.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/canalc2.py b/youtube_dl/extractor/canalc2.py
index 66a9ff093..648af2e18 100644
--- a/youtube_dl/extractor/canalc2.py
+++ b/youtube_dl/extractor/canalc2.py
@@ -31,10 +31,12 @@ class Canalc2IE(InfoExtractor):
             webpage, 'video_url')
         formats = [{'url': video_url}]
         if video_url.startswith('rtmp://'):
-            rtmp = re.search(r'^(?P<url>rtmp://[^/]+/(?P<app>.+))/(?P<play_path>mp4:.+)$', video_url)
+            rtmp = re.search(r'^(?P<url>rtmp://[^/]+/(?P<app>.+/))(?P<play_path>mp4:.+)$', video_url)
             formats[0].update({
+                'url': rtmp.group('url'),
                 'app': rtmp.group('app'),
                 'play_path': rtmp.group('play_path'),
+                'page_url': url,
             })
 
         title = self._html_search_regex(

From ef6c868f23f2fe0d493831e0d4cba71c735bd160 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 18 Oct 2015 19:23:31 +0600
Subject: [PATCH 2852/6219] [canalc2] Improve some regexes

---
 youtube_dl/extractor/canalc2.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/canalc2.py b/youtube_dl/extractor/canalc2.py
index 648af2e18..d9137e2ef 100644
--- a/youtube_dl/extractor/canalc2.py
+++ b/youtube_dl/extractor/canalc2.py
@@ -8,7 +8,7 @@ from .common import InfoExtractor
 
 class Canalc2IE(InfoExtractor):
     IE_NAME = 'canalc2.tv'
-    _VALID_URL = r'https?://(www\.)?canalc2\.tv/video/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?canalc2\.tv/video/(?P<id>\d+)'
 
     _TEST = {
         'url': 'http://www.canalc2.tv/video/12163',
@@ -27,8 +27,8 @@ class Canalc2IE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
         video_url = self._search_regex(
-            r'jwplayer\("Player"\).setup\({[^}]*file: "([^"]+)"',
-            webpage, 'video_url')
+            r'jwplayer\((["\'])Player\1\)\.setup\({[^}]*file\s*:\s*(["\'])(?P<file>.+?)\2',
+            webpage, 'video_url', group='file')
         formats = [{'url': video_url}]
         if video_url.startswith('rtmp://'):
             rtmp = re.search(r'^(?P<url>rtmp://[^/]+/(?P<app>.+/))(?P<play_path>mp4:.+)$', video_url)

From 14bddf35fbe8253e283042630e24b134996b2575 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 18 Oct 2015 19:23:52 +0600
Subject: [PATCH 2853/6219] [canalc2] Add ext

---
 youtube_dl/extractor/canalc2.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/canalc2.py b/youtube_dl/extractor/canalc2.py
index d9137e2ef..ba82bb2b7 100644
--- a/youtube_dl/extractor/canalc2.py
+++ b/youtube_dl/extractor/canalc2.py
@@ -34,6 +34,7 @@ class Canalc2IE(InfoExtractor):
             rtmp = re.search(r'^(?P<url>rtmp://[^/]+/(?P<app>.+/))(?P<play_path>mp4:.+)$', video_url)
             formats[0].update({
                 'url': rtmp.group('url'),
+                'ext': 'flv',
                 'app': rtmp.group('app'),
                 'play_path': rtmp.group('play_path'),
                 'page_url': url,

From b1bf063503893192637f95e929d1a9147de59a7e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 18 Oct 2015 19:27:05 +0600
Subject: [PATCH 2854/6219] [canalc2] Extract duration

---
 youtube_dl/extractor/canalc2.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/canalc2.py b/youtube_dl/extractor/canalc2.py
index ba82bb2b7..e326b8fbd 100644
--- a/youtube_dl/extractor/canalc2.py
+++ b/youtube_dl/extractor/canalc2.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..utils import parse_duration
 
 
 class Canalc2IE(InfoExtractor):
@@ -42,9 +43,13 @@ class Canalc2IE(InfoExtractor):
 
         title = self._html_search_regex(
             r'(?s)class="[^"]*col_description[^"]*">.*?<h3>(.*?)</h3>', webpage, 'title')
+        duration = parse_duration(self._search_regex(
+            r'id=["\']video_duree["\'][^>]*>([^<]+)',
+            webpage, 'duration', fatal=False))
 
         return {
             'id': video_id,
-            'formats': formats,
             'title': title,
+            'duration': duration,
+            'formats': formats,
         }

From 608945d44a7e47fa5115295839c993af545936eb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 18 Oct 2015 19:27:22 +0600
Subject: [PATCH 2855/6219] [canalc2] Fix test

---
 youtube_dl/extractor/canalc2.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/canalc2.py b/youtube_dl/extractor/canalc2.py
index e326b8fbd..f6a1ff381 100644
--- a/youtube_dl/extractor/canalc2.py
+++ b/youtube_dl/extractor/canalc2.py
@@ -16,8 +16,9 @@ class Canalc2IE(InfoExtractor):
         'md5': '060158428b650f896c542dfbb3d6487f',
         'info_dict': {
             'id': '12163',
-            'ext': 'mp4',
-            'title': 'Terrasses du Numérique'
+            'ext': 'flv',
+            'title': 'Terrasses du Numérique',
+            'duration': 122,
         },
         'params': {
             'skip_download': True,  # Requires rtmpdump

From dedd35c6bc33eb88f19b16eeb37498cee076c47a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 18 Oct 2015 19:59:18 +0600
Subject: [PATCH 2856/6219] [viewster] Fix failing m3u8

---
 youtube_dl/extractor/viewster.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/viewster.py b/youtube_dl/extractor/viewster.py
index 632e57fb4..7cf930d69 100644
--- a/youtube_dl/extractor/viewster.py
+++ b/youtube_dl/extractor/viewster.py
@@ -131,10 +131,11 @@ class ViewsterIE(InfoExtractor):
                 formats.extend(self._extract_f4m_formats(
                     video_url, video_id, f4m_id='hds'))
             elif ext == 'm3u8':
-                formats.extend(self._extract_m3u8_formats(
+                m3u8_formats = self._extract_m3u8_formats(
                     video_url, video_id, 'mp4', m3u8_id='hls',
-                    fatal=False  # m3u8 sometimes fail
-                ))
+                    fatal=False)  # m3u8 sometimes fail
+                if m3u8_formats:
+                    formats.extend(m3u8_formats)
             else:
                 format_id = media.get('Bitrate')
                 f = {

From e36963e0eb57294f156a98c38df891dec41ebaa4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 18 Oct 2015 20:24:33 +0600
Subject: [PATCH 2857/6219] [eagleplatform] Identify hls formats

---
 youtube_dl/extractor/eagleplatform.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/eagleplatform.py b/youtube_dl/extractor/eagleplatform.py
index e529b9b96..7bbf617d4 100644
--- a/youtube_dl/extractor/eagleplatform.py
+++ b/youtube_dl/extractor/eagleplatform.py
@@ -87,7 +87,7 @@ class EaglePlatformIE(InfoExtractor):
         m3u8_url = self._get_video_url(secure_m3u8, video_id, 'Downloading m3u8 JSON')
         formats = self._extract_m3u8_formats(
             m3u8_url, video_id,
-            'mp4', entry_protocol='m3u8_native')
+            'mp4', entry_protocol='m3u8_native', m3u8_id='hls')
 
         mp4_url = self._get_video_url(
             # Secure mp4 URL is constructed according to Player.prototype.mp4 from

From a6e0afa2bbc93d145b31911b8ce40c502994e2a1 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 18 Oct 2015 19:23:40 +0200
Subject: [PATCH 2858/6219] release 2015.10.18

---
 docs/supportedsites.md | 3 ++-
 youtube_dl/version.py  | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 47f7da86d..cfa665d88 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -588,7 +588,8 @@
  - **twitch:stream**
  - **twitch:video**
  - **twitch:vod**
- - **TwitterCard**
+ - **twitter**
+ - **twitter:card**
  - **Ubu**
  - **udemy**
  - **udemy:course**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 31d2a9dc0..660b0050b 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.10.16'
+__version__ = '2015.10.18'

From 264b23e1a42378d52f8774a07c1d906cd1cff96c Mon Sep 17 00:00:00 2001
From: kennell <kevin@fileperms.org>
Date: Sun, 18 Oct 2015 19:56:22 +0200
Subject: [PATCH 2859/6219] adds thumbnail support for ZDF Mediathek extractor

---
 youtube_dl/extractor/zdf.py | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/youtube_dl/extractor/zdf.py b/youtube_dl/extractor/zdf.py
index 98f15177b..f376025e1 100644
--- a/youtube_dl/extractor/zdf.py
+++ b/youtube_dl/extractor/zdf.py
@@ -70,6 +70,23 @@ def extract_from_xml_url(ie, video_id, xml_url):
             '_available': is_available,
         }
 
+    def xml_to_thumbnails(fnode):
+        thumbnails = list()
+        for node in fnode:
+            width_x_height = node.attrib['key']
+            thumbnail = {
+                'url': node.text,
+                'width': int(width_x_height.split('x')[0]),
+                'height': int(width_x_height.split('x')[1])
+            }
+            thumbnails.append(thumbnail)
+        return thumbnails
+
+
+    thumbnail_nodes = doc.findall('.//teaserimages/teaserimage')
+    thumbnails = xml_to_thumbnails(thumbnail_nodes)
+    thumbnail = thumbnails[-1]['url']
+
     format_nodes = doc.findall('.//formitaeten/formitaet')
     formats = list(filter(
         lambda f: f['_available'],
@@ -81,6 +98,8 @@ def extract_from_xml_url(ie, video_id, xml_url):
         'title': title,
         'description': description,
         'duration': duration,
+        'thumbnail': thumbnail,
+        'thumbnails': thumbnails,
         'uploader': uploader,
         'uploader_id': uploader_id,
         'upload_date': upload_date,

From d762f86e940ad656e8f7e7b93636292e4cf36de5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 19 Oct 2015 00:11:16 +0600
Subject: [PATCH 2860/6219] [ok] Extend _VALID_URL

---
 youtube_dl/extractor/odnoklassniki.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/odnoklassniki.py b/youtube_dl/extractor/odnoklassniki.py
index ccc88cfb1..184c7a323 100644
--- a/youtube_dl/extractor/odnoklassniki.py
+++ b/youtube_dl/extractor/odnoklassniki.py
@@ -13,7 +13,7 @@ from ..utils import (
 
 
 class OdnoklassnikiIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?(?:odnoklassniki|ok)\.ru/(?:video|web-api/video/moviePlayer)/(?P<id>[\d-]+)'
+    _VALID_URL = r'https?://(?:www\.)?(?:odnoklassniki|ok)\.ru/(?:video(?:embed)?|web-api/video/moviePlayer)/(?P<id>[\d-]+)'
     _TESTS = [{
         # metadata in JSON
         'url': 'http://ok.ru/video/20079905452',
@@ -66,6 +66,9 @@ class OdnoklassnikiIE(InfoExtractor):
     }, {
         'url': 'http://www.ok.ru/video/20648036891',
         'only_matching': True,
+    }, {
+        'url': 'http://www.ok.ru/videoembed/20648036891',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 8cc83d301dd0e8029aff804e362860d36e3d7e7a Mon Sep 17 00:00:00 2001
From: kennell <kevin@fileperms.org>
Date: Sun, 18 Oct 2015 20:47:42 +0200
Subject: [PATCH 2861/6219] use int_or_none, check if attrib exists, remove
 thumbnail

---
 youtube_dl/extractor/zdf.py | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/zdf.py b/youtube_dl/extractor/zdf.py
index f376025e1..d41c4e712 100644
--- a/youtube_dl/extractor/zdf.py
+++ b/youtube_dl/extractor/zdf.py
@@ -73,19 +73,17 @@ def extract_from_xml_url(ie, video_id, xml_url):
     def xml_to_thumbnails(fnode):
         thumbnails = list()
         for node in fnode:
-            width_x_height = node.attrib['key']
-            thumbnail = {
-                'url': node.text,
-                'width': int(width_x_height.split('x')[0]),
-                'height': int(width_x_height.split('x')[1])
-            }
+            thumbnail = {'url': node.text}
+            if 'key' in node.attrib:
+                width_x_height = node.attrib['key']
+                thumbnail['width'] = int_or_none(width_x_height.split('x')[0])
+                thumbnail['height'] = int_or_none(width_x_height.split('x')[1])
             thumbnails.append(thumbnail)
         return thumbnails
 
 
     thumbnail_nodes = doc.findall('.//teaserimages/teaserimage')
     thumbnails = xml_to_thumbnails(thumbnail_nodes)
-    thumbnail = thumbnails[-1]['url']
 
     format_nodes = doc.findall('.//formitaeten/formitaet')
     formats = list(filter(
@@ -98,7 +96,6 @@ def extract_from_xml_url(ie, video_id, xml_url):
         'title': title,
         'description': description,
         'duration': duration,
-        'thumbnail': thumbnail,
         'thumbnails': thumbnails,
         'uploader': uploader,
         'uploader_id': uploader_id,

From b243340f0ce311443a15a2dfd4356a9504e18c04 Mon Sep 17 00:00:00 2001
From: kennell <kevin@fileperms.org>
Date: Sun, 18 Oct 2015 21:07:52 +0200
Subject: [PATCH 2862/6219] check if key attrib matches resolution pattern

---
 youtube_dl/extractor/zdf.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/zdf.py b/youtube_dl/extractor/zdf.py
index d41c4e712..ed385450c 100644
--- a/youtube_dl/extractor/zdf.py
+++ b/youtube_dl/extractor/zdf.py
@@ -75,9 +75,9 @@ def extract_from_xml_url(ie, video_id, xml_url):
         for node in fnode:
             thumbnail = {'url': node.text}
             if 'key' in node.attrib:
-                width_x_height = node.attrib['key']
-                thumbnail['width'] = int_or_none(width_x_height.split('x')[0])
-                thumbnail['height'] = int_or_none(width_x_height.split('x')[1])
+                if re.match("^[0-9]+x[0-9]+$", node.attrib['key']):
+                    thumbnail['width'] = int_or_none(node.attrib['key'].split('x')[0])
+                    thumbnail['height'] = int_or_none(node.attrib['key'].split('x')[1])
             thumbnails.append(thumbnail)
         return thumbnails
 

From 2038ad6ee71c842420b83cb6c5ce3c6898e8e380 Mon Sep 17 00:00:00 2001
From: "Sergey M." <dstftw@gmail.com>
Date: Mon, 19 Oct 2015 01:12:41 +0600
Subject: [PATCH 2863/6219] [README.md] Add uploader extraction sample in
 example extractor

---
 README.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index cf4aebf3d..a6ec9619c 100644
--- a/README.md
+++ b/README.md
@@ -710,12 +710,13 @@ If you want to add support for a new site, you can follow this quick list (assum
             webpage = self._download_webpage(url, video_id)
 
             # TODO more code goes here, for example ...
-            title = self._html_search_regex(r'<h1>(.*?)</h1>', webpage, 'title')
+            title = self._html_search_regex(r'<h1>(.+?)</h1>', webpage, 'title')
 
             return {
                 'id': video_id,
                 'title': title,
                 'description': self._og_search_description(webpage),
+                'uploader': self._search_regex(r'<div[^>]+id="uploader"[^>]*>([^<]+)<', webpage, 'uploader', fatal=False),
                 # TODO more properties (see youtube_dl/extractor/common.py)
             }
     ```

From b7cedb16043c60d4032b206a83539acbd39f994f Mon Sep 17 00:00:00 2001
From: kennell <kevin@fileperms.org>
Date: Sun, 18 Oct 2015 21:25:26 +0200
Subject: [PATCH 2864/6219] simplify thumbnail dict building

---
 youtube_dl/extractor/zdf.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/zdf.py b/youtube_dl/extractor/zdf.py
index ed385450c..c2b196504 100644
--- a/youtube_dl/extractor/zdf.py
+++ b/youtube_dl/extractor/zdf.py
@@ -75,9 +75,10 @@ def extract_from_xml_url(ie, video_id, xml_url):
         for node in fnode:
             thumbnail = {'url': node.text}
             if 'key' in node.attrib:
-                if re.match("^[0-9]+x[0-9]+$", node.attrib['key']):
-                    thumbnail['width'] = int_or_none(node.attrib['key'].split('x')[0])
-                    thumbnail['height'] = int_or_none(node.attrib['key'].split('x')[1])
+                m = re.match('^([0-9]+)x([0-9]+)$', node.attrib['key'])
+                if m:
+                    thumbnail['width'] = int(m.group(1))
+                    thumbnail['height'] = int(m.group(2))
             thumbnails.append(thumbnail)
         return thumbnails
 

From 7b091c370c0f187545df8b1b1cc990fcf95df108 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 19 Oct 2015 01:48:05 +0600
Subject: [PATCH 2865/6219] [zdf] Modernize and PEP 8

---
 youtube_dl/extractor/zdf.py | 43 +++++++++++++++++++------------------
 1 file changed, 22 insertions(+), 21 deletions(-)

diff --git a/youtube_dl/extractor/zdf.py b/youtube_dl/extractor/zdf.py
index c2b196504..a795f56b3 100644
--- a/youtube_dl/extractor/zdf.py
+++ b/youtube_dl/extractor/zdf.py
@@ -9,6 +9,7 @@ from ..utils import (
     int_or_none,
     unified_strdate,
     OnDemandPagedList,
+    xpath_text,
 )
 
 
@@ -19,13 +20,11 @@ def extract_from_xml_url(ie, video_id, xml_url):
         errnote='Failed to download video info')
 
     title = doc.find('.//information/title').text
-    description = doc.find('.//information/detail').text
-    duration = int(doc.find('.//details/lengthSec').text)
-    uploader_node = doc.find('.//details/originChannelTitle')
-    uploader = None if uploader_node is None else uploader_node.text
-    uploader_id_node = doc.find('.//details/originChannelId')
-    uploader_id = None if uploader_id_node is None else uploader_id_node.text
-    upload_date = unified_strdate(doc.find('.//details/airtime').text)
+    description = xpath_text(doc, './/information/detail', 'description')
+    duration = int_or_none(xpath_text(doc, './/details/lengthSec', 'duration'))
+    uploader = xpath_text(doc, './/details/originChannelTitle', 'uploader')
+    uploader_id = xpath_text(doc, './/details/originChannelId', 'uploader id')
+    upload_date = unified_strdate(xpath_text(doc, './/details/airtime', 'upload date'))
 
     def xml_to_format(fnode):
         video_url = fnode.find('url').text
@@ -40,15 +39,14 @@ def extract_from_xml_url(ie, video_id, xml_url):
         ext = format_m.group('container')
         proto = format_m.group('proto').lower()
 
-        quality = fnode.find('./quality').text
-        abr = int(fnode.find('./audioBitrate').text) // 1000
-        vbr_node = fnode.find('./videoBitrate')
-        vbr = None if vbr_node is None else int(vbr_node.text) // 1000
+        quality = xpath_text(fnode, './quality', 'quality')
+        abr = int_or_none(xpath_text(fnode, './audioBitrate', 'abr'), 1000)
+        vbr = int_or_none(xpath_text(fnode, './videoBitrate', 'vbr'), 1000)
 
-        width_node = fnode.find('./width')
-        width = None if width_node is None else int_or_none(width_node.text)
-        height_node = fnode.find('./height')
-        height = None if height_node is None else int_or_none(height_node.text)
+        width = int_or_none(xpath_text(fnode, './width', 'width'))
+        height = int_or_none(xpath_text(fnode, './height', 'height'))
+
+        filesize = int_or_none(xpath_text(fnode, './filesize', 'filesize'))
 
         format_note = ''
         if not format_note:
@@ -64,16 +62,21 @@ def extract_from_xml_url(ie, video_id, xml_url):
             'vbr': vbr,
             'width': width,
             'height': height,
-            'filesize': int_or_none(fnode.find('./filesize').text),
+            'filesize': filesize,
             'format_note': format_note,
             'protocol': proto,
             '_available': is_available,
         }
 
     def xml_to_thumbnails(fnode):
-        thumbnails = list()
+        thumbnails = []
         for node in fnode:
-            thumbnail = {'url': node.text}
+            thumbnail_url = node.text
+            if not thumbnail_url:
+                continue
+            thumbnail = {
+                'url': thumbnail_url,
+            }
             if 'key' in node.attrib:
                 m = re.match('^([0-9]+)x([0-9]+)$', node.attrib['key'])
                 if m:
@@ -82,9 +85,7 @@ def extract_from_xml_url(ie, video_id, xml_url):
             thumbnails.append(thumbnail)
         return thumbnails
 
-
-    thumbnail_nodes = doc.findall('.//teaserimages/teaserimage')
-    thumbnails = xml_to_thumbnails(thumbnail_nodes)
+    thumbnails = xml_to_thumbnails(doc.findall('.//teaserimages/teaserimage'))
 
     format_nodes = doc.findall('.//formitaeten/formitaet')
     formats = list(filter(

From 0be30bafa42dbfa99644a9eb7fefa5cebb70f121 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Lalinsk=C3=BD?= <lukas@oxygene.sk>
Date: Mon, 19 Oct 2015 20:53:27 +0200
Subject: [PATCH 2866/6219] [vidme] Stream URL fallback, better error message
 for suspended videos

---
 youtube_dl/extractor/vidme.py | 37 +++++++++++++++++++++++++++++++++--
 1 file changed, 35 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vidme.py b/youtube_dl/extractor/vidme.py
index 382517a4a..393970a12 100644
--- a/youtube_dl/extractor/vidme.py
+++ b/youtube_dl/extractor/vidme.py
@@ -97,6 +97,31 @@ class VidmeIE(InfoExtractor):
         # nsfw, user-disabled
         'url': 'https://vid.me/dzGJ',
         'only_matching': True,
+    }, {
+        # suspended
+        'url': 'https://vid.me/Ox3G',
+        'only_matching': True,
+    }, {
+        # no formats in the API response
+        'url': 'https://vid.me/e5g',
+        'info_dict': {
+            'id': 'e5g',
+            'ext': 'mp4',
+            'title': 'e5g',
+            'thumbnail': 're:^https?://.*\.jpg',
+            'timestamp': 1401480195,
+            'upload_date': '20140530',
+            'uploader': None,
+            'uploader_id': None,
+            'age_limit': 0,
+            'duration': 483,
+            'view_count': int,
+            'like_count': int,
+            'comment_count': int,
+        },
+        'params': {
+            'skip_download': True,
+        },
     }]
 
     def _real_extract(self, url):
@@ -118,7 +143,7 @@ class VidmeIE(InfoExtractor):
 
         video = response['video']
 
-        if video.get('state') == 'user-disabled':
+        if video.get('state') in ('user-disabled', 'suspended'):
             raise ExtractorError(
                 'Vidme said: This video has been suspended either due to a copyright claim, '
                 'or for violating the terms of use.',
@@ -131,6 +156,14 @@ class VidmeIE(InfoExtractor):
             'height': int_or_none(f.get('height')),
             'preference': 0 if f.get('type', '').endswith('clip') else 1,
         } for f in video.get('formats', []) if f.get('uri')]
+
+        if not formats and video.get('complete_url'):
+            formats.append({
+                'url': video.get('complete_url'),
+                'width': int_or_none(video.get('width')),
+                'height': int_or_none(video.get('height')),
+            })
+
         self._sort_formats(formats)
 
         title = video['title']
@@ -147,7 +180,7 @@ class VidmeIE(InfoExtractor):
 
         return {
             'id': video_id,
-            'title': title,
+            'title': title or video_id,
             'description': description,
             'thumbnail': thumbnail,
             'uploader': uploader,

From 4bf56141950f3c24000381403417d20095f04460 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 20 Oct 2015 07:43:39 +0100
Subject: [PATCH 2867/6219] [cspan] move get_text_attr to CSpanIE

---
 youtube_dl/extractor/cspan.py | 21 ++++++++++-----------
 1 file changed, 10 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/cspan.py b/youtube_dl/extractor/cspan.py
index c74b35fd9..388460a32 100644
--- a/youtube_dl/extractor/cspan.py
+++ b/youtube_dl/extractor/cspan.py
@@ -14,10 +14,6 @@ from ..utils import (
 from .senateisvp import SenateISVPIE
 
 
-def get_text_attr(d, attr):
-    return d.get(attr, {}).get('#text')
-
-
 class CSpanIE(InfoExtractor):
     _VALID_URL = r'http://(?:www\.)?c-span\.org/video/\?(?P<id>[0-9a-f]+)'
     IE_DESC = 'C-SPAN'
@@ -60,6 +56,9 @@ class CSpanIE(InfoExtractor):
         }
     }]
 
+    def get_text_attr(self, d, attr):
+        return d.get(attr, {}).get('#text')
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
@@ -79,7 +78,7 @@ class CSpanIE(InfoExtractor):
             'http://www.c-span.org/assets/player/ajax-player.php?os=android&html5=%s&id=%s' % (video_type, video_id),
             video_id)['video']
         if data['@status'] != 'Success':
-            raise ExtractorError('%s said: %s' % (self.IE_NAME, get_text_attr(data, 'error')), expected=True)
+            raise ExtractorError('%s said: %s' % (self.IE_NAME, self.get_text_attr(data, 'error')), expected=True)
 
         doc = self._download_xml(
             'http://www.c-span.org/common/services/flashXml.php?%sid=%s' % (video_type, video_id),
@@ -91,17 +90,17 @@ class CSpanIE(InfoExtractor):
         thumbnail = find_xpath_attr(doc, './/string', 'name', 'poster').text
 
         files = data['files']
-        capfile = get_text_attr(data, 'capfile')
+        capfile = self.get_text_attr(data, 'capfile')
 
         entries = []
         for partnum, f in enumerate(files):
             formats = []
             for quality in f['qualities']:
                 formats.append({
-                    'format_id': '%s-%sp' % (get_text_attr(quality, 'bitrate'), get_text_attr(quality, 'height')),
-                    'url': unescapeHTML(get_text_attr(quality, 'file')),
-                    'height': int_or_none(get_text_attr(quality, 'height')),
-                    'tbr': int_or_none(get_text_attr(quality, 'bitrate')),
+                    'format_id': '%s-%sp' % (self.get_text_attr(quality, 'bitrate'), self.get_text_attr(quality, 'height')),
+                    'url': unescapeHTML(self.get_text_attr(quality, 'file')),
+                    'height': int_or_none(self.get_text_attr(quality, 'height')),
+                    'tbr': int_or_none(self.get_text_attr(quality, 'bitrate')),
                 })
             self._sort_formats(formats)
             entries.append({
@@ -112,7 +111,7 @@ class CSpanIE(InfoExtractor):
                 'formats': formats,
                 'description': description,
                 'thumbnail': thumbnail,
-                'duration': int_or_none(get_text_attr(f, 'length')),
+                'duration': int_or_none(self.get_text_attr(f, 'length')),
                 'subtitles': {
                     'en': [{
                         'url': capfile,

From b6aa99aff8278142fed94e37e500f1cfb62defd1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Lalinsk=C3=BD?= <lukas@oxygene.sk>
Date: Tue, 20 Oct 2015 10:30:31 +0200
Subject: [PATCH 2868/6219] [vimeo] Fix error parsing

---
 youtube_dl/extractor/vimeo.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 0f84656c0..bdec79341 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -273,13 +273,13 @@ class VimeoIE(VimeoBaseInfoExtractor):
         self.report_extraction(video_id)
 
         vimeo_config = self._search_regex(
-            r'vimeo\.config\s*=\s*({.+?});', webpage,
+            r'vimeo\.config\s*=\s*(?:({.+?})|_extend\([^,]+,\s+({.+?})\));', webpage,
             'vimeo config', default=None)
         if vimeo_config:
             seed_status = self._parse_json(vimeo_config, video_id).get('seed_status', {})
             if seed_status.get('state') == 'failed':
                 raise ExtractorError(
-                    '%s returned error: %s' % (self.IE_NAME, seed_status['title']),
+                    '%s said: %s' % (self.IE_NAME, seed_status['title']),
                     expected=True)
 
         # Extract the config JSON

From 4a8963770e37568c484841338cbb6761cf3cb5c3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 20 Oct 2015 20:17:54 +0600
Subject: [PATCH 2869/6219] [vidme] Use original vid.me title template for
 untitled videos

---
 youtube_dl/extractor/vidme.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vidme.py b/youtube_dl/extractor/vidme.py
index 393970a12..296e00423 100644
--- a/youtube_dl/extractor/vidme.py
+++ b/youtube_dl/extractor/vidme.py
@@ -107,7 +107,7 @@ class VidmeIE(InfoExtractor):
         'info_dict': {
             'id': 'e5g',
             'ext': 'mp4',
-            'title': 'e5g',
+            'title': 'Video upload (e5g)',
             'thumbnail': 're:^https?://.*\.jpg',
             'timestamp': 1401480195,
             'upload_date': '20140530',
@@ -180,7 +180,7 @@ class VidmeIE(InfoExtractor):
 
         return {
             'id': video_id,
-            'title': title or video_id,
+            'title': title or 'Video upload (%s)' % video_id,
             'description': description,
             'thumbnail': thumbnail,
             'uploader': uploader,

From d65889bbc0a6b4a1eafe6a8c0e0e26170dc75586 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 20 Oct 2015 20:18:23 +0600
Subject: [PATCH 2870/6219] [vidme] Update test

---
 youtube_dl/extractor/vidme.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vidme.py b/youtube_dl/extractor/vidme.py
index 296e00423..eb5cde761 100644
--- a/youtube_dl/extractor/vidme.py
+++ b/youtube_dl/extractor/vidme.py
@@ -14,7 +14,7 @@ class VidmeIE(InfoExtractor):
     _VALID_URL = r'https?://vid\.me/(?:e/)?(?P<id>[\da-zA-Z]+)'
     _TESTS = [{
         'url': 'https://vid.me/QNB',
-        'md5': 'c62f1156138dc3323902188c5b5a8bd6',
+        'md5': 'f42d05e7149aeaec5c037b17e5d3dc82',
         'info_dict': {
             'id': 'QNB',
             'ext': 'mp4',

From 8bea039b8329074af9a95fe51e7622c8074f6218 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Lalinsk=C3=BD?= <lukas@oxygene.sk>
Date: Tue, 20 Oct 2015 16:38:44 +0200
Subject: [PATCH 2871/6219] [vimeo] New test, fixed one older test

---
 youtube_dl/extractor/vimeo.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index bdec79341..2437ae1eb 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -133,7 +133,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
                 'uploader_id': 'user18948128',
                 'uploader': 'Jaime Marquínez Ferrándiz',
                 'duration': 10,
-                'description': 'This is "youtube-dl password protected test video" by Jaime Marquínez Ferrándiz on Vimeo, the home for high quality videos and the people who love them.',
+                'description': 'This is "youtube-dl password protected test video" by Jaime Marquínez Ferrándiz on Vimeo, the home for high quality videos and the people\u2026',
             },
             'params': {
                 'videopassword': 'youtube-dl',
@@ -181,6 +181,11 @@ class VimeoIE(VimeoBaseInfoExtractor):
                 'uploader_id': 'user28849593',
             },
         },
+        {
+            'url': 'https://vimeo.com/109815029',
+            'note': 'Video not completely processed, "failed" seed status',
+            'only_matching': True,
+        },
     ]
 
     @staticmethod

From d01949dc89feb2441f251e42e8a6bfa4711b9715 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 20 Oct 2015 23:09:51 +0600
Subject: [PATCH 2872/6219] [utils:js_to_json] Fix bad escape in double quoted
 strings

---
 test/test_utils.py  | 3 +++
 youtube_dl/utils.py | 4 ++--
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index a5f164c49..918a7a9ef 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -495,6 +495,9 @@ class TestUtil(unittest.TestCase):
             "playlist":[{"controls":{"all":null}}]
         }''')
 
+        inp = '''"The CW\\'s \\'Crazy Ex-Girlfriend\\'"'''
+        self.assertEqual(js_to_json(inp), '''"The CW's 'Crazy Ex-Girlfriend'"''')
+
         inp = '"SAND Number: SAND 2013-7800P\\nPresenter: Tom Russo\\nHabanero Software Training - Xyce Software\\nXyce, Sandia\\u0027s"'
         json_code = js_to_json(inp)
         self.assertEqual(json.loads(json_code), json.loads(inp))
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index db5b3698e..a61e47646 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1701,8 +1701,8 @@ def js_to_json(code):
         if v in ('true', 'false', 'null'):
             return v
         if v.startswith('"'):
-            return v
-        if v.startswith("'"):
+            v = re.sub(r"\\'", "'", v[1:-1])
+        elif v.startswith("'"):
             v = v[1:-1]
             v = re.sub(r"\\\\|\\'|\"", lambda m: {
                 '\\\\': '\\\\',

From 4211c83aa4dec0cf9874a6a485665360570e2a89 Mon Sep 17 00:00:00 2001
From: mjdubell <localhost@localhost>
Date: Mon, 19 Oct 2015 03:36:07 +0200
Subject: [PATCH 2873/6219] [stitcher] Add extractor

Stitcher review updates

Removed re import

Stitcher review updates
---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/stitcher.py | 37 ++++++++++++++++++++++++++++++++
 2 files changed, 38 insertions(+)
 create mode 100644 youtube_dl/extractor/stitcher.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index bd6eb6ae0..eac5e7d5e 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -586,6 +586,7 @@ from .spankwire import SpankwireIE
 from .spiegel import SpiegelIE, SpiegelArticleIE
 from .spiegeltv import SpiegeltvIE
 from .spike import SpikeIE
+from .stitcher import StitcherIE
 from .sport5 import Sport5IE
 from .sportbox import (
     SportBoxIE,
diff --git a/youtube_dl/extractor/stitcher.py b/youtube_dl/extractor/stitcher.py
new file mode 100644
index 000000000..a547debbd
--- /dev/null
+++ b/youtube_dl/extractor/stitcher.py
@@ -0,0 +1,37 @@
+# coding: utf-8
+from __future__ import unicode_literals
+from .common import InfoExtractor
+from ..utils import int_or_none
+
+
+class StitcherIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?stitcher\.com/podcast/[\/a-z\-]+(?P<id>\d+)'
+    _TEST = {
+        'url': 'http://www.stitcher.com/podcast/the-talking-machines/e/40789481?autoplay=true',
+        'md5': '391dd4e021e6edeb7b8e68fbf2e9e940',
+        'info_dict': {
+            'id': '40789481',
+            'ext': 'mp3',
+            'title': 'Machine Learning Mastery and Cancer Clusters from Talking Machines',
+        }
+    }
+
+    def _real_extract(self, url):
+        audio_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, audio_id)
+
+        title = self._og_search_title(webpage)
+        url = self._search_regex(r'episodeURL: "(.+?)"', webpage, 'url')
+        episode_image = self._search_regex(r'episodeImage: "(.+?)"', webpage, 'episode_image', fatal=False)
+        duration = int_or_none(self._search_regex(r'duration: (\d+?),', webpage, 'duration', fatal=False))
+
+        return {
+            'id': audio_id,
+            'url': url,
+            'title': title,
+            'duration': duration,
+            'thumbnail': episode_image,
+            'ext': 'mp3',
+            'vcodec': 'none',
+        }

From 7308b8cb3df5a2df0a86e8050c83b951004a0aca Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 20 Oct 2015 23:12:13 +0600
Subject: [PATCH 2874/6219] [stitcher] Improve (Closes #7162, closes #7228)

---
 youtube_dl/extractor/stitcher.py | 78 +++++++++++++++++++++++++-------
 1 file changed, 61 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/stitcher.py b/youtube_dl/extractor/stitcher.py
index a547debbd..971a1c466 100644
--- a/youtube_dl/extractor/stitcher.py
+++ b/youtube_dl/extractor/stitcher.py
@@ -1,37 +1,81 @@
-# coding: utf-8
 from __future__ import unicode_literals
+
+import re
+
 from .common import InfoExtractor
-from ..utils import int_or_none
+from ..utils import (
+    determine_ext,
+    int_or_none,
+    js_to_json,
+    unescapeHTML,
+)
 
 
 class StitcherIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?stitcher\.com/podcast/[\/a-z\-]+(?P<id>\d+)'
-    _TEST = {
+    _VALID_URL = r'https?://(?:www\.)?stitcher\.com/podcast/(?:[^/]+/)+e/(?:(?P<display_id>[^/#?&]+?)-)?(?P<id>\d+)(?:[/#?&]|$)'
+    _TESTS = [{
         'url': 'http://www.stitcher.com/podcast/the-talking-machines/e/40789481?autoplay=true',
         'md5': '391dd4e021e6edeb7b8e68fbf2e9e940',
         'info_dict': {
             'id': '40789481',
             'ext': 'mp3',
-            'title': 'Machine Learning Mastery and Cancer Clusters from Talking Machines',
-        }
-    }
+            'title': 'Machine Learning Mastery and Cancer Clusters',
+            'description': 'md5:55163197a44e915a14a1ac3a1de0f2d3',
+            'duration': 1604,
+            'thumbnail': 're:^https?://.*\.jpg',
+        },
+    }, {
+        'url': 'http://www.stitcher.com/podcast/panoply/vulture-tv/e/the-rare-hourlong-comedy-plus-40846275?autoplay=true',
+        'info_dict': {
+            'id': '40846275',
+            'display_id': 'the-rare-hourlong-comedy-plus',
+            'ext': 'mp3',
+            'title': "The CW's 'Crazy Ex-Girlfriend'",
+            'description': 'md5:04f1e2f98eb3f5cbb094cea0f9e19b17',
+            'duration': 2235,
+            'thumbnail': 're:^https?://.*\.jpg',
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }, {
+        # escaped title
+        'url': 'http://www.stitcher.com/podcast/marketplace-on-stitcher/e/40910226?autoplay=true',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.stitcher.com/podcast/panoply/getting-in/e/episode-2a-how-many-extracurriculars-should-i-have-40876278?autoplay=true',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
-        audio_id = self._match_id(url)
+        mobj = re.match(self._VALID_URL, url)
+        audio_id = mobj.group('id')
+        display_id = mobj.group('display_id') or audio_id
 
-        webpage = self._download_webpage(url, audio_id)
+        webpage = self._download_webpage(url, display_id)
 
-        title = self._og_search_title(webpage)
-        url = self._search_regex(r'episodeURL: "(.+?)"', webpage, 'url')
-        episode_image = self._search_regex(r'episodeImage: "(.+?)"', webpage, 'episode_image', fatal=False)
-        duration = int_or_none(self._search_regex(r'duration: (\d+?),', webpage, 'duration', fatal=False))
+        episode = self._parse_json(
+            js_to_json(self._search_regex(
+                r'(?s)var\s+stitcher\s*=\s*({.+?});\n', webpage, 'episode config')),
+            display_id)['config']['episode']
+
+        title = unescapeHTML(episode['title'])
+        formats = [{
+            'url': episode[episode_key],
+            'ext': determine_ext(episode[episode_key]) or 'mp3',
+            'vcodec': 'none',
+        } for episode_key in ('origEpisodeURL', 'episodeURL') if episode.get(episode_key)]
+        description = self._search_regex(
+            r'Episode Info:\s*</span>([^<]+)<', webpage, 'description', fatal=False)
+        duration = int_or_none(episode.get('duration'))
+        thumbnail = episode.get('episodeImage')
 
         return {
             'id': audio_id,
-            'url': url,
+            'display_id': display_id,
             'title': title,
+            'description': description,
             'duration': duration,
-            'thumbnail': episode_image,
-            'ext': 'mp3',
-            'vcodec': 'none',
+            'thumbnail': thumbnail,
+            'formats': formats,
         }

From 497f5fd93fe1efd0df8dc58d518c328ed1409457 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 21 Oct 2015 08:24:05 +0100
Subject: [PATCH 2875/6219] [bilibili] extract multiple backup_urls

---
 youtube_dl/extractor/bilibili.py | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/bilibili.py b/youtube_dl/extractor/bilibili.py
index e00efb6eb..935fcc55c 100644
--- a/youtube_dl/extractor/bilibili.py
+++ b/youtube_dl/extractor/bilibili.py
@@ -69,21 +69,22 @@ class BiliBiliIE(InfoExtractor):
             pass
 
         doc = ET.fromstring(page)
-        durls = doc.findall('./durl')
 
         entries = []
 
-        for durl in durls:
-            formats = []
-            backup_url = durl.find('./backup_url')
-            if backup_url is not None:
-                formats.append({'url': backup_url.find('./url').text})
+        for durl in doc.findall('./durl'):
             size = durl.find('./filesize|./size')
-            formats.append({
+            formats = [{
                 'url': durl.find('./url').text,
                 'filesize': int_or_none(size.text) if size else None,
                 'ext': 'flv',
-            })
+            }]
+            backup_urls = durl.find('./backup_url')
+            if backup_urls is not None:
+                for backup_url in backup_urls.findall('./url'):
+                    formats.append({'url': backup_url.text})
+            formats.reverse()
+
             entries.append({
                 'id': '%s_part%s' % (cid, durl.find('./order').text),
                 'title': title,

From cc449417c4b3835c31f89e47a5e08e0f0c42ac5a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 21 Oct 2015 20:35:22 +0600
Subject: [PATCH 2876/6219] [vine] Use _search_regex for JSON data (Closes
 #7254, closes #7255)

---
 youtube_dl/extractor/vine.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vine.py b/youtube_dl/extractor/vine.py
index be72f3147..cb2a4b0b5 100644
--- a/youtube_dl/extractor/vine.py
+++ b/youtube_dl/extractor/vine.py
@@ -85,8 +85,8 @@ class VineIE(InfoExtractor):
         webpage = self._download_webpage('https://vine.co/v/' + video_id, video_id)
 
         data = self._parse_json(
-            self._html_search_regex(
-                r'window\.POST_DATA = { %s: ({.+?}) };\s*</script>' % video_id,
+            self._search_regex(
+                r'window\.POST_DATA\s*=\s*{\s*%s\s*:\s*({.+?})\s*};\s*</script>' % video_id,
                 webpage, 'vine data'),
             video_id)
 

From 44d6dd08b299ccf17eb04901cf09a8d333769783 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 21 Oct 2015 21:35:57 +0600
Subject: [PATCH 2877/6219] [facebook] Fix extraction (Closes #7252)

---
 youtube_dl/extractor/facebook.py | 23 ++++++++++++-----------
 1 file changed, 12 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index 178a7ca4c..f53c51615 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -14,7 +14,6 @@ from ..compat import (
 )
 from ..utils import (
     ExtractorError,
-    int_or_none,
     limit_length,
     urlencode_postdata,
     get_element_by_id,
@@ -142,16 +141,20 @@ class FacebookIE(InfoExtractor):
         data = dict(json.loads(m.group(1)))
         params_raw = compat_urllib_parse_unquote(data['params'])
         params = json.loads(params_raw)
-        video_data = params['video_data'][0]
 
         formats = []
-        for quality in ['sd', 'hd']:
-            src = video_data.get('%s_src' % quality)
-            if src is not None:
-                formats.append({
-                    'format_id': quality,
-                    'url': src,
-                })
+        for format_id, f in params['video_data'].items():
+            if not f or not isinstance(f, list):
+                continue
+            for quality in ('sd', 'hd'):
+                for src_type in ('src', 'src_no_ratelimit'):
+                    src = f[0].get('%s_%s' % (quality, src_type))
+                    if src:
+                        formats.append({
+                            'format_id': '%s_%s_%s' % (format_id, quality, src_type),
+                            'url': src,
+                            'preference': -10 if format_id == 'progressive' else 0,
+                        })
         if not formats:
             raise ExtractorError('Cannot find video formats')
 
@@ -171,7 +174,5 @@ class FacebookIE(InfoExtractor):
             'id': video_id,
             'title': video_title,
             'formats': formats,
-            'duration': int_or_none(video_data.get('video_duration')),
-            'thumbnail': video_data.get('thumbnail_src'),
             'uploader': uploader,
         }

From 8c3533ba976af15ca9fac8acd68547b195dc4e8a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 21 Oct 2015 23:57:23 +0200
Subject: [PATCH 2878/6219] [adultswim] Don't default to the native m3u8
 downloader (closes #7243)

Some of the streams are encrypted, which is not supported .
---
 youtube_dl/extractor/adultswim.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/adultswim.py b/youtube_dl/extractor/adultswim.py
index 130afe791..3ae618e71 100644
--- a/youtube_dl/extractor/adultswim.py
+++ b/youtube_dl/extractor/adultswim.py
@@ -183,7 +183,7 @@ class AdultSwimIE(InfoExtractor):
                 media_url = file_el.text
                 if determine_ext(media_url) == 'm3u8':
                     formats.extend(self._extract_m3u8_formats(
-                        media_url, segment_title, 'mp4', 'm3u8_native', preference=0, m3u8_id='hls'))
+                        media_url, segment_title, 'mp4', preference=0, m3u8_id='hls'))
                 else:
                     formats.append({
                         'format_id': '%s_%s' % (bitrate, ftype),

From 89d5fbf354cf0b49098582a19f76cef67358d375 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 22 Oct 2015 17:47:11 +0800
Subject: [PATCH 2879/6219] [iqiyi] Update key

---
 youtube_dl/extractor/iqiyi.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index 0e53cb154..2df1da3f0 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -205,9 +205,9 @@ class IqiyiIE(InfoExtractor):
 
     def get_enc_key(self, swf_url, video_id):
         # TODO: automatic key extraction
-        # last update at 2015-10-10 for Zombie::bite
-        # '7239670519b6ac209a0bee4ef0446a6b24894b8ac2751506e42116212a0d0272e505'[2:66][1::2]
-        enc_key = '97596c0abee04ab49ba25564161ad225'
+        # last update at 2015-10-22 for Zombie::bite
+        # '7223c67061dbea1259d0ceb44f44b6d62288f4f80c972170de5201d2321060270e05'[2:66][0::2]
+        enc_key = '2c76de15dcb44bd28ff0927d50d31620'
         return enc_key
 
     def _real_extract(self, url):

From 7033bc1a5117068c493931cb736d53e68d50f9a1 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 22 Oct 2015 21:12:29 +0800
Subject: [PATCH 2880/6219] [bbc] Fix test_BBC_9

---
 youtube_dl/extractor/bbc.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 1b3a33e4e..ea67e3f2d 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -625,6 +625,7 @@ class BBCIE(BBCCoUkIE):
             'id': 'p02xycnp',
             'ext': 'mp4',
             'title': 'Transfers: Cristiano Ronaldo to Man Utd, Arsenal to spend?',
+            'description': 'BBC Sport\'s David Ornstein has the latest transfer gossip, including rumours of a Manchester United return for Cristiano Ronaldo.',
             'duration': 140,
         },
         'params': {

From a65402ef42c42477f78469f0a6c4af1583d97a31 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 22 Oct 2015 21:13:03 +0800
Subject: [PATCH 2881/6219] [bbc.co.uk:article] Add new extractor (#7257)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/bbc.py      | 34 ++++++++++++++++++++++++++++++--
 2 files changed, 33 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index eac5e7d5e..6318ac4a2 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -45,6 +45,7 @@ from .bambuser import BambuserIE, BambuserChannelIE
 from .bandcamp import BandcampIE, BandcampAlbumIE
 from .bbc import (
     BBCCoUkIE,
+    BBCCoUkArticleIE,
     BBCIE,
 )
 from .beeg import BeegIE
diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index ea67e3f2d..2cdce1eb9 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -20,7 +20,7 @@ from ..compat import compat_HTTPError
 class BBCCoUkIE(InfoExtractor):
     IE_NAME = 'bbc.co.uk'
     IE_DESC = 'BBC iPlayer'
-    _VALID_URL = r'https?://(?:www\.)?bbc\.co\.uk/(?:(?:(?:programmes|iplayer(?:/[^/]+)?/(?:episode|playlist))/)|music/clips[/#])(?P<id>[\da-z]{8})'
+    _VALID_URL = r'https?://(?:www\.)?bbc\.co\.uk/(?:(?:programmes/(?!articles/)|iplayer(?:/[^/]+)?/(?:episode/|playlist/))|music/clips[/#])(?P<id>[\da-z]{8})'
 
     _MEDIASELECTOR_URLS = [
         # Provides HQ HLS streams with even better quality that pc mediaset but fails
@@ -652,7 +652,7 @@ class BBCIE(BBCCoUkIE):
 
     @classmethod
     def suitable(cls, url):
-        return False if BBCCoUkIE.suitable(url) else super(BBCIE, cls).suitable(url)
+        return False if BBCCoUkIE.suitable(url) or BBCCoUkArticleIE.suitable(url) else super(BBCIE, cls).suitable(url)
 
     def _extract_from_media_meta(self, media_meta, video_id):
         # Direct links to media in media metadata (e.g.
@@ -903,3 +903,33 @@ class BBCIE(BBCCoUkIE):
             })
 
         return self.playlist_result(entries, playlist_id, playlist_title, playlist_description)
+
+
+class BBCCoUkArticleIE(InfoExtractor):
+    _VALID_URL = 'http://www.bbc.co.uk/programmes/articles/(?P<id>[a-zA-Z0-9]+)'
+    IE_NAME = 'bbc.co.uk:article'
+    IE_DESC = 'BBC articles'
+
+    _TEST = {
+        'url': 'http://www.bbc.co.uk/programmes/articles/3jNQLTMrPlYGTBn0WV6M2MS/not-your-typical-role-model-ada-lovelace-the-19th-century-programmer',
+        'info_dict': {
+            'id': '3jNQLTMrPlYGTBn0WV6M2MS',
+            'title': 'Calculating Ada: The Countess of Computing - Not your typical role model: Ada Lovelace the 19th century programmer - BBC Four',
+            'description': 'Hannah Fry reveals some of her surprising discoveries about Ada Lovelace during filming.',
+        },
+        'playlist_count': 4,
+        'add_ie': ['BBCCoUk'],
+    }
+
+    def _real_extract(self, url):
+        playlist_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, playlist_id)
+
+        title = self._og_search_title(webpage)
+        description = self._og_search_description(webpage).strip()
+
+        entries = [self.url_result(programme_url) for programme_url in re.findall(
+            r'<div[^>]+typeof="Clip"[^>]+resource="([^"]+)"', webpage)]
+
+        return self.playlist_result(entries, playlist_id, title, description)

From 769078755318896fa1a6c5c8aba6f76d6aeddf78 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 22 Oct 2015 20:34:11 +0600
Subject: [PATCH 2882/6219] [crunchyroll] Improve subtitle regex (Closes #7262)

---
 youtube_dl/extractor/crunchyroll.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index cecd0c784..f8ce10111 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -245,7 +245,7 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
 
     def _get_subtitles(self, video_id, webpage):
         subtitles = {}
-        for sub_id, sub_name in re.findall(r'\?ssid=([0-9]+)" title="([^"]+)', webpage):
+        for sub_id, sub_name in re.findall(r'\bssid=([0-9]+)"[^>]+?\btitle="([^"]+)', webpage):
             sub_page = self._download_webpage(
                 'http://www.crunchyroll.com/xml/?req=RpcApiSubtitle_GetXml&subtitle_script_id=' + sub_id,
                 video_id, note='Downloading subtitles for ' + sub_name)

From ab03c0b47c142bfb649eacb3c72ce9cb67184535 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 23 Oct 2015 09:33:05 +0200
Subject: [PATCH 2883/6219] release 2015.10.23

---
 CONTRIBUTING.md        | 3 ++-
 docs/supportedsites.md | 2 ++
 youtube_dl/version.py  | 2 +-
 3 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 32c2fd84c..aebded4ce 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -114,12 +114,13 @@ If you want to add support for a new site, you can follow this quick list (assum
             webpage = self._download_webpage(url, video_id)
 
             # TODO more code goes here, for example ...
-            title = self._html_search_regex(r'<h1>(.*?)</h1>', webpage, 'title')
+            title = self._html_search_regex(r'<h1>(.+?)</h1>', webpage, 'title')
 
             return {
                 'id': video_id,
                 'title': title,
                 'description': self._og_search_description(webpage),
+                'uploader': self._search_regex(r'<div[^>]+id="uploader"[^>]*>([^<]+)<', webpage, 'uploader', fatal=False),
                 # TODO more properties (see youtube_dl/extractor/common.py)
             }
     ```
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index cfa665d88..03561b87d 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -53,6 +53,7 @@
  - **Bandcamp:album**
  - **bbc**: BBC
  - **bbc.co.uk**: BBC iPlayer
+ - **bbc.co.uk:article**: BBC articles
  - **BeatportPro**
  - **Beeg**
  - **BehindKink**
@@ -515,6 +516,7 @@
  - **SSA**
  - **stanfordoc**: Stanford Open ClassRoom
  - **Steam**
+ - **Stitcher**
  - **streamcloud.eu**
  - **StreamCZ**
  - **StreetVoice**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 660b0050b..d5c7f338d 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.10.18'
+__version__ = '2015.10.23'

From 65d49afa48086b568364bbcbab29feef71031178 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 23 Oct 2015 14:12:46 +0200
Subject: [PATCH 2884/6219] [test/test_download] Use extract_flat =
 'in_playlist' for playlist items

Some playlist extractors return a 'url' result, which wouldn't be resolved.
---
 test/test_download.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/test_download.py b/test/test_download.py
index 284418834..a3f1c0644 100644
--- a/test/test_download.py
+++ b/test/test_download.py
@@ -102,7 +102,7 @@ def generator(test_case):
 
         params = get_params(test_case.get('params', {}))
         if is_playlist and 'playlist' not in test_case:
-            params.setdefault('extract_flat', True)
+            params.setdefault('extract_flat', 'in_playlist')
             params.setdefault('skip_download', True)
 
         ydl = YoutubeDL(params, auto_init=False)

From 9170ca5b16f3420892ff06bbe5cccf1679eb75e4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 23 Oct 2015 14:16:08 +0200
Subject: [PATCH 2885/6219] [youtube:channel] Fix test

---
 youtube_dl/extractor/youtube.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 08e821362..bae1b1117 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1644,7 +1644,8 @@ class YoutubeChannelIE(YoutubePlaylistBaseInfoExtractor):
         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
         'playlist_mincount': 91,
         'info_dict': {
-            'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
+            'id': 'UUKfVa3S1e4PHvxWcwyMMg8w',
+            'title': 'Uploads from lex will',
         }
     }]
 

From 5c43afd40f8ba101e0cf90b8fcb5713b378a62c9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 23 Oct 2015 14:23:45 +0200
Subject: [PATCH 2886/6219] [youtube:channel] Support age restricted channels
 (fixes #7277)

---
 youtube_dl/extractor/youtube.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index bae1b1117..d7eda7aa7 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1647,6 +1647,15 @@ class YoutubeChannelIE(YoutubePlaylistBaseInfoExtractor):
             'id': 'UUKfVa3S1e4PHvxWcwyMMg8w',
             'title': 'Uploads from lex will',
         }
+    }, {
+        'note': 'Age restricted channel',
+        # from https://www.youtube.com/user/DeusExOfficial
+        'url': 'https://www.youtube.com/channel/UCs0ifCMCm1icqRbqhUINa0w',
+        'playlist_mincount': 64,
+        'info_dict': {
+            'id': 'UUs0ifCMCm1icqRbqhUINa0w',
+            'title': 'Uploads from Deus Ex',
+        },
     }]
 
     def _real_extract(self, url):
@@ -1667,7 +1676,7 @@ class YoutubeChannelIE(YoutubePlaylistBaseInfoExtractor):
                 'channelId', channel_page, 'channel id', default=None)
             if not channel_playlist_id:
                 channel_playlist_id = self._search_regex(
-                    r'data-channel-external-id="([^"]+)"',
+                    r'data-(?:channel-external-|yt)id="([^"]+)"',
                     channel_page, 'channel id', default=None)
         if channel_playlist_id and channel_playlist_id.startswith('UC'):
             playlist_id = 'UU' + channel_playlist_id[2:]

From edeb3e7cb1ab2d82ff7c712a7cc1e338a9dcd8f8 Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Fri, 23 Oct 2015 15:58:24 +0000
Subject: [PATCH 2887/6219] [README.md] Fix typo

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index a6ec9619c..38db97c59 100644
--- a/README.md
+++ b/README.md
@@ -795,7 +795,7 @@ Bugs and suggestions should be reported at: <https://github.com/rg3/youtube-dl/i
 
 **Please include the full output of youtube-dl when run with `-v`**.
 
-The output (including the first lines) contain important debugging information. Issues without the full output are often not reproducible and therefore do not get solved in short order, if ever.
+The output (including the first lines) contains important debugging information. Issues without the full output are often not reproducible and therefore do not get solved in short order, if ever.
 
 Please re-read your issue once again to avoid a couple of common mistakes (you can and should use this as a checklist):
 

From dae69640d086ca1e2683ca81b60f48a0c6c83eac Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 24 Oct 2015 00:10:28 +0200
Subject: [PATCH 2888/6219] Fix py2exe build (#7276)

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 4686260e0..bfe931f5b 100644
--- a/setup.py
+++ b/setup.py
@@ -28,7 +28,7 @@ py2exe_options = {
     "compressed": 1,
     "optimize": 2,
     "dist_dir": '.',
-    "dll_excludes": ['w9xpopen.exe'],
+    "dll_excludes": ['w9xpopen.exe', 'crypt32.dll'],
 }
 
 py2exe_console = [{

From ab9c7214ee6c831a68216caf1cd1f9f3c183e4fd Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 24 Oct 2015 00:10:41 +0200
Subject: [PATCH 2889/6219] release 2015.10.24

---
 CONTRIBUTING.md       | 2 +-
 youtube_dl/version.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index aebded4ce..09ce98ca2 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -1,6 +1,6 @@
 **Please include the full output of youtube-dl when run with `-v`**.
 
-The output (including the first lines) contain important debugging information. Issues without the full output are often not reproducible and therefore do not get solved in short order, if ever.
+The output (including the first lines) contains important debugging information. Issues without the full output are often not reproducible and therefore do not get solved in short order, if ever.
 
 Please re-read your issue once again to avoid a couple of common mistakes (you can and should use this as a checklist):
 
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index d5c7f338d..125e8ccf5 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.10.23'
+__version__ = '2015.10.24'

From c93153852f342ef26005b37649d58fa944c53fe3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 24 Oct 2015 12:10:53 +0200
Subject: [PATCH 2890/6219] [mitele] Don't encode the URL query (closes #7280)

This seems to produce sporadic errors when trying to access the URL, because on python 3.x when you do '%s' % b'somedata' you get "b'somedata'".
---
 youtube_dl/extractor/mitele.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mitele.py b/youtube_dl/extractor/mitele.py
index 54993e2c9..ccb5c1467 100644
--- a/youtube_dl/extractor/mitele.py
+++ b/youtube_dl/extractor/mitele.py
@@ -56,7 +56,7 @@ class MiTeleIE(InfoExtractor):
                 'sta': '0',
             }
             media = self._download_json(
-                '%s/?%s' % (gat, compat_urllib_parse.urlencode(encode_dict(token_data)).encode('utf-8')),
+                '%s/?%s' % (gat, compat_urllib_parse.urlencode(encode_dict(token_data))),
                 display_id, 'Downloading %s JSON' % location['loc'])
             file_ = media.get('file')
             if not file_:

From 6856139705aea86ab1f950c08e605dd47f839be0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 24 Oct 2015 12:13:26 +0200
Subject: [PATCH 2891/6219] [mitele] Fix test checksum

---
 youtube_dl/extractor/mitele.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mitele.py b/youtube_dl/extractor/mitele.py
index ccb5c1467..3142fcde2 100644
--- a/youtube_dl/extractor/mitele.py
+++ b/youtube_dl/extractor/mitele.py
@@ -15,7 +15,7 @@ class MiTeleIE(InfoExtractor):
 
     _TESTS = [{
         'url': 'http://www.mitele.es/programas-tv/diario-de/la-redaccion/programa-144/',
-        'md5': 'ace7635b2a0b286aaa37d3ff192d2a8a',
+        'md5': '757b0b66cbd7e0a97226d7d3156cb3e9',
         'info_dict': {
             'id': '0NF1jJnxS1Wu3pHrmvFyw2',
             'display_id': 'programa-144',

From 0198807ef95338bec69cfdcd67249c007e4d4141 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20D=C3=B6pmann?=
 <cdoepmann@users.noreply.github.com>
Date: Sat, 24 Oct 2015 11:35:41 +0200
Subject: [PATCH 2892/6219] [spiegeltv] Fix Accept-Encoding issue (server
 chokes on gzip)

---
 youtube_dl/extractor/spiegeltv.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/spiegeltv.py b/youtube_dl/extractor/spiegeltv.py
index 27f4033c5..a85305281 100644
--- a/youtube_dl/extractor/spiegeltv.py
+++ b/youtube_dl/extractor/spiegeltv.py
@@ -83,6 +83,10 @@ class SpiegeltvIE(InfoExtractor):
                     preference=1,  # Prefer hls since it allows to workaround georestriction
                     m3u8_id='hls', fatal=False)
                 if m3u8_formats is not False:
+                    for m3u8_format in m3u8_formats:
+                        m3u8_format['http_headers'] = {
+                            'Accept-Encoding': 'deflate', # gzip causes trouble on the server side
+                        }
                     formats.extend(m3u8_formats)
             else:
                 formats.append({

From 50f01302d347738647262f9442bc4f5d06f013c6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 24 Oct 2015 16:24:08 +0600
Subject: [PATCH 2893/6219] [spiegeltv] Do not extract m3u8 formats since it's
 already a format

---
 youtube_dl/extractor/spiegeltv.py | 21 ++++++++++-----------
 1 file changed, 10 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/spiegeltv.py b/youtube_dl/extractor/spiegeltv.py
index a85305281..d976bf33c 100644
--- a/youtube_dl/extractor/spiegeltv.py
+++ b/youtube_dl/extractor/spiegeltv.py
@@ -77,17 +77,16 @@ class SpiegeltvIE(InfoExtractor):
                     'rtmp_live': True,
                 })
             elif determine_ext(endpoint) == 'm3u8':
-                m3u8_formats = self._extract_m3u8_formats(
-                    endpoint.replace('[video]', play_path),
-                    video_id, 'm4v',
-                    preference=1,  # Prefer hls since it allows to workaround georestriction
-                    m3u8_id='hls', fatal=False)
-                if m3u8_formats is not False:
-                    for m3u8_format in m3u8_formats:
-                        m3u8_format['http_headers'] = {
-                            'Accept-Encoding': 'deflate', # gzip causes trouble on the server side
-                        }
-                    formats.extend(m3u8_formats)
+                formats.append({
+                    'url': endpoint.replace('[video]', play_path),
+                    'ext': 'm4v',
+                    'format_id': 'hls',  # Prefer hls since it allows to workaround georestriction
+                    'protocol': 'm3u8',
+                    'preference': 1,
+                    'http_headers': {
+                        'Accept-Encoding': 'deflate', # gzip causes trouble on the server side
+                    },
+                })
             else:
                 formats.append({
                     'url': endpoint,

From 943a1e24b896a64c869bbd302f32fe5bd2afec96 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 24 Oct 2015 16:25:04 +0600
Subject: [PATCH 2894/6219] [extractor/common] Use more generic URLError in
 _is_valid_url

---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 6169fbbeb..720033ddf 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -842,7 +842,7 @@ class InfoExtractor(object):
             self._request_webpage(url, video_id, 'Checking %s URL' % item)
             return True
         except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError):
+            if isinstance(e.cause, compat_urllib_error.URLError):
                 self.to_screen(
                     '%s: %s URL is invalid, skipping' % (video_id, item))
                 return False

From ac21e7196856d7b689f74ed3f9953cbcbe90bee5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 24 Oct 2015 16:25:44 +0600
Subject: [PATCH 2895/6219] [spiegeltv] Check formats

---
 youtube_dl/extractor/spiegeltv.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/spiegeltv.py b/youtube_dl/extractor/spiegeltv.py
index d976bf33c..0981e325a 100644
--- a/youtube_dl/extractor/spiegeltv.py
+++ b/youtube_dl/extractor/spiegeltv.py
@@ -91,6 +91,7 @@ class SpiegeltvIE(InfoExtractor):
                 formats.append({
                     'url': endpoint,
                 })
+        self._check_formats(formats, video_id)
 
         thumbnails = []
         for image in media_json['images']:

From 865d1fbafc671815904b2ba3da76544d66c593c9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 24 Oct 2015 12:39:23 +0200
Subject: [PATCH 2896/6219] [extractor/common] Remove unused import

---
 youtube_dl/extractor/common.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 720033ddf..04b699972 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -16,7 +16,6 @@ from ..compat import (
     compat_cookiejar,
     compat_cookies,
     compat_getpass,
-    compat_HTTPError,
     compat_http_client,
     compat_urllib_error,
     compat_urllib_parse,

From 36d72810374ef2dba0232706a461d6dc4aa292d8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 24 Oct 2015 12:41:41 +0200
Subject: [PATCH 2897/6219] [spiegeltv] Fix style issue

Use two spaces before comment.
---
 youtube_dl/extractor/spiegeltv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/spiegeltv.py b/youtube_dl/extractor/spiegeltv.py
index 0981e325a..034bd47ff 100644
--- a/youtube_dl/extractor/spiegeltv.py
+++ b/youtube_dl/extractor/spiegeltv.py
@@ -84,7 +84,7 @@ class SpiegeltvIE(InfoExtractor):
                     'protocol': 'm3u8',
                     'preference': 1,
                     'http_headers': {
-                        'Accept-Encoding': 'deflate', # gzip causes trouble on the server side
+                        'Accept-Encoding': 'deflate',  # gzip causes trouble on the server side
                     },
                 })
             else:

From 7687b354c59efea076fae762206c00de273fbe04 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 23 Oct 2015 07:09:41 +0100
Subject: [PATCH 2898/6219] [abc] add support for audio extraction

---
 youtube_dl/extractor/abc.py | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/abc.py b/youtube_dl/extractor/abc.py
index f9a389f67..ae80dc529 100644
--- a/youtube_dl/extractor/abc.py
+++ b/youtube_dl/extractor/abc.py
@@ -36,6 +36,15 @@ class ABCIE(InfoExtractor):
             'title': 'Marriage Equality: Warren Entsch introduces same sex marriage bill',
         },
         'add_ie': ['Youtube'],
+    }, {
+        'url': 'http://www.abc.net.au/news/2015-10-23/nab-lifts-interest-rates-following-westpac-and-cba/6880080',
+        'md5': 'b96eee7c9edf4fc5a358a0252881cc1f',
+        'info_dict': {
+            'id': '6880080',
+            'ext': 'mp3',
+            'title': 'NAB lifts interest rates, following Westpac and CBA',
+            'description': 'md5:f13d8edc81e462fce4a0437c7dc04728',
+        },
     }]
 
     def _real_extract(self, url):
@@ -43,7 +52,7 @@ class ABCIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         mobj = re.search(
-            r'inline(?P<type>Video|YouTube)Data\.push\((?P<json_data>[^)]+)\);',
+            r'inline(?P<type>Video|Audio|YouTube)Data\.push\((?P<json_data>[^)]+)\);',
             webpage)
         if mobj is None:
             raise ExtractorError('Unable to extract video urls')
@@ -60,11 +69,13 @@ class ABCIE(InfoExtractor):
 
         formats = [{
             'url': url_info['url'],
+            'vcodec': url_info.get('codec') if mobj.group('type') == 'Video' else 'none',
             'width': int_or_none(url_info.get('width')),
             'height': int_or_none(url_info.get('height')),
             'tbr': int_or_none(url_info.get('bitrate')),
             'filesize': int_or_none(url_info.get('filesize')),
         } for url_info in urls_info]
+
         self._sort_formats(formats)
 
         return {

From d97da29da26c920ae31fde94bba5e3b4e1f5a36a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 24 Oct 2015 12:31:42 +0200
Subject: [PATCH 2899/6219] [abc] Support more URL formats

---
 youtube_dl/extractor/abc.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/abc.py b/youtube_dl/extractor/abc.py
index ae80dc529..c0e5d1abf 100644
--- a/youtube_dl/extractor/abc.py
+++ b/youtube_dl/extractor/abc.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 class ABCIE(InfoExtractor):
     IE_NAME = 'abc.net.au'
-    _VALID_URL = r'http://www\.abc\.net\.au/news/[^/]+/[^/]+/(?P<id>\d+)'
+    _VALID_URL = r'http://www\.abc\.net\.au/news/(?:[^/]+/){1,2}(?P<id>\d+)'
 
     _TESTS = [{
         'url': 'http://www.abc.net.au/news/2014-11-05/australia-to-staff-ebola-treatment-centre-in-sierra-leone/5868334',
@@ -45,6 +45,9 @@ class ABCIE(InfoExtractor):
             'title': 'NAB lifts interest rates, following Westpac and CBA',
             'description': 'md5:f13d8edc81e462fce4a0437c7dc04728',
         },
+    }, {
+        'url': 'http://www.abc.net.au/news/2015-10-19/6866214',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 50b936936dcf53b448557c35a90e4678239aaf81 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 24 Oct 2015 14:22:47 +0200
Subject: [PATCH 2900/6219] [tutv] Fix test

---
 youtube_dl/extractor/tutv.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/tutv.py b/youtube_dl/extractor/tutv.py
index fad720b68..822372ea1 100644
--- a/youtube_dl/extractor/tutv.py
+++ b/youtube_dl/extractor/tutv.py
@@ -10,10 +10,10 @@ class TutvIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?tu\.tv/videos/(?P<id>[^/?]+)'
     _TEST = {
         'url': 'http://tu.tv/videos/robots-futbolistas',
-        'md5': '627c7c124ac2a9b5ab6addb94e0e65f7',
+        'md5': '0cd9e28ad270488911b0d2a72323395d',
         'info_dict': {
             'id': '2973058',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Robots futbolistas',
         },
     }

From 3711304510d3be6a5f9b2b18084aad8687e78001 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 8 Sep 2015 19:35:41 +0100
Subject: [PATCH 2901/6219] [extractor/common] get the redirected m3u8_url in
 _extract_m3u8_formats

---
 youtube_dl/extractor/common.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 04b699972..10c0d5d1f 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -943,13 +943,14 @@ class InfoExtractor(object):
             if re.match(r'^https?://', u)
             else compat_urlparse.urljoin(m3u8_url, u))
 
-        m3u8_doc = self._download_webpage(
+        m3u8_doc, urlh = self._download_webpage_handle(
             m3u8_url, video_id,
             note=note or 'Downloading m3u8 information',
             errnote=errnote or 'Failed to download m3u8 information',
             fatal=fatal)
         if m3u8_doc is False:
             return m3u8_doc
+        m3u8_url = urlh.geturl()
         last_info = None
         last_media = None
         kv_rex = re.compile(

From 324ac0a243c14340f7e4cd909e2e7c62828a2425 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 10 Sep 2015 20:49:43 +0100
Subject: [PATCH 2902/6219] [downloader/f4m] get the redirected f4m_url and
 handle url query string properly

---
 youtube_dl/downloader/f4m.py | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index 174180db5..b8db6bf9b 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -11,6 +11,7 @@ from .fragment import FragmentFD
 from ..compat import (
     compat_urlparse,
     compat_urllib_error,
+    compat_urllib_parse_urlparse,
 )
 from ..utils import (
     encodeFilename,
@@ -285,7 +286,9 @@ class F4mFD(FragmentFD):
         man_url = info_dict['url']
         requested_bitrate = info_dict.get('tbr')
         self.to_screen('[%s] Downloading f4m manifest' % self.FD_NAME)
-        manifest = self.ydl.urlopen(man_url).read()
+        urlh = self.ydl.urlopen(man_url)
+        man_url = urlh.geturl()
+        manifest = urlh.read()
 
         doc = etree.fromstring(manifest)
         formats = [(int(f.attrib.get('bitrate', -1)), f)
@@ -329,20 +332,22 @@ class F4mFD(FragmentFD):
         if not live:
             write_metadata_tag(dest_stream, metadata)
 
+        base_url_parsed = compat_urllib_parse_urlparse(base_url)
+
         self._start_frag_download(ctx)
 
         frags_filenames = []
         while fragments_list:
             seg_i, frag_i = fragments_list.pop(0)
             name = 'Seg%d-Frag%d' % (seg_i, frag_i)
-            url = base_url + name
+            url_parsed = base_url_parsed._replace(path=base_url_parsed.path + name)
             if akamai_pv:
-                url += '?' + akamai_pv.strip(';')
+                url_parsed = url_parsed._replace(query=url_parsed.query + akamai_pv.strip(';'))
             if info_dict.get('extra_param_to_segment_url'):
-                url += info_dict.get('extra_param_to_segment_url')
+                url_parsed = url_parsed._replace(query=url_parsed.query + info_dict.get('extra_param_to_segment_url'))
             frag_filename = '%s-%s' % (ctx['tmpfilename'], name)
             try:
-                success = ctx['dl'].download(frag_filename, {'url': url})
+                success = ctx['dl'].download(frag_filename, {'url': url_parsed.geturl()})
                 if not success:
                     return False
                 (down, frag_sanitized) = sanitize_open(frag_filename, 'rb')

From 8cd9614abf81cb41055142d87158b5eda4353a4d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 24 Oct 2015 21:02:31 +0600
Subject: [PATCH 2903/6219] [downloader/f4m] More accurate fragment URL
 construction

---
 youtube_dl/downloader/f4m.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index b8db6bf9b..7f6143954 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -340,11 +340,14 @@ class F4mFD(FragmentFD):
         while fragments_list:
             seg_i, frag_i = fragments_list.pop(0)
             name = 'Seg%d-Frag%d' % (seg_i, frag_i)
-            url_parsed = base_url_parsed._replace(path=base_url_parsed.path + name)
+            query = []
+            if base_url_parsed.query:
+                query.append(base_url_parsed.query)
             if akamai_pv:
-                url_parsed = url_parsed._replace(query=url_parsed.query + akamai_pv.strip(';'))
+                query.append(akamai_pv.strip(';'))
             if info_dict.get('extra_param_to_segment_url'):
-                url_parsed = url_parsed._replace(query=url_parsed.query + info_dict.get('extra_param_to_segment_url'))
+                query.append(info_dict['extra_param_to_segment_url'])
+            url_parsed = base_url_parsed._replace(path=base_url_parsed.path + name, query='&'.join(query))
             frag_filename = '%s-%s' % (ctx['tmpfilename'], name)
             try:
                 success = ctx['dl'].download(frag_filename, {'url': url_parsed.geturl()})

From ec29539e06e156a2bb589af774a80d156b2c2f76 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 24 Oct 2015 21:03:45 +0600
Subject: [PATCH 2904/6219] [senateisvp] Pass extra param as query segment
 without `?`

---
 youtube_dl/extractor/senateisvp.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/senateisvp.py b/youtube_dl/extractor/senateisvp.py
index 9c53704ea..474ebb49b 100644
--- a/youtube_dl/extractor/senateisvp.py
+++ b/youtube_dl/extractor/senateisvp.py
@@ -121,9 +121,9 @@ class SenateISVPIE(InfoExtractor):
                 'url': compat_urlparse.urljoin(domain, filename) + '?v=3.1.0&fp=&r=&g=',
             }]
         else:
-            hdcore_sign = '?hdcore=3.1.0'
+            hdcore_sign = 'hdcore=3.1.0'
             url_params = (domain, video_id, stream_num)
-            f4m_url = '%s/z/%s_1@%s/manifest.f4m' % url_params + hdcore_sign
+            f4m_url = '%s/z/%s_1@%s/manifest.f4m?' % url_params + hdcore_sign
             m3u8_url = '%s/i/%s_1@%s/master.m3u8' % url_params
             for entry in self._extract_f4m_formats(f4m_url, video_id, f4m_id='f4m'):
                 # URLs without the extra param induce an 404 error

From 8e82ecfe8f0dc2b9dfb6a2cda68e7b5f7926b0e8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 24 Oct 2015 21:04:09 +0600
Subject: [PATCH 2905/6219] [dailymotion] Extract f4m formats

---
 youtube_dl/extractor/dailymotion.py | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index 9cd9ff17d..bc7823931 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -141,9 +141,17 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
                     type_ = media.get('type')
                     if type_ == 'application/vnd.lumberjack.manifest':
                         continue
-                    if type_ == 'application/x-mpegURL' or determine_ext(media_url) == 'm3u8':
-                        formats.extend(self._extract_m3u8_formats(
-                            media_url, video_id, 'mp4', m3u8_id='hls'))
+                    ext = determine_ext(media_url)
+                    if type_ == 'application/x-mpegURL' or ext == 'm3u8':
+                        m3u8_formats = self._extract_m3u8_formats(
+                            media_url, video_id, 'mp4', m3u8_id='hls', fatal=False)
+                        if m3u8_formats:
+                            formats.extend(m3u8_formats)
+                    elif type_ == 'application/f4m' or ext == 'f4m':
+                        f4m_formats = self._extract_f4m_formats(
+                            media_url, video_id, preference=-1, f4m_id='hds', fatal=False)
+                        if f4m_formats:
+                            formats.extend(f4m_formats)
                     else:
                         f = {
                             'url': media_url,

From 7e0dc61334cfe9c79e92fd79d9996d191990a80a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 25 Oct 2015 20:48:29 +0600
Subject: [PATCH 2906/6219] [njoy] Add support for URLs without display id

---
 youtube_dl/extractor/ndr.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/ndr.py b/youtube_dl/extractor/ndr.py
index e3cc6fde8..ba06d8a98 100644
--- a/youtube_dl/extractor/ndr.py
+++ b/youtube_dl/extractor/ndr.py
@@ -14,7 +14,8 @@ from ..utils import (
 
 class NDRBaseIE(InfoExtractor):
     def _real_extract(self, url):
-        display_id = self._match_id(url)
+        mobj = re.match(self._VALID_URL, url)
+        display_id = next(group for group in mobj.groups() if group)
         webpage = self._download_webpage(url, display_id)
         return self._extract_embed(webpage, display_id)
 
@@ -101,7 +102,7 @@ class NDRIE(NDRBaseIE):
 class NJoyIE(NDRBaseIE):
     IE_NAME = 'njoy'
     IE_DESC = 'N-JOY'
-    _VALID_URL = r'https?://www\.n-joy\.de/(?:[^/]+/)+(?P<id>[^/?#]+),[\da-z]+\.html'
+    _VALID_URL = r'https?://www\.n-joy\.de/(?:[^/]+/)+(?:(?P<display_id>[^/?#]+),)?(?P<id>[\da-z]+)\.html'
     _TESTS = [{
         # httpVideo, same content id
         'url': 'http://www.n-joy.de/entertainment/comedy/comedy_contest/Benaissa-beim-NDR-Comedy-Contest,comedycontest2480.html',
@@ -136,6 +137,9 @@ class NJoyIE(NDRBaseIE):
         'params': {
             'skip_download': True,
         },
+    }, {
+        'url': 'http://www.n-joy.de/radio/webradio/morningshow209.html',
+        'only_matching': True,
     }]
 
     def _extract_embed(self, webpage, display_id):

From e572a1010b81f5864e610808c86848db4d6ed8e4 Mon Sep 17 00:00:00 2001
From: Erik <erik@vanderleer.nl>
Date: Sat, 17 Oct 2015 19:22:47 +0200
Subject: [PATCH 2907/6219] [youporn] Fix extraction

[youporn] Added description and thumbnail

[youporn] Added uploader and date

[youporn] Removed Try and Except lines

[youporn] Fixed date, fatal, formats and /s*

[youporn] Undid removing comment about video url components & Undid and fixed removal of encrypted URL detection

[youporn] Fix: Add encrypted link to links array only if not already in it

[youporn] Fix: Add encrypted link to links array only if not already in it

[youporn] Fix: cleanup
---
 youtube_dl/extractor/youporn.py | 55 ++++++++++++++-------------------
 1 file changed, 23 insertions(+), 32 deletions(-)

diff --git a/youtube_dl/extractor/youporn.py b/youtube_dl/extractor/youporn.py
index 4ba7c36db..546985f3a 100644
--- a/youtube_dl/extractor/youporn.py
+++ b/youtube_dl/extractor/youporn.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 import json
 import re
 import sys
+import datetime
 
 from .common import InfoExtractor
 from ..compat import (
@@ -27,10 +28,11 @@ class YouPornIE(InfoExtractor):
         'info_dict': {
             'id': '505835',
             'ext': 'mp4',
-            'upload_date': '20101221',
-            'description': 'Love & Sex Answers: http://bit.ly/DanAndJenn -- Is It Unhealthy To Masturbate Daily?',
-            'uploader': 'Ask Dan And Jennifer',
             'title': 'Sex Ed: Is It Safe To Masturbate Daily?',
+            'description': 'Watch Sex Ed: Is It Safe To Masturbate Daily? at YouPorn.com - YouPorn is the biggest free porn tube site on the net!',
+            'uploader': 'Ask Dan And Jennifer',
+            'thumbnail': 'http://cdn5.image.youporn.phncdn.com/201012/17/505835/640x480/8/sex-ed-is-it-safe-to-masturbate-daily-8.jpg',
+            'date': '20101221',
             'age_limit': 18,
         }
     }
@@ -45,45 +47,34 @@ class YouPornIE(InfoExtractor):
         webpage = self._download_webpage(req, video_id)
         age_limit = self._rta_search(webpage)
 
-        # Get JSON parameters
-        json_params = self._search_regex(
-            [r'videoJa?son\s*=\s*({.+})',
-             r'var\s+currentVideo\s*=\s*new\s+Video\((.+?)\)[,;]'],
-            webpage, 'JSON parameters')
-        try:
-            params = json.loads(json_params)
-        except ValueError:
-            raise ExtractorError('Invalid JSON')
-
         self.report_extraction(video_id)
-        try:
-            video_title = params['title']
-            upload_date = unified_strdate(params['release_date_f'])
-            video_description = params['description']
-            video_uploader = params['submitted_by']
-            thumbnail = params['thumbnails'][0]['image']
-        except KeyError:
-            raise ExtractorError('Missing JSON parameter: ' + sys.exc_info()[1])
+        video_title = self._html_search_regex(r'page_params.video_title = \'(.+?)\';', webpage, 'video URL', fatal=False)
+        video_description = self._html_search_meta('description', webpage, 'video DESC', fatal=False)
+        video_thumbnail = self._html_search_regex(r'page_params.imageurl\t=\t"(.+?)";', webpage, 'video THUMB', fatal=False)
+        video_uploader = self._html_search_regex(r"<div class=\'videoInfoBy\'>By:</div>\n<a href=\"[^>]+\">(.+?)</a>", webpage, 'video UPLOADER', fatal=False)
+        video_date = self._html_search_regex(r"<div class='videoInfoTime'>\n<i class='icon-clock'></i> (.+?)\n</div>", webpage, 'video DATE', fatal=False)
+        video_date = datetime.datetime.strptime(video_date, '%B %d, %Y').strftime('%Y%m%d')
 
         # Get all of the links from the page
-        DOWNLOAD_LIST_RE = r'(?s)<ul class="downloadList">(?P<download_list>.*?)</ul>'
-        download_list_html = self._search_regex(DOWNLOAD_LIST_RE,
-                                                webpage, 'download list').strip()
-        LINK_RE = r'<a href="([^"]+)">'
+        DOWNLOAD_LIST_RE = r'(?s)sources: {\n(?P<download_list>.*?)}'
+        download_list_html = self._search_regex(DOWNLOAD_LIST_RE, webpage, 'download list').strip()
+        LINK_RE = r': \'(.+?)\','
         links = re.findall(LINK_RE, download_list_html)
 
         # Get all encrypted links
-        encrypted_links = re.findall(r'var encryptedQuality[0-9]{3}URL = \'([a-zA-Z0-9+/]+={0,2})\';', webpage)
+        encrypted_links = re.findall(r'page_params.encryptedQuality[0-9]{3,4}URL\s=\s\'([a-zA-Z0-9+/]+={0,2})\';', webpage)
         for encrypted_link in encrypted_links:
             link = aes_decrypt_text(encrypted_link, video_title, 32).decode('utf-8')
-            links.append(link)
+            # it's unclear if encryted links still differ from normal ones, so only include in links array if it's unique
+            if link not in links:
+                links.append(link)
 
         formats = []
         for link in links:
             # A link looks like this:
-            # http://cdn1.download.youporn.phncdn.com/201210/31/8004515/480p_370k_8004515/YouPorn%20-%20Nubile%20Films%20The%20Pillow%20Fight.mp4?nvb=20121113051249&nva=20121114051249&ir=1200&sr=1200&hash=014b882080310e95fb6a0
+            # http://cdn2b.public.youporn.phncdn.com/201012/17/505835/720p_1500k_505835/YouPorn%20-%20Sex%20Ed%20Is%20It%20Safe%20To%20Masturbate%20Daily.mp4?rs=200&ri=2500&s=1445599900&e=1445773500&h=5345d19ce9944ec52eb167abf24af248
             # A path looks like this:
-            # /201210/31/8004515/480p_370k_8004515/YouPorn%20-%20Nubile%20Films%20The%20Pillow%20Fight.mp4
+            # 201012/17/505835/720p_1500k_505835/YouPorn%20-%20Sex%20Ed%20Is%20It%20Safe%20To%20Masturbate%20Daily.mp4
             video_url = unescapeHTML(link)
             path = compat_urllib_parse_urlparse(video_url).path
             format_parts = path.split('/')[4].split('_')[:2]
@@ -111,11 +102,11 @@ class YouPornIE(InfoExtractor):
 
         return {
             'id': video_id,
-            'uploader': video_uploader,
-            'upload_date': upload_date,
             'title': video_title,
-            'thumbnail': thumbnail,
             'description': video_description,
+            'thumbnail': video_thumbnail,
+            'uploader': video_uploader,
+            'date': video_date,
             'age_limit': age_limit,
             'formats': formats,
         }

From 589c33dadeec18a9d50713a4a200e3e2d9e297bb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 25 Oct 2015 22:56:35 +0600
Subject: [PATCH 2908/6219] [youporn] Improve and make more robust (Closes
 #6888, closes #7214)

---
 youtube_dl/extractor/youporn.py | 173 +++++++++++++++++++-------------
 1 file changed, 102 insertions(+), 71 deletions(-)

diff --git a/youtube_dl/extractor/youporn.py b/youtube_dl/extractor/youporn.py
index 546985f3a..d10ebb0bf 100644
--- a/youtube_dl/extractor/youporn.py
+++ b/youtube_dl/extractor/youporn.py
@@ -1,112 +1,143 @@
 from __future__ import unicode_literals
 
-
-import json
 import re
-import sys
-import datetime
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse_urlparse,
-    compat_urllib_request,
-)
+from ..compat import compat_urllib_request
 from ..utils import (
-    ExtractorError,
+    int_or_none,
+    str_to_int,
     unescapeHTML,
     unified_strdate,
 )
-from ..aes import (
-    aes_decrypt_text
-)
+from ..aes import aes_decrypt_text
 
 
 class YouPornIE(InfoExtractor):
-    _VALID_URL = r'^(?P<proto>https?://)(?:www\.)?(?P<url>youporn\.com/watch/(?P<videoid>[0-9]+)/(?P<title>[^/]+))'
+    _VALID_URL = r'https?://(?:www\.)?youporn\.com/watch/(?P<id>\d+)/(?P<display_id>[^/?#&]+)'
     _TEST = {
         'url': 'http://www.youporn.com/watch/505835/sex-ed-is-it-safe-to-masturbate-daily/',
         'info_dict': {
             'id': '505835',
+            'display_id': 'sex-ed-is-it-safe-to-masturbate-daily',
             'ext': 'mp4',
             'title': 'Sex Ed: Is It Safe To Masturbate Daily?',
-            'description': 'Watch Sex Ed: Is It Safe To Masturbate Daily? at YouPorn.com - YouPorn is the biggest free porn tube site on the net!',
+            'description': 'Love & Sex Answers: http://bit.ly/DanAndJenn -- Is It Unhealthy To Masturbate Daily?',
+            'thumbnail': 're:^https?://.*\.jpg$',
             'uploader': 'Ask Dan And Jennifer',
-            'thumbnail': 'http://cdn5.image.youporn.phncdn.com/201012/17/505835/640x480/8/sex-ed-is-it-safe-to-masturbate-daily-8.jpg',
-            'date': '20101221',
+            'upload_date': '20101221',
+            'average_rating': int,
+            'view_count': int,
+            'categories': list,
+            'tags': list,
             'age_limit': 18,
         }
     }
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('videoid')
-        url = mobj.group('proto') + 'www.' + mobj.group('url')
+        video_id = mobj.group('id')
+        display_id = mobj.group('display_id')
 
-        req = compat_urllib_request.Request(url)
-        req.add_header('Cookie', 'age_verified=1')
-        webpage = self._download_webpage(req, video_id)
-        age_limit = self._rta_search(webpage)
+        request = compat_urllib_request.Request(url)
+        request.add_header('Cookie', 'age_verified=1')
+        webpage = self._download_webpage(request, display_id)
 
-        self.report_extraction(video_id)
-        video_title = self._html_search_regex(r'page_params.video_title = \'(.+?)\';', webpage, 'video URL', fatal=False)
-        video_description = self._html_search_meta('description', webpage, 'video DESC', fatal=False)
-        video_thumbnail = self._html_search_regex(r'page_params.imageurl\t=\t"(.+?)";', webpage, 'video THUMB', fatal=False)
-        video_uploader = self._html_search_regex(r"<div class=\'videoInfoBy\'>By:</div>\n<a href=\"[^>]+\">(.+?)</a>", webpage, 'video UPLOADER', fatal=False)
-        video_date = self._html_search_regex(r"<div class='videoInfoTime'>\n<i class='icon-clock'></i> (.+?)\n</div>", webpage, 'video DATE', fatal=False)
-        video_date = datetime.datetime.strptime(video_date, '%B %d, %Y').strftime('%Y%m%d')
+        title = self._search_regex(
+            [r'(?:video_titles|videoTitle)\s*[:=]\s*(["\'])(?P<title>.+?)\1',
+             r'<h1[^>]+class=["\']heading\d?["\'][^>]*>([^<])<'],
+            webpage, 'title', group='title')
 
-        # Get all of the links from the page
-        DOWNLOAD_LIST_RE = r'(?s)sources: {\n(?P<download_list>.*?)}'
-        download_list_html = self._search_regex(DOWNLOAD_LIST_RE, webpage, 'download list').strip()
-        LINK_RE = r': \'(.+?)\','
-        links = re.findall(LINK_RE, download_list_html)
+        links = []
 
-        # Get all encrypted links
-        encrypted_links = re.findall(r'page_params.encryptedQuality[0-9]{3,4}URL\s=\s\'([a-zA-Z0-9+/]+={0,2})\';', webpage)
-        for encrypted_link in encrypted_links:
-            link = aes_decrypt_text(encrypted_link, video_title, 32).decode('utf-8')
-            # it's unclear if encryted links still differ from normal ones, so only include in links array if it's unique
-            if link not in links:
+        sources = self._search_regex(
+            r'sources\s*:\s*({.+?})', webpage, 'sources', default=None)
+        if sources:
+            for _, link in re.findall(r'[^:]+\s*:\s*(["\'])(http.+?)\1', sources):
                 links.append(link)
 
+        # Fallback #1
+        for _, link in re.findall(
+                r'(?:videoUrl|videoSrc|videoIpadUrl|html5PlayerSrc)\s*[:=]\s*(["\'])(http.+?)\1', webpage):
+            links.append(link)
+
+        # Fallback #2, this also contains extra low quality 180p format
+        for _, link in re.findall(r'<a[^>]+href=(["\'])(http.+?)\1[^>]+title=["\']Download [Vv]ideo', webpage):
+            links.append(link)
+
+        # Fallback #3, encrypted links
+        for _, encrypted_link in re.findall(
+                r'encryptedQuality\d{3,4}URL\s*=\s*(["\'])([\da-zA-Z+/=]+)\1', webpage):
+            links.append(aes_decrypt_text(encrypted_link, title, 32).decode('utf-8'))
+
         formats = []
-        for link in links:
-            # A link looks like this:
-            # http://cdn2b.public.youporn.phncdn.com/201012/17/505835/720p_1500k_505835/YouPorn%20-%20Sex%20Ed%20Is%20It%20Safe%20To%20Masturbate%20Daily.mp4?rs=200&ri=2500&s=1445599900&e=1445773500&h=5345d19ce9944ec52eb167abf24af248
-            # A path looks like this:
-            # 201012/17/505835/720p_1500k_505835/YouPorn%20-%20Sex%20Ed%20Is%20It%20Safe%20To%20Masturbate%20Daily.mp4
-            video_url = unescapeHTML(link)
-            path = compat_urllib_parse_urlparse(video_url).path
-            format_parts = path.split('/')[4].split('_')[:2]
-
-            dn = compat_urllib_parse_urlparse(video_url).netloc.partition('.')[0]
-
-            resolution = format_parts[0]
-            height = int(resolution[:-len('p')])
-            bitrate = int(format_parts[1][:-len('k')])
-            format = '-'.join(format_parts) + '-' + dn
-
-            formats.append({
+        for video_url in set(unescapeHTML(link) for link in links):
+            f = {
                 'url': video_url,
-                'format': format,
-                'format_id': format,
-                'height': height,
-                'tbr': bitrate,
-                'resolution': resolution,
-            })
-
+            }
+            # Video URL's path looks like this:
+            #  /201012/17/505835/720p_1500k_505835/YouPorn%20-%20Sex%20Ed%20Is%20It%20Safe%20To%20Masturbate%20Daily.mp4
+            # We will benefit from it by extracting some metadata
+            mobj = re.search(r'/(?P<height>\d{3,4})[pP]_(?P<bitrate>\d+)[kK]_\d+/', video_url)
+            if mobj:
+                height = int(mobj.group('height'))
+                bitrate = int(mobj.group('bitrate'))
+                f.update({
+                    'format_id': '%dp-%dk' % (height, bitrate),
+                    'height': height,
+                    'tbr': bitrate,
+                })
+            formats.append(f)
         self._sort_formats(formats)
 
-        if not formats:
-            raise ExtractorError('ERROR: no known formats available for video')
+        description = self._html_search_regex(
+            r'(?s)<div[^>]+class=["\']video-description["\'][^>]*>(.+?)</div>',
+            webpage, 'description', fatal=False)
+        thumbnail = self._search_regex(
+            r'(?:imageurl\s*=|poster\s*:)\s*(["\'])(?P<thumbnail>.+?)\1',
+            webpage, 'thumbnail', fatal=False, group='thumbnail')
+
+        uploader = self._search_regex(
+            r'<div[^>]+class=["\']videoInfoBy["\'][^>]*>\s*By:\s*</div>\s*<a[^>]+href="[^"]*">([^<]+)</a>',
+            webpage, 'uploader', fatal=False)
+        upload_date = unified_strdate(self._html_search_regex(
+            r'(?s)<div[^>]+class=["\']videoInfoTime["\'][^>]*>(.+?)</div>',
+            webpage, 'upload date', fatal=False))
+
+        age_limit = self._rta_search(webpage)
+
+        average_rating = int_or_none(self._search_regex(
+            r'<div[^>]+class=["\']videoInfoRating["\'][^>]*>\s*<div[^>]+class=["\']videoRatingPercentage["\'][^>]*>(\d+)%</div>',
+            webpage, 'average rating', fatal=False))
+
+        view_count = str_to_int(self._search_regex(
+            r'(?s)<div[^>]+class=["\']videoInfoViews["\'][^>]*>.*?([\d,.]+)\s*</div>',
+            webpage, 'view count', fatal=False))
+
+        def extract_tag_box(title):
+            tag_box = self._search_regex(
+                (r'<div[^>]+class=["\']tagBoxTitle["\'][^>]*>\s*%s\b.*?</div>\s*'
+                 '<div[^>]+class=["\']tagBoxContent["\']>(.+?)</div>') % re.escape(title),
+                webpage, '%s tag box' % title, default=None)
+            if not tag_box:
+                return []
+            return re.findall(r'<a[^>]+href=[^>]+>([^<]+)', tag_box)
+
+        categories = extract_tag_box('Category')
+        tags = extract_tag_box('Tags')
 
         return {
             'id': video_id,
-            'title': video_title,
-            'description': video_description,
-            'thumbnail': video_thumbnail,
-            'uploader': video_uploader,
-            'date': video_date,
+            'display_id': display_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'uploader': uploader,
+            'upload_date': upload_date,
+            'average_rating': average_rating,
+            'view_count': view_count,
+            'categories': categories,
+            'tags': tags,
             'age_limit': age_limit,
             'formats': formats,
         }

From feb7711cf58863a19cae770a878d22a8424e3c61 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 25 Oct 2015 23:01:12 +0600
Subject: [PATCH 2909/6219] [youporn] Make description optional

Some videos does not contain any description
---
 youtube_dl/extractor/youporn.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youporn.py b/youtube_dl/extractor/youporn.py
index d10ebb0bf..db5d049d2 100644
--- a/youtube_dl/extractor/youporn.py
+++ b/youtube_dl/extractor/youporn.py
@@ -92,7 +92,7 @@ class YouPornIE(InfoExtractor):
 
         description = self._html_search_regex(
             r'(?s)<div[^>]+class=["\']video-description["\'][^>]*>(.+?)</div>',
-            webpage, 'description', fatal=False)
+            webpage, 'description', default=None)
         thumbnail = self._search_regex(
             r'(?:imageurl\s*=|poster\s*:)\s*(["\'])(?P<thumbnail>.+?)\1',
             webpage, 'thumbnail', fatal=False, group='thumbnail')

From 4f13f8f798be06bc2b3c0c42818bb0785e4cde64 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 25 Oct 2015 23:12:12 +0600
Subject: [PATCH 2910/6219] [youporn] Improve uploader extraction

---
 youtube_dl/extractor/youporn.py | 32 +++++++++++++++++++++++++++-----
 1 file changed, 27 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/youporn.py b/youtube_dl/extractor/youporn.py
index db5d049d2..b39fbb5fc 100644
--- a/youtube_dl/extractor/youporn.py
+++ b/youtube_dl/extractor/youporn.py
@@ -15,8 +15,9 @@ from ..aes import aes_decrypt_text
 
 class YouPornIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?youporn\.com/watch/(?P<id>\d+)/(?P<display_id>[^/?#&]+)'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.youporn.com/watch/505835/sex-ed-is-it-safe-to-masturbate-daily/',
+        'md5': '71ec5fcfddacf80f495efa8b6a8d9a89',
         'info_dict': {
             'id': '505835',
             'display_id': 'sex-ed-is-it-safe-to-masturbate-daily',
@@ -31,8 +32,29 @@ class YouPornIE(InfoExtractor):
             'categories': list,
             'tags': list,
             'age_limit': 18,
-        }
-    }
+        },
+    }, {
+        # Anonymous User uploader
+        'url': 'http://www.youporn.com/watch/561726/big-tits-awesome-brunette-on-amazing-webcam-show/?from=related3&al=2&from_id=561726&pos=4',
+        'info_dict': {
+            'id': '561726',
+            'display_id': 'big-tits-awesome-brunette-on-amazing-webcam-show',
+            'ext': 'mp4',
+            'title': 'Big Tits Awesome Brunette On amazing webcam show',
+            'description': 'http://sweetlivegirls.com Big Tits Awesome Brunette On amazing webcam show.mp4',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'uploader': 'Anonymous User',
+            'upload_date': '20111125',
+            'average_rating': int,
+            'view_count': int,
+            'categories': list,
+            'tags': list,
+            'age_limit': 18,
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
@@ -97,8 +119,8 @@ class YouPornIE(InfoExtractor):
             r'(?:imageurl\s*=|poster\s*:)\s*(["\'])(?P<thumbnail>.+?)\1',
             webpage, 'thumbnail', fatal=False, group='thumbnail')
 
-        uploader = self._search_regex(
-            r'<div[^>]+class=["\']videoInfoBy["\'][^>]*>\s*By:\s*</div>\s*<a[^>]+href="[^"]*">([^<]+)</a>',
+        uploader = self._html_search_regex(
+            r'(?s)<div[^>]+class=["\']videoInfoBy["\'][^>]*>\s*By:\s*</div>(.+?)</(?:a|div)>',
             webpage, 'uploader', fatal=False)
         upload_date = unified_strdate(self._html_search_regex(
             r'(?s)<div[^>]+class=["\']videoInfoTime["\'][^>]*>(.+?)</div>',

From 7b3a19e5339344037a872574780c39f334cea90e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 25 Oct 2015 23:17:23 +0600
Subject: [PATCH 2911/6219] [stitcher] Remove origEpisodeURL

It's always 404
---
 youtube_dl/extractor/stitcher.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/stitcher.py b/youtube_dl/extractor/stitcher.py
index 971a1c466..d5c852f52 100644
--- a/youtube_dl/extractor/stitcher.py
+++ b/youtube_dl/extractor/stitcher.py
@@ -64,7 +64,7 @@ class StitcherIE(InfoExtractor):
             'url': episode[episode_key],
             'ext': determine_ext(episode[episode_key]) or 'mp3',
             'vcodec': 'none',
-        } for episode_key in ('origEpisodeURL', 'episodeURL') if episode.get(episode_key)]
+        } for episode_key in ('episodeURL',) if episode.get(episode_key)]
         description = self._search_regex(
             r'Episode Info:\s*</span>([^<]+)<', webpage, 'description', fatal=False)
         duration = int_or_none(episode.get('duration'))

From 755ff8d22ca5607400c1232b194e20a004e4e9eb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 25 Oct 2015 23:41:10 +0600
Subject: [PATCH 2912/6219] [youporn] Extract comment count

---
 youtube_dl/extractor/youporn.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/youporn.py b/youtube_dl/extractor/youporn.py
index b39fbb5fc..9bf8d1eeb 100644
--- a/youtube_dl/extractor/youporn.py
+++ b/youtube_dl/extractor/youporn.py
@@ -29,6 +29,7 @@ class YouPornIE(InfoExtractor):
             'upload_date': '20101221',
             'average_rating': int,
             'view_count': int,
+            'comment_count': int,
             'categories': list,
             'tags': list,
             'age_limit': 18,
@@ -47,6 +48,7 @@ class YouPornIE(InfoExtractor):
             'upload_date': '20111125',
             'average_rating': int,
             'view_count': int,
+            'comment_count': int,
             'categories': list,
             'tags': list,
             'age_limit': 18,
@@ -135,6 +137,9 @@ class YouPornIE(InfoExtractor):
         view_count = str_to_int(self._search_regex(
             r'(?s)<div[^>]+class=["\']videoInfoViews["\'][^>]*>.*?([\d,.]+)\s*</div>',
             webpage, 'view count', fatal=False))
+        comment_count = str_to_int(self._search_regex(
+            r'>All [Cc]omments? \(([\d,.]+)\)',
+            webpage, 'comment count', fatal=False))
 
         def extract_tag_box(title):
             tag_box = self._search_regex(
@@ -158,6 +163,7 @@ class YouPornIE(InfoExtractor):
             'upload_date': upload_date,
             'average_rating': average_rating,
             'view_count': view_count,
+            'comment_count': comment_count,
             'categories': categories,
             'tags': tags,
             'age_limit': age_limit,

From 36e6f62cd0883f0f486d1666d010e5d9e6d515bd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 25 Oct 2015 20:04:55 +0100
Subject: [PATCH 2913/6219] Use a wrapper around
 xml.etree.ElementTree.fromstring in python 2.x (#7178)

Attributes aren't unicode objects, so they couldn't be directly used in info_dict fields (for example '--write-description' doesn't work with bytes).
---
 test/test_compat.py                 |  7 +++++++
 test/test_utils.py                  | 11 +++++++----
 youtube_dl/compat.py                | 25 +++++++++++++++++++++++++
 youtube_dl/downloader/f4m.py        |  4 ++--
 youtube_dl/extractor/bbc.py         |  8 +++++---
 youtube_dl/extractor/bilibili.py    |  6 ++++--
 youtube_dl/extractor/brightcove.py  |  4 ++--
 youtube_dl/extractor/common.py      |  4 ++--
 youtube_dl/extractor/crunchyroll.py |  4 ++--
 youtube_dl/extractor/vevo.py        |  6 +++---
 youtube_dl/utils.py                 |  3 ++-
 11 files changed, 61 insertions(+), 21 deletions(-)

diff --git a/test/test_compat.py b/test/test_compat.py
index 4ee0dc99d..2b0860479 100644
--- a/test/test_compat.py
+++ b/test/test_compat.py
@@ -13,8 +13,10 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 from youtube_dl.utils import get_filesystem_encoding
 from youtube_dl.compat import (
     compat_getenv,
+    compat_etree_fromstring,
     compat_expanduser,
     compat_shlex_split,
+    compat_str,
     compat_urllib_parse_unquote,
     compat_urllib_parse_unquote_plus,
 )
@@ -71,5 +73,10 @@ class TestCompat(unittest.TestCase):
     def test_compat_shlex_split(self):
         self.assertEqual(compat_shlex_split('-option "one two"'), ['-option', 'one two'])
 
+    def test_compat_etree_fromstring(self):
+        xml = '<el foo="bar"></el>'
+        doc = compat_etree_fromstring(xml.encode('utf-8'))
+        self.assertTrue(isinstance(doc.attrib['foo'], compat_str))
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/test/test_utils.py b/test/test_utils.py
index 918a7a9ef..a9e0fed7e 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -68,6 +68,9 @@ from youtube_dl.utils import (
     cli_valueless_option,
     cli_bool_option,
 )
+from youtube_dl.compat import (
+    compat_etree_fromstring,
+)
 
 
 class TestUtil(unittest.TestCase):
@@ -242,7 +245,7 @@ class TestUtil(unittest.TestCase):
             <node x="b" y="d" />
             <node x="" />
         </root>'''
-        doc = xml.etree.ElementTree.fromstring(testxml)
+        doc = compat_etree_fromstring(testxml)
 
         self.assertEqual(find_xpath_attr(doc, './/fourohfour', 'n'), None)
         self.assertEqual(find_xpath_attr(doc, './/fourohfour', 'n', 'v'), None)
@@ -263,7 +266,7 @@ class TestUtil(unittest.TestCase):
                 <url>http://server.com/download.mp3</url>
             </media:song>
         </root>'''
-        doc = xml.etree.ElementTree.fromstring(testxml)
+        doc = compat_etree_fromstring(testxml)
         find = lambda p: doc.find(xpath_with_ns(p, {'media': 'http://example.com/'}))
         self.assertTrue(find('media:song') is not None)
         self.assertEqual(find('media:song/media:author').text, 'The Author')
@@ -285,7 +288,7 @@ class TestUtil(unittest.TestCase):
                 <p>Foo</p>
             </div>
         </root>'''
-        doc = xml.etree.ElementTree.fromstring(testxml)
+        doc = compat_etree_fromstring(testxml)
         self.assertEqual(xpath_text(doc, 'div/p'), 'Foo')
         self.assertEqual(xpath_text(doc, 'div/bar', default='default'), 'default')
         self.assertTrue(xpath_text(doc, 'div/bar') is None)
@@ -297,7 +300,7 @@ class TestUtil(unittest.TestCase):
                 <p x="a">Foo</p>
             </div>
         </root>'''
-        doc = xml.etree.ElementTree.fromstring(testxml)
+        doc = compat_etree_fromstring(testxml)
         self.assertEqual(xpath_attr(doc, 'div/p', 'x'), 'a')
         self.assertEqual(xpath_attr(doc, 'div/bar', 'x'), None)
         self.assertEqual(xpath_attr(doc, 'div/p', 'y'), None)
diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index d103ab9ad..cf10835ca 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -14,6 +14,7 @@ import socket
 import subprocess
 import sys
 import itertools
+import xml.etree.ElementTree
 
 
 try:
@@ -212,6 +213,29 @@ try:
 except ImportError:  # Python 2.6
     from xml.parsers.expat import ExpatError as compat_xml_parse_error
 
+if sys.version_info[0] >= 3:
+    compat_etree_fromstring = xml.etree.ElementTree.fromstring
+else:
+    # on python 2.x the the attributes of a node are str objects instead of
+    # unicode
+    etree = xml.etree.ElementTree
+
+    # on 2.6 XML doesn't have a parser argument, function copied from CPython
+    # 2.7 source
+    def _XML(text, parser=None):
+        if not parser:
+            parser = etree.XMLParser(target=etree.TreeBuilder())
+        parser.feed(text)
+        return parser.close()
+
+    def _element_factory(*args, **kwargs):
+        el = etree.Element(*args, **kwargs)
+        for k, v in el.items():
+            el.set(k, v.decode('utf-8'))
+        return el
+
+    def compat_etree_fromstring(text):
+        return _XML(text, parser=etree.XMLParser(target=etree.TreeBuilder(element_factory=_element_factory)))
 
 try:
     from urllib.parse import parse_qs as compat_parse_qs
@@ -507,6 +531,7 @@ __all__ = [
     'compat_chr',
     'compat_cookiejar',
     'compat_cookies',
+    'compat_etree_fromstring',
     'compat_expanduser',
     'compat_get_terminal_size',
     'compat_getenv',
diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index 7f6143954..6170cc155 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -5,10 +5,10 @@ import io
 import itertools
 import os
 import time
-import xml.etree.ElementTree as etree
 
 from .fragment import FragmentFD
 from ..compat import (
+    compat_etree_fromstring,
     compat_urlparse,
     compat_urllib_error,
     compat_urllib_parse_urlparse,
@@ -290,7 +290,7 @@ class F4mFD(FragmentFD):
         man_url = urlh.geturl()
         manifest = urlh.read()
 
-        doc = etree.fromstring(manifest)
+        doc = compat_etree_fromstring(manifest)
         formats = [(int(f.attrib.get('bitrate', -1)), f)
                    for f in self._get_unencrypted_media(doc)]
         if requested_bitrate is None:
diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 2cdce1eb9..a55a6dbc9 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -2,7 +2,6 @@
 from __future__ import unicode_literals
 
 import re
-import xml.etree.ElementTree
 
 from .common import InfoExtractor
 from ..utils import (
@@ -14,7 +13,10 @@ from ..utils import (
     remove_end,
     unescapeHTML,
 )
-from ..compat import compat_HTTPError
+from ..compat import (
+    compat_etree_fromstring,
+    compat_HTTPError,
+)
 
 
 class BBCCoUkIE(InfoExtractor):
@@ -344,7 +346,7 @@ class BBCCoUkIE(InfoExtractor):
                 url, programme_id, 'Downloading media selection XML')
         except ExtractorError as ee:
             if isinstance(ee.cause, compat_HTTPError) and ee.cause.code == 403:
-                media_selection = xml.etree.ElementTree.fromstring(ee.cause.read().decode('utf-8'))
+                media_selection = compat_etree_fromstring(ee.cause.read().decode('utf-8'))
             else:
                 raise
         return self._process_media_selector(media_selection, programme_id)
diff --git a/youtube_dl/extractor/bilibili.py b/youtube_dl/extractor/bilibili.py
index ecc17ebeb..6c66a1236 100644
--- a/youtube_dl/extractor/bilibili.py
+++ b/youtube_dl/extractor/bilibili.py
@@ -4,9 +4,11 @@ from __future__ import unicode_literals
 import re
 import itertools
 import json
-import xml.etree.ElementTree as ET
 
 from .common import InfoExtractor
+from ..compat import (
+    compat_etree_fromstring,
+)
 from ..utils import (
     int_or_none,
     unified_strdate,
@@ -88,7 +90,7 @@ class BiliBiliIE(InfoExtractor):
         except ValueError:
             pass
 
-        lq_doc = ET.fromstring(lq_page)
+        lq_doc = compat_etree_fromstring(lq_page)
         lq_durls = lq_doc.findall('./durl')
 
         hq_doc = self._download_xml(
diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 4721c2293..1686cdde1 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -3,10 +3,10 @@ from __future__ import unicode_literals
 
 import re
 import json
-import xml.etree.ElementTree
 
 from .common import InfoExtractor
 from ..compat import (
+    compat_etree_fromstring,
     compat_parse_qs,
     compat_str,
     compat_urllib_parse,
@@ -119,7 +119,7 @@ class BrightcoveIE(InfoExtractor):
         object_str = fix_xml_ampersands(object_str)
 
         try:
-            object_doc = xml.etree.ElementTree.fromstring(object_str.encode('utf-8'))
+            object_doc = compat_etree_fromstring(object_str.encode('utf-8'))
         except compat_xml_parse_error:
             return
 
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 10c0d5d1f..52523d7b2 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -10,7 +10,6 @@ import re
 import socket
 import sys
 import time
-import xml.etree.ElementTree
 
 from ..compat import (
     compat_cookiejar,
@@ -23,6 +22,7 @@ from ..compat import (
     compat_urllib_request,
     compat_urlparse,
     compat_str,
+    compat_etree_fromstring,
 )
 from ..utils import (
     NO_DEFAULT,
@@ -461,7 +461,7 @@ class InfoExtractor(object):
             return xml_string
         if transform_source:
             xml_string = transform_source(xml_string)
-        return xml.etree.ElementTree.fromstring(xml_string.encode('utf-8'))
+        return compat_etree_fromstring(xml_string.encode('utf-8'))
 
     def _download_json(self, url_or_request, video_id,
                        note='Downloading JSON metadata',
diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index f8ce10111..0c9b8ca02 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -5,12 +5,12 @@ import re
 import json
 import base64
 import zlib
-import xml.etree.ElementTree
 
 from hashlib import sha1
 from math import pow, sqrt, floor
 from .common import InfoExtractor
 from ..compat import (
+    compat_etree_fromstring,
     compat_urllib_parse,
     compat_urllib_parse_unquote,
     compat_urllib_request,
@@ -234,7 +234,7 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
         return output
 
     def _extract_subtitles(self, subtitle):
-        sub_root = xml.etree.ElementTree.fromstring(subtitle)
+        sub_root = compat_etree_fromstring(subtitle)
         return [{
             'ext': 'srt',
             'data': self._convert_subtitles_to_srt(sub_root),
diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py
index c17094f81..4c0de354f 100644
--- a/youtube_dl/extractor/vevo.py
+++ b/youtube_dl/extractor/vevo.py
@@ -1,10 +1,10 @@
 from __future__ import unicode_literals
 
 import re
-import xml.etree.ElementTree
 
 from .common import InfoExtractor
 from ..compat import (
+    compat_etree_fromstring,
     compat_urllib_request,
 )
 from ..utils import (
@@ -97,7 +97,7 @@ class VevoIE(InfoExtractor):
         if last_version['version'] == -1:
             raise ExtractorError('Unable to extract last version of the video')
 
-        renditions = xml.etree.ElementTree.fromstring(last_version['data'])
+        renditions = compat_etree_fromstring(last_version['data'])
         formats = []
         # Already sorted from worst to best quality
         for rend in renditions.findall('rendition'):
@@ -114,7 +114,7 @@ class VevoIE(InfoExtractor):
 
     def _formats_from_smil(self, smil_xml):
         formats = []
-        smil_doc = xml.etree.ElementTree.fromstring(smil_xml.encode('utf-8'))
+        smil_doc = compat_etree_fromstring(smil_xml.encode('utf-8'))
         els = smil_doc.findall('.//{http://www.w3.org/2001/SMIL20/Language}video')
         for el in els:
             src = el.attrib['src']
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index a61e47646..7d846d680 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -36,6 +36,7 @@ import zlib
 from .compat import (
     compat_basestring,
     compat_chr,
+    compat_etree_fromstring,
     compat_html_entities,
     compat_http_client,
     compat_kwargs,
@@ -1974,7 +1975,7 @@ def dfxp2srt(dfxp_data):
 
         return out
 
-    dfxp = xml.etree.ElementTree.fromstring(dfxp_data.encode('utf-8'))
+    dfxp = compat_etree_fromstring(dfxp_data.encode('utf-8'))
     out = []
     paras = dfxp.findall(_x('.//ttml:p')) or dfxp.findall(_x('.//ttaf1:p')) or dfxp.findall('.//p')
 

From 387db16a789fea25795433538d80513c18d0f699 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 25 Oct 2015 20:30:54 +0100
Subject: [PATCH 2914/6219] [compat] compat_etree_fromstring: only decode bytes
 objects

---
 test/test_compat.py  | 3 ++-
 youtube_dl/compat.py | 6 +++---
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/test/test_compat.py b/test/test_compat.py
index 2b0860479..834f4bc55 100644
--- a/test/test_compat.py
+++ b/test/test_compat.py
@@ -74,9 +74,10 @@ class TestCompat(unittest.TestCase):
         self.assertEqual(compat_shlex_split('-option "one two"'), ['-option', 'one two'])
 
     def test_compat_etree_fromstring(self):
-        xml = '<el foo="bar"></el>'
+        xml = '<el foo="bar" spam="中文"></el>'
         doc = compat_etree_fromstring(xml.encode('utf-8'))
         self.assertTrue(isinstance(doc.attrib['foo'], compat_str))
+        self.assertTrue(isinstance(doc.attrib['spam'], compat_str))
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index cf10835ca..f39d4e9a9 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -216,8 +216,7 @@ except ImportError:  # Python 2.6
 if sys.version_info[0] >= 3:
     compat_etree_fromstring = xml.etree.ElementTree.fromstring
 else:
-    # on python 2.x the the attributes of a node are str objects instead of
-    # unicode
+    # on python 2.x the the attributes of a node aren't always unicode objects
     etree = xml.etree.ElementTree
 
     # on 2.6 XML doesn't have a parser argument, function copied from CPython
@@ -231,7 +230,8 @@ else:
     def _element_factory(*args, **kwargs):
         el = etree.Element(*args, **kwargs)
         for k, v in el.items():
-            el.set(k, v.decode('utf-8'))
+            if isinstance(v, bytes):
+                el.set(k, v.decode('utf-8'))
         return el
 
     def compat_etree_fromstring(text):

From 5f9f87c06fa819c75e59f5d0d491d191c229abbc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Lalinsk=C3=BD?= <lukas@oxygene.sk>
Date: Mon, 26 Oct 2015 14:42:17 +0100
Subject: [PATCH 2915/6219] [vidme] Check for deleted videos

---
 youtube_dl/extractor/vidme.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/youtube_dl/extractor/vidme.py b/youtube_dl/extractor/vidme.py
index eb5cde761..3d63ed4f0 100644
--- a/youtube_dl/extractor/vidme.py
+++ b/youtube_dl/extractor/vidme.py
@@ -101,6 +101,10 @@ class VidmeIE(InfoExtractor):
         # suspended
         'url': 'https://vid.me/Ox3G',
         'only_matching': True,
+    }, {
+        # deleted
+        'url': 'https://vid.me/KTPm',
+        'only_matching': True,
     }, {
         # no formats in the API response
         'url': 'https://vid.me/e5g',
@@ -143,6 +147,11 @@ class VidmeIE(InfoExtractor):
 
         video = response['video']
 
+        if video.get('state') == 'deleted':
+            raise ExtractorError(
+                'Vidme said: Sorry, this video has been deleted.',
+                expected=True)
+
         if video.get('state') in ('user-disabled', 'suspended'):
             raise ExtractorError(
                 'Vidme said: This video has been suspended either due to a copyright claim, '

From 5dadae079bd053c822353b081e94d9daff333208 Mon Sep 17 00:00:00 2001
From: Frans de Jonge <fransdejonge@gmail.com>
Date: Mon, 26 Oct 2015 15:11:09 +0100
Subject: [PATCH 2916/6219] [francetv] Add subtitles support

---
 youtube_dl/extractor/francetv.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index 129984a5f..eaaa43958 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -83,6 +83,16 @@ class FranceTVBaseInfoExtractor(InfoExtractor):
         if subtitle:
             title += ' - %s' % subtitle
 
+        subtitles = {}
+        for subtitle_accessibilite in info['subtitles']:
+            if subtitle_accessibilite['url'] is not '':
+                if not subtitles:
+                    subtitles['fr'] = []
+                subtitles['fr'].append({
+                    'ext': subtitle_accessibilite['format'],
+                    'url': subtitle_accessibilite['url'],
+                })
+
         return {
             'id': video_id,
             'title': title,
@@ -91,6 +101,7 @@ class FranceTVBaseInfoExtractor(InfoExtractor):
             'duration': int_or_none(info.get('real_duration')) or parse_duration(info['duree']),
             'timestamp': int_or_none(info['diffusion']['timestamp']),
             'formats': formats,
+            'subtitles': subtitles,
         }
 
 

From 6e4b8b28916aaafc6d1b4b4d69a6f667e35d413f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 26 Oct 2015 20:35:28 +0600
Subject: [PATCH 2917/6219] [francetv] Make subtitles more robust (Closes
 #7298)

---
 youtube_dl/extractor/francetv.py | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index eaaa43958..07115b9d4 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -84,14 +84,12 @@ class FranceTVBaseInfoExtractor(InfoExtractor):
             title += ' - %s' % subtitle
 
         subtitles = {}
-        for subtitle_accessibilite in info['subtitles']:
-            if subtitle_accessibilite['url'] is not '':
-                if not subtitles:
-                    subtitles['fr'] = []
-                subtitles['fr'].append({
-                    'ext': subtitle_accessibilite['format'],
-                    'url': subtitle_accessibilite['url'],
-                })
+        subtitles_list = [{
+            'url': subtitle['url'],
+            'ext': subtitle.get('format'),
+        } for subtitle in info.get('subtitles', []) if subtitle.get('url')]
+        if subtitles_list:
+            subtitles['fr'] = subtitles_list
 
         return {
             'id': video_id,

From c137cc0d33fee8369b857c8f12a9116379248127 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 26 Oct 2015 20:35:45 +0600
Subject: [PATCH 2918/6219] [francetv] Add subtitles test

---
 youtube_dl/extractor/francetv.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index 07115b9d4..a31cc3c97 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -129,6 +129,9 @@ class FranceTvInfoIE(FranceTVBaseInfoExtractor):
             'title': 'Soir 3',
             'upload_date': '20130826',
             'timestamp': 1377548400,
+            'subtitles': {
+                'fr': 'mincount:2',
+            },
         },
     }, {
         'url': 'http://www.francetvinfo.fr/elections/europeennes/direct-europeennes-regardez-le-debat-entre-les-candidats-a-la-presidence-de-la-commission_600639.html',

From f78546272cf7c4b10c8003870728ab69bec982fc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 26 Oct 2015 16:41:24 +0100
Subject: [PATCH 2919/6219] [compat] compat_etree_fromstring: also decode the
 text attribute

Deletes parse_xml from utils, because it also does it.
---
 test/test_compat.py             | 11 ++++++++++-
 youtube_dl/compat.py            | 18 ++++++++++++++++--
 youtube_dl/extractor/ard.py     |  4 ++--
 youtube_dl/extractor/generic.py |  4 ++--
 youtube_dl/utils.py             | 23 -----------------------
 5 files changed, 30 insertions(+), 30 deletions(-)

diff --git a/test/test_compat.py b/test/test_compat.py
index 834f4bc55..b6bfad05e 100644
--- a/test/test_compat.py
+++ b/test/test_compat.py
@@ -74,10 +74,19 @@ class TestCompat(unittest.TestCase):
         self.assertEqual(compat_shlex_split('-option "one two"'), ['-option', 'one two'])
 
     def test_compat_etree_fromstring(self):
-        xml = '<el foo="bar" spam="中文"></el>'
+        xml = '''
+            <root foo="bar" spam="中文">
+                <normal>foo</normal>
+                <chinese>中文</chinese>
+                <foo><bar>spam</bar></foo>
+            </root>
+        '''
         doc = compat_etree_fromstring(xml.encode('utf-8'))
         self.assertTrue(isinstance(doc.attrib['foo'], compat_str))
         self.assertTrue(isinstance(doc.attrib['spam'], compat_str))
+        self.assertTrue(isinstance(doc.find('normal').text, compat_str))
+        self.assertTrue(isinstance(doc.find('chinese').text, compat_str))
+        self.assertTrue(isinstance(doc.find('foo/bar').text, compat_str))
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index f39d4e9a9..2d43ec852 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -216,9 +216,19 @@ except ImportError:  # Python 2.6
 if sys.version_info[0] >= 3:
     compat_etree_fromstring = xml.etree.ElementTree.fromstring
 else:
-    # on python 2.x the the attributes of a node aren't always unicode objects
+    # on python 2.x the attributes and text of a node aren't always unicode
+    # objects
     etree = xml.etree.ElementTree
 
+    try:
+        _etree_iter = etree.Element.iter
+    except AttributeError:  # Python <=2.6
+        def _etree_iter(root):
+            for el in root.findall('*'):
+                yield el
+                for sub in _etree_iter(el):
+                    yield sub
+
     # on 2.6 XML doesn't have a parser argument, function copied from CPython
     # 2.7 source
     def _XML(text, parser=None):
@@ -235,7 +245,11 @@ else:
         return el
 
     def compat_etree_fromstring(text):
-        return _XML(text, parser=etree.XMLParser(target=etree.TreeBuilder(element_factory=_element_factory)))
+        doc = _XML(text, parser=etree.XMLParser(target=etree.TreeBuilder(element_factory=_element_factory)))
+        for el in _etree_iter(doc):
+            if el.text is not None and isinstance(el.text, bytes):
+                el.text = el.text.decode('utf-8')
+        return doc
 
 try:
     from urllib.parse import parse_qs as compat_parse_qs
diff --git a/youtube_dl/extractor/ard.py b/youtube_dl/extractor/ard.py
index 6f465789b..73be6d204 100644
--- a/youtube_dl/extractor/ard.py
+++ b/youtube_dl/extractor/ard.py
@@ -14,8 +14,8 @@ from ..utils import (
     parse_duration,
     unified_strdate,
     xpath_text,
-    parse_xml,
 )
+from ..compat import compat_etree_fromstring
 
 
 class ARDMediathekIE(InfoExtractor):
@@ -161,7 +161,7 @@ class ARDMediathekIE(InfoExtractor):
             raise ExtractorError('This program is only suitable for those aged 12 and older. Video %s is therefore only available between 20 pm and 6 am.' % video_id, expected=True)
 
         if re.search(r'[\?&]rss($|[=&])', url):
-            doc = parse_xml(webpage)
+            doc = compat_etree_fromstring(webpage.encode('utf-8'))
             if doc.tag == 'rss':
                 return GenericIE()._extract_rss(url, video_id, doc)
 
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index ca5fbafb2..1de96b268 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -9,6 +9,7 @@ import sys
 from .common import InfoExtractor
 from .youtube import YoutubeIE
 from ..compat import (
+    compat_etree_fromstring,
     compat_urllib_parse_unquote,
     compat_urllib_request,
     compat_urlparse,
@@ -21,7 +22,6 @@ from ..utils import (
     HEADRequest,
     is_html,
     orderedSet,
-    parse_xml,
     smuggle_url,
     unescapeHTML,
     unified_strdate,
@@ -1237,7 +1237,7 @@ class GenericIE(InfoExtractor):
 
         # Is it an RSS feed, a SMIL file or a XSPF playlist?
         try:
-            doc = parse_xml(webpage)
+            doc = compat_etree_fromstring(webpage.encode('utf-8'))
             if doc.tag == 'rss':
                 return self._extract_rss(url, video_id, doc)
             elif re.match(r'^(?:{[^}]+})?smil$', doc.tag):
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 7d846d680..c761ea22a 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1652,29 +1652,6 @@ def encode_dict(d, encoding='utf-8'):
     return dict((k.encode(encoding), v.encode(encoding)) for k, v in d.items())
 
 
-try:
-    etree_iter = xml.etree.ElementTree.Element.iter
-except AttributeError:  # Python <=2.6
-    etree_iter = lambda n: n.findall('.//*')
-
-
-def parse_xml(s):
-    class TreeBuilder(xml.etree.ElementTree.TreeBuilder):
-        def doctype(self, name, pubid, system):
-            pass  # Ignore doctypes
-
-    parser = xml.etree.ElementTree.XMLParser(target=TreeBuilder())
-    kwargs = {'parser': parser} if sys.version_info >= (2, 7) else {}
-    tree = xml.etree.ElementTree.XML(s.encode('utf-8'), **kwargs)
-    # Fix up XML parser in Python 2.x
-    if sys.version_info < (3, 0):
-        for n in etree_iter(tree):
-            if n.text is not None:
-                if not isinstance(n.text, compat_str):
-                    n.text = n.text.decode('utf-8')
-    return tree
-
-
 US_RATINGS = {
     'G': 0,
     'PG': 10,

From a526167d40983e47231d10c09c9f9064e0298604 Mon Sep 17 00:00:00 2001
From: Pierre Fenoll <pierrefenoll@gmail.com>
Date: Tue, 27 Oct 2015 11:58:59 +0100
Subject: [PATCH 2920/6219] [francetv] Accept mobile URLs

---
 youtube_dl/extractor/francetv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index a31cc3c97..d63dc4d7c 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -105,7 +105,7 @@ class FranceTVBaseInfoExtractor(InfoExtractor):
 
 class PluzzIE(FranceTVBaseInfoExtractor):
     IE_NAME = 'pluzz.francetv.fr'
-    _VALID_URL = r'https?://pluzz\.francetv\.fr/videos/(.*?)\.html'
+    _VALID_URL = r'https?://(?:m\.)?pluzz\.francetv\.fr/videos/(.*?)\.html'
 
     # Can't use tests, videos expire in 7 days
 

From 0a192fbea798c843ad6fef37106901d431f39b6e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 27 Oct 2015 21:43:29 +0600
Subject: [PATCH 2921/6219] [pluzz] Fix mobile support and modernize (Closes
 #7305)

---
 youtube_dl/extractor/francetv.py | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index d63dc4d7c..00a80ba61 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -105,15 +105,21 @@ class FranceTVBaseInfoExtractor(InfoExtractor):
 
 class PluzzIE(FranceTVBaseInfoExtractor):
     IE_NAME = 'pluzz.francetv.fr'
-    _VALID_URL = r'https?://(?:m\.)?pluzz\.francetv\.fr/videos/(.*?)\.html'
+    _VALID_URL = r'https?://(?:m\.)?pluzz\.francetv\.fr/videos/(?P<id>.+?)\.html'
 
     # Can't use tests, videos expire in 7 days
 
     def _real_extract(self, url):
-        title = re.match(self._VALID_URL, url).group(1)
-        webpage = self._download_webpage(url, title)
-        video_id = self._search_regex(
-            r'data-diffusion="(\d+)"', webpage, 'ID')
+        display_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, display_id)
+
+        video_id = self._html_search_meta(
+            'id_video', webpage, 'video id', default=None)
+        if not video_id:
+            video_id = self._search_regex(
+            r'data-diffusion=["\'](\d+)', webpage, 'video id')
+
         return self._extract_video(video_id, 'Pluzz')
 
 

From 7ccb2b84ddb65f41d01037b5b62301886be9d22c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 28 Oct 2015 08:22:04 +0100
Subject: [PATCH 2922/6219] [francetv] fix style issues reported by flake8

* Don't redefine variable in list comprehension
* Line missing indentation
---
 youtube_dl/extractor/francetv.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index 00a80ba61..8e60cf60f 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -85,9 +85,9 @@ class FranceTVBaseInfoExtractor(InfoExtractor):
 
         subtitles = {}
         subtitles_list = [{
-            'url': subtitle['url'],
-            'ext': subtitle.get('format'),
-        } for subtitle in info.get('subtitles', []) if subtitle.get('url')]
+            'url': subformat['url'],
+            'ext': subformat.get('format'),
+        } for subformat in info.get('subtitles', []) if subformat.get('url')]
         if subtitles_list:
             subtitles['fr'] = subtitles_list
 
@@ -118,7 +118,7 @@ class PluzzIE(FranceTVBaseInfoExtractor):
             'id_video', webpage, 'video id', default=None)
         if not video_id:
             video_id = self._search_regex(
-            r'data-diffusion=["\'](\d+)', webpage, 'video id')
+                r'data-diffusion=["\'](\d+)', webpage, 'video id')
 
         return self._extract_video(video_id, 'Pluzz')
 

From 4e16c1f80b009001aaea6f8baca5dfbfa9b58c11 Mon Sep 17 00:00:00 2001
From: Cian Ruane <CianLR@users.noreply.github.com>
Date: Fri, 16 Oct 2015 01:23:09 +0100
Subject: [PATCH 2923/6219] [clyp] Add extractor

Update __init__.py

[clyp.it] Extract ID idiomatically and make duration and description optional
---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/clyp.py     | 57 ++++++++++++++++++++++++++++++++
 2 files changed, 58 insertions(+)
 create mode 100644 youtube_dl/extractor/clyp.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 6318ac4a2..f98e6487e 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -90,6 +90,7 @@ from .cliphunter import CliphunterIE
 from .clipsyndicate import ClipsyndicateIE
 from .cloudy import CloudyIE
 from .clubic import ClubicIE
+from .clyp import ClypIE
 from .cmt import CMTIE
 from .cnet import CNETIE
 from .cnn import (
diff --git a/youtube_dl/extractor/clyp.py b/youtube_dl/extractor/clyp.py
new file mode 100644
index 000000000..906729b30
--- /dev/null
+++ b/youtube_dl/extractor/clyp.py
@@ -0,0 +1,57 @@
+# coding: utf-8
+
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+
+class ClypIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?clyp\.it/(?P<id>[a-z0-9]+)'
+
+    _TESTS = [{
+        'url': 'https://clyp.it/ojz2wfah',
+        'md5': '1d4961036c41247ecfdcc439c0cddcbb',
+        'info_dict': {
+            'id': 'ojz2wfah',
+            'ext': 'mp3',
+            'title': 'Krisson80 - bits wip wip',
+            'description': '#Krisson80BitsWipWip #chiptune\n#wip',
+        },
+    }, {
+        'url': 'https://clyp.it/ojz2wfah',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        audio_id = self._match_id(url)
+        api_url = 'https://api.clyp.it/' + audio_id
+        metadata = self._download_json(api_url, audio_id)
+
+        title = metadata['Title']
+
+        description = None
+        if metadata['Description']: description = metadata['Description']
+
+        duration = None
+        if metadata['Duration']: duration = int(metadata['Duration'])
+        
+        formats = [
+            {
+            'url': metadata['OggUrl'],
+            'format_id': 'ogg',
+            'preference': -2
+            },{
+            'url': metadata['Mp3Url'],
+            'format_id': 'mp3',
+            'preference': -1
+            }]
+
+        return {
+            'id': audio_id,
+            'title': title,
+            'formats': formats,
+            'description': description,
+            'duration': duration
+        }

From 52c3a6e49d2cbc1932992d816d28bbed629daadc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 28 Oct 2015 21:40:22 +0600
Subject: [PATCH 2924/6219] [utils] Improve parse_iso8601

---
 test/test_utils.py  |  2 ++
 youtube_dl/utils.py | 13 +++++++++----
 2 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 918a7a9ef..0c34f0e55 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -425,6 +425,8 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(parse_iso8601('2014-03-23T22:04:26+0000'), 1395612266)
         self.assertEqual(parse_iso8601('2014-03-23T22:04:26Z'), 1395612266)
         self.assertEqual(parse_iso8601('2014-03-23T22:04:26.1234Z'), 1395612266)
+        self.assertEqual(parse_iso8601('2015-09-29T08:27:31.727'), 1443515251)
+        self.assertEqual(parse_iso8601('2015-09-29T08-27-31.727'), None)
 
     def test_strip_jsonp(self):
         stripped = strip_jsonp('cb ([ {"id":"532cb",\n\n\n"x":\n3}\n]\n);')
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index a61e47646..558c9c7d5 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -814,9 +814,11 @@ def parse_iso8601(date_str, delimiter='T', timezone=None):
     if date_str is None:
         return None
 
+    date_str = re.sub(r'\.[0-9]+', '', date_str)
+
     if timezone is None:
         m = re.search(
-            r'(\.[0-9]+)?(?:Z$| ?(?P<sign>\+|-)(?P<hours>[0-9]{2}):?(?P<minutes>[0-9]{2})$)',
+            r'(?:Z$| ?(?P<sign>\+|-)(?P<hours>[0-9]{2}):?(?P<minutes>[0-9]{2})$)',
             date_str)
         if not m:
             timezone = datetime.timedelta()
@@ -829,9 +831,12 @@ def parse_iso8601(date_str, delimiter='T', timezone=None):
                 timezone = datetime.timedelta(
                     hours=sign * int(m.group('hours')),
                     minutes=sign * int(m.group('minutes')))
-    date_format = '%Y-%m-%d{0}%H:%M:%S'.format(delimiter)
-    dt = datetime.datetime.strptime(date_str, date_format) - timezone
-    return calendar.timegm(dt.timetuple())
+    try:
+        date_format = '%Y-%m-%d{0}%H:%M:%S'.format(delimiter)
+        dt = datetime.datetime.strptime(date_str, date_format) - timezone
+        return calendar.timegm(dt.timetuple())
+    except ValueError:
+        pass
 
 
 def unified_strdate(date_str, day_first=True):

From 03c2c162f9eaac6b474a1be9e985621f5b7b8c10 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 28 Oct 2015 21:42:01 +0600
Subject: [PATCH 2925/6219] [clyp] Improve and cleanup (Closes #7194)

---
 youtube_dl/extractor/clyp.py | 62 ++++++++++++++++++------------------
 1 file changed, 31 insertions(+), 31 deletions(-)

diff --git a/youtube_dl/extractor/clyp.py b/youtube_dl/extractor/clyp.py
index 906729b30..57e643799 100644
--- a/youtube_dl/extractor/clyp.py
+++ b/youtube_dl/extractor/clyp.py
@@ -1,16 +1,15 @@
-# coding: utf-8
-
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
+from ..utils import (
+    float_or_none,
+    parse_iso8601,
+)
 
 
 class ClypIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?clyp\.it/(?P<id>[a-z0-9]+)'
-
-    _TESTS = [{
+    _TEST = {
         'url': 'https://clyp.it/ojz2wfah',
         'md5': '1d4961036c41247ecfdcc439c0cddcbb',
         'info_dict': {
@@ -18,40 +17,41 @@ class ClypIE(InfoExtractor):
             'ext': 'mp3',
             'title': 'Krisson80 - bits wip wip',
             'description': '#Krisson80BitsWipWip #chiptune\n#wip',
+            'duration': 263.21,
+            'timestamp': 1443515251,
+            'upload_date': '20150929',
         },
-    }, {
-        'url': 'https://clyp.it/ojz2wfah',
-        'only_matching': True,
-    }]
+    }
 
     def _real_extract(self, url):
         audio_id = self._match_id(url)
-        api_url = 'https://api.clyp.it/' + audio_id
-        metadata = self._download_json(api_url, audio_id)
+
+        metadata = self._download_json(
+            'https://api.clyp.it/%s' % audio_id, audio_id)
+
+        formats = []
+        for secure in ('', 'Secure'):
+            for ext in ('Ogg', 'Mp3'):
+                format_id = '%s%s' % (secure, ext)
+                format_url = metadata.get('%sUrl' % format_id)
+                if format_url:
+                    formats.append({
+                        'url': format_url,
+                        'format_id': format_id,
+                        'vcodec': 'none',
+                    })
+        self._sort_formats(formats)
 
         title = metadata['Title']
-
-        description = None
-        if metadata['Description']: description = metadata['Description']
-
-        duration = None
-        if metadata['Duration']: duration = int(metadata['Duration'])
-        
-        formats = [
-            {
-            'url': metadata['OggUrl'],
-            'format_id': 'ogg',
-            'preference': -2
-            },{
-            'url': metadata['Mp3Url'],
-            'format_id': 'mp3',
-            'preference': -1
-            }]
+        description = metadata.get('Description')
+        duration = float_or_none(metadata.get('Duration'))
+        timestamp = parse_iso8601(metadata.get('DateCreated'))
 
         return {
             'id': audio_id,
             'title': title,
-            'formats': formats,
             'description': description,
-            'duration': duration
+            'duration': duration,
+            'timestamp': timestamp,
+            'formats': formats,
         }

From ae37338e681319a28d98dc551253d9fa1830969a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 29 Oct 2015 13:58:40 +0100
Subject: [PATCH 2926/6219] [compat] compat_etree_fromstring: clarify comment

---
 youtube_dl/compat.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 2d43ec852..a3e85264a 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -216,8 +216,8 @@ except ImportError:  # Python 2.6
 if sys.version_info[0] >= 3:
     compat_etree_fromstring = xml.etree.ElementTree.fromstring
 else:
-    # on python 2.x the attributes and text of a node aren't always unicode
-    # objects
+    # python 2.x tries to encode unicode strings with ascii (see the
+    # XMLParser._fixtext method)
     etree = xml.etree.ElementTree
 
     try:

From 6fb8ace671db2f2bdcc9cd7ac6b9f81fbd356791 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 29 Oct 2015 22:44:01 +0600
Subject: [PATCH 2927/6219] [moniker] Add support for builtin embedded videos
 (Closes #7244)

---
 youtube_dl/extractor/moniker.py | 35 ++++++++++++++++++++++-----------
 1 file changed, 24 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/moniker.py b/youtube_dl/extractor/moniker.py
index 69e4bcd1a..204c03c4a 100644
--- a/youtube_dl/extractor/moniker.py
+++ b/youtube_dl/extractor/moniker.py
@@ -17,7 +17,7 @@ from ..utils import (
 
 class MonikerIE(InfoExtractor):
     IE_DESC = 'allmyvideos.net and vidspot.net'
-    _VALID_URL = r'https?://(?:www\.)?(?:allmyvideos|vidspot)\.net/(?P<id>[a-zA-Z0-9_-]+)'
+    _VALID_URL = r'https?://(?:www\.)?(?:allmyvideos|vidspot)\.net/(?:(?:2|v)/v-)?(?P<id>[a-zA-Z0-9_-]+)'
 
     _TESTS = [{
         'url': 'http://allmyvideos.net/jih3nce3x6wn',
@@ -64,18 +64,30 @@ class MonikerIE(InfoExtractor):
             raise ExtractorError(
                 '%s returned error: %s' % (self.IE_NAME, error), expected=True)
 
-        fields = re.findall(r'type="hidden" name="(.+?)"\s* value="?(.+?)">', orig_webpage)
-        data = dict(fields)
+        builtin_url = self._search_regex(
+            r'<iframe[^>]+src=(["\'])(?P<url>.+?/builtin-.+?)\1',
+            orig_webpage, 'builtin URL', default=None, group='url')
 
-        post = compat_urllib_parse.urlencode(data)
-        headers = {
-            b'Content-Type': b'application/x-www-form-urlencoded',
-        }
-        req = compat_urllib_request.Request(url, post, headers)
-        webpage = self._download_webpage(
-            req, video_id, note='Downloading video page ...')
+        if builtin_url:
+            req = compat_urllib_request.Request(builtin_url)
+            req.add_header('Referer', url)
+            webpage = self._download_webpage(req, video_id, 'Downloading builtin page')
+            title = self._og_search_title(orig_webpage).strip()
+            description = self._og_search_description(orig_webpage).strip()
+        else:
+            fields = re.findall(r'type="hidden" name="(.+?)"\s* value="?(.+?)">', orig_webpage)
+            data = dict(fields)
 
-        title = os.path.splitext(data['fname'])[0]
+            post = compat_urllib_parse.urlencode(data)
+            headers = {
+                b'Content-Type': b'application/x-www-form-urlencoded',
+            }
+            req = compat_urllib_request.Request(url, post, headers)
+            webpage = self._download_webpage(
+                req, video_id, note='Downloading video page ...')
+
+            title = os.path.splitext(data['fname'])[0]
+            description = None
 
         # Could be several links with different quality
         links = re.findall(r'"file" : "?(.+?)",', webpage)
@@ -89,5 +101,6 @@ class MonikerIE(InfoExtractor):
         return {
             'id': video_id,
             'title': title,
+            'description': description,
             'formats': formats,
         }

From 721f5a277ca0012ee72c9d4b3e5550e52a0a596d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 29 Oct 2015 22:47:18 +0600
Subject: [PATCH 2928/6219] [moniker] Add tests for #7244

---
 youtube_dl/extractor/moniker.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/youtube_dl/extractor/moniker.py b/youtube_dl/extractor/moniker.py
index 204c03c4a..7c0c4e50e 100644
--- a/youtube_dl/extractor/moniker.py
+++ b/youtube_dl/extractor/moniker.py
@@ -46,6 +46,18 @@ class MonikerIE(InfoExtractor):
     }, {
         'url': 'https://www.vidspot.net/l2ngsmhs8ci5',
         'only_matching': True,
+    }, {
+        'url': 'http://vidspot.net/2/v-ywDf99',
+        'md5': '5f8254ce12df30479428b0152fb8e7ba',
+        'info_dict': {
+            'id': 'ywDf99',
+            'ext': 'mp4',
+            'title': 'IL FAIT LE MALIN EN PORSHE CAYENNE ( mais pas pour longtemps)',
+            'description': 'IL FAIT LE MALIN EN PORSHE CAYENNE.',
+        },
+    }, {
+        'url': 'http://allmyvideos.net/v/v-HXZm5t',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 30bd1c16c8e2a767246cc115f12c1d2e99e2f8ec Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 29 Oct 2015 19:44:26 +0100
Subject: [PATCH 2929/6219] [adobetv] use api for extraction and add support
 specific language videos

---
 youtube_dl/extractor/adobetv.py | 60 +++++++++++++--------------------
 1 file changed, 24 insertions(+), 36 deletions(-)

diff --git a/youtube_dl/extractor/adobetv.py b/youtube_dl/extractor/adobetv.py
index 5e43adc51..3d25caad6 100644
--- a/youtube_dl/extractor/adobetv.py
+++ b/youtube_dl/extractor/adobetv.py
@@ -1,23 +1,26 @@
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..utils import (
     parse_duration,
     unified_strdate,
     str_to_int,
+    int_or_none,
     float_or_none,
     ISO639Utils,
 )
 
 
 class AdobeTVIE(InfoExtractor):
-    _VALID_URL = r'https?://tv\.adobe\.com/watch/[^/]+/(?P<id>[^/]+)'
+    _VALID_URL = r'https?://tv\.adobe\.com/(?:(?P<language>fr|de|es|jp)/)?watch/(?P<show_urlname>[^/]+)/(?P<id>[^/]+)'
 
     _TEST = {
         'url': 'http://tv.adobe.com/watch/the-complete-picture-with-julieanne-kost/quick-tip-how-to-draw-a-circle-around-an-object-in-photoshop/',
         'md5': '9bc5727bcdd55251f35ad311ca74fa1e',
         'info_dict': {
-            'id': 'quick-tip-how-to-draw-a-circle-around-an-object-in-photoshop',
+            'id': '10981',
             'ext': 'mp4',
             'title': 'Quick Tip - How to Draw a Circle Around an Object in Photoshop',
             'description': 'md5:99ec318dc909d7ba2a1f2b038f7d2311',
@@ -29,46 +32,31 @@ class AdobeTVIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
+        language, show_urlname, urlname = re.match(self._VALID_URL, url).groups()
+        if not language:
+            language = 'en'
 
-        player = self._parse_json(
-            self._search_regex(r'html5player:\s*({.+?})\s*\n', webpage, 'player'),
-            video_id)
-
-        title = player.get('title') or self._search_regex(
-            r'data-title="([^"]+)"', webpage, 'title')
-        description = self._og_search_description(webpage)
-        thumbnail = self._og_search_thumbnail(webpage)
-
-        upload_date = unified_strdate(
-            self._html_search_meta('datepublished', webpage, 'upload date'))
-
-        duration = parse_duration(
-            self._html_search_meta('duration', webpage, 'duration') or
-            self._search_regex(
-                r'Runtime:\s*(\d{2}:\d{2}:\d{2})',
-                webpage, 'duration', fatal=False))
-
-        view_count = str_to_int(self._search_regex(
-            r'<div class="views">\s*Views?:\s*([\d,.]+)\s*</div>',
-            webpage, 'view count'))
+        video_data = self._download_json(
+            'http://tv.adobe.com/api/v4/episode/get/?language=%s&show_urlname=%s&urlname=%s&disclosure=standard' % (language, show_urlname, urlname),
+            urlname)['data'][0]
 
         formats = [{
-            'url': source['src'],
-            'format_id': source.get('quality') or source['src'].split('-')[-1].split('.')[0] or None,
-            'tbr': source.get('bitrate'),
-        } for source in player['sources']]
+            'url': source['url'],
+            'format_id': source.get('quality_level') or source['url'].split('-')[-1].split('.')[0] or None,
+            'width': int_or_none(source.get('width')),
+            'height': int_or_none(source.get('height')),
+            'tbr': int_or_none(source.get('video_data_rate')),
+        } for source in video_data['videos']]
         self._sort_formats(formats)
 
         return {
-            'id': video_id,
-            'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
-            'upload_date': upload_date,
-            'duration': duration,
-            'view_count': view_count,
+            'id': str(video_data['id']),
+            'title': video_data['title'],
+            'description': video_data.get('description'),
+            'thumbnail': video_data.get('thumbnail'),
+            'upload_date': unified_strdate(video_data.get('start_date')),
+            'duration': parse_duration(video_data.get('duration')),
+            'view_count': str_to_int(video_data.get('playcount')),
             'formats': formats,
         }
 

From 402ca40c9d0b20f1d04a0035b4cda0cc1184c689 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 29 Oct 2015 19:55:04 +0100
Subject: [PATCH 2930/6219] [adobetv] extract AdobeTVVideo info from json
 directly

---
 youtube_dl/extractor/adobetv.py | 27 +++++++++++++--------------
 1 file changed, 13 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/adobetv.py b/youtube_dl/extractor/adobetv.py
index 3d25caad6..383c89485 100644
--- a/youtube_dl/extractor/adobetv.py
+++ b/youtube_dl/extractor/adobetv.py
@@ -10,6 +10,7 @@ from ..utils import (
     int_or_none,
     float_or_none,
     ISO639Utils,
+    determine_ext,
 )
 
 
@@ -79,28 +80,25 @@ class AdobeTVVideoIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-
-        webpage = self._download_webpage(url, video_id)
-
-        player_params = self._parse_json(self._search_regex(
-            r'var\s+bridge\s*=\s*([^;]+);', webpage, 'player parameters'),
-            video_id)
+        video_data = self._download_json(url + '?format=json', video_id)
 
         formats = [{
+            'format_id': '%s-%s' % (determine_ext(source['src']), source.get('height')),
             'url': source['src'],
-            'width': source.get('width'),
-            'height': source.get('height'),
-            'tbr': source.get('bitrate'),
-        } for source in player_params['sources']]
+            'width': int_or_none(source.get('width')),
+            'height': int_or_none(source.get('height')),
+            'tbr': int_or_none(source.get('bitrate')),
+        } for source in video_data['sources']]
+        self._sort_formats(formats)
 
         # For both metadata and downloaded files the duration varies among
         # formats. I just pick the max one
         duration = max(filter(None, [
             float_or_none(source.get('duration'), scale=1000)
-            for source in player_params['sources']]))
+            for source in video_data['sources']]))
 
         subtitles = {}
-        for translation in player_params.get('translations', []):
+        for translation in video_data.get('translations', []):
             lang_id = translation.get('language_w3c') or ISO639Utils.long2short(translation['language_medium'])
             if lang_id not in subtitles:
                 subtitles[lang_id] = []
@@ -112,8 +110,9 @@ class AdobeTVVideoIE(InfoExtractor):
         return {
             'id': video_id,
             'formats': formats,
-            'title': player_params['title'],
-            'description': self._og_search_description(webpage),
+            'title': video_data['title'],
+            'description': video_data.get('description'),
+            'thumbnail': video_data['video'].get('poster'),
             'duration': duration,
             'subtitles': subtitles,
         }

From 9a605c8859e5ecf164719b890ea62b76afb0b874 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 29 Oct 2015 20:00:27 +0100
Subject: [PATCH 2931/6219] [adobetv] add support for show and channel
 extraction

---
 youtube_dl/extractor/__init__.py |  2 +
 youtube_dl/extractor/adobetv.py  | 72 ++++++++++++++++++++++++++++++++
 2 files changed, 74 insertions(+)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 6318ac4a2..d4b42dc25 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -6,6 +6,8 @@ from .academicearth import AcademicEarthCourseIE
 from .addanime import AddAnimeIE
 from .adobetv import (
     AdobeTVIE,
+    AdobeTVShowIE,
+    AdobeTVChannelIE,
     AdobeTVVideoIE,
 )
 from .adultswim import AdultSwimIE
diff --git a/youtube_dl/extractor/adobetv.py b/youtube_dl/extractor/adobetv.py
index 383c89485..d0bfafa45 100644
--- a/youtube_dl/extractor/adobetv.py
+++ b/youtube_dl/extractor/adobetv.py
@@ -62,6 +62,78 @@ class AdobeTVIE(InfoExtractor):
         }
 
 
+class AdobeTVPlaylistBaseIE(InfoExtractor):
+    def _parse_page_data(self, page_data):
+        return [self.url_result(self._get_element_url(element_data)) for element_data in page_data]
+
+    def _extract_playlist_entries(self, url, display_id):
+        page = self._download_json(url, display_id)
+        entries = self._parse_page_data(page['data'])
+        for page_num in range(2, page['paging']['pages'] + 1):
+            entries.extend(self._parse_page_data(
+                self._download_json(url + '&page=%d' % page_num, display_id)['data']))
+        return entries
+
+
+class AdobeTVShowIE(AdobeTVPlaylistBaseIE):
+    _VALID_URL = r'https?://tv\.adobe\.com/(?:(?P<language>fr|de|es|jp)/)?show/(?P<id>[^/]+)'
+
+    _TEST = {
+        'url': 'http://tv.adobe.com/show/the-complete-picture-with-julieanne-kost',
+        'info_dict': {
+            'id': '36',
+            'title': 'The Complete Picture with Julieanne Kost',
+            'description': 'md5:fa50867102dcd1aa0ddf2ab039311b27',
+        },
+        'playlist_mincount': 136,
+    }
+
+    def _get_element_url(self, element_data):
+        return element_data['urls'][0]
+
+    def _real_extract(self, url):
+        language, show_urlname = re.match(self._VALID_URL, url).groups()
+        if not language:
+            language = 'en'
+        query = 'language=%s&show_urlname=%s' % (language, show_urlname)
+
+        show_data = self._download_json(
+            'http://tv.adobe.com/api/v4/show/get/?%s' % query, show_urlname)['data'][0]
+
+        return self.playlist_result(
+            self._extract_playlist_entries('http://tv.adobe.com/api/v4/episode/?%s' % query, show_urlname),
+            str(show_data['id']),
+            show_data['show_name'],
+            show_data['show_description'])
+
+
+class AdobeTVChannelIE(AdobeTVPlaylistBaseIE):
+    _VALID_URL = r'https?://tv\.adobe\.com/(?:(?P<language>fr|de|es|jp)/)?channel/(?P<id>[^/]+)(?:/(?P<category_urlname>[^/]+))?'
+
+    _TEST = {
+        'url': 'http://tv.adobe.com/channel/development',
+        'info_dict': {
+            'id': 'development',
+        },
+        'playlist_mincount': 96,
+    }
+
+    def _get_element_url(self, element_data):
+        return element_data['url']
+
+    def _real_extract(self, url):
+        language, channel_urlname, category_urlname = re.match(self._VALID_URL, url).groups()
+        if not language:
+            language = 'en'
+        query = 'language=%s&channel_urlname=%s' % (language, channel_urlname)
+        if category_urlname:
+            query += '&category_urlname=%s' % category_urlname
+
+        return self.playlist_result(
+            self._extract_playlist_entries('http://tv.adobe.com/api/v4/show/?%s' % query, channel_urlname),
+            channel_urlname)
+
+
 class AdobeTVVideoIE(InfoExtractor):
     _VALID_URL = r'https?://video\.tv\.adobe\.com/v/(?P<id>\d+)'
 

From 00d24327efcac74b11dbc4d813aed74da9a501e0 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 30 Oct 2015 09:48:56 +0100
Subject: [PATCH 2932/6219] [vgtv] extract videos from FTV, Aftenposten,
 Aftonbladet using VGTVIE

---
 youtube_dl/extractor/__init__.py    |  1 -
 youtube_dl/extractor/aftenposten.py | 23 -----------
 youtube_dl/extractor/vgtv.py        | 60 ++++++++++++++++++++++-------
 3 files changed, 46 insertions(+), 38 deletions(-)
 delete mode 100644 youtube_dl/extractor/aftenposten.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index f98e6487e..f7dcabcf7 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -9,7 +9,6 @@ from .adobetv import (
     AdobeTVVideoIE,
 )
 from .adultswim import AdultSwimIE
-from .aftenposten import AftenpostenIE
 from .aftonbladet import AftonbladetIE
 from .airmozilla import AirMozillaIE
 from .aljazeera import AlJazeeraIE
diff --git a/youtube_dl/extractor/aftenposten.py b/youtube_dl/extractor/aftenposten.py
deleted file mode 100644
index 0c00acfb5..000000000
--- a/youtube_dl/extractor/aftenposten.py
+++ /dev/null
@@ -1,23 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-from .common import InfoExtractor
-
-
-class AftenpostenIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?aftenposten\.no/webtv/(?:#!/)?video/(?P<id>\d+)'
-    _TEST = {
-        'url': 'http://www.aftenposten.no/webtv/#!/video/21039/trailer-sweatshop-i-can-t-take-any-more',
-        'md5': 'fd828cd29774a729bf4d4425fe192972',
-        'info_dict': {
-            'id': '21039',
-            'ext': 'mov',
-            'title': 'TRAILER: "Sweatshop" - I can´t take any more',
-            'description': 'md5:21891f2b0dd7ec2f78d84a50e54f8238',
-            'timestamp': 1416927969,
-            'upload_date': '20141125',
-        }
-    }
-
-    def _real_extract(self, url):
-        return self.url_result('xstream:ap:%s' % self._match_id(url), 'Xstream')
diff --git a/youtube_dl/extractor/vgtv.py b/youtube_dl/extractor/vgtv.py
index f38a72fde..17213d9b6 100644
--- a/youtube_dl/extractor/vgtv.py
+++ b/youtube_dl/extractor/vgtv.py
@@ -11,16 +11,17 @@ from ..utils import (
 
 
 class VGTVIE(InfoExtractor):
-    IE_DESC = 'VGTV and BTTV'
+    IE_DESC = 'VGTV, BTTV, FTV, Aftenposten, Aftonbladet'
     _VALID_URL = r'''(?x)
                     (?:
                         vgtv:|
                         http://(?:www\.)?
                     )
-                    (?P<host>vgtv|bt)
+                    (?P<host>vgtv.no|(?:bt.no|aftenbladet.no)/tv|fvn.no/fvntv|aftenposten.no/webtv)
                     (?:
                         :|
-                        \.no/(?:tv/)?\#!/(?:video|live)/
+                        /\#!/(?:video|live)/|
+                        /embed?id=
                     )
                     (?P<id>[0-9]+)
                     '''
@@ -59,17 +60,18 @@ class VGTVIE(InfoExtractor):
                 # m3u8 download
                 'skip_download': True,
             },
+            'skip': 'Video is no longer available',
         },
         {
-            # streamType: live
+            # streamType: wasLive
             'url': 'http://www.vgtv.no/#!/live/113063/direkte-v75-fra-solvalla',
             'info_dict': {
                 'id': '113063',
-                'ext': 'flv',
-                'title': 're:^DIREKTE: V75 fra Solvalla [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
+                'ext': 'mp4',
+                'title': 'V75 fra Solvalla 30.05.15',
                 'description': 'md5:b3743425765355855f88e096acc93231',
                 'thumbnail': 're:^https?://.*\.jpg',
-                'duration': 0,
+                'duration': 25966,
                 'timestamp': 1432975582,
                 'upload_date': '20150530',
                 'view_count': int,
@@ -78,26 +80,56 @@ class VGTVIE(InfoExtractor):
                 # m3u8 download
                 'skip_download': True,
             },
+        },{
+            'url': 'http://www.aftenposten.no/webtv/#!/video/21039/trailer-sweatshop-i-can-t-take-any-more',
+            'md5': '7fbc265a3ca4933a423c7a66aa879a67',
+            'info_dict': {
+                'id': '21039',
+                'ext': 'mp4',
+                'title': 'TRAILER: «SWEATSHOP» - I can´t take any more',
+                'description': 'md5:21891f2b0dd7ec2f78d84a50e54f8238',
+                'duration': 66,
+                'timestamp': 1417002452,
+                'upload_date': '20141126',
+                'view_count': int,
+            }
         },
         {
             'url': 'http://www.bt.no/tv/#!/video/100250/norling-dette-er-forskjellen-paa-1-divisjon-og-eliteserien',
             'only_matching': True,
         },
     ]
+    _HOST_WEBSITES = {
+        'vgtv.no': {
+            'vendor': 'vgtv',
+            'appname': 'vgtv',
+        },
+        'bt.no/tv': {
+            'vendor': 'bt',
+            'appname': 'bttv',
+        },
+        'aftenbladet.no/tv': {
+            'vendor': 'sa',
+            'appname': 'satv',
+        },
+        'fvn.no/fvntv': {
+            'vendor': 'fvn',
+            'appname': 'fvntv',
+        },
+        'aftenposten.no/webtv': {
+            'vendor': 'ap',
+            'appname': 'aptv',
+        },
+    }
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
         host = mobj.group('host')
 
-        HOST_WEBSITES = {
-            'vgtv': 'vgtv',
-            'bt': 'bttv',
-        }
-
         data = self._download_json(
             'http://svp.vg.no/svp/api/v1/%s/assets/%s?appName=%s-website'
-            % (host, video_id, HOST_WEBSITES[host]),
+            % (self._HOST_WEBSITES[host]['vendor'], video_id, self._HOST_WEBSITES[host]['appname']),
             video_id, 'Downloading media JSON')
 
         if data.get('status') == 'inactive':
@@ -144,7 +176,7 @@ class VGTVIE(InfoExtractor):
 
         return {
             'id': video_id,
-            'title': self._live_title(data['title']),
+            'title': self._live_title(data['title']) if stream_type == 'live' else data['title'],
             'description': data['description'],
             'thumbnail': data['images']['main'] + '?t[]=900x506q80',
             'timestamp': data['published'],

From 804afc5871a88eaa32a6c161df67e6b37383d7d1 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 30 Oct 2015 10:20:38 +0100
Subject: [PATCH 2933/6219] [vgtv] improve _VALID_URL regex

---
 youtube_dl/extractor/vgtv.py | 20 +++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/vgtv.py b/youtube_dl/extractor/vgtv.py
index 17213d9b6..e8039ec7f 100644
--- a/youtube_dl/extractor/vgtv.py
+++ b/youtube_dl/extractor/vgtv.py
@@ -11,17 +11,19 @@ from ..utils import (
 
 
 class VGTVIE(InfoExtractor):
-    IE_DESC = 'VGTV, BTTV, FTV, Aftenposten, Aftonbladet'
+    IE_DESC = 'VGTV, BTTV, FTV, Aftenposten and Aftonbladet'
     _VALID_URL = r'''(?x)
-                    (?:
-                        vgtv:|
-                        http://(?:www\.)?
+                    http://(?:www\.)?
+                    (?P<host>
+                        vgtv.no|
+                        (?:bt|aftenbladet).no/tv|
+                        fvn.no/fvntv|
+                        aftenposten.no/webtv
                     )
-                    (?P<host>vgtv.no|(?:bt.no|aftenbladet.no)/tv|fvn.no/fvntv|aftenposten.no/webtv)
+                    /
                     (?:
-                        :|
-                        /\#!/(?:video|live)/|
-                        /embed?id=
+                        \#!/(?:video|live)/|
+                        embed?.*id=
                     )
                     (?P<id>[0-9]+)
                     '''
@@ -211,7 +213,7 @@ class BTArticleIE(InfoExtractor):
         webpage = self._download_webpage(url, self._match_id(url))
         video_id = self._search_regex(
             r'SVP\.Player\.load\(\s*(\d+)', webpage, 'video id')
-        return self.url_result('vgtv:bt:%s' % video_id, 'VGTV')
+        return self.url_result('http://bt.no/tv/embed?id=%s' % video_id, 'VGTV')
 
 
 class BTVestlendingenIE(InfoExtractor):

From 957e0db1d2d1d4b2e9fdd1c314b0c46e68ca0cb3 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 30 Oct 2015 13:56:21 +0100
Subject: [PATCH 2934/6219] [baidu] improve info extraction

---
 youtube_dl/extractor/baidu.py | 45 +++++++++++++----------------------
 1 file changed, 16 insertions(+), 29 deletions(-)

diff --git a/youtube_dl/extractor/baidu.py b/youtube_dl/extractor/baidu.py
index e37ee4440..84fab551b 100644
--- a/youtube_dl/extractor/baidu.py
+++ b/youtube_dl/extractor/baidu.py
@@ -14,8 +14,8 @@ class BaiduVideoIE(InfoExtractor):
         'url': 'http://v.baidu.com/comic/1069.htm?frp=bdbrand&q=%E4%B8%AD%E5%8D%8E%E5%B0%8F%E5%BD%93%E5%AE%B6',
         'info_dict': {
             'id': '1069',
-            'title': '中华小当家 TV版 (全52集)',
-            'description': 'md5:395a419e41215e531c857bb037bbaf80',
+            'title': '中华小当家 TV版国语',
+            'description': 'md5:40a9c1b1c7f4e05d642e7bb1c84eeda0',
         },
         'playlist_count': 52,
     }, {
@@ -25,45 +25,32 @@ class BaiduVideoIE(InfoExtractor):
             'title': 're:^奔跑吧兄弟',
             'description': 'md5:1bf88bad6d850930f542d51547c089b8',
         },
-        'playlist_mincount': 3,
+        'playlist_mincount': 12,
     }]
 
+    def _call_api(self, path, category, playlist_id):
+        return self._download_json('http://app.video.baidu.com/%s/?worktype=adnative%s&id=%s' % (path, category, playlist_id), playlist_id)
+
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        playlist_id = mobj.group('id')
-        category = category2 = mobj.group('type')
+        category, playlist_id = re.match(self._VALID_URL, url).groups()
         if category == 'show':
-            category2 = 'tvshow'
+            category = 'tvshow'
+        if category == 'tv':
+            category = 'tvplay'
 
-        webpage = self._download_webpage(url, playlist_id)
+        playlist_detail = self._call_api('xqinfo', category, playlist_id)
 
-        playlist_title = self._html_search_regex(
-            r'title\s*:\s*(["\'])(?P<title>[^\']+)\1', webpage,
-            'playlist title', group='title')
-        playlist_description = self._html_search_regex(
-            r'<input[^>]+class="j-data-intro"[^>]+value="([^"]+)"/>', webpage,
-            playlist_id, 'playlist description')
+        playlist_title = playlist_detail['title']
+        playlist_description = playlist_detail.get('intro')
 
-        site = self._html_search_regex(
-            r'filterSite\s*:\s*["\']([^"]*)["\']', webpage,
-            'primary provider site')
-        api_result = self._download_json(
-            'http://v.baidu.com/%s_intro/?dtype=%sPlayUrl&id=%s&site=%s' % (
-                category, category2, playlist_id, site),
-            playlist_id, 'Get playlist links')
+        episodes_detail = self._call_api('xqsingle', category, playlist_id)
 
         entries = []
-        for episode in api_result[0]['episodes']:
+        for episode in episodes_detail['videos']:
             episode_id = '%s_%s' % (playlist_id, episode['episode'])
 
-            redirect_page = self._download_webpage(
-                compat_urlparse.urljoin(url, episode['url']), episode_id,
-                note='Download Baidu redirect page')
-            real_url = self._html_search_regex(
-                r'location\.replace\("([^"]+)"\)', redirect_page, 'real URL')
-
             entries.append(self.url_result(
-                real_url, video_title=episode['single_title']))
+                episode['url'], video_title=episode['title']))
 
         return self.playlist_result(
             entries, playlist_id, playlist_title, playlist_description)

From 6722ebd43720e836af8217fa078fa1a604b98229 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 30 Oct 2015 21:00:36 +0600
Subject: [PATCH 2935/6219] [anitube] Relax key regex (Closes #7303)

 Another variant seen http://anitubebr.xpg.uol.com.br/embed/
---
 youtube_dl/extractor/anitube.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/anitube.py b/youtube_dl/extractor/anitube.py
index 31f0d417c..23f942ae2 100644
--- a/youtube_dl/extractor/anitube.py
+++ b/youtube_dl/extractor/anitube.py
@@ -26,8 +26,8 @@ class AnitubeIE(InfoExtractor):
         video_id = mobj.group('id')
 
         webpage = self._download_webpage(url, video_id)
-        key = self._html_search_regex(
-            r'http://www\.anitube\.se/embed/([A-Za-z0-9_-]*)', webpage, 'key')
+        key = self._search_regex(
+            r'src=["\']https?://[^/]+/embed/([A-Za-z0-9_-]+)', webpage, 'key')
 
         config_xml = self._download_xml(
             'http://www.anitube.se/nuevo/econfig.php?key=%s' % key, key)

From 240384afe6983b71d3a6cb8891312185a15619f6 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 30 Oct 2015 20:06:38 +0100
Subject: [PATCH 2936/6219] [clipfish] improve info extraction

---
 youtube_dl/extractor/clipfish.py | 56 ++++++++++++--------------------
 1 file changed, 21 insertions(+), 35 deletions(-)

diff --git a/youtube_dl/extractor/clipfish.py b/youtube_dl/extractor/clipfish.py
index 7af903571..d142e326f 100644
--- a/youtube_dl/extractor/clipfish.py
+++ b/youtube_dl/extractor/clipfish.py
@@ -4,11 +4,8 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
-    determine_ext,
     int_or_none,
-    js_to_json,
-    parse_iso8601,
-    remove_end,
+    unified_strdate,
 )
 
 
@@ -21,48 +18,37 @@ class ClipfishIE(InfoExtractor):
             'id': '3966754',
             'ext': 'mp4',
             'title': 'FIFA 14 - E3 2013 Trailer',
-            'timestamp': 1370938118,
+            'description': 'Video zu FIFA 14: E3 2013 Trailer',
             'upload_date': '20130611',
             'duration': 82,
+            'view_count': int,
         }
     }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        webpage = self._download_webpage(url, video_id)
+        video_info = self._download_json('http://www.clipfish.de/devapi/id/%s?format=json&apikey=hbbtv' % video_id, video_id)['items'][0]
 
-        video_info = self._parse_json(
-            js_to_json(self._html_search_regex(
-                '(?s)videoObject\s*=\s*({.+?});', webpage, 'video object')),
-            video_id)
-
-        formats = []
-        for video_url in re.findall(r'var\s+videourl\s*=\s*"([^"]+)"', webpage):
-            ext = determine_ext(video_url)
-            if ext == 'm3u8':
-                formats.append({
-                    'url': video_url.replace('de.hls.fra.clipfish.de', 'hls.fra.clipfish.de'),
-                    'ext': 'mp4',
-                    'format_id': 'hls',
-                })
-            else:
-                formats.append({
-                    'url': video_url,
-                    'format_id': ext,
-                })
-        self._sort_formats(formats)
-
-        title = remove_end(self._og_search_title(webpage), ' - Video')
-        thumbnail = self._og_search_thumbnail(webpage)
-        duration = int_or_none(video_info.get('length'))
-        timestamp = parse_iso8601(self._html_search_meta('uploadDate', webpage, 'upload date'))
+        formats = [{
+            'url': video_info['media_videourl_hls'].replace('de.hls.fra.clipfish.de', 'hls.fra.clipfish.de'),
+            'ext': 'mp4',
+            'format_id': 'hls',
+        },{
+            'url': video_info['media_videourl'],
+            'format_id': 'mp4',
+            'width': int_or_none(video_info.get('width')),
+            'height': int_or_none(video_info.get('height')),
+            'tbr': int_or_none(video_info.get('bitrate')),
+        }]
 
         return {
             'id': video_id,
-            'title': title,
+            'title': video_info['title'],
+            'description': video_info.get('descr'),
             'formats': formats,
-            'thumbnail': thumbnail,
-            'duration': duration,
-            'timestamp': timestamp,
+            'thumbnail': video_info.get('media_content_thumbnail_large') or video_info.get('media_thumbnail'),
+            'duration': int_or_none(video_info.get('media_length')),
+            'upload_date': unified_strdate(video_info.get('pubDate')),
+            'view_count': int_or_none(video_info.get('media_views'))
         }

From 720334659ac13d7f5bc0759662ba0534f48a1ce4 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 31 Oct 2015 01:08:37 +0100
Subject: [PATCH 2937/6219] [daum] improve info extraction

---
 youtube_dl/extractor/__init__.py |   5 +-
 youtube_dl/extractor/daum.py     | 123 +++++++++++++++++++------------
 2 files changed, 79 insertions(+), 49 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index f98e6487e..99b4be002 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -119,7 +119,10 @@ from .dailymotion import (
     DailymotionUserIE,
     DailymotionCloudIE,
 )
-from .daum import DaumIE
+from .daum import (
+    DaumIE,
+    DaumClipIE,
+)
 from .dbtv import DBTVIE
 from .dcn import DCNIE
 from .dctp import DctpTvIE
diff --git a/youtube_dl/extractor/daum.py b/youtube_dl/extractor/daum.py
index 934da765e..0f5686e07 100644
--- a/youtube_dl/extractor/daum.py
+++ b/youtube_dl/extractor/daum.py
@@ -2,16 +2,73 @@
 
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse,
-)
+from ..compat import compat_urllib_parse
+from ..utils import int_or_none
 
 
 class DaumIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:m\.)?tvpot\.daum\.net/(?:v/|.*?clipid=)(?P<id>[^?#&]+)'
+    _VALID_URL = r'https?://(?:m\.)?tvpot\.daum\.net/v/(?P<id>[^?#&]+)'
+    IE_NAME = 'daum.net'
+
+    _TESTS = [{
+        'url': 'http://tvpot.daum.net/v/vab4dyeDBysyBssyukBUjBz',
+        'info_dict': {
+            'id': 'vab4dyeDBysyBssyukBUjBz',
+            'ext': 'mp4',
+            'title': '마크 헌트 vs 안토니오 실바',
+            'description': 'Mark Hunt vs Antonio Silva',
+            'upload_date': '20131217',
+            'duration': 2117,
+        },
+    }, {
+        'url': 'http://tvpot.daum.net/v/07dXWRka62Y%24',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        query = compat_urllib_parse.urlencode({'vid': video_id})
+        info = self._download_xml(
+            'http://tvpot.daum.net/clip/ClipInfoXml.do?' + query, video_id,
+            'Downloading video info')
+        movie_data = self._download_json(
+            'http://videofarm.daum.net/controller/api/closed/v1_2/IntegratedMovieData.json?' + query,
+            video_id, 'Downloading video formats info')
+
+        formats = []
+        for format_el in movie_data['output_list']['output_list']:
+            profile = format_el['profile']
+            format_query = compat_urllib_parse.urlencode({
+                'vid': video_id,
+                'profile': profile,
+            })
+            url_doc = self._download_xml(
+                'http://videofarm.daum.net/controller/api/open/v1_2/MovieLocation.apixml?' + format_query,
+                video_id, note='Downloading video data for %s format' % profile)
+            format_url = url_doc.find('result/url').text
+            formats.append({
+                'url': format_url,
+                'format_id': profile,
+                'width': int_or_none(format_el.get('width')),
+                'height': int_or_none(format_el.get('height')),
+                'filesize': int_or_none(format_el.get('filesize')),
+            })
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': info.find('TITLE').text,
+            'formats': formats,
+            'thumbnail': info.find('THUMB_URL').text,
+            'description': info.find('CONTENTS').text,
+            'duration': int_or_none(info.find('DURATION').text),
+            'upload_date': info.find('REGDTTM').text[:8],
+        }
+
+
+class DaumClipIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:m\.)?tvpot\.daum\.net/.*?clipid=(?P<id>\d+)'
     IE_NAME = 'daum.net'
 
     _TESTS = [{
@@ -23,53 +80,23 @@ class DaumIE(InfoExtractor):
             'description': 'DOTA 2GETHER 시즌2 6회 - 2부',
             'upload_date': '20130831',
             'duration': 3868,
+            'view_count': int,
         },
-    }, {
-        'url': 'http://tvpot.daum.net/v/vab4dyeDBysyBssyukBUjBz',
-        'only_matching': True,
-    }, {
-        'url': 'http://tvpot.daum.net/v/07dXWRka62Y%24',
-        'only_matching': True,
     }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        canonical_url = 'http://tvpot.daum.net/v/%s' % video_id
-        webpage = self._download_webpage(canonical_url, video_id)
-        full_id = self._search_regex(
-            r'src=["\']http://videofarm\.daum\.net/controller/video/viewer/Video\.html\?.*?vid=(.+?)[&"\']',
-            webpage, 'full id')
-        query = compat_urllib_parse.urlencode({'vid': full_id})
-        info = self._download_xml(
-            'http://tvpot.daum.net/clip/ClipInfoXml.do?' + query, video_id,
-            'Downloading video info')
-        urls = self._download_xml(
-            'http://videofarm.daum.net/controller/api/open/v1_2/MovieData.apixml?' + query,
-            video_id, 'Downloading video formats info')
-
-        formats = []
-        for format_el in urls.findall('result/output_list/output_list'):
-            profile = format_el.attrib['profile']
-            format_query = compat_urllib_parse.urlencode({
-                'vid': full_id,
-                'profile': profile,
-            })
-            url_doc = self._download_xml(
-                'http://videofarm.daum.net/controller/api/open/v1_2/MovieLocation.apixml?' + format_query,
-                video_id, note='Downloading video data for %s format' % profile)
-            format_url = url_doc.find('result/url').text
-            formats.append({
-                'url': format_url,
-                'format_id': profile,
-            })
+        video_id = self._match_id(url)
+        clip_info = self._download_json('http://tvpot.daum.net/mypot/json/GetClipInfo.do?clipid=%s' % video_id, video_id)['clip_bean']
 
         return {
+            '_type': 'url_transparent',
             'id': video_id,
-            'title': info.find('TITLE').text,
-            'formats': formats,
-            'thumbnail': self._og_search_thumbnail(webpage),
-            'description': info.find('CONTENTS').text,
-            'duration': int(info.find('DURATION').text),
-            'upload_date': info.find('REGDTTM').text[:8],
+            'url': 'http://tvpot.daum.net/v/%s' % clip_info['vid'],
+            'title': clip_info['title'],
+            'thumbnail': clip_info.get('thumb_url'),
+            'description': clip_info.get('contents'),
+            'duration': int_or_none(clip_info.get('duration')),
+            'upload_date': clip_info.get('up_date')[:8],
+            'view_count': int_or_none(clip_info.get('play_count')),
+            'ie_key': 'Daum',
         }

From 47f2d01a5ac074f6959aa12e8bc00310f18a54e8 Mon Sep 17 00:00:00 2001
From: Lucas <mikotosc@gmail.com>
Date: Thu, 24 Sep 2015 22:19:09 +0200
Subject: [PATCH 2938/6219] Add new extractor

---
 youtube_dl/extractor/__init__.py |   1 +
 youtube_dl/extractor/kika.py     | 115 +++++++++++++++++++++++++++++++
 2 files changed, 116 insertions(+)
 create mode 100644 youtube_dl/extractor/kika.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index f98e6487e..5ad4e9c36 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -274,6 +274,7 @@ from .karrierevideos import KarriereVideosIE
 from .keezmovies import KeezMoviesIE
 from .khanacademy import KhanAcademyIE
 from .kickstarter import KickStarterIE
+from .kika import KikaIE
 from .keek import KeekIE
 from .kontrtube import KontrTubeIE
 from .krasview import KrasViewIE
diff --git a/youtube_dl/extractor/kika.py b/youtube_dl/extractor/kika.py
new file mode 100644
index 000000000..db0f333ff
--- /dev/null
+++ b/youtube_dl/extractor/kika.py
@@ -0,0 +1,115 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import ExtractorError
+
+
+class KikaIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?kika\.de/(?:[a-z-]+/)*(?:video|sendung)(?P<id>\d+).*'
+
+    _TESTS = [
+        {
+            'url': 'http://www.kika.de/baumhaus/videos/video9572.html',
+            'md5': '94fc748cf5d64916571d275a07ffe2d5',
+            'info_dict': {
+                'id': '9572',
+                'ext': 'mp4',
+                'title': 'Baumhaus vom 29. Oktober 2014',
+                'description': None
+            }
+        },
+        {
+            'url': 'http://www.kika.de/sendungen/einzelsendungen/weihnachtsprogramm/videos/video8182.html',
+            'md5': '5fe9c4dd7d71e3b238f04b8fdd588357',
+            'info_dict': {
+                'id': '8182',
+                'ext': 'mp4',
+                'title': 'Beutolomäus und der geheime Weihnachtswunsch',
+                'description': 'md5:b69d32d7b2c55cbe86945ab309d39bbd'
+            }
+        },
+        {
+            'url': 'http://www.kika.de/videos/allevideos/video9572_zc-32ca94ad_zs-3f535991.html',
+            'md5': '94fc748cf5d64916571d275a07ffe2d5',
+            'info_dict': {
+                'id': '9572',
+                'ext': 'mp4',
+                'title': 'Baumhaus vom 29. Oktober 2014',
+                'description': None
+            }
+        },
+        {
+            'url': 'http://www.kika.de/sendungen/einzelsendungen/weihnachtsprogramm/videos/sendung81244_zc-81d703f8_zs-f82d5e31.html',
+            'md5': '5fe9c4dd7d71e3b238f04b8fdd588357',
+            'info_dict': {
+                'id': '8182',
+                'ext': 'mp4',
+                'title': 'Beutolomäus und der geheime Weihnachtswunsch',
+                'description': 'md5:b69d32d7b2c55cbe86945ab309d39bbd'
+            }
+        }
+    ]
+
+    def _real_extract(self, url):
+        # broadcast_id may be the same as the video_id
+        broadcast_id = self._match_id(url)
+        webpage = self._download_webpage(url, broadcast_id)
+
+        xml_re = r'sectionArticle[ "](?:(?!sectionA[ "])(?:.|\n))*?dataURL:\'(?:/[a-z-]+?)*?/video(\d+)-avCustom\.xml'
+        video_id = self._search_regex(xml_re, webpage, "xml_url", default=None)
+        if not video_id:
+            # Video is not available online
+            err_msg = 'Video %s is not available online' % broadcast_id
+            raise ExtractorError(err_msg, expected=True)
+
+        xml_url = 'http://www.kika.de/video%s-avCustom.xml' % (video_id)
+        xml_tree = self._download_xml(xml_url, video_id)
+
+        title = xml_tree.find('title').text
+        webpage_url = xml_tree.find('htmlUrl').text
+
+        # Try to get the description, not available for all videos
+        try:
+            broadcast_elem = xml_tree.find('broadcast')
+            description = broadcast_elem.find('broadcastDescription').text
+        except AttributeError:
+            # No description available
+            description = None
+
+        # duration string format is mm:ss (even if it is >= 1 hour, e.g. 78:42)
+        tmp = xml_tree.find('duration').text.split(':')
+        duration = int(tmp[0]) * 60 + int(tmp[1])
+
+        formats_list = []
+        for elem in xml_tree.find('assets'):
+            format_dict = {}
+            format_dict['url'] = elem.find('progressiveDownloadUrl').text
+            format_dict['ext'] = elem.find('mediaType').text.lower()
+            format_dict['format'] = elem.find('profileName').text
+            width = int(elem.find('frameWidth').text)
+            height = int(elem.find('frameHeight').text)
+            format_dict['width'] = width
+            format_dict['height'] = height
+            format_dict['resolution'] = '%dx%d' % (width, height)
+            format_dict['abr'] = int(elem.find('bitrateAudio').text)
+            format_dict['vbr'] = int(elem.find('bitrateVideo').text)
+            format_dict['tbr'] = format_dict['abr'] + format_dict['vbr']
+            format_dict['filesize'] = int(elem.find('fileSize').text)
+
+            # append resolution and dict for sorting by resolution
+            formats_list.append((width * height, format_dict))
+
+        # Sort by resolution (=quality)
+        formats_list.sort()
+
+        out_list = [x[1] for x in formats_list]
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'formats': out_list,
+            'duration': duration,
+            'webpage_url': webpage_url
+        }

From 892015b088fa21915270b0a05937fcc7063ccdd2 Mon Sep 17 00:00:00 2001
From: Lucas <mikotosc@gmail.com>
Date: Mon, 28 Sep 2015 22:00:56 +0200
Subject: [PATCH 2939/6219] replaced inefficient code

---
 youtube_dl/extractor/kika.py | 18 +++++++-----------
 1 file changed, 7 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/kika.py b/youtube_dl/extractor/kika.py
index db0f333ff..871e4ea44 100644
--- a/youtube_dl/extractor/kika.py
+++ b/youtube_dl/extractor/kika.py
@@ -87,29 +87,25 @@ class KikaIE(InfoExtractor):
             format_dict['url'] = elem.find('progressiveDownloadUrl').text
             format_dict['ext'] = elem.find('mediaType').text.lower()
             format_dict['format'] = elem.find('profileName').text
-            width = int(elem.find('frameWidth').text)
-            height = int(elem.find('frameHeight').text)
-            format_dict['width'] = width
-            format_dict['height'] = height
-            format_dict['resolution'] = '%dx%d' % (width, height)
+            format_dict['width'] = int(elem.find('frameWidth').text)
+            format_dict['height'] = int(elem.find('frameHeight').text)
+            format_dict['resolution'] = '%dx%d' % (format_dict['width'],
+                                                   format_dict['height'])
             format_dict['abr'] = int(elem.find('bitrateAudio').text)
             format_dict['vbr'] = int(elem.find('bitrateVideo').text)
             format_dict['tbr'] = format_dict['abr'] + format_dict['vbr']
             format_dict['filesize'] = int(elem.find('fileSize').text)
 
-            # append resolution and dict for sorting by resolution
-            formats_list.append((width * height, format_dict))
+            formats_list.append(format_dict)
 
         # Sort by resolution (=quality)
-        formats_list.sort()
-
-        out_list = [x[1] for x in formats_list]
+        formats_list.sort(key=lambda x: x['width'] * x['height'])
 
         return {
             'id': video_id,
             'title': title,
             'description': description,
-            'formats': out_list,
+            'formats': formats_list,
             'duration': duration,
             'webpage_url': webpage_url
         }

From 78d7ee19dc417b16b26fe2fa1101124866727a85 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 31 Oct 2015 22:21:52 +0800
Subject: [PATCH 2940/6219] [democracynow] Fix _TESTS

---
 youtube_dl/extractor/democracynow.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/democracynow.py b/youtube_dl/extractor/democracynow.py
index 973bb437b..05cfc7502 100644
--- a/youtube_dl/extractor/democracynow.py
+++ b/youtube_dl/extractor/democracynow.py
@@ -36,10 +36,9 @@ class DemocracynowIE(InfoExtractor):
         if display_id == '':
             display_id = 'home'
         webpage = self._download_webpage(url, display_id)
-        re_desc = re.search(r'<meta property=.og:description. content=(["\'])(.+?)\1', webpage, re.DOTALL)
-        description = re_desc.group(2) if re_desc else ''
+        description = self._og_search_description(webpage)
 
-        jstr = self._search_regex(r'({.+?"related_video_xml".+?})', webpage, 'json', default=None)
+        jstr = self._search_regex(r'<script[^>]+type="text/json"[^>]*>\s*({[^>]+})', webpage, 'json')
         js = self._parse_json(jstr, display_id)
         video_id = None
         formats = []
@@ -56,7 +55,7 @@ class DemocracynowIE(InfoExtractor):
                 'ext': ext,
                 'url': url,
             }]
-        for key in ('file', 'audio'):
+        for key in ('file', 'audio', 'video'):
             url = js.get(key, '')
             if url == '' or url is None:
                 continue

From 50b9dd734423231f7a01ed8a156d09ca04a23a31 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 31 Oct 2015 15:40:11 +0100
Subject: [PATCH 2941/6219] [dcn] improve season info extraction

---
 youtube_dl/extractor/dcn.py | 20 +++++++++-----------
 1 file changed, 9 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/dcn.py b/youtube_dl/extractor/dcn.py
index 8b360a9d7..a9a5e94f5 100644
--- a/youtube_dl/extractor/dcn.py
+++ b/youtube_dl/extractor/dcn.py
@@ -210,16 +210,14 @@ class DCNSeasonIE(InfoExtractor):
             })
 
         show = self._download_json(request, show_id)
-        season_id = season_id or show['default_season']
-        season = {}
-        for _ in show['seasons']:
-            if _['id'] == season_id:
-                season = _
-                break
-        title = season.get('title_en') or season['title_ar']
+        if not season_id:
+            season_id = show['default_season']
+        for season in show['seasons']:
+            if season['id'] == season_id:
+                title = season.get('title_en') or season['title_ar']
 
-        entries = []
-        for video in show['videos']:
-            entries.append(self.url_result('http://www.dcndigital.ae/#/media/%s' % video['id'], 'DCNVideo'))
+                entries = []
+                for video in show['videos']:
+                    entries.append(self.url_result('http://www.dcndigital.ae/#/media/%s' % video['id'], 'DCNVideo'))
 
-        return self.playlist_result(entries, season_id, title)
+                return self.playlist_result(entries, season_id, title)

From 8c1aa28c27af204c4996260cdc70359e83c2c3d6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 31 Oct 2015 16:14:36 +0100
Subject: [PATCH 2942/6219] [kika] Replace non working tests and recognize
 'einzelsendung' urls.

---
 youtube_dl/extractor/kika.py | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/kika.py b/youtube_dl/extractor/kika.py
index 871e4ea44..c9169076a 100644
--- a/youtube_dl/extractor/kika.py
+++ b/youtube_dl/extractor/kika.py
@@ -6,16 +6,16 @@ from ..utils import ExtractorError
 
 
 class KikaIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?kika\.de/(?:[a-z-]+/)*(?:video|sendung)(?P<id>\d+).*'
+    _VALID_URL = r'https?://(?:www\.)?kika\.de/(?:[a-z-]+/)*(?:video|(?:einzel)?sendung)(?P<id>\d+).*'
 
     _TESTS = [
         {
-            'url': 'http://www.kika.de/baumhaus/videos/video9572.html',
-            'md5': '94fc748cf5d64916571d275a07ffe2d5',
+            'url': 'http://www.kika.de/baumhaus/videos/video19636.html',
+            'md5': '4930515e36b06c111213e80d1e4aad0e',
             'info_dict': {
-                'id': '9572',
+                'id': '19636',
                 'ext': 'mp4',
-                'title': 'Baumhaus vom 29. Oktober 2014',
+                'title': 'Baumhaus vom 30. Oktober 2015',
                 'description': None
             }
         },
@@ -30,17 +30,17 @@ class KikaIE(InfoExtractor):
             }
         },
         {
-            'url': 'http://www.kika.de/videos/allevideos/video9572_zc-32ca94ad_zs-3f535991.html',
-            'md5': '94fc748cf5d64916571d275a07ffe2d5',
+            'url': 'http://www.kika.de/baumhaus/sendungen/video19636_zc-fea7f8a0_zs-4bf89c60.html',
+            'md5': '4930515e36b06c111213e80d1e4aad0e',
             'info_dict': {
-                'id': '9572',
+                'id': '19636',
                 'ext': 'mp4',
-                'title': 'Baumhaus vom 29. Oktober 2014',
+                'title': 'Baumhaus vom 30. Oktober 2015',
                 'description': None
             }
         },
         {
-            'url': 'http://www.kika.de/sendungen/einzelsendungen/weihnachtsprogramm/videos/sendung81244_zc-81d703f8_zs-f82d5e31.html',
+            'url': 'http://www.kika.de/sendungen/einzelsendungen/weihnachtsprogramm/einzelsendung2534.html',
             'md5': '5fe9c4dd7d71e3b238f04b8fdd588357',
             'info_dict': {
                 'id': '8182',

From c3040bd00a43e111dab0d1ab903df03ac7d19a00 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 31 Oct 2015 16:32:35 +0100
Subject: [PATCH 2943/6219] [kika] Cleanup

Closes #6957.
---
 youtube_dl/extractor/kika.py | 54 +++++++++++++++---------------------
 1 file changed, 22 insertions(+), 32 deletions(-)

diff --git a/youtube_dl/extractor/kika.py b/youtube_dl/extractor/kika.py
index c9169076a..5337ac439 100644
--- a/youtube_dl/extractor/kika.py
+++ b/youtube_dl/extractor/kika.py
@@ -16,8 +16,8 @@ class KikaIE(InfoExtractor):
                 'id': '19636',
                 'ext': 'mp4',
                 'title': 'Baumhaus vom 30. Oktober 2015',
-                'description': None
-            }
+                'description': None,
+            },
         },
         {
             'url': 'http://www.kika.de/sendungen/einzelsendungen/weihnachtsprogramm/videos/video8182.html',
@@ -26,8 +26,8 @@ class KikaIE(InfoExtractor):
                 'id': '8182',
                 'ext': 'mp4',
                 'title': 'Beutolomäus und der geheime Weihnachtswunsch',
-                'description': 'md5:b69d32d7b2c55cbe86945ab309d39bbd'
-            }
+                'description': 'md5:b69d32d7b2c55cbe86945ab309d39bbd',
+            },
         },
         {
             'url': 'http://www.kika.de/baumhaus/sendungen/video19636_zc-fea7f8a0_zs-4bf89c60.html',
@@ -36,8 +36,8 @@ class KikaIE(InfoExtractor):
                 'id': '19636',
                 'ext': 'mp4',
                 'title': 'Baumhaus vom 30. Oktober 2015',
-                'description': None
-            }
+                'description': None,
+            },
         },
         {
             'url': 'http://www.kika.de/sendungen/einzelsendungen/weihnachtsprogramm/einzelsendung2534.html',
@@ -46,9 +46,9 @@ class KikaIE(InfoExtractor):
                 'id': '8182',
                 'ext': 'mp4',
                 'title': 'Beutolomäus und der geheime Weihnachtswunsch',
-                'description': 'md5:b69d32d7b2c55cbe86945ab309d39bbd'
-            }
-        }
+                'description': 'md5:b69d32d7b2c55cbe86945ab309d39bbd',
+            },
+        },
     ]
 
     def _real_extract(self, url):
@@ -59,7 +59,6 @@ class KikaIE(InfoExtractor):
         xml_re = r'sectionArticle[ "](?:(?!sectionA[ "])(?:.|\n))*?dataURL:\'(?:/[a-z-]+?)*?/video(\d+)-avCustom\.xml'
         video_id = self._search_regex(xml_re, webpage, "xml_url", default=None)
         if not video_id:
-            # Video is not available online
             err_msg = 'Video %s is not available online' % broadcast_id
             raise ExtractorError(err_msg, expected=True)
 
@@ -74,38 +73,29 @@ class KikaIE(InfoExtractor):
             broadcast_elem = xml_tree.find('broadcast')
             description = broadcast_elem.find('broadcastDescription').text
         except AttributeError:
-            # No description available
             description = None
 
         # duration string format is mm:ss (even if it is >= 1 hour, e.g. 78:42)
         tmp = xml_tree.find('duration').text.split(':')
         duration = int(tmp[0]) * 60 + int(tmp[1])
 
-        formats_list = []
-        for elem in xml_tree.find('assets'):
-            format_dict = {}
-            format_dict['url'] = elem.find('progressiveDownloadUrl').text
-            format_dict['ext'] = elem.find('mediaType').text.lower()
-            format_dict['format'] = elem.find('profileName').text
-            format_dict['width'] = int(elem.find('frameWidth').text)
-            format_dict['height'] = int(elem.find('frameHeight').text)
-            format_dict['resolution'] = '%dx%d' % (format_dict['width'],
-                                                   format_dict['height'])
-            format_dict['abr'] = int(elem.find('bitrateAudio').text)
-            format_dict['vbr'] = int(elem.find('bitrateVideo').text)
-            format_dict['tbr'] = format_dict['abr'] + format_dict['vbr']
-            format_dict['filesize'] = int(elem.find('fileSize').text)
-
-            formats_list.append(format_dict)
-
-        # Sort by resolution (=quality)
-        formats_list.sort(key=lambda x: x['width'] * x['height'])
+        formats = [{
+            'url': elem.find('progressiveDownloadUrl').text,
+            'ext': elem.find('mediaType').text.lower(),
+            'format': elem.find('profileName').text,
+            'width': int(elem.find('frameWidth').text),
+            'height': int(elem.find('frameHeight').text),
+            'abr': int(elem.find('bitrateAudio').text),
+            'vbr': int(elem.find('bitrateVideo').text),
+            'filesize': int(elem.find('fileSize').text),
+        } for elem in xml_tree.find('assets')]
+        self._sort_formats(formats)
 
         return {
             'id': video_id,
             'title': title,
             'description': description,
-            'formats': formats_list,
+            'formats': formats,
             'duration': duration,
-            'webpage_url': webpage_url
+            'webpage_url': webpage_url,
         }

From 2b1b2d83cacfdce19cae5eea2f9bbfd142efc7f1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 31 Oct 2015 22:17:09 +0600
Subject: [PATCH 2944/6219] [mdr] Modernize and include kika.de

---
 youtube_dl/extractor/__init__.py |   1 -
 youtube_dl/extractor/kika.py     | 101 ------------------
 youtube_dl/extractor/mdr.py      | 172 +++++++++++++++++++++++--------
 3 files changed, 131 insertions(+), 143 deletions(-)
 delete mode 100644 youtube_dl/extractor/kika.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 5ad4e9c36..f98e6487e 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -274,7 +274,6 @@ from .karrierevideos import KarriereVideosIE
 from .keezmovies import KeezMoviesIE
 from .khanacademy import KhanAcademyIE
 from .kickstarter import KickStarterIE
-from .kika import KikaIE
 from .keek import KeekIE
 from .kontrtube import KontrTubeIE
 from .krasview import KrasViewIE
diff --git a/youtube_dl/extractor/kika.py b/youtube_dl/extractor/kika.py
deleted file mode 100644
index 5337ac439..000000000
--- a/youtube_dl/extractor/kika.py
+++ /dev/null
@@ -1,101 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-from .common import InfoExtractor
-from ..utils import ExtractorError
-
-
-class KikaIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?kika\.de/(?:[a-z-]+/)*(?:video|(?:einzel)?sendung)(?P<id>\d+).*'
-
-    _TESTS = [
-        {
-            'url': 'http://www.kika.de/baumhaus/videos/video19636.html',
-            'md5': '4930515e36b06c111213e80d1e4aad0e',
-            'info_dict': {
-                'id': '19636',
-                'ext': 'mp4',
-                'title': 'Baumhaus vom 30. Oktober 2015',
-                'description': None,
-            },
-        },
-        {
-            'url': 'http://www.kika.de/sendungen/einzelsendungen/weihnachtsprogramm/videos/video8182.html',
-            'md5': '5fe9c4dd7d71e3b238f04b8fdd588357',
-            'info_dict': {
-                'id': '8182',
-                'ext': 'mp4',
-                'title': 'Beutolomäus und der geheime Weihnachtswunsch',
-                'description': 'md5:b69d32d7b2c55cbe86945ab309d39bbd',
-            },
-        },
-        {
-            'url': 'http://www.kika.de/baumhaus/sendungen/video19636_zc-fea7f8a0_zs-4bf89c60.html',
-            'md5': '4930515e36b06c111213e80d1e4aad0e',
-            'info_dict': {
-                'id': '19636',
-                'ext': 'mp4',
-                'title': 'Baumhaus vom 30. Oktober 2015',
-                'description': None,
-            },
-        },
-        {
-            'url': 'http://www.kika.de/sendungen/einzelsendungen/weihnachtsprogramm/einzelsendung2534.html',
-            'md5': '5fe9c4dd7d71e3b238f04b8fdd588357',
-            'info_dict': {
-                'id': '8182',
-                'ext': 'mp4',
-                'title': 'Beutolomäus und der geheime Weihnachtswunsch',
-                'description': 'md5:b69d32d7b2c55cbe86945ab309d39bbd',
-            },
-        },
-    ]
-
-    def _real_extract(self, url):
-        # broadcast_id may be the same as the video_id
-        broadcast_id = self._match_id(url)
-        webpage = self._download_webpage(url, broadcast_id)
-
-        xml_re = r'sectionArticle[ "](?:(?!sectionA[ "])(?:.|\n))*?dataURL:\'(?:/[a-z-]+?)*?/video(\d+)-avCustom\.xml'
-        video_id = self._search_regex(xml_re, webpage, "xml_url", default=None)
-        if not video_id:
-            err_msg = 'Video %s is not available online' % broadcast_id
-            raise ExtractorError(err_msg, expected=True)
-
-        xml_url = 'http://www.kika.de/video%s-avCustom.xml' % (video_id)
-        xml_tree = self._download_xml(xml_url, video_id)
-
-        title = xml_tree.find('title').text
-        webpage_url = xml_tree.find('htmlUrl').text
-
-        # Try to get the description, not available for all videos
-        try:
-            broadcast_elem = xml_tree.find('broadcast')
-            description = broadcast_elem.find('broadcastDescription').text
-        except AttributeError:
-            description = None
-
-        # duration string format is mm:ss (even if it is >= 1 hour, e.g. 78:42)
-        tmp = xml_tree.find('duration').text.split(':')
-        duration = int(tmp[0]) * 60 + int(tmp[1])
-
-        formats = [{
-            'url': elem.find('progressiveDownloadUrl').text,
-            'ext': elem.find('mediaType').text.lower(),
-            'format': elem.find('profileName').text,
-            'width': int(elem.find('frameWidth').text),
-            'height': int(elem.find('frameHeight').text),
-            'abr': int(elem.find('bitrateAudio').text),
-            'vbr': int(elem.find('bitrateVideo').text),
-            'filesize': int(elem.find('fileSize').text),
-        } for elem in xml_tree.find('assets')]
-        self._sort_formats(formats)
-
-        return {
-            'id': video_id,
-            'title': title,
-            'description': description,
-            'formats': formats,
-            'duration': duration,
-            'webpage_url': webpage_url,
-        }
diff --git a/youtube_dl/extractor/mdr.py b/youtube_dl/extractor/mdr.py
index fc7499958..541ddd909 100644
--- a/youtube_dl/extractor/mdr.py
+++ b/youtube_dl/extractor/mdr.py
@@ -1,64 +1,154 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
+from ..compat import compat_urlparse
+from ..utils import (
+    determine_ext,
+    int_or_none,
+    parse_duration,
+    parse_iso8601,
+    xpath_text,
+)
 
 
 class MDRIE(InfoExtractor):
-    _VALID_URL = r'^(?P<domain>https?://(?:www\.)?mdr\.de)/(?:.*)/(?P<type>video|audio)(?P<video_id>[^/_]+)(?:_|\.html)'
+    IE_DESC = 'MDR.DE and KiKA'
+    _VALID_URL = r'https?://(?:www\.)?(?:mdr|kika)\.de/(?:.*)/[a-z]+(?P<id>\d+)(?:_.+?)?\.html'
 
-    # No tests, MDR regularily deletes its videos
-    _TEST = {
+    _TESTS = [{
+        # MDR regularily deletes its videos
         'url': 'http://www.mdr.de/fakt/video189002.html',
         'only_matching': True,
-    }
+    }, {
+        'url': 'http://www.kika.de/baumhaus/videos/video19636.html',
+        'md5': '4930515e36b06c111213e80d1e4aad0e',
+        'info_dict': {
+            'id': '19636',
+            'ext': 'mp4',
+            'title': 'Baumhaus vom 30. Oktober 2015',
+            'duration': 134,
+            'uploader': 'KIKA',
+        },
+    }, {
+        'url': 'http://www.kika.de/sendungen/einzelsendungen/weihnachtsprogramm/videos/video8182.html',
+        'md5': '5fe9c4dd7d71e3b238f04b8fdd588357',
+        'info_dict': {
+            'id': '8182',
+            'ext': 'mp4',
+            'title': 'Beutolomäus und der geheime Weihnachtswunsch',
+            'description': 'md5:b69d32d7b2c55cbe86945ab309d39bbd',
+            'timestamp': 1419047100,
+            'upload_date': '20141220',
+            'duration': 4628,
+            'uploader': 'KIKA',
+        },
+    }, {
+        'url': 'http://www.kika.de/baumhaus/sendungen/video19636_zc-fea7f8a0_zs-4bf89c60.html',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.kika.de/sendungen/einzelsendungen/weihnachtsprogramm/einzelsendung2534.html',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
-        m = re.match(self._VALID_URL, url)
-        video_id = m.group('video_id')
-        domain = m.group('domain')
+        video_id = self._match_id(url)
 
-        # determine title and media streams from webpage
-        html = self._download_webpage(url, video_id)
+        webpage = self._download_webpage(url, video_id)
 
-        title = self._html_search_regex(r'<h[12]>(.*?)</h[12]>', html, 'title')
-        xmlurl = self._search_regex(
-            r'dataURL:\'(/(?:.+)/(?:video|audio)[0-9]+-avCustom.xml)', html, 'XML URL')
+        data_url = self._search_regex(
+            r'dataURL\s*:\s*(["\'])(?P<url>/.+/(?:video|audio)[0-9]+-avCustom\.xml)\1',
+            webpage, 'data url', group='url')
+
+        doc = self._download_xml(
+            compat_urlparse.urljoin(url, data_url), video_id)
+
+        title = (xpath_text(doc, './title', 'title', default=None) or
+                 xpath_text(doc, './broadcast/broadcastName', 'title'))
 
-        doc = self._download_xml(domain + xmlurl, video_id)
         formats = []
-        for a in doc.findall('./assets/asset'):
-            url_el = a.find('./progressiveDownloadUrl')
-            if url_el is None:
-                continue
-            abr = int(a.find('bitrateAudio').text) // 1000
-            media_type = a.find('mediaType').text
-            format = {
-                'abr': abr,
-                'filesize': int(a.find('fileSize').text),
-                'url': url_el.text,
-            }
+        processed_urls = []
+        for asset in doc.findall('./assets/asset'):
+            for source in (
+                    'progressiveDownload',
+                    'dynamicHttpStreamingRedirector',
+                    'adaptiveHttpStreamingRedirector'):
+                url_el = asset.find('./%sUrl' % source)
+                if url_el is None:
+                    continue
 
-            vbr_el = a.find('bitrateVideo')
-            if vbr_el is None:
-                format.update({
-                    'vcodec': 'none',
-                    'format_id': '%s-%d' % (media_type, abr),
-                })
-            else:
-                vbr = int(vbr_el.text) // 1000
-                format.update({
-                    'vbr': vbr,
-                    'width': int(a.find('frameWidth').text),
-                    'height': int(a.find('frameHeight').text),
-                    'format_id': '%s-%d' % (media_type, vbr),
-                })
-            formats.append(format)
+                video_url = url_el.text
+                if video_url in processed_urls:
+                    continue
+
+                processed_urls.append(video_url)
+
+                vbr = int_or_none(xpath_text(asset, './bitrateVideo', 'vbr'), 1000)
+                abr = int_or_none(xpath_text(asset, './bitrateAudio', 'abr'), 1000)
+
+                url_formats = []
+
+                ext = determine_ext(url_el.text)
+                if ext == 'm3u8':
+                    url_formats = self._extract_m3u8_formats(
+                        video_url, video_id, 'mp4', entry_protocol='m3u8_native',
+                        preference=0, m3u8_id='HLS', fatal=False)
+                elif ext == 'f4m':
+                    url_formats = self._extract_f4m_formats(
+                        video_url + '?hdcore=3.7.0&plugin=aasp-3.7.0.39.44', video_id,
+                        preference=0, f4m_id='HDS', fatal=False)
+                else:
+                    media_type = xpath_text(asset, './mediaType', 'media type', default='MP4')
+                    vbr = int_or_none(xpath_text(asset, './bitrateVideo', 'vbr'), 1000)
+                    abr = int_or_none(xpath_text(asset, './bitrateAudio', 'abr'), 1000)
+                    filesize = int_or_none(xpath_text(asset, './fileSize', 'file size'))
+
+                    f = {
+                        'url': video_url,
+                        'format_id': '%s-%d' % (media_type, vbr or abr),
+                        'filesize': filesize,
+                        'abr': abr,
+                        'preference': 1,
+                    }
+
+                    if vbr:
+                        width = int_or_none(xpath_text(asset, './frameWidth', 'width'))
+                        height = int_or_none(xpath_text(asset, './frameHeight', 'height'))
+                        f.update({
+                            'vbr': vbr,
+                            'width': width,
+                            'height': height,
+                        })
+
+                    url_formats.append(f)
+
+                if not vbr:
+                    for f in url_formats:
+                        abr = f.get('tbr') or abr
+                        if 'tbr' in f:
+                            del f['tbr']
+                        f.update({
+                            'abr': abr,
+                            'vcodec': 'none',
+                        })
+
+                if url_formats:
+                    formats.extend(url_formats)
         self._sort_formats(formats)
 
+        description = xpath_text(doc, './broadcast/broadcastDescription', 'description')
+        timestamp = parse_iso8601(
+            xpath_text(doc, './broadcast/broadcastDate', 'timestamp', default=None) or
+            xpath_text(doc, './broadcast/broadcastStartDate', 'timestamp', default=None))
+        duration = parse_duration(xpath_text(doc, './duration', 'duration'))
+        uploader = xpath_text(doc, './rights', 'uploader')
+
         return {
             'id': video_id,
             'title': title,
+            'description': description,
+            'timestamp': timestamp,
+            'duration': duration,
+            'uploader': uploader,
             'formats': formats,
         }

From 8cdb5c845336ad3dc48c85a0558a38bd42972b00 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 31 Oct 2015 22:24:21 +0600
Subject: [PATCH 2945/6219] [mdr] Add audio test

---
 youtube_dl/extractor/mdr.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/youtube_dl/extractor/mdr.py b/youtube_dl/extractor/mdr.py
index 541ddd909..e05577496 100644
--- a/youtube_dl/extractor/mdr.py
+++ b/youtube_dl/extractor/mdr.py
@@ -20,6 +20,17 @@ class MDRIE(InfoExtractor):
         # MDR regularily deletes its videos
         'url': 'http://www.mdr.de/fakt/video189002.html',
         'only_matching': True,
+    },  {
+        # audio
+        'url': 'http://www.mdr.de/kultur/audio1312272_zc-15948bad_zs-86171fdd.html',
+        'md5': '64c4ee50f0a791deb9479cd7bbe9d2fa',
+        'info_dict': {
+            'id': '1312272',
+            'ext': 'mp3',
+            'title': 'Feuilleton vom 30. Oktober 2015',
+            'duration': 250,
+            'uploader': 'MITTELDEUTSCHER RUNDFUNK',
+        },
     }, {
         'url': 'http://www.kika.de/baumhaus/videos/video19636.html',
         'md5': '4930515e36b06c111213e80d1e4aad0e',

From 578c074575f45ffdfd032d7b84f6fe449614f511 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 31 Oct 2015 22:39:44 +0600
Subject: [PATCH 2946/6219] [utils] Support list of xpath in xpath_element

---
 test/test_utils.py  |  7 +++++++
 youtube_dl/utils.py | 15 ++++++++++++---
 2 files changed, 19 insertions(+), 3 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 0c34f0e55..5a56ad776 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -275,9 +275,16 @@ class TestUtil(unittest.TestCase):
         p = xml.etree.ElementTree.SubElement(div, 'p')
         p.text = 'Foo'
         self.assertEqual(xpath_element(doc, 'div/p'), p)
+        self.assertEqual(xpath_element(doc, ['div/p']), p)
+        self.assertEqual(xpath_element(doc, ['div/bar', 'div/p']), p)
         self.assertEqual(xpath_element(doc, 'div/bar', default='default'), 'default')
+        self.assertEqual(xpath_element(doc, ['div/bar'], default='default'), 'default')
         self.assertTrue(xpath_element(doc, 'div/bar') is None)
+        self.assertTrue(xpath_element(doc, ['div/bar']) is None)
+        self.assertTrue(xpath_element(doc, ['div/bar'], 'div/baz') is None)
         self.assertRaises(ExtractorError, xpath_element, doc, 'div/bar', fatal=True)
+        self.assertRaises(ExtractorError, xpath_element, doc, ['div/bar'], fatal=True)
+        self.assertRaises(ExtractorError, xpath_element, doc, ['div/bar', 'div/baz'], fatal=True)
 
     def test_xpath_text(self):
         testxml = '''<root>
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 558c9c7d5..89c88a4d3 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -178,10 +178,19 @@ def xpath_with_ns(path, ns_map):
 
 
 def xpath_element(node, xpath, name=None, fatal=False, default=NO_DEFAULT):
-    if sys.version_info < (2, 7):  # Crazy 2.6
-        xpath = xpath.encode('ascii')
+    def _find_xpath(xpath):
+        if sys.version_info < (2, 7):  # Crazy 2.6
+            xpath = xpath.encode('ascii')
+        return node.find(xpath)
+
+    if isinstance(xpath, (str, compat_str)):
+        n = _find_xpath(xpath)
+    else:
+        for xp in xpath:
+            n = _find_xpath(xp)
+            if n is not None:
+                break
 
-    n = node.find(xpath)
     if n is None:
         if default is not NO_DEFAULT:
             return default

From 11465da70257663ee52c7be50debe1c1e825ec67 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 31 Oct 2015 22:45:45 +0600
Subject: [PATCH 2947/6219] [mdr] Simplify xpath

---
 youtube_dl/extractor/mdr.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/mdr.py b/youtube_dl/extractor/mdr.py
index e05577496..a63257c56 100644
--- a/youtube_dl/extractor/mdr.py
+++ b/youtube_dl/extractor/mdr.py
@@ -74,8 +74,7 @@ class MDRIE(InfoExtractor):
         doc = self._download_xml(
             compat_urlparse.urljoin(url, data_url), video_id)
 
-        title = (xpath_text(doc, './title', 'title', default=None) or
-                 xpath_text(doc, './broadcast/broadcastName', 'title'))
+        title = xpath_text(doc, ['./title', './broadcast/broadcastName'], 'title', fatal=True)
 
         formats = []
         processed_urls = []
@@ -149,8 +148,12 @@ class MDRIE(InfoExtractor):
 
         description = xpath_text(doc, './broadcast/broadcastDescription', 'description')
         timestamp = parse_iso8601(
-            xpath_text(doc, './broadcast/broadcastDate', 'timestamp', default=None) or
-            xpath_text(doc, './broadcast/broadcastStartDate', 'timestamp', default=None))
+            xpath_text(
+                doc, [
+                    './broadcast/broadcastDate',
+                    './broadcast/broadcastStartDate',
+                    './broadcast/broadcastEndDate'],
+                'timestamp', default=None))
         duration = parse_duration(xpath_text(doc, './duration', 'duration'))
         uploader = xpath_text(doc, './rights', 'uploader')
 

From 82b69a5cbb1455d31916be4f19ab327ae63f313c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 31 Oct 2015 23:00:36 +0600
Subject: [PATCH 2948/6219] [mdr] PEP 8

---
 youtube_dl/extractor/mdr.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mdr.py b/youtube_dl/extractor/mdr.py
index a63257c56..a566c6a2c 100644
--- a/youtube_dl/extractor/mdr.py
+++ b/youtube_dl/extractor/mdr.py
@@ -20,7 +20,7 @@ class MDRIE(InfoExtractor):
         # MDR regularily deletes its videos
         'url': 'http://www.mdr.de/fakt/video189002.html',
         'only_matching': True,
-    },  {
+    }, {
         # audio
         'url': 'http://www.mdr.de/kultur/audio1312272_zc-15948bad_zs-86171fdd.html',
         'md5': '64c4ee50f0a791deb9479cd7bbe9d2fa',

From e327b736ca6a6a1c880b93e09a3b310c354c2c7c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 31 Oct 2015 23:05:30 +0600
Subject: [PATCH 2949/6219] [generic] Update test

---
 youtube_dl/extractor/generic.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index ca5fbafb2..a84135032 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -141,6 +141,7 @@ class GenericIE(InfoExtractor):
                 'ext': 'mp4',
                 'title': 'Automatics, robotics and biocybernetics',
                 'description': 'md5:815fc1deb6b3a2bff99de2d5325be482',
+                'upload_date': '20130627',
                 'formats': 'mincount:16',
                 'subtitles': 'mincount:1',
             },

From ae12bc3ebb4cb377c2b4337ec255e652b36f5143 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 31 Oct 2015 23:07:37 +0600
Subject: [PATCH 2950/6219] [utils] Make unified_strdate always return unicode
 string

---
 youtube_dl/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 89c88a4d3..764a89cca 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -910,7 +910,7 @@ def unified_strdate(date_str, day_first=True):
         timetuple = email.utils.parsedate_tz(date_str)
         if timetuple:
             upload_date = datetime.datetime(*timetuple[:6]).strftime('%Y%m%d')
-    return upload_date
+    return compat_str(upload_date)
 
 
 def determine_ext(url, default_ext='unknown_video'):

From dc519b5421366a8cac681455a817ae25f7f4aa83 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 31 Oct 2015 23:12:57 +0600
Subject: [PATCH 2951/6219] [extractor/common] Make ie_key and IE_NAME return
 unicode string

---
 youtube_dl/extractor/common.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 10c0d5d1f..59c3fa8dc 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -310,11 +310,11 @@ class InfoExtractor(object):
     @classmethod
     def ie_key(cls):
         """A string for getting the InfoExtractor with get_info_extractor"""
-        return cls.__name__[:-2]
+        return compat_str(cls.__name__[:-2])
 
     @property
     def IE_NAME(self):
-        return type(self).__name__[:-2]
+        return compat_str(type(self).__name__[:-2])
 
     def _request_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True):
         """ Returns the response handle """

From 76f0c50d3d3e2eb5903b61da08829699e902916d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 1 Nov 2015 00:01:08 +0600
Subject: [PATCH 2952/6219] [mdr] Fix failed formats processing

---
 youtube_dl/extractor/mdr.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/mdr.py b/youtube_dl/extractor/mdr.py
index a566c6a2c..88334889e 100644
--- a/youtube_dl/extractor/mdr.py
+++ b/youtube_dl/extractor/mdr.py
@@ -96,8 +96,6 @@ class MDRIE(InfoExtractor):
                 vbr = int_or_none(xpath_text(asset, './bitrateVideo', 'vbr'), 1000)
                 abr = int_or_none(xpath_text(asset, './bitrateAudio', 'abr'), 1000)
 
-                url_formats = []
-
                 ext = determine_ext(url_el.text)
                 if ext == 'm3u8':
                     url_formats = self._extract_m3u8_formats(
@@ -130,7 +128,10 @@ class MDRIE(InfoExtractor):
                             'height': height,
                         })
 
-                    url_formats.append(f)
+                    url_formats = [f]
+
+                if not url_formats:
+                    continue
 
                 if not vbr:
                     for f in url_formats:
@@ -142,8 +143,8 @@ class MDRIE(InfoExtractor):
                             'vcodec': 'none',
                         })
 
-                if url_formats:
-                    formats.extend(url_formats)
+                formats.extend(url_formats)
+
         self._sort_formats(formats)
 
         description = xpath_text(doc, './broadcast/broadcastDescription', 'description')

From dbd82a1d4fff1655920e111cc25a7fd526d7bf9a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 1 Nov 2015 00:01:34 +0600
Subject: [PATCH 2953/6219] [extractor/common] Fix m3u8 extraction on failure

---
 youtube_dl/extractor/common.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 59c3fa8dc..1f09fbb47 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -943,13 +943,14 @@ class InfoExtractor(object):
             if re.match(r'^https?://', u)
             else compat_urlparse.urljoin(m3u8_url, u))
 
-        m3u8_doc, urlh = self._download_webpage_handle(
+        res = self._download_webpage_handle(
             m3u8_url, video_id,
             note=note or 'Downloading m3u8 information',
             errnote=errnote or 'Failed to download m3u8 information',
             fatal=fatal)
-        if m3u8_doc is False:
-            return m3u8_doc
+        if res is False:
+            return res
+        m3u8_doc, urlh = res
         m3u8_url = urlh.geturl()
         last_info = None
         last_media = None

From 9550ca506fccf9c9d795816cc0a7817ff262ef45 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 31 Oct 2015 19:36:04 +0100
Subject: [PATCH 2954/6219] [utils] change extract_attributes to work in python
 2

---
 youtube_dl/extractor/brightcove.py | 3 +--
 youtube_dl/utils.py                | 3 ++-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index b41cee91b..c6ad1d065 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -383,8 +383,7 @@ class BrightcoveInPageEmbedIE(InfoExtractor):
         return None
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        account_id, player_id, embed, video_id = mobj.groups()
+        account_id, player_id, embed, video_id = re.match(self._VALID_URL, url).groups()
 
         webpage = self._download_webpage('http://players.brightcove.net/%s/%s_%s/index.min.js' % (account_id, player_id, embed), video_id)
 
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index bcebf9cc5..518cea98b 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -252,7 +252,8 @@ def extract_attributes(attributes_str, attributes_regex=r'(?s)\s*([^\s=]+)\s*=\s
     attributes = re.findall(attributes_regex, attributes_str)
     attributes_dict = {}
     if attributes:
-        attributes_dict = {attribute_name: attribute_value for (attribute_name, attribute_value) in attributes}
+        for (attribute_name, attribute_value) in attributes:
+            attributes_dict[attribute_name] = attribute_value
     return attributes_dict
 
 

From 80dcee5cd5cbe623a53e0c582e3e3ae170c63e8d Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 31 Oct 2015 04:02:49 +0100
Subject: [PATCH 2955/6219] [eitb] fix info extraction

---
 youtube_dl/extractor/eitb.py | 65 ++++++++++++++++++++++++------------
 1 file changed, 44 insertions(+), 21 deletions(-)

diff --git a/youtube_dl/extractor/eitb.py b/youtube_dl/extractor/eitb.py
index 2cba82532..fc8f15544 100644
--- a/youtube_dl/extractor/eitb.py
+++ b/youtube_dl/extractor/eitb.py
@@ -1,39 +1,62 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
-from .brightcove import BrightcoveIE
-from ..utils import ExtractorError
+from ..compat import compat_urllib_request
+from ..utils import (
+    int_or_none,
+    unified_strdate,
+)
 
 
 class EitbIE(InfoExtractor):
     IE_NAME = 'eitb.tv'
-    _VALID_URL = r'https?://www\.eitb\.tv/(eu/bideoa|es/video)/[^/]+/(?P<playlist_id>\d+)/(?P<chapter_id>\d+)'
+    _VALID_URL = r'https?://www\.eitb\.tv/(eu/bideoa|es/video)/[^/]+/\d+/(?P<id>\d+)'
 
     _TEST = {
-        'add_ie': ['Brightcove'],
-        'url': 'http://www.eitb.tv/es/video/60-minutos-60-minutos-2013-2014/2677100210001/2743577154001/lasa-y-zabala-30-anos/',
+        'url': 'http://www.eitb.tv/es/video/60-minutos-60-minutos-2013-2014/4104995148001/4090227752001/lasa-y-zabala-30-anos/',
         'md5': 'edf4436247185adee3ea18ce64c47998',
         'info_dict': {
-            'id': '2743577154001',
+            'id': '4090227752001',
             'ext': 'mp4',
             'title': '60 minutos (Lasa y Zabala, 30 años)',
-            # All videos from eitb has this description in the brightcove info
-            'description': '.',
-            'uploader': 'Euskal Telebista',
+            'description': '',
+            'duration': 3996760,
+            'upload_date': '20131014',
         },
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        chapter_id = mobj.group('chapter_id')
-        webpage = self._download_webpage(url, chapter_id)
-        bc_url = BrightcoveIE._extract_brightcove_url(webpage)
-        if bc_url is None:
-            raise ExtractorError('Could not extract the Brightcove url')
-        # The BrightcoveExperience object doesn't contain the video id, we set
-        # it manually
-        bc_url += '&%40videoPlayer={0}'.format(chapter_id)
-        return self.url_result(bc_url, BrightcoveIE.ie_key())
+        video_id = self._match_id(url)
+        video_data = self._download_json('http://mam.eitb.eus/mam/REST/ServiceMultiweb/Video/MULTIWEBTV/%s/' % video_id, video_id)['web_media'][0]
+
+        formats = []
+        for rendition in video_data['RENDITIONS']:
+            formats.append({
+                'url': rendition['PMD_URL'],
+                'width': int_or_none(rendition.get('FRAME_WIDTH')),
+                'height': int_or_none(rendition.get('FRAME_HEIGHT')),
+                'tbr': int_or_none(rendition.get('ENCODING_RATE')),
+            })
+
+        # TODO: parse f4m manifest
+        request = compat_urllib_request.Request(
+            'http://mam.eitb.eus/mam/REST/ServiceMultiweb/DomainRestrictedSecurity/TokenAuth/',
+            headers={'Referer': url})
+        token_data = self._download_json(request, video_id, fatal=False)
+        if token_data:
+            m3u8_formats = self._extract_m3u8_formats('%s?hdnts=%s' % (video_data['HLS_SURL'], token_data['token']), video_id, m3u8_id='hls', fatal=False)
+            if m3u8_formats:
+                formats.extend(m3u8_formats)
+
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': video_data['NAME_ES'],
+            'description': video_data.get('SHORT_DESC_ES'),
+            'thumbnail': video_data.get('STILL_URL'),
+            'duration': int_or_none(video_data.get('LENGTH')),
+            'upload_date': unified_strdate(video_data.get('BROADCST_DATE')),
+            'formats': formats,
+        }

From 8a06999ba0f9c948f8d2a1ef89c73eedbfb09cfb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 1 Nov 2015 01:52:33 +0600
Subject: [PATCH 2956/6219] [eitb] Improve, make more robust and extract f4m
 formats (Closes #7328)

---
 youtube_dl/extractor/eitb.py | 71 +++++++++++++++++++++++++-----------
 1 file changed, 50 insertions(+), 21 deletions(-)

diff --git a/youtube_dl/extractor/eitb.py b/youtube_dl/extractor/eitb.py
index fc8f15544..0de8d3dc6 100644
--- a/youtube_dl/extractor/eitb.py
+++ b/youtube_dl/extractor/eitb.py
@@ -4,14 +4,15 @@ from __future__ import unicode_literals
 from .common import InfoExtractor
 from ..compat import compat_urllib_request
 from ..utils import (
+    float_or_none,
     int_or_none,
-    unified_strdate,
+    parse_iso8601,
 )
 
 
 class EitbIE(InfoExtractor):
     IE_NAME = 'eitb.tv'
-    _VALID_URL = r'https?://www\.eitb\.tv/(eu/bideoa|es/video)/[^/]+/\d+/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?eitb\.tv/(?:eu/bideoa|es/video)/[^/]+/\d+/(?P<id>\d+)'
 
     _TEST = {
         'url': 'http://www.eitb.tv/es/video/60-minutos-60-minutos-2013-2014/4104995148001/4090227752001/lasa-y-zabala-30-anos/',
@@ -20,43 +21,71 @@ class EitbIE(InfoExtractor):
             'id': '4090227752001',
             'ext': 'mp4',
             'title': '60 minutos (Lasa y Zabala, 30 años)',
-            'description': '',
-            'duration': 3996760,
+            'description': 'Programa de reportajes de actualidad.',
+            'duration': 3996.76,
+            'timestamp': 1381789200,
             'upload_date': '20131014',
+            'tags': list,
         },
     }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        video_data = self._download_json('http://mam.eitb.eus/mam/REST/ServiceMultiweb/Video/MULTIWEBTV/%s/' % video_id, video_id)['web_media'][0]
+
+        video = self._download_json(
+            'http://mam.eitb.eus/mam/REST/ServiceMultiweb/Video/MULTIWEBTV/%s/' % video_id,
+            video_id, 'Downloading video JSON')
+
+        media = video['web_media'][0]
 
         formats = []
-        for rendition in video_data['RENDITIONS']:
+        for rendition in media['RENDITIONS']:
+            video_url = rendition.get('PMD_URL')
+            if not video_url:
+                continue
+            tbr = float_or_none(rendition.get('ENCODING_RATE'), 1000)
+            format_id = 'http'
+            if tbr:
+                format_id += '-%d' % int(tbr)
             formats.append({
                 'url': rendition['PMD_URL'],
+                'format_id': format_id,
                 'width': int_or_none(rendition.get('FRAME_WIDTH')),
                 'height': int_or_none(rendition.get('FRAME_HEIGHT')),
-                'tbr': int_or_none(rendition.get('ENCODING_RATE')),
+                'tbr': tbr,
             })
 
-        # TODO: parse f4m manifest
-        request = compat_urllib_request.Request(
-            'http://mam.eitb.eus/mam/REST/ServiceMultiweb/DomainRestrictedSecurity/TokenAuth/',
-            headers={'Referer': url})
-        token_data = self._download_json(request, video_id, fatal=False)
-        if token_data:
-            m3u8_formats = self._extract_m3u8_formats('%s?hdnts=%s' % (video_data['HLS_SURL'], token_data['token']), video_id, m3u8_id='hls', fatal=False)
-            if m3u8_formats:
-                formats.extend(m3u8_formats)
+        hls_url = media.get('HLS_SURL')
+        if hls_url:
+            request = compat_urllib_request.Request(
+                'http://mam.eitb.eus/mam/REST/ServiceMultiweb/DomainRestrictedSecurity/TokenAuth/',
+                headers={'Referer': url})
+            token_data = self._download_json(
+                request, video_id, 'Downloading auth token', fatal=False)
+            if token_data:
+                token = token_data.get('token')
+                if token:
+                    m3u8_formats = self._extract_m3u8_formats(
+                        '%s?hdnts=%s' % (hls_url, token), video_id, m3u8_id='hls', fatal=False)
+                    if m3u8_formats:
+                        formats.extend(m3u8_formats)
+
+        hds_url = media.get('HDS_SURL').replace('euskalsvod', 'euskalvod')
+        if hds_url:
+            f4m_formats = self._extract_f4m_formats(
+                '%s?hdcore=3.7.0' % hds_url, video_id, f4m_id='hds', fatal=False)
+            if f4m_formats:
+                formats.extend(f4m_formats)
 
         self._sort_formats(formats)
 
         return {
             'id': video_id,
-            'title': video_data['NAME_ES'],
-            'description': video_data.get('SHORT_DESC_ES'),
-            'thumbnail': video_data.get('STILL_URL'),
-            'duration': int_or_none(video_data.get('LENGTH')),
-            'upload_date': unified_strdate(video_data.get('BROADCST_DATE')),
+            'title': media.get('NAME_ES') or media.get('name') or media['NAME_EU'],
+            'description': media.get('SHORT_DESC_ES') or video.get('desc_group') or media.get('SHORT_DESC_EU'),
+            'thumbnail': media.get('STILL_URL') or media.get('THUMBNAIL_URL'),
+            'duration': float_or_none(media.get('LENGTH'), 1000),
+            'timestamp': parse_iso8601(media.get('BROADCST_DATE'), ' '),
+            'tags': media.get('TAGS'),
             'formats': formats,
         }

From 02fb9804513ce1bfe28ec7c285526db7989e5844 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 1 Nov 2015 02:08:19 +0100
Subject: [PATCH 2957/6219] [flickr] extract more info and formats

---
 youtube_dl/extractor/flickr.py | 110 ++++++++++++++++++---------------
 1 file changed, 60 insertions(+), 50 deletions(-)

diff --git a/youtube_dl/extractor/flickr.py b/youtube_dl/extractor/flickr.py
index 2fe76d661..5ca754105 100644
--- a/youtube_dl/extractor/flickr.py
+++ b/youtube_dl/extractor/flickr.py
@@ -1,67 +1,77 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
-from ..compat import compat_urllib_request
+from ..compat import compat_urllib_parse
 from ..utils import (
-    ExtractorError,
-    find_xpath_attr,
+    int_or_none,
+    qualities,
 )
 
 
 class FlickrIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.|secure\.)?flickr\.com/photos/(?P<uploader_id>[\w\-_@]+)/(?P<id>\d+).*'
+    _VALID_URL = r'https?://(?:www\.|secure\.)?flickr\.com/photos/[\w\-_@]+/(?P<id>\d+)'
     _TEST = {
         'url': 'http://www.flickr.com/photos/forestwander-nature-pictures/5645318632/in/photostream/',
-        'md5': '6fdc01adbc89d72fc9c4f15b4a4ba87b',
+        'md5': '164fe3fa6c22e18d448d4d5af2330f31',
         'info_dict': {
             'id': '5645318632',
-            'ext': 'mp4',
-            "description": "Waterfalls in the Springtime at Dark Hollow Waterfalls. These are located just off of Skyline Drive in Virginia. They are only about 6/10 of a mile hike but it is a pretty steep hill and a good climb back up.",
-            "uploader_id": "forestwander-nature-pictures",
-            "title": "Dark Hollow Waterfalls"
+            'ext': 'mpg',
+            'description': 'Waterfalls in the Springtime at Dark Hollow Waterfalls. These are located just off of Skyline Drive in Virginia. They are only about 6/10 of a mile hike but it is a pretty steep hill and a good climb back up.',
+            'uploader_id': 'forestwander-nature-pictures',
+            'title': 'Dark Hollow Waterfalls',
+            'duration': 19,
+            'timestamp': 1303528740,
+            'upload_date': '20110423',
+            'uploader_id': '10922353@N03',
+            'uploader': 'Forest Wander',
+            'comment_count': int,
         }
     }
 
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
+    _API_BASE_URL = 'https://api.flickr.com/services/rest?'
+    _API_KEY = '61b16865f916058e63580a912d9143be'
 
-        video_id = mobj.group('id')
-        video_uploader_id = mobj.group('uploader_id')
-        webpage_url = 'http://www.flickr.com/photos/' + video_uploader_id + '/' + video_id
-        req = compat_urllib_request.Request(webpage_url)
-        req.add_header(
-            'User-Agent',
-            # it needs a more recent version
-            'Mozilla/5.0 (X11; Linux x86_64; rv:38.0) Gecko/20150101 Firefox/38.0 (Chrome)')
-        webpage = self._download_webpage(req, video_id)
-
-        secret = self._search_regex(r'secret"\s*:\s*"(\w+)"', webpage, 'secret')
-
-        first_url = 'https://secure.flickr.com/apps/video/video_mtl_xml.gne?v=x&photo_id=' + video_id + '&secret=' + secret + '&bitrate=700&target=_self'
-        first_xml = self._download_xml(first_url, video_id, 'Downloading first data webpage')
-
-        node_id = find_xpath_attr(
-            first_xml, './/{http://video.yahoo.com/YEP/1.0/}Item', 'id',
-            'id').text
-
-        second_url = 'https://secure.flickr.com/video_playlist.gne?node_id=' + node_id + '&tech=flash&mode=playlist&bitrate=700&secret=' + secret + '&rd=video.yahoo.com&noad=1'
-        second_xml = self._download_xml(second_url, video_id, 'Downloading second data webpage')
-
-        self.report_extraction(video_id)
-
-        stream = second_xml.find('.//STREAM')
-        if stream is None:
-            raise ExtractorError('Unable to extract video url')
-        video_url = stream.attrib['APP'] + stream.attrib['FULLPATH']
-
-        return {
-            'id': video_id,
-            'url': video_url,
-            'ext': 'mp4',
-            'title': self._og_search_title(webpage),
-            'description': self._og_search_description(webpage),
-            'thumbnail': self._og_search_thumbnail(webpage),
-            'uploader_id': video_uploader_id,
+    def _call_api(self, method, video_id, secret=None):
+        query = {
+            'photo_id': video_id,
+            'method': 'flickr.%s' % method,
+            'api_key': self._API_KEY,
+            'format': 'json',
+            'nojsoncallback': 1,
         }
+        if secret:
+            query['secret'] = secret
+        return self._download_json(self._API_BASE_URL + compat_urllib_parse.urlencode(query), video_id)
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        video_info = self._call_api('photos.getInfo', video_id)['photo']
+        if video_info['media'] == 'video':
+            streams = self._call_api('video.getStreamInfo', video_id, video_info['secret'])['streams']
+
+            preference = qualities(['iphone_wifi', '700', 'appletv', 'orig'])
+
+            formats = []
+            for stream in streams['stream']:
+                stream_type = str(stream.get('type'))
+                formats.append({
+                    'format_id': stream_type,
+                    'url': stream['_content'],
+                    'preference': preference(stream_type),
+                })
+            self._sort_formats(formats)
+
+            owner = video_info.get('owner', {})
+
+            return {
+                'id': video_id,
+                'title': video_info['title']['_content'],
+                'description': video_info.get('description', {}).get('_content'),
+                'formats': formats,
+                'timestamp': int_or_none(video_info.get('dateuploaded')),
+                'duration': int_or_none(video_info.get('video', {}).get('duration')),
+                'uploader_id': owner.get('nsid'),
+                'uploader': owner.get('realname'),
+                'comment_count': int_or_none(video_info.get('comments', {}).get('_content')),
+            }

From 999079b4543b4cd5e71a235865fbfefd349eb064 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 1 Nov 2015 15:49:11 +0600
Subject: [PATCH 2958/6219] [eitb] Improve hds extraction

---
 youtube_dl/extractor/eitb.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/eitb.py b/youtube_dl/extractor/eitb.py
index 0de8d3dc6..357a2196c 100644
--- a/youtube_dl/extractor/eitb.py
+++ b/youtube_dl/extractor/eitb.py
@@ -70,10 +70,11 @@ class EitbIE(InfoExtractor):
                     if m3u8_formats:
                         formats.extend(m3u8_formats)
 
-        hds_url = media.get('HDS_SURL').replace('euskalsvod', 'euskalvod')
+        hds_url = media.get('HDS_SURL')
         if hds_url:
             f4m_formats = self._extract_f4m_formats(
-                '%s?hdcore=3.7.0' % hds_url, video_id, f4m_id='hds', fatal=False)
+                '%s?hdcore=3.7.0' % hds_url.replace('euskalsvod', 'euskalvod'),
+                video_id, f4m_id='hds', fatal=False)
             if f4m_formats:
                 formats.extend(f4m_formats)
 

From 146672254e409bf97c82a302095fbfabf2c48928 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 1 Nov 2015 13:23:23 +0100
Subject: [PATCH 2959/6219] [flickr] extract fresh api key and remove
 duplication in test

---
 youtube_dl/extractor/flickr.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/flickr.py b/youtube_dl/extractor/flickr.py
index 5ca754105..0d5d6b0b9 100644
--- a/youtube_dl/extractor/flickr.py
+++ b/youtube_dl/extractor/flickr.py
@@ -17,7 +17,6 @@ class FlickrIE(InfoExtractor):
             'id': '5645318632',
             'ext': 'mpg',
             'description': 'Waterfalls in the Springtime at Dark Hollow Waterfalls. These are located just off of Skyline Drive in Virginia. They are only about 6/10 of a mile hike but it is a pretty steep hill and a good climb back up.',
-            'uploader_id': 'forestwander-nature-pictures',
             'title': 'Dark Hollow Waterfalls',
             'duration': 19,
             'timestamp': 1303528740,
@@ -29,26 +28,27 @@ class FlickrIE(InfoExtractor):
     }
 
     _API_BASE_URL = 'https://api.flickr.com/services/rest?'
-    _API_KEY = '61b16865f916058e63580a912d9143be'
 
-    def _call_api(self, method, video_id, secret=None):
+    def _call_api(self, method, video_id, api_key, note, secret=None):
         query = {
             'photo_id': video_id,
             'method': 'flickr.%s' % method,
-            'api_key': self._API_KEY,
+            'api_key': api_key,
             'format': 'json',
             'nojsoncallback': 1,
         }
         if secret:
             query['secret'] = secret
-        return self._download_json(self._API_BASE_URL + compat_urllib_parse.urlencode(query), video_id)
+        return self._download_json(self._API_BASE_URL + compat_urllib_parse.urlencode(query), video_id, note)
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        video_info = self._call_api('photos.getInfo', video_id)['photo']
+        api_key = self._download_json('https://www.flickr.com/hermes_error_beacon.gne', video_id, 'Downloading api key',)['site_key']
+
+        video_info = self._call_api('photos.getInfo', video_id, api_key, 'Downloading video info')['photo']
         if video_info['media'] == 'video':
-            streams = self._call_api('video.getStreamInfo', video_id, video_info['secret'])['streams']
+            streams = self._call_api('video.getStreamInfo', video_id, api_key, 'Downloading streams info', video_info['secret'])['streams']
 
             preference = qualities(['iphone_wifi', '700', 'appletv', 'orig'])
 

From f3003531a5622cc01501325b9f35dcb2424cfb70 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 1 Nov 2015 13:38:11 +0100
Subject: [PATCH 2960/6219] [flickr] handle error message

---
 youtube_dl/extractor/flickr.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/flickr.py b/youtube_dl/extractor/flickr.py
index 0d5d6b0b9..e97754d36 100644
--- a/youtube_dl/extractor/flickr.py
+++ b/youtube_dl/extractor/flickr.py
@@ -3,6 +3,7 @@ from __future__ import unicode_literals
 from .common import InfoExtractor
 from ..compat import compat_urllib_parse
 from ..utils import (
+    ExtractorError,
     int_or_none,
     qualities,
 )
@@ -39,7 +40,10 @@ class FlickrIE(InfoExtractor):
         }
         if secret:
             query['secret'] = secret
-        return self._download_json(self._API_BASE_URL + compat_urllib_parse.urlencode(query), video_id, note)
+        data = self._download_json(self._API_BASE_URL + compat_urllib_parse.urlencode(query), video_id, note)
+        if data['stat'] != 'ok':
+            raise ExtractorError(data['message'])
+        return data
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From ab6ca0480280abb2a35a54e1b380bbae07a48863 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Nov 2015 14:20:10 +0100
Subject: [PATCH 2961/6219] release 2015.11.01

---
 docs/supportedsites.md | 3 ++-
 youtube_dl/version.py  | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 03561b87d..805af14a0 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -93,6 +93,7 @@
  - **Clipsyndicate**
  - **Cloudy**
  - **Clubic**
+ - **Clyp**
  - **cmt.com**
  - **CNET**
  - **CNN**
@@ -281,7 +282,7 @@
  - **macgamestore**: MacGameStore trailers
  - **mailru**: Видео@Mail.Ru
  - **Malemotion**
- - **MDR**
+ - **MDR**: MDR.DE and KiKA
  - **media.ccc.de**
  - **metacafe**
  - **Metacritic**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 125e8ccf5..006b973c0 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.10.24'
+__version__ = '2015.11.01'

From c90d16cf36d8edf03f4dc923ee9dbeadca910844 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 2 Nov 2015 04:26:20 +0600
Subject: [PATCH 2962/6219] [utils:sanitize_path] Disallow trailing whitespace
 in path segment (Closes #7332)

---
 youtube_dl/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index efd5f4ae1..7b3f79141 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -366,7 +366,7 @@ def sanitize_path(s):
     if drive_or_unc:
         norm_path.pop(0)
     sanitized_path = [
-        path_part if path_part in ['.', '..'] else re.sub('(?:[/<>:"\\|\\\\?\\*]|\.$)', '#', path_part)
+        path_part if path_part in ['.', '..'] else re.sub('(?:[/<>:"\\|\\\\?\\*]|[\s.]$)', '#', path_part)
         for path_part in norm_path]
     if drive_or_unc:
         sanitized_path.insert(0, drive_or_unc + os.path.sep)

From eb97f46e8bd9cb04f0fe5f8a5c13aeeaabeefef5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 2 Nov 2015 12:46:10 +0100
Subject: [PATCH 2963/6219] [mitele] Fix extraction and update test checksum
 (fixes #7343)

---
 youtube_dl/extractor/mitele.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/mitele.py b/youtube_dl/extractor/mitele.py
index 3142fcde2..c595f2077 100644
--- a/youtube_dl/extractor/mitele.py
+++ b/youtube_dl/extractor/mitele.py
@@ -1,7 +1,10 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse
+from ..compat import (
+    compat_urllib_parse,
+    compat_urlparse,
+)
 from ..utils import (
     encode_dict,
     get_element_by_attribute,
@@ -15,7 +18,7 @@ class MiTeleIE(InfoExtractor):
 
     _TESTS = [{
         'url': 'http://www.mitele.es/programas-tv/diario-de/la-redaccion/programa-144/',
-        'md5': '757b0b66cbd7e0a97226d7d3156cb3e9',
+        'md5': '0ff1a13aebb35d9bc14081ff633dd324',
         'info_dict': {
             'id': '0NF1jJnxS1Wu3pHrmvFyw2',
             'display_id': 'programa-144',
@@ -34,6 +37,7 @@ class MiTeleIE(InfoExtractor):
 
         config_url = self._search_regex(
             r'data-config\s*=\s*"([^"]+)"', webpage, 'data config url')
+        config_url = compat_urlparse.urljoin(url, config_url)
 
         config = self._download_json(
             config_url, display_id, 'Downloading config JSON')

From c514b0ec655b23e7804eb18df04daa863d973f32 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 1 Nov 2015 22:12:20 +0100
Subject: [PATCH 2964/6219] [videofy.me] fix info extraction

Closes #7339.
---
 youtube_dl/extractor/videofyme.py | 40 ++++++++++++++++---------------
 1 file changed, 21 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/extractor/videofyme.py b/youtube_dl/extractor/videofyme.py
index 94f9e9be9..cd3f50a63 100644
--- a/youtube_dl/extractor/videofyme.py
+++ b/youtube_dl/extractor/videofyme.py
@@ -2,8 +2,8 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..utils import (
-    find_xpath_attr,
     int_or_none,
+    parse_iso8601,
 )
 
 
@@ -18,33 +18,35 @@ class VideofyMeIE(InfoExtractor):
             'id': '1100701',
             'ext': 'mp4',
             'title': 'This is VideofyMe',
-            'description': None,
+            'description': '',
+            'upload_date': '20130326',
+            'timestamp': 1364288959,
             'uploader': 'VideofyMe',
             'uploader_id': 'thisisvideofyme',
             'view_count': int,
+            'likes': int,
+            'comment_count': int,
         },
-
     }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        config = self._download_xml('http://sunshine.videofy.me/?videoId=%s' % video_id,
-                                    video_id)
-        video = config.find('video')
-        sources = video.find('sources')
-        url_node = next(node for node in [find_xpath_attr(sources, 'source', 'id', 'HQ %s' % key)
-                                          for key in ['on', 'av', 'off']] if node is not None)
-        video_url = url_node.find('url').text
-        view_count = int_or_none(self._search_regex(
-            r'([0-9]+)', video.find('views').text, 'view count', fatal=False))
+
+        config = self._download_json('http://vf-player-info-loader.herokuapp.com/%s.json' % video_id, video_id)['videoinfo']
+
+        video = config.get('video')
+        blog = config.get('blog', {})
 
         return {
             'id': video_id,
-            'title': video.find('title').text,
-            'url': video_url,
-            'thumbnail': video.find('thumb').text,
-            'description': video.find('description').text,
-            'uploader': config.find('blog/name').text,
-            'uploader_id': video.find('identifier').text,
-            'view_count': view_count,
+            'title': video['title'],
+            'url': video['sources']['source']['url'],
+            'thumbnail': video.get('thumb'),
+            'description': video.get('description'),
+            'timestamp': parse_iso8601(video.get('date')),
+            'uploader': blog.get('name'),
+            'uploader_id': blog.get('identifier'),
+            'view_count': int_or_none(self._search_regex(r'([0-9]+)', video.get('views'), 'view count', fatal=False)),
+            'likes': int_or_none(video.get('likes')),
+            'comment_count': int_or_none(video.get('nrOfComments')),
         }

From 6a750402787dfc1f39a9ad347f2d78ae1c94c52c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 2 Nov 2015 14:08:38 +0100
Subject: [PATCH 2965/6219] [utils] unified_strdate: Return None if the date
 format can't be recognized (fixes #7340)

This issue was introduced with ae12bc3ebb4cb377c2b4337ec255e652b36f5143, it returned 'None'.
---
 test/test_utils.py  | 1 +
 youtube_dl/utils.py | 3 ++-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 3298315d2..01829f71e 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -236,6 +236,7 @@ class TestUtil(unittest.TestCase):
             unified_strdate('2/2/2015 6:47:40 PM', day_first=False),
             '20150202')
         self.assertEqual(unified_strdate('25-09-2014'), '20140925')
+        self.assertEqual(unified_strdate('UNKNOWN DATE FORMAT'), None)
 
     def test_find_xpath_attr(self):
         testxml = '''<root>
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 7b3f79141..d39f313a4 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -911,7 +911,8 @@ def unified_strdate(date_str, day_first=True):
         timetuple = email.utils.parsedate_tz(date_str)
         if timetuple:
             upload_date = datetime.datetime(*timetuple[:6]).strftime('%Y%m%d')
-    return compat_str(upload_date)
+    if upload_date is not None:
+        return compat_str(upload_date)
 
 
 def determine_ext(url, default_ext='unknown_video'):

From a230068ff7427c19e29331fc0f2bb17d50003bca Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 2 Nov 2015 16:18:54 +0100
Subject: [PATCH 2966/6219] release 2015.11.02

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 006b973c0..6ef482b78 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.11.01'
+__version__ = '2015.11.02'

From dde9fe9788f23f168e0bddaf8ab0470f469165fa Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 3 Nov 2015 18:36:54 +0800
Subject: [PATCH 2967/6219] [democracynow] Simplify

---
 youtube_dl/extractor/democracynow.py | 86 ++++++++++++++--------------
 1 file changed, 44 insertions(+), 42 deletions(-)

diff --git a/youtube_dl/extractor/democracynow.py b/youtube_dl/extractor/democracynow.py
index 05cfc7502..824b8e2c5 100644
--- a/youtube_dl/extractor/democracynow.py
+++ b/youtube_dl/extractor/democracynow.py
@@ -2,11 +2,18 @@
 from __future__ import unicode_literals
 
 import re
+import os.path
+
 from .common import InfoExtractor
+from ..compat import compat_urlparse
+from ..utils import (
+    url_basename,
+    remove_start,
+)
 
 
 class DemocracynowIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?democracynow.org/?(?P<id>[^\?]*)'
+    _VALID_URL = r'https?://(?:www\.)?democracynow.org/(?P<id>[^\?]*)'
     IE_NAME = 'democracynow'
     _TESTS = [{
         'url': 'http://www.democracynow.org/shows/2015/7/3',
@@ -14,9 +21,7 @@ class DemocracynowIE(InfoExtractor):
             'id': '2015-0703-001',
             'ext': 'mp4',
             'title': 'July 03, 2015 - Democracy Now!',
-            'description': 'A daily independent global news hour with Amy Goodman & Juan Gonz\xe1lez "What to the Slave is 4th of July?": James Earl Jones Reads Frederick Douglass\u2019 Historic Speech : "This Flag Comes Down Today": Bree Newsome Scales SC Capitol Flagpole, Takes Down Confederate Flag : "We Shall Overcome": Remembering Folk Icon, Activist Pete Seeger in His Own Words & Songs',
-            'uploader': 'Democracy Now',
-            'upload_date': None,
+            'description': 'A daily independent global news hour with Amy Goodman & Juan González "What to the Slave is 4th of July?": James Earl Jones Reads Frederick Douglass\u2019 Historic Speech : "This Flag Comes Down Today": Bree Newsome Scales SC Capitol Flagpole, Takes Down Confederate Flag : "We Shall Overcome": Remembering Folk Icon, Activist Pete Seeger in His Own Words & Songs',
         },
     }, {
         'url': 'http://www.democracynow.org/2015/7/3/this_flag_comes_down_today_bree',
@@ -25,60 +30,57 @@ class DemocracynowIE(InfoExtractor):
             'ext': 'mp4',
             'title': '"This Flag Comes Down Today": Bree Newsome Scales SC Capitol Flagpole, Takes Down Confederate Flag',
             'description': 'md5:4d2bc4f0d29f5553c2210a4bc7761a21',
-            'uploader': 'Democracy Now',
-            'upload_date': None,
         },
     }]
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
-        base_host = re.search(r'^(.+?://[^/]+)', url).group(1)
-        if display_id == '':
-            display_id = 'home'
         webpage = self._download_webpage(url, display_id)
         description = self._og_search_description(webpage)
 
-        jstr = self._search_regex(r'<script[^>]+type="text/json"[^>]*>\s*({[^>]+})', webpage, 'json')
-        js = self._parse_json(jstr, display_id)
+        js = self._parse_json(self._search_regex(
+            r'<script[^>]+type="text/json"[^>]*>\s*({[^>]+})', webpage, 'json'),
+            display_id)
         video_id = None
         formats = []
+
+        default_lang = 'en'
+
         subtitles = {}
-        for key in ('caption_file', '.......'):
-            # ....... = pending vtt support that doesn't clobber srt 'chapter_file':
-            url = js.get(key, '')
-            if url == '' or url is None:
-                continue
-            if not re.match(r'^https?://', url):
-                url = base_host + url
-            ext = re.search(r'\.([^\.]+)$', url).group(1)
-            subtitles['eng'] = [{
-                'ext': ext,
-                'url': url,
-            }]
-        for key in ('file', 'audio', 'video'):
-            url = js.get(key, '')
-            if url == '' or url is None:
-                continue
-            if not re.match(r'^https?://', url):
-                url = base_host + url
-            purl = re.search(r'/(?P<dir>[^/]+)/(?:dn)?(?P<fn>[^/]+?)\.(?P<ext>[^\.\?]+)(?P<hasparams>\?|$)', url)
-            if video_id is None:
-                video_id = purl.group('fn')
-            if js.get('start') is not None:
-                url += '&' if purl.group('hasparams') == '?' else '?'
-                url = url + 'start=' + str(js.get('start'))
-            formats.append({
-                'format_id': purl.group('dir'),
-                'ext': purl.group('ext'),
-                'url': url,
+
+        def add_subtitle_item(lang, info_dict):
+            if lang not in subtitles:
+                subtitles[lang] = []
+            subtitles[lang].append(info_dict)
+
+        # chapter_file are not subtitles
+        if 'caption_file' in js:
+            add_subtitle_item(default_lang, {
+                'url': compat_urlparse.urljoin(url, js['caption_file']),
             })
+
+        for subtitle_item in js.get('captions', []):
+            lang = subtitle_item.get('language', '').lower() or default_lang
+            add_subtitle_item(lang, {
+                'url': compat_urlparse.urljoin(url, subtitle_item['url']),
+            })
+
+        for key in ('file', 'audio', 'video'):
+            media_url = js.get(key, '')
+            if not media_url:
+                continue
+            media_url = re.sub(r'\?.*', '', compat_urlparse.urljoin(url, media_url))
+            video_id = video_id or remove_start(os.path.splitext(url_basename(media_url))[0], 'dn')
+            formats.append({
+                'url': media_url,
+            })
+
         self._sort_formats(formats)
-        ret = {
+
+        return {
             'id': video_id,
             'title': js.get('title'),
             'description': description,
-            'uploader': 'Democracy Now',
             'subtitles': subtitles,
             'formats': formats,
         }
-        return ret

From fc68d52bb95dc81ed3d05a5c5397cd3f35ee093a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 3 Nov 2015 21:24:10 +0800
Subject: [PATCH 2968/6219] [democracynow] Add MD5 sums

---
 youtube_dl/extractor/democracynow.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/democracynow.py b/youtube_dl/extractor/democracynow.py
index 824b8e2c5..70c364e8b 100644
--- a/youtube_dl/extractor/democracynow.py
+++ b/youtube_dl/extractor/democracynow.py
@@ -17,6 +17,7 @@ class DemocracynowIE(InfoExtractor):
     IE_NAME = 'democracynow'
     _TESTS = [{
         'url': 'http://www.democracynow.org/shows/2015/7/3',
+        'md5': 'fbb8fe3d7a56a5e12431ce2f9b2fab0d',
         'info_dict': {
             'id': '2015-0703-001',
             'ext': 'mp4',
@@ -25,6 +26,7 @@ class DemocracynowIE(InfoExtractor):
         },
     }, {
         'url': 'http://www.democracynow.org/2015/7/3/this_flag_comes_down_today_bree',
+        'md5': 'fbb8fe3d7a56a5e12431ce2f9b2fab0d',
         'info_dict': {
             'id': '2015-0703-001',
             'ext': 'mp4',

From 852fad922ffa931b3c90b0b9fdb2fa1c7f965ab4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 3 Nov 2015 20:53:17 +0600
Subject: [PATCH 2969/6219] [vimeo] Fix non-ASCII video passwords (Closes
 #7352)

---
 youtube_dl/extractor/vimeo.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 2437ae1eb..cc0d337e8 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -13,6 +13,7 @@ from ..compat import (
     compat_urlparse,
 )
 from ..utils import (
+    encode_dict,
     ExtractorError,
     InAdvancePagedList,
     int_or_none,
@@ -208,10 +209,10 @@ class VimeoIE(VimeoBaseInfoExtractor):
         if password is None:
             raise ExtractorError('This video is protected by a password, use the --video-password option', expected=True)
         token, vuid = self._extract_xsrft_and_vuid(webpage)
-        data = urlencode_postdata({
+        data = urlencode_postdata(encode_dict({
             'password': password,
             'token': token,
-        })
+        }))
         if url.startswith('http://'):
             # vimeo only supports https now, but the user can give an http url
             url = url.replace('http://', 'https://')

From 0a0110fc6bbd21850e25541fd0bd4b602ce194e2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 3 Nov 2015 21:01:09 +0600
Subject: [PATCH 2970/6219] [vimeo] Fix non-ASCII video passwords (2)

---
 youtube_dl/extractor/vimeo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index cc0d337e8..fa07bd59c 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -228,7 +228,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
         password = self._downloader.params.get('videopassword', None)
         if password is None:
             raise ExtractorError('This video is protected by a password, use the --video-password option')
-        data = compat_urllib_parse.urlencode({'password': password})
+        data = urlencode_postdata(encode_dict({'password': password}))
         pass_url = url + '/check-password'
         password_request = compat_urllib_request.Request(pass_url, data)
         password_request.add_header('Content-Type', 'application/x-www-form-urlencoded')

From 3fa3ff1bc36aaf82ac0a5e880304cb7aae217b9a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 3 Nov 2015 21:06:36 +0600
Subject: [PATCH 2971/6219] [vimeo] Fix non-ASCII login

---
 youtube_dl/extractor/vimeo.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index fa07bd59c..46fb36f21 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -41,13 +41,13 @@ class VimeoBaseInfoExtractor(InfoExtractor):
         self.report_login()
         webpage = self._download_webpage(self._LOGIN_URL, None, False)
         token, vuid = self._extract_xsrft_and_vuid(webpage)
-        data = urlencode_postdata({
+        data = urlencode_postdata(encode_dict({
             'action': 'login',
             'email': username,
             'password': password,
             'service': 'vimeo',
             'token': token,
-        })
+        }))
         login_request = compat_urllib_request.Request(self._LOGIN_URL, data)
         login_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
         login_request.add_header('Cookie', 'vuid=%s' % vuid)

From bfdf891fd36811909aa5d83dc0614eacbb634fcf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 3 Nov 2015 21:09:24 +0600
Subject: [PATCH 2972/6219] [vimeo] Fix non-ASCII album passwords

---
 youtube_dl/extractor/vimeo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 46fb36f21..b608740b8 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -489,7 +489,7 @@ class VimeoChannelIE(VimeoBaseInfoExtractor):
         token, vuid = self._extract_xsrft_and_vuid(webpage)
         fields['token'] = token
         fields['password'] = password
-        post = urlencode_postdata(fields)
+        post = urlencode_postdata(encode_dict(fields))
         password_path = self._search_regex(
             r'action="([^"]+)"', login_form, 'password URL')
         password_url = compat_urlparse.urljoin(page_url, password_path)

From fd8102820c4d14fdb1ff7e090553211717012f67 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 4 Nov 2015 00:09:55 +0800
Subject: [PATCH 2973/6219] [democracynow] Rename js to json_data

---
 youtube_dl/extractor/democracynow.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/democracynow.py b/youtube_dl/extractor/democracynow.py
index 70c364e8b..72fc75d80 100644
--- a/youtube_dl/extractor/democracynow.py
+++ b/youtube_dl/extractor/democracynow.py
@@ -40,7 +40,7 @@ class DemocracynowIE(InfoExtractor):
         webpage = self._download_webpage(url, display_id)
         description = self._og_search_description(webpage)
 
-        js = self._parse_json(self._search_regex(
+        json_data = self._parse_json(self._search_regex(
             r'<script[^>]+type="text/json"[^>]*>\s*({[^>]+})', webpage, 'json'),
             display_id)
         video_id = None
@@ -56,19 +56,19 @@ class DemocracynowIE(InfoExtractor):
             subtitles[lang].append(info_dict)
 
         # chapter_file are not subtitles
-        if 'caption_file' in js:
+        if 'caption_file' in json_data:
             add_subtitle_item(default_lang, {
-                'url': compat_urlparse.urljoin(url, js['caption_file']),
+                'url': compat_urlparse.urljoin(url, json_data['caption_file']),
             })
 
-        for subtitle_item in js.get('captions', []):
+        for subtitle_item in json_data.get('captions', []):
             lang = subtitle_item.get('language', '').lower() or default_lang
             add_subtitle_item(lang, {
                 'url': compat_urlparse.urljoin(url, subtitle_item['url']),
             })
 
         for key in ('file', 'audio', 'video'):
-            media_url = js.get(key, '')
+            media_url = json_data.get(key, '')
             if not media_url:
                 continue
             media_url = re.sub(r'\?.*', '', compat_urlparse.urljoin(url, media_url))
@@ -81,7 +81,7 @@ class DemocracynowIE(InfoExtractor):
 
         return {
             'id': video_id,
-            'title': js.get('title'),
+            'title': json_data.get('title'),
             'description': description,
             'subtitles': subtitles,
             'formats': formats,

From 0aeb9a106e1aad37967e0ee666ed816a7d5eb7c2 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 4 Nov 2015 00:13:00 +0800
Subject: [PATCH 2974/6219] [democracynow] Prevent required fields to be None

---
 youtube_dl/extractor/democracynow.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/democracynow.py b/youtube_dl/extractor/democracynow.py
index 72fc75d80..6cd395e11 100644
--- a/youtube_dl/extractor/democracynow.py
+++ b/youtube_dl/extractor/democracynow.py
@@ -80,8 +80,8 @@ class DemocracynowIE(InfoExtractor):
         self._sort_formats(formats)
 
         return {
-            'id': video_id,
-            'title': json_data.get('title'),
+            'id': video_id or display_id,
+            'title': json_data['title'],
             'description': description,
             'subtitles': subtitles,
             'formats': formats,

From 66d041f250f7d3e0c4d501e3b98721f2c6588c35 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 4 Nov 2015 00:53:30 +0800
Subject: [PATCH 2975/6219] [test/subtitles] Add test for DemocracynowIE

---
 test/test_subtitles.py | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/test/test_subtitles.py b/test/test_subtitles.py
index 0343967d9..75f0ea75f 100644
--- a/test/test_subtitles.py
+++ b/test/test_subtitles.py
@@ -28,6 +28,7 @@ from youtube_dl.extractor import (
     ThePlatformFeedIE,
     RTVEALaCartaIE,
     FunnyOrDieIE,
+    DemocracynowIE,
 )
 
 
@@ -346,5 +347,25 @@ class TestFunnyOrDieSubtitles(BaseTestSubtitles):
         self.assertEqual(md5(subtitles['en']), 'c5593c193eacd353596c11c2d4f9ecc4')
 
 
+class TestDemocracynowSubtitles(BaseTestSubtitles):
+    url = 'http://www.democracynow.org/shows/2015/7/3'
+    IE = DemocracynowIE
+
+    def test_allsubtitles(self):
+        self.DL.params['writesubtitles'] = True
+        self.DL.params['allsubtitles'] = True
+        subtitles = self.getSubtitles()
+        self.assertEqual(set(subtitles.keys()), set(['en']))
+        self.assertEqual(md5(subtitles['en']), 'acaca989e24a9e45a6719c9b3d60815c')
+
+    def test_subtitles_in_page(self):
+        self.url = 'http://www.democracynow.org/2015/7/3/this_flag_comes_down_today_bree'
+        self.DL.params['writesubtitles'] = True
+        self.DL.params['allsubtitles'] = True
+        subtitles = self.getSubtitles()
+        self.assertEqual(set(subtitles.keys()), set(['en']))
+        self.assertEqual(md5(subtitles['en']), 'acaca989e24a9e45a6719c9b3d60815c')
+
+
 if __name__ == '__main__':
     unittest.main()

From ad607563a2fbb5275ea39f7a052c09ffa232e271 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 4 Nov 2015 16:46:26 +0600
Subject: [PATCH 2976/6219] [globo] Separate article extractor

---
 youtube_dl/extractor/__init__.py |   5 +-
 youtube_dl/extractor/globo.py    | 140 +++++++++++++++++--------------
 2 files changed, 79 insertions(+), 66 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 10286aa88..94150a28f 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -212,7 +212,10 @@ from .gfycat import GfycatIE
 from .giantbomb import GiantBombIE
 from .giga import GigaIE
 from .glide import GlideIE
-from .globo import GloboIE
+from .globo import (
+    GloboIE,
+    GloboArticleIE,
+)
 from .godtube import GodTubeIE
 from .goldenmoustache import GoldenMoustacheIE
 from .golem import GolemIE
diff --git a/youtube_dl/extractor/globo.py b/youtube_dl/extractor/globo.py
index 33d6432a6..828e40d76 100644
--- a/youtube_dl/extractor/globo.py
+++ b/youtube_dl/extractor/globo.py
@@ -18,75 +18,52 @@ from ..utils import (
 
 
 class GloboIE(InfoExtractor):
-    _VALID_URL = 'https?://.+?\.globo\.com/(?P<id>.+)'
+    _VALID_URL = '(?:globo:|https?://.+?\.globo\.com/(?:[^/]+/)*(?:v/(?:[^/]+/)?|videos/))(?P<id>\d{7,})'
 
     _API_URL_TEMPLATE = 'http://api.globovideos.com/videos/%s/playlist'
     _SECURITY_URL_TEMPLATE = 'http://security.video.globo.com/videos/%s/hash?player=flash&version=17.0.0.132&resource_id=%s'
 
-    _VIDEOID_REGEXES = [
-        r'\bdata-video-id="(\d+)"',
-        r'\bdata-player-videosids="(\d+)"',
-        r'<div[^>]+\bid="(\d+)"',
-    ]
-
     _RESIGN_EXPIRATION = 86400
 
-    _TESTS = [
-        {
-            'url': 'http://globotv.globo.com/sportv/futebol-nacional/v/os-gols-de-atletico-mg-3-x-2-santos-pela-24a-rodada-do-brasileirao/3654973/',
-            'md5': '03ebf41cb7ade43581608b7d9b71fab0',
-            'info_dict': {
-                'id': '3654973',
-                'ext': 'mp4',
-                'title': 'Os gols de Atlético-MG 3 x 2 Santos pela 24ª rodada do Brasileirão',
-                'duration': 251.585,
-                'uploader': 'SporTV',
-                'uploader_id': 698,
-                'like_count': int,
-            }
-        },
-        {
-            'url': 'http://g1.globo.com/carros/autoesporte/videos/t/exclusivos-do-g1/v/mercedes-benz-gla-passa-por-teste-de-colisao-na-europa/3607726/',
-            'md5': 'b3ccc801f75cd04a914d51dadb83a78d',
-            'info_dict': {
-                'id': '3607726',
-                'ext': 'mp4',
-                'title': 'Mercedes-Benz GLA passa por teste de colisão na Europa',
-                'duration': 103.204,
-                'uploader': 'Globo.com',
-                'uploader_id': 265,
-                'like_count': int,
-            }
-        },
-        {
-            'url': 'http://g1.globo.com/jornal-nacional/noticia/2014/09/novidade-na-fiscalizacao-de-bagagem-pela-receita-provoca-discussoes.html',
-            'md5': '307fdeae4390ccfe6ba1aa198cf6e72b',
-            'info_dict': {
-                'id': '3652183',
-                'ext': 'mp4',
-                'title': 'Receita Federal explica como vai fiscalizar bagagens de quem retorna ao Brasil de avião',
-                'duration': 110.711,
-                'uploader': 'Rede Globo',
-                'uploader_id': 196,
-                'like_count': int,
-            }
-        },
-        {
-            'url': 'http://globotv.globo.com/canal-brasil/sangue-latino/t/todos-os-videos/v/ator-e-diretor-argentino-ricado-darin-fala-sobre-utopias-e-suas-perdas/3928201/',
-            'md5': 'c1defca721ce25b2354e927d3e4b3dec',
-            'info_dict': {
-                'id': '3928201',
-                'ext': 'mp4',
-                'title': 'Ator e diretor argentino, Ricado Darín fala sobre utopias e suas perdas',
-                'duration': 1472.906,
-                'uploader': 'Canal Brasil',
-                'uploader_id': 705,
-                'like_count': int,
-            }
-        },
-    ]
+    _TESTS = [{
+        'url': 'http://globotv.globo.com/sportv/futebol-nacional/v/os-gols-de-atletico-mg-3-x-2-santos-pela-24a-rodada-do-brasileirao/3654973/',
+        'md5': '03ebf41cb7ade43581608b7d9b71fab0',
+        'info_dict': {
+            'id': '3654973',
+            'ext': 'mp4',
+            'title': 'Os gols de Atlético-MG 3 x 2 Santos pela 24ª rodada do Brasileirão',
+            'duration': 251.585,
+            'uploader': 'SporTV',
+            'uploader_id': 698,
+            'like_count': int,
+        }
+    }, {
+        'url': 'http://g1.globo.com/carros/autoesporte/videos/t/exclusivos-do-g1/v/mercedes-benz-gla-passa-por-teste-de-colisao-na-europa/3607726/',
+        'md5': 'b3ccc801f75cd04a914d51dadb83a78d',
+        'info_dict': {
+            'id': '3607726',
+            'ext': 'mp4',
+            'title': 'Mercedes-Benz GLA passa por teste de colisão na Europa',
+            'duration': 103.204,
+            'uploader': 'Globo.com',
+            'uploader_id': 265,
+            'like_count': int,
+        }
+    }, {
+        'url': 'http://globotv.globo.com/canal-brasil/sangue-latino/t/todos-os-videos/v/ator-e-diretor-argentino-ricado-darin-fala-sobre-utopias-e-suas-perdas/3928201/',
+        'md5': 'c1defca721ce25b2354e927d3e4b3dec',
+        'info_dict': {
+            'id': '3928201',
+            'ext': 'mp4',
+            'title': 'Ator e diretor argentino, Ricado Darín fala sobre utopias e suas perdas',
+            'duration': 1472.906,
+            'uploader': 'Canal Brasil',
+            'uploader_id': 705,
+            'like_count': int,
+        }
+    }]
 
-    class MD5():
+    class MD5:
         HEX_FORMAT_LOWERCASE = 0
         HEX_FORMAT_UPPERCASE = 1
         BASE64_PAD_CHARACTER_DEFAULT_COMPLIANCE = ''
@@ -353,9 +330,6 @@ class GloboIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        webpage = self._download_webpage(url, video_id)
-        video_id = self._search_regex(self._VIDEOID_REGEXES, webpage, 'video id')
-
         video = self._download_json(
             self._API_URL_TEMPLATE % video_id, video_id)['videos'][0]
 
@@ -417,3 +391,39 @@ class GloboIE(InfoExtractor):
             'like_count': like_count,
             'formats': formats
         }
+
+
+class GloboArticleIE(InfoExtractor):
+    _VALID_URL = 'https?://.+?\.globo\.com/(?:[^/]+/)*(?P<id>[^/]+)\.html'
+
+    _VIDEOID_REGEXES = [
+        r'\bdata-video-id=["\'](\d{7,})',
+        r'\bdata-player-videosids=["\'](\d{7,})',
+        r'\bvideosIDs\s*:\s*["\'](\d{7,})',
+        r'\bdata-id=["\'](\d{7,})',
+        r'<div[^>]+\bid=["\'](\d{7,})',
+    ]
+
+    _TEST = {
+        'url': 'http://g1.globo.com/jornal-nacional/noticia/2014/09/novidade-na-fiscalizacao-de-bagagem-pela-receita-provoca-discussoes.html',
+        'md5': '307fdeae4390ccfe6ba1aa198cf6e72b',
+        'info_dict': {
+            'id': '3652183',
+            'ext': 'mp4',
+            'title': 'Receita Federal explica como vai fiscalizar bagagens de quem retorna ao Brasil de avião',
+            'duration': 110.711,
+            'uploader': 'Rede Globo',
+            'uploader_id': 196,
+            'like_count': int,
+        }
+    }
+
+    @classmethod
+    def suitable(cls, url):
+        return False if GloboIE.suitable(url) else super(GloboArticleIE, cls).suitable(url)
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+        video_id = self._search_regex(self._VIDEOID_REGEXES, webpage, 'video id')
+        return self.url_result('globo:%s' % video_id, 'Globo')

From e3778cce0e912f803ea10cb806406f7fcafe840f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 4 Nov 2015 16:51:19 +0600
Subject: [PATCH 2977/6219] [globo] Improve m3u8 extraction

---
 youtube_dl/extractor/globo.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/globo.py b/youtube_dl/extractor/globo.py
index 828e40d76..c28899011 100644
--- a/youtube_dl/extractor/globo.py
+++ b/youtube_dl/extractor/globo.py
@@ -367,7 +367,10 @@ class GloboIE(InfoExtractor):
             resource_url = resource['url']
             signed_url = '%s?h=%s&k=%s' % (resource_url, signed_hash, 'flash')
             if resource_id.endswith('m3u8') or resource_url.endswith('.m3u8'):
-                formats.extend(self._extract_m3u8_formats(signed_url, resource_id, 'mp4'))
+                m3u8_formats = self._extract_m3u8_formats(
+                    signed_url, resource_id, 'mp4', m3u8_id='hls', fatal=False)
+                if m3u8_formats:
+                    formats.extend(m3u8_formats)
             else:
                 formats.append({
                     'url': signed_url,

From c3459d24f16056e8ae8f982db2a10871ef18e80a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 4 Nov 2015 16:53:21 +0600
Subject: [PATCH 2978/6219] [globo] Skip unsupported smooth streaming

---
 youtube_dl/extractor/globo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/globo.py b/youtube_dl/extractor/globo.py
index c28899011..ec451bb07 100644
--- a/youtube_dl/extractor/globo.py
+++ b/youtube_dl/extractor/globo.py
@@ -338,7 +338,7 @@ class GloboIE(InfoExtractor):
         formats = []
         for resource in video['resources']:
             resource_id = resource.get('_id')
-            if not resource_id:
+            if not resource_id or resource_id.endswith('manifest'):
                 continue
 
             security = self._download_json(

From 5d235ca7f66af1f82c1a4d753d238f48fc3afa40 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 4 Nov 2015 16:55:39 +0600
Subject: [PATCH 2979/6219] [globo] Prefer native m3u8

---
 youtube_dl/extractor/globo.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/globo.py b/youtube_dl/extractor/globo.py
index ec451bb07..2a805cbb2 100644
--- a/youtube_dl/extractor/globo.py
+++ b/youtube_dl/extractor/globo.py
@@ -368,7 +368,8 @@ class GloboIE(InfoExtractor):
             signed_url = '%s?h=%s&k=%s' % (resource_url, signed_hash, 'flash')
             if resource_id.endswith('m3u8') or resource_url.endswith('.m3u8'):
                 m3u8_formats = self._extract_m3u8_formats(
-                    signed_url, resource_id, 'mp4', m3u8_id='hls', fatal=False)
+                    signed_url, resource_id, 'mp4', entry_protocol='m3u8_native',
+                    m3u8_id='hls', fatal=False)
                 if m3u8_formats:
                     formats.extend(m3u8_formats)
             else:

From b4ef6a0038657c1adde565df947e42ad1e1b4195 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 4 Nov 2015 17:01:27 +0600
Subject: [PATCH 2980/6219] [globo] Remove non available test

---
 youtube_dl/extractor/globo.py | 12 ------------
 1 file changed, 12 deletions(-)

diff --git a/youtube_dl/extractor/globo.py b/youtube_dl/extractor/globo.py
index 2a805cbb2..8aada01dc 100644
--- a/youtube_dl/extractor/globo.py
+++ b/youtube_dl/extractor/globo.py
@@ -26,18 +26,6 @@ class GloboIE(InfoExtractor):
     _RESIGN_EXPIRATION = 86400
 
     _TESTS = [{
-        'url': 'http://globotv.globo.com/sportv/futebol-nacional/v/os-gols-de-atletico-mg-3-x-2-santos-pela-24a-rodada-do-brasileirao/3654973/',
-        'md5': '03ebf41cb7ade43581608b7d9b71fab0',
-        'info_dict': {
-            'id': '3654973',
-            'ext': 'mp4',
-            'title': 'Os gols de Atlético-MG 3 x 2 Santos pela 24ª rodada do Brasileirão',
-            'duration': 251.585,
-            'uploader': 'SporTV',
-            'uploader_id': 698,
-            'like_count': int,
-        }
-    }, {
         'url': 'http://g1.globo.com/carros/autoesporte/videos/t/exclusivos-do-g1/v/mercedes-benz-gla-passa-por-teste-de-colisao-na-europa/3607726/',
         'md5': 'b3ccc801f75cd04a914d51dadb83a78d',
         'info_dict': {

From aebb42d32b608eaffb424e5e7c22f1b68a491e3d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 4 Nov 2015 17:01:55 +0600
Subject: [PATCH 2981/6219] [globo] Remove like count

It's no longer provided
---
 youtube_dl/extractor/globo.py | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/youtube_dl/extractor/globo.py b/youtube_dl/extractor/globo.py
index 8aada01dc..dc89e46ac 100644
--- a/youtube_dl/extractor/globo.py
+++ b/youtube_dl/extractor/globo.py
@@ -35,7 +35,6 @@ class GloboIE(InfoExtractor):
             'duration': 103.204,
             'uploader': 'Globo.com',
             'uploader_id': 265,
-            'like_count': int,
         }
     }, {
         'url': 'http://globotv.globo.com/canal-brasil/sangue-latino/t/todos-os-videos/v/ator-e-diretor-argentino-ricado-darin-fala-sobre-utopias-e-suas-perdas/3928201/',
@@ -47,7 +46,6 @@ class GloboIE(InfoExtractor):
             'duration': 1472.906,
             'uploader': 'Canal Brasil',
             'uploader_id': 705,
-            'like_count': int,
         }
     }]
 
@@ -370,7 +368,6 @@ class GloboIE(InfoExtractor):
         self._sort_formats(formats)
 
         duration = float_or_none(video.get('duration'), 1000)
-        like_count = int_or_none(video.get('likes'))
         uploader = video.get('channel')
         uploader_id = video.get('channel_id')
 
@@ -380,7 +377,6 @@ class GloboIE(InfoExtractor):
             'duration': duration,
             'uploader': uploader,
             'uploader_id': uploader_id,
-            'like_count': like_count,
             'formats': formats
         }
 
@@ -406,7 +402,6 @@ class GloboArticleIE(InfoExtractor):
             'duration': 110.711,
             'uploader': 'Rede Globo',
             'uploader_id': 196,
-            'like_count': int,
         }
     }
 

From a4a6b7b80f18680ee0a8bba50a24c58edd3f2a73 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 4 Nov 2015 17:03:45 +0600
Subject: [PATCH 2982/6219] [globo] Improve http formats

---
 youtube_dl/extractor/globo.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/globo.py b/youtube_dl/extractor/globo.py
index dc89e46ac..64622aa5c 100644
--- a/youtube_dl/extractor/globo.py
+++ b/youtube_dl/extractor/globo.py
@@ -361,8 +361,8 @@ class GloboIE(InfoExtractor):
             else:
                 formats.append({
                     'url': signed_url,
-                    'format_id': resource_id,
-                    'height': resource.get('height'),
+                    'format_id': 'http-%s' % resource_id,
+                    'height': int_or_none(resource.get('height')),
                 })
 
         self._sort_formats(formats)

From 264cd00fff4f6d7063d43e1d476de46901bd9c5b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 4 Nov 2015 17:10:45 +0600
Subject: [PATCH 2983/6219] [globo] Update tests

---
 youtube_dl/extractor/globo.py | 32 ++++++++++++++++++++++----------
 1 file changed, 22 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/globo.py b/youtube_dl/extractor/globo.py
index 64622aa5c..0337256ed 100644
--- a/youtube_dl/extractor/globo.py
+++ b/youtube_dl/extractor/globo.py
@@ -35,18 +35,30 @@ class GloboIE(InfoExtractor):
             'duration': 103.204,
             'uploader': 'Globo.com',
             'uploader_id': 265,
-        }
+        },
+    }, {
+        'url': 'http://globoplay.globo.com/v/4581987/',
+        'md5': 'f36a1ecd6a50da1577eee6dd17f67eff',
+        'info_dict': {
+            'id': '4581987',
+            'ext': 'mp4',
+            'title': 'Acidentes de trânsito estão entre as maiores causas de queda de energia em SP',
+            'duration': 137.973,
+            'uploader': 'Rede Globo',
+            'uploader_id': 196,
+        },
+    }, {
+        'url': 'http://canalbrasil.globo.com/programas/sangue-latino/videos/3928201.html',
+        'only_matching': True,
+    }, {
+        'url': 'http://globosatplay.globo.com/globonews/v/4472924/',
+        'only_matching': True,
+    }, {
+        'url': 'http://globotv.globo.com/t/programa/v/clipe-sexo-e-as-negas-adeus/3836166/',
+        'only_matching': True,
     }, {
         'url': 'http://globotv.globo.com/canal-brasil/sangue-latino/t/todos-os-videos/v/ator-e-diretor-argentino-ricado-darin-fala-sobre-utopias-e-suas-perdas/3928201/',
-        'md5': 'c1defca721ce25b2354e927d3e4b3dec',
-        'info_dict': {
-            'id': '3928201',
-            'ext': 'mp4',
-            'title': 'Ator e diretor argentino, Ricado Darín fala sobre utopias e suas perdas',
-            'duration': 1472.906,
-            'uploader': 'Canal Brasil',
-            'uploader_id': 705,
-        }
+        'only_matching': True,
     }]
 
     class MD5:

From e7d34c03f200e178e9d6dfe4ae3f6856e382a4b9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 4 Nov 2015 17:12:42 +0600
Subject: [PATCH 2984/6219] [globo] Force uploader id to be string

---
 youtube_dl/extractor/globo.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/globo.py b/youtube_dl/extractor/globo.py
index 0337256ed..6c0fc54de 100644
--- a/youtube_dl/extractor/globo.py
+++ b/youtube_dl/extractor/globo.py
@@ -14,6 +14,7 @@ from ..utils import (
     ExtractorError,
     float_or_none,
     int_or_none,
+    str_or_none,
 )
 
 
@@ -34,7 +35,7 @@ class GloboIE(InfoExtractor):
             'title': 'Mercedes-Benz GLA passa por teste de colisão na Europa',
             'duration': 103.204,
             'uploader': 'Globo.com',
-            'uploader_id': 265,
+            'uploader_id': '265',
         },
     }, {
         'url': 'http://globoplay.globo.com/v/4581987/',
@@ -45,7 +46,7 @@ class GloboIE(InfoExtractor):
             'title': 'Acidentes de trânsito estão entre as maiores causas de queda de energia em SP',
             'duration': 137.973,
             'uploader': 'Rede Globo',
-            'uploader_id': 196,
+            'uploader_id': '196',
         },
     }, {
         'url': 'http://canalbrasil.globo.com/programas/sangue-latino/videos/3928201.html',
@@ -381,7 +382,7 @@ class GloboIE(InfoExtractor):
 
         duration = float_or_none(video.get('duration'), 1000)
         uploader = video.get('channel')
-        uploader_id = video.get('channel_id')
+        uploader_id = str_or_none(video.get('channel_id'))
 
         return {
             'id': video_id,

From c13722480bebfb1fc33169516790df2e99b3e499 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 4 Nov 2015 17:13:35 +0600
Subject: [PATCH 2985/6219] [globo:article] Fix test

---
 youtube_dl/extractor/globo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/globo.py b/youtube_dl/extractor/globo.py
index 6c0fc54de..5883be704 100644
--- a/youtube_dl/extractor/globo.py
+++ b/youtube_dl/extractor/globo.py
@@ -414,7 +414,7 @@ class GloboArticleIE(InfoExtractor):
             'title': 'Receita Federal explica como vai fiscalizar bagagens de quem retorna ao Brasil de avião',
             'duration': 110.711,
             'uploader': 'Rede Globo',
-            'uploader_id': 196,
+            'uploader_id': '196',
         }
     }
 

From 5d501a0901c36695c9d6ca3958ac4ccfdea90954 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 4 Nov 2015 17:42:11 +0600
Subject: [PATCH 2986/6219] [globo] Add more tests

---
 youtube_dl/extractor/globo.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/globo.py b/youtube_dl/extractor/globo.py
index 5883be704..c65ef6bcf 100644
--- a/youtube_dl/extractor/globo.py
+++ b/youtube_dl/extractor/globo.py
@@ -60,6 +60,9 @@ class GloboIE(InfoExtractor):
     }, {
         'url': 'http://globotv.globo.com/canal-brasil/sangue-latino/t/todos-os-videos/v/ator-e-diretor-argentino-ricado-darin-fala-sobre-utopias-e-suas-perdas/3928201/',
         'only_matching': True,
+    }, {
+        'url': 'http://canaloff.globo.com/programas/desejar-profundo/videos/4518560.html',
+        'only_matching': True,
     }]
 
     class MD5:
@@ -405,7 +408,7 @@ class GloboArticleIE(InfoExtractor):
         r'<div[^>]+\bid=["\'](\d{7,})',
     ]
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://g1.globo.com/jornal-nacional/noticia/2014/09/novidade-na-fiscalizacao-de-bagagem-pela-receita-provoca-discussoes.html',
         'md5': '307fdeae4390ccfe6ba1aa198cf6e72b',
         'info_dict': {
@@ -416,7 +419,13 @@ class GloboArticleIE(InfoExtractor):
             'uploader': 'Rede Globo',
             'uploader_id': '196',
         }
-    }
+    }, {
+        'url': 'http://gq.globo.com/Prazeres/Poder/noticia/2015/10/all-o-desafio-assista-ao-segundo-capitulo-da-serie.html',
+        'only_matching': True,
+    }, {
+        'url': 'http://gshow.globo.com/programas/tv-xuxa/O-Programa/noticia/2014/01/xuxa-e-junno-namoram-muuuito-em-luau-de-zeze-di-camargo-e-luciano.html',
+        'only_matching': True,
+    }]
 
     @classmethod
     def suitable(cls, url):

From 17d1900581ffd12866e56640080ce340d99149a2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 4 Nov 2015 17:57:46 +0600
Subject: [PATCH 2987/6219] [vk] Fix view count extraction (Closes #7353)

---
 youtube_dl/extractor/vk.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 765e9e6fd..01960b827 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -281,9 +281,13 @@ class VKIE(InfoExtractor):
             mobj.group(1) + ' ' + mobj.group(2)
             upload_date = unified_strdate(mobj.group(1) + ' ' + mobj.group(2))
 
-        view_count = str_to_int(self._search_regex(
-            r'"mv_views_count_number"[^>]*>([\d,.]+) views<',
-            info_page, 'view count', fatal=False))
+        view_count = None
+        views = self._html_search_regex(
+            r'"mv_views_count_number"[^>]*>(.+?\bviews?)<',
+            info_page, 'view count', fatal=False)
+        if views:
+            view_count = str_to_int(self._search_regex(
+                r'([\d,.]+)', views, 'view count', fatal=False))
 
         formats = [{
             'format_id': k,

From cb5a470635ea2ad91f18d33e391979aabb0755fb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 4 Nov 2015 16:18:51 +0100
Subject: [PATCH 2988/6219] [vimeo] Remove unused import

---
 youtube_dl/extractor/vimeo.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index b608740b8..ca716c8f5 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -8,7 +8,6 @@ import itertools
 from .common import InfoExtractor
 from ..compat import (
     compat_HTTPError,
-    compat_urllib_parse,
     compat_urllib_request,
     compat_urlparse,
 )

From 44b2264feae331eeb34e83eed1387def3d61a437 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 4 Nov 2015 22:12:24 +0600
Subject: [PATCH 2989/6219] [youtube] Prefer video_info with token available

---
 youtube_dl/extractor/youtube.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index d7eda7aa7..5eeb3c663 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1107,6 +1107,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     if not video_info:
                         video_info = get_video_info
                     if 'token' in get_video_info:
+                        if 'token' not in video_info:
+                            video_info = get_video_info
                         break
         if 'token' not in video_info:
             if 'reason' in video_info:

From 89ea063eebae84792a7ccb968533ff8bf6a41d56 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 4 Nov 2015 22:49:23 +0600
Subject: [PATCH 2990/6219] [youtube] Clarify rationale for preferring a video
 info with token (#7362)

---
 youtube_dl/extractor/youtube.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 5eeb3c663..e2a43299f 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1107,6 +1107,15 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     if not video_info:
                         video_info = get_video_info
                     if 'token' in get_video_info:
+                        # Different get_video_info requests may report different results, e.g.
+                        # some may report video unavailability, but some may serve it without
+                        # any complaint (see https://github.com/rg3/youtube-dl/issues/7362,
+                        # the original webpage as well as el=info and el=embedded get_video_info
+                        # requests report video unavailability due to geo restriction while
+                        # el=detailpage succeeds and returns valid data). This is probably
+                        # due to YouTube measures against IP ranges of hosting providers.
+                        # Working around by preferring the first succeeded video_info containing
+                        # the token if no such video_info yet was found.
                         if 'token' not in video_info:
                             video_info = get_video_info
                         break

From f93ded98522cc1272a8d2210738937132292afc9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 5 Nov 2015 01:54:49 +0600
Subject: [PATCH 2991/6219] [prosiebensat1] Add support for .ch domains (Closes
 #7365)

---
 youtube_dl/extractor/prosiebensat1.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/prosiebensat1.py b/youtube_dl/extractor/prosiebensat1.py
index effcf1db3..baa54a3af 100644
--- a/youtube_dl/extractor/prosiebensat1.py
+++ b/youtube_dl/extractor/prosiebensat1.py
@@ -20,7 +20,7 @@ from ..utils import (
 class ProSiebenSat1IE(InfoExtractor):
     IE_NAME = 'prosiebensat1'
     IE_DESC = 'ProSiebenSat.1 Digital'
-    _VALID_URL = r'https?://(?:www\.)?(?:(?:prosieben|prosiebenmaxx|sixx|sat1|kabeleins|the-voice-of-germany)\.(?:de|at)|ran\.de|fem\.com)/(?P<id>.+)'
+    _VALID_URL = r'https?://(?:www\.)?(?:(?:prosieben|prosiebenmaxx|sixx|sat1|kabeleins|the-voice-of-germany)\.(?:de|at|ch)|ran\.de|fem\.com)/(?P<id>.+)'
 
     _TESTS = [
         {

From b15c44cd36831f175e9dd4081b82beb8075790b3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 5 Nov 2015 02:51:30 +0600
Subject: [PATCH 2992/6219] [periscope] Add support for videos with
 broadcast_id (Closes #7359)

---
 youtube_dl/extractor/periscope.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/periscope.py b/youtube_dl/extractor/periscope.py
index 8ad936758..0f9d7576f 100644
--- a/youtube_dl/extractor/periscope.py
+++ b/youtube_dl/extractor/periscope.py
@@ -27,9 +27,10 @@ class PeriscopeIE(InfoExtractor):
         'skip': 'Expires in 24 hours',
     }
 
-    def _call_api(self, method, token):
+    def _call_api(self, method, value):
+        attribute = 'token' if len(value) > 13 else 'broadcast_id'
         return self._download_json(
-            'https://api.periscope.tv/api/v2/%s?token=%s' % (method, token), token)
+            'https://api.periscope.tv/api/v2/%s?%s=%s' % (method, attribute, value), value)
 
     def _real_extract(self, url):
         token = self._match_id(url)

From 2549e113b8750a493917436d4fd15ed74a1a4983 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 5 Nov 2015 02:55:53 +0600
Subject: [PATCH 2993/6219] [periscope] Add test for broadcast_id based URL

---
 youtube_dl/extractor/periscope.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/periscope.py b/youtube_dl/extractor/periscope.py
index 0f9d7576f..7621d9e99 100644
--- a/youtube_dl/extractor/periscope.py
+++ b/youtube_dl/extractor/periscope.py
@@ -12,7 +12,7 @@ from ..utils import parse_iso8601
 class PeriscopeIE(InfoExtractor):
     IE_DESC = 'Periscope'
     _VALID_URL = r'https?://(?:www\.)?periscope\.tv/w/(?P<id>[^/?#]+)'
-    _TEST = {
+    _TESTS = [{
         'url': 'https://www.periscope.tv/w/aJUQnjY3MjA3ODF8NTYxMDIyMDl2zCg2pECBgwTqRpQuQD352EMPTKQjT4uqlM3cgWFA-g==',
         'md5': '65b57957972e503fcbbaeed8f4fa04ca',
         'info_dict': {
@@ -25,7 +25,10 @@ class PeriscopeIE(InfoExtractor):
             'uploader_id': '1465763',
         },
         'skip': 'Expires in 24 hours',
-    }
+    }, {
+        'url': 'https://www.periscope.tv/w/1ZkKzPbMVggJv',
+        'only_matching': True,
+    }]
 
     def _call_api(self, method, value):
         attribute = 'token' if len(value) > 13 else 'broadcast_id'

From 53472df85793cc89deb779c2ffc3ae1f47292fd4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 5 Nov 2015 02:56:44 +0600
Subject: [PATCH 2994/6219] [periscope] Add note on where to find alive example
 URLs

---
 youtube_dl/extractor/periscope.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/periscope.py b/youtube_dl/extractor/periscope.py
index 7621d9e99..887c8020d 100644
--- a/youtube_dl/extractor/periscope.py
+++ b/youtube_dl/extractor/periscope.py
@@ -12,6 +12,7 @@ from ..utils import parse_iso8601
 class PeriscopeIE(InfoExtractor):
     IE_DESC = 'Periscope'
     _VALID_URL = r'https?://(?:www\.)?periscope\.tv/w/(?P<id>[^/?#]+)'
+    # Alive example URLs can be found here http://onperiscope.com/
     _TESTS = [{
         'url': 'https://www.periscope.tv/w/aJUQnjY3MjA3ODF8NTYxMDIyMDl2zCg2pECBgwTqRpQuQD352EMPTKQjT4uqlM3cgWFA-g==',
         'md5': '65b57957972e503fcbbaeed8f4fa04ca',

From b3613d36da14ab527166326707c0f911d192144d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 5 Nov 2015 04:37:51 +0600
Subject: [PATCH 2995/6219] [YoutubeDL] Sanitize path after output template
 substitution (Closes #7367)

---
 youtube_dl/YoutubeDL.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 12977bf80..1783ce01b 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -572,7 +572,7 @@ class YoutubeDL(object):
                                  if v is not None)
             template_dict = collections.defaultdict(lambda: 'NA', template_dict)
 
-            outtmpl = sanitize_path(self.params.get('outtmpl', DEFAULT_OUTTMPL))
+            outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
             tmpl = compat_expanduser(outtmpl)
             filename = tmpl % template_dict
             # Temporary fix for #4787
@@ -580,7 +580,7 @@ class YoutubeDL(object):
             # to workaround encoding issues with subprocess on python2 @ Windows
             if sys.version_info < (3, 0) and sys.platform == 'win32':
                 filename = encodeFilename(filename, True).decode(preferredencoding())
-            return filename
+            return sanitize_path(filename)
         except ValueError as err:
             self.report_error('Error in output template: ' + str(err) + ' (encoding: ' + repr(preferredencoding()) + ')')
             return None

From 967c9076a31ca2a2b43fb71082ad1a8db88116bd Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 5 Nov 2015 18:01:13 +0100
Subject: [PATCH 2996/6219] raise ExtractorError if the page doesn't contain a
 video

---
 youtube_dl/extractor/flickr.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/flickr.py b/youtube_dl/extractor/flickr.py
index e97754d36..92d2ac553 100644
--- a/youtube_dl/extractor/flickr.py
+++ b/youtube_dl/extractor/flickr.py
@@ -79,3 +79,5 @@ class FlickrIE(InfoExtractor):
                 'uploader': owner.get('realname'),
                 'comment_count': int_or_none(video_info.get('comments', {}).get('_content')),
             }
+        else:
+            raise ExtractorError('not a video', expected=True)

From 6953d8e95a78e83f087693b7353baab96b09fbdd Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 6 Nov 2015 02:09:55 +0100
Subject: [PATCH 2997/6219] [miomio] fix info extraction (fixes #7366)

---
 youtube_dl/extractor/miomio.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/miomio.py b/youtube_dl/extractor/miomio.py
index a784fc5fb..3f812e005 100644
--- a/youtube_dl/extractor/miomio.py
+++ b/youtube_dl/extractor/miomio.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 import random
 
 from .common import InfoExtractor
+from ..compat import compat_urllib_request
 from ..utils import (
     xpath_text,
     int_or_none,
@@ -60,10 +61,12 @@ class MioMioIE(InfoExtractor):
             'http://www.miomio.tv/mioplayer/mioplayerconfigfiles/xml.php?id=%s&r=%s' % (id, random.randint(100, 999)),
             video_id)
 
-        # the following xml contains the actual configuration information on the video file(s)
-        vid_config = self._download_xml(
+        vid_config_request = compat_urllib_request.Request(
             'http://www.miomio.tv/mioplayer/mioplayerconfigfiles/sina.php?{0}'.format(xml_config),
-            video_id)
+            headers={'Referer': 'http://www.miomio.tv/mioplayer/mioplayer-v3.0.swf'})
+
+        # the following xml contains the actual configuration information on the video file(s)
+        vid_config = self._download_xml(vid_config_request, video_id)
 
         http_headers = {
             'Referer': 'http://www.miomio.tv%s' % mioplayer_path,

From e68dd1921ad7528d225a8571066f99b9934b6a06 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 6 Nov 2015 06:33:05 +0100
Subject: [PATCH 2998/6219] [miomio] use the formats urls headers for
 downloading xml

---
 youtube_dl/extractor/miomio.py | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/miomio.py b/youtube_dl/extractor/miomio.py
index 3f812e005..6f40bf1b9 100644
--- a/youtube_dl/extractor/miomio.py
+++ b/youtube_dl/extractor/miomio.py
@@ -52,6 +52,8 @@ class MioMioIE(InfoExtractor):
         mioplayer_path = self._search_regex(
             r'src="(/mioplayer/[^"]+)"', webpage, 'ref_path')
 
+        http_headers = {'Referer': 'http://www.miomio.tv%s' % mioplayer_path,}
+
         xml_config = self._search_regex(
             r'flashvars="type=(?:sina|video)&amp;(.+?)&amp;',
             webpage, 'xml config')
@@ -63,15 +65,11 @@ class MioMioIE(InfoExtractor):
 
         vid_config_request = compat_urllib_request.Request(
             'http://www.miomio.tv/mioplayer/mioplayerconfigfiles/sina.php?{0}'.format(xml_config),
-            headers={'Referer': 'http://www.miomio.tv/mioplayer/mioplayer-v3.0.swf'})
+            headers=http_headers)
 
         # the following xml contains the actual configuration information on the video file(s)
         vid_config = self._download_xml(vid_config_request, video_id)
 
-        http_headers = {
-            'Referer': 'http://www.miomio.tv%s' % mioplayer_path,
-        }
-
         if not int_or_none(xpath_text(vid_config, 'timelength')):
             raise ExtractorError('Unable to load videos!', expected=True)
 

From a641b2459263228fb1dd86dfe05d6047cedbf345 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 6 Nov 2015 07:23:03 +0100
Subject: [PATCH 2999/6219] [cnet] skip hls_phone if hls_tablet is present

---
 youtube_dl/extractor/cnet.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/cnet.py b/youtube_dl/extractor/cnet.py
index 2fac0d79d..3ecf0efd4 100644
--- a/youtube_dl/extractor/cnet.py
+++ b/youtube_dl/extractor/cnet.py
@@ -57,7 +57,9 @@ class CNETIE(InfoExtractor):
         subtitles = {}
         description = vdata.get('description')
 
-        for vid in vdata['files'].values():
+        for (fkey, vid) in vdata['files'].items():
+            if fkey == 'hls_phone' and 'hls_tablet' in vdata['files']:
+                continue
             result = tp.extract(('http://link.theplatform.com/s/%s/%s' % (mpx_account, vid)))
             formats.extend(result['formats'])
             subtitles = self._merge_subtitles(subtitles, result['subtitles'])

From 5003e4283b35acb82ea9793d91bc3cd1ee679f86 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 6 Nov 2015 21:06:44 +0600
Subject: [PATCH 3000/6219] [ndr] Relax _VALID_URL (Closes #7383)

---
 youtube_dl/extractor/ndr.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ndr.py b/youtube_dl/extractor/ndr.py
index ba06d8a98..a2b51ccb3 100644
--- a/youtube_dl/extractor/ndr.py
+++ b/youtube_dl/extractor/ndr.py
@@ -23,7 +23,7 @@ class NDRBaseIE(InfoExtractor):
 class NDRIE(NDRBaseIE):
     IE_NAME = 'ndr'
     IE_DESC = 'NDR.de - Norddeutscher Rundfunk'
-    _VALID_URL = r'https?://www\.ndr\.de/(?:[^/]+/)+(?P<id>[^/?#]+),[\da-z]+\.html'
+    _VALID_URL = r'https?://www\.ndr\.de/(?:[^/]+/)*(?P<id>[^/?#]+),[\da-z]+\.html'
     _TESTS = [{
         # httpVideo, same content id
         'url': 'http://www.ndr.de/fernsehen/Party-Poette-und-Parade,hafengeburtstag988.html',

From 01003d072c20c2ed095930d87c5ce3d5610e66b1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 6 Nov 2015 21:07:52 +0600
Subject: [PATCH 3001/6219] [ndr] Add test for #7383

---
 youtube_dl/extractor/ndr.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/ndr.py b/youtube_dl/extractor/ndr.py
index a2b51ccb3..0be866681 100644
--- a/youtube_dl/extractor/ndr.py
+++ b/youtube_dl/extractor/ndr.py
@@ -78,6 +78,9 @@ class NDRIE(NDRBaseIE):
         'params': {
             'skip_download': True,
         },
+    }, {
+        'url': 'https://www.ndr.de/Fettes-Brot-Ferris-MC-und-Thees-Uhlmann-live-on-stage,festivalsommer116.html',
+        'only_matching': True,
     }]
 
     def _extract_embed(self, webpage, display_id):

From 1e2eb4b40d46f39d15c067657ecad16fa3b2121d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 6 Nov 2015 21:08:21 +0600
Subject: [PATCH 3002/6219] [njoy] Relax _VALID_URL

---
 youtube_dl/extractor/ndr.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ndr.py b/youtube_dl/extractor/ndr.py
index 0be866681..7043c7e0f 100644
--- a/youtube_dl/extractor/ndr.py
+++ b/youtube_dl/extractor/ndr.py
@@ -105,7 +105,7 @@ class NDRIE(NDRBaseIE):
 class NJoyIE(NDRBaseIE):
     IE_NAME = 'njoy'
     IE_DESC = 'N-JOY'
-    _VALID_URL = r'https?://www\.n-joy\.de/(?:[^/]+/)+(?:(?P<display_id>[^/?#]+),)?(?P<id>[\da-z]+)\.html'
+    _VALID_URL = r'https?://www\.n-joy\.de/(?:[^/]+/)*(?:(?P<display_id>[^/?#]+),)?(?P<id>[\da-z]+)\.html'
     _TESTS = [{
         # httpVideo, same content id
         'url': 'http://www.n-joy.de/entertainment/comedy/comedy_contest/Benaissa-beim-NDR-Comedy-Contest,comedycontest2480.html',

From 81413c01651eddcc5180af379f2ce3689a376051 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 6 Nov 2015 21:08:52 +0600
Subject: [PATCH 3003/6219] [ndr:embed] Relax _VALID_URL

---
 youtube_dl/extractor/ndr.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ndr.py b/youtube_dl/extractor/ndr.py
index 7043c7e0f..477ce4e6b 100644
--- a/youtube_dl/extractor/ndr.py
+++ b/youtube_dl/extractor/ndr.py
@@ -238,7 +238,7 @@ class NDREmbedBaseIE(InfoExtractor):
 
 class NDREmbedIE(NDREmbedBaseIE):
     IE_NAME = 'ndr:embed'
-    _VALID_URL = r'https?://www\.ndr\.de/(?:[^/]+/)+(?P<id>[\da-z]+)-(?:player|externalPlayer)\.html'
+    _VALID_URL = r'https?://www\.ndr\.de/(?:[^/]+/)*(?P<id>[\da-z]+)-(?:player|externalPlayer)\.html'
     _TESTS = [{
         'url': 'http://www.ndr.de/fernsehen/sendungen/ndr_aktuell/ndraktuell28488-player.html',
         'md5': '8b9306142fe65bbdefb5ce24edb6b0a9',

From 92366d189ef280b8ba0057930c54aa14b0ecdd24 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 6 Nov 2015 21:09:17 +0600
Subject: [PATCH 3004/6219] [njoy:embed] Relax _VALID_URL

---
 youtube_dl/extractor/ndr.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ndr.py b/youtube_dl/extractor/ndr.py
index 477ce4e6b..16213eed9 100644
--- a/youtube_dl/extractor/ndr.py
+++ b/youtube_dl/extractor/ndr.py
@@ -332,7 +332,7 @@ class NDREmbedIE(NDREmbedBaseIE):
 
 class NJoyEmbedIE(NDREmbedBaseIE):
     IE_NAME = 'njoy:embed'
-    _VALID_URL = r'https?://www\.n-joy\.de/(?:[^/]+/)+(?P<id>[\da-z]+)-(?:player|externalPlayer)_[^/]+\.html'
+    _VALID_URL = r'https?://www\.n-joy\.de/(?:[^/]+/)*(?P<id>[\da-z]+)-(?:player|externalPlayer)_[^/]+\.html'
     _TESTS = [{
         # httpVideo
         'url': 'http://www.n-joy.de/events/reeperbahnfestival/doku948-player_image-bc168e87-5263-4d6d-bd27-bb643005a6de_theme-n-joy.html',

From deb85c32bbd32e8d280e1919432a11c0bdaa26bb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 6 Nov 2015 21:56:31 +0600
Subject: [PATCH 3005/6219] [postprocessor/ffmpeg] Use ffmpeg as prefix since
 it's used all over the places (Closes #7371)

---
 youtube_dl/postprocessor/ffmpeg.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 4f320e124..5ed723bc6 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -272,7 +272,7 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
             return [], information
 
         try:
-            self._downloader.to_screen('[' + self.basename + '] Destination: ' + new_path)
+            self._downloader.to_screen('[ffmpeg] Destination: ' + new_path)
             self.run_ffmpeg(path, new_path, acodec, more_opts)
         except AudioConversionError as e:
             raise PostProcessingError(

From 179ffab69c3359ab7d0a7b0a2b63c94d8c70af67 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 6 Nov 2015 23:06:13 +0600
Subject: [PATCH 3006/6219] [lynda:course] Force log out (Closes #7361)

---
 youtube_dl/extractor/lynda.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index 5c973e75c..67f2025de 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -82,6 +82,11 @@ class LyndaBaseIE(InfoExtractor):
                         expected=True)
             raise ExtractorError('Unable to log in')
 
+    def _logout(self):
+        self._download_webpage(
+            'http://www.lynda.com/ajax/logout.aspx', None,
+            'Logging out', 'Unable to log out', fatal=False)
+
 
 class LyndaIE(LyndaBaseIE):
     IE_NAME = 'lynda'
@@ -210,6 +215,8 @@ class LyndaCourseIE(LyndaBaseIE):
             course_id, 'Downloading course JSON')
         course_json = json.loads(page)
 
+        self._logout()
+
         if 'Status' in course_json and course_json['Status'] == 'NotFound':
             raise ExtractorError(
                 'Course %s does not exist' % course_id, expected=True)

From 71bb016160744a80fecaadf5b75b0dc2b1e8089b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 6 Nov 2015 23:10:07 +0600
Subject: [PATCH 3007/6219] [lynda:course] Modernize and make more robust

---
 youtube_dl/extractor/lynda.py | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index 67f2025de..98474ded9 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -210,14 +210,13 @@ class LyndaCourseIE(LyndaBaseIE):
         course_path = mobj.group('coursepath')
         course_id = mobj.group('courseid')
 
-        page = self._download_webpage(
+        course = self._download_json(
             'http://www.lynda.com/ajax/player?courseId=%s&type=course' % course_id,
             course_id, 'Downloading course JSON')
-        course_json = json.loads(page)
 
         self._logout()
 
-        if 'Status' in course_json and course_json['Status'] == 'NotFound':
+        if course.get('Status') == 'NotFound':
             raise ExtractorError(
                 'Course %s does not exist' % course_id, expected=True)
 
@@ -227,12 +226,14 @@ class LyndaCourseIE(LyndaBaseIE):
         # Might want to extract videos right here from video['Formats'] as it seems 'Formats' is not provided
         # by single video API anymore
 
-        for chapter in course_json['Chapters']:
-            for video in chapter['Videos']:
-                if video['HasAccess'] is False:
+        for chapter in course['Chapters']:
+            for video in chapter.get('Videos', []):
+                if video.get('HasAccess') is False:
                     unaccessible_videos += 1
                     continue
-                videos.append(video['ID'])
+                video_id = video.get('ID')
+                if video_id:
+                    videos.append(video_id)
 
         if unaccessible_videos > 0:
             self._downloader.report_warning(
@@ -245,6 +246,6 @@ class LyndaCourseIE(LyndaBaseIE):
                 'Lynda')
             for video_id in videos]
 
-        course_title = course_json['Title']
+        course_title = course.get('Title')
 
         return self.playlist_result(entries, course_id, course_title)

From ea8ed40b2fb70fc2f01aba475128821078873d46 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 6 Nov 2015 23:24:39 +0600
Subject: [PATCH 3008/6219] [lynda] Modernize and make more robust

---
 youtube_dl/extractor/lynda.py | 52 ++++++++++++++++-------------------
 1 file changed, 24 insertions(+), 28 deletions(-)

diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index 98474ded9..c8a16842e 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -113,51 +113,47 @@ class LyndaIE(LyndaBaseIE):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        page = self._download_webpage(
+        video = self._download_json(
             'http://www.lynda.com/ajax/player?videoId=%s&type=video' % video_id,
             video_id, 'Downloading video JSON')
-        video_json = json.loads(page)
 
-        if 'Status' in video_json:
+        if 'Status' in video:
             raise ExtractorError(
-                'lynda returned error: %s' % video_json['Message'], expected=True)
+                'lynda returned error: %s' % video['Message'], expected=True)
 
-        if video_json['HasAccess'] is False:
+        if video.get('HasAccess') is False:
             self.raise_login_required('Video %s is only available for members' % video_id)
 
-        video_id = compat_str(video_json['ID'])
-        duration = video_json['DurationInSeconds']
-        title = video_json['Title']
+        video_id = compat_str(video.get('ID') or video_id)
+        duration = int_or_none(video.get('DurationInSeconds'))
+        title = video['Title']
 
         formats = []
 
-        fmts = video_json.get('Formats')
+        fmts = video.get('Formats')
         if fmts:
-            formats.extend([
-                {
-                    'url': fmt['Url'],
-                    'ext': fmt['Extension'],
-                    'width': fmt['Width'],
-                    'height': fmt['Height'],
-                    'filesize': fmt['FileSize'],
-                    'format_id': str(fmt['Resolution'])
-                } for fmt in fmts])
+            formats.extend([{
+                'url': f['Url'],
+                'ext': f.get('Extension'),
+                'width': int_or_none(f.get('Width')),
+                'height': int_or_none(f.get('Height')),
+                'filesize': int_or_none(f.get('FileSize')),
+                'format_id': compat_str(f.get('Resolution')) if f.get('Resolution') else None,
+            } for f in fmts if f.get('Url')])
 
-        prioritized_streams = video_json.get('PrioritizedStreams')
+        prioritized_streams = video.get('PrioritizedStreams')
         if prioritized_streams:
             for prioritized_stream_id, prioritized_stream in prioritized_streams.items():
-                formats.extend([
-                    {
-                        'url': video_url,
-                        'width': int_or_none(format_id),
-                        'format_id': '%s-%s' % (prioritized_stream_id, format_id),
-                    } for format_id, video_url in prioritized_stream.items()
-                ])
+                formats.extend([{
+                    'url': video_url,
+                    'width': int_or_none(format_id),
+                    'format_id': '%s-%s' % (prioritized_stream_id, format_id),
+                } for format_id, video_url in prioritized_stream.items()])
 
         self._check_formats(formats, video_id)
         self._sort_formats(formats)
 
-        subtitles = self.extract_subtitles(video_id, page)
+        subtitles = self.extract_subtitles(video_id)
 
         return {
             'id': video_id,
@@ -188,7 +184,7 @@ class LyndaIE(LyndaBaseIE):
         if srt:
             return srt
 
-    def _get_subtitles(self, video_id, webpage):
+    def _get_subtitles(self, video_id):
         url = 'http://www.lynda.com/ajax/player?videoId=%s&type=transcript' % video_id
         subs = self._download_json(url, None, False)
         if subs:

From ae4ddf9efae816f4d52fc584c93e4f0e3c79c410 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 6 Nov 2015 23:27:38 +0600
Subject: [PATCH 3009/6219] [lynda] PEP 8

---
 youtube_dl/extractor/lynda.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index c8a16842e..9a207b2cd 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -227,9 +227,8 @@ class LyndaCourseIE(LyndaBaseIE):
                 if video.get('HasAccess') is False:
                     unaccessible_videos += 1
                     continue
-                video_id = video.get('ID')
-                if video_id:
-                    videos.append(video_id)
+                if video.get('ID'):
+                    videos.append(video['ID'])
 
         if unaccessible_videos > 0:
             self._downloader.report_warning(

From 472404953a22811cc8156da110ea872a924f1f18 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 6 Nov 2015 23:28:14 +0600
Subject: [PATCH 3010/6219] [miomio] PEP 8

---
 youtube_dl/extractor/miomio.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/miomio.py b/youtube_dl/extractor/miomio.py
index 6f40bf1b9..ce391c759 100644
--- a/youtube_dl/extractor/miomio.py
+++ b/youtube_dl/extractor/miomio.py
@@ -52,7 +52,7 @@ class MioMioIE(InfoExtractor):
         mioplayer_path = self._search_regex(
             r'src="(/mioplayer/[^"]+)"', webpage, 'ref_path')
 
-        http_headers = {'Referer': 'http://www.miomio.tv%s' % mioplayer_path,}
+        http_headers = {'Referer': 'http://www.miomio.tv%s' % mioplayer_path}
 
         xml_config = self._search_regex(
             r'flashvars="type=(?:sina|video)&amp;(.+?)&amp;',

From 0fa6b17dccd2347cb0611651fc04e36839d33a4e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 6 Nov 2015 23:45:26 +0600
Subject: [PATCH 3011/6219] [pbs] Simplify and speed up player URL search

---
 youtube_dl/extractor/pbs.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 3448736a2..7b868d057 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -191,9 +191,13 @@ class PBSIE(InfoExtractor):
             if media_id:
                 return media_id, presumptive_id, upload_date
 
-            url = self._search_regex(
-                r'(?s)<iframe[^>]+?(?:[a-z-]+?=["\'].*?["\'][^>]+?)*?\bsrc=["\']([^\'"]+partnerplayer[^\'"]+)["\']',
-                webpage, 'player URL')
+            for iframe in re.findall(r'(?s)<iframe(.+?)></iframe>', webpage):
+                url = self._search_regex(
+                    r'src=(["\'])(?P<url>.+?partnerplayer.+?)\1', iframe,
+                    'player URL', default=None, group='url')
+                if url:
+                    break
+
             mobj = re.match(self._VALID_URL, url)
 
         player_id = mobj.group('player_id')

From 686f98816ecbbcb224d1336682688b05cdb051a6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 7 Nov 2015 00:39:16 +0600
Subject: [PATCH 3012/6219] [pbs] Add support for flp frontlines (Closes #7369)

---
 youtube_dl/extractor/pbs.py | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 7b868d057..3169e9c3f 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -8,6 +8,7 @@ from ..utils import (
     ExtractorError,
     determine_ext,
     int_or_none,
+    strip_jsonp,
     unified_strdate,
     US_RATINGS,
 )
@@ -191,6 +192,23 @@ class PBSIE(InfoExtractor):
             if media_id:
                 return media_id, presumptive_id, upload_date
 
+            # Fronline video embedded via flp
+            video_id = self._search_regex(
+                r'videoid\s*:\s*"([\d+a-z]{7,})"', webpage, 'videoid')
+            if video_id:
+                # pkg_id calculation is reverse engineered from
+                # http://www.pbs.org/wgbh/pages/frontline/js/flp2012.js
+                prg_id = self._search_regex(
+                    r'videoid\s*:\s*"([\d+a-z]{7,})"', webpage, 'videoid')[7:]
+                if 'q' in prg_id:
+                    prg_id = prg_id.split('q')[1]
+                prg_id = int(prg_id, 16)
+                getdir = self._download_json(
+                    'http://www.pbs.org/wgbh/pages/frontline/.json/getdir/getdir%d.json' % prg_id,
+                    presumptive_id, 'Downloading getdir JSON',
+                    transform_source=strip_jsonp)
+                return getdir['mid'], presumptive_id, upload_date
+
             for iframe in re.findall(r'(?s)<iframe(.+?)></iframe>', webpage):
                 url = self._search_regex(
                     r'src=(["\'])(?P<url>.+?partnerplayer.+?)\1', iframe,

From 8b6d9406db1d3361b006016e6aace54b05cb6fea Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 7 Nov 2015 00:42:30 +0600
Subject: [PATCH 3013/6219] [pbs] Add test for flp frontline embeds

---
 youtube_dl/extractor/pbs.py | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 3169e9c3f..a690f9c29 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -154,6 +154,22 @@ class PBSIE(InfoExtractor):
             'params': {
                 'skip_download': True,  # requires ffmpeg
             },
+        },
+        {
+            # Frontline video embedded via flp2012.js
+            'url': 'http://www.pbs.org/wgbh/pages/frontline/the-atomic-artists',
+            'info_dict': {
+                'id': '2070868960',
+                'display_id': 'the-atomic-artists',
+                'ext': 'mp4',
+                'title': 'FRONTLINE - The Atomic Artists',
+                'description': 'md5:f5bfbefadf421e8bb8647602011caf8e',
+                'duration': 723,
+                'thumbnail': 're:^https?://.*\.jpg$',
+            },
+            'params': {
+                'skip_download': True,  # requires ffmpeg
+            },
         }
     ]
     _ERRORS = {

From 21d0c33ecde573db961b97f5f0c37ba9d3c02ff3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 7 Nov 2015 01:08:40 +0600
Subject: [PATCH 3014/6219] [pbs] Make flp embed lookup non fatal

---
 youtube_dl/extractor/pbs.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index a690f9c29..8fb9b1849 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -210,7 +210,7 @@ class PBSIE(InfoExtractor):
 
             # Fronline video embedded via flp
             video_id = self._search_regex(
-                r'videoid\s*:\s*"([\d+a-z]{7,})"', webpage, 'videoid')
+                r'videoid\s*:\s*"([\d+a-z]{7,})"', webpage, 'videoid', default=None)
             if video_id:
                 # pkg_id calculation is reverse engineered from
                 # http://www.pbs.org/wgbh/pages/frontline/js/flp2012.js

From ee223abb88263bdda2d92c4b2139d1dca60ba3ae Mon Sep 17 00:00:00 2001
From: Mister Hat <misterhat144@gmail.com>
Date: Tue, 3 Nov 2015 19:13:27 -0600
Subject: [PATCH 3015/6219] [vidzi] fixed. finds url from hash and host in
 script

Closes #7386.
---
 youtube_dl/extractor/vidzi.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vidzi.py b/youtube_dl/extractor/vidzi.py
index 08a5a7b8d..2ba9f31df 100644
--- a/youtube_dl/extractor/vidzi.py
+++ b/youtube_dl/extractor/vidzi.py
@@ -20,8 +20,14 @@ class VidziIE(InfoExtractor):
         video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id)
-        video_url = self._html_search_regex(
-            r'{\s*file\s*:\s*"([^"]+)"\s*}', webpage, 'video url')
+        video_host = self._html_search_regex(
+            r'id=\'vplayer\'><img src="http://(.*?)/i', webpage,
+            'video host')
+        video_hash = self._html_search_regex(
+            r'\|([a-z0-9]+)\|hls\|type', webpage, 'video_hash')
+        ext = self._html_search_regex(
+            r'\|tracks\|([a-z0-9]+)\|', webpage, 'video ext')
+        video_url = 'http://' + video_host + '/' + video_hash + '/v.' + ext
         title = self._html_search_regex(
             r'(?s)<h2 class="video-title">(.*?)</h2>', webpage, 'title')
 

From 5d0f84d32cc038dd71673987cb6efaa85e953474 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 7 Nov 2015 06:23:00 +0600
Subject: [PATCH 3016/6219] [beeg] Skip empty URLs (Closes #7392)

---
 youtube_dl/extractor/beeg.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/beeg.py b/youtube_dl/extractor/beeg.py
index e6c928699..61bc2f744 100644
--- a/youtube_dl/extractor/beeg.py
+++ b/youtube_dl/extractor/beeg.py
@@ -33,6 +33,8 @@ class BeegIE(InfoExtractor):
 
         formats = []
         for format_id, video_url in video.items():
+            if not video_url:
+                continue
             height = self._search_regex(
                 r'^(\d+)[pP]$', format_id, 'height', default=None)
             if not height:

From 5214f1e31d5e5ba692fb1ed4803ff71ef4e480e8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 7 Nov 2015 19:25:59 +0600
Subject: [PATCH 3017/6219] [crunchyroll] Fix title extraction (Closes #7396)

---
 youtube_dl/extractor/crunchyroll.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 0c9b8ca02..4243f3e2e 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -287,7 +287,9 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
         if 'To view this, please log in to verify you are 18 or older.' in webpage:
             self.raise_login_required()
 
-        video_title = self._html_search_regex(r'<h1[^>]*>(.+?)</h1>', webpage, 'video_title', flags=re.DOTALL)
+        video_title = self._html_search_regex(
+            r'(?s)<h1[^>]*>((?:(?!<h1).)*?<span[^>]+itemprop=["\']title["\'][^>]*>(?:(?!<h1).)+?)</h1>',
+            webpage, 'video_title')
         video_title = re.sub(r' {2,}', ' ', video_title)
         video_description = self._html_search_regex(r'"description":"([^"]+)', webpage, 'video_description', default='')
         if not video_description:

From 2c740cf28d257d2a915195e7cc60f83e6690d2cc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 7 Nov 2015 19:29:09 +0600
Subject: [PATCH 3018/6219] [crunchyroll] Simplify description extraction

---
 youtube_dl/extractor/crunchyroll.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 4243f3e2e..9aa5d58b4 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -291,9 +291,8 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
             r'(?s)<h1[^>]*>((?:(?!<h1).)*?<span[^>]+itemprop=["\']title["\'][^>]*>(?:(?!<h1).)+?)</h1>',
             webpage, 'video_title')
         video_title = re.sub(r' {2,}', ' ', video_title)
-        video_description = self._html_search_regex(r'"description":"([^"]+)', webpage, 'video_description', default='')
-        if not video_description:
-            video_description = None
+        video_description = self._html_search_regex(
+            r'"description":"([^"]+)', webpage, 'video_description', default=None)
         video_upload_date = self._html_search_regex(
             [r'<div>Availability for free users:(.+?)</div>', r'<div>[^<>]+<span>\s*(.+?\d{4})\s*</span></div>'],
             webpage, 'video_upload_date', fatal=False, flags=re.DOTALL)

From 6d02b9a392d39c114d3fb58bf7965f62196ccecd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 7 Nov 2015 20:02:39 +0600
Subject: [PATCH 3019/6219] [crunchyroll] Fix description extraction

---
 youtube_dl/extractor/crunchyroll.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 9aa5d58b4..6e5999c72 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -21,6 +21,7 @@ from ..utils import (
     bytes_to_intlist,
     intlist_to_bytes,
     int_or_none,
+    lowercase_escape,
     remove_end,
     unified_strdate,
     urlencode_postdata,
@@ -104,7 +105,7 @@ class CrunchyrollIE(CrunchyrollBaseIE):
             'id': '589804',
             'ext': 'flv',
             'title': 'Culture Japan Episode 1 – Rebuilding Japan after the 3.11',
-            'description': 'md5:fe2743efedb49d279552926d0bd0cd9e',
+            'description': 'md5:2fbc01f90b87e8e9137296f37b461c12',
             'thumbnail': 're:^https?://.*\.jpg$',
             'uploader': 'Danny Choo Network',
             'upload_date': '20120213',
@@ -292,7 +293,10 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
             webpage, 'video_title')
         video_title = re.sub(r' {2,}', ' ', video_title)
         video_description = self._html_search_regex(
-            r'"description":"([^"]+)', webpage, 'video_description', default=None)
+            r'<script[^>]*>\s*.+?\[media_id=%s\].+?"description"\s*:\s*"([^"]+)' % video_id,
+            webpage, 'description', default=None)
+        if video_description:
+            video_description = lowercase_escape(video_description.replace(r'\r\n', '\n'))
         video_upload_date = self._html_search_regex(
             [r'<div>Availability for free users:(.+?)</div>', r'<div>[^<>]+<span>\s*(.+?\d{4})\s*</span></div>'],
             webpage, 'video_upload_date', fatal=False, flags=re.DOTALL)

From 3793090b1b1c1e3462b80dd3045a3573545cfb29 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 7 Nov 2015 16:54:35 +0100
Subject: [PATCH 3020/6219] [amp] Add generic extractor for Akamai AMP feeds
 and use it in dramafever and foxnews extractors

---
 youtube_dl/extractor/amp.py        | 84 ++++++++++++++++++++++++++++++
 youtube_dl/extractor/dramafever.py | 65 ++++-------------------
 youtube_dl/extractor/foxnews.py    | 64 ++++-------------------
 3 files changed, 105 insertions(+), 108 deletions(-)
 create mode 100644 youtube_dl/extractor/amp.py

diff --git a/youtube_dl/extractor/amp.py b/youtube_dl/extractor/amp.py
new file mode 100644
index 000000000..b573b9280
--- /dev/null
+++ b/youtube_dl/extractor/amp.py
@@ -0,0 +1,84 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    parse_iso8601,
+)
+
+
+class AMPIE(InfoExtractor):
+    def _get_media_node(self, item, name, default=None):
+        media_name = 'media-%s' % name
+        media_group = item.get('media-group') or item
+        return media_group.get(media_name) or item.get(media_name) or item.get(name, default)
+
+    # parse Akamai Adaptive Media Player feed
+    def _extract_feed_info(self, url):
+        item = self._download_json(
+            url, None,
+            'Downloading Akamai AMP feed',
+            'Unable to download Akamai AMP feed'
+            )['channel']['item']
+
+        video_id = item['guid']
+        
+        thumbnails = []
+        media_thumbnail = self._get_media_node(item, 'thumbnail')
+        if media_thumbnail:
+            if isinstance(media_thumbnail, dict):
+                media_thumbnail = [media_thumbnail]
+            for thumbnail_data in media_thumbnail:
+                thumbnail = thumbnail_data['@attributes']
+                thumbnails.append({
+                    'url': self._proto_relative_url(thumbnail['url'], 'http:'),
+                    'width': int_or_none(thumbnail.get('width')),
+                    'height': int_or_none(thumbnail.get('height')),
+                })
+
+        subtitles = {}
+        media_subtitle = self._get_media_node(item, 'subTitle')
+        if media_subtitle:
+            if isinstance(media_subtitle, dict):
+                media_subtitle = [media_subtitle]
+            for subtitle_data in media_subtitle:
+                subtitle = subtitle_data['@attributes']
+                lang = subtitle.get('lang') or 'en'
+                subtitles[lang] = [{'url': subtitle['href']}]
+
+        formats = []
+        media_content = self._get_media_node(item, 'content')
+        if isinstance(media_content, dict):
+            media_content = [media_content]
+        for media_data in media_content:
+            media = media_data['@attributes']
+            media_type = media['type']
+            if media_type == 'video/f4m':
+                f4m_formats = self._extract_f4m_formats(media['url'] + '?hdcore=3.4.0&plugin=aasp-3.4.0.132.124', video_id, f4m_id='hds', fatal=False)
+                if f4m_formats:
+                    formats.extend(f4m_formats)
+            elif media_type == 'application/x-mpegURL':
+                m3u8_formats = self._extract_m3u8_formats(media['url'], video_id, m3u8_id='hls', fatal=False)
+                if m3u8_formats:
+                    formats.extend(m3u8_formats)
+            else:
+                formats.append({
+                    'format_id': media_data['media-category']['@attributes']['label'],
+                    'url': media['url'],
+                    'preference': 1,
+                    'vbr': int_or_none(media.get('bitrate')),
+                    'filesize': int_or_none(media.get('fileSize')),
+                })
+
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': self._get_media_node(item, 'title'),
+            'description': self._get_media_node(item, 'description'),
+            'thumbnails': thumbnails,
+            'timestamp': parse_iso8601(item.get('pubDate'), ' '),
+            'duration': int_or_none(media_content[0].get('@attributes', {}).get('duration')),
+            'formats': formats,
+        }
diff --git a/youtube_dl/extractor/dramafever.py b/youtube_dl/extractor/dramafever.py
index 38e6597c8..80a928827 100644
--- a/youtube_dl/extractor/dramafever.py
+++ b/youtube_dl/extractor/dramafever.py
@@ -3,7 +3,7 @@ from __future__ import unicode_literals
 
 import itertools
 
-from .common import InfoExtractor
+from .amp import AMPIE
 from ..compat import (
     compat_HTTPError,
     compat_urllib_parse,
@@ -19,7 +19,7 @@ from ..utils import (
 )
 
 
-class DramaFeverBaseIE(InfoExtractor):
+class DramaFeverBaseIE(AMPIE):
     _LOGIN_URL = 'https://www.dramafever.com/accounts/login/'
     _NETRC_MACHINE = 'dramafever'
 
@@ -80,60 +80,24 @@ class DramaFeverIE(DramaFeverBaseIE):
             'timestamp': 1404336058,
             'upload_date': '20140702',
             'duration': 343,
-        }
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
     }
 
     def _real_extract(self, url):
         video_id = self._match_id(url).replace('/', '.')
 
         try:
-            feed = self._download_json(
-                'http://www.dramafever.com/amp/episode/feed.json?guid=%s' % video_id,
-                video_id, 'Downloading episode JSON')['channel']['item']
+            info = self._extract_feed_info('http://www.dramafever.com/amp/episode/feed.json?guid=%s' % video_id)
         except ExtractorError as e:
             if isinstance(e.cause, compat_HTTPError):
                 raise ExtractorError(
                     'Currently unavailable in your country.', expected=True)
             raise
 
-        media_group = feed.get('media-group', {})
-
-        formats = []
-        for media_content in media_group['media-content']:
-            src = media_content.get('@attributes', {}).get('url')
-            if not src:
-                continue
-            ext = determine_ext(src)
-            if ext == 'f4m':
-                formats.extend(self._extract_f4m_formats(
-                    src, video_id, f4m_id='hds'))
-            elif ext == 'm3u8':
-                formats.extend(self._extract_m3u8_formats(
-                    src, video_id, 'mp4', m3u8_id='hls'))
-            else:
-                formats.append({
-                    'url': src,
-                })
-        self._sort_formats(formats)
-
-        title = media_group.get('media-title')
-        description = media_group.get('media-description')
-        duration = int_or_none(media_group['media-content'][0].get('@attributes', {}).get('duration'))
-        thumbnail = self._proto_relative_url(
-            media_group.get('media-thumbnail', {}).get('@attributes', {}).get('url'))
-        timestamp = parse_iso8601(feed.get('pubDate'), ' ')
-
-        subtitles = {}
-        for media_subtitle in media_group.get('media-subTitle', []):
-            lang = media_subtitle.get('@attributes', {}).get('lang')
-            href = media_subtitle.get('@attributes', {}).get('href')
-            if not lang or not href:
-                continue
-            subtitles[lang] = [{
-                'ext': 'ttml',
-                'url': href,
-            }]
-
         series_id, episode_number = video_id.split('.')
         episode_info = self._download_json(
             # We only need a single episode info, so restricting page size to one episode
@@ -146,21 +110,12 @@ class DramaFeverIE(DramaFeverBaseIE):
             if value:
                 subfile = value[0].get('subfile') or value[0].get('new_subfile')
                 if subfile and subfile != 'http://www.dramafever.com/st/':
-                    subtitles.setdefault('English', []).append({
+                    info['subtitiles'].setdefault('English', []).append({
                         'ext': 'srt',
                         'url': subfile,
                     })
 
-        return {
-            'id': video_id,
-            'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
-            'timestamp': timestamp,
-            'duration': duration,
-            'formats': formats,
-            'subtitles': subtitles,
-        }
+        return info
 
 
 class DramaFeverSeriesIE(DramaFeverBaseIE):
diff --git a/youtube_dl/extractor/foxnews.py b/youtube_dl/extractor/foxnews.py
index 3a4a59135..0cd0f9fa8 100644
--- a/youtube_dl/extractor/foxnews.py
+++ b/youtube_dl/extractor/foxnews.py
@@ -2,14 +2,14 @@ from __future__ import unicode_literals
 
 import re
 
-from .common import InfoExtractor
+from .amp import AMPIE
 from ..utils import (
     parse_iso8601,
     int_or_none,
 )
 
 
-class FoxNewsIE(InfoExtractor):
+class FoxNewsIE(AMPIE):
     IE_DESC = 'Fox News and Fox Business Video'
     _VALID_URL = r'https?://(?P<host>video\.fox(?:news|business)\.com)/v/(?:video-embed\.html\?video_id=)?(?P<id>\d+)'
     _TESTS = [
@@ -20,10 +20,10 @@ class FoxNewsIE(InfoExtractor):
                 'id': '3937480',
                 'ext': 'flv',
                 'title': 'Frozen in Time',
-                'description': 'Doctors baffled by 16-year-old girl that is the size of a toddler',
+                'description': '16-year-old girl is size of toddler',
                 'duration': 265,
-                'timestamp': 1304411491,
-                'upload_date': '20110503',
+                #'timestamp': 1304411491,
+                #'upload_date': '20110503',
                 'thumbnail': 're:^https?://.*\.jpg$',
             },
         },
@@ -34,10 +34,10 @@ class FoxNewsIE(InfoExtractor):
                 'id': '3922535568001',
                 'ext': 'mp4',
                 'title': "Rep. Luis Gutierrez on if Obama's immigration plan is legal",
-                'description': "Congressman discusses the president's executive action",
+                'description': "Congressman discusses president's plan",
                 'duration': 292,
-                'timestamp': 1417662047,
-                'upload_date': '20141204',
+                #'timestamp': 1417662047,
+                #'upload_date': '20141204',
                 'thumbnail': 're:^https?://.*\.jpg$',
             },
         },
@@ -56,48 +56,6 @@ class FoxNewsIE(InfoExtractor):
         video_id = mobj.group('id')
         host = mobj.group('host')
 
-        video = self._download_json(
-            'http://%s/v/feed/video/%s.js?template=fox' % (host, video_id), video_id)
-
-        item = video['channel']['item']
-        title = item['title']
-        description = item['description']
-        timestamp = parse_iso8601(item['dc-date'])
-
-        media_group = item['media-group']
-        duration = None
-        formats = []
-        for media in media_group['media-content']:
-            attributes = media['@attributes']
-            video_url = attributes['url']
-            if video_url.endswith('.f4m'):
-                formats.extend(self._extract_f4m_formats(video_url + '?hdcore=3.4.0&plugin=aasp-3.4.0.132.124', video_id))
-            elif video_url.endswith('.m3u8'):
-                formats.extend(self._extract_m3u8_formats(video_url, video_id, 'flv'))
-            elif not video_url.endswith('.smil'):
-                duration = int_or_none(attributes.get('duration'))
-                formats.append({
-                    'url': video_url,
-                    'format_id': media['media-category']['@attributes']['label'],
-                    'preference': 1,
-                    'vbr': int_or_none(attributes.get('bitrate')),
-                    'filesize': int_or_none(attributes.get('fileSize'))
-                })
-        self._sort_formats(formats)
-
-        media_thumbnail = media_group['media-thumbnail']['@attributes']
-        thumbnails = [{
-            'url': media_thumbnail['url'],
-            'width': int_or_none(media_thumbnail.get('width')),
-            'height': int_or_none(media_thumbnail.get('height')),
-        }] if media_thumbnail else []
-
-        return {
-            'id': video_id,
-            'title': title,
-            'description': description,
-            'duration': duration,
-            'timestamp': timestamp,
-            'formats': formats,
-            'thumbnails': thumbnails,
-        }
+        info = self._extract_feed_info('http://%s/v/feed/video/%s.js?template=fox' % (host, video_id))
+        info['id'] = video_id
+        return info

From 63b728f06f00c2f1a45a67eddebd18bcdc36a753 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 7 Nov 2015 16:56:21 +0100
Subject: [PATCH 3021/6219] [bleacherreport] Add new Extractor

---
 youtube_dl/extractor/__init__.py       |   4 +
 youtube_dl/extractor/bleacherreport.py | 121 +++++++++++++++++++++++++
 2 files changed, 125 insertions(+)
 create mode 100644 youtube_dl/extractor/bleacherreport.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 94150a28f..4d65ece94 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -54,6 +54,10 @@ from .beatportpro import BeatportProIE
 from .bet import BetIE
 from .bild import BildIE
 from .bilibili import BiliBiliIE
+from .bleacherreport import (
+    BleacherReportIE,
+    BleacherReportCMSIE,
+)
 from .blinkx import BlinkxIE
 from .bliptv import BlipTVIE, BlipTVUserIE
 from .bloomberg import BloombergIE
diff --git a/youtube_dl/extractor/bleacherreport.py b/youtube_dl/extractor/bleacherreport.py
new file mode 100644
index 000000000..a55e696d2
--- /dev/null
+++ b/youtube_dl/extractor/bleacherreport.py
@@ -0,0 +1,121 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from .amp import AMPIE
+from ..utils import (
+    ExtractorError,
+    int_or_none,
+    parse_iso8601,
+)
+
+
+class BleacherReportIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?bleacherreport\.com/articles/(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'http://bleacherreport.com/articles/2496438-fsu-stat-projections-is-jalen-ramsey-best-defensive-player-in-college-football',
+        'md5': 'a3ffc3dc73afdbc2010f02d98f990f20',
+        'info_dict': {
+            'id': '2496438',
+            'ext': 'mp4',
+            'title': 'FSU Stat Projections: Is Jalen Ramsey Best Defensive Player in College Football?',
+            'uploader_id': 3992341,
+            'description': 'CFB, ACC, Florida State',
+            'timestamp': 1434380212,
+            'upload_date': '20150615',
+            'uploader': 'Team Stream Now ',
+        },
+        'add_ie': ['Ooyala'],
+    },{
+        'url': 'http://bleacherreport.com/articles/2586817-aussie-golfers-get-fright-of-their-lives-after-being-chased-by-angry-kangaroo',
+        'md5': 'af5f90dc9c7ba1c19d0a3eac806bbf50',
+        'info_dict': {
+            'id': '2586817',
+            'ext': 'mp4',
+            'title': 'Aussie Golfers Get Fright of Their Lives After Being Chased by Angry Kangaroo',
+            'timestamp': 1446839961,
+            'uploader': 'Sean Fay',
+            'description': 'md5:e95afafa43619816552723878b3b0a84',
+            'uploader_id': 6466954,
+            'upload_date': '20151011',
+        },
+        'add_ie': ['Youtube'],
+    },{
+        'url': 'http://bleacherreport.com/articles/2496438-fsu-stat-projections-is-jalen-ramsey-best-defensive-player-in-college-football',
+        'md5': 'a3ffc3dc73afdbc2010f02d98f990f20',
+        'info_dict': {
+            'id': '2496438',
+            'ext': 'mp4',
+            'title': 'FSU Stat Projections: Is Jalen Ramsey Best Defensive Player in College Football?',
+            'upload_date': '20150615',
+            'uploader': 'Team Stream Now ',
+            'timestamp': 1434380212,
+            'description': 'CFB, ACC, Florida State',
+            'uploader_id': 3992341,
+        },
+        'add_ie': ['Vine'],
+    }]
+
+    def _real_extract(self, url):
+        article_id = self._match_id(url)
+
+        article_data = self._download_json('http://api.bleacherreport.com/api/v1/articles/%s' % article_id, article_id)['article']
+
+        thumbnails = []
+        primary_photo = article_data.get('primaryPhoto')
+        if primary_photo:
+            thumbnails = [{
+                'url': primary_photo['url'],
+                'width': primary_photo.get('width'),
+                'height': primary_photo.get('height'),
+            }]
+
+        info = {
+            '_type': 'url_transparent',
+            'id': article_id,
+            'title': article_data['title'],
+            'uploader': article_data.get('author', {}).get('name'),
+            'uploader_id': article_data.get('authorId'),
+            'timestamp': parse_iso8601(article_data.get('createdAt')),
+            'thumbnails': thumbnails,
+            'comment_count': int_or_none(article_data.get('commentsCount')),
+            'view_count': int_or_none(article_data.get('hitCount')),
+        }
+
+        video = article_data.get('video')
+        if video:
+            video_type = video['type']
+            if video_type == 'cms.bleacherreport.com':
+                info['url'] = 'http://bleacherreport.com/video_embed?id=%s' % video['id']
+            elif video_type == 'ooyala.com':
+                info['url'] = 'ooyala:%s' % video['id']
+            elif video_type == 'youtube.com':
+                info['url'] = video['id']
+            elif video_type == 'vine.co':
+                info['url'] = 'https://vine.co/v/%s' % video['id']
+            else:
+                info['url'] = video_type + video['id']
+            return info
+        else:
+            raise ExtractorError('no video in the article', expected=True)
+
+
+class BleacherReportCMSIE(AMPIE):
+    _VALID_URL = r'https?://(?:www\.)?bleacherreport\.com/video_embed\?id=(?P<id>[0-9a-f-]{36})'
+    _TESTS = [{
+        'url': 'http://bleacherreport.com/video_embed?id=8fd44c2f-3dc5-4821-9118-2c825a98c0e1',
+        'md5': 'f0ca220af012d4df857b54f792c586bb',
+        'info_dict': {
+            'id': '8fd44c2f-3dc5-4821-9118-2c825a98c0e1',
+            'ext': 'flv',
+            'title': 'Cena vs. Rollins Would Expose the Heavyweight Division',
+            'description': 'md5:984afb4ade2f9c0db35f3267ed88b36e',
+        },
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        info = self._extract_feed_info('http://cms.bleacherreport.com/media/items/%s/akamai.json' % video_id)
+        info['id'] = video_id
+        return info

From cff551c0b0ed8eb55c1ab63ec669c07a51aa4998 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 7 Nov 2015 18:43:22 +0100
Subject: [PATCH 3022/6219] [googleplus] Fix extraction of formats

---
 youtube_dl/extractor/googleplus.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/googleplus.py b/youtube_dl/extractor/googleplus.py
index fcefe54cd..731bacd67 100644
--- a/youtube_dl/extractor/googleplus.py
+++ b/youtube_dl/extractor/googleplus.py
@@ -61,7 +61,7 @@ class GooglePlusIE(InfoExtractor):
             'width': int(width),
             'height': int(height),
         } for width, height, video_url in re.findall(
-            r'\d+,(\d+),(\d+),"(https?://redirector\.googlevideo\.com.*?)"', webpage)]
+            r'\d+,(\d+),(\d+),"(https?://[^.]+\.googleusercontent.com.*?)"', webpage)]
         self._sort_formats(formats)
 
         return {

From ee4337d100f68bbb2ae795101d4c391b522ec753 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 6 Nov 2015 20:16:14 +0100
Subject: [PATCH 3023/6219] [videolecture] add support for multi part videos

---
 youtube_dl/extractor/videolecturesnet.py | 95 +++++++++++++++++-------
 1 file changed, 70 insertions(+), 25 deletions(-)

diff --git a/youtube_dl/extractor/videolecturesnet.py b/youtube_dl/extractor/videolecturesnet.py
index 649ac9433..351706362 100644
--- a/youtube_dl/extractor/videolecturesnet.py
+++ b/youtube_dl/extractor/videolecturesnet.py
@@ -10,17 +10,19 @@ from ..compat import (
 from ..utils import (
     ExtractorError,
     parse_duration,
+    js_to_json,
+    parse_iso8601,
 )
 
 
 class VideoLecturesNetIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?videolectures\.net/(?P<id>[^/#?]+)/*(?:[#?].*)?$'
+    _VALID_URL = r'http://(?:www\.)?videolectures\.net/(?P<id>[^/]+)(?:/video/(?P<part>\d+))?'
     IE_NAME = 'videolectures.net'
 
     _TESTS = [{
         'url': 'http://videolectures.net/promogram_igor_mekjavic_eng/',
         'info_dict': {
-            'id': 'promogram_igor_mekjavic_eng',
+            'id': '20171_part1',
             'ext': 'mp4',
             'title': 'Automatics, robotics and biocybernetics',
             'description': 'md5:815fc1deb6b3a2bff99de2d5325be482',
@@ -32,7 +34,7 @@ class VideoLecturesNetIE(InfoExtractor):
         # video with invalid direct format links (HTTP 403)
         'url': 'http://videolectures.net/russir2010_filippova_nlp/',
         'info_dict': {
-            'id': 'russir2010_filippova_nlp',
+            'id': '14891_part1',
             'ext': 'flv',
             'title': 'NLP at Google',
             'description': 'md5:fc7a6d9bf0302d7cc0e53f7ca23747b3',
@@ -46,37 +48,80 @@ class VideoLecturesNetIE(InfoExtractor):
     }, {
         'url': 'http://videolectures.net/deeplearning2015_montreal/',
         'info_dict': {
-            'id': 'deeplearning2015_montreal',
+            'id': '23181',
             'title': 'Deep Learning Summer School, Montreal 2015',
-            'description': 'md5:90121a40cc6926df1bf04dcd8563ed3b',
+            'description': 'md5:0533a85e4bd918df52a01f0e1ebe87b7',
+            'timestamp': 1438560000,
         },
         'playlist_count': 30,
+    }, {
+        # multi part lecture
+        'url': 'http://videolectures.net/mlss09uk_bishop_ibi/',
+        'info_dict': {
+            'id': '9737',
+            'title': 'Introduction To Bayesian Inference',
+            'timestamp': 1251622800,
+        },
+        'playlist': [{
+            'info_dict': {
+                'id': '9737_part1',
+                'ext': 'wmv',
+                'title': 'Introduction To Bayesian Inference',
+            },
+        }, {
+            'info_dict': {
+                'id': '9737_part2',
+                'ext': 'wmv',
+                'title': 'Introduction To Bayesian Inference',
+            },
+        }],
+        'playlist_count': 2,
     }]
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        lecture_slug, part = re.match(self._VALID_URL, url).groups()
 
-        smil_url = 'http://videolectures.net/%s/video/1/smil.xml' % video_id
+        webpage = self._download_webpage(url, lecture_slug)
 
-        try:
-            smil = self._download_smil(smil_url, video_id)
-        except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 404:
-                # Probably a playlist
-                webpage = self._download_webpage(url, video_id)
-                entries = [
-                    self.url_result(compat_urlparse.urljoin(url, video_url), 'VideoLecturesNet')
-                    for _, video_url in re.findall(r'<a[^>]+href=(["\'])(.+?)\1[^>]+id=["\']lec=\d+', webpage)]
-                playlist_title = self._html_search_meta('title', webpage, 'title', fatal=True)
-                playlist_description = self._html_search_meta('description', webpage, 'description')
-                return self.playlist_result(entries, video_id, playlist_title, playlist_description)
+        cfg = self._parse_json(self._search_regex(r'cfg\s*:\s*({[^}]+})', webpage, 'cfg'), lecture_slug, js_to_json)
 
-        info = self._parse_smil(smil, smil_url, video_id)
+        lecture_id = str(cfg['obj_id'])
 
-        info['id'] = video_id
+        lecture_data = self._download_json('%s/site/api/lecture/%s?format=json' % (self._proto_relative_url(cfg['livepipe'], 'http:'), lecture_id), lecture_id)['lecture'][0]
 
-        switch = smil.find('.//switch')
-        if switch is not None:
-            info['duration'] = parse_duration(switch.attrib.get('dur'))
+        lecture_info = {
+            'id': lecture_id,
+            'display_id': lecture_slug,
+            'title': lecture_data['title'],
+            'timestamp': parse_iso8601(lecture_data.get('time')),
+            'description': lecture_data.get('description_wiki'),
+            'thumbnail': lecture_data.get('thumb'),
+        }
 
-        return info
+        entries = []
+        parts = cfg.get('videos')
+        if parts:
+            if len(parts) == 1:
+                part = str(parts[0])
+            if part:
+                smil_url = 'http://videolectures.net/%s/video/%s/smil.xml' % (lecture_slug, part)
+                smil = self._download_smil(smil_url, lecture_id)
+                info = self._parse_smil(smil, smil_url, lecture_id)
+                info['id'] = '%s_part%s' % (lecture_id, part)
+                switch = smil.find('.//switch')
+                if switch is not None:
+                    info['duration'] = parse_duration(switch.attrib.get('dur'))
+                return info
+            else:
+                for part in parts:
+                    entries.append(self.url_result('http://videolectures.net/%s/video/%s' % (lecture_slug, part), 'VideoLecturesNet'))
+                lecture_info['_type'] = 'multi_video'
+        else:
+            # Probably a playlist
+            entries = [
+                self.url_result(compat_urlparse.urljoin(url, video_url), 'VideoLecturesNet')
+                for _, video_url in re.findall(r'<a[^>]+href=(["\'])(.+?)\1[^>]+id=["\']lec=\d+', webpage)]
+            lecture_info['_type'] = 'playlist'
+
+        lecture_info['entries'] = entries
+        return lecture_info

From a06bf87a2c6009d82ec28afe566f653b3deb11bf Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 6 Nov 2015 21:23:41 +0100
Subject: [PATCH 3024/6219] [viidea] add support for sites using viidea service

---
 youtube_dl/extractor/__init__.py              |  2 +-
 .../{videolecturesnet.py => viidea.py}        | 33 ++++++++++++++-----
 2 files changed, 26 insertions(+), 9 deletions(-)
 rename youtube_dl/extractor/{videolecturesnet.py => viidea.py} (77%)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 94150a28f..0a90da73c 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -724,7 +724,6 @@ from .vh1 import VH1IE
 from .vice import ViceIE
 from .viddler import ViddlerIE
 from .videodetective import VideoDetectiveIE
-from .videolecturesnet import VideoLecturesNetIE
 from .videofyme import VideofyMeIE
 from .videomega import VideoMegaIE
 from .videopremium import VideoPremiumIE
@@ -734,6 +733,7 @@ from .vidme import VidmeIE
 from .vidzi import VidziIE
 from .vier import VierIE, VierVideosIE
 from .viewster import ViewsterIE
+from .viidea import ViideaIE
 from .vimeo import (
     VimeoIE,
     VimeoAlbumIE,
diff --git a/youtube_dl/extractor/videolecturesnet.py b/youtube_dl/extractor/viidea.py
similarity index 77%
rename from youtube_dl/extractor/videolecturesnet.py
rename to youtube_dl/extractor/viidea.py
index 351706362..71fb298e6 100644
--- a/youtube_dl/extractor/videolecturesnet.py
+++ b/youtube_dl/extractor/viidea.py
@@ -15,9 +15,23 @@ from ..utils import (
 )
 
 
-class VideoLecturesNetIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?videolectures\.net/(?P<id>[^/]+)(?:/video/(?P<part>\d+))?'
-    IE_NAME = 'videolectures.net'
+class ViideaIE(InfoExtractor):
+    _VALID_URL = r'''(?x)http://(?:www\.)?(?:
+            videolectures\.net|
+            flexilearn\.viidea\.net|
+            presentations\.ocwconsortium\.org|
+            video\.travel-zoom\.si|
+            video\.pomp-forum\.si|
+            tv\.nil\.si|
+            video\.hekovnik.com|
+            video\.szko\.si|
+            kpk\.viidea\.com|
+            inside\.viidea\.net|
+            video\.kiberpipa\.org|
+            bvvideo\.si|
+            kongres\.viidea\.net|
+            edemokracija\.viidea\.com
+        )(?:/lecture)?/(?P<id>[^/]+)(?:/video/(?P<part>\d+))?'''
 
     _TESTS = [{
         'url': 'http://videolectures.net/promogram_igor_mekjavic_eng/',
@@ -87,7 +101,9 @@ class VideoLecturesNetIE(InfoExtractor):
 
         lecture_id = str(cfg['obj_id'])
 
-        lecture_data = self._download_json('%s/site/api/lecture/%s?format=json' % (self._proto_relative_url(cfg['livepipe'], 'http:'), lecture_id), lecture_id)['lecture'][0]
+        base_url = self._proto_relative_url(cfg['livepipe'], 'http:')
+
+        lecture_data = self._download_json('%s/site/api/lecture/%s?format=json' % (base_url, lecture_id), lecture_id)['lecture'][0]
 
         lecture_info = {
             'id': lecture_id,
@@ -104,7 +120,7 @@ class VideoLecturesNetIE(InfoExtractor):
             if len(parts) == 1:
                 part = str(parts[0])
             if part:
-                smil_url = 'http://videolectures.net/%s/video/%s/smil.xml' % (lecture_slug, part)
+                smil_url = '%s/%s/video/%s/smil.xml' % (base_url, lecture_slug, part)
                 smil = self._download_smil(smil_url, lecture_id)
                 info = self._parse_smil(smil, smil_url, lecture_id)
                 info['id'] = '%s_part%s' % (lecture_id, part)
@@ -114,13 +130,14 @@ class VideoLecturesNetIE(InfoExtractor):
                 return info
             else:
                 for part in parts:
-                    entries.append(self.url_result('http://videolectures.net/%s/video/%s' % (lecture_slug, part), 'VideoLecturesNet'))
+                    entries.append(self.url_result('%s/video/%s' % (base_url, lecture_id, part), 'Viidea'))
                 lecture_info['_type'] = 'multi_video'
         else:
             # Probably a playlist
+            playlist_webpage = self._download_webpage('%s/site/ajax/drilldown/?id=%s' % (base_url, lecture_id), lecture_id)
             entries = [
-                self.url_result(compat_urlparse.urljoin(url, video_url), 'VideoLecturesNet')
-                for _, video_url in re.findall(r'<a[^>]+href=(["\'])(.+?)\1[^>]+id=["\']lec=\d+', webpage)]
+                self.url_result(compat_urlparse.urljoin(url, video_url), 'Viidea')
+                for _, video_url in re.findall(r'<a[^>]+href=(["\'])(.+?)\1[^>]+id=["\']lec=\d+', playlist_webpage)]
             lecture_info['_type'] = 'playlist'
 
         lecture_info['entries'] = entries

From 8e3a2bd6200660f9fb9d485b1c924fa5462bd566 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 7 Nov 2015 17:43:23 +0100
Subject: [PATCH 3025/6219] [viidea] fix _VALID_URL regex and tests

---
 youtube_dl/extractor/viidea.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/viidea.py b/youtube_dl/extractor/viidea.py
index 71fb298e6..ae9a42737 100644
--- a/youtube_dl/extractor/viidea.py
+++ b/youtube_dl/extractor/viidea.py
@@ -31,7 +31,7 @@ class ViideaIE(InfoExtractor):
             bvvideo\.si|
             kongres\.viidea\.net|
             edemokracija\.viidea\.com
-        )(?:/lecture)?/(?P<id>[^/]+)(?:/video/(?P<part>\d+))?'''
+        )(?:/lecture)?/(?P<id>[^/]+)(?:/video/(?P<part>\d+))?/*(?:[#?].*)?$'''
 
     _TESTS = [{
         'url': 'http://videolectures.net/promogram_igor_mekjavic_eng/',
@@ -130,7 +130,7 @@ class ViideaIE(InfoExtractor):
                 return info
             else:
                 for part in parts:
-                    entries.append(self.url_result('%s/video/%s' % (base_url, lecture_id, part), 'Viidea'))
+                    entries.append(self.url_result('%s/%s/video/%s' % (base_url, lecture_slug, part), 'Viidea'))
                 lecture_info['_type'] = 'multi_video'
         else:
             # Probably a playlist

From 6fdb39ded15c6276b49fa67cb517bf1fed63af35 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 7 Nov 2015 20:38:33 +0100
Subject: [PATCH 3026/6219] [viidia] Cleaup

[viidea] extract playlist if lecture is an event

[viidia] use compat_str
---
 youtube_dl/extractor/viidea.py | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/viidea.py b/youtube_dl/extractor/viidea.py
index ae9a42737..2541a36ed 100644
--- a/youtube_dl/extractor/viidea.py
+++ b/youtube_dl/extractor/viidea.py
@@ -4,11 +4,10 @@ import re
 
 from .common import InfoExtractor
 from ..compat import (
-    compat_HTTPError,
     compat_urlparse,
+    compat_str,
 )
 from ..utils import (
-    ExtractorError,
     parse_duration,
     js_to_json,
     parse_iso8601,
@@ -97,9 +96,9 @@ class ViideaIE(InfoExtractor):
 
         webpage = self._download_webpage(url, lecture_slug)
 
-        cfg = self._parse_json(self._search_regex(r'cfg\s*:\s*({[^}]+})', webpage, 'cfg'), lecture_slug, js_to_json)
+        cfg = self._parse_json(self._search_regex([r'cfg\s*:\s*({.+?}),[\da-zA-Z_]:\(?function', r'cfg\s*:\s*({[^}]+})'], webpage, 'cfg'), lecture_slug, js_to_json)
 
-        lecture_id = str(cfg['obj_id'])
+        lecture_id = compat_str(cfg['obj_id'])
 
         base_url = self._proto_relative_url(cfg['livepipe'], 'http:')
 
@@ -118,7 +117,7 @@ class ViideaIE(InfoExtractor):
         parts = cfg.get('videos')
         if parts:
             if len(parts) == 1:
-                part = str(parts[0])
+                part = compat_str(parts[0])
             if part:
                 smil_url = '%s/%s/video/%s/smil.xml' % (base_url, lecture_slug, part)
                 smil = self._download_smil(smil_url, lecture_id)
@@ -132,7 +131,7 @@ class ViideaIE(InfoExtractor):
                 for part in parts:
                     entries.append(self.url_result('%s/%s/video/%s' % (base_url, lecture_slug, part), 'Viidea'))
                 lecture_info['_type'] = 'multi_video'
-        else:
+        if not parts or lecture_data.get('type') == 'evt':
             # Probably a playlist
             playlist_webpage = self._download_webpage('%s/site/ajax/drilldown/?id=%s' % (base_url, lecture_id), lecture_id)
             entries = [

From e8ce2375e0851e65c4882002297404825fe1045e Mon Sep 17 00:00:00 2001
From: Sergey M? <dstftw@gmail.com>
Date: Sun, 8 Nov 2015 06:54:27 +0600
Subject: [PATCH 3027/6219] [viidea] Improve and cleanup (Closes #7390)

* Optimize requests for multipart videos
* Fix cfg regex
* Improve titles and identifiers
---
 youtube_dl/extractor/viidea.py | 99 ++++++++++++++++++++++++----------
 1 file changed, 72 insertions(+), 27 deletions(-)

diff --git a/youtube_dl/extractor/viidea.py b/youtube_dl/extractor/viidea.py
index 2541a36ed..525e303d4 100644
--- a/youtube_dl/extractor/viidea.py
+++ b/youtube_dl/extractor/viidea.py
@@ -35,35 +35,42 @@ class ViideaIE(InfoExtractor):
     _TESTS = [{
         'url': 'http://videolectures.net/promogram_igor_mekjavic_eng/',
         'info_dict': {
-            'id': '20171_part1',
+            'id': '20171',
+            'display_id': 'promogram_igor_mekjavic_eng',
             'ext': 'mp4',
             'title': 'Automatics, robotics and biocybernetics',
             'description': 'md5:815fc1deb6b3a2bff99de2d5325be482',
+            'thumbnail': 're:http://.*\.jpg',
+            'timestamp': 1372349289,
             'upload_date': '20130627',
             'duration': 565,
-            'thumbnail': 're:http://.*\.jpg',
         },
     }, {
         # video with invalid direct format links (HTTP 403)
         'url': 'http://videolectures.net/russir2010_filippova_nlp/',
         'info_dict': {
-            'id': '14891_part1',
+            'id': '14891',
+            'display_id': 'russir2010_filippova_nlp',
             'ext': 'flv',
             'title': 'NLP at Google',
             'description': 'md5:fc7a6d9bf0302d7cc0e53f7ca23747b3',
-            'duration': 5352,
             'thumbnail': 're:http://.*\.jpg',
+            'timestamp': 1284375600,
+            'upload_date': '20100913',
+            'duration': 5352,
         },
         'params': {
             # rtmp download
             'skip_download': True,
         },
     }, {
+        # event playlist
         'url': 'http://videolectures.net/deeplearning2015_montreal/',
         'info_dict': {
             'id': '23181',
             'title': 'Deep Learning Summer School, Montreal 2015',
             'description': 'md5:0533a85e4bd918df52a01f0e1ebe87b7',
+            'thumbnail': 're:http://.*\.jpg',
             'timestamp': 1438560000,
         },
         'playlist_count': 30,
@@ -72,37 +79,54 @@ class ViideaIE(InfoExtractor):
         'url': 'http://videolectures.net/mlss09uk_bishop_ibi/',
         'info_dict': {
             'id': '9737',
+            'display_id': 'mlss09uk_bishop_ibi',
             'title': 'Introduction To Bayesian Inference',
+            'thumbnail': 're:http://.*\.jpg',
             'timestamp': 1251622800,
         },
         'playlist': [{
             'info_dict': {
                 'id': '9737_part1',
+                'display_id': 'mlss09uk_bishop_ibi_part1',
                 'ext': 'wmv',
-                'title': 'Introduction To Bayesian Inference',
+                'title': 'Introduction To Bayesian Inference (Part 1)',
+                'thumbnail': 're:http://.*\.jpg',
+                'duration': 4622,
+                'timestamp': 1251622800,
+                'upload_date': '20090830',
             },
         }, {
             'info_dict': {
                 'id': '9737_part2',
+                'display_id': 'mlss09uk_bishop_ibi_part2',
                 'ext': 'wmv',
-                'title': 'Introduction To Bayesian Inference',
+                'title': 'Introduction To Bayesian Inference (Part 2)',
+                'thumbnail': 're:http://.*\.jpg',
+                'duration': 5641,
+                'timestamp': 1251622800,
+                'upload_date': '20090830',
             },
         }],
         'playlist_count': 2,
     }]
 
     def _real_extract(self, url):
-        lecture_slug, part = re.match(self._VALID_URL, url).groups()
+        lecture_slug, explicit_part_id = re.match(self._VALID_URL, url).groups()
 
         webpage = self._download_webpage(url, lecture_slug)
 
-        cfg = self._parse_json(self._search_regex([r'cfg\s*:\s*({.+?}),[\da-zA-Z_]:\(?function', r'cfg\s*:\s*({[^}]+})'], webpage, 'cfg'), lecture_slug, js_to_json)
+        cfg = self._parse_json(self._search_regex(
+            [r'cfg\s*:\s*({.+?})\s*,\s*[\da-zA-Z_]+\s*:\s*\(?\s*function',
+             r'cfg\s*:\s*({[^}]+})'],
+            webpage, 'cfg'), lecture_slug, js_to_json)
 
         lecture_id = compat_str(cfg['obj_id'])
 
         base_url = self._proto_relative_url(cfg['livepipe'], 'http:')
 
-        lecture_data = self._download_json('%s/site/api/lecture/%s?format=json' % (base_url, lecture_id), lecture_id)['lecture'][0]
+        lecture_data = self._download_json(
+            '%s/site/api/lecture/%s?format=json' % (base_url, lecture_id),
+            lecture_id)['lecture'][0]
 
         lecture_info = {
             'id': lecture_id,
@@ -113,31 +137,52 @@ class ViideaIE(InfoExtractor):
             'thumbnail': lecture_data.get('thumb'),
         }
 
-        entries = []
-        parts = cfg.get('videos')
+        playlist_entries = []
+        lecture_type = lecture_data.get('type')
+        parts = [compat_str(video) for video in cfg.get('videos', [])]
         if parts:
-            if len(parts) == 1:
-                part = compat_str(parts[0])
-            if part:
-                smil_url = '%s/%s/video/%s/smil.xml' % (base_url, lecture_slug, part)
+            multipart = len(parts) > 1
+
+            def extract_part(part_id):
+                smil_url = '%s/%s/video/%s/smil.xml' % (base_url, lecture_slug, part_id)
                 smil = self._download_smil(smil_url, lecture_id)
                 info = self._parse_smil(smil, smil_url, lecture_id)
-                info['id'] = '%s_part%s' % (lecture_id, part)
+                info['id'] = lecture_id if not multipart else '%s_part%s' % (lecture_id, part_id)
+                info['display_id'] = lecture_slug if not multipart else '%s_part%s' % (lecture_slug, part_id)
+                if multipart:
+                    info['title'] += ' (Part %s)' % part_id
                 switch = smil.find('.//switch')
                 if switch is not None:
                     info['duration'] = parse_duration(switch.attrib.get('dur'))
-                return info
+                item_info = lecture_info.copy()
+                item_info.update(info)
+                return item_info
+
+            if explicit_part_id or not multipart:
+                result = extract_part(explicit_part_id or parts[0])
             else:
-                for part in parts:
-                    entries.append(self.url_result('%s/%s/video/%s' % (base_url, lecture_slug, part), 'Viidea'))
-                lecture_info['_type'] = 'multi_video'
-        if not parts or lecture_data.get('type') == 'evt':
-            # Probably a playlist
-            playlist_webpage = self._download_webpage('%s/site/ajax/drilldown/?id=%s' % (base_url, lecture_id), lecture_id)
+                result = {
+                    '_type': 'multi_video',
+                    'entries': [extract_part(part) for part in parts],
+                }
+                result.update(lecture_info)
+
+            # Immediately return explicitly requested part or non event item
+            if explicit_part_id or lecture_type != 'evt':
+                return result
+
+            playlist_entries.append(result)
+
+        # It's probably a playlist
+        if not parts or lecture_type == 'evt':
+            playlist_webpage = self._download_webpage(
+                '%s/site/ajax/drilldown/?id=%s' % (base_url, lecture_id), lecture_id)
             entries = [
                 self.url_result(compat_urlparse.urljoin(url, video_url), 'Viidea')
-                for _, video_url in re.findall(r'<a[^>]+href=(["\'])(.+?)\1[^>]+id=["\']lec=\d+', playlist_webpage)]
-            lecture_info['_type'] = 'playlist'
+                for _, video_url in re.findall(
+                    r'<a[^>]+href=(["\'])(.+?)\1[^>]+id=["\']lec=\d+', playlist_webpage)]
+            playlist_entries.extend(entries)
 
-        lecture_info['entries'] = entries
-        return lecture_info
+        playlist = self.playlist_result(playlist_entries, lecture_id)
+        playlist.update(lecture_info)
+        return playlist

From d5c181a14e08198e400932d591b47683a630c8c1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 8 Nov 2015 11:49:51 +0100
Subject: [PATCH 3028/6219] [movieclips] Fix extraction (fixes #7404)

They use theplatform now.
Changed the test, because the old one seems to be georestricted.
---
 youtube_dl/extractor/movieclips.py | 77 ++++++++----------------------
 1 file changed, 19 insertions(+), 58 deletions(-)

diff --git a/youtube_dl/extractor/movieclips.py b/youtube_dl/extractor/movieclips.py
index 04e17d055..e06828b55 100644
--- a/youtube_dl/extractor/movieclips.py
+++ b/youtube_dl/extractor/movieclips.py
@@ -1,80 +1,41 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..compat import (
-    compat_str,
-)
-from ..utils import (
-    ExtractorError,
-    clean_html,
+    compat_urllib_request,
 )
 
 
 class MovieClipsIE(InfoExtractor):
-    _VALID_URL = r'https?://movieclips\.com/(?P<id>[\da-zA-Z]+)(?:-(?P<display_id>[\da-z-]+))?'
+    _VALID_URL = r'https?://(?:www.)?movieclips\.com/videos/(?P<id>[^/?#]+)'
     _TEST = {
-        'url': 'http://movieclips.com/Wy7ZU-my-week-with-marilyn-movie-do-you-love-me/',
+        'url': 'http://www.movieclips.com/videos/warcraft-trailer-1-561180739597?autoPlay=true&playlistId=5',
         'info_dict': {
-            'id': 'Wy7ZU',
-            'display_id': 'my-week-with-marilyn-movie-do-you-love-me',
+            'id': 'pKIGmG83AqD9',
+            'display_id': 'warcraft-trailer-1-561180739597',
             'ext': 'mp4',
-            'title': 'My Week with Marilyn - Do You Love Me?',
-            'description': 'md5:e86795bd332fe3cff461e7c8dc542acb',
+            'title': 'Warcraft Trailer 1',
+            'description': 'Watch Trailer 1 from Warcraft (2016). Legendary’s WARCRAFT is a 3D epic adventure of world-colliding conflict based.',
             'thumbnail': 're:^https?://.*\.jpg$',
         },
-        'params': {
-            # rtmp download
-            'skip_download': True,
-        }
+        'add_ie': ['ThePlatform'],
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        display_id = mobj.group('display_id')
-        show_id = display_id or video_id
+        display_id = self._match_id(url)
 
-        config = self._download_xml(
-            'http://config.movieclips.com/player/config/%s' % video_id,
-            show_id, 'Downloading player config')
-
-        if config.find('./country-region').text == 'false':
-            raise ExtractorError(
-                '%s said: %s' % (self.IE_NAME, config.find('./region_alert').text), expected=True)
-
-        properties = config.find('./video/properties')
-        smil_file = properties.attrib['smil_file']
-
-        smil = self._download_xml(smil_file, show_id, 'Downloading SMIL')
-        base_url = smil.find('./head/meta').attrib['base']
-
-        formats = []
-        for video in smil.findall('./body/switch/video'):
-            vbr = int(video.attrib['system-bitrate']) / 1000
-            src = video.attrib['src']
-            formats.append({
-                'url': base_url,
-                'play_path': src,
-                'ext': src.split(':')[0],
-                'vbr': vbr,
-                'format_id': '%dk' % vbr,
-            })
-
-        self._sort_formats(formats)
-
-        title = '%s - %s' % (properties.attrib['clip_movie_title'], properties.attrib['clip_title'])
-        description = clean_html(compat_str(properties.attrib['clip_description']))
-        thumbnail = properties.attrib['image']
-        categories = properties.attrib['clip_categories'].split(',')
+        req = compat_urllib_request.Request(url)
+        # it doesn't work if it thinks the browser it's too old
+        req.add_header('User-Agent', 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20150101 Firefox/43.0 (Chrome)')
+        webpage = self._download_webpage(req, display_id)
+        theplatform_link = self._html_search_regex(r'src="(http://player.theplatform.com/p/.*?)"', webpage, 'theplatform link')
+        title = self._html_search_regex(r'<title[^>]*>([^>]+)-\s*\d+\s*|\s*Movieclips.com</title>', webpage, 'title')
+        description = self._html_search_meta('description', webpage)
 
         return {
-            'id': video_id,
-            'display_id': display_id,
+            '_type': 'url_transparent',
+            'url': theplatform_link,
             'title': title,
+            'display_id': display_id,
             'description': description,
-            'thumbnail': thumbnail,
-            'categories': categories,
-            'formats': formats,
         }

From 937511dfc01c3d00c35a00f78c2b6f989b4d46e3 Mon Sep 17 00:00:00 2001
From: Frans de Jonge <fransdejonge@gmail.com>
Date: Sat, 7 Nov 2015 22:55:02 +0100
Subject: [PATCH 3029/6219] Added support for the RTBF OUFtivi subpage

---
 youtube_dl/extractor/rtbf.py | 41 ++++++++++++++++++++++++++----------
 1 file changed, 30 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/rtbf.py b/youtube_dl/extractor/rtbf.py
index 04a66df90..e75b45112 100644
--- a/youtube_dl/extractor/rtbf.py
+++ b/youtube_dl/extractor/rtbf.py
@@ -9,17 +9,36 @@ from ..utils import (
 
 
 class RTBFIE(InfoExtractor):
-    _VALID_URL = r'https?://www.rtbf.be/video/[^\?]+\?id=(?P<id>\d+)'
-    _TEST = {
-        'url': 'https://www.rtbf.be/video/detail_les-diables-au-coeur-episode-2?id=1921274',
-        'md5': '799f334ddf2c0a582ba80c44655be570',
-        'info_dict': {
-            'id': '1921274',
-            'ext': 'mp4',
-            'title': 'Les Diables au coeur (épisode 2)',
-            'duration': 3099,
-        }
-    }
+    _VALID_URL = r'''(?x)
+        https?://www\.rtbf\.be/
+            (?:
+                video/[^\?]+\?id=|
+                ouftivi/heros/[^&]+&videoId=
+            )
+        (?P<id>\d+)
+    '''
+    _TESTS = [
+        {
+            'url': 'https://www.rtbf.be/video/detail_les-diables-au-coeur-episode-2?id=1921274',
+            'md5': '799f334ddf2c0a582ba80c44655be570',
+            'info_dict': {
+                'id': '1921274',
+                'ext': 'mp4',
+                'title': 'Les Diables au coeur (épisode 2)',
+                'duration': 3099,
+            }
+        },
+        {
+            'url': 'http://www.rtbf.be/ouftivi/heros/detail_scooby-doo-mysteres-associes?id=1097&videoId=2057442',
+            'md5': '25aea17e949e1e0c7c41270d60d25f22',
+            'info_dict': {
+                'id': '2057442',
+                'ext': 'mp4',
+                'title': 'Scooby-Doo, myst\xe8res associ\xe9s',
+                'duration': 1279,
+            }
+        },
+    ]
 
     _QUALITIES = [
         ('mobile', 'mobile'),

From fda2717ef9d429358d5816582590d15d18f9109f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 8 Nov 2015 16:56:20 +0600
Subject: [PATCH 3030/6219] [movieclips] Add coding cookie

---
 youtube_dl/extractor/movieclips.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/movieclips.py b/youtube_dl/extractor/movieclips.py
index e06828b55..b8c43a163 100644
--- a/youtube_dl/extractor/movieclips.py
+++ b/youtube_dl/extractor/movieclips.py
@@ -1,3 +1,4 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
 from .common import InfoExtractor

From 114e6025b09e12bd01b5ce22bd2c43a3ef0ba460 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 8 Nov 2015 17:01:45 +0600
Subject: [PATCH 3031/6219] [rtbf] Expand _VALID_URL (Closes #7402)

---
 youtube_dl/extractor/rtbf.py | 48 ++++++++++++++----------------------
 1 file changed, 18 insertions(+), 30 deletions(-)

diff --git a/youtube_dl/extractor/rtbf.py b/youtube_dl/extractor/rtbf.py
index e75b45112..acf10e253 100644
--- a/youtube_dl/extractor/rtbf.py
+++ b/youtube_dl/extractor/rtbf.py
@@ -9,36 +9,24 @@ from ..utils import (
 
 
 class RTBFIE(InfoExtractor):
-    _VALID_URL = r'''(?x)
-        https?://www\.rtbf\.be/
-            (?:
-                video/[^\?]+\?id=|
-                ouftivi/heros/[^&]+&videoId=
-            )
-        (?P<id>\d+)
-    '''
-    _TESTS = [
-        {
-            'url': 'https://www.rtbf.be/video/detail_les-diables-au-coeur-episode-2?id=1921274',
-            'md5': '799f334ddf2c0a582ba80c44655be570',
-            'info_dict': {
-                'id': '1921274',
-                'ext': 'mp4',
-                'title': 'Les Diables au coeur (épisode 2)',
-                'duration': 3099,
-            }
-        },
-        {
-            'url': 'http://www.rtbf.be/ouftivi/heros/detail_scooby-doo-mysteres-associes?id=1097&videoId=2057442',
-            'md5': '25aea17e949e1e0c7c41270d60d25f22',
-            'info_dict': {
-                'id': '2057442',
-                'ext': 'mp4',
-                'title': 'Scooby-Doo, myst\xe8res associ\xe9s',
-                'duration': 1279,
-            }
-        },
-    ]
+    _VALID_URL = r'https?://www\.rtbf\.be/(?:video/[^?]+\?.*\bid=|ouftivi/(?:[^/]+/)*[^?]+\?.*\bvideoId=)(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'https://www.rtbf.be/video/detail_les-diables-au-coeur-episode-2?id=1921274',
+        'md5': '799f334ddf2c0a582ba80c44655be570',
+        'info_dict': {
+            'id': '1921274',
+            'ext': 'mp4',
+            'title': 'Les Diables au coeur (épisode 2)',
+            'duration': 3099,
+        }
+    }, {
+        # geo restricted
+        'url': 'http://www.rtbf.be/ouftivi/heros/detail_scooby-doo-mysteres-associes?id=1097&videoId=2057442',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.rtbf.be/ouftivi/niouzz?videoId=2055858',
+        'only_matching': True,
+    }]
 
     _QUALITIES = [
         ('mobile', 'mobile'),

From aa8d2d5be6a99542b85a85af3310fab1bf641e86 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 8 Nov 2015 17:03:21 +0600
Subject: [PATCH 3032/6219] [rtbf] Make www optional in _VALID_URL

---
 youtube_dl/extractor/rtbf.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/rtbf.py b/youtube_dl/extractor/rtbf.py
index acf10e253..e42b319a3 100644
--- a/youtube_dl/extractor/rtbf.py
+++ b/youtube_dl/extractor/rtbf.py
@@ -9,7 +9,7 @@ from ..utils import (
 
 
 class RTBFIE(InfoExtractor):
-    _VALID_URL = r'https?://www\.rtbf\.be/(?:video/[^?]+\?.*\bid=|ouftivi/(?:[^/]+/)*[^?]+\?.*\bvideoId=)(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?rtbf\.be/(?:video/[^?]+\?.*\bid=|ouftivi/(?:[^/]+/)*[^?]+\?.*\bvideoId=)(?P<id>\d+)'
     _TESTS = [{
         'url': 'https://www.rtbf.be/video/detail_les-diables-au-coeur-episode-2?id=1921274',
         'md5': '799f334ddf2c0a582ba80c44655be570',

From 50506cb60798fe4d2ebb9603798b3fb1cb81e55f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 8 Nov 2015 19:01:37 +0600
Subject: [PATCH 3033/6219] [extremetube] Fix extraction (Closes #7163)

---
 youtube_dl/extractor/extremetube.py | 45 +++++++++++++++++++----------
 1 file changed, 29 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/extractor/extremetube.py b/youtube_dl/extractor/extremetube.py
index c826a5404..3e11e3299 100644
--- a/youtube_dl/extractor/extremetube.py
+++ b/youtube_dl/extractor/extremetube.py
@@ -3,12 +3,9 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_parse_qs,
-    compat_urllib_request,
-)
+from ..compat import compat_urllib_request
 from ..utils import (
-    qualities,
+    int_or_none,
     str_to_int,
 )
 
@@ -49,20 +46,36 @@ class ExtremeTubeIE(InfoExtractor):
             r'Views:\s*</strong>\s*<span>([\d,\.]+)</span>',
             webpage, 'view count', fatal=False))
 
-        flash_vars = compat_parse_qs(self._search_regex(
-            r'<param[^>]+?name="flashvars"[^>]+?value="([^"]+)"', webpage, 'flash vars'))
+        flash_vars = self._parse_json(
+            self._search_regex(
+                r'var\s+flashvars\s*=\s*({.+?});', webpage, 'flash vars'),
+            video_id)
 
         formats = []
-        quality = qualities(['180p', '240p', '360p', '480p', '720p', '1080p'])
-        for k, vals in flash_vars.items():
-            m = re.match(r'quality_(?P<quality>[0-9]+p)$', k)
-            if m is not None:
-                formats.append({
-                    'format_id': m.group('quality'),
-                    'quality': quality(m.group('quality')),
-                    'url': vals[0],
+        for quality_key, video_url in flash_vars.items():
+            height = int_or_none(self._search_regex(
+                r'quality_(\d+)[pP]$', quality_key, 'height', default=None))
+            if not height:
+                continue
+            f = {
+                'url': video_url,
+            }
+            mobj = re.search(
+                r'/(?P<height>\d{3,4})[pP]_(?P<bitrate>\d+)[kK]_\d+', video_url)
+            if mobj:
+                height = int(mobj.group('height'))
+                bitrate = int(mobj.group('bitrate'))
+                f.update({
+                    'format_id': '%dp-%dk' % (height, bitrate),
+                    'height': height,
+                    'tbr': bitrate,
                 })
-
+            else:
+                f.update({
+                    'format_id': '%dp' % height,
+                    'height': height,
+                })
+            formats.append(f)
         self._sort_formats(formats)
 
         return {

From cc8034cc4c52fcbfaf9f8edf34d562c481860193 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 8 Nov 2015 19:14:39 +0600
Subject: [PATCH 3034/6219] [extremetube] Modernize

---
 youtube_dl/extractor/extremetube.py | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/extremetube.py b/youtube_dl/extractor/extremetube.py
index 3e11e3299..c5677c82b 100644
--- a/youtube_dl/extractor/extremetube.py
+++ b/youtube_dl/extractor/extremetube.py
@@ -11,12 +11,12 @@ from ..utils import (
 
 
 class ExtremeTubeIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?(?P<url>extremetube\.com/.*?video/.+?(?P<id>[0-9]+))(?:[/?&]|$)'
+    _VALID_URL = r'https?://(?:www\.)?extremetube\.com/(?:[^/]+/)?video/(?P<id>[^/#?&]+)'
     _TESTS = [{
         'url': 'http://www.extremetube.com/video/music-video-14-british-euro-brit-european-cumshots-swallow-652431',
         'md5': '344d0c6d50e2f16b06e49ca011d8ac69',
         'info_dict': {
-            'id': '652431',
+            'id': 'music-video-14-british-euro-brit-european-cumshots-swallow-652431',
             'ext': 'mp4',
             'title': 'Music Video 14 british euro brit european cumshots swallow',
             'uploader': 'unknown',
@@ -26,12 +26,16 @@ class ExtremeTubeIE(InfoExtractor):
     }, {
         'url': 'http://www.extremetube.com/gay/video/abcde-1234',
         'only_matching': True,
+    }, {
+        'url': 'http://www.extremetube.com/video/latina-slut-fucked-by-fat-black-dick',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.extremetube.com/video/652431',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        url = 'http://www.' + mobj.group('url')
+        video_id = self._match_id(url)
 
         req = compat_urllib_request.Request(url)
         req.add_header('Cookie', 'age_verified=1')

From f09a767d3198823e5c0ac187a91284c8d2736eb6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 8 Nov 2015 19:19:13 +0600
Subject: [PATCH 3035/6219] [mit] Allow external embeds (Closes #7406)

---
 youtube_dl/extractor/mit.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mit.py b/youtube_dl/extractor/mit.py
index f088ab9e2..29ca45778 100644
--- a/youtube_dl/extractor/mit.py
+++ b/youtube_dl/extractor/mit.py
@@ -86,7 +86,7 @@ class MITIE(TechTVMITIE):
         webpage = self._download_webpage(url, page_title)
         embed_url = self._search_regex(
             r'<iframe .*?src="(.+?)"', webpage, 'embed url')
-        return self.url_result(embed_url, ie='TechTVMIT')
+        return self.url_result(embed_url)
 
 
 class OCWMITIE(InfoExtractor):

From 4f5cdf7c9b21ba00822ad0134c5883f6ca366674 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 9 Nov 2015 01:48:46 +0600
Subject: [PATCH 3036/6219] [cmt] Extend _VALID_URL to support shows (Closes
 #7407)

---
 youtube_dl/extractor/cmt.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/cmt.py b/youtube_dl/extractor/cmt.py
index e96c59f71..f1311b14f 100644
--- a/youtube_dl/extractor/cmt.py
+++ b/youtube_dl/extractor/cmt.py
@@ -4,7 +4,7 @@ from .mtv import MTVIE
 
 class CMTIE(MTVIE):
     IE_NAME = 'cmt.com'
-    _VALID_URL = r'https?://www\.cmt\.com/videos/.+?/(?P<videoid>[^/]+)\.jhtml'
+    _VALID_URL = r'https?://www\.cmt\.com/(?:videos|shows)/(?:[^/]+/)*(?P<videoid>\d+)'
     _FEED_URL = 'http://www.cmt.com/sitewide/apps/player/embed/rss/'
 
     _TESTS = [{
@@ -16,4 +16,7 @@ class CMTIE(MTVIE):
             'title': 'Garth Brooks - "The Call (featuring Trisha Yearwood)"',
             'description': 'Blame It All On My Roots',
         },
+    }, {
+        'url': 'http://www.cmt.com/shows/party-down-south/party-down-south-ep-407-gone-girl/1738172/playlist/#id=1738172',
+        'only_matching': True,
     }]

From 1c31a5b0e029ae47415f7df7bea30e2b2b5df64f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 9 Nov 2015 20:49:06 +0600
Subject: [PATCH 3037/6219] [generic] Improve kaltura embed detection (Closes
 #7409)

---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index ee5419f51..5f61ab26f 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1673,7 +1673,7 @@ class GenericIE(InfoExtractor):
 
         # Look for Kaltura embeds
         mobj = (re.search(r"(?s)kWidget\.(?:thumb)?[Ee]mbed\(\{.*?'wid'\s*:\s*'_?(?P<partner_id>[^']+)',.*?'entry_id'\s*:\s*'(?P<id>[^']+)',", webpage) or
-                re.search(r'(?s)(["\'])(?:https?:)?//cdnapisec\.kaltura\.com/.*?(?:p|partner_id)/(?P<partner_id>\d+).*?\1.*?entry_id\s*:\s*(["\'])(?P<id>[^\2]+?)\2', webpage))
+                re.search(r'(?s)(?P<q1>["\'])(?:https?:)?//cdnapi(?:sec)?\.kaltura\.com/.*?(?:p|partner_id)/(?P<partner_id>\d+).*?(?P=q1).*?entry_?[Ii]d\s*:\s*(?P<q2>["\'])(?P<id>.+?)(?P=q2)', webpage))
         if mobj is not None:
             return self.url_result('kaltura:%(partner_id)s:%(id)s' % mobj.groupdict(), 'Kaltura')
 

From 6a5d6de1e36f5278b48b7dcc92a8e15adbe541ad Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 9 Nov 2015 20:49:42 +0600
Subject: [PATCH 3038/6219] [generic] Improve kaltura embed detection (2)

---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 5f61ab26f..d0b486d2a 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1672,7 +1672,7 @@ class GenericIE(InfoExtractor):
             return self.url_result(mobj.group('url'), 'Zapiks')
 
         # Look for Kaltura embeds
-        mobj = (re.search(r"(?s)kWidget\.(?:thumb)?[Ee]mbed\(\{.*?'wid'\s*:\s*'_?(?P<partner_id>[^']+)',.*?'entry_id'\s*:\s*'(?P<id>[^']+)',", webpage) or
+        mobj = (re.search(r"(?s)kWidget\.(?:thumb)?[Ee]mbed\(\{.*?'wid'\s*:\s*'_?(?P<partner_id>[^']+)',.*?'entry_?[Ii]d'\s*:\s*'(?P<id>[^']+)',", webpage) or
                 re.search(r'(?s)(?P<q1>["\'])(?:https?:)?//cdnapi(?:sec)?\.kaltura\.com/.*?(?:p|partner_id)/(?P<partner_id>\d+).*?(?P=q1).*?entry_?[Ii]d\s*:\s*(?P<q2>["\'])(?P<id>.+?)(?P=q2)', webpage))
         if mobj is not None:
             return self.url_result('kaltura:%(partner_id)s:%(id)s' % mobj.groupdict(), 'Kaltura')

From b25f7533975d7463cc772e9d106802ef29252796 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 9 Nov 2015 20:50:43 +0600
Subject: [PATCH 3039/6219] [kaltura] Relax _VALID_URL

---
 youtube_dl/extractor/kaltura.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/kaltura.py b/youtube_dl/extractor/kaltura.py
index 3dca0e566..0dcd6cd05 100644
--- a/youtube_dl/extractor/kaltura.py
+++ b/youtube_dl/extractor/kaltura.py
@@ -16,7 +16,7 @@ class KalturaIE(InfoExtractor):
                 (?:
                     kaltura:(?P<partner_id_s>\d+):(?P<id_s>[0-9a-z_]+)|
                     https?://
-                        (:?(?:www|cdnapisec)\.)?kaltura\.com/
+                        (:?(?:www|cdnapi(?:sec)?)\.)?kaltura\.com/
                         (?:
                             (?:
                                 # flash player

From ff29bf81f852941cd7bcef1c051c77eccd4857bc Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 10 Nov 2015 12:54:02 +0800
Subject: [PATCH 3040/6219] [jsinterp] Support alternative function definition
 form

---
 test/test_jsinterp.py  | 3 +++
 youtube_dl/jsinterp.py | 4 ++--
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py
index fc73e5dc2..63c350b8f 100644
--- a/test/test_jsinterp.py
+++ b/test/test_jsinterp.py
@@ -19,6 +19,9 @@ class TestJSInterpreter(unittest.TestCase):
         jsi = JSInterpreter('function x3(){return 42;}')
         self.assertEqual(jsi.call_function('x3'), 42)
 
+        jsi = JSInterpreter('var x5 = function(){return 42;}')
+        self.assertEqual(jsi.call_function('x5'), 42)
+
     def test_calc(self):
         jsi = JSInterpreter('function x4(a){return 2*a+1;}')
         self.assertEqual(jsi.call_function('x4', 3), 7)
diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index 0e0c7d90d..9bc855144 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -232,10 +232,10 @@ class JSInterpreter(object):
     def extract_function(self, funcname):
         func_m = re.search(
             r'''(?x)
-                (?:function\s+%s|[{;]%s\s*=\s*function)\s*
+                (?:function\s+%s|[{;]%s\s*=\s*function|var\s+%s\s*=\s*function)\s*
                 \((?P<args>[^)]*)\)\s*
                 \{(?P<code>[^}]+)\}''' % (
-                re.escape(funcname), re.escape(funcname)),
+                re.escape(funcname), re.escape(funcname), re.escape(funcname)),
             self.code)
         if func_m is None:
             raise ExtractorError('Could not find JS function %r' % funcname)

From 50f84a9ae171233c08ada41e03f6555c5ed95236 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 10 Nov 2015 12:55:01 +0800
Subject: [PATCH 3041/6219] [youtube] Support new base.js html5 player

---
 youtube_dl/extractor/youtube.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index e2a43299f..687e0b4db 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -703,7 +703,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
 
     def _extract_signature_function(self, video_id, player_url, example_sig):
         id_m = re.match(
-            r'.*?-(?P<id>[a-zA-Z0-9_-]+)(?:/watch_as3|/html5player(?:-new)?)?\.(?P<ext>[a-z]+)$',
+            r'.*?-(?P<id>[a-zA-Z0-9_-]+)(?:/watch_as3|/html5player(?:-new)?|/base)?\.(?P<ext>[a-z]+)$',
             player_url)
         if not id_m:
             raise ExtractorError('Cannot identify player %r' % player_url)
@@ -1343,7 +1343,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                                 player_desc = 'flash player %s' % player_version
                             else:
                                 player_version = self._search_regex(
-                                    r'html5player-([^/]+?)(?:/html5player(?:-new)?)?\.js',
+                                    [r'html5player-([^/]+?)(?:/html5player(?:-new)?)?\.js', r'(?:www|player)-([^/]+)/base\.js'],
                                     player_url,
                                     'html5 player', fatal=False)
                                 player_desc = 'html5 player %s' % player_version

From 37ca7b22b5521d7797ffefcd1456b4eb31320460 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 10 Nov 2015 11:39:06 +0100
Subject: [PATCH 3042/6219] release 2015.11.10

---
 docs/supportedsites.md | 4 +++-
 youtube_dl/version.py  | 2 +-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 805af14a0..a9820c1f5 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -123,6 +123,7 @@
  - **DctpTv**
  - **DeezerPlaylist**
  - **defense.gouv.fr**
+ - **democracynow**
  - **DHM**: Filmarchiv - Deutsches Historisches Museum
  - **Discovery**
  - **Dotsub**
@@ -195,6 +196,7 @@
  - **Giga**
  - **Glide**: Glide mobile video messages (glide.me)
  - **Globo**
+ - **GloboArticle**
  - **GodTube**
  - **GoldenMoustache**
  - **Golem**
@@ -617,7 +619,6 @@
  - **video.mit.edu**
  - **VideoDetective**
  - **videofy.me**
- - **videolectures.net**
  - **VideoMega**
  - **VideoPremium**
  - **VideoTt**: video.tt - Your True Tube
@@ -627,6 +628,7 @@
  - **vier**
  - **vier:videos**
  - **Viewster**
+ - **Viidea**
  - **viki**
  - **viki:channel**
  - **vimeo**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 6ef482b78..b3d254005 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.11.02'
+__version__ = '2015.11.10'

From d3d3e2e3aac13c3c6fbae0fcfeaa41ce5ee9144b Mon Sep 17 00:00:00 2001
From: David Ben Zakai <david.benzakai@gmail.com>
Date: Tue, 10 Nov 2015 16:31:31 +0200
Subject: [PATCH 3043/6219] Adding proxy to update procedure

---
 youtube_dl/YoutubeDL.py | 3 +++
 youtube_dl/__init__.py  | 2 +-
 youtube_dl/update.py    | 7 ++-----
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 1783ce01b..5a0cc3f9a 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1994,6 +1994,9 @@ class YoutubeDL(object):
             encoding = preferredencoding()
         return encoding
 
+    def get_opener(self):
+        return self._opener
+
     def _write_thumbnails(self, info_dict, filename):
         if self.params.get('writethumbnail', False):
             thumbnails = info_dict.get('thumbnails')
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 5e2ed4d4b..760128546 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -377,7 +377,7 @@ def _real_main(argv=None):
     with YoutubeDL(ydl_opts) as ydl:
         # Update version
         if opts.update_self:
-            update_self(ydl.to_screen, opts.verbose)
+            update_self(ydl.to_screen, opts.verbose, ydl.get_opener())
 
         # Remove cache dir
         if opts.rm_cachedir:
diff --git a/youtube_dl/update.py b/youtube_dl/update.py
index fc7ac8305..04bf0939e 100644
--- a/youtube_dl/update.py
+++ b/youtube_dl/update.py
@@ -13,7 +13,7 @@ from .compat import (
     compat_str,
     compat_urllib_request,
 )
-from .utils import make_HTTPS_handler
+
 from .version import __version__
 
 
@@ -47,7 +47,7 @@ def rsa_verify(message, signature, key):
     return True
 
 
-def update_self(to_screen, verbose):
+def update_self(to_screen, verbose, opener):
     """Update the program file with the latest version from the repository"""
 
     UPDATE_URL = "https://rg3.github.io/youtube-dl/update/"
@@ -59,9 +59,6 @@ def update_self(to_screen, verbose):
         to_screen('It looks like you installed youtube-dl with a package manager, pip, setup.py or a tarball. Please use that to update.')
         return
 
-    https_handler = make_HTTPS_handler({})
-    opener = compat_urllib_request.build_opener(https_handler)
-
     # Check if there is a new version
     try:
         newversion = opener.open(VERSION_URL).read().decode('utf-8').strip()

From 90bb5667bf32574caa7c01f66b3ca8ad2ab77e81 Mon Sep 17 00:00:00 2001
From: David Ben Zakai <david.benzakai@gmail.com>
Date: Tue, 10 Nov 2015 17:15:23 +0200
Subject: [PATCH 3044/6219] Using internal opener

---
 youtube_dl/YoutubeDL.py | 3 ---
 youtube_dl/__init__.py  | 2 +-
 2 files changed, 1 insertion(+), 4 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 5a0cc3f9a..1783ce01b 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1994,9 +1994,6 @@ class YoutubeDL(object):
             encoding = preferredencoding()
         return encoding
 
-    def get_opener(self):
-        return self._opener
-
     def _write_thumbnails(self, info_dict, filename):
         if self.params.get('writethumbnail', False):
             thumbnails = info_dict.get('thumbnails')
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 760128546..9f131f5db 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -377,7 +377,7 @@ def _real_main(argv=None):
     with YoutubeDL(ydl_opts) as ydl:
         # Update version
         if opts.update_self:
-            update_self(ydl.to_screen, opts.verbose, ydl.get_opener())
+            update_self(ydl.to_screen, opts.verbose, ydl._opener)
 
         # Remove cache dir
         if opts.rm_cachedir:

From 9b738b2caa20e13d0a56d05aa56af44be75476bc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 10 Nov 2015 21:32:54 +0600
Subject: [PATCH 3045/6219] [funnyordie] Fix extraction and extract m3u8
 formats

---
 youtube_dl/extractor/funnyordie.py | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/funnyordie.py b/youtube_dl/extractor/funnyordie.py
index f5f13689c..7f21d7410 100644
--- a/youtube_dl/extractor/funnyordie.py
+++ b/youtube_dl/extractor/funnyordie.py
@@ -45,11 +45,20 @@ class FunnyOrDieIE(InfoExtractor):
 
         links.sort(key=lambda link: 1 if link[1] == 'mp4' else 0)
 
-        bitrates = self._html_search_regex(r'<source src="[^"]+/v,((?:\d+,)+)\.mp4\.csmil', webpage, 'video bitrates')
-        bitrates = [int(b) for b in bitrates.rstrip(',').split(',')]
-        bitrates.sort()
+        m3u8_url = self._search_regex(
+            r'<source[^>]+src=(["\'])(?P<url>.+?/master\.m3u8)\1',
+            webpage, 'm3u8 url', default=None, group='url')
 
         formats = []
+
+        m3u8_formats = self._extract_m3u8_formats(
+            m3u8_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False)
+        if m3u8_formats:
+            formats.extend(m3u8_formats)
+
+        bitrates = [int(bitrate) for bitrate in re.findall(r'[,/]v(\d+)[,/]', m3u8_url)]
+        bitrates.sort()
+
         for bitrate in bitrates:
             for link in links:
                 formats.append({

From a625e5654365a8a246fa5fc5f5d347adc67c5617 Mon Sep 17 00:00:00 2001
From: Marco Ferragina <marco.ferragina@gmail.com>
Date: Wed, 14 Oct 2015 11:11:52 +0200
Subject: [PATCH 3046/6219] [vidto] Add extractor

---
 docs/supportedsites.md           |  1 +
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/vidto.py    | 82 ++++++++++++++++++++++++++++++++
 3 files changed, 84 insertions(+)
 create mode 100644 youtube_dl/extractor/vidto.py

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index a9820c1f5..510d4d627 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -624,6 +624,7 @@
  - **VideoTt**: video.tt - Your True Tube
  - **videoweed**: VideoWeed
  - **Vidme**
+ - **vidto**: VidTo.me
  - **Vidzi**
  - **vier**
  - **vier:videos**
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 0a90da73c..ef6c7f0de 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -730,6 +730,7 @@ from .videopremium import VideoPremiumIE
 from .videott import VideoTtIE
 from .videoweed import VideoWeedIE
 from .vidme import VidmeIE
+from .vidto import VidtoIE
 from .vidzi import VidziIE
 from .vier import VierIE, VierVideosIE
 from .viewster import ViewsterIE
diff --git a/youtube_dl/extractor/vidto.py b/youtube_dl/extractor/vidto.py
new file mode 100644
index 000000000..3cc585471
--- /dev/null
+++ b/youtube_dl/extractor/vidto.py
@@ -0,0 +1,82 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+import sys
+from .common import InfoExtractor
+import time
+
+from ..utils import (
+    encode_dict,
+)
+from ..compat import (
+    compat_chr,
+    compat_parse_qs,
+    compat_urllib_parse,
+    compat_urllib_parse_unquote,
+    compat_urllib_parse_unquote_plus,
+    compat_urllib_parse_urlparse,
+    compat_urllib_request,
+    compat_urlparse,
+    compat_str,
+)
+
+
+class VidtoIE(InfoExtractor):
+    IE_NAME = 'vidto'
+    IE_DESC = 'VidTo.me'
+    _VALID_URL = r'https?://(?:www\.)?vidto\.me/(?P<id>[0-9a-zA-Z]+)\.html'
+    _HOST = 'vidto.me'
+    _TEST = {
+        'url': 'http://vidto.me/ku5glz52nqe1.html',
+        'info_dict': {
+            'id': 'ku5glz52nqe1',
+            'ext': 'mp4',
+            'title': 'test.mp4'
+        }
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+
+        page = self._download_webpage(
+            'http://%s/%s.html' % (self._HOST, video_id), video_id, 'Downloading video page')
+        hash_regex = r'<input type="hidden" name="hash" value="(.*)">'
+        hash_value = self._search_regex(hash_regex, page, 'hash', fatal=True)
+        title_regex = r'<input type="hidden" name="fname" value="(.*)">'
+        title = self._search_regex(title_regex, page, 'title', fatal=False)
+        id_regex = r'<input type="hidden" name="id" value="(.*)">'
+        id_value = self._search_regex(id_regex, page, 'id', fatal=True)
+        cookies = self._get_cookies('http://%s/%s.html' % (self._HOST, video_id))
+
+
+        form_str = {
+            'op': 'download1',
+            'imhuman': 'Proceed to video',
+            'usr_login': '',
+            'id': id_value,
+            'fname': title,
+            'referer': '',
+            'hash': hash_value,
+        }
+        post_data = compat_urllib_parse.urlencode(encode_dict(form_str)).encode('ascii')
+        req = compat_urllib_request.Request(url, post_data)
+        req.add_header('Content-type', 'application/x-www-form-urlencoded')
+        for key, morsel in cookies.iteritems():
+            req.add_header('Cookie', '%s=%s' % (morsel.key, morsel.value))
+
+        print("Waiting for countdown...")
+        time.sleep(7)
+        post_result = self._download_webpage(
+            req, None,
+            note='Proceed to video...', errnote='unable to proceed', fatal=True)
+
+        file_link_regex = r'file_link ?= ?\'(https?:\/\/[0-9a-zA-z.\/\-_]+)'
+        file_link = self._search_regex(file_link_regex, post_result, 'file_link', fatal=True)
+
+        return {
+            'id': video_id,
+            'url': file_link,
+            'title': title,
+        }

From 42fc93c70928d34d46ceda644eb870e86daa8393 Mon Sep 17 00:00:00 2001
From: Marco Ferragina <marco.ferragina@gmail.com>
Date: Sat, 17 Oct 2015 19:35:41 +0200
Subject: [PATCH 3047/6219] vidto extractor: code cleanup

---
 docs/supportedsites.md        |  1 -
 youtube_dl/extractor/vidto.py | 32 ++++++++++++--------------------
 2 files changed, 12 insertions(+), 21 deletions(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 510d4d627..a9820c1f5 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -624,7 +624,6 @@
  - **VideoTt**: video.tt - Your True Tube
  - **videoweed**: VideoWeed
  - **Vidme**
- - **vidto**: VidTo.me
  - **Vidzi**
  - **vier**
  - **vier:videos**
diff --git a/youtube_dl/extractor/vidto.py b/youtube_dl/extractor/vidto.py
index 3cc585471..391f400fc 100644
--- a/youtube_dl/extractor/vidto.py
+++ b/youtube_dl/extractor/vidto.py
@@ -1,24 +1,14 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
-import sys
 from .common import InfoExtractor
+import re
 import time
 
-from ..utils import (
-    encode_dict,
-)
+from ..utils import encode_dict
 from ..compat import (
-    compat_chr,
-    compat_parse_qs,
-    compat_urllib_parse,
-    compat_urllib_parse_unquote,
-    compat_urllib_parse_unquote_plus,
-    compat_urllib_parse_urlparse,
     compat_urllib_request,
-    compat_urlparse,
-    compat_str,
+    compat_urllib_parse
 )
 
 
@@ -37,8 +27,7 @@ class VidtoIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         page = self._download_webpage(
             'http://%s/%s.html' % (self._HOST, video_id), video_id, 'Downloading video page')
@@ -63,16 +52,19 @@ class VidtoIE(InfoExtractor):
         post_data = compat_urllib_parse.urlencode(encode_dict(form_str)).encode('ascii')
         req = compat_urllib_request.Request(url, post_data)
         req.add_header('Content-type', 'application/x-www-form-urlencoded')
-        for key, morsel in cookies.iteritems():
-            req.add_header('Cookie', '%s=%s' % (morsel.key, morsel.value))
+        cookie_string = ""
+        for key in cookies.keys():
+            cookie_string += "%s=%s;" % (key, cookies[key].value)
 
-        print("Waiting for countdown...")
+        req.add_header('Cookie', '%s' % cookie_string)
+
+        self.to_screen("Waiting for countdown...")
         time.sleep(7)
         post_result = self._download_webpage(
-            req, None,
+            req, video_id,
             note='Proceed to video...', errnote='unable to proceed', fatal=True)
 
-        file_link_regex = r'file_link ?= ?\'(https?:\/\/[0-9a-zA-z.\/\-_]+)'
+        file_link_regex = r'file_link\s*=\s*\'(https?:\/\/[0-9a-zA-z.\/\-_]+)'
         file_link = self._search_regex(file_link_regex, post_result, 'file_link', fatal=True)
 
         return {

From 37120974dc7e3e7febd40b652fc7f9e1748e2ad3 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 11 Nov 2015 02:02:46 +0800
Subject: [PATCH 3048/6219] [vidto] PEP8

---
 youtube_dl/extractor/vidto.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/youtube_dl/extractor/vidto.py b/youtube_dl/extractor/vidto.py
index 391f400fc..d10002534 100644
--- a/youtube_dl/extractor/vidto.py
+++ b/youtube_dl/extractor/vidto.py
@@ -2,7 +2,6 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-import re
 import time
 
 from ..utils import encode_dict
@@ -39,7 +38,6 @@ class VidtoIE(InfoExtractor):
         id_value = self._search_regex(id_regex, page, 'id', fatal=True)
         cookies = self._get_cookies('http://%s/%s.html' % (self._HOST, video_id))
 
-
         form_str = {
             'op': 'download1',
             'imhuman': 'Proceed to video',

From d8b7e80d29a67835555a6317b2c528b911077d47 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9my=20L=C3=A9one?= <remy.leone@gmail.com>
Date: Wed, 11 Nov 2015 12:00:31 +0100
Subject: [PATCH 3049/6219] Remove duplicate key

---
 youtube_dl/extractor/wsj.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/wsj.py b/youtube_dl/extractor/wsj.py
index 2ddf29a69..5a897371d 100644
--- a/youtube_dl/extractor/wsj.py
+++ b/youtube_dl/extractor/wsj.py
@@ -84,6 +84,5 @@ class WSJIE(InfoExtractor):
             'duration': duration,
             'upload_date': upload_date,
             'title': title,
-            'formats': formats,
             'categories': categories,
         }

From 990e6e8fa346224967f123a50146377088acbd75 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 11 Nov 2015 20:13:03 +0800
Subject: [PATCH 3050/6219] [vidto] Minor fixes

1. import order
2. fatal is already True in helper functions
---
 youtube_dl/extractor/vidto.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/vidto.py b/youtube_dl/extractor/vidto.py
index d10002534..45b21e2a9 100644
--- a/youtube_dl/extractor/vidto.py
+++ b/youtube_dl/extractor/vidto.py
@@ -1,9 +1,9 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-from .common import InfoExtractor
 import time
 
+from .common import InfoExtractor
 from ..utils import encode_dict
 from ..compat import (
     compat_urllib_request,
@@ -60,10 +60,10 @@ class VidtoIE(InfoExtractor):
         time.sleep(7)
         post_result = self._download_webpage(
             req, video_id,
-            note='Proceed to video...', errnote='unable to proceed', fatal=True)
+            note='Proceed to video...', errnote='unable to proceed')
 
         file_link_regex = r'file_link\s*=\s*\'(https?:\/\/[0-9a-zA-z.\/\-_]+)'
-        file_link = self._search_regex(file_link_regex, post_result, 'file_link', fatal=True)
+        file_link = self._search_regex(file_link_regex, post_result, 'file_link')
 
         return {
             'id': video_id,

From 6abce58a1226a5c359e4eb0cb228353fa7cb2aaf Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 11 Nov 2015 20:16:18 +0800
Subject: [PATCH 3051/6219] Credit @sieben for fixing wsj extractor

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index cc552bcb2..4cdbf4a5f 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -144,3 +144,4 @@ Lee Jenkins
 Anssi Hannula
 Lukáš Lalinský
 Qijiang Fan
+Rémy Léone

From 2eb99a4b98ecece86a8d0c692066644a855f8524 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 11 Nov 2015 21:00:23 +0600
Subject: [PATCH 3052/6219] [nowvideo] Replace main host to resolvable one

---
 youtube_dl/extractor/nowvideo.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/nowvideo.py b/youtube_dl/extractor/nowvideo.py
index 17baa9679..57ee3d366 100644
--- a/youtube_dl/extractor/nowvideo.py
+++ b/youtube_dl/extractor/nowvideo.py
@@ -7,9 +7,9 @@ class NowVideoIE(NovaMovIE):
     IE_NAME = 'nowvideo'
     IE_DESC = 'NowVideo'
 
-    _VALID_URL = NovaMovIE._VALID_URL_TEMPLATE % {'host': 'nowvideo\.(?:ch|ec|sx|eu|at|ag|co|li)'}
+    _VALID_URL = NovaMovIE._VALID_URL_TEMPLATE % {'host': 'nowvideo\.(?:to|ch|ec|sx|eu|at|ag|co|li)'}
 
-    _HOST = 'www.nowvideo.ch'
+    _HOST = 'www.nowvideo.to'
 
     _FILE_DELETED_REGEX = r'>This file no longer exists on our servers.<'
     _FILEKEY_REGEX = r'var fkzd="([^"]+)";'

From 82393e2bb2c499cad285beb07c222f501302d830 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 11 Nov 2015 21:02:05 +0600
Subject: [PATCH 3053/6219] [novamov] Follow continue-to-the-video button if
 any (Closes #7330)

---
 youtube_dl/extractor/novamov.py | 38 +++++++++++++++++++++++++++------
 1 file changed, 32 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/novamov.py b/youtube_dl/extractor/novamov.py
index 04d779890..e0bf6d1bc 100644
--- a/youtube_dl/extractor/novamov.py
+++ b/youtube_dl/extractor/novamov.py
@@ -4,10 +4,14 @@ import re
 
 from .common import InfoExtractor
 from ..compat import (
+    compat_urllib_request,
     compat_urlparse,
 )
 from ..utils import (
     ExtractorError,
+    NO_DEFAULT,
+    encode_dict,
+    urlencode_postdata,
 )
 
 
@@ -41,16 +45,38 @@ class NovaMovIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
 
-        page = self._download_webpage(
-            'http://%s/video/%s' % (self._HOST, video_id), video_id, 'Downloading video page')
+        url = 'http://%s/video/%s' % (self._HOST, video_id)
 
-        if re.search(self._FILE_DELETED_REGEX, page) is not None:
+        webpage = self._download_webpage(
+            url, video_id, 'Downloading video page')
+
+        if re.search(self._FILE_DELETED_REGEX, webpage) is not None:
             raise ExtractorError('Video %s does not exist' % video_id, expected=True)
 
-        filekey = self._search_regex(self._FILEKEY_REGEX, page, 'filekey')
+        def extract_filekey(default=NO_DEFAULT):
+            return self._search_regex(
+                self._FILEKEY_REGEX, webpage, 'filekey', default=default)
 
-        title = self._html_search_regex(self._TITLE_REGEX, page, 'title', fatal=False)
-        description = self._html_search_regex(self._DESCRIPTION_REGEX, page, 'description', default='', fatal=False)
+        filekey = extract_filekey(default=None)
+
+        if not filekey:
+            fields = self._hidden_inputs(webpage)
+            post_url = self._search_regex(
+                r'<form[^>]+action=(["\'])(?P<url>.+?)\1', webpage,
+                'post url', default=url, group='url')
+            if not post_url.startswith('http'):
+                post_url = compat_urlparse.urljoin(url, post_url)
+            request = compat_urllib_request.Request(
+                post_url, urlencode_postdata(encode_dict(fields)))
+            request.add_header('Content-Type', 'application/x-www-form-urlencoded')
+            request.add_header('Referer', post_url)
+            webpage = self._download_webpage(
+                request, video_id, 'Downloading continue to the video page')
+
+        filekey = extract_filekey()
+
+        title = self._html_search_regex(self._TITLE_REGEX, webpage, 'title', fatal=False)
+        description = self._html_search_regex(self._DESCRIPTION_REGEX, webpage, 'description', default='', fatal=False)
 
         api_response = self._download_webpage(
             'http://%s/api/player.api.php?key=%s&file=%s' % (self._HOST, filekey, video_id), video_id,

From 8b8a39e279b02926197230024e64a50d7995b95d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 11 Nov 2015 23:17:59 +0800
Subject: [PATCH 3054/6219] [vidto] Several simplifications and improvements

1. Use InfoExtractor._hidden_inputs
2. Fetch title from <title> tag
3. Cookies are preserved automatically
4. Use single quotes everywhere
5. Do not declare variables for one-time use only
---
 youtube_dl/extractor/vidto.py | 51 +++++++++++++----------------------
 1 file changed, 18 insertions(+), 33 deletions(-)

diff --git a/youtube_dl/extractor/vidto.py b/youtube_dl/extractor/vidto.py
index 45b21e2a9..5ebf0f3d3 100644
--- a/youtube_dl/extractor/vidto.py
+++ b/youtube_dl/extractor/vidto.py
@@ -4,24 +4,24 @@ from __future__ import unicode_literals
 import time
 
 from .common import InfoExtractor
-from ..utils import encode_dict
-from ..compat import (
-    compat_urllib_request,
-    compat_urllib_parse
+from ..utils import (
+    encode_dict,
+    remove_end,
+    urlencode_postdata,
 )
+from ..compat import compat_urllib_request
 
 
 class VidtoIE(InfoExtractor):
     IE_NAME = 'vidto'
     IE_DESC = 'VidTo.me'
     _VALID_URL = r'https?://(?:www\.)?vidto\.me/(?P<id>[0-9a-zA-Z]+)\.html'
-    _HOST = 'vidto.me'
     _TEST = {
         'url': 'http://vidto.me/ku5glz52nqe1.html',
         'info_dict': {
             'id': 'ku5glz52nqe1',
             'ext': 'mp4',
-            'title': 'test.mp4'
+            'title': 'test'
         }
     }
 
@@ -29,41 +29,26 @@ class VidtoIE(InfoExtractor):
         video_id = self._match_id(url)
 
         page = self._download_webpage(
-            'http://%s/%s.html' % (self._HOST, video_id), video_id, 'Downloading video page')
-        hash_regex = r'<input type="hidden" name="hash" value="(.*)">'
-        hash_value = self._search_regex(hash_regex, page, 'hash', fatal=True)
-        title_regex = r'<input type="hidden" name="fname" value="(.*)">'
-        title = self._search_regex(title_regex, page, 'title', fatal=False)
-        id_regex = r'<input type="hidden" name="id" value="(.*)">'
-        id_value = self._search_regex(id_regex, page, 'id', fatal=True)
-        cookies = self._get_cookies('http://%s/%s.html' % (self._HOST, video_id))
+            'http://vidto.me/%s.html' % video_id, video_id, 'Downloading video page')
 
-        form_str = {
-            'op': 'download1',
-            'imhuman': 'Proceed to video',
-            'usr_login': '',
-            'id': id_value,
-            'fname': title,
-            'referer': '',
-            'hash': hash_value,
-        }
-        post_data = compat_urllib_parse.urlencode(encode_dict(form_str)).encode('ascii')
-        req = compat_urllib_request.Request(url, post_data)
-        req.add_header('Content-type', 'application/x-www-form-urlencoded')
-        cookie_string = ""
-        for key in cookies.keys():
-            cookie_string += "%s=%s;" % (key, cookies[key].value)
+        title = remove_end(self._html_search_regex(
+            r'<Title>\s*([^<]+)\s*</Title>', page, 'title'), ' - Vidto')
 
-        req.add_header('Cookie', '%s' % cookie_string)
+        hidden_fields = self._hidden_inputs(page)
 
-        self.to_screen("Waiting for countdown...")
+        self.to_screen('Waiting for countdown...')
         time.sleep(7)
+
+        req = compat_urllib_request.Request(
+            url, urlencode_postdata(encode_dict(hidden_fields)))
+        req.add_header('Content-type', 'application/x-www-form-urlencoded')
+
         post_result = self._download_webpage(
             req, video_id,
             note='Proceed to video...', errnote='unable to proceed')
 
-        file_link_regex = r'file_link\s*=\s*\'(https?:\/\/[0-9a-zA-z.\/\-_]+)'
-        file_link = self._search_regex(file_link_regex, post_result, 'file_link')
+        file_link = self._search_regex(
+            r'file_link\s*=\s*\'(https?:\/\/[0-9a-zA-z.\/\-_]+)', post_result, 'file_link')
 
         return {
             'id': video_id,

From 3d9c4bf09a57871f0030d6b6022b863d66616b8b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 11 Nov 2015 21:21:21 +0600
Subject: [PATCH 3055/6219] [vimeo] Fix password protected videos (Closes
 #7451)

---
 youtube_dl/extractor/vimeo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index ca716c8f5..9dc6247e8 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -217,7 +217,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
             url = url.replace('http://', 'https://')
         password_request = compat_urllib_request.Request(url + '/password', data)
         password_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
-        password_request.add_header('Cookie', 'clip_test2=1; vuid=%s' % vuid)
+        password_request.add_header('Cookie', 'clip_test_v2=0; vuid=%s' % vuid)
         password_request.add_header('Referer', url)
         return self._download_webpage(
             password_request, video_id,

From 9a8a12b7d8c09c6034a3d44ee64eef402c9bc076 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 11 Nov 2015 22:23:23 +0600
Subject: [PATCH 3056/6219] [vimeo] Append cookies instead of overriding

---
 youtube_dl/extractor/vimeo.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 9dc6247e8..abfd9b6c4 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -49,8 +49,8 @@ class VimeoBaseInfoExtractor(InfoExtractor):
         }))
         login_request = compat_urllib_request.Request(self._LOGIN_URL, data)
         login_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
-        login_request.add_header('Cookie', 'vuid=%s' % vuid)
         login_request.add_header('Referer', self._LOGIN_URL)
+        self._set_cookie('vimeo.com', 'vuid', vuid)
         self._download_webpage(login_request, None, False, 'Wrong login info')
 
     def _extract_xsrft_and_vuid(self, webpage):
@@ -217,8 +217,8 @@ class VimeoIE(VimeoBaseInfoExtractor):
             url = url.replace('http://', 'https://')
         password_request = compat_urllib_request.Request(url + '/password', data)
         password_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
-        password_request.add_header('Cookie', 'clip_test_v2=0; vuid=%s' % vuid)
         password_request.add_header('Referer', url)
+        self._set_cookie('vimeo.com', 'vuid', vuid)
         return self._download_webpage(
             password_request, video_id,
             'Verifying the password', 'Wrong password')
@@ -494,7 +494,7 @@ class VimeoChannelIE(VimeoBaseInfoExtractor):
         password_url = compat_urlparse.urljoin(page_url, password_path)
         password_request = compat_urllib_request.Request(password_url, post)
         password_request.add_header('Content-type', 'application/x-www-form-urlencoded')
-        password_request.add_header('Cookie', 'vuid=%s' % vuid)
+        self._set_cookie('vimeo.com', 'vuid', vuid)
         self._set_cookie('vimeo.com', 'xsrft', token)
 
         return self._download_webpage(

From 9eab37dca0e767a1c9a9d659dd4125ce68d049f9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 11 Nov 2015 22:32:13 +0600
Subject: [PATCH 3057/6219] [vimeo] Simplify set cookie

---
 youtube_dl/extractor/vimeo.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index abfd9b6c4..b056ba720 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -50,7 +50,7 @@ class VimeoBaseInfoExtractor(InfoExtractor):
         login_request = compat_urllib_request.Request(self._LOGIN_URL, data)
         login_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
         login_request.add_header('Referer', self._LOGIN_URL)
-        self._set_cookie('vimeo.com', 'vuid', vuid)
+        self._set_vimeo_cookie('vuid', vuid)
         self._download_webpage(login_request, None, False, 'Wrong login info')
 
     def _extract_xsrft_and_vuid(self, webpage):
@@ -62,6 +62,9 @@ class VimeoBaseInfoExtractor(InfoExtractor):
             webpage, 'vuid', group='vuid')
         return xsrft, vuid
 
+    def _set_vimeo_cookie(self, name, value):
+        self._set_cookie('vimeo.com', name, value)
+
 
 class VimeoIE(VimeoBaseInfoExtractor):
     """Information extractor for vimeo.com."""
@@ -218,7 +221,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
         password_request = compat_urllib_request.Request(url + '/password', data)
         password_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
         password_request.add_header('Referer', url)
-        self._set_cookie('vimeo.com', 'vuid', vuid)
+        self._set_vimeo_cookie('vuid', vuid)
         return self._download_webpage(
             password_request, video_id,
             'Verifying the password', 'Wrong password')
@@ -494,8 +497,8 @@ class VimeoChannelIE(VimeoBaseInfoExtractor):
         password_url = compat_urlparse.urljoin(page_url, password_path)
         password_request = compat_urllib_request.Request(password_url, post)
         password_request.add_header('Content-type', 'application/x-www-form-urlencoded')
-        self._set_cookie('vimeo.com', 'vuid', vuid)
-        self._set_cookie('vimeo.com', 'xsrft', token)
+        self._set_vimeo_cookie('vuid', vuid)
+        self._set_vimeo_cookie('xsrft', token)
 
         return self._download_webpage(
             password_request, list_id,

From 699ed30ceed8a0499d17c54742b63469e5ff08ee Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 11 Nov 2015 22:34:49 +0600
Subject: [PATCH 3058/6219] [novamov] Modernize

---
 youtube_dl/extractor/novamov.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/novamov.py b/youtube_dl/extractor/novamov.py
index e0bf6d1bc..6b15fc2e5 100644
--- a/youtube_dl/extractor/novamov.py
+++ b/youtube_dl/extractor/novamov.py
@@ -42,8 +42,7 @@ class NovaMovIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         url = 'http://%s/video/%s' % (self._HOST, video_id)
 

From 435911029f1dfbe0caaf460a09742e864e656478 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 11 Nov 2015 22:43:17 +0600
Subject: [PATCH 3059/6219] [vidto] Remove extractor

---
 youtube_dl/extractor/__init__.py |  1 -
 youtube_dl/extractor/vidto.py    | 57 --------------------------------
 2 files changed, 58 deletions(-)
 delete mode 100644 youtube_dl/extractor/vidto.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index ef6c7f0de..0a90da73c 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -730,7 +730,6 @@ from .videopremium import VideoPremiumIE
 from .videott import VideoTtIE
 from .videoweed import VideoWeedIE
 from .vidme import VidmeIE
-from .vidto import VidtoIE
 from .vidzi import VidziIE
 from .vier import VierIE, VierVideosIE
 from .viewster import ViewsterIE
diff --git a/youtube_dl/extractor/vidto.py b/youtube_dl/extractor/vidto.py
deleted file mode 100644
index 5ebf0f3d3..000000000
--- a/youtube_dl/extractor/vidto.py
+++ /dev/null
@@ -1,57 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-import time
-
-from .common import InfoExtractor
-from ..utils import (
-    encode_dict,
-    remove_end,
-    urlencode_postdata,
-)
-from ..compat import compat_urllib_request
-
-
-class VidtoIE(InfoExtractor):
-    IE_NAME = 'vidto'
-    IE_DESC = 'VidTo.me'
-    _VALID_URL = r'https?://(?:www\.)?vidto\.me/(?P<id>[0-9a-zA-Z]+)\.html'
-    _TEST = {
-        'url': 'http://vidto.me/ku5glz52nqe1.html',
-        'info_dict': {
-            'id': 'ku5glz52nqe1',
-            'ext': 'mp4',
-            'title': 'test'
-        }
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        page = self._download_webpage(
-            'http://vidto.me/%s.html' % video_id, video_id, 'Downloading video page')
-
-        title = remove_end(self._html_search_regex(
-            r'<Title>\s*([^<]+)\s*</Title>', page, 'title'), ' - Vidto')
-
-        hidden_fields = self._hidden_inputs(page)
-
-        self.to_screen('Waiting for countdown...')
-        time.sleep(7)
-
-        req = compat_urllib_request.Request(
-            url, urlencode_postdata(encode_dict(hidden_fields)))
-        req.add_header('Content-type', 'application/x-www-form-urlencoded')
-
-        post_result = self._download_webpage(
-            req, video_id,
-            note='Proceed to video...', errnote='unable to proceed')
-
-        file_link = self._search_regex(
-            r'file_link\s*=\s*\'(https?:\/\/[0-9a-zA-z.\/\-_]+)', post_result, 'file_link')
-
-        return {
-            'id': video_id,
-            'url': file_link,
-            'title': title,
-        }

From b9ad101926b928445acf4a1ab48c24e17836b22c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 11 Nov 2015 22:44:03 +0600
Subject: [PATCH 3060/6219] [gorillavid] Add support for vidto.me

---
 youtube_dl/extractor/gorillavid.py | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/gorillavid.py b/youtube_dl/extractor/gorillavid.py
index d23e3eac1..d4fd0449c 100644
--- a/youtube_dl/extractor/gorillavid.py
+++ b/youtube_dl/extractor/gorillavid.py
@@ -16,10 +16,10 @@ from ..utils import (
 
 
 class GorillaVidIE(InfoExtractor):
-    IE_DESC = 'GorillaVid.in, daclips.in, movpod.in, fastvideo.in, realvid.net and filehoot.com'
+    IE_DESC = 'GorillaVid.in, daclips.in, movpod.in, fastvideo.in, realvid.net, filehoot.com and vidto.me'
     _VALID_URL = r'''(?x)
         https?://(?P<host>(?:www\.)?
-            (?:daclips\.in|gorillavid\.in|movpod\.in|fastvideo\.in|realvid\.net|filehoot\.com))/
+            (?:daclips\.in|gorillavid\.in|movpod\.in|fastvideo\.in|realvid\.net|filehoot\.com|vidto.\me))/
         (?:embed-)?(?P<id>[0-9a-zA-Z]+)(?:-[0-9]+x[0-9]+\.html)?
     '''
 
@@ -105,12 +105,17 @@ class GorillaVidIE(InfoExtractor):
             webpage = self._download_webpage(req, video_id, 'Downloading video page')
 
         title = self._search_regex(
-            [r'style="z-index: [0-9]+;">([^<]+)</span>', r'<td nowrap>([^<]+)</td>', r'>Watch (.+) '],
+            [r'style="z-index: [0-9]+;">([^<]+)</span>',
+             r'<td nowrap>([^<]+)</td>',
+             r'>Watch (.+) ',
+             r'<h2 class="video-page-head">([^<]+)</h2>'],
             webpage, 'title', default=None) or self._og_search_title(webpage)
         video_url = self._search_regex(
-            r'file\s*:\s*["\'](http[^"\']+)["\'],', webpage, 'file url')
+            [r'file\s*:\s*["\'](http[^"\']+)["\'],',
+             r'file_link\s*=\s*\'(https?:\/\/[0-9a-zA-z.\/\-_]+)'],
+            webpage, 'file url')
         thumbnail = self._search_regex(
-            r'image\s*:\s*["\'](http[^"\']+)["\'],', webpage, 'thumbnail', fatal=False)
+            r'image\s*:\s*["\'](http[^"\']+)["\'],', webpage, 'thumbnail', default=None)
 
         formats = [{
             'format_id': 'sd',

From 668db403f97eb1b50a3f54266db4bc274f1fa874 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 11 Nov 2015 22:47:28 +0600
Subject: [PATCH 3061/6219] [gorillavid] Add test for vidto.me and strip title

---
 youtube_dl/extractor/gorillavid.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/gorillavid.py b/youtube_dl/extractor/gorillavid.py
index d4fd0449c..8c982d8e9 100644
--- a/youtube_dl/extractor/gorillavid.py
+++ b/youtube_dl/extractor/gorillavid.py
@@ -76,6 +76,13 @@ class GorillaVidIE(InfoExtractor):
             'title': 'youtube-dl test video \'äBaW_jenozKc.mp4.mp4',
             'thumbnail': 're:http://.*\.jpg',
         }
+    }, {
+        'url': 'http://vidto.me/ku5glz52nqe1.html',
+        'info_dict': {
+            'id': 'ku5glz52nqe1',
+            'ext': 'mp4',
+            'title': 'test'
+        }
     }]
 
     def _real_extract(self, url):
@@ -104,12 +111,12 @@ class GorillaVidIE(InfoExtractor):
 
             webpage = self._download_webpage(req, video_id, 'Downloading video page')
 
-        title = self._search_regex(
+        title = (self._search_regex(
             [r'style="z-index: [0-9]+;">([^<]+)</span>',
              r'<td nowrap>([^<]+)</td>',
              r'>Watch (.+) ',
              r'<h2 class="video-page-head">([^<]+)</h2>'],
-            webpage, 'title', default=None) or self._og_search_title(webpage)
+            webpage, 'title', default=None) or self._og_search_title(webpage)).strip()
         video_url = self._search_regex(
             [r'file\s*:\s*["\'](http[^"\']+)["\'],',
              r'file_link\s*=\s*\'(https?:\/\/[0-9a-zA-z.\/\-_]+)'],

From 031ec536f00af5dc4ec034d3adb7d5fe3cc02453 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 11 Nov 2015 22:58:39 +0600
Subject: [PATCH 3062/6219] [gorillavid] Rename to xfileshare

---
 youtube_dl/extractor/__init__.py                      | 2 +-
 youtube_dl/extractor/{gorillavid.py => xfileshare.py} | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)
 rename youtube_dl/extractor/{gorillavid.py => xfileshare.py} (96%)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 0a90da73c..06d25ef40 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -221,7 +221,6 @@ from .goldenmoustache import GoldenMoustacheIE
 from .golem import GolemIE
 from .googleplus import GooglePlusIE
 from .googlesearch import GoogleSearchIE
-from .gorillavid import GorillaVidIE
 from .goshgay import GoshgayIE
 from .groupon import GrouponIE
 from .hark import HarkIE
@@ -786,6 +785,7 @@ from .wrzuta import WrzutaIE
 from .wsj import WSJIE
 from .xbef import XBefIE
 from .xboxclips import XboxClipsIE
+from .xfileshare import XFileShareIE
 from .xhamster import (
     XHamsterIE,
     XHamsterEmbedIE,
diff --git a/youtube_dl/extractor/gorillavid.py b/youtube_dl/extractor/xfileshare.py
similarity index 96%
rename from youtube_dl/extractor/gorillavid.py
rename to youtube_dl/extractor/xfileshare.py
index 8c982d8e9..7610dc627 100644
--- a/youtube_dl/extractor/gorillavid.py
+++ b/youtube_dl/extractor/xfileshare.py
@@ -1,4 +1,4 @@
-# -*- coding: utf-8 -*-
+# coding: utf-8
 from __future__ import unicode_literals
 
 import re
@@ -15,8 +15,8 @@ from ..utils import (
 )
 
 
-class GorillaVidIE(InfoExtractor):
-    IE_DESC = 'GorillaVid.in, daclips.in, movpod.in, fastvideo.in, realvid.net, filehoot.com and vidto.me'
+class XFileShareIE(InfoExtractor):
+    IE_DESC = 'XFileShare based sites: GorillaVid.in, daclips.in, movpod.in, fastvideo.in, realvid.net, filehoot.com and vidto.me'
     _VALID_URL = r'''(?x)
         https?://(?P<host>(?:www\.)?
             (?:daclips\.in|gorillavid\.in|movpod\.in|fastvideo\.in|realvid\.net|filehoot\.com|vidto.\me))/

From fcd817a32636268156dfab20ffc7896fb6082f50 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 12 Nov 2015 03:56:11 +0600
Subject: [PATCH 3063/6219] [vimeo] Fix extraction (Closes #7460)

---
 youtube_dl/extractor/vimeo.py | 52 ++++++++++++-----------------------
 1 file changed, 17 insertions(+), 35 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index b056ba720..b72341a2b 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -387,47 +387,29 @@ class VimeoIE(VimeoBaseInfoExtractor):
             like_count = None
             comment_count = None
 
-        # Vimeo specific: extract request signature and timestamp
-        sig = config['request']['signature']
-        timestamp = config['request']['timestamp']
-
-        # Vimeo specific: extract video codec and quality information
-        # First consider quality, then codecs, then take everything
-        codecs = [('vp6', 'flv'), ('vp8', 'flv'), ('h264', 'mp4')]
-        files = {'hd': [], 'sd': [], 'other': []}
-        config_files = config["video"].get("files") or config["request"].get("files")
-        for codec_name, codec_extension in codecs:
-            for quality in config_files.get(codec_name, []):
-                format_id = '-'.join((codec_name, quality)).lower()
-                key = quality if quality in files else 'other'
-                video_url = None
-                if isinstance(config_files[codec_name], dict):
-                    file_info = config_files[codec_name][quality]
-                    video_url = file_info.get('url')
-                else:
-                    file_info = {}
-                if video_url is None:
-                    video_url = "http://player.vimeo.com/play_redirect?clip_id=%s&sig=%s&time=%s&quality=%s&codecs=%s&type=moogaloop_local&embed_location=" \
-                        % (video_id, sig, timestamp, quality, codec_name.upper())
-
-                files[key].append({
-                    'ext': codec_extension,
-                    'url': video_url,
-                    'format_id': format_id,
-                    'width': int_or_none(file_info.get('width')),
-                    'height': int_or_none(file_info.get('height')),
-                    'tbr': int_or_none(file_info.get('bitrate')),
-                })
         formats = []
-        m3u8_url = config_files.get('hls', {}).get('all')
+        config_files = config['video'].get('files') or config['request'].get('files', {})
+        for f in config_files.get('progressive', []):
+            video_url = f.get('url')
+            if not video_url:
+                continue
+            formats.append({
+                'url': video_url,
+                'format_id': 'http-%s' % f.get('quality'),
+                'width': int_or_none(f.get('width')),
+                'height': int_or_none(f.get('height')),
+                'fps': int_or_none(f.get('fps')),
+                'tbr': int_or_none(f.get('bitrate')),
+            })
+        m3u8_url = config_files.get('hls', {}).get('url')
         if m3u8_url:
             m3u8_formats = self._extract_m3u8_formats(
                 m3u8_url, video_id, 'mp4', 'm3u8_native', 0, 'hls', fatal=False)
             if m3u8_formats:
                 formats.extend(m3u8_formats)
-        for key in ('other', 'sd', 'hd'):
-            formats += files[key]
-        self._sort_formats(formats)
+        # Bitrates are completely broken. Single m3u8 may contain entries in kbps and bps
+        # at the same time without actual units specified. This lead to wrong sorting.
+        self._sort_formats(formats, field_preference=('height', 'width', 'fps', 'format_id'))
 
         subtitles = {}
         text_tracks = config['request'].get('text_tracks')

From cf5881fc4defe3902367fc02b8eb39d912572d5a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 12 Nov 2015 02:13:42 +0800
Subject: [PATCH 3064/6219] Credit @ferama

For providing idea for vidto.me (#7167) and extending nowvideo support (#6760)
---
 AUTHORS                         |  1 +
 youtube_dl/extractor/twitter.py | 58 +++++++++++++++++++++++++++------
 2 files changed, 49 insertions(+), 10 deletions(-)

diff --git a/AUTHORS b/AUTHORS
index 4cdbf4a5f..f465d20ed 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -145,3 +145,4 @@ Anssi Hannula
 Lukáš Lalinský
 Qijiang Fan
 Rémy Léone
+Marco Ferragina
diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 9d3e46b94..ae1212d19 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -9,6 +9,8 @@ from ..utils import (
     float_or_none,
     xpath_text,
     remove_end,
+    int_or_none,
+    ExtractorError,
 )
 
 
@@ -120,7 +122,7 @@ class TwitterIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.|m\.|mobile\.)?twitter\.com/(?P<user_id>[^/]+)/status/(?P<id>\d+)'
     _TEMPLATE_URL = 'https://twitter.com/%s/status/%s'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'https://twitter.com/freethenipple/status/643211948184596480',
         'md5': '31cd83a116fc41f99ae3d909d4caf6a0',
         'info_dict': {
@@ -133,7 +135,19 @@ class TwitterIE(InfoExtractor):
             'uploader': 'FREE THE NIPPLE',
             'uploader_id': 'freethenipple',
         },
-    }
+    }, {
+        'url': 'https://twitter.com/giphz/status/657991469417025536/photo/1',
+        'md5': 'f36dcd5fb92bf7057f155e7d927eeb42',
+        'info_dict': {
+            'id': '657991469417025536',
+            'ext': 'mp4',
+            'title': 'Gifs - tu vai cai tu vai cai tu nao eh capaz disso tu vai cai',
+            'description': 'Gifs on Twitter: "tu vai cai tu vai cai tu nao eh capaz disso tu vai cai https://t.co/tM46VHFlO5"',
+            'thumbnail': 're:^https?://.*\.png',
+            'uploader': 'Gifs',
+            'uploader_id': 'giphz',
+        },
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
@@ -150,17 +164,41 @@ class TwitterIE(InfoExtractor):
         mobj = re.match(r'“(.*)\s+(https?://[^ ]+)”', title)
         title, short_url = mobj.groups()
 
-        card_id = self._search_regex(
-            r'["\']/i/cards/tfw/v1/(\d+)', webpage, 'twitter card url')
-        card_url = 'https://twitter.com/i/cards/tfw/v1/' + card_id
-
-        return {
-            '_type': 'url_transparent',
-            'ie_key': 'TwitterCard',
+        info = {
             'uploader_id': user_id,
             'uploader': username,
-            'url': card_url,
             'webpage_url': url,
             'description': '%s on Twitter: "%s %s"' % (username, title, short_url),
             'title': username + ' - ' + title,
         }
+
+        card_id = self._search_regex(
+            r'["\']/i/cards/tfw/v1/(\d+)', webpage, 'twitter card url', default=None)
+        if card_id:
+            card_url = 'https://twitter.com/i/cards/tfw/v1/' + card_id
+            info.update({
+                '_type': 'url_transparent',
+                'ie_key': 'TwitterCard',
+                'url': card_url,
+            })
+            return info
+
+        mobj = re.search(r'''(?x)
+            <video[^>]+class="animated-gif"[^>]+
+                (?:data-height="(?P<height>\d+)")?[^>]+
+                (?:data-width="(?P<width>\d+)")?[^>]+
+                (?:poster="(?P<poster>[^"]+)")?[^>]*>\s*
+                <source[^>]+video-src="(?P<url>[^"]+)"
+        ''', webpage)
+
+        if mobj:
+            info.update({
+                'id': twid,
+                'url': mobj.group('url'),
+                'height': int_or_none(mobj.group('height')),
+                'width': int_or_none(mobj.group('width')),
+                'thumbnail': mobj.group('poster'),
+            })
+            return info
+
+        raise ExtractorError('There\'s not video in this tweet.')

From 1ebb4717dfc67890c66abd4274e201cafda48e79 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 12 Nov 2015 21:02:56 +0100
Subject: [PATCH 3065/6219] [cbsnews] Fix construction of 'play_path' in some
 videos (fixes #7394)

---
 youtube_dl/extractor/cbsnews.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/cbsnews.py b/youtube_dl/extractor/cbsnews.py
index 52e61d85b..f9a64a0a2 100644
--- a/youtube_dl/extractor/cbsnews.py
+++ b/youtube_dl/extractor/cbsnews.py
@@ -67,9 +67,12 @@ class CBSNewsIE(InfoExtractor):
                 'format_id': format_id,
             }
             if uri.startswith('rtmp'):
+                play_path = re.sub(
+                    r'{slistFilePath}', '',
+                    uri.split('<break>')[-1].split('{break}')[-1])
                 fmt.update({
                     'app': 'ondemand?auth=cbs',
-                    'play_path': 'mp4:' + uri.split('<break>')[-1],
+                    'play_path': 'mp4:' + play_path,
                     'player_url': 'http://www.cbsnews.com/[[IMPORT]]/vidtech.cbsinteractive.com/player/3_3_0/CBSI_PLAYER_HD.swf',
                     'page_url': 'http://www.cbsnews.com',
                     'ext': 'flv',

From 5d6c3d6a665ed5de89bb3b056bb051b043400897 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 13 Nov 2015 02:41:38 +0600
Subject: [PATCH 3066/6219] [ruutu] Skip NOT-USED URLs(Closes #7478)

---
 youtube_dl/extractor/ruutu.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ruutu.py b/youtube_dl/extractor/ruutu.py
index a16b73ff4..c19107cb0 100644
--- a/youtube_dl/extractor/ruutu.py
+++ b/youtube_dl/extractor/ruutu.py
@@ -57,7 +57,8 @@ class RuutuIE(InfoExtractor):
                     extract_formats(child)
                 elif child.tag.endswith('File'):
                     video_url = child.text
-                    if not video_url or video_url in processed_urls or 'NOT_USED' in video_url:
+                    if (not video_url or video_url in processed_urls or
+                            any(p in video_url for p in ('NOT_USED', 'NOT-USED'))):
                         return
                     processed_urls.append(video_url)
                     ext = determine_ext(video_url)

From 91d644b5ba7e520363d146442682b7df5a175d31 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 13 Nov 2015 02:43:27 +0600
Subject: [PATCH 3067/6219] [ruutu] Relax formats extraction

---
 youtube_dl/extractor/ruutu.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/ruutu.py b/youtube_dl/extractor/ruutu.py
index c19107cb0..e417bf661 100644
--- a/youtube_dl/extractor/ruutu.py
+++ b/youtube_dl/extractor/ruutu.py
@@ -63,11 +63,15 @@ class RuutuIE(InfoExtractor):
                     processed_urls.append(video_url)
                     ext = determine_ext(video_url)
                     if ext == 'm3u8':
-                        formats.extend(self._extract_m3u8_formats(
-                            video_url, video_id, 'mp4', m3u8_id='hls'))
+                        m3u8_formats = self._extract_m3u8_formats(
+                            video_url, video_id, 'mp4', m3u8_id='hls', fatal=False)
+                        if m3u8_formats:
+                            formats.extend(m3u8_formats)
                     elif ext == 'f4m':
-                        formats.extend(self._extract_f4m_formats(
-                            video_url, video_id, f4m_id='hds'))
+                        f4m_formats = self._extract_f4m_formats(
+                            video_url, video_id, f4m_id='hds', fatal=False)
+                        if f4m_formats:
+                            formats.extend(f4m_formats)
                     else:
                         proto = compat_urllib_parse_urlparse(video_url).scheme
                         if not child.tag.startswith('HTTP') and proto != 'rtmp':

From a1ec9a7553b6bb09b0f88ce5e987c04e2f16ba4d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 13 Nov 2015 11:07:30 +0100
Subject: [PATCH 3068/6219] release 2015.11.13

---
 docs/supportedsites.md | 2 +-
 youtube_dl/version.py  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index a9820c1f5..5016ba4bc 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -200,7 +200,6 @@
  - **GodTube**
  - **GoldenMoustache**
  - **Golem**
- - **GorillaVid**: GorillaVid.in, daclips.in, movpod.in, fastvideo.in, realvid.net and filehoot.com
  - **Goshgay**
  - **Groupon**
  - **Hark**
@@ -671,6 +670,7 @@
  - **WSJ**: Wall Street Journal
  - **XBef**
  - **XboxClips**
+ - **XFileShare**: XFileShare based sites: GorillaVid.in, daclips.in, movpod.in, fastvideo.in, realvid.net, filehoot.com and vidto.me
  - **XHamster**
  - **XHamsterEmbed**
  - **XMinus**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index b3d254005..6585d60d5 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.11.10'
+__version__ = '2015.11.13'

From b84a5f03375d086e9912cfa7c690ff824b1092a9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 13 Nov 2015 18:55:07 +0100
Subject: [PATCH 3069/6219] [twitter] Update tests checksums

---
 youtube_dl/extractor/twitter.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index ae1212d19..0db7a4440 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -20,7 +20,7 @@ class TwitterCardIE(InfoExtractor):
     _TESTS = [
         {
             'url': 'https://twitter.com/i/cards/tfw/v1/560070183650213889',
-            'md5': '7d2f6b4d2eb841a7ccc893d479bfceb4',
+            'md5': '4fa26a35f9d1bf4b646590ba8e84be19',
             'info_dict': {
                 'id': '560070183650213889',
                 'ext': 'mp4',
@@ -124,7 +124,7 @@ class TwitterIE(InfoExtractor):
 
     _TESTS = [{
         'url': 'https://twitter.com/freethenipple/status/643211948184596480',
-        'md5': '31cd83a116fc41f99ae3d909d4caf6a0',
+        'md5': 'db6612ec5d03355953c3ca9250c97e5e',
         'info_dict': {
             'id': '643211948184596480',
             'ext': 'mp4',

From b703ebeeafed3e890270633788c7340d174cde86 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 13 Nov 2015 19:09:42 +0100
Subject: [PATCH 3070/6219] [twitter] Don't fail if the description doesn't
 contain an URL (fixes #7489)

---
 youtube_dl/extractor/twitter.py | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 0db7a4440..2bd5946ac 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -147,6 +147,17 @@ class TwitterIE(InfoExtractor):
             'uploader': 'Gifs',
             'uploader_id': 'giphz',
         },
+    }, {
+        'url': 'https://twitter.com/starwars/status/665052190608723968',
+        'md5': '39b7199856dee6cd4432e72c74bc69d4',
+        'info_dict': {
+            'id': '665052190608723968',
+            'ext': 'mp4',
+            'title': 'Star Wars - A new beginning is coming December 18. Watch the official 60 second #TV spot for #StarWars: #TheForceAwakens.',
+            'description': 'Star Wars on Twitter: "A new beginning is coming December 18. Watch the official 60 second #TV spot for #StarWars: #TheForceAwakens."',
+            'uploader_id': 'starwars',
+            'uploader': 'Star Wars',
+        },
     }]
 
     def _real_extract(self, url):
@@ -158,17 +169,16 @@ class TwitterIE(InfoExtractor):
 
         username = remove_end(self._og_search_title(webpage), ' on Twitter')
 
-        title = self._og_search_description(webpage).strip('').replace('\n', ' ')
+        title = description = self._og_search_description(webpage).strip('').replace('\n', ' ').strip('“”')
 
         # strip  'https -_t.co_BJYgOjSeGA' junk from filenames
-        mobj = re.match(r'“(.*)\s+(https?://[^ ]+)”', title)
-        title, short_url = mobj.groups()
+        title = re.sub(r'\s+(https?://[^ ]+)', '', title)
 
         info = {
             'uploader_id': user_id,
             'uploader': username,
             'webpage_url': url,
-            'description': '%s on Twitter: "%s %s"' % (username, title, short_url),
+            'description': '%s on Twitter: "%s"' % (username, description),
             'title': username + ' - ' + title,
         }
 

From 4e21b3a94f1ce7ba3757d59d1ac4baf9efaeed84 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Nov 2015 00:22:32 +0600
Subject: [PATCH 3071/6219] [cbs] Use android UA for higher quality streams
 (Closes #7490)

---
 youtube_dl/extractor/cbs.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/cbs.py b/youtube_dl/extractor/cbs.py
index 75fffb156..43f05d278 100644
--- a/youtube_dl/extractor/cbs.py
+++ b/youtube_dl/extractor/cbs.py
@@ -1,6 +1,8 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..compat import compat_urllib_request
+from ..utils import smuggle_url
 
 
 class CBSIE(InfoExtractor):
@@ -46,13 +48,19 @@ class CBSIE(InfoExtractor):
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
-        webpage = self._download_webpage(url, display_id)
+        request = compat_urllib_request.Request(url)
+        # Android UA is served with higher quality (720p) streams (see
+        # https://github.com/rg3/youtube-dl/issues/7490)
+        request.add_header('User-Agent', 'Mozilla/5.0 (Linux; Android 4.4; Nexus 5)')
+        webpage = self._download_webpage(request, display_id)
         real_id = self._search_regex(
             [r"video\.settings\.pid\s*=\s*'([^']+)';", r"cbsplayer\.pid\s*=\s*'([^']+)';"],
             webpage, 'real video ID')
         return {
             '_type': 'url_transparent',
             'ie_key': 'ThePlatform',
-            'url': 'theplatform:%s' % real_id,
+            'url': smuggle_url(
+                'http://link.theplatform.com/s/dJ5BDC/%s?mbr=true&manifest=m3u' % real_id,
+                {'force_smil_url': True}),
             'display_id': display_id,
         }

From a662489877aa8d88b898a4984c2e580b9edbe7de Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Nov 2015 05:09:50 +0600
Subject: [PATCH 3072/6219] [brightcove:embedinpage] Make more robust and
 extract rtmp streams

---
 youtube_dl/extractor/brightcove.py | 117 +++++++++++++++++++----------
 1 file changed, 79 insertions(+), 38 deletions(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 2c7d968a8..2ad35ec90 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -23,6 +23,7 @@ from ..utils import (
     unescapeHTML,
     unsmuggle_url,
     js_to_json,
+    float_or_none,
     int_or_none,
     parse_iso8601,
     extract_attributes,
@@ -353,7 +354,7 @@ class BrightcoveIE(InfoExtractor):
 
 
 class BrightcoveInPageEmbedIE(InfoExtractor):
-    _VALID_URL = r'https?://players\.brightcove\.net/(?P<account_id>\d+)/([a-z0-9-]+)_([a-z]+)/index.html?.*videoId=(?P<video_id>\d+)'
+    _VALID_URL = r'https?://players\.brightcove\.net/(?P<account_id>\d+)/(?P<player_id>[\da-f-]+)_(?P<embed>[a-z]+)/index\.html\?.*videoId=(?P<video_id>\d+)'
     _TEST = {
         'url': 'http://players.brightcove.net/929656772001/e41d32dc-ec74-459e-a845-6c69f7b724ea_default/index.html?videoId=4463358922001',
         'md5': 'c8100925723840d4b0d243f7025703be',
@@ -385,59 +386,99 @@ class BrightcoveInPageEmbedIE(InfoExtractor):
     def _real_extract(self, url):
         account_id, player_id, embed, video_id = re.match(self._VALID_URL, url).groups()
 
-        webpage = self._download_webpage('http://players.brightcove.net/%s/%s_%s/index.min.js' % (account_id, player_id, embed), video_id)
+        webpage = self._download_webpage(
+            'http://players.brightcove.net/%s/%s_%s/index.min.js'
+            % (account_id, player_id, embed), video_id)
 
-        catalog = self._parse_json(
-            js_to_json(
-                self._search_regex(
-                    r'catalog\(({[^}]+})\);',
-                    webpage,
-                    'catalog'
-                )
-            ),
-            video_id
-        )
-        policy_key = catalog['policyKey']
+        policy_key = None
+
+        catalog = self._search_regex(
+            r'catalog\(({.+?})\);', webpage, 'catalog', default=None)
+        if catalog:
+            catalog = self._parse_json(
+                js_to_json(catalog), video_id, fatal=False)
+            if catalog:
+                policy_key = catalog.get('policyKey')
+
+        if not policy_key:
+            policy_key = self._search_regex(
+                r'policyKey\s*:\s*(["\'])(?P<pk>.+?)\1',
+                webpage, 'policy key', group='pk')
 
         req = compat_urllib_request.Request(
-            'https://edge.api.brightcove.com/playback/v1/accounts/%s/videos/%s' % (account_id, video_id),
+            'https://edge.api.brightcove.com/playback/v1/accounts/%s/videos/%s'
+            % (account_id, video_id),
             headers={'Accept': 'application/json;pk=%s' % policy_key})
         json_data = self._download_json(req, video_id)
 
         title = json_data['name']
+
+        formats = []
+        for source in json_data.get('sources', []):
+            source_type = source.get('type')
+            src = source.get('src')
+            if source_type == 'application/x-mpegURL':
+                if not src:
+                    continue
+                m3u8_formats = self._extract_m3u8_formats(
+                    src, video_id, 'mp4', entry_protocol='m3u8_native',
+                    m3u8_id='hls', fatal=False)
+                if m3u8_formats:
+                    formats.extend(m3u8_formats)
+            else:
+                streaming_src = source.get('streaming_src')
+                stream_name, app_name = source.get('stream_name'), source.get('app_name')
+                if not src and not streaming_src and (not stream_name or not app_name):
+                    continue
+                tbr = float_or_none(source.get('avg_bitrate'), 1000)
+                height = int_or_none(source.get('height'))
+                f = {
+                    'tbr': tbr,
+                    'width': int_or_none(source.get('width')),
+                    'height': height,
+                    'filesize': int_or_none(source.get('size')),
+                    'container': source.get('container'),
+                    'vcodec': source.get('codec'),
+                    'ext': source.get('container').lower(),
+                }
+
+                def build_format_id(kind):
+                    format_id = kind
+                    if tbr:
+                        format_id += '-%dk' % int(tbr)
+                    if height:
+                        format_id += '-%dp' % height
+                    return format_id
+
+                if src or streaming_src:
+                    f.update({
+                        'url': src or streaming_src,
+                        'format_id': build_format_id('http' if src else 'http-streaming'),
+                        'preference': 2 if src else 1,
+                    })
+                else:
+                    f.update({
+                        'url': app_name,
+                        'play_path': stream_name,
+                        'format_id': build_format_id('rtmp'),
+                    })
+                formats.append(f)
+        self._sort_formats(formats)
+
         description = json_data.get('description')
         thumbnail = json_data.get('thumbnail')
         timestamp = parse_iso8601(json_data.get('published_at'))
-        duration = int_or_none(json_data.get('duration'))
-
-        formats = []
-        for source in json_data.get('sources'):
-            source_type = source.get('type')
-            if source_type == 'application/x-mpegURL':
-                formats.extend(self._extract_m3u8_formats(source.get('src'), video_id))
-            else:
-                src = source.get('src') or source.get('streaming_src')
-                if src:
-                    formats.append({
-                        'url': src,
-                        'tbr': source.get('avg_bitrate'),
-                        'width': int_or_none(source.get('width')),
-                        'height': int_or_none(source.get('height')),
-                        'filesize': source.get('size'),
-                        'container': source.get('container'),
-                        'vcodec': source.get('codec'),
-                        'ext': source.get('container').lower(),
-                    })
-
-        self._sort_formats(formats)
+        duration = float_or_none(json_data.get('duration'), 1000)
+        tags = json_data.get('tags', [])
 
         return {
             'id': video_id,
             'title': title,
             'description': description,
             'thumbnail': thumbnail,
-            'timestamp': timestamp,
             'duration': duration,
-            'formats': formats,
+            'timestamp': timestamp,
             'uploader_id': account_id,
+            'formats': formats,
+            'tags': tags,
         }

From 536f819eda975224666374a9ce83cc3472f5aa5a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Nov 2015 05:51:05 +0600
Subject: [PATCH 3073/6219] [brightcove] Imrove extraction of new embeds

---
 youtube_dl/extractor/brightcove.py | 42 +++++++++++++++++++++---------
 1 file changed, 29 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 2ad35ec90..d494b8b67 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -354,7 +354,7 @@ class BrightcoveIE(InfoExtractor):
 
 
 class BrightcoveInPageEmbedIE(InfoExtractor):
-    _VALID_URL = r'https?://players\.brightcove\.net/(?P<account_id>\d+)/(?P<player_id>[\da-f-]+)_(?P<embed>[a-z]+)/index\.html\?.*videoId=(?P<video_id>\d+)'
+    _VALID_URL = r'https?://players\.brightcove\.net/(?P<account_id>\d+)/(?P<player_id>[^/]+)_(?P<embed>[^/]+)/index\.html\?.*videoId=(?P<video_id>\d+)'
     _TEST = {
         'url': 'http://players.brightcove.net/929656772001/e41d32dc-ec74-459e-a845-6c69f7b724ea_default/index.html?videoId=4463358922001',
         'md5': 'c8100925723840d4b0d243f7025703be',
@@ -370,18 +370,34 @@ class BrightcoveInPageEmbedIE(InfoExtractor):
         }
     }
 
-    @staticmethod
-    def _extract_url(webpage):
-        video_attributes = re.search(r'(?s)<video([^>]*)>.*?</(?:video|audio)>', webpage)
-        if video_attributes:
-            video_attributes = extract_attributes(video_attributes.group(), r'(?s)\s*data-(account|video-id|playlist-id|policy-key|player|embed)\s*=\s*["\']([^"\']+)["\']')
-            account_id = video_attributes.get('account')
-            player_id = video_attributes.get('player')
-            embed = video_attributes.get('embed')
-            video_id = video_attributes.get('video-id')
-            if account_id and player_id and embed and video_id:
-                return 'http://players.brightcove.net/%s/%s_%s/index.html?videoId=%s' % (account_id, player_id, embed, video_id)
-        return None
+    def _extract_urls(self, webpage):
+        # Reference:
+        # 1. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/publish-video.html#setvideoiniframe
+        # 2. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/publish-video.html#setvideousingjavascript)
+        # 3. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/embed-in-page.html
+
+        entries = []
+
+        # Look for iframe embeds [1]
+        for _, url in re.findall(
+                r'<iframe[^>]+src=(["\'])((?:https?:)//players\.brightcove\.net/\d+/[^/]+/index\.html.+?)\1', webpage):
+            entries.append(self.url_result(self._proto_relative_url(url)))
+        # Look for embed_in_page embeds [2]
+        # According to examples from [3] it's unclear whether video id may be optional
+        # and what to do when it is
+        for video_id, account_id, player_id, embed in re.findall(
+                r'''(?sx)
+                    <video[^>]+
+                        data-video-id=["\'](\d+)["\'][^>]*>.*?
+                    </video>.*?
+                    <script[^>]+
+                        src=["\'](?:https?:)?//players\.brightcove\.net/
+                        (\d+)/([\da-f-]+)_([^/]+)/index\.min\.js
+                ''', webpage):
+            entries.append(self.url_result(
+                'http://players.brightcove.net/%s/%s_%s/index.html?videoId=%s'
+                % (account_id, player_id, embed, video_id)))
+        return entries
 
     def _real_extract(self, url):
         account_id, player_id, embed, video_id = re.match(self._VALID_URL, url).groups()

From 4fcaa4f4a5ef328009bef53ebc491ebe76452550 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Nov 2015 05:54:16 +0600
Subject: [PATCH 3074/6219] [brightcove] Rename extractor to brightcove legacy

Old embedding approaches are now "Legacy Studio"
---
 youtube_dl/extractor/__init__.py   | 2 +-
 youtube_dl/extractor/brightcove.py | 2 +-
 youtube_dl/extractor/generic.py    | 4 ++--
 youtube_dl/extractor/nowness.py    | 4 ++--
 youtube_dl/extractor/safari.py     | 4 ++--
 youtube_dl/extractor/space.py      | 6 +++---
 youtube_dl/extractor/tlc.py        | 6 +++---
 7 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 08cb93d76..8a0f76d7e 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -61,7 +61,7 @@ from .bpb import BpbIE
 from .br import BRIE
 from .breakcom import BreakIE
 from .brightcove import (
-    BrightcoveIE,
+    BrightcoveLegacyIE,
     BrightcoveInPageEmbedIE,
 )
 from .buzzfeed import BuzzFeedIE
diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index d494b8b67..4dbc2e975 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -30,7 +30,7 @@ from ..utils import (
 )
 
 
-class BrightcoveIE(InfoExtractor):
+class BrightcoveLegacyIE(InfoExtractor):
     _VALID_URL = r'(?:https?://.*brightcove\.com/(services|viewer).*?\?|brightcove:)(?P<query>.*)'
     _FEDERATED_URL_TEMPLATE = 'http://c.brightcove.com/services/viewer/htmlFederated?%s'
 
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 34d930a2d..8f99dd9b1 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -31,7 +31,7 @@ from ..utils import (
     xpath_text,
 )
 from .brightcove import (
-    BrightcoveIE,
+    BrightcoveLegacyIE,
     BrightcoveInPageEmbedIE,
 )
 from .nbc import NBCSportsVPlayerIE
@@ -1305,7 +1305,7 @@ class GenericIE(InfoExtractor):
                 urlrs, playlist_id=video_id, playlist_title=video_title)
 
         # Look for BrightCove:
-        bc_urls = BrightcoveIE._extract_brightcove_urls(webpage)
+        bc_urls = BrightcoveLegacyIE._extract_brightcove_urls(webpage)
         if bc_urls:
             self.to_screen('Brightcove video detected.')
             entries = [{
diff --git a/youtube_dl/extractor/nowness.py b/youtube_dl/extractor/nowness.py
index b97f62fdb..dab487ea4 100644
--- a/youtube_dl/extractor/nowness.py
+++ b/youtube_dl/extractor/nowness.py
@@ -1,7 +1,7 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
-from .brightcove import BrightcoveIE
+from .brightcove import BrightcoveLegacyIE
 from .common import InfoExtractor
 from ..utils import ExtractorError
 from ..compat import (
@@ -22,7 +22,7 @@ class NownessBaseIE(InfoExtractor):
                             'http://www.nowness.com/iframe?id=%s' % video_id, video_id,
                             note='Downloading player JavaScript',
                             errnote='Unable to download player JavaScript')
-                        bc_url = BrightcoveIE._extract_brightcove_url(player_code)
+                        bc_url = BrightcoveLegacyIE._extract_brightcove_url(player_code)
                         if bc_url is None:
                             raise ExtractorError('Could not find player definition')
                         return self.url_result(bc_url, 'Brightcove')
diff --git a/youtube_dl/extractor/safari.py b/youtube_dl/extractor/safari.py
index a602af692..4f1f05c6a 100644
--- a/youtube_dl/extractor/safari.py
+++ b/youtube_dl/extractor/safari.py
@@ -4,7 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from .brightcove import BrightcoveIE
+from .brightcove import BrightcoveLegacyIE
 
 from ..compat import (
     compat_urllib_parse,
@@ -112,7 +112,7 @@ class SafariIE(SafariBaseIE):
             '%s/%s/chapter-content/%s.html' % (self._API_BASE, course_id, part),
             part)
 
-        bc_url = BrightcoveIE._extract_brightcove_url(webpage)
+        bc_url = BrightcoveLegacyIE._extract_brightcove_url(webpage)
         if not bc_url:
             raise ExtractorError('Could not extract Brightcove URL from %s' % url, expected=True)
 
diff --git a/youtube_dl/extractor/space.py b/youtube_dl/extractor/space.py
index c2d0d36a6..2f190f764 100644
--- a/youtube_dl/extractor/space.py
+++ b/youtube_dl/extractor/space.py
@@ -3,7 +3,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from .brightcove import BrightcoveIE
+from .brightcove import BrightcoveLegacyIE
 from ..utils import RegexNotFoundError, ExtractorError
 
 
@@ -31,8 +31,8 @@ class SpaceIE(InfoExtractor):
             brightcove_url = self._og_search_video_url(webpage)
         except RegexNotFoundError:
             # Other videos works fine with the info from the object
-            brightcove_url = BrightcoveIE._extract_brightcove_url(webpage)
+            brightcove_url = BrightcoveLegacyIE._extract_brightcove_url(webpage)
         if brightcove_url is None:
             raise ExtractorError(
                 'The webpage does not contain a video', expected=True)
-        return self.url_result(brightcove_url, BrightcoveIE.ie_key())
+        return self.url_result(brightcove_url, BrightcoveLegacyIE.ie_key())
diff --git a/youtube_dl/extractor/tlc.py b/youtube_dl/extractor/tlc.py
index 13263614c..d6d038a8d 100644
--- a/youtube_dl/extractor/tlc.py
+++ b/youtube_dl/extractor/tlc.py
@@ -3,7 +3,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from .brightcove import BrightcoveIE
+from .brightcove import BrightcoveLegacyIE
 from .discovery import DiscoveryIE
 from ..compat import compat_urlparse
 
@@ -66,6 +66,6 @@ class TlcDeIE(InfoExtractor):
 
         return {
             '_type': 'url',
-            'url': BrightcoveIE._extract_brightcove_url(iframe),
-            'ie': BrightcoveIE.ie_key(),
+            'url': BrightcoveLegacyIE._extract_brightcove_url(iframe),
+            'ie': BrightcoveLegacyIE.ie_key(),
         }

From 5c17f0a67a3a0518f448825eee54d16045acd63c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Nov 2015 05:55:59 +0600
Subject: [PATCH 3075/6219] [brightcove:embedinpage] Rename extractor to
 brightcove new

It's not actually embed_in_page but "New Studio" and allows both iframe and embed_in_page embeds
---
 youtube_dl/extractor/__init__.py   | 2 +-
 youtube_dl/extractor/brightcove.py | 2 +-
 youtube_dl/extractor/generic.py    | 4 ++--
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 8a0f76d7e..64ce3210b 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -62,7 +62,7 @@ from .br import BRIE
 from .breakcom import BreakIE
 from .brightcove import (
     BrightcoveLegacyIE,
-    BrightcoveInPageEmbedIE,
+    BrightcoveNewIE,
 )
 from .buzzfeed import BuzzFeedIE
 from .byutv import BYUtvIE
diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 4dbc2e975..5aca0b378 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -353,7 +353,7 @@ class BrightcoveLegacyIE(InfoExtractor):
         return info
 
 
-class BrightcoveInPageEmbedIE(InfoExtractor):
+class BrightcoveNewIE(InfoExtractor):
     _VALID_URL = r'https?://players\.brightcove\.net/(?P<account_id>\d+)/(?P<player_id>[^/]+)_(?P<embed>[^/]+)/index\.html\?.*videoId=(?P<video_id>\d+)'
     _TEST = {
         'url': 'http://players.brightcove.net/929656772001/e41d32dc-ec74-459e-a845-6c69f7b724ea_default/index.html?videoId=4463358922001',
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 8f99dd9b1..0797e1a90 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -32,7 +32,7 @@ from ..utils import (
 )
 from .brightcove import (
     BrightcoveLegacyIE,
-    BrightcoveInPageEmbedIE,
+    BrightcoveNewIE,
 )
 from .nbc import NBCSportsVPlayerIE
 from .ooyala import OoyalaIE
@@ -1322,7 +1322,7 @@ class GenericIE(InfoExtractor):
             }
 
         # Look for Brightcove In Page Embed:
-        brightcove_in_page_embed_url = BrightcoveInPageEmbedIE._extract_url(webpage)
+        brightcove_in_page_embed_url = BrightcoveNewIE._extract_url(webpage)
         if brightcove_in_page_embed_url:
             return self.url_result(brightcove_in_page_embed_url, 'BrightcoveInPageEmbed')
 

From e721d857c2b24c10c09626a4a79172d85e0dc5fb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Nov 2015 05:56:51 +0600
Subject: [PATCH 3076/6219] [brightcove] Clarify IE_NAMEs

---
 youtube_dl/extractor/brightcove.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 5aca0b378..8ee5486fe 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -31,6 +31,7 @@ from ..utils import (
 
 
 class BrightcoveLegacyIE(InfoExtractor):
+    IE_NAME = 'brightcove:legacy'
     _VALID_URL = r'(?:https?://.*brightcove\.com/(services|viewer).*?\?|brightcove:)(?P<query>.*)'
     _FEDERATED_URL_TEMPLATE = 'http://c.brightcove.com/services/viewer/htmlFederated?%s'
 
@@ -354,6 +355,7 @@ class BrightcoveLegacyIE(InfoExtractor):
 
 
 class BrightcoveNewIE(InfoExtractor):
+    IE_NAME = 'brightcove:new'
     _VALID_URL = r'https?://players\.brightcove\.net/(?P<account_id>\d+)/(?P<player_id>[^/]+)_(?P<embed>[^/]+)/index\.html\?.*videoId=(?P<video_id>\d+)'
     _TEST = {
         'url': 'http://players.brightcove.net/929656772001/e41d32dc-ec74-459e-a845-6c69f7b724ea_default/index.html?videoId=4463358922001',

From 24af85298ed1862ac809677e70ff59f3e9ee3234 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Nov 2015 06:01:56 +0600
Subject: [PATCH 3077/6219] [brightcove] Fix _extract_urls

---
 youtube_dl/extractor/brightcove.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 8ee5486fe..1c7783dcb 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -372,7 +372,8 @@ class BrightcoveNewIE(InfoExtractor):
         }
     }
 
-    def _extract_urls(self, webpage):
+    @staticmethod
+    def _extract_urls(webpage):
         # Reference:
         # 1. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/publish-video.html#setvideoiniframe
         # 2. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/publish-video.html#setvideousingjavascript)
@@ -383,7 +384,7 @@ class BrightcoveNewIE(InfoExtractor):
         # Look for iframe embeds [1]
         for _, url in re.findall(
                 r'<iframe[^>]+src=(["\'])((?:https?:)//players\.brightcove\.net/\d+/[^/]+/index\.html.+?)\1', webpage):
-            entries.append(self.url_result(self._proto_relative_url(url)))
+            entries.append(url)
         # Look for embed_in_page embeds [2]
         # According to examples from [3] it's unclear whether video id may be optional
         # and what to do when it is
@@ -396,9 +397,9 @@ class BrightcoveNewIE(InfoExtractor):
                         src=["\'](?:https?:)?//players\.brightcove\.net/
                         (\d+)/([\da-f-]+)_([^/]+)/index\.min\.js
                 ''', webpage):
-            entries.append(self.url_result(
+            entries.append(
                 'http://players.brightcove.net/%s/%s_%s/index.html?videoId=%s'
-                % (account_id, player_id, embed, video_id)))
+                % (account_id, player_id, embed, video_id))
         return entries
 
     def _real_extract(self, url):

From f6519f89b09be788549f68ba12f0cc31c55d9751 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Nov 2015 06:03:07 +0600
Subject: [PATCH 3078/6219] [generic] Extract Brightcove New Studio embeds

---
 youtube_dl/extractor/generic.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 0797e1a90..334864db3 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1321,10 +1321,10 @@ class GenericIE(InfoExtractor):
                 'entries': entries,
             }
 
-        # Look for Brightcove In Page Embed:
-        brightcove_in_page_embed_url = BrightcoveNewIE._extract_url(webpage)
-        if brightcove_in_page_embed_url:
-            return self.url_result(brightcove_in_page_embed_url, 'BrightcoveInPageEmbed')
+        # Look for Brightcove New Studio embeds
+        bc_urls = BrightcoveNewIE._extract_urls(webpage)
+        if bc_urls:
+            return _playlist_from_matches(bc_urls, ie='BrightcoveNew')
 
         # Look for embedded rtl.nl player
         matches = re.findall(

From 1f4b722b00fd5c24468cd4d072e8b5c5428ca515 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Nov 2015 06:03:32 +0600
Subject: [PATCH 3079/6219] [generic] Clarify Brightcove Legacy Studio comment

---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 334864db3..8ba0a9913 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1304,7 +1304,7 @@ class GenericIE(InfoExtractor):
             return self.playlist_result(
                 urlrs, playlist_id=video_id, playlist_title=video_title)
 
-        # Look for BrightCove:
+        # Look for Brightcove Legacy Studio embeds
         bc_urls = BrightcoveLegacyIE._extract_brightcove_urls(webpage)
         if bc_urls:
             self.to_screen('Brightcove video detected.')

From 3b7d9aa487399e06bba5dc03c90b6576c2b067b9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Nov 2015 06:05:46 +0600
Subject: [PATCH 3080/6219] Rename all references to legacy studio Brightcove
 extractor

---
 youtube_dl/extractor/aljazeera.py | 4 ++--
 youtube_dl/extractor/generic.py   | 8 ++++----
 youtube_dl/extractor/nowness.py   | 2 +-
 youtube_dl/extractor/safari.py    | 2 +-
 youtube_dl/extractor/space.py     | 2 +-
 5 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/aljazeera.py b/youtube_dl/extractor/aljazeera.py
index 184a14a4f..5b2c0dc9a 100644
--- a/youtube_dl/extractor/aljazeera.py
+++ b/youtube_dl/extractor/aljazeera.py
@@ -15,7 +15,7 @@ class AlJazeeraIE(InfoExtractor):
             'description': 'As a birth attendant advocating for family planning, Remy is on the frontline of Tondo\'s battle with overcrowding.',
             'uploader': 'Al Jazeera English',
         },
-        'add_ie': ['Brightcove'],
+        'add_ie': ['BrightcoveLegacy'],
         'skip': 'Not accessible from Travis CI server',
     }
 
@@ -32,5 +32,5 @@ class AlJazeeraIE(InfoExtractor):
                 'playerKey=AQ~~%2CAAAAmtVJIFk~%2CTVGOQ5ZTwJbeMWnq5d_H4MOM57xfzApc'
                 '&%40videoPlayer={0}'.format(brightcove_id)
             ),
-            'ie_key': 'Brightcove',
+            'ie_key': 'BrightcoveLegacy',
         }
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 8ba0a9913..51516a38a 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -278,7 +278,7 @@ class GenericIE(InfoExtractor):
         # it also tests brightcove videos that need to set the 'Referer' in the
         # http requests
         {
-            'add_ie': ['Brightcove'],
+            'add_ie': ['BrightcoveLegacy'],
             'url': 'http://www.bfmtv.com/video/bfmbusiness/cours-bourse/cours-bourse-l-analyse-technique-154522/',
             'info_dict': {
                 'id': '2765128793001',
@@ -302,7 +302,7 @@ class GenericIE(InfoExtractor):
                 'uploader': 'thestar.com',
                 'description': 'Mississauga resident David Farmer is still out of power as a result of the ice storm a month ago. To keep the house warm, Farmer cuts wood from his property for a wood burning stove downstairs.',
             },
-            'add_ie': ['Brightcove'],
+            'add_ie': ['BrightcoveLegacy'],
         },
         {
             'url': 'http://www.championat.com/video/football/v/87/87499.html',
@@ -317,7 +317,7 @@ class GenericIE(InfoExtractor):
         },
         {
             # https://github.com/rg3/youtube-dl/issues/3541
-            'add_ie': ['Brightcove'],
+            'add_ie': ['BrightcoveLegacy'],
             'url': 'http://www.kijk.nl/sbs6/leermijvrouwenkennen/videos/jqMiXKAYan2S/aflevering-1',
             'info_dict': {
                 'id': '3866516442001',
@@ -1311,7 +1311,7 @@ class GenericIE(InfoExtractor):
             entries = [{
                 '_type': 'url',
                 'url': smuggle_url(bc_url, {'Referer': url}),
-                'ie_key': 'Brightcove'
+                'ie_key': 'BrightcoveLegacy'
             } for bc_url in bc_urls]
 
             return {
diff --git a/youtube_dl/extractor/nowness.py b/youtube_dl/extractor/nowness.py
index dab487ea4..0fba55833 100644
--- a/youtube_dl/extractor/nowness.py
+++ b/youtube_dl/extractor/nowness.py
@@ -25,7 +25,7 @@ class NownessBaseIE(InfoExtractor):
                         bc_url = BrightcoveLegacyIE._extract_brightcove_url(player_code)
                         if bc_url is None:
                             raise ExtractorError('Could not find player definition')
-                        return self.url_result(bc_url, 'Brightcove')
+                        return self.url_result(bc_url, 'BrightcoveLegacy')
                     elif source == 'vimeo':
                         return self.url_result('http://vimeo.com/%s' % video_id, 'Vimeo')
                     elif source == 'youtube':
diff --git a/youtube_dl/extractor/safari.py b/youtube_dl/extractor/safari.py
index 4f1f05c6a..e9e33d0a3 100644
--- a/youtube_dl/extractor/safari.py
+++ b/youtube_dl/extractor/safari.py
@@ -116,7 +116,7 @@ class SafariIE(SafariBaseIE):
         if not bc_url:
             raise ExtractorError('Could not extract Brightcove URL from %s' % url, expected=True)
 
-        return self.url_result(smuggle_url(bc_url, {'Referer': url}), 'Brightcove')
+        return self.url_result(smuggle_url(bc_url, {'Referer': url}), 'BrightcoveLegacy')
 
 
 class SafariCourseIE(SafariBaseIE):
diff --git a/youtube_dl/extractor/space.py b/youtube_dl/extractor/space.py
index 2f190f764..ebb5d6ec0 100644
--- a/youtube_dl/extractor/space.py
+++ b/youtube_dl/extractor/space.py
@@ -10,7 +10,7 @@ from ..utils import RegexNotFoundError, ExtractorError
 class SpaceIE(InfoExtractor):
     _VALID_URL = r'https?://(?:(?:www|m)\.)?space\.com/\d+-(?P<title>[^/\.\?]*?)-video\.html'
     _TEST = {
-        'add_ie': ['Brightcove'],
+        'add_ie': ['BrightcoveLegacy'],
         'url': 'http://www.space.com/23373-huge-martian-landforms-detail-revealed-by-european-probe-video.html',
         'info_dict': {
             'id': '2780937028001',

From 75eac8961ee2ff004891ec57d5a2fec4f0b5574d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Nov 2015 06:07:24 +0600
Subject: [PATCH 3081/6219] [brightcove] Remove unused import

---
 youtube_dl/extractor/brightcove.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 1c7783dcb..ef34ae48f 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -26,7 +26,6 @@ from ..utils import (
     float_or_none,
     int_or_none,
     parse_iso8601,
-    extract_attributes,
 )
 
 

From c7b959ce383040f1d507eef0e43041029583b307 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Nov 2015 06:07:44 +0600
Subject: [PATCH 3082/6219] [utils] Remove unused function

---
 youtube_dl/utils.py | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 65556d056..d39f313a4 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -259,15 +259,6 @@ def get_element_by_attribute(attribute, value, html):
     return unescapeHTML(res)
 
 
-def extract_attributes(attributes_str, attributes_regex=r'(?s)\s*([^\s=]+)\s*=\s*["\']([^"\']+)["\']'):
-    attributes = re.findall(attributes_regex, attributes_str)
-    attributes_dict = {}
-    if attributes:
-        for (attribute_name, attribute_value) in attributes:
-            attributes_dict[attribute_name] = attribute_value
-    return attributes_dict
-
-
 def clean_html(html):
     """Clean an HTML snippet into a readable string"""
 

From fd91257c4019a1956cc59eac1232f2c413b9747d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Nov 2015 06:08:36 +0600
Subject: [PATCH 3083/6219] [brightcove] Order imports alphabetically

---
 youtube_dl/extractor/brightcove.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index ef34ae48f..f137ba8c6 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -20,12 +20,12 @@ from ..utils import (
     ExtractorError,
     find_xpath_attr,
     fix_xml_ampersands,
-    unescapeHTML,
-    unsmuggle_url,
-    js_to_json,
     float_or_none,
+    js_to_json,
     int_or_none,
     parse_iso8601,
+    unescapeHTML,
+    unsmuggle_url,
 )
 
 

From e01b432ad38b36b1ba6cb1b6dccecec51f9fc1e7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Nov 2015 06:11:17 +0600
Subject: [PATCH 3084/6219] [brightcove:new] Fix test

---
 youtube_dl/extractor/brightcove.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index f137ba8c6..6b184157c 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -366,7 +366,7 @@ class BrightcoveNewIE(InfoExtractor):
             'description': 'md5:eac376a4fe366edc70279bfb681aea16',
             'timestamp': 1441391203,
             'upload_date': '20150904',
-            'duration': 165768,
+            'duration': 165.768,
             'uploader_id': '929656772001',
         }
     }

From cb33d389ed452583638f73360599642b86617dfb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Nov 2015 06:20:09 +0600
Subject: [PATCH 3085/6219] [brightcove:new] Add test with rtmp streams

---
 youtube_dl/extractor/brightcove.py | 24 +++++++++++++++++++++---
 1 file changed, 21 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 6b184157c..9ae724c2b 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -356,7 +356,7 @@ class BrightcoveLegacyIE(InfoExtractor):
 class BrightcoveNewIE(InfoExtractor):
     IE_NAME = 'brightcove:new'
     _VALID_URL = r'https?://players\.brightcove\.net/(?P<account_id>\d+)/(?P<player_id>[^/]+)_(?P<embed>[^/]+)/index\.html\?.*videoId=(?P<video_id>\d+)'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://players.brightcove.net/929656772001/e41d32dc-ec74-459e-a845-6c69f7b724ea_default/index.html?videoId=4463358922001',
         'md5': 'c8100925723840d4b0d243f7025703be',
         'info_dict': {
@@ -364,12 +364,30 @@ class BrightcoveNewIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Meet the man behind Popcorn Time',
             'description': 'md5:eac376a4fe366edc70279bfb681aea16',
+            'duration': 165.768,
             'timestamp': 1441391203,
             'upload_date': '20150904',
-            'duration': 165.768,
             'uploader_id': '929656772001',
+            'formats': 'mincount:22',
+        },
+    }, {
+        # with rtmp streams
+        'url': 'http://players.brightcove.net/4036320279001/5d112ed9-283f-485f-a7f9-33f42e8bc042_default/index.html?videoId=4279049078001',
+        'info_dict': {
+            'id': '4279049078001',
+            'ext': 'mp4',
+            'title': 'Titansgrave: Chapter 0',
+            'description': 'Titansgrave: Chapter 0',
+            'duration': 1242.058,
+            'timestamp': 1433556729,
+            'upload_date': '20150606',
+            'uploader_id': '4036320279001',
+            'formats': 'mincount:41',
+        },
+        'params': {
+            'skip_download': True,
         }
-    }
+    }]
 
     @staticmethod
     def _extract_urls(webpage):

From d8a1caf04f1733d7c94f213af25c587addcea35b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Nov 2015 06:22:12 +0600
Subject: [PATCH 3086/6219] [brightcove:new] Style

---
 youtube_dl/extractor/brightcove.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 9ae724c2b..14ee05f21 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -402,10 +402,11 @@ class BrightcoveNewIE(InfoExtractor):
         for _, url in re.findall(
                 r'<iframe[^>]+src=(["\'])((?:https?:)//players\.brightcove\.net/\d+/[^/]+/index\.html.+?)\1', webpage):
             entries.append(url)
+
         # Look for embed_in_page embeds [2]
-        # According to examples from [3] it's unclear whether video id may be optional
-        # and what to do when it is
         for video_id, account_id, player_id, embed in re.findall(
+                # According to examples from [3] it's unclear whether video id
+                # may be optional and what to do when it is
                 r'''(?sx)
                     <video[^>]+
                         data-video-id=["\'](\d+)["\'][^>]*>.*?
@@ -417,6 +418,7 @@ class BrightcoveNewIE(InfoExtractor):
             entries.append(
                 'http://players.brightcove.net/%s/%s_%s/index.html?videoId=%s'
                 % (account_id, player_id, embed, video_id))
+
         return entries
 
     def _real_extract(self, url):

From a90189c3ad9bab0571121ecb70baea89ccac154e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Nov 2015 07:20:33 +0600
Subject: [PATCH 3087/6219] [instagram] Relax _VALID_URL (Closes #7497)

---
 youtube_dl/extractor/instagram.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/instagram.py b/youtube_dl/extractor/instagram.py
index 3d78f78c4..ddcaf76d1 100644
--- a/youtube_dl/extractor/instagram.py
+++ b/youtube_dl/extractor/instagram.py
@@ -10,7 +10,7 @@ from ..utils import (
 
 
 class InstagramIE(InfoExtractor):
-    _VALID_URL = r'https://instagram\.com/p/(?P<id>[\da-zA-Z]+)'
+    _VALID_URL = r'https://instagram\.com/p/(?P<id>[^/?#&]+)'
     _TEST = {
         'url': 'https://instagram.com/p/aye83DjauH/?foo=bar#abc',
         'md5': '0d2da106a9d2631273e192b372806516',

From 4479600d57d1a48de5a910764e2d18c30d7a4bd9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Nov 2015 07:21:20 +0600
Subject: [PATCH 3088/6219] [instagram] Add test for #7497

---
 youtube_dl/extractor/instagram.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/instagram.py b/youtube_dl/extractor/instagram.py
index ddcaf76d1..fce179000 100644
--- a/youtube_dl/extractor/instagram.py
+++ b/youtube_dl/extractor/instagram.py
@@ -11,7 +11,7 @@ from ..utils import (
 
 class InstagramIE(InfoExtractor):
     _VALID_URL = r'https://instagram\.com/p/(?P<id>[^/?#&]+)'
-    _TEST = {
+    _TESTS = [{
         'url': 'https://instagram.com/p/aye83DjauH/?foo=bar#abc',
         'md5': '0d2da106a9d2631273e192b372806516',
         'info_dict': {
@@ -21,7 +21,10 @@ class InstagramIE(InfoExtractor):
             'title': 'Video by naomipq',
             'description': 'md5:1f17f0ab29bd6fe2bfad705f58de3cb8',
         }
-    }
+    }, {
+        'url': 'https://instagram.com/p/-Cmh1cukG2/',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From 5f1b2aea8051a34dd543ba0fd17e351a8b7bf858 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 14 Nov 2015 17:02:07 +0800
Subject: [PATCH 3089/6219] [twitter:card] Support vine.co embeds (closes
 #7496)

---
 youtube_dl/extractor/twitter.py | 23 ++++++++++++++++++-----
 1 file changed, 18 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 2bd5946ac..afbe8a651 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -52,6 +52,19 @@ class TwitterCardIE(InfoExtractor):
                 'uploader': 'OMG! Ubuntu!',
                 'uploader_id': 'omgubuntu',
             },
+        },
+        {
+            'url': 'https://twitter.com/i/cards/tfw/v1/665289828897005568',
+            'md5': 'ab2745d0b0ce53319a534fccaa986439',
+            'info_dict': {
+                'id': 'iBb2x00UVlv',
+                'ext': 'mp4',
+                'upload_date': '20151113',
+                'uploader_id': '1189339351084113920',
+                'uploader': '@ArsenalTerje',
+                'title': 'Vine by @ArsenalTerje',
+            },
+            'add_ie': ['Vine'],
         }
     ]
 
@@ -71,11 +84,11 @@ class TwitterCardIE(InfoExtractor):
             request.add_header('User-Agent', user_agent)
             webpage = self._download_webpage(request, video_id)
 
-            youtube_url = self._html_search_regex(
-                r'<iframe[^>]+src="((?:https?:)?//www.youtube.com/embed/[^"]+)"',
-                webpage, 'youtube iframe', default=None)
-            if youtube_url:
-                return self.url_result(youtube_url, 'Youtube')
+            iframe_url = self._html_search_regex(
+                r'<iframe[^>]+src="((?:https?:)?//(?:www.youtube.com/embed/[^"]+|(?:www\.)?vine\.co/v/\w+/card))"',
+                webpage, 'video iframe', default=None)
+            if iframe_url:
+                return self.url_result(iframe_url)
 
             config = self._parse_json(self._html_search_regex(
                 r'data-player-config="([^"]+)"', webpage, 'data player config'),

From 31752f76f76c6ca6f9e515b69a7488103575763e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 14 Nov 2015 17:03:26 +0800
Subject: [PATCH 3090/6219] [twitter:card] Add add_ie for the external test

---
 youtube_dl/extractor/twitter.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index afbe8a651..055047340 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -52,6 +52,7 @@ class TwitterCardIE(InfoExtractor):
                 'uploader': 'OMG! Ubuntu!',
                 'uploader_id': 'omgubuntu',
             },
+            'add_ie': ['Youtube'],
         },
         {
             'url': 'https://twitter.com/i/cards/tfw/v1/665289828897005568',

From 9d584da7d0fd207d46e1592b25244cfcad4354ed Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 14 Nov 2015 17:27:00 +0800
Subject: [PATCH 3091/6219] [xfileshare] Correct _VALID_URL

---
 youtube_dl/extractor/xfileshare.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/xfileshare.py b/youtube_dl/extractor/xfileshare.py
index 7610dc627..952515c98 100644
--- a/youtube_dl/extractor/xfileshare.py
+++ b/youtube_dl/extractor/xfileshare.py
@@ -19,7 +19,7 @@ class XFileShareIE(InfoExtractor):
     IE_DESC = 'XFileShare based sites: GorillaVid.in, daclips.in, movpod.in, fastvideo.in, realvid.net, filehoot.com and vidto.me'
     _VALID_URL = r'''(?x)
         https?://(?P<host>(?:www\.)?
-            (?:daclips\.in|gorillavid\.in|movpod\.in|fastvideo\.in|realvid\.net|filehoot\.com|vidto.\me))/
+            (?:daclips\.in|gorillavid\.in|movpod\.in|fastvideo\.in|realvid\.net|filehoot\.com|vidto\.me))/
         (?:embed-)?(?P<id>[0-9a-zA-Z]+)(?:-[0-9]+x[0-9]+\.html)?
     '''
 

From 903d1369428f72397dbc698654dbad445ecaf2ac Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Nov 2015 16:43:58 +0600
Subject: [PATCH 3092/6219] [lynda] Logout only when login info present (Closes
 #7500)

---
 youtube_dl/extractor/lynda.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index 9a207b2cd..7d78a8805 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -83,6 +83,10 @@ class LyndaBaseIE(InfoExtractor):
             raise ExtractorError('Unable to log in')
 
     def _logout(self):
+        username, _ = self._get_login_info()
+        if username is None:
+            return
+
         self._download_webpage(
             'http://www.lynda.com/ajax/logout.aspx', None,
             'Logging out', 'Unable to log out', fatal=False)

From 0d85c3a7327e75173897d0e212254e496a46ea2d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Nov 2015 16:44:24 +0600
Subject: [PATCH 3093/6219] [lynda] Style

---
 youtube_dl/extractor/lynda.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index 7d78a8805..3d7e7e003 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -25,7 +25,7 @@ class LyndaBaseIE(InfoExtractor):
         self._login()
 
     def _login(self):
-        (username, password) = self._get_login_info()
+        username, password = self._get_login_info()
         if username is None:
             return
 

From dcdfeb33d2666ca07d2fb73ebf9284a77e714f69 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Nov 2015 22:32:54 +0600
Subject: [PATCH 3094/6219] [quickscope] Remove extractor

---
 youtube_dl/extractor/periscope.py | 21 ---------------------
 1 file changed, 21 deletions(-)

diff --git a/youtube_dl/extractor/periscope.py b/youtube_dl/extractor/periscope.py
index 887c8020d..f4c1b622a 100644
--- a/youtube_dl/extractor/periscope.py
+++ b/youtube_dl/extractor/periscope.py
@@ -81,24 +81,3 @@ class PeriscopeIE(InfoExtractor):
             'thumbnails': thumbnails,
             'formats': formats,
         }
-
-
-class QuickscopeIE(InfoExtractor):
-    IE_DESC = 'Quick Scope'
-    _VALID_URL = r'https?://watchonperiscope\.com/broadcast/(?P<id>\d+)'
-    _TEST = {
-        'url': 'https://watchonperiscope.com/broadcast/56180087',
-        'only_matching': True,
-    }
-
-    def _real_extract(self, url):
-        broadcast_id = self._match_id(url)
-        request = compat_urllib_request.Request(
-            'https://watchonperiscope.com/api/accessChannel', compat_urllib_parse.urlencode({
-                'broadcast_id': broadcast_id,
-                'entry_ticket': '',
-                'from_push': 'false',
-                'uses_sessions': 'true',
-            }).encode('utf-8'))
-        return self.url_result(
-            self._download_json(request, broadcast_id)['share_url'], 'Periscope')

From 3b3e8ed332e44003f410d9f97967f0773e41ed1d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Nov 2015 22:34:30 +0600
Subject: [PATCH 3095/6219] [quickscope] Remove extractor (2)

---
 youtube_dl/extractor/__init__.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 64ce3210b..eca4b00f5 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -458,10 +458,7 @@ from .orf import (
 from .parliamentliveuk import ParliamentLiveUKIE
 from .patreon import PatreonIE
 from .pbs import PBSIE
-from .periscope import (
-    PeriscopeIE,
-    QuickscopeIE,
-)
+from .periscope import PeriscopeIE
 from .philharmoniedeparis import PhilharmonieDeParisIE
 from .phoenix import PhoenixIE
 from .photobucket import PhotobucketIE

From d781e293168cbdf93ee8d76e33b65387b13a2755 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Nov 2015 23:08:13 +0600
Subject: [PATCH 3096/6219] [bbc] Allow selectionunavailable errors (Closes
 #7502)

---
 youtube_dl/extractor/bbc.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index a55a6dbc9..33b296eaf 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -27,7 +27,7 @@ class BBCCoUkIE(InfoExtractor):
     _MEDIASELECTOR_URLS = [
         # Provides HQ HLS streams with even better quality that pc mediaset but fails
         # with geolocation in some cases when it's even not geo restricted at all (e.g.
-        # http://www.bbc.co.uk/programmes/b06bp7lf)
+        # http://www.bbc.co.uk/programmes/b06bp7lf). Also may fail with selectionunavailable.
         'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/iptv-all/vpid/%s',
         'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/pc/vpid/%s',
     ]
@@ -334,7 +334,7 @@ class BBCCoUkIE(InfoExtractor):
                 return self._download_media_selector_url(
                     mediaselector_url % programme_id, programme_id)
             except BBCCoUkIE.MediaSelectionError as e:
-                if e.id in ('notukerror', 'geolocation'):
+                if e.id in ('notukerror', 'geolocation', 'selectionunavailable'):
                     last_exception = e
                     continue
                 self._raise_extractor_error(e)
@@ -345,7 +345,7 @@ class BBCCoUkIE(InfoExtractor):
             media_selection = self._download_xml(
                 url, programme_id, 'Downloading media selection XML')
         except ExtractorError as ee:
-            if isinstance(ee.cause, compat_HTTPError) and ee.cause.code == 403:
+            if isinstance(ee.cause, compat_HTTPError) and ee.cause.code in (403, 404):
                 media_selection = compat_etree_fromstring(ee.cause.read().decode('utf-8'))
             else:
                 raise

From 0f72beb5153fbd4780a1c6b52e24acf5f0515874 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 14 Nov 2015 18:31:33 +0100
Subject: [PATCH 3097/6219] [periscope] Remove unused imports

---
 youtube_dl/extractor/periscope.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/youtube_dl/extractor/periscope.py b/youtube_dl/extractor/periscope.py
index f4c1b622a..459fa2c43 100644
--- a/youtube_dl/extractor/periscope.py
+++ b/youtube_dl/extractor/periscope.py
@@ -2,10 +2,6 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse,
-    compat_urllib_request,
-)
 from ..utils import parse_iso8601
 
 

From 0c59d02bdc86056957fcbebfec591ad53b7fcda3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 15 Nov 2015 00:20:17 +0600
Subject: [PATCH 3098/6219] [periscope] Relax _VALID_URL (Closes #7503)

---
 youtube_dl/extractor/periscope.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/periscope.py b/youtube_dl/extractor/periscope.py
index 459fa2c43..63cc764bb 100644
--- a/youtube_dl/extractor/periscope.py
+++ b/youtube_dl/extractor/periscope.py
@@ -7,7 +7,7 @@ from ..utils import parse_iso8601
 
 class PeriscopeIE(InfoExtractor):
     IE_DESC = 'Periscope'
-    _VALID_URL = r'https?://(?:www\.)?periscope\.tv/w/(?P<id>[^/?#]+)'
+    _VALID_URL = r'https?://(?:www\.)?periscope\.tv/[^/]+/(?P<id>[^/?#]+)'
     # Alive example URLs can be found here http://onperiscope.com/
     _TESTS = [{
         'url': 'https://www.periscope.tv/w/aJUQnjY3MjA3ODF8NTYxMDIyMDl2zCg2pECBgwTqRpQuQD352EMPTKQjT4uqlM3cgWFA-g==',
@@ -25,6 +25,9 @@ class PeriscopeIE(InfoExtractor):
     }, {
         'url': 'https://www.periscope.tv/w/1ZkKzPbMVggJv',
         'only_matching': True,
+    }, {
+        'url': 'https://www.periscope.tv/bastaakanoggano/1OdKrlkZZjOJX',
+        'only_matching': True,
     }]
 
     def _call_api(self, method, value):

From dc0279532a218f5f43ca461aacd01058b8bc9e6d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 15 Nov 2015 02:21:24 +0600
Subject: [PATCH 3099/6219] [dumpert] Disable SSL (Closes #7504)

---
 youtube_dl/extractor/dumpert.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/dumpert.py b/youtube_dl/extractor/dumpert.py
index 1f00386fe..8811be985 100644
--- a/youtube_dl/extractor/dumpert.py
+++ b/youtube_dl/extractor/dumpert.py
@@ -28,7 +28,7 @@ class DumpertIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        url = 'https://www.dumpert.nl/mediabase/' + video_id
+        url = 'http://www.dumpert.nl/mediabase/' + video_id
         req = compat_urllib_request.Request(url)
         req.add_header('Cookie', 'nsfw=1; cpc=10')
         webpage = self._download_webpage(req, video_id)

From b2f77388305975570d57a4db3ced46aa3856f7a4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 15 Nov 2015 02:25:00 +0600
Subject: [PATCH 3100/6219] [dumpert] Use original protocol

---
 youtube_dl/extractor/dumpert.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/dumpert.py b/youtube_dl/extractor/dumpert.py
index 8811be985..f5a31058d 100644
--- a/youtube_dl/extractor/dumpert.py
+++ b/youtube_dl/extractor/dumpert.py
@@ -2,6 +2,7 @@
 from __future__ import unicode_literals
 
 import base64
+import re
 
 from .common import InfoExtractor
 from ..compat import compat_urllib_request
@@ -9,7 +10,7 @@ from ..utils import qualities
 
 
 class DumpertIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?dumpert\.nl/(?:mediabase|embed)/(?P<id>[0-9]+/[0-9a-zA-Z]+)'
+    _VALID_URL = r'(?P<protocol>https?)://(?:www\.)?dumpert\.nl/(?:mediabase|embed)/(?P<id>[0-9]+/[0-9a-zA-Z]+)'
     _TESTS = [{
         'url': 'http://www.dumpert.nl/mediabase/6646981/951bc60f/',
         'md5': '1b9318d7d5054e7dcb9dc7654f21d643',
@@ -26,9 +27,11 @@ class DumpertIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        protocol = mobj.group('protocol')
 
-        url = 'http://www.dumpert.nl/mediabase/' + video_id
+        url = '%s://www.dumpert.nl/mediabase/%s' % (protocol, video_id)
         req = compat_urllib_request.Request(url)
         req.add_header('Cookie', 'nsfw=1; cpc=10')
         webpage = self._download_webpage(req, video_id)

From 2ff7cbeaaabbc8d67caa224a5cef1fbd32761918 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 15 Nov 2015 08:30:13 +0600
Subject: [PATCH 3101/6219] [nowtv:list] Add extrator (Closes #7147)

---
 youtube_dl/extractor/__init__.py |   5 +-
 youtube_dl/extractor/nowtv.py    | 164 +++++++++++++++++++++----------
 2 files changed, 118 insertions(+), 51 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index eca4b00f5..59c82f65d 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -420,7 +420,10 @@ from .nowness import (
     NownessPlaylistIE,
     NownessSeriesIE,
 )
-from .nowtv import NowTVIE
+from .nowtv import (
+    NowTVIE,
+    NowTVListIE,
+)
 from .nowvideo import NowVideoIE
 from .npo import (
     NPOIE,
diff --git a/youtube_dl/extractor/nowtv.py b/youtube_dl/extractor/nowtv.py
index b0bdffc4e..67e34b294 100644
--- a/youtube_dl/extractor/nowtv.py
+++ b/youtube_dl/extractor/nowtv.py
@@ -1,6 +1,8 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..compat import compat_str
 from ..utils import (
@@ -13,8 +15,63 @@ from ..utils import (
 )
 
 
-class NowTVIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?nowtv\.(?:de|at|ch)/(?:rtl|rtl2|rtlnitro|superrtl|ntv|vox)/(?P<id>.+?)/(?:player|preview)'
+class NowTVBaseIE(InfoExtractor):
+    _VIDEO_FIELDS = (
+        'id', 'title', 'free', 'geoblocked', 'articleLong', 'articleShort',
+        'broadcastStartDate', 'seoUrl', 'duration', 'files',
+        'format.defaultImage169Format', 'format.defaultImage169Logo')
+
+    def _extract_video(self, info, display_id=None):
+        video_id = compat_str(info['id'])
+
+        files = info['files']
+        if not files:
+            if info.get('geoblocked', False):
+                raise ExtractorError(
+                    'Video %s is not available from your location due to geo restriction' % video_id,
+                    expected=True)
+            if not info.get('free', True):
+                raise ExtractorError(
+                    'Video %s is not available for free' % video_id, expected=True)
+
+        formats = []
+        for item in files['items']:
+            if determine_ext(item['path']) != 'f4v':
+                continue
+            app, play_path = remove_start(item['path'], '/').split('/', 1)
+            formats.append({
+                'url': 'rtmpe://fms.rtl.de',
+                'app': app,
+                'play_path': 'mp4:%s' % play_path,
+                'ext': 'flv',
+                'page_url': 'http://rtlnow.rtl.de',
+                'player_url': 'http://cdn.static-fra.de/now/vodplayer.swf',
+                'tbr': int_or_none(item.get('bitrate')),
+            })
+        self._sort_formats(formats)
+
+        title = info['title']
+        description = info.get('articleLong') or info.get('articleShort')
+        timestamp = parse_iso8601(info.get('broadcastStartDate'), ' ')
+        duration = parse_duration(info.get('duration'))
+
+        f = info.get('format', {})
+        thumbnail = f.get('defaultImage169Format') or f.get('defaultImage169Logo')
+
+        return {
+            'id': video_id,
+            'display_id': display_id or info.get('seoUrl'),
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'timestamp': timestamp,
+            'duration': duration,
+            'formats': formats,
+        }
+
+
+class NowTVIE(NowTVBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?nowtv\.(?:de|at|ch)/(?:rtl|rtl2|rtlnitro|superrtl|ntv|vox)/(?P<show_id>[^/]+)/(?:list/[^/]+/)?(?P<id>[^/]+)/(?:player|preview)'
 
     _TESTS = [{
         # rtl
@@ -23,7 +80,7 @@ class NowTVIE(InfoExtractor):
             'id': '203519',
             'display_id': 'bauer-sucht-frau/die-neuen-bauern-und-eine-hochzeit',
             'ext': 'flv',
-            'title': 'Die neuen Bauern und eine Hochzeit',
+            'title': 'Inka Bause stellt die neuen Bauern vor',
             'description': 'md5:e234e1ed6d63cf06be5c070442612e7e',
             'thumbnail': 're:^https?://.*\.jpg$',
             'timestamp': 1432580700,
@@ -136,58 +193,65 @@ class NowTVIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        display_id = self._match_id(url)
-        display_id_split = display_id.split('/')
-        if len(display_id) > 2:
-            display_id = '/'.join((display_id_split[0], display_id_split[-1]))
+        mobj = re.match(self._VALID_URL, url)
+        display_id = '%s/%s' % (mobj.group('show_id'), mobj.group('id'))
 
         info = self._download_json(
-            'https://api.nowtv.de/v3/movies/%s?fields=id,title,free,geoblocked,articleLong,articleShort,broadcastStartDate,seoUrl,duration,format,files' % display_id,
-            display_id)
+            'https://api.nowtv.de/v3/movies/%s?fields=%s'
+            % (display_id, ','.join(self._VIDEO_FIELDS)), display_id)
 
-        video_id = compat_str(info['id'])
+        return self._extract_video(info, display_id)
 
-        files = info['files']
-        if not files:
-            if info.get('geoblocked', False):
-                raise ExtractorError(
-                    'Video %s is not available from your location due to geo restriction' % video_id,
-                    expected=True)
-            if not info.get('free', True):
-                raise ExtractorError(
-                    'Video %s is not available for free' % video_id, expected=True)
 
-        formats = []
-        for item in files['items']:
-            if determine_ext(item['path']) != 'f4v':
-                continue
-            app, play_path = remove_start(item['path'], '/').split('/', 1)
-            formats.append({
-                'url': 'rtmpe://fms.rtl.de',
-                'app': app,
-                'play_path': 'mp4:%s' % play_path,
-                'ext': 'flv',
-                'page_url': 'http://rtlnow.rtl.de',
-                'player_url': 'http://cdn.static-fra.de/now/vodplayer.swf',
-                'tbr': int_or_none(item.get('bitrate')),
-            })
-        self._sort_formats(formats)
+class NowTVListIE(NowTVBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?nowtv\.(?:de|at|ch)/(?:rtl|rtl2|rtlnitro|superrtl|ntv|vox)/(?P<show_id>[^/]+)/list/(?P<id>[^?/#&]+)$'
 
-        title = info['title']
-        description = info.get('articleLong') or info.get('articleShort')
-        timestamp = parse_iso8601(info.get('broadcastStartDate'), ' ')
-        duration = parse_duration(info.get('duration'))
+    _SHOW_FIELDS = ('title', )
+    _SEASON_FIELDS = ('id', 'headline', 'seoheadline', )
 
-        f = info.get('format', {})
-        thumbnail = f.get('defaultImage169Format') or f.get('defaultImage169Logo')
+    _TESTS = [{
+        'url': 'http://www.nowtv.at/rtl/stern-tv/list/aktuell',
+        'info_dict': {
+            'id': '17006',
+            'title': 'stern TV - Aktuell',
+        },
+        'playlist_count': 1,
+    }, {
+        'url': 'http://www.nowtv.at/rtl/das-supertalent/list/free-staffel-8',
+        'info_dict': {
+            'id': '20716',
+            'title': 'Das Supertalent - FREE Staffel 8',
+        },
+        'playlist_count': 14,
+    }]
 
-        return {
-            'id': video_id,
-            'display_id': display_id,
-            'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
-            'timestamp': timestamp,
-            'duration': duration,
-            'formats': formats,
-        }
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        show_id = mobj.group('show_id')
+        season_id = mobj.group('id')
+
+        fields = []
+        fields.extend(self._SHOW_FIELDS)
+        fields.extend('formatTabs.%s' % field for field in self._SEASON_FIELDS)
+        fields.extend(
+            'formatTabs.formatTabPages.container.movies.%s' % field
+            for field in self._VIDEO_FIELDS)
+
+        list_info = self._download_json(
+            'https://api.nowtv.de/v3/formats/seo?fields=%s&name=%s.php'
+            % (','.join(fields), show_id),
+            season_id)
+
+        season = next(
+            season for season in list_info['formatTabs']['items']
+            if season.get('seoheadline') == season_id)
+
+        title = '%s - %s' % (list_info['title'], season['headline'])
+
+        entries = []
+        for container in season['formatTabPages']['items']:
+            for info in ((container.get('container') or {}).get('movies') or {}).get('items') or []:
+                entries.append(self._extract_video(info))
+
+        return self.playlist_result(
+            entries, compat_str(season.get('id') or season_id), title)

From 828b2a5cd971ae04469aeb7b11dfeaab7962c4d9 Mon Sep 17 00:00:00 2001
From: David Ben Zakai <david.benzakai@gmail.com>
Date: Sun, 15 Nov 2015 09:40:32 +0200
Subject: [PATCH 3102/6219] Removing an unnecessary import

---
 youtube_dl/update.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/youtube_dl/update.py b/youtube_dl/update.py
index 04bf0939e..074eb64a7 100644
--- a/youtube_dl/update.py
+++ b/youtube_dl/update.py
@@ -9,10 +9,7 @@ import subprocess
 import sys
 from zipimport import zipimporter
 
-from .compat import (
-    compat_str,
-    compat_urllib_request,
-)
+from .compat import compat_str
 
 from .version import __version__
 

From ad1f4e7902464532ba11f3a09ac6ff2312b01661 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 15 Nov 2015 23:43:23 +0600
Subject: [PATCH 3103/6219] [theplatform] Handle explicitly specified SMIL
 (#7385)

---
 youtube_dl/extractor/theplatform.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index 25edc3100..114c70b7c 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -193,6 +193,15 @@ class ThePlatformIE(ThePlatformBaseIE):
 
         if smuggled_data.get('force_smil_url', False):
             smil_url = url
+        # Explicitly specified SMIL (see https://github.com/rg3/youtube-dl/issues/7385)
+        elif '/guid/' in url:
+            webpage = self._download_webpage(url, video_id)
+            smil_url = self._search_regex(
+                r'<link[^>]+href=(["\'])(?P<url>.+?)\1[^>]+type=["\']application/smil\+xml',
+                webpage, 'smil url', group='url')
+            path = self._search_regex(
+                r'link\.theplatform\.com/s/((?:[^/?#&]+/)+[^/?#&]+)', smil_url, 'path')
+            smil_url += '?' if '?' not in smil_url else '&' + 'formats=m3u,mpeg4&format=SMIL'
         elif mobj.group('config'):
             config_url = url + '&form=json'
             config_url = config_url.replace('swf/', 'config/')

From 9a4acbfaf58f7334bdd8a3b66dadf331f1ce1d7b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 16 Nov 2015 00:28:04 +0600
Subject: [PATCH 3104/6219] [theplatform] Add test for #7385

---
 youtube_dl/extractor/theplatform.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index 114c70b7c..43315e75d 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -139,6 +139,11 @@ class ThePlatformIE(ThePlatformBaseIE):
             'upload_date': '20150701',
             'categories': ['Today/Shows/Orange Room', 'Today/Sections/Money', 'Today/Topics/Tech', "Today/Topics/Editor's picks"],
         },
+    }, {
+        # From http://www.nbc.com/the-blacklist/video/sir-crispin-crandall/2928790?onid=137781#vc137781=1
+        # geo-restricted (US), HLS encrypted with AES-128
+        'url': 'http://player.theplatform.com/p/NnzsPC/onsite_universal/select/media/guid/2410887629/2928790?fwsitesection=nbc_the_blacklist_video_library&autoPlay=true&carouselID=137781',
+        'only_matching': True,
     }]
 
     @staticmethod

From bd1512d19649c280197729814766d590ea6c023b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 15 Nov 2015 22:16:08 +0100
Subject: [PATCH 3105/6219] release 2015.11.15

---
 docs/supportedsites.md | 5 +++--
 youtube_dl/version.py  | 2 +-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 5016ba4bc..2e5283747 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -67,7 +67,8 @@
  - **Bpb**: Bundeszentrale für politische Bildung
  - **BR**: Bayerischer Rundfunk Mediathek
  - **Break**
- - **Brightcove**
+ - **brightcove:legacy**
+ - **brightcove:new**
  - **bt:article**: Bergens Tidende Articles
  - **bt:vestlendingen**: Bergens Tidende - Vestlendingen
  - **BuzzFeed**
@@ -366,6 +367,7 @@
  - **nowness:playlist**
  - **nowness:series**
  - **NowTV**
+ - **NowTVList**
  - **nowvideo**: NowVideo
  - **npo**: npo.nl and ntr.nl
  - **npo.nl:live**
@@ -425,7 +427,6 @@
  - **qqmusic:playlist**: QQ音乐 - 歌单
  - **qqmusic:singer**: QQ音乐 - 歌手
  - **qqmusic:toplist**: QQ音乐 - 排行榜
- - **Quickscope**: Quick Scope
  - **QuickVid**
  - **R7**
  - **radio.de**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 6585d60d5..6f601cbb1 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.11.13'
+__version__ = '2015.11.15'

From 76adc820682b78b67ffc9cb1544807f7e021e73b Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Mon, 16 Nov 2015 11:39:18 +0800
Subject: [PATCH 3106/6219] [neteasemusic] Fixes #7301

---
 youtube_dl/extractor/neteasemusic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/neteasemusic.py b/youtube_dl/extractor/neteasemusic.py
index a8e0a64ed..bb3362069 100644
--- a/youtube_dl/extractor/neteasemusic.py
+++ b/youtube_dl/extractor/neteasemusic.py
@@ -40,7 +40,7 @@ class NetEaseMusicBaseIE(InfoExtractor):
             if not details:
                 continue
             formats.append({
-                'url': 'http://m1.music.126.net/%s/%s.%s' %
+                'url': 'http://m5.music.126.net/%s/%s.%s' %
                        (cls._encrypt(details['dfsId']), details['dfsId'],
                         details['extension']),
                 'ext': details.get('extension'),

From 741dd8ea65b276997f3eadae43b2879e9a229a80 Mon Sep 17 00:00:00 2001
From: Rastislav Barlik <barlik@zoho.com>
Date: Mon, 16 Nov 2015 14:15:25 +0000
Subject: [PATCH 3107/6219] Clarify that automatic subtitles are generated.

It wasn't clear what automatic word mean.
---
 README.md               | 4 ++--
 youtube_dl/YoutubeDL.py | 2 +-
 youtube_dl/options.py   | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index 38db97c59..b286651cd 100644
--- a/README.md
+++ b/README.md
@@ -329,8 +329,8 @@ which means you can modify it, redistribute it or use it however you like.
 
 ## Subtitle Options:
     --write-sub                      Write subtitle file
-    --write-auto-sub                 Write automatic subtitle file (YouTube
-                                     only)
+    --write-auto-sub                 Write automatically generated subtitle file
+                                     (YouTube only)
     --all-subs                       Download all the available subtitles of the
                                      video
     --list-subs                      List all available subtitles for the video
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 1783ce01b..422f3ffca 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -156,7 +156,7 @@ class YoutubeDL(object):
     writethumbnail:    Write the thumbnail image to a file
     write_all_thumbnails:  Write all thumbnail formats to files
     writesubtitles:    Write the video subtitles to a file
-    writeautomaticsub: Write the automatic subtitles to a file
+    writeautomaticsub: Write the automatically generated subtitles to a file
     allsubtitles:      Downloads all the subtitles of the video
                        (requires writesubtitles or writeautomaticsub)
     listsubtitles:     Lists all available subtitles for the video
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 3dd6d290b..079fe7e8a 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -363,7 +363,7 @@ def parseOpts(overrideArguments=None):
     subtitles.add_option(
         '--write-auto-sub', '--write-automatic-sub',
         action='store_true', dest='writeautomaticsub', default=False,
-        help='Write automatic subtitle file (YouTube only)')
+        help='Write automatically generated subtitle file (YouTube only)')
     subtitles.add_option(
         '--all-subs',
         action='store_true', dest='allsubtitles', default=False,

From 7aefc49c4013efb5056b2c1237e22c52cb5d3c49 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 16 Nov 2015 20:20:16 +0600
Subject: [PATCH 3108/6219] [utils] Skip invalid/non HTML entities (Closes
 #7518)

---
 test/test_utils.py  | 4 ++--
 youtube_dl/utils.py | 6 +++++-
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 01829f71e..ea1ff0547 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -210,8 +210,8 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(unescapeHTML('%20;'), '%20;')
         self.assertEqual(unescapeHTML('&#x2F;'), '/')
         self.assertEqual(unescapeHTML('&#47;'), '/')
-        self.assertEqual(
-            unescapeHTML('&eacute;'), 'é')
+        self.assertEqual(unescapeHTML('&eacute;'), 'é')
+        self.assertEqual(unescapeHTML('&#2013266066;'), '&#2013266066;')
 
     def test_daterange(self):
         _20century = DateRange("19000101", "20000101")
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index d39f313a4..b7013a6aa 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -396,7 +396,11 @@ def _htmlentity_transform(entity):
             numstr = '0%s' % numstr
         else:
             base = 10
-        return compat_chr(int(numstr, base))
+        # See https://github.com/rg3/youtube-dl/issues/7518
+        try:
+            return compat_chr(int(numstr, base))
+        except ValueError:
+            pass
 
     # Unknown entity in name, return its literal representation
     return ('&%s;' % entity)

From 7a3f0c00ad138eef396ae8fd1583fe29b4c4c684 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 16 Nov 2015 20:24:09 +0600
Subject: [PATCH 3109/6219] [utils] Style

---
 youtube_dl/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index b7013a6aa..d00b14b86 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -403,7 +403,7 @@ def _htmlentity_transform(entity):
             pass
 
     # Unknown entity in name, return its literal representation
-    return ('&%s;' % entity)
+    return '&%s;' % entity
 
 
 def unescapeHTML(s):

From 9b464929fe62d405daccf866cdfdfa6257208839 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 17 Nov 2015 21:11:42 +0600
Subject: [PATCH 3110/6219] [rtve.es:alacarta] Fix extraction

---
 youtube_dl/extractor/rtve.py | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/rtve.py b/youtube_dl/extractor/rtve.py
index 5b97d33ca..81fcfedc7 100644
--- a/youtube_dl/extractor/rtve.py
+++ b/youtube_dl/extractor/rtve.py
@@ -107,15 +107,9 @@ class RTVEALaCartaIE(InfoExtractor):
         png = self._download_webpage(png_request, video_id, 'Downloading url information')
         video_url = _decrypt_url(png)
         if not video_url.endswith('.f4m'):
-            auth_url = video_url.replace(
+            video_url = video_url.replace(
                 'resources/', 'auth/resources/'
             ).replace('.net.rtve', '.multimedia.cdn.rtve')
-            video_path = self._download_webpage(
-                auth_url, video_id, 'Getting video url')
-            # Use mvod1.akcdn instead of flash.akamaihd.multimedia.cdn to get
-            # the right Content-Length header and the mp4 format
-            video_url = compat_urlparse.urljoin(
-                'http://mvod1.akcdn.rtve.es/', video_path)
 
         subtitles = None
         if info.get('sbtFile') is not None:

From e156e70281332b14c3aa47a71fd4a7a16be1a225 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 17 Nov 2015 16:23:29 +0100
Subject: [PATCH 3111/6219] [rtve] Remove unused import

---
 youtube_dl/extractor/rtve.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/rtve.py b/youtube_dl/extractor/rtve.py
index 81fcfedc7..0fe6356db 100644
--- a/youtube_dl/extractor/rtve.py
+++ b/youtube_dl/extractor/rtve.py
@@ -6,7 +6,7 @@ import re
 import time
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_request, compat_urlparse
+from ..compat import compat_urllib_request
 from ..utils import (
     ExtractorError,
     float_or_none,

From 4cd759f73d734882b6d74e764abb068e64a99635 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 17 Nov 2015 17:52:29 +0100
Subject: [PATCH 3112/6219] [dplay] Add extractor (closes #7515)

Since I haven't figured out how to download the hds stream, we use the hls one instead.
---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/dplay.py    | 50 ++++++++++++++++++++++++++++++++
 2 files changed, 51 insertions(+)
 create mode 100644 youtube_dl/extractor/dplay.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 59c82f65d..26e5745d6 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -132,6 +132,7 @@ from .dfb import DFBIE
 from .dhm import DHMIE
 from .dotsub import DotsubIE
 from .douyutv import DouyuTVIE
+from .dplay import DPlayIE
 from .dramafever import (
     DramaFeverIE,
     DramaFeverSeriesIE,
diff --git a/youtube_dl/extractor/dplay.py b/youtube_dl/extractor/dplay.py
new file mode 100644
index 000000000..93768c294
--- /dev/null
+++ b/youtube_dl/extractor/dplay.py
@@ -0,0 +1,50 @@
+from __future__ import unicode_literals
+
+import time
+
+from .common import InfoExtractor
+from ..utils import int_or_none
+
+
+class DPlayIE(InfoExtractor):
+    _VALID_URL = r'http://www\.dplay\.se/[^/]+/(?P<id>[^/?#]+)'
+
+    _TEST = {
+        'url': 'http://www.dplay.se/nugammalt-77-handelser-som-format-sverige/season-1-svensken-lar-sig-njuta-av-livet/',
+        'info_dict': {
+            'id': '3172',
+            'ext': 'mp4',
+            'display_id': 'season-1-svensken-lar-sig-njuta-av-livet',
+            'title': 'Svensken lär sig njuta av livet',
+            'duration': 2650,
+        },
+    }
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+        video_id = self._search_regex(
+            r'data-video-id="(\d+)"', webpage, 'video id')
+
+        info = self._download_json(
+            'http://www.dplay.se/api/v2/ajax/videos?video_id=' + video_id,
+            video_id)['data'][0]
+
+        self._set_cookie(
+            'secure.dplay.se', 'dsc-geo',
+            '{"countryCode":"NL","expiry":%d}' % ((time.time() + 20 * 60) * 1000))
+        # TODO: consider adding support for 'stream_type=hds', it seems to
+        # require setting some cookies
+        manifest_url = self._download_json(
+            'https://secure.dplay.se/secure/api/v2/user/authorization/stream/%s?stream_type=hls' % video_id,
+            video_id, 'Getting manifest url for hls stream')['hls']
+        formats = self._extract_m3u8_formats(
+            manifest_url, video_id, ext='mp4', entry_protocol='m3u8_native')
+
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            'title': info['title'],
+            'formats': formats,
+            'duration': int_or_none(info.get('video_metadata_length'), scale=1000),
+        }

From 609af1ae1ce578b0fdd1dd7ec6e56f1272e107f6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 17 Nov 2015 17:57:40 +0100
Subject: [PATCH 3113/6219] [dplay] Add 'encoding: utf-8' line

---
 youtube_dl/extractor/dplay.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/dplay.py b/youtube_dl/extractor/dplay.py
index 93768c294..6cda56a7f 100644
--- a/youtube_dl/extractor/dplay.py
+++ b/youtube_dl/extractor/dplay.py
@@ -1,3 +1,4 @@
+# encoding: utf-8
 from __future__ import unicode_literals
 
 import time

From 312a3f389b4c788556cfcceec1556580c9a9520f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 18 Nov 2015 00:46:41 +0600
Subject: [PATCH 3114/6219] [pbs] Extend _VALID_URL

---
 youtube_dl/extractor/pbs.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 8fb9b1849..33d5c1cf7 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -22,7 +22,7 @@ class PBSIE(InfoExtractor):
            # Article with embedded player (or direct video)
            (?:www\.)?pbs\.org/(?:[^/]+/){2,5}(?P<presumptive_id>[^/]+?)(?:\.html)?/?(?:$|[?\#]) |
            # Player
-           video\.pbs\.org/(?:widget/)?partnerplayer/(?P<player_id>[^/]+)/
+           (?:video|player)\.pbs\.org/(?:widget/)?partnerplayer/(?P<player_id>[^/]+)/
         )
     '''
 
@@ -170,6 +170,10 @@ class PBSIE(InfoExtractor):
             'params': {
                 'skip_download': True,  # requires ffmpeg
             },
+        },
+        {
+            'url': 'http://player.pbs.org/widget/partnerplayer/2365297708/?start=0&end=0&chapterbar=false&endscreen=false&topbar=true',
+            'only_matching': True,
         }
     ]
     _ERRORS = {

From 63b4295d20a5b98a9fca4dc3ce132b26408110d6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 18 Nov 2015 18:28:05 +0100
Subject: [PATCH 3115/6219] [youtube:playlist] fix title extraction (fixes
 #7544 and #7545)

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 687e0b4db..364ca102a 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1615,7 +1615,7 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor, YoutubePlaylistBaseInfoExtract
                 self.report_warning('Youtube gives an alert message: ' + match)
 
         playlist_title = self._html_search_regex(
-            r'(?s)<h1 class="pl-header-title[^"]*">\s*(.*?)\s*</h1>',
+            r'(?s)<h1 class="pl-header-title[^"]*"[^>]*>\s*(.*?)\s*</h1>',
             page, 'title')
 
         return self.playlist_result(self._entries(page, playlist_id), playlist_id, playlist_title)

From 82beaabb41f015b4aaa1b7460a881a9653e398e9 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 18 Nov 2015 19:23:04 +0100
Subject: [PATCH 3116/6219] release 2015.11.18

---
 docs/supportedsites.md | 1 +
 youtube_dl/version.py  | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 2e5283747..f56a6986b 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -129,6 +129,7 @@
  - **Discovery**
  - **Dotsub**
  - **DouyuTV**: 斗鱼
+ - **DPlay**
  - **dramafever**
  - **dramafever:series**
  - **DRBonanza**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 6f601cbb1..b394cb880 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.11.15'
+__version__ = '2015.11.18'

From a9c09a7c62df3024b0774de7204e9262bb2d7d4a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 19 Nov 2015 20:25:28 +0600
Subject: [PATCH 3117/6219] [pbs] Update API URL (Closes #7565)

---
 youtube_dl/extractor/pbs.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 33d5c1cf7..b787e2a73 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -263,7 +263,7 @@ class PBSIE(InfoExtractor):
             return self.playlist_result(entries, display_id)
 
         info = self._download_json(
-            'http://video.pbs.org/videoInfo/%s?format=json&type=partner' % video_id,
+            'http://player.pbs.org/videoInfo/%s?format=json&type=partner' % video_id,
             display_id)
 
         formats = []

From c39fd7b1ca3e81901aa28d36523fe698295e58d5 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 19 Nov 2015 22:22:57 +0800
Subject: [PATCH 3118/6219] [UDNEmbed] Fix generic UDN pages

Closes #7547
---
 youtube_dl/extractor/generic.py | 2 +-
 youtube_dl/extractor/udn.py     | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 51516a38a..b483eba65 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1739,7 +1739,7 @@ class GenericIE(InfoExtractor):
 
         # Look for UDN embeds
         mobj = re.search(
-            r'<iframe[^>]+src="(?P<url>%s)"' % UDNEmbedIE._VALID_URL, webpage)
+            r'<iframe[^>]+src="(?P<url>%s)"' % UDNEmbedIE._PROTOCOL_RELATIVE_VALID_URL, webpage)
         if mobj is not None:
             return self.url_result(
                 compat_urlparse.urljoin(url, mobj.group('url')), 'UDNEmbed')
diff --git a/youtube_dl/extractor/udn.py b/youtube_dl/extractor/udn.py
index 2151f8338..ee35b7227 100644
--- a/youtube_dl/extractor/udn.py
+++ b/youtube_dl/extractor/udn.py
@@ -12,7 +12,8 @@ from ..compat import compat_urlparse
 
 class UDNEmbedIE(InfoExtractor):
     IE_DESC = '聯合影音'
-    _VALID_URL = r'https?://video\.udn\.com/(?:embed|play)/news/(?P<id>\d+)'
+    _PROTOCOL_RELATIVE_VALID_URL = r'//video\.udn\.com/(?:embed|play)/news/(?P<id>\d+)'
+    _VALID_URL = r'https?:' + _PROTOCOL_RELATIVE_VALID_URL
     _TESTS = [{
         'url': 'http://video.udn.com/embed/news/300040',
         'md5': 'de06b4c90b042c128395a88f0384817e',

From e8110b81254075462fd39590dd05d7cd0214cf22 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 19 Nov 2015 15:35:13 +0100
Subject: [PATCH 3119/6219] release 2015.11.19

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index b394cb880..b3b1b90ba 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.11.18'
+__version__ = '2015.11.19'

From 2c94198eb62a9ce46a09badfdb232f8b5243a893 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 19 Nov 2015 21:29:32 +0600
Subject: [PATCH 3120/6219] [vimeo] Improve playlists extraction

---
 youtube_dl/extractor/vimeo.py | 19 +++++++++----------
 1 file changed, 9 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index b72341a2b..cb8c0d484 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -486,8 +486,7 @@ class VimeoChannelIE(VimeoBaseInfoExtractor):
             password_request, list_id,
             'Verifying the password', 'Wrong password')
 
-    def _extract_videos(self, list_id, base_url):
-        video_ids = []
+    def _title_and_entries(self, list_id, base_url):
         for pagenum in itertools.count(1):
             page_url = self._page_url(base_url, pagenum)
             webpage = self._download_webpage(
@@ -496,18 +495,18 @@ class VimeoChannelIE(VimeoBaseInfoExtractor):
 
             if pagenum == 1:
                 webpage = self._login_list_password(page_url, list_id, webpage)
+                yield self._extract_list_title(webpage)
+
+            for video_id in re.findall(r'id="clip_(\d+?)"', webpage):
+                yield self.url_result('https://vimeo.com/%s' % video_id, 'Vimeo')
 
-            video_ids.extend(re.findall(r'id="clip_(\d+?)"', webpage))
             if re.search(self._MORE_PAGES_INDICATOR, webpage, re.DOTALL) is None:
                 break
 
-        entries = [self.url_result('https://vimeo.com/%s' % video_id, 'Vimeo')
-                   for video_id in video_ids]
-        return {'_type': 'playlist',
-                'id': list_id,
-                'title': self._extract_list_title(webpage),
-                'entries': entries,
-                }
+    def _extract_videos(self, list_id, base_url):
+        title_and_entries = self._title_and_entries(list_id, base_url)
+        list_title = next(title_and_entries)
+        return self.playlist_result(title_and_entries, list_id, list_title)
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)

From fdb20a27a335dddeebd2e6535a1b9c12c08aa42f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 19 Nov 2015 22:30:58 +0600
Subject: [PATCH 3121/6219] [vimeo:group] Improve _VALID_URL (Closes #7552)

---
 youtube_dl/extractor/vimeo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index cb8c0d484..836a7312b 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -567,7 +567,7 @@ class VimeoAlbumIE(VimeoChannelIE):
 
 class VimeoGroupsIE(VimeoAlbumIE):
     IE_NAME = 'vimeo:group'
-    _VALID_URL = r'https://vimeo\.com/groups/(?P<name>[^/]+)'
+    _VALID_URL = r'https://vimeo\.com/groups/(?P<name>[^/]+)(?:/(?!videos?/\d+)|$)'
     _TESTS = [{
         'url': 'https://vimeo.com/groups/rolexawards',
         'info_dict': {

From 6b7ceee1b9124521d2f583fb0d0717999747fdcf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 19 Nov 2015 22:31:16 +0600
Subject: [PATCH 3122/6219] [vimeo] Add test for #7552

---
 youtube_dl/extractor/vimeo.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 836a7312b..057c72f39 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -189,6 +189,10 @@ class VimeoIE(VimeoBaseInfoExtractor):
             'note': 'Video not completely processed, "failed" seed status',
             'only_matching': True,
         },
+        {
+            'url': 'https://vimeo.com/groups/travelhd/videos/22439234',
+            'only_matching': True,
+        },
     ]
 
     @staticmethod

From 371c3b796cb4168dbb2cb1ad48e3cf12745a8601 Mon Sep 17 00:00:00 2001
From: hedii <hedi.chaibs@gmail.com>
Date: Tue, 17 Nov 2015 16:41:59 +0100
Subject: [PATCH 3123/6219] [YoutubeDL] Add playlist finished downloading
 message (Closes #7517)

Conflicts:
	youtube_dl/YoutubeDL.py
---
 youtube_dl/YoutubeDL.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 1783ce01b..9d626049a 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -833,6 +833,7 @@ class YoutubeDL(object):
                                                       extra_info=extra)
                 playlist_results.append(entry_result)
             ie_result['entries'] = playlist_results
+            self.to_screen('[download] Finished downloading playlist: %s' % playlist)
             return ie_result
         elif result_type == 'compat_list':
             self.report_warning(

From 342609a1b452c8b8a02e86c3de2ec2a55b6e809b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 19 Nov 2015 22:55:06 +0600
Subject: [PATCH 3124/6219] [bloomberg] Reax _VALID_URL (Closes #7546)

---
 youtube_dl/extractor/bloomberg.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/bloomberg.py b/youtube_dl/extractor/bloomberg.py
index 0dca29b71..11ace91dd 100644
--- a/youtube_dl/extractor/bloomberg.py
+++ b/youtube_dl/extractor/bloomberg.py
@@ -6,9 +6,9 @@ from .common import InfoExtractor
 
 
 class BloombergIE(InfoExtractor):
-    _VALID_URL = r'https?://www\.bloomberg\.com/news/videos/[^/]+/(?P<id>[^/?#]+)'
+    _VALID_URL = r'https?://www\.bloomberg\.com/news/[^/]+/[^/]+/(?P<id>[^/?#]+)'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.bloomberg.com/news/videos/b/aaeae121-5949-481e-a1ce-4562db6f5df2',
         # The md5 checksum changes
         'info_dict': {
@@ -17,7 +17,10 @@ class BloombergIE(InfoExtractor):
             'title': 'Shah\'s Presentation on Foreign-Exchange Strategies',
             'description': 'md5:a8ba0302912d03d246979735c17d2761',
         },
-    }
+    }, {
+        'url': 'http://www.bloomberg.com/news/articles/2015-11-12/five-strange-things-that-have-been-happening-in-financial-markets',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         name = self._match_id(url)

From ee5cd8418e7d6503bfa33ef44e2eb863a55a8c8c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 19 Nov 2015 22:58:29 +0600
Subject: [PATCH 3125/6219] [theplatform] Handle protocolless feed URLs (Closes
 #7532)

---
 youtube_dl/extractor/theplatform.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index 43315e75d..d52f9ae4b 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -187,7 +187,8 @@ class ThePlatformIE(ThePlatformBaseIE):
             # Seems there's no pattern for the interested script filename, so
             # I try one by one
             for script in reversed(scripts):
-                feed_script = self._download_webpage(script, video_id, 'Downloading feed script')
+                feed_script = self._download_webpage(
+                    self._proto_relative_url(script, 'http:'), video_id, 'Downloading feed script')
                 feed_id = self._search_regex(r'defaultFeedId\s*:\s*"([^"]+)"', feed_script, 'default feed id', default=None)
                 if feed_id is not None:
                     break

From 325bb615a79928541aabfbab366ef813b4f72309 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 19 Nov 2015 22:58:43 +0600
Subject: [PATCH 3126/6219] [theplatform] Style

---
 youtube_dl/extractor/theplatform.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index d52f9ae4b..1555aa77c 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -188,8 +188,11 @@ class ThePlatformIE(ThePlatformBaseIE):
             # I try one by one
             for script in reversed(scripts):
                 feed_script = self._download_webpage(
-                    self._proto_relative_url(script, 'http:'), video_id, 'Downloading feed script')
-                feed_id = self._search_regex(r'defaultFeedId\s*:\s*"([^"]+)"', feed_script, 'default feed id', default=None)
+                    self._proto_relative_url(script, 'http:'),
+                    video_id, 'Downloading feed script')
+                feed_id = self._search_regex(
+                    r'defaultFeedId\s*:\s*"([^"]+)"', feed_script,
+                    'default feed id', default=None)
                 if feed_id is not None:
                     break
             if feed_id is None:

From 67446fd49b447531720d62aa339fc364a1699e2b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 20 Nov 2015 04:07:39 +0600
Subject: [PATCH 3127/6219] [instagram] Improve _VALID_URL (Closes #7568)

---
 youtube_dl/extractor/instagram.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/instagram.py b/youtube_dl/extractor/instagram.py
index fce179000..c158f2064 100644
--- a/youtube_dl/extractor/instagram.py
+++ b/youtube_dl/extractor/instagram.py
@@ -10,7 +10,7 @@ from ..utils import (
 
 
 class InstagramIE(InfoExtractor):
-    _VALID_URL = r'https://instagram\.com/p/(?P<id>[^/?#&]+)'
+    _VALID_URL = r'https?://(?:www\.)?instagram\.com/p/(?P<id>[^/?#&]+)'
     _TESTS = [{
         'url': 'https://instagram.com/p/aye83DjauH/?foo=bar#abc',
         'md5': '0d2da106a9d2631273e192b372806516',

From 17cc1534359aad2c180cfd8a4f66d9d0eccb9e01 Mon Sep 17 00:00:00 2001
From: ashutosh-mishra <ashutosh.mishra@oneconvergence.com>
Date: Fri, 20 Nov 2015 22:51:46 +0530
Subject: [PATCH 3128/6219] Typo fix, found while going through the code.

---
 youtube_dl/YoutubeDL.py         | 2 +-
 youtube_dl/downloader/common.py | 2 +-
 youtube_dl/downloader/rtmp.py   | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 2e824117a..fba99af8d 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -938,7 +938,7 @@ class YoutubeDL(object):
                     filter_parts.append(string)
 
         def _remove_unused_ops(tokens):
-            # Remove operators that we don't use and join them with the sourrounding strings
+            # Remove operators that we don't use and join them with the surrounding strings
             # for example: 'mp4' '-' 'baseline' '-' '16x9' is converted to 'mp4-baseline-16x9'
             ALLOWED_OPS = ('/', '+', ',', '(', ')')
             last_string, last_start, last_end, last_line = None, None, None, None
diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index 29a4500d3..b8bf8daf8 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -42,7 +42,7 @@ class FileDownloader(object):
     min_filesize:       Skip files smaller than this size
     max_filesize:       Skip files larger than this size
     xattr_set_filesize: Set ytdl.filesize user xattribute with expected size.
-                        (experimenatal)
+                        (experimental)
     external_downloader_args:  A list of additional command-line arguments for the
                         external downloader.
 
diff --git a/youtube_dl/downloader/rtmp.py b/youtube_dl/downloader/rtmp.py
index f1d219ba9..14d56db47 100644
--- a/youtube_dl/downloader/rtmp.py
+++ b/youtube_dl/downloader/rtmp.py
@@ -117,7 +117,7 @@ class RtmpFD(FileDownloader):
             return False
 
         # Download using rtmpdump. rtmpdump returns exit code 2 when
-        # the connection was interrumpted and resuming appears to be
+        # the connection was interrupted and resuming appears to be
         # possible. This is part of rtmpdump's normal usage, AFAIK.
         basic_args = [
             'rtmpdump', '--verbose', '-r', url,

From 01b06aedcf9692e03a80efe7e9857560a2a6eb45 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Nov 2015 01:34:02 +0600
Subject: [PATCH 3129/6219] [kaltura] Add support for referrer protected videos
 (#7409)

---
 youtube_dl/extractor/kaltura.py | 47 ++++++++++++++++++++++++---------
 1 file changed, 34 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/kaltura.py b/youtube_dl/extractor/kaltura.py
index 0dcd6cd05..39e038659 100644
--- a/youtube_dl/extractor/kaltura.py
+++ b/youtube_dl/extractor/kaltura.py
@@ -2,12 +2,17 @@
 from __future__ import unicode_literals
 
 import re
+import base64
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse
+from ..compat import (
+    compat_urllib_parse,
+    compat_urlparse,
+)
 from ..utils import (
     ExtractorError,
     int_or_none,
+    unsmuggle_url,
 )
 
 
@@ -121,24 +126,40 @@ class KalturaIE(InfoExtractor):
             video_id, actions, note='Downloading video info JSON')
 
     def _real_extract(self, url):
+        url, smuggled_data = unsmuggle_url(url, {})
+
         mobj = re.match(self._VALID_URL, url)
         partner_id = mobj.group('partner_id_s') or mobj.group('partner_id') or mobj.group('partner_id_html5')
         entry_id = mobj.group('id_s') or mobj.group('id') or mobj.group('id_html5')
 
         info, source_data = self._get_video_info(entry_id, partner_id)
 
-        formats = [{
-            'format_id': '%(fileExt)s-%(bitrate)s' % f,
-            'ext': f['fileExt'],
-            'tbr': f['bitrate'],
-            'fps': f.get('frameRate'),
-            'filesize_approx': int_or_none(f.get('size'), invscale=1024),
-            'container': f.get('containerFormat'),
-            'vcodec': f.get('videoCodecId'),
-            'height': f.get('height'),
-            'width': f.get('width'),
-            'url': '%s/flavorId/%s' % (info['dataUrl'], f['id']),
-        } for f in source_data['flavorAssets']]
+        source_url = smuggled_data.get('source_url')
+        if source_url:
+            referrer = base64.b64encode(
+                '://'.join(compat_urlparse.urlparse(source_url)[:2])
+                .encode('utf-8')).decode('utf-8')
+        else:
+            referrer = None
+
+        formats = []
+        for f in source_data['flavorAssets']:
+            video_url = '%s/flavorId/%s' % (info['dataUrl'], f['id'])
+            if referrer:
+                video_url += '?referrer=%s' % referrer
+            formats.append({
+                'format_id': '%(fileExt)s-%(bitrate)s' % f,
+                'ext': f['fileExt'],
+                'tbr': f['bitrate'],
+                'fps': f.get('frameRate'),
+                'filesize_approx': int_or_none(f.get('size'), invscale=1024),
+                'container': f.get('containerFormat'),
+                'vcodec': f.get('videoCodecId'),
+                'height': f.get('height'),
+                'width': f.get('width'),
+                'url': video_url,
+            })
+        self._check_formats(formats, entry_id)
         self._sort_formats(formats)
 
         return {

From 5b5fae5f2004e5a1be36551bf477f74161d4f9ac Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Nov 2015 01:35:58 +0600
Subject: [PATCH 3130/6219] [generic] Use referrer from source kaltura embed
 URLs (#7409)

---
 youtube_dl/extractor/generic.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index b483eba65..c7cee5487 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1694,7 +1694,9 @@ class GenericIE(InfoExtractor):
         mobj = (re.search(r"(?s)kWidget\.(?:thumb)?[Ee]mbed\(\{.*?'wid'\s*:\s*'_?(?P<partner_id>[^']+)',.*?'entry_?[Ii]d'\s*:\s*'(?P<id>[^']+)',", webpage) or
                 re.search(r'(?s)(?P<q1>["\'])(?:https?:)?//cdnapi(?:sec)?\.kaltura\.com/.*?(?:p|partner_id)/(?P<partner_id>\d+).*?(?P=q1).*?entry_?[Ii]d\s*:\s*(?P<q2>["\'])(?P<id>.+?)(?P=q2)', webpage))
         if mobj is not None:
-            return self.url_result('kaltura:%(partner_id)s:%(id)s' % mobj.groupdict(), 'Kaltura')
+            return self.url_result(smuggle_url(
+                'kaltura:%(partner_id)s:%(id)s' % mobj.groupdict(),
+                {'source_url': url}), 'Kaltura')
 
         # Look for Eagle.Platform embeds
         mobj = re.search(

From d80a39cec89e8ddfc3090c3d1b10d3fe105791e1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Nov 2015 01:38:08 +0600
Subject: [PATCH 3131/6219] [kaltura] Improve

---
 youtube_dl/extractor/kaltura.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/kaltura.py b/youtube_dl/extractor/kaltura.py
index 39e038659..1eaa09fea 100644
--- a/youtube_dl/extractor/kaltura.py
+++ b/youtube_dl/extractor/kaltura.py
@@ -149,14 +149,14 @@ class KalturaIE(InfoExtractor):
                 video_url += '?referrer=%s' % referrer
             formats.append({
                 'format_id': '%(fileExt)s-%(bitrate)s' % f,
-                'ext': f['fileExt'],
-                'tbr': f['bitrate'],
-                'fps': f.get('frameRate'),
+                'ext': f.get('fileExt'),
+                'tbr': int_or_none(f['bitrate']),
+                'fps': int_or_none(f.get('frameRate')),
                 'filesize_approx': int_or_none(f.get('size'), invscale=1024),
                 'container': f.get('containerFormat'),
                 'vcodec': f.get('videoCodecId'),
-                'height': f.get('height'),
-                'width': f.get('width'),
+                'height': int_or_none(f.get('height')),
+                'width': int_or_none(f.get('width')),
                 'url': video_url,
             })
         self._check_formats(formats, entry_id)

From bdceea7afde4fe02fff90f84ef1f2fb6ebbac9d3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Nov 2015 01:39:29 +0600
Subject: [PATCH 3132/6219] [kaltura] Clean description

---
 youtube_dl/extractor/kaltura.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/kaltura.py b/youtube_dl/extractor/kaltura.py
index 1eaa09fea..583b1a5ad 100644
--- a/youtube_dl/extractor/kaltura.py
+++ b/youtube_dl/extractor/kaltura.py
@@ -10,6 +10,7 @@ from ..compat import (
     compat_urlparse,
 )
 from ..utils import (
+    clean_html,
     ExtractorError,
     int_or_none,
     unsmuggle_url,
@@ -166,7 +167,7 @@ class KalturaIE(InfoExtractor):
             'id': entry_id,
             'title': info['name'],
             'formats': formats,
-            'description': info.get('description'),
+            'description': clean_html(info.get('description')),
             'thumbnail': info.get('thumbnailUrl'),
             'duration': info.get('duration'),
             'timestamp': info.get('createdAt'),

From 6da620de58bd1a4221bcdae5f4f2098b247038bf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Nov 2015 01:40:28 +0600
Subject: [PATCH 3133/6219] [kaltura] Add test for referrer protected video
 (#7409)

---
 youtube_dl/extractor/generic.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index c7cee5487..e512c1e14 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -823,6 +823,19 @@ class GenericIE(InfoExtractor):
                 'title': 'Os Guinness // Is It Fools Talk? // Unbelievable? Conference 2014',
             },
         },
+        # Kaltura embed protected with referrer
+        {
+            'url': 'http://www.disney.nl/disney-channel/filmpjes/achter-de-schermen#/videoId/violetta-achter-de-schermen-ruggero',
+            'info_dict': {
+                'id': '1_g4fbemnq',
+                'ext': 'mp4',
+                'title': 'Violetta - Achter De Schermen - Ruggero',
+                'description': 'Achter de schermen met Ruggero',
+                'timestamp': 1435133761,
+                'upload_date': '20150624',
+                'uploader_id': 'echojecka',
+            },
+        },
         # Eagle.Platform embed (generic URL)
         {
             'url': 'http://lenta.ru/news/2015/03/06/navalny/',

From 71bd93b89c667b7ca852b3a536dd771da7b67f67 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Nov 2015 08:08:34 +0600
Subject: [PATCH 3134/6219] [pluralsight] Do not rely on argument order in
 query (Closes #7583)

---
 youtube_dl/extractor/pluralsight.py | 24 +++++++++++++++---------
 1 file changed, 15 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index fd32836cc..6cef7c829 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -1,6 +1,5 @@
 from __future__ import unicode_literals
 
-import re
 import json
 
 from .common import InfoExtractor
@@ -19,11 +18,11 @@ from ..utils import (
 
 class PluralsightIE(InfoExtractor):
     IE_NAME = 'pluralsight'
-    _VALID_URL = r'https?://(?:www\.)?pluralsight\.com/training/player\?author=(?P<author>[^&]+)&name=(?P<name>[^&]+)(?:&mode=live)?&clip=(?P<clip>\d+)&course=(?P<course>[^&]+)'
+    _VALID_URL = r'https?://(?:(?:www|app)\.)?pluralsight\.com/training/player\?'
     _LOGIN_URL = 'https://www.pluralsight.com/id/'
     _NETRC_MACHINE = 'pluralsight'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.pluralsight.com/training/player?author=mike-mckeown&name=hosting-sql-server-windows-azure-iaas-m7-mgmt&mode=live&clip=3&course=hosting-sql-server-windows-azure-iaas',
         'md5': '4d458cf5cf4c593788672419a8dd4cf8',
         'info_dict': {
@@ -33,7 +32,10 @@ class PluralsightIE(InfoExtractor):
             'duration': 338,
         },
         'skip': 'Requires pluralsight account credentials',
-    }
+    }, {
+        'url': 'https://app.pluralsight.com/training/player?course=angularjs-get-started&author=scott-allen&name=angularjs-get-started-m1-introduction&clip=0&mode=live',
+        'only_matching': True,
+    }]
 
     def _real_initialize(self):
         self._login()
@@ -74,11 +76,15 @@ class PluralsightIE(InfoExtractor):
             raise ExtractorError('Unable to login: %s' % error, expected=True)
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        author = mobj.group('author')
-        name = mobj.group('name')
-        clip_id = mobj.group('clip')
-        course = mobj.group('course')
+        qs = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
+
+        author = qs.get('author', [None])[0]
+        name = qs.get('name', [None])[0]
+        clip_id = qs.get('clip', [None])[0]
+        course = qs.get('course', [None])[0]
+
+        if any(not f for f in (author, name, clip_id, course,)):
+            raise ExtractorError('Invalid URL', expected=True)
 
         display_id = '%s-%s' % (name, clip_id)
 

From 651acffbe57a016ae3b6b2c842aa691b1feb2d5b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Nov 2015 08:21:33 +0600
Subject: [PATCH 3135/6219] [pluralsight] Update ViewClip URL

---
 youtube_dl/extractor/pluralsight.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index 6cef7c829..caba8fb79 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -138,7 +138,7 @@ class PluralsightIE(InfoExtractor):
                     'q': '%dx%d' % (f['width'], f['height']),
                 }
                 request = compat_urllib_request.Request(
-                    'http://www.pluralsight.com/training/Player/ViewClip',
+                    'http://app.pluralsight.com/training/Player/ViewClip',
                     json.dumps(clip_post).encode('utf-8'))
                 request.add_header('Content-Type', 'application/json;charset=utf-8')
                 format_id = '%s-%s' % (ext, quality)

From c23e266427ef7fdcff3dc02e0d9978c58addea6c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Nov 2015 08:25:52 +0600
Subject: [PATCH 3136/6219] [pluralsight] Do not require pluralsight account

Looks like some courses are available without pluralsight account
---
 youtube_dl/extractor/pluralsight.py | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index caba8fb79..c6fd41d5a 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -35,6 +35,10 @@ class PluralsightIE(InfoExtractor):
     }, {
         'url': 'https://app.pluralsight.com/training/player?course=angularjs-get-started&author=scott-allen&name=angularjs-get-started-m1-introduction&clip=0&mode=live',
         'only_matching': True,
+    }, {
+        # available without pluralsight account
+        'url': 'http://app.pluralsight.com/training/player?author=scott-allen&name=angularjs-get-started-m1-introduction&mode=live&clip=0&course=angularjs-get-started',
+        'only_matching': True,
     }]
 
     def _real_initialize(self):
@@ -43,7 +47,7 @@ class PluralsightIE(InfoExtractor):
     def _login(self):
         (username, password) = self._get_login_info()
         if username is None:
-            self.raise_login_required('Pluralsight account is required')
+            return
 
         login_page = self._download_webpage(
             self._LOGIN_URL, None, 'Downloading login page')
@@ -172,7 +176,7 @@ class PluralsightIE(InfoExtractor):
 class PluralsightCourseIE(InfoExtractor):
     IE_NAME = 'pluralsight:course'
     _VALID_URL = r'https?://(?:www\.)?pluralsight\.com/courses/(?P<id>[^/]+)'
-    _TEST = {
+    _TESTS = [{
         # Free course from Pluralsight Starter Subscription for Microsoft TechNet
         # https://offers.pluralsight.com/technet?loc=zTS3z&prod=zOTprodz&tech=zOttechz&prog=zOTprogz&type=zSOz&media=zOTmediaz&country=zUSz
         'url': 'http://www.pluralsight.com/courses/hosting-sql-server-windows-azure-iaas',
@@ -182,7 +186,11 @@ class PluralsightCourseIE(InfoExtractor):
             'description': 'md5:61b37e60f21c4b2f91dc621a977d0986',
         },
         'playlist_count': 31,
-    }
+    }, {
+        # available without pluralsight account
+        'url': 'https://www.pluralsight.com/courses/angularjs-get-started',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         course_id = self._match_id(url)

From a5cd0eb8a4f2661cfe4863e958172d8f8f42d0f1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Nov 2015 08:32:48 +0600
Subject: [PATCH 3137/6219] [pluralsight:course] Improve _VALID_URL

---
 youtube_dl/extractor/pluralsight.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index c6fd41d5a..93244f41d 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -175,7 +175,7 @@ class PluralsightIE(InfoExtractor):
 
 class PluralsightCourseIE(InfoExtractor):
     IE_NAME = 'pluralsight:course'
-    _VALID_URL = r'https?://(?:www\.)?pluralsight\.com/courses/(?P<id>[^/]+)'
+    _VALID_URL = r'https?://(?:(?:www|app)\.)?pluralsight\.com/(?:library/)?courses/(?P<id>[^/]+)'
     _TESTS = [{
         # Free course from Pluralsight Starter Subscription for Microsoft TechNet
         # https://offers.pluralsight.com/technet?loc=zTS3z&prod=zOTprodz&tech=zOttechz&prog=zOTprogz&type=zSOz&media=zOTmediaz&country=zUSz
@@ -190,6 +190,9 @@ class PluralsightCourseIE(InfoExtractor):
         # available without pluralsight account
         'url': 'https://www.pluralsight.com/courses/angularjs-get-started',
         'only_matching': True,
+    }, {
+        'url': 'https://app.pluralsight.com/library/courses/understanding-microsoft-azure-amazon-aws/table-of-contents',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 6cc37c69e25f886c466d4b89f0734639e20e279b Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 21 Nov 2015 14:12:34 +0800
Subject: [PATCH 3138/6219] [generic] Unescape URLs from JWPlayer (#7582)

---
 youtube_dl/extractor/generic.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index e512c1e14..6cffde20d 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1874,6 +1874,7 @@ class GenericIE(InfoExtractor):
 
         entries = []
         for video_url in found:
+            video_url = video_url.replace('\\/', '/')
             video_url = compat_urlparse.urljoin(url, video_url)
             video_id = compat_urllib_parse_unquote(os.path.basename(video_url))
 

From 28602e747c13a7979aedd517e491bada3856cb12 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 21 Nov 2015 16:08:54 +0800
Subject: [PATCH 3139/6219] [generic] Refactor

---
 youtube_dl/extractor/generic.py | 25 +++++++++++--------------
 1 file changed, 11 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 6cffde20d..1991a8684 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1886,25 +1886,22 @@ class GenericIE(InfoExtractor):
             # here's a fun little line of code for you:
             video_id = os.path.splitext(video_id)[0]
 
+            entry_info_dict = {
+                'id': video_id,
+                'uploader': video_uploader,
+                'title': video_title,
+                'age_limit': age_limit,
+            }
+
             ext = determine_ext(video_url)
             if ext == 'smil':
-                entries.append({
-                    'id': video_id,
-                    'formats': self._extract_smil_formats(video_url, video_id),
-                    'uploader': video_uploader,
-                    'title': video_title,
-                    'age_limit': age_limit,
-                })
+                entry_info_dict['formats'] = self._extract_smil_formats(video_url, video_id)
             elif ext == 'xspf':
                 return self.playlist_result(self._extract_xspf_playlist(video_url, video_id), video_id)
             else:
-                entries.append({
-                    'id': video_id,
-                    'url': video_url,
-                    'uploader': video_uploader,
-                    'title': video_title,
-                    'age_limit': age_limit,
-                })
+                entry_info_dict['url'] = video_url
+
+            entries.append(entry_info_dict)
 
         if len(entries) == 1:
             return entries[0]

From 750b9ff0324b04f252b05cfadd802e8b1f2cc19d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 21 Nov 2015 16:43:01 +0800
Subject: [PATCH 3140/6219] [generic] Extract M3U8 formats (closes #7582)

---
 youtube_dl/extractor/generic.py | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 1991a8684..2b934148d 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1058,6 +1058,20 @@ class GenericIE(InfoExtractor):
                 'description': 'Tabletop: Dread, Last Thoughts',
                 'duration': 51690,
             },
+        },
+        # JWPlayer with M3U8
+        {
+            'url': 'http://ren.tv/novosti/2015-09-25/sluchaynyy-prohozhiy-poymal-avtougonshchika-v-murmanske-video',
+            'info_dict': {
+                'id': 'playlist',
+                'ext': 'mp4',
+                'title': 'Случайный прохожий поймал автоугонщика в Мурманске. ВИДЕО | РЕН ТВ',
+                'uploader': 'ren.tv',
+            },
+            'params': {
+                # m3u8 downloads
+                'skip_download': True,
+            }
         }
     ]
 
@@ -1898,6 +1912,8 @@ class GenericIE(InfoExtractor):
                 entry_info_dict['formats'] = self._extract_smil_formats(video_url, video_id)
             elif ext == 'xspf':
                 return self.playlist_result(self._extract_xspf_playlist(video_url, video_id), video_id)
+            elif ext == 'm3u8':
+                entry_info_dict['formats'] = self._extract_m3u8_formats(video_url, video_id, ext='mp4')
             else:
                 entry_info_dict['url'] = video_url
 

From f52183a8780a7b9e58ffb4510e234a7392ad8a2f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Nov 2015 17:39:24 +0600
Subject: [PATCH 3141/6219] [rutube:embed] Extend _VALID_URL (Closes #7588)

---
 youtube_dl/extractor/rutube.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/rutube.py b/youtube_dl/extractor/rutube.py
index d94dc7399..e824129b4 100644
--- a/youtube_dl/extractor/rutube.py
+++ b/youtube_dl/extractor/rutube.py
@@ -74,9 +74,9 @@ class RutubeIE(InfoExtractor):
 class RutubeEmbedIE(InfoExtractor):
     IE_NAME = 'rutube:embed'
     IE_DESC = 'Rutube embedded videos'
-    _VALID_URL = 'https?://rutube\.ru/video/embed/(?P<id>[0-9]+)'
+    _VALID_URL = 'https?://rutube\.ru/(?:video|play)/embed/(?P<id>[0-9]+)'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://rutube.ru/video/embed/6722881?vk_puid37=&vk_puid38=',
         'info_dict': {
             'id': 'a10e53b86e8f349080f718582ce4c661',
@@ -90,7 +90,10 @@ class RutubeEmbedIE(InfoExtractor):
         'params': {
             'skip_download': 'Requires ffmpeg',
         },
-    }
+    }, {
+        'url': 'http://rutube.ru/play/embed/8083783',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         embed_id = self._match_id(url)

From 019839faaa28d6233097a669d063a3a42f9fbbc7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Nov 2015 18:01:39 +0600
Subject: [PATCH 3142/6219] [extractor/common] Use baseURL from f4m manifest
 for recursive manifest extraction

---
 youtube_dl/extractor/common.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 5e263f8b5..71bdcad5a 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -891,6 +891,11 @@ class InfoExtractor(object):
         if not media_nodes:
             manifest_version = '2.0'
             media_nodes = manifest.findall('{http://ns.adobe.com/f4m/2.0}media')
+        base_url = xpath_text(
+            manifest, ['{http://ns.adobe.com/f4m/1.0}baseURL', '{http://ns.adobe.com/f4m/2.0}baseURL'],
+            'base URL', default=None)
+        if base_url:
+            base_url = base_url.strip()
         for i, media_el in enumerate(media_nodes):
             if manifest_version == '2.0':
                 media_url = media_el.attrib.get('href') or media_el.attrib.get('url')
@@ -898,7 +903,7 @@ class InfoExtractor(object):
                     continue
                 manifest_url = (
                     media_url if media_url.startswith('http://') or media_url.startswith('https://')
-                    else ('/'.join(manifest_url.split('/')[:-1]) + '/' + media_url))
+                    else ((base_url or '/'.join(manifest_url.split('/')[:-1])) + '/' + media_url))
                 # If media_url is itself a f4m manifest do the recursive extraction
                 # since bitrates in parent manifest (this one) and media_url manifest
                 # may differ leading to inability to resolve the format by requested

From 413719689996b2642f2484dd572d551376f0d104 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Nov 2015 18:02:52 +0600
Subject: [PATCH 3143/6219] [rutube] Extract all formats

---
 youtube_dl/extractor/rutube.py | 26 +++++++++++++++++++++-----
 1 file changed, 21 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/rutube.py b/youtube_dl/extractor/rutube.py
index e824129b4..0db8c410d 100644
--- a/youtube_dl/extractor/rutube.py
+++ b/youtube_dl/extractor/rutube.py
@@ -9,7 +9,7 @@ from ..compat import (
     compat_str,
 )
 from ..utils import (
-    ExtractorError,
+    determine_ext,
     unified_strdate,
 )
 
@@ -51,10 +51,26 @@ class RutubeIE(InfoExtractor):
             'http://rutube.ru/api/play/options/%s/?format=json' % video_id,
             video_id, 'Downloading options JSON')
 
-        m3u8_url = options['video_balancer'].get('m3u8')
-        if m3u8_url is None:
-            raise ExtractorError('Couldn\'t find m3u8 manifest url')
-        formats = self._extract_m3u8_formats(m3u8_url, video_id, ext='mp4')
+        formats = []
+        for format_id, format_url in options['video_balancer'].items():
+            ext = determine_ext(format_url)
+            print(ext)
+            if ext == 'm3u8':
+                m3u8_formats = self._extract_m3u8_formats(
+                    format_url, video_id, 'mp4', m3u8_id=format_id, fatal=False)
+                if m3u8_formats:
+                    formats.extend(m3u8_formats)
+            elif ext == 'f4m':
+                f4m_formats = self._extract_f4m_formats(
+                    format_url, video_id, f4m_id=format_id, fatal=False)
+                if f4m_formats:
+                    formats.extend(f4m_formats)
+            else:
+                formats.append({
+                    'url': format_url,
+                    'format_id': format_id,
+                })
+        self._sort_formats(formats)
 
         return {
             'id': video['id'],

From 2abf7cab80a2d12a3157afef05d61f8404bce45d Mon Sep 17 00:00:00 2001
From: reiv <metareiv@gmail.com>
Date: Sat, 17 Oct 2015 18:23:46 +0200
Subject: [PATCH 3144/6219] [soundcloud] Add Soundcloud search extractor

---
 youtube_dl/extractor/__init__.py   |  3 +-
 youtube_dl/extractor/soundcloud.py | 88 +++++++++++++++++++++++++++++-
 2 files changed, 89 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 26e5745d6..232bcd89a 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -576,7 +576,8 @@ from .soundcloud import (
     SoundcloudIE,
     SoundcloudSetIE,
     SoundcloudUserIE,
-    SoundcloudPlaylistIE
+    SoundcloudPlaylistIE,
+    SoundcloudSearchIE
 )
 from .soundgasm import (
     SoundgasmIE,
diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 2b60d354a..7395a9848 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -4,7 +4,10 @@ from __future__ import unicode_literals
 import re
 import itertools
 
-from .common import InfoExtractor
+from .common import (
+    InfoExtractor,
+    SearchInfoExtractor
+)
 from ..compat import (
     compat_str,
     compat_urlparse,
@@ -469,3 +472,86 @@ class SoundcloudPlaylistIE(SoundcloudIE):
             'description': data.get('description'),
             'entries': entries,
         }
+
+
+class SoundcloudSearchIE(SearchInfoExtractor, SoundcloudIE):
+    IE_NAME = 'soundcloud:search'
+    IE_DESC = 'Soundcloud search'
+    _MAX_RESULTS = 200
+    _TESTS = [{
+        'url': 'scsearch15:post-avant jazzcore',
+        'info_dict': {
+            'title': 'post-avant jazzcore',
+        },
+        'playlist_count': 15,
+    }]
+
+    _SEARCH_KEY = 'scsearch'
+    _RESULTS_PER_PAGE = 50
+
+    def _get_collection(self, endpoint, collection_id, **query):
+        import itertools
+
+        query['limit'] = self._RESULTS_PER_PAGE
+        query['client_id'] = self._CLIENT_ID
+        query['linked_partitioning'] = '1'
+
+        api_base_url = '{0}//api-v2.soundcloud.com'.format(self.http_scheme())
+
+        total_results = self._MAX_RESULTS
+        collected_results = 0
+
+        next_url = None
+
+        for i in itertools.count():
+
+            if not next_url:
+                query['offset'] = i * self._RESULTS_PER_PAGE
+                data = compat_urllib_parse.urlencode(query)
+                next_url = '{0}{1}?{2}'.format(api_base_url, endpoint, data)
+
+            response = self._download_json(next_url,
+                    video_id=collection_id,
+                    note='Downloading page {0}'.format(i+1),
+                    errnote='Unable to download API page')
+
+            total_results = int(response.get(
+                u'total_results', total_results))
+
+            collection = response['collection']
+            collected_results += len(collection)
+
+            for item in filter(bool, collection):
+                yield item
+
+            if collected_results >= total_results or not collection:
+                break
+
+            next_url = response.get(u'next_href', None)
+
+    def _get_n_results(self, query, n):
+
+        results = []
+
+        tracks = self._get_collection('/search/tracks',
+            collection_id='Query "{}"'.format(query),
+            q=query.encode('utf-8'))
+
+        for _ in range(n):
+            try:
+                track = next(tracks)
+            except StopIteration:
+                break
+            uri = track[u'uri']
+            title = track[u'title']
+            username = track[u'user'][u'username']
+            results.append(self.url_result(
+                url=uri,
+                video_title='{0} - {1}'.format(username, title)))
+
+        if not results:
+            raise ExtractorError(
+                '[soundcloud] No track results', expected=True)
+        
+        return self.playlist_result(results[:n], playlist_title=query)
+

From c30943b1c06d0f8d927031fc3a7f3cda30c4dae1 Mon Sep 17 00:00:00 2001
From: reiv <metareiv@gmail.com>
Date: Sat, 17 Oct 2015 21:18:42 +0200
Subject: [PATCH 3145/6219] Fix some compatibility issues, cleanup.

---
 youtube_dl/extractor/soundcloud.py | 24 ++++++++----------------
 1 file changed, 8 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 7395a9848..3fe991849 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -488,27 +488,23 @@ class SoundcloudSearchIE(SearchInfoExtractor, SoundcloudIE):
 
     _SEARCH_KEY = 'scsearch'
     _RESULTS_PER_PAGE = 50
+    _API_V2_BASE = 'https://api-v2.soundcloud.com'
 
     def _get_collection(self, endpoint, collection_id, **query):
-        import itertools
-
         query['limit'] = self._RESULTS_PER_PAGE
         query['client_id'] = self._CLIENT_ID
         query['linked_partitioning'] = '1'
 
-        api_base_url = '{0}//api-v2.soundcloud.com'.format(self.http_scheme())
-
         total_results = self._MAX_RESULTS
         collected_results = 0
 
         next_url = None
 
         for i in itertools.count():
-
             if not next_url:
                 query['offset'] = i * self._RESULTS_PER_PAGE
                 data = compat_urllib_parse.urlencode(query)
-                next_url = '{0}{1}?{2}'.format(api_base_url, endpoint, data)
+                next_url = '{0}{1}?{2}'.format(self._API_V2_BASE, endpoint, data)
 
             response = self._download_json(next_url,
                     video_id=collection_id,
@@ -516,7 +512,7 @@ class SoundcloudSearchIE(SearchInfoExtractor, SoundcloudIE):
                     errnote='Unable to download API page')
 
             total_results = int(response.get(
-                u'total_results', total_results))
+                'total_results', total_results))
 
             collection = response['collection']
             collected_results += len(collection)
@@ -527,14 +523,13 @@ class SoundcloudSearchIE(SearchInfoExtractor, SoundcloudIE):
             if collected_results >= total_results or not collection:
                 break
 
-            next_url = response.get(u'next_href', None)
+            next_url = response.get('next_href', None)
 
     def _get_n_results(self, query, n):
-
         results = []
 
         tracks = self._get_collection('/search/tracks',
-            collection_id='Query "{}"'.format(query),
+            collection_id='Query "{0}"'.format(query),
             q=query.encode('utf-8'))
 
         for _ in range(n):
@@ -542,12 +537,9 @@ class SoundcloudSearchIE(SearchInfoExtractor, SoundcloudIE):
                 track = next(tracks)
             except StopIteration:
                 break
-            uri = track[u'uri']
-            title = track[u'title']
-            username = track[u'user'][u'username']
-            results.append(self.url_result(
-                url=uri,
-                video_title='{0} - {1}'.format(username, title)))
+            uri = track['uri']
+            title = track['title']
+            results.append(self.url_result(url=uri))
 
         if not results:
             raise ExtractorError(

From b54b08c91bb7f59a9ea720abbf944809d1ea8956 Mon Sep 17 00:00:00 2001
From: reiv <metareiv@gmail.com>
Date: Sat, 17 Oct 2015 22:36:08 +0200
Subject: [PATCH 3146/6219] Simplify with itertools.islice().

---
 youtube_dl/extractor/soundcloud.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 3fe991849..959f27975 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -532,11 +532,7 @@ class SoundcloudSearchIE(SearchInfoExtractor, SoundcloudIE):
             collection_id='Query "{0}"'.format(query),
             q=query.encode('utf-8'))
 
-        for _ in range(n):
-            try:
-                track = next(tracks)
-            except StopIteration:
-                break
+        for track in itertools.islice(tracks, n):
             uri = track['uri']
             title = track['title']
             results.append(self.url_result(url=uri))

From 6ea7190a3e6165f3d5c49e1e142eb85951b44712 Mon Sep 17 00:00:00 2001
From: reiv <metareiv@gmail.com>
Date: Sat, 17 Oct 2015 22:47:16 +0200
Subject: [PATCH 3147/6219] Rewrite as list comprehension.

---
 youtube_dl/extractor/soundcloud.py | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 959f27975..c623bb6de 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -526,16 +526,12 @@ class SoundcloudSearchIE(SearchInfoExtractor, SoundcloudIE):
             next_url = response.get('next_href', None)
 
     def _get_n_results(self, query, n):
-        results = []
-
         tracks = self._get_collection('/search/tracks',
             collection_id='Query "{0}"'.format(query),
             q=query.encode('utf-8'))
 
-        for track in itertools.islice(tracks, n):
-            uri = track['uri']
-            title = track['title']
-            results.append(self.url_result(url=uri))
+        results = [self.url_result(url=track['uri'])
+            for track in itertools.islice(tracks, n)]
 
         if not results:
             raise ExtractorError(

From 417b4536999daaae2e2cc105012cd1c1e8046c15 Mon Sep 17 00:00:00 2001
From: reiv <metareiv@gmail.com>
Date: Sun, 18 Oct 2015 12:43:25 +0200
Subject: [PATCH 3148/6219] [soundcloud] Use correct error message conventions

---
 youtube_dl/extractor/soundcloud.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index c623bb6de..6b510a43b 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -535,7 +535,7 @@ class SoundcloudSearchIE(SearchInfoExtractor, SoundcloudIE):
 
         if not results:
             raise ExtractorError(
-                '[soundcloud] No track results', expected=True)
+                'Soundcloud said: No track results', expected=True)
         
-        return self.playlist_result(results[:n], playlist_title=query)
+        return self.playlist_result(results, playlist_title=query)
 

From 328a22e175a4e056258d50ebffe28026b9411285 Mon Sep 17 00:00:00 2001
From: reiv <metareiv@gmail.com>
Date: Fri, 30 Oct 2015 23:56:07 +0100
Subject: [PATCH 3149/6219] [soundcloud] Remove limit on search results

---
 youtube_dl/extractor/soundcloud.py | 21 +++++++++++++--------
 1 file changed, 13 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 6b510a43b..bba29d845 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -477,7 +477,7 @@ class SoundcloudPlaylistIE(SoundcloudIE):
 class SoundcloudSearchIE(SearchInfoExtractor, SoundcloudIE):
     IE_NAME = 'soundcloud:search'
     IE_DESC = 'Soundcloud search'
-    _MAX_RESULTS = 200
+    _MAX_RESULTS = float('inf')
     _TESTS = [{
         'url': 'scsearch15:post-avant jazzcore',
         'info_dict': {
@@ -487,24 +487,28 @@ class SoundcloudSearchIE(SearchInfoExtractor, SoundcloudIE):
     }]
 
     _SEARCH_KEY = 'scsearch'
-    _RESULTS_PER_PAGE = 50
+    _MAX_RESULTS_PER_PAGE = 200
+    _DEFAULT_RESULTS_PER_PAGE = 50
     _API_V2_BASE = 'https://api-v2.soundcloud.com'
 
     def _get_collection(self, endpoint, collection_id, **query):
-        query['limit'] = self._RESULTS_PER_PAGE
+        query['limit'] = results_per_page = min(
+            query.get('limit', self._DEFAULT_RESULTS_PER_PAGE),
+            self._MAX_RESULTS_PER_PAGE)
         query['client_id'] = self._CLIENT_ID
         query['linked_partitioning'] = '1'
 
-        total_results = self._MAX_RESULTS
+        total_results = None
         collected_results = 0
 
         next_url = None
 
         for i in itertools.count():
             if not next_url:
-                query['offset'] = i * self._RESULTS_PER_PAGE
+                query['offset'] = i * results_per_page
                 data = compat_urllib_parse.urlencode(query)
-                next_url = '{0}{1}?{2}'.format(self._API_V2_BASE, endpoint, data)
+                next_url = '{0}{1}?{2}'.format(
+                    self._API_V2_BASE, endpoint, data)
 
             response = self._download_json(next_url,
                     video_id=collection_id,
@@ -520,7 +524,8 @@ class SoundcloudSearchIE(SearchInfoExtractor, SoundcloudIE):
             for item in filter(bool, collection):
                 yield item
 
-            if collected_results >= total_results or not collection:
+            if (total_results is not None and
+                collected_results >= total_results) or not collection:
                 break
 
             next_url = response.get('next_href', None)
@@ -528,7 +533,7 @@ class SoundcloudSearchIE(SearchInfoExtractor, SoundcloudIE):
     def _get_n_results(self, query, n):
         tracks = self._get_collection('/search/tracks',
             collection_id='Query "{0}"'.format(query),
-            q=query.encode('utf-8'))
+            limit=n, q=query)
 
         results = [self.url_result(url=track['uri'])
             for track in itertools.islice(tracks, n)]

From 7e3472758bfaa75aa413368b29a26c2615e5231b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Nov 2015 20:04:35 +0600
Subject: [PATCH 3150/6219] [soundcloud:search] PEP 8

---
 youtube_dl/extractor/soundcloud.py | 23 +++++++++--------------
 1 file changed, 9 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index bba29d845..a5c40514b 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -510,10 +510,9 @@ class SoundcloudSearchIE(SearchInfoExtractor, SoundcloudIE):
                 next_url = '{0}{1}?{2}'.format(
                     self._API_V2_BASE, endpoint, data)
 
-            response = self._download_json(next_url,
-                    video_id=collection_id,
-                    note='Downloading page {0}'.format(i+1),
-                    errnote='Unable to download API page')
+            response = self._download_json(
+                next_url, collection_id, 'Downloading page {0}'.format(i + 1),
+                'Unable to download API page')
 
             total_results = int(response.get(
                 'total_results', total_results))
@@ -524,23 +523,19 @@ class SoundcloudSearchIE(SearchInfoExtractor, SoundcloudIE):
             for item in filter(bool, collection):
                 yield item
 
-            if (total_results is not None and
-                collected_results >= total_results) or not collection:
+            if (total_results is not None and collected_results >= total_results) or not collection:
                 break
 
-            next_url = response.get('next_href', None)
+            next_url = response.get('next_href')
 
     def _get_n_results(self, query, n):
-        tracks = self._get_collection('/search/tracks',
-            collection_id='Query "{0}"'.format(query),
-            limit=n, q=query)
+        tracks = self._get_collection(
+            '/search/tracks', collection_id='Query "{0}"'.format(query), limit=n, q=query)
 
-        results = [self.url_result(url=track['uri'])
-            for track in itertools.islice(tracks, n)]
+        results = [self.url_result(track['uri']) for track in itertools.islice(tracks, n)]
 
         if not results:
             raise ExtractorError(
                 'Soundcloud said: No track results', expected=True)
-        
-        return self.playlist_result(results, playlist_title=query)
 
+        return self.playlist_result(results, playlist_title=query)

From 7e1f5447e76e57af58bf45ce565742c813c80b99 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Nov 2015 20:45:50 +0600
Subject: [PATCH 3151/6219] [utils] Improve encode_dict

---
 youtube_dl/utils.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index d00b14b86..bff59eb73 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1668,7 +1668,9 @@ def urlencode_postdata(*args, **kargs):
 
 
 def encode_dict(d, encoding='utf-8'):
-    return dict((k.encode(encoding), v.encode(encoding)) for k, v in d.items())
+    def encode(v):
+        return v.encode(encoding) if isinstance(v, compat_basestring) else v
+    return dict((encode(k), encode(v)) for k, v in d.items())
 
 
 US_RATINGS = {

From 4e3b3030168927bd589343c51278040a5cec6203 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Nov 2015 20:55:48 +0600
Subject: [PATCH 3152/6219] [soundcloud:search] Fix non-ASCII searches

---
 youtube_dl/extractor/soundcloud.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index a5c40514b..66bb1cb54 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -14,6 +14,7 @@ from ..compat import (
     compat_urllib_parse,
 )
 from ..utils import (
+    encode_dict,
     ExtractorError,
     int_or_none,
     unified_strdate,
@@ -506,7 +507,7 @@ class SoundcloudSearchIE(SearchInfoExtractor, SoundcloudIE):
         for i in itertools.count():
             if not next_url:
                 query['offset'] = i * results_per_page
-                data = compat_urllib_parse.urlencode(query)
+                data = compat_urllib_parse.urlencode(encode_dict(query))
                 next_url = '{0}{1}?{2}'.format(
                     self._API_V2_BASE, endpoint, data)
 

From 7dc011c063c5c51712605ac08cf1c01841d5867c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Nov 2015 21:00:42 +0600
Subject: [PATCH 3153/6219] [soundcloud:search] Remove no track results message

---
 youtube_dl/extractor/soundcloud.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 66bb1cb54..e880872ae 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -535,8 +535,4 @@ class SoundcloudSearchIE(SearchInfoExtractor, SoundcloudIE):
 
         results = [self.url_result(track['uri']) for track in itertools.islice(tracks, n)]
 
-        if not results:
-            raise ExtractorError(
-                'Soundcloud said: No track results', expected=True)
-
         return self.playlist_result(results, playlist_title=query)

From f6c903e708bf8d9156ce5b999a220c4480a7aca5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Nov 2015 21:21:21 +0600
Subject: [PATCH 3154/6219] [soundcloud:search] Simplify (Closes #7213)

---
 youtube_dl/extractor/soundcloud.py | 42 +++++++++++++-----------------
 1 file changed, 18 insertions(+), 24 deletions(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index e880872ae..9b252c444 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -493,46 +493,40 @@ class SoundcloudSearchIE(SearchInfoExtractor, SoundcloudIE):
     _API_V2_BASE = 'https://api-v2.soundcloud.com'
 
     def _get_collection(self, endpoint, collection_id, **query):
-        query['limit'] = results_per_page = min(
+        limit = results_per_page = min(
             query.get('limit', self._DEFAULT_RESULTS_PER_PAGE),
             self._MAX_RESULTS_PER_PAGE)
+        query['limit'] = limit
         query['client_id'] = self._CLIENT_ID
         query['linked_partitioning'] = '1'
+        query['offset'] = 0
+        data = compat_urllib_parse.urlencode(encode_dict(query))
+        next_url = '{0}{1}?{2}'.format(self._API_V2_BASE, endpoint, data)
 
-        total_results = None
         collected_results = 0
 
-        next_url = None
-
-        for i in itertools.count():
-            if not next_url:
-                query['offset'] = i * results_per_page
-                data = compat_urllib_parse.urlencode(encode_dict(query))
-                next_url = '{0}{1}?{2}'.format(
-                    self._API_V2_BASE, endpoint, data)
-
+        for i in itertools.count(1):
             response = self._download_json(
-                next_url, collection_id, 'Downloading page {0}'.format(i + 1),
+                next_url, collection_id, 'Downloading page {0}'.format(i),
                 'Unable to download API page')
 
-            total_results = int(response.get(
-                'total_results', total_results))
+            collection = response.get('collection', [])
+            if not collection:
+                break
 
-            collection = response['collection']
+            collection = list(filter(bool, collection))
             collected_results += len(collection)
 
-            for item in filter(bool, collection):
-                yield item
+            for item in collection:
+                yield self.url_result(item['uri'], SoundcloudIE.ie_key())
 
-            if (total_results is not None and collected_results >= total_results) or not collection:
+            if not collection or collected_results >= limit:
                 break
 
             next_url = response.get('next_href')
+            if not next_url:
+                break
 
     def _get_n_results(self, query, n):
-        tracks = self._get_collection(
-            '/search/tracks', collection_id='Query "{0}"'.format(query), limit=n, q=query)
-
-        results = [self.url_result(track['uri']) for track in itertools.islice(tracks, n)]
-
-        return self.playlist_result(results, playlist_title=query)
+        tracks = self._get_collection('/search/tracks', query, limit=n, q=query)
+        return self.playlist_result(tracks, playlist_title=query)

From c3a227d1c406d0af8fdf6afd9e33366e903d9a8a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Nov 2015 21:25:48 +0600
Subject: [PATCH 3155/6219] [pluralsight] Update _LOGIN_URL

---
 youtube_dl/extractor/pluralsight.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index 93244f41d..417dd965c 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -19,7 +19,7 @@ from ..utils import (
 class PluralsightIE(InfoExtractor):
     IE_NAME = 'pluralsight'
     _VALID_URL = r'https?://(?:(?:www|app)\.)?pluralsight\.com/training/player\?'
-    _LOGIN_URL = 'https://www.pluralsight.com/id/'
+    _LOGIN_URL = 'https://app.pluralsight.com/id/'
     _NETRC_MACHINE = 'pluralsight'
 
     _TESTS = [{

From 0533915aad95d76b6bf4f529d39167f22a7c4b90 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Nov 2015 21:35:08 +0600
Subject: [PATCH 3156/6219] [pluralsight] Update some more URLs

---
 youtube_dl/extractor/pluralsight.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index 417dd965c..fe6850aac 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -20,6 +20,7 @@ class PluralsightIE(InfoExtractor):
     IE_NAME = 'pluralsight'
     _VALID_URL = r'https?://(?:(?:www|app)\.)?pluralsight\.com/training/player\?'
     _LOGIN_URL = 'https://app.pluralsight.com/id/'
+    _API_BASE = 'http://app.pluralsight.com'
     _NETRC_MACHINE = 'pluralsight'
 
     _TESTS = [{
@@ -142,7 +143,7 @@ class PluralsightIE(InfoExtractor):
                     'q': '%dx%d' % (f['width'], f['height']),
                 }
                 request = compat_urllib_request.Request(
-                    'http://app.pluralsight.com/training/Player/ViewClip',
+                    '%s/training/Player/ViewClip' % self._API_BASE,
                     json.dumps(clip_post).encode('utf-8'))
                 request.add_header('Content-Type', 'application/json;charset=utf-8')
                 format_id = '%s-%s' % (ext, quality)
@@ -201,14 +202,14 @@ class PluralsightCourseIE(InfoExtractor):
         # TODO: PSM cookie
 
         course = self._download_json(
-            'http://www.pluralsight.com/data/course/%s' % course_id,
+            '%s/data/course/%s' % (self._API_BASE, course_id),
             course_id, 'Downloading course JSON')
 
         title = course['title']
         description = course.get('description') or course.get('shortDescription')
 
         course_data = self._download_json(
-            'http://www.pluralsight.com/data/course/content/%s' % course_id,
+            '%s/data/course/content/%s' % (self._API_BASE, course_id),
             course_id, 'Downloading course data JSON')
 
         entries = []
@@ -218,7 +219,7 @@ class PluralsightCourseIE(InfoExtractor):
                 if not player_parameters:
                     continue
                 entries.append(self.url_result(
-                    'http://www.pluralsight.com/training/player?%s' % player_parameters,
+                    '%s/training/player?%s' % (self._API_BASE, player_parameters),
                     'Pluralsight'))
 
         return self.playlist_result(entries, course_id, title, description)

From 563772eda4a42b1fce1f3740fe91bf74e63bd347 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Nov 2015 21:37:29 +0600
Subject: [PATCH 3157/6219] [pluralsight] Extract base class

---
 youtube_dl/extractor/pluralsight.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index fe6850aac..de7dc739b 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -16,11 +16,15 @@ from ..utils import (
 )
 
 
-class PluralsightIE(InfoExtractor):
+class PluralsightBaseIE(InfoExtractor):
+    _API_BASE = 'http://app.pluralsight.com'
+
+
+class PluralsightIE(PluralsightBaseIE):
     IE_NAME = 'pluralsight'
     _VALID_URL = r'https?://(?:(?:www|app)\.)?pluralsight\.com/training/player\?'
     _LOGIN_URL = 'https://app.pluralsight.com/id/'
-    _API_BASE = 'http://app.pluralsight.com'
+
     _NETRC_MACHINE = 'pluralsight'
 
     _TESTS = [{
@@ -174,7 +178,7 @@ class PluralsightIE(InfoExtractor):
         }
 
 
-class PluralsightCourseIE(InfoExtractor):
+class PluralsightCourseIE(PluralsightBaseIE):
     IE_NAME = 'pluralsight:course'
     _VALID_URL = r'https?://(?:(?:www|app)\.)?pluralsight\.com/(?:library/)?courses/(?P<id>[^/]+)'
     _TESTS = [{

From 7e508ff2cf8f8f8b1784db6fb33994839841d122 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Nov 2015 21:49:37 +0600
Subject: [PATCH 3158/6219] [pluralsight] Improve login detection

---
 youtube_dl/extractor/pluralsight.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index de7dc739b..7693282a5 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -84,6 +84,9 @@ class PluralsightIE(PluralsightBaseIE):
         if error:
             raise ExtractorError('Unable to login: %s' % error, expected=True)
 
+        if all(p not in response for p in ('__INITIAL_STATE__', '"currentUser"')):
+            raise ExtractorError('Unable to log in')
+
     def _real_extract(self, url):
         qs = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
 

From bea56c95699af594586095e5ea88e9857049c6a1 Mon Sep 17 00:00:00 2001
From: Andrzej Lichnerowicz <andrzej@lichnerowicz.pl>
Date: Sat, 26 Sep 2015 21:58:33 +0200
Subject: [PATCH 3159/6219] [pluralsight] prevent error 429 when sensing video
 formats

---
 youtube_dl/extractor/pluralsight.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index 7693282a5..d542a9e0e 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -156,6 +156,9 @@ class PluralsightIE(PluralsightBaseIE):
                 format_id = '%s-%s' % (ext, quality)
                 clip_url = self._download_webpage(
                     request, display_id, 'Downloading %s URL' % format_id, fatal=False)
+                # #6989: sleep 3 seconds to avoid 429 errors.
+                # should help with #6842.
+                self._sleep(3, display_id)
                 if not clip_url:
                     continue
                 f.update({

From 38eb2968abce9078f28904046b8e26b0d87ada61 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Nov 2015 00:07:09 +0600
Subject: [PATCH 3160/6219] [pluralsight] Clarify and randomize ViewClip sleep
 interval

---
 youtube_dl/extractor/pluralsight.py | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index d542a9e0e..8481a10b0 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -1,6 +1,7 @@
 from __future__ import unicode_literals
 
 import json
+import random
 
 from .common import InfoExtractor
 from ..compat import (
@@ -156,9 +157,17 @@ class PluralsightIE(PluralsightBaseIE):
                 format_id = '%s-%s' % (ext, quality)
                 clip_url = self._download_webpage(
                     request, display_id, 'Downloading %s URL' % format_id, fatal=False)
-                # #6989: sleep 3 seconds to avoid 429 errors.
-                # should help with #6842.
-                self._sleep(3, display_id)
+
+                # Pluralsight tracks multiple sequential calls to ViewClip API and start
+                # to return 429 HTTP errors after some time (see
+                # https://github.com/rg3/youtube-dl/pull/6989). Moreover it may even lead
+                # to account ban (see https://github.com/rg3/youtube-dl/issues/6842).
+                # To somewhat reduce the probability of these consequences
+                # we will sleep random amount of time before each call to ViewClip.
+                self._sleep(
+                    random.randint(2, 5), display_id,
+                    '%(video_id)s: Waiting for %(timeout)s seconds to avoid throttling')
+
                 if not clip_url:
                     continue
                 f.update({

From 4c57b4853d8c1b41d729314e316cc1b83129ea18 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Nov 2015 00:42:58 +0600
Subject: [PATCH 3161/6219] [pluralsight] Until listing formats request only
 single format

---
 youtube_dl/extractor/pluralsight.py | 24 +++++++++++++++++++++---
 1 file changed, 21 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index 8481a10b0..16de0fd1c 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -2,6 +2,7 @@ from __future__ import unicode_literals
 
 import json
 import random
+import collections
 
 from .common import InfoExtractor
 from ..compat import (
@@ -131,13 +132,30 @@ class PluralsightIE(PluralsightBaseIE):
             'high': {'width': 1024, 'height': 768},
         }
 
+        AllowedQuality = collections.namedtuple('AllowedQuality', ['ext', 'qualities'])
+
         ALLOWED_QUALITIES = (
-            ('webm', ('high',)),
-            ('mp4', ('low', 'medium', 'high',)),
+            AllowedQuality('webm', ('high',)),
+            AllowedQuality('mp4', ('low', 'medium', 'high',)),
         )
 
+        if self._downloader.params.get('listformats', False):
+            allowed_qualities = ALLOWED_QUALITIES
+        else:
+            def guess_allowed_qualities():
+                req_format = self._downloader.params.get('format') or 'best'
+                req_format_split = req_format.split('-')
+                if len(req_format_split) > 1:
+                    req_ext, req_quality = req_format_split
+                    for allowed_quality in ALLOWED_QUALITIES:
+                        if req_ext == allowed_quality.ext and req_quality in allowed_quality.qualities:
+                            return (AllowedQuality(req_ext, (req_quality, )), )
+                req_ext = 'webm' if self._downloader.params.get('prefer_free_formats') else 'mp4'
+                return (AllowedQuality(req_ext, ('high', )), )
+            allowed_qualities = guess_allowed_qualities()
+
         formats = []
-        for ext, qualities in ALLOWED_QUALITIES:
+        for ext, qualities in allowed_qualities:
             for quality in qualities:
                 f = QUALITIES[quality].copy()
                 clip_post = {

From a3372437bfe57f5b450bfdd60105bb6120595928 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Nov 2015 00:49:58 +0600
Subject: [PATCH 3162/6219] [soundcloud] Remove unused variable

---
 youtube_dl/extractor/soundcloud.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 9b252c444..02e64e094 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -493,7 +493,7 @@ class SoundcloudSearchIE(SearchInfoExtractor, SoundcloudIE):
     _API_V2_BASE = 'https://api-v2.soundcloud.com'
 
     def _get_collection(self, endpoint, collection_id, **query):
-        limit = results_per_page = min(
+        limit = min(
             query.get('limit', self._DEFAULT_RESULTS_PER_PAGE),
             self._MAX_RESULTS_PER_PAGE)
         query['limit'] = limit

From cf186b77a79a0bf6f322bc1e2f610471eab3b01c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Nov 2015 00:56:40 +0600
Subject: [PATCH 3163/6219] [pluralsight] Clarify allowed qualities guessing
 rationale

---
 youtube_dl/extractor/pluralsight.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index 16de0fd1c..3dee616c1 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -139,6 +139,9 @@ class PluralsightIE(PluralsightBaseIE):
             AllowedQuality('mp4', ('low', 'medium', 'high',)),
         )
 
+        # In order to minimize the number of calls to ViewClip API and reduce
+        # the probability of being throttled or banned by Pluralsight we will request
+        # only single format until explicit listformats was requested.
         if self._downloader.params.get('listformats', False):
             allowed_qualities = ALLOWED_QUALITIES
         else:

From 0eebf34d9dd793bce9148dae267d690d50d89f88 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Nov 2015 00:58:25 +0600
Subject: [PATCH 3164/6219] [pluralsight] Rephrase

---
 youtube_dl/extractor/pluralsight.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index 3dee616c1..a11cebaf1 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -141,7 +141,7 @@ class PluralsightIE(PluralsightBaseIE):
 
         # In order to minimize the number of calls to ViewClip API and reduce
         # the probability of being throttled or banned by Pluralsight we will request
-        # only single format until explicit listformats was requested.
+        # only single format until formats listing was explicitly requested.
         if self._downloader.params.get('listformats', False):
             allowed_qualities = ALLOWED_QUALITIES
         else:

From 0c14841585db1baa2f9a4a5ff263035977cf0964 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Nov 2015 04:17:07 +0600
Subject: [PATCH 3165/6219] [youtube:user:playlists] Add extractor (Closes
 #3817)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/youtube.py  | 26 ++++++++++++++++++++++++++
 2 files changed, 27 insertions(+)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 232bcd89a..947b83683 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -834,6 +834,7 @@ from .youtube import (
     YoutubeTruncatedIDIE,
     YoutubeTruncatedURLIE,
     YoutubeUserIE,
+    YoutubeUserPlaylistsIE,
     YoutubeWatchLaterIE,
 )
 from .zapiks import ZapiksIE
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 364ca102a..abc67f07f 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -224,6 +224,17 @@ class YoutubePlaylistBaseInfoExtractor(InfoExtractor):
         return zip(ids_in_page, titles_in_page)
 
 
+class YoutubePlaylistsBaseInfoExtractor(InfoExtractor):
+    def _real_extract(self, url):
+        playlist_id = self._match_id(url)
+        webpage = self._download_webpage(url, playlist_id)
+        entries = [
+            self.url_result(compat_urlparse.urljoin(url, playlist), 'YoutubePlaylist')
+            for playlist in re.findall(r'href="(/playlist\?list=.+?)"', webpage)]
+        title = self._og_search_title(webpage, fatal=False)
+        return self.playlist_result(entries, playlist_id, title)
+
+
 class YoutubeIE(YoutubeBaseInfoExtractor):
     IE_DESC = 'YouTube.com'
     _VALID_URL = r"""(?x)^
@@ -1742,6 +1753,21 @@ class YoutubeUserIE(YoutubeChannelIE):
             return super(YoutubeUserIE, cls).suitable(url)
 
 
+class YoutubeUserPlaylistsIE(YoutubePlaylistsBaseInfoExtractor):
+    IE_DESC = 'YouTube.com user playlists'
+    _VALID_URL = r'https?://(?:\w+\.)?youtube\.com/user/(?P<id>[^/]+)/playlists'
+    IE_NAME = 'youtube:user:playlists'
+
+    _TEST = {
+        'url': 'http://www.youtube.com/user/ThirstForScience/playlists',
+        'playlist_mincount': 4,
+        'info_dict': {
+            'id': 'ThirstForScience',
+            'title': 'Thirst for Science',
+        },
+    }
+
+
 class YoutubeSearchIE(SearchInfoExtractor, YoutubePlaylistIE):
     IE_DESC = 'YouTube.com searches'
     # there doesn't appear to be a real limit, for example if you search for

From 136dadde9543a80f490b26c822dcfdff5541c335 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Nov 2015 04:18:20 +0600
Subject: [PATCH 3166/6219] [youtube:show] Rework in terms of playlists base
 extractor

---
 youtube_dl/extractor/youtube.py | 25 ++++---------------------
 1 file changed, 4 insertions(+), 21 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index abc67f07f..c56f8a0a2 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1863,7 +1863,7 @@ class YoutubeSearchURLIE(InfoExtractor):
         }
 
 
-class YoutubeShowIE(InfoExtractor):
+class YoutubeShowIE(YoutubePlaylistsBaseInfoExtractor):
     IE_DESC = 'YouTube.com (multi-season) shows'
     _VALID_URL = r'https?://www\.youtube\.com/show/(?P<id>[^?#]*)'
     IE_NAME = 'youtube:show'
@@ -1877,26 +1877,9 @@ class YoutubeShowIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        playlist_id = mobj.group('id')
-        webpage = self._download_webpage(
-            'https://www.youtube.com/show/%s/playlists' % playlist_id, playlist_id, 'Downloading show webpage')
-        # There's one playlist for each season of the show
-        m_seasons = list(re.finditer(r'href="(/playlist\?list=.*?)"', webpage))
-        self.to_screen('%s: Found %s seasons' % (playlist_id, len(m_seasons)))
-        entries = [
-            self.url_result(
-                'https://www.youtube.com' + season.group(1), 'YoutubePlaylist')
-            for season in m_seasons
-        ]
-        title = self._og_search_title(webpage, fatal=False)
-
-        return {
-            '_type': 'playlist',
-            'id': playlist_id,
-            'title': title,
-            'entries': entries,
-        }
+        playlist_id = self._match_id(url)
+        return super(YoutubeShowIE, self)._real_extract(
+            'https://www.youtube.com/show/%s/playlists' % playlist_id)
 
 
 class YoutubeFeedsInfoExtractor(YoutubeBaseInfoExtractor):

From 82c4d7b0ce49b13a9dd8491dd32e962ca307c974 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 21 Nov 2015 23:36:27 +0100
Subject: [PATCH 3167/6219] release 2015.11.21

---
 docs/supportedsites.md | 2 ++
 youtube_dl/version.py  | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index f56a6986b..1df408610 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -494,6 +494,7 @@
  - **soompi:show**
  - **soundcloud**
  - **soundcloud:playlist**
+ - **soundcloud:search**: Soundcloud search
  - **soundcloud:set**
  - **soundcloud:user**
  - **soundgasm**
@@ -707,6 +708,7 @@
  - **youtube:show**: YouTube.com (multi-season) shows
  - **youtube:subscriptions**: YouTube.com subscriptions feed, "ytsubs" keyword (requires authentication)
  - **youtube:user**: YouTube.com user videos (URL or "ytuser" keyword)
+ - **youtube:user:playlists**: YouTube.com user playlists
  - **youtube:watchlater**: Youtube watch later list, ":ytwatchlater" for short (requires authentication)
  - **Zapiks**
  - **ZDF**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index b3b1b90ba..2baf1ac42 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.11.19'
+__version__ = '2015.11.21'

From 061a75edd6bab2f30978b458fe7402ff9e9c02a5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Nov 2015 05:01:01 +0600
Subject: [PATCH 3168/6219] [youtube] Extract base for entry list extractors
 and support multi page lists of playlists

---
 youtube_dl/extractor/youtube.py | 28 +++++++++++++++++-----------
 1 file changed, 17 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index c56f8a0a2..8352ad1da 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -178,15 +178,13 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
             return
 
 
-class YoutubePlaylistBaseInfoExtractor(InfoExtractor):
-    # Extract the video ids from the playlist pages
+class YoutubeEntryListBaseInfoExtractor(InfoExtractor):
+    # Extract entries from page with "Load more" button
     def _entries(self, page, playlist_id):
         more_widget_html = content_html = page
         for page_num in itertools.count(1):
-            for video_id, video_title in self.extract_videos_from_page(content_html):
-                yield self.url_result(
-                    video_id, 'Youtube', video_id=video_id,
-                    video_title=video_title)
+            for entry in self._process_page(content_html):
+                yield entry
 
             mobj = re.search(r'data-uix-load-more-href="/?(?P<more>[^"]+)"', more_widget_html)
             if not mobj:
@@ -203,6 +201,12 @@ class YoutubePlaylistBaseInfoExtractor(InfoExtractor):
                 break
             more_widget_html = more['load_more_widget_html']
 
+
+class YoutubePlaylistBaseInfoExtractor(YoutubeEntryListBaseInfoExtractor):
+    def _process_page(self, content):
+        for video_id, video_title in self.extract_videos_from_page(content):
+            yield self.url_result(video_id, 'Youtube', video_id, video_title)
+
     def extract_videos_from_page(self, page):
         ids_in_page = []
         titles_in_page = []
@@ -224,15 +228,17 @@ class YoutubePlaylistBaseInfoExtractor(InfoExtractor):
         return zip(ids_in_page, titles_in_page)
 
 
-class YoutubePlaylistsBaseInfoExtractor(InfoExtractor):
+class YoutubePlaylistsBaseInfoExtractor(YoutubeEntryListBaseInfoExtractor):
+    def _process_page(self, content):
+        for playlist_id in re.findall(r'href="/?playlist\?list=(.+?)"', content):
+            yield self.url_result(
+                'https://www.youtube.com/playlist?list=%s' % playlist_id, 'YoutubePlaylist')
+
     def _real_extract(self, url):
         playlist_id = self._match_id(url)
         webpage = self._download_webpage(url, playlist_id)
-        entries = [
-            self.url_result(compat_urlparse.urljoin(url, playlist), 'YoutubePlaylist')
-            for playlist in re.findall(r'href="(/playlist\?list=.+?)"', webpage)]
         title = self._og_search_title(webpage, fatal=False)
-        return self.playlist_result(entries, playlist_id, title)
+        return self.playlist_result(self._entries(webpage, playlist_id), playlist_id, title)
 
 
 class YoutubeIE(YoutubeBaseInfoExtractor):

From e568c2233e7b4b27c9a5c56322ab7633a5f0b1f7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Nov 2015 05:03:23 +0600
Subject: [PATCH 3169/6219] [youtube] Add test for multi page list of playlists

---
 youtube_dl/extractor/youtube.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 8352ad1da..4a0ff6e9c 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1764,14 +1764,22 @@ class YoutubeUserPlaylistsIE(YoutubePlaylistsBaseInfoExtractor):
     _VALID_URL = r'https?://(?:\w+\.)?youtube\.com/user/(?P<id>[^/]+)/playlists'
     IE_NAME = 'youtube:user:playlists'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.youtube.com/user/ThirstForScience/playlists',
         'playlist_mincount': 4,
         'info_dict': {
             'id': 'ThirstForScience',
             'title': 'Thirst for Science',
         },
-    }
+    }, {
+        # with "Load more" button
+        'url': 'http://www.youtube.com/user/igorkle1/playlists?view=1&sort=dd',
+        'playlist_mincount': 70,
+        'info_dict': {
+            'id': 'igorkle1',
+            'title': 'Игорь Клейнер',
+        },
+    }]
 
 
 class YoutubeSearchIE(SearchInfoExtractor, YoutubePlaylistIE):

From 3e12bc583af9d5abf5f144ed6e092c59f4b83fdf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Nov 2015 06:29:39 +0600
Subject: [PATCH 3170/6219] [utils] Improve determine_ext (Closes #7593)

---
 youtube_dl/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index bff59eb73..7dab60bb8 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -922,7 +922,7 @@ def unified_strdate(date_str, day_first=True):
 def determine_ext(url, default_ext='unknown_video'):
     if url is None:
         return default_ext
-    guess = url.partition('?')[0].rpartition('.')[2]
+    guess = url.partition('?')[0].rpartition('.')[2].rstrip('/')
     if re.match(r'^[A-Za-z0-9]+$', guess):
         return guess
     else:

From 5035536e3f32d4c47b2d3067c12e074cb9a4a199 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Nov 2015 06:33:52 +0600
Subject: [PATCH 3171/6219] [test_utils] Add tests for determine_ext

---
 test/test_utils.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/test/test_utils.py b/test/test_utils.py
index ea1ff0547..4f0d9e481 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -21,6 +21,7 @@ from youtube_dl.utils import (
     clean_html,
     DateRange,
     detect_exe_version,
+    determine_ext,
     encodeFilename,
     escape_rfc3986,
     escape_url,
@@ -238,6 +239,10 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(unified_strdate('25-09-2014'), '20140925')
         self.assertEqual(unified_strdate('UNKNOWN DATE FORMAT'), None)
 
+    def test_determine_ext(self):
+        self.assertEqual(determine_ext('http://example.com/foo/bar.mp4/?download'), 'mp4')
+        self.assertEqual(determine_ext('http://example.com/foo/bar/?download', None), None)
+
     def test_find_xpath_attr(self):
         testxml = '''<root>
             <node/>

From 9cb9a5df7794579c38efff1c4b1451a7d13da3c1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Nov 2015 17:27:13 +0600
Subject: [PATCH 3172/6219] [utils] Check ext with trailing slash against the
 list of known extensions

---
 test/test_utils.py  |  3 +++
 youtube_dl/utils.py | 17 ++++++++++++++++-
 2 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 4f0d9e481..501355c74 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -242,6 +242,9 @@ class TestUtil(unittest.TestCase):
     def test_determine_ext(self):
         self.assertEqual(determine_ext('http://example.com/foo/bar.mp4/?download'), 'mp4')
         self.assertEqual(determine_ext('http://example.com/foo/bar/?download', None), None)
+        self.assertEqual(determine_ext('http://example.com/foo/bar.nonext/?download', None), None)
+        self.assertEqual(determine_ext('http://example.com/foo/bar/mp4?download', None), None)
+        self.assertEqual(determine_ext('http://example.com/foo/bar.m3u8//?download'), 'm3u8')
 
     def test_find_xpath_attr(self):
         testxml = '''<root>
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 7dab60bb8..c0325f054 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -922,9 +922,24 @@ def unified_strdate(date_str, day_first=True):
 def determine_ext(url, default_ext='unknown_video'):
     if url is None:
         return default_ext
-    guess = url.partition('?')[0].rpartition('.')[2].rstrip('/')
+    guess = url.partition('?')[0].rpartition('.')[2]
     if re.match(r'^[A-Za-z0-9]+$', guess):
         return guess
+    elif guess.rstrip('/') in (
+            'mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v', 'aac',
+            'flv', 'f4v', 'f4a', 'f4b',
+            'webm', 'ogg', 'ogv', 'oga', 'ogx', 'spx', 'opus',
+            'mkv', 'mka', 'mk3d',
+            'avi', 'divx',
+            'mov',
+            'asf', 'wmv', 'wma',
+            '3gp', '3g2',
+            'mp3',
+            'flac',
+            'ape',
+            'wav',
+            'f4f', 'f4m', 'm3u8', 'smil'):
+        return guess.rstrip('/')
     else:
         return default_ext
 

From 1b38185361e096d6e34db11adac7333ac9dadca0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Nov 2015 18:08:30 +0600
Subject: [PATCH 3173/6219] [pornhd] Fix title extraction (Closes #7596)

---
 youtube_dl/extractor/pornhd.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pornhd.py b/youtube_dl/extractor/pornhd.py
index dbb2c3bd9..57c78ba52 100644
--- a/youtube_dl/extractor/pornhd.py
+++ b/youtube_dl/extractor/pornhd.py
@@ -36,7 +36,8 @@ class PornHdIE(InfoExtractor):
         webpage = self._download_webpage(url, display_id or video_id)
 
         title = self._html_search_regex(
-            r'<title>(.+) porn HD.+?</title>', webpage, 'title')
+            [r'<span[^>]+class=["\']video-name["\'][^>]*>([^<]+)',
+             r'<title>(.+?) - .*?[Pp]ornHD.*?</title>'], webpage, 'title')
         description = self._html_search_regex(
             r'<div class="description">([^<]+)</div>', webpage, 'description', fatal=False)
         view_count = int_or_none(self._html_search_regex(

From 3cfd000849208b58dab4f78d1486d3f24552009e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Lalinsk=C3=BD?= <lukas@oxygene.sk>
Date: Sun, 22 Nov 2015 13:14:35 +0100
Subject: [PATCH 3174/6219] [youtube] More explicit player config JSON
 extraction (fixes #7468)

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 687e0b4db..21731188a 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1074,7 +1074,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             age_gate = False
             video_info = None
             # Try looking directly into the video webpage
-            mobj = re.search(r';ytplayer\.config\s*=\s*({.*?});', video_webpage)
+            mobj = re.search(r';ytplayer\.config\s*=\s*({.*?});ytplayer', video_webpage)
             if mobj:
                 json_code = uppercase_escape(mobj.group(1))
                 ytplayer_config = json.loads(json_code)

From 4a7d108ab3c8b5ac82b8740179dae6a454218a38 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Nov 2015 18:24:17 +0600
Subject: [PATCH 3175/6219] [rutube] Remove unnecessary print

---
 youtube_dl/extractor/rutube.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/rutube.py b/youtube_dl/extractor/rutube.py
index 0db8c410d..6b09550b0 100644
--- a/youtube_dl/extractor/rutube.py
+++ b/youtube_dl/extractor/rutube.py
@@ -54,7 +54,6 @@ class RutubeIE(InfoExtractor):
         formats = []
         for format_id, format_url in options['video_balancer'].items():
             ext = determine_ext(format_url)
-            print(ext)
             if ext == 'm3u8':
                 m3u8_formats = self._extract_m3u8_formats(
                     format_url, video_id, 'mp4', m3u8_id=format_id, fatal=False)

From 0e49d9a6b0216555c2a3ee063ae3d1c6d09edbd4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Lalinsk=C3=BD?= <lukas@oxygene.sk>
Date: Sun, 22 Nov 2015 13:49:33 +0100
Subject: [PATCH 3176/6219] [youtube] Fall back to the original regex for
 ytplayer.config

---
 youtube_dl/extractor/youtube.py | 39 ++++++++++++++++++++++++++-------
 1 file changed, 31 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 21731188a..7e74d2368 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -674,7 +674,23 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         {
             'url': 'http://vid.plus/FlRa-iH7PGw',
             'only_matching': True,
-        }
+        },
+        {
+            # Title with JS-like syntax "};"
+            'url': 'https://www.youtube.com/watch?v=lsguqyKfVQg',
+            'info_dict': {
+                'id': 'lsguqyKfVQg',
+                'ext': 'mp4',
+                'title': '{dark walk}; Loki/AC/Dishonored; collab w/Elflover21',
+                'description': 'md5:8085699c11dc3f597ce0410b0dcbb34a',
+                'upload_date': '20151119',
+                'uploader_id': 'IronSoulElf',
+                'uploader': 'IronSoulElf',
+            },
+            'params': {
+                'skip_download': True,
+            },
+        },
     ]
 
     def __init__(self, *args, **kwargs):
@@ -858,16 +874,25 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             return {}
         return sub_lang_list
 
+    def _get_ytplayer_config(self, webpage):
+        patterns = [
+            r';ytplayer\.config\s*=\s*({.*?});ytplayer',
+            r';ytplayer\.config\s*=\s*({.*?});',
+        ]
+        for pattern in patterns:
+            config = self._search_regex(pattern, webpage, 'ytconfig.player', default=None)
+            if config is not None:
+                return json.loads(uppercase_escape(config))
+
     def _get_automatic_captions(self, video_id, webpage):
         """We need the webpage for getting the captions url, pass it as an
            argument to speed up the process."""
         self.to_screen('%s: Looking for automatic captions' % video_id)
-        mobj = re.search(r';ytplayer.config = ({.*?});', webpage)
+        player_config = self._get_ytplayer_config(webpage)
         err_msg = 'Couldn\'t find automatic captions for %s' % video_id
-        if mobj is None:
+        if player_config is None:
             self._downloader.report_warning(err_msg)
             return {}
-        player_config = json.loads(mobj.group(1))
         try:
             args = player_config['args']
             caption_url = args['ttsurl']
@@ -1074,10 +1099,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             age_gate = False
             video_info = None
             # Try looking directly into the video webpage
-            mobj = re.search(r';ytplayer\.config\s*=\s*({.*?});ytplayer', video_webpage)
-            if mobj:
-                json_code = uppercase_escape(mobj.group(1))
-                ytplayer_config = json.loads(json_code)
+            ytplayer_config = self._get_ytplayer_config(video_webpage)
+            if ytplayer_config is not None:
                 args = ytplayer_config['args']
                 if args.get('url_encoded_fmt_stream_map'):
                     # Convert to the same format returned by compat_parse_qs

From b41631c4e6e56afb2427513c84df1b13681cf4c8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Lalinsk=C3=BD?= <lukas@oxygene.sk>
Date: Sun, 22 Nov 2015 13:53:26 +0100
Subject: [PATCH 3177/6219] [youtube] Send the list of patterns directly to
 _search_regex

---
 youtube_dl/extractor/youtube.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 7e74d2368..247769067 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -879,10 +879,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             r';ytplayer\.config\s*=\s*({.*?});ytplayer',
             r';ytplayer\.config\s*=\s*({.*?});',
         ]
-        for pattern in patterns:
-            config = self._search_regex(pattern, webpage, 'ytconfig.player', default=None)
-            if config is not None:
-                return json.loads(uppercase_escape(config))
+        config = self._search_regex(patterns, webpage, 'ytconfig.player', default=None)
+        if config is not None:
+            return json.loads(uppercase_escape(config))
 
     def _get_automatic_captions(self, video_id, webpage):
         """We need the webpage for getting the captions url, pass it as an

From 02f0da20b00fa0efee6abcd1842c183b3a4cc36c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 23 Nov 2015 03:08:38 +0600
Subject: [PATCH 3178/6219] [pluralsight] Add support for alternative webpage
 layout (Closes #7607)

---
 youtube_dl/extractor/pluralsight.py | 22 ++++++++++++++++------
 1 file changed, 16 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index a11cebaf1..792316db8 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -104,19 +104,29 @@ class PluralsightIE(PluralsightBaseIE):
 
         webpage = self._download_webpage(url, display_id)
 
-        collection = self._parse_json(
-            self._search_regex(
-                r'moduleCollection\s*:\s*new\s+ModuleCollection\((\[.+?\])\s*,\s*\$rootScope\)',
-                webpage, 'modules'),
-            display_id)
+        modules = self._search_regex(
+            r'moduleCollection\s*:\s*new\s+ModuleCollection\((\[.+?\])\s*,\s*\$rootScope\)',
+            webpage, 'modules', default=None)
+
+        if modules:
+            collection = self._parse_json(modules, display_id)
+        else:
+            # Webpage may be served in different layout (see
+            # https://github.com/rg3/youtube-dl/issues/7607)
+            collection = self._parse_json(
+                self._search_regex(
+                    r'var\s+initialState\s*=\s*({.+?});\n', webpage, 'initial state'),
+                display_id)['course']['modules']
 
         module, clip = None, None
 
         for module_ in collection:
-            if module_.get('moduleName') == name:
+            if name in (module_.get('moduleName'), module_.get('name')):
                 module = module_
                 for clip_ in module_.get('clips', []):
                     clip_index = clip_.get('clipIndex')
+                    if clip_index is None:
+                        clip_index = clip_.get('index')
                     if clip_index is None:
                         continue
                     if compat_str(clip_index) == clip_id:

From a72778d364022612ba88bdfd9affef0d7b0ca864 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 23 Nov 2015 21:00:06 +0600
Subject: [PATCH 3179/6219] [youtube] Improve ytplayer.config extraction

---
 youtube_dl/extractor/youtube.py | 26 ++++++++++++++------------
 1 file changed, 14 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 1580c54fe..052f6922a 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -891,22 +891,24 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             return {}
         return sub_lang_list
 
-    def _get_ytplayer_config(self, webpage):
-        patterns = [
-            r';ytplayer\.config\s*=\s*({.*?});ytplayer',
-            r';ytplayer\.config\s*=\s*({.*?});',
-        ]
-        config = self._search_regex(patterns, webpage, 'ytconfig.player', default=None)
-        if config is not None:
-            return json.loads(uppercase_escape(config))
+    def _get_ytplayer_config(self, video_id, webpage):
+        patterns = (
+            r';ytplayer\.config\s*=\s*({.+?});ytplayer',
+            r';ytplayer\.config\s*=\s*({.+?});',
+        )
+        config = self._search_regex(
+            patterns, webpage, 'ytplayer.config', default=None)
+        if config:
+            return self._parse_json(
+                uppercase_escape(config), video_id, fatal=False)
 
     def _get_automatic_captions(self, video_id, webpage):
         """We need the webpage for getting the captions url, pass it as an
            argument to speed up the process."""
         self.to_screen('%s: Looking for automatic captions' % video_id)
-        player_config = self._get_ytplayer_config(webpage)
+        player_config = self._get_ytplayer_config(video_id, webpage)
         err_msg = 'Couldn\'t find automatic captions for %s' % video_id
-        if player_config is None:
+        if not player_config:
             self._downloader.report_warning(err_msg)
             return {}
         try:
@@ -1115,8 +1117,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             age_gate = False
             video_info = None
             # Try looking directly into the video webpage
-            ytplayer_config = self._get_ytplayer_config(video_webpage)
-            if ytplayer_config is not None:
+            ytplayer_config = self._get_ytplayer_config(video_id, video_webpage)
+            if ytplayer_config:
                 args = ytplayer_config['args']
                 if args.get('url_encoded_fmt_stream_map'):
                     # Convert to the same format returned by compat_parse_qs

From 61f92af1cfacb9a5a6e368d0093fb71dbac0af6b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 23 Nov 2015 21:02:37 +0600
Subject: [PATCH 3180/6219] [youtube] Add test with '};' in tags

---
 youtube_dl/extractor/youtube.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 052f6922a..824335d0a 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -693,7 +693,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             'only_matching': True,
         },
         {
-            # Title with JS-like syntax "};"
+            # Title with JS-like syntax "};" (see https://github.com/rg3/youtube-dl/issues/7468)
             'url': 'https://www.youtube.com/watch?v=lsguqyKfVQg',
             'info_dict': {
                 'id': 'lsguqyKfVQg',
@@ -708,6 +708,11 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'skip_download': True,
             },
         },
+        {
+            # Tags with '};' (see https://github.com/rg3/youtube-dl/issues/7468)
+            'url': 'https://www.youtube.com/watch?v=Ms7iBXnlUO8',
+            'only_matching': True,
+        },
     ]
 
     def __init__(self, *args, **kwargs):

From 526b3b071632bc3c840ae4dd3579e015f41df6f5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 23 Nov 2015 21:14:03 +0600
Subject: [PATCH 3181/6219] [youtube] Clarify ytplayer.config extraction
 rationale

---
 youtube_dl/extractor/youtube.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 824335d0a..5482aac3b 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -898,6 +898,13 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
 
     def _get_ytplayer_config(self, video_id, webpage):
         patterns = (
+            # User data may contain arbitrary character sequences that may affect
+            # JSON extraction with regex, e.g. when '};' is contained the second
+            # regex won't capture the whole JSON. Yet working around by trying more
+            # concrete regex first keeping in mind proper quoted string handling
+            # to be implemented in future that will replace this workaround (see
+            # https://github.com/rg3/youtube-dl/issues/7468,
+            # https://github.com/rg3/youtube-dl/pull/7599)
             r';ytplayer\.config\s*=\s*({.+?});ytplayer',
             r';ytplayer\.config\s*=\s*({.+?});',
         )

From 94bfcd23b7e2488b5e4bbf076965ab9ed980f1ca Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 23 Nov 2015 21:35:23 +0600
Subject: [PATCH 3182/6219] [youtube] Fix test

---
 youtube_dl/extractor/youtube.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 5482aac3b..e0e345496 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -426,7 +426,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'title': 'Principal Sexually Assaults A Teacher - Episode 117 - 8th June 2012',
                 'description': 'md5:09b78bd971f1e3e289601dfba15ca4f7',
                 'uploader': 'SET India',
-                'uploader_id': 'setindia'
+                'uploader_id': 'setindia',
+                'age_limit': 18,
             }
         },
         {

From 9022726446c659f2bc38556105991e9797e0c8c8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 23 Nov 2015 21:37:21 +0600
Subject: [PATCH 3183/6219] [youtube] Fix test

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index e0e345496..0246050c2 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -564,7 +564,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             'info_dict': {
                 'id': 'lqQg6PlCWgI',
                 'ext': 'mp4',
-                'upload_date': '20120724',
+                'upload_date': '20150827',
                 'uploader_id': 'olympic',
                 'description': 'HO09  - Women -  GER-AUS - Hockey - 31 July 2012 - London 2012 Olympic Games',
                 'uploader': 'Olympics',

From 13a10d5aa336be7c301a6d09eb4e9d7b50f51191 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 20 Nov 2015 03:08:01 +0600
Subject: [PATCH 3184/6219] [compat] Add compat_urllib_request_Request

This is actually not a compatibility routine but rather a workaround for URLs without protocol specified.
The protocol-less URL is treated as HTTP one since it's most probable scenario and it will most likely to
redirect to HTTPS if HTTPS was actually expected. This routine could also be useful for any Request
preprocessing that may be added in future.
---
 youtube_dl/compat.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index a3e85264a..4e3de7f51 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -198,6 +198,14 @@ except ImportError:  # Python < 3.4
 
             return compat_urllib_response.addinfourl(io.BytesIO(data), headers, url)
 
+
+# Prepend protocol-less URLs with `http:` scheme in order to mitigate the number of
+# unwanted failures due to missing protocol
+def compat_urllib_request_Request(url, *args, **kwargs):
+    return compat_urllib_request.Request(
+        'http:%s' % url if url.startswith('//') else url, *args, **kwargs)
+
+
 try:
     compat_basestring = basestring  # Python 2
 except NameError:

From 82d8a8b6e2eadb2c1ac8684ce9876f40925f4e2e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 20 Nov 2015 03:08:34 +0600
Subject: [PATCH 3185/6219] [YoutubeDL] Wrap plain-text URL requests in
 compat_urllib_request_Request

---
 youtube_dl/YoutubeDL.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index fba99af8d..b20837ce2 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -28,6 +28,7 @@ if os.name == 'nt':
     import ctypes
 
 from .compat import (
+    compat_basestring,
     compat_cookiejar,
     compat_expanduser,
     compat_get_terminal_size,
@@ -38,6 +39,7 @@ from .compat import (
     compat_urllib_error,
     compat_urllib_request,
     compat_urllib_request_DataHandler,
+    compat_urllib_request_Request,
 )
 from .utils import (
     ContentTooShortError,
@@ -1871,6 +1873,8 @@ class YoutubeDL(object):
 
     def urlopen(self, req):
         """ Start an HTTP download """
+        if isinstance(req, compat_basestring):
+            req = compat_urllib_request_Request(req)
         return self._opener.open(req, timeout=self._socket_timeout)
 
     def print_debug_header(self):

From e4c4bcf36f4e0bb575526701f852152f4fd976c4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 20 Nov 2015 03:12:54 +0600
Subject: [PATCH 3186/6219] [vimeo] Use compat_urllib_request_Request

---
 youtube_dl/extractor/vimeo.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 057c72f39..d1daf9672 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -8,7 +8,7 @@ import itertools
 from .common import InfoExtractor
 from ..compat import (
     compat_HTTPError,
-    compat_urllib_request,
+    compat_urllib_request_Request,
     compat_urlparse,
 )
 from ..utils import (
@@ -47,7 +47,7 @@ class VimeoBaseInfoExtractor(InfoExtractor):
             'service': 'vimeo',
             'token': token,
         }))
-        login_request = compat_urllib_request.Request(self._LOGIN_URL, data)
+        login_request = compat_urllib_request_Request(self._LOGIN_URL, data)
         login_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
         login_request.add_header('Referer', self._LOGIN_URL)
         self._set_vimeo_cookie('vuid', vuid)
@@ -222,7 +222,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
         if url.startswith('http://'):
             # vimeo only supports https now, but the user can give an http url
             url = url.replace('http://', 'https://')
-        password_request = compat_urllib_request.Request(url + '/password', data)
+        password_request = compat_urllib_request_Request(url + '/password', data)
         password_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
         password_request.add_header('Referer', url)
         self._set_vimeo_cookie('vuid', vuid)
@@ -236,7 +236,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
             raise ExtractorError('This video is protected by a password, use the --video-password option')
         data = urlencode_postdata(encode_dict({'password': password}))
         pass_url = url + '/check-password'
-        password_request = compat_urllib_request.Request(pass_url, data)
+        password_request = compat_urllib_request_Request(pass_url, data)
         password_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
         return self._download_json(
             password_request, video_id,
@@ -265,7 +265,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
             url = 'https://vimeo.com/' + video_id
 
         # Retrieve video webpage to extract further information
-        request = compat_urllib_request.Request(url, None, headers)
+        request = compat_urllib_request_Request(url, None, headers)
         try:
             webpage = self._download_webpage(request, video_id)
         except ExtractorError as ee:
@@ -481,7 +481,7 @@ class VimeoChannelIE(VimeoBaseInfoExtractor):
         password_path = self._search_regex(
             r'action="([^"]+)"', login_form, 'password URL')
         password_url = compat_urlparse.urljoin(page_url, password_path)
-        password_request = compat_urllib_request.Request(password_url, post)
+        password_request = compat_urllib_request_Request(password_url, post)
         password_request.add_header('Content-type', 'application/x-www-form-urlencoded')
         self._set_vimeo_cookie('vuid', vuid)
         self._set_vimeo_cookie('xsrft', token)
@@ -640,7 +640,7 @@ class VimeoWatchLaterIE(VimeoChannelIE):
 
     def _page_url(self, base_url, pagenum):
         url = '%s/page:%d/' % (base_url, pagenum)
-        request = compat_urllib_request.Request(url)
+        request = compat_urllib_request_Request(url)
         # Set the header to get a partial html page with the ids,
         # the normal page doesn't contain them.
         request.add_header('X-Requested-With', 'XMLHttpRequest')

From 67dda51722f1ce12b956782d43047b3fff390115 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 20 Nov 2015 20:33:49 +0600
Subject: [PATCH 3187/6219] Rename compat_urllib_request_Request to
 sanitized_Request and move to utils

---
 youtube_dl/YoutubeDL.py       |  4 ++--
 youtube_dl/compat.py          |  8 --------
 youtube_dl/extractor/vimeo.py | 14 +++++++-------
 youtube_dl/utils.py           |  7 +++++++
 4 files changed, 16 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index b20837ce2..eedab37a7 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -39,7 +39,6 @@ from .compat import (
     compat_urllib_error,
     compat_urllib_request,
     compat_urllib_request_DataHandler,
-    compat_urllib_request_Request,
 )
 from .utils import (
     ContentTooShortError,
@@ -65,6 +64,7 @@ from .utils import (
     SameFileError,
     sanitize_filename,
     sanitize_path,
+    sanitized_Request,
     std_headers,
     subtitles_filename,
     UnavailableVideoError,
@@ -1874,7 +1874,7 @@ class YoutubeDL(object):
     def urlopen(self, req):
         """ Start an HTTP download """
         if isinstance(req, compat_basestring):
-            req = compat_urllib_request_Request(req)
+            req = sanitized_Request(req)
         return self._opener.open(req, timeout=self._socket_timeout)
 
     def print_debug_header(self):
diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 4e3de7f51..a3e85264a 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -198,14 +198,6 @@ except ImportError:  # Python < 3.4
 
             return compat_urllib_response.addinfourl(io.BytesIO(data), headers, url)
 
-
-# Prepend protocol-less URLs with `http:` scheme in order to mitigate the number of
-# unwanted failures due to missing protocol
-def compat_urllib_request_Request(url, *args, **kwargs):
-    return compat_urllib_request.Request(
-        'http:%s' % url if url.startswith('//') else url, *args, **kwargs)
-
-
 try:
     compat_basestring = basestring  # Python 2
 except NameError:
diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index d1daf9672..f392ccf1c 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -8,7 +8,6 @@ import itertools
 from .common import InfoExtractor
 from ..compat import (
     compat_HTTPError,
-    compat_urllib_request_Request,
     compat_urlparse,
 )
 from ..utils import (
@@ -17,6 +16,7 @@ from ..utils import (
     InAdvancePagedList,
     int_or_none,
     RegexNotFoundError,
+    sanitized_Request,
     smuggle_url,
     std_headers,
     unified_strdate,
@@ -47,7 +47,7 @@ class VimeoBaseInfoExtractor(InfoExtractor):
             'service': 'vimeo',
             'token': token,
         }))
-        login_request = compat_urllib_request_Request(self._LOGIN_URL, data)
+        login_request = sanitized_Request(self._LOGIN_URL, data)
         login_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
         login_request.add_header('Referer', self._LOGIN_URL)
         self._set_vimeo_cookie('vuid', vuid)
@@ -222,7 +222,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
         if url.startswith('http://'):
             # vimeo only supports https now, but the user can give an http url
             url = url.replace('http://', 'https://')
-        password_request = compat_urllib_request_Request(url + '/password', data)
+        password_request = sanitized_Request(url + '/password', data)
         password_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
         password_request.add_header('Referer', url)
         self._set_vimeo_cookie('vuid', vuid)
@@ -236,7 +236,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
             raise ExtractorError('This video is protected by a password, use the --video-password option')
         data = urlencode_postdata(encode_dict({'password': password}))
         pass_url = url + '/check-password'
-        password_request = compat_urllib_request_Request(pass_url, data)
+        password_request = sanitized_Request(pass_url, data)
         password_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
         return self._download_json(
             password_request, video_id,
@@ -265,7 +265,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
             url = 'https://vimeo.com/' + video_id
 
         # Retrieve video webpage to extract further information
-        request = compat_urllib_request_Request(url, None, headers)
+        request = sanitized_Request(url, None, headers)
         try:
             webpage = self._download_webpage(request, video_id)
         except ExtractorError as ee:
@@ -481,7 +481,7 @@ class VimeoChannelIE(VimeoBaseInfoExtractor):
         password_path = self._search_regex(
             r'action="([^"]+)"', login_form, 'password URL')
         password_url = compat_urlparse.urljoin(page_url, password_path)
-        password_request = compat_urllib_request_Request(password_url, post)
+        password_request = sanitized_Request(password_url, post)
         password_request.add_header('Content-type', 'application/x-www-form-urlencoded')
         self._set_vimeo_cookie('vuid', vuid)
         self._set_vimeo_cookie('xsrft', token)
@@ -640,7 +640,7 @@ class VimeoWatchLaterIE(VimeoChannelIE):
 
     def _page_url(self, base_url, pagenum):
         url = '%s/page:%d/' % (base_url, pagenum)
-        request = compat_urllib_request_Request(url)
+        request = sanitized_Request(url)
         # Set the header to get a partial html page with the ids,
         # the normal page doesn't contain them.
         request.add_header('X-Requested-With', 'XMLHttpRequest')
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index c0325f054..d7b737e21 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -373,6 +373,13 @@ def sanitize_path(s):
     return os.path.join(*sanitized_path)
 
 
+# Prepend protocol-less URLs with `http:` scheme in order to mitigate the number of
+# unwanted failures due to missing protocol
+def sanitized_Request(url, *args, **kwargs):
+    return compat_urllib_request.Request(
+        'http:%s' % url if url.startswith('//') else url, *args, **kwargs)
+
+
 def orderedSet(iterable):
     """ Remove all duplicates from the input iterable """
     res = []

From 5c2266df4b9aeb7881ed8c026a038e2a25e43734 Mon Sep 17 00:00:00 2001
From: Sergey M? <dstftw@gmail.com>
Date: Sat, 21 Nov 2015 22:18:17 +0600
Subject: [PATCH 3188/6219] Switch codebase to use sanitized_Request instead of
 compat_urllib_request.Request

[downloader/dash] Use sanitized_Request

[downloader/http] Use sanitized_Request

[atresplayer] Use sanitized_Request

[bambuser] Use sanitized_Request

[bliptv] Use sanitized_Request

[brightcove] Use sanitized_Request

[cbs] Use sanitized_Request

[ceskatelevize] Use sanitized_Request

[collegerama] Use sanitized_Request

[extractor/common] Use sanitized_Request

[crunchyroll] Use sanitized_Request

[dailymotion] Use sanitized_Request

[dcn] Use sanitized_Request

[dramafever] Use sanitized_Request

[dumpert] Use sanitized_Request

[eitb] Use sanitized_Request

[escapist] Use sanitized_Request

[everyonesmixtape] Use sanitized_Request

[extremetube] Use sanitized_Request

[facebook] Use sanitized_Request

[fc2] Use sanitized_Request

[flickr] Use sanitized_Request

[4tube] Use sanitized_Request

[gdcvault] Use sanitized_Request

[extractor/generic] Use sanitized_Request

[hearthisat] Use sanitized_Request

[hotnewhiphop] Use sanitized_Request

[hypem] Use sanitized_Request

[iprima] Use sanitized_Request

[ivi] Use sanitized_Request

[keezmovies] Use sanitized_Request

[letv] Use sanitized_Request

[lynda] Use sanitized_Request

[metacafe] Use sanitized_Request

[minhateca] Use sanitized_Request

[miomio] Use sanitized_Request

[meovideo] Use sanitized_Request

[mofosex] Use sanitized_Request

[moniker] Use sanitized_Request

[mooshare] Use sanitized_Request

[movieclips] Use sanitized_Request

[mtv] Use sanitized_Request

[myvideo] Use sanitized_Request

[neteasemusic] Use sanitized_Request

[nfb] Use sanitized_Request

[niconico] Use sanitized_Request

[noco] Use sanitized_Request

[nosvideo] Use sanitized_Request

[novamov] Use sanitized_Request

[nowness] Use sanitized_Request

[nuvid] Use sanitized_Request

[played] Use sanitized_Request

[pluralsight] Use sanitized_Request

[pornhub] Use sanitized_Request

[pornotube] Use sanitized_Request

[primesharetv] Use sanitized_Request

[promptfile] Use sanitized_Request

[qqmusic] Use sanitized_Request

[rtve] Use sanitized_Request

[safari] Use sanitized_Request

[sandia] Use sanitized_Request

[shared] Use sanitized_Request

[sharesix] Use sanitized_Request

[sina] Use sanitized_Request

[smotri] Use sanitized_Request

[sohu] Use sanitized_Request

[spankwire] Use sanitized_Request

[sportdeutschland] Use sanitized_Request

[streamcloud] Use sanitized_Request

[streamcz] Use sanitized_Request

[tapely] Use sanitized_Request

[tube8] Use sanitized_Request

[tubitv] Use sanitized_Request

[twitch] Use sanitized_Request

[twitter] Use sanitized_Request

[udemy] Use sanitized_Request

[vbox7] Use sanitized_Request

[veoh] Use sanitized_Request

[vessel] Use sanitized_Request

[vevo] Use sanitized_Request

[viddler] Use sanitized_Request

[videomega] Use sanitized_Request

[viewvster] Use sanitized_Request

[viki] Use sanitized_Request

[vk] Use sanitized_Request

[vodlocker] Use sanitized_Request

[voicerepublic] Use sanitized_Request

[wistia] Use sanitized_Request

[xfileshare] Use sanitized_Request

[xtube] Use sanitized_Request

[xvideos] Use sanitized_Request

[yandexmusic] Use sanitized_Request

[youku] Use sanitized_Request

[youporn] Use sanitized_Request

[youtube] Use sanitized_Request

[patreon] Use sanitized_Request

[extractor/common] Remove unused import

[nfb] PEP 8
---
 youtube_dl/YoutubeDL.py                  |  2 +-
 youtube_dl/downloader/dash.py            |  4 ++--
 youtube_dl/downloader/http.py            | 10 ++++------
 youtube_dl/extractor/atresplayer.py      |  6 +++---
 youtube_dl/extractor/bambuser.py         |  6 +++---
 youtube_dl/extractor/bliptv.py           |  8 +++-----
 youtube_dl/extractor/brightcove.py       |  6 +++---
 youtube_dl/extractor/cbs.py              |  8 +++++---
 youtube_dl/extractor/ceskatelevize.py    |  6 +++---
 youtube_dl/extractor/collegerama.py      |  4 ++--
 youtube_dl/extractor/common.py           |  4 ++--
 youtube_dl/extractor/crunchyroll.py      |  9 +++++----
 youtube_dl/extractor/dailymotion.py      |  8 +++-----
 youtube_dl/extractor/dcn.py              |  8 +++-----
 youtube_dl/extractor/dramafever.py       |  4 ++--
 youtube_dl/extractor/dumpert.py          |  8 +++++---
 youtube_dl/extractor/eitb.py             |  4 ++--
 youtube_dl/extractor/escapist.py         |  5 ++---
 youtube_dl/extractor/everyonesmixtape.py |  8 +++-----
 youtube_dl/extractor/extremetube.py      |  4 ++--
 youtube_dl/extractor/facebook.py         |  8 ++++----
 youtube_dl/extractor/fc2.py              |  5 +++--
 youtube_dl/extractor/flickr.py           |  4 ++--
 youtube_dl/extractor/fourtube.py         |  6 ++----
 youtube_dl/extractor/gdcvault.py         |  8 +++-----
 youtube_dl/extractor/generic.py          |  6 +++---
 youtube_dl/extractor/hearthisat.py       |  8 +++-----
 youtube_dl/extractor/hotnewhiphop.py     |  8 +++-----
 youtube_dl/extractor/hypem.py            | 10 ++++------
 youtube_dl/extractor/iprima.py           |  6 ++----
 youtube_dl/extractor/ivi.py              |  6 ++----
 youtube_dl/extractor/keezmovies.py       |  8 +++-----
 youtube_dl/extractor/letv.py             |  4 ++--
 youtube_dl/extractor/lynda.py            |  6 +++---
 youtube_dl/extractor/metacafe.py         |  6 +++---
 youtube_dl/extractor/minhateca.py        |  8 +++-----
 youtube_dl/extractor/miomio.py           |  4 ++--
 youtube_dl/extractor/moevideo.py         |  8 +++-----
 youtube_dl/extractor/mofosex.py          |  4 ++--
 youtube_dl/extractor/moniker.py          | 10 ++++------
 youtube_dl/extractor/mooshare.py         |  8 +++-----
 youtube_dl/extractor/movieclips.py       |  6 ++----
 youtube_dl/extractor/mtv.py              |  4 ++--
 youtube_dl/extractor/myvideo.py          |  4 ++--
 youtube_dl/extractor/neteasemusic.py     |  4 ++--
 youtube_dl/extractor/nfb.py              | 11 +++++------
 youtube_dl/extractor/niconico.py         |  6 +++---
 youtube_dl/extractor/noco.py             |  4 ++--
 youtube_dl/extractor/nosvideo.py         |  6 ++----
 youtube_dl/extractor/novamov.py          |  8 +++-----
 youtube_dl/extractor/nowness.py          | 10 +++++-----
 youtube_dl/extractor/nuvid.py            |  6 ++----
 youtube_dl/extractor/patreon.py          |  6 ++----
 youtube_dl/extractor/played.py           |  8 +++-----
 youtube_dl/extractor/pluralsight.py      |  6 +++---
 youtube_dl/extractor/pornhub.py          |  4 ++--
 youtube_dl/extractor/pornotube.py        | 10 ++++------
 youtube_dl/extractor/primesharetv.py     | 10 +++++-----
 youtube_dl/extractor/promptfile.py       |  8 +++-----
 youtube_dl/extractor/qqmusic.py          |  4 ++--
 youtube_dl/extractor/rtve.py             |  4 ++--
 youtube_dl/extractor/safari.py           |  8 +++-----
 youtube_dl/extractor/sandia.py           |  8 +++-----
 youtube_dl/extractor/shared.py           |  8 +++-----
 youtube_dl/extractor/sharesix.py         |  8 +++-----
 youtube_dl/extractor/sina.py             |  8 +++-----
 youtube_dl/extractor/smotri.py           | 10 ++++------
 youtube_dl/extractor/sohu.py             |  4 ++--
 youtube_dl/extractor/spankwire.py        |  4 ++--
 youtube_dl/extractor/sportdeutschland.py |  6 ++----
 youtube_dl/extractor/streamcloud.py      |  8 +++-----
 youtube_dl/extractor/streamcz.py         |  6 ++----
 youtube_dl/extractor/tapely.py           |  6 ++----
 youtube_dl/extractor/tube8.py            |  8 +++-----
 youtube_dl/extractor/tubitv.py           |  8 +++-----
 youtube_dl/extractor/twitch.py           |  6 +++---
 youtube_dl/extractor/twitter.py          |  4 ++--
 youtube_dl/extractor/udemy.py            |  5 +++--
 youtube_dl/extractor/vbox7.py            |  4 ++--
 youtube_dl/extractor/veoh.py             |  6 ++----
 youtube_dl/extractor/vessel.py           |  4 ++--
 youtube_dl/extractor/vevo.py             |  8 +++-----
 youtube_dl/extractor/viddler.py          |  6 ++----
 youtube_dl/extractor/videomega.py        |  4 ++--
 youtube_dl/extractor/viewster.py         |  4 ++--
 youtube_dl/extractor/viki.py             |  6 +++---
 youtube_dl/extractor/vk.py               |  4 ++--
 youtube_dl/extractor/vodlocker.py        |  8 +++-----
 youtube_dl/extractor/voicerepublic.py    |  8 +++-----
 youtube_dl/extractor/wistia.py           |  8 +++++---
 youtube_dl/extractor/xfileshare.py       |  8 +++-----
 youtube_dl/extractor/xtube.py            |  8 +++-----
 youtube_dl/extractor/xvideos.py          |  8 +++-----
 youtube_dl/extractor/yandexmusic.py      |  4 ++--
 youtube_dl/extractor/youku.py            |  9 +++++----
 youtube_dl/extractor/youporn.py          |  4 ++--
 youtube_dl/extractor/youtube.py          |  6 +++---
 97 files changed, 271 insertions(+), 353 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index eedab37a7..9a8c7da05 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1189,7 +1189,7 @@ class YoutubeDL(object):
         return res
 
     def _calc_cookies(self, info_dict):
-        pr = compat_urllib_request.Request(info_dict['url'])
+        pr = sanitized_Request(info_dict['url'])
         self.cookiejar.add_cookie_header(pr)
         return pr.get_header('Cookie')
 
diff --git a/youtube_dl/downloader/dash.py b/youtube_dl/downloader/dash.py
index 8b6fa2753..535f2a7fc 100644
--- a/youtube_dl/downloader/dash.py
+++ b/youtube_dl/downloader/dash.py
@@ -3,7 +3,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import FileDownloader
-from ..compat import compat_urllib_request
+from ..utils import sanitized_Request
 
 
 class DashSegmentsFD(FileDownloader):
@@ -22,7 +22,7 @@ class DashSegmentsFD(FileDownloader):
 
         def append_url_to_file(outf, target_url, target_name, remaining_bytes=None):
             self.to_screen('[DashSegments] %s: Downloading %s' % (info_dict['id'], target_name))
-            req = compat_urllib_request.Request(target_url)
+            req = sanitized_Request(target_url)
             if remaining_bytes is not None:
                 req.add_header('Range', 'bytes=0-%d' % (remaining_bytes - 1))
 
diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index a29f5cf31..56840e026 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -7,14 +7,12 @@ import time
 import re
 
 from .common import FileDownloader
-from ..compat import (
-    compat_urllib_request,
-    compat_urllib_error,
-)
+from ..compat import compat_urllib_error
 from ..utils import (
     ContentTooShortError,
     encodeFilename,
     sanitize_open,
+    sanitized_Request,
 )
 
 
@@ -29,8 +27,8 @@ class HttpFD(FileDownloader):
         add_headers = info_dict.get('http_headers')
         if add_headers:
             headers.update(add_headers)
-        basic_request = compat_urllib_request.Request(url, None, headers)
-        request = compat_urllib_request.Request(url, None, headers)
+        basic_request = sanitized_Request(url, None, headers)
+        request = sanitized_Request(url, None, headers)
 
         is_test = self.params.get('test', False)
 
diff --git a/youtube_dl/extractor/atresplayer.py b/youtube_dl/extractor/atresplayer.py
index 29f8795d3..50e47ba0a 100644
--- a/youtube_dl/extractor/atresplayer.py
+++ b/youtube_dl/extractor/atresplayer.py
@@ -7,11 +7,11 @@ from .common import InfoExtractor
 from ..compat import (
     compat_str,
     compat_urllib_parse,
-    compat_urllib_request,
 )
 from ..utils import (
     int_or_none,
     float_or_none,
+    sanitized_Request,
     xpath_text,
     ExtractorError,
 )
@@ -63,7 +63,7 @@ class AtresPlayerIE(InfoExtractor):
             'j_password': password,
         }
 
-        request = compat_urllib_request.Request(
+        request = sanitized_Request(
             self._LOGIN_URL, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
         request.add_header('Content-Type', 'application/x-www-form-urlencoded')
         response = self._download_webpage(
@@ -94,7 +94,7 @@ class AtresPlayerIE(InfoExtractor):
 
         formats = []
         for fmt in ['windows', 'android_tablet']:
-            request = compat_urllib_request.Request(
+            request = sanitized_Request(
                 self._URL_VIDEO_TEMPLATE.format(fmt, episode_id, timestamp_shifted, token))
             request.add_header('User-Agent', self._USER_AGENT)
 
diff --git a/youtube_dl/extractor/bambuser.py b/youtube_dl/extractor/bambuser.py
index 8dff1d6e3..da986e063 100644
--- a/youtube_dl/extractor/bambuser.py
+++ b/youtube_dl/extractor/bambuser.py
@@ -6,13 +6,13 @@ import itertools
 from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse,
-    compat_urllib_request,
     compat_str,
 )
 from ..utils import (
     ExtractorError,
     int_or_none,
     float_or_none,
+    sanitized_Request,
 )
 
 
@@ -57,7 +57,7 @@ class BambuserIE(InfoExtractor):
             'pass': password,
         }
 
-        request = compat_urllib_request.Request(
+        request = sanitized_Request(
             self._LOGIN_URL, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
         request.add_header('Referer', self._LOGIN_URL)
         response = self._download_webpage(
@@ -126,7 +126,7 @@ class BambuserChannelIE(InfoExtractor):
                 '&sort=created&access_mode=0%2C1%2C2&limit={count}'
                 '&method=broadcast&format=json&vid_older_than={last}'
             ).format(user=user, count=self._STEP, last=last_id)
-            req = compat_urllib_request.Request(req_url)
+            req = sanitized_Request(req_url)
             # Without setting this header, we wouldn't get any result
             req.add_header('Referer', 'http://bambuser.com/channel/%s' % user)
             data = self._download_json(
diff --git a/youtube_dl/extractor/bliptv.py b/youtube_dl/extractor/bliptv.py
index c3296283d..35375f7b1 100644
--- a/youtube_dl/extractor/bliptv.py
+++ b/youtube_dl/extractor/bliptv.py
@@ -4,14 +4,12 @@ import re
 
 from .common import InfoExtractor
 
-from ..compat import (
-    compat_urllib_request,
-    compat_urlparse,
-)
+from ..compat import compat_urlparse
 from ..utils import (
     clean_html,
     int_or_none,
     parse_iso8601,
+    sanitized_Request,
     unescapeHTML,
     xpath_text,
     xpath_with_ns,
@@ -219,7 +217,7 @@ class BlipTVIE(InfoExtractor):
         for lang, url in subtitles_urls.items():
             # For some weird reason, blip.tv serves a video instead of subtitles
             # when we request with a common UA
-            req = compat_urllib_request.Request(url)
+            req = sanitized_Request(url)
             req.add_header('User-Agent', 'youtube-dl')
             subtitles[lang] = [{
                 # The extension is 'srt' but it's actually an 'ass' file
diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 14ee05f21..f5ebae1e6 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -11,7 +11,6 @@ from ..compat import (
     compat_str,
     compat_urllib_parse,
     compat_urllib_parse_urlparse,
-    compat_urllib_request,
     compat_urlparse,
     compat_xml_parse_error,
 )
@@ -24,6 +23,7 @@ from ..utils import (
     js_to_json,
     int_or_none,
     parse_iso8601,
+    sanitized_Request,
     unescapeHTML,
     unsmuggle_url,
 )
@@ -250,7 +250,7 @@ class BrightcoveLegacyIE(InfoExtractor):
 
     def _get_video_info(self, video_id, query_str, query, referer=None):
         request_url = self._FEDERATED_URL_TEMPLATE % query_str
-        req = compat_urllib_request.Request(request_url)
+        req = sanitized_Request(request_url)
         linkBase = query.get('linkBaseURL')
         if linkBase is not None:
             referer = linkBase[0]
@@ -443,7 +443,7 @@ class BrightcoveNewIE(InfoExtractor):
                 r'policyKey\s*:\s*(["\'])(?P<pk>.+?)\1',
                 webpage, 'policy key', group='pk')
 
-        req = compat_urllib_request.Request(
+        req = sanitized_Request(
             'https://edge.api.brightcove.com/playback/v1/accounts/%s/videos/%s'
             % (account_id, video_id),
             headers={'Accept': 'application/json;pk=%s' % policy_key})
diff --git a/youtube_dl/extractor/cbs.py b/youtube_dl/extractor/cbs.py
index 43f05d278..40d07ab18 100644
--- a/youtube_dl/extractor/cbs.py
+++ b/youtube_dl/extractor/cbs.py
@@ -1,8 +1,10 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_request
-from ..utils import smuggle_url
+from ..utils import (
+    sanitized_Request,
+    smuggle_url,
+)
 
 
 class CBSIE(InfoExtractor):
@@ -48,7 +50,7 @@ class CBSIE(InfoExtractor):
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
-        request = compat_urllib_request.Request(url)
+        request = sanitized_Request(url)
         # Android UA is served with higher quality (720p) streams (see
         # https://github.com/rg3/youtube-dl/issues/7490)
         request.add_header('User-Agent', 'Mozilla/5.0 (Linux; Android 4.4; Nexus 5)')
diff --git a/youtube_dl/extractor/ceskatelevize.py b/youtube_dl/extractor/ceskatelevize.py
index e857e66f4..6f7b2a70d 100644
--- a/youtube_dl/extractor/ceskatelevize.py
+++ b/youtube_dl/extractor/ceskatelevize.py
@@ -5,7 +5,6 @@ import re
 
 from .common import InfoExtractor
 from ..compat import (
-    compat_urllib_request,
     compat_urllib_parse,
     compat_urllib_parse_unquote,
     compat_urllib_parse_urlparse,
@@ -13,6 +12,7 @@ from ..compat import (
 from ..utils import (
     ExtractorError,
     float_or_none,
+    sanitized_Request,
 )
 
 
@@ -100,7 +100,7 @@ class CeskaTelevizeIE(InfoExtractor):
             'requestSource': 'iVysilani',
         }
 
-        req = compat_urllib_request.Request(
+        req = sanitized_Request(
             'http://www.ceskatelevize.cz/ivysilani/ajax/get-client-playlist',
             data=compat_urllib_parse.urlencode(data))
 
@@ -115,7 +115,7 @@ class CeskaTelevizeIE(InfoExtractor):
         if playlist_url == 'error_region':
             raise ExtractorError(NOT_AVAILABLE_STRING, expected=True)
 
-        req = compat_urllib_request.Request(compat_urllib_parse_unquote(playlist_url))
+        req = sanitized_Request(compat_urllib_parse_unquote(playlist_url))
         req.add_header('Referer', url)
 
         playlist_title = self._og_search_title(webpage)
diff --git a/youtube_dl/extractor/collegerama.py b/youtube_dl/extractor/collegerama.py
index fedd48490..40667a0f1 100644
--- a/youtube_dl/extractor/collegerama.py
+++ b/youtube_dl/extractor/collegerama.py
@@ -3,10 +3,10 @@ from __future__ import unicode_literals
 import json
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_request
 from ..utils import (
     float_or_none,
     int_or_none,
+    sanitized_Request,
 )
 
 
@@ -52,7 +52,7 @@ class CollegeRamaIE(InfoExtractor):
             }
         }
 
-        request = compat_urllib_request.Request(
+        request = sanitized_Request(
             'http://collegerama.tudelft.nl/Mediasite/PlayerService/PlayerService.svc/json/GetPlayerOptions',
             json.dumps(player_options_request))
         request.add_header('Content-Type', 'application/json')
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 71bdcad5a..eb9bfa3d1 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -19,7 +19,6 @@ from ..compat import (
     compat_urllib_error,
     compat_urllib_parse,
     compat_urllib_parse_urlparse,
-    compat_urllib_request,
     compat_urlparse,
     compat_str,
     compat_etree_fromstring,
@@ -37,6 +36,7 @@ from ..utils import (
     int_or_none,
     RegexNotFoundError,
     sanitize_filename,
+    sanitized_Request,
     unescapeHTML,
     unified_strdate,
     url_basename,
@@ -1285,7 +1285,7 @@ class InfoExtractor(object):
 
     def _get_cookies(self, url):
         """ Return a compat_cookies.SimpleCookie with the cookies for the url """
-        req = compat_urllib_request.Request(url)
+        req = sanitized_Request(url)
         self._downloader.cookiejar.add_cookie_header(req)
         return compat_cookies.SimpleCookie(req.get_header('Cookie'))
 
diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 6e5999c72..00d943f77 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -23,6 +23,7 @@ from ..utils import (
     int_or_none,
     lowercase_escape,
     remove_end,
+    sanitized_Request,
     unified_strdate,
     urlencode_postdata,
     xpath_text,
@@ -46,7 +47,7 @@ class CrunchyrollBaseIE(InfoExtractor):
             'name': username,
             'password': password,
         })
-        login_request = compat_urllib_request.Request(login_url, data)
+        login_request = sanitized_Request(login_url, data)
         login_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
         self._download_webpage(login_request, None, False, 'Wrong login info')
 
@@ -55,7 +56,7 @@ class CrunchyrollBaseIE(InfoExtractor):
 
     def _download_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True, tries=1, timeout=5, encoding=None):
         request = (url_or_request if isinstance(url_or_request, compat_urllib_request.Request)
-                   else compat_urllib_request.Request(url_or_request))
+                   else sanitized_Request(url_or_request))
         # Accept-Language must be set explicitly to accept any language to avoid issues
         # similar to https://github.com/rg3/youtube-dl/issues/6797.
         # Along with IP address Crunchyroll uses Accept-Language to guess whether georestriction
@@ -307,7 +308,7 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
             'video_uploader', fatal=False)
 
         playerdata_url = compat_urllib_parse_unquote(self._html_search_regex(r'"config_url":"([^"]+)', webpage, 'playerdata_url'))
-        playerdata_req = compat_urllib_request.Request(playerdata_url)
+        playerdata_req = sanitized_Request(playerdata_url)
         playerdata_req.data = compat_urllib_parse.urlencode({'current_page': webpage_url})
         playerdata_req.add_header('Content-Type', 'application/x-www-form-urlencoded')
         playerdata = self._download_webpage(playerdata_req, video_id, note='Downloading media info')
@@ -319,7 +320,7 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
         for fmt in re.findall(r'showmedia\.([0-9]{3,4})p', webpage):
             stream_quality, stream_format = self._FORMAT_IDS[fmt]
             video_format = fmt + 'p'
-            streamdata_req = compat_urllib_request.Request(
+            streamdata_req = sanitized_Request(
                 'http://www.crunchyroll.com/xml/?req=RpcApiVideoPlayer_GetStandardConfig&media_id=%s&video_format=%s&video_quality=%s'
                 % (stream_id, stream_format, stream_quality),
                 compat_urllib_parse.urlencode({'current_page': url}).encode('utf-8'))
diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index bc7823931..ab7f3aec4 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -7,15 +7,13 @@ import itertools
 
 from .common import InfoExtractor
 
-from ..compat import (
-    compat_str,
-    compat_urllib_request,
-)
+from ..compat import compat_str
 from ..utils import (
     ExtractorError,
     determine_ext,
     int_or_none,
     parse_iso8601,
+    sanitized_Request,
     str_to_int,
     unescapeHTML,
 )
@@ -25,7 +23,7 @@ class DailymotionBaseInfoExtractor(InfoExtractor):
     @staticmethod
     def _build_request(url):
         """Build a request with the family filter disabled"""
-        request = compat_urllib_request.Request(url)
+        request = sanitized_Request(url)
         request.add_header('Cookie', 'family_filter=off; ff=off')
         return request
 
diff --git a/youtube_dl/extractor/dcn.py b/youtube_dl/extractor/dcn.py
index 6f2fea5ff..9737cff14 100644
--- a/youtube_dl/extractor/dcn.py
+++ b/youtube_dl/extractor/dcn.py
@@ -2,13 +2,11 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse,
-    compat_urllib_request,
-)
+from ..compat import compat_urllib_parse
 from ..utils import (
     int_or_none,
     parse_iso8601,
+    sanitized_Request,
 )
 
 
@@ -36,7 +34,7 @@ class DCNIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        request = compat_urllib_request.Request(
+        request = sanitized_Request(
             'http://admin.mangomolo.com/analytics/index.php/plus/video?id=%s' % video_id,
             headers={'Origin': 'http://www.dcndigital.ae'})
 
diff --git a/youtube_dl/extractor/dramafever.py b/youtube_dl/extractor/dramafever.py
index 38e6597c8..d836c1a6c 100644
--- a/youtube_dl/extractor/dramafever.py
+++ b/youtube_dl/extractor/dramafever.py
@@ -7,7 +7,6 @@ from .common import InfoExtractor
 from ..compat import (
     compat_HTTPError,
     compat_urllib_parse,
-    compat_urllib_request,
     compat_urlparse,
 )
 from ..utils import (
@@ -16,6 +15,7 @@ from ..utils import (
     determine_ext,
     int_or_none,
     parse_iso8601,
+    sanitized_Request,
 )
 
 
@@ -51,7 +51,7 @@ class DramaFeverBaseIE(InfoExtractor):
             'password': password,
         }
 
-        request = compat_urllib_request.Request(
+        request = sanitized_Request(
             self._LOGIN_URL, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
         response = self._download_webpage(
             request, None, 'Logging in as %s' % username)
diff --git a/youtube_dl/extractor/dumpert.py b/youtube_dl/extractor/dumpert.py
index f5a31058d..e5aadcd25 100644
--- a/youtube_dl/extractor/dumpert.py
+++ b/youtube_dl/extractor/dumpert.py
@@ -5,8 +5,10 @@ import base64
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_request
-from ..utils import qualities
+from ..utils import (
+    qualities,
+    sanitized_Request,
+)
 
 
 class DumpertIE(InfoExtractor):
@@ -32,7 +34,7 @@ class DumpertIE(InfoExtractor):
         protocol = mobj.group('protocol')
 
         url = '%s://www.dumpert.nl/mediabase/%s' % (protocol, video_id)
-        req = compat_urllib_request.Request(url)
+        req = sanitized_Request(url)
         req.add_header('Cookie', 'nsfw=1; cpc=10')
         webpage = self._download_webpage(req, video_id)
 
diff --git a/youtube_dl/extractor/eitb.py b/youtube_dl/extractor/eitb.py
index 357a2196c..c83845fc2 100644
--- a/youtube_dl/extractor/eitb.py
+++ b/youtube_dl/extractor/eitb.py
@@ -2,11 +2,11 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_request
 from ..utils import (
     float_or_none,
     int_or_none,
     parse_iso8601,
+    sanitized_Request,
 )
 
 
@@ -57,7 +57,7 @@ class EitbIE(InfoExtractor):
 
         hls_url = media.get('HLS_SURL')
         if hls_url:
-            request = compat_urllib_request.Request(
+            request = sanitized_Request(
                 'http://mam.eitb.eus/mam/REST/ServiceMultiweb/DomainRestrictedSecurity/TokenAuth/',
                 headers={'Referer': url})
             token_data = self._download_json(
diff --git a/youtube_dl/extractor/escapist.py b/youtube_dl/extractor/escapist.py
index c85b4c458..a3d7bbbcb 100644
--- a/youtube_dl/extractor/escapist.py
+++ b/youtube_dl/extractor/escapist.py
@@ -3,13 +3,12 @@ from __future__ import unicode_literals
 import json
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_request
-
 from ..utils import (
     determine_ext,
     clean_html,
     int_or_none,
     float_or_none,
+    sanitized_Request,
 )
 
 
@@ -75,7 +74,7 @@ class EscapistIE(InfoExtractor):
         video_id = ims_video['videoID']
         key = ims_video['hash']
 
-        config_req = compat_urllib_request.Request(
+        config_req = sanitized_Request(
             'http://www.escapistmagazine.com/videos/'
             'vidconfig.php?videoID=%s&hash=%s' % (video_id, key))
         config_req.add_header('Referer', url)
diff --git a/youtube_dl/extractor/everyonesmixtape.py b/youtube_dl/extractor/everyonesmixtape.py
index d872d828f..493d38af8 100644
--- a/youtube_dl/extractor/everyonesmixtape.py
+++ b/youtube_dl/extractor/everyonesmixtape.py
@@ -3,11 +3,9 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_request,
-)
 from ..utils import (
     ExtractorError,
+    sanitized_Request,
 )
 
 
@@ -42,7 +40,7 @@ class EveryonesMixtapeIE(InfoExtractor):
         playlist_id = mobj.group('id')
 
         pllist_url = 'http://everyonesmixtape.com/mixtape.php?a=getMixes&u=-1&linked=%s&explore=' % playlist_id
-        pllist_req = compat_urllib_request.Request(pllist_url)
+        pllist_req = sanitized_Request(pllist_url)
         pllist_req.add_header('X-Requested-With', 'XMLHttpRequest')
 
         playlist_list = self._download_json(
@@ -55,7 +53,7 @@ class EveryonesMixtapeIE(InfoExtractor):
             raise ExtractorError('Playlist id not found')
 
         pl_url = 'http://everyonesmixtape.com/mixtape.php?a=getMix&id=%s&userId=null&code=' % playlist_no
-        pl_req = compat_urllib_request.Request(pl_url)
+        pl_req = sanitized_Request(pl_url)
         pl_req.add_header('X-Requested-With', 'XMLHttpRequest')
         playlist = self._download_json(
             pl_req, playlist_id, note='Downloading playlist info')
diff --git a/youtube_dl/extractor/extremetube.py b/youtube_dl/extractor/extremetube.py
index c5677c82b..3403581fd 100644
--- a/youtube_dl/extractor/extremetube.py
+++ b/youtube_dl/extractor/extremetube.py
@@ -3,9 +3,9 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_request
 from ..utils import (
     int_or_none,
+    sanitized_Request,
     str_to_int,
 )
 
@@ -37,7 +37,7 @@ class ExtremeTubeIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        req = compat_urllib_request.Request(url)
+        req = sanitized_Request(url)
         req.add_header('Cookie', 'age_verified=1')
         webpage = self._download_webpage(req, video_id)
 
diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index f53c51615..fd854411b 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -10,11 +10,11 @@ from ..compat import (
     compat_str,
     compat_urllib_error,
     compat_urllib_parse_unquote,
-    compat_urllib_request,
 )
 from ..utils import (
     ExtractorError,
     limit_length,
+    sanitized_Request,
     urlencode_postdata,
     get_element_by_id,
     clean_html,
@@ -73,7 +73,7 @@ class FacebookIE(InfoExtractor):
         if useremail is None:
             return
 
-        login_page_req = compat_urllib_request.Request(self._LOGIN_URL)
+        login_page_req = sanitized_Request(self._LOGIN_URL)
         login_page_req.add_header('Cookie', 'locale=en_US')
         login_page = self._download_webpage(login_page_req, None,
                                             note='Downloading login page',
@@ -94,7 +94,7 @@ class FacebookIE(InfoExtractor):
             'timezone': '-60',
             'trynum': '1',
         }
-        request = compat_urllib_request.Request(self._LOGIN_URL, urlencode_postdata(login_form))
+        request = sanitized_Request(self._LOGIN_URL, urlencode_postdata(login_form))
         request.add_header('Content-Type', 'application/x-www-form-urlencoded')
         try:
             login_results = self._download_webpage(request, None,
@@ -109,7 +109,7 @@ class FacebookIE(InfoExtractor):
                     r'name="h"\s+(?:\w+="[^"]+"\s+)*?value="([^"]+)"', login_results, 'h'),
                 'name_action_selected': 'dont_save',
             }
-            check_req = compat_urllib_request.Request(self._CHECKPOINT_URL, urlencode_postdata(check_form))
+            check_req = sanitized_Request(self._CHECKPOINT_URL, urlencode_postdata(check_form))
             check_req.add_header('Content-Type', 'application/x-www-form-urlencoded')
             check_response = self._download_webpage(check_req, None,
                                                     note='Confirming login')
diff --git a/youtube_dl/extractor/fc2.py b/youtube_dl/extractor/fc2.py
index a406945e8..92e8c571f 100644
--- a/youtube_dl/extractor/fc2.py
+++ b/youtube_dl/extractor/fc2.py
@@ -12,6 +12,7 @@ from ..compat import (
 from ..utils import (
     encode_dict,
     ExtractorError,
+    sanitized_Request,
 )
 
 
@@ -57,7 +58,7 @@ class FC2IE(InfoExtractor):
         }
 
         login_data = compat_urllib_parse.urlencode(encode_dict(login_form_strs)).encode('utf-8')
-        request = compat_urllib_request.Request(
+        request = sanitized_Request(
             'https://secure.id.fc2.com/index.php?mode=login&switch_language=en', login_data)
 
         login_results = self._download_webpage(request, None, note='Logging in', errnote='Unable to log in')
@@ -66,7 +67,7 @@ class FC2IE(InfoExtractor):
             return False
 
         # this is also needed
-        login_redir = compat_urllib_request.Request('http://id.fc2.com/?mode=redirect&login=done')
+        login_redir = sanitized_Request('http://id.fc2.com/?mode=redirect&login=done')
         self._download_webpage(
             login_redir, None, note='Login redirect', errnote='Login redirect failed')
 
diff --git a/youtube_dl/extractor/flickr.py b/youtube_dl/extractor/flickr.py
index 2fe76d661..91cd46e76 100644
--- a/youtube_dl/extractor/flickr.py
+++ b/youtube_dl/extractor/flickr.py
@@ -3,10 +3,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_request
 from ..utils import (
     ExtractorError,
     find_xpath_attr,
+    sanitized_Request,
 )
 
 
@@ -30,7 +30,7 @@ class FlickrIE(InfoExtractor):
         video_id = mobj.group('id')
         video_uploader_id = mobj.group('uploader_id')
         webpage_url = 'http://www.flickr.com/photos/' + video_uploader_id + '/' + video_id
-        req = compat_urllib_request.Request(webpage_url)
+        req = sanitized_Request(webpage_url)
         req.add_header(
             'User-Agent',
             # it needs a more recent version
diff --git a/youtube_dl/extractor/fourtube.py b/youtube_dl/extractor/fourtube.py
index fb6d108c0..fc4a5a0fb 100644
--- a/youtube_dl/extractor/fourtube.py
+++ b/youtube_dl/extractor/fourtube.py
@@ -3,12 +3,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_request,
-)
 from ..utils import (
     parse_duration,
     parse_iso8601,
+    sanitized_Request,
     str_to_int,
 )
 
@@ -93,7 +91,7 @@ class FourTubeIE(InfoExtractor):
             b'Content-Type': b'application/x-www-form-urlencoded',
             b'Origin': b'http://www.4tube.com',
         }
-        token_req = compat_urllib_request.Request(token_url, b'{}', headers)
+        token_req = sanitized_Request(token_url, b'{}', headers)
         tokens = self._download_json(token_req, video_id)
         formats = [{
             'url': tokens[format]['token'],
diff --git a/youtube_dl/extractor/gdcvault.py b/youtube_dl/extractor/gdcvault.py
index a6834db43..3befd3e7b 100644
--- a/youtube_dl/extractor/gdcvault.py
+++ b/youtube_dl/extractor/gdcvault.py
@@ -3,13 +3,11 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse,
-    compat_urllib_request,
-)
+from ..compat import compat_urllib_parse
 from ..utils import (
     remove_end,
     HEADRequest,
+    sanitized_Request,
 )
 
 
@@ -125,7 +123,7 @@ class GDCVaultIE(InfoExtractor):
             'password': password,
         }
 
-        request = compat_urllib_request.Request(login_url, compat_urllib_parse.urlencode(login_form))
+        request = sanitized_Request(login_url, compat_urllib_parse.urlencode(login_form))
         request.add_header('Content-Type', 'application/x-www-form-urlencoded')
         self._download_webpage(request, display_id, 'Logging in')
         start_page = self._download_webpage(webpage_url, display_id, 'Getting authenticated video page')
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 2b934148d..5075d131e 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -11,7 +11,6 @@ from .youtube import YoutubeIE
 from ..compat import (
     compat_etree_fromstring,
     compat_urllib_parse_unquote,
-    compat_urllib_request,
     compat_urlparse,
     compat_xml_parse_error,
 )
@@ -22,6 +21,7 @@ from ..utils import (
     HEADRequest,
     is_html,
     orderedSet,
+    sanitized_Request,
     smuggle_url,
     unescapeHTML,
     unified_strdate,
@@ -1215,7 +1215,7 @@ class GenericIE(InfoExtractor):
 
         full_response = None
         if head_response is False:
-            request = compat_urllib_request.Request(url)
+            request = sanitized_Request(url)
             request.add_header('Accept-Encoding', '*')
             full_response = self._request_webpage(request, video_id)
             head_response = full_response
@@ -1244,7 +1244,7 @@ class GenericIE(InfoExtractor):
                 '%s on generic information extractor.' % ('Forcing' if force else 'Falling back'))
 
         if not full_response:
-            request = compat_urllib_request.Request(url)
+            request = sanitized_Request(url)
             # Some webservers may serve compressed content of rather big size (e.g. gzipped flac)
             # making it impossible to download only chunk of the file (yet we need only 512kB to
             # test whether it's HTML or not). According to youtube-dl default Accept-Encoding
diff --git a/youtube_dl/extractor/hearthisat.py b/youtube_dl/extractor/hearthisat.py
index a19b31ac0..7d8698655 100644
--- a/youtube_dl/extractor/hearthisat.py
+++ b/youtube_dl/extractor/hearthisat.py
@@ -4,12 +4,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_request,
-    compat_urlparse,
-)
+from ..compat import compat_urlparse
 from ..utils import (
     HEADRequest,
+    sanitized_Request,
     str_to_int,
     urlencode_postdata,
     urlhandle_detect_ext,
@@ -47,7 +45,7 @@ class HearThisAtIE(InfoExtractor):
             r'intTrackId\s*=\s*(\d+)', webpage, 'track ID')
 
         payload = urlencode_postdata({'tracks[]': track_id})
-        req = compat_urllib_request.Request(self._PLAYLIST_URL, payload)
+        req = sanitized_Request(self._PLAYLIST_URL, payload)
         req.add_header('Content-type', 'application/x-www-form-urlencoded')
 
         track = self._download_json(req, track_id, 'Downloading playlist')[0]
diff --git a/youtube_dl/extractor/hotnewhiphop.py b/youtube_dl/extractor/hotnewhiphop.py
index 651784b73..31e219945 100644
--- a/youtube_dl/extractor/hotnewhiphop.py
+++ b/youtube_dl/extractor/hotnewhiphop.py
@@ -3,13 +3,11 @@ from __future__ import unicode_literals
 import base64
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse,
-    compat_urllib_request,
-)
+from ..compat import compat_urllib_parse
 from ..utils import (
     ExtractorError,
     HEADRequest,
+    sanitized_Request,
 )
 
 
@@ -41,7 +39,7 @@ class HotNewHipHopIE(InfoExtractor):
             ('mediaType', 's'),
             ('mediaId', video_id),
         ])
-        r = compat_urllib_request.Request(
+        r = sanitized_Request(
             'http://www.hotnewhiphop.com/ajax/media/getActions/', data=reqdata)
         r.add_header('Content-Type', 'application/x-www-form-urlencoded')
         mkd = self._download_json(
diff --git a/youtube_dl/extractor/hypem.py b/youtube_dl/extractor/hypem.py
index aa0724a02..cca3dd498 100644
--- a/youtube_dl/extractor/hypem.py
+++ b/youtube_dl/extractor/hypem.py
@@ -4,12 +4,10 @@ import json
 import time
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse,
-    compat_urllib_request,
-)
+from ..compat import compat_urllib_parse
 from ..utils import (
     ExtractorError,
+    sanitized_Request,
 )
 
 
@@ -32,7 +30,7 @@ class HypemIE(InfoExtractor):
         data = {'ax': 1, 'ts': time.time()}
         data_encoded = compat_urllib_parse.urlencode(data)
         complete_url = url + "?" + data_encoded
-        request = compat_urllib_request.Request(complete_url)
+        request = sanitized_Request(complete_url)
         response, urlh = self._download_webpage_handle(
             request, track_id, 'Downloading webpage with the url')
         cookie = urlh.headers.get('Set-Cookie', '')
@@ -52,7 +50,7 @@ class HypemIE(InfoExtractor):
         title = track['song']
 
         serve_url = "http://hypem.com/serve/source/%s/%s" % (track_id, key)
-        request = compat_urllib_request.Request(
+        request = sanitized_Request(
             serve_url, '', {'Content-Type': 'application/json'})
         request.add_header('cookie', cookie)
         song_data = self._download_json(request, track_id, 'Downloading metadata')
diff --git a/youtube_dl/extractor/iprima.py b/youtube_dl/extractor/iprima.py
index 821c8ec10..36baf3245 100644
--- a/youtube_dl/extractor/iprima.py
+++ b/youtube_dl/extractor/iprima.py
@@ -6,12 +6,10 @@ from random import random
 from math import floor
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_request,
-)
 from ..utils import (
     ExtractorError,
     remove_end,
+    sanitized_Request,
 )
 
 
@@ -61,7 +59,7 @@ class IPrimaIE(InfoExtractor):
             (floor(random() * 1073741824), floor(random() * 1073741824))
         )
 
-        req = compat_urllib_request.Request(player_url)
+        req = sanitized_Request(player_url)
         req.add_header('Referer', url)
         playerpage = self._download_webpage(req, video_id)
 
diff --git a/youtube_dl/extractor/ivi.py b/youtube_dl/extractor/ivi.py
index e82594444..029878d24 100644
--- a/youtube_dl/extractor/ivi.py
+++ b/youtube_dl/extractor/ivi.py
@@ -5,11 +5,9 @@ import re
 import json
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_request,
-)
 from ..utils import (
     ExtractorError,
+    sanitized_Request,
 )
 
 
@@ -78,7 +76,7 @@ class IviIE(InfoExtractor):
             ]
         }
 
-        request = compat_urllib_request.Request(api_url, json.dumps(data))
+        request = sanitized_Request(api_url, json.dumps(data))
 
         video_json_page = self._download_webpage(
             request, video_id, 'Downloading video JSON')
diff --git a/youtube_dl/extractor/keezmovies.py b/youtube_dl/extractor/keezmovies.py
index 82eddec51..d79261bb5 100644
--- a/youtube_dl/extractor/keezmovies.py
+++ b/youtube_dl/extractor/keezmovies.py
@@ -4,10 +4,8 @@ import os
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse_urlparse,
-    compat_urllib_request,
-)
+from ..compat import compat_urllib_parse_urlparse
+from ..utils import sanitized_Request
 
 
 class KeezMoviesIE(InfoExtractor):
@@ -26,7 +24,7 @@ class KeezMoviesIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        req = compat_urllib_request.Request(url)
+        req = sanitized_Request(url)
         req.add_header('Cookie', 'age_verified=1')
         webpage = self._download_webpage(req, video_id)
 
diff --git a/youtube_dl/extractor/letv.py b/youtube_dl/extractor/letv.py
index effd9eb92..be648000e 100644
--- a/youtube_dl/extractor/letv.py
+++ b/youtube_dl/extractor/letv.py
@@ -8,13 +8,13 @@ import time
 from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse,
-    compat_urllib_request,
     compat_ord,
 )
 from ..utils import (
     determine_ext,
     ExtractorError,
     parse_iso8601,
+    sanitized_Request,
     int_or_none,
     encode_data_uri,
 )
@@ -114,7 +114,7 @@ class LetvIE(InfoExtractor):
             'tkey': self.calc_time_key(int(time.time())),
             'domain': 'www.letv.com'
         }
-        play_json_req = compat_urllib_request.Request(
+        play_json_req = sanitized_Request(
             'http://api.letv.com/mms/out/video/playJson?' + compat_urllib_parse.urlencode(params)
         )
         cn_verification_proxy = self._downloader.params.get('cn_verification_proxy')
diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index 3d7e7e003..d4e1ae99d 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -7,12 +7,12 @@ from .common import InfoExtractor
 from ..compat import (
     compat_str,
     compat_urllib_parse,
-    compat_urllib_request,
 )
 from ..utils import (
     ExtractorError,
     clean_html,
     int_or_none,
+    sanitized_Request,
 )
 
 
@@ -35,7 +35,7 @@ class LyndaBaseIE(InfoExtractor):
             'remember': 'false',
             'stayPut': 'false'
         }
-        request = compat_urllib_request.Request(
+        request = sanitized_Request(
             self._LOGIN_URL, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
         login_page = self._download_webpage(
             request, None, 'Logging in as %s' % username)
@@ -64,7 +64,7 @@ class LyndaBaseIE(InfoExtractor):
                     'remember': 'false',
                     'stayPut': 'false',
                 }
-                request = compat_urllib_request.Request(
+                request = sanitized_Request(
                     self._LOGIN_URL, compat_urllib_parse.urlencode(confirm_form).encode('utf-8'))
                 login_page = self._download_webpage(
                     request, None,
diff --git a/youtube_dl/extractor/metacafe.py b/youtube_dl/extractor/metacafe.py
index 6e2e73a51..3c786a36d 100644
--- a/youtube_dl/extractor/metacafe.py
+++ b/youtube_dl/extractor/metacafe.py
@@ -7,12 +7,12 @@ from ..compat import (
     compat_parse_qs,
     compat_urllib_parse,
     compat_urllib_parse_unquote,
-    compat_urllib_request,
 )
 from ..utils import (
     determine_ext,
     ExtractorError,
     int_or_none,
+    sanitized_Request,
 )
 
 
@@ -117,7 +117,7 @@ class MetacafeIE(InfoExtractor):
             'filters': '0',
             'submit': "Continue - I'm over 18",
         }
-        request = compat_urllib_request.Request(self._FILTER_POST, compat_urllib_parse.urlencode(disclaimer_form))
+        request = sanitized_Request(self._FILTER_POST, compat_urllib_parse.urlencode(disclaimer_form))
         request.add_header('Content-Type', 'application/x-www-form-urlencoded')
         self.report_age_confirmation()
         self._download_webpage(request, None, False, 'Unable to confirm age')
@@ -142,7 +142,7 @@ class MetacafeIE(InfoExtractor):
                 return self.url_result('theplatform:%s' % ext_id, 'ThePlatform')
 
         # Retrieve video webpage to extract further information
-        req = compat_urllib_request.Request('http://www.metacafe.com/watch/%s/' % video_id)
+        req = sanitized_Request('http://www.metacafe.com/watch/%s/' % video_id)
 
         # AnyClip videos require the flashversion cookie so that we get the link
         # to the mp4 file
diff --git a/youtube_dl/extractor/minhateca.py b/youtube_dl/extractor/minhateca.py
index 14934b7ec..e46b23a6f 100644
--- a/youtube_dl/extractor/minhateca.py
+++ b/youtube_dl/extractor/minhateca.py
@@ -2,14 +2,12 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse,
-    compat_urllib_request,
-)
+from ..compat import compat_urllib_parse
 from ..utils import (
     int_or_none,
     parse_duration,
     parse_filesize,
+    sanitized_Request,
 )
 
 
@@ -39,7 +37,7 @@ class MinhatecaIE(InfoExtractor):
             ('fileId', video_id),
             ('__RequestVerificationToken', token),
         ]
-        req = compat_urllib_request.Request(
+        req = sanitized_Request(
             'http://minhateca.com.br/action/License/Download',
             data=compat_urllib_parse.urlencode(token_data))
         req.add_header('Content-Type', 'application/x-www-form-urlencoded')
diff --git a/youtube_dl/extractor/miomio.py b/youtube_dl/extractor/miomio.py
index ce391c759..170ebd9eb 100644
--- a/youtube_dl/extractor/miomio.py
+++ b/youtube_dl/extractor/miomio.py
@@ -4,11 +4,11 @@ from __future__ import unicode_literals
 import random
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_request
 from ..utils import (
     xpath_text,
     int_or_none,
     ExtractorError,
+    sanitized_Request,
 )
 
 
@@ -63,7 +63,7 @@ class MioMioIE(InfoExtractor):
             'http://www.miomio.tv/mioplayer/mioplayerconfigfiles/xml.php?id=%s&r=%s' % (id, random.randint(100, 999)),
             video_id)
 
-        vid_config_request = compat_urllib_request.Request(
+        vid_config_request = sanitized_Request(
             'http://www.miomio.tv/mioplayer/mioplayerconfigfiles/sina.php?{0}'.format(xml_config),
             headers=http_headers)
 
diff --git a/youtube_dl/extractor/moevideo.py b/youtube_dl/extractor/moevideo.py
index 5a66302f6..d930b9634 100644
--- a/youtube_dl/extractor/moevideo.py
+++ b/youtube_dl/extractor/moevideo.py
@@ -5,13 +5,11 @@ import json
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse,
-    compat_urllib_request,
-)
+from ..compat import compat_urllib_parse
 from ..utils import (
     ExtractorError,
     int_or_none,
+    sanitized_Request,
 )
 
 
@@ -80,7 +78,7 @@ class MoeVideoIE(InfoExtractor):
         ]
         r_json = json.dumps(r)
         post = compat_urllib_parse.urlencode({'r': r_json})
-        req = compat_urllib_request.Request(self._API_URL, post)
+        req = sanitized_Request(self._API_URL, post)
         req.add_header('Content-type', 'application/x-www-form-urlencoded')
 
         response = self._download_json(req, video_id)
diff --git a/youtube_dl/extractor/mofosex.py b/youtube_dl/extractor/mofosex.py
index 9bf99a54a..f8226cbb2 100644
--- a/youtube_dl/extractor/mofosex.py
+++ b/youtube_dl/extractor/mofosex.py
@@ -7,8 +7,8 @@ from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse_unquote,
     compat_urllib_parse_urlparse,
-    compat_urllib_request,
 )
+from ..utils import sanitized_Request
 
 
 class MofosexIE(InfoExtractor):
@@ -29,7 +29,7 @@ class MofosexIE(InfoExtractor):
         video_id = mobj.group('id')
         url = 'http://www.' + mobj.group('url')
 
-        req = compat_urllib_request.Request(url)
+        req = sanitized_Request(url)
         req.add_header('Cookie', 'age_verified=1')
         webpage = self._download_webpage(req, video_id)
 
diff --git a/youtube_dl/extractor/moniker.py b/youtube_dl/extractor/moniker.py
index 7c0c4e50e..f6bf94f2f 100644
--- a/youtube_dl/extractor/moniker.py
+++ b/youtube_dl/extractor/moniker.py
@@ -5,13 +5,11 @@ import os.path
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse,
-    compat_urllib_request,
-)
+from ..compat import compat_urllib_parse
 from ..utils import (
     ExtractorError,
     remove_start,
+    sanitized_Request,
 )
 
 
@@ -81,7 +79,7 @@ class MonikerIE(InfoExtractor):
             orig_webpage, 'builtin URL', default=None, group='url')
 
         if builtin_url:
-            req = compat_urllib_request.Request(builtin_url)
+            req = sanitized_Request(builtin_url)
             req.add_header('Referer', url)
             webpage = self._download_webpage(req, video_id, 'Downloading builtin page')
             title = self._og_search_title(orig_webpage).strip()
@@ -94,7 +92,7 @@ class MonikerIE(InfoExtractor):
             headers = {
                 b'Content-Type': b'application/x-www-form-urlencoded',
             }
-            req = compat_urllib_request.Request(url, post, headers)
+            req = sanitized_Request(url, post, headers)
             webpage = self._download_webpage(
                 req, video_id, note='Downloading video page ...')
 
diff --git a/youtube_dl/extractor/mooshare.py b/youtube_dl/extractor/mooshare.py
index 7603af5e2..7cc7f054f 100644
--- a/youtube_dl/extractor/mooshare.py
+++ b/youtube_dl/extractor/mooshare.py
@@ -3,12 +3,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_request,
-    compat_urllib_parse,
-)
+from ..compat import compat_urllib_parse
 from ..utils import (
     ExtractorError,
+    sanitized_Request,
 )
 
 
@@ -59,7 +57,7 @@ class MooshareIE(InfoExtractor):
             'hash': hash_key,
         }
 
-        request = compat_urllib_request.Request(
+        request = sanitized_Request(
             'http://mooshare.biz/%s' % video_id, compat_urllib_parse.urlencode(download_form))
         request.add_header('Content-Type', 'application/x-www-form-urlencoded')
 
diff --git a/youtube_dl/extractor/movieclips.py b/youtube_dl/extractor/movieclips.py
index b8c43a163..1564cb71f 100644
--- a/youtube_dl/extractor/movieclips.py
+++ b/youtube_dl/extractor/movieclips.py
@@ -2,9 +2,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_request,
-)
+from ..utils import sanitized_Request
 
 
 class MovieClipsIE(InfoExtractor):
@@ -25,7 +23,7 @@ class MovieClipsIE(InfoExtractor):
     def _real_extract(self, url):
         display_id = self._match_id(url)
 
-        req = compat_urllib_request.Request(url)
+        req = sanitized_Request(url)
         # it doesn't work if it thinks the browser it's too old
         req.add_header('User-Agent', 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20150101 Firefox/43.0 (Chrome)')
         webpage = self._download_webpage(req, display_id)
diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index 302c9bf35..d887583e6 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -5,7 +5,6 @@ import re
 from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse,
-    compat_urllib_request,
     compat_str,
 )
 from ..utils import (
@@ -13,6 +12,7 @@ from ..utils import (
     find_xpath_attr,
     fix_xml_ampersands,
     HEADRequest,
+    sanitized_Request,
     unescapeHTML,
     url_basename,
     RegexNotFoundError,
@@ -53,7 +53,7 @@ class MTVServicesInfoExtractor(InfoExtractor):
 
     def _extract_mobile_video_formats(self, mtvn_id):
         webpage_url = self._MOBILE_TEMPLATE % mtvn_id
-        req = compat_urllib_request.Request(webpage_url)
+        req = sanitized_Request(webpage_url)
         # Otherwise we get a webpage that would execute some javascript
         req.add_header('User-Agent', 'curl/7')
         webpage = self._download_webpage(req, mtvn_id,
diff --git a/youtube_dl/extractor/myvideo.py b/youtube_dl/extractor/myvideo.py
index c96f472a3..36ab388b2 100644
--- a/youtube_dl/extractor/myvideo.py
+++ b/youtube_dl/extractor/myvideo.py
@@ -11,10 +11,10 @@ from ..compat import (
     compat_ord,
     compat_urllib_parse,
     compat_urllib_parse_unquote,
-    compat_urllib_request,
 )
 from ..utils import (
     ExtractorError,
+    sanitized_Request,
 )
 
 
@@ -83,7 +83,7 @@ class MyVideoIE(InfoExtractor):
 
         mobj = re.search(r'data-video-service="/service/data/video/%s/config' % video_id, webpage)
         if mobj is not None:
-            request = compat_urllib_request.Request('http://www.myvideo.de/service/data/video/%s/config' % video_id, '')
+            request = sanitized_Request('http://www.myvideo.de/service/data/video/%s/config' % video_id, '')
             response = self._download_webpage(request, video_id,
                                               'Downloading video info')
             info = json.loads(base64.b64decode(response).decode('utf-8'))
diff --git a/youtube_dl/extractor/neteasemusic.py b/youtube_dl/extractor/neteasemusic.py
index bb3362069..15eca825a 100644
--- a/youtube_dl/extractor/neteasemusic.py
+++ b/youtube_dl/extractor/neteasemusic.py
@@ -8,11 +8,11 @@ import re
 
 from .common import InfoExtractor
 from ..compat import (
-    compat_urllib_request,
     compat_urllib_parse,
     compat_str,
     compat_itertools_count,
 )
+from ..utils import sanitized_Request
 
 
 class NetEaseMusicBaseIE(InfoExtractor):
@@ -56,7 +56,7 @@ class NetEaseMusicBaseIE(InfoExtractor):
         return int(round(ms / 1000.0))
 
     def query_api(self, endpoint, video_id, note):
-        req = compat_urllib_request.Request('%s%s' % (self._API_BASE, endpoint))
+        req = sanitized_Request('%s%s' % (self._API_BASE, endpoint))
         req.add_header('Referer', self._API_BASE)
         return self._download_json(req, video_id, note)
 
diff --git a/youtube_dl/extractor/nfb.py b/youtube_dl/extractor/nfb.py
index ea077254b..5bd15f7a7 100644
--- a/youtube_dl/extractor/nfb.py
+++ b/youtube_dl/extractor/nfb.py
@@ -1,10 +1,8 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_request,
-    compat_urllib_parse,
-)
+from ..compat import compat_urllib_parse
+from ..utils import sanitized_Request
 
 
 class NFBIE(InfoExtractor):
@@ -40,8 +38,9 @@ class NFBIE(InfoExtractor):
         uploader = self._html_search_regex(r'<em class="director-name" itemprop="name">([^<]+)</em>',
                                            page, 'director name', fatal=False)
 
-        request = compat_urllib_request.Request('https://www.nfb.ca/film/%s/player_config' % video_id,
-                                                compat_urllib_parse.urlencode({'getConfig': 'true'}).encode('ascii'))
+        request = sanitized_Request(
+            'https://www.nfb.ca/film/%s/player_config' % video_id,
+            compat_urllib_parse.urlencode({'getConfig': 'true'}).encode('ascii'))
         request.add_header('Content-Type', 'application/x-www-form-urlencoded')
         request.add_header('X-NFB-Referer', 'http://www.nfb.ca/medias/flash/NFBVideoPlayer.swf')
 
diff --git a/youtube_dl/extractor/niconico.py b/youtube_dl/extractor/niconico.py
index bda1cff05..586e52a4a 100644
--- a/youtube_dl/extractor/niconico.py
+++ b/youtube_dl/extractor/niconico.py
@@ -8,7 +8,6 @@ import datetime
 from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse,
-    compat_urllib_request,
     compat_urlparse,
 )
 from ..utils import (
@@ -17,6 +16,7 @@ from ..utils import (
     int_or_none,
     parse_duration,
     parse_iso8601,
+    sanitized_Request,
     xpath_text,
     determine_ext,
 )
@@ -102,7 +102,7 @@ class NiconicoIE(InfoExtractor):
             'password': password,
         }
         login_data = compat_urllib_parse.urlencode(encode_dict(login_form_strs)).encode('utf-8')
-        request = compat_urllib_request.Request(
+        request = sanitized_Request(
             'https://secure.nicovideo.jp/secure/login', login_data)
         login_results = self._download_webpage(
             request, None, note='Logging in', errnote='Unable to log in')
@@ -145,7 +145,7 @@ class NiconicoIE(InfoExtractor):
                 'k': thumb_play_key,
                 'v': video_id
             })
-            flv_info_request = compat_urllib_request.Request(
+            flv_info_request = sanitized_Request(
                 'http://ext.nicovideo.jp/thumb_watch', flv_info_data,
                 {'Content-Type': 'application/x-www-form-urlencoded'})
             flv_info_webpage = self._download_webpage(
diff --git a/youtube_dl/extractor/noco.py b/youtube_dl/extractor/noco.py
index a53e27b27..76bd21e6d 100644
--- a/youtube_dl/extractor/noco.py
+++ b/youtube_dl/extractor/noco.py
@@ -9,7 +9,6 @@ from .common import InfoExtractor
 from ..compat import (
     compat_str,
     compat_urllib_parse,
-    compat_urllib_request,
 )
 from ..utils import (
     clean_html,
@@ -17,6 +16,7 @@ from ..utils import (
     int_or_none,
     float_or_none,
     parse_iso8601,
+    sanitized_Request,
 )
 
 
@@ -74,7 +74,7 @@ class NocoIE(InfoExtractor):
             'username': username,
             'password': password,
         }
-        request = compat_urllib_request.Request(self._LOGIN_URL, compat_urllib_parse.urlencode(login_form))
+        request = sanitized_Request(self._LOGIN_URL, compat_urllib_parse.urlencode(login_form))
         request.add_header('Content-Type', 'application/x-www-form-urlencoded; charset=UTF-8')
 
         login = self._download_json(request, None, 'Logging in as %s' % username)
diff --git a/youtube_dl/extractor/nosvideo.py b/youtube_dl/extractor/nosvideo.py
index f5ef856db..eab816e49 100644
--- a/youtube_dl/extractor/nosvideo.py
+++ b/youtube_dl/extractor/nosvideo.py
@@ -4,11 +4,9 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_request,
-)
 from ..utils import (
     ExtractorError,
+    sanitized_Request,
     urlencode_postdata,
     xpath_text,
     xpath_with_ns,
@@ -41,7 +39,7 @@ class NosVideoIE(InfoExtractor):
             'op': 'download1',
             'method_free': 'Continue to Video',
         }
-        req = compat_urllib_request.Request(url, urlencode_postdata(fields))
+        req = sanitized_Request(url, urlencode_postdata(fields))
         req.add_header('Content-type', 'application/x-www-form-urlencoded')
         webpage = self._download_webpage(req, video_id,
                                          'Downloading download page')
diff --git a/youtube_dl/extractor/novamov.py b/youtube_dl/extractor/novamov.py
index 6b15fc2e5..6163e8855 100644
--- a/youtube_dl/extractor/novamov.py
+++ b/youtube_dl/extractor/novamov.py
@@ -3,14 +3,12 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_request,
-    compat_urlparse,
-)
+from ..compat import compat_urlparse
 from ..utils import (
     ExtractorError,
     NO_DEFAULT,
     encode_dict,
+    sanitized_Request,
     urlencode_postdata,
 )
 
@@ -65,7 +63,7 @@ class NovaMovIE(InfoExtractor):
                 'post url', default=url, group='url')
             if not post_url.startswith('http'):
                 post_url = compat_urlparse.urljoin(url, post_url)
-            request = compat_urllib_request.Request(
+            request = sanitized_Request(
                 post_url, urlencode_postdata(encode_dict(fields)))
             request.add_header('Content-Type', 'application/x-www-form-urlencoded')
             request.add_header('Referer', post_url)
diff --git a/youtube_dl/extractor/nowness.py b/youtube_dl/extractor/nowness.py
index 0fba55833..d480fb58c 100644
--- a/youtube_dl/extractor/nowness.py
+++ b/youtube_dl/extractor/nowness.py
@@ -3,10 +3,10 @@ from __future__ import unicode_literals
 
 from .brightcove import BrightcoveLegacyIE
 from .common import InfoExtractor
-from ..utils import ExtractorError
-from ..compat import (
-    compat_str,
-    compat_urllib_request,
+from ..compat import compat_str
+from ..utils import (
+    ExtractorError,
+    sanitized_Request,
 )
 
 
@@ -37,7 +37,7 @@ class NownessBaseIE(InfoExtractor):
 
     def _api_request(self, url, request_path):
         display_id = self._match_id(url)
-        request = compat_urllib_request.Request(
+        request = sanitized_Request(
             'http://api.nowness.com/api/' + request_path % display_id,
             headers={
                 'X-Nowness-Language': 'zh-cn' if 'cn.nowness.com' in url else 'en-us',
diff --git a/youtube_dl/extractor/nuvid.py b/youtube_dl/extractor/nuvid.py
index 57928f2ae..9fa7cefad 100644
--- a/youtube_dl/extractor/nuvid.py
+++ b/youtube_dl/extractor/nuvid.py
@@ -3,11 +3,9 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_request,
-)
 from ..utils import (
     parse_duration,
+    sanitized_Request,
     unified_strdate,
 )
 
@@ -33,7 +31,7 @@ class NuvidIE(InfoExtractor):
         formats = []
 
         for dwnld_speed, format_id in [(0, '3gp'), (5, 'mp4')]:
-            request = compat_urllib_request.Request(
+            request = sanitized_Request(
                 'http://m.nuvid.com/play/%s' % video_id)
             request.add_header('Cookie', 'skip_download_page=1; dwnld_speed=%d; adv_show=1' % dwnld_speed)
             webpage = self._download_webpage(
diff --git a/youtube_dl/extractor/patreon.py b/youtube_dl/extractor/patreon.py
index 6cdc2638b..ec8876c28 100644
--- a/youtube_dl/extractor/patreon.py
+++ b/youtube_dl/extractor/patreon.py
@@ -2,9 +2,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..utils import (
-    js_to_json,
-)
+from ..utils import js_to_json
 
 
 class PatreonIE(InfoExtractor):
@@ -65,7 +63,7 @@ class PatreonIE(InfoExtractor):
             'password': password,
         }
 
-        request = compat_urllib_request.Request(
+        request = sanitized_Request(
             'https://www.patreon.com/processLogin',
             compat_urllib_parse.urlencode(login_form).encode('utf-8')
         )
diff --git a/youtube_dl/extractor/played.py b/youtube_dl/extractor/played.py
index 8a1c296dd..2856af96f 100644
--- a/youtube_dl/extractor/played.py
+++ b/youtube_dl/extractor/played.py
@@ -5,12 +5,10 @@ import re
 import os.path
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse,
-    compat_urllib_request,
-)
+from ..compat import compat_urllib_parse
 from ..utils import (
     ExtractorError,
+    sanitized_Request,
 )
 
 
@@ -46,7 +44,7 @@ class PlayedIE(InfoExtractor):
         headers = {
             b'Content-Type': b'application/x-www-form-urlencoded',
         }
-        req = compat_urllib_request.Request(url, post, headers)
+        req = sanitized_Request(url, post, headers)
         webpage = self._download_webpage(
             req, video_id, note='Downloading video page ...')
 
diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index 792316db8..aa7dbcb63 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -8,13 +8,13 @@ from .common import InfoExtractor
 from ..compat import (
     compat_str,
     compat_urllib_parse,
-    compat_urllib_request,
     compat_urlparse,
 )
 from ..utils import (
     ExtractorError,
     int_or_none,
     parse_duration,
+    sanitized_Request,
 )
 
 
@@ -73,7 +73,7 @@ class PluralsightIE(PluralsightBaseIE):
         if not post_url.startswith('http'):
             post_url = compat_urlparse.urljoin(self._LOGIN_URL, post_url)
 
-        request = compat_urllib_request.Request(
+        request = sanitized_Request(
             post_url, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
         request.add_header('Content-Type', 'application/x-www-form-urlencoded')
 
@@ -181,7 +181,7 @@ class PluralsightIE(PluralsightBaseIE):
                     'mt': ext,
                     'q': '%dx%d' % (f['width'], f['height']),
                 }
-                request = compat_urllib_request.Request(
+                request = sanitized_Request(
                     '%s/training/Player/ViewClip' % self._API_BASE,
                     json.dumps(clip_post).encode('utf-8'))
                 request.add_header('Content-Type', 'application/json;charset=utf-8')
diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index a656ad85a..965940a4b 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -8,10 +8,10 @@ from ..compat import (
     compat_urllib_parse_unquote,
     compat_urllib_parse_unquote_plus,
     compat_urllib_parse_urlparse,
-    compat_urllib_request,
 )
 from ..utils import (
     ExtractorError,
+    sanitized_Request,
     str_to_int,
 )
 from ..aes import (
@@ -53,7 +53,7 @@ class PornHubIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        req = compat_urllib_request.Request(
+        req = sanitized_Request(
             'http://www.pornhub.com/view_video.php?viewkey=%s' % video_id)
         req.add_header('Cookie', 'age_verified=1')
         webpage = self._download_webpage(req, video_id)
diff --git a/youtube_dl/extractor/pornotube.py b/youtube_dl/extractor/pornotube.py
index 34735c51e..5398e708b 100644
--- a/youtube_dl/extractor/pornotube.py
+++ b/youtube_dl/extractor/pornotube.py
@@ -3,11 +3,9 @@ from __future__ import unicode_literals
 import json
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_request,
-)
 from ..utils import (
     int_or_none,
+    sanitized_Request,
 )
 
 
@@ -46,7 +44,7 @@ class PornotubeIE(InfoExtractor):
             'authenticationSpaceKey': originAuthenticationSpaceKey,
             'credentials': 'Clip Application',
         }
-        token_req = compat_urllib_request.Request(
+        token_req = sanitized_Request(
             'https://api.aebn.net/auth/v1/token/primal',
             data=json.dumps(token_req_data).encode('utf-8'))
         token_req.add_header('Content-Type', 'application/json')
@@ -56,7 +54,7 @@ class PornotubeIE(InfoExtractor):
         token = token_answer['tokenKey']
 
         # Get video URL
-        delivery_req = compat_urllib_request.Request(
+        delivery_req = sanitized_Request(
             'https://api.aebn.net/delivery/v1/clips/%s/MP4' % video_id)
         delivery_req.add_header('Authorization', token)
         delivery_info = self._download_json(
@@ -64,7 +62,7 @@ class PornotubeIE(InfoExtractor):
         video_url = delivery_info['mediaUrl']
 
         # Get additional info (title etc.)
-        info_req = compat_urllib_request.Request(
+        info_req = sanitized_Request(
             'https://api.aebn.net/content/v1/clips/%s?expand='
             'title,description,primaryImageNumber,startSecond,endSecond,'
             'movie.title,movie.MovieId,movie.boxCoverFront,movie.stars,'
diff --git a/youtube_dl/extractor/primesharetv.py b/youtube_dl/extractor/primesharetv.py
index 304359dc5..85aae9576 100644
--- a/youtube_dl/extractor/primesharetv.py
+++ b/youtube_dl/extractor/primesharetv.py
@@ -1,11 +1,11 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse,
-    compat_urllib_request,
+from ..compat import compat_urllib_parse
+from ..utils import (
+    ExtractorError,
+    sanitized_Request,
 )
-from ..utils import ExtractorError
 
 
 class PrimeShareTVIE(InfoExtractor):
@@ -41,7 +41,7 @@ class PrimeShareTVIE(InfoExtractor):
             webpage, 'wait time', default=7)) + 1
         self._sleep(wait_time, video_id)
 
-        req = compat_urllib_request.Request(
+        req = sanitized_Request(
             url, compat_urllib_parse.urlencode(fields), headers)
         video_page = self._download_webpage(
             req, video_id, 'Downloading video page')
diff --git a/youtube_dl/extractor/promptfile.py b/youtube_dl/extractor/promptfile.py
index 8190ed676..d5357283a 100644
--- a/youtube_dl/extractor/promptfile.py
+++ b/youtube_dl/extractor/promptfile.py
@@ -4,13 +4,11 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse,
-    compat_urllib_request,
-)
+from ..compat import compat_urllib_parse
 from ..utils import (
     determine_ext,
     ExtractorError,
+    sanitized_Request,
 )
 
 
@@ -37,7 +35,7 @@ class PromptFileIE(InfoExtractor):
 
         fields = self._hidden_inputs(webpage)
         post = compat_urllib_parse.urlencode(fields)
-        req = compat_urllib_request.Request(url, post)
+        req = sanitized_Request(url, post)
         req.add_header('Content-type', 'application/x-www-form-urlencoded')
         webpage = self._download_webpage(
             req, video_id, 'Downloading video page')
diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index c98539f6a..1ba3bbddf 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -7,11 +7,11 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
+    sanitized_Request,
     strip_jsonp,
     unescapeHTML,
     clean_html,
 )
-from ..compat import compat_urllib_request
 
 
 class QQMusicIE(InfoExtractor):
@@ -201,7 +201,7 @@ class QQMusicSingerIE(QQPlaylistBaseIE):
         singer_desc = None
 
         if singer_id:
-            req = compat_urllib_request.Request(
+            req = sanitized_Request(
                 'http://s.plcloud.music.qq.com/fcgi-bin/fcg_get_singer_desc.fcg?utf8=1&outCharset=utf-8&format=xml&singerid=%s' % singer_id)
             req.add_header(
                 'Referer', 'http://s.plcloud.music.qq.com/xhr_proxy_utf8.html')
diff --git a/youtube_dl/extractor/rtve.py b/youtube_dl/extractor/rtve.py
index 0fe6356db..603d7bd00 100644
--- a/youtube_dl/extractor/rtve.py
+++ b/youtube_dl/extractor/rtve.py
@@ -6,11 +6,11 @@ import re
 import time
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_request
 from ..utils import (
     ExtractorError,
     float_or_none,
     remove_end,
+    sanitized_Request,
     std_headers,
     struct_unpack,
 )
@@ -102,7 +102,7 @@ class RTVEALaCartaIE(InfoExtractor):
         if info['state'] == 'DESPU':
             raise ExtractorError('The video is no longer available', expected=True)
         png_url = 'http://www.rtve.es/ztnr/movil/thumbnail/%s/videos/%s.png' % (self._manager, video_id)
-        png_request = compat_urllib_request.Request(png_url)
+        png_request = sanitized_Request(png_url)
         png_request.add_header('Referer', url)
         png = self._download_webpage(png_request, video_id, 'Downloading url information')
         video_url = _decrypt_url(png)
diff --git a/youtube_dl/extractor/safari.py b/youtube_dl/extractor/safari.py
index e9e33d0a3..919704261 100644
--- a/youtube_dl/extractor/safari.py
+++ b/youtube_dl/extractor/safari.py
@@ -6,12 +6,10 @@ import re
 from .common import InfoExtractor
 from .brightcove import BrightcoveLegacyIE
 
-from ..compat import (
-    compat_urllib_parse,
-    compat_urllib_request,
-)
+from ..compat import compat_urllib_parse
 from ..utils import (
     ExtractorError,
+    sanitized_Request,
     smuggle_url,
     std_headers,
 )
@@ -58,7 +56,7 @@ class SafariBaseIE(InfoExtractor):
             'next': '',
         }
 
-        request = compat_urllib_request.Request(
+        request = sanitized_Request(
             self._LOGIN_URL, compat_urllib_parse.urlencode(login_form), headers=headers)
         login_page = self._download_webpage(
             request, None, 'Logging in as %s' % username)
diff --git a/youtube_dl/extractor/sandia.py b/youtube_dl/extractor/sandia.py
index 9c88167f0..759898a49 100644
--- a/youtube_dl/extractor/sandia.py
+++ b/youtube_dl/extractor/sandia.py
@@ -6,14 +6,12 @@ import json
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_request,
-    compat_urlparse,
-)
+from ..compat import compat_urlparse
 from ..utils import (
     int_or_none,
     js_to_json,
     mimetype2ext,
+    sanitized_Request,
     unified_strdate,
 )
 
@@ -37,7 +35,7 @@ class SandiaIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        req = compat_urllib_request.Request(url)
+        req = sanitized_Request(url)
         req.add_header('Cookie', 'MediasitePlayerCaps=ClientPlugins=4')
         webpage = self._download_webpage(req, video_id)
 
diff --git a/youtube_dl/extractor/shared.py b/youtube_dl/extractor/shared.py
index c5636e8e9..8eda3c864 100644
--- a/youtube_dl/extractor/shared.py
+++ b/youtube_dl/extractor/shared.py
@@ -3,13 +3,11 @@ from __future__ import unicode_literals
 import base64
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse,
-    compat_urllib_request,
-)
+from ..compat import compat_urllib_parse
 from ..utils import (
     ExtractorError,
     int_or_none,
+    sanitized_Request,
 )
 
 
@@ -46,7 +44,7 @@ class SharedIE(InfoExtractor):
                 'Video %s does not exist' % video_id, expected=True)
 
         download_form = self._hidden_inputs(webpage)
-        request = compat_urllib_request.Request(
+        request = sanitized_Request(
             url, compat_urllib_parse.urlencode(download_form))
         request.add_header('Content-Type', 'application/x-www-form-urlencoded')
 
diff --git a/youtube_dl/extractor/sharesix.py b/youtube_dl/extractor/sharesix.py
index ac3e3adf2..f1ea9bdb2 100644
--- a/youtube_dl/extractor/sharesix.py
+++ b/youtube_dl/extractor/sharesix.py
@@ -4,12 +4,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse,
-    compat_urllib_request,
-)
+from ..compat import compat_urllib_parse
 from ..utils import (
     parse_duration,
+    sanitized_Request,
 )
 
 
@@ -50,7 +48,7 @@ class ShareSixIE(InfoExtractor):
             'method_free': 'Free'
         }
         post = compat_urllib_parse.urlencode(fields)
-        req = compat_urllib_request.Request(url, post)
+        req = sanitized_Request(url, post)
         req.add_header('Content-type', 'application/x-www-form-urlencoded')
 
         webpage = self._download_webpage(req, video_id,
diff --git a/youtube_dl/extractor/sina.py b/youtube_dl/extractor/sina.py
index 0891a441f..b2258a0f6 100644
--- a/youtube_dl/extractor/sina.py
+++ b/youtube_dl/extractor/sina.py
@@ -4,10 +4,8 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_request,
-    compat_urllib_parse,
-)
+from ..compat import compat_urllib_parse
+from ..utils import sanitized_Request
 
 
 class SinaIE(InfoExtractor):
@@ -61,7 +59,7 @@ class SinaIE(InfoExtractor):
         if mobj.group('token') is not None:
             # The video id is in the redirected url
             self.to_screen('Getting video id')
-            request = compat_urllib_request.Request(url)
+            request = sanitized_Request(url)
             request.get_method = lambda: 'HEAD'
             (_, urlh) = self._download_webpage_handle(request, 'NA', False)
             return self._real_extract(urlh.geturl())
diff --git a/youtube_dl/extractor/smotri.py b/youtube_dl/extractor/smotri.py
index 35a81ee87..30210c8a3 100644
--- a/youtube_dl/extractor/smotri.py
+++ b/youtube_dl/extractor/smotri.py
@@ -7,13 +7,11 @@ import hashlib
 import uuid
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse,
-    compat_urllib_request,
-)
+from ..compat import compat_urllib_parse
 from ..utils import (
     ExtractorError,
     int_or_none,
+    sanitized_Request,
     unified_strdate,
 )
 
@@ -176,7 +174,7 @@ class SmotriIE(InfoExtractor):
         if video_password:
             video_form['pass'] = hashlib.md5(video_password.encode('utf-8')).hexdigest()
 
-        request = compat_urllib_request.Request(
+        request = sanitized_Request(
             'http://smotri.com/video/view/url/bot/', compat_urllib_parse.urlencode(video_form))
         request.add_header('Content-Type', 'application/x-www-form-urlencoded')
 
@@ -339,7 +337,7 @@ class SmotriBroadcastIE(InfoExtractor):
                 'password': password,
             }
 
-            request = compat_urllib_request.Request(
+            request = sanitized_Request(
                 broadcast_url + '/?no_redirect=1', compat_urllib_parse.urlencode(login_form))
             request.add_header('Content-Type', 'application/x-www-form-urlencoded')
             broadcast_page = self._download_webpage(
diff --git a/youtube_dl/extractor/sohu.py b/youtube_dl/extractor/sohu.py
index ba2d5e19b..daf6ad555 100644
--- a/youtube_dl/extractor/sohu.py
+++ b/youtube_dl/extractor/sohu.py
@@ -6,11 +6,11 @@ import re
 from .common import InfoExtractor
 from ..compat import (
     compat_str,
-    compat_urllib_request,
     compat_urllib_parse,
 )
 from ..utils import (
     ExtractorError,
+    sanitized_Request,
 )
 
 
@@ -96,7 +96,7 @@ class SohuIE(InfoExtractor):
             else:
                 base_data_url = 'http://hot.vrs.sohu.com/vrs_flash.action?vid='
 
-            req = compat_urllib_request.Request(base_data_url + vid_id)
+            req = sanitized_Request(base_data_url + vid_id)
 
             cn_verification_proxy = self._downloader.params.get('cn_verification_proxy')
             if cn_verification_proxy:
diff --git a/youtube_dl/extractor/spankwire.py b/youtube_dl/extractor/spankwire.py
index 9e8fb35b2..692fd78e8 100644
--- a/youtube_dl/extractor/spankwire.py
+++ b/youtube_dl/extractor/spankwire.py
@@ -6,9 +6,9 @@ from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse_unquote,
     compat_urllib_parse_urlparse,
-    compat_urllib_request,
 )
 from ..utils import (
+    sanitized_Request,
     str_to_int,
     unified_strdate,
 )
@@ -51,7 +51,7 @@ class SpankwireIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
 
-        req = compat_urllib_request.Request('http://www.' + mobj.group('url'))
+        req = sanitized_Request('http://www.' + mobj.group('url'))
         req.add_header('Cookie', 'age_verified=1')
         webpage = self._download_webpage(req, video_id)
 
diff --git a/youtube_dl/extractor/sportdeutschland.py b/youtube_dl/extractor/sportdeutschland.py
index 7ec6c613f..ebb75f059 100644
--- a/youtube_dl/extractor/sportdeutschland.py
+++ b/youtube_dl/extractor/sportdeutschland.py
@@ -4,11 +4,9 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_request,
-)
 from ..utils import (
     parse_iso8601,
+    sanitized_Request,
 )
 
 
@@ -54,7 +52,7 @@ class SportDeutschlandIE(InfoExtractor):
 
         api_url = 'http://proxy.vidibusdynamic.net/sportdeutschland.tv/api/permalinks/%s/%s?access_token=true' % (
             sport_id, video_id)
-        req = compat_urllib_request.Request(api_url, headers={
+        req = sanitized_Request(api_url, headers={
             'Accept': 'application/vnd.vidibus.v2.html+json',
             'Referer': url,
         })
diff --git a/youtube_dl/extractor/streamcloud.py b/youtube_dl/extractor/streamcloud.py
index d4e134015..77841b946 100644
--- a/youtube_dl/extractor/streamcloud.py
+++ b/youtube_dl/extractor/streamcloud.py
@@ -4,10 +4,8 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse,
-    compat_urllib_request,
-)
+from ..compat import compat_urllib_parse
+from ..utils import sanitized_Request
 
 
 class StreamcloudIE(InfoExtractor):
@@ -43,7 +41,7 @@ class StreamcloudIE(InfoExtractor):
         headers = {
             b'Content-Type': b'application/x-www-form-urlencoded',
         }
-        req = compat_urllib_request.Request(url, post, headers)
+        req = sanitized_Request(url, post, headers)
 
         webpage = self._download_webpage(
             req, video_id, note='Downloading video page ...')
diff --git a/youtube_dl/extractor/streamcz.py b/youtube_dl/extractor/streamcz.py
index e92b93285..d3d2b7eb7 100644
--- a/youtube_dl/extractor/streamcz.py
+++ b/youtube_dl/extractor/streamcz.py
@@ -5,11 +5,9 @@ import hashlib
 import time
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_request,
-)
 from ..utils import (
     int_or_none,
+    sanitized_Request,
 )
 
 
@@ -54,7 +52,7 @@ class StreamCZIE(InfoExtractor):
         video_id = self._match_id(url)
         api_path = '/episode/%s' % video_id
 
-        req = compat_urllib_request.Request(self._API_URL + api_path)
+        req = sanitized_Request(self._API_URL + api_path)
         req.add_header('Api-Password', _get_api_key(api_path))
         data = self._download_json(req, video_id)
 
diff --git a/youtube_dl/extractor/tapely.py b/youtube_dl/extractor/tapely.py
index 744f9db38..ed560bd24 100644
--- a/youtube_dl/extractor/tapely.py
+++ b/youtube_dl/extractor/tapely.py
@@ -4,14 +4,12 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_request,
-)
 from ..utils import (
     clean_html,
     ExtractorError,
     float_or_none,
     parse_iso8601,
+    sanitized_Request,
 )
 
 
@@ -53,7 +51,7 @@ class TapelyIE(InfoExtractor):
         display_id = mobj.group('id')
 
         playlist_url = self._API_URL.format(display_id)
-        request = compat_urllib_request.Request(playlist_url)
+        request = sanitized_Request(playlist_url)
         request.add_header('X-Requested-With', 'XMLHttpRequest')
         request.add_header('Accept', 'application/json')
         request.add_header('Referer', url)
diff --git a/youtube_dl/extractor/tube8.py b/youtube_dl/extractor/tube8.py
index c9cb69333..46ef61ff5 100644
--- a/youtube_dl/extractor/tube8.py
+++ b/youtube_dl/extractor/tube8.py
@@ -4,12 +4,10 @@ import json
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse_urlparse,
-    compat_urllib_request,
-)
+from ..compat import compat_urllib_parse_urlparse
 from ..utils import (
     int_or_none,
+    sanitized_Request,
     str_to_int,
 )
 from ..aes import aes_decrypt_text
@@ -42,7 +40,7 @@ class Tube8IE(InfoExtractor):
         video_id = mobj.group('id')
         display_id = mobj.group('display_id')
 
-        req = compat_urllib_request.Request(url)
+        req = sanitized_Request(url)
         req.add_header('Cookie', 'age_verified=1')
         webpage = self._download_webpage(req, display_id)
 
diff --git a/youtube_dl/extractor/tubitv.py b/youtube_dl/extractor/tubitv.py
index 4f86b3ee9..6d78b5dfe 100644
--- a/youtube_dl/extractor/tubitv.py
+++ b/youtube_dl/extractor/tubitv.py
@@ -5,13 +5,11 @@ import codecs
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse,
-    compat_urllib_request
-)
+from ..compat import compat_urllib_parse
 from ..utils import (
     ExtractorError,
     int_or_none,
+    sanitized_Request,
 )
 
 
@@ -44,7 +42,7 @@ class TubiTvIE(InfoExtractor):
             'password': password,
         }
         payload = compat_urllib_parse.urlencode(form_data).encode('utf-8')
-        request = compat_urllib_request.Request(self._LOGIN_URL, payload)
+        request = sanitized_Request(self._LOGIN_URL, payload)
         request.add_header('Content-Type', 'application/x-www-form-urlencoded')
         login_page = self._download_webpage(
             request, None, False, 'Wrong login info')
diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 3ec08b674..69882da63 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -11,7 +11,6 @@ from ..compat import (
     compat_str,
     compat_urllib_parse,
     compat_urllib_parse_urlparse,
-    compat_urllib_request,
     compat_urlparse,
 )
 from ..utils import (
@@ -20,6 +19,7 @@ from ..utils import (
     int_or_none,
     parse_duration,
     parse_iso8601,
+    sanitized_Request,
 )
 
 
@@ -48,7 +48,7 @@ class TwitchBaseIE(InfoExtractor):
         for cookie in self._downloader.cookiejar:
             if cookie.name == 'api_token':
                 headers['Twitch-Api-Token'] = cookie.value
-        request = compat_urllib_request.Request(url, headers=headers)
+        request = sanitized_Request(url, headers=headers)
         response = super(TwitchBaseIE, self)._download_json(request, video_id, note)
         self._handle_error(response)
         return response
@@ -80,7 +80,7 @@ class TwitchBaseIE(InfoExtractor):
         if not post_url.startswith('http'):
             post_url = compat_urlparse.urljoin(redirect_url, post_url)
 
-        request = compat_urllib_request.Request(
+        request = sanitized_Request(
             post_url, compat_urllib_parse.urlencode(encode_dict(login_form)).encode('utf-8'))
         request.add_header('Referer', redirect_url)
         response = self._download_webpage(
diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 055047340..a161f046b 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -4,13 +4,13 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_request
 from ..utils import (
     float_or_none,
     xpath_text,
     remove_end,
     int_or_none,
     ExtractorError,
+    sanitized_Request,
 )
 
 
@@ -81,7 +81,7 @@ class TwitterCardIE(InfoExtractor):
         config = None
         formats = []
         for user_agent in USER_AGENTS:
-            request = compat_urllib_request.Request(url)
+            request = sanitized_Request(url)
             request.add_header('User-Agent', user_agent)
             webpage = self._download_webpage(request, video_id)
 
diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index 365d8b4bf..825172806 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -9,6 +9,7 @@ from ..compat import (
 )
 from ..utils import (
     ExtractorError,
+    sanitized_Request,
 )
 
 
@@ -58,7 +59,7 @@ class UdemyIE(InfoExtractor):
             for header, value in headers.items():
                 url_or_request.add_header(header, value)
         else:
-            url_or_request = compat_urllib_request.Request(url_or_request, headers=headers)
+            url_or_request = sanitized_Request(url_or_request, headers=headers)
 
         response = super(UdemyIE, self)._download_json(url_or_request, video_id, note)
         self._handle_error(response)
@@ -89,7 +90,7 @@ class UdemyIE(InfoExtractor):
             'password': password.encode('utf-8'),
         })
 
-        request = compat_urllib_request.Request(
+        request = sanitized_Request(
             self._LOGIN_URL, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
         request.add_header('Referer', self._ORIGIN_URL)
         request.add_header('Origin', self._ORIGIN_URL)
diff --git a/youtube_dl/extractor/vbox7.py b/youtube_dl/extractor/vbox7.py
index 722eb5236..1e740fbe6 100644
--- a/youtube_dl/extractor/vbox7.py
+++ b/youtube_dl/extractor/vbox7.py
@@ -4,11 +4,11 @@ from __future__ import unicode_literals
 from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse,
-    compat_urllib_request,
     compat_urlparse,
 )
 from ..utils import (
     ExtractorError,
+    sanitized_Request,
 )
 
 
@@ -49,7 +49,7 @@ class Vbox7IE(InfoExtractor):
 
         info_url = "http://vbox7.com/play/magare.do"
         data = compat_urllib_parse.urlencode({'as3': '1', 'vid': video_id})
-        info_request = compat_urllib_request.Request(info_url, data)
+        info_request = sanitized_Request(info_url, data)
         info_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
         info_response = self._download_webpage(info_request, video_id, 'Downloading info webpage')
         if info_response is None:
diff --git a/youtube_dl/extractor/veoh.py b/youtube_dl/extractor/veoh.py
index 01e258e32..9633f7ffe 100644
--- a/youtube_dl/extractor/veoh.py
+++ b/youtube_dl/extractor/veoh.py
@@ -4,12 +4,10 @@ import re
 import json
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_request,
-)
 from ..utils import (
     int_or_none,
     ExtractorError,
+    sanitized_Request,
 )
 
 
@@ -110,7 +108,7 @@ class VeohIE(InfoExtractor):
         if 'class="adultwarning-container"' in webpage:
             self.report_age_confirmation()
             age_limit = 18
-            request = compat_urllib_request.Request(url)
+            request = sanitized_Request(url)
             request.add_header('Cookie', 'confirmedAdult=true')
             webpage = self._download_webpage(request, video_id)
 
diff --git a/youtube_dl/extractor/vessel.py b/youtube_dl/extractor/vessel.py
index 3c8d2a943..1a0ff3395 100644
--- a/youtube_dl/extractor/vessel.py
+++ b/youtube_dl/extractor/vessel.py
@@ -4,10 +4,10 @@ from __future__ import unicode_literals
 import json
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_request
 from ..utils import (
     ExtractorError,
     parse_iso8601,
+    sanitized_Request,
 )
 
 
@@ -33,7 +33,7 @@ class VesselIE(InfoExtractor):
     @staticmethod
     def make_json_request(url, data):
         payload = json.dumps(data).encode('utf-8')
-        req = compat_urllib_request.Request(url, payload)
+        req = sanitized_Request(url, payload)
         req.add_header('Content-Type', 'application/json; charset=utf-8')
         return req
 
diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py
index 4c0de354f..571289421 100644
--- a/youtube_dl/extractor/vevo.py
+++ b/youtube_dl/extractor/vevo.py
@@ -3,13 +3,11 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_etree_fromstring,
-    compat_urllib_request,
-)
+from ..compat import compat_etree_fromstring
 from ..utils import (
     ExtractorError,
     int_or_none,
+    sanitized_Request,
 )
 
 
@@ -73,7 +71,7 @@ class VevoIE(InfoExtractor):
     _SMIL_BASE_URL = 'http://smil.lvl3.vevo.com/'
 
     def _real_initialize(self):
-        req = compat_urllib_request.Request(
+        req = sanitized_Request(
             'http://www.vevo.com/auth', data=b'')
         webpage = self._download_webpage(
             req, None,
diff --git a/youtube_dl/extractor/viddler.py b/youtube_dl/extractor/viddler.py
index 8516a2940..40ffbad2a 100644
--- a/youtube_dl/extractor/viddler.py
+++ b/youtube_dl/extractor/viddler.py
@@ -4,9 +4,7 @@ from .common import InfoExtractor
 from ..utils import (
     float_or_none,
     int_or_none,
-)
-from ..compat import (
-    compat_urllib_request
+    sanitized_Request,
 )
 
 
@@ -65,7 +63,7 @@ class ViddlerIE(InfoExtractor):
             'http://api.viddler.com/api/v2/viddler.videos.getPlaybackDetails.json?video_id=%s&key=v0vhrt7bg2xq1vyxhkct' %
             video_id)
         headers = {'Referer': 'http://static.cdn-ec.viddler.com/js/arpeggio/v2/embed.html'}
-        request = compat_urllib_request.Request(json_url, None, headers)
+        request = sanitized_Request(json_url, None, headers)
         data = self._download_json(request, video_id)['video']
 
         formats = []
diff --git a/youtube_dl/extractor/videomega.py b/youtube_dl/extractor/videomega.py
index 78ff6310a..87aca327b 100644
--- a/youtube_dl/extractor/videomega.py
+++ b/youtube_dl/extractor/videomega.py
@@ -4,7 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_request
+from ..utils import sanitized_Request
 
 
 class VideoMegaIE(InfoExtractor):
@@ -30,7 +30,7 @@ class VideoMegaIE(InfoExtractor):
         video_id = self._match_id(url)
 
         iframe_url = 'http://videomega.tv/cdn.php?ref=%s' % video_id
-        req = compat_urllib_request.Request(iframe_url)
+        req = sanitized_Request(iframe_url)
         req.add_header('Referer', url)
         req.add_header('Cookie', 'noadvtday=0')
         webpage = self._download_webpage(req, video_id)
diff --git a/youtube_dl/extractor/viewster.py b/youtube_dl/extractor/viewster.py
index 7cf930d69..185b1c119 100644
--- a/youtube_dl/extractor/viewster.py
+++ b/youtube_dl/extractor/viewster.py
@@ -4,7 +4,6 @@ from __future__ import unicode_literals
 from .common import InfoExtractor
 from ..compat import (
     compat_HTTPError,
-    compat_urllib_request,
     compat_urllib_parse,
     compat_urllib_parse_unquote,
 )
@@ -13,6 +12,7 @@ from ..utils import (
     ExtractorError,
     int_or_none,
     parse_iso8601,
+    sanitized_Request,
     HEADRequest,
 )
 
@@ -76,7 +76,7 @@ class ViewsterIE(InfoExtractor):
     _ACCEPT_HEADER = 'application/json, text/javascript, */*; q=0.01'
 
     def _download_json(self, url, video_id, note='Downloading JSON metadata', fatal=True):
-        request = compat_urllib_request.Request(url)
+        request = sanitized_Request(url)
         request.add_header('Accept', self._ACCEPT_HEADER)
         request.add_header('Auth-token', self._AUTH_TOKEN)
         return super(ViewsterIE, self)._download_json(request, video_id, note, fatal=fatal)
diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index ddbd395c8..a63c23617 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -7,14 +7,14 @@ import hmac
 import hashlib
 import itertools
 
+from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
     int_or_none,
     parse_age_limit,
     parse_iso8601,
+    sanitized_Request,
 )
-from ..compat import compat_urllib_request
-from .common import InfoExtractor
 
 
 class VikiBaseIE(InfoExtractor):
@@ -43,7 +43,7 @@ class VikiBaseIE(InfoExtractor):
             hashlib.sha1
         ).hexdigest()
         url = self._API_URL_TEMPLATE % (query, sig)
-        return compat_urllib_request.Request(
+        return sanitized_Request(
             url, json.dumps(post_data).encode('utf-8')) if post_data else url
 
     def _call_api(self, path, video_id, note, timestamp=None, post_data=None):
diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 01960b827..d99a42a9f 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -8,11 +8,11 @@ from .common import InfoExtractor
 from ..compat import (
     compat_str,
     compat_urllib_parse,
-    compat_urllib_request,
 )
 from ..utils import (
     ExtractorError,
     orderedSet,
+    sanitized_Request,
     str_to_int,
     unescapeHTML,
     unified_strdate,
@@ -182,7 +182,7 @@ class VKIE(InfoExtractor):
             'pass': password.encode('cp1251'),
         })
 
-        request = compat_urllib_request.Request(
+        request = sanitized_Request(
             'https://login.vk.com/?act=login',
             compat_urllib_parse.urlencode(login_form).encode('utf-8'))
         login_page = self._download_webpage(
diff --git a/youtube_dl/extractor/vodlocker.py b/youtube_dl/extractor/vodlocker.py
index ccf1928b5..be0a2780f 100644
--- a/youtube_dl/extractor/vodlocker.py
+++ b/youtube_dl/extractor/vodlocker.py
@@ -2,10 +2,8 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse,
-    compat_urllib_request,
-)
+from ..compat import compat_urllib_parse
+from ..utils import sanitized_Request
 
 
 class VodlockerIE(InfoExtractor):
@@ -31,7 +29,7 @@ class VodlockerIE(InfoExtractor):
         if fields['op'] == 'download1':
             self._sleep(3, video_id)  # they do detect when requests happen too fast!
             post = compat_urllib_parse.urlencode(fields)
-            req = compat_urllib_request.Request(url, post)
+            req = sanitized_Request(url, post)
             req.add_header('Content-type', 'application/x-www-form-urlencoded')
             webpage = self._download_webpage(
                 req, video_id, 'Downloading video page')
diff --git a/youtube_dl/extractor/voicerepublic.py b/youtube_dl/extractor/voicerepublic.py
index 254383d6c..93d15a556 100644
--- a/youtube_dl/extractor/voicerepublic.py
+++ b/youtube_dl/extractor/voicerepublic.py
@@ -3,14 +3,12 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_request,
-    compat_urlparse,
-)
+from ..compat import compat_urlparse
 from ..utils import (
     ExtractorError,
     determine_ext,
     int_or_none,
+    sanitized_Request,
 )
 
 
@@ -37,7 +35,7 @@ class VoiceRepublicIE(InfoExtractor):
     def _real_extract(self, url):
         display_id = self._match_id(url)
 
-        req = compat_urllib_request.Request(
+        req = sanitized_Request(
             compat_urlparse.urljoin(url, '/talks/%s' % display_id))
         # Older versions of Firefox get redirected to an "upgrade browser" page
         req.add_header('User-Agent', 'youtube-dl')
diff --git a/youtube_dl/extractor/wistia.py b/youtube_dl/extractor/wistia.py
index 13a079151..fdb16d91c 100644
--- a/youtube_dl/extractor/wistia.py
+++ b/youtube_dl/extractor/wistia.py
@@ -1,8 +1,10 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_request
-from ..utils import ExtractorError
+from ..utils import (
+    ExtractorError,
+    sanitized_Request,
+)
 
 
 class WistiaIE(InfoExtractor):
@@ -23,7 +25,7 @@ class WistiaIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        request = compat_urllib_request.Request(self._API_URL.format(video_id))
+        request = sanitized_Request(self._API_URL.format(video_id))
         request.add_header('Referer', url)  # Some videos require this.
         data_json = self._download_json(request, video_id)
         if data_json.get('error'):
diff --git a/youtube_dl/extractor/xfileshare.py b/youtube_dl/extractor/xfileshare.py
index 952515c98..a3236e66c 100644
--- a/youtube_dl/extractor/xfileshare.py
+++ b/youtube_dl/extractor/xfileshare.py
@@ -4,14 +4,12 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse,
-    compat_urllib_request,
-)
+from ..compat import compat_urllib_parse
 from ..utils import (
     ExtractorError,
     encode_dict,
     int_or_none,
+    sanitized_Request,
 )
 
 
@@ -106,7 +104,7 @@ class XFileShareIE(InfoExtractor):
 
             post = compat_urllib_parse.urlencode(encode_dict(fields))
 
-            req = compat_urllib_request.Request(url, post)
+            req = sanitized_Request(url, post)
             req.add_header('Content-type', 'application/x-www-form-urlencoded')
 
             webpage = self._download_webpage(req, video_id, 'Downloading video page')
diff --git a/youtube_dl/extractor/xtube.py b/youtube_dl/extractor/xtube.py
index 779e4f46a..a1fe24050 100644
--- a/youtube_dl/extractor/xtube.py
+++ b/youtube_dl/extractor/xtube.py
@@ -3,12 +3,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_request,
-    compat_urllib_parse_unquote,
-)
+from ..compat import compat_urllib_parse_unquote
 from ..utils import (
     parse_duration,
+    sanitized_Request,
     str_to_int,
 )
 
@@ -32,7 +30,7 @@ class XTubeIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        req = compat_urllib_request.Request(url)
+        req = sanitized_Request(url)
         req.add_header('Cookie', 'age_verified=1')
         webpage = self._download_webpage(req, video_id)
 
diff --git a/youtube_dl/extractor/xvideos.py b/youtube_dl/extractor/xvideos.py
index 5dcf2fdd1..710ad5041 100644
--- a/youtube_dl/extractor/xvideos.py
+++ b/youtube_dl/extractor/xvideos.py
@@ -3,14 +3,12 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse_unquote,
-    compat_urllib_request,
-)
+from ..compat import compat_urllib_parse_unquote
 from ..utils import (
     clean_html,
     ExtractorError,
     determine_ext,
+    sanitized_Request,
 )
 
 
@@ -48,7 +46,7 @@ class XVideosIE(InfoExtractor):
             'url': video_url,
         }]
 
-        android_req = compat_urllib_request.Request(url)
+        android_req = sanitized_Request(url)
         android_req.add_header('User-Agent', self._ANDROID_USER_AGENT)
         android_webpage = self._download_webpage(android_req, video_id, fatal=False)
 
diff --git a/youtube_dl/extractor/yandexmusic.py b/youtube_dl/extractor/yandexmusic.py
index 08dc81f3a..d3cc1a29f 100644
--- a/youtube_dl/extractor/yandexmusic.py
+++ b/youtube_dl/extractor/yandexmusic.py
@@ -8,11 +8,11 @@ from .common import InfoExtractor
 from ..compat import (
     compat_str,
     compat_urllib_parse,
-    compat_urllib_request,
 )
 from ..utils import (
     int_or_none,
     float_or_none,
+    sanitized_Request,
 )
 
 
@@ -154,7 +154,7 @@ class YandexMusicPlaylistIE(YandexMusicPlaylistBaseIE):
         if len(tracks) < len(track_ids):
             present_track_ids = set([compat_str(track['id']) for track in tracks if track.get('id')])
             missing_track_ids = set(map(compat_str, track_ids)) - set(present_track_ids)
-            request = compat_urllib_request.Request(
+            request = sanitized_Request(
                 'https://music.yandex.ru/handlers/track-entries.jsx',
                 compat_urllib_parse.urlencode({
                     'entries': ','.join(missing_track_ids),
diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index 2e81d9223..69ecc837a 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -4,12 +4,13 @@ from __future__ import unicode_literals
 import base64
 
 from .common import InfoExtractor
-from ..utils import ExtractorError
-
 from ..compat import (
     compat_urllib_parse,
     compat_ord,
-    compat_urllib_request,
+)
+from ..utils import (
+    ExtractorError,
+    sanitized_Request,
 )
 
 
@@ -187,7 +188,7 @@ class YoukuIE(InfoExtractor):
         video_id = self._match_id(url)
 
         def retrieve_data(req_url, note):
-            req = compat_urllib_request.Request(req_url)
+            req = sanitized_Request(req_url)
 
             cn_verification_proxy = self._downloader.params.get('cn_verification_proxy')
             if cn_verification_proxy:
diff --git a/youtube_dl/extractor/youporn.py b/youtube_dl/extractor/youporn.py
index 9bf8d1eeb..dd724085a 100644
--- a/youtube_dl/extractor/youporn.py
+++ b/youtube_dl/extractor/youporn.py
@@ -3,9 +3,9 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_request
 from ..utils import (
     int_or_none,
+    sanitized_Request,
     str_to_int,
     unescapeHTML,
     unified_strdate,
@@ -63,7 +63,7 @@ class YouPornIE(InfoExtractor):
         video_id = mobj.group('id')
         display_id = mobj.group('display_id')
 
-        request = compat_urllib_request.Request(url)
+        request = sanitized_Request(url)
         request.add_header('Cookie', 'age_verified=1')
         webpage = self._download_webpage(request, display_id)
 
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 0246050c2..cfe9eed55 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -20,7 +20,6 @@ from ..compat import (
     compat_urllib_parse_unquote,
     compat_urllib_parse_unquote_plus,
     compat_urllib_parse_urlparse,
-    compat_urllib_request,
     compat_urlparse,
     compat_str,
 )
@@ -35,6 +34,7 @@ from ..utils import (
     orderedSet,
     parse_duration,
     remove_start,
+    sanitized_Request,
     smuggle_url,
     str_to_int,
     unescapeHTML,
@@ -114,7 +114,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
 
         login_data = compat_urllib_parse.urlencode(encode_dict(login_form_strs)).encode('ascii')
 
-        req = compat_urllib_request.Request(self._LOGIN_URL, login_data)
+        req = sanitized_Request(self._LOGIN_URL, login_data)
         login_results = self._download_webpage(
             req, None,
             note='Logging in', errnote='unable to log in', fatal=False)
@@ -147,7 +147,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
 
             tfa_data = compat_urllib_parse.urlencode(encode_dict(tfa_form_strs)).encode('ascii')
 
-            tfa_req = compat_urllib_request.Request(self._TWOFACTOR_URL, tfa_data)
+            tfa_req = sanitized_Request(self._TWOFACTOR_URL, tfa_data)
             tfa_results = self._download_webpage(
                 tfa_req, None,
                 note='Submitting TFA code', errnote='unable to submit tfa', fatal=False)

From 20e98bf6c0df1dc10651d5b835d42385db9bbdc7 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 23 Nov 2015 18:07:58 +0100
Subject: [PATCH 3189/6219] release 2015.11.23

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 2baf1ac42..3e2c0b4ac 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.11.21'
+__version__ = '2015.11.23'

From 032f2f260f7ef7a04d1499b26b97c1643fe993b0 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 24 Nov 2015 03:38:46 +0100
Subject: [PATCH 3190/6219] README: Document which other programs may be
 helpful (Fixes #7621)

---
 README.md | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/README.md b/README.md
index b286651cd..957059f8c 100644
--- a/README.md
+++ b/README.md
@@ -534,6 +534,12 @@ Most people asking this question are not aware that youtube-dl now defaults to d
 
 Apparently YouTube requires you to pass a CAPTCHA test if you download too much. We're [considering to provide a way to let you solve the CAPTCHA](https://github.com/rg3/youtube-dl/issues/154), but at the moment, your best course of action is pointing a webbrowser to the youtube URL, solving the CAPTCHA, and restart youtube-dl.
 
+### Do I need any other programs?
+
+youtube-dl works fine on its own on most sites. However, if you want to convert video/audio, you'll need [avconv](https://libav.org/) or [ffmpeg](https://www.ffmpeg.org/). On some sites - most notably YouTube - videos can be retrieved in a higher quality format without sound. youtube-dl will detect whether avconv/ffmpeg is present and automatically pick the best option.
+
+Some videos or video formats can also be only downloaded when [rtmpdump](https://rtmpdump.mplayerhq.hu/) is installed.
+
 ### I have downloaded a video but how can I play it?
 
 Once the video is fully downloaded, use any video player, such as [vlc](http://www.videolan.org) or [mplayer](http://www.mplayerhq.hu/).

From 4c7d816dd70a7b10dc096d815da55d41c46f9b59 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 24 Nov 2015 07:45:02 +0100
Subject: [PATCH 3191/6219] [jsinterp] Adapt to updated YouTube code generation
 (Fixes #7623, fixes #7624, fixes #7625, fixes #7626)

---
 youtube_dl/jsinterp.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index 9bc855144..2191e8b89 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -214,7 +214,7 @@ class JSInterpreter(object):
         obj = {}
         obj_m = re.search(
             (r'(?:var\s+)?%s\s*=\s*\{' % re.escape(objname)) +
-            r'\s*(?P<fields>([a-zA-Z$0-9]+\s*:\s*function\(.*?\)\s*\{.*?\})*)' +
+            r'\s*(?P<fields>([a-zA-Z$0-9]+\s*:\s*function\(.*?\)\s*\{.*?\}(?:,\s*)?)*)' +
             r'\}\s*;',
             self.code)
         fields = obj_m.group('fields')

From ba7a92b0ced8cce0c800e287777f1654b728f17c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 24 Nov 2015 07:46:38 +0100
Subject: [PATCH 3192/6219] release 2015.11.24

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 3e2c0b4ac..5ecff39fb 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.11.23'
+__version__ = '2015.11.24'

From 7689413e42334ff79e671a4c90869077e9f9835b Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Tue, 24 Nov 2015 23:06:21 +0600
Subject: [PATCH 3193/6219] [README.md] Mention mplayer and mpv in "other
 programs" question

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 957059f8c..b85e08e78 100644
--- a/README.md
+++ b/README.md
@@ -538,7 +538,7 @@ Apparently YouTube requires you to pass a CAPTCHA test if you download too much.
 
 youtube-dl works fine on its own on most sites. However, if you want to convert video/audio, you'll need [avconv](https://libav.org/) or [ffmpeg](https://www.ffmpeg.org/). On some sites - most notably YouTube - videos can be retrieved in a higher quality format without sound. youtube-dl will detect whether avconv/ffmpeg is present and automatically pick the best option.
 
-Some videos or video formats can also be only downloaded when [rtmpdump](https://rtmpdump.mplayerhq.hu/) is installed.
+Videos or video formats streamed via RTMP protocol can only be downloaded when [rtmpdump](https://rtmpdump.mplayerhq.hu/) is installed. Downloading MMS and RTSP videos requires either [mplayer](http://mplayerhq.hu/) or [mpv](https://mpv.io/) to be installed.
 
 ### I have downloaded a video but how can I play it?
 

From 527ca1da4f778c5ad13ff3c74a8d311adb9aaff3 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 26 Nov 2015 21:24:10 +0100
Subject: [PATCH 3194/6219] [audimedia] Add new extractor(closes #7654)

---
 youtube_dl/extractor/__init__.py  |  1 +
 youtube_dl/extractor/audimedia.py | 80 +++++++++++++++++++++++++++++++
 2 files changed, 81 insertions(+)
 create mode 100644 youtube_dl/extractor/audimedia.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 947b83683..f17c22691 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -38,6 +38,7 @@ from .arte import (
 )
 from .atresplayer import AtresPlayerIE
 from .atttechchannel import ATTTechChannelIE
+from .audimedia import AudiMediaIE
 from .audiomack import AudiomackIE, AudiomackAlbumIE
 from .azubu import AzubuIE
 from .baidu import BaiduVideoIE
diff --git a/youtube_dl/extractor/audimedia.py b/youtube_dl/extractor/audimedia.py
new file mode 100644
index 000000000..cad503510
--- /dev/null
+++ b/youtube_dl/extractor/audimedia.py
@@ -0,0 +1,80 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    parse_iso8601,
+    sanitized_Request,
+)
+
+
+class AudiMediaIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?audimedia\.tv/(?:en|de)/vid/(?P<id>[^/?#]+)'
+    _TEST = {
+        'url': 'https://audimedia.tv/en/vid/60-seconds-of-audi-sport-104-2015-wec-bahrain-rookie-test',
+        'md5': '79a8b71c46d49042609795ab59779b66',
+        'info_dict': {
+            'id': '1564',
+            'ext': 'mp4',
+            'title': '60 Seconds of Audi Sport 104/2015 - WEC Bahrain, Rookie Test',
+            'description': 'md5:60e5d30a78ced725f7b8d34370762941',
+            'upload_date': '20151124',
+            'timestamp': 1448354940,
+            'duration': 74022,
+            'view_count': int,
+        }
+    }
+    # extracted from https://audimedia.tv/assets/embed/embedded-player.js (dataSourceAuthToken)
+    _AUTH_TOKEN = 'e25b42847dba18c6c8816d5d8ce94c326e06823ebf0859ed164b3ba169be97f2'
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+        
+        raw_payload = self._search_regex(r'<script[^>]+class="amtv-embed"[^>]+id="([^"]+)"', webpage, 'raw payload');
+        _, stage_mode, video_id, lang = raw_payload.split('-')
+
+        # TODO: handle s and e stage_mode (live streams and ended live streams)
+        if stage_mode not in ('s', 'e'):
+            request = sanitized_Request(
+                'https://audimedia.tv/api/video/v1/videos/%s?embed[]=video_versions&embed[]=thumbnail_image&where[content_language_iso]=%s' % (video_id, lang),
+                headers={'X-Auth-Token': self._AUTH_TOKEN})
+            json_data = self._download_json(request, video_id)['results']
+            formats = []
+
+            stream_url_hls = json_data.get('stream_url_hls')
+            if stream_url_hls:
+                m3u8_formats = self._extract_m3u8_formats(stream_url_hls, video_id, 'mp4', entry_protocol='m3u8_native', m3u8_id='hls', fatal=False)
+                if m3u8_formats:
+                    formats.extend(m3u8_formats)
+
+            stream_url_hds = json_data.get('stream_url_hds')
+            if stream_url_hds:
+                f4m_formats = self._extract_f4m_formats(json_data.get('stream_url_hds') + '?hdcore=3.4.0', video_id, -1, f4m_id='hds', fatal=False)
+                if f4m_formats:
+                    formats.extend(f4m_formats)
+
+            for video_version in json_data.get('video_versions'):
+                video_version_url = video_version.get('download_url') or video_version.get('stream_url')
+                if not video_version_url:
+                    continue
+                formats.append({
+                    'url': video_version_url,
+                    'width': int_or_none(video_version.get('width')),
+                    'height': int_or_none(video_version.get('height')),
+                    'abr': int_or_none(video_version.get('audio_bitrate')),
+                    'vbr': int_or_none(video_version.get('video_bitrate')),
+                })
+            self._sort_formats(formats)
+
+            return {
+                'id': video_id,
+                'title': json_data['title'],
+                'description': json_data.get('subtitle'),
+                'thumbnail': json_data.get('thumbnail_image', {}).get('file'),
+                'timestamp': parse_iso8601(json_data.get('publication_date')),
+                'duration': int_or_none(json_data.get('duration')),
+                'view_count': int_or_none(json_data.get('view_count')),
+                'formats': formats,
+            }

From 60121eb5141ab8ad527ec94373a180ecd5030d78 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 26 Nov 2015 22:43:31 +0100
Subject: [PATCH 3195/6219] [gameinformer] Add new extractor

---
 youtube_dl/extractor/__init__.py     |  1 +
 youtube_dl/extractor/gameinformer.py | 42 ++++++++++++++++++++++++++++
 2 files changed, 43 insertions(+)
 create mode 100644 youtube_dl/extractor/gameinformer.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 947b83683..4fbe57208 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -200,6 +200,7 @@ from .freesound import FreesoundIE
 from .freespeech import FreespeechIE
 from .freevideo import FreeVideoIE
 from .funnyordie import FunnyOrDieIE
+from .gameinformer import GameInformerIE
 from .gamekings import GamekingsIE
 from .gameone import (
     GameOneIE,
diff --git a/youtube_dl/extractor/gameinformer.py b/youtube_dl/extractor/gameinformer.py
new file mode 100644
index 000000000..8bb764703
--- /dev/null
+++ b/youtube_dl/extractor/gameinformer.py
@@ -0,0 +1,42 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..compat import compat_str
+from ..utils import int_or_none
+
+
+class GameInformerIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?gameinformer\.com/(?:[^/]+/)*(?P<id>.+)\.aspx'
+    _TEST = {
+        'url': 'http://www.gameinformer.com/b/features/archive/2015/09/26/replay-animal-crossing.aspx',
+        'md5': 'TODO: md5 sum of the first 10241 bytes of the video file (use --test)',
+        'info_dict': {
+            'id': '4515472681001',
+            'ext': 'm3u8',
+            'title': 'Replay - Animal Crossing',
+            'description': 'md5:2e211891b215c85d061adc7a4dd2d930',
+            'timestamp': 1443457610706,
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+
+        bc_api_url = self._search_regex(r"getVideo\('([^']+)'", webpage, 'brightcove api url')
+        json_data = self._download_json(bc_api_url + '&video_fields=id,name,shortDescription,publishedDate,videoStillURL,length,IOSRenditions', display_id)
+
+        return {
+            'id': compat_str(json_data['id']),
+            'display_id': display_id,
+            'url': json_data['IOSRenditions'][0]['url'],
+            'title': json_data['name'],
+            'description': json_data.get('shortDescription'),
+            'timestamp': int_or_none(json_data.get('publishedDate')),
+            'duration': int_or_none(json_data.get('length')),
+        }

From 9bff48a0e7128b4d274a5dab4aaac8da0c900ede Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 27 Nov 2015 21:24:39 +0800
Subject: [PATCH 3196/6219] [options] Clarify --list-formats needs videos
 (closes #7669)

---
 youtube_dl/options.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 079fe7e8a..359e8d300 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -338,7 +338,7 @@ def parseOpts(overrideArguments=None):
     video_format.add_option(
         '-F', '--list-formats',
         action='store_true', dest='listformats',
-        help='List all available formats')
+        help='List all available formats of specified videos')
     video_format.add_option(
         '--youtube-include-dash-manifest',
         action='store_true', dest='youtube_include_dash_manifest', default=True,

From 9d0e3668804fe91e77275e6bd859cbd0d784ab33 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 27 Nov 2015 21:37:45 +0800
Subject: [PATCH 3197/6219] [downloader/hls] Remove Accept-encoding from
 headers passed to ffmpeg

Fails for Youtube Gaming live streams (#7671)
---
 youtube_dl/downloader/hls.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index 9a83a73dd..92765a3f9 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -35,7 +35,7 @@ class HlsFD(FileDownloader):
             # [http @ 00000000003d2fa0] No trailing CRLF found in HTTP header.
             args += [
                 '-headers',
-                ''.join('%s: %s\r\n' % (key, val) for key, val in info_dict['http_headers'].items())]
+                ''.join('%s: %s\r\n' % (key, val) for key, val in info_dict['http_headers'].items() if key.lower() != 'accept-encoding')]
 
         args += ['-i', url, '-f', 'mp4', '-c', 'copy', '-bsf:a', 'aac_adtstoasc']
 

From bb6ac8369858e79b04e30e6ea25016500083628f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 27 Nov 2015 16:32:51 +0100
Subject: [PATCH 3198/6219] release 2015.11.27

---
 README.md             | 3 ++-
 youtube_dl/version.py | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index b85e08e78..df419abe8 100644
--- a/README.md
+++ b/README.md
@@ -319,7 +319,8 @@ which means you can modify it, redistribute it or use it however you like.
     --all-formats                    Download all available video formats
     --prefer-free-formats            Prefer free video formats unless a specific
                                      one is requested
-    -F, --list-formats               List all available formats
+    -F, --list-formats               List all available formats of specified
+                                     videos
     --youtube-skip-dash-manifest     Do not download the DASH manifests and
                                      related data on YouTube videos
     --merge-output-format FORMAT     If a merge is required (e.g.
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 5ecff39fb..288b76e28 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.11.24'
+__version__ = '2015.11.27'

From 78a55d7a28662a65cba36c48eb69ecc4b150b336 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 27 Nov 2015 16:39:59 +0100
Subject: [PATCH 3199/6219] release 2015.11.27.1

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 288b76e28..bd0de9f53 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.11.27'
+__version__ = '2015.11.27.1'

From 313dfc45f500db3acc348d58f431197ce7c153a3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Lalinsk=C3=BD?= <lukas@oxygene.sk>
Date: Sat, 28 Nov 2015 01:07:07 +0100
Subject: [PATCH 3200/6219] [youtube] Ignore yt:stretch with zero width/height

---
 youtube_dl/extractor/youtube.py | 27 +++++++++++++++++++++++----
 1 file changed, 23 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index cfe9eed55..726b5ba0a 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -714,6 +714,22 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             'url': 'https://www.youtube.com/watch?v=Ms7iBXnlUO8',
             'only_matching': True,
         },
+        {
+            # Video with yt:stretch=17:0
+            'url': 'https://www.youtube.com/watch?v=Q39EVAstoRM',
+            'info_dict': {
+                'id': 'Q39EVAstoRM',
+                'ext': 'mp4',
+                'title': 'Clash Of Clans#14 Dicas De Ataque Para CV 4',
+                'description': 'md5:ee18a25c350637c8faff806845bddee9',
+                'upload_date': '20151107',
+                'uploader_id': 'UCCr7TALkRbo3EtFzETQF1LA',
+                'uploader': 'CH GAMER DROID',
+            },
+            'params': {
+                'skip_download': True,
+            },
+        },
     ]
 
     def __init__(self, *args, **kwargs):
@@ -1496,10 +1512,13 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             r'<meta\s+property="og:video:tag".*?content="yt:stretch=(?P<w>[0-9]+):(?P<h>[0-9]+)">',
             video_webpage)
         if stretched_m:
-            ratio = float(stretched_m.group('w')) / float(stretched_m.group('h'))
-            for f in formats:
-                if f.get('vcodec') != 'none':
-                    f['stretched_ratio'] = ratio
+            w = float(stretched_m.group('w'))
+            h = float(stretched_m.group('h'))
+            if w > 0 and h > 0:
+                ratio = float(stretched_m.group('w')) / float(stretched_m.group('h'))
+                for f in formats:
+                    if f.get('vcodec') != 'none':
+                        f['stretched_ratio'] = ratio
 
         self._sort_formats(formats)
 

From 7ac40086f5f9fdc01c92d1b76acb742c661cf0ff Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 28 Nov 2015 08:44:13 +0600
Subject: [PATCH 3201/6219] [dbtv] Expand _VALID_URL (Closes #7645)

---
 youtube_dl/extractor/dbtv.py | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/dbtv.py b/youtube_dl/extractor/dbtv.py
index 212217625..133cdc50b 100644
--- a/youtube_dl/extractor/dbtv.py
+++ b/youtube_dl/extractor/dbtv.py
@@ -13,8 +13,8 @@ from ..utils import (
 
 
 class DBTVIE(InfoExtractor):
-    _VALID_URL = r'http://dbtv\.no/(?P<id>[0-9]+)#(?P<display_id>.+)'
-    _TEST = {
+    _VALID_URL = r'https?://(?:www\.)?dbtv\.no/(?:(?:lazyplayer|player)/)?(?P<id>[0-9]+)(?:#(?P<display_id>.+))?'
+    _TESTS = [{
         'url': 'http://dbtv.no/3649835190001#Skulle_teste_ut_fornøyelsespark,_men_kollegaen_var_bare_opptatt_av_bikinikroppen',
         'md5': 'b89953ed25dacb6edb3ef6c6f430f8bc',
         'info_dict': {
@@ -30,12 +30,18 @@ class DBTVIE(InfoExtractor):
             'view_count': int,
             'categories': list,
         }
-    }
+    }, {
+        'url': 'http://dbtv.no/3649835190001',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.dbtv.no/lazyplayer/4631135248001',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
-        display_id = mobj.group('display_id')
+        display_id = mobj.group('display_id') or video_id
 
         data = self._download_json(
             'http://api.dbtv.no/discovery/%s' % video_id, display_id)

From 4b3fbafdd261405c12e044a59d20554838c9c4c2 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 28 Nov 2015 14:14:20 +0800
Subject: [PATCH 3202/6219] [options] Changed wording for --list-formats

As proposed by @dstftw at 9bff48a0e7128b4d274a5dab4aaac8da0c900ede
---
 youtube_dl/options.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 359e8d300..c46e136bf 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -338,7 +338,7 @@ def parseOpts(overrideArguments=None):
     video_format.add_option(
         '-F', '--list-formats',
         action='store_true', dest='listformats',
-        help='List all available formats of specified videos')
+        help='List all available formats of requested videos')
     video_format.add_option(
         '--youtube-include-dash-manifest',
         action='store_true', dest='youtube_include_dash_manifest', default=True,

From 41f24c321d04108a32f457d5d5445f2cfce705a8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Lalinsk=C3=BD?= <lukas@oxygene.sk>
Date: Sat, 28 Nov 2015 08:16:46 +0100
Subject: [PATCH 3203/6219] [youtube] Use the existing `w` and `h` variables

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 726b5ba0a..9da8d4bc5 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1515,7 +1515,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             w = float(stretched_m.group('w'))
             h = float(stretched_m.group('h'))
             if w > 0 and h > 0:
-                ratio = float(stretched_m.group('w')) / float(stretched_m.group('h'))
+                ratio = w / h
                 for f in formats:
                     if f.get('vcodec') != 'none':
                         f['stretched_ratio'] = ratio

From 0cc71785461d273294f2ab96d50854172ba29c04 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 28 Nov 2015 11:48:18 +0100
Subject: [PATCH 3204/6219] [skynewsarabia] Add new extractor

---
 youtube_dl/extractor/__init__.py      |   4 +
 youtube_dl/extractor/skynewsarabia.py | 115 ++++++++++++++++++++++++++
 2 files changed, 119 insertions(+)
 create mode 100644 youtube_dl/extractor/skynewsarabia.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 947b83683..d981acb8e 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -554,6 +554,10 @@ from .shahid import ShahidIE
 from .shared import SharedIE
 from .sharesix import ShareSixIE
 from .sina import SinaIE
+from .skynewsarabia import (
+    SkyNewsArabiaIE,
+    SkyNewsArabiaArticleIE,
+)
 from .slideshare import SlideshareIE
 from .slutload import SlutloadIE
 from .smotri import (
diff --git a/youtube_dl/extractor/skynewsarabia.py b/youtube_dl/extractor/skynewsarabia.py
new file mode 100644
index 000000000..3499b8cd5
--- /dev/null
+++ b/youtube_dl/extractor/skynewsarabia.py
@@ -0,0 +1,115 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..compat import compat_str
+from ..utils import (
+    parse_iso8601,
+    parse_duration,
+)
+
+
+class SkyNewArabiaBaseIE(InfoExtractor):
+    _IMAGE_BASE_URL = 'http://www.skynewsarabia.com/web/images'
+
+    def _call_api(self, path, value):
+        return self._download_json('http://api.skynewsarabia.com/web/rest/v2/%s/%s.json' % (path, value), value)
+
+    def _get_limelight_media_id(self, url):
+        return self._search_regex(r'/media/[^/]+/([a-z0-9]{32})', url, 'limelight media id')
+
+    def _get_image_url(self, image_path_template, width='1600', height='1200'):
+        return self._IMAGE_BASE_URL + image_path_template.format(width=width, height=height)
+
+    def _extract_video_info(self, video_data):
+        video_id = compat_str(video_data['id'])
+        return {
+            '_type': 'url_transparent',
+            'url': 'limelight:media:%s' % self._get_limelight_media_id(video_data['videoUrl'][0]['url']),
+            'id': video_id,
+            'title': video_data['headline'],
+            'description': video_data.get('summary'),
+            'thumbnail': self._get_image_url(video_data['mediaAsset']['imageUrl']),
+            'timestamp': parse_iso8601(video_data.get('date')),
+            'duration': parse_duration(video_data.get('runTime')),
+            'tags': video_data.get('tags'),
+            'categories': [video_data.get('topicTitle')],
+            'webpage_url': 'http://www.skynewsarabia.com/web/video/%s' % video_id,
+            'ie_key': 'LimelightMedia',
+        }
+
+
+class SkyNewsArabiaIE(SkyNewArabiaBaseIE):
+    _IE_NAME = 'skynewsarabia:video'
+    _VALID_URL = r'https?://(?:www\.)?skynewsarabia\.com/web/video/(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'http://www.skynewsarabia.com/web/video/794902/%D9%86%D8%B5%D9%81-%D9%85%D9%84%D9%8A%D9%88%D9%86-%D9%85%D8%B5%D8%A8%D8%A7%D8%AD-%D8%B4%D8%AC%D8%B1%D8%A9-%D9%83%D8%B1%D9%8A%D8%B3%D9%85%D8%A7%D8%B3',
+        'info_dict': {
+            'id': '794902',
+            'ext': 'flv',
+            'title': 'نصف مليون مصباح على شجرة كريسماس',
+            'description': 'md5:22f1b27f0850eeb10c7e59b1f16eb7c6',
+            'upload_date': '20151128',
+            'timestamp': 1448697198,
+            'duration': 2119,
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        video_data = self._call_api('video', video_id)
+        return self._extract_video_info(video_data)
+
+
+class SkyNewsArabiaArticleIE(SkyNewArabiaBaseIE):
+    _IE_NAME = 'skynewsarabia:video'
+    _VALID_URL = r'https?://(?:www\.)?skynewsarabia\.com/web/article/(?P<id>[0-9]+)'
+    _TESTS = [{
+        'url': 'http://www.skynewsarabia.com/web/article/794549/%D8%A7%D9%94%D8%AD%D8%AF%D8%A7%D8%AB-%D8%A7%D9%84%D8%B4%D8%B1%D9%82-%D8%A7%D9%84%D8%A7%D9%94%D9%88%D8%B3%D8%B7-%D8%AE%D8%B1%D9%8A%D8%B7%D8%A9-%D8%A7%D9%84%D8%A7%D9%94%D9%84%D8%B9%D8%A7%D8%A8-%D8%A7%D9%84%D8%B0%D9%83%D9%8A%D8%A9',
+        'info_dict': {
+            'id': '794549',
+            'ext': 'flv',
+            'title': 'بالفيديو.. ألعاب ذكية تحاكي واقع المنطقة',
+            'description': 'md5:0c373d29919a851e080ee4edd0c5d97f',
+            'upload_date': '20151126',
+            'timestamp': 1448559336,
+            'duration': 281.6,
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        },
+    }, {
+        'url': 'http://www.skynewsarabia.com/web/article/794844/%D8%A7%D8%B3%D8%AA%D9%87%D8%AF%D8%A7%D9%81-%D9%82%D9%88%D8%A7%D8%B1%D8%A8-%D8%A7%D9%94%D8%B3%D9%84%D8%AD%D8%A9-%D9%84%D9%85%D9%8A%D9%84%D9%8A%D8%B4%D9%8A%D8%A7%D8%AA-%D8%A7%D9%84%D8%AD%D9%88%D8%AB%D9%8A-%D9%88%D8%B5%D8%A7%D9%84%D8%AD',
+        'info_dict': {
+            'id': '794844',
+            'title': 'إحباط تهريب أسلحة لميليشيات الحوثي وصالح بجنوب اليمن',
+            'description': 'md5:5c927b8b2e805796e7f693538d96fc7e',
+        },
+        'playlist_mincount': 2,
+    }]
+
+    def _real_extract(self, url):
+        article_id = self._match_id(url)
+        article_data = self._call_api('article', article_id)
+        media_asset = article_data['mediaAsset']
+        if media_asset['type'] == 'VIDEO':
+            return {
+                '_type': 'url_transparent',
+                'url': 'limelight:media:%s' % self._get_limelight_media_id(media_asset['videoUrl'][0]['url']),
+                'id': article_id,
+                'title': article_data['headline'],
+                'description': article_data.get('summary'),
+                'thumbnail': self._get_image_url(media_asset['imageUrl']),
+                'timestamp': parse_iso8601(article_data.get('date')),
+                'tags': article_data.get('tags'),
+                'categories': [article_data.get('topicTitle')],
+                'webpage_url': url,
+                'ie_key': 'LimelightMedia',
+            }
+        entries = [self._extract_video_info(item) for item in article_data.get('inlineItems', []) if item['type'] == 'VIDEO']
+        return self.playlist_result(entries, article_id, article_data['headline'], article_data.get('summary'))

From 4975650e002f3b9c20cc54940684c39c8f68e786 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 28 Nov 2015 12:20:39 +0100
Subject: [PATCH 3205/6219] [skynewsarabia] fix IE_NAME

---
 youtube_dl/extractor/skynewsarabia.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/skynewsarabia.py b/youtube_dl/extractor/skynewsarabia.py
index 3499b8cd5..d1c737020 100644
--- a/youtube_dl/extractor/skynewsarabia.py
+++ b/youtube_dl/extractor/skynewsarabia.py
@@ -40,7 +40,7 @@ class SkyNewArabiaBaseIE(InfoExtractor):
 
 
 class SkyNewsArabiaIE(SkyNewArabiaBaseIE):
-    _IE_NAME = 'skynewsarabia:video'
+    IE_NAME = 'skynewsarabia:video'
     _VALID_URL = r'https?://(?:www\.)?skynewsarabia\.com/web/video/(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://www.skynewsarabia.com/web/video/794902/%D9%86%D8%B5%D9%81-%D9%85%D9%84%D9%8A%D9%88%D9%86-%D9%85%D8%B5%D8%A8%D8%A7%D8%AD-%D8%B4%D8%AC%D8%B1%D8%A9-%D9%83%D8%B1%D9%8A%D8%B3%D9%85%D8%A7%D8%B3',
@@ -66,7 +66,7 @@ class SkyNewsArabiaIE(SkyNewArabiaBaseIE):
 
 
 class SkyNewsArabiaArticleIE(SkyNewArabiaBaseIE):
-    _IE_NAME = 'skynewsarabia:video'
+    IE_NAME = 'skynewsarabia:video'
     _VALID_URL = r'https?://(?:www\.)?skynewsarabia\.com/web/article/(?P<id>[0-9]+)'
     _TESTS = [{
         'url': 'http://www.skynewsarabia.com/web/article/794549/%D8%A7%D9%94%D8%AD%D8%AF%D8%A7%D8%AB-%D8%A7%D9%84%D8%B4%D8%B1%D9%82-%D8%A7%D9%84%D8%A7%D9%94%D9%88%D8%B3%D8%B7-%D8%AE%D8%B1%D9%8A%D8%B7%D8%A9-%D8%A7%D9%84%D8%A7%D9%94%D9%84%D8%B9%D8%A7%D8%A8-%D8%A7%D9%84%D8%B0%D9%83%D9%8A%D8%A9',

From 5faf9fed7e1c7922578467cfd48db5867ef9b91b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 28 Nov 2015 18:50:21 +0600
Subject: [PATCH 3206/6219] [youtube] Clarify rationale for yt:stretch
 validation

---
 youtube_dl/extractor/youtube.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 9da8d4bc5..1c2420a33 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1514,6 +1514,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         if stretched_m:
             w = float(stretched_m.group('w'))
             h = float(stretched_m.group('h'))
+            # yt:stretch may hold invalid ratio data (e.g. for Q39EVAstoRM ratio is 17:0).
+            # We will only process correct ratios.
             if w > 0 and h > 0:
                 ratio = w / h
                 for f in formats:

From 9945c4994c8cdbfa9971df942332f2c75bc2734d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 28 Nov 2015 20:21:03 +0600
Subject: [PATCH 3207/6219] Credit @reiv for soundcloud:search

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index f465d20ed..cdb56de3b 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -146,3 +146,4 @@ Lukáš Lalinský
 Qijiang Fan
 Rémy Léone
 Marco Ferragina
+reiv

From 7ad4258add964b2604ada611d380b5594126d85c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 28 Nov 2015 22:39:36 +0600
Subject: [PATCH 3208/6219] [bloomberg] Relax _VALID_URL even more (Closes
 #7685)

---
 youtube_dl/extractor/bloomberg.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bloomberg.py b/youtube_dl/extractor/bloomberg.py
index 11ace91dd..8b2f94211 100644
--- a/youtube_dl/extractor/bloomberg.py
+++ b/youtube_dl/extractor/bloomberg.py
@@ -6,7 +6,7 @@ from .common import InfoExtractor
 
 
 class BloombergIE(InfoExtractor):
-    _VALID_URL = r'https?://www\.bloomberg\.com/news/[^/]+/[^/]+/(?P<id>[^/?#]+)'
+    _VALID_URL = r'https?://(?:www\.)?bloomberg\.com/(?:[^/]+/)*(?P<id>[^/?#]+)'
 
     _TESTS = [{
         'url': 'http://www.bloomberg.com/news/videos/b/aaeae121-5949-481e-a1ce-4562db6f5df2',
@@ -20,6 +20,9 @@ class BloombergIE(InfoExtractor):
     }, {
         'url': 'http://www.bloomberg.com/news/articles/2015-11-12/five-strange-things-that-have-been-happening-in-financial-markets',
         'only_matching': True,
+    }, {
+        'url': 'http://www.bloomberg.com/politics/videos/2015-11-25/karl-rove-on-jeb-bush-s-struggles-stopping-trump',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 9a4f12be98a0d59de9b5da65d2cc24b04fe26929 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 28 Nov 2015 22:40:29 +0600
Subject: [PATCH 3209/6219] [bloomberg] Modernize

---
 youtube_dl/extractor/bloomberg.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bloomberg.py b/youtube_dl/extractor/bloomberg.py
index 8b2f94211..b6197d3f1 100644
--- a/youtube_dl/extractor/bloomberg.py
+++ b/youtube_dl/extractor/bloomberg.py
@@ -35,7 +35,7 @@ class BloombergIE(InfoExtractor):
             'http://www.bloomberg.com/api/embed?id=%s' % video_id, video_id)
         formats = []
         for stream in embed_info['streams']:
-            if stream["muxing_format"] == "TS":
+            if stream['muxing_format'] == 'TS':
                 formats.extend(self._extract_m3u8_formats(stream['url'], video_id))
             else:
                 formats.extend(self._extract_f4m_formats(stream['url'], video_id))

From 4191fdf147729728ca9c487f25ca66bf3d037cd5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 28 Nov 2015 22:41:39 +0600
Subject: [PATCH 3210/6219] [bloomberg] Improve video id regex

---
 youtube_dl/extractor/bloomberg.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bloomberg.py b/youtube_dl/extractor/bloomberg.py
index b6197d3f1..c28ed0c79 100644
--- a/youtube_dl/extractor/bloomberg.py
+++ b/youtube_dl/extractor/bloomberg.py
@@ -28,7 +28,9 @@ class BloombergIE(InfoExtractor):
     def _real_extract(self, url):
         name = self._match_id(url)
         webpage = self._download_webpage(url, name)
-        video_id = self._search_regex(r'"bmmrId":"(.+?)"', webpage, 'id')
+        video_id = self._search_regex(
+            r'["\']bmmrId["\']\s*:\s*(["\'])(?P<url>.+?)\1',
+            webpage, 'id', group='url')
         title = re.sub(': Video$', '', self._og_search_title(webpage))
 
         embed_info = self._download_json(

From b7faebbac87430106d0249a09fcca55a5ab112de Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 28 Nov 2015 22:45:19 +0600
Subject: [PATCH 3211/6219] [bloomberg] Improve formats extraction

---
 youtube_dl/extractor/bloomberg.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/bloomberg.py b/youtube_dl/extractor/bloomberg.py
index c28ed0c79..ebeef8f2a 100644
--- a/youtube_dl/extractor/bloomberg.py
+++ b/youtube_dl/extractor/bloomberg.py
@@ -37,10 +37,19 @@ class BloombergIE(InfoExtractor):
             'http://www.bloomberg.com/api/embed?id=%s' % video_id, video_id)
         formats = []
         for stream in embed_info['streams']:
+            stream_url = stream.get('url')
+            if not stream_url:
+                continue
             if stream['muxing_format'] == 'TS':
-                formats.extend(self._extract_m3u8_formats(stream['url'], video_id))
+                m3u8_formats = self._extract_m3u8_formats(
+                    stream_url, video_id, 'mp4', m3u8_id='hls', fatal=False)
+                if m3u8_formats:
+                    formats.extend(m3u8_formats)
             else:
-                formats.extend(self._extract_f4m_formats(stream['url'], video_id))
+                f4m_formats = self._extract_f4m_formats(
+                    stream_url, video_id, f4m_id='hds', fatal=False)
+                if f4m_formats:
+                    formats.extend(f4m_formats)
         self._sort_formats(formats)
 
         return {

From 50e12e9df186b475d0e500f61426d9e72d5b4320 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 28 Nov 2015 18:10:37 +0100
Subject: [PATCH 3212/6219] [acast] Add new extractor

---
 youtube_dl/extractor/__init__.py |  4 ++
 youtube_dl/extractor/acast.py    | 66 ++++++++++++++++++++++++++++++++
 2 files changed, 70 insertions(+)
 create mode 100644 youtube_dl/extractor/acast.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 947b83683..f7594832f 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -3,6 +3,10 @@ from __future__ import unicode_literals
 from .abc import ABCIE
 from .abc7news import Abc7NewsIE
 from .academicearth import AcademicEarthCourseIE
+from .acast import (
+    ACastIE,
+    ACastChannelIE,
+)
 from .addanime import AddAnimeIE
 from .adobetv import (
     AdobeTVIE,
diff --git a/youtube_dl/extractor/acast.py b/youtube_dl/extractor/acast.py
new file mode 100644
index 000000000..c2719ca32
--- /dev/null
+++ b/youtube_dl/extractor/acast.py
@@ -0,0 +1,66 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..compat import compat_str
+from ..utils import int_or_none
+
+
+class ACastBaseIE(InfoExtractor):
+    _API_BASE_URL = 'https://www.acast.com/api/'
+
+
+class ACastIE(ACastBaseIE):
+    IE_NAME = 'acast'
+    _VALID_URL = r'https?://(?:www\.)?acast\.com/(?P<channel>[^/]+)/(?P<id>[^/#?]+)'
+    _TEST = {
+        'url': 'https://www.acast.com/gardenersquestiontime/liverpool',
+        'md5': '9e9cd59c3a8a7d8d5407605f51093050',
+        'info_dict': {
+            'id': '43da2262-ade7-420c-8564-f6367da7c010',
+            'ext': 'mp3',
+            'title': 'Liverpool',
+            'timestamp': 1446163200000,
+            'description': 'md5:170432c9956eec0670d7080a75000d5b',
+            'duration': 2520,
+        }
+    }
+
+    def _real_extract(self, url):
+        channel, display_id = re.match(self._VALID_URL, url).groups()
+        cast_data = self._download_json(self._API_BASE_URL + 'channels/%s/acasts/%s/playback' % (channel, display_id), display_id)
+
+        return {
+            'id': compat_str(cast_data['id']),
+            'display_id': display_id,
+            'url': cast_data['blings'][0]['audio'],
+            'title': cast_data['name'],
+            'description': cast_data.get('description'),
+            'thumbnail': cast_data.get('image'),
+            'timestamp': int_or_none(cast_data.get('publishingDate')),
+            'duration': int_or_none(cast_data.get('duration')),
+        }
+
+
+class ACastChannelIE(ACastBaseIE):
+    IE_NAME = 'acast:channel'
+    _VALID_URL = r'https?://(?:www\.)?acast\.com/(?P<id>[^/#?]+)'
+    _TEST = {
+        'url': 'https://www.acast.com/gardenersquestiontime',
+        'info_dict': {
+            'id': '9d8f6f73-6b9d-4d16-9399-52bf88f8f611',
+            'title': 'Gardeners\' Question Time',
+            'description': 'md5:c7ef18049da6a52b63d371b3edccce90',
+        },
+        'playlist_mincount': 5,
+    }
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        channel_data = self._download_json(self._API_BASE_URL + 'channels/%s' % display_id, display_id)
+        casts = self._download_json(self._API_BASE_URL + 'channels/%s/acasts' % display_id, display_id)
+        entries = [self.url_result('https://www.acast.com/%s/%s' % (display_id, cast['url']), 'ACast') for cast in casts]
+
+        return self.playlist_result(entries, compat_str(channel_data['id']), channel_data['name'], channel_data.get('description'))

From f4c7ef98624bd407a8f8f91215cb95a2c5db9472 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 28 Nov 2015 18:20:44 +0100
Subject: [PATCH 3213/6219] [skynewsarabia] return empty categories array if
 there is no topic

---
 youtube_dl/extractor/skynewsarabia.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/skynewsarabia.py b/youtube_dl/extractor/skynewsarabia.py
index d1c737020..f09fee102 100644
--- a/youtube_dl/extractor/skynewsarabia.py
+++ b/youtube_dl/extractor/skynewsarabia.py
@@ -23,6 +23,7 @@ class SkyNewArabiaBaseIE(InfoExtractor):
 
     def _extract_video_info(self, video_data):
         video_id = compat_str(video_data['id'])
+        topic = video_data.get('topicTitle')
         return {
             '_type': 'url_transparent',
             'url': 'limelight:media:%s' % self._get_limelight_media_id(video_data['videoUrl'][0]['url']),
@@ -32,8 +33,8 @@ class SkyNewArabiaBaseIE(InfoExtractor):
             'thumbnail': self._get_image_url(video_data['mediaAsset']['imageUrl']),
             'timestamp': parse_iso8601(video_data.get('date')),
             'duration': parse_duration(video_data.get('runTime')),
-            'tags': video_data.get('tags'),
-            'categories': [video_data.get('topicTitle')],
+            'tags': video_data.get('tags', []),
+            'categories': [topic] if topic else [],
             'webpage_url': 'http://www.skynewsarabia.com/web/video/%s' % video_id,
             'ie_key': 'LimelightMedia',
         }
@@ -98,6 +99,7 @@ class SkyNewsArabiaArticleIE(SkyNewArabiaBaseIE):
         article_data = self._call_api('article', article_id)
         media_asset = article_data['mediaAsset']
         if media_asset['type'] == 'VIDEO':
+            topic = article_data.get('topicTitle')
             return {
                 '_type': 'url_transparent',
                 'url': 'limelight:media:%s' % self._get_limelight_media_id(media_asset['videoUrl'][0]['url']),
@@ -106,8 +108,8 @@ class SkyNewsArabiaArticleIE(SkyNewArabiaBaseIE):
                 'description': article_data.get('summary'),
                 'thumbnail': self._get_image_url(media_asset['imageUrl']),
                 'timestamp': parse_iso8601(article_data.get('date')),
-                'tags': article_data.get('tags'),
-                'categories': [article_data.get('topicTitle')],
+                'tags': article_data.get('tags', []),
+                'categories': [topic] if topic else [],
                 'webpage_url': url,
                 'ie_key': 'LimelightMedia',
             }

From 2a776f978849e0c66f70133747e7fd244f516f7f Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 28 Nov 2015 20:22:31 +0100
Subject: [PATCH 3214/6219] [cspan] change into a function

---
 youtube_dl/extractor/cspan.py | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/cspan.py b/youtube_dl/extractor/cspan.py
index 388460a32..7b685d157 100644
--- a/youtube_dl/extractor/cspan.py
+++ b/youtube_dl/extractor/cspan.py
@@ -56,9 +56,6 @@ class CSpanIE(InfoExtractor):
         }
     }]
 
-    def get_text_attr(self, d, attr):
-        return d.get(attr, {}).get('#text')
-
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
@@ -74,11 +71,14 @@ class CSpanIE(InfoExtractor):
                 surl = smuggle_url(senate_isvp_url, {'force_title': title})
                 return self.url_result(surl, 'SenateISVP', video_id, title)
 
+        def get_text_attr(d, attr):
+            return d.get(attr, {}).get('#text')
+
         data = self._download_json(
             'http://www.c-span.org/assets/player/ajax-player.php?os=android&html5=%s&id=%s' % (video_type, video_id),
             video_id)['video']
         if data['@status'] != 'Success':
-            raise ExtractorError('%s said: %s' % (self.IE_NAME, self.get_text_attr(data, 'error')), expected=True)
+            raise ExtractorError('%s said: %s' % (self.IE_NAME, get_text_attr(data, 'error')), expected=True)
 
         doc = self._download_xml(
             'http://www.c-span.org/common/services/flashXml.php?%sid=%s' % (video_type, video_id),
@@ -90,17 +90,17 @@ class CSpanIE(InfoExtractor):
         thumbnail = find_xpath_attr(doc, './/string', 'name', 'poster').text
 
         files = data['files']
-        capfile = self.get_text_attr(data, 'capfile')
+        capfile = get_text_attr(data, 'capfile')
 
         entries = []
         for partnum, f in enumerate(files):
             formats = []
             for quality in f['qualities']:
                 formats.append({
-                    'format_id': '%s-%sp' % (self.get_text_attr(quality, 'bitrate'), self.get_text_attr(quality, 'height')),
-                    'url': unescapeHTML(self.get_text_attr(quality, 'file')),
-                    'height': int_or_none(self.get_text_attr(quality, 'height')),
-                    'tbr': int_or_none(self.get_text_attr(quality, 'bitrate')),
+                    'format_id': '%s-%sp' % (get_text_attr(quality, 'bitrate'), get_text_attr(quality, 'height')),
+                    'url': unescapeHTML(get_text_attr(quality, 'file')),
+                    'height': int_or_none(get_text_attr(quality, 'height')),
+                    'tbr': int_or_none(get_text_attr(quality, 'bitrate')),
                 })
             self._sort_formats(formats)
             entries.append({
@@ -111,7 +111,7 @@ class CSpanIE(InfoExtractor):
                 'formats': formats,
                 'description': description,
                 'thumbnail': thumbnail,
-                'duration': int_or_none(self.get_text_attr(f, 'length')),
+                'duration': int_or_none(get_text_attr(f, 'length')),
                 'subtitles': {
                     'en': [{
                         'url': capfile,

From 87f0e62d94e0486598d123e26db3173e6f1d18e6 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 29 Nov 2015 12:42:50 +0800
Subject: [PATCH 3215/6219] [utils] Separate codes for handling Youtubedl-*
 headers

---
 youtube_dl/utils.py | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index d7b737e21..653a49055 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -663,6 +663,15 @@ def _create_http_connection(ydl_handler, http_class, is_https, *args, **kwargs):
     return hc
 
 
+def handle_youtubedl_headers(headers):
+    if 'Youtubedl-no-compression' in headers:
+        filtered_headers = dict((k, v) for k, v in headers.items() if k.lower() != 'accept-encoding')
+        del filtered_headers['Youtubedl-no-compression']
+        return filtered_headers
+
+    return headers
+
+
 class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
     """Handler for HTTP requests and responses.
 
@@ -731,10 +740,8 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
             # The dict keys are capitalized because of this bug by urllib
             if h.capitalize() not in req.headers:
                 req.add_header(h, v)
-        if 'Youtubedl-no-compression' in req.headers:
-            if 'Accept-encoding' in req.headers:
-                del req.headers['Accept-encoding']
-            del req.headers['Youtubedl-no-compression']
+
+        req.headers = handle_youtubedl_headers(req.headers)
 
         if sys.version_info < (2, 7) and '#' in req.get_full_url():
             # Python 2.6 is brain-dead when it comes to fragments

From 94e8c8047353eb541fa20dcf55819cc6ee6d3303 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 29 Nov 2015 12:43:59 +0800
Subject: [PATCH 3216/6219] [downloader/hls] Respect Youtubedl-* headers

---
 youtube_dl/downloader/hls.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index 92765a3f9..b5a3e1167 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -13,6 +13,7 @@ from ..utils import (
     encodeArgument,
     encodeFilename,
     sanitize_open,
+    handle_youtubedl_headers,
 )
 
 
@@ -33,9 +34,10 @@ class HlsFD(FileDownloader):
         if info_dict['http_headers'] and re.match(r'^https?://', url):
             # Trailing \r\n after each HTTP header is important to prevent warning from ffmpeg/avconv:
             # [http @ 00000000003d2fa0] No trailing CRLF found in HTTP header.
+            headers = handle_youtubedl_headers(info_dict['http_headers'])
             args += [
                 '-headers',
-                ''.join('%s: %s\r\n' % (key, val) for key, val in info_dict['http_headers'].items() if key.lower() != 'accept-encoding')]
+                ''.join('%s: %s\r\n' % (key, val) for key, val in headers.items())]
 
         args += ['-i', url, '-f', 'mp4', '-c', 'copy', '-bsf:a', 'aac_adtstoasc']
 

From ac5a69af45307b583a9a6088abe5939bec18d562 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 29 Nov 2015 12:44:24 +0800
Subject: [PATCH 3217/6219] [youtube] Disable compression for live streams

---
 youtube_dl/extractor/youtube.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 1c2420a33..52f4fe36d 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1475,6 +1475,11 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             manifest_url = video_info['hlsvp'][0]
             url_map = self._extract_from_m3u8(manifest_url, video_id)
             formats = _map_to_format_list(url_map)
+            # Accept-Encoding header causes failures in live streams on Youtube and Youtube Gaming
+            for a_format in formats:
+                if 'http_headers' not in a_format:
+                    a_format['http_headers'] = {}
+                a_format['http_headers']['Youtubedl-no-compression'] = True
         else:
             raise ExtractorError('no conn, hlsvp or url_encoded_fmt_stream_map information found in video info')
 

From 0424ec307bb920a2a7c217a741241f3d2af84efa Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 29 Nov 2015 12:46:04 +0800
Subject: [PATCH 3218/6219] [utils] Correct docstring of YoutubeDLHandler

---
 youtube_dl/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 653a49055..c43e9e3a1 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -679,7 +679,7 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
     the standard headers to every HTTP request and handles gzipped and
     deflated responses from web servers. If compression is to be avoided in
     a particular request, the original request in the program code only has
-    to include the HTTP header "Youtubedl-No-Compression", which will be
+    to include the HTTP header "Youtubedl-no-compression", which will be
     removed before making the real request.
 
     Part of this code was copied from:

From 8639f89f516c5bd1e4fda38c40e2a5a9b940ad85 Mon Sep 17 00:00:00 2001
From: Ryan Schmidt <github@ryandesign.com>
Date: Sat, 28 Nov 2015 22:56:24 -0600
Subject: [PATCH 3219/6219] Always use PYTHON env var in Makefile

---
 Makefile | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/Makefile b/Makefile
index 0636fc4cb..337a2eefb 100644
--- a/Makefile
+++ b/Makefile
@@ -60,34 +60,34 @@ youtube-dl: youtube_dl/*.py youtube_dl/*/*.py
 	chmod a+x youtube-dl
 
 README.md: youtube_dl/*.py youtube_dl/*/*.py
-	COLUMNS=80 python youtube_dl/__main__.py --help | python devscripts/make_readme.py
+	COLUMNS=80 $(PYTHON) youtube_dl/__main__.py --help | $(PYTHON) devscripts/make_readme.py
 
 CONTRIBUTING.md: README.md
-	python devscripts/make_contributing.py README.md CONTRIBUTING.md
+	$(PYTHON) devscripts/make_contributing.py README.md CONTRIBUTING.md
 
 supportedsites:
-	python devscripts/make_supportedsites.py docs/supportedsites.md
+	$(PYTHON) devscripts/make_supportedsites.py docs/supportedsites.md
 
 README.txt: README.md
 	pandoc -f markdown -t plain README.md -o README.txt
 
 youtube-dl.1: README.md
-	python devscripts/prepare_manpage.py >youtube-dl.1.temp.md
+	$(PYTHON) devscripts/prepare_manpage.py >youtube-dl.1.temp.md
 	pandoc -s -f markdown -t man youtube-dl.1.temp.md -o youtube-dl.1
 	rm -f youtube-dl.1.temp.md
 
 youtube-dl.bash-completion: youtube_dl/*.py youtube_dl/*/*.py devscripts/bash-completion.in
-	python devscripts/bash-completion.py
+	$(PYTHON) devscripts/bash-completion.py
 
 bash-completion: youtube-dl.bash-completion
 
 youtube-dl.zsh: youtube_dl/*.py youtube_dl/*/*.py devscripts/zsh-completion.in
-	python devscripts/zsh-completion.py
+	$(PYTHON) devscripts/zsh-completion.py
 
 zsh-completion: youtube-dl.zsh
 
 youtube-dl.fish: youtube_dl/*.py youtube_dl/*/*.py devscripts/fish-completion.in
-	python devscripts/fish-completion.py
+	$(PYTHON) devscripts/fish-completion.py
 
 fish-completion: youtube-dl.fish
 

From 992fc9d6e124b910ff3d720e252ef9aad99b2a8b Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 29 Nov 2015 12:58:29 +0800
Subject: [PATCH 3220/6219] [utils] Refactor handle_youtubedl_headers for
 future extension

---
 youtube_dl/utils.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index c43e9e3a1..d0606b4bc 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -664,12 +664,13 @@ def _create_http_connection(ydl_handler, http_class, is_https, *args, **kwargs):
 
 
 def handle_youtubedl_headers(headers):
-    if 'Youtubedl-no-compression' in headers:
-        filtered_headers = dict((k, v) for k, v in headers.items() if k.lower() != 'accept-encoding')
-        del filtered_headers['Youtubedl-no-compression']
-        return filtered_headers
+    filtered_headers = headers
 
-    return headers
+    if 'Youtubedl-no-compression' in filtered_headers:
+        filtered_headers = dict((k, v) for k, v in filtered_headers.items() if k.lower() != 'accept-encoding')
+        del filtered_headers['Youtubedl-no-compression']
+
+    return filtered_headers
 
 
 class YoutubeDLHandler(compat_urllib_request.HTTPHandler):

From bf2c8c8f82ff54f9594673c48e531661a72dbdcd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 29 Nov 2015 17:03:33 +0600
Subject: [PATCH 3221/6219] [spiegel] Fix extraction (Closes #7693)

---
 youtube_dl/extractor/spiegel.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/spiegel.py b/youtube_dl/extractor/spiegel.py
index 5bd3c0087..39a7aaf9d 100644
--- a/youtube_dl/extractor/spiegel.py
+++ b/youtube_dl/extractor/spiegel.py
@@ -58,7 +58,8 @@ class SpiegelIE(InfoExtractor):
         description = self._html_search_meta('description', webpage, 'description')
 
         base_url = self._search_regex(
-            r'var\s+server\s*=\s*"([^"]+)\"', webpage, 'server URL')
+            [r'server\s*:\s*(["\'])(?P<url>.+?)\1', r'var\s+server\s*=\s*"(?P<url>[^"]+)\"'],
+            webpage, 'server URL', group='url')
 
         xml_url = base_url + video_id + '.xml'
         idoc = self._download_xml(xml_url, video_id)

From 049d71d8745014bf5ec23e25e51d6b92556baa8c Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 29 Nov 2015 19:52:48 +0800
Subject: [PATCH 3222/6219] [youtube] Simplify and make sure header values are
 strings

---
 youtube_dl/extractor/youtube.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 52f4fe36d..4f375e2c8 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1477,9 +1477,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             formats = _map_to_format_list(url_map)
             # Accept-Encoding header causes failures in live streams on Youtube and Youtube Gaming
             for a_format in formats:
-                if 'http_headers' not in a_format:
-                    a_format['http_headers'] = {}
-                a_format['http_headers']['Youtubedl-no-compression'] = True
+                a_format.setdefault('http_headers', {})['Youtubedl-no-compression'] = 'True'
         else:
             raise ExtractorError('no conn, hlsvp or url_encoded_fmt_stream_map information found in video info')
 

From 040ac686798fdc922157cca64d654933e3f6d096 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 29 Nov 2015 21:01:59 +0600
Subject: [PATCH 3223/6219] [youtube] Extend _VALID_URL (Closes #7694)

---
 youtube_dl/extractor/youtube.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 4f375e2c8..55a06eb68 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -258,7 +258,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                              |(?:                                             # or the v= param in all its forms
                                  (?:(?:watch|movie)(?:_popup)?(?:\.php)?/?)?  # preceding watch(_popup|.php) or nothing (like /?v=xxxx)
                                  (?:\?|\#!?)                                  # the params delimiter ? or # or #!
-                                 (?:.*?&)??                                   # any other preceding param (like /?s=tuff&v=xxxx)
+                                 (?:.*?[&;])??                                # any other preceding param (like /?s=tuff&v=xxxx or ?s=tuff&amp;v=V36LpHqtcDY)
                                  v=
                              )
                          ))
@@ -730,6 +730,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'skip_download': True,
             },
         },
+        {
+            'url': 'https://www.youtube.com/watch?feature=player_embedded&amp;amp;v=V36LpHqtcDY',
+            'only_matching': True,
+        }
     ]
 
     def __init__(self, *args, **kwargs):

From 2e1b92854000662e554413df0c34c1cbc0d7ffc8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 29 Nov 2015 21:04:11 +0600
Subject: [PATCH 3224/6219] [youtube:playlist] Extend _VALID_URL

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 55a06eb68..032691e7f 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1566,7 +1566,7 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor, YoutubePlaylistBaseInfoExtract
                         youtube\.com/
                         (?:
                            (?:course|view_play_list|my_playlists|artist|playlist|watch|embed/videoseries)
-                           \? (?:.*?&)*? (?:p|a|list)=
+                           \? (?:.*?[&;])*? (?:p|a|list)=
                         |  p/
                         )
                         (

From d53a4af1a49413a38d639aeb7f522c4ebff8f5c8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 30 Nov 2015 03:47:01 +0600
Subject: [PATCH 3225/6219] [pornhub:playlist] Allow alphanumeric viewkeys
 (Closes #7695)

---
 youtube_dl/extractor/pornhub.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index 965940a4b..08275687d 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -147,7 +147,8 @@ class PornHubPlaylistIE(InfoExtractor):
 
         entries = [
             self.url_result('http://www.pornhub.com/%s' % video_url, 'PornHub')
-            for video_url in set(re.findall('href="/?(view_video\.php\?viewkey=\d+[^"]*)"', webpage))
+            for video_url in set(re.findall(
+                r'href="/?(view_video\.php\?.*\bviewkey=[\da-z]+[^"]*)"', webpage))
         ]
 
         playlist = self._parse_json(

From af284305d58a9915a8ef00d056484b3a59548dda Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 30 Nov 2015 03:58:39 +0600
Subject: [PATCH 3226/6219] [vodlocker] Capture file not found error (Closes
 #7696)

---
 youtube_dl/extractor/vodlocker.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vodlocker.py b/youtube_dl/extractor/vodlocker.py
index be0a2780f..357594a11 100644
--- a/youtube_dl/extractor/vodlocker.py
+++ b/youtube_dl/extractor/vodlocker.py
@@ -3,11 +3,14 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..compat import compat_urllib_parse
-from ..utils import sanitized_Request
+from ..utils import (
+    ExtractorError,
+    sanitized_Request,
+)
 
 
 class VodlockerIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?vodlocker\.com/(?P<id>[0-9a-zA-Z]+)(?:\..*?)?'
+    _VALID_URL = r'https?://(?:www\.)?vodlocker\.com/(?:embed-)?(?P<id>[0-9a-zA-Z]+)(?:\..*?)?'
 
     _TESTS = [{
         'url': 'http://vodlocker.com/e8wvyzz4sl42',
@@ -24,6 +27,12 @@ class VodlockerIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
+        if any(p in webpage for p in (
+                '>THIS FILE WAS DELETED<',
+                '>File Not Found<',
+                'The file you were looking for could not be found, sorry for any inconvenience.<')):
+            raise ExtractorError('Video %s does not exist' % video_id, expected=True)
+
         fields = self._hidden_inputs(webpage)
 
         if fields['op'] == 'download1':

From 59ee8a86471af488c2ee16dcacf7a913636f0150 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 30 Nov 2015 20:10:09 +0600
Subject: [PATCH 3227/6219] [facebook] Make alternative title optional (Closes
 #7700)

---
 youtube_dl/extractor/facebook.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index fd854411b..321eec59e 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -164,7 +164,7 @@ class FacebookIE(InfoExtractor):
         if not video_title:
             video_title = self._html_search_regex(
                 r'(?s)<span class="fbPhotosPhotoCaption".*?id="fbPhotoPageCaption"><span class="hasCaption">(.*?)</span>',
-                webpage, 'alternative title', fatal=False)
+                webpage, 'alternative title', default=None)
             video_title = limit_length(video_title, 80)
         if not video_title:
             video_title = 'Facebook video #%s' % video_id

From 4c6b4764f0260808d321cfb6cca1daa5e3eb13d0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 30 Nov 2015 20:42:05 +0600
Subject: [PATCH 3228/6219] [youtube] Clarify itag 272 possible resolutions
 (#7699)

---
 youtube_dl/extractor/youtube.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 032691e7f..9b39505ba 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -346,6 +346,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         '247': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
         '248': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
         '271': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
+        # itag 272 videos are either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug)
         '272': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
         '302': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'vp9'},
         '303': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'vp9'},

From 874ae0354e96a578dc44846fffd91b01079ccf1e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 1 Dec 2015 18:35:24 +0600
Subject: [PATCH 3229/6219] [nrk] Extract f4m formats and impose geo
 restriction only when not media URL (Closes #7715)

---
 youtube_dl/extractor/nrk.py | 24 +++++++++++++++++-------
 1 file changed, 17 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index 8ac38a174..6ff13050d 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -6,6 +6,7 @@ import re
 from .common import InfoExtractor
 from ..compat import compat_urlparse
 from ..utils import (
+    determine_ext,
     ExtractorError,
     float_or_none,
     parse_duration,
@@ -48,12 +49,22 @@ class NRKIE(InfoExtractor):
             'http://v8.psapi.nrk.no/mediaelement/%s' % video_id,
             video_id, 'Downloading media JSON')
 
-        if data['usageRights']['isGeoBlocked']:
-            raise ExtractorError(
-                'NRK har ikke rettigheter til å vise dette programmet utenfor Norge',
-                expected=True)
+        media_url = data.get('mediaUrl')
 
-        video_url = data['mediaUrl'] + '?hdcore=3.5.0&plugin=aasp-3.5.0.151.81'
+        if not media_url:
+            if data['usageRights']['isGeoBlocked']:
+                raise ExtractorError(
+                    'NRK har ikke rettigheter til å vise dette programmet utenfor Norge',
+                    expected=True)
+
+        if determine_ext(media_url) == 'f4m':
+            formats = self._extract_f4m_formats(
+                media_url + '?hdcore=3.5.0&plugin=aasp-3.5.0.151.81', video_id, f4m_id='hds')
+        else:
+            formats = [{
+                'url': media_url,
+                'ext': 'flv',
+            }]
 
         duration = parse_duration(data.get('duration'))
 
@@ -67,12 +78,11 @@ class NRKIE(InfoExtractor):
 
         return {
             'id': video_id,
-            'url': video_url,
-            'ext': 'flv',
             'title': data['title'],
             'description': data['description'],
             'duration': duration,
             'thumbnail': thumbnail,
+            'formats': formats,
         }
 
 

From 3b35c3425e2c4d9836a3efe5cf73e35a60f674d1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 1 Dec 2015 20:35:46 +0600
Subject: [PATCH 3230/6219] [udemy] Extract formats from data.outputs (#7704)

---
 youtube_dl/extractor/udemy.py | 46 ++++++++++++++++++++++++++---------
 1 file changed, 34 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index 825172806..9d31a3f5b 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -9,6 +9,7 @@ from ..compat import (
 )
 from ..utils import (
     ExtractorError,
+    int_or_none,
     sanitized_Request,
 )
 
@@ -127,26 +128,47 @@ class UdemyIE(InfoExtractor):
 
         video_id = asset['id']
         thumbnail = asset.get('thumbnailUrl') or asset.get('thumbnail_url')
-        duration = asset['data']['duration']
+        duration = int_or_none(asset.get('data', {}).get('duration'))
 
         download_url = asset.get('downloadUrl') or asset.get('download_url')
 
         video = download_url.get('Video') or download_url.get('video')
         video_480p = download_url.get('Video480p') or download_url.get('video_480p')
 
-        formats = [
-            {
-                'url': video_480p[0],
-                'format_id': '360p',
-            },
-            {
-                'url': video[0],
-                'format_id': '720p',
-            },
-        ]
+        formats = [{
+            'url': video_480p[0],
+            'format_id': 'download-360p',
+        }, {
+            'url': video[0],
+            'format_id': 'download-720p',
+        }]
+
+        # Some videos also contain formats in asset['data']['outputs'] (e.g.
+        # https://www.udemy.com/ios9-swift/learn/#/lecture/3383208)
+        outputs = asset.get('data', {}).get('outputs')
+        if isinstance(outputs, dict):
+            for format_id, f in outputs.items():
+                video_url = f.get('url')
+                if video_url:
+                    formats.append({
+                        'url': video_url,
+                        'format_id': '%sp' % (f.get('labe1l') or format_id),
+                        'width': int_or_none(f.get('width')),
+                        'height': int_or_none(f.get('height')),
+                        'vbr': int_or_none(f.get('video_bitrate_in_kbps')),
+                        'vcodec': f.get('video_codec'),
+                        'fps': int_or_none(f.get('frame_rate')),
+                        'abr': int_or_none(f.get('audio_bitrate_in_kbps')),
+                        'acodec': f.get('audio_codec'),
+                        'asr': int_or_none(f.get('audio_sample_rate')),
+                        'tbr': int_or_none(f.get('total_bitrate_in_kbps')),
+                        'filesize': int_or_none(f.get('file_size_in_bytes')),
+                    })
+
+        self._sort_formats(formats)
 
         title = lecture['title']
-        description = lecture['description']
+        description = lecture.get('description')
 
         return {
             'id': video_id,

From 78717fc32802cb1e0f240cfbdf02207c6356ff2b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 1 Dec 2015 22:10:10 +0600
Subject: [PATCH 3231/6219] [udemy] Allow authentication via cookies

---
 youtube_dl/extractor/udemy.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index 9d31a3f5b..e743c0262 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -72,7 +72,7 @@ class UdemyIE(InfoExtractor):
     def _login(self):
         (username, password) = self._get_login_info()
         if username is None:
-            self.raise_login_required('Udemy account is required')
+            return
 
         login_popup = self._download_webpage(
             self._LOGIN_URL, None, 'Downloading login popup')

From 328f82d59a0d3b743280c3ab532949c9220383ed Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 2 Dec 2015 00:48:27 +0600
Subject: [PATCH 3232/6219] [udemy] Semi-switch to api 2.0 (Closes #7704)

* Use api 2.0 to get lectures since it provides more formats
* Fix authorization for api 2.0
* Autotry enrolling in the course for single lectures
* Extract additional metadata rom asset['data']['outputs']
---
 youtube_dl/extractor/udemy.py | 160 ++++++++++++++++++++--------------
 1 file changed, 95 insertions(+), 65 deletions(-)

diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index e743c0262..26ae8cc36 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -1,14 +1,15 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..compat import (
+    compat_HTTPError,
     compat_urllib_parse,
     compat_urllib_request,
+    compat_urlparse,
 )
 from ..utils import (
     ExtractorError,
+    float_or_none,
     int_or_none,
     sanitized_Request,
 )
@@ -19,6 +20,8 @@ class UdemyIE(InfoExtractor):
     _VALID_URL = r'https?://www\.udemy\.com/(?:[^#]+#/lecture/|lecture/view/?\?lectureId=)(?P<id>\d+)'
     _LOGIN_URL = 'https://www.udemy.com/join/login-popup/?displayType=ajax&showSkipButton=1'
     _ORIGIN_URL = 'https://www.udemy.com'
+    _SUCCESSFULLY_ENROLLED = '>You have enrolled in this course!<'
+    _ALREADY_ENROLLED = '>You are already taking this course.<'
     _NETRC_MACHINE = 'udemy'
 
     _TESTS = [{
@@ -34,6 +37,29 @@ class UdemyIE(InfoExtractor):
         'skip': 'Requires udemy account credentials',
     }]
 
+    def _enroll_course(self, webpage, course_id):
+        enroll_url = self._search_regex(
+            r'href=(["\'])(?P<url>https?://(?:www\.)?udemy\.com/course/subscribe/.+?)\1',
+            webpage, 'enroll url', group='url',
+            default='https://www.udemy.com/course/subscribe/?courseId=%s' % course_id)
+        webpage = self._download_webpage(enroll_url, course_id, 'Enrolling in the course')
+        if self._SUCCESSFULLY_ENROLLED in webpage:
+            self.to_screen('%s: Successfully enrolled in' % course_id)
+        elif self._ALREADY_ENROLLED in webpage:
+            self.to_screen('%s: Already enrolled in' % course_id)
+
+    def _download_lecture(self, course_id, lecture_id):
+        return self._download_json(
+            'https://www.udemy.com/api-2.0/users/me/subscribed-courses/%s/lectures/%s?%s' % (
+                course_id, lecture_id, compat_urllib_parse.urlencode({
+                    'video_only': '',
+                    'auto_play': '',
+                    'fields[lecture]': 'title,description,asset',
+                    'fields[asset]': 'asset_type,stream_url,thumbnail_url,download_urls,data',
+                    'instructorPreviewMode': 'False',
+                })),
+            lecture_id, 'Downloading lecture JSON', fatal=False)
+
     def _handle_error(self, response):
         if not isinstance(response, dict):
             return
@@ -45,7 +71,7 @@ class UdemyIE(InfoExtractor):
                 error_str += ' - %s' % error_data.get('formErrors')
             raise ExtractorError(error_str, expected=True)
 
-    def _download_json(self, url_or_request, video_id, note='Downloading JSON metadata'):
+    def _download_json(self, url_or_request, video_id, note='Downloading JSON metadata', *args, **kwargs):
         headers = {
             'X-Udemy-Snail-Case': 'true',
             'X-Requested-With': 'XMLHttpRequest',
@@ -55,6 +81,7 @@ class UdemyIE(InfoExtractor):
                 headers['X-Udemy-Client-Id'] = cookie.value
             elif cookie.name == 'access_token':
                 headers['X-Udemy-Bearer-Token'] = cookie.value
+                headers['X-Udemy-Authorization'] = 'Bearer %s' % cookie.value
 
         if isinstance(url_or_request, compat_urllib_request.Request):
             for header, value in headers.items():
@@ -62,7 +89,7 @@ class UdemyIE(InfoExtractor):
         else:
             url_or_request = sanitized_Request(url_or_request, headers=headers)
 
-        response = super(UdemyIE, self)._download_json(url_or_request, video_id, note)
+        response = super(UdemyIE, self)._download_json(url_or_request, video_id, note, *args, **kwargs)
         self._handle_error(response)
         return response
 
@@ -110,66 +137,77 @@ class UdemyIE(InfoExtractor):
     def _real_extract(self, url):
         lecture_id = self._match_id(url)
 
-        lecture = self._download_json(
-            'https://www.udemy.com/api-1.1/lectures/%s' % lecture_id,
-            lecture_id, 'Downloading lecture JSON')
+        webpage = self._download_webpage(url, lecture_id)
 
-        asset_type = lecture.get('assetType') or lecture.get('asset_type')
+        course_id = self._search_regex(
+            r'data-course-id=["\'](\d+)', webpage, 'course id')
+
+        try:
+            lecture = self._download_lecture(course_id, lecture_id)
+        except ExtractorError as e:
+            # Error could possibly mean we are not enrolled in the course
+            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
+                self._enroll_course(webpage, course_id)
+                lecture_id = self._download_lecture(course_id, lecture_id)
+            else:
+                raise
+
+        title = lecture['title']
+        description = lecture.get('description')
+
+        asset = lecture['asset']
+
+        asset_type = asset.get('assetType') or asset.get('asset_type')
         if asset_type != 'Video':
             raise ExtractorError(
                 'Lecture %s is not a video' % lecture_id, expected=True)
 
-        asset = lecture['asset']
-
         stream_url = asset.get('streamUrl') or asset.get('stream_url')
-        mobj = re.search(r'(https?://www\.youtube\.com/watch\?v=.*)', stream_url)
-        if mobj:
-            return self.url_result(mobj.group(1), 'Youtube')
+        if stream_url:
+            youtube_url = self._search_regex(
+                r'(https?://www\.youtube\.com/watch\?v=.*)', stream_url, 'youtube URL', default=None)
+            if youtube_url:
+                return self.url_result(youtube_url, 'Youtube')
 
         video_id = asset['id']
         thumbnail = asset.get('thumbnailUrl') or asset.get('thumbnail_url')
-        duration = int_or_none(asset.get('data', {}).get('duration'))
+        duration = float_or_none(asset.get('data', {}).get('duration'))
+        outputs = asset.get('data', {}).get('outputs', {})
 
-        download_url = asset.get('downloadUrl') or asset.get('download_url')
-
-        video = download_url.get('Video') or download_url.get('video')
-        video_480p = download_url.get('Video480p') or download_url.get('video_480p')
-
-        formats = [{
-            'url': video_480p[0],
-            'format_id': 'download-360p',
-        }, {
-            'url': video[0],
-            'format_id': 'download-720p',
-        }]
-
-        # Some videos also contain formats in asset['data']['outputs'] (e.g.
-        # https://www.udemy.com/ios9-swift/learn/#/lecture/3383208)
-        outputs = asset.get('data', {}).get('outputs')
-        if isinstance(outputs, dict):
-            for format_id, f in outputs.items():
-                video_url = f.get('url')
-                if video_url:
-                    formats.append({
-                        'url': video_url,
-                        'format_id': '%sp' % (f.get('labe1l') or format_id),
-                        'width': int_or_none(f.get('width')),
-                        'height': int_or_none(f.get('height')),
-                        'vbr': int_or_none(f.get('video_bitrate_in_kbps')),
-                        'vcodec': f.get('video_codec'),
-                        'fps': int_or_none(f.get('frame_rate')),
-                        'abr': int_or_none(f.get('audio_bitrate_in_kbps')),
-                        'acodec': f.get('audio_codec'),
-                        'asr': int_or_none(f.get('audio_sample_rate')),
-                        'tbr': int_or_none(f.get('total_bitrate_in_kbps')),
-                        'filesize': int_or_none(f.get('file_size_in_bytes')),
+        formats = []
+        for format_ in asset.get('download_urls', {}).get('Video', []):
+            video_url = format_.get('file')
+            if not video_url:
+                continue
+            format_id = format_.get('label')
+            f = {
+                'url': format_['file'],
+                'height': int_or_none(format_id),
+            }
+            if format_id:
+                # Some videos contain additional metadata (e.g.
+                # https://www.udemy.com/ios9-swift/learn/#/lecture/3383208)
+                output = outputs.get(format_id)
+                if isinstance(output, dict):
+                    f.update({
+                        'format_id': '%sp' % (output.get('label') or format_id),
+                        'width': int_or_none(output.get('width')),
+                        'height': int_or_none(output.get('height')),
+                        'vbr': int_or_none(output.get('video_bitrate_in_kbps')),
+                        'vcodec': output.get('video_codec'),
+                        'fps': int_or_none(output.get('frame_rate')),
+                        'abr': int_or_none(output.get('audio_bitrate_in_kbps')),
+                        'acodec': output.get('audio_codec'),
+                        'asr': int_or_none(output.get('audio_sample_rate')),
+                        'tbr': int_or_none(output.get('total_bitrate_in_kbps')),
+                        'filesize': int_or_none(output.get('file_size_in_bytes')),
                     })
+                else:
+                    f['format_id'] = '%sp' % format_id
+            formats.append(f)
 
         self._sort_formats(formats)
 
-        title = lecture['title']
-        description = lecture.get('description')
-
         return {
             'id': video_id,
             'title': title,
@@ -182,9 +220,7 @@ class UdemyIE(InfoExtractor):
 
 class UdemyCourseIE(UdemyIE):
     IE_NAME = 'udemy:course'
-    _VALID_URL = r'https?://www\.udemy\.com/(?P<coursepath>[\da-z-]+)'
-    _SUCCESSFULLY_ENROLLED = '>You have enrolled in this course!<'
-    _ALREADY_ENROLLED = '>You are already taking this course.<'
+    _VALID_URL = r'https?://www\.udemy\.com/(?P<id>[\da-z-]+)'
     _TESTS = []
 
     @classmethod
@@ -192,24 +228,18 @@ class UdemyCourseIE(UdemyIE):
         return False if UdemyIE.suitable(url) else super(UdemyCourseIE, cls).suitable(url)
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        course_path = mobj.group('coursepath')
+        course_path = self._match_id(url)
+
+        webpage = self._download_webpage(url, course_path)
 
         response = self._download_json(
             'https://www.udemy.com/api-1.1/courses/%s' % course_path,
             course_path, 'Downloading course JSON')
 
-        course_id = int(response['id'])
-        course_title = response['title']
+        course_id = response['id']
+        course_title = response.get('title')
 
-        webpage = self._download_webpage(
-            'https://www.udemy.com/course/subscribe/?courseId=%s' % course_id,
-            course_id, 'Enrolling in the course')
-
-        if self._SUCCESSFULLY_ENROLLED in webpage:
-            self.to_screen('%s: Successfully enrolled in' % course_id)
-        elif self._ALREADY_ENROLLED in webpage:
-            self.to_screen('%s: Already enrolled in' % course_id)
+        self._enroll_course(webpage, course_id)
 
         response = self._download_json(
             'https://www.udemy.com/api-1.1/courses/%s/curriculum' % course_id,

From 9fc87fa767425ee7d56b2ce4650c9b06b480e005 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 2 Dec 2015 00:51:47 +0600
Subject: [PATCH 3233/6219] [udemy] Remove unused import

---
 youtube_dl/extractor/udemy.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index 26ae8cc36..2a54f3764 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -5,7 +5,6 @@ from ..compat import (
     compat_HTTPError,
     compat_urllib_parse,
     compat_urllib_request,
-    compat_urlparse,
 )
 from ..utils import (
     ExtractorError,

From 24121bc703152312cfbb70f01ebd39e2fe1197e5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 2 Dec 2015 00:53:03 +0600
Subject: [PATCH 3234/6219] [udemy] Make lecture downloading fatal

---
 youtube_dl/extractor/udemy.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index 2a54f3764..59832b1ec 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -57,7 +57,7 @@ class UdemyIE(InfoExtractor):
                     'fields[asset]': 'asset_type,stream_url,thumbnail_url,download_urls,data',
                     'instructorPreviewMode': 'False',
                 })),
-            lecture_id, 'Downloading lecture JSON', fatal=False)
+            lecture_id, 'Downloading lecture JSON')
 
     def _handle_error(self, response):
         if not isinstance(response, dict):
@@ -70,7 +70,7 @@ class UdemyIE(InfoExtractor):
                 error_str += ' - %s' % error_data.get('formErrors')
             raise ExtractorError(error_str, expected=True)
 
-    def _download_json(self, url_or_request, video_id, note='Downloading JSON metadata', *args, **kwargs):
+    def _download_json(self, url_or_request, video_id, note='Downloading JSON metadata'):
         headers = {
             'X-Udemy-Snail-Case': 'true',
             'X-Requested-With': 'XMLHttpRequest',
@@ -88,7 +88,7 @@ class UdemyIE(InfoExtractor):
         else:
             url_or_request = sanitized_Request(url_or_request, headers=headers)
 
-        response = super(UdemyIE, self)._download_json(url_or_request, video_id, note, *args, **kwargs)
+        response = super(UdemyIE, self)._download_json(url_or_request, video_id, note)
         self._handle_error(response)
         return response
 

From 22d7368dfb384e7698faad6d2891b4aaceab3d7c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 2 Dec 2015 02:34:31 +0600
Subject: [PATCH 3235/6219] [bbc] Extract _ID_REGEX and ad one more video id
 pattern (Closes #7724)

---
 youtube_dl/extractor/bbc.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 33b296eaf..5f7265a06 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -22,7 +22,8 @@ from ..compat import (
 class BBCCoUkIE(InfoExtractor):
     IE_NAME = 'bbc.co.uk'
     IE_DESC = 'BBC iPlayer'
-    _VALID_URL = r'https?://(?:www\.)?bbc\.co\.uk/(?:(?:programmes/(?!articles/)|iplayer(?:/[^/]+)?/(?:episode/|playlist/))|music/clips[/#])(?P<id>[\da-z]{8})'
+    _ID_REGEX = r'[pb][\da-z]{7}'
+    _VALID_URL = r'https?://(?:(?:www\.)?bbc\.co\.uk/(?:(?:programmes/(?!articles/)|iplayer(?:/[^/]+)?/(?:episode/|playlist/))|music/clips[/#])|)(?P<id>%s)' % _ID_REGEX
 
     _MEDIASELECTOR_URLS = [
         # Provides HQ HLS streams with even better quality that pc mediaset but fails
@@ -465,7 +466,7 @@ class BBCCoUkIE(InfoExtractor):
 
         if not programme_id:
             programme_id = self._search_regex(
-                r'"vpid"\s*:\s*"([\da-z]{8})"', webpage, 'vpid', fatal=False, default=None)
+                r'"vpid"\s*:\s*"(%s)"' % self._ID_REGEX, webpage, 'vpid', fatal=False, default=None)
 
         if programme_id:
             formats, subtitles = self._download_media_selector(programme_id)
@@ -780,8 +781,9 @@ class BBCIE(BBCCoUkIE):
 
         # single video story (e.g. http://www.bbc.com/travel/story/20150625-sri-lankas-spicy-secret)
         programme_id = self._search_regex(
-            [r'data-video-player-vpid="([\da-z]{8})"',
-             r'<param[^>]+name="externalIdentifier"[^>]+value="([\da-z]{8})"'],
+            [r'data-video-player-vpid="(%s)"' % self._ID_REGEX,
+             r'<param[^>]+name="externalIdentifier"[^>]+value="(%s)"' % self._ID_REGEX,
+             r'videoId\s*:\s*["\'](%s)["\']' % self._ID_REGEX],
             webpage, 'vpid', default=None)
 
         if programme_id:
@@ -816,7 +818,7 @@ class BBCIE(BBCCoUkIE):
 
         # Multiple video article (e.g.
         # http://www.bbc.co.uk/blogs/adamcurtis/entries/3662a707-0af9-3149-963f-47bea720b460)
-        EMBED_URL = r'https?://(?:www\.)?bbc\.co\.uk/(?:[^/]+/)+[\da-z]{8}(?:\b[^"]+)?'
+        EMBED_URL = r'https?://(?:www\.)?bbc\.co\.uk/(?:[^/]+/)+%s(?:\b[^"]+)?' % self._ID_REGEX
         entries = []
         for match in extract_all(r'new\s+SMP\(({.+?})\)'):
             embed_url = match.get('playerSettings', {}).get('externalEmbedUrl')

From 7b1e379ca9316433ee911f2f36c7808522fff13d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 3 Dec 2015 13:47:21 +0100
Subject: [PATCH 3236/6219] [gametrailers] Fix extraction (fixes #7722)

They have stopped using the MTV system.
---
 youtube_dl/extractor/gametrailers.py | 61 ++++++++++++++++++++++++----
 1 file changed, 52 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/gametrailers.py b/youtube_dl/extractor/gametrailers.py
index a6ab795ae..c3f031d9c 100644
--- a/youtube_dl/extractor/gametrailers.py
+++ b/youtube_dl/extractor/gametrailers.py
@@ -1,19 +1,62 @@
 from __future__ import unicode_literals
 
-from .mtv import MTVServicesInfoExtractor
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    parse_age_limit,
+    url_basename,
+)
 
 
-class GametrailersIE(MTVServicesInfoExtractor):
-    _VALID_URL = r'http://www\.gametrailers\.com/(?P<type>videos|reviews|full-episodes)/(?P<id>.*?)/(?P<title>.*)'
+class GametrailersIE(InfoExtractor):
+    _VALID_URL = r'http://www\.gametrailers\.com/videos/view/[^/]+/(?P<id>.+)'
+
     _TEST = {
-        'url': 'http://www.gametrailers.com/videos/zbvr8i/mirror-s-edge-2-e3-2013--debut-trailer',
-        'md5': '4c8e67681a0ea7ec241e8c09b3ea8cf7',
+        'url': 'http://www.gametrailers.com/videos/view/gametrailers-com/116437-Just-Cause-3-Review',
+        'md5': 'f28c4efa0bdfaf9b760f6507955b6a6a',
         'info_dict': {
-            'id': '70e9a5d7-cf25-4a10-9104-6f3e7342ae0d',
+            'id': '2983958',
             'ext': 'mp4',
-            'title': 'E3 2013: Debut Trailer',
-            'description': 'Faith is back!  Check out the World Premiere trailer for Mirror\'s Edge 2 straight from the EA Press Conference at E3 2013!',
+            'display_id': '116437-Just-Cause-3-Review',
+            'title': 'Just Cause 3 - Review',
+            'description': 'It\'s a lot of fun to shoot at things and then watch them explode in Just Cause 3, but should there be more to the experience than that?',
         },
     }
 
-    _FEED_URL = 'http://www.gametrailers.com/feeds/mrss'
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+        title = self._html_search_regex(
+            r'<title>(.+?)\|', webpage, 'title').strip()
+        embed_url = self._proto_relative_url(
+            self._search_regex(
+                r'src=\'(//embed.gametrailers.com/embed/[^\']+)\'', webpage,
+                'embed url'),
+            scheme='http:')
+        video_id = url_basename(embed_url)
+        embed_page = self._download_webpage(embed_url, video_id)
+        embed_vars_json = self._search_regex(
+            r'(?s)var embedVars = (\{.*?\})\s*</script>', embed_page,
+            'embed vars')
+        info = self._parse_json(embed_vars_json, video_id)
+
+        formats = []
+        for media in info['media']:
+            if media['mediaPurpose'] == 'play':
+                formats.append({
+                    'url': media['uri'],
+                    'height': media['height'],
+                    'width:': media['width'],
+                })
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            'title': title,
+            'formats': formats,
+            'thumbnail': info.get('thumbUri'),
+            'description': self._og_search_description(webpage),
+            'duration': int_or_none(info.get('videoLengthInSeconds')),
+            'age_limit': parse_age_limit(info.get('audienceRating')),
+        }

From 49358274d7145788f2e99471dbb7c7c4301a3d3d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 3 Dec 2015 20:49:14 +0600
Subject: [PATCH 3237/6219] [bbc] Fix _VALID_URL

---
 youtube_dl/extractor/bbc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 5f7265a06..7fb80aa38 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -23,7 +23,7 @@ class BBCCoUkIE(InfoExtractor):
     IE_NAME = 'bbc.co.uk'
     IE_DESC = 'BBC iPlayer'
     _ID_REGEX = r'[pb][\da-z]{7}'
-    _VALID_URL = r'https?://(?:(?:www\.)?bbc\.co\.uk/(?:(?:programmes/(?!articles/)|iplayer(?:/[^/]+)?/(?:episode/|playlist/))|music/clips[/#])|)(?P<id>%s)' % _ID_REGEX
+    _VALID_URL = r'https?://(?:www\.)?bbc\.co\.uk/(?:(?:programmes/(?!articles/)|iplayer(?:/[^/]+)?/(?:episode/|playlist/))|music/clips[/#])(?P<id>%s)' % _ID_REGEX
 
     _MEDIASELECTOR_URLS = [
         # Provides HQ HLS streams with even better quality that pc mediaset but fails

From 62d231c00493c329a78a30b15a336f131fc392f9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 3 Dec 2015 20:55:02 +0600
Subject: [PATCH 3238/6219] [extractor/common] Clarify duration can be float

---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index eb9bfa3d1..6ab2d68d6 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -167,7 +167,7 @@ class InfoExtractor(object):
                     "ext" will be calculated from URL if missing
     automatic_captions: Like 'subtitles', used by the YoutubeIE for
                     automatically generated captions
-    duration:       Length of the video in seconds, as an integer.
+    duration:       Length of the video in seconds, as an integer or float.
     view_count:     How many users have watched the video on the platform.
     like_count:     Number of positive ratings of the video
     dislike_count:  Number of negative ratings of the video

From af93fcfa0549bdfe26c39adecbb1bcc98a9345af Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 3 Dec 2015 23:23:36 +0600
Subject: [PATCH 3239/6219] [beeg] Update API URL (Closes #7736)

---
 youtube_dl/extractor/beeg.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/beeg.py b/youtube_dl/extractor/beeg.py
index 61bc2f744..1ee4a8b05 100644
--- a/youtube_dl/extractor/beeg.py
+++ b/youtube_dl/extractor/beeg.py
@@ -29,7 +29,7 @@ class BeegIE(InfoExtractor):
         video_id = self._match_id(url)
 
         video = self._download_json(
-            'http://beeg.com/api/v1/video/%s' % video_id, video_id)
+            'http://beeg.com/api/v3/video/%s' % video_id, video_id)
 
         formats = []
         for format_id, video_url in video.items():

From e0977d7686e5df524b1a024484e7a4bb9cfa261d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 4 Dec 2015 00:59:32 +0600
Subject: [PATCH 3240/6219] [beeg] Decrypt URL (Closes #7736)

---
 youtube_dl/extractor/beeg.py | 24 +++++++++++++++++++++++-
 1 file changed, 23 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/beeg.py b/youtube_dl/extractor/beeg.py
index 1ee4a8b05..e63c2ac00 100644
--- a/youtube_dl/extractor/beeg.py
+++ b/youtube_dl/extractor/beeg.py
@@ -1,6 +1,11 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..compat import (
+    compat_chr,
+    compat_ord,
+    compat_urllib_parse_unquote,
+)
 from ..utils import (
     int_or_none,
     parse_iso8601,
@@ -31,6 +36,23 @@ class BeegIE(InfoExtractor):
         video = self._download_json(
             'http://beeg.com/api/v3/video/%s' % video_id, video_id)
 
+        def decrypt_key(key):
+            # Reverse engineered from http://static.beeg.com/cpl/1067.js
+            a = '8RPUUCS35ZWp3ADnKcSmpH71ZusrROo'
+            e = compat_urllib_parse_unquote(key)
+            return ''.join([
+                compat_chr(compat_ord(e[n]) - compat_ord(a[n % len(a)]) % 25)
+                for n in range(len(e))])
+
+        def decrypt_url(encrypted_url):
+            encrypted_url = self._proto_relative_url(
+                encrypted_url.replace('{DATA_MARKERS}', ''), 'http:')
+            key = self._search_regex(
+                r'/key=(.*?)%2Cend=', encrypted_url, 'key', default=None)
+            if not key:
+                return encrypted_url
+            return encrypted_url.replace(key, decrypt_key(key))
+
         formats = []
         for format_id, video_url in video.items():
             if not video_url:
@@ -40,7 +62,7 @@ class BeegIE(InfoExtractor):
             if not height:
                 continue
             formats.append({
-                'url': self._proto_relative_url(video_url.replace('{DATA_MARKERS}', ''), 'http:'),
+                'url': decrypt_url(video_url),
                 'format_id': format_id,
                 'height': int(height),
             })

From 76ab842d9b1308e4c3409cfdfcd925fcdb92e14f Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 3 Dec 2015 22:01:32 +0100
Subject: [PATCH 3241/6219] [bilibili] use xpath_text and catch errors in xml
 document

---
 youtube_dl/extractor/bilibili.py | 25 +++++++++++--------------
 1 file changed, 11 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/bilibili.py b/youtube_dl/extractor/bilibili.py
index 1c3644587..b5aaa41e7 100644
--- a/youtube_dl/extractor/bilibili.py
+++ b/youtube_dl/extractor/bilibili.py
@@ -7,11 +7,13 @@ import json
 from .common import InfoExtractor
 from ..compat import (
     compat_etree_fromstring,
+    compat_str,
 )
 from ..utils import (
     int_or_none,
     unescapeHTML,
     ExtractorError,
+    xpath_text,
 )
 
 
@@ -58,27 +60,22 @@ class BiliBiliIE(InfoExtractor):
         cid = view_data['cid']
         title = unescapeHTML(view_data['title'])
 
-        page = self._download_webpage(
+        doc = self._download_xml(
             'http://interface.bilibili.com/v_cdn_play?appkey=8e9fc618fbd41e28&cid=%s' % cid,
             cid,
             'Downloading page %s/%s' % (page_num, view_data['pages'])
         )
-        try:
-            err_info = json.loads(page)
-            raise ExtractorError(
-                'BiliBili said: ' + err_info['error_text'], expected=True)
-        except ValueError:
-            pass
 
-        doc = compat_etree_fromstring(page)
+        if xpath_text(doc, './result') == 'error':
+            raise ExtractorError('%s said: %s' % (self.IE_NAME, xpath_text(doc, './message')), expected=True)
 
         entries = []
 
         for durl in doc.findall('./durl'):
-            size = durl.find('./filesize|./size')
+            size = xpath_text(durl, ['./filesize', './size'])
             formats = [{
                 'url': durl.find('./url').text,
-                'filesize': int_or_none(size.text) if size else None,
+                'filesize': int_or_none(size),
                 'ext': 'flv',
             }]
             backup_urls = durl.find('./backup_url')
@@ -88,21 +85,21 @@ class BiliBiliIE(InfoExtractor):
             formats.reverse()
 
             entries.append({
-                'id': '%s_part%s' % (cid, durl.find('./order').text),
+                'id': '%s_part%s' % (cid, xpath_text(durl, './order')),
                 'title': title,
-                'duration': int_or_none(durl.find('./length').text) // 1000,
+                'duration': int_or_none(xpath_text(durl, './length'), 1000),
                 'formats': formats,
             })
 
         info = {
-            'id': str(cid),
+            'id': compat_str(cid),
             'title': title,
             'description': view_data.get('description'),
             'thumbnail': view_data.get('pic'),
             'uploader': view_data.get('author'),
             'timestamp': int_or_none(view_data.get('created')),
             'view_count': view_data.get('play'),
-            'duration': int_or_none(doc.find('./timelength').text),
+            'duration': int_or_none(xpath_text(doc, './timelength')),
         }
 
         if len(entries) == 1:

From e5b4225f7c891c4971fe5acdf20459594d5ff9c2 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 3 Dec 2015 22:25:08 +0100
Subject: [PATCH 3242/6219] [audimedia] flake8

---
 youtube_dl/extractor/audimedia.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/audimedia.py b/youtube_dl/extractor/audimedia.py
index cad503510..b0b089dee 100644
--- a/youtube_dl/extractor/audimedia.py
+++ b/youtube_dl/extractor/audimedia.py
@@ -31,8 +31,8 @@ class AudiMediaIE(InfoExtractor):
     def _real_extract(self, url):
         display_id = self._match_id(url)
         webpage = self._download_webpage(url, display_id)
-        
-        raw_payload = self._search_regex(r'<script[^>]+class="amtv-embed"[^>]+id="([^"]+)"', webpage, 'raw payload');
+
+        raw_payload = self._search_regex(r'<script[^>]+class="amtv-embed"[^>]+id="([^"]+)"', webpage, 'raw payload')
         _, stage_mode, video_id, lang = raw_payload.split('-')
 
         # TODO: handle s and e stage_mode (live streams and ended live streams)

From 6612a34939d49d75b2f35d46b1ae9657c11e0f97 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 3 Dec 2015 22:43:19 +0100
Subject: [PATCH 3243/6219] [bilibili] flake8

---
 youtube_dl/extractor/bilibili.py | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/bilibili.py b/youtube_dl/extractor/bilibili.py
index b5aaa41e7..59beb11bc 100644
--- a/youtube_dl/extractor/bilibili.py
+++ b/youtube_dl/extractor/bilibili.py
@@ -2,13 +2,9 @@
 from __future__ import unicode_literals
 
 import re
-import json
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_etree_fromstring,
-    compat_str,
-)
+from ..compat import compat_str
 from ..utils import (
     int_or_none,
     unescapeHTML,
@@ -98,7 +94,7 @@ class BiliBiliIE(InfoExtractor):
             'thumbnail': view_data.get('pic'),
             'uploader': view_data.get('author'),
             'timestamp': int_or_none(view_data.get('created')),
-            'view_count': view_data.get('play'),
+            'view_count': int_or_none(view_data.get('play')),
             'duration': int_or_none(xpath_text(doc, './timelength')),
         }
 

From 4ccabf93db1839c07b9fa64e669ce54f33199057 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 4 Dec 2015 00:51:02 +0100
Subject: [PATCH 3244/6219] [trilulilu] fix info extraction

---
 youtube_dl/extractor/trilulilu.py | 118 ++++++++++++++++--------------
 1 file changed, 64 insertions(+), 54 deletions(-)

diff --git a/youtube_dl/extractor/trilulilu.py b/youtube_dl/extractor/trilulilu.py
index 185accc4b..3f523ec19 100644
--- a/youtube_dl/extractor/trilulilu.py
+++ b/youtube_dl/extractor/trilulilu.py
@@ -1,80 +1,90 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
-from ..utils import ExtractorError
+from ..utils import (
+    ExtractorError,
+    int_or_none,
+    parse_iso8601,
+)
 
 
 class TriluliluIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?trilulilu\.ro/(?:video-[^/]+/)?(?P<id>[^/#\?]+)'
-    _TEST = {
-        'url': 'http://www.trilulilu.ro/video-animatie/big-buck-bunny-1',
-        'md5': 'c1450a00da251e2769b74b9005601cac',
+    _VALID_URL = r'https?://(?:(?:www|m)\.)?trilulilu\.ro/(?:[^/]+/)?(?P<id>[^/#\?]+)'
+    _TESTS = [{
+        'url': 'http://www.trilulilu.ro/big-buck-bunny-1',
+        'md5': '68da087b676a6196a413549212f60cc6',
         'info_dict': {
             'id': 'ae2899e124140b',
             'ext': 'mp4',
             'title': 'Big Buck Bunny',
             'description': ':) pentru copilul din noi',
+            'uploader_id': 'chipy',
+            'upload_date': '20120304',
+            'timestamp': 1330830647,
+            'uploader': 'chipy',
+            'view_count': int,
+            'like_count': int,
+            'comment_count': int,
         },
-    }
+    }, {
+        'url': 'http://www.trilulilu.ro/adena-ft-morreti-inocenta',
+        'md5': '929dfb8729dc71750463af88bbbbf4a4',
+        'info_dict': {
+            'id': 'f299710e3c91c5',
+            'ext': 'mp4',
+            'title': 'Adena ft. Morreti - Inocenta',
+            'description': 'pop music',
+            'uploader_id': 'VEVOmixt',
+            'upload_date': '20151204',
+            'uploader': 'VEVOmixt',
+            'timestamp': 1449187937,
+            'view_count': int,
+            'like_count': int,
+            'comment_count': int,
+        },
+    }]
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
-        webpage = self._download_webpage(url, display_id)
+        media_info = self._download_json('http://m.trilulilu.ro/%s?format=json' % display_id, display_id)
+        user = media_info.get('user', {})
 
-        if re.search(r'Fişierul nu este disponibil pentru vizionare în ţara dumneavoastră', webpage):
-            raise ExtractorError(
-                'This video is not available in your country.', expected=True)
-        elif re.search('Fişierul poate fi accesat doar de către prietenii lui', webpage):
-            raise ExtractorError('This video is private.', expected=True)
+        thumbnail = media_info.get('cover_url')
+        if thumbnail:
+            thumbnail.format(width='1600', height='1200')
 
-        flashvars_str = self._search_regex(
-            r'block_flash_vars\s*=\s*(\{[^\}]+\})', webpage, 'flashvars', fatal=False, default=None)
-
-        if flashvars_str:
-            flashvars = self._parse_json(flashvars_str, display_id)
+        media_class = media_info.get('class')
+        if media_class in ('video', 'audio'):
+            # TODO: get correct ext for audio files
+            stream_type = media_info.get('stream_type')
+            formats = [{
+                'url': media_info['href'],
+                'ext': stream_type,
+            }]
+            if media_info.get('is_hd'):
+                formats.append({
+                    'url': media_info['hrefhd'],
+                    'ext': stream_type,
+                })
         else:
-            raise ExtractorError(
-                'This page does not contain videos', expected=True)
+            raise ExtractorError('not a video or an audio')
 
-        if flashvars['isMP3'] == 'true':
-            raise ExtractorError(
-                'Audio downloads are currently not supported', expected=True)
-
-        video_id = flashvars['hash']
-        title = self._og_search_title(webpage)
-        thumbnail = self._og_search_thumbnail(webpage)
-        description = self._og_search_description(webpage, default=None)
-
-        format_url = ('http://fs%(server)s.trilulilu.ro/%(hash)s/'
-                      'video-formats2' % flashvars)
-        format_doc = self._download_xml(
-            format_url, video_id,
-            note='Downloading formats',
-            errnote='Error while downloading formats')
-
-        video_url_template = (
-            'http://fs%(server)s.trilulilu.ro/stream.php?type=video'
-            '&source=site&hash=%(hash)s&username=%(userid)s&'
-            'key=ministhebest&format=%%s&sig=&exp=' %
-            flashvars)
-        formats = [
-            {
-                'format_id': fnode.text.partition('-')[2],
-                'url': video_url_template % fnode.text,
-                'ext': fnode.text.partition('-')[0]
-            }
-
-            for fnode in format_doc.findall('./formats/format')
-        ]
+        if media_class == 'audio':
+            formats[0]['vcodec'] = 'none'
 
         return {
-            'id': video_id,
+            'id': media_info['identifier'].split('|')[1],
             'display_id': display_id,
             'formats': formats,
-            'title': title,
-            'description': description,
+            'title': media_info['title'],
+            'description': media_info.get('description'),
             'thumbnail': thumbnail,
+            'uploader_id': user.get('username'),
+            'uploader': user.get('fullname'),
+            'timestamp': parse_iso8601(media_info.get('published'), ' '),
+            'duration': int_or_none(media_info.get('duration')),
+            'view_count': int_or_none(media_info.get('count_views')),
+            'like_count': int_or_none(media_info.get('count_likes')),
+            'comment_count': int_or_none(media_info.get('count_comments')),
         }

From 71d9fe7818b8c766c3b1190017746d8e51db3081 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 4 Dec 2015 06:53:33 +0100
Subject: [PATCH 3245/6219] [trilulilu] improve extraction

---
 youtube_dl/extractor/trilulilu.py | 33 +++++++++++++++++--------------
 1 file changed, 18 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/trilulilu.py b/youtube_dl/extractor/trilulilu.py
index 3f523ec19..515632527 100644
--- a/youtube_dl/extractor/trilulilu.py
+++ b/youtube_dl/extractor/trilulilu.py
@@ -48,30 +48,33 @@ class TriluliluIE(InfoExtractor):
     def _real_extract(self, url):
         display_id = self._match_id(url)
         media_info = self._download_json('http://m.trilulilu.ro/%s?format=json' % display_id, display_id)
+
+        media_class = media_info.get('class')
+        if media_class not in ('video', 'audio'):
+            raise ExtractorError('not a video or an audio')
+
         user = media_info.get('user', {})
 
         thumbnail = media_info.get('cover_url')
         if thumbnail:
             thumbnail.format(width='1600', height='1200')
 
-        media_class = media_info.get('class')
-        if media_class in ('video', 'audio'):
-            # TODO: get correct ext for audio files
-            stream_type = media_info.get('stream_type')
-            formats = [{
-                'url': media_info['href'],
+        # TODO: get correct ext for audio files
+        stream_type = media_info.get('stream_type')
+        formats = [{
+            'url': media_info['href'],
+            'ext': stream_type,
+        }]
+        if media_info.get('is_hd'):
+            formats.append({
+                'format_id': 'hd',
+                'url': media_info['hrefhd'],
                 'ext': stream_type,
-            }]
-            if media_info.get('is_hd'):
-                formats.append({
-                    'url': media_info['hrefhd'],
-                    'ext': stream_type,
-                })
-        else:
-            raise ExtractorError('not a video or an audio')
-
+            })
         if media_class == 'audio':
             formats[0]['vcodec'] = 'none'
+        else:
+            formats[0]['format_id'] = 'sd'
 
         return {
             'id': media_info['identifier'].split('|')[1],

From 46cc1c65a4b5c57555f2549534788ad004c46c03 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 4 Dec 2015 07:09:48 +0100
Subject: [PATCH 3246/6219] [nba] use xpath utils

---
 youtube_dl/extractor/nba.py | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/nba.py b/youtube_dl/extractor/nba.py
index a0cc58c12..77a3b49ef 100644
--- a/youtube_dl/extractor/nba.py
+++ b/youtube_dl/extractor/nba.py
@@ -6,6 +6,8 @@ from .common import InfoExtractor
 from ..utils import (
     parse_duration,
     int_or_none,
+    xpath_text,
+    xpath_attr,
 )
 
 
@@ -26,7 +28,7 @@ class NBAIE(InfoExtractor):
     }, {
         'url': 'http://www.nba.com/video/games/hornets/2014/12/05/0021400276-nyk-cha-play5.nba/',
         'only_matching': True,
-    },{
+    }, {
         'url': 'http://watch.nba.com/video/channels/playoffs/2015/05/20/0041400301-cle-atl-recap.nba',
         'md5': 'b2b39b81cf28615ae0c3360a3f9668c4',
         'info_dict': {
@@ -43,11 +45,11 @@ class NBAIE(InfoExtractor):
     def _real_extract(self, url):
         path, video_id = re.match(self._VALID_URL, url).groups()
         video_info = self._download_xml('http://www.nba.com/%s.xml' % path, video_id)
-        video_id = video_info.find('slug').text
-        title = video_info.find('headline').text
-        description = video_info.find('description').text
-        duration = parse_duration(video_info.find('length').text)
-        timestamp = int_or_none(video_info.find('dateCreated').attrib.get('uts'))
+        video_id = xpath_text(video_info, 'slug')
+        title = xpath_text(video_info, 'headline')
+        description = xpath_text(video_info, 'description')
+        duration = parse_duration(xpath_text(video_info, 'length'))
+        timestamp = int_or_none(xpath_attr(video_info, 'dateCreated', 'uts'))
 
         thumbnails = []
         for image in video_info.find('images'):

From fad2428f47428b75d7728c338ba61407e2b281bc Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 4 Dec 2015 08:24:04 +0100
Subject: [PATCH 3247/6219] [gameinformer] split long line

---
 youtube_dl/extractor/gameinformer.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/gameinformer.py b/youtube_dl/extractor/gameinformer.py
index 8bb764703..25870c131 100644
--- a/youtube_dl/extractor/gameinformer.py
+++ b/youtube_dl/extractor/gameinformer.py
@@ -10,7 +10,6 @@ class GameInformerIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?gameinformer\.com/(?:[^/]+/)*(?P<id>.+)\.aspx'
     _TEST = {
         'url': 'http://www.gameinformer.com/b/features/archive/2015/09/26/replay-animal-crossing.aspx',
-        'md5': 'TODO: md5 sum of the first 10241 bytes of the video file (use --test)',
         'info_dict': {
             'id': '4515472681001',
             'ext': 'm3u8',
@@ -29,7 +28,9 @@ class GameInformerIE(InfoExtractor):
         webpage = self._download_webpage(url, display_id)
 
         bc_api_url = self._search_regex(r"getVideo\('([^']+)'", webpage, 'brightcove api url')
-        json_data = self._download_json(bc_api_url + '&video_fields=id,name,shortDescription,publishedDate,videoStillURL,length,IOSRenditions', display_id)
+        json_data = self._download_json(
+            bc_api_url + '&video_fields=id,name,shortDescription,publishedDate,videoStillURL,length,IOSRenditions',
+            display_id)
 
         return {
             'id': compat_str(json_data['id']),

From 2c3b9f35703def516e455448666af0b4c5b31ec9 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 4 Dec 2015 08:37:08 +0100
Subject: [PATCH 3248/6219] [adobetv] use a variable for api base url

---
 youtube_dl/extractor/adobetv.py | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/adobetv.py b/youtube_dl/extractor/adobetv.py
index d0bfafa45..d65826932 100644
--- a/youtube_dl/extractor/adobetv.py
+++ b/youtube_dl/extractor/adobetv.py
@@ -14,7 +14,11 @@ from ..utils import (
 )
 
 
-class AdobeTVIE(InfoExtractor):
+class AdobeTVBaseIE(InfoExtractor):
+    _API_BASE_URL = 'http://tv.adobe.com/api/v4/'
+
+
+class AdobeTVIE(AdobeTVBaseIE):
     _VALID_URL = r'https?://tv\.adobe\.com/(?:(?P<language>fr|de|es|jp)/)?watch/(?P<show_urlname>[^/]+)/(?P<id>[^/]+)'
 
     _TEST = {
@@ -38,7 +42,7 @@ class AdobeTVIE(InfoExtractor):
             language = 'en'
 
         video_data = self._download_json(
-            'http://tv.adobe.com/api/v4/episode/get/?language=%s&show_urlname=%s&urlname=%s&disclosure=standard' % (language, show_urlname, urlname),
+            self._API_BASE_URL + 'episode/get/?language=%s&show_urlname=%s&urlname=%s&disclosure=standard' % (language, show_urlname, urlname),
             urlname)['data'][0]
 
         formats = [{
@@ -62,7 +66,7 @@ class AdobeTVIE(InfoExtractor):
         }
 
 
-class AdobeTVPlaylistBaseIE(InfoExtractor):
+class AdobeTVPlaylistBaseIE(AdobeTVBaseIE):
     def _parse_page_data(self, page_data):
         return [self.url_result(self._get_element_url(element_data)) for element_data in page_data]
 
@@ -97,11 +101,10 @@ class AdobeTVShowIE(AdobeTVPlaylistBaseIE):
             language = 'en'
         query = 'language=%s&show_urlname=%s' % (language, show_urlname)
 
-        show_data = self._download_json(
-            'http://tv.adobe.com/api/v4/show/get/?%s' % query, show_urlname)['data'][0]
+        show_data = self._download_json(self._API_BASE_URL + 'show/get/?%s' % query, show_urlname)['data'][0]
 
         return self.playlist_result(
-            self._extract_playlist_entries('http://tv.adobe.com/api/v4/episode/?%s' % query, show_urlname),
+            self._extract_playlist_entries(self._API_BASE_URL + 'episode/?%s' % query, show_urlname),
             str(show_data['id']),
             show_data['show_name'],
             show_data['show_description'])
@@ -130,7 +133,7 @@ class AdobeTVChannelIE(AdobeTVPlaylistBaseIE):
             query += '&category_urlname=%s' % category_urlname
 
         return self.playlist_result(
-            self._extract_playlist_entries('http://tv.adobe.com/api/v4/show/?%s' % query, channel_urlname),
+            self._extract_playlist_entries(self._API_BASE_URL + 'show/?%s' % query, channel_urlname),
             channel_urlname)
 
 

From 7079f8ff1fe524a5711957a426066b94a22a605e Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 4 Dec 2015 08:44:18 +0100
Subject: [PATCH 3249/6219] [adobetv] use compat_str

---
 youtube_dl/extractor/adobetv.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/adobetv.py b/youtube_dl/extractor/adobetv.py
index d65826932..8753ee2cf 100644
--- a/youtube_dl/extractor/adobetv.py
+++ b/youtube_dl/extractor/adobetv.py
@@ -3,6 +3,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import compat_str
 from ..utils import (
     parse_duration,
     unified_strdate,
@@ -55,7 +56,7 @@ class AdobeTVIE(AdobeTVBaseIE):
         self._sort_formats(formats)
 
         return {
-            'id': str(video_data['id']),
+            'id': compat_str(video_data['id']),
             'title': video_data['title'],
             'description': video_data.get('description'),
             'thumbnail': video_data.get('thumbnail'),
@@ -105,7 +106,7 @@ class AdobeTVShowIE(AdobeTVPlaylistBaseIE):
 
         return self.playlist_result(
             self._extract_playlist_entries(self._API_BASE_URL + 'episode/?%s' % query, show_urlname),
-            str(show_data['id']),
+            compat_str(show_data['id']),
             show_data['show_name'],
             show_data['show_description'])
 

From 2a63b0f1109f3f1c1f2d8fd48bfa49b080048c98 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 4 Dec 2015 14:26:34 +0100
Subject: [PATCH 3250/6219] [mixcloud] Fix extraction of the audio url (fixes
 #7751)

---
 youtube_dl/extractor/mixcloud.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dl/extractor/mixcloud.py
index d47aeceda..c2b7ed9ab 100644
--- a/youtube_dl/extractor/mixcloud.py
+++ b/youtube_dl/extractor/mixcloud.py
@@ -64,7 +64,8 @@ class MixcloudIE(InfoExtractor):
 
         preview_url = self._search_regex(
             r'\s(?:data-preview-url|m-preview)="([^"]+)"', webpage, 'preview url')
-        song_url = preview_url.replace('/previews/', '/c/originals/')
+        song_url = re.sub(r'audiocdn(\d+)', r'stream\1', preview_url)
+        song_url = song_url.replace('/previews/', '/c/originals/')
         if not self._check_url(song_url, track_id, 'mp3'):
             song_url = song_url.replace('.mp3', '.m4a').replace('originals/', 'm4a/64/')
             if not self._check_url(song_url, track_id, 'm4a'):

From 55bec9b658ab6a06f7568ccaec4ef212543aa4f8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 4 Dec 2015 14:29:37 +0100
Subject: [PATCH 3251/6219] [clipfish] Remove unused import and style fix

---
 youtube_dl/extractor/clipfish.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/clipfish.py b/youtube_dl/extractor/clipfish.py
index d142e326f..7d8eb27a1 100644
--- a/youtube_dl/extractor/clipfish.py
+++ b/youtube_dl/extractor/clipfish.py
@@ -1,7 +1,5 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..utils import (
     int_or_none,
@@ -34,7 +32,7 @@ class ClipfishIE(InfoExtractor):
             'url': video_info['media_videourl_hls'].replace('de.hls.fra.clipfish.de', 'hls.fra.clipfish.de'),
             'ext': 'mp4',
             'format_id': 'hls',
-        },{
+        }, {
             'url': video_info['media_videourl'],
             'format_id': 'mp4',
             'width': int_or_none(video_info.get('width')),

From b80d4bebf307a18ea0e4991503f91ef601cb2091 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 4 Dec 2015 16:04:22 +0100
Subject: [PATCH 3252/6219] [nba] fix extraction errors

---
 youtube_dl/extractor/nba.py | 20 +++++++++++++-------
 1 file changed, 13 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/nba.py b/youtube_dl/extractor/nba.py
index 77a3b49ef..d8f901b55 100644
--- a/youtube_dl/extractor/nba.py
+++ b/youtube_dl/extractor/nba.py
@@ -44,6 +44,8 @@ class NBAIE(InfoExtractor):
 
     def _real_extract(self, url):
         path, video_id = re.match(self._VALID_URL, url).groups()
+        if path.startswith('nba/'):
+            path = path[3:]
         video_info = self._download_xml('http://www.nba.com/%s.xml' % path, video_id)
         video_id = xpath_text(video_info, 'slug')
         title = xpath_text(video_info, 'headline')
@@ -61,23 +63,27 @@ class NBAIE(InfoExtractor):
             })
 
         formats = []
-        for video_file in video_info.find('files').iter('file'):
+        for video_file in video_info.findall('.//file'):
             video_url = video_file.text
             if video_url.startswith('/'):
                 continue
             if video_url.endswith('.m3u8'):
-                formats.extend(self._extract_m3u8_formats(video_url, video_id, m3u8_id='hls'))
+                m3u8_formats = self._extract_m3u8_formats(video_url, video_id, m3u8_id='hls', fatal=False)
+                if m3u8_formats:
+                    formats.extend(m3u8_formats)
             elif video_url.endswith('.f4m'):
-                formats.extend(self._extract_f4m_formats(video_url + '?hdcore=3.4.1.1', video_id, f4m_id='hds'))
+                f4m_formats = self._extract_f4m_formats(video_url + '?hdcore=3.4.1.1', video_id, f4m_id='hds', fatal=False)
+                if f4m_formats:
+                    formats.extend(f4m_formats)
             else:
                 key = video_file.attrib.get('bitrate')
-                width, height, bitrate = re.search(r'(\d+)x(\d+)(?:_(\d+))?', key).groups()
+                mobj = re.search(r'(\d+)x(\d+)(?:_(\d+))?', key)
                 formats.append({
                     'format_id': key,
                     'url': video_url,
-                    'width': int_or_none(width),
-                    'height': int_or_none(height),
-                    'tbr': int_or_none(bitrate),
+                    'width': int_or_none(mobj.group(1)),
+                    'height': int_or_none(mobj.group(2)),
+                    'tbr': int_or_none(mobj.group(3)),
                 })
         self._sort_formats(formats)
 

From 53e06b25070580b8b0dab7dd54ddff82f3bd2583 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 4 Dec 2015 16:18:02 +0100
Subject: [PATCH 3253/6219] [ooyala] fix duration scale

---
 youtube_dl/extractor/byutv.py           | 2 +-
 youtube_dl/extractor/generic.py         | 6 +++---
 youtube_dl/extractor/groupon.py         | 2 +-
 youtube_dl/extractor/howcast.py         | 2 +-
 youtube_dl/extractor/ooyala.py          | 8 ++++----
 youtube_dl/extractor/teachingchannel.py | 2 +-
 youtube_dl/extractor/vice.py            | 2 +-
 7 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/byutv.py b/youtube_dl/extractor/byutv.py
index ce25816f0..dda98059e 100644
--- a/youtube_dl/extractor/byutv.py
+++ b/youtube_dl/extractor/byutv.py
@@ -17,7 +17,7 @@ class BYUtvIE(InfoExtractor):
             'description': 'md5:e07269172baff037f8e8bf9956bc9747',
             'title': 'Season 5 Episode 5',
             'thumbnail': 're:^https?://.*\.jpg$',
-            'duration': 1486486,
+            'duration': 1486.486,
         },
         'params': {
             'skip_download': True,
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index b60684f98..e3bdff2d8 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -339,7 +339,7 @@ class GenericIE(InfoExtractor):
                 'id': 'BwY2RxaTrTkslxOfcan0UCf0YqyvWysJ',
                 'ext': 'mp4',
                 'title': '2cc213299525360.mov',  # that's what we get
-                'duration': 238231,
+                'duration': 238.231,
             },
             'add_ie': ['Ooyala'],
         },
@@ -351,7 +351,7 @@ class GenericIE(InfoExtractor):
                 'ext': 'mp4',
                 'title': '"Steve Jobs: Man in the Machine" trailer',
                 'description': 'The first trailer for the Alex Gibney documentary "Steve Jobs: Man in the Machine."',
-                'duration': 135427,
+                'duration': 135.427,
             },
             'params': {
                 'skip_download': True,
@@ -964,7 +964,7 @@ class GenericIE(InfoExtractor):
                 'ext': 'mp4',
                 'description': 'VIDEO: INDEX/MATCH versus VLOOKUP.',
                 'title': 'This is what separates the Excel masters from the wannabes',
-                'duration': 191933,
+                'duration': 191.933,
             },
             'params': {
                 # m3u8 downloads
diff --git a/youtube_dl/extractor/groupon.py b/youtube_dl/extractor/groupon.py
index 22ff7182f..63c05b6a6 100644
--- a/youtube_dl/extractor/groupon.py
+++ b/youtube_dl/extractor/groupon.py
@@ -19,7 +19,7 @@ class GrouponIE(InfoExtractor):
                 'ext': 'mp4',
                 'title': 'Bikram Yoga Huntington Beach | Orange County',
                 'description': 'md5:d41d8cd98f00b204e9800998ecf8427e',
-                'duration': 44961,
+                'duration': 44.961,
             },
         }],
         'params': {
diff --git a/youtube_dl/extractor/howcast.py b/youtube_dl/extractor/howcast.py
index 165b9f39e..e8f51e545 100644
--- a/youtube_dl/extractor/howcast.py
+++ b/youtube_dl/extractor/howcast.py
@@ -16,7 +16,7 @@ class HowcastIE(InfoExtractor):
             'description': 'md5:dbe792e5f6f1489027027bf2eba188a3',
             'timestamp': 1276081287,
             'upload_date': '20100609',
-            'duration': 56823,
+            'duration': 56.823,
         },
         'params': {
             # m3u8 download
diff --git a/youtube_dl/extractor/ooyala.py b/youtube_dl/extractor/ooyala.py
index 3b692e903..35067e271 100644
--- a/youtube_dl/extractor/ooyala.py
+++ b/youtube_dl/extractor/ooyala.py
@@ -24,7 +24,7 @@ class OoyalaBaseIE(InfoExtractor):
             'title': metadata['title'],
             'description': metadata.get('description'),
             'thumbnail': metadata.get('thumbnail_image') or metadata.get('promo_image'),
-            'duration': int_or_none(metadata.get('duration')),
+            'duration': float_or_none(metadata.get('duration'), 1000),
         }
 
         formats = []
@@ -78,7 +78,7 @@ class OoyalaIE(OoyalaBaseIE):
                 'ext': 'mp4',
                 'title': 'Explaining Data Recovery from Hard Drives and SSDs',
                 'description': 'How badly damaged does a drive have to be to defeat Russell and his crew? Apparently, smashed to bits.',
-                'duration': 853386,
+                'duration': 853.386,
             },
         }, {
             # Only available for ipad
@@ -87,7 +87,7 @@ class OoyalaIE(OoyalaBaseIE):
                 'id': 'x1b3lqZDq9y_7kMyC2Op5qo-p077tXD0',
                 'ext': 'mp4',
                 'title': 'Simulation Overview - Levels of Simulation',
-                'duration': 194948,
+                'duration': 194.948,
             },
         },
         {
@@ -99,7 +99,7 @@ class OoyalaIE(OoyalaBaseIE):
                 'id': 'FiOG81ZTrvckcchQxmalf4aQj590qTEx',
                 'ext': 'mp4',
                 'title': 'Divide Tool Path.mp4',
-                'duration': 204405,
+                'duration': 204.405,
             }
         }
     ]
diff --git a/youtube_dl/extractor/teachingchannel.py b/youtube_dl/extractor/teachingchannel.py
index 36a6fc679..e0477382c 100644
--- a/youtube_dl/extractor/teachingchannel.py
+++ b/youtube_dl/extractor/teachingchannel.py
@@ -16,7 +16,7 @@ class TeachingChannelIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'A History of Teaming',
             'description': 'md5:2a9033db8da81f2edffa4c99888140b3',
-            'duration': 422255,
+            'duration': 422.255,
         },
         'params': {
             # m3u8 download
diff --git a/youtube_dl/extractor/vice.py b/youtube_dl/extractor/vice.py
index 7df87c31c..3db6286e4 100644
--- a/youtube_dl/extractor/vice.py
+++ b/youtube_dl/extractor/vice.py
@@ -15,7 +15,7 @@ class ViceIE(InfoExtractor):
                 'id': '43cW1mYzpia9IlestBjVpd23Yu3afAfp',
                 'ext': 'mp4',
                 'title': 'VICE_COWBOYCAPITALISTS_PART01_v1_VICE_WM_1080p.mov',
-                'duration': 725983,
+                'duration': 725.983,
             },
             'params': {
                 # Requires ffmpeg (m3u8 manifest)

From 96929dd1e835842cd3c5fad0255a517731f7008d Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 4 Dec 2015 16:23:44 +0100
Subject: [PATCH 3254/6219] [skynewsarabia] fix extractor name

---
 youtube_dl/extractor/skynewsarabia.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/skynewsarabia.py b/youtube_dl/extractor/skynewsarabia.py
index f09fee102..05e1b02ad 100644
--- a/youtube_dl/extractor/skynewsarabia.py
+++ b/youtube_dl/extractor/skynewsarabia.py
@@ -9,7 +9,7 @@ from ..utils import (
 )
 
 
-class SkyNewArabiaBaseIE(InfoExtractor):
+class SkyNewsArabiaBaseIE(InfoExtractor):
     _IMAGE_BASE_URL = 'http://www.skynewsarabia.com/web/images'
 
     def _call_api(self, path, value):
@@ -40,7 +40,7 @@ class SkyNewArabiaBaseIE(InfoExtractor):
         }
 
 
-class SkyNewsArabiaIE(SkyNewArabiaBaseIE):
+class SkyNewsArabiaIE(SkyNewsArabiaBaseIE):
     IE_NAME = 'skynewsarabia:video'
     _VALID_URL = r'https?://(?:www\.)?skynewsarabia\.com/web/video/(?P<id>[0-9]+)'
     _TEST = {
@@ -66,7 +66,7 @@ class SkyNewsArabiaIE(SkyNewArabiaBaseIE):
         return self._extract_video_info(video_data)
 
 
-class SkyNewsArabiaArticleIE(SkyNewArabiaBaseIE):
+class SkyNewsArabiaArticleIE(SkyNewsArabiaBaseIE):
     IE_NAME = 'skynewsarabia:video'
     _VALID_URL = r'https?://(?:www\.)?skynewsarabia\.com/web/article/(?P<id>[0-9]+)'
     _TESTS = [{

From bee4c5571a18ac52aada80ac7ce9c3e409c18293 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 4 Dec 2015 16:38:05 +0100
Subject: [PATCH 3255/6219] [clipfish] improve extraction

---
 youtube_dl/extractor/clipfish.py | 34 +++++++++++++++++++++-----------
 1 file changed, 22 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/clipfish.py b/youtube_dl/extractor/clipfish.py
index 7d8eb27a1..3a47f6fa4 100644
--- a/youtube_dl/extractor/clipfish.py
+++ b/youtube_dl/extractor/clipfish.py
@@ -26,19 +26,29 @@ class ClipfishIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        video_info = self._download_json('http://www.clipfish.de/devapi/id/%s?format=json&apikey=hbbtv' % video_id, video_id)['items'][0]
+        video_info = self._download_json(
+            'http://www.clipfish.de/devapi/id/%s?format=json&apikey=hbbtv' % video_id,
+            video_id)['items'][0]
 
-        formats = [{
-            'url': video_info['media_videourl_hls'].replace('de.hls.fra.clipfish.de', 'hls.fra.clipfish.de'),
-            'ext': 'mp4',
-            'format_id': 'hls',
-        }, {
-            'url': video_info['media_videourl'],
-            'format_id': 'mp4',
-            'width': int_or_none(video_info.get('width')),
-            'height': int_or_none(video_info.get('height')),
-            'tbr': int_or_none(video_info.get('bitrate')),
-        }]
+        formats = []
+
+        m3u8_url = video_info.get('media_videourl_hls')
+        if m3u8_url:
+            formats.append({
+                'url': m3u8_url.replace('de.hls.fra.clipfish.de', 'hls.fra.clipfish.de'),
+                'ext': 'mp4',
+                'format_id': 'hls',
+            })
+
+        mp4_url = video_info.get('media_videourl')
+        if mp4_url:
+            formats.append({
+                'url': mp4_url,
+                'format_id': 'mp4',
+                'width': int_or_none(video_info.get('width')),
+                'height': int_or_none(video_info.get('height')),
+                'tbr': int_or_none(video_info.get('bitrate')),
+            })
 
         return {
             'id': video_id,

From 64b7e89c0c1f52502ce4fdb7fda4cc1531e578fd Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 5 Dec 2015 16:26:30 +0800
Subject: [PATCH 3256/6219] [srf] Support audios (closes #7760)

---
 youtube_dl/extractor/srf.py | 26 +++++++++++++++++++++-----
 1 file changed, 21 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/srf.py b/youtube_dl/extractor/srf.py
index 77eec0bc7..16e1bf2d6 100644
--- a/youtube_dl/extractor/srf.py
+++ b/youtube_dl/extractor/srf.py
@@ -11,7 +11,7 @@ from ..utils import (
 
 
 class SrfIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.srf\.ch/play(?:er)?/tv/[^/]+/video/(?P<display_id>[^?]+)\?id=|tp\.srgssr\.ch/p/flash\?urn=urn:srf:ais:video:)(?P<id>[0-9a-f\-]{36})'
+    _VALID_URL = r'https?://(?:www\.srf\.ch/play(?:er)?/(?:tv|radio)/[^/]+/(?P<media_type>video|audio)/(?P<display_id>[^?]+)\?id=|tp\.srgssr\.ch/p/flash\?urn=urn:srf:ais:video:)(?P<id>[0-9a-f\-]{36})'
     _TESTS = [{
         'url': 'http://www.srf.ch/play/tv/10vor10/video/snowden-beantragt-asyl-in-russland?id=28e1a57d-5b76-4399-8ab3-9097f071e6c5',
         'md5': '4cd93523723beff51bb4bee974ee238d',
@@ -35,6 +35,20 @@ class SrfIE(InfoExtractor):
             'title': 'Jaguar XK120, Shadow und Tornado-Dampflokomotive',
             'timestamp': 1373493600,
         },
+    }, {
+        'url': 'http://www.srf.ch/play/radio/hoerspielarchiv-srf-musikwelle/audio/saegel-ohni-wind-von-jakob-stebler?id=415bf3d3-6429-4de7-968d-95866e37cfbc',
+        'md5': '',
+        'info_dict': {
+            'id': '415bf3d3-6429-4de7-968d-95866e37cfbc',
+            'display_id': 'saegel-ohni-wind-von-jakob-stebler',
+            'ext': 'mp3',
+            'upload_date': '20080518',
+            'title': '«Sägel ohni Wind» von Jakob Stebler',
+            'timestamp': 1211112000,
+        },
+        'params': {
+            'skip_download': True,  # requires rtmpdump
+        },
     }, {
         'url': 'http://www.srf.ch/player/tv/10vor10/video/snowden-beantragt-asyl-in-russland?id=28e1a57d-5b76-4399-8ab3-9097f071e6c5',
         'only_matching': True,
@@ -44,11 +58,13 @@ class SrfIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-        display_id = re.match(self._VALID_URL, url).group('display_id') or video_id
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        media_type = mobj.group('media_type')
+        display_id = mobj.group('display_id') or video_id
 
         video_data = self._download_xml(
-            'http://il.srgssr.ch/integrationlayer/1.0/ue/srf/video/play/%s.xml' % video_id,
+            'http://il.srgssr.ch/integrationlayer/1.0/ue/srf/%s/play/%s.xml' % (media_type, video_id),
             display_id)
 
         title = xpath_text(
@@ -64,7 +80,7 @@ class SrfIE(InfoExtractor):
             for url_node in item.findall('url'):
                 quality = url_node.attrib['quality']
                 full_url = url_node.text
-                original_ext = determine_ext(full_url)
+                original_ext = determine_ext(full_url).lower()
                 format_id = '%s-%s' % (quality, item.attrib['protocol'])
                 if original_ext == 'f4m':
                     formats.extend(self._extract_f4m_formats(

From 02e3952f3b2a289dd69ee2f922d7e8f5d8759e4d Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 5 Dec 2015 09:29:14 +0100
Subject: [PATCH 3257/6219] [trilulilu] handle errors

---
 youtube_dl/extractor/trilulilu.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/youtube_dl/extractor/trilulilu.py b/youtube_dl/extractor/trilulilu.py
index 515632527..a800449e9 100644
--- a/youtube_dl/extractor/trilulilu.py
+++ b/youtube_dl/extractor/trilulilu.py
@@ -49,6 +49,15 @@ class TriluliluIE(InfoExtractor):
         display_id = self._match_id(url)
         media_info = self._download_json('http://m.trilulilu.ro/%s?format=json' % display_id, display_id)
 
+        age_limit = 0
+        errors = media_info.get('errors', {})
+        if errors.get('friends'):
+            raise ExtractorError('This video is private.', expected=True)
+        elif errors.get('geoblock'):
+            raise ExtractorError('This video is not available in your country.', expected=True)
+        elif errors.get('xxx_unlogged'):
+            age_limit = 18
+
         media_class = media_info.get('class')
         if media_class not in ('video', 'audio'):
             raise ExtractorError('not a video or an audio')
@@ -90,4 +99,5 @@ class TriluliluIE(InfoExtractor):
             'view_count': int_or_none(media_info.get('count_views')),
             'like_count': int_or_none(media_info.get('count_likes')),
             'comment_count': int_or_none(media_info.get('count_comments')),
+            'age_limit': age_limit,
         }

From 679bacf0b5bea22944cf95acd1fdeb064b98d7c7 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 5 Dec 2015 16:45:24 +0800
Subject: [PATCH 3258/6219] [bbc.co.uk] Fix test_BBCCoUk

This is similar to the one in #7756, So also fixes #7756.
---
 youtube_dl/extractor/bbc.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 7fb80aa38..cd129ba71 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -47,9 +47,8 @@ class BBCCoUkIE(InfoExtractor):
             'info_dict': {
                 'id': 'b039d07m',
                 'ext': 'flv',
-                'title': 'Kaleidoscope, Leonard Cohen',
+                'title': 'Leonard Cohen, Kaleidoscope - BBC Radio 4',
                 'description': 'The Canadian poet and songwriter reflects on his musical career.',
-                'duration': 1740,
             },
             'params': {
                 # rtmp download
@@ -454,6 +453,7 @@ class BBCCoUkIE(InfoExtractor):
         webpage = self._download_webpage(url, group_id, 'Downloading video page')
 
         programme_id = None
+        duration = None
 
         tviplayer = self._search_regex(
             r'mediator\.bind\(({.+?})\s*,\s*document\.getElementById',
@@ -473,7 +473,9 @@ class BBCCoUkIE(InfoExtractor):
             title = self._og_search_title(webpage)
             description = self._search_regex(
                 r'<p class="[^"]*medium-description[^"]*">([^<]+)</p>',
-                webpage, 'description', fatal=False)
+                webpage, 'description', default=None)
+            if not description:
+                description = self._html_search_meta('description', webpage)
         else:
             programme_id, title, description, duration, formats, subtitles = self._download_playlist(group_id)
 

From a3ef0e1cdd05698453f30f9250361c42a9fa58df Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 5 Dec 2015 16:51:13 +0800
Subject: [PATCH 3259/6219] [bbc.co.uk] Skip removed test video

---
 youtube_dl/extractor/bbc.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index cd129ba71..d789c3c5b 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -111,7 +111,8 @@ class BBCCoUkIE(InfoExtractor):
             'params': {
                 # rtmp download
                 'skip_download': True,
-            }
+            },
+            'skip': 'Episode is no longer available on BBC iPlayer Radio',
         }, {
             'url': 'http://www.bbc.co.uk/music/clips/p02frcc3',
             'note': 'Audio',

From 0bc4ee60e09ba8d94cfd4092f373b7b877fa053c Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 5 Dec 2015 16:54:25 +0800
Subject: [PATCH 3260/6219] [bbc] Fix test_BBC_6

---
 youtube_dl/extractor/bbc.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index d789c3c5b..d89e34ba0 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -590,6 +590,7 @@ class BBCIE(BBCCoUkIE):
             'ext': 'mp4',
             'title': '''Judge Mindy Glazer: "I'm sorry to see you here... I always wondered what happened to you"''',
             'duration': 56,
+            'description': '''Judge Mindy Glazer: "I'm sorry to see you here... I always wondered what happened to you"''',
         },
         'params': {
             'skip_download': True,

From ab3b773bbecfa4c3a009f141e720a455b1073b61 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 5 Dec 2015 10:14:34 +0100
Subject: [PATCH 3261/6219] [acast] change tests into more stable casts and
 work with channel extractor only if it didn't match cast regex

---
 youtube_dl/extractor/acast.py | 28 ++++++++++++++++------------
 1 file changed, 16 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/acast.py b/youtube_dl/extractor/acast.py
index c2719ca32..be7913bc7 100644
--- a/youtube_dl/extractor/acast.py
+++ b/youtube_dl/extractor/acast.py
@@ -16,15 +16,15 @@ class ACastIE(ACastBaseIE):
     IE_NAME = 'acast'
     _VALID_URL = r'https?://(?:www\.)?acast\.com/(?P<channel>[^/]+)/(?P<id>[^/#?]+)'
     _TEST = {
-        'url': 'https://www.acast.com/gardenersquestiontime/liverpool',
-        'md5': '9e9cd59c3a8a7d8d5407605f51093050',
+        'url': 'https://www.acast.com/condenasttraveler/-where-are-you-taipei-101-taiwan',
+        'md5': 'ada3de5a1e3a2a381327d749854788bb',
         'info_dict': {
-            'id': '43da2262-ade7-420c-8564-f6367da7c010',
+            'id': '57de3baa-4bb0-487e-9418-2692c1277a34',
             'ext': 'mp3',
-            'title': 'Liverpool',
-            'timestamp': 1446163200000,
-            'description': 'md5:170432c9956eec0670d7080a75000d5b',
-            'duration': 2520,
+            'title': '"Where Are You?": Taipei 101, Taiwan',
+            'timestamp': 1196172000000,
+            'description': 'md5:0c5d8201dfea2b93218ea986c91eee6e',
+            'duration': 211,
         }
     }
 
@@ -48,15 +48,19 @@ class ACastChannelIE(ACastBaseIE):
     IE_NAME = 'acast:channel'
     _VALID_URL = r'https?://(?:www\.)?acast\.com/(?P<id>[^/#?]+)'
     _TEST = {
-        'url': 'https://www.acast.com/gardenersquestiontime',
+        'url': 'https://www.acast.com/condenasttraveler',
         'info_dict': {
-            'id': '9d8f6f73-6b9d-4d16-9399-52bf88f8f611',
-            'title': 'Gardeners\' Question Time',
-            'description': 'md5:c7ef18049da6a52b63d371b3edccce90',
+            'id': '50544219-29bb-499e-a083-6087f4cb7797',
+            'title': 'Condé Nast Traveler Podcast',
+            'description': 'md5:98646dee22a5b386626ae31866638fbd',
         },
-        'playlist_mincount': 5,
+        'playlist_mincount': 20,
     }
 
+    @classmethod
+    def suitable(cls, url):
+        return False if ACastIE.suitable(url) else super(ACastChannelIE, cls).suitable(url)
+
     def _real_extract(self, url):
         display_id = self._match_id(url)
         channel_data = self._download_json(self._API_BASE_URL + 'channels/%s' % display_id, display_id)

From db9bd5267f31225fb149b11941bf89818eacf4f7 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 5 Dec 2015 17:25:14 +0800
Subject: [PATCH 3262/6219] [keezmovies] Fix extraction

Also fixes #7752
---
 youtube_dl/extractor/keezmovies.py | 38 +++++++++++++++++++-----------
 1 file changed, 24 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/keezmovies.py b/youtube_dl/extractor/keezmovies.py
index d79261bb5..126ca13df 100644
--- a/youtube_dl/extractor/keezmovies.py
+++ b/youtube_dl/extractor/keezmovies.py
@@ -1,23 +1,25 @@
 from __future__ import unicode_literals
 
-import os
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse_urlparse
-from ..utils import sanitized_Request
+from ..utils import (
+    sanitized_Request,
+    url_basename,
+)
 
 
 class KeezMoviesIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?keezmovies\.com/video/.+?(?P<id>[0-9]+)(?:[/?&]|$)'
     _TEST = {
         'url': 'http://www.keezmovies.com/video/petite-asian-lady-mai-playing-in-bathtub-1214711',
-        'md5': '6e297b7e789329923fcf83abb67c9289',
+        'md5': '1c1e75d22ffa53320f45eeb07bc4cdc0',
         'info_dict': {
             'id': '1214711',
             'ext': 'mp4',
             'title': 'Petite Asian Lady Mai Playing In Bathtub',
             'age_limit': 18,
+            'thumbnail': 're:^https?://.*\.jpg$',
         }
     }
 
@@ -36,21 +38,29 @@ class KeezMoviesIE(InfoExtractor):
 
         video_title = self._html_search_regex(
             r'<h1 [^>]*>([^<]+)', webpage, 'title')
-        video_url = self._html_search_regex(
-            r'(?s)html5VideoPlayer = .*?src="([^"]+)"', webpage, 'video URL')
-        path = compat_urllib_parse_urlparse(video_url).path
-        extension = os.path.splitext(path)[1][1:]
-        format = path.split('/')[4].split('_')[:2]
-        format = "-".join(format)
+        flashvars = self._parse_json(self._search_regex(
+            r'var\s+flashvars\s*=\s*([^;]+);', webpage, 'flashvars'), video_id)
+
+        formats = []
+        for height in (180, 240, 480):
+            if flashvars.get('quality_%dp' % height):
+                video_url = flashvars['quality_%dp' % height]
+                a_format = {
+                    'url': video_url,
+                    'height': height,
+                    'format_id': '%dp' % height,
+                }
+                filename_parts = url_basename(video_url).split('_')
+                if len(filename_parts) >= 2 and re.match(r'\d+[Kk]', filename_parts[1]):
+                    a_format['tbr'] = int(filename_parts[1][:-1])
+                formats.append(a_format)
 
         age_limit = self._rta_search(webpage)
 
         return {
             'id': video_id,
             'title': video_title,
-            'url': video_url,
-            'ext': extension,
-            'format': format,
-            'format_id': format,
+            'formats': formats,
             'age_limit': age_limit,
+            'thumbnail': flashvars.get('image_url')
         }

From 38d05d17e54ead57eefd7b33c5ff261f127315a8 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 5 Dec 2015 21:10:26 +0800
Subject: [PATCH 3263/6219] [fc2] Fix test_FC2_1

---
 youtube_dl/extractor/fc2.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/fc2.py b/youtube_dl/extractor/fc2.py
index 92e8c571f..4c81271d3 100644
--- a/youtube_dl/extractor/fc2.py
+++ b/youtube_dl/extractor/fc2.py
@@ -37,8 +37,8 @@ class FC2IE(InfoExtractor):
         'params': {
             'username': 'ytdl@yt-dl.org',
             'password': '(snip)',
-            'skip': 'requires actual password'
-        }
+        },
+        'skip': 'requires actual password',
     }, {
         'url': 'http://video.fc2.com/en/a/content/20130926eZpARwsF',
         'only_matching': True,

From ea14422ff12a5184794880befe50f1cac4686ea6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 5 Dec 2015 20:42:21 +0600
Subject: [PATCH 3264/6219] [hypem] Correctly handle cookies (Closes #7762)

---
 youtube_dl/extractor/hypem.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/youtube_dl/extractor/hypem.py b/youtube_dl/extractor/hypem.py
index cca3dd498..c8a74bfc7 100644
--- a/youtube_dl/extractor/hypem.py
+++ b/youtube_dl/extractor/hypem.py
@@ -33,7 +33,6 @@ class HypemIE(InfoExtractor):
         request = sanitized_Request(complete_url)
         response, urlh = self._download_webpage_handle(
             request, track_id, 'Downloading webpage with the url')
-        cookie = urlh.headers.get('Set-Cookie', '')
 
         html_tracks = self._html_search_regex(
             r'(?ms)<script type="application/json" id="displayList-data">\s*(.*?)\s*</script>',
@@ -52,7 +51,6 @@ class HypemIE(InfoExtractor):
         serve_url = "http://hypem.com/serve/source/%s/%s" % (track_id, key)
         request = sanitized_Request(
             serve_url, '', {'Content-Type': 'application/json'})
-        request.add_header('cookie', cookie)
         song_data = self._download_json(request, track_id, 'Downloading metadata')
         final_url = song_data["url"]
 

From 960e0388869a0cb383ee86fa01dce6cc8abc380a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 5 Dec 2015 20:46:57 +0600
Subject: [PATCH 3265/6219] [hypem] Modernize

---
 youtube_dl/extractor/hypem.py | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/hypem.py b/youtube_dl/extractor/hypem.py
index c8a74bfc7..b3706fe6d 100644
--- a/youtube_dl/extractor/hypem.py
+++ b/youtube_dl/extractor/hypem.py
@@ -28,14 +28,12 @@ class HypemIE(InfoExtractor):
         track_id = self._match_id(url)
 
         data = {'ax': 1, 'ts': time.time()}
-        data_encoded = compat_urllib_parse.urlencode(data)
-        complete_url = url + "?" + data_encoded
-        request = sanitized_Request(complete_url)
+        request = sanitized_Request(url + '?' + compat_urllib_parse.urlencode(data))
         response, urlh = self._download_webpage_handle(
             request, track_id, 'Downloading webpage with the url')
 
         html_tracks = self._html_search_regex(
-            r'(?ms)<script type="application/json" id="displayList-data">\s*(.*?)\s*</script>',
+            r'(?ms)<script type="application/json" id="displayList-data">(.+?)</script>',
             response, 'tracks')
         try:
             track_list = json.loads(html_tracks)
@@ -45,14 +43,14 @@ class HypemIE(InfoExtractor):
 
         key = track['key']
         track_id = track['id']
-        artist = track['artist']
         title = track['song']
 
-        serve_url = "http://hypem.com/serve/source/%s/%s" % (track_id, key)
         request = sanitized_Request(
-            serve_url, '', {'Content-Type': 'application/json'})
+            'http://hypem.com/serve/source/%s/%s' % (track_id, key),
+            '', {'Content-Type': 'application/json'})
         song_data = self._download_json(request, track_id, 'Downloading metadata')
-        final_url = song_data["url"]
+        final_url = song_data['url']
+        artist = track.get('artist')
 
         return {
             'id': track_id,

From 7db2897ded3d5970131fb34210f8510a02c672b8 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 5 Dec 2015 15:57:10 +0100
Subject: [PATCH 3266/6219] [srgssr] handle all play urls only in SRGSSRIE and
 keep RTSIE for articles

---
 youtube_dl/extractor/rts.py    | 97 +++++++++++++++-------------------
 youtube_dl/extractor/srgssr.py | 49 ++++++++++++-----
 2 files changed, 79 insertions(+), 67 deletions(-)

diff --git a/youtube_dl/extractor/rts.py b/youtube_dl/extractor/rts.py
index 12639f08b..dc10fd88b 100644
--- a/youtube_dl/extractor/rts.py
+++ b/youtube_dl/extractor/rts.py
@@ -3,7 +3,7 @@ from __future__ import unicode_literals
 
 import re
 
-from .common import InfoExtractor
+from .srgssr import SRGSSRIE
 from ..compat import (
     compat_str,
     compat_urllib_parse_urlparse,
@@ -17,27 +17,18 @@ from ..utils import (
 )
 
 
-class RTSIE(InfoExtractor):
+class RTSIE(SRGSSRIE):
     IE_DESC = 'RTS.ch'
-    _VALID_URL = r'''(?x)
-                    (?:
-                        rts:(?P<rts_id>\d+)|
-                        https?://
-                            (?:www\.)?rts\.ch/
-                            (?:
-                                (?:[^/]+/){2,}(?P<id>[0-9]+)-(?P<display_id>.+?)\.html|
-                                play/tv/[^/]+/video/(?P<display_id_new>.+?)\?id=(?P<id_new>[0-9]+)
-                            )
-                    )'''
+    _VALID_URL = r'rts:(?P<rts_id>\d+)|https?://(?:www\.)?rts\.ch/(?:[^/]+/){2,}(?P<id>[0-9]+)-(?P<display_id>.+?)\.html'
 
     _TESTS = [
         {
             'url': 'http://www.rts.ch/archives/tv/divers/3449373-les-enfants-terribles.html',
-            'md5': '753b877968ad8afaeddccc374d4256a5',
+            'md5': 'f254c4b26fb1d3c183793d52bc40d3e7',
             'info_dict': {
                 'id': '3449373',
                 'display_id': 'les-enfants-terribles',
-                'ext': 'mp4',
+                'ext': 'flv',
                 'duration': 1488,
                 'title': 'Les Enfants Terribles',
                 'description': 'France Pommier et sa soeur Luce Feral, les deux filles de ce groupe de 5.',
@@ -50,11 +41,11 @@ class RTSIE(InfoExtractor):
         },
         {
             'url': 'http://www.rts.ch/emissions/passe-moi-les-jumelles/5624067-entre-ciel-et-mer.html',
-            'md5': 'c148457a27bdc9e5b1ffe081a7a8337b',
+            'md5': 'f1077ac5af686c76528dc8d7c5df29ba',
             'info_dict': {
-                'id': '5624067',
-                'display_id': 'entre-ciel-et-mer',
-                'ext': 'mp4',
+                'id': '5742494',
+                'display_id': '5742494',
+                'ext': 'flv',
                 'duration': 3720,
                 'title': 'Les yeux dans les cieux - Mon homard au Canada',
                 'description': 'md5:d22ee46f5cc5bac0912e5a0c6d44a9f7',
@@ -85,11 +76,11 @@ class RTSIE(InfoExtractor):
         },
         {
             'url': 'http://www.rts.ch/video/info/journal-continu/5745356-londres-cachee-par-un-epais-smog.html',
-            'md5': '9bb06503773c07ce83d3cbd793cebb91',
+            'md5': '9f713382f15322181bb366cc8c3a4ff0',
             'info_dict': {
                 'id': '5745356',
                 'display_id': 'londres-cachee-par-un-epais-smog',
-                'ext': 'mp4',
+                'ext': 'flv',
                 'duration': 33,
                 'title': 'Londres cachée par un épais smog',
                 'description': 'Un important voile de smog recouvre Londres depuis mercredi, provoqué par la pollution et du sable du Sahara.',
@@ -114,23 +105,6 @@ class RTSIE(InfoExtractor):
                 'timestamp': 1396551600,
             },
         },
-        {
-            'url': 'http://www.rts.ch/play/tv/-/video/le-19h30?id=6348260',
-            'md5': '968777c8779e5aa2434be96c54e19743',
-            'info_dict': {
-                'id': '6348260',
-                'display_id': 'le-19h30',
-                'ext': 'mp4',
-                'duration': 1796,
-                'title': 'Le 19h30',
-                'description': '',
-                'uploader': 'Le 19h30',
-                'upload_date': '20141201',
-                'timestamp': 1417458600,
-                'thumbnail': 're:^https?://.*\.image',
-                'view_count': int,
-            },
-        },
         {
             # article with videos on rhs
             'url': 'http://www.rts.ch/sport/hockey/6693917-hockey-davos-decroche-son-31e-titre-de-champion-de-suisse.html',
@@ -139,42 +113,47 @@ class RTSIE(InfoExtractor):
                 'title': 'Hockey: Davos décroche son 31e titre de champion de Suisse',
             },
             'playlist_mincount': 5,
-        },
-        {
-            'url': 'http://www.rts.ch/play/tv/le-19h30/video/le-chantier-du-nouveau-parlement-vaudois-a-permis-une-trouvaille-historique?id=6348280',
-            'only_matching': True,
         }
     ]
 
     def _real_extract(self, url):
         m = re.match(self._VALID_URL, url)
-        video_id = m.group('rts_id') or m.group('id') or m.group('id_new')
-        display_id = m.group('display_id') or m.group('display_id_new')
+        media_id = m.group('rts_id') or m.group('id')
+        display_id = m.group('display_id') or media_id
 
         def download_json(internal_id):
             return self._download_json(
                 'http://www.rts.ch/a/%s.html?f=json/article' % internal_id,
                 display_id)
 
-        all_info = download_json(video_id)
+        all_info = download_json(media_id)
 
-        # video_id extracted out of URL is not always a real id
+        # media_id extracted out of URL is not always a real id
         if 'video' not in all_info and 'audio' not in all_info:
             page = self._download_webpage(url, display_id)
 
             # article with videos on rhs
             videos = re.findall(
-                r'<article[^>]+class="content-item"[^>]*>\s*<a[^>]+data-video-urn="urn:rts:video:(\d+)"',
+                r'<article[^>]+class="content-item"[^>]*>\s*<a[^>]+data-video-urn="urn:([^"]+)"',
                 page)
+            if not videos:
+                videos = re.findall(
+                    r'(?s)<iframe[^>]+class="srg-player"[^>]+src="[^"]+urn:([^"]+)"',
+                    page)
             if videos:
-                entries = [self.url_result('rts:%s' % video_urn, 'RTS') for video_urn in videos]
-                return self.playlist_result(entries, video_id, self._og_search_title(page))
+                entries = [self.url_result('srgssr:%s' % video_urn, 'SRGSSR') for video_urn in videos]
+                return self.playlist_result(entries, media_id, self._og_search_title(page))
 
             internal_id = self._html_search_regex(
                 r'<(?:video|audio) data-id="([0-9]+)"', page,
                 'internal video id')
             all_info = download_json(internal_id)
 
+        media_type = 'video' if 'video' in all_info else 'audio'
+
+        # check for errors
+        self.get_media_data('rts', media_type, media_id)
+
         info = all_info['video']['JSONinfo'] if 'video' in all_info else all_info['audio']
 
         upload_timestamp = parse_iso8601(info.get('broadcast_date'))
@@ -190,19 +169,27 @@ class RTSIE(InfoExtractor):
 
         formats = []
         for format_id, format_url in info['streams'].items():
+            if format_id == 'hds_sd' and 'hds' in info['streams']:
+                continue
+            if format_id == 'hls_sd' and 'hls' in info['streams']:
+                continue
             if format_url.endswith('.f4m'):
                 token = self._download_xml(
                     'http://tp.srgssr.ch/token/akahd.xml?stream=%s/*' % compat_urllib_parse_urlparse(format_url).path,
-                    video_id, 'Downloading %s token' % format_id)
+                    media_id, 'Downloading %s token' % format_id)
                 auth_params = xpath_text(token, './/authparams', 'auth params')
                 if not auth_params:
                     continue
-                formats.extend(self._extract_f4m_formats(
+                f4m_formats = self._extract_f4m_formats(
                     '%s?%s&hdcore=3.4.0&plugin=aasp-3.4.0.132.66' % (format_url, auth_params),
-                    video_id, f4m_id=format_id))
+                    media_id, f4m_id=format_id, fatal=False)
+                if f4m_formats:
+                    formats.extend(f4m_formats)
             elif format_url.endswith('.m3u8'):
-                formats.extend(self._extract_m3u8_formats(
-                    format_url, video_id, 'mp4', m3u8_id=format_id))
+                m3u8_formats = self._extract_m3u8_formats(
+                    format_url, media_id, 'mp4', m3u8_id=format_id, fatal=False)
+                if m3u8_formats:
+                    formats.extend(m3u8_formats)
             else:
                 formats.append({
                     'format_id': format_id,
@@ -217,11 +204,11 @@ class RTSIE(InfoExtractor):
                 'tbr': media['rate'] or extract_bitrate(media['url']),
             } for media in info['media'] if media.get('rate')])
 
-        self._check_formats(formats, video_id)
+        self._check_formats(formats, media_id)
         self._sort_formats(formats)
 
         return {
-            'id': video_id,
+            'id': media_id,
             'display_id': display_id,
             'formats': formats,
             'title': info['title'],
diff --git a/youtube_dl/extractor/srgssr.py b/youtube_dl/extractor/srgssr.py
index f759e5600..563851b3f 100644
--- a/youtube_dl/extractor/srgssr.py
+++ b/youtube_dl/extractor/srgssr.py
@@ -17,25 +17,29 @@ class SRGSSRIE(InfoExtractor):
     _ERRORS = {
         'AGERATING12': 'To protect children under the age of 12, this video is only available between 8 p.m. and 6 a.m.',
         'AGERATING18': 'To protect children under the age of 18, this video is only available between 11 p.m. and 5 a.m.',
-#        'ENDDATE': 'For legal reasons, this video was only available for a specified period of time.',
+        # 'ENDDATE': 'For legal reasons, this video was only available for a specified period of time.',
         'GEOBLOCK': 'For legal reasons, this video is only available in Switzerland.',
         'LEGAL': 'The video cannot be transmitted for legal reasons.',
         'STARTDATE': 'This video is not yet available. Please try again later.',
     }
 
-    def _real_extract(self, url):
-        bu, media_type, media_id = re.match(self._VALID_URL, url).groups()
-
+    def get_media_data(self, bu, media_type, media_id):
         media_data = self._download_json(
             'http://il.srgssr.ch/integrationlayer/1.0/ue/%s/%s/play/%s.json' % (bu, media_type, media_id),
             media_id)[media_type.capitalize()]
 
         if media_data.get('block') and media_data['block'] in self._ERRORS:
-            raise ExtractorError(
-                    '%s said: %s' % (
-                        self.IE_NAME,
-                        self._ERRORS[media_data['block']]),
-                    expected=True)
+            raise ExtractorError('%s said: %s' % (self.IE_NAME, self._ERRORS[media_data['block']]), expected=True)
+
+        return media_data
+
+    def _real_extract(self, url):
+        bu, media_type, media_id = re.match(self._VALID_URL, url).groups()
+
+        if bu == 'rts':
+            return self.url_result('rts:%s' % media_id, 'RTS')
+
+        media_data = self.get_media_data(bu, media_type, media_id)
 
         metadata = media_data['AssetMetadatas']['AssetMetadata'][0]
         title = metadata['title']
@@ -61,9 +65,13 @@ class SRGSSRIE(InfoExtractor):
                     assets[quality['@quality']] = quality['text']
                 asset_url = assets.get('HD') or assets.get('HQ') or assets.get('SD') or assets.get('MQ') or assets.get('LQ')
                 if '.f4m' in asset_url:
-                    formats.extend(self._extract_f4m_formats(asset_url + '?hdcore=3.4.0', media_id, f4m_id='hds'))
+                    f4m_formats = formats.extend(self._extract_f4m_formats(asset_url + '?hdcore=3.4.0', media_id, f4m_id='hds', fatal=False))
+                    if f4m_formats:
+                        formats.extend(f4m_formats)
                 elif '.m3u8' in asset_url:
-                    formats.extend(self._extract_m3u8_formats(asset_url, media_id, m3u8_id='hls'))
+                    m3u8_formats = formats.extend(self._extract_m3u8_formats(asset_url, media_id, m3u8_id='hls', fatal=False))
+                    if m3u8_formats:
+                        formats.extend(m3u8_formats)
             else:
                 for asset in source['url']:
                     asset_url = asset['text']
@@ -122,7 +130,7 @@ class SRGSSRPlayIE(InfoExtractor):
             'description': 'md5:88604432b60d5a38787f152dec89cd56',
             'timestamp': 1373493600,
         },
-    },{
+    }, {
         'url': 'http://www.rtr.ch/play/radio/actualitad/audio/saira-tujetsch-tuttina-cuntinuar-cun-sedrun-muster-turissem?id=63cb0778-27f8-49af-9284-8c7a8c6d15fc',
         'info_dict': {
             'id': '63cb0778-27f8-49af-9284-8c7a8c6d15fc',
@@ -135,8 +143,25 @@ class SRGSSRPlayIE(InfoExtractor):
             # rtmp download
             'skip_download': True,
         },
+    }, {
+        'url': 'http://www.rts.ch/play/tv/-/video/le-19h30?id=6348260',
+        'md5': '67a2a9ae4e8e62a68d0e9820cc9782df',
+        'info_dict': {
+            'id': '6348260',
+            'display_id': '6348260',
+            'ext': 'flv',
+            'duration': 1796,
+            'title': 'Le 19h30',
+            'description': '',
+            'uploader': '19h30',
+            'upload_date': '20141201',
+            'timestamp': 1417458600,
+            'thumbnail': 're:^https?://.*\.image',
+            'view_count': int,
+        },
     }]
 
     def _real_extract(self, url):
         bu, media_type, media_id = re.match(self._VALID_URL, url).groups()
+        # other info can be extracted from url + '&layout=json'
         return self.url_result('srgssr:%s:%s:%s' % (bu[:3], media_type, media_id), 'SRGSSR')

From 535d7b681b05a37a3c2549c7501eb7fe9c8a5f4e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 5 Dec 2015 16:01:37 +0100
Subject: [PATCH 3267/6219] release 2015.12.05

---
 README.md              | 2 +-
 docs/supportedsites.md | 8 ++++++++
 youtube_dl/version.py  | 2 +-
 3 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index df419abe8..0b224f0be 100644
--- a/README.md
+++ b/README.md
@@ -319,7 +319,7 @@ which means you can modify it, redistribute it or use it however you like.
     --all-formats                    Download all available video formats
     --prefer-free-formats            Prefer free video formats unless a specific
                                      one is requested
-    -F, --list-formats               List all available formats of specified
+    -F, --list-formats               List all available formats of requested
                                      videos
     --youtube-skip-dash-manifest     Do not download the DASH manifests and
                                      related data on YouTube videos
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 1df408610..880da0e47 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -15,8 +15,12 @@
  - **abc.net.au**
  - **Abc7News**
  - **AcademicEarth:Course**
+ - **acast**
+ - **acast:channel**
  - **AddAnime**
  - **AdobeTV**
+ - **AdobeTVChannel**
+ - **AdobeTVShow**
  - **AdobeTVVideo**
  - **AdultSwim**
  - **Aftenposten**
@@ -43,6 +47,7 @@
  - **arte.tv:future**
  - **AtresPlayer**
  - **ATTTechChannel**
+ - **AudiMedia**
  - **audiomack**
  - **audiomack:album**
  - **Azubu**
@@ -183,6 +188,7 @@
  - **freespeech.org**
  - **FreeVideo**
  - **FunnyOrDie**
+ - **GameInformer**
  - **Gamekings**
  - **GameOne**
  - **gameone:playlist**
@@ -480,6 +486,8 @@
  - **Shared**: shared.sx and vivo.sx
  - **ShareSix**
  - **Sina**
+ - **skynewsarabia:video**
+ - **skynewsarabia:video**
  - **Slideshare**
  - **Slutload**
  - **smotri**: Smotri.com
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index bd0de9f53..12ca0241d 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.11.27.1'
+__version__ = '2015.12.05'

From 796db212952605606502fd40aa83007f08524cb2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 5 Dec 2015 16:12:02 +0100
Subject: [PATCH 3268/6219] [metacafe] Fix video url extraction (closes #7763)

---
 youtube_dl/extractor/metacafe.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/metacafe.py b/youtube_dl/extractor/metacafe.py
index 3c786a36d..5b0b42608 100644
--- a/youtube_dl/extractor/metacafe.py
+++ b/youtube_dl/extractor/metacafe.py
@@ -154,10 +154,10 @@ class MetacafeIE(InfoExtractor):
         # Extract URL, uploader and title from webpage
         self.report_extraction(video_id)
         video_url = None
-        mobj = re.search(r'(?m)&mediaURL=([^&]+)', webpage)
+        mobj = re.search(r'(?m)&(?:media|video)URL=([^&]+)', webpage)
         if mobj is not None:
             mediaURL = compat_urllib_parse_unquote(mobj.group(1))
-            video_ext = mediaURL[-3:]
+            video_ext = determine_ext(mediaURL)
 
             # Extract gdaKey if available
             mobj = re.search(r'(?m)&gdaKey=(.*?)&', webpage)

From 77334ccb44c06ea2d9308b334aecc612c2f3240e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 5 Dec 2015 16:12:50 +0100
Subject: [PATCH 3269/6219] [metacafe] Fix age limit extraction

---
 youtube_dl/extractor/metacafe.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/metacafe.py b/youtube_dl/extractor/metacafe.py
index 5b0b42608..67d6271e1 100644
--- a/youtube_dl/extractor/metacafe.py
+++ b/youtube_dl/extractor/metacafe.py
@@ -229,7 +229,7 @@ class MetacafeIE(InfoExtractor):
 
         age_limit = (
             18
-            if re.search(r'"contentRating":"restricted"', webpage)
+            if re.search(r'(?:"contentRating":|"rating",)"restricted"', webpage)
             else 0)
 
         if isinstance(video_url, list):

From cb160dd531f328b2c741e4712fae560e1cec3f90 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 5 Dec 2015 18:47:15 +0100
Subject: [PATCH 3270/6219] [nba] handle format info properly

---
 youtube_dl/extractor/nba.py | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/nba.py b/youtube_dl/extractor/nba.py
index d8f901b55..5a3eb0833 100644
--- a/youtube_dl/extractor/nba.py
+++ b/youtube_dl/extractor/nba.py
@@ -77,14 +77,18 @@ class NBAIE(InfoExtractor):
                     formats.extend(f4m_formats)
             else:
                 key = video_file.attrib.get('bitrate')
-                mobj = re.search(r'(\d+)x(\d+)(?:_(\d+))?', key)
-                formats.append({
+                format_info = {
                     'format_id': key,
                     'url': video_url,
-                    'width': int_or_none(mobj.group(1)),
-                    'height': int_or_none(mobj.group(2)),
-                    'tbr': int_or_none(mobj.group(3)),
-                })
+                }
+                mobj = re.search(r'(\d+)x(\d+)(?:_(\d+))?', key)
+                if mobj:
+                    format_info.update({
+                        'width': int_or_none(mobj.group(1)),
+                        'height': int_or_none(mobj.group(2)),
+                        'tbr': int_or_none(mobj.group(3)),
+                    })
+                formats.append(format_info)
         self._sort_formats(formats)
 
         return {

From 756926ff00b1b0bf5face8116657c99d8ae22172 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Dec 2015 01:39:28 +0600
Subject: [PATCH 3271/6219] [pluralsight] Add support for widescreen videos
 (Closes #7766)

---
 youtube_dl/extractor/pluralsight.py | 26 +++++++++++++++++++++-----
 1 file changed, 21 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index aa7dbcb63..336374bf7 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -1,5 +1,6 @@
 from __future__ import unicode_literals
 
+import re
 import json
 import random
 import collections
@@ -14,6 +15,7 @@ from ..utils import (
     ExtractorError,
     int_or_none,
     parse_duration,
+    qualities,
     sanitized_Request,
 )
 
@@ -140,15 +142,28 @@ class PluralsightIE(PluralsightBaseIE):
             'low': {'width': 640, 'height': 480},
             'medium': {'width': 848, 'height': 640},
             'high': {'width': 1024, 'height': 768},
+            'high-widescreen': {'width': 1280, 'height': 720},
         }
 
+        QUALITIES_PREFERENCE = ('low', 'medium', 'high', 'high-widescreen',)
+        quality_key = qualities(QUALITIES_PREFERENCE)
+
         AllowedQuality = collections.namedtuple('AllowedQuality', ['ext', 'qualities'])
 
         ALLOWED_QUALITIES = (
-            AllowedQuality('webm', ('high',)),
-            AllowedQuality('mp4', ('low', 'medium', 'high',)),
+            AllowedQuality('webm', ['high', ]),
+            AllowedQuality('mp4', ['low', 'medium', 'high', ]),
         )
 
+        # Some courses also offer widescreen resolution for high quality (see
+        # https://github.com/rg3/youtube-dl/issues/7766)
+        widescreen = True if re.search(
+            r'courseSupportsWidescreenVideoFormats\s*:\s*true', webpage) else False
+        best_quality = 'high-widescreen' if widescreen else 'high'
+        if widescreen:
+            for allowed_quality in ALLOWED_QUALITIES:
+                allowed_quality.qualities.append(best_quality)
+
         # In order to minimize the number of calls to ViewClip API and reduce
         # the probability of being throttled or banned by Pluralsight we will request
         # only single format until formats listing was explicitly requested.
@@ -164,12 +179,12 @@ class PluralsightIE(PluralsightBaseIE):
                         if req_ext == allowed_quality.ext and req_quality in allowed_quality.qualities:
                             return (AllowedQuality(req_ext, (req_quality, )), )
                 req_ext = 'webm' if self._downloader.params.get('prefer_free_formats') else 'mp4'
-                return (AllowedQuality(req_ext, ('high', )), )
+                return (AllowedQuality(req_ext, (best_quality, )), )
             allowed_qualities = guess_allowed_qualities()
 
         formats = []
-        for ext, qualities in allowed_qualities:
-            for quality in qualities:
+        for ext, qualities_ in allowed_qualities:
+            for quality in qualities_:
                 f = QUALITIES[quality].copy()
                 clip_post = {
                     'a': author,
@@ -205,6 +220,7 @@ class PluralsightIE(PluralsightBaseIE):
                     'url': clip_url,
                     'ext': ext,
                     'format_id': format_id,
+                    'quality': quality_key(quality),
                 })
                 formats.append(f)
         self._sort_formats(formats)

From edc70f4aaf08009c95cb5e91878461cc7b508f68 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Dec 2015 01:40:13 +0600
Subject: [PATCH 3272/6219] [pluralsight] Fix format code split while guessing
 quality

---
 youtube_dl/extractor/pluralsight.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index 336374bf7..55c11b3bf 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -172,7 +172,7 @@ class PluralsightIE(PluralsightBaseIE):
         else:
             def guess_allowed_qualities():
                 req_format = self._downloader.params.get('format') or 'best'
-                req_format_split = req_format.split('-')
+                req_format_split = req_format.split('-', 1)
                 if len(req_format_split) > 1:
                     req_ext, req_quality = req_format_split
                     for allowed_quality in ALLOWED_QUALITIES:

From 0017486dcab67583db7896a9d754b195bcee3bad Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 5 Dec 2015 20:58:44 +0100
Subject: [PATCH 3273/6219] [nba] use int instead of int_or_none

---
 youtube_dl/extractor/nba.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/nba.py b/youtube_dl/extractor/nba.py
index 5a3eb0833..9f99dbf36 100644
--- a/youtube_dl/extractor/nba.py
+++ b/youtube_dl/extractor/nba.py
@@ -84,9 +84,9 @@ class NBAIE(InfoExtractor):
                 mobj = re.search(r'(\d+)x(\d+)(?:_(\d+))?', key)
                 if mobj:
                     format_info.update({
-                        'width': int_or_none(mobj.group(1)),
-                        'height': int_or_none(mobj.group(2)),
-                        'tbr': int_or_none(mobj.group(3)),
+                        'width': int(mobj.group(1)),
+                        'height': int(mobj.group(2)),
+                        'tbr': int(mobj.group(3)),
                     })
                 formats.append(format_info)
         self._sort_formats(formats)

From 51c4fec0d58af709a06bd75338b23eace412362f Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 5 Dec 2015 21:04:22 +0100
Subject: [PATCH 3274/6219] [nba] use int_or_none for tbr

---
 youtube_dl/extractor/nba.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nba.py b/youtube_dl/extractor/nba.py
index 9f99dbf36..7c6b7841d 100644
--- a/youtube_dl/extractor/nba.py
+++ b/youtube_dl/extractor/nba.py
@@ -86,7 +86,7 @@ class NBAIE(InfoExtractor):
                     format_info.update({
                         'width': int(mobj.group(1)),
                         'height': int(mobj.group(2)),
-                        'tbr': int(mobj.group(3)),
+                        'tbr': int_or_none(mobj.group(3)),
                     })
                 formats.append(format_info)
         self._sort_formats(formats)

From 33d152b6cc04bfdf658691dbf12c1d976cfa0ba3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Dec 2015 09:29:41 +0600
Subject: [PATCH 3275/6219] [novamov] Move all novamov based extractors to a
 single place

For easier navigation
---
 youtube_dl/extractor/__init__.py  | 10 +++--
 youtube_dl/extractor/movshare.py  | 27 ------------
 youtube_dl/extractor/novamov.py   | 72 +++++++++++++++++++++++++++++++
 youtube_dl/extractor/nowvideo.py  | 28 ------------
 youtube_dl/extractor/videoweed.py | 26 -----------
 5 files changed, 78 insertions(+), 85 deletions(-)
 delete mode 100644 youtube_dl/extractor/movshare.py
 delete mode 100644 youtube_dl/extractor/nowvideo.py
 delete mode 100644 youtube_dl/extractor/videoweed.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index c1dd87550..fefaff35b 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -357,7 +357,6 @@ from .motherless import MotherlessIE
 from .motorsport import MotorsportIE
 from .movieclips import MovieClipsIE
 from .moviezine import MoviezineIE
-from .movshare import MovShareIE
 from .mtv import (
     MTVIE,
     MTVServicesEmbeddedIE,
@@ -423,7 +422,12 @@ from .noco import NocoIE
 from .normalboots import NormalbootsIE
 from .nosvideo import NosVideoIE
 from .nova import NovaIE
-from .novamov import NovaMovIE
+from .novamov import (
+    NovaMovIE,
+    MovShareIE,
+    NowVideoIE,
+    VideoWeedIE,
+)
 from .nowness import (
     NownessIE,
     NownessPlaylistIE,
@@ -433,7 +437,6 @@ from .nowtv import (
     NowTVIE,
     NowTVListIE,
 )
-from .nowvideo import NowVideoIE
 from .npo import (
     NPOIE,
     NPOLiveIE,
@@ -744,7 +747,6 @@ from .videofyme import VideofyMeIE
 from .videomega import VideoMegaIE
 from .videopremium import VideoPremiumIE
 from .videott import VideoTtIE
-from .videoweed import VideoWeedIE
 from .vidme import VidmeIE
 from .vidzi import VidziIE
 from .vier import VierIE, VierVideosIE
diff --git a/youtube_dl/extractor/movshare.py b/youtube_dl/extractor/movshare.py
deleted file mode 100644
index 6101063f2..000000000
--- a/youtube_dl/extractor/movshare.py
+++ /dev/null
@@ -1,27 +0,0 @@
-from __future__ import unicode_literals
-
-from .novamov import NovaMovIE
-
-
-class MovShareIE(NovaMovIE):
-    IE_NAME = 'movshare'
-    IE_DESC = 'MovShare'
-
-    _VALID_URL = NovaMovIE._VALID_URL_TEMPLATE % {'host': 'movshare\.(?:net|sx|ag)'}
-
-    _HOST = 'www.movshare.net'
-
-    _FILE_DELETED_REGEX = r'>This file no longer exists on our servers.<'
-    _TITLE_REGEX = r'<strong>Title:</strong> ([^<]+)</p>'
-    _DESCRIPTION_REGEX = r'<strong>Description:</strong> ([^<]+)</p>'
-
-    _TEST = {
-        'url': 'http://www.movshare.net/video/559e28be54d96',
-        'md5': 'abd31a2132947262c50429e1d16c1bfd',
-        'info_dict': {
-            'id': '559e28be54d96',
-            'ext': 'flv',
-            'title': 'dissapeared image',
-            'description': 'optical illusion  dissapeared image  magic illusion',
-        }
-    }
diff --git a/youtube_dl/extractor/novamov.py b/youtube_dl/extractor/novamov.py
index 6163e8855..0605c0e0d 100644
--- a/youtube_dl/extractor/novamov.py
+++ b/youtube_dl/extractor/novamov.py
@@ -92,3 +92,75 @@ class NovaMovIE(InfoExtractor):
             'title': title,
             'description': description
         }
+
+
+class MovShareIE(NovaMovIE):
+    IE_NAME = 'movshare'
+    IE_DESC = 'MovShare'
+
+    _VALID_URL = NovaMovIE._VALID_URL_TEMPLATE % {'host': 'movshare\.(?:net|sx|ag)'}
+
+    _HOST = 'www.movshare.net'
+
+    _FILE_DELETED_REGEX = r'>This file no longer exists on our servers.<'
+    _TITLE_REGEX = r'<strong>Title:</strong> ([^<]+)</p>'
+    _DESCRIPTION_REGEX = r'<strong>Description:</strong> ([^<]+)</p>'
+
+    _TEST = {
+        'url': 'http://www.movshare.net/video/559e28be54d96',
+        'md5': 'abd31a2132947262c50429e1d16c1bfd',
+        'info_dict': {
+            'id': '559e28be54d96',
+            'ext': 'flv',
+            'title': 'dissapeared image',
+            'description': 'optical illusion  dissapeared image  magic illusion',
+        }
+    }
+
+
+class NowVideoIE(NovaMovIE):
+    IE_NAME = 'nowvideo'
+    IE_DESC = 'NowVideo'
+
+    _VALID_URL = NovaMovIE._VALID_URL_TEMPLATE % {'host': 'nowvideo\.(?:to|ch|ec|sx|eu|at|ag|co|li)'}
+
+    _HOST = 'www.nowvideo.to'
+
+    _FILE_DELETED_REGEX = r'>This file no longer exists on our servers.<'
+    _FILEKEY_REGEX = r'var fkzd="([^"]+)";'
+    _TITLE_REGEX = r'<h4>([^<]+)</h4>'
+    _DESCRIPTION_REGEX = r'</h4>\s*<p>([^<]+)</p>'
+
+    _TEST = {
+        'url': 'http://www.nowvideo.ch/video/0mw0yow7b6dxa',
+        'md5': 'f8fbbc8add72bd95b7850c6a02fc8817',
+        'info_dict': {
+            'id': '0mw0yow7b6dxa',
+            'ext': 'flv',
+            'title': 'youtubedl test video _BaW_jenozKc.mp4',
+            'description': 'Description',
+        }
+    }
+
+
+class VideoWeedIE(NovaMovIE):
+    IE_NAME = 'videoweed'
+    IE_DESC = 'VideoWeed'
+
+    _VALID_URL = NovaMovIE._VALID_URL_TEMPLATE % {'host': 'videoweed\.(?:es|com)'}
+
+    _HOST = 'www.videoweed.es'
+
+    _FILE_DELETED_REGEX = r'>This file no longer exists on our servers.<'
+    _TITLE_REGEX = r'<h1 class="text_shadow">([^<]+)</h1>'
+
+    _TEST = {
+        'url': 'http://www.videoweed.es/file/b42178afbea14',
+        'md5': 'abd31a2132947262c50429e1d16c1bfd',
+        'info_dict': {
+            'id': 'b42178afbea14',
+            'ext': 'flv',
+            'title': 'optical illusion  dissapeared image magic illusion',
+            'description': ''
+        },
+    }
diff --git a/youtube_dl/extractor/nowvideo.py b/youtube_dl/extractor/nowvideo.py
deleted file mode 100644
index 57ee3d366..000000000
--- a/youtube_dl/extractor/nowvideo.py
+++ /dev/null
@@ -1,28 +0,0 @@
-from __future__ import unicode_literals
-
-from .novamov import NovaMovIE
-
-
-class NowVideoIE(NovaMovIE):
-    IE_NAME = 'nowvideo'
-    IE_DESC = 'NowVideo'
-
-    _VALID_URL = NovaMovIE._VALID_URL_TEMPLATE % {'host': 'nowvideo\.(?:to|ch|ec|sx|eu|at|ag|co|li)'}
-
-    _HOST = 'www.nowvideo.to'
-
-    _FILE_DELETED_REGEX = r'>This file no longer exists on our servers.<'
-    _FILEKEY_REGEX = r'var fkzd="([^"]+)";'
-    _TITLE_REGEX = r'<h4>([^<]+)</h4>'
-    _DESCRIPTION_REGEX = r'</h4>\s*<p>([^<]+)</p>'
-
-    _TEST = {
-        'url': 'http://www.nowvideo.ch/video/0mw0yow7b6dxa',
-        'md5': 'f8fbbc8add72bd95b7850c6a02fc8817',
-        'info_dict': {
-            'id': '0mw0yow7b6dxa',
-            'ext': 'flv',
-            'title': 'youtubedl test video _BaW_jenozKc.mp4',
-            'description': 'Description',
-        }
-    }
diff --git a/youtube_dl/extractor/videoweed.py b/youtube_dl/extractor/videoweed.py
deleted file mode 100644
index ca2e50935..000000000
--- a/youtube_dl/extractor/videoweed.py
+++ /dev/null
@@ -1,26 +0,0 @@
-from __future__ import unicode_literals
-
-from .novamov import NovaMovIE
-
-
-class VideoWeedIE(NovaMovIE):
-    IE_NAME = 'videoweed'
-    IE_DESC = 'VideoWeed'
-
-    _VALID_URL = NovaMovIE._VALID_URL_TEMPLATE % {'host': 'videoweed\.(?:es|com)'}
-
-    _HOST = 'www.videoweed.es'
-
-    _FILE_DELETED_REGEX = r'>This file no longer exists on our servers.<'
-    _TITLE_REGEX = r'<h1 class="text_shadow">([^<]+)</h1>'
-
-    _TEST = {
-        'url': 'http://www.videoweed.es/file/b42178afbea14',
-        'md5': 'abd31a2132947262c50429e1d16c1bfd',
-        'info_dict': {
-            'id': 'b42178afbea14',
-            'ext': 'flv',
-            'title': 'optical illusion  dissapeared image magic illusion',
-            'description': ''
-        },
-    }

From 636aa83ed3fa2b2ef01b6919fca4e4d603b9773a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Dec 2015 09:37:38 +0600
Subject: [PATCH 3276/6219] [cloudtime] Add extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/novamov.py  | 14 ++++++++++++++
 2 files changed, 15 insertions(+)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index fefaff35b..ed85d0ee5 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -427,6 +427,7 @@ from .novamov import (
     MovShareIE,
     NowVideoIE,
     VideoWeedIE,
+    CloudTimeIE,
 )
 from .nowness import (
     NownessIE,
diff --git a/youtube_dl/extractor/novamov.py b/youtube_dl/extractor/novamov.py
index 0605c0e0d..53d656f9d 100644
--- a/youtube_dl/extractor/novamov.py
+++ b/youtube_dl/extractor/novamov.py
@@ -164,3 +164,17 @@ class VideoWeedIE(NovaMovIE):
             'description': ''
         },
     }
+
+
+class CloudTimeIE(NovaMovIE):
+    IE_NAME = 'cloudtime'
+    IE_DESC = 'CloudTime'
+
+    _VALID_URL = NovaMovIE._VALID_URL_TEMPLATE % {'host': 'cloudtime\.to'}
+
+    _HOST = 'www.cloudtime.to'
+
+    _FILE_DELETED_REGEX = r'>This file no longer exists on our servers.<'
+    _TITLE_REGEX = r'<div[^>]+class=["\']video_det["\'][^>]*>\s*<strong>([^<]+)</strong>'
+
+    _TEST = None

From 36066dd3ee3f90c3feb136ec4c85a03ece5b8585 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Dec 2015 09:42:00 +0600
Subject: [PATCH 3277/6219] [movshare] Rename to wholecloud

---
 youtube_dl/extractor/__init__.py |  2 +-
 youtube_dl/extractor/novamov.py  | 12 ++++++------
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index ed85d0ee5..3db5cd6d9 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -424,7 +424,7 @@ from .nosvideo import NosVideoIE
 from .nova import NovaIE
 from .novamov import (
     NovaMovIE,
-    MovShareIE,
+    WholeCloudIE,
     NowVideoIE,
     VideoWeedIE,
     CloudTimeIE,
diff --git a/youtube_dl/extractor/novamov.py b/youtube_dl/extractor/novamov.py
index 53d656f9d..bbbb8fe58 100644
--- a/youtube_dl/extractor/novamov.py
+++ b/youtube_dl/extractor/novamov.py
@@ -94,20 +94,20 @@ class NovaMovIE(InfoExtractor):
         }
 
 
-class MovShareIE(NovaMovIE):
-    IE_NAME = 'movshare'
-    IE_DESC = 'MovShare'
+class WholeCloudIE(NovaMovIE):
+    IE_NAME = 'wholecloud'
+    IE_DESC = 'WholeCloud'
 
-    _VALID_URL = NovaMovIE._VALID_URL_TEMPLATE % {'host': 'movshare\.(?:net|sx|ag)'}
+    _VALID_URL = NovaMovIE._VALID_URL_TEMPLATE % {'host': '(?:wholecloud\.net|movshare\.(?:net|sx|ag))'}
 
-    _HOST = 'www.movshare.net'
+    _HOST = 'www.wholecloud.net'
 
     _FILE_DELETED_REGEX = r'>This file no longer exists on our servers.<'
     _TITLE_REGEX = r'<strong>Title:</strong> ([^<]+)</p>'
     _DESCRIPTION_REGEX = r'<strong>Description:</strong> ([^<]+)</p>'
 
     _TEST = {
-        'url': 'http://www.movshare.net/video/559e28be54d96',
+        'url': 'http://www.wholecloud.net/video/559e28be54d96',
         'md5': 'abd31a2132947262c50429e1d16c1bfd',
         'info_dict': {
             'id': '559e28be54d96',

From 7ac40e552181ad43b3e2ad9919a643414d8a8434 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Dec 2015 09:42:20 +0600
Subject: [PATCH 3278/6219] [nowvideo] Update test

---
 youtube_dl/extractor/novamov.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/novamov.py b/youtube_dl/extractor/novamov.py
index bbbb8fe58..837c91559 100644
--- a/youtube_dl/extractor/novamov.py
+++ b/youtube_dl/extractor/novamov.py
@@ -132,7 +132,7 @@ class NowVideoIE(NovaMovIE):
     _DESCRIPTION_REGEX = r'</h4>\s*<p>([^<]+)</p>'
 
     _TEST = {
-        'url': 'http://www.nowvideo.ch/video/0mw0yow7b6dxa',
+        'url': 'http://www.nowvideo.to/video/0mw0yow7b6dxa',
         'md5': 'f8fbbc8add72bd95b7850c6a02fc8817',
         'info_dict': {
             'id': '0mw0yow7b6dxa',

From 9f6b517671a7a9b16ccf2bc3f5153673a73895ff Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 6 Dec 2015 07:43:46 +0100
Subject: [PATCH 3279/6219] [vgtv] extract all formats and improve extraction

---
 youtube_dl/extractor/vgtv.py | 51 +++++++++++++++++++-----------------
 1 file changed, 27 insertions(+), 24 deletions(-)

diff --git a/youtube_dl/extractor/vgtv.py b/youtube_dl/extractor/vgtv.py
index e8039ec7f..8ce3d8b8f 100644
--- a/youtube_dl/extractor/vgtv.py
+++ b/youtube_dl/extractor/vgtv.py
@@ -13,7 +13,7 @@ from ..utils import (
 class VGTVIE(InfoExtractor):
     IE_DESC = 'VGTV, BTTV, FTV, Aftenposten and Aftonbladet'
     _VALID_URL = r'''(?x)
-                    http://(?:www\.)?
+                    https?://(?:www\.)?
                     (?P<host>
                         vgtv.no|
                         (?:bt|aftenbladet).no/tv|
@@ -25,7 +25,7 @@ class VGTVIE(InfoExtractor):
                         \#!/(?:video|live)/|
                         embed?.*id=
                     )
-                    (?P<id>[0-9]+)
+                    (?P<id>\d+)
                     '''
     _TESTS = [
         {
@@ -82,7 +82,8 @@ class VGTVIE(InfoExtractor):
                 # m3u8 download
                 'skip_download': True,
             },
-        },{
+        },
+        {
             'url': 'http://www.aftenposten.no/webtv/#!/video/21039/trailer-sweatshop-i-can-t-take-any-more',
             'md5': '7fbc265a3ca4933a423c7a66aa879a67',
             'info_dict': {
@@ -145,35 +146,37 @@ class VGTVIE(InfoExtractor):
 
         hls_url = streams.get('hls')
         if hls_url:
-            formats.extend(self._extract_m3u8_formats(
-                hls_url, video_id, 'mp4', m3u8_id='hls'))
+            m3u8_formats = self._extract_m3u8_formats(hls_url, video_id, 'mp4', m3u8_id='hls', fatal=False)
+            if m3u8_formats:
+                formats.extend(m3u8_formats)
 
         hds_url = streams.get('hds')
         # wasLive hds are always 404
         if hds_url and stream_type != 'wasLive':
-            formats.extend(self._extract_f4m_formats(
-                hds_url + '?hdcore=3.2.0&plugin=aasp-3.2.0.77.18',
-                video_id, f4m_id='hds'))
+            f4m_formats = self._extract_f4m_formats(hds_url + '?hdcore=3.2.0&plugin=aasp-3.2.0.77.18', video_id, f4m_id='hds', fatal=False)
+            if f4m_formats:
+                formats.extend(f4m_formats)
 
+        mp4_urls = streams.get('pseudostreaming') or []
         mp4_url = streams.get('mp4')
         if mp4_url:
-            _url = hls_url or hds_url
-            MP4_URL_TEMPLATE = '%s/%%s.%s' % (mp4_url.rpartition('/')[0], mp4_url.rpartition('.')[-1])
-            for mp4_format in _url.split(','):
-                m = re.search('(?P<width>\d+)_(?P<height>\d+)_(?P<vbr>\d+)', mp4_format)
-                if not m:
-                    continue
-                width = int(m.group('width'))
-                height = int(m.group('height'))
-                vbr = int(m.group('vbr'))
-                formats.append({
-                    'url': MP4_URL_TEMPLATE % mp4_format,
-                    'format_id': 'mp4-%s' % vbr,
-                    'width': width,
-                    'height': height,
+            mp4_urls.append(mp4_url)
+        for mp4_url in mp4_urls:
+            format_info = {
+                'url': mp4_url,
+                'preference': 1,
+            }
+            mobj = re.search('(\d+)_(\d+)_(\d+)', mp4_url)
+            if mobj:
+                vbr = int(mobj.group(3))
+                format_info.update({
+                    'width': int(mobj.group(1)),
+                    'height': int(mobj.group(2)),
                     'vbr': vbr,
-                    'preference': 1,
+                    'format_id': 'mp4-%s' % vbr,
                 })
+            formats.append(format_info)
+
         self._sort_formats(formats)
 
         return {
@@ -234,4 +237,4 @@ class BTVestlendingenIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        return self.url_result('xstream:btno:%s' % self._match_id(url), 'Xstream')
+        return self.url_result('http://bt.no/tv/embed?id=%s' % self._match_id(url), 'VGTV')

From 8364b6b0b17f4ea20db9bbd160740801951170ca Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 6 Dec 2015 16:40:35 +0800
Subject: [PATCH 3280/6219] [iqiyi] Update key

Closes #7772
---
 youtube_dl/extractor/iqiyi.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index 2df1da3f0..f96e12e69 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -205,9 +205,8 @@ class IqiyiIE(InfoExtractor):
 
     def get_enc_key(self, swf_url, video_id):
         # TODO: automatic key extraction
-        # last update at 2015-10-22 for Zombie::bite
-        # '7223c67061dbea1259d0ceb44f44b6d62288f4f80c972170de5201d2321060270e05'[2:66][0::2]
-        enc_key = '2c76de15dcb44bd28ff0927d50d31620'
+        # last update at 2015-12-06 for Zombie::bite
+        enc_key = '3719f6a1da83ee0aee3488d8802d7696'[::-1]
         return enc_key
 
     def _real_extract(self, url):

From 7d682f0acb44d72a2c1d0d52fae151f3c273874d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Dec 2015 16:16:59 +0600
Subject: [PATCH 3281/6219] [nowtv] Extend _VALID_URL to support jahr URLs
 (Closes #7755)

---
 youtube_dl/extractor/nowtv.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nowtv.py b/youtube_dl/extractor/nowtv.py
index 67e34b294..fd107aca2 100644
--- a/youtube_dl/extractor/nowtv.py
+++ b/youtube_dl/extractor/nowtv.py
@@ -71,7 +71,7 @@ class NowTVBaseIE(InfoExtractor):
 
 
 class NowTVIE(NowTVBaseIE):
-    _VALID_URL = r'https?://(?:www\.)?nowtv\.(?:de|at|ch)/(?:rtl|rtl2|rtlnitro|superrtl|ntv|vox)/(?P<show_id>[^/]+)/(?:list/[^/]+/)?(?P<id>[^/]+)/(?:player|preview)'
+    _VALID_URL = r'https?://(?:www\.)?nowtv\.(?:de|at|ch)/(?:rtl|rtl2|rtlnitro|superrtl|ntv|vox)/(?P<show_id>[^/]+)/(?:(?:list/[^/]+|jahr/\d{4}/\d{1,2})/)?(?P<id>[^/]+)/(?:player|preview)'
 
     _TESTS = [{
         # rtl
@@ -190,6 +190,9 @@ class NowTVIE(NowTVBaseIE):
     }, {
         'url': 'http://www.nowtv.de/rtl2/echtzeit/list/aktuell/schnelles-geld-am-ende-der-welt/player',
         'only_matching': True,
+    }, {
+        'url': 'http://www.nowtv.de/rtl2/zuhause-im-glueck/jahr/2015/11/eine-erschuetternde-diagnose/player',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 222e11d4ae7d424cd71d9ba87762ce169c0a8c87 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Dec 2015 16:41:12 +0600
Subject: [PATCH 3282/6219] [bbc] Add another pattern for playlist.sxml (Closes
 #7743)

---
 youtube_dl/extractor/bbc.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index d89e34ba0..691aecc0d 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -733,6 +733,7 @@ class BBCIE(BBCCoUkIE):
         # article with multiple videos embedded with playlist.sxml (e.g.
         # http://www.bbc.com/sport/0/football/34475836)
         playlists = re.findall(r'<param[^>]+name="playlist"[^>]+value="([^"]+)"', webpage)
+        playlists.extend(re.findall(r'data-media-id="([^"]+/playlist\.sxml)"', webpage))
         if playlists:
             entries = [
                 self._extract_from_playlist_sxml(playlist_url, playlist_id, timestamp)

From fd8e559c3a9861121d5d89f9095c8d45aa1d9dcd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Dec 2015 23:47:10 +0600
Subject: [PATCH 3283/6219] [beeg] Switch to api v4 (Closes #7774)

---
 youtube_dl/extractor/beeg.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/beeg.py b/youtube_dl/extractor/beeg.py
index e63c2ac00..d151d38c9 100644
--- a/youtube_dl/extractor/beeg.py
+++ b/youtube_dl/extractor/beeg.py
@@ -34,7 +34,7 @@ class BeegIE(InfoExtractor):
         video_id = self._match_id(url)
 
         video = self._download_json(
-            'http://beeg.com/api/v3/video/%s' % video_id, video_id)
+            'http://beeg.com/api/v4/video/%s' % video_id, video_id)
 
         def decrypt_key(key):
             # Reverse engineered from http://static.beeg.com/cpl/1067.js

From bc92621adea280c93ed7ed86cc8f2bec2c27bb36 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 6 Dec 2015 18:51:25 +0100
Subject: [PATCH 3284/6219] release 2015.12.06

---
 docs/supportedsites.md | 3 ++-
 youtube_dl/version.py  | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 880da0e47..bf26fecd7 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -97,6 +97,7 @@
  - **Clipfish**
  - **cliphunter**
  - **Clipsyndicate**
+ - **cloudtime**: CloudTime
  - **Cloudy**
  - **Clubic**
  - **Clyp**
@@ -313,7 +314,6 @@
  - **MovieClips**
  - **MovieFap**
  - **Moviezine**
- - **movshare**: MovShare
  - **MPORA**
  - **MSNBC**
  - **MTV**
@@ -673,6 +673,7 @@
  - **WebOfStories**
  - **WebOfStoriesPlaylist**
  - **Weibo**
+ - **wholecloud**: WholeCloud
  - **Wimp**
  - **Wistia**
  - **WNL**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 12ca0241d..a4e9d7072 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.12.05'
+__version__ = '2015.12.06'

From db7c9da871def2ab6c8520760512a976ea0fe0e7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 7 Dec 2015 22:02:45 +0600
Subject: [PATCH 3285/6219] [pladform] Add _extract_url routine

---
 youtube_dl/extractor/pladform.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/youtube_dl/extractor/pladform.py b/youtube_dl/extractor/pladform.py
index 551c8c9f0..bc559d1df 100644
--- a/youtube_dl/extractor/pladform.py
+++ b/youtube_dl/extractor/pladform.py
@@ -1,6 +1,8 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
@@ -44,6 +46,13 @@ class PladformIE(InfoExtractor):
         'only_matching': True,
     }]
 
+    @staticmethod
+    def _extract_url(webpage):
+        mobj = re.search(
+            r'<iframe[^>]+src="(?P<url>(?:https?:)?//out\.pladform\.ru/player\?.+?)"', webpage)
+        if mobj:
+            return mobj.group('url')
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
 

From 45dad7ba1b28321299f973fc6a42bd7de64481d7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 7 Dec 2015 22:03:21 +0600
Subject: [PATCH 3286/6219] [extractor/generic] Use _extract_url for pladform

---
 youtube_dl/extractor/generic.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index e3bdff2d8..c2e8f9b62 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -54,6 +54,7 @@ from .onionstudios import OnionStudiosIE
 from .snagfilms import SnagFilmsEmbedIE
 from .screenwavemedia import ScreenwaveMediaIE
 from .mtv import MTVServicesEmbeddedIE
+from .pladform import PladformIE
 
 
 class GenericIE(InfoExtractor):
@@ -1741,10 +1742,9 @@ class GenericIE(InfoExtractor):
             return self.url_result('eagleplatform:%(host)s:%(id)s' % mobj.groupdict(), 'EaglePlatform')
 
         # Look for Pladform embeds
-        mobj = re.search(
-            r'<iframe[^>]+src="(?P<url>https?://out\.pladform\.ru/player\?.+?)"', webpage)
-        if mobj is not None:
-            return self.url_result(mobj.group('url'), 'Pladform')
+        pladform_url = PladformIE._extract_url(webpage)
+        if pladform_url:
+            return self.url_result(pladform_url)
 
         # Look for Playwire embeds
         mobj = re.search(

From c4737bea1716386e8b3455edfbd783fd415c192d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 7 Dec 2015 22:03:52 +0600
Subject: [PATCH 3287/6219] [vk] Add support for pladform embeds (Closes #7780)

---
 youtube_dl/extractor/vk.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index d99a42a9f..9632c85fa 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -18,6 +18,7 @@ from ..utils import (
     unified_strdate,
 )
 from .vimeo import VimeoIE
+from .pladform import PladformIE
 
 
 class VKIE(InfoExtractor):
@@ -254,6 +255,10 @@ class VKIE(InfoExtractor):
         if vimeo_url is not None:
             return self.url_result(vimeo_url)
 
+        pladform_url = PladformIE._extract_url(info_page)
+        if pladform_url:
+            return self.url_result(pladform_url)
+
         m_rutube = re.search(
             r'\ssrc="((?:https?:)?//rutube\.ru\\?/video\\?/embed(?:.*?))\\?"', info_page)
         if m_rutube is not None:

From 291a93bafa066530c19244c4d156422cf41e140a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 7 Dec 2015 22:04:47 +0600
Subject: [PATCH 3288/6219] [vk] Remove unnecessary message

---
 youtube_dl/extractor/vk.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 9632c85fa..d719096ff 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -262,7 +262,6 @@ class VKIE(InfoExtractor):
         m_rutube = re.search(
             r'\ssrc="((?:https?:)?//rutube\.ru\\?/video\\?/embed(?:.*?))\\?"', info_page)
         if m_rutube is not None:
-            self.to_screen('rutube video detected')
             rutube_url = self._proto_relative_url(
                 m_rutube.group(1).replace('\\', ''))
             return self.url_result(rutube_url)

From a5e52a1fd486db46e2115b949da29ec6c03b9bac Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 7 Dec 2015 22:05:54 +0600
Subject: [PATCH 3289/6219] [vk] Add test for pladform embed

---
 youtube_dl/extractor/vk.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index d719096ff..90557fa61 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -165,6 +165,11 @@ class VKIE(InfoExtractor):
             # vk wrapper
             'url': 'http://www.biqle.ru/watch/847655_160197695',
             'only_matching': True,
+        },
+        {
+            # pladform embed
+            'url': 'https://vk.com/video-76116461_171554880',
+            'only_matching': True,
         }
     ]
 

From 0d2d967cc77e445c781bf7bd9a00b75b3bc2236e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 7 Dec 2015 22:14:45 +0600
Subject: [PATCH 3290/6219] [wimp] Fix extraction (Closes #7784)

---
 youtube_dl/extractor/wimp.py | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/wimp.py b/youtube_dl/extractor/wimp.py
index e4f50e64c..022a3d1b5 100644
--- a/youtube_dl/extractor/wimp.py
+++ b/youtube_dl/extractor/wimp.py
@@ -5,7 +5,7 @@ from .youtube import YoutubeIE
 
 
 class WimpIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?wimp\.com/(?P<id>[^/]+)/'
+    _VALID_URL = r'http://(?:www\.)?wimp\.com/(?P<id>[^/]+)'
     _TESTS = [{
         'url': 'http://www.wimp.com/maruexhausted/',
         'md5': 'ee21217ffd66d058e8b16be340b74883',
@@ -28,18 +28,23 @@ class WimpIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
+
         webpage = self._download_webpage(url, video_id)
-        video_url = self._search_regex(
-            [r"[\"']file[\"']\s*[:,]\s*[\"'](.+?)[\"']", r"videoId\s*:\s*[\"']([^\"']+)[\"']"],
-            webpage, 'video URL')
-        if YoutubeIE.suitable(video_url):
-            self.to_screen('Found YouTube video')
+
+        youtube_id = self._search_regex(
+            r"videoId\s*:\s*[\"']([0-9A-Za-z_-]{11})[\"']",
+            webpage, 'video URL', default=None)
+        if youtube_id:
             return {
                 '_type': 'url',
-                'url': video_url,
+                'url': youtube_id,
                 'ie_key': YoutubeIE.ie_key(),
             }
 
+        video_url = self._search_regex(
+            r'<video[^>]+>\s*<source src=(["\'])(?P<url>.+?)\1',
+            webpage, 'video URL', group='url')
+
         return {
             'id': video_id,
             'url': video_url,

From 1775612512aba3c38455e534d423c36c330af35d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 7 Dec 2015 22:18:00 +0600
Subject: [PATCH 3291/6219] [wimp] Improve video URL regex

---
 youtube_dl/extractor/wimp.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/wimp.py b/youtube_dl/extractor/wimp.py
index 022a3d1b5..041ff6c55 100644
--- a/youtube_dl/extractor/wimp.py
+++ b/youtube_dl/extractor/wimp.py
@@ -42,7 +42,7 @@ class WimpIE(InfoExtractor):
             }
 
         video_url = self._search_regex(
-            r'<video[^>]+>\s*<source src=(["\'])(?P<url>.+?)\1',
+            r'<video[^>]+>\s*<source[^>]+src=(["\'])(?P<url>.+?)\1',
             webpage, 'video URL', group='url')
 
         return {

From 3d24bbfbe433e3a2b2257096264bf4e846f6d889 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 7 Dec 2015 23:10:57 +0600
Subject: [PATCH 3292/6219] [YoutubeDL] Check formats for merge to be opposite
 (#7786)

---
 youtube_dl/YoutubeDL.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 9a8c7da05..c642a1fbf 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1110,6 +1110,12 @@ class YoutubeDL(object):
                                           'contain the video, try using '
                                           '"-f %s+%s"' % (format_2, format_1))
                         return
+                    # Formats must be opposite (video+audio)
+                    if formats_info[0].get('acodec') == 'none' and formats_info[1].get('acodec') == 'none':
+                        self.report_error(
+                            'Both formats %s and %s are video-only, you must specify "-f video+audio"'
+                            % (format_1, format_2))
+                        return
                     output_ext = (
                         formats_info[0]['ext']
                         if self.params.get('merge_output_format') is None

From 5fd6cd64f9d4b9e7fadbea08fef1f3d131e6f041 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E8=99=BE=E5=93=A5=E5=93=A5?= <jfk_jin@sina.com>
Date: Tue, 8 Dec 2015 14:12:32 +0800
Subject: [PATCH 3293/6219] [sohu]fix 403 forbidden

---
 youtube_dl/extractor/sohu.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/sohu.py b/youtube_dl/extractor/sohu.py
index daf6ad555..ea8fc258d 100644
--- a/youtube_dl/extractor/sohu.py
+++ b/youtube_dl/extractor/sohu.py
@@ -158,6 +158,7 @@ class SohuIE(InfoExtractor):
                         'file': clips_url[i],
                         'new': su[i],
                         'prod': 'flash',
+                        'rb': 1,
                     }
 
                     if cdnId is not None:

From 18e4088fad97de0d79e29e0a49a0df3ce719a441 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 8 Dec 2015 21:15:45 +0600
Subject: [PATCH 3294/6219] [theplatform] Add support for referer protected
 videos wuth explicit SMIL

---
 youtube_dl/extractor/theplatform.py | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index 1555aa77c..0bf6726b5 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -16,11 +16,12 @@ from ..compat import (
 from ..utils import (
     determine_ext,
     ExtractorError,
-    xpath_with_ns,
-    unsmuggle_url,
-    int_or_none,
-    url_basename,
     float_or_none,
+    int_or_none,
+    sanitized_Request,
+    unsmuggle_url,
+    url_basename,
+    xpath_with_ns,
 )
 
 default_ns = 'http://www.w3.org/2005/SMIL21/Language'
@@ -204,7 +205,12 @@ class ThePlatformIE(ThePlatformBaseIE):
             smil_url = url
         # Explicitly specified SMIL (see https://github.com/rg3/youtube-dl/issues/7385)
         elif '/guid/' in url:
-            webpage = self._download_webpage(url, video_id)
+            headers = {}
+            source_url = smuggled_data.get('source_url')
+            if source_url:
+                headers['Referer'] = source_url
+            request = sanitized_Request(url, headers=headers)
+            webpage = self._download_webpage(request, video_id)
             smil_url = self._search_regex(
                 r'<link[^>]+href=(["\'])(?P<url>.+?)\1[^>]+type=["\']application/smil\+xml',
                 webpage, 'smil url', group='url')

From b46b65ed370340159e77bf875e99fb93133b6492 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 8 Dec 2015 21:16:14 +0600
Subject: [PATCH 3295/6219] [nbc] Smuggle referer (Closes #7791)

---
 youtube_dl/extractor/nbc.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index e683d24c4..7de86a969 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -11,6 +11,7 @@ from ..utils import (
     ExtractorError,
     find_xpath_attr,
     lowercase_escape,
+    smuggle_url,
     unescapeHTML,
 )
 
@@ -67,7 +68,7 @@ class NBCIE(InfoExtractor):
             webpage, 'theplatform url').replace('_no_endcard', '').replace('\\/', '/')))
         if theplatform_url.startswith('//'):
             theplatform_url = 'http:' + theplatform_url
-        return self.url_result(theplatform_url)
+        return self.url_result(smuggle_url(theplatform_url, {'source_url': url}))
 
 
 class NBCSportsVPlayerIE(InfoExtractor):

From 3348243b7ba94e83dfb9d675fa0e701f97d3e220 Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Tue, 8 Dec 2015 21:34:26 +0600
Subject: [PATCH 3296/6219] [README.md] Clarify verbose log requirements

---
 README.md | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 0b224f0be..e39f71281 100644
--- a/README.md
+++ b/README.md
@@ -800,7 +800,21 @@ with youtube_dl.YoutubeDL(ydl_opts) as ydl:
 
 Bugs and suggestions should be reported at: <https://github.com/rg3/youtube-dl/issues> . Unless you were prompted so or there is another pertinent reason (e.g. GitHub fails to accept the bug report), please do not send bug reports via personal email. For discussions, join us in the irc channel #youtube-dl on freenode.
 
-**Please include the full output of youtube-dl when run with `-v`**.
+**Please include the full output of youtube-dl when run with `-v`**, i.e. add `-v` flag to your command line, copy the **whole** output and post it in the issue body wrapped in \`\`\` for better formatting. It should look similar to this:
+```
+$ youtube-dl -v http://www.youtube.com/watch?v=BaW_jenozKcj
+[debug] System config: []
+[debug] User config: []
+[debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
+[debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
+[debug] youtube-dl version 2015.12.06
+[debug] Git HEAD: 135392e
+[debug] Python version 2.6.6 - Windows-2003Server-5.2.3790-SP2
+[debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
+[debug] Proxy map: {}
+...
+```
+**Do not post screenshots of verbose log only plain text is acceptable.**
 
 The output (including the first lines) contains important debugging information. Issues without the full output are often not reproducible and therefore do not get solved in short order, if ever.
 

From b09c122373ad3a1e04148aeeac735443fd91e39d Mon Sep 17 00:00:00 2001
From: Sergey M? <dstftw@gmail.com>
Date: Tue, 8 Dec 2015 21:16:58 +0600
Subject: [PATCH 3297/6219] [nbc] Add another theplatform pattern

---
 youtube_dl/extractor/nbc.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index 7de86a969..4c1eca96f 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -63,6 +63,7 @@ class NBCIE(InfoExtractor):
         theplatform_url = unescapeHTML(lowercase_escape(self._html_search_regex(
             [
                 r'(?:class="video-player video-player-full" data-mpx-url|class="player" src)="(.*?)"',
+                r'<iframe[^>]+src="((?:https?:)?//player\.theplatform\.com/[^"]+)"',
                 r'"embedURL"\s*:\s*"([^"]+)"'
             ],
             webpage, 'theplatform url').replace('_no_endcard', '').replace('\\/', '/')))

From 39fa4cc10775d2d2b8a49d2268b11e34f32b3d2c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 8 Dec 2015 21:56:00 +0600
Subject: [PATCH 3298/6219] [cliphunter] Fix extraction (Closes #7796)

---
 youtube_dl/extractor/cliphunter.py | 43 +++++++++++++-----------------
 1 file changed, 19 insertions(+), 24 deletions(-)

diff --git a/youtube_dl/extractor/cliphunter.py b/youtube_dl/extractor/cliphunter.py
index d46592cc5..2996b6b09 100644
--- a/youtube_dl/extractor/cliphunter.py
+++ b/youtube_dl/extractor/cliphunter.py
@@ -1,7 +1,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..utils import determine_ext
+from ..utils import int_or_none
 
 
 _translation_table = {
@@ -42,31 +42,26 @@ class CliphunterIE(InfoExtractor):
         video_title = self._search_regex(
             r'mediaTitle = "([^"]+)"', webpage, 'title')
 
-        fmts = {}
-        for fmt in ('mp4', 'flv'):
-            fmt_list = self._parse_json(self._search_regex(
-                r'var %sjson\s*=\s*(\[.*?\]);' % fmt, webpage, '%s formats' % fmt), video_id)
-            for f in fmt_list:
-                fmts[f['fname']] = _decode(f['sUrl'])
-
-        qualities = self._parse_json(self._search_regex(
-            r'var player_btns\s*=\s*(.*?);\n', webpage, 'quality info'), video_id)
+        gexo_files = self._parse_json(
+            self._search_regex(
+                r'var\s+gexoFiles\s*=\s*({.+?});', webpage, 'gexo files'),
+            video_id)
 
         formats = []
-        for fname, url in fmts.items():
-            f = {
-                'url': url,
-            }
-            if fname in qualities:
-                qual = qualities[fname]
-                f.update({
-                    'format_id': '%s_%sp' % (determine_ext(url), qual['h']),
-                    'width': qual['w'],
-                    'height': qual['h'],
-                    'tbr': qual['br'],
-                })
-            formats.append(f)
-
+        for format_id, f in gexo_files.items():
+            video_url = f.get('url')
+            if not video_url:
+                continue
+            fmt = f.get('fmt')
+            height = f.get('h')
+            format_id = '%s_%sp' % (fmt, height) if fmt and height else format_id
+            formats.append({
+                'url': _decode(video_url),
+                'format_id': format_id,
+                'width': int_or_none(f.get('w')),
+                'height': int_or_none(height),
+                'tbr': int_or_none(f.get('br')),
+            })
         self._sort_formats(formats)
 
         thumbnail = self._search_regex(

From 612d83b51df2712327a2643880a80c62e4296d71 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 8 Dec 2015 23:28:36 +0600
Subject: [PATCH 3299/6219] [pbs] Extend _VALID_URL

---
 youtube_dl/extractor/pbs.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index b787e2a73..7a4ed23a1 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -18,7 +18,7 @@ class PBSIE(InfoExtractor):
     _VALID_URL = r'''(?x)https?://
         (?:
            # Direct video URL
-           video\.pbs\.org/(?:viralplayer|video)/(?P<id>[0-9]+)/? |
+           (?:video\.pbs|watch\.knpb)\.org/(?:viralplayer|video)/(?P<id>[0-9]+)/? |
            # Article with embedded player (or direct video)
            (?:www\.)?pbs\.org/(?:[^/]+/){2,5}(?P<presumptive_id>[^/]+?)(?:\.html)?/?(?:$|[?\#]) |
            # Player
@@ -174,6 +174,10 @@ class PBSIE(InfoExtractor):
         {
             'url': 'http://player.pbs.org/widget/partnerplayer/2365297708/?start=0&end=0&chapterbar=false&endscreen=false&topbar=true',
             'only_matching': True,
+        },
+        {
+            'url': 'http://watch.knpb.org/video/2365616055/',
+            'only_matching': True,
         }
     ]
     _ERRORS = {

From d0c8b279daa83261b5644407ffb4a1fdb3e1b0e3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 8 Dec 2015 23:34:43 +0600
Subject: [PATCH 3300/6219] [pbs] Add another coveplayer pattern (Closes #7674)

---
 youtube_dl/extractor/pbs.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 7a4ed23a1..ab3c43cfe 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -208,6 +208,7 @@ class PBSIE(InfoExtractor):
             MEDIA_ID_REGEXES = [
                 r"div\s*:\s*'videoembed'\s*,\s*mediaid\s*:\s*'(\d+)'",  # frontline video embed
                 r'class="coveplayerid">([^<]+)<',                       # coveplayer
+                r'<section[^>]+data-coveid="(\d+)"',                    # coveplayer from http://www.pbs.org/wgbh/frontline/film/real-csi/
                 r'<input type="hidden" id="pbs_video_id_[0-9]+" value="([0-9]+)"/>',  # jwplayer
             ]
 

From e15e2ef7a00362fc1e89d9959241efad46325c54 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 9 Dec 2015 01:51:34 +0600
Subject: [PATCH 3301/6219] [pbs] Add support for all member stations (#7674)

---
 youtube_dl/extractor/pbs.py | 477 +++++++++++++++++++++++++++++++++++-
 1 file changed, 475 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index ab3c43cfe..decc376a2 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -15,16 +15,489 @@ from ..utils import (
 
 
 class PBSIE(InfoExtractor):
+    _STATIONS = (
+        ('pbs.org', 'PBS: Public Broadcasting Service'),  # http://www.pbs.org/
+        ('aptv.org', 'APT - Alabama Public Television (WBIQ)'),  # AL, http://aptv.org/
+        ('gpb.org', 'GPB/Georgia Public Broadcasting (WGTV)'),  # AL, http://www.gpb.org/
+        ('mpbonline.org', 'Mississippi Public Broadcasting (WMPN)'),  # AL, http://www.mpbonline.org
+        ('wnpt.org', 'Nashville Public Television (WNPT)'),  # AL, http://www.wnpt.org
+        ('wfsu.org', 'WFSU-TV (WFSU)'),  # AL, http://wfsu.org/
+        ('wsre.org', 'WSRE (WSRE)'),  # AL, http://www.wsre.org
+        ('wtcitv.org', 'WTCI (WTCI)'),  # AL, http://www.wtcitv.org
+        ('pba.org', 'WPBA/Channel 30 (WPBA)'),  # AL, http://pba.org/
+        ('alaskapublic.org', 'Alaska Public Media (KAKM)'),  # AK, http://alaskapublic.org/kakm
+        ('kuac.org', 'KUAC (KUAC)'),  # AK, http://kuac.org/kuac-tv/
+        ('ktoo.org', '360 North (KTOO)'),  # AK, http://www.ktoo.org/
+        ('azpm.org', 'KUAT 6 (KUAT)'),  # AZ, http://www.azpm.org/
+        ('azpbs.org', 'Arizona PBS (KAET)'),  # AZ, http://www.azpbs.org
+        ('newmexicopbs.org', 'KNME-TV/Channel 5 (KNME)'),  # AZ, http://www.newmexicopbs.org/
+        ('vegaspbs.org', 'Vegas PBS (KLVX)'),  # AZ, http://vegaspbs.org/
+        ('aetn.org', 'AETN/ARKANSAS ETV NETWORK (KETS)'),  # AR, http://www.aetn.org/
+        ('ket.org', 'KET (WKLE)'),  # AR, http://www.ket.org/
+        ('wkno.org', 'WKNO/Channel 10 (WKNO)'),  # AR, http://www.wkno.org/
+        ('lpb.org', 'LPB/LOUISIANA PUBLIC BROADCASTING (WLPB)'),  # AR, http://www.lpb.org/
+        ('mpbonline.org', 'Mississippi Public Broadcasting (WMPN)'),  # AR, http://www.mpbonline.org
+        ('oeta.tv', 'OETA (KETA)'),  # AR, http://www.oeta.tv
+        ('optv.org', 'Ozarks Public Television (KOZK)'),  # AR, http://www.optv.org/
+        ('wsiu.org', 'WSIU Public Broadcasting (WSIU)'),  # AR, http://www.wsiu.org/
+        ('keet.org', 'KEET TV (KEET)'),  # CA, http://www.keet.org
+        ('kixe.org', 'KIXE/Channel 9 (KIXE)'),  # CA, http://kixe.org/
+        ('kpbs.org', 'KPBS San Diego (KPBS)'),  # CA, http://www.kpbs.org/
+        ('kqed.org', 'KQED (KQED)'),  # CA, http://www.kqed.org
+        ('kvie.org', 'KVIE Public Television (KVIE)'),  # CA, http://www.kvie.org
+        ('pbssocal.org', 'PBS SoCal/KOCE (KOCE)'),  # CA, http://www.pbssocal.org/
+        ('valleypbs.org', 'ValleyPBS (KVPT)'),  # CA, http://www.valleypbs.org/
+        ('aptv.org', 'APT - Alabama Public Television (WBIQ)'),  # CA, http://aptv.org/
+        ('cptv.org', 'CONNECTICUT PUBLIC TELEVISION (WEDH)'),  # CA, http://cptv.org
+        ('azpbs.org', 'Arizona PBS (KAET)'),  # CA, http://www.azpbs.org
+        ('knpb.org', 'KNPB Channel 5 (KNPB)'),  # CA, http://www.knpb.org/
+        ('soptv.org', 'SOPTV (KSYS)'),  # CA, http://www.soptv.org
+        ('klcs.org', 'KLCS/Channel 58 (KLCS)'),  # CA, http://www.klcs.org
+        ('krcb.org', 'KRCB Television & Radio (KRCB)'),  # CA, http://www.krcb.org
+        ('kvcr.org', 'KVCR TV/DT/FM :: Vision for the Future (KVCR)'),  # CA, http://kvcr.org
+        ('rmpbs.org', 'Rocky Mountain PBS (KRMA)'),  # CO, http://www.rmpbs.org
+        ('kenw.org', 'KENW-TV3 (KENW)'),  # CO, http://www.kenw.org
+        ('kued.org', 'KUED Channel 7 (KUED)'),  # CO, http://www.kued.org
+        ('newmexicopbs.org', 'KNME-TV/Channel 5 (KNME)'),  # CO, http://www.newmexicopbs.org/
+        ('wyomingpbs.org', 'Wyoming PBS (KCWC)'),  # CO, http://www.wyomingpbs.org
+        ('cpt12.org', 'Colorado Public Television / KBDI 12 (KBDI)'),  # CO, http://www.cpt12.org/
+        ('kbyutv.org', 'KBYU-TV (KBYU)'),  # CO, http://www.kbyutv.org/
+        ('cptv.org', 'CONNECTICUT PUBLIC TELEVISION (WEDH)'),  # CT, http://cptv.org
+        ('thirteen.org', 'Thirteen/WNET New York (WNET)'),  # CT, http://www.thirteen.org
+        ('wgbh.org', 'WGBH/Channel 2 (WGBH)'),  # CT, http://wgbh.org
+        ('wgby.org', 'WGBY (WGBY)'),  # CT, http://www.wgby.org
+        ('njtvonline.org', 'NJTV Public Media NJ (WNJT)'),  # CT, http://www.njtvonline.org/
+        ('ripbs.org', 'Rhode Island PBS (WSBE)'),  # CT, http://www.ripbs.org/home/
+        ('wliw.org', 'WLIW21 (WLIW)'),  # CT, http://www.wliw.org/
+        ('mpt.org', 'mpt/Maryland Public Television (WMPB)'),  # DE, http://www.mpt.org
+        ('weta.org', 'WETA Television and Radio (WETA)'),  # DE, http://www.weta.org
+        ('whyy.org', 'WHYY (WHYY)'),  # DE, http://www.whyy.org
+        ('njtvonline.org', 'NJTV Public Media NJ (WNJT)'),  # DE, http://www.njtvonline.org/
+        ('wlvt.org', 'PBS 39 (WLVT)'),  # DE, http://www.wlvt.org/
+        ('wliw.org', 'WLIW21 (WLIW)'),  # DE, http://www.wliw.org/
+        ('weta.org', 'WETA Television and Radio (WETA)'),  # DC, http://www.weta.org
+        ('mpt.org', 'mpt/Maryland Public Television (WMPB)'),  # DC, http://www.mpt.org
+        ('wvpt.net', 'WVPT - Your Source for PBS and More! (WVPT)'),  # DC, http://www.wvpt.net
+        ('whut.org', 'Howard University Television (WHUT)'),  # DC, http://www.whut.org
+        ('wedu.org', 'WEDU PBS (WEDU)'),  # FL, http://www.wedu.org
+        ('wfsu.org', 'WFSU-TV (WFSU)'),  # FL, http://wfsu.org/
+        ('wgcu.org', 'WGCU Public Media (WGCU)'),  # FL, http://www.wgcu.org/
+        ('wjct.org', 'WJCT Public Broadcasting (WJCT)'),  # FL, http://www.wjct.org
+        ('wpbt2.org', 'WPBT2 (WPBT)'),  # FL, http://www.wpbt2.org
+        ('wsre.org', 'WSRE (WSRE)'),  # FL, http://www.wsre.org
+        ('wucftv.org', 'WUCF TV (WUCF)'),  # FL, http://wucftv.org
+        ('wuft.org', 'WUFT/Channel 5 (WUFT)'),  # FL, http://www.wuft.org
+        ('wxel.org', 'WXEL/Channel 42 (WXEL)'),  # FL, http://www.wxel.org/home/
+        ('aptv.org', 'APT - Alabama Public Television (WBIQ)'),  # FL, http://aptv.org/
+        ('gpb.org', 'GPB/Georgia Public Broadcasting (WGTV)'),  # FL, http://www.gpb.org/
+        ('wlrn.org', 'WLRN/Channel 17 (WLRN)'),  # FL, http://www.wlrn.org/
+        ('wusf.org', 'WUSF Public Broadcasting (WUSF)'),  # FL, http://wusf.org/
+        ('njtvonline.org', 'NJTV Public Media NJ (WNJT)'),  # FL, http://www.njtvonline.org/
+        ('gpb.org', 'GPB/Georgia Public Broadcasting (WGTV)'),  # GA, http://www.gpb.org/
+        ('aptv.org', 'APT - Alabama Public Television (WBIQ)'),  # GA, http://aptv.org/
+        ('scetv.org', 'ETV (WRLK)'),  # GA, http://www.scetv.org
+        ('unctv.org', 'UNC-TV (WUNC)'),  # GA, http://www.unctv.org/
+        ('wfsu.org', 'WFSU-TV (WFSU)'),  # GA, http://wfsu.org/
+        ('wjct.org', 'WJCT Public Broadcasting (WJCT)'),  # GA, http://www.wjct.org
+        ('wtcitv.org', 'WTCI (WTCI)'),  # GA, http://www.wtcitv.org
+        ('pba.org', 'WPBA/Channel 30 (WPBA)'),  # GA, http://pba.org/
+        ('pbsguam.org', 'PBS Guam (KGTF)'),  # GU, http://www.pbsguam.org/
+        ('pbshawaii.org', 'PBS Hawaii - Oceanic Cable Channel 10 (KHET)'),  # HI, http://www.pbshawaii.org/
+        ('idahoptv.org', 'Idaho Public Television (KAID)'),  # ID, http://idahoptv.org
+        ('ksps.org', 'KSPS (KSPS)'),  # ID, http://www.ksps.org/home/
+        ('kued.org', 'KUED Channel 7 (KUED)'),  # ID, http://www.kued.org
+        ('opb.org', 'OPB (KOPB)'),  # ID, http://www.opb.org
+        ('soptv.org', 'SOPTV (KSYS)'),  # ID, http://www.soptv.org
+        ('wyomingpbs.org', 'Wyoming PBS (KCWC)'),  # ID, http://www.wyomingpbs.org
+        ('cpt12.org', 'Colorado Public Television / KBDI 12 (KBDI)'),  # ID, http://www.cpt12.org/
+        ('kbyutv.org', 'KBYU-TV (KBYU)'),  # ID, http://www.kbyutv.org/
+        ('kwsu.org', 'KWSU/Channel 10 & KTNW/Channel 31 (KWSU)'),  # ID, http://www.kwsu.org
+        ('illinois.edu', 'WILL-TV (WILL)'),  # IL, http://will.illinois.edu/
+        ('wsec.tv', 'Network Knowledge - WSEC/Springfield (WSEC)'),  # IL, http://www.wsec.tv
+        ('wsiu.org', 'WSIU Public Broadcasting (WSIU)'),  # IL, http://www.wsiu.org/
+        ('wttw.com', 'WTTW11 (WTTW)'),  # IL, http://www.wttw.com/
+        ('wtvp.org', 'WTVP & WTVP.org, Public Media for Central Illinois (WTVP)'),  # IL, http://www.wtvp.org/
+        ('iptv.org', 'Iowa Public Television/IPTV (KDIN)'),  # IL, http://www.iptv.org/
+        ('ket.org', 'KET (WKLE)'),  # IL, http://www.ket.org/
+        ('ninenet.org', 'Nine Network (KETC)'),  # IL, http://www.ninenet.org
+        ('wfwa.org', 'PBS39 Fort Wayne (WFWA)'),  # IL, http://wfwa.org/
+        ('wfyi.org', 'WFYI Indianapolis (WFYI)'),  # IL, http://www.wfyi.org
+        ('mptv.org', 'Milwaukee Public Television (WMVS)'),  # IL, http://www.mptv.org
+        ('wnin.org', 'WNIN (WNIN)'),  # IL, http://www.wnin.org/
+        ('wnit.org', 'WNIT Public Television (WNIT)'),  # IL, http://www.wnit.org/
+        ('wpt.org', 'WPT (WPNE)'),  # IL, http://www.wpt.org/
+        ('wvut.org', 'WVUT/Channel 22 (WVUT)'),  # IL, http://wvut.org/
+        ('weiu.net', 'WEIU/Channel 51 (WEIU)'),  # IL, http://www.weiu.net
+        ('wqpt.org', 'WQPT-TV (WQPT)'),  # IL, http://www.wqpt.org
+        ('wycc.org', 'WYCC PBS Chicago (WYCC)'),  # IL, http://www.wycc.org
+        ('lakeshorepublicmedia.org', 'Lakeshore Public Television (WYIN)'),  # IL, http://lakeshorepublicmedia.org/
+        ('wfwa.org', 'PBS39 Fort Wayne (WFWA)'),  # IN, http://wfwa.org/
+        ('wfyi.org', 'WFYI Indianapolis (WFYI)'),  # IN, http://www.wfyi.org
+        ('wipb.org', 'WIPB-TV (WIPB)'),  # IN, http://wipb.org
+        ('wnin.org', 'WNIN (WNIN)'),  # IN, http://www.wnin.org/
+        ('wnit.org', 'WNIT Public Television (WNIT)'),  # IN, http://www.wnit.org/
+        ('indianapublicmedia.org', 'WTIU (WTIU)'),  # IN, http://indianapublicmedia.org/tv/
+        ('wvut.org', 'WVUT/Channel 22 (WVUT)'),  # IN, http://wvut.org/
+        ('cetconnect.org', 'CET  (WCET)'),  # IN, http://www.cetconnect.org
+        ('ket.org', 'KET (WKLE)'),  # IN, http://www.ket.org/
+        ('thinktv.org', 'ThinkTVNetwork (WPTD)'),  # IN, http://www.thinktv.org
+        ('wbgu.org', 'WBGU-TV (WBGU)'),  # IN, http://wbgu.org
+        ('wgvu.org', 'WGVU TV (WGVU)'),  # IN, http://www.wgvu.org/
+        ('illinois.edu', 'WILL-TV (WILL)'),  # IN, http://will.illinois.edu/
+        ('wsec.tv', 'Network Knowledge - WSEC/Springfield (WSEC)'),  # IN, http://www.wsec.tv
+        ('wsiu.org', 'WSIU Public Broadcasting (WSIU)'),  # IN, http://www.wsiu.org/
+        ('wttw.com', 'WTTW11 (WTTW)'),  # IN, http://www.wttw.com/
+        ('lakeshorepublicmedia.org', 'Lakeshore Public Television (WYIN)'),  # IN, http://lakeshorepublicmedia.org/
+        ('weiu.net', 'WEIU/Channel 51 (WEIU)'),  # IN, http://www.weiu.net
+        ('wycc.org', 'WYCC PBS Chicago (WYCC)'),  # IN, http://www.wycc.org
+        ('iptv.org', 'Iowa Public Television/IPTV (KDIN)'),  # IA, http://www.iptv.org/
+        ('netnebraska.org', 'NET1 (KUON)'),  # IA, http://netnebraska.org
+        ('pioneer.org', 'Pioneer Public Television (KWCM)'),  # IA, http://www.pioneer.org
+        ('sdpb.org', 'SDPB Television (KUSD)'),  # IA, http://www.sdpb.org
+        ('tpt.org', 'TPT (KTCA)'),  # IA, http://www.tpt.org
+        ('wsec.tv', 'Network Knowledge - WSEC/Springfield (WSEC)'),  # IA, http://www.wsec.tv
+        ('wpt.org', 'WPT (WPNE)'),  # IA, http://www.wpt.org/
+        ('ksmq.org', 'KSMQ (KSMQ)'),  # IA, http://www.ksmq.org/
+        ('wqpt.org', 'WQPT-TV (WQPT)'),  # IA, http://www.wqpt.org
+        ('kpts.org', 'KPTS/Channel 8 (KPTS)'),  # KS, http://www.kpts.org/
+        ('ktwu.org', 'KTWU/Channel 11 (KTWU)'),  # KS, http://ktwu.org
+        ('shptv.org', 'Smoky Hills Public Television (KOOD)'),  # KS, http://www.shptv.org
+        ('kcpt.org', 'KCPT Kansas City Public Television (KCPT)'),  # KS, http://kcpt.org/
+        ('netnebraska.org', 'NET1 (KUON)'),  # KS, http://netnebraska.org
+        ('oeta.tv', 'OETA (KETA)'),  # KS, http://www.oeta.tv
+        ('optv.org', 'Ozarks Public Television (KOZK)'),  # KS, http://www.optv.org/
+        ('rmpbs.org', 'Rocky Mountain PBS (KRMA)'),  # KS, http://www.rmpbs.org
+        ('cpt12.org', 'Colorado Public Television / KBDI 12 (KBDI)'),  # KS, http://www.cpt12.org/
+        ('ket.org', 'KET (WKLE)'),  # KY, http://www.ket.org/
+        ('blueridgepbs.org', 'Blue Ridge PBS (WBRA)'),  # KY, http://www.blueridgepbs.org/
+        ('cetconnect.org', 'CET  (WCET)'),  # KY, http://www.cetconnect.org
+        ('easttennesseepbs.org', 'East Tennessee PBS (WSJK)'),  # KY, http://easttennesseepbs.org
+        ('wnpt.org', 'Nashville Public Television (WNPT)'),  # KY, http://www.wnpt.org
+        ('thinktv.org', 'ThinkTVNetwork (WPTD)'),  # KY, http://www.thinktv.org
+        ('wcte.org', 'WCTE-TV (WCTE)'),  # KY, http://www.wcte.org
+        ('wljt.org', 'WLJT, Channel 11 (WLJT)'),  # KY, http://wljt.org/
+        ('wnin.org', 'WNIN (WNIN)'),  # KY, http://www.wnin.org/
+        ('wosu.org', 'WOSU TV (WOSU)'),  # KY, http://wosu.org/
+        ('woub.org', 'WOUB/WOUC (WOUB)'),  # KY, http://woub.org/tv/index.php?section=5
+        ('wsiu.org', 'WSIU Public Broadcasting (WSIU)'),  # KY, http://www.wsiu.org/
+        ('wvpublic.org', 'WVPB (WVPB)'),  # KY, http://wvpublic.org/
+        ('wkyupbs.org', 'WKYU-PBS (WKYU)'),  # KY, http://www.wkyupbs.org
+        ('lpb.org', 'LPB/LOUISIANA PUBLIC BROADCASTING (WLPB)'),  # LA, http://www.lpb.org/
+        ('wyes.org', 'WYES-TV/New Orleans (WYES)'),  # LA, http://www.wyes.org
+        ('aetn.org', 'AETN/ARKANSAS ETV NETWORK (KETS)'),  # LA, http://www.aetn.org/
+        ('kera.org', 'KERA 13 (KERA)'),  # LA, http://www.kera.org/
+        ('mpbonline.org', 'Mississippi Public Broadcasting (WMPN)'),  # LA, http://www.mpbonline.org
+        ('mpbn.net', 'MPBN (WCBB)'),  # ME, http://www.mpbn.net/
+        ('mountainlake.org', 'Mountain Lake PBS (WCFE)'),  # ME, http://www.mountainlake.org/
+        ('nhptv.org', 'NHPTV (WENH)'),  # ME, http://nhptv.org/
+        ('vpt.org', 'Vermont PBS (WETK)'),  # ME, http://www.vpt.org
+        ('wgbh.org', 'WGBH/Channel 2 (WGBH)'),  # ME, http://wgbh.org
+        ('mpt.org', 'mpt/Maryland Public Television (WMPB)'),  # MD, http://www.mpt.org
+        ('thirteen.org', 'Thirteen/WNET New York (WNET)'),  # MD, http://www.thirteen.org
+        ('weta.org', 'WETA Television and Radio (WETA)'),  # MD, http://www.weta.org
+        ('whyy.org', 'WHYY (WHYY)'),  # MD, http://www.whyy.org
+        ('witf.org', 'witf (WITF)'),  # MD, http://www.witf.org
+        ('wqed.org', 'WQED Multimedia (WQED)'),  # MD, http://www.wqed.org/
+        ('wvpublic.org', 'WVPB (WVPB)'),  # MD, http://wvpublic.org/
+        ('wvpt.net', 'WVPT - Your Source for PBS and More! (WVPT)'),  # MD, http://www.wvpt.net
+        ('whut.org', 'Howard University Television (WHUT)'),  # MD, http://www.whut.org
+        ('wliw.org', 'WLIW21 (WLIW)'),  # MD, http://www.wliw.org/
+        ('wgbh.org', 'WGBH/Channel 2 (WGBH)'),  # MA, http://wgbh.org
+        ('wgby.org', 'WGBY (WGBY)'),  # MA, http://www.wgby.org
+        ('cptv.org', 'CONNECTICUT PUBLIC TELEVISION (WEDH)'),  # MA, http://cptv.org
+        ('nhptv.org', 'NHPTV (WENH)'),  # MA, http://nhptv.org/
+        ('vpt.org', 'Vermont PBS (WETK)'),  # MA, http://www.vpt.org
+        ('wmht.org', 'WMHT Educational Telecommunications (WMHT)'),  # MA, http://www.wmht.org/home/
+        ('ripbs.org', 'Rhode Island PBS (WSBE)'),  # MA, http://www.ripbs.org/home/
+        ('deltabroadcasting.org', 'Q-TV (WDCQ)'),  # MI, http://www.deltabroadcasting.org
+        ('dptv.org', 'WTVS Detroit Public TV (WTVS)'),  # MI, http://www.dptv.org/
+        ('wcmu.org', 'CMU Public Television (WCMU)'),  # MI, http://www.wcmu.org
+        ('wgvu.org', 'WGVU TV (WGVU)'),  # MI, http://www.wgvu.org/
+        ('wkar.org', 'WKAR-TV (WKAR)'),  # MI, http://wkar.org/
+        ('nmu.edu', 'WNMU-TV Public TV 13 (WNMU)'),  # MI, http://wnmutv.nmu.edu
+        ('wdse.org', 'WDSE - WRPT (WDSE)'),  # MI, http://www.wdse.org/
+        ('wbgu.org', 'WBGU-TV (WBGU)'),  # MI, http://wbgu.org
+        ('wgte.org', 'WGTE TV (WGTE)'),  # MI, http://www.wgte.org
+        ('wnit.org', 'WNIT Public Television (WNIT)'),  # MI, http://www.wnit.org/
+        ('wpt.org', 'WPT (WPNE)'),  # MI, http://www.wpt.org/
+        ('wttw.com', 'WTTW11 (WTTW)'),  # MI, http://www.wttw.com/
+        ('wycc.org', 'WYCC PBS Chicago (WYCC)'),  # MI, http://www.wycc.org
+        ('lakelandptv.org', 'Lakeland Public Television (KAWE)'),  # MN, http://www.lakelandptv.org
+        ('wdse.org', 'WDSE - WRPT (WDSE)'),  # MN, http://www.wdse.org/
+        ('pioneer.org', 'Pioneer Public Television (KWCM)'),  # MN, http://www.pioneer.org
+        ('tpt.org', 'TPT (KTCA)'),  # MN, http://www.tpt.org
+        ('iptv.org', 'Iowa Public Television/IPTV (KDIN)'),  # MN, http://www.iptv.org/
+        ('prairiepublic.org', 'PRAIRIE PUBLIC (KFME)'),  # MN, http://www.prairiepublic.org/
+        ('ninenet.org', 'Nine Network (KETC)'),  # MN, http://www.ninenet.org
+        ('sdpb.org', 'SDPB Television (KUSD)'),  # MN, http://www.sdpb.org
+        ('wpt.org', 'WPT (WPNE)'),  # MN, http://www.wpt.org/
+        ('ksmq.org', 'KSMQ (KSMQ)'),  # MN, http://www.ksmq.org/
+        ('mpbonline.org', 'Mississippi Public Broadcasting (WMPN)'),  # MS, http://www.mpbonline.org
+        ('aetn.org', 'AETN/ARKANSAS ETV NETWORK (KETS)'),  # MS, http://www.aetn.org/
+        ('aptv.org', 'APT - Alabama Public Television (WBIQ)'),  # MS, http://aptv.org/
+        ('wkno.org', 'WKNO/Channel 10 (WKNO)'),  # MS, http://www.wkno.org/
+        ('lpb.org', 'LPB/LOUISIANA PUBLIC BROADCASTING (WLPB)'),  # MS, http://www.lpb.org/
+        ('wsre.org', 'WSRE (WSRE)'),  # MS, http://www.wsre.org
+        ('wyes.org', 'WYES-TV/New Orleans (WYES)'),  # MS, http://www.wyes.org
+        ('kcpt.org', 'KCPT Kansas City Public Television (KCPT)'),  # MO, http://kcpt.org/
+        ('kmos.org', 'KMOS-TV - Channels 6.1, 6.2 and 6.3 (KMOS)'),  # MO, http://www.kmos.org/
+        ('ninenet.org', 'Nine Network (KETC)'),  # MO, http://www.ninenet.org
+        ('optv.org', 'Ozarks Public Television (KOZK)'),  # MO, http://www.optv.org/
+        ('aetn.org', 'AETN/ARKANSAS ETV NETWORK (KETS)'),  # MO, http://www.aetn.org/
+        ('dptv.org', 'WTVS Detroit Public TV (WTVS)'),  # MO, http://www.dptv.org/
+        ('iptv.org', 'Iowa Public Television/IPTV (KDIN)'),  # MO, http://www.iptv.org/
+        ('ket.org', 'KET (WKLE)'),  # MO, http://www.ket.org/
+        ('wkno.org', 'WKNO/Channel 10 (WKNO)'),  # MO, http://www.wkno.org/
+        ('ktwu.org', 'KTWU/Channel 11 (KTWU)'),  # MO, http://ktwu.org
+        ('netnebraska.org', 'NET1 (KUON)'),  # MO, http://netnebraska.org
+        ('rmpbs.org', 'Rocky Mountain PBS (KRMA)'),  # MO, http://www.rmpbs.org
+        ('wsec.tv', 'Network Knowledge - WSEC/Springfield (WSEC)'),  # MO, http://www.wsec.tv
+        ('wsiu.org', 'WSIU Public Broadcasting (WSIU)'),  # MO, http://www.wsiu.org/
+        ('montanapbs.org', 'MontanaPBS (KUSM)'),  # MT, http://montanapbs.org
+        ('azpbs.org', 'Arizona PBS (KAET)'),  # MT, http://www.azpbs.org
+        ('idahoptv.org', 'Idaho Public Television (KAID)'),  # MT, http://idahoptv.org
+        ('prairiepublic.org', 'PRAIRIE PUBLIC (KFME)'),  # MT, http://www.prairiepublic.org/
+        ('ksps.org', 'KSPS (KSPS)'),  # MT, http://www.ksps.org/home/
+        ('rmpbs.org', 'Rocky Mountain PBS (KRMA)'),  # MT, http://www.rmpbs.org
+        ('sdpb.org', 'SDPB Television (KUSD)'),  # MT, http://www.sdpb.org
+        ('netnebraska.org', 'NET1 (KUON)'),  # NE, http://netnebraska.org
+        ('iptv.org', 'Iowa Public Television/IPTV (KDIN)'),  # NE, http://www.iptv.org/
+        ('ktwu.org', 'KTWU/Channel 11 (KTWU)'),  # NE, http://ktwu.org
+        ('pioneer.org', 'Pioneer Public Television (KWCM)'),  # NE, http://www.pioneer.org
+        ('rmpbs.org', 'Rocky Mountain PBS (KRMA)'),  # NE, http://www.rmpbs.org
+        ('sdpb.org', 'SDPB Television (KUSD)'),  # NE, http://www.sdpb.org
+        ('wyomingpbs.org', 'Wyoming PBS (KCWC)'),  # NE, http://www.wyomingpbs.org
+        ('cpt12.org', 'Colorado Public Television / KBDI 12 (KBDI)'),  # NE, http://www.cpt12.org/
+        ('knpb.org', 'KNPB Channel 5 (KNPB)'),  # NV, http://www.knpb.org/
+        ('vegaspbs.org', 'Vegas PBS (KLVX)'),  # NV, http://vegaspbs.org/
+        ('azpbs.org', 'Arizona PBS (KAET)'),  # NV, http://www.azpbs.org
+        ('kued.org', 'KUED Channel 7 (KUED)'),  # NV, http://www.kued.org
+        ('pbssocal.org', 'PBS SoCal/KOCE (KOCE)'),  # NV, http://www.pbssocal.org/
+        ('kbyutv.org', 'KBYU-TV (KBYU)'),  # NV, http://www.kbyutv.org/
+        ('nhptv.org', 'NHPTV (WENH)'),  # NH, http://nhptv.org/
+        ('scetv.org', 'ETV (WRLK)'),  # NH, http://www.scetv.org
+        ('mountainlake.org', 'Mountain Lake PBS (WCFE)'),  # NH, http://www.mountainlake.org/
+        ('mpbn.net', 'MPBN (WCBB)'),  # NH, http://www.mpbn.net/
+        ('vpt.org', 'Vermont PBS (WETK)'),  # NH, http://www.vpt.org
+        ('wgbh.org', 'WGBH/Channel 2 (WGBH)'),  # NH, http://wgbh.org
+        ('wgby.org', 'WGBY (WGBY)'),  # NH, http://www.wgby.org
+        ('cptv.org', 'CONNECTICUT PUBLIC TELEVISION (WEDH)'),  # NJ, http://cptv.org
+        ('thirteen.org', 'Thirteen/WNET New York (WNET)'),  # NJ, http://www.thirteen.org
+        ('whyy.org', 'WHYY (WHYY)'),  # NJ, http://www.whyy.org
+        ('njtvonline.org', 'NJTV Public Media NJ (WNJT)'),  # NJ, http://www.njtvonline.org/
+        ('wlvt.org', 'PBS 39 (WLVT)'),  # NJ, http://www.wlvt.org/
+        ('wliw.org', 'WLIW21 (WLIW)'),  # NJ, http://www.wliw.org/
+        ('kenw.org', 'KENW-TV3 (KENW)'),  # NM, http://www.kenw.org
+        ('krwg.org', 'KRWG/Channel 22 (KRWG)'),  # NM, http://www.krwg.org
+        ('newmexicopbs.org', 'KNME-TV/Channel 5 (KNME)'),  # NM, http://www.newmexicopbs.org/
+        ('azpm.org', 'KUAT 6 (KUAT)'),  # NM, http://www.azpm.org/
+        ('panhandlepbs.org', 'KACV (KACV)'),  # NM, http://www.panhandlepbs.org/home/
+        ('kcostv.org', 'KCOS/Channel 13 (KCOS)'),  # NM, www.kcostv.org
+        ('kued.org', 'KUED Channel 7 (KUED)'),  # NM, http://www.kued.org
+        ('oeta.tv', 'OETA (KETA)'),  # NM, http://www.oeta.tv
+        ('rmpbs.org', 'Rocky Mountain PBS (KRMA)'),  # NM, http://www.rmpbs.org
+        ('cpt12.org', 'Colorado Public Television / KBDI 12 (KBDI)'),  # NM, http://www.cpt12.org/
+        ('kbyutv.org', 'KBYU-TV (KBYU)'),  # NM, http://www.kbyutv.org/
+        ('mountainlake.org', 'Mountain Lake PBS (WCFE)'),  # NY, http://www.mountainlake.org/
+        ('thirteen.org', 'Thirteen/WNET New York (WNET)'),  # NY, http://www.thirteen.org
+        ('wcny.org', 'WCNY/Channel 24 (WCNY)'),  # NY, http://www.wcny.org
+        ('wmht.org', 'WMHT Educational Telecommunications (WMHT)'),  # NY, http://www.wmht.org/home/
+        ('wned.org', 'WNED (WNED)'),  # NY, http://www.wned.org/
+        ('wpbstv.org', 'WPBS (WPBS)'),  # NY, http://www.wpbstv.org
+        ('wskg.org', 'WSKG Public TV (WSKG)'),  # NY, http://wskg.org
+        ('wxxi.org', 'WXXI (WXXI)'),  # NY, http://wxxi.org
+        ('cptv.org', 'CONNECTICUT PUBLIC TELEVISION (WEDH)'),  # NY, http://cptv.org
+        ('nhptv.org', 'NHPTV (WENH)'),  # NY, http://nhptv.org/
+        ('vpt.org', 'Vermont PBS (WETK)'),  # NY, http://www.vpt.org
+        ('wgbh.org', 'WGBH/Channel 2 (WGBH)'),  # NY, http://wgbh.org
+        ('wpsu.org', 'WPSU (WPSU)'),  # NY, http://www.wpsu.org
+        ('wqln.org', 'WQLN/Channel 54 (WQLN)'),  # NY, http://www.wqln.org
+        ('wvia.org', 'WVIA Public Media Studios (WVIA)'),  # NY, http://www.wvia.org/
+        ('wliw.org', 'WLIW21 (WLIW)'),  # NY, http://www.wliw.org/
+        ('njtvonline.org', 'NJTV Public Media NJ (WNJT)'),  # NY, http://www.njtvonline.org/
+        ('unctv.org', 'UNC-TV (WUNC)'),  # NC, http://www.unctv.org/
+        ('wtvi.org', 'WTVI (WTVI)'),  # NC, http://www.wtvi.org/
+        ('blueridgepbs.org', 'Blue Ridge PBS (WBRA)'),  # NC, http://www.blueridgepbs.org/
+        ('scetv.org', 'ETV (WRLK)'),  # NC, http://www.scetv.org
+        ('gpb.org', 'GPB/Georgia Public Broadcasting (WGTV)'),  # NC, http://www.gpb.org/
+        ('whro.org', 'WHRO (WHRO)'),  # NC, http://whro.org
+        ('wtcitv.org', 'WTCI (WTCI)'),  # NC, http://www.wtcitv.org
+        ('pba.org', 'WPBA/Channel 30 (WPBA)'),  # NC, http://pba.org/
+        ('prairiepublic.org', 'PRAIRIE PUBLIC (KFME)'),  # ND, http://www.prairiepublic.org/
+        ('sdpb.org', 'SDPB Television (KUSD)'),  # ND, http://www.sdpb.org
+        ('cetconnect.org', 'CET  (WCET)'),  # OH, http://www.cetconnect.org
+        ('thinktv.org', 'ThinkTVNetwork (WPTD)'),  # OH, http://www.thinktv.org
+        ('wbgu.org', 'WBGU-TV (WBGU)'),  # OH, http://wbgu.org
+        ('wgte.org', 'WGTE TV (WGTE)'),  # OH, http://www.wgte.org
+        ('WesternReservePublicMedia.org', 'Western Reserve PBS (WNEO)'),  # OH, http://www.WesternReservePublicMedia.org/
+        ('wosu.org', 'WOSU TV (WOSU)'),  # OH, http://wosu.org/
+        ('woub.org', 'WOUB/WOUC (WOUB)'),  # OH, http://woub.org/tv/index.php?section=5
+        ('wviz.org', 'WVIZ/PBS ideastream (WVIZ)'),  # OH, http://www.wviz.org/
+        ('dptv.org', 'WTVS Detroit Public TV (WTVS)'),  # OH, http://www.dptv.org/
+        ('ket.org', 'KET (WKLE)'),  # OH, http://www.ket.org/
+        ('wfwa.org', 'PBS39 Fort Wayne (WFWA)'),  # OH, http://wfwa.org/
+        ('wipb.org', 'WIPB-TV (WIPB)'),  # OH, http://wipb.org
+        ('wqed.org', 'WQED Multimedia (WQED)'),  # OH, http://www.wqed.org/
+        ('wqln.org', 'WQLN/Channel 54 (WQLN)'),  # OH, http://www.wqln.org
+        ('wvpublic.org', 'WVPB (WVPB)'),  # OH, http://wvpublic.org/
+        ('oeta.tv', 'OETA (KETA)'),  # OK, http://www.oeta.tv
+        ('aetn.org', 'AETN/ARKANSAS ETV NETWORK (KETS)'),  # OK, http://www.aetn.org/
+        ('panhandlepbs.org', 'KACV (KACV)'),  # OK, http://www.panhandlepbs.org/home/
+        ('kera.org', 'KERA 13 (KERA)'),  # OK, http://www.kera.org/
+        ('lpb.org', 'LPB/LOUISIANA PUBLIC BROADCASTING (WLPB)'),  # OK, http://www.lpb.org/
+        ('optv.org', 'Ozarks Public Television (KOZK)'),  # OK, http://www.optv.org/
+        ('opb.org', 'OPB (KOPB)'),  # OR, http://www.opb.org
+        ('soptv.org', 'SOPTV (KSYS)'),  # OR, http://www.soptv.org
+        ('idahoptv.org', 'Idaho Public Television (KAID)'),  # OR, http://idahoptv.org
+        ('kcts9.org', 'KCTS 9 (KCTS)'),  # OR, http://kcts9.org/
+        ('keet.org', 'KEET TV (KEET)'),  # OR, http://www.keet.org
+        ('ksps.org', 'KSPS (KSPS)'),  # OR, http://www.ksps.org/home/
+        ('kwsu.org', 'KWSU/Channel 10 & KTNW/Channel 31 (KWSU)'),  # OR, http://www.kwsu.org
+        ('whyy.org', 'WHYY (WHYY)'),  # PA, http://www.whyy.org
+        ('witf.org', 'witf (WITF)'),  # PA, http://www.witf.org
+        ('wpsu.org', 'WPSU (WPSU)'),  # PA, http://www.wpsu.org
+        ('wqed.org', 'WQED Multimedia (WQED)'),  # PA, http://www.wqed.org/
+        ('wqln.org', 'WQLN/Channel 54 (WQLN)'),  # PA, http://www.wqln.org
+        ('wvia.org', 'WVIA Public Media Studios (WVIA)'),  # PA, http://www.wvia.org/
+        ('cptv.org', 'CONNECTICUT PUBLIC TELEVISION (WEDH)'),  # PA, http://cptv.org
+        ('mpt.org', 'mpt/Maryland Public Television (WMPB)'),  # PA, http://www.mpt.org
+        ('thirteen.org', 'Thirteen/WNET New York (WNET)'),  # PA, http://www.thirteen.org
+        ('weta.org', 'WETA Television and Radio (WETA)'),  # PA, http://www.weta.org
+        ('wned.org', 'WNED (WNED)'),  # PA, http://www.wned.org/
+        ('WesternReservePublicMedia.org', 'Western Reserve PBS (WNEO)'),  # PA, http://www.WesternReservePublicMedia.org/
+        ('wskg.org', 'WSKG Public TV (WSKG)'),  # PA, http://wskg.org
+        ('wviz.org', 'WVIZ/PBS ideastream (WVIZ)'),  # PA, http://www.wviz.org/
+        ('wvpublic.org', 'WVPB (WVPB)'),  # PA, http://wvpublic.org/
+        ('wvpt.net', 'WVPT - Your Source for PBS and More! (WVPT)'),  # PA, http://www.wvpt.net
+        ('wlvt.org', 'PBS 39 (WLVT)'),  # PA, http://www.wlvt.org/
+        ('njtvonline.org', 'NJTV Public Media NJ (WNJT)'),  # PA, http://www.njtvonline.org/
+        ('whut.org', 'Howard University Television (WHUT)'),  # PA, http://www.whut.org
+        ('wliw.org', 'WLIW21 (WLIW)'),  # PA, http://www.wliw.org/
+        ('cptv.org', 'CONNECTICUT PUBLIC TELEVISION (WEDH)'),  # RI, http://cptv.org
+        ('nhptv.org', 'NHPTV (WENH)'),  # RI, http://nhptv.org/
+        ('wgbh.org', 'WGBH/Channel 2 (WGBH)'),  # RI, http://wgbh.org
+        ('ripbs.org', 'Rhode Island PBS (WSBE)'),  # RI, http://www.ripbs.org/home/
+        ('scetv.org', 'ETV (WRLK)'),  # SC, http://www.scetv.org
+        ('gpb.org', 'GPB/Georgia Public Broadcasting (WGTV)'),  # SC, http://www.gpb.org/
+        ('unctv.org', 'UNC-TV (WUNC)'),  # SC, http://www.unctv.org/
+        ('wtvi.org', 'WTVI (WTVI)'),  # SC, http://www.wtvi.org/
+        ('pba.org', 'WPBA/Channel 30 (WPBA)'),  # SC, http://pba.org/
+        ('sdpb.org', 'SDPB Television (KUSD)'),  # SD, http://www.sdpb.org
+        ('iptv.org', 'Iowa Public Television/IPTV (KDIN)'),  # SD, http://www.iptv.org/
+        ('prairiepublic.org', 'PRAIRIE PUBLIC (KFME)'),  # SD, http://www.prairiepublic.org/
+        ('netnebraska.org', 'NET1 (KUON)'),  # SD, http://netnebraska.org
+        ('pioneer.org', 'Pioneer Public Television (KWCM)'),  # SD, http://www.pioneer.org
+        ('easttennesseepbs.org', 'East Tennessee PBS (WSJK)'),  # TN, http://easttennesseepbs.org
+        ('wkno.org', 'WKNO/Channel 10 (WKNO)'),  # TN, http://www.wkno.org/
+        ('wnpt.org', 'Nashville Public Television (WNPT)'),  # TN, http://www.wnpt.org
+        ('wcte.org', 'WCTE-TV (WCTE)'),  # TN, http://www.wcte.org
+        ('wljt.org', 'WLJT, Channel 11 (WLJT)'),  # TN, http://wljt.org/
+        ('wtcitv.org', 'WTCI (WTCI)'),  # TN, http://www.wtcitv.org
+        ('aptv.org', 'APT - Alabama Public Television (WBIQ)'),  # TN, http://aptv.org/
+        ('blueridgepbs.org', 'Blue Ridge PBS (WBRA)'),  # TN, http://www.blueridgepbs.org/
+        ('gpb.org', 'GPB/Georgia Public Broadcasting (WGTV)'),  # TN, http://www.gpb.org/
+        ('lakelandptv.org', 'Lakeland Public Television (KAWE)'),  # TN, http://www.lakelandptv.org
+        ('ket.org', 'KET (WKLE)'),  # TN, http://www.ket.org/
+        ('mpbonline.org', 'Mississippi Public Broadcasting (WMPN)'),  # TN, http://www.mpbonline.org
+        ('unctv.org', 'UNC-TV (WUNC)'),  # TN, http://www.unctv.org/
+        ('WesternReservePublicMedia.org', 'Western Reserve PBS (WNEO)'),  # TN, http://www.WesternReservePublicMedia.org/
+        ('wsiu.org', 'WSIU Public Broadcasting (WSIU)'),  # TN, http://www.wsiu.org/
+        ('wviz.org', 'WVIZ/PBS ideastream (WVIZ)'),  # TN, http://www.wviz.org/
+        ('wkyupbs.org', 'WKYU-PBS (WKYU)'),  # TN, http://www.wkyupbs.org
+        ('pba.org', 'WPBA/Channel 30 (WPBA)'),  # TN, http://pba.org/
+        ('basinpbs.org', 'Basin PBS (KPBT)'),  # TX, http://www.basinpbs.org
+        ('houstonpublicmedia.org', 'KUHT / Channel 8 (KUHT)'),  # TX, http://www.houstonpublicmedia.org/
+        ('panhandlepbs.org', 'KACV (KACV)'),  # TX, http://www.panhandlepbs.org/home/
+        ('tamu.edu', 'KAMU - TV (KAMU)'),  # TX, http://KAMU.tamu.edu
+        ('kcostv.org', 'KCOS/Channel 13 (KCOS)'),  # TX, www.kcostv.org
+        ('kedt.org', 'KEDT/Channel 16 (KEDT)'),  # TX, http://www.kedt.org
+        ('kera.org', 'KERA 13 (KERA)'),  # TX, http://www.kera.org/
+        ('klrn.org', 'KLRN (KLRN)'),  # TX, http://www.klrn.org
+        ('klru.org', 'KLRU (KLRU)'),  # TX, http://www.klru.org
+        ('kmbh.org', 'KMBH-TV (KMBH)'),  # TX, http://www.kmbh.org
+        ('knct.org', 'KNCT (KNCT)'),  # TX, http://www.knct.org
+        ('ktxt.org', 'KTTZ-TV (KTXT)'),  # TX, http://www.ktxt.org
+        ('aetn.org', 'AETN/ARKANSAS ETV NETWORK (KETS)'),  # TX, http://www.aetn.org/
+        ('kenw.org', 'KENW-TV3 (KENW)'),  # TX, http://www.kenw.org
+        ('krwg.org', 'KRWG/Channel 22 (KRWG)'),  # TX, http://www.krwg.org
+        ('lpb.org', 'LPB/LOUISIANA PUBLIC BROADCASTING (WLPB)'),  # TX, http://www.lpb.org/
+        ('netnebraska.org', 'NET1 (KUON)'),  # TX, http://netnebraska.org
+        ('newmexicopbs.org', 'KNME-TV/Channel 5 (KNME)'),  # TX, http://www.newmexicopbs.org/
+        ('wnpt.org', 'Nashville Public Television (WNPT)'),  # TX, http://www.wnpt.org
+        ('oeta.tv', 'OETA (KETA)'),  # TX, http://www.oeta.tv
+        ('rmpbs.org', 'Rocky Mountain PBS (KRMA)'),  # TX, http://www.rmpbs.org
+        ('kued.org', 'KUED Channel 7 (KUED)'),  # UT, http://www.kued.org
+        ('idahoptv.org', 'Idaho Public Television (KAID)'),  # UT, http://idahoptv.org
+        ('vegaspbs.org', 'Vegas PBS (KLVX)'),  # UT, http://vegaspbs.org/
+        ('kbyutv.org', 'KBYU-TV (KBYU)'),  # UT, http://www.kbyutv.org/
+        ('vpt.org', 'Vermont PBS (WETK)'),  # VT, http://www.vpt.org
+        ('mountainlake.org', 'Mountain Lake PBS (WCFE)'),  # VT, http://www.mountainlake.org/
+        ('nhptv.org', 'NHPTV (WENH)'),  # VT, http://nhptv.org/
+        ('wgbh.org', 'WGBH/Channel 2 (WGBH)'),  # VT, http://wgbh.org
+        ('wgby.org', 'WGBY (WGBY)'),  # VT, http://www.wgby.org
+        ('wmht.org', 'WMHT Educational Telecommunications (WMHT)'),  # VT, http://www.wmht.org/home/
+        ('wtjx.org', 'WTJX Channel 12 (WTJX)'),  # VI, http://www.wtjx.org/
+        ('blueridgepbs.org', 'Blue Ridge PBS (WBRA)'),  # VA, http://www.blueridgepbs.org/
+        ('ideastations.org', 'WCVE PBS (WCVE)'),  # VA, http://ideastations.org/
+        ('whro.org', 'WHRO (WHRO)'),  # VA, http://whro.org
+        ('wvpt.net', 'WVPT - Your Source for PBS and More! (WVPT)'),  # VA, http://www.wvpt.net
+        ('easttennesseepbs.org', 'East Tennessee PBS (WSJK)'),  # VA, http://easttennesseepbs.org
+        ('ket.org', 'KET (WKLE)'),  # VA, http://www.ket.org/
+        ('mpt.org', 'mpt/Maryland Public Television (WMPB)'),  # VA, http://www.mpt.org
+        ('unctv.org', 'UNC-TV (WUNC)'),  # VA, http://www.unctv.org/
+        ('weta.org', 'WETA Television and Radio (WETA)'),  # VA, http://www.weta.org
+        ('whut.org', 'Howard University Television (WHUT)'),  # VA, http://www.whut.org
+        ('kcts9.org', 'KCTS 9 (KCTS)'),  # WA, http://kcts9.org/
+        ('ksps.org', 'KSPS (KSPS)'),  # WA, http://www.ksps.org/home/
+        ('idahoptv.org', 'Idaho Public Television (KAID)'),  # WA, http://idahoptv.org
+        ('opb.org', 'OPB (KOPB)'),  # WA, http://www.opb.org
+        ('kbtc.org', 'KBTC Public Television (KBTC)'),  # WA, http://kbtc.org
+        ('kwsu.org', 'KWSU/Channel 10 & KTNW/Channel 31 (KWSU)'),  # WA, http://www.kwsu.org
+        ('wvpublic.org', 'WVPB (WVPB)'),  # WV, http://wvpublic.org/
+        ('blueridgepbs.org', 'Blue Ridge PBS (WBRA)'),  # WV, http://www.blueridgepbs.org/
+        ('ket.org', 'KET (WKLE)'),  # WV, http://www.ket.org/
+        ('mpt.org', 'mpt/Maryland Public Television (WMPB)'),  # WV, http://www.mpt.org
+        ('weta.org', 'WETA Television and Radio (WETA)'),  # WV, http://www.weta.org
+        ('WesternReservePublicMedia.org', 'Western Reserve PBS (WNEO)'),  # WV, http://www.WesternReservePublicMedia.org/
+        ('wosu.org', 'WOSU TV (WOSU)'),  # WV, http://wosu.org/
+        ('woub.org', 'WOUB/WOUC (WOUB)'),  # WV, http://woub.org/tv/index.php?section=5
+        ('wqed.org', 'WQED Multimedia (WQED)'),  # WV, http://www.wqed.org/
+        ('wvpt.net', 'WVPT - Your Source for PBS and More! (WVPT)'),  # WV, http://www.wvpt.net
+        ('whut.org', 'Howard University Television (WHUT)'),  # WV, http://www.whut.org
+        ('mptv.org', 'Milwaukee Public Television (WMVS)'),  # WI, http://www.mptv.org
+        ('wpt.org', 'WPT (WPNE)'),  # WI, http://www.wpt.org/
+        ('iptv.org', 'Iowa Public Television/IPTV (KDIN)'),  # WI, http://www.iptv.org/
+        ('lakelandptv.org', 'Lakeland Public Television (KAWE)'),  # WI, http://www.lakelandptv.org
+        ('wdse.org', 'WDSE - WRPT (WDSE)'),  # WI, http://www.wdse.org/
+        ('tpt.org', 'TPT (KTCA)'),  # WI, http://www.tpt.org
+        ('nmu.edu', 'WNMU-TV Public TV 13 (WNMU)'),  # WI, http://wnmutv.nmu.edu
+        ('wttw.com', 'WTTW11 (WTTW)'),  # WI, http://www.wttw.com/
+        ('ksmq.org', 'KSMQ (KSMQ)'),  # WI, http://www.ksmq.org/
+        ('wycc.org', 'WYCC PBS Chicago (WYCC)'),  # WI, http://www.wycc.org
+        ('wyomingpbs.org', 'Wyoming PBS (KCWC)'),  # WY, http://www.wyomingpbs.org
+        ('idahoptv.org', 'Idaho Public Television (KAID)'),  # WY, http://idahoptv.org
+        ('kued.org', 'KUED Channel 7 (KUED)'),  # WY, http://www.kued.org
+        ('montanapbs.org', 'MontanaPBS (KUSM)'),  # WY, http://montanapbs.org
+        ('netnebraska.org', 'NET1 (KUON)'),  # WY, http://netnebraska.org
+        ('rmpbs.org', 'Rocky Mountain PBS (KRMA)'),  # WY, http://www.rmpbs.org
+        ('sdpb.org', 'SDPB Television (KUSD)'),  # WY, http://www.sdpb.org
+        ('cpt12.org', 'Colorado Public Television / KBDI 12 (KBDI)'),  # WY, http://www.cpt12.org/
+        ('kbyutv.org', 'KBYU-TV (KBYU)'),  # WY, http://www.kbyutv.org/
+    )
+
+    IE_NAME = 'pbs'
+    IE_DESC = 'Public Broadcasting Service (PBS) and member stations: %s' % ', '.join(list(zip(*_STATIONS))[1])
+
     _VALID_URL = r'''(?x)https?://
         (?:
            # Direct video URL
-           (?:video\.pbs|watch\.knpb)\.org/(?:viralplayer|video)/(?P<id>[0-9]+)/? |
+           (?:videos?|watch)\.(?:%s)/(?:viralplayer|video)/(?P<id>[0-9]+)/? |
            # Article with embedded player (or direct video)
            (?:www\.)?pbs\.org/(?:[^/]+/){2,5}(?P<presumptive_id>[^/]+?)(?:\.html)?/?(?:$|[?\#]) |
            # Player
            (?:video|player)\.pbs\.org/(?:widget/)?partnerplayer/(?P<player_id>[^/]+)/
         )
-    '''
+    ''' % '|'.join(list(zip(*_STATIONS))[0])
 
     _TESTS = [
         {

From 47f48f5d851cf2f0acd22ad3fb00a2b69378c7bc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 8 Dec 2015 21:12:13 +0100
Subject: [PATCH 3302/6219] [test/test_all_urls] Update pbs extractor name

It's in lowercase now (since e15e2ef7a00362fc1e89d9959241efad46325c54).
---
 test/test_all_urls.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/test_all_urls.py b/test/test_all_urls.py
index a9db42b30..a0c11e6c1 100644
--- a/test/test_all_urls.py
+++ b/test/test_all_urls.py
@@ -121,8 +121,8 @@ class TestAllURLsMatching(unittest.TestCase):
 
     def test_pbs(self):
         # https://github.com/rg3/youtube-dl/issues/2350
-        self.assertMatch('http://video.pbs.org/viralplayer/2365173446/', ['PBS'])
-        self.assertMatch('http://video.pbs.org/widget/partnerplayer/980042464/', ['PBS'])
+        self.assertMatch('http://video.pbs.org/viralplayer/2365173446/', ['pbs'])
+        self.assertMatch('http://video.pbs.org/widget/partnerplayer/980042464/', ['pbs'])
 
     def test_yahoo_https(self):
         # https://github.com/rg3/youtube-dl/issues/2701

From 41c3b34b1f1f94c723f3af760ccddc4d7119464f Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 9 Dec 2015 10:50:11 +0100
Subject: [PATCH 3303/6219] [vgtv] add sortcut expressions to use the extractor

---
 youtube_dl/extractor/vgtv.py | 68 ++++++++++++++++++------------------
 1 file changed, 34 insertions(+), 34 deletions(-)

diff --git a/youtube_dl/extractor/vgtv.py b/youtube_dl/extractor/vgtv.py
index 8ce3d8b8f..1fba37578 100644
--- a/youtube_dl/extractor/vgtv.py
+++ b/youtube_dl/extractor/vgtv.py
@@ -12,21 +12,39 @@ from ..utils import (
 
 class VGTVIE(InfoExtractor):
     IE_DESC = 'VGTV, BTTV, FTV, Aftenposten and Aftonbladet'
+
+    _HOST_TO_APPNAME = {
+        'vgtv.no': 'vgtv',
+        'bt.no/tv': 'bttv',
+        'aftenbladet.no/tv': 'satv',
+        'fvn.no/fvntv': 'fvntv',
+        'aftenposten.no/webtv': 'aptv',
+    }
+
+    _APP_NAME_TO_VENDOR = {
+        'vgtv': 'vgtv',
+        'bttv': 'bt',
+        'satv': 'sa',
+        'fvntv': 'fvn',
+        'aptv': 'ap',
+    }
+
     _VALID_URL = r'''(?x)
-                    https?://(?:www\.)?
+                    (?:https?://(?:www\.)?
                     (?P<host>
-                        vgtv.no|
-                        (?:bt|aftenbladet).no/tv|
-                        fvn.no/fvntv|
-                        aftenposten.no/webtv
+                        %s
                     )
                     /
                     (?:
                         \#!/(?:video|live)/|
                         embed?.*id=
-                    )
+                    )|
+                    (?P<appname>
+                        %s
+                    ):)
                     (?P<id>\d+)
-                    '''
+                    ''' % ('|'.join(_HOST_TO_APPNAME.keys()), '|'.join(_APP_NAME_TO_VENDOR.keys()))
+
     _TESTS = [
         {
             # streamType: vod
@@ -102,37 +120,17 @@ class VGTVIE(InfoExtractor):
             'only_matching': True,
         },
     ]
-    _HOST_WEBSITES = {
-        'vgtv.no': {
-            'vendor': 'vgtv',
-            'appname': 'vgtv',
-        },
-        'bt.no/tv': {
-            'vendor': 'bt',
-            'appname': 'bttv',
-        },
-        'aftenbladet.no/tv': {
-            'vendor': 'sa',
-            'appname': 'satv',
-        },
-        'fvn.no/fvntv': {
-            'vendor': 'fvn',
-            'appname': 'fvntv',
-        },
-        'aftenposten.no/webtv': {
-            'vendor': 'ap',
-            'appname': 'aptv',
-        },
-    }
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
         host = mobj.group('host')
+        appname = self._HOST_TO_APPNAME[host] if host else mobj.group('appname')
+        vendor = self._APP_NAME_TO_VENDOR[appname]
 
         data = self._download_json(
             'http://svp.vg.no/svp/api/v1/%s/assets/%s?appName=%s-website'
-            % (self._HOST_WEBSITES[host]['vendor'], video_id, self._HOST_WEBSITES[host]['appname']),
+            % (vendor, video_id, appname),
             video_id, 'Downloading media JSON')
 
         if data.get('status') == 'inactive':
@@ -146,14 +144,16 @@ class VGTVIE(InfoExtractor):
 
         hls_url = streams.get('hls')
         if hls_url:
-            m3u8_formats = self._extract_m3u8_formats(hls_url, video_id, 'mp4', m3u8_id='hls', fatal=False)
+            m3u8_formats = self._extract_m3u8_formats(
+                hls_url, video_id, 'mp4', m3u8_id='hls', fatal=False)
             if m3u8_formats:
                 formats.extend(m3u8_formats)
 
         hds_url = streams.get('hds')
         # wasLive hds are always 404
         if hds_url and stream_type != 'wasLive':
-            f4m_formats = self._extract_f4m_formats(hds_url + '?hdcore=3.2.0&plugin=aasp-3.2.0.77.18', video_id, f4m_id='hds', fatal=False)
+            f4m_formats = self._extract_f4m_formats(
+                hds_url + '?hdcore=3.2.0&plugin=aasp-3.2.0.77.18', video_id, f4m_id='hds', fatal=False)
             if f4m_formats:
                 formats.extend(f4m_formats)
 
@@ -216,7 +216,7 @@ class BTArticleIE(InfoExtractor):
         webpage = self._download_webpage(url, self._match_id(url))
         video_id = self._search_regex(
             r'SVP\.Player\.load\(\s*(\d+)', webpage, 'video id')
-        return self.url_result('http://bt.no/tv/embed?id=%s' % video_id, 'VGTV')
+        return self.url_result('bttv:%s' % video_id, 'VGTV')
 
 
 class BTVestlendingenIE(InfoExtractor):
@@ -237,4 +237,4 @@ class BTVestlendingenIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        return self.url_result('http://bt.no/tv/embed?id=%s' % self._match_id(url), 'VGTV')
+        return self.url_result('bttv:%s' % self._match_id(url), 'VGTV')

From 86e8c89488c4d41009aa8f1044681c3f217ec1b8 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 9 Dec 2015 15:32:26 +0100
Subject: [PATCH 3304/6219] release 2015.12.09

---
 CONTRIBUTING.md        | 16 +++++++++++++++-
 docs/supportedsites.md |  2 +-
 youtube_dl/version.py  |  2 +-
 3 files changed, 17 insertions(+), 3 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 09ce98ca2..28aae94f8 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -1,4 +1,18 @@
-**Please include the full output of youtube-dl when run with `-v`**.
+**Please include the full output of youtube-dl when run with `-v`**, i.e. add `-v` flag to your command line, copy the **whole** output and post it in the issue body wrapped in \`\`\` for better formatting. It should look similar to this:
+```
+$ youtube-dl -v http://www.youtube.com/watch?v=BaW_jenozKcj
+[debug] System config: []
+[debug] User config: []
+[debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
+[debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
+[debug] youtube-dl version 2015.12.06
+[debug] Git HEAD: 135392e
+[debug] Python version 2.6.6 - Windows-2003Server-5.2.3790-SP2
+[debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
+[debug] Proxy map: {}
+...
+```
+**Do not post screenshots of verbose log only plain text is acceptable.**
 
 The output (including the first lines) contains important debugging information. Issues without the full output are often not reproducible and therefore do not get solved in short order, if ever.
 
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index bf26fecd7..4f19cb6f5 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -400,7 +400,7 @@
  - **orf:tvthek**: ORF TVthek
  - **parliamentlive.tv**: UK parliament videos
  - **Patreon**
- - **PBS**
+ - **pbs**: Public Broadcasting Service (PBS) and member stations: PBS: Public Broadcasting Service, APT - Alabama Public Television (WBIQ), GPB/Georgia Public Broadcasting (WGTV), Mississippi Public Broadcasting (WMPN), Nashville Public Television (WNPT), WFSU-TV (WFSU), WSRE (WSRE), WTCI (WTCI), WPBA/Channel 30 (WPBA), Alaska Public Media (KAKM), KUAC (KUAC), 360 North (KTOO), KUAT 6 (KUAT), Arizona PBS (KAET), KNME-TV/Channel 5 (KNME), Vegas PBS (KLVX), AETN/ARKANSAS ETV NETWORK (KETS), KET (WKLE), WKNO/Channel 10 (WKNO), LPB/LOUISIANA PUBLIC BROADCASTING (WLPB), Mississippi Public Broadcasting (WMPN), OETA (KETA), Ozarks Public Television (KOZK), WSIU Public Broadcasting (WSIU), KEET TV (KEET), KIXE/Channel 9 (KIXE), KPBS San Diego (KPBS), KQED (KQED), KVIE Public Television (KVIE), PBS SoCal/KOCE (KOCE), ValleyPBS (KVPT), APT - Alabama Public Television (WBIQ), CONNECTICUT PUBLIC TELEVISION (WEDH), Arizona PBS (KAET), KNPB Channel 5 (KNPB), SOPTV (KSYS), KLCS/Channel 58 (KLCS), KRCB Television & Radio (KRCB), KVCR TV/DT/FM :: Vision for the Future (KVCR), Rocky Mountain PBS (KRMA), KENW-TV3 (KENW), KUED Channel 7 (KUED), KNME-TV/Channel 5 (KNME), Wyoming PBS (KCWC), Colorado Public Television / KBDI 12 (KBDI), KBYU-TV (KBYU), CONNECTICUT PUBLIC TELEVISION (WEDH), Thirteen/WNET New York (WNET), WGBH/Channel 2 (WGBH), WGBY (WGBY), NJTV Public Media NJ (WNJT), Rhode Island PBS (WSBE), WLIW21 (WLIW), mpt/Maryland Public Television (WMPB), WETA Television and Radio (WETA), WHYY (WHYY), NJTV Public Media NJ (WNJT), PBS 39 (WLVT), WLIW21 (WLIW), WETA Television and Radio (WETA), mpt/Maryland Public Television (WMPB), WVPT - Your Source for PBS and More! (WVPT), Howard University Television (WHUT), WEDU PBS (WEDU), WFSU-TV (WFSU), WGCU Public Media (WGCU), WJCT Public Broadcasting (WJCT), WPBT2 (WPBT), WSRE (WSRE), WUCF TV (WUCF), WUFT/Channel 5 (WUFT), WXEL/Channel 42 (WXEL), APT - Alabama Public Television (WBIQ), GPB/Georgia Public Broadcasting (WGTV), WLRN/Channel 17 (WLRN), WUSF Public Broadcasting (WUSF), NJTV Public Media NJ (WNJT), GPB/Georgia Public Broadcasting (WGTV), APT - Alabama Public Television (WBIQ), ETV (WRLK), UNC-TV (WUNC), WFSU-TV (WFSU), WJCT Public Broadcasting (WJCT), WTCI (WTCI), WPBA/Channel 30 (WPBA), PBS Guam (KGTF), PBS Hawaii - Oceanic Cable Channel 10 (KHET), Idaho Public Television (KAID), KSPS (KSPS), KUED Channel 7 (KUED), OPB (KOPB), SOPTV (KSYS), Wyoming PBS (KCWC), Colorado Public Television / KBDI 12 (KBDI), KBYU-TV (KBYU), KWSU/Channel 10 & KTNW/Channel 31 (KWSU), WILL-TV (WILL), Network Knowledge - WSEC/Springfield (WSEC), WSIU Public Broadcasting (WSIU), WTTW11 (WTTW), WTVP & WTVP.org, Public Media for Central Illinois (WTVP), Iowa Public Television/IPTV (KDIN), KET (WKLE), Nine Network (KETC), PBS39 Fort Wayne (WFWA), WFYI Indianapolis (WFYI), Milwaukee Public Television (WMVS), WNIN (WNIN), WNIT Public Television (WNIT), WPT (WPNE), WVUT/Channel 22 (WVUT), WEIU/Channel 51 (WEIU), WQPT-TV (WQPT), WYCC PBS Chicago (WYCC), Lakeshore Public Television (WYIN), PBS39 Fort Wayne (WFWA), WFYI Indianapolis (WFYI), WIPB-TV (WIPB), WNIN (WNIN), WNIT Public Television (WNIT), WTIU (WTIU), WVUT/Channel 22 (WVUT), CET  (WCET), KET (WKLE), ThinkTVNetwork (WPTD), WBGU-TV (WBGU), WGVU TV (WGVU), WILL-TV (WILL), Network Knowledge - WSEC/Springfield (WSEC), WSIU Public Broadcasting (WSIU), WTTW11 (WTTW), Lakeshore Public Television (WYIN), WEIU/Channel 51 (WEIU), WYCC PBS Chicago (WYCC), Iowa Public Television/IPTV (KDIN), NET1 (KUON), Pioneer Public Television (KWCM), SDPB Television (KUSD), TPT (KTCA), Network Knowledge - WSEC/Springfield (WSEC), WPT (WPNE), KSMQ (KSMQ), WQPT-TV (WQPT), KPTS/Channel 8 (KPTS), KTWU/Channel 11 (KTWU), Smoky Hills Public Television (KOOD), KCPT Kansas City Public Television (KCPT), NET1 (KUON), OETA (KETA), Ozarks Public Television (KOZK), Rocky Mountain PBS (KRMA), Colorado Public Television / KBDI 12 (KBDI), KET (WKLE), Blue Ridge PBS (WBRA), CET  (WCET), East Tennessee PBS (WSJK), Nashville Public Television (WNPT), ThinkTVNetwork (WPTD), WCTE-TV (WCTE), WLJT, Channel 11 (WLJT), WNIN (WNIN), WOSU TV (WOSU), WOUB/WOUC (WOUB), WSIU Public Broadcasting (WSIU), WVPB (WVPB), WKYU-PBS (WKYU), LPB/LOUISIANA PUBLIC BROADCASTING (WLPB), WYES-TV/New Orleans (WYES), AETN/ARKANSAS ETV NETWORK (KETS), KERA 13 (KERA), Mississippi Public Broadcasting (WMPN), MPBN (WCBB), Mountain Lake PBS (WCFE), NHPTV (WENH), Vermont PBS (WETK), WGBH/Channel 2 (WGBH), mpt/Maryland Public Television (WMPB), Thirteen/WNET New York (WNET), WETA Television and Radio (WETA), WHYY (WHYY), witf (WITF), WQED Multimedia (WQED), WVPB (WVPB), WVPT - Your Source for PBS and More! (WVPT), Howard University Television (WHUT), WLIW21 (WLIW), WGBH/Channel 2 (WGBH), WGBY (WGBY), CONNECTICUT PUBLIC TELEVISION (WEDH), NHPTV (WENH), Vermont PBS (WETK), WMHT Educational Telecommunications (WMHT), Rhode Island PBS (WSBE), Q-TV (WDCQ), WTVS Detroit Public TV (WTVS), CMU Public Television (WCMU), WGVU TV (WGVU), WKAR-TV (WKAR), WNMU-TV Public TV 13 (WNMU), WDSE - WRPT (WDSE), WBGU-TV (WBGU), WGTE TV (WGTE), WNIT Public Television (WNIT), WPT (WPNE), WTTW11 (WTTW), WYCC PBS Chicago (WYCC), Lakeland Public Television (KAWE), WDSE - WRPT (WDSE), Pioneer Public Television (KWCM), TPT (KTCA), Iowa Public Television/IPTV (KDIN), PRAIRIE PUBLIC (KFME), Nine Network (KETC), SDPB Television (KUSD), WPT (WPNE), KSMQ (KSMQ), Mississippi Public Broadcasting (WMPN), AETN/ARKANSAS ETV NETWORK (KETS), APT - Alabama Public Television (WBIQ), WKNO/Channel 10 (WKNO), LPB/LOUISIANA PUBLIC BROADCASTING (WLPB), WSRE (WSRE), WYES-TV/New Orleans (WYES), KCPT Kansas City Public Television (KCPT), KMOS-TV - Channels 6.1, 6.2 and 6.3 (KMOS), Nine Network (KETC), Ozarks Public Television (KOZK), AETN/ARKANSAS ETV NETWORK (KETS), WTVS Detroit Public TV (WTVS), Iowa Public Television/IPTV (KDIN), KET (WKLE), WKNO/Channel 10 (WKNO), KTWU/Channel 11 (KTWU), NET1 (KUON), Rocky Mountain PBS (KRMA), Network Knowledge - WSEC/Springfield (WSEC), WSIU Public Broadcasting (WSIU), MontanaPBS (KUSM), Arizona PBS (KAET), Idaho Public Television (KAID), PRAIRIE PUBLIC (KFME), KSPS (KSPS), Rocky Mountain PBS (KRMA), SDPB Television (KUSD), NET1 (KUON), Iowa Public Television/IPTV (KDIN), KTWU/Channel 11 (KTWU), Pioneer Public Television (KWCM), Rocky Mountain PBS (KRMA), SDPB Television (KUSD), Wyoming PBS (KCWC), Colorado Public Television / KBDI 12 (KBDI), KNPB Channel 5 (KNPB), Vegas PBS (KLVX), Arizona PBS (KAET), KUED Channel 7 (KUED), PBS SoCal/KOCE (KOCE), KBYU-TV (KBYU), NHPTV (WENH), ETV (WRLK), Mountain Lake PBS (WCFE), MPBN (WCBB), Vermont PBS (WETK), WGBH/Channel 2 (WGBH), WGBY (WGBY), CONNECTICUT PUBLIC TELEVISION (WEDH), Thirteen/WNET New York (WNET), WHYY (WHYY), NJTV Public Media NJ (WNJT), PBS 39 (WLVT), WLIW21 (WLIW), KENW-TV3 (KENW), KRWG/Channel 22 (KRWG), KNME-TV/Channel 5 (KNME), KUAT 6 (KUAT), KACV (KACV), KCOS/Channel 13 (KCOS), KUED Channel 7 (KUED), OETA (KETA), Rocky Mountain PBS (KRMA), Colorado Public Television / KBDI 12 (KBDI), KBYU-TV (KBYU), Mountain Lake PBS (WCFE), Thirteen/WNET New York (WNET), WCNY/Channel 24 (WCNY), WMHT Educational Telecommunications (WMHT), WNED (WNED), WPBS (WPBS), WSKG Public TV (WSKG), WXXI (WXXI), CONNECTICUT PUBLIC TELEVISION (WEDH), NHPTV (WENH), Vermont PBS (WETK), WGBH/Channel 2 (WGBH), WPSU (WPSU), WQLN/Channel 54 (WQLN), WVIA Public Media Studios (WVIA), WLIW21 (WLIW), NJTV Public Media NJ (WNJT), UNC-TV (WUNC), WTVI (WTVI), Blue Ridge PBS (WBRA), ETV (WRLK), GPB/Georgia Public Broadcasting (WGTV), WHRO (WHRO), WTCI (WTCI), WPBA/Channel 30 (WPBA), PRAIRIE PUBLIC (KFME), SDPB Television (KUSD), CET  (WCET), ThinkTVNetwork (WPTD), WBGU-TV (WBGU), WGTE TV (WGTE), Western Reserve PBS (WNEO), WOSU TV (WOSU), WOUB/WOUC (WOUB), WVIZ/PBS ideastream (WVIZ), WTVS Detroit Public TV (WTVS), KET (WKLE), PBS39 Fort Wayne (WFWA), WIPB-TV (WIPB), WQED Multimedia (WQED), WQLN/Channel 54 (WQLN), WVPB (WVPB), OETA (KETA), AETN/ARKANSAS ETV NETWORK (KETS), KACV (KACV), KERA 13 (KERA), LPB/LOUISIANA PUBLIC BROADCASTING (WLPB), Ozarks Public Television (KOZK), OPB (KOPB), SOPTV (KSYS), Idaho Public Television (KAID), KCTS 9 (KCTS), KEET TV (KEET), KSPS (KSPS), KWSU/Channel 10 & KTNW/Channel 31 (KWSU), WHYY (WHYY), witf (WITF), WPSU (WPSU), WQED Multimedia (WQED), WQLN/Channel 54 (WQLN), WVIA Public Media Studios (WVIA), CONNECTICUT PUBLIC TELEVISION (WEDH), mpt/Maryland Public Television (WMPB), Thirteen/WNET New York (WNET), WETA Television and Radio (WETA), WNED (WNED), Western Reserve PBS (WNEO), WSKG Public TV (WSKG), WVIZ/PBS ideastream (WVIZ), WVPB (WVPB), WVPT - Your Source for PBS and More! (WVPT), PBS 39 (WLVT), NJTV Public Media NJ (WNJT), Howard University Television (WHUT), WLIW21 (WLIW), CONNECTICUT PUBLIC TELEVISION (WEDH), NHPTV (WENH), WGBH/Channel 2 (WGBH), Rhode Island PBS (WSBE), ETV (WRLK), GPB/Georgia Public Broadcasting (WGTV), UNC-TV (WUNC), WTVI (WTVI), WPBA/Channel 30 (WPBA), SDPB Television (KUSD), Iowa Public Television/IPTV (KDIN), PRAIRIE PUBLIC (KFME), NET1 (KUON), Pioneer Public Television (KWCM), East Tennessee PBS (WSJK), WKNO/Channel 10 (WKNO), Nashville Public Television (WNPT), WCTE-TV (WCTE), WLJT, Channel 11 (WLJT), WTCI (WTCI), APT - Alabama Public Television (WBIQ), Blue Ridge PBS (WBRA), GPB/Georgia Public Broadcasting (WGTV), Lakeland Public Television (KAWE), KET (WKLE), Mississippi Public Broadcasting (WMPN), UNC-TV (WUNC), Western Reserve PBS (WNEO), WSIU Public Broadcasting (WSIU), WVIZ/PBS ideastream (WVIZ), WKYU-PBS (WKYU), WPBA/Channel 30 (WPBA), Basin PBS (KPBT), KUHT / Channel 8 (KUHT), KACV (KACV), KAMU - TV (KAMU), KCOS/Channel 13 (KCOS), KEDT/Channel 16 (KEDT), KERA 13 (KERA), KLRN (KLRN), KLRU (KLRU), KMBH-TV (KMBH), KNCT (KNCT), KTTZ-TV (KTXT), AETN/ARKANSAS ETV NETWORK (KETS), KENW-TV3 (KENW), KRWG/Channel 22 (KRWG), LPB/LOUISIANA PUBLIC BROADCASTING (WLPB), NET1 (KUON), KNME-TV/Channel 5 (KNME), Nashville Public Television (WNPT), OETA (KETA), Rocky Mountain PBS (KRMA), KUED Channel 7 (KUED), Idaho Public Television (KAID), Vegas PBS (KLVX), KBYU-TV (KBYU), Vermont PBS (WETK), Mountain Lake PBS (WCFE), NHPTV (WENH), WGBH/Channel 2 (WGBH), WGBY (WGBY), WMHT Educational Telecommunications (WMHT), WTJX Channel 12 (WTJX), Blue Ridge PBS (WBRA), WCVE PBS (WCVE), WHRO (WHRO), WVPT - Your Source for PBS and More! (WVPT), East Tennessee PBS (WSJK), KET (WKLE), mpt/Maryland Public Television (WMPB), UNC-TV (WUNC), WETA Television and Radio (WETA), Howard University Television (WHUT), KCTS 9 (KCTS), KSPS (KSPS), Idaho Public Television (KAID), OPB (KOPB), KBTC Public Television (KBTC), KWSU/Channel 10 & KTNW/Channel 31 (KWSU), WVPB (WVPB), Blue Ridge PBS (WBRA), KET (WKLE), mpt/Maryland Public Television (WMPB), WETA Television and Radio (WETA), Western Reserve PBS (WNEO), WOSU TV (WOSU), WOUB/WOUC (WOUB), WQED Multimedia (WQED), WVPT - Your Source for PBS and More! (WVPT), Howard University Television (WHUT), Milwaukee Public Television (WMVS), WPT (WPNE), Iowa Public Television/IPTV (KDIN), Lakeland Public Television (KAWE), WDSE - WRPT (WDSE), TPT (KTCA), WNMU-TV Public TV 13 (WNMU), WTTW11 (WTTW), KSMQ (KSMQ), WYCC PBS Chicago (WYCC), Wyoming PBS (KCWC), Idaho Public Television (KAID), KUED Channel 7 (KUED), MontanaPBS (KUSM), NET1 (KUON), Rocky Mountain PBS (KRMA), SDPB Television (KUSD), Colorado Public Television / KBDI 12 (KBDI), KBYU-TV (KBYU)
  - **Periscope**: Periscope
  - **PhilharmonieDeParis**: Philharmonie de Paris
  - **Phoenix**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index a4e9d7072..0f5571006 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.12.06'
+__version__ = '2015.12.09'

From b51b10804556659af181541e6cd0d46347a8859f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 9 Dec 2015 21:23:19 +0600
Subject: [PATCH 3305/6219] [pbs] Clean up stations list from duplicates

---
 youtube_dl/extractor/pbs.py | 625 +++++++++---------------------------
 1 file changed, 158 insertions(+), 467 deletions(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index decc376a2..6aae7f028 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -16,473 +16,164 @@ from ..utils import (
 
 class PBSIE(InfoExtractor):
     _STATIONS = (
-        ('pbs.org', 'PBS: Public Broadcasting Service'),  # http://www.pbs.org/
-        ('aptv.org', 'APT - Alabama Public Television (WBIQ)'),  # AL, http://aptv.org/
-        ('gpb.org', 'GPB/Georgia Public Broadcasting (WGTV)'),  # AL, http://www.gpb.org/
-        ('mpbonline.org', 'Mississippi Public Broadcasting (WMPN)'),  # AL, http://www.mpbonline.org
-        ('wnpt.org', 'Nashville Public Television (WNPT)'),  # AL, http://www.wnpt.org
-        ('wfsu.org', 'WFSU-TV (WFSU)'),  # AL, http://wfsu.org/
-        ('wsre.org', 'WSRE (WSRE)'),  # AL, http://www.wsre.org
-        ('wtcitv.org', 'WTCI (WTCI)'),  # AL, http://www.wtcitv.org
-        ('pba.org', 'WPBA/Channel 30 (WPBA)'),  # AL, http://pba.org/
-        ('alaskapublic.org', 'Alaska Public Media (KAKM)'),  # AK, http://alaskapublic.org/kakm
-        ('kuac.org', 'KUAC (KUAC)'),  # AK, http://kuac.org/kuac-tv/
-        ('ktoo.org', '360 North (KTOO)'),  # AK, http://www.ktoo.org/
-        ('azpm.org', 'KUAT 6 (KUAT)'),  # AZ, http://www.azpm.org/
-        ('azpbs.org', 'Arizona PBS (KAET)'),  # AZ, http://www.azpbs.org
-        ('newmexicopbs.org', 'KNME-TV/Channel 5 (KNME)'),  # AZ, http://www.newmexicopbs.org/
-        ('vegaspbs.org', 'Vegas PBS (KLVX)'),  # AZ, http://vegaspbs.org/
-        ('aetn.org', 'AETN/ARKANSAS ETV NETWORK (KETS)'),  # AR, http://www.aetn.org/
-        ('ket.org', 'KET (WKLE)'),  # AR, http://www.ket.org/
-        ('wkno.org', 'WKNO/Channel 10 (WKNO)'),  # AR, http://www.wkno.org/
-        ('lpb.org', 'LPB/LOUISIANA PUBLIC BROADCASTING (WLPB)'),  # AR, http://www.lpb.org/
-        ('mpbonline.org', 'Mississippi Public Broadcasting (WMPN)'),  # AR, http://www.mpbonline.org
-        ('oeta.tv', 'OETA (KETA)'),  # AR, http://www.oeta.tv
-        ('optv.org', 'Ozarks Public Television (KOZK)'),  # AR, http://www.optv.org/
-        ('wsiu.org', 'WSIU Public Broadcasting (WSIU)'),  # AR, http://www.wsiu.org/
-        ('keet.org', 'KEET TV (KEET)'),  # CA, http://www.keet.org
-        ('kixe.org', 'KIXE/Channel 9 (KIXE)'),  # CA, http://kixe.org/
-        ('kpbs.org', 'KPBS San Diego (KPBS)'),  # CA, http://www.kpbs.org/
-        ('kqed.org', 'KQED (KQED)'),  # CA, http://www.kqed.org
-        ('kvie.org', 'KVIE Public Television (KVIE)'),  # CA, http://www.kvie.org
-        ('pbssocal.org', 'PBS SoCal/KOCE (KOCE)'),  # CA, http://www.pbssocal.org/
-        ('valleypbs.org', 'ValleyPBS (KVPT)'),  # CA, http://www.valleypbs.org/
-        ('aptv.org', 'APT - Alabama Public Television (WBIQ)'),  # CA, http://aptv.org/
-        ('cptv.org', 'CONNECTICUT PUBLIC TELEVISION (WEDH)'),  # CA, http://cptv.org
-        ('azpbs.org', 'Arizona PBS (KAET)'),  # CA, http://www.azpbs.org
-        ('knpb.org', 'KNPB Channel 5 (KNPB)'),  # CA, http://www.knpb.org/
-        ('soptv.org', 'SOPTV (KSYS)'),  # CA, http://www.soptv.org
-        ('klcs.org', 'KLCS/Channel 58 (KLCS)'),  # CA, http://www.klcs.org
-        ('krcb.org', 'KRCB Television & Radio (KRCB)'),  # CA, http://www.krcb.org
-        ('kvcr.org', 'KVCR TV/DT/FM :: Vision for the Future (KVCR)'),  # CA, http://kvcr.org
-        ('rmpbs.org', 'Rocky Mountain PBS (KRMA)'),  # CO, http://www.rmpbs.org
-        ('kenw.org', 'KENW-TV3 (KENW)'),  # CO, http://www.kenw.org
-        ('kued.org', 'KUED Channel 7 (KUED)'),  # CO, http://www.kued.org
-        ('newmexicopbs.org', 'KNME-TV/Channel 5 (KNME)'),  # CO, http://www.newmexicopbs.org/
-        ('wyomingpbs.org', 'Wyoming PBS (KCWC)'),  # CO, http://www.wyomingpbs.org
-        ('cpt12.org', 'Colorado Public Television / KBDI 12 (KBDI)'),  # CO, http://www.cpt12.org/
-        ('kbyutv.org', 'KBYU-TV (KBYU)'),  # CO, http://www.kbyutv.org/
-        ('cptv.org', 'CONNECTICUT PUBLIC TELEVISION (WEDH)'),  # CT, http://cptv.org
-        ('thirteen.org', 'Thirteen/WNET New York (WNET)'),  # CT, http://www.thirteen.org
-        ('wgbh.org', 'WGBH/Channel 2 (WGBH)'),  # CT, http://wgbh.org
-        ('wgby.org', 'WGBY (WGBY)'),  # CT, http://www.wgby.org
-        ('njtvonline.org', 'NJTV Public Media NJ (WNJT)'),  # CT, http://www.njtvonline.org/
-        ('ripbs.org', 'Rhode Island PBS (WSBE)'),  # CT, http://www.ripbs.org/home/
-        ('wliw.org', 'WLIW21 (WLIW)'),  # CT, http://www.wliw.org/
-        ('mpt.org', 'mpt/Maryland Public Television (WMPB)'),  # DE, http://www.mpt.org
-        ('weta.org', 'WETA Television and Radio (WETA)'),  # DE, http://www.weta.org
-        ('whyy.org', 'WHYY (WHYY)'),  # DE, http://www.whyy.org
-        ('njtvonline.org', 'NJTV Public Media NJ (WNJT)'),  # DE, http://www.njtvonline.org/
-        ('wlvt.org', 'PBS 39 (WLVT)'),  # DE, http://www.wlvt.org/
-        ('wliw.org', 'WLIW21 (WLIW)'),  # DE, http://www.wliw.org/
-        ('weta.org', 'WETA Television and Radio (WETA)'),  # DC, http://www.weta.org
-        ('mpt.org', 'mpt/Maryland Public Television (WMPB)'),  # DC, http://www.mpt.org
-        ('wvpt.net', 'WVPT - Your Source for PBS and More! (WVPT)'),  # DC, http://www.wvpt.net
-        ('whut.org', 'Howard University Television (WHUT)'),  # DC, http://www.whut.org
-        ('wedu.org', 'WEDU PBS (WEDU)'),  # FL, http://www.wedu.org
-        ('wfsu.org', 'WFSU-TV (WFSU)'),  # FL, http://wfsu.org/
-        ('wgcu.org', 'WGCU Public Media (WGCU)'),  # FL, http://www.wgcu.org/
-        ('wjct.org', 'WJCT Public Broadcasting (WJCT)'),  # FL, http://www.wjct.org
-        ('wpbt2.org', 'WPBT2 (WPBT)'),  # FL, http://www.wpbt2.org
-        ('wsre.org', 'WSRE (WSRE)'),  # FL, http://www.wsre.org
-        ('wucftv.org', 'WUCF TV (WUCF)'),  # FL, http://wucftv.org
-        ('wuft.org', 'WUFT/Channel 5 (WUFT)'),  # FL, http://www.wuft.org
-        ('wxel.org', 'WXEL/Channel 42 (WXEL)'),  # FL, http://www.wxel.org/home/
-        ('aptv.org', 'APT - Alabama Public Television (WBIQ)'),  # FL, http://aptv.org/
-        ('gpb.org', 'GPB/Georgia Public Broadcasting (WGTV)'),  # FL, http://www.gpb.org/
-        ('wlrn.org', 'WLRN/Channel 17 (WLRN)'),  # FL, http://www.wlrn.org/
-        ('wusf.org', 'WUSF Public Broadcasting (WUSF)'),  # FL, http://wusf.org/
-        ('njtvonline.org', 'NJTV Public Media NJ (WNJT)'),  # FL, http://www.njtvonline.org/
-        ('gpb.org', 'GPB/Georgia Public Broadcasting (WGTV)'),  # GA, http://www.gpb.org/
-        ('aptv.org', 'APT - Alabama Public Television (WBIQ)'),  # GA, http://aptv.org/
-        ('scetv.org', 'ETV (WRLK)'),  # GA, http://www.scetv.org
-        ('unctv.org', 'UNC-TV (WUNC)'),  # GA, http://www.unctv.org/
-        ('wfsu.org', 'WFSU-TV (WFSU)'),  # GA, http://wfsu.org/
-        ('wjct.org', 'WJCT Public Broadcasting (WJCT)'),  # GA, http://www.wjct.org
-        ('wtcitv.org', 'WTCI (WTCI)'),  # GA, http://www.wtcitv.org
-        ('pba.org', 'WPBA/Channel 30 (WPBA)'),  # GA, http://pba.org/
-        ('pbsguam.org', 'PBS Guam (KGTF)'),  # GU, http://www.pbsguam.org/
-        ('pbshawaii.org', 'PBS Hawaii - Oceanic Cable Channel 10 (KHET)'),  # HI, http://www.pbshawaii.org/
-        ('idahoptv.org', 'Idaho Public Television (KAID)'),  # ID, http://idahoptv.org
-        ('ksps.org', 'KSPS (KSPS)'),  # ID, http://www.ksps.org/home/
-        ('kued.org', 'KUED Channel 7 (KUED)'),  # ID, http://www.kued.org
-        ('opb.org', 'OPB (KOPB)'),  # ID, http://www.opb.org
-        ('soptv.org', 'SOPTV (KSYS)'),  # ID, http://www.soptv.org
-        ('wyomingpbs.org', 'Wyoming PBS (KCWC)'),  # ID, http://www.wyomingpbs.org
-        ('cpt12.org', 'Colorado Public Television / KBDI 12 (KBDI)'),  # ID, http://www.cpt12.org/
-        ('kbyutv.org', 'KBYU-TV (KBYU)'),  # ID, http://www.kbyutv.org/
-        ('kwsu.org', 'KWSU/Channel 10 & KTNW/Channel 31 (KWSU)'),  # ID, http://www.kwsu.org
-        ('illinois.edu', 'WILL-TV (WILL)'),  # IL, http://will.illinois.edu/
-        ('wsec.tv', 'Network Knowledge - WSEC/Springfield (WSEC)'),  # IL, http://www.wsec.tv
-        ('wsiu.org', 'WSIU Public Broadcasting (WSIU)'),  # IL, http://www.wsiu.org/
-        ('wttw.com', 'WTTW11 (WTTW)'),  # IL, http://www.wttw.com/
-        ('wtvp.org', 'WTVP & WTVP.org, Public Media for Central Illinois (WTVP)'),  # IL, http://www.wtvp.org/
-        ('iptv.org', 'Iowa Public Television/IPTV (KDIN)'),  # IL, http://www.iptv.org/
-        ('ket.org', 'KET (WKLE)'),  # IL, http://www.ket.org/
-        ('ninenet.org', 'Nine Network (KETC)'),  # IL, http://www.ninenet.org
-        ('wfwa.org', 'PBS39 Fort Wayne (WFWA)'),  # IL, http://wfwa.org/
-        ('wfyi.org', 'WFYI Indianapolis (WFYI)'),  # IL, http://www.wfyi.org
-        ('mptv.org', 'Milwaukee Public Television (WMVS)'),  # IL, http://www.mptv.org
-        ('wnin.org', 'WNIN (WNIN)'),  # IL, http://www.wnin.org/
-        ('wnit.org', 'WNIT Public Television (WNIT)'),  # IL, http://www.wnit.org/
-        ('wpt.org', 'WPT (WPNE)'),  # IL, http://www.wpt.org/
-        ('wvut.org', 'WVUT/Channel 22 (WVUT)'),  # IL, http://wvut.org/
-        ('weiu.net', 'WEIU/Channel 51 (WEIU)'),  # IL, http://www.weiu.net
-        ('wqpt.org', 'WQPT-TV (WQPT)'),  # IL, http://www.wqpt.org
-        ('wycc.org', 'WYCC PBS Chicago (WYCC)'),  # IL, http://www.wycc.org
-        ('lakeshorepublicmedia.org', 'Lakeshore Public Television (WYIN)'),  # IL, http://lakeshorepublicmedia.org/
-        ('wfwa.org', 'PBS39 Fort Wayne (WFWA)'),  # IN, http://wfwa.org/
-        ('wfyi.org', 'WFYI Indianapolis (WFYI)'),  # IN, http://www.wfyi.org
-        ('wipb.org', 'WIPB-TV (WIPB)'),  # IN, http://wipb.org
-        ('wnin.org', 'WNIN (WNIN)'),  # IN, http://www.wnin.org/
-        ('wnit.org', 'WNIT Public Television (WNIT)'),  # IN, http://www.wnit.org/
-        ('indianapublicmedia.org', 'WTIU (WTIU)'),  # IN, http://indianapublicmedia.org/tv/
-        ('wvut.org', 'WVUT/Channel 22 (WVUT)'),  # IN, http://wvut.org/
-        ('cetconnect.org', 'CET  (WCET)'),  # IN, http://www.cetconnect.org
-        ('ket.org', 'KET (WKLE)'),  # IN, http://www.ket.org/
-        ('thinktv.org', 'ThinkTVNetwork (WPTD)'),  # IN, http://www.thinktv.org
-        ('wbgu.org', 'WBGU-TV (WBGU)'),  # IN, http://wbgu.org
-        ('wgvu.org', 'WGVU TV (WGVU)'),  # IN, http://www.wgvu.org/
-        ('illinois.edu', 'WILL-TV (WILL)'),  # IN, http://will.illinois.edu/
-        ('wsec.tv', 'Network Knowledge - WSEC/Springfield (WSEC)'),  # IN, http://www.wsec.tv
-        ('wsiu.org', 'WSIU Public Broadcasting (WSIU)'),  # IN, http://www.wsiu.org/
-        ('wttw.com', 'WTTW11 (WTTW)'),  # IN, http://www.wttw.com/
-        ('lakeshorepublicmedia.org', 'Lakeshore Public Television (WYIN)'),  # IN, http://lakeshorepublicmedia.org/
-        ('weiu.net', 'WEIU/Channel 51 (WEIU)'),  # IN, http://www.weiu.net
-        ('wycc.org', 'WYCC PBS Chicago (WYCC)'),  # IN, http://www.wycc.org
-        ('iptv.org', 'Iowa Public Television/IPTV (KDIN)'),  # IA, http://www.iptv.org/
-        ('netnebraska.org', 'NET1 (KUON)'),  # IA, http://netnebraska.org
-        ('pioneer.org', 'Pioneer Public Television (KWCM)'),  # IA, http://www.pioneer.org
-        ('sdpb.org', 'SDPB Television (KUSD)'),  # IA, http://www.sdpb.org
-        ('tpt.org', 'TPT (KTCA)'),  # IA, http://www.tpt.org
-        ('wsec.tv', 'Network Knowledge - WSEC/Springfield (WSEC)'),  # IA, http://www.wsec.tv
-        ('wpt.org', 'WPT (WPNE)'),  # IA, http://www.wpt.org/
-        ('ksmq.org', 'KSMQ (KSMQ)'),  # IA, http://www.ksmq.org/
-        ('wqpt.org', 'WQPT-TV (WQPT)'),  # IA, http://www.wqpt.org
-        ('kpts.org', 'KPTS/Channel 8 (KPTS)'),  # KS, http://www.kpts.org/
-        ('ktwu.org', 'KTWU/Channel 11 (KTWU)'),  # KS, http://ktwu.org
-        ('shptv.org', 'Smoky Hills Public Television (KOOD)'),  # KS, http://www.shptv.org
-        ('kcpt.org', 'KCPT Kansas City Public Television (KCPT)'),  # KS, http://kcpt.org/
-        ('netnebraska.org', 'NET1 (KUON)'),  # KS, http://netnebraska.org
-        ('oeta.tv', 'OETA (KETA)'),  # KS, http://www.oeta.tv
-        ('optv.org', 'Ozarks Public Television (KOZK)'),  # KS, http://www.optv.org/
-        ('rmpbs.org', 'Rocky Mountain PBS (KRMA)'),  # KS, http://www.rmpbs.org
-        ('cpt12.org', 'Colorado Public Television / KBDI 12 (KBDI)'),  # KS, http://www.cpt12.org/
-        ('ket.org', 'KET (WKLE)'),  # KY, http://www.ket.org/
-        ('blueridgepbs.org', 'Blue Ridge PBS (WBRA)'),  # KY, http://www.blueridgepbs.org/
-        ('cetconnect.org', 'CET  (WCET)'),  # KY, http://www.cetconnect.org
-        ('easttennesseepbs.org', 'East Tennessee PBS (WSJK)'),  # KY, http://easttennesseepbs.org
-        ('wnpt.org', 'Nashville Public Television (WNPT)'),  # KY, http://www.wnpt.org
-        ('thinktv.org', 'ThinkTVNetwork (WPTD)'),  # KY, http://www.thinktv.org
-        ('wcte.org', 'WCTE-TV (WCTE)'),  # KY, http://www.wcte.org
-        ('wljt.org', 'WLJT, Channel 11 (WLJT)'),  # KY, http://wljt.org/
-        ('wnin.org', 'WNIN (WNIN)'),  # KY, http://www.wnin.org/
-        ('wosu.org', 'WOSU TV (WOSU)'),  # KY, http://wosu.org/
-        ('woub.org', 'WOUB/WOUC (WOUB)'),  # KY, http://woub.org/tv/index.php?section=5
-        ('wsiu.org', 'WSIU Public Broadcasting (WSIU)'),  # KY, http://www.wsiu.org/
-        ('wvpublic.org', 'WVPB (WVPB)'),  # KY, http://wvpublic.org/
-        ('wkyupbs.org', 'WKYU-PBS (WKYU)'),  # KY, http://www.wkyupbs.org
-        ('lpb.org', 'LPB/LOUISIANA PUBLIC BROADCASTING (WLPB)'),  # LA, http://www.lpb.org/
-        ('wyes.org', 'WYES-TV/New Orleans (WYES)'),  # LA, http://www.wyes.org
-        ('aetn.org', 'AETN/ARKANSAS ETV NETWORK (KETS)'),  # LA, http://www.aetn.org/
-        ('kera.org', 'KERA 13 (KERA)'),  # LA, http://www.kera.org/
-        ('mpbonline.org', 'Mississippi Public Broadcasting (WMPN)'),  # LA, http://www.mpbonline.org
-        ('mpbn.net', 'MPBN (WCBB)'),  # ME, http://www.mpbn.net/
-        ('mountainlake.org', 'Mountain Lake PBS (WCFE)'),  # ME, http://www.mountainlake.org/
-        ('nhptv.org', 'NHPTV (WENH)'),  # ME, http://nhptv.org/
-        ('vpt.org', 'Vermont PBS (WETK)'),  # ME, http://www.vpt.org
-        ('wgbh.org', 'WGBH/Channel 2 (WGBH)'),  # ME, http://wgbh.org
-        ('mpt.org', 'mpt/Maryland Public Television (WMPB)'),  # MD, http://www.mpt.org
-        ('thirteen.org', 'Thirteen/WNET New York (WNET)'),  # MD, http://www.thirteen.org
-        ('weta.org', 'WETA Television and Radio (WETA)'),  # MD, http://www.weta.org
-        ('whyy.org', 'WHYY (WHYY)'),  # MD, http://www.whyy.org
-        ('witf.org', 'witf (WITF)'),  # MD, http://www.witf.org
-        ('wqed.org', 'WQED Multimedia (WQED)'),  # MD, http://www.wqed.org/
-        ('wvpublic.org', 'WVPB (WVPB)'),  # MD, http://wvpublic.org/
-        ('wvpt.net', 'WVPT - Your Source for PBS and More! (WVPT)'),  # MD, http://www.wvpt.net
-        ('whut.org', 'Howard University Television (WHUT)'),  # MD, http://www.whut.org
-        ('wliw.org', 'WLIW21 (WLIW)'),  # MD, http://www.wliw.org/
-        ('wgbh.org', 'WGBH/Channel 2 (WGBH)'),  # MA, http://wgbh.org
-        ('wgby.org', 'WGBY (WGBY)'),  # MA, http://www.wgby.org
-        ('cptv.org', 'CONNECTICUT PUBLIC TELEVISION (WEDH)'),  # MA, http://cptv.org
-        ('nhptv.org', 'NHPTV (WENH)'),  # MA, http://nhptv.org/
-        ('vpt.org', 'Vermont PBS (WETK)'),  # MA, http://www.vpt.org
-        ('wmht.org', 'WMHT Educational Telecommunications (WMHT)'),  # MA, http://www.wmht.org/home/
-        ('ripbs.org', 'Rhode Island PBS (WSBE)'),  # MA, http://www.ripbs.org/home/
-        ('deltabroadcasting.org', 'Q-TV (WDCQ)'),  # MI, http://www.deltabroadcasting.org
-        ('dptv.org', 'WTVS Detroit Public TV (WTVS)'),  # MI, http://www.dptv.org/
-        ('wcmu.org', 'CMU Public Television (WCMU)'),  # MI, http://www.wcmu.org
-        ('wgvu.org', 'WGVU TV (WGVU)'),  # MI, http://www.wgvu.org/
-        ('wkar.org', 'WKAR-TV (WKAR)'),  # MI, http://wkar.org/
-        ('nmu.edu', 'WNMU-TV Public TV 13 (WNMU)'),  # MI, http://wnmutv.nmu.edu
-        ('wdse.org', 'WDSE - WRPT (WDSE)'),  # MI, http://www.wdse.org/
-        ('wbgu.org', 'WBGU-TV (WBGU)'),  # MI, http://wbgu.org
-        ('wgte.org', 'WGTE TV (WGTE)'),  # MI, http://www.wgte.org
-        ('wnit.org', 'WNIT Public Television (WNIT)'),  # MI, http://www.wnit.org/
-        ('wpt.org', 'WPT (WPNE)'),  # MI, http://www.wpt.org/
-        ('wttw.com', 'WTTW11 (WTTW)'),  # MI, http://www.wttw.com/
-        ('wycc.org', 'WYCC PBS Chicago (WYCC)'),  # MI, http://www.wycc.org
-        ('lakelandptv.org', 'Lakeland Public Television (KAWE)'),  # MN, http://www.lakelandptv.org
-        ('wdse.org', 'WDSE - WRPT (WDSE)'),  # MN, http://www.wdse.org/
-        ('pioneer.org', 'Pioneer Public Television (KWCM)'),  # MN, http://www.pioneer.org
-        ('tpt.org', 'TPT (KTCA)'),  # MN, http://www.tpt.org
-        ('iptv.org', 'Iowa Public Television/IPTV (KDIN)'),  # MN, http://www.iptv.org/
-        ('prairiepublic.org', 'PRAIRIE PUBLIC (KFME)'),  # MN, http://www.prairiepublic.org/
-        ('ninenet.org', 'Nine Network (KETC)'),  # MN, http://www.ninenet.org
-        ('sdpb.org', 'SDPB Television (KUSD)'),  # MN, http://www.sdpb.org
-        ('wpt.org', 'WPT (WPNE)'),  # MN, http://www.wpt.org/
-        ('ksmq.org', 'KSMQ (KSMQ)'),  # MN, http://www.ksmq.org/
-        ('mpbonline.org', 'Mississippi Public Broadcasting (WMPN)'),  # MS, http://www.mpbonline.org
-        ('aetn.org', 'AETN/ARKANSAS ETV NETWORK (KETS)'),  # MS, http://www.aetn.org/
-        ('aptv.org', 'APT - Alabama Public Television (WBIQ)'),  # MS, http://aptv.org/
-        ('wkno.org', 'WKNO/Channel 10 (WKNO)'),  # MS, http://www.wkno.org/
-        ('lpb.org', 'LPB/LOUISIANA PUBLIC BROADCASTING (WLPB)'),  # MS, http://www.lpb.org/
-        ('wsre.org', 'WSRE (WSRE)'),  # MS, http://www.wsre.org
-        ('wyes.org', 'WYES-TV/New Orleans (WYES)'),  # MS, http://www.wyes.org
-        ('kcpt.org', 'KCPT Kansas City Public Television (KCPT)'),  # MO, http://kcpt.org/
-        ('kmos.org', 'KMOS-TV - Channels 6.1, 6.2 and 6.3 (KMOS)'),  # MO, http://www.kmos.org/
-        ('ninenet.org', 'Nine Network (KETC)'),  # MO, http://www.ninenet.org
-        ('optv.org', 'Ozarks Public Television (KOZK)'),  # MO, http://www.optv.org/
-        ('aetn.org', 'AETN/ARKANSAS ETV NETWORK (KETS)'),  # MO, http://www.aetn.org/
-        ('dptv.org', 'WTVS Detroit Public TV (WTVS)'),  # MO, http://www.dptv.org/
-        ('iptv.org', 'Iowa Public Television/IPTV (KDIN)'),  # MO, http://www.iptv.org/
-        ('ket.org', 'KET (WKLE)'),  # MO, http://www.ket.org/
-        ('wkno.org', 'WKNO/Channel 10 (WKNO)'),  # MO, http://www.wkno.org/
-        ('ktwu.org', 'KTWU/Channel 11 (KTWU)'),  # MO, http://ktwu.org
-        ('netnebraska.org', 'NET1 (KUON)'),  # MO, http://netnebraska.org
-        ('rmpbs.org', 'Rocky Mountain PBS (KRMA)'),  # MO, http://www.rmpbs.org
-        ('wsec.tv', 'Network Knowledge - WSEC/Springfield (WSEC)'),  # MO, http://www.wsec.tv
-        ('wsiu.org', 'WSIU Public Broadcasting (WSIU)'),  # MO, http://www.wsiu.org/
-        ('montanapbs.org', 'MontanaPBS (KUSM)'),  # MT, http://montanapbs.org
-        ('azpbs.org', 'Arizona PBS (KAET)'),  # MT, http://www.azpbs.org
-        ('idahoptv.org', 'Idaho Public Television (KAID)'),  # MT, http://idahoptv.org
-        ('prairiepublic.org', 'PRAIRIE PUBLIC (KFME)'),  # MT, http://www.prairiepublic.org/
-        ('ksps.org', 'KSPS (KSPS)'),  # MT, http://www.ksps.org/home/
-        ('rmpbs.org', 'Rocky Mountain PBS (KRMA)'),  # MT, http://www.rmpbs.org
-        ('sdpb.org', 'SDPB Television (KUSD)'),  # MT, http://www.sdpb.org
-        ('netnebraska.org', 'NET1 (KUON)'),  # NE, http://netnebraska.org
-        ('iptv.org', 'Iowa Public Television/IPTV (KDIN)'),  # NE, http://www.iptv.org/
-        ('ktwu.org', 'KTWU/Channel 11 (KTWU)'),  # NE, http://ktwu.org
-        ('pioneer.org', 'Pioneer Public Television (KWCM)'),  # NE, http://www.pioneer.org
-        ('rmpbs.org', 'Rocky Mountain PBS (KRMA)'),  # NE, http://www.rmpbs.org
-        ('sdpb.org', 'SDPB Television (KUSD)'),  # NE, http://www.sdpb.org
-        ('wyomingpbs.org', 'Wyoming PBS (KCWC)'),  # NE, http://www.wyomingpbs.org
-        ('cpt12.org', 'Colorado Public Television / KBDI 12 (KBDI)'),  # NE, http://www.cpt12.org/
-        ('knpb.org', 'KNPB Channel 5 (KNPB)'),  # NV, http://www.knpb.org/
-        ('vegaspbs.org', 'Vegas PBS (KLVX)'),  # NV, http://vegaspbs.org/
-        ('azpbs.org', 'Arizona PBS (KAET)'),  # NV, http://www.azpbs.org
-        ('kued.org', 'KUED Channel 7 (KUED)'),  # NV, http://www.kued.org
-        ('pbssocal.org', 'PBS SoCal/KOCE (KOCE)'),  # NV, http://www.pbssocal.org/
-        ('kbyutv.org', 'KBYU-TV (KBYU)'),  # NV, http://www.kbyutv.org/
-        ('nhptv.org', 'NHPTV (WENH)'),  # NH, http://nhptv.org/
-        ('scetv.org', 'ETV (WRLK)'),  # NH, http://www.scetv.org
-        ('mountainlake.org', 'Mountain Lake PBS (WCFE)'),  # NH, http://www.mountainlake.org/
-        ('mpbn.net', 'MPBN (WCBB)'),  # NH, http://www.mpbn.net/
-        ('vpt.org', 'Vermont PBS (WETK)'),  # NH, http://www.vpt.org
-        ('wgbh.org', 'WGBH/Channel 2 (WGBH)'),  # NH, http://wgbh.org
-        ('wgby.org', 'WGBY (WGBY)'),  # NH, http://www.wgby.org
-        ('cptv.org', 'CONNECTICUT PUBLIC TELEVISION (WEDH)'),  # NJ, http://cptv.org
-        ('thirteen.org', 'Thirteen/WNET New York (WNET)'),  # NJ, http://www.thirteen.org
-        ('whyy.org', 'WHYY (WHYY)'),  # NJ, http://www.whyy.org
-        ('njtvonline.org', 'NJTV Public Media NJ (WNJT)'),  # NJ, http://www.njtvonline.org/
-        ('wlvt.org', 'PBS 39 (WLVT)'),  # NJ, http://www.wlvt.org/
-        ('wliw.org', 'WLIW21 (WLIW)'),  # NJ, http://www.wliw.org/
-        ('kenw.org', 'KENW-TV3 (KENW)'),  # NM, http://www.kenw.org
-        ('krwg.org', 'KRWG/Channel 22 (KRWG)'),  # NM, http://www.krwg.org
-        ('newmexicopbs.org', 'KNME-TV/Channel 5 (KNME)'),  # NM, http://www.newmexicopbs.org/
-        ('azpm.org', 'KUAT 6 (KUAT)'),  # NM, http://www.azpm.org/
-        ('panhandlepbs.org', 'KACV (KACV)'),  # NM, http://www.panhandlepbs.org/home/
-        ('kcostv.org', 'KCOS/Channel 13 (KCOS)'),  # NM, www.kcostv.org
-        ('kued.org', 'KUED Channel 7 (KUED)'),  # NM, http://www.kued.org
-        ('oeta.tv', 'OETA (KETA)'),  # NM, http://www.oeta.tv
-        ('rmpbs.org', 'Rocky Mountain PBS (KRMA)'),  # NM, http://www.rmpbs.org
-        ('cpt12.org', 'Colorado Public Television / KBDI 12 (KBDI)'),  # NM, http://www.cpt12.org/
-        ('kbyutv.org', 'KBYU-TV (KBYU)'),  # NM, http://www.kbyutv.org/
-        ('mountainlake.org', 'Mountain Lake PBS (WCFE)'),  # NY, http://www.mountainlake.org/
-        ('thirteen.org', 'Thirteen/WNET New York (WNET)'),  # NY, http://www.thirteen.org
-        ('wcny.org', 'WCNY/Channel 24 (WCNY)'),  # NY, http://www.wcny.org
-        ('wmht.org', 'WMHT Educational Telecommunications (WMHT)'),  # NY, http://www.wmht.org/home/
-        ('wned.org', 'WNED (WNED)'),  # NY, http://www.wned.org/
-        ('wpbstv.org', 'WPBS (WPBS)'),  # NY, http://www.wpbstv.org
-        ('wskg.org', 'WSKG Public TV (WSKG)'),  # NY, http://wskg.org
-        ('wxxi.org', 'WXXI (WXXI)'),  # NY, http://wxxi.org
-        ('cptv.org', 'CONNECTICUT PUBLIC TELEVISION (WEDH)'),  # NY, http://cptv.org
-        ('nhptv.org', 'NHPTV (WENH)'),  # NY, http://nhptv.org/
-        ('vpt.org', 'Vermont PBS (WETK)'),  # NY, http://www.vpt.org
-        ('wgbh.org', 'WGBH/Channel 2 (WGBH)'),  # NY, http://wgbh.org
-        ('wpsu.org', 'WPSU (WPSU)'),  # NY, http://www.wpsu.org
-        ('wqln.org', 'WQLN/Channel 54 (WQLN)'),  # NY, http://www.wqln.org
-        ('wvia.org', 'WVIA Public Media Studios (WVIA)'),  # NY, http://www.wvia.org/
-        ('wliw.org', 'WLIW21 (WLIW)'),  # NY, http://www.wliw.org/
-        ('njtvonline.org', 'NJTV Public Media NJ (WNJT)'),  # NY, http://www.njtvonline.org/
-        ('unctv.org', 'UNC-TV (WUNC)'),  # NC, http://www.unctv.org/
-        ('wtvi.org', 'WTVI (WTVI)'),  # NC, http://www.wtvi.org/
-        ('blueridgepbs.org', 'Blue Ridge PBS (WBRA)'),  # NC, http://www.blueridgepbs.org/
-        ('scetv.org', 'ETV (WRLK)'),  # NC, http://www.scetv.org
-        ('gpb.org', 'GPB/Georgia Public Broadcasting (WGTV)'),  # NC, http://www.gpb.org/
-        ('whro.org', 'WHRO (WHRO)'),  # NC, http://whro.org
-        ('wtcitv.org', 'WTCI (WTCI)'),  # NC, http://www.wtcitv.org
-        ('pba.org', 'WPBA/Channel 30 (WPBA)'),  # NC, http://pba.org/
-        ('prairiepublic.org', 'PRAIRIE PUBLIC (KFME)'),  # ND, http://www.prairiepublic.org/
-        ('sdpb.org', 'SDPB Television (KUSD)'),  # ND, http://www.sdpb.org
-        ('cetconnect.org', 'CET  (WCET)'),  # OH, http://www.cetconnect.org
-        ('thinktv.org', 'ThinkTVNetwork (WPTD)'),  # OH, http://www.thinktv.org
-        ('wbgu.org', 'WBGU-TV (WBGU)'),  # OH, http://wbgu.org
-        ('wgte.org', 'WGTE TV (WGTE)'),  # OH, http://www.wgte.org
-        ('WesternReservePublicMedia.org', 'Western Reserve PBS (WNEO)'),  # OH, http://www.WesternReservePublicMedia.org/
-        ('wosu.org', 'WOSU TV (WOSU)'),  # OH, http://wosu.org/
-        ('woub.org', 'WOUB/WOUC (WOUB)'),  # OH, http://woub.org/tv/index.php?section=5
-        ('wviz.org', 'WVIZ/PBS ideastream (WVIZ)'),  # OH, http://www.wviz.org/
-        ('dptv.org', 'WTVS Detroit Public TV (WTVS)'),  # OH, http://www.dptv.org/
-        ('ket.org', 'KET (WKLE)'),  # OH, http://www.ket.org/
-        ('wfwa.org', 'PBS39 Fort Wayne (WFWA)'),  # OH, http://wfwa.org/
-        ('wipb.org', 'WIPB-TV (WIPB)'),  # OH, http://wipb.org
-        ('wqed.org', 'WQED Multimedia (WQED)'),  # OH, http://www.wqed.org/
-        ('wqln.org', 'WQLN/Channel 54 (WQLN)'),  # OH, http://www.wqln.org
-        ('wvpublic.org', 'WVPB (WVPB)'),  # OH, http://wvpublic.org/
-        ('oeta.tv', 'OETA (KETA)'),  # OK, http://www.oeta.tv
-        ('aetn.org', 'AETN/ARKANSAS ETV NETWORK (KETS)'),  # OK, http://www.aetn.org/
-        ('panhandlepbs.org', 'KACV (KACV)'),  # OK, http://www.panhandlepbs.org/home/
-        ('kera.org', 'KERA 13 (KERA)'),  # OK, http://www.kera.org/
-        ('lpb.org', 'LPB/LOUISIANA PUBLIC BROADCASTING (WLPB)'),  # OK, http://www.lpb.org/
-        ('optv.org', 'Ozarks Public Television (KOZK)'),  # OK, http://www.optv.org/
-        ('opb.org', 'OPB (KOPB)'),  # OR, http://www.opb.org
-        ('soptv.org', 'SOPTV (KSYS)'),  # OR, http://www.soptv.org
-        ('idahoptv.org', 'Idaho Public Television (KAID)'),  # OR, http://idahoptv.org
-        ('kcts9.org', 'KCTS 9 (KCTS)'),  # OR, http://kcts9.org/
-        ('keet.org', 'KEET TV (KEET)'),  # OR, http://www.keet.org
-        ('ksps.org', 'KSPS (KSPS)'),  # OR, http://www.ksps.org/home/
-        ('kwsu.org', 'KWSU/Channel 10 & KTNW/Channel 31 (KWSU)'),  # OR, http://www.kwsu.org
-        ('whyy.org', 'WHYY (WHYY)'),  # PA, http://www.whyy.org
-        ('witf.org', 'witf (WITF)'),  # PA, http://www.witf.org
-        ('wpsu.org', 'WPSU (WPSU)'),  # PA, http://www.wpsu.org
-        ('wqed.org', 'WQED Multimedia (WQED)'),  # PA, http://www.wqed.org/
-        ('wqln.org', 'WQLN/Channel 54 (WQLN)'),  # PA, http://www.wqln.org
-        ('wvia.org', 'WVIA Public Media Studios (WVIA)'),  # PA, http://www.wvia.org/
-        ('cptv.org', 'CONNECTICUT PUBLIC TELEVISION (WEDH)'),  # PA, http://cptv.org
-        ('mpt.org', 'mpt/Maryland Public Television (WMPB)'),  # PA, http://www.mpt.org
-        ('thirteen.org', 'Thirteen/WNET New York (WNET)'),  # PA, http://www.thirteen.org
-        ('weta.org', 'WETA Television and Radio (WETA)'),  # PA, http://www.weta.org
-        ('wned.org', 'WNED (WNED)'),  # PA, http://www.wned.org/
-        ('WesternReservePublicMedia.org', 'Western Reserve PBS (WNEO)'),  # PA, http://www.WesternReservePublicMedia.org/
-        ('wskg.org', 'WSKG Public TV (WSKG)'),  # PA, http://wskg.org
-        ('wviz.org', 'WVIZ/PBS ideastream (WVIZ)'),  # PA, http://www.wviz.org/
-        ('wvpublic.org', 'WVPB (WVPB)'),  # PA, http://wvpublic.org/
-        ('wvpt.net', 'WVPT - Your Source for PBS and More! (WVPT)'),  # PA, http://www.wvpt.net
-        ('wlvt.org', 'PBS 39 (WLVT)'),  # PA, http://www.wlvt.org/
-        ('njtvonline.org', 'NJTV Public Media NJ (WNJT)'),  # PA, http://www.njtvonline.org/
-        ('whut.org', 'Howard University Television (WHUT)'),  # PA, http://www.whut.org
-        ('wliw.org', 'WLIW21 (WLIW)'),  # PA, http://www.wliw.org/
-        ('cptv.org', 'CONNECTICUT PUBLIC TELEVISION (WEDH)'),  # RI, http://cptv.org
-        ('nhptv.org', 'NHPTV (WENH)'),  # RI, http://nhptv.org/
-        ('wgbh.org', 'WGBH/Channel 2 (WGBH)'),  # RI, http://wgbh.org
-        ('ripbs.org', 'Rhode Island PBS (WSBE)'),  # RI, http://www.ripbs.org/home/
-        ('scetv.org', 'ETV (WRLK)'),  # SC, http://www.scetv.org
-        ('gpb.org', 'GPB/Georgia Public Broadcasting (WGTV)'),  # SC, http://www.gpb.org/
-        ('unctv.org', 'UNC-TV (WUNC)'),  # SC, http://www.unctv.org/
-        ('wtvi.org', 'WTVI (WTVI)'),  # SC, http://www.wtvi.org/
-        ('pba.org', 'WPBA/Channel 30 (WPBA)'),  # SC, http://pba.org/
-        ('sdpb.org', 'SDPB Television (KUSD)'),  # SD, http://www.sdpb.org
-        ('iptv.org', 'Iowa Public Television/IPTV (KDIN)'),  # SD, http://www.iptv.org/
-        ('prairiepublic.org', 'PRAIRIE PUBLIC (KFME)'),  # SD, http://www.prairiepublic.org/
-        ('netnebraska.org', 'NET1 (KUON)'),  # SD, http://netnebraska.org
-        ('pioneer.org', 'Pioneer Public Television (KWCM)'),  # SD, http://www.pioneer.org
-        ('easttennesseepbs.org', 'East Tennessee PBS (WSJK)'),  # TN, http://easttennesseepbs.org
-        ('wkno.org', 'WKNO/Channel 10 (WKNO)'),  # TN, http://www.wkno.org/
-        ('wnpt.org', 'Nashville Public Television (WNPT)'),  # TN, http://www.wnpt.org
-        ('wcte.org', 'WCTE-TV (WCTE)'),  # TN, http://www.wcte.org
-        ('wljt.org', 'WLJT, Channel 11 (WLJT)'),  # TN, http://wljt.org/
-        ('wtcitv.org', 'WTCI (WTCI)'),  # TN, http://www.wtcitv.org
-        ('aptv.org', 'APT - Alabama Public Television (WBIQ)'),  # TN, http://aptv.org/
-        ('blueridgepbs.org', 'Blue Ridge PBS (WBRA)'),  # TN, http://www.blueridgepbs.org/
-        ('gpb.org', 'GPB/Georgia Public Broadcasting (WGTV)'),  # TN, http://www.gpb.org/
-        ('lakelandptv.org', 'Lakeland Public Television (KAWE)'),  # TN, http://www.lakelandptv.org
-        ('ket.org', 'KET (WKLE)'),  # TN, http://www.ket.org/
-        ('mpbonline.org', 'Mississippi Public Broadcasting (WMPN)'),  # TN, http://www.mpbonline.org
-        ('unctv.org', 'UNC-TV (WUNC)'),  # TN, http://www.unctv.org/
-        ('WesternReservePublicMedia.org', 'Western Reserve PBS (WNEO)'),  # TN, http://www.WesternReservePublicMedia.org/
-        ('wsiu.org', 'WSIU Public Broadcasting (WSIU)'),  # TN, http://www.wsiu.org/
-        ('wviz.org', 'WVIZ/PBS ideastream (WVIZ)'),  # TN, http://www.wviz.org/
-        ('wkyupbs.org', 'WKYU-PBS (WKYU)'),  # TN, http://www.wkyupbs.org
-        ('pba.org', 'WPBA/Channel 30 (WPBA)'),  # TN, http://pba.org/
-        ('basinpbs.org', 'Basin PBS (KPBT)'),  # TX, http://www.basinpbs.org
-        ('houstonpublicmedia.org', 'KUHT / Channel 8 (KUHT)'),  # TX, http://www.houstonpublicmedia.org/
-        ('panhandlepbs.org', 'KACV (KACV)'),  # TX, http://www.panhandlepbs.org/home/
-        ('tamu.edu', 'KAMU - TV (KAMU)'),  # TX, http://KAMU.tamu.edu
-        ('kcostv.org', 'KCOS/Channel 13 (KCOS)'),  # TX, www.kcostv.org
-        ('kedt.org', 'KEDT/Channel 16 (KEDT)'),  # TX, http://www.kedt.org
-        ('kera.org', 'KERA 13 (KERA)'),  # TX, http://www.kera.org/
-        ('klrn.org', 'KLRN (KLRN)'),  # TX, http://www.klrn.org
-        ('klru.org', 'KLRU (KLRU)'),  # TX, http://www.klru.org
-        ('kmbh.org', 'KMBH-TV (KMBH)'),  # TX, http://www.kmbh.org
-        ('knct.org', 'KNCT (KNCT)'),  # TX, http://www.knct.org
-        ('ktxt.org', 'KTTZ-TV (KTXT)'),  # TX, http://www.ktxt.org
-        ('aetn.org', 'AETN/ARKANSAS ETV NETWORK (KETS)'),  # TX, http://www.aetn.org/
-        ('kenw.org', 'KENW-TV3 (KENW)'),  # TX, http://www.kenw.org
-        ('krwg.org', 'KRWG/Channel 22 (KRWG)'),  # TX, http://www.krwg.org
-        ('lpb.org', 'LPB/LOUISIANA PUBLIC BROADCASTING (WLPB)'),  # TX, http://www.lpb.org/
-        ('netnebraska.org', 'NET1 (KUON)'),  # TX, http://netnebraska.org
-        ('newmexicopbs.org', 'KNME-TV/Channel 5 (KNME)'),  # TX, http://www.newmexicopbs.org/
-        ('wnpt.org', 'Nashville Public Television (WNPT)'),  # TX, http://www.wnpt.org
-        ('oeta.tv', 'OETA (KETA)'),  # TX, http://www.oeta.tv
-        ('rmpbs.org', 'Rocky Mountain PBS (KRMA)'),  # TX, http://www.rmpbs.org
-        ('kued.org', 'KUED Channel 7 (KUED)'),  # UT, http://www.kued.org
-        ('idahoptv.org', 'Idaho Public Television (KAID)'),  # UT, http://idahoptv.org
-        ('vegaspbs.org', 'Vegas PBS (KLVX)'),  # UT, http://vegaspbs.org/
-        ('kbyutv.org', 'KBYU-TV (KBYU)'),  # UT, http://www.kbyutv.org/
-        ('vpt.org', 'Vermont PBS (WETK)'),  # VT, http://www.vpt.org
-        ('mountainlake.org', 'Mountain Lake PBS (WCFE)'),  # VT, http://www.mountainlake.org/
-        ('nhptv.org', 'NHPTV (WENH)'),  # VT, http://nhptv.org/
-        ('wgbh.org', 'WGBH/Channel 2 (WGBH)'),  # VT, http://wgbh.org
-        ('wgby.org', 'WGBY (WGBY)'),  # VT, http://www.wgby.org
-        ('wmht.org', 'WMHT Educational Telecommunications (WMHT)'),  # VT, http://www.wmht.org/home/
-        ('wtjx.org', 'WTJX Channel 12 (WTJX)'),  # VI, http://www.wtjx.org/
-        ('blueridgepbs.org', 'Blue Ridge PBS (WBRA)'),  # VA, http://www.blueridgepbs.org/
-        ('ideastations.org', 'WCVE PBS (WCVE)'),  # VA, http://ideastations.org/
-        ('whro.org', 'WHRO (WHRO)'),  # VA, http://whro.org
-        ('wvpt.net', 'WVPT - Your Source for PBS and More! (WVPT)'),  # VA, http://www.wvpt.net
-        ('easttennesseepbs.org', 'East Tennessee PBS (WSJK)'),  # VA, http://easttennesseepbs.org
-        ('ket.org', 'KET (WKLE)'),  # VA, http://www.ket.org/
-        ('mpt.org', 'mpt/Maryland Public Television (WMPB)'),  # VA, http://www.mpt.org
-        ('unctv.org', 'UNC-TV (WUNC)'),  # VA, http://www.unctv.org/
-        ('weta.org', 'WETA Television and Radio (WETA)'),  # VA, http://www.weta.org
-        ('whut.org', 'Howard University Television (WHUT)'),  # VA, http://www.whut.org
-        ('kcts9.org', 'KCTS 9 (KCTS)'),  # WA, http://kcts9.org/
-        ('ksps.org', 'KSPS (KSPS)'),  # WA, http://www.ksps.org/home/
-        ('idahoptv.org', 'Idaho Public Television (KAID)'),  # WA, http://idahoptv.org
-        ('opb.org', 'OPB (KOPB)'),  # WA, http://www.opb.org
-        ('kbtc.org', 'KBTC Public Television (KBTC)'),  # WA, http://kbtc.org
-        ('kwsu.org', 'KWSU/Channel 10 & KTNW/Channel 31 (KWSU)'),  # WA, http://www.kwsu.org
-        ('wvpublic.org', 'WVPB (WVPB)'),  # WV, http://wvpublic.org/
-        ('blueridgepbs.org', 'Blue Ridge PBS (WBRA)'),  # WV, http://www.blueridgepbs.org/
-        ('ket.org', 'KET (WKLE)'),  # WV, http://www.ket.org/
-        ('mpt.org', 'mpt/Maryland Public Television (WMPB)'),  # WV, http://www.mpt.org
-        ('weta.org', 'WETA Television and Radio (WETA)'),  # WV, http://www.weta.org
-        ('WesternReservePublicMedia.org', 'Western Reserve PBS (WNEO)'),  # WV, http://www.WesternReservePublicMedia.org/
-        ('wosu.org', 'WOSU TV (WOSU)'),  # WV, http://wosu.org/
-        ('woub.org', 'WOUB/WOUC (WOUB)'),  # WV, http://woub.org/tv/index.php?section=5
-        ('wqed.org', 'WQED Multimedia (WQED)'),  # WV, http://www.wqed.org/
-        ('wvpt.net', 'WVPT - Your Source for PBS and More! (WVPT)'),  # WV, http://www.wvpt.net
-        ('whut.org', 'Howard University Television (WHUT)'),  # WV, http://www.whut.org
-        ('mptv.org', 'Milwaukee Public Television (WMVS)'),  # WI, http://www.mptv.org
-        ('wpt.org', 'WPT (WPNE)'),  # WI, http://www.wpt.org/
-        ('iptv.org', 'Iowa Public Television/IPTV (KDIN)'),  # WI, http://www.iptv.org/
-        ('lakelandptv.org', 'Lakeland Public Television (KAWE)'),  # WI, http://www.lakelandptv.org
-        ('wdse.org', 'WDSE - WRPT (WDSE)'),  # WI, http://www.wdse.org/
-        ('tpt.org', 'TPT (KTCA)'),  # WI, http://www.tpt.org
-        ('nmu.edu', 'WNMU-TV Public TV 13 (WNMU)'),  # WI, http://wnmutv.nmu.edu
-        ('wttw.com', 'WTTW11 (WTTW)'),  # WI, http://www.wttw.com/
-        ('ksmq.org', 'KSMQ (KSMQ)'),  # WI, http://www.ksmq.org/
-        ('wycc.org', 'WYCC PBS Chicago (WYCC)'),  # WI, http://www.wycc.org
-        ('wyomingpbs.org', 'Wyoming PBS (KCWC)'),  # WY, http://www.wyomingpbs.org
-        ('idahoptv.org', 'Idaho Public Television (KAID)'),  # WY, http://idahoptv.org
-        ('kued.org', 'KUED Channel 7 (KUED)'),  # WY, http://www.kued.org
-        ('montanapbs.org', 'MontanaPBS (KUSM)'),  # WY, http://montanapbs.org
-        ('netnebraska.org', 'NET1 (KUON)'),  # WY, http://netnebraska.org
-        ('rmpbs.org', 'Rocky Mountain PBS (KRMA)'),  # WY, http://www.rmpbs.org
-        ('sdpb.org', 'SDPB Television (KUSD)'),  # WY, http://www.sdpb.org
-        ('cpt12.org', 'Colorado Public Television / KBDI 12 (KBDI)'),  # WY, http://www.cpt12.org/
-        ('kbyutv.org', 'KBYU-TV (KBYU)'),  # WY, http://www.kbyutv.org/
+        ('aptv.org', 'APT - Alabama Public Television (WBIQ)'),  # http://aptv.org/
+        ('gpb.org', 'GPB/Georgia Public Broadcasting (WGTV)'),  # http://www.gpb.org/
+        ('mpbonline.org', 'Mississippi Public Broadcasting (WMPN)'),  # http://www.mpbonline.org
+        ('wnpt.org', 'Nashville Public Television (WNPT)'),  # http://www.wnpt.org
+        ('wfsu.org', 'WFSU-TV (WFSU)'),  # http://wfsu.org/
+        ('wsre.org', 'WSRE (WSRE)'),  # http://www.wsre.org
+        ('wtcitv.org', 'WTCI (WTCI)'),  # http://www.wtcitv.org
+        ('pba.org', 'WPBA/Channel 30 (WPBA)'),  # http://pba.org/
+        ('alaskapublic.org', 'Alaska Public Media (KAKM)'),  # http://alaskapublic.org/kakm
+        ('kuac.org', 'KUAC (KUAC)'),  # http://kuac.org/kuac-tv/
+        ('ktoo.org', '360 North (KTOO)'),  # http://www.ktoo.org/
+        ('azpm.org', 'KUAT 6 (KUAT)'),  # http://www.azpm.org/
+        ('azpbs.org', 'Arizona PBS (KAET)'),  # http://www.azpbs.org
+        ('newmexicopbs.org', 'KNME-TV/Channel 5 (KNME)'),  # http://www.newmexicopbs.org/
+        ('vegaspbs.org', 'Vegas PBS (KLVX)'),  # http://vegaspbs.org/
+        ('aetn.org', 'AETN/ARKANSAS ETV NETWORK (KETS)'),  # http://www.aetn.org/
+        ('ket.org', 'KET (WKLE)'),  # http://www.ket.org/
+        ('wkno.org', 'WKNO/Channel 10 (WKNO)'),  # http://www.wkno.org/
+        ('lpb.org', 'LPB/LOUISIANA PUBLIC BROADCASTING (WLPB)'),  # http://www.lpb.org/
+        ('oeta.tv', 'OETA (KETA)'),  # http://www.oeta.tv
+        ('optv.org', 'Ozarks Public Television (KOZK)'),  # http://www.optv.org/
+        ('wsiu.org', 'WSIU Public Broadcasting (WSIU)'),  # http://www.wsiu.org/
+        ('keet.org', 'KEET TV (KEET)'),  # http://www.keet.org
+        ('kixe.org', 'KIXE/Channel 9 (KIXE)'),  # http://kixe.org/
+        ('kpbs.org', 'KPBS San Diego (KPBS)'),  # http://www.kpbs.org/
+        ('kqed.org', 'KQED (KQED)'),  # http://www.kqed.org
+        ('kvie.org', 'KVIE Public Television (KVIE)'),  # http://www.kvie.org
+        ('pbssocal.org', 'PBS SoCal/KOCE (KOCE)'),  # http://www.pbssocal.org/
+        ('valleypbs.org', 'ValleyPBS (KVPT)'),  # http://www.valleypbs.org/
+        ('cptv.org', 'CONNECTICUT PUBLIC TELEVISION (WEDH)'),  # http://cptv.org
+        ('knpb.org', 'KNPB Channel 5 (KNPB)'),  # http://www.knpb.org/
+        ('soptv.org', 'SOPTV (KSYS)'),  # http://www.soptv.org
+        ('klcs.org', 'KLCS/Channel 58 (KLCS)'),  # http://www.klcs.org
+        ('krcb.org', 'KRCB Television & Radio (KRCB)'),  # http://www.krcb.org
+        ('kvcr.org', 'KVCR TV/DT/FM :: Vision for the Future (KVCR)'),  # http://kvcr.org
+        ('rmpbs.org', 'Rocky Mountain PBS (KRMA)'),  # http://www.rmpbs.org
+        ('kenw.org', 'KENW-TV3 (KENW)'),  # http://www.kenw.org
+        ('kued.org', 'KUED Channel 7 (KUED)'),  # http://www.kued.org
+        ('wyomingpbs.org', 'Wyoming PBS (KCWC)'),  # http://www.wyomingpbs.org
+        ('cpt12.org', 'Colorado Public Television / KBDI 12 (KBDI)'),  # http://www.cpt12.org/
+        ('kbyutv.org', 'KBYU-TV (KBYU)'),  # http://www.kbyutv.org/
+        ('thirteen.org', 'Thirteen/WNET New York (WNET)'),  # http://www.thirteen.org
+        ('wgbh.org', 'WGBH/Channel 2 (WGBH)'),  # http://wgbh.org
+        ('wgby.org', 'WGBY (WGBY)'),  # http://www.wgby.org
+        ('njtvonline.org', 'NJTV Public Media NJ (WNJT)'),  # http://www.njtvonline.org/
+        ('ripbs.org', 'Rhode Island PBS (WSBE)'),  # http://www.ripbs.org/home/
+        ('wliw.org', 'WLIW21 (WLIW)'),  # http://www.wliw.org/
+        ('mpt.org', 'mpt/Maryland Public Television (WMPB)'),  # http://www.mpt.org
+        ('weta.org', 'WETA Television and Radio (WETA)'),  # http://www.weta.org
+        ('whyy.org', 'WHYY (WHYY)'),  # http://www.whyy.org
+        ('wlvt.org', 'PBS 39 (WLVT)'),  # http://www.wlvt.org/
+        ('wvpt.net', 'WVPT - Your Source for PBS and More! (WVPT)'),  # http://www.wvpt.net
+        ('whut.org', 'Howard University Television (WHUT)'),  # http://www.whut.org
+        ('wedu.org', 'WEDU PBS (WEDU)'),  # http://www.wedu.org
+        ('wgcu.org', 'WGCU Public Media (WGCU)'),  # http://www.wgcu.org/
+        ('wjct.org', 'WJCT Public Broadcasting (WJCT)'),  # http://www.wjct.org
+        ('wpbt2.org', 'WPBT2 (WPBT)'),  # http://www.wpbt2.org
+        ('wucftv.org', 'WUCF TV (WUCF)'),  # http://wucftv.org
+        ('wuft.org', 'WUFT/Channel 5 (WUFT)'),  # http://www.wuft.org
+        ('wxel.org', 'WXEL/Channel 42 (WXEL)'),  # http://www.wxel.org/home/
+        ('wlrn.org', 'WLRN/Channel 17 (WLRN)'),  # http://www.wlrn.org/
+        ('wusf.org', 'WUSF Public Broadcasting (WUSF)'),  # http://wusf.org/
+        ('scetv.org', 'ETV (WRLK)'),  # http://www.scetv.org
+        ('unctv.org', 'UNC-TV (WUNC)'),  # http://www.unctv.org/
+        ('pbsguam.org', 'PBS Guam (KGTF)'),  # http://www.pbsguam.org/
+        ('pbshawaii.org', 'PBS Hawaii - Oceanic Cable Channel 10 (KHET)'),  # http://www.pbshawaii.org/
+        ('idahoptv.org', 'Idaho Public Television (KAID)'),  # http://idahoptv.org
+        ('ksps.org', 'KSPS (KSPS)'),  # http://www.ksps.org/home/
+        ('opb.org', 'OPB (KOPB)'),  # http://www.opb.org
+        ('kwsu.org', 'KWSU/Channel 10 & KTNW/Channel 31 (KWSU)'),  # http://www.kwsu.org
+        ('illinois.edu', 'WILL-TV (WILL)'),  # http://will.illinois.edu/
+        ('wsec.tv', 'Network Knowledge - WSEC/Springfield (WSEC)'),  # http://www.wsec.tv
+        ('wttw.com', 'WTTW11 (WTTW)'),  # http://www.wttw.com/
+        ('wtvp.org', 'WTVP & WTVP.org, Public Media for Central Illinois (WTVP)'),  # http://www.wtvp.org/
+        ('iptv.org', 'Iowa Public Television/IPTV (KDIN)'),  # http://www.iptv.org/
+        ('ninenet.org', 'Nine Network (KETC)'),  # http://www.ninenet.org
+        ('wfwa.org', 'PBS39 Fort Wayne (WFWA)'),  # http://wfwa.org/
+        ('wfyi.org', 'WFYI Indianapolis (WFYI)'),  # http://www.wfyi.org
+        ('mptv.org', 'Milwaukee Public Television (WMVS)'),  # http://www.mptv.org
+        ('wnin.org', 'WNIN (WNIN)'),  # http://www.wnin.org/
+        ('wnit.org', 'WNIT Public Television (WNIT)'),  # http://www.wnit.org/
+        ('wpt.org', 'WPT (WPNE)'),  # http://www.wpt.org/
+        ('wvut.org', 'WVUT/Channel 22 (WVUT)'),  # http://wvut.org/
+        ('weiu.net', 'WEIU/Channel 51 (WEIU)'),  # http://www.weiu.net
+        ('wqpt.org', 'WQPT-TV (WQPT)'),  # http://www.wqpt.org
+        ('wycc.org', 'WYCC PBS Chicago (WYCC)'),  # http://www.wycc.org
+        ('lakeshorepublicmedia.org', 'Lakeshore Public Television (WYIN)'),  # http://lakeshorepublicmedia.org/
+        ('wipb.org', 'WIPB-TV (WIPB)'),  # http://wipb.org
+        ('indianapublicmedia.org', 'WTIU (WTIU)'),  # http://indianapublicmedia.org/tv/
+        ('cetconnect.org', 'CET  (WCET)'),  # http://www.cetconnect.org
+        ('thinktv.org', 'ThinkTVNetwork (WPTD)'),  # http://www.thinktv.org
+        ('wbgu.org', 'WBGU-TV (WBGU)'),  # http://wbgu.org
+        ('wgvu.org', 'WGVU TV (WGVU)'),  # http://www.wgvu.org/
+        ('netnebraska.org', 'NET1 (KUON)'),  # http://netnebraska.org
+        ('pioneer.org', 'Pioneer Public Television (KWCM)'),  # http://www.pioneer.org
+        ('sdpb.org', 'SDPB Television (KUSD)'),  # http://www.sdpb.org
+        ('tpt.org', 'TPT (KTCA)'),  # http://www.tpt.org
+        ('ksmq.org', 'KSMQ (KSMQ)'),  # http://www.ksmq.org/
+        ('kpts.org', 'KPTS/Channel 8 (KPTS)'),  # http://www.kpts.org/
+        ('ktwu.org', 'KTWU/Channel 11 (KTWU)'),  # http://ktwu.org
+        ('shptv.org', 'Smoky Hills Public Television (KOOD)'),  # http://www.shptv.org
+        ('kcpt.org', 'KCPT Kansas City Public Television (KCPT)'),  # http://kcpt.org/
+        ('blueridgepbs.org', 'Blue Ridge PBS (WBRA)'),  # http://www.blueridgepbs.org/
+        ('easttennesseepbs.org', 'East Tennessee PBS (WSJK)'),  # http://easttennesseepbs.org
+        ('wcte.org', 'WCTE-TV (WCTE)'),  # http://www.wcte.org
+        ('wljt.org', 'WLJT, Channel 11 (WLJT)'),  # http://wljt.org/
+        ('wosu.org', 'WOSU TV (WOSU)'),  # http://wosu.org/
+        ('woub.org', 'WOUB/WOUC (WOUB)'),  # http://woub.org/tv/index.php?section=5
+        ('wvpublic.org', 'WVPB (WVPB)'),  # http://wvpublic.org/
+        ('wkyupbs.org', 'WKYU-PBS (WKYU)'),  # http://www.wkyupbs.org
+        ('wyes.org', 'WYES-TV/New Orleans (WYES)'),  # http://www.wyes.org
+        ('kera.org', 'KERA 13 (KERA)'),  # http://www.kera.org/
+        ('mpbn.net', 'MPBN (WCBB)'),  # http://www.mpbn.net/
+        ('mountainlake.org', 'Mountain Lake PBS (WCFE)'),  # http://www.mountainlake.org/
+        ('nhptv.org', 'NHPTV (WENH)'),  # http://nhptv.org/
+        ('vpt.org', 'Vermont PBS (WETK)'),  # http://www.vpt.org
+        ('witf.org', 'witf (WITF)'),  # http://www.witf.org
+        ('wqed.org', 'WQED Multimedia (WQED)'),  # http://www.wqed.org/
+        ('wmht.org', 'WMHT Educational Telecommunications (WMHT)'),  # http://www.wmht.org/home/
+        ('deltabroadcasting.org', 'Q-TV (WDCQ)'),  # http://www.deltabroadcasting.org
+        ('dptv.org', 'WTVS Detroit Public TV (WTVS)'),  # http://www.dptv.org/
+        ('wcmu.org', 'CMU Public Television (WCMU)'),  # http://www.wcmu.org
+        ('wkar.org', 'WKAR-TV (WKAR)'),  # http://wkar.org/
+        ('nmu.edu', 'WNMU-TV Public TV 13 (WNMU)'),  # http://wnmutv.nmu.edu
+        ('wdse.org', 'WDSE - WRPT (WDSE)'),  # http://www.wdse.org/
+        ('wgte.org', 'WGTE TV (WGTE)'),  # http://www.wgte.org
+        ('lakelandptv.org', 'Lakeland Public Television (KAWE)'),  # http://www.lakelandptv.org
+        ('prairiepublic.org', 'PRAIRIE PUBLIC (KFME)'),  # http://www.prairiepublic.org/
+        ('kmos.org', 'KMOS-TV - Channels 6.1, 6.2 and 6.3 (KMOS)'),  # http://www.kmos.org/
+        ('montanapbs.org', 'MontanaPBS (KUSM)'),  # http://montanapbs.org
+        ('krwg.org', 'KRWG/Channel 22 (KRWG)'),  # http://www.krwg.org
+        ('panhandlepbs.org', 'KACV (KACV)'),  # http://www.panhandlepbs.org/home/
+        ('kcostv.org', 'KCOS/Channel 13 (KCOS)'),  # www.kcostv.org
+        ('wcny.org', 'WCNY/Channel 24 (WCNY)'),  # http://www.wcny.org
+        ('wned.org', 'WNED (WNED)'),  # http://www.wned.org/
+        ('wpbstv.org', 'WPBS (WPBS)'),  # http://www.wpbstv.org
+        ('wskg.org', 'WSKG Public TV (WSKG)'),  # http://wskg.org
+        ('wxxi.org', 'WXXI (WXXI)'),  # http://wxxi.org
+        ('wpsu.org', 'WPSU (WPSU)'),  # http://www.wpsu.org
+        ('wqln.org', 'WQLN/Channel 54 (WQLN)'),  # http://www.wqln.org
+        ('wvia.org', 'WVIA Public Media Studios (WVIA)'),  # http://www.wvia.org/
+        ('wtvi.org', 'WTVI (WTVI)'),  # http://www.wtvi.org/
+        ('whro.org', 'WHRO (WHRO)'),  # http://whro.org
+        ('WesternReservePublicMedia.org', 'Western Reserve PBS (WNEO)'),  # http://www.WesternReservePublicMedia.org/
+        ('wviz.org', 'WVIZ/PBS ideastream (WVIZ)'),  # http://www.wviz.org/
+        ('kcts9.org', 'KCTS 9 (KCTS)'),  # http://kcts9.org/
+        ('basinpbs.org', 'Basin PBS (KPBT)'),  # http://www.basinpbs.org
+        ('houstonpublicmedia.org', 'KUHT / Channel 8 (KUHT)'),  # http://www.houstonpublicmedia.org/
+        ('tamu.edu', 'KAMU - TV (KAMU)'),  # http://KAMU.tamu.edu
+        ('kedt.org', 'KEDT/Channel 16 (KEDT)'),  # http://www.kedt.org
+        ('klrn.org', 'KLRN (KLRN)'),  # http://www.klrn.org
+        ('klru.org', 'KLRU (KLRU)'),  # http://www.klru.org
+        ('kmbh.org', 'KMBH-TV (KMBH)'),  # http://www.kmbh.org
+        ('knct.org', 'KNCT (KNCT)'),  # http://www.knct.org
+        ('ktxt.org', 'KTTZ-TV (KTXT)'),  # http://www.ktxt.org
+        ('wtjx.org', 'WTJX Channel 12 (WTJX)'),  # http://www.wtjx.org/
+        ('ideastations.org', 'WCVE PBS (WCVE)'),  # http://ideastations.org/
+        ('kbtc.org', 'KBTC Public Television (KBTC)'),  # http://kbtc.org
     )
 
     IE_NAME = 'pbs'

From 6c7b26e13f1aebe7bd19a7899b01e49b052febe5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 9 Dec 2015 21:28:04 +0600
Subject: [PATCH 3306/6219] [pbs] Make URLs lowercase

---
 youtube_dl/extractor/pbs.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 6aae7f028..d71d7eb76 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -159,7 +159,7 @@ class PBSIE(InfoExtractor):
         ('wvia.org', 'WVIA Public Media Studios (WVIA)'),  # http://www.wvia.org/
         ('wtvi.org', 'WTVI (WTVI)'),  # http://www.wtvi.org/
         ('whro.org', 'WHRO (WHRO)'),  # http://whro.org
-        ('WesternReservePublicMedia.org', 'Western Reserve PBS (WNEO)'),  # http://www.WesternReservePublicMedia.org/
+        ('westernreservepublicmedia.org', 'Western Reserve PBS (WNEO)'),  # http://www.WesternReservePublicMedia.org/
         ('wviz.org', 'WVIZ/PBS ideastream (WVIZ)'),  # http://www.wviz.org/
         ('kcts9.org', 'KCTS 9 (KCTS)'),  # http://kcts9.org/
         ('basinpbs.org', 'Basin PBS (KPBT)'),  # http://www.basinpbs.org

From 374c761e77b09b07918f525ad74724650309015f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Juan=20M=20Mart=C3=ADnez?= <joksnet@gmail.com>
Date: Wed, 9 Dec 2015 20:56:00 -0300
Subject: [PATCH 3307/6219] [FFmpegPostProcessor] Default of prefer ffmpeg

When no `downloader` is passed to `FFmpegPostProcessor`
an exception was raised trying to get the prefer ffmpeg param.

    AttributeError: 'NoneType' object has no attribute 'params'

This fixes and defaults to `False`.
---
 youtube_dl/postprocessor/ffmpeg.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 5ed723bc6..daca5d814 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -52,7 +52,7 @@ class FFmpegPostProcessor(PostProcessor):
 
     def _determine_executables(self):
         programs = ['avprobe', 'avconv', 'ffmpeg', 'ffprobe']
-        prefer_ffmpeg = self._downloader.params.get('prefer_ffmpeg', False)
+        prefer_ffmpeg = False
 
         self.basename = None
         self.probe_basename = None
@@ -60,6 +60,7 @@ class FFmpegPostProcessor(PostProcessor):
         self._paths = None
         self._versions = None
         if self._downloader:
+            prefer_ffmpeg = self._downloader.params.get('prefer_ffmpeg', False)
             location = self._downloader.params.get('ffmpeg_location')
             if location is not None:
                 if not os.path.exists(location):

From 46047c58d033f71839477d338e128b459d4d4376 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?David=20Haberthu=CC=88r?= <email@davidhaberthuer.ch>
Date: Thu, 10 Dec 2015 13:17:31 +0100
Subject: [PATCH 3308/6219] Updating README.md

- Harmonizing mentions of **youtube-dl** in the text
- Removing unnecessary Markdown markup for headers
- Adding some links
---
 README.md | 110 +++++++++++++++++++++++++++---------------------------
 1 file changed, 55 insertions(+), 55 deletions(-)

diff --git a/README.md b/README.md
index e39f71281..7a4a4f50f 100644
--- a/README.md
+++ b/README.md
@@ -1,4 +1,4 @@
-youtube-dl - download videos from youtube.com or other video platforms
+**youtube-dl** - download videos from youtube.com or other video platforms
 
 - [INSTALLATION](#installation)
 - [DESCRIPTION](#description)
@@ -35,7 +35,7 @@ You can also use pip:
 
     sudo pip install youtube-dl
 
-Alternatively, refer to the [developer instructions](#developer-instructions) for how to check out and work with the git repository. For further options, including PGP signatures, see https://rg3.github.io/youtube-dl/download.html .
+Alternatively, refer to the [developer instructions](#developer-instructions) for how to check out and work with the git repository. For further options, including PGP signatures, see the [**youtube-dl** Download Page](https://rg3.github.io/youtube-dl/download.html).
 
 # DESCRIPTION
 **youtube-dl** is a small command-line program to download videos from
@@ -405,18 +405,18 @@ which means you can modify it, redistribute it or use it however you like.
 
 # CONFIGURATION
 
-You can configure youtube-dl by placing any supported command line option to a configuration file. On Linux, the system wide configuration file is located at `/etc/youtube-dl.conf` and the user wide configuration file at `~/.config/youtube-dl/config`. On Windows, the user wide configuration file locations are `%APPDATA%\youtube-dl\config.txt` or `C:\Users\<user name>\youtube-dl.conf`. For example, with the following configuration file youtube-dl will always extract the audio, not copy the mtime and use a proxy:
+You can configure **youtube-dl** by placing any supported command line option to a configuration file. On Linux, the system wide configuration file is located at `/etc/youtube-dl.conf` and the user wide configuration file at `~/.config/youtube-dl/config`. On Windows, the user wide configuration file locations are `%APPDATA%\youtube-dl\config.txt` or `C:\Users\<user name>\youtube-dl.conf`. For example, with the following configuration file **youtube-dl** will always extract the audio, not copy the mtime and use a proxy:
 ```
 --extract-audio
 --no-mtime
 --proxy 127.0.0.1:3128
 ```
 
-You can use `--ignore-config` if you want to disable the configuration file for a particular youtube-dl run.
+You can use `--ignore-config` if you want to disable the configuration file for a particular **youtube-dl** run.
 
-### Authentication with `.netrc` file ###
+### Authentication with `.netrc` file
 
-You may also want to configure automatic credentials storage for extractors that support authentication (by providing login and password with `--username` and `--password`) in order not to pass credentials as command line arguments on every youtube-dl execution and prevent tracking plain text passwords in the shell command history. You can achieve this using a [`.netrc` file](http://stackoverflow.com/tags/.netrc/info) on per extractor basis. For that you will need to create a`.netrc` file in your `$HOME` and restrict permissions to read/write by you only:
+You may also want to configure automatic credentials storage for extractors that support authentication (by providing login and password with `--username` and `--password`) in order not to pass credentials as command line arguments on every **youtube-dl** execution and prevent tracking plain text passwords in the shell command history. You can achieve this using a [`.netrc` file](http://stackoverflow.com/tags/.netrc/info) on per extractor basis. For that you will need to create a`.netrc` file in your `$HOME` and restrict permissions to read/write by you only:
 ```
 touch $HOME/.netrc
 chmod a-rwx,u+rw $HOME/.netrc
@@ -430,7 +430,7 @@ For example:
 machine youtube login myaccount@gmail.com password my_youtube_password
 machine twitch login my_twitch_account_name password my_twitch_password
 ```
-To activate authentication with the `.netrc` file you should pass `--netrc` to youtube-dl or place it in the [configuration file](#configuration).
+To activate authentication with the `.netrc` file you should pass `--netrc` to **youtube-dl** or place it in the [configuration file](#configuration).
 
 On Windows you may also need to setup the `%HOME%` environment variable manually.
 
@@ -464,14 +464,14 @@ youtube-dl_test_video_.mp4          # A simple file name
 
 # FORMAT SELECTION
 
-By default youtube-dl tries to download the best quality, but sometimes you may want to download in a different format.
+By default **youtube-dl** tries to download the best quality, but sometimes you may want to download in a different format.
 The simplest case is requesting a specific format, for example `-f 22`. You can get the list of available formats using `--list-formats`, you can also use a file extension (currently it supports aac, m4a, mp3, mp4, ogg, wav, webm) or the special names `best`, `bestvideo`, `bestaudio` and `worst`.
 
 If you want to download multiple videos and they don't have the same formats available, you can specify the order of preference using slashes, as in `-f 22/17/18`. You can also filter the video results by putting a condition in brackets, as in `-f "best[height=720]"` (or `-f "[filesize>10M]"`).  This works for filesize, height, width, tbr, abr, vbr, asr, and fps and the comparisons <, <=, >, >=, =, != and for ext, acodec, vcodec, container, and protocol and the comparisons =, != . Formats for which the value is not known are excluded unless you put a question mark (?) after the operator. You can combine format filters, so  `-f "[height <=? 720][tbr>500]"` selects up to 720p videos (or videos where the height is not known) with a bitrate of at least 500 KBit/s. Use commas to download multiple formats, such as `-f 136/137/mp4/bestvideo,140/m4a/bestaudio`. You can merge the video and audio of two formats into a single file using `-f <video-format>+<audio-format>` (requires ffmpeg or avconv), for example `-f bestvideo+bestaudio`. Format selectors can also be grouped using parentheses, for example if you want to download the best mp4 and webm formats with a height lower than 480 you can use `-f '(mp4,webm)[height<480]'`.
 
-Since the end of April 2015 and version 2015.04.26 youtube-dl uses `-f bestvideo+bestaudio/best` as default format selection (see #5447, #5456). If ffmpeg or avconv are installed this results in downloading `bestvideo` and `bestaudio` separately and muxing them together into a single file giving the best overall quality available. Otherwise it falls back to `best` and results in downloading the best available quality served as a single file. `best` is also needed for videos that don't come from YouTube because they don't provide the audio and video in two different files. If you want to only download some dash formats (for example if you are not interested in getting videos with a resolution higher than 1080p), you can add `-f bestvideo[height<=?1080]+bestaudio/best` to your configuration file. Note that if you use youtube-dl to stream to `stdout` (and most likely to pipe it to your media player then), i.e. you explicitly specify output template as `-o -`, youtube-dl still uses `-f best` format selection in order to start content delivery immediately to your player and not to wait until `bestvideo` and `bestaudio` are downloaded and muxed.
+Since the end of April 2015 and version 2015.04.26 **youtube-dl** uses `-f bestvideo+bestaudio/best` as default format selection (see #5447, #5456). If ffmpeg or avconv are installed this results in downloading `bestvideo` and `bestaudio` separately and muxing them together into a single file giving the best overall quality available. Otherwise it falls back to `best` and results in downloading the best available quality served as a single file. `best` is also needed for videos that don't come from YouTube because they don't provide the audio and video in two different files. If you want to only download some dash formats (for example if you are not interested in getting videos with a resolution higher than 1080p), you can add `-f bestvideo[height<=?1080]+bestaudio/best` to your configuration file. Note that if you use **youtube-dl** to stream to `stdout` (and most likely to pipe it to your media player then), i.e. you explicitly specify output template as `-o -`, **youtube-dl** still uses `-f best` format selection in order to start content delivery immediately to your player and not to wait until `bestvideo` and `bestaudio` are downloaded and muxed.
 
-If you want to preserve the old format selection behavior (prior to youtube-dl 2015.04.26), i.e. you want to download the best available quality media served as a single file, you should explicitly specify your choice with `-f best`. You may want to add it to the [configuration file](#configuration) in order not to type it every time you run youtube-dl.
+If you want to preserve the old format selection behavior (prior to **youtube-dl** 2015.04.26), i.e. you want to download the best available quality media served as a single file, you should explicitly specify your choice with `-f best`. You may want to add it to the [configuration file](#configuration) in order not to type it every time you run **youtube-dl**.
 
 # VIDEO SELECTION
 
@@ -501,7 +501,7 @@ If you've followed [our manual installation instructions](http://rg3.github.io/y
 
 If you have used pip, a simple `sudo pip install -U youtube-dl` is sufficient to update.
 
-If you have installed youtube-dl using a package manager like *apt-get* or *yum*, use the standard system update mechanism to update. Note that distribution packages are often outdated. As a rule of thumb, youtube-dl releases at least once a month, and often weekly or even daily. Simply go to http://yt-dl.org/ to find out the current version. Unfortunately, there is nothing we youtube-dl developers can do if your distribution serves a really outdated version. You can (and should) complain to your distribution in their bugtracker or support forum.
+If you have installed **youtube-dl** using a package manager like *apt-get* or *yum*, use the standard system update mechanism to update. Note that distribution packages are often outdated. As a rule of thumb, **youtube-dl** releases at least once a month, and often weekly or even daily. Simply go to http://yt-dl.org/ to find out the current version. Unfortunately, there is nothing we **youtube-dl** developers can do if your distribution serves a really outdated version. You can (and should) complain to your distribution in their bugtracker or support forum.
 
 As a last resort, you can also uninstall the version installed by your package manager and follow our manual installation instructions. For that, remove the distribution's package, with a line like
 
@@ -519,25 +519,25 @@ Again, from then on you'll be able to update with `sudo youtube-dl -U`.
 
 ### I'm getting an error `Unable to extract OpenGraph title` on YouTube playlists
 
-YouTube changed their playlist format in March 2014 and later on, so you'll need at least youtube-dl 2014.07.25 to download all YouTube videos.
+YouTube changed their playlist format in March 2014 and later on, so you'll need at least **youtube-dl** 2014.07.25 to download all YouTube videos.
 
-If you have installed youtube-dl with a package manager, pip, setup.py or a tarball, please use that to update. Note that Ubuntu packages do not seem to get updated anymore. Since we are not affiliated with Ubuntu, there is little we can do. Feel free to [report bugs](https://bugs.launchpad.net/ubuntu/+source/youtube-dl/+filebug) to the [Ubuntu packaging guys](mailto:ubuntu-motu@lists.ubuntu.com?subject=outdated%20version%20of%20youtube-dl) - all they have to do is update the package to a somewhat recent version. See above for a way to update.
+If you have installed **youtube-dl** with a package manager, pip, setup.py or a tarball, please use that to update. Note that Ubuntu packages do not seem to get updated anymore. Since we are not affiliated with Ubuntu, there is little we can do. Feel free to [report bugs](https://bugs.launchpad.net/ubuntu/+source/youtube-dl/+filebug) to the [Ubuntu packaging guys](mailto:ubuntu-motu@lists.ubuntu.com?subject=outdated%20version%20of%20youtube-dl) - all they have to do is update the package to a somewhat recent version. See above for a way to update.
 
 ### Do I always have to pass `-citw`?
 
-By default, youtube-dl intends to have the best options (incidentally, if you have a convincing case that these should be different, [please file an issue where you explain that](https://yt-dl.org/bug)). Therefore, it is unnecessary and sometimes harmful to copy long option strings from webpages. In particular, the only option out of `-citw` that is regularly useful is `-i`.
+By default, **youtube-dl** intends to have the best options (incidentally, if you have a convincing case that these should be different, [please file an issue where you explain that](https://yt-dl.org/bug)). Therefore, it is unnecessary and sometimes harmful to copy long option strings from webpages. In particular, the only option out of `-citw` that is regularly useful is `-i`.
 
 ### Can you please put the `-b` option back?
 
-Most people asking this question are not aware that youtube-dl now defaults to downloading the highest available quality as reported by YouTube, which will be 1080p or 720p in some cases, so you no longer need the `-b` option. For some specific videos, maybe YouTube does not report them to be available in a specific high quality format you're interested in. In that case, simply request it with the `-f` option and youtube-dl will try to download it.
+Most people asking this question are not aware that **youtube-dl** now defaults to downloading the highest available quality as reported by YouTube, which will be 1080p or 720p in some cases, so you no longer need the `-b` option. For some specific videos, maybe YouTube does not report them to be available in a specific high quality format you're interested in. In that case, simply request it with the `-f` option and **youtube-dl** will try to download it.
 
 ### I get HTTP error 402 when trying to download a video. What's this?
 
-Apparently YouTube requires you to pass a CAPTCHA test if you download too much. We're [considering to provide a way to let you solve the CAPTCHA](https://github.com/rg3/youtube-dl/issues/154), but at the moment, your best course of action is pointing a webbrowser to the youtube URL, solving the CAPTCHA, and restart youtube-dl.
+Apparently YouTube requires you to pass a CAPTCHA test if you download too much. We're [considering to provide a way to let you solve the CAPTCHA](https://github.com/rg3/youtube-dl/issues/154), but at the moment, your best course of action is pointing a webbrowser to the youtube URL, solving the CAPTCHA, and restart **youtube-dl**.
 
 ### Do I need any other programs?
 
-youtube-dl works fine on its own on most sites. However, if you want to convert video/audio, you'll need [avconv](https://libav.org/) or [ffmpeg](https://www.ffmpeg.org/). On some sites - most notably YouTube - videos can be retrieved in a higher quality format without sound. youtube-dl will detect whether avconv/ffmpeg is present and automatically pick the best option.
+**youtube-dl** works fine on its own on most sites. However, if you want to convert video/audio, you'll need [avconv](https://libav.org/) or [ffmpeg](https://www.ffmpeg.org/). On some sites - most notably YouTube - videos can be retrieved in a higher quality format without sound. **youtube-dl** will detect whether avconv/ffmpeg is present and automatically pick the best option.
 
 Videos or video formats streamed via RTMP protocol can only be downloaded when [rtmpdump](https://rtmpdump.mplayerhq.hu/) is installed. Downloading MMS and RTSP videos requires either [mplayer](http://mplayerhq.hu/) or [mpv](https://mpv.io/) to be installed.
 
@@ -547,25 +547,25 @@ Once the video is fully downloaded, use any video player, such as [vlc](http://w
 
 ### I extracted a video URL with `-g`, but it does not play on another machine / in my webbrowser.
 
-It depends a lot on the service. In many cases, requests for the video (to download/play it) must come from the same IP address and with the same cookies.  Use the `--cookies` option to write the required cookies into a file, and advise your downloader to read cookies from that file. Some sites also require a common user agent to be used, use `--dump-user-agent` to see the one in use by youtube-dl.
+It depends a lot on the service. In many cases, requests for the video (to download/play it) must come from the same IP address and with the same cookies.  Use the `--cookies` option to write the required cookies into a file, and advise your downloader to read cookies from that file. Some sites also require a common user agent to be used, use `--dump-user-agent` to see the one in use by **youtube-dl**.
 
 It may be beneficial to use IPv6; in some cases, the restrictions are only applied to IPv4. Some services (sometimes only for a subset of videos) do not restrict the video URL by IP address, cookie, or user-agent, but these are the exception rather than the rule.
 
 Please bear in mind that some URL protocols are **not** supported by browsers out of the box, including RTMP. If you are using `-g`, your own downloader must support these as well.
 
-If you want to play the video on a machine that is not running youtube-dl, you can relay the video content from the machine that runs youtube-dl. You can use `-o -` to let youtube-dl stream a video to stdout, or simply allow the player to download the files written by youtube-dl in turn.
+If you want to play the video on a machine that is not running **youtube-dl**, you can relay the video content from the machine that runs **youtube-dl**. You can use `-o -` to let **youtube-dl** stream a video to stdout, or simply allow the player to download the files written by **youtube-dl** in turn.
 
 ### ERROR: no fmt_url_map or conn information found in video info
 
-YouTube has switched to a new video info format in July 2011 which is not supported by old versions of youtube-dl. See [above](#how-do-i-update-youtube-dl) for how to update youtube-dl.
+YouTube has switched to a new video info format in July 2011 which is not supported by old versions of **youtube-dl**. See [above](#how-do-i-update-youtube-dl) for how to update **youtube-dl**.
 
-### ERROR: unable to download video ###
+### ERROR: unable to download video
 
-YouTube requires an additional signature since September 2012 which is not supported by old versions of youtube-dl. See [above](#how-do-i-update-youtube-dl) for how to update youtube-dl.
+YouTube requires an additional signature since September 2012 which is not supported by old versions of youtube-dl. See [above](#how-do-i-update-youtube-dl) for how to update **youtube-dl**.
 
-### Video URL contains an ampersand and I'm getting some strange output `[1] 2839` or `'v' is not recognized as an internal or external command` ###
+### Video URL contains an ampersand and I'm getting some strange output `[1] 2839` or `'v' is not recognized as an internal or external command`
 
-That's actually the output from your shell. Since ampersand is one of the special shell characters it's interpreted by the shell preventing you from passing the whole URL to youtube-dl. To disable your shell from interpreting the ampersands (or any other special characters) you have to either put the whole URL in quotes or escape them with a backslash (which approach will work depends on your shell).
+That's actually the output from your shell. Since ampersand is one of the special shell characters it's interpreted by the shell preventing you from passing the whole URL to **youtube-dl**. To disable your shell from interpreting the ampersands (or any other special characters) you have to either put the whole URL in quotes or escape them with a backslash (which approach will work depends on your shell).
 
 For example if your URL is https://www.youtube.com/watch?t=4&v=BaW_jenozKc you should end up with following command:
 
@@ -581,13 +581,13 @@ For Windows you have to use the double quotes:
 
 ### ExtractorError: Could not find JS function u'OF'
 
-In February 2015, the new YouTube player contained a character sequence in a string that was misinterpreted by old versions of youtube-dl. See [above](#how-do-i-update-youtube-dl) for how to update youtube-dl.
+In February 2015, the new YouTube player contained a character sequence in a string that was misinterpreted by old versions of **youtube-dl**. See [above](#how-do-i-update-youtube-dl) for how to update **youtube-dl**.
 
 ### HTTP Error 429: Too Many Requests or 402: Payment Required
 
 These two error codes indicate that the service is blocking your IP address because of overuse. Contact the service and ask them to unblock your IP address, or - if you have acquired a whitelisted IP address already - use the [`--proxy` or `--source-address` options](#network-options) to select another IP address.
 
-### SyntaxError: Non-ASCII character ###
+### SyntaxError: Non-ASCII character
 
 The error
 
@@ -598,7 +598,7 @@ means you're using an outdated version of Python. Please update to Python 2.6 or
 
 ### What is this binary file? Where has the code gone?
 
-Since June 2012 (#342) youtube-dl is packed as an executable zipfile, simply unzip it (might need renaming to `youtube-dl.zip` first on some systems) or clone the git repository, as laid out above. If you modify the code, you can run it by executing the `__main__.py` file. To recompile the executable, run `make youtube-dl`.
+Since June 2012 (#342) **youtube-dl** is packed as an executable zipfile, simply unzip it (might need renaming to `youtube-dl.zip` first on some systems) or clone the git repository, as laid out above. If you modify the code, you can run it by executing the `__main__.py` file. To recompile the executable, run `make youtube-dl`.
 
 ### The exe throws a *Runtime error from Visual C++*
 
@@ -606,17 +606,17 @@ To run the exe you need to install first the [Microsoft Visual C++ 2008 Redistri
 
 ### On Windows, how should I set up ffmpeg and youtube-dl? Where should I put the exe files?
 
-If you put youtube-dl and ffmpeg in the same directory that you're running the command from, it will work, but that's rather cumbersome.
+If you put **youtube-dl** and ffmpeg in the same directory that you're running the command from, it will work, but that's rather cumbersome.
 
-To make a different directory work - either for ffmpeg, or for youtube-dl, or for both - simply create the directory (say, `C:\bin`, or `C:\Users\<User name>\bin`), put all the executables directly in there, and then [set your PATH environment variable](https://www.java.com/en/download/help/path.xml) to include that directory.
+To make a different directory work - either for ffmpeg, or for **youtube-dl**, or for both - simply create the directory (say, `C:\bin`, or `C:\Users\<User name>\bin`), put all the executables directly in there, and then [set your PATH environment variable](https://www.java.com/en/download/help/path.xml) to include that directory.
 
-From then on, after restarting your shell, you will be able to access both youtube-dl and ffmpeg (and youtube-dl will be able to find ffmpeg) by simply typing `youtube-dl` or `ffmpeg`, no matter what directory you're in.
+From then on, after restarting your shell, you will be able to access both **youtube-dl** and ffmpeg (and **youtube-dl** will be able to find ffmpeg) by simply typing `youtube-dl` or `ffmpeg`, no matter what directory you're in.
 
 ### How do I put downloads into a specific folder?
 
 Use the `-o` to specify an [output template](#output-template), for example `-o "/home/user/videos/%(title)s-%(id)s.%(ext)s"`. If you want this for all of your downloads, put the option into your [configuration file](#configuration).
 
-### How do I download a video starting with a `-` ?
+### How do I download a video starting with a `-`?
 
 Either prepend `http://www.youtube.com/watch?v=` or separate the ID from the options with `--`:
 
@@ -627,41 +627,41 @@ Either prepend `http://www.youtube.com/watch?v=` or separate the ID from the opt
 
 Use the `--cookies` option, for example `--cookies /path/to/cookies/file.txt`. Note that the cookies file must be in Mozilla/Netscape format and the first line of the cookies file must be either `# HTTP Cookie File` or `# Netscape HTTP Cookie File`. Make sure you have correct [newline format](https://en.wikipedia.org/wiki/Newline) in the cookies file and convert newlines if necessary to correspond with your OS, namely `CRLF` (`\r\n`) for Windows, `LF` (`\n`) for Linux and `CR` (`\r`) for Mac OS. `HTTP Error 400: Bad Request` when using `--cookies` is a good sign of invalid newline format.
 
-Passing cookies to youtube-dl is a good way to workaround login when a particular extractor does not implement it explicitly.
+Passing cookies to **youtube-dl** is a good way to workaround login when a particular extractor does not implement it explicitly.
 
 ### Can you add support for this anime video site, or site which shows current movies for free?
 
-As a matter of policy (as well as legality), youtube-dl does not include support for services that specialize in infringing copyright. As a rule of thumb, if you cannot easily find a video that the service is quite obviously allowed to distribute (i.e. that has been uploaded by the creator, the creator's distributor, or is published under a free license), the service is probably unfit for inclusion to youtube-dl.
+As a matter of policy (as well as legality), **youtube-dl** does not include support for services that specialize in infringing copyright. As a rule of thumb, if you cannot easily find a video that the service is quite obviously allowed to distribute (i.e. that has been uploaded by the creator, the creator's distributor, or is published under a free license), the service is probably unfit for inclusion to **youtube-dl**.
 
-A note on the service that they don't host the infringing content, but just link to those who do, is evidence that the service should **not** be included into youtube-dl. The same goes for any DMCA note when the whole front page of the service is filled with videos they are not allowed to distribute. A "fair use" note is equally unconvincing if the service shows copyright-protected videos in full without authorization.
+A note on the service that they don't host the infringing content, but just link to those who do, is evidence that the service should **not** be included into **youtube-dl**. The same goes for any DMCA note when the whole front page of the service is filled with videos they are not allowed to distribute. A "fair use" note is equally unconvincing if the service shows copyright-protected videos in full without authorization.
 
 Support requests for services that **do** purchase the rights to distribute their content are perfectly fine though. If in doubt, you can simply include a source that mentions the legitimate purchase of content.
 
 ### How can I speed up work on my issue?
 
-(Also known as: Help, my important issue not being solved!) The youtube-dl core developer team is quite small. While we do our best to solve as many issues as possible, sometimes that can take quite a while. To speed up your issue, here's what you can do:
+(Also known as: Help, my important issue not being solved!) The **youtube-dl** core developer team is quite small. While we do our best to solve as many issues as possible, sometimes that can take quite a while. To speed up your issue, here's what you can do:
 
-First of all, please do report the issue [at our issue tracker](https://yt-dl.org/bugs). That allows us to coordinate all efforts by users and developers, and serves as a unified point. Unfortunately, the youtube-dl project has grown too large to use personal email as an effective communication channel.
+First of all, please do report the issue [at our issue tracker](https://yt-dl.org/bugs). That allows us to coordinate all efforts by users and developers, and serves as a unified point. Unfortunately, the **youtube-dl** project has grown too large to use personal email as an effective communication channel.
 
-Please read the [bug reporting instructions](#bugs) below. A lot of bugs lack all the necessary information. If you can, offer proxy, VPN, or shell access to the youtube-dl developers. If you are able to, test the issue from multiple computers in multiple countries to exclude local censorship or misconfiguration issues.
+Please read the [bug reporting instructions](#bugs) below. A lot of bugs lack all the necessary information. If you can, offer proxy, VPN, or shell access to the **youtube-dl** developers. If you are able to, test the issue from multiple computers in multiple countries to exclude local censorship or misconfiguration issues.
 
 If nobody is interested in solving your issue, you are welcome to take matters into your own hands and submit a pull request (or coerce/pay somebody else to do so).
 
-Feel free to bump the issue from time to time by writing a small comment ("Issue is still present in youtube-dl version ...from France, but fixed from Belgium"), but please not more than once a month. Please do not declare your issue as `important` or `urgent`.
+Feel free to bump the issue from time to time by writing a small comment ("Issue is still present in **youtube-dl** version ...from France, but fixed from Belgium"), but please not more than once a month. Please do not declare your issue as `important` or `urgent`.
 
 ### How can I detect whether a given URL is supported by youtube-dl?
 
-For one, have a look at the [list of supported sites](docs/supportedsites.md). Note that it can sometimes happen that the site changes its URL scheme (say, from http://example.com/video/1234567 to http://example.com/v/1234567 ) and youtube-dl reports an URL of a service in that list as unsupported. In that case, simply report a bug.
+For one, have a look at the [list of supported sites](docs/supportedsites.md). Note that it can sometimes happen that the site changes its URL scheme (say, from http://example.com/video/1234567 to http://example.com/v/1234567 ) and **youtube-dl** reports an URL of a service in that list as unsupported. In that case, simply report a bug.
 
-It is *not* possible to detect whether a URL is supported or not. That's because youtube-dl contains a generic extractor which matches **all** URLs. You may be tempted to disable, exclude, or remove the generic extractor, but the generic extractor not only allows users to extract videos from lots of websites that embed a video from another service, but may also be used to extract video from a service that it's hosting itself. Therefore, we neither recommend nor support disabling, excluding, or removing the generic extractor.
+It is *not* possible to detect whether a URL is supported or not. That's because **youtube-dl** contains a generic extractor which matches **all** URLs. You may be tempted to disable, exclude, or remove the generic extractor, but the generic extractor not only allows users to extract videos from lots of websites that embed a video from another service, but may also be used to extract video from a service that it's hosting itself. Therefore, we neither recommend nor support disabling, excluding, or removing the generic extractor.
 
-If you want to find out whether a given URL is supported, simply call youtube-dl with it. If you get no videos back, chances are the URL is either not referring to a video or unsupported. You can find out which by examining the output (if you run youtube-dl on the console) or catching an `UnsupportedError` exception if you run it from a Python program.
+If you want to find out whether a given URL is supported, simply call **youtube-dl** with it. If you get no videos back, chances are the URL is either not referring to a video or unsupported. You can find out which by examining the output (if you run **youtube-dl** on the console) or catching an `UnsupportedError` exception if you run it from a Python program.
 
 # DEVELOPER INSTRUCTIONS
 
-Most users do not need to build youtube-dl and can [download the builds](http://rg3.github.io/youtube-dl/download.html) or get them from their distribution.
+Most users do not need to build **youtube-dl** and can [download the builds](http://rg3.github.io/youtube-dl/download.html) or get them from their distribution.
 
-To run youtube-dl as a developer, you don't need to build anything either. Simply execute
+To run **youtube-dl** as a developer, you don't need to build anything either. Simply execute
 
     python -m youtube_dl
 
@@ -671,7 +671,7 @@ To run the test, simply invoke your favorite test runner, or execute a test file
     python test/test_download.py
     nosetests
 
-If you want to create a build of youtube-dl yourself, you'll need
+If you want to create a build of **youtube-dl** yourself, you'll need
 
 * python
 * make
@@ -744,9 +744,9 @@ In any case, thank you very much for your contributions!
 
 # EMBEDDING YOUTUBE-DL
 
-youtube-dl makes the best effort to be a good command-line program, and thus should be callable from any programming language. If you encounter any problems parsing its output, feel free to [create a report](https://github.com/rg3/youtube-dl/issues/new).
+**youtube-dl** makes the best effort to be a good command-line program, and thus should be callable from any programming language. If you encounter any problems parsing its output, feel free to [create a report](https://github.com/rg3/youtube-dl/issues/new).
 
-From a Python program, you can embed youtube-dl in a more powerful fashion, like this:
+From a Python program, you can embed **youtube-dl** in a more powerful fashion, like this:
 
 ```python
 from __future__ import unicode_literals
@@ -757,7 +757,7 @@ with youtube_dl.YoutubeDL(ydl_opts) as ydl:
     ydl.download(['http://www.youtube.com/watch?v=BaW_jenozKc'])
 ```
 
-Most likely, you'll want to use various options. For a list of what can be done, have a look at [youtube_dl/YoutubeDL.py](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/YoutubeDL.py#L117-L265). For a start, if you want to intercept youtube-dl's output, set a `logger` object.
+Most likely, you'll want to use various options. For a list of what can be done, have a look at [youtube_dl/YoutubeDL.py](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/YoutubeDL.py#L117-L265). For a start, if you want to intercept **youtube-dl**'s output, set a `logger` object.
 
 Here's a more complete example of a program that outputs only errors (and a short message after the download is finished), and downloads/converts the video to an mp3 file:
 
@@ -798,7 +798,7 @@ with youtube_dl.YoutubeDL(ydl_opts) as ydl:
 
 # BUGS
 
-Bugs and suggestions should be reported at: <https://github.com/rg3/youtube-dl/issues> . Unless you were prompted so or there is another pertinent reason (e.g. GitHub fails to accept the bug report), please do not send bug reports via personal email. For discussions, join us in the irc channel #youtube-dl on freenode.
+Bugs and suggestions should be reported at: <https://github.com/rg3/youtube-dl/issues>. Unless you were prompted so or there is another pertinent reason (e.g. GitHub fails to accept the bug report), please do not send bug reports via personal email. For discussions, join us in the IRC channel [#youtube-dl](irc://chat.freenode.net/#youtube-dl) on freenode ([webchat](http://webchat.freenode.net/?randomnick=1&channels=youtube-dl)).
 
 **Please include the full output of youtube-dl when run with `-v`**, i.e. add `-v` flag to your command line, copy the **whole** output and post it in the issue body wrapped in \`\`\` for better formatting. It should look similar to this:
 ```
@@ -832,11 +832,11 @@ So please elaborate on what feature you are requesting, or what bug you want to
 
 If your report is shorter than two lines, it is almost certainly missing some of these, which makes it hard for us to respond to it. We're often too polite to close the issue outright, but the missing info makes misinterpretation likely. As a commiter myself, I often get frustrated by these issues, since the only possible way for me to move forward on them is to ask for clarification over and over.
 
-For bug reports, this means that your report should contain the *complete* output of youtube-dl when called with the `-v` flag. The error message you get for (most) bugs even says so, but you would not believe how many of our bug reports do not contain this information.
+For bug reports, this means that your report should contain the *complete* output of **youtube-dl** when called with the `-v` flag. The error message you get for (most) bugs even says so, but you would not believe how many of our bug reports do not contain this information.
 
 If your server has multiple IPs or you suspect censorship, adding `--call-home` may be a good idea to get more diagnostics. If the error is `ERROR: Unable to extract ...` and you cannot reproduce it from multiple countries, add `--dump-pages` (warning: this will yield a rather large output, redirect it to the file `log.txt` by adding `>log.txt 2>&1` to your command-line) or upload the `.dump` files you get when you add `--write-pages` [somewhere](https://gist.github.com/).
 
-**Site support requests must contain an example URL**. An example URL is a URL you might want to download, like http://www.youtube.com/watch?v=BaW_jenozKc . There should be an obvious video present. Except under very special circumstances, the main page of a video service (e.g. http://www.youtube.com/ ) is *not* an example URL.
+**Site support requests must contain an example URL**. An example URL is a URL you might want to download, like `http://www.youtube.com/watch?v=BaW_jenozKc`. There should be an obvious video present. Except under very special circumstances, the main page of a video service (e.g. `http://www.youtube.com/`) is *not* an example URL.
 
 ###  Are you using the latest version?
 
@@ -844,7 +844,7 @@ Before reporting any issue, type `youtube-dl -U`. This should report that you're
 
 ###  Is the issue already documented?
 
-Make sure that someone has not already opened the issue you're trying to open. Search at the top of the window or at https://github.com/rg3/youtube-dl/search?type=Issues . If there is an issue, feel free to write something along the lines of "This affects me as well, with version 2015.01.01. Here is some more information on the issue: ...". While some issues may be old, a new post into them often spurs rapid activity.
+Make sure that someone has not already opened the issue you're trying to open. Search at the top of the window or browse the [GitHub Issues](https://github.com/rg3/youtube-dl/search?type=Issues) of this repository. If there is an issue, feel free to write something along the lines of "This affects me as well, with version 2015.01.01. Here is some more information on the issue: ...". While some issues may be old, a new post into them often spurs rapid activity.
 
 ###  Why are existing options not enough?
 
@@ -860,7 +860,7 @@ We are then presented with a very complicated request when the original problem
 
 Some of our users seem to think there is a limit of issues they can or should open. There is no limit of issues they can or should open. While it may seem appealing to be able to dump all your issues into one ticket, that means that someone who solves one of your issues cannot mark the issue as closed. Typically, reporting a bunch of issues leads to the ticket lingering since nobody wants to attack that behemoth, until someone mercifully splits the issue into multiple ones.
 
-In particular, every site support request issue should only pertain to services at one site (generally under a common domain, but always using the same backend technology). Do not request support for vimeo user videos, Whitehouse podcasts, and Google Plus pages in the same issue. Also, make sure that you don't post bug reports alongside feature requests. As a rule of thumb, a feature request does not include outputs of youtube-dl that are not immediately related to the feature at hand. Do not post reports of a network error alongside the request for a new video service.
+In particular, every site support request issue should only pertain to services at one site (generally under a common domain, but always using the same backend technology). Do not request support for vimeo user videos, Whitehouse podcasts, and Google Plus pages in the same issue. Also, make sure that you don't post bug reports alongside feature requests. As a rule of thumb, a feature request does not include outputs of **youtube-dl** that are not immediately related to the feature at hand. Do not post reports of a network error alongside the request for a new video service.
 
 ###  Is anyone going to need the feature?
 
@@ -868,10 +868,10 @@ Only post features that you (or an incapacitated friend you can personally talk
 
 ###  Is your question about youtube-dl?
 
-It may sound strange, but some bug reports we receive are completely unrelated to youtube-dl and relate to a different or even the reporter's own application. Please make sure that you are actually using youtube-dl. If you are using a UI for youtube-dl, report the bug to the maintainer of the actual application providing the UI. On the other hand, if your UI for youtube-dl fails in some way you believe is related to youtube-dl, by all means, go ahead and report the bug.
+It may sound strange, but some bug reports we receive are completely unrelated to **youtube-dl** and relate to a different or even the reporter's own application. Please make sure that you are actually using **youtube-dl**. If you are using a UI for **youtube-dl**, report the bug to the maintainer of the actual application providing the UI. On the other hand, if your UI for **youtube-dl** fails in some way you believe is related to **youtube-dl**, by all means, go ahead and report the bug.
 
 # COPYRIGHT
 
-youtube-dl is released into the public domain by the copyright holders.
+**youtube-dl** is released into the public domain by the copyright holders.
 
-This README file was originally written by Daniel Bolton (<https://github.com/dbbolton>) and is likewise released into the public domain.
+This README file was originally written by [Daniel Bolton](https://github.com/dbbolton) and is likewise released into the public domain.

From a1b8d815f5716783f5ec9c2b4f60dcdd4d8192d1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?David=20Haberthu=CC=88r?= <email@davidhaberthuer.ch>
Date: Thu, 10 Dec 2015 13:45:53 +0100
Subject: [PATCH 3309/6219] Reverting markup changes

---
 README.md | 92 +++++++++++++++++++++++++++----------------------------
 1 file changed, 46 insertions(+), 46 deletions(-)

diff --git a/README.md b/README.md
index 7a4a4f50f..5c3915d64 100644
--- a/README.md
+++ b/README.md
@@ -1,4 +1,4 @@
-**youtube-dl** - download videos from youtube.com or other video platforms
+youtube-dl - download videos from youtube.com or other video platforms
 
 - [INSTALLATION](#installation)
 - [DESCRIPTION](#description)
@@ -35,7 +35,7 @@ You can also use pip:
 
     sudo pip install youtube-dl
 
-Alternatively, refer to the [developer instructions](#developer-instructions) for how to check out and work with the git repository. For further options, including PGP signatures, see the [**youtube-dl** Download Page](https://rg3.github.io/youtube-dl/download.html).
+Alternatively, refer to the [developer instructions](#developer-instructions) for how to check out and work with the git repository. For further options, including PGP signatures, see the [youtube-dl Download Page](https://rg3.github.io/youtube-dl/download.html).
 
 # DESCRIPTION
 **youtube-dl** is a small command-line program to download videos from
@@ -405,18 +405,18 @@ which means you can modify it, redistribute it or use it however you like.
 
 # CONFIGURATION
 
-You can configure **youtube-dl** by placing any supported command line option to a configuration file. On Linux, the system wide configuration file is located at `/etc/youtube-dl.conf` and the user wide configuration file at `~/.config/youtube-dl/config`. On Windows, the user wide configuration file locations are `%APPDATA%\youtube-dl\config.txt` or `C:\Users\<user name>\youtube-dl.conf`. For example, with the following configuration file **youtube-dl** will always extract the audio, not copy the mtime and use a proxy:
+You can configure youtube-dl by placing any supported command line option to a configuration file. On Linux, the system wide configuration file is located at `/etc/youtube-dl.conf` and the user wide configuration file at `~/.config/youtube-dl/config`. On Windows, the user wide configuration file locations are `%APPDATA%\youtube-dl\config.txt` or `C:\Users\<user name>\youtube-dl.conf`. For example, with the following configuration file youtube-dl will always extract the audio, not copy the mtime and use a proxy:
 ```
 --extract-audio
 --no-mtime
 --proxy 127.0.0.1:3128
 ```
 
-You can use `--ignore-config` if you want to disable the configuration file for a particular **youtube-dl** run.
+You can use `--ignore-config` if you want to disable the configuration file for a particular youtube-dl run.
 
 ### Authentication with `.netrc` file
 
-You may also want to configure automatic credentials storage for extractors that support authentication (by providing login and password with `--username` and `--password`) in order not to pass credentials as command line arguments on every **youtube-dl** execution and prevent tracking plain text passwords in the shell command history. You can achieve this using a [`.netrc` file](http://stackoverflow.com/tags/.netrc/info) on per extractor basis. For that you will need to create a`.netrc` file in your `$HOME` and restrict permissions to read/write by you only:
+You may also want to configure automatic credentials storage for extractors that support authentication (by providing login and password with `--username` and `--password`) in order not to pass credentials as command line arguments on every youtube-dl execution and prevent tracking plain text passwords in the shell command history. You can achieve this using a [`.netrc` file](http://stackoverflow.com/tags/.netrc/info) on per extractor basis. For that you will need to create a`.netrc` file in your `$HOME` and restrict permissions to read/write by you only:
 ```
 touch $HOME/.netrc
 chmod a-rwx,u+rw $HOME/.netrc
@@ -430,7 +430,7 @@ For example:
 machine youtube login myaccount@gmail.com password my_youtube_password
 machine twitch login my_twitch_account_name password my_twitch_password
 ```
-To activate authentication with the `.netrc` file you should pass `--netrc` to **youtube-dl** or place it in the [configuration file](#configuration).
+To activate authentication with the `.netrc` file you should pass `--netrc` to youtube-dl or place it in the [configuration file](#configuration).
 
 On Windows you may also need to setup the `%HOME%` environment variable manually.
 
@@ -464,14 +464,14 @@ youtube-dl_test_video_.mp4          # A simple file name
 
 # FORMAT SELECTION
 
-By default **youtube-dl** tries to download the best quality, but sometimes you may want to download in a different format.
+By default youtube-dl tries to download the best quality, but sometimes you may want to download in a different format.
 The simplest case is requesting a specific format, for example `-f 22`. You can get the list of available formats using `--list-formats`, you can also use a file extension (currently it supports aac, m4a, mp3, mp4, ogg, wav, webm) or the special names `best`, `bestvideo`, `bestaudio` and `worst`.
 
 If you want to download multiple videos and they don't have the same formats available, you can specify the order of preference using slashes, as in `-f 22/17/18`. You can also filter the video results by putting a condition in brackets, as in `-f "best[height=720]"` (or `-f "[filesize>10M]"`).  This works for filesize, height, width, tbr, abr, vbr, asr, and fps and the comparisons <, <=, >, >=, =, != and for ext, acodec, vcodec, container, and protocol and the comparisons =, != . Formats for which the value is not known are excluded unless you put a question mark (?) after the operator. You can combine format filters, so  `-f "[height <=? 720][tbr>500]"` selects up to 720p videos (or videos where the height is not known) with a bitrate of at least 500 KBit/s. Use commas to download multiple formats, such as `-f 136/137/mp4/bestvideo,140/m4a/bestaudio`. You can merge the video and audio of two formats into a single file using `-f <video-format>+<audio-format>` (requires ffmpeg or avconv), for example `-f bestvideo+bestaudio`. Format selectors can also be grouped using parentheses, for example if you want to download the best mp4 and webm formats with a height lower than 480 you can use `-f '(mp4,webm)[height<480]'`.
 
-Since the end of April 2015 and version 2015.04.26 **youtube-dl** uses `-f bestvideo+bestaudio/best` as default format selection (see #5447, #5456). If ffmpeg or avconv are installed this results in downloading `bestvideo` and `bestaudio` separately and muxing them together into a single file giving the best overall quality available. Otherwise it falls back to `best` and results in downloading the best available quality served as a single file. `best` is also needed for videos that don't come from YouTube because they don't provide the audio and video in two different files. If you want to only download some dash formats (for example if you are not interested in getting videos with a resolution higher than 1080p), you can add `-f bestvideo[height<=?1080]+bestaudio/best` to your configuration file. Note that if you use **youtube-dl** to stream to `stdout` (and most likely to pipe it to your media player then), i.e. you explicitly specify output template as `-o -`, **youtube-dl** still uses `-f best` format selection in order to start content delivery immediately to your player and not to wait until `bestvideo` and `bestaudio` are downloaded and muxed.
+Since the end of April 2015 and version 2015.04.26 youtube-dl uses `-f bestvideo+bestaudio/best` as default format selection (see #5447, #5456). If ffmpeg or avconv are installed this results in downloading `bestvideo` and `bestaudio` separately and muxing them together into a single file giving the best overall quality available. Otherwise it falls back to `best` and results in downloading the best available quality served as a single file. `best` is also needed for videos that don't come from YouTube because they don't provide the audio and video in two different files. If you want to only download some dash formats (for example if you are not interested in getting videos with a resolution higher than 1080p), you can add `-f bestvideo[height<=?1080]+bestaudio/best` to your configuration file. Note that if you use youtube-dl to stream to `stdout` (and most likely to pipe it to your media player then), i.e. you explicitly specify output template as `-o -`, youtube-dl still uses `-f best` format selection in order to start content delivery immediately to your player and not to wait until `bestvideo` and `bestaudio` are downloaded and muxed.
 
-If you want to preserve the old format selection behavior (prior to **youtube-dl** 2015.04.26), i.e. you want to download the best available quality media served as a single file, you should explicitly specify your choice with `-f best`. You may want to add it to the [configuration file](#configuration) in order not to type it every time you run **youtube-dl**.
+If you want to preserve the old format selection behavior (prior to youtube-dl 2015.04.26), i.e. you want to download the best available quality media served as a single file, you should explicitly specify your choice with `-f best`. You may want to add it to the [configuration file](#configuration) in order not to type it every time you run youtube-dl.
 
 # VIDEO SELECTION
 
@@ -501,7 +501,7 @@ If you've followed [our manual installation instructions](http://rg3.github.io/y
 
 If you have used pip, a simple `sudo pip install -U youtube-dl` is sufficient to update.
 
-If you have installed **youtube-dl** using a package manager like *apt-get* or *yum*, use the standard system update mechanism to update. Note that distribution packages are often outdated. As a rule of thumb, **youtube-dl** releases at least once a month, and often weekly or even daily. Simply go to http://yt-dl.org/ to find out the current version. Unfortunately, there is nothing we **youtube-dl** developers can do if your distribution serves a really outdated version. You can (and should) complain to your distribution in their bugtracker or support forum.
+If you have installed youtube-dl using a package manager like *apt-get* or *yum*, use the standard system update mechanism to update. Note that distribution packages are often outdated. As a rule of thumb, youtube-dl releases at least once a month, and often weekly or even daily. Simply go to http://yt-dl.org/ to find out the current version. Unfortunately, there is nothing we youtube-dl developers can do if your distribution serves a really outdated version. You can (and should) complain to your distribution in their bugtracker or support forum.
 
 As a last resort, you can also uninstall the version installed by your package manager and follow our manual installation instructions. For that, remove the distribution's package, with a line like
 
@@ -519,25 +519,25 @@ Again, from then on you'll be able to update with `sudo youtube-dl -U`.
 
 ### I'm getting an error `Unable to extract OpenGraph title` on YouTube playlists
 
-YouTube changed their playlist format in March 2014 and later on, so you'll need at least **youtube-dl** 2014.07.25 to download all YouTube videos.
+YouTube changed their playlist format in March 2014 and later on, so you'll need at least youtube-dl 2014.07.25 to download all YouTube videos.
 
-If you have installed **youtube-dl** with a package manager, pip, setup.py or a tarball, please use that to update. Note that Ubuntu packages do not seem to get updated anymore. Since we are not affiliated with Ubuntu, there is little we can do. Feel free to [report bugs](https://bugs.launchpad.net/ubuntu/+source/youtube-dl/+filebug) to the [Ubuntu packaging guys](mailto:ubuntu-motu@lists.ubuntu.com?subject=outdated%20version%20of%20youtube-dl) - all they have to do is update the package to a somewhat recent version. See above for a way to update.
+If you have installed youtube-dl with a package manager, pip, setup.py or a tarball, please use that to update. Note that Ubuntu packages do not seem to get updated anymore. Since we are not affiliated with Ubuntu, there is little we can do. Feel free to [report bugs](https://bugs.launchpad.net/ubuntu/+source/youtube-dl/+filebug) to the [Ubuntu packaging guys](mailto:ubuntu-motu@lists.ubuntu.com?subject=outdated%20version%20of%20youtube-dl) - all they have to do is update the package to a somewhat recent version. See above for a way to update.
 
 ### Do I always have to pass `-citw`?
 
-By default, **youtube-dl** intends to have the best options (incidentally, if you have a convincing case that these should be different, [please file an issue where you explain that](https://yt-dl.org/bug)). Therefore, it is unnecessary and sometimes harmful to copy long option strings from webpages. In particular, the only option out of `-citw` that is regularly useful is `-i`.
+By default, youtube-dl intends to have the best options (incidentally, if you have a convincing case that these should be different, [please file an issue where you explain that](https://yt-dl.org/bug)). Therefore, it is unnecessary and sometimes harmful to copy long option strings from webpages. In particular, the only option out of `-citw` that is regularly useful is `-i`.
 
 ### Can you please put the `-b` option back?
 
-Most people asking this question are not aware that **youtube-dl** now defaults to downloading the highest available quality as reported by YouTube, which will be 1080p or 720p in some cases, so you no longer need the `-b` option. For some specific videos, maybe YouTube does not report them to be available in a specific high quality format you're interested in. In that case, simply request it with the `-f` option and **youtube-dl** will try to download it.
+Most people asking this question are not aware that youtube-dl now defaults to downloading the highest available quality as reported by YouTube, which will be 1080p or 720p in some cases, so you no longer need the `-b` option. For some specific videos, maybe YouTube does not report them to be available in a specific high quality format you're interested in. In that case, simply request it with the `-f` option and youtube-dl will try to download it.
 
 ### I get HTTP error 402 when trying to download a video. What's this?
 
-Apparently YouTube requires you to pass a CAPTCHA test if you download too much. We're [considering to provide a way to let you solve the CAPTCHA](https://github.com/rg3/youtube-dl/issues/154), but at the moment, your best course of action is pointing a webbrowser to the youtube URL, solving the CAPTCHA, and restart **youtube-dl**.
+Apparently YouTube requires you to pass a CAPTCHA test if you download too much. We're [considering to provide a way to let you solve the CAPTCHA](https://github.com/rg3/youtube-dl/issues/154), but at the moment, your best course of action is pointing a webbrowser to the youtube URL, solving the CAPTCHA, and restart youtube-dl.
 
 ### Do I need any other programs?
 
-**youtube-dl** works fine on its own on most sites. However, if you want to convert video/audio, you'll need [avconv](https://libav.org/) or [ffmpeg](https://www.ffmpeg.org/). On some sites - most notably YouTube - videos can be retrieved in a higher quality format without sound. **youtube-dl** will detect whether avconv/ffmpeg is present and automatically pick the best option.
+youtube-dl works fine on its own on most sites. However, if you want to convert video/audio, you'll need [avconv](https://libav.org/) or [ffmpeg](https://www.ffmpeg.org/). On some sites - most notably YouTube - videos can be retrieved in a higher quality format without sound. youtube-dl will detect whether avconv/ffmpeg is present and automatically pick the best option.
 
 Videos or video formats streamed via RTMP protocol can only be downloaded when [rtmpdump](https://rtmpdump.mplayerhq.hu/) is installed. Downloading MMS and RTSP videos requires either [mplayer](http://mplayerhq.hu/) or [mpv](https://mpv.io/) to be installed.
 
@@ -547,25 +547,25 @@ Once the video is fully downloaded, use any video player, such as [vlc](http://w
 
 ### I extracted a video URL with `-g`, but it does not play on another machine / in my webbrowser.
 
-It depends a lot on the service. In many cases, requests for the video (to download/play it) must come from the same IP address and with the same cookies.  Use the `--cookies` option to write the required cookies into a file, and advise your downloader to read cookies from that file. Some sites also require a common user agent to be used, use `--dump-user-agent` to see the one in use by **youtube-dl**.
+It depends a lot on the service. In many cases, requests for the video (to download/play it) must come from the same IP address and with the same cookies.  Use the `--cookies` option to write the required cookies into a file, and advise your downloader to read cookies from that file. Some sites also require a common user agent to be used, use `--dump-user-agent` to see the one in use by youtube-dl.
 
 It may be beneficial to use IPv6; in some cases, the restrictions are only applied to IPv4. Some services (sometimes only for a subset of videos) do not restrict the video URL by IP address, cookie, or user-agent, but these are the exception rather than the rule.
 
 Please bear in mind that some URL protocols are **not** supported by browsers out of the box, including RTMP. If you are using `-g`, your own downloader must support these as well.
 
-If you want to play the video on a machine that is not running **youtube-dl**, you can relay the video content from the machine that runs **youtube-dl**. You can use `-o -` to let **youtube-dl** stream a video to stdout, or simply allow the player to download the files written by **youtube-dl** in turn.
+If you want to play the video on a machine that is not running youtube-dl, you can relay the video content from the machine that runs youtube-dl. You can use `-o -` to let youtube-dl stream a video to stdout, or simply allow the player to download the files written by youtube-dl in turn.
 
 ### ERROR: no fmt_url_map or conn information found in video info
 
-YouTube has switched to a new video info format in July 2011 which is not supported by old versions of **youtube-dl**. See [above](#how-do-i-update-youtube-dl) for how to update **youtube-dl**.
+YouTube has switched to a new video info format in July 2011 which is not supported by old versions of youtube-dl. See [above](#how-do-i-update-youtube-dl) for how to update youtube-dl.
 
 ### ERROR: unable to download video
 
-YouTube requires an additional signature since September 2012 which is not supported by old versions of youtube-dl. See [above](#how-do-i-update-youtube-dl) for how to update **youtube-dl**.
+YouTube requires an additional signature since September 2012 which is not supported by old versions of youtube-dl. See [above](#how-do-i-update-youtube-dl) for how to update youtube-dl.
 
 ### Video URL contains an ampersand and I'm getting some strange output `[1] 2839` or `'v' is not recognized as an internal or external command`
 
-That's actually the output from your shell. Since ampersand is one of the special shell characters it's interpreted by the shell preventing you from passing the whole URL to **youtube-dl**. To disable your shell from interpreting the ampersands (or any other special characters) you have to either put the whole URL in quotes or escape them with a backslash (which approach will work depends on your shell).
+That's actually the output from your shell. Since ampersand is one of the special shell characters it's interpreted by the shell preventing you from passing the whole URL to youtube-dl. To disable your shell from interpreting the ampersands (or any other special characters) you have to either put the whole URL in quotes or escape them with a backslash (which approach will work depends on your shell).
 
 For example if your URL is https://www.youtube.com/watch?t=4&v=BaW_jenozKc you should end up with following command:
 
@@ -581,7 +581,7 @@ For Windows you have to use the double quotes:
 
 ### ExtractorError: Could not find JS function u'OF'
 
-In February 2015, the new YouTube player contained a character sequence in a string that was misinterpreted by old versions of **youtube-dl**. See [above](#how-do-i-update-youtube-dl) for how to update **youtube-dl**.
+In February 2015, the new YouTube player contained a character sequence in a string that was misinterpreted by old versions of youtube-dl. See [above](#how-do-i-update-youtube-dl) for how to update youtube-dl.
 
 ### HTTP Error 429: Too Many Requests or 402: Payment Required
 
@@ -598,7 +598,7 @@ means you're using an outdated version of Python. Please update to Python 2.6 or
 
 ### What is this binary file? Where has the code gone?
 
-Since June 2012 (#342) **youtube-dl** is packed as an executable zipfile, simply unzip it (might need renaming to `youtube-dl.zip` first on some systems) or clone the git repository, as laid out above. If you modify the code, you can run it by executing the `__main__.py` file. To recompile the executable, run `make youtube-dl`.
+Since June 2012 (#342) youtube-dl is packed as an executable zipfile, simply unzip it (might need renaming to `youtube-dl.zip` first on some systems) or clone the git repository, as laid out above. If you modify the code, you can run it by executing the `__main__.py` file. To recompile the executable, run `make youtube-dl`.
 
 ### The exe throws a *Runtime error from Visual C++*
 
@@ -606,11 +606,11 @@ To run the exe you need to install first the [Microsoft Visual C++ 2008 Redistri
 
 ### On Windows, how should I set up ffmpeg and youtube-dl? Where should I put the exe files?
 
-If you put **youtube-dl** and ffmpeg in the same directory that you're running the command from, it will work, but that's rather cumbersome.
+If you put youtube-dl and ffmpeg in the same directory that you're running the command from, it will work, but that's rather cumbersome.
 
-To make a different directory work - either for ffmpeg, or for **youtube-dl**, or for both - simply create the directory (say, `C:\bin`, or `C:\Users\<User name>\bin`), put all the executables directly in there, and then [set your PATH environment variable](https://www.java.com/en/download/help/path.xml) to include that directory.
+To make a different directory work - either for ffmpeg, or for youtube-dl, or for both - simply create the directory (say, `C:\bin`, or `C:\Users\<User name>\bin`), put all the executables directly in there, and then [set your PATH environment variable](https://www.java.com/en/download/help/path.xml) to include that directory.
 
-From then on, after restarting your shell, you will be able to access both **youtube-dl** and ffmpeg (and **youtube-dl** will be able to find ffmpeg) by simply typing `youtube-dl` or `ffmpeg`, no matter what directory you're in.
+From then on, after restarting your shell, you will be able to access both youtube-dl and ffmpeg (and youtube-dl will be able to find ffmpeg) by simply typing `youtube-dl` or `ffmpeg`, no matter what directory you're in.
 
 ### How do I put downloads into a specific folder?
 
@@ -627,41 +627,41 @@ Either prepend `http://www.youtube.com/watch?v=` or separate the ID from the opt
 
 Use the `--cookies` option, for example `--cookies /path/to/cookies/file.txt`. Note that the cookies file must be in Mozilla/Netscape format and the first line of the cookies file must be either `# HTTP Cookie File` or `# Netscape HTTP Cookie File`. Make sure you have correct [newline format](https://en.wikipedia.org/wiki/Newline) in the cookies file and convert newlines if necessary to correspond with your OS, namely `CRLF` (`\r\n`) for Windows, `LF` (`\n`) for Linux and `CR` (`\r`) for Mac OS. `HTTP Error 400: Bad Request` when using `--cookies` is a good sign of invalid newline format.
 
-Passing cookies to **youtube-dl** is a good way to workaround login when a particular extractor does not implement it explicitly.
+Passing cookies to youtube-dl is a good way to workaround login when a particular extractor does not implement it explicitly.
 
 ### Can you add support for this anime video site, or site which shows current movies for free?
 
-As a matter of policy (as well as legality), **youtube-dl** does not include support for services that specialize in infringing copyright. As a rule of thumb, if you cannot easily find a video that the service is quite obviously allowed to distribute (i.e. that has been uploaded by the creator, the creator's distributor, or is published under a free license), the service is probably unfit for inclusion to **youtube-dl**.
+As a matter of policy (as well as legality), youtube-dl does not include support for services that specialize in infringing copyright. As a rule of thumb, if you cannot easily find a video that the service is quite obviously allowed to distribute (i.e. that has been uploaded by the creator, the creator's distributor, or is published under a free license), the service is probably unfit for inclusion to youtube-dl.
 
-A note on the service that they don't host the infringing content, but just link to those who do, is evidence that the service should **not** be included into **youtube-dl**. The same goes for any DMCA note when the whole front page of the service is filled with videos they are not allowed to distribute. A "fair use" note is equally unconvincing if the service shows copyright-protected videos in full without authorization.
+A note on the service that they don't host the infringing content, but just link to those who do, is evidence that the service should **not** be included into youtube-dl. The same goes for any DMCA note when the whole front page of the service is filled with videos they are not allowed to distribute. A "fair use" note is equally unconvincing if the service shows copyright-protected videos in full without authorization.
 
 Support requests for services that **do** purchase the rights to distribute their content are perfectly fine though. If in doubt, you can simply include a source that mentions the legitimate purchase of content.
 
 ### How can I speed up work on my issue?
 
-(Also known as: Help, my important issue not being solved!) The **youtube-dl** core developer team is quite small. While we do our best to solve as many issues as possible, sometimes that can take quite a while. To speed up your issue, here's what you can do:
+(Also known as: Help, my important issue not being solved!) The youtube-dl core developer team is quite small. While we do our best to solve as many issues as possible, sometimes that can take quite a while. To speed up your issue, here's what you can do:
 
-First of all, please do report the issue [at our issue tracker](https://yt-dl.org/bugs). That allows us to coordinate all efforts by users and developers, and serves as a unified point. Unfortunately, the **youtube-dl** project has grown too large to use personal email as an effective communication channel.
+First of all, please do report the issue [at our issue tracker](https://yt-dl.org/bugs). That allows us to coordinate all efforts by users and developers, and serves as a unified point. Unfortunately, the youtube-dl project has grown too large to use personal email as an effective communication channel.
 
-Please read the [bug reporting instructions](#bugs) below. A lot of bugs lack all the necessary information. If you can, offer proxy, VPN, or shell access to the **youtube-dl** developers. If you are able to, test the issue from multiple computers in multiple countries to exclude local censorship or misconfiguration issues.
+Please read the [bug reporting instructions](#bugs) below. A lot of bugs lack all the necessary information. If you can, offer proxy, VPN, or shell access to the youtube-dl developers. If you are able to, test the issue from multiple computers in multiple countries to exclude local censorship or misconfiguration issues.
 
 If nobody is interested in solving your issue, you are welcome to take matters into your own hands and submit a pull request (or coerce/pay somebody else to do so).
 
-Feel free to bump the issue from time to time by writing a small comment ("Issue is still present in **youtube-dl** version ...from France, but fixed from Belgium"), but please not more than once a month. Please do not declare your issue as `important` or `urgent`.
+Feel free to bump the issue from time to time by writing a small comment ("Issue is still present in youtube-dl version ...from France, but fixed from Belgium"), but please not more than once a month. Please do not declare your issue as `important` or `urgent`.
 
 ### How can I detect whether a given URL is supported by youtube-dl?
 
-For one, have a look at the [list of supported sites](docs/supportedsites.md). Note that it can sometimes happen that the site changes its URL scheme (say, from http://example.com/video/1234567 to http://example.com/v/1234567 ) and **youtube-dl** reports an URL of a service in that list as unsupported. In that case, simply report a bug.
+For one, have a look at the [list of supported sites](docs/supportedsites.md). Note that it can sometimes happen that the site changes its URL scheme (say, from http://example.com/video/1234567 to http://example.com/v/1234567 ) and youtube-dl reports an URL of a service in that list as unsupported. In that case, simply report a bug.
 
-It is *not* possible to detect whether a URL is supported or not. That's because **youtube-dl** contains a generic extractor which matches **all** URLs. You may be tempted to disable, exclude, or remove the generic extractor, but the generic extractor not only allows users to extract videos from lots of websites that embed a video from another service, but may also be used to extract video from a service that it's hosting itself. Therefore, we neither recommend nor support disabling, excluding, or removing the generic extractor.
+It is *not* possible to detect whether a URL is supported or not. That's because youtube-dl contains a generic extractor which matches **all** URLs. You may be tempted to disable, exclude, or remove the generic extractor, but the generic extractor not only allows users to extract videos from lots of websites that embed a video from another service, but may also be used to extract video from a service that it's hosting itself. Therefore, we neither recommend nor support disabling, excluding, or removing the generic extractor.
 
-If you want to find out whether a given URL is supported, simply call **youtube-dl** with it. If you get no videos back, chances are the URL is either not referring to a video or unsupported. You can find out which by examining the output (if you run **youtube-dl** on the console) or catching an `UnsupportedError` exception if you run it from a Python program.
+If you want to find out whether a given URL is supported, simply call youtube-dl with it. If you get no videos back, chances are the URL is either not referring to a video or unsupported. You can find out which by examining the output (if you run youtube-dl on the console) or catching an `UnsupportedError` exception if you run it from a Python program.
 
 # DEVELOPER INSTRUCTIONS
 
-Most users do not need to build **youtube-dl** and can [download the builds](http://rg3.github.io/youtube-dl/download.html) or get them from their distribution.
+Most users do not need to build youtube-dl and can [download the builds](http://rg3.github.io/youtube-dl/download.html) or get them from their distribution.
 
-To run **youtube-dl** as a developer, you don't need to build anything either. Simply execute
+To run youtube-dl as a developer, you don't need to build anything either. Simply execute
 
     python -m youtube_dl
 
@@ -671,7 +671,7 @@ To run the test, simply invoke your favorite test runner, or execute a test file
     python test/test_download.py
     nosetests
 
-If you want to create a build of **youtube-dl** yourself, you'll need
+If you want to create a build of youtube-dl yourself, you'll need
 
 * python
 * make
@@ -744,9 +744,9 @@ In any case, thank you very much for your contributions!
 
 # EMBEDDING YOUTUBE-DL
 
-**youtube-dl** makes the best effort to be a good command-line program, and thus should be callable from any programming language. If you encounter any problems parsing its output, feel free to [create a report](https://github.com/rg3/youtube-dl/issues/new).
+youtube-dl makes the best effort to be a good command-line program, and thus should be callable from any programming language. If you encounter any problems parsing its output, feel free to [create a report](https://github.com/rg3/youtube-dl/issues/new).
 
-From a Python program, you can embed **youtube-dl** in a more powerful fashion, like this:
+From a Python program, you can embed youtube-dl in a more powerful fashion, like this:
 
 ```python
 from __future__ import unicode_literals
@@ -757,7 +757,7 @@ with youtube_dl.YoutubeDL(ydl_opts) as ydl:
     ydl.download(['http://www.youtube.com/watch?v=BaW_jenozKc'])
 ```
 
-Most likely, you'll want to use various options. For a list of what can be done, have a look at [youtube_dl/YoutubeDL.py](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/YoutubeDL.py#L117-L265). For a start, if you want to intercept **youtube-dl**'s output, set a `logger` object.
+Most likely, you'll want to use various options. For a list of what can be done, have a look at [youtube_dl/YoutubeDL.py](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/YoutubeDL.py#L117-L265). For a start, if you want to intercept youtube-dl's output, set a `logger` object.
 
 Here's a more complete example of a program that outputs only errors (and a short message after the download is finished), and downloads/converts the video to an mp3 file:
 
@@ -832,7 +832,7 @@ So please elaborate on what feature you are requesting, or what bug you want to
 
 If your report is shorter than two lines, it is almost certainly missing some of these, which makes it hard for us to respond to it. We're often too polite to close the issue outright, but the missing info makes misinterpretation likely. As a commiter myself, I often get frustrated by these issues, since the only possible way for me to move forward on them is to ask for clarification over and over.
 
-For bug reports, this means that your report should contain the *complete* output of **youtube-dl** when called with the `-v` flag. The error message you get for (most) bugs even says so, but you would not believe how many of our bug reports do not contain this information.
+For bug reports, this means that your report should contain the *complete* output of youtube-dl when called with the `-v` flag. The error message you get for (most) bugs even says so, but you would not believe how many of our bug reports do not contain this information.
 
 If your server has multiple IPs or you suspect censorship, adding `--call-home` may be a good idea to get more diagnostics. If the error is `ERROR: Unable to extract ...` and you cannot reproduce it from multiple countries, add `--dump-pages` (warning: this will yield a rather large output, redirect it to the file `log.txt` by adding `>log.txt 2>&1` to your command-line) or upload the `.dump` files you get when you add `--write-pages` [somewhere](https://gist.github.com/).
 
@@ -860,7 +860,7 @@ We are then presented with a very complicated request when the original problem
 
 Some of our users seem to think there is a limit of issues they can or should open. There is no limit of issues they can or should open. While it may seem appealing to be able to dump all your issues into one ticket, that means that someone who solves one of your issues cannot mark the issue as closed. Typically, reporting a bunch of issues leads to the ticket lingering since nobody wants to attack that behemoth, until someone mercifully splits the issue into multiple ones.
 
-In particular, every site support request issue should only pertain to services at one site (generally under a common domain, but always using the same backend technology). Do not request support for vimeo user videos, Whitehouse podcasts, and Google Plus pages in the same issue. Also, make sure that you don't post bug reports alongside feature requests. As a rule of thumb, a feature request does not include outputs of **youtube-dl** that are not immediately related to the feature at hand. Do not post reports of a network error alongside the request for a new video service.
+In particular, every site support request issue should only pertain to services at one site (generally under a common domain, but always using the same backend technology). Do not request support for vimeo user videos, Whitehouse podcasts, and Google Plus pages in the same issue. Also, make sure that you don't post bug reports alongside feature requests. As a rule of thumb, a feature request does not include outputs of youtube-dl that are not immediately related to the feature at hand. Do not post reports of a network error alongside the request for a new video service.
 
 ###  Is anyone going to need the feature?
 
@@ -868,10 +868,10 @@ Only post features that you (or an incapacitated friend you can personally talk
 
 ###  Is your question about youtube-dl?
 
-It may sound strange, but some bug reports we receive are completely unrelated to **youtube-dl** and relate to a different or even the reporter's own application. Please make sure that you are actually using **youtube-dl**. If you are using a UI for **youtube-dl**, report the bug to the maintainer of the actual application providing the UI. On the other hand, if your UI for **youtube-dl** fails in some way you believe is related to **youtube-dl**, by all means, go ahead and report the bug.
+It may sound strange, but some bug reports we receive are completely unrelated to youtube-dl and relate to a different or even the reporter's own application. Please make sure that you are actually using youtube-dl. If you are using a UI for youtube-dl, report the bug to the maintainer of the actual application providing the UI. On the other hand, if your UI for youtube-dl fails in some way you believe is related to youtube-dl, by all means, go ahead and report the bug.
 
 # COPYRIGHT
 
-**youtube-dl** is released into the public domain by the copyright holders.
+youtube-dl is released into the public domain by the copyright holders.
 
 This README file was originally written by [Daniel Bolton](https://github.com/dbbolton) and is likewise released into the public domain.

From 51094b1b0885462c19cc2c75f545e621f8a3aa0a Mon Sep 17 00:00:00 2001
From: Celthi <txtlu0@gmail.com>
Date: Thu, 10 Dec 2015 21:42:12 +0800
Subject: [PATCH 3310/6219] add cookie and referer in headers, change the video
 url

---
 youtube_dl/extractor/youku.py | 21 +++++++++++++++++----
 1 file changed, 17 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index 69ecc837a..d077e03db 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -6,6 +6,9 @@ import base64
 from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse,
+    compat_cookiejar,
+    compat_cookies,
+    compat_urllib_request,
     compat_ord,
 )
 from ..utils import (
@@ -188,19 +191,28 @@ class YoukuIE(InfoExtractor):
         video_id = self._match_id(url)
 
         def retrieve_data(req_url, note):
-            req = sanitized_Request(req_url)
+            
+
+            headers = {
+                    'Referer': req_url,
+                }
+            self._set_cookie('youku.com','xreferrer','http://www.youku.com')
+            req = sanitized_Request(req_url,headers=headers)
 
             cn_verification_proxy = self._downloader.params.get('cn_verification_proxy')
             if cn_verification_proxy:
                 req.add_header('Ytdl-request-proxy', cn_verification_proxy)
 
             raw_data = self._download_json(req, video_id, note=note)
-            return raw_data['data'][0]
+
+            return raw_data['data']['security']
+
 
         video_password = self._downloader.params.get('videopassword', None)
 
         # request basic data
-        basic_data_url = 'http://v.youku.com/player/getPlayList/VideoIDS/%s' % video_id
+        #basic_data_url = 'http://v.youku.com/player/getPlayList/VideoIDS/%s' % video_id
+        basic_data_url = "http://play.youku.com/play/get.json?vid=%s&ct=12" % video_id
         if video_password:
             basic_data_url += '?password=%s' % video_password
 
@@ -208,7 +220,8 @@ class YoukuIE(InfoExtractor):
             basic_data_url,
             'Downloading JSON metadata 1')
         data2 = retrieve_data(
-            'http://v.youku.com/player/getPlayList/VideoIDS/%s/Pf/4/ctype/12/ev/1' % video_id,
+            #'http://v.youku.com/player/getPlayList/VideoIDS/%s/Pf/4/ctype/12/ev/1' % video_id,
+            "http://play.youku.com/play/get.json?vid=%s&ct=12" % video_id,
             'Downloading JSON metadata 2')
 
         error_code = data1.get('error_code')

From 526a20bd16ed734fdff0bf56ec79537d7e4d7b96 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 10 Dec 2015 21:04:26 +0600
Subject: [PATCH 3311/6219] [pbs] Clarify member stations' URLs

---
 youtube_dl/extractor/pbs.py | 321 ++++++++++++++++++------------------
 1 file changed, 161 insertions(+), 160 deletions(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index d71d7eb76..d030236d4 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -16,164 +16,165 @@ from ..utils import (
 
 class PBSIE(InfoExtractor):
     _STATIONS = (
-        ('aptv.org', 'APT - Alabama Public Television (WBIQ)'),  # http://aptv.org/
-        ('gpb.org', 'GPB/Georgia Public Broadcasting (WGTV)'),  # http://www.gpb.org/
-        ('mpbonline.org', 'Mississippi Public Broadcasting (WMPN)'),  # http://www.mpbonline.org
-        ('wnpt.org', 'Nashville Public Television (WNPT)'),  # http://www.wnpt.org
-        ('wfsu.org', 'WFSU-TV (WFSU)'),  # http://wfsu.org/
-        ('wsre.org', 'WSRE (WSRE)'),  # http://www.wsre.org
-        ('wtcitv.org', 'WTCI (WTCI)'),  # http://www.wtcitv.org
-        ('pba.org', 'WPBA/Channel 30 (WPBA)'),  # http://pba.org/
-        ('alaskapublic.org', 'Alaska Public Media (KAKM)'),  # http://alaskapublic.org/kakm
-        ('kuac.org', 'KUAC (KUAC)'),  # http://kuac.org/kuac-tv/
-        ('ktoo.org', '360 North (KTOO)'),  # http://www.ktoo.org/
-        ('azpm.org', 'KUAT 6 (KUAT)'),  # http://www.azpm.org/
-        ('azpbs.org', 'Arizona PBS (KAET)'),  # http://www.azpbs.org
-        ('newmexicopbs.org', 'KNME-TV/Channel 5 (KNME)'),  # http://www.newmexicopbs.org/
-        ('vegaspbs.org', 'Vegas PBS (KLVX)'),  # http://vegaspbs.org/
-        ('aetn.org', 'AETN/ARKANSAS ETV NETWORK (KETS)'),  # http://www.aetn.org/
-        ('ket.org', 'KET (WKLE)'),  # http://www.ket.org/
-        ('wkno.org', 'WKNO/Channel 10 (WKNO)'),  # http://www.wkno.org/
-        ('lpb.org', 'LPB/LOUISIANA PUBLIC BROADCASTING (WLPB)'),  # http://www.lpb.org/
-        ('oeta.tv', 'OETA (KETA)'),  # http://www.oeta.tv
-        ('optv.org', 'Ozarks Public Television (KOZK)'),  # http://www.optv.org/
-        ('wsiu.org', 'WSIU Public Broadcasting (WSIU)'),  # http://www.wsiu.org/
-        ('keet.org', 'KEET TV (KEET)'),  # http://www.keet.org
-        ('kixe.org', 'KIXE/Channel 9 (KIXE)'),  # http://kixe.org/
-        ('kpbs.org', 'KPBS San Diego (KPBS)'),  # http://www.kpbs.org/
-        ('kqed.org', 'KQED (KQED)'),  # http://www.kqed.org
-        ('kvie.org', 'KVIE Public Television (KVIE)'),  # http://www.kvie.org
-        ('pbssocal.org', 'PBS SoCal/KOCE (KOCE)'),  # http://www.pbssocal.org/
-        ('valleypbs.org', 'ValleyPBS (KVPT)'),  # http://www.valleypbs.org/
-        ('cptv.org', 'CONNECTICUT PUBLIC TELEVISION (WEDH)'),  # http://cptv.org
-        ('knpb.org', 'KNPB Channel 5 (KNPB)'),  # http://www.knpb.org/
-        ('soptv.org', 'SOPTV (KSYS)'),  # http://www.soptv.org
-        ('klcs.org', 'KLCS/Channel 58 (KLCS)'),  # http://www.klcs.org
-        ('krcb.org', 'KRCB Television & Radio (KRCB)'),  # http://www.krcb.org
-        ('kvcr.org', 'KVCR TV/DT/FM :: Vision for the Future (KVCR)'),  # http://kvcr.org
-        ('rmpbs.org', 'Rocky Mountain PBS (KRMA)'),  # http://www.rmpbs.org
-        ('kenw.org', 'KENW-TV3 (KENW)'),  # http://www.kenw.org
-        ('kued.org', 'KUED Channel 7 (KUED)'),  # http://www.kued.org
-        ('wyomingpbs.org', 'Wyoming PBS (KCWC)'),  # http://www.wyomingpbs.org
-        ('cpt12.org', 'Colorado Public Television / KBDI 12 (KBDI)'),  # http://www.cpt12.org/
-        ('kbyutv.org', 'KBYU-TV (KBYU)'),  # http://www.kbyutv.org/
-        ('thirteen.org', 'Thirteen/WNET New York (WNET)'),  # http://www.thirteen.org
-        ('wgbh.org', 'WGBH/Channel 2 (WGBH)'),  # http://wgbh.org
-        ('wgby.org', 'WGBY (WGBY)'),  # http://www.wgby.org
-        ('njtvonline.org', 'NJTV Public Media NJ (WNJT)'),  # http://www.njtvonline.org/
-        ('ripbs.org', 'Rhode Island PBS (WSBE)'),  # http://www.ripbs.org/home/
-        ('wliw.org', 'WLIW21 (WLIW)'),  # http://www.wliw.org/
-        ('mpt.org', 'mpt/Maryland Public Television (WMPB)'),  # http://www.mpt.org
-        ('weta.org', 'WETA Television and Radio (WETA)'),  # http://www.weta.org
-        ('whyy.org', 'WHYY (WHYY)'),  # http://www.whyy.org
-        ('wlvt.org', 'PBS 39 (WLVT)'),  # http://www.wlvt.org/
-        ('wvpt.net', 'WVPT - Your Source for PBS and More! (WVPT)'),  # http://www.wvpt.net
-        ('whut.org', 'Howard University Television (WHUT)'),  # http://www.whut.org
-        ('wedu.org', 'WEDU PBS (WEDU)'),  # http://www.wedu.org
-        ('wgcu.org', 'WGCU Public Media (WGCU)'),  # http://www.wgcu.org/
-        ('wjct.org', 'WJCT Public Broadcasting (WJCT)'),  # http://www.wjct.org
-        ('wpbt2.org', 'WPBT2 (WPBT)'),  # http://www.wpbt2.org
-        ('wucftv.org', 'WUCF TV (WUCF)'),  # http://wucftv.org
-        ('wuft.org', 'WUFT/Channel 5 (WUFT)'),  # http://www.wuft.org
-        ('wxel.org', 'WXEL/Channel 42 (WXEL)'),  # http://www.wxel.org/home/
-        ('wlrn.org', 'WLRN/Channel 17 (WLRN)'),  # http://www.wlrn.org/
-        ('wusf.org', 'WUSF Public Broadcasting (WUSF)'),  # http://wusf.org/
-        ('scetv.org', 'ETV (WRLK)'),  # http://www.scetv.org
-        ('unctv.org', 'UNC-TV (WUNC)'),  # http://www.unctv.org/
-        ('pbsguam.org', 'PBS Guam (KGTF)'),  # http://www.pbsguam.org/
-        ('pbshawaii.org', 'PBS Hawaii - Oceanic Cable Channel 10 (KHET)'),  # http://www.pbshawaii.org/
-        ('idahoptv.org', 'Idaho Public Television (KAID)'),  # http://idahoptv.org
-        ('ksps.org', 'KSPS (KSPS)'),  # http://www.ksps.org/home/
-        ('opb.org', 'OPB (KOPB)'),  # http://www.opb.org
-        ('kwsu.org', 'KWSU/Channel 10 & KTNW/Channel 31 (KWSU)'),  # http://www.kwsu.org
-        ('illinois.edu', 'WILL-TV (WILL)'),  # http://will.illinois.edu/
-        ('wsec.tv', 'Network Knowledge - WSEC/Springfield (WSEC)'),  # http://www.wsec.tv
-        ('wttw.com', 'WTTW11 (WTTW)'),  # http://www.wttw.com/
-        ('wtvp.org', 'WTVP & WTVP.org, Public Media for Central Illinois (WTVP)'),  # http://www.wtvp.org/
-        ('iptv.org', 'Iowa Public Television/IPTV (KDIN)'),  # http://www.iptv.org/
-        ('ninenet.org', 'Nine Network (KETC)'),  # http://www.ninenet.org
-        ('wfwa.org', 'PBS39 Fort Wayne (WFWA)'),  # http://wfwa.org/
-        ('wfyi.org', 'WFYI Indianapolis (WFYI)'),  # http://www.wfyi.org
-        ('mptv.org', 'Milwaukee Public Television (WMVS)'),  # http://www.mptv.org
-        ('wnin.org', 'WNIN (WNIN)'),  # http://www.wnin.org/
-        ('wnit.org', 'WNIT Public Television (WNIT)'),  # http://www.wnit.org/
-        ('wpt.org', 'WPT (WPNE)'),  # http://www.wpt.org/
-        ('wvut.org', 'WVUT/Channel 22 (WVUT)'),  # http://wvut.org/
-        ('weiu.net', 'WEIU/Channel 51 (WEIU)'),  # http://www.weiu.net
-        ('wqpt.org', 'WQPT-TV (WQPT)'),  # http://www.wqpt.org
-        ('wycc.org', 'WYCC PBS Chicago (WYCC)'),  # http://www.wycc.org
-        ('lakeshorepublicmedia.org', 'Lakeshore Public Television (WYIN)'),  # http://lakeshorepublicmedia.org/
-        ('wipb.org', 'WIPB-TV (WIPB)'),  # http://wipb.org
-        ('indianapublicmedia.org', 'WTIU (WTIU)'),  # http://indianapublicmedia.org/tv/
-        ('cetconnect.org', 'CET  (WCET)'),  # http://www.cetconnect.org
-        ('thinktv.org', 'ThinkTVNetwork (WPTD)'),  # http://www.thinktv.org
-        ('wbgu.org', 'WBGU-TV (WBGU)'),  # http://wbgu.org
-        ('wgvu.org', 'WGVU TV (WGVU)'),  # http://www.wgvu.org/
-        ('netnebraska.org', 'NET1 (KUON)'),  # http://netnebraska.org
-        ('pioneer.org', 'Pioneer Public Television (KWCM)'),  # http://www.pioneer.org
-        ('sdpb.org', 'SDPB Television (KUSD)'),  # http://www.sdpb.org
-        ('tpt.org', 'TPT (KTCA)'),  # http://www.tpt.org
-        ('ksmq.org', 'KSMQ (KSMQ)'),  # http://www.ksmq.org/
-        ('kpts.org', 'KPTS/Channel 8 (KPTS)'),  # http://www.kpts.org/
-        ('ktwu.org', 'KTWU/Channel 11 (KTWU)'),  # http://ktwu.org
-        ('shptv.org', 'Smoky Hills Public Television (KOOD)'),  # http://www.shptv.org
-        ('kcpt.org', 'KCPT Kansas City Public Television (KCPT)'),  # http://kcpt.org/
-        ('blueridgepbs.org', 'Blue Ridge PBS (WBRA)'),  # http://www.blueridgepbs.org/
-        ('easttennesseepbs.org', 'East Tennessee PBS (WSJK)'),  # http://easttennesseepbs.org
-        ('wcte.org', 'WCTE-TV (WCTE)'),  # http://www.wcte.org
-        ('wljt.org', 'WLJT, Channel 11 (WLJT)'),  # http://wljt.org/
-        ('wosu.org', 'WOSU TV (WOSU)'),  # http://wosu.org/
-        ('woub.org', 'WOUB/WOUC (WOUB)'),  # http://woub.org/tv/index.php?section=5
-        ('wvpublic.org', 'WVPB (WVPB)'),  # http://wvpublic.org/
-        ('wkyupbs.org', 'WKYU-PBS (WKYU)'),  # http://www.wkyupbs.org
-        ('wyes.org', 'WYES-TV/New Orleans (WYES)'),  # http://www.wyes.org
-        ('kera.org', 'KERA 13 (KERA)'),  # http://www.kera.org/
-        ('mpbn.net', 'MPBN (WCBB)'),  # http://www.mpbn.net/
-        ('mountainlake.org', 'Mountain Lake PBS (WCFE)'),  # http://www.mountainlake.org/
-        ('nhptv.org', 'NHPTV (WENH)'),  # http://nhptv.org/
-        ('vpt.org', 'Vermont PBS (WETK)'),  # http://www.vpt.org
-        ('witf.org', 'witf (WITF)'),  # http://www.witf.org
-        ('wqed.org', 'WQED Multimedia (WQED)'),  # http://www.wqed.org/
-        ('wmht.org', 'WMHT Educational Telecommunications (WMHT)'),  # http://www.wmht.org/home/
-        ('deltabroadcasting.org', 'Q-TV (WDCQ)'),  # http://www.deltabroadcasting.org
-        ('dptv.org', 'WTVS Detroit Public TV (WTVS)'),  # http://www.dptv.org/
-        ('wcmu.org', 'CMU Public Television (WCMU)'),  # http://www.wcmu.org
-        ('wkar.org', 'WKAR-TV (WKAR)'),  # http://wkar.org/
-        ('nmu.edu', 'WNMU-TV Public TV 13 (WNMU)'),  # http://wnmutv.nmu.edu
-        ('wdse.org', 'WDSE - WRPT (WDSE)'),  # http://www.wdse.org/
-        ('wgte.org', 'WGTE TV (WGTE)'),  # http://www.wgte.org
-        ('lakelandptv.org', 'Lakeland Public Television (KAWE)'),  # http://www.lakelandptv.org
-        ('prairiepublic.org', 'PRAIRIE PUBLIC (KFME)'),  # http://www.prairiepublic.org/
-        ('kmos.org', 'KMOS-TV - Channels 6.1, 6.2 and 6.3 (KMOS)'),  # http://www.kmos.org/
-        ('montanapbs.org', 'MontanaPBS (KUSM)'),  # http://montanapbs.org
-        ('krwg.org', 'KRWG/Channel 22 (KRWG)'),  # http://www.krwg.org
-        ('panhandlepbs.org', 'KACV (KACV)'),  # http://www.panhandlepbs.org/home/
-        ('kcostv.org', 'KCOS/Channel 13 (KCOS)'),  # www.kcostv.org
-        ('wcny.org', 'WCNY/Channel 24 (WCNY)'),  # http://www.wcny.org
-        ('wned.org', 'WNED (WNED)'),  # http://www.wned.org/
-        ('wpbstv.org', 'WPBS (WPBS)'),  # http://www.wpbstv.org
-        ('wskg.org', 'WSKG Public TV (WSKG)'),  # http://wskg.org
-        ('wxxi.org', 'WXXI (WXXI)'),  # http://wxxi.org
-        ('wpsu.org', 'WPSU (WPSU)'),  # http://www.wpsu.org
-        ('wqln.org', 'WQLN/Channel 54 (WQLN)'),  # http://www.wqln.org
-        ('wvia.org', 'WVIA Public Media Studios (WVIA)'),  # http://www.wvia.org/
-        ('wtvi.org', 'WTVI (WTVI)'),  # http://www.wtvi.org/
-        ('whro.org', 'WHRO (WHRO)'),  # http://whro.org
-        ('westernreservepublicmedia.org', 'Western Reserve PBS (WNEO)'),  # http://www.WesternReservePublicMedia.org/
-        ('wviz.org', 'WVIZ/PBS ideastream (WVIZ)'),  # http://www.wviz.org/
-        ('kcts9.org', 'KCTS 9 (KCTS)'),  # http://kcts9.org/
-        ('basinpbs.org', 'Basin PBS (KPBT)'),  # http://www.basinpbs.org
-        ('houstonpublicmedia.org', 'KUHT / Channel 8 (KUHT)'),  # http://www.houstonpublicmedia.org/
-        ('tamu.edu', 'KAMU - TV (KAMU)'),  # http://KAMU.tamu.edu
-        ('kedt.org', 'KEDT/Channel 16 (KEDT)'),  # http://www.kedt.org
-        ('klrn.org', 'KLRN (KLRN)'),  # http://www.klrn.org
-        ('klru.org', 'KLRU (KLRU)'),  # http://www.klru.org
-        ('kmbh.org', 'KMBH-TV (KMBH)'),  # http://www.kmbh.org
-        ('knct.org', 'KNCT (KNCT)'),  # http://www.knct.org
-        ('ktxt.org', 'KTTZ-TV (KTXT)'),  # http://www.ktxt.org
-        ('wtjx.org', 'WTJX Channel 12 (WTJX)'),  # http://www.wtjx.org/
-        ('ideastations.org', 'WCVE PBS (WCVE)'),  # http://ideastations.org/
-        ('kbtc.org', 'KBTC Public Television (KBTC)'),  # http://kbtc.org
+        ('video.pbs.org', 'PBS: Public Broadcasting Service'),  # http://www.pbs.org/
+        ('video.aptv.org', 'APT - Alabama Public Television (WBIQ)'),  # http://aptv.org/
+        ('video.gpb.org', 'GPB/Georgia Public Broadcasting (WGTV)'),  # http://www.gpb.org/
+        ('video.mpbonline.org', 'Mississippi Public Broadcasting (WMPN)'),  # http://www.mpbonline.org
+        ('video.wnpt.org', 'Nashville Public Television (WNPT)'),  # http://www.wnpt.org
+        ('video.wfsu.org', 'WFSU-TV (WFSU)'),  # http://wfsu.org/
+        ('video.wsre.org', 'WSRE (WSRE)'),  # http://www.wsre.org
+        ('video.wtcitv.org', 'WTCI (WTCI)'),  # http://www.wtcitv.org
+        ('video.pba.org', 'WPBA/Channel 30 (WPBA)'),  # http://pba.org/
+        ('video.alaskapublic.org', 'Alaska Public Media (KAKM)'),  # http://alaskapublic.org/kakm
+        #('kuac.org', 'KUAC (KUAC)'),  # http://kuac.org/kuac-tv/
+        #('ktoo.org', '360 North (KTOO)'),  # http://www.ktoo.org/
+        #('azpm.org', 'KUAT 6 (KUAT)'),  # http://www.azpm.org/
+        ('video.azpbs.org', 'Arizona PBS (KAET)'),  # http://www.azpbs.org
+        ('portal.knme.org', 'KNME-TV/Channel 5 (KNME)'),  # http://www.newmexicopbs.org/
+        ('video.vegaspbs.org', 'Vegas PBS (KLVX)'),  # http://vegaspbs.org/
+        ('watch.aetn.org', 'AETN/ARKANSAS ETV NETWORK (KETS)'),  # http://www.aetn.org/
+        ('video.ket.org', 'KET (WKLE)'),  # http://www.ket.org/
+        ('video.wkno.org', 'WKNO/Channel 10 (WKNO)'),  # http://www.wkno.org/
+        ('video.lpb.org', 'LPB/LOUISIANA PUBLIC BROADCASTING (WLPB)'),  # http://www.lpb.org/
+        ('videos.oeta.tv', 'OETA (KETA)'),  # http://www.oeta.tv
+        ('video.optv.org', 'Ozarks Public Television (KOZK)'),  # http://www.optv.org/
+        ('watch.wsiu.org', 'WSIU Public Broadcasting (WSIU)'),  # http://www.wsiu.org/
+        ('video.keet.org', 'KEET TV (KEET)'),  # http://www.keet.org
+        ('pbs.kixe.org', 'KIXE/Channel 9 (KIXE)'),  # http://kixe.org/
+        ('video.kpbs.org', 'KPBS San Diego (KPBS)'),  # http://www.kpbs.org/
+        ('video.kqed.org', 'KQED (KQED)'),  # http://www.kqed.org
+        ('vids.kvie.org', 'KVIE Public Television (KVIE)'),  # http://www.kvie.org
+        ('video.pbssocal.org', 'PBS SoCal/KOCE (KOCE)'),  # http://www.pbssocal.org/
+        ('video.valleypbs.org', 'ValleyPBS (KVPT)'),  # http://www.valleypbs.org/
+        ('video.cptv.org', 'CONNECTICUT PUBLIC TELEVISION (WEDH)'),  # http://cptv.org
+        ('watch.knpb.org', 'KNPB Channel 5 (KNPB)'),  # http://www.knpb.org/
+        ('video.soptv.org', 'SOPTV (KSYS)'),  # http://www.soptv.org
+        #('klcs.org', 'KLCS/Channel 58 (KLCS)'),  # http://www.klcs.org
+        #('krcb.org', 'KRCB Television & Radio (KRCB)'),  # http://www.krcb.org
+        #('kvcr.org', 'KVCR TV/DT/FM :: Vision for the Future (KVCR)'),  # http://kvcr.org
+        ('video.rmpbs.org', 'Rocky Mountain PBS (KRMA)'),  # http://www.rmpbs.org
+        ('video.kenw.org', 'KENW-TV3 (KENW)'),  # http://www.kenw.org
+        ('video.kued.org', 'KUED Channel 7 (KUED)'),  # http://www.kued.org
+        ('video.wyomingpbs.org', 'Wyoming PBS (KCWC)'),  # http://www.wyomingpbs.org
+        ('video.cpt12.org', 'Colorado Public Television / KBDI 12 (KBDI)'),  # http://www.cpt12.org/
+        ('video.kbyueleven.org', 'KBYU-TV (KBYU)'),  # http://www.kbyutv.org/
+        ('video.thirteen.org', 'Thirteen/WNET New York (WNET)'),  # http://www.thirteen.org
+        ('video.wgbh.org', 'WGBH/Channel 2 (WGBH)'),  # http://wgbh.org
+        ('video.wgby.org', 'WGBY (WGBY)'),  # http://www.wgby.org
+        ('watch.njtvonline.org', 'NJTV Public Media NJ (WNJT)'),  # http://www.njtvonline.org/
+        #('ripbs.org', 'Rhode Island PBS (WSBE)'),  # http://www.ripbs.org/home/
+        ('watch.wliw.org', 'WLIW21 (WLIW)'),  # http://www.wliw.org/
+        ('video.mpt.tv', 'mpt/Maryland Public Television (WMPB)'),  # http://www.mpt.org
+        ('watch.weta.org', 'WETA Television and Radio (WETA)'),  # http://www.weta.org
+        ('video.whyy.org', 'WHYY (WHYY)'),  # http://www.whyy.org
+        ('video.wlvt.org', 'PBS 39 (WLVT)'),  # http://www.wlvt.org/
+        ('video.wvpt.net', 'WVPT - Your Source for PBS and More! (WVPT)'),  # http://www.wvpt.net
+        ('video.whut.org', 'Howard University Television (WHUT)'),  # http://www.whut.org
+        ('video.wedu.org', 'WEDU PBS (WEDU)'),  # http://www.wedu.org
+        ('video.wgcu.org', 'WGCU Public Media (WGCU)'),  # http://www.wgcu.org/
+        #('wjct.org', 'WJCT Public Broadcasting (WJCT)'),  # http://www.wjct.org
+        ('video.wpbt2.org', 'WPBT2 (WPBT)'),  # http://www.wpbt2.org
+        ('video.wucftv.org', 'WUCF TV (WUCF)'),  # http://wucftv.org
+        ('video.wuft.org', 'WUFT/Channel 5 (WUFT)'),  # http://www.wuft.org
+        ('watch.wxel.org', 'WXEL/Channel 42 (WXEL)'),  # http://www.wxel.org/home/
+        ('video.wlrn.org', 'WLRN/Channel 17 (WLRN)'),  # http://www.wlrn.org/
+        ('video.wusf.usf.edu', 'WUSF Public Broadcasting (WUSF)'),  # http://wusf.org/
+        ('video.scetv.org', 'ETV (WRLK)'),  # http://www.scetv.org
+        ('video.unctv.org', 'UNC-TV (WUNC)'),  # http://www.unctv.org/
+        #('pbsguam.org', 'PBS Guam (KGTF)'),  # http://www.pbsguam.org/
+        ('video.pbshawaii.org', 'PBS Hawaii - Oceanic Cable Channel 10 (KHET)'),  # http://www.pbshawaii.org/
+        ('video.idahoptv.org', 'Idaho Public Television (KAID)'),  # http://idahoptv.org
+        ('video.ksps.org', 'KSPS (KSPS)'),  # http://www.ksps.org/home/
+        ('watch.opb.org', 'OPB (KOPB)'),  # http://www.opb.org
+        ('watch.nwptv.org', 'KWSU/Channel 10 & KTNW/Channel 31 (KWSU)'),  # http://www.kwsu.org
+        ('video.will.illinois.edu', 'WILL-TV (WILL)'),  # http://will.illinois.edu/
+        ('video.networkknowledge.tv', 'Network Knowledge - WSEC/Springfield (WSEC)'),  # http://www.wsec.tv
+        ('video.wttw.com', 'WTTW11 (WTTW)'),  # http://www.wttw.com/
+        #('wtvp.org', 'WTVP & WTVP.org, Public Media for Central Illinois (WTVP)'),  # http://www.wtvp.org/
+        ('video.iptv.org', 'Iowa Public Television/IPTV (KDIN)'),  # http://www.iptv.org/
+        ('video.ninenet.org', 'Nine Network (KETC)'),  # http://www.ninenet.org
+        ('video.wfwa.org', 'PBS39 Fort Wayne (WFWA)'),  # http://wfwa.org/
+        ('video.wfyi.org', 'WFYI Indianapolis (WFYI)'),  # http://www.wfyi.org
+        ('video.mptv.org', 'Milwaukee Public Television (WMVS)'),  # http://www.mptv.org
+        ('video.wnin.org', 'WNIN (WNIN)'),  # http://www.wnin.org/
+        ('video.wnit.org', 'WNIT Public Television (WNIT)'),  # http://www.wnit.org/
+        ('video.wpt.org', 'WPT (WPNE)'),  # http://www.wpt.org/
+        ('video.wvut.org', 'WVUT/Channel 22 (WVUT)'),  # http://wvut.org/
+        ('video.weiu.net', 'WEIU/Channel 51 (WEIU)'),  # http://www.weiu.net
+        ('video.wqpt.org', 'WQPT-TV (WQPT)'),  # http://www.wqpt.org
+        ('video.wycc.org', 'WYCC PBS Chicago (WYCC)'),  # http://www.wycc.org
+        #('lakeshorepublicmedia.org', 'Lakeshore Public Television (WYIN)'),  # http://lakeshorepublicmedia.org/
+        ('video.wipb.org', 'WIPB-TV (WIPB)'),  # http://wipb.org
+        ('video.indianapublicmedia.org', 'WTIU (WTIU)'),  # http://indianapublicmedia.org/tv/
+        ('watch.cetconnect.org', 'CET  (WCET)'),  # http://www.cetconnect.org
+        ('video.thinktv.org', 'ThinkTVNetwork (WPTD)'),  # http://www.thinktv.org
+        ('video.wbgu.org', 'WBGU-TV (WBGU)'),  # http://wbgu.org
+        ('video.wgvu.org', 'WGVU TV (WGVU)'),  # http://www.wgvu.org/
+        ('video.netnebraska.org', 'NET1 (KUON)'),  # http://netnebraska.org
+        ('video.pioneer.org', 'Pioneer Public Television (KWCM)'),  # http://www.pioneer.org
+        ('watch.sdpb.org', 'SDPB Television (KUSD)'),  # http://www.sdpb.org
+        ('video.tpt.org', 'TPT (KTCA)'),  # http://www.tpt.org
+        ('watch.ksmq.org', 'KSMQ (KSMQ)'),  # http://www.ksmq.org/
+        ('watch.kpts.org', 'KPTS/Channel 8 (KPTS)'),  # http://www.kpts.org/
+        ('watch.ktwu.org', 'KTWU/Channel 11 (KTWU)'),  # http://ktwu.org
+        #('shptv.org', 'Smoky Hills Public Television (KOOD)'),  # http://www.shptv.org
+        #('kcpt.org', 'KCPT Kansas City Public Television (KCPT)'),  # http://kcpt.org/
+        #('blueridgepbs.org', 'Blue Ridge PBS (WBRA)'),  # http://www.blueridgepbs.org/
+        ('watch.easttennesseepbs.org', 'East Tennessee PBS (WSJK)'),  # http://easttennesseepbs.org
+        ('video.wcte.tv', 'WCTE-TV (WCTE)'),  # http://www.wcte.org
+        ('video.wljt.org', 'WLJT, Channel 11 (WLJT)'),  # http://wljt.org/
+        ('video.wosu.org', 'WOSU TV (WOSU)'),  # http://wosu.org/
+        ('video.woub.org', 'WOUB/WOUC (WOUB)'),  # http://woub.org/tv/index.php?section=5
+        ('video.wvpublic.org', 'WVPB (WVPB)'),  # http://wvpublic.org/
+        ('video.wkyupbs.org', 'WKYU-PBS (WKYU)'),  # http://www.wkyupbs.org
+        #('wyes.org', 'WYES-TV/New Orleans (WYES)'),  # http://www.wyes.org
+        ('video.kera.org', 'KERA 13 (KERA)'),  # http://www.kera.org/
+        ('video.mpbn.net', 'MPBN (WCBB)'),  # http://www.mpbn.net/
+        ('video.mountainlake.org', 'Mountain Lake PBS (WCFE)'),  # http://www.mountainlake.org/
+        ('video.nhptv.org', 'NHPTV (WENH)'),  # http://nhptv.org/
+        ('video.vpt.org', 'Vermont PBS (WETK)'),  # http://www.vpt.org
+        ('video.witf.org', 'witf (WITF)'),  # http://www.witf.org
+        ('watch.wqed.org', 'WQED Multimedia (WQED)'),  # http://www.wqed.org/
+        ('video.wmht.org', 'WMHT Educational Telecommunications (WMHT)'),  # http://www.wmht.org/home/
+        ('video.deltabroadcasting.org', 'Q-TV (WDCQ)'),  # http://www.deltabroadcasting.org
+        ('video.dptv.org', 'WTVS Detroit Public TV (WTVS)'),  # http://www.dptv.org/
+        ('video.wcmu.org', 'CMU Public Television (WCMU)'),  # http://www.wcmu.org
+        ('video.wkar.org', 'WKAR-TV (WKAR)'),  # http://wkar.org/
+        ('wnmuvideo.nmu.edu', 'WNMU-TV Public TV 13 (WNMU)'),  # http://wnmutv.nmu.edu
+        ('video.wdse.org', 'WDSE - WRPT (WDSE)'),  # http://www.wdse.org/
+        ('video.wgte.org', 'WGTE TV (WGTE)'),  # http://www.wgte.org
+        ('video.lptv.org', 'Lakeland Public Television (KAWE)'),  # http://www.lakelandptv.org
+        #('prairiepublic.org', 'PRAIRIE PUBLIC (KFME)'),  # http://www.prairiepublic.org/
+        ('video.kmos.org', 'KMOS-TV - Channels 6.1, 6.2 and 6.3 (KMOS)'),  # http://www.kmos.org/
+        ('watch.montanapbs.org', 'MontanaPBS (KUSM)'),  # http://montanapbs.org
+        ('video.krwg.org', 'KRWG/Channel 22 (KRWG)'),  # http://www.krwg.org
+        ('video.kacvtv.org', 'KACV (KACV)'),  # http://www.panhandlepbs.org/home/
+        ('video.kcostv.org', 'KCOS/Channel 13 (KCOS)'),  # www.kcostv.org
+        ('video.wcny.org', 'WCNY/Channel 24 (WCNY)'),  # http://www.wcny.org
+        ('video.wned.org', 'WNED (WNED)'),  # http://www.wned.org/
+        ('watch.wpbstv.org', 'WPBS (WPBS)'),  # http://www.wpbstv.org
+        ('video.wskg.org', 'WSKG Public TV (WSKG)'),  # http://wskg.org
+        ('video.wxxi.org', 'WXXI (WXXI)'),  # http://wxxi.org
+        ('video.wpsu.org', 'WPSU (WPSU)'),  # http://www.wpsu.org
+        #('wqln.org', 'WQLN/Channel 54 (WQLN)'),  # http://www.wqln.org
+        ('on-demand.wvia.org', 'WVIA Public Media Studios (WVIA)'),  # http://www.wvia.org/
+        ('video.wtvi.org', 'WTVI (WTVI)'),  # http://www.wtvi.org/
+        #('whro.org', 'WHRO (WHRO)'),  # http://whro.org
+        ('video.westernreservepublicmedia.org', 'Western Reserve PBS (WNEO)'),  # http://www.WesternReservePublicMedia.org/
+        ('video.ideastream.org', 'WVIZ/PBS ideastream (WVIZ)'),  # http://www.wviz.org/
+        ('video.kcts9.org', 'KCTS 9 (KCTS)'),  # http://kcts9.org/
+        ('video.basinpbs.org', 'Basin PBS (KPBT)'),  # http://www.basinpbs.org
+        ('video.houstonpbs.org', 'KUHT / Channel 8 (KUHT)'),  # http://www.houstonpublicmedia.org/
+        #('tamu.edu', 'KAMU - TV (KAMU)'),  # http://KAMU.tamu.edu
+        #('kedt.org', 'KEDT/Channel 16 (KEDT)'),  # http://www.kedt.org
+        ('video.klrn.org', 'KLRN (KLRN)'),  # http://www.klrn.org
+        ('video.klru.tv', 'KLRU (KLRU)'),  # http://www.klru.org
+        #('kmbh.org', 'KMBH-TV (KMBH)'),  # http://www.kmbh.org
+        #('knct.org', 'KNCT (KNCT)'),  # http://www.knct.org
+        #('ktxt.org', 'KTTZ-TV (KTXT)'),  # http://www.ktxt.org
+        ('video.wtjx.org', 'WTJX Channel 12 (WTJX)'),  # http://www.wtjx.org/
+        ('video.ideastations.org', 'WCVE PBS (WCVE)'),  # http://ideastations.org/
+        ('video.kbtc.org', 'KBTC Public Television (KBTC)'),  # http://kbtc.org
     )
 
     IE_NAME = 'pbs'
@@ -182,13 +183,13 @@ class PBSIE(InfoExtractor):
     _VALID_URL = r'''(?x)https?://
         (?:
            # Direct video URL
-           (?:videos?|watch)\.(?:%s)/(?:viralplayer|video)/(?P<id>[0-9]+)/? |
+           (?:%s)/(?:viralplayer|video)/(?P<id>[0-9]+)/? |
            # Article with embedded player (or direct video)
            (?:www\.)?pbs\.org/(?:[^/]+/){2,5}(?P<presumptive_id>[^/]+?)(?:\.html)?/?(?:$|[?\#]) |
            # Player
            (?:video|player)\.pbs\.org/(?:widget/)?partnerplayer/(?P<player_id>[^/]+)/
         )
-    ''' % '|'.join(list(zip(*_STATIONS))[0])
+    ''' % '|'.join(re.escape(p) for p in list(zip(*_STATIONS))[0])
 
     _TESTS = [
         {

From 0b534d2adc56ead4fcb085d886e165dfeaf2852f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 10 Dec 2015 21:27:47 +0600
Subject: [PATCH 3312/6219] [dailymotion] Restrict player v5 regex (Closes
 #7826)

---
 youtube_dl/extractor/dailymotion.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index ab7f3aec4..fd6d14d23 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -122,7 +122,9 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
             webpage, 'comment count', fatal=False))
 
         player_v5 = self._search_regex(
-            [r'buildPlayer\(({.+?})\);', r'playerV5\s*=\s*dmp\.create\([^,]+?,\s*({.+?})\);'],
+            [r'buildPlayer\(({.+?})\);\n',  # See https://github.com/rg3/youtube-dl/issues/7826
+             r'playerV5\s*=\s*dmp\.create\([^,]+?,\s*({.+?})\);',
+             r'buildPlayer\(({.+?})\);'],
             webpage, 'player v5', default=None)
         if player_v5:
             player = self._parse_json(player_v5, video_id)

From cbd2ffd031ea1a3388dce96db1d8093daa3c3b77 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 10 Dec 2015 21:29:07 +0600
Subject: [PATCH 3313/6219] [dailymotion] Fix subtitles extraction

---
 youtube_dl/extractor/dailymotion.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index fd6d14d23..e260a0c59 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -174,11 +174,13 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
             uploader_id = metadata.get('owner', {}).get('id')
 
             subtitles = {}
-            for subtitle_lang, subtitle in metadata.get('subtitles', {}).get('data', {}).items():
-                subtitles[subtitle_lang] = [{
-                    'ext': determine_ext(subtitle_url),
-                    'url': subtitle_url,
-                } for subtitle_url in subtitle.get('urls', [])]
+            subtitles_data = metadata.get('subtitles', {}).get('data', {})
+            if subtitles_data and isinstance(subtitles_data, dict):
+                for subtitle_lang, subtitle in subtitles_data.items():
+                    subtitles[subtitle_lang] = [{
+                        'ext': determine_ext(subtitle_url),
+                        'url': subtitle_url,
+                    } for subtitle_url in subtitle.get('urls', [])]
 
             return {
                 'id': video_id,

From 176ccefcd89af5393b7cab14237097c54dd0ee14 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 10 Dec 2015 21:33:40 +0600
Subject: [PATCH 3314/6219] [pbs] PEP 8

---
 youtube_dl/extractor/pbs.py | 46 ++++++++++++++++++-------------------
 1 file changed, 23 insertions(+), 23 deletions(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index d030236d4..604a0dd22 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -26,9 +26,9 @@ class PBSIE(InfoExtractor):
         ('video.wtcitv.org', 'WTCI (WTCI)'),  # http://www.wtcitv.org
         ('video.pba.org', 'WPBA/Channel 30 (WPBA)'),  # http://pba.org/
         ('video.alaskapublic.org', 'Alaska Public Media (KAKM)'),  # http://alaskapublic.org/kakm
-        #('kuac.org', 'KUAC (KUAC)'),  # http://kuac.org/kuac-tv/
-        #('ktoo.org', '360 North (KTOO)'),  # http://www.ktoo.org/
-        #('azpm.org', 'KUAT 6 (KUAT)'),  # http://www.azpm.org/
+        # ('kuac.org', 'KUAC (KUAC)'),  # http://kuac.org/kuac-tv/
+        # ('ktoo.org', '360 North (KTOO)'),  # http://www.ktoo.org/
+        # ('azpm.org', 'KUAT 6 (KUAT)'),  # http://www.azpm.org/
         ('video.azpbs.org', 'Arizona PBS (KAET)'),  # http://www.azpbs.org
         ('portal.knme.org', 'KNME-TV/Channel 5 (KNME)'),  # http://www.newmexicopbs.org/
         ('video.vegaspbs.org', 'Vegas PBS (KLVX)'),  # http://vegaspbs.org/
@@ -49,9 +49,9 @@ class PBSIE(InfoExtractor):
         ('video.cptv.org', 'CONNECTICUT PUBLIC TELEVISION (WEDH)'),  # http://cptv.org
         ('watch.knpb.org', 'KNPB Channel 5 (KNPB)'),  # http://www.knpb.org/
         ('video.soptv.org', 'SOPTV (KSYS)'),  # http://www.soptv.org
-        #('klcs.org', 'KLCS/Channel 58 (KLCS)'),  # http://www.klcs.org
-        #('krcb.org', 'KRCB Television & Radio (KRCB)'),  # http://www.krcb.org
-        #('kvcr.org', 'KVCR TV/DT/FM :: Vision for the Future (KVCR)'),  # http://kvcr.org
+        # ('klcs.org', 'KLCS/Channel 58 (KLCS)'),  # http://www.klcs.org
+        # ('krcb.org', 'KRCB Television & Radio (KRCB)'),  # http://www.krcb.org
+        # ('kvcr.org', 'KVCR TV/DT/FM :: Vision for the Future (KVCR)'),  # http://kvcr.org
         ('video.rmpbs.org', 'Rocky Mountain PBS (KRMA)'),  # http://www.rmpbs.org
         ('video.kenw.org', 'KENW-TV3 (KENW)'),  # http://www.kenw.org
         ('video.kued.org', 'KUED Channel 7 (KUED)'),  # http://www.kued.org
@@ -62,7 +62,7 @@ class PBSIE(InfoExtractor):
         ('video.wgbh.org', 'WGBH/Channel 2 (WGBH)'),  # http://wgbh.org
         ('video.wgby.org', 'WGBY (WGBY)'),  # http://www.wgby.org
         ('watch.njtvonline.org', 'NJTV Public Media NJ (WNJT)'),  # http://www.njtvonline.org/
-        #('ripbs.org', 'Rhode Island PBS (WSBE)'),  # http://www.ripbs.org/home/
+        # ('ripbs.org', 'Rhode Island PBS (WSBE)'),  # http://www.ripbs.org/home/
         ('watch.wliw.org', 'WLIW21 (WLIW)'),  # http://www.wliw.org/
         ('video.mpt.tv', 'mpt/Maryland Public Television (WMPB)'),  # http://www.mpt.org
         ('watch.weta.org', 'WETA Television and Radio (WETA)'),  # http://www.weta.org
@@ -72,7 +72,7 @@ class PBSIE(InfoExtractor):
         ('video.whut.org', 'Howard University Television (WHUT)'),  # http://www.whut.org
         ('video.wedu.org', 'WEDU PBS (WEDU)'),  # http://www.wedu.org
         ('video.wgcu.org', 'WGCU Public Media (WGCU)'),  # http://www.wgcu.org/
-        #('wjct.org', 'WJCT Public Broadcasting (WJCT)'),  # http://www.wjct.org
+        # ('wjct.org', 'WJCT Public Broadcasting (WJCT)'),  # http://www.wjct.org
         ('video.wpbt2.org', 'WPBT2 (WPBT)'),  # http://www.wpbt2.org
         ('video.wucftv.org', 'WUCF TV (WUCF)'),  # http://wucftv.org
         ('video.wuft.org', 'WUFT/Channel 5 (WUFT)'),  # http://www.wuft.org
@@ -81,7 +81,7 @@ class PBSIE(InfoExtractor):
         ('video.wusf.usf.edu', 'WUSF Public Broadcasting (WUSF)'),  # http://wusf.org/
         ('video.scetv.org', 'ETV (WRLK)'),  # http://www.scetv.org
         ('video.unctv.org', 'UNC-TV (WUNC)'),  # http://www.unctv.org/
-        #('pbsguam.org', 'PBS Guam (KGTF)'),  # http://www.pbsguam.org/
+        # ('pbsguam.org', 'PBS Guam (KGTF)'),  # http://www.pbsguam.org/
         ('video.pbshawaii.org', 'PBS Hawaii - Oceanic Cable Channel 10 (KHET)'),  # http://www.pbshawaii.org/
         ('video.idahoptv.org', 'Idaho Public Television (KAID)'),  # http://idahoptv.org
         ('video.ksps.org', 'KSPS (KSPS)'),  # http://www.ksps.org/home/
@@ -90,7 +90,7 @@ class PBSIE(InfoExtractor):
         ('video.will.illinois.edu', 'WILL-TV (WILL)'),  # http://will.illinois.edu/
         ('video.networkknowledge.tv', 'Network Knowledge - WSEC/Springfield (WSEC)'),  # http://www.wsec.tv
         ('video.wttw.com', 'WTTW11 (WTTW)'),  # http://www.wttw.com/
-        #('wtvp.org', 'WTVP & WTVP.org, Public Media for Central Illinois (WTVP)'),  # http://www.wtvp.org/
+        # ('wtvp.org', 'WTVP & WTVP.org, Public Media for Central Illinois (WTVP)'),  # http://www.wtvp.org/
         ('video.iptv.org', 'Iowa Public Television/IPTV (KDIN)'),  # http://www.iptv.org/
         ('video.ninenet.org', 'Nine Network (KETC)'),  # http://www.ninenet.org
         ('video.wfwa.org', 'PBS39 Fort Wayne (WFWA)'),  # http://wfwa.org/
@@ -103,7 +103,7 @@ class PBSIE(InfoExtractor):
         ('video.weiu.net', 'WEIU/Channel 51 (WEIU)'),  # http://www.weiu.net
         ('video.wqpt.org', 'WQPT-TV (WQPT)'),  # http://www.wqpt.org
         ('video.wycc.org', 'WYCC PBS Chicago (WYCC)'),  # http://www.wycc.org
-        #('lakeshorepublicmedia.org', 'Lakeshore Public Television (WYIN)'),  # http://lakeshorepublicmedia.org/
+        # ('lakeshorepublicmedia.org', 'Lakeshore Public Television (WYIN)'),  # http://lakeshorepublicmedia.org/
         ('video.wipb.org', 'WIPB-TV (WIPB)'),  # http://wipb.org
         ('video.indianapublicmedia.org', 'WTIU (WTIU)'),  # http://indianapublicmedia.org/tv/
         ('watch.cetconnect.org', 'CET  (WCET)'),  # http://www.cetconnect.org
@@ -117,9 +117,9 @@ class PBSIE(InfoExtractor):
         ('watch.ksmq.org', 'KSMQ (KSMQ)'),  # http://www.ksmq.org/
         ('watch.kpts.org', 'KPTS/Channel 8 (KPTS)'),  # http://www.kpts.org/
         ('watch.ktwu.org', 'KTWU/Channel 11 (KTWU)'),  # http://ktwu.org
-        #('shptv.org', 'Smoky Hills Public Television (KOOD)'),  # http://www.shptv.org
-        #('kcpt.org', 'KCPT Kansas City Public Television (KCPT)'),  # http://kcpt.org/
-        #('blueridgepbs.org', 'Blue Ridge PBS (WBRA)'),  # http://www.blueridgepbs.org/
+        # ('shptv.org', 'Smoky Hills Public Television (KOOD)'),  # http://www.shptv.org
+        # ('kcpt.org', 'KCPT Kansas City Public Television (KCPT)'),  # http://kcpt.org/
+        # ('blueridgepbs.org', 'Blue Ridge PBS (WBRA)'),  # http://www.blueridgepbs.org/
         ('watch.easttennesseepbs.org', 'East Tennessee PBS (WSJK)'),  # http://easttennesseepbs.org
         ('video.wcte.tv', 'WCTE-TV (WCTE)'),  # http://www.wcte.org
         ('video.wljt.org', 'WLJT, Channel 11 (WLJT)'),  # http://wljt.org/
@@ -127,7 +127,7 @@ class PBSIE(InfoExtractor):
         ('video.woub.org', 'WOUB/WOUC (WOUB)'),  # http://woub.org/tv/index.php?section=5
         ('video.wvpublic.org', 'WVPB (WVPB)'),  # http://wvpublic.org/
         ('video.wkyupbs.org', 'WKYU-PBS (WKYU)'),  # http://www.wkyupbs.org
-        #('wyes.org', 'WYES-TV/New Orleans (WYES)'),  # http://www.wyes.org
+        # ('wyes.org', 'WYES-TV/New Orleans (WYES)'),  # http://www.wyes.org
         ('video.kera.org', 'KERA 13 (KERA)'),  # http://www.kera.org/
         ('video.mpbn.net', 'MPBN (WCBB)'),  # http://www.mpbn.net/
         ('video.mountainlake.org', 'Mountain Lake PBS (WCFE)'),  # http://www.mountainlake.org/
@@ -144,7 +144,7 @@ class PBSIE(InfoExtractor):
         ('video.wdse.org', 'WDSE - WRPT (WDSE)'),  # http://www.wdse.org/
         ('video.wgte.org', 'WGTE TV (WGTE)'),  # http://www.wgte.org
         ('video.lptv.org', 'Lakeland Public Television (KAWE)'),  # http://www.lakelandptv.org
-        #('prairiepublic.org', 'PRAIRIE PUBLIC (KFME)'),  # http://www.prairiepublic.org/
+        # ('prairiepublic.org', 'PRAIRIE PUBLIC (KFME)'),  # http://www.prairiepublic.org/
         ('video.kmos.org', 'KMOS-TV - Channels 6.1, 6.2 and 6.3 (KMOS)'),  # http://www.kmos.org/
         ('watch.montanapbs.org', 'MontanaPBS (KUSM)'),  # http://montanapbs.org
         ('video.krwg.org', 'KRWG/Channel 22 (KRWG)'),  # http://www.krwg.org
@@ -156,22 +156,22 @@ class PBSIE(InfoExtractor):
         ('video.wskg.org', 'WSKG Public TV (WSKG)'),  # http://wskg.org
         ('video.wxxi.org', 'WXXI (WXXI)'),  # http://wxxi.org
         ('video.wpsu.org', 'WPSU (WPSU)'),  # http://www.wpsu.org
-        #('wqln.org', 'WQLN/Channel 54 (WQLN)'),  # http://www.wqln.org
+        # ('wqln.org', 'WQLN/Channel 54 (WQLN)'),  # http://www.wqln.org
         ('on-demand.wvia.org', 'WVIA Public Media Studios (WVIA)'),  # http://www.wvia.org/
         ('video.wtvi.org', 'WTVI (WTVI)'),  # http://www.wtvi.org/
-        #('whro.org', 'WHRO (WHRO)'),  # http://whro.org
+        # ('whro.org', 'WHRO (WHRO)'),  # http://whro.org
         ('video.westernreservepublicmedia.org', 'Western Reserve PBS (WNEO)'),  # http://www.WesternReservePublicMedia.org/
         ('video.ideastream.org', 'WVIZ/PBS ideastream (WVIZ)'),  # http://www.wviz.org/
         ('video.kcts9.org', 'KCTS 9 (KCTS)'),  # http://kcts9.org/
         ('video.basinpbs.org', 'Basin PBS (KPBT)'),  # http://www.basinpbs.org
         ('video.houstonpbs.org', 'KUHT / Channel 8 (KUHT)'),  # http://www.houstonpublicmedia.org/
-        #('tamu.edu', 'KAMU - TV (KAMU)'),  # http://KAMU.tamu.edu
-        #('kedt.org', 'KEDT/Channel 16 (KEDT)'),  # http://www.kedt.org
+        # ('tamu.edu', 'KAMU - TV (KAMU)'),  # http://KAMU.tamu.edu
+        # ('kedt.org', 'KEDT/Channel 16 (KEDT)'),  # http://www.kedt.org
         ('video.klrn.org', 'KLRN (KLRN)'),  # http://www.klrn.org
         ('video.klru.tv', 'KLRU (KLRU)'),  # http://www.klru.org
-        #('kmbh.org', 'KMBH-TV (KMBH)'),  # http://www.kmbh.org
-        #('knct.org', 'KNCT (KNCT)'),  # http://www.knct.org
-        #('ktxt.org', 'KTTZ-TV (KTXT)'),  # http://www.ktxt.org
+        # ('kmbh.org', 'KMBH-TV (KMBH)'),  # http://www.kmbh.org
+        # ('knct.org', 'KNCT (KNCT)'),  # http://www.knct.org
+        # ('ktxt.org', 'KTTZ-TV (KTXT)'),  # http://www.ktxt.org
         ('video.wtjx.org', 'WTJX Channel 12 (WTJX)'),  # http://www.wtjx.org/
         ('video.ideastations.org', 'WCVE PBS (WCVE)'),  # http://ideastations.org/
         ('video.kbtc.org', 'KBTC Public Television (KBTC)'),  # http://kbtc.org

From a8abf124c80cac3f2a75a2261523a3836376888d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 10 Dec 2015 21:54:48 +0600
Subject: [PATCH 3315/6219] [dailymotion] Add subtitles test URL for reference

---
 youtube_dl/extractor/dailymotion.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index e260a0c59..428556213 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -99,6 +99,11 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
         {
             'url': 'http://www.dailymotion.com/video/xhza0o',
             'only_matching': True,
+        },
+        # with subtitles
+        {
+            'url': 'http://www.dailymotion.com/video/x20su5f_the-power-of-nightmares-1-the-rise-of-the-politics-of-fear-bbc-2004_news',
+            'only_matching': True,
         }
     ]
 

From f7b8dd63f0b99a6953f1ab0645c923ca2f1e157d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 10 Dec 2015 17:05:13 +0100
Subject: [PATCH 3316/6219] release 2015.12.10

---
 CONTRIBUTING.md        | 4 ++--
 docs/supportedsites.md | 2 +-
 youtube_dl/version.py  | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 28aae94f8..f3fe0d432 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -34,7 +34,7 @@ For bug reports, this means that your report should contain the *complete* outpu
 
 If your server has multiple IPs or you suspect censorship, adding `--call-home` may be a good idea to get more diagnostics. If the error is `ERROR: Unable to extract ...` and you cannot reproduce it from multiple countries, add `--dump-pages` (warning: this will yield a rather large output, redirect it to the file `log.txt` by adding `>log.txt 2>&1` to your command-line) or upload the `.dump` files you get when you add `--write-pages` [somewhere](https://gist.github.com/).
 
-**Site support requests must contain an example URL**. An example URL is a URL you might want to download, like http://www.youtube.com/watch?v=BaW_jenozKc . There should be an obvious video present. Except under very special circumstances, the main page of a video service (e.g. http://www.youtube.com/ ) is *not* an example URL.
+**Site support requests must contain an example URL**. An example URL is a URL you might want to download, like `http://www.youtube.com/watch?v=BaW_jenozKc`. There should be an obvious video present. Except under very special circumstances, the main page of a video service (e.g. `http://www.youtube.com/`) is *not* an example URL.
 
 ###  Are you using the latest version?
 
@@ -42,7 +42,7 @@ Before reporting any issue, type `youtube-dl -U`. This should report that you're
 
 ###  Is the issue already documented?
 
-Make sure that someone has not already opened the issue you're trying to open. Search at the top of the window or at https://github.com/rg3/youtube-dl/search?type=Issues . If there is an issue, feel free to write something along the lines of "This affects me as well, with version 2015.01.01. Here is some more information on the issue: ...". While some issues may be old, a new post into them often spurs rapid activity.
+Make sure that someone has not already opened the issue you're trying to open. Search at the top of the window or browse the [GitHub Issues](https://github.com/rg3/youtube-dl/search?type=Issues) of this repository. If there is an issue, feel free to write something along the lines of "This affects me as well, with version 2015.01.01. Here is some more information on the issue: ...". While some issues may be old, a new post into them often spurs rapid activity.
 
 ###  Why are existing options not enough?
 
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 4f19cb6f5..3bb3abb25 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -400,7 +400,7 @@
  - **orf:tvthek**: ORF TVthek
  - **parliamentlive.tv**: UK parliament videos
  - **Patreon**
- - **pbs**: Public Broadcasting Service (PBS) and member stations: PBS: Public Broadcasting Service, APT - Alabama Public Television (WBIQ), GPB/Georgia Public Broadcasting (WGTV), Mississippi Public Broadcasting (WMPN), Nashville Public Television (WNPT), WFSU-TV (WFSU), WSRE (WSRE), WTCI (WTCI), WPBA/Channel 30 (WPBA), Alaska Public Media (KAKM), KUAC (KUAC), 360 North (KTOO), KUAT 6 (KUAT), Arizona PBS (KAET), KNME-TV/Channel 5 (KNME), Vegas PBS (KLVX), AETN/ARKANSAS ETV NETWORK (KETS), KET (WKLE), WKNO/Channel 10 (WKNO), LPB/LOUISIANA PUBLIC BROADCASTING (WLPB), Mississippi Public Broadcasting (WMPN), OETA (KETA), Ozarks Public Television (KOZK), WSIU Public Broadcasting (WSIU), KEET TV (KEET), KIXE/Channel 9 (KIXE), KPBS San Diego (KPBS), KQED (KQED), KVIE Public Television (KVIE), PBS SoCal/KOCE (KOCE), ValleyPBS (KVPT), APT - Alabama Public Television (WBIQ), CONNECTICUT PUBLIC TELEVISION (WEDH), Arizona PBS (KAET), KNPB Channel 5 (KNPB), SOPTV (KSYS), KLCS/Channel 58 (KLCS), KRCB Television & Radio (KRCB), KVCR TV/DT/FM :: Vision for the Future (KVCR), Rocky Mountain PBS (KRMA), KENW-TV3 (KENW), KUED Channel 7 (KUED), KNME-TV/Channel 5 (KNME), Wyoming PBS (KCWC), Colorado Public Television / KBDI 12 (KBDI), KBYU-TV (KBYU), CONNECTICUT PUBLIC TELEVISION (WEDH), Thirteen/WNET New York (WNET), WGBH/Channel 2 (WGBH), WGBY (WGBY), NJTV Public Media NJ (WNJT), Rhode Island PBS (WSBE), WLIW21 (WLIW), mpt/Maryland Public Television (WMPB), WETA Television and Radio (WETA), WHYY (WHYY), NJTV Public Media NJ (WNJT), PBS 39 (WLVT), WLIW21 (WLIW), WETA Television and Radio (WETA), mpt/Maryland Public Television (WMPB), WVPT - Your Source for PBS and More! (WVPT), Howard University Television (WHUT), WEDU PBS (WEDU), WFSU-TV (WFSU), WGCU Public Media (WGCU), WJCT Public Broadcasting (WJCT), WPBT2 (WPBT), WSRE (WSRE), WUCF TV (WUCF), WUFT/Channel 5 (WUFT), WXEL/Channel 42 (WXEL), APT - Alabama Public Television (WBIQ), GPB/Georgia Public Broadcasting (WGTV), WLRN/Channel 17 (WLRN), WUSF Public Broadcasting (WUSF), NJTV Public Media NJ (WNJT), GPB/Georgia Public Broadcasting (WGTV), APT - Alabama Public Television (WBIQ), ETV (WRLK), UNC-TV (WUNC), WFSU-TV (WFSU), WJCT Public Broadcasting (WJCT), WTCI (WTCI), WPBA/Channel 30 (WPBA), PBS Guam (KGTF), PBS Hawaii - Oceanic Cable Channel 10 (KHET), Idaho Public Television (KAID), KSPS (KSPS), KUED Channel 7 (KUED), OPB (KOPB), SOPTV (KSYS), Wyoming PBS (KCWC), Colorado Public Television / KBDI 12 (KBDI), KBYU-TV (KBYU), KWSU/Channel 10 & KTNW/Channel 31 (KWSU), WILL-TV (WILL), Network Knowledge - WSEC/Springfield (WSEC), WSIU Public Broadcasting (WSIU), WTTW11 (WTTW), WTVP & WTVP.org, Public Media for Central Illinois (WTVP), Iowa Public Television/IPTV (KDIN), KET (WKLE), Nine Network (KETC), PBS39 Fort Wayne (WFWA), WFYI Indianapolis (WFYI), Milwaukee Public Television (WMVS), WNIN (WNIN), WNIT Public Television (WNIT), WPT (WPNE), WVUT/Channel 22 (WVUT), WEIU/Channel 51 (WEIU), WQPT-TV (WQPT), WYCC PBS Chicago (WYCC), Lakeshore Public Television (WYIN), PBS39 Fort Wayne (WFWA), WFYI Indianapolis (WFYI), WIPB-TV (WIPB), WNIN (WNIN), WNIT Public Television (WNIT), WTIU (WTIU), WVUT/Channel 22 (WVUT), CET  (WCET), KET (WKLE), ThinkTVNetwork (WPTD), WBGU-TV (WBGU), WGVU TV (WGVU), WILL-TV (WILL), Network Knowledge - WSEC/Springfield (WSEC), WSIU Public Broadcasting (WSIU), WTTW11 (WTTW), Lakeshore Public Television (WYIN), WEIU/Channel 51 (WEIU), WYCC PBS Chicago (WYCC), Iowa Public Television/IPTV (KDIN), NET1 (KUON), Pioneer Public Television (KWCM), SDPB Television (KUSD), TPT (KTCA), Network Knowledge - WSEC/Springfield (WSEC), WPT (WPNE), KSMQ (KSMQ), WQPT-TV (WQPT), KPTS/Channel 8 (KPTS), KTWU/Channel 11 (KTWU), Smoky Hills Public Television (KOOD), KCPT Kansas City Public Television (KCPT), NET1 (KUON), OETA (KETA), Ozarks Public Television (KOZK), Rocky Mountain PBS (KRMA), Colorado Public Television / KBDI 12 (KBDI), KET (WKLE), Blue Ridge PBS (WBRA), CET  (WCET), East Tennessee PBS (WSJK), Nashville Public Television (WNPT), ThinkTVNetwork (WPTD), WCTE-TV (WCTE), WLJT, Channel 11 (WLJT), WNIN (WNIN), WOSU TV (WOSU), WOUB/WOUC (WOUB), WSIU Public Broadcasting (WSIU), WVPB (WVPB), WKYU-PBS (WKYU), LPB/LOUISIANA PUBLIC BROADCASTING (WLPB), WYES-TV/New Orleans (WYES), AETN/ARKANSAS ETV NETWORK (KETS), KERA 13 (KERA), Mississippi Public Broadcasting (WMPN), MPBN (WCBB), Mountain Lake PBS (WCFE), NHPTV (WENH), Vermont PBS (WETK), WGBH/Channel 2 (WGBH), mpt/Maryland Public Television (WMPB), Thirteen/WNET New York (WNET), WETA Television and Radio (WETA), WHYY (WHYY), witf (WITF), WQED Multimedia (WQED), WVPB (WVPB), WVPT - Your Source for PBS and More! (WVPT), Howard University Television (WHUT), WLIW21 (WLIW), WGBH/Channel 2 (WGBH), WGBY (WGBY), CONNECTICUT PUBLIC TELEVISION (WEDH), NHPTV (WENH), Vermont PBS (WETK), WMHT Educational Telecommunications (WMHT), Rhode Island PBS (WSBE), Q-TV (WDCQ), WTVS Detroit Public TV (WTVS), CMU Public Television (WCMU), WGVU TV (WGVU), WKAR-TV (WKAR), WNMU-TV Public TV 13 (WNMU), WDSE - WRPT (WDSE), WBGU-TV (WBGU), WGTE TV (WGTE), WNIT Public Television (WNIT), WPT (WPNE), WTTW11 (WTTW), WYCC PBS Chicago (WYCC), Lakeland Public Television (KAWE), WDSE - WRPT (WDSE), Pioneer Public Television (KWCM), TPT (KTCA), Iowa Public Television/IPTV (KDIN), PRAIRIE PUBLIC (KFME), Nine Network (KETC), SDPB Television (KUSD), WPT (WPNE), KSMQ (KSMQ), Mississippi Public Broadcasting (WMPN), AETN/ARKANSAS ETV NETWORK (KETS), APT - Alabama Public Television (WBIQ), WKNO/Channel 10 (WKNO), LPB/LOUISIANA PUBLIC BROADCASTING (WLPB), WSRE (WSRE), WYES-TV/New Orleans (WYES), KCPT Kansas City Public Television (KCPT), KMOS-TV - Channels 6.1, 6.2 and 6.3 (KMOS), Nine Network (KETC), Ozarks Public Television (KOZK), AETN/ARKANSAS ETV NETWORK (KETS), WTVS Detroit Public TV (WTVS), Iowa Public Television/IPTV (KDIN), KET (WKLE), WKNO/Channel 10 (WKNO), KTWU/Channel 11 (KTWU), NET1 (KUON), Rocky Mountain PBS (KRMA), Network Knowledge - WSEC/Springfield (WSEC), WSIU Public Broadcasting (WSIU), MontanaPBS (KUSM), Arizona PBS (KAET), Idaho Public Television (KAID), PRAIRIE PUBLIC (KFME), KSPS (KSPS), Rocky Mountain PBS (KRMA), SDPB Television (KUSD), NET1 (KUON), Iowa Public Television/IPTV (KDIN), KTWU/Channel 11 (KTWU), Pioneer Public Television (KWCM), Rocky Mountain PBS (KRMA), SDPB Television (KUSD), Wyoming PBS (KCWC), Colorado Public Television / KBDI 12 (KBDI), KNPB Channel 5 (KNPB), Vegas PBS (KLVX), Arizona PBS (KAET), KUED Channel 7 (KUED), PBS SoCal/KOCE (KOCE), KBYU-TV (KBYU), NHPTV (WENH), ETV (WRLK), Mountain Lake PBS (WCFE), MPBN (WCBB), Vermont PBS (WETK), WGBH/Channel 2 (WGBH), WGBY (WGBY), CONNECTICUT PUBLIC TELEVISION (WEDH), Thirteen/WNET New York (WNET), WHYY (WHYY), NJTV Public Media NJ (WNJT), PBS 39 (WLVT), WLIW21 (WLIW), KENW-TV3 (KENW), KRWG/Channel 22 (KRWG), KNME-TV/Channel 5 (KNME), KUAT 6 (KUAT), KACV (KACV), KCOS/Channel 13 (KCOS), KUED Channel 7 (KUED), OETA (KETA), Rocky Mountain PBS (KRMA), Colorado Public Television / KBDI 12 (KBDI), KBYU-TV (KBYU), Mountain Lake PBS (WCFE), Thirteen/WNET New York (WNET), WCNY/Channel 24 (WCNY), WMHT Educational Telecommunications (WMHT), WNED (WNED), WPBS (WPBS), WSKG Public TV (WSKG), WXXI (WXXI), CONNECTICUT PUBLIC TELEVISION (WEDH), NHPTV (WENH), Vermont PBS (WETK), WGBH/Channel 2 (WGBH), WPSU (WPSU), WQLN/Channel 54 (WQLN), WVIA Public Media Studios (WVIA), WLIW21 (WLIW), NJTV Public Media NJ (WNJT), UNC-TV (WUNC), WTVI (WTVI), Blue Ridge PBS (WBRA), ETV (WRLK), GPB/Georgia Public Broadcasting (WGTV), WHRO (WHRO), WTCI (WTCI), WPBA/Channel 30 (WPBA), PRAIRIE PUBLIC (KFME), SDPB Television (KUSD), CET  (WCET), ThinkTVNetwork (WPTD), WBGU-TV (WBGU), WGTE TV (WGTE), Western Reserve PBS (WNEO), WOSU TV (WOSU), WOUB/WOUC (WOUB), WVIZ/PBS ideastream (WVIZ), WTVS Detroit Public TV (WTVS), KET (WKLE), PBS39 Fort Wayne (WFWA), WIPB-TV (WIPB), WQED Multimedia (WQED), WQLN/Channel 54 (WQLN), WVPB (WVPB), OETA (KETA), AETN/ARKANSAS ETV NETWORK (KETS), KACV (KACV), KERA 13 (KERA), LPB/LOUISIANA PUBLIC BROADCASTING (WLPB), Ozarks Public Television (KOZK), OPB (KOPB), SOPTV (KSYS), Idaho Public Television (KAID), KCTS 9 (KCTS), KEET TV (KEET), KSPS (KSPS), KWSU/Channel 10 & KTNW/Channel 31 (KWSU), WHYY (WHYY), witf (WITF), WPSU (WPSU), WQED Multimedia (WQED), WQLN/Channel 54 (WQLN), WVIA Public Media Studios (WVIA), CONNECTICUT PUBLIC TELEVISION (WEDH), mpt/Maryland Public Television (WMPB), Thirteen/WNET New York (WNET), WETA Television and Radio (WETA), WNED (WNED), Western Reserve PBS (WNEO), WSKG Public TV (WSKG), WVIZ/PBS ideastream (WVIZ), WVPB (WVPB), WVPT - Your Source for PBS and More! (WVPT), PBS 39 (WLVT), NJTV Public Media NJ (WNJT), Howard University Television (WHUT), WLIW21 (WLIW), CONNECTICUT PUBLIC TELEVISION (WEDH), NHPTV (WENH), WGBH/Channel 2 (WGBH), Rhode Island PBS (WSBE), ETV (WRLK), GPB/Georgia Public Broadcasting (WGTV), UNC-TV (WUNC), WTVI (WTVI), WPBA/Channel 30 (WPBA), SDPB Television (KUSD), Iowa Public Television/IPTV (KDIN), PRAIRIE PUBLIC (KFME), NET1 (KUON), Pioneer Public Television (KWCM), East Tennessee PBS (WSJK), WKNO/Channel 10 (WKNO), Nashville Public Television (WNPT), WCTE-TV (WCTE), WLJT, Channel 11 (WLJT), WTCI (WTCI), APT - Alabama Public Television (WBIQ), Blue Ridge PBS (WBRA), GPB/Georgia Public Broadcasting (WGTV), Lakeland Public Television (KAWE), KET (WKLE), Mississippi Public Broadcasting (WMPN), UNC-TV (WUNC), Western Reserve PBS (WNEO), WSIU Public Broadcasting (WSIU), WVIZ/PBS ideastream (WVIZ), WKYU-PBS (WKYU), WPBA/Channel 30 (WPBA), Basin PBS (KPBT), KUHT / Channel 8 (KUHT), KACV (KACV), KAMU - TV (KAMU), KCOS/Channel 13 (KCOS), KEDT/Channel 16 (KEDT), KERA 13 (KERA), KLRN (KLRN), KLRU (KLRU), KMBH-TV (KMBH), KNCT (KNCT), KTTZ-TV (KTXT), AETN/ARKANSAS ETV NETWORK (KETS), KENW-TV3 (KENW), KRWG/Channel 22 (KRWG), LPB/LOUISIANA PUBLIC BROADCASTING (WLPB), NET1 (KUON), KNME-TV/Channel 5 (KNME), Nashville Public Television (WNPT), OETA (KETA), Rocky Mountain PBS (KRMA), KUED Channel 7 (KUED), Idaho Public Television (KAID), Vegas PBS (KLVX), KBYU-TV (KBYU), Vermont PBS (WETK), Mountain Lake PBS (WCFE), NHPTV (WENH), WGBH/Channel 2 (WGBH), WGBY (WGBY), WMHT Educational Telecommunications (WMHT), WTJX Channel 12 (WTJX), Blue Ridge PBS (WBRA), WCVE PBS (WCVE), WHRO (WHRO), WVPT - Your Source for PBS and More! (WVPT), East Tennessee PBS (WSJK), KET (WKLE), mpt/Maryland Public Television (WMPB), UNC-TV (WUNC), WETA Television and Radio (WETA), Howard University Television (WHUT), KCTS 9 (KCTS), KSPS (KSPS), Idaho Public Television (KAID), OPB (KOPB), KBTC Public Television (KBTC), KWSU/Channel 10 & KTNW/Channel 31 (KWSU), WVPB (WVPB), Blue Ridge PBS (WBRA), KET (WKLE), mpt/Maryland Public Television (WMPB), WETA Television and Radio (WETA), Western Reserve PBS (WNEO), WOSU TV (WOSU), WOUB/WOUC (WOUB), WQED Multimedia (WQED), WVPT - Your Source for PBS and More! (WVPT), Howard University Television (WHUT), Milwaukee Public Television (WMVS), WPT (WPNE), Iowa Public Television/IPTV (KDIN), Lakeland Public Television (KAWE), WDSE - WRPT (WDSE), TPT (KTCA), WNMU-TV Public TV 13 (WNMU), WTTW11 (WTTW), KSMQ (KSMQ), WYCC PBS Chicago (WYCC), Wyoming PBS (KCWC), Idaho Public Television (KAID), KUED Channel 7 (KUED), MontanaPBS (KUSM), NET1 (KUON), Rocky Mountain PBS (KRMA), SDPB Television (KUSD), Colorado Public Television / KBDI 12 (KBDI), KBYU-TV (KBYU)
+ - **pbs**: Public Broadcasting Service (PBS) and member stations: PBS: Public Broadcasting Service, APT - Alabama Public Television (WBIQ), GPB/Georgia Public Broadcasting (WGTV), Mississippi Public Broadcasting (WMPN), Nashville Public Television (WNPT), WFSU-TV (WFSU), WSRE (WSRE), WTCI (WTCI), WPBA/Channel 30 (WPBA), Alaska Public Media (KAKM), Arizona PBS (KAET), KNME-TV/Channel 5 (KNME), Vegas PBS (KLVX), AETN/ARKANSAS ETV NETWORK (KETS), KET (WKLE), WKNO/Channel 10 (WKNO), LPB/LOUISIANA PUBLIC BROADCASTING (WLPB), OETA (KETA), Ozarks Public Television (KOZK), WSIU Public Broadcasting (WSIU), KEET TV (KEET), KIXE/Channel 9 (KIXE), KPBS San Diego (KPBS), KQED (KQED), KVIE Public Television (KVIE), PBS SoCal/KOCE (KOCE), ValleyPBS (KVPT), CONNECTICUT PUBLIC TELEVISION (WEDH), KNPB Channel 5 (KNPB), SOPTV (KSYS), Rocky Mountain PBS (KRMA), KENW-TV3 (KENW), KUED Channel 7 (KUED), Wyoming PBS (KCWC), Colorado Public Television / KBDI 12 (KBDI), KBYU-TV (KBYU), Thirteen/WNET New York (WNET), WGBH/Channel 2 (WGBH), WGBY (WGBY), NJTV Public Media NJ (WNJT), WLIW21 (WLIW), mpt/Maryland Public Television (WMPB), WETA Television and Radio (WETA), WHYY (WHYY), PBS 39 (WLVT), WVPT - Your Source for PBS and More! (WVPT), Howard University Television (WHUT), WEDU PBS (WEDU), WGCU Public Media (WGCU), WPBT2 (WPBT), WUCF TV (WUCF), WUFT/Channel 5 (WUFT), WXEL/Channel 42 (WXEL), WLRN/Channel 17 (WLRN), WUSF Public Broadcasting (WUSF), ETV (WRLK), UNC-TV (WUNC), PBS Hawaii - Oceanic Cable Channel 10 (KHET), Idaho Public Television (KAID), KSPS (KSPS), OPB (KOPB), KWSU/Channel 10 & KTNW/Channel 31 (KWSU), WILL-TV (WILL), Network Knowledge - WSEC/Springfield (WSEC), WTTW11 (WTTW), Iowa Public Television/IPTV (KDIN), Nine Network (KETC), PBS39 Fort Wayne (WFWA), WFYI Indianapolis (WFYI), Milwaukee Public Television (WMVS), WNIN (WNIN), WNIT Public Television (WNIT), WPT (WPNE), WVUT/Channel 22 (WVUT), WEIU/Channel 51 (WEIU), WQPT-TV (WQPT), WYCC PBS Chicago (WYCC), WIPB-TV (WIPB), WTIU (WTIU), CET  (WCET), ThinkTVNetwork (WPTD), WBGU-TV (WBGU), WGVU TV (WGVU), NET1 (KUON), Pioneer Public Television (KWCM), SDPB Television (KUSD), TPT (KTCA), KSMQ (KSMQ), KPTS/Channel 8 (KPTS), KTWU/Channel 11 (KTWU), East Tennessee PBS (WSJK), WCTE-TV (WCTE), WLJT, Channel 11 (WLJT), WOSU TV (WOSU), WOUB/WOUC (WOUB), WVPB (WVPB), WKYU-PBS (WKYU), KERA 13 (KERA), MPBN (WCBB), Mountain Lake PBS (WCFE), NHPTV (WENH), Vermont PBS (WETK), witf (WITF), WQED Multimedia (WQED), WMHT Educational Telecommunications (WMHT), Q-TV (WDCQ), WTVS Detroit Public TV (WTVS), CMU Public Television (WCMU), WKAR-TV (WKAR), WNMU-TV Public TV 13 (WNMU), WDSE - WRPT (WDSE), WGTE TV (WGTE), Lakeland Public Television (KAWE), KMOS-TV - Channels 6.1, 6.2 and 6.3 (KMOS), MontanaPBS (KUSM), KRWG/Channel 22 (KRWG), KACV (KACV), KCOS/Channel 13 (KCOS), WCNY/Channel 24 (WCNY), WNED (WNED), WPBS (WPBS), WSKG Public TV (WSKG), WXXI (WXXI), WPSU (WPSU), WVIA Public Media Studios (WVIA), WTVI (WTVI), Western Reserve PBS (WNEO), WVIZ/PBS ideastream (WVIZ), KCTS 9 (KCTS), Basin PBS (KPBT), KUHT / Channel 8 (KUHT), KLRN (KLRN), KLRU (KLRU), WTJX Channel 12 (WTJX), WCVE PBS (WCVE), KBTC Public Television (KBTC)
  - **Periscope**: Periscope
  - **PhilharmonieDeParis**: Philharmonie de Paris
  - **Phoenix**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 0f5571006..6cd748b48 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.12.09'
+__version__ = '2015.12.10'

From 17b786ae73848a8f26ecc6e95947c3f65edc667f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 10 Dec 2015 22:59:50 +0600
Subject: [PATCH 3317/6219] [downloader/f4m] Fix malformed manifests (Closes
 #7823)

---
 youtube_dl/downloader/f4m.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index 6170cc155..7819a91dc 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -15,6 +15,7 @@ from ..compat import (
 )
 from ..utils import (
     encodeFilename,
+    fix_xml_ampersands,
     sanitize_open,
     struct_pack,
     struct_unpack,
@@ -288,7 +289,10 @@ class F4mFD(FragmentFD):
         self.to_screen('[%s] Downloading f4m manifest' % self.FD_NAME)
         urlh = self.ydl.urlopen(man_url)
         man_url = urlh.geturl()
-        manifest = urlh.read()
+        # Some manifests may be malformed, e.g. prosiebensat1 generated manifests
+        # (see https://github.com/rg3/youtube-dl/issues/6215#issuecomment-121704244
+        # and https://github.com/rg3/youtube-dl/issues/7823)
+        manifest = fix_xml_ampersands(urlh.read()).strip()
 
         doc = compat_etree_fromstring(manifest)
         formats = [(int(f.attrib.get('bitrate', -1)), f)

From 75ed53320e4a11b648dd9ae0bea292f3d1000f00 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 10 Dec 2015 19:08:16 +0100
Subject: [PATCH 3318/6219] [ooyala] improve extraction

---
 youtube_dl/extractor/ooyala.py | 25 +++++++++++++++++--------
 1 file changed, 17 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/ooyala.py b/youtube_dl/extractor/ooyala.py
index 35067e271..8603fd692 100644
--- a/youtube_dl/extractor/ooyala.py
+++ b/youtube_dl/extractor/ooyala.py
@@ -27,6 +27,7 @@ class OoyalaBaseIE(InfoExtractor):
             'duration': float_or_none(metadata.get('duration'), 1000),
         }
 
+        urls = []
         formats = []
         for supported_format in ('mp4', 'm3u8', 'hds', 'rtmp'):
             auth_data = self._download_json(
@@ -38,20 +39,28 @@ class OoyalaBaseIE(InfoExtractor):
             if cur_auth_data['authorized']:
                 for stream in cur_auth_data['streams']:
                     url = base64.b64decode(stream['url']['data'].encode('ascii')).decode('utf-8')
+                    if url in urls:
+                        continue
+                    urls.append(url)
                     delivery_type = stream['delivery_type']
-                    if delivery_type == 'remote_asset':
-                        video_info['url'] = url
-                        return video_info
-                    if delivery_type == 'hls':
-                        formats.extend(self._extract_m3u8_formats(url, embed_code, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
-                    elif delivery_type == 'hds':
-                        formats.extend(self._extract_f4m_formats(url, embed_code, -1, 'hds', fatal=False))
+                    if delivery_type == 'hls' or '.m3u8' in url:
+                        m3u8_formats = self._extract_m3u8_formats(url, embed_code, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False)
+                        if m3u8_formats:
+                            formats.extend(m3u8_formats)
+                    elif delivery_type == 'hds' or '.f4m' in url:
+                        f4m_formats = self._extract_f4m_formats(url, embed_code, f4m_id='hds', fatal=False)
+                        if f4m_formats:
+                            formats.extend(f4m_formats)
+                    elif '.smil' in url:
+                        smil_formats = self._extract_smil_formats(url, embed_code, fatal=False)
+                        if smil_formats:
+                            formats.extend(smil_formats)
                     else:
                         formats.append({
                             'url': url,
                             'ext': stream.get('delivery_type'),
                             'vcodec': stream.get('video_codec'),
-                            'format_id': '%s-%s-%sp' % (stream.get('profile'), delivery_type, stream.get('height')),
+                            'format_id': delivery_type,
                             'width': int_or_none(stream.get('width')),
                             'height': int_or_none(stream.get('height')),
                             'abr': int_or_none(stream.get('audio_bitrate')),

From d50116b8ac2484ce9e8bc6a3d885c5a4a09b4e47 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 10 Dec 2015 22:18:42 +0100
Subject: [PATCH 3319/6219] [vgtv] extract 5 digit length video ids using both
 xstream and vgtv

---
 youtube_dl/extractor/vgtv.py    | 22 +++++++++++++++++-----
 youtube_dl/extractor/xstream.py | 14 +++++++++-----
 2 files changed, 26 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/vgtv.py b/youtube_dl/extractor/vgtv.py
index 1fba37578..347410a78 100644
--- a/youtube_dl/extractor/vgtv.py
+++ b/youtube_dl/extractor/vgtv.py
@@ -4,13 +4,14 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from .xstream import XstreamIE
 from ..utils import (
     ExtractorError,
     float_or_none,
 )
 
 
-class VGTVIE(InfoExtractor):
+class VGTVIE(XstreamIE):
     IE_DESC = 'VGTV, BTTV, FTV, Aftenposten and Aftonbladet'
 
     _HOST_TO_APPNAME = {
@@ -137,6 +138,15 @@ class VGTVIE(InfoExtractor):
             raise ExtractorError(
                 'Video %s is no longer available' % video_id, expected=True)
 
+        info = {
+            'formats': [],
+        }
+        if len(video_id) == 5:
+            if appname == 'bttv':
+                info = self._extract_video_info('btno', video_id)
+            elif appname == 'aptv':
+                info = self._extract_video_info('ap', video_id)
+
         streams = data['streamUrls']
         stream_type = data.get('streamType')
 
@@ -177,9 +187,11 @@ class VGTVIE(InfoExtractor):
                 })
             formats.append(format_info)
 
-        self._sort_formats(formats)
+        info['formats'].extend(formats)
 
-        return {
+        self._sort_formats(info['formats'])
+
+        info.update({
             'id': video_id,
             'title': self._live_title(data['title']) if stream_type == 'live' else data['title'],
             'description': data['description'],
@@ -187,9 +199,9 @@ class VGTVIE(InfoExtractor):
             'timestamp': data['published'],
             'duration': float_or_none(data['duration'], 1000),
             'view_count': data['displays'],
-            'formats': formats,
             'is_live': True if stream_type == 'live' else False,
-        }
+        })
+        return info
 
 
 class BTArticleIE(InfoExtractor):
diff --git a/youtube_dl/extractor/xstream.py b/youtube_dl/extractor/xstream.py
index 71584c291..436f8978b 100644
--- a/youtube_dl/extractor/xstream.py
+++ b/youtube_dl/extractor/xstream.py
@@ -42,11 +42,7 @@ class XstreamIE(InfoExtractor):
         'only_matching': True,
     }]
 
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        partner_id = mobj.group('partner_id')
-        video_id = mobj.group('id')
-
+    def _extract_video_info(self, partner_id, video_id):
         data = self._download_xml(
             'http://frontend.xstream.dk/%s/feed/video/?platform=web&id=%s'
             % (partner_id, video_id),
@@ -97,6 +93,7 @@ class XstreamIE(InfoExtractor):
             formats.append({
                 'url': link.get('href'),
                 'format_id': link.get('rel'),
+                'preference': 2,
             })
 
         thumbnails = [{
@@ -113,3 +110,10 @@ class XstreamIE(InfoExtractor):
             'formats': formats,
             'thumbnails': thumbnails,
         }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        partner_id = mobj.group('partner_id')
+        video_id = mobj.group('id')
+
+        return self._extract_video_info(partner_id, video_id)

From 3da3999612311ff193890072727208892b9e93f0 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 10 Dec 2015 23:04:28 +0100
Subject: [PATCH 3320/6219] [ultimedia] keep direct support for ultimedia
 videos

---
 youtube_dl/extractor/ultimedia.py | 31 +++++++++++++++++++++++++++----
 1 file changed, 27 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/ultimedia.py b/youtube_dl/extractor/ultimedia.py
index 45201332d..60328123c 100644
--- a/youtube_dl/extractor/ultimedia.py
+++ b/youtube_dl/extractor/ultimedia.py
@@ -8,10 +8,29 @@ from ..utils import int_or_none
 
 
 class UltimediaIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?ultimedia\.com/deliver/(?P<type>generic|musique)(?:/[^/]+)*/(?:src|article)/(?P<id>[\d+a-z]+)'
+    _VALID_URL = r'''(?x)
+        https?://(?:www\.)?ultimedia\.com/
+        (?:
+            deliver/
+            (?P<embed_type>
+                generic|
+                musique
+            )
+            (?:/[^/]+)*/
+            (?:
+                src|
+                article
+            )|
+            default/index/video
+            (?P<site_type>
+                generic|
+                music
+            )
+            /id
+        )/(?P<id>[\d+a-z]+)'''
     _TESTS = [{
         # news
-        'url': 'https://www.ultimedia.com/deliver/generic/iframe/mdtk/01601930/zone/1/src/s8uk0r/autoplay/yes/ad/no/width/714/height/435',
+        'url': 'https://www.ultimedia.com/default/index/videogeneric/id/s8uk0r',
         'md5': '276a0e49de58c7e85d32b057837952a2',
         'info_dict': {
             'id': 's8uk0r',
@@ -25,7 +44,7 @@ class UltimediaIE(InfoExtractor):
         },
     }, {
         # music
-        'url': 'https://www.ultimedia.com/deliver/musique/iframe/mdtk/01601930/zone/1/article/xvpfp8/autoplay/yes/ad/no/width/714/height/435',
+        'url': 'https://www.ultimedia.com/default/index/videomusic/id/xvpfp8',
         'md5': '2ea3513813cf230605c7e2ffe7eca61c',
         'info_dict': {
             'id': 'xvpfp8',
@@ -48,7 +67,11 @@ class UltimediaIE(InfoExtractor):
             return mobj.group('url')
 
     def _real_extract(self, url):
-        video_type, video_id = re.match(self._VALID_URL, url).groups()
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        video_type = mobj.group('embed_type') or mobj.group('site_type')
+        if video_type == 'music':
+            video_type = 'musique'
 
         deliver_info = self._download_json(
             'http://www.ultimedia.com/deliver/video?video=%s&topic=%s' % (video_id, video_type),

From e3e166d8cfa2be039b320d0c733b8233e95d3dcf Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 11 Dec 2015 00:08:54 +0100
Subject: [PATCH 3321/6219] [ign] improve extraction and extract uploader_id

---
 youtube_dl/extractor/ign.py | 26 +++++++++++++++++++-------
 1 file changed, 19 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/ign.py b/youtube_dl/extractor/ign.py
index fb2753738..6160f03d1 100644
--- a/youtube_dl/extractor/ign.py
+++ b/youtube_dl/extractor/ign.py
@@ -32,6 +32,7 @@ class IGNIE(InfoExtractor):
                 'description': 'md5:c8946d4260a4d43a00d5ae8ed998870c',
                 'timestamp': 1370440800,
                 'upload_date': '20130605',
+                'uploader_id': 'cberidon@ign.com',
             }
         },
         {
@@ -48,6 +49,7 @@ class IGNIE(InfoExtractor):
                         'description': 'Rockstar drops the mic on this generation of games. Watch our review of the masterly Grand Theft Auto V.',
                         'timestamp': 1379339880,
                         'upload_date': '20130916',
+                        'uploader_id': 'danieljkrupa@gmail.com',
                     },
                 },
                 {
@@ -58,6 +60,7 @@ class IGNIE(InfoExtractor):
                         'description': 'The twisted beauty of GTA 5 in stunning slow motion.',
                         'timestamp': 1386878820,
                         'upload_date': '20131212',
+                        'uploader_id': 'togilvie@ign.com',
                     },
                 },
             ],
@@ -75,6 +78,7 @@ class IGNIE(InfoExtractor):
                 'description': 'Brian and Jared explore Michel Ancel\'s captivating new preview.',
                 'timestamp': 1408047180,
                 'upload_date': '20140814',
+                'uploader_id': 'jamesduggan1990@gmail.com',
             },
         },
         {
@@ -104,8 +108,8 @@ class IGNIE(InfoExtractor):
         webpage = self._download_webpage(url, name_or_id)
         if page_type != 'video':
             multiple_urls = re.findall(
-                '<param name="flashvars"[^>]*value="[^"]*?url=(https?://www\.ign\.com/videos/.*?)["&]',
-                webpage)
+                r'<param name="flashvars"[^>]*value="[^"]*?url=(https?://www\.ign\.com/videos/.*?)["&]',
+                 webpage)
             if multiple_urls:
                 entries = [self.url_result(u, ie='IGN') for u in multiple_urls]
                 return {
@@ -125,10 +129,14 @@ class IGNIE(InfoExtractor):
         formats = []
         m3u8_url = api_data['refs'].get('m3uUrl')
         if m3u8_url:
-            formats.extend(self._extract_m3u8_formats(m3u8_url, video_id))
+            m3u8_formats = self._extract_m3u8_formats(m3u8_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False)
+            if m3u8_formats:
+                formats.extend(m3u8_formats)
         f4m_url = api_data['refs'].get('f4mUrl')
         if f4m_url:
-            formats.extend(self._extract_f4m_formats(f4m_url, video_id))
+            f4m_formats = self._extract_f4m_formats(f4m_url, video_id, f4m_id='hds', fatal=False)
+            if f4m_formats:
+                formats.extend(f4m_formats)
         for asset in api_data['assets']:
             formats.append({
                 'url': asset['url'],
@@ -139,9 +147,9 @@ class IGNIE(InfoExtractor):
             })
         self._sort_formats(formats)
 
-        thumbnails = []
-        for thumbnail in api_data['thumbnails']:
-            thumbnails.append({'url': thumbnail['url']})
+        thumbnails = [{
+            'url': thumbnail['url']
+        } for thumbnail in api_data.get('thumbnails', [])]
 
         metadata = api_data['metadata']
 
@@ -152,6 +160,7 @@ class IGNIE(InfoExtractor):
             'timestamp': parse_iso8601(metadata.get('publishDate')),
             'duration': int_or_none(metadata.get('duration')),
             'display_id': metadata.get('slug') or video_id,
+            'uploader_id': metadata.get('creator'),
             'thumbnails': thumbnails,
             'formats': formats,
         }
@@ -171,6 +180,7 @@ class OneUPIE(IGNIE):
             'description': 'md5:bf0516c5ee32a3217aa703e9b1bc7826',
             'timestamp': 1313099220,
             'upload_date': '20110811',
+            'uploader_id': 'IGN',
         }
     }]
 
@@ -197,6 +207,7 @@ class PCMagIE(IGNIE):
             'description': 'md5:a7071ae64d2f68cc821c729d4ded6bb3',
             'timestamp': 1420571160,
             'upload_date': '20150106',
+            'uploader_id': 'cozzipix@gmail.com',
         }
     },{
         'url': 'http://www.pcmag.com/article2/0,2817,2470156,00.asp',
@@ -208,5 +219,6 @@ class PCMagIE(IGNIE):
             'description': 'md5:53433c45df96d2ea5d0fda18be2ca908',
             'timestamp': 1412953920,
             'upload_date': '20141010',
+            'uploader_id': 'chris_snyder@pcmag.com',
         }
     }]

From 4bf99ade15c2b9dc0ec0ce4e6127d4cd56f87896 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 11 Dec 2015 14:25:01 +0800
Subject: [PATCH 3322/6219] [vevo] Catch the georestriction message (#7802)

---
 youtube_dl/extractor/vevo.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py
index 571289421..61813301a 100644
--- a/youtube_dl/extractor/vevo.py
+++ b/youtube_dl/extractor/vevo.py
@@ -81,6 +81,9 @@ class VevoIE(InfoExtractor):
         if webpage is False:
             self._oauth_token = None
         else:
+            if 'THIS PAGE IS CURRENTLY UNAVAILABLE IN YOUR REGION' in webpage:
+                raise ExtractorError('%s said: This page is currently unavailable in your region.' % self.IE_NAME, expected=True)
+
             self._oauth_token = self._search_regex(
                 r'access_token":\s*"([^"]+)"',
                 webpage, 'access token', fatal=False)

From 4b942883019968c556a6c6b989b346cecc474ac2 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 11 Dec 2015 17:32:29 +0800
Subject: [PATCH 3323/6219] [vevo] Use _match_id

---
 youtube_dl/extractor/vevo.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py
index 61813301a..1d834970c 100644
--- a/youtube_dl/extractor/vevo.py
+++ b/youtube_dl/extractor/vevo.py
@@ -169,8 +169,7 @@ class VevoIE(InfoExtractor):
             preference=0)
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         json_url = 'http://videoplayer.vevo.com/VideoService/AuthenticateVideo?isrc=%s' % video_id
         response = self._download_json(json_url, video_id)

From fdf01663d1fd362d9886dbe2e4136bca23cc1543 Mon Sep 17 00:00:00 2001
From: Celthi <txtlu0@gmail.com>
Date: Fri, 11 Dec 2015 17:48:40 +0800
Subject: [PATCH 3324/6219] able to download first part of the video, but fail
 in the left part

---
 youtube_dl/extractor/youku.py | 57 +++++++++++++++++++++--------------
 1 file changed, 35 insertions(+), 22 deletions(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index d077e03db..155286fc5 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -1,6 +1,7 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import json
 import base64
 
 from .common import InfoExtractor
@@ -84,17 +85,18 @@ class YoukuIE(InfoExtractor):
             return bytes(s)
 
         sid, token = yk_t(
-            b'becaf9be', base64.b64decode(data2['ep'].encode('ascii'))
+            b'becaf9be', base64.b64decode(data2['security']['encrypt_string'].encode('ascii'))
         ).decode('ascii').split('_')
 
         # get oip
-        oip = data2['ip']
+        oip = data2['security']['ip']
 
         # get fileid
         string_ls = list(
             'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ/\:._-1234567890')
         shuffled_string_ls = []
-        seed = data1['seed']
+        seed = data1.get('seed')
+        seed = 0
         N = len(string_ls)
         for ii in range(N):
             seed = (seed * 0xd3 + 0x754f) % 0x10000
@@ -103,15 +105,19 @@ class YoukuIE(InfoExtractor):
             del string_ls[idx]
 
         fileid_dict = {}
-        for format in data1['streamtypes']:
-            streamfileid = [
-                int(i) for i in data1['streamfileids'][format].strip('*').split('*')]
-            fileid = ''.join(
-                [shuffled_string_ls[i] for i in streamfileid])
-            fileid_dict[format] = fileid[:8] + '%s' + fileid[10:]
+        for stream in data1['stream']:
+            format = stream.get('stream_type')
+            #streamfileid = [
+            #    int(i) for i in data1['stream']['streamfileids']#[format].strip('*').split('*')]
+            #fileid = ''.join(
+            #    [shuffled_string_ls[i] for i in streamfileid])
+            fileid = stream['stream_fileid']
+            fileid_dict[format] = fileid
+            #fileid_dict[format] = fileid[:8] + '%s' + fileid[10:]
 
         def get_fileid(format, n):
-            fileid = fileid_dict[format] % hex(int(n))[2:].upper().zfill(2)
+            #fileid = fileid_dict[format] % hex(int(n))[2:].upper().zfill(2)
+            fileid = fileid_dict[format]
             return fileid
 
         # get ep
@@ -126,15 +132,16 @@ class YoukuIE(InfoExtractor):
 
         # generate video_urls
         video_urls_dict = {}
-        for format in data1['streamtypes']:
+        for stream in data1['stream']:
+            format = stream.get('stream_type')
             video_urls = []
-            for dt in data1['segs'][format]:
-                n = str(int(dt['no']))
+            for dt in stream['segs']:
+                n = str(int(dt['size']))
                 param = {
-                    'K': dt['k'],
+                    'K': dt['key'],
                     'hd': self.get_hd(format),
                     'myp': 0,
-                    'ts': dt['seconds'],
+                    'ts': dt['total_milliseconds_video'],
                     'ypp': 0,
                     'ctype': 12,
                     'ev': 1,
@@ -161,7 +168,8 @@ class YoukuIE(InfoExtractor):
             'hd2': '2',
             'hd3': '3',
             '3gp': '0',
-            '3gphd': '1'
+            '3gphd': '1',
+            'flvhd': '0'
         }
         return hd_id_dict[fm]
 
@@ -172,7 +180,8 @@ class YoukuIE(InfoExtractor):
             'hd2': 'flv',
             'hd3': 'flv',
             '3gp': 'flv',
-            '3gphd': 'mp4'
+            '3gphd': 'mp4',
+            'flvhd': 'flv'
         }
         return ext_dict[fm]
 
@@ -180,6 +189,7 @@ class YoukuIE(InfoExtractor):
         _dict = {
             '3gp': 'h6',
             '3gphd': 'h5',
+            'flvhd': 'h4',
             'flv': 'h4',
             'mp4': 'h3',
             'hd2': 'h2',
@@ -204,8 +214,9 @@ class YoukuIE(InfoExtractor):
                 req.add_header('Ytdl-request-proxy', cn_verification_proxy)
 
             raw_data = self._download_json(req, video_id, note=note)
+            jsonDumpIn = json.dumps(raw_data,indent = 1)
 
-            return raw_data['data']['security']
+            return raw_data['data']
 
 
         video_password = self._downloader.params.get('videopassword', None)
@@ -236,7 +247,8 @@ class YoukuIE(InfoExtractor):
                     msg += ': ' + error
                 raise ExtractorError(msg)
 
-        title = data1['title']
+        #title = data1['title']
+        title = data1['video']['title']
 
         # generate video_urls_dict
         video_urls_dict = self.construct_video_urls(data1, data2)
@@ -248,10 +260,11 @@ class YoukuIE(InfoExtractor):
             'formats': [],
             # some formats are not available for all parts, we have to detect
             # which one has all
-        } for i in range(max(len(v) for v in data1['segs'].values()))]
-        for fm in data1['streamtypes']:
+        } for i in range(max(len(v) for v in data1['stream']))]
+        for stream in data1['stream']:
+            fm = stream.get('stream_type')
             video_urls = video_urls_dict[fm]
-            for video_url, seg, entry in zip(video_urls, data1['segs'][fm], entries):
+            for video_url, seg, entry in zip(video_urls, stream['segs'], entries):
                 entry['formats'].append({
                     'url': video_url,
                     'format_id': self.get_format_name(fm),

From ff51983e1563db08734e43a07d5671a517f99ef6 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 11 Dec 2015 18:52:03 +0800
Subject: [PATCH 3325/6219] [vevo] Handle videos without video_info (#7802)

---
 youtube_dl/extractor/vevo.py | 47 ++++++++++++++++++++++++++++--------
 1 file changed, 37 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py
index 1d834970c..38d382581 100644
--- a/youtube_dl/extractor/vevo.py
+++ b/youtube_dl/extractor/vevo.py
@@ -67,6 +67,17 @@ class VevoIE(InfoExtractor):
         'params': {
             'skip_download': 'true',
         }
+    }, {
+        'note': 'No video_info',
+        'url': 'http://www.vevo.com/watch/k-camp-1/Till-I-Die/USUV71503000',
+        'md5': '8b83cc492d72fc9cf74a02acee7dc1b0',
+        'info_dict': {
+            'id': 'USUV71503000',
+            'ext': 'mp4',
+            'title': 'Till I Die - K Camp ft. T.I.',
+            'duration': 193,
+        },
+        'expected_warnings': ['HTTP Error 404'],
     }]
     _SMIL_BASE_URL = 'http://smil.lvl3.vevo.com/'
 
@@ -89,6 +100,9 @@ class VevoIE(InfoExtractor):
                 webpage, 'access token', fatal=False)
 
     def _formats_from_json(self, video_info):
+        if not video_info:
+            return []
+
         last_version = {'version': -1}
         for version in video_info['videoVersions']:
             # These are the HTTP downloads, other types are for different manifests
@@ -171,15 +185,24 @@ class VevoIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
+        webpage = None
+
         json_url = 'http://videoplayer.vevo.com/VideoService/AuthenticateVideo?isrc=%s' % video_id
         response = self._download_json(json_url, video_id)
-        video_info = response['video']
+        video_info = response['video'] or {}
 
-        if not video_info:
+        if not video_info and response.get('statusCode') != 909:
             if 'statusMessage' in response:
                 raise ExtractorError('%s said: %s' % (self.IE_NAME, response['statusMessage']), expected=True)
             raise ExtractorError('Unable to extract videos')
 
+        if not video_info:
+            if url.startswith('vevo:'):
+                raise ExtractorError('Please specify full Vevo URL for downloading', expected=True)
+            webpage = self._download_webpage(url, video_id)
+
+        title = video_info.get('title') or self._og_search_title(webpage)
+
         formats = self._formats_from_json(video_info)
 
         is_explicit = video_info.get('isExplicit')
@@ -195,7 +218,7 @@ class VevoIE(InfoExtractor):
 
         # Download SMIL
         smil_blocks = sorted((
-            f for f in video_info['videoVersions']
+            f for f in video_info.get('videoVersions', [])
             if f['sourceType'] == 13),
             key=lambda f: f['version'])
         smil_url = '%s/Video/V2/VFILE/%s/%sr.smil' % (
@@ -213,17 +236,21 @@ class VevoIE(InfoExtractor):
                 formats.extend(self._formats_from_smil(smil_xml))
 
         self._sort_formats(formats)
-        timestamp_ms = int_or_none(self._search_regex(
+        timestamp = int_or_none(self._search_regex(
             r'/Date\((\d+)\)/',
-            video_info['launchDate'], 'launch date', fatal=False))
+            video_info['launchDate'], 'launch date', fatal=False),
+            scale=1000) if video_info else None
+
+        duration = video_info.get('duration') or int_or_none(
+            self._html_search_meta('video:duration', webpage))
 
         return {
             'id': video_id,
-            'title': video_info['title'],
+            'title': title,
             'formats': formats,
-            'thumbnail': video_info['imageUrl'],
-            'timestamp': timestamp_ms // 1000,
-            'uploader': video_info['mainArtists'][0]['artistName'],
-            'duration': video_info['duration'],
+            'thumbnail': video_info.get('imageUrl'),
+            'timestamp': timestamp,
+            'uploader': video_info['mainArtists'][0]['artistName'] if video_info else None,
+            'duration': duration,
             'age_limit': age_limit,
         }

From ad30dc1e20979842b6b81ee3f6d2364c78acf5f1 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 11 Dec 2015 19:05:51 +0800
Subject: [PATCH 3326/6219] [vevo] Allow calling API without https

Not all proxies allow CONNECT
---
 youtube_dl/extractor/vevo.py | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py
index 38d382581..3e3b25da0 100644
--- a/youtube_dl/extractor/vevo.py
+++ b/youtube_dl/extractor/vevo.py
@@ -3,7 +3,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_etree_fromstring
+from ..compat import (
+    compat_etree_fromstring,
+    compat_urlparse,
+)
 from ..utils import (
     ExtractorError,
     int_or_none,
@@ -162,14 +165,14 @@ class VevoIE(InfoExtractor):
             })
         return formats
 
-    def _download_api_formats(self, video_id):
+    def _download_api_formats(self, video_id, video_url):
         if not self._oauth_token:
             self._downloader.report_warning(
                 'No oauth token available, skipping API HLS download')
             return []
 
-        api_url = 'https://apiv2.vevo.com/video/%s/streams/hls?token=%s' % (
-            video_id, self._oauth_token)
+        api_url = compat_urlparse.urljoin(video_url, '//apiv2.vevo.com/video/%s/streams/hls?token=%s' % (
+            video_id, self._oauth_token))
         api_data = self._download_json(
             api_url, video_id,
             note='Downloading HLS formats',
@@ -214,7 +217,7 @@ class VevoIE(InfoExtractor):
             age_limit = None
 
         # Download via HLS API
-        formats.extend(self._download_api_formats(video_id))
+        formats.extend(self._download_api_formats(video_id, url))
 
         # Download SMIL
         smil_blocks = sorted((

From b6afc225c89e3fc70a588e5822bbb2d86705bd08 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 11 Dec 2015 19:16:51 +0800
Subject: [PATCH 3327/6219] [vevo] Use _download_smil to provide informative
 error messages

---
 youtube_dl/extractor/vevo.py | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py
index 3e3b25da0..02dfd36f4 100644
--- a/youtube_dl/extractor/vevo.py
+++ b/youtube_dl/extractor/vevo.py
@@ -80,7 +80,7 @@ class VevoIE(InfoExtractor):
             'title': 'Till I Die - K Camp ft. T.I.',
             'duration': 193,
         },
-        'expected_warnings': ['HTTP Error 404'],
+        'expected_warnings': ['Unable to download SMIL file'],
     }]
     _SMIL_BASE_URL = 'http://smil.lvl3.vevo.com/'
 
@@ -130,9 +130,8 @@ class VevoIE(InfoExtractor):
             })
         return formats
 
-    def _formats_from_smil(self, smil_xml):
+    def _formats_from_smil(self, smil_doc):
         formats = []
-        smil_doc = compat_etree_fromstring(smil_xml.encode('utf-8'))
         els = smil_doc.findall('.//{http://www.w3.org/2001/SMIL20/Language}video')
         for el in els:
             src = el.attrib['src']
@@ -233,10 +232,9 @@ class VevoIE(InfoExtractor):
             if smil_url_m is not None:
                 smil_url = smil_url_m
         if smil_url:
-            smil_xml = self._download_webpage(
-                smil_url, video_id, 'Downloading SMIL info', fatal=False)
-            if smil_xml:
-                formats.extend(self._formats_from_smil(smil_xml))
+            smil_doc = self._download_smil(smil_url, video_id, fatal=False)
+            if smil_doc:
+                formats.extend(self._formats_from_smil(smil_doc))
 
         self._sort_formats(formats)
         timestamp = int_or_none(self._search_regex(

From 98c3806b15adfcbd7a24fa67c8702c8a1adb0783 Mon Sep 17 00:00:00 2001
From: Celthi <txtlu0@gmail.com>
Date: Fri, 11 Dec 2015 19:18:14 +0800
Subject: [PATCH 3328/6219] fix some not important codesnips

---
 youtube_dl/extractor/youku.py | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index 155286fc5..0e955db8a 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -1,15 +1,11 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import json
 import base64
 
 from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse,
-    compat_cookiejar,
-    compat_cookies,
-    compat_urllib_request,
     compat_ord,
 )
 from ..utils import (
@@ -89,7 +85,7 @@ class YoukuIE(InfoExtractor):
         ).decode('ascii').split('_')
 
         # get oip
-        oip = data2['security']['ip']
+        oip = data1['security']['ip']
 
         # get fileid
         string_ls = list(
@@ -136,7 +132,8 @@ class YoukuIE(InfoExtractor):
             format = stream.get('stream_type')
             video_urls = []
             for dt in stream['segs']:
-                n = str(int(dt['size']))
+                #n = str(int(dt['size']))
+                n = str(stream['segs'].index(dt))
                 param = {
                     'K': dt['key'],
                     'hd': self.get_hd(format),
@@ -177,6 +174,8 @@ class YoukuIE(InfoExtractor):
         ext_dict = {
             'flv': 'flv',
             'mp4': 'mp4',
+            'mp4hd': 'mp4',
+            'mp4hd2': 'mp4',
             'hd2': 'flv',
             'hd3': 'flv',
             '3gp': 'flv',
@@ -214,7 +213,6 @@ class YoukuIE(InfoExtractor):
                 req.add_header('Ytdl-request-proxy', cn_verification_proxy)
 
             raw_data = self._download_json(req, video_id, note=note)
-            jsonDumpIn = json.dumps(raw_data,indent = 1)
 
             return raw_data['data']
 
@@ -260,7 +258,7 @@ class YoukuIE(InfoExtractor):
             'formats': [],
             # some formats are not available for all parts, we have to detect
             # which one has all
-        } for i in range(max(len(v) for v in data1['stream']))]
+        } for i in range(max(len(v.get('segs')) for v in data1['stream']))]
         for stream in data1['stream']:
             fm = stream.get('stream_type')
             video_urls = video_urls_dict[fm]

From 5333842a1dd89320cfc0b15a483dba068e6c6af2 Mon Sep 17 00:00:00 2001
From: Celthi <txtlu0@gmail.com>
Date: Fri, 11 Dec 2015 20:08:14 +0800
Subject: [PATCH 3329/6219] According the blog and you-get fixed the issues
 #7627.

---
 youtube_dl/extractor/youku.py | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index 0e955db8a..430a2bb8d 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -112,8 +112,13 @@ class YoukuIE(InfoExtractor):
             #fileid_dict[format] = fileid[:8] + '%s' + fileid[10:]
 
         def get_fileid(format, n):
+            number = hex(int(str(n), 10))[2:].upper()
+            if len(number) == 1:
+                number = '0' + number
+            streamfileids = fileid_dict[format]
+            fileid = streamfileids[0:8] + number + streamfileids[10:]
             #fileid = fileid_dict[format] % hex(int(n))[2:].upper().zfill(2)
-            fileid = fileid_dict[format]
+            #fileid = fileid_dict[format]
             return fileid
 
         # get ep
@@ -138,7 +143,7 @@ class YoukuIE(InfoExtractor):
                     'K': dt['key'],
                     'hd': self.get_hd(format),
                     'myp': 0,
-                    'ts': dt['total_milliseconds_video'],
+                    #'ts': dt['total_milliseconds_video'],
                     'ypp': 0,
                     'ctype': 12,
                     'ev': 1,
@@ -149,7 +154,7 @@ class YoukuIE(InfoExtractor):
                 video_url = \
                     'http://k.youku.com/player/getFlvPath/' + \
                     'sid/' + sid + \
-                    '_' + str(int(n) + 1).zfill(2) + \
+                    '_00'+ \
                     '/st/' + self.parse_ext_l(format) + \
                     '/fileid/' + get_fileid(format, n) + '?' + \
                     compat_urllib_parse.urlencode(param)

From 4db43567e8833ca0eb89d694260e5640c414ca07 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 11 Dec 2015 20:28:44 +0600
Subject: [PATCH 3330/6219] [downloader/f4m] Decode manifest before fixing

---
 youtube_dl/downloader/f4m.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index 7819a91dc..aaf0c49c8 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -292,7 +292,7 @@ class F4mFD(FragmentFD):
         # Some manifests may be malformed, e.g. prosiebensat1 generated manifests
         # (see https://github.com/rg3/youtube-dl/issues/6215#issuecomment-121704244
         # and https://github.com/rg3/youtube-dl/issues/7823)
-        manifest = fix_xml_ampersands(urlh.read()).strip()
+        manifest = fix_xml_ampersands(urlh.read().decode('utf-8', 'ignore')).strip()
 
         doc = compat_etree_fromstring(manifest)
         formats = [(int(f.attrib.get('bitrate', -1)), f)

From 2559b9d017985b1fc553028c4059b6c58eb78ea7 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 11 Dec 2015 17:31:33 +0100
Subject: [PATCH 3331/6219] [wdr] extract all formats(closes #7788)

---
 youtube_dl/extractor/wdr.py | 58 ++++++++++++++++++++++++++++---------
 1 file changed, 44 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/wdr.py b/youtube_dl/extractor/wdr.py
index b46802306..ef096cbd2 100644
--- a/youtube_dl/extractor/wdr.py
+++ b/youtube_dl/extractor/wdr.py
@@ -10,8 +10,8 @@ from ..compat import (
     compat_urlparse,
 )
 from ..utils import (
-    determine_ext,
     unified_strdate,
+    qualities,
 )
 
 
@@ -33,6 +33,7 @@ class WDRIE(InfoExtractor):
             'params': {
                 'skip_download': True,
             },
+            'skip': 'Page Not Found',
         },
         {
             'url': 'http://www1.wdr.de/themen/av/videomargaspiegelisttot101-videoplayer.html',
@@ -47,6 +48,7 @@ class WDRIE(InfoExtractor):
             'params': {
                 'skip_download': True,
             },
+            'skip': 'Page Not Found',
         },
         {
             'url': 'http://www1.wdr.de/themen/kultur/audioerlebtegeschichtenmargaspiegel100-audioplayer.html',
@@ -71,6 +73,7 @@ class WDRIE(InfoExtractor):
                 'upload_date': '20140717',
                 'is_live': False
             },
+            'skip': 'Page Not Found',
         },
         {
             'url': 'http://www1.wdr.de/mediathek/video/sendungen/quarks_und_co/filterseite-quarks-und-co100.html',
@@ -83,10 +86,10 @@ class WDRIE(InfoExtractor):
             'url': 'http://www1.wdr.de/mediathek/video/livestream/index.html',
             'info_dict': {
                 'id': 'mdb-103364',
-                'title': 're:^WDR Fernsehen [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
+                'title': 're:^WDR Fernsehen Live [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
                 'description': 'md5:ae2ff888510623bf8d4b115f95a9b7c9',
                 'ext': 'flv',
-                'upload_date': '20150212',
+                'upload_date': '20150101',
                 'is_live': True
             },
             'params': {
@@ -150,25 +153,52 @@ class WDRIE(InfoExtractor):
         if upload_date:
             upload_date = unified_strdate(upload_date)
 
+        formats = []
+        preference = qualities(['S', 'M', 'L', 'XL'])
+
         if video_url.endswith('.f4m'):
-            video_url += '?hdcore=3.2.0&plugin=aasp-3.2.0.77.18'
-            ext = 'flv'
+            f4m_formats = self._extract_f4m_formats(video_url + '?hdcore=3.2.0&plugin=aasp-3.2.0.77.18', page_id, f4m_id='hds', fatal=False)
+            if f4m_formats:
+                formats.extend(f4m_formats)
         elif video_url.endswith('.smil'):
-            fmt = self._extract_smil_formats(video_url, page_id)[0]
-            video_url = fmt['url']
-            sep = '&' if '?' in video_url else '?'
-            video_url += sep
-            video_url += 'hdcore=3.3.0&plugin=aasp-3.3.0.99.43'
-            ext = fmt['ext']
+            smil_formats = self._extract_smil_formats(video_url, page_id, False, {
+                'hdcore': '3.3.0',
+                'plugin': 'aasp-3.3.0.99.43',
+            })
+            if smil_formats:
+                formats.extend(smil_formats)
         else:
-            ext = determine_ext(video_url)
+            formats.append({
+                'url': video_url,
+                'http_headers': {
+                    'User-Agent': 'mobile',
+                },
+            })
+
+        m3u8_url = self._search_regex(r'rel="adaptiv"[^>]+href="([^"]+)"', webpage, 'm3u8 url', default=None)
+        if m3u8_url:
+            m3u8_formats = self._extract_m3u8_formats(m3u8_url, page_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False)
+            if m3u8_formats:
+                formats.extend(m3u8_formats)
+
+        direct_urls = re.findall(r'rel="web(S|M|L|XL)"[^>]+href="([^"]+)"', webpage)
+        if direct_urls:
+            for quality, video_url in direct_urls:
+                formats.append({
+                    'url': video_url,
+                    'preference': preference(quality),
+                    'http_headers': {
+                        'User-Agent': 'mobile',
+                    },
+                })
+
+        self._sort_formats(formats)
 
         description = self._html_search_meta('Description', webpage, 'description')
 
         return {
             'id': page_id,
-            'url': video_url,
-            'ext': ext,
+            'formats': formats,
             'title': title,
             'description': description,
             'thumbnail': thumbnail,

From ab4bdc913fafc71217c1196602525e0382c4eea3 Mon Sep 17 00:00:00 2001
From: Muratcan Simsek <mail@muratcansimsek.com>
Date: Mon, 7 Dec 2015 00:15:19 +0200
Subject: [PATCH 3332/6219] [funimation] Add new extractor

Update funimation.py

Update funimation.py

Removed unnecessary lines.

Update funimation.py

Added thumbnail and description.

Filename improvement.

fixed TEST.
---
 youtube_dl/extractor/__init__.py   |  1 +
 youtube_dl/extractor/funimation.py | 77 ++++++++++++++++++++++++++++++
 2 files changed, 78 insertions(+)
 create mode 100644 youtube_dl/extractor/funimation.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 3db5cd6d9..3cd95ba01 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -206,6 +206,7 @@ from .francetv import (
 from .freesound import FreesoundIE
 from .freespeech import FreespeechIE
 from .freevideo import FreeVideoIE
+from .funimation import FunimationIE
 from .funnyordie import FunnyOrDieIE
 from .gameinformer import GameInformerIE
 from .gamekings import GamekingsIE
diff --git a/youtube_dl/extractor/funimation.py b/youtube_dl/extractor/funimation.py
new file mode 100644
index 000000000..773d80fe8
--- /dev/null
+++ b/youtube_dl/extractor/funimation.py
@@ -0,0 +1,77 @@
+# coding: utf-8
+from __future__ import unicode_literals
+from .common import InfoExtractor
+from ..compat import compat_HTTPError
+from ..utils import (
+    encode_dict,
+    sanitized_Request,
+    ExtractorError,
+    urlencode_postdata
+)
+import re
+
+class FunimationIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?funimation\.com/shows/.+[^ ]/videos/official/(?P<id>[^?]+)'
+
+    _TEST = {
+        'url': 'http://www.funimation.com/shows/air/videos/official/breeze',
+        'info_dict': {
+            'id': 'AIRENG0001',
+            'title': 'Air - 1 - Breeze ',
+            'ext': 'mp4',
+            'thumbnail': 'http://www.funimation.com/admin/uploads/default/recap_thumbnails/7555590/home_spotlight/AIR0001.jpg',
+            'description': 'Travelling puppeteer Yukito arrives in a small town where he hopes to earn money through the magic of his puppets. When a young girl named Misuzu lures him to her home with the promise of food, his life changes forever. ',
+        }
+    }
+
+    def _login(self):
+        (username, password) = self._get_login_info()
+        if username is None:
+            return
+        login_url = 'http://www.funimation.com/login'
+        data = urlencode_postdata(encode_dict({
+            'loginForm2': 'loginform',
+            'email_field': username,
+            'password_field': password,
+        }))
+        login_request = sanitized_Request(login_url, data)
+        login_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
+        try:
+            login = self._download_webpage(login_request, login_url)
+        except ExtractorError as e:
+            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
+                raise ExtractorError('Funimation is not available in your region.', expected=True)
+            raise
+        if re.search(r'<meta property="og:url" content="http://www.funimation.com/login"/>', login) is not None:
+                raise ExtractorError('Unable to login, wrong username or password.', expected=True)
+
+    def _real_initialize(self):
+        self._login()
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        try:
+            webpage = self._download_webpage(url, video_id)
+        except ExtractorError as e:
+            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
+                raise ExtractorError('Funimation is not available in your region.', expected=True)
+            raise
+        if re.search(r'"sdUrl":"http', webpage) is None:
+                raise ExtractorError('You are not logged-in or the stream requires subscription.', expected=True)
+
+        m3u8 = self._search_regex(r'".+Url":"(.+?m3u8)"', webpage, 'm3u8') + self._search_regex(r'"authToken":"(.+?)"', webpage, 'm3u8')
+        formats = self._extract_m3u8_formats(m3u8.replace('\\', ''), video_id, ext='mp4', entry_protocol='m3u8_native')
+
+        video_show = self._search_regex(r'"artist":"(.+?)"', webpage, 'video_show')
+        video_track = self._search_regex(r'"videoNumber":"(\d+).0"', webpage, 'video_track')
+        video_title = self._search_regex(r'"title":"({0}.+?)"'.format(video_track), webpage, 'video_title')
+        video_id = self._search_regex(r'"FUNImationID":"(.+?)"', webpage, 'video_id')
+
+        return {
+            'id': video_id,
+            'title': video_show + ' - ' + video_title + ' ',
+            'formats': formats,
+            'thumbnail': self._og_search_thumbnail(webpage),
+            'description': self._og_search_description(webpage)
+        }

From b4c299bad032c51736f52117fe8154751aecd6da Mon Sep 17 00:00:00 2001
From: Sergey M? <dstftw@gmail.com>
Date: Fri, 11 Dec 2015 21:11:45 +0600
Subject: [PATCH 3333/6219] [funimation] PEP 8

---
 youtube_dl/extractor/funimation.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/funimation.py b/youtube_dl/extractor/funimation.py
index 773d80fe8..6f6aaf30f 100644
--- a/youtube_dl/extractor/funimation.py
+++ b/youtube_dl/extractor/funimation.py
@@ -10,6 +10,7 @@ from ..utils import (
 )
 import re
 
+
 class FunimationIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?funimation\.com/shows/.+[^ ]/videos/official/(?P<id>[^?]+)'
 
@@ -43,7 +44,7 @@ class FunimationIE(InfoExtractor):
                 raise ExtractorError('Funimation is not available in your region.', expected=True)
             raise
         if re.search(r'<meta property="og:url" content="http://www.funimation.com/login"/>', login) is not None:
-                raise ExtractorError('Unable to login, wrong username or password.', expected=True)
+            raise ExtractorError('Unable to login, wrong username or password.', expected=True)
 
     def _real_initialize(self):
         self._login()

From 411e5b88c9d87f7deac35130d77f0c7243ff2a9d Mon Sep 17 00:00:00 2001
From: Sergey M? <dstftw@gmail.com>
Date: Fri, 11 Dec 2015 21:13:49 +0600
Subject: [PATCH 3334/6219] [funimation] Fix login message

---
 youtube_dl/extractor/funimation.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/funimation.py b/youtube_dl/extractor/funimation.py
index 6f6aaf30f..8de28029f 100644
--- a/youtube_dl/extractor/funimation.py
+++ b/youtube_dl/extractor/funimation.py
@@ -38,7 +38,8 @@ class FunimationIE(InfoExtractor):
         login_request = sanitized_Request(login_url, data)
         login_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
         try:
-            login = self._download_webpage(login_request, login_url)
+            login = self._download_webpage(
+                login_request, None, 'Logging in as %s' % username)
         except ExtractorError as e:
             if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
                 raise ExtractorError('Funimation is not available in your region.', expected=True)

From 40ca5b04f4845d3ab334d6cc9fb0272fe25df8b3 Mon Sep 17 00:00:00 2001
From: Sergey M? <dstftw@gmail.com>
Date: Fri, 11 Dec 2015 21:33:17 +0600
Subject: [PATCH 3335/6219] [funimation] Remove unnecessary login form field

---
 youtube_dl/extractor/funimation.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/funimation.py b/youtube_dl/extractor/funimation.py
index 8de28029f..74923c3cf 100644
--- a/youtube_dl/extractor/funimation.py
+++ b/youtube_dl/extractor/funimation.py
@@ -31,7 +31,6 @@ class FunimationIE(InfoExtractor):
             return
         login_url = 'http://www.funimation.com/login'
         data = urlencode_postdata(encode_dict({
-            'loginForm2': 'loginform',
             'email_field': username,
             'password_field': password,
         }))

From 59a4ff482a330ddf241ca4606aed66b5d029c055 Mon Sep 17 00:00:00 2001
From: Sergey M? <dstftw@gmail.com>
Date: Fri, 11 Dec 2015 21:34:30 +0600
Subject: [PATCH 3336/6219] [funimation] Real UA is required for login

---
 youtube_dl/extractor/funimation.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/funimation.py b/youtube_dl/extractor/funimation.py
index 74923c3cf..d0759f3b6 100644
--- a/youtube_dl/extractor/funimation.py
+++ b/youtube_dl/extractor/funimation.py
@@ -34,8 +34,10 @@ class FunimationIE(InfoExtractor):
             'email_field': username,
             'password_field': password,
         }))
-        login_request = sanitized_Request(login_url, data)
-        login_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
+        login_request = sanitized_Request(login_url, data, headers={
+            'User-Agent': 'Mozilla/5.0 (Windows NT 5.2; WOW64; rv:42.0) Gecko/20100101 Firefox/42.0',
+            'Content-Type': 'application/x-www-form-urlencoded'
+        })
         try:
             login = self._download_webpage(
                 login_request, None, 'Logging in as %s' % username)

From f542a3d26bfb70d59a0418387e508130e753b82e Mon Sep 17 00:00:00 2001
From: Sergey M? <dstftw@gmail.com>
Date: Fri, 11 Dec 2015 23:00:22 +0600
Subject: [PATCH 3337/6219] [funimation] Improve extraction (Closes #7775)

---
 youtube_dl/extractor/funimation.py | 137 ++++++++++++++++++++++-------
 1 file changed, 105 insertions(+), 32 deletions(-)

diff --git a/youtube_dl/extractor/funimation.py b/youtube_dl/extractor/funimation.py
index d0759f3b6..06a57779a 100644
--- a/youtube_dl/extractor/funimation.py
+++ b/youtube_dl/extractor/funimation.py
@@ -1,18 +1,22 @@
 # coding: utf-8
 from __future__ import unicode_literals
+
+import re
+
 from .common import InfoExtractor
-from ..compat import compat_HTTPError
+from ..compat import compat_urllib_request
 from ..utils import (
+    clean_html,
+    determine_ext,
     encode_dict,
     sanitized_Request,
     ExtractorError,
     urlencode_postdata
 )
-import re
 
 
 class FunimationIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?funimation\.com/shows/.+[^ ]/videos/official/(?P<id>[^?]+)'
+    _VALID_URL = r'https?://(?:www\.)?funimation\.com/shows/[^/]+/videos/official/(?P<id>[^?]+)'
 
     _TEST = {
         'url': 'http://www.funimation.com/shows/air/videos/official/breeze',
@@ -25,26 +29,31 @@ class FunimationIE(InfoExtractor):
         }
     }
 
+    def _download_webpage(self, url_or_request, video_id, note='Downloading webpage'):
+        HEADERS = {
+            'User-Agent': 'Mozilla/5.0 (Windows NT 5.2; WOW64; rv:42.0) Gecko/20100101 Firefox/42.0',
+        }
+        if isinstance(url_or_request, compat_urllib_request.Request):
+            for header, value in HEADERS.items():
+                url_or_request.add_header(header, value)
+        else:
+            url_or_request = sanitized_Request(url_or_request, headers=HEADERS)
+        response = super(FunimationIE, self)._download_webpage(url_or_request, video_id, note)
+        return response
+
     def _login(self):
         (username, password) = self._get_login_info()
         if username is None:
             return
-        login_url = 'http://www.funimation.com/login'
         data = urlencode_postdata(encode_dict({
             'email_field': username,
             'password_field': password,
         }))
-        login_request = sanitized_Request(login_url, data, headers={
-            'User-Agent': 'Mozilla/5.0 (Windows NT 5.2; WOW64; rv:42.0) Gecko/20100101 Firefox/42.0',
+        login_request = sanitized_Request('http://www.funimation.com/login', data, headers={
             'Content-Type': 'application/x-www-form-urlencoded'
         })
-        try:
-            login = self._download_webpage(
-                login_request, None, 'Logging in as %s' % username)
-        except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
-                raise ExtractorError('Funimation is not available in your region.', expected=True)
-            raise
+        login = self._download_webpage(
+            login_request, None, 'Logging in as %s' % username)
         if re.search(r'<meta property="og:url" content="http://www.funimation.com/login"/>', login) is not None:
             raise ExtractorError('Unable to login, wrong username or password.', expected=True)
 
@@ -52,29 +61,93 @@ class FunimationIE(InfoExtractor):
         self._login()
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        try:
-            webpage = self._download_webpage(url, video_id)
-        except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
-                raise ExtractorError('Funimation is not available in your region.', expected=True)
-            raise
-        if re.search(r'"sdUrl":"http', webpage) is None:
-                raise ExtractorError('You are not logged-in or the stream requires subscription.', expected=True)
+        display_id = self._match_id(url)
 
-        m3u8 = self._search_regex(r'".+Url":"(.+?m3u8)"', webpage, 'm3u8') + self._search_regex(r'"authToken":"(.+?)"', webpage, 'm3u8')
-        formats = self._extract_m3u8_formats(m3u8.replace('\\', ''), video_id, ext='mp4', entry_protocol='m3u8_native')
+        webpage = self._download_webpage(url, display_id)
 
-        video_show = self._search_regex(r'"artist":"(.+?)"', webpage, 'video_show')
-        video_track = self._search_regex(r'"videoNumber":"(\d+).0"', webpage, 'video_track')
-        video_title = self._search_regex(r'"title":"({0}.+?)"'.format(video_track), webpage, 'video_title')
-        video_id = self._search_regex(r'"FUNImationID":"(.+?)"', webpage, 'video_id')
+        items = self._parse_json(
+            self._search_regex(
+                r'var\s+playersData\s*=\s*(\[.+?\]);\n',
+                webpage, 'players data'),
+            display_id)[0]['playlist'][0]['items']
+
+        item = next(item for item in items if item.get('itemAK') == display_id)
+
+        ERRORS_MAP = {
+            'ERROR_MATURE_CONTENT_LOGGED_IN': 'matureContentLoggedIn',
+            'ERROR_MATURE_CONTENT_LOGGED_OUT': 'matureContentLoggedOut',
+            'ERROR_SUBSCRIPTION_LOGGED_OUT': 'subscriptionLoggedOut',
+            'ERROR_VIDEO_EXPIRED': 'videoExpired',
+            'ERROR_TERRITORY_UNAVAILABLE': 'territoryUnavailable',
+            'SVODBASIC_SUBSCRIPTION_IN_PLAYER': 'basicSubscription',
+            'SVODNON_SUBSCRIPTION_IN_PLAYER': 'nonSubscription',
+            'ERROR_PLAYER_NOT_RESPONDING': 'playerNotResponding',
+            'ERROR_UNABLE_TO_CONNECT_TO_CDN': 'unableToConnectToCDN',
+            'ERROR_STREAM_NOT_FOUND': 'streamNotFound',
+        }
+
+        error_messages = {}
+        video_error_messages = self._search_regex(
+            r'var\s+videoErrorMessages\s*=\s*({.+?});\n',
+            webpage, 'error messages', default=None)
+        if video_error_messages:
+            error_messages_json = self._parse_json(video_error_messages, display_id, fatal=False)
+            if error_messages_json:
+                for _, error in error_messages_json.items():
+                    type_ = error.get('type')
+                    description = error.get('description')
+                    content = error.get('content')
+                    if type_ == 'text' and description and content:
+                        error_message = ERRORS_MAP.get(description)
+                        if error_message:
+                            error_messages[error_message] = content
+
+        errors = []
+        formats = []
+        for video in item['videoSet']:
+            auth_token = video.get('authToken')
+            if not auth_token:
+                continue
+            funimation_id = video.get('FUNImationID') or video.get('videoId')
+            if not auth_token.startswith('?'):
+                auth_token = '?%s' % auth_token
+            for quality in ('sd', 'hd', 'hd1080'):
+                format_url = video.get('%sUrl' % quality)
+                if not format_url:
+                    continue
+                if not format_url.startswith(('http', '//')):
+                    errors.append(format_url)
+                    continue
+                if determine_ext(format_url) == 'm3u8':
+                    m3u8_formats = self._extract_m3u8_formats(
+                        format_url + auth_token, display_id, 'mp4', entry_protocol='m3u8_native',
+                        m3u8_id=funimation_id or 'hls', fatal=False)
+                    if m3u8_formats:
+                        formats.extend(m3u8_formats)
+                else:
+                    formats.append({
+                        'url': format_url + auth_token,
+                    })
+
+        if not formats and errors:
+            raise ExtractorError(
+                '%s returned error: %s'
+                % (self.IE_NAME, clean_html(error_messages.get(errors[0], errors[0]))),
+                expected=True)
+
+        title = item['title']
+        artist = item.get('artist')
+        if artist:
+            title = '%s - %s' % (artist, title)
+        description = self._og_search_description(webpage) or item.get('description')
+        thumbnail = self._og_search_thumbnail(webpage) or item.get('posterUrl')
+        video_id = item.get('itemId') or display_id
 
         return {
             'id': video_id,
-            'title': video_show + ' - ' + video_title + ' ',
+            'display_id': display_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
             'formats': formats,
-            'thumbnail': self._og_search_thumbnail(webpage),
-            'description': self._og_search_description(webpage)
         }

From d357bbd375c9e30b882dfb8cd2cded7e19a9936c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 11 Dec 2015 23:06:44 +0600
Subject: [PATCH 3338/6219] [funimation] Update test

---
 youtube_dl/extractor/funimation.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/funimation.py b/youtube_dl/extractor/funimation.py
index 06a57779a..e47fe9b73 100644
--- a/youtube_dl/extractor/funimation.py
+++ b/youtube_dl/extractor/funimation.py
@@ -21,12 +21,13 @@ class FunimationIE(InfoExtractor):
     _TEST = {
         'url': 'http://www.funimation.com/shows/air/videos/official/breeze',
         'info_dict': {
-            'id': 'AIRENG0001',
-            'title': 'Air - 1 - Breeze ',
+            'id': '658',
+            'display_id': 'breeze',
             'ext': 'mp4',
-            'thumbnail': 'http://www.funimation.com/admin/uploads/default/recap_thumbnails/7555590/home_spotlight/AIR0001.jpg',
-            'description': 'Travelling puppeteer Yukito arrives in a small town where he hopes to earn money through the magic of his puppets. When a young girl named Misuzu lures him to her home with the promise of food, his life changes forever. ',
-        }
+            'title': 'Air - 1 - Breeze',
+            'description': 'md5:1769f43cd5fc130ace8fd87232207892',
+            'thumbnail': 're:https?://.*\.jpg',
+        },
     }
 
     def _download_webpage(self, url_or_request, video_id, note='Downloading webpage'):

From 9c163950da803b11f50bff4562b38effaa000cd0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 11 Dec 2015 23:20:10 +0600
Subject: [PATCH 3339/6219] [funimation] Improve _VALID_URL

---
 youtube_dl/extractor/funimation.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/funimation.py b/youtube_dl/extractor/funimation.py
index e47fe9b73..4f382ea6f 100644
--- a/youtube_dl/extractor/funimation.py
+++ b/youtube_dl/extractor/funimation.py
@@ -16,7 +16,7 @@ from ..utils import (
 
 
 class FunimationIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?funimation\.com/shows/[^/]+/videos/official/(?P<id>[^?]+)'
+    _VALID_URL = r'https?://(?:www\.)?funimation\.com/shows/[^/]+/videos/official/(?P<id>[^/?#&]+)'
 
     _TEST = {
         'url': 'http://www.funimation.com/shows/air/videos/official/breeze',

From b59623ef434cb7868faa4b5e6e0dd2a56449f369 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 12 Dec 2015 00:38:58 +0600
Subject: [PATCH 3340/6219] [funimation] Use mobile webpage for workaround hulu
 error

---
 youtube_dl/extractor/funimation.py | 151 +++++++++++++++++------------
 1 file changed, 88 insertions(+), 63 deletions(-)

diff --git a/youtube_dl/extractor/funimation.py b/youtube_dl/extractor/funimation.py
index 4f382ea6f..8af2e907c 100644
--- a/youtube_dl/extractor/funimation.py
+++ b/youtube_dl/extractor/funimation.py
@@ -18,7 +18,7 @@ from ..utils import (
 class FunimationIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?funimation\.com/shows/[^/]+/videos/official/(?P<id>[^/?#&]+)'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.funimation.com/shows/air/videos/official/breeze',
         'info_dict': {
             'id': '658',
@@ -28,19 +28,17 @@ class FunimationIE(InfoExtractor):
             'description': 'md5:1769f43cd5fc130ace8fd87232207892',
             'thumbnail': 're:https?://.*\.jpg',
         },
-    }
-
-    def _download_webpage(self, url_or_request, video_id, note='Downloading webpage'):
-        HEADERS = {
-            'User-Agent': 'Mozilla/5.0 (Windows NT 5.2; WOW64; rv:42.0) Gecko/20100101 Firefox/42.0',
-        }
-        if isinstance(url_or_request, compat_urllib_request.Request):
-            for header, value in HEADERS.items():
-                url_or_request.add_header(header, value)
-        else:
-            url_or_request = sanitized_Request(url_or_request, headers=HEADERS)
-        response = super(FunimationIE, self)._download_webpage(url_or_request, video_id, note)
-        return response
+    }, {
+        'url': 'http://www.funimation.com/shows/hacksign/videos/official/role-play',
+        'info_dict': {
+            'id': '31128',
+            'display_id': 'role-play',
+            'ext': 'mp4',
+            'title': '.hack//SIGN - 1 - Role Play',
+            'description': 'md5:b602bdc15eef4c9bbb201bb6e6a4a2dd',
+            'thumbnail': 're:https?://.*\.jpg',
+        },
+    }]
 
     def _login(self):
         (username, password) = self._get_login_info()
@@ -51,6 +49,7 @@ class FunimationIE(InfoExtractor):
             'password_field': password,
         }))
         login_request = sanitized_Request('http://www.funimation.com/login', data, headers={
+            'User-Agent': 'Mozilla/5.0 (Windows NT 5.2; WOW64; rv:42.0) Gecko/20100101 Firefox/42.0',
             'Content-Type': 'application/x-www-form-urlencoded'
         })
         login = self._download_webpage(
@@ -64,15 +63,8 @@ class FunimationIE(InfoExtractor):
     def _real_extract(self, url):
         display_id = self._match_id(url)
 
-        webpage = self._download_webpage(url, display_id)
-
-        items = self._parse_json(
-            self._search_regex(
-                r'var\s+playersData\s*=\s*(\[.+?\]);\n',
-                webpage, 'players data'),
-            display_id)[0]['playlist'][0]['items']
-
-        item = next(item for item in items if item.get('itemAK') == display_id)
+        errors = []
+        formats = []
 
         ERRORS_MAP = {
             'ERROR_MATURE_CONTENT_LOGGED_IN': 'matureContentLoggedIn',
@@ -87,48 +79,79 @@ class FunimationIE(InfoExtractor):
             'ERROR_STREAM_NOT_FOUND': 'streamNotFound',
         }
 
-        error_messages = {}
-        video_error_messages = self._search_regex(
-            r'var\s+videoErrorMessages\s*=\s*({.+?});\n',
-            webpage, 'error messages', default=None)
-        if video_error_messages:
-            error_messages_json = self._parse_json(video_error_messages, display_id, fatal=False)
-            if error_messages_json:
-                for _, error in error_messages_json.items():
-                    type_ = error.get('type')
-                    description = error.get('description')
-                    content = error.get('content')
-                    if type_ == 'text' and description and content:
-                        error_message = ERRORS_MAP.get(description)
-                        if error_message:
-                            error_messages[error_message] = content
+        USER_AGENTS = (
+            # PC UA is served with m3u8 that provides some bonus lower quality formats
+            ('pc', 'Mozilla/5.0 (Windows NT 5.2; WOW64; rv:42.0) Gecko/20100101 Firefox/42.0'),
+            # Mobile UA allows to extract direct links and also does not fail when
+            # PC UA fails with hulu error (e.g.
+            # http://www.funimation.com/shows/hacksign/videos/official/role-play)
+            ('mobile', 'Mozilla/5.0 (Linux; Android 4.4.2; Nexus 4 Build/KOT49H) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.114 Mobile Safari/537.36'),
+        )
 
-        errors = []
-        formats = []
-        for video in item['videoSet']:
-            auth_token = video.get('authToken')
-            if not auth_token:
-                continue
-            funimation_id = video.get('FUNImationID') or video.get('videoId')
-            if not auth_token.startswith('?'):
-                auth_token = '?%s' % auth_token
-            for quality in ('sd', 'hd', 'hd1080'):
-                format_url = video.get('%sUrl' % quality)
-                if not format_url:
+        for kind, user_agent in USER_AGENTS:
+            request = sanitized_Request(url)
+            request.add_header('User-Agent', user_agent)
+            webpage = self._download_webpage(
+                request, display_id, 'Downloading %s webpage' % kind)
+
+            items = self._parse_json(
+                self._search_regex(
+                    r'var\s+playersData\s*=\s*(\[.+?\]);\n',
+                    webpage, 'players data'),
+                display_id)[0]['playlist'][0]['items']
+
+            item = next(item for item in items if item.get('itemAK') == display_id)
+
+            error_messages = {}
+            video_error_messages = self._search_regex(
+                r'var\s+videoErrorMessages\s*=\s*({.+?});\n',
+                webpage, 'error messages', default=None)
+            if video_error_messages:
+                error_messages_json = self._parse_json(video_error_messages, display_id, fatal=False)
+                if error_messages_json:
+                    for _, error in error_messages_json.items():
+                        type_ = error.get('type')
+                        description = error.get('description')
+                        content = error.get('content')
+                        if type_ == 'text' and description and content:
+                            error_message = ERRORS_MAP.get(description)
+                            if error_message:
+                                error_messages[error_message] = content
+
+            for video in item.get('videoSet', []):
+                auth_token = video.get('authToken')
+                if not auth_token:
                     continue
-                if not format_url.startswith(('http', '//')):
-                    errors.append(format_url)
-                    continue
-                if determine_ext(format_url) == 'm3u8':
-                    m3u8_formats = self._extract_m3u8_formats(
-                        format_url + auth_token, display_id, 'mp4', entry_protocol='m3u8_native',
-                        m3u8_id=funimation_id or 'hls', fatal=False)
-                    if m3u8_formats:
-                        formats.extend(m3u8_formats)
-                else:
-                    formats.append({
-                        'url': format_url + auth_token,
-                    })
+                funimation_id = video.get('FUNImationID') or video.get('videoId')
+                preference = 1 if video.get('languageMode') == 'dub' else 0
+                if not auth_token.startswith('?'):
+                    auth_token = '?%s' % auth_token
+                for quality in ('sd', 'hd', 'hd1080'):
+                    format_url = video.get('%sUrl' % quality)
+                    if not format_url:
+                        continue
+                    if not format_url.startswith(('http', '//')):
+                        errors.append(format_url)
+                        continue
+                    if determine_ext(format_url) == 'm3u8':
+                        m3u8_formats = self._extract_m3u8_formats(
+                            format_url + auth_token, display_id, 'mp4', entry_protocol='m3u8_native',
+                            preference=preference, m3u8_id=funimation_id or 'hls', fatal=False)
+                        if m3u8_formats:
+                            formats.extend(m3u8_formats)
+                    else:
+                        f = {
+                            'url': format_url + auth_token,
+                            'format_id': funimation_id,
+                            'preference': preference,
+                        }
+                        mobj = re.search(r'(?P<height>\d+)-(?P<tbr>\d+)[Kk]', format_url)
+                        if mobj:
+                            f.update({
+                                'height': int(mobj.group('height')),
+                                'tbr': int(mobj.group('tbr')),
+                            })
+                        formats.append(f)
 
         if not formats and errors:
             raise ExtractorError(
@@ -136,6 +159,8 @@ class FunimationIE(InfoExtractor):
                 % (self.IE_NAME, clean_html(error_messages.get(errors[0], errors[0]))),
                 expected=True)
 
+        self._sort_formats(formats)
+
         title = item['title']
         artist = item.get('artist')
         if artist:

From b323a3cbff32df7188fc4fbb899dcc1c44e5d064 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 12 Dec 2015 00:39:44 +0600
Subject: [PATCH 3341/6219] [funimation] Remove unused import

---
 youtube_dl/extractor/funimation.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/funimation.py b/youtube_dl/extractor/funimation.py
index 8af2e907c..35328997b 100644
--- a/youtube_dl/extractor/funimation.py
+++ b/youtube_dl/extractor/funimation.py
@@ -4,7 +4,6 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_request
 from ..utils import (
     clean_html,
     determine_ext,

From b091529a3ccfdcc638939b71f53708f944450df1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 12 Dec 2015 00:43:03 +0600
Subject: [PATCH 3342/6219] [funimation] Extend _VALID_URL to match promotional
 videos

---
 youtube_dl/extractor/funimation.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/funimation.py b/youtube_dl/extractor/funimation.py
index 35328997b..0b4279612 100644
--- a/youtube_dl/extractor/funimation.py
+++ b/youtube_dl/extractor/funimation.py
@@ -15,7 +15,7 @@ from ..utils import (
 
 
 class FunimationIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?funimation\.com/shows/[^/]+/videos/official/(?P<id>[^/?#&]+)'
+    _VALID_URL = r'https?://(?:www\.)?funimation\.com/shows/[^/]+/videos/(?:official|promotional)/(?P<id>[^/?#&]+)'
 
     _TESTS = [{
         'url': 'http://www.funimation.com/shows/air/videos/official/breeze',
@@ -37,6 +37,9 @@ class FunimationIE(InfoExtractor):
             'description': 'md5:b602bdc15eef4c9bbb201bb6e6a4a2dd',
             'thumbnail': 're:https?://.*\.jpg',
         },
+    }, {
+        'url': 'http://www.funimation.com/shows/attack-on-titan-junior-high/videos/promotional/broadcast-dub-preview',
+        'only_matching': True,
     }]
 
     def _login(self):

From f208e52a76231b112c6a9d64e2fd32712e7812e0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 12 Dec 2015 00:48:09 +0600
Subject: [PATCH 3343/6219] [funimation] Fix promotional videos extraction

---
 youtube_dl/extractor/funimation.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/funimation.py b/youtube_dl/extractor/funimation.py
index 0b4279612..5322e4e93 100644
--- a/youtube_dl/extractor/funimation.py
+++ b/youtube_dl/extractor/funimation.py
@@ -96,12 +96,13 @@ class FunimationIE(InfoExtractor):
             webpage = self._download_webpage(
                 request, display_id, 'Downloading %s webpage' % kind)
 
-            items = self._parse_json(
+            playlist = self._parse_json(
                 self._search_regex(
                     r'var\s+playersData\s*=\s*(\[.+?\]);\n',
                     webpage, 'players data'),
-                display_id)[0]['playlist'][0]['items']
+                display_id)[0]['playlist']
 
+            items = next(item['items'] for item in playlist if item.get('items'))
             item = next(item for item in items if item.get('itemAK') == display_id)
 
             error_messages = {}

From 0b1bb1ac3ac25dd4d248c71fec16b2a995ecb935 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 12 Dec 2015 00:52:00 +0600
Subject: [PATCH 3344/6219] [funimation] Add test for promotional video

---
 youtube_dl/extractor/funimation.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/funimation.py b/youtube_dl/extractor/funimation.py
index 5322e4e93..35125b631 100644
--- a/youtube_dl/extractor/funimation.py
+++ b/youtube_dl/extractor/funimation.py
@@ -39,7 +39,14 @@ class FunimationIE(InfoExtractor):
         },
     }, {
         'url': 'http://www.funimation.com/shows/attack-on-titan-junior-high/videos/promotional/broadcast-dub-preview',
-        'only_matching': True,
+        'info_dict': {
+            'id': '9635',
+            'display_id': 'broadcast-dub-preview',
+            'ext': 'mp4',
+            'title': 'Attack on Titan: Junior High - Broadcast Dub Preview',
+            'description': 'md5:f8ec49c0aff702a7832cd81b8a44f803',
+            'thumbnail': 're:https?://.*\.(?:jpg|png)',
+        },
     }]
 
     def _login(self):

From f377f44dae78ff6837d9301e7a086dfb1e601df2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 12 Dec 2015 01:02:54 +0600
Subject: [PATCH 3345/6219] [funimation] Improve extraction

---
 youtube_dl/extractor/funimation.py | 22 ++++++++++------------
 1 file changed, 10 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/funimation.py b/youtube_dl/extractor/funimation.py
index 35125b631..8a65343c5 100644
--- a/youtube_dl/extractor/funimation.py
+++ b/youtube_dl/extractor/funimation.py
@@ -8,6 +8,7 @@ from ..utils import (
     clean_html,
     determine_ext,
     encode_dict,
+    int_or_none,
     sanitized_Request,
     ExtractorError,
     urlencode_postdata
@@ -136,7 +137,7 @@ class FunimationIE(InfoExtractor):
                 preference = 1 if video.get('languageMode') == 'dub' else 0
                 if not auth_token.startswith('?'):
                     auth_token = '?%s' % auth_token
-                for quality in ('sd', 'hd', 'hd1080'):
+                for quality, height in (('sd', 480), ('hd', 720), ('hd1080', 1080)):
                     format_url = video.get('%sUrl' % quality)
                     if not format_url:
                         continue
@@ -146,22 +147,19 @@ class FunimationIE(InfoExtractor):
                     if determine_ext(format_url) == 'm3u8':
                         m3u8_formats = self._extract_m3u8_formats(
                             format_url + auth_token, display_id, 'mp4', entry_protocol='m3u8_native',
-                            preference=preference, m3u8_id=funimation_id or 'hls', fatal=False)
+                            preference=preference, m3u8_id='%s-hls' % funimation_id, fatal=False)
                         if m3u8_formats:
                             formats.extend(m3u8_formats)
                     else:
-                        f = {
+                        tbr = int_or_none(self._search_regex(
+                            r'-(\d+)[Kk]', format_url, 'tbr', default=None))
+                        formats.append({
                             'url': format_url + auth_token,
-                            'format_id': funimation_id,
+                            'format_id': '%s-http-%dp' % (funimation_id, height),
+                            'height': height,
+                            'tbr': tbr,
                             'preference': preference,
-                        }
-                        mobj = re.search(r'(?P<height>\d+)-(?P<tbr>\d+)[Kk]', format_url)
-                        if mobj:
-                            f.update({
-                                'height': int(mobj.group('height')),
-                                'tbr': int(mobj.group('tbr')),
-                            })
-                        formats.append(f)
+                        })
 
         if not formats and errors:
             raise ExtractorError(

From d63cfc3f0faa7d52900f2f1959e8ad81bcef2775 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 12 Dec 2015 02:52:20 +0600
Subject: [PATCH 3346/6219] [beeg] API v5 (Closes #7846)

---
 youtube_dl/extractor/beeg.py | 24 +++++++++++++++++++-----
 1 file changed, 19 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/beeg.py b/youtube_dl/extractor/beeg.py
index d151d38c9..c8d921daf 100644
--- a/youtube_dl/extractor/beeg.py
+++ b/youtube_dl/extractor/beeg.py
@@ -34,15 +34,29 @@ class BeegIE(InfoExtractor):
         video_id = self._match_id(url)
 
         video = self._download_json(
-            'http://beeg.com/api/v4/video/%s' % video_id, video_id)
+            'http://beeg.com/api/v5/video/%s' % video_id, video_id)
+
+        def split(o, e):
+            def cut(s, x):
+                n.append(s[:x])
+                return s[x:]
+            n = []
+            r = len(o) % e
+            if r > 0:
+                o = cut(o, r)
+            while len(o) > e:
+                o = cut(o, e)
+            n.append(o)
+            return n
 
         def decrypt_key(key):
-            # Reverse engineered from http://static.beeg.com/cpl/1067.js
-            a = '8RPUUCS35ZWp3ADnKcSmpH71ZusrROo'
+            # Reverse engineered from http://static.beeg.com/cpl/1105.js
+            a = '5ShMcIQlssOd7zChAIOlmeTZDaUxULbJRnywYaiB'
             e = compat_urllib_parse_unquote(key)
-            return ''.join([
-                compat_chr(compat_ord(e[n]) - compat_ord(a[n % len(a)]) % 25)
+            o = ''.join([
+                compat_chr(compat_ord(e[n]) - compat_ord(a[n % len(a)]) % 21)
                 for n in range(len(e))])
+            return ''.join(split(o, 3)[::-1])
 
         def decrypt_url(encrypted_url):
             encrypted_url = self._proto_relative_url(

From bb4b8c57b986ce9aa1f21f437c8e1de98f8cb7bf Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 11 Dec 2015 23:40:12 +0100
Subject: [PATCH 3347/6219] [kaltura] extract more formats

---
 youtube_dl/extractor/ellentv.py |  6 +++---
 youtube_dl/extractor/kaltura.py | 23 +++++++++++++++--------
 2 files changed, 18 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/ellentv.py b/youtube_dl/extractor/ellentv.py
index 02c6a4615..476cce2d0 100644
--- a/youtube_dl/extractor/ellentv.py
+++ b/youtube_dl/extractor/ellentv.py
@@ -13,12 +13,12 @@ class EllenTVIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?(?:ellentv|ellentube)\.com/videos/(?P<id>[a-z0-9_-]+)'
     _TEST = {
         'url': 'http://www.ellentv.com/videos/0-ipq1gsai/',
-        'md5': '8e3c576bf2e9bfff4d76565f56f94c9c',
+        'md5': '4294cf98bc165f218aaa0b89e0fd8042',
         'info_dict': {
             'id': '0_ipq1gsai',
-            'ext': 'mp4',
+            'ext': 'mov',
             'title': 'Fast Fingers of Fate',
-            'description': 'md5:587e79fbbd0d73b148bc596d99ce48e6',
+            'description': 'md5:3539013ddcbfa64b2a6d1b38d910868a',
             'timestamp': 1428035648,
             'upload_date': '20150403',
             'uploader_id': 'batchUser',
diff --git a/youtube_dl/extractor/kaltura.py b/youtube_dl/extractor/kaltura.py
index 583b1a5ad..353e960b0 100644
--- a/youtube_dl/extractor/kaltura.py
+++ b/youtube_dl/extractor/kaltura.py
@@ -45,7 +45,7 @@ class KalturaIE(InfoExtractor):
             'info_dict': {
                 'id': '1_1jc2y3e4',
                 'ext': 'mp4',
-                'title': 'Track 4',
+                'title': 'Straight from the Heart',
                 'upload_date': '20131219',
                 'uploader_id': 'mlundberg@wolfgangsvault.com',
                 'description': 'The Allman Brothers Band, 12/16/1981',
@@ -115,12 +115,9 @@ class KalturaIE(InfoExtractor):
                 'version': '-1',
             },
             {
-                'action': 'getContextData',
-                'contextDataParams:objectType': 'KalturaEntryContextDataParams',
-                'contextDataParams:referrer': 'http://www.kaltura.com/',
-                'contextDataParams:streamerType': 'http',
+                'action': 'getbyentryid',
                 'entryId': video_id,
-                'service': 'baseentry',
+                'service': 'flavorAsset',
             },
         ]
         return self._kaltura_api_call(
@@ -133,7 +130,7 @@ class KalturaIE(InfoExtractor):
         partner_id = mobj.group('partner_id_s') or mobj.group('partner_id') or mobj.group('partner_id_html5')
         entry_id = mobj.group('id_s') or mobj.group('id') or mobj.group('id_html5')
 
-        info, source_data = self._get_video_info(entry_id, partner_id)
+        info, flavor_assets = self._get_video_info(entry_id, partner_id)
 
         source_url = smuggled_data.get('source_url')
         if source_url:
@@ -144,7 +141,10 @@ class KalturaIE(InfoExtractor):
             referrer = None
 
         formats = []
-        for f in source_data['flavorAssets']:
+        for f in flavor_assets:
+            # Continue if asset is not ready
+            if f['status'] != 2:
+                continue
             video_url = '%s/flavorId/%s' % (info['dataUrl'], f['id'])
             if referrer:
                 video_url += '?referrer=%s' % referrer
@@ -160,6 +160,13 @@ class KalturaIE(InfoExtractor):
                 'width': int_or_none(f.get('width')),
                 'url': video_url,
             })
+
+        m3u8_formats = self._extract_m3u8_formats(
+            info['dataUrl'].replace('format/url', 'format/applehttp'),
+            entry_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False)
+        if m3u8_formats:
+            formats.extend(m3u8_formats)
+
         self._check_formats(formats, entry_id)
         self._sort_formats(formats)
 

From 8696a7fd1396e051a3762ff241d1b4e157b7b08a Mon Sep 17 00:00:00 2001
From: Celthi <txtlu0@gmail.com>
Date: Sat, 12 Dec 2015 10:44:21 +0800
Subject: [PATCH 3348/6219] fix the keyerror(mp4hd), todo support download the
 video protected by password

---
 youtube_dl/extractor/youku.py | 35 +++++++++++++----------------------
 1 file changed, 13 insertions(+), 22 deletions(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index 430a2bb8d..9fd0e1e23 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -2,6 +2,7 @@
 from __future__ import unicode_literals
 
 import base64
+import json
 
 from .common import InfoExtractor
 from ..compat import (
@@ -90,26 +91,12 @@ class YoukuIE(InfoExtractor):
         # get fileid
         string_ls = list(
             'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ/\:._-1234567890')
-        shuffled_string_ls = []
-        seed = data1.get('seed')
-        seed = 0
-        N = len(string_ls)
-        for ii in range(N):
-            seed = (seed * 0xd3 + 0x754f) % 0x10000
-            idx = seed * len(string_ls) // 0x10000
-            shuffled_string_ls.append(string_ls[idx])
-            del string_ls[idx]
 
         fileid_dict = {}
         for stream in data1['stream']:
             format = stream.get('stream_type')
-            #streamfileid = [
-            #    int(i) for i in data1['stream']['streamfileids']#[format].strip('*').split('*')]
-            #fileid = ''.join(
-            #    [shuffled_string_ls[i] for i in streamfileid])
             fileid = stream['stream_fileid']
             fileid_dict[format] = fileid
-            #fileid_dict[format] = fileid[:8] + '%s' + fileid[10:]
 
         def get_fileid(format, n):
             number = hex(int(str(n), 10))[2:].upper()
@@ -117,8 +104,6 @@ class YoukuIE(InfoExtractor):
                 number = '0' + number
             streamfileids = fileid_dict[format]
             fileid = streamfileids[0:8] + number + streamfileids[10:]
-            #fileid = fileid_dict[format] % hex(int(n))[2:].upper().zfill(2)
-            #fileid = fileid_dict[format]
             return fileid
 
         # get ep
@@ -171,7 +156,9 @@ class YoukuIE(InfoExtractor):
             'hd3': '3',
             '3gp': '0',
             '3gphd': '1',
-            'flvhd': '0'
+            'flvhd': '0',
+            'mp4hd': '1',
+            'mp4hd2': '1'
         }
         return hd_id_dict[fm]
 
@@ -180,7 +167,8 @@ class YoukuIE(InfoExtractor):
             'flv': 'flv',
             'mp4': 'mp4',
             'mp4hd': 'mp4',
-            'mp4hd2': 'mp4',
+            'mp4hd2': 'flv',
+            'mp4hd3': 'flv',
             'hd2': 'flv',
             'hd3': 'flv',
             '3gp': 'flv',
@@ -197,7 +185,10 @@ class YoukuIE(InfoExtractor):
             'flv': 'h4',
             'mp4': 'h3',
             'hd2': 'h2',
-            'hd3': 'h1'
+            'hd3': 'h1',
+            'mp4hd': 'h3',
+            'mp4hd3': 'h4',
+            'mp4hd2': 'h4'
         }
         return _dict[fm]
 
@@ -218,6 +209,7 @@ class YoukuIE(InfoExtractor):
                 req.add_header('Ytdl-request-proxy', cn_verification_proxy)
 
             raw_data = self._download_json(req, video_id, note=note)
+            js = json.dumps(raw_data)
 
             return raw_data['data']
 
@@ -225,7 +217,6 @@ class YoukuIE(InfoExtractor):
         video_password = self._downloader.params.get('videopassword', None)
 
         # request basic data
-        #basic_data_url = 'http://v.youku.com/player/getPlayList/VideoIDS/%s' % video_id
         basic_data_url = "http://play.youku.com/play/get.json?vid=%s&ct=12" % video_id
         if video_password:
             basic_data_url += '?password=%s' % video_password
@@ -234,7 +225,6 @@ class YoukuIE(InfoExtractor):
             basic_data_url,
             'Downloading JSON metadata 1')
         data2 = retrieve_data(
-            #'http://v.youku.com/player/getPlayList/VideoIDS/%s/Pf/4/ctype/12/ev/1' % video_id,
             "http://play.youku.com/play/get.json?vid=%s&ct=12" % video_id,
             'Downloading JSON metadata 2')
 
@@ -250,9 +240,10 @@ class YoukuIE(InfoExtractor):
                     msg += ': ' + error
                 raise ExtractorError(msg)
 
-        #title = data1['title']
+        #get video title
         title = data1['video']['title']
 
+
         # generate video_urls_dict
         video_urls_dict = self.construct_video_urls(data1, data2)
 

From 14c17cafa18292112e5ea09e23f8598892ab3a0b Mon Sep 17 00:00:00 2001
From: Celthi <txtlu0@gmail.com>
Date: Sat, 12 Dec 2015 11:21:44 +0800
Subject: [PATCH 3349/6219] add support to video protected by password

---
 youtube_dl/extractor/youku.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index 9fd0e1e23..f1bb482c3 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -219,25 +219,25 @@ class YoukuIE(InfoExtractor):
         # request basic data
         basic_data_url = "http://play.youku.com/play/get.json?vid=%s&ct=12" % video_id
         if video_password:
-            basic_data_url += '?password=%s' % video_password
+            basic_data_url += '&pwd=%s' % video_password
 
         data1 = retrieve_data(
             basic_data_url,
             'Downloading JSON metadata 1')
         data2 = retrieve_data(
-            "http://play.youku.com/play/get.json?vid=%s&ct=12" % video_id,
+            basic_data_url,
             'Downloading JSON metadata 2')
 
-        error_code = data1.get('error_code')
-        if error_code:
-            error = data1.get('error')
-            if error is not None and '因版权原因无法观看此视频' in error:
+        error = data1.get('error')
+        if error:
+            error_note = error.get('note')
+            if error_note is not None and '因版权原因无法观看此视频' in error_note:
                 raise ExtractorError(
                     'Youku said: Sorry, this video is available in China only', expected=True)
             else:
-                msg = 'Youku server reported error %i' % error_code
+                msg = 'Youku server reported error %i' % error.get('code')
                 if error is not None:
-                    msg += ': ' + error
+                    msg += ': ' + error_note
                 raise ExtractorError(msg)
 
         #get video title

From 7e37c394858ac215b02d1c4ea425d20a17bf4655 Mon Sep 17 00:00:00 2001
From: Celthi <txtlu0@gmail.com>
Date: Sat, 12 Dec 2015 11:26:15 +0800
Subject: [PATCH 3350/6219] merge data1 and data2

---
 youtube_dl/extractor/youku.py | 25 +++++++++++--------------
 1 file changed, 11 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index f1bb482c3..5110efcb4 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -64,7 +64,7 @@ class YoukuIE(InfoExtractor):
         },
     }]
 
-    def construct_video_urls(self, data1, data2):
+    def construct_video_urls(self, data):
         # get sid, token
         def yk_t(s1, s2):
             ls = list(range(256))
@@ -82,18 +82,18 @@ class YoukuIE(InfoExtractor):
             return bytes(s)
 
         sid, token = yk_t(
-            b'becaf9be', base64.b64decode(data2['security']['encrypt_string'].encode('ascii'))
+            b'becaf9be', base64.b64decode(data['security']['encrypt_string'].encode('ascii'))
         ).decode('ascii').split('_')
 
         # get oip
-        oip = data1['security']['ip']
+        oip = data['security']['ip']
 
         # get fileid
         string_ls = list(
             'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ/\:._-1234567890')
 
         fileid_dict = {}
-        for stream in data1['stream']:
+        for stream in data['stream']:
             format = stream.get('stream_type')
             fileid = stream['stream_fileid']
             fileid_dict[format] = fileid
@@ -118,7 +118,7 @@ class YoukuIE(InfoExtractor):
 
         # generate video_urls
         video_urls_dict = {}
-        for stream in data1['stream']:
+        for stream in data['stream']:
             format = stream.get('stream_type')
             video_urls = []
             for dt in stream['segs']:
@@ -221,14 +221,11 @@ class YoukuIE(InfoExtractor):
         if video_password:
             basic_data_url += '&pwd=%s' % video_password
 
-        data1 = retrieve_data(
+        data = retrieve_data(
             basic_data_url,
             'Downloading JSON metadata 1')
-        data2 = retrieve_data(
-            basic_data_url,
-            'Downloading JSON metadata 2')
 
-        error = data1.get('error')
+        error = data.get('error')
         if error:
             error_note = error.get('note')
             if error_note is not None and '因版权原因无法观看此视频' in error_note:
@@ -241,11 +238,11 @@ class YoukuIE(InfoExtractor):
                 raise ExtractorError(msg)
 
         #get video title
-        title = data1['video']['title']
+        title = data['video']['title']
 
 
         # generate video_urls_dict
-        video_urls_dict = self.construct_video_urls(data1, data2)
+        video_urls_dict = self.construct_video_urls(data)
 
         # construct info
         entries = [{
@@ -254,8 +251,8 @@ class YoukuIE(InfoExtractor):
             'formats': [],
             # some formats are not available for all parts, we have to detect
             # which one has all
-        } for i in range(max(len(v.get('segs')) for v in data1['stream']))]
-        for stream in data1['stream']:
+        } for i in range(max(len(v.get('segs')) for v in data['stream']))]
+        for stream in data['stream']:
             fm = stream.get('stream_type')
             video_urls = video_urls_dict[fm]
             for video_url, seg, entry in zip(video_urls, stream['segs'], entries):

From 8773f3158fb45fcadadc0b5e8e9dfd5c9915fa39 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 11 Dec 2015 23:59:25 +0800
Subject: [PATCH 3351/6219] [safari] Use postdata_urlencode (#7465)

---
 youtube_dl/extractor/safari.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/safari.py b/youtube_dl/extractor/safari.py
index 919704261..7de7b7273 100644
--- a/youtube_dl/extractor/safari.py
+++ b/youtube_dl/extractor/safari.py
@@ -6,12 +6,12 @@ import re
 from .common import InfoExtractor
 from .brightcove import BrightcoveLegacyIE
 
-from ..compat import compat_urllib_parse
 from ..utils import (
     ExtractorError,
     sanitized_Request,
     smuggle_url,
     std_headers,
+    urlencode_postdata,
 )
 
 
@@ -57,7 +57,7 @@ class SafariBaseIE(InfoExtractor):
         }
 
         request = sanitized_Request(
-            self._LOGIN_URL, compat_urllib_parse.urlencode(login_form), headers=headers)
+            self._LOGIN_URL, urlencode_postdata(login_form), headers=headers)
         login_page = self._download_webpage(
             request, None, 'Logging in as %s' % username)
 

From f133fd326bf691db389acd8871b5fc7c6d4bc978 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 12 Dec 2015 14:41:53 +0800
Subject: [PATCH 3352/6219] [youku] Cleanup and PEP8

---
 youtube_dl/extractor/youku.py | 24 ++++++------------------
 1 file changed, 6 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index 5110efcb4..bec29196a 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -2,7 +2,6 @@
 from __future__ import unicode_literals
 
 import base64
-import json
 
 from .common import InfoExtractor
 from ..compat import (
@@ -88,10 +87,6 @@ class YoukuIE(InfoExtractor):
         # get oip
         oip = data['security']['ip']
 
-        # get fileid
-        string_ls = list(
-            'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ/\:._-1234567890')
-
         fileid_dict = {}
         for stream in data['stream']:
             format = stream.get('stream_type')
@@ -122,13 +117,11 @@ class YoukuIE(InfoExtractor):
             format = stream.get('stream_type')
             video_urls = []
             for dt in stream['segs']:
-                #n = str(int(dt['size']))
                 n = str(stream['segs'].index(dt))
                 param = {
                     'K': dt['key'],
                     'hd': self.get_hd(format),
                     'myp': 0,
-                    #'ts': dt['total_milliseconds_video'],
                     'ypp': 0,
                     'ctype': 12,
                     'ev': 1,
@@ -139,7 +132,7 @@ class YoukuIE(InfoExtractor):
                 video_url = \
                     'http://k.youku.com/player/getFlvPath/' + \
                     'sid/' + sid + \
-                    '_00'+ \
+                    '_00' + \
                     '/st/' + self.parse_ext_l(format) + \
                     '/fileid/' + get_fileid(format, n) + '?' + \
                     compat_urllib_parse.urlencode(param)
@@ -196,24 +189,20 @@ class YoukuIE(InfoExtractor):
         video_id = self._match_id(url)
 
         def retrieve_data(req_url, note):
-            
-
             headers = {
-                    'Referer': req_url,
-                }
-            self._set_cookie('youku.com','xreferrer','http://www.youku.com')
-            req = sanitized_Request(req_url,headers=headers)
+                'Referer': req_url,
+            }
+            self._set_cookie('youku.com', 'xreferrer', 'http://www.youku.com')
+            req = sanitized_Request(req_url, headers=headers)
 
             cn_verification_proxy = self._downloader.params.get('cn_verification_proxy')
             if cn_verification_proxy:
                 req.add_header('Ytdl-request-proxy', cn_verification_proxy)
 
             raw_data = self._download_json(req, video_id, note=note)
-            js = json.dumps(raw_data)
 
             return raw_data['data']
 
-
         video_password = self._downloader.params.get('videopassword', None)
 
         # request basic data
@@ -237,10 +226,9 @@ class YoukuIE(InfoExtractor):
                     msg += ': ' + error_note
                 raise ExtractorError(msg)
 
-        #get video title
+        # get video title
         title = data['video']['title']
 
-
         # generate video_urls_dict
         video_urls_dict = self.construct_video_urls(data)
 

From c683454e7e35a67a901526071136f8932adab3ea Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 12 Dec 2015 14:48:46 +0800
Subject: [PATCH 3353/6219] [youku] MD5 is unstable

---
 youtube_dl/extractor/youku.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index bec29196a..2a65ae627 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -25,8 +25,8 @@ class YoukuIE(InfoExtractor):
     '''
 
     _TESTS = [{
+        # MD5 is unstable
         'url': 'http://v.youku.com/v_show/id_XMTc1ODE5Njcy.html',
-        'md5': '5f3af4192eabacc4501508d54a8cabd7',
         'info_dict': {
             'id': 'XMTc1ODE5Njcy_part1',
             'title': '★Smile﹗♡ Git Fresh -Booty Music舞蹈.',

From 4d77550cf0d968344a6762c90496ffbe7528cbe7 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 12 Dec 2015 14:57:14 +0800
Subject: [PATCH 3354/6219] [youku] Fix tests

---
 youtube_dl/extractor/youku.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index 2a65ae627..b6129ecfa 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -42,6 +42,7 @@ class YoukuIE(InfoExtractor):
             'title': '武媚娘传奇 85',
         },
         'playlist_count': 11,
+        'skip': 'Available in China only',
     }, {
         'url': 'http://v.youku.com/v_show/id_XMTI1OTczNDM5Mg==.html',
         'info_dict': {
@@ -49,7 +50,6 @@ class YoukuIE(InfoExtractor):
             'title': '花千骨 04',
         },
         'playlist_count': 13,
-        'skip': 'Available in China only',
     }, {
         'url': 'http://v.youku.com/v_show/id_XNjA1NzA2Njgw.html',
         'note': 'Video protected with password',

From ade23409717b45c14fda8446389f6dd8c2ca3bae Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 12 Dec 2015 15:19:14 +0800
Subject: [PATCH 3355/6219] [youku] Simplify

---
 youtube_dl/extractor/youku.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index b6129ecfa..1b1878f37 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -210,9 +210,7 @@ class YoukuIE(InfoExtractor):
         if video_password:
             basic_data_url += '&pwd=%s' % video_password
 
-        data = retrieve_data(
-            basic_data_url,
-            'Downloading JSON metadata 1')
+        data = retrieve_data(basic_data_url, 'Downloading JSON metadata')
 
         error = data.get('error')
         if error:

From dbb7d7e26cfba1cb8c9f78164757f594a2a50fd1 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 12 Dec 2015 15:24:58 +0800
Subject: [PATCH 3356/6219] [youku] Reorder format items

---
 youtube_dl/extractor/youku.py | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index 1b1878f37..b6cc4bb4c 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -143,30 +143,30 @@ class YoukuIE(InfoExtractor):
 
     def get_hd(self, fm):
         hd_id_dict = {
-            'flv': '0',
-            'mp4': '1',
-            'hd2': '2',
-            'hd3': '3',
             '3gp': '0',
             '3gphd': '1',
+            'flv': '0',
             'flvhd': '0',
+            'mp4': '1',
             'mp4hd': '1',
-            'mp4hd2': '1'
+            'mp4hd2': '1',
+            'hd2': '2',
+            'hd3': '3',
         }
         return hd_id_dict[fm]
 
     def parse_ext_l(self, fm):
         ext_dict = {
+            '3gp': 'flv',
+            '3gphd': 'mp4',
             'flv': 'flv',
+            'flvhd': 'flv',
             'mp4': 'mp4',
             'mp4hd': 'mp4',
             'mp4hd2': 'flv',
             'mp4hd3': 'flv',
             'hd2': 'flv',
             'hd3': 'flv',
-            '3gp': 'flv',
-            '3gphd': 'mp4',
-            'flvhd': 'flv'
         }
         return ext_dict[fm]
 
@@ -174,14 +174,14 @@ class YoukuIE(InfoExtractor):
         _dict = {
             '3gp': 'h6',
             '3gphd': 'h5',
-            'flvhd': 'h4',
             'flv': 'h4',
+            'flvhd': 'h4',
             'mp4': 'h3',
+            'mp4hd': 'h3',
+            'mp4hd2': 'h4',
+            'mp4hd3': 'h4',
             'hd2': 'h2',
             'hd3': 'h1',
-            'mp4hd': 'h3',
-            'mp4hd3': 'h4',
-            'mp4hd2': 'h4'
         }
         return _dict[fm]
 

From 174964a7bc4b8d759a4019e366520c9121fcbf15 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 12 Dec 2015 15:34:40 +0800
Subject: [PATCH 3357/6219] Credit @Celthi for fixing Youku extractor

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index cdb56de3b..e0fb098e6 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -147,3 +147,4 @@ Qijiang Fan
 Rémy Léone
 Marco Ferragina
 reiv
+Evan Lu

From deb1e8d20e8680d8e217e2d376e05836541a46ca Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 12 Dec 2015 15:49:19 +0800
Subject: [PATCH 3358/6219] [youku] Put the missing item to get_hd

---
 youtube_dl/extractor/youku.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index b6cc4bb4c..d33caa79e 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -150,6 +150,7 @@ class YoukuIE(InfoExtractor):
             'mp4': '1',
             'mp4hd': '1',
             'mp4hd2': '1',
+            'mp4hd3': '1',
             'hd2': '2',
             'hd3': '3',
         }

From c03943f39440d80e82239c82958b5d2f48f25410 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 12 Dec 2015 15:19:23 +0600
Subject: [PATCH 3359/6219] Credit @Slyneth for funimation (#7775)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index e0fb098e6..ce350e96c 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -147,4 +147,5 @@ Qijiang Fan
 Rémy Léone
 Marco Ferragina
 reiv
+Muratcan Simsek
 Evan Lu

From 0014ffa829ad0d39d7231f664ea4436d2edd5c6a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 13 Dec 2015 07:17:42 +0600
Subject: [PATCH 3360/6219] [funimation] Improve login

---
 youtube_dl/extractor/funimation.py | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/funimation.py b/youtube_dl/extractor/funimation.py
index 8a65343c5..d1a95d87f 100644
--- a/youtube_dl/extractor/funimation.py
+++ b/youtube_dl/extractor/funimation.py
@@ -1,8 +1,6 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..utils import (
     clean_html,
@@ -18,6 +16,8 @@ from ..utils import (
 class FunimationIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?funimation\.com/shows/[^/]+/videos/(?:official|promotional)/(?P<id>[^/?#&]+)'
 
+    _NETRC_MACHINE = 'funimation'
+
     _TESTS = [{
         'url': 'http://www.funimation.com/shows/air/videos/official/breeze',
         'info_dict': {
@@ -62,10 +62,16 @@ class FunimationIE(InfoExtractor):
             'User-Agent': 'Mozilla/5.0 (Windows NT 5.2; WOW64; rv:42.0) Gecko/20100101 Firefox/42.0',
             'Content-Type': 'application/x-www-form-urlencoded'
         })
-        login = self._download_webpage(
+        login_page = self._download_webpage(
             login_request, None, 'Logging in as %s' % username)
-        if re.search(r'<meta property="og:url" content="http://www.funimation.com/login"/>', login) is not None:
-            raise ExtractorError('Unable to login, wrong username or password.', expected=True)
+        if any(p in login_page for p in ('funimation.com/logout', '>Log Out<')):
+            return
+        error = self._html_search_regex(
+            r'(?s)<div[^>]+id=["\']errorMessages["\'][^>]*>(.+?)</div>',
+            login_page, 'error messages', default=None)
+        if error:
+            raise ExtractorError('Unable to login: %s' % error, expected=True)
+        raise ExtractorError('Unable to log in')
 
     def _real_initialize(self):
         self._login()

From 47c7f3d99504325c43651b781376ea470e7e28bd Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 13 Dec 2015 15:33:11 +0800
Subject: [PATCH 3361/6219] [novamov] Fix filekey extraction (closes #7764)

---
 youtube_dl/extractor/novamov.py | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/novamov.py b/youtube_dl/extractor/novamov.py
index 837c91559..0c4ff90f2 100644
--- a/youtube_dl/extractor/novamov.py
+++ b/youtube_dl/extractor/novamov.py
@@ -23,9 +23,10 @@ class NovaMovIE(InfoExtractor):
     _HOST = 'www.novamov.com'
 
     _FILE_DELETED_REGEX = r'This file no longer exists on our servers!</h2>'
-    _FILEKEY_REGEX = r'flashvars\.filekey="(?P<filekey>[^"]+)";'
+    _FILEKEY_REGEX = r'flashvars\.filekey=(?P<filekey>"?[^"]+"?);'
     _TITLE_REGEX = r'(?s)<div class="v_tab blockborder rounded5" id="v_tab1">\s*<h3>([^<]+)</h3>'
     _DESCRIPTION_REGEX = r'(?s)<div class="v_tab blockborder rounded5" id="v_tab1">\s*<h3>[^<]+</h3><p>([^<]+)</p>'
+    _URL_TEMPLATE = 'http://%s/video/%s'
 
     _TEST = {
         'url': 'http://www.novamov.com/video/4rurhn9x446jj',
@@ -42,7 +43,7 @@ class NovaMovIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        url = 'http://%s/video/%s' % (self._HOST, video_id)
+        url = self._URL_TEMPLATE % (self._HOST, video_id)
 
         webpage = self._download_webpage(
             url, video_id, 'Downloading video page')
@@ -51,8 +52,13 @@ class NovaMovIE(InfoExtractor):
             raise ExtractorError('Video %s does not exist' % video_id, expected=True)
 
         def extract_filekey(default=NO_DEFAULT):
-            return self._search_regex(
+            filekey = self._search_regex(
                 self._FILEKEY_REGEX, webpage, 'filekey', default=default)
+            if filekey is not default and (filekey[0] != '"' or filekey[-1] != '"'):
+                return self._search_regex(
+                    r'var\s*%s\s*=\s*"([^"]+)"', webpage, 'filekey', default=default)
+            else:
+                return filekey
 
         filekey = extract_filekey(default=None)
 
@@ -127,7 +133,6 @@ class NowVideoIE(NovaMovIE):
     _HOST = 'www.nowvideo.to'
 
     _FILE_DELETED_REGEX = r'>This file no longer exists on our servers.<'
-    _FILEKEY_REGEX = r'var fkzd="([^"]+)";'
     _TITLE_REGEX = r'<h4>([^<]+)</h4>'
     _DESCRIPTION_REGEX = r'</h4>\s*<p>([^<]+)</p>'
 
@@ -153,6 +158,7 @@ class VideoWeedIE(NovaMovIE):
 
     _FILE_DELETED_REGEX = r'>This file no longer exists on our servers.<'
     _TITLE_REGEX = r'<h1 class="text_shadow">([^<]+)</h1>'
+    _URL_TEMPLATE = 'http://%s/file/%s'
 
     _TEST = {
         'url': 'http://www.videoweed.es/file/b42178afbea14',

From b5424acdb9983dbb761aed2f014dadeeac355930 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 13 Dec 2015 15:37:52 +0800
Subject: [PATCH 3362/6219] [novamov] Improve existence checking

---
 youtube_dl/extractor/novamov.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/novamov.py b/youtube_dl/extractor/novamov.py
index 0c4ff90f2..a2a527ac3 100644
--- a/youtube_dl/extractor/novamov.py
+++ b/youtube_dl/extractor/novamov.py
@@ -40,6 +40,10 @@ class NovaMovIE(InfoExtractor):
         'skip': '"Invalid token" errors abound (in web interface as well as youtube-dl, there is nothing we can do about it.)'
     }
 
+    def _check_existence(self, webpage, video_id):
+        if re.search(self._FILE_DELETED_REGEX, webpage) is not None:
+            raise ExtractorError('Video %s does not exist' % video_id, expected=True)
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
@@ -48,8 +52,7 @@ class NovaMovIE(InfoExtractor):
         webpage = self._download_webpage(
             url, video_id, 'Downloading video page')
 
-        if re.search(self._FILE_DELETED_REGEX, webpage) is not None:
-            raise ExtractorError('Video %s does not exist' % video_id, expected=True)
+        self._check_existence(webpage, video_id)
 
         def extract_filekey(default=NO_DEFAULT):
             filekey = self._search_regex(
@@ -75,6 +78,7 @@ class NovaMovIE(InfoExtractor):
             request.add_header('Referer', post_url)
             webpage = self._download_webpage(
                 request, video_id, 'Downloading continue to the video page')
+            self._check_existence(webpage, video_id)
 
         filekey = extract_filekey()
 

From f6abca506ef5274cfab151027bb3624d7b0e6a6b Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 13 Dec 2015 15:42:34 +0800
Subject: [PATCH 3363/6219] [nowvideo] Skip deleted test case

---
 youtube_dl/extractor/novamov.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/novamov.py b/youtube_dl/extractor/novamov.py
index a2a527ac3..422fb993d 100644
--- a/youtube_dl/extractor/novamov.py
+++ b/youtube_dl/extractor/novamov.py
@@ -148,7 +148,8 @@ class NowVideoIE(NovaMovIE):
             'ext': 'flv',
             'title': 'youtubedl test video _BaW_jenozKc.mp4',
             'description': 'Description',
-        }
+        },
+        'skip': 'Video 0mw0yow7b6dxa does not exist',
     }
 
 

From 22d07ba4e4fc1237cc015c971370ea9faf900610 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 13 Dec 2015 17:29:27 +0800
Subject: [PATCH 3364/6219] [infoq] Fix extraction for HTTP URLs (closes #7739)

---
 youtube_dl/extractor/infoq.py | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/infoq.py b/youtube_dl/extractor/infoq.py
index 71cfd12c5..89bc3039c 100644
--- a/youtube_dl/extractor/infoq.py
+++ b/youtube_dl/extractor/infoq.py
@@ -3,10 +3,7 @@ from __future__ import unicode_literals
 import base64
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse_unquote,
-    compat_urlparse,
-)
+from ..compat import compat_urllib_parse_unquote
 
 
 class InfoQIE(InfoExtractor):
@@ -45,9 +42,11 @@ class InfoQIE(InfoExtractor):
         video_filename = playpath.split('/')[-1]
         video_id, extension = video_filename.split('.')
 
-        http_base = self._search_regex(
-            r'EXPRESSINSTALL_SWF\s*=\s*[^"]*"((?:https?:)?//[^/"]+/)', webpage,
-            'HTTP base URL')
+        http_video_url = self._search_regex(r'P\.s\s*=\s*\'([^\']+)\'', webpage, 'video URL')
+
+        policy = self._search_regex(r'InfoQConstants.scp\s*=\s*\'([^\']+)\'', webpage, 'policy')
+        signature = self._search_regex(r'InfoQConstants.scs\s*=\s*\'([^\']+)\'', webpage, 'signature')
+        key_pair_id = self._search_regex(r'InfoQConstants.sck\s*=\s*\'([^\']+)\'', webpage, 'key-pair-id')
 
         formats = [{
             'format_id': 'rtmp',
@@ -56,7 +55,11 @@ class InfoQIE(InfoExtractor):
             'play_path': playpath,
         }, {
             'format_id': 'http',
-            'url': compat_urlparse.urljoin(url, http_base) + real_id,
+            'url': http_video_url,
+            'http_headers': {
+                'Cookie': 'CloudFront-Policy=%s; CloudFront-Signature=%s; CloudFront-Key-Pair-Id=%s' % (
+                    policy, signature, key_pair_id),
+            },
         }]
         self._sort_formats(formats)
 

From f3711edcf133bb621f6f58efa6753965573d6629 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 13 Dec 2015 10:52:59 +0100
Subject: [PATCH 3365/6219] release 2015.12.13

---
 docs/supportedsites.md | 1 +
 youtube_dl/version.py  | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 3bb3abb25..8253335e3 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -188,6 +188,7 @@
  - **Freesound**
  - **freespeech.org**
  - **FreeVideo**
+ - **Funimation**
  - **FunnyOrDie**
  - **GameInformer**
  - **Gamekings**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 6cd748b48..20b44b94d 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.12.10'
+__version__ = '2015.12.13'

From 02a63fadc357190ea116690a50610f04b61da976 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 13 Dec 2015 19:15:45 +0800
Subject: [PATCH 3366/6219] [infoq] Refactor and support the Chinese version

Closes #7576
---
 youtube_dl/extractor/infoq.py | 108 ++++++++++++++++++++++++----------
 1 file changed, 76 insertions(+), 32 deletions(-)

diff --git a/youtube_dl/extractor/infoq.py b/youtube_dl/extractor/infoq.py
index 89bc3039c..016af2084 100644
--- a/youtube_dl/extractor/infoq.py
+++ b/youtube_dl/extractor/infoq.py
@@ -1,9 +1,15 @@
+# coding: utf-8
+
 from __future__ import unicode_literals
 
 import base64
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse_unquote
+from ..compat import (
+    compat_urllib_parse_unquote,
+    compat_parse_qs,
+)
+from ..utils import determine_ext
 
 
 class InfoQIE(InfoExtractor):
@@ -13,7 +19,7 @@ class InfoQIE(InfoExtractor):
         'url': 'http://www.infoq.com/presentations/A-Few-of-My-Favorite-Python-Things',
         'md5': 'b5ca0e0a8c1fed93b0e65e48e462f9a2',
         'info_dict': {
-            'id': '12-jan-pythonthings',
+            'id': 'A-Few-of-My-Favorite-Python-Things',
             'ext': 'mp4',
             'description': 'Mike Pirnat presents some tips and tricks, standard libraries and third party packages that make programming in Python a richer experience.',
             'title': 'A Few of My Favorite [Python] Things',
@@ -21,8 +27,71 @@ class InfoQIE(InfoExtractor):
     }, {
         'url': 'http://www.infoq.com/fr/presentations/changez-avis-sur-javascript',
         'only_matching': True,
+    }, {
+        'url': 'http://www.infoq.com/cn/presentations/openstack-continued-delivery',
+        'md5': '4918d0cca1497f2244572caf626687ef',
+        'info_dict': {
+            'id': 'openstack-continued-delivery',
+            'title': 'OpenStack持续交付之路',
+            'ext': 'flv',
+            'description': 'md5:308d981fb28fa42f49f9568322c683ff',
+        },
     }]
 
+    def _extract_bokecc_videos(self, webpage, video_id):
+        # TODO: bokecc.com is a Chinese video cloud platform
+        # It should have an independent extractor but I don't have other
+        # examples using bokecc
+        player_params_str = self._html_search_regex(
+            r'<script[^>]+src="http://p\.bokecc\.com/player\?([^"]+)',
+            webpage, 'player params', default=None)
+
+        player_params = compat_parse_qs(player_params_str)
+
+        info_xml = self._download_xml(
+            'http://p.bokecc.com/servlet/playinfo?uid=%s&vid=%s&m=1' % (
+                player_params['siteid'][0], player_params['vid'][0]), video_id)
+
+        return [{
+            'format_id': 'bokecc',
+            'url': quality.find('./copy').attrib['playurl'],
+            'preference': int(quality.attrib['value']),
+        } for quality in info_xml.findall('./video/quality')]
+
+    def _extract_rtmp_videos(self, webpage):
+        # The server URL is hardcoded
+        video_url = 'rtmpe://video.infoq.com/cfx/st/'
+
+        # Extract video URL
+        encoded_id = self._search_regex(
+            r"jsclassref\s*=\s*'([^']*)'", webpage, 'encoded id', default=None)
+
+        real_id = compat_urllib_parse_unquote(base64.b64decode(encoded_id.encode('ascii')).decode('utf-8'))
+        playpath = 'mp4:' + real_id
+
+        return [{
+            'format_id': 'rtmp',
+            'url': video_url,
+            'ext': determine_ext(playpath),
+            'play_path': playpath,
+        }]
+
+    def _extract_http_videos(self, webpage):
+        http_video_url = self._search_regex(r'P\.s\s*=\s*\'([^\']+)\'', webpage, 'video URL')
+
+        policy = self._search_regex(r'InfoQConstants.scp\s*=\s*\'([^\']+)\'', webpage, 'policy')
+        signature = self._search_regex(r'InfoQConstants.scs\s*=\s*\'([^\']+)\'', webpage, 'signature')
+        key_pair_id = self._search_regex(r'InfoQConstants.sck\s*=\s*\'([^\']+)\'', webpage, 'key-pair-id')
+
+        return [{
+            'format_id': 'http',
+            'url': http_video_url,
+            'http_headers': {
+                'Cookie': 'CloudFront-Policy=%s; CloudFront-Signature=%s; CloudFront-Key-Pair-Id=%s' % (
+                    policy, signature, key_pair_id),
+            },
+        }]
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
@@ -30,37 +99,12 @@ class InfoQIE(InfoExtractor):
         video_title = self._html_search_regex(r'<title>(.*?)</title>', webpage, 'title')
         video_description = self._html_search_meta('description', webpage, 'description')
 
-        # The server URL is hardcoded
-        video_url = 'rtmpe://video.infoq.com/cfx/st/'
+        if '/cn/' in url:
+            # for China videos, HTTP video URL exists but always fails with 403
+            formats = self._extract_bokecc_videos(webpage, video_id)
+        else:
+            formats = self._extract_rtmp_videos(webpage) + self._extract_http_videos(webpage)
 
-        # Extract video URL
-        encoded_id = self._search_regex(
-            r"jsclassref\s*=\s*'([^']*)'", webpage, 'encoded id')
-        real_id = compat_urllib_parse_unquote(base64.b64decode(encoded_id.encode('ascii')).decode('utf-8'))
-        playpath = 'mp4:' + real_id
-
-        video_filename = playpath.split('/')[-1]
-        video_id, extension = video_filename.split('.')
-
-        http_video_url = self._search_regex(r'P\.s\s*=\s*\'([^\']+)\'', webpage, 'video URL')
-
-        policy = self._search_regex(r'InfoQConstants.scp\s*=\s*\'([^\']+)\'', webpage, 'policy')
-        signature = self._search_regex(r'InfoQConstants.scs\s*=\s*\'([^\']+)\'', webpage, 'signature')
-        key_pair_id = self._search_regex(r'InfoQConstants.sck\s*=\s*\'([^\']+)\'', webpage, 'key-pair-id')
-
-        formats = [{
-            'format_id': 'rtmp',
-            'url': video_url,
-            'ext': extension,
-            'play_path': playpath,
-        }, {
-            'format_id': 'http',
-            'url': http_video_url,
-            'http_headers': {
-                'Cookie': 'CloudFront-Policy=%s; CloudFront-Signature=%s; CloudFront-Key-Pair-Id=%s' % (
-                    policy, signature, key_pair_id),
-            },
-        }]
         self._sort_formats(formats)
 
         return {

From ee86e2c6d714bf9641075623bf01effac5240fa8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 13 Dec 2015 19:16:01 +0600
Subject: [PATCH 3367/6219] [novamov] Add support for mobile URLs

---
 youtube_dl/extractor/novamov.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/novamov.py b/youtube_dl/extractor/novamov.py
index 422fb993d..97f0549d3 100644
--- a/youtube_dl/extractor/novamov.py
+++ b/youtube_dl/extractor/novamov.py
@@ -17,7 +17,7 @@ class NovaMovIE(InfoExtractor):
     IE_NAME = 'novamov'
     IE_DESC = 'NovaMov'
 
-    _VALID_URL_TEMPLATE = r'http://(?:(?:www\.)?%(host)s/(?:file|video)/|(?:(?:embed|www)\.)%(host)s/embed\.php\?(?:.*?&)?v=)(?P<id>[a-z\d]{13})'
+    _VALID_URL_TEMPLATE = r'http://(?:(?:www\.)?%(host)s/(?:file|video|mobile/#/videos)/|(?:(?:embed|www)\.)%(host)s/embed\.php\?(?:.*?&)?v=)(?P<id>[a-z\d]{13})'
     _VALID_URL = _VALID_URL_TEMPLATE % {'host': 'novamov\.com'}
 
     _HOST = 'www.novamov.com'

From 2d9295643ef2a94880ab96f485f2d7c261856c6e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 13 Dec 2015 23:55:10 +0600
Subject: [PATCH 3368/6219] [footyroom] Skip test

---
 youtube_dl/extractor/footyroom.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/footyroom.py b/youtube_dl/extractor/footyroom.py
index 4c7dbca40..370fd006f 100644
--- a/youtube_dl/extractor/footyroom.py
+++ b/youtube_dl/extractor/footyroom.py
@@ -13,6 +13,7 @@ class FootyRoomIE(InfoExtractor):
             'title': 'Schalke 04 0 – 2 Real Madrid',
         },
         'playlist_count': 3,
+        'skip': 'Video for this match is not available',
     }, {
         'url': 'http://footyroom.com/georgia-0-2-germany-2015-03/',
         'info_dict': {

From 6583c741cd31b0ebd605efdbde15a81dd6f81de1 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 14 Dec 2015 02:33:40 +0800
Subject: [PATCH 3369/6219] [novamov] Fix filekey extraction and reupload test
 video

---
 youtube_dl/extractor/novamov.py | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/novamov.py b/youtube_dl/extractor/novamov.py
index 97f0549d3..bbb5be492 100644
--- a/youtube_dl/extractor/novamov.py
+++ b/youtube_dl/extractor/novamov.py
@@ -59,7 +59,7 @@ class NovaMovIE(InfoExtractor):
                 self._FILEKEY_REGEX, webpage, 'filekey', default=default)
             if filekey is not default and (filekey[0] != '"' or filekey[-1] != '"'):
                 return self._search_regex(
-                    r'var\s*%s\s*=\s*"([^"]+)"', webpage, 'filekey', default=default)
+                    r'var\s*%s\s*=\s*"([^"]+)"' % re.escape(filekey), webpage, 'filekey', default=default)
             else:
                 return filekey
 
@@ -141,15 +141,14 @@ class NowVideoIE(NovaMovIE):
     _DESCRIPTION_REGEX = r'</h4>\s*<p>([^<]+)</p>'
 
     _TEST = {
-        'url': 'http://www.nowvideo.to/video/0mw0yow7b6dxa',
-        'md5': 'f8fbbc8add72bd95b7850c6a02fc8817',
+        'url': 'http://www.nowvideo.sx/video/f1d6fce9a968b',
+        'md5': '12c82cad4f2084881d8bc60ee29df092',
         'info_dict': {
-            'id': '0mw0yow7b6dxa',
+            'id': 'f1d6fce9a968b',
             'ext': 'flv',
-            'title': 'youtubedl test video _BaW_jenozKc.mp4',
+            'title': 'youtubedl test video BaWjenozKc',
             'description': 'Description',
         },
-        'skip': 'Video 0mw0yow7b6dxa does not exist',
     }
 
 

From eb0bdc2c3e112801bbc7517def25647b8a9fe4a4 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 14 Dec 2015 02:50:59 +0800
Subject: [PATCH 3370/6219] [novamov] Fix again

---
 youtube_dl/extractor/novamov.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/novamov.py b/youtube_dl/extractor/novamov.py
index bbb5be492..d68c1ad79 100644
--- a/youtube_dl/extractor/novamov.py
+++ b/youtube_dl/extractor/novamov.py
@@ -59,7 +59,7 @@ class NovaMovIE(InfoExtractor):
                 self._FILEKEY_REGEX, webpage, 'filekey', default=default)
             if filekey is not default and (filekey[0] != '"' or filekey[-1] != '"'):
                 return self._search_regex(
-                    r'var\s*%s\s*=\s*"([^"]+)"' % re.escape(filekey), webpage, 'filekey', default=default)
+                    r'var\s+%s\s*=\s*"([^"]+)"' % re.escape(filekey), webpage, 'filekey', default=default)
             else:
                 return filekey
 

From 31b2051e211f3e2691a186d16733cf91eb4ab391 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 14 Dec 2015 21:30:58 +0600
Subject: [PATCH 3371/6219] [utils] Add remove_quotes

---
 test/test_utils.py  | 10 ++++++++++
 youtube_dl/utils.py |  9 +++++++++
 2 files changed, 19 insertions(+)

diff --git a/test/test_utils.py b/test/test_utils.py
index 501355c74..8fc74e591 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -43,6 +43,7 @@ from youtube_dl.utils import (
     sanitize_path,
     prepend_extension,
     replace_extension,
+    remove_quotes,
     shell_quote,
     smuggle_url,
     str_to_int,
@@ -200,6 +201,15 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(replace_extension('.abc', 'temp'), '.abc.temp')
         self.assertEqual(replace_extension('.abc.ext', 'temp'), '.abc.temp')
 
+    def test_remove_quotes(self):
+        self.assertEqual(remove_quotes(None), None)
+        self.assertEqual(remove_quotes('"'), '"')
+        self.assertEqual(remove_quotes("'"), "'")
+        self.assertEqual(remove_quotes(';'), ';')
+        self.assertEqual(remove_quotes('";'), '";')
+        self.assertEqual(remove_quotes('""'), '')
+        self.assertEqual(remove_quotes('";"'), ';')
+
     def test_ordered_set(self):
         self.assertEqual(orderedSet([1, 1, 2, 3, 4, 4, 5, 6, 7, 3, 5]), [1, 2, 3, 4, 5, 6, 7])
         self.assertEqual(orderedSet([]), [])
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index d0606b4bc..91917fc96 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1406,6 +1406,15 @@ def remove_end(s, end):
     return s
 
 
+def remove_quotes(s):
+    if s is None or len(s) < 2:
+        return s
+    for quote in ('"', "'", ):
+        if s[0] == quote and s[-1] == quote:
+            return s[1:-1]
+    return s
+
+
 def url_basename(url):
     path = compat_urlparse.urlparse(url).path
     return path.strip('/').split('/')[-1]

From 0cb58b0259de0b0f44b0326d492b98a8eeb6316e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 14 Dec 2015 21:31:53 +0600
Subject: [PATCH 3372/6219] [youtube] Extract alt_title and creator for music
 videos (Closes #7862)

---
 youtube_dl/extractor/youtube.py | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 9b39505ba..4556a16fb 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -33,6 +33,7 @@ from ..utils import (
     int_or_none,
     orderedSet,
     parse_duration,
+    remove_quotes,
     remove_start,
     sanitized_Request,
     smuggle_url,
@@ -395,12 +396,14 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'ext': 'mp4',
                 'upload_date': '20120506',
                 'title': 'Icona Pop - I Love It (feat. Charli XCX) [OFFICIAL VIDEO]',
+                'alt_title': 'I Love It (feat. Charli XCX)',
                 'description': 'md5:782e8651347686cba06e58f71ab51773',
                 'tags': ['Icona Pop i love it', 'sweden', 'pop music', 'big beat records', 'big beat', 'charli',
                          'xcx', 'charli xcx', 'girls', 'hbo', 'i love it', "i don't care", 'icona', 'pop',
                          'iconic ep', 'iconic', 'love', 'it'],
                 'uploader': 'Icona Pop',
                 'uploader_id': 'IconaPop',
+                'creator': 'Icona Pop',
             }
         },
         {
@@ -411,9 +414,11 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'ext': 'mp4',
                 'upload_date': '20130703',
                 'title': 'Justin Timberlake - Tunnel Vision (Explicit)',
+                'alt_title': 'Tunnel Vision',
                 'description': 'md5:64249768eec3bc4276236606ea996373',
                 'uploader': 'justintimberlakeVEVO',
                 'uploader_id': 'justintimberlakeVEVO',
+                'creator': 'Justin Timberlake',
                 'age_limit': 18,
             }
         },
@@ -492,10 +497,12 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'id': 'nfWlot6h_JM',
                 'ext': 'm4a',
                 'title': 'Taylor Swift - Shake It Off',
+                'alt_title': 'Shake It Off',
                 'description': 'md5:95f66187cd7c8b2c13eb78e1223b63c3',
                 'uploader': 'TaylorSwiftVEVO',
                 'uploader_id': 'TaylorSwiftVEVO',
                 'upload_date': '20140818',
+                'creator': 'Taylor Swift',
             },
             'params': {
                 'youtube_include_dash_manifest': True,
@@ -551,9 +558,11 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'ext': 'mp4',
                 'upload_date': '20100430',
                 'uploader_id': 'deadmau5',
+                'creator': 'deadmau5',
                 'description': 'md5:12c56784b8032162bb936a5f76d55360',
                 'uploader': 'deadmau5',
                 'title': 'Deadmau5 - Some Chords (HD)',
+                'alt_title': 'Some Chords',
             },
             'expected_warnings': [
                 'DASH manifest missing',
@@ -701,10 +710,12 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'id': 'lsguqyKfVQg',
                 'ext': 'mp4',
                 'title': '{dark walk}; Loki/AC/Dishonored; collab w/Elflover21',
+                'alt_title': 'Dark Walk',
                 'description': 'md5:8085699c11dc3f597ce0410b0dcbb34a',
                 'upload_date': '20151119',
                 'uploader_id': 'IronSoulElf',
                 'uploader': 'IronSoulElf',
+                'creator': 'Todd Haberman, Daniel Law Heath & Aaron Kaplan',
             },
             'params': {
                 'skip_download': True,
@@ -1308,6 +1319,15 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 upload_date = ' '.join(re.sub(r'[/,-]', r' ', mobj.group(1)).split())
         upload_date = unified_strdate(upload_date)
 
+        m_music = re.search(
+            r'<h4[^>]+class="title"[^>]*>\s*Music\s*</h4>\s*<ul[^>]*>\s*<li>(?P<title>.+?) by (?P<creator>.+?)(?:\(.+?\))?</li',
+            video_webpage)
+        if m_music:
+            video_alt_title = remove_quotes(unescapeHTML(m_music.group('title')))
+            video_creator = clean_html(m_music.group('creator'))
+        else:
+            video_alt_title = video_creator = None
+
         m_cat_container = self._search_regex(
             r'(?s)<h4[^>]*>\s*Category\s*</h4>\s*<ul[^>]*>(.*?)</ul>',
             video_webpage, 'categories', default=None)
@@ -1537,7 +1557,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             'uploader': video_uploader,
             'uploader_id': video_uploader_id,
             'upload_date': upload_date,
+            'creator': video_creator,
             'title': video_title,
+            'alt_title': video_alt_title,
             'thumbnail': video_thumbnail,
             'description': video_description,
             'categories': video_categories,

From d7ffcfcf9703f1f02e642d0855c0977056e2d0fc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 15 Dec 2015 21:09:14 +0600
Subject: [PATCH 3373/6219] [tf1] Fix extraction (Closes #7873)

---
 youtube_dl/extractor/tf1.py | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/tf1.py b/youtube_dl/extractor/tf1.py
index 3a68eaa80..32f5f78bf 100644
--- a/youtube_dl/extractor/tf1.py
+++ b/youtube_dl/extractor/tf1.py
@@ -43,11 +43,9 @@ class TF1IE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
-        embed_url = self._html_search_regex(
-            r'["\'](https?://www.wat.tv/embedframe/.*?)["\']', webpage, 'embed url')
-        embed_page = self._download_webpage(embed_url, video_id,
-                                            'Downloading embed player page')
-        wat_id = self._search_regex(r'UVID=(.*?)&', embed_page, 'wat id')
+        wat_id = self._html_search_regex(
+            r'(["\'])(?:https?:)?//www\.wat\.tv/embedframe/.*?(?P<id>\d+)\1',
+            webpage, 'wat id', group='id')
         wat_info = self._download_json(
             'http://www.wat.tv/interface/contentv3/%s' % wat_id, video_id)
         return self.url_result(wat_info['media']['url'], 'Wat')

From ae5e94808e70e608e6bbdf71c0fb8436bcaa76e2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 15 Dec 2015 21:11:52 +0600
Subject: [PATCH 3374/6219] [tf1] Fix extraction (2)

---
 youtube_dl/extractor/tf1.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tf1.py b/youtube_dl/extractor/tf1.py
index 32f5f78bf..2d84c9bf8 100644
--- a/youtube_dl/extractor/tf1.py
+++ b/youtube_dl/extractor/tf1.py
@@ -44,7 +44,7 @@ class TF1IE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
         wat_id = self._html_search_regex(
-            r'(["\'])(?:https?:)?//www\.wat\.tv/embedframe/.*?(?P<id>\d+)\1',
+            r'(["\'])(?:https?:)?//www\.wat\.tv/embedframe/.*?(?P<id>\d{8})\1',
             webpage, 'wat id', group='id')
         wat_info = self._download_json(
             'http://www.wat.tv/interface/contentv3/%s' % wat_id, video_id)

From 05467d5a5211adb5e2473decaef82769e778ffc4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 15 Dec 2015 21:31:58 +0600
Subject: [PATCH 3375/6219] [tf1] Relax _VALID_URL

---
 youtube_dl/extractor/tf1.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tf1.py b/youtube_dl/extractor/tf1.py
index 2d84c9bf8..80cd92938 100644
--- a/youtube_dl/extractor/tf1.py
+++ b/youtube_dl/extractor/tf1.py
@@ -6,7 +6,7 @@ from .common import InfoExtractor
 
 class TF1IE(InfoExtractor):
     """TF1 uses the wat.tv player."""
-    _VALID_URL = r'http://(?:(?:videos|www|lci)\.tf1|www\.tfou)\.fr/.*?-(?P<id>\d+)(?:-\d+)?\.html'
+    _VALID_URL = r'http://(?:(?:videos|www|lci)\.tf1|www\.tfou)\.fr/(?:[^/]+/)*(?P<id>.+?)\.html'
     _TESTS = [{
         'url': 'http://videos.tf1.fr/auto-moto/citroen-grand-c4-picasso-2013-presentation-officielle-8062060.html',
         'info_dict': {
@@ -38,6 +38,9 @@ class TF1IE(InfoExtractor):
     }, {
         'url': 'http://lci.tf1.fr/sept-a-huit/videos/sept-a-huit-du-24-mai-2015-8611550.html',
         'only_matching': True,
+    }, {
+        'url': 'http://www.tf1.fr/hd1/documentaire/videos/mylene-farmer-d-une-icone.html',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From ad1b6017cd48c62896a2d6c978f21e27f5d96d2f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 15 Dec 2015 21:36:59 +0600
Subject: [PATCH 3376/6219] [tf1] Fix tests

---
 youtube_dl/extractor/tf1.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tf1.py b/youtube_dl/extractor/tf1.py
index 80cd92938..6890021cf 100644
--- a/youtube_dl/extractor/tf1.py
+++ b/youtube_dl/extractor/tf1.py
@@ -22,7 +22,7 @@ class TF1IE(InfoExtractor):
     }, {
         'url': 'http://www.tfou.fr/chuggington/videos/le-grand-mysterioso-chuggington-7085291-739.html',
         'info_dict': {
-            'id': '12043945',
+            'id': 'le-grand-mysterioso-chuggington-7085291-739',
             'ext': 'mp4',
             'title': 'Le grand Mystérioso - Chuggington',
             'description': 'Le grand Mystérioso - Emery rêve qu\'un article lui soit consacré dans le journal.',
@@ -32,6 +32,7 @@ class TF1IE(InfoExtractor):
             # Sometimes wat serves the whole file with the --test option
             'skip_download': True,
         },
+        'skip': 'HTTP Error 410: Gone',
     }, {
         'url': 'http://www.tf1.fr/tf1/koh-lanta/videos/replay-koh-lanta-22-mai-2015.html',
         'only_matching': True,

From 2d3b70271ca427cb4bbe3cf7bb8b6dc0b471cae5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 16 Dec 2015 04:44:17 +0600
Subject: [PATCH 3377/6219] [rutube] Extend _VALID_URL

---
 youtube_dl/extractor/rutube.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/rutube.py b/youtube_dl/extractor/rutube.py
index 6b09550b0..9db62adb1 100644
--- a/youtube_dl/extractor/rutube.py
+++ b/youtube_dl/extractor/rutube.py
@@ -17,9 +17,9 @@ from ..utils import (
 class RutubeIE(InfoExtractor):
     IE_NAME = 'rutube'
     IE_DESC = 'Rutube videos'
-    _VALID_URL = r'https?://rutube\.ru/video/(?P<id>[\da-z]{32})'
+    _VALID_URL = r'https?://rutube\.ru/(?:video|play/embed)/(?P<id>[\da-z]{32})'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://rutube.ru/video/3eac3b4561676c17df9132a9a1e62e3e/',
         'info_dict': {
             'id': '3eac3b4561676c17df9132a9a1e62e3e',
@@ -36,7 +36,10 @@ class RutubeIE(InfoExtractor):
             # It requires ffmpeg (m3u8 download)
             'skip_download': True,
         },
-    }
+    }, {
+        'url': 'http://rutube.ru/play/embed/a10e53b86e8f349080f718582ce4c661',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From eb4f27405b172e6ab19ed622a0296ec8d65abe0f Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 16 Dec 2015 09:43:53 +0100
Subject: [PATCH 3378/6219] [vimeo] extract source file(closes #1072)

---
 youtube_dl/extractor/vimeo.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index f392ccf1c..cf854a0f6 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -23,6 +23,7 @@ from ..utils import (
     unsmuggle_url,
     urlencode_postdata,
     unescapeHTML,
+    parse_filesize,
 )
 
 
@@ -392,6 +393,20 @@ class VimeoIE(VimeoBaseInfoExtractor):
             comment_count = None
 
         formats = []
+        download_request = sanitized_Request('https://vimeo.com/%s?action=load_download_config' % video_id, headers={
+            'X-Requested-With': 'XMLHttpRequest'})
+        download_data = self._download_json(download_request, video_id, fatal=False)
+        if download_data:
+            source_file = download_data.get('source_file')
+            if source_file and not source_file.get('is_cold') and not source_file.get('is_defrosting'):
+                formats.append({
+                    'url': source_file['download_url'],
+                    'ext': source_file['extension'].lower(),
+                    'width': int_or_none(source_file.get('width')),
+                    'height': int_or_none(source_file.get('height')),
+                    'filesize': parse_filesize(source_file.get('size')),
+                    'format_id': source_file.get('public_name', 'Original'),
+                })
         config_files = config['video'].get('files') or config['request'].get('files', {})
         for f in config_files.get('progressive', []):
             video_url = f.get('url')

From 8534bf1f0051640399b7c65c7d33e38b02f598fd Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 16 Dec 2015 16:36:25 +0100
Subject: [PATCH 3379/6219] [vimeo] prefer original format

---
 youtube_dl/extractor/vimeo.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index cf854a0f6..715ede34e 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -406,6 +406,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
                     'height': int_or_none(source_file.get('height')),
                     'filesize': parse_filesize(source_file.get('size')),
                     'format_id': source_file.get('public_name', 'Original'),
+                    'preference': 1,
                 })
         config_files = config['video'].get('files') or config['request'].get('files', {})
         for f in config_files.get('progressive', []):
@@ -423,12 +424,12 @@ class VimeoIE(VimeoBaseInfoExtractor):
         m3u8_url = config_files.get('hls', {}).get('url')
         if m3u8_url:
             m3u8_formats = self._extract_m3u8_formats(
-                m3u8_url, video_id, 'mp4', 'm3u8_native', 0, 'hls', fatal=False)
+                m3u8_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False)
             if m3u8_formats:
                 formats.extend(m3u8_formats)
         # Bitrates are completely broken. Single m3u8 may contain entries in kbps and bps
         # at the same time without actual units specified. This lead to wrong sorting.
-        self._sort_formats(formats, field_preference=('height', 'width', 'fps', 'format_id'))
+        self._sort_formats(formats, field_preference=('preference', 'height', 'width', 'fps', 'format_id'))
 
         subtitles = {}
         text_tracks = config['request'].get('text_tracks')

From 323f82a7e05afcc3518706dc18016ebd97fbc052 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 16 Dec 2015 17:00:17 +0100
Subject: [PATCH 3380/6219] [vimeo] add test for original format

---
 youtube_dl/extractor/vimeo.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 715ede34e..ce08e6955 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -185,6 +185,20 @@ class VimeoIE(VimeoBaseInfoExtractor):
                 'uploader_id': 'user28849593',
             },
         },
+        {
+            # contains original format
+            'url': 'https://vimeo.com/33951933',
+            'md5': '53c688fa95a55bf4b7293d37a89c5c53',
+            'info_dict': {
+                'id': '33951933',
+                'ext': 'mp4',
+                'title': 'FOX CLASSICS - Forever Classic ID - A Full Minute',
+                'uploader': 'The DMCI',
+                'uploader_id': 'dmci',
+                'upload_date': '20111220',
+                'description': 'md5:ae23671e82d05415868f7ad1aec21147',
+            },
+        },
         {
             'url': 'https://vimeo.com/109815029',
             'note': 'Video not completely processed, "failed" seed status',

From 35e22b6b32d0e662d486c9be8c76f6ea86f7cdd4 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 17 Dec 2015 12:51:50 +0100
Subject: [PATCH 3381/6219] [youku] check for the correct variable

---
 youtube_dl/extractor/youku.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index d33caa79e..3a3432be8 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -221,7 +221,7 @@ class YoukuIE(InfoExtractor):
                     'Youku said: Sorry, this video is available in China only', expected=True)
             else:
                 msg = 'Youku server reported error %i' % error.get('code')
-                if error is not None:
+                if error_note is not None:
                     msg += ': ' + error_note
                 raise ExtractorError(msg)
 

From 8f0afda028e64f7620d8ce21e9cfa4d686e3e6c6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 17 Dec 2015 20:24:33 +0600
Subject: [PATCH 3382/6219] [pbs] Extend _VALID_URL (Closes #7889)

---
 youtube_dl/extractor/pbs.py | 320 ++++++++++++++++++------------------
 1 file changed, 160 insertions(+), 160 deletions(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 604a0dd22..744e4a09a 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -16,165 +16,165 @@ from ..utils import (
 
 class PBSIE(InfoExtractor):
     _STATIONS = (
-        ('video.pbs.org', 'PBS: Public Broadcasting Service'),  # http://www.pbs.org/
-        ('video.aptv.org', 'APT - Alabama Public Television (WBIQ)'),  # http://aptv.org/
-        ('video.gpb.org', 'GPB/Georgia Public Broadcasting (WGTV)'),  # http://www.gpb.org/
-        ('video.mpbonline.org', 'Mississippi Public Broadcasting (WMPN)'),  # http://www.mpbonline.org
-        ('video.wnpt.org', 'Nashville Public Television (WNPT)'),  # http://www.wnpt.org
-        ('video.wfsu.org', 'WFSU-TV (WFSU)'),  # http://wfsu.org/
-        ('video.wsre.org', 'WSRE (WSRE)'),  # http://www.wsre.org
-        ('video.wtcitv.org', 'WTCI (WTCI)'),  # http://www.wtcitv.org
-        ('video.pba.org', 'WPBA/Channel 30 (WPBA)'),  # http://pba.org/
-        ('video.alaskapublic.org', 'Alaska Public Media (KAKM)'),  # http://alaskapublic.org/kakm
-        # ('kuac.org', 'KUAC (KUAC)'),  # http://kuac.org/kuac-tv/
-        # ('ktoo.org', '360 North (KTOO)'),  # http://www.ktoo.org/
-        # ('azpm.org', 'KUAT 6 (KUAT)'),  # http://www.azpm.org/
-        ('video.azpbs.org', 'Arizona PBS (KAET)'),  # http://www.azpbs.org
-        ('portal.knme.org', 'KNME-TV/Channel 5 (KNME)'),  # http://www.newmexicopbs.org/
-        ('video.vegaspbs.org', 'Vegas PBS (KLVX)'),  # http://vegaspbs.org/
-        ('watch.aetn.org', 'AETN/ARKANSAS ETV NETWORK (KETS)'),  # http://www.aetn.org/
-        ('video.ket.org', 'KET (WKLE)'),  # http://www.ket.org/
-        ('video.wkno.org', 'WKNO/Channel 10 (WKNO)'),  # http://www.wkno.org/
-        ('video.lpb.org', 'LPB/LOUISIANA PUBLIC BROADCASTING (WLPB)'),  # http://www.lpb.org/
-        ('videos.oeta.tv', 'OETA (KETA)'),  # http://www.oeta.tv
-        ('video.optv.org', 'Ozarks Public Television (KOZK)'),  # http://www.optv.org/
-        ('watch.wsiu.org', 'WSIU Public Broadcasting (WSIU)'),  # http://www.wsiu.org/
-        ('video.keet.org', 'KEET TV (KEET)'),  # http://www.keet.org
-        ('pbs.kixe.org', 'KIXE/Channel 9 (KIXE)'),  # http://kixe.org/
-        ('video.kpbs.org', 'KPBS San Diego (KPBS)'),  # http://www.kpbs.org/
-        ('video.kqed.org', 'KQED (KQED)'),  # http://www.kqed.org
-        ('vids.kvie.org', 'KVIE Public Television (KVIE)'),  # http://www.kvie.org
-        ('video.pbssocal.org', 'PBS SoCal/KOCE (KOCE)'),  # http://www.pbssocal.org/
-        ('video.valleypbs.org', 'ValleyPBS (KVPT)'),  # http://www.valleypbs.org/
-        ('video.cptv.org', 'CONNECTICUT PUBLIC TELEVISION (WEDH)'),  # http://cptv.org
-        ('watch.knpb.org', 'KNPB Channel 5 (KNPB)'),  # http://www.knpb.org/
-        ('video.soptv.org', 'SOPTV (KSYS)'),  # http://www.soptv.org
-        # ('klcs.org', 'KLCS/Channel 58 (KLCS)'),  # http://www.klcs.org
-        # ('krcb.org', 'KRCB Television & Radio (KRCB)'),  # http://www.krcb.org
-        # ('kvcr.org', 'KVCR TV/DT/FM :: Vision for the Future (KVCR)'),  # http://kvcr.org
-        ('video.rmpbs.org', 'Rocky Mountain PBS (KRMA)'),  # http://www.rmpbs.org
-        ('video.kenw.org', 'KENW-TV3 (KENW)'),  # http://www.kenw.org
-        ('video.kued.org', 'KUED Channel 7 (KUED)'),  # http://www.kued.org
-        ('video.wyomingpbs.org', 'Wyoming PBS (KCWC)'),  # http://www.wyomingpbs.org
-        ('video.cpt12.org', 'Colorado Public Television / KBDI 12 (KBDI)'),  # http://www.cpt12.org/
-        ('video.kbyueleven.org', 'KBYU-TV (KBYU)'),  # http://www.kbyutv.org/
-        ('video.thirteen.org', 'Thirteen/WNET New York (WNET)'),  # http://www.thirteen.org
-        ('video.wgbh.org', 'WGBH/Channel 2 (WGBH)'),  # http://wgbh.org
-        ('video.wgby.org', 'WGBY (WGBY)'),  # http://www.wgby.org
-        ('watch.njtvonline.org', 'NJTV Public Media NJ (WNJT)'),  # http://www.njtvonline.org/
-        # ('ripbs.org', 'Rhode Island PBS (WSBE)'),  # http://www.ripbs.org/home/
-        ('watch.wliw.org', 'WLIW21 (WLIW)'),  # http://www.wliw.org/
-        ('video.mpt.tv', 'mpt/Maryland Public Television (WMPB)'),  # http://www.mpt.org
-        ('watch.weta.org', 'WETA Television and Radio (WETA)'),  # http://www.weta.org
-        ('video.whyy.org', 'WHYY (WHYY)'),  # http://www.whyy.org
-        ('video.wlvt.org', 'PBS 39 (WLVT)'),  # http://www.wlvt.org/
-        ('video.wvpt.net', 'WVPT - Your Source for PBS and More! (WVPT)'),  # http://www.wvpt.net
-        ('video.whut.org', 'Howard University Television (WHUT)'),  # http://www.whut.org
-        ('video.wedu.org', 'WEDU PBS (WEDU)'),  # http://www.wedu.org
-        ('video.wgcu.org', 'WGCU Public Media (WGCU)'),  # http://www.wgcu.org/
-        # ('wjct.org', 'WJCT Public Broadcasting (WJCT)'),  # http://www.wjct.org
-        ('video.wpbt2.org', 'WPBT2 (WPBT)'),  # http://www.wpbt2.org
-        ('video.wucftv.org', 'WUCF TV (WUCF)'),  # http://wucftv.org
-        ('video.wuft.org', 'WUFT/Channel 5 (WUFT)'),  # http://www.wuft.org
-        ('watch.wxel.org', 'WXEL/Channel 42 (WXEL)'),  # http://www.wxel.org/home/
-        ('video.wlrn.org', 'WLRN/Channel 17 (WLRN)'),  # http://www.wlrn.org/
-        ('video.wusf.usf.edu', 'WUSF Public Broadcasting (WUSF)'),  # http://wusf.org/
-        ('video.scetv.org', 'ETV (WRLK)'),  # http://www.scetv.org
-        ('video.unctv.org', 'UNC-TV (WUNC)'),  # http://www.unctv.org/
-        # ('pbsguam.org', 'PBS Guam (KGTF)'),  # http://www.pbsguam.org/
-        ('video.pbshawaii.org', 'PBS Hawaii - Oceanic Cable Channel 10 (KHET)'),  # http://www.pbshawaii.org/
-        ('video.idahoptv.org', 'Idaho Public Television (KAID)'),  # http://idahoptv.org
-        ('video.ksps.org', 'KSPS (KSPS)'),  # http://www.ksps.org/home/
-        ('watch.opb.org', 'OPB (KOPB)'),  # http://www.opb.org
-        ('watch.nwptv.org', 'KWSU/Channel 10 & KTNW/Channel 31 (KWSU)'),  # http://www.kwsu.org
-        ('video.will.illinois.edu', 'WILL-TV (WILL)'),  # http://will.illinois.edu/
-        ('video.networkknowledge.tv', 'Network Knowledge - WSEC/Springfield (WSEC)'),  # http://www.wsec.tv
-        ('video.wttw.com', 'WTTW11 (WTTW)'),  # http://www.wttw.com/
-        # ('wtvp.org', 'WTVP & WTVP.org, Public Media for Central Illinois (WTVP)'),  # http://www.wtvp.org/
-        ('video.iptv.org', 'Iowa Public Television/IPTV (KDIN)'),  # http://www.iptv.org/
-        ('video.ninenet.org', 'Nine Network (KETC)'),  # http://www.ninenet.org
-        ('video.wfwa.org', 'PBS39 Fort Wayne (WFWA)'),  # http://wfwa.org/
-        ('video.wfyi.org', 'WFYI Indianapolis (WFYI)'),  # http://www.wfyi.org
-        ('video.mptv.org', 'Milwaukee Public Television (WMVS)'),  # http://www.mptv.org
-        ('video.wnin.org', 'WNIN (WNIN)'),  # http://www.wnin.org/
-        ('video.wnit.org', 'WNIT Public Television (WNIT)'),  # http://www.wnit.org/
-        ('video.wpt.org', 'WPT (WPNE)'),  # http://www.wpt.org/
-        ('video.wvut.org', 'WVUT/Channel 22 (WVUT)'),  # http://wvut.org/
-        ('video.weiu.net', 'WEIU/Channel 51 (WEIU)'),  # http://www.weiu.net
-        ('video.wqpt.org', 'WQPT-TV (WQPT)'),  # http://www.wqpt.org
-        ('video.wycc.org', 'WYCC PBS Chicago (WYCC)'),  # http://www.wycc.org
-        # ('lakeshorepublicmedia.org', 'Lakeshore Public Television (WYIN)'),  # http://lakeshorepublicmedia.org/
-        ('video.wipb.org', 'WIPB-TV (WIPB)'),  # http://wipb.org
-        ('video.indianapublicmedia.org', 'WTIU (WTIU)'),  # http://indianapublicmedia.org/tv/
-        ('watch.cetconnect.org', 'CET  (WCET)'),  # http://www.cetconnect.org
-        ('video.thinktv.org', 'ThinkTVNetwork (WPTD)'),  # http://www.thinktv.org
-        ('video.wbgu.org', 'WBGU-TV (WBGU)'),  # http://wbgu.org
-        ('video.wgvu.org', 'WGVU TV (WGVU)'),  # http://www.wgvu.org/
-        ('video.netnebraska.org', 'NET1 (KUON)'),  # http://netnebraska.org
-        ('video.pioneer.org', 'Pioneer Public Television (KWCM)'),  # http://www.pioneer.org
-        ('watch.sdpb.org', 'SDPB Television (KUSD)'),  # http://www.sdpb.org
-        ('video.tpt.org', 'TPT (KTCA)'),  # http://www.tpt.org
-        ('watch.ksmq.org', 'KSMQ (KSMQ)'),  # http://www.ksmq.org/
-        ('watch.kpts.org', 'KPTS/Channel 8 (KPTS)'),  # http://www.kpts.org/
-        ('watch.ktwu.org', 'KTWU/Channel 11 (KTWU)'),  # http://ktwu.org
-        # ('shptv.org', 'Smoky Hills Public Television (KOOD)'),  # http://www.shptv.org
-        # ('kcpt.org', 'KCPT Kansas City Public Television (KCPT)'),  # http://kcpt.org/
-        # ('blueridgepbs.org', 'Blue Ridge PBS (WBRA)'),  # http://www.blueridgepbs.org/
-        ('watch.easttennesseepbs.org', 'East Tennessee PBS (WSJK)'),  # http://easttennesseepbs.org
-        ('video.wcte.tv', 'WCTE-TV (WCTE)'),  # http://www.wcte.org
-        ('video.wljt.org', 'WLJT, Channel 11 (WLJT)'),  # http://wljt.org/
-        ('video.wosu.org', 'WOSU TV (WOSU)'),  # http://wosu.org/
-        ('video.woub.org', 'WOUB/WOUC (WOUB)'),  # http://woub.org/tv/index.php?section=5
-        ('video.wvpublic.org', 'WVPB (WVPB)'),  # http://wvpublic.org/
-        ('video.wkyupbs.org', 'WKYU-PBS (WKYU)'),  # http://www.wkyupbs.org
-        # ('wyes.org', 'WYES-TV/New Orleans (WYES)'),  # http://www.wyes.org
-        ('video.kera.org', 'KERA 13 (KERA)'),  # http://www.kera.org/
-        ('video.mpbn.net', 'MPBN (WCBB)'),  # http://www.mpbn.net/
-        ('video.mountainlake.org', 'Mountain Lake PBS (WCFE)'),  # http://www.mountainlake.org/
-        ('video.nhptv.org', 'NHPTV (WENH)'),  # http://nhptv.org/
-        ('video.vpt.org', 'Vermont PBS (WETK)'),  # http://www.vpt.org
-        ('video.witf.org', 'witf (WITF)'),  # http://www.witf.org
-        ('watch.wqed.org', 'WQED Multimedia (WQED)'),  # http://www.wqed.org/
-        ('video.wmht.org', 'WMHT Educational Telecommunications (WMHT)'),  # http://www.wmht.org/home/
-        ('video.deltabroadcasting.org', 'Q-TV (WDCQ)'),  # http://www.deltabroadcasting.org
-        ('video.dptv.org', 'WTVS Detroit Public TV (WTVS)'),  # http://www.dptv.org/
-        ('video.wcmu.org', 'CMU Public Television (WCMU)'),  # http://www.wcmu.org
-        ('video.wkar.org', 'WKAR-TV (WKAR)'),  # http://wkar.org/
-        ('wnmuvideo.nmu.edu', 'WNMU-TV Public TV 13 (WNMU)'),  # http://wnmutv.nmu.edu
-        ('video.wdse.org', 'WDSE - WRPT (WDSE)'),  # http://www.wdse.org/
-        ('video.wgte.org', 'WGTE TV (WGTE)'),  # http://www.wgte.org
-        ('video.lptv.org', 'Lakeland Public Television (KAWE)'),  # http://www.lakelandptv.org
-        # ('prairiepublic.org', 'PRAIRIE PUBLIC (KFME)'),  # http://www.prairiepublic.org/
-        ('video.kmos.org', 'KMOS-TV - Channels 6.1, 6.2 and 6.3 (KMOS)'),  # http://www.kmos.org/
-        ('watch.montanapbs.org', 'MontanaPBS (KUSM)'),  # http://montanapbs.org
-        ('video.krwg.org', 'KRWG/Channel 22 (KRWG)'),  # http://www.krwg.org
-        ('video.kacvtv.org', 'KACV (KACV)'),  # http://www.panhandlepbs.org/home/
-        ('video.kcostv.org', 'KCOS/Channel 13 (KCOS)'),  # www.kcostv.org
-        ('video.wcny.org', 'WCNY/Channel 24 (WCNY)'),  # http://www.wcny.org
-        ('video.wned.org', 'WNED (WNED)'),  # http://www.wned.org/
-        ('watch.wpbstv.org', 'WPBS (WPBS)'),  # http://www.wpbstv.org
-        ('video.wskg.org', 'WSKG Public TV (WSKG)'),  # http://wskg.org
-        ('video.wxxi.org', 'WXXI (WXXI)'),  # http://wxxi.org
-        ('video.wpsu.org', 'WPSU (WPSU)'),  # http://www.wpsu.org
-        # ('wqln.org', 'WQLN/Channel 54 (WQLN)'),  # http://www.wqln.org
-        ('on-demand.wvia.org', 'WVIA Public Media Studios (WVIA)'),  # http://www.wvia.org/
-        ('video.wtvi.org', 'WTVI (WTVI)'),  # http://www.wtvi.org/
-        # ('whro.org', 'WHRO (WHRO)'),  # http://whro.org
-        ('video.westernreservepublicmedia.org', 'Western Reserve PBS (WNEO)'),  # http://www.WesternReservePublicMedia.org/
-        ('video.ideastream.org', 'WVIZ/PBS ideastream (WVIZ)'),  # http://www.wviz.org/
-        ('video.kcts9.org', 'KCTS 9 (KCTS)'),  # http://kcts9.org/
-        ('video.basinpbs.org', 'Basin PBS (KPBT)'),  # http://www.basinpbs.org
-        ('video.houstonpbs.org', 'KUHT / Channel 8 (KUHT)'),  # http://www.houstonpublicmedia.org/
-        # ('tamu.edu', 'KAMU - TV (KAMU)'),  # http://KAMU.tamu.edu
-        # ('kedt.org', 'KEDT/Channel 16 (KEDT)'),  # http://www.kedt.org
-        ('video.klrn.org', 'KLRN (KLRN)'),  # http://www.klrn.org
-        ('video.klru.tv', 'KLRU (KLRU)'),  # http://www.klru.org
-        # ('kmbh.org', 'KMBH-TV (KMBH)'),  # http://www.kmbh.org
-        # ('knct.org', 'KNCT (KNCT)'),  # http://www.knct.org
-        # ('ktxt.org', 'KTTZ-TV (KTXT)'),  # http://www.ktxt.org
-        ('video.wtjx.org', 'WTJX Channel 12 (WTJX)'),  # http://www.wtjx.org/
-        ('video.ideastations.org', 'WCVE PBS (WCVE)'),  # http://ideastations.org/
-        ('video.kbtc.org', 'KBTC Public Television (KBTC)'),  # http://kbtc.org
+        (r'(?:video|www)\.pbs\.org', 'PBS: Public Broadcasting Service'),  # http://www.pbs.org/
+        (r'video\.aptv\.org', 'APT - Alabama Public Television (WBIQ)'),  # http://aptv.org/
+        (r'video\.gpb\.org', 'GPB/Georgia Public Broadcasting (WGTV)'),  # http://www.gpb.org/
+        (r'video\.mpbonline\.org', 'Mississippi Public Broadcasting (WMPN)'),  # http://www.mpbonline.org
+        (r'video\.wnpt\.org', 'Nashville Public Television (WNPT)'),  # http://www.wnpt.org
+        (r'video\.wfsu\.org', 'WFSU-TV (WFSU)'),  # http://wfsu.org/
+        (r'video\.wsre\.org', 'WSRE (WSRE)'),  # http://www.wsre.org
+        (r'video\.wtcitv\.org', 'WTCI (WTCI)'),  # http://www.wtcitv.org
+        (r'video\.pba\.org', 'WPBA/Channel 30 (WPBA)'),  # http://pba.org/
+        (r'video\.alaskapublic\.org', 'Alaska Public Media (KAKM)'),  # http://alaskapublic.org/kakm
+        # (r'kuac\.org', 'KUAC (KUAC)'),  # http://kuac.org/kuac-tv/
+        # (r'ktoo\.org', '360 North (KTOO)'),  # http://www.ktoo.org/
+        # (r'azpm\.org', 'KUAT 6 (KUAT)'),  # http://www.azpm.org/
+        (r'video\.azpbs\.org', 'Arizona PBS (KAET)'),  # http://www.azpbs.org
+        (r'portal\.knme\.org', 'KNME-TV/Channel 5 (KNME)'),  # http://www.newmexicopbs.org/
+        (r'video\.vegaspbs\.org', 'Vegas PBS (KLVX)'),  # http://vegaspbs.org/
+        (r'watch\.aetn\.org', 'AETN/ARKANSAS ETV NETWORK (KETS)'),  # http://www.aetn.org/
+        (r'video\.ket\.org', 'KET (WKLE)'),  # http://www.ket.org/
+        (r'video\.wkno\.org', 'WKNO/Channel 10 (WKNO)'),  # http://www.wkno.org/
+        (r'video\.lpb\.org', 'LPB/LOUISIANA PUBLIC BROADCASTING (WLPB)'),  # http://www.lpb.org/
+        (r'videos\.oeta\.tv', 'OETA (KETA)'),  # http://www.oeta.tv
+        (r'video\.optv\.org', 'Ozarks Public Television (KOZK)'),  # http://www.optv.org/
+        (r'watch\.wsiu\.org', 'WSIU Public Broadcasting (WSIU)'),  # http://www.wsiu.org/
+        (r'video\.keet\.org', 'KEET TV (KEET)'),  # http://www.keet.org
+        (r'pbs\.kixe\.org', 'KIXE/Channel 9 (KIXE)'),  # http://kixe.org/
+        (r'video\.kpbs\.org', 'KPBS San Diego (KPBS)'),  # http://www.kpbs.org/
+        (r'video\.kqed\.org', 'KQED (KQED)'),  # http://www.kqed.org
+        (r'vids\.kvie\.org', 'KVIE Public Television (KVIE)'),  # http://www.kvie.org
+        (r'video\.pbssocal\.org', 'PBS SoCal/KOCE (KOCE)'),  # http://www.pbssocal.org/
+        (r'video\.valleypbs\.org', 'ValleyPBS (KVPT)'),  # http://www.valleypbs.org/
+        (r'video\.cptv\.org', 'CONNECTICUT PUBLIC TELEVISION (WEDH)'),  # http://cptv.org
+        (r'watch\.knpb\.org', 'KNPB Channel 5 (KNPB)'),  # http://www.knpb.org/
+        (r'video\.soptv\.org', 'SOPTV (KSYS)'),  # http://www.soptv.org
+        # (r'klcs\.org', 'KLCS/Channel 58 (KLCS)'),  # http://www.klcs.org
+        # (r'krcb\.org', 'KRCB Television & Radio (KRCB)'),  # http://www.krcb.org
+        # (r'kvcr\.org', 'KVCR TV/DT/FM :: Vision for the Future (KVCR)'),  # http://kvcr.org
+        (r'video\.rmpbs\.org', 'Rocky Mountain PBS (KRMA)'),  # http://www.rmpbs.org
+        (r'video\.kenw\.org', 'KENW-TV3 (KENW)'),  # http://www.kenw.org
+        (r'video\.kued\.org', 'KUED Channel 7 (KUED)'),  # http://www.kued.org
+        (r'video\.wyomingpbs\.org', 'Wyoming PBS (KCWC)'),  # http://www.wyomingpbs.org
+        (r'video\.cpt12\.org', 'Colorado Public Television / KBDI 12 (KBDI)'),  # http://www.cpt12.org/
+        (r'video\.kbyueleven\.org', 'KBYU-TV (KBYU)'),  # http://www.kbyutv.org/
+        (r'video\.thirteen\.org', 'Thirteen/WNET New York (WNET)'),  # http://www.thirteen.org
+        (r'video\.wgbh\.org', 'WGBH/Channel 2 (WGBH)'),  # http://wgbh.org
+        (r'video\.wgby\.org', 'WGBY (WGBY)'),  # http://www.wgby.org
+        (r'watch\.njtvonline\.org', 'NJTV Public Media NJ (WNJT)'),  # http://www.njtvonline.org/
+        # (r'ripbs\.org', 'Rhode Island PBS (WSBE)'),  # http://www.ripbs.org/home/
+        (r'watch\.wliw\.org', 'WLIW21 (WLIW)'),  # http://www.wliw.org/
+        (r'video\.mpt\.tv', 'mpt/Maryland Public Television (WMPB)'),  # http://www.mpt.org
+        (r'watch\.weta\.org', 'WETA Television and Radio (WETA)'),  # http://www.weta.org
+        (r'video\.whyy\.org', 'WHYY (WHYY)'),  # http://www.whyy.org
+        (r'video\.wlvt\.org', 'PBS 39 (WLVT)'),  # http://www.wlvt.org/
+        (r'video\.wvpt\.net', 'WVPT - Your Source for PBS and More! (WVPT)'),  # http://www.wvpt.net
+        (r'video\.whut\.org', 'Howard University Television (WHUT)'),  # http://www.whut.org
+        (r'video\.wedu\.org', 'WEDU PBS (WEDU)'),  # http://www.wedu.org
+        (r'video\.wgcu\.org', 'WGCU Public Media (WGCU)'),  # http://www.wgcu.org/
+        # (r'wjct\.org', 'WJCT Public Broadcasting (WJCT)'),  # http://www.wjct.org
+        (r'video\.wpbt2\.org', 'WPBT2 (WPBT)'),  # http://www.wpbt2.org
+        (r'video\.wucftv\.org', 'WUCF TV (WUCF)'),  # http://wucftv.org
+        (r'video\.wuft\.org', 'WUFT/Channel 5 (WUFT)'),  # http://www.wuft.org
+        (r'watch\.wxel\.org', 'WXEL/Channel 42 (WXEL)'),  # http://www.wxel.org/home/
+        (r'video\.wlrn\.org', 'WLRN/Channel 17 (WLRN)'),  # http://www.wlrn.org/
+        (r'video\.wusf\.usf\.edu', 'WUSF Public Broadcasting (WUSF)'),  # http://wusf.org/
+        (r'video\.scetv\.org', 'ETV (WRLK)'),  # http://www.scetv.org
+        (r'video\.unctv\.org', 'UNC-TV (WUNC)'),  # http://www.unctv.org/
+        # (r'pbsguam\.org', 'PBS Guam (KGTF)'),  # http://www.pbsguam.org/
+        (r'video\.pbshawaii\.org', 'PBS Hawaii - Oceanic Cable Channel 10 (KHET)'),  # http://www.pbshawaii.org/
+        (r'video\.idahoptv\.org', 'Idaho Public Television (KAID)'),  # http://idahoptv.org
+        (r'video\.ksps\.org', 'KSPS (KSPS)'),  # http://www.ksps.org/home/
+        (r'watch\.opb\.org', 'OPB (KOPB)'),  # http://www.opb.org
+        (r'watch\.nwptv\.org', 'KWSU/Channel 10 & KTNW/Channel 31 (KWSU)'),  # http://www.kwsu.org
+        (r'video\.will\.illinois\.edu', 'WILL-TV (WILL)'),  # http://will.illinois.edu/
+        (r'video\.networkknowledge\.tv', 'Network Knowledge - WSEC/Springfield (WSEC)'),  # http://www.wsec.tv
+        (r'video\.wttw\.com', 'WTTW11 (WTTW)'),  # http://www.wttw.com/
+        # (r'wtvp\.org', 'WTVP & WTVP.org, Public Media for Central Illinois (WTVP)'),  # http://www.wtvp.org/
+        (r'video\.iptv\.org', 'Iowa Public Television/IPTV (KDIN)'),  # http://www.iptv.org/
+        (r'video\.ninenet\.org', 'Nine Network (KETC)'),  # http://www.ninenet.org
+        (r'video\.wfwa\.org', 'PBS39 Fort Wayne (WFWA)'),  # http://wfwa.org/
+        (r'video\.wfyi\.org', 'WFYI Indianapolis (WFYI)'),  # http://www.wfyi.org
+        (r'video\.mptv\.org', 'Milwaukee Public Television (WMVS)'),  # http://www.mptv.org
+        (r'video\.wnin\.org', 'WNIN (WNIN)'),  # http://www.wnin.org/
+        (r'video\.wnit\.org', 'WNIT Public Television (WNIT)'),  # http://www.wnit.org/
+        (r'video\.wpt\.org', 'WPT (WPNE)'),  # http://www.wpt.org/
+        (r'video\.wvut\.org', 'WVUT/Channel 22 (WVUT)'),  # http://wvut.org/
+        (r'video\.weiu\.net', 'WEIU/Channel 51 (WEIU)'),  # http://www.weiu.net
+        (r'video\.wqpt\.org', 'WQPT-TV (WQPT)'),  # http://www.wqpt.org
+        (r'video\.wycc\.org', 'WYCC PBS Chicago (WYCC)'),  # http://www.wycc.org
+        # (r'lakeshorepublicmedia\.org', 'Lakeshore Public Television (WYIN)'),  # http://lakeshorepublicmedia.org/
+        (r'video\.wipb\.org', 'WIPB-TV (WIPB)'),  # http://wipb.org
+        (r'video\.indianapublicmedia\.org', 'WTIU (WTIU)'),  # http://indianapublicmedia.org/tv/
+        (r'watch\.cetconnect\.org', 'CET  (WCET)'),  # http://www.cetconnect.org
+        (r'video\.thinktv\.org', 'ThinkTVNetwork (WPTD)'),  # http://www.thinktv.org
+        (r'video\.wbgu\.org', 'WBGU-TV (WBGU)'),  # http://wbgu.org
+        (r'video\.wgvu\.org', 'WGVU TV (WGVU)'),  # http://www.wgvu.org/
+        (r'video\.netnebraska\.org', 'NET1 (KUON)'),  # http://netnebraska.org
+        (r'video\.pioneer\.org', 'Pioneer Public Television (KWCM)'),  # http://www.pioneer.org
+        (r'watch\.sdpb\.org', 'SDPB Television (KUSD)'),  # http://www.sdpb.org
+        (r'video\.tpt\.org', 'TPT (KTCA)'),  # http://www.tpt.org
+        (r'watch\.ksmq\.org', 'KSMQ (KSMQ)'),  # http://www.ksmq.org/
+        (r'watch\.kpts\.org', 'KPTS/Channel 8 (KPTS)'),  # http://www.kpts.org/
+        (r'watch\.ktwu\.org', 'KTWU/Channel 11 (KTWU)'),  # http://ktwu.org
+        # (r'shptv\.org', 'Smoky Hills Public Television (KOOD)'),  # http://www.shptv.org
+        # (r'kcpt\.org', 'KCPT Kansas City Public Television (KCPT)'),  # http://kcpt.org/
+        # (r'blueridgepbs\.org', 'Blue Ridge PBS (WBRA)'),  # http://www.blueridgepbs.org/
+        (r'watch\.easttennesseepbs\.org', 'East Tennessee PBS (WSJK)'),  # http://easttennesseepbs.org
+        (r'video\.wcte\.tv', 'WCTE-TV (WCTE)'),  # http://www.wcte.org
+        (r'video\.wljt\.org', 'WLJT, Channel 11 (WLJT)'),  # http://wljt.org/
+        (r'video\.wosu\.org', 'WOSU TV (WOSU)'),  # http://wosu.org/
+        (r'video\.woub\.org', 'WOUB/WOUC (WOUB)'),  # http://woub.org/tv/index.php?section=5
+        (r'video\.wvpublic\.org', 'WVPB (WVPB)'),  # http://wvpublic.org/
+        (r'video\.wkyupbs\.org', 'WKYU-PBS (WKYU)'),  # http://www.wkyupbs.org
+        # (r'wyes\.org', 'WYES-TV/New Orleans (WYES)'),  # http://www.wyes.org
+        (r'video\.kera\.org', 'KERA 13 (KERA)'),  # http://www.kera.org/
+        (r'video\.mpbn\.net', 'MPBN (WCBB)'),  # http://www.mpbn.net/
+        (r'video\.mountainlake\.org', 'Mountain Lake PBS (WCFE)'),  # http://www.mountainlake.org/
+        (r'video\.nhptv\.org', 'NHPTV (WENH)'),  # http://nhptv.org/
+        (r'video\.vpt\.org', 'Vermont PBS (WETK)'),  # http://www.vpt.org
+        (r'video\.witf\.org', 'witf (WITF)'),  # http://www.witf.org
+        (r'watch\.wqed\.org', 'WQED Multimedia (WQED)'),  # http://www.wqed.org/
+        (r'video\.wmht\.org', 'WMHT Educational Telecommunications (WMHT)'),  # http://www.wmht.org/home/
+        (r'video\.deltabroadcasting\.org', 'Q-TV (WDCQ)'),  # http://www.deltabroadcasting.org
+        (r'video\.dptv\.org', 'WTVS Detroit Public TV (WTVS)'),  # http://www.dptv.org/
+        (r'video\.wcmu\.org', 'CMU Public Television (WCMU)'),  # http://www.wcmu.org
+        (r'video\.wkar\.org', 'WKAR-TV (WKAR)'),  # http://wkar.org/
+        (r'wnmuvideo\.nmu\.edu', 'WNMU-TV Public TV 13 (WNMU)'),  # http://wnmutv.nmu.edu
+        (r'video\.wdse\.org', 'WDSE - WRPT (WDSE)'),  # http://www.wdse.org/
+        (r'video\.wgte\.org', 'WGTE TV (WGTE)'),  # http://www.wgte.org
+        (r'video\.lptv\.org', 'Lakeland Public Television (KAWE)'),  # http://www.lakelandptv.org
+        # (r'prairiepublic\.org', 'PRAIRIE PUBLIC (KFME)'),  # http://www.prairiepublic.org/
+        (r'video\.kmos\.org', 'KMOS-TV - Channels 6.1, 6.2 and 6.3 (KMOS)'),  # http://www.kmos.org/
+        (r'watch\.montanapbs\.org', 'MontanaPBS (KUSM)'),  # http://montanapbs.org
+        (r'video\.krwg\.org', 'KRWG/Channel 22 (KRWG)'),  # http://www.krwg.org
+        (r'video\.kacvtv\.org', 'KACV (KACV)'),  # http://www.panhandlepbs.org/home/
+        (r'video\.kcostv\.org', 'KCOS/Channel 13 (KCOS)'),  # www.kcostv.org
+        (r'video\.wcny\.org', 'WCNY/Channel 24 (WCNY)'),  # http://www.wcny.org
+        (r'video\.wned\.org', 'WNED (WNED)'),  # http://www.wned.org/
+        (r'watch\.wpbstv\.org', 'WPBS (WPBS)'),  # http://www.wpbstv.org
+        (r'video\.wskg\.org', 'WSKG Public TV (WSKG)'),  # http://wskg.org
+        (r'video\.wxxi\.org', 'WXXI (WXXI)'),  # http://wxxi.org
+        (r'video\.wpsu\.org', 'WPSU (WPSU)'),  # http://www.wpsu.org
+        # (r'wqln\.org', 'WQLN/Channel 54 (WQLN)'),  # http://www.wqln.org
+        (r'on-demand\.wvia\.org', 'WVIA Public Media Studios (WVIA)'),  # http://www.wvia.org/
+        (r'video\.wtvi\.org', 'WTVI (WTVI)'),  # http://www.wtvi.org/
+        # (r'whro\.org', 'WHRO (WHRO)'),  # http://whro.org
+        (r'video\.westernreservepublicmedia\.org', 'Western Reserve PBS (WNEO)'),  # http://www.WesternReservePublicMedia.org/
+        (r'video\.ideastream\.org', 'WVIZ/PBS ideastream (WVIZ)'),  # http://www.wviz.org/
+        (r'video\.kcts9\.org', 'KCTS 9 (KCTS)'),  # http://kcts9.org/
+        (r'video\.basinpbs\.org', 'Basin PBS (KPBT)'),  # http://www.basinpbs.org
+        (r'video\.houstonpbs\.org', 'KUHT / Channel 8 (KUHT)'),  # http://www.houstonpublicmedia.org/
+        # (r'tamu\.edu', 'KAMU - TV (KAMU)'),  # http://KAMU.tamu.edu
+        # (r'kedt\.org', 'KEDT/Channel 16 (KEDT)'),  # http://www.kedt.org
+        (r'video\.klrn\.org', 'KLRN (KLRN)'),  # http://www.klrn.org
+        (r'video\.klru\.tv', 'KLRU (KLRU)'),  # http://www.klru.org
+        # (r'kmbh\.org', 'KMBH-TV (KMBH)'),  # http://www.kmbh.org
+        # (r'knct\.org', 'KNCT (KNCT)'),  # http://www.knct.org
+        # (r'ktxt\.org', 'KTTZ-TV (KTXT)'),  # http://www.ktxt.org
+        (r'video\.wtjx\.org', 'WTJX Channel 12 (WTJX)'),  # http://www.wtjx.org/
+        (r'video\.ideastations\.org', 'WCVE PBS (WCVE)'),  # http://ideastations.org/
+        (r'video\.kbtc\.org', 'KBTC Public Television (KBTC)'),  # http://kbtc.org
     )
 
     IE_NAME = 'pbs'
@@ -189,7 +189,7 @@ class PBSIE(InfoExtractor):
            # Player
            (?:video|player)\.pbs\.org/(?:widget/)?partnerplayer/(?P<player_id>[^/]+)/
         )
-    ''' % '|'.join(re.escape(p) for p in list(zip(*_STATIONS))[0])
+    ''' % '|'.join(list(zip(*_STATIONS))[0])
 
     _TESTS = [
         {

From 2469a6aecbc819b87915d31dd2fb5940d84d11d2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 17 Dec 2015 22:16:22 +0600
Subject: [PATCH 3383/6219] [noco] Adjust timestamp according to server time
 (Closes #7864)

---
 youtube_dl/extractor/noco.py | 27 +++++++++++++++++++++++++--
 1 file changed, 25 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/noco.py b/youtube_dl/extractor/noco.py
index 76bd21e6d..63b97f170 100644
--- a/youtube_dl/extractor/noco.py
+++ b/youtube_dl/extractor/noco.py
@@ -9,6 +9,7 @@ from .common import InfoExtractor
 from ..compat import (
     compat_str,
     compat_urllib_parse,
+    compat_urlparse,
 )
 from ..utils import (
     clean_html,
@@ -82,14 +83,21 @@ class NocoIE(InfoExtractor):
         if 'erreur' in login:
             raise ExtractorError('Unable to login: %s' % clean_html(login['erreur']), expected=True)
 
+    @staticmethod
+    def _ts():
+        return int(time.time() * 1000)
+
     def _call_api(self, path, video_id, note, sub_lang=None):
-        ts = compat_str(int(time.time() * 1000))
+        ts = compat_str(self._ts() + self._ts_offset)
         tk = hashlib.md5((hashlib.md5(ts.encode('ascii')).hexdigest() + '#8S?uCraTedap6a').encode('ascii')).hexdigest()
         url = self._API_URL_TEMPLATE % (path, ts, tk)
         if sub_lang:
             url += self._SUB_LANG_TEMPLATE % sub_lang
 
-        resp = self._download_json(url, video_id, note)
+        request = sanitized_Request(url)
+        request.add_header('Referer', self._referer)
+
+        resp = self._download_json(request, video_id, note)
 
         if isinstance(resp, dict) and resp.get('error'):
             self._raise_error(resp['error'], resp['description'])
@@ -105,6 +113,21 @@ class NocoIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
 
+        # Timestamp adjustment offset between server time and local time
+        # must be calculated in order to use timestamps closest to server's
+        # in all API requests (see https://github.com/rg3/youtube-dl/issues/7864)
+        webpage = self._download_webpage(url, video_id)
+
+        player_url = self._search_regex(
+            r'(["\'])(?P<player>https?://noco\.tv/(?:[^/]+/)+NocoPlayer.+?\.swf.*?)\1',
+            webpage, 'noco player', group='player',
+            default='http://noco.tv/cdata/js/player/NocoPlayer-v1.2.40.swf')
+
+        qs = compat_urlparse.parse_qs(compat_urlparse.urlparse(player_url).query)
+        ts = int_or_none(qs.get('ts', [None])[0])
+        self._ts_offset = ts - self._ts() if ts else 0
+        self._referer = player_url
+
         medias = self._call_api(
             'shows/%s/medias' % video_id,
             video_id, 'Downloading video JSON')

From 7824e1f6a61e97c3c79c21db65de33a6380e29c3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 17 Dec 2015 22:16:58 +0600
Subject: [PATCH 3384/6219] [noco] Modernize

---
 youtube_dl/extractor/noco.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/noco.py b/youtube_dl/extractor/noco.py
index 63b97f170..daba2e12f 100644
--- a/youtube_dl/extractor/noco.py
+++ b/youtube_dl/extractor/noco.py
@@ -110,8 +110,7 @@ class NocoIE(InfoExtractor):
             expected=True)
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         # Timestamp adjustment offset between server time and local time
         # must be calculated in order to use timestamps closest to server's

From 9dc1d94a0ccf39c4dab686cbd5941d75208248fb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 17 Dec 2015 22:18:28 +0600
Subject: [PATCH 3385/6219] [noco] Fix bitrates

---
 youtube_dl/extractor/noco.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/noco.py b/youtube_dl/extractor/noco.py
index daba2e12f..d440313d5 100644
--- a/youtube_dl/extractor/noco.py
+++ b/youtube_dl/extractor/noco.py
@@ -177,8 +177,8 @@ class NocoIE(InfoExtractor):
                         'format_id': format_id_extended,
                         'width': int_or_none(fmt.get('res_width')),
                         'height': int_or_none(fmt.get('res_lines')),
-                        'abr': int_or_none(fmt.get('audiobitrate')),
-                        'vbr': int_or_none(fmt.get('videobitrate')),
+                        'abr': int_or_none(fmt.get('audiobitrate'), 1000),
+                        'vbr': int_or_none(fmt.get('videobitrate'), 1000),
                         'filesize': int_or_none(fmt.get('filesize')),
                         'format_note': qualities[format_id].get('quality_name'),
                         'quality': qualities[format_id].get('priority'),

From 64ccbf18c0e4849618d82add2ff13879f9db67ab Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 17 Dec 2015 20:56:54 +0100
Subject: [PATCH 3386/6219] [livestream] improve extraction, add support for
 live streams and extract more info and formats

---
 youtube_dl/extractor/livestream.py | 276 +++++++++++++++++------------
 1 file changed, 167 insertions(+), 109 deletions(-)

diff --git a/youtube_dl/extractor/livestream.py b/youtube_dl/extractor/livestream.py
index 6d7733e41..b95c23c8c 100644
--- a/youtube_dl/extractor/livestream.py
+++ b/youtube_dl/extractor/livestream.py
@@ -1,27 +1,29 @@
 from __future__ import unicode_literals
 
 import re
-import json
 import itertools
 
 from .common import InfoExtractor
 from ..compat import (
     compat_str,
-    compat_urllib_parse_urlparse,
     compat_urlparse,
 )
 from ..utils import (
-    ExtractorError,
     find_xpath_attr,
-    int_or_none,
-    orderedSet,
+    xpath_attr,
     xpath_with_ns,
+    xpath_text,
+    orderedSet,
+    int_or_none,
+    float_or_none,
+    parse_iso8601,
+    determine_ext,
 )
 
 
 class LivestreamIE(InfoExtractor):
     IE_NAME = 'livestream'
-    _VALID_URL = r'https?://(?:new\.)?livestream\.com/.*?/(?P<event_name>.*?)(/videos/(?P<id>[0-9]+)(?:/player)?)?/?(?:$|[?#])'
+    _VALID_URL = r'https?://(?:new\.)?livestream\.com/(?:accounts/(?P<account_id>\d+)|(?P<account_name>[^/]+))/(?:events/(?P<event_id>\d+)|(?P<event_name>[^/]+))(?:/videos/(?P<id>\d+))?'
     _TESTS = [{
         'url': 'http://new.livestream.com/CoheedandCambria/WebsterHall/videos/4719370',
         'md5': '53274c76ba7754fb0e8d072716f2292b',
@@ -29,7 +31,9 @@ class LivestreamIE(InfoExtractor):
             'id': '4719370',
             'ext': 'mp4',
             'title': 'Live from Webster Hall NYC',
+            'timestamp': 1350008072,
             'upload_date': '20121012',
+            'duration': 5968.0,
             'like_count': int,
             'view_count': int,
             'thumbnail': 're:^http://.*\.jpg$'
@@ -55,39 +59,19 @@ class LivestreamIE(InfoExtractor):
         'url': 'http://livestream.com/bsww/concacafbeachsoccercampeonato2015',
         'only_matching': True,
     }]
+    _API_URL_TEMPLATE = 'http://livestream.com/api/accounts/%s/events/%s'
+
+    def _parse_smil_formats(self, smil, smil_url, video_id, namespace=None, f4m_params=None, transform_rtmp_url=None):
+        base_ele = find_xpath_attr(smil, self._xpath_ns('.//meta', namespace), 'name', 'httpBase')
+        base = base_ele.get('content') if base_ele else 'http://livestreamvod-f.akamaihd.net/'
 
-    def _parse_smil(self, video_id, smil_url):
         formats = []
-        _SWITCH_XPATH = (
-            './/{http://www.w3.org/2001/SMIL20/Language}body/'
-            '{http://www.w3.org/2001/SMIL20/Language}switch')
-        smil_doc = self._download_xml(
-            smil_url, video_id,
-            note='Downloading SMIL information',
-            errnote='Unable to download SMIL information',
-            fatal=False)
-        if smil_doc is False:  # Download failed
-            return formats
-        title_node = find_xpath_attr(
-            smil_doc, './/{http://www.w3.org/2001/SMIL20/Language}meta',
-            'name', 'title')
-        if title_node is None:
-            self.report_warning('Cannot find SMIL id')
-            switch_node = smil_doc.find(_SWITCH_XPATH)
-        else:
-            title_id = title_node.attrib['content']
-            switch_node = find_xpath_attr(
-                smil_doc, _SWITCH_XPATH, 'id', title_id)
-        if switch_node is None:
-            raise ExtractorError('Cannot find switch node')
-        video_nodes = switch_node.findall(
-            '{http://www.w3.org/2001/SMIL20/Language}video')
+        video_nodes = smil.findall(self._xpath_ns('.//video', namespace))
 
         for vn in video_nodes:
-            tbr = int_or_none(vn.attrib.get('system-bitrate'))
+            tbr = int_or_none(vn.attrib.get('system-bitrate'), 1000)
             furl = (
-                'http://livestream-f.akamaihd.net/%s?v=3.0.3&fp=WIN%%2014,0,0,145' %
-                (vn.attrib['src']))
+                '%s%s?v=3.0.3&fp=WIN%%2014,0,0,145' % (base, vn.attrib['src']))
             if 'clipBegin' in vn.attrib:
                 furl += '&ssek=' + vn.attrib['clipBegin']
             formats.append({
@@ -106,89 +90,136 @@ class LivestreamIE(InfoExtractor):
             ('sd', 'progressive_url'),
             ('hd', 'progressive_url_hd'),
         )
-        formats = [{
-            'format_id': format_id,
-            'url': video_data[key],
-            'quality': i + 1,
-        } for i, (format_id, key) in enumerate(FORMAT_KEYS)
-            if video_data.get(key)]
+
+        formats = []
+        for format_id, key in FORMAT_KEYS:
+            video_url = video_data.get(key)
+            if video_url:
+                ext = determine_ext(video_url)
+                bitrate = int_or_none(self._search_regex(r'(\d+)\.%s' % ext, video_url, 'bitrate', default=None))
+                formats.append({
+                    'url': video_url,
+                    'format_id': format_id,
+                    'tbr': bitrate,
+                    'ext': ext,
+                })
 
         smil_url = video_data.get('smil_url')
         if smil_url:
-            formats.extend(self._parse_smil(video_id, smil_url))
+            smil_formats = self._extract_smil_formats(smil_url, video_id)
+            if smil_formats:
+                formats.extend(smil_formats)
+
+        m3u8_url = video_data.get('m3u8_url')
+        if m3u8_url:
+            m3u8_formats = self._extract_m3u8_formats(
+                m3u8_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False)
+            if m3u8_formats:
+                formats.extend(m3u8_formats)
+
+        f4m_url = video_data.get('f4m_url')
+        if f4m_url:
+            f4m_formats = self._extract_f4m_formats(f4m_url, video_id, f4m_id='hds', fatal=False)
+            if f4m_formats:
+                formats.extend(f4m_formats)
         self._sort_formats(formats)
 
+        comments = [{
+            'author_id': comment.get('author_id'),
+            'author': comment.get('author', {}).get('full_name'),
+            'id': comment.get('id'),
+            'text': comment['text'],
+            'timestamp': parse_iso8601(comment.get('created_at')),
+        } for comment in video_data.get('comments', {}).get('data', [])]
+
         return {
             'id': video_id,
             'formats': formats,
             'title': video_data['caption'],
+            'description': video_data.get('description'),
             'thumbnail': video_data.get('thumbnail_url'),
-            'upload_date': video_data['updated_at'].replace('-', '')[:8],
+            'duration': float_or_none(video_data.get('duration'), 1000),
+            'timestamp': parse_iso8601(video_data.get('publish_at')),
             'like_count': video_data.get('likes', {}).get('total'),
+            'comment_count': video_data.get('comments', {}).get('total'),
             'view_count': video_data.get('views'),
+            'comments': comments,
         }
 
-    def _extract_event(self, info):
-        event_id = compat_str(info['id'])
-        account = compat_str(info['owner_account_id'])
-        root_url = (
-            'https://new.livestream.com/api/accounts/{account}/events/{event}/'
-            'feed.json'.format(account=account, event=event_id))
+    def _extract_stream_info(self, stream_info):
+        broadcast_id = stream_info['broadcast_id']
+        is_live = stream_info.get('is_live')
 
-        def _extract_videos():
-            last_video = None
-            for i in itertools.count(1):
-                if last_video is None:
-                    info_url = root_url
-                else:
-                    info_url = '{root}?&id={id}&newer=-1&type=video'.format(
-                        root=root_url, id=last_video)
-                videos_info = self._download_json(info_url, event_id, 'Downloading page {0}'.format(i))['data']
-                videos_info = [v['data'] for v in videos_info if v['type'] == 'video']
-                if not videos_info:
-                    break
-                for v in videos_info:
-                    yield self._extract_video_info(v)
-                last_video = videos_info[-1]['id']
-        return self.playlist_result(_extract_videos(), event_id, info['full_name'])
+        formats = []
+        smil_url = stream_info.get('play_url')
+        if smil_url:
+            smil_formats = self._extract_smil_formats(smil_url, broadcast_id)
+            if smil_formats:
+                formats.extend(smil_formats)
+
+        m3u8_url = stream_info.get('m3u8_url')
+        if m3u8_url:
+            m3u8_formats = self._extract_m3u8_formats(
+                m3u8_url, broadcast_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False)
+            if m3u8_formats:
+                formats.extend(m3u8_formats)
+
+        rtsp_url = stream_info.get('rtsp_url')
+        if rtsp_url:
+            formats.append({
+                'url': rtsp_url,
+                'format_id': 'rtsp',
+            })
+        self._sort_formats(formats)
+
+        return {
+            'id': broadcast_id,
+            'formats': formats,
+            'title': self._live_title(stream_info['stream_title']) if is_live else stream_info['stream_title'],
+            'thumbnail': stream_info.get('thumbnail_url'),
+            'is_live': is_live,
+        }
+
+    def _extract_event(self, event_data):
+        event_id = compat_str(event_data['id'])
+        account_id = compat_str(event_data['owner_account_id'])
+        feed_root_url = self._API_URL_TEMPLATE % (account_id, event_id) + '/feed.json'
+
+        stream_info = event_data.get('stream_info')
+        if stream_info:
+            return self._extract_stream_info(stream_info)
+
+        last_video = None
+        entries = []
+        for i in itertools.count(1):
+            if last_video is None:
+                info_url = feed_root_url
+            else:
+                info_url = '{root}?&id={id}&newer=-1&type=video'.format(
+                    root=feed_root_url, id=last_video)
+            videos_info = self._download_json(info_url, event_id, 'Downloading page {0}'.format(i))['data']
+            videos_info = [v['data'] for v in videos_info if v['type'] == 'video']
+            if not videos_info:
+                break
+            for v in videos_info:
+                entries.append(self.url_result(
+                    'http://livestream.com/accounts/%s/events/%s/videos/%s' % (account_id, event_id, v['id']),
+                    'Livestream', v['id'], v['caption']))
+            last_video = videos_info[-1]['id']
+        return self.playlist_result(entries, event_id, event_data['full_name'])
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
-        event_name = mobj.group('event_name')
-        webpage = self._download_webpage(url, video_id or event_name)
-
-        og_video = self._og_search_video_url(
-            webpage, 'player url', fatal=False, default=None)
-        if og_video is not None:
-            query_str = compat_urllib_parse_urlparse(og_video).query
-            query = compat_urlparse.parse_qs(query_str)
-            if 'play_url' in query:
-                api_url = query['play_url'][0].replace('.smil', '')
-                info = json.loads(self._download_webpage(
-                    api_url, video_id, 'Downloading video info'))
-                return self._extract_video_info(info)
-
-        config_json = self._search_regex(
-            r'window.config = ({.*?});', webpage, 'window config')
-        info = json.loads(config_json)['event']
-
-        def is_relevant(vdata, vid):
-            result = vdata['type'] == 'video'
-            if video_id is not None:
-                result = result and compat_str(vdata['data']['id']) == vid
-            return result
-
-        if video_id is None:
-            # This is an event page:
-            return self._extract_event(info)
+        event = mobj.group('event_id') or mobj.group('event_name')
+        account = mobj.group('account_id') or mobj.group('account_name')
+        api_url = self._API_URL_TEMPLATE % (account, event)
+        if video_id:
+            video_data = self._download_json(api_url + '/videos/%s' % video_id, video_id)
+            return self._extract_video_info(video_data)
         else:
-            videos = [self._extract_video_info(video_data['data'])
-                      for video_data in info['feed']['data']
-                      if is_relevant(video_data, video_id)]
-            if not videos:
-                raise ExtractorError('Cannot find video %s' % video_id)
-            return videos[0]
+            event_data = self._download_json(api_url, video_id)
+            return self._extract_event(event_data)
 
 
 # The original version of Livestream uses a different system
@@ -204,6 +235,8 @@ class LivestreamOriginalIE(InfoExtractor):
             'id': 'pla_8aa4a3f1-ba15-46a4-893b-902210e138fb',
             'ext': 'mp4',
             'title': 'Spark 1 (BitCoin) with Cameron Winklevoss & Tyler Winklevoss of Winklevoss Capital',
+            'duration': 771.301,
+            'view_count': int,
         },
     }, {
         'url': 'https://original.livestream.com/newplay/folder?dirId=a07bf706-d0e4-4e75-a747-b021d84f2fd3',
@@ -220,15 +253,42 @@ class LivestreamOriginalIE(InfoExtractor):
         # this url is used on mobile devices
         stream_url = 'http://x{0}x.api.channel.livestream.com/3.0/getstream.json?id={1}'.format(user, video_id)
         stream_info = self._download_json(stream_url, video_id)
+        is_live = stream_info.get('isLive')
         item = info.find('channel').find('item')
-        ns = {'media': 'http://search.yahoo.com/mrss'}
-        thumbnail_url = item.find(xpath_with_ns('media:thumbnail', ns)).attrib['url']
+        media_ns = {'media': 'http://search.yahoo.com/mrss'}
+        thumbnail_url = xpath_attr(item, xpath_with_ns('media:thumbnail', media_ns), 'url')
+        duration = float_or_none(xpath_attr(item, xpath_with_ns('media:content', media_ns), 'duration'))
+        ls_ns = {'ls': 'http://api.channel.livestream.com/2.0'}
+        view_count = int_or_none(xpath_text(item, xpath_with_ns('ls:viewsCount', ls_ns)))
+
+        formats = [{
+            'url': stream_info['progressiveUrl'],
+            'format_id': 'http',
+        }]
+
+        m3u8_url = stream_info.get('httpUrl')
+        if m3u8_url:
+            m3u8_formats = self._extract_m3u8_formats(
+                m3u8_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False)
+            if m3u8_formats:
+                formats.extend(m3u8_formats)
+
+        rtsp_url = stream_info.get('rtspUrl')
+        if rtsp_url:
+            formats.append({
+                'url': rtsp_url,
+                'format_id': 'rtsp',
+            })
+        self._sort_formats(formats)
 
         return {
             'id': video_id,
-            'title': item.find('title').text,
-            'url': stream_info['progressiveUrl'],
+            'title': self._live_title(xpath_text(item, 'title')) if is_live else xpath_text(item, 'title'),
+            'formats': formats,
             'thumbnail': thumbnail_url,
+            'duration': duration,
+            'view_count': view_count,
+            'is_live': is_live,
         }
 
     def _extract_folder(self, url, folder_id):
@@ -239,14 +299,12 @@ class LivestreamOriginalIE(InfoExtractor):
                 <a\s+href="(?=https?://livestre\.am/)
             )([^"]+)"''', webpage))
 
-        return {
-            '_type': 'playlist',
-            'id': folder_id,
-            'entries': [{
-                '_type': 'url',
-                'url': compat_urlparse.urljoin(url, p),
-            } for p in paths],
-        }
+        entries = [{
+            '_type': 'url',
+            'url': compat_urlparse.urljoin(url, p),
+        } for p in paths]
+
+        return self.playlist_result(entries, folder_id)
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)

From 45dad8bab98e30eb3fb1cf0f20770c1dce74610b Mon Sep 17 00:00:00 2001
From: Gautam M <Blue9@users.noreply.github.com>
Date: Fri, 18 Dec 2015 03:16:36 -0500
Subject: [PATCH 3387/6219] Fix hyperlink to youtube-dl options

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 5c3915d64..7002f45e0 100644
--- a/README.md
+++ b/README.md
@@ -757,7 +757,7 @@ with youtube_dl.YoutubeDL(ydl_opts) as ydl:
     ydl.download(['http://www.youtube.com/watch?v=BaW_jenozKc'])
 ```
 
-Most likely, you'll want to use various options. For a list of what can be done, have a look at [youtube_dl/YoutubeDL.py](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/YoutubeDL.py#L117-L265). For a start, if you want to intercept youtube-dl's output, set a `logger` object.
+Most likely, you'll want to use various options. For a list of what can be done, have a look at [`youtube_dl/YoutubeDL.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/YoutubeDL.py#L121-L269). For a start, if you want to intercept youtube-dl's output, set a `logger` object.
 
 Here's a more complete example of a program that outputs only errors (and a short message after the download is finished), and downloads/converts the video to an mp3 file:
 

From 10171468d94e68f92505a77938be11d266ccabd8 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 18 Dec 2015 18:20:41 +0800
Subject: [PATCH 3388/6219] [iqiyi] Update key (closes #7896)

---
 youtube_dl/extractor/iqiyi.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index f96e12e69..c3731a110 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -205,8 +205,8 @@ class IqiyiIE(InfoExtractor):
 
     def get_enc_key(self, swf_url, video_id):
         # TODO: automatic key extraction
-        # last update at 2015-12-06 for Zombie::bite
-        enc_key = '3719f6a1da83ee0aee3488d8802d7696'[::-1]
+        # last update at 2015-12-18 for Zombie::bite
+        enc_key = '8b6b683780897eb8d9a48a02ccc4817d'[::-1]
         return enc_key
 
     def _real_extract(self, url):

From b95779be21929c1aed1c0edbc3f9a5d11826973f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 18 Dec 2015 18:57:49 +0600
Subject: [PATCH 3389/6219] [jsinterp] Extend function regex (Closes #7900,
 closes #7901)

---
 youtube_dl/jsinterp.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index 2191e8b89..a7440c582 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -232,7 +232,7 @@ class JSInterpreter(object):
     def extract_function(self, funcname):
         func_m = re.search(
             r'''(?x)
-                (?:function\s+%s|[{;]%s\s*=\s*function|var\s+%s\s*=\s*function)\s*
+                (?:function\s+%s|[{;,]%s\s*=\s*function|var\s+%s\s*=\s*function)\s*
                 \((?P<args>[^)]*)\)\s*
                 \{(?P<code>[^}]+)\}''' % (
                 re.escape(funcname), re.escape(funcname), re.escape(funcname)),

From 016dd820505e635ed1d4e5890f370cf800b25c7f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 18 Dec 2015 14:21:30 +0100
Subject: [PATCH 3390/6219] release 2015.12.18

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 20b44b94d..01607693e 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.12.13'
+__version__ = '2015.12.18'

From 9796a9b20ccc46217c27523774f5088d74e672bb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 18 Dec 2015 21:34:17 +0600
Subject: [PATCH 3391/6219] [ndr] Fix description and upload date extraction
 (Closes #7893)

---
 youtube_dl/extractor/ndr.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/ndr.py b/youtube_dl/extractor/ndr.py
index 16213eed9..894c51399 100644
--- a/youtube_dl/extractor/ndr.py
+++ b/youtube_dl/extractor/ndr.py
@@ -88,10 +88,10 @@ class NDRIE(NDRBaseIE):
             'embedURL', webpage, 'embed URL', fatal=True)
         description = self._search_regex(
             r'<p[^>]+itemprop="description">([^<]+)</p>',
-            webpage, 'description', fatal=False)
+            webpage, 'description', default=None) or self._og_search_description(webpage)
         timestamp = parse_iso8601(
             self._search_regex(
-                r'<span itemprop="datePublished" content="([^"]+)">',
+                r'<span[^>]+itemprop="(?:datePublished|uploadDate)"[^>]+content="([^"]+)"',
                 webpage, 'upload date', fatal=False))
         return {
             '_type': 'url_transparent',

From 7234d1d9c795bd43799de47952f65f72952e0564 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 18 Dec 2015 22:05:32 +0600
Subject: [PATCH 3392/6219] [brightcove:new] Add _extract_url

---
 youtube_dl/extractor/brightcove.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index f5ebae1e6..66b8d2dff 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -389,6 +389,11 @@ class BrightcoveNewIE(InfoExtractor):
         }
     }]
 
+    @staticmethod
+    def _extract_url(webpage):
+        urls = BrightcoveNewIE._extract_urls(webpage)
+        return urls[0] if urls else None
+
     @staticmethod
     def _extract_urls(webpage):
         # Reference:

From 15d50aca64421c97d251bcd86e9bb308265dfdaa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 18 Dec 2015 22:05:56 +0600
Subject: [PATCH 3393/6219] [nowness] Add support for brightcove:new videos
 (Closes #7884)

---
 youtube_dl/extractor/nowness.py | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/nowness.py b/youtube_dl/extractor/nowness.py
index d480fb58c..446f5901c 100644
--- a/youtube_dl/extractor/nowness.py
+++ b/youtube_dl/extractor/nowness.py
@@ -1,7 +1,10 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
-from .brightcove import BrightcoveLegacyIE
+from .brightcove import (
+    BrightcoveLegacyIE,
+    BrightcoveNewIE,
+)
 from .common import InfoExtractor
 from ..compat import compat_str
 from ..utils import (
@@ -23,9 +26,12 @@ class NownessBaseIE(InfoExtractor):
                             note='Downloading player JavaScript',
                             errnote='Unable to download player JavaScript')
                         bc_url = BrightcoveLegacyIE._extract_brightcove_url(player_code)
-                        if bc_url is None:
-                            raise ExtractorError('Could not find player definition')
-                        return self.url_result(bc_url, 'BrightcoveLegacy')
+                        if bc_url:
+                            return self.url_result(bc_url, BrightcoveLegacyIE.ie_key())
+                        bc_url = BrightcoveNewIE._extract_url(player_code)
+                        if bc_url:
+                            return self.url_result(bc_url, BrightcoveNewIE.ie_key())
+                        raise ExtractorError('Could not find player definition')
                     elif source == 'vimeo':
                         return self.url_result('http://vimeo.com/%s' % video_id, 'Vimeo')
                     elif source == 'youtube':

From 9fd0f676788f94768b80fe5ff87895b607c84345 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 18 Dec 2015 22:18:55 +0600
Subject: [PATCH 3394/6219] [brightcove:new] Add support for ref: preffixed
 video ids (Closes #7794)

---
 youtube_dl/extractor/brightcove.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 66b8d2dff..d77590195 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -355,7 +355,7 @@ class BrightcoveLegacyIE(InfoExtractor):
 
 class BrightcoveNewIE(InfoExtractor):
     IE_NAME = 'brightcove:new'
-    _VALID_URL = r'https?://players\.brightcove\.net/(?P<account_id>\d+)/(?P<player_id>[^/]+)_(?P<embed>[^/]+)/index\.html\?.*videoId=(?P<video_id>\d+)'
+    _VALID_URL = r'https?://players\.brightcove\.net/(?P<account_id>\d+)/(?P<player_id>[^/]+)_(?P<embed>[^/]+)/index\.html\?.*videoId=(?P<video_id>(?:ref:)?\d+)'
     _TESTS = [{
         'url': 'http://players.brightcove.net/929656772001/e41d32dc-ec74-459e-a845-6c69f7b724ea_default/index.html?videoId=4463358922001',
         'md5': 'c8100925723840d4b0d243f7025703be',
@@ -414,7 +414,7 @@ class BrightcoveNewIE(InfoExtractor):
                 # may be optional and what to do when it is
                 r'''(?sx)
                     <video[^>]+
-                        data-video-id=["\'](\d+)["\'][^>]*>.*?
+                        data-video-id=["\']((?:ref:)?\d+)["\'][^>]*>.*?
                     </video>.*?
                     <script[^>]+
                         src=["\'](?:https?:)?//players\.brightcove\.net/

From f81ccbb3df205c612275f8881ddf21115eb590b4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 18 Dec 2015 22:20:44 +0600
Subject: [PATCH 3395/6219] [brightcove:new] Fix typo

---
 youtube_dl/extractor/brightcove.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index d77590195..b0536ddbd 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -398,7 +398,7 @@ class BrightcoveNewIE(InfoExtractor):
     def _extract_urls(webpage):
         # Reference:
         # 1. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/publish-video.html#setvideoiniframe
-        # 2. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/publish-video.html#setvideousingjavascript)
+        # 2. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/publish-video.html#setvideousingjavascript
         # 3. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/embed-in-page.html
 
         entries = []

From 5b72fda140488685be55215d584f764c39577b02 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 18 Dec 2015 22:22:41 +0600
Subject: [PATCH 3396/6219] [brightcove:new] Clarify ref: prefix

---
 youtube_dl/extractor/brightcove.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index b0536ddbd..4e983fa10 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -400,6 +400,7 @@ class BrightcoveNewIE(InfoExtractor):
         # 1. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/publish-video.html#setvideoiniframe
         # 2. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/publish-video.html#setvideousingjavascript
         # 3. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/embed-in-page.html
+        # 4. https://support.brightcove.com/en/video-cloud/docs/dynamically-assigning-videos-player
 
         entries = []
 
@@ -412,6 +413,7 @@ class BrightcoveNewIE(InfoExtractor):
         for video_id, account_id, player_id, embed in re.findall(
                 # According to examples from [3] it's unclear whether video id
                 # may be optional and what to do when it is
+                # According to [4] data-video-id may be prefixed with ref:
                 r'''(?sx)
                     <video[^>]+
                         data-video-id=["\']((?:ref:)?\d+)["\'][^>]*>.*?

From 4f29fa99069760dc47ef9ca5dbf607a567d2982f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 18 Dec 2015 22:31:48 +0600
Subject: [PATCH 3397/6219] [brightcove:new] Add test for ref: prefixed video
 id

---
 youtube_dl/extractor/brightcove.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 4e983fa10..03a4f446e 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -387,6 +387,10 @@ class BrightcoveNewIE(InfoExtractor):
         'params': {
             'skip_download': True,
         }
+    }, {
+        # ref: prefixed video id
+        'url': 'http://players.brightcove.net/3910869709001/21519b5c-4b3b-4363-accb-bdc8f358f823_default/index.html?videoId=ref:7069442',
+        'only_matching': True,
     }]
 
     @staticmethod

From d631d5f9f27f93767226192e4288990413fa9dbd Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 19 Dec 2015 18:21:42 +0800
Subject: [PATCH 3398/6219] [utils] Fix TTML conversion

Tolerate invalid timestamps (closes #7909)
---
 test/test_utils.py  |  7 +++++--
 youtube_dl/utils.py | 11 ++++++++---
 2 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 8fc74e591..86045e680 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -661,8 +661,8 @@ ffmpeg version 2.4.4 Copyright (c) 2000-2014 the FFmpeg ...'''), '2.4.4')
             {'like_count': 190, 'dislike_count': 10}))
 
     def test_parse_dfxp_time_expr(self):
-        self.assertEqual(parse_dfxp_time_expr(None), 0.0)
-        self.assertEqual(parse_dfxp_time_expr(''), 0.0)
+        self.assertEqual(parse_dfxp_time_expr(None), None)
+        self.assertEqual(parse_dfxp_time_expr(''), None)
         self.assertEqual(parse_dfxp_time_expr('0.1'), 0.1)
         self.assertEqual(parse_dfxp_time_expr('0.1s'), 0.1)
         self.assertEqual(parse_dfxp_time_expr('00:00:01'), 1.0)
@@ -676,6 +676,9 @@ ffmpeg version 2.4.4 Copyright (c) 2000-2014 the FFmpeg ...'''), '2.4.4')
                     <p begin="0" end="1">The following line contains Chinese characters and special symbols</p>
                     <p begin="1" end="2">第二行<br/>♪♪</p>
                     <p begin="2" dur="1"><span>Third<br/>Line</span></p>
+                    <p begin="3" end="-1">Lines with invalid timestamps are ignored</p>
+                    <p begin="-1" end="-1">Ignore, two</p>
+                    <p begin="3" dur="-1">Ignored, three</p>
                 </div>
             </body>
             </tt>'''
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 91917fc96..ee20c3d9b 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1976,7 +1976,7 @@ def match_filter_func(filter_str):
 
 def parse_dfxp_time_expr(time_expr):
     if not time_expr:
-        return 0.0
+        return
 
     mobj = re.match(r'^(?P<time_offset>\d+(?:\.\d+)?)s?$', time_expr)
     if mobj:
@@ -2020,10 +2020,15 @@ def dfxp2srt(dfxp_data):
         raise ValueError('Invalid dfxp/TTML subtitle')
 
     for para, index in zip(paras, itertools.count(1)):
-        begin_time = parse_dfxp_time_expr(para.attrib['begin'])
+        begin_time = parse_dfxp_time_expr(para.attrib.get('begin'))
         end_time = parse_dfxp_time_expr(para.attrib.get('end'))
+        dur = parse_dfxp_time_expr(para.attrib.get('dur'))
+        if begin_time is None:
+            continue
         if not end_time:
-            end_time = begin_time + parse_dfxp_time_expr(para.attrib['dur'])
+            if not dur:
+                continue
+            end_time = begin_time + dur
         out.append('%d\n%s --> %s\n%s\n\n' % (
             index,
             srt_subtitles_timecode(begin_time),

From db2fe38b5508cbd28b89893219d9cccd41406851 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 19 Dec 2015 19:29:51 +0800
Subject: [PATCH 3399/6219] [utils] Support alternative timestamp format in
 TTML

Fixes #7608
---
 test/test_utils.py  | 1 +
 youtube_dl/utils.py | 4 ++--
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 86045e680..e4e8d3825 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -667,6 +667,7 @@ ffmpeg version 2.4.4 Copyright (c) 2000-2014 the FFmpeg ...'''), '2.4.4')
         self.assertEqual(parse_dfxp_time_expr('0.1s'), 0.1)
         self.assertEqual(parse_dfxp_time_expr('00:00:01'), 1.0)
         self.assertEqual(parse_dfxp_time_expr('00:00:01.100'), 1.1)
+        self.assertEqual(parse_dfxp_time_expr('00:00:01:100'), 1.1)
 
     def test_dfxp2srt(self):
         dfxp_data = '''<?xml version="1.0" encoding="UTF-8"?>
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index ee20c3d9b..5b396ede8 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1982,9 +1982,9 @@ def parse_dfxp_time_expr(time_expr):
     if mobj:
         return float(mobj.group('time_offset'))
 
-    mobj = re.match(r'^(\d+):(\d\d):(\d\d(?:\.\d+)?)$', time_expr)
+    mobj = re.match(r'^(\d+):(\d\d):(\d\d(?:(?:\.|:)\d+)?)$', time_expr)
     if mobj:
-        return 3600 * int(mobj.group(1)) + 60 * int(mobj.group(2)) + float(mobj.group(3))
+        return 3600 * int(mobj.group(1)) + 60 * int(mobj.group(2)) + float(mobj.group(3).replace(':', '.'))
 
 
 def srt_subtitles_timecode(seconds):

From ee0f0393cfd3daeff729f5f5594e4c5b1bec5436 Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Thu, 17 Sep 2015 13:51:50 +0800
Subject: [PATCH 3400/6219] [togglesg] New extractor for toggle.sg

---
 youtube_dl/extractor/__init__.py |   1 +
 youtube_dl/extractor/togglesg.py | 159 +++++++++++++++++++++++++++++++
 2 files changed, 160 insertions(+)
 create mode 100644 youtube_dl/extractor/togglesg.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 3cd95ba01..b88fbcc4d 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -675,6 +675,7 @@ from .tnaflix import (
     EMPFlixIE,
     MovieFapIE,
 )
+from .togglesg import ToggleSgIE
 from .thvideo import (
     THVideoIE,
     THVideoPlaylistIE
diff --git a/youtube_dl/extractor/togglesg.py b/youtube_dl/extractor/togglesg.py
new file mode 100644
index 000000000..56ef4b464
--- /dev/null
+++ b/youtube_dl/extractor/togglesg.py
@@ -0,0 +1,159 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import json
+import re
+import itertools
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    int_or_none,
+    determine_ext,
+    parse_iso8601,
+    remove_end
+)
+from ..compat import compat_urllib_request
+
+
+class ToggleSgIE(InfoExtractor):
+    IE_NAME = 'togglesg'
+    _VALID_URL = r'https?://video\.toggle\.sg/(?:(en|zh))/(?:(series|clips|movies))/.+?/(?P<id>[0-9]+)'
+    _TESTS = [{
+        'url': 'http://video.toggle.sg/en/series/lion-moms-tif/trailers/lion-moms-premier/343115',
+        'info_dict': {
+            'id': '343115',
+            'ext': 'mp4',
+            'title': 'Lion Moms Premiere',
+            'description': 'md5:aea1149404bff4d7f7b6da11fafd8e6b',
+            'upload_date': '20150910',
+            'timestamp': 1441858274,
+        },
+        'params': {
+            'skip_download': 'm3u8 download',
+        }
+    }, {
+        'note': 'DRM-protected video',
+        'url': 'http://video.toggle.sg/en/movies/dug-s-special-mission/341413',
+        'info_dict': {
+            'id': '341413',
+            'ext': 'wvm',
+            'title': 'Dug\'s Special Mission',
+            'description': 'md5:e86c6f4458214905c1772398fabc93e0',
+            'upload_date': '20150827',
+            'timestamp': 1440644006,
+        },
+        'params': {
+            'skip_download': 'DRM-protected wvm download',
+        }
+    }, {
+        'note': 'm3u8 links are geo-restricted, but Android/mp4 is okay',
+        'url': 'http://video.toggle.sg/en/series/28th-sea-games-5-show/ep11/332861',
+        'info_dict': {
+            'id': '332861',
+            'ext': 'mp4',
+            'title': '28th SEA Games (5 Show) -  Episode  11',
+            'description': 'md5:3cd4f5f56c7c3b1340c50a863f896faa',
+            'upload_date': '20150605',
+            'timestamp': 1433480166,
+        },
+        'params': {
+            'skip_download': 'DRM-protected wvm download',
+        },
+        'skip': 'm3u8 links are geo-restricted'
+    }, {
+        'url': 'http://video.toggle.sg/en/clips/seraph-sun-aloysius-will-suddenly-sing-some-old-songs-in-high-pitch-on-set/343331',
+        'only_matching': True,
+    }, {
+        'url': 'http://video.toggle.sg/zh/series/zero-calling-s2-hd/ep13/336367',
+        'only_matching': True,
+    }, {
+        'url': 'http://video.toggle.sg/en/series/vetri-s2/webisodes/jeeva-is-an-orphan-vetri-s2-webisode-7/342302',
+        'only_matching': True,
+    }, {
+        'url': 'http://video.toggle.sg/en/movies/seven-days/321936',
+        'only_matching': True,
+    }]
+
+    _FORMAT_PREFERENCES = {
+        'wvm-STBMain': -10,
+        'wvm-iPadMain': -20,
+        'wvm-iPhoneMain': -30,
+        'wvm-Android': -40,
+    }
+    _API_USER = 'tvpapi_147'
+    _API_PASS = '11111'
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id, note='Downloading video page')
+
+        api_user = self._search_regex(
+            r'apiUser:\s*"([^"]+)"', webpage, 'apiUser', default=self._API_USER, fatal=False)
+        api_pass = self._search_regex(
+            r'apiPass:\s*"([^"]+)"', webpage, 'apiPass', default=self._API_PASS, fatal=False)
+
+        params = {
+            'initObj': {
+                'Locale': {
+                    'LocaleLanguage': '', 'LocaleCountry': '',
+                    'LocaleDevice': '', 'LocaleUserState': 0
+                },
+                'Platform': 0, 'SiteGuid': 0, 'DomainID': '0', 'UDID': '',
+                'ApiUser': api_user, 'ApiPass': api_pass
+            },
+            'MediaID': video_id,
+            'mediaType': 0,
+        }
+
+        req = compat_urllib_request.Request(
+            'http://tvpapi.as.tvinci.com/v2_9/gateways/jsonpostgw.aspx?m=GetMediaInfo',
+            json.dumps(params).encode('utf-8'))
+        info = self._download_json(req, video_id, 'Downloading video info json')
+
+        title = info['MediaName']
+        duration = int_or_none(info.get('Duration'))
+        thumbnail = info.get('PicURL')
+        description = info.get('Description')
+        created_at = parse_iso8601(info.get('CreationDate') or None)
+        formats = []
+
+        for video_file in info.get('Files', []):
+            ext = determine_ext(video_file['URL'])
+            vid_format = video_file['Format'].replace(' ', '')
+            # if geo-restricted, m3u8 is inaccessible, but mp4 is okay
+            if ext == 'm3u8':
+                m3u8_formats = self._extract_m3u8_formats(
+                    video_file['URL'], video_id, ext='mp4', m3u8_id=vid_format,
+                    note='Downloading %s m3u8 information' % vid_format,
+                    errnote='Failed to download %s m3u8 information' % vid_format,
+                    fatal=False
+                )
+                if m3u8_formats:
+                    formats.extend(m3u8_formats)
+            if ext in ['mp4', 'wvm']:
+                # wvm are drm-protected files
+                formats.append({
+                    'ext': ext,
+                    'url': video_file['URL'],
+                    'format_id': vid_format,
+                    'preference': self._FORMAT_PREFERENCES.get(ext + '-' + vid_format) or -1,
+                    'format_note': 'DRM-protected video' if ext == 'wvm' else None
+                })
+
+        if not formats:
+            # Most likely because geo-blocked
+            raise ExtractorError('No downloadable videos found', expected=True)
+
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'duration': duration,
+            'timestamp': created_at,
+            'thumbnail': thumbnail,
+            'formats': formats,
+        }

From ed370ff0e6f52aaf3647b32cdbba9d25ce8533e5 Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Fri, 18 Sep 2015 00:51:41 +0800
Subject: [PATCH 3401/6219] [togglesg] Fixes

---
 youtube_dl/extractor/togglesg.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/togglesg.py b/youtube_dl/extractor/togglesg.py
index 56ef4b464..9f958d453 100644
--- a/youtube_dl/extractor/togglesg.py
+++ b/youtube_dl/extractor/togglesg.py
@@ -18,7 +18,7 @@ from ..compat import compat_urllib_request
 
 class ToggleSgIE(InfoExtractor):
     IE_NAME = 'togglesg'
-    _VALID_URL = r'https?://video\.toggle\.sg/(?:(en|zh))/(?:(series|clips|movies))/.+?/(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://video\.toggle\.sg/(?:en|zh)/(?:series|clips|movies)/.+?/(?P<id>[0-9]+)'
     _TESTS = [{
         'url': 'http://video.toggle.sg/en/series/lion-moms-tif/trailers/lion-moms-premier/343115',
         'info_dict': {
@@ -90,9 +90,9 @@ class ToggleSgIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id, note='Downloading video page')
 
         api_user = self._search_regex(
-            r'apiUser:\s*"([^"]+)"', webpage, 'apiUser', default=self._API_USER, fatal=False)
+            r'apiUser:\s*"([^"]+)"', webpage, 'apiUser', default=self._API_USER)
         api_pass = self._search_regex(
-            r'apiPass:\s*"([^"]+)"', webpage, 'apiPass', default=self._API_PASS, fatal=False)
+            r'apiPass:\s*"([^"]+)"', webpage, 'apiPass', default=self._API_PASS)
 
         params = {
             'initObj': {

From f8253af561655c7c470323e631e8b66283bcf623 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 19 Dec 2015 19:03:55 +0600
Subject: [PATCH 3402/6219] [toggle] Use sanitized_Request

---
 youtube_dl/extractor/togglesg.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/togglesg.py b/youtube_dl/extractor/togglesg.py
index 9f958d453..05a9e5439 100644
--- a/youtube_dl/extractor/togglesg.py
+++ b/youtube_dl/extractor/togglesg.py
@@ -11,7 +11,8 @@ from ..utils import (
     int_or_none,
     determine_ext,
     parse_iso8601,
-    remove_end
+    remove_end,
+    sanitized_Request,
 )
 from ..compat import compat_urllib_request
 
@@ -107,7 +108,7 @@ class ToggleSgIE(InfoExtractor):
             'mediaType': 0,
         }
 
-        req = compat_urllib_request.Request(
+        req = sanitized_Request(
             'http://tvpapi.as.tvinci.com/v2_9/gateways/jsonpostgw.aspx?m=GetMediaInfo',
             json.dumps(params).encode('utf-8'))
         info = self._download_json(req, video_id, 'Downloading video info json')

From c82a8dd14c813e1b022188925fc33fb4d831bc3d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 19 Dec 2015 19:04:38 +0600
Subject: [PATCH 3403/6219] [toggle] Remove unused imports

---
 youtube_dl/extractor/togglesg.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/togglesg.py b/youtube_dl/extractor/togglesg.py
index 05a9e5439..ea660960e 100644
--- a/youtube_dl/extractor/togglesg.py
+++ b/youtube_dl/extractor/togglesg.py
@@ -2,19 +2,15 @@
 from __future__ import unicode_literals
 
 import json
-import re
-import itertools
 
 from .common import InfoExtractor
 from ..utils import (
+    determine_ext,
     ExtractorError,
     int_or_none,
-    determine_ext,
     parse_iso8601,
-    remove_end,
     sanitized_Request,
 )
-from ..compat import compat_urllib_request
 
 
 class ToggleSgIE(InfoExtractor):

From 74c730174fa872ae37b4709a3a452391ee6273fb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 19 Dec 2015 19:06:05 +0600
Subject: [PATCH 3404/6219] [toggle] Style

---
 youtube_dl/extractor/togglesg.py | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/togglesg.py b/youtube_dl/extractor/togglesg.py
index ea660960e..d3903b58d 100644
--- a/youtube_dl/extractor/togglesg.py
+++ b/youtube_dl/extractor/togglesg.py
@@ -94,11 +94,17 @@ class ToggleSgIE(InfoExtractor):
         params = {
             'initObj': {
                 'Locale': {
-                    'LocaleLanguage': '', 'LocaleCountry': '',
-                    'LocaleDevice': '', 'LocaleUserState': 0
+                    'LocaleLanguage': '',
+                    'LocaleCountry': '',
+                    'LocaleDevice': '',
+                    'LocaleUserState': 0
                 },
-                'Platform': 0, 'SiteGuid': 0, 'DomainID': '0', 'UDID': '',
-                'ApiUser': api_user, 'ApiPass': api_pass
+                'Platform': 0,
+                'SiteGuid': 0,
+                'DomainID': '0',
+                'UDID': '',
+                'ApiUser': api_user,
+                'ApiPass': api_pass
             },
             'MediaID': video_id,
             'mediaType': 0,

From ffaf6e66e3830d0e4750aec2cfdaff1a6bd9c2ad Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 19 Dec 2015 19:08:47 +0600
Subject: [PATCH 3405/6219] [toggle] Improve

---
 youtube_dl/extractor/togglesg.py | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/togglesg.py b/youtube_dl/extractor/togglesg.py
index d3903b58d..244c79e8d 100644
--- a/youtube_dl/extractor/togglesg.py
+++ b/youtube_dl/extractor/togglesg.py
@@ -84,12 +84,15 @@ class ToggleSgIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        webpage = self._download_webpage(url, video_id, note='Downloading video page')
+        webpage = self._download_webpage(
+            url, video_id, note='Downloading video page')
 
         api_user = self._search_regex(
-            r'apiUser:\s*"([^"]+)"', webpage, 'apiUser', default=self._API_USER)
+            r'apiUser\s*:\s*(["\'])(?P<user>.+?)\1', webpage, 'apiUser',
+            default=self._API_USER, group='user')
         api_pass = self._search_regex(
-            r'apiPass:\s*"([^"]+)"', webpage, 'apiPass', default=self._API_PASS)
+            r'apiPass\s*:\s*(["\'])(?P<pass>.+?)\1', webpage, 'apiPass',
+            default=self._API_PASS, group='pass')
 
         params = {
             'initObj': {
@@ -131,11 +134,10 @@ class ToggleSgIE(InfoExtractor):
                     video_file['URL'], video_id, ext='mp4', m3u8_id=vid_format,
                     note='Downloading %s m3u8 information' % vid_format,
                     errnote='Failed to download %s m3u8 information' % vid_format,
-                    fatal=False
-                )
+                    fatal=False)
                 if m3u8_formats:
                     formats.extend(m3u8_formats)
-            if ext in ['mp4', 'wvm']:
+            elif ext in ('mp4', 'wvm'):
                 # wvm are drm-protected files
                 formats.append({
                     'ext': ext,

From c40dbb19ab475e8a0e1b29548130adf9ce13ea43 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 19 Dec 2015 19:19:26 +0600
Subject: [PATCH 3406/6219] [toggle] Extract thumbnails

---
 youtube_dl/extractor/togglesg.py | 34 ++++++++++++++++++++++++--------
 1 file changed, 26 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/togglesg.py b/youtube_dl/extractor/togglesg.py
index 244c79e8d..a2b89d6bb 100644
--- a/youtube_dl/extractor/togglesg.py
+++ b/youtube_dl/extractor/togglesg.py
@@ -2,6 +2,7 @@
 from __future__ import unicode_literals
 
 import json
+import re
 
 from .common import InfoExtractor
 from ..utils import (
@@ -119,12 +120,8 @@ class ToggleSgIE(InfoExtractor):
         info = self._download_json(req, video_id, 'Downloading video info json')
 
         title = info['MediaName']
-        duration = int_or_none(info.get('Duration'))
-        thumbnail = info.get('PicURL')
-        description = info.get('Description')
-        created_at = parse_iso8601(info.get('CreationDate') or None)
-        formats = []
 
+        formats = []
         for video_file in info.get('Files', []):
             ext = determine_ext(video_file['URL'])
             vid_format = video_file['Format'].replace(' ', '')
@@ -146,19 +143,40 @@ class ToggleSgIE(InfoExtractor):
                     'preference': self._FORMAT_PREFERENCES.get(ext + '-' + vid_format) or -1,
                     'format_note': 'DRM-protected video' if ext == 'wvm' else None
                 })
-
         if not formats:
             # Most likely because geo-blocked
             raise ExtractorError('No downloadable videos found', expected=True)
-
         self._sort_formats(formats)
 
+        duration = int_or_none(info.get('Duration'))
+        description = info.get('Description')
+        created_at = parse_iso8601(info.get('CreationDate') or None)
+
+        thumbnails = []
+        for picture in info.get('Pictures', []):
+            if not isinstance(picture, dict):
+                continue
+            pic_url = picture.get('URL')
+            if not pic_url:
+                continue
+            thumbnail = {
+                'url': pic_url,
+            }
+            pic_size = picture.get('PicSize', '')
+            m = re.search(r'(?P<width>\d+)[xX](?P<height>\d+)', pic_size)
+            if m:
+                thumbnail.update({
+                    'width': int(m.group('width')),
+                    'height': int(m.group('height')),
+                })
+            thumbnails.append(thumbnail)
+
         return {
             'id': video_id,
             'title': title,
             'description': description,
             'duration': duration,
             'timestamp': created_at,
-            'thumbnail': thumbnail,
+            'thumbnails': thumbnails,
             'formats': formats,
         }

From 8f097af4ec5d6ff72a80c744d67f947336fecb28 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 19 Dec 2015 19:23:28 +0600
Subject: [PATCH 3407/6219] [toggle] Extract counters

---
 youtube_dl/extractor/togglesg.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/youtube_dl/extractor/togglesg.py b/youtube_dl/extractor/togglesg.py
index a2b89d6bb..36367971f 100644
--- a/youtube_dl/extractor/togglesg.py
+++ b/youtube_dl/extractor/togglesg.py
@@ -8,6 +8,7 @@ from .common import InfoExtractor
 from ..utils import (
     determine_ext,
     ExtractorError,
+    float_or_none,
     int_or_none,
     parse_iso8601,
     sanitized_Request,
@@ -152,6 +153,10 @@ class ToggleSgIE(InfoExtractor):
         description = info.get('Description')
         created_at = parse_iso8601(info.get('CreationDate') or None)
 
+        average_rating = float_or_none(info.get('Rating'))
+        view_count = int_or_none(info.get('ViewCounter') or info.get('view_counter'))
+        like_count = int_or_none(info.get('LikeCounter') or info.get('like_counter'))
+
         thumbnails = []
         for picture in info.get('Pictures', []):
             if not isinstance(picture, dict):
@@ -177,6 +182,9 @@ class ToggleSgIE(InfoExtractor):
             'description': description,
             'duration': duration,
             'timestamp': created_at,
+            'average_rating': average_rating,
+            'view_count': view_count,
+            'like_count': like_count,
             'thumbnails': thumbnails,
             'formats': formats,
         }

From 989e9f8eadc936ab47fcd0b7ba63aac13020a526 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 19 Dec 2015 19:52:37 +0600
Subject: [PATCH 3408/6219] [toggle] Improve formats extraction robustness

---
 youtube_dl/extractor/togglesg.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/togglesg.py b/youtube_dl/extractor/togglesg.py
index 36367971f..47b2bfcb8 100644
--- a/youtube_dl/extractor/togglesg.py
+++ b/youtube_dl/extractor/togglesg.py
@@ -124,12 +124,15 @@ class ToggleSgIE(InfoExtractor):
 
         formats = []
         for video_file in info.get('Files', []):
-            ext = determine_ext(video_file['URL'])
-            vid_format = video_file['Format'].replace(' ', '')
+            video_url, vid_format = video_file.get('URL'), video_file.get('Format')
+            if not video_url or not vid_format:
+                continue
+            ext = determine_ext(video_url)
+            vid_format = vid_format.replace(' ', '')
             # if geo-restricted, m3u8 is inaccessible, but mp4 is okay
             if ext == 'm3u8':
                 m3u8_formats = self._extract_m3u8_formats(
-                    video_file['URL'], video_id, ext='mp4', m3u8_id=vid_format,
+                    video_url, video_id, ext='mp4', m3u8_id=vid_format,
                     note='Downloading %s m3u8 information' % vid_format,
                     errnote='Failed to download %s m3u8 information' % vid_format,
                     fatal=False)
@@ -139,7 +142,7 @@ class ToggleSgIE(InfoExtractor):
                 # wvm are drm-protected files
                 formats.append({
                     'ext': ext,
-                    'url': video_file['URL'],
+                    'url': video_url,
                     'format_id': vid_format,
                     'preference': self._FORMAT_PREFERENCES.get(ext + '-' + vid_format) or -1,
                     'format_note': 'DRM-protected video' if ext == 'wvm' else None

From e33c9cba7c6c9d4a04d749c88ab447a57bcfe423 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 19 Dec 2015 19:58:18 +0600
Subject: [PATCH 3409/6219] [toggle] Improve _VALID_URL

---
 youtube_dl/extractor/togglesg.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/togglesg.py b/youtube_dl/extractor/togglesg.py
index 47b2bfcb8..22c6c91b9 100644
--- a/youtube_dl/extractor/togglesg.py
+++ b/youtube_dl/extractor/togglesg.py
@@ -17,7 +17,7 @@ from ..utils import (
 
 class ToggleSgIE(InfoExtractor):
     IE_NAME = 'togglesg'
-    _VALID_URL = r'https?://video\.toggle\.sg/(?:en|zh)/(?:series|clips|movies)/.+?/(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://video\.toggle\.sg/(?:en|zh)/(?:series|clips|movies)/(?:[^/]+/)+(?P<id>[0-9]+)'
     _TESTS = [{
         'url': 'http://video.toggle.sg/en/series/lion-moms-tif/trailers/lion-moms-premier/343115',
         'info_dict': {
@@ -46,8 +46,9 @@ class ToggleSgIE(InfoExtractor):
             'skip_download': 'DRM-protected wvm download',
         }
     }, {
+        # this also tests correct video id extraction
         'note': 'm3u8 links are geo-restricted, but Android/mp4 is okay',
-        'url': 'http://video.toggle.sg/en/series/28th-sea-games-5-show/ep11/332861',
+        'url': 'http://video.toggle.sg/en/series/28th-sea-games-5-show/28th-sea-games-5-show-ep11/332861',
         'info_dict': {
             'id': '332861',
             'ext': 'mp4',

From cc0f378d5429225caf0477f5d10b342b1ef136d5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 19 Dec 2015 19:59:00 +0600
Subject: [PATCH 3410/6219] [toggle] Rename to toggle

---
 youtube_dl/extractor/__init__.py                | 2 +-
 youtube_dl/extractor/{togglesg.py => toggle.py} | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)
 rename youtube_dl/extractor/{togglesg.py => toggle.py} (99%)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index b88fbcc4d..fef2cff62 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -675,7 +675,7 @@ from .tnaflix import (
     EMPFlixIE,
     MovieFapIE,
 )
-from .togglesg import ToggleSgIE
+from .toggle import ToggleIE
 from .thvideo import (
     THVideoIE,
     THVideoPlaylistIE
diff --git a/youtube_dl/extractor/togglesg.py b/youtube_dl/extractor/toggle.py
similarity index 99%
rename from youtube_dl/extractor/togglesg.py
rename to youtube_dl/extractor/toggle.py
index 22c6c91b9..c9d6e52e1 100644
--- a/youtube_dl/extractor/togglesg.py
+++ b/youtube_dl/extractor/toggle.py
@@ -15,7 +15,7 @@ from ..utils import (
 )
 
 
-class ToggleSgIE(InfoExtractor):
+class ToggleIE(InfoExtractor):
     IE_NAME = 'togglesg'
     _VALID_URL = r'https?://video\.toggle\.sg/(?:en|zh)/(?:series|clips|movies)/(?:[^/]+/)+(?P<id>[0-9]+)'
     _TESTS = [{

From 0f206ee81447349f9ec8c430de4868a86020564c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 19 Dec 2015 23:11:23 +0600
Subject: [PATCH 3411/6219] [toggle] Change IE_NAME

---
 youtube_dl/extractor/toggle.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/toggle.py b/youtube_dl/extractor/toggle.py
index c9d6e52e1..a47239952 100644
--- a/youtube_dl/extractor/toggle.py
+++ b/youtube_dl/extractor/toggle.py
@@ -16,7 +16,7 @@ from ..utils import (
 
 
 class ToggleIE(InfoExtractor):
-    IE_NAME = 'togglesg'
+    IE_NAME = 'toggle'
     _VALID_URL = r'https?://video\.toggle\.sg/(?:en|zh)/(?:series|clips|movies)/(?:[^/]+/)+(?P<id>[0-9]+)'
     _TESTS = [{
         'url': 'http://video.toggle.sg/en/series/lion-moms-tif/trailers/lion-moms-premier/343115',

From e0f06eae432e592b159237f2ce9813449431cc7b Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 19 Dec 2015 18:26:28 +0100
Subject: [PATCH 3412/6219] [fktv] fix info extraction

---
 youtube_dl/extractor/fktv.py | 30 +++++++++++++-----------------
 1 file changed, 13 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/fktv.py b/youtube_dl/extractor/fktv.py
index 40ea27895..5f6e65dae 100644
--- a/youtube_dl/extractor/fktv.py
+++ b/youtube_dl/extractor/fktv.py
@@ -1,12 +1,10 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..utils import (
     clean_html,
     determine_ext,
-    ExtractorError,
+    js_to_json,
 )
 
 
@@ -32,24 +30,22 @@ class FKTVIE(InfoExtractor):
             'http://fernsehkritik.tv/folge-%s/play' % episode, episode)
         title = clean_html(self._html_search_regex(
             '<h3>([^<]+)</h3>', webpage, 'title'))
-        matches = re.search(
-            r'(?s)<video(?:(?!poster)[^>])+(?:poster="([^"]+)")?[^>]*>(.*)</video>',
-            webpage)
-        if matches is None:
-            raise ExtractorError('Unable to extract the video')
+        thumbnail = self._search_regex(r'POSTER\s*=\s*"([^"]+)', webpage, 'thumbnail', fatal=False)
+        sources = self._parse_json(self._search_regex(r'(?s)MEDIA\s*=\s*(\[.+?\]);', webpage, 'media'), episode, js_to_json)
 
-        poster, sources = matches.groups()
-        if poster is None:
-            self.report_warning('unable to extract thumbnail')
+        formats = []
+        for source in sources:
+            furl = source.get('src')
+            if furl:
+                formats.append({
+                    'url': furl,
+                    'format_id': determine_ext(furl),
+                })
+        self._sort_formats(formats)
 
-        urls = re.findall(r'<source[^>]+src="([^"]+)"', sources)
-        formats = [{
-            'url': furl,
-            'format_id': determine_ext(furl),
-        } for furl in urls]
         return {
             'id': episode,
             'title': title,
             'formats': formats,
-            'thumbnail': poster,
+            'thumbnail': thumbnail,
         }

From fa64a8431166ad7ca7bd2881306ee379b6e2abfd Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 19 Dec 2015 19:02:04 +0100
Subject: [PATCH 3413/6219] [faz] fix info extraction

---
 youtube_dl/extractor/faz.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/faz.py b/youtube_dl/extractor/faz.py
index cebdd0193..d9a868119 100644
--- a/youtube_dl/extractor/faz.py
+++ b/youtube_dl/extractor/faz.py
@@ -38,7 +38,7 @@ class FazIE(InfoExtractor):
 
         webpage = self._download_webpage(url, video_id)
         config_xml_url = self._search_regex(
-            r'writeFLV\(\'(.+?)\',', webpage, 'config xml url')
+            r'(?:var\s+)?videoXMLURL\s*=\s*"([^"]+)', webpage, 'config xml url')
         config = self._download_xml(
             config_xml_url, video_id, 'Downloading config xml')
 

From dd85e4d70787dd0c106fb05bdb0a38f51037fd39 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 20 Dec 2015 02:43:50 +0600
Subject: [PATCH 3414/6219] [extractor/common] Properly decode error string on
 python 2 (Closes #1354, closes #3957, closes #4037, closes #6449)

---
 youtube_dl/extractor/common.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 6ab2d68d6..3ab72ff76 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -34,6 +34,7 @@ from ..utils import (
     fix_xml_ampersands,
     float_or_none,
     int_or_none,
+    preferredencoding,
     RegexNotFoundError,
     sanitize_filename,
     sanitized_Request,
@@ -332,7 +333,12 @@ class InfoExtractor(object):
                 return False
             if errnote is None:
                 errnote = 'Unable to download webpage'
-            errmsg = '%s: %s' % (errnote, compat_str(err))
+            err_str = str(err)
+            # On python 2 error byte string must be decoded with proper
+            # encoding rather than ascii
+            if sys.version_info[0] < 3:
+                err_str = err_str.decode(preferredencoding())
+            errmsg = '%s: %s' % (errnote, err_str)
             if fatal:
                 raise ExtractorError(errmsg, sys.exc_info()[2], cause=err)
             else:

From ec6504b39c9b3d9766c551691a96e481278fa659 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 19 Dec 2015 23:28:54 +0100
Subject: [PATCH 3415/6219] [gputechconf] Add new extractor(closes #5775)

---
 youtube_dl/extractor/__init__.py    |  1 +
 youtube_dl/extractor/gputechconf.py | 57 +++++++++++++++++++++++++++++
 2 files changed, 58 insertions(+)
 create mode 100644 youtube_dl/extractor/gputechconf.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index fef2cff62..65c5e1c92 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -235,6 +235,7 @@ from .golem import GolemIE
 from .googleplus import GooglePlusIE
 from .googlesearch import GoogleSearchIE
 from .goshgay import GoshgayIE
+from .gputechconf import GPUTechConfIE
 from .groupon import GrouponIE
 from .hark import HarkIE
 from .hearthisat import HearThisAtIE
diff --git a/youtube_dl/extractor/gputechconf.py b/youtube_dl/extractor/gputechconf.py
new file mode 100644
index 000000000..becc61f1e
--- /dev/null
+++ b/youtube_dl/extractor/gputechconf.py
@@ -0,0 +1,57 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    xpath_element,
+    xpath_text,
+    int_or_none,
+    parse_duration,
+)
+
+
+class GPUTechConfIE(InfoExtractor):
+    _VALID_URL = r'https?://on-demand\.gputechconf\.com/gtc/2015/video/S(?P<id>\d+)\.html'
+    _TEST = {
+        'url': 'http://on-demand.gputechconf.com/gtc/2015/video/S5156.html',
+        'md5': 'a8862a00a0fd65b8b43acc5b8e33f798',
+        'info_dict': {
+            'id': '5156',
+            'ext': 'mp4',
+            'title': 'Coordinating More Than 3 Million CUDA Threads for Social Network Analysis',
+            'duration': 1219,
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        root_path = self._search_regex(r'var\s+rootPath\s*=\s*"([^"]+)', webpage, 'root path', 'http://evt.dispeak.com/nvidia/events/gtc15/')
+        xml_file_id = self._search_regex(r'var\s+xmlFileId\s*=\s*"([^"]+)', webpage, 'xml file id')
+
+        doc = self._download_xml('%sxml/%s.xml' % (root_path, xml_file_id), video_id)
+
+        metadata = xpath_element(doc, 'metadata')
+        http_host = xpath_text(metadata, 'httpHost')
+        mbr_videos = xpath_element(metadata, 'MBRVideos')
+
+        formats = []
+        for mbr_video in mbr_videos.findall('MBRVideo'):
+            stream_name = xpath_text(mbr_video, 'streamName')
+            if stream_name:
+                bitrate = int_or_none(xpath_text(mbr_video, 'bitrate'))
+                formats.append({
+                    'url': 'http://%s/%s' % (http_host, stream_name.replace('mp4:', '')),
+                    'tbr': bitrate,
+                    'format_id': 'http-%d' % bitrate,
+                })
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': xpath_text(metadata, 'title'),
+            'duration': parse_duration(xpath_text(metadata, 'endTime')),
+            'creator': xpath_text(metadata, 'speaker'),
+            'formats': formats,
+        }

From 1deb710f26b0ccd14db13103cb400b663e243c57 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 19 Dec 2015 23:59:00 +0100
Subject: [PATCH 3416/6219] [gputechconf] improve extraction

---
 youtube_dl/extractor/gputechconf.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/gputechconf.py b/youtube_dl/extractor/gputechconf.py
index becc61f1e..145b55bf3 100644
--- a/youtube_dl/extractor/gputechconf.py
+++ b/youtube_dl/extractor/gputechconf.py
@@ -33,18 +33,16 @@ class GPUTechConfIE(InfoExtractor):
         doc = self._download_xml('%sxml/%s.xml' % (root_path, xml_file_id), video_id)
 
         metadata = xpath_element(doc, 'metadata')
-        http_host = xpath_text(metadata, 'httpHost')
+        http_host = xpath_text(metadata, 'httpHost', 'http host', True)
         mbr_videos = xpath_element(metadata, 'MBRVideos')
 
         formats = []
         for mbr_video in mbr_videos.findall('MBRVideo'):
             stream_name = xpath_text(mbr_video, 'streamName')
             if stream_name:
-                bitrate = int_or_none(xpath_text(mbr_video, 'bitrate'))
                 formats.append({
                     'url': 'http://%s/%s' % (http_host, stream_name.replace('mp4:', '')),
-                    'tbr': bitrate,
-                    'format_id': 'http-%d' % bitrate,
+                    'tbr': int_or_none(xpath_text(mbr_video, 'bitrate')),
                 })
         self._sort_formats(formats)
 

From fdae2358581be6f80ac5fd20b1062e87a8797f46 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 20 Dec 2015 05:26:47 +0600
Subject: [PATCH 3417/6219] [utils] Add error_to_str

---
 youtube_dl/utils.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 5b396ede8..6d3119760 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1806,6 +1806,15 @@ def args_to_str(args):
     return ' '.join(shlex_quote(a) for a in args)
 
 
+def error_to_str(err):
+    err_str = str(err)
+    # On python 2 error byte string must be decoded with proper
+    # encoding rather than ascii
+    if sys.version_info[0] < 3:
+        err_str = err_str.decode(preferredencoding())
+    return err_str
+
+
 def mimetype2ext(mt):
     _, _, res = mt.rpartition('/')
 

From 7f8b271465df75bb1f83cb181dc45fee5fe02cf9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 20 Dec 2015 05:27:38 +0600
Subject: [PATCH 3418/6219] Properly convert errors to strings

---
 youtube_dl/YoutubeDL.py             |  7 ++++---
 youtube_dl/downloader/common.py     |  4 ++--
 youtube_dl/extractor/common.py      | 12 ++++--------
 youtube_dl/extractor/dailymotion.py |  6 +++---
 youtube_dl/extractor/facebook.py    |  4 ++--
 youtube_dl/extractor/youtube.py     |  3 ++-
 6 files changed, 17 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index c642a1fbf..26b3adb02 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -48,6 +48,7 @@ from .utils import (
     determine_ext,
     DownloadError,
     encodeFilename,
+    error_to_str,
     ExtractorError,
     format_bytes,
     formatSeconds,
@@ -681,7 +682,7 @@ class YoutubeDL(object):
                 raise
             except Exception as e:
                 if self.params.get('ignoreerrors', False):
-                    self.report_error(compat_str(e), tb=compat_str(traceback.format_exc()))
+                    self.report_error(error_to_str(e), tb=compat_str(traceback.format_exc()))
                     break
                 else:
                     raise
@@ -1459,7 +1460,7 @@ class YoutubeDL(object):
             if dn and not os.path.exists(dn):
                 os.makedirs(dn)
         except (OSError, IOError) as err:
-            self.report_error('unable to create directory ' + compat_str(err))
+            self.report_error('unable to create directory ' + error_to_str(err))
             return
 
         if self.params.get('writedescription', False):
@@ -2039,4 +2040,4 @@ class YoutubeDL(object):
                                    (info_dict['extractor'], info_dict['id'], thumb_display_id, thumb_filename))
                 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
                     self.report_warning('Unable to download thumbnail "%s": %s' %
-                                        (t['url'], compat_str(err)))
+                                        (t['url'], error_to_str(err)))
diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index b8bf8daf8..eb63ccffd 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -5,9 +5,9 @@ import re
 import sys
 import time
 
-from ..compat import compat_str
 from ..utils import (
     encodeFilename,
+    error_to_str,
     decodeArgument,
     format_bytes,
     timeconvert,
@@ -186,7 +186,7 @@ class FileDownloader(object):
                 return
             os.rename(encodeFilename(old_filename), encodeFilename(new_filename))
         except (IOError, OSError) as err:
-            self.report_error('unable to rename file: %s' % compat_str(err))
+            self.report_error('unable to rename file: %s' % error_to_str(err))
 
     def try_utime(self, filename, last_modified_hdr):
         """Try to set the last-modified time of the given file."""
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 3ab72ff76..5659e40c8 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -30,11 +30,11 @@ from ..utils import (
     clean_html,
     compiled_regex_type,
     determine_ext,
+    error_to_str,
     ExtractorError,
     fix_xml_ampersands,
     float_or_none,
     int_or_none,
-    preferredencoding,
     RegexNotFoundError,
     sanitize_filename,
     sanitized_Request,
@@ -333,12 +333,8 @@ class InfoExtractor(object):
                 return False
             if errnote is None:
                 errnote = 'Unable to download webpage'
-            err_str = str(err)
-            # On python 2 error byte string must be decoded with proper
-            # encoding rather than ascii
-            if sys.version_info[0] < 3:
-                err_str = err_str.decode(preferredencoding())
-            errmsg = '%s: %s' % (errnote, err_str)
+
+            errmsg = '%s: %s' % (errnote, error_to_str(err))
             if fatal:
                 raise ExtractorError(errmsg, sys.exc_info()[2], cause=err)
             else:
@@ -628,7 +624,7 @@ class InfoExtractor(object):
                 else:
                     raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE)
             except (IOError, netrc.NetrcParseError) as err:
-                self._downloader.report_warning('parsing .netrc: %s' % compat_str(err))
+                self._downloader.report_warning('parsing .netrc: %s' % error_to_str(err))
 
         return (username, password)
 
diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index 428556213..7d66baf96 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -7,10 +7,10 @@ import itertools
 
 from .common import InfoExtractor
 
-from ..compat import compat_str
 from ..utils import (
-    ExtractorError,
     determine_ext,
+    error_to_str,
+    ExtractorError,
     int_or_none,
     parse_iso8601,
     sanitized_Request,
@@ -278,7 +278,7 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
                 'https://api.dailymotion.com/video/%s/subtitles?fields=id,language,url' % video_id,
                 video_id, note=False)
         except ExtractorError as err:
-            self._downloader.report_warning('unable to download video subtitles: %s' % compat_str(err))
+            self._downloader.report_warning('unable to download video subtitles: %s' % error_to_str(err))
             return {}
         info = json.loads(sub_list)
         if (info['total'] > 0):
diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index 321eec59e..296d3dd77 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -7,11 +7,11 @@ import socket
 from .common import InfoExtractor
 from ..compat import (
     compat_http_client,
-    compat_str,
     compat_urllib_error,
     compat_urllib_parse_unquote,
 )
 from ..utils import (
+    error_to_str,
     ExtractorError,
     limit_length,
     sanitized_Request,
@@ -116,7 +116,7 @@ class FacebookIE(InfoExtractor):
             if re.search(r'id="checkpointSubmitButton"', check_response) is not None:
                 self._downloader.report_warning('Unable to confirm login, you have to login in your brower and authorize the login.')
         except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
-            self._downloader.report_warning('unable to log in: %s' % compat_str(err))
+            self._downloader.report_warning('unable to log in: %s' % error_to_str(err))
             return
 
     def _real_initialize(self):
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 4556a16fb..d63aa5f4e 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -26,6 +26,7 @@ from ..compat import (
 from ..utils import (
     clean_html,
     encode_dict,
+    error_to_str,
     ExtractorError,
     float_or_none,
     get_element_by_attribute,
@@ -903,7 +904,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'https://video.google.com/timedtext?hl=en&type=list&v=%s' % video_id,
                 video_id, note=False)
         except ExtractorError as err:
-            self._downloader.report_warning('unable to download video subtitles: %s' % compat_str(err))
+            self._downloader.report_warning('unable to download video subtitles: %s' % error_to_str(err))
             return {}
 
         sub_lang_list = {}

From fb043a6e4ee98a2ea9fbb0975e701341bbd34bc3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 20 Dec 2015 06:16:19 +0600
Subject: [PATCH 3419/6219] [YoutubeDL] Use error_to_str

---
 youtube_dl/YoutubeDL.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 26b3adb02..216b5a5db 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -675,8 +675,8 @@ class YoutubeDL(object):
                     return self.process_ie_result(ie_result, download, extra_info)
                 else:
                     return ie_result
-            except ExtractorError as de:  # An error we somewhat expected
-                self.report_error(compat_str(de), de.format_traceback())
+            except ExtractorError as e:  # An error we somewhat expected
+                self.report_error(error_to_str(e), e.format_traceback())
                 break
             except MaxDownloadsReached:
                 raise

From 8900ab4d9bc6bf8a05a1a2608efaff08561945e8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 20 Dec 2015 06:22:01 +0600
Subject: [PATCH 3420/6219] [YoutubeDL] More error_to_str

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 216b5a5db..57df6a279 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1511,7 +1511,7 @@ class YoutubeDL(object):
                             sub_info['url'], info_dict['id'], note=False)
                     except ExtractorError as err:
                         self.report_warning('Unable to download subtitle for "%s": %s' %
-                                            (sub_lang, compat_str(err.cause)))
+                                            (sub_lang, error_to_str(err.cause)))
                         continue
                 try:
                     sub_filename = subtitles_filename(filename, sub_lang, sub_format)

From 8e60dc7526596f456c0b5d7dc48daa6cae08ebb6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 20 Dec 2015 06:26:26 +0600
Subject: [PATCH 3421/6219] [utils] Add encode_compat_str

---
 youtube_dl/utils.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 6d3119760..9a2dd1439 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1712,6 +1712,10 @@ def encode_dict(d, encoding='utf-8'):
     return dict((encode(k), encode(v)) for k, v in d.items())
 
 
+def encode_compat_str(string, encoding=preferredencoding(), errors='strict'):
+    return string if isinstance(string, compat_str) else compat_str(string, encoding, errors)
+
+
 US_RATINGS = {
     'G': 0,
     'PG': 10,

From c0384f221e5a8383c377a3c43f634cf9807eb634 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 20 Dec 2015 06:29:36 +0600
Subject: [PATCH 3422/6219] Use proper encoding on compat_str construction when
 necessary

---
 youtube_dl/YoutubeDL.py |  5 +++--
 youtube_dl/update.py    | 16 ++++++++--------
 2 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 57df6a279..1795097ae 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -47,6 +47,7 @@ from .utils import (
     DEFAULT_OUTTMPL,
     determine_ext,
     DownloadError,
+    encode_compat_str,
     encodeFilename,
     error_to_str,
     ExtractorError,
@@ -496,7 +497,7 @@ class YoutubeDL(object):
                     tb = ''
                     if hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
                         tb += ''.join(traceback.format_exception(*sys.exc_info()[1].exc_info))
-                    tb += compat_str(traceback.format_exc())
+                    tb += encode_compat_str(traceback.format_exc())
                 else:
                     tb_data = traceback.format_list(traceback.extract_stack())
                     tb = ''.join(tb_data)
@@ -682,7 +683,7 @@ class YoutubeDL(object):
                 raise
             except Exception as e:
                 if self.params.get('ignoreerrors', False):
-                    self.report_error(error_to_str(e), tb=compat_str(traceback.format_exc()))
+                    self.report_error(error_to_str(e), tb=encode_compat_str(traceback.format_exc()))
                     break
                 else:
                     raise
diff --git a/youtube_dl/update.py b/youtube_dl/update.py
index 074eb64a7..995b8ed96 100644
--- a/youtube_dl/update.py
+++ b/youtube_dl/update.py
@@ -9,7 +9,7 @@ import subprocess
 import sys
 from zipimport import zipimporter
 
-from .compat import compat_str
+from .utils import encode_compat_str
 
 from .version import __version__
 
@@ -61,7 +61,7 @@ def update_self(to_screen, verbose, opener):
         newversion = opener.open(VERSION_URL).read().decode('utf-8').strip()
     except Exception:
         if verbose:
-            to_screen(compat_str(traceback.format_exc()))
+            to_screen(encode_compat_str(traceback.format_exc()))
         to_screen('ERROR: can\'t find the current version. Please try again later.')
         return
     if newversion == __version__:
@@ -74,7 +74,7 @@ def update_self(to_screen, verbose, opener):
         versions_info = json.loads(versions_info)
     except Exception:
         if verbose:
-            to_screen(compat_str(traceback.format_exc()))
+            to_screen(encode_compat_str(traceback.format_exc()))
         to_screen('ERROR: can\'t obtain versions info. Please try again later.')
         return
     if 'signature' not in versions_info:
@@ -123,7 +123,7 @@ def update_self(to_screen, verbose, opener):
             urlh.close()
         except (IOError, OSError):
             if verbose:
-                to_screen(compat_str(traceback.format_exc()))
+                to_screen(encode_compat_str(traceback.format_exc()))
             to_screen('ERROR: unable to download latest version')
             return
 
@@ -137,7 +137,7 @@ def update_self(to_screen, verbose, opener):
                 outf.write(newcontent)
         except (IOError, OSError):
             if verbose:
-                to_screen(compat_str(traceback.format_exc()))
+                to_screen(encode_compat_str(traceback.format_exc()))
             to_screen('ERROR: unable to write the new version')
             return
 
@@ -157,7 +157,7 @@ start /b "" cmd /c del "%%~f0"&exit /b"
             return  # Do not show premature success messages
         except (IOError, OSError):
             if verbose:
-                to_screen(compat_str(traceback.format_exc()))
+                to_screen(encode_compat_str(traceback.format_exc()))
             to_screen('ERROR: unable to overwrite current version')
             return
 
@@ -169,7 +169,7 @@ start /b "" cmd /c del "%%~f0"&exit /b"
             urlh.close()
         except (IOError, OSError):
             if verbose:
-                to_screen(compat_str(traceback.format_exc()))
+                to_screen(encode_compat_str(traceback.format_exc()))
             to_screen('ERROR: unable to download latest version')
             return
 
@@ -183,7 +183,7 @@ start /b "" cmd /c del "%%~f0"&exit /b"
                 outf.write(newcontent)
         except (IOError, OSError):
             if verbose:
-                to_screen(compat_str(traceback.format_exc()))
+                to_screen(encode_compat_str(traceback.format_exc()))
             to_screen('ERROR: unable to overwrite current version')
             return
 

From 2c74e6fa77935804b8e63807e23762e97a25f4d5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 20 Dec 2015 06:35:58 +0600
Subject: [PATCH 3423/6219] [YoutubeDL] Revert error_to_str for ExtractorError

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 1795097ae..bf05e9340 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -677,7 +677,7 @@ class YoutubeDL(object):
                 else:
                     return ie_result
             except ExtractorError as e:  # An error we somewhat expected
-                self.report_error(error_to_str(e), e.format_traceback())
+                self.report_error(compat_str(e), e.format_traceback())
                 break
             except MaxDownloadsReached:
                 raise

From d890b4cc0ac6b88daa917a1a791afe7541cd0411 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 20 Dec 2015 06:43:42 +0600
Subject: [PATCH 3424/6219] [nbc:news] Remove unnecessary compat_str

---
 youtube_dl/extractor/nbc.py | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index 4c1eca96f..340c922bd 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -3,10 +3,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_str,
-    compat_HTTPError,
-)
+from ..compat import compat_HTTPError
 from ..utils import (
     ExtractorError,
     find_xpath_attr,
@@ -189,7 +186,7 @@ class NBCNewsIE(InfoExtractor):
                 'title': info.find('headline').text,
                 'ext': 'flv',
                 'url': find_xpath_attr(info, 'media', 'type', 'flashVideo').text,
-                'description': compat_str(info.find('caption').text),
+                'description': info.find('caption').text,
                 'thumbnail': find_xpath_attr(info, 'media', 'type', 'thumbnail').text,
             }
         else:

From 9b9c5355e4c0fe6f9028aaa3b1b8beb28a085433 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 20 Dec 2015 07:00:39 +0600
Subject: [PATCH 3425/6219] Rename error_to_str to error_to_compat_str

---
 youtube_dl/YoutubeDL.py             | 10 +++++-----
 youtube_dl/downloader/common.py     |  4 ++--
 youtube_dl/extractor/common.py      |  6 +++---
 youtube_dl/extractor/dailymotion.py |  4 ++--
 youtube_dl/extractor/facebook.py    |  4 ++--
 youtube_dl/extractor/youtube.py     |  4 ++--
 youtube_dl/utils.py                 |  2 +-
 7 files changed, 17 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index bf05e9340..50425b8d7 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -49,7 +49,7 @@ from .utils import (
     DownloadError,
     encode_compat_str,
     encodeFilename,
-    error_to_str,
+    error_to_compat_str,
     ExtractorError,
     format_bytes,
     formatSeconds,
@@ -683,7 +683,7 @@ class YoutubeDL(object):
                 raise
             except Exception as e:
                 if self.params.get('ignoreerrors', False):
-                    self.report_error(error_to_str(e), tb=encode_compat_str(traceback.format_exc()))
+                    self.report_error(error_to_compat_str(e), tb=encode_compat_str(traceback.format_exc()))
                     break
                 else:
                     raise
@@ -1461,7 +1461,7 @@ class YoutubeDL(object):
             if dn and not os.path.exists(dn):
                 os.makedirs(dn)
         except (OSError, IOError) as err:
-            self.report_error('unable to create directory ' + error_to_str(err))
+            self.report_error('unable to create directory ' + error_to_compat_str(err))
             return
 
         if self.params.get('writedescription', False):
@@ -1512,7 +1512,7 @@ class YoutubeDL(object):
                             sub_info['url'], info_dict['id'], note=False)
                     except ExtractorError as err:
                         self.report_warning('Unable to download subtitle for "%s": %s' %
-                                            (sub_lang, error_to_str(err.cause)))
+                                            (sub_lang, error_to_compat_str(err.cause)))
                         continue
                 try:
                     sub_filename = subtitles_filename(filename, sub_lang, sub_format)
@@ -2041,4 +2041,4 @@ class YoutubeDL(object):
                                    (info_dict['extractor'], info_dict['id'], thumb_display_id, thumb_filename))
                 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
                     self.report_warning('Unable to download thumbnail "%s": %s' %
-                                        (t['url'], error_to_str(err)))
+                                        (t['url'], error_to_compat_str(err)))
diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index eb63ccffd..beae8c4d0 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -7,7 +7,7 @@ import time
 
 from ..utils import (
     encodeFilename,
-    error_to_str,
+    error_to_compat_str,
     decodeArgument,
     format_bytes,
     timeconvert,
@@ -186,7 +186,7 @@ class FileDownloader(object):
                 return
             os.rename(encodeFilename(old_filename), encodeFilename(new_filename))
         except (IOError, OSError) as err:
-            self.report_error('unable to rename file: %s' % error_to_str(err))
+            self.report_error('unable to rename file: %s' % error_to_compat_str(err))
 
     def try_utime(self, filename, last_modified_hdr):
         """Try to set the last-modified time of the given file."""
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 5659e40c8..828f58f12 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -30,7 +30,7 @@ from ..utils import (
     clean_html,
     compiled_regex_type,
     determine_ext,
-    error_to_str,
+    error_to_compat_str,
     ExtractorError,
     fix_xml_ampersands,
     float_or_none,
@@ -334,7 +334,7 @@ class InfoExtractor(object):
             if errnote is None:
                 errnote = 'Unable to download webpage'
 
-            errmsg = '%s: %s' % (errnote, error_to_str(err))
+            errmsg = '%s: %s' % (errnote, error_to_compat_str(err))
             if fatal:
                 raise ExtractorError(errmsg, sys.exc_info()[2], cause=err)
             else:
@@ -624,7 +624,7 @@ class InfoExtractor(object):
                 else:
                     raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE)
             except (IOError, netrc.NetrcParseError) as err:
-                self._downloader.report_warning('parsing .netrc: %s' % error_to_str(err))
+                self._downloader.report_warning('parsing .netrc: %s' % error_to_compat_str(err))
 
         return (username, password)
 
diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index 7d66baf96..0c5b6617f 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -9,7 +9,7 @@ from .common import InfoExtractor
 
 from ..utils import (
     determine_ext,
-    error_to_str,
+    error_to_compat_str,
     ExtractorError,
     int_or_none,
     parse_iso8601,
@@ -278,7 +278,7 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
                 'https://api.dailymotion.com/video/%s/subtitles?fields=id,language,url' % video_id,
                 video_id, note=False)
         except ExtractorError as err:
-            self._downloader.report_warning('unable to download video subtitles: %s' % error_to_str(err))
+            self._downloader.report_warning('unable to download video subtitles: %s' % error_to_compat_str(err))
             return {}
         info = json.loads(sub_list)
         if (info['total'] > 0):
diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index 296d3dd77..39c481068 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -11,7 +11,7 @@ from ..compat import (
     compat_urllib_parse_unquote,
 )
 from ..utils import (
-    error_to_str,
+    error_to_compat_str,
     ExtractorError,
     limit_length,
     sanitized_Request,
@@ -116,7 +116,7 @@ class FacebookIE(InfoExtractor):
             if re.search(r'id="checkpointSubmitButton"', check_response) is not None:
                 self._downloader.report_warning('Unable to confirm login, you have to login in your brower and authorize the login.')
         except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
-            self._downloader.report_warning('unable to log in: %s' % error_to_str(err))
+            self._downloader.report_warning('unable to log in: %s' % error_to_compat_str(err))
             return
 
     def _real_initialize(self):
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index d63aa5f4e..89759a1cb 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -26,7 +26,7 @@ from ..compat import (
 from ..utils import (
     clean_html,
     encode_dict,
-    error_to_str,
+    error_to_compat_str,
     ExtractorError,
     float_or_none,
     get_element_by_attribute,
@@ -904,7 +904,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'https://video.google.com/timedtext?hl=en&type=list&v=%s' % video_id,
                 video_id, note=False)
         except ExtractorError as err:
-            self._downloader.report_warning('unable to download video subtitles: %s' % error_to_str(err))
+            self._downloader.report_warning('unable to download video subtitles: %s' % error_to_compat_str(err))
             return {}
 
         sub_lang_list = {}
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 9a2dd1439..1737ac5f6 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1810,7 +1810,7 @@ def args_to_str(args):
     return ' '.join(shlex_quote(a) for a in args)
 
 
-def error_to_str(err):
+def error_to_compat_str(err):
     err_str = str(err)
     # On python 2 error byte string must be decoded with proper
     # encoding rather than ascii

From 6b77d52b1f8ca3777284e4164e5363c110aeba66 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 20 Dec 2015 07:07:14 +0600
Subject: [PATCH 3426/6219] [test_utils] Add tests for encode_compat_str

---
 test/test_utils.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/test/test_utils.py b/test/test_utils.py
index e4e8d3825..1c3290d9b 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -22,6 +22,7 @@ from youtube_dl.utils import (
     DateRange,
     detect_exe_version,
     determine_ext,
+    encode_compat_str,
     encodeFilename,
     escape_rfc3986,
     escape_url,
@@ -449,6 +450,10 @@ class TestUtil(unittest.TestCase):
         data = urlencode_postdata({'username': 'foo@bar.com', 'password': '1234'})
         self.assertTrue(isinstance(data, bytes))
 
+    def test_encode_compat_str(self):
+        self.assertEqual(encode_compat_str(b'\xd1\x82\xd0\xb5\xd1\x81\xd1\x82', 'utf-8'), 'тест')
+        self.assertEqual(encode_compat_str('тест', 'utf-8'), 'тест')
+
     def test_parse_iso8601(self):
         self.assertEqual(parse_iso8601('2014-03-23T23:04:26+0100'), 1395612266)
         self.assertEqual(parse_iso8601('2014-03-23T22:04:26+0000'), 1395612266)

From e462474e1d944e8b1547edc619a13a7ee2abbc2b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 20 Dec 2015 07:48:16 +0600
Subject: [PATCH 3427/6219] [youtube] Generalize playlists extractor

---
 youtube_dl/extractor/__init__.py |  2 +-
 youtube_dl/extractor/youtube.py  | 19 +++++++++++++++----
 2 files changed, 16 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 65c5e1c92..eac50eda5 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -852,7 +852,7 @@ from .youtube import (
     YoutubeTruncatedIDIE,
     YoutubeTruncatedURLIE,
     YoutubeUserIE,
-    YoutubeUserPlaylistsIE,
+    YoutubePlaylistsIE,
     YoutubeWatchLaterIE,
 )
 from .zapiks import ZapiksIE
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 89759a1cb..4aac2cc03 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1775,6 +1775,10 @@ class YoutubeChannelIE(YoutubePlaylistBaseInfoExtractor):
         },
     }]
 
+    @classmethod
+    def suitable(cls, url):
+        return False if YoutubePlaylistsIE.suitable(url) else super(YoutubeChannelIE, cls).suitable(url)
+
     def _real_extract(self, url):
         channel_id = self._match_id(url)
 
@@ -1848,10 +1852,10 @@ class YoutubeUserIE(YoutubeChannelIE):
             return super(YoutubeUserIE, cls).suitable(url)
 
 
-class YoutubeUserPlaylistsIE(YoutubePlaylistsBaseInfoExtractor):
-    IE_DESC = 'YouTube.com user playlists'
-    _VALID_URL = r'https?://(?:\w+\.)?youtube\.com/user/(?P<id>[^/]+)/playlists'
-    IE_NAME = 'youtube:user:playlists'
+class YoutubePlaylistsIE(YoutubePlaylistsBaseInfoExtractor):
+    IE_DESC = 'YouTube.com user/channel playlists'
+    _VALID_URL = r'https?://(?:\w+\.)?youtube\.com/(?:user|channel)/(?P<id>[^/]+)/playlists'
+    IE_NAME = 'youtube:playlists'
 
     _TESTS = [{
         'url': 'http://www.youtube.com/user/ThirstForScience/playlists',
@@ -1868,6 +1872,13 @@ class YoutubeUserPlaylistsIE(YoutubePlaylistsBaseInfoExtractor):
             'id': 'igorkle1',
             'title': 'Игорь Клейнер',
         },
+    }, {
+        'url': 'https://www.youtube.com/channel/UCiU1dHvZObB2iP6xkJ__Icw/playlists',
+        'playlist_mincount': 17,
+        'info_dict': {
+            'id': 'UCiU1dHvZObB2iP6xkJ__Icw',
+            'title': 'Chem Player',
+        },
     }]
 
 

From c6ed6fadc248bd4957fd7cccc1eeb0f109ac3ffa Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 20 Dec 2015 12:43:00 +0100
Subject: [PATCH 3428/6219] [cnet] improve extraction

- relex data json regex
- extract the platform metadata once
- extract hds formats
- extract duration
- extract thumbnail
---
 youtube_dl/extractor/cnet.py | 41 ++++++++++++++++++++----------------
 1 file changed, 23 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/cnet.py b/youtube_dl/extractor/cnet.py
index 3ecf0efd4..5c3908f72 100644
--- a/youtube_dl/extractor/cnet.py
+++ b/youtube_dl/extractor/cnet.py
@@ -1,33 +1,33 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import json
-
-from .common import InfoExtractor
 from .theplatform import ThePlatformIE
+from ..utils import int_or_none
 
 
-class CNETIE(InfoExtractor):
+class CNETIE(ThePlatformIE):
     _VALID_URL = r'https?://(?:www\.)?cnet\.com/videos/(?P<id>[^/]+)/'
     _TESTS = [{
         'url': 'http://www.cnet.com/videos/hands-on-with-microsofts-windows-8-1-update/',
         'info_dict': {
             'id': '56f4ea68-bd21-4852-b08c-4de5b8354c60',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': 'Hands-on with Microsoft Windows 8.1 Update',
             'description': 'The new update to the Windows 8 OS brings improved performance for mouse and keyboard users.',
             'uploader_id': '6085384d-619e-11e3-b231-14feb5ca9861',
             'uploader': 'Sarah Mitroff',
+            'duration': 70,
         },
     }, {
         'url': 'http://www.cnet.com/videos/whiny-pothole-tweets-at-local-government-when-hit-by-cars-tomorrow-daily-187/',
         'info_dict': {
             'id': '56527b93-d25d-44e3-b738-f989ce2e49ba',
-            'ext': 'mp4',
+            'ext': 'flv',
+            'title': 'Whiny potholes tweet at local government when hit by cars (Tomorrow Daily 187)',
             'description': 'Khail and Ashley wonder what other civic woes can be solved by self-tweeting objects, investigate a new kind of VR camera and watch an origami robot self-assemble, walk, climb, dig and dissolve. #TDPothole',
             'uploader_id': 'b163284d-6b73-44fc-b3e6-3da66c392d40',
             'uploader': 'Ashley Esqueda',
-            'title': 'Whiny potholes tweet at local government when hit by cars (Tomorrow Daily 187)',
+            'duration': 1482,
         },
     }]
 
@@ -36,10 +36,10 @@ class CNETIE(InfoExtractor):
         webpage = self._download_webpage(url, display_id)
 
         data_json = self._html_search_regex(
-            r"<div class=\"videoPlayer\"\s+.*?data-cnet-video-uvp-options='([^']+)'",
+            r"data-cnet-video(?:-uvp)?-options='([^']+)'",
             webpage, 'data json')
-        data = json.loads(data_json)
-        vdata = data['videos'][0]
+        data = self._parse_json(data_json, display_id)
+        vdata = data.get('video') or data['videos'][0]
 
         video_id = vdata['id']
         title = vdata['title']
@@ -52,19 +52,22 @@ class CNETIE(InfoExtractor):
             uploader_id = None
 
         mpx_account = data['config']['uvpConfig']['default']['mpx_account']
-        tp = ThePlatformIE(self._downloader)
+
+        metadata = self.get_metadata('%s/%s' % (mpx_account, list(vdata['files'].values())[0]), video_id)
+        description = vdata.get('description') or metadata.get('description')
+        duration = int_or_none(vdata.get('duration')) or metadata.get('duration')
+
         formats = []
         subtitles = {}
-        description = vdata.get('description')
-
         for (fkey, vid) in vdata['files'].items():
             if fkey == 'hls_phone' and 'hls_tablet' in vdata['files']:
                 continue
-            result = tp.extract(('http://link.theplatform.com/s/%s/%s' % (mpx_account, vid)))
-            formats.extend(result['formats'])
-            subtitles = self._merge_subtitles(subtitles, result['subtitles'])
-            description = description or result.get('description')
-
+            release_url = 'http://link.theplatform.com/s/%s/%s?format=SMIL&mbr=true' % (mpx_account, vid)
+            if fkey == 'hds':
+                release_url += '&manifest=f4m'
+            tp_formats, tp_subtitles = self._extract_theplatform_smil(release_url, video_id, 'Downloading %s SMIL data' % fkey)
+            formats.extend(tp_formats)
+            subtitles = self._merge_subtitles(subtitles, tp_subtitles)
         self._sort_formats(formats)
 
         return {
@@ -72,6 +75,8 @@ class CNETIE(InfoExtractor):
             'display_id': display_id,
             'title': title,
             'description': description,
+            'thumbnail': metadata.get('thumbnail'),
+            'duration': duration,
             'uploader': uploader,
             'uploader_id': uploader_id,
             'subtitles': subtitles,

From 6882c0870eb89a17eb81d3d273a9f9e42a8bea5e Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 20 Dec 2015 15:48:19 +0100
Subject: [PATCH 3429/6219] [tele13] improve extraction

- improve jwplayer setup regex
- sort formats
- remove duplicate formats
- update youtube test
---
 youtube_dl/extractor/tele13.py | 64 +++++++++++++++++-----------------
 1 file changed, 32 insertions(+), 32 deletions(-)

diff --git a/youtube_dl/extractor/tele13.py b/youtube_dl/extractor/tele13.py
index f1764eb2f..a363b4d40 100644
--- a/youtube_dl/extractor/tele13.py
+++ b/youtube_dl/extractor/tele13.py
@@ -1,10 +1,13 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
-from ..utils import js_to_json
+from .youtube import YoutubeIE
+from ..utils import (
+    js_to_json,
+    qualities,
+    determine_ext,
+)
 
 
 class Tele13IE(InfoExtractor):
@@ -25,12 +28,12 @@ class Tele13IE(InfoExtractor):
         },
         {
             'url': 'http://www.t13.cl/videos/mundo/tendencias/video-captan-misteriosa-bola-fuego-cielos-bangkok',
-            'md5': '65d1ae54812c96f4b345dd21d3bb1adc',
+            'md5': '867adf6a3b3fef932c68a71d70b70946',
             'info_dict': {
                 'id': 'rOoKv2OMpOw',
                 'ext': 'mp4',
                 'title': 'Shooting star seen on 7-Sep-2015',
-                'description': 'md5:a1cd2e74f6ee6851552c9cf5851d6b06',
+                'description': 'md5:7292ff2a34b2f673da77da222ae77e1e',
                 'uploader': 'Porjai Jaturongkhakun',
                 'upload_date': '20150906',
                 'uploader_id': 'UCnLY_3ezwNcDSC_Wc6suZxw',
@@ -41,41 +44,38 @@ class Tele13IE(InfoExtractor):
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
-
         webpage = self._download_webpage(url, display_id)
 
-        setup_js = self._parse_json(
-            js_to_json(
-                self._search_regex(
-                    r"jwplayer\('player-vivo'\).setup\((\{.*?\})\)",
-                    webpage,
-                    'setup code',
-                    flags=re.DOTALL
-                ).replace('\n//', '')
-            ),
-            display_id
-        )
-        title = setup_js['title']
-        thumbnail = setup_js.get('image') or setup_js['playlist'][0].get('image')
-        description = self._html_search_meta(
-            'description', webpage, 'description')
+        setup_js = self._search_regex(r"(?s)jwplayer\('player-vivo'\).setup\((\{.*?\})\)", webpage, 'setup code')
+        sources = self._parse_json(self._search_regex(r'sources\s*:\s*(\[[^\]]+\])', setup_js, 'sources'), display_id, js_to_json)
 
+        preference = qualities(['Móvil', 'SD', 'HD'])
         formats = []
-        for f in setup_js['playlist'][0]['sources']:
+        urls = []
+        for f in sources:
             format_url = f['file']
-            if format_url != '':
-                if '.m3u8' in format_url:
-                    formats.extend(self._extract_m3u8_formats(format_url, display_id))
+            if format_url and format_url not in urls:
+                ext = determine_ext(format_url)
+                if ext == 'm3u8':
+                    m3u8_formats = self._extract_m3u8_formats(format_url, display_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False)
+                    if m3u8_formats:
+                        formats.extend(m3u8_formats)
+                elif YoutubeIE.suitable(format_url):
+                    return self.url_result(format_url, 'Youtube')
                 else:
-                    if 'youtube.com' in format_url:
-                        return self.url_result(format_url, 'Youtube')
-                    else:
-                        formats.append({'url': format_url, 'format_id': f.get('label')})
+                    formats.append({
+                        'url': format_url,
+                        'format_id': f.get('label'),
+                        'preference': preference(f.get('label')),
+                        'ext': ext,
+                    })
+                urls.append(format_url)
+        self._sort_formats(formats)
 
         return {
             'id': display_id,
-            'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
+            'title': self._search_regex(r'title\s*:\s*"([^"]+)"', setup_js, 'title'),
+            'description': self._html_search_meta('description', webpage, 'description'),
+            'thumbnail': self._search_regex(r'image\s*:\s*"([^"]+)"', setup_js, 'thumbnail', default=None),
             'formats': formats,
         }

From ecbccea703cc7812b66c6dd3a543d60b5be8aa48 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 20 Dec 2015 21:38:30 +0100
Subject: [PATCH 3430/6219] [faz] extract duration and bitrate and use
 xpath_element and xpath_text for extraction

---
 youtube_dl/extractor/faz.py | 36 +++++++++++++++++++++---------------
 1 file changed, 21 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/faz.py b/youtube_dl/extractor/faz.py
index d9a868119..6f9b003c2 100644
--- a/youtube_dl/extractor/faz.py
+++ b/youtube_dl/extractor/faz.py
@@ -2,6 +2,11 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..utils import (
+    xpath_element,
+    xpath_text,
+    int_or_none,
+)
 
 
 class FazIE(InfoExtractor):
@@ -37,31 +42,32 @@ class FazIE(InfoExtractor):
         video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id)
+        description = self._og_search_description(webpage)
         config_xml_url = self._search_regex(
-            r'(?:var\s+)?videoXMLURL\s*=\s*"([^"]+)', webpage, 'config xml url')
+            r'videoXMLURL\s*=\s*"([^"]+)', webpage, 'config xml url')
         config = self._download_xml(
             config_xml_url, video_id, 'Downloading config xml')
 
-        encodings = config.find('ENCODINGS')
+        encodings = xpath_element(config, 'ENCODINGS', 'encodings', True)
         formats = []
         for pref, code in enumerate(['LOW', 'HIGH', 'HQ']):
-            encoding = encodings.find(code)
-            if encoding is None:
-                continue
-            encoding_url = encoding.find('FILENAME').text
-            formats.append({
-                'url': encoding_url,
-                'format_id': code.lower(),
-                'quality': pref,
-            })
+            encoding = xpath_element(encodings, code)
+            if encoding:
+                encoding_url = xpath_text(encoding, 'FILENAME')
+                if encoding_url:
+                    formats.append({
+                        'url': encoding_url,
+                        'format_id': code.lower(),
+                        'quality': pref,
+                        'tbr': int_or_none(xpath_text(encoding, 'AVERAGEBITRATE')),
+                    })
         self._sort_formats(formats)
 
-        descr = self._html_search_regex(
-            r'<p class="Content Copy">(.*?)</p>', webpage, 'description', fatal=False)
         return {
             'id': video_id,
             'title': self._og_search_title(webpage),
             'formats': formats,
-            'description': descr,
-            'thumbnail': config.find('STILL/STILL_BIG').text,
+            'description': description.strip() if description else None,
+            'thumbnail': xpath_text(config, 'STILL/STILL_BIG'),
+            'duration': int_or_none(xpath_text(config, 'DURATION')),
         }

From 5b95419ca59e6238ab0041d48a53a9e51485e1cc Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 21 Dec 2015 00:20:22 +0100
Subject: [PATCH 3431/6219] [flickr] extract views_count and tags

---
 youtube_dl/extractor/flickr.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/flickr.py b/youtube_dl/extractor/flickr.py
index 92d2ac553..452b27b26 100644
--- a/youtube_dl/extractor/flickr.py
+++ b/youtube_dl/extractor/flickr.py
@@ -25,6 +25,8 @@ class FlickrIE(InfoExtractor):
             'uploader_id': '10922353@N03',
             'uploader': 'Forest Wander',
             'comment_count': int,
+            'view_count': int,
+            'tags': list,
         }
     }
 
@@ -78,6 +80,8 @@ class FlickrIE(InfoExtractor):
                 'uploader_id': owner.get('nsid'),
                 'uploader': owner.get('realname'),
                 'comment_count': int_or_none(video_info.get('comments', {}).get('_content')),
+                'view_count': int_or_none(video_info.get('views')),
+                'tags': [tag.get('_content') for tag in video_info.get('tags', {}).get('tag', [])]
             }
         else:
             raise ExtractorError('not a video', expected=True)

From 5b251628e9f45c89c1becb3f62c4212874eb74ea Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 21 Dec 2015 03:05:34 +0100
Subject: [PATCH 3432/6219] [googledrive] Modernize

---
 youtube_dl/extractor/__init__.py    |   5 +-
 youtube_dl/extractor/generic.py     |   4 +-
 youtube_dl/extractor/googledrive.py | 146 ++++++++++------------------
 3 files changed, 54 insertions(+), 101 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 02e18a0da..6655d7eb5 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -209,10 +209,7 @@ from .globo import GloboIE
 from .godtube import GodTubeIE
 from .goldenmoustache import GoldenMoustacheIE
 from .golem import GolemIE
-from .googledrive import (
-    GoogleDriveEmbedIE,
-    GoogleDriveIE,
-)
+from .googledrive import GoogleDriveIE
 from .googleplus import GooglePlusIE
 from .googlesearch import GoogleSearchIE
 from .gorillavid import GorillaVidIE
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 3f7b094db..abd98e500 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -48,7 +48,7 @@ from .vimeo import VimeoIE
 from .dailymotion import DailymotionCloudIE
 from .onionstudios import OnionStudiosIE
 from .snagfilms import SnagFilmsEmbedIE
-from .googledrive import GoogleDriveEmbedIE
+from .googledrive import GoogleDriveIE
 
 
 class GenericIE(InfoExtractor):
@@ -1601,7 +1601,7 @@ class GenericIE(InfoExtractor):
             return self.url_result(nbc_sports_url, 'NBCSportsVPlayer')
 
         # Look for Google Drive embeds
-        google_drive_url = GoogleDriveEmbedIE._extract_url(webpage)
+        google_drive_url = GoogleDriveIE._extract_url(webpage)
         if google_drive_url:
             return self.url_result(google_drive_url, 'GoogleDrive')
 
diff --git a/youtube_dl/extractor/googledrive.py b/youtube_dl/extractor/googledrive.py
index 7bc7b7a0d..f354c9c7a 100644
--- a/youtube_dl/extractor/googledrive.py
+++ b/youtube_dl/extractor/googledrive.py
@@ -1,132 +1,88 @@
+from __future__ import unicode_literals
+
 import re
 
 from .common import InfoExtractor
 from ..utils import (
-    RegexNotFoundError,
     ExtractorError,
+    int_or_none,
 )
 
-class GoogleDriveEmbedIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:video\.google\.com/get_player\?.*?docid=|(?:docs|drive)\.google\.com/file/d/)(?P<id>[a-zA-Z0-9_-]{28})'
+
+class GoogleDriveIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:(?:docs|drive)\.google\.com/(?:uc\?.*?id=|file/d/)|video\.google\.com/get_player\?.*?docid=)(?P<id>[a-zA-Z0-9_-]{28})'
     _TEST = {
-        'url': 'https://docs.google.com/file/d/0B8KB9DRosYGKMXNoeWxqa3JYclE/preview',
+        'url': 'https://drive.google.com/file/d/0ByeS4oOUV-49Zzh4R1J6R09zazQ/edit?pli=1',
+        'md5': '881f7700aec4f538571fa1e0eed4a7b6',
         'info_dict': {
-            'id': '0B8KB9DRosYGKMXNoeWxqa3JYclE',
+            'id': '0ByeS4oOUV-49Zzh4R1J6R09zazQ',
             'ext': 'mp4',
-            'title': 'Jimmy Fallon Sings Since You\'ve Been Gone.wmv',
+            'title': 'Big Buck Bunny.mp4',
+            'duration': 46,
         }
     }
+    _FORMATS_EXT = {
+        '5': 'flv',
+        '6': 'flv',
+        '13': '3gp',
+        '17': '3gp',
+        '18': 'mp4',
+        '22': 'mp4',
+        '34': 'flv',
+        '35': 'flv',
+        '36': '3gp',
+        '37': 'mp4',
+        '38': 'mp4',
+        '43': 'webm',
+        '44': 'webm',
+        '45': 'webm',
+        '46': 'webm',
+        '59': 'mp4',
+    }
 
     @staticmethod
     def _extract_url(webpage):
         mobj = re.search(
-            r'<iframe src="https?://(?:video\.google\.com/get_player\?.*?docid=|(?:docs|drive)\.google\.com/file/d/)(?P<id>[a-zA-Z0-9_-]{28})',
+            r'<iframe[^>]+src="https?://(?:video\.google\.com/get_player\?.*?docid=|(?:docs|drive)\.google\.com/file/d/)(?P<id>[a-zA-Z0-9_-]{28})',
             webpage)
         if mobj:
             return 'https://drive.google.com/file/d/%s' % mobj.group('id')
 
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        return {
-            '_type': 'url',
-            'ie_key': 'GoogleDrive',
-            'url': 'https://drive.google.com/file/d/%s' % video_id
-        }
-
-class GoogleDriveIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:docs|drive)\.google\.com/(?:uc\?.*?id=|file/d/)(?P<id>[a-zA-Z0-9_-]{28})'
-    _TEST = {
-        'url': 'https://drive.google.com/file/d/0ByeS4oOUV-49Zzh4R1J6R09zazQ/edit?pli=1',
-        'info_dict': {
-            'id': '0ByeS4oOUV-49Zzh4R1J6R09zazQ',
-            'ext': 'mp4',
-            'title': 'Big Buck Bunny.mp4',
-        }
-    }
-    _formats = {
-        '5': {'ext': 'flv'},
-        '6': {'ext': 'flv'},
-        '13': {'ext': '3gp'},
-        '17': {'ext': '3gp'},
-        '18': {'ext': 'mp4'},
-        '22': {'ext': 'mp4'},
-        '34': {'ext': 'flv'},
-        '35': {'ext': 'flv'},
-        '36': {'ext': '3gp'},
-        '37': {'ext': 'mp4'},
-        '38': {'ext': 'mp4'},
-        '43': {'ext': 'webm'},
-        '44': {'ext': 'webm'},
-        '45': {'ext': 'webm'},
-        '46': {'ext': 'webm'},
-        '59': {'ext': 'mp4'}
-    }
-
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(
-            'http://docs.google.com/file/d/' + video_id, video_id, encoding='unicode_escape'
-        )
-        try:
-            title = self._html_search_regex(
-                r'"title"\s*,\s*"([^"]+)',
-                webpage,
-                'title'
-            )
-            fmt_stream_map = self._html_search_regex(
-                r'"fmt_stream_map"\s*,\s*"([^"]+)',
-                webpage,
-                'fmt_stream_map'
-            )
-            fmt_list = self._html_search_regex(
-                r'"fmt_list"\s*,\s*"([^"]+)',
-                webpage,
-                'fmt_list'
-            )
-#			timestamp = self._html_search_regex(
-#				r'"timestamp"\s*,\s*"([^"]+)',
-#				webpage,
-#				'timestamp'
-#			)
-            length_seconds = self._html_search_regex(
-                r'"length_seconds"\s*,\s*"([^"]+)',
-                webpage,
-                'length_seconds'
-            )
-        except RegexNotFoundError:
-            try:
-                reason = self._html_search_regex(
-                    r'"reason","([^"]+)',
-                    webpage,
-                    'reason'
-                )
-                raise ExtractorError(reason)
-                return
-            except RegexNotFoundError:
-                raise ExtractorError('not a video')
-                return
+            'http://docs.google.com/file/d/%s' % video_id, video_id, encoding='unicode_escape')
+
+        reason = self._search_regex(r'"reason"\s*,\s*"([^"]+)', webpage, 'reason', default=None)
+        if reason:
+            raise ExtractorError(reason)
+
+        title = self._search_regex(r'"title"\s*,\s*"([^"]+)', webpage, 'title')
+        duration = int_or_none(self._search_regex(
+            r'"length_seconds"\s*,\s*"([^"]+)', webpage, 'length seconds', default=None))
+        fmt_stream_map = self._search_regex(
+            r'"fmt_stream_map"\s*,\s*"([^"]+)', webpage, 'fmt stream map').split(',')
+        fmt_list = self._search_regex(r'"fmt_list"\s*,\s*"([^"]+)', webpage, 'fmt_list').split(',')
 
-        fmt_stream_map = fmt_stream_map.split(',')
-        fmt_list = fmt_list.split(',')
         formats = []
-        for i in range(len(fmt_stream_map)):
-            fmt_id, fmt_url = fmt_stream_map[i].split('|')
-            resolution = fmt_list[i].split('/')[1]
+        for fmt, fmt_stream in zip(fmt_list, fmt_stream_map):
+            fmt_id, fmt_url = fmt_stream.split('|')
+            resolution = fmt.split('/')[1]
             width, height = resolution.split('x')
             formats.append({
                 'url': fmt_url,
                 'format_id': fmt_id,
                 'resolution': resolution,
-                'width': int(width),
-                'height': int(height),
-                'ext': self._formats[fmt_id]['ext']
+                'width': int_or_none(width),
+                'height': int_or_none(height),
+                'ext': self._FORMATS_EXT[fmt_id],
             })
         self._sort_formats(formats)
 
         return {
             'id': video_id,
             'title': title,
-#           'timestamp': int(timestamp),
-            'duration': int(length_seconds),
-            'formats': formats
+            'thumbnail': self._og_search_thumbnail(webpage),
+            'duration': duration,
+            'formats': formats,
         }

From 7cb09524749d4a061acc801d0a2d6ad08463e549 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 21 Dec 2015 04:24:58 +0100
Subject: [PATCH 3433/6219] [makertv] improve extraction

---
 youtube_dl/extractor/generic.py    |  6 ++++++
 youtube_dl/extractor/jwplatform.py | 10 +++++++---
 youtube_dl/extractor/makertv.py    | 13 +++++++++----
 3 files changed, 22 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 4d38b0c9d..f5dd88f54 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -53,6 +53,7 @@ from .onionstudios import OnionStudiosIE
 from .snagfilms import SnagFilmsEmbedIE
 from .screenwavemedia import ScreenwaveMediaIE
 from .mtv import MTVServicesEmbeddedIE
+from .jwplatform import JWPlatformIE
 
 
 class GenericIE(InfoExtractor):
@@ -1787,6 +1788,11 @@ class GenericIE(InfoExtractor):
         if snagfilms_url:
             return self.url_result(snagfilms_url)
 
+        # Look for JWPlatform embeds
+        jwplatform_url = JWPlatformIE._extract_url(webpage)
+        if jwplatform_url:
+            return self.url_result(jwplatform_url, 'JWPlatform')
+
         # Look for ScreenwaveMedia embeds
         mobj = re.search(ScreenwaveMediaIE.EMBED_PATTERN, webpage)
         if mobj is not None:
diff --git a/youtube_dl/extractor/jwplatform.py b/youtube_dl/extractor/jwplatform.py
index 3a3dc439a..cdc095a79 100644
--- a/youtube_dl/extractor/jwplatform.py
+++ b/youtube_dl/extractor/jwplatform.py
@@ -1,6 +1,8 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..utils import int_or_none
 
@@ -23,7 +25,7 @@ class JWPlatformIE(InfoExtractor):
     @staticmethod
     def _extract_url(webpage):
         mobj = re.search(
-            r'<script[^>]+?src=["\'](?P<url>(?:https?:)?//content.jwplatform.com/players/[a-zA-Z0-9]{8}',
+            r'<script[^>]+?src=["\'](?P<url>(?:https?:)?//content.jwplatform.com/players/[a-zA-Z0-9]{8})',
             webpage)
         if mobj:
             return mobj.group('url')
@@ -42,7 +44,9 @@ class JWPlatformIE(InfoExtractor):
             source_url = self._proto_relative_url(source['file'])
             source_type = source.get('type') or ''
             if source_type == 'application/vnd.apple.mpegurl':
-                formats.extend(self._extract_m3u8_formats(source_url, video_id, 'mp4', 'm3u8_native', fatal=None))
+                m3u8_formats = self._extract_m3u8_formats(source_url, video_id, 'mp4', 'm3u8_native', fatal=None)
+                if m3u8_formats:
+                    formats.extend(m3u8_formats)
             elif source_type.startswith('audio'):
                 formats.append({
                     'url': source_url,
@@ -57,7 +61,7 @@ class JWPlatformIE(InfoExtractor):
         self._sort_formats(formats)
 
         return {
-            'id': video_data['mediaid'],
+            'id': video_id,
             'title': video_data['title'],
             'description': video_data.get('description'),
             'thumbnail': self._proto_relative_url(video_data.get('image')),
diff --git a/youtube_dl/extractor/makertv.py b/youtube_dl/extractor/makertv.py
index 0256e4e24..3c34d4604 100644
--- a/youtube_dl/extractor/makertv.py
+++ b/youtube_dl/extractor/makertv.py
@@ -5,12 +5,12 @@ from .common import InfoExtractor
 
 
 class MakerTVIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:(?:www\.)?maker\.tv/(?:[^/]+/)?video|http://makerplayer.com/embed/maker)/(?P<id>[a-zA-Z0-9]{12})'
+    _VALID_URL = r'https?://(?:(?:www\.)?maker\.tv/(?:[^/]+/)*video|makerplayer.com/embed/maker)/(?P<id>[a-zA-Z0-9]{12})'
     _TEST = {
         'url': 'http://www.maker.tv/video/Fh3QgymL9gsc',
         'md5': 'ca237a53a8eb20b6dc5bd60564d4ab3e',
         'info_dict': {
-            'id': 'brOEcGut',
+            'id': 'Fh3QgymL9gsc',
             'ext': 'mp4',
             'title': 'Maze Runner: The Scorch Trials Official Movie Review',
             'description': 'md5:11ff3362d7ef1d679fdb649f6413975a',
@@ -22,6 +22,11 @@ class MakerTVIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
-        jwplatform_id = self._search_regex([r'jwid="([^"]+)"', r'Maker.jw_id\s*=\s*"([^"]+)";'], webpage, 'jwplatform id')
+        jwplatform_id = self._search_regex(r'jw_?id="([^"]+)"', webpage, 'jwplatform id')
 
-        return self.url_result('jwplatform:%s' % jwplatform_id, 'JWPlatform')
+        return {
+            '_type': 'url_transparent',
+            'id': video_id,
+            'url': 'jwplatform:%s' % jwplatform_id,
+            'ie_key': 'JWPlatform',
+        }

From c7fa5fa42cb91aa32c5b21c6821f6d190ec047a2 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 21 Dec 2015 11:12:58 +0100
Subject: [PATCH 3434/6219] [bleacherreport] fix style issues and simplify

---
 youtube_dl/extractor/amp.py            | 38 +++++++++++++-------------
 youtube_dl/extractor/bleacherreport.py | 19 ++-----------
 youtube_dl/extractor/dramafever.py     |  6 ++--
 youtube_dl/extractor/foxnews.py        | 19 +++++--------
 4 files changed, 30 insertions(+), 52 deletions(-)

diff --git a/youtube_dl/extractor/amp.py b/youtube_dl/extractor/amp.py
index b573b9280..dcc3c97f1 100644
--- a/youtube_dl/extractor/amp.py
+++ b/youtube_dl/extractor/amp.py
@@ -9,23 +9,21 @@ from ..utils import (
 
 
 class AMPIE(InfoExtractor):
-    def _get_media_node(self, item, name, default=None):
-        media_name = 'media-%s' % name
-        media_group = item.get('media-group') or item
-        return media_group.get(media_name) or item.get(media_name) or item.get(name, default)
-
     # parse Akamai Adaptive Media Player feed
     def _extract_feed_info(self, url):
         item = self._download_json(
-            url, None,
-            'Downloading Akamai AMP feed',
-            'Unable to download Akamai AMP feed'
-            )['channel']['item']
+            url, None, 'Downloading Akamai AMP feed',
+            'Unable to download Akamai AMP feed')['channel']['item']
 
         video_id = item['guid']
-        
+
+        def get_media_node(name, default=None):
+            media_name = 'media-%s' % name
+            media_group = item.get('media-group') or item
+            return media_group.get(media_name) or item.get(media_name) or item.get(name, default)
+
         thumbnails = []
-        media_thumbnail = self._get_media_node(item, 'thumbnail')
+        media_thumbnail = get_media_node('thumbnail')
         if media_thumbnail:
             if isinstance(media_thumbnail, dict):
                 media_thumbnail = [media_thumbnail]
@@ -38,7 +36,7 @@ class AMPIE(InfoExtractor):
                 })
 
         subtitles = {}
-        media_subtitle = self._get_media_node(item, 'subTitle')
+        media_subtitle = get_media_node('subTitle')
         if media_subtitle:
             if isinstance(media_subtitle, dict):
                 media_subtitle = [media_subtitle]
@@ -48,26 +46,28 @@ class AMPIE(InfoExtractor):
                 subtitles[lang] = [{'url': subtitle['href']}]
 
         formats = []
-        media_content = self._get_media_node(item, 'content')
+        media_content = get_media_node('content')
         if isinstance(media_content, dict):
             media_content = [media_content]
         for media_data in media_content:
             media = media_data['@attributes']
             media_type = media['type']
             if media_type == 'video/f4m':
-                f4m_formats = self._extract_f4m_formats(media['url'] + '?hdcore=3.4.0&plugin=aasp-3.4.0.132.124', video_id, f4m_id='hds', fatal=False)
+                f4m_formats = self._extract_f4m_formats(
+                    media['url'] + '?hdcore=3.4.0&plugin=aasp-3.4.0.132.124',
+                    video_id, f4m_id='hds', fatal=False)
                 if f4m_formats:
                     formats.extend(f4m_formats)
             elif media_type == 'application/x-mpegURL':
-                m3u8_formats = self._extract_m3u8_formats(media['url'], video_id, m3u8_id='hls', fatal=False)
+                m3u8_formats = self._extract_m3u8_formats(
+                    media['url'], video_id, 'mp4', m3u8_id='hls', fatal=False)
                 if m3u8_formats:
                     formats.extend(m3u8_formats)
             else:
                 formats.append({
                     'format_id': media_data['media-category']['@attributes']['label'],
                     'url': media['url'],
-                    'preference': 1,
-                    'vbr': int_or_none(media.get('bitrate')),
+                    'tbr': int_or_none(media.get('bitrate')),
                     'filesize': int_or_none(media.get('fileSize')),
                 })
 
@@ -75,8 +75,8 @@ class AMPIE(InfoExtractor):
 
         return {
             'id': video_id,
-            'title': self._get_media_node(item, 'title'),
-            'description': self._get_media_node(item, 'description'),
+            'title': get_media_node('title'),
+            'description': get_media_node('description'),
             'thumbnails': thumbnails,
             'timestamp': parse_iso8601(item.get('pubDate'), ' '),
             'duration': int_or_none(media_content[0].get('@attributes', {}).get('duration')),
diff --git a/youtube_dl/extractor/bleacherreport.py b/youtube_dl/extractor/bleacherreport.py
index a55e696d2..bd2a6340b 100644
--- a/youtube_dl/extractor/bleacherreport.py
+++ b/youtube_dl/extractor/bleacherreport.py
@@ -26,7 +26,7 @@ class BleacherReportIE(InfoExtractor):
             'uploader': 'Team Stream Now ',
         },
         'add_ie': ['Ooyala'],
-    },{
+    }, {
         'url': 'http://bleacherreport.com/articles/2586817-aussie-golfers-get-fright-of-their-lives-after-being-chased-by-angry-kangaroo',
         'md5': 'af5f90dc9c7ba1c19d0a3eac806bbf50',
         'info_dict': {
@@ -35,25 +35,11 @@ class BleacherReportIE(InfoExtractor):
             'title': 'Aussie Golfers Get Fright of Their Lives After Being Chased by Angry Kangaroo',
             'timestamp': 1446839961,
             'uploader': 'Sean Fay',
-            'description': 'md5:e95afafa43619816552723878b3b0a84',
+            'description': 'md5:825e94e0f3521df52fa83b2ed198fa20',
             'uploader_id': 6466954,
             'upload_date': '20151011',
         },
         'add_ie': ['Youtube'],
-    },{
-        'url': 'http://bleacherreport.com/articles/2496438-fsu-stat-projections-is-jalen-ramsey-best-defensive-player-in-college-football',
-        'md5': 'a3ffc3dc73afdbc2010f02d98f990f20',
-        'info_dict': {
-            'id': '2496438',
-            'ext': 'mp4',
-            'title': 'FSU Stat Projections: Is Jalen Ramsey Best Defensive Player in College Football?',
-            'upload_date': '20150615',
-            'uploader': 'Team Stream Now ',
-            'timestamp': 1434380212,
-            'description': 'CFB, ACC, Florida State',
-            'uploader_id': 3992341,
-        },
-        'add_ie': ['Vine'],
     }]
 
     def _real_extract(self, url):
@@ -115,7 +101,6 @@ class BleacherReportCMSIE(AMPIE):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-
         info = self._extract_feed_info('http://cms.bleacherreport.com/media/items/%s/akamai.json' % video_id)
         info['id'] = video_id
         return info
diff --git a/youtube_dl/extractor/dramafever.py b/youtube_dl/extractor/dramafever.py
index 80a928827..2a4310754 100644
--- a/youtube_dl/extractor/dramafever.py
+++ b/youtube_dl/extractor/dramafever.py
@@ -13,9 +13,6 @@ from ..compat import (
 from ..utils import (
     ExtractorError,
     clean_html,
-    determine_ext,
-    int_or_none,
-    parse_iso8601,
 )
 
 
@@ -91,7 +88,8 @@ class DramaFeverIE(DramaFeverBaseIE):
         video_id = self._match_id(url).replace('/', '.')
 
         try:
-            info = self._extract_feed_info('http://www.dramafever.com/amp/episode/feed.json?guid=%s' % video_id)
+            info = self._extract_feed_info(
+                'http://www.dramafever.com/amp/episode/feed.json?guid=%s' % video_id)
         except ExtractorError as e:
             if isinstance(e.cause, compat_HTTPError):
                 raise ExtractorError(
diff --git a/youtube_dl/extractor/foxnews.py b/youtube_dl/extractor/foxnews.py
index 0cd0f9fa8..318ac013d 100644
--- a/youtube_dl/extractor/foxnews.py
+++ b/youtube_dl/extractor/foxnews.py
@@ -3,10 +3,6 @@ from __future__ import unicode_literals
 import re
 
 from .amp import AMPIE
-from ..utils import (
-    parse_iso8601,
-    int_or_none,
-)
 
 
 class FoxNewsIE(AMPIE):
@@ -22,8 +18,8 @@ class FoxNewsIE(AMPIE):
                 'title': 'Frozen in Time',
                 'description': '16-year-old girl is size of toddler',
                 'duration': 265,
-                #'timestamp': 1304411491,
-                #'upload_date': '20110503',
+                # 'timestamp': 1304411491,
+                # 'upload_date': '20110503',
                 'thumbnail': 're:^https?://.*\.jpg$',
             },
         },
@@ -36,8 +32,8 @@ class FoxNewsIE(AMPIE):
                 'title': "Rep. Luis Gutierrez on if Obama's immigration plan is legal",
                 'description': "Congressman discusses president's plan",
                 'duration': 292,
-                #'timestamp': 1417662047,
-                #'upload_date': '20141204',
+                # 'timestamp': 1417662047,
+                # 'upload_date': '20141204',
                 'thumbnail': 're:^https?://.*\.jpg$',
             },
         },
@@ -52,10 +48,9 @@ class FoxNewsIE(AMPIE):
     ]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        host = mobj.group('host')
+        host, video_id = re.match(self._VALID_URL, url).groups()
 
-        info = self._extract_feed_info('http://%s/v/feed/video/%s.js?template=fox' % (host, video_id))
+        info = self._extract_feed_info(
+            'http://%s/v/feed/video/%s.js?template=fox' % (host, video_id))
         info['id'] = video_id
         return info

From 0197004f786eba88238c5dd638dc1923b85b2b4c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 21 Dec 2015 11:42:25 +0100
Subject: [PATCH 3435/6219] release 2015.12.21

---
 docs/supportedsites.md | 13 +++++++++----
 youtube_dl/version.py  |  2 +-
 2 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 8253335e3..299bc5e72 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -65,9 +65,9 @@
  - **Bet**
  - **Bild**: Bild.de
  - **BiliBili**
+ - **BleacherReport**
+ - **BleacherReportCMS**
  - **blinkx**
- - **blip.tv:user**
- - **BlipTV**
  - **Bloomberg**
  - **Bpb**: Bundeszentrale für politische Bildung
  - **BR**: Bayerischer Rundfunk Mediathek
@@ -80,7 +80,6 @@
  - **BYUtv**
  - **Camdemy**
  - **CamdemyFolder**
- - **Canal13cl**
  - **canalc2.tv**
  - **Canalplus**: canalplus.fr, piwiplus.fr and d8.tv
  - **CBS**
@@ -210,7 +209,9 @@
  - **GodTube**
  - **GoldenMoustache**
  - **Golem**
+ - **GoogleDrive**
  - **Goshgay**
+ - **GPUTechConf**
  - **Groupon**
  - **Hark**
  - **HearThisAt**
@@ -252,6 +253,7 @@
  - **Jove**
  - **jpopsuki.tv**
  - **Jukebox**
+ - **JWPlatform**
  - **Kaltura**
  - **KanalPlay**: Kanal 5/9/11 Play
  - **Kankan**
@@ -292,6 +294,7 @@
  - **m6**
  - **macgamestore**: MacGameStore trailers
  - **mailru**: Видео@Mail.Ru
+ - **MakerTV**
  - **Malemotion**
  - **MDR**: MDR.DE and KiKA
  - **media.ccc.de**
@@ -551,6 +554,7 @@
  - **TechTalks**
  - **techtv.mit.edu**
  - **ted**
+ - **Tele13**
  - **TeleBruxelles**
  - **Telecinco**: telecinco.es, cuatro.com and mediaset.es
  - **Telegraaf**
@@ -573,6 +577,7 @@
  - **TMZ**
  - **TMZArticle**
  - **TNAFlix**
+ - **toggle**
  - **tou.tv**
  - **Toypics**: Toypics user profile
  - **ToypicsUser**: Toypics user profile
@@ -711,6 +716,7 @@
  - **youtube:favorites**: YouTube.com favourite videos, ":ytfav" for short (requires authentication)
  - **youtube:history**: Youtube watch history, ":ythistory" for short (requires authentication)
  - **youtube:playlist**: YouTube.com playlists
+ - **youtube:playlists**: YouTube.com user/channel playlists
  - **youtube:recommended**: YouTube.com recommended videos, ":ytrec" for short (requires authentication)
  - **youtube:search**: YouTube.com searches
  - **youtube:search:date**: YouTube.com searches, newest videos first
@@ -718,7 +724,6 @@
  - **youtube:show**: YouTube.com (multi-season) shows
  - **youtube:subscriptions**: YouTube.com subscriptions feed, "ytsubs" keyword (requires authentication)
  - **youtube:user**: YouTube.com user videos (URL or "ytuser" keyword)
- - **youtube:user:playlists**: YouTube.com user playlists
  - **youtube:watchlater**: Youtube watch later list, ":ytwatchlater" for short (requires authentication)
  - **Zapiks**
  - **ZDF**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 01607693e..7095033c5 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.12.18'
+__version__ = '2015.12.21'

From 5c5a3ecf1b5337510378b782a14525584769e6d6 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 21 Dec 2015 13:07:52 +0100
Subject: [PATCH 3436/6219] [abc] detect expired state and update tests

---
 youtube_dl/extractor/abc.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/abc.py b/youtube_dl/extractor/abc.py
index c0e5d1abf..6a29e587f 100644
--- a/youtube_dl/extractor/abc.py
+++ b/youtube_dl/extractor/abc.py
@@ -23,6 +23,7 @@ class ABCIE(InfoExtractor):
             'title': 'Australia to help staff Ebola treatment centre in Sierra Leone',
             'description': 'md5:809ad29c67a05f54eb41f2a105693a67',
         },
+        'skip': 'this video has expired',
     }, {
         'url': 'http://www.abc.net.au/news/2015-08-17/warren-entsch-introduces-same-sex-marriage-bill/6702326',
         'md5': 'db2a5369238b51f9811ad815b69dc086',
@@ -36,6 +37,7 @@ class ABCIE(InfoExtractor):
             'title': 'Marriage Equality: Warren Entsch introduces same sex marriage bill',
         },
         'add_ie': ['Youtube'],
+        'skip': 'Not accessible from Travis CI server',
     }, {
         'url': 'http://www.abc.net.au/news/2015-10-23/nab-lifts-interest-rates-following-westpac-and-cba/6880080',
         'md5': 'b96eee7c9edf4fc5a358a0252881cc1f',
@@ -58,6 +60,9 @@ class ABCIE(InfoExtractor):
             r'inline(?P<type>Video|Audio|YouTube)Data\.push\((?P<json_data>[^)]+)\);',
             webpage)
         if mobj is None:
+            expired = self._html_search_regex(r'(?s)class="expired-(?:video|audio)".+?<span>(.+?)</span>', webpage, 'expired', None)
+            if expired:
+                raise ExtractorError('%s said: %s' % (self.IE_NAME, expired), expected=True)
             raise ExtractorError('Unable to extract video urls')
 
         urls_info = self._parse_json(

From 61ebb401b7ff8f15850bfc5778cdc08ab72145e1 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 21 Dec 2015 16:26:40 +0100
Subject: [PATCH 3437/6219] [atresplayer] improve extraction

- select hashlib.md5 constructor as digestmod(in python 3.4+ MD5 as
implicit default digest for digestmod is deprecated.)
- extract hls formats
- update tests
- extract errors
---
 youtube_dl/extractor/atresplayer.py | 117 +++++++++++++++++++---------
 1 file changed, 81 insertions(+), 36 deletions(-)

diff --git a/youtube_dl/extractor/atresplayer.py b/youtube_dl/extractor/atresplayer.py
index 50e47ba0a..7ac3044c7 100644
--- a/youtube_dl/extractor/atresplayer.py
+++ b/youtube_dl/extractor/atresplayer.py
@@ -2,6 +2,8 @@ from __future__ import unicode_literals
 
 import time
 import hmac
+import hashlib
+import re
 
 from .common import InfoExtractor
 from ..compat import (
@@ -32,6 +34,19 @@ class AtresPlayerIE(InfoExtractor):
                 'duration': 5527.6,
                 'thumbnail': 're:^https?://.*\.jpg$',
             },
+            'skip': 'This video is only available for registered users'
+        },
+        {
+            'url': 'http://www.atresplayer.com/television/especial/videoencuentros/temporada-1/capitulo-112-david-bustamante_2014121600375.html',
+            'md5': '0d0e918533bbd4b263f2de4d197d4aac',
+            'info_dict': {
+                'id': 'capitulo-112-david-bustamante',
+                'ext': 'flv',
+                'title': 'David Bustamante',
+                'description': 'md5:f33f1c0a05be57f6708d4dd83a3b81c6',
+                'duration': 1439.0,
+                'thumbnail': 're:^https?://.*\.jpg$',
+            },
         },
         {
             'url': 'http://www.atresplayer.com/television/series/el-secreto-de-puente-viejo/el-chico-de-los-tres-lunares/capitulo-977-29-12-14_2014122400174.html',
@@ -50,6 +65,13 @@ class AtresPlayerIE(InfoExtractor):
 
     _LOGIN_URL = 'https://servicios.atresplayer.com/j_spring_security_check'
 
+    _ERRORS = {
+        'UNPUBLISHED': 'We\'re sorry, but this video is not yet available.',
+        'DELETED': 'This video has expired and is no longer available for online streaming.',
+        'GEOUNPUBLISHED': 'We\'re sorry, but this video is not available in your region due to right restrictions.',
+        # 'PREMIUM': 'PREMIUM',
+    }
+
     def _real_initialize(self):
         self._login()
 
@@ -83,58 +105,81 @@ class AtresPlayerIE(InfoExtractor):
         episode_id = self._search_regex(
             r'episode="([^"]+)"', webpage, 'episode id')
 
+        request = sanitized_Request(
+            self._PLAYER_URL_TEMPLATE % episode_id,
+            headers={'User-Agent': self._USER_AGENT})
+        player = self._download_json(request, episode_id, 'Downloading player JSON')
+
+        episode_type = player.get('typeOfEpisode')
+        error_message = self._ERRORS.get(episode_type)
+        if error_message:
+            raise ExtractorError(
+                '%s returned error: %s' % (self.IE_NAME, error_message), expected=True)
+
+        formats = []
+        video_url = player.get('urlVideo')
+        if video_url:
+            format_info = {
+                'url': video_url,
+                'format_id': 'http',
+            }
+            mobj = re.search(r'(?P<bitrate>\d+)K_(?P<width>\d+)x(?P<height>\d+)', video_url)
+            if mobj:
+                format_info.update({
+                    'width': int_or_none(mobj.group('width')),
+                    'height': int_or_none(mobj.group('height')),
+                    'tbr': int_or_none(mobj.group('bitrate')),
+                })
+            formats.append(format_info)
+
+        m3u8_url = player.get('urlVideoHls')
+        if m3u8_url:
+            m3u8_formats = self._extract_m3u8_formats(
+                m3u8_url, episode_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False)
+            if m3u8_formats:
+                formats.extend(m3u8_formats)
+
         timestamp = int_or_none(self._download_webpage(
             self._TIME_API_URL,
             video_id, 'Downloading timestamp', fatal=False), 1000, time.time())
         timestamp_shifted = compat_str(timestamp + self._TIMESTAMP_SHIFT)
         token = hmac.new(
             self._MAGIC.encode('ascii'),
-            (episode_id + timestamp_shifted).encode('utf-8')
+            (episode_id + timestamp_shifted).encode('utf-8'), hashlib.md5
         ).hexdigest()
 
-        formats = []
-        for fmt in ['windows', 'android_tablet']:
-            request = sanitized_Request(
-                self._URL_VIDEO_TEMPLATE.format(fmt, episode_id, timestamp_shifted, token))
-            request.add_header('User-Agent', self._USER_AGENT)
+        request = sanitized_Request(
+            self._URL_VIDEO_TEMPLATE.format('windows', episode_id, timestamp_shifted, token),
+            headers={'User-Agent': self._USER_AGENT})
 
-            fmt_json = self._download_json(
-                request, video_id, 'Downloading %s video JSON' % fmt)
+        fmt_json = self._download_json(
+            request, video_id, 'Downloading windows video JSON')
 
-            result = fmt_json.get('resultDes')
-            if result.lower() != 'ok':
-                raise ExtractorError(
-                    '%s returned error: %s' % (self.IE_NAME, result), expected=True)
+        result = fmt_json.get('resultDes')
+        if result.lower() != 'ok':
+            raise ExtractorError(
+                '%s returned error: %s' % (self.IE_NAME, result), expected=True)
 
-            for format_id, video_url in fmt_json['resultObject'].items():
-                if format_id == 'token' or not video_url.startswith('http'):
-                    continue
-                if video_url.endswith('/Manifest'):
-                    if 'geodeswowsmpra3player' in video_url:
-                        f4m_path = video_url.split('smil:', 1)[-1].split('free_', 1)[0]
-                        f4m_url = 'http://drg.antena3.com/{0}hds/es/sd.f4m'.format(f4m_path)
-                        # this videos are protected by DRM, the f4m downloader doesn't support them
-                        continue
-                    else:
-                        f4m_url = video_url[:-9] + '/manifest.f4m'
-                    formats.extend(self._extract_f4m_formats(f4m_url, video_id))
-                else:
-                    formats.append({
-                        'url': video_url,
-                        'format_id': 'android-%s' % format_id,
-                        'preference': 1,
-                    })
+        for format_id, video_url in fmt_json['resultObject'].items():
+            if format_id == 'token' or not video_url.startswith('http'):
+                continue
+            if 'geodeswowsmpra3player' in video_url:
+                f4m_path = video_url.split('smil:', 1)[-1].split('free_', 1)[0]
+                f4m_url = 'http://drg.antena3.com/{0}hds/es/sd.f4m'.format(f4m_path)
+                # this videos are protected by DRM, the f4m downloader doesn't support them
+                continue
+            else:
+                f4m_url = video_url[:-9] + '/manifest.f4m'
+            f4m_formats = self._extract_f4m_formats(f4m_url, video_id, f4m_id='hds', fatal=False)
+            if f4m_formats:
+                formats.extend(f4m_formats)
         self._sort_formats(formats)
 
-        player = self._download_json(
-            self._PLAYER_URL_TEMPLATE % episode_id,
-            episode_id)
-
         path_data = player.get('pathData')
 
         episode = self._download_xml(
-            self._EPISODE_URL_TEMPLATE % path_data,
-            video_id, 'Downloading episode XML')
+            self._EPISODE_URL_TEMPLATE % path_data, video_id,
+            'Downloading episode XML')
 
         duration = float_or_none(xpath_text(
             episode, './media/asset/info/technical/contentDuration', 'duration'))

From f11d00fa4109ce754c58c4a7b8bef04fc14a257a Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 21 Dec 2015 16:52:47 +0100
Subject: [PATCH 3438/6219] [test_subtitles] remove BlipTV test

---
 test/test_subtitles.py | 13 -------------
 1 file changed, 13 deletions(-)

diff --git a/test/test_subtitles.py b/test/test_subtitles.py
index 75f0ea75f..9ed9fe622 100644
--- a/test/test_subtitles.py
+++ b/test/test_subtitles.py
@@ -11,7 +11,6 @@ from test.helper import FakeYDL, md5
 
 
 from youtube_dl.extractor import (
-    BlipTVIE,
     YoutubeIE,
     DailymotionIE,
     TEDIE,
@@ -145,18 +144,6 @@ class TestTedSubtitles(BaseTestSubtitles):
             self.assertTrue(subtitles.get(lang) is not None, 'Subtitles for \'%s\' not extracted' % lang)
 
 
-class TestBlipTVSubtitles(BaseTestSubtitles):
-    url = 'http://blip.tv/a/a-6603250'
-    IE = BlipTVIE
-
-    def test_allsubtitles(self):
-        self.DL.params['writesubtitles'] = True
-        self.DL.params['allsubtitles'] = True
-        subtitles = self.getSubtitles()
-        self.assertEqual(set(subtitles.keys()), set(['en']))
-        self.assertEqual(md5(subtitles['en']), '5b75c300af65fe4476dff79478bb93e4')
-
-
 class TestVimeoSubtitles(BaseTestSubtitles):
     url = 'http://vimeo.com/76979871'
     IE = VimeoIE

From 0f15ad7b9b6cae4e8719d2b960d8e882963f7a5b Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 21 Dec 2015 17:07:19 +0100
Subject: [PATCH 3439/6219] [adultswim] update test

---
 youtube_dl/extractor/adultswim.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/adultswim.py b/youtube_dl/extractor/adultswim.py
index 3ae618e71..bf21a6887 100644
--- a/youtube_dl/extractor/adultswim.py
+++ b/youtube_dl/extractor/adultswim.py
@@ -68,7 +68,7 @@ class AdultSwimIE(InfoExtractor):
                 'md5': '3e346a2ab0087d687a05e1e7f3b3e529',
                 'info_dict': {
                     'id': 'sY3cMUR_TbuE4YmdjzbIcQ-0',
-                    'ext': 'flv',
+                    'ext': 'mp4',
                     'title': 'Tim and Eric Awesome Show Great Job! - Dr. Steve Brule, For Your Wine',
                     'description': 'Dr. Brule reports live from Wine Country with a special report on wines.  \r\nWatch Tim and Eric Awesome Show Great Job! episode #20, "Embarrassed" on Adult Swim.\r\n\r\n',
                 },
@@ -79,6 +79,10 @@ class AdultSwimIE(InfoExtractor):
             'title': 'Tim and Eric Awesome Show Great Job! - Dr. Steve Brule, For Your Wine',
             'description': 'Dr. Brule reports live from Wine Country with a special report on wines.  \r\nWatch Tim and Eric Awesome Show Great Job! episode #20, "Embarrassed" on Adult Swim.\r\n\r\n',
         },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        }
     }]
 
     @staticmethod

From 5ef5d25b150a383362aab046394849ed523b3730 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 21 Dec 2015 22:51:58 +0600
Subject: [PATCH 3440/6219] [audiomack] Fix typo (Closes #7936)

---
 youtube_dl/extractor/audiomack.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/audiomack.py b/youtube_dl/extractor/audiomack.py
index 693ba22c6..3eed91279 100644
--- a/youtube_dl/extractor/audiomack.py
+++ b/youtube_dl/extractor/audiomack.py
@@ -56,7 +56,7 @@ class AudiomackIE(InfoExtractor):
 
         # API is inconsistent with errors
         if 'url' not in api_response or not api_response['url'] or 'error' in api_response:
-            raise ExtractorError('Invalid url %s', url)
+            raise ExtractorError('Invalid url %s' % url)
 
         # Audiomack wraps a lot of soundcloud tracks in their branded wrapper
         # if so, pass the work off to the soundcloud extractor

From 5625bd0617d6c661c66049b420556ae442102ffc Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 21 Dec 2015 21:06:10 +0100
Subject: [PATCH 3441/6219] [br] add support for br-klassik.de and improve
 extraction

- extend _VALID_URL to match both br.de and br-klassik.de
- extract all formats(hls,hds and rtmp)
- use xpath_element and xpath_text for xml info extraction
---
 youtube_dl/extractor/br.py | 127 +++++++++++++++++++++++--------------
 1 file changed, 78 insertions(+), 49 deletions(-)

diff --git a/youtube_dl/extractor/br.py b/youtube_dl/extractor/br.py
index 66e394e10..e66854538 100644
--- a/youtube_dl/extractor/br.py
+++ b/youtube_dl/extractor/br.py
@@ -1,18 +1,21 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
     int_or_none,
     parse_duration,
+    xpath_element,
+    xpath_text,
 )
 
 
 class BRIE(InfoExtractor):
     IE_DESC = 'Bayerischer Rundfunk Mediathek'
-    _VALID_URL = r'https?://(?:www\.)?br\.de/(?:[a-z0-9\-_]+/)+(?P<id>[a-z0-9\-_]+)\.html'
-    _BASE_URL = 'http://www.br.de'
+    _VALID_URL = r'(?P<base_url>https?://(?:www\.)?br(?:-klassik)?\.de)/(?:[a-z0-9\-_]+/)+(?P<id>[a-z0-9\-_]+)\.html'
 
     _TESTS = [
         {
@@ -22,7 +25,7 @@ class BRIE(InfoExtractor):
                 'id': '48f656ef-287e-486f-be86-459122db22cc',
                 'ext': 'mp4',
                 'title': 'Die böse Überraschung',
-                'description': 'Betriebliche Altersvorsorge: Die böse Überraschung',
+                'description': 'md5:ce9ac81b466ce775b8018f6801b48ac9',
                 'duration': 180,
                 'uploader': 'Reinhard Weber',
                 'upload_date': '20150422',
@@ -30,23 +33,23 @@ class BRIE(InfoExtractor):
         },
         {
             'url': 'http://www.br.de/nachrichten/oberbayern/inhalt/muenchner-polizeipraesident-schreiber-gestorben-100.html',
-            'md5': 'a44396d73ab6a68a69a568fae10705bb',
+            'md5': 'af3a3a4aa43ff0ce6a89504c67f427ef',
             'info_dict': {
                 'id': 'a4b83e34-123d-4b81-9f4e-c0d3121a4e05',
-                'ext': 'mp4',
+                'ext': 'flv',
                 'title': 'Manfred Schreiber ist tot',
-                'description': 'Abendschau kompakt: Manfred Schreiber ist tot',
+                'description': 'md5:b454d867f2a9fc524ebe88c3f5092d97',
                 'duration': 26,
             }
         },
         {
-            'url': 'http://www.br.de/radio/br-klassik/sendungen/allegro/premiere-urauffuehrung-the-land-2015-dance-festival-muenchen-100.html',
+            'url': 'https://www.br-klassik.de/audio/peeping-tom-premierenkritik-dance-festival-muenchen-100.html',
             'md5': '8b5b27c0b090f3b35eac4ab3f7a73d3d',
             'info_dict': {
                 'id': '74c603c9-26d3-48bb-b85b-079aeed66e0b',
                 'ext': 'aac',
                 'title': 'Kurzweilig und sehr bewegend',
-                'description': '"The Land" von Peeping Tom: Kurzweilig und sehr bewegend',
+                'description': 'md5:0351996e3283d64adeb38ede91fac54e',
                 'duration': 296,
             }
         },
@@ -57,7 +60,7 @@ class BRIE(InfoExtractor):
                 'id': '6ba73750-d405-45d3-861d-1ce8c524e059',
                 'ext': 'mp4',
                 'title': 'Umweltbewusster Häuslebauer',
-                'description': 'Uwe Erdelt: Umweltbewusster Häuslebauer',
+                'description': 'md5:d52dae9792d00226348c1dbb13c9bae2',
                 'duration': 116,
             }
         },
@@ -68,7 +71,7 @@ class BRIE(InfoExtractor):
                 'id': 'd982c9ce-8648-4753-b358-98abb8aec43d',
                 'ext': 'mp4',
                 'title': 'Folge 1 - Metaphysik',
-                'description': 'Kant für Anfänger: Folge 1 - Metaphysik',
+                'description': 'md5:bb659990e9e59905c3d41e369db1fbe3',
                 'duration': 893,
                 'uploader': 'Eva Maria Steimle',
                 'upload_date': '20140117',
@@ -77,28 +80,31 @@ class BRIE(InfoExtractor):
     ]
 
     def _real_extract(self, url):
-        display_id = self._match_id(url)
+        base_url, display_id = re.search(self._VALID_URL, url).groups()
         page = self._download_webpage(url, display_id)
         xml_url = self._search_regex(
             r"return BRavFramework\.register\(BRavFramework\('avPlayer_(?:[a-f0-9-]{36})'\)\.setup\({dataURL:'(/(?:[a-z0-9\-]+/)+[a-z0-9/~_.-]+)'}\)\);", page, 'XMLURL')
-        xml = self._download_xml(self._BASE_URL + xml_url, None)
+        xml = self._download_xml(base_url + xml_url, display_id)
 
         medias = []
 
         for xml_media in xml.findall('video') + xml.findall('audio'):
+            media_id = xml_media.get('externalId')
             media = {
-                'id': xml_media.get('externalId'),
-                'title': xml_media.find('title').text,
-                'duration': parse_duration(xml_media.find('duration').text),
-                'formats': self._extract_formats(xml_media.find('assets')),
-                'thumbnails': self._extract_thumbnails(xml_media.find('teaserImage/variants')),
-                'description': ' '.join(xml_media.find('shareTitle').text.splitlines()),
-                'webpage_url': xml_media.find('permalink').text
+                'id': media_id,
+                'title': xpath_text(xml_media, 'title', 'title', True),
+                'duration': parse_duration(xpath_text(xml_media, 'duration')),
+                'formats': self._extract_formats(xpath_element(
+                    xml_media, 'assets'), media_id),
+                'thumbnails': self._extract_thumbnails(xpath_element(
+                    xml_media, 'teaserImage/variants'), base_url),
+                'description': xpath_text(xml_media, 'desc'),
+                'webpage_url': xpath_text(xml_media, 'permalink'),
+                'uploader': xpath_text(xml_media, 'author'),
             }
-            if xml_media.find('author').text:
-                media['uploader'] = xml_media.find('author').text
-            if xml_media.find('broadcastDate').text:
-                media['upload_date'] = ''.join(reversed(xml_media.find('broadcastDate').text.split('.')))
+            broadcast_date = xpath_text(xml_media, 'broadcastDate')
+            if broadcast_date:
+                media['upload_date'] = ''.join(reversed(broadcast_date.split('.')))
             medias.append(media)
 
         if len(medias) > 1:
@@ -109,35 +115,58 @@ class BRIE(InfoExtractor):
             raise ExtractorError('No media entries found')
         return medias[0]
 
-    def _extract_formats(self, assets):
-
-        def text_or_none(asset, tag):
-            elem = asset.find(tag)
-            return None if elem is None else elem.text
-
-        formats = [{
-            'url': text_or_none(asset, 'downloadUrl'),
-            'ext': text_or_none(asset, 'mediaType'),
-            'format_id': asset.get('type'),
-            'width': int_or_none(text_or_none(asset, 'frameWidth')),
-            'height': int_or_none(text_or_none(asset, 'frameHeight')),
-            'tbr': int_or_none(text_or_none(asset, 'bitrateVideo')),
-            'abr': int_or_none(text_or_none(asset, 'bitrateAudio')),
-            'vcodec': text_or_none(asset, 'codecVideo'),
-            'acodec': text_or_none(asset, 'codecAudio'),
-            'container': text_or_none(asset, 'mediaType'),
-            'filesize': int_or_none(text_or_none(asset, 'size')),
-        } for asset in assets.findall('asset')
-            if asset.find('downloadUrl') is not None]
-
+    def _extract_formats(self, assets, media_id):
+        formats = []
+        for asset in assets.findall('asset'):
+            format_url = xpath_text(asset, ['downloadUrl', 'url'])
+            asset_type = asset.get('type')
+            if asset_type == 'HDS':
+                f4m_formats = self._extract_f4m_formats(
+                    format_url + '?hdcore=3.2.0', media_id, f4m_id='hds', fatal=False)
+                if f4m_formats:
+                    formats.extend(f4m_formats)
+            elif asset_type == 'HLS':
+                m3u8_formats = self._extract_m3u8_formats(
+                    format_url, media_id, 'mp4', 'm3u8_native', m3u8_id='hds', fatal=False)
+                if m3u8_formats:
+                    formats.extend(m3u8_formats)
+            else:
+                format_info = {
+                    'ext': xpath_text(asset, 'mediaType'),
+                    'width': int_or_none(xpath_text(asset, 'frameWidth')),
+                    'height': int_or_none(xpath_text(asset, 'frameHeight')),
+                    'tbr': int_or_none(xpath_text(asset, 'bitrateVideo')),
+                    'abr': int_or_none(xpath_text(asset, 'bitrateAudio')),
+                    'vcodec': xpath_text(asset, 'codecVideo'),
+                    'acodec': xpath_text(asset, 'codecAudio'),
+                    'container': xpath_text(asset, 'mediaType'),
+                    'filesize': int_or_none(xpath_text(asset, 'size')),
+                }
+                format_url = self._proto_relative_url(format_url)
+                if format_url:
+                    http_format_info = format_info.copy()
+                    http_format_info.update({
+                        'url': format_url,
+                        'format_id': 'http-%s' % asset_type,
+                    })
+                    formats.append(http_format_info)
+                server_prefix = xpath_text(asset, 'serverPrefix')
+                if server_prefix:
+                    rtmp_format_info = format_info.copy()
+                    rtmp_format_info.update({
+                        'url': server_prefix,
+                        'play_path': xpath_text(asset, 'fileName'),
+                        'format_id': 'rtmp-%s' % asset_type,
+                    })
+                    formats.append(rtmp_format_info)
         self._sort_formats(formats)
         return formats
 
-    def _extract_thumbnails(self, variants):
+    def _extract_thumbnails(self, variants, base_url):
         thumbnails = [{
-            'url': self._BASE_URL + variant.find('url').text,
-            'width': int_or_none(variant.find('width').text),
-            'height': int_or_none(variant.find('height').text),
-        } for variant in variants.findall('variant')]
+            'url': base_url + xpath_text(variant, 'url'),
+            'width': int_or_none(xpath_text(variant, 'width')),
+            'height': int_or_none(xpath_text(variant, 'height')),
+        } for variant in variants.findall('variant') if xpath_text(variant, 'url')]
         thumbnails.sort(key=lambda x: x['width'] * x['height'], reverse=True)
         return thumbnails

From eed30fea755f454582eb5ff555517cce3870dc87 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 21 Dec 2015 22:10:16 +0100
Subject: [PATCH 3442/6219] [flickr] fix format sorting

---
 youtube_dl/extractor/flickr.py | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/flickr.py b/youtube_dl/extractor/flickr.py
index 452b27b26..18f439df9 100644
--- a/youtube_dl/extractor/flickr.py
+++ b/youtube_dl/extractor/flickr.py
@@ -50,13 +50,19 @@ class FlickrIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        api_key = self._download_json('https://www.flickr.com/hermes_error_beacon.gne', video_id, 'Downloading api key',)['site_key']
+        api_key = self._download_json(
+            'https://www.flickr.com/hermes_error_beacon.gne', video_id,
+            'Downloading api key')['site_key']
 
-        video_info = self._call_api('photos.getInfo', video_id, api_key, 'Downloading video info')['photo']
+        video_info = self._call_api(
+            'photos.getInfo', video_id, api_key, 'Downloading video info')['photo']
         if video_info['media'] == 'video':
-            streams = self._call_api('video.getStreamInfo', video_id, api_key, 'Downloading streams info', video_info['secret'])['streams']
+            streams = self._call_api(
+                'video.getStreamInfo', video_id, api_key,
+                'Downloading streams info', video_info['secret'])['streams']
 
-            preference = qualities(['iphone_wifi', '700', 'appletv', 'orig'])
+            preference = qualities(
+                ['288p', 'iphone_wifi', '100', '300', '700', '360p', 'appletv', '720p', '1080p', 'orig'])
 
             formats = []
             for stream in streams['stream']:

From c7224074d61adf0dda362332a27f8ae20c4502cd Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 21 Dec 2015 23:02:55 +0100
Subject: [PATCH 3443/6219] [audimedia] correct test case id

---
 youtube_dl/extractor/audimedia.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/audimedia.py b/youtube_dl/extractor/audimedia.py
index b0b089dee..4382a302b 100644
--- a/youtube_dl/extractor/audimedia.py
+++ b/youtube_dl/extractor/audimedia.py
@@ -15,7 +15,7 @@ class AudiMediaIE(InfoExtractor):
         'url': 'https://audimedia.tv/en/vid/60-seconds-of-audi-sport-104-2015-wec-bahrain-rookie-test',
         'md5': '79a8b71c46d49042609795ab59779b66',
         'info_dict': {
-            'id': '1564',
+            'id': '1565',
             'ext': 'mp4',
             'title': '60 Seconds of Audi Sport 104/2015 - WEC Bahrain, Rookie Test',
             'description': 'md5:60e5d30a78ced725f7b8d34370762941',

From 5f432ac8f552b06e715c5e17165328dc76d9c1b5 Mon Sep 17 00:00:00 2001
From: Founder Fang <founder.fang@gmail.com>
Date: Sun, 20 Dec 2015 19:09:45 +0800
Subject: [PATCH 3444/6219] [Weiqitv] Add new extractor

---
 youtube_dl/extractor/__init__.py |  4 ++-
 youtube_dl/extractor/letv.py     | 57 ++++++++++++++++++++++++++++++++
 youtube_dl/extractor/weiqitv.py  | 54 ++++++++++++++++++++++++++++++
 3 files changed, 114 insertions(+), 1 deletion(-)
 create mode 100644 youtube_dl/extractor/weiqitv.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index e46d73ed7..9dcd252f8 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -331,7 +331,8 @@ from .lecture2go import Lecture2GoIE
 from .letv import (
     LetvIE,
     LetvTvIE,
-    LetvPlaylistIE
+    LetvPlaylistIE,
+    LetvCloudIE,
 )
 from .libsyn import LibsynIE
 from .lifenews import (
@@ -834,6 +835,7 @@ from .webofstories import (
     WebOfStoriesPlaylistIE,
 )
 from .weibo import WeiboIE
+from .weiqitv import WeiqitvIE
 from .wimp import WimpIE
 from .wistia import WistiaIE
 from .worldstarhiphop import WorldStarHipHopIE
diff --git a/youtube_dl/extractor/letv.py b/youtube_dl/extractor/letv.py
index be648000e..c096cb1ab 100644
--- a/youtube_dl/extractor/letv.py
+++ b/youtube_dl/extractor/letv.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 import datetime
 import re
 import time
+import base64
 
 from .common import InfoExtractor
 from ..compat import (
@@ -16,6 +17,7 @@ from ..utils import (
     parse_iso8601,
     sanitized_Request,
     int_or_none,
+    str_or_none,
     encode_data_uri,
 )
 
@@ -239,3 +241,58 @@ class LetvPlaylistIE(LetvTvIE):
         },
         'playlist_mincount': 7
     }]
+
+
+class LetvCloudIE(InfoExtractor):
+    IE_DESC = '乐视云'
+    _VALID_URL = r'http://yuntv\.letv\.com/bcloud.html\?.*$'
+
+    _TESTS = [{
+        'url': 'http://yuntv.letv.com/bcloud.html?uu=p7jnfw5hw9&vu=467623dedf',
+        'md5': '26450599afd64c513bc77030ad15db44',
+        'info_dict': {
+            'id': 'p7jnfw5hw9_467623dedf',
+            'ext': 'mp4',
+            'title': 'p7jnfw5hw9_467623dedf',
+        },
+    }, {
+        'url': 'http://yuntv.letv.com/bcloud.html?uu=p7jnfw5hw9&vu=ec93197892&pu=2c7cd40209&auto_play=1&gpcflag=1&width=640&height=360',
+        'info_dict': {
+            'id': 'p7jnfw5hw9_ec93197892',
+            'ext': 'mp4',
+            'title': 'p7jnfw5hw9_ec93197892',
+        },
+    }, {
+        'url': 'http://yuntv.letv.com/bcloud.html?uu=p7jnfw5hw9&vu=187060b6fd',
+        'info_dict': {
+            'id': 'p7jnfw5hw9_187060b6fd',
+            'ext': 'mp4',
+            'title': 'p7jnfw5hw9_187060b6fd',
+        },
+    }]
+
+    def _real_extract(self, url):
+        uu = re.search('uu=([\w]+)', url).group(1)
+        vu = re.search('vu=([\w]+)', url).group(1)
+        media_id = uu + '_' + vu
+
+        play_json_req = sanitized_Request(
+            'http://api.letvcloud.com/gpc.php?cf=html5&sign=signxxxxx&ver=2.2&format=json&' +
+            "uu=" + uu + "&vu=" + vu)
+        play_json = self._download_json(play_json_req, media_id, 'Downloading playJson data')
+
+        formats = [{
+            'url': base64.b64decode(media['play_url']['main_url'].encode('utf-8')).decode("utf-8"),
+            'ext': 'mp4',
+            'format_id': int_or_none(media.get('play_url', {}).get('vtype')),
+            'format_note': str_or_none(media.get('play_url', {}).get('definition')),
+            'width': int_or_none(media.get('play_url', {}).get('vwidth')),
+            'height': int_or_none(media.get('play_url', {}).get('vheight')),
+        } for media in play_json['data']['video_info']['media'].values()]
+        self._sort_formats(formats)
+
+        return {
+            'id': media_id,
+            'title': media_id,
+            'formats': formats,
+        }
diff --git a/youtube_dl/extractor/weiqitv.py b/youtube_dl/extractor/weiqitv.py
new file mode 100644
index 000000000..da3b3d145
--- /dev/null
+++ b/youtube_dl/extractor/weiqitv.py
@@ -0,0 +1,54 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class WeiqitvIE(InfoExtractor):
+    IE_DESC = 'WQTV'
+    _VALID_URL = r'http://www\.weiqitv\.com/index/video_play\?videoId=(?P<id>[A-Za-z0-9]+)'
+
+    _TESTS = [{
+        'url': 'http://www.weiqitv.com/index/video_play?videoId=53c744f09874f0e76a8b46f3',
+        'md5': '26450599afd64c513bc77030ad15db44',
+        'info_dict': {
+            'id': '53c744f09874f0e76a8b46f3',
+            'ext': 'mp4',
+            'title': '2013年度盘点',
+        },
+    }, {
+        'url': 'http://www.weiqitv.com/index/video_play?videoId=567379a2d4c36cca518b4569',
+        'info_dict': {
+            'id': '567379a2d4c36cca518b4569',
+            'ext': 'mp4',
+            'title': '民国围棋史',
+        },
+    }, {
+        'url': 'http://www.weiqitv.com/index/video_play?videoId=5430220a9874f088658b4567',
+        'info_dict': {
+            'id': '5430220a9874f088658b4567',
+            'ext': 'mp4',
+            'title': '二路托过的手段和运用',
+        },
+    }]
+
+    def _real_extract(self, url):
+        media_id = self._match_id(url)
+        page = self._download_webpage(url, media_id)
+
+        info_json_str = self._search_regex(
+            'var\s+video\s*=\s*(.+});',
+            page, 'info_json_str')
+        info_json = self._parse_json(info_json_str, media_id)
+
+        letvcloud_url = self._search_regex(
+            'var\s+letvurl\s*=\s*"([^"]+)',
+            page, 'letvcloud_url')
+
+        return {
+            '_type': 'url_transparent',
+            "ie_key": 'LetvCloud',
+            'url': letvcloud_url,
+            'title': info_json['name'],
+            'id': media_id,
+        }

From ed63cbd6ee392a01d10f0ca3f36ac9442622f0a9 Mon Sep 17 00:00:00 2001
From: Austin Adams <git@austinjadams.com>
Date: Mon, 21 Dec 2015 20:26:15 -0500
Subject: [PATCH 3445/6219] [comcarcoff] adjust for json updates

---
 youtube_dl/extractor/comcarcoff.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/comcarcoff.py b/youtube_dl/extractor/comcarcoff.py
index 81f3d7697..4391b7ce4 100644
--- a/youtube_dl/extractor/comcarcoff.py
+++ b/youtube_dl/extractor/comcarcoff.py
@@ -32,8 +32,8 @@ class ComCarCoffIE(InfoExtractor):
         webpage = self._download_webpage(url, display_id)
 
         full_data = json.loads(self._search_regex(
-            r'<script type="application/json" id="videoData">(?P<json>.+?)</script>',
-            webpage, 'full data json'))
+            r'\nwindow.app = (?P<json>.+?);\n',
+            webpage, 'full data json'))['videoData']
 
         video_id = full_data['activeVideo']['video']
         video_data = full_data.get('videos', {}).get(video_id) or full_data['singleshots'][video_id]

From ff43d2365ff5569c98df8e01250d34706e266c44 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 22 Dec 2015 07:58:33 +0100
Subject: [PATCH 3446/6219] [soompi] remove extractor

http://tv.soompi.com now redirect to viki.com because Viki has acquired
Soompi
http://www.soompi.com/2015/08/19/we-got-married-soompi-joins-viki/
---
 youtube_dl/extractor/__init__.py |   4 -
 youtube_dl/extractor/soompi.py   | 146 -------------------------------
 2 files changed, 150 deletions(-)
 delete mode 100644 youtube_dl/extractor/soompi.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 760b65441..702cbc6e2 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -591,10 +591,6 @@ from .snagfilms import (
 )
 from .snotr import SnotrIE
 from .sohu import SohuIE
-from .soompi import (
-    SoompiIE,
-    SoompiShowIE,
-)
 from .soundcloud import (
     SoundcloudIE,
     SoundcloudSetIE,
diff --git a/youtube_dl/extractor/soompi.py b/youtube_dl/extractor/soompi.py
deleted file mode 100644
index 5da66ca9e..000000000
--- a/youtube_dl/extractor/soompi.py
+++ /dev/null
@@ -1,146 +0,0 @@
-# encoding: utf-8
-from __future__ import unicode_literals
-
-import re
-
-from .crunchyroll import CrunchyrollIE
-
-from .common import InfoExtractor
-from ..compat import compat_HTTPError
-from ..utils import (
-    ExtractorError,
-    int_or_none,
-    remove_start,
-    xpath_text,
-)
-
-
-class SoompiBaseIE(InfoExtractor):
-    def _get_episodes(self, webpage, episode_filter=None):
-        episodes = self._parse_json(
-            self._search_regex(
-                r'VIDEOS\s*=\s*(\[.+?\]);', webpage, 'episodes JSON'),
-            None)
-        return list(filter(episode_filter, episodes))
-
-
-class SoompiIE(SoompiBaseIE, CrunchyrollIE):
-    IE_NAME = 'soompi'
-    _VALID_URL = r'https?://tv\.soompi\.com/(?:en/)?watch/(?P<id>[0-9]+)'
-    _TESTS = [{
-        'url': 'http://tv.soompi.com/en/watch/29235',
-        'info_dict': {
-            'id': '29235',
-            'ext': 'mp4',
-            'title': 'Episode 1096',
-            'description': '2015-05-20'
-        },
-        'params': {
-            'skip_download': True,
-        },
-    }]
-
-    def _get_episode(self, webpage, video_id):
-        return self._get_episodes(webpage, lambda x: x['id'] == video_id)[0]
-
-    def _get_subtitles(self, config, video_id):
-        sub_langs = {}
-        for subtitle in config.findall('./{default}preload/subtitles/subtitle'):
-            sub_langs[subtitle.attrib['id']] = subtitle.attrib['title']
-
-        subtitles = {}
-        for s in config.findall('./{default}preload/subtitle'):
-            lang_code = sub_langs.get(s.attrib['id'])
-            if not lang_code:
-                continue
-            sub_id = s.get('id')
-            data = xpath_text(s, './data', 'data')
-            iv = xpath_text(s, './iv', 'iv')
-            if not id or not iv or not data:
-                continue
-            subtitle = self._decrypt_subtitles(data, iv, sub_id).decode('utf-8')
-            subtitles[lang_code] = self._extract_subtitles(subtitle)
-        return subtitles
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        try:
-            webpage = self._download_webpage(
-                url, video_id, 'Downloading episode page')
-        except ExtractorError as ee:
-            if isinstance(ee.cause, compat_HTTPError) and ee.cause.code == 403:
-                webpage = ee.cause.read()
-                block_message = self._html_search_regex(
-                    r'(?s)<div class="block-message">(.+?)</div>', webpage,
-                    'block message', default=None)
-                if block_message:
-                    raise ExtractorError(block_message, expected=True)
-            raise
-
-        formats = []
-        config = None
-        for format_id in re.findall(r'\?quality=([0-9a-zA-Z]+)', webpage):
-            config = self._download_xml(
-                'http://tv.soompi.com/en/show/_/%s-config.xml?mode=hls&quality=%s' % (video_id, format_id),
-                video_id, 'Downloading %s XML' % format_id)
-            m3u8_url = xpath_text(
-                config, './{default}preload/stream_info/file',
-                '%s m3u8 URL' % format_id)
-            if not m3u8_url:
-                continue
-            formats.extend(self._extract_m3u8_formats(
-                m3u8_url, video_id, 'mp4', m3u8_id=format_id))
-        self._sort_formats(formats)
-
-        episode = self._get_episode(webpage, video_id)
-
-        title = episode['name']
-        description = episode.get('description')
-        duration = int_or_none(episode.get('duration'))
-
-        thumbnails = [{
-            'id': thumbnail_id,
-            'url': thumbnail_url,
-        } for thumbnail_id, thumbnail_url in episode.get('img_url', {}).items()]
-
-        subtitles = self.extract_subtitles(config, video_id)
-
-        return {
-            'id': video_id,
-            'title': title,
-            'description': description,
-            'thumbnails': thumbnails,
-            'duration': duration,
-            'formats': formats,
-            'subtitles': subtitles
-        }
-
-
-class SoompiShowIE(SoompiBaseIE):
-    IE_NAME = 'soompi:show'
-    _VALID_URL = r'https?://tv\.soompi\.com/en/shows/(?P<id>[0-9a-zA-Z\-_]+)'
-    _TESTS = [{
-        'url': 'http://tv.soompi.com/en/shows/liar-game',
-        'info_dict': {
-            'id': 'liar-game',
-            'title': 'Liar Game',
-            'description': 'md5:52c02bce0c1a622a95823591d0589b66',
-        },
-        'playlist_count': 14,
-    }]
-
-    def _real_extract(self, url):
-        show_id = self._match_id(url)
-
-        webpage = self._download_webpage(
-            url, show_id, 'Downloading show page')
-
-        title = remove_start(self._og_search_title(webpage), 'SoompiTV | ')
-        description = self._og_search_description(webpage)
-
-        entries = [
-            self.url_result('http://tv.soompi.com/en/watch/%s' % episode['id'], 'Soompi')
-            for episode in self._get_episodes(webpage)]
-
-        return self.playlist_result(entries, show_id, title, description)

From dc016bf5216d4c0d5b5fb2cd707e1d08fa4b0517 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 22 Dec 2015 09:55:25 +0100
Subject: [PATCH 3447/6219] [viki] detect errors and fix formats extraction

---
 youtube_dl/extractor/viki.py | 24 ++++++++++++++++++++++--
 1 file changed, 22 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index a63c23617..ca3f20a3d 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -30,6 +30,12 @@ class VikiBaseIE(InfoExtractor):
 
     _token = None
 
+    _ERRORS = {
+        'geo': 'Sorry, this content is not available in your region.',
+        'upcoming': 'Sorry, this content is not yet available.',
+        # 'paywall': 'paywall',
+    }
+
     def _prepare_call(self, path, timestamp=None, post_data=None):
         path += '?' if '?' not in path else '&'
         if not timestamp:
@@ -67,6 +73,12 @@ class VikiBaseIE(InfoExtractor):
             '%s returned error: %s' % (self.IE_NAME, error),
             expected=True)
 
+    def _check_errors(self, data):
+        for reason, status in data.get('blocking', {}).items():
+            if status and reason in self._ERRORS:
+                raise ExtractorError('%s said: %s' % (
+                    self.IE_NAME, self._ERRORS[reason]), expected=True)
+
     def _real_initialize(self):
         self._login()
 
@@ -193,6 +205,7 @@ class VikiIE(VikiBaseIE):
             'timestamp': 1321985454,
             'description': 'md5:44b1e46619df3a072294645c770cef36',
             'title': 'Love In Magic',
+            'age_limit': 13,
         },
     }]
 
@@ -202,6 +215,8 @@ class VikiIE(VikiBaseIE):
         video = self._call_api(
             'videos/%s.json' % video_id, video_id, 'Downloading video JSON')
 
+        self._check_errors(video)
+
         title = self.dict_selection(video.get('titles', {}), 'en')
         if not title:
             title = 'Episode %d' % video.get('number') if video.get('type') == 'episode' else video.get('id') or video_id
@@ -262,8 +277,11 @@ class VikiIE(VikiBaseIE):
                 r'^(\d+)[pP]$', format_id, 'height', default=None))
             for protocol, format_dict in stream_dict.items():
                 if format_id == 'm3u8':
-                    formats = self._extract_m3u8_formats(
-                        format_dict['url'], video_id, 'mp4', m3u8_id='m3u8-%s' % protocol)
+                    m3u8_formats = self._extract_m3u8_formats(
+                        format_dict['url'], video_id, 'mp4', 'm3u8_native',
+                        m3u8_id='m3u8-%s' % protocol, fatal=None)
+                    if m3u8_formats:
+                        formats.extend(m3u8_formats)
                 else:
                     formats.append({
                         'url': format_dict['url'],
@@ -315,6 +333,8 @@ class VikiChannelIE(VikiBaseIE):
             'containers/%s.json' % channel_id, channel_id,
             'Downloading channel JSON')
 
+        self._check_errors(channel)
+
         title = self.dict_selection(channel['titles'], 'en')
 
         description = self.dict_selection(channel['descriptions'], 'en')

From 48a6c984b806141dc6d3da0a96df2e553bb815e0 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 22 Dec 2015 10:14:57 +0100
Subject: [PATCH 3448/6219] [bleacherreport] update test

---
 youtube_dl/extractor/bleacherreport.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bleacherreport.py b/youtube_dl/extractor/bleacherreport.py
index bd2a6340b..38bda3af5 100644
--- a/youtube_dl/extractor/bleacherreport.py
+++ b/youtube_dl/extractor/bleacherreport.py
@@ -90,7 +90,7 @@ class BleacherReportCMSIE(AMPIE):
     _VALID_URL = r'https?://(?:www\.)?bleacherreport\.com/video_embed\?id=(?P<id>[0-9a-f-]{36})'
     _TESTS = [{
         'url': 'http://bleacherreport.com/video_embed?id=8fd44c2f-3dc5-4821-9118-2c825a98c0e1',
-        'md5': 'f0ca220af012d4df857b54f792c586bb',
+        'md5': '8c2c12e3af7805152675446c905d159b',
         'info_dict': {
             'id': '8fd44c2f-3dc5-4821-9118-2c825a98c0e1',
             'ext': 'flv',

From 220bc3f0e3777b89de335cdbc58a7d105584f06b Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 22 Dec 2015 11:27:18 +0100
Subject: [PATCH 3449/6219] [franceinter] fix title extraction

---
 youtube_dl/extractor/franceinter.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/franceinter.py b/youtube_dl/extractor/franceinter.py
index 6613ee17a..90a17815d 100644
--- a/youtube_dl/extractor/franceinter.py
+++ b/youtube_dl/extractor/franceinter.py
@@ -33,7 +33,7 @@ class FranceInterIE(InfoExtractor):
         video_url = 'http://www.franceinter.fr/' + path
 
         title = self._html_search_regex(
-            r'<span class="title">(.+?)</span>', webpage, 'title')
+            r'<span class="title-diffusion">(.+?)</span>', webpage, 'title')
         description = self._html_search_regex(
             r'<span class="description">(.*?)</span>',
             webpage, 'description', fatal=False)

From 2db5806991145ee293c964ecd85623c093d2e429 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 22 Dec 2015 11:30:35 +0100
Subject: [PATCH 3450/6219] [franceinter] use _match_id

---
 youtube_dl/extractor/franceinter.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/franceinter.py b/youtube_dl/extractor/franceinter.py
index 90a17815d..fdc51f44f 100644
--- a/youtube_dl/extractor/franceinter.py
+++ b/youtube_dl/extractor/franceinter.py
@@ -1,8 +1,6 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..utils import int_or_none
 
@@ -23,8 +21,7 @@ class FranceInterIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id)
 

From 2be689b7e27df89648d1d98fa74c297f0e06cbc1 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Mon, 21 Dec 2015 02:26:37 +0100
Subject: [PATCH 3451/6219] [theintercept] Add new extractor

---
 youtube_dl/extractor/__init__.py     |  1 +
 youtube_dl/extractor/theintercept.py | 68 ++++++++++++++++++++++++++++
 2 files changed, 69 insertions(+)
 create mode 100644 youtube_dl/extractor/theintercept.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index eac50eda5..042b1e921 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -657,6 +657,7 @@ from .tenplay import TenPlayIE
 from .testurl import TestURLIE
 from .testtube import TestTubeIE
 from .tf1 import TF1IE
+from .theintercept import TheInterceptIE
 from .theonion import TheOnionIE
 from .theplatform import (
     ThePlatformIE,
diff --git a/youtube_dl/extractor/theintercept.py b/youtube_dl/extractor/theintercept.py
new file mode 100644
index 000000000..b096a28de
--- /dev/null
+++ b/youtube_dl/extractor/theintercept.py
@@ -0,0 +1,68 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+from ..utils import (
+    ExtractorError,
+)
+
+class TheInterceptIE(InfoExtractor):
+    _VALID_URL = r'https://theintercept.com/fieldofvision/(?P<id>.+?)/'
+    _TESTS = [{
+        'url': 'https://theintercept.com/fieldofvision/thisisacoup-episode-four-surrender-or-die/',
+        'info_dict': {
+            'id': 'thisisacoup-episode-four-surrender-or-die',
+            'ext': 'mp4',
+            'title': '#ThisIsACoup – Episode Four: Surrender or Die',
+            'upload_date': '20151218',
+            'description': 'md5:74dd27f0e2fbd50817829f97eaa33140',
+        }
+    }]
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+
+        mobj = re.search(r'initialStoreTree =(?P<json_data>.+})', webpage)
+        if mobj is None:
+            raise ExtractorError('Unable to extract initialStoreTree')
+        json_data = self._parse_json(mobj.group('json_data'), display_id)
+
+        info = None
+        for post in json_data['resources']['posts'].values():
+            if post['slug'] == display_id:
+                info = post
+                break
+        if info is None:
+            raise ExtractorError('Unable to find info for %s'%display_id)
+
+        title = info['title']
+        description = info['excerpt']
+        upload_date = info['date'][:10].replace('-', '')
+        video_id = info['fov_videoid']
+        creator = ','.join([a['display_name'] for a in info['authors']])
+        thumbnail = self._og_search_property('image', webpage)
+        content_id = thumbnail.split('/')[-1].split('.')[0]
+        content_url = 'https://content.jwplatform.com/jw6/{content_id}.xml'.format(content_id=content_id)
+        content = self._download_xml(content_url, video_id)
+
+        formats = []
+        for source in content.findall('.//{http://rss.jwpcdn.com/}source'):
+            if source.attrib['file'].endswith('.m3u8'):
+                formats.extend(self._extract_m3u8_formats(
+                    source.attrib['file'], video_id, 'mp4', preference=1, m3u8_id='hls'))
+
+        return {
+            'creator': creator,
+            'description': description,
+            'display_id': display_id,
+            'formats': formats,
+            'id': video_id,
+            'id': video_id,
+            'thumbnail': thumbnail,
+            'title': title,
+            'upload_date': upload_date,
+        }

From 3b68efdc6ae109a840ff5f15f0e28910c2463b3f Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 22 Dec 2015 15:54:51 +0100
Subject: [PATCH 3452/6219] [vgtv] update tests and correct format sorting

---
 youtube_dl/extractor/vgtv.py    | 31 +++++++++++++++++++++----------
 youtube_dl/extractor/xstream.py |  2 +-
 2 files changed, 22 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/vgtv.py b/youtube_dl/extractor/vgtv.py
index 347410a78..811ee197d 100644
--- a/youtube_dl/extractor/vgtv.py
+++ b/youtube_dl/extractor/vgtv.py
@@ -104,10 +104,10 @@ class VGTVIE(XstreamIE):
         },
         {
             'url': 'http://www.aftenposten.no/webtv/#!/video/21039/trailer-sweatshop-i-can-t-take-any-more',
-            'md5': '7fbc265a3ca4933a423c7a66aa879a67',
+            'md5': 'fd828cd29774a729bf4d4425fe192972',
             'info_dict': {
                 'id': '21039',
-                'ext': 'mp4',
+                'ext': 'mov',
                 'title': 'TRAILER: «SWEATSHOP» - I can´t take any more',
                 'description': 'md5:21891f2b0dd7ec2f78d84a50e54f8238',
                 'duration': 66,
@@ -174,16 +174,15 @@ class VGTVIE(XstreamIE):
         for mp4_url in mp4_urls:
             format_info = {
                 'url': mp4_url,
-                'preference': 1,
             }
             mobj = re.search('(\d+)_(\d+)_(\d+)', mp4_url)
             if mobj:
-                vbr = int(mobj.group(3))
+                tbr = int(mobj.group(3))
                 format_info.update({
                     'width': int(mobj.group(1)),
                     'height': int(mobj.group(2)),
-                    'vbr': vbr,
-                    'format_id': 'mp4-%s' % vbr,
+                    'tbr': tbr,
+                    'format_id': 'mp4-%s' % tbr,
                 })
             formats.append(format_info)
 
@@ -210,7 +209,7 @@ class BTArticleIE(InfoExtractor):
     _VALID_URL = 'http://(?:www\.)?bt\.no/(?:[^/]+/)+(?P<id>[^/]+)-\d+\.html'
     _TEST = {
         'url': 'http://www.bt.no/nyheter/lokalt/Kjemper-for-internatet-1788214.html',
-        'md5': 'd055e8ee918ef2844745fcfd1a4175fb',
+        'md5': '2acbe8ad129b3469d5ae51b1158878df',
         'info_dict': {
             'id': '23199',
             'ext': 'mp4',
@@ -227,7 +226,7 @@ class BTArticleIE(InfoExtractor):
     def _real_extract(self, url):
         webpage = self._download_webpage(url, self._match_id(url))
         video_id = self._search_regex(
-            r'SVP\.Player\.load\(\s*(\d+)', webpage, 'video id')
+            r'<video[^>]+data-id="(\d+)"', webpage, 'video id')
         return self.url_result('bttv:%s' % video_id, 'VGTV')
 
 
@@ -235,7 +234,7 @@ class BTVestlendingenIE(InfoExtractor):
     IE_NAME = 'bt:vestlendingen'
     IE_DESC = 'Bergens Tidende - Vestlendingen'
     _VALID_URL = 'http://(?:www\.)?bt\.no/spesial/vestlendingen/#!/(?P<id>\d+)'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.bt.no/spesial/vestlendingen/#!/86588',
         'md5': 'd7d17e3337dc80de6d3a540aefbe441b',
         'info_dict': {
@@ -246,7 +245,19 @@ class BTVestlendingenIE(InfoExtractor):
             'timestamp': 1430473209,
             'upload_date': '20150501',
         },
-    }
+        'skip': '404 Error',
+    }, {
+        'url': 'http://www.bt.no/spesial/vestlendingen/#!/86255',
+        'md5': 'a2893f8632e96389f4bdf36aa9463ceb',
+        'info_dict': {
+            'id': '86255',
+            'ext': 'mov',
+            'title': 'Du må tåle å fryse og være sulten',
+            'description': 'md5:b8046f4d022d5830ddab04865791d063',
+            'upload_date': '20150321',
+            'timestamp': 1426942023,
+        },
+    }]
 
     def _real_extract(self, url):
         return self.url_result('bttv:%s' % self._match_id(url), 'VGTV')
diff --git a/youtube_dl/extractor/xstream.py b/youtube_dl/extractor/xstream.py
index 436f8978b..76c91bd92 100644
--- a/youtube_dl/extractor/xstream.py
+++ b/youtube_dl/extractor/xstream.py
@@ -93,7 +93,7 @@ class XstreamIE(InfoExtractor):
             formats.append({
                 'url': link.get('href'),
                 'format_id': link.get('rel'),
-                'preference': 2,
+                'preference': 1,
             })
 
         thumbnails = [{

From dbee18b5521edbfa1642c683ad2d317ba06e9d5b Mon Sep 17 00:00:00 2001
From: Abhishek Kedia <kedia.abhishek10@gmail.com>
Date: Mon, 21 Dec 2015 01:50:07 +0100
Subject: [PATCH 3453/6219] Improve extraction (Closes #7918)

remove outer parentheses in if

Conflicts:
	youtube_dl/extractor/imgur.py

checked code with flake8

not returning list in case of single images.

using the fact that id with length 5 are albums and more are single videos.
Also for single videos ie ImgurIE both urls - http://imgur.com/gallery/oWeAMW2 and http://imgur.com/oWeAMW2 are equally fine. Change regex to allow thuis.
For albums urls - http://imgur.com/gallery/Q95ko and http://imgur.com/Q95ko are ok. Change regex to allow this also.

update description in ImgurIE Tests.
Also move single video test 'https://imgur.com/gallery/YcAQlkx' from ImgurAlbumIE to ImgurIE.
---
 youtube_dl/extractor/imgur.py | 32 ++++++++++++++++++++++----------
 1 file changed, 22 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/imgur.py b/youtube_dl/extractor/imgur.py
index 70c8ca64e..88423f179 100644
--- a/youtube_dl/extractor/imgur.py
+++ b/youtube_dl/extractor/imgur.py
@@ -13,7 +13,7 @@ from ..utils import (
 
 
 class ImgurIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:i\.)?imgur\.com/(?!gallery)(?P<id>[a-zA-Z0-9]+)'
+    _VALID_URL = r'https?://(?:i\.)?imgur\.com/(gallery/)?(?P<id>[a-zA-Z0-9]{6,})'
 
     _TESTS = [{
         'url': 'https://i.imgur.com/A61SaA1.gifv',
@@ -21,7 +21,7 @@ class ImgurIE(InfoExtractor):
             'id': 'A61SaA1',
             'ext': 'mp4',
             'title': 're:Imgur GIF$|MRW gifv is up and running without any bugs$',
-            'description': 're:The origin of the Internet\'s most viral images$|The Internet\'s visual storytelling community\. Explore, share, and discuss the best visual stories the Internet has to offer\.$',
+            'description': 'Imgur: The most awesome images on the Internet.',
         },
     }, {
         'url': 'https://imgur.com/A61SaA1',
@@ -29,8 +29,17 @@ class ImgurIE(InfoExtractor):
             'id': 'A61SaA1',
             'ext': 'mp4',
             'title': 're:Imgur GIF$|MRW gifv is up and running without any bugs$',
-            'description': 're:The origin of the Internet\'s most viral images$|The Internet\'s visual storytelling community\. Explore, share, and discuss the best visual stories the Internet has to offer\.$',
+            'description': 'Imgur: The most awesome images on the Internet.',
         },
+    }, {
+        'url': 'https://imgur.com/gallery/YcAQlkx',
+        'info_dict': {
+            'id': 'YcAQlkx',
+            'ext': 'mp4',
+            'title': 'Classic Steve Carell gif...cracks me up everytime....damn the repost downvotes....',
+            'description': 'Imgur: The most awesome images on the Internet.'
+
+        }
     }]
 
     def _real_extract(self, url):
@@ -100,7 +109,7 @@ class ImgurIE(InfoExtractor):
 
 
 class ImgurAlbumIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:i\.)?imgur\.com/gallery/(?P<id>[a-zA-Z0-9]+)'
+    _VALID_URL = r'https?://(?:i\.)?imgur\.com/(gallery/)?(?P<id>[a-zA-Z0-9]{5})(?![a-zA-Z0-9])'
 
     _TEST = {
         'url': 'http://imgur.com/gallery/Q95ko',
@@ -113,12 +122,15 @@ class ImgurAlbumIE(InfoExtractor):
     def _real_extract(self, url):
         album_id = self._match_id(url)
 
-        album_images = self._download_json(
-            'http://imgur.com/gallery/%s/album_images/hit.json?all=true' % album_id,
-            album_id)['data']['images']
+        album_img_data = self._download_json(
+            'http://imgur.com/gallery/%s/album_images/hit.json?all=true' % album_id, album_id)['data']
 
-        entries = [
-            self.url_result('http://imgur.com/%s' % image['hash'])
-            for image in album_images if image.get('hash')]
+        if len(album_img_data) == 0:
+            return self.url_result('http://imgur.com/%s' % album_id)
+        else:
+            album_images = album_img_data['images']
+            entries = [
+                self.url_result('http://imgur.com/%s' % image['hash'])
+                for image in album_images if image.get('hash')]
 
         return self.playlist_result(entries, album_id)

From 774ce35571c08a1532fe4079224239adfdb80e43 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 22 Dec 2015 21:48:48 +0600
Subject: [PATCH 3454/6219] [imgur] Improve (Closes #7928)

---
 youtube_dl/extractor/imgur.py | 41 +++++++++++++++++++++++------------
 1 file changed, 27 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/imgur.py b/youtube_dl/extractor/imgur.py
index 88423f179..85e9344aa 100644
--- a/youtube_dl/extractor/imgur.py
+++ b/youtube_dl/extractor/imgur.py
@@ -13,7 +13,7 @@ from ..utils import (
 
 
 class ImgurIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:i\.)?imgur\.com/(gallery/)?(?P<id>[a-zA-Z0-9]{6,})'
+    _VALID_URL = r'https?://(?:i\.)?imgur\.com/(?:(?:gallery|topic/[^/]+)/)?(?P<id>[a-zA-Z0-9]{6,})(?:[/?#&]+|\.[a-z]+)?$'
 
     _TESTS = [{
         'url': 'https://i.imgur.com/A61SaA1.gifv',
@@ -40,6 +40,9 @@ class ImgurIE(InfoExtractor):
             'description': 'Imgur: The most awesome images on the Internet.'
 
         }
+    }, {
+        'url': 'http://imgur.com/topic/Funny/N8rOudd',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
@@ -109,28 +112,38 @@ class ImgurIE(InfoExtractor):
 
 
 class ImgurAlbumIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:i\.)?imgur\.com/(gallery/)?(?P<id>[a-zA-Z0-9]{5})(?![a-zA-Z0-9])'
+    _VALID_URL = r'https?://(?:i\.)?imgur\.com/(?:(?:a|gallery|topic/[^/]+)/)?(?P<id>[a-zA-Z0-9]{5})(?:[/?#&]+)?$'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://imgur.com/gallery/Q95ko',
         'info_dict': {
             'id': 'Q95ko',
         },
         'playlist_count': 25,
-    }
+    }, {
+        'url': 'http://imgur.com/a/j6Orj',
+        'only_matching': True,
+    }, {
+        'url': 'http://imgur.com/topic/Aww/ll5Vk',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         album_id = self._match_id(url)
 
-        album_img_data = self._download_json(
-            'http://imgur.com/gallery/%s/album_images/hit.json?all=true' % album_id, album_id)['data']
+        album_images = self._download_json(
+            'http://imgur.com/gallery/%s/album_images/hit.json?all=true' % album_id,
+            album_id, fatal=False)
 
-        if len(album_img_data) == 0:
-            return self.url_result('http://imgur.com/%s' % album_id)
-        else:
-            album_images = album_img_data['images']
-            entries = [
-                self.url_result('http://imgur.com/%s' % image['hash'])
-                for image in album_images if image.get('hash')]
+        if album_images:
+            data = album_images.get('data')
+            if data and isinstance(data, dict):
+                images = data.get('images')
+                if images and isinstance(images, list):
+                    entries = [
+                        self.url_result('http://imgur.com/%s' % image['hash'])
+                        for image in images if image.get('hash')]
+                    return self.playlist_result(entries, album_id)
 
-        return self.playlist_result(entries, album_id)
+        # Fallback to single video
+        return self.url_result('http://imgur.com/%s' % album_id, ImgurIE.ie_key())

From 4c24ed94640b148882f1ceb400127b3b3afcafd4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 23 Dec 2015 01:10:31 +0600
Subject: [PATCH 3455/6219] [comcarcoff] Improve json data regex and modernize

---
 youtube_dl/extractor/comcarcoff.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/comcarcoff.py b/youtube_dl/extractor/comcarcoff.py
index 4391b7ce4..edf5b29a0 100644
--- a/youtube_dl/extractor/comcarcoff.py
+++ b/youtube_dl/extractor/comcarcoff.py
@@ -31,9 +31,10 @@ class ComCarCoffIE(InfoExtractor):
             display_id = 'comediansincarsgettingcoffee.com'
         webpage = self._download_webpage(url, display_id)
 
-        full_data = json.loads(self._search_regex(
-            r'\nwindow.app = (?P<json>.+?);\n',
-            webpage, 'full data json'))['videoData']
+        full_data = self._parse_json(
+            self._search_regex(
+                r'window\.app\s*=\s*({.+?});\n', webpage, 'full data json'),
+            display_id)['videoData']
 
         video_id = full_data['activeVideo']['video']
         video_data = full_data.get('videos', {}).get(video_id) or full_data['singleshots'][video_id]

From cfe9e5aa6c5b14016ae454649b1a9df9c7c18b3c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 23 Dec 2015 01:18:14 +0600
Subject: [PATCH 3456/6219] [comcarcoff] Extract duration

---
 youtube_dl/extractor/comcarcoff.py | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/comcarcoff.py b/youtube_dl/extractor/comcarcoff.py
index edf5b29a0..2efa200b5 100644
--- a/youtube_dl/extractor/comcarcoff.py
+++ b/youtube_dl/extractor/comcarcoff.py
@@ -1,10 +1,12 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
-import json
-
 from .common import InfoExtractor
-from ..utils import parse_iso8601
+from ..utils import (
+    int_or_none,
+    parse_duration,
+    parse_iso8601,
+)
 
 
 class ComCarCoffIE(InfoExtractor):
@@ -16,6 +18,7 @@ class ComCarCoffIE(InfoExtractor):
             'ext': 'mp4',
             'upload_date': '20141127',
             'timestamp': 1417107600,
+            'duration': 1232,
             'title': 'Happy Thanksgiving Miranda',
             'description': 'Jerry Seinfeld and his special guest Miranda Sings cruise around town in search of coffee, complaining and apologizing along the way.',
             'thumbnail': 'http://ccc.crackle.com/images/s5e4_thumb.jpg',
@@ -46,12 +49,18 @@ class ComCarCoffIE(InfoExtractor):
         formats = self._extract_m3u8_formats(
             video_data['mediaUrl'], video_id, ext='mp4')
 
+        timestamp = int_or_none(video_data.get('pubDateTime')) or parse_iso8601(
+            video_data.get('pubDate'))
+        duration = int_or_none(video_data.get('durationSeconds')) or parse_duration(
+            video_data.get('duration'))
+
         return {
             'id': video_id,
             'display_id': display_id,
             'title': video_data['title'],
             'description': video_data.get('description'),
-            'timestamp': parse_iso8601(video_data.get('pubDate')),
+            'timestamp': timestamp,
+            'duration': duration,
             'thumbnails': thumbnails,
             'formats': formats,
             'webpage_url': 'http://comediansincarsgettingcoffee.com/%s' % (video_data.get('urlSlug', video_data.get('slug'))),

From 89abf7bf4d5dfc8c161924067f4430b7d81a8b32 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 23 Dec 2015 02:09:50 +0600
Subject: [PATCH 3457/6219] [periscope] Fix token based extraction (Closes
 #7943)

---
 youtube_dl/extractor/periscope.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/periscope.py b/youtube_dl/extractor/periscope.py
index 63cc764bb..514e9b433 100644
--- a/youtube_dl/extractor/periscope.py
+++ b/youtube_dl/extractor/periscope.py
@@ -31,9 +31,8 @@ class PeriscopeIE(InfoExtractor):
     }]
 
     def _call_api(self, method, value):
-        attribute = 'token' if len(value) > 13 else 'broadcast_id'
         return self._download_json(
-            'https://api.periscope.tv/api/v2/%s?%s=%s' % (method, attribute, value), value)
+            'https://api.periscope.tv/api/v2/%s?broadcast_id=%s' % (method, value), value)
 
     def _real_extract(self, url):
         token = self._match_id(url)

From 3a70ed9ebeac782b922cc3cb3b74cd999e60845a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 23 Dec 2015 02:54:32 +0600
Subject: [PATCH 3458/6219] [daum] Fix extraction (Closes #7949)

---
 youtube_dl/extractor/daum.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/daum.py b/youtube_dl/extractor/daum.py
index 934da765e..e3fc639b0 100644
--- a/youtube_dl/extractor/daum.py
+++ b/youtube_dl/extractor/daum.py
@@ -37,9 +37,11 @@ class DaumIE(InfoExtractor):
         video_id = mobj.group('id')
         canonical_url = 'http://tvpot.daum.net/v/%s' % video_id
         webpage = self._download_webpage(canonical_url, video_id)
+        og_url = self._og_search_url(webpage, default=None) or self._search_regex(
+            r'<link[^>]+rel=(["\'])canonical\1[^>]+href=(["\'])(?P<url>.+?)\2',
+            webpage, 'canonical url', group='url')
         full_id = self._search_regex(
-            r'src=["\']http://videofarm\.daum\.net/controller/video/viewer/Video\.html\?.*?vid=(.+?)[&"\']',
-            webpage, 'full id')
+            r'tvpot\.daum\.net/v/([^/]+)', og_url, 'full id')
         query = compat_urllib_parse.urlencode({'vid': full_id})
         info = self._download_xml(
             'http://tvpot.daum.net/clip/ClipInfoXml.do?' + query, video_id,

From 178b47e6af73521da50e9eec04af7fdceb236e39 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 23 Dec 2015 02:59:49 +0600
Subject: [PATCH 3459/6219] [daum] Add test for #7949

---
 youtube_dl/extractor/daum.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/youtube_dl/extractor/daum.py b/youtube_dl/extractor/daum.py
index e3fc639b0..9a94cf361 100644
--- a/youtube_dl/extractor/daum.py
+++ b/youtube_dl/extractor/daum.py
@@ -24,6 +24,18 @@ class DaumIE(InfoExtractor):
             'upload_date': '20130831',
             'duration': 3868,
         },
+    }, {
+        # Test for https://github.com/rg3/youtube-dl/issues/7949
+        'url': 'http://tvpot.daum.net/mypot/View.do?ownerid=M1O35s8HPOo0&clipid=73147290',
+        'md5': 'c92d78bcee4424451f1667f275c1dc97',
+        'info_dict': {
+            'id': '73147290',
+            'ext': 'mp4',
+            'title': '싸이 - 나팔바지 [유희열의 스케치북] 299회 20151218',
+            'description': '싸이 - 나팔바지',
+            'upload_date': '20151219',
+            'duration': 232,
+        },
     }, {
         'url': 'http://tvpot.daum.net/v/vab4dyeDBysyBssyukBUjBz',
         'only_matching': True,

From 60427f63d129919d5c98e1176ea8136d0eedd0f4 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 23 Dec 2015 10:40:45 +0100
Subject: [PATCH 3460/6219] [appletrailers] Add support for AppleTrailers
 Section

---
 youtube_dl/extractor/__init__.py      |  5 +-
 youtube_dl/extractor/appletrailers.py | 74 +++++++++++++++++++++++++++
 2 files changed, 78 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index fede5ff0d..3b541a538 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -25,7 +25,10 @@ from .aol import AolIE
 from .allocine import AllocineIE
 from .aparat import AparatIE
 from .appleconnect import AppleConnectIE
-from .appletrailers import AppleTrailersIE
+from .appletrailers import (
+    AppleTrailersIE,
+    AppleTrailersSectionIE,
+)
 from .archiveorg import ArchiveOrgIE
 from .ard import (
     ARDIE,
diff --git a/youtube_dl/extractor/appletrailers.py b/youtube_dl/extractor/appletrailers.py
index f68dc3236..ca9a70924 100644
--- a/youtube_dl/extractor/appletrailers.py
+++ b/youtube_dl/extractor/appletrailers.py
@@ -11,6 +11,7 @@ from ..utils import (
 
 
 class AppleTrailersIE(InfoExtractor):
+    IE_NAME = 'appletrailers'
     _VALID_URL = r'https?://(?:www\.)?trailers\.apple\.com/(?:trailers|ca)/(?P<company>[^/]+)/(?P<movie>[^/]+)'
     _TESTS = [{
         'url': 'http://trailers.apple.com/trailers/wb/manofsteel/',
@@ -144,3 +145,76 @@ class AppleTrailersIE(InfoExtractor):
             'id': movie,
             'entries': playlist,
         }
+
+
+class AppleTrailersSectionIE(InfoExtractor):
+    IE_NAME = 'appletrailers:section'
+    _SECTIONS = {
+        'justadded': {
+            'feed_path': 'just_added',
+            'title': 'Just Added',
+        },
+        'exclusive': {
+            'feed_path': 'exclusive',
+            'title': 'Exclusive',
+        },
+        'justhd': {
+            'feed_path': 'just_hd',
+            'title': 'Just HD',
+        },
+        'mostpopular': {
+            'feed_path': 'most_pop',
+            'title': 'Most Popular',
+        },
+        'moviestudios': {
+            'feed_path': 'studios',
+            'title': 'Movie Studios',
+        },
+    }
+    _VALID_URL = r'https?://(?:www\.)?trailers\.apple\.com/#section=(?P<id>%s)' % '|'.join(_SECTIONS)
+    _TESTS = [{
+        'url': 'http://trailers.apple.com/#section=justadded',
+        'info_dict': {
+            'title': 'Just Added',
+            'id': 'justadded',
+        },
+        'playlist_mincount': 80,
+    }, {
+        'url': 'http://trailers.apple.com/#section=exclusive',
+        'info_dict': {
+            'title': 'Exclusive',
+            'id': 'exclusive',
+        },
+        'playlist_mincount': 80,
+    }, {
+        'url': 'http://trailers.apple.com/#section=justhd',
+        'info_dict': {
+            'title': 'Just HD',
+            'id': 'justhd',
+        },
+        'playlist_mincount': 80,
+    }, {
+        'url': 'http://trailers.apple.com/#section=mostpopular',
+        'info_dict': {
+            'title': 'Most Popular',
+            'id': 'mostpopular',
+        },
+        'playlist_mincount': 80,
+    }, {
+        'url': 'http://trailers.apple.com/#section=moviestudios',
+        'info_dict': {
+            'title': 'Movie Studios',
+            'id': 'moviestudios',
+        },
+        'playlist_mincount': 80,
+    }]
+
+    def _real_extract(self, url):
+        section = self._match_id(url)
+        section_data = self._download_json(
+            'http://trailers.apple.com/trailers/home/feeds/%s.json' % self._SECTIONS[section]['feed_path'],
+            section)
+        entries = [
+            self.url_result('http://trailers.apple.com' + e['location'])
+            for e in section_data]
+        return self.playlist_result(entries, section, self._SECTIONS[section]['title'])

From f10c27b8cb35c72b5f9633956f71f96da72ada31 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 23 Dec 2015 14:05:06 +0100
Subject: [PATCH 3461/6219] release 2015.12.23

---
 docs/supportedsites.md | 8 +++-----
 youtube_dl/version.py  | 2 +-
 2 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 299bc5e72..1a5c7cde9 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -23,7 +23,6 @@
  - **AdobeTVShow**
  - **AdobeTVVideo**
  - **AdultSwim**
- - **Aftenposten**
  - **Aftonbladet**
  - **AirMozilla**
  - **AlJazeera**
@@ -34,7 +33,8 @@
  - **Aparat**
  - **AppleConnect**
  - **AppleDaily**: 臺灣蘋果日報
- - **AppleTrailers**
+ - **appletrailers**
+ - **appletrailers:section**
  - **archive.org**: archive.org videos
  - **ARD**
  - **ARD:mediathek**
@@ -502,8 +502,6 @@
  - **SnagFilmsEmbed**
  - **Snotr**
  - **Sohu**
- - **soompi**
- - **soompi:show**
  - **soundcloud**
  - **soundcloud:playlist**
  - **soundcloud:search**: Soundcloud search
@@ -627,7 +625,7 @@
  - **Vessel**
  - **Vesti**: Вести.Ru
  - **Vevo**
- - **VGTV**: VGTV and BTTV
+ - **VGTV**: VGTV, BTTV, FTV, Aftenposten and Aftonbladet
  - **vh1.com**
  - **Vice**
  - **Viddler**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 7095033c5..255d64269 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.12.21'
+__version__ = '2015.12.23'

From 7fe37d8a05609229332d5a156cb9b7cf4bba2790 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 23 Dec 2015 14:48:40 +0100
Subject: [PATCH 3462/6219] [appletrailers] Improve regex for fixing '<img>'
 tags (#7953)

---
 youtube_dl/extractor/appletrailers.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/appletrailers.py b/youtube_dl/extractor/appletrailers.py
index ca9a70924..82beed2ce 100644
--- a/youtube_dl/extractor/appletrailers.py
+++ b/youtube_dl/extractor/appletrailers.py
@@ -80,7 +80,7 @@ class AppleTrailersIE(InfoExtractor):
 
         def fix_html(s):
             s = re.sub(r'(?s)<script[^<]*?>.*?</script>', '', s)
-            s = re.sub(r'<img ([^<]*?)>', r'<img \1/>', s)
+            s = re.sub(r'<img ([^<]*?)/?>', r'<img \1/>', s)
             # The ' in the onClick attributes are not escaped, it couldn't be parsed
             # like: http://trailers.apple.com/trailers/wb/gravity/
 

From 747b028412828c66080c7f165b461a7ea490fead Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 23 Dec 2015 20:42:36 +0600
Subject: [PATCH 3463/6219] [24video] Fix extraction (Closes #7956)

---
 youtube_dl/extractor/twentyfourvideo.py | 35 +++++++++----------------
 1 file changed, 12 insertions(+), 23 deletions(-)

diff --git a/youtube_dl/extractor/twentyfourvideo.py b/youtube_dl/extractor/twentyfourvideo.py
index c1ee1decc..cb9e5f1b5 100644
--- a/youtube_dl/extractor/twentyfourvideo.py
+++ b/youtube_dl/extractor/twentyfourvideo.py
@@ -64,33 +64,22 @@ class TwentyFourVideoIE(InfoExtractor):
             r'<div class="comments-title" id="comments-count">(\d+) комментари',
             webpage, 'comment count', fatal=False))
 
-        formats = []
+        # Sets some cookies
+        self._download_xml(
+            r'http://www.24video.net/video/xml/%s?mode=init' % video_id,
+            video_id, 'Downloading init XML')
 
-        pc_video = self._download_xml(
+        video = self._download_xml(
             'http://www.24video.net/video/xml/%s?mode=play' % video_id,
-            video_id, 'Downloading PC video URL').find('.//video')
+            video_id, 'Downloading video XML').find('.//video')
 
-        formats.append({
-            'url': pc_video.attrib['url'],
-            'format_id': 'pc',
-            'quality': 1,
-        })
+        formats = [{
+            'url': video.attrib['url'],
+        }]
 
-        like_count = int_or_none(pc_video.get('ratingPlus'))
-        dislike_count = int_or_none(pc_video.get('ratingMinus'))
-        age_limit = 18 if pc_video.get('adult') == 'true' else 0
-
-        mobile_video = self._download_xml(
-            'http://www.24video.net/video/xml/%s' % video_id,
-            video_id, 'Downloading mobile video URL').find('.//video')
-
-        formats.append({
-            'url': mobile_video.attrib['url'],
-            'format_id': 'mobile',
-            'quality': 0,
-        })
-
-        self._sort_formats(formats)
+        like_count = int_or_none(video.get('ratingPlus'))
+        dislike_count = int_or_none(video.get('ratingMinus'))
+        age_limit = 18 if video.get('adult') == 'true' else 0
 
         return {
             'id': video_id,

From 128eb31d90583113083ba1fe329eb4cf42c2989f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 23 Dec 2015 20:49:41 +0600
Subject: [PATCH 3464/6219] [24video] Fix extraction on python 2.6

---
 youtube_dl/extractor/twentyfourvideo.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/twentyfourvideo.py b/youtube_dl/extractor/twentyfourvideo.py
index cb9e5f1b5..68e2277a4 100644
--- a/youtube_dl/extractor/twentyfourvideo.py
+++ b/youtube_dl/extractor/twentyfourvideo.py
@@ -5,6 +5,8 @@ from .common import InfoExtractor
 from ..utils import (
     parse_iso8601,
     int_or_none,
+    xpath_attr,
+    xpath_element,
 )
 
 
@@ -69,12 +71,14 @@ class TwentyFourVideoIE(InfoExtractor):
             r'http://www.24video.net/video/xml/%s?mode=init' % video_id,
             video_id, 'Downloading init XML')
 
-        video = self._download_xml(
+        video_xml = self._download_xml(
             'http://www.24video.net/video/xml/%s?mode=play' % video_id,
-            video_id, 'Downloading video XML').find('.//video')
+            video_id, 'Downloading video XML')
+
+        video = xpath_element(video_xml, './/video', 'video', fatal=True)
 
         formats = [{
-            'url': video.attrib['url'],
+            'url': xpath_attr(video, '', 'url', 'video URL', fatal=True),
         }]
 
         like_count = int_or_none(video.get('ratingPlus'))

From be514c856cf9f95fe3e0d45b1df0319a0872b911 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 23 Dec 2015 20:49:52 +0600
Subject: [PATCH 3465/6219] [24video] Fix test

---
 youtube_dl/extractor/twentyfourvideo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/twentyfourvideo.py b/youtube_dl/extractor/twentyfourvideo.py
index 68e2277a4..e03e2dbaa 100644
--- a/youtube_dl/extractor/twentyfourvideo.py
+++ b/youtube_dl/extractor/twentyfourvideo.py
@@ -17,7 +17,7 @@ class TwentyFourVideoIE(InfoExtractor):
     _TESTS = [
         {
             'url': 'http://www.24video.net/video/view/1044982',
-            'md5': 'd041af8b5b4246ea466226a0d6693345',
+            'md5': 'e09fc0901d9eaeedac872f154931deeb',
             'info_dict': {
                 'id': '1044982',
                 'ext': 'mp4',

From dcdc352371115007028632da6ae377d0ad39b62a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 23 Dec 2015 21:13:31 +0600
Subject: [PATCH 3466/6219] [instagram:user] Improve _VALID_URL (Closes #7955)

---
 youtube_dl/extractor/instagram.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/instagram.py b/youtube_dl/extractor/instagram.py
index c158f2064..e5e16ca3b 100644
--- a/youtube_dl/extractor/instagram.py
+++ b/youtube_dl/extractor/instagram.py
@@ -47,7 +47,7 @@ class InstagramIE(InfoExtractor):
 
 
 class InstagramUserIE(InfoExtractor):
-    _VALID_URL = r'https://instagram\.com/(?P<username>[^/]{2,})/?(?:$|[?#])'
+    _VALID_URL = r'https?://(?:www\.)?instagram\.com/(?P<username>[^/]{2,})/?(?:$|[?#])'
     IE_DESC = 'Instagram user profile'
     IE_NAME = 'instagram:user'
     _TEST = {

From 261b4c23c70c7c5dc4fe9fd22cf9e867b7456c40 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 23 Dec 2015 17:48:37 +0100
Subject: [PATCH 3467/6219] [appletrailers] skip clips with empty url

---
 youtube_dl/extractor/appletrailers.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/appletrailers.py b/youtube_dl/extractor/appletrailers.py
index 82beed2ce..62ed0c918 100644
--- a/youtube_dl/extractor/appletrailers.py
+++ b/youtube_dl/extractor/appletrailers.py
@@ -64,6 +64,12 @@ class AppleTrailersIE(InfoExtractor):
                 },
             },
         ]
+    }, {
+        'url': 'http://trailers.apple.com/trailers/magnolia/blackthorn/',
+        'info_dict': {
+            'id': 'blackthorn',
+        },
+        'playlist_mincount': 2,
     }, {
         'url': 'http://trailers.apple.com/ca/metropole/autrui/',
         'only_matching': True,
@@ -97,6 +103,9 @@ class AppleTrailersIE(InfoExtractor):
             trailer_info_json = self._search_regex(self._JSON_RE,
                                                    on_click, 'trailer info')
             trailer_info = json.loads(trailer_info_json)
+            first_url = trailer_info.get('url')
+            if not first_url:
+                continue
             title = trailer_info['title']
             video_id = movie + '-' + re.sub(r'[^a-zA-Z0-9]', '', title).lower()
             thumbnail = li.find('.//img').attrib['src']
@@ -108,7 +117,6 @@ class AppleTrailersIE(InfoExtractor):
             if m:
                 duration = 60 * int(m.group('minutes')) + int(m.group('seconds'))
 
-            first_url = trailer_info['url']
             trailer_id = first_url.split('/')[-1].rpartition('_')[0].lower()
             settings_json_url = compat_urlparse.urljoin(url, 'includes/settings/%s.json' % trailer_id)
             settings = self._download_json(settings_json_url, trailer_id, 'Downloading settings json')

From a8f1d167f6741485b7cbec2cb355315c3774d5bc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 23 Dec 2015 17:55:58 +0100
Subject: [PATCH 3468/6219] [arte] Prefer json URLs that contain the video id
 from the 'vid' parameter in the URL (fixes #7920)

---
 youtube_dl/extractor/arte.py | 20 +++++++++++++++-----
 1 file changed, 15 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index 2a00da3ee..10301a8ea 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -68,9 +68,13 @@ class ArteTVPlus7IE(InfoExtractor):
     def _extract_url_info(cls, url):
         mobj = re.match(cls._VALID_URL, url)
         lang = mobj.group('lang')
-        # This is not a real id, it can be for example AJT for the news
-        # http://www.arte.tv/guide/fr/emissions/AJT/arte-journal
-        video_id = mobj.group('id')
+        query = compat_parse_qs(compat_urllib_parse_urlparse(url).query)
+        if 'vid' in query:
+            video_id = query['vid'][0]
+        else:
+            # This is not a real id, it can be for example AJT for the news
+            # http://www.arte.tv/guide/fr/emissions/AJT/arte-journal
+            video_id = mobj.group('id')
         return video_id, lang
 
     def _real_extract(self, url):
@@ -79,9 +83,15 @@ class ArteTVPlus7IE(InfoExtractor):
         return self._extract_from_webpage(webpage, video_id, lang)
 
     def _extract_from_webpage(self, webpage, video_id, lang):
+        patterns_templates = (r'arte_vp_url=["\'](.*?%s.*?)["\']', r'data-url=["\']([^"]+%s[^"]+)["\']')
+        ids = (video_id, '')
+        # some pages contain multiple videos (like
+        # http://www.arte.tv/guide/de/sendungen/XEN/xenius/?vid=055918-015_PLUS7-D),
+        # so we first try to look for json URLs that contain the video id from
+        # the 'vid' parameter.
+        patterns = [t % re.escape(_id) for _id in ids for t in patterns_templates]
         json_url = self._html_search_regex(
-            [r'arte_vp_url=["\'](.*?)["\']', r'data-url=["\']([^"]+)["\']'],
-            webpage, 'json vp url', default=None)
+            patterns, webpage, 'json vp url', default=None)
         if not json_url:
             iframe_url = self._html_search_regex(
                 r'<iframe[^>]+src=(["\'])(?P<url>.+\bjson_url=.+?)\1',

From 2c566d02febb0cf137a8dce8646957beb1415770 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 23 Dec 2015 23:22:47 +0600
Subject: [PATCH 3469/6219] [pbs] Extend PBS station regex (Closes #7964)

---
 youtube_dl/extractor/pbs.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 744e4a09a..97e8ffc97 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -16,7 +16,7 @@ from ..utils import (
 
 class PBSIE(InfoExtractor):
     _STATIONS = (
-        (r'(?:video|www)\.pbs\.org', 'PBS: Public Broadcasting Service'),  # http://www.pbs.org/
+        (r'(?:video|www|player)\.pbs\.org', 'PBS: Public Broadcasting Service'),  # http://www.pbs.org/
         (r'video\.aptv\.org', 'APT - Alabama Public Television (WBIQ)'),  # http://aptv.org/
         (r'video\.gpb\.org', 'GPB/Georgia Public Broadcasting (WGTV)'),  # http://www.gpb.org/
         (r'video\.mpbonline\.org', 'Mississippi Public Broadcasting (WMPN)'),  # http://www.mpbonline.org

From fc383f199e73358f88ecf24b7e804dda7400afae Mon Sep 17 00:00:00 2001
From: Jakub Wilk <jwilk@jwilk.net>
Date: Wed, 23 Dec 2015 17:35:10 +0100
Subject: [PATCH 3470/6219] Fix typos

---
 CONTRIBUTING.md | 2 +-
 README.md       | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index f3fe0d432..d15267d7e 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -28,7 +28,7 @@ So please elaborate on what feature you are requesting, or what bug you want to
 - How it could be fixed
 - How your proposed solution would look like
 
-If your report is shorter than two lines, it is almost certainly missing some of these, which makes it hard for us to respond to it. We're often too polite to close the issue outright, but the missing info makes misinterpretation likely. As a commiter myself, I often get frustrated by these issues, since the only possible way for me to move forward on them is to ask for clarification over and over.
+If your report is shorter than two lines, it is almost certainly missing some of these, which makes it hard for us to respond to it. We're often too polite to close the issue outright, but the missing info makes misinterpretation likely. As a committer myself, I often get frustrated by these issues, since the only possible way for me to move forward on them is to ask for clarification over and over.
 
 For bug reports, this means that your report should contain the *complete* output of youtube-dl when called with the `-v` flag. The error message you get for (most) bugs even says so, but you would not believe how many of our bug reports do not contain this information.
 
diff --git a/README.md b/README.md
index 7002f45e0..3a4707227 100644
--- a/README.md
+++ b/README.md
@@ -830,7 +830,7 @@ So please elaborate on what feature you are requesting, or what bug you want to
 - How it could be fixed
 - How your proposed solution would look like
 
-If your report is shorter than two lines, it is almost certainly missing some of these, which makes it hard for us to respond to it. We're often too polite to close the issue outright, but the missing info makes misinterpretation likely. As a commiter myself, I often get frustrated by these issues, since the only possible way for me to move forward on them is to ask for clarification over and over.
+If your report is shorter than two lines, it is almost certainly missing some of these, which makes it hard for us to respond to it. We're often too polite to close the issue outright, but the missing info makes misinterpretation likely. As a committer myself, I often get frustrated by these issues, since the only possible way for me to move forward on them is to ask for clarification over and over.
 
 For bug reports, this means that your report should contain the *complete* output of youtube-dl when called with the `-v` flag. The error message you get for (most) bugs even says so, but you would not believe how many of our bug reports do not contain this information.
 

From 96db61ffb83de9d912003a4778e9ce7c4d46e848 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 23 Dec 2015 22:36:53 +0100
Subject: [PATCH 3471/6219] [theintercept] improve extraction

---
 youtube_dl/extractor/theintercept.py | 67 ++++++++++------------------
 1 file changed, 24 insertions(+), 43 deletions(-)

diff --git a/youtube_dl/extractor/theintercept.py b/youtube_dl/extractor/theintercept.py
index b096a28de..8cb3c3669 100644
--- a/youtube_dl/extractor/theintercept.py
+++ b/youtube_dl/extractor/theintercept.py
@@ -1,24 +1,28 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
-
+from ..compat import compat_str
 from ..utils import (
+    parse_iso8601,
+    int_or_none,
     ExtractorError,
 )
 
+
 class TheInterceptIE(InfoExtractor):
-    _VALID_URL = r'https://theintercept.com/fieldofvision/(?P<id>.+?)/'
+    _VALID_URL = r'https://theintercept.com/fieldofvision/(?P<id>[^/?#]+)'
     _TESTS = [{
         'url': 'https://theintercept.com/fieldofvision/thisisacoup-episode-four-surrender-or-die/',
+        'md5': '145f28b41d44aab2f87c0a4ac8ec95bd',
         'info_dict': {
-            'id': 'thisisacoup-episode-four-surrender-or-die',
+            'id': '46214',
             'ext': 'mp4',
             'title': '#ThisIsACoup – Episode Four: Surrender or Die',
-            'upload_date': '20151218',
             'description': 'md5:74dd27f0e2fbd50817829f97eaa33140',
+            'timestamp': 1450429239,
+            'upload_date': '20151218',
+            'comment_count': int,
         }
     }]
 
@@ -26,43 +30,20 @@ class TheInterceptIE(InfoExtractor):
         display_id = self._match_id(url)
         webpage = self._download_webpage(url, display_id)
 
-        mobj = re.search(r'initialStoreTree =(?P<json_data>.+})', webpage)
-        if mobj is None:
-            raise ExtractorError('Unable to extract initialStoreTree')
-        json_data = self._parse_json(mobj.group('json_data'), display_id)
+        json_data = self._parse_json(self._search_regex(
+            r'initialStoreTree\s*=\s*(?P<json_data>{.+})', webpage,
+            'initialStoreTree'), display_id)
 
-        info = None
         for post in json_data['resources']['posts'].values():
             if post['slug'] == display_id:
-                info = post
-                break
-        if info is None:
-            raise ExtractorError('Unable to find info for %s'%display_id)
-
-        title = info['title']
-        description = info['excerpt']
-        upload_date = info['date'][:10].replace('-', '')
-        video_id = info['fov_videoid']
-        creator = ','.join([a['display_name'] for a in info['authors']])
-        thumbnail = self._og_search_property('image', webpage)
-        content_id = thumbnail.split('/')[-1].split('.')[0]
-        content_url = 'https://content.jwplatform.com/jw6/{content_id}.xml'.format(content_id=content_id)
-        content = self._download_xml(content_url, video_id)
-
-        formats = []
-        for source in content.findall('.//{http://rss.jwpcdn.com/}source'):
-            if source.attrib['file'].endswith('.m3u8'):
-                formats.extend(self._extract_m3u8_formats(
-                    source.attrib['file'], video_id, 'mp4', preference=1, m3u8_id='hls'))
-
-        return {
-            'creator': creator,
-            'description': description,
-            'display_id': display_id,
-            'formats': formats,
-            'id': video_id,
-            'id': video_id,
-            'thumbnail': thumbnail,
-            'title': title,
-            'upload_date': upload_date,
-        }
+                return {
+                    '_type': 'url_transparent',
+                    'url': 'jwplatform:%s' % post['fov_videoid'],
+                    'id': compat_str(post['ID']),
+                    'display_id': display_id,
+                    'title': post['title'],
+                    'description': post.get('excerpt'),
+                    'timestamp': parse_iso8601(post.get('date')),
+                    'comment_count': int_or_none(post.get('comments_number')),
+                }
+        raise ExtractorError('Unable to find the current post')

From 8a609c32fdf8d99f7c868c74d64a05d9a936044d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 24 Dec 2015 20:09:48 +0600
Subject: [PATCH 3472/6219] [chaturbate] Improve error extraction (Closes
 #7989)

---
 youtube_dl/extractor/chaturbate.py | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/chaturbate.py b/youtube_dl/extractor/chaturbate.py
index 0b67ba67d..242fba311 100644
--- a/youtube_dl/extractor/chaturbate.py
+++ b/youtube_dl/extractor/chaturbate.py
@@ -23,6 +23,8 @@ class ChaturbateIE(InfoExtractor):
         'only_matching': True,
     }]
 
+    _ROOM_OFFLINE = 'Room is currently offline'
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
@@ -34,9 +36,16 @@ class ChaturbateIE(InfoExtractor):
 
         if not m3u8_url:
             error = self._search_regex(
-                r'<span[^>]+class=(["\'])desc_span\1[^>]*>(?P<error>[^<]+)</span>',
-                webpage, 'error', group='error')
-            raise ExtractorError(error, expected=True)
+                [r'<span[^>]+class=(["\'])desc_span\1[^>]*>(?P<error>[^<]+)</span>',
+                 r'<div[^>]+id=(["\'])defchat\1[^>]*>\s*<p><strong>(?P<error>[^<]+)<'],
+                webpage, 'error', group='error', default=None)
+            if not error:
+                if any(p not in webpage for p in (
+                        self._ROOM_OFFLINE, 'offline_tipping', 'tip_offline')):
+                    error = self._ROOM_OFFLINE
+            if error:
+                raise ExtractorError(error, expected=True)
+            raise ExtractorError('Unable to find stream URL')
 
         formats = self._extract_m3u8_formats(m3u8_url, video_id, ext='mp4')
 

From e41604227ce14ded0b87385a17490bfa65d35fde Mon Sep 17 00:00:00 2001
From: Boris Wachtmeister <boris-code@gmx.com>
Date: Thu, 24 Dec 2015 08:27:25 +0100
Subject: [PATCH 3473/6219] [zdf] expand valid-url pattern for channels

The webpage also creates URLs which include additional text that defines
the sorting order on the page like "aktuellste" (most current) and
"meist-gesehen" (most seen), e.g.:

http://www.zdf.de/ZDFmediathek/kanaluebersicht/aktuellste/332
http://www.zdf.de/ZDFmediathek/kanaluebersicht/meist-gesehen/332
---
 youtube_dl/extractor/zdf.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/zdf.py b/youtube_dl/extractor/zdf.py
index a795f56b3..b581813d6 100644
--- a/youtube_dl/extractor/zdf.py
+++ b/youtube_dl/extractor/zdf.py
@@ -131,7 +131,7 @@ class ZDFIE(InfoExtractor):
 
 
 class ZDFChannelIE(InfoExtractor):
-    _VALID_URL = r'(?:zdf:topic:|https?://www\.zdf\.de/ZDFmediathek(?:#)?/.*kanaluebersicht/)(?P<id>[0-9]+)'
+    _VALID_URL = r'(?:zdf:topic:|https?://www\.zdf\.de/ZDFmediathek(?:#)?/.*kanaluebersicht/(?:(?:aktuellste|meist-gesehen)/)?)(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://www.zdf.de/ZDFmediathek#/kanaluebersicht/1586442/sendung/Titanic',
         'info_dict': {

From 67ba388efb5a6fa925ba3472697484553f6ddcc9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 24 Dec 2015 20:42:29 +0600
Subject: [PATCH 3474/6219] [zdf:channel] Relax _VALID_URL

---
 youtube_dl/extractor/zdf.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/zdf.py b/youtube_dl/extractor/zdf.py
index b581813d6..d852ffd07 100644
--- a/youtube_dl/extractor/zdf.py
+++ b/youtube_dl/extractor/zdf.py
@@ -131,7 +131,7 @@ class ZDFIE(InfoExtractor):
 
 
 class ZDFChannelIE(InfoExtractor):
-    _VALID_URL = r'(?:zdf:topic:|https?://www\.zdf\.de/ZDFmediathek(?:#)?/.*kanaluebersicht/(?:(?:aktuellste|meist-gesehen)/)?)(?P<id>[0-9]+)'
+    _VALID_URL = r'(?:zdf:topic:|https?://www\.zdf\.de/ZDFmediathek(?:#)?/.*kanaluebersicht/(?:[^/]+/)?)(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://www.zdf.de/ZDFmediathek#/kanaluebersicht/1586442/sendung/Titanic',
         'info_dict': {

From c24044635b6e7f67274899e7c5e7a5efcce947af Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 24 Dec 2015 20:44:49 +0600
Subject: [PATCH 3475/6219] [zdf:channel] Add more tests

---
 youtube_dl/extractor/zdf.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/zdf.py b/youtube_dl/extractor/zdf.py
index d852ffd07..9a3331a69 100644
--- a/youtube_dl/extractor/zdf.py
+++ b/youtube_dl/extractor/zdf.py
@@ -132,13 +132,22 @@ class ZDFIE(InfoExtractor):
 
 class ZDFChannelIE(InfoExtractor):
     _VALID_URL = r'(?:zdf:topic:|https?://www\.zdf\.de/ZDFmediathek(?:#)?/.*kanaluebersicht/(?:[^/]+/)?)(?P<id>[0-9]+)'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.zdf.de/ZDFmediathek#/kanaluebersicht/1586442/sendung/Titanic',
         'info_dict': {
             'id': '1586442',
         },
         'playlist_count': 3,
-    }
+    }, {
+        'url': 'http://www.zdf.de/ZDFmediathek/kanaluebersicht/aktuellste/332',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.zdf.de/ZDFmediathek/kanaluebersicht/meist-gesehen/332',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.zdf.de/ZDFmediathek/kanaluebersicht/_/1798716?bc=nrt;nrm?flash=off',
+        'only_matching': True,
+    }]
     _PAGE_SIZE = 50
 
     def _fetch_page(self, channel_id, page):

From fb8e402ad252bebc54c36c5acca1b7ef05416c9b Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 25 Dec 2015 01:59:56 +0100
Subject: [PATCH 3476/6219] [hotstar] Add new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/hotstar.py  | 79 ++++++++++++++++++++++++++++++++
 2 files changed, 80 insertions(+)
 create mode 100644 youtube_dl/extractor/hotstar.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 365c0b86f..dd7e23d80 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -253,6 +253,7 @@ from .history import HistoryIE
 from .hitbox import HitboxIE, HitboxLiveIE
 from .hornbunny import HornBunnyIE
 from .hotnewhiphop import HotNewHipHopIE
+from .hotstar import HotStarIE
 from .howcast import HowcastIE
 from .howstuffworks import HowStuffWorksIE
 from .huffpost import HuffPostIE
diff --git a/youtube_dl/extractor/hotstar.py b/youtube_dl/extractor/hotstar.py
new file mode 100644
index 000000000..05d27e75d
--- /dev/null
+++ b/youtube_dl/extractor/hotstar.py
@@ -0,0 +1,79 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    determine_ext,
+    int_or_none,
+)
+
+
+class HotStarIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?hotstar\.com/.*?[/-](?P<id>\d{10})'
+    _TEST = {
+        'url': 'http://www.hotstar.com/on-air-with-aib--english-1000076273',
+        'info_dict': {
+            'id': '1000076273',
+            'ext': 'mp4',
+            'title': 'On Air With AIB - English',
+            'description': 'md5:c957d8868e9bc793ccb813691cc4c434',
+            'timestamp': 1447227000,
+            'upload_date': '20151111',
+            'duration': 381,
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        }
+    }
+
+    _GET_CONTENT_TEMPLATE = 'http://account.hotstar.com/AVS/besc?action=GetAggregatedContentDetails&channel=PCTV&contentId=%s'
+    _GET_CDN_TEMPLATE = 'http://getcdn.hotstar.com/AVS/besc?action=GetCDN&asJson=Y&channel=%s&id=%s&type=%s'
+
+    def _download_json(self, url_or_request, video_id, note='Downloading JSON metadata', fatal=True):
+        json_data = super(HotStarIE, self)._download_json(url_or_request, video_id, note, fatal=fatal)
+        if json_data['resultCode'] != 'OK':
+            if fatal:
+                raise ExtractorError(json_data['errorDescription'])
+            return None
+        return json_data['resultObj']
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        video_data = self._download_json(
+            self._GET_CONTENT_TEMPLATE % video_id,
+            video_id)['contentInfo'][0]
+
+        formats = []
+        # PCTV for extracting f4m manifest
+        for f in ('TABLET',):
+            format_data = self._download_json(
+                self._GET_CDN_TEMPLATE % (f, video_id, 'VOD'),
+                video_id, 'Downloading %s JSON metadata' % f, fatal=False)
+            if format_data:
+                format_url = format_data['src']
+                ext = determine_ext(format_url)
+                if ext == 'm3u8':
+                    m3u8_formats = self._extract_m3u8_formats(format_url, video_id, 'mp4', m3u8_id='hls', fatal=False)
+                    if m3u8_formats:
+                        formats.extend(m3u8_formats)
+                elif ext == 'f4m':
+                    # produce broken files
+                    continue
+                else:
+                    formats.append({
+                        'url': format_url,
+                        'width': int_or_none(format_data.get('width')),
+                        'height': int_or_none(format_data.get('height')),
+                    })
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': video_data['episodeTitle'],
+            'description': video_data.get('description'),
+            'duration': int_or_none(video_data.get('duration')),
+            'timestamp': int_or_none(video_data.get('broadcastDate')),
+            'formats': formats,
+        }

From 06d5556dface3901a86419b6b125ef377116448f Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 25 Dec 2015 15:38:12 +0100
Subject: [PATCH 3477/6219] [rai] improve extraction

---
 youtube_dl/extractor/__init__.py |   5 +-
 youtube_dl/extractor/rai.py      | 203 ++++++++++++++++++-------------
 2 files changed, 121 insertions(+), 87 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index dd7e23d80..4784db70b 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -528,7 +528,10 @@ from .radiode import RadioDeIE
 from .radiojavan import RadioJavanIE
 from .radiobremen import RadioBremenIE
 from .radiofrance import RadioFranceIE
-from .rai import RaiIE
+from .rai import (
+    RaiTVIE,
+    RaiIE,
+)
 from .rbmaradio import RBMARadioIE
 from .rds import RDSIE
 from .redtube import RedTubeIE
diff --git a/youtube_dl/extractor/rai.py b/youtube_dl/extractor/rai.py
index 7ff1d06c4..14f1ccbb4 100644
--- a/youtube_dl/extractor/rai.py
+++ b/youtube_dl/extractor/rai.py
@@ -8,20 +8,24 @@ from ..compat import (
     compat_urlparse,
 )
 from ..utils import (
+    ExtractorError,
+    determine_ext,
     parse_duration,
     unified_strdate,
+    int_or_none,
+    xpath_text,
 )
 
 
-class RaiIE(InfoExtractor):
-    _VALID_URL = r'(?P<url>(?P<host>http://(?:.+?\.)?(?:rai\.it|rai\.tv|rainews\.it))/dl/.+?-(?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})(?:-.+?)?\.html)'
+class RaiTVIE(InfoExtractor):
+    _VALID_URL = r'http://(?:.+?\.)?(?:rai\.it|rai\.tv|rainews\.it)/dl/(?:[^/]+/)+media/.+?-(?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})(?:-.+?)?\.html'
     _TESTS = [
         {
             'url': 'http://www.rai.tv/dl/RaiTV/programmi/media/ContentItem-cb27157f-9dd0-4aee-b788-b1f67643a391.html',
-            'md5': 'c064c0b2d09c278fb293116ef5d0a32d',
+            'md5': '96382709b61dd64a6b88e0f791e6df4c',
             'info_dict': {
                 'id': 'cb27157f-9dd0-4aee-b788-b1f67643a391',
-                'ext': 'mp4',
+                'ext': 'flv',
                 'title': 'Report del 07/04/2014',
                 'description': 'md5:f27c544694cacb46a078db84ec35d2d9',
                 'upload_date': '20140407',
@@ -30,16 +34,14 @@ class RaiIE(InfoExtractor):
         },
         {
             'url': 'http://www.raisport.rai.it/dl/raiSport/media/rassegna-stampa-04a9f4bd-b563-40cf-82a6-aad3529cb4a9.html',
-            'md5': '8bb9c151924ce241b74dd52ef29ceafa',
+            'md5': 'd9751b78eac9710d62c2447b224dea39',
             'info_dict': {
                 'id': '04a9f4bd-b563-40cf-82a6-aad3529cb4a9',
-                'ext': 'mp4',
+                'ext': 'flv',
                 'title': 'TG PRIMO TEMPO',
-                'description': '',
                 'upload_date': '20140612',
                 'duration': 1758,
             },
-            'skip': 'Error 404',
         },
         {
             'url': 'http://www.rainews.it/dl/rainews/media/state-of-the-net-Antonella-La-Carpia-regole-virali-7aafdea9-0e5d-49d5-88a6-7e65da67ae13.html',
@@ -55,110 +57,106 @@ class RaiIE(InfoExtractor):
         },
         {
             'url': 'http://www.rai.tv/dl/RaiTV/programmi/media/ContentItem-b4a49761-e0cc-4b14-8736-2729f6f73132-tg2.html',
-            'md5': '35694f062977fe6619943f08ed935730',
             'info_dict': {
                 'id': 'b4a49761-e0cc-4b14-8736-2729f6f73132',
                 'ext': 'mp4',
                 'title': 'Alluvione in Sardegna e dissesto idrogeologico',
                 'description': 'Edizione delle ore 20:30 ',
-            }
+            },
+            'skip': 'invalid urls',
         },
         {
             'url': 'http://www.ilcandidato.rai.it/dl/ray/media/Il-Candidato---Primo-episodio-Le-Primarie-28e5525a-b495-45e8-a7c3-bc48ba45d2b6.html',
-            'md5': '02b64456f7cc09f96ff14e7dd489017e',
+            'md5': '496ab63e420574447f70d02578333437',
             'info_dict': {
                 'id': '28e5525a-b495-45e8-a7c3-bc48ba45d2b6',
                 'ext': 'flv',
                 'title': 'Il Candidato - Primo episodio: "Le Primarie"',
-                'description': 'Primo appuntamento con "Il candidato" con Filippo Timi, alias Piero Zucca presidente!',
-                'uploader': 'RaiTre',
+                'description': 'md5:364b604f7db50594678f483353164fb8',
+                'upload_date': '20140923',
+                'duration': 386,
             }
         },
-        {
-            'url': 'http://www.report.rai.it/dl/Report/puntata/ContentItem-0c7a664b-d0f4-4b2c-8835-3f82e46f433e.html',
-            'md5': '037104d2c14132887e5e4cf114569214',
-            'info_dict': {
-                'id': '0c7a664b-d0f4-4b2c-8835-3f82e46f433e',
-                'ext': 'flv',
-                'title': 'Il pacco',
-                'description': 'md5:4b1afae1364115ce5d78ed83cd2e5b3a',
-                'uploader': 'RaiTre',
-                'upload_date': '20141221',
-            },
-        }
     ]
 
-    def _extract_relinker_url(self, webpage):
-        return self._proto_relative_url(self._search_regex(
-            [r'name="videourl" content="([^"]+)"', r'var\s+videoURL(?:_MP4)?\s*=\s*"([^"]+)"'],
-            webpage, 'relinker url', default=None))
-
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        host = mobj.group('host')
+        video_id = self._match_id(url)
+        media = self._download_json(
+            'http://www.rai.tv/dl/RaiTV/programmi/media/ContentItem-%s.html?json' % video_id,
+            video_id, 'Downloading video JSON')
 
-        webpage = self._download_webpage(url, video_id)
+        thumbnails = []
+        for image_type in ('image', 'image_medium', 'image_300'):
+            thumbnail_url = media.get(image_type)
+            if thumbnail_url:
+                thumbnails.append({
+                    'url': thumbnail_url,
+                })
 
-        relinker_url = self._extract_relinker_url(webpage)
+        subtitles = []
+        formats = []
+        media_type = media['type']
+        if 'Audio' in media_type:
+            formats.append({
+                'format_id': media.get('formatoAudio'),
+                'url': media['audioUrl'],
+                'ext': media.get('formatoAudio'),
+            })
+        elif 'Video' in media_type:
+            def fix_xml(xml):
+                return xml.replace(' tag elementi', '').replace('>/', '</')
 
-        if not relinker_url:
-            iframe_url = self._search_regex(
-                [r'<iframe[^>]+src="([^"]*/dl/[^"]+\?iframe\b[^"]*)"',
-                 r'drawMediaRaiTV\(["\'](.+?)["\']'],
-                webpage, 'iframe')
-            if not iframe_url.startswith('http'):
-                iframe_url = compat_urlparse.urljoin(url, iframe_url)
-            webpage = self._download_webpage(
-                iframe_url, video_id)
-            relinker_url = self._extract_relinker_url(webpage)
+            relinker = self._download_xml(
+                media['mediaUri'] + '&output=43', video_id, transform_source=fix_xml)
 
-        relinker = self._download_json(
-            '%s&output=47' % relinker_url, video_id)
+            has_subtitle = False
 
-        media_url = relinker['video'][0]
-        ct = relinker.get('ct')
-        if ct == 'f4m':
-            formats = self._extract_f4m_formats(
-                media_url + '&hdcore=3.7.0&plugin=aasp-3.7.0.39.44', video_id)
+            for element in relinker.findall('element'):
+                media_url = xpath_text(element, 'url')
+                ext = determine_ext(media_url)
+                content_type = xpath_text(element, 'content-type')
+                if ext == 'm3u8':
+                    m3u8_formats = self._extract_m3u8_formats(
+                        media_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls',
+                        fatal=None)
+                    if m3u8_formats:
+                        formats.extend(m3u8_formats)
+                elif ext == 'f4m':
+                    f4m_formats = self._extract_f4m_formats(
+                        media_url + '?hdcore=3.7.0&plugin=aasp-3.7.0.39.44',
+                        video_id, f4m_id='hds', fatal=None)
+                    if f4m_formats:
+                        formats.extend(f4m_formats)
+                elif ext == 'stl':
+                    has_subtitle = True
+                elif content_type.startswith('video/'):
+                    bitrate = int_or_none(xpath_text(element, 'bitrate'))
+                    formats.append({
+                        'url': media_url,
+                        'tbr': bitrate if bitrate > 0 else None,
+                        'format_id': 'http-%d' % bitrate if bitrate > 0 else 'http',
+                    })
+                elif content_type.startswith('image/'):
+                    thumbnails.append({
+                        'url': media_url,
+                    })
+
+            self._sort_formats(formats)
+
+            if has_subtitle:
+                webpage = self._download_webpage(url, video_id)
+                subtitles = self._get_subtitles(video_id, webpage)
         else:
-            formats = [{
-                'url': media_url,
-                'format_id': ct,
-            }]
-
-        json_link = self._html_search_meta(
-            'jsonlink', webpage, 'JSON link', default=None)
-        if json_link:
-            media = self._download_json(
-                host + json_link, video_id, 'Downloading video JSON')
-            title = media.get('name')
-            description = media.get('desc')
-            thumbnail = media.get('image_300') or media.get('image_medium') or media.get('image')
-            duration = parse_duration(media.get('length'))
-            uploader = media.get('author')
-            upload_date = unified_strdate(media.get('date'))
-        else:
-            title = (self._search_regex(
-                r'var\s+videoTitolo\s*=\s*"(.+?)";',
-                webpage, 'title', default=None) or self._og_search_title(webpage)).replace('\\"', '"')
-            description = self._og_search_description(webpage)
-            thumbnail = self._og_search_thumbnail(webpage)
-            duration = None
-            uploader = self._html_search_meta('Editore', webpage, 'uploader')
-            upload_date = unified_strdate(self._html_search_meta(
-                'item-date', webpage, 'upload date', default=None))
-
-        subtitles = self.extract_subtitles(video_id, webpage)
+            raise ExtractorError('not a media file')
 
         return {
             'id': video_id,
-            'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
-            'uploader': uploader,
-            'upload_date': upload_date,
-            'duration': duration,
+            'title': media['name'],
+            'description': media.get('desc'),
+            'thumbnails': thumbnails,
+            'uploader': media.get('author'),
+            'upload_date': unified_strdate(media.get('date')),
+            'duration': parse_duration(media.get('length')),
             'formats': formats,
             'subtitles': subtitles,
         }
@@ -177,3 +175,36 @@ class RaiIE(InfoExtractor):
                 'url': 'http://www.rai.tv%s' % compat_urllib_parse.quote(captions),
             }]
         return subtitles
+
+
+class RaiIE(InfoExtractor):
+    _VALID_URL = r'http://(?:.+?\.)?(?:rai\.it|rai\.tv|rainews\.it)/dl/.+?-(?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})(?:-.+?)?\.html'
+    _TESTS = [
+        {
+            'url': 'http://www.report.rai.it/dl/Report/puntata/ContentItem-0c7a664b-d0f4-4b2c-8835-3f82e46f433e.html',
+            'md5': 'e0e7a8a131e249d1aa0ebf270d1d8db7',
+            'info_dict': {
+                'id': '59d69d28-6bb6-409d-a4b5-ed44096560af',
+                'ext': 'flv',
+                'title': 'Il pacco',
+                'description': 'md5:4b1afae1364115ce5d78ed83cd2e5b3a',
+                'upload_date': '20141221',
+            },
+        }
+    ]
+
+    @classmethod
+    def suitable(cls, url):
+        return False if RaiTVIE.suitable(url) else super(RaiIE, cls).suitable(url)
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        iframe_url = self._search_regex(
+            [r'<iframe[^>]+src="([^"]*/dl/[^"]+\?iframe\b[^"]*)"',
+             r'drawMediaRaiTV\(["\'](.+?)["\']'],
+            webpage, 'iframe')
+        if not iframe_url.startswith('http'):
+            iframe_url = compat_urlparse.urljoin(url, iframe_url)
+        return self.url_result(iframe_url)

From 6418b2439b81ceec77b50879b4d9d395893d8eba Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 25 Dec 2015 21:14:00 +0600
Subject: [PATCH 3478/6219] [rutv] Fix extraction (Closes #8004)

---
 youtube_dl/extractor/rutv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/rutv.py b/youtube_dl/extractor/rutv.py
index d9df06861..f7fe1fece 100644
--- a/youtube_dl/extractor/rutv.py
+++ b/youtube_dl/extractor/rutv.py
@@ -131,7 +131,7 @@ class RUTVIE(InfoExtractor):
         is_live = video_type == 'live'
 
         json_data = self._download_json(
-            'http://player.rutv.ru/iframe/%splay/id/%s' % ('live-' if is_live else '', video_id),
+            'http://player.rutv.ru/iframe/data%s/id/%s' % ('live' if is_live else 'video', video_id),
             video_id, 'Downloading JSON')
 
         if json_data['errors']:

From 1fc0b47fdf9367aa71e6b81076666f137e68f637 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 25 Dec 2015 17:37:50 +0100
Subject: [PATCH 3479/6219] [srmediathek] improve extraction

---
 youtube_dl/extractor/ard.py         | 14 ++++++---
 youtube_dl/extractor/srmediathek.py | 49 ++++++++++++++++++-----------
 2 files changed, 41 insertions(+), 22 deletions(-)

diff --git a/youtube_dl/extractor/ard.py b/youtube_dl/extractor/ard.py
index 73be6d204..687eb9f82 100644
--- a/youtube_dl/extractor/ard.py
+++ b/youtube_dl/extractor/ard.py
@@ -110,13 +110,19 @@ class ARDMediathekIE(InfoExtractor):
                 server = stream.get('_server')
                 for stream_url in stream_urls:
                     ext = determine_ext(stream_url)
+                    if quality != 'auto' and ext in ('f4m', 'm3u8'):
+                        continue
                     if ext == 'f4m':
-                        formats.extend(self._extract_f4m_formats(
+                        f4m_formats = self._extract_f4m_formats(
                             stream_url + '?hdcore=3.1.1&plugin=aasp-3.1.1.69.124',
-                            video_id, preference=-1, f4m_id='hds'))
+                            video_id, preference=-1, f4m_id='hds', fatal=False)
+                        if f4m_formats:
+                            formats.extend(f4m_formats)
                     elif ext == 'm3u8':
-                        formats.extend(self._extract_m3u8_formats(
-                            stream_url, video_id, 'mp4', preference=1, m3u8_id='hls'))
+                        m3u8_formats = self._extract_m3u8_formats(
+                            stream_url, video_id, 'mp4', preference=1, m3u8_id='hls', fatal=False)
+                        if m3u8_formats:
+                            formats.extend(m3u8_formats)
                     else:
                         if server and server.startswith('rtmp'):
                             f = {
diff --git a/youtube_dl/extractor/srmediathek.py b/youtube_dl/extractor/srmediathek.py
index 5d583c720..74d01183f 100644
--- a/youtube_dl/extractor/srmediathek.py
+++ b/youtube_dl/extractor/srmediathek.py
@@ -1,17 +1,18 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
-import json
-
-from .common import InfoExtractor
-from ..utils import js_to_json
+from .ard import ARDMediathekIE
+from ..utils import (
+    ExtractorError,
+    get_element_by_attribute,
+)
 
 
-class SRMediathekIE(InfoExtractor):
+class SRMediathekIE(ARDMediathekIE):
     IE_DESC = 'Saarländischer Rundfunk'
     _VALID_URL = r'https?://sr-mediathek\.sr-online\.de/index\.php\?.*?&id=(?P<id>[0-9]+)'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://sr-mediathek.sr-online.de/index.php?seite=7&id=28455',
         'info_dict': {
             'id': '28455',
@@ -20,24 +21,36 @@ class SRMediathekIE(InfoExtractor):
             'description': 'Ringen: KSV Köllerbach gegen Aachen-Walheim; Frauen-Fußball: 1. FC Saarbrücken gegen Sindelfingen; Motorsport: Rallye in Losheim; dazu: Interview mit Timo Bernhard; Turnen: TG Saar; Reitsport: Deutscher Voltigier-Pokal; Badminton: Interview mit Michael Fuchs ',
             'thumbnail': 're:^https?://.*\.jpg$',
         },
-    }
+        'skip': 'no longer available',
+    }, {
+        'url': 'http://sr-mediathek.sr-online.de/index.php?seite=7&id=37682',
+        'info_dict': {
+            'id': '37682',
+            'ext': 'mp4',
+            'title': 'Love, Cakes and Rock\'n\'Roll',
+            'description': 'md5:18bf9763631c7d326c22603681e1123d',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+        'expected_warnings': ['Unable to download f4m manifest']
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        murls = json.loads(js_to_json(self._search_regex(
-            r'var mediaURLs\s*=\s*(.*?);\n', webpage, 'video URLs')))
-        formats = [{'url': murl} for murl in murls]
-        self._sort_formats(formats)
+        if '>Der gew&uuml;nschte Beitrag ist leider nicht mehr verf&uuml;gbar.<' in webpage:
+            raise ExtractorError('Video %s is no longer available' % video_id, expected=True)
 
-        title = json.loads(js_to_json(self._search_regex(
-            r'var mediaTitles\s*=\s*(.*?);\n', webpage, 'title')))[0]
-
-        return {
+        media_collection_url = self._search_regex(
+            r'data-mediacollection-ardplayer="([^"]+)"', webpage, 'media collection url')
+        info = self._extract_media_info(media_collection_url, webpage, video_id)
+        info.update({
             'id': video_id,
-            'title': title,
-            'formats': formats,
+            'title': get_element_by_attribute('class', 'ardplayer-title', webpage),
             'description': self._og_search_description(webpage),
             'thumbnail': self._og_search_thumbnail(webpage),
-        }
+        })
+        return info

From 607d65fbceebe7df6d45bb02d6eab378896f099b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 26 Dec 2015 03:17:56 +0600
Subject: [PATCH 3480/6219] [ign] flake8

---
 youtube_dl/extractor/ign.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/ign.py b/youtube_dl/extractor/ign.py
index 6160f03d1..a2e18c8a7 100644
--- a/youtube_dl/extractor/ign.py
+++ b/youtube_dl/extractor/ign.py
@@ -109,7 +109,7 @@ class IGNIE(InfoExtractor):
         if page_type != 'video':
             multiple_urls = re.findall(
                 r'<param name="flashvars"[^>]*value="[^"]*?url=(https?://www\.ign\.com/videos/.*?)["&]',
-                 webpage)
+                webpage)
             if multiple_urls:
                 entries = [self.url_result(u, ie='IGN') for u in multiple_urls]
                 return {
@@ -209,7 +209,7 @@ class PCMagIE(IGNIE):
             'upload_date': '20150106',
             'uploader_id': 'cozzipix@gmail.com',
         }
-    },{
+    }, {
         'url': 'http://www.pcmag.com/article2/0,2817,2470156,00.asp',
         'md5': '94130c1ca07ba0adb6088350681f16c1',
         'info_dict': {

From 85367c3a478241a9d47be76ff5418b512cd40eed Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Giedrius=20Statkevi=C4=8Dius?=
 <giedrius.statkevicius@gmail.com>
Date: Sat, 28 Nov 2015 23:14:38 +0200
Subject: [PATCH 3481/6219] [lrt] fix duration parsing

---
 youtube_dl/extractor/lrt.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/lrt.py b/youtube_dl/extractor/lrt.py
index e3236f7b5..24a9a4c91 100644
--- a/youtube_dl/extractor/lrt.py
+++ b/youtube_dl/extractor/lrt.py
@@ -37,8 +37,7 @@ class LRTIE(InfoExtractor):
         thumbnail = self._og_search_thumbnail(webpage)
         description = self._og_search_description(webpage)
         duration = parse_duration(self._search_regex(
-            r"'duration':\s*'([^']+)',", webpage,
-            'duration', fatal=False, default=None))
+            r"var record_len = '([0-9]+:[0-9]+:[0-9]+)';", webpage, 'record_len', fatal=False, default=None))
 
         formats = []
         for js in re.findall(r'(?s)config:\s*(\{.*?\})', webpage):

From 339b1944e7c8c28d1fc7cc8721af73fd23937a1c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Giedrius=20Statkevi=C4=8Dius?=
 <giedrius.statkevicius@gmail.com>
Date: Sun, 29 Nov 2015 02:58:52 +0200
Subject: [PATCH 3482/6219] [lrt] fix the rest of extractor

Closes #7690.
---
 youtube_dl/extractor/lrt.py | 25 +++----------------------
 1 file changed, 3 insertions(+), 22 deletions(-)

diff --git a/youtube_dl/extractor/lrt.py b/youtube_dl/extractor/lrt.py
index 24a9a4c91..42e768ce7 100644
--- a/youtube_dl/extractor/lrt.py
+++ b/youtube_dl/extractor/lrt.py
@@ -1,12 +1,8 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..utils import (
-    determine_ext,
-    js_to_json,
     parse_duration,
     remove_end,
 )
@@ -25,7 +21,7 @@ class LRTIE(InfoExtractor):
             'duration': 1783,
         },
         'params': {
-            'skip_download': True,  # HLS download
+            'skip_download': True,  # m3u8 download
         },
     }
 
@@ -39,23 +35,8 @@ class LRTIE(InfoExtractor):
         duration = parse_duration(self._search_regex(
             r"var record_len = '([0-9]+:[0-9]+:[0-9]+)';", webpage, 'record_len', fatal=False, default=None))
 
-        formats = []
-        for js in re.findall(r'(?s)config:\s*(\{.*?\})', webpage):
-            data = self._parse_json(js, video_id, transform_source=js_to_json)
-            if 'provider' not in data:
-                continue
-            if data['provider'] == 'rtmp':
-                formats.append({
-                    'format_id': 'rtmp',
-                    'ext': determine_ext(data['file']),
-                    'url': data['streamer'],
-                    'play_path': 'mp4:%s' % data['file'],
-                    'preference': -1,
-                    'rtmp_real_time': True,
-                })
-            else:
-                formats.extend(
-                    self._extract_m3u8_formats(data['file'], video_id, 'mp4'))
+        link = self._search_regex(r'file: "(.*)" \+ location\.hash\.substring\(1\)', webpage, 'link to m3u8')
+        formats = self._extract_m3u8_formats(link, video_id, "mp4")
 
         return {
             'id': video_id,

From f7e1d82d407ccb0be23999ba8af27ae2549d39aa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 27 Dec 2015 12:16:55 +0600
Subject: [PATCH 3483/6219] [lrt] Improve

---
 youtube_dl/extractor/lrt.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/lrt.py b/youtube_dl/extractor/lrt.py
index 42e768ce7..f8d27598a 100644
--- a/youtube_dl/extractor/lrt.py
+++ b/youtube_dl/extractor/lrt.py
@@ -30,13 +30,16 @@ class LRTIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         title = remove_end(self._og_search_title(webpage), ' - LRT')
+        m3u8_url = self._search_regex(
+            r'file\s*:\s*(["\'])(?P<url>.+?)\1\s*\+\s*location\.hash\.substring\(1\)',
+            webpage, 'm3u8 url', group='url')
+        formats = self._extract_m3u8_formats(m3u8_url, video_id, 'mp4')
+
         thumbnail = self._og_search_thumbnail(webpage)
         description = self._og_search_description(webpage)
         duration = parse_duration(self._search_regex(
-            r"var record_len = '([0-9]+:[0-9]+:[0-9]+)';", webpage, 'record_len', fatal=False, default=None))
-
-        link = self._search_regex(r'file: "(.*)" \+ location\.hash\.substring\(1\)', webpage, 'link to m3u8')
-        formats = self._extract_m3u8_formats(link, video_id, "mp4")
+            r'var\s+record_len\s*=\s*(["\'])(?P<duration>[0-9]+:[0-9]+:[0-9]+)\1',
+            webpage, 'duration', default=None, group='duration'))
 
         return {
             'id': video_id,

From 15aad84dc5ea9d60ffd4a1fd5d67f90afd11b4c6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 27 Dec 2015 12:26:48 +0600
Subject: [PATCH 3484/6219] [lrt] Extract counters

---
 youtube_dl/extractor/lrt.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/youtube_dl/extractor/lrt.py b/youtube_dl/extractor/lrt.py
index f8d27598a..863efd896 100644
--- a/youtube_dl/extractor/lrt.py
+++ b/youtube_dl/extractor/lrt.py
@@ -3,6 +3,7 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..utils import (
+    int_or_none,
     parse_duration,
     remove_end,
 )
@@ -19,6 +20,8 @@ class LRTIE(InfoExtractor):
             'title': 'Septynios Kauno dienos',
             'description': 'md5:24d84534c7dc76581e59f5689462411a',
             'duration': 1783,
+            'view_count': int,
+            'like_count': int,
         },
         'params': {
             'skip_download': True,  # m3u8 download
@@ -41,6 +44,13 @@ class LRTIE(InfoExtractor):
             r'var\s+record_len\s*=\s*(["\'])(?P<duration>[0-9]+:[0-9]+:[0-9]+)\1',
             webpage, 'duration', default=None, group='duration'))
 
+        view_count = int_or_none(self._html_search_regex(
+            r'<div[^>]+class=(["\']).*?record-desc-seen.*?\1[^>]*>(?P<count>.+?)</div>',
+            webpage, 'view count', fatal=False, group='count'))
+        like_count = int_or_none(self._search_regex(
+            r'<span[^>]+id=(["\'])flikesCount.*?\1>(?P<count>\d+)<',
+            webpage, 'like count', fatal=False, group='count'))
+
         return {
             'id': video_id,
             'title': title,
@@ -48,4 +58,6 @@ class LRTIE(InfoExtractor):
             'thumbnail': thumbnail,
             'description': description,
             'duration': duration,
+            'view_count': view_count,
+            'like_count': like_count,
         }

From 6afe044b51e99a3a0e638492e118634a2ca3cdad Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 27 Dec 2015 09:56:15 +0100
Subject: [PATCH 3485/6219] [dcn] improve extraction

---
 youtube_dl/extractor/__init__.py |   2 +-
 youtube_dl/extractor/dcn.py      | 148 ++++++++++++++-----------------
 2 files changed, 67 insertions(+), 83 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 677c75564..abf36caf3 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -119,7 +119,7 @@ from .dailymotion import (
 from .daum import DaumIE
 from .dbtv import DBTVIE
 from .dcn import (
-    DCNGeneralIE,
+    DCNIE,
     DCNVideoIE,
     DCNLiveIE,
     DCNSeasonIE,
diff --git a/youtube_dl/extractor/dcn.py b/youtube_dl/extractor/dcn.py
index a9a5e94f5..3857ba334 100644
--- a/youtube_dl/extractor/dcn.py
+++ b/youtube_dl/extractor/dcn.py
@@ -17,24 +17,61 @@ from ..utils import (
 )
 
 
-class DCNGeneralIE(InfoExtractor):
+class DCNIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?dcndigital\.ae/(?:#/)?show/(?P<show_id>\d+)/[^/]+(?:/(?P<video_id>\d+)/(?P<season_id>\d+))?'
 
     def _real_extract(self, url):
         show_id, video_id, season_id = re.match(self._VALID_URL, url).groups()
-        url = ''
-        ie_key = ''
         if video_id and int(video_id) > 0:
-            return self.url_result('http://www.dcndigital.ae/#/media/%s' % video_id, 'DCNVideo')
+            return self.url_result(
+                'http://www.dcndigital.ae/media/%s' % video_id, 'DCNVideo')
+        elif season_id and int(season_id) > 0:
+            return self.url_result(smuggle_url(
+                'http://www.dcndigital.ae/program/season/%s' % season_id,
+                {'show_id': show_id}), 'DCNSeason')
         else:
-            if season_id and int(season_id) > 0:
-                url = smuggle_url('http://www.dcndigital.ae/#/program/season/%s' % season_id, {'show_id': show_id})
-            else:
-                url = 'http://www.dcndigital.ae/#/program/%s' % show_id
-            return self.url_result(url, 'DCNSeason')
+            return self.url_result(
+                'http://www.dcndigital.ae/program/%s' % show_id, 'DCNSeason')
 
 
-class DCNVideoIE(InfoExtractor):
+class DCNBaseIE(InfoExtractor):
+    def _extract_video_info(self, video_data, video_id, is_live):
+        title = video_data.get('title_en') or video_data['title_ar']
+        img = video_data.get('img')
+        thumbnail = 'http://admin.mangomolo.com/analytics/%s' % img if img else None
+        duration = int_or_none(video_data.get('duration'))
+        description = video_data.get('description_en') or video_data.get('description_ar')
+        timestamp = parse_iso8601(video_data.get('create_time'), ' ')
+
+        return {
+            'id': video_id,
+            'title': self._live_title(title) if is_live else title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'duration': duration,
+            'timestamp': timestamp,
+            'is_live': is_live,
+        }
+
+    def _extract_video_formats(self, webpage, video_id, entry_protocol):
+        m3u8_url = self._html_search_regex(
+            r'file\s*:\s*"([^"]+)', webpage, 'm3u8 url')
+        formats = self._extract_m3u8_formats(
+            m3u8_url, video_id, 'mp4', entry_protocol, m3u8_id='hls')
+
+        rtsp_url = self._search_regex(
+            r'<a[^>]+href="(rtsp://[^"]+)"', webpage, 'rtsp url', fatal=False)
+        if rtsp_url:
+            formats.append({
+                'url': rtsp_url,
+                'format_id': 'rtsp',
+            })
+
+        self._sort_formats(formats)
+        return formats
+
+
+class DCNVideoIE(DCNBaseIE):
     IE_NAME = 'dcn:video'
     _VALID_URL = r'https?://(?:www\.)?dcndigital\.ae/(?:#/)?(?:video/[^/]+|media|catchup/[^/]+/[^/]+)/(?P<id>\d+)'
     _TEST = {
@@ -45,7 +82,6 @@ class DCNVideoIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'رحلة العمر : الحلقة 1',
             'description': 'md5:0156e935d870acb8ef0a66d24070c6d6',
-            'thumbnail': 're:^https?://.*\.jpg$',
             'duration': 2041,
             'timestamp': 1227504126,
             'upload_date': '20081124',
@@ -62,51 +98,23 @@ class DCNVideoIE(InfoExtractor):
         request = compat_urllib_request.Request(
             'http://admin.mangomolo.com/analytics/index.php/plus/video?id=%s' % video_id,
             headers={'Origin': 'http://www.dcndigital.ae'})
-
-        video = self._download_json(request, video_id)
-        title = video.get('title_en') or video['title_ar']
-        img = video.get('img')
-        thumbnail = 'http://admin.mangomolo.com/analytics/%s' % img if img else None
-        duration = int_or_none(video.get('duration'))
-        description = video.get('description_en') or video.get('description_ar')
-        timestamp = parse_iso8601(video.get('create_time') or video.get('update_time'), ' ')
+        video_data = self._download_json(request, video_id)
+        info = self._extract_video_info(video_data, video_id, False)
 
         webpage = self._download_webpage(
-            'http://admin.mangomolo.com/analytics/index.php/customers/embed/video?'
-            + compat_urllib_parse.urlencode({
-                'id': video['id'],
-                'user_id': video['user_id'],
-                'signature': video['signature'],
+            'http://admin.mangomolo.com/analytics/index.php/customers/embed/video?' +
+            compat_urllib_parse.urlencode({
+                'id': video_data['id'],
+                'user_id': video_data['user_id'],
+                'signature': video_data['signature'],
                 'countries': 'Q0M=',
                 'filter': 'DENY',
             }), video_id)
-
-        m3u8_url = self._html_search_regex(r'file:\s*"([^"]+)', webpage, 'm3u8 url')
-        formats = self._extract_m3u8_formats(
-            m3u8_url, video_id, 'mp4', entry_protocol='m3u8_native', m3u8_id='hls')
-
-        rtsp_url = self._search_regex(
-            r'<a[^>]+href="(rtsp://[^"]+)"', webpage, 'rtsp url', fatal=False)
-        if rtsp_url:
-            formats.append({
-                'url': rtsp_url,
-                'format_id': 'rtsp',
-            })
-
-        self._sort_formats(formats)
-
-        return {
-            'id': video_id,
-            'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
-            'duration': duration,
-            'timestamp': timestamp,
-            'formats': formats,
-        }
+        info['formats'] = self._extract_video_formats(webpage, video_id, 'm3u8_native')
+        return info
 
 
-class DCNLiveIE(InfoExtractor):
+class DCNLiveIE(DCNBaseIE):
     IE_NAME = 'dcn:live'
     _VALID_URL = r'https?://(?:www\.)?dcndigital\.ae/(?:#/)?live/(?P<id>\d+)'
     _TEST = {
@@ -132,45 +140,20 @@ class DCNLiveIE(InfoExtractor):
             'http://admin.mangomolo.com/analytics/index.php/plus/getchanneldetails?channel_id=%s' % channel_id,
             headers={'Origin': 'http://www.dcndigital.ae'})
 
-        channel = self._download_json(request, channel_id)
-        title = channel.get('title_en') or channel['title_ar']
-        img = channel.get('thumbnail')
-        thumbnail = 'http://admin.mangomolo.com/analytics/%s' % img if img else None
-        description = channel.get('description_en') or channel.get('description_ar')
-        timestamp = parse_iso8601(channel.get('create_time') or channel.get('update_time'), ' ')
+        channel_data = self._download_json(request, channel_id)
+        info = self._extract_video_info(channel_data, channel_id, True)
 
         webpage = self._download_webpage(
             'http://admin.mangomolo.com/analytics/index.php/customers/embed/index?' +
             compat_urllib_parse.urlencode({
-                'id': base64.b64encode(channel['user_id'].encode()).decode(),
-                'channelid': base64.b64encode(channel['id'].encode()).decode(),
-                'signature': channel['signature'],
+                'id': base64.b64encode(channel_data['user_id'].encode()).decode(),
+                'channelid': base64.b64encode(channel_data['id'].encode()).decode(),
+                'signature': channel_data['signature'],
                 'countries': 'Q0M=',
                 'filter': 'DENY',
             }), channel_id)
-
-        m3u8_url = self._html_search_regex(r'file:\s*"([^"]+)', webpage, 'm3u8 url')
-        formats = self._extract_m3u8_formats(
-            m3u8_url, channel_id, 'mp4', entry_protocol='m3u8_native', m3u8_id='hls')
-
-        rtsp_url = self._search_regex(
-            r'<a[^>]+href="(rtsp://[^"]+)"', webpage, 'rtsp url', fatal=False)
-        if rtsp_url:
-            formats.append({
-                'url': rtsp_url,
-                'format_id': 'rtsp',
-            })
-
-        self._sort_formats(formats)
-
-        return {
-            'id': channel_id,
-            'title': self._live_title(title),
-            'description': description,
-            'thumbnail': thumbnail,
-            'formats': formats,
-            'is_live': True,
-        }
+        info['formats'] = self._extract_video_formats(webpage, channel_id, 'm3u8')
+        return info
 
 
 class DCNSeasonIE(InfoExtractor):
@@ -218,6 +201,7 @@ class DCNSeasonIE(InfoExtractor):
 
                 entries = []
                 for video in show['videos']:
-                    entries.append(self.url_result('http://www.dcndigital.ae/#/media/%s' % video['id'], 'DCNVideo'))
+                    entries.append(self.url_result(
+                        'http://www.dcndigital.ae/media/%s' % video['id'], 'DCNVideo'))
 
                 return self.playlist_result(entries, season_id, title)

From 608cc3b85cbe5acc045274c22e3fdaca08b90d45 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 27 Dec 2015 10:19:44 +0100
Subject: [PATCH 3486/6219] [kaltura] add referrer to m3u8 url

---
 youtube_dl/extractor/kaltura.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/kaltura.py b/youtube_dl/extractor/kaltura.py
index 353e960b0..4807c8110 100644
--- a/youtube_dl/extractor/kaltura.py
+++ b/youtube_dl/extractor/kaltura.py
@@ -160,10 +160,11 @@ class KalturaIE(InfoExtractor):
                 'width': int_or_none(f.get('width')),
                 'url': video_url,
             })
-
+        m3u8_url = info['dataUrl'].replace('format/url', 'format/applehttp')
+        if referrer:
+            m3u8_url += '?referrer=%s' % referrer
         m3u8_formats = self._extract_m3u8_formats(
-            info['dataUrl'].replace('format/url', 'format/applehttp'),
-            entry_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False)
+            m3u8_url, entry_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False)
         if m3u8_formats:
             formats.extend(m3u8_formats)
 

From 5b025168c73e00a45f1da857694835c52b73f0fa Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 27 Dec 2015 12:57:39 +0100
Subject: [PATCH 3487/6219] [livestream] improve extraction

- split long lines
- use m3u8 entry protocol for live streams
- extend _VALID_URL regex for livestream original
- extract livestream original live streams
---
 youtube_dl/extractor/livestream.py | 114 +++++++++++++++++++----------
 1 file changed, 76 insertions(+), 38 deletions(-)

diff --git a/youtube_dl/extractor/livestream.py b/youtube_dl/extractor/livestream.py
index b95c23c8c..11168fc4d 100644
--- a/youtube_dl/extractor/livestream.py
+++ b/youtube_dl/extractor/livestream.py
@@ -62,7 +62,8 @@ class LivestreamIE(InfoExtractor):
     _API_URL_TEMPLATE = 'http://livestream.com/api/accounts/%s/events/%s'
 
     def _parse_smil_formats(self, smil, smil_url, video_id, namespace=None, f4m_params=None, transform_rtmp_url=None):
-        base_ele = find_xpath_attr(smil, self._xpath_ns('.//meta', namespace), 'name', 'httpBase')
+        base_ele = find_xpath_attr(
+            smil, self._xpath_ns('.//meta', namespace), 'name', 'httpBase')
         base = base_ele.get('content') if base_ele else 'http://livestreamvod-f.akamaihd.net/'
 
         formats = []
@@ -96,7 +97,8 @@ class LivestreamIE(InfoExtractor):
             video_url = video_data.get(key)
             if video_url:
                 ext = determine_ext(video_url)
-                bitrate = int_or_none(self._search_regex(r'(\d+)\.%s' % ext, video_url, 'bitrate', default=None))
+                bitrate = int_or_none(self._search_regex(
+                    r'(\d+)\.%s' % ext, video_url, 'bitrate', default=None))
                 formats.append({
                     'url': video_url,
                     'format_id': format_id,
@@ -119,7 +121,8 @@ class LivestreamIE(InfoExtractor):
 
         f4m_url = video_data.get('f4m_url')
         if f4m_url:
-            f4m_formats = self._extract_f4m_formats(f4m_url, video_id, f4m_id='hds', fatal=False)
+            f4m_formats = self._extract_f4m_formats(
+                f4m_url, video_id, f4m_id='hds', fatal=False)
             if f4m_formats:
                 formats.extend(f4m_formats)
         self._sort_formats(formats)
@@ -157,10 +160,11 @@ class LivestreamIE(InfoExtractor):
             if smil_formats:
                 formats.extend(smil_formats)
 
+        entry_protocol = 'm3u8' if is_live else 'm3u8_native'
         m3u8_url = stream_info.get('m3u8_url')
         if m3u8_url:
             m3u8_formats = self._extract_m3u8_formats(
-                m3u8_url, broadcast_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False)
+                m3u8_url, broadcast_id, 'mp4', entry_protocol, m3u8_id='hls', fatal=False)
             if m3u8_formats:
                 formats.extend(m3u8_formats)
 
@@ -197,7 +201,8 @@ class LivestreamIE(InfoExtractor):
             else:
                 info_url = '{root}?&id={id}&newer=-1&type=video'.format(
                     root=feed_root_url, id=last_video)
-            videos_info = self._download_json(info_url, event_id, 'Downloading page {0}'.format(i))['data']
+            videos_info = self._download_json(
+                info_url, event_id, 'Downloading page {0}'.format(i))['data']
             videos_info = [v['data'] for v in videos_info if v['type'] == 'video']
             if not videos_info:
                 break
@@ -215,7 +220,8 @@ class LivestreamIE(InfoExtractor):
         account = mobj.group('account_id') or mobj.group('account_name')
         api_url = self._API_URL_TEMPLATE % (account, event)
         if video_id:
-            video_data = self._download_json(api_url + '/videos/%s' % video_id, video_id)
+            video_data = self._download_json(
+                api_url + '/videos/%s' % video_id, video_id)
             return self._extract_video_info(video_data)
         else:
             event_data = self._download_json(api_url, video_id)
@@ -226,8 +232,8 @@ class LivestreamIE(InfoExtractor):
 class LivestreamOriginalIE(InfoExtractor):
     IE_NAME = 'livestream:original'
     _VALID_URL = r'''(?x)https?://original\.livestream\.com/
-        (?P<user>[^/]+)/(?P<type>video|folder)
-        (?:\?.*?Id=|/)(?P<id>.*?)(&|$)
+        (?P<user>[^/\?#]+)(?:/(?P<type>video|folder)
+        (?:(?:\?.*?Id=|/)(?P<id>.*?)(&|$))?)?
         '''
     _TESTS = [{
         'url': 'http://original.livestream.com/dealbook/video?clipId=pla_8aa4a3f1-ba15-46a4-893b-902210e138fb',
@@ -244,52 +250,61 @@ class LivestreamOriginalIE(InfoExtractor):
             'id': 'a07bf706-d0e4-4e75-a747-b021d84f2fd3',
         },
         'playlist_mincount': 4,
+    }, {
+        # live stream
+        'url': 'http://www.livestream.com/znsbahamas',
+        'only_matching': True,
     }]
 
-    def _extract_video(self, user, video_id):
-        api_url = 'http://x{0}x.api.channel.livestream.com/2.0/clipdetails?extendedInfo=true&id={1}'.format(user, video_id)
-
+    def _extract_video_info(self, user, video_id):
+        api_url = 'http://x%sx.api.channel.livestream.com/2.0/clipdetails?extendedInfo=true&id=%s' % (user, video_id)
         info = self._download_xml(api_url, video_id)
-        # this url is used on mobile devices
-        stream_url = 'http://x{0}x.api.channel.livestream.com/3.0/getstream.json?id={1}'.format(user, video_id)
-        stream_info = self._download_json(stream_url, video_id)
-        is_live = stream_info.get('isLive')
+
         item = info.find('channel').find('item')
+        title = xpath_text(item, 'title')
         media_ns = {'media': 'http://search.yahoo.com/mrss'}
-        thumbnail_url = xpath_attr(item, xpath_with_ns('media:thumbnail', media_ns), 'url')
-        duration = float_or_none(xpath_attr(item, xpath_with_ns('media:content', media_ns), 'duration'))
+        thumbnail_url = xpath_attr(
+            item, xpath_with_ns('media:thumbnail', media_ns), 'url')
+        duration = float_or_none(xpath_attr(
+            item, xpath_with_ns('media:content', media_ns), 'duration'))
         ls_ns = {'ls': 'http://api.channel.livestream.com/2.0'}
-        view_count = int_or_none(xpath_text(item, xpath_with_ns('ls:viewsCount', ls_ns)))
+        view_count = int_or_none(xpath_text(
+            item, xpath_with_ns('ls:viewsCount', ls_ns)))
 
-        formats = [{
-            'url': stream_info['progressiveUrl'],
-            'format_id': 'http',
-        }]
+        return {
+            'id': video_id,
+            'title': title,
+            'thumbnail': thumbnail_url,
+            'duration': duration,
+            'view_count': view_count,
+        }
 
-        m3u8_url = stream_info.get('httpUrl')
+    def _extract_video_formats(self, video_data, video_id, entry_protocol):
+        formats = []
+
+        progressive_url = video_data.get('progressiveUrl')
+        if progressive_url:
+            formats.append({
+                'url': progressive_url,
+                'format_id': 'http',
+            })
+
+        m3u8_url = video_data.get('httpUrl')
         if m3u8_url:
             m3u8_formats = self._extract_m3u8_formats(
-                m3u8_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False)
+                m3u8_url, video_id, 'mp4', entry_protocol, m3u8_id='hls', fatal=False)
             if m3u8_formats:
                 formats.extend(m3u8_formats)
 
-        rtsp_url = stream_info.get('rtspUrl')
+        rtsp_url = video_data.get('rtspUrl')
         if rtsp_url:
             formats.append({
                 'url': rtsp_url,
                 'format_id': 'rtsp',
             })
-        self._sort_formats(formats)
 
-        return {
-            'id': video_id,
-            'title': self._live_title(xpath_text(item, 'title')) if is_live else xpath_text(item, 'title'),
-            'formats': formats,
-            'thumbnail': thumbnail_url,
-            'duration': duration,
-            'view_count': view_count,
-            'is_live': is_live,
-        }
+        self._sort_formats(formats)
+        return formats
 
     def _extract_folder(self, url, folder_id):
         webpage = self._download_webpage(url, folder_id)
@@ -308,13 +323,36 @@ class LivestreamOriginalIE(InfoExtractor):
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        id = mobj.group('id')
         user = mobj.group('user')
         url_type = mobj.group('type')
+        content_id = mobj.group('id')
         if url_type == 'folder':
-            return self._extract_folder(url, id)
+            return self._extract_folder(url, content_id)
         else:
-            return self._extract_video(user, id)
+            # this url is used on mobile devices
+            stream_url = 'http://x%sx.api.channel.livestream.com/3.0/getstream.json' % user
+            info = {}
+            if content_id:
+                stream_url += '?id=%s' % content_id
+                info = self._extract_video_info(user, content_id)
+            else:
+                content_id = user
+                webpage = self._download_webpage(url, content_id)
+                info = {
+                    'title': self._og_search_title(webpage),
+                    'description': self._og_search_description(webpage),
+                    'thumbnail': self._search_regex(r'channelLogo.src\s*=\s*"([^"]+)"', webpage, 'thumbnail', None),
+                }
+            video_data = self._download_json(stream_url, content_id)
+            is_live = video_data.get('isLive')
+            entry_protocol = 'm3u8' if is_live else 'm3u8_native'
+            info.update({
+                'id': content_id,
+                'title': self._live_title(info['title']) if is_live else info['title'],
+                'formats': self._extract_video_formats(video_data, content_id, entry_protocol),
+                'is_live': is_live,
+            })
+            return info
 
 
 # The server doesn't support HEAD request, the generic extractor can't detect

From 9f610f3a9e0e7536fe9c0ca167ea084aa2cd54df Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 27 Dec 2015 13:11:53 +0100
Subject: [PATCH 3488/6219] [sportdeutschland] Do not abort if meta info is
 missing

This fixes http://sportdeutschland.tv/badminton/yonex-copenhagen-masters-2015 . No testcase though since the event will be over by 2016.
---
 youtube_dl/extractor/sportdeutschland.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/sportdeutschland.py b/youtube_dl/extractor/sportdeutschland.py
index ebb75f059..a9927f6e2 100644
--- a/youtube_dl/extractor/sportdeutschland.py
+++ b/youtube_dl/extractor/sportdeutschland.py
@@ -70,10 +70,12 @@ class SportDeutschlandIE(InfoExtractor):
 
             smil_doc = self._download_xml(
                 smil_url, video_id, note='Downloading SMIL metadata')
-            base_url = smil_doc.find('./head/meta').attrib['base']
+            base_url_el = smil_doc.find('./head/meta')
+            if base_url_el:
+                base_url = base_url_el.attrib['base']
             formats.extend([{
                 'format_id': 'rmtp',
-                'url': base_url,
+                'url': base_url if base_url_el else n.attrib['src'],
                 'play_path': n.attrib['src'],
                 'ext': 'flv',
                 'preference': -100,

From 7b81316508d676393d89a99b42fc15e3dcfeab73 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 27 Dec 2015 14:58:57 +0100
Subject: [PATCH 3489/6219] [livestream] skip m3u8 manifest in progressive_urls

---
 youtube_dl/extractor/livestream.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/livestream.py b/youtube_dl/extractor/livestream.py
index 11168fc4d..9c8d826c4 100644
--- a/youtube_dl/extractor/livestream.py
+++ b/youtube_dl/extractor/livestream.py
@@ -97,6 +97,8 @@ class LivestreamIE(InfoExtractor):
             video_url = video_data.get(key)
             if video_url:
                 ext = determine_ext(video_url)
+                if ext == 'm3u8':
+                    continue
                 bitrate = int_or_none(self._search_regex(
                     r'(\d+)\.%s' % ext, video_url, 'bitrate', default=None))
                 formats.append({

From 8d29e47f543152bf91db0167a313e56ea2f132e3 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 27 Dec 2015 15:33:39 +0100
Subject: [PATCH 3490/6219] [common] simplify the use of _extract_m3u8_formats
 and _extract_f4m_formats

---
 youtube_dl/extractor/common.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 828f58f12..3d39781a4 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -885,7 +885,7 @@ class InfoExtractor(object):
             fatal=fatal)
 
         if manifest is False:
-            return manifest
+            return []
 
         formats = []
         manifest_version = '1.0'
@@ -956,7 +956,7 @@ class InfoExtractor(object):
             errnote=errnote or 'Failed to download m3u8 information',
             fatal=fatal)
         if res is False:
-            return res
+            return []
         m3u8_doc, urlh = res
         m3u8_url = urlh.geturl()
         last_info = None

From e73277c7e811a93a318a7707f82636c8093942b2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 27 Dec 2015 21:41:21 +0600
Subject: [PATCH 3491/6219] [abc7news] Remove redundant formats sorting

---
 youtube_dl/extractor/abc7news.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/abc7news.py b/youtube_dl/extractor/abc7news.py
index c04949c21..122dc9099 100644
--- a/youtube_dl/extractor/abc7news.py
+++ b/youtube_dl/extractor/abc7news.py
@@ -44,7 +44,6 @@ class Abc7NewsIE(InfoExtractor):
             'contentURL', webpage, 'm3u8 url', fatal=True)
 
         formats = self._extract_m3u8_formats(m3u8, display_id, 'mp4')
-        self._sort_formats(formats)
 
         title = self._og_search_title(webpage).strip()
         description = self._og_search_description(webpage).strip()

From 62bdc9feccbab0265243f8554bdbc7a84b5cca65 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 27 Dec 2015 21:41:36 +0600
Subject: [PATCH 3492/6219] [esri] Fix typo

---
 youtube_dl/extractor/esri.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/esri.py b/youtube_dl/extractor/esri.py
index bf5d2019f..d4205d7fb 100644
--- a/youtube_dl/extractor/esri.py
+++ b/youtube_dl/extractor/esri.py
@@ -61,7 +61,7 @@ class EsriVideoIE(InfoExtractor):
             webpage, 'duration', fatal=False))
 
         upload_date = unified_strdate(self._html_search_meta(
-            'last-modified', webpage, 'upload date', fatal=None))
+            'last-modified', webpage, 'upload date', fatal=False))
 
         return {
             'id': video_id,

From e0898585a188eb0538624b227eb00c90c2da26d8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 27 Dec 2015 21:41:55 +0600
Subject: [PATCH 3493/6219] [jwplatform] Fix typo

---
 youtube_dl/extractor/jwplatform.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/jwplatform.py b/youtube_dl/extractor/jwplatform.py
index cdc095a79..29eba77fc 100644
--- a/youtube_dl/extractor/jwplatform.py
+++ b/youtube_dl/extractor/jwplatform.py
@@ -44,7 +44,7 @@ class JWPlatformIE(InfoExtractor):
             source_url = self._proto_relative_url(source['file'])
             source_type = source.get('type') or ''
             if source_type == 'application/vnd.apple.mpegurl':
-                m3u8_formats = self._extract_m3u8_formats(source_url, video_id, 'mp4', 'm3u8_native', fatal=None)
+                m3u8_formats = self._extract_m3u8_formats(source_url, video_id, 'mp4', 'm3u8_native', fatal=False)
                 if m3u8_formats:
                     formats.extend(m3u8_formats)
             elif source_type.startswith('audio'):

From 30f51acbc8b0f4f3161102501527ed81d162d9e9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 27 Dec 2015 21:42:12 +0600
Subject: [PATCH 3494/6219] [rai] Fix typos

---
 youtube_dl/extractor/rai.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/rai.py b/youtube_dl/extractor/rai.py
index 14f1ccbb4..278b1d2bf 100644
--- a/youtube_dl/extractor/rai.py
+++ b/youtube_dl/extractor/rai.py
@@ -118,13 +118,13 @@ class RaiTVIE(InfoExtractor):
                 if ext == 'm3u8':
                     m3u8_formats = self._extract_m3u8_formats(
                         media_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls',
-                        fatal=None)
+                        fatal=False)
                     if m3u8_formats:
                         formats.extend(m3u8_formats)
                 elif ext == 'f4m':
                     f4m_formats = self._extract_f4m_formats(
                         media_url + '?hdcore=3.7.0&plugin=aasp-3.7.0.39.44',
-                        video_id, f4m_id='hds', fatal=None)
+                        video_id, f4m_id='hds', fatal=False)
                     if f4m_formats:
                         formats.extend(f4m_formats)
                 elif ext == 'stl':

From 0b0a17ae9d8c53b75f7f3cfa7e6fd236ba83e280 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 27 Dec 2015 21:42:26 +0600
Subject: [PATCH 3495/6219] [viki] Fix typo

---
 youtube_dl/extractor/viki.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index ca3f20a3d..9a1c377a4 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -279,7 +279,7 @@ class VikiIE(VikiBaseIE):
                 if format_id == 'm3u8':
                     m3u8_formats = self._extract_m3u8_formats(
                         format_dict['url'], video_id, 'mp4', 'm3u8_native',
-                        m3u8_id='m3u8-%s' % protocol, fatal=None)
+                        m3u8_id='m3u8-%s' % protocol, fatal=False)
                     if m3u8_formats:
                         formats.extend(m3u8_formats)
                 else:

From a95c26a06a88ab464638d16b42d1659a15e71c3c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 27 Dec 2015 21:43:14 +0600
Subject: [PATCH 3496/6219] [jwplatform] Carry long line

---
 youtube_dl/extractor/jwplatform.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/jwplatform.py b/youtube_dl/extractor/jwplatform.py
index 29eba77fc..a92adf2b3 100644
--- a/youtube_dl/extractor/jwplatform.py
+++ b/youtube_dl/extractor/jwplatform.py
@@ -44,7 +44,8 @@ class JWPlatformIE(InfoExtractor):
             source_url = self._proto_relative_url(source['file'])
             source_type = source.get('type') or ''
             if source_type == 'application/vnd.apple.mpegurl':
-                m3u8_formats = self._extract_m3u8_formats(source_url, video_id, 'mp4', 'm3u8_native', fatal=False)
+                m3u8_formats = self._extract_m3u8_formats(
+                    source_url, video_id, 'mp4', 'm3u8_native', fatal=False)
                 if m3u8_formats:
                     formats.extend(m3u8_formats)
             elif source_type.startswith('audio'):

From 82e02ea5fce6bbc595dbfefc8eafce21e2bb53cc Mon Sep 17 00:00:00 2001
From: gam2046 <gam2046@qq.com>
Date: Fri, 18 Dec 2015 14:18:56 +0800
Subject: [PATCH 3497/6219] Update iqiyi.py

Fix part of the address can not be resolved.
eg:http://www.iqiyi.com/w_19rt6o8t9p.html
---
 youtube_dl/extractor/iqiyi.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index c3731a110..f0e2e5104 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -16,7 +16,7 @@ class IqiyiIE(InfoExtractor):
     IE_NAME = 'iqiyi'
     IE_DESC = '爱奇艺'
 
-    _VALID_URL = r'http://(?:www\.)iqiyi.com/v_.+?\.html'
+    _VALID_URL = r'http://(?:www\.)iqiyi.com/._.+?\.html'
 
     _TESTS = [{
         'url': 'http://www.iqiyi.com/v_19rrojlavg.html',

From bee83e84f64248a47f2f0b79e8c2698bc89abf55 Mon Sep 17 00:00:00 2001
From: forDream <gam2046@users.noreply.github.com>
Date: Tue, 22 Dec 2015 09:28:12 +0800
Subject: [PATCH 3498/6219] [iqiyi]fix valid url

eg:
http://yule.iqiyi.com/zbj.html
---
 youtube_dl/extractor/iqiyi.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index f0e2e5104..38348e0e7 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -16,7 +16,7 @@ class IqiyiIE(InfoExtractor):
     IE_NAME = 'iqiyi'
     IE_DESC = '爱奇艺'
 
-    _VALID_URL = r'http://(?:www\.)iqiyi.com/._.+?\.html'
+    _VALID_URL = r'http://(?:[^.]+\.)?iqiyi\.com/.+\.html'
 
     _TESTS = [{
         'url': 'http://www.iqiyi.com/v_19rrojlavg.html',

From 59185202c6854b39ceb5fe6ea960ad1b308ceca8 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 28 Dec 2015 00:19:36 +0800
Subject: [PATCH 3499/6219] [iqiyi] Add tests for #7894

---
 youtube_dl/extractor/iqiyi.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index 38348e0e7..66a70a181 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -84,6 +84,15 @@ class IqiyiIE(InfoExtractor):
         'params': {
             'skip_download': True,
         },
+    }, {
+        'url': 'http://www.iqiyi.com/w_19rt6o8t9p.html',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.iqiyi.com/a_19rrhbc6kt.html',
+        'only_matching': True,
+    }, {
+        'url': 'http://yule.iqiyi.com/pcb.html',
+        'only_matching': True,
     }]
 
     _FORMATS_MAP = [

From c71d2e2087e6daf2196bb26ba7fa7e00c10f2d53 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 27 Dec 2015 17:27:20 +0100
Subject: [PATCH 3500/6219] [livestream] change test url

---
 youtube_dl/extractor/livestream.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/livestream.py b/youtube_dl/extractor/livestream.py
index 9c8d826c4..688eb2308 100644
--- a/youtube_dl/extractor/livestream.py
+++ b/youtube_dl/extractor/livestream.py
@@ -254,7 +254,7 @@ class LivestreamOriginalIE(InfoExtractor):
         'playlist_mincount': 4,
     }, {
         # live stream
-        'url': 'http://www.livestream.com/znsbahamas',
+        'url': 'http://original.livestream.com/znsbahamas',
         'only_matching': True,
     }]
 

From 97f18fac3a90da163bb6692038be560a5b93b525 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 27 Dec 2015 17:44:55 +0100
Subject: [PATCH 3501/6219] [vgtv] fix f4m downloading(fixes #7843)

---
 youtube_dl/extractor/vgtv.py | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/vgtv.py b/youtube_dl/extractor/vgtv.py
index 811ee197d..129668a99 100644
--- a/youtube_dl/extractor/vgtv.py
+++ b/youtube_dl/extractor/vgtv.py
@@ -86,9 +86,10 @@ class VGTVIE(XstreamIE):
         {
             # streamType: wasLive
             'url': 'http://www.vgtv.no/#!/live/113063/direkte-v75-fra-solvalla',
+            'md5': '458f4841239dab414343b50e5af8869c',
             'info_dict': {
                 'id': '113063',
-                'ext': 'mp4',
+                'ext': 'flv',
                 'title': 'V75 fra Solvalla 30.05.15',
                 'description': 'md5:b3743425765355855f88e096acc93231',
                 'thumbnail': 're:^https?://.*\.jpg',
@@ -97,10 +98,6 @@ class VGTVIE(XstreamIE):
                 'upload_date': '20150530',
                 'view_count': int,
             },
-            'params': {
-                # m3u8 download
-                'skip_download': True,
-            },
         },
         {
             'url': 'http://www.aftenposten.no/webtv/#!/video/21039/trailer-sweatshop-i-can-t-take-any-more',
@@ -160,12 +157,15 @@ class VGTVIE(XstreamIE):
                 formats.extend(m3u8_formats)
 
         hds_url = streams.get('hds')
-        # wasLive hds are always 404
-        if hds_url and stream_type != 'wasLive':
+        if hds_url:
+            hdcore_sign = 'hdcore=3.7.0'
             f4m_formats = self._extract_f4m_formats(
-                hds_url + '?hdcore=3.2.0&plugin=aasp-3.2.0.77.18', video_id, f4m_id='hds', fatal=False)
+                hds_url + '?%s' % hdcore_sign, video_id, f4m_id='hds', fatal=False)
             if f4m_formats:
-                formats.extend(f4m_formats)
+                for entry in f4m_formats:
+                    # URLs without the extra param induce an 404 error
+                    entry.update({'extra_param_to_segment_url': hdcore_sign})
+                    formats.append(entry)
 
         mp4_urls = streams.get('pseudostreaming') or []
         mp4_url = streams.get('mp4')

From c047270c02bef6c8303c985d1e1b8da5be394235 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 28 Dec 2015 01:09:18 +0800
Subject: [PATCH 3502/6219] [utils] Remove Content-encoding from headers after
 decompression

With cn_verification_proxy, our http_response() is called twice, one from
PerRequestProxyHandler.proxy_open() and another from normal
YoutubeDL.urlopen(). As a result, for proxies honoring Accept-Encoding, the
following bug occurs:

$ youtube-dl -vs --cn-verification-proxy https://secure.uku.im:993 "test:letv"
[debug] System config: []
[debug] User config: []
[debug] Command-line args: ['-vs', '--cn-verification-proxy', 'https://secure.uku.im:993', 'test:letv']
[debug] Encodings: locale UTF-8, fs utf-8, out UTF-8, pref UTF-8
[debug] youtube-dl version 2015.12.23
[debug] Git HEAD: 97f18fa
[debug] Python version 3.5.1 - Linux-4.3.3-1-ARCH-x86_64-with-arch-Arch-Linux
[debug] exe versions: ffmpeg 2.8.4, ffprobe 2.8.4, rtmpdump 2.4
[debug] Proxy map: {}
[TestURL] Test URL: http://www.letv.com/ptv/vplay/22005890.html
[Letv] 22005890: Downloading webpage
[Letv] 22005890: Downloading playJson data
ERROR: Unable to download JSON metadata: Not a gzipped file (b'{"') (caused by OSError('Not a gzipped file (b\'{"\')',)); please report this issue on https://yt-dl.org/bug . Make sure you are using the latest version; see  https://yt-dl.org/update  on how to update. Be sure to call youtube-dl with the --verbose flag and include its complete output.
  File "/home/yen/Executables/Multimedia/youtube-dl/youtube_dl/extractor/common.py", line 330, in _request_webpage
    return self._downloader.urlopen(url_or_request)
  File "/home/yen/Executables/Multimedia/youtube-dl/youtube_dl/YoutubeDL.py", line 1886, in urlopen
    return self._opener.open(req, timeout=self._socket_timeout)
  File "/usr/lib/python3.5/urllib/request.py", line 471, in open
    response = meth(req, response)
  File "/home/yen/Executables/Multimedia/youtube-dl/youtube_dl/utils.py", line 773, in http_response
    raise original_ioerror
  File "/home/yen/Executables/Multimedia/youtube-dl/youtube_dl/utils.py", line 761, in http_response
    uncompressed = io.BytesIO(gz.read())
  File "/usr/lib/python3.5/gzip.py", line 274, in read
    return self._buffer.read(size)
  File "/usr/lib/python3.5/gzip.py", line 461, in read
    if not self._read_gzip_header():
  File "/usr/lib/python3.5/gzip.py", line 409, in _read_gzip_header
    raise OSError('Not a gzipped file (%r)' % magic)
---
 youtube_dl/utils.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 1737ac5f6..0ed6c45c8 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -773,11 +773,13 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
                     raise original_ioerror
             resp = self.addinfourl_wrapper(uncompressed, old_resp.headers, old_resp.url, old_resp.code)
             resp.msg = old_resp.msg
+            del resp.headers['Content-encoding']
         # deflate
         if resp.headers.get('Content-encoding', '') == 'deflate':
             gz = io.BytesIO(self.deflate(resp.read()))
             resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
             resp.msg = old_resp.msg
+            del resp.headers['Content-encoding']
         # Percent-encode redirect URL of Location HTTP header to satisfy RFC 3986 (see
         # https://github.com/rg3/youtube-dl/issues/6457).
         if 300 <= resp.code < 400:

From bd3f9ecabeb5e189f16580c61c56d93148079363 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 28 Dec 2015 00:36:57 +0100
Subject: [PATCH 3503/6219] [tunein] add support for tunein topic,clip and
 program(fixes #7348)

---
 youtube_dl/extractor/__init__.py |   8 +-
 youtube_dl/extractor/tunein.py   | 187 +++++++++++++++++++++----------
 2 files changed, 136 insertions(+), 59 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 165835f63..c1dbf9f21 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -703,7 +703,13 @@ from .tube8 import Tube8IE
 from .tubitv import TubiTvIE
 from .tudou import TudouIE
 from .tumblr import TumblrIE
-from .tunein import TuneInIE
+from .tunein import (
+    TuneInClipIE,
+    TuneInStationIE,
+    TuneInProgramIE,
+    TuneInTopicIE,
+    TuneInShortenerIE,
+)
 from .turbo import TurboIE
 from .tutv import TutvIE
 from .tv2 import (
diff --git a/youtube_dl/extractor/tunein.py b/youtube_dl/extractor/tunein.py
index b6b1f2568..8322cc14d 100644
--- a/youtube_dl/extractor/tunein.py
+++ b/youtube_dl/extractor/tunein.py
@@ -2,74 +2,33 @@
 from __future__ import unicode_literals
 
 import json
-import re
 
 from .common import InfoExtractor
 from ..utils import ExtractorError
+from ..compat import compat_urlparse
 
 
-class TuneInIE(InfoExtractor):
-    _VALID_URL = r'''(?x)https?://(?:www\.)?
-    (?:
-        tunein\.com/
-        (?:
-            radio/.*?-s|
-            station/.*?StationId\=
-        )(?P<id>[0-9]+)
-        |tun\.in/(?P<redirect_id>[A-Za-z0-9]+)
-    )
-    '''
-    _API_URL_TEMPLATE = 'http://tunein.com/tuner/tune/?stationId={0:}&tuneType=Station'
-
-    _INFO_DICT = {
-        'id': '34682',
-        'title': 'Jazz 24 on 88.5 Jazz24 - KPLU-HD2',
-        'ext': 'aac',
-        'thumbnail': 're:^https?://.*\.png$',
-        'location': 'Tacoma, WA',
-    }
-    _TESTS = [
-        {
-            'url': 'http://tunein.com/radio/Jazz24-885-s34682/',
-            'info_dict': _INFO_DICT,
-            'params': {
-                'skip_download': True,  # live stream
-            },
-        },
-        {  # test redirection
-            'url': 'http://tun.in/ser7s',
-            'info_dict': _INFO_DICT,
-            'params': {
-                'skip_download': True,  # live stream
-            },
-        },
-    ]
+class TuneInBaseIE(InfoExtractor):
+    _API_BASE_URL = 'http://tunein.com/tuner/tune/'
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        redirect_id = mobj.group('redirect_id')
-        if redirect_id:
-            # The server doesn't support HEAD requests
-            urlh = self._request_webpage(
-                url, redirect_id, note='Downloading redirect page')
-            url = urlh.geturl()
-            self.to_screen('Following redirect: %s' % url)
-            mobj = re.match(self._VALID_URL, url)
-        station_id = mobj.group('id')
+        content_id = self._match_id(url)
 
-        station_info = self._download_json(
-            self._API_URL_TEMPLATE.format(station_id),
-            station_id, note='Downloading station JSON')
+        content_info = self._download_json(
+            self._API_BASE_URL + self._API_URL_QUERY % content_id,
+            content_id, note='Downloading JSON metadata')
 
-        title = station_info['Title']
-        thumbnail = station_info.get('Logo')
-        location = station_info.get('Location')
-        streams_url = station_info.get('StreamUrl')
+        title = content_info['Title']
+        thumbnail = content_info.get('Logo')
+        location = content_info.get('Location')
+        streams_url = content_info.get('StreamUrl')
         if not streams_url:
-            raise ExtractorError('No downloadable streams found',
-                                 expected=True)
+            raise ExtractorError('No downloadable streams found', expected=True)
+        if not streams_url.startswith('http://'):
+            streams_url = compat_urlparse.urljoin(url, streams_url)
+
         stream_data = self._download_webpage(
-            streams_url, station_id, note='Downloading stream data')
+            streams_url, content_id, note='Downloading stream data')
         streams = json.loads(self._search_regex(
             r'\((.*)\);', stream_data, 'stream info'))['Streams']
 
@@ -97,10 +56,122 @@ class TuneInIE(InfoExtractor):
         self._sort_formats(formats)
 
         return {
-            'id': station_id,
+            'id': content_id,
             'title': title,
             'formats': formats,
             'thumbnail': thumbnail,
             'location': location,
             'is_live': is_live,
         }
+
+
+class TuneInClipIE(TuneInBaseIE):
+    IE_NAME = 'tunein:clip'
+    _VALID_URL = r'https?://(?:www\.)?tunein\.com/station/.*?audioClipId\=(?P<id>\d+)'
+    _API_URL_QUERY = '?tuneType=AudioClip&audioclipId=%s'
+
+    _TESTS = [
+        {
+            'url': 'http://tunein.com/station/?stationId=246119&audioClipId=816',
+            'md5': '99f00d772db70efc804385c6b47f4e77',
+            'info_dict': {
+                'id': '816',
+                'title': '32m',
+                'ext': 'mp3',
+            },
+        },
+    ]
+
+
+class TuneInStationIE(TuneInBaseIE):
+    IE_NAME = 'tunein:station'
+    _VALID_URL = r'https?://(?:www\.)?tunein\.com/(?:radio/.*?-s|station/.*?StationId\=)(?P<id>\d+)'
+    _API_URL_QUERY = '?tuneType=Station&stationId=%s'
+
+    @classmethod
+    def suitable(cls, url):
+        return False if TuneInClipIE.suitable(url) else super(TuneInStationIE, cls).suitable(url)
+
+    _TESTS = [
+        {
+            'url': 'http://tunein.com/radio/Jazz24-885-s34682/',
+            'info_dict': {
+                'id': '34682',
+                'title': 'Jazz 24 on 88.5 Jazz24 - KPLU-HD2',
+                'ext': 'mp3',
+                'location': 'Tacoma, WA',
+            },
+            'params': {
+                'skip_download': True,  # live stream
+            },
+        },
+    ]
+
+
+class TuneInProgramIE(TuneInBaseIE):
+    IE_NAME = 'tunein:program'
+    _VALID_URL = r'https?://(?:www\.)?tunein\.com/(?:radio/.*?-p|program/.*?ProgramId\=)(?P<id>\d+)'
+    _API_URL_QUERY = '?tuneType=Program&programId=%s'
+
+    _TESTS = [
+        {
+            'url': 'http://tunein.com/radio/Jazz-24-p2506/',
+            'info_dict': {
+                'id': '2506',
+                'title': 'Jazz 24 on 91.3 WUKY-HD3',
+                'ext': 'mp3',
+                'location': 'Lexington, KY',
+            },
+            'params': {
+                'skip_download': True,  # live stream
+            },
+        },
+    ]
+
+
+class TuneInTopicIE(TuneInBaseIE):
+    IE_NAME = 'tunein:topic'
+    _VALID_URL = r'https?://(?:www\.)?tunein\.com/topic/.*?TopicId\=(?P<id>\d+)'
+    _API_URL_QUERY = '?tuneType=Topic&topicId=%s'
+
+    _TESTS = [
+        {
+            'url': 'http://tunein.com/topic/?TopicId=101830576',
+            'md5': 'c31a39e6f988d188252eae7af0ef09c9',
+            'info_dict': {
+                'id': '101830576',
+                'title': 'Votez pour moi du 29 octobre 2015 (29/10/15)',
+                'ext': 'mp3',
+                'location': 'Belgium',
+            },
+        },
+    ]
+
+
+class TuneInShortenerIE(InfoExtractor):
+    IE_NAME = 'tunein:shortener'
+    IE_DESC = False  # Do not list
+    _VALID_URL = r'https?://tun\.in/(?P<id>[A-Za-z0-9]+)'
+
+    _TEST = {
+        # test redirection
+        'url': 'http://tun.in/ser7s',
+        'info_dict': {
+            'id': '34682',
+            'title': 'Jazz 24 on 88.5 Jazz24 - KPLU-HD2',
+            'ext': 'mp3',
+            'location': 'Tacoma, WA',
+        },
+        'params': {
+            'skip_download': True,  # live stream
+        },
+    }
+
+    def _real_extract(self, url):
+        redirect_id = self._match_id(url)
+        # The server doesn't support HEAD requests
+        urlh = self._request_webpage(
+            url, redirect_id, note='Downloading redirect page')
+        url = urlh.geturl()
+        self.to_screen('Following redirect: %s' % url)
+        return self.url_result(url)

From bca9bea1c1a13d2f9ad3244a2b11979220c30484 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 28 Dec 2015 10:27:17 +0100
Subject: [PATCH 3504/6219] [dcn] make m3u8 formats extraction non fatal

---
 youtube_dl/extractor/dcn.py | 25 +++++++------------------
 1 file changed, 7 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/dcn.py b/youtube_dl/extractor/dcn.py
index 3857ba334..d9485cd86 100644
--- a/youtube_dl/extractor/dcn.py
+++ b/youtube_dl/extractor/dcn.py
@@ -54,10 +54,14 @@ class DCNBaseIE(InfoExtractor):
         }
 
     def _extract_video_formats(self, webpage, video_id, entry_protocol):
+        formats = []
         m3u8_url = self._html_search_regex(
-            r'file\s*:\s*"([^"]+)', webpage, 'm3u8 url')
-        formats = self._extract_m3u8_formats(
-            m3u8_url, video_id, 'mp4', entry_protocol, m3u8_id='hls')
+            r'file\s*:\s*"([^"]+)', webpage, 'm3u8 url', fatal=False)
+        if m3u8_url:
+            m3u8_formats = self._extract_m3u8_formats(
+                m3u8_url, video_id, 'mp4', entry_protocol, m3u8_id='hls', fatal=None)
+            if m3u8_formats:
+                formats.extend(m3u8_formats)
 
         rtsp_url = self._search_regex(
             r'<a[^>]+href="(rtsp://[^"]+)"', webpage, 'rtsp url', fatal=False)
@@ -117,21 +121,6 @@ class DCNVideoIE(DCNBaseIE):
 class DCNLiveIE(DCNBaseIE):
     IE_NAME = 'dcn:live'
     _VALID_URL = r'https?://(?:www\.)?dcndigital\.ae/(?:#/)?live/(?P<id>\d+)'
-    _TEST = {
-        'url': 'http://www.dcndigital.ae/#/live/6/dubai-tv',
-        'info_dict':
-        {
-            'id': '6',
-            'ext': 'mp4',
-            'title': 're:^Dubai Al Oula [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
-            'thumbnail': 're:^https?://.*\.png$',
-            'is_live': True,
-        },
-        'params': {
-            # m3u8 download
-            'skip_download': True,
-        },
-    }
 
     def _real_extract(self, url):
         channel_id = self._match_id(url)

From 04e24906be1dd0b6626b50de7e65609219a88116 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 28 Dec 2015 13:06:30 +0100
Subject: [PATCH 3505/6219] [cspan] Initialize 'video_type' to avoid
 'UnboundLocalError' exceptions (#8032)

---
 youtube_dl/extractor/cspan.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/cspan.py b/youtube_dl/extractor/cspan.py
index 7b685d157..154bddb43 100644
--- a/youtube_dl/extractor/cspan.py
+++ b/youtube_dl/extractor/cspan.py
@@ -58,6 +58,7 @@ class CSpanIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
+        video_type = None
         webpage = self._download_webpage(url, video_id)
         matches = re.search(r'data-(prog|clip)id=\'([0-9]+)\'', webpage)
         if matches:
@@ -70,6 +71,8 @@ class CSpanIE(InfoExtractor):
                 title = self._og_search_title(webpage)
                 surl = smuggle_url(senate_isvp_url, {'force_title': title})
                 return self.url_result(surl, 'SenateISVP', video_id, title)
+        if video_type is None or video_id is None:
+            raise ExtractorError('unable to find video id and type')
 
         def get_text_attr(d, attr):
             return d.get(attr, {}).get('#text')

From 6c6b8bd5ccef10aad73ee5a556830f9b00b93661 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 28 Dec 2015 13:48:10 +0100
Subject: [PATCH 3506/6219] [cspan] Fix extraction (fixes #8032)

---
 youtube_dl/extractor/cspan.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/cspan.py b/youtube_dl/extractor/cspan.py
index 154bddb43..b3ee67018 100644
--- a/youtube_dl/extractor/cspan.py
+++ b/youtube_dl/extractor/cspan.py
@@ -60,11 +60,13 @@ class CSpanIE(InfoExtractor):
         video_id = self._match_id(url)
         video_type = None
         webpage = self._download_webpage(url, video_id)
-        matches = re.search(r'data-(prog|clip)id=\'([0-9]+)\'', webpage)
-        if matches:
+        # We first look for clipid, because clipprog always appears before
+        patterns = [r'id=\'clip(%s)\'\s*value=\'([0-9]+)\'' % t for t in ('id', 'prog')]
+        results = list(filter(None, (re.search(p, webpage) for p in patterns)))
+        if results:
+            matches = results[0]
             video_type, video_id = matches.groups()
-            if video_type == 'prog':
-                video_type = 'program'
+            video_type = 'clip' if video_type == 'id' else 'program'
         else:
             senate_isvp_url = SenateISVPIE._search_iframe_url(webpage)
             if senate_isvp_url:

From 9f66931e160ec7c4cc3be8bf731da32b0bfa9043 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 28 Dec 2015 21:20:09 +0600
Subject: [PATCH 3507/6219] [facebook] Extract login error

---
 youtube_dl/extractor/facebook.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index 39c481068..1692ec1cb 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -100,6 +100,11 @@ class FacebookIE(InfoExtractor):
             login_results = self._download_webpage(request, None,
                                                    note='Logging in', errnote='unable to fetch login page')
             if re.search(r'<form(.*)name="login"(.*)</form>', login_results) is not None:
+                error = self._html_search_regex(
+                    r'(?s)<div[^>]+class=(["\']).*?login_error_box.*?\1[^>]*><div[^>]*>.*?</div><div[^>]*>(?P<error>.+?)</div>',
+                    login_results, 'login error', default=None, group='error')
+                if error:
+                    raise ExtractorError('Unable to login: %s' % error, expected=True)
                 self._downloader.report_warning('unable to log in: bad username/password, or exceded login rate limit (~3/min). Check credentials or wait.')
                 return
 

From c1f49e16843bb9bdfdf132a472eb0fc2c6b920e4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 28 Dec 2015 21:37:02 +0600
Subject: [PATCH 3508/6219] [facebook] Fix authentication

---
 youtube_dl/extractor/facebook.py | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index 1692ec1cb..5e43f2359 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -74,7 +74,7 @@ class FacebookIE(InfoExtractor):
             return
 
         login_page_req = sanitized_Request(self._LOGIN_URL)
-        login_page_req.add_header('Cookie', 'locale=en_US')
+        self._set_cookie('facebook.com', 'locale', 'en_US')
         login_page = self._download_webpage(login_page_req, None,
                                             note='Downloading login page',
                                             errnote='Unable to download login page')
@@ -108,10 +108,17 @@ class FacebookIE(InfoExtractor):
                 self._downloader.report_warning('unable to log in: bad username/password, or exceded login rate limit (~3/min). Check credentials or wait.')
                 return
 
+            fb_dtsg = self._search_regex(
+                r'name="fb_dtsg" value="(.+?)"', login_results, 'fb_dtsg', default=None)
+            h = self._search_regex(
+                r'name="h"\s+(?:\w+="[^"]+"\s+)*?value="([^"]+)"', login_results, 'h', default=None)
+
+            if not fb_dtsg or not h:
+                return
+
             check_form = {
-                'fb_dtsg': self._search_regex(r'name="fb_dtsg" value="(.+?)"', login_results, 'fb_dtsg'),
-                'h': self._search_regex(
-                    r'name="h"\s+(?:\w+="[^"]+"\s+)*?value="([^"]+)"', login_results, 'h'),
+                'fb_dtsg': fb_dtsg,
+                'h': h,
                 'name_action_selected': 'dont_save',
             }
             check_req = sanitized_Request(self._CHECKPOINT_URL, urlencode_postdata(check_form))

From a5c1d95500898541cd6b19b32963883db421c4b4 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 28 Dec 2015 17:49:59 +0100
Subject: [PATCH 3509/6219] [zdf] fix formats extraction

---
 youtube_dl/extractor/phoenix.py |   9 +-
 youtube_dl/extractor/zdf.py     | 253 ++++++++++++++++++++------------
 2 files changed, 161 insertions(+), 101 deletions(-)

diff --git a/youtube_dl/extractor/phoenix.py b/youtube_dl/extractor/phoenix.py
index 46cebc0d7..6ce2ec19d 100644
--- a/youtube_dl/extractor/phoenix.py
+++ b/youtube_dl/extractor/phoenix.py
@@ -1,10 +1,9 @@
 from __future__ import unicode_literals
 
-from .common import InfoExtractor
-from .zdf import extract_from_xml_url
+from .zdf import ZDFIE
 
 
-class PhoenixIE(InfoExtractor):
+class PhoenixIE(ZDFIE):
     _VALID_URL = r'''(?x)https?://(?:www\.)?phoenix\.de/content/
         (?:
             phoenix/die_sendungen/(?:[^/]+/)?
@@ -41,5 +40,5 @@ class PhoenixIE(InfoExtractor):
             r'<div class="phx_vod" id="phx_vod_([0-9]+)"',
             webpage, 'internal video ID')
 
-        api_url = 'http://www.phoenix.de/php/zdfplayer-v1.3/data/beitragsDetails.php?ak=web&id=%s' % internal_id
-        return extract_from_xml_url(self, video_id, api_url)
+        api_url = 'http://www.phoenix.de/php/mediaplayer/data/beitrags_details.php?ak=web&id=%s' % internal_id
+        return self.extract_from_xml_url(video_id, api_url)
diff --git a/youtube_dl/extractor/zdf.py b/youtube_dl/extractor/zdf.py
index 9a3331a69..92c12bac6 100644
--- a/youtube_dl/extractor/zdf.py
+++ b/youtube_dl/extractor/zdf.py
@@ -10,106 +10,16 @@ from ..utils import (
     unified_strdate,
     OnDemandPagedList,
     xpath_text,
+    determine_ext,
+    qualities,
+    float_or_none,
 )
 
 
-def extract_from_xml_url(ie, video_id, xml_url):
-    doc = ie._download_xml(
-        xml_url, video_id,
-        note='Downloading video info',
-        errnote='Failed to download video info')
-
-    title = doc.find('.//information/title').text
-    description = xpath_text(doc, './/information/detail', 'description')
-    duration = int_or_none(xpath_text(doc, './/details/lengthSec', 'duration'))
-    uploader = xpath_text(doc, './/details/originChannelTitle', 'uploader')
-    uploader_id = xpath_text(doc, './/details/originChannelId', 'uploader id')
-    upload_date = unified_strdate(xpath_text(doc, './/details/airtime', 'upload date'))
-
-    def xml_to_format(fnode):
-        video_url = fnode.find('url').text
-        is_available = 'http://www.metafilegenerator' not in video_url
-
-        format_id = fnode.attrib['basetype']
-        format_m = re.match(r'''(?x)
-            (?P<vcodec>[^_]+)_(?P<acodec>[^_]+)_(?P<container>[^_]+)_
-            (?P<proto>[^_]+)_(?P<index>[^_]+)_(?P<indexproto>[^_]+)
-        ''', format_id)
-
-        ext = format_m.group('container')
-        proto = format_m.group('proto').lower()
-
-        quality = xpath_text(fnode, './quality', 'quality')
-        abr = int_or_none(xpath_text(fnode, './audioBitrate', 'abr'), 1000)
-        vbr = int_or_none(xpath_text(fnode, './videoBitrate', 'vbr'), 1000)
-
-        width = int_or_none(xpath_text(fnode, './width', 'width'))
-        height = int_or_none(xpath_text(fnode, './height', 'height'))
-
-        filesize = int_or_none(xpath_text(fnode, './filesize', 'filesize'))
-
-        format_note = ''
-        if not format_note:
-            format_note = None
-
-        return {
-            'format_id': format_id + '-' + quality,
-            'url': video_url,
-            'ext': ext,
-            'acodec': format_m.group('acodec'),
-            'vcodec': format_m.group('vcodec'),
-            'abr': abr,
-            'vbr': vbr,
-            'width': width,
-            'height': height,
-            'filesize': filesize,
-            'format_note': format_note,
-            'protocol': proto,
-            '_available': is_available,
-        }
-
-    def xml_to_thumbnails(fnode):
-        thumbnails = []
-        for node in fnode:
-            thumbnail_url = node.text
-            if not thumbnail_url:
-                continue
-            thumbnail = {
-                'url': thumbnail_url,
-            }
-            if 'key' in node.attrib:
-                m = re.match('^([0-9]+)x([0-9]+)$', node.attrib['key'])
-                if m:
-                    thumbnail['width'] = int(m.group(1))
-                    thumbnail['height'] = int(m.group(2))
-            thumbnails.append(thumbnail)
-        return thumbnails
-
-    thumbnails = xml_to_thumbnails(doc.findall('.//teaserimages/teaserimage'))
-
-    format_nodes = doc.findall('.//formitaeten/formitaet')
-    formats = list(filter(
-        lambda f: f['_available'],
-        map(xml_to_format, format_nodes)))
-    ie._sort_formats(formats)
-
-    return {
-        'id': video_id,
-        'title': title,
-        'description': description,
-        'duration': duration,
-        'thumbnails': thumbnails,
-        'uploader': uploader,
-        'uploader_id': uploader_id,
-        'upload_date': upload_date,
-        'formats': formats,
-    }
-
-
 class ZDFIE(InfoExtractor):
     _VALID_URL = r'(?:zdf:|zdf:video:|https?://www\.zdf\.de/ZDFmediathek(?:#)?/(.*beitrag/(?:video/)?))(?P<id>[0-9]+)(?:/[^/?]+)?(?:\?.*)?'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.zdf.de/ZDFmediathek/beitrag/video/2037704/ZDFspezial---Ende-des-Machtpokers--?bc=sts;stt',
         'info_dict': {
             'id': '2037704',
@@ -122,12 +32,163 @@ class ZDFIE(InfoExtractor):
             'upload_date': '20131127',
         },
         'skip': 'Videos on ZDF.de are depublicised in short order',
-    }
+    }]
+
+    def _parse_smil_formats(self, smil, smil_url, video_id, namespace=None, f4m_params=None, transform_rtmp_url=None):
+        param_groups = {}
+        for param_group in smil.findall(self._xpath_ns('./head/paramGroup', namespace)):
+            group_id = param_group.attrib.get(self._xpath_ns('id', 'http://www.w3.org/XML/1998/namespace'))
+            params = {}
+            for param in param_group:
+                params[param.get('name')] = param.get('value')
+            param_groups[group_id] = params
+
+        formats = []
+        for video in smil.findall(self._xpath_ns('.//video', namespace)):
+            src = video.get('src')
+            if not src:
+                continue
+            bitrate = float_or_none(video.get('system-bitrate') or video.get('systemBitrate'), 1000)
+            group_id = video.get('paramGroup')
+            param_group = param_groups[group_id]
+            for proto in param_group['protocols'].split(','):
+                formats.append({
+                    'url': '%s://%s' % (proto, param_group['host']),
+                    'app': param_group['app'],
+                    'play_path': src,
+                    'ext': 'flv',
+                    'format_id': '%s-%d' % (proto, bitrate),
+                    'tbr': bitrate,
+                    'protocol': proto,
+                })
+        self._sort_formats(formats)
+        return formats
+
+    def extract_from_xml_url(self, video_id, xml_url):
+        doc = self._download_xml(
+            xml_url, video_id,
+            note='Downloading video info',
+            errnote='Failed to download video info')
+
+        title = doc.find('.//information/title').text
+        description = xpath_text(doc, './/information/detail', 'description')
+        duration = int_or_none(xpath_text(doc, './/details/lengthSec', 'duration'))
+        uploader = xpath_text(doc, './/details/originChannelTitle', 'uploader')
+        uploader_id = xpath_text(doc, './/details/originChannelId', 'uploader id')
+        upload_date = unified_strdate(xpath_text(doc, './/details/airtime', 'upload date'))
+
+        def xml_to_thumbnails(fnode):
+            thumbnails = []
+            for node in fnode:
+                thumbnail_url = node.text
+                if not thumbnail_url:
+                    continue
+                thumbnail = {
+                    'url': thumbnail_url,
+                }
+                if 'key' in node.attrib:
+                    m = re.match('^([0-9]+)x([0-9]+)$', node.attrib['key'])
+                    if m:
+                        thumbnail['width'] = int(m.group(1))
+                        thumbnail['height'] = int(m.group(2))
+                thumbnails.append(thumbnail)
+            return thumbnails
+
+        thumbnails = xml_to_thumbnails(doc.findall('.//teaserimages/teaserimage'))
+
+        format_nodes = doc.findall('.//formitaeten/formitaet')
+        quality = qualities(['veryhigh', 'high', 'med', 'low'])
+
+        def get_quality(elem):
+            return quality(xpath_text(elem, 'quality'))
+        format_nodes.sort(key=get_quality)
+        format_ids = []
+        formats = []
+        for fnode in format_nodes:
+            video_url = fnode.find('url').text
+            is_available = 'http://www.metafilegenerator' not in video_url
+            if not is_available:
+                continue
+            format_id = fnode.attrib['basetype']
+            quality = xpath_text(fnode, './quality', 'quality')
+            format_m = re.match(r'''(?x)
+                (?P<vcodec>[^_]+)_(?P<acodec>[^_]+)_(?P<container>[^_]+)_
+                (?P<proto>[^_]+)_(?P<index>[^_]+)_(?P<indexproto>[^_]+)
+            ''', format_id)
+
+            ext = determine_ext(video_url, None) or format_m.group('container')
+            if ext not in ('smil', 'f4m', 'm3u8'):
+                format_id = format_id + '-' + quality
+            if format_id in format_ids:
+                continue
+
+            if ext == 'meta':
+                continue
+            elif ext == 'smil':
+                smil_formats = self._extract_smil_formats(
+                    video_url, video_id, fatal=False)
+                if smil_formats:
+                    formats.extend(smil_formats)
+            elif ext == 'm3u8':
+                m3u8_formats = self._extract_m3u8_formats(
+                    video_url, video_id, 'mp4', m3u8_id='hls', fatal=False)
+                if m3u8_formats:
+                    formats.extend(m3u8_formats)
+            elif ext == 'f4m':
+                f4m_formats = self._extract_f4m_formats(
+                    video_url, video_id, f4m_id='hds', fatal=False)
+                if f4m_formats:
+                    formats.extend(f4m_formats)
+            else:
+                proto = format_m.group('proto').lower()
+
+                abr = int_or_none(xpath_text(fnode, './audioBitrate', 'abr'), 1000)
+                vbr = int_or_none(xpath_text(fnode, './videoBitrate', 'vbr'), 1000)
+
+                width = int_or_none(xpath_text(fnode, './width', 'width'))
+                height = int_or_none(xpath_text(fnode, './height', 'height'))
+
+                filesize = int_or_none(xpath_text(fnode, './filesize', 'filesize'))
+
+                format_note = ''
+                if not format_note:
+                    format_note = None
+
+                formats.append({
+                    'format_id': format_id,
+                    'url': video_url,
+                    'ext': ext,
+                    'acodec': format_m.group('acodec'),
+                    'vcodec': format_m.group('vcodec'),
+                    'abr': abr,
+                    'vbr': vbr,
+                    'width': width,
+                    'height': height,
+                    'filesize': filesize,
+                    'format_note': format_note,
+                    'protocol': proto,
+                    '_available': is_available,
+                })
+            format_ids.append(format_id)
+
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'duration': duration,
+            'thumbnails': thumbnails,
+            'uploader': uploader,
+            'uploader_id': uploader_id,
+            'upload_date': upload_date,
+            'formats': formats,
+        }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
         xml_url = 'http://www.zdf.de/ZDFmediathek/xmlservice/web/beitragsDetails?ak=web&id=%s' % video_id
-        return extract_from_xml_url(self, video_id, xml_url)
+        return self.extract_from_xml_url(video_id, xml_url)
 
 
 class ZDFChannelIE(InfoExtractor):

From d497a201cab0eb588e9b7f75494f5494c9663e67 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 5 Dec 2015 21:14:43 +0100
Subject: [PATCH 3510/6219] [common] use specific variable for protocol
 preference in _sort_formats

---
 youtube_dl/extractor/common.py | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 6ab2d68d6..76f5b8b05 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -18,7 +18,6 @@ from ..compat import (
     compat_http_client,
     compat_urllib_error,
     compat_urllib_parse,
-    compat_urllib_parse_urlparse,
     compat_urlparse,
     compat_str,
     compat_etree_fromstring,
@@ -42,6 +41,7 @@ from ..utils import (
     url_basename,
     xpath_text,
     xpath_with_ns,
+    determine_protocol,
 )
 
 
@@ -776,14 +776,12 @@ class InfoExtractor(object):
 
             preference = f.get('preference')
             if preference is None:
-                proto = f.get('protocol')
-                if proto is None:
-                    proto = compat_urllib_parse_urlparse(f.get('url', '')).scheme
-
-                preference = 0 if proto in ['http', 'https'] else -0.1
+                preference = 0
                 if f.get('ext') in ['f4f', 'f4m']:  # Not yet supported
                     preference -= 0.5
 
+            proto_preference = 0 if determine_protocol(f) in ['http', 'https'] else -0.1
+
             if f.get('vcodec') == 'none':  # audio only
                 if self._downloader.params.get('prefer_free_formats'):
                     ORDER = ['aac', 'mp3', 'm4a', 'webm', 'ogg', 'opus']
@@ -814,6 +812,7 @@ class InfoExtractor(object):
                 f.get('vbr') if f.get('vbr') is not None else -1,
                 f.get('height') if f.get('height') is not None else -1,
                 f.get('width') if f.get('width') is not None else -1,
+                proto_preference,
                 ext_preference,
                 f.get('abr') if f.get('abr') is not None else -1,
                 audio_ext_preference,

From 7e5edcfd333f05c610bd70c33f3d3fd0ff6a7cec Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 29 Dec 2015 00:58:24 +0600
Subject: [PATCH 3511/6219] Simplify formats accumulation for f4m/m3u8/smil
 formats

Now all _extract_*_formats routines return a list
---
 youtube_dl/extractor/amp.py         | 12 ++++-------
 youtube_dl/extractor/ard.py         | 12 ++++-------
 youtube_dl/extractor/atresplayer.py | 10 +++------
 youtube_dl/extractor/audimedia.py   |  8 ++------
 youtube_dl/extractor/bbc.py         |  6 ++----
 youtube_dl/extractor/bloomberg.py   | 12 ++++-------
 youtube_dl/extractor/br.py          | 12 ++++-------
 youtube_dl/extractor/brightcove.py  |  6 ++----
 youtube_dl/extractor/common.py      | 16 +++++----------
 youtube_dl/extractor/dailymotion.py | 12 ++++-------
 youtube_dl/extractor/dcn.py         |  6 ++----
 youtube_dl/extractor/eitb.py        | 12 ++++-------
 youtube_dl/extractor/funimation.py  |  6 ++----
 youtube_dl/extractor/funnyordie.py  |  6 ++----
 youtube_dl/extractor/globo.py       |  6 ++----
 youtube_dl/extractor/hotstar.py     |  4 +---
 youtube_dl/extractor/ign.py         |  8 ++------
 youtube_dl/extractor/jwplatform.py  |  6 ++----
 youtube_dl/extractor/kaltura.py     |  6 ++----
 youtube_dl/extractor/livestream.py  | 32 +++++++++--------------------
 youtube_dl/extractor/nba.py         |  8 ++------
 youtube_dl/extractor/ooyala.py      | 12 +++--------
 youtube_dl/extractor/rai.py         | 12 ++++-------
 youtube_dl/extractor/rutube.py      | 12 ++++-------
 youtube_dl/extractor/ruutu.py       | 12 ++++-------
 youtube_dl/extractor/tele13.py      |  4 +---
 youtube_dl/extractor/toggle.py      |  6 ++----
 youtube_dl/extractor/vgtv.py        |  6 ++----
 youtube_dl/extractor/viki.py        |  6 ++----
 youtube_dl/extractor/vimeo.py       |  6 ++----
 youtube_dl/extractor/wdr.py         | 14 ++++---------
 youtube_dl/extractor/zdf.py         | 18 ++++++----------
 32 files changed, 99 insertions(+), 215 deletions(-)

diff --git a/youtube_dl/extractor/amp.py b/youtube_dl/extractor/amp.py
index dcc3c97f1..1035d1c48 100644
--- a/youtube_dl/extractor/amp.py
+++ b/youtube_dl/extractor/amp.py
@@ -53,16 +53,12 @@ class AMPIE(InfoExtractor):
             media = media_data['@attributes']
             media_type = media['type']
             if media_type == 'video/f4m':
-                f4m_formats = self._extract_f4m_formats(
+                formats.extend(self._extract_f4m_formats(
                     media['url'] + '?hdcore=3.4.0&plugin=aasp-3.4.0.132.124',
-                    video_id, f4m_id='hds', fatal=False)
-                if f4m_formats:
-                    formats.extend(f4m_formats)
+                    video_id, f4m_id='hds', fatal=False))
             elif media_type == 'application/x-mpegURL':
-                m3u8_formats = self._extract_m3u8_formats(
-                    media['url'], video_id, 'mp4', m3u8_id='hls', fatal=False)
-                if m3u8_formats:
-                    formats.extend(m3u8_formats)
+                formats.extend(self._extract_m3u8_formats(
+                    media['url'], video_id, 'mp4', m3u8_id='hls', fatal=False))
             else:
                 formats.append({
                     'format_id': media_data['media-category']['@attributes']['label'],
diff --git a/youtube_dl/extractor/ard.py b/youtube_dl/extractor/ard.py
index 687eb9f82..9fb84911a 100644
--- a/youtube_dl/extractor/ard.py
+++ b/youtube_dl/extractor/ard.py
@@ -113,16 +113,12 @@ class ARDMediathekIE(InfoExtractor):
                     if quality != 'auto' and ext in ('f4m', 'm3u8'):
                         continue
                     if ext == 'f4m':
-                        f4m_formats = self._extract_f4m_formats(
+                        formats.extend(self._extract_f4m_formats(
                             stream_url + '?hdcore=3.1.1&plugin=aasp-3.1.1.69.124',
-                            video_id, preference=-1, f4m_id='hds', fatal=False)
-                        if f4m_formats:
-                            formats.extend(f4m_formats)
+                            video_id, preference=-1, f4m_id='hds', fatal=False))
                     elif ext == 'm3u8':
-                        m3u8_formats = self._extract_m3u8_formats(
-                            stream_url, video_id, 'mp4', preference=1, m3u8_id='hls', fatal=False)
-                        if m3u8_formats:
-                            formats.extend(m3u8_formats)
+                        formats.extend(self._extract_m3u8_formats(
+                            stream_url, video_id, 'mp4', preference=1, m3u8_id='hls', fatal=False))
                     else:
                         if server and server.startswith('rtmp'):
                             f = {
diff --git a/youtube_dl/extractor/atresplayer.py b/youtube_dl/extractor/atresplayer.py
index 7ac3044c7..3fb042cea 100644
--- a/youtube_dl/extractor/atresplayer.py
+++ b/youtube_dl/extractor/atresplayer.py
@@ -134,10 +134,8 @@ class AtresPlayerIE(InfoExtractor):
 
         m3u8_url = player.get('urlVideoHls')
         if m3u8_url:
-            m3u8_formats = self._extract_m3u8_formats(
-                m3u8_url, episode_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False)
-            if m3u8_formats:
-                formats.extend(m3u8_formats)
+            formats.extend(self._extract_m3u8_formats(
+                m3u8_url, episode_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
 
         timestamp = int_or_none(self._download_webpage(
             self._TIME_API_URL,
@@ -170,9 +168,7 @@ class AtresPlayerIE(InfoExtractor):
                 continue
             else:
                 f4m_url = video_url[:-9] + '/manifest.f4m'
-            f4m_formats = self._extract_f4m_formats(f4m_url, video_id, f4m_id='hds', fatal=False)
-            if f4m_formats:
-                formats.extend(f4m_formats)
+            formats.extend(self._extract_f4m_formats(f4m_url, video_id, f4m_id='hds', fatal=False))
         self._sort_formats(formats)
 
         path_data = player.get('pathData')
diff --git a/youtube_dl/extractor/audimedia.py b/youtube_dl/extractor/audimedia.py
index 4382a302b..9b037bb0c 100644
--- a/youtube_dl/extractor/audimedia.py
+++ b/youtube_dl/extractor/audimedia.py
@@ -45,15 +45,11 @@ class AudiMediaIE(InfoExtractor):
 
             stream_url_hls = json_data.get('stream_url_hls')
             if stream_url_hls:
-                m3u8_formats = self._extract_m3u8_formats(stream_url_hls, video_id, 'mp4', entry_protocol='m3u8_native', m3u8_id='hls', fatal=False)
-                if m3u8_formats:
-                    formats.extend(m3u8_formats)
+                formats.extend(self._extract_m3u8_formats(stream_url_hls, video_id, 'mp4', entry_protocol='m3u8_native', m3u8_id='hls', fatal=False))
 
             stream_url_hds = json_data.get('stream_url_hds')
             if stream_url_hds:
-                f4m_formats = self._extract_f4m_formats(json_data.get('stream_url_hds') + '?hdcore=3.4.0', video_id, -1, f4m_id='hds', fatal=False)
-                if f4m_formats:
-                    formats.extend(f4m_formats)
+                formats.extend(self._extract_f4m_formats(json_data.get('stream_url_hds') + '?hdcore=3.4.0', video_id, -1, f4m_id='hds', fatal=False))
 
             for video_version in json_data.get('video_versions'):
                 video_version_url = video_version.get('download_url') or video_version.get('stream_url')
diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 691aecc0d..923273fb2 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -223,11 +223,9 @@ class BBCCoUkIE(InfoExtractor):
             elif transfer_format == 'dash':
                 pass
             elif transfer_format == 'hls':
-                m3u8_formats = self._extract_m3u8_formats(
+                formats.extend(self._extract_m3u8_formats(
                     href, programme_id, ext='mp4', entry_protocol='m3u8_native',
-                    m3u8_id=supplier, fatal=False)
-                if m3u8_formats:
-                    formats.extend(m3u8_formats)
+                    m3u8_id=supplier, fatal=False))
             # Direct link
             else:
                 formats.append({
diff --git a/youtube_dl/extractor/bloomberg.py b/youtube_dl/extractor/bloomberg.py
index ebeef8f2a..13343bc25 100644
--- a/youtube_dl/extractor/bloomberg.py
+++ b/youtube_dl/extractor/bloomberg.py
@@ -41,15 +41,11 @@ class BloombergIE(InfoExtractor):
             if not stream_url:
                 continue
             if stream['muxing_format'] == 'TS':
-                m3u8_formats = self._extract_m3u8_formats(
-                    stream_url, video_id, 'mp4', m3u8_id='hls', fatal=False)
-                if m3u8_formats:
-                    formats.extend(m3u8_formats)
+                formats.extend(self._extract_m3u8_formats(
+                    stream_url, video_id, 'mp4', m3u8_id='hls', fatal=False))
             else:
-                f4m_formats = self._extract_f4m_formats(
-                    stream_url, video_id, f4m_id='hds', fatal=False)
-                if f4m_formats:
-                    formats.extend(f4m_formats)
+                formats.extend(self._extract_f4m_formats(
+                    stream_url, video_id, f4m_id='hds', fatal=False))
         self._sort_formats(formats)
 
         return {
diff --git a/youtube_dl/extractor/br.py b/youtube_dl/extractor/br.py
index e66854538..11cf49851 100644
--- a/youtube_dl/extractor/br.py
+++ b/youtube_dl/extractor/br.py
@@ -121,15 +121,11 @@ class BRIE(InfoExtractor):
             format_url = xpath_text(asset, ['downloadUrl', 'url'])
             asset_type = asset.get('type')
             if asset_type == 'HDS':
-                f4m_formats = self._extract_f4m_formats(
-                    format_url + '?hdcore=3.2.0', media_id, f4m_id='hds', fatal=False)
-                if f4m_formats:
-                    formats.extend(f4m_formats)
+                formats.extend(self._extract_f4m_formats(
+                    format_url + '?hdcore=3.2.0', media_id, f4m_id='hds', fatal=False))
             elif asset_type == 'HLS':
-                m3u8_formats = self._extract_m3u8_formats(
-                    format_url, media_id, 'mp4', 'm3u8_native', m3u8_id='hds', fatal=False)
-                if m3u8_formats:
-                    formats.extend(m3u8_formats)
+                formats.extend(self._extract_m3u8_formats(
+                    format_url, media_id, 'mp4', 'm3u8_native', m3u8_id='hds', fatal=False))
             else:
                 format_info = {
                     'ext': xpath_text(asset, 'mediaType'),
diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 03a4f446e..c947337f9 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -469,11 +469,9 @@ class BrightcoveNewIE(InfoExtractor):
             if source_type == 'application/x-mpegURL':
                 if not src:
                     continue
-                m3u8_formats = self._extract_m3u8_formats(
+                formats.extend(self._extract_m3u8_formats(
                     src, video_id, 'mp4', entry_protocol='m3u8_native',
-                    m3u8_id='hls', fatal=False)
-                if m3u8_formats:
-                    formats.extend(m3u8_formats)
+                    m3u8_id='hls', fatal=False))
             else:
                 streaming_src = source.get('streaming_src')
                 stream_name, app_name = source.get('stream_name'), source.get('app_name')
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 34a28c126..655207447 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -910,10 +910,8 @@ class InfoExtractor(object):
                 # may differ leading to inability to resolve the format by requested
                 # bitrate in f4m downloader
                 if determine_ext(manifest_url) == 'f4m':
-                    f4m_formats = self._extract_f4m_formats(
-                        manifest_url, video_id, preference, f4m_id, fatal=fatal)
-                    if f4m_formats:
-                        formats.extend(f4m_formats)
+                    formats.extend(self._extract_f4m_formats(
+                        manifest_url, video_id, preference, f4m_id, fatal=fatal))
                     continue
             tbr = int_or_none(media_el.attrib.get('bitrate'))
             formats.append({
@@ -1147,10 +1145,8 @@ class InfoExtractor(object):
             src_url = src if src.startswith('http') else compat_urlparse.urljoin(base, src)
 
             if proto == 'm3u8' or src_ext == 'm3u8':
-                m3u8_formats = self._extract_m3u8_formats(
-                    src_url, video_id, ext or 'mp4', m3u8_id='hls', fatal=False)
-                if m3u8_formats:
-                    formats.extend(m3u8_formats)
+                formats.extend(self._extract_m3u8_formats(
+                    src_url, video_id, ext or 'mp4', m3u8_id='hls', fatal=False))
                 continue
 
             if src_ext == 'f4m':
@@ -1162,9 +1158,7 @@ class InfoExtractor(object):
                     }
                 f4m_url += '&' if '?' in f4m_url else '?'
                 f4m_url += compat_urllib_parse.urlencode(f4m_params)
-                f4m_formats = self._extract_f4m_formats(f4m_url, video_id, f4m_id='hds', fatal=False)
-                if f4m_formats:
-                    formats.extend(f4m_formats)
+                formats.extend(self._extract_f4m_formats(f4m_url, video_id, f4m_id='hds', fatal=False))
                 continue
 
             if src_url.startswith('http') and self._is_valid_url(src, video_id):
diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index 0c5b6617f..439fd42e8 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -148,15 +148,11 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
                         continue
                     ext = determine_ext(media_url)
                     if type_ == 'application/x-mpegURL' or ext == 'm3u8':
-                        m3u8_formats = self._extract_m3u8_formats(
-                            media_url, video_id, 'mp4', m3u8_id='hls', fatal=False)
-                        if m3u8_formats:
-                            formats.extend(m3u8_formats)
+                        formats.extend(self._extract_m3u8_formats(
+                            media_url, video_id, 'mp4', m3u8_id='hls', fatal=False))
                     elif type_ == 'application/f4m' or ext == 'f4m':
-                        f4m_formats = self._extract_f4m_formats(
-                            media_url, video_id, preference=-1, f4m_id='hds', fatal=False)
-                        if f4m_formats:
-                            formats.extend(f4m_formats)
+                        formats.extend(self._extract_f4m_formats(
+                            media_url, video_id, preference=-1, f4m_id='hds', fatal=False))
                     else:
                         f = {
                             'url': media_url,
diff --git a/youtube_dl/extractor/dcn.py b/youtube_dl/extractor/dcn.py
index 0d140f12f..8f48571de 100644
--- a/youtube_dl/extractor/dcn.py
+++ b/youtube_dl/extractor/dcn.py
@@ -56,10 +56,8 @@ class DCNBaseIE(InfoExtractor):
         m3u8_url = self._html_search_regex(
             r'file\s*:\s*"([^"]+)', webpage, 'm3u8 url', fatal=False)
         if m3u8_url:
-            m3u8_formats = self._extract_m3u8_formats(
-                m3u8_url, video_id, 'mp4', entry_protocol, m3u8_id='hls', fatal=None)
-            if m3u8_formats:
-                formats.extend(m3u8_formats)
+            formats.extend(self._extract_m3u8_formats(
+                m3u8_url, video_id, 'mp4', entry_protocol, m3u8_id='hls', fatal=None))
 
         rtsp_url = self._search_regex(
             r'<a[^>]+href="(rtsp://[^"]+)"', webpage, 'rtsp url', fatal=False)
diff --git a/youtube_dl/extractor/eitb.py b/youtube_dl/extractor/eitb.py
index c83845fc2..713cb7b32 100644
--- a/youtube_dl/extractor/eitb.py
+++ b/youtube_dl/extractor/eitb.py
@@ -65,18 +65,14 @@ class EitbIE(InfoExtractor):
             if token_data:
                 token = token_data.get('token')
                 if token:
-                    m3u8_formats = self._extract_m3u8_formats(
-                        '%s?hdnts=%s' % (hls_url, token), video_id, m3u8_id='hls', fatal=False)
-                    if m3u8_formats:
-                        formats.extend(m3u8_formats)
+                    formats.extend(self._extract_m3u8_formats(
+                        '%s?hdnts=%s' % (hls_url, token), video_id, m3u8_id='hls', fatal=False))
 
         hds_url = media.get('HDS_SURL')
         if hds_url:
-            f4m_formats = self._extract_f4m_formats(
+            formats.extend(self._extract_f4m_formats(
                 '%s?hdcore=3.7.0' % hds_url.replace('euskalsvod', 'euskalvod'),
-                video_id, f4m_id='hds', fatal=False)
-            if f4m_formats:
-                formats.extend(f4m_formats)
+                video_id, f4m_id='hds', fatal=False))
 
         self._sort_formats(formats)
 
diff --git a/youtube_dl/extractor/funimation.py b/youtube_dl/extractor/funimation.py
index d1a95d87f..0f37ed786 100644
--- a/youtube_dl/extractor/funimation.py
+++ b/youtube_dl/extractor/funimation.py
@@ -151,11 +151,9 @@ class FunimationIE(InfoExtractor):
                         errors.append(format_url)
                         continue
                     if determine_ext(format_url) == 'm3u8':
-                        m3u8_formats = self._extract_m3u8_formats(
+                        formats.extend(self._extract_m3u8_formats(
                             format_url + auth_token, display_id, 'mp4', entry_protocol='m3u8_native',
-                            preference=preference, m3u8_id='%s-hls' % funimation_id, fatal=False)
-                        if m3u8_formats:
-                            formats.extend(m3u8_formats)
+                            preference=preference, m3u8_id='%s-hls' % funimation_id, fatal=False))
                     else:
                         tbr = int_or_none(self._search_regex(
                             r'-(\d+)[Kk]', format_url, 'tbr', default=None))
diff --git a/youtube_dl/extractor/funnyordie.py b/youtube_dl/extractor/funnyordie.py
index 7f21d7410..4c4a87e2a 100644
--- a/youtube_dl/extractor/funnyordie.py
+++ b/youtube_dl/extractor/funnyordie.py
@@ -51,10 +51,8 @@ class FunnyOrDieIE(InfoExtractor):
 
         formats = []
 
-        m3u8_formats = self._extract_m3u8_formats(
-            m3u8_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False)
-        if m3u8_formats:
-            formats.extend(m3u8_formats)
+        formats.extend(self._extract_m3u8_formats(
+            m3u8_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
 
         bitrates = [int(bitrate) for bitrate in re.findall(r'[,/]v(\d+)[,/]', m3u8_url)]
         bitrates.sort()
diff --git a/youtube_dl/extractor/globo.py b/youtube_dl/extractor/globo.py
index c65ef6bcf..b241c4868 100644
--- a/youtube_dl/extractor/globo.py
+++ b/youtube_dl/extractor/globo.py
@@ -369,11 +369,9 @@ class GloboIE(InfoExtractor):
             resource_url = resource['url']
             signed_url = '%s?h=%s&k=%s' % (resource_url, signed_hash, 'flash')
             if resource_id.endswith('m3u8') or resource_url.endswith('.m3u8'):
-                m3u8_formats = self._extract_m3u8_formats(
+                formats.extend(self._extract_m3u8_formats(
                     signed_url, resource_id, 'mp4', entry_protocol='m3u8_native',
-                    m3u8_id='hls', fatal=False)
-                if m3u8_formats:
-                    formats.extend(m3u8_formats)
+                    m3u8_id='hls', fatal=False))
             else:
                 formats.append({
                     'url': signed_url,
diff --git a/youtube_dl/extractor/hotstar.py b/youtube_dl/extractor/hotstar.py
index 05d27e75d..a7c3ce4ab 100644
--- a/youtube_dl/extractor/hotstar.py
+++ b/youtube_dl/extractor/hotstar.py
@@ -55,9 +55,7 @@ class HotStarIE(InfoExtractor):
                 format_url = format_data['src']
                 ext = determine_ext(format_url)
                 if ext == 'm3u8':
-                    m3u8_formats = self._extract_m3u8_formats(format_url, video_id, 'mp4', m3u8_id='hls', fatal=False)
-                    if m3u8_formats:
-                        formats.extend(m3u8_formats)
+                    formats.extend(self._extract_m3u8_formats(format_url, video_id, 'mp4', m3u8_id='hls', fatal=False))
                 elif ext == 'f4m':
                     # produce broken files
                     continue
diff --git a/youtube_dl/extractor/ign.py b/youtube_dl/extractor/ign.py
index a2e18c8a7..d1c1c210c 100644
--- a/youtube_dl/extractor/ign.py
+++ b/youtube_dl/extractor/ign.py
@@ -129,14 +129,10 @@ class IGNIE(InfoExtractor):
         formats = []
         m3u8_url = api_data['refs'].get('m3uUrl')
         if m3u8_url:
-            m3u8_formats = self._extract_m3u8_formats(m3u8_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False)
-            if m3u8_formats:
-                formats.extend(m3u8_formats)
+            formats.extend(self._extract_m3u8_formats(m3u8_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
         f4m_url = api_data['refs'].get('f4mUrl')
         if f4m_url:
-            f4m_formats = self._extract_f4m_formats(f4m_url, video_id, f4m_id='hds', fatal=False)
-            if f4m_formats:
-                formats.extend(f4m_formats)
+            formats.extend(self._extract_f4m_formats(f4m_url, video_id, f4m_id='hds', fatal=False))
         for asset in api_data['assets']:
             formats.append({
                 'url': asset['url'],
diff --git a/youtube_dl/extractor/jwplatform.py b/youtube_dl/extractor/jwplatform.py
index a92adf2b3..8e90d5986 100644
--- a/youtube_dl/extractor/jwplatform.py
+++ b/youtube_dl/extractor/jwplatform.py
@@ -44,10 +44,8 @@ class JWPlatformIE(InfoExtractor):
             source_url = self._proto_relative_url(source['file'])
             source_type = source.get('type') or ''
             if source_type == 'application/vnd.apple.mpegurl':
-                m3u8_formats = self._extract_m3u8_formats(
-                    source_url, video_id, 'mp4', 'm3u8_native', fatal=False)
-                if m3u8_formats:
-                    formats.extend(m3u8_formats)
+                formats.extend(self._extract_m3u8_formats(
+                    source_url, video_id, 'mp4', 'm3u8_native', fatal=False))
             elif source_type.startswith('audio'):
                 formats.append({
                     'url': source_url,
diff --git a/youtube_dl/extractor/kaltura.py b/youtube_dl/extractor/kaltura.py
index 4807c8110..ccbc39c66 100644
--- a/youtube_dl/extractor/kaltura.py
+++ b/youtube_dl/extractor/kaltura.py
@@ -163,10 +163,8 @@ class KalturaIE(InfoExtractor):
         m3u8_url = info['dataUrl'].replace('format/url', 'format/applehttp')
         if referrer:
             m3u8_url += '?referrer=%s' % referrer
-        m3u8_formats = self._extract_m3u8_formats(
-            m3u8_url, entry_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False)
-        if m3u8_formats:
-            formats.extend(m3u8_formats)
+        formats.extend(self._extract_m3u8_formats(
+            m3u8_url, entry_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
 
         self._check_formats(formats, entry_id)
         self._sort_formats(formats)
diff --git a/youtube_dl/extractor/livestream.py b/youtube_dl/extractor/livestream.py
index 688eb2308..38fb3d9e4 100644
--- a/youtube_dl/extractor/livestream.py
+++ b/youtube_dl/extractor/livestream.py
@@ -110,23 +110,17 @@ class LivestreamIE(InfoExtractor):
 
         smil_url = video_data.get('smil_url')
         if smil_url:
-            smil_formats = self._extract_smil_formats(smil_url, video_id)
-            if smil_formats:
-                formats.extend(smil_formats)
+            formats.extend(self._extract_smil_formats(smil_url, video_id))
 
         m3u8_url = video_data.get('m3u8_url')
         if m3u8_url:
-            m3u8_formats = self._extract_m3u8_formats(
-                m3u8_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False)
-            if m3u8_formats:
-                formats.extend(m3u8_formats)
+            formats.extend(self._extract_m3u8_formats(
+                m3u8_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
 
         f4m_url = video_data.get('f4m_url')
         if f4m_url:
-            f4m_formats = self._extract_f4m_formats(
-                f4m_url, video_id, f4m_id='hds', fatal=False)
-            if f4m_formats:
-                formats.extend(f4m_formats)
+            formats.extend(self._extract_f4m_formats(
+                f4m_url, video_id, f4m_id='hds', fatal=False))
         self._sort_formats(formats)
 
         comments = [{
@@ -158,17 +152,13 @@ class LivestreamIE(InfoExtractor):
         formats = []
         smil_url = stream_info.get('play_url')
         if smil_url:
-            smil_formats = self._extract_smil_formats(smil_url, broadcast_id)
-            if smil_formats:
-                formats.extend(smil_formats)
+            formats.extend(self._extract_smil_formats(smil_url, broadcast_id))
 
         entry_protocol = 'm3u8' if is_live else 'm3u8_native'
         m3u8_url = stream_info.get('m3u8_url')
         if m3u8_url:
-            m3u8_formats = self._extract_m3u8_formats(
-                m3u8_url, broadcast_id, 'mp4', entry_protocol, m3u8_id='hls', fatal=False)
-            if m3u8_formats:
-                formats.extend(m3u8_formats)
+            formats.extend(self._extract_m3u8_formats(
+                m3u8_url, broadcast_id, 'mp4', entry_protocol, m3u8_id='hls', fatal=False))
 
         rtsp_url = stream_info.get('rtsp_url')
         if rtsp_url:
@@ -293,10 +283,8 @@ class LivestreamOriginalIE(InfoExtractor):
 
         m3u8_url = video_data.get('httpUrl')
         if m3u8_url:
-            m3u8_formats = self._extract_m3u8_formats(
-                m3u8_url, video_id, 'mp4', entry_protocol, m3u8_id='hls', fatal=False)
-            if m3u8_formats:
-                formats.extend(m3u8_formats)
+            formats.extend(self._extract_m3u8_formats(
+                m3u8_url, video_id, 'mp4', entry_protocol, m3u8_id='hls', fatal=False))
 
         rtsp_url = video_data.get('rtspUrl')
         if rtsp_url:
diff --git a/youtube_dl/extractor/nba.py b/youtube_dl/extractor/nba.py
index 7c6b7841d..9d26030d3 100644
--- a/youtube_dl/extractor/nba.py
+++ b/youtube_dl/extractor/nba.py
@@ -68,13 +68,9 @@ class NBAIE(InfoExtractor):
             if video_url.startswith('/'):
                 continue
             if video_url.endswith('.m3u8'):
-                m3u8_formats = self._extract_m3u8_formats(video_url, video_id, m3u8_id='hls', fatal=False)
-                if m3u8_formats:
-                    formats.extend(m3u8_formats)
+                formats.extend(self._extract_m3u8_formats(video_url, video_id, m3u8_id='hls', fatal=False))
             elif video_url.endswith('.f4m'):
-                f4m_formats = self._extract_f4m_formats(video_url + '?hdcore=3.4.1.1', video_id, f4m_id='hds', fatal=False)
-                if f4m_formats:
-                    formats.extend(f4m_formats)
+                formats.extend(self._extract_f4m_formats(video_url + '?hdcore=3.4.1.1', video_id, f4m_id='hds', fatal=False))
             else:
                 key = video_file.attrib.get('bitrate')
                 format_info = {
diff --git a/youtube_dl/extractor/ooyala.py b/youtube_dl/extractor/ooyala.py
index 8603fd692..3960d522e 100644
--- a/youtube_dl/extractor/ooyala.py
+++ b/youtube_dl/extractor/ooyala.py
@@ -44,17 +44,11 @@ class OoyalaBaseIE(InfoExtractor):
                     urls.append(url)
                     delivery_type = stream['delivery_type']
                     if delivery_type == 'hls' or '.m3u8' in url:
-                        m3u8_formats = self._extract_m3u8_formats(url, embed_code, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False)
-                        if m3u8_formats:
-                            formats.extend(m3u8_formats)
+                        formats.extend(self._extract_m3u8_formats(url, embed_code, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
                     elif delivery_type == 'hds' or '.f4m' in url:
-                        f4m_formats = self._extract_f4m_formats(url, embed_code, f4m_id='hds', fatal=False)
-                        if f4m_formats:
-                            formats.extend(f4m_formats)
+                        formats.extend(self._extract_f4m_formats(url, embed_code, f4m_id='hds', fatal=False))
                     elif '.smil' in url:
-                        smil_formats = self._extract_smil_formats(url, embed_code, fatal=False)
-                        if smil_formats:
-                            formats.extend(smil_formats)
+                        formats.extend(self._extract_smil_formats(url, embed_code, fatal=False))
                     else:
                         formats.append({
                             'url': url,
diff --git a/youtube_dl/extractor/rai.py b/youtube_dl/extractor/rai.py
index 278b1d2bf..f2679591b 100644
--- a/youtube_dl/extractor/rai.py
+++ b/youtube_dl/extractor/rai.py
@@ -116,17 +116,13 @@ class RaiTVIE(InfoExtractor):
                 ext = determine_ext(media_url)
                 content_type = xpath_text(element, 'content-type')
                 if ext == 'm3u8':
-                    m3u8_formats = self._extract_m3u8_formats(
+                    formats.extend(self._extract_m3u8_formats(
                         media_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls',
-                        fatal=False)
-                    if m3u8_formats:
-                        formats.extend(m3u8_formats)
+                        fatal=False))
                 elif ext == 'f4m':
-                    f4m_formats = self._extract_f4m_formats(
+                    formats.extend(self._extract_f4m_formats(
                         media_url + '?hdcore=3.7.0&plugin=aasp-3.7.0.39.44',
-                        video_id, f4m_id='hds', fatal=False)
-                    if f4m_formats:
-                        formats.extend(f4m_formats)
+                        video_id, f4m_id='hds', fatal=False))
                 elif ext == 'stl':
                     has_subtitle = True
                 elif content_type.startswith('video/'):
diff --git a/youtube_dl/extractor/rutube.py b/youtube_dl/extractor/rutube.py
index 9db62adb1..c5c47d01e 100644
--- a/youtube_dl/extractor/rutube.py
+++ b/youtube_dl/extractor/rutube.py
@@ -58,15 +58,11 @@ class RutubeIE(InfoExtractor):
         for format_id, format_url in options['video_balancer'].items():
             ext = determine_ext(format_url)
             if ext == 'm3u8':
-                m3u8_formats = self._extract_m3u8_formats(
-                    format_url, video_id, 'mp4', m3u8_id=format_id, fatal=False)
-                if m3u8_formats:
-                    formats.extend(m3u8_formats)
+                formats.extend(self._extract_m3u8_formats(
+                    format_url, video_id, 'mp4', m3u8_id=format_id, fatal=False))
             elif ext == 'f4m':
-                f4m_formats = self._extract_f4m_formats(
-                    format_url, video_id, f4m_id=format_id, fatal=False)
-                if f4m_formats:
-                    formats.extend(f4m_formats)
+                formats.extend(self._extract_f4m_formats(
+                    format_url, video_id, f4m_id=format_id, fatal=False))
             else:
                 formats.append({
                     'url': format_url,
diff --git a/youtube_dl/extractor/ruutu.py b/youtube_dl/extractor/ruutu.py
index e417bf661..41fddc375 100644
--- a/youtube_dl/extractor/ruutu.py
+++ b/youtube_dl/extractor/ruutu.py
@@ -63,15 +63,11 @@ class RuutuIE(InfoExtractor):
                     processed_urls.append(video_url)
                     ext = determine_ext(video_url)
                     if ext == 'm3u8':
-                        m3u8_formats = self._extract_m3u8_formats(
-                            video_url, video_id, 'mp4', m3u8_id='hls', fatal=False)
-                        if m3u8_formats:
-                            formats.extend(m3u8_formats)
+                        formats.extend(self._extract_m3u8_formats(
+                            video_url, video_id, 'mp4', m3u8_id='hls', fatal=False))
                     elif ext == 'f4m':
-                        f4m_formats = self._extract_f4m_formats(
-                            video_url, video_id, f4m_id='hds', fatal=False)
-                        if f4m_formats:
-                            formats.extend(f4m_formats)
+                        formats.extend(self._extract_f4m_formats(
+                            video_url, video_id, f4m_id='hds', fatal=False))
                     else:
                         proto = compat_urllib_parse_urlparse(video_url).scheme
                         if not child.tag.startswith('HTTP') and proto != 'rtmp':
diff --git a/youtube_dl/extractor/tele13.py b/youtube_dl/extractor/tele13.py
index a363b4d40..fe11c20df 100644
--- a/youtube_dl/extractor/tele13.py
+++ b/youtube_dl/extractor/tele13.py
@@ -57,9 +57,7 @@ class Tele13IE(InfoExtractor):
             if format_url and format_url not in urls:
                 ext = determine_ext(format_url)
                 if ext == 'm3u8':
-                    m3u8_formats = self._extract_m3u8_formats(format_url, display_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False)
-                    if m3u8_formats:
-                        formats.extend(m3u8_formats)
+                    formats.extend(self._extract_m3u8_formats(format_url, display_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
                 elif YoutubeIE.suitable(format_url):
                     return self.url_result(format_url, 'Youtube')
                 else:
diff --git a/youtube_dl/extractor/toggle.py b/youtube_dl/extractor/toggle.py
index a47239952..c54b876d3 100644
--- a/youtube_dl/extractor/toggle.py
+++ b/youtube_dl/extractor/toggle.py
@@ -132,13 +132,11 @@ class ToggleIE(InfoExtractor):
             vid_format = vid_format.replace(' ', '')
             # if geo-restricted, m3u8 is inaccessible, but mp4 is okay
             if ext == 'm3u8':
-                m3u8_formats = self._extract_m3u8_formats(
+                formats.extend(self._extract_m3u8_formats(
                     video_url, video_id, ext='mp4', m3u8_id=vid_format,
                     note='Downloading %s m3u8 information' % vid_format,
                     errnote='Failed to download %s m3u8 information' % vid_format,
-                    fatal=False)
-                if m3u8_formats:
-                    formats.extend(m3u8_formats)
+                    fatal=False))
             elif ext in ('mp4', 'wvm'):
                 # wvm are drm-protected files
                 formats.append({
diff --git a/youtube_dl/extractor/vgtv.py b/youtube_dl/extractor/vgtv.py
index 129668a99..86ba70ed9 100644
--- a/youtube_dl/extractor/vgtv.py
+++ b/youtube_dl/extractor/vgtv.py
@@ -151,10 +151,8 @@ class VGTVIE(XstreamIE):
 
         hls_url = streams.get('hls')
         if hls_url:
-            m3u8_formats = self._extract_m3u8_formats(
-                hls_url, video_id, 'mp4', m3u8_id='hls', fatal=False)
-            if m3u8_formats:
-                formats.extend(m3u8_formats)
+            formats.extend(self._extract_m3u8_formats(
+                hls_url, video_id, 'mp4', m3u8_id='hls', fatal=False))
 
         hds_url = streams.get('hds')
         if hds_url:
diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index 9a1c377a4..433fc9914 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -277,11 +277,9 @@ class VikiIE(VikiBaseIE):
                 r'^(\d+)[pP]$', format_id, 'height', default=None))
             for protocol, format_dict in stream_dict.items():
                 if format_id == 'm3u8':
-                    m3u8_formats = self._extract_m3u8_formats(
+                    formats.extend(self._extract_m3u8_formats(
                         format_dict['url'], video_id, 'mp4', 'm3u8_native',
-                        m3u8_id='m3u8-%s' % protocol, fatal=False)
-                    if m3u8_formats:
-                        formats.extend(m3u8_formats)
+                        m3u8_id='m3u8-%s' % protocol, fatal=False))
                 else:
                     formats.append({
                         'url': format_dict['url'],
diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index ce08e6955..7af699982 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -437,10 +437,8 @@ class VimeoIE(VimeoBaseInfoExtractor):
             })
         m3u8_url = config_files.get('hls', {}).get('url')
         if m3u8_url:
-            m3u8_formats = self._extract_m3u8_formats(
-                m3u8_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False)
-            if m3u8_formats:
-                formats.extend(m3u8_formats)
+            formats.extend(self._extract_m3u8_formats(
+                m3u8_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
         # Bitrates are completely broken. Single m3u8 may contain entries in kbps and bps
         # at the same time without actual units specified. This lead to wrong sorting.
         self._sort_formats(formats, field_preference=('preference', 'height', 'width', 'fps', 'format_id'))
diff --git a/youtube_dl/extractor/wdr.py b/youtube_dl/extractor/wdr.py
index ef096cbd2..e8511398c 100644
--- a/youtube_dl/extractor/wdr.py
+++ b/youtube_dl/extractor/wdr.py
@@ -157,16 +157,12 @@ class WDRIE(InfoExtractor):
         preference = qualities(['S', 'M', 'L', 'XL'])
 
         if video_url.endswith('.f4m'):
-            f4m_formats = self._extract_f4m_formats(video_url + '?hdcore=3.2.0&plugin=aasp-3.2.0.77.18', page_id, f4m_id='hds', fatal=False)
-            if f4m_formats:
-                formats.extend(f4m_formats)
+            formats.extend(self._extract_f4m_formats(video_url + '?hdcore=3.2.0&plugin=aasp-3.2.0.77.18', page_id, f4m_id='hds', fatal=False))
         elif video_url.endswith('.smil'):
-            smil_formats = self._extract_smil_formats(video_url, page_id, False, {
+            formats.extend(self._extract_smil_formats(video_url, page_id, False, {
                 'hdcore': '3.3.0',
                 'plugin': 'aasp-3.3.0.99.43',
-            })
-            if smil_formats:
-                formats.extend(smil_formats)
+            }))
         else:
             formats.append({
                 'url': video_url,
@@ -177,9 +173,7 @@ class WDRIE(InfoExtractor):
 
         m3u8_url = self._search_regex(r'rel="adaptiv"[^>]+href="([^"]+)"', webpage, 'm3u8 url', default=None)
         if m3u8_url:
-            m3u8_formats = self._extract_m3u8_formats(m3u8_url, page_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False)
-            if m3u8_formats:
-                formats.extend(m3u8_formats)
+            formats.extend(self._extract_m3u8_formats(m3u8_url, page_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
 
         direct_urls = re.findall(r'rel="web(S|M|L|XL)"[^>]+href="([^"]+)"', webpage)
         if direct_urls:
diff --git a/youtube_dl/extractor/zdf.py b/youtube_dl/extractor/zdf.py
index 92c12bac6..2a1f2f6d1 100644
--- a/youtube_dl/extractor/zdf.py
+++ b/youtube_dl/extractor/zdf.py
@@ -125,20 +125,14 @@ class ZDFIE(InfoExtractor):
             if ext == 'meta':
                 continue
             elif ext == 'smil':
-                smil_formats = self._extract_smil_formats(
-                    video_url, video_id, fatal=False)
-                if smil_formats:
-                    formats.extend(smil_formats)
+                formats.extend(self._extract_smil_formats(
+                    video_url, video_id, fatal=False))
             elif ext == 'm3u8':
-                m3u8_formats = self._extract_m3u8_formats(
-                    video_url, video_id, 'mp4', m3u8_id='hls', fatal=False)
-                if m3u8_formats:
-                    formats.extend(m3u8_formats)
+                formats.extend(self._extract_m3u8_formats(
+                    video_url, video_id, 'mp4', m3u8_id='hls', fatal=False))
             elif ext == 'f4m':
-                f4m_formats = self._extract_f4m_formats(
-                    video_url, video_id, f4m_id='hds', fatal=False)
-                if f4m_formats:
-                    formats.extend(f4m_formats)
+                formats.extend(self._extract_f4m_formats(
+                    video_url, video_id, f4m_id='hds', fatal=False))
             else:
                 proto = format_m.group('proto').lower()
 

From 7447661e9b433d8871d0a898c9af271d5a49ec82 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 28 Dec 2015 19:23:54 +0100
Subject: [PATCH 3512/6219] [franceculture] Fix test

---
 youtube_dl/extractor/franceculture.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/franceculture.py b/youtube_dl/extractor/franceculture.py
index 1e83a4e7e..2ac6157cd 100644
--- a/youtube_dl/extractor/franceculture.py
+++ b/youtube_dl/extractor/franceculture.py
@@ -22,8 +22,8 @@ class FranceCultureIE(InfoExtractor):
             'alt_title': 'Carnet nomade | 13-14',
             'vcodec': 'none',
             'upload_date': '20140301',
-            'thumbnail': r're:^http://www\.franceculture\.fr/.*/images/player/Carnet-nomade\.jpg$',
-            'description': 'startswith:Avec :Jean-Baptiste Péretié pour son documentaire sur Arte "La revanche des « geeks », une enquête menée aux Etats',
+            'thumbnail': r're:^http://static\.franceculture\.fr/.*/images/player/Carnet-nomade\.jpg$',
+            'description': 'startswith:Avec :Jean-Baptiste Péretié pour son documentaire sur Arte "La revanche',
             'timestamp': 1393700400,
         }
     }

From ecf17d165317078d0e31191e40903a74fc3da3e8 Mon Sep 17 00:00:00 2001
From: flatgreen <flatgreen@gmail.com>
Date: Sun, 27 Dec 2015 15:30:45 +0100
Subject: [PATCH 3513/6219] [franceculture] Add extractor for '/emission-*'
 urls (closes #3777, closes #8022)

---
 youtube_dl/extractor/__init__.py      |  5 +++-
 youtube_dl/extractor/franceculture.py | 40 +++++++++++++++++++++++++--
 2 files changed, 42 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 971047ad4..68c9c1288 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -203,7 +203,10 @@ from .fourtube import FourTubeIE
 from .foxgay import FoxgayIE
 from .foxnews import FoxNewsIE
 from .foxsports import FoxSportsIE
-from .franceculture import FranceCultureIE
+from .franceculture import (
+    FranceCultureIE,
+    FranceCultureEmissionIE,
+)
 from .franceinter import FranceInterIE
 from .francetv import (
     PluzzIE,
diff --git a/youtube_dl/extractor/franceculture.py b/youtube_dl/extractor/franceculture.py
index 2ac6157cd..e2ca96283 100644
--- a/youtube_dl/extractor/franceculture.py
+++ b/youtube_dl/extractor/franceculture.py
@@ -8,6 +8,7 @@ from ..compat import (
 from ..utils import (
     determine_ext,
     int_or_none,
+    ExtractorError,
 )
 
 
@@ -28,8 +29,7 @@ class FranceCultureIE(InfoExtractor):
         }
     }
 
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
+    def _extract_from_player(self, url, video_id):
         webpage = self._download_webpage(url, video_id)
 
         video_path = self._search_regex(
@@ -42,6 +42,9 @@ class FranceCultureIE(InfoExtractor):
             r'<a id="player".*?>\s+<img src="([^"]+)"',
             webpage, 'thumbnail', fatal=False)
 
+        display_id = self._search_regex(
+            r'<span class="path-diffusion">emission-(.*?)</span>', webpage, 'display_id')
+
         title = self._html_search_regex(
             r'<span class="title-diffusion">(.*?)</span>', webpage, 'title')
         alt_title = self._html_search_regex(
@@ -66,4 +69,37 @@ class FranceCultureIE(InfoExtractor):
             'alt_title': alt_title,
             'thumbnail': thumbnail,
             'description': description,
+            'display_id': display_id,
         }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        return self._extract_from_player(url, video_id)
+
+
+class FranceCultureEmissionIE(FranceCultureIE):
+    _VALID_URL = r'https?://(?:www\.)?franceculture\.fr/emission-(?P<id>[^?#]+)'
+    _TEST = {
+        'url': 'http://www.franceculture.fr/emission-les-carnets-de-la-creation-jean-gabriel-periot-cineaste-2015-10-13',
+        'info_dict': {
+            'title': 'Jean-Gabriel Périot, cinéaste',
+            'alt_title': 'Les Carnets de la création',
+            'id': '5093239',
+            'display_id': 'les-carnets-de-la-creation-jean-gabriel-periot-cineaste-2015-10-13',
+            'ext': 'mp3',
+            'timestamp': 1444762500,
+            'upload_date': '20151013',
+            'description': 'startswith:Aujourd\'hui dans "Les carnets de la création", le cinéaste',
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        video_path = self._html_search_regex(
+            r'<a class="rf-player-open".*?href="([^"]+)"', webpage, 'video path', 'no_path_player')
+        if video_path == 'no_path_player':
+            raise ExtractorError('no player : no sound in this page.', expected=True)
+        new_id = self._search_regex('play=(?P<id>[0-9]+)', video_path, 'new_id', group='id')
+        video_url = compat_urlparse.urljoin(url, video_path)
+        return self._extract_from_player(video_url, new_id)

From e974356f32aa31ac8d9e5f2a664c6c521fb45892 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 29 Dec 2015 11:02:58 +0100
Subject: [PATCH 3514/6219] release 2015.12.29

---
 docs/supportedsites.md | 16 ++++++++++++++--
 youtube_dl/version.py  |  2 +-
 2 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 1a5c7cde9..1646277ec 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -37,6 +37,7 @@
  - **appletrailers:section**
  - **archive.org**: archive.org videos
  - **ARD**
+ - **ARD:mediathek**: Saarländischer Rundfunk
  - **ARD:mediathek**
  - **arte.tv**
  - **arte.tv:+7**
@@ -124,8 +125,12 @@
  - **dailymotion:user**
  - **DailymotionCloud**
  - **daum.net**
+ - **daum.net**
  - **DBTV**
  - **DCN**
+ - **dcn:live**
+ - **dcn:season**
+ - **dcn:video**
  - **DctpTv**
  - **DeezerPlaylist**
  - **defense.gouv.fr**
@@ -181,6 +186,7 @@
  - **FoxSports**
  - **france2.fr:generation-quoi**
  - **FranceCulture**
+ - **FranceCultureEmission**
  - **FranceInter**
  - **francetv**: France 2, 3, 4, 5 and Ô
  - **francetvinfo.fr**
@@ -225,6 +231,7 @@
  - **hitbox:live**
  - **HornBunny**
  - **HotNewHipHop**
+ - **HotStar**
  - **Howcast**
  - **HowStuffWorks**
  - **HuffPost**: Huffington Post
@@ -405,6 +412,7 @@
  - **parliamentlive.tv**: UK parliament videos
  - **Patreon**
  - **pbs**: Public Broadcasting Service (PBS) and member stations: PBS: Public Broadcasting Service, APT - Alabama Public Television (WBIQ), GPB/Georgia Public Broadcasting (WGTV), Mississippi Public Broadcasting (WMPN), Nashville Public Television (WNPT), WFSU-TV (WFSU), WSRE (WSRE), WTCI (WTCI), WPBA/Channel 30 (WPBA), Alaska Public Media (KAKM), Arizona PBS (KAET), KNME-TV/Channel 5 (KNME), Vegas PBS (KLVX), AETN/ARKANSAS ETV NETWORK (KETS), KET (WKLE), WKNO/Channel 10 (WKNO), LPB/LOUISIANA PUBLIC BROADCASTING (WLPB), OETA (KETA), Ozarks Public Television (KOZK), WSIU Public Broadcasting (WSIU), KEET TV (KEET), KIXE/Channel 9 (KIXE), KPBS San Diego (KPBS), KQED (KQED), KVIE Public Television (KVIE), PBS SoCal/KOCE (KOCE), ValleyPBS (KVPT), CONNECTICUT PUBLIC TELEVISION (WEDH), KNPB Channel 5 (KNPB), SOPTV (KSYS), Rocky Mountain PBS (KRMA), KENW-TV3 (KENW), KUED Channel 7 (KUED), Wyoming PBS (KCWC), Colorado Public Television / KBDI 12 (KBDI), KBYU-TV (KBYU), Thirteen/WNET New York (WNET), WGBH/Channel 2 (WGBH), WGBY (WGBY), NJTV Public Media NJ (WNJT), WLIW21 (WLIW), mpt/Maryland Public Television (WMPB), WETA Television and Radio (WETA), WHYY (WHYY), PBS 39 (WLVT), WVPT - Your Source for PBS and More! (WVPT), Howard University Television (WHUT), WEDU PBS (WEDU), WGCU Public Media (WGCU), WPBT2 (WPBT), WUCF TV (WUCF), WUFT/Channel 5 (WUFT), WXEL/Channel 42 (WXEL), WLRN/Channel 17 (WLRN), WUSF Public Broadcasting (WUSF), ETV (WRLK), UNC-TV (WUNC), PBS Hawaii - Oceanic Cable Channel 10 (KHET), Idaho Public Television (KAID), KSPS (KSPS), OPB (KOPB), KWSU/Channel 10 & KTNW/Channel 31 (KWSU), WILL-TV (WILL), Network Knowledge - WSEC/Springfield (WSEC), WTTW11 (WTTW), Iowa Public Television/IPTV (KDIN), Nine Network (KETC), PBS39 Fort Wayne (WFWA), WFYI Indianapolis (WFYI), Milwaukee Public Television (WMVS), WNIN (WNIN), WNIT Public Television (WNIT), WPT (WPNE), WVUT/Channel 22 (WVUT), WEIU/Channel 51 (WEIU), WQPT-TV (WQPT), WYCC PBS Chicago (WYCC), WIPB-TV (WIPB), WTIU (WTIU), CET  (WCET), ThinkTVNetwork (WPTD), WBGU-TV (WBGU), WGVU TV (WGVU), NET1 (KUON), Pioneer Public Television (KWCM), SDPB Television (KUSD), TPT (KTCA), KSMQ (KSMQ), KPTS/Channel 8 (KPTS), KTWU/Channel 11 (KTWU), East Tennessee PBS (WSJK), WCTE-TV (WCTE), WLJT, Channel 11 (WLJT), WOSU TV (WOSU), WOUB/WOUC (WOUB), WVPB (WVPB), WKYU-PBS (WKYU), KERA 13 (KERA), MPBN (WCBB), Mountain Lake PBS (WCFE), NHPTV (WENH), Vermont PBS (WETK), witf (WITF), WQED Multimedia (WQED), WMHT Educational Telecommunications (WMHT), Q-TV (WDCQ), WTVS Detroit Public TV (WTVS), CMU Public Television (WCMU), WKAR-TV (WKAR), WNMU-TV Public TV 13 (WNMU), WDSE - WRPT (WDSE), WGTE TV (WGTE), Lakeland Public Television (KAWE), KMOS-TV - Channels 6.1, 6.2 and 6.3 (KMOS), MontanaPBS (KUSM), KRWG/Channel 22 (KRWG), KACV (KACV), KCOS/Channel 13 (KCOS), WCNY/Channel 24 (WCNY), WNED (WNED), WPBS (WPBS), WSKG Public TV (WSKG), WXXI (WXXI), WPSU (WPSU), WVIA Public Media Studios (WVIA), WTVI (WTVI), Western Reserve PBS (WNEO), WVIZ/PBS ideastream (WVIZ), KCTS 9 (KCTS), Basin PBS (KPBT), KUHT / Channel 8 (KUHT), KLRN (KLRN), KLRU (KLRU), WTJX Channel 12 (WTJX), WCVE PBS (WCVE), KBTC Public Television (KBTC)
+ - **pcmag**
  - **Periscope**: Periscope
  - **PhilharmonieDeParis**: Philharmonie de Paris
  - **Phoenix**
@@ -445,6 +453,7 @@
  - **radiofrance**
  - **RadioJavan**
  - **Rai**
+ - **RaiTV**
  - **RBMARadio**
  - **RDS**: RDS.ca
  - **RedTube**
@@ -527,7 +536,6 @@
  - **SportDeutschland**
  - **Sportschau**
  - **Srf**
- - **SRMediathek**: Saarländischer Rundfunk
  - **SSA**
  - **stanfordoc**: Stanford Open ClassRoom
  - **Steam**
@@ -561,6 +569,7 @@
  - **TenPlay**
  - **TestTube**
  - **TF1**
+ - **TheIntercept**
  - **TheOnion**
  - **ThePlatform**
  - **ThePlatformFeed**
@@ -586,7 +595,10 @@
  - **TubiTv**
  - **Tudou**
  - **Tumblr**
- - **TuneIn**
+ - **tunein:clip**
+ - **tunein:program**
+ - **tunein:station**
+ - **tunein:topic**
  - **Turbo**
  - **Tutv**
  - **tv.dfb.de**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 255d64269..a62baa305 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.12.23'
+__version__ = '2015.12.29'

From 422f7c112c12d92bf06c95ff33782c8e9a0df43e Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 29 Dec 2015 11:36:04 +0100
Subject: [PATCH 3515/6219] [srgssr] update tests

---
 youtube_dl/extractor/rts.py    | 30 +++++++++++++++++----------
 youtube_dl/extractor/srgssr.py | 38 ++++++++++++----------------------
 2 files changed, 32 insertions(+), 36 deletions(-)

diff --git a/youtube_dl/extractor/rts.py b/youtube_dl/extractor/rts.py
index dc10fd88b..f063ab5dd 100644
--- a/youtube_dl/extractor/rts.py
+++ b/youtube_dl/extractor/rts.py
@@ -28,7 +28,7 @@ class RTSIE(SRGSSRIE):
             'info_dict': {
                 'id': '3449373',
                 'display_id': 'les-enfants-terribles',
-                'ext': 'flv',
+                'ext': 'mp4',
                 'duration': 1488,
                 'title': 'Les Enfants Terribles',
                 'description': 'France Pommier et sa soeur Luce Feral, les deux filles de ce groupe de 5.',
@@ -38,6 +38,10 @@ class RTSIE(SRGSSRIE):
                 'thumbnail': 're:^https?://.*\.image',
                 'view_count': int,
             },
+            'params': {
+                # m3u8 download
+                'skip_download': True,
+            }
         },
         {
             'url': 'http://www.rts.ch/emissions/passe-moi-les-jumelles/5624067-entre-ciel-et-mer.html',
@@ -45,7 +49,7 @@ class RTSIE(SRGSSRIE):
             'info_dict': {
                 'id': '5742494',
                 'display_id': '5742494',
-                'ext': 'flv',
+                'ext': 'mp4',
                 'duration': 3720,
                 'title': 'Les yeux dans les cieux - Mon homard au Canada',
                 'description': 'md5:d22ee46f5cc5bac0912e5a0c6d44a9f7',
@@ -55,6 +59,10 @@ class RTSIE(SRGSSRIE):
                 'thumbnail': 're:^https?://.*\.image',
                 'view_count': int,
             },
+            'params': {
+                # m3u8 download
+                'skip_download': True,
+            }
         },
         {
             'url': 'http://www.rts.ch/video/sport/hockey/5745975-1-2-kloten-fribourg-5-2-second-but-pour-gotteron-par-kwiatowski.html',
@@ -80,7 +88,7 @@ class RTSIE(SRGSSRIE):
             'info_dict': {
                 'id': '5745356',
                 'display_id': 'londres-cachee-par-un-epais-smog',
-                'ext': 'flv',
+                'ext': 'mp4',
                 'duration': 33,
                 'title': 'Londres cachée par un épais smog',
                 'description': 'Un important voile de smog recouvre Londres depuis mercredi, provoqué par la pollution et du sable du Sahara.',
@@ -90,6 +98,10 @@ class RTSIE(SRGSSRIE):
                 'thumbnail': 're:^https?://.*\.image',
                 'view_count': int,
             },
+            'params': {
+                # m3u8 download
+                'skip_download': True,
+            }
         },
         {
             'url': 'http://www.rts.ch/audio/couleur3/programmes/la-belle-video-de-stephane-laurenceau/5706148-urban-hippie-de-damien-krisl-03-04-2014.html',
@@ -180,16 +192,12 @@ class RTSIE(SRGSSRIE):
                 auth_params = xpath_text(token, './/authparams', 'auth params')
                 if not auth_params:
                     continue
-                f4m_formats = self._extract_f4m_formats(
+                formats.extend(self._extract_f4m_formats(
                     '%s?%s&hdcore=3.4.0&plugin=aasp-3.4.0.132.66' % (format_url, auth_params),
-                    media_id, f4m_id=format_id, fatal=False)
-                if f4m_formats:
-                    formats.extend(f4m_formats)
+                    media_id, f4m_id=format_id, fatal=False))
             elif format_url.endswith('.m3u8'):
-                m3u8_formats = self._extract_m3u8_formats(
-                    format_url, media_id, 'mp4', m3u8_id=format_id, fatal=False)
-                if m3u8_formats:
-                    formats.extend(m3u8_formats)
+                formats.extend(self._extract_m3u8_formats(
+                    format_url, media_id, 'mp4', m3u8_id=format_id, fatal=False))
             else:
                 formats.append({
                     'format_id': format_id,
diff --git a/youtube_dl/extractor/srgssr.py b/youtube_dl/extractor/srgssr.py
index 563851b3f..4962fe885 100644
--- a/youtube_dl/extractor/srgssr.py
+++ b/youtube_dl/extractor/srgssr.py
@@ -47,31 +47,24 @@ class SRGSSRIE(InfoExtractor):
         created_date = media_data.get('createdDate') or metadata.get('createdDate')
         timestamp = parse_iso8601(created_date)
 
-        thumbnails = []
-        if 'Image' in media_data:
-            for image in media_data['Image']['ImageRepresentations']['ImageRepresentation']:
-                thumbnails.append({
-                    'id': image.get('id'),
-                    'url': image['url'],
-                })
+        thumbnails = [{
+            'id': image.get('id'),
+            'url': image['url'],
+        } for image in media_data.get('Image', {}).get('ImageRepresentations', {}).get('ImageRepresentation', [])]
 
         preference = qualities(['LQ', 'MQ', 'SD', 'HQ', 'HD'])
         formats = []
-        for source in media_data['Playlists']['Playlist']:
-            assets = {}
+        for source in media_data.get('Playlists', {}).get('Playlist', []) + media_data.get('Downloads', {}).get('Download', []):
             protocol = source.get('@protocol')
             if protocol in ('HTTP-HDS', 'HTTP-HLS'):
+                assets = {}
                 for quality in source['url']:
                     assets[quality['@quality']] = quality['text']
                 asset_url = assets.get('HD') or assets.get('HQ') or assets.get('SD') or assets.get('MQ') or assets.get('LQ')
                 if '.f4m' in asset_url:
-                    f4m_formats = formats.extend(self._extract_f4m_formats(asset_url + '?hdcore=3.4.0', media_id, f4m_id='hds', fatal=False))
-                    if f4m_formats:
-                        formats.extend(f4m_formats)
+                    formats.extend(self._extract_f4m_formats(asset_url + '?hdcore=3.4.0', media_id, f4m_id='hds', fatal=False))
                 elif '.m3u8' in asset_url:
-                    m3u8_formats = formats.extend(self._extract_m3u8_formats(asset_url, media_id, m3u8_id='hls', fatal=False))
-                    if m3u8_formats:
-                        formats.extend(m3u8_formats)
+                    formats.extend(self._extract_m3u8_formats(asset_url, media_id, m3u8_id='hls', fatal=False))
             else:
                 for asset in source['url']:
                     asset_url = asset['text']
@@ -84,15 +77,6 @@ class SRGSSRIE(InfoExtractor):
                         'preference': preference(asset['@quality']),
                         'ext': ext,
                     })
-
-        if 'Downloads' in media_data:
-            for source in media_data['Downloads']['Download']:
-                for asset in source['url']:
-                    formats.append({
-                        'format_id': asset['@quality'],
-                        'url': asset['text'],
-                        'preference': preference(asset['@quality'])
-                    })
         self._sort_formats(formats)
 
         return {
@@ -149,7 +133,7 @@ class SRGSSRPlayIE(InfoExtractor):
         'info_dict': {
             'id': '6348260',
             'display_id': '6348260',
-            'ext': 'flv',
+            'ext': 'mp4',
             'duration': 1796,
             'title': 'Le 19h30',
             'description': '',
@@ -159,6 +143,10 @@ class SRGSSRPlayIE(InfoExtractor):
             'thumbnail': 're:^https?://.*\.image',
             'view_count': int,
         },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        }
     }]
 
     def _real_extract(self, url):

From db710571fd7bf6b697a7999fa1ea76dd2b8b1382 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 29 Dec 2015 11:59:27 +0100
Subject: [PATCH 3516/6219] [daum] fix IE_NAME

---
 youtube_dl/extractor/daum.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/daum.py b/youtube_dl/extractor/daum.py
index a083cc0dc..f08f57157 100644
--- a/youtube_dl/extractor/daum.py
+++ b/youtube_dl/extractor/daum.py
@@ -77,7 +77,7 @@ class DaumIE(InfoExtractor):
 
 class DaumClipIE(InfoExtractor):
     _VALID_URL = r'https?://(?:m\.)?tvpot\.daum\.net/(?:clip/ClipView.do|mypot/View.do)\?.*?clipid=(?P<id>\d+)'
-    IE_NAME = 'daum.net'
+    IE_NAME = 'daum.net:clip'
 
     _TESTS = [{
         'url': 'http://tvpot.daum.net/clip/ClipView.do?clipid=52554690',

From 6c4d6609ad14d33ef1a9b13a938a9413ac438e2c Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 29 Dec 2015 12:00:52 +0100
Subject: [PATCH 3517/6219] [phoenix] fix IE_NAME

---
 youtube_dl/extractor/phoenix.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/phoenix.py b/youtube_dl/extractor/phoenix.py
index 6ce2ec19d..ac009f60f 100644
--- a/youtube_dl/extractor/phoenix.py
+++ b/youtube_dl/extractor/phoenix.py
@@ -4,6 +4,7 @@ from .zdf import ZDFIE
 
 
 class PhoenixIE(ZDFIE):
+    IE_NAME = 'phoenix.de'
     _VALID_URL = r'''(?x)https?://(?:www\.)?phoenix\.de/content/
         (?:
             phoenix/die_sendungen/(?:[^/]+/)?

From ce7d243c7e6b6f5eb5129a3e85aa75a3773b3a73 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 29 Dec 2015 12:01:22 +0100
Subject: [PATCH 3518/6219] [srgssr] fix IE_DESC

---
 youtube_dl/extractor/srgssr.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/srgssr.py b/youtube_dl/extractor/srgssr.py
index 4962fe885..2c8b4276b 100644
--- a/youtube_dl/extractor/srgssr.py
+++ b/youtube_dl/extractor/srgssr.py
@@ -90,6 +90,7 @@ class SRGSSRIE(InfoExtractor):
 
 
 class SRGSSRPlayIE(InfoExtractor):
+    IE_DESC = 'srf.ch, rts.ch, rsi.ch, rtr.ch and swissinfo.ch play sites'
     _VALID_URL = r'https?://(?:(?:www|play)\.)?(?P<bu>srf|rts|rsi|rtr|swissinfo)\.ch/play/(?:tv|radio)/[^/]+/(?P<type>video|audio)/[^?]+\?id=(?P<id>[0-9a-f\-]{36}|\d+)'
 
     _TESTS = [{

From eccde5e9de26dbe0c289b5bd1bf40b0482da59b1 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 29 Dec 2015 14:53:06 +0100
Subject: [PATCH 3519/6219] [audimedia] split long lines

---
 youtube_dl/extractor/audimedia.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/audimedia.py b/youtube_dl/extractor/audimedia.py
index 9b037bb0c..3b2effa15 100644
--- a/youtube_dl/extractor/audimedia.py
+++ b/youtube_dl/extractor/audimedia.py
@@ -45,11 +45,15 @@ class AudiMediaIE(InfoExtractor):
 
             stream_url_hls = json_data.get('stream_url_hls')
             if stream_url_hls:
-                formats.extend(self._extract_m3u8_formats(stream_url_hls, video_id, 'mp4', entry_protocol='m3u8_native', m3u8_id='hls', fatal=False))
+                formats.extend(self._extract_m3u8_formats(
+                    stream_url_hls, video_id, 'mp4',
+                    entry_protocol='m3u8_native', m3u8_id='hls', fatal=False))
 
             stream_url_hds = json_data.get('stream_url_hds')
             if stream_url_hds:
-                formats.extend(self._extract_f4m_formats(json_data.get('stream_url_hds') + '?hdcore=3.4.0', video_id, -1, f4m_id='hds', fatal=False))
+                formats.extend(self._extract_f4m_formats(
+                    stream_url_hds + '?hdcore=3.4.0',
+                    video_id, f4m_id='hds', fatal=False))
 
             for video_version in json_data.get('video_versions'):
                 video_version_url = video_version.get('download_url') or video_version.get('stream_url')

From f889ac45b83cd771919fba29c14cee9d24e637b7 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 29 Dec 2015 14:55:36 +0100
Subject: [PATCH 3520/6219] [ign] split long lines

---
 youtube_dl/extractor/ign.py | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/ign.py b/youtube_dl/extractor/ign.py
index d1c1c210c..c45c68c1d 100644
--- a/youtube_dl/extractor/ign.py
+++ b/youtube_dl/extractor/ign.py
@@ -120,19 +120,24 @@ class IGNIE(InfoExtractor):
 
         video_id = self._find_video_id(webpage)
         if not video_id:
-            return self.url_result(self._search_regex(self._EMBED_RE, webpage, 'embed url'))
+            return self.url_result(self._search_regex(
+                self._EMBED_RE, webpage, 'embed url'))
         return self._get_video_info(video_id)
 
     def _get_video_info(self, video_id):
-        api_data = self._download_json(self._API_URL_TEMPLATE % video_id, video_id)
+        api_data = self._download_json(
+            self._API_URL_TEMPLATE % video_id, video_id)
 
         formats = []
         m3u8_url = api_data['refs'].get('m3uUrl')
         if m3u8_url:
-            formats.extend(self._extract_m3u8_formats(m3u8_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
+            formats.extend(self._extract_m3u8_formats(
+                m3u8_url, video_id, 'mp4', 'm3u8_native',
+                m3u8_id='hls', fatal=False))
         f4m_url = api_data['refs'].get('f4mUrl')
         if f4m_url:
-            formats.extend(self._extract_f4m_formats(f4m_url, video_id, f4m_id='hds', fatal=False))
+            formats.extend(self._extract_f4m_formats(
+                f4m_url, video_id, f4m_id='hds', fatal=False))
         for asset in api_data['assets']:
             formats.append({
                 'url': asset['url'],

From f2017cb020efe0956049bda11df4eec7c5985fda Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 29 Dec 2015 14:58:22 +0100
Subject: [PATCH 3521/6219] [srgssr] split long lines and use m3u8_native

---
 youtube_dl/extractor/rts.py    |  2 +-
 youtube_dl/extractor/srgssr.py | 11 ++++++++---
 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/rts.py b/youtube_dl/extractor/rts.py
index f063ab5dd..3cc32847b 100644
--- a/youtube_dl/extractor/rts.py
+++ b/youtube_dl/extractor/rts.py
@@ -197,7 +197,7 @@ class RTSIE(SRGSSRIE):
                     media_id, f4m_id=format_id, fatal=False))
             elif format_url.endswith('.m3u8'):
                 formats.extend(self._extract_m3u8_formats(
-                    format_url, media_id, 'mp4', m3u8_id=format_id, fatal=False))
+                    format_url, media_id, 'mp4', 'm3u8_native', m3u8_id=format_id, fatal=False))
             else:
                 formats.append({
                     'format_id': format_id,
diff --git a/youtube_dl/extractor/srgssr.py b/youtube_dl/extractor/srgssr.py
index 2c8b4276b..d06d34316 100644
--- a/youtube_dl/extractor/srgssr.py
+++ b/youtube_dl/extractor/srgssr.py
@@ -29,7 +29,8 @@ class SRGSSRIE(InfoExtractor):
             media_id)[media_type.capitalize()]
 
         if media_data.get('block') and media_data['block'] in self._ERRORS:
-            raise ExtractorError('%s said: %s' % (self.IE_NAME, self._ERRORS[media_data['block']]), expected=True)
+            raise ExtractorError('%s said: %s' % (
+                self.IE_NAME, self._ERRORS[media_data['block']]), expected=True)
 
         return media_data
 
@@ -62,9 +63,13 @@ class SRGSSRIE(InfoExtractor):
                     assets[quality['@quality']] = quality['text']
                 asset_url = assets.get('HD') or assets.get('HQ') or assets.get('SD') or assets.get('MQ') or assets.get('LQ')
                 if '.f4m' in asset_url:
-                    formats.extend(self._extract_f4m_formats(asset_url + '?hdcore=3.4.0', media_id, f4m_id='hds', fatal=False))
+                    formats.extend(self._extract_f4m_formats(
+                        asset_url + '?hdcore=3.4.0', media_id,
+                        f4m_id='hds', fatal=False))
                 elif '.m3u8' in asset_url:
-                    formats.extend(self._extract_m3u8_formats(asset_url, media_id, m3u8_id='hls', fatal=False))
+                    formats.extend(self._extract_m3u8_formats(
+                        asset_url, media_id, 'mp4', 'm3u8_native',
+                        m3u8_id='hls', fatal=False))
             else:
                 for asset in source['url']:
                     asset_url = asset['text']

From 98d7c0f4f7d88cdfed8dfd3ad81efec36090c7fb Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 29 Dec 2015 15:02:18 +0100
Subject: [PATCH 3522/6219] [tele13] split long lines

---
 youtube_dl/extractor/tele13.py | 21 +++++++++++++++------
 1 file changed, 15 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/tele13.py b/youtube_dl/extractor/tele13.py
index fe11c20df..4e860db0a 100644
--- a/youtube_dl/extractor/tele13.py
+++ b/youtube_dl/extractor/tele13.py
@@ -46,8 +46,12 @@ class Tele13IE(InfoExtractor):
         display_id = self._match_id(url)
         webpage = self._download_webpage(url, display_id)
 
-        setup_js = self._search_regex(r"(?s)jwplayer\('player-vivo'\).setup\((\{.*?\})\)", webpage, 'setup code')
-        sources = self._parse_json(self._search_regex(r'sources\s*:\s*(\[[^\]]+\])', setup_js, 'sources'), display_id, js_to_json)
+        setup_js = self._search_regex(
+            r"(?s)jwplayer\('player-vivo'\).setup\((\{.*?\})\)",
+            webpage, 'setup code')
+        sources = self._parse_json(self._search_regex(
+            r'sources\s*:\s*(\[[^\]]+\])', setup_js, 'sources'),
+            display_id, js_to_json)
 
         preference = qualities(['Móvil', 'SD', 'HD'])
         formats = []
@@ -57,7 +61,9 @@ class Tele13IE(InfoExtractor):
             if format_url and format_url not in urls:
                 ext = determine_ext(format_url)
                 if ext == 'm3u8':
-                    formats.extend(self._extract_m3u8_formats(format_url, display_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
+                    formats.extend(self._extract_m3u8_formats(
+                        format_url, display_id, 'mp4', 'm3u8_native',
+                        m3u8_id='hls', fatal=False))
                 elif YoutubeIE.suitable(format_url):
                     return self.url_result(format_url, 'Youtube')
                 else:
@@ -72,8 +78,11 @@ class Tele13IE(InfoExtractor):
 
         return {
             'id': display_id,
-            'title': self._search_regex(r'title\s*:\s*"([^"]+)"', setup_js, 'title'),
-            'description': self._html_search_meta('description', webpage, 'description'),
-            'thumbnail': self._search_regex(r'image\s*:\s*"([^"]+)"', setup_js, 'thumbnail', default=None),
+            'title': self._search_regex(
+                r'title\s*:\s*"([^"]+)"', setup_js, 'title'),
+            'description': self._html_search_meta(
+                'description', webpage, 'description'),
+            'thumbnail': self._search_regex(
+                r'image\s*:\s*"([^"]+)"', setup_js, 'thumbnail', default=None),
             'formats': formats,
         }

From bf96b45238b221a01ea9e07d7747313272cde142 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 29 Dec 2015 15:03:14 +0100
Subject: [PATCH 3523/6219] [rai] split long lines

---
 youtube_dl/extractor/rai.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/rai.py b/youtube_dl/extractor/rai.py
index f2679591b..a4dc5c335 100644
--- a/youtube_dl/extractor/rai.py
+++ b/youtube_dl/extractor/rai.py
@@ -107,7 +107,8 @@ class RaiTVIE(InfoExtractor):
                 return xml.replace(' tag elementi', '').replace('>/', '</')
 
             relinker = self._download_xml(
-                media['mediaUri'] + '&output=43', video_id, transform_source=fix_xml)
+                media['mediaUri'] + '&output=43',
+                video_id, transform_source=fix_xml)
 
             has_subtitle = False
 
@@ -117,8 +118,8 @@ class RaiTVIE(InfoExtractor):
                 content_type = xpath_text(element, 'content-type')
                 if ext == 'm3u8':
                     formats.extend(self._extract_m3u8_formats(
-                        media_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls',
-                        fatal=False))
+                        media_url, video_id, 'mp4', 'm3u8_native',
+                        m3u8_id='hls', fatal=False))
                 elif ext == 'f4m':
                     formats.extend(self._extract_f4m_formats(
                         media_url + '?hdcore=3.7.0&plugin=aasp-3.7.0.39.44',

From 6101f45ef90e5b893477c684baa0d797d15bff29 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 29 Dec 2015 15:05:21 +0100
Subject: [PATCH 3524/6219] [ooyala] split long lines, fix test duration and
 add hdcode param to hds url

---
 youtube_dl/extractor/ooyala.py | 31 ++++++++++++++++++++++---------
 1 file changed, 22 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/ooyala.py b/youtube_dl/extractor/ooyala.py
index 3960d522e..20b984288 100644
--- a/youtube_dl/extractor/ooyala.py
+++ b/youtube_dl/extractor/ooyala.py
@@ -13,6 +13,9 @@ from ..compat import compat_urllib_parse
 
 
 class OoyalaBaseIE(InfoExtractor):
+    _PLAYER_BASE = 'http://player.ooyala.com/'
+    _CONTENT_TREE_BASE = _PLAYER_BASE + 'player_api/v1/content_tree/'
+    _AUTHORIZATION_URL_TEMPLATE = _PLAYER_BASE + 'sas/player_api/v1/authorization/embed_code/%s/%s?'
 
     def _extract(self, content_tree_url, video_id, domain='example.org'):
         content_tree = self._download_json(content_tree_url, video_id)['content_tree']
@@ -31,24 +34,33 @@ class OoyalaBaseIE(InfoExtractor):
         formats = []
         for supported_format in ('mp4', 'm3u8', 'hds', 'rtmp'):
             auth_data = self._download_json(
-                'http://player.ooyala.com/sas/player_api/v1/authorization/embed_code/%s/%s?' % (pcode, embed_code) + compat_urllib_parse.urlencode({'domain': domain, 'supportedFormats': supported_format}),
+                self._AUTHORIZATION_URL_TEMPLATE % (pcode, embed_code) +
+                compat_urllib_parse.urlencode({
+                    'domain': domain,
+                    'supportedFormats': supported_format
+                }),
                 video_id, 'Downloading %s JSON' % supported_format)
 
             cur_auth_data = auth_data['authorization_data'][embed_code]
 
             if cur_auth_data['authorized']:
                 for stream in cur_auth_data['streams']:
-                    url = base64.b64decode(stream['url']['data'].encode('ascii')).decode('utf-8')
+                    url = base64.b64decode(
+                        stream['url']['data'].encode('ascii')).decode('utf-8')
                     if url in urls:
                         continue
                     urls.append(url)
                     delivery_type = stream['delivery_type']
                     if delivery_type == 'hls' or '.m3u8' in url:
-                        formats.extend(self._extract_m3u8_formats(url, embed_code, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
+                        formats.extend(self._extract_m3u8_formats(
+                            url, embed_code, 'mp4', 'm3u8_native',
+                            m3u8_id='hls', fatal=False))
                     elif delivery_type == 'hds' or '.f4m' in url:
-                        formats.extend(self._extract_f4m_formats(url, embed_code, f4m_id='hds', fatal=False))
+                        formats.extend(self._extract_f4m_formats(
+                            url + '?hdcore=3.7.0', embed_code, f4m_id='hds', fatal=False))
                     elif '.smil' in url:
-                        formats.extend(self._extract_smil_formats(url, embed_code, fatal=False))
+                        formats.extend(self._extract_smil_formats(
+                            url, embed_code, fatal=False))
                     else:
                         formats.append({
                             'url': url,
@@ -62,7 +74,8 @@ class OoyalaBaseIE(InfoExtractor):
                             'fps': float_or_none(stream.get('framerate')),
                         })
             else:
-                raise ExtractorError('%s said: %s' % (self.IE_NAME, cur_auth_data['message']), expected=True)
+                raise ExtractorError('%s said: %s' % (
+                    self.IE_NAME, cur_auth_data['message']), expected=True)
         self._sort_formats(formats)
 
         video_info['formats'] = formats
@@ -120,7 +133,7 @@ class OoyalaIE(OoyalaBaseIE):
         url, smuggled_data = unsmuggle_url(url, {})
         embed_code = self._match_id(url)
         domain = smuggled_data.get('domain')
-        content_tree_url = 'http://player.ooyala.com/player_api/v1/content_tree/embed_code/%s/%s' % (embed_code, embed_code)
+        content_tree_url = self._CONTENT_TREE_BASE + 'embed_code/%s/%s' % (embed_code, embed_code)
         return self._extract(content_tree_url, embed_code, domain)
 
 
@@ -147,7 +160,7 @@ class OoyalaExternalIE(OoyalaBaseIE):
             'id': 'FkYWtmazr6Ed8xmvILvKLWjd4QvYZpzG',
             'ext': 'mp4',
             'title': 'dm_140128_30for30Shorts___JudgingJewellv2',
-            'duration': 1302000,
+            'duration': 1302.0,
         },
         'params': {
             # m3u8 download
@@ -157,5 +170,5 @@ class OoyalaExternalIE(OoyalaBaseIE):
 
     def _real_extract(self, url):
         partner_id, video_id, pcode = re.match(self._VALID_URL, url).groups()
-        content_tree_url = 'http://player.ooyala.com/player_api/v1/content_tree/external_id/%s/%s:%s' % (pcode, partner_id, video_id)
+        content_tree_url = self._CONTENT_TREE_BASE + 'external_id/%s/%s:%s' % (pcode, partner_id, video_id)
         return self._extract(content_tree_url, video_id)

From 0368181998cd5ec365bb8ed10c5da0188fbed798 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 29 Dec 2015 15:07:21 +0100
Subject: [PATCH 3525/6219] [wdr] split long lines

---
 youtube_dl/extractor/wdr.py | 35 +++++++++++++++++++++++------------
 1 file changed, 23 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/wdr.py b/youtube_dl/extractor/wdr.py
index e8511398c..a851578e0 100644
--- a/youtube_dl/extractor/wdr.py
+++ b/youtube_dl/extractor/wdr.py
@@ -108,7 +108,9 @@ class WDRIE(InfoExtractor):
         if mobj.group('player') is None:
             entries = [
                 self.url_result(page_url + href, 'WDR')
-                for href in re.findall(r'<a href="/?(.+?%s\.html)" rel="nofollow"' % self._PLAYER_REGEX, webpage)
+                for href in re.findall(
+                    r'<a href="/?(.+?%s\.html)" rel="nofollow"' % self._PLAYER_REGEX,
+                    webpage)
             ]
 
             if entries:  # Playlist page
@@ -133,8 +135,8 @@ class WDRIE(InfoExtractor):
                     note='Downloading playlist page %d' % page_num)
             return self.playlist_result(entries, page_id)
 
-        flashvars = compat_parse_qs(
-            self._html_search_regex(r'<param name="flashvars" value="([^"]+)"', webpage, 'flashvars'))
+        flashvars = compat_parse_qs(self._html_search_regex(
+            r'<param name="flashvars" value="([^"]+)"', webpage, 'flashvars'))
 
         page_id = flashvars['trackerClipId'][0]
         video_url = flashvars['dslSrc'][0]
@@ -148,7 +150,8 @@ class WDRIE(InfoExtractor):
         if 'trackerClipAirTime' in flashvars:
             upload_date = flashvars['trackerClipAirTime'][0]
         else:
-            upload_date = self._html_search_meta('DC.Date', webpage, 'upload date')
+            upload_date = self._html_search_meta(
+                'DC.Date', webpage, 'upload date')
 
         if upload_date:
             upload_date = unified_strdate(upload_date)
@@ -157,12 +160,15 @@ class WDRIE(InfoExtractor):
         preference = qualities(['S', 'M', 'L', 'XL'])
 
         if video_url.endswith('.f4m'):
-            formats.extend(self._extract_f4m_formats(video_url + '?hdcore=3.2.0&plugin=aasp-3.2.0.77.18', page_id, f4m_id='hds', fatal=False))
+            formats.extend(self._extract_f4m_formats(
+                video_url + '?hdcore=3.2.0&plugin=aasp-3.2.0.77.18', page_id,
+                f4m_id='hds', fatal=False))
         elif video_url.endswith('.smil'):
-            formats.extend(self._extract_smil_formats(video_url, page_id, False, {
-                'hdcore': '3.3.0',
-                'plugin': 'aasp-3.3.0.99.43',
-            }))
+            formats.extend(self._extract_smil_formats(
+                video_url, page_id, False, {
+                    'hdcore': '3.3.0',
+                    'plugin': 'aasp-3.3.0.99.43',
+                }))
         else:
             formats.append({
                 'url': video_url,
@@ -171,11 +177,16 @@ class WDRIE(InfoExtractor):
                 },
             })
 
-        m3u8_url = self._search_regex(r'rel="adaptiv"[^>]+href="([^"]+)"', webpage, 'm3u8 url', default=None)
+        m3u8_url = self._search_regex(
+            r'rel="adaptiv"[^>]+href="([^"]+)"',
+            webpage, 'm3u8 url', default=None)
         if m3u8_url:
-            formats.extend(self._extract_m3u8_formats(m3u8_url, page_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
+            formats.extend(self._extract_m3u8_formats(
+                m3u8_url, page_id, 'mp4', 'm3u8_native',
+                m3u8_id='hls', fatal=False))
 
-        direct_urls = re.findall(r'rel="web(S|M|L|XL)"[^>]+href="([^"]+)"', webpage)
+        direct_urls = re.findall(
+            r'rel="web(S|M|L|XL)"[^>]+href="([^"]+)"', webpage)
         if direct_urls:
             for quality, video_url in direct_urls:
                 formats.append({

From 53365f74a7142c0e16eeb8db2c08f1f200de2406 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 29 Dec 2015 16:14:05 +0100
Subject: [PATCH 3526/6219] Credit @flatgreen for FranceCultureEmission (#8022)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index ce350e96c..20f51009e 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -149,3 +149,4 @@ Marco Ferragina
 reiv
 Muratcan Simsek
 Evan Lu
+flatgreen

From 0d8a0fdc30845aecfb9c98ccd4884dd1456dda86 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 29 Dec 2015 16:38:06 +0100
Subject: [PATCH 3527/6219] [srgssr] use SRFIE format ids

---
 youtube_dl/extractor/srgssr.py | 27 ++++++++++++---------------
 1 file changed, 12 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/srgssr.py b/youtube_dl/extractor/srgssr.py
index d06d34316..4707029ca 100644
--- a/youtube_dl/extractor/srgssr.py
+++ b/youtube_dl/extractor/srgssr.py
@@ -57,29 +57,26 @@ class SRGSSRIE(InfoExtractor):
         formats = []
         for source in media_data.get('Playlists', {}).get('Playlist', []) + media_data.get('Downloads', {}).get('Download', []):
             protocol = source.get('@protocol')
-            if protocol in ('HTTP-HDS', 'HTTP-HLS'):
-                assets = {}
-                for quality in source['url']:
-                    assets[quality['@quality']] = quality['text']
-                asset_url = assets.get('HD') or assets.get('HQ') or assets.get('SD') or assets.get('MQ') or assets.get('LQ')
-                if '.f4m' in asset_url:
+            for asset in source['url']:
+                asset_url = asset['text']
+                quality = asset['@quality']
+                format_id = '%s-%s' % (protocol, quality)
+                if protocol == 'HTTP-HDS':
                     formats.extend(self._extract_f4m_formats(
                         asset_url + '?hdcore=3.4.0', media_id,
-                        f4m_id='hds', fatal=False))
-                elif '.m3u8' in asset_url:
+                        f4m_id=format_id, fatal=False))
+                elif protocol == 'HTTP-HLS':
                     formats.extend(self._extract_m3u8_formats(
                         asset_url, media_id, 'mp4', 'm3u8_native',
-                        m3u8_id='hls', fatal=False))
-            else:
-                for asset in source['url']:
-                    asset_url = asset['text']
+                        m3u8_id=format_id, fatal=False))
+                else:
                     ext = None
-                    if asset_url.startswith('rtmp'):
+                    if protocol == 'RTMP':
                         ext = self._search_regex(r'([a-z0-9]+):[^/]+', asset_url, 'ext')
                     formats.append({
-                        'format_id': asset['@quality'],
+                        'format_id': format_id,
                         'url': asset_url,
-                        'preference': preference(asset['@quality']),
+                        'preference': preference(quality),
                         'ext': ext,
                     })
         self._sort_formats(formats)

From 030dfb04e0deff7e824194a7ddea2b55cf050e34 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 29 Dec 2015 23:57:46 +0600
Subject: [PATCH 3528/6219] [videomore] Add extractor (Closes #8040)

---
 youtube_dl/extractor/__init__.py  |   5 +
 youtube_dl/extractor/videomore.py | 186 ++++++++++++++++++++++++++++++
 2 files changed, 191 insertions(+)
 create mode 100644 youtube_dl/extractor/videomore.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 75d191d5d..fb7151443 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -781,6 +781,11 @@ from .viddler import ViddlerIE
 from .videodetective import VideoDetectiveIE
 from .videofyme import VideofyMeIE
 from .videomega import VideoMegaIE
+from .videomore import (
+    VideomoreIE,
+    VideomoreVideoIE,
+    VideomoreSeasonIE,
+)
 from .videopremium import VideoPremiumIE
 from .videott import VideoTtIE
 from .vidme import VidmeIE
diff --git a/youtube_dl/extractor/videomore.py b/youtube_dl/extractor/videomore.py
new file mode 100644
index 000000000..dbc9dbdb7
--- /dev/null
+++ b/youtube_dl/extractor/videomore.py
@@ -0,0 +1,186 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    parse_age_limit,
+    parse_iso8601,
+    xpath_text,
+)
+
+
+class VideomoreIE(InfoExtractor):
+    _VALID_URL = r'videomore:(?P<sid>\d+)$|https?://videomore\.ru/(?:(?:embed|[^/]+/[^/]+)/|[^/]+\?.*\btrack_id=)(?P<id>\d+)(?:[/?#&]|\.(?:xml|json)|$)'
+    _TESTS = [{
+        'url': 'http://videomore.ru/kino_v_detalayah/5_sezon/367617',
+        'md5': '70875fbf57a1cd004709920381587185',
+        'info_dict': {
+            'id': '367617',
+            'ext': 'flv',
+            'title': 'В гостях Алексей Чумаков и Юлия Ковальчук',
+            'description': 'В гостях – лучшие романтические комедии года, «Выживший» Иньярриту и «Стив Джобс» Дэнни Бойла.',
+            'thumbnail': 're:^https?://.*\.jpg',
+            'duration': 2910,
+            'age_limit': 16,
+            'view_count': int,
+        },
+    }, {
+        'url': 'http://videomore.ru/elki_3?track_id=364623',
+        'only_matching': True,
+    }, {
+        'url': 'http://videomore.ru/embed/364623',
+        'only_matching': True,
+    }, {
+        'url': 'http://videomore.ru/video/tracks/364623.xml',
+        'only_matching': True,
+    }, {
+        'url': 'http://videomore.ru/video/tracks/364623.json',
+        'only_matching': True,
+    }, {
+        'url': 'http://videomore.ru/video/tracks/158031/quotes/33248',
+        'only_matching': True,
+    }, {
+        'url': 'videomore:367617',
+        'only_matching': True,
+    }]
+
+    @staticmethod
+    def _extract_url(webpage):
+        mobj = re.search(
+            r'<object[^>]+data=(["\'])https?://videomore.ru/player\.swf\?.*config=(?P<url>https?://videomore\.ru/(?:[^/]+/)+\d+\.xml).*\1',
+            webpage)
+        if mobj:
+            return mobj.group('url')
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('sid') or mobj.group('id')
+
+        video = self._download_xml(
+            'http://videomore.ru/video/tracks/%s.xml' % video_id,
+            video_id, 'Downloading video XML')
+
+        video_url = xpath_text(video, './/video_url', 'video url', fatal=True)
+        formats = self._extract_f4m_formats(video_url, video_id, f4m_id='hds')
+
+        data = self._download_json(
+            'http://videomore.ru/video/tracks/%s.json' % video_id,
+            video_id, 'Downloadinng video JSON')
+
+        title = data.get('title') or data['project_title']
+        description = data.get('description') or data.get('description_raw')
+        timestamp = parse_iso8601(data.get('published_at'))
+        duration = int_or_none(data.get('duration'))
+        view_count = int_or_none(data.get('views'))
+        age_limit = parse_age_limit(data.get('min_age'))
+        thumbnails = [{
+            'url': thumbnail,
+        } for thumbnail in data.get('big_thumbnail_urls', [])]
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'thumbnails': thumbnails,
+            'timestamp': timestamp,
+            'duration': duration,
+            'view_count': view_count,
+            'age_limit': age_limit,
+            'formats': formats,
+        }
+
+
+class VideomoreVideoIE(InfoExtractor):
+    IE_NAME = 'videomore:video'
+    _VALID_URL = r'https?://videomore\.ru/(?:(?:[^/]+/){2})?(?P<id>[^/?#&]+)[/?#&]*$'
+    _TESTS = [{
+        # single video with og:video:iframe
+        'url': 'http://videomore.ru/elki_3',
+        'info_dict': {
+            'id': '364623',
+            'ext': 'flv',
+            'title': 'Ёлки 3',
+            'description': '',
+            'thumbnail': 're:^https?://.*\.jpg',
+            'duration': 5579,
+            'age_limit': 6,
+            'view_count': int,
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }, {
+        # season single serie with og:video:iframe
+        'url': 'http://videomore.ru/poslednii_ment/1_sezon/14_seriya',
+        'only_matching': True,
+    }, {
+        'url': 'http://videomore.ru/sejchas_v_seti/serii_221-240/226_vypusk',
+        'only_matching': True,
+    }, {
+        # single video without og:video:iframe
+        'url': 'http://videomore.ru/marin_i_ego_druzya',
+        'info_dict': {
+            'id': '359073',
+            'ext': 'flv',
+            'title': '1 серия. Здравствуй, Аквавилль!',
+            'description': 'md5:c6003179538b5d353e7bcd5b1372b2d7',
+            'thumbnail': 're:^https?://.*\.jpg',
+            'duration': 754,
+            'age_limit': 6,
+            'view_count': int,
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }]
+
+    @classmethod
+    def suitable(cls, url):
+        return False if VideomoreIE.suitable(url) else super(VideomoreVideoIE, cls).suitable(url)
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, display_id)
+
+        video_url = self._og_search_property(
+            'video:iframe', webpage, 'video url', default=None)
+
+        if not video_url:
+            video_id = self._search_regex(
+                (r'config\s*:\s*["\']https?://videomore\.ru/video/tracks/(\d+)\.xml',
+                 r'track-id=["\'](\d+)',
+                 r'xcnt_product_id\s*=\s*(\d+)'), webpage, 'video id')
+            video_url = 'videomore:%s' % video_id
+
+        return self.url_result(video_url, VideomoreIE.ie_key())
+
+
+class VideomoreSeasonIE(InfoExtractor):
+    IE_NAME = 'videomore:season'
+    _VALID_URL = r'https?://videomore\.ru/(?!embed)(?P<id>[^/]+/[^/?#&]+)[/?#&]*$'
+    _TESTS = [{
+        'url': 'http://videomore.ru/molodezhka/sezon_promo',
+        'info_dict': {
+            'id': 'molodezhka/sezon_promo',
+            'title': 'Молодежка Промо',
+        },
+        'playlist_mincount': 12,
+    }]
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, display_id)
+
+        title = self._og_search_title(webpage)
+
+        entries = [
+            self.url_result(item) for item in re.findall(
+                r'<a[^>]+href="((?:https?:)?//videomore\.ru/%s/[^/]+)"[^>]+class="widget-item-desc"'
+                % display_id, webpage)]
+
+        return self.playlist_result(entries, display_id, title)

From ff18735cb2440dd2e628f0bf497325b50848353b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 29 Dec 2015 23:58:23 +0600
Subject: [PATCH 3529/6219] [extractor/generic] Add support for videomore
 embeds

---
 youtube_dl/extractor/generic.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 70a8d8eb9..d79e1adc9 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -54,6 +54,7 @@ from .snagfilms import SnagFilmsEmbedIE
 from .screenwavemedia import ScreenwaveMediaIE
 from .mtv import MTVServicesEmbeddedIE
 from .pladform import PladformIE
+from .videomore import VideomoreIE
 from .googledrive import GoogleDriveIE
 from .jwplatform import JWPlatformIE
 from .ultimedia import UltimediaIE
@@ -1743,6 +1744,11 @@ class GenericIE(InfoExtractor):
         if pladform_url:
             return self.url_result(pladform_url)
 
+        # Look for Videomore embeds
+        videomore_url = VideomoreIE._extract_url(webpage)
+        if videomore_url:
+            return self.url_result(videomore_url)
+
         # Look for Playwire embeds
         mobj = re.search(
             r'<script[^>]+data-config=(["\'])(?P<url>(?:https?:)?//config\.playwire\.com/.+?)\1', webpage)

From 453fe2a345358064609df9adeee302e19aa636bd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 30 Dec 2015 00:13:00 +0600
Subject: [PATCH 3530/6219] [dramafever] Fix subtitles extraction (Closes
 #8049)

---
 youtube_dl/extractor/dramafever.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/dramafever.py b/youtube_dl/extractor/dramafever.py
index 60ed438f8..b3b21d65f 100644
--- a/youtube_dl/extractor/dramafever.py
+++ b/youtube_dl/extractor/dramafever.py
@@ -108,7 +108,7 @@ class DramaFeverIE(DramaFeverBaseIE):
             if value:
                 subfile = value[0].get('subfile') or value[0].get('new_subfile')
                 if subfile and subfile != 'http://www.dramafever.com/st/':
-                    info['subtitiles'].setdefault('English', []).append({
+                    info.setdefault('subtitles', {}).setdefault('English', []).append({
                         'ext': 'srt',
                         'url': subfile,
                     })

From 69759a5990ad897bdb1af9c44d5c73a2be1e92c6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 30 Dec 2015 00:44:07 +0600
Subject: [PATCH 3531/6219] [videomore] Set IE_NAME

---
 youtube_dl/extractor/videomore.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/videomore.py b/youtube_dl/extractor/videomore.py
index dbc9dbdb7..16078ac19 100644
--- a/youtube_dl/extractor/videomore.py
+++ b/youtube_dl/extractor/videomore.py
@@ -13,6 +13,7 @@ from ..utils import (
 
 
 class VideomoreIE(InfoExtractor):
+    IE_NAME = 'videomore'
     _VALID_URL = r'videomore:(?P<sid>\d+)$|https?://videomore\.ru/(?:(?:embed|[^/]+/[^/]+)/|[^/]+\?.*\btrack_id=)(?P<id>\d+)(?:[/?#&]|\.(?:xml|json)|$)'
     _TESTS = [{
         'url': 'http://videomore.ru/kino_v_detalayah/5_sezon/367617',

From f5bc4b5f95393b5fc83f5c64d44ee78093f9d798 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 30 Dec 2015 23:12:35 +0600
Subject: [PATCH 3532/6219] [options] Prefer --convert-subs spelling

---
 youtube_dl/options.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index c46e136bf..ade58c375 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -752,7 +752,7 @@ def parseOpts(overrideArguments=None):
         metavar='CMD', dest='exec_cmd',
         help='Execute a command on the file after downloading, similar to find\'s -exec syntax. Example: --exec \'adb push {} /sdcard/Music/ && rm {}\'')
     postproc.add_option(
-        '--convert-subtitles', '--convert-subs',
+        '--convert-subs', '--convert-subtitles',
         metavar='FORMAT', dest='convertsubtitles', default=None,
         help='Convert the subtitles to other format (currently supported: srt|ass|vtt)')
 

From 3092fc4035dd3a0841a92227e9bd80b7b9e4d871 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 31 Dec 2015 01:09:21 +0600
Subject: [PATCH 3533/6219] [udemy] Fix typo

---
 youtube_dl/extractor/udemy.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index 59832b1ec..ce7e211fc 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -147,7 +147,7 @@ class UdemyIE(InfoExtractor):
             # Error could possibly mean we are not enrolled in the course
             if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
                 self._enroll_course(webpage, course_id)
-                lecture_id = self._download_lecture(course_id, lecture_id)
+                lecture = self._download_lecture(course_id, lecture_id)
             else:
                 raise
 

From 7109903e6140e01b5d5980fc80c6ef1b6c86797a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 31 Dec 2015 03:10:44 +0600
Subject: [PATCH 3534/6219] [extractor/common] Document chapter and series
 fields

---
 youtube_dl/extractor/common.py | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 655207447..c443daf20 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -200,6 +200,26 @@ class InfoExtractor(object):
     end_time:       Time in seconds where the reproduction should end, as
                     specified in the URL.
 
+    The following fields should only be used when the video belongs to some logical
+    chapter or section:
+
+    chapter:        Name or title of the chapter the video belongs to.
+    chapter_id:     Number or id of the chapter the video belongs to, as an integer
+                    or unicode string.
+
+    The following fields should only be used when the video is an episode of some
+    series or programme:
+
+    series:         Title of the series or programme the video episode belongs to.
+    season:         Title of the season the video episode belongs to.
+    season_id:      Number or id of the season the video episode belongs to, as an
+                    integer or unicode string.
+    episode:        Title of the video episode. Unlike mandatory video title field,
+                    this field should denote the exact title of the video episode
+                    without any kind of decoration.
+    episode_id:     Number or id of the video episode within a season, as an integer
+                    or unicode string.
+
     Unless mentioned otherwise, the fields should be Unicode strings.
 
     Unless mentioned otherwise, None is equivalent to absence of information.

From 4d402db52140458b1b4707c30fb01bb92861a8cc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 31 Dec 2015 03:11:21 +0600
Subject: [PATCH 3535/6219] [udemy] Extract chapter info

---
 youtube_dl/extractor/udemy.py | 25 ++++++++++++++++++++-----
 1 file changed, 20 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index ce7e211fc..41097829d 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -244,10 +244,25 @@ class UdemyCourseIE(UdemyIE):
             'https://www.udemy.com/api-1.1/courses/%s/curriculum' % course_id,
             course_id, 'Downloading course curriculum')
 
-        entries = [
-            self.url_result(
-                'https://www.udemy.com/%s/#/lecture/%s' % (course_path, asset['id']), 'Udemy')
-            for asset in response if asset.get('assetType') or asset.get('asset_type') == 'Video'
-        ]
+        entries = []
+        chapter, chapter_id = None, None
+        for asset in response:
+            asset_type = asset.get('assetType') or asset.get('asset_type')
+            if asset_type == 'Video':
+                asset_id = asset.get('id')
+                if asset_id:
+                    entry = {
+                        '_type': 'url_transparent',
+                        'url': 'https://www.udemy.com/%s/#/lecture/%s' % (course_path, asset['id']),
+                        'ie_key': UdemyIE.ie_key(),
+                    }
+                    if chapter_id:
+                        entry['chapter_id'] = chapter_id
+                    if chapter:
+                        entry['chapter'] = chapter
+                    entries.append(entry)
+            elif asset.get('type') == 'chapter':
+                chapter_id = asset.get('index') or asset.get('object_index')
+                chapter = asset.get('title')
 
         return self.playlist_result(entries, course_id, course_title)

From 7a0b07c719b621b1572120ed3f04870fa5737bfb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 31 Dec 2015 03:13:02 +0600
Subject: [PATCH 3536/6219] [videomore] Extract series info

---
 youtube_dl/extractor/videomore.py | 56 +++++++++++++++++++++++++++++++
 1 file changed, 56 insertions(+)

diff --git a/youtube_dl/extractor/videomore.py b/youtube_dl/extractor/videomore.py
index 16078ac19..3bd96e445 100644
--- a/youtube_dl/extractor/videomore.py
+++ b/youtube_dl/extractor/videomore.py
@@ -23,11 +23,56 @@ class VideomoreIE(InfoExtractor):
             'ext': 'flv',
             'title': 'В гостях Алексей Чумаков и Юлия Ковальчук',
             'description': 'В гостях – лучшие романтические комедии года, «Выживший» Иньярриту и «Стив Джобс» Дэнни Бойла.',
+            'series': 'Кино в деталях',
+            'episode': 'В гостях Алексей Чумаков и Юлия Ковальчук',
+            'episode_id': None,
+            'season': 'Сезон 2015',
+            'season_id': 5,
             'thumbnail': 're:^https?://.*\.jpg',
             'duration': 2910,
             'age_limit': 16,
             'view_count': int,
         },
+    }, {
+        'url': 'http://videomore.ru/embed/259974',
+        'info_dict': {
+            'id': '259974',
+            'ext': 'flv',
+            'title': '80 серия',
+            'description': '«Медведей» ждет решающий матч. Макеев выясняет отношения со Стрельцовым. Парни узнают подробности прошлого Макеева.',
+            'series': 'Молодежка',
+            'episode': '80 серия',
+            'episode_id': 40,
+            'season': '2 сезон',
+            'season_id': 2,
+            'thumbnail': 're:^https?://.*\.jpg',
+            'duration': 2809,
+            'age_limit': 16,
+            'view_count': int,
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }, {
+        'url': 'http://videomore.ru/molodezhka/sezon_promo/341073',
+        'info_dict': {
+            'id': '341073',
+            'ext': 'flv',
+            'title': 'Команда проиграла из-за Бакина?',
+            'description': 'Молодежка 3 сезон скоро',
+            'series': 'Молодежка',
+            'episode': 'Команда проиграла из-за Бакина?',
+            'episode_id': None,
+            'season': 'Промо',
+            'season_id': 99,
+            'thumbnail': 're:^https?://.*\.jpg',
+            'duration': 29,
+            'age_limit': 16,
+            'view_count': int,
+        },
+        'params': {
+            'skip_download': True,
+        },
     }, {
         'url': 'http://videomore.ru/elki_3?track_id=364623',
         'only_matching': True,
@@ -81,10 +126,21 @@ class VideomoreIE(InfoExtractor):
             'url': thumbnail,
         } for thumbnail in data.get('big_thumbnail_urls', [])]
 
+        series = data.get('project_title')
+        episode = data.get('title')
+        episode_id = data.get('episode_of_season') or None
+        season = data.get('season_title')
+        season_id = data.get('season_pos') or None
+
         return {
             'id': video_id,
             'title': title,
             'description': description,
+            'series': series,
+            'episode': episode,
+            'episode_id': episode_id,
+            'season': season,
+            'season_id': season_id,
             'thumbnails': thumbnails,
             'timestamp': timestamp,
             'duration': duration,

From c7932289e73f62d5d73b942175c66510acc2be7e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 30 Dec 2015 23:57:19 +0100
Subject: [PATCH 3537/6219] [cbsnews] Fix extraction of the URL for the
 'RtmpDesktop' format (fixes #8048)

---
 youtube_dl/extractor/cbsnews.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/cbsnews.py b/youtube_dl/extractor/cbsnews.py
index f9a64a0a2..d211ec23b 100644
--- a/youtube_dl/extractor/cbsnews.py
+++ b/youtube_dl/extractor/cbsnews.py
@@ -5,6 +5,7 @@ import re
 import json
 
 from .common import InfoExtractor
+from ..utils import remove_start
 
 
 class CBSNewsIE(InfoExtractor):
@@ -62,6 +63,7 @@ class CBSNewsIE(InfoExtractor):
             uri = item.get('media' + format_id + 'URI')
             if not uri:
                 continue
+            uri = remove_start(uri, '{manifest:none}')
             fmt = {
                 'url': uri,
                 'format_id': format_id,
@@ -70,6 +72,8 @@ class CBSNewsIE(InfoExtractor):
                 play_path = re.sub(
                     r'{slistFilePath}', '',
                     uri.split('<break>')[-1].split('{break}')[-1])
+                play_path = re.sub(
+                    r'{manifest:.+}.*$', '', play_path)
                 fmt.update({
                     'app': 'ondemand?auth=cbs',
                     'play_path': 'mp4:' + play_path,

From 29f36839016e989f4a335fdd51d63f083e26d555 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 31 Dec 2015 05:31:01 +0600
Subject: [PATCH 3538/6219] [espn] Remove broken flag

---
 youtube_dl/extractor/espn.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/espn.py b/youtube_dl/extractor/espn.py
index e6f8f0337..7d758102c 100644
--- a/youtube_dl/extractor/espn.py
+++ b/youtube_dl/extractor/espn.py
@@ -5,7 +5,6 @@ from .common import InfoExtractor
 
 class ESPNIE(InfoExtractor):
     _VALID_URL = r'https?://espn\.go\.com/(?:[^/]+/)*(?P<id>[^/]+)'
-    _WORKING = False
     _TESTS = [{
         'url': 'http://espn.go.com/video/clip?id=10365079',
         'info_dict': {

From df827a983aa94ba99542230c6f7585b36ab49713 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 31 Dec 2015 06:06:36 +0600
Subject: [PATCH 3539/6219] [discovery] Allow https (Closes #8065)

---
 youtube_dl/extractor/discovery.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/discovery.py b/youtube_dl/extractor/discovery.py
index d6723ecf2..507f4bdad 100644
--- a/youtube_dl/extractor/discovery.py
+++ b/youtube_dl/extractor/discovery.py
@@ -9,7 +9,7 @@ from ..compat import compat_str
 
 
 class DiscoveryIE(InfoExtractor):
-    _VALID_URL = r'http://www\.discovery\.com\/[a-zA-Z0-9\-]*/[a-zA-Z0-9\-]*/videos/(?P<id>[a-zA-Z0-9_\-]*)(?:\.htm)?'
+    _VALID_URL = r'https?://www\.discovery\.com\/[a-zA-Z0-9\-]*/[a-zA-Z0-9\-]*/videos/(?P<id>[a-zA-Z0-9_\-]*)(?:\.htm)?'
     _TESTS = [{
         'url': 'http://www.discovery.com/tv-shows/mythbusters/videos/mission-impossible-outtakes.htm',
         'info_dict': {

From d5f6429de87da4bffa0be7703d774681393f1ffb Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 31 Dec 2015 04:02:08 +0100
Subject: [PATCH 3540/6219] [canalplus] improve extraction(fixes #6301)

- extract data from json instead of xml
- fix http format urls
- extract more metadata
- update tests
- make m3u8 and f4m format extraction non fatal
- use m3u8_native implementation
---
 youtube_dl/extractor/canalplus.py | 59 ++++++++++++++++++-------------
 1 file changed, 34 insertions(+), 25 deletions(-)

diff --git a/youtube_dl/extractor/canalplus.py b/youtube_dl/extractor/canalplus.py
index 004372f8d..25b2d4efe 100644
--- a/youtube_dl/extractor/canalplus.py
+++ b/youtube_dl/extractor/canalplus.py
@@ -10,13 +10,14 @@ from ..utils import (
     unified_strdate,
     url_basename,
     qualities,
+    int_or_none,
 )
 
 
 class CanalplusIE(InfoExtractor):
     IE_DESC = 'canalplus.fr, piwiplus.fr and d8.tv'
     _VALID_URL = r'https?://(?:www\.(?P<site>canalplus\.fr|piwiplus\.fr|d8\.tv|itele\.fr)/.*?/(?P<path>.*)|player\.canalplus\.fr/#/(?P<id>[0-9]+))'
-    _VIDEO_INFO_TEMPLATE = 'http://service.canal-plus.com/video/rest/getVideosLiees/%s/%s'
+    _VIDEO_INFO_TEMPLATE = 'http://service.canal-plus.com/video/rest/getVideosLiees/%s/%s?format=json'
     _SITE_ID_MAP = {
         'canalplus.fr': 'cplus',
         'piwiplus.fr': 'teletoon',
@@ -26,10 +27,10 @@ class CanalplusIE(InfoExtractor):
 
     _TESTS = [{
         'url': 'http://www.canalplus.fr/c-emissions/pid1830-c-zapping.html?vid=1263092',
-        'md5': 'b3481d7ca972f61e37420798d0a9d934',
+        'md5': '12164a6f14ff6df8bd628e8ba9b10b78',
         'info_dict': {
             'id': '1263092',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Le Zapping - 13/05/15',
             'description': 'md5:09738c0d06be4b5d06a0940edb0da73f',
             'upload_date': '20150513',
@@ -56,10 +57,10 @@ class CanalplusIE(InfoExtractor):
         'skip': 'videos get deleted after a while',
     }, {
         'url': 'http://www.itele.fr/france/video/aubervilliers-un-lycee-en-colere-111559',
-        'md5': 'f3a46edcdf28006598ffaf5b30e6a2d4',
+        'md5': '38b8f7934def74f0d6f3ba6c036a5f82',
         'info_dict': {
             'id': '1213714',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Aubervilliers : un lycée en colère - Le 11/02/2015 à 06h45',
             'description': 'md5:8216206ec53426ea6321321f3b3c16db',
             'upload_date': '20150211',
@@ -82,15 +83,16 @@ class CanalplusIE(InfoExtractor):
                 webpage, 'video id', group='id')
 
         info_url = self._VIDEO_INFO_TEMPLATE % (site_id, video_id)
-        doc = self._download_xml(info_url, video_id, 'Downloading video XML')
+        video_data = self._download_json(info_url, video_id, 'Downloading video JSON')
 
-        video_info = [video for video in doc if video.find('ID').text == video_id][0]
-        media = video_info.find('MEDIA')
-        infos = video_info.find('INFOS')
+        if isinstance(video_data, list):
+            video_data = [video for video in video_data if video.get('ID') == video_id][0]
+        media = video_data['MEDIA']
+        infos = video_data['INFOS']
 
-        preference = qualities(['MOBILE', 'BAS_DEBIT', 'HAUT_DEBIT', 'HD', 'HLS', 'HDS'])
+        preference = qualities(['MOBILE', 'BAS_DEBIT', 'HAUT_DEBIT', 'HD'])
 
-        fmt_url = next(iter(media.find('VIDEOS'))).text
+        fmt_url = next(iter(media.get('VIDEOS')))
         if '/geo' in fmt_url.lower():
             response = self._request_webpage(
                 HEADRequest(fmt_url), video_id,
@@ -101,35 +103,42 @@ class CanalplusIE(InfoExtractor):
                     expected=True)
 
         formats = []
-        for fmt in media.find('VIDEOS'):
-            format_url = fmt.text
+        for format_id, format_url in media['VIDEOS'].items():
             if not format_url:
                 continue
-            format_id = fmt.tag
             if format_id == 'HLS':
                 formats.extend(self._extract_m3u8_formats(
-                    format_url, video_id, 'mp4', preference=preference(format_id)))
+                    format_url, video_id, 'mp4', 'm3u8_native', m3u8_id=format_id, fatal=False))
             elif format_id == 'HDS':
                 formats.extend(self._extract_f4m_formats(
-                    format_url + '?hdcore=2.11.3', video_id, preference=preference(format_id)))
+                    format_url + '?hdcore=2.11.3', video_id, f4m_id=format_id, fatal=False))
             else:
                 formats.append({
-                    'url': format_url,
+                    # the secret extracted ya function in http://player.canalplus.fr/common/js/canalPlayer.js
+                    'url': format_url + '?secret=pqzerjlsmdkjfoiuerhsdlfknaes',
                     'format_id': format_id,
                     'preference': preference(format_id),
                 })
         self._sort_formats(formats)
 
+        thumbnails = [{
+            'id': image_id,
+            'url': image_url,
+        } for image_id, image_url in media.get('images', {}).items()]
+
+        titrage = infos['TITRAGE']
+
         return {
             'id': video_id,
             'display_id': display_id,
-            'title': '%s - %s' % (infos.find('TITRAGE/TITRE').text,
-                                  infos.find('TITRAGE/SOUS_TITRE').text),
-            'upload_date': unified_strdate(infos.find('PUBLICATION/DATE').text),
-            'thumbnail': media.find('IMAGES/GRAND').text,
-            'description': infos.find('DESCRIPTION').text,
-            'view_count': int(infos.find('NB_VUES').text),
-            'like_count': int(infos.find('NB_LIKES').text),
-            'comment_count': int(infos.find('NB_COMMENTS').text),
+            'title': '%s - %s' % (titrage['TITRE'],
+                                  titrage['SOUS_TITRE']),
+            'upload_date': unified_strdate(infos.get('PUBLICATION', {}).get('DATE')),
+            'thumbnails': thumbnails,
+            'description': infos.get('DESCRIPTION'),
+            'duration': int_or_none(infos.get('DURATION')),
+            'view_count': int_or_none(infos.get('NB_VUES')),
+            'like_count': int_or_none(infos.get('NB_LIKES')),
+            'comment_count': int_or_none(infos.get('NB_COMMENTS')),
             'formats': formats,
         }

From 9787c5f4c89714acd36685b84671e6ea69307a6f Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 31 Dec 2015 12:02:33 +0100
Subject: [PATCH 3541/6219] [fox] Add new extractor(closes #3063)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/fox.py      | 39 ++++++++++++++++++++++++++++++++
 2 files changed, 40 insertions(+)
 create mode 100644 youtube_dl/extractor/fox.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index fb7151443..2063ef633 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -203,6 +203,7 @@ from .flickr import FlickrIE
 from .folketinget import FolketingetIE
 from .footyroom import FootyRoomIE
 from .fourtube import FourTubeIE
+from .fox import FOXIE
 from .foxgay import FoxgayIE
 from .foxnews import FoxNewsIE
 from .foxsports import FoxSportsIE
diff --git a/youtube_dl/extractor/fox.py b/youtube_dl/extractor/fox.py
new file mode 100644
index 000000000..ab97b3196
--- /dev/null
+++ b/youtube_dl/extractor/fox.py
@@ -0,0 +1,39 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import smuggle_url
+
+
+class FOXIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?fox\.com/watch/(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'http://www.fox.com/watch/255180355939/7684182528',
+        'info_dict': {
+            'id': '255180355939',
+            'ext': 'mp4',
+            'title': 'Official Trailer: Gotham',
+            'description': 'Tracing the rise of the great DC Comics Super-Villains and vigilantes, Gotham reveals an entirely new chapter that has never been told.',
+            'duration': 129,
+        },
+        'add_ie': ['ThePlatform'],
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        release_url = self._parse_json(self._search_regex(
+            r'"fox_pdk_player"\s*:\s*({[^}]+?})', webpage, 'fox_pdk_player'),
+            video_id)['release_url'] + '&manifest=m3u'
+
+        return {
+            '_type': 'url_transparent',
+            'ie_key': 'ThePlatform',
+            'url': smuggle_url(release_url, {'force_smil_url': True}),
+            'id': video_id,
+        }

From 80b8b72cb847bd286eea01819bf43fd8bbe6aa92 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 31 Dec 2015 13:36:07 +0100
Subject: [PATCH 3542/6219] [animalplanet] Add new extractor(closes #5303)

---
 youtube_dl/extractor/__init__.py     |  1 +
 youtube_dl/extractor/animalplanet.py | 53 ++++++++++++++++++++++++++++
 2 files changed, 54 insertions(+)
 create mode 100644 youtube_dl/extractor/animalplanet.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 2063ef633..abfabc7da 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -19,6 +19,7 @@ from .aftonbladet import AftonbladetIE
 from .airmozilla import AirMozillaIE
 from .aljazeera import AlJazeeraIE
 from .alphaporno import AlphaPornoIE
+from .animalplanet import AnimalPlanetIE
 from .anitube import AnitubeIE
 from .anysex import AnySexIE
 from .aol import AolIE
diff --git a/youtube_dl/extractor/animalplanet.py b/youtube_dl/extractor/animalplanet.py
new file mode 100644
index 000000000..0cebc8185
--- /dev/null
+++ b/youtube_dl/extractor/animalplanet.py
@@ -0,0 +1,53 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..compat import compat_str
+from ..utils import (
+    parse_duration,
+    parse_iso8601,
+)
+
+
+class AnimalPlanetIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?animalplanet\.com/([^/]+/)*(?P<id>[^/\?#]+)'
+    _TESTS = [{
+        'url': 'http://www.animalplanet.com/tv-shows/i-shouldnt-be-alive/videos/dog-saves-injured-owner/',
+        'info_dict': {
+            'id': '10608',
+            'ext': 'mp4',
+            'title': 'Dog Saves Injured Owner',
+            'description': 'A world class athlete is put to the test when she falls into a canyon and breaks her hip. Her only companion is her dog, Taz, who is on a mission to save her!',
+            'upload_date': '20100410',
+            'timestamp': 1270857727,
+            'duration': 220,
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        }
+    }, {
+        'url': 'http://www.animalplanet.com/longfin-eels-maneaters/',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+
+        video_data = self._parse_json(self._search_regex(
+            r'initialVideoData\s*=\s*({.+?});',
+            webpage, 'initialVideoData'), display_id)['playlist'][0]
+
+        return {
+            'id': compat_str(video_data['id']),
+            'display_id': display_id,
+            'title': video_data['title'],
+            'description': video_data.get('description'),
+            'thumbnail': video_data.get('thumbnailURL'),
+            'duration': parse_duration(video_data.get('video_length')),
+            'timestamp': parse_iso8601(video_data.get('publishedDate')),
+            'formats': self._extract_m3u8_formats(
+                video_data['src'], display_id, 'mp4',
+                'm3u8_native', m3u8_id='hls')
+        }

From 9d46608efabe25454288b5decaa96f9216506770 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 31 Dec 2015 16:35:51 +0100
Subject: [PATCH 3543/6219] [ora] Add new extractor(closes #7732)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/ora.py      | 75 ++++++++++++++++++++++++++++++++
 2 files changed, 76 insertions(+)
 create mode 100644 youtube_dl/extractor/ora.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index abfabc7da..3df80bf00 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -491,6 +491,7 @@ from .ooyala import (
     OoyalaIE,
     OoyalaExternalIE,
 )
+from .ora import OraTVIE
 from .orf import (
     ORFTVthekIE,
     ORFOE1IE,
diff --git a/youtube_dl/extractor/ora.py b/youtube_dl/extractor/ora.py
new file mode 100644
index 000000000..9c4255a2d
--- /dev/null
+++ b/youtube_dl/extractor/ora.py
@@ -0,0 +1,75 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+from .common import InfoExtractor
+from ..compat import compat_urlparse
+from ..utils import (
+    get_element_by_attribute,
+    qualities,
+    unescapeHTML,
+)
+
+
+class OraTVIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?ora\.tv/([^/]+/)*(?P<id>[^/\?#]+)'
+    _TEST = {
+        'url': 'https://www.ora.tv/larrykingnow/2015/12/16/vine-youtube-stars-zach-king-king-bach-on-their-viral-videos-0_36jupg6090pq',
+        'md5': 'fa33717591c631ec93b04b0e330df786',
+        'info_dict': {
+            'id': '50178',
+            'ext': 'mp4',
+            'title': 'Vine & YouTube Stars Zach King & King Bach On Their Viral Videos!',
+            'description': 'md5:ebbc5b1424dd5dba7be7538148287ac1',
+            'duration': 1477,
+        }
+    }
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+
+        video_data = self._search_regex(
+            r'"current"\s*:\s*({[^}]+?})', webpage, 'current video')
+        m3u8_url = self._search_regex(
+            r'"hls_stream"\s*:\s*"([^"]+)', video_data, 'm3u8 url', None)
+        if m3u8_url:
+            formats = self._extract_m3u8_formats(
+                m3u8_url, display_id, 'mp4', 'm3u8_native',
+                m3u8_id='hls', fatal=False)
+            # simular to GameSpotIE
+            m3u8_path = compat_urlparse.urlparse(m3u8_url).path
+            QUALITIES_RE = r'((,[a-z]+\d+)+,?)'
+            available_qualities = self._search_regex(
+                QUALITIES_RE, m3u8_path, 'qualities').strip(',').split(',')
+            http_path = m3u8_path[1:].split('/', 1)[1]
+            http_template = re.sub(QUALITIES_RE, r'%s', http_path)
+            http_template = http_template.replace('.csmil/master.m3u8', '')
+            http_template = compat_urlparse.urljoin(
+                'http://videocdn-pmd.ora.tv/', http_template)
+            preference = qualities(
+                ['mobile400', 'basic400', 'basic600', 'sd900', 'sd1200', 'sd1500', 'hd720', 'hd1080'])
+            for q in available_qualities:
+                formats.append({
+                    'url': http_template % q,
+                    'format_id': q,
+                    'preference': preference(q),
+                })
+            self._sort_formats(formats)
+        else:
+            return self.url_result(self._search_regex(
+                r'"youtube_id"\s*:\s*"([^"]+)', webpage, 'youtube id'), 'Youtube')
+
+        return {
+            'id': self._search_regex(
+                r'"video_id"\s*:\s*(\d+)', video_data, 'video id'),
+            'display_id': display_id,
+            'title': unescapeHTML(self._og_search_title(webpage)),
+            'description': get_element_by_attribute(
+                'class', 'video_txt_decription', webpage),
+            'thumbnail': self._proto_relative_url(self._search_regex(
+                r'"thumb"\s*:\s*"([^"]+)', video_data, 'thumbnail', None)),
+            'duration': int(self._search_regex(
+                r'"duration"\s*:\s*(\d+)', video_data, 'duration')),
+            'formats': formats,
+        }

From 7de81fcc530bfad9023a1a89622c15e4443de5eb Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 31 Dec 2015 16:50:53 +0100
Subject: [PATCH 3544/6219] release 2015.12.31

---
 README.md              |  2 +-
 docs/supportedsites.md | 16 +++++++++++-----
 youtube_dl/version.py  |  2 +-
 3 files changed, 13 insertions(+), 7 deletions(-)

diff --git a/README.md b/README.md
index 3a4707227..4fc83b8e3 100644
--- a/README.md
+++ b/README.md
@@ -400,7 +400,7 @@ which means you can modify it, redistribute it or use it however you like.
                                      downloading, similar to find's -exec
                                      syntax. Example: --exec 'adb push {}
                                      /sdcard/Music/ && rm {}'
-    --convert-subtitles FORMAT       Convert the subtitles to other format
+    --convert-subs FORMAT            Convert the subtitles to other format
                                      (currently supported: srt|ass|vtt)
 
 # CONFIGURATION
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 1646277ec..2297edc6b 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -28,6 +28,7 @@
  - **AlJazeera**
  - **Allocine**
  - **AlphaPorno**
+ - **AnimalPlanet**
  - **anitube.se**
  - **AnySex**
  - **Aparat**
@@ -125,7 +126,7 @@
  - **dailymotion:user**
  - **DailymotionCloud**
  - **daum.net**
- - **daum.net**
+ - **daum.net:clip**
  - **DBTV**
  - **DCN**
  - **dcn:live**
@@ -164,7 +165,7 @@
  - **Eporner**
  - **EroProfile**
  - **Escapist**
- - **ESPN** (Currently broken)
+ - **ESPN**
  - **EsriVideo**
  - **Europa**
  - **EveryonesMixtape**
@@ -181,6 +182,7 @@
  - **Flickr**
  - **Folketinget**: Folketinget (ft.dk; Danish parliament)
  - **FootyRoom**
+ - **FOX**
  - **Foxgay**
  - **FoxNews**: Fox News and Fox Business Video
  - **FoxSports**
@@ -259,7 +261,6 @@
  - **JeuxVideo**
  - **Jove**
  - **jpopsuki.tv**
- - **Jukebox**
  - **JWPlatform**
  - **Kaltura**
  - **KanalPlay**: Kanal 5/9/11 Play
@@ -405,6 +406,7 @@
  - **OnionStudios**
  - **Ooyala**
  - **OoyalaExternal**
+ - **OraTV**
  - **orf:fm4**: radio FM4
  - **orf:iptv**: iptv.ORF.at
  - **orf:oe1**: Radio Österreich 1
@@ -415,7 +417,7 @@
  - **pcmag**
  - **Periscope**: Periscope
  - **PhilharmonieDeParis**: Philharmonie de Paris
- - **Phoenix**
+ - **phoenix.de**
  - **Photobucket**
  - **Pinkbike**
  - **Pladform**
@@ -535,7 +537,8 @@
  - **SportBoxEmbed**
  - **SportDeutschland**
  - **Sportschau**
- - **Srf**
+ - **SRGSSR**
+ - **SRGSSRPlay**: srf.ch, rts.ch, rsi.ch, rtr.ch and swissinfo.ch play sites
  - **SSA**
  - **stanfordoc**: Stanford Open ClassRoom
  - **Steam**
@@ -646,6 +649,9 @@
  - **VideoDetective**
  - **videofy.me**
  - **VideoMega**
+ - **videomore**
+ - **videomore:season**
+ - **videomore:video**
  - **VideoPremium**
  - **VideoTt**: video.tt - Your True Tube
  - **videoweed**: VideoWeed
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index a62baa305..cf6c4d74d 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.12.29'
+__version__ = '2015.12.31'

From c43fda4c1a6cc6d1a1726c3f37c82a0b256625e8 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Sun, 20 Dec 2015 03:17:23 +0100
Subject: [PATCH 3545/6219] [regiotv] Add new extractor (closes #7797)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/regiotv.py  | 61 ++++++++++++++++++++++++++++++++
 2 files changed, 62 insertions(+)
 create mode 100644 youtube_dl/extractor/regiotv.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 3df80bf00..732229c14 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -552,6 +552,7 @@ from .rai import (
 from .rbmaradio import RBMARadioIE
 from .rds import RDSIE
 from .redtube import RedTubeIE
+from .regiotv import RegioTVIE
 from .restudy import RestudyIE
 from .reverbnation import ReverbNationIE
 from .ringtv import RingTVIE
diff --git a/youtube_dl/extractor/regiotv.py b/youtube_dl/extractor/regiotv.py
new file mode 100644
index 000000000..a5c21c085
--- /dev/null
+++ b/youtube_dl/extractor/regiotv.py
@@ -0,0 +1,61 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+from ..utils import (
+    sanitized_Request,
+    xpath_with_ns,
+)
+
+
+class RegioTVIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?regio-tv\.de/video/(?P<id>[0-9]+).html'
+    _TESTS = [
+        {
+            'url': 'http://www.regio-tv.de/video/395808.html',
+            'info_dict': {
+                'id': '395808',
+                'ext': 'mp4',
+                'title': u'Wir in Ludwigsburg',
+                'description': u'Mit unseren zuckers\xfc\xdfen Adventskindern, au\xdferdem besuchen wir die Abendsterne!',
+            }
+        },
+    ]
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+
+        webpage = self._download_webpage(url, video_id)
+        key = self._html_search_regex(r''',key: "(.*?)"''', webpage, 'key')
+
+        title = self._html_search_regex(
+            r'<meta property="og:title" content="\s*(.*?)\s*"\s*/?\s*>',
+            webpage, 'title')
+
+        soapxml = '<?xml version="1.0" encoding="utf-8"?><soap:Envelope xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:soap="http://schemas.xmlsoap.org/soap/envelope/"><soap:Body><getHTML5VideoData xmlns="http://v.telvi.de/"><key xsi:type="xsd:string">%s</key></getHTML5VideoData></soap:Body></soap:Envelope>' % key
+        request = sanitized_Request('http://v.telvi.de/?wsdl', soapxml)
+        request.add_header('Origin', 'http://www.regio-tv.de')
+        request.add_header('Referer', url)
+        video_data = self._download_xml(request, video_id, 'video data')
+
+        NS_MAP = {
+            'xsi': 'http://www.w3.org/2001/XMLSchema-instance',
+            'soap': 'http://schemas.xmlsoap.org/soap/envelope/',
+        }
+
+        url = video_data.find(xpath_with_ns('.//video', NS_MAP)).text
+        thumbnail = video_data.find(xpath_with_ns('.//image', NS_MAP)).text
+
+        description = self._html_search_meta('description', webpage)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'url': url,
+            'thumbnail': thumbnail,
+            'description': description,
+        }

From 34a9da136f1a29889af01815b696672b9dd5b57d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 31 Dec 2015 22:12:47 +0600
Subject: [PATCH 3546/6219] [regiotv] Improve extraction (Closes #7915)

---
 youtube_dl/extractor/regiotv.py | 65 +++++++++++++++++----------------
 1 file changed, 33 insertions(+), 32 deletions(-)

diff --git a/youtube_dl/extractor/regiotv.py b/youtube_dl/extractor/regiotv.py
index a5c21c085..e250a52f0 100644
--- a/youtube_dl/extractor/regiotv.py
+++ b/youtube_dl/extractor/regiotv.py
@@ -1,61 +1,62 @@
-# -*- coding: utf-8 -*-
+# coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 
 from ..utils import (
     sanitized_Request,
+    xpath_text,
     xpath_with_ns,
 )
 
 
 class RegioTVIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?regio-tv\.de/video/(?P<id>[0-9]+).html'
-    _TESTS = [
-        {
-            'url': 'http://www.regio-tv.de/video/395808.html',
-            'info_dict': {
-                'id': '395808',
-                'ext': 'mp4',
-                'title': u'Wir in Ludwigsburg',
-                'description': u'Mit unseren zuckers\xfc\xdfen Adventskindern, au\xdferdem besuchen wir die Abendsterne!',
-            }
-        },
-    ]
+    _VALID_URL = r'https?://(?:www\.)?regio-tv\.de/video/(?P<id>[0-9]+)'
+    _TESTS = [{
+        'url': 'http://www.regio-tv.de/video/395808.html',
+        'info_dict': {
+            'id': '395808',
+            'ext': 'mp4',
+            'title': 'Wir in Ludwigsburg',
+            'description': 'Mit unseren zuckersüßen Adventskindern, außerdem besuchen wir die Abendsterne!',
+        }
+    }, {
+        'url': 'http://www.regio-tv.de/video/395808',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id)
-        key = self._html_search_regex(r''',key: "(.*?)"''', webpage, 'key')
 
-        title = self._html_search_regex(
-            r'<meta property="og:title" content="\s*(.*?)\s*"\s*/?\s*>',
-            webpage, 'title')
+        key = self._search_regex(
+            r'key\s*:\s*(["\'])(?P<key>.+?)\1', webpage, 'key', group='key')
+        title = self._og_search_title(webpage)
 
-        soapxml = '<?xml version="1.0" encoding="utf-8"?><soap:Envelope xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:soap="http://schemas.xmlsoap.org/soap/envelope/"><soap:Body><getHTML5VideoData xmlns="http://v.telvi.de/"><key xsi:type="xsd:string">%s</key></getHTML5VideoData></soap:Body></soap:Envelope>' % key
-        request = sanitized_Request('http://v.telvi.de/?wsdl', soapxml)
-        request.add_header('Origin', 'http://www.regio-tv.de')
-        request.add_header('Referer', url)
-        video_data = self._download_xml(request, video_id, 'video data')
+        SOAP_TEMPLATE = '<?xml version="1.0" encoding="utf-8"?><soap:Envelope xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:soap="http://schemas.xmlsoap.org/soap/envelope/"><soap:Body><{0} xmlns="http://v.telvi.de/"><key xsi:type="xsd:string">{1}</key></{0}></soap:Body></soap:Envelope>'
+
+        request = sanitized_Request(
+            'http://v.telvi.de/',
+            SOAP_TEMPLATE.format('GetHTML5VideoData', key).encode('utf-8'))
+        video_data = self._download_xml(request, video_id, 'Downloading video XML')
 
         NS_MAP = {
             'xsi': 'http://www.w3.org/2001/XMLSchema-instance',
             'soap': 'http://schemas.xmlsoap.org/soap/envelope/',
         }
 
-        url = video_data.find(xpath_with_ns('.//video', NS_MAP)).text
-        thumbnail = video_data.find(xpath_with_ns('.//image', NS_MAP)).text
-
-        description = self._html_search_meta('description', webpage)
+        video_url = xpath_text(
+            video_data, xpath_with_ns('.//video', NS_MAP), 'video url', fatal=True)
+        thumbnail = xpath_text(
+            video_data, xpath_with_ns('.//image', NS_MAP), 'thumbnail')
+        description = self._og_search_description(
+            webpage) or self._html_search_meta('description', webpage)
 
         return {
             'id': video_id,
+            'url': video_url,
             'title': title,
-            'url': url,
-            'thumbnail': thumbnail,
             'description': description,
+            'thumbnail': thumbnail,
         }

From fec040e7549d06e91dae31da53bee855630d0adf Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 31 Dec 2015 17:29:37 +0100
Subject: [PATCH 3547/6219] [discovery] add support for discovery related sites

- investigationdiscovery.com
- discoverylife.com
- animalplanet.com
- ahctv.com
- destinationamerica.com
- sciencechannel.com
- tlc.com
- velocity.com
---
 youtube_dl/extractor/__init__.py     |  1 -
 youtube_dl/extractor/animalplanet.py | 53 ----------------------------
 youtube_dl/extractor/discovery.py    | 39 +++++++++++++++-----
 3 files changed, 30 insertions(+), 63 deletions(-)
 delete mode 100644 youtube_dl/extractor/animalplanet.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 732229c14..bae151e58 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -19,7 +19,6 @@ from .aftonbladet import AftonbladetIE
 from .airmozilla import AirMozillaIE
 from .aljazeera import AlJazeeraIE
 from .alphaporno import AlphaPornoIE
-from .animalplanet import AnimalPlanetIE
 from .anitube import AnitubeIE
 from .anysex import AnySexIE
 from .aol import AolIE
diff --git a/youtube_dl/extractor/animalplanet.py b/youtube_dl/extractor/animalplanet.py
deleted file mode 100644
index 0cebc8185..000000000
--- a/youtube_dl/extractor/animalplanet.py
+++ /dev/null
@@ -1,53 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-from .common import InfoExtractor
-from ..compat import compat_str
-from ..utils import (
-    parse_duration,
-    parse_iso8601,
-)
-
-
-class AnimalPlanetIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?animalplanet\.com/([^/]+/)*(?P<id>[^/\?#]+)'
-    _TESTS = [{
-        'url': 'http://www.animalplanet.com/tv-shows/i-shouldnt-be-alive/videos/dog-saves-injured-owner/',
-        'info_dict': {
-            'id': '10608',
-            'ext': 'mp4',
-            'title': 'Dog Saves Injured Owner',
-            'description': 'A world class athlete is put to the test when she falls into a canyon and breaks her hip. Her only companion is her dog, Taz, who is on a mission to save her!',
-            'upload_date': '20100410',
-            'timestamp': 1270857727,
-            'duration': 220,
-        },
-        'params': {
-            # m3u8 download
-            'skip_download': True,
-        }
-    }, {
-        'url': 'http://www.animalplanet.com/longfin-eels-maneaters/',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        display_id = self._match_id(url)
-        webpage = self._download_webpage(url, display_id)
-
-        video_data = self._parse_json(self._search_regex(
-            r'initialVideoData\s*=\s*({.+?});',
-            webpage, 'initialVideoData'), display_id)['playlist'][0]
-
-        return {
-            'id': compat_str(video_data['id']),
-            'display_id': display_id,
-            'title': video_data['title'],
-            'description': video_data.get('description'),
-            'thumbnail': video_data.get('thumbnailURL'),
-            'duration': parse_duration(video_data.get('video_length')),
-            'timestamp': parse_iso8601(video_data.get('publishedDate')),
-            'formats': self._extract_m3u8_formats(
-                video_data['src'], display_id, 'mp4',
-                'm3u8_native', m3u8_id='hls')
-        }
diff --git a/youtube_dl/extractor/discovery.py b/youtube_dl/extractor/discovery.py
index 507f4bdad..2626d85e3 100644
--- a/youtube_dl/extractor/discovery.py
+++ b/youtube_dl/extractor/discovery.py
@@ -9,7 +9,17 @@ from ..compat import compat_str
 
 
 class DiscoveryIE(InfoExtractor):
-    _VALID_URL = r'https?://www\.discovery\.com\/[a-zA-Z0-9\-]*/[a-zA-Z0-9\-]*/videos/(?P<id>[a-zA-Z0-9_\-]*)(?:\.htm)?'
+    _VALID_URL = r'''(?x)http://(?:www\.)?(?:
+            discovery|
+            investigationdiscovery|
+            discoverylife|
+            animalplanet|
+            ahctv|
+            destinationamerica|
+            sciencechannel|
+            tlc|
+            velocity
+        )\.com/([^/]+/)*(?P<id>[^\./\?#]+)'''
     _TESTS = [{
         'url': 'http://www.discovery.com/tv-shows/mythbusters/videos/mission-impossible-outtakes.htm',
         'info_dict': {
@@ -21,8 +31,8 @@ class DiscoveryIE(InfoExtractor):
                             'don\'t miss Adam moon-walking as Jamie ... behind Jamie\'s'
                             ' back.'),
             'duration': 156,
-            'timestamp': 1303099200,
-            'upload_date': '20110418',
+            'timestamp': 1302032462,
+            'upload_date': '20110405',
         },
         'params': {
             'skip_download': True,  # requires ffmpeg
@@ -33,27 +43,38 @@ class DiscoveryIE(InfoExtractor):
             'id': 'mythbusters-the-simpsons',
             'title': 'MythBusters: The Simpsons',
         },
-        'playlist_count': 9,
+        'playlist_mincount': 10,
+    }, {
+        'url': 'http://www.animalplanet.com/longfin-eels-maneaters/',
+        'info_dict': {
+            'id': '78326',
+            'ext': 'mp4',
+            'title': 'Longfin Eels: Maneaters?',
+            'description': 'Jeremy Wade tests whether or not New Zealand\'s longfin eels are man-eaters by covering himself in fish guts and getting in the water with them.',
+            'upload_date': '20140725',
+            'timestamp': 1406246400,
+            'duration': 116,
+        },
     }]
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-        info = self._download_json(url + '?flat=1', video_id)
+        display_id = self._match_id(url)
+        info = self._download_json(url + '?flat=1', display_id)
 
         video_title = info.get('playlist_title') or info.get('video_title')
 
         entries = [{
             'id': compat_str(video_info['id']),
             'formats': self._extract_m3u8_formats(
-                video_info['src'], video_id, ext='mp4',
+                video_info['src'], display_id, 'mp4', 'm3u8_native', m3u8_id='hls',
                 note='Download m3u8 information for video %d' % (idx + 1)),
             'title': video_info['title'],
             'description': video_info.get('description'),
             'duration': parse_duration(video_info.get('video_length')),
-            'webpage_url': video_info.get('href'),
+            'webpage_url': video_info.get('href') or video_info.get('url'),
             'thumbnail': video_info.get('thumbnailURL'),
             'alt_title': video_info.get('secondary_title'),
             'timestamp': parse_iso8601(video_info.get('publishedDate')),
         } for idx, video_info in enumerate(info['playlist'])]
 
-        return self.playlist_result(entries, video_id, video_title)
+        return self.playlist_result(entries, display_id, video_title)

From b05641ce405ef281b395c26b094208c17141d027 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 31 Dec 2015 18:24:49 +0100
Subject: [PATCH 3548/6219] [discovery] improve _VALID_URL regex

---
 youtube_dl/extractor/discovery.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/discovery.py b/youtube_dl/extractor/discovery.py
index 2626d85e3..ce680a9f3 100644
--- a/youtube_dl/extractor/discovery.py
+++ b/youtube_dl/extractor/discovery.py
@@ -19,7 +19,7 @@ class DiscoveryIE(InfoExtractor):
             sciencechannel|
             tlc|
             velocity
-        )\.com/([^/]+/)*(?P<id>[^\./\?#]+)'''
+        )\.com/(?:[^/]+/)*(?P<id>[^./?#]+)'''
     _TESTS = [{
         'url': 'http://www.discovery.com/tv-shows/mythbusters/videos/mission-impossible-outtakes.htm',
         'info_dict': {

From f1e21efe63bc08abbc5cec0f8d9c3112c580bd48 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 31 Dec 2015 18:33:40 +0100
Subject: [PATCH 3549/6219] [tlc] remove TlcIE

---
 youtube_dl/extractor/__init__.py |  2 +-
 youtube_dl/extractor/tlc.py      | 23 -----------------------
 2 files changed, 1 insertion(+), 24 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index bae151e58..f3f9cd978 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -695,7 +695,7 @@ from .thesixtyone import TheSixtyOneIE
 from .thisamericanlife import ThisAmericanLifeIE
 from .thisav import ThisAVIE
 from .tinypic import TinyPicIE
-from .tlc import TlcIE, TlcDeIE
+from .tlc import TlcDeIE
 from .tmz import (
     TMZIE,
     TMZArticleIE,
diff --git a/youtube_dl/extractor/tlc.py b/youtube_dl/extractor/tlc.py
index d6d038a8d..adc05ed5f 100644
--- a/youtube_dl/extractor/tlc.py
+++ b/youtube_dl/extractor/tlc.py
@@ -4,32 +4,9 @@ import re
 
 from .common import InfoExtractor
 from .brightcove import BrightcoveLegacyIE
-from .discovery import DiscoveryIE
 from ..compat import compat_urlparse
 
 
-class TlcIE(DiscoveryIE):
-    IE_NAME = 'tlc.com'
-    _VALID_URL = r'http://www\.tlc\.com\/[a-zA-Z0-9\-]*/[a-zA-Z0-9\-]*/videos/(?P<id>[a-zA-Z0-9\-]*)(.htm)?'
-
-    # DiscoveryIE has _TESTS
-    _TESTS = [{
-        'url': 'http://www.tlc.com/tv-shows/cake-boss/videos/too-big-to-fly.htm',
-        'info_dict': {
-            'id': '104493',
-            'ext': 'mp4',
-            'title': 'Too Big to Fly',
-            'description': 'Buddy has taken on a high flying task.',
-            'duration': 119,
-            'timestamp': 1393365060,
-            'upload_date': '20140225',
-        },
-        'params': {
-            'skip_download': True,  # requires ffmpef
-        },
-    }]
-
-
 class TlcDeIE(InfoExtractor):
     IE_NAME = 'tlc.de'
     _VALID_URL = r'http://www\.tlc\.de/sendungen/[^/]+/videos/(?P<title>[^/?]+)'

From 9accfed4e724d3048e894c2323b3a478951d670d Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Sun, 20 Dec 2015 15:37:57 +0100
Subject: [PATCH 3550/6219] [pandoratv] Add new extractor (closes #6884)

---
 youtube_dl/extractor/__init__.py  |  1 +
 youtube_dl/extractor/pandoratv.py | 56 +++++++++++++++++++++++++++++++
 2 files changed, 57 insertions(+)
 create mode 100644 youtube_dl/extractor/pandoratv.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index f3f9cd978..6aed59d16 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -497,6 +497,7 @@ from .orf import (
     ORFFM4IE,
     ORFIPTVIE,
 )
+from .pandoratv import PandoraTVIE
 from .parliamentliveuk import ParliamentLiveUKIE
 from .patreon import PatreonIE
 from .pbs import PBSIE
diff --git a/youtube_dl/extractor/pandoratv.py b/youtube_dl/extractor/pandoratv.py
new file mode 100644
index 000000000..a0a0c114a
--- /dev/null
+++ b/youtube_dl/extractor/pandoratv.py
@@ -0,0 +1,56 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+from ..compat import (
+    compat_urlparse,
+)
+from ..utils import (
+    ExtractorError,
+)
+
+
+class PandoraTVIE(InfoExtractor):
+    _VALID_URL = r'http://(?:.+?\.)?channel.pandora.tv/channel/video.ptv\?'
+    _TESTS = [{
+        'url': 'http://jp.channel.pandora.tv/channel/video.ptv?c1=&prgid=53294230&ch_userid=mikakim&ref=main&lot=cate_01_2',
+        'info_dict': {
+            'description': '\u982d\u3092\u64ab\u3067\u3066\u304f\u308c\u308b\uff1f',
+            'ext': 'mp4',
+            'id': '53294230',
+            'title': '\u982d\u3092\u64ab\u3067\u3066\u304f\u308c\u308b\uff1f',
+            'upload_date': '20151218',
+        }
+    }]
+
+
+    def _real_extract(self, url):
+        qs = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
+        video_id = qs.get('prgid', [None])[0]
+        user_id = qs.get('ch_userid', [None])[0]
+        if any(not f for f in (video_id, user_id,)):
+            raise ExtractorError('Invalid URL', expected=True)
+
+        data_url ='http://m.pandora.tv/?c=view&m=viewJsonApi&ch_userid={userid}&prgid={prgid}'.format(userid=user_id,prgid=video_id)
+        data = self._download_json(data_url, video_id)
+        info = data['data']['rows']['vod_play_info']['result']
+
+        formats = []
+        for format_id in sorted([k for k in info if k.startswith('v') and k.endswith('Url') and info[k]]):
+            formats.append({
+                'format_id': format_id,
+                'url': info[format_id],
+                'ext': 'mp4',
+                'height': int(format_id[1:-3]),
+            })
+
+        return {
+            'description': info['body'],
+            'thumbnail': info['thumbnail'],
+            'formats': formats,
+            'id': video_id,
+            'title': info['subject'],
+            'upload_date': info['fid'][:8],
+            'view_count': info['hit'],
+        }

From e4bd63f9c061cbf5c57e7aba29d0f46bffbf05d8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 1 Jan 2016 00:40:27 +0600
Subject: [PATCH 3551/6219] [pandoratv] Improve extraction (Closes #7921)

---
 youtube_dl/extractor/pandoratv.py | 60 ++++++++++++++++++++-----------
 1 file changed, 40 insertions(+), 20 deletions(-)

diff --git a/youtube_dl/extractor/pandoratv.py b/youtube_dl/extractor/pandoratv.py
index a0a0c114a..34cede4c1 100644
--- a/youtube_dl/extractor/pandoratv.py
+++ b/youtube_dl/extractor/pandoratv.py
@@ -2,28 +2,36 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-
 from ..compat import (
+    compat_str,
     compat_urlparse,
 )
 from ..utils import (
     ExtractorError,
+    float_or_none,
+    parse_duration,
+    str_to_int,
 )
 
 
 class PandoraTVIE(InfoExtractor):
-    _VALID_URL = r'http://(?:.+?\.)?channel.pandora.tv/channel/video.ptv\?'
-    _TESTS = [{
+    _VALID_URL = r'https?://(?:.+?\.)?channel\.pandora\.tv/channel/video\.ptv\?'
+    _TEST = {
         'url': 'http://jp.channel.pandora.tv/channel/video.ptv?c1=&prgid=53294230&ch_userid=mikakim&ref=main&lot=cate_01_2',
         'info_dict': {
-            'description': '\u982d\u3092\u64ab\u3067\u3066\u304f\u308c\u308b\uff1f',
-            'ext': 'mp4',
             'id': '53294230',
-            'title': '\u982d\u3092\u64ab\u3067\u3066\u304f\u308c\u308b\uff1f',
+            'ext': 'flv',
+            'title': '頭を撫でてくれる？',
+            'description': '頭を撫でてくれる？',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 39,
             'upload_date': '20151218',
+            'uploader': 'カワイイ動物まとめ',
+            'uploader_id': 'mikakim',
+            'view_count': int,
+            'like_count': int,
         }
-    }]
-
+    }
 
     def _real_extract(self, url):
         qs = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
@@ -32,25 +40,37 @@ class PandoraTVIE(InfoExtractor):
         if any(not f for f in (video_id, user_id,)):
             raise ExtractorError('Invalid URL', expected=True)
 
-        data_url ='http://m.pandora.tv/?c=view&m=viewJsonApi&ch_userid={userid}&prgid={prgid}'.format(userid=user_id,prgid=video_id)
-        data = self._download_json(data_url, video_id)
+        data = self._download_json(
+            'http://m.pandora.tv/?c=view&m=viewJsonApi&ch_userid=%s&prgid=%s'
+            % (user_id, video_id), video_id)
+
         info = data['data']['rows']['vod_play_info']['result']
 
         formats = []
-        for format_id in sorted([k for k in info if k.startswith('v') and k.endswith('Url') and info[k]]):
+        for format_id, format_url in info.items():
+            if not format_url:
+                continue
+            height = self._search_regex(
+                r'^v(\d+)[Uu]rl$', format_id, 'height', default=None)
+            if not height:
+                continue
             formats.append({
-                'format_id': format_id,
-                'url': info[format_id],
-                'ext': 'mp4',
-                'height': int(format_id[1:-3]),
+                'format_id': '%sp' % height,
+                'url': format_url,
+                'height': int(height),
             })
+        self._sort_formats(formats)
 
         return {
-            'description': info['body'],
-            'thumbnail': info['thumbnail'],
-            'formats': formats,
             'id': video_id,
             'title': info['subject'],
-            'upload_date': info['fid'][:8],
-            'view_count': info['hit'],
+            'description': info.get('body'),
+            'thumbnail': info.get('thumbnail') or info.get('poster'),
+            'duration': float_or_none(info.get('runtime'), 1000) or parse_duration(info.get('time')),
+            'upload_date': info['fid'][:8] if isinstance(info.get('fid'), compat_str) else None,
+            'uploader': info.get('nickname'),
+            'uploader_id': info.get('upload_userid'),
+            'view_count': str_to_int(info.get('hit')),
+            'like_count': str_to_int(info.get('likecnt')),
+            'formats': formats,
         }

From 72528252e303a084c4b95ae07c7a7213e53cad8a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 1 Jan 2016 00:42:42 +0600
Subject: [PATCH 3552/6219] [pandoratv] Add IE names

---
 youtube_dl/extractor/pandoratv.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/pandoratv.py b/youtube_dl/extractor/pandoratv.py
index 34cede4c1..8d49f5c4a 100644
--- a/youtube_dl/extractor/pandoratv.py
+++ b/youtube_dl/extractor/pandoratv.py
@@ -15,6 +15,8 @@ from ..utils import (
 
 
 class PandoraTVIE(InfoExtractor):
+    IE_NAME = 'pandora.tv'
+    IE_DESC = '판도라TV'
     _VALID_URL = r'https?://(?:.+?\.)?channel\.pandora\.tv/channel/video\.ptv\?'
     _TEST = {
         'url': 'http://jp.channel.pandora.tv/channel/video.ptv?c1=&prgid=53294230&ch_userid=mikakim&ref=main&lot=cate_01_2',

From 9f9f7664b543f2691a9100920a8b5962d4608bf9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 31 Dec 2015 19:52:48 +0100
Subject: [PATCH 3553/6219] [espn] Update test

---
 youtube_dl/extractor/espn.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/espn.py b/youtube_dl/extractor/espn.py
index 7d758102c..24780b82d 100644
--- a/youtube_dl/extractor/espn.py
+++ b/youtube_dl/extractor/espn.py
@@ -11,7 +11,7 @@ class ESPNIE(InfoExtractor):
             'id': 'FkYWtmazr6Ed8xmvILvKLWjd4QvYZpzG',
             'ext': 'mp4',
             'title': 'dm_140128_30for30Shorts___JudgingJewellv2',
-            'description': '',
+            'description': None,
         },
         'params': {
             # m3u8 download

From 930087f2f69d887d768f234c5dc4563c5ad86ad6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 31 Dec 2015 19:56:54 +0100
Subject: [PATCH 3554/6219] [espn] Support 'intl' videos (#7858)

---
 youtube_dl/extractor/espn.py | 20 ++++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/espn.py b/youtube_dl/extractor/espn.py
index 24780b82d..0a917f9f6 100644
--- a/youtube_dl/extractor/espn.py
+++ b/youtube_dl/extractor/espn.py
@@ -17,6 +17,18 @@ class ESPNIE(InfoExtractor):
             # m3u8 download
             'skip_download': True,
         },
+    }, {
+        # intl video, from http://www.espnfc.us/video/mls-highlights/150/video/2743663/must-see-moments-best-of-the-mls-season
+        'url': 'http://espn.go.com/video/clip?id=2743663',
+        'info_dict': {
+            'id': '50NDFkeTqRHB0nXBOK-RGdSG5YQPuxHg',
+            'ext': 'mp4',
+            'title': 'int_151206_Must_See_Moments_Best_of_MLS_2015_season',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
     }, {
         'url': 'https://espn.go.com/video/iframe/twitter/?cms=espn&id=10365079',
         'only_matching': True,
@@ -43,12 +55,16 @@ class ESPNIE(InfoExtractor):
             r'class="video-play-button"[^>]+data-id="(\d+)',
             webpage, 'video id')
 
+        cms = 'espn'
+        if 'data-source="intl"' in webpage:
+            cms = 'intl'
+        player_url = 'https://espn.go.com/video/iframe/twitter/?id=%s&cms=%s' % (video_id, cms)
         player = self._download_webpage(
-            'https://espn.go.com/video/iframe/twitter/?id=%s' % video_id, video_id)
+            player_url, video_id)
 
         pcode = self._search_regex(
             r'["\']pcode=([^"\']+)["\']', player, 'pcode')
 
         return self.url_result(
-            'ooyalaexternal:espn:%s:%s' % (video_id, pcode),
+            'ooyalaexternal:%s:%s:%s' % (cms, video_id, pcode),
             'OoyalaExternal')

From 69f85952568def9ce13f71f3e8bac65f0c5f5f36 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 31 Dec 2015 20:06:21 +0100
Subject: [PATCH 3555/6219] [espn] Extract better titles

---
 youtube_dl/extractor/espn.py | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/espn.py b/youtube_dl/extractor/espn.py
index 0a917f9f6..3762d8748 100644
--- a/youtube_dl/extractor/espn.py
+++ b/youtube_dl/extractor/espn.py
@@ -1,6 +1,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..utils import remove_end
 
 
 class ESPNIE(InfoExtractor):
@@ -10,7 +11,7 @@ class ESPNIE(InfoExtractor):
         'info_dict': {
             'id': 'FkYWtmazr6Ed8xmvILvKLWjd4QvYZpzG',
             'ext': 'mp4',
-            'title': 'dm_140128_30for30Shorts___JudgingJewellv2',
+            'title': '30 for 30 Shorts: Judging Jewell',
             'description': None,
         },
         'params': {
@@ -23,7 +24,7 @@ class ESPNIE(InfoExtractor):
         'info_dict': {
             'id': '50NDFkeTqRHB0nXBOK-RGdSG5YQPuxHg',
             'ext': 'mp4',
-            'title': 'int_151206_Must_See_Moments_Best_of_MLS_2015_season',
+            'title': 'Must-See Moments: Best of the MLS season',
         },
         'params': {
             # m3u8 download
@@ -65,6 +66,13 @@ class ESPNIE(InfoExtractor):
         pcode = self._search_regex(
             r'["\']pcode=([^"\']+)["\']', player, 'pcode')
 
-        return self.url_result(
-            'ooyalaexternal:%s:%s:%s' % (cms, video_id, pcode),
-            'OoyalaExternal')
+        title = remove_end(
+            self._og_search_title(webpage),
+            '- ESPN Video').strip()
+
+        return {
+            '_type': 'url_transparent',
+            'url': 'ooyalaexternal:%s:%s:%s' % (cms, video_id, pcode),
+            'ie_key': 'OoyalaExternal',
+            'title': title,
+        }

From 91e274546c4492dfa8f216f311356f641859a7cc Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 31 Dec 2015 20:23:48 +0100
Subject: [PATCH 3556/6219] [tvland] Add new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/tvland.py   | 70 ++++++++++++++++++++++++++++++++
 2 files changed, 71 insertions(+)
 create mode 100644 youtube_dl/extractor/tvland.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 6aed59d16..e20d90bac 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -739,6 +739,7 @@ from .tvc import (
     TVCArticleIE,
 )
 from .tvigle import TvigleIE
+from .tvland import TVLandIE
 from .tvp import TvpIE, TvpSeriesIE
 from .tvplay import TVPlayIE
 from .tweakers import TweakersIE
diff --git a/youtube_dl/extractor/tvland.py b/youtube_dl/extractor/tvland.py
new file mode 100644
index 000000000..7f049e076
--- /dev/null
+++ b/youtube_dl/extractor/tvland.py
@@ -0,0 +1,70 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .mtv import MTVIE
+
+
+class TVLandIE(MTVIE):
+    IE_NAME = 'tvland.com'
+    _VALID_URL = r'https?://(?:www\.)?tvland\.com/(?:video-clips|episodes)/(?P<id>[^/?#.]+)'
+    _FEED_URL = 'http://www.tvland.com/feeds/mrss/?uri='
+    _TESTS = [{
+        'url': 'http://www.tvland.com/episodes/hqhps2/everybody-loves-raymond-the-invasion-ep-048',
+        'playlist': [
+            {
+                'md5': '227e9723b9669c05bf51098b10287aa7',
+                'info_dict': {
+                    'id': 'bcbd3a83-3aca-4dca-809b-f78a87dcccdd',
+                    'ext': 'mp4',
+                    'title': 'Everybody Loves Raymond|Everybody Loves Raymond 048 HD, Part 1 of 5',
+                }
+            },
+            {
+                'md5': '9fa2b764ec0e8194fb3ebb01a83df88b',
+                'info_dict': {
+                    'id': 'f4279548-6e13-40dd-92e8-860d27289197',
+                    'ext': 'mp4',
+                    'title': 'Everybody Loves Raymond|Everybody Loves Raymond 048 HD, Part 2 of 5',
+                }
+            },
+            {
+                'md5': 'fde4c3bccd7cc7e3576b338734153cec',
+                'info_dict': {
+                    'id': '664e4a38-53ef-4115-9bc9-d0f789ec6334',
+                    'ext': 'mp4',
+                    'title': 'Everybody Loves Raymond|Everybody Loves Raymond 048 HD, Part 3 of 5',
+                }
+            },
+            {
+                'md5': '247f6780cda6891f2e49b8ae2b10e017',
+                'info_dict': {
+                    'id': '9146ecf5-b15a-4d78-879c-6679b77f4960',
+                    'ext': 'mp4',
+                    'title': 'Everybody Loves Raymond|Everybody Loves Raymond 048 HD, Part 4 of 5',
+                }
+            },
+            {
+                'md5': 'fd269f33256e47bad5eb6c40de089ff6',
+                'info_dict': {
+                    'id': '04334a2e-9a47-4214-a8c2-ae5792e2fab7',
+                    'ext': 'mp4',
+                    'title': 'Everybody Loves Raymond|Everybody Loves Raymond 048 HD, Part 5 of 5',
+                }
+            }
+        ],
+    }, {
+        'url': 'http://www.tvland.com/video-clips/zea2ev/younger-younger--hilary-duff---little-lies',
+        'md5': 'e2c6389401cf485df26c79c247b08713',
+        'info_dict': {
+            'id': 'b8697515-4bbe-4e01-83d5-fa705ce5fa88',
+            'ext': 'mp4',
+            'title': 'Younger|Younger: Hilary Duff - Little Lies',
+            'description': 'md5:7d192f56ca8d958645c83f0de8ef0269'
+        },
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        mgid = self._search_regex(r'data-mgid="([^"]+)"', webpage, 'mgid')
+        return self._get_videos_info_from_url(self._FEED_URL + mgid, video_id)

From 7f9134fb2db5559124a0f658bacb6bad54261043 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 31 Dec 2015 20:52:47 +0100
Subject: [PATCH 3557/6219] [tvland] inherit from MTVServicesInfoExtractor

---
 youtube_dl/extractor/tvland.py | 12 +++---------
 1 file changed, 3 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/tvland.py b/youtube_dl/extractor/tvland.py
index 7f049e076..b73279dec 100644
--- a/youtube_dl/extractor/tvland.py
+++ b/youtube_dl/extractor/tvland.py
@@ -1,13 +1,13 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-from .mtv import MTVIE
+from .mtv import MTVServicesInfoExtractor
 
 
-class TVLandIE(MTVIE):
+class TVLandIE(MTVServicesInfoExtractor):
     IE_NAME = 'tvland.com'
     _VALID_URL = r'https?://(?:www\.)?tvland\.com/(?:video-clips|episodes)/(?P<id>[^/?#.]+)'
-    _FEED_URL = 'http://www.tvland.com/feeds/mrss/?uri='
+    _FEED_URL = 'http://www.tvland.com/feeds/mrss/'
     _TESTS = [{
         'url': 'http://www.tvland.com/episodes/hqhps2/everybody-loves-raymond-the-invasion-ep-048',
         'playlist': [
@@ -62,9 +62,3 @@ class TVLandIE(MTVIE):
             'description': 'md5:7d192f56ca8d958645c83f0de8ef0269'
         },
     }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-        mgid = self._search_regex(r'data-mgid="([^"]+)"', webpage, 'mgid')
-        return self._get_videos_info_from_url(self._FEED_URL + mgid, video_id)

From 0416006a3051b15e4bebbb096960ca4fb8ffd0a9 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Tue, 15 Dec 2015 21:37:47 +0100
Subject: [PATCH 3558/6219] Fix einthusan parser

---
 youtube_dl/extractor/einthusan.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/einthusan.py b/youtube_dl/extractor/einthusan.py
index 5dfea0d39..bc6def65e 100644
--- a/youtube_dl/extractor/einthusan.py
+++ b/youtube_dl/extractor/einthusan.py
@@ -41,9 +41,12 @@ class EinthusanIE(InfoExtractor):
         video_title = self._html_search_regex(
             r'<h1><a class="movie-title".*?>(.*?)</a></h1>', webpage, 'title')
 
-        video_url = self._html_search_regex(
-            r'''(?s)jwplayer\("mediaplayer"\)\.setup\({.*?'file': '([^']+)'.*?}\);''',
-            webpage, 'video url')
+        movieid = self._html_search_regex(
+            r'data-movieid="(.*?)"', webpage, 'movieid')
+
+        location = 'Washington'
+        geturl = 'http://cdn.einthusan.com/geturl/%s/hd/%s' % (movieid, location)
+        video_url = self._download_webpage(geturl, video_id)
 
         description = self._html_search_meta('description', webpage)
         thumbnail = self._html_search_regex(

From b26afec81f408af37cca6298109ca2a59688bf79 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 1 Jan 2016 02:23:03 +0600
Subject: [PATCH 3559/6219] [einthusan] Improve extraction (Closes #7877)

---
 youtube_dl/extractor/einthusan.py | 34 ++++++++++++++++++-------------
 1 file changed, 20 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/einthusan.py b/youtube_dl/extractor/einthusan.py
index bc6def65e..f7339702c 100644
--- a/youtube_dl/extractor/einthusan.py
+++ b/youtube_dl/extractor/einthusan.py
@@ -1,9 +1,12 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
+from ..compat import compat_urlparse
+from ..utils import (
+    remove_start,
+    sanitized_Request,
+)
 
 
 class EinthusanIE(InfoExtractor):
@@ -34,30 +37,33 @@ class EinthusanIE(InfoExtractor):
     ]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        webpage = self._download_webpage(url, video_id)
+        video_id = self._match_id(url)
 
-        video_title = self._html_search_regex(
-            r'<h1><a class="movie-title".*?>(.*?)</a></h1>', webpage, 'title')
+        request = sanitized_Request(url)
+        request.add_header('User-Agent', 'Mozilla/5.0 (Windows NT 5.2; WOW64; rv:43.0) Gecko/20100101 Firefox/43.0')
+        webpage = self._download_webpage(request, video_id)
 
-        movieid = self._html_search_regex(
-            r'data-movieid="(.*?)"', webpage, 'movieid')
+        title = self._html_search_regex(
+            r'<h1><a[^>]+class=["\']movie-title["\'][^>]*>(.+?)</a></h1>',
+            webpage, 'title')
 
-        location = 'Washington'
-        geturl = 'http://cdn.einthusan.com/geturl/%s/hd/%s' % (movieid, location)
-        video_url = self._download_webpage(geturl, video_id)
+        video_id = self._search_regex(
+            r'data-movieid=["\'](\d+)', webpage, 'video id', default=video_id)
+
+        video_url = self._download_webpage(
+            'http://cdn.einthusan.com/geturl/%s/hd/London,Washington,Toronto,Dallas,San,Sydney/'
+            % video_id, video_id)
 
         description = self._html_search_meta('description', webpage)
         thumbnail = self._html_search_regex(
             r'''<a class="movie-cover-wrapper".*?><img src=["'](.*?)["'].*?/></a>''',
             webpage, "thumbnail url", fatal=False)
         if thumbnail is not None:
-            thumbnail = thumbnail.replace('..', 'http://www.einthusan.com')
+            thumbnail = compat_urlparse.urljoin(url, remove_start(thumbnail, '..'))
 
         return {
             'id': video_id,
-            'title': video_title,
+            'title': title,
             'url': video_url,
             'thumbnail': thumbnail,
             'description': description,

From a0d7ede350f13096f94a94f982925ba4ffac91e3 Mon Sep 17 00:00:00 2001
From: j <j@mailb.org>
Date: Tue, 15 Dec 2015 21:37:47 +0100
Subject: [PATCH 3560/6219] Fix einthusan parser

---
 youtube_dl/extractor/einthusan.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/einthusan.py b/youtube_dl/extractor/einthusan.py
index 5dfea0d39..bc6def65e 100644
--- a/youtube_dl/extractor/einthusan.py
+++ b/youtube_dl/extractor/einthusan.py
@@ -41,9 +41,12 @@ class EinthusanIE(InfoExtractor):
         video_title = self._html_search_regex(
             r'<h1><a class="movie-title".*?>(.*?)</a></h1>', webpage, 'title')
 
-        video_url = self._html_search_regex(
-            r'''(?s)jwplayer\("mediaplayer"\)\.setup\({.*?'file': '([^']+)'.*?}\);''',
-            webpage, 'video url')
+        movieid = self._html_search_regex(
+            r'data-movieid="(.*?)"', webpage, 'movieid')
+
+        location = 'Washington'
+        geturl = 'http://cdn.einthusan.com/geturl/%s/hd/%s' % (movieid, location)
+        video_url = self._download_webpage(geturl, video_id)
 
         description = self._html_search_meta('description', webpage)
         thumbnail = self._html_search_regex(

From fec09bf15d632bc1694b32981d1468011e9d7da6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 1 Jan 2016 02:39:00 +0600
Subject: [PATCH 3561/6219] [einthusan] Improve extraction (Closes #7877)

---
 youtube_dl/extractor/einthusan.py | 34 ++++++++++++++++++-------------
 1 file changed, 20 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/einthusan.py b/youtube_dl/extractor/einthusan.py
index bc6def65e..f7339702c 100644
--- a/youtube_dl/extractor/einthusan.py
+++ b/youtube_dl/extractor/einthusan.py
@@ -1,9 +1,12 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
+from ..compat import compat_urlparse
+from ..utils import (
+    remove_start,
+    sanitized_Request,
+)
 
 
 class EinthusanIE(InfoExtractor):
@@ -34,30 +37,33 @@ class EinthusanIE(InfoExtractor):
     ]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        webpage = self._download_webpage(url, video_id)
+        video_id = self._match_id(url)
 
-        video_title = self._html_search_regex(
-            r'<h1><a class="movie-title".*?>(.*?)</a></h1>', webpage, 'title')
+        request = sanitized_Request(url)
+        request.add_header('User-Agent', 'Mozilla/5.0 (Windows NT 5.2; WOW64; rv:43.0) Gecko/20100101 Firefox/43.0')
+        webpage = self._download_webpage(request, video_id)
 
-        movieid = self._html_search_regex(
-            r'data-movieid="(.*?)"', webpage, 'movieid')
+        title = self._html_search_regex(
+            r'<h1><a[^>]+class=["\']movie-title["\'][^>]*>(.+?)</a></h1>',
+            webpage, 'title')
 
-        location = 'Washington'
-        geturl = 'http://cdn.einthusan.com/geturl/%s/hd/%s' % (movieid, location)
-        video_url = self._download_webpage(geturl, video_id)
+        video_id = self._search_regex(
+            r'data-movieid=["\'](\d+)', webpage, 'video id', default=video_id)
+
+        video_url = self._download_webpage(
+            'http://cdn.einthusan.com/geturl/%s/hd/London,Washington,Toronto,Dallas,San,Sydney/'
+            % video_id, video_id)
 
         description = self._html_search_meta('description', webpage)
         thumbnail = self._html_search_regex(
             r'''<a class="movie-cover-wrapper".*?><img src=["'](.*?)["'].*?/></a>''',
             webpage, "thumbnail url", fatal=False)
         if thumbnail is not None:
-            thumbnail = thumbnail.replace('..', 'http://www.einthusan.com')
+            thumbnail = compat_urlparse.urljoin(url, remove_start(thumbnail, '..'))
 
         return {
             'id': video_id,
-            'title': video_title,
+            'title': title,
             'url': video_url,
             'thumbnail': thumbnail,
             'description': description,

From c1e90619bde0c8b3b17d938195cfeb5ed7803125 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 31 Dec 2015 22:10:00 +0100
Subject: [PATCH 3562/6219] [mtv] extract mgid extraction and query building
 into separate methods

---
 youtube_dl/extractor/mtv.py | 21 +++++++++++++--------
 1 file changed, 13 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index d887583e6..e8bb527b8 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -167,14 +167,16 @@ class MTVServicesInfoExtractor(InfoExtractor):
             'description': description,
         }
 
+    def _get_feed_query(self, uri):
+        data = {'uri': uri}
+        if self._LANG:
+            data['lang'] = self._LANG
+        return compat_urllib_parse.urlencode(data)
+
     def _get_videos_info(self, uri):
         video_id = self._id_from_uri(uri)
         feed_url = self._get_feed_url(uri)
-        data = compat_urllib_parse.urlencode({'uri': uri})
-        info_url = feed_url + '?'
-        if self._LANG:
-            info_url += 'lang=%s&' % self._LANG
-        info_url += data
+        info_url = feed_url + '?' + self._get_feed_query(uri)
         return self._get_videos_info_from_url(info_url, video_id)
 
     def _get_videos_info_from_url(self, url, video_id):
@@ -184,9 +186,7 @@ class MTVServicesInfoExtractor(InfoExtractor):
         return self.playlist_result(
             [self._get_video_info(item) for item in idoc.findall('.//item')])
 
-    def _real_extract(self, url):
-        title = url_basename(url)
-        webpage = self._download_webpage(url, title)
+    def _extract_mgid(self, webpage):
         try:
             # the url can be http://media.mtvnservices.com/fb/{mgid}.swf
             # or http://media.mtvnservices.com/{mgid}
@@ -207,7 +207,12 @@ class MTVServicesInfoExtractor(InfoExtractor):
                 'sm4:video:embed', webpage, 'sm4 embed', default='')
             mgid = self._search_regex(
                 r'embed/(mgid:.+?)["\'&?/]', sm4_embed, 'mgid')
+        return mgid
 
+    def _real_extract(self, url):
+        title = url_basename(url)
+        webpage = self._download_webpage(url, title)
+        mgid = self._extract_mgid(webpage)
         videos_info = self._get_videos_info(mgid)
         return videos_info
 

From a0e5beb0fbfa5f300087c18b12ff67fade51d578 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 31 Dec 2015 22:12:05 +0100
Subject: [PATCH 3563/6219] [nick] Add new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/nick.py     | 63 ++++++++++++++++++++++++++++++++
 2 files changed, 64 insertions(+)
 create mode 100644 youtube_dl/extractor/nick.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index e20d90bac..120ed9340 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -441,6 +441,7 @@ from .nhl import (
     NHLNewsIE,
     NHLVideocenterIE,
 )
+from .nick import NickIE
 from .niconico import NiconicoIE, NiconicoPlaylistIE
 from .ninegag import NineGagIE
 from .noco import NocoIE
diff --git a/youtube_dl/extractor/nick.py b/youtube_dl/extractor/nick.py
new file mode 100644
index 000000000..4840368ac
--- /dev/null
+++ b/youtube_dl/extractor/nick.py
@@ -0,0 +1,63 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .mtv import MTVServicesInfoExtractor
+from ..compat import compat_urllib_parse
+
+
+class NickIE(MTVServicesInfoExtractor):
+    IE_NAME = 'nick.com'
+    _VALID_URL = r'https?://(?:www\.)?nick\.com/videos/clip/(?P<id>[^/?#.]+)'
+    _FEED_URL = 'http://udat.mtvnservices.com/service1/dispatch.htm'
+    _TESTS = [{
+        'url': 'http://www.nick.com/videos/clip/alvinnn-and-the-chipmunks-112-full-episode.html',
+        'playlist': [
+            {
+                'md5': '6e5adc1e28253bbb1b28ab05403dd4d4',
+                'info_dict': {
+                    'id': 'be6a17b0-412d-11e5-8ff7-0026b9414f30',
+                    'ext': 'mp4',
+                    'title': 'ALVINNN!!! and The Chipmunks: "Mojo Missing/Who\'s The Animal" S1',
+                    'description': 'Alvin is convinced his mojo was in a cap he gave to a fan, and must find a way to get his hat back before the Chipmunks’ big concert.\nDuring a costume visit to the zoo, Alvin finds himself mistaken for the real Tasmanian devil.',
+
+                }
+            },
+            {
+                'md5': 'd7be441fc53a1d4882fa9508a1e5b3ce',
+                'info_dict': {
+                    'id': 'be6b8f96-412d-11e5-8ff7-0026b9414f30',
+                    'ext': 'mp4',
+                    'title': 'ALVINNN!!! and The Chipmunks: "Mojo Missing/Who\'s The Animal" S2',
+                    'description': 'Alvin is convinced his mojo was in a cap he gave to a fan, and must find a way to get his hat back before the Chipmunks’ big concert.\nDuring a costume visit to the zoo, Alvin finds himself mistaken for the real Tasmanian devil.',
+
+                }
+            },
+            {
+                'md5': 'efffe1728a234b2b0d2f2b343dd1946f',
+                'info_dict': {
+                    'id': 'be6cf7e6-412d-11e5-8ff7-0026b9414f30',
+                    'ext': 'mp4',
+                    'title': 'ALVINNN!!! and The Chipmunks: "Mojo Missing/Who\'s The Animal" S3',
+                    'description': 'Alvin is convinced his mojo was in a cap he gave to a fan, and must find a way to get his hat back before the Chipmunks’ big concert.\nDuring a costume visit to the zoo, Alvin finds himself mistaken for the real Tasmanian devil.',
+                }
+            },
+            {
+                'md5': '1ec6690733ab9f41709e274a1d5c7556',
+                'info_dict': {
+                    'id': 'be6e3354-412d-11e5-8ff7-0026b9414f30',
+                    'ext': 'mp4',
+                    'title': 'ALVINNN!!! and The Chipmunks: "Mojo Missing/Who\'s The Animal" S4',
+                    'description': 'Alvin is convinced his mojo was in a cap he gave to a fan, and must find a way to get his hat back before the Chipmunks’ big concert.\nDuring a costume visit to the zoo, Alvin finds himself mistaken for the real Tasmanian devil.',
+                }
+            },
+        ],
+    }]
+
+    def _get_feed_query(self, uri):
+        return compat_urllib_parse.urlencode({
+            'feed': 'nick_arc_player_prime',
+            'mgid': uri
+        })
+
+    def _extract_mgid(self, webpage):
+        return self._search_regex(r'data-contenturi="([^"]+)', webpage, 'mgid')

From e565cf6048739fdfdfbab19e4b35c37a53865807 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 31 Dec 2015 22:47:18 +0100
Subject: [PATCH 3564/6219] [nextmovie] Add new extractor

---
 youtube_dl/extractor/__init__.py  |  1 +
 youtube_dl/extractor/nextmovie.py | 30 ++++++++++++++++++++++++++++++
 youtube_dl/extractor/nick.py      |  2 +-
 3 files changed, 32 insertions(+), 1 deletion(-)
 create mode 100644 youtube_dl/extractor/nextmovie.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 120ed9340..b3f7059e4 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -434,6 +434,7 @@ from .nextmedia import (
     NextMediaActionNewsIE,
     AppleDailyIE,
 )
+from .nextmovie import NextMovieIE
 from .nfb import NFBIE
 from .nfl import NFLIE
 from .nhl import (
diff --git a/youtube_dl/extractor/nextmovie.py b/youtube_dl/extractor/nextmovie.py
new file mode 100644
index 000000000..657ae77a0
--- /dev/null
+++ b/youtube_dl/extractor/nextmovie.py
@@ -0,0 +1,30 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .mtv import MTVServicesInfoExtractor
+from ..compat import compat_urllib_parse
+
+
+class NextMovieIE(MTVServicesInfoExtractor):
+    IE_NAME = 'nextmovie.com'
+    _VALID_URL = r'https?://(?:www\.)?nextmovie\.com/shows/[^/]+/\d{4}-\d{2}-\d{2}/(?P<id>[^/?#]+)'
+    _FEED_URL = 'http://lite.dextr.mtvi.com/service1/dispatch.htm'
+    _TESTS = [{
+        'url': 'http://www.nextmovie.com/shows/exclusives/2013-03-10/mgid:uma:videolist:nextmovie.com:1715019/',
+        'md5': '09a9199f2f11f10107d04fcb153218aa',
+        'info_dict': {
+            'id': '961726',
+            'ext': 'mp4',
+            'title': 'The Muppets\' Gravity',
+        },
+    }]
+
+    def _get_feed_query(self, uri):
+        return compat_urllib_parse.urlencode({
+            'feed': '1505',
+            'mgid': uri,
+        })
+
+    def _real_extract(self, url):
+        mgid = self._match_id(url)
+        return self._get_videos_info(mgid)
diff --git a/youtube_dl/extractor/nick.py b/youtube_dl/extractor/nick.py
index 4840368ac..b62819ae5 100644
--- a/youtube_dl/extractor/nick.py
+++ b/youtube_dl/extractor/nick.py
@@ -56,7 +56,7 @@ class NickIE(MTVServicesInfoExtractor):
     def _get_feed_query(self, uri):
         return compat_urllib_parse.urlencode({
             'feed': 'nick_arc_player_prime',
-            'mgid': uri
+            'mgid': uri,
         })
 
     def _extract_mgid(self, webpage):

From 034caf70b24da1d1de12b00e9ac5620fb7664220 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 31 Dec 2015 13:05:46 +0800
Subject: [PATCH 3565/6219] [youku] Fix extraction (#8068)

---
 youtube_dl/extractor/youku.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index 3a3432be8..f767fa15f 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -2,6 +2,9 @@
 from __future__ import unicode_literals
 
 import base64
+import random
+import string
+import time
 
 from .common import InfoExtractor
 from ..compat import (
@@ -141,6 +144,11 @@ class YoukuIE(InfoExtractor):
 
         return video_urls_dict
 
+    @staticmethod
+    def get_ysuid():
+        return '%d%s' % (int(time.time()), ''.join([
+            random.choice(string.ascii_letters) for i in range(3)]))
+
     def get_hd(self, fm):
         hd_id_dict = {
             '3gp': '0',
@@ -189,6 +197,8 @@ class YoukuIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
+        self._set_cookie('youku.com', '__ysuid', self.get_ysuid())
+
         def retrieve_data(req_url, note):
             headers = {
                 'Referer': req_url,

From 0d5095fc65d92251f7aeefd9cb504434590cee8b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 1 Jan 2016 15:14:41 +0600
Subject: [PATCH 3566/6219] [ccc] Update _VALID_URL (Closes #8097)

---
 youtube_dl/extractor/ccc.py | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/ccc.py b/youtube_dl/extractor/ccc.py
index 6924eac70..2e0a0b29f 100644
--- a/youtube_dl/extractor/ccc.py
+++ b/youtube_dl/extractor/ccc.py
@@ -12,13 +12,13 @@ from ..utils import (
 
 class CCCIE(InfoExtractor):
     IE_NAME = 'media.ccc.de'
-    _VALID_URL = r'https?://(?:www\.)?media\.ccc\.de/[^?#]+/[^?#/]*?_(?P<id>[0-9]{8,})._[^?#/]*\.html'
+    _VALID_URL = r'https?://(?:www\.)?media\.ccc\.de/v/(?P<id>[^/?#&]+)'
 
-    _TEST = {
-        'url': 'http://media.ccc.de/browse/congress/2013/30C3_-_5443_-_en_-_saal_g_-_201312281830_-_introduction_to_processor_design_-_byterazor.html#video',
+    _TESTS = [{
+        'url': 'https://media.ccc.de/v/30C3_-_5443_-_en_-_saal_g_-_201312281830_-_introduction_to_processor_design_-_byterazor#video',
         'md5': '3a1eda8f3a29515d27f5adb967d7e740',
         'info_dict': {
-            'id': '20131228183',
+            'id': '30C3_-_5443_-_en_-_saal_g_-_201312281830_-_introduction_to_processor_design_-_byterazor',
             'ext': 'mp4',
             'title': 'Introduction to Processor Design',
             'description': 'md5:5ddbf8c734800267f2cee4eab187bc1b',
@@ -26,7 +26,10 @@ class CCCIE(InfoExtractor):
             'view_count': int,
             'upload_date': '20131229',
         }
-    }
+    }, {
+        'url': 'https://media.ccc.de/v/32c3-7368-shopshifting#download',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From c9154514c495bd2259df2250d4cf72a2564a9136 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 1 Jan 2016 15:22:22 +0600
Subject: [PATCH 3567/6219] [ccc] Fix upload date extraction

---
 youtube_dl/extractor/ccc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ccc.py b/youtube_dl/extractor/ccc.py
index 2e0a0b29f..5b549b343 100644
--- a/youtube_dl/extractor/ccc.py
+++ b/youtube_dl/extractor/ccc.py
@@ -46,7 +46,7 @@ class CCCIE(InfoExtractor):
             r"(?s)<p class='description'>(.*?)</p>",
             webpage, 'description', fatal=False)
         upload_date = unified_strdate(self._html_search_regex(
-            r"(?s)<span class='[^']*fa-calendar-o'></span>(.*?)</li>",
+            r"(?s)<span[^>]+class='[^']*fa-calendar-o'[^>]*>(.+?)</span>",
             webpage, 'upload date', fatal=False))
         view_count = int_or_none(self._html_search_regex(
             r"(?s)<span class='[^']*fa-eye'></span>(.*?)</li>",

From 8499d211583f3534c0d0e0f086d471c52ac6c803 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 1 Jan 2016 15:29:42 +0600
Subject: [PATCH 3568/6219] [ccc] Fix description extraction and update test

---
 youtube_dl/extractor/ccc.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/ccc.py b/youtube_dl/extractor/ccc.py
index 5b549b343..45115c838 100644
--- a/youtube_dl/extractor/ccc.py
+++ b/youtube_dl/extractor/ccc.py
@@ -21,10 +21,10 @@ class CCCIE(InfoExtractor):
             'id': '30C3_-_5443_-_en_-_saal_g_-_201312281830_-_introduction_to_processor_design_-_byterazor',
             'ext': 'mp4',
             'title': 'Introduction to Processor Design',
-            'description': 'md5:5ddbf8c734800267f2cee4eab187bc1b',
+            'description': 'md5:80be298773966f66d56cb11260b879af',
             'thumbnail': 're:^https?://.*\.jpg$',
             'view_count': int,
-            'upload_date': '20131229',
+            'upload_date': '20131228',
         }
     }, {
         'url': 'https://media.ccc.de/v/32c3-7368-shopshifting#download',
@@ -43,7 +43,7 @@ class CCCIE(InfoExtractor):
         title = self._html_search_regex(
             r'(?s)<h1>(.*?)</h1>', webpage, 'title')
         description = self._html_search_regex(
-            r"(?s)<p class='description'>(.*?)</p>",
+            r"(?s)<h3>About</h3>(.+?)<h3>",
             webpage, 'description', fatal=False)
         upload_date = unified_strdate(self._html_search_regex(
             r"(?s)<span[^>]+class='[^']*fa-calendar-o'[^>]*>(.+?)</span>",

From 82597f0ec0b8ec208e77d27f62904de7804b914d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 1 Jan 2016 15:41:52 +0600
Subject: [PATCH 3569/6219] [ccc] Extract duration

---
 youtube_dl/extractor/ccc.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/ccc.py b/youtube_dl/extractor/ccc.py
index 45115c838..710e5919c 100644
--- a/youtube_dl/extractor/ccc.py
+++ b/youtube_dl/extractor/ccc.py
@@ -5,6 +5,7 @@ import re
 from .common import InfoExtractor
 from ..utils import (
     int_or_none,
+    parse_duration,
     qualities,
     unified_strdate,
 )
@@ -25,6 +26,7 @@ class CCCIE(InfoExtractor):
             'thumbnail': 're:^https?://.*\.jpg$',
             'view_count': int,
             'upload_date': '20131228',
+            'duration': 3660,
         }
     }, {
         'url': 'https://media.ccc.de/v/32c3-7368-shopshifting#download',
@@ -51,6 +53,9 @@ class CCCIE(InfoExtractor):
         view_count = int_or_none(self._html_search_regex(
             r"(?s)<span class='[^']*fa-eye'></span>(.*?)</li>",
             webpage, 'view count', fatal=False))
+        duration = parse_duration(self._html_search_regex(
+            r'(?s)<span[^>]+class=(["\']).*?fa-clock-o.*?\1[^>]*></span>(?P<duration>.+?)</li',
+            webpage, 'duration', fatal=False, group='duration'))
 
         matches = re.finditer(r'''(?xs)
             <(?:span|div)\s+class='label\s+filetype'>(?P<format>.*?)</(?:span|div)>\s*
@@ -98,5 +103,6 @@ class CCCIE(InfoExtractor):
             'thumbnail': thumbnail,
             'view_count': view_count,
             'upload_date': upload_date,
+            'duration': duration,
             'formats': formats,
         }

From 190ef0798103908d986e07f54323299084726d42 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 1 Jan 2016 12:17:10 +0100
Subject: [PATCH 3570/6219] release 2016.01.01

---
 docs/supportedsites.md | 7 +++++--
 youtube_dl/version.py  | 2 +-
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 2297edc6b..84c166805 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -28,7 +28,6 @@
  - **AlJazeera**
  - **Allocine**
  - **AlphaPorno**
- - **AnimalPlanet**
  - **anitube.se**
  - **AnySex**
  - **Aparat**
@@ -368,11 +367,13 @@
  - **Newstube**
  - **NextMedia**: 蘋果日報
  - **NextMediaActionNews**: 蘋果日報 - 動新聞
+ - **nextmovie.com**
  - **nfb**: National Film Board of Canada
  - **nfl.com**
  - **nhl.com**
  - **nhl.com:news**: NHL news
  - **nhl.com:videocenter**: NHL videocenter category
+ - **nick.com**
  - **niconico**: ニコニコ動画
  - **NiconicoPlaylist**
  - **njoy**: N-JOY
@@ -411,6 +412,7 @@
  - **orf:iptv**: iptv.ORF.at
  - **orf:oe1**: Radio Österreich 1
  - **orf:tvthek**: ORF TVthek
+ - **pandora.tv**: 판도라TV
  - **parliamentlive.tv**: UK parliament videos
  - **Patreon**
  - **pbs**: Public Broadcasting Service (PBS) and member stations: PBS: Public Broadcasting Service, APT - Alabama Public Television (WBIQ), GPB/Georgia Public Broadcasting (WGTV), Mississippi Public Broadcasting (WMPN), Nashville Public Television (WNPT), WFSU-TV (WFSU), WSRE (WSRE), WTCI (WTCI), WPBA/Channel 30 (WPBA), Alaska Public Media (KAKM), Arizona PBS (KAET), KNME-TV/Channel 5 (KNME), Vegas PBS (KLVX), AETN/ARKANSAS ETV NETWORK (KETS), KET (WKLE), WKNO/Channel 10 (WKNO), LPB/LOUISIANA PUBLIC BROADCASTING (WLPB), OETA (KETA), Ozarks Public Television (KOZK), WSIU Public Broadcasting (WSIU), KEET TV (KEET), KIXE/Channel 9 (KIXE), KPBS San Diego (KPBS), KQED (KQED), KVIE Public Television (KVIE), PBS SoCal/KOCE (KOCE), ValleyPBS (KVPT), CONNECTICUT PUBLIC TELEVISION (WEDH), KNPB Channel 5 (KNPB), SOPTV (KSYS), Rocky Mountain PBS (KRMA), KENW-TV3 (KENW), KUED Channel 7 (KUED), Wyoming PBS (KCWC), Colorado Public Television / KBDI 12 (KBDI), KBYU-TV (KBYU), Thirteen/WNET New York (WNET), WGBH/Channel 2 (WGBH), WGBY (WGBY), NJTV Public Media NJ (WNJT), WLIW21 (WLIW), mpt/Maryland Public Television (WMPB), WETA Television and Radio (WETA), WHYY (WHYY), PBS 39 (WLVT), WVPT - Your Source for PBS and More! (WVPT), Howard University Television (WHUT), WEDU PBS (WEDU), WGCU Public Media (WGCU), WPBT2 (WPBT), WUCF TV (WUCF), WUFT/Channel 5 (WUFT), WXEL/Channel 42 (WXEL), WLRN/Channel 17 (WLRN), WUSF Public Broadcasting (WUSF), ETV (WRLK), UNC-TV (WUNC), PBS Hawaii - Oceanic Cable Channel 10 (KHET), Idaho Public Television (KAID), KSPS (KSPS), OPB (KOPB), KWSU/Channel 10 & KTNW/Channel 31 (KWSU), WILL-TV (WILL), Network Knowledge - WSEC/Springfield (WSEC), WTTW11 (WTTW), Iowa Public Television/IPTV (KDIN), Nine Network (KETC), PBS39 Fort Wayne (WFWA), WFYI Indianapolis (WFYI), Milwaukee Public Television (WMVS), WNIN (WNIN), WNIT Public Television (WNIT), WPT (WPNE), WVUT/Channel 22 (WVUT), WEIU/Channel 51 (WEIU), WQPT-TV (WQPT), WYCC PBS Chicago (WYCC), WIPB-TV (WIPB), WTIU (WTIU), CET  (WCET), ThinkTVNetwork (WPTD), WBGU-TV (WBGU), WGVU TV (WGVU), NET1 (KUON), Pioneer Public Television (KWCM), SDPB Television (KUSD), TPT (KTCA), KSMQ (KSMQ), KPTS/Channel 8 (KPTS), KTWU/Channel 11 (KTWU), East Tennessee PBS (WSJK), WCTE-TV (WCTE), WLJT, Channel 11 (WLJT), WOSU TV (WOSU), WOUB/WOUC (WOUB), WVPB (WVPB), WKYU-PBS (WKYU), KERA 13 (KERA), MPBN (WCBB), Mountain Lake PBS (WCFE), NHPTV (WENH), Vermont PBS (WETK), witf (WITF), WQED Multimedia (WQED), WMHT Educational Telecommunications (WMHT), Q-TV (WDCQ), WTVS Detroit Public TV (WTVS), CMU Public Television (WCMU), WKAR-TV (WKAR), WNMU-TV Public TV 13 (WNMU), WDSE - WRPT (WDSE), WGTE TV (WGTE), Lakeland Public Television (KAWE), KMOS-TV - Channels 6.1, 6.2 and 6.3 (KMOS), MontanaPBS (KUSM), KRWG/Channel 22 (KRWG), KACV (KACV), KCOS/Channel 13 (KCOS), WCNY/Channel 24 (WCNY), WNED (WNED), WPBS (WPBS), WSKG Public TV (WSKG), WXXI (WXXI), WPSU (WPSU), WVIA Public Media Studios (WVIA), WTVI (WTVI), Western Reserve PBS (WNEO), WVIZ/PBS ideastream (WVIZ), KCTS 9 (KCTS), Basin PBS (KPBT), KUHT / Channel 8 (KUHT), KLRN (KLRN), KLRU (KLRU), WTJX Channel 12 (WTJX), WCVE PBS (WCVE), KBTC Public Television (KBTC)
@@ -459,6 +461,7 @@
  - **RBMARadio**
  - **RDS**: RDS.ca
  - **RedTube**
+ - **RegioTV**
  - **Restudy**
  - **ReverbNation**
  - **RingTV**
@@ -582,7 +585,6 @@
  - **THVideo**
  - **THVideoPlaylist**
  - **tinypic**: tinypic.com videos
- - **tlc.com**
  - **tlc.de**
  - **TMZ**
  - **TMZArticle**
@@ -611,6 +613,7 @@
  - **TVC**
  - **TVCArticle**
  - **tvigle**: Интернет-телевидение Tvigle.ru
+ - **tvland.com**
  - **tvp.pl**
  - **tvp.pl:Series**
  - **TVPlay**: TV3Play and related services
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index cf6c4d74d..790bd5b3b 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.12.31'
+__version__ = '2016.01.01'

From 32f9036447d1211f9ce0750203d71671f0ee99dc Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 1 Jan 2016 13:28:45 +0100
Subject: [PATCH 3571/6219] [ccc] Add language information to formats

---
 youtube_dl/YoutubeDL.py        | 4 ++++
 youtube_dl/extractor/ccc.py    | 8 ++++++--
 youtube_dl/extractor/common.py | 5 +++--
 3 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 50425b8d7..3b2be3159 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1791,6 +1791,10 @@ class YoutubeDL(object):
         res = ''
         if fdict.get('ext') in ['f4f', 'f4m']:
             res += '(unsupported) '
+        if fdict.get('language'):
+            if res:
+                res += ' '
+            res += '[%s]' % fdict['language']
         if fdict.get('format_note') is not None:
             res += fdict['format_note'] + ' '
         if fdict.get('tbr') is not None:
diff --git a/youtube_dl/extractor/ccc.py b/youtube_dl/extractor/ccc.py
index 710e5919c..e94b1e35b 100644
--- a/youtube_dl/extractor/ccc.py
+++ b/youtube_dl/extractor/ccc.py
@@ -58,11 +58,12 @@ class CCCIE(InfoExtractor):
             webpage, 'duration', fatal=False, group='duration'))
 
         matches = re.finditer(r'''(?xs)
-            <(?:span|div)\s+class='label\s+filetype'>(?P<format>.*?)</(?:span|div)>\s*
+            <(?:span|div)\s+class='label\s+filetype'>(?P<format>[^<]*)</(?:span|div)>\s*
+            <(?:span|div)\s+class='label\s+filetype'>(?P<lang>[^<]*)</(?:span|div)>\s*
             <a\s+download\s+href='(?P<http_url>[^']+)'>\s*
             (?:
                 .*?
-                <a\s+href='(?P<torrent_url>[^']+\.torrent)'
+                <a\s+(?:download\s+)?href='(?P<torrent_url>[^']+\.torrent)'
             )?''', webpage)
         formats = []
         for m in matches:
@@ -70,12 +71,15 @@ class CCCIE(InfoExtractor):
             format_id = self._search_regex(
                 r'.*/([a-z0-9_-]+)/[^/]*$',
                 m.group('http_url'), 'format id', default=None)
+            if format_id:
+                format_id = m.group('lang') + '-' + format_id
             vcodec = 'h264' if 'h264' in format_id else (
                 'none' if format_id in ('mp3', 'opus') else None
             )
             formats.append({
                 'format_id': format_id,
                 'format': format,
+                'language': m.group('lang'),
                 'url': m.group('http_url'),
                 'vcodec': vcodec,
                 'preference': preference(format_id),
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 655207447..2823b1d18 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -108,8 +108,9 @@ class InfoExtractor(object):
                                  -2 or smaller for less than default.
                                  < -1000 to hide the format (if there is
                                     another one which is strictly better)
-                    * language_preference  Is this in the correct requested
-                                 language?
+                    * language   Language code, e.g. "de" or "en-US".
+                    * language_preference  Is this in the language mentioned in
+                                 the URL?
                                  10 if it's what the URL is about,
                                  -1 for default (don't know),
                                  -10 otherwise, other values reserved for now.

From ca227c8698340ad9170698cef81ab4bf4d832a55 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 1 Jan 2016 14:32:00 +0100
Subject: [PATCH 3572/6219] [yahoo] Support pages that use an alias (fixes
 #8084)

---
 youtube_dl/extractor/yahoo.py | 34 ++++++++++++++++++++++++++--------
 1 file changed, 26 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/yahoo.py b/youtube_dl/extractor/yahoo.py
index fca5ddc69..4a492f784 100644
--- a/youtube_dl/extractor/yahoo.py
+++ b/youtube_dl/extractor/yahoo.py
@@ -155,7 +155,16 @@ class YahooIE(InfoExtractor):
                 'description': 'md5:8fc39608213295748e1e289807838c97',
                 'duration': 1646,
             },
-        }
+        }, {
+            # it uses an alias to get the video_id
+            'url': 'https://www.yahoo.com/movies/the-stars-of-daddys-home-have-very-different-212843197.html',
+            'info_dict': {
+                'id': '40eda9c8-8e5f-3552-8745-830f67d0c737',
+                'ext': 'mp4',
+                'title': 'Will Ferrell & Mark Wahlberg Are Pro-Spanking',
+                'description': 'While they play feuding fathers in \'Daddy\'s Home,\' star Will Ferrell & Mark Wahlberg share their true feelings on parenthood.',
+            },
+        },
     ]
 
     def _real_extract(self, url):
@@ -199,13 +208,22 @@ class YahooIE(InfoExtractor):
             r'mediaItems: ({.*?})$', webpage, 'items', flags=re.MULTILINE,
             default=None)
         if items_json is None:
-            CONTENT_ID_REGEXES = [
-                r'YUI\.namespace\("Media"\)\.CONTENT_ID\s*=\s*"([^"]+)"',
-                r'root\.App\.Cache\.context\.videoCache\.curVideo = \{"([^"]+)"',
-                r'"first_videoid"\s*:\s*"([^"]+)"',
-                r'%s[^}]*"ccm_id"\s*:\s*"([^"]+)"' % re.escape(page_id),
-            ]
-            video_id = self._search_regex(CONTENT_ID_REGEXES, webpage, 'content ID')
+            alias = self._search_regex(
+                r'"aliases":{"video":"(.*?)"', webpage, 'alias', default=None)
+            if alias is not None:
+                alias_info = self._download_json(
+                    'https://www.yahoo.com/_td/api/resource/VideoService.videos;video_aliases=["%s"]' % alias,
+                    display_id, 'Downloading alias info')
+                video_id = alias_info[0]['id']
+            else:
+                CONTENT_ID_REGEXES = [
+                    r'YUI\.namespace\("Media"\)\.CONTENT_ID\s*=\s*"([^"]+)"',
+                    r'root\.App\.Cache\.context\.videoCache\.curVideo = \{"([^"]+)"',
+                    r'"first_videoid"\s*:\s*"([^"]+)"',
+                    r'%s[^}]*"ccm_id"\s*:\s*"([^"]+)"' % re.escape(page_id),
+                ]
+                video_id = self._search_regex(
+                    CONTENT_ID_REGEXES, webpage, 'content ID')
         else:
             items = json.loads(items_json)
             info = items['mediaItems']['query']['results']['mediaObj'][0]

From 27bfd4e5266623d9000ba17e6bb2a72eefb0e22d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 1 Jan 2016 20:26:56 +0600
Subject: [PATCH 3573/6219] [extractor/common] Introduce number fields for
 chapters and series

---
 youtube_dl/extractor/common.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index c443daf20..c63157619 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -204,21 +204,21 @@ class InfoExtractor(object):
     chapter or section:
 
     chapter:        Name or title of the chapter the video belongs to.
-    chapter_id:     Number or id of the chapter the video belongs to, as an integer
-                    or unicode string.
+    chapter_number: Number of the chapter the video belongs to, as an integer.
+    chapter_id:     Id of the chapter the video belongs to, as a unicode string.
 
     The following fields should only be used when the video is an episode of some
     series or programme:
 
     series:         Title of the series or programme the video episode belongs to.
     season:         Title of the season the video episode belongs to.
-    season_id:      Number or id of the season the video episode belongs to, as an
-                    integer or unicode string.
+    season_number:  Number of the season the video episode belongs to, as an integer.
+    season_id:      Id of the season the video episode belongs to, as a unicode string.
     episode:        Title of the video episode. Unlike mandatory video title field,
                     this field should denote the exact title of the video episode
                     without any kind of decoration.
-    episode_id:     Number or id of the video episode within a season, as an integer
-                    or unicode string.
+    episode_number: Number of the video episode within a season, as an integer.
+    episode_id:     Id of the video episode, as a unicode string.
 
     Unless mentioned otherwise, the fields should be Unicode strings.
 

From 306c51c66922abb1846640b81c9b7c4d6a570d37 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 1 Jan 2016 20:30:08 +0600
Subject: [PATCH 3574/6219] [videomore] Use number fields for series

---
 youtube_dl/extractor/videomore.py | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/videomore.py b/youtube_dl/extractor/videomore.py
index 3bd96e445..a66d6de23 100644
--- a/youtube_dl/extractor/videomore.py
+++ b/youtube_dl/extractor/videomore.py
@@ -25,9 +25,9 @@ class VideomoreIE(InfoExtractor):
             'description': 'В гостях – лучшие романтические комедии года, «Выживший» Иньярриту и «Стив Джобс» Дэнни Бойла.',
             'series': 'Кино в деталях',
             'episode': 'В гостях Алексей Чумаков и Юлия Ковальчук',
-            'episode_id': None,
+            'episode_number': None,
             'season': 'Сезон 2015',
-            'season_id': 5,
+            'season_number': 5,
             'thumbnail': 're:^https?://.*\.jpg',
             'duration': 2910,
             'age_limit': 16,
@@ -42,9 +42,9 @@ class VideomoreIE(InfoExtractor):
             'description': '«Медведей» ждет решающий матч. Макеев выясняет отношения со Стрельцовым. Парни узнают подробности прошлого Макеева.',
             'series': 'Молодежка',
             'episode': '80 серия',
-            'episode_id': 40,
+            'episode_number': 40,
             'season': '2 сезон',
-            'season_id': 2,
+            'season_number': 2,
             'thumbnail': 're:^https?://.*\.jpg',
             'duration': 2809,
             'age_limit': 16,
@@ -62,9 +62,9 @@ class VideomoreIE(InfoExtractor):
             'description': 'Молодежка 3 сезон скоро',
             'series': 'Молодежка',
             'episode': 'Команда проиграла из-за Бакина?',
-            'episode_id': None,
+            'episode_number': None,
             'season': 'Промо',
-            'season_id': 99,
+            'season_number': 99,
             'thumbnail': 're:^https?://.*\.jpg',
             'duration': 29,
             'age_limit': 16,
@@ -128,9 +128,9 @@ class VideomoreIE(InfoExtractor):
 
         series = data.get('project_title')
         episode = data.get('title')
-        episode_id = data.get('episode_of_season') or None
+        episode_number = int_or_none(data.get('episode_of_season') or None)
         season = data.get('season_title')
-        season_id = data.get('season_pos') or None
+        season_number = int_or_none(data.get('season_pos') or None)
 
         return {
             'id': video_id,
@@ -138,9 +138,9 @@ class VideomoreIE(InfoExtractor):
             'description': description,
             'series': series,
             'episode': episode,
-            'episode_id': episode_id,
+            'episode_number': episode_number,
             'season': season,
-            'season_id': season_id,
+            'season_number': season_number,
             'thumbnails': thumbnails,
             'timestamp': timestamp,
             'duration': duration,

From 5bafcf65255261c2dc6482de6b1e545e5de1cbee Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 1 Jan 2016 20:34:29 +0600
Subject: [PATCH 3575/6219] [udemy] Use chapter_number

---
 youtube_dl/extractor/udemy.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index 41097829d..1df636779 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -245,7 +245,7 @@ class UdemyCourseIE(UdemyIE):
             course_id, 'Downloading course curriculum')
 
         entries = []
-        chapter, chapter_id = None, None
+        chapter, chapter_number = None, None
         for asset in response:
             asset_type = asset.get('assetType') or asset.get('asset_type')
             if asset_type == 'Video':
@@ -256,13 +256,13 @@ class UdemyCourseIE(UdemyIE):
                         'url': 'https://www.udemy.com/%s/#/lecture/%s' % (course_path, asset['id']),
                         'ie_key': UdemyIE.ie_key(),
                     }
-                    if chapter_id:
-                        entry['chapter_id'] = chapter_id
+                    if chapter_number:
+                        entry['chapter_number'] = chapter_number
                     if chapter:
                         entry['chapter'] = chapter
                     entries.append(entry)
             elif asset.get('type') == 'chapter':
-                chapter_id = asset.get('index') or asset.get('object_index')
+                chapter_number = asset.get('index') or asset.get('object_index')
                 chapter = asset.get('title')
 
         return self.playlist_result(entries, course_id, course_title)

From 054479754c661f40140f5db7f58916a0006c6b10 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 1 Jan 2016 21:03:16 +0100
Subject: [PATCH 3576/6219] [revision3] Add new extractor(closes #6388)

- revision3.com
- testtube.com
- animalist.com
---
 youtube_dl/extractor/__init__.py  |   1 +
 youtube_dl/extractor/revision3.py | 103 ++++++++++++++++++++++++++++++
 2 files changed, 104 insertions(+)
 create mode 100644 youtube_dl/extractor/revision3.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index b3f7059e4..7adce5499 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -557,6 +557,7 @@ from .redtube import RedTubeIE
 from .regiotv import RegioTVIE
 from .restudy import RestudyIE
 from .reverbnation import ReverbNationIE
+from .revision3 import Revision3IE
 from .ringtv import RingTVIE
 from .ro220 import Ro220IE
 from .rottentomatoes import RottenTomatoesIE
diff --git a/youtube_dl/extractor/revision3.py b/youtube_dl/extractor/revision3.py
new file mode 100644
index 000000000..25fe4ef16
--- /dev/null
+++ b/youtube_dl/extractor/revision3.py
@@ -0,0 +1,103 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..compat import compat_str
+from ..utils import (
+    int_or_none,
+    parse_iso8601,
+    unescapeHTML,
+)
+
+
+class Revision3IE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?(?P<domain>(?:revision3|testtube|animalist)\.com)/(?P<id>[^/]+(?:/[^/?#]+)?)'
+    _TESTS = [{
+        'url': 'http://www.revision3.com/technobuffalo/5-google-predictions-for-2016',
+        'md5': 'd94a72d85d0a829766de4deb8daaf7df',
+        'info_dict': {
+            'id': '73034',
+            'ext': 'webm',
+            'title': '5 Google Predictions for 2016',
+            'description': 'Google had a great 2015, but it\'s already time to look ahead. Here are our five predictions for 2016.',
+            'upload_date': '20151228',
+            'timestamp': 1451325600,
+            'duration': 187,
+        }
+    }, {
+        'url': 'http://testtube.com/brainstuff',
+        'info_dict': {
+            'id': '251',
+            'title': 'BrainStuff',
+            'description': 'Whether the topic is popcorn or particle physics, you can count on the HowStuffWorks team to explore-and explain-the everyday science in the world around us on BrainStuff.',
+        },
+        'playlist_mincount': 93,
+    }]
+    _PAGE_DATA_TEMPLATE = 'http://www.%s/apiProxy/ddn/%s?domain=%s'
+    _API_KEY = 'ba9c741bce1b9d8e3defcc22193f3651b8867e62'
+
+    def _real_extract(self, url):
+        domain, display_id = re.match(self._VALID_URL, url).groups()
+        page_info = self._download_json(
+            self._PAGE_DATA_TEMPLATE % (domain, display_id, domain), display_id)
+
+        if page_info['data']['type'] == 'episode':
+            episode_data = page_info['data']
+            video_id = compat_str(episode_data['video']['data']['id'])
+            video_data = self._download_json(
+                'http://revision3.com/api/getPlaylist.json?api_key=%s&codecs=h264,vp8,theora&video_id=%s' % (self._API_KEY, video_id),
+                video_id)['items'][0]
+
+            formats = []
+            for media_type, media in video_data['media'].items():
+                for quality_id, quality in media.items():
+                    if quality_id == 'hls':
+                        formats.extend(self._extract_m3u8_formats(
+                            quality['url'], video_id, 'mp4',
+                            'm3u8_native', m3u8_id='hls', fatal=False))
+                    else:
+                        formats.append({
+                            'url': quality['url'],
+                            'format_id': '%s-%s' % (media_type, quality_id),
+                            'tbr': int_or_none(quality.get('bitrate')),
+                        })
+            self._sort_formats(formats)
+
+            thumbnails = [{
+                'url': image_url,
+                'id': image_id,
+            } for image_id, image_url in video_data.get('images', {}).items()]
+
+            return {
+                'id': video_id,
+                'title': unescapeHTML(video_data['title']),
+                'description': unescapeHTML(video_data.get('summary')),
+                'timestamp': parse_iso8601(episode_data.get('publishTime'), ' '),
+                'author': episode_data.get('author'),
+                'duration': int_or_none(video_data.get('duration')),
+                'thumbnails': thumbnails,
+                'formats': formats,
+            }
+        else:
+            show_data = page_info['show']['data']
+            episodes_data = page_info['episodes']['data']
+            num_episodes = page_info['meta']['totalEpisodes']
+            processed_episodes = 0
+            entries = []
+            page_num = 1
+            while True:
+                entries.extend([self.url_result(
+                    url + '/%s' % episode['slug']) for episode in episodes_data])
+                processed_episodes += len(episodes_data)
+                if processed_episodes == num_episodes:
+                    break
+                page_num += 1
+                episodes_data = self._download_json(self._PAGE_DATA_TEMPLATE % (
+                    domain, display_id + '/' + compat_str(page_num), domain),
+                    display_id)['episodes']['data']
+
+            return self.playlist_result(
+                entries, compat_str(show_data['id']),
+                show_data.get('name'), show_data.get('summary'))

From 8af2804a5d09e08d8916e7a7d2eab40864e9ecd8 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 1 Jan 2016 21:53:19 +0100
Subject: [PATCH 3577/6219] [testtube] Remove Extractor

---
 youtube_dl/extractor/revision3.py | 30 +++++++++--
 youtube_dl/extractor/testtube.py  | 90 -------------------------------
 2 files changed, 27 insertions(+), 93 deletions(-)
 delete mode 100644 youtube_dl/extractor/testtube.py

diff --git a/youtube_dl/extractor/revision3.py b/youtube_dl/extractor/revision3.py
index 25fe4ef16..b1b8800b9 100644
--- a/youtube_dl/extractor/revision3.py
+++ b/youtube_dl/extractor/revision3.py
@@ -9,6 +9,7 @@ from ..utils import (
     int_or_none,
     parse_iso8601,
     unescapeHTML,
+    qualities,
 )
 
 
@@ -19,12 +20,15 @@ class Revision3IE(InfoExtractor):
         'md5': 'd94a72d85d0a829766de4deb8daaf7df',
         'info_dict': {
             'id': '73034',
+            'display_id': 'technobuffalo/5-google-predictions-for-2016',
             'ext': 'webm',
             'title': '5 Google Predictions for 2016',
             'description': 'Google had a great 2015, but it\'s already time to look ahead. Here are our five predictions for 2016.',
             'upload_date': '20151228',
             'timestamp': 1451325600,
             'duration': 187,
+            'uploader': 'TechnoBuffalo',
+            'uploader_id': 'technobuffalo',
         }
     }, {
         'url': 'http://testtube.com/brainstuff',
@@ -34,6 +38,20 @@ class Revision3IE(InfoExtractor):
             'description': 'Whether the topic is popcorn or particle physics, you can count on the HowStuffWorks team to explore-and explain-the everyday science in the world around us on BrainStuff.',
         },
         'playlist_mincount': 93,
+    }, {
+        'url': 'https://testtube.com/dnews/5-weird-ways-plants-can-eat-animals?utm_source=FB&utm_medium=DNews&utm_campaign=DNewsSocial',
+        'info_dict': {
+            'id': '60163',
+            'display_id': 'dnews/5-weird-ways-plants-can-eat-animals',
+            'duration': 275,
+            'ext': 'webm',
+            'title': '5 Weird Ways Plants Can Eat Animals',
+            'description': 'Why have some plants evolved to eat meat?',
+            'upload_date': '20150120',
+            'timestamp': 1421763300,
+            'uploader': 'DNews',
+            'uploader_id': 'dnews',
+        },
     }]
     _PAGE_DATA_TEMPLATE = 'http://www.%s/apiProxy/ddn/%s?domain=%s'
     _API_KEY = 'ba9c741bce1b9d8e3defcc22193f3651b8867e62'
@@ -51,7 +69,7 @@ class Revision3IE(InfoExtractor):
                 video_id)['items'][0]
 
             formats = []
-            for media_type, media in video_data['media'].items():
+            for vcodec, media in video_data['media'].items():
                 for quality_id, quality in media.items():
                     if quality_id == 'hls':
                         formats.extend(self._extract_m3u8_formats(
@@ -60,22 +78,28 @@ class Revision3IE(InfoExtractor):
                     else:
                         formats.append({
                             'url': quality['url'],
-                            'format_id': '%s-%s' % (media_type, quality_id),
+                            'format_id': '%s-%s' % (vcodec, quality_id),
                             'tbr': int_or_none(quality.get('bitrate')),
+                            'vcodec': vcodec,
                         })
             self._sort_formats(formats)
 
+            preference = qualities(['mini', 'small', 'medium', 'large'])
             thumbnails = [{
                 'url': image_url,
                 'id': image_id,
+                'preference': preference(image_id)
             } for image_id, image_url in video_data.get('images', {}).items()]
 
             return {
                 'id': video_id,
+                'display_id': display_id,
                 'title': unescapeHTML(video_data['title']),
                 'description': unescapeHTML(video_data.get('summary')),
                 'timestamp': parse_iso8601(episode_data.get('publishTime'), ' '),
                 'author': episode_data.get('author'),
+                'uploader': video_data.get('show', {}).get('name'),
+                'uploader_id': video_data.get('show', {}).get('slug'),
                 'duration': int_or_none(video_data.get('duration')),
                 'thumbnails': thumbnails,
                 'formats': formats,
@@ -89,7 +113,7 @@ class Revision3IE(InfoExtractor):
             page_num = 1
             while True:
                 entries.extend([self.url_result(
-                    url + '/%s' % episode['slug']) for episode in episodes_data])
+                    'http://%s/%s/%s' % (domain, display_id, episode['slug'])) for episode in episodes_data])
                 processed_episodes += len(episodes_data)
                 if processed_episodes == num_episodes:
                     break
diff --git a/youtube_dl/extractor/testtube.py b/youtube_dl/extractor/testtube.py
deleted file mode 100644
index 26655d690..000000000
--- a/youtube_dl/extractor/testtube.py
+++ /dev/null
@@ -1,90 +0,0 @@
-from __future__ import unicode_literals
-
-from .common import InfoExtractor
-from ..utils import (
-    int_or_none,
-    qualities,
-)
-
-
-class TestTubeIE(InfoExtractor):
-    _VALID_URL = r'https?://testtube\.com/[^/?#]+/(?P<id>[^/?#]+)'
-    _TESTS = [{
-        'url': 'https://testtube.com/dnews/5-weird-ways-plants-can-eat-animals?utm_source=FB&utm_medium=DNews&utm_campaign=DNewsSocial',
-        'info_dict': {
-            'id': '60163',
-            'display_id': '5-weird-ways-plants-can-eat-animals',
-            'duration': 275,
-            'ext': 'webm',
-            'title': '5 Weird Ways Plants Can Eat Animals',
-            'description': 'Why have some plants evolved to eat meat?',
-            'thumbnail': 're:^https?://.*\.jpg$',
-            'uploader': 'DNews',
-            'uploader_id': 'dnews',
-        },
-    }, {
-        'url': 'https://testtube.com/iflscience/insane-jet-ski-flipping',
-        'info_dict': {
-            'id': 'fAGfJ4YjVus',
-            'ext': 'mp4',
-            'title': 'Flipping Jet-Ski Skills | Outrageous Acts of Science',
-            'uploader': 'Science Channel',
-            'uploader_id': 'ScienceChannel',
-            'upload_date': '20150203',
-            'description': 'md5:e61374030015bae1d2e22f096d4769d6',
-        }
-    }]
-
-    def _real_extract(self, url):
-        display_id = self._match_id(url)
-
-        webpage = self._download_webpage(url, display_id)
-
-        youtube_url = self._html_search_regex(
-            r'<iframe[^>]+src="((?:https?:)?//www.youtube.com/embed/[^"]+)"',
-            webpage, 'youtube iframe', default=None)
-        if youtube_url:
-            return self.url_result(youtube_url, 'Youtube', video_id=display_id)
-
-        video_id = self._search_regex(
-            r"player\.loadRevision3Item\('video_id',\s*([0-9]+)\);",
-            webpage, 'video ID')
-
-        all_info = self._download_json(
-            'https://testtube.com/api/getPlaylist.json?api_key=ba9c741bce1b9d8e3defcc22193f3651b8867e62&codecs=h264,vp8,theora&video_id=%s' % video_id,
-            video_id)
-        info = all_info['items'][0]
-
-        formats = []
-        for vcodec, fdatas in info['media'].items():
-            for name, fdata in fdatas.items():
-                formats.append({
-                    'format_id': '%s-%s' % (vcodec, name),
-                    'url': fdata['url'],
-                    'vcodec': vcodec,
-                    'tbr': fdata.get('bitrate'),
-                })
-        self._sort_formats(formats)
-
-        duration = int_or_none(info.get('duration'))
-        images = info.get('images')
-        thumbnails = None
-        preference = qualities(['mini', 'small', 'medium', 'large'])
-        if images:
-            thumbnails = [{
-                'id': thumbnail_id,
-                'url': img_url,
-                'preference': preference(thumbnail_id)
-            } for thumbnail_id, img_url in images.items()]
-
-        return {
-            'id': video_id,
-            'display_id': display_id,
-            'title': info['title'],
-            'description': info.get('summary'),
-            'thumbnails': thumbnails,
-            'uploader': info.get('show', {}).get('name'),
-            'uploader_id': info.get('show', {}).get('slug'),
-            'duration': duration,
-            'formats': formats,
-        }

From 94de6cf59cb1933e395775f9ffca4fa311adb0dc Mon Sep 17 00:00:00 2001
From: pingtux <pingtux@users.noreply.github.com>
Date: Sat, 2 Jan 2016 01:35:09 +0100
Subject: [PATCH 3578/6219] Remove testtube import

Extractor got deleted in remitamine/youtube-dl@8af2804
---
 youtube_dl/extractor/__init__.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 7adce5499..4c7e5223d 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -687,7 +687,6 @@ from .telemb import TeleMBIE
 from .teletask import TeleTaskIE
 from .tenplay import TenPlayIE
 from .testurl import TestURLIE
-from .testtube import TestTubeIE
 from .tf1 import TF1IE
 from .theintercept import TheInterceptIE
 from .theonion import TheOnionIE

From 76a353c9e5511a522b5331cb3f74d4215341791c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 2 Jan 2016 07:44:30 +0600
Subject: [PATCH 3579/6219] [ruutu] Fix extraction (Closes #8107)

---
 youtube_dl/extractor/ruutu.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ruutu.py b/youtube_dl/extractor/ruutu.py
index 41fddc375..ffea438cc 100644
--- a/youtube_dl/extractor/ruutu.py
+++ b/youtube_dl/extractor/ruutu.py
@@ -75,9 +75,12 @@ class RuutuIE(InfoExtractor):
                         preference = -1 if proto == 'rtmp' else 1
                         label = child.get('label')
                         tbr = int_or_none(child.get('bitrate'))
+                        format_id = '%s-%s' % (proto, label if label else tbr) if label or tbr else proto
+                        if not self._is_valid_url(video_url, video_id, format_id):
+                            continue
                         width, height = [int_or_none(x) for x in child.get('resolution', 'x').split('x')[:2]]
                         formats.append({
-                            'format_id': '%s-%s' % (proto, label if label else tbr),
+                            'format_id': format_id,
                             'url': video_url,
                             'width': width,
                             'height': height,

From f20a11ed257f21a1c658f827e0c2129a72582adf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 2 Jan 2016 19:22:39 +0600
Subject: [PATCH 3580/6219] [bbccouk] Extend _VALID_URL (Closes #8116)

---
 youtube_dl/extractor/bbc.py | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 923273fb2..5c621a32b 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -23,7 +23,17 @@ class BBCCoUkIE(InfoExtractor):
     IE_NAME = 'bbc.co.uk'
     IE_DESC = 'BBC iPlayer'
     _ID_REGEX = r'[pb][\da-z]{7}'
-    _VALID_URL = r'https?://(?:www\.)?bbc\.co\.uk/(?:(?:programmes/(?!articles/)|iplayer(?:/[^/]+)?/(?:episode/|playlist/))|music/clips[/#])(?P<id>%s)' % _ID_REGEX
+    _VALID_URL = r'''(?x)
+                    https?://
+                        (?:www\.)?bbc\.co\.uk/
+                        (?:
+                            programmes/(?!articles/)|
+                            iplayer(?:/[^/]+)?/(?:episode/|playlist/)|
+                            music/clips[/#]|
+                            radio/player/
+                        )
+                        (?P<id>%s)
+                    ''' % _ID_REGEX
 
     _MEDIASELECTOR_URLS = [
         # Provides HQ HLS streams with even better quality that pc mediaset but fails
@@ -193,6 +203,9 @@ class BBCCoUkIE(InfoExtractor):
         }, {
             'url': 'http://www.bbc.co.uk/iplayer/cbeebies/episode/b0480276/bing-14-atchoo',
             'only_matching': True,
+        }, {
+            'url': 'http://www.bbc.co.uk/radio/player/p03cchwf',
+            'only_matching': True,
         }
     ]
 

From c579c5e967cd223baa2c1f593b6ed65ac8643e42 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 2 Jan 2016 21:31:02 +0800
Subject: [PATCH 3581/6219] [baidu] Cleanups

---
 youtube_dl/extractor/baidu.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/youtube_dl/extractor/baidu.py b/youtube_dl/extractor/baidu.py
index 84fab551b..734b50d1d 100644
--- a/youtube_dl/extractor/baidu.py
+++ b/youtube_dl/extractor/baidu.py
@@ -4,7 +4,6 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urlparse
 
 
 class BaiduVideoIE(InfoExtractor):
@@ -47,8 +46,6 @@ class BaiduVideoIE(InfoExtractor):
 
         entries = []
         for episode in episodes_detail['videos']:
-            episode_id = '%s_%s' % (playlist_id, episode['episode'])
-
             entries.append(self.url_result(
                 episode['url'], video_title=episode['title']))
 

From a1d9f6c5dc15f1d62e0ab8c9dd6feaed4ff83a34 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 2 Jan 2016 21:36:35 +0800
Subject: [PATCH 3582/6219] [baidu] Improve playlist description

---
 youtube_dl/extractor/baidu.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/baidu.py b/youtube_dl/extractor/baidu.py
index 734b50d1d..d60cb0fdd 100644
--- a/youtube_dl/extractor/baidu.py
+++ b/youtube_dl/extractor/baidu.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..utils import unescapeHTML
 
 
 class BaiduVideoIE(InfoExtractor):
@@ -14,7 +15,7 @@ class BaiduVideoIE(InfoExtractor):
         'info_dict': {
             'id': '1069',
             'title': '中华小当家 TV版国语',
-            'description': 'md5:40a9c1b1c7f4e05d642e7bb1c84eeda0',
+            'description': 'md5:51be07afe461cf99fa61231421b5397c',
         },
         'playlist_count': 52,
     }, {
@@ -40,7 +41,7 @@ class BaiduVideoIE(InfoExtractor):
         playlist_detail = self._call_api('xqinfo', category, playlist_id)
 
         playlist_title = playlist_detail['title']
-        playlist_description = playlist_detail.get('intro')
+        playlist_description = unescapeHTML(playlist_detail.get('intro'))
 
         episodes_detail = self._call_api('xqsingle', category, playlist_id)
 

From 88fb59d91bf5fd33dbe7863fbf1e4a9a641fa483 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 2 Jan 2016 19:42:11 +0600
Subject: [PATCH 3583/6219] [bbccouk] Extend title extraction

---
 youtube_dl/extractor/bbc.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 5c621a32b..7b169881a 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -482,7 +482,8 @@ class BBCCoUkIE(InfoExtractor):
 
         if programme_id:
             formats, subtitles = self._download_media_selector(programme_id)
-            title = self._og_search_title(webpage)
+            title = self._og_search_title(webpage, default=None) or self._html_search_regex(
+                r'<h2[^>]+id="parent-title"[^>]*>(.+?)</h2>', webpage, 'title')
             description = self._search_regex(
                 r'<p class="[^"]*medium-description[^"]*">([^<]+)</p>',
                 webpage, 'description', default=None)

From 03116772583103ba97eaf8ce2cbabba9742e1929 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 2 Jan 2016 21:44:49 +0800
Subject: [PATCH 3584/6219] [baidu] Add notes for API calls

---
 youtube_dl/extractor/baidu.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/baidu.py b/youtube_dl/extractor/baidu.py
index d60cb0fdd..d35116a7c 100644
--- a/youtube_dl/extractor/baidu.py
+++ b/youtube_dl/extractor/baidu.py
@@ -28,8 +28,9 @@ class BaiduVideoIE(InfoExtractor):
         'playlist_mincount': 12,
     }]
 
-    def _call_api(self, path, category, playlist_id):
-        return self._download_json('http://app.video.baidu.com/%s/?worktype=adnative%s&id=%s' % (path, category, playlist_id), playlist_id)
+    def _call_api(self, path, category, playlist_id, note):
+        return self._download_json('http://app.video.baidu.com/%s/?worktype=adnative%s&id=%s' % (
+            path, category, playlist_id), playlist_id, note)
 
     def _real_extract(self, url):
         category, playlist_id = re.match(self._VALID_URL, url).groups()
@@ -38,12 +39,14 @@ class BaiduVideoIE(InfoExtractor):
         if category == 'tv':
             category = 'tvplay'
 
-        playlist_detail = self._call_api('xqinfo', category, playlist_id)
+        playlist_detail = self._call_api(
+            'xqinfo', category, playlist_id, 'Download playlist JSON metadata')
 
         playlist_title = playlist_detail['title']
         playlist_description = unescapeHTML(playlist_detail.get('intro'))
 
-        episodes_detail = self._call_api('xqsingle', category, playlist_id)
+        episodes_detail = self._call_api(
+            'xqsingle', category, playlist_id, 'Download episodes JSON metadata')
 
         entries = []
         for episode in episodes_detail['videos']:

From b7546397f080e65fed42c69701f423e4f57f43f5 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 2 Jan 2016 21:46:40 +0800
Subject: [PATCH 3585/6219] [baidu] Use list comprehension

---
 youtube_dl/extractor/baidu.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/baidu.py b/youtube_dl/extractor/baidu.py
index d35116a7c..76b21e596 100644
--- a/youtube_dl/extractor/baidu.py
+++ b/youtube_dl/extractor/baidu.py
@@ -48,10 +48,9 @@ class BaiduVideoIE(InfoExtractor):
         episodes_detail = self._call_api(
             'xqsingle', category, playlist_id, 'Download episodes JSON metadata')
 
-        entries = []
-        for episode in episodes_detail['videos']:
-            entries.append(self.url_result(
-                episode['url'], video_title=episode['title']))
+        entries = [self.url_result(
+            episode['url'], video_title=episode['title']
+        ) for episode in episodes_detail['videos']]
 
         return self.playlist_result(
             entries, playlist_id, playlist_title, playlist_description)

From 2fffb1dcd0790da264f86f4a8d9c54a127269aca Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 2 Jan 2016 22:33:33 +0800
Subject: [PATCH 3586/6219] [qqmusic:playlist] Capture errors and update tests

---
 youtube_dl/extractor/qqmusic.py | 29 +++++++++++++++++++++++------
 1 file changed, 23 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index 1ba3bbddf..03f04b724 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -11,6 +11,7 @@ from ..utils import (
     strip_jsonp,
     unescapeHTML,
     clean_html,
+    ExtractorError,
 )
 
 
@@ -315,7 +316,7 @@ class QQMusicPlaylistIE(QQPlaylistBaseIE):
     IE_DESC = 'QQ音乐 - 歌单'
     _VALID_URL = r'http://y\.qq\.com/#type=taoge&id=(?P<id>[0-9]+)'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://y.qq.com/#type=taoge&id=3462654915',
         'info_dict': {
             'id': '3462654915',
@@ -323,7 +324,16 @@ class QQMusicPlaylistIE(QQPlaylistBaseIE):
             'description': 'md5:d2c9d758a96b9888cf4fe82f603121d4',
         },
         'playlist_count': 40,
-    }
+        'skip': 'playlist gone',
+    }, {
+        'url': 'http://y.qq.com/#type=taoge&id=1374105607',
+        'info_dict': {
+            'id': '1374105607',
+            'title': '易入人心的华语民谣',
+            'description': '民谣的歌曲易于传唱、、歌词朗朗伤口、旋律简单温馨。属于那种才入耳孔。却上心头的感觉。没有太多的复杂情绪。简单而直接地表达乐者的情绪，就是这样的简单才易入人心。',
+        },
+        'playlist_count': 20,
+    }]
 
     def _real_extract(self, url):
         list_id = self._match_id(url)
@@ -331,14 +341,21 @@ class QQMusicPlaylistIE(QQPlaylistBaseIE):
         list_json = self._download_json(
             'http://i.y.qq.com/qzone-music/fcg-bin/fcg_ucc_getcdinfo_byids_cp.fcg?type=1&json=1&utf8=1&onlysong=0&disstid=%s'
             % list_id, list_id, 'Download list page',
-            transform_source=strip_jsonp)['cdlist'][0]
+            transform_source=strip_jsonp)
+        if not len(list_json.get('cdlist', [])):
+            if list_json.get('code'):
+                raise ExtractorError(
+                    'QQ Music said: error %d in fetching playlist info' % list_json['code'],
+                    expected=True)
+            raise ExtractorError('Unable to get playlist info')
 
+        cdlist = list_json['cdlist'][0]
         entries = [
             self.url_result(
                 'http://y.qq.com/#type=song&mid=' + song['songmid'], 'QQMusic', song['songmid']
-            ) for song in list_json['songlist']
+            ) for song in cdlist['songlist']
         ]
 
-        list_name = list_json.get('dissname')
-        list_description = clean_html(unescapeHTML(list_json.get('desc')))
+        list_name = cdlist.get('dissname')
+        list_description = clean_html(unescapeHTML(cdlist.get('desc')))
         return self.playlist_result(entries, list_id, list_name, list_description)

From 141a273a8b7a95fac5ccc0b5141c8d2eac48e3f9 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 2 Jan 2016 22:39:09 +0800
Subject: [PATCH 3587/6219] [qqmusic] Update tests

---
 youtube_dl/extractor/qqmusic.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index 03f04b724..45a3c41c5 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -178,7 +178,7 @@ class QQMusicSingerIE(QQPlaylistBaseIE):
         'info_dict': {
             'id': '001BLpXF2DyJe2',
             'title': '林俊杰',
-            'description': 'md5:2a222d89ba4455a3af19940c0481bb78',
+            'description': 'md5:870ec08f7d8547c29c93010899103751',
         },
         'playlist_count': 12,
     }
@@ -273,7 +273,7 @@ class QQMusicToplistIE(QQPlaylistBaseIE):
         'url': 'http://y.qq.com/#type=toplist&p=top_3',
         'info_dict': {
             'id': 'top_3',
-            'title': 'QQ音乐巅峰榜·欧美',
+            'title': '巅峰榜·欧美',
             'description': 'QQ音乐巅峰榜·欧美根据用户收听行为自动生成，集结当下最流行的欧美新歌！:更新时间：每周四22点|统'
                            '计周期：一周（上周四至本周三）|统计对象：三个月内发行的欧美歌曲|统计数量：100首|统计算法：根据'
                            '歌曲在一周内的有效播放次数，由高到低取前100名（同一歌手最多允许5首歌曲同时上榜）|有效播放次数：'

From 6b461026616b0e3ede9a302c74fc437541e19343 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 2 Jan 2016 21:24:57 +0100
Subject: [PATCH 3588/6219] [zdf] fix rtmpt format downloading handle errors

---
 youtube_dl/extractor/zdf.py | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/zdf.py b/youtube_dl/extractor/zdf.py
index 2a1f2f6d1..c619a75e2 100644
--- a/youtube_dl/extractor/zdf.py
+++ b/youtube_dl/extractor/zdf.py
@@ -13,6 +13,7 @@ from ..utils import (
     determine_ext,
     qualities,
     float_or_none,
+    ExtractorError,
 )
 
 
@@ -59,7 +60,6 @@ class ZDFIE(InfoExtractor):
                     'ext': 'flv',
                     'format_id': '%s-%d' % (proto, bitrate),
                     'tbr': bitrate,
-                    'protocol': proto,
                 })
         self._sort_formats(formats)
         return formats
@@ -70,6 +70,15 @@ class ZDFIE(InfoExtractor):
             note='Downloading video info',
             errnote='Failed to download video info')
 
+        status_code = doc.find('./status/statuscode')
+        if status_code is not None and status_code.text != 'ok':
+            code = status_code.text
+            if code == 'notVisibleAnymore':
+                message = 'Video %s is not available' % video_id
+            else:
+                message = '%s returned error: %s' % (self.IE_NAME, code)
+            raise ExtractorError(message, expected=True)
+
         title = doc.find('.//information/title').text
         description = xpath_text(doc, './/information/detail', 'description')
         duration = int_or_none(xpath_text(doc, './/details/lengthSec', 'duration'))
@@ -129,10 +138,10 @@ class ZDFIE(InfoExtractor):
                     video_url, video_id, fatal=False))
             elif ext == 'm3u8':
                 formats.extend(self._extract_m3u8_formats(
-                    video_url, video_id, 'mp4', m3u8_id='hls', fatal=False))
+                    video_url, video_id, 'mp4', m3u8_id=format_id, fatal=False))
             elif ext == 'f4m':
                 formats.extend(self._extract_f4m_formats(
-                    video_url, video_id, f4m_id='hds', fatal=False))
+                    video_url, video_id, f4m_id=format_id, fatal=False))
             else:
                 proto = format_m.group('proto').lower()
 

From 4059eabd58dd5e68aa1229c0ba0ffce0262ef7d8 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 2 Jan 2016 21:29:10 +0100
Subject: [PATCH 3589/6219] [dreisat] use extract_from_xml_url from ZDFIE for
 info extraction(fixes #7680)(fixes #8104)(closes #8121)

---
 youtube_dl/extractor/dreisat.py | 59 ++-------------------------------
 1 file changed, 3 insertions(+), 56 deletions(-)

diff --git a/youtube_dl/extractor/dreisat.py b/youtube_dl/extractor/dreisat.py
index 8ac8587be..028144f20 100644
--- a/youtube_dl/extractor/dreisat.py
+++ b/youtube_dl/extractor/dreisat.py
@@ -2,14 +2,10 @@ from __future__ import unicode_literals
 
 import re
 
-from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
-    unified_strdate,
-)
+from .zdf import ZDFIE
 
 
-class DreiSatIE(InfoExtractor):
+class DreiSatIE(ZDFIE):
     IE_NAME = '3sat'
     _VALID_URL = r'(?:http://)?(?:www\.)?3sat\.de/mediathek/(?:index\.php|mediathek\.php)?\?(?:(?:mode|display)=[^&]+&)*obj=(?P<id>[0-9]+)$'
     _TESTS = [
@@ -35,53 +31,4 @@ class DreiSatIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
         details_url = 'http://www.3sat.de/mediathek/xmlservice/web/beitragsDetails?ak=web&id=%s' % video_id
-        details_doc = self._download_xml(details_url, video_id, 'Downloading video details')
-
-        status_code = details_doc.find('./status/statuscode')
-        if status_code is not None and status_code.text != 'ok':
-            code = status_code.text
-            if code == 'notVisibleAnymore':
-                message = 'Video %s is not available' % video_id
-            else:
-                message = '%s returned error: %s' % (self.IE_NAME, code)
-            raise ExtractorError(message, expected=True)
-
-        thumbnail_els = details_doc.findall('.//teaserimage')
-        thumbnails = [{
-            'width': int(te.attrib['key'].partition('x')[0]),
-            'height': int(te.attrib['key'].partition('x')[2]),
-            'url': te.text,
-        } for te in thumbnail_els]
-
-        information_el = details_doc.find('.//information')
-        video_title = information_el.find('./title').text
-        video_description = information_el.find('./detail').text
-
-        details_el = details_doc.find('.//details')
-        video_uploader = details_el.find('./channel').text
-        upload_date = unified_strdate(details_el.find('./airtime').text)
-
-        format_els = details_doc.findall('.//formitaet')
-        formats = [{
-            'format_id': fe.attrib['basetype'],
-            'width': int(fe.find('./width').text),
-            'height': int(fe.find('./height').text),
-            'url': fe.find('./url').text,
-            'filesize': int(fe.find('./filesize').text),
-            'video_bitrate': int(fe.find('./videoBitrate').text),
-        } for fe in format_els
-            if not fe.find('./url').text.startswith('http://www.metafilegenerator.de/')]
-
-        self._sort_formats(formats)
-
-        return {
-            '_type': 'video',
-            'id': video_id,
-            'title': video_title,
-            'formats': formats,
-            'description': video_description,
-            'thumbnails': thumbnails,
-            'thumbnail': thumbnails[-1]['url'],
-            'uploader': video_uploader,
-            'upload_date': upload_date,
-        }
+        return self.extract_from_xml_url(video_id, details_url)

From e1a0bfdffe25dda494a9da8b02fba0c9ad39f4fe Mon Sep 17 00:00:00 2001
From: dyn888 <dyn.8.8.8+github@gmail.com>
Date: Sun, 3 Jan 2016 04:11:19 +0100
Subject: [PATCH 3590/6219] [youtube] added vcodec/acodec/abr for multiple
 itags

Should make downloading with filters more precise and easier, ie. bestvideo[vcodec=h264]. By default a lot of codecs are specified as avc1.xxxxxx and unique for each format, which makes them unusable for bestvideo selection.
---
 youtube_dl/extractor/youtube.py | 120 ++++++++++++++++----------------
 1 file changed, 60 insertions(+), 60 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 4aac2cc03..64386f34a 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -277,55 +277,55 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                      $"""
     _NEXT_URL_RE = r'[\?&]next_url=([^&]+)'
     _formats = {
-        '5': {'ext': 'flv', 'width': 400, 'height': 240},
-        '6': {'ext': 'flv', 'width': 450, 'height': 270},
-        '13': {'ext': '3gp'},
-        '17': {'ext': '3gp', 'width': 176, 'height': 144},
-        '18': {'ext': 'mp4', 'width': 640, 'height': 360},
-        '22': {'ext': 'mp4', 'width': 1280, 'height': 720},
-        '34': {'ext': 'flv', 'width': 640, 'height': 360},
-        '35': {'ext': 'flv', 'width': 854, 'height': 480},
-        '36': {'ext': '3gp', 'width': 320, 'height': 240},
-        '37': {'ext': 'mp4', 'width': 1920, 'height': 1080},
-        '38': {'ext': 'mp4', 'width': 4096, 'height': 3072},
-        '43': {'ext': 'webm', 'width': 640, 'height': 360},
-        '44': {'ext': 'webm', 'width': 854, 'height': 480},
-        '45': {'ext': 'webm', 'width': 1280, 'height': 720},
-        '46': {'ext': 'webm', 'width': 1920, 'height': 1080},
-        '59': {'ext': 'mp4', 'width': 854, 'height': 480},
-        '78': {'ext': 'mp4', 'width': 854, 'height': 480},
+        '5': {'ext': 'flv', 'width': 400, 'height': 240, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
+        '6': {'ext': 'flv', 'width': 450, 'height': 270, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
+        '13': {'ext': '3gp', 'acodec': 'aac', 'vcodec': 'mp4v'},
+        '17': {'ext': '3gp', 'width': 176, 'height': 144, 'acodec': 'aac', 'abr': 24, 'vcodec': 'mp4v'},
+        '18': {'ext': 'mp4', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 96, 'vcodec': 'h264'},
+        '22': {'ext': 'mp4', 'width': 1280, 'height': 720, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
+        '34': {'ext': 'flv', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
+        '35': {'ext': 'flv', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
+        '36': {'ext': '3gp', 'width': 320, 'height': 240, 'acodec': 'aac', 'abr': 32, 'vcodec': 'mp4v'},
+        '37': {'ext': 'mp4', 'width': 1920, 'height': 1080, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
+        '38': {'ext': 'mp4', 'width': 4096, 'height': 3072, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
+        '43': {'ext': 'webm', 'width': 640, 'height': 360, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
+        '44': {'ext': 'webm', 'width': 854, 'height': 480, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
+        '45': {'ext': 'webm', 'width': 1280, 'height': 720, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
+        '46': {'ext': 'webm', 'width': 1920, 'height': 1080, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
+        '59': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
+        '78': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
 
 
-        # 3d videos
-        '82': {'ext': 'mp4', 'height': 360, 'format_note': '3D', 'preference': -20},
-        '83': {'ext': 'mp4', 'height': 480, 'format_note': '3D', 'preference': -20},
-        '84': {'ext': 'mp4', 'height': 720, 'format_note': '3D', 'preference': -20},
-        '85': {'ext': 'mp4', 'height': 1080, 'format_note': '3D', 'preference': -20},
-        '100': {'ext': 'webm', 'height': 360, 'format_note': '3D', 'preference': -20},
-        '101': {'ext': 'webm', 'height': 480, 'format_note': '3D', 'preference': -20},
-        '102': {'ext': 'webm', 'height': 720, 'format_note': '3D', 'preference': -20},
+        # 3D videos
+        '82': {'ext': 'mp4', 'height': 360, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
+        '83': {'ext': 'mp4', 'height': 480, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
+        '84': {'ext': 'mp4', 'height': 720, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
+        '85': {'ext': 'mp4', 'height': 1080, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
+        '100': {'ext': 'webm', 'height': 360, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8', 'preference': -20},
+        '101': {'ext': 'webm', 'height': 480, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
+        '102': {'ext': 'webm', 'height': 720, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
 
         # Apple HTTP Live Streaming
-        '92': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'preference': -10},
-        '93': {'ext': 'mp4', 'height': 360, 'format_note': 'HLS', 'preference': -10},
-        '94': {'ext': 'mp4', 'height': 480, 'format_note': 'HLS', 'preference': -10},
-        '95': {'ext': 'mp4', 'height': 720, 'format_note': 'HLS', 'preference': -10},
-        '96': {'ext': 'mp4', 'height': 1080, 'format_note': 'HLS', 'preference': -10},
-        '132': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'preference': -10},
-        '151': {'ext': 'mp4', 'height': 72, 'format_note': 'HLS', 'preference': -10},
+        '92': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
+        '93': {'ext': 'mp4', 'height': 360, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
+        '94': {'ext': 'mp4', 'height': 480, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
+        '95': {'ext': 'mp4', 'height': 720, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
+        '96': {'ext': 'mp4', 'height': 1080, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
+        '132': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
+        '151': {'ext': 'mp4', 'height': 72, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 24, 'vcodec': 'h264', 'preference': -10},
 
         # DASH mp4 video
-        '133': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
-        '134': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
-        '135': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
-        '136': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
-        '137': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
-        '138': {'ext': 'mp4', 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},  # Height can vary (https://github.com/rg3/youtube-dl/issues/4559)
-        '160': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
-        '264': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
-        '298': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'h264'},
-        '299': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'h264'},
-        '266': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'vcodec': 'h264'},
+        '133': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'h264', 'preference': -40},
+        '134': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'h264', 'preference': -40},
+        '135': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'h264', 'preference': -40},
+        '136': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'h264', 'preference': -40},
+        '137': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'h264', 'preference': -40},
+        '138': {'ext': 'mp4', 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'h264', 'preference': -40},  # Height can vary (https://github.com/rg3/youtube-dl/issues/4559)
+        '160': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'h264', 'preference': -40},
+        '264': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'h264', 'preference': -40},
+        '298': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'h264', 'fps': 60, 'preference': -40},
+        '299': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'h264', 'fps': 60, 'preference': -40},
+        '266': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'h264', 'preference': -40},
 
         # Dash mp4 audio
         '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'vcodec': 'none', 'abr': 48, 'preference': -50, 'container': 'm4a_dash'},
@@ -339,26 +339,26 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         '170': {'ext': 'webm', 'height': 1080, 'width': 1920, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
         '218': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
         '219': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
-        '278': {'ext': 'webm', 'height': 144, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'container': 'webm', 'vcodec': 'vp9'},
-        '242': {'ext': 'webm', 'height': 240, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
-        '243': {'ext': 'webm', 'height': 360, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
-        '244': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
-        '245': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
-        '246': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
-        '247': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
-        '248': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
-        '271': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
+        '278': {'ext': 'webm', 'height': 144, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'vp9', 'preference': -40},
+        '242': {'ext': 'webm', 'height': 240, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'vp9', 'preference': -40},
+        '243': {'ext': 'webm', 'height': 360, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'vp9', 'preference': -40},
+        '244': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'vp9', 'preference': -40},
+        '245': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'vp9', 'preference': -40},
+        '246': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'vp9', 'preference': -40},
+        '247': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'vp9', 'preference': -40},
+        '248': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'vp9', 'preference': -40},
+        '271': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'vp9', 'preference': -40},
         # itag 272 videos are either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug)
-        '272': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
-        '302': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'vp9'},
-        '303': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'vp9'},
-        '308': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'vp9'},
-        '313': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'vcodec': 'vp9'},
-        '315': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'vp9'},
+        '272': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'vp9', 'preference': -40},
+        '302': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'vp9', 'fps': 60, 'preference': -40},
+        '303': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'vp9', 'fps': 60, 'preference': -40},
+        '308': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'vp9', 'fps': 60, 'preference': -40},
+        '313': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'vp9', 'preference': -40},
+        '315': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'vp9', 'fps': 60, 'preference': -40},
 
         # Dash webm audio
-        '171': {'ext': 'webm', 'vcodec': 'none', 'format_note': 'DASH audio', 'abr': 128, 'preference': -50},
-        '172': {'ext': 'webm', 'vcodec': 'none', 'format_note': 'DASH audio', 'abr': 256, 'preference': -50},
+        '171': {'ext': 'webm', 'acodec': 'vorbis', 'vcodec': 'none', 'format_note': 'DASH audio', 'abr': 128, 'preference': -50},
+        '172': {'ext': 'webm', 'acodec': 'vorbis', 'vcodec': 'none', 'format_note': 'DASH audio', 'abr': 256, 'preference': -50},
 
         # Dash webm audio with opus inside
         '249': {'ext': 'webm', 'vcodec': 'none', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 50, 'preference': -50},

From ea6abd740f8dd1c559e186bc1cbcfa73854a90d1 Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Sun, 3 Jan 2016 10:12:13 +0500
Subject: [PATCH 3591/6219] [nowtv] Mark broken

---
 youtube_dl/extractor/nowtv.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/nowtv.py b/youtube_dl/extractor/nowtv.py
index fd107aca2..916a102bf 100644
--- a/youtube_dl/extractor/nowtv.py
+++ b/youtube_dl/extractor/nowtv.py
@@ -71,6 +71,7 @@ class NowTVBaseIE(InfoExtractor):
 
 
 class NowTVIE(NowTVBaseIE):
+    _WORKING = False
     _VALID_URL = r'https?://(?:www\.)?nowtv\.(?:de|at|ch)/(?:rtl|rtl2|rtlnitro|superrtl|ntv|vox)/(?P<show_id>[^/]+)/(?:(?:list/[^/]+|jahr/\d{4}/\d{1,2})/)?(?P<id>[^/]+)/(?:player|preview)'
 
     _TESTS = [{

From a7aaa39863a6b1ab89a56a55521fd4779c6ac900 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 4 Jan 2016 01:08:34 +0600
Subject: [PATCH 3592/6219] [utils] Extract known extensions for reuse

---
 youtube_dl/utils.py | 31 +++++++++++++++++--------------
 1 file changed, 17 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 0ed6c45c8..da4ec7f20 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -70,6 +70,21 @@ ENGLISH_MONTH_NAMES = [
     'January', 'February', 'March', 'April', 'May', 'June',
     'July', 'August', 'September', 'October', 'November', 'December']
 
+KNOWN_EXTENSIONS = (
+    'mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v', 'aac',
+    'flv', 'f4v', 'f4a', 'f4b',
+    'webm', 'ogg', 'ogv', 'oga', 'ogx', 'spx', 'opus',
+    'mkv', 'mka', 'mk3d',
+    'avi', 'divx',
+    'mov',
+    'asf', 'wmv', 'wma',
+    '3gp', '3g2',
+    'mp3',
+    'flac',
+    'ape',
+    'wav',
+    'f4f', 'f4m', 'm3u8', 'smil')
+
 
 def preferredencoding():
     """Get preferred encoding.
@@ -942,20 +957,8 @@ def determine_ext(url, default_ext='unknown_video'):
     guess = url.partition('?')[0].rpartition('.')[2]
     if re.match(r'^[A-Za-z0-9]+$', guess):
         return guess
-    elif guess.rstrip('/') in (
-            'mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v', 'aac',
-            'flv', 'f4v', 'f4a', 'f4b',
-            'webm', 'ogg', 'ogv', 'oga', 'ogx', 'spx', 'opus',
-            'mkv', 'mka', 'mk3d',
-            'avi', 'divx',
-            'mov',
-            'asf', 'wmv', 'wma',
-            '3gp', '3g2',
-            'mp3',
-            'flac',
-            'ape',
-            'wav',
-            'f4f', 'f4m', 'm3u8', 'smil'):
+    # Try extract ext from URLs like http://example.com/foo/bar.mp4/?download
+    elif guess.rstrip('/') in KNOWN_EXTENSIONS:
         return guess.rstrip('/')
     else:
         return default_ext

From e54c44eeab9e088c344ec379cdc4b1fbc63ff324 Mon Sep 17 00:00:00 2001
From: pingtux <pingtux@users.noreply.github.com>
Date: Sat, 2 Jan 2016 01:13:49 +0100
Subject: [PATCH 3593/6219] [20min.ch] Add new extractor (closes #5977)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/min20.py    | 40 ++++++++++++++++++++++++++++++++
 2 files changed, 41 insertions(+)
 create mode 100644 youtube_dl/extractor/min20.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 4c7e5223d..64e1fd334 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -364,6 +364,7 @@ from .mdr import MDRIE
 from .metacafe import MetacafeIE
 from .metacritic import MetacriticIE
 from .mgoon import MgoonIE
+from .min20 import Min20IE
 from .minhateca import MinhatecaIE
 from .ministrygrid import MinistryGridIE
 from .miomio import MioMioIE
diff --git a/youtube_dl/extractor/min20.py b/youtube_dl/extractor/min20.py
new file mode 100644
index 000000000..a3f91671e
--- /dev/null
+++ b/youtube_dl/extractor/min20.py
@@ -0,0 +1,40 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class Min20IE(InfoExtractor):
+    _VALID_URL = r'http://www\.20min\.ch/.+?-(?P<id>[0-9]+)$'
+    _TEST = {
+        'url': 'http://www.20min.ch/schweiz/news/story/-Wir-muessen-mutig-nach-vorne-schauen--22050469',
+        'md5': 'cd4cbb99b94130cff423e967cd275e5e',
+        'info_dict': {
+            'id': '22050469',
+            'ext': 'flv',
+            'title': '«Wir müssen mutig nach vorne schauen»',
+            'description': 'Kein Land sei innovativer als die Schweiz, sagte Johann Schneider-Ammann in seiner Neujahrsansprache. Das Land müsse aber seine Hausaufgaben machen.',
+            'thumbnail': 'http://www.20min.ch/images/content/2/2/0/22050469/10/teaserbreit.jpg'
+        }
+    }
+
+    # location of the flv videos, can't be extracted from the web page
+    _BASE_URL = "http://flv-rr.20min-tv.ch/videos/"
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        title = self._html_search_regex(r'<h1><span>(.+?)</span></h1>', webpage, 'title')
+        flash_id = self._search_regex(r"so\.addVariable\(\"file1\",\"([0-9]+)\"\)", webpage, 'flash_id')
+
+        description = self._html_search_regex(r'<meta name="description" content="(.+?)" />', webpage, 'description')
+        thumbnail = self._html_search_regex(r'<meta property="og:image" content="(.+?)" />', webpage, 'thumbnail')
+        url = self._BASE_URL + flash_id + "m.flv"
+
+        return {
+            'id': video_id,
+            'url': url,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail
+        }

From 66295fa4a6a712d4d41ff54ee9c361c7c83b44bc Mon Sep 17 00:00:00 2001
From: pingtux <pingtux@users.noreply.github.com>
Date: Sat, 2 Jan 2016 16:19:05 +0100
Subject: [PATCH 3594/6219] [20min.ch] Added support for videoportal

---
 youtube_dl/extractor/min20.py | 26 +++++++++++++++++++++-----
 1 file changed, 21 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/min20.py b/youtube_dl/extractor/min20.py
index a3f91671e..23aead19d 100644
--- a/youtube_dl/extractor/min20.py
+++ b/youtube_dl/extractor/min20.py
@@ -1,12 +1,14 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 
 
 class Min20IE(InfoExtractor):
-    _VALID_URL = r'http://www\.20min\.ch/.+?-(?P<id>[0-9]+)$'
-    _TEST = {
+    _VALID_URL = r'http://www\.20min\.ch/(videotv/\?vid=(?P<video_id>[0-9]+)|.+?-(?P<page_id>[0-9]+)$)'
+    _TESTS = [{
         'url': 'http://www.20min.ch/schweiz/news/story/-Wir-muessen-mutig-nach-vorne-schauen--22050469',
         'md5': 'cd4cbb99b94130cff423e967cd275e5e',
         'info_dict': {
@@ -16,15 +18,29 @@ class Min20IE(InfoExtractor):
             'description': 'Kein Land sei innovativer als die Schweiz, sagte Johann Schneider-Ammann in seiner Neujahrsansprache. Das Land müsse aber seine Hausaufgaben machen.',
             'thumbnail': 'http://www.20min.ch/images/content/2/2/0/22050469/10/teaserbreit.jpg'
         }
-    }
+    }, {
+        'url': 'http://www.20min.ch/videotv/?vid=469148&cid=2',
+        'md5': 'b52d6bc6ea6398e6a38f12cfd418149c',
+        'info_dict': {
+            'id': '469148',
+            'ext': 'flv',
+            'title': '85 000 Franken für 15 perfekte Minuten',
+            'description': 'Was die Besucher vom Silvesterzauber erwarten können. (Video: Alice Grosjean/Murat Temel)',
+            'thumbnail': 'http://thumbnails.20min-tv.ch/server063/469148/frame-72-469148.jpg'
+        }
+    }]
 
     # location of the flv videos, can't be extracted from the web page
     _BASE_URL = "http://flv-rr.20min-tv.ch/videos/"
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('page_id')
+        if video_id is None:
+            # URL from the videoportal
+            video_id = mobj.group('video_id')
         webpage = self._download_webpage(url, video_id)
-        title = self._html_search_regex(r'<h1><span>(.+?)</span></h1>', webpage, 'title')
+        title = self._html_search_regex(r'<h1>.*<span>(.+?)</span></h1>', webpage, 'title')
         flash_id = self._search_regex(r"so\.addVariable\(\"file1\",\"([0-9]+)\"\)", webpage, 'flash_id')
 
         description = self._html_search_regex(r'<meta name="description" content="(.+?)" />', webpage, 'description')

From 133b1886fc6721090ea5d3be3d382626e2602b48 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 4 Jan 2016 02:33:08 +0600
Subject: [PATCH 3595/6219] [20min] Improve (Closes #8110)

---
 youtube_dl/extractor/__init__.py  |  2 +-
 youtube_dl/extractor/min20.py     | 56 ------------------------
 youtube_dl/extractor/twentymin.py | 73 +++++++++++++++++++++++++++++++
 3 files changed, 74 insertions(+), 57 deletions(-)
 delete mode 100644 youtube_dl/extractor/min20.py
 create mode 100644 youtube_dl/extractor/twentymin.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 64e1fd334..625b0bf16 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -364,7 +364,6 @@ from .mdr import MDRIE
 from .metacafe import MetacafeIE
 from .metacritic import MetacriticIE
 from .mgoon import MgoonIE
-from .min20 import Min20IE
 from .minhateca import MinhatecaIE
 from .ministrygrid import MinistryGridIE
 from .miomio import MioMioIE
@@ -747,6 +746,7 @@ from .tvp import TvpIE, TvpSeriesIE
 from .tvplay import TVPlayIE
 from .tweakers import TweakersIE
 from .twentyfourvideo import TwentyFourVideoIE
+from .twentymin import TwentyMinutenIE
 from .twentytwotracks import (
     TwentyTwoTracksIE,
     TwentyTwoTracksGenreIE
diff --git a/youtube_dl/extractor/min20.py b/youtube_dl/extractor/min20.py
deleted file mode 100644
index 23aead19d..000000000
--- a/youtube_dl/extractor/min20.py
+++ /dev/null
@@ -1,56 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-import re
-
-from .common import InfoExtractor
-
-
-class Min20IE(InfoExtractor):
-    _VALID_URL = r'http://www\.20min\.ch/(videotv/\?vid=(?P<video_id>[0-9]+)|.+?-(?P<page_id>[0-9]+)$)'
-    _TESTS = [{
-        'url': 'http://www.20min.ch/schweiz/news/story/-Wir-muessen-mutig-nach-vorne-schauen--22050469',
-        'md5': 'cd4cbb99b94130cff423e967cd275e5e',
-        'info_dict': {
-            'id': '22050469',
-            'ext': 'flv',
-            'title': '«Wir müssen mutig nach vorne schauen»',
-            'description': 'Kein Land sei innovativer als die Schweiz, sagte Johann Schneider-Ammann in seiner Neujahrsansprache. Das Land müsse aber seine Hausaufgaben machen.',
-            'thumbnail': 'http://www.20min.ch/images/content/2/2/0/22050469/10/teaserbreit.jpg'
-        }
-    }, {
-        'url': 'http://www.20min.ch/videotv/?vid=469148&cid=2',
-        'md5': 'b52d6bc6ea6398e6a38f12cfd418149c',
-        'info_dict': {
-            'id': '469148',
-            'ext': 'flv',
-            'title': '85 000 Franken für 15 perfekte Minuten',
-            'description': 'Was die Besucher vom Silvesterzauber erwarten können. (Video: Alice Grosjean/Murat Temel)',
-            'thumbnail': 'http://thumbnails.20min-tv.ch/server063/469148/frame-72-469148.jpg'
-        }
-    }]
-
-    # location of the flv videos, can't be extracted from the web page
-    _BASE_URL = "http://flv-rr.20min-tv.ch/videos/"
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('page_id')
-        if video_id is None:
-            # URL from the videoportal
-            video_id = mobj.group('video_id')
-        webpage = self._download_webpage(url, video_id)
-        title = self._html_search_regex(r'<h1>.*<span>(.+?)</span></h1>', webpage, 'title')
-        flash_id = self._search_regex(r"so\.addVariable\(\"file1\",\"([0-9]+)\"\)", webpage, 'flash_id')
-
-        description = self._html_search_regex(r'<meta name="description" content="(.+?)" />', webpage, 'description')
-        thumbnail = self._html_search_regex(r'<meta property="og:image" content="(.+?)" />', webpage, 'thumbnail')
-        url = self._BASE_URL + flash_id + "m.flv"
-
-        return {
-            'id': video_id,
-            'url': url,
-            'title': title,
-            'description': description,
-            'thumbnail': thumbnail
-        }
diff --git a/youtube_dl/extractor/twentymin.py b/youtube_dl/extractor/twentymin.py
new file mode 100644
index 000000000..ca7d953b8
--- /dev/null
+++ b/youtube_dl/extractor/twentymin.py
@@ -0,0 +1,73 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import remove_end
+
+
+class TwentyMinutenIE(InfoExtractor):
+    IE_NAME = '20min'
+    _VALID_URL = r'https?://(?:www\.)?20min\.ch/(?:videotv/*\?.*\bvid=(?P<id>\d+)|(?:[^/]+/)*(?P<display_id>[^/#?]+))'
+    _TESTS = [{
+        # regular video
+        'url': 'http://www.20min.ch/videotv/?vid=469148&cid=2',
+        'md5': 'b52d6bc6ea6398e6a38f12cfd418149c',
+        'info_dict': {
+            'id': '469148',
+            'ext': 'flv',
+            'title': '85 000 Franken für 15 perfekte Minuten',
+            'description': 'Was die Besucher vom Silvesterzauber erwarten können. (Video: Alice Grosjean/Murat Temel)',
+            'thumbnail': 'http://thumbnails.20min-tv.ch/server063/469148/frame-72-469148.jpg'
+        }
+    }, {
+        # news article with video
+        'url': 'http://www.20min.ch/schweiz/news/story/-Wir-muessen-mutig-nach-vorne-schauen--22050469',
+        'md5': 'cd4cbb99b94130cff423e967cd275e5e',
+        'info_dict': {
+            'id': '469408',
+            'display_id': '-Wir-muessen-mutig-nach-vorne-schauen--22050469',
+            'ext': 'flv',
+            'title': '«Wir müssen mutig nach vorne schauen»',
+            'description': 'Kein Land sei innovativer als die Schweiz, sagte Johann Schneider-Ammann in seiner Neujahrsansprache. Das Land müsse aber seine Hausaufgaben machen.',
+            'thumbnail': 'http://www.20min.ch/images/content/2/2/0/22050469/10/teaserbreit.jpg'
+        }
+    }, {
+        'url': 'http://www.20min.ch/videotv/?cid=44&vid=468738',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.20min.ch/ro/sortir/cinema/story/Grandir-au-bahut--c-est-dur-18927411',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        display_id = mobj.group('display_id') or video_id
+
+        webpage = self._download_webpage(url, display_id)
+
+        title = self._html_search_regex(
+            r'<h1>.*?<span>(.+?)</span></h1>',
+            webpage, 'title', default=None)
+        if not title:
+            title = remove_end(re.sub(
+                r'^20 [Mm]inuten.*? -', '', self._og_search_title(webpage)), ' - News')
+
+        if not video_id:
+            video_id = self._search_regex(
+                r'"file\d?"\s*,\s*\"(\d+)', webpage, 'video id')
+
+        description = self._html_search_meta(
+            'description', webpage, 'description')
+        thumbnail = self._og_search_thumbnail(webpage)
+
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            'url': 'http://speed.20min-tv.ch/%sm.flv' % video_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+        }

From 5aa535c32982f1f9b2f689097a64be78d25e7d90 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 4 Jan 2016 02:55:25 +0600
Subject: [PATCH 3596/6219] [bbccouk] Update tests (Closes #8090)

---
 youtube_dl/extractor/bbc.py | 19 +++++++++----------
 1 file changed, 9 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 7b169881a..ce99a34ab 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -124,14 +124,14 @@ class BBCCoUkIE(InfoExtractor):
             },
             'skip': 'Episode is no longer available on BBC iPlayer Radio',
         }, {
-            'url': 'http://www.bbc.co.uk/music/clips/p02frcc3',
+            'url': 'http://www.bbc.co.uk/music/clips/p022h44b',
             'note': 'Audio',
             'info_dict': {
-                'id': 'p02frcch',
+                'id': 'p022h44j',
                 'ext': 'flv',
-                'title': 'Pete Tong, Past, Present and Future Special, Madeon - After Hours mix',
-                'description': 'French house superstar Madeon takes us out of the club and onto the after party.',
-                'duration': 3507,
+                'title': 'BBC Proms Music Guides, Rachmaninov: Symphonic Dances',
+                'description': "In this Proms Music Guide, Andrew McGregor looks at Rachmaninov's Symphonic Dances.",
+                'duration': 227,
             },
             'params': {
                 # rtmp download
@@ -182,13 +182,12 @@ class BBCCoUkIE(InfoExtractor):
         }, {
             # iptv-all mediaset fails with geolocation however there is no geo restriction
             # for this programme at all
-            'url': 'http://www.bbc.co.uk/programmes/b06bp7lf',
+            'url': 'http://www.bbc.co.uk/programmes/b06rkn85',
             'info_dict': {
-                'id': 'b06bp7kf',
+                'id': 'b06rkms3',
                 'ext': 'flv',
-                'title': "Annie Mac's Friday Night, B.Traits sits in for Annie",
-                'description': 'B.Traits sits in for Annie Mac with a Mini-Mix from Disclosure.',
-                'duration': 10800,
+                'title': "Best of the Mini-Mixes 2015: Part 3, Annie Mac's Friday Night - BBC Radio 1",
+                'description': "Annie has part three in the Best of the Mini-Mixes 2015, plus the year's Most Played!",
             },
             'params': {
                 # rtmp download

From ab3176af3472917537635bd96b860a974094b0af Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 4 Jan 2016 03:34:15 +0600
Subject: [PATCH 3597/6219] [ivi] Fix extraction and modernize

---
 youtube_dl/extractor/ivi.py | 69 ++++++++++++++++++-------------------
 1 file changed, 33 insertions(+), 36 deletions(-)

diff --git a/youtube_dl/extractor/ivi.py b/youtube_dl/extractor/ivi.py
index 029878d24..216c534b5 100644
--- a/youtube_dl/extractor/ivi.py
+++ b/youtube_dl/extractor/ivi.py
@@ -7,6 +7,7 @@ import json
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
+    int_or_none,
     sanitized_Request,
 )
 
@@ -27,7 +28,7 @@ class IviIE(InfoExtractor):
                 'title': 'Иван Васильевич меняет профессию',
                 'description': 'md5:b924063ea1677c8fe343d8a72ac2195f',
                 'duration': 5498,
-                'thumbnail': 'http://thumbs.ivi.ru/f20.vcp.digitalaccess.ru/contents/d/1/c3c885163a082c29bceeb7b5a267a6.jpg',
+                'thumbnail': 're:^https?://.*\.jpg$',
             },
             'skip': 'Only works from Russia',
         },
@@ -38,33 +39,23 @@ class IviIE(InfoExtractor):
             'info_dict': {
                 'id': '9549',
                 'ext': 'mp4',
-                'title': 'Двое из ларца - Серия 1',
+                'title': 'Двое из ларца - Дело Гольдберга (1 часть)',
+                'series': 'Двое из ларца',
+                'episode': 'Дело Гольдберга (1 часть)',
+                'episode_number': 1,
                 'duration': 2655,
-                'thumbnail': 'http://thumbs.ivi.ru/f15.vcp.digitalaccess.ru/contents/8/4/0068dc0677041f3336b7c2baad8fc0.jpg',
+                'thumbnail': 're:^https?://.*\.jpg$',
             },
             'skip': 'Only works from Russia',
         }
     ]
 
     # Sorted by quality
-    _known_formats = ['MP4-low-mobile', 'MP4-mobile', 'FLV-lo', 'MP4-lo', 'FLV-hi', 'MP4-hi', 'MP4-SHQ']
-
-    # Sorted by size
-    _known_thumbnails = ['Thumb-120x90', 'Thumb-160', 'Thumb-640x480']
-
-    def _extract_description(self, html):
-        m = re.search(r'<meta name="description" content="(?P<description>[^"]+)"/>', html)
-        return m.group('description') if m is not None else None
-
-    def _extract_comment_count(self, html):
-        m = re.search('(?s)<a href="#" id="view-comments" class="action-button dim gradient">\s*Комментарии:\s*(?P<commentcount>\d+)\s*</a>', html)
-        return int(m.group('commentcount')) if m is not None else 0
+    _KNOWN_FORMATS = ['MP4-low-mobile', 'MP4-mobile', 'FLV-lo', 'MP4-lo', 'FLV-hi', 'MP4-hi', 'MP4-SHQ']
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        api_url = 'http://api.digitalaccess.ru/api/json/'
-
         data = {
             'method': 'da.content.get',
             'params': [
@@ -76,11 +67,10 @@ class IviIE(InfoExtractor):
             ]
         }
 
-        request = sanitized_Request(api_url, json.dumps(data))
-
-        video_json_page = self._download_webpage(
+        request = sanitized_Request(
+            'http://api.digitalaccess.ru/api/json/', json.dumps(data))
+        video_json = self._download_json(
             request, video_id, 'Downloading video JSON')
-        video_json = json.loads(video_json_page)
 
         if 'error' in video_json:
             error = video_json['error']
@@ -95,35 +85,42 @@ class IviIE(InfoExtractor):
         formats = [{
             'url': x['url'],
             'format_id': x['content_format'],
-            'preference': self._known_formats.index(x['content_format']),
-        } for x in result['files'] if x['content_format'] in self._known_formats]
+            'preference': self._KNOWN_FORMATS.index(x['content_format']),
+        } for x in result['files'] if x['content_format'] in self._KNOWN_FORMATS]
 
         self._sort_formats(formats)
 
-        if not formats:
-            raise ExtractorError('No media links available for %s' % video_id)
-
-        duration = result['duration']
-        compilation = result['compilation']
         title = result['title']
 
+        duration = int_or_none(result.get('duration'))
+        compilation = result.get('compilation')
+        episode = title if compilation else None
+
         title = '%s - %s' % (compilation, title) if compilation is not None else title
 
-        previews = result['preview']
-        previews.sort(key=lambda fmt: self._known_thumbnails.index(fmt['content_format']))
-        thumbnail = previews[-1]['url'] if len(previews) > 0 else None
+        thumbnails = [{
+            'url': preview['url'],
+            'id': preview.get('content_format'),
+        } for preview in result.get('preview', []) if preview.get('url')]
 
-        video_page = self._download_webpage(url, video_id, 'Downloading video page')
-        description = self._extract_description(video_page)
-        comment_count = self._extract_comment_count(video_page)
+        webpage = self._download_webpage(url, video_id)
+
+        episode_number = int_or_none(self._search_regex(
+            r'<meta[^>]+itemprop="episode"[^>]*>\s*<meta[^>]+itemprop="episodeNumber"[^>]+content="(\d+)',
+            webpage, 'episode number', default=None))
+
+        description = self._og_search_description(webpage, default=None) or self._html_search_meta(
+            'description', webpage, 'description', default=None)
 
         return {
             'id': video_id,
             'title': title,
-            'thumbnail': thumbnail,
+            'series': compilation,
+            'episode': episode,
+            'episode_number': episode_number,
+            'thumbnails': thumbnails,
             'description': description,
             'duration': duration,
-            'comment_count': comment_count,
             'formats': formats,
         }
 

From c6270b2ed5be9e83bcb8114511641f9c5ad9008d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 4 Jan 2016 03:49:18 +0600
Subject: [PATCH 3598/6219] [ivi:compilation] Fix extraction

---
 youtube_dl/extractor/ivi.py | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/ivi.py b/youtube_dl/extractor/ivi.py
index 216c534b5..9ccfee173 100644
--- a/youtube_dl/extractor/ivi.py
+++ b/youtube_dl/extractor/ivi.py
@@ -146,8 +146,11 @@ class IviCompilationIE(InfoExtractor):
     }]
 
     def _extract_entries(self, html, compilation_id):
-        return [self.url_result('http://www.ivi.ru/watch/%s/%s' % (compilation_id, serie), 'Ivi')
-                for serie in re.findall(r'<strong><a href="/watch/%s/(\d+)">(?:[^<]+)</a></strong>' % compilation_id, html)]
+        return [
+            self.url_result(
+                'http://www.ivi.ru/watch/%s/%s' % (compilation_id, serie), IviIE.ie_key())
+            for serie in re.findall(
+                r'<a href="/watch/%s/(\d+)"[^>]+data-id="\1"' % compilation_id, html)]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
@@ -155,7 +158,8 @@ class IviCompilationIE(InfoExtractor):
         season_id = mobj.group('seasonid')
 
         if season_id is not None:  # Season link
-            season_page = self._download_webpage(url, compilation_id, 'Downloading season %s web page' % season_id)
+            season_page = self._download_webpage(
+                url, compilation_id, 'Downloading season %s web page' % season_id)
             playlist_id = '%s/season%s' % (compilation_id, season_id)
             playlist_title = self._html_search_meta('title', season_page, 'title')
             entries = self._extract_entries(season_page, compilation_id)
@@ -163,8 +167,9 @@ class IviCompilationIE(InfoExtractor):
             compilation_page = self._download_webpage(url, compilation_id, 'Downloading compilation web page')
             playlist_id = compilation_id
             playlist_title = self._html_search_meta('title', compilation_page, 'title')
-            seasons = re.findall(r'<a href="/watch/%s/season(\d+)">[^<]+</a>' % compilation_id, compilation_page)
-            if len(seasons) == 0:  # No seasons in this compilation
+            seasons = re.findall(
+                r'<a href="/watch/%s/season(\d+)' % compilation_id, compilation_page)
+            if not seasons:  # No seasons in this compilation
                 entries = self._extract_entries(compilation_page, compilation_id)
             else:
                 entries = []

From 1463c5b9ac16bd50d5e30d2145ec584d1e6e74ce Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 4 Jan 2016 03:54:52 +0600
Subject: [PATCH 3599/6219] [ivi] Extract season info

---
 youtube_dl/extractor/ivi.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/youtube_dl/extractor/ivi.py b/youtube_dl/extractor/ivi.py
index 9ccfee173..d0f00cdea 100644
--- a/youtube_dl/extractor/ivi.py
+++ b/youtube_dl/extractor/ivi.py
@@ -41,6 +41,8 @@ class IviIE(InfoExtractor):
                 'ext': 'mp4',
                 'title': 'Двое из ларца - Дело Гольдберга (1 часть)',
                 'series': 'Двое из ларца',
+                'season': 'Сезон 1',
+                'season_number': 1,
                 'episode': 'Дело Гольдберга (1 часть)',
                 'episode_number': 1,
                 'duration': 2655,
@@ -105,6 +107,13 @@ class IviIE(InfoExtractor):
 
         webpage = self._download_webpage(url, video_id)
 
+        season = self._search_regex(
+            r'<li[^>]+class="season active"[^>]*><a[^>]+>([^<]+)',
+            webpage, 'season', default=None)
+        season_number = int_or_none(self._search_regex(
+            r'<li[^>]+class="season active"[^>]*><a[^>]+data-season(?:-index)?="(\d+)"',
+            webpage, 'season number', default=None))
+
         episode_number = int_or_none(self._search_regex(
             r'<meta[^>]+itemprop="episode"[^>]*>\s*<meta[^>]+itemprop="episodeNumber"[^>]+content="(\d+)',
             webpage, 'episode number', default=None))
@@ -116,6 +125,8 @@ class IviIE(InfoExtractor):
             'id': video_id,
             'title': title,
             'series': compilation,
+            'season': season,
+            'season_number': season_number,
             'episode': episode,
             'episode_number': episode_number,
             'thumbnails': thumbnails,

From 896c7a23cd9fbfc589f26e9a7bbada38a325e2ad Mon Sep 17 00:00:00 2001
From: bpfoley <bpfoley>
Date: Wed, 30 Dec 2015 21:50:26 +0000
Subject: [PATCH 3600/6219] [extractor/rte.py] Add support for RTE radio player

While here, stop RteIE changing filename extensions to .mp4. The files
saved are .flv containers with h264 video.
---
 youtube_dl/extractor/__init__.py |  2 +-
 youtube_dl/extractor/rte.py      | 77 ++++++++++++++++++++++++++++----
 2 files changed, 70 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index fb7151443..fe3c32874 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -556,7 +556,7 @@ from .ro220 import Ro220IE
 from .rottentomatoes import RottenTomatoesIE
 from .roxwel import RoxwelIE
 from .rtbf import RTBFIE
-from .rte import RteIE
+from .rte import RteIE, RteRadioIE
 from .rtlnl import RtlNlIE
 from .rtl2 import RTL2IE
 from .rtp import RTPIE
diff --git a/youtube_dl/extractor/rte.py b/youtube_dl/extractor/rte.py
index d9cfbf180..25abcee92 100644
--- a/youtube_dl/extractor/rte.py
+++ b/youtube_dl/extractor/rte.py
@@ -5,16 +5,19 @@ from .common import InfoExtractor
 
 from ..utils import (
     float_or_none,
+    unescapeHTML,
 )
 
 
 class RteIE(InfoExtractor):
+    IE_NAME = 'rte'
+    IE_DESC = 'Raidió Teilifís Éireann TV'
     _VALID_URL = r'https?://(?:www\.)?rte\.ie/player/[^/]{2,3}/show/[^/]+/(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://www.rte.ie/player/ie/show/iwitness-862/10478715/',
         'info_dict': {
             'id': '10478715',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': 'Watch iWitness  online',
             'thumbnail': 're:^https?://.*\.jpg$',
             'description': 'iWitness : The spirit of Ireland, one voice and one minute at a time.',
@@ -44,13 +47,6 @@ class RteIE(InfoExtractor):
         # f4m_url = server + relative_url
         f4m_url = json_string['shows'][0]['media:group'][0]['rte:server'] + json_string['shows'][0]['media:group'][0]['url']
         f4m_formats = self._extract_f4m_formats(f4m_url, video_id)
-        f4m_formats = [{
-            'format_id': f['format_id'],
-            'url': f['url'],
-            'ext': 'mp4',
-            'width': f['width'],
-            'height': f['height'],
-        } for f in f4m_formats]
 
         return {
             'id': video_id,
@@ -60,3 +56,68 @@ class RteIE(InfoExtractor):
             'thumbnail': thumbnail,
             'duration': duration,
         }
+
+
+
+class RteRadioIE(InfoExtractor):
+    IE_NAME = 'rte:radio'
+    IE_DESC = 'Raidió Teilifís Éireann radio'
+    # Radioplayer URLs have the specifier #!rii=<channel_id>:<id>:<playable_item_id>:<date>:
+    # where the IDs are int/empty, the date is DD-MM-YYYY, and the specifier may be truncated.
+    # An <id> uniquely defines an individual recording, and is the only part we require.
+    _VALID_URL = r'https?://(?:www\.)?rte\.ie/radio/utils/radioplayer/rteradioweb\.html#!rii=(?:[0-9]*)(?:%3A|:)(?P<id>[0-9]+)'
+
+    _TEST = {
+        'url': 'http://www.rte.ie/radio/utils/radioplayer/rteradioweb.html#!rii=16:10507902:2414:27-12-2015:',
+        'info_dict': {
+            'id': '10507902',
+            'ext': 'flv',
+            'title': 'Gloria',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'description': 'Tim Thurston guides you through a millennium of sacred music featuring Gregorian chant, pure solo voices and choral masterpieces, framed around the glorious music of J.S. Bach.',
+            'duration': 7230.0,
+        },
+        'params': {
+            'skip_download': 'f4m fails with --test atm'
+        }
+    }
+
+    def _real_extract(self, url):
+        item_id = self._match_id(url)
+        feeds_url = 'http://www.rte.ie/rteavgen/getplaylist/?type=web&format=json&id=' + item_id
+        json_string = self._download_json(feeds_url, item_id)
+
+        # NB the string values in the JSON are stored using XML escaping(!)
+        show = json_string['shows'][0]
+        title = unescapeHTML(show['title'])
+        description = unescapeHTML(show.get('description'))
+        thumbnail = show.get('thumbnail')
+        duration = float_or_none(show.get('duration'), 1000)
+
+        mg = show['media:group'][0]
+
+        formats = []
+
+        if mg.get('url') and not mg['url'].startswith('rtmpe:'):
+            formats.append({'url': mg.get('url')})
+
+        if mg.get('hls_server') and mg.get('hls_url'):
+            hls_url = mg['hls_server'] +  mg['hls_url']
+            hls_formats = self._extract_m3u8_formats(
+                    hls_url, item_id, 'mp4', m3u8_id='hls', fatal=False)
+            formats.extend(hls_formats)
+
+        if mg.get('hds_server') and mg.get('hds_url'):
+            f4m_url = mg['hds_server'] + mg['hds_url']
+            f4m_formats = self._extract_f4m_formats(
+                    f4m_url, item_id, f4m_id='hds', fatal=False)
+            formats.extend(f4m_formats)
+
+        return {
+            'id': item_id,
+            'title': title,
+            'formats': formats,
+            'description': description,
+            'thumbnail': thumbnail,
+            'duration': duration,
+        }

From fb588f6a5663811da83cde68d17ff4eb041022ab Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 4 Jan 2016 04:32:47 +0600
Subject: [PATCH 3601/6219] Credit @bpfoley for rte:radio (#8063)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 20f51009e..e39d07efe 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -150,3 +150,4 @@ reiv
 Muratcan Simsek
 Evan Lu
 flatgreen
+Brian Foley

From 0238451fc0f3c5f06827d5197a904981890a3d3b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 4 Jan 2016 04:49:13 +0600
Subject: [PATCH 3602/6219] [rte] PEP 8

---
 youtube_dl/extractor/rte.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/rte.py b/youtube_dl/extractor/rte.py
index 25abcee92..673399f04 100644
--- a/youtube_dl/extractor/rte.py
+++ b/youtube_dl/extractor/rte.py
@@ -58,7 +58,6 @@ class RteIE(InfoExtractor):
         }
 
 
-
 class RteRadioIE(InfoExtractor):
     IE_NAME = 'rte:radio'
     IE_DESC = 'Raidió Teilifís Éireann radio'
@@ -102,15 +101,15 @@ class RteRadioIE(InfoExtractor):
             formats.append({'url': mg.get('url')})
 
         if mg.get('hls_server') and mg.get('hls_url'):
-            hls_url = mg['hls_server'] +  mg['hls_url']
+            hls_url = mg['hls_server'] + mg['hls_url']
             hls_formats = self._extract_m3u8_formats(
-                    hls_url, item_id, 'mp4', m3u8_id='hls', fatal=False)
+                hls_url, item_id, 'mp4', m3u8_id='hls', fatal=False)
             formats.extend(hls_formats)
 
         if mg.get('hds_server') and mg.get('hds_url'):
             f4m_url = mg['hds_server'] + mg['hds_url']
             f4m_formats = self._extract_f4m_formats(
-                    f4m_url, item_id, f4m_id='hds', fatal=False)
+                f4m_url, item_id, f4m_id='hds', fatal=False)
             formats.extend(f4m_formats)
 
         return {

From 9746f4314ad0fe55076eb35fd70413623b0c8ab0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 4 Jan 2016 05:01:32 +0600
Subject: [PATCH 3603/6219] [rte:radio] Simplify

---
 youtube_dl/extractor/rte.py | 19 +++++++++----------
 1 file changed, 9 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/rte.py b/youtube_dl/extractor/rte.py
index 673399f04..c17ef2cc2 100644
--- a/youtube_dl/extractor/rte.py
+++ b/youtube_dl/extractor/rte.py
@@ -2,7 +2,6 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-
 from ..utils import (
     float_or_none,
     unescapeHTML,
@@ -98,19 +97,19 @@ class RteRadioIE(InfoExtractor):
         formats = []
 
         if mg.get('url') and not mg['url'].startswith('rtmpe:'):
-            formats.append({'url': mg.get('url')})
+            formats.append({'url': mg['url']})
 
         if mg.get('hls_server') and mg.get('hls_url'):
-            hls_url = mg['hls_server'] + mg['hls_url']
-            hls_formats = self._extract_m3u8_formats(
-                hls_url, item_id, 'mp4', m3u8_id='hls', fatal=False)
-            formats.extend(hls_formats)
+            formats.extend(self._extract_m3u8_formats(
+                mg['hls_server'] + mg['hls_url'], item_id, 'mp4',
+                entry_protocol='m3u8_native', m3u8_id='hls', fatal=False))
 
         if mg.get('hds_server') and mg.get('hds_url'):
-            f4m_url = mg['hds_server'] + mg['hds_url']
-            f4m_formats = self._extract_f4m_formats(
-                f4m_url, item_id, f4m_id='hds', fatal=False)
-            formats.extend(f4m_formats)
+            formats.extend(self._extract_f4m_formats(
+                mg['hds_server'] + mg['hds_url'], item_id,
+                f4m_id='hds', fatal=False))
+
+        self._sort_formats(formats)
 
         return {
             'id': item_id,

From 9938a17f92ad8242d8841528a89df647ff759183 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 4 Jan 2016 05:04:48 +0600
Subject: [PATCH 3604/6219] [rte:radio] Extract timestamp

---
 youtube_dl/extractor/rte.py | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/rte.py b/youtube_dl/extractor/rte.py
index c17ef2cc2..47c8331fe 100644
--- a/youtube_dl/extractor/rte.py
+++ b/youtube_dl/extractor/rte.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 from .common import InfoExtractor
 from ..utils import (
     float_or_none,
+    parse_iso8601,
     unescapeHTML,
 )
 
@@ -69,10 +70,12 @@ class RteRadioIE(InfoExtractor):
         'url': 'http://www.rte.ie/radio/utils/radioplayer/rteradioweb.html#!rii=16:10507902:2414:27-12-2015:',
         'info_dict': {
             'id': '10507902',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Gloria',
             'thumbnail': 're:^https?://.*\.jpg$',
-            'description': 'Tim Thurston guides you through a millennium of sacred music featuring Gregorian chant, pure solo voices and choral masterpieces, framed around the glorious music of J.S. Bach.',
+            'description': 'md5:9ce124a7fb41559ec68f06387cabddf0',
+            'timestamp': 1451203200,
+            'upload_date': '20151227',
             'duration': 7230.0,
         },
         'params': {
@@ -82,8 +85,10 @@ class RteRadioIE(InfoExtractor):
 
     def _real_extract(self, url):
         item_id = self._match_id(url)
-        feeds_url = 'http://www.rte.ie/rteavgen/getplaylist/?type=web&format=json&id=' + item_id
-        json_string = self._download_json(feeds_url, item_id)
+
+        json_string = self._download_json(
+            'http://www.rte.ie/rteavgen/getplaylist/?type=web&format=json&id=' + item_id,
+            item_id)
 
         # NB the string values in the JSON are stored using XML escaping(!)
         show = json_string['shows'][0]
@@ -91,6 +96,7 @@ class RteRadioIE(InfoExtractor):
         description = unescapeHTML(show.get('description'))
         thumbnail = show.get('thumbnail')
         duration = float_or_none(show.get('duration'), 1000)
+        timestamp = parse_iso8601(show.get('published'))
 
         mg = show['media:group'][0]
 
@@ -114,8 +120,9 @@ class RteRadioIE(InfoExtractor):
         return {
             'id': item_id,
             'title': title,
-            'formats': formats,
             'description': description,
             'thumbnail': thumbnail,
+            'timestamp': timestamp,
             'duration': duration,
+            'formats': formats,
         }

From 3f17c357d9958bfae18f06676c89ffdb63553509 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 4 Jan 2016 18:35:31 +0100
Subject: [PATCH 3605/6219] [downloader/hls] Don't let ffmpeg read from stdin
 (#8139)

If you run 'while read aurl ; do youtube-dl "${aurl}"; done < path_to_batch_file'  (batch_file contains one url per line that uses the hls downloader) each call to youtube-dl consumed some characters and 'read' would assing to 'aurl' a non valid url

(This is the same problem that was fixed for the ffmpeg postprocessors in cffcbc02de504d84e1c2677bb525c00b03e04f53)
---
 youtube_dl/downloader/hls.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index b5a3e1167..d186d402d 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -46,7 +46,7 @@ class HlsFD(FileDownloader):
 
         self._debug_cmd(args)
 
-        retval = subprocess.call(args)
+        retval = subprocess.call(args, stdin=subprocess.PIPE)
         if retval == 0:
             fsize = os.path.getsize(encodeFilename(tmpfilename))
             self.to_screen('\r[%s] %s bytes' % (args[0], fsize))

From fff79f1867c5ac50eb7f88201dd3f728996d52a4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 5 Jan 2016 01:05:37 +0600
Subject: [PATCH 3606/6219] [amp] Add missing subtitles to info dict

---
 youtube_dl/extractor/amp.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/amp.py b/youtube_dl/extractor/amp.py
index 1035d1c48..69e6baff7 100644
--- a/youtube_dl/extractor/amp.py
+++ b/youtube_dl/extractor/amp.py
@@ -76,5 +76,6 @@ class AMPIE(InfoExtractor):
             'thumbnails': thumbnails,
             'timestamp': parse_iso8601(item.get('pubDate'), ' '),
             'duration': int_or_none(media_content[0].get('@attributes', {}).get('duration')),
+            'subtitles': subtitles,
             'formats': formats,
         }

From 1dcc38b233f33112e00e1ddf0de6eb8ba1ef028e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 5 Jan 2016 01:11:07 +0600
Subject: [PATCH 3607/6219] [dramafever] Improve subtitles extraction (Closes
 #8136)

---
 youtube_dl/extractor/dramafever.py | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/dramafever.py b/youtube_dl/extractor/dramafever.py
index b3b21d65f..0e9e6f7ba 100644
--- a/youtube_dl/extractor/dramafever.py
+++ b/youtube_dl/extractor/dramafever.py
@@ -105,13 +105,16 @@ class DramaFeverIE(DramaFeverBaseIE):
             video_id, 'Downloading episode info JSON', fatal=False)
         if episode_info:
             value = episode_info.get('value')
-            if value:
-                subfile = value[0].get('subfile') or value[0].get('new_subfile')
-                if subfile and subfile != 'http://www.dramafever.com/st/':
-                    info.setdefault('subtitles', {}).setdefault('English', []).append({
-                        'ext': 'srt',
-                        'url': subfile,
-                    })
+            if isinstance(value, list):
+                for v in value:
+                    if v.get('type') == 'Episode':
+                        subfile = v.get('subfile') or v.get('new_subfile')
+                        if subfile and subfile != 'http://www.dramafever.com/st/':
+                            info.setdefault('subtitles', {}).setdefault('English', []).append({
+                                'ext': 'srt',
+                                'url': subfile,
+                            })
+                        break
 
         return info
 

From 8f4c56f33412db909e92218f4933d5b08beb9f21 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 5 Jan 2016 01:17:33 +0600
Subject: [PATCH 3608/6219] [dramafever] Extract episode number

---
 youtube_dl/extractor/dramafever.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/dramafever.py b/youtube_dl/extractor/dramafever.py
index 0e9e6f7ba..8d71d6769 100644
--- a/youtube_dl/extractor/dramafever.py
+++ b/youtube_dl/extractor/dramafever.py
@@ -12,6 +12,7 @@ from ..compat import (
 from ..utils import (
     ExtractorError,
     clean_html,
+    int_or_none,
     sanitized_Request,
 )
 
@@ -114,6 +115,7 @@ class DramaFeverIE(DramaFeverBaseIE):
                                 'ext': 'srt',
                                 'url': subfile,
                             })
+                        info['episode_number'] = int_or_none(v.get('number'))
                         break
 
         return info

From bd19aa0ed32ca4d041e691d47697ad7b82454b69 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 5 Jan 2016 01:28:48 +0600
Subject: [PATCH 3609/6219] [dramafever] Extract episode

---
 youtube_dl/extractor/dramafever.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/dramafever.py b/youtube_dl/extractor/dramafever.py
index 8d71d6769..a14021a9e 100644
--- a/youtube_dl/extractor/dramafever.py
+++ b/youtube_dl/extractor/dramafever.py
@@ -115,7 +115,12 @@ class DramaFeverIE(DramaFeverBaseIE):
                                 'ext': 'srt',
                                 'url': subfile,
                             })
-                        info['episode_number'] = int_or_none(v.get('number'))
+                        episode_number = int_or_none(v.get('number'))
+                        episode_fallback = 'Episode'
+                        if episode_number:
+                            episode_fallback += ' %d' % episode_number
+                        info['episode'] = v.get('title', episode_fallback)
+                        info['episode_number'] = episode_number
                         break
 
         return info

From a2e51e7b49a193b326291842d03aa49dd08326f2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 5 Jan 2016 01:36:38 +0600
Subject: [PATCH 3610/6219] [dramafever] Fix episode fallback

---
 youtube_dl/extractor/dramafever.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/dramafever.py b/youtube_dl/extractor/dramafever.py
index a14021a9e..9f5e82f56 100644
--- a/youtube_dl/extractor/dramafever.py
+++ b/youtube_dl/extractor/dramafever.py
@@ -119,7 +119,7 @@ class DramaFeverIE(DramaFeverBaseIE):
                         episode_fallback = 'Episode'
                         if episode_number:
                             episode_fallback += ' %d' % episode_number
-                        info['episode'] = v.get('title', episode_fallback)
+                        info['episode'] = v.get('title') or episode_fallback
                         info['episode_number'] = episode_number
                         break
 

From 33cee6c7f6260dd33a9fbafd9f374c70454ff0f3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 5 Jan 2016 01:41:18 +0600
Subject: [PATCH 3611/6219] [dramafever] Add test for custom episode title

---
 youtube_dl/extractor/dramafever.py | 26 +++++++++++++++++++++++---
 1 file changed, 23 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/dramafever.py b/youtube_dl/extractor/dramafever.py
index 9f5e82f56..d35e88881 100644
--- a/youtube_dl/extractor/dramafever.py
+++ b/youtube_dl/extractor/dramafever.py
@@ -67,13 +67,15 @@ class DramaFeverBaseIE(AMPIE):
 class DramaFeverIE(DramaFeverBaseIE):
     IE_NAME = 'dramafever'
     _VALID_URL = r'https?://(?:www\.)?dramafever\.com/drama/(?P<id>[0-9]+/[0-9]+)(?:/|$)'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.dramafever.com/drama/4512/1/Cooking_with_Shin/',
         'info_dict': {
             'id': '4512.1',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Cooking with Shin 4512.1',
             'description': 'md5:a8eec7942e1664a6896fcd5e1287bfd0',
+            'episode': 'Episode 1',
+            'episode_number': 1,
             'thumbnail': 're:^https?://.*\.jpg',
             'timestamp': 1404336058,
             'upload_date': '20140702',
@@ -83,7 +85,25 @@ class DramaFeverIE(DramaFeverBaseIE):
             # m3u8 download
             'skip_download': True,
         },
-    }
+    }, {
+        'url': 'http://www.dramafever.com/drama/4826/4/Mnet_Asian_Music_Awards_2015/?ap=1',
+        'info_dict': {
+            'id': '4826.4',
+            'ext': 'mp4',
+            'title': 'Mnet Asian Music Awards 2015 4826.4',
+            'description': 'md5:3ff2ee8fedaef86e076791c909cf2e91',
+            'episode': 'Mnet Asian Music Awards 2015 - Part 3',
+            'episode_number': 4,
+            'thumbnail': 're:^https?://.*\.jpg',
+            'timestamp': 1450213200,
+            'upload_date': '20151215',
+            'duration': 5602,
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url).replace('/', '.')

From 18c782ab26a53958a7d54d39e06eba0cd920178b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 5 Jan 2016 01:58:25 +0600
Subject: [PATCH 3612/6219] [vrt] Extend _VALUD_URL

---
 youtube_dl/extractor/vrt.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vrt.py b/youtube_dl/extractor/vrt.py
index bbd3bbf7b..ee158b7b3 100644
--- a/youtube_dl/extractor/vrt.py
+++ b/youtube_dl/extractor/vrt.py
@@ -8,7 +8,7 @@ from ..utils import float_or_none
 
 
 class VRTIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:deredactie|sporza|cobra)\.be/cm/(?:[^/]+/)+(?P<id>[^/]+)/*'
+    _VALID_URL = r'https?://(?:deredactie|sporza|cobra(?:\.canvas)?)\.be/cm/(?:[^/]+/)+(?P<id>[^/]+)/*'
     _TESTS = [
         # deredactie.be
         {
@@ -52,6 +52,10 @@ class VRTIE(InfoExtractor):
                 'duration': 661,
             }
         },
+        {
+            'url': 'http://cobra.canvas.be/cm/cobra/videozone/rubriek/film-videozone/1.2377055',
+            'only_matching': True,
+        }
     ]
 
     def _real_extract(self, url):

From 2f546d0a3cc9b9fa7f022df68574f7009d7d1ffb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 5 Jan 2016 01:59:45 +0600
Subject: [PATCH 3613/6219] [vrt] Prefix format ids

---
 youtube_dl/extractor/vrt.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vrt.py b/youtube_dl/extractor/vrt.py
index ee158b7b3..01891ac4c 100644
--- a/youtube_dl/extractor/vrt.py
+++ b/youtube_dl/extractor/vrt.py
@@ -73,11 +73,11 @@ class VRTIE(InfoExtractor):
         if mobj:
             formats.extend(self._extract_m3u8_formats(
                 '%s/%s' % (mobj.group('server'), mobj.group('path')),
-                video_id, 'mp4'))
+                video_id, 'mp4', m3u8_id='hls'))
         mobj = re.search(r'data-video-src="(?P<src>[^"]+)"', webpage)
         if mobj:
             formats.extend(self._extract_f4m_formats(
-                '%s/manifest.f4m' % mobj.group('src'), video_id))
+                '%s/manifest.f4m' % mobj.group('src'), video_id, f4m_id='hds'))
         self._sort_formats(formats)
 
         title = self._og_search_title(webpage)

From 40f796288afe634055c4600993b353da58e29a45 Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Tue, 5 Jan 2016 02:17:12 +0600
Subject: [PATCH 3614/6219] [README.md] Clarify cookies usage

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 4fc83b8e3..75253199c 100644
--- a/README.md
+++ b/README.md
@@ -627,7 +627,7 @@ Either prepend `http://www.youtube.com/watch?v=` or separate the ID from the opt
 
 Use the `--cookies` option, for example `--cookies /path/to/cookies/file.txt`. Note that the cookies file must be in Mozilla/Netscape format and the first line of the cookies file must be either `# HTTP Cookie File` or `# Netscape HTTP Cookie File`. Make sure you have correct [newline format](https://en.wikipedia.org/wiki/Newline) in the cookies file and convert newlines if necessary to correspond with your OS, namely `CRLF` (`\r\n`) for Windows, `LF` (`\n`) for Linux and `CR` (`\r`) for Mac OS. `HTTP Error 400: Bad Request` when using `--cookies` is a good sign of invalid newline format.
 
-Passing cookies to youtube-dl is a good way to workaround login when a particular extractor does not implement it explicitly.
+Passing cookies to youtube-dl is a good way to workaround login when a particular extractor does not implement it explicitly. Another use case is working around [CAPTCHA](https://en.wikipedia.org/wiki/CAPTCHA) some websites require you to solve in particular cases in order to get access (e.g. YouTube, CloudFlare).
 
 ### Can you add support for this anime video site, or site which shows current movies for free?
 

From 17b2d7ca772da5b709e00ca01e96e893807f7b66 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 6 Jan 2016 00:02:21 +0600
Subject: [PATCH 3615/6219] [udemy] Detect non free courses (Closes #8138)

---
 youtube_dl/extractor/udemy.py | 25 +++++++++++++++----------
 1 file changed, 15 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index 1df636779..e19c1f762 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -11,6 +11,7 @@ from ..utils import (
     float_or_none,
     int_or_none,
     sanitized_Request,
+    unescapeHTML,
 )
 
 
@@ -19,8 +20,6 @@ class UdemyIE(InfoExtractor):
     _VALID_URL = r'https?://www\.udemy\.com/(?:[^#]+#/lecture/|lecture/view/?\?lectureId=)(?P<id>\d+)'
     _LOGIN_URL = 'https://www.udemy.com/join/login-popup/?displayType=ajax&showSkipButton=1'
     _ORIGIN_URL = 'https://www.udemy.com'
-    _SUCCESSFULLY_ENROLLED = '>You have enrolled in this course!<'
-    _ALREADY_ENROLLED = '>You are already taking this course.<'
     _NETRC_MACHINE = 'udemy'
 
     _TESTS = [{
@@ -37,15 +36,21 @@ class UdemyIE(InfoExtractor):
     }]
 
     def _enroll_course(self, webpage, course_id):
-        enroll_url = self._search_regex(
+        checkout_url = unescapeHTML(self._search_regex(
+            r'href=(["\'])(?P<url>https?://(?:www\.)?udemy\.com/payment/checkout/.+?)\1',
+            webpage, 'checkout url', group='url', default=None))
+        if checkout_url:
+            raise ExtractorError(
+                'Course %s is not free. You have to pay for it before you can download.'
+                'Use this URL to confirm purchase: %s' % (course_id, checkout_url), expected=True)
+
+        enroll_url = unescapeHTML(self._search_regex(
             r'href=(["\'])(?P<url>https?://(?:www\.)?udemy\.com/course/subscribe/.+?)\1',
-            webpage, 'enroll url', group='url',
-            default='https://www.udemy.com/course/subscribe/?courseId=%s' % course_id)
-        webpage = self._download_webpage(enroll_url, course_id, 'Enrolling in the course')
-        if self._SUCCESSFULLY_ENROLLED in webpage:
-            self.to_screen('%s: Successfully enrolled in' % course_id)
-        elif self._ALREADY_ENROLLED in webpage:
-            self.to_screen('%s: Already enrolled in' % course_id)
+            webpage, 'enroll url', group='url', default=None))
+        if enroll_url:
+            webpage = self._download_webpage(enroll_url, course_id, 'Enrolling in the course')
+            if '>You have enrolled in' in webpage:
+                self.to_screen('%s: Successfully enrolled in the course' % course_id)
 
     def _download_lecture(self, course_id, lecture_id):
         return self._download_json(

From f20756fb10ec560177282f032684327f600acc34 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 6 Jan 2016 00:03:39 +0600
Subject: [PATCH 3616/6219] [udemy] Fix non free course message

---
 youtube_dl/extractor/udemy.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index e19c1f762..f5b5e7fd6 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -41,7 +41,7 @@ class UdemyIE(InfoExtractor):
             webpage, 'checkout url', group='url', default=None))
         if checkout_url:
             raise ExtractorError(
-                'Course %s is not free. You have to pay for it before you can download.'
+                'Course %s is not free. You have to pay for it before you can download. '
                 'Use this URL to confirm purchase: %s' % (course_id, checkout_url), expected=True)
 
         enroll_url = unescapeHTML(self._search_regex(

From 76048b23e8a4aac93a33a96356fe64a9bcf78421 Mon Sep 17 00:00:00 2001
From: kaspi <je326@hotmail.com>
Date: Sat, 17 Oct 2015 23:27:03 -0400
Subject: [PATCH 3617/6219] [npr] Add extractor

removed md5 from _TEST

moved from xml data to json

test

changed _TEST url to one that will not expire, so tests would not be failing
---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/npr.py      | 71 ++++++++++++++++++++++++++++++++
 2 files changed, 72 insertions(+)
 create mode 100644 youtube_dl/extractor/npr.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 19cea5e99..c46e19eae 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -473,6 +473,7 @@ from .npo import (
     VPROIE,
     WNLIE
 )
+from .npr import NprIE
 from .nrk import (
     NRKIE,
     NRKPlaylistIE,
diff --git a/youtube_dl/extractor/npr.py b/youtube_dl/extractor/npr.py
new file mode 100644
index 000000000..a823bc096
--- /dev/null
+++ b/youtube_dl/extractor/npr.py
@@ -0,0 +1,71 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import os.path
+import re
+
+from ..compat import compat_urllib_parse_unquote
+from ..utils import url_basename
+from .common import InfoExtractor
+
+class NprIE(InfoExtractor):
+    _VALID_URL = r'http://(?:www\.)?npr\.org/player/v2/mediaPlayer.html?.*id=(?P<id>[0-9]+)'
+    _TEST = {
+    'url': 'http://www.npr.org/player/v2/mediaPlayer.html?id=449974205',
+    'info_dict': {
+        'id': '449974205',
+        'ext': 'mp4',
+        'title': 'New Music From Beach House, Chairlift, CMJ Discoveries And More'
+    }
+}
+
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        webpage_url = 'http://www.npr.org/player/v2/mediaPlayer.html?id=' + video_id
+        webpage = self._download_webpage(webpage_url, video_id)
+        key = 'MDAzMzQ2MjAyMDEyMzk4MTU1MDg3ZmM3MQ010'
+        xml_url = 'http://api.npr.org/query?id=%s&apiKey=%s' % (video_id, key)
+        json_url = 'http://api.npr.org/query?id=%s&apiKey=%s&format=json' % (video_id, key)
+
+        formats = []
+        entries = []
+
+        config = self._download_json(json_url, video_id)
+
+        content = config["list"]["story"]
+
+        album_title = config["list"]["story"][0]['song'][0]['album']['albumTitle']
+        print album_title['$text']
+
+        for key in content:
+            if "audio" in key:
+                for x in key['audio']:
+                    if x['type'] == 'standard':
+                        playlist = True
+                        song_duration = x["duration"]['$text']
+                        song_title = x["title"]["$text"]
+                        song_id = x["id"]
+
+                        for k in x["format"]:
+                            if type(x["format"][k]) is list:
+                                for z in x["format"][k]:
+                                    formats.append({ 'format': z['type'], 
+                                                     'url'   : z['$text']
+                                              })
+                            else:
+                                formats.append({ 'format': k, 
+                                                 'url'   : x["format"][k]['$text']
+                                      })
+
+                        entries.append({ "title":song_title,
+                                         "id":song_id,
+                                         "duration": song_duration ,
+                                         "formats":formats})
+                        formats = []
+
+        return {    '_type': 'playlist',
+                    'id' : video_id,
+                    'title' : album_title,
+                    'entries': entries  }

From 51d3045de26fccbdcf0076c5cab8ab8152548bba Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 7 Jan 2016 01:57:36 +0600
Subject: [PATCH 3618/6219] [npr] Fix extractor (Closes #7218)

---
 youtube_dl/extractor/npr.py | 133 +++++++++++++++++++-----------------
 1 file changed, 72 insertions(+), 61 deletions(-)

diff --git a/youtube_dl/extractor/npr.py b/youtube_dl/extractor/npr.py
index a823bc096..125c7010b 100644
--- a/youtube_dl/extractor/npr.py
+++ b/youtube_dl/extractor/npr.py
@@ -1,71 +1,82 @@
-# coding: utf-8
 from __future__ import unicode_literals
 
-import os.path
-import re
-
-from ..compat import compat_urllib_parse_unquote
-from ..utils import url_basename
 from .common import InfoExtractor
+from ..compat import compat_urllib_parse
+from ..utils import (
+    int_or_none,
+    qualities,
+)
+
 
 class NprIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?npr\.org/player/v2/mediaPlayer.html?.*id=(?P<id>[0-9]+)'
-    _TEST = {
-    'url': 'http://www.npr.org/player/v2/mediaPlayer.html?id=449974205',
-    'info_dict': {
-        'id': '449974205',
-        'ext': 'mp4',
-        'title': 'New Music From Beach House, Chairlift, CMJ Discoveries And More'
-    }
-}
-
+    _VALID_URL = r'http://(?:www\.)?npr\.org/player/v2/mediaPlayer\.html\?.*\bid=(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'http://www.npr.org/player/v2/mediaPlayer.html?id=449974205',
+        'info_dict': {
+            'id': '449974205',
+            'title': 'New Music From Beach House, Chairlift, CMJ Discoveries And More'
+        },
+        'playlist_count': 7,
+    }, {
+        'url': 'http://www.npr.org/player/v2/mediaPlayer.html?action=1&t=1&islist=false&id=446928052&m=446929930&live=1',
+        'info_dict': {
+            'id': '446928052',
+            'title': "Songs We Love: Tigran Hamasyan, 'Your Mercy is Boundless'"
+        },
+        'playlist': [{
+            'md5': '12fa60cb2d3ed932f53609d4aeceabf1',
+            'info_dict': {
+                'id': '446929930',
+                'ext': 'mp3',
+                'title': 'Your Mercy is Boundless (Bazum en Qo gtutyunqd)',
+                'duration': 402,
+            },
+        }],
+    }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        webpage_url = 'http://www.npr.org/player/v2/mediaPlayer.html?id=' + video_id
-        webpage = self._download_webpage(webpage_url, video_id)
-        key = 'MDAzMzQ2MjAyMDEyMzk4MTU1MDg3ZmM3MQ010'
-        xml_url = 'http://api.npr.org/query?id=%s&apiKey=%s' % (video_id, key)
-        json_url = 'http://api.npr.org/query?id=%s&apiKey=%s&format=json' % (video_id, key)
+        playlist_id = self._match_id(url)
+
+        config = self._download_json(
+            'http://api.npr.org/query?%s' % compat_urllib_parse.urlencode({
+                'id': playlist_id,
+                'fields': 'titles,audio,show',
+                'format': 'json',
+                'apiKey': 'MDAzMzQ2MjAyMDEyMzk4MTU1MDg3ZmM3MQ010',
+            }), playlist_id)
+
+        story = config['list']['story'][0]
+
+        KNOWN_FORMATS = ('threegp', 'mp4', 'mp3')
+        quality = qualities(KNOWN_FORMATS)
 
-        formats = []
         entries = []
+        for audio in story.get('audio', []):
+            title = audio.get('title', {}).get('$text')
+            duration = int_or_none(audio.get('duration', {}).get('$text'))
+            formats = []
+            for format_id, formats_entry in audio.get('format', {}).items():
+                if not formats_entry:
+                    continue
+                if isinstance(formats_entry, list):
+                    formats_entry = formats_entry[0]
+                format_url = formats_entry.get('$text')
+                if not format_url:
+                    continue
+                if format_id in KNOWN_FORMATS:
+                    formats.append({
+                        'url': format_url,
+                        'format_id': format_id,
+                        'ext': formats_entry.get('type'),
+                        'quality': quality(format_id),
+                    })
+            self._sort_formats(formats)
+            entries.append({
+                'id': audio['id'],
+                'title': title,
+                'duration': duration,
+                'formats': formats,
+            })
 
-        config = self._download_json(json_url, video_id)
-
-        content = config["list"]["story"]
-
-        album_title = config["list"]["story"][0]['song'][0]['album']['albumTitle']
-        print album_title['$text']
-
-        for key in content:
-            if "audio" in key:
-                for x in key['audio']:
-                    if x['type'] == 'standard':
-                        playlist = True
-                        song_duration = x["duration"]['$text']
-                        song_title = x["title"]["$text"]
-                        song_id = x["id"]
-
-                        for k in x["format"]:
-                            if type(x["format"][k]) is list:
-                                for z in x["format"][k]:
-                                    formats.append({ 'format': z['type'], 
-                                                     'url'   : z['$text']
-                                              })
-                            else:
-                                formats.append({ 'format': k, 
-                                                 'url'   : x["format"][k]['$text']
-                                      })
-
-                        entries.append({ "title":song_title,
-                                         "id":song_id,
-                                         "duration": song_duration ,
-                                         "formats":formats})
-                        formats = []
-
-        return {    '_type': 'playlist',
-                    'id' : video_id,
-                    'title' : album_title,
-                    'entries': entries  }
+        playlist_title = story.get('title', {}).get('$text')
+        return self.playlist_result(entries, playlist_id, playlist_title)

From e4f49a87533bd6e3ebf11b43c3baf0e32db2f5ca Mon Sep 17 00:00:00 2001
From: oittaa <oittaa@users.noreply.github.com>
Date: Thu, 7 Jan 2016 07:26:14 +0200
Subject: [PATCH 3619/6219] check video_play_path and use xpath_text

"This check should take place earlier and should be more general if not video_url:. Same should be done for video_play_path. Also these fields better extracted with xpath_text."

Suggestions by @dstftw
---
 youtube_dl/extractor/crunchyroll.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 00d943f77..785594df8 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -329,8 +329,10 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
                 streamdata_req, video_id,
                 note='Downloading media info for %s' % video_format)
             stream_info = streamdata.find('./{default}preload/stream_info')
-            video_url = stream_info.find('./host').text
-            video_play_path = stream_info.find('./file').text
+            video_url = xpath_text(stream_info, './host')
+            video_play_path = xpath_text(stream_info, './file')
+            if not video_url or not video_play_path:
+                continue
             metadata = stream_info.find('./metadata')
             format_info = {
                 'format': video_format,

From 2e02ecbccc9751af54ae2aab9ce973712c78af74 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 7 Jan 2016 12:24:32 +0600
Subject: [PATCH 3620/6219] [ivideon] Add extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/ivideon.py  | 75 ++++++++++++++++++++++++++++++++
 2 files changed, 76 insertions(+)
 create mode 100644 youtube_dl/extractor/ivideon.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index c46e19eae..a9d23b8f4 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -299,6 +299,7 @@ from .ivi import (
     IviIE,
     IviCompilationIE
 )
+from .ivideon import IvideonIE
 from .izlesene import IzleseneIE
 from .jadorecettepub import JadoreCettePubIE
 from .jeuxvideo import JeuxVideoIE
diff --git a/youtube_dl/extractor/ivideon.py b/youtube_dl/extractor/ivideon.py
new file mode 100644
index 000000000..aa05f5413
--- /dev/null
+++ b/youtube_dl/extractor/ivideon.py
@@ -0,0 +1,75 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..compat import compat_urllib_parse
+from ..utils import qualities
+
+
+class IvideonIE(InfoExtractor):
+    IE_NAME = 'ivideon'
+    IE_DESC = 'Ivideon TV'
+    _VALID_URL = r'https?://(?:www\.)?ivideon\.com/tv/camera/(?P<id>\d+-[\da-f]+)/(?P<camera_id>\d+)'
+    _TESTS = [{
+        'url': 'https://www.ivideon.com/tv/camera/100-916ca13b5c4ad9f564266424a026386d/0/',
+        'info_dict': {
+            'id': '100-916ca13b5c4ad9f564266424a026386d',
+            'ext': 'flv',
+            'title': 're:^Касса [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
+            'description': 'Основное предназначение - запись действий кассиров. Плюс общий вид.',
+            'is_live': True,
+        },
+        'params': {
+            'skip_download': True,
+        }
+    }, {
+        'url': 'https://www.ivideon.com/tv/camera/100-c4ee4cb9ede885cf62dfbe93d7b53783/589824/?lang=ru',
+        'only_matching': True,
+    }]
+
+    _QUALITIES = ('low', 'mid', 'hi')
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        server_id, camera_id = mobj.group('id'), mobj.group('camera_id')
+        camera_name, description = None, None
+
+        webpage = self._download_webpage(url, server_id, fatal=False)
+        if webpage:
+            config_string = self._search_regex(
+                r'var\s+config\s*=\s*({.+?});', webpage, 'config', default=None)
+            if config_string:
+                config = self._parse_json(config_string, server_id, fatal=False)
+                camera_info = config.get('ivTvAppOptions', {}).get('currentCameraInfo')
+                if camera_info:
+                    camera_name = camera_info.get('camera_name')
+                    description = camera_info.get('misc', {}).get('description')
+            if not camera_name:
+                camera_name = self._html_search_meta(
+                    'name', webpage, 'camera name', default=None) or self._search_regex(
+                    r'<h1[^>]+class="b-video-title"[^>]*>([^<]+)', webpage, 'camera name', default=None)
+
+        quality = qualities(self._QUALITIES)
+
+        formats = [{
+            'url': 'https://streaming.ivideon.com/flv/live?%s' % compat_urllib_parse.urlencode({
+                'server': server_id,
+                'camera': camera_id,
+                'sessionId': 'demo',
+                'q': quality(format_id),
+            }),
+            'format_id': format_id,
+            'ext': 'flv',
+            'quality': quality(format_id),
+        } for format_id in self._QUALITIES]
+        self._sort_formats(formats)
+
+        return {
+            'id': server_id,
+            'title': self._live_title(camera_name or server_id),
+            'description': description,
+            'is_live': True,
+            'formats': formats,
+        }

From 23f13e97548cc4ff3d11408ee5bc77f682e642dc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 8 Jan 2016 00:52:55 +0600
Subject: [PATCH 3621/6219] [youtube] Support expanding alternative format of
 links in description (Closes #8164)

---
 youtube_dl/extractor/youtube.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 4aac2cc03..897641e42 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1235,10 +1235,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             video_description = re.sub(r'''(?x)
                 <a\s+
                     (?:[a-zA-Z-]+="[^"]+"\s+)*?
-                    title="([^"]+)"\s+
+                    (?:title|href)="([^"]+)"\s+
                     (?:[a-zA-Z-]+="[^"]+"\s+)*?
-                    class="yt-uix-redirect-link"\s*>
-                [^<]+
+                    class="(?:yt-uix-redirect-link|yt-uix-sessionlink[^"]*)".*?>
+                [^<]+\.{3}\s*
                 </a>
             ''', r'\1', video_description)
             video_description = clean_html(video_description)

From 97afd99a18e4723e4ff588df456c7aec62967b35 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 8 Jan 2016 01:54:31 +0600
Subject: [PATCH 3622/6219] [soundcloud:likes] Adapt to API changes (Closes
 #8166)

---
 youtube_dl/extractor/soundcloud.py | 38 +++++++++++++++---------------
 1 file changed, 19 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 02e64e094..b2d5487ca 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -384,27 +384,24 @@ class SoundcloudUserIE(SoundcloudIE):
         resource = mobj.group('rsrc') or 'all'
         base_url = self._BASE_URL_MAP[resource] % user['id']
 
-        next_href = None
+        COMMON_QUERY = {
+            'limit': 50,
+            'client_id': self._CLIENT_ID,
+            'linked_partitioning': '1',
+        }
+
+        query = COMMON_QUERY.copy()
+        query['offset'] = 0
+
+        next_href = base_url + '?' + compat_urllib_parse.urlencode(query)
 
         entries = []
         for i in itertools.count():
-            if not next_href:
-                data = compat_urllib_parse.urlencode({
-                    'offset': i * 50,
-                    'limit': 50,
-                    'client_id': self._CLIENT_ID,
-                    'linked_partitioning': '1',
-                    'representation': 'speedy',
-                })
-                next_href = base_url + '?' + data
-
             response = self._download_json(
                 next_href, uploader, 'Downloading track page %s' % (i + 1))
 
             collection = response['collection']
-
             if not collection:
-                self.to_screen('%s: End page received' % uploader)
                 break
 
             def resolve_permalink_url(candidates):
@@ -419,12 +416,15 @@ class SoundcloudUserIE(SoundcloudIE):
                 if permalink_url:
                     entries.append(self.url_result(permalink_url))
 
-            if 'next_href' in response:
-                next_href = response['next_href']
-                if not next_href:
-                    break
-            else:
-                next_href = None
+            next_href = response.get('next_href')
+            if not next_href:
+                break
+
+            parsed_next_href = compat_urlparse.urlparse(response['next_href'])
+            qs = compat_urlparse.parse_qs(parsed_next_href.query)
+            qs.update(COMMON_QUERY)
+            next_href = compat_urlparse.urlunparse(
+                parsed_next_href._replace(query=compat_urllib_parse.urlencode(qs, True)))
 
         return {
             '_type': 'playlist',

From 18e6c97c48f883911649d9b3d64127379a8b1df4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 8 Jan 2016 03:19:47 +0600
Subject: [PATCH 3623/6219] [adultswim] Skip georestricted hls (Closes #8168)

---
 youtube_dl/extractor/adultswim.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/adultswim.py b/youtube_dl/extractor/adultswim.py
index bf21a6887..8157da2cb 100644
--- a/youtube_dl/extractor/adultswim.py
+++ b/youtube_dl/extractor/adultswim.py
@@ -187,7 +187,8 @@ class AdultSwimIE(InfoExtractor):
                 media_url = file_el.text
                 if determine_ext(media_url) == 'm3u8':
                     formats.extend(self._extract_m3u8_formats(
-                        media_url, segment_title, 'mp4', preference=0, m3u8_id='hls'))
+                        media_url, segment_title, 'mp4', preference=0,
+                        m3u8_id='hls', fatal=False))
                 else:
                     formats.append({
                         'format_id': '%s_%s' % (bitrate, ftype),

From 4cf096a4a99c2cc29708f23244e4433d91c83b65 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 8 Jan 2016 05:11:23 +0600
Subject: [PATCH 3624/6219] [ivideon] Add support for map bound URLs

---
 youtube_dl/extractor/ivideon.py | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/ivideon.py b/youtube_dl/extractor/ivideon.py
index aa05f5413..617dc8c07 100644
--- a/youtube_dl/extractor/ivideon.py
+++ b/youtube_dl/extractor/ivideon.py
@@ -4,14 +4,17 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse
+from ..compat import (
+    compat_urllib_parse,
+    compat_urlparse,
+)
 from ..utils import qualities
 
 
 class IvideonIE(InfoExtractor):
     IE_NAME = 'ivideon'
     IE_DESC = 'Ivideon TV'
-    _VALID_URL = r'https?://(?:www\.)?ivideon\.com/tv/camera/(?P<id>\d+-[\da-f]+)/(?P<camera_id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?ivideon\.com/tv/(?:[^/]+/)*camera/(?P<id>\d+-[\da-f]+)/(?P<camera_id>\d+)'
     _TESTS = [{
         'url': 'https://www.ivideon.com/tv/camera/100-916ca13b5c4ad9f564266424a026386d/0/',
         'info_dict': {
@@ -27,6 +30,9 @@ class IvideonIE(InfoExtractor):
     }, {
         'url': 'https://www.ivideon.com/tv/camera/100-c4ee4cb9ede885cf62dfbe93d7b53783/589824/?lang=ru',
         'only_matching': True,
+    }, {
+        'url': 'https://www.ivideon.com/tv/map/22.917923/-31.816406/16/camera/100-e7bc16c7d4b5bbd633fd5350b66dfa9a/0',
+        'only_matching': True,
     }]
 
     _QUALITIES = ('low', 'mid', 'hi')
@@ -35,8 +41,10 @@ class IvideonIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         server_id, camera_id = mobj.group('id'), mobj.group('camera_id')
         camera_name, description = None, None
+        camera_url = compat_urlparse.urljoin(
+            url, '/tv/camera/%s/%s/' % (server_id, camera_id))
 
-        webpage = self._download_webpage(url, server_id, fatal=False)
+        webpage = self._download_webpage(camera_url, server_id, fatal=False)
         if webpage:
             config_string = self._search_regex(
                 r'var\s+config\s*=\s*({.+?});', webpage, 'config', default=None)

From 5dbe81a1d35ae704b5ea208698a6bb785923d71a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 8 Jan 2016 10:41:24 +0600
Subject: [PATCH 3625/6219] [vimeo] Automatically pickup full movie when rented
 (Closes #8171)

---
 youtube_dl/extractor/vimeo.py | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 7af699982..776e8cce4 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -217,7 +217,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
             r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//player\.vimeo\.com/video/.+?)\1', webpage)
         if mobj:
             player_url = unescapeHTML(mobj.group('url'))
-            surl = smuggle_url(player_url, {'Referer': url})
+            surl = smuggle_url(player_url, {'http_headers': {'Referer': url}})
             return surl
         # Look for embedded (swf embed) Vimeo player
         mobj = re.search(
@@ -262,11 +262,11 @@ class VimeoIE(VimeoBaseInfoExtractor):
         self._login()
 
     def _real_extract(self, url):
-        url, data = unsmuggle_url(url)
+        url, data = unsmuggle_url(url, {})
         headers = std_headers
-        if data is not None:
+        if 'http_headers' in data:
             headers = headers.copy()
-            headers.update(data)
+            headers.update(data['http_headers'])
         if 'Referer' not in headers:
             headers['Referer'] = url
 
@@ -342,7 +342,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
                 raise ExtractorError('The author has restricted the access to this video, try with the "--referer" option')
 
             if re.search(r'<form[^>]+?id="pw_form"', webpage) is not None:
-                if data and '_video_password_verified' in data:
+                if '_video_password_verified' in data:
                     raise ExtractorError('video password verification failed!')
                 self._verify_video_password(url, video_id, webpage)
                 return self._real_extract(
@@ -354,6 +354,13 @@ class VimeoIE(VimeoBaseInfoExtractor):
             if config.get('view') == 4:
                 config = self._verify_player_video_password(url, video_id)
 
+        if '>You rented this title.<' in webpage:
+            feature_id = config.get('video', {}).get('vod', {}).get('feature_id')
+            if feature_id and not data.get('force_feature_id', False):
+                return self.url_result(smuggle_url(
+                    'https://player.vimeo.com/player/%s' % feature_id,
+                    {'force_feature_id': True}), 'Vimeo')
+
         # Extract title
         video_title = config["video"]["title"]
 

From 5777f5d386d14407a19bc86c31e0bf2b5ae4a87f Mon Sep 17 00:00:00 2001
From: atomic83 <atomic83@riseup.net>
Date: Fri, 8 Jan 2016 12:58:05 +0100
Subject: [PATCH 3626/6219] Extract xHamster title fix

---
 youtube_dl/extractor/xhamster.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index 8938c0e45..261d323a7 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -65,7 +65,7 @@ class XHamsterIE(InfoExtractor):
 
         title = self._html_search_regex(
             [r'<title>(?P<title>.+?)(?:, (?:[^,]+? )?Porn: xHamster| - xHamster\.com)</title>',
-             r'<h1>([^<]+)</h1>'], webpage, 'title')
+             r'<h1(?: itemprop="name")?>([^<]+)</h1>'], webpage, 'title')
 
         # Only a few videos have an description
         mobj = re.search(r'<span>Description: </span>([^<]+)', webpage)

From 6609b3ce371d08c0464a3caa30c21b61de221508 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 9 Jan 2016 00:19:36 +0600
Subject: [PATCH 3627/6219] [xhamster] Improve title extraction

---
 youtube_dl/extractor/xhamster.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index 261d323a7..97355d17f 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -64,8 +64,9 @@ class XHamsterIE(InfoExtractor):
         webpage = self._download_webpage(mrss_url, video_id)
 
         title = self._html_search_regex(
-            [r'<title>(?P<title>.+?)(?:, (?:[^,]+? )?Porn: xHamster| - xHamster\.com)</title>',
-             r'<h1(?: itemprop="name")?>([^<]+)</h1>'], webpage, 'title')
+            [r'<title[^>]*>(.+?)(?:,\s*[^,]*?\s*Porn\s*[^,]*?:\s*xHamster[^<]*| - xHamster\.com)</title>',
+             r'<h1[^>]*>([^<]+)</h1>',
+             r'<meta[^>]+itemprop=".*?caption.*?"[^>]+content="(.+?)"'], webpage, 'title')
 
         # Only a few videos have an description
         mobj = re.search(r'<span>Description: </span>([^<]+)', webpage)

From 4763b624a6655bc2333157031c73858f6b918f61 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 9 Jan 2016 00:21:57 +0600
Subject: [PATCH 3628/6219] [xhamster] Fix upload date extraction

---
 youtube_dl/extractor/xhamster.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index 97355d17f..560c38e26 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -72,10 +72,9 @@ class XHamsterIE(InfoExtractor):
         mobj = re.search(r'<span>Description: </span>([^<]+)', webpage)
         description = mobj.group(1) if mobj else None
 
-        upload_date = self._html_search_regex(r'hint=\'(\d{4}-\d{2}-\d{2}) \d{2}:\d{2}:\d{2} [A-Z]{3,4}\'',
-                                              webpage, 'upload date', fatal=False)
-        if upload_date:
-            upload_date = unified_strdate(upload_date)
+        upload_date = unified_strdate(self._search_regex(
+            r'hint=["\'](\d{4}-\d{2}-\d{2}) \d{2}:\d{2}:\d{2} [A-Z]{3,4}',
+            webpage, 'upload date', fatal=False))
 
         uploader = self._html_search_regex(
             r"<a href='[^']+xhamster\.com/user/[^>]+>(?P<uploader>[^<]+)",

From 44731e308cdcc89fda1e613e094aa23de33f5c9e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 9 Jan 2016 00:26:37 +0600
Subject: [PATCH 3629/6219] [xhamster] Fix duration extraction

---
 youtube_dl/extractor/xhamster.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index 560c38e26..2b9ac2419 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -4,10 +4,10 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
-    unified_strdate,
-    str_to_int,
+    float_or_none,
     int_or_none,
-    parse_duration,
+    str_to_int,
+    unified_strdate,
 )
 
 
@@ -85,8 +85,9 @@ class XHamsterIE(InfoExtractor):
              r'''<video[^>]+poster=(?P<q>["'])(?P<thumbnail>.+?)(?P=q)[^>]*>'''],
             webpage, 'thumbnail', fatal=False, group='thumbnail')
 
-        duration = parse_duration(self._html_search_regex(r'<span>Runtime:</span> (\d+:\d+)</div>',
-                                                          webpage, 'duration', fatal=False))
+        duration = float_or_none(self._search_regex(
+            r'(["\'])duration\1\s*:\s*(["\'])(?P<duration>.+?)\2',
+            webpage, 'duration', fatal=False, group='duration'))
 
         view_count = self._html_search_regex(r'<span>Views:</span> ([^<]+)</div>', webpage, 'view count', fatal=False)
         if view_count:

From 6a16fd4a1ad1dbd9372f75ddffce1e9fe95b002c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 9 Jan 2016 00:29:10 +0600
Subject: [PATCH 3630/6219] [xhamster] Fix view count extraction

---
 youtube_dl/extractor/xhamster.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index 2b9ac2419..ccee77359 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -89,9 +89,9 @@ class XHamsterIE(InfoExtractor):
             r'(["\'])duration\1\s*:\s*(["\'])(?P<duration>.+?)\2',
             webpage, 'duration', fatal=False, group='duration'))
 
-        view_count = self._html_search_regex(r'<span>Views:</span> ([^<]+)</div>', webpage, 'view count', fatal=False)
-        if view_count:
-            view_count = str_to_int(view_count)
+        view_count = int_or_none(self._search_regex(
+            r'content=["\']User(?:View|Play)s:(\d+)',
+            webpage, 'view count', fatal=False))
 
         mobj = re.search(r"hint='(?P<likecount>\d+) Likes / (?P<dislikecount>\d+) Dislikes'", webpage)
         (like_count, dislike_count) = (mobj.group('likecount'), mobj.group('dislikecount')) if mobj else (None, None)

From 1a6d92847f4e5fe1d94f2f159f903a414d5ff62c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 9 Jan 2016 00:31:24 +0600
Subject: [PATCH 3631/6219] [xhamster] Change title regex precedence

---
 youtube_dl/extractor/xhamster.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index ccee77359..c9108c345 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -64,9 +64,10 @@ class XHamsterIE(InfoExtractor):
         webpage = self._download_webpage(mrss_url, video_id)
 
         title = self._html_search_regex(
-            [r'<title[^>]*>(.+?)(?:,\s*[^,]*?\s*Porn\s*[^,]*?:\s*xHamster[^<]*| - xHamster\.com)</title>',
-             r'<h1[^>]*>([^<]+)</h1>',
-             r'<meta[^>]+itemprop=".*?caption.*?"[^>]+content="(.+?)"'], webpage, 'title')
+            [r'<h1[^>]*>([^<]+)</h1>',
+             r'<meta[^>]+itemprop=".*?caption.*?"[^>]+content="(.+?)"',
+             r'<title[^>]*>(.+?)(?:,\s*[^,]*?\s*Porn\s*[^,]*?:\s*xHamster[^<]*| - xHamster\.com)</title>'],
+            webpage, 'title')
 
         # Only a few videos have an description
         mobj = re.search(r'<span>Description: </span>([^<]+)', webpage)

From bcac2a071040fd89585a806ca66e086c109406c1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 9 Jan 2016 00:36:19 +0600
Subject: [PATCH 3632/6219] [xhamster] Fix uploader extraction

---
 youtube_dl/extractor/xhamster.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index c9108c345..981ee284f 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -78,7 +78,7 @@ class XHamsterIE(InfoExtractor):
             webpage, 'upload date', fatal=False))
 
         uploader = self._html_search_regex(
-            r"<a href='[^']+xhamster\.com/user/[^>]+>(?P<uploader>[^<]+)",
+            r'<span[^>]+itemprop=["\']author[^>]+><a[^>]+href=["\'].+?xhamster\.com/user/[^>]+>(?P<uploader>.+?)</a>',
             webpage, 'uploader', default='anonymous')
 
         thumbnail = self._search_regex(

From 14b4f038c0123138a642a25bf6a3851dd610d1d7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 9 Jan 2016 00:36:43 +0600
Subject: [PATCH 3633/6219] [xhamster] Update tests

---
 youtube_dl/extractor/xhamster.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index 981ee284f..066af184f 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -22,7 +22,7 @@ class XHamsterIE(InfoExtractor):
                 'title': 'FemaleAgent Shy beauty takes the bait',
                 'upload_date': '20121014',
                 'uploader': 'Ruseful2011',
-                'duration': 893,
+                'duration': 893.52,
                 'age_limit': 18,
             }
         },
@@ -34,7 +34,7 @@ class XHamsterIE(InfoExtractor):
                 'title': 'Britney Spears  Sexy Booty',
                 'upload_date': '20130914',
                 'uploader': 'jojo747400',
-                'duration': 200,
+                'duration': 200.48,
                 'age_limit': 18,
             }
         },

From d5f071afb5ce302058d99a14989f3f6afb88b1a7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 9 Jan 2016 03:06:09 +0600
Subject: [PATCH 3634/6219] [vimeo] Check source file URL (Closes #8187)

---
 youtube_dl/extractor/vimeo.py | 26 ++++++++++++++++----------
 1 file changed, 16 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 776e8cce4..5cb01907b 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -11,6 +11,7 @@ from ..compat import (
     compat_urlparse,
 )
 from ..utils import (
+    determine_ext,
     encode_dict,
     ExtractorError,
     InAdvancePagedList,
@@ -419,16 +420,21 @@ class VimeoIE(VimeoBaseInfoExtractor):
         download_data = self._download_json(download_request, video_id, fatal=False)
         if download_data:
             source_file = download_data.get('source_file')
-            if source_file and not source_file.get('is_cold') and not source_file.get('is_defrosting'):
-                formats.append({
-                    'url': source_file['download_url'],
-                    'ext': source_file['extension'].lower(),
-                    'width': int_or_none(source_file.get('width')),
-                    'height': int_or_none(source_file.get('height')),
-                    'filesize': parse_filesize(source_file.get('size')),
-                    'format_id': source_file.get('public_name', 'Original'),
-                    'preference': 1,
-                })
+            if isinstance(source_file, dict):
+                download_url = source_file.get('download_url')
+                if download_url and not source_file.get('is_cold') and not source_file.get('is_defrosting'):
+                    source_name = source_file.get('public_name', 'Original')
+                    if self._is_valid_url(download_url, video_id, '%s video' % source_name):
+                        ext = source_file.get('extension', determine_ext(download_url)).lower(),
+                        formats.append({
+                            'url': download_url,
+                            'ext': ext,
+                            'width': int_or_none(source_file.get('width')),
+                            'height': int_or_none(source_file.get('height')),
+                            'filesize': parse_filesize(source_file.get('size')),
+                            'format_id': source_name,
+                            'preference': 1,
+                        })
         config_files = config['video'].get('files') or config['request'].get('files', {})
         for f in config_files.get('progressive', []):
             video_url = f.get('url')

From 16f1131a4d46c4437485d4b075e9d1243ca8d60c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 9 Jan 2016 03:07:29 +0600
Subject: [PATCH 3635/6219] [vimeo] Add test for #8187

---
 youtube_dl/extractor/vimeo.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 5cb01907b..76e681bc3 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -209,6 +209,11 @@ class VimeoIE(VimeoBaseInfoExtractor):
             'url': 'https://vimeo.com/groups/travelhd/videos/22439234',
             'only_matching': True,
         },
+        {
+            # source file returns 403: Forbidden
+            'url': 'https://vimeo.com/7809605',
+            'only_matching': True,
+        },
     ]
 
     @staticmethod

From b374af6ebdf6298ab1593bee56dba5b0a1daf9b7 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 9 Jan 2016 01:16:08 +0100
Subject: [PATCH 3636/6219] release 2016.01.09

---
 docs/supportedsites.md | 10 +++++++---
 youtube_dl/version.py  |  2 +-
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 84c166805..8d0c7b97a 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -1,6 +1,7 @@
 # Supported sites
  - **1tv**: Первый канал
  - **1up.com**
+ - **20min**
  - **220.ro**
  - **22tracks:genre**
  - **22tracks:track**
@@ -255,6 +256,7 @@
  - **Ir90Tv**
  - **ivi**: ivi.ru
  - **ivi:compilation**: ivi.ru compilations
+ - **ivideon**: Ivideon TV
  - **Izlesene**
  - **JadoreCettePub**
  - **JeuxVideo**
@@ -386,13 +388,14 @@
  - **nowness**
  - **nowness:playlist**
  - **nowness:series**
- - **NowTV**
+ - **NowTV** (Currently broken)
  - **NowTVList**
  - **nowvideo**: NowVideo
  - **npo**: npo.nl and ntr.nl
  - **npo.nl:live**
  - **npo.nl:radio**
  - **npo.nl:radio:fragment**
+ - **Npr**
  - **NRK**
  - **NRKPlaylist**
  - **NRKTV**: NRK TV and NRK Radio
@@ -464,11 +467,13 @@
  - **RegioTV**
  - **Restudy**
  - **ReverbNation**
+ - **Revision3**
  - **RingTV**
  - **RottenTomatoes**
  - **Roxwel**
  - **RTBF**
- - **Rte**
+ - **rte**: Raidió Teilifís Éireann TV
+ - **rte:radio**: Raidió Teilifís Éireann radio
  - **rtl.nl**: rtl.nl and rtlxl.nl
  - **RTL2**
  - **RTP**
@@ -573,7 +578,6 @@
  - **TeleMB**
  - **TeleTask**
  - **TenPlay**
- - **TestTube**
  - **TF1**
  - **TheIntercept**
  - **TheOnion**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 790bd5b3b..7030903c0 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.01.01'
+__version__ = '2016.01.09'

From 27783821af5a85fb20266834cff4f07eb0695cb5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 9 Jan 2016 11:16:23 +0100
Subject: [PATCH 3637/6219] [xhamster] Remove unused import

---
 youtube_dl/extractor/xhamster.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index 066af184f..fd43e8854 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -6,7 +6,6 @@ from .common import InfoExtractor
 from ..utils import (
     float_or_none,
     int_or_none,
-    str_to_int,
     unified_strdate,
 )
 

From 7a34302e95ee66f770b1eba720847831c40121a2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 10 Jan 2016 01:37:10 +0600
Subject: [PATCH 3638/6219] [canalc2] Fix extraction (Closes #8191)

---
 youtube_dl/extractor/canalc2.py | 46 +++++++++++++++++++++------------
 1 file changed, 29 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/canalc2.py b/youtube_dl/extractor/canalc2.py
index f6a1ff381..f1f128c45 100644
--- a/youtube_dl/extractor/canalc2.py
+++ b/youtube_dl/extractor/canalc2.py
@@ -9,9 +9,9 @@ from ..utils import parse_duration
 
 class Canalc2IE(InfoExtractor):
     IE_NAME = 'canalc2.tv'
-    _VALID_URL = r'https?://(?:www\.)?canalc2\.tv/video/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:(?:www\.)?canalc2\.tv/video/|archives-canalc2\.u-strasbg\.fr/video\.asp\?.*\bidVideo=)(?P<id>\d+)'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.canalc2.tv/video/12163',
         'md5': '060158428b650f896c542dfbb3d6487f',
         'info_dict': {
@@ -23,24 +23,36 @@ class Canalc2IE(InfoExtractor):
         'params': {
             'skip_download': True,  # Requires rtmpdump
         }
-    }
+    }, {
+        'url': 'http://archives-canalc2.u-strasbg.fr/video.asp?idVideo=11427&voir=oui',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-        video_url = self._search_regex(
-            r'jwplayer\((["\'])Player\1\)\.setup\({[^}]*file\s*:\s*(["\'])(?P<file>.+?)\2',
-            webpage, 'video_url', group='file')
-        formats = [{'url': video_url}]
-        if video_url.startswith('rtmp://'):
-            rtmp = re.search(r'^(?P<url>rtmp://[^/]+/(?P<app>.+/))(?P<play_path>mp4:.+)$', video_url)
-            formats[0].update({
-                'url': rtmp.group('url'),
-                'ext': 'flv',
-                'app': rtmp.group('app'),
-                'play_path': rtmp.group('play_path'),
-                'page_url': url,
-            })
+
+        webpage = self._download_webpage(
+            'http://www.canalc2.tv/video/%s' % video_id, video_id)
+
+        formats = []
+        for _, video_url in re.findall(r'file\s*=\s*(["\'])(.+?)\1', webpage):
+            if video_url.startswith('rtmp://'):
+                rtmp = re.search(
+                    r'^(?P<url>rtmp://[^/]+/(?P<app>.+/))(?P<play_path>mp4:.+)$', video_url)
+                formats.append({
+                    'url': rtmp.group('url'),
+                    'format_id': 'rtmp',
+                    'ext': 'flv',
+                    'app': rtmp.group('app'),
+                    'play_path': rtmp.group('play_path'),
+                    'page_url': url,
+                })
+            else:
+                formats.append({
+                    'url': video_url,
+                    'format_id': 'http',
+                })
+        self._sort_formats(formats)
 
         title = self._html_search_regex(
             r'(?s)class="[^"]*col_description[^"]*">.*?<h3>(.*?)</h3>', webpage, 'title')

From 0a899a1448328648927e18e43c6e2274d2706396 Mon Sep 17 00:00:00 2001
From: Vignesh Venkat <vigneshacitizenofindia@gmail.com>
Date: Sat, 9 Jan 2016 15:31:50 -0800
Subject: [PATCH 3639/6219] [Bigflix] Add new extractor for bigflix.com

Add an IE to support bigflix.com. It uses some sort of silverlight
plugin whose video url is being populated using base64 encoded
flashvars. So it is quite straightforward to extract.
---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/bigflix.py  | 42 ++++++++++++++++++++++++++++++++
 2 files changed, 43 insertions(+)
 create mode 100644 youtube_dl/extractor/bigflix.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index a9d23b8f4..40c42d4d2 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -61,6 +61,7 @@ from .beeg import BeegIE
 from .behindkink import BehindKinkIE
 from .beatportpro import BeatportProIE
 from .bet import BetIE
+from .bigflix import BigflixIE
 from .bild import BildIE
 from .bilibili import BiliBiliIE
 from .bleacherreport import (
diff --git a/youtube_dl/extractor/bigflix.py b/youtube_dl/extractor/bigflix.py
new file mode 100644
index 000000000..aeea1a002
--- /dev/null
+++ b/youtube_dl/extractor/bigflix.py
@@ -0,0 +1,42 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from base64 import b64decode
+
+from .common import InfoExtractor
+from ..compat import compat_urllib_parse_unquote
+
+
+class BigflixIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?bigflix\.com/.*/(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'http://www.bigflix.com/Hindi-movies/Action-movies/Singham-Returns/16537',
+        'md5': 'ec76aa9b1129e2e5b301a474e54fab74',
+        'info_dict': {
+            'id': '16537',
+            'ext': 'mp4',
+            'title': 'Singham Returns',
+            'description': 'md5:3d2ba5815f14911d5cc6a501ae0cf65d',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        title = self._html_search_regex(
+            r'<div[^>]+class=["\']pagetitle["\'][^>]*>(.+?)</div>',
+            webpage, 'title')
+
+        video_url = b64decode(compat_urllib_parse_unquote(self._search_regex(
+            r'file=([^&]+)', webpage, 'video url')).encode('ascii')).decode('utf-8')
+
+        description = self._html_search_meta('description', webpage)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'url': video_url,
+            'description': description,
+        }

From 15b1c6656fd8b384bd974f4d943a04b8bf8ca915 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 10 Jan 2016 10:03:56 +0600
Subject: [PATCH 3640/6219] Credit @vickyg3 for bigflix (#8194)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index e39d07efe..3d8bebbb0 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -151,3 +151,4 @@ Muratcan Simsek
 Evan Lu
 flatgreen
 Brian Foley
+Vignesh Venkat

From 6e99d5762a844b44cad4cae144045ed0537c084b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 10 Jan 2016 10:31:36 +0600
Subject: [PATCH 3641/6219] [bigflix] Extract all formats

---
 youtube_dl/extractor/bigflix.py | 42 +++++++++++++++++++++++++++------
 1 file changed, 35 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/bigflix.py b/youtube_dl/extractor/bigflix.py
index aeea1a002..b7e498436 100644
--- a/youtube_dl/extractor/bigflix.py
+++ b/youtube_dl/extractor/bigflix.py
@@ -1,15 +1,16 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-from base64 import b64decode
+import base64
+import re
 
 from .common import InfoExtractor
 from ..compat import compat_urllib_parse_unquote
 
 
 class BigflixIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?bigflix\.com/.*/(?P<id>[0-9]+)'
-    _TEST = {
+    _VALID_URL = r'https?://(?:www\.)?bigflix\.com/.+/(?P<id>[0-9]+)'
+    _TESTS = [{
         'url': 'http://www.bigflix.com/Hindi-movies/Action-movies/Singham-Returns/16537',
         'md5': 'ec76aa9b1129e2e5b301a474e54fab74',
         'info_dict': {
@@ -18,7 +19,20 @@ class BigflixIE(InfoExtractor):
             'title': 'Singham Returns',
             'description': 'md5:3d2ba5815f14911d5cc6a501ae0cf65d',
         }
-    }
+    }, {
+        # multiple formats
+        'url': 'http://www.bigflix.com/Tamil-movies/Drama-movies/Madarasapatinam/16070',
+        'info_dict': {
+            'id': '16070',
+            'ext': 'mp4',
+            'title': 'Madarasapatinam',
+            'description': 'md5:63b9b8ed79189c6f0418c26d9a3452ca',
+            'formats': 'mincount:2',
+        },
+        'params': {
+            'skip_download': True,
+        }
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -29,14 +43,28 @@ class BigflixIE(InfoExtractor):
             r'<div[^>]+class=["\']pagetitle["\'][^>]*>(.+?)</div>',
             webpage, 'title')
 
-        video_url = b64decode(compat_urllib_parse_unquote(self._search_regex(
-            r'file=([^&]+)', webpage, 'video url')).encode('ascii')).decode('utf-8')
+        def decode_url(quoted_b64_url):
+            return base64.b64decode(compat_urllib_parse_unquote(
+                quoted_b64_url)).encode('ascii').decode('utf-8')
+
+        formats = [{
+            'url': decode_url(encoded_url),
+            'format_id': '%sp' % height,
+            'height': int(height),
+        } for height, encoded_url in re.findall(
+            r'ContentURL_(\d{3,4})[pP][^=]+=([^&]+)', webpage)]
+
+        if not formats:
+            formats.append({
+                'url': decode_url(self._search_regex(
+                    r'file=([^&]+)', webpage, 'video url')),
+            })
 
         description = self._html_search_meta('description', webpage)
 
         return {
             'id': video_id,
             'title': title,
-            'url': video_url,
             'description': description,
+            'formats': formats
         }

From a9bbd26f1d2bb45205f9fbd2626569522049e40e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 10 Jan 2016 10:49:27 +0600
Subject: [PATCH 3642/6219] [bigflix] Improve formats extraction

---
 youtube_dl/extractor/bigflix.py | 41 ++++++++++++++++++++++-----------
 1 file changed, 28 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/bigflix.py b/youtube_dl/extractor/bigflix.py
index b7e498436..24720de86 100644
--- a/youtube_dl/extractor/bigflix.py
+++ b/youtube_dl/extractor/bigflix.py
@@ -20,7 +20,7 @@ class BigflixIE(InfoExtractor):
             'description': 'md5:3d2ba5815f14911d5cc6a501ae0cf65d',
         }
     }, {
-        # multiple formats
+        # 2 formats
         'url': 'http://www.bigflix.com/Tamil-movies/Drama-movies/Madarasapatinam/16070',
         'info_dict': {
             'id': '16070',
@@ -32,6 +32,10 @@ class BigflixIE(InfoExtractor):
         'params': {
             'skip_download': True,
         }
+    }, {
+        # multiple formats
+        'url': 'http://www.bigflix.com/Malayalam-movies/Drama-movies/Indian-Rupee/15967',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
@@ -45,20 +49,31 @@ class BigflixIE(InfoExtractor):
 
         def decode_url(quoted_b64_url):
             return base64.b64decode(compat_urllib_parse_unquote(
-                quoted_b64_url)).encode('ascii').decode('utf-8')
+                quoted_b64_url).encode('ascii')).decode('utf-8')
 
-        formats = [{
-            'url': decode_url(encoded_url),
-            'format_id': '%sp' % height,
-            'height': int(height),
-        } for height, encoded_url in re.findall(
-            r'ContentURL_(\d{3,4})[pP][^=]+=([^&]+)', webpage)]
+        formats = []
+        for height, encoded_url in re.findall(
+            r'ContentURL_(\d{3,4})[pP][^=]+=([^&]+)', webpage):
+            video_url = decode_url(encoded_url)
+            f = {
+                'url': video_url,
+                'format_id': '%sp' % height,
+                'height': int(height),
+            }
+            if video_url.startswith('rtmp'):
+                f['ext'] = 'flv'
+            formats.append(f)
 
-        if not formats:
-            formats.append({
-                'url': decode_url(self._search_regex(
-                    r'file=([^&]+)', webpage, 'video url')),
-            })
+        file_url = self._search_regex(
+            r'file=([^&]+)', webpage, 'video url', default=None)
+        if file_url:
+            video_url = decode_url(file_url)
+            if all(f['url'] != video_url for f in formats):
+                formats.append({
+                    'url': decode_url(file_url),
+                })
+
+        self._sort_formats(formats)
 
         description = self._html_search_meta('description', webpage)
 

From 3fc088f8c7f61026943a62fc28e051a7d3a6bdd5 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 10 Jan 2016 07:45:41 +0100
Subject: [PATCH 3643/6219] [dcn] extract video ids in season entries

---
 youtube_dl/extractor/dcn.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/dcn.py b/youtube_dl/extractor/dcn.py
index 8f48571de..15a1c40f7 100644
--- a/youtube_dl/extractor/dcn.py
+++ b/youtube_dl/extractor/dcn.py
@@ -5,7 +5,10 @@ import re
 import base64
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse
+from ..compat import (
+    compat_urllib_parse,
+    compat_str,
+)
 from ..utils import (
     int_or_none,
     parse_iso8601,
@@ -186,7 +189,8 @@ class DCNSeasonIE(InfoExtractor):
 
                 entries = []
                 for video in show['videos']:
+                    video_id = compat_str(video['id'])
                     entries.append(self.url_result(
-                        'http://www.dcndigital.ae/media/%s' % video['id'], 'DCNVideo'))
+                        'http://www.dcndigital.ae/media/%s' % video_id, 'DCNVideo', video_id))
 
                 return self.playlist_result(entries, season_id, title)

From 2334762b03dec5da4d6788539e3e11192eb97010 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 10 Jan 2016 07:55:58 +0100
Subject: [PATCH 3644/6219] [shahid] raise ExtractorError if the video is DRM
 protected

---
 youtube_dl/extractor/shahid.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/shahid.py b/youtube_dl/extractor/shahid.py
index f76fb12c0..1178b7a27 100644
--- a/youtube_dl/extractor/shahid.py
+++ b/youtube_dl/extractor/shahid.py
@@ -73,6 +73,9 @@ class ShahidIE(InfoExtractor):
             'https://shahid.mbc.net/arContent/getPlayerContent-param-.id-%s.type-%s.html'
             % (video_id, api_vars['type']), video_id, 'Downloading player JSON')
 
+        if player.get('drm'):
+            raise ExtractorError('This video is DRM protected.', expected=True)
+
         formats = self._extract_m3u8_formats(player['url'], video_id, 'mp4')
 
         video = self._download_json(

From 7e8a800f29d52cbc6057638595df05ac69f622bb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 10 Jan 2016 14:26:27 +0100
Subject: [PATCH 3645/6219] [bigflix] Use correct indentation to make flake8
 happy

---
 youtube_dl/extractor/bigflix.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bigflix.py b/youtube_dl/extractor/bigflix.py
index 24720de86..33762ad93 100644
--- a/youtube_dl/extractor/bigflix.py
+++ b/youtube_dl/extractor/bigflix.py
@@ -53,7 +53,7 @@ class BigflixIE(InfoExtractor):
 
         formats = []
         for height, encoded_url in re.findall(
-            r'ContentURL_(\d{3,4})[pP][^=]+=([^&]+)', webpage):
+                r'ContentURL_(\d{3,4})[pP][^=]+=([^&]+)', webpage):
             video_url = decode_url(encoded_url)
             f = {
                 'url': video_url,

From 3c91e41614a0f3bfab8710e861c895b78d93bce8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 10 Jan 2016 14:32:53 +0100
Subject: [PATCH 3646/6219] [downloader/fragment] Don't fail if the
 'Content-Length' header is missing

In some dailymotion videos (like http://www.dailymotion.com/video/x3k0dtv from #8156) the segments URLs don't have the 'Content-Length' header and HttpFD sets the 'totat_bytes' field to None, so we also use '0' in that case (since we do different math operations with it).
---
 youtube_dl/downloader/fragment.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
index 5a64b29ee..3da554622 100644
--- a/youtube_dl/downloader/fragment.py
+++ b/youtube_dl/downloader/fragment.py
@@ -66,7 +66,7 @@ class FragmentFD(FileDownloader):
             if s['status'] not in ('downloading', 'finished'):
                 return
 
-            frag_total_bytes = s.get('total_bytes', 0)
+            frag_total_bytes = s.get('total_bytes') or 0
             if s['status'] == 'finished':
                 state['downloaded_bytes'] += frag_total_bytes
                 state['frag_index'] += 1

From dfb1b1468cef4ddc7ecc43776abce03763f8e426 Mon Sep 17 00:00:00 2001
From: Jakub Wilk <jwilk@jwilk.net>
Date: Sun, 10 Jan 2016 16:17:47 +0100
Subject: [PATCH 3647/6219] Fix typos

Closes #8200.
---
 devscripts/gh-pages/update-copyright.py | 2 +-
 test/test_write_annotations.py          | 2 +-
 youtube_dl/YoutubeDL.py                 | 2 +-
 youtube_dl/extractor/common.py          | 4 ++--
 youtube_dl/extractor/facebook.py        | 4 ++--
 youtube_dl/extractor/generic.py         | 4 ++--
 youtube_dl/extractor/ivi.py             | 2 +-
 youtube_dl/extractor/mdr.py             | 2 +-
 youtube_dl/extractor/nbc.py             | 2 +-
 youtube_dl/extractor/nhl.py             | 2 +-
 youtube_dl/extractor/ora.py             | 2 +-
 youtube_dl/extractor/testurl.py         | 2 +-
 youtube_dl/extractor/tv4.py             | 2 +-
 youtube_dl/extractor/videomore.py       | 2 +-
 youtube_dl/swfinterp.py                 | 2 +-
 youtube_dl/utils.py                     | 4 ++--
 16 files changed, 20 insertions(+), 20 deletions(-)

diff --git a/devscripts/gh-pages/update-copyright.py b/devscripts/gh-pages/update-copyright.py
index 3663c8afe..e6c3abc8d 100755
--- a/devscripts/gh-pages/update-copyright.py
+++ b/devscripts/gh-pages/update-copyright.py
@@ -5,7 +5,7 @@ from __future__ import with_statement, unicode_literals
 
 import datetime
 import glob
-import io  # For Python 2 compatibilty
+import io  # For Python 2 compatibility
 import os
 import re
 
diff --git a/test/test_write_annotations.py b/test/test_write_annotations.py
index 84b8f39e0..8de08f2d6 100644
--- a/test/test_write_annotations.py
+++ b/test/test_write_annotations.py
@@ -66,7 +66,7 @@ class TestAnnotations(unittest.TestCase):
                 textTag = a.find('TEXT')
                 text = textTag.text
                 self.assertTrue(text in expected)  # assertIn only added in python 2.7
-                # remove the first occurance, there could be more than one annotation with the same text
+                # remove the first occurrence, there could be more than one annotation with the same text
                 expected.remove(text)
         # We should have seen (and removed) all the expected annotation texts.
         self.assertEqual(len(expected), 0, 'Not all expected annotations were found.')
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 3b2be3159..d50b7cfed 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1312,7 +1312,7 @@ class YoutubeDL(object):
             # only set the 'formats' fields if the original info_dict list them
             # otherwise we end up with a circular reference, the first (and unique)
             # element in the 'formats' field in info_dict is info_dict itself,
-            # wich can't be exported to json
+            # which can't be exported to json
             info_dict['formats'] = formats
         if self.params.get('listformats'):
             self.list_formats(info_dict)
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 0719c7bcd..b05b22a94 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -313,9 +313,9 @@ class InfoExtractor(object):
         except ExtractorError:
             raise
         except compat_http_client.IncompleteRead as e:
-            raise ExtractorError('A network error has occured.', cause=e, expected=True)
+            raise ExtractorError('A network error has occurred.', cause=e, expected=True)
         except (KeyError, StopIteration) as e:
-            raise ExtractorError('An extractor error has occured.', cause=e)
+            raise ExtractorError('An extractor error has occurred.', cause=e)
 
     def set_downloader(self, downloader):
         """Sets the downloader for this IE."""
diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index 5e43f2359..ec699ba54 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -105,7 +105,7 @@ class FacebookIE(InfoExtractor):
                     login_results, 'login error', default=None, group='error')
                 if error:
                     raise ExtractorError('Unable to login: %s' % error, expected=True)
-                self._downloader.report_warning('unable to log in: bad username/password, or exceded login rate limit (~3/min). Check credentials or wait.')
+                self._downloader.report_warning('unable to log in: bad username/password, or exceeded login rate limit (~3/min). Check credentials or wait.')
                 return
 
             fb_dtsg = self._search_regex(
@@ -126,7 +126,7 @@ class FacebookIE(InfoExtractor):
             check_response = self._download_webpage(check_req, None,
                                                     note='Confirming login')
             if re.search(r'id="checkpointSubmitButton"', check_response) is not None:
-                self._downloader.report_warning('Unable to confirm login, you have to login in your brower and authorize the login.')
+                self._downloader.report_warning('Unable to confirm login, you have to login in your browser and authorize the login.')
         except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
             self._downloader.report_warning('unable to log in: %s' % error_to_compat_str(err))
             return
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index d79e1adc9..b3f8efc80 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -487,7 +487,7 @@ class GenericIE(InfoExtractor):
                 'description': 'md5:8145d19d320ff3e52f28401f4c4283b9',
             }
         },
-        # Embeded Ustream video
+        # Embedded Ustream video
         {
             'url': 'http://www.american.edu/spa/pti/nsa-privacy-janus-2014.cfm',
             'md5': '27b99cdb639c9b12a79bca876a073417',
@@ -1644,7 +1644,7 @@ class GenericIE(InfoExtractor):
         if myvi_url:
             return self.url_result(myvi_url)
 
-        # Look for embeded soundcloud player
+        # Look for embedded soundcloud player
         mobj = re.search(
             r'<iframe\s+(?:[a-zA-Z0-9_-]+="[^"]+"\s+)*src="(?P<url>https?://(?:w\.)?soundcloud\.com/player[^"]+)"',
             webpage)
diff --git a/youtube_dl/extractor/ivi.py b/youtube_dl/extractor/ivi.py
index d0f00cdea..472d72b4c 100644
--- a/youtube_dl/extractor/ivi.py
+++ b/youtube_dl/extractor/ivi.py
@@ -32,7 +32,7 @@ class IviIE(InfoExtractor):
             },
             'skip': 'Only works from Russia',
         },
-        # Serial's serie
+        # Serial's series
         {
             'url': 'http://www.ivi.ru/watch/dvoe_iz_lartsa/9549',
             'md5': '221f56b35e3ed815fde2df71032f4b3e',
diff --git a/youtube_dl/extractor/mdr.py b/youtube_dl/extractor/mdr.py
index 88334889e..425fc9e2a 100644
--- a/youtube_dl/extractor/mdr.py
+++ b/youtube_dl/extractor/mdr.py
@@ -17,7 +17,7 @@ class MDRIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?(?:mdr|kika)\.de/(?:.*)/[a-z]+(?P<id>\d+)(?:_.+?)?\.html'
 
     _TESTS = [{
-        # MDR regularily deletes its videos
+        # MDR regularly deletes its videos
         'url': 'http://www.mdr.de/fakt/video189002.html',
         'only_matching': True,
     }, {
diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index 340c922bd..1dd54c2f1 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -100,7 +100,7 @@ class NBCSportsVPlayerIE(InfoExtractor):
 
 
 class NBCSportsIE(InfoExtractor):
-    # Does not include https becuase its certificate is invalid
+    # Does not include https because its certificate is invalid
     _VALID_URL = r'http://www\.nbcsports\.com//?(?:[^/]+/)+(?P<id>[0-9a-z-]+)'
 
     _TEST = {
diff --git a/youtube_dl/extractor/nhl.py b/youtube_dl/extractor/nhl.py
index e98a5ef89..8d5ce46ad 100644
--- a/youtube_dl/extractor/nhl.py
+++ b/youtube_dl/extractor/nhl.py
@@ -223,7 +223,7 @@ class NHLVideocenterIE(NHLBaseInfoExtractor):
         response = self._download_webpage(request_url, playlist_title)
         response = self._fix_json(response)
         if not response.strip():
-            self._downloader.report_warning('Got an empty reponse, trying '
+            self._downloader.report_warning('Got an empty response, trying '
                                             'adding the "newvideos" parameter')
             response = self._download_webpage(request_url + '&newvideos=true',
                                               playlist_title)
diff --git a/youtube_dl/extractor/ora.py b/youtube_dl/extractor/ora.py
index 9c4255a2d..02de1502a 100644
--- a/youtube_dl/extractor/ora.py
+++ b/youtube_dl/extractor/ora.py
@@ -37,7 +37,7 @@ class OraTVIE(InfoExtractor):
             formats = self._extract_m3u8_formats(
                 m3u8_url, display_id, 'mp4', 'm3u8_native',
                 m3u8_id='hls', fatal=False)
-            # simular to GameSpotIE
+            # similar to GameSpotIE
             m3u8_path = compat_urlparse.urlparse(m3u8_url).path
             QUALITIES_RE = r'((,[a-z]+\d+)+,?)'
             available_qualities = self._search_regex(
diff --git a/youtube_dl/extractor/testurl.py b/youtube_dl/extractor/testurl.py
index c7d559315..46918adb0 100644
--- a/youtube_dl/extractor/testurl.py
+++ b/youtube_dl/extractor/testurl.py
@@ -7,7 +7,7 @@ from ..utils import ExtractorError
 
 
 class TestURLIE(InfoExtractor):
-    """ Allows adressing of the test cases as test:yout.*be_1 """
+    """ Allows addressing of the test cases as test:yout.*be_1 """
 
     IE_DESC = False  # Do not list
     _VALID_URL = r'test(?:url)?:(?P<id>(?P<extractor>.+?)(?:_(?P<num>[0-9]+))?)$'
diff --git a/youtube_dl/extractor/tv4.py b/youtube_dl/extractor/tv4.py
index 1c4b6d635..343edf206 100644
--- a/youtube_dl/extractor/tv4.py
+++ b/youtube_dl/extractor/tv4.py
@@ -67,7 +67,7 @@ class TV4IE(InfoExtractor):
         info = self._download_json(
             'http://www.tv4play.se/player/assets/%s.json' % video_id, video_id, 'Downloading video info JSON')
 
-        # If is_geo_restricted is true, it doesn't neceserally mean we can't download it
+        # If is_geo_restricted is true, it doesn't necessarily mean we can't download it
         if info['is_geo_restricted']:
             self.report_warning('This content might not be available in your country due to licensing restrictions.')
         if info['requires_subscription']:
diff --git a/youtube_dl/extractor/videomore.py b/youtube_dl/extractor/videomore.py
index a66d6de23..fcee940e6 100644
--- a/youtube_dl/extractor/videomore.py
+++ b/youtube_dl/extractor/videomore.py
@@ -170,7 +170,7 @@ class VideomoreVideoIE(InfoExtractor):
             'skip_download': True,
         },
     }, {
-        # season single serie with og:video:iframe
+        # season single series with og:video:iframe
         'url': 'http://videomore.ru/poslednii_ment/1_sezon/14_seriya',
         'only_matching': True,
     }, {
diff --git a/youtube_dl/swfinterp.py b/youtube_dl/swfinterp.py
index e60505ace..06c1d6cc1 100644
--- a/youtube_dl/swfinterp.py
+++ b/youtube_dl/swfinterp.py
@@ -689,7 +689,7 @@ class SWFInterpreter(object):
                     elif mname in _builtin_classes:
                         res = _builtin_classes[mname]
                     else:
-                        # Assume unitialized
+                        # Assume uninitialized
                         # TODO warn here
                         res = undefined
                     stack.append(res)
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index da4ec7f20..9c1c0e0bd 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -984,7 +984,7 @@ def date_from_str(date_str):
         if sign == '-':
             time = -time
         unit = match.group('unit')
-        # A bad aproximation?
+        # A bad approximation?
         if unit == 'month':
             unit = 'day'
             time *= 30
@@ -1307,7 +1307,7 @@ def parse_filesize(s):
     if s is None:
         return None
 
-    # The lower-case forms are of course incorrect and inofficial,
+    # The lower-case forms are of course incorrect and unofficial,
     # but we support those too
     _UNIT_TABLE = {
         'B': 1,

From 36a0e46c39ea4f211dea9944177976e8f8364736 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 10 Jan 2016 17:55:41 +0100
Subject: [PATCH 3648/6219] fix typos

---
 youtube_dl/YoutubeDL.py | 4 ++--
 youtube_dl/utils.py     | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index d50b7cfed..6afc1b730 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -241,7 +241,7 @@ class YoutubeDL(object):
                        - "detect_or_warn": check whether we can do anything
                                            about it, warn otherwise (default)
     source_address:    (Experimental) Client-side IP address to bind to.
-    call_home:         Boolean, true iff we are allowed to contact the
+    call_home:         Boolean, true if we are allowed to contact the
                        youtube-dl servers for debugging.
     sleep_interval:    Number of seconds to sleep before each download.
     listformats:       Print an overview of available video formats and exit.
@@ -590,7 +590,7 @@ class YoutubeDL(object):
             return None
 
     def _match_entry(self, info_dict, incomplete):
-        """ Returns None iff the file should be downloaded """
+        """ Returns None if the file should be downloaded """
 
         video_title = info_dict.get('title', info_dict.get('id', 'video'))
         if 'title' in info_dict:
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 9c1c0e0bd..e583299c5 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1857,7 +1857,7 @@ def encode_data_uri(data, mime_type):
 
 
 def age_restricted(content_limit, age_limit):
-    """ Returns True iff the content should be blocked """
+    """ Returns True if the content should be blocked """
 
     if age_limit is None:  # No limit set
         return False

From 6ec6cb4e956abbd76a6ff42336821770a5fbbcc7 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 10 Jan 2016 19:27:22 +0100
Subject: [PATCH 3649/6219] Revert "fix typos"

This reverts commit 36a0e46c39ea4f211dea9944177976e8f8364736.
---
 youtube_dl/YoutubeDL.py | 4 ++--
 youtube_dl/utils.py     | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 6afc1b730..d50b7cfed 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -241,7 +241,7 @@ class YoutubeDL(object):
                        - "detect_or_warn": check whether we can do anything
                                            about it, warn otherwise (default)
     source_address:    (Experimental) Client-side IP address to bind to.
-    call_home:         Boolean, true if we are allowed to contact the
+    call_home:         Boolean, true iff we are allowed to contact the
                        youtube-dl servers for debugging.
     sleep_interval:    Number of seconds to sleep before each download.
     listformats:       Print an overview of available video formats and exit.
@@ -590,7 +590,7 @@ class YoutubeDL(object):
             return None
 
     def _match_entry(self, info_dict, incomplete):
-        """ Returns None if the file should be downloaded """
+        """ Returns None iff the file should be downloaded """
 
         video_title = info_dict.get('title', info_dict.get('id', 'video'))
         if 'title' in info_dict:
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index e583299c5..9c1c0e0bd 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1857,7 +1857,7 @@ def encode_data_uri(data, mime_type):
 
 
 def age_restricted(content_limit, age_limit):
-    """ Returns True if the content should be blocked """
+    """ Returns True iff the content should be blocked """
 
     if age_limit is None:  # No limit set
         return False

From 192b9a571cd50b0fd924e9ecb926b28633e8c19c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 11 Jan 2016 21:56:19 +0600
Subject: [PATCH 3650/6219] [videomega] Mark broken

---
 youtube_dl/extractor/videomega.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/videomega.py b/youtube_dl/extractor/videomega.py
index 87aca327b..5e2e7cbac 100644
--- a/youtube_dl/extractor/videomega.py
+++ b/youtube_dl/extractor/videomega.py
@@ -8,6 +8,7 @@ from ..utils import sanitized_Request
 
 
 class VideoMegaIE(InfoExtractor):
+    _WORKING = False
     _VALID_URL = r'(?:videomega:|https?://(?:www\.)?videomega\.tv/(?:(?:view|iframe|cdn)\.php)?\?ref=)(?P<id>[A-Za-z0-9]+)'
     _TESTS = [{
         'url': 'http://videomega.tv/cdn.php?ref=AOSQBJYKIDDIKYJBQSOA',

From b924bfad684ac7e9209e7165df3e13af6545596d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 11 Jan 2016 21:58:32 +0600
Subject: [PATCH 3651/6219] [videott] Mark broken

---
 youtube_dl/extractor/videott.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/videott.py b/youtube_dl/extractor/videott.py
index 591024ead..2cd36508a 100644
--- a/youtube_dl/extractor/videott.py
+++ b/youtube_dl/extractor/videott.py
@@ -11,6 +11,7 @@ from ..utils import (
 
 
 class VideoTtIE(InfoExtractor):
+    _WORKING = False
     ID_NAME = 'video.tt'
     IE_DESC = 'video.tt - Your True Tube'
     _VALID_URL = r'http://(?:www\.)?video\.tt/(?:(?:video|embed)/|watch_video\.php\?v=)(?P<id>[\da-zA-Z]{9})'

From be27283ef68e96d2461875a2bfb8c9ce962cfe61 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 11 Jan 2016 22:00:17 +0600
Subject: [PATCH 3652/6219] [iprima] Mark broken

---
 youtube_dl/extractor/iprima.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/iprima.py b/youtube_dl/extractor/iprima.py
index 36baf3245..073777f34 100644
--- a/youtube_dl/extractor/iprima.py
+++ b/youtube_dl/extractor/iprima.py
@@ -14,6 +14,7 @@ from ..utils import (
 
 
 class IPrimaIE(InfoExtractor):
+    _WORKING = False
     _VALID_URL = r'https?://play\.iprima\.cz/(?:[^/]+/)*(?P<id>[^?#]+)'
 
     _TESTS = [{

From 9cb1a06b6c8fbeb6cfdbf0533ea60a6624fa1246 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 12 Jan 2016 22:09:38 +0600
Subject: [PATCH 3653/6219] [downloader/fragment] Remove unused code and fix
 zero division error

---
 youtube_dl/downloader/fragment.py | 13 +++----------
 1 file changed, 3 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
index 3da554622..d236ac737 100644
--- a/youtube_dl/downloader/fragment.py
+++ b/youtube_dl/downloader/fragment.py
@@ -78,17 +78,10 @@ class FragmentFD(FileDownloader):
             state['total_bytes_estimate'] = estimated_size
             state['elapsed'] = time_now - start
 
-            if s['status'] == 'finished':
-                progress = self.calc_percent(state['frag_index'], total_frags)
-            else:
-                frag_downloaded_bytes = s['downloaded_bytes']
-                frag_progress = self.calc_percent(frag_downloaded_bytes,
-                                                  frag_total_bytes)
-                progress = self.calc_percent(state['frag_index'], total_frags)
-                progress += frag_progress / float(total_frags)
-
+            if s['status'] != 'finished':
                 state['eta'] = self.calc_eta(
-                    start, time_now, estimated_size, state['downloaded_bytes'] + frag_downloaded_bytes)
+                    start, time_now, estimated_size,
+                    state['downloaded_bytes'] + s['downloaded_bytes'])
                 state['speed'] = s.get('speed')
             self._hook_progress(state)
 

From 709185a2648c991cc143272dda67e4b42e89c03b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 12 Jan 2016 23:18:38 +0600
Subject: [PATCH 3654/6219] [downloader/fragment] More smooth calculations

`downloaded_bytes` is now updated on each fragment progress hook invocation
---
 youtube_dl/downloader/fragment.py | 22 ++++++++++++++++------
 1 file changed, 16 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
index d236ac737..b2597f1e5 100644
--- a/youtube_dl/downloader/fragment.py
+++ b/youtube_dl/downloader/fragment.py
@@ -58,6 +58,11 @@ class FragmentFD(FileDownloader):
             'frag_count': total_frags,
             'filename': ctx['filename'],
             'tmpfilename': ctx['tmpfilename'],
+            # Total complete fragments downloaded so far in bytes
+            '_complete_frags_downloaded_bytes': 0,
+            # Amount of fragment's bytes downloaded by the time of the previous
+            # frag progress hook invocation
+            '_prev_frag_downloaded_bytes': 0,
         }
         start = time.time()
         ctx['started'] = start
@@ -67,22 +72,27 @@ class FragmentFD(FileDownloader):
                 return
 
             frag_total_bytes = s.get('total_bytes') or 0
-            if s['status'] == 'finished':
-                state['downloaded_bytes'] += frag_total_bytes
-                state['frag_index'] += 1
 
             estimated_size = (
-                (state['downloaded_bytes'] + frag_total_bytes) /
+                (state['_complete_frags_downloaded_bytes'] + frag_total_bytes) /
                 (state['frag_index'] + 1) * total_frags)
             time_now = time.time()
             state['total_bytes_estimate'] = estimated_size
             state['elapsed'] = time_now - start
 
-            if s['status'] != 'finished':
+            if s['status'] == 'finished':
+                state['frag_index'] += 1
+                state['downloaded_bytes'] += frag_total_bytes - state['_prev_frag_downloaded_bytes']
+                state['_complete_frags_downloaded_bytes'] = state['downloaded_bytes']
+                state['_prev_frag_downloaded_bytes'] = 0
+            else:
+                frag_downloaded_bytes = s['downloaded_bytes']
+                state['downloaded_bytes'] += frag_downloaded_bytes - state['_prev_frag_downloaded_bytes']
                 state['eta'] = self.calc_eta(
                     start, time_now, estimated_size,
-                    state['downloaded_bytes'] + s['downloaded_bytes'])
+                    state['downloaded_bytes'])
                 state['speed'] = s.get('speed')
+                state['_prev_frag_downloaded_bytes'] = frag_downloaded_bytes
             self._hook_progress(state)
 
         ctx['dl'].add_progress_hook(frag_progress_hook)

From 16a348475cb7ab90e80c31a01eec1280f4fc4fcb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 12 Jan 2016 23:23:39 +0600
Subject: [PATCH 3655/6219] [dailymotion] Prefer direct links (Closes #8156)

---
 youtube_dl/extractor/dailymotion.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index 439fd42e8..b687ec4d6 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -149,14 +149,15 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
                     ext = determine_ext(media_url)
                     if type_ == 'application/x-mpegURL' or ext == 'm3u8':
                         formats.extend(self._extract_m3u8_formats(
-                            media_url, video_id, 'mp4', m3u8_id='hls', fatal=False))
+                            media_url, video_id, 'mp4', preference=-1,
+                            m3u8_id='hls', fatal=False))
                     elif type_ == 'application/f4m' or ext == 'f4m':
                         formats.extend(self._extract_f4m_formats(
                             media_url, video_id, preference=-1, f4m_id='hds', fatal=False))
                     else:
                         f = {
                             'url': media_url,
-                            'format_id': quality,
+                            'format_id': 'http-%s' % quality,
                         }
                         m = re.search(r'H264-(?P<width>\d+)x(?P<height>\d+)', media_url)
                         if m:

From b83b782dc4afeabbd356a329fd302b4410afa626 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 13 Jan 2016 00:00:31 +0600
Subject: [PATCH 3656/6219] [downloader/fragment] Move helper data to context
 dict

---
 youtube_dl/downloader/fragment.py | 25 ++++++++++++++-----------
 1 file changed, 14 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
index b2597f1e5..0c9113d0f 100644
--- a/youtube_dl/downloader/fragment.py
+++ b/youtube_dl/downloader/fragment.py
@@ -58,14 +58,17 @@ class FragmentFD(FileDownloader):
             'frag_count': total_frags,
             'filename': ctx['filename'],
             'tmpfilename': ctx['tmpfilename'],
+        }
+
+        start = time.time()
+        ctx.update({
+            'started': start,
             # Total complete fragments downloaded so far in bytes
-            '_complete_frags_downloaded_bytes': 0,
+            'complete_frags_downloaded_bytes': 0,
             # Amount of fragment's bytes downloaded by the time of the previous
             # frag progress hook invocation
-            '_prev_frag_downloaded_bytes': 0,
-        }
-        start = time.time()
-        ctx['started'] = start
+            'prev_frag_downloaded_bytes': 0,
+        })
 
         def frag_progress_hook(s):
             if s['status'] not in ('downloading', 'finished'):
@@ -74,7 +77,7 @@ class FragmentFD(FileDownloader):
             frag_total_bytes = s.get('total_bytes') or 0
 
             estimated_size = (
-                (state['_complete_frags_downloaded_bytes'] + frag_total_bytes) /
+                (ctx['complete_frags_downloaded_bytes'] + frag_total_bytes) /
                 (state['frag_index'] + 1) * total_frags)
             time_now = time.time()
             state['total_bytes_estimate'] = estimated_size
@@ -82,17 +85,17 @@ class FragmentFD(FileDownloader):
 
             if s['status'] == 'finished':
                 state['frag_index'] += 1
-                state['downloaded_bytes'] += frag_total_bytes - state['_prev_frag_downloaded_bytes']
-                state['_complete_frags_downloaded_bytes'] = state['downloaded_bytes']
-                state['_prev_frag_downloaded_bytes'] = 0
+                state['downloaded_bytes'] += frag_total_bytes - ctx['prev_frag_downloaded_bytes']
+                ctx['complete_frags_downloaded_bytes'] = state['downloaded_bytes']
+                ctx['prev_frag_downloaded_bytes'] = 0
             else:
                 frag_downloaded_bytes = s['downloaded_bytes']
-                state['downloaded_bytes'] += frag_downloaded_bytes - state['_prev_frag_downloaded_bytes']
+                state['downloaded_bytes'] += frag_downloaded_bytes - ctx['prev_frag_downloaded_bytes']
                 state['eta'] = self.calc_eta(
                     start, time_now, estimated_size,
                     state['downloaded_bytes'])
                 state['speed'] = s.get('speed')
-                state['_prev_frag_downloaded_bytes'] = frag_downloaded_bytes
+                ctx['prev_frag_downloaded_bytes'] = frag_downloaded_bytes
             self._hook_progress(state)
 
         ctx['dl'].add_progress_hook(frag_progress_hook)

From bc0550c262db0bc756bb7e08fb945f3285fb196e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 13 Jan 2016 08:18:37 +0600
Subject: [PATCH 3657/6219] [pluralsight] Fix new player (Closes #8215)

---
 youtube_dl/extractor/pluralsight.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index 55c11b3bf..12e1c2862 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -232,7 +232,7 @@ class PluralsightIE(PluralsightBaseIE):
         # { a = author, cn = clip_id, lc = end, m = name }
 
         return {
-            'id': clip['clipName'],
+            'id': clip.get('clipName') or clip['name'],
             'title': '%s - %s' % (module['title'], clip['title']),
             'duration': int_or_none(clip.get('duration')) or parse_duration(clip.get('formattedDuration')),
             'creator': author,

From cc28492d31556b55bbd8fc574bf5d890305b22d2 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 13 Jan 2016 17:05:38 +0800
Subject: [PATCH 3658/6219] [youtube] Fix acodec and vcodec order

In RFC6381, there's no rule stating that the first part of codecs should
be video and the second part should be audio, while it seems the case
for data reported by YouTube.
---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 897641e42..e4f227f19 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1487,7 +1487,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                             if codecs:
                                 codecs = codecs.split(',')
                                 if len(codecs) == 2:
-                                    acodec, vcodec = codecs[0], codecs[1]
+                                    acodec, vcodec = codecs[1], codecs[0]
                                 else:
                                     acodec, vcodec = (codecs[0], 'none') if kind == 'audio' else ('none', codecs[0])
                                 dct.update({

From 40cf7fcbd2e30747065ca7b8bf4467a9582a4aa9 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 13 Jan 2016 13:29:00 +0100
Subject: [PATCH 3659/6219] [tudou] Add support for Albums and Playlists and
 extract more metadata

---
 youtube_dl/extractor/__init__.py |  6 +-
 youtube_dl/extractor/tudou.py    | 94 ++++++++++++++++++++++++++------
 2 files changed, 81 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 40c42d4d2..e4ae9332d 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -723,7 +723,11 @@ from .trilulilu import TriluliluIE
 from .trutube import TruTubeIE
 from .tube8 import Tube8IE
 from .tubitv import TubiTvIE
-from .tudou import TudouIE
+from .tudou import (
+    TudouIE,
+    TudouPlaylistIE,
+    TudouAlbumIE,
+)
 from .tumblr import TumblrIE
 from .tunein import (
     TuneInClipIE,
diff --git a/youtube_dl/extractor/tudou.py b/youtube_dl/extractor/tudou.py
index 5f7ac4b35..da3cd76f7 100644
--- a/youtube_dl/extractor/tudou.py
+++ b/youtube_dl/extractor/tudou.py
@@ -4,10 +4,16 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..compat import compat_str
+from ..utils import (
+    int_or_none,
+    float_or_none,
+    unescapeHTML,
+)
 
 
 class TudouIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?tudou\.com/(?:listplay|programs(?:/view)?|albumplay)/([^/]+/)*(?P<id>[^/?#]+?)(?:\.html)?/?(?:$|[?#])'
+    IE_NAME = 'tudou'
+    _VALID_URL = r'https?://(?:www\.)?tudou\.com/(?:(?:programs|wlplay)/view|(?:listplay|albumplay)/[\w-]{11})/(?P<id>[\w-]{11})'
     _TESTS = [{
         'url': 'http://www.tudou.com/listplay/zzdE77v6Mmo/2xN2duXMxmw.html',
         'md5': '140a49ed444bd22f93330985d8475fcb',
@@ -16,6 +22,11 @@ class TudouIE(InfoExtractor):
             'ext': 'f4v',
             'title': '卡马乔国足开大脚长传冲吊集锦',
             'thumbnail': 're:^https?://.*\.jpg$',
+            'timestamp': 1372113489000,
+            'description': '卡马乔卡家军，开大脚先进战术不完全集锦！',
+            'duration': 289.04,
+            'view_count': int,
+            'filesize': int,
         }
     }, {
         'url': 'http://www.tudou.com/programs/view/ajX3gyhL0pc/',
@@ -24,10 +35,12 @@ class TudouIE(InfoExtractor):
             'ext': 'f4v',
             'title': 'La Sylphide-Bolshoi-Ekaterina Krysanova & Vyacheslav Lopatin 2012',
             'thumbnail': 're:^https?://.*\.jpg$',
+            'timestamp': 1349207518000,
+            'description': 'md5:294612423894260f2dcd5c6c04fe248b',
+            'duration': 5478.33,
+            'view_count': int,
+            'filesize': int,
         }
-    }, {
-        'url': 'http://www.tudou.com/albumplay/cJAHGih4yYg.html',
-        'only_matching': True,
     }]
 
     _PLAYER_URL = 'http://js.tudouui.com/bin/lingtong/PortalPlayer_177.swf'
@@ -42,24 +55,20 @@ class TudouIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
+        item_data = self._download_json(
+            'http://www.tudou.com/tvp/getItemInfo.action?ic=%s' % video_id, video_id)
 
-        youku_vcode = self._search_regex(
-            r'vcode\s*:\s*[\'"]([^\'"]*)[\'"]', webpage, 'youku vcode', default=None)
+        youku_vcode = item_data.get('vcode')
         if youku_vcode:
             return self.url_result('youku:' + youku_vcode, ie='Youku')
 
-        title = self._search_regex(
-            r',kw\s*:\s*[\'"]([^\'"]+)[\'"]', webpage, 'title')
-        thumbnail_url = self._search_regex(
-            r',pic\s*:\s*[\'"]([^\'"]+)[\'"]', webpage, 'thumbnail URL', fatal=False)
+        title = unescapeHTML(item_data['kw'])
+        description = item_data.get('desc')
+        thumbnail_url = item_data.get('pic')
+        view_count = int_or_none(item_data.get('playTimes'))
+        timestamp = int_or_none(item_data.get('pt'))
 
-        player_url = self._search_regex(
-            r'playerUrl\s*:\s*[\'"]([^\'"]+\.swf)[\'"]',
-            webpage, 'player URL', default=self._PLAYER_URL)
-
-        segments = self._parse_json(self._search_regex(
-            r'segs: \'([^\']+)\'', webpage, 'segments'), video_id)
+        segments = self._parse_json(item_data['itemSegs'], video_id)
         # It looks like the keys are the arguments that have to be passed as
         # the hd field in the request url, we pick the higher
         # Also, filter non-number qualities (see issue #3643).
@@ -80,8 +89,13 @@ class TudouIE(InfoExtractor):
                 'ext': ext,
                 'title': title,
                 'thumbnail': thumbnail_url,
+                'description': description,
+                'view_count': view_count,
+                'timestamp': timestamp,
+                'duration': float_or_none(part.get('seconds'), 1000),
+                'filesize': int_or_none(part.get('size')),
                 'http_headers': {
-                    'Referer': player_url,
+                    'Referer': self._PLAYER_URL,
                 },
             }
             result.append(part_info)
@@ -92,3 +106,47 @@ class TudouIE(InfoExtractor):
             'id': video_id,
             'title': title,
         }
+
+
+class TudouPlaylistIE(InfoExtractor):
+    IE_NAME = 'tudou:playlist'
+    _VALID_URL = r'https?://(?:www\.)?tudou\.com/listplay/(?P<id>[\w-]{11})\.html'
+    _TESTS = [{
+        'url': 'http://www.tudou.com/listplay/zzdE77v6Mmo.html',
+        'info_dict': {
+            'id': 'zzdE77v6Mmo',
+        },
+        'playlist_mincount': 209,
+    }]
+
+    def _real_extract(self, url):
+        playlist_id = self._match_id(url)
+        playlist_data = self._download_json(
+            'http://www.tudou.com/tvp/plist.action?lcode=%s' % playlist_id, playlist_id)
+        entries = [self.url_result(
+            'http://www.tudou.com/programs/view/%s' % item['icode'],
+            'Tudou', item['icode'],
+            item['kw']) for item in playlist_data['items']]
+        return self.playlist_result(entries, playlist_id)
+
+
+class TudouAlbumIE(InfoExtractor):
+    IE_NAME = 'tudou:album'
+    _VALID_URL = r'https?://(?:www\.)?tudou\.com/album(?:cover|play)/(?P<id>[\w-]{11})'
+    _TESTS = [{
+        'url': 'http://www.tudou.com/albumplay/v5qckFJvNJg.html',
+        'info_dict': {
+            'id': 'v5qckFJvNJg',
+        },
+        'playlist_mincount': 45,
+    }]
+
+    def _real_extract(self, url):
+        album_id = self._match_id(url)
+        album_data = self._download_json(
+            'http://www.tudou.com/tvp/alist.action?acode=%s' % album_id, album_id)
+        entries = [self.url_result(
+            'http://www.tudou.com/programs/view/%s' % item['icode'],
+            'Tudou', item['icode'],
+            item['kw']) for item in album_data['items']]
+        return self.playlist_result(entries, album_id)

From e37afbe0b8a1222cb214ad0bec9a53bb7953531d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 14 Jan 2016 00:16:23 +0100
Subject: [PATCH 3660/6219] [YoutubeDL] urlopen: disable the 'file:' protocol
 (#8227)

If someone is running youtube-dl on a server to deliver files, the user could input 'file:///some/important/file' and youtube-dl would save that file as a video giving access to sensitive information to the user.
'file:' urls can be filtered, but the user can use an URL to a crafted m3u8 manifest like:

    #EXTM3U
    #EXT-X-MEDIA-SEQUENCE:0
    #EXTINF:10.0
    file:///etc/passwd
    #EXT-X-ENDLIST

With this patch 'file:' URLs raise URLError like for unknown protocols.
---
 test/test_YoutubeDL.py  |  7 ++++++-
 youtube_dl/YoutubeDL.py | 10 ++++++++--
 2 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 0388c0bf3..0caa43843 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -12,7 +12,7 @@ import copy
 
 from test.helper import FakeYDL, assertRegexpMatches
 from youtube_dl import YoutubeDL
-from youtube_dl.compat import compat_str
+from youtube_dl.compat import compat_str, compat_urllib_error
 from youtube_dl.extractor import YoutubeIE
 from youtube_dl.postprocessor.common import PostProcessor
 from youtube_dl.utils import ExtractorError, match_filter_func
@@ -631,6 +631,11 @@ class TestYoutubeDL(unittest.TestCase):
         result = get_ids({'playlist_items': '10'})
         self.assertEqual(result, [])
 
+    def test_urlopen_no_file_protocol(self):
+        # see https://github.com/rg3/youtube-dl/issues/8227
+        ydl = YDL()
+        self.assertRaises(compat_urllib_error.URLError, ydl.urlopen, 'file:///etc/passwd')
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index d50b7cfed..e8ce58604 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1986,8 +1986,14 @@ class YoutubeDL(object):
         https_handler = make_HTTPS_handler(self.params, debuglevel=debuglevel)
         ydlh = YoutubeDLHandler(self.params, debuglevel=debuglevel)
         data_handler = compat_urllib_request_DataHandler()
-        opener = compat_urllib_request.build_opener(
-            proxy_handler, https_handler, cookie_processor, ydlh, data_handler)
+        unknown_handler = compat_urllib_request.UnknownHandler()
+        handlers = (proxy_handler, https_handler, cookie_processor, ydlh, data_handler, unknown_handler)
+        # we don't use build_opener because it automatically adds FileHandler,
+        # which can be used for malicious purposes (see
+        # https://github.com/rg3/youtube-dl/issues/8227)
+        opener = compat_urllib_request.OpenerDirector()
+        for handler in handlers:
+            opener.add_handler(handler)
 
         # Delete the default user-agent header, which would otherwise apply in
         # cases where our custom HTTP handler doesn't come into play

From 6240b0a278781a3b584a9dd6d57191b2472c0fd8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 14 Jan 2016 08:14:01 +0100
Subject: [PATCH 3661/6219] [YoutubeDL] urlopen: use build_opener again

Otherwise we would need to manually add handlers like HTTPRedirectHandler, instead we add a customized FileHandler instance that raises an error.
---
 youtube_dl/YoutubeDL.py | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index e8ce58604..ccad5f2ea 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1986,14 +1986,19 @@ class YoutubeDL(object):
         https_handler = make_HTTPS_handler(self.params, debuglevel=debuglevel)
         ydlh = YoutubeDLHandler(self.params, debuglevel=debuglevel)
         data_handler = compat_urllib_request_DataHandler()
-        unknown_handler = compat_urllib_request.UnknownHandler()
-        handlers = (proxy_handler, https_handler, cookie_processor, ydlh, data_handler, unknown_handler)
-        # we don't use build_opener because it automatically adds FileHandler,
-        # which can be used for malicious purposes (see
+
+        # When passing our own FileHandler instance, build_opener won't add the
+        # default FileHandler and allows us to disable the file protocol, which
+        # can be used for malicious purposes (see
         # https://github.com/rg3/youtube-dl/issues/8227)
-        opener = compat_urllib_request.OpenerDirector()
-        for handler in handlers:
-            opener.add_handler(handler)
+        file_handler = compat_urllib_request.FileHandler()
+
+        def file_open(*args, **kwargs):
+            raise compat_urllib_error.URLError('file protocol is disabled')
+        file_handler.file_open = file_open
+
+        opener = compat_urllib_request.build_opener(
+            proxy_handler, https_handler, cookie_processor, ydlh, data_handler, file_handler)
 
         # Delete the default user-agent header, which would otherwise apply in
         # cases where our custom HTTP handler doesn't come into play

From 4240d504963bb6d1c7bd7c288a7874f9d8dc042b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 14 Jan 2016 14:07:54 +0100
Subject: [PATCH 3662/6219] [YoutubeDL] improve error message for file:/// URLs

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index ccad5f2ea..4915fbd45 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1994,7 +1994,7 @@ class YoutubeDL(object):
         file_handler = compat_urllib_request.FileHandler()
 
         def file_open(*args, **kwargs):
-            raise compat_urllib_error.URLError('file protocol is disabled')
+            raise compat_urllib_error.URLError('file:/// protocol is explicitly disabled in youtube-dl for security reasons')
         file_handler.file_open = file_open
 
         opener = compat_urllib_request.build_opener(

From 4511c1976d0a06394a000333a020a4d3668072fe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 14 Jan 2016 19:57:20 +0600
Subject: [PATCH 3663/6219] [beeg] Fix extraction (Closes #8225)

---
 youtube_dl/extractor/beeg.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/beeg.py b/youtube_dl/extractor/beeg.py
index c8d921daf..d0174b818 100644
--- a/youtube_dl/extractor/beeg.py
+++ b/youtube_dl/extractor/beeg.py
@@ -60,7 +60,7 @@ class BeegIE(InfoExtractor):
 
         def decrypt_url(encrypted_url):
             encrypted_url = self._proto_relative_url(
-                encrypted_url.replace('{DATA_MARKERS}', ''), 'http:')
+                encrypted_url.replace('{DATA_MARKERS}', ''), 'https:')
             key = self._search_regex(
                 r'/key=(.*?)%2Cend=', encrypted_url, 'key', default=None)
             if not key:

From abb893e6e45b0b0c6ec0e3a1d29dbd1746cbee96 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 14 Jan 2016 19:57:56 +0600
Subject: [PATCH 3664/6219] [beeg] Update API URL

---
 youtube_dl/extractor/beeg.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/beeg.py b/youtube_dl/extractor/beeg.py
index d0174b818..34c2a756f 100644
--- a/youtube_dl/extractor/beeg.py
+++ b/youtube_dl/extractor/beeg.py
@@ -34,7 +34,7 @@ class BeegIE(InfoExtractor):
         video_id = self._match_id(url)
 
         video = self._download_json(
-            'http://beeg.com/api/v5/video/%s' % video_id, video_id)
+            'https://api.beeg.com/api/v5/video/%s' % video_id, video_id)
 
         def split(o, e):
             def cut(s, x):

From 11c60089a8772a2d12288f0ff382866e516f9a4b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 14 Jan 2016 15:43:21 +0100
Subject: [PATCH 3665/6219] release 2016.01.14

---
 docs/supportedsites.md | 11 +++++++----
 youtube_dl/version.py  |  2 +-
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 8d0c7b97a..eb160bd2f 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -65,6 +65,7 @@
  - **Beeg**
  - **BehindKink**
  - **Bet**
+ - **Bigflix**
  - **Bild**: Bild.de
  - **BiliBili**
  - **BleacherReport**
@@ -251,7 +252,7 @@
  - **Instagram**
  - **instagram:user**: Instagram user profile
  - **InternetVideoArchive**
- - **IPrima**
+ - **IPrima** (Currently broken)
  - **iqiyi**: 爱奇艺
  - **Ir90Tv**
  - **ivi**: ivi.ru
@@ -602,7 +603,9 @@
  - **TruTube**
  - **Tube8**
  - **TubiTv**
- - **Tudou**
+ - **tudou**
+ - **tudou:album**
+ - **tudou:playlist**
  - **Tumblr**
  - **tunein:clip**
  - **tunein:program**
@@ -655,12 +658,12 @@
  - **video.mit.edu**
  - **VideoDetective**
  - **videofy.me**
- - **VideoMega**
+ - **VideoMega** (Currently broken)
  - **videomore**
  - **videomore:season**
  - **videomore:video**
  - **VideoPremium**
- - **VideoTt**: video.tt - Your True Tube
+ - **VideoTt**: video.tt - Your True Tube (Currently broken)
  - **videoweed**: VideoWeed
  - **Vidme**
  - **Vidzi**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 7030903c0..4d433b667 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.01.09'
+__version__ = '2016.01.14'

From 30e2f2d76f6dd52803effce14fa14f3a8051c84a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 14 Jan 2016 16:28:46 +0100
Subject: [PATCH 3666/6219] [YoutubeDL] use a more correct terminology in the
 error message for file:// URLs

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 4915fbd45..6b73b8e06 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1994,7 +1994,7 @@ class YoutubeDL(object):
         file_handler = compat_urllib_request.FileHandler()
 
         def file_open(*args, **kwargs):
-            raise compat_urllib_error.URLError('file:/// protocol is explicitly disabled in youtube-dl for security reasons')
+            raise compat_urllib_error.URLError('file:// scheme is explicitly disabled in youtube-dl for security reasons')
         file_handler.file_open = file_open
 
         opener = compat_urllib_request.build_opener(

From fbd90643cb123011a224da58b4ff1c4ba1c4f8f9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 14 Jan 2016 21:48:08 +0600
Subject: [PATCH 3667/6219] [vodlocker] Fix extraction (Closes #8231)

---
 youtube_dl/extractor/vodlocker.py | 26 +++++++++++++++++++++-----
 1 file changed, 21 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/vodlocker.py b/youtube_dl/extractor/vodlocker.py
index 357594a11..a97995a6d 100644
--- a/youtube_dl/extractor/vodlocker.py
+++ b/youtube_dl/extractor/vodlocker.py
@@ -5,12 +5,13 @@ from .common import InfoExtractor
 from ..compat import compat_urllib_parse
 from ..utils import (
     ExtractorError,
+    NO_DEFAULT,
     sanitized_Request,
 )
 
 
 class VodlockerIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?vodlocker\.com/(?:embed-)?(?P<id>[0-9a-zA-Z]+)(?:\..*?)?'
+    _VALID_URL = r'https?://(?:www\.)?vodlocker\.(?:com|city)/(?:embed-)?(?P<id>[0-9a-zA-Z]+)(?:\..*?)?'
 
     _TESTS = [{
         'url': 'http://vodlocker.com/e8wvyzz4sl42',
@@ -43,16 +44,31 @@ class VodlockerIE(InfoExtractor):
             webpage = self._download_webpage(
                 req, video_id, 'Downloading video page')
 
+        def extract_file_url(html, default=NO_DEFAULT):
+            return self._search_regex(
+                r'file:\s*"(http[^\"]+)",', html, 'file url', default=default)
+
+        video_url = extract_file_url(webpage, default=None)
+
+        if not video_url:
+            embed_url = self._search_regex(
+                r'<iframe[^>]+src=(["\'])(?P<url>(?:https?://)?vodlocker\.(?:com|city)/embed-.+?)\1',
+                webpage, 'embed url', group='url')
+            embed_webpage = self._download_webpage(
+                embed_url, video_id, 'Downloading embed webpage')
+            video_url = extract_file_url(embed_webpage)
+            thumbnail_webpage = embed_webpage
+        else:
+            thumbnail_webpage = webpage
+
         title = self._search_regex(
             r'id="file_title".*?>\s*(.*?)\s*<(?:br|span)', webpage, 'title')
         thumbnail = self._search_regex(
-            r'image:\s*"(http[^\"]+)",', webpage, 'thumbnail')
-        url = self._search_regex(
-            r'file:\s*"(http[^\"]+)",', webpage, 'file url')
+            r'image:\s*"(http[^\"]+)",', thumbnail_webpage, 'thumbnail', fatal=False)
 
         formats = [{
             'format_id': 'sd',
-            'url': url,
+            'url': video_url,
         }]
 
         return {

From 5cc9c5dfa8f731b6582b092e06f78cccbaefc3c4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 14 Jan 2016 21:53:24 +0600
Subject: [PATCH 3668/6219] [unistra] Fix extraction

---
 youtube_dl/extractor/unistra.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/unistra.py b/youtube_dl/extractor/unistra.py
index f70978299..594bee4f9 100644
--- a/youtube_dl/extractor/unistra.py
+++ b/youtube_dl/extractor/unistra.py
@@ -38,7 +38,7 @@ class UnistraIE(InfoExtractor):
 
         webpage = self._download_webpage(url, video_id)
 
-        files = set(re.findall(r'file\s*:\s*"([^"]+)"', webpage))
+        files = set(re.findall(r'file\s*:\s*"(/[^"]+)"', webpage))
 
         quality = qualities(['SD', 'HD'])
         formats = []

From 163e8369b0d2f6b8cc59dd1e93b20a980590648f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 14 Jan 2016 22:05:04 +0600
Subject: [PATCH 3669/6219] [ntvde] Fix extraction

---
 youtube_dl/extractor/ntvde.py | 31 +++++++++++++++++++------------
 1 file changed, 19 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/ntvde.py b/youtube_dl/extractor/ntvde.py
index d2cfe0961..8268eace7 100644
--- a/youtube_dl/extractor/ntvde.py
+++ b/youtube_dl/extractor/ntvde.py
@@ -2,6 +2,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..compat import compat_urlparse
 from ..utils import (
     int_or_none,
     js_to_json,
@@ -42,18 +43,24 @@ class NTVDeIE(InfoExtractor):
             webpage, 'player data'),
             video_id, transform_source=js_to_json)
         duration = parse_duration(vdata.get('duration'))
-        formats = [{
-            'format_id': 'flash',
-            'url': 'rtmp://fms.n-tv.de/' + vdata['video'],
-        }, {
-            'format_id': 'mobile',
-            'url': 'http://video.n-tv.de' + vdata['videoMp4'],
-            'tbr': 400,  # estimation
-        }]
-        m3u8_url = 'http://video.n-tv.de' + vdata['videoM3u8']
-        formats.extend(self._extract_m3u8_formats(
-            m3u8_url, video_id, ext='mp4',
-            entry_protocol='m3u8_native', preference=0))
+
+        formats = []
+        if vdata.get('video'):
+            formats.append({
+                'format_id': 'flash',
+                'url': 'rtmp://fms.n-tv.de/%s' % vdata['video'],
+            })
+        if vdata.get('videoMp4'):
+            formats.append({
+                'format_id': 'mobile',
+                'url': compat_urlparse.urljoin('http://video.n-tv.de', vdata['videoMp4']),
+                'tbr': 400,  # estimation
+            })
+        if vdata.get('videoM3u8'):
+            m3u8_url = compat_urlparse.urljoin('http://video.n-tv.de', vdata['videoM3u8'])
+            formats.extend(self._extract_m3u8_formats(
+                m3u8_url, video_id, ext='mp4', entry_protocol='m3u8_native',
+                preference=0, m3u8_id='hls', fatal=False))
         self._sort_formats(formats)
 
         return {

From 4654c1d01613e26d782c95b13ce60e5fdd84892a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 14 Jan 2016 22:07:42 +0600
Subject: [PATCH 3670/6219] [orf:fm4] Extend _VALID_URL (Closes #8234)

---
 youtube_dl/extractor/orf.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/orf.py b/youtube_dl/extractor/orf.py
index 2e6c9872b..da598e7f7 100644
--- a/youtube_dl/extractor/orf.py
+++ b/youtube_dl/extractor/orf.py
@@ -170,7 +170,7 @@ class ORFOE1IE(InfoExtractor):
 class ORFFM4IE(InfoExtractor):
     IE_NAME = 'orf:fm4'
     IE_DESC = 'radio FM4'
-    _VALID_URL = r'http://fm4\.orf\.at/7tage/?#(?P<date>[0-9]+)/(?P<show>\w+)'
+    _VALID_URL = r'http://fm4\.orf\.at/(?:7tage/?#|player/)(?P<date>[0-9]+)/(?P<show>\w+)'
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)

From 986986064ec102b0d97b4ab008ae38ede6358796 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 14 Jan 2016 22:11:33 +0600
Subject: [PATCH 3671/6219] [orf:fm4] Add test

---
 youtube_dl/extractor/orf.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/youtube_dl/extractor/orf.py b/youtube_dl/extractor/orf.py
index da598e7f7..c54775d54 100644
--- a/youtube_dl/extractor/orf.py
+++ b/youtube_dl/extractor/orf.py
@@ -172,6 +172,20 @@ class ORFFM4IE(InfoExtractor):
     IE_DESC = 'radio FM4'
     _VALID_URL = r'http://fm4\.orf\.at/(?:7tage/?#|player/)(?P<date>[0-9]+)/(?P<show>\w+)'
 
+    _TEST = {
+        'url': 'http://fm4.orf.at/player/20160110/IS/',
+        'md5': '01e736e8f1cef7e13246e880a59ad298',
+        'info_dict': {
+            'id': '2016-01-10_2100_tl_54_7DaysSun13_11244',
+            'ext': 'mp3',
+            'title': 'Im Sumpf',
+            'description': 'md5:384c543f866c4e422a55f66a62d669cd',
+            'duration': 7173,
+            'timestamp': 1452456073,
+            'upload_date': '20160110',
+        },
+    }
+
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         show_date = mobj.group('date')

From 6b559c2fbcf70158bd84b3b5892ecd5fc4b03e91 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 14 Jan 2016 22:12:24 +0600
Subject: [PATCH 3672/6219] [ntvde] Improve regex

---
 youtube_dl/extractor/ntvde.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ntvde.py b/youtube_dl/extractor/ntvde.py
index 8268eace7..a83e85cb8 100644
--- a/youtube_dl/extractor/ntvde.py
+++ b/youtube_dl/extractor/ntvde.py
@@ -35,7 +35,7 @@ class NTVDeIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         info = self._parse_json(self._search_regex(
-            r'(?s)ntv.pageInfo.article =\s(\{.*?\});', webpage, 'info'),
+            r'(?s)ntv\.pageInfo\.article\s*=\s*(\{.*?\});', webpage, 'info'),
             video_id, transform_source=js_to_json)
         timestamp = int_or_none(info.get('publishedDateAsUnixTimeStamp'))
         vdata = self._parse_json(self._search_regex(

From 0baedd1851692a4b9f94c08b3eae5d57acf07f09 Mon Sep 17 00:00:00 2001
From: Lucas <mikotosc@gmail.com>
Date: Wed, 13 Jan 2016 16:11:49 +0100
Subject: [PATCH 3673/6219] [prosiebensat1] add support for 7tv.de

---
 youtube_dl/extractor/prosiebensat1.py | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/prosiebensat1.py b/youtube_dl/extractor/prosiebensat1.py
index baa54a3af..953df3efc 100644
--- a/youtube_dl/extractor/prosiebensat1.py
+++ b/youtube_dl/extractor/prosiebensat1.py
@@ -20,7 +20,7 @@ from ..utils import (
 class ProSiebenSat1IE(InfoExtractor):
     IE_NAME = 'prosiebensat1'
     IE_DESC = 'ProSiebenSat.1 Digital'
-    _VALID_URL = r'https?://(?:www\.)?(?:(?:prosieben|prosiebenmaxx|sixx|sat1|kabeleins|the-voice-of-germany)\.(?:de|at|ch)|ran\.de|fem\.com)/(?P<id>.+)'
+    _VALID_URL = r'https?://(?:www\.)?(?:(?:prosieben|prosiebenmaxx|sixx|sat1|kabeleins|the-voice-of-germany|7tv)\.(?:de|at|ch)|ran\.de|fem\.com)/(?P<id>.+)'
 
     _TESTS = [
         {
@@ -172,6 +172,20 @@ class ProSiebenSat1IE(InfoExtractor):
             },
             'playlist_count': 2,
         },
+        {
+            'url': 'http://www.7tv.de/circus-halligalli/615-best-of-circus-halligalli-ganze-folge',
+            'info_dict': {
+                'id': '4187506',
+                'ext': 'flv',
+                'title': 'Best of Circus HalliGalli',
+                'description': 'md5:8849752efd90b9772c9db6fdf87fb9e9',
+                'upload_date': '20151229',
+            },
+            'params': {
+                # rtmp download
+                'skip_download': True,
+            },
+        },
     ]
 
     _CLIPID_REGEXES = [
@@ -186,12 +200,14 @@ class ProSiebenSat1IE(InfoExtractor):
         r'<!-- start video -->\s*<h1>(.+?)</h1>',
         r'<h1 class="att-name">\s*(.+?)</h1>',
         r'<header class="module_header">\s*<h2>([^<]+)</h2>\s*</header>',
+        r'<h2 class="video-title" itemprop="name">\s*(.+?)</h2>',
     ]
     _DESCRIPTION_REGEXES = [
         r'<p itemprop="description">\s*(.+?)</p>',
         r'<div class="videoDecription">\s*<p><strong>Beschreibung</strong>: (.+?)</p>',
         r'<div class="g-plusone" data-size="medium"></div>\s*</div>\s*</header>\s*(.+?)\s*<footer>',
         r'<p class="att-description">\s*(.+?)\s*</p>',
+        r'<p class="video-description" itemprop="description">\s*(.+?)</p>',
     ]
     _UPLOAD_DATE_REGEXES = [
         r'<meta property="og:published_time" content="(.+?)">',

From 81549898c00a6f12eff4786f3e42e1a53e70c97c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 14 Jan 2016 22:45:09 +0600
Subject: [PATCH 3674/6219] [prosiebensat1] Fix some extraction and update
 tests

---
 youtube_dl/extractor/prosiebensat1.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/prosiebensat1.py b/youtube_dl/extractor/prosiebensat1.py
index 953df3efc..670e6950f 100644
--- a/youtube_dl/extractor/prosiebensat1.py
+++ b/youtube_dl/extractor/prosiebensat1.py
@@ -32,7 +32,7 @@ class ProSiebenSat1IE(InfoExtractor):
             'url': 'http://www.prosieben.de/tv/circus-halligalli/videos/218-staffel-2-episode-18-jahresrueckblick-ganze-folge',
             'info_dict': {
                 'id': '2104602',
-                'ext': 'mp4',
+                'ext': 'flv',
                 'title': 'Episode 18 - Staffel 2',
                 'description': 'md5:8733c81b702ea472e069bc48bb658fc1',
                 'upload_date': '20131231',
@@ -138,14 +138,13 @@ class ProSiebenSat1IE(InfoExtractor):
             'url': 'http://www.the-voice-of-germany.de/video/31-andreas-kuemmert-rocket-man-clip',
             'info_dict': {
                 'id': '2572814',
-                'ext': 'mp4',
+                'ext': 'flv',
                 'title': 'Andreas Kümmert: Rocket Man',
                 'description': 'md5:6ddb02b0781c6adf778afea606652e38',
                 'upload_date': '20131017',
                 'duration': 469.88,
             },
             'params': {
-                # rtmp download
                 'skip_download': True,
             },
         },
@@ -153,13 +152,12 @@ class ProSiebenSat1IE(InfoExtractor):
             'url': 'http://www.fem.com/wellness/videos/wellness-video-clip-kurztripps-zum-valentinstag.html',
             'info_dict': {
                 'id': '2156342',
-                'ext': 'mp4',
+                'ext': 'flv',
                 'title': 'Kurztrips zum Valentinstag',
-                'description': 'Romantischer Kurztrip zum Valentinstag? Wir verraten, was sich hier wirklich lohnt.',
+                'description': 'Romantischer Kurztrip zum Valentinstag? Nina Heinemann verrät, was sich hier wirklich lohnt.',
                 'duration': 307.24,
             },
             'params': {
-                # rtmp download
                 'skip_download': True,
             },
         },
@@ -182,7 +180,6 @@ class ProSiebenSat1IE(InfoExtractor):
                 'upload_date': '20151229',
             },
             'params': {
-                # rtmp download
                 'skip_download': True,
             },
         },
@@ -192,6 +189,7 @@ class ProSiebenSat1IE(InfoExtractor):
         r'"clip_id"\s*:\s+"(\d+)"',
         r'clipid: "(\d+)"',
         r'clip[iI]d=(\d+)',
+        r'clip[iI]d\s*=\s*["\'](\d+)',
         r"'itemImageUrl'\s*:\s*'/dynamic/thumbnails/full/\d+/(\d+)",
     ]
     _TITLE_REGEXES = [
@@ -201,6 +199,7 @@ class ProSiebenSat1IE(InfoExtractor):
         r'<h1 class="att-name">\s*(.+?)</h1>',
         r'<header class="module_header">\s*<h2>([^<]+)</h2>\s*</header>',
         r'<h2 class="video-title" itemprop="name">\s*(.+?)</h2>',
+        r'<div[^>]+id="veeseoTitle"[^>]*>(.+?)</div>',
     ]
     _DESCRIPTION_REGEXES = [
         r'<p itemprop="description">\s*(.+?)</p>',
@@ -208,6 +207,7 @@ class ProSiebenSat1IE(InfoExtractor):
         r'<div class="g-plusone" data-size="medium"></div>\s*</div>\s*</header>\s*(.+?)\s*<footer>',
         r'<p class="att-description">\s*(.+?)\s*</p>',
         r'<p class="video-description" itemprop="description">\s*(.+?)</p>',
+        r'<div[^>]+id="veeseoDescription"[^>]*>(.+?)</div>',
     ]
     _UPLOAD_DATE_REGEXES = [
         r'<meta property="og:published_time" content="(.+?)">',

From be2d40a58a4f89682859bb23789b75330ee756a6 Mon Sep 17 00:00:00 2001
From: Tom Gijselinck <tomgijselinck@gmail.com>
Date: Mon, 12 Oct 2015 09:07:18 +0200
Subject: [PATCH 3675/6219] [Canvas] Add new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/canvas.py   | 80 ++++++++++++++++++++++++++++++++
 2 files changed, 81 insertions(+)
 create mode 100644 youtube_dl/extractor/canvas.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index e4ae9332d..5621c9eb0 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -86,6 +86,7 @@ from .camdemy import (
 )
 from .canalplus import CanalplusIE
 from .canalc2 import Canalc2IE
+from .canvas import CanvasIE
 from .cbs import CBSIE
 from .cbsnews import CBSNewsIE
 from .cbssports import CBSSportsIE
diff --git a/youtube_dl/extractor/canvas.py b/youtube_dl/extractor/canvas.py
new file mode 100644
index 000000000..a37720e98
--- /dev/null
+++ b/youtube_dl/extractor/canvas.py
@@ -0,0 +1,80 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import os
+import urlparse
+
+from youtube_dl import utils
+from .common import InfoExtractor
+
+
+class CanvasIE(InfoExtractor):
+    _VALID_URL = r'(?:https?://)?(?:www\.)?canvas\.be/video/(?P<id>.+)'
+    _TEST = {
+        'url': 'http://www.canvas.be/video/de-afspraak/najaar-2015/de-afspraak-veilt-voor-de-warmste-week',
+        'md5': 'ea838375a547ac787d4064d8c7860a6c',
+        'info_dict': {
+            'id': 'de-afspraak/najaar-2015/de-afspraak-veilt-voor-de-warmste-week',
+            'title': 'De afspraak veilt voor de Warmste Week',
+            'ext': 'mp4',
+            'duration': 49,
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        title = self._search_regex(
+            r'<h1 class="video__body__header__title">(.+?)</h1>', webpage,
+            'title')
+        data_video = self._html_search_regex(
+            r'data-video=(["\'])(?P<id>.+?)\1', webpage, 'data-video', group='id')
+        json_url = 'https://mediazone.vrt.be/api/v1/canvas/assets/' + data_video
+        data = self._download_json(json_url, video_id)
+
+        formats = []
+        for target in data['targetUrls']:
+            if 'type' and 'url' in target:
+                extension = utils.determine_ext(target['url'])
+                if target['type'] == 'PROGRESSIVE_DOWNLOAD':
+                    formats.append({
+                        'format_id': extension,
+                        'url': target['url'],
+                        'protocol': 'http',
+                    })
+                elif target['type'] == 'HLS':
+                    formats.extend(self._extract_m3u8_formats(
+                        target['url'], video_id, entry_protocol='m3u8_native',
+                        ext='mp4',
+                        preference=0,
+                        fatal=False,
+                        m3u8_id='hls'))
+                elif target['type'] == 'HDS':
+                    formats.append({
+                        'format_id': extension,
+                        'url': target['url'],
+                        'protocol': 'HDS',
+                    })
+                elif target['type'] == 'RTMP':
+                    formats.append({
+                        'format_id': extension,
+                        'url': target['url'],
+                        'protocol': 'rtmp',
+                    })
+                elif target['type'] == 'RTSP':
+                    formats.append({
+                        'format_id': extension,
+                        'url': target['url'],
+                        'protocol': 'rtsp',
+                    })
+
+        self._sort_formats(formats)
+        duration = utils.int_or_none(data.get('duration')) / 1000
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': formats,
+            'duration': duration,
+        }

From 4e2743abd9a3f685374cfc1eb4031fb0bc398c58 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 14 Jan 2016 23:15:12 +0600
Subject: [PATCH 3676/6219] [canvas] Improve (Closes #7145)

---
 youtube_dl/extractor/canvas.py | 91 ++++++++++++++--------------------
 1 file changed, 38 insertions(+), 53 deletions(-)

diff --git a/youtube_dl/extractor/canvas.py b/youtube_dl/extractor/canvas.py
index a37720e98..ee19ff836 100644
--- a/youtube_dl/extractor/canvas.py
+++ b/youtube_dl/extractor/canvas.py
@@ -1,80 +1,65 @@
-# coding: utf-8
 from __future__ import unicode_literals
 
-import os
-import urlparse
-
-from youtube_dl import utils
 from .common import InfoExtractor
+from ..utils import float_or_none
 
 
 class CanvasIE(InfoExtractor):
-    _VALID_URL = r'(?:https?://)?(?:www\.)?canvas\.be/video/(?P<id>.+)'
+    _VALID_URL = r'https?://(?:www\.)?canvas\.be/video/(?:[^/]+/)*(?P<id>[^/?#&]+)'
     _TEST = {
         'url': 'http://www.canvas.be/video/de-afspraak/najaar-2015/de-afspraak-veilt-voor-de-warmste-week',
         'md5': 'ea838375a547ac787d4064d8c7860a6c',
         'info_dict': {
-            'id': 'de-afspraak/najaar-2015/de-afspraak-veilt-voor-de-warmste-week',
-            'title': 'De afspraak veilt voor de Warmste Week',
+            'id': 'mz-ast-5e5f90b6-2d72-4c40-82c2-e134f884e93e',
+            'display_id': 'de-afspraak-veilt-voor-de-warmste-week',
             'ext': 'mp4',
-            'duration': 49,
+            'title': 'De afspraak veilt voor de Warmste Week',
+            'description': 'md5:24cb860c320dc2be7358e0e5aa317ba6',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 49.02,
         }
     }
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        display_id = self._match_id(url)
 
-        webpage = self._download_webpage(url, video_id)
+        webpage = self._download_webpage(url, display_id)
 
         title = self._search_regex(
-            r'<h1 class="video__body__header__title">(.+?)</h1>', webpage,
-            'title')
-        data_video = self._html_search_regex(
-            r'data-video=(["\'])(?P<id>.+?)\1', webpage, 'data-video', group='id')
-        json_url = 'https://mediazone.vrt.be/api/v1/canvas/assets/' + data_video
-        data = self._download_json(json_url, video_id)
+            r'<h1[^>]+class="video__body__header__title"[^>]*>(.+?)</h1>',
+            webpage, 'title', default=None) or self._og_search_title(webpage)
+
+        video_id = self._html_search_regex(
+            r'data-video=(["\'])(?P<id>.+?)\1', webpage, 'video id', group='id')
+
+        data = self._download_json(
+            'https://mediazone.vrt.be/api/v1/canvas/assets/%s' % video_id, display_id)
 
         formats = []
         for target in data['targetUrls']:
-            if 'type' and 'url' in target:
-                extension = utils.determine_ext(target['url'])
-                if target['type'] == 'PROGRESSIVE_DOWNLOAD':
-                    formats.append({
-                        'format_id': extension,
-                        'url': target['url'],
-                        'protocol': 'http',
-                    })
-                elif target['type'] == 'HLS':
-                    formats.extend(self._extract_m3u8_formats(
-                        target['url'], video_id, entry_protocol='m3u8_native',
-                        ext='mp4',
-                        preference=0,
-                        fatal=False,
-                        m3u8_id='hls'))
-                elif target['type'] == 'HDS':
-                    formats.append({
-                        'format_id': extension,
-                        'url': target['url'],
-                        'protocol': 'HDS',
-                    })
-                elif target['type'] == 'RTMP':
-                    formats.append({
-                        'format_id': extension,
-                        'url': target['url'],
-                        'protocol': 'rtmp',
-                    })
-                elif target['type'] == 'RTSP':
-                    formats.append({
-                        'format_id': extension,
-                        'url': target['url'],
-                        'protocol': 'rtsp',
-                    })
-
+            format_url, format_type = target.get('url'), target.get('type')
+            if not format_url or not format_type:
+                continue
+            if format_type == 'HLS':
+                formats.extend(self._extract_m3u8_formats(
+                    format_url, display_id, entry_protocol='m3u8_native',
+                    ext='mp4', preference=0, fatal=False, m3u8_id=format_type))
+            elif format_type == 'HDS':
+                formats.extend(self._extract_f4m_formats(
+                    format_url, display_id, f4m_id=format_type, fatal=False))
+            else:
+                formats.append({
+                    'format_id': format_type,
+                    'url': format_url,
+                })
         self._sort_formats(formats)
-        duration = utils.int_or_none(data.get('duration')) / 1000
+
         return {
             'id': video_id,
+            'display_id': display_id,
             'title': title,
+            'description': self._og_search_description(webpage),
             'formats': formats,
-            'duration': duration,
+            'duration': float_or_none(data.get('duration'), 1000),
+            'thumbnail': data.get('posterImageUrl'),
         }

From 5769b68bc018654b959580624b545a9e7e857c5b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 14 Jan 2016 23:15:26 +0600
Subject: [PATCH 3677/6219] Credit @TomGijselinck for canvas (#7145)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 3d8bebbb0..f4238e1d3 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -152,3 +152,4 @@ Evan Lu
 flatgreen
 Brian Foley
 Vignesh Venkat
+Tom Gijselinck

From ffbc0baf72f456b1f4f4ac989e4a7b75b69c7a54 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 15 Jan 2016 12:34:28 +0100
Subject: [PATCH 3678/6219] [history] fix signature and media url
 extraction(fixes #8240)

---
 youtube_dl/extractor/history.py     | 17 +++++++++++++----
 youtube_dl/extractor/theplatform.py |  2 +-
 2 files changed, 14 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/history.py b/youtube_dl/extractor/history.py
index f86164afe..1866ca679 100644
--- a/youtube_dl/extractor/history.py
+++ b/youtube_dl/extractor/history.py
@@ -2,6 +2,7 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..utils import smuggle_url
+from ..compat import compat_urllib_parse
 
 
 class HistoryIE(InfoExtractor):
@@ -19,13 +20,21 @@ class HistoryIE(InfoExtractor):
         'add_ie': ['ThePlatform'],
     }]
 
+    _VIDEO_URL_RE = [
+        r"media_url\s*=\s*'[^']'",
+        r'data-mediaurl="([^"]+)"'
+    ]
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id)
 
-        video_url = self._search_regex(
-            r'data-href="[^"]*/%s"[^>]+data-release-url="([^"]+)"' % video_id,
-            webpage, 'video url')
+        media_url = self._search_regex(self._VIDEO_URL_RE, webpage, 'video url')
 
-        return self.url_result(smuggle_url(video_url, {'sig': {'key': 'crazyjava', 'secret': 's3cr3t'}}))
+        pdk_signature = webpage = self._download_webpage(
+            'https://signature.video.aetndigital.com/?' +
+            compat_urllib_parse.urlencode({'url': media_url}), video_id)
+
+        return self.url_result(smuggle_url(
+            media_url + '?mbr=true&format=smil&sig=' + pdk_signature, {'force_smil_url': True}))
diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index 0bf6726b5..10f2cad55 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -85,7 +85,7 @@ class ThePlatformBaseIE(InfoExtractor):
 class ThePlatformIE(ThePlatformBaseIE):
     _VALID_URL = r'''(?x)
         (?:https?://(?:link|player)\.theplatform\.com/[sp]/(?P<provider_id>[^/]+)/
-           (?:(?P<media>(?:[^/]+/)+select/media/)|(?P<config>(?:[^/\?]+/(?:swf|config)|onsite)/select/))?
+           (?:(?P<media>(?:(?:[^/]+/)+select/)?media/)|(?P<config>(?:[^/\?]+/(?:swf|config)|onsite)/select/))?
          |theplatform:)(?P<id>[^/\?&]+)'''
 
     _TESTS = [{

From fad7a336a162cda2a53a77bf24fe9f666ec3ce07 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 15 Jan 2016 14:54:39 +0100
Subject: [PATCH 3679/6219] Revert "[history] fix signature and media url
 extraction(fixes #8240)"

This reverts commit ffbc0baf72f456b1f4f4ac989e4a7b75b69c7a54.
---
 youtube_dl/extractor/history.py     | 17 ++++-------------
 youtube_dl/extractor/theplatform.py |  2 +-
 2 files changed, 5 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/history.py b/youtube_dl/extractor/history.py
index 1866ca679..f86164afe 100644
--- a/youtube_dl/extractor/history.py
+++ b/youtube_dl/extractor/history.py
@@ -2,7 +2,6 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..utils import smuggle_url
-from ..compat import compat_urllib_parse
 
 
 class HistoryIE(InfoExtractor):
@@ -20,21 +19,13 @@ class HistoryIE(InfoExtractor):
         'add_ie': ['ThePlatform'],
     }]
 
-    _VIDEO_URL_RE = [
-        r"media_url\s*=\s*'[^']'",
-        r'data-mediaurl="([^"]+)"'
-    ]
-
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id)
 
-        media_url = self._search_regex(self._VIDEO_URL_RE, webpage, 'video url')
+        video_url = self._search_regex(
+            r'data-href="[^"]*/%s"[^>]+data-release-url="([^"]+)"' % video_id,
+            webpage, 'video url')
 
-        pdk_signature = webpage = self._download_webpage(
-            'https://signature.video.aetndigital.com/?' +
-            compat_urllib_parse.urlencode({'url': media_url}), video_id)
-
-        return self.url_result(smuggle_url(
-            media_url + '?mbr=true&format=smil&sig=' + pdk_signature, {'force_smil_url': True}))
+        return self.url_result(smuggle_url(video_url, {'sig': {'key': 'crazyjava', 'secret': 's3cr3t'}}))
diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index 10f2cad55..0bf6726b5 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -85,7 +85,7 @@ class ThePlatformBaseIE(InfoExtractor):
 class ThePlatformIE(ThePlatformBaseIE):
     _VALID_URL = r'''(?x)
         (?:https?://(?:link|player)\.theplatform\.com/[sp]/(?P<provider_id>[^/]+)/
-           (?:(?P<media>(?:(?:[^/]+/)+select/)?media/)|(?P<config>(?:[^/\?]+/(?:swf|config)|onsite)/select/))?
+           (?:(?P<media>(?:[^/]+/)+select/media/)|(?P<config>(?:[^/\?]+/(?:swf|config)|onsite)/select/))?
          |theplatform:)(?P<id>[^/\?&]+)'''
 
     _TESTS = [{

From 014b5c59d827c413663394c23384524ef6998f0b Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 15 Jan 2016 15:12:35 +0100
Subject: [PATCH 3680/6219] [theplatform] extend _VALID_URL regex

---
 youtube_dl/extractor/theplatform.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index 0bf6726b5..10f2cad55 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -85,7 +85,7 @@ class ThePlatformBaseIE(InfoExtractor):
 class ThePlatformIE(ThePlatformBaseIE):
     _VALID_URL = r'''(?x)
         (?:https?://(?:link|player)\.theplatform\.com/[sp]/(?P<provider_id>[^/]+)/
-           (?:(?P<media>(?:[^/]+/)+select/media/)|(?P<config>(?:[^/\?]+/(?:swf|config)|onsite)/select/))?
+           (?:(?P<media>(?:(?:[^/]+/)+select/)?media/)|(?P<config>(?:[^/\?]+/(?:swf|config)|onsite)/select/))?
          |theplatform:)(?P<id>[^/\?&]+)'''
 
     _TESTS = [{

From 52767c1ba0ddd515f52bfdc9169cd2037d3eb403 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 15 Jan 2016 15:16:57 +0100
Subject: [PATCH 3681/6219] [history] add support for episode pages(fixes
 #8240)

---
 youtube_dl/extractor/history.py | 26 ++++++++++++++++++++++----
 1 file changed, 22 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/history.py b/youtube_dl/extractor/history.py
index f86164afe..9ec2e92cf 100644
--- a/youtube_dl/extractor/history.py
+++ b/youtube_dl/extractor/history.py
@@ -9,13 +9,29 @@ class HistoryIE(InfoExtractor):
 
     _TESTS = [{
         'url': 'http://www.history.com/topics/valentines-day/history-of-valentines-day/videos/bet-you-didnt-know-valentines-day?m=528e394da93ae&s=undefined&f=1&free=false',
-        'md5': '6fe632d033c92aa10b8d4a9be047a7c5',
         'info_dict': {
-            'id': 'bLx5Dv5Aka1G',
+            'id': 'g12m5Gyt3fdR',
             'ext': 'mp4',
             'title': "Bet You Didn't Know: Valentine's Day",
             'description': 'md5:7b57ea4829b391995b405fa60bd7b5f7',
         },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+        'add_ie': ['ThePlatform'],
+    }, {
+        'url': 'http://www.history.com/shows/mountain-men/season-1/episode-1',
+        'info_dict': {
+            'id': 'eg47EERs_JsZ',
+            'ext': 'mp4',
+            'title': "Winter Is Coming",
+            'description': 'md5:a40e370925074260b1c8a633c632c63a',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
         'add_ie': ['ThePlatform'],
     }]
 
@@ -24,8 +40,10 @@ class HistoryIE(InfoExtractor):
 
         webpage = self._download_webpage(url, video_id)
 
-        video_url = self._search_regex(
+        video_url_re = [
             r'data-href="[^"]*/%s"[^>]+data-release-url="([^"]+)"' % video_id,
-            webpage, 'video url')
+            r"media_url\s*=\s*'([^']+)'"
+        ]
+        video_url = self._search_regex(video_url_re, webpage, 'video url')
 
         return self.url_result(smuggle_url(video_url, {'sig': {'key': 'crazyjava', 'secret': 's3cr3t'}}))

From 587dfd44a4df3fac573e1152ca70f34eabe42611 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 15 Jan 2016 16:18:07 +0100
Subject: [PATCH 3682/6219] [ae] Add support for fyi.tv, aetv.com and
 mylifetime.com(closes #3599)

---
 youtube_dl/extractor/{history.py => ae.py} | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)
 rename youtube_dl/extractor/{history.py => ae.py} (73%)

diff --git a/youtube_dl/extractor/history.py b/youtube_dl/extractor/ae.py
similarity index 73%
rename from youtube_dl/extractor/history.py
rename to youtube_dl/extractor/ae.py
index 9ec2e92cf..880c65350 100644
--- a/youtube_dl/extractor/history.py
+++ b/youtube_dl/extractor/ae.py
@@ -4,8 +4,8 @@ from .common import InfoExtractor
 from ..utils import smuggle_url
 
 
-class HistoryIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?history\.com/(?:[^/]+/)+(?P<id>[^/]+?)(?:$|[?#])'
+class AEIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?(?:(?:history|aetv|mylifetime)\.com|fyi.tv)/(?:[^/]+/)+(?P<id>[^/]+?)(?:$|[?#])'
 
     _TESTS = [{
         'url': 'http://www.history.com/topics/valentines-day/history-of-valentines-day/videos/bet-you-didnt-know-valentines-day?m=528e394da93ae&s=undefined&f=1&free=false',
@@ -33,6 +33,15 @@ class HistoryIE(InfoExtractor):
             'skip_download': True,
         },
         'add_ie': ['ThePlatform'],
+    }, {
+        'url': 'http://www.aetv.com/shows/duck-dynasty/video/inlawful-entry',
+        'only_matching': True
+    }, {
+        'url': 'http://www.fyi.tv/shows/tiny-house-nation/videos/207-sq-ft-minnesota-prairie-cottage',
+        'only_matching': True
+    }, {
+        'url': 'http://www.mylifetime.com/shows/project-runway-junior/video/season-1/episode-6/superstar-clients',
+        'only_matching': True
     }]
 
     def _real_extract(self, url):

From b3f502cdb971c8cc827d8646d8dcb659caa77149 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 15 Jan 2016 21:28:36 +0600
Subject: [PATCH 3683/6219] [xtube] Add shortcut

---
 youtube_dl/extractor/xtube.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/xtube.py b/youtube_dl/extractor/xtube.py
index a1fe24050..2254cd238 100644
--- a/youtube_dl/extractor/xtube.py
+++ b/youtube_dl/extractor/xtube.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 
 class XTubeIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?(?P<url>xtube\.com/watch\.php\?v=(?P<id>[^/?&#]+))'
+    _VALID_URL = r'(?:xtube:|https?://(?:www\.)?xtube\.com/watch\.php\?.*\bv=)(?P<id>[^/?&#]+)'
     _TEST = {
         'url': 'http://www.xtube.com/watch.php?v=kVTUy_G222_',
         'md5': '092fbdd3cbe292c920ef6fc6a8a9cdab',
@@ -30,7 +30,7 @@ class XTubeIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        req = sanitized_Request(url)
+        req = sanitized_Request('http://www.xtube.com/watch.php?v=%s' % video_id)
         req.add_header('Cookie', 'age_verified=1')
         webpage = self._download_webpage(req, video_id)
 

From 34dd81c03a23ef7e4db31f8aa9a61473f2d8f727 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 15 Jan 2016 21:35:20 +0600
Subject: [PATCH 3684/6219] [xtube:user] Fix extraction (Closes #8224)

---
 youtube_dl/extractor/xtube.py | 60 +++++++++++++++++------------------
 1 file changed, 30 insertions(+), 30 deletions(-)

diff --git a/youtube_dl/extractor/xtube.py b/youtube_dl/extractor/xtube.py
index 2254cd238..8cd3a0687 100644
--- a/youtube_dl/extractor/xtube.py
+++ b/youtube_dl/extractor/xtube.py
@@ -1,10 +1,12 @@
 from __future__ import unicode_literals
 
+import itertools
 import re
 
 from .common import InfoExtractor
 from ..compat import compat_urllib_parse_unquote
 from ..utils import (
+    int_or_none,
     parse_duration,
     sanitized_Request,
     str_to_int,
@@ -88,45 +90,43 @@ class XTubeIE(InfoExtractor):
 
 class XTubeUserIE(InfoExtractor):
     IE_DESC = 'XTube user profile'
-    _VALID_URL = r'https?://(?:www\.)?xtube\.com/community/profile\.php\?(.*?)user=(?P<username>[^&#]+)(?:$|[&#])'
+    _VALID_URL = r'https?://(?:www\.)?xtube\.com/profile/(?P<id>[^/]+-\d+)'
     _TEST = {
-        'url': 'http://www.xtube.com/community/profile.php?user=greenshowers',
+        'url': 'http://www.xtube.com/profile/greenshowers-4056496',
         'info_dict': {
-            'id': 'greenshowers',
+            'id': 'greenshowers-4056496',
             'age_limit': 18,
         },
         'playlist_mincount': 155,
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        username = mobj.group('username')
+        user_id = self._match_id(url)
 
-        profile_page = self._download_webpage(
-            url, username, note='Retrieving profile page')
+        entries = []
+        for pagenum in itertools.count(1):
+            request = sanitized_Request(
+                'http://www.xtube.com/profile/%s/videos/%d' % (user_id, pagenum),
+                headers={
+                    'Cookie': 'popunder=4',
+                    'X-Requested-With': 'XMLHttpRequest',
+                    'Referer': url,
+                })
 
-        video_count = int(self._search_regex(
-            r'<strong>%s\'s Videos \(([0-9]+)\)</strong>' % username, profile_page,
-            'video count'))
+            page = self._download_json(
+                request, user_id, 'Downloading videos JSON page %d' % pagenum)
 
-        PAGE_SIZE = 25
-        urls = []
-        page_count = (video_count + PAGE_SIZE + 1) // PAGE_SIZE
-        for n in range(1, page_count + 1):
-            lpage_url = 'http://www.xtube.com/user_videos.php?page=%d&u=%s' % (n, username)
-            lpage = self._download_webpage(
-                lpage_url, username,
-                note='Downloading page %d/%d' % (n, page_count))
-            urls.extend(
-                re.findall(r'addthis:url="([^"]+)"', lpage))
+            html = page.get('html')
+            if not html:
+                break
 
-        return {
-            '_type': 'playlist',
-            'id': username,
-            'age_limit': 18,
-            'entries': [{
-                '_type': 'url',
-                'url': eurl,
-                'ie_key': 'XTube',
-            } for eurl in urls]
-        }
+            for _, video_id in re.findall(r'data-plid=(["\'])(.+?)\1', html):
+                entries.append(self.url_result('xtube:%s' % video_id, XTubeIE.ie_key()))
+
+            page_count = int_or_none(page.get('pageCount'))
+            if not page_count or pagenum == page_count:
+                break
+
+        playlist = self.playlist_result(entries, user_id)
+        playlist['age_limit'] = 18
+        return playlist

From 3a421c724f60b7cc9fe870e343e8eb62d587a620 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 15 Jan 2016 22:10:07 +0600
Subject: [PATCH 3685/6219] [history] Remove import (Closes #8243)

---
 youtube_dl/extractor/__init__.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 5621c9eb0..f50a5d625 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -263,7 +263,6 @@ from .hellporno import HellPornoIE
 from .helsinki import HelsinkiIE
 from .hentaistigma import HentaiStigmaIE
 from .historicfilms import HistoricFilmsIE
-from .history import HistoryIE
 from .hitbox import HitboxIE, HitboxLiveIE
 from .hornbunny import HornBunnyIE
 from .hotnewhiphop import HotNewHipHopIE

From 567f9a5809daf51ed48cf8292ab741f765937d56 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 15 Jan 2016 22:12:51 +0600
Subject: [PATCH 3686/6219] [ae] Add extractor import

---
 youtube_dl/extractor/__init__.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index f50a5d625..3975354da 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -15,6 +15,7 @@ from .adobetv import (
     AdobeTVVideoIE,
 )
 from .adultswim import AdultSwimIE
+from .ae import AEIE
 from .aftonbladet import AftonbladetIE
 from .airmozilla import AirMozillaIE
 from .aljazeera import AlJazeeraIE

From a78d6a9bb1387db4aa29f1ac1e4edad8d2061783 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 15 Jan 2016 22:13:48 +0600
Subject: [PATCH 3687/6219] [ae] Improve _VALID_URL

---
 youtube_dl/extractor/ae.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ae.py b/youtube_dl/extractor/ae.py
index 880c65350..a3b9f1ca4 100644
--- a/youtube_dl/extractor/ae.py
+++ b/youtube_dl/extractor/ae.py
@@ -5,7 +5,7 @@ from ..utils import smuggle_url
 
 
 class AEIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?(?:(?:history|aetv|mylifetime)\.com|fyi.tv)/(?:[^/]+/)+(?P<id>[^/]+?)(?:$|[?#])'
+    _VALID_URL = r'https?://(?:www\.)?(?:(?:history|aetv|mylifetime)\.com|fyi\.tv)/(?:[^/]+/)+(?P<id>[^/]+?)(?:$|[?#])'
 
     _TESTS = [{
         'url': 'http://www.history.com/topics/valentines-day/history-of-valentines-day/videos/bet-you-didnt-know-valentines-day?m=528e394da93ae&s=undefined&f=1&free=false',

From 27a95f51aaf75275ed0f7a7d2e8763d874c649a4 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 15 Jan 2016 17:45:51 +0100
Subject: [PATCH 3688/6219] [cwtv] Add new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/cwtv.py     | 88 ++++++++++++++++++++++++++++++++
 2 files changed, 89 insertions(+)
 create mode 100644 youtube_dl/extractor/cwtv.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 3975354da..d7a9df202 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -130,6 +130,7 @@ from .crunchyroll import (
 )
 from .cspan import CSpanIE
 from .ctsnews import CtsNewsIE
+from .cwtv import CWTVIE
 from .dailymotion import (
     DailymotionIE,
     DailymotionPlaylistIE,
diff --git a/youtube_dl/extractor/cwtv.py b/youtube_dl/extractor/cwtv.py
new file mode 100644
index 000000000..36af67013
--- /dev/null
+++ b/youtube_dl/extractor/cwtv.py
@@ -0,0 +1,88 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    parse_iso8601,
+)
+
+
+class CWTVIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?cw(?:tv|seed)\.com/shows/(?:[^/]+/){2}\?play=(?P<id>[a-z0-9]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12})'
+    _TESTS = [{
+        'url': 'http://cwtv.com/shows/arrow/legends-of-yesterday/?play=6b15e985-9345-4f60-baf8-56e96be57c63',
+        'info_dict': {
+            'id': '6b15e985-9345-4f60-baf8-56e96be57c63',
+            'ext': 'mp4',
+            'title': 'Legends of Yesterday',
+            'description': 'Oliver and Barry Allen take Kendra Saunders and Carter Hall to a remote location to keep them hidden from Vandal Savage while they figure out how to defeat him.',
+            'duration': 2665,
+            'series': 'Arrow',
+            'season_number': 4,
+            'season': '4',
+            'episode_number': 8,
+            'upload_date': '20151203',
+            'timestamp': 1449122100,
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        }
+    }, {
+        'url': 'http://www.cwseed.com/shows/whose-line-is-it-anyway/jeff-davis-4/?play=24282b12-ead2-42f2-95ad-26770c2c6088',
+        'info_dict': {
+            'id': '24282b12-ead2-42f2-95ad-26770c2c6088',
+            'ext': 'mp4',
+            'title': 'Jeff Davis 4',
+            'description': 'Jeff Davis is back to make you laugh.',
+            'duration': 1263,
+            'series': 'Whose Line Is It Anyway?',
+            'season_number': 11,
+            'season': '11',
+            'episode_number': 20,
+            'upload_date': '20151006',
+            'timestamp': 1444107300,
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        }
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        video_data = self._download_json(
+            'http://metaframe.digitalsmiths.tv/v2/CWtv/assets/%s/partner/132?format=json' % video_id, video_id)
+
+        formats = self._extract_m3u8_formats(
+            video_data['videos']['variantplaylist']['uri'], video_id, 'mp4')
+
+        thumbnails = [{
+            'url': image['uri'],
+            'width': image.get('width'),
+            'height': image.get('height'),
+        } for image_id, image in video_data['images'].items() if image.get('uri')] if video_data.get('images') else None
+
+        video_metadata = video_data['assetFields']
+
+        subtitles = {
+            'en': [{
+                'url': video_metadata['UnicornCcUrl'],
+            }],
+        } if video_metadata.get('UnicornCcUrl') else None
+
+        return {
+            'id': video_id,
+            'title': video_metadata['title'],
+            'description': video_metadata.get('description'),
+            'duration': int_or_none(video_metadata.get('duration')),
+            'series': video_metadata.get('seriesName'),
+            'season_number': int_or_none(video_metadata.get('seasonNumber')),
+            'season': video_metadata.get('seasonName'),
+            'episode_number': int_or_none(video_metadata.get('episodeNumber')),
+            'timestamp': parse_iso8601(video_data.get('startTime')),
+            'thumbnails': thumbnails,
+            'formats': formats,
+            'subtitles': subtitles,
+        }

From 33d2fc2f640c32b4b5469ecb3a48e7e209844407 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 16 Jan 2016 00:09:54 +0600
Subject: [PATCH 3689/6219] [YoutubeDL] Auto generate title fields
 corresponding to the *_number fields

Auto generate title fields corresponding to the *_number fields when missing in order to always have clean titles. This is very common for TV series.
---
 youtube_dl/YoutubeDL.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 6b73b8e06..0748fbba0 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1244,6 +1244,12 @@ class YoutubeDL(object):
             except (ValueError, OverflowError, OSError):
                 pass
 
+        # Auto generate title fields corresponding to the *_number fields when missing
+        # in order to always have clean titles. This is very common for TV series.
+        for field in ('chapter', 'season', 'episode'):
+            if info_dict.get('%s_number' % field) is not None and not info_dict.get(field):
+                info_dict[field] = '%s %d' % (field.capitalize(), info_dict['%s_number' % field])
+
         subtitles = info_dict.get('subtitles')
         if subtitles:
             for _, subtitle in subtitles.items():

From 4ca2a3cf3cf0227025f09cfe6e75a37f884442a5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 16 Jan 2016 00:36:02 +0600
Subject: [PATCH 3690/6219] [extractor/common] Add initial support for JSON-LD
 metadata extraction into info_dict

---
 youtube_dl/extractor/common.py | 37 ++++++++++++++++++++++++++++++++++
 1 file changed, 37 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index b05b22a94..92e2e4f43 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -34,6 +34,7 @@ from ..utils import (
     fix_xml_ampersands,
     float_or_none,
     int_or_none,
+    parse_iso8601,
     RegexNotFoundError,
     sanitize_filename,
     sanitized_Request,
@@ -762,6 +763,42 @@ class InfoExtractor(object):
         return self._html_search_meta('twitter:player', html,
                                       'twitter card player')
 
+    def _search_json_ld(self, html, video_id, fatal=True):
+        json_ld = self._search_regex(
+            r'(?s)<script[^>]+type=(["\'])application/ld\+json\1[^>]*>(?P<json_ld>.+?)</script>',
+            html, 'JSON-LD', fatal=fatal, group='json_ld')
+        if not json_ld:
+            return {}
+        return self._json_ld(json_ld, video_id, fatal=fatal)
+
+    def _json_ld(self, json_ld, video_id, fatal=True):
+        if isinstance(json_ld, compat_str):
+            json_ld = self._parse_json(json_ld, video_id, fatal=fatal)
+        if not json_ld:
+            return {}
+        info = {}
+        if json_ld.get('@context') == 'http://schema.org':
+            item_type = json_ld.get('@type')
+            if item_type == 'TVEpisode':
+                info.update({
+                    'episode': unescapeHTML(json_ld.get('name')),
+                    'episode_number': int_or_none(json_ld.get('episodeNumber')),
+                    'description': unescapeHTML(json_ld.get('description')),
+                })
+                part_of_season = json_ld.get('partOfSeason')
+                if isinstance(part_of_season, dict) and part_of_season.get('@type') == 'TVSeason':
+                    info['season_number'] = int_or_none(part_of_season.get('seasonNumber'))
+                part_of_series = json_ld.get('partOfSeries')
+                if isinstance(part_of_series, dict) and part_of_series.get('@type') == 'TVSeries':
+                    info['series'] = unescapeHTML(part_of_series.get('name'))
+            elif item_type == 'Article':
+                info.update({
+                    'timestamp': parse_iso8601(json_ld.get('datePublished')),
+                    'title': unescapeHTML(json_ld.get('headline')),
+                    'description': unescapeHTML(json_ld.get('articleBody')),
+                })
+        return dict((k, v) for k, v in info.items() if v is not None)
+
     @staticmethod
     def _hidden_inputs(html):
         html = re.sub(r'<!--(?:(?!<!--).)*-->', '', html)

From b4e44234bc481747b88c330b0e30a13dbc0fc99a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 16 Jan 2016 00:36:49 +0600
Subject: [PATCH 3691/6219] [ae] Use JSON-LD for TV series metadata

---
 youtube_dl/extractor/ae.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ae.py b/youtube_dl/extractor/ae.py
index a3b9f1ca4..3bc7c12fc 100644
--- a/youtube_dl/extractor/ae.py
+++ b/youtube_dl/extractor/ae.py
@@ -55,4 +55,9 @@ class AEIE(InfoExtractor):
         ]
         video_url = self._search_regex(video_url_re, webpage, 'video url')
 
-        return self.url_result(smuggle_url(video_url, {'sig': {'key': 'crazyjava', 'secret': 's3cr3t'}}))
+        info = self._search_json_ld(webpage, video_id, fatal=False)
+        info.update({
+            '_type': 'url_transparent',
+            'url': smuggle_url(video_url, {'sig': {'key': 'crazyjava', 'secret': 's3cr3t'}}),
+        })
+        return info

From f3ed5df6118169894a27ee70a15c6202ba3823b4 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 15 Jan 2016 19:43:04 +0100
Subject: [PATCH 3692/6219] release 2016.01.15

---
 docs/supportedsites.md | 4 +++-
 youtube_dl/version.py  | 2 +-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index eb160bd2f..99b1e2731 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -24,6 +24,7 @@
  - **AdobeTVShow**
  - **AdobeTVVideo**
  - **AdultSwim**
+ - **AE**
  - **Aftonbladet**
  - **AirMozilla**
  - **AlJazeera**
@@ -85,6 +86,7 @@
  - **CamdemyFolder**
  - **canalc2.tv**
  - **Canalplus**: canalplus.fr, piwiplus.fr and d8.tv
+ - **Canvas**
  - **CBS**
  - **CBSNews**: CBS News
  - **CBSSports**
@@ -122,6 +124,7 @@
  - **CSpan**: C-SPAN
  - **CtsNews**: 華視新聞
  - **culturebox.francetvinfo.fr**
+ - **CWTV**
  - **dailymotion**
  - **dailymotion:playlist**
  - **dailymotion:user**
@@ -229,7 +232,6 @@
  - **Helsinki**: helsinki.fi
  - **HentaiStigma**
  - **HistoricFilms**
- - **History**
  - **hitbox**
  - **hitbox:live**
  - **HornBunny**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 4d433b667..02c438f3a 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.01.14'
+__version__ = '2016.01.15'

From 2ddfd26f1be72bc8e92ec539cdd969440f301f67 Mon Sep 17 00:00:00 2001
From: ckuu <cku@gmx.com>
Date: Sun, 10 Jan 2016 15:20:59 +0400
Subject: [PATCH 3693/6219] '[ZippCast] Add new extractor' Closes
 rg3/youtube-dl#6591

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/zippcast.py | 58 ++++++++++++++++++++++++++++++++
 2 files changed, 59 insertions(+)
 create mode 100644 youtube_dl/extractor/zippcast.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index d7a9df202..5b8e16a29 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -915,6 +915,7 @@ from .zingmp3 import (
     ZingMp3SongIE,
     ZingMp3AlbumIE,
 )
+from .zippcast import ZippCastIE
 
 _ALL_CLASSES = [
     klass
diff --git a/youtube_dl/extractor/zippcast.py b/youtube_dl/extractor/zippcast.py
new file mode 100644
index 000000000..afeab4114
--- /dev/null
+++ b/youtube_dl/extractor/zippcast.py
@@ -0,0 +1,58 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class ZippCastIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?zippcast\.com/video/(?P<id>[0-9a-zA-Z]+)'
+    _TESTS = [{
+        'url': 'http://www.zippcast.com/video/c9cfd5c7e44dbc29c81',
+        'md5': 'f2aea8659962d9155031aaeac53f7c54',
+        'info_dict': {
+            'id': 'c9cfd5c7e44dbc29c81',
+            'ext': 'mp4',
+            'title': '[Vinesauce] Vinny - Digital Space Traveler',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'uploader': 'vinesauce',
+            'description': 'Muted on youtube, but now uploaded in it\'s original form.',
+            'categories': ['Entertainment'],
+            'view_count': int,
+        },
+    }, {
+        'url': 'http://www.zippcast.com/video/b79c0a233e9c6581775',
+        'md5': 'b8631f0cc48ed15387f9179988d0c97c',
+        'info_dict': {
+            'id': 'b79c0a233e9c6581775',
+            'ext': 'mp4',
+            'title': 'Battlefield Hardline Trailer',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'uploader': 'IGXGaming',
+            'description': 'Battlefield Hardline Trailer',
+            'categories': ['Gaming'],
+            'view_count': int,
+        },
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        title = self._html_search_regex(r'title="(.+?)"', webpage, 'title')
+        uploader = self._html_search_regex(r'http://www.zippcast.com/profile/(.+?)">', webpage, 'uploader')
+        url = self._html_search_regex(r'<source src="(.+?)" type="', webpage, 'url')
+        description = self._html_search_regex(r'<span class="vdescr".+>(.+?)</span>', webpage, 'description')
+        thumbnail = self._html_search_regex(r'poster="(.+?)" controls>', webpage, 'thumbnail')
+        categories = self._html_search_regex(r'<a href="http://www.zippcast.com/categories/(.+?)"', webpage, 'categories')
+        view_count = self._html_search_regex(r'<td align="right"><h3>(.+?) views!', webpage, 'view_count')
+
+        return {
+            'id': video_id,
+            'title': title,
+            'url': url,
+            'description': description,
+            'uploader': uploader,
+            'thumbnail': thumbnail,
+            'categories': [categories],
+            'view_count': int(view_count.replace(',', '')),
+        }

From 315bdae00a906b0e07d400b63754abadec34dab5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 16 Jan 2016 06:25:25 +0600
Subject: [PATCH 3694/6219] [zippcast] Improve (Closes #8198)

---
 youtube_dl/extractor/zippcast.py | 92 ++++++++++++++++++++++----------
 1 file changed, 64 insertions(+), 28 deletions(-)

diff --git a/youtube_dl/extractor/zippcast.py b/youtube_dl/extractor/zippcast.py
index afeab4114..de819376d 100644
--- a/youtube_dl/extractor/zippcast.py
+++ b/youtube_dl/extractor/zippcast.py
@@ -1,58 +1,94 @@
-# coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
+from ..utils import (
+    determine_ext,
+    str_to_int,
+)
 
 
 class ZippCastIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?zippcast\.com/video/(?P<id>[0-9a-zA-Z]+)'
+    _VALID_URL = r'https?://(?:www\.)?zippcast\.com/(?:video/|videoview\.php\?.*\bvplay=)(?P<id>[0-9a-zA-Z]+)'
     _TESTS = [{
+        # m3u8, hq direct link
         'url': 'http://www.zippcast.com/video/c9cfd5c7e44dbc29c81',
-        'md5': 'f2aea8659962d9155031aaeac53f7c54',
+        'md5': '5ea0263b5606866c4d6cda0fc5e8c6b6',
         'info_dict': {
             'id': 'c9cfd5c7e44dbc29c81',
             'ext': 'mp4',
             'title': '[Vinesauce] Vinny - Digital Space Traveler',
+            'description': 'Muted on youtube, but now uploaded in it\'s original form.',
             'thumbnail': 're:^https?://.*\.jpg$',
             'uploader': 'vinesauce',
-            'description': 'Muted on youtube, but now uploaded in it\'s original form.',
-            'categories': ['Entertainment'],
             'view_count': int,
+            'categories': ['Entertainment'],
+            'tags': list,
         },
     }, {
+        # f4m, lq ipod direct link
         'url': 'http://www.zippcast.com/video/b79c0a233e9c6581775',
-        'md5': 'b8631f0cc48ed15387f9179988d0c97c',
-        'info_dict': {
-            'id': 'b79c0a233e9c6581775',
-            'ext': 'mp4',
-            'title': 'Battlefield Hardline Trailer',
-            'thumbnail': 're:^https?://.*\.jpg$',
-            'uploader': 'IGXGaming',
-            'description': 'Battlefield Hardline Trailer',
-            'categories': ['Gaming'],
-            'view_count': int,
-        },
+        'only_matching': True,
+    }, {
+        'url': 'http://www.zippcast.com/videoview.php?vplay=c9cfd5c7e44dbc29c81&auto=no',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
 
-        title = self._html_search_regex(r'title="(.+?)"', webpage, 'title')
-        uploader = self._html_search_regex(r'http://www.zippcast.com/profile/(.+?)">', webpage, 'uploader')
-        url = self._html_search_regex(r'<source src="(.+?)" type="', webpage, 'url')
-        description = self._html_search_regex(r'<span class="vdescr".+>(.+?)</span>', webpage, 'description')
-        thumbnail = self._html_search_regex(r'poster="(.+?)" controls>', webpage, 'thumbnail')
-        categories = self._html_search_regex(r'<a href="http://www.zippcast.com/categories/(.+?)"', webpage, 'categories')
-        view_count = self._html_search_regex(r'<td align="right"><h3>(.+?) views!', webpage, 'view_count')
+        webpage = self._download_webpage(
+            'http://www.zippcast.com/video/%s' % video_id, video_id)
+
+        formats = []
+        video_url = self._search_regex(
+            r'<source[^>]+src=(["\'])(?P<url>.+?)\1', webpage,
+            'video url', default=None, group='url')
+        if video_url:
+            formats.append({
+                'url': video_url,
+                'format_id': 'http',
+                'preference': 0,  # direct link is almost always of worse quality
+            })
+        src_url = self._search_regex(
+            r'src\s*:\s*(?:escape\()?(["\'])(?P<url>http://.+?)\1',
+            webpage, 'src', default=None, group='url')
+        ext = determine_ext(src_url)
+        if ext == 'm3u8':
+            formats.extend(self._extract_m3u8_formats(
+                src_url, video_id, 'mp4', entry_protocol='m3u8_native',
+                m3u8_id='hls', fatal=False))
+        elif ext == 'f4m':
+            formats.extend(self._extract_f4m_formats(
+                src_url, video_id, f4m_id='hds', fatal=False))
+        self._sort_formats(formats)
+
+        title = self._og_search_title(webpage)
+        description = self._og_search_description(webpage) or self._html_search_meta(
+            'description', webpage)
+        uploader = self._search_regex(
+            r'<a[^>]+href="https?://[^/]+/profile/[^>]+>([^<]+)</a>',
+            webpage, 'uploader', fatal=False)
+        thumbnail = self._og_search_thumbnail(webpage)
+        view_count = str_to_int(self._search_regex(
+            r'>([\d,.]+) views!', webpage, 'view count', fatal=False))
+
+        categories = re.findall(
+            r'<a[^>]+href="https?://[^/]+/categories/[^"]+">([^<]+),?<',
+            webpage)
+        tags = re.findall(
+            r'<a[^>]+href="https?://[^/]+/search/tags/[^"]+">([^<]+),?<',
+            webpage)
 
         return {
             'id': video_id,
             'title': title,
-            'url': url,
             'description': description,
-            'uploader': uploader,
             'thumbnail': thumbnail,
-            'categories': [categories],
-            'view_count': int(view_count.replace(',', '')),
+            'uploader': uploader,
+            'view_count': view_count,
+            'categories': categories,
+            'tags': tags,
+            'formats': formats,
         }

From a50a8003a08bc67bc99363f6738147a394a43210 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 16 Jan 2016 07:10:51 +0600
Subject: [PATCH 3695/6219] [cultureunplugged] Improve (Closes #8060)

---
 youtube_dl/extractor/__init__.py         |  1 +
 youtube_dl/extractor/cultureunplugged.py | 63 ++++++++++++++++++++++++
 2 files changed, 64 insertions(+)
 create mode 100644 youtube_dl/extractor/cultureunplugged.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 5b8e16a29..2c56797a5 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -130,6 +130,7 @@ from .crunchyroll import (
 )
 from .cspan import CSpanIE
 from .ctsnews import CtsNewsIE
+from .cultureunplugged import CultureUnpluggedIE
 from .cwtv import CWTVIE
 from .dailymotion import (
     DailymotionIE,
diff --git a/youtube_dl/extractor/cultureunplugged.py b/youtube_dl/extractor/cultureunplugged.py
new file mode 100644
index 000000000..9c764fe68
--- /dev/null
+++ b/youtube_dl/extractor/cultureunplugged.py
@@ -0,0 +1,63 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import int_or_none
+
+
+class CultureUnpluggedIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?cultureunplugged\.com/documentary/watch-online/play/(?P<id>\d+)(?:/(?P<display_id>[^/]+))?'
+    _TESTS = [{
+        'url': 'http://www.cultureunplugged.com/documentary/watch-online/play/53662/The-Next--Best-West',
+        'md5': 'ac6c093b089f7d05e79934dcb3d228fc',
+        'info_dict': {
+            'id': '53662',
+            'display_id': 'The-Next--Best-West',
+            'ext': 'mp4',
+            'title': 'The Next, Best West',
+            'description': 'md5:0423cd00833dea1519cf014e9d0903b1',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'creator': 'Coldstream Creative',
+            'duration': 2203,
+            'view_count': int,
+        }
+    }, {
+        'url': 'http://www.cultureunplugged.com/documentary/watch-online/play/53662',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        display_id = mobj.group('display_id') or video_id
+
+        movie_data = self._download_json(
+            'http://www.cultureunplugged.com/movie-data/cu-%s.json' % video_id, display_id)
+
+        video_url = movie_data['url']
+        title = movie_data['title']
+
+        description = movie_data.get('synopsis')
+        creator = movie_data.get('producer')
+        duration = int_or_none(movie_data.get('duration'))
+        view_count = int_or_none(movie_data.get('views'))
+
+        thumbnails = [{
+            'url': movie_data['%s_thumb' % size],
+            'id': size,
+            'preference': preference,
+        } for preference, size in enumerate((
+            'small', 'large')) if movie_data.get('%s_thumb' % size)]
+
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            'url': video_url,
+            'title': title,
+            'description': description,
+            'creator': creator,
+            'duration': duration,
+            'view_count': view_count,
+            'thumbnails': thumbnails,
+        }

From 10d33b347391ed0af453a88d2860a8ad7d8d18c7 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 13 Jan 2016 16:24:48 +0800
Subject: [PATCH 3696/6219] [YoutubeDL] Introduce CSS3 like string operators

---
 youtube_dl/YoutubeDL.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 0748fbba0..18b1443b8 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -898,6 +898,9 @@ class YoutubeDL(object):
             STR_OPERATORS = {
                 '=': operator.eq,
                 '!=': operator.ne,
+                '^=': lambda attr, value: attr.startswith(value),
+                '$=': lambda attr, value: attr.endswith(value),
+                '*=': lambda attr, value: value in attr,
             }
             str_operator_rex = re.compile(r'''(?x)
                 \s*(?P<key>ext|acodec|vcodec|container|protocol)

From 15870bbd018415231c40d9e4319ef43e563ab7b8 Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Sat, 16 Jan 2016 07:31:14 +0600
Subject: [PATCH 3697/6219] [README.md] Mention new string operators for format
 selection

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 75253199c..af6907be3 100644
--- a/README.md
+++ b/README.md
@@ -467,7 +467,7 @@ youtube-dl_test_video_.mp4          # A simple file name
 By default youtube-dl tries to download the best quality, but sometimes you may want to download in a different format.
 The simplest case is requesting a specific format, for example `-f 22`. You can get the list of available formats using `--list-formats`, you can also use a file extension (currently it supports aac, m4a, mp3, mp4, ogg, wav, webm) or the special names `best`, `bestvideo`, `bestaudio` and `worst`.
 
-If you want to download multiple videos and they don't have the same formats available, you can specify the order of preference using slashes, as in `-f 22/17/18`. You can also filter the video results by putting a condition in brackets, as in `-f "best[height=720]"` (or `-f "[filesize>10M]"`).  This works for filesize, height, width, tbr, abr, vbr, asr, and fps and the comparisons <, <=, >, >=, =, != and for ext, acodec, vcodec, container, and protocol and the comparisons =, != . Formats for which the value is not known are excluded unless you put a question mark (?) after the operator. You can combine format filters, so  `-f "[height <=? 720][tbr>500]"` selects up to 720p videos (or videos where the height is not known) with a bitrate of at least 500 KBit/s. Use commas to download multiple formats, such as `-f 136/137/mp4/bestvideo,140/m4a/bestaudio`. You can merge the video and audio of two formats into a single file using `-f <video-format>+<audio-format>` (requires ffmpeg or avconv), for example `-f bestvideo+bestaudio`. Format selectors can also be grouped using parentheses, for example if you want to download the best mp4 and webm formats with a height lower than 480 you can use `-f '(mp4,webm)[height<480]'`.
+If you want to download multiple videos and they don't have the same formats available, you can specify the order of preference using slashes, as in `-f 22/17/18`. You can also filter the video results by putting a condition in brackets, as in `-f "best[height=720]"` (or `-f "[filesize>10M]"`).  This works for filesize, height, width, tbr, abr, vbr, asr, and fps and the comparisons <, <=, >, >=, =, != and for ext, acodec, vcodec, container, and protocol and the comparisons =, !=, ^= (begins with), $= (ends with), *= (contains) . Formats for which the value is not known are excluded unless you put a question mark (?) after the operator. You can combine format filters, so  `-f "[height <=? 720][tbr>500]"` selects up to 720p videos (or videos where the height is not known) with a bitrate of at least 500 KBit/s. Use commas to download multiple formats, such as `-f 136/137/mp4/bestvideo,140/m4a/bestaudio`. You can merge the video and audio of two formats into a single file using `-f <video-format>+<audio-format>` (requires ffmpeg or avconv), for example `-f bestvideo+bestaudio`. Format selectors can also be grouped using parentheses, for example if you want to download the best mp4 and webm formats with a height lower than 480 you can use `-f '(mp4,webm)[height<480]'`.
 
 Since the end of April 2015 and version 2015.04.26 youtube-dl uses `-f bestvideo+bestaudio/best` as default format selection (see #5447, #5456). If ffmpeg or avconv are installed this results in downloading `bestvideo` and `bestaudio` separately and muxing them together into a single file giving the best overall quality available. Otherwise it falls back to `best` and results in downloading the best available quality served as a single file. `best` is also needed for videos that don't come from YouTube because they don't provide the audio and video in two different files. If you want to only download some dash formats (for example if you are not interested in getting videos with a resolution higher than 1080p), you can add `-f bestvideo[height<=?1080]+bestaudio/best` to your configuration file. Note that if you use youtube-dl to stream to `stdout` (and most likely to pipe it to your media player then), i.e. you explicitly specify output template as `-o -`, youtube-dl still uses `-f best` format selection in order to start content delivery immediately to your player and not to wait until `bestvideo` and `bestaudio` are downloaded and muxed.
 

From b2dca40d8138cb044bd6c6f75a4e87ecb633fc00 Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Sat, 16 Jan 2016 09:49:54 +0600
Subject: [PATCH 3698/6219] [README.md] Improve format selection documentation

---
 README.md | 67 +++++++++++++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 63 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index af6907be3..efb51027b 100644
--- a/README.md
+++ b/README.md
@@ -464,15 +464,74 @@ youtube-dl_test_video_.mp4          # A simple file name
 
 # FORMAT SELECTION
 
-By default youtube-dl tries to download the best quality, but sometimes you may want to download in a different format.
-The simplest case is requesting a specific format, for example `-f 22`. You can get the list of available formats using `--list-formats`, you can also use a file extension (currently it supports aac, m4a, mp3, mp4, ogg, wav, webm) or the special names `best`, `bestvideo`, `bestaudio` and `worst`.
+By default youtube-dl tries to download the best available quality, i.e. if you want the best quality you **don't need** to pass any special options, youtube-dl will guess it for you by **default**.
 
-If you want to download multiple videos and they don't have the same formats available, you can specify the order of preference using slashes, as in `-f 22/17/18`. You can also filter the video results by putting a condition in brackets, as in `-f "best[height=720]"` (or `-f "[filesize>10M]"`).  This works for filesize, height, width, tbr, abr, vbr, asr, and fps and the comparisons <, <=, >, >=, =, != and for ext, acodec, vcodec, container, and protocol and the comparisons =, !=, ^= (begins with), $= (ends with), *= (contains) . Formats for which the value is not known are excluded unless you put a question mark (?) after the operator. You can combine format filters, so  `-f "[height <=? 720][tbr>500]"` selects up to 720p videos (or videos where the height is not known) with a bitrate of at least 500 KBit/s. Use commas to download multiple formats, such as `-f 136/137/mp4/bestvideo,140/m4a/bestaudio`. You can merge the video and audio of two formats into a single file using `-f <video-format>+<audio-format>` (requires ffmpeg or avconv), for example `-f bestvideo+bestaudio`. Format selectors can also be grouped using parentheses, for example if you want to download the best mp4 and webm formats with a height lower than 480 you can use `-f '(mp4,webm)[height<480]'`.
+But sometimes you may want to download in a different format, for example when you are on a slow or intermittent connection. The key mechanism for achieving this is so called *format selection* based on which you can explicitly specify desired format, select formats based on some criterion or criteria, setup precedence and much more.
 
-Since the end of April 2015 and version 2015.04.26 youtube-dl uses `-f bestvideo+bestaudio/best` as default format selection (see #5447, #5456). If ffmpeg or avconv are installed this results in downloading `bestvideo` and `bestaudio` separately and muxing them together into a single file giving the best overall quality available. Otherwise it falls back to `best` and results in downloading the best available quality served as a single file. `best` is also needed for videos that don't come from YouTube because they don't provide the audio and video in two different files. If you want to only download some dash formats (for example if you are not interested in getting videos with a resolution higher than 1080p), you can add `-f bestvideo[height<=?1080]+bestaudio/best` to your configuration file. Note that if you use youtube-dl to stream to `stdout` (and most likely to pipe it to your media player then), i.e. you explicitly specify output template as `-o -`, youtube-dl still uses `-f best` format selection in order to start content delivery immediately to your player and not to wait until `bestvideo` and `bestaudio` are downloaded and muxed.
+The general syntax for format selection is `--format FORMAT` or shorter `-f FORMAT` where `FORMAT` is a *selector expression*, i.e. an expression that describes format or formats you would like to download.
+
+The simplest case is requesting a specific format, for example with `-f 22` you can download the format with format code equal to 22. You can get the list of available format codes for particular video using `--list-formats` or `-F`. Note that these format codes are extractor specific. 
+
+You can also use a file extension (currently `3gp`, `aac`, `flv`, `m4a`, `mp3`, `mp4`, `ogg`, `wav`, `webm` are supported) to download best quality format of particular file extension served as a single file, e.g. `-f webm` will download best quality format with `webm` extension served as a single file.
+
+You can also use special names to select particular edge case format:
+ - `best`: Select best quality format represented by single file with video and audio
+ - `worst`: Select worst quality format represented by single file with video and audio
+ - `bestvideo`: Select best quality video only format (e.g. DASH video), may not be available
+ - `worstvideo`: Select worst quality video only format, may not be available
+ - `bestaudio`: Select best quality audio only format, may not be available
+ - `worstaudio`: Select worst quality audio only format, may not be available
+
+For example, to download worst quality video only format you can use `-f worstvideo`.
+
+If you want to download multiple videos and they don't have the same formats available, you can specify the order of preference using slashes. Note that slash is left-associative, i.e. formats on the left hand side are preferred, for example `-f 22/17/18` will download format 22 if it's available, otherwise it will download format 17 if it's available, otherwise it will download format 18 if it's available, otherwise it will complain that no suitable formats are available for download.
+
+If you want to download several formats of the same video use comma as a separator, e.g. `-f 22,17,18` will download all these three formats, of course if they are available. Or more sophisticated example combined with precedence feature `-f 136/137/mp4/bestvideo,140/m4a/bestaudio`.
+
+You can also filter the video formats by putting a condition in brackets, as in `-f "best[height=720]"` (or `-f "[filesize>10M]"`).
+
+The following numeric meta fields can be used with comparisons `<`, `<=`, `>`, `>=`, `=` (equals), `!=` (not equals):
+ - `filesize`: The number of bytes, if known in advance
+ - `width`: Width of the video, if known
+ - `height`: Height of the video, if known
+ - `tbr`: Average bitrate of audio and video in KBit/s
+ - `abr`: Average audio bitrate in KBit/s
+ - `vbr`: Average video bitrate in KBit/s
+ - `asr`: Audio sampling rate in Hertz
+ - `fps`: Frame rate
+
+Also filtering work for comparisons `=` (equals), `!=` (not equals), `^=` (begins with), `$=` (ends with), `*=` (contains) and following string meta fields:
+ - `ext`: File extension
+ - `acodec`: Name of the audio codec in use
+ - `vcodec`: Name of the video codec in use
+ - `container`: Name of the container format
+ - `protocol`: The protocol that will be used for the actual download, lower-case. `http`, `https`, `rtsp`, `rtmp`, `rtmpe`, `m3u8`, or `m3u8_native`
+
+Note that none of the aforementioned meta fields are guaranteed to be present since this solely depends on the metadata obtained by particular extractor, i.e. the metadata offered by video hoster.
+
+Formats for which the value is not known are excluded unless you put a question mark (`?`) after the operator. You can combine format filters, so `-f "[height <=? 720][tbr>500]"` selects up to 720p videos (or videos where the height is not known) with a bitrate of at least 500 KBit/s.
+
+You can merge the video and audio of two formats into a single file using `-f <video-format>+<audio-format>` (requires ffmpeg or avconv installed), for example `-f bestvideo+bestaudio` will download best video only format, best audio only format and mux them together with ffmpeg/avconv.
+
+Format selectors can also be grouped using parentheses, for example if you want to download the best mp4 and webm formats with a height lower than 480 you can use `-f '(mp4,webm)[height<480]'`.
+
+Since the end of April 2015 and version 2015.04.26 youtube-dl uses `-f bestvideo+bestaudio/best` as default format selection (see #5447, #5456). If ffmpeg or avconv are installed this results in downloading `bestvideo` and `bestaudio` separately and muxing them together into a single file giving the best overall quality available. Otherwise it falls back to `best` and results in downloading the best available quality served as a single file. `best` is also needed for videos that don't come from YouTube because they don't provide the audio and video in two different files. If you want to only download some DASH formats (for example if you are not interested in getting videos with a resolution higher than 1080p), you can add `-f bestvideo[height<=?1080]+bestaudio/best` to your configuration file. Note that if you use youtube-dl to stream to `stdout` (and most likely to pipe it to your media player then), i.e. you explicitly specify output template as `-o -`, youtube-dl still uses `-f best` format selection in order to start content delivery immediately to your player and not to wait until `bestvideo` and `bestaudio` are downloaded and muxed.
 
 If you want to preserve the old format selection behavior (prior to youtube-dl 2015.04.26), i.e. you want to download the best available quality media served as a single file, you should explicitly specify your choice with `-f best`. You may want to add it to the [configuration file](#configuration) in order not to type it every time you run youtube-dl.
 
+Examples (note on Windows you may need to use double quotes instead of single):
+```bash
+# Download best mp4 format available or any other best if no mp4 available
+$ youtube-dl -f 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/best[ext=mp4]/best'
+
+# Download best format available but not better that 480p
+$ youtube-dl -f 'bestvideo[height<=480]+bestaudio/best[height<=480]'
+
+# Download best video only format but no bigger that 50 MB
+$ youtube-dl -f 'best[filesize<50M]'
+```
+
+
 # VIDEO SELECTION
 
 Videos can be filtered by their upload date using the options `--date`, `--datebefore` or `--dateafter`. They accept dates in two formats:

From b555942428726bd619744a208b5e6cfceb99d8d9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 16 Jan 2016 10:10:28 +0600
Subject: [PATCH 3699/6219] [YoutubeDL] Ensure protocol is always present

---
 youtube_dl/YoutubeDL.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 18b1443b8..09d2b18f2 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -46,6 +46,7 @@ from .utils import (
     DateRange,
     DEFAULT_OUTTMPL,
     determine_ext,
+    determine_protocol,
     DownloadError,
     encode_compat_str,
     encodeFilename,
@@ -1309,6 +1310,10 @@ class YoutubeDL(object):
             # Automatically determine file extension if missing
             if 'ext' not in format:
                 format['ext'] = determine_ext(format['url']).lower()
+            # Automatically determine protocol if missing (useful for format
+            # selection purposes)
+            if 'protocol' not in format:
+                format['protocol'] = determine_protocol(format)
             # Add HTTP headers, so that external programs can use them from the
             # json output
             full_format_info = info_dict.copy()

From 6be16ed24bba86c0bb22a3eca6640e727153f1fd Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Sat, 16 Jan 2016 10:15:24 +0600
Subject: [PATCH 3700/6219] [README.md] Add protocol usage example in format
 selection

---
 README.md | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/README.md b/README.md
index efb51027b..9dbeae1bc 100644
--- a/README.md
+++ b/README.md
@@ -529,6 +529,9 @@ $ youtube-dl -f 'bestvideo[height<=480]+bestaudio/best[height<=480]'
 
 # Download best video only format but no bigger that 50 MB
 $ youtube-dl -f 'best[filesize<50M]'
+
+# Download best format available via direct link over HTTP/HTTPS protocol
+$ youtube-dl -f '(bestvideo+bestaudio/best)[protocol^=http]'
 ```
 
 

From 3a0a78731b5223b232beaabc3cdc771b9cebb078 Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Sat, 16 Jan 2016 12:17:07 +0800
Subject: [PATCH 3701/6219] Fixes #8239

---
 youtube_dl/extractor/neteasemusic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/neteasemusic.py b/youtube_dl/extractor/neteasemusic.py
index 15eca825a..848ebdea0 100644
--- a/youtube_dl/extractor/neteasemusic.py
+++ b/youtube_dl/extractor/neteasemusic.py
@@ -40,7 +40,7 @@ class NetEaseMusicBaseIE(InfoExtractor):
             if not details:
                 continue
             formats.append({
-                'url': 'http://m5.music.126.net/%s/%s.%s' %
+                'url': 'http://203.130.59.9/m1.music.126.net/%s/%s.%s' %
                        (cls._encrypt(details['dfsId']), details['dfsId'],
                         details['extension']),
                 'ext': details.get('extension'),

From 0b26ba3fc8eee0bb047c5e78e6f1a8ba59fa9457 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 16 Jan 2016 20:45:36 +0800
Subject: [PATCH 3702/6219] [extractor/common] Allow passing more parameters to
 _search_json_ld

---
 youtube_dl/extractor/common.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 92e2e4f43..8da70ae14 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -763,13 +763,13 @@ class InfoExtractor(object):
         return self._html_search_meta('twitter:player', html,
                                       'twitter card player')
 
-    def _search_json_ld(self, html, video_id, fatal=True):
+    def _search_json_ld(self, html, video_id, **kwargs):
         json_ld = self._search_regex(
             r'(?s)<script[^>]+type=(["\'])application/ld\+json\1[^>]*>(?P<json_ld>.+?)</script>',
-            html, 'JSON-LD', fatal=fatal, group='json_ld')
+            html, 'JSON-LD', group='json_ld', **kwargs)
         if not json_ld:
             return {}
-        return self._json_ld(json_ld, video_id, fatal=fatal)
+        return self._json_ld(json_ld, video_id, fatal=kwargs.get('fatal', True))
 
     def _json_ld(self, json_ld, video_id, fatal=True):
         if isinstance(json_ld, compat_str):

From 350e02d40d93c341d73b23d630f458b14638fcb6 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 16 Jan 2016 20:46:28 +0800
Subject: [PATCH 3703/6219] [bbc] Use _search_json_ld

---
 youtube_dl/extractor/bbc.py | 17 ++++-------------
 1 file changed, 4 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index ce99a34ab..1c493b72d 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -718,19 +718,10 @@ class BBCIE(BBCCoUkIE):
 
         webpage = self._download_webpage(url, playlist_id)
 
-        timestamp = None
-        playlist_title = None
-        playlist_description = None
-
-        ld = self._parse_json(
-            self._search_regex(
-                r'(?s)<script type="application/ld\+json">(.+?)</script>',
-                webpage, 'ld json', default='{}'),
-            playlist_id, fatal=False)
-        if ld:
-            timestamp = parse_iso8601(ld.get('datePublished'))
-            playlist_title = ld.get('headline')
-            playlist_description = ld.get('articleBody')
+        json_ld_info = self._search_json_ld(webpage, playlist_id, default=None)
+        timestamp = json_ld_info.get('timestamp')
+        playlist_title = json_ld_info.get('title')
+        playlist_description = json_ld_info.get('description')
 
         if not timestamp:
             timestamp = parse_iso8601(self._search_regex(

From 1358b94163899d38cfe1106eb4d734996f42328e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 16 Jan 2016 20:56:53 +0800
Subject: [PATCH 3704/6219] [ae] Fix _TESTS

---
 youtube_dl/extractor/ae.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ae.py b/youtube_dl/extractor/ae.py
index 3bc7c12fc..01c189d78 100644
--- a/youtube_dl/extractor/ae.py
+++ b/youtube_dl/extractor/ae.py
@@ -20,13 +20,14 @@ class AEIE(InfoExtractor):
             'skip_download': True,
         },
         'add_ie': ['ThePlatform'],
+        'expected_warnings': ['JSON-LD'],
     }, {
         'url': 'http://www.history.com/shows/mountain-men/season-1/episode-1',
         'info_dict': {
             'id': 'eg47EERs_JsZ',
             'ext': 'mp4',
             'title': "Winter Is Coming",
-            'description': 'md5:a40e370925074260b1c8a633c632c63a',
+            'description': 'md5:641f424b7a19d8e24f26dea22cf59d74',
         },
         'params': {
             # m3u8 download

From 614db89ae39a922ddbbfbd34e667933bc05c6b04 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 16 Jan 2016 22:17:31 +0800
Subject: [PATCH 3705/6219] [compat] Clarify the versions requiring
 compat_kwargs

It's supported since 2.7.0 alpha 1 and 2.6.5 rc 1. See
https://hg.python.org/cpython/file/v2.7a1/Misc/NEWS#l337
https://hg.python.org/cpython/file/v2.6.5rc1/Misc/NEWS#l28
---
 youtube_dl/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index a3e85264a..8ab688001 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -433,7 +433,7 @@ if sys.version_info < (3, 0) and sys.platform == 'win32':
 else:
     compat_getpass = getpass.getpass
 
-# Old 2.6 and 2.7 releases require kwargs to be bytes
+# Python < 2.6.5 require kwargs to be bytes
 try:
     def _testfunc(x):
         pass

From 855f90fa6f348d8128d9c581db4756e26db02944 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 17 Jan 2016 03:02:45 +0600
Subject: [PATCH 3706/6219] [ae] Rename to aenetworks and clarify extractor
 name and description

---
 youtube_dl/extractor/__init__.py              | 2 +-
 youtube_dl/extractor/{ae.py => aenetworks.py} | 4 +++-
 2 files changed, 4 insertions(+), 2 deletions(-)
 rename youtube_dl/extractor/{ae.py => aenetworks.py} (94%)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 2c56797a5..0ba0d226f 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -15,7 +15,7 @@ from .adobetv import (
     AdobeTVVideoIE,
 )
 from .adultswim import AdultSwimIE
-from .ae import AEIE
+from .aenetworks import AENetworksIE
 from .aftonbladet import AftonbladetIE
 from .airmozilla import AirMozillaIE
 from .aljazeera import AlJazeeraIE
diff --git a/youtube_dl/extractor/ae.py b/youtube_dl/extractor/aenetworks.py
similarity index 94%
rename from youtube_dl/extractor/ae.py
rename to youtube_dl/extractor/aenetworks.py
index 01c189d78..43d7b0523 100644
--- a/youtube_dl/extractor/ae.py
+++ b/youtube_dl/extractor/aenetworks.py
@@ -4,7 +4,9 @@ from .common import InfoExtractor
 from ..utils import smuggle_url
 
 
-class AEIE(InfoExtractor):
+class AENetworksIE(InfoExtractor):
+    IE_NAME = 'aenetworks'
+    IE_DESC = 'A+E Networks: A&E, Lifetime, History.com, FYI Network'
     _VALID_URL = r'https?://(?:www\.)?(?:(?:history|aetv|mylifetime)\.com|fyi\.tv)/(?:[^/]+/)+(?P<id>[^/]+?)(?:$|[?#])'
 
     _TESTS = [{

From e0ef13ddebfab39d8c59c8cd9c11882cfc604070 Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Sun, 17 Jan 2016 07:48:46 +0800
Subject: [PATCH 3707/6219] [neteasemusic] Fallback to alt hosts if
 m5.music.126.net doesn't work

---
 youtube_dl/extractor/neteasemusic.py | 35 +++++++++++++++++-----------
 1 file changed, 22 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/neteasemusic.py b/youtube_dl/extractor/neteasemusic.py
index 848ebdea0..435cd8e2e 100644
--- a/youtube_dl/extractor/neteasemusic.py
+++ b/youtube_dl/extractor/neteasemusic.py
@@ -32,23 +32,32 @@ class NetEaseMusicBaseIE(InfoExtractor):
         result = b64encode(m.digest()).decode('ascii')
         return result.replace('/', '_').replace('+', '-')
 
-    @classmethod
-    def extract_formats(cls, info):
+    def extract_formats(self, info):
         formats = []
-        for song_format in cls._FORMATS:
+        for song_format in self._FORMATS:
             details = info.get(song_format)
             if not details:
                 continue
-            formats.append({
-                'url': 'http://203.130.59.9/m1.music.126.net/%s/%s.%s' %
-                       (cls._encrypt(details['dfsId']), details['dfsId'],
-                        details['extension']),
-                'ext': details.get('extension'),
-                'abr': details.get('bitrate', 0) / 1000,
-                'format_id': song_format,
-                'filesize': details.get('size'),
-                'asr': details.get('sr')
-            })
+            song_file_path = '/%s/%s.%s' % (
+                self._encrypt(details['dfsId']), details['dfsId'], details['extension'])
+
+            # 203.130.59.9, 124.40.233.182, 115.231.74.139, etc is a reverse proxy-like feature
+            # from NetEase's CDN provider that can be used if m5.music.126.net does not
+            # work, especially for users outside of Mainland China
+            # via: https://github.com/JixunMoe/unblock-163/issues/3#issuecomment-163115880
+            for host in ('http://m5.music.126.net', 'http://115.231.74.139/m1.music.126.net',
+                         'http://124.40.233.182/m1.music.126.net', 'http://203.130.59.9/m1.music.126.net'):
+                song_url = host + song_file_path
+                if self._is_valid_url(song_url, info['id'], 'song'):
+                    formats.append({
+                        'url': song_url,
+                        'ext': details.get('extension'),
+                        'abr': details.get('bitrate', 0) / 1000,
+                        'format_id': song_format,
+                        'filesize': details.get('size'),
+                        'asr': details.get('sr')
+                    })
+                    break
         return formats
 
     @classmethod

From e5a66240c0ddb326b3f885cc29b6f00058481739 Mon Sep 17 00:00:00 2001
From: Jakub Wilk <jwilk@jwilk.net>
Date: Sun, 17 Jan 2016 15:06:00 +0100
Subject: [PATCH 3708/6219] [ustream] Use HTTPS for GitHub URL

---
 youtube_dl/extractor/ustream.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ustream.py b/youtube_dl/extractor/ustream.py
index 73b05ecab..b5fe753d7 100644
--- a/youtube_dl/extractor/ustream.py
+++ b/youtube_dl/extractor/ustream.py
@@ -47,7 +47,7 @@ class UstreamIE(InfoExtractor):
         m = re.match(self._VALID_URL, url)
         video_id = m.group('id')
 
-        # some sites use this embed format (see: http://github.com/rg3/youtube-dl/issues/2990)
+        # some sites use this embed format (see: https://github.com/rg3/youtube-dl/issues/2990)
         if m.group('type') == 'embed/recorded':
             video_id = m.group('id')
             desktop_url = 'http://www.ustream.tv/recorded/' + video_id

From dae503afaa8dbbf745a3c796dab1506cd736de83 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 17 Jan 2016 22:14:07 +0600
Subject: [PATCH 3709/6219] [atresplayer] Skip HLS completely (Closes #8261)

---
 youtube_dl/extractor/atresplayer.py | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/youtube_dl/extractor/atresplayer.py b/youtube_dl/extractor/atresplayer.py
index 3fb042cea..b8f9ae005 100644
--- a/youtube_dl/extractor/atresplayer.py
+++ b/youtube_dl/extractor/atresplayer.py
@@ -132,11 +132,6 @@ class AtresPlayerIE(InfoExtractor):
                 })
             formats.append(format_info)
 
-        m3u8_url = player.get('urlVideoHls')
-        if m3u8_url:
-            formats.extend(self._extract_m3u8_formats(
-                m3u8_url, episode_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
-
         timestamp = int_or_none(self._download_webpage(
             self._TIME_API_URL,
             video_id, 'Downloading timestamp', fatal=False), 1000, time.time())

From 096b5339823fc65beeca6cff96867ed03e6234db Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 18 Jan 2016 23:17:45 +0600
Subject: [PATCH 3710/6219] [youtube] Fix URL expansion in video description

Fixes test_Youtube_18
---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index e4f227f19..50c1363cd 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1237,7 +1237,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     (?:[a-zA-Z-]+="[^"]+"\s+)*?
                     (?:title|href)="([^"]+)"\s+
                     (?:[a-zA-Z-]+="[^"]+"\s+)*?
-                    class="(?:yt-uix-redirect-link|yt-uix-sessionlink[^"]*)".*?>
+                    class="(?:yt-uix-redirect-link|yt-uix-sessionlink[^"]*)"[^>]*>
                 [^<]+\.{3}\s*
                 </a>
             ''', r'\1', video_description)

From a8776b107b0137fc13d8fd4350035d02e7bb7761 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 18 Jan 2016 23:19:38 +0600
Subject: [PATCH 3711/6219] [youtube] Clarify test_Youtube_18

---
 youtube_dl/extractor/youtube.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 50c1363cd..2045f5386 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -706,6 +706,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         },
         {
             # Title with JS-like syntax "};" (see https://github.com/rg3/youtube-dl/issues/7468)
+            # Also tests cut-off URL expansion in video description (see
+            # https://github.com/rg3/youtube-dl/issues/1892,
+            # https://github.com/rg3/youtube-dl/issues/8164)
             'url': 'https://www.youtube.com/watch?v=lsguqyKfVQg',
             'info_dict': {
                 'id': 'lsguqyKfVQg',

From 8ceabd4df332d9c39f71a62bbd60cbea9fcb6621 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 19 Jan 2016 20:54:43 +0600
Subject: [PATCH 3712/6219] [youtube] Capture and output unavailable message

---
 youtube_dl/extractor/youtube.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 2045f5386..2a0c2e5a6 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1508,6 +1508,11 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             for a_format in formats:
                 a_format.setdefault('http_headers', {})['Youtubedl-no-compression'] = 'True'
         else:
+            unavailable_message = self._html_search_regex(
+                r'(?s)<h1[^>]+id="unavailable-message"[^>]*>(.+?)</h1>',
+                video_webpage, 'unavailable message', default=None)
+            if unavailable_message:
+                raise ExtractorError(unavailable_message, expected=True)
             raise ExtractorError('no conn, hlsvp or url_encoded_fmt_stream_map information found in video info')
 
         # Look for the DASH manifest

From e323cf3ff37b1268f585b66159de20bd8ffe8dc0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 19 Jan 2016 20:56:04 +0600
Subject: [PATCH 3713/6219] [youtube] Skip test

---
 youtube_dl/extractor/youtube.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 2a0c2e5a6..d31161d21 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -613,7 +613,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             },
             'params': {
                 'skip_download': 'requires avconv',
-            }
+            },
+            'skip': 'This live event has ended.',
         },
         # Extraction from multiple DASH manifests (https://github.com/rg3/youtube-dl/pull/6097)
         {

From 9fbd0822aa9bccb319446e6d75a76ab4f23ddf46 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 19 Jan 2016 21:20:14 +0600
Subject: [PATCH 3714/6219] [dailymotion] Extend _VALID_URL

---
 youtube_dl/extractor/dailymotion.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index b687ec4d6..6e462af69 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -37,7 +37,7 @@ class DailymotionBaseInfoExtractor(InfoExtractor):
 
 
 class DailymotionIE(DailymotionBaseInfoExtractor):
-    _VALID_URL = r'(?i)(?:https?://)?(?:(www|touch)\.)?dailymotion\.[a-z]{2,3}/(?:(embed|#)/)?video/(?P<id>[^/?_]+)'
+    _VALID_URL = r'(?i)(?:https?://)?(?:(www|touch)\.)?dailymotion\.[a-z]{2,3}/(?:(?:embed|swf|#)/)?video/(?P<id>[^/?_]+)'
     IE_NAME = 'dailymotion'
 
     _FORMATS = [
@@ -104,6 +104,10 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
         {
             'url': 'http://www.dailymotion.com/video/x20su5f_the-power-of-nightmares-1-the-rise-of-the-politics-of-fear-bbc-2004_news',
             'only_matching': True,
+        },
+        {
+            'url': 'http://www.dailymotion.com/swf/video/x3n92nf',
+            'only_matching': True,
         }
     ]
 
@@ -336,7 +340,7 @@ class DailymotionPlaylistIE(DailymotionBaseInfoExtractor):
 
 class DailymotionUserIE(DailymotionPlaylistIE):
     IE_NAME = 'dailymotion:user'
-    _VALID_URL = r'https?://(?:www\.)?dailymotion\.[a-z]{2,3}/(?!(?:embed|#|video|playlist)/)(?:(?:old/)?user/)?(?P<user>[^/]+)'
+    _VALID_URL = r'https?://(?:www\.)?dailymotion\.[a-z]{2,3}/(?!(?:embed|swf|#|video|playlist)/)(?:(?:old/)?user/)?(?P<user>[^/]+)'
     _PAGE_TEMPLATE = 'http://www.dailymotion.com/user/%s/%s'
     _TESTS = [{
         'url': 'https://www.dailymotion.com/user/nqtv',

From 1fa2b9841ddafd41c3aebe40976445b3b7087f9e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 19 Jan 2016 21:20:45 +0600
Subject: [PATCH 3715/6219] [extractor/generic] Extend dailymotion embed regex

---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index b3f8efc80..0baa17e8d 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1402,7 +1402,7 @@ class GenericIE(InfoExtractor):
 
         # Look for embedded Dailymotion player
         matches = re.findall(
-            r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?dailymotion\.com/embed/video/.+?)\1', webpage)
+            r'<(?:embed|iframe)[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?dailymotion\.com/(?:embed|swf)/video/.+?)\1', webpage)
         if matches:
             return _playlist_from_matches(
                 matches, lambda m: unescapeHTML(m[1]))

From 942d46196ffc18bc7444bbc2c822824efcd12bdc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 19 Jan 2016 21:47:06 +0600
Subject: [PATCH 3716/6219] [ultimedia] Extend _VALID_URL to support digiteka

---
 youtube_dl/extractor/ultimedia.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ultimedia.py b/youtube_dl/extractor/ultimedia.py
index 60328123c..eef8705e0 100644
--- a/youtube_dl/extractor/ultimedia.py
+++ b/youtube_dl/extractor/ultimedia.py
@@ -9,7 +9,7 @@ from ..utils import int_or_none
 
 class UltimediaIE(InfoExtractor):
     _VALID_URL = r'''(?x)
-        https?://(?:www\.)?ultimedia\.com/
+        https?://(?:www\.)?(?:digiteka\.net|ultimedia\.com)/
         (?:
             deliver/
             (?P<embed_type>
@@ -56,6 +56,9 @@ class UltimediaIE(InfoExtractor):
             'timestamp': 1424760500,
             'uploader_id': '3rfzk',
         },
+    }, {
+        'url': 'https://www.digiteka.net/deliver/generic/iframe/mdtk/01637594/src/lqm3kl/zone/1/showtitle/1/autoplay/yes',
+        'only_matching': True,
     }]
 
     @staticmethod

From aecfcd4e595121d17b173f1ee9f4444ee9c0fda2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 19 Jan 2016 21:51:46 +0600
Subject: [PATCH 3717/6219] [ultimedia] Rename to digiteka

---
 youtube_dl/extractor/__init__.py                   |  2 +-
 youtube_dl/extractor/{ultimedia.py => digiteka.py} |  2 +-
 youtube_dl/extractor/generic.py                    | 10 +++++-----
 3 files changed, 7 insertions(+), 7 deletions(-)
 rename youtube_dl/extractor/{ultimedia.py => digiteka.py} (99%)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 0ba0d226f..1d838cdd0 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -777,7 +777,7 @@ from .udemy import (
     UdemyCourseIE
 )
 from .udn import UDNEmbedIE
-from .ultimedia import UltimediaIE
+from .digiteka import DigitekaIE
 from .unistra import UnistraIE
 from .urort import UrortIE
 from .ustream import UstreamIE, UstreamChannelIE
diff --git a/youtube_dl/extractor/ultimedia.py b/youtube_dl/extractor/digiteka.py
similarity index 99%
rename from youtube_dl/extractor/ultimedia.py
rename to youtube_dl/extractor/digiteka.py
index eef8705e0..7bb79ffda 100644
--- a/youtube_dl/extractor/ultimedia.py
+++ b/youtube_dl/extractor/digiteka.py
@@ -7,7 +7,7 @@ from .common import InfoExtractor
 from ..utils import int_or_none
 
 
-class UltimediaIE(InfoExtractor):
+class DigitekaIE(InfoExtractor):
     _VALID_URL = r'''(?x)
         https?://(?:www\.)?(?:digiteka\.net|ultimedia\.com)/
         (?:
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 0baa17e8d..26d3698c8 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -57,7 +57,7 @@ from .pladform import PladformIE
 from .videomore import VideomoreIE
 from .googledrive import GoogleDriveIE
 from .jwplatform import JWPlatformIE
-from .ultimedia import UltimediaIE
+from .digiteka import DigitekaIE
 
 
 class GenericIE(InfoExtractor):
@@ -1814,10 +1814,10 @@ class GenericIE(InfoExtractor):
         if mobj is not None:
             return self.url_result(unescapeHTML(mobj.group('url')), 'ScreenwaveMedia')
 
-        # Look for Ulltimedia embeds
-        ultimedia_url = UltimediaIE._extract_url(webpage)
-        if ultimedia_url:
-            return self.url_result(self._proto_relative_url(ultimedia_url), 'Ultimedia')
+        # Look for Digiteka embeds
+        digiteka_url = DigitekaIE._extract_url(webpage)
+        if digiteka_url:
+            return self.url_result(self._proto_relative_url(digiteka_url), DigitekaIE.ie_key())
 
         # Look for AdobeTVVideo embeds
         mobj = re.search(

From b605ebb609dc208a814ab44fff84463bf0c5369a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 19 Jan 2016 22:09:55 +0600
Subject: [PATCH 3718/6219] [lemonde] Add extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/lemonde.py  | 34 ++++++++++++++++++++++++++++++++
 2 files changed, 35 insertions(+)
 create mode 100644 youtube_dl/extractor/lemonde.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 1d838cdd0..0cc339a37 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -333,6 +333,7 @@ from .kuwo import (
 from .la7 import LA7IE
 from .laola1tv import Laola1TvIE
 from .lecture2go import Lecture2GoIE
+from .lemonde import LemondeIE
 from .letv import (
     LetvIE,
     LetvTvIE,
diff --git a/youtube_dl/extractor/lemonde.py b/youtube_dl/extractor/lemonde.py
new file mode 100644
index 000000000..be66fff03
--- /dev/null
+++ b/youtube_dl/extractor/lemonde.py
@@ -0,0 +1,34 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class LemondeIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:.+?\.)?lemonde\.fr/(?:[^/]+/)*(?P<id>[^/]+)\.html'
+    _TESTS = [{
+        'url': 'http://www.lemonde.fr/police-justice/video/2016/01/19/comprendre-l-affaire-bygmalion-en-cinq-minutes_4849702_1653578.html',
+        'md5': '01fb3c92de4c12c573343d63e163d302',
+        'info_dict': {
+            'id': 'lqm3kl',
+            'ext': 'mp4',
+            'title': "Comprendre l'affaire Bygmalion en 5 minutes",
+            'thumbnail': 're:^https?://.*\.jpg',
+            'duration': 320,
+            'upload_date': '20160119',
+            'timestamp': 1453194778,
+            'uploader_id': '3pmkp',
+        },
+    }, {
+        'url': 'http://redaction.actu.lemonde.fr/societe/video/2016/01/18/calais-debut-des-travaux-de-defrichement-dans-la-jungle_4849233_3224.html',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, display_id)
+
+        digiteka_url = self._proto_relative_url(self._search_regex(
+            r'url\s*:\s*(["\'])(?P<url>(?:https?://)?//(?:www\.)?(?:digiteka\.net|ultimedia\.com)/deliver/.+?)\1',
+            webpage, 'digiteka url', group='url'))
+        return self.url_result(digiteka_url, 'Digiteka')

From 485139c15c91af03bdb89c7ac07563d9fb979363 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 20 Jan 2016 00:02:46 +0600
Subject: [PATCH 3719/6219] [viewster] Tolerate missing synopsis (Closes #8274)

---
 youtube_dl/extractor/viewster.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/viewster.py b/youtube_dl/extractor/viewster.py
index 185b1c119..fe94a4793 100644
--- a/youtube_dl/extractor/viewster.py
+++ b/youtube_dl/extractor/viewster.py
@@ -155,10 +155,10 @@ class ViewsterIE(InfoExtractor):
 
         self._sort_formats(formats)
 
-        synopsis = info.get('Synopsis', {})
+        synopsis = info.get('Synopsis') or {}
         # Prefer title outside synopsis since it's less messy
         title = (info.get('Title') or synopsis['Title']).strip()
-        description = synopsis.get('Detailed') or info.get('Synopsis', {}).get('Short')
+        description = synopsis.get('Detailed') or (info.get('Synopsis') or {}).get('Short')
         duration = int_or_none(info.get('Duration'))
         timestamp = parse_iso8601(info.get('ReleaseDate'))
 

From 10defdd06a1ea878087f5bad6ee99da3b2f9d3f6 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 20 Jan 2016 03:17:35 +0800
Subject: [PATCH 3720/6219] [letv] Reduce duplicated codes

---
 youtube_dl/extractor/letv.py | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/letv.py b/youtube_dl/extractor/letv.py
index c096cb1ab..d43b4e808 100644
--- a/youtube_dl/extractor/letv.py
+++ b/youtube_dl/extractor/letv.py
@@ -281,14 +281,17 @@ class LetvCloudIE(InfoExtractor):
             "uu=" + uu + "&vu=" + vu)
         play_json = self._download_json(play_json_req, media_id, 'Downloading playJson data')
 
-        formats = [{
-            'url': base64.b64decode(media['play_url']['main_url'].encode('utf-8')).decode("utf-8"),
-            'ext': 'mp4',
-            'format_id': int_or_none(media.get('play_url', {}).get('vtype')),
-            'format_note': str_or_none(media.get('play_url', {}).get('definition')),
-            'width': int_or_none(media.get('play_url', {}).get('vwidth')),
-            'height': int_or_none(media.get('play_url', {}).get('vheight')),
-        } for media in play_json['data']['video_info']['media'].values()]
+        formats = []
+        for media in play_json['data']['video_info']['media'].values():
+            play_url = media['play_url']
+            formats.append({
+                'url': base64.b64decode(play_url['main_url'].encode('utf-8')).decode("utf-8"),
+                'ext': 'mp4',
+                'format_id': int_or_none(play_url.get('vtype')),
+                'format_note': str_or_none(play_url.get('definition')),
+                'width': int_or_none(play_url.get('vwidth')),
+                'height': int_or_none(play_url.get('vheight')),
+            })
         self._sort_formats(formats)
 
         return {

From 8fff4f61e5741e56890ccb108f42b0c7dc6607e8 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 20 Jan 2016 03:18:54 +0800
Subject: [PATCH 3721/6219] [letv] Use single quotes

---
 youtube_dl/extractor/letv.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/letv.py b/youtube_dl/extractor/letv.py
index d43b4e808..be6c75967 100644
--- a/youtube_dl/extractor/letv.py
+++ b/youtube_dl/extractor/letv.py
@@ -278,14 +278,14 @@ class LetvCloudIE(InfoExtractor):
 
         play_json_req = sanitized_Request(
             'http://api.letvcloud.com/gpc.php?cf=html5&sign=signxxxxx&ver=2.2&format=json&' +
-            "uu=" + uu + "&vu=" + vu)
+            'uu=' + uu + '&vu=' + vu)
         play_json = self._download_json(play_json_req, media_id, 'Downloading playJson data')
 
         formats = []
         for media in play_json['data']['video_info']['media'].values():
             play_url = media['play_url']
             formats.append({
-                'url': base64.b64decode(play_url['main_url'].encode('utf-8')).decode("utf-8"),
+                'url': base64.b64decode(play_url['main_url'].encode('utf-8')).decode('utf-8'),
                 'ext': 'mp4',
                 'format_id': int_or_none(play_url.get('vtype')),
                 'format_note': str_or_none(play_url.get('definition')),

From e0690782b8531f9962950693fe33a5d4a4f494f6 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 20 Jan 2016 03:25:12 +0800
Subject: [PATCH 3722/6219] [letv] LetvCloud: guard against invalid URLs

---
 youtube_dl/extractor/letv.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/letv.py b/youtube_dl/extractor/letv.py
index be6c75967..b8d4f5bb8 100644
--- a/youtube_dl/extractor/letv.py
+++ b/youtube_dl/extractor/letv.py
@@ -272,8 +272,14 @@ class LetvCloudIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        uu = re.search('uu=([\w]+)', url).group(1)
-        vu = re.search('vu=([\w]+)', url).group(1)
+        uu_mobj = re.search('uu=([\w]+)', url)
+        vu_mobj = re.search('vu=([\w]+)', url)
+
+        if not uu_mobj or not vu_mobj:
+            raise ExtractorError('Invalid URL: %s' % url, expected=True)
+
+        uu = uu_mobj.group(1)
+        vu = vu_mobj.group(1)
         media_id = uu + '_' + vu
 
         play_json_req = sanitized_Request(

From 26de1bba83b3269653b571682cf962899b6231b7 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 20 Jan 2016 03:31:34 +0800
Subject: [PATCH 3723/6219] [letv] LetvCloud: check error messages from server

---
 youtube_dl/extractor/letv.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/youtube_dl/extractor/letv.py b/youtube_dl/extractor/letv.py
index b8d4f5bb8..177d53bbb 100644
--- a/youtube_dl/extractor/letv.py
+++ b/youtube_dl/extractor/letv.py
@@ -287,6 +287,14 @@ class LetvCloudIE(InfoExtractor):
             'uu=' + uu + '&vu=' + vu)
         play_json = self._download_json(play_json_req, media_id, 'Downloading playJson data')
 
+        if not play_json.get('data'):
+            if play_json.get('message'):
+                raise ExtractorError('Letv cloud said: %s' % play_json['message'], expected=True)
+            elif play_json.get('code'):
+                raise ExtractorError('Letv cloud returned error %d' % play_json['code'], expected=True)
+            else:
+                raise ExtractorError('Letv cloud returned an unknwon error')
+
         formats = []
         for media in play_json['data']['video_info']['media'].values():
             play_url = media['play_url']

From 73e74424561b3e277c2f071fd88b17c845a1162f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 20 Jan 2016 03:39:45 +0800
Subject: [PATCH 3724/6219] [letv] LetvCloud: simplify and improve _VALID_URL

---
 youtube_dl/extractor/letv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/letv.py b/youtube_dl/extractor/letv.py
index 177d53bbb..ff74b5141 100644
--- a/youtube_dl/extractor/letv.py
+++ b/youtube_dl/extractor/letv.py
@@ -245,7 +245,7 @@ class LetvPlaylistIE(LetvTvIE):
 
 class LetvCloudIE(InfoExtractor):
     IE_DESC = '乐视云'
-    _VALID_URL = r'http://yuntv\.letv\.com/bcloud.html\?.*$'
+    _VALID_URL = r'https?://yuntv\.letv\.com/bcloud.html\?.+'
 
     _TESTS = [{
         'url': 'http://yuntv.letv.com/bcloud.html?uu=p7jnfw5hw9&vu=467623dedf',

From 0428106da33419302ac8ad31306f9357549f086e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 20 Jan 2016 03:53:17 +0800
Subject: [PATCH 3725/6219] [letv] LetvCloud: make title looks like a title

---
 youtube_dl/extractor/letv.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/letv.py b/youtube_dl/extractor/letv.py
index ff74b5141..b15ac163e 100644
--- a/youtube_dl/extractor/letv.py
+++ b/youtube_dl/extractor/letv.py
@@ -253,21 +253,21 @@ class LetvCloudIE(InfoExtractor):
         'info_dict': {
             'id': 'p7jnfw5hw9_467623dedf',
             'ext': 'mp4',
-            'title': 'p7jnfw5hw9_467623dedf',
+            'title': 'Video p7jnfw5hw9_467623dedf',
         },
     }, {
         'url': 'http://yuntv.letv.com/bcloud.html?uu=p7jnfw5hw9&vu=ec93197892&pu=2c7cd40209&auto_play=1&gpcflag=1&width=640&height=360',
         'info_dict': {
             'id': 'p7jnfw5hw9_ec93197892',
             'ext': 'mp4',
-            'title': 'p7jnfw5hw9_ec93197892',
+            'title': 'Video p7jnfw5hw9_ec93197892',
         },
     }, {
         'url': 'http://yuntv.letv.com/bcloud.html?uu=p7jnfw5hw9&vu=187060b6fd',
         'info_dict': {
             'id': 'p7jnfw5hw9_187060b6fd',
             'ext': 'mp4',
-            'title': 'p7jnfw5hw9_187060b6fd',
+            'title': 'Video p7jnfw5hw9_187060b6fd',
         },
     }]
 
@@ -310,6 +310,6 @@ class LetvCloudIE(InfoExtractor):
 
         return {
             'id': media_id,
-            'title': media_id,
+            'title': 'Video %s' % media_id,
             'formats': formats,
         }

From bec30224ff86d11c3f61808dbe683f97940ba23e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 20 Jan 2016 04:00:37 +0800
Subject: [PATCH 3726/6219] [letv] LetvCloud: Detect ext instead of the
 hardcoded one

---
 youtube_dl/extractor/letv.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/letv.py b/youtube_dl/extractor/letv.py
index b15ac163e..08bdae8a2 100644
--- a/youtube_dl/extractor/letv.py
+++ b/youtube_dl/extractor/letv.py
@@ -19,6 +19,7 @@ from ..utils import (
     int_or_none,
     str_or_none,
     encode_data_uri,
+    url_basename,
 )
 
 
@@ -295,12 +296,17 @@ class LetvCloudIE(InfoExtractor):
             else:
                 raise ExtractorError('Letv cloud returned an unknwon error')
 
+        def b64decode(s):
+            return base64.b64decode(s.encode('utf-8')).decode('utf-8')
+
         formats = []
         for media in play_json['data']['video_info']['media'].values():
             play_url = media['play_url']
+            url = b64decode(play_url['main_url'])
+            decoded_url = b64decode(url_basename(url))
             formats.append({
-                'url': base64.b64decode(play_url['main_url'].encode('utf-8')).decode('utf-8'),
-                'ext': 'mp4',
+                'url': url,
+                'ext': determine_ext(decoded_url),
                 'format_id': int_or_none(play_url.get('vtype')),
                 'format_note': str_or_none(play_url.get('definition')),
                 'width': int_or_none(play_url.get('vwidth')),

From 65ced034b851f890f894ce79d8069de852208dfc Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 20 Jan 2016 04:02:30 +0800
Subject: [PATCH 3727/6219] [weiqitv] Make codes shorter

---
 youtube_dl/extractor/weiqitv.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/weiqitv.py b/youtube_dl/extractor/weiqitv.py
index da3b3d145..b72df1e8d 100644
--- a/youtube_dl/extractor/weiqitv.py
+++ b/youtube_dl/extractor/weiqitv.py
@@ -37,13 +37,11 @@ class WeiqitvIE(InfoExtractor):
         page = self._download_webpage(url, media_id)
 
         info_json_str = self._search_regex(
-            'var\s+video\s*=\s*(.+});',
-            page, 'info_json_str')
+            'var\s+video\s*=\s*(.+});', page, 'info json str')
         info_json = self._parse_json(info_json_str, media_id)
 
         letvcloud_url = self._search_regex(
-            'var\s+letvurl\s*=\s*"([^"]+)',
-            page, 'letvcloud_url')
+            'var\s+letvurl\s*=\s*"([^"]+)', page, 'letvcloud url')
 
         return {
             '_type': 'url_transparent',

From f15a9ca301799c44300d16ac7e61e5b36d67fd31 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 20 Jan 2016 04:03:57 +0800
Subject: [PATCH 3728/6219] [weiqitv] Rename the extractor - capitilize 'TV'

---
 youtube_dl/extractor/__init__.py | 2 +-
 youtube_dl/extractor/weiqitv.py  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index aeb67354e..5ec017d5d 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -857,7 +857,7 @@ from .webofstories import (
     WebOfStoriesPlaylistIE,
 )
 from .weibo import WeiboIE
-from .weiqitv import WeiqitvIE
+from .weiqitv import WeiqiTVIE
 from .wimp import WimpIE
 from .wistia import WistiaIE
 from .worldstarhiphop import WorldStarHipHopIE
diff --git a/youtube_dl/extractor/weiqitv.py b/youtube_dl/extractor/weiqitv.py
index b72df1e8d..24389d4bb 100644
--- a/youtube_dl/extractor/weiqitv.py
+++ b/youtube_dl/extractor/weiqitv.py
@@ -4,7 +4,7 @@ from __future__ import unicode_literals
 from .common import InfoExtractor
 
 
-class WeiqitvIE(InfoExtractor):
+class WeiqiTVIE(InfoExtractor):
     IE_DESC = 'WQTV'
     _VALID_URL = r'http://www\.weiqitv\.com/index/video_play\?videoId=(?P<id>[A-Za-z0-9]+)'
 

From 8652bd22f1ee955940793874b5c4060ef8e273b3 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 20 Jan 2016 04:04:39 +0800
Subject: [PATCH 3729/6219] [weiqitv] Use single quotes

---
 youtube_dl/extractor/weiqitv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/weiqitv.py b/youtube_dl/extractor/weiqitv.py
index 24389d4bb..e333ae345 100644
--- a/youtube_dl/extractor/weiqitv.py
+++ b/youtube_dl/extractor/weiqitv.py
@@ -45,7 +45,7 @@ class WeiqiTVIE(InfoExtractor):
 
         return {
             '_type': 'url_transparent',
-            "ie_key": 'LetvCloud',
+            'ie_key': 'LetvCloud',
             'url': letvcloud_url,
             'title': info_json['name'],
             'id': media_id,

From fe6856b0595b85c8070d7ccc154cd8ac2ff85e2c Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 20 Jan 2016 04:21:51 +0800
Subject: [PATCH 3730/6219] [neteasemusic] Use float_or_none

---
 youtube_dl/extractor/neteasemusic.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/neteasemusic.py b/youtube_dl/extractor/neteasemusic.py
index 435cd8e2e..7830616f8 100644
--- a/youtube_dl/extractor/neteasemusic.py
+++ b/youtube_dl/extractor/neteasemusic.py
@@ -12,7 +12,10 @@ from ..compat import (
     compat_str,
     compat_itertools_count,
 )
-from ..utils import sanitized_Request
+from ..utils import (
+    sanitized_Request,
+    float_or_none,
+)
 
 
 class NetEaseMusicBaseIE(InfoExtractor):
@@ -52,7 +55,7 @@ class NetEaseMusicBaseIE(InfoExtractor):
                     formats.append({
                         'url': song_url,
                         'ext': details.get('extension'),
-                        'abr': details.get('bitrate', 0) / 1000,
+                        'abr': float_or_none(details.get('bitrate'), scale=1000),
                         'format_id': song_format,
                         'filesize': details.get('size'),
                         'asr': details.get('sr')

From 9d90e7de03c20b8365ffbcbb17cad58cbfbe35a5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 16 Jan 2016 21:51:25 +0100
Subject: [PATCH 3731/6219] [downloader/hls] Ask ffmpeg to quit when
 interrupting youtube-dl with 'Ctrl+C' (#8252)

Otherwise the mp4 file can't be played.
---
 youtube_dl/downloader/hls.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index d186d402d..10b83c6b2 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -46,7 +46,16 @@ class HlsFD(FileDownloader):
 
         self._debug_cmd(args)
 
-        retval = subprocess.call(args, stdin=subprocess.PIPE)
+        proc = subprocess.Popen(args, stdin=subprocess.PIPE)
+        try:
+            retval = proc.wait()
+        except KeyboardInterrupt:
+            # subprocces.run would send the SIGKILL signal to ffmpeg and the
+            # mp4 file couldn't be played, but if we ask ffmpeg to quit it
+            # produces a file that is playable (this is mostly useful for live
+            # streams)
+            proc.communicate(b'q')
+            raise
         if retval == 0:
             fsize = os.path.getsize(encodeFilename(tmpfilename))
             self.to_screen('\r[%s] %s bytes' % (args[0], fsize))

From 56aa074538afc4941b437e2d91a3e298d5f20eb9 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 20 Jan 2016 13:20:03 +0800
Subject: [PATCH 3732/6219] Credit @FounderSG for WeiqiTV and LetvCloud (#7994)

[ci skip]
---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index f4238e1d3..73fcafeb5 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -153,3 +153,4 @@ flatgreen
 Brian Foley
 Vignesh Venkat
 Tom Gijselinck
+Founder Fang

From 27b83249c90fa0500bff68b5aee7874100bba5e5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 20 Jan 2016 20:00:51 +0600
Subject: [PATCH 3733/6219] [tube8] Fix extraction and extract all formats
 (Closes #8281)

---
 youtube_dl/extractor/tube8.py | 34 +++++++++++++++++++++++-----------
 1 file changed, 23 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/tube8.py b/youtube_dl/extractor/tube8.py
index 46ef61ff5..6d7ca4874 100644
--- a/youtube_dl/extractor/tube8.py
+++ b/youtube_dl/extractor/tube8.py
@@ -1,10 +1,9 @@
 from __future__ import unicode_literals
 
-import json
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse_urlparse
+from ..compat import compat_str
 from ..utils import (
     int_or_none,
     sanitized_Request,
@@ -44,14 +43,28 @@ class Tube8IE(InfoExtractor):
         req.add_header('Cookie', 'age_verified=1')
         webpage = self._download_webpage(req, display_id)
 
-        flashvars = json.loads(self._html_search_regex(
-            r'flashvars\s*=\s*({.+?});\r?\n', webpage, 'flashvars'))
+        flashvars = self._parse_json(
+            self._search_regex(
+                r'flashvars\s*=\s*({.+?});\r?\n', webpage, 'flashvars'),
+            video_id)
 
-        video_url = flashvars['video_url']
-        if flashvars.get('encrypted') is True:
-            video_url = aes_decrypt_text(video_url, flashvars['video_title'], 32).decode('utf-8')
-        path = compat_urllib_parse_urlparse(video_url).path
-        format_id = '-'.join(path.split('/')[4].split('_')[:2])
+        formats = []
+        for key, video_url in flashvars.items():
+            if not isinstance(video_url, compat_str) or not video_url.startswith('http'):
+                continue
+            height = self._search_regex(
+                r'quality_(\d+)[pP]', key, 'height', default=None)
+            if not height:
+                continue
+            if flashvars.get('encrypted') is True:
+                video_url = aes_decrypt_text(
+                    video_url, flashvars['video_title'], 32).decode('utf-8')
+            formats.append({
+                'url': video_url,
+                'format_id': '%sp' % height,
+                'height': int(height),
+            })
+        self._sort_formats(formats)
 
         thumbnail = flashvars.get('image_url')
 
@@ -79,15 +92,14 @@ class Tube8IE(InfoExtractor):
         return {
             'id': video_id,
             'display_id': display_id,
-            'url': video_url,
             'title': title,
             'description': description,
             'thumbnail': thumbnail,
             'uploader': uploader,
-            'format_id': format_id,
             'view_count': view_count,
             'like_count': like_count,
             'dislike_count': dislike_count,
             'comment_count': comment_count,
             'age_limit': 18,
+            'formats': formats,
         }

From 7dde358adcc3d6ef9e7a174cbeadde4d27029669 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 20 Jan 2016 20:07:32 +0600
Subject: [PATCH 3734/6219] [tube8] Extract duration and modernize

---
 youtube_dl/extractor/tube8.py | 56 +++++++++++++++++------------------
 1 file changed, 27 insertions(+), 29 deletions(-)

diff --git a/youtube_dl/extractor/tube8.py b/youtube_dl/extractor/tube8.py
index 6d7ca4874..378cb0570 100644
--- a/youtube_dl/extractor/tube8.py
+++ b/youtube_dl/extractor/tube8.py
@@ -14,25 +14,23 @@ from ..aes import aes_decrypt_text
 
 class Tube8IE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?tube8\.com/(?:[^/]+/)+(?P<display_id>[^/]+)/(?P<id>\d+)'
-    _TESTS = [
-        {
-            'url': 'http://www.tube8.com/teen/kasia-music-video/229795/',
-            'md5': '44bf12b98313827dd52d35b8706a4ea0',
-            'info_dict': {
-                'id': '229795',
-                'display_id': 'kasia-music-video',
-                'ext': 'mp4',
-                'description': 'hot teen Kasia grinding',
-                'uploader': 'unknown',
-                'title': 'Kasia music video',
-                'age_limit': 18,
-            }
-        },
-        {
-            'url': 'http://www.tube8.com/shemale/teen/blonde-cd-gets-kidnapped-by-two-blacks-and-punished-for-being-a-slutty-girl/19569151/',
-            'only_matching': True,
-        },
-    ]
+    _TESTS = [{
+        'url': 'http://www.tube8.com/teen/kasia-music-video/229795/',
+        'md5': '65e20c48e6abff62ed0c3965fff13a39',
+        'info_dict': {
+            'id': '229795',
+            'display_id': 'kasia-music-video',
+            'ext': 'mp4',
+            'description': 'hot teen Kasia grinding',
+            'uploader': 'unknown',
+            'title': 'Kasia music video',
+            'age_limit': 18,
+            'duration': 230,
+        }
+    },{
+        'url': 'http://www.tube8.com/shemale/teen/blonde-cd-gets-kidnapped-by-two-blacks-and-punished-for-being-a-slutty-girl/19569151/',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
@@ -75,19 +73,18 @@ class Tube8IE(InfoExtractor):
         uploader = self._html_search_regex(
             r'<span class="username">\s*(.+?)\s*<',
             webpage, 'uploader', fatal=False)
+        duration = int_or_none(flashvars.get('video_duration'))
 
-        like_count = int_or_none(self._html_search_regex(
+        like_count = int_or_none(self._search_regex(
             r'rupVar\s*=\s*"(\d+)"', webpage, 'like count', fatal=False))
-        dislike_count = int_or_none(self._html_search_regex(
+        dislike_count = int_or_none(self._search_regex(
             r'rdownVar\s*=\s*"(\d+)"', webpage, 'dislike count', fatal=False))
-        view_count = self._html_search_regex(
-            r'<strong>Views: </strong>([\d,\.]+)\s*</li>', webpage, 'view count', fatal=False)
-        if view_count:
-            view_count = str_to_int(view_count)
-        comment_count = self._html_search_regex(
-            r'<span id="allCommentsCount">(\d+)</span>', webpage, 'comment count', fatal=False)
-        if comment_count:
-            comment_count = str_to_int(comment_count)
+        view_count = str_to_int(self._search_regex(
+            r'<strong>Views: </strong>([\d,\.]+)\s*</li>',
+            webpage, 'view count', fatal=False))
+        comment_count = str_to_int(self._search_regex(
+            r'<span id="allCommentsCount">(\d+)</span>',
+            webpage, 'comment count', fatal=False))
 
         return {
             'id': video_id,
@@ -96,6 +93,7 @@ class Tube8IE(InfoExtractor):
             'description': description,
             'thumbnail': thumbnail,
             'uploader': uploader,
+            'duration': duration,
             'view_count': view_count,
             'like_count': like_count,
             'dislike_count': dislike_count,

From 1f78ed189aab9b7ea17594ac1f33662d37433455 Mon Sep 17 00:00:00 2001
From: CeruleanSky <CeruleanSky@users.noreply.github.com>
Date: Mon, 18 Jan 2016 17:10:16 -0500
Subject: [PATCH 3735/6219] [OraTV] update extractor

"current" is now "video"
"hls_stream" is now hls_stream without quotes
video_id is now id
duration for current video is not present(for other videos it is)

modified regex to find hls_stream variable to work reguardless of whether it is quoted or not.

[ora] Improve (Closes #8273)
---
 youtube_dl/extractor/ora.py | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/ora.py b/youtube_dl/extractor/ora.py
index 02de1502a..8545fb1b8 100644
--- a/youtube_dl/extractor/ora.py
+++ b/youtube_dl/extractor/ora.py
@@ -21,7 +21,6 @@ class OraTVIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Vine & YouTube Stars Zach King & King Bach On Their Viral Videos!',
             'description': 'md5:ebbc5b1424dd5dba7be7538148287ac1',
-            'duration': 1477,
         }
     }
 
@@ -30,9 +29,9 @@ class OraTVIE(InfoExtractor):
         webpage = self._download_webpage(url, display_id)
 
         video_data = self._search_regex(
-            r'"current"\s*:\s*({[^}]+?})', webpage, 'current video')
+            r'"(?:video|current)"\s*:\s*({[^}]+?})', webpage, 'current video')
         m3u8_url = self._search_regex(
-            r'"hls_stream"\s*:\s*"([^"]+)', video_data, 'm3u8 url', None)
+            r'hls_stream"?\s*:\s*"([^"]+)', video_data, 'm3u8 url', None)
         if m3u8_url:
             formats = self._extract_m3u8_formats(
                 m3u8_url, display_id, 'mp4', 'm3u8_native',
@@ -62,14 +61,12 @@ class OraTVIE(InfoExtractor):
 
         return {
             'id': self._search_regex(
-                r'"video_id"\s*:\s*(\d+)', video_data, 'video id'),
+                r'"id"\s*:\s*(\d+)', video_data, 'video id', default=display_id),
             'display_id': display_id,
             'title': unescapeHTML(self._og_search_title(webpage)),
             'description': get_element_by_attribute(
                 'class', 'video_txt_decription', webpage),
             'thumbnail': self._proto_relative_url(self._search_regex(
                 r'"thumb"\s*:\s*"([^"]+)', video_data, 'thumbnail', None)),
-            'duration': int(self._search_regex(
-                r'"duration"\s*:\s*(\d+)', video_data, 'duration')),
             'formats': formats,
         }

From a616f65471743abfacccb2639a299ea356839909 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 20 Jan 2016 21:30:29 +0600
Subject: [PATCH 3736/6219] [tube8] PEP 8

---
 youtube_dl/extractor/tube8.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tube8.py b/youtube_dl/extractor/tube8.py
index 378cb0570..1d9271d1e 100644
--- a/youtube_dl/extractor/tube8.py
+++ b/youtube_dl/extractor/tube8.py
@@ -27,7 +27,7 @@ class Tube8IE(InfoExtractor):
             'age_limit': 18,
             'duration': 230,
         }
-    },{
+    }, {
         'url': 'http://www.tube8.com/shemale/teen/blonde-cd-gets-kidnapped-by-two-blacks-and-punished-for-being-a-slutty-girl/19569151/',
         'only_matching': True,
     }]

From 4519c1f43cd2d0634e85041aa8f950ddb745e285 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 21 Jan 2016 12:43:45 +0100
Subject: [PATCH 3737/6219] [vimeo] 'ext' must be a string, not a tuple (fixes
 #8288)

There was an ',' at the end of the line.
---
 youtube_dl/extractor/vimeo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 76e681bc3..2389e7f0f 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -430,7 +430,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
                 if download_url and not source_file.get('is_cold') and not source_file.get('is_defrosting'):
                     source_name = source_file.get('public_name', 'Original')
                     if self._is_valid_url(download_url, video_id, '%s video' % source_name):
-                        ext = source_file.get('extension', determine_ext(download_url)).lower(),
+                        ext = source_file.get('extension', determine_ext(download_url)).lower()
                         formats.append({
                             'url': download_url,
                             'ext': ext,

From 383496e65ed2a965ab82e5cc3012fd3e0ca2afbf Mon Sep 17 00:00:00 2001
From: igv <igv.versn@gmail.com>
Date: Thu, 21 Jan 2016 12:36:09 +0300
Subject: [PATCH 3738/6219] Additional regex for yahoo extractor

---
 youtube_dl/extractor/yahoo.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/yahoo.py b/youtube_dl/extractor/yahoo.py
index 4a492f784..92b50f6f7 100644
--- a/youtube_dl/extractor/yahoo.py
+++ b/youtube_dl/extractor/yahoo.py
@@ -221,6 +221,7 @@ class YahooIE(InfoExtractor):
                     r'root\.App\.Cache\.context\.videoCache\.curVideo = \{"([^"]+)"',
                     r'"first_videoid"\s*:\s*"([^"]+)"',
                     r'%s[^}]*"ccm_id"\s*:\s*"([^"]+)"' % re.escape(page_id),
+                    r'yahoo:\/\/article\/view\?uuid=([^&]+)&',
                 ]
                 video_id = self._search_regex(
                     CONTENT_ID_REGEXES, webpage, 'content ID')

From e87d98b0ddbb2cfa873b4d0c13c751bb12224290 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 21 Jan 2016 22:42:50 +0600
Subject: [PATCH 3739/6219] [yahoo] Add improve content id regexes (Closes
 #8290)

---
 youtube_dl/extractor/yahoo.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/yahoo.py b/youtube_dl/extractor/yahoo.py
index 92b50f6f7..4c6142927 100644
--- a/youtube_dl/extractor/yahoo.py
+++ b/youtube_dl/extractor/yahoo.py
@@ -221,7 +221,8 @@ class YahooIE(InfoExtractor):
                     r'root\.App\.Cache\.context\.videoCache\.curVideo = \{"([^"]+)"',
                     r'"first_videoid"\s*:\s*"([^"]+)"',
                     r'%s[^}]*"ccm_id"\s*:\s*"([^"]+)"' % re.escape(page_id),
-                    r'yahoo:\/\/article\/view\?uuid=([^&]+)&',
+                    r'<article[^>]data-uuid=["\']([^"\']+)',
+                    r'yahoo://article/view\?.*\buuid=([^&"\']+)',
                 ]
                 video_id = self._search_regex(
                     CONTENT_ID_REGEXES, webpage, 'content ID')

From 582f4f834e5cb6fc783eeb51bd977aa237330e79 Mon Sep 17 00:00:00 2001
From: Dimitre Liotev <dl@liotev.com>
Date: Thu, 21 Jan 2016 08:10:41 +0200
Subject: [PATCH 3740/6219] Fix issue #8109 (error when downloading automatic
 captions)

---
 youtube_dl/extractor/youtube.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index d31161d21..567877920 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -964,6 +964,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         try:
             args = player_config['args']
             caption_url = args['ttsurl']
+            if caption_url is None or caption_url == "" or caption_url.isspace():
+                self._downloader.report_warning("No automatic captions")
+                return {}
             timestamp = args['timestamp']
             # We get the available subtitles
             list_params = compat_urllib_parse.urlencode({

From 51290d8457aa8460382407796740063ced464481 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 21 Jan 2016 22:58:03 +0600
Subject: [PATCH 3741/6219] [youtube] Simplify automatic captions URL check
 (Closes #8287)

---
 youtube_dl/extractor/youtube.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 567877920..8e8fc14d2 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -964,8 +964,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         try:
             args = player_config['args']
             caption_url = args['ttsurl']
-            if caption_url is None or caption_url == "" or caption_url.isspace():
-                self._downloader.report_warning("No automatic captions")
+            if not caption_url:
+                self._downloader.report_warning(err_msg)
                 return {}
             timestamp = args['timestamp']
             # We get the available subtitles

From 1e10d02fec559618f877e6694b6eeec53d59a65a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 21 Jan 2016 23:28:22 +0600
Subject: [PATCH 3742/6219] [hitbox] Skip subscribe only formats (Closes #8217)

---
 youtube_dl/extractor/hitbox.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/hitbox.py b/youtube_dl/extractor/hitbox.py
index 421f55bbe..ff797438d 100644
--- a/youtube_dl/extractor/hitbox.py
+++ b/youtube_dl/extractor/hitbox.py
@@ -159,6 +159,9 @@ class HitboxLiveIE(HitboxIE):
         cdns = player_config.get('cdns')
         servers = []
         for cdn in cdns:
+            # Subscribe URLs are not playable
+            if cdn.get('rtmpSubscribe') is True:
+                continue
             base_url = cdn.get('netConnectionUrl')
             host = re.search('.+\.([^\.]+\.[^\./]+)/.+', base_url).group(1)
             if base_url not in servers:

From 6b45f9aba2dad6e965ab51b4d18f4bb05336eaf1 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 22 Jan 2016 02:14:47 +0800
Subject: [PATCH 3743/6219] [iqiyi] Update key (closes #8292)

---
 youtube_dl/extractor/iqiyi.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index 66a70a181..691cb66d6 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -214,8 +214,8 @@ class IqiyiIE(InfoExtractor):
 
     def get_enc_key(self, swf_url, video_id):
         # TODO: automatic key extraction
-        # last update at 2015-12-18 for Zombie::bite
-        enc_key = '8b6b683780897eb8d9a48a02ccc4817d'[::-1]
+        # last update at 2016-01-22 for Zombie::bite
+        enc_key = '6ab6d0280511493ba85594779759d4ed'
         return enc_key
 
     def _real_extract(self, url):

From 4d318be1951d6bbae0eae7aff69a58de353c8337 Mon Sep 17 00:00:00 2001
From: Filippo Valsorda <filippo@cloudflare.com>
Date: Mon, 14 Dec 2015 02:18:13 +0000
Subject: [PATCH 3744/6219] [update] fix (unexploitable) BB'06 vulnerability in
 rsa_verify

The rsa_verify code was vulnerable to a BB'06 attack, allowing to forge
signatures for arbitrary messages if and only if the public key exponent is
3.  Since the updates key is hardcoded to 65537, there is no risk for
youtube-dl, but I don't want vulnerable code in the wild.

The new function adopts a way safer approach of encoding-and-comparing to
replace the dangerous parsing code.
---
 test/test_update.py  | 30 ++++++++++++++++++++++++++++++
 test/versions.json   | 34 ++++++++++++++++++++++++++++++++++
 youtube_dl/update.py | 32 ++++++++------------------------
 3 files changed, 72 insertions(+), 24 deletions(-)
 create mode 100644 test/test_update.py
 create mode 100644 test/versions.json

diff --git a/test/test_update.py b/test/test_update.py
new file mode 100644
index 000000000..d9c71511d
--- /dev/null
+++ b/test/test_update.py
@@ -0,0 +1,30 @@
+#!/usr/bin/env python
+
+from __future__ import unicode_literals
+
+# Allow direct execution
+import os
+import sys
+import unittest
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+
+import json
+from youtube_dl.update import rsa_verify
+
+
+class TestUpdate(unittest.TestCase):
+    def test_rsa_verify(self):
+        UPDATES_RSA_KEY = (0x9d60ee4d8f805312fdb15a62f87b95bd66177b91df176765d13514a0f1754bcd2057295c5b6f1d35daa6742c3ffc9a82d3e118861c207995a8031e151d863c9927e304576bc80692bc8e094896fcf11b66f3e29e04e3a71e9a11558558acea1840aec37fc396fb6b65dc81a1c4144e03bd1c011de62e3f1357b327d08426fe93, 65537)
+        with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'versions.json'), 'rb') as f:
+            versions_info = f.read().decode()
+        versions_info = json.loads(versions_info)
+        signature = versions_info['signature']
+        del versions_info['signature']
+        self.assertTrue(rsa_verify(
+            json.dumps(versions_info, sort_keys=True).encode('utf-8'),
+            signature, UPDATES_RSA_KEY))
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/test/versions.json b/test/versions.json
new file mode 100644
index 000000000..6cccc2259
--- /dev/null
+++ b/test/versions.json
@@ -0,0 +1,34 @@
+{
+    "latest": "2013.01.06", 
+    "signature": "72158cdba391628569ffdbea259afbcf279bbe3d8aeb7492690735dc1cfa6afa754f55c61196f3871d429599ab22f2667f1fec98865527b32632e7f4b3675a7ef0f0fbe084d359256ae4bba68f0d33854e531a70754712f244be71d4b92e664302aa99653ee4df19800d955b6c4149cd2b3f24288d6e4b40b16126e01f4c8ce6", 
+    "versions": {
+        "2013.01.02": {
+            "bin": [
+                "http://youtube-dl.org/downloads/2013.01.02/youtube-dl", 
+                "f5b502f8aaa77675c4884938b1e4871ebca2611813a0c0e74f60c0fbd6dcca6b"
+            ], 
+            "exe": [
+                "http://youtube-dl.org/downloads/2013.01.02/youtube-dl.exe", 
+                "75fa89d2ce297d102ff27675aa9d92545bbc91013f52ec52868c069f4f9f0422"
+            ], 
+            "tar": [
+                "http://youtube-dl.org/downloads/2013.01.02/youtube-dl-2013.01.02.tar.gz", 
+                "6a66d022ac8e1c13da284036288a133ec8dba003b7bd3a5179d0c0daca8c8196"
+            ]
+        }, 
+        "2013.01.06": {
+            "bin": [
+                "http://youtube-dl.org/downloads/2013.01.06/youtube-dl", 
+                "64b6ed8865735c6302e836d4d832577321b4519aa02640dc508580c1ee824049"
+            ], 
+            "exe": [
+                "http://youtube-dl.org/downloads/2013.01.06/youtube-dl.exe", 
+                "58609baf91e4389d36e3ba586e21dab882daaaee537e4448b1265392ae86ff84"
+            ], 
+            "tar": [
+                "http://youtube-dl.org/downloads/2013.01.06/youtube-dl-2013.01.06.tar.gz", 
+                "fe77ab20a95d980ed17a659aa67e371fdd4d656d19c4c7950e7b720b0c2f1a86"
+            ]
+        }
+    }
+}
\ No newline at end of file
diff --git a/youtube_dl/update.py b/youtube_dl/update.py
index 995b8ed96..e4a1aaa64 100644
--- a/youtube_dl/update.py
+++ b/youtube_dl/update.py
@@ -15,33 +15,17 @@ from .version import __version__
 
 
 def rsa_verify(message, signature, key):
-    from struct import pack
     from hashlib import sha256
-
     assert isinstance(message, bytes)
-    block_size = 0
-    n = key[0]
-    while n:
-        block_size += 1
-        n >>= 8
-    signature = pow(int(signature, 16), key[1], key[0])
-    raw_bytes = []
-    while signature:
-        raw_bytes.insert(0, pack("B", signature & 0xFF))
-        signature >>= 8
-    signature = (block_size - len(raw_bytes)) * b'\x00' + b''.join(raw_bytes)
-    if signature[0:2] != b'\x00\x01':
+    byte_size = (len(bin(key[0])) - 2 + 8 - 1) // 8
+    signature = ('%x' % pow(int(signature, 16), key[1], key[0])).encode()
+    signature = (byte_size * 2 - len(signature)) * b'0' + signature
+    asn1 = b'3031300d060960864801650304020105000420'
+    asn1 += sha256(message).hexdigest().encode()
+    if byte_size < len(asn1) // 2 + 11:
         return False
-    signature = signature[2:]
-    if b'\x00' not in signature:
-        return False
-    signature = signature[signature.index(b'\x00') + 1:]
-    if not signature.startswith(b'\x30\x31\x30\x0D\x06\x09\x60\x86\x48\x01\x65\x03\x04\x02\x01\x05\x00\x04\x20'):
-        return False
-    signature = signature[19:]
-    if signature != sha256(message).digest():
-        return False
-    return True
+    expected = b'0001' + (byte_size - len(asn1) // 2 - 3) * b'ff' + b'00' + asn1
+    return expected == signature
 
 
 def update_self(to_screen, verbose, opener):

From 32d77eeb04657529b118159db4d23fd671fe04c7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 22 Jan 2016 14:49:17 +0100
Subject: [PATCH 3745/6219] [downloader/common] report_retry: Don't crash when
 retries is infinite (fixes #8299)

---
 youtube_dl/downloader/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index beae8c4d0..fc7521598 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -295,7 +295,7 @@ class FileDownloader(object):
 
     def report_retry(self, count, retries):
         """Report retry in case of HTTP error 5xx"""
-        self.to_screen('[download] Got server HTTP error. Retrying (attempt %d of %d)...' % (count, retries))
+        self.to_screen('[download] Got server HTTP error. Retrying (attempt %d of %.0f)...' % (count, retries))
 
     def report_file_already_downloaded(self, file_name):
         """Report file has already been fully downloaded."""

From 4118cc02c1694a7fd355b75c806a69d75b3850bb Mon Sep 17 00:00:00 2001
From: John Assael <iassael@gmail.com>
Date: Thu, 21 Jan 2016 19:51:48 +0000
Subject: [PATCH 3746/6219] [cbsnews] Extract subtitles

added test function for CBS News subtitles
---
 youtube_dl/extractor/cbsnews.py | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/youtube_dl/extractor/cbsnews.py b/youtube_dl/extractor/cbsnews.py
index d211ec23b..7b9dd67fd 100644
--- a/youtube_dl/extractor/cbsnews.py
+++ b/youtube_dl/extractor/cbsnews.py
@@ -41,6 +41,21 @@ class CBSNewsIE(InfoExtractor):
                 'skip_download': True,
             },
         },
+        {
+            'url': 'http://www.cbsnews.com/videos/mountain-lions-of-l-a/',
+            'info_dict': {
+                'id': 'Mountain Lions of L.A.',
+                'ext': 'flv',
+                'title': 'Fort Hood shooting: Army downplays mental illness as cause of attack',
+                'thumbnail': 're:^http?://.*\.jpg$',
+                'subtitles': 're:^http?://.*\.xml$',
+                'duration': 787,
+            },
+            'params': {
+                # rtmp download
+                'skip_download': True,
+            },
+        },
     ]
 
     def _real_extract(self, url):
@@ -85,10 +100,21 @@ class CBSNewsIE(InfoExtractor):
                 fmt['ext'] = 'mp4'
             formats.append(fmt)
 
+        if 'mpxRefId' in video_info:
+            cap_url = 'http://www.cbsnews.com/videos/captions/%s.adb_xml' % video_info['mpxRefId']
+            subtitles = {
+                'en': [{
+                    'url': cap_url,
+                    'ext': 'xml'
+                }], }
+        else:
+            subtitles = {}
+
         return {
             'id': video_id,
             'title': title,
             'thumbnail': thumbnail,
             'duration': duration,
             'formats': formats,
+            'subtitles': subtitles,
         }

From 220ee33f2b8a22d2fe992c26b32a046b77ce2a4a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 22 Jan 2016 22:23:21 +0600
Subject: [PATCH 3747/6219] [cbsnews] Simplify subtitles extraction and fix
 test (Closes #8295)

---
 youtube_dl/extractor/cbsnews.py | 33 ++++++++++-----------------------
 1 file changed, 10 insertions(+), 23 deletions(-)

diff --git a/youtube_dl/extractor/cbsnews.py b/youtube_dl/extractor/cbsnews.py
index 7b9dd67fd..480435e26 100644
--- a/youtube_dl/extractor/cbsnews.py
+++ b/youtube_dl/extractor/cbsnews.py
@@ -35,21 +35,11 @@ class CBSNewsIE(InfoExtractor):
                 'title': 'Fort Hood shooting: Army downplays mental illness as cause of attack',
                 'thumbnail': 're:^https?://.*\.jpg$',
                 'duration': 205,
-            },
-            'params': {
-                # rtmp download
-                'skip_download': True,
-            },
-        },
-        {
-            'url': 'http://www.cbsnews.com/videos/mountain-lions-of-l-a/',
-            'info_dict': {
-                'id': 'Mountain Lions of L.A.',
-                'ext': 'flv',
-                'title': 'Fort Hood shooting: Army downplays mental illness as cause of attack',
-                'thumbnail': 're:^http?://.*\.jpg$',
-                'subtitles': 're:^http?://.*\.xml$',
-                'duration': 787,
+                'subtitles': {
+                    'en': [{
+                        'ext': 'ttml',
+                    }],
+                },
             },
             'params': {
                 # rtmp download
@@ -100,15 +90,12 @@ class CBSNewsIE(InfoExtractor):
                 fmt['ext'] = 'mp4'
             formats.append(fmt)
 
+        subtitles = {}
         if 'mpxRefId' in video_info:
-            cap_url = 'http://www.cbsnews.com/videos/captions/%s.adb_xml' % video_info['mpxRefId']
-            subtitles = {
-                'en': [{
-                    'url': cap_url,
-                    'ext': 'xml'
-                }], }
-        else:
-            subtitles = {}
+            subtitles['en'] = [{
+                'ext': 'ttml',
+                'url': 'http://www.cbsnews.com/videos/captions/%s.adb_xml' % video_info['mpxRefId'],
+            }]
 
         return {
             'id': video_id,

From 24114fee747c5cec4aae41f93581561fcb902e10 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fran=C3=A7ois=20Charlier?= <francois.charlier@redhat.com>
Date: Thu, 21 Jan 2016 18:47:43 +0100
Subject: [PATCH 3748/6219] [arte:future] Fix extraction

[arte] Add support for more "Arte Future" uri
---
 youtube_dl/extractor/arte.py | 35 ++++++++++++++++++-----------------
 1 file changed, 18 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index 10301a8ea..964d38fdf 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -199,25 +199,26 @@ class ArteTVCreativeIE(ArteTVPlus7IE):
 
 class ArteTVFutureIE(ArteTVPlus7IE):
     IE_NAME = 'arte.tv:future'
-    _VALID_URL = r'https?://future\.arte\.tv/(?P<lang>fr|de)/(thema|sujet)/.*?#article-anchor-(?P<id>\d+)'
+    _VALID_URL = r'https?://future\.arte\.tv/(?P<lang>fr|de)/(?P<id>.+)'
 
-    _TEST = {
-        'url': 'http://future.arte.tv/fr/sujet/info-sciences#article-anchor-7081',
-        'info_dict': {
-            'id': '5201',
-            'ext': 'mp4',
-            'title': 'Les champignons au secours de la planète',
-            'upload_date': '20131101',
+    _TESTS = [
+        {
+            'url': 'http://future.arte.tv/fr/info-sciences/les-ecrevisses-aussi-sont-anxieuses',
+            'info_dict': {
+                'id': '050940-028-A',
+                'ext': 'mp4',
+                'title': 'Les écrevisses aussi peuvent être anxieuses',
+            },
         },
-    }
-
-    def _real_extract(self, url):
-        anchor_id, lang = self._extract_url_info(url)
-        webpage = self._download_webpage(url, anchor_id)
-        row = self._search_regex(
-            r'(?s)id="%s"[^>]*>.+?(<div[^>]*arte_vp_url[^>]*>)' % anchor_id,
-            webpage, 'row')
-        return self._extract_from_webpage(row, anchor_id, lang)
+        {
+            'url': 'http://future.arte.tv/fr/la-science-est-elle-responsable',
+            'info_dict': {
+                'id': '061982-002-A',
+                'ext': 'mp4',
+                'title': 'Brian P. Schmidt - Prix Nobel de physique 2011',
+            },
+        }
+    ]
 
 
 class ArteTVDDCIE(ArteTVPlus7IE):

From 9c54ae3387b883bec5f014e2ac864a9a6c109163 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 22 Jan 2016 23:00:05 +0600
Subject: [PATCH 3749/6219] [arte:future] Make duplicated test matching only

---
 youtube_dl/extractor/arte.py | 27 ++++++++++-----------------
 1 file changed, 10 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index 964d38fdf..7ef42e092 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -201,24 +201,17 @@ class ArteTVFutureIE(ArteTVPlus7IE):
     IE_NAME = 'arte.tv:future'
     _VALID_URL = r'https?://future\.arte\.tv/(?P<lang>fr|de)/(?P<id>.+)'
 
-    _TESTS = [
-        {
-            'url': 'http://future.arte.tv/fr/info-sciences/les-ecrevisses-aussi-sont-anxieuses',
-            'info_dict': {
-                'id': '050940-028-A',
-                'ext': 'mp4',
-                'title': 'Les écrevisses aussi peuvent être anxieuses',
-            },
+    _TESTS = [{
+        'url': 'http://future.arte.tv/fr/info-sciences/les-ecrevisses-aussi-sont-anxieuses',
+        'info_dict': {
+            'id': '050940-028-A',
+            'ext': 'mp4',
+            'title': 'Les écrevisses aussi peuvent être anxieuses',
         },
-        {
-            'url': 'http://future.arte.tv/fr/la-science-est-elle-responsable',
-            'info_dict': {
-                'id': '061982-002-A',
-                'ext': 'mp4',
-                'title': 'Brian P. Schmidt - Prix Nobel de physique 2011',
-            },
-        }
-    ]
+    }, {
+        'url': 'http://future.arte.tv/fr/la-science-est-elle-responsable',
+        'only_matching': True,
+    }]
 
 
 class ArteTVDDCIE(ArteTVPlus7IE):

From 4fcd9d147df9b06d954b8f8a1749b50609529ed4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 22 Jan 2016 23:00:50 +0600
Subject: [PATCH 3750/6219] [arte:cinema] Add extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/arte.py     | 17 +++++++++++++++++
 2 files changed, 18 insertions(+)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index bab3d7b46..cee5cfe7c 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -42,6 +42,7 @@ from .arte import (
     ArteTVCreativeIE,
     ArteTVConcertIE,
     ArteTVFutureIE,
+    ArteTVCinemaIE,
     ArteTVDDCIE,
     ArteTVEmbedIE,
 )
diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index 7ef42e092..b9e07f0ef 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -249,6 +249,23 @@ class ArteTVConcertIE(ArteTVPlus7IE):
     }
 
 
+class ArteTVCinemaIE(ArteTVPlus7IE):
+    IE_NAME = 'arte.tv:cinema'
+    _VALID_URL = r'https?://cinema\.arte\.tv/(?P<lang>de|fr)/(?P<id>.+)'
+
+    _TEST = {
+        'url': 'http://cinema.arte.tv/de/node/38291',
+        'md5': '6b275511a5107c60bacbeeda368c3aa1',
+        'info_dict': {
+            'id': '055876-000_PWA12025-D',
+            'ext': 'mp4',
+            'title': 'Tod auf dem Nil',
+            'upload_date': '20160122',
+            'description': 'md5:7f749bbb77d800ef2be11d54529b96bc',
+        },
+    }
+
+
 class ArteTVEmbedIE(ArteTVPlus7IE):
     IE_NAME = 'arte.tv:embed'
     _VALID_URL = r'''(?x)

From d570746e45cff3c0f89654bf748e44a5da75a924 Mon Sep 17 00:00:00 2001
From: "Andrew \"Akari\" Alexeyew" <akari@dbc.1gb.ua>
Date: Wed, 2 Dec 2015 06:00:47 +0200
Subject: [PATCH 3751/6219] [nuevo] Generalize nuevo extractor and add support
 for trollvids

Supports only the nuevo player for now (most common).

[trollvids] convert duration to an int

[trollvids] added a test

[trollvids] made flake8 shut up

Generalized the Nuevo extractor

Affects: anitube, trollvids, trutube

[nuevo] Complied with the code comments.
---
 youtube_dl/extractor/__init__.py  |  1 +
 youtube_dl/extractor/anitube.py   | 34 +++------------------
 youtube_dl/extractor/nuevo.py     | 37 +++++++++++++++++++++++
 youtube_dl/extractor/trollvids.py | 49 +++++++++++++++++++++++++++++++
 youtube_dl/extractor/trutube.py   | 23 +++++----------
 5 files changed, 98 insertions(+), 46 deletions(-)
 create mode 100644 youtube_dl/extractor/nuevo.py
 create mode 100644 youtube_dl/extractor/trollvids.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index cee5cfe7c..6f2b35cf1 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -726,6 +726,7 @@ from .toutv import TouTvIE
 from .toypics import ToypicsUserIE, ToypicsIE
 from .traileraddict import TrailerAddictIE
 from .trilulilu import TriluliluIE
+from .trollvids import TrollvidsIE
 from .trutube import TruTubeIE
 from .tube8 import Tube8IE
 from .tubitv import TubiTvIE
diff --git a/youtube_dl/extractor/anitube.py b/youtube_dl/extractor/anitube.py
index 23f942ae2..73690df82 100644
--- a/youtube_dl/extractor/anitube.py
+++ b/youtube_dl/extractor/anitube.py
@@ -2,10 +2,10 @@ from __future__ import unicode_literals
 
 import re
 
-from .common import InfoExtractor
+from .nuevo import NuevoBaseIE
 
 
-class AnitubeIE(InfoExtractor):
+class AnitubeIE(NuevoBaseIE):
     IE_NAME = 'anitube.se'
     _VALID_URL = r'https?://(?:www\.)?anitube\.se/video/(?P<id>\d+)'
 
@@ -29,31 +29,5 @@ class AnitubeIE(InfoExtractor):
         key = self._search_regex(
             r'src=["\']https?://[^/]+/embed/([A-Za-z0-9_-]+)', webpage, 'key')
 
-        config_xml = self._download_xml(
-            'http://www.anitube.se/nuevo/econfig.php?key=%s' % key, key)
-
-        video_title = config_xml.find('title').text
-        thumbnail = config_xml.find('image').text
-        duration = float(config_xml.find('duration').text)
-
-        formats = []
-        video_url = config_xml.find('file')
-        if video_url is not None:
-            formats.append({
-                'format_id': 'sd',
-                'url': video_url.text,
-            })
-        video_url = config_xml.find('filehd')
-        if video_url is not None:
-            formats.append({
-                'format_id': 'hd',
-                'url': video_url.text,
-            })
-
-        return {
-            'id': video_id,
-            'title': video_title,
-            'thumbnail': thumbnail,
-            'duration': duration,
-            'formats': formats
-        }
+        config_url = 'http://www.anitube.se/nuevo/econfig.php?key=%s' % key
+        return self._extract_nuevo(config_url, video_id)
diff --git a/youtube_dl/extractor/nuevo.py b/youtube_dl/extractor/nuevo.py
new file mode 100644
index 000000000..ccc697e4f
--- /dev/null
+++ b/youtube_dl/extractor/nuevo.py
@@ -0,0 +1,37 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+from ..utils import (
+    float_or_none,
+    xpath_text
+)
+
+
+class NuevoBaseIE(InfoExtractor):
+    def _extract_nuevo(self, config_url, video_id):
+        tree = self._download_xml(config_url, video_id, transform_source=lambda s: s.strip())
+
+        title = xpath_text(tree, './title')
+        if title:
+            title = title.strip()
+
+        thumbnail = xpath_text(tree, './image')
+        duration = float_or_none(xpath_text(tree, './duration'))
+
+        formats = []
+        for element_name, format_id in (('file', 'sd'), ('filehd', 'hd')):
+            video_url = tree.find(element_name)
+            video_url is None or formats.append({
+                'format_id': format_id,
+                'url': video_url.text
+            })
+
+        return {
+            'id': video_id,
+            'title': title,
+            'thumbnail': thumbnail,
+            'duration': duration,
+            'formats': formats
+        }
diff --git a/youtube_dl/extractor/trollvids.py b/youtube_dl/extractor/trollvids.py
new file mode 100644
index 000000000..e4fe620f7
--- /dev/null
+++ b/youtube_dl/extractor/trollvids.py
@@ -0,0 +1,49 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+from .nuevo import NuevoBaseIE
+
+from ..compat import (
+    compat_urllib_parse_unquote
+)
+
+import re
+
+
+class TrollvidsIE(NuevoBaseIE):
+    _VALID_URL = r'http://(?:www\.)?trollvids\.com/+video/+(?P<id>[0-9]+)/+(?P<title>[^?&]+)'
+    IE_NAME = 'trollvids'
+
+    def _real_extract(self, url):
+        match = re.match(self._VALID_URL, url)
+
+        video_id = match.group('id')
+        raw_video_title = match.group('title')
+        url = 'http://trollvids.com/video/%s/%s' % (video_id, raw_video_title)
+        config_url = 'http://trollvids.com/nuevo/player/config.php?v=%s' % video_id
+
+        info = self._extract_nuevo(config_url, video_id)
+
+        info.update({
+            'webpage_url': url,
+            'age_limit': 18
+        })
+
+        if 'title' not in info:
+            info['title'] = compat_urllib_parse_unquote(raw_video_title)
+
+        return info
+
+    _TESTS = [
+        {
+            'url': 'http://trollvids.com/video/2349002/%E3%80%90MMD-R-18%E3%80%91%E3%82%AC%E3%83%BC%E3%83%AB%E3%83%95%E3%83%AC%E3%83%B3%E3%83%89-carrymeoff',
+            'md5': '1d53866b2c514b23ed69e4352fdc9839',
+            'info_dict': {
+                'id': '2349002',
+                'ext': 'mp4',
+                'title': "【MMD R-18】ガールフレンド carry_me_off",
+                'age_limit': 18,
+                'duration': 216.78,
+            },
+        },
+    ]
diff --git a/youtube_dl/extractor/trutube.py b/youtube_dl/extractor/trutube.py
index e7b79243a..d7ec2ec26 100644
--- a/youtube_dl/extractor/trutube.py
+++ b/youtube_dl/extractor/trutube.py
@@ -1,10 +1,9 @@
 from __future__ import unicode_literals
 
-from .common import InfoExtractor
-from ..utils import xpath_text
+from .nuevo import NuevoBaseIE
 
 
-class TruTubeIE(InfoExtractor):
+class TruTubeIE(NuevoBaseIE):
     _VALID_URL = r'https?://(?:www\.)?trutube\.tv/(?:video/|nuevo/player/embed\.php\?v=)(?P<id>[0-9]+)'
     _TESTS = [{
         'url': 'http://trutube.tv/video/14880/Ramses-II-Proven-To-Be-A-Red-Headed-Caucasoid-',
@@ -22,19 +21,11 @@ class TruTubeIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
+        config_url = 'https://trutube.tv/nuevo/player/config.php?v=%s' % video_id
 
-        config = self._download_xml(
-            'https://trutube.tv/nuevo/player/config.php?v=%s' % video_id,
-            video_id, transform_source=lambda s: s.strip())
+        info = self._extract_nuevo(config_url, video_id)
 
-        # filehd is always 404
-        video_url = xpath_text(config, './file', 'video URL', fatal=True)
-        title = xpath_text(config, './title', 'title').strip()
-        thumbnail = xpath_text(config, './image', ' thumbnail')
+        # filehd always 404s
+        info['formats'] = info['formats'][:1]
 
-        return {
-            'id': video_id,
-            'url': video_url,
-            'title': title,
-            'thumbnail': thumbnail,
-        }
+        return info

From 10677ece81b7ed05bb84a0dbaf5bd237107eeb62 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 Jan 2016 00:04:33 +0600
Subject: [PATCH 3752/6219] [nuevo] Simplify nuevo extractors (Closes #7728)

---
 youtube_dl/extractor/anitube.py   |  9 ++---
 youtube_dl/extractor/nuevo.py     | 25 +++++++-------
 youtube_dl/extractor/trollvids.py | 55 ++++++++++++-------------------
 youtube_dl/extractor/trutube.py   | 13 +++-----
 4 files changed, 41 insertions(+), 61 deletions(-)

diff --git a/youtube_dl/extractor/anitube.py b/youtube_dl/extractor/anitube.py
index 73690df82..2fd912da4 100644
--- a/youtube_dl/extractor/anitube.py
+++ b/youtube_dl/extractor/anitube.py
@@ -1,7 +1,5 @@
 from __future__ import unicode_literals
 
-import re
-
 from .nuevo import NuevoBaseIE
 
 
@@ -22,12 +20,11 @@ class AnitubeIE(NuevoBaseIE):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id)
         key = self._search_regex(
             r'src=["\']https?://[^/]+/embed/([A-Za-z0-9_-]+)', webpage, 'key')
 
-        config_url = 'http://www.anitube.se/nuevo/econfig.php?key=%s' % key
-        return self._extract_nuevo(config_url, video_id)
+        return self._extract_nuevo(
+            'http://www.anitube.se/nuevo/econfig.php?key=%s' % key, video_id)
diff --git a/youtube_dl/extractor/nuevo.py b/youtube_dl/extractor/nuevo.py
index ccc697e4f..225da033c 100644
--- a/youtube_dl/extractor/nuevo.py
+++ b/youtube_dl/extractor/nuevo.py
@@ -11,22 +11,23 @@ from ..utils import (
 
 class NuevoBaseIE(InfoExtractor):
     def _extract_nuevo(self, config_url, video_id):
-        tree = self._download_xml(config_url, video_id, transform_source=lambda s: s.strip())
+        config = self._download_xml(
+            config_url, video_id, transform_source=lambda s: s.strip())
 
-        title = xpath_text(tree, './title')
-        if title:
-            title = title.strip()
-
-        thumbnail = xpath_text(tree, './image')
-        duration = float_or_none(xpath_text(tree, './duration'))
+        title = xpath_text(config, './title', 'title', fatal=True).strip()
+        video_id = xpath_text(config, './mediaid', default=video_id)
+        thumbnail = xpath_text(config, './image')
+        duration = float_or_none(xpath_text(config, './duration'))
 
         formats = []
         for element_name, format_id in (('file', 'sd'), ('filehd', 'hd')):
-            video_url = tree.find(element_name)
-            video_url is None or formats.append({
-                'format_id': format_id,
-                'url': video_url.text
-            })
+            video_url = xpath_text(config, element_name)
+            if video_url:
+                formats.append({
+                    'url': video_url,
+                    'format_id': format_id,
+                })
+        self._check_formats(formats, video_id)
 
         return {
             'id': video_id,
diff --git a/youtube_dl/extractor/trollvids.py b/youtube_dl/extractor/trollvids.py
index e4fe620f7..d239949a6 100644
--- a/youtube_dl/extractor/trollvids.py
+++ b/youtube_dl/extractor/trollvids.py
@@ -1,49 +1,36 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
-from .nuevo import NuevoBaseIE
-
-from ..compat import (
-    compat_urllib_parse_unquote
-)
-
 import re
 
+from .nuevo import NuevoBaseIE
+
 
 class TrollvidsIE(NuevoBaseIE):
-    _VALID_URL = r'http://(?:www\.)?trollvids\.com/+video/+(?P<id>[0-9]+)/+(?P<title>[^?&]+)'
+    _VALID_URL = r'http://(?:www\.)?trollvids\.com/video/(?P<id>\d+)/(?P<display_id>[^/?#&]+)'
     IE_NAME = 'trollvids'
+    _TEST = {
+        'url': 'http://trollvids.com/video/2349002/%E3%80%90MMD-R-18%E3%80%91%E3%82%AC%E3%83%BC%E3%83%AB%E3%83%95%E3%83%AC%E3%83%B3%E3%83%89-carrymeoff',
+        'md5': '1d53866b2c514b23ed69e4352fdc9839',
+        'info_dict': {
+            'id': '2349002',
+            'ext': 'mp4',
+            'title': '【MMD R-18】ガールフレンド carry_me_off',
+            'age_limit': 18,
+            'duration': 216.78,
+        },
+    }
 
     def _real_extract(self, url):
-        match = re.match(self._VALID_URL, url)
-
-        video_id = match.group('id')
-        raw_video_title = match.group('title')
-        url = 'http://trollvids.com/video/%s/%s' % (video_id, raw_video_title)
-        config_url = 'http://trollvids.com/nuevo/player/config.php?v=%s' % video_id
-
-        info = self._extract_nuevo(config_url, video_id)
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        display_id = mobj.group('display_id')
 
+        info = self._extract_nuevo(
+            'http://trollvids.com/nuevo/player/config.php?v=%s' % video_id,
+            video_id)
         info.update({
-            'webpage_url': url,
+            'display_id': display_id,
             'age_limit': 18
         })
-
-        if 'title' not in info:
-            info['title'] = compat_urllib_parse_unquote(raw_video_title)
-
         return info
-
-    _TESTS = [
-        {
-            'url': 'http://trollvids.com/video/2349002/%E3%80%90MMD-R-18%E3%80%91%E3%82%AC%E3%83%BC%E3%83%AB%E3%83%95%E3%83%AC%E3%83%B3%E3%83%89-carrymeoff',
-            'md5': '1d53866b2c514b23ed69e4352fdc9839',
-            'info_dict': {
-                'id': '2349002',
-                'ext': 'mp4',
-                'title': "【MMD R-18】ガールフレンド carry_me_off",
-                'age_limit': 18,
-                'duration': 216.78,
-            },
-        },
-    ]
diff --git a/youtube_dl/extractor/trutube.py b/youtube_dl/extractor/trutube.py
index d7ec2ec26..d55e0c563 100644
--- a/youtube_dl/extractor/trutube.py
+++ b/youtube_dl/extractor/trutube.py
@@ -4,7 +4,7 @@ from .nuevo import NuevoBaseIE
 
 
 class TruTubeIE(NuevoBaseIE):
-    _VALID_URL = r'https?://(?:www\.)?trutube\.tv/(?:video/|nuevo/player/embed\.php\?v=)(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?trutube\.tv/(?:video/|nuevo/player/embed\.php\?v=)(?P<id>\d+)'
     _TESTS = [{
         'url': 'http://trutube.tv/video/14880/Ramses-II-Proven-To-Be-A-Red-Headed-Caucasoid-',
         'md5': 'c5b6e301b0a2040b074746cbeaa26ca1',
@@ -21,11 +21,6 @@ class TruTubeIE(NuevoBaseIE):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        config_url = 'https://trutube.tv/nuevo/player/config.php?v=%s' % video_id
-
-        info = self._extract_nuevo(config_url, video_id)
-
-        # filehd always 404s
-        info['formats'] = info['formats'][:1]
-
-        return info
+        return self._extract_nuevo(
+            'https://trutube.tv/nuevo/player/config.php?v=%s' % video_id,
+            video_id)

From b9698135480e156c89b30b4da60df1aae1c2a660 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 Jan 2016 00:10:49 +0600
Subject: [PATCH 3753/6219] Credit @nexAkari for trollvids and nuevo (#7728)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 73fcafeb5..bb1f2d8d9 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -154,3 +154,4 @@ Brian Foley
 Vignesh Venkat
 Tom Gijselinck
 Founder Fang
+Andrew Alexeyew

From 1257b049bcd9857e2993a4366af41219c36baa3e Mon Sep 17 00:00:00 2001
From: Dankryn <dankryn@weg-werf-email.de>
Date: Mon, 11 Jan 2016 21:17:30 +0100
Subject: [PATCH 3754/6219] [ruleporn] Add new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/ruleporn.py | 31 +++++++++++++++++++++++++++++++
 2 files changed, 32 insertions(+)
 create mode 100644 youtube_dl/extractor/ruleporn.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 6f2b35cf1..dc8679d46 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -580,6 +580,7 @@ from .rts import RTSIE
 from .rtve import RTVEALaCartaIE, RTVELiveIE, RTVEInfantilIE
 from .rtvnh import RTVNHIE
 from .ruhd import RUHDIE
+from .ruleporn import RulepornIE
 from .rutube import (
     RutubeIE,
     RutubeChannelIE,
diff --git a/youtube_dl/extractor/ruleporn.py b/youtube_dl/extractor/ruleporn.py
new file mode 100644
index 000000000..9e6a9125a
--- /dev/null
+++ b/youtube_dl/extractor/ruleporn.py
@@ -0,0 +1,31 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class RulepornIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?ruleporn\.com/(?:[a-z]+(?:-[a-z]+)+)'
+    _TEST = {
+        'url': 'http://ruleporn.com/brunette-nympho-chick-takes-her-boyfriend-in-every-angle/',
+        'md5': '86861ebc624a1097c7c10eaf06d7d505',
+        'info_dict': {
+            'id': '48212',
+            'ext': 'mp4',
+            'title': 'Brunette Nympho Chick Takes Her Boyfriend In Every Angle',
+        }
+    }
+
+    def _real_extract(self, url):
+        webpage = self._download_webpage(url, None)
+
+        video_id = self._search_regex(r'http://lovehomeporn.com/embed/([0-9]+)', webpage, 'video_id', fatal=True)
+        title = self._search_regex(r'<h2 title="((?:\w|\s|\d)+)">', webpage, 'title', fatal=True)
+        info_xml = self._download_xml('http://lovehomeporn.com/media/nuevo/econfig.php?key=%s&rp=true' % video_id, video_id)
+        url = info_xml.find('file').text
+
+        return {
+            'id': video_id,
+            'title': title,
+            'url': url,
+        }

From ea178204326c0d4203cc3ff88057d2ffd25a68a1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 Jan 2016 00:38:58 +0600
Subject: [PATCH 3755/6219] [nuevo] Improve thumbnail extraction

---
 youtube_dl/extractor/nuevo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nuevo.py b/youtube_dl/extractor/nuevo.py
index 225da033c..ef093dec2 100644
--- a/youtube_dl/extractor/nuevo.py
+++ b/youtube_dl/extractor/nuevo.py
@@ -16,7 +16,7 @@ class NuevoBaseIE(InfoExtractor):
 
         title = xpath_text(config, './title', 'title', fatal=True).strip()
         video_id = xpath_text(config, './mediaid', default=video_id)
-        thumbnail = xpath_text(config, './image')
+        thumbnail = xpath_text(config, ['./image', './thumb'])
         duration = float_or_none(xpath_text(config, './duration'))
 
         formats = []

From b2c6528baf990d1fc0ad7b435595102ae2d8ba2d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 Jan 2016 00:40:11 +0600
Subject: [PATCH 3756/6219] [ruleporn] Rework in terms of nuevo (Closes #8206)

---
 youtube_dl/extractor/__init__.py |  2 +-
 youtube_dl/extractor/ruleporn.py | 39 +++++++++++++++++++++-----------
 2 files changed, 27 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index dc8679d46..4ea6a3f71 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -580,7 +580,7 @@ from .rts import RTSIE
 from .rtve import RTVEALaCartaIE, RTVELiveIE, RTVEInfantilIE
 from .rtvnh import RTVNHIE
 from .ruhd import RUHDIE
-from .ruleporn import RulepornIE
+from .ruleporn import RulePornIE
 from .rutube import (
     RutubeIE,
     RutubeChannelIE,
diff --git a/youtube_dl/extractor/ruleporn.py b/youtube_dl/extractor/ruleporn.py
index 9e6a9125a..ebf9808d5 100644
--- a/youtube_dl/extractor/ruleporn.py
+++ b/youtube_dl/extractor/ruleporn.py
@@ -1,31 +1,44 @@
-# coding: utf-8
 from __future__ import unicode_literals
 
-from .common import InfoExtractor
+from .nuevo import NuevoBaseIE
 
 
-class RulepornIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?ruleporn\.com/(?:[a-z]+(?:-[a-z]+)+)'
+class RulePornIE(NuevoBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?ruleporn\.com/(?:[^/?#&]+/)*(?P<id>[^/?#&]+)'
     _TEST = {
         'url': 'http://ruleporn.com/brunette-nympho-chick-takes-her-boyfriend-in-every-angle/',
         'md5': '86861ebc624a1097c7c10eaf06d7d505',
         'info_dict': {
             'id': '48212',
+            'display_id': 'brunette-nympho-chick-takes-her-boyfriend-in-every-angle',
             'ext': 'mp4',
             'title': 'Brunette Nympho Chick Takes Her Boyfriend In Every Angle',
+            'description': 'md5:6d28be231b981fff1981deaaa03a04d5',
+            'age_limit': 18,
+            'duration': 635.1,
         }
     }
 
     def _real_extract(self, url):
-        webpage = self._download_webpage(url, None)
+        display_id = self._match_id(url)
 
-        video_id = self._search_regex(r'http://lovehomeporn.com/embed/([0-9]+)', webpage, 'video_id', fatal=True)
-        title = self._search_regex(r'<h2 title="((?:\w|\s|\d)+)">', webpage, 'title', fatal=True)
-        info_xml = self._download_xml('http://lovehomeporn.com/media/nuevo/econfig.php?key=%s&rp=true' % video_id, video_id)
-        url = info_xml.find('file').text
+        webpage = self._download_webpage(url, display_id)
 
-        return {
-            'id': video_id,
+        video_id = self._search_regex(
+            r'lovehomeporn\.com/embed/(\d+)', webpage, 'video id')
+
+        title = self._search_regex(
+            r'<h2[^>]+title=(["\'])(?P<url>.+?)\1',
+            webpage, 'title', group='url')
+        description = self._html_search_meta('description', webpage)
+
+        info = self._extract_nuevo(
+            'http://lovehomeporn.com/media/nuevo/econfig.php?key=%s&rp=true' % video_id,
+            video_id)
+        info.update({
+            'display_id': display_id,
             'title': title,
-            'url': url,
-        }
+            'description': description,
+            'age_limit': 18
+        })
+        return info

From 4c0d13df9bdf7222fbfa6dde543ffcdb47696392 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 Jan 2016 00:52:23 +0600
Subject: [PATCH 3757/6219] [lovehomeporn] Add extractor

---
 youtube_dl/extractor/__init__.py     |  1 +
 youtube_dl/extractor/lovehomeporn.py | 37 ++++++++++++++++++++++++++++
 2 files changed, 38 insertions(+)
 create mode 100644 youtube_dl/extractor/lovehomeporn.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 4ea6a3f71..245e4d044 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -358,6 +358,7 @@ from .livestream import (
     LivestreamShortenerIE,
 )
 from .lnkgo import LnkGoIE
+from .lovehomeporn import LoveHomePornIE
 from .lrt import LRTIE
 from .lynda import (
     LyndaIE,
diff --git a/youtube_dl/extractor/lovehomeporn.py b/youtube_dl/extractor/lovehomeporn.py
new file mode 100644
index 000000000..8f65a3c03
--- /dev/null
+++ b/youtube_dl/extractor/lovehomeporn.py
@@ -0,0 +1,37 @@
+from __future__ import unicode_literals
+
+import re
+
+from .nuevo import NuevoBaseIE
+
+
+class LoveHomePornIE(NuevoBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?lovehomeporn\.com/video/(?P<id>\d+)(?:/(?P<display_id>[^/?#&]+))?'
+    _TEST = {
+        'url': 'http://lovehomeporn.com/video/48483/stunning-busty-brunette-girlfriend-sucking-and-riding-a-big-dick#menu',
+        'info_dict': {
+            'id': '48483',
+            'display_id': 'stunning-busty-brunette-girlfriend-sucking-and-riding-a-big-dick',
+            'ext': 'mp4',
+            'title': 'Stunning busty brunette girlfriend sucking and riding a big dick',
+            'age_limit': 18,
+            'duration': 238.47,
+        },
+        'params': {
+            'skip_download': True,
+        }
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        display_id = mobj.group('display_id')
+
+        info = self._extract_nuevo(
+            'http://lovehomeporn.com/media/nuevo/config.php?key=%s' % video_id,
+            video_id)
+        info.update({
+            'display_id': display_id,
+            'age_limit': 18
+        })
+        return info

From 1f16b958b11c4c8503a3555d46bd205540d7aae8 Mon Sep 17 00:00:00 2001
From: Marian Sigler <m@qjym.de>
Date: Tue, 12 Jan 2016 17:08:55 +0100
Subject: [PATCH 3758/6219] [SVTPlay] Add subtitle support

---
 youtube_dl/extractor/svt.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/youtube_dl/extractor/svt.py b/youtube_dl/extractor/svt.py
index fc20f664b..7ff5ee11a 100644
--- a/youtube_dl/extractor/svt.py
+++ b/youtube_dl/extractor/svt.py
@@ -37,6 +37,12 @@ class SVTBaseIE(InfoExtractor):
                 })
         self._sort_formats(formats)
 
+        # SVT does not tell us the language, so we assume swedish.
+        subtitles = {}
+        for sr in video_info['subtitleReferences']:
+            if 'url' in sr:
+                subtitles.setdefault('sv', []).append({'url': sr['url']})
+
         duration = video_info.get('materialLength')
         age_limit = 18 if video_info.get('inappropriateForChildren') else 0
 
@@ -44,6 +50,7 @@ class SVTBaseIE(InfoExtractor):
             'id': video_id,
             'title': title,
             'formats': formats,
+            'subtitles': subtitles,
             'thumbnail': thumbnail,
             'duration': duration,
             'age_limit': age_limit,

From 594c4d79a5cc988af99cbd0e3c00d1752e277dd1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 Jan 2016 01:47:54 +0600
Subject: [PATCH 3759/6219] [svt] Improve subtitles extraction and add test
 (Closes #8265)

---
 youtube_dl/extractor/svt.py | 43 ++++++++++++++++---------------------
 1 file changed, 19 insertions(+), 24 deletions(-)

diff --git a/youtube_dl/extractor/svt.py b/youtube_dl/extractor/svt.py
index 7ff5ee11a..399c3b8ee 100644
--- a/youtube_dl/extractor/svt.py
+++ b/youtube_dl/extractor/svt.py
@@ -37,11 +37,13 @@ class SVTBaseIE(InfoExtractor):
                 })
         self._sort_formats(formats)
 
-        # SVT does not tell us the language, so we assume swedish.
         subtitles = {}
-        for sr in video_info['subtitleReferences']:
-            if 'url' in sr:
-                subtitles.setdefault('sv', []).append({'url': sr['url']})
+        subtitle_references = video_info.get('subtitleReferences')
+        if isinstance(subtitle_references, list):
+            for sr in subtitle_references:
+                subtitle_url = sr.get('url')
+                if subtitle_url:
+                    subtitles.setdefault('sv', []).append({'url': subtitle_url})
 
         duration = video_info.get('materialLength')
         age_limit = 18 if video_info.get('inappropriateForChildren') else 0
@@ -90,30 +92,23 @@ class SVTIE(SVTBaseIE):
 class SVTPlayIE(SVTBaseIE):
     IE_DESC = 'SVT Play and Öppet arkiv'
     _VALID_URL = r'https?://(?:www\.)?(?P<host>svtplay|oppetarkiv)\.se/video/(?P<id>[0-9]+)'
-    _TESTS = [{
-        'url': 'http://www.svtplay.se/video/2609989/sm-veckan/sm-veckan-rally-final-sasong-1-sm-veckan-rally-final',
-        'md5': 'ade3def0643fa1c40587a422f98edfd9',
+    _TEST = {
+        'url': 'http://www.svtplay.se/video/5996901/flygplan-till-haile-selassie/flygplan-till-haile-selassie-2',
+        'md5': '2b6704fe4a28801e1a098bbf3c5ac611',
         'info_dict': {
-            'id': '2609989',
-            'ext': 'flv',
-            'title': 'SM veckan vinter, Örebro - Rally, final',
-            'duration': 4500,
+            'id': '5996901',
+            'ext': 'mp4',
+            'title': 'Flygplan till Haile Selassie',
+            'duration': 3527,
             'thumbnail': 're:^https?://.*[\.-]jpg$',
             'age_limit': 0,
+            'subtitles': {
+                'sv': [{
+                    'ext': 'wsrt',
+                }]
+            },
         },
-    }, {
-        'url': 'http://www.oppetarkiv.se/video/1058509/rederiet-sasong-1-avsnitt-1-av-318',
-        'md5': 'c3101a17ce9634f4c1f9800f0746c187',
-        'info_dict': {
-            'id': '1058509',
-            'ext': 'flv',
-            'title': 'Farlig kryssning',
-            'duration': 2566,
-            'thumbnail': 're:^https?://.*[\.-]jpg$',
-            'age_limit': 0,
-        },
-        'skip': 'Only works from Sweden',
-    }]
+    }
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)

From 1ca59daca964b789105b50aa42835aa87c982342 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 Jan 2016 01:50:06 +0600
Subject: [PATCH 3760/6219] [options] Clarify language tags

---
 youtube_dl/options.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index ade58c375..433245f00 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -380,7 +380,7 @@ def parseOpts(overrideArguments=None):
         '--sub-lang', '--sub-langs', '--srt-lang',
         action='callback', dest='subtitleslangs', metavar='LANGS', type='str',
         default=[], callback=_comma_separated_values_options_callback,
-        help='Languages of the subtitles to download (optional) separated by commas, use IETF language tags like \'en,pt\'')
+        help='Languages of the subtitles to download (optional) separated by commas, use --list-subs for available language tags')
 
     downloader = optparse.OptionGroup(parser, 'Download Options')
     downloader.add_option(

From 5ca01bb9e4803d82734420445d49ea7f98579fe6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 Jan 2016 01:51:18 +0600
Subject: [PATCH 3761/6219] [kanalplay] Use IETF language tag

---
 youtube_dl/extractor/kanalplay.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/kanalplay.py b/youtube_dl/extractor/kanalplay.py
index 4597d1b96..6c3498c67 100644
--- a/youtube_dl/extractor/kanalplay.py
+++ b/youtube_dl/extractor/kanalplay.py
@@ -49,7 +49,7 @@ class KanalPlayIE(InfoExtractor):
         subs = self._download_json(
             'http://www.kanal%splay.se/api/subtitles/%s' % (channel_id, video_id),
             video_id, 'Downloading subtitles JSON', fatal=False)
-        return {'se': [{'ext': 'srt', 'data': self._fix_subtitles(subs)}]} if subs else {}
+        return {'sv': [{'ext': 'srt', 'data': self._fix_subtitles(subs)}]} if subs else {}
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)

From 6fa73386cbb9b3d39dafde9c61bd950bd99b7155 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 Jan 2016 01:54:00 +0600
Subject: [PATCH 3762/6219] [drtv] Use IETF language tag

---
 youtube_dl/extractor/drtv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/drtv.py b/youtube_dl/extractor/drtv.py
index baa24c6d1..2d74ff855 100644
--- a/youtube_dl/extractor/drtv.py
+++ b/youtube_dl/extractor/drtv.py
@@ -91,7 +91,7 @@ class DRTVIE(InfoExtractor):
                 subtitles_list = asset.get('SubtitlesList')
                 if isinstance(subtitles_list, list):
                     LANGS = {
-                        'Danish': 'dk',
+                        'Danish': 'da',
                     }
                     for subs in subtitles_list:
                         lang = subs['Language']

From f733b05302f44776d369f45c730da4a5b558e21b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 23 Jan 2016 12:03:12 +0100
Subject: [PATCH 3763/6219] release 2016.01.23

---
 README.md              |  4 ++--
 docs/supportedsites.md | 13 +++++++++++--
 youtube_dl/version.py  |  2 +-
 3 files changed, 14 insertions(+), 5 deletions(-)

diff --git a/README.md b/README.md
index 9dbeae1bc..724fb17d1 100644
--- a/README.md
+++ b/README.md
@@ -339,8 +339,8 @@ which means you can modify it, redistribute it or use it however you like.
                                      preference, for example: "srt" or
                                      "ass/srt/best"
     --sub-lang LANGS                 Languages of the subtitles to download
-                                     (optional) separated by commas, use IETF
-                                     language tags like 'en,pt'
+                                     (optional) separated by commas, use --list-
+                                     subs for available language tags
 
 ## Authentication Options:
     -u, --username USERNAME          Login with this account ID
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 99b1e2731..e86467cfa 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -24,7 +24,7 @@
  - **AdobeTVShow**
  - **AdobeTVVideo**
  - **AdultSwim**
- - **AE**
+ - **aenetworks**: A+E Networks: A&E, Lifetime, History.com, FYI Network
  - **Aftonbladet**
  - **AirMozilla**
  - **AlJazeera**
@@ -43,6 +43,7 @@
  - **ARD:mediathek**
  - **arte.tv**
  - **arte.tv:+7**
+ - **arte.tv:cinema**
  - **arte.tv:concert**
  - **arte.tv:creative**
  - **arte.tv:ddc**
@@ -124,6 +125,7 @@
  - **CSpan**: C-SPAN
  - **CtsNews**: 華視新聞
  - **culturebox.francetvinfo.fr**
+ - **CultureUnplugged**
  - **CWTV**
  - **dailymotion**
  - **dailymotion:playlist**
@@ -141,6 +143,7 @@
  - **defense.gouv.fr**
  - **democracynow**
  - **DHM**: Filmarchiv - Deutsches Historisches Museum
+ - **Digiteka**
  - **Discovery**
  - **Dotsub**
  - **DouyuTV**: 斗鱼
@@ -287,7 +290,9 @@
  - **la7.tv**
  - **Laola1Tv**
  - **Lecture2Go**
+ - **Lemonde**
  - **Letv**: 乐视网
+ - **LetvCloud**: 乐视云
  - **LetvPlaylist**
  - **LetvTv**
  - **Libsyn**
@@ -300,6 +305,7 @@
  - **livestream**
  - **livestream:original**
  - **LnkGo**
+ - **LoveHomePorn**
  - **lrt.lt**
  - **lynda**: lynda.com videos
  - **lynda:course**: lynda.com online courses
@@ -486,6 +492,7 @@
  - **rtve.es:live**: RTVE.es live streams
  - **RTVNH**
  - **RUHD**
+ - **RulePorn**
  - **rutube**: Rutube videos
  - **rutube:channel**: Rutube channels
  - **rutube:embed**: Rutube embedded videos
@@ -602,6 +609,7 @@
  - **ToypicsUser**: Toypics user profile
  - **TrailerAddict** (Currently broken)
  - **Trilulilu**
+ - **trollvids**
  - **TruTube**
  - **Tube8**
  - **TubiTv**
@@ -640,7 +648,6 @@
  - **udemy**
  - **udemy:course**
  - **UDNEmbed**: 聯合影音
- - **Ultimedia**
  - **Unistra**
  - **Urort**: NRK P3 Urørt
  - **ustream**
@@ -707,6 +714,7 @@
  - **WebOfStories**
  - **WebOfStoriesPlaylist**
  - **Weibo**
+ - **WeiqiTV**: WQTV
  - **wholecloud**: WholeCloud
  - **Wimp**
  - **Wistia**
@@ -758,3 +766,4 @@
  - **ZDFChannel**
  - **zingmp3:album**: mp3.zing.vn albums
  - **zingmp3:song**: mp3.zing.vn songs
+ - **ZippCast**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 02c438f3a..d5bf73815 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.01.15'
+__version__ = '2016.01.23'

From f6861ec96f33722c9b1ba4f2d8ca307dcbe64ac1 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 25 Jan 2016 00:58:53 +0800
Subject: [PATCH 3764/6219] [utils] Add more items to mimetype2ext (#8293)

These are used in Youtube formats
---
 youtube_dl/utils.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 9c1c0e0bd..178d1dcb3 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1831,6 +1831,8 @@ def mimetype2ext(mt):
         'x-ms-wmv': 'wmv',
         'x-mp4-fragmented': 'mp4',
         'ttml+xml': 'ttml',
+        '3gpp': '3gp',
+        'x-flv': 'flv',
     }.get(res, res)
 
 

From a0d8d704df5ff3098b86e741f238d53de1711198 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 25 Jan 2016 01:01:15 +0800
Subject: [PATCH 3765/6219] [utils] Reorder items in mimetype2ext
 alphabetically

---
 youtube_dl/utils.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 178d1dcb3..c63b61598 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1828,11 +1828,11 @@ def mimetype2ext(mt):
     _, _, res = mt.rpartition('/')
 
     return {
-        'x-ms-wmv': 'wmv',
-        'x-mp4-fragmented': 'mp4',
-        'ttml+xml': 'ttml',
         '3gpp': '3gp',
+        'ttml+xml': 'ttml',
         'x-flv': 'flv',
+        'x-mp4-fragmented': 'mp4',
+        'x-ms-wmv': 'wmv',
     }.get(res, res)
 
 

From 94278f720272c5ad2cd5900f59f8e71f31d46633 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 25 Jan 2016 01:02:19 +0800
Subject: [PATCH 3766/6219] [youtube] Prefer info from YouTube than _formats
 (#8293)

---
 youtube_dl/extractor/youtube.py | 28 ++++++++++++++++++----------
 1 file changed, 18 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 8e8fc14d2..6f0665775 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -32,6 +32,7 @@ from ..utils import (
     get_element_by_attribute,
     get_element_by_id,
     int_or_none,
+    mimetype2ext,
     orderedSet,
     parse_duration,
     remove_quotes,
@@ -1090,9 +1091,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                         full_info.update(f)
                         codecs = r.attrib.get('codecs')
                         if codecs:
-                            if full_info.get('acodec') == 'none' and 'vcodec' not in full_info:
+                            if full_info.get('acodec') == 'none':
                                 full_info['vcodec'] = codecs
-                            elif full_info.get('vcodec') == 'none' and 'acodec' not in full_info:
+                            elif full_info.get('vcodec') == 'none':
                                 full_info['acodec'] = codecs
                         formats.append(full_info)
                     else:
@@ -1461,15 +1462,21 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 if 'ratebypass' not in url:
                     url += '&ratebypass=yes'
 
+                dct = {
+                    'format_id': format_id,
+                    'url': url,
+                    'player_url': player_url,
+                }
+                if format_id in self._formats:
+                    dct.update(self._formats[format_id])
+
                 # Some itags are not included in DASH manifest thus corresponding formats will
                 # lack metadata (see https://github.com/rg3/youtube-dl/pull/5993).
                 # Trying to extract metadata from url_encoded_fmt_stream_map entry.
                 mobj = re.search(r'^(?P<width>\d+)[xX](?P<height>\d+)$', url_data.get('size', [''])[0])
                 width, height = (int(mobj.group('width')), int(mobj.group('height'))) if mobj else (None, None)
-                dct = {
-                    'format_id': format_id,
-                    'url': url,
-                    'player_url': player_url,
+
+                more_fields = {
                     'filesize': int_or_none(url_data.get('clen', [None])[0]),
                     'tbr': float_or_none(url_data.get('bitrate', [None])[0], 1000),
                     'width': width,
@@ -1477,13 +1484,16 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     'fps': int_or_none(url_data.get('fps', [None])[0]),
                     'format_note': url_data.get('quality_label', [None])[0] or url_data.get('quality', [None])[0],
                 }
+                for key, value in more_fields.items():
+                    if value:
+                        dct[key] = value
                 type_ = url_data.get('type', [None])[0]
                 if type_:
                     type_split = type_.split(';')
                     kind_ext = type_split[0].split('/')
                     if len(kind_ext) == 2:
-                        kind, ext = kind_ext
-                        dct['ext'] = ext
+                        kind, _ = kind_ext
+                        dct['ext'] = mimetype2ext(type_split[0])
                         if kind in ('audio', 'video'):
                             codecs = None
                             for mobj in re.finditer(
@@ -1501,8 +1511,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                                     'acodec': acodec,
                                     'vcodec': vcodec,
                                 })
-                if format_id in self._formats:
-                    dct.update(self._formats[format_id])
                 formats.append(dct)
         elif video_info.get('hlsvp'):
             manifest_url = video_info['hlsvp'][0]

From 77f785076f9a2fcfc35de49c8e612716bbb37f6c Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 25 Jan 2016 01:03:46 +0800
Subject: [PATCH 3767/6219] [common] Keep full codec name from m3u8 manifests

See #8293. This is for consistency between YouTube and HLS formats.
---
 youtube_dl/extractor/common.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 8da70ae14..2f574054d 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1058,9 +1058,9 @@ class InfoExtractor(object):
                     # TODO: looks like video codec is not always necessarily goes first
                     va_codecs = codecs.split(',')
                     if va_codecs[0]:
-                        f['vcodec'] = va_codecs[0].partition('.')[0]
+                        f['vcodec'] = va_codecs[0]
                     if len(va_codecs) > 1 and va_codecs[1]:
-                        f['acodec'] = va_codecs[1].partition('.')[0]
+                        f['acodec'] = va_codecs[1]
                 resolution = last_info.get('RESOLUTION')
                 if resolution:
                     width_str, height_str = resolution.split('x')

From 2e6e742c3cda5ed4846bff4ef894aac21434e3d4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 25 Jan 2016 22:15:21 +0600
Subject: [PATCH 3768/6219] [facebook] Add shortcut and reformat _VALID_URL

---
 youtube_dl/extractor/facebook.py | 29 ++++++++++++++++++++---------
 1 file changed, 20 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index ec699ba54..f9fd8ed4e 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -23,15 +23,23 @@ from ..utils import (
 
 class FacebookIE(InfoExtractor):
     _VALID_URL = r'''(?x)
-        https?://(?:\w+\.)?facebook\.com/
-        (?:[^#]*?\#!/)?
-        (?:
-            (?:video/video\.php|photo\.php|video\.php|video/embed)\?(?:.*?)
-            (?:v|video_id)=|
-            [^/]+/videos/(?:[^/]+/)?
-        )
-        (?P<id>[0-9]+)
-        (?:.*)'''
+                (?:
+                    https?://
+                        (?:\w+\.)?facebook\.com/
+                        (?:[^#]*?\#!/)?
+                        (?:
+                            (?:
+                                video/video\.php|
+                                photo\.php|
+                                video\.php|
+                                video/embed
+                            )\?(?:.*?)(?:v|video_id)=|
+                            [^/]+/videos/(?:[^/]+/)?
+                        )|
+                    facebook:
+                )
+                (?P<id>[0-9]+)
+                '''
     _LOGIN_URL = 'https://www.facebook.com/login.php?next=http%3A%2F%2Ffacebook.com%2Fhome.php&login_attempt=1'
     _CHECKPOINT_URL = 'https://www.facebook.com/checkpoint/?next=http%3A%2F%2Ffacebook.com%2Fhome.php&_fb_noscript=1'
     _NETRC_MACHINE = 'facebook'
@@ -66,6 +74,9 @@ class FacebookIE(InfoExtractor):
     }, {
         'url': 'https://www.facebook.com/ChristyClarkForBC/videos/vb.22819070941/10153870694020942/?type=2&theater',
         'only_matching': True,
+    }, {
+        'url': 'facebook:544765982287235',
+        'only_matching': True,
     }]
 
     def _login(self):

From de691a498dd334ee5f8d237a4fc7ced314d86f44 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 25 Jan 2016 22:18:34 +0600
Subject: [PATCH 3769/6219] [facebook:post] Add extractor (Closes #8321)

---
 youtube_dl/extractor/__init__.py |  5 ++++-
 youtube_dl/extractor/facebook.py | 30 ++++++++++++++++++++++++++++++
 2 files changed, 34 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 245e4d044..532be7e4c 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -196,7 +196,10 @@ from .everyonesmixtape import EveryonesMixtapeIE
 from .exfm import ExfmIE
 from .expotv import ExpoTVIE
 from .extremetube import ExtremeTubeIE
-from .facebook import FacebookIE
+from .facebook import (
+    FacebookIE,
+    FacebookPostIE,
+)
 from .faz import FazIE
 from .fc2 import FC2IE
 from .fczenit import FczenitIE
diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index f9fd8ed4e..cb5dd57fb 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -199,3 +199,33 @@ class FacebookIE(InfoExtractor):
             'formats': formats,
             'uploader': uploader,
         }
+
+
+class FacebookPostIE(InfoExtractor):
+    IE_NAME = 'facebook:post'
+    _VALID_URL = r'https?://(?:\w+\.)?facebook\.com/[^/]+/posts/(?P<id>\d+)'
+    _TEST = {
+        'url': 'https://www.facebook.com/maxlayn/posts/10153807558977570',
+        'md5': '037b1fa7f3c2d02b7a0d7bc16031ecc6',
+        'info_dict': {
+            'id': '544765982287235',
+            'ext': 'mp4',
+            'title': '"What are you doing running in the snow?"',
+            'uploader': 'FailArmy',
+        }
+    }
+
+    def _real_extract(self, url):
+        post_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, post_id)
+
+        entries = [
+            self.url_result('facebook:%s' % video_id, FacebookIE.ie_key())
+            for video_id in self._parse_json(
+                self._search_regex(
+                    r'(["\'])video_ids\1\s*:\s*(?P<ids>\[.+?\])',
+                    webpage, 'video ids', group='ids'),
+                post_id)]
+
+        return self.playlist_result(entries, post_id)

From 2975fe1a7be39b66d14fc7ef68ea7db948f0e40f Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 25 Jan 2016 22:35:06 +0100
Subject: [PATCH 3770/6219] [vevo] extract all formats and bypass geo
 restriction

---
 youtube_dl/extractor/vevo.py | 234 +++++++++++++++--------------------
 1 file changed, 102 insertions(+), 132 deletions(-)

diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py
index 02dfd36f4..ff854e74b 100644
--- a/youtube_dl/extractor/vevo.py
+++ b/youtube_dl/extractor/vevo.py
@@ -3,22 +3,18 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_etree_fromstring,
-    compat_urlparse,
-)
+from ..compat import compat_etree_fromstring
 from ..utils import (
     ExtractorError,
     int_or_none,
-    sanitized_Request,
 )
 
 
 class VevoIE(InfoExtractor):
-    """
+    '''
     Accepts urls from vevo.com or in the format 'vevo:{id}'
     (currently used by MTVIE and MySpaceIE)
-    """
+    '''
     _VALID_URL = r'''(?x)
         (?:https?://www\.vevo\.com/watch/(?:[^/]+/(?:[^/]+/)?)?|
            https?://cache\.vevo\.com/m/html/embed\.html\?video=|
@@ -28,48 +24,51 @@ class VevoIE(InfoExtractor):
 
     _TESTS = [{
         'url': 'http://www.vevo.com/watch/hurts/somebody-to-die-for/GB1101300280',
-        "md5": "95ee28ee45e70130e3ab02b0f579ae23",
+        'md5': '2dbc7e9fd4f1c60436c9aa73a5406193',
         'info_dict': {
-            'id': 'GB1101300280',
+            'id': 'Pt1kc_FniKM',
             'ext': 'mp4',
-            "upload_date": "20130624",
-            "uploader": "Hurts",
-            "title": "Somebody to Die For",
-            "duration": 230.12,
-            "width": 1920,
-            "height": 1080,
-            # timestamp and upload_date are often incorrect; seem to change randomly
-            'timestamp': int,
-        }
+            'title': 'Hurts - Somebody to Die For',
+            'description': 'md5:13e925b89af6b01c7e417332bd23c4bf',
+            'uploader_id': 'HurtsVEVO',
+            'uploader': 'HurtsVEVO',
+            'upload_date': '20130624',
+            'duration': 230,
+        },
+        'add_ie': ['Youtube'],
     }, {
         'note': 'v3 SMIL format',
         'url': 'http://www.vevo.com/watch/cassadee-pope/i-wish-i-could-break-your-heart/USUV71302923',
-        'md5': 'f6ab09b034f8c22969020b042e5ac7fc',
+        'md5': '13d5204f520af905eeffa675040b8e76',
         'info_dict': {
-            'id': 'USUV71302923',
+            'id': 'ByGmQn1uxJw',
             'ext': 'mp4',
+            'title': 'Cassadee Pope - I Wish I Could Break Your Heart',
+            'description': 'md5:5e9721c92ef117a6f69d00e9b42ceba7',
+            'uploader_id': 'CassadeeVEVO',
+            'uploader': 'CassadeeVEVO',
             'upload_date': '20140219',
-            'uploader': 'Cassadee Pope',
-            'title': 'I Wish I Could Break Your Heart',
-            'duration': 226.101,
+            'duration': 226,
             'age_limit': 0,
-            'timestamp': int,
-        }
+        },
+        'add_ie': ['Youtube'],
     }, {
         'note': 'Age-limited video',
         'url': 'https://www.vevo.com/watch/justin-timberlake/tunnel-vision-explicit/USRV81300282',
         'info_dict': {
-            'id': 'USRV81300282',
+            'id': '07FYdnEawAQ',
             'ext': 'mp4',
             'age_limit': 18,
-            'title': 'Tunnel Vision (Explicit)',
-            'uploader': 'Justin Timberlake',
-            'upload_date': 're:2013070[34]',
-            'timestamp': int,
+            'title': 'Justin Timberlake - Tunnel Vision (Explicit)',
+            'description': 'md5:64249768eec3bc4276236606ea996373',
+            'uploader_id': 'justintimberlakeVEVO',
+            'uploader': 'justintimberlakeVEVO',
+            'upload_date': '20130703',
         },
         'params': {
             'skip_download': 'true',
-        }
+        },
+        'add_ie': ['Youtube'],
     }, {
         'note': 'No video_info',
         'url': 'http://www.vevo.com/watch/k-camp-1/Till-I-Die/USUV71503000',
@@ -80,66 +79,40 @@ class VevoIE(InfoExtractor):
             'title': 'Till I Die - K Camp ft. T.I.',
             'duration': 193,
         },
-        'expected_warnings': ['Unable to download SMIL file'],
     }]
-    _SMIL_BASE_URL = 'http://smil.lvl3.vevo.com/'
+    _SMIL_BASE_URL = 'http://smil.lvl3.vevo.com'
+    _SOURCE_TYPES = {
+        0: 'youtube',
+        1: 'brightcove',
+        2: 'http',
+        3: 'hls_ios',
+        4: 'hls',
+        5: 'smil',  # http
+        7: 'f4m_cc',
+        8: 'f4m_ak',
+        9: 'f4m_l3',
+        10: 'ism',
+        13: 'smil',  # rtmp
+        18: 'dash',
+    }
+    _VERSIONS = {
+        0: 'youtube',
+        1: 'level3',
+        2: 'akamai',
+        3: 'level3',
+        4: 'amazon',
+    }
 
-    def _real_initialize(self):
-        req = sanitized_Request(
-            'http://www.vevo.com/auth', data=b'')
-        webpage = self._download_webpage(
-            req, None,
-            note='Retrieving oauth token',
-            errnote='Unable to retrieve oauth token',
-            fatal=False)
-        if webpage is False:
-            self._oauth_token = None
-        else:
-            if 'THIS PAGE IS CURRENTLY UNAVAILABLE IN YOUR REGION' in webpage:
-                raise ExtractorError('%s said: This page is currently unavailable in your region.' % self.IE_NAME, expected=True)
-
-            self._oauth_token = self._search_regex(
-                r'access_token":\s*"([^"]+)"',
-                webpage, 'access token', fatal=False)
-
-    def _formats_from_json(self, video_info):
-        if not video_info:
-            return []
-
-        last_version = {'version': -1}
-        for version in video_info['videoVersions']:
-            # These are the HTTP downloads, other types are for different manifests
-            if version['sourceType'] == 2:
-                if version['version'] > last_version['version']:
-                    last_version = version
-        if last_version['version'] == -1:
-            raise ExtractorError('Unable to extract last version of the video')
-
-        renditions = compat_etree_fromstring(last_version['data'])
+    def _parse_smil_formats(self, smil, smil_url, video_id, namespace=None, f4m_params=None, transform_rtmp_url=None):
         formats = []
-        # Already sorted from worst to best quality
-        for rend in renditions.findall('rendition'):
-            attr = rend.attrib
-            format_note = '%(videoCodec)s@%(videoBitrate)4sk, %(audioCodec)s@%(audioBitrate)3sk' % attr
-            formats.append({
-                'url': attr['url'],
-                'format_id': attr['name'],
-                'format_note': format_note,
-                'height': int(attr['frameheight']),
-                'width': int(attr['frameWidth']),
-            })
-        return formats
-
-    def _formats_from_smil(self, smil_doc):
-        formats = []
-        els = smil_doc.findall('.//{http://www.w3.org/2001/SMIL20/Language}video')
+        els = smil.findall('.//{http://www.w3.org/2001/SMIL20/Language}video')
         for el in els:
             src = el.attrib['src']
             m = re.match(r'''(?xi)
                 (?P<ext>[a-z0-9]+):
                 (?P<path>
                     [/a-z0-9]+     # The directory and main part of the URL
-                    _(?P<cbr>[0-9]+)k
+                    _(?P<tbr>[0-9]+)k
                     _(?P<width>[0-9]+)x(?P<height>[0-9]+)
                     _(?P<vcodec>[a-z0-9]+)
                     _(?P<vbr>[0-9]+)
@@ -153,9 +126,10 @@ class VevoIE(InfoExtractor):
             format_url = self._SMIL_BASE_URL + m.group('path')
             formats.append({
                 'url': format_url,
-                'format_id': 'SMIL_' + m.group('cbr'),
+                'format_id': 'smil_' + m.group('tbr'),
                 'vcodec': m.group('vcodec'),
                 'acodec': m.group('acodec'),
+                'tbr': int(m.group('tbr')),
                 'vbr': int(m.group('vbr')),
                 'abr': int(m.group('abr')),
                 'ext': m.group('ext'),
@@ -164,26 +138,6 @@ class VevoIE(InfoExtractor):
             })
         return formats
 
-    def _download_api_formats(self, video_id, video_url):
-        if not self._oauth_token:
-            self._downloader.report_warning(
-                'No oauth token available, skipping API HLS download')
-            return []
-
-        api_url = compat_urlparse.urljoin(video_url, '//apiv2.vevo.com/video/%s/streams/hls?token=%s' % (
-            video_id, self._oauth_token))
-        api_data = self._download_json(
-            api_url, video_id,
-            note='Downloading HLS formats',
-            errnote='Failed to download HLS format list', fatal=False)
-        if api_data is None:
-            return []
-
-        m3u8_url = api_data[0]['url']
-        return self._extract_m3u8_formats(
-            m3u8_url, video_id, entry_protocol='m3u8_native', ext='mp4',
-            preference=0)
-
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
@@ -193,19 +147,57 @@ class VevoIE(InfoExtractor):
         response = self._download_json(json_url, video_id)
         video_info = response['video'] or {}
 
-        if not video_info and response.get('statusCode') != 909:
-            if 'statusMessage' in response:
-                raise ExtractorError('%s said: %s' % (self.IE_NAME, response['statusMessage']), expected=True)
-            raise ExtractorError('Unable to extract videos')
-
         if not video_info:
+            ytid = response.get('errorInfo', {}).get('ytid')
+            if ytid:
+                return self.url_result(ytid, 'Youtube', ytid)
+
+            if response.get('statusCode') != 909:
+                if 'statusMessage' in response:
+                    raise ExtractorError('%s said: %s' % (
+                        self.IE_NAME, response['statusMessage']), expected=True)
+                raise ExtractorError('Unable to extract videos')
+
             if url.startswith('vevo:'):
-                raise ExtractorError('Please specify full Vevo URL for downloading', expected=True)
+                raise ExtractorError(
+                    'Please specify full Vevo URL for downloading', expected=True)
             webpage = self._download_webpage(url, video_id)
 
         title = video_info.get('title') or self._og_search_title(webpage)
 
-        formats = self._formats_from_json(video_info)
+        smil_parsed = False
+        formats = []
+        for video_version in video_info['videoVersions']:
+            version = self._VERSIONS.get(video_version['version'])
+            if version == 'youtube':
+                return self.url_result(
+                    video_version['id'], 'Youtube', video_version['id'])
+            else:
+                source_type = self._SOURCE_TYPES.get(video_version['sourceType'])
+                renditions = compat_etree_fromstring(video_version['data'])
+                if source_type == 'http':
+                    for rend in renditions.findall('rendition'):
+                        attr = rend.attrib
+                        formats.append({
+                            'url': attr['url'],
+                            'format_id': '%s-%s' % (version, attr['name']),
+                            'height': int_or_none(attr.get('frameheight')),
+                            'width': int_or_none(attr.get('frameWidth')),
+                            'tbr': int_or_none(attr.get('totalBitrate')),
+                            'vbr': int_or_none(attr.get('videoBitrate')),
+                            'abr': int_or_none(attr.get('audioBitrate')),
+                            'vcodec': attr.get('videoCodec'),
+                            'acodec': attr.get('audioCodec'),
+                        })
+                elif source_type == 'hls':
+                    formats.extend(self._extract_m3u8_formats(
+                        renditions.find('rendition').attrib['url'], video_id,
+                        'mp4', 'm3u8_native', m3u8_id='hls-%s' % version, fatal=False))
+                elif source_type == 'smil' and not smil_parsed:
+                    formats.extend(self._extract_smil_formats(
+                        renditions.find('rendition').attrib['url'], video_id, False))
+                    smil_parsed = True
+        self._sort_formats(formats)
 
         is_explicit = video_info.get('isExplicit')
         if is_explicit is True:
@@ -215,28 +207,6 @@ class VevoIE(InfoExtractor):
         else:
             age_limit = None
 
-        # Download via HLS API
-        formats.extend(self._download_api_formats(video_id, url))
-
-        # Download SMIL
-        smil_blocks = sorted((
-            f for f in video_info.get('videoVersions', [])
-            if f['sourceType'] == 13),
-            key=lambda f: f['version'])
-        smil_url = '%s/Video/V2/VFILE/%s/%sr.smil' % (
-            self._SMIL_BASE_URL, video_id, video_id.lower())
-        if smil_blocks:
-            smil_url_m = self._search_regex(
-                r'url="([^"]+)"', smil_blocks[-1]['data'], 'SMIL URL',
-                default=None)
-            if smil_url_m is not None:
-                smil_url = smil_url_m
-        if smil_url:
-            smil_doc = self._download_smil(smil_url, video_id, fatal=False)
-            if smil_doc:
-                formats.extend(self._formats_from_smil(smil_doc))
-
-        self._sort_formats(formats)
         timestamp = int_or_none(self._search_regex(
             r'/Date\((\d+)\)/',
             video_info['launchDate'], 'launch date', fatal=False),

From 9165d6bab9ae3cc98984ebeb249ef320b9f4db83 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 26 Jan 2016 13:46:58 +0100
Subject: [PATCH 3771/6219] [vevo] extract metadata and formats from api if
 videoinfo is empty

these was fixed by @yan12125 in ff51983e1563db08734e43a07d5671a517f99ef6
i only added some code to extract video metadata and more formats from
api
---
 youtube_dl/extractor/vevo.py | 189 ++++++++++++++++++++++++++---------
 1 file changed, 140 insertions(+), 49 deletions(-)

diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py
index ff854e74b..9c955c895 100644
--- a/youtube_dl/extractor/vevo.py
+++ b/youtube_dl/extractor/vevo.py
@@ -7,6 +7,8 @@ from ..compat import compat_etree_fromstring
 from ..utils import (
     ExtractorError,
     int_or_none,
+    sanitized_Request,
+    parse_iso8601,
 )
 
 
@@ -64,6 +66,7 @@ class VevoIE(InfoExtractor):
             'uploader_id': 'justintimberlakeVEVO',
             'uploader': 'justintimberlakeVEVO',
             'upload_date': '20130703',
+            'duration': 419,
         },
         'params': {
             'skip_download': 'true',
@@ -72,13 +75,18 @@ class VevoIE(InfoExtractor):
     }, {
         'note': 'No video_info',
         'url': 'http://www.vevo.com/watch/k-camp-1/Till-I-Die/USUV71503000',
-        'md5': '8b83cc492d72fc9cf74a02acee7dc1b0',
+        'md5': 'a8b84d1d1957cd01046441b701b270fb',
         'info_dict': {
-            'id': 'USUV71503000',
+            'id': 'Lad2jHtJCqY',
             'ext': 'mp4',
-            'title': 'Till I Die - K Camp ft. T.I.',
+            'title': 'K Camp - Till I Die ft. T.I.',
+            'description': 'md5:0694920ededdee4a14cfc39695cc8ec3',
+            'uploader_id': 'KCampVEVO',
+            'uploader': 'KCampVEVO',
+            'upload_date': '20151207',
             'duration': 193,
         },
+        'add_ie': ['Youtube'],
     }]
     _SMIL_BASE_URL = 'http://smil.lvl3.vevo.com'
     _SOURCE_TYPES = {
@@ -96,7 +104,7 @@ class VevoIE(InfoExtractor):
         18: 'dash',
     }
     _VERSIONS = {
-        0: 'youtube',
+        0: 'youtube',  # only in AuthenticateVideo videoVersions
         1: 'level3',
         2: 'akamai',
         3: 'level3',
@@ -138,14 +146,34 @@ class VevoIE(InfoExtractor):
             })
         return formats
 
+    def _initialize_api(self, video_url, video_id):
+        req = sanitized_Request(
+            'http://www.vevo.com/auth', data=b'')
+        webpage = self._download_webpage(
+            req, None,
+            note='Retrieving oauth token',
+            errnote='Unable to retrieve oauth token')
+
+        if 'THIS PAGE IS CURRENTLY UNAVAILABLE IN YOUR REGION' in webpage:
+            raise ExtractorError('%s said: This page is currently unavailable in your region.' % self.IE_NAME, expected=True)
+
+        auth_info = self._parse_json(webpage, video_id)
+        self._api_url_template = self.http_scheme() + '//apiv2.vevo.com/%s?token=' + auth_info['access_token']
+
+    def _call_api(self, path, video_id, note, errnote, fatal=True):
+        return self._download_json(self._api_url_template % path, video_id, note, errnote)
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        webpage = None
-
         json_url = 'http://videoplayer.vevo.com/VideoService/AuthenticateVideo?isrc=%s' % video_id
-        response = self._download_json(json_url, video_id)
-        video_info = response['video'] or {}
+        response = self._download_json(json_url, video_id, 'Downloading video info', 'Unable to download info')
+        video_info = response.get('video') or {}
+        video_versions = video_info.get('videoVersions')
+        uploader = None
+        timestamp = None
+        view_count = None
+        formats = []
 
         if not video_info:
             ytid = response.get('errorInfo', {}).get('ytid')
@@ -161,44 +189,112 @@ class VevoIE(InfoExtractor):
             if url.startswith('vevo:'):
                 raise ExtractorError(
                     'Please specify full Vevo URL for downloading', expected=True)
-            webpage = self._download_webpage(url, video_id)
 
-        title = video_info.get('title') or self._og_search_title(webpage)
+            self._initialize_api(url, video_id)
+            video_info = self._call_api(
+                'video/%s' % video_id, video_id, 'Downloading api video info',
+                'Failed to download video info')
 
-        smil_parsed = False
-        formats = []
-        for video_version in video_info['videoVersions']:
-            version = self._VERSIONS.get(video_version['version'])
-            if version == 'youtube':
+            ytid = video_info.get('youTubeId')
+            if ytid:
                 return self.url_result(
-                    video_version['id'], 'Youtube', video_version['id'])
-            else:
-                source_type = self._SOURCE_TYPES.get(video_version['sourceType'])
-                renditions = compat_etree_fromstring(video_version['data'])
-                if source_type == 'http':
-                    for rend in renditions.findall('rendition'):
-                        attr = rend.attrib
-                        formats.append({
-                            'url': attr['url'],
-                            'format_id': '%s-%s' % (version, attr['name']),
-                            'height': int_or_none(attr.get('frameheight')),
-                            'width': int_or_none(attr.get('frameWidth')),
-                            'tbr': int_or_none(attr.get('totalBitrate')),
-                            'vbr': int_or_none(attr.get('videoBitrate')),
-                            'abr': int_or_none(attr.get('audioBitrate')),
-                            'vcodec': attr.get('videoCodec'),
-                            'acodec': attr.get('audioCodec'),
-                        })
-                elif source_type == 'hls':
+                    ytid, 'Youtube', ytid)
+
+            video_versions = self._call_api(
+                'video/%s/streams' % video_id, video_id,
+                'Downloading video versions info',
+                'Failed to download video versions info')
+
+            timestamp = parse_iso8601(video_info.get('releaseDate'))
+            artists = video_info.get('artists')
+            if artists:
+                uploader = artists[0]['name']
+            view_count = int_or_none(video_info.get('views', {}).get('total'))
+
+            for video_version in video_versions:
+                version = self._VERSIONS.get(video_version['version'])
+                version_url = video_version.get('url')
+                if not version_url:
+                        continue
+
+                if '.mpd' in version_url or '.ism' in version_url:
+                    continue
+                elif '.m3u8' in version_url:
                     formats.extend(self._extract_m3u8_formats(
-                        renditions.find('rendition').attrib['url'], video_id,
-                        'mp4', 'm3u8_native', m3u8_id='hls-%s' % version, fatal=False))
-                elif source_type == 'smil' and not smil_parsed:
-                    formats.extend(self._extract_smil_formats(
-                        renditions.find('rendition').attrib['url'], video_id, False))
-                    smil_parsed = True
+                        version_url, video_id, 'mp4', 'm3u8_native',
+                        m3u8_id='hls-%s' % version,
+                        note='Downloading %s m3u8 information' % version,
+                        errnote='Failed to download %s m3u8 information' % version,
+                        fatal=False))
+                else:
+                    m = re.search(r'''(?xi)
+                        _(?P<width>[0-9]+)x(?P<height>[0-9]+)
+                        _(?P<vcodec>[a-z0-9]+)
+                        _(?P<vbr>[0-9]+)
+                        _(?P<acodec>[a-z0-9]+)
+                        _(?P<abr>[0-9]+)
+                        \.(?P<ext>[a-z0-9]+)''', version_url)
+                    if not m:
+                        continue
+
+                    formats.append({
+                        'url': version_url,
+                        'format_id': 'http-%s-%s' % (version, video_version['quality']),
+                        'vcodec': m.group('vcodec'),
+                        'acodec': m.group('acodec'),
+                        'vbr': int(m.group('vbr')),
+                        'abr': int(m.group('abr')),
+                        'ext': m.group('ext'),
+                        'width': int(m.group('width')),
+                        'height': int(m.group('height')),
+                    })
+        else:
+            timestamp = int_or_none(self._search_regex(
+                r'/Date\((\d+)\)/',
+                video_info['releaseDate'], 'release date', fatal=False),
+                scale=1000)
+            artists = video_info.get('mainArtists')
+            if artists:
+                uploader = artists[0]['artistName']
+
+            smil_parsed = False
+            for video_version in video_info['videoVersions']:
+                version = self._VERSIONS.get(video_version['version'])
+                if version == 'youtube':
+                    return self.url_result(
+                        video_version['id'], 'Youtube', video_version['id'])
+                else:
+                    source_type = self._SOURCE_TYPES.get(video_version['sourceType'])
+                    renditions = compat_etree_fromstring(video_version['data'])
+                    if source_type == 'http':
+                        for rend in renditions.findall('rendition'):
+                            attr = rend.attrib
+                            formats.append({
+                                'url': attr['url'],
+                                'format_id': 'http-%s-%s' % (version, attr['name']),
+                                'height': int_or_none(attr.get('frameheight')),
+                                'width': int_or_none(attr.get('frameWidth')),
+                                'tbr': int_or_none(attr.get('totalBitrate')),
+                                'vbr': int_or_none(attr.get('videoBitrate')),
+                                'abr': int_or_none(attr.get('audioBitrate')),
+                                'vcodec': attr.get('videoCodec'),
+                                'acodec': attr.get('audioCodec'),
+                            })
+                    elif source_type == 'hls':
+                        formats.extend(self._extract_m3u8_formats(
+                            renditions.find('rendition').attrib['url'], video_id,
+                            'mp4', 'm3u8_native', m3u8_id='hls-%s' % version,
+                            note='Downloading %s m3u8 information' % version,
+                            errnote='Failed to download %s m3u8 information' % version,
+                            fatal=False))
+                    elif source_type == 'smil' and not smil_parsed:
+                        formats.extend(self._extract_smil_formats(
+                            renditions.find('rendition').attrib['url'], video_id, False))
+                        smil_parsed = True
         self._sort_formats(formats)
 
+        title = video_info['title']
+
         is_explicit = video_info.get('isExplicit')
         if is_explicit is True:
             age_limit = 18
@@ -207,21 +303,16 @@ class VevoIE(InfoExtractor):
         else:
             age_limit = None
 
-        timestamp = int_or_none(self._search_regex(
-            r'/Date\((\d+)\)/',
-            video_info['launchDate'], 'launch date', fatal=False),
-            scale=1000) if video_info else None
-
-        duration = video_info.get('duration') or int_or_none(
-            self._html_search_meta('video:duration', webpage))
+        duration = video_info.get('duration')
 
         return {
             'id': video_id,
             'title': title,
             'formats': formats,
-            'thumbnail': video_info.get('imageUrl'),
+            'thumbnail': video_info.get('imageUrl') or video_info.get('thumbnailUrl'),
             'timestamp': timestamp,
-            'uploader': video_info['mainArtists'][0]['artistName'] if video_info else None,
+            'uploader': uploader,
             'duration': duration,
+            'view_count': view_count,
             'age_limit': age_limit,
         }

From f693213567010ecc108447cba4615ae2932d1c18 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 26 Jan 2016 20:42:20 +0600
Subject: [PATCH 3772/6219] [cspan] Fix clip/prog id extraction (#8317)

---
 youtube_dl/extractor/cspan.py | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/cspan.py b/youtube_dl/extractor/cspan.py
index b3ee67018..f78cbbd7e 100644
--- a/youtube_dl/extractor/cspan.py
+++ b/youtube_dl/extractor/cspan.py
@@ -68,11 +68,16 @@ class CSpanIE(InfoExtractor):
             video_type, video_id = matches.groups()
             video_type = 'clip' if video_type == 'id' else 'program'
         else:
-            senate_isvp_url = SenateISVPIE._search_iframe_url(webpage)
-            if senate_isvp_url:
-                title = self._og_search_title(webpage)
-                surl = smuggle_url(senate_isvp_url, {'force_title': title})
-                return self.url_result(surl, 'SenateISVP', video_id, title)
+            m = re.search(r'data-(?P<type>clip|prog)id=["\'](?P<id>\d+)', webpage)
+            if m:
+                video_id = m.group('id')
+                video_type = 'program' if m.group('type') == 'prog' else 'clip'
+            else:
+                senate_isvp_url = SenateISVPIE._search_iframe_url(webpage)
+                if senate_isvp_url:
+                    title = self._og_search_title(webpage)
+                    surl = smuggle_url(senate_isvp_url, {'force_title': title})
+                    return self.url_result(surl, 'SenateISVP', video_id, title)
         if video_type is None or video_id is None:
             raise ExtractorError('unable to find video id and type')
 

From 682f8c43b50d8b5e2c02e34187cbddb0d5f8d3ed Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 26 Jan 2016 15:54:32 +0100
Subject: [PATCH 3773/6219] [vevo] fallback to youtube video only if vevo video
 is geo restricted(fixes 8263)(fixes 2874)

---
 youtube_dl/extractor/vevo.py | 93 ++++++++++++++----------------------
 1 file changed, 36 insertions(+), 57 deletions(-)

diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py
index 9c955c895..35fcff1b2 100644
--- a/youtube_dl/extractor/vevo.py
+++ b/youtube_dl/extractor/vevo.py
@@ -26,67 +26,52 @@ class VevoIE(InfoExtractor):
 
     _TESTS = [{
         'url': 'http://www.vevo.com/watch/hurts/somebody-to-die-for/GB1101300280',
-        'md5': '2dbc7e9fd4f1c60436c9aa73a5406193',
+        'md5': '95ee28ee45e70130e3ab02b0f579ae23',
         'info_dict': {
-            'id': 'Pt1kc_FniKM',
+            'id': 'GB1101300280',
             'ext': 'mp4',
-            'title': 'Hurts - Somebody to Die For',
-            'description': 'md5:13e925b89af6b01c7e417332bd23c4bf',
-            'uploader_id': 'HurtsVEVO',
-            'uploader': 'HurtsVEVO',
+            'title': 'Somebody to Die For',
             'upload_date': '20130624',
-            'duration': 230,
+            'uploader': 'Hurts',
+            'timestamp': 1372057200,
         },
-        'add_ie': ['Youtube'],
     }, {
         'note': 'v3 SMIL format',
         'url': 'http://www.vevo.com/watch/cassadee-pope/i-wish-i-could-break-your-heart/USUV71302923',
-        'md5': '13d5204f520af905eeffa675040b8e76',
+        'md5': 'f6ab09b034f8c22969020b042e5ac7fc',
         'info_dict': {
-            'id': 'ByGmQn1uxJw',
+            'id': 'USUV71302923',
             'ext': 'mp4',
-            'title': 'Cassadee Pope - I Wish I Could Break Your Heart',
-            'description': 'md5:5e9721c92ef117a6f69d00e9b42ceba7',
-            'uploader_id': 'CassadeeVEVO',
-            'uploader': 'CassadeeVEVO',
+            'title': 'I Wish I Could Break Your Heart',
             'upload_date': '20140219',
-            'duration': 226,
-            'age_limit': 0,
+            'uploader': 'Cassadee Pope',
+            'timestamp': 1392796919,
         },
-        'add_ie': ['Youtube'],
     }, {
         'note': 'Age-limited video',
         'url': 'https://www.vevo.com/watch/justin-timberlake/tunnel-vision-explicit/USRV81300282',
         'info_dict': {
-            'id': '07FYdnEawAQ',
+            'id': 'USRV81300282',
             'ext': 'mp4',
-            'age_limit': 18,
-            'title': 'Justin Timberlake - Tunnel Vision (Explicit)',
-            'description': 'md5:64249768eec3bc4276236606ea996373',
-            'uploader_id': 'justintimberlakeVEVO',
-            'uploader': 'justintimberlakeVEVO',
+            'title': 'Tunnel Vision (Explicit)',
             'upload_date': '20130703',
-            'duration': 419,
+            'age_limit': 18,
+            'uploader': 'Justin Timberlake',
+            'timestamp': 1372888800,
         },
-        'params': {
-            'skip_download': 'true',
-        },
-        'add_ie': ['Youtube'],
     }, {
         'note': 'No video_info',
         'url': 'http://www.vevo.com/watch/k-camp-1/Till-I-Die/USUV71503000',
-        'md5': 'a8b84d1d1957cd01046441b701b270fb',
+        'md5': '8b83cc492d72fc9cf74a02acee7dc1b0',
         'info_dict': {
-            'id': 'Lad2jHtJCqY',
+            'id': 'USUV71503000',
             'ext': 'mp4',
-            'title': 'K Camp - Till I Die ft. T.I.',
-            'description': 'md5:0694920ededdee4a14cfc39695cc8ec3',
-            'uploader_id': 'KCampVEVO',
-            'uploader': 'KCampVEVO',
+            'title': 'Till I Die',
             'upload_date': '20151207',
-            'duration': 193,
+            'age_limit': 18,
+            'uploader': 'K Camp',
+            'timestamp': 1449468000,
         },
-        'add_ie': ['Youtube'],
     }]
     _SMIL_BASE_URL = 'http://smil.lvl3.vevo.com'
     _SOURCE_TYPES = {
@@ -146,7 +131,7 @@ class VevoIE(InfoExtractor):
             })
         return formats
 
-    def _initialize_api(self, video_url, video_id):
+    def _initialize_api(self, video_id):
         req = sanitized_Request(
             'http://www.vevo.com/auth', data=b'')
         webpage = self._download_webpage(
@@ -155,7 +140,8 @@ class VevoIE(InfoExtractor):
             errnote='Unable to retrieve oauth token')
 
         if 'THIS PAGE IS CURRENTLY UNAVAILABLE IN YOUR REGION' in webpage:
-            raise ExtractorError('%s said: This page is currently unavailable in your region.' % self.IE_NAME, expected=True)
+            raise ExtractorError(
+                '%s said: This page is currently unavailable in your region.' % self.IE_NAME, expected=True)
 
         auth_info = self._parse_json(webpage, video_id)
         self._api_url_template = self.http_scheme() + '//apiv2.vevo.com/%s?token=' + auth_info['access_token']
@@ -167,7 +153,8 @@ class VevoIE(InfoExtractor):
         video_id = self._match_id(url)
 
         json_url = 'http://videoplayer.vevo.com/VideoService/AuthenticateVideo?isrc=%s' % video_id
-        response = self._download_json(json_url, video_id, 'Downloading video info', 'Unable to download info')
+        response = self._download_json(
+            json_url, video_id, 'Downloading video info', 'Unable to download info')
         video_info = response.get('video') or {}
         video_versions = video_info.get('videoVersions')
         uploader = None
@@ -176,30 +163,23 @@ class VevoIE(InfoExtractor):
         formats = []
 
         if not video_info:
-            ytid = response.get('errorInfo', {}).get('ytid')
-            if ytid:
-                return self.url_result(ytid, 'Youtube', ytid)
-
             if response.get('statusCode') != 909:
+                ytid = response.get('errorInfo', {}).get('ytid')
+                if ytid:
+                    self.report_warning(
+                        'Video is geoblocked, trying with the YouTube video %s' % ytid)
+                    return self.url_result(ytid, 'Youtube', ytid)
+
                 if 'statusMessage' in response:
                     raise ExtractorError('%s said: %s' % (
                         self.IE_NAME, response['statusMessage']), expected=True)
                 raise ExtractorError('Unable to extract videos')
 
-            if url.startswith('vevo:'):
-                raise ExtractorError(
-                    'Please specify full Vevo URL for downloading', expected=True)
-
-            self._initialize_api(url, video_id)
+            self._initialize_api(video_id)
             video_info = self._call_api(
                 'video/%s' % video_id, video_id, 'Downloading api video info',
                 'Failed to download video info')
 
-            ytid = video_info.get('youTubeId')
-            if ytid:
-                return self.url_result(
-                    ytid, 'Youtube', ytid)
-
             video_versions = self._call_api(
                 'video/%s/streams' % video_id, video_id,
                 'Downloading video versions info',
@@ -215,7 +195,7 @@ class VevoIE(InfoExtractor):
                 version = self._VERSIONS.get(video_version['version'])
                 version_url = video_version.get('url')
                 if not version_url:
-                        continue
+                    continue
 
                 if '.mpd' in version_url or '.ism' in version_url:
                     continue
@@ -261,8 +241,7 @@ class VevoIE(InfoExtractor):
             for video_version in video_info['videoVersions']:
                 version = self._VERSIONS.get(video_version['version'])
                 if version == 'youtube':
-                    return self.url_result(
-                        video_version['id'], 'Youtube', video_version['id'])
+                    continue
                 else:
                     source_type = self._SOURCE_TYPES.get(video_version['sourceType'])
                     renditions = compat_etree_fromstring(video_version['data'])
@@ -287,7 +266,7 @@ class VevoIE(InfoExtractor):
                             note='Downloading %s m3u8 information' % version,
                             errnote='Failed to download %s m3u8 information' % version,
                             fatal=False))
-                    elif source_type == 'smil' and not smil_parsed:
+                    elif source_type == 'smil' and version == 'level3' and not smil_parsed:
                         formats.extend(self._extract_smil_formats(
                             renditions.find('rendition').attrib['url'], video_id, False))
                         smil_parsed = True

From af9c2a07aea530b3bee560a953e94ac92fcd49c9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 26 Jan 2016 21:29:42 +0600
Subject: [PATCH 3774/6219] [cspan] Extract from path when no qualities (Closes
 #8317)

---
 youtube_dl/extractor/cspan.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/youtube_dl/extractor/cspan.py b/youtube_dl/extractor/cspan.py
index f78cbbd7e..b78edf729 100644
--- a/youtube_dl/extractor/cspan.py
+++ b/youtube_dl/extractor/cspan.py
@@ -112,6 +112,13 @@ class CSpanIE(InfoExtractor):
                     'height': int_or_none(get_text_attr(quality, 'height')),
                     'tbr': int_or_none(get_text_attr(quality, 'bitrate')),
                 })
+            if not formats:
+                path = get_text_attr(f, 'path')
+                if not path:
+                    continue
+                formats = self._extract_m3u8_formats(
+                    path, video_id, 'mp4', entry_protocol='m3u8_native',
+                    m3u8_id='hls') if determine_ext(path) == 'm3u8' else [{'url': path, }]
             self._sort_formats(formats)
             entries.append({
                 'id': '%s_%d' % (video_id, partnum + 1),

From fb4b3458000bff87f2b083f9a5e6853c043d032d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 26 Jan 2016 21:46:51 +0600
Subject: [PATCH 3775/6219] [instagram] Make description optional (Closes
 #8326)

---
 youtube_dl/extractor/instagram.py | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/instagram.py b/youtube_dl/extractor/instagram.py
index e5e16ca3b..ed3e07118 100644
--- a/youtube_dl/extractor/instagram.py
+++ b/youtube_dl/extractor/instagram.py
@@ -21,6 +21,18 @@ class InstagramIE(InfoExtractor):
             'title': 'Video by naomipq',
             'description': 'md5:1f17f0ab29bd6fe2bfad705f58de3cb8',
         }
+    }, {
+        # missing description
+        'url': 'https://www.instagram.com/p/BA-pQFBG8HZ/?taken-by=britneyspears',
+        'info_dict': {
+            'id': 'BA-pQFBG8HZ',
+            'ext': 'mp4',
+            'uploader_id': 'britneyspears',
+            'title': 'Video by britneyspears',
+        },
+        'params': {
+            'skip_download': True,
+        },
     }, {
         'url': 'https://instagram.com/p/-Cmh1cukG2/',
         'only_matching': True,
@@ -32,8 +44,8 @@ class InstagramIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
         uploader_id = self._search_regex(r'"owner":{"username":"(.+?)"',
                                          webpage, 'uploader id', fatal=False)
-        desc = self._search_regex(r'"caption":"(.*?)"', webpage, 'description',
-                                  fatal=False)
+        desc = self._search_regex(
+            r'"caption":"(.+?)"', webpage, 'description', default=None)
 
         return {
             'id': video_id,

From b6c33fd544adddf399a80842eb9a7d2250aa04b6 Mon Sep 17 00:00:00 2001
From: ping <liping.ong@gmail.com>
Date: Wed, 27 Jan 2016 12:48:00 +0800
Subject: [PATCH 3776/6219] [daum.net] Fixes #8331

---
 youtube_dl/extractor/daum.py | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/daum.py b/youtube_dl/extractor/daum.py
index f08f57157..c84302c43 100644
--- a/youtube_dl/extractor/daum.py
+++ b/youtube_dl/extractor/daum.py
@@ -2,6 +2,8 @@
 
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..compat import compat_urllib_parse
 from ..utils import (
@@ -28,6 +30,15 @@ class DaumIE(InfoExtractor):
             'comment_count': int,
         },
     }, {
+        'url': 'http://m.tvpot.daum.net/v/65139429',
+        'info_dict': {
+            'id': '65139429',
+            'ext': 'mp4',
+            'title': 'md5:a100d65d09cec246d8aa9bde7de45aed',
+            'description': 'md5:79794514261164ff27e36a21ad229fc5',
+            'upload_date': '20150604',
+            'duration': 154
+        },    }, {
         'url': 'http://tvpot.daum.net/v/07dXWRka62Y%24',
         'only_matching': True,
     }]
@@ -42,6 +53,10 @@ class DaumIE(InfoExtractor):
             'http://videofarm.daum.net/controller/api/closed/v1_2/IntegratedMovieData.json?' + query,
             video_id, 'Downloading video formats info')
 
+        # For urls like http://m.tvpot.daum.net/v/65139429, where the video_id is really a clipid
+        if not movie_data.get('output_list', {}).get('output_list') and re.match(r'^\d+$', video_id):
+            return self.url_result('http://tvpot.daum.net/clip/ClipView.do?clipid=%s' % video_id)
+
         formats = []
         for format_el in movie_data['output_list']['output_list']:
             profile = format_el['profile']
@@ -76,7 +91,7 @@ class DaumIE(InfoExtractor):
 
 
 class DaumClipIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:m\.)?tvpot\.daum\.net/(?:clip/ClipView.do|mypot/View.do)\?.*?clipid=(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:m\.)?tvpot\.daum\.net/(?:clip/ClipView.(?:do|tv)|mypot/View.do)\?.*?clipid=(?P<id>\d+)'
     IE_NAME = 'daum.net:clip'
 
     _TESTS = [{
@@ -90,6 +105,9 @@ class DaumClipIE(InfoExtractor):
             'duration': 3868,
             'view_count': int,
         },
+    }, {
+        'url': 'http://m.tvpot.daum.net/clip/ClipView.tv?clipid=54999425',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From fab6f0e65b0084ce8b29c0810b874d132ffcd47c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 27 Jan 2016 08:32:03 +0100
Subject: [PATCH 3777/6219] release 2016.01.27

---
 docs/supportedsites.md | 1 +
 youtube_dl/version.py  | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index e86467cfa..eb68c23b5 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -180,6 +180,7 @@
  - **ExpoTV**
  - **ExtremeTube**
  - **facebook**
+ - **facebook:post**
  - **faz.net**
  - **fc2**
  - **Fczenit**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index d5bf73815..4ac7f9e93 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.01.23'
+__version__ = '2016.01.27'

From b0d21deda95e7e49e162c12aa305f27e477e0ce9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 27 Jan 2016 21:11:17 +0600
Subject: [PATCH 3778/6219] [extractor/common] Auto calculate tbr when missing

---
 youtube_dl/extractor/common.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 2f574054d..f5a358388 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -825,6 +825,12 @@ class InfoExtractor(object):
         if not formats:
             raise ExtractorError('No video formats found')
 
+        for f in formats:
+            # Automatically determine tbr when missing based on abr and vbr (improves
+            # formats sorting in some cases)
+            if 'tbr' not in f and 'abr' in f and 'vbr' in f:
+                f['tbr'] = f['abr'] + f['vbr']
+
         def _formats_key(f):
             # TODO remove the following workaround
             from ..utils import determine_ext

From 9339774af274b04a054b0fa151347619e40b5b21 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 27 Jan 2016 21:49:39 +0600
Subject: [PATCH 3779/6219] [spankbang] Fix formats extraction

---
 youtube_dl/extractor/spankbang.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/spankbang.py b/youtube_dl/extractor/spankbang.py
index 7f060b15b..a67db5905 100644
--- a/youtube_dl/extractor/spankbang.py
+++ b/youtube_dl/extractor/spankbang.py
@@ -34,7 +34,7 @@ class SpankBangIE(InfoExtractor):
             'ext': 'mp4',
             'format_id': '%sp' % height,
             'height': int(height),
-        } for height in re.findall(r'<span[^>]+q_(\d+)p', webpage)]
+        } for height in re.findall(r'<(?:span|li)[^>]+q_(\d+)p', webpage)]
         self._sort_formats(formats)
 
         title = self._html_search_regex(

From c3111ab34fed3af16f645c6ee4dcd6e5d35b69dd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 27 Jan 2016 21:49:56 +0600
Subject: [PATCH 3780/6219] [spankbang] Fix title extraction (Closes #8329)

---
 youtube_dl/extractor/spankbang.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/spankbang.py b/youtube_dl/extractor/spankbang.py
index a67db5905..3cfa671ed 100644
--- a/youtube_dl/extractor/spankbang.py
+++ b/youtube_dl/extractor/spankbang.py
@@ -38,7 +38,7 @@ class SpankBangIE(InfoExtractor):
         self._sort_formats(formats)
 
         title = self._html_search_regex(
-            r'(?s)<h1>(.+?)</h1>', webpage, 'title')
+            r'(?s)<h1[^>]*>(.+?)</h1>', webpage, 'title')
         description = self._search_regex(
             r'class="desc"[^>]*>([^<]+)',
             webpage, 'description', default=None)

From 7f32e5dc350e2a641e6304855727623f95bef31c Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 26 Jan 2016 17:44:44 +0100
Subject: [PATCH 3781/6219] [extractor/common] detect media playlist in
 _extract_m3u8_formats

---
 youtube_dl/extractor/common.py | 26 ++++++++++++++++++++++++--
 1 file changed, 24 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 2f574054d..11191c173 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1014,6 +1014,18 @@ class InfoExtractor(object):
             return []
         m3u8_doc, urlh = res
         m3u8_url = urlh.geturl()
+        # A Media Playlist Tag MUST NOT appear in a Master Playlist
+        # https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.3
+        # The EXT-X-TARGETDURATION tag is REQUIRED for every M3U8 Media Playlists
+        # https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.3.1
+        if '#EXT-X-TARGETDURATION' in m3u8_doc:
+            return [{
+                'url': m3u8_url,
+                'format_id': m3u8_id,
+                'ext': ext,
+                'protocol': entry_protocol,
+                'preference': preference,
+            }]
         last_info = None
         last_media = None
         kv_rex = re.compile(
@@ -1164,6 +1176,7 @@ class InfoExtractor(object):
         formats = []
         rtmp_count = 0
         http_count = 0
+        m3u8_count = 0
 
         videos = smil.findall(self._xpath_ns('.//video', namespace))
         for video in videos:
@@ -1203,8 +1216,17 @@ class InfoExtractor(object):
             src_url = src if src.startswith('http') else compat_urlparse.urljoin(base, src)
 
             if proto == 'm3u8' or src_ext == 'm3u8':
-                formats.extend(self._extract_m3u8_formats(
-                    src_url, video_id, ext or 'mp4', m3u8_id='hls', fatal=False))
+                m3u8_formats = self._extract_m3u8_formats(
+                    src_url, video_id, ext or 'mp4', m3u8_id='hls', fatal=False)
+                if len(m3u8_formats) == 1:
+                    m3u8_count += 1
+                    m3u8_formats[0].update({
+                        'format_id': 'hls-%d' % (m3u8_count if bitrate is None else bitrate),
+                        'tbr': bitrate,
+                        'width': width,
+                        'height': height,
+                    })
+                formats.extend(m3u8_formats)
                 continue
 
             if src_ext == 'f4m':

From f125d9115b262ccead54a31c4e64b0c71ad65721 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 27 Jan 2016 19:10:34 +0100
Subject: [PATCH 3782/6219] [cbsnews] extract all formats

---
 youtube_dl/extractor/cbsnews.py | 46 +++++++++++----------------------
 1 file changed, 15 insertions(+), 31 deletions(-)

diff --git a/youtube_dl/extractor/cbsnews.py b/youtube_dl/extractor/cbsnews.py
index 480435e26..6db66e886 100644
--- a/youtube_dl/extractor/cbsnews.py
+++ b/youtube_dl/extractor/cbsnews.py
@@ -4,11 +4,11 @@ from __future__ import unicode_literals
 import re
 import json
 
-from .common import InfoExtractor
+from .theplatform import ThePlatformIE
 from ..utils import remove_start
 
 
-class CBSNewsIE(InfoExtractor):
+class CBSNewsIE(ThePlatformIE):
     IE_DESC = 'CBS News'
     _VALID_URL = r'http://(?:www\.)?cbsnews\.com/(?:[^/]+/)+(?P<id>[\da-z_-]+)'
 
@@ -31,7 +31,7 @@ class CBSNewsIE(InfoExtractor):
             'url': 'http://www.cbsnews.com/videos/fort-hood-shooting-army-downplays-mental-illness-as-cause-of-attack/',
             'info_dict': {
                 'id': 'fort-hood-shooting-army-downplays-mental-illness-as-cause-of-attack',
-                'ext': 'flv',
+                'ext': 'mp4',
                 'title': 'Fort Hood shooting: Army downplays mental illness as cause of attack',
                 'thumbnail': 're:^https?://.*\.jpg$',
                 'duration': 205,
@@ -42,7 +42,7 @@ class CBSNewsIE(InfoExtractor):
                 },
             },
             'params': {
-                # rtmp download
+                # m3u8 download
                 'skip_download': True,
             },
         },
@@ -63,33 +63,6 @@ class CBSNewsIE(InfoExtractor):
         duration = item.get('duration')
         thumbnail = item.get('mediaImage') or item.get('thumbnail')
 
-        formats = []
-        for format_id in ['RtmpMobileLow', 'RtmpMobileHigh', 'Hls', 'RtmpDesktop']:
-            uri = item.get('media' + format_id + 'URI')
-            if not uri:
-                continue
-            uri = remove_start(uri, '{manifest:none}')
-            fmt = {
-                'url': uri,
-                'format_id': format_id,
-            }
-            if uri.startswith('rtmp'):
-                play_path = re.sub(
-                    r'{slistFilePath}', '',
-                    uri.split('<break>')[-1].split('{break}')[-1])
-                play_path = re.sub(
-                    r'{manifest:.+}.*$', '', play_path)
-                fmt.update({
-                    'app': 'ondemand?auth=cbs',
-                    'play_path': 'mp4:' + play_path,
-                    'player_url': 'http://www.cbsnews.com/[[IMPORT]]/vidtech.cbsinteractive.com/player/3_3_0/CBSI_PLAYER_HD.swf',
-                    'page_url': 'http://www.cbsnews.com',
-                    'ext': 'flv',
-                })
-            elif uri.endswith('.m3u8'):
-                fmt['ext'] = 'mp4'
-            formats.append(fmt)
-
         subtitles = {}
         if 'mpxRefId' in video_info:
             subtitles['en'] = [{
@@ -97,6 +70,17 @@ class CBSNewsIE(InfoExtractor):
                 'url': 'http://www.cbsnews.com/videos/captions/%s.adb_xml' % video_info['mpxRefId'],
             }]
 
+        formats = []
+        for format_id in ['RtmpMobileLow', 'RtmpMobileHigh', 'Hls', 'RtmpDesktop']:
+            pid = item.get('media' + format_id)
+            if not pid:
+                continue
+            release_url = 'http://link.theplatform.com/s/dJ5BDC/%s?format=SMIL&mbr=true' % pid
+            tp_formats, tp_subtitles = self._extract_theplatform_smil(release_url, video_id, 'Downloading %s SMIL data' % pid)
+            formats.extend(tp_formats)
+            subtitles = self._merge_subtitles(subtitles, tp_subtitles)
+        self._sort_formats(formats)
+
         return {
             'id': video_id,
             'title': title,

From ed7cd1e859cf97e975a28a5e8c58a1d1aca819fe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 28 Jan 2016 00:42:04 +0600
Subject: [PATCH 3783/6219] [cbsnews] Remove unused import

---
 youtube_dl/extractor/cbsnews.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/cbsnews.py b/youtube_dl/extractor/cbsnews.py
index 6db66e886..cabf7e73b 100644
--- a/youtube_dl/extractor/cbsnews.py
+++ b/youtube_dl/extractor/cbsnews.py
@@ -5,7 +5,6 @@ import re
 import json
 
 from .theplatform import ThePlatformIE
-from ..utils import remove_start
 
 
 class CBSNewsIE(ThePlatformIE):

From b0df5223be5429da0378841dc33eecb532f6898f Mon Sep 17 00:00:00 2001
From: dyn888 <dyn.8.8.8+github@gmail.com>
Date: Thu, 28 Jan 2016 12:07:15 +0100
Subject: [PATCH 3784/6219] Update YoutubeDL.py

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 09d2b18f2..e1bd40843 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -906,7 +906,7 @@ class YoutubeDL(object):
             str_operator_rex = re.compile(r'''(?x)
                 \s*(?P<key>ext|acodec|vcodec|container|protocol)
                 \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?
-                \s*(?P<value>[a-zA-Z0-9_-]+)
+                \s*(?P<value>[a-zA-Z0-9._-]+)
                 \s*$
                 ''' % '|'.join(map(re.escape, STR_OPERATORS.keys())))
             m = str_operator_rex.search(filter_spec)

From 2b14cb566fde3e5482ce9a63b2be7103cec939e0 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 28 Jan 2016 12:38:34 +0100
Subject: [PATCH 3785/6219] [utils] fix dfxp2srt text extraction(fixes #8055)

---
 youtube_dl/utils.py | 33 ++++++++++++++++++++-------------
 1 file changed, 20 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index c63b61598..18dbe28bb 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2017,20 +2017,27 @@ def dfxp2srt(dfxp_data):
         'ttaf1': 'http://www.w3.org/2006/10/ttaf1',
     })
 
+    class TTMLPElementParser:
+        out = ''
+
+        def start(self, tag, attrib):
+            if tag in (_x('ttml:br'), _x('ttaf1:br'), 'br'):
+                self.out += '\n'
+
+        def end(self, tag):
+            pass
+
+        def data(self, data):
+            self.out += data
+
+        def close(self):
+            return self.out.strip()
+
     def parse_node(node):
-        str_or_empty = functools.partial(str_or_none, default='')
-
-        out = str_or_empty(node.text)
-
-        for child in node:
-            if child.tag in (_x('ttml:br'), _x('ttaf1:br'), 'br'):
-                out += '\n' + str_or_empty(child.tail)
-            elif child.tag in (_x('ttml:span'), _x('ttaf1:span'), 'span'):
-                out += str_or_empty(parse_node(child))
-            else:
-                out += str_or_empty(xml.etree.ElementTree.tostring(child))
-
-        return out
+        target = TTMLPElementParser()
+        parser = xml.etree.ElementTree.XMLParser(target=target)
+        parser.feed(xml.etree.ElementTree.tostring(node))
+        return parser.close()
 
     dfxp = compat_etree_fromstring(dfxp_data.encode('utf-8'))
     out = []

From b913348d5fa1130ab1c65576e458407f262df6e1 Mon Sep 17 00:00:00 2001
From: dyn888 <dyn.8.8.8+github@gmail.com>
Date: Thu, 28 Jan 2016 15:07:33 +0100
Subject: [PATCH 3786/6219] Test codec with a dot '.' in name selection.

---
 test/test_YoutubeDL.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 0caa43843..b53cfbe78 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -221,6 +221,16 @@ class TestFormatSelection(unittest.TestCase):
         downloaded = ydl.downloaded_info_dicts[0]
         self.assertEqual(downloaded['format_id'], 'dash-video-low')
 
+        formats = [
+            {'format_id': 'vid-vcodec-dot', 'ext': 'mp4', 'preference': 1, 'vcodec': 'avc1.123456', 'acodec': 'none', 'url': TEST_URL},
+        ]
+        info_dict = _make_result(formats)
+
+        ydl = YDL({'format': 'bestvideo[vcodec=avc1.123456]'})
+        ydl.process_ie_result(info_dict.copy())
+        downloaded = ydl.downloaded_info_dicts[0]
+        self.assertEqual(downloaded['format_id'], 'vid-vcodec-dot')
+
     def test_youtube_format_selection(self):
         order = [
             '38', '37', '46', '22', '45', '35', '44', '18', '34', '43', '6', '5', '36', '17', '13',

From 29f46c2bee3641efcf2c06c3d173baf7a41010af Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 28 Jan 2016 22:56:00 +0800
Subject: [PATCH 3787/6219] Credit @dyn888 for improving format selection

[ci skip]
---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index bb1f2d8d9..a46799506 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -155,3 +155,4 @@ Vignesh Venkat
 Tom Gijselinck
 Founder Fang
 Andrew Alexeyew
+Saso Bezlaj

From 38c84acae512f1b218e5146f4b2ddcac6b1a5e13 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 28 Jan 2016 22:50:18 +0600
Subject: [PATCH 3788/6219] [ndr:embed:base] Add missing ext for m3u8

---
 youtube_dl/extractor/ndr.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ndr.py b/youtube_dl/extractor/ndr.py
index 894c51399..0cded6b5c 100644
--- a/youtube_dl/extractor/ndr.py
+++ b/youtube_dl/extractor/ndr.py
@@ -193,7 +193,7 @@ class NDREmbedBaseIE(InfoExtractor):
                     src + '?hdcore=3.7.0&plugin=aasp-3.7.0.39.44', video_id, f4m_id='hds'))
             elif ext == 'm3u8':
                 formats.extend(self._extract_m3u8_formats(
-                    src, video_id, m3u8_id='hls', entry_protocol='m3u8_native'))
+                    src, video_id, 'mp4', m3u8_id='hls', entry_protocol='m3u8_native'))
             else:
                 quality = f.get('quality')
                 ff = {

From 10e6ed93417fb51c606af2e3e47b9a5a094dd6ee Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 28 Jan 2016 22:56:49 +0600
Subject: [PATCH 3789/6219] [ok] Add support for mobile URLs (Closes #8345)

---
 youtube_dl/extractor/odnoklassniki.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/odnoklassniki.py b/youtube_dl/extractor/odnoklassniki.py
index 184c7a323..f9e064a60 100644
--- a/youtube_dl/extractor/odnoklassniki.py
+++ b/youtube_dl/extractor/odnoklassniki.py
@@ -13,7 +13,7 @@ from ..utils import (
 
 
 class OdnoklassnikiIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?(?:odnoklassniki|ok)\.ru/(?:video(?:embed)?|web-api/video/moviePlayer)/(?P<id>[\d-]+)'
+    _VALID_URL = r'https?://(?:(?:www|m|mobile)\.)?(?:odnoklassniki|ok)\.ru/(?:video(?:embed)?|web-api/video/moviePlayer)/(?P<id>[\d-]+)'
     _TESTS = [{
         # metadata in JSON
         'url': 'http://ok.ru/video/20079905452',
@@ -69,6 +69,12 @@ class OdnoklassnikiIE(InfoExtractor):
     }, {
         'url': 'http://www.ok.ru/videoembed/20648036891',
         'only_matching': True,
+    }, {
+        'url': 'http://m.ok.ru/video/20079905452',
+        'only_matching': True,
+    }, {
+        'url': 'http://mobile.ok.ru/video/20079905452',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 50e989e2636fc59ed896cc021b1b594bd10e9e17 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 28 Jan 2016 23:19:53 +0600
Subject: [PATCH 3790/6219] [bbc] Add another title regex (Closes #8340)

---
 youtube_dl/extractor/bbc.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 1c493b72d..210ce568b 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -482,7 +482,8 @@ class BBCCoUkIE(InfoExtractor):
         if programme_id:
             formats, subtitles = self._download_media_selector(programme_id)
             title = self._og_search_title(webpage, default=None) or self._html_search_regex(
-                r'<h2[^>]+id="parent-title"[^>]*>(.+?)</h2>', webpage, 'title')
+                (r'<h2[^>]+id="parent-title"[^>]*>(.+?)</h2>',
+                 r'<div[^>]+class="info"[^>]*>\s*<h1>(.+?)</h1>'), webpage, 'title')
             description = self._search_regex(
                 r'<p class="[^"]*medium-description[^"]*">([^<]+)</p>',
                 webpage, 'description', default=None)

From a853427427269549593a0217db203305332ac983 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 28 Jan 2016 23:23:13 +0600
Subject: [PATCH 3791/6219] [bbc] Add another description regex

---
 youtube_dl/extractor/bbc.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 210ce568b..5be0ff476 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -485,7 +485,8 @@ class BBCCoUkIE(InfoExtractor):
                 (r'<h2[^>]+id="parent-title"[^>]*>(.+?)</h2>',
                  r'<div[^>]+class="info"[^>]*>\s*<h1>(.+?)</h1>'), webpage, 'title')
             description = self._search_regex(
-                r'<p class="[^"]*medium-description[^"]*">([^<]+)</p>',
+                (r'<p class="[^"]*medium-description[^"]*">([^<]+)</p>',
+                 r'<div[^>]+class="info_+synopsis"[^>]*>([^<]+)</div>'),
                 webpage, 'description', default=None)
             if not description:
                 description = self._html_search_meta('description', webpage)

From 1ac6e794cb36af612db97007006fc7cf1468e049 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 28 Jan 2016 23:27:48 +0600
Subject: [PATCH 3792/6219] [bbc] Add test for #8147

---
 youtube_dl/extractor/bbc.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 5be0ff476..6ddee686c 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -193,6 +193,19 @@ class BBCCoUkIE(InfoExtractor):
                 # rtmp download
                 'skip_download': True,
             },
+        }, {
+            # compact player (https://github.com/rg3/youtube-dl/issues/8147)
+            'url': 'http://www.bbc.co.uk/programmes/p028bfkf/player',
+            'info_dict': {
+                'id': 'p028bfkj',
+                'ext': 'flv',
+                'title': 'Extract from BBC documentary Look Stranger - Giant Leeks and Magic Brews',
+                'description': 'Extract from BBC documentary Look Stranger - Giant Leeks and Magic Brews',
+            },
+            'params': {
+                # rtmp download
+                'skip_download': True,
+            },
         }, {
             'url': 'http://www.bbc.co.uk/iplayer/playlist/p01dvks4',
             'only_matching': True,

From cf57433bbd190143694b44ac4cd5f56e6eb625ee Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 28 Jan 2016 18:57:32 +0100
Subject: [PATCH 3793/6219] [ffmpeg] fix adding metadata when using
 m3u8_native(fixes #8350)

---
 youtube_dl/postprocessor/ffmpeg.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index daca5d814..48f86e21c 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -391,6 +391,10 @@ class FFmpegMetadataPP(FFmpegPostProcessor):
         for (name, value) in metadata.items():
             options.extend(['-metadata', '%s=%s' % (name, value)])
 
+        # https://github.com/rg3/youtube-dl/issues/8350
+        if info['protocol'] == 'm3u8_native':
+            options.extend(['-bsf:a', 'aac_adtstoasc'])
+
         self._downloader.to_screen('[ffmpeg] Adding metadata to \'%s\'' % filename)
         self.run_ffmpeg(filename, temp_filename, options)
         os.remove(encodeFilename(filename))

From 70029bc348f27294b7f3e369f953167c1893c2bd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 29 Jan 2016 11:27:11 +0100
Subject: [PATCH 3794/6219] [youtube:user] Require 'https?://' in the url
 (fixes #8356)

It was matching www.youtube.com/embed/WpfukLMe1TM.
The generic extractor automatically adds http:// if it's missing.
---
 test/test_all_urls.py           | 2 +-
 youtube_dl/extractor/youtube.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/test_all_urls.py b/test/test_all_urls.py
index a0c11e6c1..f5af184e6 100644
--- a/test/test_all_urls.py
+++ b/test/test_all_urls.py
@@ -56,7 +56,7 @@ class TestAllURLsMatching(unittest.TestCase):
         assertChannel('https://www.youtube.com/channel/HCtnHdj3df7iM/videos')
 
     def test_youtube_user_matching(self):
-        self.assertMatch('www.youtube.com/NASAgovVideo/videos', ['youtube:user'])
+        self.assertMatch('http://www.youtube.com/NASAgovVideo/videos', ['youtube:user'])
 
     def test_youtube_feeds(self):
         self.assertMatch('https://www.youtube.com/feed/watch_later', ['youtube:watchlater'])
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 92b9f3ae4..a24c73584 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1846,7 +1846,7 @@ class YoutubeChannelIE(YoutubePlaylistBaseInfoExtractor):
 
 class YoutubeUserIE(YoutubeChannelIE):
     IE_DESC = 'YouTube.com user videos (URL or "ytuser" keyword)'
-    _VALID_URL = r'(?:(?:(?:https?://)?(?:\w+\.)?youtube\.com/(?:user/)?(?!(?:attribution_link|watch|results)(?:$|[^a-z_A-Z0-9-])))|ytuser:)(?!feed/)(?P<id>[A-Za-z0-9_-]+)'
+    _VALID_URL = r'(?:(?:https?://(?:\w+\.)?youtube\.com/(?:user/)?(?!(?:attribution_link|watch|results)(?:$|[^a-z_A-Z0-9-])))|ytuser:)(?!feed/)(?P<id>[A-Za-z0-9_-]+)'
     _TEMPLATE_URL = 'https://www.youtube.com/user/%s/videos'
     IE_NAME = 'youtube:user'
 

From 055f4172781dd2a43d60f17a91a1d0c1a5f3e6b9 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 29 Jan 2016 12:20:08 +0100
Subject: [PATCH 3795/6219] release 2016.01.29

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 4ac7f9e93..d9f1e22b0 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.01.27'
+__version__ = '2016.01.29'

From 2b4f5e68d1517bcadac4b25ecbac3b143104b1c4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 29 Jan 2016 15:36:33 +0100
Subject: [PATCH 3796/6219] [azubu] Add extractor for live streams (closes
 #8343)

---
 youtube_dl/extractor/__init__.py |  2 +-
 youtube_dl/extractor/azubu.py    | 40 +++++++++++++++++++++++++++++++-
 2 files changed, 40 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 532be7e4c..5e0d7d3dc 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -50,7 +50,7 @@ from .atresplayer import AtresPlayerIE
 from .atttechchannel import ATTTechChannelIE
 from .audimedia import AudiMediaIE
 from .audiomack import AudiomackIE, AudiomackAlbumIE
-from .azubu import AzubuIE
+from .azubu import AzubuIE, AzubuLiveIE
 from .baidu import BaiduVideoIE
 from .bambuser import BambuserIE, BambuserChannelIE
 from .bandcamp import BandcampIE, BandcampAlbumIE
diff --git a/youtube_dl/extractor/azubu.py b/youtube_dl/extractor/azubu.py
index 0961d339f..011edf128 100644
--- a/youtube_dl/extractor/azubu.py
+++ b/youtube_dl/extractor/azubu.py
@@ -3,7 +3,11 @@ from __future__ import unicode_literals
 import json
 
 from .common import InfoExtractor
-from ..utils import float_or_none
+from ..utils import (
+    ExtractorError,
+    float_or_none,
+    sanitized_Request,
+)
 
 
 class AzubuIE(InfoExtractor):
@@ -91,3 +95,37 @@ class AzubuIE(InfoExtractor):
             'view_count': view_count,
             'formats': formats,
         }
+
+
+class AzubuLiveIE(InfoExtractor):
+    _VALID_URL = r'http://www.azubu.tv/(?P<id>[^/]+)$'
+
+    _TEST = {
+        'url': 'http://www.azubu.tv/MarsTVMDLen',
+        'only_matching': True,
+    }
+
+    def _real_extract(self, url):
+        user = self._match_id(url)
+
+        info = self._download_json(
+            'http://api.azubu.tv/public/modules/last-video/{0}/info'.format(user),
+            user)['data']
+        if info['type'] != 'STREAM':
+            raise ExtractorError('{0} is not streaming live'.format(user), expected=True)
+
+        req = sanitized_Request(
+            'https://edge-elb.api.brightcove.com/playback/v1/accounts/3361910549001/videos/ref:' + info['reference_id'])
+        req.add_header('Accept', 'application/json;pk=BCpkADawqM1gvI0oGWg8dxQHlgT8HkdE2LnAlWAZkOlznO39bSZX726u4JqnDsK3MDXcO01JxXK2tZtJbgQChxgaFzEVdHRjaDoxaOu8hHOO8NYhwdxw9BzvgkvLUlpbDNUuDoc4E4wxDToV')
+        bc_info = self._download_json(req, user)
+        m3u8_url = next(source['src'] for source in bc_info['sources'] if source['container'] == 'M2TS')
+        formats = self._extract_m3u8_formats(m3u8_url, user, ext='mp4')
+
+        return {
+            'id': info['id'],
+            'title': self._live_title(info['title']),
+            'uploader_id': user,
+            'formats': formats,
+            'is_live': True,
+            'thumbnail': bc_info['poster'],
+        }

From 68a0ea15b4c20ed0174a82ee79a6d3c3474b0f69 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 30 Jan 2016 00:26:33 +0600
Subject: [PATCH 3797/6219] [cspan] Unescape path (Closes #8365)

---
 youtube_dl/extractor/cspan.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/cspan.py b/youtube_dl/extractor/cspan.py
index b78edf729..b8b9d058d 100644
--- a/youtube_dl/extractor/cspan.py
+++ b/youtube_dl/extractor/cspan.py
@@ -113,7 +113,7 @@ class CSpanIE(InfoExtractor):
                     'tbr': int_or_none(get_text_attr(quality, 'bitrate')),
                 })
             if not formats:
-                path = get_text_attr(f, 'path')
+                path = unescapeHTML(get_text_attr(f, 'path'))
                 if not path:
                     continue
                 formats = self._extract_m3u8_formats(

From 350cf045d8f86497e8d79ae193b40cc44c8e670c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 30 Jan 2016 01:47:46 +0600
Subject: [PATCH 3798/6219] [extractor/common] Restrict checks when auto
 calculating tbr

---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 33290fd74..b3d57dfce 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -828,7 +828,7 @@ class InfoExtractor(object):
         for f in formats:
             # Automatically determine tbr when missing based on abr and vbr (improves
             # formats sorting in some cases)
-            if 'tbr' not in f and 'abr' in f and 'vbr' in f:
+            if 'tbr' not in f and f.get('abr') is not None and f.get('vbr') is not None:
                 f['tbr'] = f['abr'] + f['vbr']
 
         def _formats_key(f):

From 83ab8a79ccc3b6ef143f7d636c0118f7c3e5777b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 30 Jan 2016 01:48:54 +0600
Subject: [PATCH 3799/6219] [espn] Improve video id extraction (Closes #8368)

---
 youtube_dl/extractor/espn.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/espn.py b/youtube_dl/extractor/espn.py
index 3762d8748..db4b263bc 100644
--- a/youtube_dl/extractor/espn.py
+++ b/youtube_dl/extractor/espn.py
@@ -53,8 +53,8 @@ class ESPNIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         video_id = self._search_regex(
-            r'class="video-play-button"[^>]+data-id="(\d+)',
-            webpage, 'video id')
+            r'class=(["\']).*?video-play-button.*?\1[^>]+data-id=["\'](?P<id>\d+)',
+            webpage, 'video id', group='id')
 
         cms = 'espn'
         if 'data-source="intl"' in webpage:

From e047922be01ca346cee8f53b7972773d64eed6f4 Mon Sep 17 00:00:00 2001
From: ping <liping.ong@gmail.com>
Date: Sat, 30 Jan 2016 11:04:11 +0800
Subject: [PATCH 3800/6219] [daum] Fix copy-paste mistake

---
 youtube_dl/extractor/daum.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/daum.py b/youtube_dl/extractor/daum.py
index c84302c43..0d74e5207 100644
--- a/youtube_dl/extractor/daum.py
+++ b/youtube_dl/extractor/daum.py
@@ -38,7 +38,8 @@ class DaumIE(InfoExtractor):
             'description': 'md5:79794514261164ff27e36a21ad229fc5',
             'upload_date': '20150604',
             'duration': 154
-        },    }, {
+        },
+    }, {
         'url': 'http://tvpot.daum.net/v/07dXWRka62Y%24',
         'only_matching': True,
     }]

From f5d30d521c63b4eec4aa4386365054222e354231 Mon Sep 17 00:00:00 2001
From: ping <liping.ong@gmail.com>
Date: Sat, 30 Jan 2016 11:09:30 +0800
Subject: [PATCH 3801/6219] [daum] Fix add view_count, comment_count to test

---
 youtube_dl/extractor/daum.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/daum.py b/youtube_dl/extractor/daum.py
index 0d74e5207..567abdc39 100644
--- a/youtube_dl/extractor/daum.py
+++ b/youtube_dl/extractor/daum.py
@@ -37,7 +37,9 @@ class DaumIE(InfoExtractor):
             'title': 'md5:a100d65d09cec246d8aa9bde7de45aed',
             'description': 'md5:79794514261164ff27e36a21ad229fc5',
             'upload_date': '20150604',
-            'duration': 154
+            'duration': 154,
+            'view_count': int,
+            'comment_count': int,
         },
     }, {
         'url': 'http://tvpot.daum.net/v/07dXWRka62Y%24',

From 06ffa3348590f9aa6ddca7ca8f78424c49314560 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 30 Jan 2016 16:23:37 +0800
Subject: [PATCH 3802/6219] [daum.net] Move the request to ClipInfoXml.do

To reduce the number of wasted requests
---
 youtube_dl/extractor/daum.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/daum.py b/youtube_dl/extractor/daum.py
index 567abdc39..543eb5c3a 100644
--- a/youtube_dl/extractor/daum.py
+++ b/youtube_dl/extractor/daum.py
@@ -49,9 +49,6 @@ class DaumIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
         query = compat_urllib_parse.urlencode({'vid': video_id})
-        info = self._download_xml(
-            'http://tvpot.daum.net/clip/ClipInfoXml.do?' + query, video_id,
-            'Downloading video info')
         movie_data = self._download_json(
             'http://videofarm.daum.net/controller/api/closed/v1_2/IntegratedMovieData.json?' + query,
             video_id, 'Downloading video formats info')
@@ -60,6 +57,10 @@ class DaumIE(InfoExtractor):
         if not movie_data.get('output_list', {}).get('output_list') and re.match(r'^\d+$', video_id):
             return self.url_result('http://tvpot.daum.net/clip/ClipView.do?clipid=%s' % video_id)
 
+        info = self._download_xml(
+            'http://tvpot.daum.net/clip/ClipInfoXml.do?' + query, video_id,
+            'Downloading video info')
+
         formats = []
         for format_el in movie_data['output_list']['output_list']:
             profile = format_el['profile']

From 830afe85dc79e034ffbe7f3ec1d049482ed651de Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 30 Jan 2016 16:50:13 +0800
Subject: [PATCH 3803/6219] [daum.net] Support VodPlayer.swf URLs (closes
 #8173)

---
 youtube_dl/extractor/daum.py | 22 +++++++++++++++++++---
 1 file changed, 19 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/daum.py b/youtube_dl/extractor/daum.py
index 543eb5c3a..ea6a9d848 100644
--- a/youtube_dl/extractor/daum.py
+++ b/youtube_dl/extractor/daum.py
@@ -5,7 +5,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse
+from ..compat import (
+    compat_urllib_parse,
+    compat_urllib_parse_unquote,
+)
 from ..utils import (
     int_or_none,
     str_to_int,
@@ -14,7 +17,7 @@ from ..utils import (
 
 
 class DaumIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:m\.)?tvpot\.daum\.net/v/(?P<id>[^?#&]+)'
+    _VALID_URL = r'https?://(?:(?:m\.)?tvpot\.daum\.net/v/|videofarm\.daum\.net/controller/player/VodPlayer\.swf\?vid=)(?P<id>[^?#&]+)'
     IE_NAME = 'daum.net'
 
     _TESTS = [{
@@ -44,10 +47,23 @@ class DaumIE(InfoExtractor):
     }, {
         'url': 'http://tvpot.daum.net/v/07dXWRka62Y%24',
         'only_matching': True,
+    }, {
+        'url': 'http://videofarm.daum.net/controller/player/VodPlayer.swf?vid=vwIpVpCQsT8%24&ref=',
+        'info_dict': {
+            'id': 'vwIpVpCQsT8$',
+            'ext': 'flv',
+            'title': '01-Korean War ( Trouble on the horizon )',
+            'description': '\nKorean War 01\nTrouble on the horizon\n전쟁의 먹구름',
+            'upload_date': '20080223',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 249,
+            'view_count': int,
+            'comment_count': int,
+        },
     }]
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        video_id = compat_urllib_parse_unquote(self._match_id(url))
         query = compat_urllib_parse.urlencode({'vid': video_id})
         movie_data = self._download_json(
             'http://videofarm.daum.net/controller/api/closed/v1_2/IntegratedMovieData.json?' + query,

From 0179f6a8301e8dee1f435289f89cf1c748a13b16 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 30 Jan 2016 16:54:14 +0800
Subject: [PATCH 3804/6219] [daum] Add 'thumbnail' to all _TESTS

---
 youtube_dl/extractor/daum.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/daum.py b/youtube_dl/extractor/daum.py
index ea6a9d848..9bc345f60 100644
--- a/youtube_dl/extractor/daum.py
+++ b/youtube_dl/extractor/daum.py
@@ -28,6 +28,7 @@ class DaumIE(InfoExtractor):
             'title': '마크 헌트 vs 안토니오 실바',
             'description': 'Mark Hunt vs Antonio Silva',
             'upload_date': '20131217',
+            'thumbnail': 're:^https?://.*\.(?:jpg|png)',
             'duration': 2117,
             'view_count': int,
             'comment_count': int,
@@ -40,6 +41,7 @@ class DaumIE(InfoExtractor):
             'title': 'md5:a100d65d09cec246d8aa9bde7de45aed',
             'description': 'md5:79794514261164ff27e36a21ad229fc5',
             'upload_date': '20150604',
+            'thumbnail': 're:^https?://.*\.(?:jpg|png)',
             'duration': 154,
             'view_count': int,
             'comment_count': int,
@@ -55,7 +57,7 @@ class DaumIE(InfoExtractor):
             'title': '01-Korean War ( Trouble on the horizon )',
             'description': '\nKorean War 01\nTrouble on the horizon\n전쟁의 먹구름',
             'upload_date': '20080223',
-            'thumbnail': 're:^https?://.*\.jpg$',
+            'thumbnail': 're:^https?://.*\.(?:jpg|png)',
             'duration': 249,
             'view_count': int,
             'comment_count': int,
@@ -122,6 +124,7 @@ class DaumClipIE(InfoExtractor):
             'title': 'DOTA 2GETHER 시즌2 6회 - 2부',
             'description': 'DOTA 2GETHER 시즌2 6회 - 2부',
             'upload_date': '20130831',
+            'thumbnail': 're:^https?://.*\.(?:jpg|png)',
             'duration': 3868,
             'view_count': int,
         },

From 7d106a65ca30cc3201f147cb96aa98a15a48d6d2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 30 Jan 2016 12:26:40 +0100
Subject: [PATCH 3805/6219] Add --hls-use-mpegts option

When using the mpegts container hls vidoes can be played while being downloaded (useful if you are recording a live stream).
VLC and mpv play them file, but QuickTime doesn't.
---
 youtube_dl/YoutubeDL.py         | 2 +-
 youtube_dl/__init__.py          | 1 +
 youtube_dl/downloader/common.py | 1 +
 youtube_dl/downloader/hls.py    | 6 +++++-
 youtube_dl/options.py           | 5 +++++
 5 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index e1bd40843..2a3d6cd4a 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -263,7 +263,7 @@ class YoutubeDL(object):
     the downloader (see youtube_dl/downloader/common.py):
     nopart, updatetime, buffersize, ratelimit, min_filesize, max_filesize, test,
     noresizebuffer, retries, continuedl, noprogress, consoletitle,
-    xattr_set_filesize, external_downloader_args.
+    xattr_set_filesize, external_downloader_args, hls_use_mpegts.
 
     The following options are used by the post processors:
     prefer_ffmpeg:     If True, use ffmpeg instead of avconv if both are available,
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 9f131f5db..f5f064241 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -369,6 +369,7 @@ def _real_main(argv=None):
         'no_color': opts.no_color,
         'ffmpeg_location': opts.ffmpeg_location,
         'hls_prefer_native': opts.hls_prefer_native,
+        'hls_use_mpegts': opts.hls_use_mpegts,
         'external_downloader_args': external_downloader_args,
         'postprocessor_args': postprocessor_args,
         'cn_verification_proxy': opts.cn_verification_proxy,
diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index fc7521598..de815612c 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -45,6 +45,7 @@ class FileDownloader(object):
                         (experimental)
     external_downloader_args:  A list of additional command-line arguments for the
                         external downloader.
+    hls_use_mpegts:     Use the mpegts container for HLS videos.
 
     Subclasses of this one must re-define the real_download method.
     """
diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index 10b83c6b2..cb34dc4ab 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -39,7 +39,11 @@ class HlsFD(FileDownloader):
                 '-headers',
                 ''.join('%s: %s\r\n' % (key, val) for key, val in headers.items())]
 
-        args += ['-i', url, '-f', 'mp4', '-c', 'copy', '-bsf:a', 'aac_adtstoasc']
+        args += ['-i', url, '-c', 'copy']
+        if self.params.get('hls_use_mpegts', False):
+            args += ['-f', 'mpegts']
+        else:
+            args += ['-f', 'mp4', '-bsf:a', 'aac_adtstoasc']
 
         args = [encodeArgument(opt) for opt in args]
         args.append(encodeFilename(ffpp._ffmpeg_filename_argument(tmpfilename), True))
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 433245f00..39fc4306a 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -415,6 +415,11 @@ def parseOpts(overrideArguments=None):
         '--hls-prefer-native',
         dest='hls_prefer_native', action='store_true',
         help='Use the native HLS downloader instead of ffmpeg (experimental)')
+    downloader.add_option(
+        '--hls-use-mpegts',
+        dest='hls_use_mpegts', action='store_true',
+        help='Use the mpegts container for HLS videos, allowing to play the '
+             'video while downloading (some players may not be able to play it')
     downloader.add_option(
         '--external-downloader',
         dest='external_downloader', metavar='COMMAND',

From c1406299959b780e7b86b567bd4eeecc8556dce0 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 30 Jan 2016 19:30:39 +0800
Subject: [PATCH 3806/6219] [facebook] Support alternative webpage form

Fixes #8371
---
 youtube_dl/extractor/facebook.py | 29 ++++++++++++++++++++++++-----
 1 file changed, 24 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index cb5dd57fb..899b0896b 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -150,10 +150,32 @@ class FacebookIE(InfoExtractor):
         url = 'https://www.facebook.com/video/video.php?v=%s' % video_id
         webpage = self._download_webpage(url, video_id)
 
+        video_data = None
+
         BEFORE = '{swf.addParam(param[0], param[1]);});\n'
         AFTER = '.forEach(function(variable) {swf.addVariable(variable[0], variable[1]);});'
         m = re.search(re.escape(BEFORE) + '(.*?)' + re.escape(AFTER), webpage)
-        if not m:
+        if m:
+            data = dict(json.loads(m.group(1)))
+            params_raw = compat_urllib_parse_unquote(data['params'])
+            video_data = json.loads(params_raw)['video_data']
+
+        def video_data_list2dict(video_data):
+            ret = {}
+            for item in video_data:
+                format_id = item['stream_type']
+                ret.setdefault(format_id, []).append(item)
+            return ret
+
+        if not video_data:
+            server_js_data = self._parse_json(self._search_regex(
+                r'handleServerJS\(({.+})\);', webpage, 'server js data'), video_id)
+            for item in server_js_data['instances']:
+                if item[1][0] == 'VideoConfig':
+                    video_data = video_data_list2dict(item[2][0]['videoData'])
+                    break
+
+        if not video_data:
             m_msg = re.search(r'class="[^"]*uiInterstitialContent[^"]*"><div>(.*?)</div>', webpage)
             if m_msg is not None:
                 raise ExtractorError(
@@ -161,12 +183,9 @@ class FacebookIE(InfoExtractor):
                     expected=True)
             else:
                 raise ExtractorError('Cannot parse data')
-        data = dict(json.loads(m.group(1)))
-        params_raw = compat_urllib_parse_unquote(data['params'])
-        params = json.loads(params_raw)
 
         formats = []
-        for format_id, f in params['video_data'].items():
+        for format_id, f in video_data.items():
             if not f or not isinstance(f, list):
                 continue
             for quality in ('sd', 'hd'):

From 05d0d131a76e34af843d982dd47b556ddc8b9e44 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 30 Jan 2016 20:05:56 +0800
Subject: [PATCH 3807/6219] [youtube] Move decrypt_sig out of
 _parse_dash_manifest

---
 youtube_dl/extractor/youtube.py | 32 ++++++++++++++------------------
 1 file changed, 14 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index a24c73584..08b7e15c4 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1035,22 +1035,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         url = 'https://www.youtube.com/annotations_invideo?features=1&legacy=1&video_id=%s' % video_id
         return self._download_webpage(url, video_id, note='Searching for annotations.', errnote='Unable to download video annotations.')
 
-    def _parse_dash_manifest(
-            self, video_id, dash_manifest_url, player_url, age_gate, fatal=True):
-        def decrypt_sig(mobj):
-            s = mobj.group(1)
-            dec_s = self._decrypt_signature(s, video_id, player_url, age_gate)
-            return '/signature/%s' % dec_s
-        dash_manifest_url = re.sub(r'/s/([a-fA-F0-9\.]+)', decrypt_sig, dash_manifest_url)
-        dash_doc = self._download_xml(
-            dash_manifest_url, video_id,
-            note='Downloading DASH manifest',
-            errnote='Could not download DASH manifest',
-            fatal=fatal)
-
-        if dash_doc is False:
-            return []
-
+    def _parse_dash_manifest(self, video_id, dash_doc, fatal=True):
         formats = []
         for a in dash_doc.findall('.//{urn:mpeg:DASH:schema:MPD:2011}AdaptationSet'):
             mime_type = a.attrib.get('mimeType')
@@ -1533,8 +1518,19 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             for dash_manifest_url in dash_mpds:
                 dash_formats = {}
                 try:
-                    for df in self._parse_dash_manifest(
-                            video_id, dash_manifest_url, player_url, age_gate, dash_mpd_fatal):
+                    def decrypt_sig(mobj):
+                        s = mobj.group(1)
+                        dec_s = self._decrypt_signature(s, video_id, player_url, age_gate)
+                        return '/signature/%s' % dec_s
+
+                    dash_manifest_url = re.sub(r'/s/([a-fA-F0-9\.]+)', decrypt_sig, dash_manifest_url)
+                    dash_doc = self._download_xml(
+                        dash_manifest_url, video_id,
+                        note='Downloading DASH manifest',
+                        errnote='Could not download DASH manifest',
+                        fatal=dash_mpd_fatal)
+
+                    for df in self._parse_dash_manifest(video_id, dash_doc, dash_mpd_fatal):
                         # Do not overwrite DASH format found in some previous DASH manifest
                         if df['format_id'] not in dash_formats:
                             dash_formats[df['format_id']] = df

From 2f0a33d8a3c926f3a81d0e76bd669a8bc57a2cf8 Mon Sep 17 00:00:00 2001
From: ping <liping.ong@gmail.com>
Date: Sat, 30 Jan 2016 20:10:36 +0800
Subject: [PATCH 3808/6219] [daum.net] Support for playlists, user channels

---
 youtube_dl/extractor/__init__.py |   2 +
 youtube_dl/extractor/daum.py     | 119 ++++++++++++++++++++++++++++++-
 2 files changed, 120 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 5e0d7d3dc..e05d986ef 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -142,6 +142,8 @@ from .dailymotion import (
 from .daum import (
     DaumIE,
     DaumClipIE,
+    DaumPlaylistIE,
+    DaumUserIE,
 )
 from .dbtv import DBTVIE
 from .dcn import (
diff --git a/youtube_dl/extractor/daum.py b/youtube_dl/extractor/daum.py
index 9bc345f60..cf158c42d 100644
--- a/youtube_dl/extractor/daum.py
+++ b/youtube_dl/extractor/daum.py
@@ -3,6 +3,7 @@
 from __future__ import unicode_literals
 
 import re
+import itertools
 
 from .common import InfoExtractor
 from ..compat import (
@@ -13,6 +14,7 @@ from ..utils import (
     int_or_none,
     str_to_int,
     xpath_text,
+    unescapeHTML,
 )
 
 
@@ -143,7 +145,7 @@ class DaumClipIE(InfoExtractor):
             '_type': 'url_transparent',
             'id': video_id,
             'url': 'http://tvpot.daum.net/v/%s' % clip_info['vid'],
-            'title': clip_info['title'],
+            'title': unescapeHTML(clip_info['title']),
             'thumbnail': clip_info.get('thumb_url'),
             'description': clip_info.get('contents'),
             'duration': int_or_none(clip_info.get('duration')),
@@ -151,3 +153,118 @@ class DaumClipIE(InfoExtractor):
             'view_count': int_or_none(clip_info.get('play_count')),
             'ie_key': 'Daum',
         }
+
+
+class DaumListIE(InfoExtractor):
+    def _get_entries(self, list_id, list_id_type):
+        name = None
+        entries = []
+        for pagenum in itertools.count(1):
+            list_info = self._download_json(
+                'http://tvpot.daum.net/mypot/json/GetClipInfo.do?size=48&init=true&order=date&page=%d&%s=%s' % (
+                    pagenum, list_id_type, list_id), list_id,'Downloading list info - %s' % pagenum)
+            
+            entries.extend([
+                self.url_result(
+                    'http://tvpot.daum.net/v/%s' % clip['vid'])
+                for clip in list_info['clip_list']
+            ])
+
+            if not name:
+                name = list_info.get('playlist_bean', {}).get('name') or \
+                    list_info.get('potInfo', {}).get('name')
+
+            if not list_info.get('has_more'):
+                break
+
+        return name, entries
+
+
+class DaumPlaylistIE(DaumListIE):
+    _VALID_URL = r'https?://(?:m\.)?tvpot\.daum\.net/mypot/(?:View\.do|Top\.tv)\?.*?playlistid=(?P<id>[0-9]+)'
+    IE_NAME = 'daum.net:playlist'
+
+    _TESTS = [{
+        'note': 'Playlist url with clipid',
+        'url': 'http://tvpot.daum.net/mypot/View.do?playlistid=6213966&clipid=73806844',
+        'info_dict': {
+            'id': '6213966',
+            'title': 'Woorissica Official',
+        },
+        'playlist_mincount': 181
+    }, {
+        'note': 'Playlist url with clipid - noplaylist',
+        'url': 'http://tvpot.daum.net/mypot/View.do?playlistid=6213966&clipid=73806844',
+        'info_dict': {
+            'id': '73806844',
+            'ext': 'mp4',
+            'title': '151017 Airport',
+            'upload_date': '20160117',
+        },
+        'params': {
+            'noplaylist': True,
+            'skip_download': True,
+        }
+    }]
+
+    def _real_extract(self, url):
+        if DaumClipIE.suitable(url) and self._downloader.params.get('noplaylist'):
+            return self.url_result(url, 'DaumClip')
+
+        list_id = self._match_id(url)
+        self.to_screen('Downloading playlist %s - add --no-playlist to just download video' % list_id)
+
+        name, entries = self._get_entries(list_id, 'playlistid')
+
+        return self.playlist_result(entries, list_id, name)
+
+
+class DaumUserIE(DaumListIE):
+    _VALID_URL = r'https?://(?:m\.)?tvpot\.daum\.net/mypot/(?:View|Top)\.do\?.*?ownerid=(?P<id>[0-9a-zA-Z]+)'
+    IE_NAME = 'daum.net:user'
+
+    _TESTS = [{
+        'url': 'http://tvpot.daum.net/mypot/View.do?ownerid=o2scDLIVbHc0',
+        'info_dict': {
+            'id': 'o2scDLIVbHc0',
+            'title': '마이 리틀 텔레비전',
+        },
+        'playlist_mincount': 213
+    }, {
+        'url': 'http://tvpot.daum.net/mypot/View.do?ownerid=o2scDLIVbHc0&clipid=73801156',
+        'info_dict': {
+            'id': '73801156',
+            'ext': 'mp4',
+            'title': '[미공개] 김구라, 오만석이 부릅니다 \'오케피\' - 마이 리틀 텔레비전 20160116',
+            'upload_date': '20160117',
+            'description': 'md5:5e91d2d6747f53575badd24bd62b9f36'
+        },
+        'params': {
+            'noplaylist': True,
+            'skip_download': True,
+        }
+    }, {
+        'note': 'Playlist url has ownerid and playlistid, playlistid takes precedence',
+        'url': 'http://tvpot.daum.net/mypot/View.do?ownerid=o2scDLIVbHc0&playlistid=6196631',
+        'info_dict': {
+            'id': '6196631',
+            'title': '마이 리틀 텔레비전 - 20160109',
+        },
+        'playlist_count': 11
+    }, {
+        'url': 'http://tvpot.daum.net/mypot/Top.do?ownerid=o2scDLIVbHc0',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        if DaumClipIE.suitable(url) and self._downloader.params.get('noplaylist'):
+            return self.url_result(url, 'DaumClip')
+
+        if DaumPlaylistIE.suitable(url):
+            return self.url_result(url, 'DaumPlaylist')
+
+        list_id = self._match_id(url)
+        self.to_screen('Downloading playlist %s - add --no-playlist to just download video' % list_id)
+        name, entries = self._get_entries(list_id, 'ownerid')
+
+        return self.playlist_result(entries, list_id, name)

From 8ab3fe81d84cb541eac3af73404e71d979967dcf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 30 Jan 2016 18:28:38 +0600
Subject: [PATCH 3809/6219] [downloader/f4m] Prefer bootstrap url attribute
 over inline bootstrap info

---
 youtube_dl/downloader/f4m.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index aaf0c49c8..f8da04f69 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -273,15 +273,21 @@ class F4mFD(FragmentFD):
         return fragments_list
 
     def _parse_bootstrap_node(self, node, base_url):
-        if node.text is None:
+        # Sometimes non empty inline bootstrap info can be specified along
+        # with bootstrap url attribute (e.g. dummy inline bootstrap info
+        # contains whitespace characters in [1]). We will prefer bootstrap
+        # url over inline bootstrap info when present.
+        # 1. http://live-1-1.rutube.ru/stream/1024/HDS/SD/C2NKsS85HQNckgn5HdEmOQ/1454167650/S-s604419906/move/four/dirs/upper/1024-576p.f4m
+        bootstrap_url = node.get('url')
+        if bootstrap_url:
             bootstrap_url = compat_urlparse.urljoin(
-                base_url, node.attrib['url'])
+                base_url, bootstrap_url)
             boot_info = self._get_bootstrap_from_url(bootstrap_url)
         else:
             bootstrap_url = None
             bootstrap = base64.b64decode(node.text.encode('ascii'))
             boot_info = read_bootstrap_info(bootstrap)
-        return (boot_info, bootstrap_url)
+        return boot_info, bootstrap_url
 
     def real_download(self, filename, info_dict):
         man_url = info_dict['url']

From c3deacd5621d9686a32614e550eb4726e66c3899 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 30 Jan 2016 18:30:27 +0600
Subject: [PATCH 3810/6219] [matchtv] Add extractor (Closes #8313)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/matchtv.py  | 55 ++++++++++++++++++++++++++++++++
 2 files changed, 56 insertions(+)
 create mode 100644 youtube_dl/extractor/matchtv.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 5e0d7d3dc..e15495ec8 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -372,6 +372,7 @@ from .macgamestore import MacGameStoreIE
 from .mailru import MailRuIE
 from .makertv import MakerTVIE
 from .malemotion import MalemotionIE
+from .matchtv import MatchTVIE
 from .mdr import MDRIE
 from .metacafe import MetacafeIE
 from .metacritic import MetacriticIE
diff --git a/youtube_dl/extractor/matchtv.py b/youtube_dl/extractor/matchtv.py
new file mode 100644
index 000000000..28e0dfe63
--- /dev/null
+++ b/youtube_dl/extractor/matchtv.py
@@ -0,0 +1,55 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import random
+
+from .common import InfoExtractor
+from ..compat import compat_urllib_parse
+from ..utils import (
+    sanitized_Request,
+    xpath_text,
+)
+
+
+class MatchTVIE(InfoExtractor):
+    _VALID_URL = r'https?://matchtv\.ru/?#live-player'
+    _TEST = {
+        'url': 'http://matchtv.ru/#live-player',
+        'info_dict': {
+            'id': 'matchtv-live',
+            'ext': 'flv',
+            'title': 're:^Матч ТВ - Прямой эфир \d{4}-\d{2}-\d{2} \d{2}:\d{2}$',
+            'is_live': True,
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = 'matchtv-live'
+        request = sanitized_Request(
+            'http://player.matchtv.ntvplus.tv/player/smil?%s' % compat_urllib_parse.urlencode({
+                'ts': '',
+                'quality': 'SD',
+                'contentId': '561d2c0df7159b37178b4567',
+                'sign': '',
+                'includeHighlights': '0',
+                'userId': '',
+                'sessionId': random.randint(1, 1000000000),
+                'contentType': 'channel',
+                'timeShift': '0',
+                'platform': 'portal',
+            }),
+            headers={
+                'Referer': 'http://player.matchtv.ntvplus.tv/embed-player/NTVEmbedPlayer.swf',
+            })
+        video_url = self._download_json(request, video_id)['data']['videoUrl']
+        f4m_url = xpath_text(self._download_xml(video_url, video_id), './to')
+        formats = self._extract_f4m_formats(f4m_url, video_id)
+        return {
+            'id': video_id,
+            'title': self._live_title('Матч ТВ - Прямой эфир'),
+            'is_live': True,
+            'formats': formats,
+        }

From 53be8894e476f6e3a49373fbd34ecff157440607 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 30 Jan 2016 18:44:22 +0600
Subject: [PATCH 3811/6219] [options] Add missing closing parenthesis

---
 youtube_dl/options.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 39fc4306a..2137dfb3f 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -419,7 +419,7 @@ def parseOpts(overrideArguments=None):
         '--hls-use-mpegts',
         dest='hls_use_mpegts', action='store_true',
         help='Use the mpegts container for HLS videos, allowing to play the '
-             'video while downloading (some players may not be able to play it')
+             'video while downloading (some players may not be able to play it)')
     downloader.add_option(
         '--external-downloader',
         dest='external_downloader', metavar='COMMAND',

From 17b598d30cae2c287f3556f874ddf0fc5d028aec Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 30 Jan 2016 21:05:55 +0800
Subject: [PATCH 3812/6219] [common] _parse_dash_manifest() from youtube.py

---
 youtube_dl/extractor/common.py  | 52 +++++++++++++++++++++++++++++++++
 youtube_dl/extractor/youtube.py | 52 ---------------------------------
 2 files changed, 52 insertions(+), 52 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index b3d57dfce..7ad255672 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1330,6 +1330,58 @@ class InfoExtractor(object):
             })
         return entries
 
+    def _parse_dash_manifest(self, video_id, dash_doc, fatal=True):
+        formats = []
+        for a in dash_doc.findall('.//{urn:mpeg:DASH:schema:MPD:2011}AdaptationSet'):
+            mime_type = a.attrib.get('mimeType')
+            for r in a.findall('{urn:mpeg:DASH:schema:MPD:2011}Representation'):
+                url_el = r.find('{urn:mpeg:DASH:schema:MPD:2011}BaseURL')
+                if url_el is None:
+                    continue
+                if mime_type == 'text/vtt':
+                    # TODO implement WebVTT downloading
+                    pass
+                elif mime_type.startswith('audio/') or mime_type.startswith('video/'):
+                    segment_list = r.find('{urn:mpeg:DASH:schema:MPD:2011}SegmentList')
+                    format_id = r.attrib['id']
+                    video_url = url_el.text
+                    filesize = int_or_none(url_el.attrib.get('{http://youtube.com/yt/2012/10/10}contentLength'))
+                    f = {
+                        'format_id': format_id,
+                        'url': video_url,
+                        'width': int_or_none(r.attrib.get('width')),
+                        'height': int_or_none(r.attrib.get('height')),
+                        'tbr': int_or_none(r.attrib.get('bandwidth'), 1000),
+                        'asr': int_or_none(r.attrib.get('audioSamplingRate')),
+                        'filesize': filesize,
+                        'fps': int_or_none(r.attrib.get('frameRate')),
+                    }
+                    if segment_list is not None:
+                        f.update({
+                            'initialization_url': segment_list.find('{urn:mpeg:DASH:schema:MPD:2011}Initialization').attrib['sourceURL'],
+                            'segment_urls': [segment.attrib.get('media') for segment in segment_list.findall('{urn:mpeg:DASH:schema:MPD:2011}SegmentURL')],
+                            'protocol': 'http_dash_segments',
+                        })
+                    try:
+                        existing_format = next(
+                            fo for fo in formats
+                            if fo['format_id'] == format_id)
+                    except StopIteration:
+                        full_info = self._formats.get(format_id, {}).copy()
+                        full_info.update(f)
+                        codecs = r.attrib.get('codecs')
+                        if codecs:
+                            if full_info.get('acodec') == 'none':
+                                full_info['vcodec'] = codecs
+                            elif full_info.get('vcodec') == 'none':
+                                full_info['acodec'] = codecs
+                        formats.append(full_info)
+                    else:
+                        existing_format.update(f)
+                else:
+                    self.report_warning('Unknown MIME type %s in DASH manifest' % mime_type)
+        return formats
+
     def _live_title(self, name):
         """ Generate the title for a live video """
         now = datetime.datetime.now()
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 08b7e15c4..acd5cf2c3 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1035,58 +1035,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         url = 'https://www.youtube.com/annotations_invideo?features=1&legacy=1&video_id=%s' % video_id
         return self._download_webpage(url, video_id, note='Searching for annotations.', errnote='Unable to download video annotations.')
 
-    def _parse_dash_manifest(self, video_id, dash_doc, fatal=True):
-        formats = []
-        for a in dash_doc.findall('.//{urn:mpeg:DASH:schema:MPD:2011}AdaptationSet'):
-            mime_type = a.attrib.get('mimeType')
-            for r in a.findall('{urn:mpeg:DASH:schema:MPD:2011}Representation'):
-                url_el = r.find('{urn:mpeg:DASH:schema:MPD:2011}BaseURL')
-                if url_el is None:
-                    continue
-                if mime_type == 'text/vtt':
-                    # TODO implement WebVTT downloading
-                    pass
-                elif mime_type.startswith('audio/') or mime_type.startswith('video/'):
-                    segment_list = r.find('{urn:mpeg:DASH:schema:MPD:2011}SegmentList')
-                    format_id = r.attrib['id']
-                    video_url = url_el.text
-                    filesize = int_or_none(url_el.attrib.get('{http://youtube.com/yt/2012/10/10}contentLength'))
-                    f = {
-                        'format_id': format_id,
-                        'url': video_url,
-                        'width': int_or_none(r.attrib.get('width')),
-                        'height': int_or_none(r.attrib.get('height')),
-                        'tbr': int_or_none(r.attrib.get('bandwidth'), 1000),
-                        'asr': int_or_none(r.attrib.get('audioSamplingRate')),
-                        'filesize': filesize,
-                        'fps': int_or_none(r.attrib.get('frameRate')),
-                    }
-                    if segment_list is not None:
-                        f.update({
-                            'initialization_url': segment_list.find('{urn:mpeg:DASH:schema:MPD:2011}Initialization').attrib['sourceURL'],
-                            'segment_urls': [segment.attrib.get('media') for segment in segment_list.findall('{urn:mpeg:DASH:schema:MPD:2011}SegmentURL')],
-                            'protocol': 'http_dash_segments',
-                        })
-                    try:
-                        existing_format = next(
-                            fo for fo in formats
-                            if fo['format_id'] == format_id)
-                    except StopIteration:
-                        full_info = self._formats.get(format_id, {}).copy()
-                        full_info.update(f)
-                        codecs = r.attrib.get('codecs')
-                        if codecs:
-                            if full_info.get('acodec') == 'none':
-                                full_info['vcodec'] = codecs
-                            elif full_info.get('vcodec') == 'none':
-                                full_info['acodec'] = codecs
-                        formats.append(full_info)
-                    else:
-                        existing_format.update(f)
-                else:
-                    self.report_warning('Unknown MIME type %s in DASH manifest' % mime_type)
-        return formats
-
     def _real_extract(self, url):
         url, smuggled_data = unsmuggle_url(url, {})
 

From 5fa1702ca6d8a57b4f0ff108c6fe7488d09b448c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 30 Jan 2016 19:20:52 +0600
Subject: [PATCH 3813/6219] [downloader/fragment] Do not report total bytes
 estimation and eta for live streams

---
 youtube_dl/downloader/fragment.py | 25 ++++++++++++++++---------
 1 file changed, 16 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
index 0c9113d0f..e2f34eec6 100644
--- a/youtube_dl/downloader/fragment.py
+++ b/youtube_dl/downloader/fragment.py
@@ -26,7 +26,11 @@ class FragmentFD(FileDownloader):
         self._start_frag_download(ctx)
 
     def _prepare_frag_download(self, ctx):
-        self.to_screen('[%s] Total fragments: %d' % (self.FD_NAME, ctx['total_frags']))
+        if 'live' not in ctx:
+            ctx['live'] = False
+        self.to_screen(
+            '[%s] Total fragments: %s'
+            % (self.FD_NAME, ctx['total_frags'] if not ctx['live'] else 'unknown (live)'))
         self.report_destination(ctx['filename'])
         dl = HttpQuietDownloader(
             self.ydl,
@@ -74,13 +78,15 @@ class FragmentFD(FileDownloader):
             if s['status'] not in ('downloading', 'finished'):
                 return
 
+            time_now = time.time()
+
             frag_total_bytes = s.get('total_bytes') or 0
 
-            estimated_size = (
-                (ctx['complete_frags_downloaded_bytes'] + frag_total_bytes) /
-                (state['frag_index'] + 1) * total_frags)
-            time_now = time.time()
-            state['total_bytes_estimate'] = estimated_size
+            if not ctx['live']:
+                estimated_size = (
+                    (ctx['complete_frags_downloaded_bytes'] + frag_total_bytes) /
+                    (state['frag_index'] + 1) * total_frags)
+                state['total_bytes_estimate'] = estimated_size
             state['elapsed'] = time_now - start
 
             if s['status'] == 'finished':
@@ -91,9 +97,10 @@ class FragmentFD(FileDownloader):
             else:
                 frag_downloaded_bytes = s['downloaded_bytes']
                 state['downloaded_bytes'] += frag_downloaded_bytes - ctx['prev_frag_downloaded_bytes']
-                state['eta'] = self.calc_eta(
-                    start, time_now, estimated_size,
-                    state['downloaded_bytes'])
+                if not ctx['live']:
+                    state['eta'] = self.calc_eta(
+                        start, time_now, estimated_size,
+                        state['downloaded_bytes'])
                 state['speed'] = s.get('speed')
                 ctx['prev_frag_downloaded_bytes'] = frag_downloaded_bytes
             self._hook_progress(state)

From 09104e9930cff6cc5e9d3ab1951fdbcbb5e96840 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 30 Jan 2016 19:22:15 +0600
Subject: [PATCH 3814/6219] [downloader/f4m] Add live stream flag to context

Now download progress for f4m livestreams is reported correctly
---
 youtube_dl/downloader/f4m.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index f8da04f69..581fa7b71 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -332,6 +332,7 @@ class F4mFD(FragmentFD):
         ctx = {
             'filename': filename,
             'total_frags': total_frags,
+            'live': live,
         }
 
         self._prepare_frag_download(ctx)

From b323e1707d8f058b88a5f15f3418b31cf969399d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 30 Jan 2016 21:27:43 +0800
Subject: [PATCH 3815/6219] [common] Modify _parse_dash_manifest for use in
 Facebook

---
 youtube_dl/extractor/common.py | 29 +++++++++++++++++------------
 1 file changed, 17 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 7ad255672..83628a68f 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1330,22 +1330,24 @@ class InfoExtractor(object):
             })
         return entries
 
-    def _parse_dash_manifest(self, video_id, dash_doc, fatal=True):
+    def _parse_dash_manifest(self, video_id, dash_doc, default_ns='urn:mpeg:DASH:schema:MPD:2011', formats_dict={}, fatal=True):
+        def _add_ns(tag):
+            return '{%s}%s' % (default_ns, tag)
+
         formats = []
-        for a in dash_doc.findall('.//{urn:mpeg:DASH:schema:MPD:2011}AdaptationSet'):
+        for a in dash_doc.findall('.//' + _add_ns('AdaptationSet')):
             mime_type = a.attrib.get('mimeType')
-            for r in a.findall('{urn:mpeg:DASH:schema:MPD:2011}Representation'):
-                url_el = r.find('{urn:mpeg:DASH:schema:MPD:2011}BaseURL')
-                if url_el is None:
-                    continue
+            for r in a.findall(_add_ns('Representation')):
+                mime_type = r.attrib.get('mimeType') or mime_type
+                url_el = r.find(_add_ns('BaseURL'))
                 if mime_type == 'text/vtt':
                     # TODO implement WebVTT downloading
                     pass
                 elif mime_type.startswith('audio/') or mime_type.startswith('video/'):
-                    segment_list = r.find('{urn:mpeg:DASH:schema:MPD:2011}SegmentList')
+                    segment_list = r.find(_add_ns('SegmentList'))
                     format_id = r.attrib['id']
-                    video_url = url_el.text
-                    filesize = int_or_none(url_el.attrib.get('{http://youtube.com/yt/2012/10/10}contentLength'))
+                    video_url = url_el.text if url_el else None
+                    filesize = int_or_none(url_el.attrib.get('{http://youtube.com/yt/2012/10/10}contentLength') if url_el else None)
                     f = {
                         'format_id': format_id,
                         'url': video_url,
@@ -1357,17 +1359,20 @@ class InfoExtractor(object):
                         'fps': int_or_none(r.attrib.get('frameRate')),
                     }
                     if segment_list is not None:
+                        initialization_url = segment_list.find(_add_ns('Initialization')).attrib['sourceURL']
                         f.update({
-                            'initialization_url': segment_list.find('{urn:mpeg:DASH:schema:MPD:2011}Initialization').attrib['sourceURL'],
-                            'segment_urls': [segment.attrib.get('media') for segment in segment_list.findall('{urn:mpeg:DASH:schema:MPD:2011}SegmentURL')],
+                            'initialization_url': initialization_url,
+                            'segment_urls': [segment.attrib.get('media') for segment in segment_list.findall(_add_ns('SegmentURL'))],
                             'protocol': 'http_dash_segments',
                         })
+                        if not f.get('url'):
+                            f['url'] = initialization_url
                     try:
                         existing_format = next(
                             fo for fo in formats
                             if fo['format_id'] == format_id)
                     except StopIteration:
-                        full_info = self._formats.get(format_id, {}).copy()
+                        full_info = formats_dict.get(format_id, {}).copy()
                         full_info.update(f)
                         codecs = r.attrib.get('codecs')
                         if codecs:

From 2c2f1efdcd37f88196bb2b546e6d8bc7c22f3b6a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 30 Jan 2016 19:30:31 +0600
Subject: [PATCH 3816/6219] [downloader/fragment] Remove superfluous whitespace

---
 youtube_dl/downloader/fragment.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
index e2f34eec6..8b96eceb9 100644
--- a/youtube_dl/downloader/fragment.py
+++ b/youtube_dl/downloader/fragment.py
@@ -79,15 +79,13 @@ class FragmentFD(FileDownloader):
                 return
 
             time_now = time.time()
-
+            state['elapsed'] = time_now - start
             frag_total_bytes = s.get('total_bytes') or 0
-
             if not ctx['live']:
                 estimated_size = (
                     (ctx['complete_frags_downloaded_bytes'] + frag_total_bytes) /
                     (state['frag_index'] + 1) * total_frags)
                 state['total_bytes_estimate'] = estimated_size
-            state['elapsed'] = time_now - start
 
             if s['status'] == 'finished':
                 state['frag_index'] += 1

From 0803753fea578355db68ae9f2d915db04cde7557 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 30 Jan 2016 21:31:53 +0800
Subject: [PATCH 3817/6219] [facebook] Add support for DASH manifests

---
 youtube_dl/extractor/facebook.py | 26 ++++++++++++++++++++++++--
 1 file changed, 24 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index 899b0896b..1f3e270d9 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -6,9 +6,11 @@ import socket
 
 from .common import InfoExtractor
 from ..compat import (
+    compat_etree_fromstring,
     compat_http_client,
     compat_urllib_error,
     compat_urllib_parse_unquote,
+    compat_urllib_parse_unquote_plus,
 )
 from ..utils import (
     error_to_compat_str,
@@ -44,6 +46,9 @@ class FacebookIE(InfoExtractor):
     _CHECKPOINT_URL = 'https://www.facebook.com/checkpoint/?next=http%3A%2F%2Ffacebook.com%2Fhome.php&_fb_noscript=1'
     _NETRC_MACHINE = 'facebook'
     IE_NAME = 'facebook'
+
+    _CHROME_USER_AGENT = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.97 Safari/537.36'
+
     _TESTS = [{
         'url': 'https://www.facebook.com/video.php?v=637842556329505&fref=nf',
         'md5': '6a40d33c0eccbb1af76cf0485a052659',
@@ -65,6 +70,15 @@ class FacebookIE(InfoExtractor):
         'expected_warnings': [
             'title'
         ]
+    }, {
+        'note': 'Video with DASH manifest',
+        'url': 'https://www.facebook.com/video.php?v=957955867617029',
+        'info_dict': {
+            'id': '957955867617029',
+            'ext': 'mp4',
+            'title': 'When you post epic content on instagram.com/433 8 million followers, this is ...',
+            'uploader': 'Demy de Zeeuw',
+        },
     }, {
         'url': 'https://www.facebook.com/video.php?v=10204634152394104',
         'only_matching': True,
@@ -147,8 +161,9 @@ class FacebookIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        url = 'https://www.facebook.com/video/video.php?v=%s' % video_id
-        webpage = self._download_webpage(url, video_id)
+        req = sanitized_Request('https://www.facebook.com/video/video.php?v=%s' % video_id)
+        req.add_header('User-Agent', self._CHROME_USER_AGENT)
+        webpage = self._download_webpage(req, video_id)
 
         video_data = None
 
@@ -197,9 +212,16 @@ class FacebookIE(InfoExtractor):
                             'url': src,
                             'preference': -10 if format_id == 'progressive' else 0,
                         })
+            dash_manifest = f[0].get('dash_manifest')
+            if dash_manifest:
+                formats.extend(self._parse_dash_manifest(
+                    video_id, compat_etree_fromstring(compat_urllib_parse_unquote_plus(dash_manifest)),
+                    default_ns='urn:mpeg:dash:schema:mpd:2011'))
         if not formats:
             raise ExtractorError('Cannot find video formats')
 
+        self._sort_formats(formats)
+
         video_title = self._html_search_regex(
             r'<h2\s+[^>]*class="uiHeaderTitle"[^>]*>([^<]*)</h2>', webpage, 'title',
             default=None)

From 5d2c0fd9ba2251a5c91b7d8b1dd285b427a2251e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 30 Jan 2016 21:32:15 +0800
Subject: [PATCH 3818/6219] [youtube] Pass self._formats to
 _parse_dash_manifest

---
 youtube_dl/extractor/youtube.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index acd5cf2c3..c1d570cb3 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1478,7 +1478,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                         errnote='Could not download DASH manifest',
                         fatal=dash_mpd_fatal)
 
-                    for df in self._parse_dash_manifest(video_id, dash_doc, dash_mpd_fatal):
+                    for df in self._parse_dash_manifest(
+                            video_id, dash_doc, formats_dict=self._formats, fatal=dash_mpd_fatal):
                         # Do not overwrite DASH format found in some previous DASH manifest
                         if df['format_id'] not in dash_formats:
                             dash_formats[df['format_id']] = df

From 5ea1eb78f5e8d9b2ede35504dc3b999f5f89bbaa Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 30 Jan 2016 21:36:01 +0800
Subject: [PATCH 3819/6219] [common] Fix for youtube

---
 youtube_dl/extractor/common.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 83628a68f..243db71dc 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1346,8 +1346,8 @@ class InfoExtractor(object):
                 elif mime_type.startswith('audio/') or mime_type.startswith('video/'):
                     segment_list = r.find(_add_ns('SegmentList'))
                     format_id = r.attrib['id']
-                    video_url = url_el.text if url_el else None
-                    filesize = int_or_none(url_el.attrib.get('{http://youtube.com/yt/2012/10/10}contentLength') if url_el else None)
+                    video_url = url_el.text if url_el is not None else None
+                    filesize = int_or_none(url_el.attrib.get('{http://youtube.com/yt/2012/10/10}contentLength') if url_el is not None else None)
                     f = {
                         'format_id': format_id,
                         'url': video_url,

From df374b52228e8a083d045f9bc56847e418ef452e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 30 Jan 2016 21:42:27 +0800
Subject: [PATCH 3820/6219] [common] Prefer the manifest than formats_dict in
 determining codecs

---
 youtube_dl/extractor/common.py | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 243db71dc..f1313ef04 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1376,10 +1376,15 @@ class InfoExtractor(object):
                         full_info.update(f)
                         codecs = r.attrib.get('codecs')
                         if codecs:
-                            if full_info.get('acodec') == 'none':
-                                full_info['vcodec'] = codecs
-                            elif full_info.get('vcodec') == 'none':
-                                full_info['acodec'] = codecs
+                            if mime_type.startswith('video/'):
+                                vcodec, acodec = codecs, 'none'
+                            else:  # mime_type.startswith('audio/')
+                                vcodec, acodec = 'none', codecs
+
+                            full_info.update({
+                                'vcodec': vcodec,
+                                'acodec': acodec,
+                            })
                         formats.append(full_info)
                     else:
                         existing_format.update(f)

From b8c9926c0a9d68fb636bf99e873d6a88de41f76b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 30 Jan 2016 19:43:25 +0600
Subject: [PATCH 3821/6219] [downloader/f4m] Do not update fragment list while
 test

---
 youtube_dl/downloader/f4m.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index 581fa7b71..fc9642905 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -322,7 +322,8 @@ class F4mFD(FragmentFD):
             metadata = None
 
         fragments_list = build_fragments_list(boot_info)
-        if self.params.get('test', False):
+        test = self.params.get('test', False)
+        if test:
             # We only download the first fragment
             fragments_list = fragments_list[:1]
         total_frags = len(fragments_list)
@@ -387,7 +388,7 @@ class F4mFD(FragmentFD):
                 else:
                     raise
 
-            if not fragments_list and live and bootstrap_url:
+            if not fragments_list and not test and live and bootstrap_url:
                 fragments_list = self._update_live_fragments(bootstrap_url, frag_i)
                 total_frags += len(fragments_list)
                 if fragments_list and (fragments_list[0][1] > frag_i + 1):

From a6c2c24479e5f4827ceb06f64d855329c0a6f593 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 30 Jan 2016 22:28:53 +0800
Subject: [PATCH 3822/6219] [youtube] Remove '(v|a)codec': 'none' entries

Not used anymore
---
 youtube_dl/extractor/youtube.py | 80 ++++++++++++++++-----------------
 1 file changed, 40 insertions(+), 40 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index c1d570cb3..f992c3624 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -316,55 +316,55 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         '151': {'ext': 'mp4', 'height': 72, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 24, 'vcodec': 'h264', 'preference': -10},
 
         # DASH mp4 video
-        '133': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'h264', 'preference': -40},
-        '134': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'h264', 'preference': -40},
-        '135': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'h264', 'preference': -40},
-        '136': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'h264', 'preference': -40},
-        '137': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'h264', 'preference': -40},
-        '138': {'ext': 'mp4', 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'h264', 'preference': -40},  # Height can vary (https://github.com/rg3/youtube-dl/issues/4559)
-        '160': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'h264', 'preference': -40},
-        '264': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'h264', 'preference': -40},
-        '298': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'h264', 'fps': 60, 'preference': -40},
-        '299': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'h264', 'fps': 60, 'preference': -40},
-        '266': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'h264', 'preference': -40},
+        '133': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
+        '134': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
+        '135': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
+        '136': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
+        '137': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
+        '138': {'ext': 'mp4', 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},  # Height can vary (https://github.com/rg3/youtube-dl/issues/4559)
+        '160': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
+        '264': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
+        '298': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60, 'preference': -40},
+        '299': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60, 'preference': -40},
+        '266': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
 
         # Dash mp4 audio
-        '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'vcodec': 'none', 'abr': 48, 'preference': -50, 'container': 'm4a_dash'},
-        '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'vcodec': 'none', 'abr': 128, 'preference': -50, 'container': 'm4a_dash'},
-        '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'vcodec': 'none', 'abr': 256, 'preference': -50, 'container': 'm4a_dash'},
+        '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 48, 'preference': -50, 'container': 'm4a_dash'},
+        '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 128, 'preference': -50, 'container': 'm4a_dash'},
+        '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 256, 'preference': -50, 'container': 'm4a_dash'},
 
         # Dash webm
-        '167': {'ext': 'webm', 'height': 360, 'width': 640, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
-        '168': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
-        '169': {'ext': 'webm', 'height': 720, 'width': 1280, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
-        '170': {'ext': 'webm', 'height': 1080, 'width': 1920, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
-        '218': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
-        '219': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
-        '278': {'ext': 'webm', 'height': 144, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'vp9', 'preference': -40},
-        '242': {'ext': 'webm', 'height': 240, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'vp9', 'preference': -40},
-        '243': {'ext': 'webm', 'height': 360, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'vp9', 'preference': -40},
-        '244': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'vp9', 'preference': -40},
-        '245': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'vp9', 'preference': -40},
-        '246': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'vp9', 'preference': -40},
-        '247': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'vp9', 'preference': -40},
-        '248': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'vp9', 'preference': -40},
-        '271': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'vp9', 'preference': -40},
+        '167': {'ext': 'webm', 'height': 360, 'width': 640, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
+        '168': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
+        '169': {'ext': 'webm', 'height': 720, 'width': 1280, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
+        '170': {'ext': 'webm', 'height': 1080, 'width': 1920, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
+        '218': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
+        '219': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
+        '278': {'ext': 'webm', 'height': 144, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp9', 'preference': -40},
+        '242': {'ext': 'webm', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40},
+        '243': {'ext': 'webm', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40},
+        '244': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40},
+        '245': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40},
+        '246': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40},
+        '247': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40},
+        '248': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40},
+        '271': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40},
         # itag 272 videos are either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug)
-        '272': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'vp9', 'preference': -40},
-        '302': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'vp9', 'fps': 60, 'preference': -40},
-        '303': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'vp9', 'fps': 60, 'preference': -40},
-        '308': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'vp9', 'fps': 60, 'preference': -40},
-        '313': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'vp9', 'preference': -40},
-        '315': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'vcodec': 'vp9', 'fps': 60, 'preference': -40},
+        '272': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40},
+        '302': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60, 'preference': -40},
+        '303': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60, 'preference': -40},
+        '308': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60, 'preference': -40},
+        '313': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40},
+        '315': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60, 'preference': -40},
 
         # Dash webm audio
-        '171': {'ext': 'webm', 'acodec': 'vorbis', 'vcodec': 'none', 'format_note': 'DASH audio', 'abr': 128, 'preference': -50},
-        '172': {'ext': 'webm', 'acodec': 'vorbis', 'vcodec': 'none', 'format_note': 'DASH audio', 'abr': 256, 'preference': -50},
+        '171': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 128, 'preference': -50},
+        '172': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 256, 'preference': -50},
 
         # Dash webm audio with opus inside
-        '249': {'ext': 'webm', 'vcodec': 'none', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 50, 'preference': -50},
-        '250': {'ext': 'webm', 'vcodec': 'none', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 70, 'preference': -50},
-        '251': {'ext': 'webm', 'vcodec': 'none', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 160, 'preference': -50},
+        '249': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 50, 'preference': -50},
+        '250': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 70, 'preference': -50},
+        '251': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 160, 'preference': -50},
 
         # RTMP (unnamed)
         '_rtmp': {'protocol': 'rtmp'},

From 16f38a699f7c6d2820720d704a2373ba28c568b1 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 30 Jan 2016 22:40:13 +0800
Subject: [PATCH 3823/6219] [common] Rename to namespace

For consistency with _parse_smil_*
---
 youtube_dl/extractor/common.py   | 6 +++---
 youtube_dl/extractor/facebook.py | 2 +-
 youtube_dl/extractor/youtube.py  | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index f1313ef04..a05efec9e 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1330,9 +1330,9 @@ class InfoExtractor(object):
             })
         return entries
 
-    def _parse_dash_manifest(self, video_id, dash_doc, default_ns='urn:mpeg:DASH:schema:MPD:2011', formats_dict={}, fatal=True):
-        def _add_ns(tag):
-            return '{%s}%s' % (default_ns, tag)
+    def _parse_dash_manifest(self, video_id, dash_doc, namespace=None, formats_dict={}, fatal=True):
+        def _add_ns(path):
+            return self._xpath_ns(path, namespace)
 
         formats = []
         for a in dash_doc.findall('.//' + _add_ns('AdaptationSet')):
diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index 1f3e270d9..af38eaff1 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -216,7 +216,7 @@ class FacebookIE(InfoExtractor):
             if dash_manifest:
                 formats.extend(self._parse_dash_manifest(
                     video_id, compat_etree_fromstring(compat_urllib_parse_unquote_plus(dash_manifest)),
-                    default_ns='urn:mpeg:dash:schema:mpd:2011'))
+                    namespace='urn:mpeg:dash:schema:mpd:2011'))
         if not formats:
             raise ExtractorError('Cannot find video formats')
 
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index f992c3624..8d3fcfa5f 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1479,7 +1479,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                         fatal=dash_mpd_fatal)
 
                     for df in self._parse_dash_manifest(
-                            video_id, dash_doc, formats_dict=self._formats, fatal=dash_mpd_fatal):
+                            video_id, dash_doc, namespace='urn:mpeg:DASH:schema:MPD:2011', formats_dict=self._formats, fatal=dash_mpd_fatal):
                         # Do not overwrite DASH format found in some previous DASH manifest
                         if df['format_id'] not in dash_formats:
                             dash_formats[df['format_id']] = df

From c94678957fbe4483b2b7c8b3e6824cb7a215d42d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 30 Jan 2016 22:45:16 +0800
Subject: [PATCH 3824/6219] [common] Remove unused arguments

---
 youtube_dl/extractor/common.py   | 2 +-
 youtube_dl/extractor/facebook.py | 2 +-
 youtube_dl/extractor/youtube.py  | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index a05efec9e..5a2b7a721 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1330,7 +1330,7 @@ class InfoExtractor(object):
             })
         return entries
 
-    def _parse_dash_manifest(self, video_id, dash_doc, namespace=None, formats_dict={}, fatal=True):
+    def _parse_dash_manifest(self, dash_doc, namespace=None, formats_dict={}):
         def _add_ns(path):
             return self._xpath_ns(path, namespace)
 
diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index af38eaff1..bd65e43f2 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -215,7 +215,7 @@ class FacebookIE(InfoExtractor):
             dash_manifest = f[0].get('dash_manifest')
             if dash_manifest:
                 formats.extend(self._parse_dash_manifest(
-                    video_id, compat_etree_fromstring(compat_urllib_parse_unquote_plus(dash_manifest)),
+                    compat_etree_fromstring(compat_urllib_parse_unquote_plus(dash_manifest)),
                     namespace='urn:mpeg:dash:schema:mpd:2011'))
         if not formats:
             raise ExtractorError('Cannot find video formats')
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 8d3fcfa5f..a2f776050 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1479,7 +1479,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                         fatal=dash_mpd_fatal)
 
                     for df in self._parse_dash_manifest(
-                            video_id, dash_doc, namespace='urn:mpeg:DASH:schema:MPD:2011', formats_dict=self._formats, fatal=dash_mpd_fatal):
+                            dash_doc, namespace='urn:mpeg:DASH:schema:MPD:2011', formats_dict=self._formats):
                         # Do not overwrite DASH format found in some previous DASH manifest
                         if df['format_id'] not in dash_formats:
                             dash_formats[df['format_id']] = df

From 2d2fa82d172a10a49fb5449fa35bc409de778f05 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 30 Jan 2016 22:52:23 +0800
Subject: [PATCH 3825/6219] [common] Add _extract_dash_manifest_formats

---
 youtube_dl/extractor/common.py  | 15 +++++++++++++++
 youtube_dl/extractor/youtube.py | 10 +++-------
 2 files changed, 18 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 5a2b7a721..199a04d1c 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1330,6 +1330,21 @@ class InfoExtractor(object):
             })
         return entries
 
+    def _download_dash_manifest(self, dash_manifest_url, video_id, fatal=True):
+        return self._download_xml(
+            dash_manifest_url, video_id,
+            note='Downloading DASH manifest',
+            errnote='Could not download DASH manifest',
+            fatal=fatal)
+
+    def _extract_dash_manifest_formats(self, dash_manifest_url, video_id, fatal=True, namespace=None, formats_dict={}):
+        dash_doc = self._download_dash_manifest(dash_manifest_url, video_id, fatal)
+        if dash_doc is False:
+            return []
+
+        return self._parse_dash_manifest(
+            dash_doc, namespace=namespace, formats_dict=formats_dict)
+
     def _parse_dash_manifest(self, dash_doc, namespace=None, formats_dict={}):
         def _add_ns(path):
             return self._xpath_ns(path, namespace)
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index a2f776050..d6fef39e9 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1472,14 +1472,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                         return '/signature/%s' % dec_s
 
                     dash_manifest_url = re.sub(r'/s/([a-fA-F0-9\.]+)', decrypt_sig, dash_manifest_url)
-                    dash_doc = self._download_xml(
-                        dash_manifest_url, video_id,
-                        note='Downloading DASH manifest',
-                        errnote='Could not download DASH manifest',
-                        fatal=dash_mpd_fatal)
 
-                    for df in self._parse_dash_manifest(
-                            dash_doc, namespace='urn:mpeg:DASH:schema:MPD:2011', formats_dict=self._formats):
+                    for df in self._extract_dash_manifest_formats(
+                            dash_manifest_url, video_id, fatal=dash_mpd_fatal,
+                            namespace='urn:mpeg:DASH:schema:MPD:2011', formats_dict=self._formats):
                         # Do not overwrite DASH format found in some previous DASH manifest
                         if df['format_id'] not in dash_formats:
                             dash_formats[df['format_id']] = df

From 248ae880b6900a753e38253f149c013c3f95c0c5 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 30 Jan 2016 23:01:19 +0800
Subject: [PATCH 3826/6219] [facebook] Add md5 for the test case with DASH

---
 youtube_dl/extractor/facebook.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index bd65e43f2..b6d1180f0 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -73,6 +73,7 @@ class FacebookIE(InfoExtractor):
     }, {
         'note': 'Video with DASH manifest',
         'url': 'https://www.facebook.com/video.php?v=957955867617029',
+        'md5': '54706e4db4f5ad58fbad82dde1f1213f',
         'info_dict': {
             'id': '957955867617029',
             'ext': 'mp4',

From 1bf996fa5cd996c5f586089388853e324dc3850d Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 30 Jan 2016 20:45:56 +0100
Subject: [PATCH 3827/6219] [generic] Add support for Limelight API

---
 youtube_dl/extractor/generic.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 26d3698c8..b18e734c4 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1819,6 +1819,17 @@ class GenericIE(InfoExtractor):
         if digiteka_url:
             return self.url_result(self._proto_relative_url(digiteka_url), DigitekaIE.ie_key())
 
+        # Look for Limelight embeds
+        mobj = re.search(r'LimelightPlayer\.doLoad(Media|Channel|ChannelList)\(["\'](?P<id>[a-z0-9]{32})', webpage)
+        if mobj:
+            lm = {
+                'Media': 'media',
+                'Channel': 'channel',
+                'ChannelList': 'channel_list',
+            }
+            return self.url_result('limelight:%s:%s' % (
+                lm[mobj.group(1)], mobj.group(2)), 'Limelight%s' % mobj.group(1), mobj.group(2))
+
         # Look for AdobeTVVideo embeds
         mobj = re.search(
             r'<iframe[^>]+src=[\'"]((?:https?:)?//video\.tv\.adobe\.com/v/\d+[^"]+)[\'"]',

From 8f1fddc816dac7c0e9b216e85474aa8eb0d847bf Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 30 Jan 2016 20:51:47 +0100
Subject: [PATCH 3828/6219] [limelight] fix format sorting and make m3u8 and
 f4m extraction non fatal

---
 youtube_dl/extractor/limelight.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/limelight.py b/youtube_dl/extractor/limelight.py
index fb03dd527..1a0625ac3 100644
--- a/youtube_dl/extractor/limelight.py
+++ b/youtube_dl/extractor/limelight.py
@@ -40,7 +40,8 @@ class LimelightBaseIE(InfoExtractor):
             if not stream_url:
                 continue
             if '.f4m' in stream_url:
-                formats.extend(self._extract_f4m_formats(stream_url, video_id))
+                formats.extend(self._extract_f4m_formats(
+                    stream_url, video_id, fatal=False))
             else:
                 fmt = {
                     'url': stream_url,
@@ -72,8 +73,8 @@ class LimelightBaseIE(InfoExtractor):
             format_id = mobile_url.get('targetMediaPlatform')
             if determine_ext(media_url) == 'm3u8':
                 formats.extend(self._extract_m3u8_formats(
-                    media_url, video_id, 'mp4', entry_protocol='m3u8_native',
-                    preference=-1, m3u8_id=format_id))
+                    media_url, video_id, 'mp4', 'm3u8_native',
+                    m3u8_id=format_id, fatal=False))
             else:
                 formats.append({
                     'url': media_url,

From a7685b3a6b02be62ebd53e2978ab43b1cde3ec4b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 31 Jan 2016 02:38:28 +0600
Subject: [PATCH 3829/6219] [npo] Add extension for m3u8

---
 youtube_dl/extractor/npo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index eb12fb810..7cd5c9303 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -189,7 +189,7 @@ class NPOIE(NPOBaseIE):
                 if not video_url:
                     continue
                 if format_id == 'adaptive':
-                    formats.extend(self._extract_m3u8_formats(video_url, video_id))
+                    formats.extend(self._extract_m3u8_formats(video_url, video_id, 'mp4'))
                 else:
                     formats.append({
                         'url': video_url,

From 60ad3eb9706861d4182ab44ee19d64350ca2e36e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 31 Jan 2016 03:21:33 +0800
Subject: [PATCH 3830/6219] [viidea] Skip download for the test case requiring
 ffmpeg

---
 youtube_dl/extractor/viidea.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/viidea.py b/youtube_dl/extractor/viidea.py
index 525e303d4..315984bf9 100644
--- a/youtube_dl/extractor/viidea.py
+++ b/youtube_dl/extractor/viidea.py
@@ -45,6 +45,10 @@ class ViideaIE(InfoExtractor):
             'upload_date': '20130627',
             'duration': 565,
         },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
     }, {
         # video with invalid direct format links (HTTP 403)
         'url': 'http://videolectures.net/russir2010_filippova_nlp/',

From 7c38af48b91d050a5694e878a789a028fbcc0a3b Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 31 Jan 2016 03:31:03 +0800
Subject: [PATCH 3831/6219] [vgtv] Fix test_VGTV_2

---
 youtube_dl/extractor/vgtv.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vgtv.py b/youtube_dl/extractor/vgtv.py
index 86ba70ed9..14e945d49 100644
--- a/youtube_dl/extractor/vgtv.py
+++ b/youtube_dl/extractor/vgtv.py
@@ -86,10 +86,9 @@ class VGTVIE(XstreamIE):
         {
             # streamType: wasLive
             'url': 'http://www.vgtv.no/#!/live/113063/direkte-v75-fra-solvalla',
-            'md5': '458f4841239dab414343b50e5af8869c',
             'info_dict': {
                 'id': '113063',
-                'ext': 'flv',
+                'ext': 'mp4',
                 'title': 'V75 fra Solvalla 30.05.15',
                 'description': 'md5:b3743425765355855f88e096acc93231',
                 'thumbnail': 're:^https?://.*\.jpg',
@@ -98,6 +97,10 @@ class VGTVIE(XstreamIE):
                 'upload_date': '20150530',
                 'view_count': int,
             },
+            'params': {
+                # m3u8 download
+                'skip_download': True,
+            },
         },
         {
             'url': 'http://www.aftenposten.no/webtv/#!/video/21039/trailer-sweatshop-i-can-t-take-any-more',

From 809e1857c5155fb41d033b3c0215708eb41ddb55 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 31 Jan 2016 03:38:20 +0800
Subject: [PATCH 3832/6219] [screenwavemedia] Fix HLS extension and
 test_TeamFour

---
 youtube_dl/extractor/screenwavemedia.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/screenwavemedia.py b/youtube_dl/extractor/screenwavemedia.py
index 05f93904c..e5d62a139 100644
--- a/youtube_dl/extractor/screenwavemedia.py
+++ b/youtube_dl/extractor/screenwavemedia.py
@@ -71,7 +71,7 @@ class ScreenwaveMediaIE(InfoExtractor):
         formats = []
         for source in sources:
             if source['type'] == 'hls':
-                formats.extend(self._extract_m3u8_formats(source['file'], video_id))
+                formats.extend(self._extract_m3u8_formats(source['file'], video_id, ext='mp4'))
             else:
                 file_ = source.get('file')
                 if not file_:
@@ -107,7 +107,11 @@ class TeamFourIE(InfoExtractor):
             'upload_date': '20130401',
             'description': 'Check out this and more on our website: http://teamfourstar.com\nTFS Store: http://sharkrobot.com/team-four-star\nFollow on Twitter: http://twitter.com/teamfourstar\nLike on FB: http://facebook.com/teamfourstar',
             'title': 'A Moment With TFS Episode 4',
-        }
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
     }
 
     def _real_extract(self, url):

From ed1a390583f7adfcf38c4af5432b4b2e611004ab Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 31 Jan 2016 03:42:34 +0800
Subject: [PATCH 3833/6219] [tv2] Fix test_TV2

---
 youtube_dl/extractor/tv2.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/tv2.py b/youtube_dl/extractor/tv2.py
index fa338b936..1457e524e 100644
--- a/youtube_dl/extractor/tv2.py
+++ b/youtube_dl/extractor/tv2.py
@@ -17,18 +17,21 @@ class TV2IE(InfoExtractor):
     _VALID_URL = 'http://(?:www\.)?tv2\.no/v/(?P<id>\d+)'
     _TEST = {
         'url': 'http://www.tv2.no/v/916509/',
-        'md5': '9cb9e3410b18b515d71892f27856e9b1',
         'info_dict': {
             'id': '916509',
-            'ext': 'flv',
-            'title': 'Se Gryttens hyllest av Steven Gerrard',
+            'ext': 'mp4',
+            'title': 'Se Frode Gryttens hyllest av Steven Gerrard',
             'description': 'TV 2 Sportens huspoet tar avskjed med Liverpools kaptein Steven Gerrard.',
             'timestamp': 1431715610,
             'upload_date': '20150515',
             'duration': 156.967,
             'view_count': int,
             'categories': list,
-        }
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
     }
 
     def _real_extract(self, url):

From eb6fc7d32a6f909496cc1a4b46fdcbf2cc6f344e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 31 Jan 2016 03:45:09 +0800
Subject: [PATCH 3834/6219] [senateisvp] Fix test_SenateISVP and
 test_SenateISVP_1

---
 youtube_dl/extractor/senateisvp.py | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/senateisvp.py b/youtube_dl/extractor/senateisvp.py
index 474ebb49b..990ea0fa8 100644
--- a/youtube_dl/extractor/senateisvp.py
+++ b/youtube_dl/extractor/senateisvp.py
@@ -53,17 +53,25 @@ class SenateISVPIE(InfoExtractor):
         'url': 'http://www.senate.gov/isvp/?comm=judiciary&type=live&stt=&filename=judiciary031715&auto_play=false&wmode=transparent&poster=http%3A%2F%2Fwww.judiciary.senate.gov%2Fthemes%2Fjudiciary%2Fimages%2Fvideo-poster-flash-fit.png',
         'info_dict': {
             'id': 'judiciary031715',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Integrated Senate Video Player',
             'thumbnail': 're:^https?://.*\.(?:jpg|png)$',
-        }
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
     }, {
         'url': 'http://www.senate.gov/isvp/?type=live&comm=commerce&filename=commerce011514.mp4&auto_play=false',
         'info_dict': {
             'id': 'commerce011514',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Integrated Senate Video Player'
-        }
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
     }, {
         'url': 'http://www.senate.gov/isvp/?type=arch&comm=intel&filename=intel090613&hc_location=ufi',
         # checksum differs each time

From 670ad51ade60141a7948aa68686db49dd2e30145 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 31 Jan 2016 04:01:45 +0800
Subject: [PATCH 3835/6219] [nrktv] Fix _TESTS

---
 youtube_dl/extractor/nrk.py | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index 6ff13050d..a126f5054 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -133,26 +133,32 @@ class NRKTVIE(InfoExtractor):
     _TESTS = [
         {
             'url': 'https://tv.nrk.no/serie/20-spoersmaal-tv/MUHH48000314/23-05-2014',
-            'md5': 'adf2c5454fa2bf032f47a9f8fb351342',
             'info_dict': {
                 'id': 'MUHH48000314',
-                'ext': 'flv',
+                'ext': 'mp4',
                 'title': '20 spørsmål',
                 'description': 'md5:bdea103bc35494c143c6a9acdd84887a',
                 'upload_date': '20140523',
                 'duration': 1741.52,
             },
+            'params': {
+                # m3u8 download
+                'skip_download': True,
+            },
         },
         {
             'url': 'https://tv.nrk.no/program/mdfp15000514',
-            'md5': '383650ece2b25ecec996ad7b5bb2a384',
             'info_dict': {
                 'id': 'mdfp15000514',
-                'ext': 'flv',
-                'title': 'Kunnskapskanalen: Grunnlovsjubiléet - Stor ståhei for ingenting',
+                'ext': 'mp4',
+                'title': 'Grunnlovsjubiléet - Stor ståhei for ingenting',
                 'description': 'md5:654c12511f035aed1e42bdf5db3b206a',
                 'upload_date': '20140524',
-                'duration': 4605.0,
+                'duration': 4605.08,
+            },
+            'params': {
+                # m3u8 download
+                'skip_download': True,
             },
         },
         {

From e881c4bcabb64f84e382a75bd9c03379859105d0 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 31 Jan 2016 04:34:46 +0800
Subject: [PATCH 3836/6219] [nbc] Use NBC's id and fix _TESTS

ThePlatform URL gives the same ID for all _TESTS
---
 youtube_dl/extractor/nbc.py | 27 +++++++++++++++++++--------
 1 file changed, 19 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index 1dd54c2f1..18d01f423 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -19,32 +19,39 @@ class NBCIE(InfoExtractor):
     _TESTS = [
         {
             'url': 'http://www.nbc.com/the-tonight-show/segments/112966',
-            # md5 checksum is not stable
             'info_dict': {
-                'id': 'c9xnCo0YPOPH',
-                'ext': 'flv',
+                'id': '112966',
+                'ext': 'mp4',
                 'title': 'Jimmy Fallon Surprises Fans at Ben & Jerry\'s',
                 'description': 'Jimmy gives out free scoops of his new "Tonight Dough" ice cream flavor by surprising customers at the Ben & Jerry\'s scoop shop.',
             },
+            'params': {
+                # m3u8 download
+                'skip_download': True,
+            },
         },
         {
             'url': 'http://www.nbc.com/the-tonight-show/episodes/176',
             'info_dict': {
-                'id': 'XwU9KZkp98TH',
+                'id': '176',
                 'ext': 'flv',
                 'title': 'Ricky Gervais, Steven Van Zandt, ILoveMakonnen',
                 'description': 'A brand new episode of The Tonight Show welcomes Ricky Gervais, Steven Van Zandt and ILoveMakonnen.',
             },
-            'skip': 'Only works from US',
+            'skip': '404 Not Found',
         },
         {
             'url': 'http://www.nbc.com/saturday-night-live/video/star-wars-teaser/2832821',
             'info_dict': {
-                'id': '8iUuyzWDdYUZ',
-                'ext': 'flv',
+                'id': '2832821',
+                'ext': 'mp4',
                 'title': 'Star Wars Teaser',
                 'description': 'md5:0b40f9cbde5b671a7ff62fceccc4f442',
             },
+            'params': {
+                # m3u8 download
+                'skip_download': True,
+            },
             'skip': 'Only works from US',
         },
         {
@@ -66,7 +73,11 @@ class NBCIE(InfoExtractor):
             webpage, 'theplatform url').replace('_no_endcard', '').replace('\\/', '/')))
         if theplatform_url.startswith('//'):
             theplatform_url = 'http:' + theplatform_url
-        return self.url_result(smuggle_url(theplatform_url, {'source_url': url}))
+        return {
+            '_type': 'url_transparent',
+            'url': smuggle_url(theplatform_url, {'source_url': url}),
+            'id': video_id,
+        }
 
 
 class NBCSportsVPlayerIE(InfoExtractor):

From db9b1dbcd918bee41b3b38f1f1106ad5dec07bb7 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 31 Jan 2016 04:49:04 +0800
Subject: [PATCH 3837/6219] [nba] Add ext for hls formats and fix test_NBA

---
 youtube_dl/extractor/nba.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/nba.py b/youtube_dl/extractor/nba.py
index 9d26030d3..a071378b6 100644
--- a/youtube_dl/extractor/nba.py
+++ b/youtube_dl/extractor/nba.py
@@ -18,13 +18,17 @@ class NBAIE(InfoExtractor):
         'md5': '9e7729d3010a9c71506fd1248f74e4f4',
         'info_dict': {
             'id': '0021200253-okc-bkn-recap',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Thunder vs. Nets',
             'description': 'Kevin Durant scores 32 points and dishes out six assists as the Thunder beat the Nets in Brooklyn.',
             'duration': 181,
             'timestamp': 1354638466,
             'upload_date': '20121204',
         },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
     }, {
         'url': 'http://www.nba.com/video/games/hornets/2014/12/05/0021400276-nyk-cha-play5.nba/',
         'only_matching': True,
@@ -68,7 +72,7 @@ class NBAIE(InfoExtractor):
             if video_url.startswith('/'):
                 continue
             if video_url.endswith('.m3u8'):
-                formats.extend(self._extract_m3u8_formats(video_url, video_id, m3u8_id='hls', fatal=False))
+                formats.extend(self._extract_m3u8_formats(video_url, video_id, ext='mp4', m3u8_id='hls', fatal=False))
             elif video_url.endswith('.f4m'):
                 formats.extend(self._extract_f4m_formats(video_url + '?hdcore=3.4.1.1', video_id, f4m_id='hds', fatal=False))
             else:

From 89f2602880198b94f98bc41109771a52105e3cf8 Mon Sep 17 00:00:00 2001
From: rrooij <rderooij685@gmail.com>
Date: Sat, 30 Jan 2016 20:31:31 +0100
Subject: [PATCH 3838/6219] [schooltv] Add extractor for SchoolTV playlists

This closes #8163
---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/npo.py      | 31 +++++++++++++++++++++++++++++++
 2 files changed, 32 insertions(+)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index e15495ec8..dbdfb86c0 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -483,6 +483,7 @@ from .npo import (
     NPOLiveIE,
     NPORadioIE,
     NPORadioFragmentIE,
+    SchoolTVIE,
     VPROIE,
     WNLIE
 )
diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index eb12fb810..ab418edf4 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -406,6 +406,37 @@ class NPORadioFragmentIE(InfoExtractor):
         }
 
 
+class SchoolTVIE(InfoExtractor):
+    IE_NAME = 'schooltv'
+    _VALID_URL = r'https?://(?:www\.)?schooltv\.nl/video/(?P<id>[^/?#&]+)'
+
+    _TEST = {
+        'url': 'http://www.schooltv.nl/video/ademhaling-de-hele-dag-haal-je-adem-maar-wat-gebeurt-er-dan-eigenlijk-in-je-lichaam/',
+        'info_dict': {
+            'id': 'WO_NTR_429477',
+            'display_id': 'ademhaling-de-hele-dag-haal-je-adem-maar-wat-gebeurt-er-dan-eigenlijk-in-je-lichaam',
+            'title': 'Ademhaling: De hele dag haal je adem. Maar wat gebeurt er dan eigenlijk in je lichaam?',
+            'ext': 'mp4',
+            'description': 'md5:abfa0ff690adb73fd0297fd033aaa631'
+        },
+        'params': {
+            # Skip because of m3u8 download
+            'skip_download': True
+        }
+    }
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+        video_id = self._search_regex(r'data-mid="([^"]+)"', webpage, 'video_id')
+        return {
+            '_type': 'url_transparent',
+            'ie_key': 'NPO',
+            'url': 'npo:%s' % video_id,
+            'display_id': display_id
+        }
+
+
 class VPROIE(NPOIE):
     IE_NAME = 'vpro'
     _VALID_URL = r'https?://(?:www\.)?(?:tegenlicht\.)?vpro\.nl/(?:[^/]+/){2,}(?P<id>[^/]+)\.html'

From 673fb82e65e28935ad16d85771cefe15013a12bd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 31 Jan 2016 04:41:18 +0600
Subject: [PATCH 3839/6219] [schooltv] Improve video id regex

---
 youtube_dl/extractor/npo.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index a4363c16e..87f5675c7 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -428,7 +428,8 @@ class SchoolTVIE(InfoExtractor):
     def _real_extract(self, url):
         display_id = self._match_id(url)
         webpage = self._download_webpage(url, display_id)
-        video_id = self._search_regex(r'data-mid="([^"]+)"', webpage, 'video_id')
+        video_id = self._search_regex(
+            r'data-mid=(["\'])(?P<id>.+?)\1', webpage, 'video_id', group='id')
         return {
             '_type': 'url_transparent',
             'ie_key': 'NPO',

From 14823decf3f27d5e7f17abf85444575f7600f4e4 Mon Sep 17 00:00:00 2001
From: rrooij <rderooij685@gmail.com>
Date: Sun, 31 Jan 2016 00:03:23 +0100
Subject: [PATCH 3840/6219] [Gamekings] Fix url from .tv to .nl

Gamekings doesn't use the .tv top level domain anymore, but the regular
domain for Dutch sites.
---
 youtube_dl/extractor/gamekings.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/gamekings.py b/youtube_dl/extractor/gamekings.py
index 027f55eb2..e7747a3ae 100644
--- a/youtube_dl/extractor/gamekings.py
+++ b/youtube_dl/extractor/gamekings.py
@@ -9,9 +9,9 @@ from ..utils import (
 
 
 class GamekingsIE(InfoExtractor):
-    _VALID_URL = r'http://www\.gamekings\.tv/(?:videos|nieuws)/(?P<id>[^/]+)'
+    _VALID_URL = r'http://www\.gamekings\.nl/(?:videos|nieuws)/(?P<id>[^/]+)'
     _TESTS = [{
-        'url': 'http://www.gamekings.tv/videos/phoenix-wright-ace-attorney-dual-destinies-review/',
+        'url': 'http://www.gamekings.nl/videos/phoenix-wright-ace-attorney-dual-destinies-review/',
         # MD5 is flaky, seems to change regularly
         # 'md5': '2f32b1f7b80fdc5cb616efb4f387f8a3',
         'info_dict': {
@@ -23,7 +23,7 @@ class GamekingsIE(InfoExtractor):
         },
     }, {
         # vimeo video
-        'url': 'http://www.gamekings.tv/videos/the-legend-of-zelda-majoras-mask/',
+        'url': 'http://www.gamekings.nl/videos/the-legend-of-zelda-majoras-mask/',
         'md5': '12bf04dfd238e70058046937657ea68d',
         'info_dict': {
             'id': 'the-legend-of-zelda-majoras-mask',
@@ -33,7 +33,7 @@ class GamekingsIE(InfoExtractor):
             'thumbnail': 're:^https?://.*\.jpg$',
         },
     }, {
-        'url': 'http://www.gamekings.tv/nieuws/gamekings-extra-shelly-en-david-bereiden-zich-voor-op-de-livestream/',
+        'url': 'http://www.gamekings.nl/nieuws/gamekings-extra-shelly-en-david-bereiden-zich-voor-op-de-livestream/',
         'only_matching': True,
     }]
 

From 7b7507d6e166bcbad4d42b7fac7608b2db276e3f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 31 Jan 2016 07:13:21 +0800
Subject: [PATCH 3841/6219] [letv] Fix LetvCloud extraction

---
 youtube_dl/extractor/letv.py | 61 ++++++++++++++++++++++++++++--------
 1 file changed, 48 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/letv.py b/youtube_dl/extractor/letv.py
index 08bdae8a2..9665ece89 100644
--- a/youtube_dl/extractor/letv.py
+++ b/youtube_dl/extractor/letv.py
@@ -5,11 +5,13 @@ import datetime
 import re
 import time
 import base64
+import hashlib
 
 from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse,
     compat_ord,
+    compat_str,
 )
 from ..utils import (
     determine_ext,
@@ -258,6 +260,7 @@ class LetvCloudIE(InfoExtractor):
         },
     }, {
         'url': 'http://yuntv.letv.com/bcloud.html?uu=p7jnfw5hw9&vu=ec93197892&pu=2c7cd40209&auto_play=1&gpcflag=1&width=640&height=360',
+        'md5': 'e03d9cc8d9c13191e1caf277e42dbd31',
         'info_dict': {
             'id': 'p7jnfw5hw9_ec93197892',
             'ext': 'mp4',
@@ -265,6 +268,7 @@ class LetvCloudIE(InfoExtractor):
         },
     }, {
         'url': 'http://yuntv.letv.com/bcloud.html?uu=p7jnfw5hw9&vu=187060b6fd',
+        'md5': 'cb988699a776b22d4a41b9d43acfb3ac',
         'info_dict': {
             'id': 'p7jnfw5hw9_187060b6fd',
             'ext': 'mp4',
@@ -272,21 +276,37 @@ class LetvCloudIE(InfoExtractor):
         },
     }]
 
-    def _real_extract(self, url):
-        uu_mobj = re.search('uu=([\w]+)', url)
-        vu_mobj = re.search('vu=([\w]+)', url)
+    @staticmethod
+    def sign_data(obj):
+        if obj['cf'] == 'flash':
+            salt = '2f9d6924b33a165a6d8b5d3d42f4f987'
+            items = ['cf', 'format', 'ran', 'uu', 'ver', 'vu']
+        elif obj['cf'] == 'html5':
+            salt = 'fbeh5player12c43eccf2bec3300344'
+            items = ['cf', 'ran', 'uu', 'bver', 'vu']
+        input_data = ''.join([item + obj[item] for item in items]) + salt
+        obj['sign'] = hashlib.md5(input_data.encode('utf-8')).hexdigest()
 
-        if not uu_mobj or not vu_mobj:
-            raise ExtractorError('Invalid URL: %s' % url, expected=True)
+    def _get_formats(self, cf, uu, vu, media_id):
+        def get_play_json(cf, timestamp):
+            data = {
+                'cf': cf,
+                'ver': '2.2',
+                'bver': 'firefox44.0',
+                'format': 'json',
+                'uu': uu,
+                'vu': vu,
+                'ran': compat_str(timestamp),
+            }
+            self.sign_data(data)
+            return self._download_json(
+                'http://api.letvcloud.com/gpc.php?' + compat_urllib_parse.urlencode(data),
+                media_id, 'Downloading playJson data for type %s' % cf)
 
-        uu = uu_mobj.group(1)
-        vu = vu_mobj.group(1)
-        media_id = uu + '_' + vu
-
-        play_json_req = sanitized_Request(
-            'http://api.letvcloud.com/gpc.php?cf=html5&sign=signxxxxx&ver=2.2&format=json&' +
-            'uu=' + uu + '&vu=' + vu)
-        play_json = self._download_json(play_json_req, media_id, 'Downloading playJson data')
+        play_json = get_play_json(cf, time.time())
+        # The server time may be different from local time
+        if play_json.get('code') == 10071:
+            play_json = get_play_json(cf, play_json['timestamp'])
 
         if not play_json.get('data'):
             if play_json.get('message'):
@@ -312,6 +332,21 @@ class LetvCloudIE(InfoExtractor):
                 'width': int_or_none(play_url.get('vwidth')),
                 'height': int_or_none(play_url.get('vheight')),
             })
+
+        return formats
+
+    def _real_extract(self, url):
+        uu_mobj = re.search('uu=([\w]+)', url)
+        vu_mobj = re.search('vu=([\w]+)', url)
+
+        if not uu_mobj or not vu_mobj:
+            raise ExtractorError('Invalid URL: %s' % url, expected=True)
+
+        uu = uu_mobj.group(1)
+        vu = vu_mobj.group(1)
+        media_id = uu + '_' + vu
+
+        formats = self._get_formats('flash', uu, vu, media_id) + self._get_formats('html5', uu, vu, media_id)
         self._sort_formats(formats)
 
         return {

From ce5879fa1475d9b0a74bd4e644ea944083177b2d Mon Sep 17 00:00:00 2001
From: rrooij <rderooij685@gmail.com>
Date: Sun, 31 Jan 2016 00:12:45 +0100
Subject: [PATCH 3842/6219] [Gamekings] Fix viewing of old videos

Some old videos that aren't on Vimeo are being uploaded to YouTube under the
'Gamekings Vault' channel. They use YouTube now for some videos as video
hosting instead of Vimeo or their own hosting. The first test failed to
succeed under the existing code, but works now by using the YouTube
extractor.

The Regex is changed to find the new gogoVideo JavaScript line with the
YouTube embed. Checking if there is a YouTube embed is done by a String
find, which is probably not the best method of checking this.
---
 youtube_dl/extractor/gamekings.py | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/gamekings.py b/youtube_dl/extractor/gamekings.py
index e7747a3ae..efe31c10d 100644
--- a/youtube_dl/extractor/gamekings.py
+++ b/youtube_dl/extractor/gamekings.py
@@ -15,11 +15,14 @@ class GamekingsIE(InfoExtractor):
         # MD5 is flaky, seems to change regularly
         # 'md5': '2f32b1f7b80fdc5cb616efb4f387f8a3',
         'info_dict': {
-            'id': 'phoenix-wright-ace-attorney-dual-destinies-review',
+            'id': 'HkSQKetlGOU',
             'ext': 'mp4',
-            'title': 'Phoenix Wright: Ace Attorney \u2013 Dual Destinies Review',
-            'description': 'md5:36fd701e57e8c15ac8682a2374c99731',
+            'title': 'Phoenix Wright: Ace Attorney - Dual Destinies Review',
+            'description': 'md5:db88c0e7f47e9ea50df3271b9dc72e1d',
             'thumbnail': 're:^https?://.*\.jpg$',
+            'uploader_id': 'UCJugRGo4STYMeFr5RoOShtQ',
+            'uploader': 'Gamekings Vault',
+            'upload_date': '20151123',
         },
     }, {
         # vimeo video
@@ -43,7 +46,11 @@ class GamekingsIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         playlist_id = self._search_regex(
-            r'gogoVideo\(\s*\d+\s*,\s*"([^"]+)', webpage, 'playlist id')
+            r'gogoVideo\(.*,\s*"([^"]+)', webpage, 'playlist id')
+
+        # Check if a YouTube embed is used
+        if playlist_id.find('youtube') != -1:
+            return self.url_result(playlist_id, ie='Youtube')
 
         playlist = self._download_xml(
             'http://www.gamekings.tv/wp-content/themes/gk2010/rss_playlist.php?id=%s' % playlist_id,

From 8e7aad20756efffa649cf073d89f22a57f93048c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 31 Jan 2016 17:49:59 +0600
Subject: [PATCH 3843/6219] [youtube] Use authentication for entry list base
 extractor (Closes #8380)

---
 youtube_dl/extractor/youtube.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index a24c73584..2941da467 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -181,7 +181,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
             return
 
 
-class YoutubeEntryListBaseInfoExtractor(InfoExtractor):
+class YoutubeEntryListBaseInfoExtractor(YoutubeBaseInfoExtractor):
     # Extract entries from page with "Load more" button
     def _entries(self, page, playlist_id):
         more_widget_html = content_html = page
@@ -1602,7 +1602,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         }
 
 
-class YoutubePlaylistIE(YoutubeBaseInfoExtractor, YoutubePlaylistBaseInfoExtractor):
+class YoutubePlaylistIE(YoutubePlaylistBaseInfoExtractor):
     IE_DESC = 'YouTube.com playlists'
     _VALID_URL = r"""(?x)(?:
                         (?:https?://)?

From 9acd33094d36acde60a42837ed775c68ad3a327d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 31 Jan 2016 17:52:02 +0600
Subject: [PATCH 3844/6219] [youtube] Filter duplicates in playlists base
 extractor

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 2941da467..9a64c1d11 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -233,7 +233,7 @@ class YoutubePlaylistBaseInfoExtractor(YoutubeEntryListBaseInfoExtractor):
 
 class YoutubePlaylistsBaseInfoExtractor(YoutubeEntryListBaseInfoExtractor):
     def _process_page(self, content):
-        for playlist_id in re.findall(r'href="/?playlist\?list=(.+?)"', content):
+        for playlist_id in set(re.findall(r'href="/?playlist\?list=([0-9A-Za-z-_]{10,})"', content)):
             yield self.url_result(
                 'https://www.youtube.com/playlist?list=%s' % playlist_id, 'YoutubePlaylist')
 

From a69bee4762044b657e1fd8bd4cf0baa58d950bc3 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 31 Jan 2016 12:57:18 +0100
Subject: [PATCH 3845/6219] release 2016.01.31

---
 README.md              | 4 ++++
 docs/supportedsites.md | 3 +++
 youtube_dl/version.py  | 2 +-
 3 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 724fb17d1..7c582511f 100644
--- a/README.md
+++ b/README.md
@@ -173,6 +173,10 @@ which means you can modify it, redistribute it or use it however you like.
                                      expected filesize (experimental)
     --hls-prefer-native              Use the native HLS downloader instead of
                                      ffmpeg (experimental)
+    --hls-use-mpegts                 Use the mpegts container for HLS videos,
+                                     allowing to play the video while
+                                     downloading (some players may not be able
+                                     to play it)
     --external-downloader COMMAND    Use the specified external downloader.
                                      Currently supports
                                      aria2c,axel,curl,httpie,wget
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index eb68c23b5..0644436a8 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -55,6 +55,7 @@
  - **audiomack**
  - **audiomack:album**
  - **Azubu**
+ - **AzubuLive**
  - **BaiduVideo**: 百度视频
  - **bambuser**
  - **bambuser:channel**
@@ -315,6 +316,7 @@
  - **mailru**: Видео@Mail.Ru
  - **MakerTV**
  - **Malemotion**
+ - **MatchTV**
  - **MDR**: MDR.DE and KiKA
  - **media.ccc.de**
  - **metacafe**
@@ -507,6 +509,7 @@
  - **Sapo**: SAPO Vídeos
  - **savefrom.net**
  - **SBS**: sbs.com.au
+ - **schooltv**
  - **SciVee**
  - **screen.yahoo:search**: Yahoo screen search
  - **Screencast**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index d9f1e22b0..006b960b3 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.01.29'
+__version__ = '2016.01.31'

From a7aab0c23e537d9df3d0946c0b960cc92e35d2ff Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 31 Jan 2016 19:49:54 +0800
Subject: [PATCH 3846/6219] [test_youtube_lists] Fix
 TestYoutubeLists.test_youtube_course

Youtube entries are now generators
---
 test/test_youtube_lists.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/test_youtube_lists.py b/test/test_youtube_lists.py
index 26aadb34f..47df0f348 100644
--- a/test/test_youtube_lists.py
+++ b/test/test_youtube_lists.py
@@ -34,7 +34,7 @@ class TestYoutubeLists(unittest.TestCase):
         ie = YoutubePlaylistIE(dl)
         # TODO find a > 100 (paginating?) videos course
         result = ie.extract('https://www.youtube.com/course?list=ECUl4u3cNGP61MdtwGTqZA0MreSaDybji8')
-        entries = result['entries']
+        entries = list(result['entries'])
         self.assertEqual(YoutubeIE().extract_id(entries[0]['url']), 'j9WZyLZCBzs')
         self.assertEqual(len(entries), 25)
         self.assertEqual(YoutubeIE().extract_id(entries[-1]['url']), 'rYefUsYuEp0')

From 189d72d5fd16176a8990a9447c0165478a384676 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 31 Jan 2016 19:53:22 +0800
Subject: [PATCH 3847/6219] [test_subtitles] Fix TestRaiSubtitles

RaiIE is renamed to RaiTVIE in 06d5556dface3901a86419b6b125ef377116448f
---
 test/test_subtitles.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/test_subtitles.py b/test/test_subtitles.py
index 9ed9fe622..9a695c4e8 100644
--- a/test/test_subtitles.py
+++ b/test/test_subtitles.py
@@ -21,7 +21,7 @@ from youtube_dl.extractor import (
     NPOIE,
     ComedyCentralIE,
     NRKTVIE,
-    RaiIE,
+    RaiTVIE,
     VikiIE,
     ThePlatformIE,
     ThePlatformFeedIE,
@@ -260,7 +260,7 @@ class TestNRKSubtitles(BaseTestSubtitles):
 
 class TestRaiSubtitles(BaseTestSubtitles):
     url = 'http://www.rai.tv/dl/RaiTV/programmi/media/ContentItem-cb27157f-9dd0-4aee-b788-b1f67643a391.html'
-    IE = RaiIE
+    IE = RaiTVIE
 
     def test_allsubtitles(self):
         self.DL.params['writesubtitles'] = True

From b99d88c6a1d91f57e1804bd512fed1bdbf31a384 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 31 Jan 2016 20:00:07 +0800
Subject: [PATCH 3848/6219] [youporn] Fix uploader and description

---
 youtube_dl/extractor/youporn.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/youporn.py b/youtube_dl/extractor/youporn.py
index dd724085a..b29baafc4 100644
--- a/youtube_dl/extractor/youporn.py
+++ b/youtube_dl/extractor/youporn.py
@@ -114,15 +114,13 @@ class YouPornIE(InfoExtractor):
             formats.append(f)
         self._sort_formats(formats)
 
-        description = self._html_search_regex(
-            r'(?s)<div[^>]+class=["\']video-description["\'][^>]*>(.+?)</div>',
-            webpage, 'description', default=None)
+        description = self._og_search_description(webpage, default=None)
         thumbnail = self._search_regex(
             r'(?:imageurl\s*=|poster\s*:)\s*(["\'])(?P<thumbnail>.+?)\1',
             webpage, 'thumbnail', fatal=False, group='thumbnail')
 
         uploader = self._html_search_regex(
-            r'(?s)<div[^>]+class=["\']videoInfoBy["\'][^>]*>\s*By:\s*</div>(.+?)</(?:a|div)>',
+            r'(?s)<div[^>]+class=["\']videoInfoBy(?:\s+[^"\']+)?["\'][^>]*>\s*By:\s*</div>(.+?)</(?:a|div)>',
             webpage, 'uploader', fatal=False)
         upload_date = unified_strdate(self._html_search_regex(
             r'(?s)<div[^>]+class=["\']videoInfoTime["\'][^>]*>(.+?)</div>',

From 231ea2a3bbc6324fbd0e5dddd224646c91b0f035 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 31 Jan 2016 20:21:57 +0800
Subject: [PATCH 3849/6219] [xuite] Replace the test case with my uploaded one

---
 youtube_dl/extractor/xuite.py | 21 +++++++++++----------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/xuite.py b/youtube_dl/extractor/xuite.py
index 8bbac54e2..2466410fa 100644
--- a/youtube_dl/extractor/xuite.py
+++ b/youtube_dl/extractor/xuite.py
@@ -34,19 +34,20 @@ class XuiteIE(InfoExtractor):
         },
     }, {
         # Video with only one format
-        'url': 'http://vlog.xuite.net/play/TkRZNjhULTM0NDE2MjkuZmx2',
-        'md5': 'c45737fc8ac5dc8ac2f92ecbcecf505e',
+        'url': 'http://vlog.xuite.net/play/WUxxR2xCLTI1OTI1MDk5LmZsdg==',
+        'md5': '21f7b39c009b5a4615b4463df6eb7a46',
         'info_dict': {
-            'id': '3441629',
+            'id': '25925099',
             'ext': 'mp4',
-            'title': '孫燕姿 - 眼淚成詩',
+            'title': 'BigBuckBunny_320x180',
             'thumbnail': 're:^https?://.*\.jpg$',
-            'duration': 217.399,
-            'timestamp': 1299383640,
-            'upload_date': '20110306',
-            'uploader': 'Valen',
-            'uploader_id': '10400126',
-            'categories': ['影視娛樂'],
+            'duration': 596.458,
+            'timestamp': 1454242500,
+            'upload_date': '20160131',
+            'uploader': 'yan12125',
+            'uploader_id': '12158353',
+            'categories': ['個人短片'],
+            'description': 'http://download.blender.org/peach/bigbuckbunny_movies/BigBuckBunny_320x180.mp4',
         },
     }, {
         # Video with two formats

From e04398e397d4a4f7e0d0994355d5a94c22441e7c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 31 Jan 2016 14:22:36 +0100
Subject: [PATCH 3850/6219] [FFmpegSubtitlesConvertorPP] delete old subtitle
 files (fixes #8382)

---
 youtube_dl/postprocessor/ffmpeg.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index daca5d814..16a64802a 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -479,6 +479,7 @@ class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor):
             self._downloader.to_screen('[ffmpeg] There aren\'t any subtitles to convert')
             return [], info
         self._downloader.to_screen('[ffmpeg] Converting subtitles')
+        sub_filenames = []
         for lang, sub in subs.items():
             ext = sub['ext']
             if ext == new_ext:
@@ -486,6 +487,8 @@ class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor):
                     '[ffmpeg] Subtitle file for %s is already in the requested'
                     'format' % new_ext)
                 continue
+            old_file = subtitles_filename(filename, lang, ext)
+            sub_filenames.append(old_file)
             new_file = subtitles_filename(filename, lang, new_ext)
 
             if ext == 'dfxp' or ext == 'ttml':
@@ -493,7 +496,7 @@ class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor):
                     'You have requested to convert dfxp (TTML) subtitles into another format, '
                     'which results in style information loss')
 
-                dfxp_file = subtitles_filename(filename, lang, ext)
+                dfxp_file = old_file
                 srt_file = subtitles_filename(filename, lang, 'srt')
 
                 with io.open(dfxp_file, 'rt', encoding='utf-8') as f:
@@ -511,9 +514,7 @@ class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor):
                 if new_ext == 'srt':
                     continue
 
-            self.run_ffmpeg(
-                subtitles_filename(filename, lang, ext),
-                new_file, ['-f', new_format])
+            self.run_ffmpeg(old_file, new_file, ['-f', new_format])
 
             with io.open(new_file, 'rt', encoding='utf-8') as f:
                 subs[lang] = {
@@ -521,4 +522,4 @@ class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor):
                     'data': f.read(),
                 }
 
-        return [], info
+        return sub_filenames, info

From 3ccb0655c1f27953ba00ee3f8bc290ca9bfda7ce Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 31 Jan 2016 15:11:00 +0100
Subject: [PATCH 3851/6219] [youtube] Use 'orderedSet' instead of 'set' to
 preserve the order

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 9a64c1d11..bd87c75b6 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -233,7 +233,7 @@ class YoutubePlaylistBaseInfoExtractor(YoutubeEntryListBaseInfoExtractor):
 
 class YoutubePlaylistsBaseInfoExtractor(YoutubeEntryListBaseInfoExtractor):
     def _process_page(self, content):
-        for playlist_id in set(re.findall(r'href="/?playlist\?list=([0-9A-Za-z-_]{10,})"', content)):
+        for playlist_id in orderedSet(re.findall(r'href="/?playlist\?list=([0-9A-Za-z-_]{10,})"', content)):
             yield self.url_result(
                 'https://www.youtube.com/playlist?list=%s' % playlist_id, 'YoutubePlaylist')
 

From 163da6a484f57c52afdf300d368136d164316803 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 1 Feb 2016 02:13:11 +0800
Subject: [PATCH 3852/6219] [gamekings] Add MD5 back

The test is now a YouTube video, whose MD5 should be stable
---
 youtube_dl/extractor/gamekings.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/gamekings.py b/youtube_dl/extractor/gamekings.py
index efe31c10d..c011e9d22 100644
--- a/youtube_dl/extractor/gamekings.py
+++ b/youtube_dl/extractor/gamekings.py
@@ -12,8 +12,7 @@ class GamekingsIE(InfoExtractor):
     _VALID_URL = r'http://www\.gamekings\.nl/(?:videos|nieuws)/(?P<id>[^/]+)'
     _TESTS = [{
         'url': 'http://www.gamekings.nl/videos/phoenix-wright-ace-attorney-dual-destinies-review/',
-        # MD5 is flaky, seems to change regularly
-        # 'md5': '2f32b1f7b80fdc5cb616efb4f387f8a3',
+        'md5': '5208d3a17adeaef829a7861887cb9029',
         'info_dict': {
             'id': 'HkSQKetlGOU',
             'ext': 'mp4',

From eab3c2895c66a8d2f5da181d3ccba35a901b813f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 1 Feb 2016 02:15:25 +0800
Subject: [PATCH 3853/6219] [gamekings] add_ie

---
 youtube_dl/extractor/gamekings.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/gamekings.py b/youtube_dl/extractor/gamekings.py
index c011e9d22..df385e338 100644
--- a/youtube_dl/extractor/gamekings.py
+++ b/youtube_dl/extractor/gamekings.py
@@ -11,6 +11,7 @@ from ..utils import (
 class GamekingsIE(InfoExtractor):
     _VALID_URL = r'http://www\.gamekings\.nl/(?:videos|nieuws)/(?P<id>[^/]+)'
     _TESTS = [{
+        # YouTube embed video
         'url': 'http://www.gamekings.nl/videos/phoenix-wright-ace-attorney-dual-destinies-review/',
         'md5': '5208d3a17adeaef829a7861887cb9029',
         'info_dict': {
@@ -23,6 +24,7 @@ class GamekingsIE(InfoExtractor):
             'uploader': 'Gamekings Vault',
             'upload_date': '20151123',
         },
+        'add_ie': ['Youtube'],
     }, {
         # vimeo video
         'url': 'http://www.gamekings.nl/videos/the-legend-of-zelda-majoras-mask/',

From 0e1b1a011d1772bc1a7069bad8ad71a53798a212 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 1 Feb 2016 02:19:03 +0800
Subject: [PATCH 3854/6219] [gamekings] Stricter checks

---
 youtube_dl/extractor/gamekings.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/gamekings.py b/youtube_dl/extractor/gamekings.py
index df385e338..f6b9046f9 100644
--- a/youtube_dl/extractor/gamekings.py
+++ b/youtube_dl/extractor/gamekings.py
@@ -6,6 +6,7 @@ from ..utils import (
     xpath_text,
     xpath_with_ns,
 )
+from .youtube import YoutubeIE
 
 
 class GamekingsIE(InfoExtractor):
@@ -47,10 +48,10 @@ class GamekingsIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         playlist_id = self._search_regex(
-            r'gogoVideo\(.*,\s*"([^"]+)', webpage, 'playlist id')
+            r'gogoVideo\([^,]+,\s*"([^"]+)', webpage, 'playlist id')
 
         # Check if a YouTube embed is used
-        if playlist_id.find('youtube') != -1:
+        if YoutubeIE.suitable(playlist_id):
             return self.url_result(playlist_id, ie='Youtube')
 
         playlist = self._download_xml(

From a8aad210019b50540cf1bfd28390b7e7f2573a31 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 1 Feb 2016 03:07:04 +0800
Subject: [PATCH 3855/6219] [acast] Fix extraction

---
 youtube_dl/extractor/acast.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/acast.py b/youtube_dl/extractor/acast.py
index be7913bc7..f2465f8f8 100644
--- a/youtube_dl/extractor/acast.py
+++ b/youtube_dl/extractor/acast.py
@@ -23,14 +23,19 @@ class ACastIE(ACastBaseIE):
             'ext': 'mp3',
             'title': '"Where Are You?": Taipei 101, Taiwan',
             'timestamp': 1196172000000,
-            'description': 'md5:0c5d8201dfea2b93218ea986c91eee6e',
+            'description': 'md5:a0b4ef3634e63866b542e5b1199a1a0e',
             'duration': 211,
         }
     }
 
     def _real_extract(self, url):
         channel, display_id = re.match(self._VALID_URL, url).groups()
-        cast_data = self._download_json(self._API_BASE_URL + 'channels/%s/acasts/%s/playback' % (channel, display_id), display_id)
+
+        embed_page = self._download_webpage(
+            re.sub('(?:www\.)?acast\.com', 'embedcdn.acast.com', url), display_id)
+        cast_data = self._parse_json(self._search_regex(
+            r'window\[\'acast/queries\'\]\s*=\s*([^;]+);', embed_page, 'acast data'),
+            display_id)['GetAcast/%s/%s' % (channel, display_id)]
 
         return {
             'id': compat_str(cast_data['id']),

From 9934fe76be616cf468038a6b04771b3fcc514765 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 1 Feb 2016 03:08:46 +0800
Subject: [PATCH 3856/6219] [acast] Remove ACastBaseIE

No longer necessary as _API_BASE_URL is used by ACastChannelIE only
---
 youtube_dl/extractor/acast.py | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/acast.py b/youtube_dl/extractor/acast.py
index f2465f8f8..92eee8119 100644
--- a/youtube_dl/extractor/acast.py
+++ b/youtube_dl/extractor/acast.py
@@ -8,11 +8,7 @@ from ..compat import compat_str
 from ..utils import int_or_none
 
 
-class ACastBaseIE(InfoExtractor):
-    _API_BASE_URL = 'https://www.acast.com/api/'
-
-
-class ACastIE(ACastBaseIE):
+class ACastIE(InfoExtractor):
     IE_NAME = 'acast'
     _VALID_URL = r'https?://(?:www\.)?acast\.com/(?P<channel>[^/]+)/(?P<id>[^/#?]+)'
     _TEST = {
@@ -49,7 +45,7 @@ class ACastIE(ACastBaseIE):
         }
 
 
-class ACastChannelIE(ACastBaseIE):
+class ACastChannelIE(InfoExtractor):
     IE_NAME = 'acast:channel'
     _VALID_URL = r'https?://(?:www\.)?acast\.com/(?P<id>[^/#?]+)'
     _TEST = {
@@ -61,6 +57,7 @@ class ACastChannelIE(ACastBaseIE):
         },
         'playlist_mincount': 20,
     }
+    _API_BASE_URL = 'https://www.acast.com/api/'
 
     @classmethod
     def suitable(cls, url):

From 7a0ed06909f8951f8be3049ab089d61ead626158 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 1 Feb 2016 03:31:58 +0800
Subject: [PATCH 3857/6219] [allocine] Fix extraction of test_allocine_1 and
 update tests

---
 youtube_dl/extractor/allocine.py | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/allocine.py b/youtube_dl/extractor/allocine.py
index 7d65b8193..f94da1a05 100644
--- a/youtube_dl/extractor/allocine.py
+++ b/youtube_dl/extractor/allocine.py
@@ -8,6 +8,7 @@ from .common import InfoExtractor
 from ..compat import compat_str
 from ..utils import (
     qualities,
+    unescapeHTML,
 )
 
 
@@ -31,7 +32,7 @@ class AllocineIE(InfoExtractor):
             'id': '19540403',
             'ext': 'mp4',
             'title': 'Planes 2 Bande-annonce VF',
-            'description': 'md5:eeaffe7c2d634525e21159b93acf3b1e',
+            'description': 'Regardez la bande annonce du film Planes 2 (Planes 2 Bande-annonce VF). Planes 2, un film de Roberts Gannaway',
             'thumbnail': 're:http://.*\.jpg',
         },
     }, {
@@ -41,7 +42,7 @@ class AllocineIE(InfoExtractor):
             'id': '19544709',
             'ext': 'mp4',
             'title': 'Dragons 2 - Bande annonce finale VF',
-            'description': 'md5:71742e3a74b0d692c7fce0dd2017a4ac',
+            'description': 'md5:601d15393ac40f249648ef000720e7e3',
             'thumbnail': 're:http://.*\.jpg',
         },
     }, {
@@ -59,10 +60,14 @@ class AllocineIE(InfoExtractor):
         if typ == 'film':
             video_id = self._search_regex(r'href="/video/player_gen_cmedia=([0-9]+).+"', webpage, 'video id')
         else:
-            player = self._search_regex(r'data-player=\'([^\']+)\'>', webpage, 'data player')
-
-            player_data = json.loads(player)
-            video_id = compat_str(player_data['refMedia'])
+            player = self._search_regex(r'data-player=\'([^\']+)\'>', webpage, 'data player', default=None)
+            if player:
+                player_data = json.loads(player)
+                video_id = compat_str(player_data['refMedia'])
+            else:
+                model = self._search_regex(r'data-model="([^"]+)">', webpage, 'data model')
+                model_data = self._parse_json(unescapeHTML(model), display_id)
+                video_id = compat_str(model_data['id'])
 
         xml = self._download_xml('http://www.allocine.fr/ws/AcVisiondataV4.ashx?media=%s' % video_id, display_id)
 

From f63757ec35bb310d05966de03dcfae72b427eac3 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 1 Feb 2016 03:34:02 +0800
Subject: [PATCH 3858/6219] [allocine] Fix for Python 2.6

Python 2.6 does not support .// syntax in find(). Fortunately, the
interested node is at the top level
---
 youtube_dl/extractor/allocine.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/allocine.py b/youtube_dl/extractor/allocine.py
index f94da1a05..f6cd3a85f 100644
--- a/youtube_dl/extractor/allocine.py
+++ b/youtube_dl/extractor/allocine.py
@@ -71,7 +71,7 @@ class AllocineIE(InfoExtractor):
 
         xml = self._download_xml('http://www.allocine.fr/ws/AcVisiondataV4.ashx?media=%s' % video_id, display_id)
 
-        video = xml.find('.//AcVisionVideo').attrib
+        video = xml.find('./AcVisionVideo').attrib
         quality = qualities(['ld', 'md', 'hd'])
 
         formats = []

From 566bda51f2d943d58fbe37d88ea0fceb8ddefea8 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 1 Feb 2016 05:00:09 +0800
Subject: [PATCH 3859/6219] [bpb] Fix extraction and update tests

---
 youtube_dl/extractor/bpb.py | 30 +++++++++++++++++++++++++-----
 1 file changed, 25 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/bpb.py b/youtube_dl/extractor/bpb.py
index 510813f76..c28e72927 100644
--- a/youtube_dl/extractor/bpb.py
+++ b/youtube_dl/extractor/bpb.py
@@ -1,7 +1,13 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
+from ..utils import (
+    js_to_json,
+    determine_ext,
+)
 
 
 class BpbIE(InfoExtractor):
@@ -10,7 +16,8 @@ class BpbIE(InfoExtractor):
 
     _TEST = {
         'url': 'http://www.bpb.de/mediathek/297/joachim-gauck-zu-1989-und-die-erinnerung-an-die-ddr',
-        'md5': '0792086e8e2bfbac9cdf27835d5f2093',
+        # md5 fails in Python 2.6 due to buggy server response and wrong handling of urllib2
+        'md5': 'c4f84c8a8044ca9ff68bb8441d300b3f',
         'info_dict': {
             'id': '297',
             'ext': 'mp4',
@@ -25,13 +32,26 @@ class BpbIE(InfoExtractor):
 
         title = self._html_search_regex(
             r'<h2 class="white">(.*?)</h2>', webpage, 'title')
-        video_url = self._html_search_regex(
-            r'(http://film\.bpb\.de/player/dokument_[0-9]+\.mp4)',
-            webpage, 'video URL')
+        video_info_dicts = re.findall(
+            r"({\s*src:\s*'http://film\.bpb\.de/[^}]+})", webpage)
+
+        formats = []
+        for video_info in video_info_dicts:
+            video_info = self._parse_json(video_info, video_id, transform_source=js_to_json)
+            quality = video_info['quality']
+            video_url = video_info['src']
+            formats.append({
+                'url': video_url,
+                'preference': 10 if quality == 'high' else 0,
+                'format_note': quality,
+                'format_id': '%s-%s' % (quality, determine_ext(video_url)),
+            })
+
+        self._sort_formats(formats)
 
         return {
             'id': video_id,
-            'url': video_url,
+            'formats': formats,
             'title': title,
             'description': self._og_search_description(webpage),
         }

From dc4fe5c6d713ac09cd35f8b8dbfbe46a67a67f10 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 1 Feb 2016 05:32:28 +0800
Subject: [PATCH 3860/6219] [allocine] Use xpath_element

---
 youtube_dl/extractor/allocine.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/allocine.py b/youtube_dl/extractor/allocine.py
index f6cd3a85f..190bc2cc8 100644
--- a/youtube_dl/extractor/allocine.py
+++ b/youtube_dl/extractor/allocine.py
@@ -9,6 +9,7 @@ from ..compat import compat_str
 from ..utils import (
     qualities,
     unescapeHTML,
+    xpath_element,
 )
 
 
@@ -71,7 +72,7 @@ class AllocineIE(InfoExtractor):
 
         xml = self._download_xml('http://www.allocine.fr/ws/AcVisiondataV4.ashx?media=%s' % video_id, display_id)
 
-        video = xml.find('./AcVisionVideo').attrib
+        video = xpath_element(xml, './/AcVisionVideo').attrib
         quality = qualities(['ld', 'md', 'hd'])
 
         formats = []

From 92769650fae8827f242a1517565cceef5604eeb0 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 1 Feb 2016 15:40:42 +0800
Subject: [PATCH 3861/6219] [vidzi] Fix extraction

Closes #8386.

Vidzi.tv now uses jwplayer, which can be handled by GenericIE
---
 youtube_dl/extractor/vidzi.py | 14 +++++---------
 1 file changed, 5 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/vidzi.py b/youtube_dl/extractor/vidzi.py
index 2ba9f31df..210a738a6 100644
--- a/youtube_dl/extractor/vidzi.py
+++ b/youtube_dl/extractor/vidzi.py
@@ -2,6 +2,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..utils import smuggle_url
 
 
 class VidziIE(InfoExtractor):
@@ -20,19 +21,14 @@ class VidziIE(InfoExtractor):
         video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id)
-        video_host = self._html_search_regex(
-            r'id=\'vplayer\'><img src="http://(.*?)/i', webpage,
-            'video host')
-        video_hash = self._html_search_regex(
-            r'\|([a-z0-9]+)\|hls\|type', webpage, 'video_hash')
-        ext = self._html_search_regex(
-            r'\|tracks\|([a-z0-9]+)\|', webpage, 'video ext')
-        video_url = 'http://' + video_host + '/' + video_hash + '/v.' + ext
         title = self._html_search_regex(
             r'(?s)<h2 class="video-title">(.*?)</h2>', webpage, 'title')
 
+        # Vidzi now uses jwplayer, which can be handled by GenericIE
         return {
+            '_type': 'url_transparent',
             'id': video_id,
             'title': title,
-            'url': video_url,
+            'url': smuggle_url(url, {'to_generic': True}),
+            'ie_key': 'Generic',
         }

From bd93a12e852f34a1c9d8c86023972c6812dc80d5 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 1 Feb 2016 17:03:31 +0800
Subject: [PATCH 3862/6219] [vidzi] Fix _TESTS

---
 youtube_dl/extractor/vidzi.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/vidzi.py b/youtube_dl/extractor/vidzi.py
index 210a738a6..7c6e98026 100644
--- a/youtube_dl/extractor/vidzi.py
+++ b/youtube_dl/extractor/vidzi.py
@@ -14,6 +14,11 @@ class VidziIE(InfoExtractor):
             'id': 'cghql9yq6emu',
             'ext': 'mp4',
             'title': 'youtube-dl test video  1\\\\2\'3/4<5\\\\6ä7↭',
+            'uploader': 'vidzi.tv',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
         },
     }
 

From b286f201a8424e00cc91d27adcf53a5249b8a3b9 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 1 Feb 2016 17:05:48 +0800
Subject: [PATCH 3863/6219] [YoutubeDL] Do not override ie_key in
 url_transparent

---
 test/test_YoutubeDL.py  | 37 +++++++++++++++++++++++++++++++++++++
 youtube_dl/YoutubeDL.py |  3 +--
 2 files changed, 38 insertions(+), 2 deletions(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index b53cfbe78..73910eaec 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -14,6 +14,7 @@ from test.helper import FakeYDL, assertRegexpMatches
 from youtube_dl import YoutubeDL
 from youtube_dl.compat import compat_str, compat_urllib_error
 from youtube_dl.extractor import YoutubeIE
+from youtube_dl.extractor.common import InfoExtractor
 from youtube_dl.postprocessor.common import PostProcessor
 from youtube_dl.utils import ExtractorError, match_filter_func
 
@@ -646,6 +647,42 @@ class TestYoutubeDL(unittest.TestCase):
         ydl = YDL()
         self.assertRaises(compat_urllib_error.URLError, ydl.urlopen, 'file:///etc/passwd')
 
+    def test_do_not_override_ie_key_in_url_transparent(self):
+        ydl = YDL()
+
+        class Foo1IE(InfoExtractor):
+            _VALID_URL = r'foo1:'
+
+            def _real_extract(self, url):
+                return {
+                    '_type': 'url_transparent',
+                    'url': 'foo2:',
+                    'ie_key': 'Foo2',
+                }
+
+        class Foo2IE(InfoExtractor):
+            _VALID_URL = r'foo2:'
+
+            def _real_extract(self, url):
+                return {
+                    '_type': 'url',
+                    'url': 'foo3:',
+                    'ie_key': 'Foo3',
+                }
+
+        class Foo3IE(InfoExtractor):
+            _VALID_URL = r'foo3:'
+
+            def _real_extract(self, url):
+                return _make_result([{'url': TEST_URL}])
+
+        ydl.add_info_extractor(Foo1IE(ydl))
+        ydl.add_info_extractor(Foo2IE(ydl))
+        ydl.add_info_extractor(Foo3IE(ydl))
+        ydl.extract_info('foo1:')
+        downloaded = ydl.downloaded_info_dicts[0]
+        self.assertEqual(downloaded['url'], TEST_URL)
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 2a3d6cd4a..50228bb32 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -707,7 +707,6 @@ class YoutubeDL(object):
         It will also download the videos if 'download'.
         Returns the resolved ie_result.
         """
-
         result_type = ie_result.get('_type', 'video')
 
         if result_type in ('url', 'url_transparent'):
@@ -736,7 +735,7 @@ class YoutubeDL(object):
 
             force_properties = dict(
                 (k, v) for k, v in ie_result.items() if v is not None)
-            for f in ('_type', 'url'):
+            for f in ('_type', 'url', 'ie_key'):
                 if f in force_properties:
                     del force_properties[f]
             new_result = info.copy()

From abe694ca95fe3344db6b3ebcf483911203fcdc3f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 1 Feb 2016 17:10:11 +0800
Subject: [PATCH 3864/6219] [kickstarter] Eliminate the warning message and
 add_ie

---
 youtube_dl/extractor/kickstarter.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/kickstarter.py b/youtube_dl/extractor/kickstarter.py
index 1d391e69f..e703fc597 100644
--- a/youtube_dl/extractor/kickstarter.py
+++ b/youtube_dl/extractor/kickstarter.py
@@ -2,6 +2,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..utils import smuggle_url
 
 
 class KickStarterIE(InfoExtractor):
@@ -27,7 +28,8 @@ class KickStarterIE(InfoExtractor):
             'uploader_id': 'pebble',
             'uploader': 'Pebble Technology',
             'title': 'Pebble iOS Notifications',
-        }
+        },
+        'add_ie': ['Vimeo'],
     }, {
         'url': 'https://www.kickstarter.com/projects/1420158244/power-drive-2000/widget/video.html',
         'info_dict': {
@@ -52,7 +54,7 @@ class KickStarterIE(InfoExtractor):
             return {
                 '_type': 'url_transparent',
                 'ie_key': 'Generic',
-                'url': url,
+                'url': smuggle_url(url, {'to_generic': True}),
                 'title': title,
             }
 

From f021acee496b0bd91ceb5e7a7e115c54c2f99f5b Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 1 Feb 2016 17:21:40 +0800
Subject: [PATCH 3865/6219] [kickstarter] Fix title and test_kickstarter

It's the description page that contains a video. The original URL is now
the calendar.
---
 youtube_dl/extractor/kickstarter.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/kickstarter.py b/youtube_dl/extractor/kickstarter.py
index e703fc597..9f1ade2e4 100644
--- a/youtube_dl/extractor/kickstarter.py
+++ b/youtube_dl/extractor/kickstarter.py
@@ -8,7 +8,7 @@ from ..utils import smuggle_url
 class KickStarterIE(InfoExtractor):
     _VALID_URL = r'https?://www\.kickstarter\.com/projects/(?P<id>[^/]*)/.*'
     _TESTS = [{
-        'url': 'https://www.kickstarter.com/projects/1404461844/intersection-the-story-of-josh-grant?ref=home_location',
+        'url': 'https://www.kickstarter.com/projects/1404461844/intersection-the-story-of-josh-grant/description',
         'md5': 'c81addca81327ffa66c642b5d8b08cab',
         'info_dict': {
             'id': '1404461844',
@@ -45,7 +45,7 @@ class KickStarterIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         title = self._html_search_regex(
-            r'<title>\s*(.*?)(?:\s*&mdash; Kickstarter)?\s*</title>',
+            r'<title>\s*(.*?)(?:\s*&mdash;\s*Kickstarter)?\s*</title>',
             webpage, 'title')
         video_url = self._search_regex(
             r'data-video-url="(.*?)"',

From 7537b35fb80061287f6901bcdb6b3f6aae558036 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 1 Feb 2016 17:40:35 +0800
Subject: [PATCH 3866/6219] [daum] PEP8

---
 youtube_dl/extractor/daum.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/daum.py b/youtube_dl/extractor/daum.py
index cf158c42d..67c4c7c6b 100644
--- a/youtube_dl/extractor/daum.py
+++ b/youtube_dl/extractor/daum.py
@@ -162,8 +162,8 @@ class DaumListIE(InfoExtractor):
         for pagenum in itertools.count(1):
             list_info = self._download_json(
                 'http://tvpot.daum.net/mypot/json/GetClipInfo.do?size=48&init=true&order=date&page=%d&%s=%s' % (
-                    pagenum, list_id_type, list_id), list_id,'Downloading list info - %s' % pagenum)
-            
+                    pagenum, list_id_type, list_id), list_id, 'Downloading list info - %s' % pagenum)
+
             entries.extend([
                 self.url_result(
                     'http://tvpot.daum.net/v/%s' % clip['vid'])

From 11a4efc505fdb6456b00e848609523bb7b8e344c Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 1 Feb 2016 18:15:53 +0800
Subject: [PATCH 3867/6219] [daum] Do not match a single URL with multiple info
 extractors

---
 youtube_dl/extractor/daum.py | 47 +++++++++++++++++++++++++++---------
 1 file changed, 36 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/daum.py b/youtube_dl/extractor/daum.py
index 67c4c7c6b..5a7565e96 100644
--- a/youtube_dl/extractor/daum.py
+++ b/youtube_dl/extractor/daum.py
@@ -7,8 +7,10 @@ import itertools
 
 from .common import InfoExtractor
 from ..compat import (
+    compat_parse_qs,
     compat_urllib_parse,
     compat_urllib_parse_unquote,
+    compat_urlparse,
 )
 from ..utils import (
     int_or_none,
@@ -117,6 +119,7 @@ class DaumIE(InfoExtractor):
 class DaumClipIE(InfoExtractor):
     _VALID_URL = r'https?://(?:m\.)?tvpot\.daum\.net/(?:clip/ClipView.(?:do|tv)|mypot/View.do)\?.*?clipid=(?P<id>\d+)'
     IE_NAME = 'daum.net:clip'
+    _URL_TEMPLATE = 'http://tvpot.daum.net/clip/ClipView.do?clipid=%s'
 
     _TESTS = [{
         'url': 'http://tvpot.daum.net/clip/ClipView.do?clipid=52554690',
@@ -135,6 +138,10 @@ class DaumClipIE(InfoExtractor):
         'only_matching': True,
     }]
 
+    @classmethod
+    def suitable(cls, url):
+        return False if DaumPlaylistIE.suitable(url) or DaumUserIE.suitable(url) else super(DaumClipIE, cls).suitable(url)
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
         clip_info = self._download_json(
@@ -179,10 +186,21 @@ class DaumListIE(InfoExtractor):
 
         return name, entries
 
+    def _check_clip(self, url, list_id):
+        query_dict = compat_parse_qs(compat_urlparse.urlparse(url).query)
+        if 'clipid' in query_dict:
+            clip_id = query_dict['clipid'][0]
+            if self._downloader.params.get('noplaylist'):
+                self.to_screen('Downloading just video %s because of --no-playlist' % clip_id)
+                return self.url_result(DaumClipIE._URL_TEMPLATE % clip_id, 'DaumClip')
+            else:
+                self.to_screen('Downloading playlist %s - add --no-playlist to just download video' % list_id)
+
 
 class DaumPlaylistIE(DaumListIE):
     _VALID_URL = r'https?://(?:m\.)?tvpot\.daum\.net/mypot/(?:View\.do|Top\.tv)\?.*?playlistid=(?P<id>[0-9]+)'
     IE_NAME = 'daum.net:playlist'
+    _URL_TEMPLATE = 'http://tvpot.daum.net/mypot/View.do?playlistid=%s'
 
     _TESTS = [{
         'note': 'Playlist url with clipid',
@@ -207,12 +225,16 @@ class DaumPlaylistIE(DaumListIE):
         }
     }]
 
-    def _real_extract(self, url):
-        if DaumClipIE.suitable(url) and self._downloader.params.get('noplaylist'):
-            return self.url_result(url, 'DaumClip')
+    @classmethod
+    def suitable(cls, url):
+        return False if DaumUserIE.suitable(url) else super(DaumPlaylistIE, cls).suitable(url)
 
+    def _real_extract(self, url):
         list_id = self._match_id(url)
-        self.to_screen('Downloading playlist %s - add --no-playlist to just download video' % list_id)
+
+        clip_result = self._check_clip(url, list_id)
+        if clip_result:
+            return clip_result
 
         name, entries = self._get_entries(list_id, 'playlistid')
 
@@ -257,14 +279,17 @@ class DaumUserIE(DaumListIE):
     }]
 
     def _real_extract(self, url):
-        if DaumClipIE.suitable(url) and self._downloader.params.get('noplaylist'):
-            return self.url_result(url, 'DaumClip')
-
-        if DaumPlaylistIE.suitable(url):
-            return self.url_result(url, 'DaumPlaylist')
-
         list_id = self._match_id(url)
-        self.to_screen('Downloading playlist %s - add --no-playlist to just download video' % list_id)
+
+        clip_result = self._check_clip(url, list_id)
+        if clip_result:
+            return clip_result
+
+        query_dict = compat_parse_qs(compat_urlparse.urlparse(url).query)
+        if 'playlistid' in query_dict:
+            playlist_id = query_dict['playlistid'][0]
+            return self.url_result(DaumPlaylistIE._URL_TEMPLATE % playlist_id, 'DaumPlaylist')
+
         name, entries = self._get_entries(list_id, 'ownerid')
 
         return self.playlist_result(entries, list_id, name)

From 58250eff2b1cd8968d17d88c8e8fbefdfd9b0a68 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 1 Feb 2016 18:19:02 +0800
Subject: [PATCH 3868/6219] [daum] Update test_daum_1

---
 youtube_dl/extractor/daum.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/daum.py b/youtube_dl/extractor/daum.py
index 5a7565e96..9fce845e1 100644
--- a/youtube_dl/extractor/daum.py
+++ b/youtube_dl/extractor/daum.py
@@ -42,7 +42,7 @@ class DaumIE(InfoExtractor):
         'info_dict': {
             'id': '65139429',
             'ext': 'mp4',
-            'title': 'md5:a100d65d09cec246d8aa9bde7de45aed',
+            'title': '1297회, \'아빠 아들로 태어나길 잘 했어\' 민수, 감동의 눈물[아빠 어디가] 20150118',
             'description': 'md5:79794514261164ff27e36a21ad229fc5',
             'upload_date': '20150604',
             'thumbnail': 're:^https?://.*\.(?:jpg|png)',

From dc724e0c8bceee1eda28513d5e9531bc926d9a3b Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 1 Feb 2016 18:26:23 +0800
Subject: [PATCH 3869/6219] [daum.net:user] Match more URLs (#1952)

---
 youtube_dl/extractor/daum.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/daum.py b/youtube_dl/extractor/daum.py
index 9fce845e1..c84c51058 100644
--- a/youtube_dl/extractor/daum.py
+++ b/youtube_dl/extractor/daum.py
@@ -242,7 +242,7 @@ class DaumPlaylistIE(DaumListIE):
 
 
 class DaumUserIE(DaumListIE):
-    _VALID_URL = r'https?://(?:m\.)?tvpot\.daum\.net/mypot/(?:View|Top)\.do\?.*?ownerid=(?P<id>[0-9a-zA-Z]+)'
+    _VALID_URL = r'https?://(?:m\.)?tvpot\.daum\.net/mypot/(?:View|Top)\.(?:do|tv)\?.*?ownerid=(?P<id>[0-9a-zA-Z]+)'
     IE_NAME = 'daum.net:user'
 
     _TESTS = [{
@@ -276,6 +276,9 @@ class DaumUserIE(DaumListIE):
     }, {
         'url': 'http://tvpot.daum.net/mypot/Top.do?ownerid=o2scDLIVbHc0',
         'only_matching': True,
+    }, {
+        'url': 'http://m.tvpot.daum.net/mypot/Top.tv?ownerid=45x1okb1If50&playlistid=3569733',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From ae0db349c16a205ec571c388239999873be4d7ba Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 1 Feb 2016 12:00:32 +0100
Subject: [PATCH 3870/6219] release 2016.02.01

---
 docs/supportedsites.md | 2 ++
 youtube_dl/version.py  | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 0644436a8..61be9990d 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -134,6 +134,8 @@
  - **DailymotionCloud**
  - **daum.net**
  - **daum.net:clip**
+ - **daum.net:playlist**
+ - **daum.net:user**
  - **DBTV**
  - **DCN**
  - **dcn:live**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 006b960b3..6da42c5a5 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.01.31'
+__version__ = '2016.02.01'

From 0436157b95ddd6d7c415549ba48b7c0305b2c0df Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 2 Feb 2016 00:52:37 +0600
Subject: [PATCH 3871/6219] [vk:uservideos] Improve _VALID_URL (Closes #8389)

---
 youtube_dl/extractor/vk.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 90557fa61..11014865a 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -321,7 +321,7 @@ class VKIE(InfoExtractor):
 class VKUserVideosIE(InfoExtractor):
     IE_NAME = 'vk:uservideos'
     IE_DESC = "VK - User's Videos"
-    _VALID_URL = r'https?://vk\.com/videos(?P<id>-?[0-9]+)$'
+    _VALID_URL = r'https?://vk\.com/videos(?P<id>-?[0-9]+)(?!\?.*\bz=video)(?:[/?#&]|$)'
     _TEMPLATE_URL = 'https://vk.com/videos'
     _TESTS = [{
         'url': 'http://vk.com/videos205387401',
@@ -333,6 +333,9 @@ class VKUserVideosIE(InfoExtractor):
     }, {
         'url': 'http://vk.com/videos-77521',
         'only_matching': True,
+    }, {
+        'url': 'http://vk.com/videos-97664626?section=all',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 1bac34556fcff324fa17c679004284f03123d64d Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 2 Feb 2016 18:07:07 +0100
Subject: [PATCH 3872/6219] [common] add a generic support for mpd manifests

---
 youtube_dl/extractor/common.py | 206 ++++++++++++++++++++++-----------
 1 file changed, 136 insertions(+), 70 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 199a04d1c..1758ab29b 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -10,6 +10,7 @@ import re
 import socket
 import sys
 import time
+import math
 
 from ..compat import (
     compat_cookiejar,
@@ -44,6 +45,7 @@ from ..utils import (
     xpath_text,
     xpath_with_ns,
     determine_protocol,
+    parse_duration,
 )
 
 
@@ -1330,81 +1332,145 @@ class InfoExtractor(object):
             })
         return entries
 
-    def _download_dash_manifest(self, dash_manifest_url, video_id, fatal=True):
-        return self._download_xml(
-            dash_manifest_url, video_id,
-            note='Downloading DASH manifest',
-            errnote='Could not download DASH manifest',
+    def _extract_mpd_formats(self, mpd_url, video_id, mpd_id=None, note=None, errnote=None, fatal=True, formats_dict={}):
+        res = self._download_webpage_handle(
+            mpd_url, video_id,
+            note=note or 'Downloading MPD manifest',
+            errnote=errnote or 'Failed to download MPD manifest',
             fatal=fatal)
+        if res is False:
+            return []
+        mpd, urlh = res
+        mpd_base_url = re.match(r'https?://.+/', urlh.geturl()).group()
 
-    def _extract_dash_manifest_formats(self, dash_manifest_url, video_id, fatal=True, namespace=None, formats_dict={}):
-        dash_doc = self._download_dash_manifest(dash_manifest_url, video_id, fatal)
-        if dash_doc is False:
+        return self._parse_mpd(
+            compat_etree_fromstring(mpd.encode('utf-8')), mpd_id, mpd_base_url, formats_dict=formats_dict)
+
+    def _parse_mpd(self, mpd_doc, mpd_id=None, mpd_base_url='', formats_dict={}):
+        if mpd_doc.get('type') == 'dynamic':
             return []
 
-        return self._parse_dash_manifest(
-            dash_doc, namespace=namespace, formats_dict=formats_dict)
-
-    def _parse_dash_manifest(self, dash_doc, namespace=None, formats_dict={}):
-        def _add_ns(path):
-            return self._xpath_ns(path, namespace)
-
-        formats = []
-        for a in dash_doc.findall('.//' + _add_ns('AdaptationSet')):
-            mime_type = a.attrib.get('mimeType')
-            for r in a.findall(_add_ns('Representation')):
-                mime_type = r.attrib.get('mimeType') or mime_type
-                url_el = r.find(_add_ns('BaseURL'))
-                if mime_type == 'text/vtt':
-                    # TODO implement WebVTT downloading
-                    pass
-                elif mime_type.startswith('audio/') or mime_type.startswith('video/'):
-                    segment_list = r.find(_add_ns('SegmentList'))
-                    format_id = r.attrib['id']
-                    video_url = url_el.text if url_el is not None else None
-                    filesize = int_or_none(url_el.attrib.get('{http://youtube.com/yt/2012/10/10}contentLength') if url_el is not None else None)
-                    f = {
-                        'format_id': format_id,
-                        'url': video_url,
-                        'width': int_or_none(r.attrib.get('width')),
-                        'height': int_or_none(r.attrib.get('height')),
-                        'tbr': int_or_none(r.attrib.get('bandwidth'), 1000),
-                        'asr': int_or_none(r.attrib.get('audioSamplingRate')),
-                        'filesize': filesize,
-                        'fps': int_or_none(r.attrib.get('frameRate')),
-                    }
-                    if segment_list is not None:
-                        initialization_url = segment_list.find(_add_ns('Initialization')).attrib['sourceURL']
-                        f.update({
-                            'initialization_url': initialization_url,
-                            'segment_urls': [segment.attrib.get('media') for segment in segment_list.findall(_add_ns('SegmentURL'))],
-                            'protocol': 'http_dash_segments',
-                        })
-                        if not f.get('url'):
-                            f['url'] = initialization_url
-                    try:
-                        existing_format = next(
-                            fo for fo in formats
-                            if fo['format_id'] == format_id)
-                    except StopIteration:
-                        full_info = formats_dict.get(format_id, {}).copy()
-                        full_info.update(f)
-                        codecs = r.attrib.get('codecs')
-                        if codecs:
-                            if mime_type.startswith('video/'):
-                                vcodec, acodec = codecs, 'none'
-                            else:  # mime_type.startswith('audio/')
-                                vcodec, acodec = 'none', codecs
-
-                            full_info.update({
-                                'vcodec': vcodec,
-                                'acodec': acodec,
-                            })
-                        formats.append(full_info)
+        def extract_multisegment_info(element, ms_parent_info):
+            ms_info = ms_parent_info.copy()
+            segment_list = element.find(self._xpath_ns('SegmentList', namespace))
+            if segment_list is not None:
+                segment_urls_e = segment_list.findall(self._xpath_ns('SegmentURL', namespace))
+                if segment_urls_e:
+                    ms_info['segment_urls'] = [segment.attrib['media'] for segment in segment_urls_e]
+                initialization = segment_list.find(self._xpath_ns('Initialization', namespace))
+                if initialization is not None:
+                    ms_info['initialization_url'] = initialization.attrib['sourceURL']
+            else:
+                segment_template = element.find(self._xpath_ns('SegmentTemplate', namespace))
+                if segment_template is not None:
+                    start_number = segment_template.get('startNumber')
+                    if start_number:
+                        ms_info['start_number'] = int(start_number)
+                    segment_timeline = segment_template.find(self._xpath_ns('SegmentTimeline', namespace))
+                    if segment_timeline is not None:
+                        s_e = segment_timeline.findall(self._xpath_ns('S', namespace))
+                        if s_e:
+                            ms_info['total_number'] = 0
+                            for s in s_e:
+                                ms_info['total_number'] += 1 + int(s.get('r', '0'))
                     else:
-                        existing_format.update(f)
-                else:
-                    self.report_warning('Unknown MIME type %s in DASH manifest' % mime_type)
+                        timescale = segment_template.get('timescale')
+                        if timescale:
+                            ms_info['timescale'] = int(timescale)
+                        segment_duration = segment_template.get('duration')
+                        if segment_duration:
+                            ms_info['segment_duration'] = int(segment_duration)
+                    media_template = segment_template.get('media')
+                    if media_template:
+                        ms_info['media_template'] = media_template
+                    initialization = segment_template.get('initialization')
+                    if initialization:
+                        ms_info['initialization_url'] = initialization
+                    else:
+                        initialization = segment_template.find(self._xpath_ns('Initialization', namespace))
+                        if initialization is not None:
+                            ms_info['initialization_url'] = initialization.attrib['sourceURL']
+            return ms_info
+
+        namespace = self._search_regex(r'(?i)^{([^}]+)?}MPD$', mpd_doc.tag, 'namespace')
+        mpd_duration = parse_duration(mpd_doc.get('mediaPresentationDuration'))
+        formats = []
+        for period in mpd_doc.findall(self._xpath_ns('Period', namespace)):
+            period_duration = parse_duration(period.get('duration')) or mpd_duration
+            period_ms_info = extract_multisegment_info(period, {
+                'start_number': 1,
+                'timescale': 1,
+            })
+            for adaptation_set in period.findall(self._xpath_ns('AdaptationSet', namespace)):
+                adaption_set_ms_info = extract_multisegment_info(adaptation_set, period_ms_info)
+                for representation in adaptation_set.findall(self._xpath_ns('Representation', namespace)):
+                    representation_attrib = adaptation_set.attrib.copy()
+                    representation_attrib.update(representation.attrib)
+                    mime_type = representation_attrib.get('mimeType')
+                    content_type = mime_type.split('/')[0] if mime_type else representation_attrib.get('contentType')
+                    if content_type == 'text':
+                        # TODO implement WebVTT downloading
+                        pass
+                    elif content_type == 'video' or content_type == 'audio':
+                        base_url = ''
+                        for element in (representation, adaptation_set, period, mpd_doc):
+                            base_url_e = element.find(self._xpath_ns('BaseURL', namespace))
+                            if base_url_e is not None:
+                                base_url = base_url_e.text + base_url
+                                if re.match(r'^https?://', base_url):
+                                    break
+                        if not re.match(r'^https?://', base_url):
+                            base_url = mpd_base_url + base_url
+                        representation_id = representation_attrib.get('id')
+                        f = {
+                            'format_id': mpd_id or representation_id,
+                            'url': base_url,
+                            'width': int_or_none(representation_attrib.get('width')),
+                            'height': int_or_none(representation_attrib.get('height')),
+                            'tbr': int_or_none(representation_attrib.get('bandwidth'), 1000),
+                            'asr': int_or_none(representation_attrib.get('audioSamplingRate')),
+                            'fps': int_or_none(representation_attrib.get('frameRate')),
+                            'vcodec': 'none' if content_type == 'audio' else representation_attrib.get('codecs'),
+                            'acodec': 'none' if content_type == 'video' else representation_attrib.get('codecs'),
+                            'language': representation_attrib.get('lang'),
+                            'format_note': 'DASH %s' % content_type,
+                        }
+                        representation_ms_info = extract_multisegment_info(representation, adaption_set_ms_info)
+                        if 'segment_urls' not in representation_ms_info and 'media_template' in representation_ms_info:
+                            if 'total_number' not in representation_ms_info and 'segment_duration':
+                                segment_duration = representation_ms_info['segment_duration'] / representation_ms_info['timescale']
+                                representation_ms_info['total_number'] = int(math.ceil(period_duration / segment_duration))
+                            media_template = representation_ms_info['media_template']
+                            media_template = media_template.replace('$RepresentationID$', representation_id)
+                            media_template = re.sub(r'\$(Bandwidth)(?:%(0\d+d))?\$', r'%(\1)\2', media_template)
+                            media_template = media_template % {'Bandwidth': representation_attrib.get('bandwidth')}
+                            media_template = re.sub(r'\$(Number)(?:%(0\d+d))?\$', r'%(\1)\2', media_template)
+                            media_template.replace('$$', '$')
+                            representation_ms_info['segment_urls'] = [media_template % {'Number': segment_number} for segment_number in range(representation_ms_info['start_number'], representation_ms_info['total_number'] + representation_ms_info['start_number'])]
+                        if 'segment_urls' in representation_ms_info:
+                            f.update({
+                                'segment_urls': representation_ms_info['segment_urls'],
+                                'protocol': 'http_dash_segments',
+                            })
+                            if 'initialization_url' in representation_ms_info:
+                                initialization_url = representation_ms_info['initialization_url'].replace('$RepresentationID$', representation_id)
+                                f.update({
+                                    'initialization_url': initialization_url,
+                                })
+                                if not f.get('url'):
+                                    f['url'] = initialization_url
+                        try:
+                            existing_format = next(
+                                fo for fo in formats
+                                if fo['format_id'] == representation_id)
+                        except StopIteration:
+                            full_info = formats_dict.get(representation_id, {}).copy()
+                            full_info.update(f)
+                            formats.append(full_info)
+                        else:
+                            existing_format.update(f)
+                    else:
+                        self.report_warning('Unknown MIME type %s in DASH manifest' % mime_type)
         return formats
 
     def _live_title(self, name):

From 8ff648e4f97c9d2692cd3f97db404911c23792f5 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 2 Feb 2016 18:10:23 +0100
Subject: [PATCH 3873/6219] [youtube] use _extract_mpd_formats

---
 youtube_dl/extractor/youtube.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index a7f8c968e..828f5d1f4 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1463,7 +1463,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         # Look for the DASH manifest
         if self._downloader.params.get('youtube_include_dash_manifest', True):
             dash_mpd_fatal = True
-            for dash_manifest_url in dash_mpds:
+            for mpd_url in dash_mpds:
                 dash_formats = {}
                 try:
                     def decrypt_sig(mobj):
@@ -1471,11 +1471,11 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                         dec_s = self._decrypt_signature(s, video_id, player_url, age_gate)
                         return '/signature/%s' % dec_s
 
-                    dash_manifest_url = re.sub(r'/s/([a-fA-F0-9\.]+)', decrypt_sig, dash_manifest_url)
+                    mpd_url = re.sub(r'/s/([a-fA-F0-9\.]+)', decrypt_sig, mpd_url)
 
-                    for df in self._extract_dash_manifest_formats(
-                            dash_manifest_url, video_id, fatal=dash_mpd_fatal,
-                            namespace='urn:mpeg:DASH:schema:MPD:2011', formats_dict=self._formats):
+                    for df in self._extract_mpd_formats(
+                            mpd_url, video_id, fatal=dash_mpd_fatal,
+                            formats_dict=self._formats):
                         # Do not overwrite DASH format found in some previous DASH manifest
                         if df['format_id'] not in dash_formats:
                             dash_formats[df['format_id']] = df

From c11875b3287fe7dbacdb296eac4a21d93bc1830e Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 2 Feb 2016 18:11:16 +0100
Subject: [PATCH 3874/6219] [facebook] use _parse_mpd

---
 youtube_dl/extractor/facebook.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index b6d1180f0..9c751178f 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -215,9 +215,8 @@ class FacebookIE(InfoExtractor):
                         })
             dash_manifest = f[0].get('dash_manifest')
             if dash_manifest:
-                formats.extend(self._parse_dash_manifest(
-                    compat_etree_fromstring(compat_urllib_parse_unquote_plus(dash_manifest)),
-                    namespace='urn:mpeg:dash:schema:mpd:2011'))
+                formats.extend(self._parse_mpd(
+                    compat_etree_fromstring(compat_urllib_parse_unquote_plus(dash_manifest))))
         if not formats:
             raise ExtractorError('Cannot find video formats')
 

From a1a22572fb968aa90e110798b34c444d894ad24a Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 2 Feb 2016 18:12:32 +0100
Subject: [PATCH 3875/6219] [downloader/dash] make initialization_url optional

---
 youtube_dl/downloader/dash.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/downloader/dash.py b/youtube_dl/downloader/dash.py
index 535f2a7fc..b0070aead 100644
--- a/youtube_dl/downloader/dash.py
+++ b/youtube_dl/downloader/dash.py
@@ -40,9 +40,10 @@ class DashSegmentsFD(FileDownloader):
             return '%s%s%s' % (base_url, '' if base_url.endswith('/') else '/', target_url)
 
         with open(tmpfilename, 'wb') as outf:
-            append_url_to_file(
-                outf, combine_url(base_url, info_dict['initialization_url']),
-                'initialization segment')
+            if info_dict.get('initialization_url'):
+                append_url_to_file(
+                    outf, combine_url(base_url, info_dict['initialization_url']),
+                    'initialization segment')
             for i, segment_url in enumerate(segment_urls):
                 segment_len = append_url_to_file(
                     outf, combine_url(base_url, segment_url),

From 5976e7ab579c4d1e2fae26acbaa628009b2d83a8 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 2 Feb 2016 18:13:01 +0100
Subject: [PATCH 3876/6219] [vevo] add support for dash formats

---
 youtube_dl/extractor/vevo.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py
index 35fcff1b2..152fef42e 100644
--- a/youtube_dl/extractor/vevo.py
+++ b/youtube_dl/extractor/vevo.py
@@ -197,8 +197,14 @@ class VevoIE(InfoExtractor):
                 if not version_url:
                     continue
 
-                if '.mpd' in version_url or '.ism' in version_url:
+                if '.ism' in version_url:
                     continue
+                elif '.mpd' in version_url:
+                    formats.extend(self._extract_mpd_formats(
+                        version_url, video_id, mpd_id='dash-%s' % version,
+                        note='Downloading %s MPD information' % version,
+                        errnote='Failed to download %s MPD information' % version,
+                        fatal=False))
                 elif '.m3u8' in version_url:
                     formats.extend(self._extract_m3u8_formats(
                         version_url, video_id, 'mp4', 'm3u8_native',

From 9c74423510003a5d6d03e6da1f73f9230df9281f Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 2 Feb 2016 18:30:31 +0100
Subject: [PATCH 3877/6219] [common] fix media template regex

---
 youtube_dl/extractor/common.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 1758ab29b..eda16a315 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1442,9 +1442,9 @@ class InfoExtractor(object):
                                 representation_ms_info['total_number'] = int(math.ceil(period_duration / segment_duration))
                             media_template = representation_ms_info['media_template']
                             media_template = media_template.replace('$RepresentationID$', representation_id)
-                            media_template = re.sub(r'\$(Bandwidth)(?:%(0\d+d))?\$', r'%(\1)\2', media_template)
+                            media_template = re.sub(r'\$(Bandwidth)(?:%(0\d+)d)?\$', r'%(\1)\2d', media_template)
                             media_template = media_template % {'Bandwidth': representation_attrib.get('bandwidth')}
-                            media_template = re.sub(r'\$(Number)(?:%(0\d+d))?\$', r'%(\1)\2', media_template)
+                            media_template = re.sub(r'\$(Number)(?:%(0\d+)d)?\$', r'%(\1)\2d', media_template)
                             media_template.replace('$$', '$')
                             representation_ms_info['segment_urls'] = [media_template % {'Number': segment_number} for segment_number in range(representation_ms_info['start_number'], representation_ms_info['total_number'] + representation_ms_info['start_number'])]
                         if 'segment_urls' in representation_ms_info:

From 1df41411968b52d7fd6d0f0d233ad760aa5e6368 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 3 Feb 2016 03:42:37 +0800
Subject: [PATCH 3878/6219] [test_YoutubeDL] Fix test_youtube_format_selection

Broken since a6c2c24479e5f4827ceb06f64d855329c0a6f593. Thanks to
@jaimeMF and @anisse for pointing that out
---
 test/test_YoutubeDL.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 73910eaec..88c63010e 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -248,6 +248,17 @@ class TestFormatSelection(unittest.TestCase):
 
         def format_info(f_id):
             info = YoutubeIE._formats[f_id].copy()
+
+            # XXX: In real cases InfoExtractor._parse_mpd() fills up 'acodec'
+            # and 'vcodec', while in tests such information is incomplete since
+            # commit a6c2c24479e5f4827ceb06f64d855329c0a6f593
+            # test_YoutubeDL.test_youtube_format_selection is broken without
+            # this fix
+            if 'acodec' in info and 'vcodec' not in info:
+                info['vcodec'] = 'none'
+            elif 'vcodec' in info and 'acodec' not in info:
+                info['acodec'] = 'none'
+
             info['format_id'] = f_id
             info['url'] = 'url:' + f_id
             return info

From f14be228164a2da299d64e18f9081330d1104106 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 2 Feb 2016 22:02:08 +0100
Subject: [PATCH 3879/6219] [common] remove duplicate reference to namespace

---
 youtube_dl/extractor/common.py | 28 ++++++++++++++++------------
 1 file changed, 16 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index eda16a315..caca32d06 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1350,25 +1350,30 @@ class InfoExtractor(object):
         if mpd_doc.get('type') == 'dynamic':
             return []
 
+        namespace = self._search_regex(r'(?i)^{([^}]+)?}MPD$', mpd_doc.tag, 'namespace')
+
+        def _add_ns(path):
+            return self._xpath_ns(path, namespace)
+
         def extract_multisegment_info(element, ms_parent_info):
             ms_info = ms_parent_info.copy()
-            segment_list = element.find(self._xpath_ns('SegmentList', namespace))
+            segment_list = element.find(_add_ns('SegmentList'))
             if segment_list is not None:
-                segment_urls_e = segment_list.findall(self._xpath_ns('SegmentURL', namespace))
+                segment_urls_e = segment_list.findall(_add_ns('SegmentURL'))
                 if segment_urls_e:
                     ms_info['segment_urls'] = [segment.attrib['media'] for segment in segment_urls_e]
-                initialization = segment_list.find(self._xpath_ns('Initialization', namespace))
+                initialization = segment_list.find(_add_ns('Initialization'))
                 if initialization is not None:
                     ms_info['initialization_url'] = initialization.attrib['sourceURL']
             else:
-                segment_template = element.find(self._xpath_ns('SegmentTemplate', namespace))
+                segment_template = element.find(_add_ns('SegmentTemplate'))
                 if segment_template is not None:
                     start_number = segment_template.get('startNumber')
                     if start_number:
                         ms_info['start_number'] = int(start_number)
-                    segment_timeline = segment_template.find(self._xpath_ns('SegmentTimeline', namespace))
+                    segment_timeline = segment_template.find(_add_ns('SegmentTimeline'))
                     if segment_timeline is not None:
-                        s_e = segment_timeline.findall(self._xpath_ns('S', namespace))
+                        s_e = segment_timeline.findall(_add_ns('S'))
                         if s_e:
                             ms_info['total_number'] = 0
                             for s in s_e:
@@ -1387,23 +1392,22 @@ class InfoExtractor(object):
                     if initialization:
                         ms_info['initialization_url'] = initialization
                     else:
-                        initialization = segment_template.find(self._xpath_ns('Initialization', namespace))
+                        initialization = segment_template.find(_add_ns('Initialization'))
                         if initialization is not None:
                             ms_info['initialization_url'] = initialization.attrib['sourceURL']
             return ms_info
 
-        namespace = self._search_regex(r'(?i)^{([^}]+)?}MPD$', mpd_doc.tag, 'namespace')
         mpd_duration = parse_duration(mpd_doc.get('mediaPresentationDuration'))
         formats = []
-        for period in mpd_doc.findall(self._xpath_ns('Period', namespace)):
+        for period in mpd_doc.findall(_add_ns('Period')):
             period_duration = parse_duration(period.get('duration')) or mpd_duration
             period_ms_info = extract_multisegment_info(period, {
                 'start_number': 1,
                 'timescale': 1,
             })
-            for adaptation_set in period.findall(self._xpath_ns('AdaptationSet', namespace)):
+            for adaptation_set in period.findall(_add_ns('AdaptationSet')):
                 adaption_set_ms_info = extract_multisegment_info(adaptation_set, period_ms_info)
-                for representation in adaptation_set.findall(self._xpath_ns('Representation', namespace)):
+                for representation in adaptation_set.findall(_add_ns('Representation')):
                     representation_attrib = adaptation_set.attrib.copy()
                     representation_attrib.update(representation.attrib)
                     mime_type = representation_attrib.get('mimeType')
@@ -1414,7 +1418,7 @@ class InfoExtractor(object):
                     elif content_type == 'video' or content_type == 'audio':
                         base_url = ''
                         for element in (representation, adaptation_set, period, mpd_doc):
-                            base_url_e = element.find(self._xpath_ns('BaseURL', namespace))
+                            base_url_e = element.find(_add_ns('BaseURL'))
                             if base_url_e is not None:
                                 base_url = base_url_e.text + base_url
                                 if re.match(r'^https?://', base_url):

From 6fba62c87a39686c0424ce9acd0fd29700a21076 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 2 Feb 2016 22:14:23 +0100
Subject: [PATCH 3880/6219] [ffmpeg] fix adding metadata when using
 --hls-prefer-native(#8350)

---
 youtube_dl/postprocessor/ffmpeg.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index d3d9d4f1d..e293e03e6 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -392,7 +392,7 @@ class FFmpegMetadataPP(FFmpegPostProcessor):
             options.extend(['-metadata', '%s=%s' % (name, value)])
 
         # https://github.com/rg3/youtube-dl/issues/8350
-        if info['protocol'] == 'm3u8_native':
+        if info['protocol'] == 'm3u8_native' or self._downloader.params.get('hls_prefer_native', False):
             options.extend(['-bsf:a', 'aac_adtstoasc'])
 
         self._downloader.to_screen('[ffmpeg] Adding metadata to \'%s\'' % filename)

From 87de7069b90a5154c8fb625510891cf3fff6f713 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 2 Feb 2016 22:30:13 +0100
Subject: [PATCH 3881/6219] [utils] dfxp2srt: make TTMLPElementParser inherit
 from object

For consistency between python 2 and 3.
---
 youtube_dl/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 18dbe28bb..4262ad6ac 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2017,7 +2017,7 @@ def dfxp2srt(dfxp_data):
         'ttaf1': 'http://www.w3.org/2006/10/ttaf1',
     })
 
-    class TTMLPElementParser:
+    class TTMLPElementParser(object):
         out = ''
 
         def start(self, tag, attrib):

From fd3a1f3d608857ee36b1c5b455bbdbd3dedc17b1 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 2 Feb 2016 23:02:18 +0100
Subject: [PATCH 3882/6219] [cbsnews] add support for live videos(fixes #7010)

---
 youtube_dl/extractor/__init__.py |  5 ++-
 youtube_dl/extractor/cbsnews.py  | 52 +++++++++++++++++++++++++++-----
 2 files changed, 48 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index e61a88de7..8d2fa604b 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -90,7 +90,10 @@ from .canalplus import CanalplusIE
 from .canalc2 import Canalc2IE
 from .canvas import CanvasIE
 from .cbs import CBSIE
-from .cbsnews import CBSNewsIE
+from .cbsnews import (
+    CBSNewsIE,
+    CBSNewsLiveVideoIE,
+)
 from .cbssports import CBSSportsIE
 from .ccc import CCCIE
 from .ceskatelevize import CeskaTelevizeIE
diff --git a/youtube_dl/extractor/cbsnews.py b/youtube_dl/extractor/cbsnews.py
index cabf7e73b..8f864699f 100644
--- a/youtube_dl/extractor/cbsnews.py
+++ b/youtube_dl/extractor/cbsnews.py
@@ -1,15 +1,14 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
-import re
-import json
-
+from .common import InfoExtractor
 from .theplatform import ThePlatformIE
+from ..utils import parse_duration
 
 
 class CBSNewsIE(ThePlatformIE):
     IE_DESC = 'CBS News'
-    _VALID_URL = r'http://(?:www\.)?cbsnews\.com/(?:[^/]+/)+(?P<id>[\da-z_-]+)'
+    _VALID_URL = r'http://(?:www\.)?cbsnews\.com/(?:news|videos)/(?P<id>[\da-z_-]+)'
 
     _TESTS = [
         {
@@ -48,14 +47,13 @@ class CBSNewsIE(ThePlatformIE):
     ]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id)
 
-        video_info = json.loads(self._html_search_regex(
+        video_info = self._parse_json(self._html_search_regex(
             r'(?:<ul class="media-list items" id="media-related-items"><li data-video-info|<div id="cbsNewsVideoPlayer" data-video-player-options)=\'({.+?})\'',
-            webpage, 'video JSON info'))
+            webpage, 'video JSON info'), video_id)
 
         item = video_info['item'] if 'item' in video_info else video_info
         title = item.get('articleTitle') or item.get('hed')
@@ -88,3 +86,41 @@ class CBSNewsIE(ThePlatformIE):
             'formats': formats,
             'subtitles': subtitles,
         }
+
+
+class CBSNewsLiveVideoIE(InfoExtractor):
+    IE_DESC = 'CBS News Live Videos'
+    _VALID_URL = r'http://(?:www\.)?cbsnews\.com/live/video/(?P<id>[\da-z_-]+)'
+
+    _TEST = {
+        'url': 'http://www.cbsnews.com/live/video/clinton-sanders-prepare-to-face-off-in-nh/',
+        'info_dict': {
+            'id': 'clinton-sanders-prepare-to-face-off-in-nh',
+            'ext': 'flv',
+            'title': 'Clinton, Sanders Prepare To Face Off In NH',
+            'duration': 334,
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        video_info = self._parse_json(self._html_search_regex(
+            r'data-story-obj=\'({.+?})\'', webpage, 'video JSON info'), video_id)['story']
+
+        hdcore_sign = 'hdcore=3.3.1'
+        f4m_formats = self._extract_f4m_formats(video_info['url'] + '&' + hdcore_sign, video_id)
+        if f4m_formats:
+            for entry in f4m_formats:
+                # URLs without the extra param induce an 404 error
+                entry.update({'extra_param_to_segment_url': hdcore_sign})
+
+        return {
+            'id': video_id,
+            'title': video_info['headline'],
+            'thumbnail': video_info.get('thumbnail_url_hd') or video_info.get('thumbnail_url_sd'),
+            'duration': parse_duration(video_info.get('segmentDur')),
+            'formats': f4m_formats,
+        }

From 6ad2b01e14fe22e4607ea16cebbd13b4270505ca Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 2 Feb 2016 23:09:50 +0100
Subject: [PATCH 3883/6219] [srgssr] use flv as ext for rtmp formats

---
 youtube_dl/extractor/srgssr.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/srgssr.py b/youtube_dl/extractor/srgssr.py
index 4707029ca..246970c4d 100644
--- a/youtube_dl/extractor/srgssr.py
+++ b/youtube_dl/extractor/srgssr.py
@@ -70,14 +70,11 @@ class SRGSSRIE(InfoExtractor):
                         asset_url, media_id, 'mp4', 'm3u8_native',
                         m3u8_id=format_id, fatal=False))
                 else:
-                    ext = None
-                    if protocol == 'RTMP':
-                        ext = self._search_regex(r'([a-z0-9]+):[^/]+', asset_url, 'ext')
                     formats.append({
                         'format_id': format_id,
                         'url': asset_url,
                         'preference': preference(quality),
-                        'ext': ext,
+                        'ext': 'flv' if protocol == 'RTMP' else None,
                     })
         self._sort_formats(formats)
 

From d577c79632e96e900f8ff5b89c98c3e60475d4b0 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 3 Feb 2016 13:24:07 +0100
Subject: [PATCH 3884/6219] [common] ignore ISO 639-2 generic codes

---
 youtube_dl/extractor/common.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index caca32d06..ee0e3d8d1 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1426,6 +1426,7 @@ class InfoExtractor(object):
                         if not re.match(r'^https?://', base_url):
                             base_url = mpd_base_url + base_url
                         representation_id = representation_attrib.get('id')
+                        lang = representation_attrib.get('lang')
                         f = {
                             'format_id': mpd_id or representation_id,
                             'url': base_url,
@@ -1436,7 +1437,7 @@ class InfoExtractor(object):
                             'fps': int_or_none(representation_attrib.get('frameRate')),
                             'vcodec': 'none' if content_type == 'audio' else representation_attrib.get('codecs'),
                             'acodec': 'none' if content_type == 'video' else representation_attrib.get('codecs'),
-                            'language': representation_attrib.get('lang'),
+                            'language': lang if lang not in ('mul', 'und', 'zxx', 'mis') else None,
                             'format_note': 'DASH %s' % content_type,
                         }
                         representation_ms_info = extract_multisegment_info(representation, adaption_set_ms_info)

From c84d3a557de8eddc8e63adbd11ab40877f52c502 Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Wed, 3 Feb 2016 19:18:25 +0500
Subject: [PATCH 3885/6219] [README.md] Clarify unavailable sequences in output
 format

---
 README.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/README.md b/README.md
index 7c582511f..79cd08df4 100644
--- a/README.md
+++ b/README.md
@@ -455,6 +455,8 @@ The `-o` option allows users to indicate a template for the output file names. T
  - `format_id`: The sequence will be replaced by the format code specified by `--format`.
  - `duration`: The sequence will be replaced by the length of the video in seconds.
 
+Note that some of the aforementioned sequences are not guaranteed to be present since they depend on the metadata obtained by particular extractor, such sequences will be replaced with `NA`.
+
 The current default template is `%(title)s-%(id)s.%(ext)s`.
 
 In some cases, you don't want special characters such as 中, spaces, or &, such as when transferring the downloaded filename to a Windows system or the filename through an 8bit-unsafe channel. In these cases, add the `--restrict-filenames` flag to get a shorter title:

From 58be922079a09bc56ff331864b094ba38beaf26e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 4 Feb 2016 01:26:25 +0800
Subject: [PATCH 3886/6219] [kuwo] Check for georestriction

---
 youtube_dl/extractor/kuwo.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/kuwo.py b/youtube_dl/extractor/kuwo.py
index 0c8ed5d07..f641edef8 100644
--- a/youtube_dl/extractor/kuwo.py
+++ b/youtube_dl/extractor/kuwo.py
@@ -31,6 +31,10 @@ class KuwoBaseIE(InfoExtractor):
                 (file_format['ext'], file_format.get('br', ''), song_id),
                 song_id, note='Download %s url info' % file_format['format'],
             )
+
+            if song_url == 'IPDeny':
+                raise ExtractorError('This song is blocked in this region', expected=True)
+
             if song_url.startswith('http://') or song_url.startswith('https://'):
                 formats.append({
                     'url': song_url,

From 675d001633c9446e9d53db2794614862d1d82607 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 3 Feb 2016 18:44:43 +0100
Subject: [PATCH 3887/6219] [common] skip drm protected dash formats

---
 youtube_dl/extractor/common.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index ee0e3d8d1..4859b911a 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1355,6 +1355,9 @@ class InfoExtractor(object):
         def _add_ns(path):
             return self._xpath_ns(path, namespace)
 
+        def is_drm_protected(element):
+            return element.find(_add_ns('ContentProtection')) is not None
+
         def extract_multisegment_info(element, ms_parent_info):
             ms_info = ms_parent_info.copy()
             segment_list = element.find(_add_ns('SegmentList'))
@@ -1406,8 +1409,12 @@ class InfoExtractor(object):
                 'timescale': 1,
             })
             for adaptation_set in period.findall(_add_ns('AdaptationSet')):
+                if is_drm_protected(adaptation_set):
+                    continue
                 adaption_set_ms_info = extract_multisegment_info(adaptation_set, period_ms_info)
                 for representation in adaptation_set.findall(_add_ns('Representation')):
+                    if is_drm_protected(representation):
+                        continue
                     representation_attrib = adaptation_set.attrib.copy()
                     representation_attrib.update(representation.attrib)
                     mime_type = representation_attrib.get('mimeType')

From 53c269c6fd1af690570d35acae1d170b125d9d3c Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 3 Feb 2016 23:54:34 +0100
Subject: [PATCH 3888/6219] [common] fix media_template string formating

---
 youtube_dl/extractor/common.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 4859b911a..b384e412e 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1454,11 +1454,9 @@ class InfoExtractor(object):
                                 representation_ms_info['total_number'] = int(math.ceil(period_duration / segment_duration))
                             media_template = representation_ms_info['media_template']
                             media_template = media_template.replace('$RepresentationID$', representation_id)
-                            media_template = re.sub(r'\$(Bandwidth)(?:%(0\d+)d)?\$', r'%(\1)\2d', media_template)
-                            media_template = media_template % {'Bandwidth': representation_attrib.get('bandwidth')}
-                            media_template = re.sub(r'\$(Number)(?:%(0\d+)d)?\$', r'%(\1)\2d', media_template)
+                            media_template = re.sub(r'\$(Number|Bandwidth)(?:%(0\d+)d)?\$', r'%(\1)\2d', media_template)
                             media_template.replace('$$', '$')
-                            representation_ms_info['segment_urls'] = [media_template % {'Number': segment_number} for segment_number in range(representation_ms_info['start_number'], representation_ms_info['total_number'] + representation_ms_info['start_number'])]
+                            representation_ms_info['segment_urls'] = [media_template % {'Number': segment_number, 'Bandwidth': representation_attrib.get('bandwidth')} for segment_number in range(representation_ms_info['start_number'], representation_ms_info['total_number'] + representation_ms_info['start_number'])]
                         if 'segment_urls' in representation_ms_info:
                             f.update({
                                 'segment_urls': representation_ms_info['segment_urls'],

From 255732f0d33268aeababb1b3ce37a1defb5bc965 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 3 Feb 2016 23:57:08 +0100
Subject: [PATCH 3889/6219] [common] fix segment duration calculation

---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index b384e412e..d9f31daaa 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1,4 +1,4 @@
-from __future__ import unicode_literals
+from __future__ import unicode_literals, division
 
 import base64
 import datetime

From eadc3ccd504a13487a97172cb8468e12fd8bc068 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 4 Feb 2016 01:25:36 +0100
Subject: [PATCH 3890/6219] [generic] extract m3u8 formats when mpegurl content
 type detected

---
 youtube_dl/extractor/generic.py | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index b18e734c4..c02fe201c 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1229,19 +1229,24 @@ class GenericIE(InfoExtractor):
 
         # Check for direct link to a video
         content_type = head_response.headers.get('Content-Type', '')
-        m = re.match(r'^(?P<type>audio|video|application(?=/ogg$))/(?P<format_id>.+)$', content_type)
+        m = re.match(r'^(?P<type>audio|video|application(?=/(?:ogg$|(?:vnd\.apple\.|x-)?mpegurl)))/(?P<format_id>.+)$', content_type)
         if m:
             upload_date = unified_strdate(
                 head_response.headers.get('Last-Modified'))
+            formats = []
+            if m.group('format_id').endswith('mpegurl'):
+                formats = self._extract_m3u8_formats(url, video_id, 'mp4')
+            else:
+                formats = [{
+                    'format_id': m.group('format_id'),
+                    'url': url,
+                    'vcodec': 'none' if m.group('type') == 'audio' else None
+                }]
             return {
                 'id': video_id,
                 'title': compat_urllib_parse_unquote(os.path.splitext(url_basename(url))[0]),
                 'direct': True,
-                'formats': [{
-                    'format_id': m.group('format_id'),
-                    'url': url,
-                    'vcodec': 'none' if m.group('type') == 'audio' else None
-                }],
+                'formats': formats,
                 'upload_date': upload_date,
             }
 

From 09be85b8dd987ff4c3b3161dc58c36959c33da5e Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 4 Feb 2016 08:28:37 +0100
Subject: [PATCH 3891/6219] [youtube] fix subtitle extraction(fixes #8415)

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index a7f8c968e..1fef4cfd4 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -918,7 +918,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             if lang in sub_lang_list:
                 continue
             sub_formats = []
-            for ext in ['sbv', 'vtt', 'srt']:
+            for ext in ['vtt', 'ttml']:
                 params = compat_urllib_parse.urlencode({
                     'lang': lang,
                     'v': video_id,

From 2fb35d1c28f6dcac4bf35cc533fbc1699983b740 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 4 Feb 2016 08:39:01 +0100
Subject: [PATCH 3892/6219] [youtube] fix subtitle order

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 1fef4cfd4..e9ed77e42 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -918,7 +918,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             if lang in sub_lang_list:
                 continue
             sub_formats = []
-            for ext in ['vtt', 'ttml']:
+            for ext in ['ttml', 'vtt']:
                 params = compat_urllib_parse.urlencode({
                     'lang': lang,
                     'v': video_id,

From 920d21b9d375327760afbfbf257c8719106d7325 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 4 Feb 2016 08:50:55 +0100
Subject: [PATCH 3893/6219] [test_subtitles] update youtube subtitles tests

---
 test/test_subtitles.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/test/test_subtitles.py b/test/test_subtitles.py
index 9a695c4e8..27e763edd 100644
--- a/test/test_subtitles.py
+++ b/test/test_subtitles.py
@@ -65,16 +65,16 @@ class TestYoutubeSubtitles(BaseTestSubtitles):
         self.DL.params['allsubtitles'] = True
         subtitles = self.getSubtitles()
         self.assertEqual(len(subtitles.keys()), 13)
-        self.assertEqual(md5(subtitles['en']), '4cd9278a35ba2305f47354ee13472260')
-        self.assertEqual(md5(subtitles['it']), '164a51f16f260476a05b50fe4c2f161d')
-        for lang in ['it', 'fr', 'de']:
+        self.assertEqual(md5(subtitles['en']), '3cb210999d3e021bd6c7f0ea751eab06')
+        self.assertEqual(md5(subtitles['it']), '6d752b98c31f1cf8d597050c7a2cb4b5')
+        for lang in ['fr', 'de']:
             self.assertTrue(subtitles.get(lang) is not None, 'Subtitles for \'%s\' not extracted' % lang)
 
-    def test_youtube_subtitles_sbv_format(self):
+    def test_youtube_subtitles_ttml_format(self):
         self.DL.params['writesubtitles'] = True
-        self.DL.params['subtitlesformat'] = 'sbv'
+        self.DL.params['subtitlesformat'] = 'ttml'
         subtitles = self.getSubtitles()
-        self.assertEqual(md5(subtitles['en']), '13aeaa0c245a8bed9a451cb643e3ad8b')
+        self.assertEqual(md5(subtitles['en']), 'e306f8c42842f723447d9f63ad65df54')
 
     def test_youtube_subtitles_vtt_format(self):
         self.DL.params['writesubtitles'] = True

From f1ed3acae56a5874b6f3965e0735104bbccf8ce2 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 4 Feb 2016 13:39:26 +0100
Subject: [PATCH 3894/6219] release 2016.02.04

---
 docs/supportedsites.md | 1 +
 youtube_dl/version.py  | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 61be9990d..ee34adf26 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -91,6 +91,7 @@
  - **Canvas**
  - **CBS**
  - **CBSNews**: CBS News
+ - **CBSNewsLiveVideo**: CBS News Live Videos
  - **CBSSports**
  - **CeskaTelevize**
  - **channel9**: Channel 9
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 6da42c5a5..d4135e88e 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.02.01'
+__version__ = '2016.02.04'

From 3f7e8750d4658702b00237370da1a1c680fdb548 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 4 Feb 2016 20:16:47 +0100
Subject: [PATCH 3895/6219] [arte.tv:+7] Fix extraction (fixes #8427)

---
 youtube_dl/extractor/arte.py | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index b9e07f0ef..6ed855a57 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -13,6 +13,7 @@ from ..utils import (
     unified_strdate,
     get_element_by_attribute,
     int_or_none,
+    NO_DEFAULT,
     qualities,
 )
 
@@ -93,9 +94,18 @@ class ArteTVPlus7IE(InfoExtractor):
         json_url = self._html_search_regex(
             patterns, webpage, 'json vp url', default=None)
         if not json_url:
-            iframe_url = self._html_search_regex(
-                r'<iframe[^>]+src=(["\'])(?P<url>.+\bjson_url=.+?)\1',
-                webpage, 'iframe url', group='url')
+            def find_iframe_url(webpage, default=NO_DEFAULT):
+                return self._html_search_regex(
+                    r'<iframe[^>]+src=(["\'])(?P<url>.+\bjson_url=.+?)\1',
+                    webpage, 'iframe url', group='url', default=default)
+
+            iframe_url = find_iframe_url(webpage, None)
+            if not iframe_url:
+                embed_url = self._html_search_regex(
+                    r'arte_vp_url_oembed=\'([^\']+?)\'', webpage, 'embed url')
+                player = self._download_json(
+                    embed_url, video_id, 'Downloading player page')
+                iframe_url = find_iframe_url(player['html'])
             json_url = compat_parse_qs(
                 compat_urllib_parse_urlparse(iframe_url).query)['json_url'][0]
         return self._extract_from_json_url(json_url, video_id, lang)

From 008ab0f814a84caebeb7551b86cc0e7422eda6b7 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 5 Feb 2016 11:04:00 +0100
Subject: [PATCH 3896/6219] release 2016.02.05

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index d4135e88e..3b3e7e029 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.02.04'
+__version__ = '2016.02.05'

From 593e0f43b43d5a90d6e3430aeb94357ef81fb1ab Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 5 Feb 2016 15:12:06 +0100
Subject: [PATCH 3897/6219] [ffmpeg] fix condition(fixes #8440)

---
 youtube_dl/postprocessor/ffmpeg.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index e293e03e6..7e85a702d 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -392,7 +392,7 @@ class FFmpegMetadataPP(FFmpegPostProcessor):
             options.extend(['-metadata', '%s=%s' % (name, value)])
 
         # https://github.com/rg3/youtube-dl/issues/8350
-        if info['protocol'] == 'm3u8_native' or self._downloader.params.get('hls_prefer_native', False):
+        if info.get('protocol') == 'm3u8_native' or self._downloader.params.get('hls_prefer_native', False):
             options.extend(['-bsf:a', 'aac_adtstoasc'])
 
         self._downloader.to_screen('[ffmpeg] Adding metadata to \'%s\'' % filename)

From e06e75c7e7794c759077f2bb462dcf5d94a21760 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 5 Feb 2016 15:17:31 +0100
Subject: [PATCH 3898/6219] release 2016.02.05.1

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 3b3e7e029..3fec14ab1 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.02.05'
+__version__ = '2016.02.05.1'

From 7d3a035ee0a940021cb9c1a8cf0a8e5ab67f0c43 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 5 Feb 2016 17:12:49 +0100
Subject: [PATCH 3899/6219] [ffmpeg] check for m3u8 protocol in
 FFmpegMetadataPP

---
 youtube_dl/postprocessor/ffmpeg.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 7e85a702d..22d7ac65a 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -392,7 +392,7 @@ class FFmpegMetadataPP(FFmpegPostProcessor):
             options.extend(['-metadata', '%s=%s' % (name, value)])
 
         # https://github.com/rg3/youtube-dl/issues/8350
-        if info.get('protocol') == 'm3u8_native' or self._downloader.params.get('hls_prefer_native', False):
+        if info.get('protocol') == 'm3u8_native' or info.get('protocol') == 'm3u8' and self._downloader.params.get('hls_prefer_native', False):
             options.extend(['-bsf:a', 'aac_adtstoasc'])
 
         self._downloader.to_screen('[ffmpeg] Adding metadata to \'%s\'' % filename)

From 58ae24336a0177567ac142734ca8f302f019107b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 5 Feb 2016 22:16:12 +0600
Subject: [PATCH 3900/6219] [spankbang] Extend format id regex (Closes #8398)

---
 youtube_dl/extractor/spankbang.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/spankbang.py b/youtube_dl/extractor/spankbang.py
index 3cfa671ed..b14c01230 100644
--- a/youtube_dl/extractor/spankbang.py
+++ b/youtube_dl/extractor/spankbang.py
@@ -34,7 +34,7 @@ class SpankBangIE(InfoExtractor):
             'ext': 'mp4',
             'format_id': '%sp' % height,
             'height': int(height),
-        } for height in re.findall(r'<(?:span|li)[^>]+q_(\d+)p', webpage)]
+        } for height in re.findall(r'<(?:span|li|p)[^>]+[qb]_(\d+)p', webpage)]
         self._sort_formats(formats)
 
         title = self._html_search_regex(

From c773c232d80580c6662b5c957c830f75ccbb766b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 5 Feb 2016 22:16:40 +0600
Subject: [PATCH 3901/6219] [spankbang] Check formats (#8398)

---
 youtube_dl/extractor/spankbang.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/spankbang.py b/youtube_dl/extractor/spankbang.py
index b14c01230..6299bfd91 100644
--- a/youtube_dl/extractor/spankbang.py
+++ b/youtube_dl/extractor/spankbang.py
@@ -35,6 +35,7 @@ class SpankBangIE(InfoExtractor):
             'format_id': '%sp' % height,
             'height': int(height),
         } for height in re.findall(r'<(?:span|li|p)[^>]+[qb]_(\d+)p', webpage)]
+        self._check_formats(formats, video_id)
         self._sort_formats(formats)
 
         title = self._html_search_regex(

From d9e543b68030c8bd24d1b0352938d54312823345 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 5 Feb 2016 22:16:56 +0600
Subject: [PATCH 3902/6219] [spankbang] Add test with single format (#8398)

---
 youtube_dl/extractor/spankbang.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/spankbang.py b/youtube_dl/extractor/spankbang.py
index 6299bfd91..50433d0f6 100644
--- a/youtube_dl/extractor/spankbang.py
+++ b/youtube_dl/extractor/spankbang.py
@@ -7,7 +7,7 @@ from .common import InfoExtractor
 
 class SpankBangIE(InfoExtractor):
     _VALID_URL = r'https?://(?:(?:www|[a-z]{2})\.)?spankbang\.com/(?P<id>[\da-z]+)/video'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://spankbang.com/3vvn/video/fantasy+solo',
         'md5': '1cc433e1d6aa14bc376535b8679302f7',
         'info_dict': {
@@ -19,7 +19,11 @@ class SpankBangIE(InfoExtractor):
             'uploader': 'silly2587',
             'age_limit': 18,
         }
-    }
+    }, {
+        # 480p only
+        'url': 'http://spankbang.com/1vt0/video/solvane+gangbang',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From b7b365067f492e00166ef7ee0ebebcc53b4b6b9c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 5 Feb 2016 23:32:22 +0600
Subject: [PATCH 3903/6219] [vidme:user] Add extractor (Closes #8416)

---
 youtube_dl/extractor/__init__.py |  5 +++-
 youtube_dl/extractor/vidme.py    | 48 +++++++++++++++++++++++++++++++-
 2 files changed, 51 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 8d2fa604b..bedd2397c 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -822,7 +822,10 @@ from .videomore import (
 )
 from .videopremium import VideoPremiumIE
 from .videott import VideoTtIE
-from .vidme import VidmeIE
+from .vidme import (
+    VidmeIE,
+    VidmeUserIE,
+)
 from .vidzi import VidziIE
 from .vier import VierIE, VierVideosIE
 from .viewster import ViewsterIE
diff --git a/youtube_dl/extractor/vidme.py b/youtube_dl/extractor/vidme.py
index 3d63ed4f0..91c8349c2 100644
--- a/youtube_dl/extractor/vidme.py
+++ b/youtube_dl/extractor/vidme.py
@@ -1,5 +1,7 @@
 from __future__ import unicode_literals
 
+import itertools
+
 from .common import InfoExtractor
 from ..compat import compat_HTTPError
 from ..utils import (
@@ -11,7 +13,7 @@ from ..utils import (
 
 
 class VidmeIE(InfoExtractor):
-    _VALID_URL = r'https?://vid\.me/(?:e/)?(?P<id>[\da-zA-Z]+)'
+    _VALID_URL = r'https?://vid\.me/(?:e/)?(?P<id>[\da-zA-Z]{,5})(?:[^\da-zA-Z]|$)'
     _TESTS = [{
         'url': 'https://vid.me/QNB',
         'md5': 'f42d05e7149aeaec5c037b17e5d3dc82',
@@ -202,3 +204,47 @@ class VidmeIE(InfoExtractor):
             'comment_count': comment_count,
             'formats': formats,
         }
+
+
+class VidmeUserIE(InfoExtractor):
+    _VALID_URL = r'https?://vid\.me/(?:e/)?(?P<id>[\da-zA-Z]{6,})'
+    _TEST = {
+        'url': 'https://vid.me/EFARCHIVE',
+        'info_dict': {
+            'id': '3834632',
+            'title': 'EFARCHIVE',
+        },
+        'playlist_mincount': 238,
+    }
+
+    # Max possible limit according to https://docs.vid.me/#api-Videos-List
+    _LIMIT = 100
+
+    def _entries(self, user_id, user_name):
+        for page_num in itertools.count(1):
+            page = self._download_json(
+                'https://api.vid.me/videos/list?user=%s&limit=%d&offset=%d'
+                % (user_id, self._LIMIT, (page_num - 1) * self._LIMIT), user_name,
+                'Downloading user page %d' % page_num)
+
+            videos = page.get('videos', [])
+            if not videos:
+                break
+
+            for video in videos:
+                video_url = video.get('full_url') or video.get('embed_url')
+                if video_url:
+                    yield self.url_result(video_url, VidmeIE.ie_key())
+
+            total = int_or_none(page.get('page', {}).get('total'))
+            if total and self._LIMIT * page_num >= total:
+                break
+
+    def _real_extract(self, url):
+        user_name = self._match_id(url)
+
+        user_id = self._download_json(
+            'https://api.vid.me/userByUsername?username=%s' % user_name,
+            user_name)['user']['user_id']
+
+        return self.playlist_result(self._entries(user_id, user_name), user_id, user_name)

From eb2533ec4cc55f1a100e508ea683f345cc84dac9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 5 Feb 2016 23:57:35 +0600
Subject: [PATCH 3904/6219] [vidme:user:likes] Add extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/vidme.py    | 50 ++++++++++++++++++++++----------
 2 files changed, 36 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index bedd2397c..2fbc7f812 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -825,6 +825,7 @@ from .videott import VideoTtIE
 from .vidme import (
     VidmeIE,
     VidmeUserIE,
+    VidmeUserLikesIE,
 )
 from .vidzi import VidziIE
 from .vier import VierIE, VierVideosIE
diff --git a/youtube_dl/extractor/vidme.py b/youtube_dl/extractor/vidme.py
index 91c8349c2..e39f52204 100644
--- a/youtube_dl/extractor/vidme.py
+++ b/youtube_dl/extractor/vidme.py
@@ -206,26 +206,16 @@ class VidmeIE(InfoExtractor):
         }
 
 
-class VidmeUserIE(InfoExtractor):
-    _VALID_URL = r'https?://vid\.me/(?:e/)?(?P<id>[\da-zA-Z]{6,})'
-    _TEST = {
-        'url': 'https://vid.me/EFARCHIVE',
-        'info_dict': {
-            'id': '3834632',
-            'title': 'EFARCHIVE',
-        },
-        'playlist_mincount': 238,
-    }
-
+class VidmeListBaseIE(InfoExtractor):
     # Max possible limit according to https://docs.vid.me/#api-Videos-List
     _LIMIT = 100
 
     def _entries(self, user_id, user_name):
         for page_num in itertools.count(1):
             page = self._download_json(
-                'https://api.vid.me/videos/list?user=%s&limit=%d&offset=%d'
-                % (user_id, self._LIMIT, (page_num - 1) * self._LIMIT), user_name,
-                'Downloading user page %d' % page_num)
+                'https://api.vid.me/videos/%s?user=%s&limit=%d&offset=%d'
+                % (self._API_ITEM, user_id, self._LIMIT, (page_num - 1) * self._LIMIT),
+                user_name, 'Downloading user %s page %d' % (self._API_ITEM, page_num))
 
             videos = page.get('videos', [])
             if not videos:
@@ -247,4 +237,34 @@ class VidmeUserIE(InfoExtractor):
             'https://api.vid.me/userByUsername?username=%s' % user_name,
             user_name)['user']['user_id']
 
-        return self.playlist_result(self._entries(user_id, user_name), user_id, user_name)
+        return self.playlist_result(
+            self._entries(user_id, user_name), user_id,
+            '%s - %s' % (user_name, self._TITLE))
+
+
+class VidmeUserIE(VidmeListBaseIE):
+    _VALID_URL = r'https?://vid\.me/(?:e/)?(?P<id>[\da-zA-Z]{6,})(?!/likes)(?:[^\da-zA-Z]|$)'
+    _API_ITEM = 'list'
+    _TITLE = 'Videos'
+    _TEST = {
+        'url': 'https://vid.me/EFARCHIVE',
+        'info_dict': {
+            'id': '3834632',
+            'title': 'EFARCHIVE - %s' % _TITLE,
+        },
+        'playlist_mincount': 238,
+    }
+
+
+class VidmeUserLikesIE(VidmeListBaseIE):
+    _VALID_URL = r'https?://vid\.me/(?:e/)?(?P<id>[\da-zA-Z]{6,})/likes'
+    _API_ITEM = 'likes'
+    _TITLE = 'Likes'
+    _TEST = {
+        'url': 'https://vid.me/ErinAlexis/likes',
+        'info_dict': {
+            'id': '6483530',
+            'title': 'ErinAlexis - %s' % _TITLE,
+        },
+        'playlist_mincount': 415,
+    }

From d97b0e324152a1313675fcaea22af72b6593e365 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 5 Feb 2016 23:58:26 +0600
Subject: [PATCH 3905/6219] [vidme] Clarify IE_NAMEs

---
 youtube_dl/extractor/vidme.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/vidme.py b/youtube_dl/extractor/vidme.py
index e39f52204..b1156d531 100644
--- a/youtube_dl/extractor/vidme.py
+++ b/youtube_dl/extractor/vidme.py
@@ -13,6 +13,7 @@ from ..utils import (
 
 
 class VidmeIE(InfoExtractor):
+    IE_NAME = 'vidme'
     _VALID_URL = r'https?://vid\.me/(?:e/)?(?P<id>[\da-zA-Z]{,5})(?:[^\da-zA-Z]|$)'
     _TESTS = [{
         'url': 'https://vid.me/QNB',
@@ -243,6 +244,7 @@ class VidmeListBaseIE(InfoExtractor):
 
 
 class VidmeUserIE(VidmeListBaseIE):
+    IE_NAME = 'vidme:user'
     _VALID_URL = r'https?://vid\.me/(?:e/)?(?P<id>[\da-zA-Z]{6,})(?!/likes)(?:[^\da-zA-Z]|$)'
     _API_ITEM = 'list'
     _TITLE = 'Videos'
@@ -257,6 +259,7 @@ class VidmeUserIE(VidmeListBaseIE):
 
 
 class VidmeUserLikesIE(VidmeListBaseIE):
+    IE_NAME = 'vidme:user:likes'
     _VALID_URL = r'https?://vid\.me/(?:e/)?(?P<id>[\da-zA-Z]{6,})/likes'
     _API_ITEM = 'likes'
     _TITLE = 'Likes'

From 23d17e4bebdff9a5defa012e4fa5cd99db605919 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 6 Feb 2016 06:44:38 +0600
Subject: [PATCH 3906/6219] [youtube] Fix automatic captions

---
 youtube_dl/extractor/youtube.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index e9ed77e42..9b346d27a 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -369,6 +369,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         # RTMP (unnamed)
         '_rtmp': {'protocol': 'rtmp'},
     }
+    _SUBTITLE_FORMATS = ('ttml', 'vtt')
 
     IE_NAME = 'youtube'
     _TESTS = [
@@ -918,7 +919,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             if lang in sub_lang_list:
                 continue
             sub_formats = []
-            for ext in ['ttml', 'vtt']:
+            for ext in self._SUBTITLE_FORMATS:
                 params = compat_urllib_parse.urlencode({
                     'lang': lang,
                     'v': video_id,
@@ -988,7 +989,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             for lang_node in caption_list.findall('target'):
                 sub_lang = lang_node.attrib['lang_code']
                 sub_formats = []
-                for ext in ['sbv', 'vtt', 'srt']:
+                for ext in self._SUBTITLE_FORMATS:
                     params = compat_urllib_parse.urlencode({
                         'lang': original_lang,
                         'tlang': sub_lang,

From bcbbb98bfe5c5309f447901c2de4f793647752fc Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 6 Feb 2016 06:47:38 +0100
Subject: [PATCH 3907/6219] [generic] extract dash formats detected using
 content type

---
 youtube_dl/extractor/generic.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index c02fe201c..4c2c76dc5 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1229,13 +1229,15 @@ class GenericIE(InfoExtractor):
 
         # Check for direct link to a video
         content_type = head_response.headers.get('Content-Type', '')
-        m = re.match(r'^(?P<type>audio|video|application(?=/(?:ogg$|(?:vnd\.apple\.|x-)?mpegurl)))/(?P<format_id>.+)$', content_type)
+        m = re.match(r'^(?P<type>audio|video|application(?=/(?:ogg$|dash\+xml|(?:vnd\.apple\.|x-)?mpegurl)))/(?P<format_id>.+)$', content_type)
         if m:
             upload_date = unified_strdate(
                 head_response.headers.get('Last-Modified'))
             formats = []
             if m.group('format_id').endswith('mpegurl'):
                 formats = self._extract_m3u8_formats(url, video_id, 'mp4')
+            elif m.group('format_id').startswith('dash+xml'):
+                formats = self._extract_mpd_formats(url, video_id)
             else:
                 formats = [{
                     'format_id': m.group('format_id'),

From 0826a0b555a06e0cc9047a7779c0f55612c8f3a3 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 6 Feb 2016 06:52:48 +0100
Subject: [PATCH 3908/6219] [common] sort dash formats

---
 youtube_dl/extractor/common.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index d9f31daaa..5287fa4f6 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1481,6 +1481,7 @@ class InfoExtractor(object):
                             existing_format.update(f)
                     else:
                         self.report_warning('Unknown MIME type %s in DASH manifest' % mime_type)
+        self._sort_formats(formats)
         return formats
 
     def _live_title(self, name):

From 91cb6b50652f1ff395b1c38834ce33e99fa1975c Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 6 Feb 2016 14:03:48 +0100
Subject: [PATCH 3909/6219] rename _parse_mpd to _parse_mpd_formats and add
 default value for mpd namespace

---
 test/test_YoutubeDL.py           | 2 +-
 youtube_dl/extractor/common.py   | 6 +++---
 youtube_dl/extractor/facebook.py | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 88c63010e..02caf5908 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -249,7 +249,7 @@ class TestFormatSelection(unittest.TestCase):
         def format_info(f_id):
             info = YoutubeIE._formats[f_id].copy()
 
-            # XXX: In real cases InfoExtractor._parse_mpd() fills up 'acodec'
+            # XXX: In real cases InfoExtractor._parse_mpd_formats() fills up 'acodec'
             # and 'vcodec', while in tests such information is incomplete since
             # commit a6c2c24479e5f4827ceb06f64d855329c0a6f593
             # test_YoutubeDL.test_youtube_format_selection is broken without
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 5287fa4f6..271ba3739 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1343,14 +1343,14 @@ class InfoExtractor(object):
         mpd, urlh = res
         mpd_base_url = re.match(r'https?://.+/', urlh.geturl()).group()
 
-        return self._parse_mpd(
+        return self._parse_mpd_formats(
             compat_etree_fromstring(mpd.encode('utf-8')), mpd_id, mpd_base_url, formats_dict=formats_dict)
 
-    def _parse_mpd(self, mpd_doc, mpd_id=None, mpd_base_url='', formats_dict={}):
+    def _parse_mpd_formats(self, mpd_doc, mpd_id=None, mpd_base_url='', formats_dict={}):
         if mpd_doc.get('type') == 'dynamic':
             return []
 
-        namespace = self._search_regex(r'(?i)^{([^}]+)?}MPD$', mpd_doc.tag, 'namespace')
+        namespace = self._search_regex(r'(?i)^{([^}]+)?}MPD$', mpd_doc.tag, 'namespace', default=None)
 
         def _add_ns(path):
             return self._xpath_ns(path, namespace)
diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index 9c751178f..ed237f081 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -215,7 +215,7 @@ class FacebookIE(InfoExtractor):
                         })
             dash_manifest = f[0].get('dash_manifest')
             if dash_manifest:
-                formats.extend(self._parse_mpd(
+                formats.extend(self._parse_mpd_formats(
                     compat_etree_fromstring(compat_urllib_parse_unquote_plus(dash_manifest))))
         if not formats:
             raise ExtractorError('Cannot find video formats')

From 6a3828fddd6f6cec28ec8676bcc3918d583db77c Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 6 Feb 2016 14:27:04 +0100
Subject: [PATCH 3910/6219] [common] use float conversion instead of using
 division from __future__

---
 youtube_dl/extractor/common.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 271ba3739..9c0421de7 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1,4 +1,4 @@
-from __future__ import unicode_literals, division
+from __future__ import unicode_literals
 
 import base64
 import datetime
@@ -1450,8 +1450,8 @@ class InfoExtractor(object):
                         representation_ms_info = extract_multisegment_info(representation, adaption_set_ms_info)
                         if 'segment_urls' not in representation_ms_info and 'media_template' in representation_ms_info:
                             if 'total_number' not in representation_ms_info and 'segment_duration':
-                                segment_duration = representation_ms_info['segment_duration'] / representation_ms_info['timescale']
-                                representation_ms_info['total_number'] = int(math.ceil(period_duration / segment_duration))
+                                segment_duration = float(representation_ms_info['segment_duration']) / float(representation_ms_info['timescale'])
+                                representation_ms_info['total_number'] = int(math.ceil(float(period_duration) / segment_duration))
                             media_template = representation_ms_info['media_template']
                             media_template = media_template.replace('$RepresentationID$', representation_id)
                             media_template = re.sub(r'\$(Number|Bandwidth)(?:%(0\d+)d)?\$', r'%(\1)\2d', media_template)

From 1b840245bdcc7789f996c9bb163af79efbebc813 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 6 Feb 2016 19:35:32 +0600
Subject: [PATCH 3911/6219] [extractor/generic] Detect DASH manifests and
 extract mpd formats

---
 youtube_dl/extractor/generic.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 4c2c76dc5..6b52e75dc 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1291,7 +1291,7 @@ class GenericIE(InfoExtractor):
 
         self.report_extraction(video_id)
 
-        # Is it an RSS feed, a SMIL file or a XSPF playlist?
+        # Is it an RSS feed, a SMIL file, an XSPF playlist or a MPD manifest?
         try:
             doc = compat_etree_fromstring(webpage.encode('utf-8'))
             if doc.tag == 'rss':
@@ -1300,6 +1300,12 @@ class GenericIE(InfoExtractor):
                 return self._parse_smil(doc, url, video_id)
             elif doc.tag == '{http://xspf.org/ns/0/}playlist':
                 return self.playlist_result(self._parse_xspf(doc, video_id), video_id)
+            elif re.match(r'(?i)^(?:{[^}]+})?MPD$', doc.tag):
+                return {
+                    'id': video_id,
+                    'title': compat_urllib_parse_unquote(os.path.splitext(url_basename(url))[0]),
+                    'formats': self._parse_mpd_formats(doc, video_id),
+                }
         except compat_xml_parse_error:
             pass
 

From 79a350857911d21bb04c90b2db04d7f6f0a3676b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 6 Feb 2016 19:42:03 +0600
Subject: [PATCH 3912/6219] [extractor/generic] Detect DASH manifests in found
 URLs and extract mpd formats

---
 youtube_dl/extractor/generic.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 6b52e75dc..6de777c41 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1959,6 +1959,8 @@ class GenericIE(InfoExtractor):
                 return self.playlist_result(self._extract_xspf_playlist(video_url, video_id), video_id)
             elif ext == 'm3u8':
                 entry_info_dict['formats'] = self._extract_m3u8_formats(video_url, video_id, ext='mp4')
+            elif ext == 'mpd':
+                entry_info_dict['formats'] = self._extract_mpd_formats(video_url, video_id)
             else:
                 entry_info_dict['url'] = video_url
 

From 28ad7df65ddb78c16ac008886d14ae2914aea6be Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 6 Feb 2016 14:51:45 +0100
Subject: [PATCH 3913/6219] [generic] detect MPD manfiest only from the content

---
 youtube_dl/extractor/generic.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 6de777c41..bf61ab2e7 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1229,15 +1229,13 @@ class GenericIE(InfoExtractor):
 
         # Check for direct link to a video
         content_type = head_response.headers.get('Content-Type', '')
-        m = re.match(r'^(?P<type>audio|video|application(?=/(?:ogg$|dash\+xml|(?:vnd\.apple\.|x-)?mpegurl)))/(?P<format_id>.+)$', content_type)
+        m = re.match(r'^(?P<type>audio|video|application(?=/(?:ogg$|(?:vnd\.apple\.|x-)?mpegurl)))/(?P<format_id>.+)$', content_type)
         if m:
             upload_date = unified_strdate(
                 head_response.headers.get('Last-Modified'))
             formats = []
             if m.group('format_id').endswith('mpegurl'):
                 formats = self._extract_m3u8_formats(url, video_id, 'mp4')
-            elif m.group('format_id').startswith('dash+xml'):
-                formats = self._extract_mpd_formats(url, video_id)
             else:
                 formats = [{
                     'format_id': m.group('format_id'),

From 3ceccade87bf9ef8bccf92b353381e04cb32604d Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Sat, 6 Feb 2016 22:33:49 +0600
Subject: [PATCH 3914/6219] [README.md] Improve output template documentation
 and add more examples

---
 README.md | 56 ++++++++++++++++++++++++++++++++++++++++++-------------
 1 file changed, 43 insertions(+), 13 deletions(-)

diff --git a/README.md b/README.md
index 79cd08df4..c73b250fb 100644
--- a/README.md
+++ b/README.md
@@ -442,30 +442,60 @@ On Windows you may also need to setup the `%HOME%` environment variable manually
 
 The `-o` option allows users to indicate a template for the output file names. The basic usage is not to set any template arguments when downloading a single file, like in `youtube-dl -o funny_video.flv "http://some/video"`. However, it may contain special sequences that will be replaced when downloading each video. The special sequences have the format `%(NAME)s`. To clarify, that is a percent symbol followed by a name in parentheses, followed by a lowercase S. Allowed names are:
 
- - `id`: The sequence will be replaced by the video identifier.
- - `url`: The sequence will be replaced by the video URL.
- - `uploader`: The sequence will be replaced by the nickname of the person who uploaded the video.
- - `upload_date`: The sequence will be replaced by the upload date in YYYYMMDD format.
- - `title`: The sequence will be replaced by the video title.
- - `ext`: The sequence will be replaced by the appropriate extension (like flv or mp4).
- - `epoch`: The sequence will be replaced by the Unix epoch when creating the file.
- - `autonumber`: The sequence will be replaced by a five-digit number that will be increased with each download, starting at zero.
- - `playlist`: The sequence will be replaced by the name or the id of the playlist that contains the video.
- - `playlist_index`: The sequence will be replaced by the index of the video in the playlist padded with leading zeros according to the total length of the playlist.
- - `format_id`: The sequence will be replaced by the format code specified by `--format`.
- - `duration`: The sequence will be replaced by the length of the video in seconds.
+ - `id`: Video identifier.
+ - `url`: Video URL.
+ - `uploader`: Nickname of the person who uploaded the video.
+ - `upload_date`: Upload date in YYYYMMDD format.
+ - `title`: Video title.
+ - `ext`: Video filename extension.
+ - `epoch`: Unix epoch when creating the file.
+ - `autonumber`: Five-digit number that will be increased with each download, starting at zero.
+ - `playlist`: Name or id of the playlist that contains the video.
+ - `playlist_index`: Index of the video in the playlist padded with leading zeros according to the total length of the playlist.
+ - `format_id`: Format code specified by `--format`.
+ - `duration`: Length of the video in seconds.
 
-Note that some of the aforementioned sequences are not guaranteed to be present since they depend on the metadata obtained by particular extractor, such sequences will be replaced with `NA`.
+Available for the video that belong to some logical chapter or section:
+ - `chapter`: Name or title of the chapter the video belongs to.
+ - `chapter_number`: Number of the chapter the video belongs to, as an integer.
+ - `chapter_id`: Id of the chapter the video belongs to, as a unicode string.
+
+Available for the video that is an episode of some series or programme:
+ - `series`: Title of the series or programme the video episode belongs to.
+ - `season`: Title of the season the video episode belongs to.
+ - `season_number`: Number of the season the video episode belongs to, as an integer.
+ - `season_id`:Id of the season the video episode belongs to, as a unicode string.
+ - `episode`: Title of the video episode.
+ - `episode_number`: Number of the video episode within a season, as an integer.
+ - `episode_id`: Id of the video episode, as a unicode string.
+
+Each aforementioned sequence when referenced in output template will be replaced by the actual value corresponding to the sequence name. Note that some of the sequences are not guaranteed to be present since they depend on the metadata obtained by particular extractor, such sequences will be replaced with `NA`.
+
+For example for `-o %(title)s-%(id)s.%(ext)s` and mp4 video with title `youtube-dl test video` and id `BaW_jenozKcj` this will result in a `youtube-dl test video-BaW_jenozKcj.mp4` file created in the current directory.
+
+Output template can also contain arbitrary hierarchical path, e.g. `-o %(playlist)s/%(playlist_index)s - %(title)s.%(ext)s` that will result in downloading each video in a directory corresponding to this path template. Any missing directory will be automatically created for you.
 
 The current default template is `%(title)s-%(id)s.%(ext)s`.
 
 In some cases, you don't want special characters such as 中, spaces, or &, such as when transferring the downloaded filename to a Windows system or the filename through an 8bit-unsafe channel. In these cases, add the `--restrict-filenames` flag to get a shorter title:
 
+Examples:
+
 ```bash
 $ youtube-dl --get-filename -o "%(title)s.%(ext)s" BaW_jenozKc
 youtube-dl test video ''_ä↭𝕐.mp4    # All kinds of weird characters
+
 $ youtube-dl --get-filename -o "%(title)s.%(ext)s" BaW_jenozKc --restrict-filenames
 youtube-dl_test_video_.mp4          # A simple file name
+
+# Download YouTube playlist videos in separate directory indexed by video order in a playlist
+$ youtube-dl -o '%(playlist)s/%(playlist_index)s - %(title)s.%(ext)s' https://www.youtube.com/playlist?list=PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re
+
+# Download Udemy course keeping each chapter in separate directory under MyVideos directory in your home
+$ youtube-dl -u user -p password -o '~/MyVideos/%(playlist)s/%(chapter_number)s - %(chapter)s/%(title)s.%(ext)s' https://www.udemy.com/java-tutorial/
+
+# Download entire series season keeping each series and each season in separate directory under C:/MyVideos
+$ youtube-dl -o 'C:/MyVideos/%(series)s/%(season_number)s - %(season)s/%(episode_number)s - %(episode)s.%(ext)s' http://videomore.ru/kino_v_detalayah/5_sezon/367617
 ```
 
 # FORMAT SELECTION

From 4340727e6cbdb55c2d190fe41f38f614a3e3780b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 6 Feb 2016 22:36:30 +0600
Subject: [PATCH 3915/6219] [videomore] Fix typo

---
 youtube_dl/extractor/videomore.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/videomore.py b/youtube_dl/extractor/videomore.py
index fcee940e6..0bd1e1eec 100644
--- a/youtube_dl/extractor/videomore.py
+++ b/youtube_dl/extractor/videomore.py
@@ -114,7 +114,7 @@ class VideomoreIE(InfoExtractor):
 
         data = self._download_json(
             'http://videomore.ru/video/tracks/%s.json' % video_id,
-            video_id, 'Downloadinng video JSON')
+            video_id, 'Downloading video JSON')
 
         title = data.get('title') or data['project_title']
         description = data.get('description') or data.get('description_raw')

From bd6b6f662268a8a59a1cb9e72ac38a84dfa07822 Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Sat, 6 Feb 2016 22:36:30 +0600
Subject: [PATCH 3916/6219] [README.md] Fix typo

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index c73b250fb..85cd02a01 100644
--- a/README.md
+++ b/README.md
@@ -455,7 +455,7 @@ The `-o` option allows users to indicate a template for the output file names. T
  - `format_id`: Format code specified by `--format`.
  - `duration`: Length of the video in seconds.
 
-Available for the video that belong to some logical chapter or section:
+Available for the video that belongs to some logical chapter or section:
  - `chapter`: Name or title of the chapter the video belongs to.
  - `chapter_number`: Number of the chapter the video belongs to, as an integer.
  - `chapter_id`: Id of the chapter the video belongs to, as a unicode string.

From 4b8874975cdcd9ea656ce603f87d94b871d2849c Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Sat, 6 Feb 2016 22:39:50 +0600
Subject: [PATCH 3917/6219] [README.md] Remove non-relevant info

---
 README.md | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/README.md b/README.md
index 85cd02a01..7b8a1a22f 100644
--- a/README.md
+++ b/README.md
@@ -457,16 +457,16 @@ The `-o` option allows users to indicate a template for the output file names. T
 
 Available for the video that belongs to some logical chapter or section:
  - `chapter`: Name or title of the chapter the video belongs to.
- - `chapter_number`: Number of the chapter the video belongs to, as an integer.
- - `chapter_id`: Id of the chapter the video belongs to, as a unicode string.
+ - `chapter_number`: Number of the chapter the video belongs to.
+ - `chapter_id`: Id of the chapter the video belongs to.
 
 Available for the video that is an episode of some series or programme:
  - `series`: Title of the series or programme the video episode belongs to.
  - `season`: Title of the season the video episode belongs to.
- - `season_number`: Number of the season the video episode belongs to, as an integer.
- - `season_id`:Id of the season the video episode belongs to, as a unicode string.
+ - `season_number`: Number of the season the video episode belongs to.
+ - `season_id`:Id of the season the video episode belongs to.
  - `episode`: Title of the video episode.
- - `episode_number`: Number of the video episode within a season, as an integer.
+ - `episode_number`: Number of the video episode within a season.
  - `episode_id`: Id of the video episode, as a unicode string.
 
 Each aforementioned sequence when referenced in output template will be replaced by the actual value corresponding to the sequence name. Note that some of the sequences are not guaranteed to be present since they depend on the metadata obtained by particular extractor, such sequences will be replaced with `NA`.

From 860db2d508f2a18c1b48adb9b1eddc753721bd47 Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Sat, 6 Feb 2016 22:40:20 +0600
Subject: [PATCH 3918/6219] [README.md] Fix typo

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 7b8a1a22f..7a180dccf 100644
--- a/README.md
+++ b/README.md
@@ -464,7 +464,7 @@ Available for the video that is an episode of some series or programme:
  - `series`: Title of the series or programme the video episode belongs to.
  - `season`: Title of the season the video episode belongs to.
  - `season_number`: Number of the season the video episode belongs to.
- - `season_id`:Id of the season the video episode belongs to.
+ - `season_id`: Id of the season the video episode belongs to.
  - `episode`: Title of the video episode.
  - `episode_number`: Number of the video episode within a season.
  - `episode_id`: Id of the video episode, as a unicode string.

From 56401e1e5f53e126bf124699b2f62bb670078fd3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 6 Feb 2016 23:24:22 +0600
Subject: [PATCH 3919/6219] [downloader/hls] Do not send 'q' to ffmpeg on
 Windows (Closes #8300)

---
 youtube_dl/downloader/hls.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index cb34dc4ab..2a775bf00 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -3,6 +3,7 @@ from __future__ import unicode_literals
 import os
 import re
 import subprocess
+import sys
 
 from .common import FileDownloader
 from .fragment import FragmentFD
@@ -57,8 +58,10 @@ class HlsFD(FileDownloader):
             # subprocces.run would send the SIGKILL signal to ffmpeg and the
             # mp4 file couldn't be played, but if we ask ffmpeg to quit it
             # produces a file that is playable (this is mostly useful for live
-            # streams)
-            proc.communicate(b'q')
+            # streams). Note that Windows is not affected and produces playable
+            # files (see https://github.com/rg3/youtube-dl/issues/8300).
+            if sys.platform != 'win32':
+                proc.communicate(b'q')
             raise
         if retval == 0:
             fsize = os.path.getsize(encodeFilename(tmpfilename))

From 7e62c2eb6d13ebf0fdd24c90c2f439177d485081 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 6 Feb 2016 18:51:05 +0100
Subject: [PATCH 3920/6219] [FFmpegSubtitlesConvertorPP] fix not working when
 srt is used as the intermediate format between ttml/dfxp and other format

It was trying to use the ttml/dfxp file with ffmpeg, which doesn't have support for them.
I broke it in e04398e397d4a4f7e0d0994355d5a94c22441e7c.
---
 youtube_dl/postprocessor/ffmpeg.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 22d7ac65a..6daa6a8b4 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -508,6 +508,7 @@ class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor):
 
                 with io.open(srt_file, 'wt', encoding='utf-8') as f:
                     f.write(srt_data)
+                old_file = srt_file
 
                 ext = 'srt'
                 subs[lang] = {

From 3547d26587214f0e8ade1295dab0438d88a567de Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 6 Feb 2016 18:58:18 +0100
Subject: [PATCH 3921/6219] [FFmpegSubtitlesConvertorPP] correctly update the
 extension (fixes #8444)

---
 youtube_dl/postprocessor/ffmpeg.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 6daa6a8b4..05f2f0623 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -523,7 +523,7 @@ class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor):
 
             with io.open(new_file, 'rt', encoding='utf-8') as f:
                 subs[lang] = {
-                    'ext': ext,
+                    'ext': new_ext,
                     'data': f.read(),
                 }
 

From 7b8b007cd9b55326b45598130c9cdde8839d5340 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 6 Feb 2016 19:04:18 +0100
Subject: [PATCH 3922/6219] [FFmpegSubtitlesConvertorPP] remove intermediate
 srt files

---
 youtube_dl/postprocessor/ffmpeg.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 05f2f0623..6955b01f9 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -518,6 +518,8 @@ class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor):
 
                 if new_ext == 'srt':
                     continue
+                else:
+                    sub_filenames.append(srt_file)
 
             self.run_ffmpeg(old_file, new_file, ['-f', new_format])
 

From d4f8e834042ba120db5662b0150e1a01afe81ca8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 6 Feb 2016 19:04:53 +0100
Subject: [PATCH 3923/6219] [FFmpegSubtitlesConvertorPP] remove unused variable

---
 youtube_dl/postprocessor/ffmpeg.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 6955b01f9..380bc6f29 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -510,7 +510,6 @@ class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor):
                     f.write(srt_data)
                 old_file = srt_file
 
-                ext = 'srt'
                 subs[lang] = {
                     'ext': 'srt',
                     'data': srt_data

From 5dd4b3468f3a400b865db2e7868bbcbd7c559626 Mon Sep 17 00:00:00 2001
From: Brian Foley <bpfoley@gmail.com>
Date: Sat, 6 Feb 2016 18:42:57 +0000
Subject: [PATCH 3924/6219] [rte:radio] Add support for RTMP downloads,
 alternate URL style

This is useful as
a) RTMP downloads are a good deal faster to download
b) Older items are available only as RTMP streams
---
 youtube_dl/extractor/rte.py | 43 +++++++++++++++++++++++++++++++------
 1 file changed, 37 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/rte.py b/youtube_dl/extractor/rte.py
index 47c8331fe..73c9788be 100644
--- a/youtube_dl/extractor/rte.py
+++ b/youtube_dl/extractor/rte.py
@@ -1,6 +1,8 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..utils import (
     float_or_none,
@@ -61,12 +63,15 @@ class RteIE(InfoExtractor):
 class RteRadioIE(InfoExtractor):
     IE_NAME = 'rte:radio'
     IE_DESC = 'Raidió Teilifís Éireann radio'
-    # Radioplayer URLs have the specifier #!rii=<channel_id>:<id>:<playable_item_id>:<date>:
+    # Radioplayer URLs have two distinct specifier formats,
+    # the old format #!rii=<channel_id>:<id>:<playable_item_id>:<date>:
+    # the new format #!rii=b<channel_id>_<id>_<playable_item_id>_<date>_
     # where the IDs are int/empty, the date is DD-MM-YYYY, and the specifier may be truncated.
     # An <id> uniquely defines an individual recording, and is the only part we require.
-    _VALID_URL = r'https?://(?:www\.)?rte\.ie/radio/utils/radioplayer/rteradioweb\.html#!rii=(?:[0-9]*)(?:%3A|:)(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?rte\.ie/radio/utils/radioplayer/rteradioweb\.html#!rii=(?:b?[0-9]*)(?:%3A|:|%5F|_)(?P<id>[0-9]+)'
 
-    _TEST = {
+    _TESTS = [{
+        # Old-style player URL; HLS and RTMPE formats
         'url': 'http://www.rte.ie/radio/utils/radioplayer/rteradioweb.html#!rii=16:10507902:2414:27-12-2015:',
         'info_dict': {
             'id': '10507902',
@@ -81,7 +86,23 @@ class RteRadioIE(InfoExtractor):
         'params': {
             'skip_download': 'f4m fails with --test atm'
         }
-    }
+    }, {
+        # New-style player URL; RTMPE formats only
+        'url': 'http://rte.ie/radio/utils/radioplayer/rteradioweb.html#!rii=b16_3250678_8861_06-04-2012_',
+        'info_dict': {
+            'id': '3250678',
+            'ext': 'flv',
+            'title': 'The Lyric Concert with Paul Herriott',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'description': '',
+            'timestamp': 1333742400,
+            'upload_date': '20120406',
+            'duration': 7199.016,
+        },
+        'params': {
+            'skip_download': 'f4m fails with --test atm'
+        }
+    }]
 
     def _real_extract(self, url):
         item_id = self._match_id(url)
@@ -102,8 +123,18 @@ class RteRadioIE(InfoExtractor):
 
         formats = []
 
-        if mg.get('url') and not mg['url'].startswith('rtmpe:'):
-            formats.append({'url': mg['url']})
+        if mg.get('url'):
+            m = re.match(r'(?P<url>rtmpe?://[^/]+)/(?P<app>.+)/(?P<playpath>mp4:.*)', mg['url'])
+            if m:
+                m = m.groupdict()
+                formats.append({
+                    'url': m['url'] + '/' + m['app'],
+                    'app': m['app'],
+                    'play_path': m['playpath'],
+                    'player_url': url,
+                    'ext': 'flv',
+                    'format_id': 'rtmp',
+                })
 
         if mg.get('hls_server') and mg.get('hls_url'):
             formats.extend(self._extract_m3u8_formats(

From f406c78785b57afb584cc0ee8ef4e84e568b0614 Mon Sep 17 00:00:00 2001
From: pulpe <Pulpan3@gmail.com>
Date: Sat, 6 Feb 2016 21:23:41 +0100
Subject: [PATCH 3925/6219] [IPrima] Fix extractor (fixes #7617)

---
 youtube_dl/extractor/iprima.py | 99 ++++++++--------------------------
 1 file changed, 22 insertions(+), 77 deletions(-)

diff --git a/youtube_dl/extractor/iprima.py b/youtube_dl/extractor/iprima.py
index 073777f34..093a7e551 100644
--- a/youtube_dl/extractor/iprima.py
+++ b/youtube_dl/extractor/iprima.py
@@ -2,46 +2,30 @@
 from __future__ import unicode_literals
 
 import re
-from random import random
-from math import floor
+import time
 
 from .common import InfoExtractor
 from ..utils import (
-    ExtractorError,
-    remove_end,
     sanitized_Request,
 )
 
 
 class IPrimaIE(InfoExtractor):
-    _WORKING = False
-    _VALID_URL = r'https?://play\.iprima\.cz/(?:[^/]+/)*(?P<id>[^?#]+)'
+    _VALID_URL = r'https?://play\.iprima\.cz/(?:.+/)?(?P<id>[^?#]+)'
 
     _TESTS = [{
+        'url': 'http://play.iprima.cz/gondici-s-r-o-33',
+        'info_dict': {
+            'id': 'p136534',
+            'ext': 'mp4',
+            'title': 'Gondíci s. r. o. (34)',
+            'description': 'md5:16577c629d006aa91f59ca8d8e7f99bd',
+        },
+        'params': {
+            'skip_download': True,  # m3u8 download
+        },
+    }, {
         'url': 'http://play.iprima.cz/particka/particka-92',
-        'info_dict': {
-            'id': '39152',
-            'ext': 'flv',
-            'title': 'Partička (92)',
-            'description': 'md5:74e9617e51bca67c3ecfb2c6f9766f45',
-            'thumbnail': 'http://play.iprima.cz/sites/default/files/image_crops/image_620x349/3/491483_particka-92_image_620x349.jpg',
-        },
-        'params': {
-            'skip_download': True,  # requires rtmpdump
-        },
-    }, {
-        'url': 'http://play.iprima.cz/particka/tchibo-particka-jarni-moda',
-        'info_dict': {
-            'id': '9718337',
-            'ext': 'flv',
-            'title': 'Tchibo Partička - Jarní móda',
-            'thumbnail': 're:^http:.*\.jpg$',
-        },
-        'params': {
-            'skip_download': True,  # requires rtmpdump
-        },
-    }, {
-        'url': 'http://play.iprima.cz/zpravy-ftv-prima-2752015',
         'only_matching': True,
     }]
 
@@ -51,62 +35,23 @@ class IPrimaIE(InfoExtractor):
 
         webpage = self._download_webpage(url, video_id)
 
-        if re.search(r'Nemáte oprávnění přistupovat na tuto stránku\.\s*</div>', webpage):
-            raise ExtractorError(
-                '%s said: You do not have permission to access this page' % self.IE_NAME, expected=True)
+        video_id = self._search_regex(r'data-product="([^"]+)">', webpage, 'real id')
 
-        player_url = (
-            'http://embed.livebox.cz/iprimaplay/player-embed-v2.js?__tok%s__=%s' %
-            (floor(random() * 1073741824), floor(random() * 1073741824))
-        )
-
-        req = sanitized_Request(player_url)
+        req = sanitized_Request('http://play.iprima.cz/prehravac/init?_infuse=1'
+            '&_ts=%s&productId=%s' % (round(time.time()), 'p22201'))
         req.add_header('Referer', url)
-        playerpage = self._download_webpage(req, video_id)
+        playerpage = self._download_webpage(req, video_id, note='Downloading player')
 
-        base_url = ''.join(re.findall(r"embed\['stream'\] = '(.+?)'.+'(\?auth=)'.+'(.+?)';", playerpage)[1])
+        m3u8_url = self._search_regex(r"'src': '([^']+\.m3u8)'", playerpage, 'm3u8 url')
 
-        zoneGEO = self._html_search_regex(r'"zoneGEO":(.+?),', webpage, 'zoneGEO')
-        if zoneGEO != '0':
-            base_url = base_url.replace('token', 'token_' + zoneGEO)
-
-        formats = []
-        for format_id in ['lq', 'hq', 'hd']:
-            filename = self._html_search_regex(
-                r'"%s_id":(.+?),' % format_id, webpage, 'filename')
-
-            if filename == 'null':
-                continue
-
-            real_id = self._search_regex(
-                r'Prima-(?:[0-9]{10}|WEB)-([0-9]+)[-_]',
-                filename, 'real video id')
-
-            if format_id == 'lq':
-                quality = 0
-            elif format_id == 'hq':
-                quality = 1
-            elif format_id == 'hd':
-                quality = 2
-                filename = 'hq/' + filename
-
-            formats.append({
-                'format_id': format_id,
-                'url': base_url,
-                'quality': quality,
-                'play_path': 'mp4:' + filename.replace('"', '')[:-4],
-                'rtmp_live': True,
-                'ext': 'flv',
-            })
+        formats = self._extract_m3u8_formats(m3u8_url, video_id, ext='mp4')
 
         self._sort_formats(formats)
 
         return {
-            'id': real_id,
-            'title': remove_end(self._og_search_title(webpage), ' | Prima PLAY'),
+            'id': video_id,
+            'title': self._og_search_title(webpage),
             'thumbnail': self._og_search_thumbnail(webpage),
             'formats': formats,
-            'description': self._search_regex(
-                r'<p[^>]+itemprop="description"[^>]*>([^<]+)',
-                webpage, 'description', default=None),
+            'description': self._og_search_description(webpage),
         }

From 70328330117e3c4c0e413b8c41bfd9e8e55fc62a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 6 Feb 2016 21:37:28 +0100
Subject: [PATCH 3926/6219] [iprima] Follow pep8

---
 youtube_dl/extractor/iprima.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/iprima.py b/youtube_dl/extractor/iprima.py
index 093a7e551..d922f905a 100644
--- a/youtube_dl/extractor/iprima.py
+++ b/youtube_dl/extractor/iprima.py
@@ -37,7 +37,8 @@ class IPrimaIE(InfoExtractor):
 
         video_id = self._search_regex(r'data-product="([^"]+)">', webpage, 'real id')
 
-        req = sanitized_Request('http://play.iprima.cz/prehravac/init?_infuse=1'
+        req = sanitized_Request(
+            'http://play.iprima.cz/prehravac/init?_infuse=1'
             '&_ts=%s&productId=%s' % (round(time.time()), 'p22201'))
         req.add_header('Referer', url)
         playerpage = self._download_webpage(req, video_id, note='Downloading player')

From 59b35c674597b913454b3efc82d60f903fb83fcb Mon Sep 17 00:00:00 2001
From: pulpe <Pulpan3@gmail.com>
Date: Sat, 6 Feb 2016 21:42:24 +0100
Subject: [PATCH 3927/6219] [IPrima] Remove test video_id

---
 youtube_dl/extractor/iprima.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/iprima.py b/youtube_dl/extractor/iprima.py
index d922f905a..61a0de472 100644
--- a/youtube_dl/extractor/iprima.py
+++ b/youtube_dl/extractor/iprima.py
@@ -39,7 +39,7 @@ class IPrimaIE(InfoExtractor):
 
         req = sanitized_Request(
             'http://play.iprima.cz/prehravac/init?_infuse=1'
-            '&_ts=%s&productId=%s' % (round(time.time()), 'p22201'))
+            '&_ts=%s&productId=%s' % (round(time.time()), video_id))
         req.add_header('Referer', url)
         playerpage = self._download_webpage(req, video_id, note='Downloading player')
 

From b8b465af3e83fb19c1818c2fa83f0c5f753dd917 Mon Sep 17 00:00:00 2001
From: Erwin de Haan <git@erayan.eu>
Date: Sat, 6 Feb 2016 23:37:55 +0100
Subject: [PATCH 3928/6219] [vlive] Updated to new V App/VLive api.

More robust with getting keys and ids from website.
---
 youtube_dl/extractor/vlive.py | 44 ++++++++++++++++-------------------
 1 file changed, 20 insertions(+), 24 deletions(-)

diff --git a/youtube_dl/extractor/vlive.py b/youtube_dl/extractor/vlive.py
index 86c1cb5ef..3e1f8ef07 100644
--- a/youtube_dl/extractor/vlive.py
+++ b/youtube_dl/extractor/vlive.py
@@ -9,17 +9,18 @@ from time import time
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
-    determine_ext
+    determine_ext,
+    int_or_none
 )
 from ..compat import compat_urllib_parse
 
 
 class VLiveIE(InfoExtractor):
     IE_NAME = 'vlive'
-    # www.vlive.tv/video/ links redirect to m.vlive.tv/video/ for mobile devices
+    # vlive.tv/video/ links redirect to www.vlive.tv/video/ 
     _VALID_URL = r'https?://(?:(www|m)\.)?vlive\.tv/video/(?P<id>[0-9]+)'
     _TEST = {
-        'url': 'http://m.vlive.tv/video/1326',
+        'url': 'http://www.vlive.tv/video/1326',
         'md5': 'cc7314812855ce56de70a06a27314983',
         'info_dict': {
             'id': '1326',
@@ -28,50 +29,45 @@ class VLiveIE(InfoExtractor):
             'creator': 'Girl\'s Day',
         },
     }
-    _SECRET = 'rFkwZet6pqk1vQt6SxxUkAHX7YL3lmqzUMrU4IDusTo4jEBdtOhNfT4BYYAdArwH'
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
         webpage = self._download_webpage(
-            'http://m.vlive.tv/video/%s' % video_id,
+            'http://www.vlive.tv/video/%s' % video_id,
             video_id, note='Download video page')
 
+        long_video_id = self._search_regex(
+            r'vlive\.tv\.video\.ajax\.request\.handler\.init\("[0-9]+",\s?"[^"]*",\s?"([^"]+)",\s?"[^"]+",\s?"[^"]*",\s?"[^"]*"\)', webpage, 'long_video_id')
+
+        key = self._search_regex(
+            r'vlive\.tv\.video\.ajax\.request\.handler\.init\("[0-9]+",\s?"[^"]*",\s?"[^"]+",\s?"([^"]+)",\s?"[^"]*",\s?"[^"]*"\)', webpage, 'key')
+
         title = self._og_search_title(webpage)
         thumbnail = self._og_search_thumbnail(webpage)
         creator = self._html_search_regex(
-            r'<span[^>]+class="name">([^<>]+)</span>', webpage, 'creator')
+            r'<div class="info_area">\s*<strong[^>]+class="name">([^<>]+)</strong>', webpage, 'creator',fatal=False)
 
-        url = 'http://global.apis.naver.com/globalV/globalV/vod/%s/playinfo?' % video_id
-        msgpad = '%.0f' % (time() * 1000)
-        md = b64encode(
-            hmac.new(self._SECRET.encode('ascii'),
-                     (url[:255] + msgpad).encode('ascii'), sha1).digest()
-        )
-        url += '&' + compat_urllib_parse.urlencode({'msgpad': msgpad, 'md': md})
+        # doct = document type (xml or json), cpt = caption type (vtt or ttml)
+        url = "http://global.apis.naver.com/rmcnmv/rmcnmv/vod_play_videoInfo.json?videoId=%s&key=%s&ptc=http&doct=json&cpt=vtt" % (long_video_id, key)
+        
         playinfo = self._download_json(url, video_id, 'Downloading video json')
 
-        if playinfo.get('message', '') != 'success':
-            raise ExtractorError(playinfo.get('message', 'JSON request unsuccessful'))
-
-        if not playinfo.get('result'):
-            raise ExtractorError('No videos found.')
-
         formats = []
-        for vid in playinfo['result'].get('videos', {}).get('list', []):
+        for vid in playinfo.get('videos', {}).get('list', []):
             formats.append({
                 'url': vid['source'],
                 'ext': 'mp4',
                 'abr': vid.get('bitrate', {}).get('audio'),
                 'vbr': vid.get('bitrate', {}).get('video'),
-                'format_id': vid['encodingOption']['name'],
-                'height': vid.get('height'),
-                'width': vid.get('width'),
+                'format_id': vid.get('encodingOption', {}).get('name'),
+                'height': int_or_none(vid.get('encodingOption', {}).get('height')),
+                'width': int_or_none(vid.get('encodingOption', {}).get('width')),
             })
         self._sort_formats(formats)
 
         subtitles = {}
-        for caption in playinfo['result'].get('captions', {}).get('list', []):
+        for caption in playinfo.get('captions', {}).get('list', []):
             subtitles[caption['language']] = [
                 {'ext': determine_ext(caption['source'], default_ext='vtt'),
                  'url': caption['source']}]

From cbecc9b9039d5166185a41ca4d9d6c4d11595c52 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 7 Feb 2016 06:12:53 +0600
Subject: [PATCH 3929/6219] [utils] Add dict_get convenience method

---
 test/test_utils.py  | 14 ++++++++++++++
 youtube_dl/utils.py |  9 +++++++++
 2 files changed, 23 insertions(+)

diff --git a/test/test_utils.py b/test/test_utils.py
index 1c3290d9b..e3dd019af 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -22,6 +22,7 @@ from youtube_dl.utils import (
     DateRange,
     detect_exe_version,
     determine_ext,
+    dict_get,
     encode_compat_str,
     encodeFilename,
     escape_rfc3986,
@@ -450,6 +451,19 @@ class TestUtil(unittest.TestCase):
         data = urlencode_postdata({'username': 'foo@bar.com', 'password': '1234'})
         self.assertTrue(isinstance(data, bytes))
 
+    def test_dict_get(self):
+        d = {
+            'a': 42,
+        }
+        self.assertEqual(dict_get(d, 'a'), 42)
+        self.assertEqual(dict_get(d, 'b'), None)
+        self.assertEqual(dict_get(d, 'b', 42), 42)
+        self.assertEqual(dict_get(d, ('a', )), 42)
+        self.assertEqual(dict_get(d, ('b', 'a', )), 42)
+        self.assertEqual(dict_get(d, ('b', 'c', 'a', 'd', )), 42)
+        self.assertEqual(dict_get(d, ('b', 'c', )), None)
+        self.assertEqual(dict_get(d, ('b', 'c', ), 42), 42)
+
     def test_encode_compat_str(self):
         self.assertEqual(encode_compat_str(b'\xd1\x82\xd0\xb5\xd1\x81\xd1\x82', 'utf-8'), 'тест')
         self.assertEqual(encode_compat_str('тест', 'utf-8'), 'тест')
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 4262ad6ac..652dba59d 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1717,6 +1717,15 @@ def encode_dict(d, encoding='utf-8'):
     return dict((encode(k), encode(v)) for k, v in d.items())
 
 
+def dict_get(d, key_or_keys, default=None):
+    if isinstance(key_or_keys, (list, tuple)):
+        for key in key_or_keys:
+            if d.get(key):
+                return d[key]
+        return default
+    return d.get(key_or_keys, default)
+
+
 def encode_compat_str(string, encoding=preferredencoding(), errors='strict'):
     return string if isinstance(string, compat_str) else compat_str(string, encoding, errors)
 

From 81e0b4f2d182d05ccadc2ef5140fabf2e8cbd60f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 7 Feb 2016 06:14:26 +0600
Subject: [PATCH 3930/6219] Credit @EraYaN for vlive update (#8446)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index a46799506..7cac337a7 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -156,3 +156,4 @@ Tom Gijselinck
 Founder Fang
 Andrew Alexeyew
 Saso Bezlaj
+Erwin de Haan

From 52f5889f77e04420634d70145d689660ea6bfe24 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 7 Feb 2016 06:17:40 +0600
Subject: [PATCH 3931/6219] [vlive] Improve and extract more metadata (Closes
 #8446)

---
 youtube_dl/extractor/vlive.py | 80 +++++++++++++++++++----------------
 1 file changed, 43 insertions(+), 37 deletions(-)

diff --git a/youtube_dl/extractor/vlive.py b/youtube_dl/extractor/vlive.py
index 3e1f8ef07..9e2aa58bd 100644
--- a/youtube_dl/extractor/vlive.py
+++ b/youtube_dl/extractor/vlive.py
@@ -1,32 +1,27 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import hmac
-from hashlib import sha1
-from base64 import b64encode
-from time import time
-
 from .common import InfoExtractor
 from ..utils import (
-    ExtractorError,
-    determine_ext,
-    int_or_none
+    dict_get,
+    float_or_none,
+    int_or_none,
 )
 from ..compat import compat_urllib_parse
 
 
 class VLiveIE(InfoExtractor):
     IE_NAME = 'vlive'
-    # vlive.tv/video/ links redirect to www.vlive.tv/video/ 
-    _VALID_URL = r'https?://(?:(www|m)\.)?vlive\.tv/video/(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://(?:(?:www|m)\.)?vlive\.tv/video/(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://www.vlive.tv/video/1326',
         'md5': 'cc7314812855ce56de70a06a27314983',
         'info_dict': {
             'id': '1326',
             'ext': 'mp4',
-            'title': '[V] Girl\'s Day\'s Broadcast',
-            'creator': 'Girl\'s Day',
+            'title': "[V] Girl's Day's Broadcast",
+            'creator': "Girl's Day",
+            'view_count': int,
         },
     }
 
@@ -34,49 +29,60 @@ class VLiveIE(InfoExtractor):
         video_id = self._match_id(url)
 
         webpage = self._download_webpage(
-            'http://www.vlive.tv/video/%s' % video_id,
-            video_id, note='Download video page')
+            'http://www.vlive.tv/video/%s' % video_id, video_id)
 
         long_video_id = self._search_regex(
-            r'vlive\.tv\.video\.ajax\.request\.handler\.init\("[0-9]+",\s?"[^"]*",\s?"([^"]+)",\s?"[^"]+",\s?"[^"]*",\s?"[^"]*"\)', webpage, 'long_video_id')
+            r'vlive\.tv\.video\.ajax\.request\.handler\.init\(\s*"[0-9]+"\s*,\s*"[^"]*"\s*,\s*"([^"]+)"',
+            webpage, 'long video id')
 
         key = self._search_regex(
-            r'vlive\.tv\.video\.ajax\.request\.handler\.init\("[0-9]+",\s?"[^"]*",\s?"[^"]+",\s?"([^"]+)",\s?"[^"]*",\s?"[^"]*"\)', webpage, 'key')
+            r'vlive\.tv\.video\.ajax\.request\.handler\.init\(\s*"[0-9]+"\s*,\s*"[^"]*"\s*,\s*"[^"]+"\s*,\s*"([^"]+)"',
+            webpage, 'key')
 
         title = self._og_search_title(webpage)
+
+        playinfo = self._download_json(
+            'http://global.apis.naver.com/rmcnmv/rmcnmv/vod_play_videoInfo.json?%s'
+            % compat_urllib_parse.urlencode({
+                'videoId': long_video_id,
+                'key': key,
+                'ptc': 'http',
+                'doct': 'json',  # document type (xml or json)
+                'cpt': 'vtt',  # captions type (vtt or ttml)
+            }), video_id)
+
+        formats = [{
+            'url': vid['source'],
+            'format_id': vid.get('encodingOption', {}).get('name'),
+            'abr': float_or_none(vid.get('bitrate', {}).get('audio')),
+            'vbr': float_or_none(vid.get('bitrate', {}).get('video')),
+            'width': int_or_none(vid.get('encodingOption', {}).get('width')),
+            'height': int_or_none(vid.get('encodingOption', {}).get('height')),
+            'filesize': int_or_none(vid.get('size')),
+        } for vid in playinfo.get('videos', {}).get('list', []) if vid.get('source')]
+        self._sort_formats(formats)
+
         thumbnail = self._og_search_thumbnail(webpage)
         creator = self._html_search_regex(
-            r'<div class="info_area">\s*<strong[^>]+class="name">([^<>]+)</strong>', webpage, 'creator',fatal=False)
+            r'<div[^>]+class="info_area"[^>]*>\s*<strong[^>]+class="name"[^>]*>([^<]+)</strong>',
+            webpage, 'creator', fatal=False)
 
-        # doct = document type (xml or json), cpt = caption type (vtt or ttml)
-        url = "http://global.apis.naver.com/rmcnmv/rmcnmv/vod_play_videoInfo.json?videoId=%s&key=%s&ptc=http&doct=json&cpt=vtt" % (long_video_id, key)
-        
-        playinfo = self._download_json(url, video_id, 'Downloading video json')
-
-        formats = []
-        for vid in playinfo.get('videos', {}).get('list', []):
-            formats.append({
-                'url': vid['source'],
-                'ext': 'mp4',
-                'abr': vid.get('bitrate', {}).get('audio'),
-                'vbr': vid.get('bitrate', {}).get('video'),
-                'format_id': vid.get('encodingOption', {}).get('name'),
-                'height': int_or_none(vid.get('encodingOption', {}).get('height')),
-                'width': int_or_none(vid.get('encodingOption', {}).get('width')),
-            })
-        self._sort_formats(formats)
+        view_count = int_or_none(playinfo.get('meta', {}).get('count'))
 
         subtitles = {}
         for caption in playinfo.get('captions', {}).get('list', []):
-            subtitles[caption['language']] = [
-                {'ext': determine_ext(caption['source'], default_ext='vtt'),
-                 'url': caption['source']}]
+            lang = dict_get(caption, ('language', 'locale', 'country', 'label'))
+            if lang and caption.get('source'):
+                subtitles[lang] = [{
+                    'ext': 'vtt',
+                    'url': caption['source']}]
 
         return {
             'id': video_id,
             'title': title,
             'creator': creator,
             'thumbnail': thumbnail,
+            'view_count': view_count,
             'formats': formats,
             'subtitles': subtitles,
         }

From 86296ad2cd5702a66b05aae79ec9196b62e41e9a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 7 Feb 2016 08:13:04 +0600
Subject: [PATCH 3932/6219] [utils] Add ability to control skipping false
 values in dict_get

---
 test/test_utils.py  | 13 +++++++++++--
 youtube_dl/utils.py |  7 ++++---
 2 files changed, 15 insertions(+), 5 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index e3dd019af..909d0e51d 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -452,9 +452,15 @@ class TestUtil(unittest.TestCase):
         self.assertTrue(isinstance(data, bytes))
 
     def test_dict_get(self):
-        d = {
-            'a': 42,
+        FALSE_VALUES = {
+            'none': None,
+            'false': False,
+            'zero': 0,
+            'empty_string': '',
+            'empty_list': [],
         }
+        d = FALSE_VALUES.copy()
+        d['a'] = 42
         self.assertEqual(dict_get(d, 'a'), 42)
         self.assertEqual(dict_get(d, 'b'), None)
         self.assertEqual(dict_get(d, 'b', 42), 42)
@@ -463,6 +469,9 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(dict_get(d, ('b', 'c', 'a', 'd', )), 42)
         self.assertEqual(dict_get(d, ('b', 'c', )), None)
         self.assertEqual(dict_get(d, ('b', 'c', ), 42), 42)
+        for key, false_value in FALSE_VALUES.items():
+            self.assertEqual(dict_get(d, ('b', 'c', key, )), None)
+            self.assertEqual(dict_get(d, ('b', 'c', key, ), skip_false_values=False), false_value)
 
     def test_encode_compat_str(self):
         self.assertEqual(encode_compat_str(b'\xd1\x82\xd0\xb5\xd1\x81\xd1\x82', 'utf-8'), 'тест')
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 652dba59d..f3b0180ab 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1717,11 +1717,12 @@ def encode_dict(d, encoding='utf-8'):
     return dict((encode(k), encode(v)) for k, v in d.items())
 
 
-def dict_get(d, key_or_keys, default=None):
+def dict_get(d, key_or_keys, default=None, skip_false_values=True):
     if isinstance(key_or_keys, (list, tuple)):
         for key in key_or_keys:
-            if d.get(key):
-                return d[key]
+            if key not in d or d[key] is None or skip_false_values and not d[key]:
+                continue
+            return d[key]
         return default
     return d.get(key_or_keys, default)
 

From f0905c6ec3e1ad20f284edcd1d9c58a43b23bac9 Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Sun, 7 Feb 2016 18:45:44 +0600
Subject: [PATCH 3933/6219] [README.md] Document more sequences in output
 template

---
 README.md | 58 ++++++++++++++++++++++++++++++++++---------------------
 1 file changed, 36 insertions(+), 22 deletions(-)

diff --git a/README.md b/README.md
index 7a180dccf..78fc42c28 100644
--- a/README.md
+++ b/README.md
@@ -442,32 +442,46 @@ On Windows you may also need to setup the `%HOME%` environment variable manually
 
 The `-o` option allows users to indicate a template for the output file names. The basic usage is not to set any template arguments when downloading a single file, like in `youtube-dl -o funny_video.flv "http://some/video"`. However, it may contain special sequences that will be replaced when downloading each video. The special sequences have the format `%(NAME)s`. To clarify, that is a percent symbol followed by a name in parentheses, followed by a lowercase S. Allowed names are:
 
- - `id`: Video identifier.
- - `url`: Video URL.
- - `uploader`: Nickname of the person who uploaded the video.
- - `upload_date`: Upload date in YYYYMMDD format.
- - `title`: Video title.
- - `ext`: Video filename extension.
- - `epoch`: Unix epoch when creating the file.
- - `autonumber`: Five-digit number that will be increased with each download, starting at zero.
- - `playlist`: Name or id of the playlist that contains the video.
- - `playlist_index`: Index of the video in the playlist padded with leading zeros according to the total length of the playlist.
- - `format_id`: Format code specified by `--format`.
- - `duration`: Length of the video in seconds.
+ - `id`: Video identifier
+ - `title`: Video title
+ - `url`: Video URL
+ - `ext`: Video filename extension
+ - `alt_title`: A secondary title of the video
+ - `display_id`: An alternative identifier for the video
+ - `uploader`: Full name of the video uploader
+ - `creator`: The main artist who created the video
+ - `release_date`: The date (YYYYMMDD) when the video was released
+ - `timestamp`: UNIX timestamp of the moment the video became available
+ - `upload_date`: Video upload date (YYYYMMDD)
+ - `uploader_id`: Nickname or id of the video uploader
+ - `location`: Physical location where the video was filmed
+ - `duration`: Length of the video in seconds
+ - `view_count`: How many users have watched the video on the platform
+ - `like_count`: Number of positive ratings of the video
+ - `dislike_count`: Number of negative ratings of the video
+ - `repost_count`: Number of reposts of the video
+ - `average_rating`: Average rating give by users, the scale used depends on the webpage
+ - `comment_count`: Number of comments on the video
+ - `age_limit`: Age restriction for the video (years)
+ - `epoch`: Unix epoch when creating the file
+ - `autonumber`: Five-digit number that will be increased with each download, starting at zero
+ - `playlist`: Name or id of the playlist that contains the video
+ - `playlist_index`: Index of the video in the playlist padded with leading zeros according to the total length of the playlist
+ - `format_id`: Format code specified by `--format`
 
 Available for the video that belongs to some logical chapter or section:
- - `chapter`: Name or title of the chapter the video belongs to.
- - `chapter_number`: Number of the chapter the video belongs to.
- - `chapter_id`: Id of the chapter the video belongs to.
+ - `chapter`: Name or title of the chapter the video belongs to
+ - `chapter_number`: Number of the chapter the video belongs to
+ - `chapter_id`: Id of the chapter the video belongs to
 
 Available for the video that is an episode of some series or programme:
- - `series`: Title of the series or programme the video episode belongs to.
- - `season`: Title of the season the video episode belongs to.
- - `season_number`: Number of the season the video episode belongs to.
- - `season_id`: Id of the season the video episode belongs to.
- - `episode`: Title of the video episode.
- - `episode_number`: Number of the video episode within a season.
- - `episode_id`: Id of the video episode, as a unicode string.
+ - `series`: Title of the series or programme the video episode belongs to
+ - `season`: Title of the season the video episode belongs to
+ - `season_number`: Number of the season the video episode belongs to
+ - `season_id`: Id of the season the video episode belongs to
+ - `episode`: Title of the video episode
+ - `episode_number`: Number of the video episode within a season
+ - `episode_id`: Id of the video episode
 
 Each aforementioned sequence when referenced in output template will be replaced by the actual value corresponding to the sequence name. Note that some of the sequences are not guaranteed to be present since they depend on the metadata obtained by particular extractor, such sequences will be replaced with `NA`.
 

From 0474512e3029a7a3121fbe8f08c9ff226deecc7d Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Sun, 7 Feb 2016 19:00:59 +0600
Subject: [PATCH 3934/6219] [README.md] Document even more sequences in output
 template

---
 README.md | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 78fc42c28..75f53e51f 100644
--- a/README.md
+++ b/README.md
@@ -463,11 +463,27 @@ The `-o` option allows users to indicate a template for the output file names. T
  - `average_rating`: Average rating give by users, the scale used depends on the webpage
  - `comment_count`: Number of comments on the video
  - `age_limit`: Age restriction for the video (years)
+ - `format`: A human-readable description of the format 
+ - `format_id`: Format code specified by `--format`
+ - `format_note`: Additional info about the format
+ - `width`: Width of the video
+ - `height`: Height of the video
+ - `resolution`: Textual description of width and height
+ - `tbr`: Average bitrate of audio and video in KBit/s
+ - `abr`: Average audio bitrate in KBit/s
+ - `acodec`: Name of the audio codec in use
+ - `asr`: Audio sampling rate in Hertz
+ - `vbr`: Average video bitrate in KBit/s
+ - `fps`: Frame rate
+ - `vcodec`: Name of the video codec in use
+ - `container`: Name of the container format
+ - `filesize`: The number of bytes, if known in advance
+ - `filesize_approx`: An estimate for the number of bytes
+ - `protocol`: The protocol that will be used for the actual download
  - `epoch`: Unix epoch when creating the file
  - `autonumber`: Five-digit number that will be increased with each download, starting at zero
  - `playlist`: Name or id of the playlist that contains the video
  - `playlist_index`: Index of the video in the playlist padded with leading zeros according to the total length of the playlist
- - `format_id`: Format code specified by `--format`
 
 Available for the video that belongs to some logical chapter or section:
  - `chapter`: Name or title of the chapter the video belongs to

From 3b799ca14cfeb1d17da637b2539044758d7355e6 Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Sun, 7 Feb 2016 19:06:42 +0600
Subject: [PATCH 3935/6219] [README.md] Clarify percent literal and output to
 stdout

---
 README.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/README.md b/README.md
index 75f53e51f..93a88e79b 100644
--- a/README.md
+++ b/README.md
@@ -505,6 +505,8 @@ For example for `-o %(title)s-%(id)s.%(ext)s` and mp4 video with title `youtube-
 
 Output template can also contain arbitrary hierarchical path, e.g. `-o %(playlist)s/%(playlist_index)s - %(title)s.%(ext)s` that will result in downloading each video in a directory corresponding to this path template. Any missing directory will be automatically created for you.
 
+To specify percent literal in output template use `%%`. To output to stdout use `-o -`.
+
 The current default template is `%(title)s-%(id)s.%(ext)s`.
 
 In some cases, you don't want special characters such as 中, spaces, or &, such as when transferring the downloaded filename to a Windows system or the filename through an 8bit-unsafe channel. In these cases, add the `--restrict-filenames` flag to get a shorter title:

From 72b9ebc65da3f1cb2f85d85b002674c8e5ae36ce Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Sun, 7 Feb 2016 19:08:54 +0600
Subject: [PATCH 3936/6219] [README.md] Document extractor sequences in output
 template

---
 README.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/README.md b/README.md
index 93a88e79b..fe301ef53 100644
--- a/README.md
+++ b/README.md
@@ -480,6 +480,8 @@ The `-o` option allows users to indicate a template for the output file names. T
  - `filesize`: The number of bytes, if known in advance
  - `filesize_approx`: An estimate for the number of bytes
  - `protocol`: The protocol that will be used for the actual download
+ - `extractor`: Name of the extractor
+ - `extractor_key`: Key name of the extractor
  - `epoch`: Unix epoch when creating the file
  - `autonumber`: Five-digit number that will be increased with each download, starting at zero
  - `playlist`: Name or id of the playlist that contains the video

From 8411229bd5e147ef1ac3420000c7c28ba660df2d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 7 Feb 2016 19:47:09 +0600
Subject: [PATCH 3937/6219] [utils] Allow dot in strip_jsonp

---
 test/test_utils.py  | 4 ++++
 youtube_dl/utils.py | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 909d0e51d..a1e416dd5 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -494,6 +494,10 @@ class TestUtil(unittest.TestCase):
         d = json.loads(stripped)
         self.assertEqual(d, {'STATUS': 'OK'})
 
+        stripped = strip_jsonp('ps.embedHandler({"status": "success"});')
+        d = json.loads(stripped)
+        self.assertEqual(d, {'status': 'success'})
+
     def test_uppercase_escape(self):
         self.assertEqual(uppercase_escape('aä'), 'aä')
         self.assertEqual(uppercase_escape('\\U0001d550'), '𝕐')
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index f3b0180ab..dce67de8a 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1749,7 +1749,7 @@ def parse_age_limit(s):
 
 def strip_jsonp(code):
     return re.sub(
-        r'(?s)^[a-zA-Z0-9_]+\s*\(\s*(.*)\);?\s*?(?://[^\n]*)*$', r'\1', code)
+        r'(?s)^[a-zA-Z0-9_.]+\s*\(\s*(.*)\);?\s*?(?://[^\n]*)*$', r'\1', code)
 
 
 def js_to_json(code):

From 8628d26f3888d2f0541a5d0e5d7c35fa5cac3fc0 Mon Sep 17 00:00:00 2001
From: ovitei <sh33p0@gmail.com>
Date: Sun, 31 Jan 2016 03:08:34 +0100
Subject: [PATCH 3938/6219] [KonserthusetPlay] Add new extractor (partial
 support)

---
 youtube_dl/extractor/__init__.py         |  1 +
 youtube_dl/extractor/konserthusetplay.py | 45 ++++++++++++++++++++++++
 2 files changed, 46 insertions(+)
 create mode 100644 youtube_dl/extractor/konserthusetplay.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 2fbc7f812..18951c287 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -328,6 +328,7 @@ from .keezmovies import KeezMoviesIE
 from .khanacademy import KhanAcademyIE
 from .kickstarter import KickStarterIE
 from .keek import KeekIE
+from .konserthusetplay import KonserthusetPlayIE
 from .kontrtube import KontrTubeIE
 from .krasview import KrasViewIE
 from .ku6 import Ku6IE
diff --git a/youtube_dl/extractor/konserthusetplay.py b/youtube_dl/extractor/konserthusetplay.py
new file mode 100644
index 000000000..8200942d3
--- /dev/null
+++ b/youtube_dl/extractor/konserthusetplay.py
@@ -0,0 +1,45 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class KonserthusetPlayIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?konserthusetplay\.se/\?m=(?P<id>[0-9A-Za-z_-]+)'
+
+    _TESTS = [{
+        'url': 'http://www.konserthusetplay.se/?m=CKDDnlCY-dhWAAqiMERd-A',
+        'md5': 'e272a765e0d12a0226199e5f32d76116',
+        'info_dict': {
+            'id': 'CKDDnlCY-dhWAAqiMERd-A',
+            'ext': 'mp4',
+            'title': 'Orkesterns instrument: Valthornen',
+            'description': 'md5:f10e1f0030202020396a4d712d2fa827',
+            'thumbnail': 'http://csp.picsearch.com/img/C/K/D/D/title_CKDDnlCY-dhWAAqiMERd-A'
+        }
+    }, {
+        'url': 'http://www.konserthusetplay.se/?m=IyQcMOEpmKqT91SVT5OP8Q',
+        'md5': 'c4adb8ca76fdd33d4cbdcc7c3d181f22',
+        'info_dict': {
+            'id': 'IyQcMOEpmKqT91SVT5OP8Q',
+            'ext': 'mp4',
+            'title': 'Eliasson Einsame Fahrt, violinkonsert',
+            'description': 'md5:a8dcc8dfd9473d52433b2c5f588ba191',
+            'thumbnail': 'http://csp.picsearch.com/img/I/y/Q/c/title_IyQcMOEpmKqT91SVT5OP8Q'
+        }
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        description = self._og_search_description(webpage)
+        title = self._og_search_title(webpage)
+        main_video = self._html_search_regex(r'<link rel="video_src" href="(.+?)" />', webpage, 'url')
+        thumbnail = self._og_search_thumbnail(webpage)
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'url': main_video,
+            'thumbnail': thumbnail
+        }

From 5a7699bb2ef77c2548bcc2112e042481a809ba35 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 7 Feb 2016 21:11:59 +0600
Subject: [PATCH 3939/6219] [konserthusetplay] Improve and extract all formats
 (Closes #8381)

---
 youtube_dl/extractor/konserthusetplay.py | 111 ++++++++++++++++++-----
 1 file changed, 87 insertions(+), 24 deletions(-)

diff --git a/youtube_dl/extractor/konserthusetplay.py b/youtube_dl/extractor/konserthusetplay.py
index 8200942d3..f21d050d7 100644
--- a/youtube_dl/extractor/konserthusetplay.py
+++ b/youtube_dl/extractor/konserthusetplay.py
@@ -2,44 +2,107 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..utils import (
+    float_or_none,
+    int_or_none,
+)
 
 
 class KonserthusetPlayIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?konserthusetplay\.se/\?m=(?P<id>[0-9A-Za-z_-]+)'
-
-    _TESTS = [{
+    _VALID_URL = r'https?://(?:www\.)?konserthusetplay\.se/\?m=(?P<id>[^&]+)'
+    _TEST = {
         'url': 'http://www.konserthusetplay.se/?m=CKDDnlCY-dhWAAqiMERd-A',
-        'md5': 'e272a765e0d12a0226199e5f32d76116',
         'info_dict': {
             'id': 'CKDDnlCY-dhWAAqiMERd-A',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': 'Orkesterns instrument: Valthornen',
             'description': 'md5:f10e1f0030202020396a4d712d2fa827',
-            'thumbnail': 'http://csp.picsearch.com/img/C/K/D/D/title_CKDDnlCY-dhWAAqiMERd-A'
-        }
-    }, {
-        'url': 'http://www.konserthusetplay.se/?m=IyQcMOEpmKqT91SVT5OP8Q',
-        'md5': 'c4adb8ca76fdd33d4cbdcc7c3d181f22',
-        'info_dict': {
-            'id': 'IyQcMOEpmKqT91SVT5OP8Q',
-            'ext': 'mp4',
-            'title': 'Eliasson Einsame Fahrt, violinkonsert',
-            'description': 'md5:a8dcc8dfd9473d52433b2c5f588ba191',
-            'thumbnail': 'http://csp.picsearch.com/img/I/y/Q/c/title_IyQcMOEpmKqT91SVT5OP8Q'
-        }
-    }]
+            'thumbnail': 're:^https?://.*$',
+            'duration': 398.8,
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        },
+    }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
+
         webpage = self._download_webpage(url, video_id)
-        description = self._og_search_description(webpage)
-        title = self._og_search_title(webpage)
-        main_video = self._html_search_regex(r'<link rel="video_src" href="(.+?)" />', webpage, 'url')
-        thumbnail = self._og_search_thumbnail(webpage)
+
+        e = self._search_regex(
+            r'https?://csp\.picsearch\.com/rest\?.*\be=(.+?)[&"\']', webpage, 'e')
+
+        rest = self._download_json(
+            'http://csp.picsearch.com/rest?e=%s&containerId=mediaplayer&i=object' % e,
+            video_id, transform_source=lambda s: s[s.index('{'):s.rindex('}') + 1])
+
+        media = rest['media']
+        player_config = media['playerconfig']
+        playlist = player_config['playlist']
+
+        source = next(f for f in playlist if f.get('bitrates'))
+
+        connection_url = (player_config.get('rtmp', {}).get(
+            'netConnectionUrl') or player_config.get(
+            'plugins', {}).get('bwcheck', {}).get('netConnectionUrl'))
+
+        FORMAT_ID_REGEX = r'_([^_]+)_h264m\.mp4'
+
+        formats = []
+
+        fallback_url = source.get('fallbackUrl')
+        fallback_format_id = None
+        if fallback_url:
+            fallback_format_id = self._search_regex(
+                FORMAT_ID_REGEX, fallback_url, 'format id', default=None)
+
+        if connection_url:
+            for f in source['bitrates']:
+                video_url = f.get('url')
+                if not video_url:
+                    continue
+                format_id = self._search_regex(
+                    FORMAT_ID_REGEX, video_url, 'format id', default=None)
+                f_common = {
+                    'vbr': int_or_none(f.get('bitrate')),
+                    'width': int_or_none(f.get('width')),
+                    'height': int_or_none(f.get('height')),
+                }
+                f = f_common.copy()
+                f.update({
+                    'url': connection_url,
+                    'play_path': video_url,
+                    'format_id': 'rtmp-%s' % format_id if format_id else 'rtmp',
+                    'ext': 'flv',
+                })
+                formats.append(f)
+                if format_id and format_id == fallback_format_id:
+                    f = f_common.copy()
+                    f.update({
+                        'url': fallback_url,
+                        'format_id': 'http-%s' % format_id if format_id else 'http',
+                    })
+                    formats.append(f)
+
+        if not formats and fallback_url:
+            formats.append({
+                'url': fallback_url,
+            })
+
+        self._sort_formats(formats)
+
+        title = player_config.get('title') or media['title']
+        description = player_config.get('mediaInfo', {}).get('description')
+        thumbnail = media.get('image')
+        duration = float_or_none(media.get('duration'), 1000)
+
         return {
             'id': video_id,
             'title': title,
             'description': description,
-            'url': main_video,
-            'thumbnail': thumbnail
+            'thumbnail': thumbnail,
+            'duration': duration,
+            'formats': formats,
         }

From f070197bd7b0de49672c4ff151887355d5e6a2ec Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 7 Feb 2016 21:16:31 +0600
Subject: [PATCH 3940/6219] [konserthusetplay] Improve _VALID_URL

---
 youtube_dl/extractor/konserthusetplay.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/konserthusetplay.py b/youtube_dl/extractor/konserthusetplay.py
index f21d050d7..e1395b24e 100644
--- a/youtube_dl/extractor/konserthusetplay.py
+++ b/youtube_dl/extractor/konserthusetplay.py
@@ -9,7 +9,7 @@ from ..utils import (
 
 
 class KonserthusetPlayIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?konserthusetplay\.se/\?m=(?P<id>[^&]+)'
+    _VALID_URL = r'https?://(?:www\.)?konserthusetplay\.se/\?.*\bm=(?P<id>[^&]+)'
     _TEST = {
         'url': 'http://www.konserthusetplay.se/?m=CKDDnlCY-dhWAAqiMERd-A',
         'info_dict': {

From b6bfa6fb7953b8dc477a932acea2df2d73175bc2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 7 Feb 2016 21:18:32 +0600
Subject: [PATCH 3941/6219] [konserthusetplay] Reorder code pieces

---
 youtube_dl/extractor/konserthusetplay.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/konserthusetplay.py b/youtube_dl/extractor/konserthusetplay.py
index e1395b24e..55291c66f 100644
--- a/youtube_dl/extractor/konserthusetplay.py
+++ b/youtube_dl/extractor/konserthusetplay.py
@@ -44,10 +44,6 @@ class KonserthusetPlayIE(InfoExtractor):
 
         source = next(f for f in playlist if f.get('bitrates'))
 
-        connection_url = (player_config.get('rtmp', {}).get(
-            'netConnectionUrl') or player_config.get(
-            'plugins', {}).get('bwcheck', {}).get('netConnectionUrl'))
-
         FORMAT_ID_REGEX = r'_([^_]+)_h264m\.mp4'
 
         formats = []
@@ -58,6 +54,9 @@ class KonserthusetPlayIE(InfoExtractor):
             fallback_format_id = self._search_regex(
                 FORMAT_ID_REGEX, fallback_url, 'format id', default=None)
 
+        connection_url = (player_config.get('rtmp', {}).get(
+            'netConnectionUrl') or player_config.get(
+            'plugins', {}).get('bwcheck', {}).get('netConnectionUrl'))
         if connection_url:
             for f in source['bitrates']:
                 video_url = f.get('url')

From 915dd777830e7e309349c589f91f21e29224530c Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Sun, 7 Feb 2016 21:21:14 +0600
Subject: [PATCH 3942/6219] [README.md] Add output template example for
 streaming to stdout

---
 README.md | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/README.md b/README.md
index fe301ef53..4070aebf8 100644
--- a/README.md
+++ b/README.md
@@ -530,6 +530,9 @@ $ youtube-dl -u user -p password -o '~/MyVideos/%(playlist)s/%(chapter_number)s
 
 # Download entire series season keeping each series and each season in separate directory under C:/MyVideos
 $ youtube-dl -o 'C:/MyVideos/%(series)s/%(season_number)s - %(season)s/%(episode_number)s - %(episode)s.%(ext)s' http://videomore.ru/kino_v_detalayah/5_sezon/367617
+
+# Stream the video being downloaded to stdout
+$ youtube-dl -o - BaW_jenozKc
 ```
 
 # FORMAT SELECTION

From 11c0339bec676a56ad8c5582bb145a084d04b94d Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Sun, 7 Feb 2016 21:22:33 +0600
Subject: [PATCH 3943/6219] [README.md] Clarify quotes in output template

---
 README.md | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/README.md b/README.md
index 4070aebf8..a027bb7f9 100644
--- a/README.md
+++ b/README.md
@@ -505,7 +505,7 @@ Each aforementioned sequence when referenced in output template will be replaced
 
 For example for `-o %(title)s-%(id)s.%(ext)s` and mp4 video with title `youtube-dl test video` and id `BaW_jenozKcj` this will result in a `youtube-dl test video-BaW_jenozKcj.mp4` file created in the current directory.
 
-Output template can also contain arbitrary hierarchical path, e.g. `-o %(playlist)s/%(playlist_index)s - %(title)s.%(ext)s` that will result in downloading each video in a directory corresponding to this path template. Any missing directory will be automatically created for you.
+Output template can also contain arbitrary hierarchical path, e.g. `-o '%(playlist)s/%(playlist_index)s - %(title)s.%(ext)s'` that will result in downloading each video in a directory corresponding to this path template. Any missing directory will be automatically created for you.
 
 To specify percent literal in output template use `%%`. To output to stdout use `-o -`.
 
@@ -513,13 +513,13 @@ The current default template is `%(title)s-%(id)s.%(ext)s`.
 
 In some cases, you don't want special characters such as 中, spaces, or &, such as when transferring the downloaded filename to a Windows system or the filename through an 8bit-unsafe channel. In these cases, add the `--restrict-filenames` flag to get a shorter title:
 
-Examples:
+Examples (note on Windows you may need to use double quotes instead of single)::
 
 ```bash
-$ youtube-dl --get-filename -o "%(title)s.%(ext)s" BaW_jenozKc
+$ youtube-dl --get-filename -o '%(title)s.%(ext)s' BaW_jenozKc
 youtube-dl test video ''_ä↭𝕐.mp4    # All kinds of weird characters
 
-$ youtube-dl --get-filename -o "%(title)s.%(ext)s" BaW_jenozKc --restrict-filenames
+$ youtube-dl --get-filename -o '%(title)s.%(ext)s' BaW_jenozKc --restrict-filenames
 youtube-dl_test_video_.mp4          # A simple file name
 
 # Download YouTube playlist videos in separate directory indexed by video order in a playlist
@@ -529,7 +529,7 @@ $ youtube-dl -o '%(playlist)s/%(playlist_index)s - %(title)s.%(ext)s' https://ww
 $ youtube-dl -u user -p password -o '~/MyVideos/%(playlist)s/%(chapter_number)s - %(chapter)s/%(title)s.%(ext)s' https://www.udemy.com/java-tutorial/
 
 # Download entire series season keeping each series and each season in separate directory under C:/MyVideos
-$ youtube-dl -o 'C:/MyVideos/%(series)s/%(season_number)s - %(season)s/%(episode_number)s - %(episode)s.%(ext)s' http://videomore.ru/kino_v_detalayah/5_sezon/367617
+$ youtube-dl -o "C:/MyVideos/%(series)s/%(season_number)s - %(season)s/%(episode_number)s - %(episode)s.%(ext)s" http://videomore.ru/kino_v_detalayah/5_sezon/367617
 
 # Stream the video being downloaded to stdout
 $ youtube-dl -o - BaW_jenozKc

From 8d0a2a2a4e3e930f9916a96d8ef7bf9c970be088 Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Sun, 7 Feb 2016 21:23:29 +0600
Subject: [PATCH 3944/6219] [README.md] Fix typo

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index a027bb7f9..65e0ee963 100644
--- a/README.md
+++ b/README.md
@@ -513,7 +513,7 @@ The current default template is `%(title)s-%(id)s.%(ext)s`.
 
 In some cases, you don't want special characters such as 中, spaces, or &, such as when transferring the downloaded filename to a Windows system or the filename through an 8bit-unsafe channel. In these cases, add the `--restrict-filenames` flag to get a shorter title:
 
-Examples (note on Windows you may need to use double quotes instead of single)::
+Examples (note on Windows you may need to use double quotes instead of single):
 
 ```bash
 $ youtube-dl --get-filename -o '%(title)s.%(ext)s' BaW_jenozKc

From 3834d3e35c5ccd957a3bc7bb1c12607f9d96ccbd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 8 Feb 2016 01:30:57 +0600
Subject: [PATCH 3945/6219] [youtube] Clarify itag 36 height and abr (Closes
 #8457)

---
 youtube_dl/extractor/youtube.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 63abe5477..b9a91dea2 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -286,7 +286,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         '22': {'ext': 'mp4', 'width': 1280, 'height': 720, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
         '34': {'ext': 'flv', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
         '35': {'ext': 'flv', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
-        '36': {'ext': '3gp', 'width': 320, 'height': 240, 'acodec': 'aac', 'abr': 32, 'vcodec': 'mp4v'},
+        # itag 36 videos are either 320x180 (BaW_jenozKc) or 320x240 (__2ABJjxzNo), abr varies as well
+        '36': {'ext': '3gp', 'width': 320, 'acodec': 'aac', 'vcodec': 'mp4v'},
         '37': {'ext': 'mp4', 'width': 1920, 'height': 1080, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
         '38': {'ext': 'mp4', 'width': 4096, 'height': 3072, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
         '43': {'ext': 'webm', 'width': 640, 'height': 360, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},

From 35972ba17214608d8929d6532831abf7402afcf8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 8 Feb 2016 21:30:23 +0600
Subject: [PATCH 3946/6219] [vk] Improve rutube embeds detection (Closes #8461)

---
 youtube_dl/extractor/vk.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 11014865a..0805e3c08 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -265,7 +265,7 @@ class VKIE(InfoExtractor):
             return self.url_result(pladform_url)
 
         m_rutube = re.search(
-            r'\ssrc="((?:https?:)?//rutube\.ru\\?/video\\?/embed(?:.*?))\\?"', info_page)
+            r'\ssrc="((?:https?:)?//rutube\.ru\\?/(?:video|play)\\?/embed(?:.*?))\\?"', info_page)
         if m_rutube is not None:
             rutube_url = self._proto_relative_url(
                 m_rutube.group(1).replace('\\', ''))

From 085ad71157dd6588cece2df0bfba6815f234564f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 9 Feb 2016 12:57:51 +0100
Subject: [PATCH 3947/6219] release 2016.02.09

---
 docs/supportedsites.md | 7 +++++--
 youtube_dl/version.py  | 2 +-
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index ee34adf26..c48561eff 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -262,7 +262,7 @@
  - **Instagram**
  - **instagram:user**: Instagram user profile
  - **InternetVideoArchive**
- - **IPrima** (Currently broken)
+ - **IPrima**
  - **iqiyi**: 爱奇艺
  - **Ir90Tv**
  - **ivi**: ivi.ru
@@ -283,6 +283,7 @@
  - **KeezMovies**
  - **KhanAcademy**
  - **KickStarter**
+ - **KonserthusetPlay**
  - **kontrtube**: KontrTube.ru - Труба зовёт
  - **KrasView**: Красвью
  - **Ku6**
@@ -681,7 +682,9 @@
  - **VideoPremium**
  - **VideoTt**: video.tt - Your True Tube (Currently broken)
  - **videoweed**: VideoWeed
- - **Vidme**
+ - **vidme**
+ - **vidme:user**
+ - **vidme:user:likes**
  - **Vidzi**
  - **vier**
  - **vier:videos**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 3fec14ab1..53b210290 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.02.05.1'
+__version__ = '2016.02.09'

From 16f1430ba6fc1dc3fe0baf587f4461f37b62466a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 9 Feb 2016 21:14:02 +0600
Subject: [PATCH 3948/6219] [mailru] Prefer metaUrl API (Closes #8474)

---
 youtube_dl/extractor/mailru.py | 36 ++++++++++++++++++++++++++++++++--
 1 file changed, 34 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/mailru.py b/youtube_dl/extractor/mailru.py
index ab1300185..09424620b 100644
--- a/youtube_dl/extractor/mailru.py
+++ b/youtube_dl/extractor/mailru.py
@@ -42,6 +42,22 @@ class MailRuIE(InfoExtractor):
             },
             'skip': 'Not accessible from Travis CI server',
         },
+        {
+            # only available via metaUrl API
+            'url': 'http://my.mail.ru/mail/720pizle/video/_myvideo/502.html',
+            'md5': '3b26d2491c6949d031a32b96bd97c096',
+            'info_dict': {
+                'id': '56664382_502',
+                'ext': 'mp4',
+                'title': ':8336',
+                'timestamp': 1449094163,
+                'upload_date': '20151202',
+                'uploader': '720pizle@mail.ru',
+                'uploader_id': '720pizle@mail.ru',
+                'duration': 6001,
+            },
+            'skip': 'Not accessible from Travis CI server',
+        }
     ]
 
     def _real_extract(self, url):
@@ -51,8 +67,24 @@ class MailRuIE(InfoExtractor):
         if not video_id:
             video_id = mobj.group('idv2prefix') + mobj.group('idv2suffix')
 
-        video_data = self._download_json(
-            'http://api.video.mail.ru/videos/%s.json?new=1' % video_id, video_id, 'Downloading video JSON')
+        webpage = self._download_webpage(url, video_id)
+
+        video_data = None
+
+        page_config = self._parse_json(self._search_regex(
+            r'(?s)<script[^>]+class="sp-video__page-config"[^>]*>(.+?)</script>',
+            webpage, 'page config', default='{}'), video_id, fatal=False)
+        if page_config:
+            meta_url = page_config.get('metaUrl') or page_config.get('video', {}).get('metaUrl')
+            if meta_url:
+                video_data = self._download_json(
+                    meta_url, video_id, 'Downloading video meta JSON', fatal=False)
+
+        # Fallback old approach
+        if not video_data:
+            video_data = self._download_json(
+                'http://api.video.mail.ru/videos/%s.json?new=1' % video_id,
+                video_id, 'Downloading video JSON')
 
         author = video_data['author']
         uploader = author['name']

From b081350bd943d7595acda70240d2aaa749fc34b3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 9 Feb 2016 21:30:48 +0600
Subject: [PATCH 3949/6219] [mailru] Improve and modernize

---
 youtube_dl/extractor/mailru.py | 51 +++++++++++++++++++++-------------
 1 file changed, 31 insertions(+), 20 deletions(-)

diff --git a/youtube_dl/extractor/mailru.py b/youtube_dl/extractor/mailru.py
index 09424620b..b7671f16e 100644
--- a/youtube_dl/extractor/mailru.py
+++ b/youtube_dl/extractor/mailru.py
@@ -4,6 +4,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    remove_end,
+)
 
 
 class MailRuIE(InfoExtractor):
@@ -86,29 +90,36 @@ class MailRuIE(InfoExtractor):
                 'http://api.video.mail.ru/videos/%s.json?new=1' % video_id,
                 video_id, 'Downloading video JSON')
 
-        author = video_data['author']
-        uploader = author['name']
-        uploader_id = author.get('id') or author.get('email')
-        view_count = video_data.get('views_count')
+        formats = []
+        for f in video_data['videos']:
+            video_url = f.get('url')
+            if not video_url:
+                continue
+            format_id = f.get('key')
+            height = int_or_none(self._search_regex(
+                r'^(\d+)[pP]$', format_id, 'height', default=None)) if format_id else None
+            formats.append({
+                'url': video_url,
+                'format_id': format_id,
+                'height': height,
+            })
+        self._sort_formats(formats)
 
         meta_data = video_data['meta']
-        content_id = '%s_%s' % (
-            meta_data.get('accId', ''), meta_data['itemId'])
-        title = meta_data['title']
-        if title.endswith('.mp4'):
-            title = title[:-4]
-        thumbnail = meta_data['poster']
-        duration = meta_data['duration']
-        timestamp = meta_data['timestamp']
+        title = remove_end(meta_data['title'], '.mp4')
 
-        formats = [
-            {
-                'url': video['url'],
-                'format_id': video['key'],
-                'height': int(video['key'].rstrip('p'))
-            } for video in video_data['videos']
-        ]
-        self._sort_formats(formats)
+        author = video_data.get('author')
+        uploader = author.get('name')
+        uploader_id = author.get('id') or author.get('email')
+        view_count = int_or_none(video_data.get('viewsCount') or video_data.get('views_count'))
+
+        acc_id = meta_data.get('accId')
+        item_id = meta_data.get('itemId')
+        content_id = '%s_%s' % (acc_id, item_id) if acc_id and item_id else video_id
+
+        thumbnail = meta_data.get('poster')
+        duration = int_or_none(meta_data.get('duration'))
+        timestamp = int_or_none(meta_data.get('timestamp'))
 
         return {
             'id': content_id,

From 3967a761f484b8092e83e8c255f75adeb9ce3d90 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 9 Feb 2016 21:31:51 +0600
Subject: [PATCH 3950/6219] [mailru] Fix tests

---
 youtube_dl/extractor/mailru.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/mailru.py b/youtube_dl/extractor/mailru.py
index b7671f16e..71085f279 100644
--- a/youtube_dl/extractor/mailru.py
+++ b/youtube_dl/extractor/mailru.py
@@ -38,9 +38,9 @@ class MailRuIE(InfoExtractor):
                 'id': '46843144_1263',
                 'ext': 'mp4',
                 'title': 'Samsung Galaxy S5 Hammer Smash Fail Battery Explosion',
-                'timestamp': 1397217632,
-                'upload_date': '20140411',
-                'uploader': 'hitech',
+                'timestamp': 1397039888,
+                'upload_date': '20140409',
+                'uploader': 'hitech@corp.mail.ru',
                 'uploader_id': 'hitech@corp.mail.ru',
                 'duration': 245,
             },

From 1bedf4de06300ea67655aaec8d83cf4af36a156f Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 9 Feb 2016 17:12:34 +0100
Subject: [PATCH 3951/6219] [fox] extract http formats

---
 youtube_dl/extractor/fox.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/fox.py b/youtube_dl/extractor/fox.py
index ab97b3196..4a8acd53d 100644
--- a/youtube_dl/extractor/fox.py
+++ b/youtube_dl/extractor/fox.py
@@ -29,7 +29,7 @@ class FOXIE(InfoExtractor):
 
         release_url = self._parse_json(self._search_regex(
             r'"fox_pdk_player"\s*:\s*({[^}]+?})', webpage, 'fox_pdk_player'),
-            video_id)['release_url'] + '&manifest=m3u'
+            video_id)['release_url'] + '&switch=http'
 
         return {
             '_type': 'url_transparent',

From d413095f7e14f146f66d5f0cb828fb55aa4cb323 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 9 Feb 2016 17:15:41 +0100
Subject: [PATCH 3952/6219] [extractor/common] remove duplicated formats and
 subtiles in smil manifests

---
 youtube_dl/extractor/common.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 9c0421de7..1143f6dbb 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1186,6 +1186,7 @@ class InfoExtractor(object):
         http_count = 0
         m3u8_count = 0
 
+        src_urls = []
         videos = smil.findall(self._xpath_ns('.//video', namespace))
         for video in videos:
             src = video.get('src')
@@ -1222,6 +1223,9 @@ class InfoExtractor(object):
                 continue
 
             src_url = src if src.startswith('http') else compat_urlparse.urljoin(base, src)
+            if src_url in src_urls:
+                continue
+            src_urls.append(src_url)
 
             if proto == 'm3u8' or src_ext == 'm3u8':
                 m3u8_formats = self._extract_m3u8_formats(
@@ -1267,11 +1271,13 @@ class InfoExtractor(object):
         return formats
 
     def _parse_smil_subtitles(self, smil, namespace=None, subtitles_lang='en'):
+        urls = []
         subtitles = {}
         for num, textstream in enumerate(smil.findall(self._xpath_ns('.//textstream', namespace))):
             src = textstream.get('src')
-            if not src:
+            if not src or src in urls:
                 continue
+            urls.append(src)
             ext = textstream.get('ext') or determine_ext(src)
             if not ext:
                 type_ = textstream.get('type')

From c43fe0268cc4f7eee0e189b1b3183950574e3e19 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 9 Feb 2016 17:25:02 +0100
Subject: [PATCH 3953/6219] [downloader/dash] Implement dashsegments fd in
 terms of fragment fd

---
 youtube_dl/downloader/dash.py | 79 ++++++++++++++++-------------------
 1 file changed, 35 insertions(+), 44 deletions(-)

diff --git a/youtube_dl/downloader/dash.py b/youtube_dl/downloader/dash.py
index b0070aead..4f3eeb386 100644
--- a/youtube_dl/downloader/dash.py
+++ b/youtube_dl/downloader/dash.py
@@ -1,67 +1,58 @@
 from __future__ import unicode_literals
 
+import os
 import re
 
-from .common import FileDownloader
-from ..utils import sanitized_Request
+from .fragment import FragmentFD
+from ..utils import (
+    sanitize_open,
+    encodeFilename,
+)
 
 
-class DashSegmentsFD(FileDownloader):
+class DashSegmentsFD(FragmentFD):
     """
     Download segments in a DASH manifest
     """
+
+    FD_NAME = 'dashsegments'
+
     def real_download(self, filename, info_dict):
-        self.report_destination(filename)
-        tmpfilename = self.temp_name(filename)
         base_url = info_dict['url']
-        segment_urls = info_dict['segment_urls']
+        segment_urls = [info_dict['segment_urls'][0]] if self.params.get('test', False) else info_dict['segment_urls']
+        initialization_url = info_dict.get('initialization_url')
 
-        is_test = self.params.get('test', False)
-        remaining_bytes = self._TEST_FILE_SIZE if is_test else None
-        byte_counter = 0
+        ctx = {
+            'filename': filename,
+            'total_frags': len(segment_urls) + (1 if initialization_url else 0),
+        }
 
-        def append_url_to_file(outf, target_url, target_name, remaining_bytes=None):
-            self.to_screen('[DashSegments] %s: Downloading %s' % (info_dict['id'], target_name))
-            req = sanitized_Request(target_url)
-            if remaining_bytes is not None:
-                req.add_header('Range', 'bytes=0-%d' % (remaining_bytes - 1))
-
-            data = self.ydl.urlopen(req).read()
-
-            if remaining_bytes is not None:
-                data = data[:remaining_bytes]
-
-            outf.write(data)
-            return len(data)
+        self._prepare_and_start_frag_download(ctx)
 
         def combine_url(base_url, target_url):
             if re.match(r'^https?://', target_url):
                 return target_url
             return '%s%s%s' % (base_url, '' if base_url.endswith('/') else '/', target_url)
 
-        with open(tmpfilename, 'wb') as outf:
-            if info_dict.get('initialization_url'):
-                append_url_to_file(
-                    outf, combine_url(base_url, info_dict['initialization_url']),
-                    'initialization segment')
-            for i, segment_url in enumerate(segment_urls):
-                segment_len = append_url_to_file(
-                    outf, combine_url(base_url, segment_url),
-                    'segment %d / %d' % (i + 1, len(segment_urls)),
-                    remaining_bytes)
-                byte_counter += segment_len
-                if remaining_bytes is not None:
-                    remaining_bytes -= segment_len
-                    if remaining_bytes <= 0:
-                        break
+        segments_filenames = []
+        def append_url_to_file(target_url, target_filename):
+            success = ctx['dl'].download(target_filename, {'url': combine_url(base_url, target_url)})
+            if not success:
+                return False
+            down, target_sanitized = sanitize_open(target_filename, 'rb')
+            ctx['dest_stream'].write(down.read())
+            down.close()
+            segments_filenames.append(target_sanitized)
 
-        self.try_rename(tmpfilename, filename)
+        if initialization_url:
+            append_url_to_file(initialization_url, ctx['tmpfilename'] + '-Init')
+        for i, segment_url in enumerate(segment_urls):
+            segment_filename = '%s-Seg%d' % (ctx['tmpfilename'], i)
+            append_url_to_file(segment_url, segment_filename)
 
-        self._hook_progress({
-            'downloaded_bytes': byte_counter,
-            'total_bytes': byte_counter,
-            'filename': filename,
-            'status': 'finished',
-        })
+        self._finish_frag_download(ctx)
+
+        for segment_file in segments_filenames:
+            os.remove(encodeFilename(segment_file))
 
         return True

From 5e3a6fec3397e729991f360a4af9e2dd1e91d45c Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 9 Feb 2016 17:30:42 +0100
Subject: [PATCH 3954/6219] [fox] update test

---
 youtube_dl/extractor/fox.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/fox.py b/youtube_dl/extractor/fox.py
index 4a8acd53d..fa05af50d 100644
--- a/youtube_dl/extractor/fox.py
+++ b/youtube_dl/extractor/fox.py
@@ -9,6 +9,7 @@ class FOXIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?fox\.com/watch/(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://www.fox.com/watch/255180355939/7684182528',
+        'md5': 'ebd296fcc41dd4b19f8115d8461a3165',
         'info_dict': {
             'id': '255180355939',
             'ext': 'mp4',
@@ -17,10 +18,6 @@ class FOXIE(InfoExtractor):
             'duration': 129,
         },
         'add_ie': ['ThePlatform'],
-        'params': {
-            # m3u8 download
-            'skip_download': True,
-        },
     }
 
     def _real_extract(self, url):

From 51e9094f4a769187b11621c6477b19ad0e2418f9 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 9 Feb 2016 20:05:39 +0100
Subject: [PATCH 3955/6219] [extractor/common] extract youtube dash formats
 filesize(fixes #8480)

---
 youtube_dl/extractor/common.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 1143f6dbb..a9497851c 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1440,6 +1440,8 @@ class InfoExtractor(object):
                             base_url = mpd_base_url + base_url
                         representation_id = representation_attrib.get('id')
                         lang = representation_attrib.get('lang')
+                        url_el = representation.find(_add_ns('BaseURL'))
+                        filesize = int_or_none(url_el.attrib.get('{http://youtube.com/yt/2012/10/10}contentLength') if url_el is not None else None)
                         f = {
                             'format_id': mpd_id or representation_id,
                             'url': base_url,
@@ -1452,6 +1454,7 @@ class InfoExtractor(object):
                             'acodec': 'none' if content_type == 'video' else representation_attrib.get('codecs'),
                             'language': lang if lang not in ('mul', 'und', 'zxx', 'mis') else None,
                             'format_note': 'DASH %s' % content_type,
+                            'filesize': filesize,
                         }
                         representation_ms_info = extract_multisegment_info(representation, adaption_set_ms_info)
                         if 'segment_urls' not in representation_ms_info and 'media_template' in representation_ms_info:

From 9c15869c2846d955f4a5968924feed81121cffbb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 10 Feb 2016 01:09:07 +0600
Subject: [PATCH 3956/6219] [viddler] Add support for secret videos (Closes
 #8481)

---
 youtube_dl/extractor/viddler.py | 38 +++++++++++++++++++++++++++++----
 1 file changed, 34 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/viddler.py b/youtube_dl/extractor/viddler.py
index 40ffbad2a..8ba54ad39 100644
--- a/youtube_dl/extractor/viddler.py
+++ b/youtube_dl/extractor/viddler.py
@@ -1,6 +1,10 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..compat import (
+    compat_urllib_parse,
+    compat_urlparse,
+)
 from ..utils import (
     float_or_none,
     int_or_none,
@@ -54,16 +58,42 @@ class ViddlerIE(InfoExtractor):
             'view_count': int,
             'comment_count': int,
         }
+    }, {
+        # secret protected
+        'url': 'http://www.viddler.com/v/890c0985?secret=34051570',
+        'info_dict': {
+            'id': '890c0985',
+            'ext': 'mp4',
+            'title': 'Complete Property Training - Traineeships',
+            'description': ' ',
+            'upload_date': '20130606',
+            'uploader': 'TiffanyBowtell',
+            'timestamp': 1370496993,
+            'view_count': int,
+            'comment_count': int,
+        },
+        'params': {
+            'skip_download': True,
+        },
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        json_url = (
-            'http://api.viddler.com/api/v2/viddler.videos.getPlaybackDetails.json?video_id=%s&key=v0vhrt7bg2xq1vyxhkct' %
-            video_id)
+        query = {
+            'video_id': video_id,
+            'key': 'v0vhrt7bg2xq1vyxhkct',
+        }
+
+        qs = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
+        secret = qs.get('secret', [None])[0]
+        if secret:
+            query['secret'] = secret
+
         headers = {'Referer': 'http://static.cdn-ec.viddler.com/js/arpeggio/v2/embed.html'}
-        request = sanitized_Request(json_url, None, headers)
+        request = sanitized_Request(
+            'http://api.viddler.com/api/v2/viddler.videos.getPlaybackDetails.json?%s'
+            % compat_urllib_parse.urlencode(query), None, headers)
         data = self._download_json(request, video_id)['video']
 
         formats = []

From 47246ae26c7e19764ffa5c5c922fd549f562d42c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 10 Feb 2016 01:12:47 +0600
Subject: [PATCH 3957/6219] [viddler] Update tests

---
 youtube_dl/extractor/viddler.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/viddler.py b/youtube_dl/extractor/viddler.py
index 8ba54ad39..6bfbd4d85 100644
--- a/youtube_dl/extractor/viddler.py
+++ b/youtube_dl/extractor/viddler.py
@@ -16,10 +16,10 @@ class ViddlerIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?viddler\.com/(?:v|embed|player)/(?P<id>[a-z0-9]+)'
     _TESTS = [{
         'url': 'http://www.viddler.com/v/43903784',
-        'md5': 'ae43ad7cb59431ce043f0ff7fa13cbf4',
+        'md5': '9eee21161d2c7f5b39690c3e325fab2f',
         'info_dict': {
             'id': '43903784',
-            'ext': 'mp4',
+            'ext': 'mov',
             'title': 'Video Made Easy',
             'description': 'md5:6a697ebd844ff3093bd2e82c37b409cd',
             'uploader': 'viddler',
@@ -33,10 +33,10 @@ class ViddlerIE(InfoExtractor):
         }
     }, {
         'url': 'http://www.viddler.com/v/4d03aad9/',
-        'md5': 'faa71fbf70c0bee7ab93076fd007f4b0',
+        'md5': 'f12c5a7fa839c47a79363bfdf69404fb',
         'info_dict': {
             'id': '4d03aad9',
-            'ext': 'mp4',
+            'ext': 'ts',
             'title': 'WALL-TO-GORTAT',
             'upload_date': '20150126',
             'uploader': 'deadspin',
@@ -46,10 +46,10 @@ class ViddlerIE(InfoExtractor):
         }
     }, {
         'url': 'http://www.viddler.com/player/221ebbbd/0/',
-        'md5': '0defa2bd0ea613d14a6e9bd1db6be326',
+        'md5': '740511f61d3d1bb71dc14a0fe01a1c10',
         'info_dict': {
             'id': '221ebbbd',
-            'ext': 'mp4',
+            'ext': 'mov',
             'title': 'LETeens-Grammar-snack-third-conditional',
             'description': ' ',
             'upload_date': '20140929',

From b92071ef00d3a74b3c7145883587f5bf4d14498b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 9 Feb 2016 20:12:36 +0100
Subject: [PATCH 3958/6219] release 2016.02.09.1

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 53b210290..ce2c58e52 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.02.09'
+__version__ = '2016.02.09.1'

From 89d23f37f26c2a245c76fb6a0842cc945f68de19 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 10 Feb 2016 04:43:00 +0600
Subject: [PATCH 3959/6219] [hotstar] Relax _VALID_URL (Closes #8487)

---
 youtube_dl/extractor/hotstar.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/hotstar.py b/youtube_dl/extractor/hotstar.py
index a7c3ce4ab..f05d765d6 100644
--- a/youtube_dl/extractor/hotstar.py
+++ b/youtube_dl/extractor/hotstar.py
@@ -10,8 +10,8 @@ from ..utils import (
 
 
 class HotStarIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?hotstar\.com/.*?[/-](?P<id>\d{10})'
-    _TEST = {
+    _VALID_URL = r'https?://(?:www\.)?hotstar\.com/(?:.+?[/-])?(?P<id>\d{10})'
+    _TESTS = [{
         'url': 'http://www.hotstar.com/on-air-with-aib--english-1000076273',
         'info_dict': {
             'id': '1000076273',
@@ -26,7 +26,13 @@ class HotStarIE(InfoExtractor):
             # m3u8 download
             'skip_download': True,
         }
-    }
+    }, {
+        'url': 'http://www.hotstar.com/sports/cricket/rajitha-sizzles-on-debut-with-329/2001477583',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.hotstar.com/1000000515',
+        'only_matching': True,
+    }]
 
     _GET_CONTENT_TEMPLATE = 'http://account.hotstar.com/AVS/besc?action=GetAggregatedContentDetails&channel=PCTV&contentId=%s'
     _GET_CDN_TEMPLATE = 'http://getcdn.hotstar.com/AVS/besc?action=GetCDN&asJson=Y&channel=%s&id=%s&type=%s'

From cc799437ea828af8ca74cd5ba7dcfaf763f6e960 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 10 Feb 2016 20:05:17 +0600
Subject: [PATCH 3960/6219] [youku] Report private videos (Closes #8498)

---
 youtube_dl/extractor/youku.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index f767fa15f..49687371a 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -229,6 +229,9 @@ class YoukuIE(InfoExtractor):
             if error_note is not None and '因版权原因无法观看此视频' in error_note:
                 raise ExtractorError(
                     'Youku said: Sorry, this video is available in China only', expected=True)
+            elif error_note and '该视频被设为私密' in error_note:
+                raise ExtractorError(
+                    'Youku said: Sorry, this video is private', expected=True)
             else:
                 msg = 'Youku server reported error %i' % error.get('code')
                 if error_note is not None:

From 7fcea295c59f3321c5a70cb8171e6d5de1f378e4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 10 Feb 2016 20:46:38 +0600
Subject: [PATCH 3961/6219] [pbs] Switch to portal player by default (Closes
 #8491)

---
 youtube_dl/extractor/pbs.py | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 97e8ffc97..cbd1efea0 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -8,6 +8,7 @@ from ..utils import (
     ExtractorError,
     determine_ext,
     int_or_none,
+    js_to_json,
     strip_jsonp,
     unified_strdate,
     US_RATINGS,
@@ -432,9 +433,20 @@ class PBSIE(InfoExtractor):
                 for vid_id in video_id]
             return self.playlist_result(entries, display_id)
 
-        info = self._download_json(
-            'http://player.pbs.org/videoInfo/%s?format=json&type=partner' % video_id,
-            display_id)
+        player = self._download_webpage(
+            'http://player.pbs.org/portalplayer/%s' % video_id, display_id)
+
+        info = self._parse_json(
+            self._search_regex(
+                r'(?s)PBS\.videoData\s*=\s*({.+?});\n',
+                player, 'video data', default='{}'),
+            display_id, transform_source=js_to_json, fatal=False)
+
+        # Fallback to old videoInfo API
+        if not info:
+            info = self._download_json(
+                'http://player.pbs.org/videoInfo/%s?format=json&type=partner' % video_id,
+                display_id, 'Downloading video info JSON')
 
         formats = []
         for encoding_name in ('recommended_encoding', 'alternate_encoding'):

From e2effb08a46817104b56a6242616fd60848251d9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 10 Feb 2016 21:16:58 +0600
Subject: [PATCH 3962/6219] [YoutubeDL] Sanitize format_id (Closes #8494)

---
 youtube_dl/YoutubeDL.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 50228bb32..3cabfbc03 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1288,6 +1288,9 @@ class YoutubeDL(object):
 
             if format.get('format_id') is None:
                 format['format_id'] = compat_str(i)
+            else:
+                # Sanitize format_id from characters used in format selector expression
+                format['format_id'] = re.sub('[\s,/+\[\]()]', '_', format['format_id'])
             format_id = format['format_id']
             if format_id not in formats_dict:
                 formats_dict[format_id] = []

From f817d9bec180a932541564bdb9baf3db43e47faa Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 10 Feb 2016 16:17:38 +0100
Subject: [PATCH 3963/6219] release 2016.02.10

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index ce2c58e52..2d0d67b0f 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.02.09.1'
+__version__ = '2016.02.10'

From 80f772c28a3277376620ed7f50308e12437e358d Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 10 Feb 2016 22:16:21 +0100
Subject: [PATCH 3964/6219] [crackle] Add new extractor

---
 youtube_dl/extractor/__init__.py   |  1 +
 youtube_dl/extractor/comcarcoff.py | 16 +++---
 youtube_dl/extractor/crackle.py    | 92 ++++++++++++++++++++++++++++++
 3 files changed, 102 insertions(+), 7 deletions(-)
 create mode 100644 youtube_dl/extractor/crackle.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 18951c287..f08f27480 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -126,6 +126,7 @@ from .comcarcoff import ComCarCoffIE
 from .commonmistakes import CommonMistakesIE, UnicodeBOMIE
 from .condenast import CondeNastIE
 from .cracked import CrackedIE
+from .crackle import CrackleIE
 from .criterion import CriterionIE
 from .crooksandliars import CrooksAndLiarsIE
 from .crunchyroll import (
diff --git a/youtube_dl/extractor/comcarcoff.py b/youtube_dl/extractor/comcarcoff.py
index 2efa200b5..7dff68492 100644
--- a/youtube_dl/extractor/comcarcoff.py
+++ b/youtube_dl/extractor/comcarcoff.py
@@ -2,6 +2,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..compat import compat_str
 from ..utils import (
     int_or_none,
     parse_duration,
@@ -14,14 +15,13 @@ class ComCarCoffIE(InfoExtractor):
     _TESTS = [{
         'url': 'http://comediansincarsgettingcoffee.com/miranda-sings-happy-thanksgiving-miranda/',
         'info_dict': {
-            'id': 'miranda-sings-happy-thanksgiving-miranda',
+            'id': '2494164',
             'ext': 'mp4',
             'upload_date': '20141127',
             'timestamp': 1417107600,
             'duration': 1232,
             'title': 'Happy Thanksgiving Miranda',
             'description': 'Jerry Seinfeld and his special guest Miranda Sings cruise around town in search of coffee, complaining and apologizing along the way.',
-            'thumbnail': 'http://ccc.crackle.com/images/s5e4_thumb.jpg',
         },
         'params': {
             'skip_download': 'requires ffmpeg',
@@ -39,15 +39,14 @@ class ComCarCoffIE(InfoExtractor):
                 r'window\.app\s*=\s*({.+?});\n', webpage, 'full data json'),
             display_id)['videoData']
 
-        video_id = full_data['activeVideo']['video']
-        video_data = full_data.get('videos', {}).get(video_id) or full_data['singleshots'][video_id]
+        display_id = full_data['activeVideo']['video']
+        video_data = full_data.get('videos', {}).get(display_id) or full_data['singleshots'][display_id]
+        video_id = compat_str(video_data['mediaId'])
         thumbnails = [{
             'url': video_data['images']['thumb'],
         }, {
             'url': video_data['images']['poster'],
         }]
-        formats = self._extract_m3u8_formats(
-            video_data['mediaUrl'], video_id, ext='mp4')
 
         timestamp = int_or_none(video_data.get('pubDateTime')) or parse_iso8601(
             video_data.get('pubDate'))
@@ -55,6 +54,8 @@ class ComCarCoffIE(InfoExtractor):
             video_data.get('duration'))
 
         return {
+            '_type': 'url_transparent',
+            'url': 'crackle:%s' % video_id,
             'id': video_id,
             'display_id': display_id,
             'title': video_data['title'],
@@ -62,6 +63,7 @@ class ComCarCoffIE(InfoExtractor):
             'timestamp': timestamp,
             'duration': duration,
             'thumbnails': thumbnails,
-            'formats': formats,
+            'season_number': int_or_none(video_data.get('season')),
+            'episode_number': int_or_none(video_data.get('episode')),
             'webpage_url': 'http://comediansincarsgettingcoffee.com/%s' % (video_data.get('urlSlug', video_data.get('slug'))),
         }
diff --git a/youtube_dl/extractor/crackle.py b/youtube_dl/extractor/crackle.py
new file mode 100644
index 000000000..a478333a2
--- /dev/null
+++ b/youtube_dl/extractor/crackle.py
@@ -0,0 +1,92 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import int_or_none
+
+
+class CrackleIE(InfoExtractor):
+    _VALID_URL = r'(?:crackle:|https?://(?:www\.)?crackle\.com/(?:playlist/\d+/|(?:[^/]+/)+))(?P<id>\d+)'
+    _TEST = {
+        'url': 'http://www.crackle.com/the-art-of-more/2496419',
+        'info_dict': {
+            'id': '2496419',
+            'ext': 'mp4',
+            'title': 'Heavy Lies the Head',
+            'description': 'md5:bb56aa0708fe7b9a4861535f15c3abca',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        }
+    }
+
+    # extracted from http://legacyweb-us.crackle.com/flash/QueryReferrer.ashx
+    _SUBTITLE_SERVER = 'http://web-us-az.crackle.com'
+    _UPLYNK_OWNER_ID = 'e8773f7770a44dbd886eee4fca16a66b'
+    _THUMBNAIL_TEMPLATE = 'http://images-us-am.crackle.com/%stnl_1920x1080.jpg?ts=20140107233116?c=635333335057637614'
+
+    # extracted from http://legacyweb-us.crackle.com/flash/ReferrerRedirect.ashx
+    _MEDIA_FILE_SLOTS = {
+        'c544.flv': {
+            'width': 544,
+            'height': 306,
+        },
+        '360p.mp4': {
+            'width': 640,
+            'height': 360,
+        },
+        '480p.mp4': {
+            'width': 852,
+            'height': 478,
+        },
+        '480p_1mbps.mp4': {
+            'width': 852,
+            'height': 478,
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        item = self._download_xml(
+            'http://legacyweb-us.crackle.com/app/revamp/vidwallcache.aspx?flags=-1&fm=%s' % video_id, video_id).find('i')
+        title = item.attrib['t']
+
+        thumbnail = None
+        subtitles = {}
+        formats = self._extract_m3u8_formats('http://content.uplynk.com/ext/%s/%s.m3u8' % (self._UPLYNK_OWNER_ID, video_id), video_id, 'mp4', fatal=None)
+        path = item.attrib.get('p')
+        if path:
+            thumbnail = self._THUMBNAIL_TEMPLATE % path
+            http_base_url = 'http://ahttp.crackle.com/' + path
+            for mfs_path, mfs_info in self._MEDIA_FILE_SLOTS.items():
+                formats.append({
+                    'url': http_base_url + mfs_path,
+                    'format_id': mfs_path.split('.')[0],
+                    'width': mfs_info['width'],
+                    'height': mfs_info['height'],
+                })
+            for cc in item.findall('cc'):
+                locale = cc.attrib.get('l')
+                v = cc.attrib.get('v')
+                if locale and v:
+                    if locale not in subtitles:
+                        subtitles[locale] = []
+                    subtitles[locale] = [{
+                        'url': '%s/%s%s_%s.xml' % (self._SUBTITLE_SERVER, path, locale, v),
+                        'ext': 'ttml',
+                    }]
+        self._sort_formats(formats, ('width', 'height', 'tbr'))
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': item.attrib.get('d'),
+            'duration': int(item.attrib.get('r'), 16) if item.attrib.get('r') else None,
+            'series': item.attrib.get('sn'),
+            'season_number': int_or_none(item.attrib.get('se')),
+            'episode_number': int_or_none(item.attrib.get('ep')),
+            'thumbnail': thumbnail,
+            'subtitles': subtitles,
+            'formats': formats,
+        }

From 47d205a6460a696514f6485a516358513ab880b6 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 10 Feb 2016 22:23:56 +0100
Subject: [PATCH 3965/6219] [crackle] improve format sorting

---
 youtube_dl/extractor/crackle.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/crackle.py b/youtube_dl/extractor/crackle.py
index a478333a2..70f3f14c3 100644
--- a/youtube_dl/extractor/crackle.py
+++ b/youtube_dl/extractor/crackle.py
@@ -49,12 +49,15 @@ class CrackleIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
         item = self._download_xml(
-            'http://legacyweb-us.crackle.com/app/revamp/vidwallcache.aspx?flags=-1&fm=%s' % video_id, video_id).find('i')
+            'http://legacyweb-us.crackle.com/app/revamp/vidwallcache.aspx?flags=-1&fm=%s' % video_id,
+            video_id).find('i')
         title = item.attrib['t']
 
         thumbnail = None
         subtitles = {}
-        formats = self._extract_m3u8_formats('http://content.uplynk.com/ext/%s/%s.m3u8' % (self._UPLYNK_OWNER_ID, video_id), video_id, 'mp4', fatal=None)
+        formats = self._extract_m3u8_formats(
+            'http://content.uplynk.com/ext/%s/%s.m3u8' % (self._UPLYNK_OWNER_ID, video_id),
+            video_id, 'mp4', fatal=None)
         path = item.attrib.get('p')
         if path:
             thumbnail = self._THUMBNAIL_TEMPLATE % path
@@ -76,7 +79,7 @@ class CrackleIE(InfoExtractor):
                         'url': '%s/%s%s_%s.xml' % (self._SUBTITLE_SERVER, path, locale, v),
                         'ext': 'ttml',
                     }]
-        self._sort_formats(formats, ('width', 'height', 'tbr'))
+        self._sort_formats(formats, ('width', 'height', 'tbr', 'format_id'))
 
         return {
             'id': video_id,

From a1188d0ed06a619f4d0274eca602114648d7f33b Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 10 Feb 2016 22:39:33 +0100
Subject: [PATCH 3966/6219] [crackle] add prefix to format ids

---
 youtube_dl/extractor/crackle.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/crackle.py b/youtube_dl/extractor/crackle.py
index 70f3f14c3..79238cce7 100644
--- a/youtube_dl/extractor/crackle.py
+++ b/youtube_dl/extractor/crackle.py
@@ -57,7 +57,7 @@ class CrackleIE(InfoExtractor):
         subtitles = {}
         formats = self._extract_m3u8_formats(
             'http://content.uplynk.com/ext/%s/%s.m3u8' % (self._UPLYNK_OWNER_ID, video_id),
-            video_id, 'mp4', fatal=None)
+            video_id, 'mp4', m3u8_id='hls', fatal=None)
         path = item.attrib.get('p')
         if path:
             thumbnail = self._THUMBNAIL_TEMPLATE % path
@@ -65,7 +65,7 @@ class CrackleIE(InfoExtractor):
             for mfs_path, mfs_info in self._MEDIA_FILE_SLOTS.items():
                 formats.append({
                     'url': http_base_url + mfs_path,
-                    'format_id': mfs_path.split('.')[0],
+                    'format_id': 'http-' + mfs_path.split('.')[0],
                     'width': mfs_info['width'],
                     'height': mfs_info['height'],
                 })

From d1ea5e171f817d94bc356a96e7eb6967919cc18d Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 11 Feb 2016 10:30:31 +0100
Subject: [PATCH 3967/6219] [plays] Add new extractor(#8458)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/plays.py    | 50 ++++++++++++++++++++++++++++++++
 2 files changed, 51 insertions(+)
 create mode 100644 youtube_dl/extractor/plays.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index f08f27480..f1e5a5e86 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -534,6 +534,7 @@ from .planetaplay import PlanetaPlayIE
 from .pladform import PladformIE
 from .played import PlayedIE
 from .playfm import PlayFMIE
+from .plays import PlaysTVIE
 from .playtvak import PlaytvakIE
 from .playvid import PlayvidIE
 from .playwire import PlaywireIE
diff --git a/youtube_dl/extractor/plays.py b/youtube_dl/extractor/plays.py
new file mode 100644
index 000000000..2aba7cb9c
--- /dev/null
+++ b/youtube_dl/extractor/plays.py
@@ -0,0 +1,50 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import int_or_none
+
+
+class PlaysTVIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?plays\.tv/video/(?P<id>[0-9a-f]{18})'
+    _TEST = {
+        'url': 'http://plays.tv/video/56af17f56c95335490/when-you-outplay-the-azir-wall',
+        'md5': 'dfeac1198506652b5257a62762cec7bc',
+        'info_dict': {
+            'id': '56af17f56c95335490',
+            'ext': 'mp4',
+            'title': 'When you outplay the Azir wall',
+            'description': 'Posted by Bjergsen',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        title = self._og_search_title(webpage)
+        content = self._parse_json(
+            self._search_regex(r'R\.bindContent\(({.+?})\);', webpage,
+            'content'), video_id)['content']
+        mpd_url, sources = re.search(
+            r'(?s)<video[^>]+data-mpd="([^"]+)"[^>]*>(.+?)</video>',
+            content).groups()
+        formats = self._extract_mpd_formats(
+            self._proto_relative_url(mpd_url), video_id, mpd_id='DASH')
+        for format_id, height, format_url in re.findall(r'<source\s+res="((\d+)h?)"\s+src="([^"]+)"', sources):
+            formats.append({
+                'url': self._proto_relative_url(format_url),
+                'format_id': 'http-' + format_id,
+                'height': int_or_none(height),
+            })
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': self._og_search_description(webpage),
+            'thumbnail': self._og_search_thumbnail(webpage),
+            'formats': formats,
+        }

From 154c209e2db2bd6f51998503b15d9a050bc1e738 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 11 Feb 2016 10:33:26 +0100
Subject: [PATCH 3968/6219] [extractor/common] improve dash format ids

---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index a9497851c..00645feed 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1443,7 +1443,7 @@ class InfoExtractor(object):
                         url_el = representation.find(_add_ns('BaseURL'))
                         filesize = int_or_none(url_el.attrib.get('{http://youtube.com/yt/2012/10/10}contentLength') if url_el is not None else None)
                         f = {
-                            'format_id': mpd_id or representation_id,
+                            'format_id': '%s-%s' % (mpd_id, representation_id) if mpd_id else representation_id,
                             'url': base_url,
                             'width': int_or_none(representation_attrib.get('width')),
                             'height': int_or_none(representation_attrib.get('height')),

From dd867805969126ed0bd2ab8fe69eaf61fbf44ab7 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 11 Feb 2016 10:55:50 +0100
Subject: [PATCH 3969/6219] [extractor/common] fix dash formats sorting

---
 youtube_dl/extractor/common.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 00645feed..cd7087bec 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -851,6 +851,7 @@ class InfoExtractor(object):
             proto_preference = 0 if determine_protocol(f) in ['http', 'https'] else -0.1
 
             if f.get('vcodec') == 'none':  # audio only
+                preference -= 50
                 if self._downloader.params.get('prefer_free_formats'):
                     ORDER = ['aac', 'mp3', 'm4a', 'webm', 'ogg', 'opus']
                 else:
@@ -861,6 +862,8 @@ class InfoExtractor(object):
                 except ValueError:
                     audio_ext_preference = -1
             else:
+                if f.get('acodec') == 'none':  # video only
+                    preference -= 40
                 if self._downloader.params.get('prefer_free_formats'):
                     ORDER = ['flv', 'mp4', 'webm']
                 else:

From 9a61dfba0cfb9b978278fe2e1c823c205fcd72dd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 11 Feb 2016 21:22:57 +0600
Subject: [PATCH 3970/6219] [pbs] Revert prefer portalplayer

---
 youtube_dl/extractor/pbs.py | 24 +++++++++++++-----------
 1 file changed, 13 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index cbd1efea0..0d4d709de 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import compat_HTTPError
 from ..utils import (
     ExtractorError,
     determine_ext,
@@ -433,20 +434,21 @@ class PBSIE(InfoExtractor):
                 for vid_id in video_id]
             return self.playlist_result(entries, display_id)
 
-        player = self._download_webpage(
-            'http://player.pbs.org/portalplayer/%s' % video_id, display_id)
-
-        info = self._parse_json(
-            self._search_regex(
-                r'(?s)PBS\.videoData\s*=\s*({.+?});\n',
-                player, 'video data', default='{}'),
-            display_id, transform_source=js_to_json, fatal=False)
-
-        # Fallback to old videoInfo API
-        if not info:
+        try:
             info = self._download_json(
                 'http://player.pbs.org/videoInfo/%s?format=json&type=partner' % video_id,
                 display_id, 'Downloading video info JSON')
+        except ExtractorError as e:
+            if not isinstance(e.cause, compat_HTTPError) or e.cause.code != 404:
+                raise
+            # videoInfo API may not work for some videos, fallback to portalplayer API
+            player = self._download_webpage(
+                'http://player.pbs.org/portalplayer/%s' % video_id, display_id)
+            info = self._parse_json(
+                self._search_regex(
+                    r'(?s)PBS\.videoData\s*=\s*({.+?});\n',
+                    player, 'video data', default='{}'),
+                display_id, transform_source=js_to_json, fatal=False)
 
         formats = []
         for encoding_name in ('recommended_encoding', 'alternate_encoding'):

From b14d5e26f6c5c63f8de600e5159bf9cf12813e8b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 11 Feb 2016 21:24:07 +0600
Subject: [PATCH 3971/6219] [pbs] Improve description extraction

---
 youtube_dl/extractor/pbs.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 0d4d709de..0cdae6721 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -507,7 +507,7 @@ class PBSIE(InfoExtractor):
             'id': video_id,
             'display_id': display_id,
             'title': info['title'],
-            'description': info['program'].get('description'),
+            'description': info.get('description') or info.get('program', {}).get('description'),
             'thumbnail': info.get('image_url'),
             'duration': int_or_none(info.get('duration')),
             'age_limit': age_limit,

From a7ab46375b81dfab6b46535796f3f63296c7bb2c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 11 Feb 2016 21:43:01 +0600
Subject: [PATCH 3972/6219] [pbs] Update some tests

---
 youtube_dl/extractor/pbs.py | 27 +++++++++++++--------------
 1 file changed, 13 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 0cdae6721..c159c0071 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -201,7 +201,7 @@ class PBSIE(InfoExtractor):
                 'id': '2365006249',
                 'ext': 'mp4',
                 'title': 'Constitution USA with Peter Sagal - A More Perfect Union',
-                'description': 'md5:ba0c207295339c8d6eced00b7c363c6a',
+                'description': 'md5:36f341ae62e251b8f5bd2b754b95a071',
                 'duration': 3190,
             },
             'params': {
@@ -215,7 +215,7 @@ class PBSIE(InfoExtractor):
                 'id': '2365297690',
                 'ext': 'mp4',
                 'title': 'FRONTLINE - Losing Iraq',
-                'description': 'md5:f5bfbefadf421e8bb8647602011caf8e',
+                'description': 'md5:4d3eaa01f94e61b3e73704735f1196d9',
                 'duration': 5050,
             },
             'params': {
@@ -229,7 +229,7 @@ class PBSIE(InfoExtractor):
                 'id': '2201174722',
                 'ext': 'mp4',
                 'title': 'PBS NewsHour - Cyber Schools Gain Popularity, but Quality Questions Persist',
-                'description': 'md5:5871c15cba347c1b3d28ac47a73c7c28',
+                'description': 'md5:95a19f568689d09a166dff9edada3301',
                 'duration': 801,
             },
         },
@@ -239,8 +239,8 @@ class PBSIE(InfoExtractor):
             'info_dict': {
                 'id': '2365297708',
                 'ext': 'mp4',
-                'description': 'md5:68d87ef760660eb564455eb30ca464fe',
                 'title': 'Great Performances - Dudamel Conducts Verdi Requiem at the Hollywood Bowl - Full',
+                'description': 'md5:657897370e09e2bc6bf0f8d2cd313c6b',
                 'duration': 6559,
                 'thumbnail': 're:^https?://.*\.jpg$',
             },
@@ -280,7 +280,7 @@ class PBSIE(InfoExtractor):
                 'display_id': 'player',
                 'ext': 'mp4',
                 'title': 'American Experience - Death and the Civil War, Chapter 1',
-                'description': 'American Experience, TV’s most-watched history series, brings to life the compelling stories from our past that inform our understanding of the world today.',
+                'description': 'md5:1b80a74e0380ed2a4fb335026de1600d',
                 'duration': 682,
                 'thumbnail': 're:^https?://.*\.jpg$',
             },
@@ -289,20 +289,19 @@ class PBSIE(InfoExtractor):
             },
         },
         {
-            'url': 'http://video.pbs.org/video/2365367186/',
+            'url': 'http://www.pbs.org/video/2365245528/',
             'info_dict': {
-                'id': '2365367186',
-                'display_id': '2365367186',
+                'id': '2365245528',
+                'display_id': '2365245528',
                 'ext': 'mp4',
-                'title': 'To Catch A Comet - Full Episode',
-                'description': 'On November 12, 2014, billions of kilometers from Earth, spacecraft orbiter Rosetta and lander Philae did what no other had dared to attempt \u2014 land on the volatile surface of a comet as it zooms around the sun at 67,000 km/hr. The European Space Agency hopes this mission can help peer into our past and unlock secrets of our origins.',
-                'duration': 3342,
+                'title': 'FRONTLINE - United States of Secrets (Part One)',
+                'description': 'md5:55756bd5c551519cc4b7703e373e217e',
+                'duration': 6851,
                 'thumbnail': 're:^https?://.*\.jpg$',
             },
             'params': {
                 'skip_download': True,  # requires ffmpeg
             },
-            'skip': 'Expired',
         },
         {
             # Video embedded in iframe containing angle brackets as attribute's value (e.g.
@@ -314,7 +313,7 @@ class PBSIE(InfoExtractor):
                 'display_id': 'a-chefs-life-season-3-episode-5-prickly-business',
                 'ext': 'mp4',
                 'title': "A Chef's Life - Season 3, Ep. 5: Prickly Business",
-                'description': 'md5:61db2ddf27c9912f09c241014b118ed1',
+                'description': 'md5:54033c6baa1f9623607c6e2ed245888b',
                 'duration': 1480,
                 'thumbnail': 're:^https?://.*\.jpg$',
             },
@@ -330,7 +329,7 @@ class PBSIE(InfoExtractor):
                 'display_id': 'the-atomic-artists',
                 'ext': 'mp4',
                 'title': 'FRONTLINE - The Atomic Artists',
-                'description': 'md5:f5bfbefadf421e8bb8647602011caf8e',
+                'description': 'md5:1a2481e86b32b2e12ec1905dd473e2c1',
                 'duration': 723,
                 'thumbnail': 're:^https?://.*\.jpg$',
             },

From 6b3fbd3425180a00d99e91e6f2b9b5896c567ea4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 11 Feb 2016 22:02:37 +0600
Subject: [PATCH 3973/6219] [pbs] Fix multi part videos extraction

---
 youtube_dl/extractor/pbs.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index c159c0071..cca012953 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -366,10 +366,14 @@ class PBSIE(InfoExtractor):
                 webpage, 'upload date', default=None))
 
             # tabbed frontline videos
-            tabbed_videos = re.findall(
-                r'<div[^>]+class="videotab[^"]*"[^>]+vid="(\d+)"', webpage)
-            if tabbed_videos:
-                return tabbed_videos, presumptive_id, upload_date
+            MULTI_PART_REGEXES = (
+                r'<div[^>]+class="videotab[^"]*"[^>]+vid="(\d+)"',
+                r'<a[^>]+href=["\']#video-\d+["\'][^>]+data-coveid=["\'](\d+)',
+            )
+            for p in MULTI_PART_REGEXES:
+                tabbed_videos = re.findall(p, webpage)
+                if tabbed_videos:
+                    return tabbed_videos, presumptive_id, upload_date
 
             MEDIA_ID_REGEXES = [
                 r"div\s*:\s*'videoembed'\s*,\s*mediaid\s*:\s*'(\d+)'",  # frontline video embed

From 28280e8ded11ca13d67b705a5a56fb930ba26205 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 11 Feb 2016 22:02:57 +0600
Subject: [PATCH 3974/6219] [plays] PEP 8

---
 youtube_dl/extractor/plays.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/plays.py b/youtube_dl/extractor/plays.py
index 2aba7cb9c..c3c38cf4a 100644
--- a/youtube_dl/extractor/plays.py
+++ b/youtube_dl/extractor/plays.py
@@ -26,8 +26,9 @@ class PlaysTVIE(InfoExtractor):
 
         title = self._og_search_title(webpage)
         content = self._parse_json(
-            self._search_regex(r'R\.bindContent\(({.+?})\);', webpage,
-            'content'), video_id)['content']
+            self._search_regex(
+                r'R\.bindContent\(({.+?})\);', webpage,
+                'content'), video_id)['content']
         mpd_url, sources = re.search(
             r'(?s)<video[^>]+data-mpd="([^"]+)"[^>]*>(.+?)</video>',
             content).groups()

From b67d63149d79f98a84bec0ecf8028282d523d1f2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 11 Feb 2016 22:33:08 +0600
Subject: [PATCH 3975/6219] [youtube] Fix typos

---
 youtube_dl/extractor/youtube.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index b9a91dea2..382d52cb6 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -375,7 +375,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
     IE_NAME = 'youtube'
     _TESTS = [
         {
-            'url': 'http://www.youtube.com/watch?v=BaW_jenozKcj&t=1s&end=9',
+            'url': 'http://www.youtube.com/watch?v=BaW_jenozKc&t=1s&end=9',
             'info_dict': {
                 'id': 'BaW_jenozKc',
                 'ext': 'mp4',
@@ -441,7 +441,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             }
         },
         {
-            'url': 'http://www.youtube.com/watch?v=BaW_jenozKcj&v=UxxajLWwzqY',
+            'url': 'http://www.youtube.com/watch?v=BaW_jenozKc&v=UxxajLWwzqY',
             'note': 'Use the first video ID in the URL',
             'info_dict': {
                 'id': 'BaW_jenozKc',

From 388ae76b52ff85dba5c47f5958fbe3e662abd346 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 11 Feb 2016 22:46:13 +0600
Subject: [PATCH 3976/6219] [YoutubeDL] Fix format resolution when height is
 missing

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 3cabfbc03..e5b75f0f5 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1798,7 +1798,7 @@ class YoutubeDL(object):
             else:
                 res = '%sp' % format['height']
         elif format.get('width') is not None:
-            res = '?x%d' % format['width']
+            res = '%dx?' % format['width']
         else:
             res = default
         return res

From 81e1c4e2fc260c77de8afabc73090b549d6ba259 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 11 Feb 2016 17:58:48 +0100
Subject: [PATCH 3977/6219] [extractor/common] remove duplicate rtmp formats in
 smil manifest

---
 youtube_dl/extractor/common.py | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 00645feed..6b40a8323 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1186,12 +1186,13 @@ class InfoExtractor(object):
         http_count = 0
         m3u8_count = 0
 
-        src_urls = []
+        srcs = []
         videos = smil.findall(self._xpath_ns('.//video', namespace))
         for video in videos:
             src = video.get('src')
-            if not src:
+            if not src or src in srcs:
                 continue
+            srcs.append(src)
 
             bitrate = float_or_none(video.get('system-bitrate') or video.get('systemBitrate'), 1000)
             filesize = int_or_none(video.get('size') or video.get('fileSize'))
@@ -1223,9 +1224,6 @@ class InfoExtractor(object):
                 continue
 
             src_url = src if src.startswith('http') else compat_urlparse.urljoin(base, src)
-            if src_url in src_urls:
-                continue
-            src_urls.append(src_url)
 
             if proto == 'm3u8' or src_ext == 'm3u8':
                 m3u8_formats = self._extract_m3u8_formats(

From 8b85ac3fd95dbc3296801036f43c8d2ebfefcb6f Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 11 Feb 2016 18:10:32 +0100
Subject: [PATCH 3978/6219] [cbc] Add new extractor(closes #3803)(closes
 #4731)(closes #5309)

---
 youtube_dl/extractor/__init__.py |   4 ++
 youtube_dl/extractor/cbc.py      | 113 +++++++++++++++++++++++++++++++
 2 files changed, 117 insertions(+)
 create mode 100644 youtube_dl/extractor/cbc.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index f1e5a5e86..7056cf323 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -89,6 +89,10 @@ from .camdemy import (
 from .canalplus import CanalplusIE
 from .canalc2 import Canalc2IE
 from .canvas import CanvasIE
+from .cbc import (
+    CBCIE,
+    CBCPlayerIE,
+)
 from .cbs import CBSIE
 from .cbsnews import (
     CBSNewsIE,
diff --git a/youtube_dl/extractor/cbc.py b/youtube_dl/extractor/cbc.py
new file mode 100644
index 000000000..d8aa31038
--- /dev/null
+++ b/youtube_dl/extractor/cbc.py
@@ -0,0 +1,113 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import js_to_json
+
+
+class CBCIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?cbc\.ca/(?:[^/]+/)+(?P<id>[^/?#]+)'
+    _TESTS = [{
+        # with mediaId
+        'url': 'http://www.cbc.ca/22minutes/videos/clips-season-23/don-cherry-play-offs',
+        'info_dict': {
+            'id': '2682904050',
+            'ext': 'flv',
+            'title': 'Don Cherry – All-Stars',
+            'description': 'Don Cherry has a bee in his bonnet about AHL player John Scott because that guy’s got heart.',
+            'timestamp': 1454475540,
+            'upload_date': '20160203',
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        },
+    }, {
+        # with clipId
+        'url': 'http://www.cbc.ca/archives/entry/1978-robin-williams-freestyles-on-90-minutes-live',
+        'info_dict': {
+            'id': '2487345465',
+            'ext': 'flv',
+            'title': 'Robin Williams freestyles on 90 Minutes Live',
+            'description': 'Wacky American comedian Robin Williams shows off his infamous "freestyle" comedic talents while being interviewed on CBC\'s 90 Minutes Live.',
+            'upload_date': '19700101',
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        },
+    }, {
+        # multiple iframes
+        'url': 'http://www.cbc.ca/natureofthings/blog/birds-eye-view-from-vancouvers-burrard-street-bridge-how-we-got-the-shot',
+        'playlist': [{
+            'info_dict': {
+                'id': '2680832926',
+                'ext': 'flv',
+                'title': 'An Eagle\'s-Eye View Off Burrard Bridge',
+                'description': 'Hercules the eagle flies from Vancouver\'s Burrard Bridge down to a nearby park with a mini-camera strapped to his back.',
+                'upload_date': '19700101',
+            },
+        }, {
+            'info_dict': {
+                'id': '2658915080',
+                'ext': 'flv',
+                'title': 'Fly like an eagle!',
+                'description': 'Eagle equipped with a mini camera flies from the world\'s tallest tower',
+                'upload_date': '19700101',
+            },
+        }],
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        },
+    }]
+
+    @classmethod
+    def suitable(cls, url):
+        return False if CBCPlayerIE.suitable(url) else super(CBCIE, cls).suitable(url)
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+        player_init = self._search_regex(
+            r'CBC\.APP\.Caffeine\.initInstance\(({.+?})\);', webpage, 'player init',
+            default=None)
+        if player_init:
+            player_info = self._parse_json(player_init, display_id, js_to_json)
+            media_id = player_info.get('mediaId')
+            if not media_id:
+                clip_id = player_info['clipId']
+                media_id = self._download_json(
+                    'http://feed.theplatform.com/f/h9dtGB/punlNGjMlc1F?fields=id&byContent=byReleases%3DbyId%253D' + clip_id,
+                    clip_id)['entries'][0]['id'].split('/')[-1]
+            return self.url_result('cbcplayer:%s' % media_id, 'CBCPlayer', media_id)
+        else:
+            entries = [self.url_result('cbcplayer:%s' % media_id, 'CBCPlayer', media_id) for media_id in re.findall(r'<iframe[^>]+src="[^"]+?mediaId=(\d+)"', webpage)]
+            return self.playlist_result(entries)
+
+
+class CBCPlayerIE(InfoExtractor):
+    _VALID_URL = r'(?:cbcplayer:|https?://(?:www\.)?cbc\.ca/(?:player/play/|i/caffeine/syndicate/\?mediaId=))(?P<id>\d+)'
+    _TEST = {
+        'url': 'http://www.cbc.ca/player/play/2683190193',
+        'info_dict': {
+            'id': '2683190193',
+            'ext': 'flv',
+            'title': 'Gerry Runs a Sweat Shop',
+            'description': 'md5:b457e1c01e8ff408d9d801c1c2cd29b0',
+            'timestamp': 1455067800,
+            'upload_date': '20160210',
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        return self.url_result(
+            'http://feed.theplatform.com/f/ExhSPC/vms_5akSXx4Ng_Zn?byGuid=%s' % video_id,
+            'ThePlatformFeed', video_id)

From 9c7b38981cb6429ba690aaef313e36fb694f9902 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 11 Feb 2016 23:12:30 +0600
Subject: [PATCH 3979/6219] [utils] Bump Firefox version in User-Agent

Old version number causes Youtube not to serve some formats in ytplayer.config
---
 youtube_dl/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index dce67de8a..a82a262a0 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -56,7 +56,7 @@ from .compat import (
 compiled_regex_type = type(re.compile(''))
 
 std_headers = {
-    'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20150101 Firefox/20.0 (Chrome)',
+    'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20150101 Firefox/44.0 (Chrome)',
     'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
     'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
     'Accept-Encoding': 'gzip, deflate',

From 3dc71d82ce628d08327a098d8e87af0905d5b59f Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 11 Feb 2016 18:12:38 +0100
Subject: [PATCH 3980/6219] [theplatform] fix pid extraction in the platform
 feed

---
 youtube_dl/extractor/theplatform.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index 10f2cad55..f2e975ab0 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -284,7 +284,7 @@ class ThePlatformFeedIE(ThePlatformBaseIE):
         duration = None
         for item in entry['media$content']:
             smil_url = item['plfile$url'] + '&format=SMIL&Tracking=true&Embedded=true&formats=MPEG4,F4M'
-            cur_video_id = url_basename(smil_url)
+            cur_video_id = ThePlatformIE._match_id(smil_url)
             if first_video_id is None:
                 first_video_id = cur_video_id
                 duration = float_or_none(item.get('plfile$duration'))

From a7cab4d03927c90b8db12fda627022996e560c78 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 11 Feb 2016 18:50:14 +0100
Subject: [PATCH 3981/6219] [theplatform] remove unused import and change smil
 url for ThePlatformFeedIE

---
 youtube_dl/extractor/theplatform.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index f2e975ab0..755f816ff 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -20,7 +20,6 @@ from ..utils import (
     int_or_none,
     sanitized_Request,
     unsmuggle_url,
-    url_basename,
     xpath_with_ns,
 )
 
@@ -283,7 +282,7 @@ class ThePlatformFeedIE(ThePlatformBaseIE):
         first_video_id = None
         duration = None
         for item in entry['media$content']:
-            smil_url = item['plfile$url'] + '&format=SMIL&Tracking=true&Embedded=true&formats=MPEG4,F4M'
+            smil_url = item['plfile$url'] + '&format=SMIL&mbr=true'
             cur_video_id = ThePlatformIE._match_id(smil_url)
             if first_video_id is None:
                 first_video_id = cur_video_id

From b826035dd5948d598e29bc17c9d08e3c4d77abbc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 12 Feb 2016 03:16:26 +0600
Subject: [PATCH 3982/6219] [vimeo] Fix authentication (Closes #8520)

---
 youtube_dl/extractor/vimeo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 2389e7f0f..6a8f9b49d 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -57,7 +57,7 @@ class VimeoBaseInfoExtractor(InfoExtractor):
 
     def _extract_xsrft_and_vuid(self, webpage):
         xsrft = self._search_regex(
-            r'xsrft\s*[=:]\s*(?P<q>["\'])(?P<xsrft>.+?)(?P=q)',
+            r'(?:(?P<q1>["\'])xsrft(?P=q1)\s*:|xsrft\s*[=:])\s*(?P<q>["\'])(?P<xsrft>.+?)(?P=q)',
             webpage, 'login token', group='xsrft')
         vuid = self._search_regex(
             r'["\']vuid["\']\s*:\s*(["\'])(?P<vuid>.+?)\1',

From 5a4905924d15bb2bdc0a80f6fd41a869a9787d38 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 12 Feb 2016 22:03:10 +0600
Subject: [PATCH 3983/6219] [extractor/generic] Improve dailymotion embed
 detection (Closes #8521, closes #8325)

---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index bf61ab2e7..ede9801bb 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1413,7 +1413,7 @@ class GenericIE(InfoExtractor):
 
         # Look for embedded Dailymotion player
         matches = re.findall(
-            r'<(?:embed|iframe)[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?dailymotion\.com/(?:embed|swf)/video/.+?)\1', webpage)
+            r'<(?:(?:embed|iframe)[^>]+?src=|input[^>]+id=[\'"]dmcloudUrlEmissionSelect[\'"][^>]+value=)(["\'])(?P<url>(?:https?:)?//(?:www\.)?dailymotion\.com/(?:embed|swf)/video/.+?)\1', webpage)
         if matches:
             return _playlist_from_matches(
                 matches, lambda m: unescapeHTML(m[1]))

From c349456ef60d2f638be808bb023635c63df4107c Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 12 Feb 2016 17:38:48 +0100
Subject: [PATCH 3984/6219] [extractor/common] strip http urls in smil manifest

---
 youtube_dl/extractor/common.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 6b40a8323..359bdd299 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1224,6 +1224,7 @@ class InfoExtractor(object):
                 continue
 
             src_url = src if src.startswith('http') else compat_urlparse.urljoin(base, src)
+            src_url = src_url.strip()
 
             if proto == 'm3u8' or src_ext == 'm3u8':
                 m3u8_formats = self._extract_m3u8_formats(

From 3d98f97c6408326dd841fae767a721ed91a1cae8 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 12 Feb 2016 17:39:32 +0100
Subject: [PATCH 3985/6219] [aenetworks] extract http formats

---
 youtube_dl/extractor/aenetworks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/aenetworks.py b/youtube_dl/extractor/aenetworks.py
index 43d7b0523..e192b5ac5 100644
--- a/youtube_dl/extractor/aenetworks.py
+++ b/youtube_dl/extractor/aenetworks.py
@@ -61,6 +61,6 @@ class AENetworksIE(InfoExtractor):
         info = self._search_json_ld(webpage, video_id, fatal=False)
         info.update({
             '_type': 'url_transparent',
-            'url': smuggle_url(video_url, {'sig': {'key': 'crazyjava', 'secret': 's3cr3t'}}),
+            'url': smuggle_url(video_url + '?mbr=true&format=SMIL&assetTypes=medium_video_s3', {'sig': {'key': 'crazyjava', 'secret': 's3cr3t'}, 'force_smil_url': True}),
         })
         return info

From bcbb1b08b24215731d3444297508a1bd6b7011ff Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 12 Feb 2016 17:56:06 +0100
Subject: [PATCH 3986/6219] Revert "[aenetworks] extract http formats"

This reverts commit 3d98f97c6408326dd841fae767a721ed91a1cae8.
---
 youtube_dl/extractor/aenetworks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/aenetworks.py b/youtube_dl/extractor/aenetworks.py
index e192b5ac5..43d7b0523 100644
--- a/youtube_dl/extractor/aenetworks.py
+++ b/youtube_dl/extractor/aenetworks.py
@@ -61,6 +61,6 @@ class AENetworksIE(InfoExtractor):
         info = self._search_json_ld(webpage, video_id, fatal=False)
         info.update({
             '_type': 'url_transparent',
-            'url': smuggle_url(video_url + '?mbr=true&format=SMIL&assetTypes=medium_video_s3', {'sig': {'key': 'crazyjava', 'secret': 's3cr3t'}, 'force_smil_url': True}),
+            'url': smuggle_url(video_url, {'sig': {'key': 'crazyjava', 'secret': 's3cr3t'}}),
         })
         return info

From bb20526b64c1624639b18e1b582fbdee1bd5da64 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 13 Feb 2016 00:13:56 +0600
Subject: [PATCH 3987/6219] [extractor/common] Improve base url construction

---
 youtube_dl/extractor/common.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 359bdd299..444d412d9 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1435,7 +1435,9 @@ class InfoExtractor(object):
                                 base_url = base_url_e.text + base_url
                                 if re.match(r'^https?://', base_url):
                                     break
-                        if not re.match(r'^https?://', base_url):
+                        if mpd_base_url and not re.match(r'^https?://', base_url):
+                            if not mpd_base_url.endswith('/') and not base_url.startswith('/'):
+                                mpd_base_url += '/'
                             base_url = mpd_base_url + base_url
                         representation_id = representation_attrib.get('id')
                         lang = representation_attrib.get('lang')

From 7be747b921f0e31f4db3db2fd7e1c90d265ed566 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 13 Feb 2016 00:15:59 +0600
Subject: [PATCH 3988/6219] [extractor/generic] Pass mpd base url to
 _parse_mpd_formats

---
 youtube_dl/extractor/generic.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index ede9801bb..d6d0a9ed0 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1302,7 +1302,8 @@ class GenericIE(InfoExtractor):
                 return {
                     'id': video_id,
                     'title': compat_urllib_parse_unquote(os.path.splitext(url_basename(url))[0]),
-                    'formats': self._parse_mpd_formats(doc, video_id),
+                    'formats': self._parse_mpd_formats(
+                        doc, video_id, mpd_base_url=url.rpartition('/')[0]),
                 }
         except compat_xml_parse_error:
             pass

From 4c77d3f52ad16d9d9ac0ff0f15a62beef3158ca4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 13 Feb 2016 00:23:14 +0600
Subject: [PATCH 3989/6219] [YoutubeDL] Allow bestvideo+bestaudio for any
 extractor

---
 youtube_dl/YoutubeDL.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index e5b75f0f5..0072c7d35 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1341,7 +1341,6 @@ class YoutubeDL(object):
         if req_format is None:
             req_format_list = []
             if (self.params.get('outtmpl', DEFAULT_OUTTMPL) != '-' and
-                    info_dict['extractor'] in ['youtube', 'ted'] and
                     not info_dict.get('is_live')):
                 merger = FFmpegMergerPP(self)
                 if merger.available and merger.can_merge():

From 9d939cec48f06a401fb79eb078c1fc50b2aefbe1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 13 Feb 2016 00:36:47 +0600
Subject: [PATCH 3990/6219] [extractor/generic] Add direct mpd url test

---
 youtube_dl/extractor/generic.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index d6d0a9ed0..45adbb7a3 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -224,6 +224,20 @@ class GenericIE(InfoExtractor):
                 'skip_download': True,
             },
         },
+        # MPD from http://dash-mse-test.appspot.com/media.html
+        {
+            'url': 'http://yt-dash-mse-test.commondatastorage.googleapis.com/media/car-20120827-manifest.mpd',
+            'md5': '4b57baab2e30d6eb3a6a09f0ba57ef53',
+            'info_dict': {
+                'id': 'car-20120827-manifest',
+                'ext': 'mp4',
+                'title': 'car-20120827-manifest',
+                'formats': 'mincount:9',
+            },
+            'params': {
+                'format': 'bestvideo',
+            },
+        },
         # google redirect
         {
             'url': 'http://www.google.com/url?sa=t&rct=j&q=&esrc=s&source=web&cd=1&cad=rja&ved=0CCUQtwIwAA&url=http%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DcmQHVoWB5FY&ei=F-sNU-LLCaXk4QT52ICQBQ&usg=AFQjCNEw4hL29zgOohLXvpJ-Bdh2bils1Q&bvm=bv.61965928,d.bGE',

From 6863631c2696f93123ae5ceca1e261311f0d8925 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 13 Feb 2016 05:01:20 +0600
Subject: [PATCH 3991/6219] [youtube] Improve multifeed videos extraction
 (Closes #8536)

---
 youtube_dl/extractor/youtube.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 382d52cb6..97d621907 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1196,9 +1196,12 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             if not self._downloader.params.get('noplaylist'):
                 entries = []
                 feed_ids = []
-                multifeed_metadata_list = compat_urllib_parse_unquote_plus(video_info['multifeed_metadata_list'][0])
+                multifeed_metadata_list = video_info['multifeed_metadata_list'][0]
                 for feed in multifeed_metadata_list.split(','):
-                    feed_data = compat_parse_qs(feed)
+                    # Unquote should take place before split on comma (,) since textual
+                    # fields may contain comma as well (see
+                    # https://github.com/rg3/youtube-dl/issues/8536)
+                    feed_data = compat_parse_qs(compat_urllib_parse_unquote_plus(feed))
                     entries.append({
                         '_type': 'url_transparent',
                         'ie_key': 'Youtube',

From f9f49d87c2d731e220ee4b5cf83fbbd17e3bfa5d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 13 Feb 2016 05:18:58 +0600
Subject: [PATCH 3992/6219] [youtube] Add test for #8536

---
 youtube_dl/extractor/youtube.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 97d621907..18f7d37f4 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -704,6 +704,15 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'skip_download': True,
             },
         },
+        {
+            # Multifeed video with comma in title (see https://github.com/rg3/youtube-dl/issues/8536)
+            'url': 'https://www.youtube.com/watch?v=gVfLd0zydlo',
+            'info_dict': {
+                'id': 'gVfLd0zydlo',
+                'title': 'DevConf.cz 2016 Day 2 Workshops 1 14:00 - 15:30',
+            },
+            'playlist_count': 2,
+        },
         {
             'url': 'http://vid.plus/FlRa-iH7PGw',
             'only_matching': True,

From 3e18700d45fe4117bd2560a58129e763c408b2ef Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 13 Feb 2016 07:45:32 +0600
Subject: [PATCH 3993/6219] [nbc] Correct test

---
 youtube_dl/extractor/nbc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index 18d01f423..2202cfa33 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -57,7 +57,7 @@ class NBCIE(InfoExtractor):
         {
             # This video has expired but with an escaped embedURL
             'url': 'http://www.nbc.com/parenthood/episode-guide/season-5/just-like-at-home/515',
-            'skip': 'Expired'
+            'only_matching': True,
         }
     ]
 

From 35ced3985acb23282b96771ac8b6570aab180768 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 13 Feb 2016 08:25:05 +0100
Subject: [PATCH 3994/6219] release 2016.02.13

---
 docs/supportedsites.md | 4 ++++
 youtube_dl/version.py  | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index c48561eff..b384a3165 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -89,6 +89,8 @@
  - **canalc2.tv**
  - **Canalplus**: canalplus.fr, piwiplus.fr and d8.tv
  - **Canvas**
+ - **CBC**
+ - **CBCPlayer**
  - **CBS**
  - **CBSNews**: CBS News
  - **CBSNewsLiveVideo**: CBS News Live Videos
@@ -120,6 +122,7 @@
  - **ComedyCentralShows**: The Daily Show / The Colbert Report
  - **CondeNast**: Condé Nast media group: Allure, Architectural Digest, Ars Technica, Bon Appétit, Brides, Condé Nast, Condé Nast Traveler, Details, Epicurious, GQ, Glamour, Golf Digest, SELF, Teen Vogue, The New Yorker, Vanity Fair, Vogue, W Magazine, WIRED
  - **Cracked**
+ - **Crackle**
  - **Criterion**
  - **CrooksAndLiars**
  - **Crunchyroll**
@@ -445,6 +448,7 @@
  - **PlanetaPlay**
  - **play.fm**
  - **played.to**
+ - **PlaysTV**
  - **Playtvak**: Playtvak.cz, iDNES.cz and Lidovky.cz
  - **Playvid**
  - **Playwire**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 2d0d67b0f..9aca8001a 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.02.10'
+__version__ = '2016.02.13'

From e2bd68c90197ae629c1bbea899b6fc6db7bfffdf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 13 Feb 2016 13:30:31 +0600
Subject: [PATCH 3995/6219] [animeondemand][wip] Add extractor (#8518)

---
 youtube_dl/extractor/__init__.py      |   1 +
 youtube_dl/extractor/animeondemand.py | 159 ++++++++++++++++++++++++++
 2 files changed, 160 insertions(+)
 create mode 100644 youtube_dl/extractor/animeondemand.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 7056cf323..6937f28d3 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -20,6 +20,7 @@ from .aftonbladet import AftonbladetIE
 from .airmozilla import AirMozillaIE
 from .aljazeera import AlJazeeraIE
 from .alphaporno import AlphaPornoIE
+from .animeondemand import AnimeOnDemandIE
 from .anitube import AnitubeIE
 from .anysex import AnySexIE
 from .aol import AolIE
diff --git a/youtube_dl/extractor/animeondemand.py b/youtube_dl/extractor/animeondemand.py
new file mode 100644
index 000000000..0d0516ba6
--- /dev/null
+++ b/youtube_dl/extractor/animeondemand.py
@@ -0,0 +1,159 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..compat import compat_urlparse
+from ..utils import (
+    determine_ext,
+    encode_dict,
+    ExtractorError,
+    sanitized_Request,
+    urlencode_postdata,
+)
+
+
+class AnimeOnDemandIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?anime-on-demand\.de/anime/(?P<id>\d+)'
+    _LOGIN_URL = 'https://www.anime-on-demand.de/users/sign_in'
+    _APPLY_HTML5_URL = 'https://www.anime-on-demand.de/html5apply'
+    _TEST = {
+        'url': 'https://www.anime-on-demand.de/anime/161',
+        'info_dict': {
+            'id': '161',
+            'title': 'Grimgar, Ashes and Illusions (OmU)',
+            'description': 'md5:6681ce3c07c7189d255ac6ab23812d31',
+        },
+        'playlist_mincount': 4,
+    }
+
+    def _login(self):
+        (username, password) = self._get_login_info()
+        if username is None:
+            return
+
+        login_page = self._download_webpage(
+            self._LOGIN_URL, None, 'Downloading login page')
+
+        login_form = self._form_hidden_inputs('new_user', login_page)
+
+        login_form.update({
+            'user[login]': username,
+            'user[password]': password,
+        })
+
+        post_url = self._search_regex(
+            r'<form[^>]+action=(["\'])(?P<url>.+?)\1', login_page,
+            'post url', default=self._LOGIN_URL, group='url')
+
+        if not post_url.startswith('http'):
+            post_url = compat_urlparse.urljoin(self._LOGIN_URL, post_url)
+
+        request = sanitized_Request(
+            post_url, urlencode_postdata(encode_dict(login_form)))
+        request.add_header('Referer', self._LOGIN_URL)
+
+        response = self._download_webpage(
+            request, None, 'Logging in as %s' % username)
+
+        if all(p not in response for p in ('>Logout<', 'href="/users/sign_out"')):
+            error = self._search_regex(
+                r'<p class="alert alert-danger">(.+?)</p>',
+                response, 'error', default=None)
+            if error:
+                raise ExtractorError('Unable to login: %s' % error, expected=True)
+            raise ExtractorError('Unable to log in')
+
+    def _real_initialize(self):
+        self._login()
+
+    def _real_extract(self, url):
+        anime_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, anime_id)
+
+        if 'data-playlist=' not in webpage:
+            self._download_webpage(
+                self._APPLY_HTML5_URL, anime_id,
+                'Activating HTML5 beta', 'Unable to apply HTML5 beta')
+            webpage = self._download_webpage(url, anime_id)
+
+        csrf_token = self._html_search_meta(
+            'csrf-token', webpage, 'csrf token', fatal=True)
+
+        anime_title = self._html_search_regex(
+            r'(?s)<h1[^>]+itemprop="name"[^>]*>(.+?)</h1>',
+            webpage, 'anime name')
+        anime_description = self._html_search_regex(
+            r'(?s)<div[^>]+itemprop="description"[^>]*>(.+?)</div>',
+            webpage, 'anime description', default=None)
+
+        entries = []
+
+        for episode_html in re.findall(r'(?s)<h3[^>]+class="episodebox-title".+?>Episodeninhalt<', webpage):
+            m = re.search(
+                r'class="episodebox-title"[^>]+title="Episode (?P<number>\d+) - (?P<title>.+?)"', episode_html)
+            if not m:
+                continue
+
+            episode_number = int(m.group('number'))
+            episode_title = m.group('title')
+            video_id = 'episode-%d' % episode_number
+
+            common_info = {
+                'id': video_id,
+                'series': anime_title,
+                'episode': episode_title,
+                'episode_number': episode_number,
+            }
+
+            formats = []
+
+            playlist_url = self._search_regex(
+                r'data-playlist=(["\'])(?P<url>.+?)\1',
+                episode_html, 'data playlist', default=None, group='url')
+            if playlist_url:
+                request = sanitized_Request(
+                    compat_urlparse.urljoin(url, playlist_url),
+                    headers={
+                        'X-Requested-With': 'XMLHttpRequest',
+                        'X-CSRF-Token': csrf_token,
+                        'Referer': url,
+                        'Accept': 'application/json, text/javascript, */*; q=0.01',
+                    })
+
+                playlist = self._download_json(
+                    request, video_id, 'Downloading playlist JSON', fatal=False)
+                if playlist:
+                    playlist = playlist['playlist'][0]
+                    title = playlist['title']
+                    description = playlist.get('description')
+                    for source in playlist.get('sources', []):
+                        file_ = source.get('file')
+                        if file_ and determine_ext(file_) == 'm3u8':
+                            formats = self._extract_m3u8_formats(
+                                file_, video_id, 'mp4',
+                                entry_protocol='m3u8_native', m3u8_id='hls')
+
+            if formats:
+                f = common_info.copy()
+                f.update({
+                    'title': title,
+                    'description': description,
+                    'formats': formats,
+                })
+                entries.append(f)
+
+            m = re.search(
+                r'data-dialog-header=(["\'])(?P<title>.+?)\1[^>]+href=(["\'])(?P<href>.+?)\3[^>]*>Teaser<',
+                episode_html)
+            if m:
+                f = common_info.copy()
+                f.update({
+                    'id': '%s-teaser' % f['id'],
+                    'title': m.group('title'),
+                    'url': compat_urlparse.urljoin(url, m.group('href')),
+                })
+                entries.append(f)
+
+        return self.playlist_result(entries, anime_id, anime_title, anime_description)

From 702325123964c767eafa5667aa3d3844f8133e8a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 13 Feb 2016 12:26:27 +0100
Subject: [PATCH 3996/6219] [comedycentral] Support /shows URLs (fixes #8405)

---
 youtube_dl/extractor/comedycentral.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/comedycentral.py b/youtube_dl/extractor/comedycentral.py
index 3e4bd10b6..055c9eec5 100644
--- a/youtube_dl/extractor/comedycentral.py
+++ b/youtube_dl/extractor/comedycentral.py
@@ -16,11 +16,11 @@ from ..utils import (
 
 class ComedyCentralIE(MTVServicesInfoExtractor):
     _VALID_URL = r'''(?x)https?://(?:www\.)?cc\.com/
-        (video-clips|episodes|cc-studios|video-collections|full-episodes)
+        (video-clips|episodes|cc-studios|video-collections|full-episodes|shows)
         /(?P<title>.*)'''
     _FEED_URL = 'http://comedycentral.com/feeds/mrss/'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.cc.com/video-clips/kllhuv/stand-up-greg-fitzsimmons--uncensored---too-good-of-a-mother',
         'md5': 'c4f48e9eda1b16dd10add0744344b6d8',
         'info_dict': {
@@ -29,7 +29,10 @@ class ComedyCentralIE(MTVServicesInfoExtractor):
             'title': 'CC:Stand-Up|Greg Fitzsimmons: Life on Stage|Uncensored - Too Good of a Mother',
             'description': 'After a certain point, breastfeeding becomes c**kblocking.',
         },
-    }
+    }, {
+        'url': 'http://www.cc.com/shows/the-daily-show-with-trevor-noah/interviews/6yx39d/exclusive-rand-paul-extended-interview',
+        'only_matching': True,
+    }]
 
 
 class ComedyCentralShowsIE(MTVServicesInfoExtractor):

From b4561e857f7bbeb6571d36c9991b3fc5c109352d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 13 Feb 2016 22:41:58 +0600
Subject: [PATCH 3997/6219] [animeondemand] Add .netrc

---
 youtube_dl/extractor/animeondemand.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/animeondemand.py b/youtube_dl/extractor/animeondemand.py
index 0d0516ba6..a7d8daf7b 100644
--- a/youtube_dl/extractor/animeondemand.py
+++ b/youtube_dl/extractor/animeondemand.py
@@ -17,6 +17,7 @@ class AnimeOnDemandIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?anime-on-demand\.de/anime/(?P<id>\d+)'
     _LOGIN_URL = 'https://www.anime-on-demand.de/users/sign_in'
     _APPLY_HTML5_URL = 'https://www.anime-on-demand.de/html5apply'
+    _NETRC_MACHINE = 'animeondemand'
     _TEST = {
         'url': 'https://www.anime-on-demand.de/anime/161',
         'info_dict': {

From b45f2b1d6ec2fb4081d261f862e9413c5870bc16 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 14 Feb 2016 11:24:57 +0600
Subject: [PATCH 3998/6219] [myvideo] Mark broken

---
 youtube_dl/extractor/myvideo.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/myvideo.py b/youtube_dl/extractor/myvideo.py
index 36ab388b2..1e21cf98a 100644
--- a/youtube_dl/extractor/myvideo.py
+++ b/youtube_dl/extractor/myvideo.py
@@ -19,6 +19,7 @@ from ..utils import (
 
 
 class MyVideoIE(InfoExtractor):
+    _WORKING = False
     _VALID_URL = r'http://(?:www\.)?myvideo\.de/(?:[^/]+/)?watch/(?P<id>[0-9]+)/[^?/]+.*'
     IE_NAME = 'myvideo'
     _TEST = {

From fca2e6d5a6fd402bc14dd381c1060f6a2cc38eb8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 14 Feb 2016 13:44:23 +0600
Subject: [PATCH 3999/6219] [dailymotion:cloud] Use idiomatic name for
 classmethod's first argument

---
 youtube_dl/extractor/dailymotion.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index 6e462af69..7ae9f2359 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -396,13 +396,13 @@ class DailymotionCloudIE(DailymotionBaseInfoExtractor):
     }]
 
     @classmethod
-    def _extract_dmcloud_url(self, webpage):
-        mobj = re.search(r'<iframe[^>]+src=[\'"](%s)[\'"]' % self._VALID_EMBED_URL, webpage)
+    def _extract_dmcloud_url(cls, webpage):
+        mobj = re.search(r'<iframe[^>]+src=[\'"](%s)[\'"]' % cls._VALID_EMBED_URL, webpage)
         if mobj:
             return mobj.group(1)
 
         mobj = re.search(
-            r'<input[^>]+id=[\'"]dmcloudUrlEmissionSelect[\'"][^>]+value=[\'"](%s)[\'"]' % self._VALID_EMBED_URL,
+            r'<input[^>]+id=[\'"]dmcloudUrlEmissionSelect[\'"][^>]+value=[\'"](%s)[\'"]' % cls._VALID_EMBED_URL,
             webpage)
         if mobj:
             return mobj.group(1)

From 080e09557d0c323a9abf5f72e9d307f8bb545be6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 14 Feb 2016 14:01:43 +0600
Subject: [PATCH 4000/6219] [aes] Switch to new-style classes

---
 youtube_dl/aes.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/aes.py b/youtube_dl/aes.py
index 7817adcfd..a01c367de 100644
--- a/youtube_dl/aes.py
+++ b/youtube_dl/aes.py
@@ -161,7 +161,7 @@ def aes_decrypt_text(data, password, key_size_bytes):
     nonce = data[:NONCE_LENGTH_BYTES]
     cipher = data[NONCE_LENGTH_BYTES:]
 
-    class Counter:
+    class Counter(object):
         __value = nonce + [0] * (BLOCK_SIZE_BYTES - NONCE_LENGTH_BYTES)
 
         def next_value(self):

From e76394f36cdff2f701af7c2420b812e7ea045098 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 14 Feb 2016 14:02:12 +0600
Subject: [PATCH 4001/6219] [globo] Switch to new-style classes

---
 youtube_dl/extractor/globo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/globo.py b/youtube_dl/extractor/globo.py
index b241c4868..3de8356f6 100644
--- a/youtube_dl/extractor/globo.py
+++ b/youtube_dl/extractor/globo.py
@@ -65,7 +65,7 @@ class GloboIE(InfoExtractor):
         'only_matching': True,
     }]
 
-    class MD5:
+    class MD5(object):
         HEX_FORMAT_LOWERCASE = 0
         HEX_FORMAT_UPPERCASE = 1
         BASE64_PAD_CHARACTER_DEFAULT_COMPLIANCE = ''

From c78c9cd10dfbb4fc3fd49df4f9c98e9c94c9aae9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 14 Feb 2016 14:13:09 +0600
Subject: [PATCH 4002/6219] [downloader/dash] PEP 8

---
 youtube_dl/downloader/dash.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/downloader/dash.py b/youtube_dl/downloader/dash.py
index 4f3eeb386..8b1b17c6e 100644
--- a/youtube_dl/downloader/dash.py
+++ b/youtube_dl/downloader/dash.py
@@ -35,6 +35,7 @@ class DashSegmentsFD(FragmentFD):
             return '%s%s%s' % (base_url, '' if base_url.endswith('/') else '/', target_url)
 
         segments_filenames = []
+
         def append_url_to_file(target_url, target_filename):
             success = ctx['dl'].download(target_filename, {'url': combine_url(base_url, target_url)})
             if not success:

From d800609c62703e4e6edd2891a8432306462e4db3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 14 Feb 2016 14:25:04 +0600
Subject: [PATCH 4003/6219] [refactor] Do not specify redundant None as second
 argument in dict.get()

---
 youtube_dl/YoutubeDL.py           | 10 +++++-----
 youtube_dl/downloader/common.py   |  2 +-
 youtube_dl/downloader/fragment.py |  2 +-
 youtube_dl/downloader/http.py     |  4 ++--
 youtube_dl/downloader/rtmp.py     | 16 ++++++++--------
 youtube_dl/extractor/common.py    |  8 ++++----
 youtube_dl/extractor/smotri.py    |  4 ++--
 youtube_dl/extractor/vimeo.py     |  6 +++---
 youtube_dl/extractor/youku.py     |  2 +-
 youtube_dl/utils.py               |  2 +-
 10 files changed, 28 insertions(+), 28 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 0072c7d35..d34f77a6d 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -605,12 +605,12 @@ class YoutubeDL(object):
             if rejecttitle:
                 if re.search(rejecttitle, title, re.IGNORECASE):
                     return '"' + title + '" title matched reject pattern "' + rejecttitle + '"'
-        date = info_dict.get('upload_date', None)
+        date = info_dict.get('upload_date')
         if date is not None:
             dateRange = self.params.get('daterange', DateRange())
             if date not in dateRange:
                 return '%s upload date is not in range %s' % (date_from_str(date).isoformat(), dateRange)
-        view_count = info_dict.get('view_count', None)
+        view_count = info_dict.get('view_count')
         if view_count is not None:
             min_views = self.params.get('min_views')
             if min_views is not None and view_count < min_views:
@@ -747,18 +747,18 @@ class YoutubeDL(object):
                 new_result, download=download, extra_info=extra_info)
         elif result_type == 'playlist' or result_type == 'multi_video':
             # We process each entry in the playlist
-            playlist = ie_result.get('title', None) or ie_result.get('id', None)
+            playlist = ie_result.get('title') or ie_result.get('id')
             self.to_screen('[download] Downloading playlist: %s' % playlist)
 
             playlist_results = []
 
             playliststart = self.params.get('playliststart', 1) - 1
-            playlistend = self.params.get('playlistend', None)
+            playlistend = self.params.get('playlistend')
             # For backwards compatibility, interpret -1 as whole list
             if playlistend == -1:
                 playlistend = None
 
-            playlistitems_str = self.params.get('playlist_items', None)
+            playlistitems_str = self.params.get('playlist_items')
             playlistitems = None
             if playlistitems_str is not None:
                 def iter_playlistitems(format):
diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index de815612c..2d5154051 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -157,7 +157,7 @@ class FileDownloader(object):
 
     def slow_down(self, start_time, now, byte_counter):
         """Sleep if the download speed is over the rate limit."""
-        rate_limit = self.params.get('ratelimit', None)
+        rate_limit = self.params.get('ratelimit')
         if rate_limit is None or byte_counter == 0:
             return
         if now is None:
diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
index 8b96eceb9..5bc99492b 100644
--- a/youtube_dl/downloader/fragment.py
+++ b/youtube_dl/downloader/fragment.py
@@ -38,7 +38,7 @@ class FragmentFD(FileDownloader):
                 'continuedl': True,
                 'quiet': True,
                 'noprogress': True,
-                'ratelimit': self.params.get('ratelimit', None),
+                'ratelimit': self.params.get('ratelimit'),
                 'retries': self.params.get('retries', 0),
                 'test': self.params.get('test', False),
             }
diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index 56840e026..7089983ce 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -140,8 +140,8 @@ class HttpFD(FileDownloader):
 
         if data_len is not None:
             data_len = int(data_len) + resume_len
-            min_data_len = self.params.get("min_filesize", None)
-            max_data_len = self.params.get("max_filesize", None)
+            min_data_len = self.params.get("min_filesize")
+            max_data_len = self.params.get("max_filesize")
             if min_data_len is not None and data_len < min_data_len:
                 self.to_screen('\r[download] File is smaller than min-filesize (%s bytes < %s bytes). Aborting.' % (data_len, min_data_len))
                 return False
diff --git a/youtube_dl/downloader/rtmp.py b/youtube_dl/downloader/rtmp.py
index 14d56db47..9de6e70bb 100644
--- a/youtube_dl/downloader/rtmp.py
+++ b/youtube_dl/downloader/rtmp.py
@@ -94,15 +94,15 @@ class RtmpFD(FileDownloader):
             return proc.returncode
 
         url = info_dict['url']
-        player_url = info_dict.get('player_url', None)
-        page_url = info_dict.get('page_url', None)
-        app = info_dict.get('app', None)
-        play_path = info_dict.get('play_path', None)
-        tc_url = info_dict.get('tc_url', None)
-        flash_version = info_dict.get('flash_version', None)
+        player_url = info_dict.get('player_url')
+        page_url = info_dict.get('page_url')
+        app = info_dict.get('app')
+        play_path = info_dict.get('play_path')
+        tc_url = info_dict.get('tc_url')
+        flash_version = info_dict.get('flash_version')
         live = info_dict.get('rtmp_live', False)
-        conn = info_dict.get('rtmp_conn', None)
-        protocol = info_dict.get('rtmp_protocol', None)
+        conn = info_dict.get('rtmp_conn')
+        protocol = info_dict.get('rtmp_protocol')
         real_time = info_dict.get('rtmp_real_time', False)
         no_resume = info_dict.get('no_resume', False)
         continue_dl = self.params.get('continuedl', True)
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 444d412d9..144d8c6b6 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -636,7 +636,7 @@ class InfoExtractor(object):
         downloader_params = self._downloader.params
 
         # Attempt to use provided username and password or .netrc data
-        if downloader_params.get('username', None) is not None:
+        if downloader_params.get('username') is not None:
             username = downloader_params['username']
             password = downloader_params['password']
         elif downloader_params.get('usenetrc', False):
@@ -663,7 +663,7 @@ class InfoExtractor(object):
             return None
         downloader_params = self._downloader.params
 
-        if downloader_params.get('twofactor', None) is not None:
+        if downloader_params.get('twofactor') is not None:
             return downloader_params['twofactor']
 
         return compat_getpass('Type %s and press [Return]: ' % note)
@@ -744,7 +744,7 @@ class InfoExtractor(object):
             'mature': 17,
             'restricted': 19,
         }
-        return RATING_TABLE.get(rating.lower(), None)
+        return RATING_TABLE.get(rating.lower())
 
     def _family_friendly_search(self, html):
         # See http://schema.org/VideoObject
@@ -759,7 +759,7 @@ class InfoExtractor(object):
             '0': 18,
             'false': 18,
         }
-        return RATING_TABLE.get(family_friendly.lower(), None)
+        return RATING_TABLE.get(family_friendly.lower())
 
     def _twitter_search_player(self, html):
         return self._html_search_meta('twitter:player', html,
diff --git a/youtube_dl/extractor/smotri.py b/youtube_dl/extractor/smotri.py
index 30210c8a3..015ef75f3 100644
--- a/youtube_dl/extractor/smotri.py
+++ b/youtube_dl/extractor/smotri.py
@@ -170,7 +170,7 @@ class SmotriIE(InfoExtractor):
             'getvideoinfo': '1',
         }
 
-        video_password = self._downloader.params.get('videopassword', None)
+        video_password = self._downloader.params.get('videopassword')
         if video_password:
             video_form['pass'] = hashlib.md5(video_password.encode('utf-8')).hexdigest()
 
@@ -356,7 +356,7 @@ class SmotriBroadcastIE(InfoExtractor):
 
         url = 'http://smotri.com/broadcast/view/url/?ticket=%s' % ticket
 
-        broadcast_password = self._downloader.params.get('videopassword', None)
+        broadcast_password = self._downloader.params.get('videopassword')
         if broadcast_password:
             url += '&pass=%s' % hashlib.md5(broadcast_password.encode('utf-8')).hexdigest()
 
diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 6a8f9b49d..c7df6b0c5 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -232,7 +232,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
             return mobj.group(1)
 
     def _verify_video_password(self, url, video_id, webpage):
-        password = self._downloader.params.get('videopassword', None)
+        password = self._downloader.params.get('videopassword')
         if password is None:
             raise ExtractorError('This video is protected by a password, use the --video-password option', expected=True)
         token, vuid = self._extract_xsrft_and_vuid(webpage)
@@ -252,7 +252,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
             'Verifying the password', 'Wrong password')
 
     def _verify_player_video_password(self, url, video_id):
-        password = self._downloader.params.get('videopassword', None)
+        password = self._downloader.params.get('videopassword')
         if password is None:
             raise ExtractorError('This video is protected by a password, use the --video-password option')
         data = urlencode_postdata(encode_dict({'password': password}))
@@ -516,7 +516,7 @@ class VimeoChannelIE(VimeoBaseInfoExtractor):
         if not login_form:
             return webpage
 
-        password = self._downloader.params.get('videopassword', None)
+        password = self._downloader.params.get('videopassword')
         if password is None:
             raise ExtractorError('This album is protected by a password, use the --video-password option', expected=True)
         fields = self._hidden_inputs(login_form)
diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index 49687371a..5c1f84a09 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -214,7 +214,7 @@ class YoukuIE(InfoExtractor):
 
             return raw_data['data']
 
-        video_password = self._downloader.params.get('videopassword', None)
+        video_password = self._downloader.params.get('videopassword')
 
         # request basic data
         basic_data_url = "http://play.youku.com/play/get.json?vid=%s&ct=12" % video_id
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index a82a262a0..3e4219b17 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1744,7 +1744,7 @@ def parse_age_limit(s):
     if s is None:
         return None
     m = re.match(r'^(?P<age>\d{1,2})\+?$', s)
-    return int(m.group('age')) if m else US_RATINGS.get(s, None)
+    return int(m.group('age')) if m else US_RATINGS.get(s)
 
 
 def strip_jsonp(code):

From 611c1dd96efc36a788475e14cc4de64d554d28a0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 14 Feb 2016 15:37:17 +0600
Subject: [PATCH 4004/6219] [refactor] Single quotes consistency

---
 youtube_dl/YoutubeDL.py                       |   6 +-
 youtube_dl/__main__.py                        |   2 +-
 youtube_dl/compat.py                          |  14 +--
 youtube_dl/downloader/http.py                 |   4 +-
 youtube_dl/extractor/aenetworks.py            |   2 +-
 youtube_dl/extractor/bbc.py                   |   2 +-
 youtube_dl/extractor/ccc.py                   |   2 +-
 youtube_dl/extractor/ceskatelevize.py         |   8 +-
 youtube_dl/extractor/cnn.py                   |  12 +-
 youtube_dl/extractor/collegerama.py           |   6 +-
 youtube_dl/extractor/comedycentral.py         |   2 +-
 youtube_dl/extractor/common.py                |   8 +-
 youtube_dl/extractor/crunchyroll.py           |  74 ++++++------
 youtube_dl/extractor/drbonanza.py             |  10 +-
 youtube_dl/extractor/eighttracks.py           | 108 +++++++++---------
 youtube_dl/extractor/ellentv.py               |   2 +-
 youtube_dl/extractor/everyonesmixtape.py      |  12 +-
 youtube_dl/extractor/exfm.py                  |   2 +-
 youtube_dl/extractor/fc2.py                   |   2 +-
 youtube_dl/extractor/franceinter.py           |   2 +-
 youtube_dl/extractor/freevideo.py             |   4 +-
 youtube_dl/extractor/hentaistigma.py          |   4 +-
 youtube_dl/extractor/kankan.py                |   2 +-
 youtube_dl/extractor/liveleak.py              |   2 +-
 youtube_dl/extractor/mofosex.py               |   2 +-
 youtube_dl/extractor/myspass.py               |   4 +-
 youtube_dl/extractor/nerdcubed.py             |   4 +-
 youtube_dl/extractor/pornhub.py               |   8 +-
 youtube_dl/extractor/pornovoisines.py         |   2 +-
 youtube_dl/extractor/radiobremen.py           |  12 +-
 youtube_dl/extractor/radiofrance.py           |   6 +-
 youtube_dl/extractor/rbmaradio.py             |  10 +-
 youtube_dl/extractor/reverbnation.py          |  12 +-
 youtube_dl/extractor/ringtv.py                |  14 +--
 youtube_dl/extractor/rte.py                   |   2 +-
 youtube_dl/extractor/rtl2.py                  |   2 +-
 youtube_dl/extractor/screenwavemedia.py       |   2 +-
 youtube_dl/extractor/senateisvp.py            |  62 +++++-----
 youtube_dl/extractor/slutload.py              |   4 +-
 youtube_dl/extractor/snotr.py                 |   2 +-
 youtube_dl/extractor/soundcloud.py            |   2 +-
 youtube_dl/extractor/steam.py                 |  16 +--
 youtube_dl/extractor/tenplay.py               |   8 +-
 youtube_dl/extractor/thesixtyone.py           |  32 +++---
 youtube_dl/extractor/traileraddict.py         |   8 +-
 youtube_dl/extractor/tudou.py                 |   2 +-
 youtube_dl/extractor/vbox7.py                 |   2 +-
 youtube_dl/extractor/videopremium.py          |  10 +-
 youtube_dl/extractor/vimeo.py                 |  18 +--
 youtube_dl/extractor/vine.py                  |   6 +-
 youtube_dl/extractor/worldstarhiphop.py       |  14 +--
 youtube_dl/extractor/youjizz.py               |   4 +-
 youtube_dl/extractor/youku.py                 |   2 +-
 youtube_dl/options.py                         |   2 +-
 youtube_dl/postprocessor/execafterdownload.py |   2 +-
 youtube_dl/postprocessor/metadatafromtitle.py |   2 +-
 youtube_dl/postprocessor/xattrpp.py           |  10 +-
 youtube_dl/update.py                          |   8 +-
 youtube_dl/utils.py                           |  16 +--
 59 files changed, 302 insertions(+), 302 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index d34f77a6d..f4324039c 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -782,7 +782,7 @@ class YoutubeDL(object):
                     entries = ie_entries[playliststart:playlistend]
                 n_entries = len(entries)
                 self.to_screen(
-                    "[%s] playlist %s: Collected %d video ids (downloading %d of them)" %
+                    '[%s] playlist %s: Collected %d video ids (downloading %d of them)' %
                     (ie_result['extractor'], playlist, n_all_entries, n_entries))
             elif isinstance(ie_entries, PagedList):
                 if playlistitems:
@@ -796,7 +796,7 @@ class YoutubeDL(object):
                         playliststart, playlistend)
                 n_entries = len(entries)
                 self.to_screen(
-                    "[%s] playlist %s: Downloading %d videos" %
+                    '[%s] playlist %s: Downloading %d videos' %
                     (ie_result['extractor'], playlist, n_entries))
             else:  # iterable
                 if playlistitems:
@@ -807,7 +807,7 @@ class YoutubeDL(object):
                         ie_entries, playliststart, playlistend))
                 n_entries = len(entries)
                 self.to_screen(
-                    "[%s] playlist %s: Downloading %d videos" %
+                    '[%s] playlist %s: Downloading %d videos' %
                     (ie_result['extractor'], playlist, n_entries))
 
             if self.params.get('playlistreverse', False):
diff --git a/youtube_dl/__main__.py b/youtube_dl/__main__.py
index 42a0f8c6f..138f5fbec 100755
--- a/youtube_dl/__main__.py
+++ b/youtube_dl/__main__.py
@@ -7,7 +7,7 @@ from __future__ import unicode_literals
 
 import sys
 
-if __package__ is None and not hasattr(sys, "frozen"):
+if __package__ is None and not hasattr(sys, 'frozen'):
     # direct call of __main__.py
     import os.path
     path = os.path.realpath(os.path.abspath(__file__))
diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 8ab688001..b497da696 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -181,20 +181,20 @@ except ImportError:  # Python < 3.4
             # parameter := attribute "=" value
             url = req.get_full_url()
 
-            scheme, data = url.split(":", 1)
-            mediatype, data = data.split(",", 1)
+            scheme, data = url.split(':', 1)
+            mediatype, data = data.split(',', 1)
 
             # even base64 encoded data URLs might be quoted so unquote in any case:
             data = compat_urllib_parse_unquote_to_bytes(data)
-            if mediatype.endswith(";base64"):
+            if mediatype.endswith(';base64'):
                 data = binascii.a2b_base64(data)
                 mediatype = mediatype[:-7]
 
             if not mediatype:
-                mediatype = "text/plain;charset=US-ASCII"
+                mediatype = 'text/plain;charset=US-ASCII'
 
             headers = email.message_from_string(
-                "Content-type: %s\nContent-length: %d\n" % (mediatype, len(data)))
+                'Content-type: %s\nContent-length: %d\n' % (mediatype, len(data)))
 
             return compat_urllib_response.addinfourl(io.BytesIO(data), headers, url)
 
@@ -268,7 +268,7 @@ except ImportError:  # Python 2
             nv = name_value.split('=', 1)
             if len(nv) != 2:
                 if strict_parsing:
-                    raise ValueError("bad query field: %r" % (name_value,))
+                    raise ValueError('bad query field: %r' % (name_value,))
                 # Handle case of a control-name with no equal sign
                 if keep_blank_values:
                     nv.append('')
@@ -466,7 +466,7 @@ if sys.version_info < (2, 7):
         if err is not None:
             raise err
         else:
-            raise socket.error("getaddrinfo returns an empty list")
+            raise socket.error('getaddrinfo returns an empty list')
 else:
     compat_socket_create_connection = socket.create_connection
 
diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index 7089983ce..f8b69d186 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -140,8 +140,8 @@ class HttpFD(FileDownloader):
 
         if data_len is not None:
             data_len = int(data_len) + resume_len
-            min_data_len = self.params.get("min_filesize")
-            max_data_len = self.params.get("max_filesize")
+            min_data_len = self.params.get('min_filesize')
+            max_data_len = self.params.get('max_filesize')
             if min_data_len is not None and data_len < min_data_len:
                 self.to_screen('\r[download] File is smaller than min-filesize (%s bytes < %s bytes). Aborting.' % (data_len, min_data_len))
                 return False
diff --git a/youtube_dl/extractor/aenetworks.py b/youtube_dl/extractor/aenetworks.py
index 43d7b0523..6018ae79a 100644
--- a/youtube_dl/extractor/aenetworks.py
+++ b/youtube_dl/extractor/aenetworks.py
@@ -28,7 +28,7 @@ class AENetworksIE(InfoExtractor):
         'info_dict': {
             'id': 'eg47EERs_JsZ',
             'ext': 'mp4',
-            'title': "Winter Is Coming",
+            'title': 'Winter Is Coming',
             'description': 'md5:641f424b7a19d8e24f26dea22cf59d74',
         },
         'params': {
diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 6ddee686c..9d0dfb961 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -86,7 +86,7 @@ class BBCCoUkIE(InfoExtractor):
                 'id': 'b00yng1d',
                 'ext': 'flv',
                 'title': 'The Voice UK: Series 3: Blind Auditions 5',
-                'description': "Emma Willis and Marvin Humes present the fifth set of blind auditions in the singing competition, as the coaches continue to build their teams based on voice alone.",
+                'description': 'Emma Willis and Marvin Humes present the fifth set of blind auditions in the singing competition, as the coaches continue to build their teams based on voice alone.',
                 'duration': 5100,
             },
             'params': {
diff --git a/youtube_dl/extractor/ccc.py b/youtube_dl/extractor/ccc.py
index e94b1e35b..dda2c0959 100644
--- a/youtube_dl/extractor/ccc.py
+++ b/youtube_dl/extractor/ccc.py
@@ -45,7 +45,7 @@ class CCCIE(InfoExtractor):
         title = self._html_search_regex(
             r'(?s)<h1>(.*?)</h1>', webpage, 'title')
         description = self._html_search_regex(
-            r"(?s)<h3>About</h3>(.+?)<h3>",
+            r'(?s)<h3>About</h3>(.+?)<h3>',
             webpage, 'description', fatal=False)
         upload_date = unified_strdate(self._html_search_regex(
             r"(?s)<span[^>]+class='[^']*fa-calendar-o'[^>]*>(.+?)</span>",
diff --git a/youtube_dl/extractor/ceskatelevize.py b/youtube_dl/extractor/ceskatelevize.py
index 6f7b2a70d..b27b4e670 100644
--- a/youtube_dl/extractor/ceskatelevize.py
+++ b/youtube_dl/extractor/ceskatelevize.py
@@ -177,16 +177,16 @@ class CeskaTelevizeIE(InfoExtractor):
             for divider in [1000, 60, 60, 100]:
                 components.append(msec % divider)
                 msec //= divider
-            return "{3:02}:{2:02}:{1:02},{0:03}".format(*components)
+            return '{3:02}:{2:02}:{1:02},{0:03}'.format(*components)
 
         def _fix_subtitle(subtitle):
             for line in subtitle.splitlines():
-                m = re.match(r"^\s*([0-9]+);\s*([0-9]+)\s+([0-9]+)\s*$", line)
+                m = re.match(r'^\s*([0-9]+);\s*([0-9]+)\s+([0-9]+)\s*$', line)
                 if m:
                     yield m.group(1)
                     start, stop = (_msectotimecode(int(t)) for t in m.groups()[1:])
-                    yield "{0} --> {1}".format(start, stop)
+                    yield '{0} --> {1}'.format(start, stop)
                 else:
                     yield line
 
-        return "\r\n".join(_fix_subtitle(subtitles))
+        return '\r\n'.join(_fix_subtitle(subtitles))
diff --git a/youtube_dl/extractor/cnn.py b/youtube_dl/extractor/cnn.py
index 3b1bd4033..53489a14e 100644
--- a/youtube_dl/extractor/cnn.py
+++ b/youtube_dl/extractor/cnn.py
@@ -26,14 +26,14 @@ class CNNIE(InfoExtractor):
             'upload_date': '20130609',
         },
     }, {
-        "url": "http://edition.cnn.com/video/?/video/us/2013/08/21/sot-student-gives-epic-speech.georgia-institute-of-technology&utm_source=feedburner&utm_medium=feed&utm_campaign=Feed%3A+rss%2Fcnn_topstories+%28RSS%3A+Top+Stories%29",
-        "md5": "b5cc60c60a3477d185af8f19a2a26f4e",
-        "info_dict": {
+        'url': 'http://edition.cnn.com/video/?/video/us/2013/08/21/sot-student-gives-epic-speech.georgia-institute-of-technology&utm_source=feedburner&utm_medium=feed&utm_campaign=Feed%3A+rss%2Fcnn_topstories+%28RSS%3A+Top+Stories%29',
+        'md5': 'b5cc60c60a3477d185af8f19a2a26f4e',
+        'info_dict': {
             'id': 'us/2013/08/21/sot-student-gives-epic-speech.georgia-institute-of-technology',
             'ext': 'mp4',
-            "title": "Student's epic speech stuns new freshmen",
-            "description": "A Georgia Tech student welcomes the incoming freshmen with an epic speech backed by music from \"2001: A Space Odyssey.\"",
-            "upload_date": "20130821",
+            'title': "Student's epic speech stuns new freshmen",
+            'description': "A Georgia Tech student welcomes the incoming freshmen with an epic speech backed by music from \"2001: A Space Odyssey.\"",
+            'upload_date': '20130821',
         }
     }, {
         'url': 'http://www.cnn.com/video/data/2.0/video/living/2014/12/22/growing-america-nashville-salemtown-board-episode-1.hln.html',
diff --git a/youtube_dl/extractor/collegerama.py b/youtube_dl/extractor/collegerama.py
index 40667a0f1..f9e84193d 100644
--- a/youtube_dl/extractor/collegerama.py
+++ b/youtube_dl/extractor/collegerama.py
@@ -46,9 +46,9 @@ class CollegeRamaIE(InfoExtractor):
         video_id = self._match_id(url)
 
         player_options_request = {
-            "getPlayerOptionsRequest": {
-                "ResourceId": video_id,
-                "QueryString": "",
+            'getPlayerOptionsRequest': {
+                'ResourceId': video_id,
+                'QueryString': '',
             }
         }
 
diff --git a/youtube_dl/extractor/comedycentral.py b/youtube_dl/extractor/comedycentral.py
index 055c9eec5..5b1b99675 100644
--- a/youtube_dl/extractor/comedycentral.py
+++ b/youtube_dl/extractor/comedycentral.py
@@ -195,7 +195,7 @@ class ComedyCentralShowsIE(MTVServicesInfoExtractor):
             if len(altMovieParams) == 0:
                 raise ExtractorError('unable to find Flash URL in webpage ' + url)
             else:
-                mMovieParams = [("http://media.mtvnservices.com/" + altMovieParams[0], altMovieParams[0])]
+                mMovieParams = [('http://media.mtvnservices.com/' + altMovieParams[0], altMovieParams[0])]
 
         uri = mMovieParams[0][1]
         # Correct cc.com in uri
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 144d8c6b6..f411ea763 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1497,7 +1497,7 @@ class InfoExtractor(object):
     def _live_title(self, name):
         """ Generate the title for a live video """
         now = datetime.datetime.now()
-        now_str = now.strftime("%Y-%m-%d %H:%M")
+        now_str = now.strftime('%Y-%m-%d %H:%M')
         return name + ' ' + now_str
 
     def _int(self, v, name, fatal=False, **kwargs):
@@ -1570,7 +1570,7 @@ class InfoExtractor(object):
         return {}
 
     def _get_subtitles(self, *args, **kwargs):
-        raise NotImplementedError("This method must be implemented by subclasses")
+        raise NotImplementedError('This method must be implemented by subclasses')
 
     @staticmethod
     def _merge_subtitle_items(subtitle_list1, subtitle_list2):
@@ -1596,7 +1596,7 @@ class InfoExtractor(object):
         return {}
 
     def _get_automatic_captions(self, *args, **kwargs):
-        raise NotImplementedError("This method must be implemented by subclasses")
+        raise NotImplementedError('This method must be implemented by subclasses')
 
 
 class SearchInfoExtractor(InfoExtractor):
@@ -1636,7 +1636,7 @@ class SearchInfoExtractor(InfoExtractor):
 
     def _get_n_results(self, query, n):
         """Get a specified number of results for a query"""
-        raise NotImplementedError("This method must be implemented by subclasses")
+        raise NotImplementedError('This method must be implemented by subclasses')
 
     @property
     def SEARCH_KEY(self):
diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 785594df8..c7032ffa2 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -180,40 +180,40 @@ class CrunchyrollIE(CrunchyrollBaseIE):
             return assvalue
 
         output = '[Script Info]\n'
-        output += 'Title: %s\n' % sub_root.attrib["title"]
+        output += 'Title: %s\n' % sub_root.attrib['title']
         output += 'ScriptType: v4.00+\n'
-        output += 'WrapStyle: %s\n' % sub_root.attrib["wrap_style"]
-        output += 'PlayResX: %s\n' % sub_root.attrib["play_res_x"]
-        output += 'PlayResY: %s\n' % sub_root.attrib["play_res_y"]
+        output += 'WrapStyle: %s\n' % sub_root.attrib['wrap_style']
+        output += 'PlayResX: %s\n' % sub_root.attrib['play_res_x']
+        output += 'PlayResY: %s\n' % sub_root.attrib['play_res_y']
         output += """ScaledBorderAndShadow: yes
 
 [V4+ Styles]
 Format: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding
 """
         for style in sub_root.findall('./styles/style'):
-            output += 'Style: ' + style.attrib["name"]
-            output += ',' + style.attrib["font_name"]
-            output += ',' + style.attrib["font_size"]
-            output += ',' + style.attrib["primary_colour"]
-            output += ',' + style.attrib["secondary_colour"]
-            output += ',' + style.attrib["outline_colour"]
-            output += ',' + style.attrib["back_colour"]
-            output += ',' + ass_bool(style.attrib["bold"])
-            output += ',' + ass_bool(style.attrib["italic"])
-            output += ',' + ass_bool(style.attrib["underline"])
-            output += ',' + ass_bool(style.attrib["strikeout"])
-            output += ',' + style.attrib["scale_x"]
-            output += ',' + style.attrib["scale_y"]
-            output += ',' + style.attrib["spacing"]
-            output += ',' + style.attrib["angle"]
-            output += ',' + style.attrib["border_style"]
-            output += ',' + style.attrib["outline"]
-            output += ',' + style.attrib["shadow"]
-            output += ',' + style.attrib["alignment"]
-            output += ',' + style.attrib["margin_l"]
-            output += ',' + style.attrib["margin_r"]
-            output += ',' + style.attrib["margin_v"]
-            output += ',' + style.attrib["encoding"]
+            output += 'Style: ' + style.attrib['name']
+            output += ',' + style.attrib['font_name']
+            output += ',' + style.attrib['font_size']
+            output += ',' + style.attrib['primary_colour']
+            output += ',' + style.attrib['secondary_colour']
+            output += ',' + style.attrib['outline_colour']
+            output += ',' + style.attrib['back_colour']
+            output += ',' + ass_bool(style.attrib['bold'])
+            output += ',' + ass_bool(style.attrib['italic'])
+            output += ',' + ass_bool(style.attrib['underline'])
+            output += ',' + ass_bool(style.attrib['strikeout'])
+            output += ',' + style.attrib['scale_x']
+            output += ',' + style.attrib['scale_y']
+            output += ',' + style.attrib['spacing']
+            output += ',' + style.attrib['angle']
+            output += ',' + style.attrib['border_style']
+            output += ',' + style.attrib['outline']
+            output += ',' + style.attrib['shadow']
+            output += ',' + style.attrib['alignment']
+            output += ',' + style.attrib['margin_l']
+            output += ',' + style.attrib['margin_r']
+            output += ',' + style.attrib['margin_v']
+            output += ',' + style.attrib['encoding']
             output += '\n'
 
         output += """
@@ -222,15 +222,15 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
 """
         for event in sub_root.findall('./events/event'):
             output += 'Dialogue: 0'
-            output += ',' + event.attrib["start"]
-            output += ',' + event.attrib["end"]
-            output += ',' + event.attrib["style"]
-            output += ',' + event.attrib["name"]
-            output += ',' + event.attrib["margin_l"]
-            output += ',' + event.attrib["margin_r"]
-            output += ',' + event.attrib["margin_v"]
-            output += ',' + event.attrib["effect"]
-            output += ',' + event.attrib["text"]
+            output += ',' + event.attrib['start']
+            output += ',' + event.attrib['end']
+            output += ',' + event.attrib['style']
+            output += ',' + event.attrib['name']
+            output += ',' + event.attrib['margin_l']
+            output += ',' + event.attrib['margin_r']
+            output += ',' + event.attrib['margin_v']
+            output += ',' + event.attrib['effect']
+            output += ',' + event.attrib['text']
             output += '\n'
 
         return output
@@ -376,7 +376,7 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
 
 
 class CrunchyrollShowPlaylistIE(CrunchyrollBaseIE):
-    IE_NAME = "crunchyroll:playlist"
+    IE_NAME = 'crunchyroll:playlist'
     _VALID_URL = r'https?://(?:(?P<prefix>www|m)\.)?(?P<url>crunchyroll\.com/(?!(?:news|anime-news|library|forum|launchcalendar|lineup|store|comics|freetrial|login))(?P<id>[\w\-]+))/?(?:\?|$)'
 
     _TESTS = [{
diff --git a/youtube_dl/extractor/drbonanza.py b/youtube_dl/extractor/drbonanza.py
index 8b98b013a..01271f8f0 100644
--- a/youtube_dl/extractor/drbonanza.py
+++ b/youtube_dl/extractor/drbonanza.py
@@ -87,7 +87,7 @@ class DRBonanzaIE(InfoExtractor):
 
         formats = []
         for file in info['Files']:
-            if info['Type'] == "Video":
+            if info['Type'] == 'Video':
                 if file['Type'] in video_types:
                     format = parse_filename_info(file['Location'])
                     format.update({
@@ -101,10 +101,10 @@ class DRBonanzaIE(InfoExtractor):
                         if '/bonanza/' in rtmp_url:
                             format['play_path'] = rtmp_url.split('/bonanza/')[1]
                     formats.append(format)
-                elif file['Type'] == "Thumb":
+                elif file['Type'] == 'Thumb':
                     thumbnail = file['Location']
-            elif info['Type'] == "Audio":
-                if file['Type'] == "Audio":
+            elif info['Type'] == 'Audio':
+                if file['Type'] == 'Audio':
                     format = parse_filename_info(file['Location'])
                     format.update({
                         'url': file['Location'],
@@ -112,7 +112,7 @@ class DRBonanzaIE(InfoExtractor):
                         'vcodec': 'none',
                     })
                     formats.append(format)
-                elif file['Type'] == "Thumb":
+                elif file['Type'] == 'Thumb':
                     thumbnail = file['Location']
 
         description = '%s\n%s\n%s\n' % (
diff --git a/youtube_dl/extractor/eighttracks.py b/youtube_dl/extractor/eighttracks.py
index 0b61ea0ba..9a44f89f3 100644
--- a/youtube_dl/extractor/eighttracks.py
+++ b/youtube_dl/extractor/eighttracks.py
@@ -17,85 +17,85 @@ class EightTracksIE(InfoExtractor):
     IE_NAME = '8tracks'
     _VALID_URL = r'https?://8tracks\.com/(?P<user>[^/]+)/(?P<id>[^/#]+)(?:#.*)?$'
     _TEST = {
-        "name": "EightTracks",
-        "url": "http://8tracks.com/ytdl/youtube-dl-test-tracks-a",
-        "info_dict": {
+        'name': 'EightTracks',
+        'url': 'http://8tracks.com/ytdl/youtube-dl-test-tracks-a',
+        'info_dict': {
             'id': '1336550',
             'display_id': 'youtube-dl-test-tracks-a',
-            "description": "test chars:  \"'/\\ä↭",
-            "title": "youtube-dl test tracks \"'/\\ä↭<>",
+            'description': "test chars:  \"'/\\ä↭",
+            'title': "youtube-dl test tracks \"'/\\ä↭<>",
         },
-        "playlist": [
+        'playlist': [
             {
-                "md5": "96ce57f24389fc8734ce47f4c1abcc55",
-                "info_dict": {
-                    "id": "11885610",
-                    "ext": "m4a",
-                    "title": "youtue-dl project<>\"' - youtube-dl test track 1 \"'/\\\u00e4\u21ad",
-                    "uploader_id": "ytdl"
+                'md5': '96ce57f24389fc8734ce47f4c1abcc55',
+                'info_dict': {
+                    'id': '11885610',
+                    'ext': 'm4a',
+                    'title': "youtue-dl project<>\"' - youtube-dl test track 1 \"'/\\\u00e4\u21ad",
+                    'uploader_id': 'ytdl'
                 }
             },
             {
-                "md5": "4ab26f05c1f7291ea460a3920be8021f",
-                "info_dict": {
-                    "id": "11885608",
-                    "ext": "m4a",
-                    "title": "youtube-dl project - youtube-dl test track 2 \"'/\\\u00e4\u21ad",
-                    "uploader_id": "ytdl"
+                'md5': '4ab26f05c1f7291ea460a3920be8021f',
+                'info_dict': {
+                    'id': '11885608',
+                    'ext': 'm4a',
+                    'title': "youtube-dl project - youtube-dl test track 2 \"'/\\\u00e4\u21ad",
+                    'uploader_id': 'ytdl'
                 }
             },
             {
-                "md5": "d30b5b5f74217410f4689605c35d1fd7",
-                "info_dict": {
-                    "id": "11885679",
-                    "ext": "m4a",
-                    "title": "youtube-dl project as well - youtube-dl test track 3 \"'/\\\u00e4\u21ad",
-                    "uploader_id": "ytdl"
+                'md5': 'd30b5b5f74217410f4689605c35d1fd7',
+                'info_dict': {
+                    'id': '11885679',
+                    'ext': 'm4a',
+                    'title': "youtube-dl project as well - youtube-dl test track 3 \"'/\\\u00e4\u21ad",
+                    'uploader_id': 'ytdl'
                 }
             },
             {
-                "md5": "4eb0a669317cd725f6bbd336a29f923a",
-                "info_dict": {
-                    "id": "11885680",
-                    "ext": "m4a",
-                    "title": "youtube-dl project as well - youtube-dl test track 4 \"'/\\\u00e4\u21ad",
-                    "uploader_id": "ytdl"
+                'md5': '4eb0a669317cd725f6bbd336a29f923a',
+                'info_dict': {
+                    'id': '11885680',
+                    'ext': 'm4a',
+                    'title': "youtube-dl project as well - youtube-dl test track 4 \"'/\\\u00e4\u21ad",
+                    'uploader_id': 'ytdl'
                 }
             },
             {
-                "md5": "1893e872e263a2705558d1d319ad19e8",
-                "info_dict": {
-                    "id": "11885682",
-                    "ext": "m4a",
-                    "title": "PH - youtube-dl test track 5 \"'/\\\u00e4\u21ad",
-                    "uploader_id": "ytdl"
+                'md5': '1893e872e263a2705558d1d319ad19e8',
+                'info_dict': {
+                    'id': '11885682',
+                    'ext': 'm4a',
+                    'title': "PH - youtube-dl test track 5 \"'/\\\u00e4\u21ad",
+                    'uploader_id': 'ytdl'
                 }
             },
             {
-                "md5": "b673c46f47a216ab1741ae8836af5899",
-                "info_dict": {
-                    "id": "11885683",
-                    "ext": "m4a",
-                    "title": "PH - youtube-dl test track 6 \"'/\\\u00e4\u21ad",
-                    "uploader_id": "ytdl"
+                'md5': 'b673c46f47a216ab1741ae8836af5899',
+                'info_dict': {
+                    'id': '11885683',
+                    'ext': 'm4a',
+                    'title': "PH - youtube-dl test track 6 \"'/\\\u00e4\u21ad",
+                    'uploader_id': 'ytdl'
                 }
             },
             {
-                "md5": "1d74534e95df54986da7f5abf7d842b7",
-                "info_dict": {
-                    "id": "11885684",
-                    "ext": "m4a",
-                    "title": "phihag - youtube-dl test track 7 \"'/\\\u00e4\u21ad",
-                    "uploader_id": "ytdl"
+                'md5': '1d74534e95df54986da7f5abf7d842b7',
+                'info_dict': {
+                    'id': '11885684',
+                    'ext': 'm4a',
+                    'title': "phihag - youtube-dl test track 7 \"'/\\\u00e4\u21ad",
+                    'uploader_id': 'ytdl'
                 }
             },
             {
-                "md5": "f081f47af8f6ae782ed131d38b9cd1c0",
-                "info_dict": {
-                    "id": "11885685",
-                    "ext": "m4a",
-                    "title": "phihag - youtube-dl test track 8 \"'/\\\u00e4\u21ad",
-                    "uploader_id": "ytdl"
+                'md5': 'f081f47af8f6ae782ed131d38b9cd1c0',
+                'info_dict': {
+                    'id': '11885685',
+                    'ext': 'm4a',
+                    'title': "phihag - youtube-dl test track 8 \"'/\\\u00e4\u21ad",
+                    'uploader_id': 'ytdl'
                 }
             }
         ]
diff --git a/youtube_dl/extractor/ellentv.py b/youtube_dl/extractor/ellentv.py
index 476cce2d0..4c8190d68 100644
--- a/youtube_dl/extractor/ellentv.py
+++ b/youtube_dl/extractor/ellentv.py
@@ -72,7 +72,7 @@ class EllenTVClipsIE(InfoExtractor):
     def _extract_playlist(self, webpage):
         json_string = self._search_regex(r'playerView.addClips\(\[\{(.*?)\}\]\);', webpage, 'json')
         try:
-            return json.loads("[{" + json_string + "}]")
+            return json.loads('[{' + json_string + '}]')
         except ValueError as ve:
             raise ExtractorError('Failed to download JSON', cause=ve)
 
diff --git a/youtube_dl/extractor/everyonesmixtape.py b/youtube_dl/extractor/everyonesmixtape.py
index 493d38af8..84a9b750e 100644
--- a/youtube_dl/extractor/everyonesmixtape.py
+++ b/youtube_dl/extractor/everyonesmixtape.py
@@ -14,14 +14,14 @@ class EveryonesMixtapeIE(InfoExtractor):
 
     _TESTS = [{
         'url': 'http://everyonesmixtape.com/#/mix/m7m0jJAbMQi/5',
-        "info_dict": {
+        'info_dict': {
             'id': '5bfseWNmlds',
             'ext': 'mp4',
-            "title": "Passion Pit - \"Sleepyhead\" (Official Music Video)",
-            "uploader": "FKR.TV",
-            "uploader_id": "frenchkissrecords",
-            "description": "Music video for \"Sleepyhead\" from Passion Pit's debut EP Chunk Of Change.\nBuy on iTunes: https://itunes.apple.com/us/album/chunk-of-change-ep/id300087641\n\nDirected by The Wilderness.\n\nhttp://www.passionpitmusic.com\nhttp://www.frenchkissrecords.com",
-            "upload_date": "20081015"
+            'title': "Passion Pit - \"Sleepyhead\" (Official Music Video)",
+            'uploader': 'FKR.TV',
+            'uploader_id': 'frenchkissrecords',
+            'description': "Music video for \"Sleepyhead\" from Passion Pit's debut EP Chunk Of Change.\nBuy on iTunes: https://itunes.apple.com/us/album/chunk-of-change-ep/id300087641\n\nDirected by The Wilderness.\n\nhttp://www.passionpitmusic.com\nhttp://www.frenchkissrecords.com",
+            'upload_date': '20081015'
         },
         'params': {
             'skip_download': True,  # This is simply YouTube
diff --git a/youtube_dl/extractor/exfm.py b/youtube_dl/extractor/exfm.py
index 4de02aee9..0c0fe6d65 100644
--- a/youtube_dl/extractor/exfm.py
+++ b/youtube_dl/extractor/exfm.py
@@ -41,7 +41,7 @@ class ExfmIE(InfoExtractor):
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         song_id = mobj.group('id')
-        info_url = "http://ex.fm/api/v3/song/%s" % song_id
+        info_url = 'http://ex.fm/api/v3/song/%s' % song_id
         info = self._download_json(info_url, song_id)['song']
         song_url = info['url']
         if re.match(self._SOUNDCLOUD_URL, song_url) is not None:
diff --git a/youtube_dl/extractor/fc2.py b/youtube_dl/extractor/fc2.py
index 4c81271d3..9580f5c0c 100644
--- a/youtube_dl/extractor/fc2.py
+++ b/youtube_dl/extractor/fc2.py
@@ -87,7 +87,7 @@ class FC2IE(InfoExtractor):
         mimi = hashlib.md5((video_id + '_gGddgPfeaf_gzyr').encode('utf-8')).hexdigest()
 
         info_url = (
-            "http://video.fc2.com/ginfo.php?mimi={1:s}&href={2:s}&v={0:s}&fversion=WIN%2011%2C6%2C602%2C180&from=2&otag=0&upid={0:s}&tk=null&".
+            'http://video.fc2.com/ginfo.php?mimi={1:s}&href={2:s}&v={0:s}&fversion=WIN%2011%2C6%2C602%2C180&from=2&otag=0&upid={0:s}&tk=null&'.
             format(video_id, mimi, compat_urllib_request.quote(refer, safe=b'').replace('.', '%2E')))
 
         info_webpage = self._download_webpage(
diff --git a/youtube_dl/extractor/franceinter.py b/youtube_dl/extractor/franceinter.py
index fdc51f44f..0388ba00c 100644
--- a/youtube_dl/extractor/franceinter.py
+++ b/youtube_dl/extractor/franceinter.py
@@ -10,7 +10,7 @@ class FranceInterIE(InfoExtractor):
     _TEST = {
         'url': 'http://www.franceinter.fr/player/reecouter?play=793962',
         'md5': '4764932e466e6f6c79c317d2e74f6884',
-        "info_dict": {
+        'info_dict': {
             'id': '793962',
             'ext': 'mp3',
             'title': 'L’Histoire dans les jeux vidéo',
diff --git a/youtube_dl/extractor/freevideo.py b/youtube_dl/extractor/freevideo.py
index f755e3c4a..c7bec027b 100644
--- a/youtube_dl/extractor/freevideo.py
+++ b/youtube_dl/extractor/freevideo.py
@@ -12,8 +12,8 @@ class FreeVideoIE(InfoExtractor):
         'info_dict': {
             'id': 'vysukany-zadecek-22033',
             'ext': 'mp4',
-            "title": "vysukany-zadecek-22033",
-            "age_limit": 18,
+            'title': 'vysukany-zadecek-22033',
+            'age_limit': 18,
         },
         'skip': 'Blocked outside .cz',
     }
diff --git a/youtube_dl/extractor/hentaistigma.py b/youtube_dl/extractor/hentaistigma.py
index f5aa73d18..86a93de4d 100644
--- a/youtube_dl/extractor/hentaistigma.py
+++ b/youtube_dl/extractor/hentaistigma.py
@@ -11,8 +11,8 @@ class HentaiStigmaIE(InfoExtractor):
         'info_dict': {
             'id': 'inyouchuu-etsu-bonus',
             'ext': 'mp4',
-            "title": "Inyouchuu Etsu Bonus",
-            "age_limit": 18,
+            'title': 'Inyouchuu Etsu Bonus',
+            'age_limit': 18,
         }
     }
 
diff --git a/youtube_dl/extractor/kankan.py b/youtube_dl/extractor/kankan.py
index 364dc878e..a677ff447 100644
--- a/youtube_dl/extractor/kankan.py
+++ b/youtube_dl/extractor/kankan.py
@@ -28,7 +28,7 @@ class KankanIE(InfoExtractor):
 
         title = self._search_regex(r'(?:G_TITLE=|G_MOVIE_TITLE = )[\'"](.+?)[\'"]', webpage, 'video title')
         surls = re.search(r'surls:\[\'.+?\'\]|lurl:\'.+?\.flv\'', webpage).group(0)
-        gcids = re.findall(r"http://.+?/.+?/(.+?)/", surls)
+        gcids = re.findall(r'http://.+?/.+?/(.+?)/', surls)
         gcid = gcids[-1]
 
         info_url = 'http://p2s.cl.kankan.com/getCdnresource_flv?gcid=%s' % gcid
diff --git a/youtube_dl/extractor/liveleak.py b/youtube_dl/extractor/liveleak.py
index 857edfde2..4684994e1 100644
--- a/youtube_dl/extractor/liveleak.py
+++ b/youtube_dl/extractor/liveleak.py
@@ -47,7 +47,7 @@ class LiveLeakIE(InfoExtractor):
         'info_dict': {
             'id': '801_1409392012',
             'ext': 'mp4',
-            'description': "Happened on 27.7.2014. \r\nAt 0:53 you can see people still swimming at near beach.",
+            'description': 'Happened on 27.7.2014. \r\nAt 0:53 you can see people still swimming at near beach.',
             'uploader': 'bony333',
             'title': 'Crazy Hungarian tourist films close call waterspout in Croatia'
         }
diff --git a/youtube_dl/extractor/mofosex.py b/youtube_dl/extractor/mofosex.py
index f8226cbb2..e47c80119 100644
--- a/youtube_dl/extractor/mofosex.py
+++ b/youtube_dl/extractor/mofosex.py
@@ -38,7 +38,7 @@ class MofosexIE(InfoExtractor):
         path = compat_urllib_parse_urlparse(video_url).path
         extension = os.path.splitext(path)[1][1:]
         format = path.split('/')[5].split('_')[:2]
-        format = "-".join(format)
+        format = '-'.join(format)
 
         age_limit = self._rta_search(webpage)
 
diff --git a/youtube_dl/extractor/myspass.py b/youtube_dl/extractor/myspass.py
index 4557a2b13..f936b92bb 100644
--- a/youtube_dl/extractor/myspass.py
+++ b/youtube_dl/extractor/myspass.py
@@ -18,8 +18,8 @@ class MySpassIE(InfoExtractor):
         'info_dict': {
             'id': '11741',
             'ext': 'mp4',
-            "description": "Wer kann in die Fu\u00dfstapfen von Wolfgang Kubicki treten und die Mehrheit der Zuschauer hinter sich versammeln? Wird vielleicht sogar die Absolute Mehrheit geknackt und der Jackpot von 200.000 Euro mit nach Hause genommen?",
-            "title": "Absolute Mehrheit vom 17.02.2013 - Die Highlights, Teil 2",
+            'description': 'Wer kann in die Fu\u00dfstapfen von Wolfgang Kubicki treten und die Mehrheit der Zuschauer hinter sich versammeln? Wird vielleicht sogar die Absolute Mehrheit geknackt und der Jackpot von 200.000 Euro mit nach Hause genommen?',
+            'title': 'Absolute Mehrheit vom 17.02.2013 - Die Highlights, Teil 2',
         },
     }
 
diff --git a/youtube_dl/extractor/nerdcubed.py b/youtube_dl/extractor/nerdcubed.py
index dff78e486..9feccc672 100644
--- a/youtube_dl/extractor/nerdcubed.py
+++ b/youtube_dl/extractor/nerdcubed.py
@@ -18,14 +18,14 @@ class NerdCubedFeedIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        feed = self._download_json(url, url, "Downloading NerdCubed JSON feed")
+        feed = self._download_json(url, url, 'Downloading NerdCubed JSON feed')
 
         entries = [{
             '_type': 'url',
             'title': feed_entry['title'],
             'uploader': feed_entry['source']['name'] if feed_entry['source'] else None,
             'upload_date': datetime.datetime.strptime(feed_entry['date'], '%Y-%m-%d').strftime('%Y%m%d'),
-            'url': "http://www.youtube.com/watch?v=" + feed_entry['youtube_id'],
+            'url': 'http://www.youtube.com/watch?v=' + feed_entry['youtube_id'],
         } for feed_entry in feed]
 
         return {
diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index 08275687d..91e574dc2 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -27,9 +27,9 @@ class PornHubIE(InfoExtractor):
         'info_dict': {
             'id': '648719015',
             'ext': 'mp4',
-            "uploader": "Babes",
-            "title": "Seductive Indian beauty strips down and fingers her pink pussy",
-            "age_limit": 18
+            'uploader': 'Babes',
+            'title': 'Seductive Indian beauty strips down and fingers her pink pussy',
+            'age_limit': 18
         }
     }, {
         'url': 'http://www.pornhub.com/view_video.php?viewkey=ph557bbb6676d2d',
@@ -95,7 +95,7 @@ class PornHubIE(InfoExtractor):
             path = compat_urllib_parse_urlparse(video_url).path
             extension = os.path.splitext(path)[1][1:]
             format = path.split('/')[5].split('_')[:2]
-            format = "-".join(format)
+            format = '-'.join(format)
 
             m = re.match(r'^(?P<height>[0-9]+)[pP]-(?P<tbr>[0-9]+)[kK]$', format)
             if m is None:
diff --git a/youtube_dl/extractor/pornovoisines.py b/youtube_dl/extractor/pornovoisines.py
index eba4dfbb3..1a53fd71c 100644
--- a/youtube_dl/extractor/pornovoisines.py
+++ b/youtube_dl/extractor/pornovoisines.py
@@ -56,7 +56,7 @@ class PornoVoisinesIE(InfoExtractor):
             r'<h1>(.+?)</h1>', webpage, 'title', flags=re.DOTALL)
         description = self._html_search_regex(
             r'<article id="descriptif">(.+?)</article>',
-            webpage, "description", fatal=False, flags=re.DOTALL)
+            webpage, 'description', fatal=False, flags=re.DOTALL)
 
         thumbnail = self._search_regex(
             r'<div id="mediaspace%s">\s*<img src="/?([^"]+)"' % video_id,
diff --git a/youtube_dl/extractor/radiobremen.py b/youtube_dl/extractor/radiobremen.py
index 0d706312e..0cbb15f08 100644
--- a/youtube_dl/extractor/radiobremen.py
+++ b/youtube_dl/extractor/radiobremen.py
@@ -28,16 +28,16 @@ class RadioBremenIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        meta_url = "http://www.radiobremen.de/apps/php/mediathek/metadaten.php?id=%s" % video_id
+        meta_url = 'http://www.radiobremen.de/apps/php/mediathek/metadaten.php?id=%s' % video_id
         meta_doc = self._download_webpage(
             meta_url, video_id, 'Downloading metadata')
         title = self._html_search_regex(
-            r"<h1.*>(?P<title>.+)</h1>", meta_doc, "title")
+            r'<h1.*>(?P<title>.+)</h1>', meta_doc, 'title')
         description = self._html_search_regex(
-            r"<p>(?P<description>.*)</p>", meta_doc, "description", fatal=False)
+            r'<p>(?P<description>.*)</p>', meta_doc, 'description', fatal=False)
         duration = parse_duration(self._html_search_regex(
-            r"L&auml;nge:</td>\s+<td>(?P<duration>[0-9]+:[0-9]+)</td>",
-            meta_doc, "duration", fatal=False))
+            r'L&auml;nge:</td>\s+<td>(?P<duration>[0-9]+:[0-9]+)</td>',
+            meta_doc, 'duration', fatal=False))
 
         page_doc = self._download_webpage(
             url, video_id, 'Downloading video information')
@@ -51,7 +51,7 @@ class RadioBremenIE(InfoExtractor):
         formats = [{
             'url': video_url,
             'ext': 'mp4',
-            'width': int(mobj.group("width")),
+            'width': int(mobj.group('width')),
         }]
         return {
             'id': video_id,
diff --git a/youtube_dl/extractor/radiofrance.py b/youtube_dl/extractor/radiofrance.py
index 09352ed82..a8afc0014 100644
--- a/youtube_dl/extractor/radiofrance.py
+++ b/youtube_dl/extractor/radiofrance.py
@@ -16,9 +16,9 @@ class RadioFranceIE(InfoExtractor):
         'info_dict': {
             'id': 'one-one',
             'ext': 'ogg',
-            "title": "One to one",
-            "description": "Plutôt que d'imaginer la radio de demain comme technologie ou comme création de contenu, je veux montrer que quelles que soient ses évolutions, j'ai l'intime conviction que la radio continuera d'être un grand média de proximité pour les auditeurs.",
-            "uploader": "Thomas Hercouët",
+            'title': 'One to one',
+            'description': "Plutôt que d'imaginer la radio de demain comme technologie ou comme création de contenu, je veux montrer que quelles que soient ses évolutions, j'ai l'intime conviction que la radio continuera d'être un grand média de proximité pour les auditeurs.",
+            'uploader': 'Thomas Hercouët',
         },
     }
 
diff --git a/youtube_dl/extractor/rbmaradio.py b/youtube_dl/extractor/rbmaradio.py
index 0f8f3ebde..7932af6ef 100644
--- a/youtube_dl/extractor/rbmaradio.py
+++ b/youtube_dl/extractor/rbmaradio.py
@@ -18,11 +18,11 @@ class RBMARadioIE(InfoExtractor):
         'info_dict': {
             'id': 'ford-lopatin-live-at-primavera-sound-2011',
             'ext': 'mp3',
-            "uploader_id": "ford-lopatin",
-            "location": "Spain",
-            "description": "Joel Ford and Daniel ’Oneohtrix Point Never’ Lopatin fly their midified pop extravaganza to Spain. Live at Primavera Sound 2011.",
-            "uploader": "Ford & Lopatin",
-            "title": "Live at Primavera Sound 2011",
+            'uploader_id': 'ford-lopatin',
+            'location': 'Spain',
+            'description': 'Joel Ford and Daniel ’Oneohtrix Point Never’ Lopatin fly their midified pop extravaganza to Spain. Live at Primavera Sound 2011.',
+            'uploader': 'Ford & Lopatin',
+            'title': 'Live at Primavera Sound 2011',
         },
     }
 
diff --git a/youtube_dl/extractor/reverbnation.py b/youtube_dl/extractor/reverbnation.py
index ec7e7df7b..3c6725aeb 100644
--- a/youtube_dl/extractor/reverbnation.py
+++ b/youtube_dl/extractor/reverbnation.py
@@ -12,12 +12,12 @@ class ReverbNationIE(InfoExtractor):
         'url': 'http://www.reverbnation.com/alkilados/song/16965047-mona-lisa',
         'md5': '3da12ebca28c67c111a7f8b262d3f7a7',
         'info_dict': {
-            "id": "16965047",
-            "ext": "mp3",
-            "title": "MONA LISA",
-            "uploader": "ALKILADOS",
-            "uploader_id": "216429",
-            "thumbnail": "re:^https://gp1\.wac\.edgecastcdn\.net/.*?\.jpg$"
+            'id': '16965047',
+            'ext': 'mp3',
+            'title': 'MONA LISA',
+            'uploader': 'ALKILADOS',
+            'uploader_id': '216429',
+            'thumbnail': 're:^https://gp1\.wac\.edgecastcdn\.net/.*?\.jpg$'
         },
     }]
 
diff --git a/youtube_dl/extractor/ringtv.py b/youtube_dl/extractor/ringtv.py
index efa4afeb6..508758075 100644
--- a/youtube_dl/extractor/ringtv.py
+++ b/youtube_dl/extractor/ringtv.py
@@ -8,13 +8,13 @@ from .common import InfoExtractor
 class RingTVIE(InfoExtractor):
     _VALID_URL = r'http://(?:www\.)?ringtv\.craveonline\.com/(?P<type>news|videos/video)/(?P<id>[^/?#]+)'
     _TEST = {
-        "url": "http://ringtv.craveonline.com/news/310833-luis-collazo-says-victor-ortiz-better-not-quit-on-jan-30",
-        "md5": "d25945f5df41cdca2d2587165ac28720",
-        "info_dict": {
+        'url': 'http://ringtv.craveonline.com/news/310833-luis-collazo-says-victor-ortiz-better-not-quit-on-jan-30',
+        'md5': 'd25945f5df41cdca2d2587165ac28720',
+        'info_dict': {
             'id': '857645',
             'ext': 'mp4',
-            "title": 'Video: Luis Collazo says Victor Ortiz "better not quit on Jan. 30" - Ring TV',
-            "description": 'Luis Collazo is excited about his Jan. 30 showdown with fellow former welterweight titleholder Victor Ortiz at Barclays Center in his hometown of Brooklyn. The SuperBowl week fight headlines a Golden Boy Live! card on Fox Sports 1.',
+            'title': 'Video: Luis Collazo says Victor Ortiz "better not quit on Jan. 30" - Ring TV',
+            'description': 'Luis Collazo is excited about his Jan. 30 showdown with fellow former welterweight titleholder Victor Ortiz at Barclays Center in his hometown of Brooklyn. The SuperBowl week fight headlines a Golden Boy Live! card on Fox Sports 1.',
         }
     }
 
@@ -32,8 +32,8 @@ class RingTVIE(InfoExtractor):
         description = self._html_search_regex(
             r'addthis:description="([^"]+)"',
             webpage, 'description', fatal=False)
-        final_url = "http://ringtv.craveonline.springboardplatform.com/storage/ringtv.craveonline.com/conversion/%s.mp4" % video_id
-        thumbnail_url = "http://ringtv.craveonline.springboardplatform.com/storage/ringtv.craveonline.com/snapshots/%s.jpg" % video_id
+        final_url = 'http://ringtv.craveonline.springboardplatform.com/storage/ringtv.craveonline.com/conversion/%s.mp4' % video_id
+        thumbnail_url = 'http://ringtv.craveonline.springboardplatform.com/storage/ringtv.craveonline.com/snapshots/%s.jpg' % video_id
 
         return {
             'id': video_id,
diff --git a/youtube_dl/extractor/rte.py b/youtube_dl/extractor/rte.py
index 73c9788be..042bc8dab 100644
--- a/youtube_dl/extractor/rte.py
+++ b/youtube_dl/extractor/rte.py
@@ -43,7 +43,7 @@ class RteIE(InfoExtractor):
             r'<meta name="thumbnail" content="uri:irus:(.*?)" />', webpage, 'thumbnail')
         thumbnail = 'http://img.rasset.ie/' + thumbnail_id + '.jpg'
 
-        feeds_url = self._html_search_meta("feeds-prefix", webpage, 'feeds url') + video_id
+        feeds_url = self._html_search_meta('feeds-prefix', webpage, 'feeds url') + video_id
         json_string = self._download_json(feeds_url, video_id)
 
         # f4m_url = server + relative_url
diff --git a/youtube_dl/extractor/rtl2.py b/youtube_dl/extractor/rtl2.py
index 25f7faf76..de004671d 100644
--- a/youtube_dl/extractor/rtl2.py
+++ b/youtube_dl/extractor/rtl2.py
@@ -63,7 +63,7 @@ class RTL2IE(InfoExtractor):
         download_url = video_info['streamurl']
         download_url = download_url.replace('\\', '')
         stream_url = 'mp4:' + self._html_search_regex(r'ondemand/(.*)', download_url, 'stream URL')
-        rtmp_conn = ["S:connect", "O:1", "NS:pageUrl:" + url, "NB:fpad:0", "NN:videoFunction:1", "O:0"]
+        rtmp_conn = ['S:connect', 'O:1', 'NS:pageUrl:' + url, 'NB:fpad:0', 'NN:videoFunction:1', 'O:0']
 
         formats = [{
             'url': download_url,
diff --git a/youtube_dl/extractor/screenwavemedia.py b/youtube_dl/extractor/screenwavemedia.py
index e5d62a139..2cf210e0d 100644
--- a/youtube_dl/extractor/screenwavemedia.py
+++ b/youtube_dl/extractor/screenwavemedia.py
@@ -40,7 +40,7 @@ class ScreenwaveMediaIE(InfoExtractor):
                 re.sub(
                     r'(?s)/\*.*?\*/', '',
                     self._search_regex(
-                        r"sources\s*:\s*(\[[^\]]+?\])", playerconfig,
+                        r'sources\s*:\s*(\[[^\]]+?\])', playerconfig,
                         'sources',
                     ).replace(
                         "' + thisObj.options.videoserver + '",
diff --git a/youtube_dl/extractor/senateisvp.py b/youtube_dl/extractor/senateisvp.py
index 990ea0fa8..4d3b58522 100644
--- a/youtube_dl/extractor/senateisvp.py
+++ b/youtube_dl/extractor/senateisvp.py
@@ -15,37 +15,37 @@ from ..compat import (
 
 class SenateISVPIE(InfoExtractor):
     _COMM_MAP = [
-        ["ag", "76440", "http://ag-f.akamaihd.net"],
-        ["aging", "76442", "http://aging-f.akamaihd.net"],
-        ["approps", "76441", "http://approps-f.akamaihd.net"],
-        ["armed", "76445", "http://armed-f.akamaihd.net"],
-        ["banking", "76446", "http://banking-f.akamaihd.net"],
-        ["budget", "76447", "http://budget-f.akamaihd.net"],
-        ["cecc", "76486", "http://srs-f.akamaihd.net"],
-        ["commerce", "80177", "http://commerce1-f.akamaihd.net"],
-        ["csce", "75229", "http://srs-f.akamaihd.net"],
-        ["dpc", "76590", "http://dpc-f.akamaihd.net"],
-        ["energy", "76448", "http://energy-f.akamaihd.net"],
-        ["epw", "76478", "http://epw-f.akamaihd.net"],
-        ["ethics", "76449", "http://ethics-f.akamaihd.net"],
-        ["finance", "76450", "http://finance-f.akamaihd.net"],
-        ["foreign", "76451", "http://foreign-f.akamaihd.net"],
-        ["govtaff", "76453", "http://govtaff-f.akamaihd.net"],
-        ["help", "76452", "http://help-f.akamaihd.net"],
-        ["indian", "76455", "http://indian-f.akamaihd.net"],
-        ["intel", "76456", "http://intel-f.akamaihd.net"],
-        ["intlnarc", "76457", "http://intlnarc-f.akamaihd.net"],
-        ["jccic", "85180", "http://jccic-f.akamaihd.net"],
-        ["jec", "76458", "http://jec-f.akamaihd.net"],
-        ["judiciary", "76459", "http://judiciary-f.akamaihd.net"],
-        ["rpc", "76591", "http://rpc-f.akamaihd.net"],
-        ["rules", "76460", "http://rules-f.akamaihd.net"],
-        ["saa", "76489", "http://srs-f.akamaihd.net"],
-        ["smbiz", "76461", "http://smbiz-f.akamaihd.net"],
-        ["srs", "75229", "http://srs-f.akamaihd.net"],
-        ["uscc", "76487", "http://srs-f.akamaihd.net"],
-        ["vetaff", "76462", "http://vetaff-f.akamaihd.net"],
-        ["arch", "", "http://ussenate-f.akamaihd.net/"]
+        ['ag', '76440', 'http://ag-f.akamaihd.net'],
+        ['aging', '76442', 'http://aging-f.akamaihd.net'],
+        ['approps', '76441', 'http://approps-f.akamaihd.net'],
+        ['armed', '76445', 'http://armed-f.akamaihd.net'],
+        ['banking', '76446', 'http://banking-f.akamaihd.net'],
+        ['budget', '76447', 'http://budget-f.akamaihd.net'],
+        ['cecc', '76486', 'http://srs-f.akamaihd.net'],
+        ['commerce', '80177', 'http://commerce1-f.akamaihd.net'],
+        ['csce', '75229', 'http://srs-f.akamaihd.net'],
+        ['dpc', '76590', 'http://dpc-f.akamaihd.net'],
+        ['energy', '76448', 'http://energy-f.akamaihd.net'],
+        ['epw', '76478', 'http://epw-f.akamaihd.net'],
+        ['ethics', '76449', 'http://ethics-f.akamaihd.net'],
+        ['finance', '76450', 'http://finance-f.akamaihd.net'],
+        ['foreign', '76451', 'http://foreign-f.akamaihd.net'],
+        ['govtaff', '76453', 'http://govtaff-f.akamaihd.net'],
+        ['help', '76452', 'http://help-f.akamaihd.net'],
+        ['indian', '76455', 'http://indian-f.akamaihd.net'],
+        ['intel', '76456', 'http://intel-f.akamaihd.net'],
+        ['intlnarc', '76457', 'http://intlnarc-f.akamaihd.net'],
+        ['jccic', '85180', 'http://jccic-f.akamaihd.net'],
+        ['jec', '76458', 'http://jec-f.akamaihd.net'],
+        ['judiciary', '76459', 'http://judiciary-f.akamaihd.net'],
+        ['rpc', '76591', 'http://rpc-f.akamaihd.net'],
+        ['rules', '76460', 'http://rules-f.akamaihd.net'],
+        ['saa', '76489', 'http://srs-f.akamaihd.net'],
+        ['smbiz', '76461', 'http://smbiz-f.akamaihd.net'],
+        ['srs', '75229', 'http://srs-f.akamaihd.net'],
+        ['uscc', '76487', 'http://srs-f.akamaihd.net'],
+        ['vetaff', '76462', 'http://vetaff-f.akamaihd.net'],
+        ['arch', '', 'http://ussenate-f.akamaihd.net/']
     ]
     _IE_NAME = 'senate.gov'
     _VALID_URL = r'http://www\.senate\.gov/isvp/?\?(?P<qs>.+)'
diff --git a/youtube_dl/extractor/slutload.py b/youtube_dl/extractor/slutload.py
index 3df71304d..7efb29f65 100644
--- a/youtube_dl/extractor/slutload.py
+++ b/youtube_dl/extractor/slutload.py
@@ -13,8 +13,8 @@ class SlutloadIE(InfoExtractor):
         'info_dict': {
             'id': 'TD73btpBqSxc',
             'ext': 'mp4',
-            "title": "virginie baisee en cam",
-            "age_limit": 18,
+            'title': 'virginie baisee en cam',
+            'age_limit': 18,
             'thumbnail': 're:https?://.*?\.jpg'
         }
     }
diff --git a/youtube_dl/extractor/snotr.py b/youtube_dl/extractor/snotr.py
index da3b05a8d..0d1ab07f8 100644
--- a/youtube_dl/extractor/snotr.py
+++ b/youtube_dl/extractor/snotr.py
@@ -43,7 +43,7 @@ class SnotrIE(InfoExtractor):
         title = self._og_search_title(webpage)
 
         description = self._og_search_description(webpage)
-        video_url = "http://cdn.videos.snotr.com/%s.flv" % video_id
+        video_url = 'http://cdn.videos.snotr.com/%s.flv' % video_id
 
         view_count = str_to_int(self._html_search_regex(
             r'<p>\n<strong>Views:</strong>\n([\d,\.]+)</p>',
diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index b2d5487ca..1efb2b980 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -222,7 +222,7 @@ class SoundcloudIE(InfoExtractor):
             full_title = track_id
             token = mobj.group('secret_token')
             if token:
-                info_json_url += "&secret_token=" + token
+                info_json_url += '&secret_token=' + token
         elif mobj.group('player'):
             query = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
             real_url = query['url'][0]
diff --git a/youtube_dl/extractor/steam.py b/youtube_dl/extractor/steam.py
index 183dcb03c..1a831ef6d 100644
--- a/youtube_dl/extractor/steam.py
+++ b/youtube_dl/extractor/steam.py
@@ -22,23 +22,23 @@ class SteamIE(InfoExtractor):
     _VIDEO_PAGE_TEMPLATE = 'http://store.steampowered.com/video/%s/'
     _AGECHECK_TEMPLATE = 'http://store.steampowered.com/agecheck/video/%s/?snr=1_agecheck_agecheck__age-gate&ageDay=1&ageMonth=January&ageYear=1970'
     _TESTS = [{
-        "url": "http://store.steampowered.com/video/105600/",
-        "playlist": [
+        'url': 'http://store.steampowered.com/video/105600/',
+        'playlist': [
             {
-                "md5": "f870007cee7065d7c76b88f0a45ecc07",
-                "info_dict": {
+                'md5': 'f870007cee7065d7c76b88f0a45ecc07',
+                'info_dict': {
                     'id': '81300',
                     'ext': 'flv',
-                    "title": "Terraria 1.1 Trailer",
+                    'title': 'Terraria 1.1 Trailer',
                     'playlist_index': 1,
                 }
             },
             {
-                "md5": "61aaf31a5c5c3041afb58fb83cbb5751",
-                "info_dict": {
+                'md5': '61aaf31a5c5c3041afb58fb83cbb5751',
+                'info_dict': {
                     'id': '80859',
                     'ext': 'flv',
-                    "title": "Terraria Trailer",
+                    'title': 'Terraria Trailer',
                     'playlist_index': 2,
                 }
             }
diff --git a/youtube_dl/extractor/tenplay.py b/youtube_dl/extractor/tenplay.py
index f6694149b..02a31a609 100644
--- a/youtube_dl/extractor/tenplay.py
+++ b/youtube_dl/extractor/tenplay.py
@@ -27,10 +27,10 @@ class TenPlayIE(InfoExtractor):
     }
 
     _video_fields = [
-        "id", "name", "shortDescription", "longDescription", "creationDate",
-        "publishedDate", "lastModifiedDate", "customFields", "videoStillURL",
-        "thumbnailURL", "referenceId", "length", "playsTotal",
-        "playsTrailingWeek", "renditions", "captioning", "startDate", "endDate"]
+        'id', 'name', 'shortDescription', 'longDescription', 'creationDate',
+        'publishedDate', 'lastModifiedDate', 'customFields', 'videoStillURL',
+        'thumbnailURL', 'referenceId', 'length', 'playsTotal',
+        'playsTrailingWeek', 'renditions', 'captioning', 'startDate', 'endDate']
 
     def _real_extract(self, url):
         webpage = self._download_webpage(url, url)
diff --git a/youtube_dl/extractor/thesixtyone.py b/youtube_dl/extractor/thesixtyone.py
index 5d09eb9a8..d8b1fd281 100644
--- a/youtube_dl/extractor/thesixtyone.py
+++ b/youtube_dl/extractor/thesixtyone.py
@@ -48,22 +48,22 @@ class TheSixtyOneIE(InfoExtractor):
     ]
 
     _DECODE_MAP = {
-        "x": "a",
-        "m": "b",
-        "w": "c",
-        "q": "d",
-        "n": "e",
-        "p": "f",
-        "a": "0",
-        "h": "1",
-        "e": "2",
-        "u": "3",
-        "s": "4",
-        "i": "5",
-        "o": "6",
-        "y": "7",
-        "r": "8",
-        "c": "9"
+        'x': 'a',
+        'm': 'b',
+        'w': 'c',
+        'q': 'd',
+        'n': 'e',
+        'p': 'f',
+        'a': '0',
+        'h': '1',
+        'e': '2',
+        'u': '3',
+        's': '4',
+        'i': '5',
+        'o': '6',
+        'y': '7',
+        'r': '8',
+        'c': '9'
     }
 
     def _real_extract(self, url):
diff --git a/youtube_dl/extractor/traileraddict.py b/youtube_dl/extractor/traileraddict.py
index 1c53a3fd0..0e01b15fc 100644
--- a/youtube_dl/extractor/traileraddict.py
+++ b/youtube_dl/extractor/traileraddict.py
@@ -38,12 +38,12 @@ class TrailerAddictIE(InfoExtractor):
 
         # Presence of (no)watchplus function indicates HD quality is available
         if re.search(r'function (no)?watchplus()', webpage):
-            fvar = "fvarhd"
+            fvar = 'fvarhd'
         else:
-            fvar = "fvar"
+            fvar = 'fvar'
 
-        info_url = "http://www.traileraddict.com/%s.php?tid=%s" % (fvar, str(video_id))
-        info_webpage = self._download_webpage(info_url, video_id, "Downloading the info webpage")
+        info_url = 'http://www.traileraddict.com/%s.php?tid=%s' % (fvar, str(video_id))
+        info_webpage = self._download_webpage(info_url, video_id, 'Downloading the info webpage')
 
         final_url = self._search_regex(r'&fileurl=(.+)',
                                        info_webpage, 'Download url').replace('%3F', '?')
diff --git a/youtube_dl/extractor/tudou.py b/youtube_dl/extractor/tudou.py
index da3cd76f7..f56b66d06 100644
--- a/youtube_dl/extractor/tudou.py
+++ b/youtube_dl/extractor/tudou.py
@@ -49,7 +49,7 @@ class TudouIE(InfoExtractor):
         info_url = 'http://v2.tudou.com/f?id=' + compat_str(video_id)
         if quality:
             info_url += '&hd' + quality
-        xml_data = self._download_xml(info_url, video_id, "Opening the info XML page")
+        xml_data = self._download_xml(info_url, video_id, 'Opening the info XML page')
         final_url = xml_data.text
         return final_url
 
diff --git a/youtube_dl/extractor/vbox7.py b/youtube_dl/extractor/vbox7.py
index 1e740fbe6..3794bcded 100644
--- a/youtube_dl/extractor/vbox7.py
+++ b/youtube_dl/extractor/vbox7.py
@@ -47,7 +47,7 @@ class Vbox7IE(InfoExtractor):
         title = self._html_search_regex(r'<title>(.*)</title>',
                                         webpage, 'title').split('/')[0].strip()
 
-        info_url = "http://vbox7.com/play/magare.do"
+        info_url = 'http://vbox7.com/play/magare.do'
         data = compat_urllib_parse.urlencode({'as3': '1', 'vid': video_id})
         info_request = sanitized_Request(info_url, data)
         info_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
diff --git a/youtube_dl/extractor/videopremium.py b/youtube_dl/extractor/videopremium.py
index 3176e3b9d..5de8273c3 100644
--- a/youtube_dl/extractor/videopremium.py
+++ b/youtube_dl/extractor/videopremium.py
@@ -26,7 +26,7 @@ class VideoPremiumIE(InfoExtractor):
         webpage_url = 'http://videopremium.tv/' + video_id
         webpage = self._download_webpage(webpage_url, video_id)
 
-        if re.match(r"^<html><head><script[^>]*>window.location\s*=", webpage):
+        if re.match(r'^<html><head><script[^>]*>window.location\s*=', webpage):
             # Download again, we need a cookie
             webpage = self._download_webpage(
                 webpage_url, video_id,
@@ -37,10 +37,10 @@ class VideoPremiumIE(InfoExtractor):
 
         return {
             'id': video_id,
-            'url': "rtmp://e%d.md.iplay.md/play" % random.randint(1, 16),
-            'play_path': "mp4:%s.f4v" % video_id,
-            'page_url': "http://videopremium.tv/" + video_id,
-            'player_url': "http://videopremium.tv/uplayer/uppod.swf",
+            'url': 'rtmp://e%d.md.iplay.md/play' % random.randint(1, 16),
+            'play_path': 'mp4:%s.f4v' % video_id,
+            'page_url': 'http://videopremium.tv/' + video_id,
+            'player_url': 'http://videopremium.tv/uplayer/uppod.swf',
             'ext': 'f4v',
             'title': video_title,
         }
diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index c7df6b0c5..3049dffb6 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -368,16 +368,16 @@ class VimeoIE(VimeoBaseInfoExtractor):
                     {'force_feature_id': True}), 'Vimeo')
 
         # Extract title
-        video_title = config["video"]["title"]
+        video_title = config['video']['title']
 
         # Extract uploader and uploader_id
-        video_uploader = config["video"]["owner"]["name"]
-        video_uploader_id = config["video"]["owner"]["url"].split('/')[-1] if config["video"]["owner"]["url"] else None
+        video_uploader = config['video']['owner']['name']
+        video_uploader_id = config['video']['owner']['url'].split('/')[-1] if config['video']['owner']['url'] else None
 
         # Extract video thumbnail
-        video_thumbnail = config["video"].get("thumbnail")
+        video_thumbnail = config['video'].get('thumbnail')
         if video_thumbnail is None:
-            video_thumbs = config["video"].get("thumbs")
+            video_thumbs = config['video'].get('thumbs')
             if video_thumbs and isinstance(video_thumbs, dict):
                 _, video_thumbnail = sorted((int(width if width.isdigit() else 0), t_url) for (width, t_url) in video_thumbs.items())[-1]
 
@@ -401,7 +401,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
             self._downloader.report_warning('Cannot find video description')
 
         # Extract video duration
-        video_duration = int_or_none(config["video"].get("duration"))
+        video_duration = int_or_none(config['video'].get('duration'))
 
         # Extract upload date
         video_upload_date = None
@@ -703,10 +703,10 @@ class VimeoLikesIE(InfoExtractor):
     _TEST = {
         'url': 'https://vimeo.com/user755559/likes/',
         'playlist_mincount': 293,
-        "info_dict": {
+        'info_dict': {
             'id': 'user755559_likes',
-            "description": "See all the videos urza likes",
-            "title": 'Videos urza likes',
+            'description': 'See all the videos urza likes',
+            'title': 'Videos urza likes',
         },
     }
 
diff --git a/youtube_dl/extractor/vine.py b/youtube_dl/extractor/vine.py
index cb2a4b0b5..a6a6cc479 100644
--- a/youtube_dl/extractor/vine.py
+++ b/youtube_dl/extractor/vine.py
@@ -119,7 +119,7 @@ class VineIE(InfoExtractor):
 class VineUserIE(InfoExtractor):
     IE_NAME = 'vine:user'
     _VALID_URL = r'(?:https?://)?vine\.co/(?P<u>u/)?(?P<user>[^/]+)/?(\?.*)?$'
-    _VINE_BASE_URL = "https://vine.co/"
+    _VINE_BASE_URL = 'https://vine.co/'
     _TESTS = [
         {
             'url': 'https://vine.co/Visa',
@@ -139,7 +139,7 @@ class VineUserIE(InfoExtractor):
         user = mobj.group('user')
         u = mobj.group('u')
 
-        profile_url = "%sapi/users/profiles/%s%s" % (
+        profile_url = '%sapi/users/profiles/%s%s' % (
             self._VINE_BASE_URL, 'vanity/' if not u else '', user)
         profile_data = self._download_json(
             profile_url, user, note='Downloading user profile data')
@@ -147,7 +147,7 @@ class VineUserIE(InfoExtractor):
         user_id = profile_data['data']['userId']
         timeline_data = []
         for pagenum in itertools.count(1):
-            timeline_url = "%sapi/timelines/users/%s?page=%s&size=100" % (
+            timeline_url = '%sapi/timelines/users/%s?page=%s&size=100' % (
                 self._VINE_BASE_URL, user_id, pagenum)
             timeline_page = self._download_json(
                 timeline_url, user, note='Downloading page %d' % pagenum)
diff --git a/youtube_dl/extractor/worldstarhiphop.py b/youtube_dl/extractor/worldstarhiphop.py
index a3ea26feb..09415b589 100644
--- a/youtube_dl/extractor/worldstarhiphop.py
+++ b/youtube_dl/extractor/worldstarhiphop.py
@@ -8,12 +8,12 @@ from .common import InfoExtractor
 class WorldStarHipHopIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www|m)\.worldstar(?:candy|hiphop)\.com/(?:videos|android)/video\.php\?v=(?P<id>.*)'
     _TESTS = [{
-        "url": "http://www.worldstarhiphop.com/videos/video.php?v=wshh6a7q1ny0G34ZwuIO",
-        "md5": "9d04de741161603bf7071bbf4e883186",
-        "info_dict": {
-            "id": "wshh6a7q1ny0G34ZwuIO",
-            "ext": "mp4",
-            "title": "KO Of The Week: MMA Fighter Gets Knocked Out By Swift Head Kick!"
+        'url': 'http://www.worldstarhiphop.com/videos/video.php?v=wshh6a7q1ny0G34ZwuIO',
+        'md5': '9d04de741161603bf7071bbf4e883186',
+        'info_dict': {
+            'id': 'wshh6a7q1ny0G34ZwuIO',
+            'ext': 'mp4',
+            'title': 'KO Of The Week: MMA Fighter Gets Knocked Out By Swift Head Kick!'
         }
     }, {
         'url': 'http://m.worldstarhiphop.com/android/video.php?v=wshh6a7q1ny0G34ZwuIO',
@@ -21,7 +21,7 @@ class WorldStarHipHopIE(InfoExtractor):
         'info_dict': {
             'id': 'wshh6a7q1ny0G34ZwuIO',
             'ext': 'mp4',
-            "title": "KO Of The Week: MMA Fighter Gets Knocked Out By Swift Head Kick!"
+            'title': 'KO Of The Week: MMA Fighter Gets Knocked Out By Swift Head Kick!'
         }
     }]
 
diff --git a/youtube_dl/extractor/youjizz.py b/youtube_dl/extractor/youjizz.py
index c642075dc..4150b28da 100644
--- a/youtube_dl/extractor/youjizz.py
+++ b/youtube_dl/extractor/youjizz.py
@@ -16,8 +16,8 @@ class YouJizzIE(InfoExtractor):
         'info_dict': {
             'id': '2189178',
             'ext': 'flv',
-            "title": "Zeichentrick 1",
-            "age_limit": 18,
+            'title': 'Zeichentrick 1',
+            'age_limit': 18,
         }
     }
 
diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index 5c1f84a09..900eb2aba 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -217,7 +217,7 @@ class YoukuIE(InfoExtractor):
         video_password = self._downloader.params.get('videopassword')
 
         # request basic data
-        basic_data_url = "http://play.youku.com/play/get.json?vid=%s&ct=12" % video_id
+        basic_data_url = 'http://play.youku.com/play/get.json?vid=%s&ct=12' % video_id
         if video_password:
             basic_data_url += '&pwd=%s' % video_password
 
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 2137dfb3f..3afa8bb6f 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -85,7 +85,7 @@ def parseOpts(overrideArguments=None):
         if option.takes_value():
             opts.append(' %s' % option.metavar)
 
-        return "".join(opts)
+        return ''.join(opts)
 
     def _comma_separated_values_options_callback(option, opt_str, value, parser):
         setattr(parser.values, option.dest, value.split(','))
diff --git a/youtube_dl/postprocessor/execafterdownload.py b/youtube_dl/postprocessor/execafterdownload.py
index 13794b7ba..74f66d669 100644
--- a/youtube_dl/postprocessor/execafterdownload.py
+++ b/youtube_dl/postprocessor/execafterdownload.py
@@ -19,7 +19,7 @@ class ExecAfterDownloadPP(PostProcessor):
 
         cmd = cmd.replace('{}', shlex_quote(information['filepath']))
 
-        self._downloader.to_screen("[exec] Executing command: %s" % cmd)
+        self._downloader.to_screen('[exec] Executing command: %s' % cmd)
         retCode = subprocess.call(cmd, shell=True)
         if retCode != 0:
             raise PostProcessingError(
diff --git a/youtube_dl/postprocessor/metadatafromtitle.py b/youtube_dl/postprocessor/metadatafromtitle.py
index a56077f20..42377fa0f 100644
--- a/youtube_dl/postprocessor/metadatafromtitle.py
+++ b/youtube_dl/postprocessor/metadatafromtitle.py
@@ -24,7 +24,7 @@ class MetadataFromTitlePP(PostProcessor):
            '(?P<title>.+)\ \-\ (?P<artist>.+)'
         """
         lastpos = 0
-        regex = ""
+        regex = ''
         # replace %(..)s with regex group and escape other string parts
         for match in re.finditer(r'%\((\w+)\)s', fmt):
             regex += re.escape(fmt[lastpos:match.start()])
diff --git a/youtube_dl/postprocessor/xattrpp.py b/youtube_dl/postprocessor/xattrpp.py
index 7d88e1308..480d48d05 100644
--- a/youtube_dl/postprocessor/xattrpp.py
+++ b/youtube_dl/postprocessor/xattrpp.py
@@ -80,15 +80,15 @@ class XAttrMetadataPP(PostProcessor):
                     assert ':' not in key
                     assert os.path.exists(path)
 
-                    ads_fn = path + ":" + key
+                    ads_fn = path + ':' + key
                     try:
-                        with open(ads_fn, "wb") as f:
+                        with open(ads_fn, 'wb') as f:
                             f.write(value)
                     except EnvironmentError as e:
                         raise XAttrMetadataError(e.errno, e.strerror)
             else:
-                user_has_setfattr = check_executable("setfattr", ['--version'])
-                user_has_xattr = check_executable("xattr", ['-h'])
+                user_has_setfattr = check_executable('setfattr', ['--version'])
+                user_has_xattr = check_executable('xattr', ['-h'])
 
                 if user_has_setfattr or user_has_xattr:
 
@@ -150,7 +150,7 @@ class XAttrMetadataPP(PostProcessor):
                 value = info.get(infoname)
 
                 if value:
-                    if infoname == "upload_date":
+                    if infoname == 'upload_date':
                         value = hyphenate_date(value)
 
                     byte_value = value.encode('utf-8')
diff --git a/youtube_dl/update.py b/youtube_dl/update.py
index e4a1aaa64..676ebe1c4 100644
--- a/youtube_dl/update.py
+++ b/youtube_dl/update.py
@@ -31,12 +31,12 @@ def rsa_verify(message, signature, key):
 def update_self(to_screen, verbose, opener):
     """Update the program file with the latest version from the repository"""
 
-    UPDATE_URL = "https://rg3.github.io/youtube-dl/update/"
+    UPDATE_URL = 'https://rg3.github.io/youtube-dl/update/'
     VERSION_URL = UPDATE_URL + 'LATEST_VERSION'
     JSON_URL = UPDATE_URL + 'versions.json'
     UPDATES_RSA_KEY = (0x9d60ee4d8f805312fdb15a62f87b95bd66177b91df176765d13514a0f1754bcd2057295c5b6f1d35daa6742c3ffc9a82d3e118861c207995a8031e151d863c9927e304576bc80692bc8e094896fcf11b66f3e29e04e3a71e9a11558558acea1840aec37fc396fb6b65dc81a1c4144e03bd1c011de62e3f1357b327d08426fe93, 65537)
 
-    if not isinstance(globals().get('__loader__'), zipimporter) and not hasattr(sys, "frozen"):
+    if not isinstance(globals().get('__loader__'), zipimporter) and not hasattr(sys, 'frozen'):
         to_screen('It looks like you installed youtube-dl with a package manager, pip, setup.py or a tarball. Please use that to update.')
         return
 
@@ -85,7 +85,7 @@ def update_self(to_screen, verbose, opener):
 
     filename = sys.argv[0]
     # Py2EXE: Filename could be different
-    if hasattr(sys, "frozen") and not os.path.isfile(filename):
+    if hasattr(sys, 'frozen') and not os.path.isfile(filename):
         if os.path.isfile(filename + '.exe'):
             filename += '.exe'
 
@@ -94,7 +94,7 @@ def update_self(to_screen, verbose, opener):
         return
 
     # Py2EXE
-    if hasattr(sys, "frozen"):
+    if hasattr(sys, 'frozen'):
         exe = os.path.abspath(filename)
         directory = os.path.dirname(exe)
         if not os.access(directory, os.W_OK):
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 3e4219b17..672ce05ea 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -248,7 +248,7 @@ def xpath_attr(node, xpath, key, name=None, fatal=False, default=NO_DEFAULT):
 
 def get_element_by_id(id, html):
     """Return the content of the tag with the specified ID in the passed HTML document"""
-    return get_element_by_attribute("id", id, html)
+    return get_element_by_attribute('id', id, html)
 
 
 def get_element_by_attribute(attribute, value, html):
@@ -994,7 +994,7 @@ def date_from_str(date_str):
         unit += 's'
         delta = datetime.timedelta(**{unit: time})
         return today + delta
-    return datetime.datetime.strptime(date_str, "%Y%m%d").date()
+    return datetime.datetime.strptime(date_str, '%Y%m%d').date()
 
 
 def hyphenate_date(date_str):
@@ -1074,22 +1074,22 @@ def _windows_write_string(s, out):
 
     GetStdHandle = ctypes.WINFUNCTYPE(
         ctypes.wintypes.HANDLE, ctypes.wintypes.DWORD)(
-        (b"GetStdHandle", ctypes.windll.kernel32))
+        (b'GetStdHandle', ctypes.windll.kernel32))
     h = GetStdHandle(WIN_OUTPUT_IDS[fileno])
 
     WriteConsoleW = ctypes.WINFUNCTYPE(
         ctypes.wintypes.BOOL, ctypes.wintypes.HANDLE, ctypes.wintypes.LPWSTR,
         ctypes.wintypes.DWORD, ctypes.POINTER(ctypes.wintypes.DWORD),
-        ctypes.wintypes.LPVOID)((b"WriteConsoleW", ctypes.windll.kernel32))
+        ctypes.wintypes.LPVOID)((b'WriteConsoleW', ctypes.windll.kernel32))
     written = ctypes.wintypes.DWORD(0)
 
-    GetFileType = ctypes.WINFUNCTYPE(ctypes.wintypes.DWORD, ctypes.wintypes.DWORD)((b"GetFileType", ctypes.windll.kernel32))
+    GetFileType = ctypes.WINFUNCTYPE(ctypes.wintypes.DWORD, ctypes.wintypes.DWORD)((b'GetFileType', ctypes.windll.kernel32))
     FILE_TYPE_CHAR = 0x0002
     FILE_TYPE_REMOTE = 0x8000
     GetConsoleMode = ctypes.WINFUNCTYPE(
         ctypes.wintypes.BOOL, ctypes.wintypes.HANDLE,
         ctypes.POINTER(ctypes.wintypes.DWORD))(
-        (b"GetConsoleMode", ctypes.windll.kernel32))
+        (b'GetConsoleMode', ctypes.windll.kernel32))
     INVALID_HANDLE_VALUE = ctypes.wintypes.DWORD(-1).value
 
     def not_a_console(handle):
@@ -1387,7 +1387,7 @@ def fix_xml_ampersands(xml_str):
 def setproctitle(title):
     assert isinstance(title, compat_str)
     try:
-        libc = ctypes.cdll.LoadLibrary("libc.so.6")
+        libc = ctypes.cdll.LoadLibrary('libc.so.6')
     except OSError:
         return
     title_bytes = title.encode('utf-8')
@@ -1427,7 +1427,7 @@ def url_basename(url):
 
 class HEADRequest(compat_urllib_request.Request):
     def get_method(self):
-        return "HEAD"
+        return 'HEAD'
 
 
 def int_or_none(v, scale=1, default=None, get_attr=None, invscale=1):

From 5e19323ed91a2ac3014c8f803fa65d2774a6a4f4 Mon Sep 17 00:00:00 2001
From: Jens Wille <jens.wille@gmail.com>
Date: Tue, 9 Feb 2016 16:25:30 +0100
Subject: [PATCH 4005/6219] [laola1tv] Fixes for changed site layout.

* Fixed valid URLs (w/ tests).
* Fixed iframe URL extraction.
* Fixed token URL extraction.
* Fixed variable extraction.
* Fixed uploader spelling.
* Added upload_date to result dictionary.
---
 youtube_dl/extractor/laola1tv.py | 99 ++++++++++++++++++++------------
 1 file changed, 63 insertions(+), 36 deletions(-)

diff --git a/youtube_dl/extractor/laola1tv.py b/youtube_dl/extractor/laola1tv.py
index b459559b0..c1f248b6b 100644
--- a/youtube_dl/extractor/laola1tv.py
+++ b/youtube_dl/extractor/laola1tv.py
@@ -1,86 +1,113 @@
 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
-import random
 import re
 
 from .common import InfoExtractor
 from ..utils import (
+    compat_urlparse,
     ExtractorError,
+    sanitized_Request,
+    xpath_element,
     xpath_text,
+    unified_strdate,
+    urlencode_postdata,
 )
 
 
 class Laola1TvIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?laola1\.tv/(?P<lang>[a-z]+)-(?P<portal>[a-z]+)/.*?/(?P<id>[0-9]+)\.html'
-    _TEST = {
+    _VALID_URL = r'https?://(?:www\.)?laola1\.tv/(?P<lang>[a-z]+)-(?P<portal>[a-z]+)/.*?/(?P<slug>[\w-]+)'
+    _TESTS = [{
         'url': 'http://www.laola1.tv/de-de/video/straubing-tigers-koelner-haie/227883.html',
         'info_dict': {
-            'id': '227883',
-            'ext': 'mp4',
-            'title': 'Straubing Tigers - Kölner Haie',
             'categories': ['Eishockey'],
+            'ext': 'flv',
+            'id': '227883',
             'is_live': False,
+            'title': 'Straubing Tigers - Kölner Haie',
+            'upload_date': '20140912',
         },
         'params': {
             'skip_download': True,
         }
-    }
+    }, {
+        'url': 'http://www.laola1.tv/de-de/video/straubing-tigers-koelner-haie',
+        'info_dict': {
+            'categories': ['Eishockey'],
+            'ext': 'flv',
+            'id': '464602',
+            'is_live': False,
+            'title': 'Straubing Tigers - Kölner Haie',
+            'upload_date': '20160129',
+        },
+        'params': {
+            'skip_download': True,
+        }
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
         lang = mobj.group('lang')
         portal = mobj.group('portal')
 
-        webpage = self._download_webpage(url, video_id)
+        webpage = self._download_webpage(url, mobj.group('slug'))
         iframe_url = self._search_regex(
-            r'<iframe[^>]*?class="main_tv_player"[^>]*?src="([^"]+)"',
+            r'<iframe[^>]*?id="videoplayer"[^>]*?src="([^"]+)"',
             webpage, 'iframe URL')
 
-        iframe = self._download_webpage(
-            iframe_url, video_id, note='Downloading iframe')
-        flashvars_m = re.findall(
-            r'flashvars\.([_a-zA-Z0-9]+)\s*=\s*"([^"]*)";', iframe)
-        flashvars = dict((m[0], m[1]) for m in flashvars_m)
+        video_id = self._search_regex(
+            r'videoid=(\d+)', iframe_url, 'video ID')
+
+        iframe = self._download_webpage(compat_urlparse.urljoin(
+            url, iframe_url), video_id, note='Downloading iframe')
 
         partner_id = self._search_regex(
-            r'partnerid\s*:\s*"([^"]+)"', iframe, 'partner id')
+            r'partnerid\s*:\s*"([^"]+)"', iframe, 'partner ID')
 
         xml_url = ('http://www.laola1.tv/server/hd_video.php?' +
                    'play=%s&partner=%s&portal=%s&v5ident=&lang=%s' % (
                        video_id, partner_id, portal, lang))
         hd_doc = self._download_xml(xml_url, video_id)
 
-        title = xpath_text(hd_doc, './/video/title', fatal=True)
-        flash_url = xpath_text(hd_doc, './/video/url', fatal=True)
-        uploader = xpath_text(hd_doc, './/video/meta_organistation')
-        is_live = xpath_text(hd_doc, './/video/islive') == 'true'
+        _v = lambda x, **k: xpath_text(hd_doc, './/video/' + x, **k)
+        title = _v('title', fatal=True)
 
-        categories = xpath_text(hd_doc, './/video/meta_sports')
+        categories = _v('meta_sports')
         if categories:
             categories = categories.split(',')
 
-        ident = random.randint(10000000, 99999999)
-        token_url = '%s&ident=%s&klub=0&unikey=0&timestamp=%s&auth=%s' % (
-            flash_url, ident, flashvars['timestamp'], flashvars['auth'])
+        time_date = _v('time_date')
+        time_start = _v('time_start')
+        upload_date = None
+        if time_date and time_start:
+            upload_date = unified_strdate(time_date + ' ' + time_start)
 
+        json_url = ('https://club.laola1.tv/sp/laola1/api/v3/user/session' +
+                    '/premium/player/stream-access?videoId=%s&target=2' +
+                    '&label=laola1tv&area=%s') % (video_id, _v('area'))
+        req = sanitized_Request(json_url, urlencode_postdata(
+            dict((i, v) for i, v in enumerate(_v('req_liga_abos').split(',')))))
+
+        token_url = self._download_json(req, video_id)['data']['stream-access'][0]
         token_doc = self._download_xml(
             token_url, video_id, note='Downloading token')
-        token_attrib = token_doc.find('.//token').attrib
-        if token_attrib.get('auth') in ('blocked', 'restricted'):
-            raise ExtractorError(
-                'Token error: %s' % token_attrib.get('comment'), expected=True)
 
-        video_url = '%s?hdnea=%s&hdcore=3.2.0' % (
-            token_attrib['url'], token_attrib['auth'])
+        token_attrib = xpath_element(token_doc, './/token').attrib
+        token_auth = token_attrib['auth']
+
+        if token_auth in ('blocked', 'restricted'):
+            raise ExtractorError(
+                'Token error: %s' % token_attrib['comment'], expected=True)
+
+        video_url = '%s?hdnea=%s&hdcore=3.2.0' % (token_attrib['url'], token_auth)
 
         return {
-            'id': video_id,
-            'is_live': is_live,
-            'title': title,
-            'url': video_url,
-            'uploader': uploader,
             'categories': categories,
-            'ext': 'mp4',
+            'formats': self._extract_f4m_formats(
+                video_url, video_id, f4m_id='hds'),
+            'id': video_id,
+            'is_live': _v('islive') == 'true',
+            'title': title,
+            'upload_date': upload_date,
+            'uploader': _v('meta_organisation'),
         }

From d9da74bc06d4b82d990bac501aedd52004d83b63 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 14 Feb 2016 23:39:49 +0600
Subject: [PATCH 4006/6219] Credit @blackwinter for laola1tv (#8478)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 7cac337a7..ad3fc021b 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -157,3 +157,4 @@ Founder Fang
 Andrew Alexeyew
 Saso Bezlaj
 Erwin de Haan
+Jens Wille

From 9dcefb23a12470390baa5e2ca5f3319d1479a571 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 14 Feb 2016 23:40:26 +0600
Subject: [PATCH 4007/6219] [laola1tv] Improve (Closes #8478)

---
 youtube_dl/extractor/laola1tv.py | 102 +++++++++++++++++--------------
 1 file changed, 57 insertions(+), 45 deletions(-)

diff --git a/youtube_dl/extractor/laola1tv.py b/youtube_dl/extractor/laola1tv.py
index c1f248b6b..5d8ebbeb3 100644
--- a/youtube_dl/extractor/laola1tv.py
+++ b/youtube_dl/extractor/laola1tv.py
@@ -1,31 +1,35 @@
-# -*- coding: utf-8 -*-
+# coding: utf-8
 from __future__ import unicode_literals
 
 import re
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
+    compat_urllib_parse,
     compat_urlparse,
+)
+from ..utils import (
     ExtractorError,
     sanitized_Request,
-    xpath_element,
-    xpath_text,
     unified_strdate,
     urlencode_postdata,
+    xpath_element,
+    xpath_text,
 )
 
 
 class Laola1TvIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?laola1\.tv/(?P<lang>[a-z]+)-(?P<portal>[a-z]+)/.*?/(?P<slug>[\w-]+)'
+    _VALID_URL = r'https?://(?:www\.)?laola1\.tv/(?P<lang>[a-z]+)-(?P<portal>[a-z]+)/[^/]+/(?P<slug>[^/?#&]+)'
     _TESTS = [{
         'url': 'http://www.laola1.tv/de-de/video/straubing-tigers-koelner-haie/227883.html',
         'info_dict': {
-            'categories': ['Eishockey'],
-            'ext': 'flv',
             'id': '227883',
-            'is_live': False,
+            'display_id': 'straubing-tigers-koelner-haie',
+            'ext': 'flv',
             'title': 'Straubing Tigers - Kölner Haie',
             'upload_date': '20140912',
+            'is_live': False,
+            'categories': ['Eishockey'],
         },
         'params': {
             'skip_download': True,
@@ -33,12 +37,13 @@ class Laola1TvIE(InfoExtractor):
     }, {
         'url': 'http://www.laola1.tv/de-de/video/straubing-tigers-koelner-haie',
         'info_dict': {
-            'categories': ['Eishockey'],
-            'ext': 'flv',
             'id': '464602',
-            'is_live': False,
+            'display_id': 'straubing-tigers-koelner-haie',
+            'ext': 'flv',
             'title': 'Straubing Tigers - Kölner Haie',
             'upload_date': '20160129',
+            'is_live': False,
+            'categories': ['Eishockey'],
         },
         'params': {
             'skip_download': True,
@@ -47,67 +52,74 @@ class Laola1TvIE(InfoExtractor):
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
+        display_id = mobj.group('slug')
         lang = mobj.group('lang')
         portal = mobj.group('portal')
 
-        webpage = self._download_webpage(url, mobj.group('slug'))
+        webpage = self._download_webpage(url, display_id)
+
         iframe_url = self._search_regex(
             r'<iframe[^>]*?id="videoplayer"[^>]*?src="([^"]+)"',
-            webpage, 'iframe URL')
+            webpage, 'iframe url')
 
         video_id = self._search_regex(
-            r'videoid=(\d+)', iframe_url, 'video ID')
+            r'videoid=(\d+)', iframe_url, 'video id')
 
         iframe = self._download_webpage(compat_urlparse.urljoin(
-            url, iframe_url), video_id, note='Downloading iframe')
+            url, iframe_url), display_id, 'Downloading iframe')
 
         partner_id = self._search_regex(
-            r'partnerid\s*:\s*"([^"]+)"', iframe, 'partner ID')
+            r'partnerid\s*:\s*(["\'])(?P<partner_id>.+?)\1',
+            iframe, 'partner id', group='partner_id')
 
-        xml_url = ('http://www.laola1.tv/server/hd_video.php?' +
-                   'play=%s&partner=%s&portal=%s&v5ident=&lang=%s' % (
-                       video_id, partner_id, portal, lang))
-        hd_doc = self._download_xml(xml_url, video_id)
+        hd_doc = self._download_xml(
+            'http://www.laola1.tv/server/hd_video.php?%s'
+            % compat_urllib_parse.urlencode({
+                'play': video_id,
+                'partner': partner_id,
+                'portal': portal,
+                'lang': lang,
+                'v5ident': '',
+            }), display_id)
 
         _v = lambda x, **k: xpath_text(hd_doc, './/video/' + x, **k)
         title = _v('title', fatal=True)
 
-        categories = _v('meta_sports')
-        if categories:
-            categories = categories.split(',')
+        req = sanitized_Request(
+            'https://club.laola1.tv/sp/laola1/api/v3/user/session/premium/player/stream-access?%s' %
+            compat_urllib_parse.urlencode({
+                'videoId': video_id,
+                'target': '2',
+                'label': 'laola1tv',
+                'area': _v('area'),
+            }),
+            urlencode_postdata(
+                dict((i, v) for i, v in enumerate(_v('req_liga_abos').split(',')))))
 
-        time_date = _v('time_date')
-        time_start = _v('time_start')
-        upload_date = None
-        if time_date and time_start:
-            upload_date = unified_strdate(time_date + ' ' + time_start)
-
-        json_url = ('https://club.laola1.tv/sp/laola1/api/v3/user/session' +
-                    '/premium/player/stream-access?videoId=%s&target=2' +
-                    '&label=laola1tv&area=%s') % (video_id, _v('area'))
-        req = sanitized_Request(json_url, urlencode_postdata(
-            dict((i, v) for i, v in enumerate(_v('req_liga_abos').split(',')))))
-
-        token_url = self._download_json(req, video_id)['data']['stream-access'][0]
-        token_doc = self._download_xml(
-            token_url, video_id, note='Downloading token')
+        token_url = self._download_json(req, display_id)['data']['stream-access'][0]
+        token_doc = self._download_xml(token_url, display_id, 'Downloading token')
 
         token_attrib = xpath_element(token_doc, './/token').attrib
         token_auth = token_attrib['auth']
 
-        if token_auth in ('blocked', 'restricted'):
+        if token_auth in ('blocked', 'restricted', 'error'):
             raise ExtractorError(
                 'Token error: %s' % token_attrib['comment'], expected=True)
 
-        video_url = '%s?hdnea=%s&hdcore=3.2.0' % (token_attrib['url'], token_auth)
+        formats = self._extract_f4m_formats(
+            '%s?hdnea=%s&hdcore=3.2.0' % (token_attrib['url'], token_auth),
+            video_id, f4m_id='hds')
+
+        categories_str = _v('meta_sports')
+        categories = categories_str.split(',') if categories_str else []
 
         return {
-            'categories': categories,
-            'formats': self._extract_f4m_formats(
-                video_url, video_id, f4m_id='hds'),
             'id': video_id,
-            'is_live': _v('islive') == 'true',
+            'display_id': display_id,
             'title': title,
-            'upload_date': upload_date,
+            'upload_date': unified_strdate(_v('time_date')),
             'uploader': _v('meta_organisation'),
+            'categories': categories,
+            'is_live': _v('islive') == 'true',
+            'formats': formats,
         }

From a517bb4b1e3eaea67973ee6a0718f2f4cf2b760e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 15 Feb 2016 00:07:16 +0100
Subject: [PATCH 4008/6219] [noz] Add new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/noz.py      | 69 ++++++++++++++++++++++++++++++++
 2 files changed, 70 insertions(+)
 create mode 100644 youtube_dl/extractor/noz.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 6937f28d3..a02113c86 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -490,6 +490,7 @@ from .nowtv import (
     NowTVIE,
     NowTVListIE,
 )
+from .noz import NozIE
 from .npo import (
     NPOIE,
     NPOLiveIE,
diff --git a/youtube_dl/extractor/noz.py b/youtube_dl/extractor/noz.py
new file mode 100644
index 000000000..0ffb44b47
--- /dev/null
+++ b/youtube_dl/extractor/noz.py
@@ -0,0 +1,69 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..compat import compat_urllib_parse_unquote
+from ..utils import (
+    int_or_none,
+    xpath_text,
+)
+
+
+class NozIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?noz\.de/video/(?P<id>[0-9]+)/'
+    _TESTS = [{
+        'url': 'http://www.noz.de/video/25151/32-Deutschland-gewinnt-Badminton-Lnderspiel-in-Melle',
+        'info_dict': {
+            'id': '25151',
+            'ext': 'mp4',
+            'duration': 215,
+            'title': '3:2 - Deutschland gewinnt Badminton-Länderspiel in Melle',
+            'description': 'Vor rund 370 Zuschauern gewinnt die deutsche Badminton-Nationalmannschaft am Donnerstag ein EM-Vorbereitungsspiel gegen Frankreich in Melle. Video Moritz Frankenberg.',
+            'thumbnail': 're:^http://.*\.jpg',
+        },
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        description = self._og_search_description(webpage)
+
+        edge_url = self._html_search_regex(
+            r'<script\s+(?:type="text/javascript"\s+)?src="(.*?/videojs_.*?)"',
+            webpage, 'edge URL')
+        edge_content = self._download_webpage(edge_url, 'meta configuration')
+
+        config_url_encoded = self._search_regex(
+            r'so\.addVariable\("config_url","[^,]*,(.*?)"',
+            edge_content, 'config URL'
+        )
+        config_url = compat_urllib_parse_unquote(config_url_encoded)
+
+        doc = self._download_xml(config_url, 'video configuration')
+        title = xpath_text(doc, './/title')
+        thumbnail = xpath_text(doc, './/article/thumbnail/url')
+        duration = int_or_none(xpath_text(
+            doc, './/article/movie/file/duration'))
+        formats = []
+        for qnode in doc.findall('.//article/movie/file/qualities/qual'):
+            video_node = qnode.find('./html_urls/video_url[@format="video/mp4"]')
+            if video_node is None:
+                continue  # auto
+            formats.append({
+                'url': video_node.text,
+                'format_name': xpath_text(qnode, './name'),
+                'format_id': xpath_text(qnode, './id'),
+                'height': int_or_none(xpath_text(qnode, './height')),
+                'width': int_or_none(xpath_text(qnode, './width')),
+                'tbr': int_or_none(xpath_text(qnode, './bitrate'), scale=1000),
+            })
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'formats': formats,
+            'title': title,
+            'duration': duration,
+            'description': description,
+            'thumbnail': thumbnail,
+        }

From 8e0548e180ee960eff6325b65e6cb2b91ee4d336 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 15 Feb 2016 19:58:24 +0800
Subject: [PATCH 4009/6219] [iqiyi] Partial support for VIP-only videos

See #8569 and #8019. Currently only 6-min preview are supported
---
 youtube_dl/extractor/iqiyi.py | 106 ++++++++++++++++++++++++++++++----
 1 file changed, 94 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index 691cb66d6..04db252df 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -3,13 +3,23 @@ from __future__ import unicode_literals
 
 import hashlib
 import math
+import os
 import random
 import time
 import uuid
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse
-from ..utils import ExtractorError
+from ..compat import (
+    compat_parse_qs,
+    compat_urllib_parse,
+    compat_urllib_parse_urlparse,
+)
+from ..utils import (
+    ExtractorError,
+    sanitized_Request,
+    urlencode_postdata,
+    url_basename,
+)
 
 
 class IqiyiIE(InfoExtractor):
@@ -93,6 +103,29 @@ class IqiyiIE(InfoExtractor):
     }, {
         'url': 'http://yule.iqiyi.com/pcb.html',
         'only_matching': True,
+    }, {
+        # VIP-only video. The first 2 parts (6 minutes) are available without login
+        'url': 'http://www.iqiyi.com/v_19rrny4w8w.html',
+        'info_dict': {
+            'id': 'f3cf468b39dddb30d676f89a91200dc1',
+            'title': '泰坦尼克号',
+        },
+        'playlist': [{
+            'md5': '436bcde6e1307307d5ba1549715b0bbd',
+            'info_dict': {
+                'id': 'f3cf468b39dddb30d676f89a91200dc1_part1',
+                'ext': 'f4v',
+                'title': '泰坦尼克号',
+            },
+        }, {
+            'md5': 'bfc5e332f4900fde547c69372385649e',
+            'info_dict': {
+                'id': 'f3cf468b39dddb30d676f89a91200dc1_part2',
+                'ext': 'f4v',
+                'title': '泰坦尼克号',
+            },
+        }],
+        'expected_warnings': ['Needs a VIP account for full video'],
     }]
 
     _FORMATS_MAP = [
@@ -108,7 +141,40 @@ class IqiyiIE(InfoExtractor):
     def md5_text(text):
         return hashlib.md5(text.encode('utf-8')).hexdigest()
 
-    def construct_video_urls(self, data, video_id, _uuid):
+    def _authenticate_vip_video(self, api_video_url, video_id, tvid, _uuid, do_report_warning):
+        auth_params = {
+            # version and platform hard-coded in com/qiyi/player/core/model/remote/AuthenticationRemote.as
+            'version': '2.0',
+            'platform': 'b6c13e26323c537d',
+            'aid': tvid,
+            'tvid': tvid,
+            'uid': '',
+            'deviceId': _uuid,
+            'playType': 'main',  # XXX: always main?
+            'filename': os.path.splitext(url_basename(api_video_url))[0],
+        }
+
+        qd_items = compat_parse_qs(compat_urllib_parse_urlparse(api_video_url).query)
+        for key, val in qd_items.items():
+            auth_params[key] = val[0]
+
+        auth_req = sanitized_Request(
+            'http://api.vip.iqiyi.com/services/ckn.action',
+            urlencode_postdata(auth_params))
+        # iQiyi server throws HTTP 405 error without the following header
+        auth_req.add_header('Content-Type', 'application/x-www-form-urlencoded')
+        auth_result = self._download_json(
+            auth_req, video_id,
+            note='Downloading video authentication JSON',
+            errnote='Unable to download video authentication JSON')
+        if auth_result['code'] == 'Q00506':  # requires a VIP account
+            if do_report_warning:
+                self.report_warning('Needs a VIP account for full video')
+            return False
+
+        return auth_result
+
+    def construct_video_urls(self, data, video_id, _uuid, tvid):
         def do_xor(x, y):
             a = y % 3
             if a == 1:
@@ -137,6 +203,7 @@ class IqiyiIE(InfoExtractor):
             return self.md5_text(t + mg + x)
 
         video_urls_dict = {}
+        need_vip_warning_report = True
         for format_item in data['vp']['tkl'][0]['vs']:
             if 0 < int(format_item['bid']) <= 10:
                 format_id = self.get_format(format_item['bid'])
@@ -155,11 +222,13 @@ class IqiyiIE(InfoExtractor):
                 vl = segment['l']
                 if not vl.startswith('/'):
                     vl = get_encode_code(vl)
-                key = get_path_key(
-                    vl.split('/')[-1].split('.')[0], format_id, segment_index)
+                is_vip_video = '/vip/' in vl
                 filesize = segment['b']
                 base_url = data['vp']['du'].split('/')
-                base_url.insert(-1, key)
+                if not is_vip_video:
+                    key = get_path_key(
+                        vl.split('/')[-1].split('.')[0], format_id, segment_index)
+                    base_url.insert(-1, key)
                 base_url = '/'.join(base_url)
                 param = {
                     'su': _uuid,
@@ -170,8 +239,23 @@ class IqiyiIE(InfoExtractor):
                     'ct': '',
                     'tn': str(int(time.time()))
                 }
-                api_video_url = base_url + vl + '?' + \
-                    compat_urllib_parse.urlencode(param)
+                api_video_url = base_url + vl
+                if is_vip_video:
+                    api_video_url = api_video_url.replace('.f4v', '.hml')
+                    auth_result = self._authenticate_vip_video(
+                        api_video_url, video_id, tvid, _uuid, need_vip_warning_report)
+                    if auth_result is False:
+                        need_vip_warning_report = False
+                        break
+                    param.update({
+                        't': auth_result['data']['t'],
+                        # cid is hard-coded in com/qiyi/player/core/player/RuntimeData.as
+                        'cid': 'afbe8fd3d73448c9',
+                        'vid': video_id,
+                        'QY00001': auth_result['data']['u'],
+                    })
+                api_video_url += '?' if '?' not in api_video_url else '&'
+                api_video_url += compat_urllib_parse.urlencode(param)
                 js = self._download_json(
                     api_video_url, video_id,
                     note='Download video info of segment %d for format %s' % (segment_index + 1, format_id))
@@ -205,6 +289,7 @@ class IqiyiIE(InfoExtractor):
             'tn': random.random(),
             'um': 0,
             'authkey': self.md5_text(self.md5_text('') + tail),
+            'k_tag': 1,
         }
 
         api_url = 'http://cache.video.qiyi.com/vms' + '?' + \
@@ -236,16 +321,13 @@ class IqiyiIE(InfoExtractor):
         if raw_data['code'] != 'A000000':
             raise ExtractorError('Unable to load data. Error code: ' + raw_data['code'])
 
-        if not raw_data['data']['vp']['tkl']:
-            raise ExtractorError('No support iQiqy VIP video')
-
         data = raw_data['data']
 
         title = data['vi']['vn']
 
         # generate video_urls_dict
         video_urls_dict = self.construct_video_urls(
-            data, video_id, _uuid)
+            data, video_id, _uuid, tvid)
 
         # construct info
         entries = []

From 8d3eeb36d781183d52c093153e7fab6c17427dc7 Mon Sep 17 00:00:00 2001
From: Robin Houtevelts <robin@houtevelts.com>
Date: Sat, 13 Feb 2016 01:12:44 +0100
Subject: [PATCH 4010/6219] [Canvas] Add subtitles

---
 youtube_dl/extractor/canvas.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/youtube_dl/extractor/canvas.py b/youtube_dl/extractor/canvas.py
index ee19ff836..16f458d6c 100644
--- a/youtube_dl/extractor/canvas.py
+++ b/youtube_dl/extractor/canvas.py
@@ -53,6 +53,15 @@ class CanvasIE(InfoExtractor):
                     'url': format_url,
                 })
         self._sort_formats(formats)
+        
+        subtitles = {}
+        for target in data['subtitleUrls']:
+            format_url, format_type = target.get('url'), target.get('type')
+            if format_type == 'CLOSED':
+                subtitles['nl'] = [{
+                    'ext': 'vtt',
+                    'url': format_url,
+                }]
 
         return {
             'id': video_id,
@@ -62,4 +71,5 @@ class CanvasIE(InfoExtractor):
             'formats': formats,
             'duration': float_or_none(data.get('duration'), 1000),
             'thumbnail': data.get('posterImageUrl'),
+            'subtitles': subtitles,
         }

From fd7a3ea4a41ccad37c932662d39e159854311357 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 15 Feb 2016 20:54:01 +0600
Subject: [PATCH 4011/6219] [canvas] Improve subtitles (Closes #8537)

---
 youtube_dl/extractor/canvas.py | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/canvas.py b/youtube_dl/extractor/canvas.py
index 16f458d6c..54b535ae9 100644
--- a/youtube_dl/extractor/canvas.py
+++ b/youtube_dl/extractor/canvas.py
@@ -53,15 +53,14 @@ class CanvasIE(InfoExtractor):
                     'url': format_url,
                 })
         self._sort_formats(formats)
-        
+
         subtitles = {}
-        for target in data['subtitleUrls']:
-            format_url, format_type = target.get('url'), target.get('type')
-            if format_type == 'CLOSED':
-                subtitles['nl'] = [{
-                    'ext': 'vtt',
-                    'url': format_url,
-                }]
+        subtitle_urls = data.get('subtitleUrls')
+        if isinstance(subtitle_urls, list):
+            for subtitle in subtitle_urls:
+                subtitle_url = subtitle.get('url')
+                if subtitle_url and subtitle.get('type') == 'CLOSED':
+                    subtitles.setdefault('nl', []).append({'url': subtitle_url})
 
         return {
             'id': video_id,

From 6eff2605d645664ba52fe873bb28cea77744e7f6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 15 Feb 2016 20:59:16 +0600
Subject: [PATCH 4012/6219] [canvas] Add subtitles test (#8537)

---
 youtube_dl/extractor/canvas.py | 24 ++++++++++++++++++++++--
 1 file changed, 22 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/canvas.py b/youtube_dl/extractor/canvas.py
index 54b535ae9..ec6d24d96 100644
--- a/youtube_dl/extractor/canvas.py
+++ b/youtube_dl/extractor/canvas.py
@@ -6,7 +6,7 @@ from ..utils import float_or_none
 
 class CanvasIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?canvas\.be/video/(?:[^/]+/)*(?P<id>[^/?#&]+)'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.canvas.be/video/de-afspraak/najaar-2015/de-afspraak-veilt-voor-de-warmste-week',
         'md5': 'ea838375a547ac787d4064d8c7860a6c',
         'info_dict': {
@@ -18,7 +18,27 @@ class CanvasIE(InfoExtractor):
             'thumbnail': 're:^https?://.*\.jpg$',
             'duration': 49.02,
         }
-    }
+    }, {
+        # with subtitles
+        'url': 'http://www.canvas.be/video/panorama/2016/pieter-0167',
+        'info_dict': {
+            'id': 'mz-ast-5240ff21-2d30-4101-bba6-92b5ec67c625',
+            'display_id': 'pieter-0167',
+            'ext': 'mp4',
+            'title': 'Pieter 0167',
+            'description': 'md5:943cd30f48a5d29ba02c3a104dc4ec4e',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 2553.08,
+            'subtitles': {
+                'nl': [{
+                    'ext': 'vtt',
+                }],
+            },
+        },
+        'params': {
+            'skip_download': True,
+        }
+    }]
 
     def _real_extract(self, url):
         display_id = self._match_id(url)

From b9e8d7140aa8584f8516c3575c20b21172278fef Mon Sep 17 00:00:00 2001
From: Patrick Griffis <tingping@tingping.se>
Date: Wed, 10 Feb 2016 15:33:19 -0500
Subject: [PATCH 4013/6219] [screenjunkies] Add new extractor

This doesn't handle the plus only videos yet

Closes #8492
---
 youtube_dl/extractor/__init__.py      |  1 +
 youtube_dl/extractor/screenjunkies.py | 74 +++++++++++++++++++++++++++
 2 files changed, 75 insertions(+)
 create mode 100644 youtube_dl/extractor/screenjunkies.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index a02113c86..537d25777 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -621,6 +621,7 @@ from .sbs import SBSIE
 from .scivee import SciVeeIE
 from .screencast import ScreencastIE
 from .screencastomatic import ScreencastOMaticIE
+from .screenjunkies import ScreenJunkiesIE
 from .screenwavemedia import ScreenwaveMediaIE, TeamFourIE
 from .senateisvp import SenateISVPIE
 from .servingsys import ServingSysIE
diff --git a/youtube_dl/extractor/screenjunkies.py b/youtube_dl/extractor/screenjunkies.py
new file mode 100644
index 000000000..5da3f9bcd
--- /dev/null
+++ b/youtube_dl/extractor/screenjunkies.py
@@ -0,0 +1,74 @@
+from __future__ import unicode_literals
+
+import re
+import json
+
+from .common import (
+    InfoExtractor,
+    ExtractorError,
+)
+
+from ..utils import (
+    int_or_none,
+)
+
+class ScreenJunkiesIE(InfoExtractor):
+    _VALID_URL = r'http://www.screenjunkies.com/video/(.+-(?P<id>\d+)|.+)'
+
+    _TESTS = [{
+        'url': 'http://www.screenjunkies.com/video/best-quentin-tarantino-movie-2841915',
+        'info_dict': {
+            'id': '2841915',
+            'ext': 'mp4',
+            'title': 'Best Quentin Tarantino Movie',
+        },
+    }, {
+        'url': 'http://www.screenjunkies.com/video/honest-trailers-the-dark-knight',
+        'info_dict': {
+            'id': '2348808',
+            'ext': 'mp4',
+            'title': "Honest Trailers - 'The Dark Knight'",
+        },
+    }, {
+        'url': 'http://www.screenjunkies.com/video/knocking-dead-ep-1-the-show-so-far-3003285',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        if not video_id: # Older urls didn't have the id in them, but we can grab it manually
+            webpage = self._download_webpage(url, url)
+            video_id = self._html_search_regex(r'src="/embed/(\d+)"', webpage, 'video id')
+
+        webpage = self._download_webpage('http://www.screenjunkies.com/embed/%s' %video_id, video_id)
+        video_vars_str = self._html_search_regex(r'embedVars = (\{.+\})\s*</script>', webpage, 'video variables', flags=re.S)
+        video_vars = self._parse_json(video_vars_str, video_id)
+
+        # TODO: Figure out required cookies
+        if video_vars['subscriptionLevel'] > 0:
+            raise ExtractorError('This video requires ScreenJunkiesPlus', expected=True)
+
+        formats = []
+        for f in video_vars['media']:
+            if not f['mediaPurpose'] == 'play':
+                continue
+
+            formats.append({
+                'url': f['uri'],
+                'width': int_or_none(f.get('width')),
+                'height': int_or_none(f.get('height')),
+                'tbr': int_or_none(f.get('bitRate')),
+                'format': 'mp4',
+            })
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': video_vars['contentName'],
+            'formats': formats,
+            'duration': int_or_none(video_vars.get('videoLengthInSeconds')),
+            'thumbnail': video_vars.get('thumbUri'),
+            'tags': video_vars['tags'].split(',') if 'tags' in video_vars else [],
+        }
+

From c8a80fd8184cddbd30ac8188ce04486886156365 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 15 Feb 2016 22:29:28 +0600
Subject: [PATCH 4014/6219] [screenjunkies] Improve, extract more metadata and
 workaround subscription (Closes #8505)

---
 youtube_dl/extractor/screenjunkies.py | 122 ++++++++++++++++++++------
 1 file changed, 93 insertions(+), 29 deletions(-)

diff --git a/youtube_dl/extractor/screenjunkies.py b/youtube_dl/extractor/screenjunkies.py
index 5da3f9bcd..e0db180b8 100644
--- a/youtube_dl/extractor/screenjunkies.py
+++ b/youtube_dl/extractor/screenjunkies.py
@@ -1,74 +1,138 @@
 from __future__ import unicode_literals
 
 import re
-import json
-
-from .common import (
-    InfoExtractor,
-    ExtractorError,
-)
 
+from .common import InfoExtractor
+from ..compat import compat_str
 from ..utils import (
     int_or_none,
+    parse_age_limit,
 )
 
-class ScreenJunkiesIE(InfoExtractor):
-    _VALID_URL = r'http://www.screenjunkies.com/video/(.+-(?P<id>\d+)|.+)'
 
+class ScreenJunkiesIE(InfoExtractor):
+    _VALID_URL = r'http://www.screenjunkies.com/video/(?P<display_id>[^/]+?)(?:-(?P<id>\d+))?(?:[/?#&]|$)'
     _TESTS = [{
         'url': 'http://www.screenjunkies.com/video/best-quentin-tarantino-movie-2841915',
+        'md5': '5c2b686bec3d43de42bde9ec047536b0',
         'info_dict': {
             'id': '2841915',
+            'display_id': 'best-quentin-tarantino-movie',
             'ext': 'mp4',
             'title': 'Best Quentin Tarantino Movie',
+            'thumbnail': 're:^https?://.*\.jpg',
+            'duration': 3671,
+            'age_limit': 13,
+            'tags': list,
         },
     }, {
         'url': 'http://www.screenjunkies.com/video/honest-trailers-the-dark-knight',
         'info_dict': {
             'id': '2348808',
+            'display_id': 'honest-trailers-the-dark-knight',
             'ext': 'mp4',
-            'title': "Honest Trailers - 'The Dark Knight'",
+            'title': "Honest Trailers: 'The Dark Knight'",
+            'thumbnail': 're:^https?://.*\.jpg',
+            'age_limit': 10,
+            'tags': list,
         },
     }, {
+        # requires subscription but worked around
         'url': 'http://www.screenjunkies.com/video/knocking-dead-ep-1-the-show-so-far-3003285',
-        'only_matching': True,
+        'info_dict': {
+            'id': '3003285',
+            'display_id': 'knocking-dead-ep-1-the-show-so-far',
+            'ext': 'mp4',
+            'title': 'Knocking Dead Ep 1: State of The Dead Recap',
+            'thumbnail': 're:^https?://.*\.jpg',
+            'duration': 3307,
+            'age_limit': 13,
+            'tags': list,
+        },
     }]
 
+    _DEFAULT_BITRATES = (48, 150, 496, 864, 2240)
+
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        display_id = mobj.group('display_id')
 
-        if not video_id: # Older urls didn't have the id in them, but we can grab it manually
-            webpage = self._download_webpage(url, url)
-            video_id = self._html_search_regex(r'src="/embed/(\d+)"', webpage, 'video id')
+        if not video_id:
+            webpage = self._download_webpage(url, display_id)
+            video_id = self._search_regex(
+                (r'src=["\']/embed/(\d+)', r'data-video-content-id=["\'](\d+)'),
+                webpage, 'video id')
 
-        webpage = self._download_webpage('http://www.screenjunkies.com/embed/%s' %video_id, video_id)
-        video_vars_str = self._html_search_regex(r'embedVars = (\{.+\})\s*</script>', webpage, 'video variables', flags=re.S)
-        video_vars = self._parse_json(video_vars_str, video_id)
+        webpage = self._download_webpage(
+            'http://www.screenjunkies.com/embed/%s' % video_id,
+            display_id, 'Downloading video embed page')
+        embed_vars = self._parse_json(
+            self._search_regex(
+                r'(?s)embedVars\s*=\s*({.+?})\s*</script>', webpage, 'embed vars'),
+            display_id)
 
-        # TODO: Figure out required cookies
-        if video_vars['subscriptionLevel'] > 0:
-            raise ExtractorError('This video requires ScreenJunkiesPlus', expected=True)
+        title = embed_vars['contentName']
 
         formats = []
-        for f in video_vars['media']:
-            if not f['mediaPurpose'] == 'play':
+        bitrates = []
+        for f in embed_vars.get('media', []):
+            if not f.get('uri') or f.get('mediaPurpose') != 'play':
                 continue
-
+            bitrate = int_or_none(f.get('bitRate'))
+            if bitrate:
+                bitrates.append(bitrate)
             formats.append({
                 'url': f['uri'],
+                'format_id': 'http-%d' % bitrate if bitrate else 'http',
                 'width': int_or_none(f.get('width')),
                 'height': int_or_none(f.get('height')),
-                'tbr': int_or_none(f.get('bitRate')),
+                'tbr': bitrate,
                 'format': 'mp4',
             })
+
+        if not bitrates:
+            # When subscriptionLevel > 0, i.e. plus subscription is required
+            # media list will be empty. However, hds and hls uri are still
+            # available. We can grabbing them assuming bitrates to be default.
+            bitrates = self._DEFAULT_BITRATES
+
+        auth_token = embed_vars.get('AuthToken')
+
+        def construct_manifest_url(base_url, ext):
+            pieces = [base_url]
+            pieces.extend([compat_str(b) for b in bitrates])
+            pieces.append('_kbps.mp4.%s?%s' % (ext, auth_token))
+            return ','.join(pieces)
+
+        if bitrates and auth_token:
+            hds_url = embed_vars.get('hdsUri')
+            if hds_url:
+                f4m_formats = self._extract_f4m_formats(
+                    construct_manifest_url(hds_url, 'f4m'),
+                    display_id, f4m_id='hds', fatal=False)
+                if len(f4m_formats) == len(bitrates):
+                    for f, bitrate in zip(f4m_formats, bitrates):
+                        if not f.get('tbr'):
+                            f['format_id'] = 'hds-%d' % bitrate
+                            f['tbr'] = bitrate
+                # TODO: fix f4m downloader to handle manifests without bitrates if possible
+                # formats.extend(f4m_formats)
+
+            hls_url = embed_vars.get('hlsUri')
+            if hls_url:
+                formats.extend(self._extract_m3u8_formats(
+                    construct_manifest_url(hls_url, 'm3u8'),
+                    display_id, 'mp4', entry_protocol='m3u8_native', m3u8_id='hls', fatal=False))
         self._sort_formats(formats)
 
         return {
             'id': video_id,
-            'title': video_vars['contentName'],
+            'display_id': display_id,
+            'title': title,
+            'thumbnail': embed_vars.get('thumbUri'),
+            'duration': int_or_none(embed_vars.get('videoLengthInSeconds')) or None,
+            'age_limit': parse_age_limit(embed_vars.get('audienceRating')),
+            'tags': embed_vars.get('tags', '').split(','),
             'formats': formats,
-            'duration': int_or_none(video_vars.get('videoLengthInSeconds')),
-            'thumbnail': video_vars.get('thumbUri'),
-            'tags': video_vars['tags'].split(',') if 'tags' in video_vars else [],
         }
-

From 8eacae8cf9a1226ccd9b186034b03756ca607921 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 15 Feb 2016 22:33:32 +0600
Subject: [PATCH 4015/6219] Credit @RobinHoutevelts for canvas subtiltes
 (#8537)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index ad3fc021b..f8890eb27 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -158,3 +158,4 @@ Andrew Alexeyew
 Saso Bezlaj
 Erwin de Haan
 Jens Wille
+Robin Houtevelts

From d2c1f79f20007981432fbef6d8610094a45b0196 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 16 Feb 2016 00:29:51 +0600
Subject: [PATCH 4016/6219] [youtube:searchurl] Extend _VALID_URL

---
 youtube_dl/extractor/youtube.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 18f7d37f4..45ad88152 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1912,13 +1912,16 @@ class YoutubeSearchDateIE(YoutubeSearchIE):
 class YoutubeSearchURLIE(InfoExtractor):
     IE_DESC = 'YouTube.com search URLs'
     IE_NAME = 'youtube:search_url'
-    _VALID_URL = r'https?://(?:www\.)?youtube\.com/results\?(.*?&)?search_query=(?P<query>[^&]+)(?:[&]|$)'
+    _VALID_URL = r'https?://(?:www\.)?youtube\.com/results\?(.*?&)?(?:search_query|q)=(?P<query>[^&]+)(?:[&]|$)'
     _TESTS = [{
         'url': 'https://www.youtube.com/results?baz=bar&search_query=youtube-dl+test+video&filters=video&lclk=video',
         'playlist_mincount': 5,
         'info_dict': {
             'title': 'youtube-dl test video',
         }
+    }, {
+        'url': 'https://www.youtube.com/results?q=test&sp=EgQIBBgB',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 1932476c137363d7674abb84286c9d1aca3a0e70 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 16 Feb 2016 01:47:07 +0800
Subject: [PATCH 4017/6219] [iqiyi] Omit MD5 sums for the VIP-only video

---
 youtube_dl/extractor/iqiyi.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index 04db252df..9046705a5 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -105,20 +105,19 @@ class IqiyiIE(InfoExtractor):
         'only_matching': True,
     }, {
         # VIP-only video. The first 2 parts (6 minutes) are available without login
+        # MD5 sums omitted as values are different on Travis CI and my machine
         'url': 'http://www.iqiyi.com/v_19rrny4w8w.html',
         'info_dict': {
             'id': 'f3cf468b39dddb30d676f89a91200dc1',
             'title': '泰坦尼克号',
         },
         'playlist': [{
-            'md5': '436bcde6e1307307d5ba1549715b0bbd',
             'info_dict': {
                 'id': 'f3cf468b39dddb30d676f89a91200dc1_part1',
                 'ext': 'f4v',
                 'title': '泰坦尼克号',
             },
         }, {
-            'md5': 'bfc5e332f4900fde547c69372385649e',
             'info_dict': {
                 'id': 'f3cf468b39dddb30d676f89a91200dc1_part2',
                 'ext': 'f4v',

From fe80df30804736db5e644ad15ac09149c57ddf33 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 16 Feb 2016 01:24:57 +0600
Subject: [PATCH 4018/6219] Credit @TingPing for screenjunkies (#8505)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index f8890eb27..0e90a3ecb 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -159,3 +159,4 @@ Saso Bezlaj
 Erwin de Haan
 Jens Wille
 Robin Houtevelts
+Patrick Griffis

From f66a3c7bc2c675b0a053c6cfab6f755cb9f5bbda Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 16 Feb 2016 01:30:00 +0600
Subject: [PATCH 4019/6219] [screenjunkies] Fix spelling

---
 youtube_dl/extractor/screenjunkies.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/screenjunkies.py b/youtube_dl/extractor/screenjunkies.py
index e0db180b8..f2af15f6b 100644
--- a/youtube_dl/extractor/screenjunkies.py
+++ b/youtube_dl/extractor/screenjunkies.py
@@ -93,8 +93,8 @@ class ScreenJunkiesIE(InfoExtractor):
 
         if not bitrates:
             # When subscriptionLevel > 0, i.e. plus subscription is required
-            # media list will be empty. However, hds and hls uri are still
-            # available. We can grabbing them assuming bitrates to be default.
+            # media list will be empty. However, hds and hls uris are still
+            # available. We can grab them assuming bitrates to be default.
             bitrates = self._DEFAULT_BITRATES
 
         auth_token = embed_vars.get('AuthToken')

From cb64ccc71531fd7491f65ad12fa1d68ac5de70e1 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 16 Feb 2016 09:07:38 +0800
Subject: [PATCH 4020/6219] [facebook] Improve error handling (#8572)

---
 youtube_dl/extractor/facebook.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index ed237f081..0a9a5ca71 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -186,7 +186,7 @@ class FacebookIE(InfoExtractor):
         if not video_data:
             server_js_data = self._parse_json(self._search_regex(
                 r'handleServerJS\(({.+})\);', webpage, 'server js data'), video_id)
-            for item in server_js_data['instances']:
+            for item in server_js_data.get('instances', []):
                 if item[1][0] == 'VideoConfig':
                     video_data = video_data_list2dict(item[2][0]['videoData'])
                     break

From 86be3cdc2a623a522500ca073835579b853a1ae1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 16 Feb 2016 22:05:23 +0600
Subject: [PATCH 4021/6219] [xtube] Fix extraction (Closes #8565)

---
 youtube_dl/extractor/xtube.py | 90 +++++++++++++++++------------------
 1 file changed, 44 insertions(+), 46 deletions(-)

diff --git a/youtube_dl/extractor/xtube.py b/youtube_dl/extractor/xtube.py
index 8cd3a0687..24df5c48e 100644
--- a/youtube_dl/extractor/xtube.py
+++ b/youtube_dl/extractor/xtube.py
@@ -7,15 +7,16 @@ from .common import InfoExtractor
 from ..compat import compat_urllib_parse_unquote
 from ..utils import (
     int_or_none,
-    parse_duration,
     sanitized_Request,
     str_to_int,
 )
 
 
 class XTubeIE(InfoExtractor):
-    _VALID_URL = r'(?:xtube:|https?://(?:www\.)?xtube\.com/watch\.php\?.*\bv=)(?P<id>[^/?&#]+)'
-    _TEST = {
+    _VALID_URL = r'(?:xtube:|https?://(?:www\.)?xtube\.com/(?:watch\.php\?.*\bv=|video-watch/(?P<display_id>[^/]+)-))(?P<id>[^/?&#]+)'
+
+    _TESTS = [{
+        # old URL schema
         'url': 'http://www.xtube.com/watch.php?v=kVTUy_G222_',
         'md5': '092fbdd3cbe292c920ef6fc6a8a9cdab',
         'info_dict': {
@@ -27,63 +28,60 @@ class XTubeIE(InfoExtractor):
             'duration': 450,
             'age_limit': 18,
         }
-    }
+    }, {
+        # new URL schema
+        'url': 'http://www.xtube.com/video-watch/strange-erotica-625837',
+        'only_matching': True,
+    }, {
+        'url': 'xtube:625837',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        display_id = mobj.group('display_id')
 
-        req = sanitized_Request('http://www.xtube.com/watch.php?v=%s' % video_id)
-        req.add_header('Cookie', 'age_verified=1')
-        webpage = self._download_webpage(req, video_id)
+        if not display_id:
+            display_id = video_id
+            url = 'http://www.xtube.com/watch.php?v=%s' % video_id
 
-        video_title = self._html_search_regex(
-            r'<p class="title">([^<]+)', webpage, 'title')
-        video_uploader = self._html_search_regex(
-            [r"var\s+contentOwnerId\s*=\s*'([^']+)",
-             r'By:\s*<a href="/community/profile\.php\?user=([^"]+)'],
+        req = sanitized_Request(url)
+        req.add_header('Cookie', 'age_verified=1; cookiesAccepted=1')
+        webpage = self._download_webpage(req, display_id)
+
+        flashvars = self._parse_json(
+            self._search_regex(
+                r'xt\.playerOps\s*=\s*({.+?});', webpage, 'player ops'),
+            video_id)['flashvars']
+
+        title = flashvars.get('title') or self._search_regex(
+            r'<h1>([^<]+)</h1>', webpage, 'title')
+        video_url = compat_urllib_parse_unquote(flashvars['video_url'])
+        duration = int_or_none(flashvars.get('video_duration'))
+
+        uploader = self._search_regex(
+            r'<input[^>]+name="contentOwnerId"[^>]+value="([^"]+)"',
             webpage, 'uploader', fatal=False)
-        video_description = self._html_search_regex(
-            r'<p class="fieldsDesc">([^<]+)',
-            webpage, 'description', fatal=False)
-        duration = parse_duration(self._html_search_regex(
-            r'<span class="bold">Runtime:</span> ([^<]+)</p>',
-            webpage, 'duration', fatal=False))
-        view_count = str_to_int(self._html_search_regex(
-            r'<span class="bold">Views:</span> ([\d,\.]+)</p>',
+        description = self._search_regex(
+            r'</h1>\s*<p>([^<]+)', webpage, 'description', fatal=False)
+        view_count = str_to_int(self._search_regex(
+            r'<dt>Views:</dt>\s*<dd>([\d,\.]+)</dd>',
             webpage, 'view count', fatal=False))
         comment_count = str_to_int(self._html_search_regex(
-            r'<div id="commentBar">([\d,\.]+) Comments</div>',
+            r'>Comments? \(([\d,\.]+)\)<',
             webpage, 'comment count', fatal=False))
 
-        formats = []
-        for format_id, video_url in re.findall(
-                r'flashvars\.quality_(.+?)\s*=\s*"([^"]+)"', webpage):
-            fmt = {
-                'url': compat_urllib_parse_unquote(video_url),
-                'format_id': format_id,
-            }
-            m = re.search(r'^(?P<height>\d+)[pP]', format_id)
-            if m:
-                fmt['height'] = int(m.group('height'))
-            formats.append(fmt)
-
-        if not formats:
-            video_url = compat_urllib_parse_unquote(self._search_regex(
-                r'flashvars\.video_url\s*=\s*"([^"]+)"',
-                webpage, 'video URL'))
-            formats.append({'url': video_url})
-
-        self._sort_formats(formats)
-
         return {
             'id': video_id,
-            'title': video_title,
-            'uploader': video_uploader,
-            'description': video_description,
+            'display_id': display_id,
+            'url': video_url,
+            'title': title,
+            'description': description,
+            'uploader': uploader,
             'duration': duration,
             'view_count': view_count,
             'comment_count': comment_count,
-            'formats': formats,
             'age_limit': 18,
         }
 

From f4db09178a2c531005a3502a497b8dbf0bfe20e2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 16 Feb 2016 22:06:26 +0600
Subject: [PATCH 4022/6219] [xtube:user] Remove duplicated video ids

---
 youtube_dl/extractor/xtube.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/xtube.py b/youtube_dl/extractor/xtube.py
index 24df5c48e..4075b8a4f 100644
--- a/youtube_dl/extractor/xtube.py
+++ b/youtube_dl/extractor/xtube.py
@@ -7,6 +7,7 @@ from .common import InfoExtractor
 from ..compat import compat_urllib_parse_unquote
 from ..utils import (
     int_or_none,
+    orderedSet,
     sanitized_Request,
     str_to_int,
 )
@@ -118,7 +119,8 @@ class XTubeUserIE(InfoExtractor):
             if not html:
                 break
 
-            for _, video_id in re.findall(r'data-plid=(["\'])(.+?)\1', html):
+            for video_id in orderedSet([video_id for _, video_id in re.findall(
+                    r'data-plid=(["\'])(.+?)\1', html)]):
                 entries.append(self.url_result('xtube:%s' % video_id, XTubeIE.ie_key()))
 
             page_count = int_or_none(page.get('pageCount'))

From e277f2a63b836c774009c8f6c8e45a7867159b73 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 16 Feb 2016 22:23:38 +0600
Subject: [PATCH 4023/6219] [orf:tvthek] Check formats (Closes #8580)

---
 youtube_dl/extractor/orf.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/orf.py b/youtube_dl/extractor/orf.py
index c54775d54..958eb398b 100644
--- a/youtube_dl/extractor/orf.py
+++ b/youtube_dl/extractor/orf.py
@@ -112,6 +112,7 @@ class ORFTVthekIE(InfoExtractor):
                             % geo_str),
                         fatal=False)
 
+            self._check_formats(formats, video_id)
             self._sort_formats(formats)
 
             upload_date = unified_strdate(sd['created_date'])

From 199e72429106375218902102812e26c2fc6624b5 Mon Sep 17 00:00:00 2001
From: mutantmonkey <mutantmonkey@mutantmonkey.in>
Date: Mon, 15 Feb 2016 17:30:53 -0800
Subject: [PATCH 4024/6219] [KUSI] Add new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/kusi.py     | 61 ++++++++++++++++++++++++++++++++
 2 files changed, 62 insertions(+)
 create mode 100644 youtube_dl/extractor/kusi.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 537d25777..bfc2008be 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -338,6 +338,7 @@ from .konserthusetplay import KonserthusetPlayIE
 from .kontrtube import KontrTubeIE
 from .krasview import KrasViewIE
 from .ku6 import Ku6IE
+from .kusi import KUSIIE
 from .kuwo import (
     KuwoIE,
     KuwoAlbumIE,
diff --git a/youtube_dl/extractor/kusi.py b/youtube_dl/extractor/kusi.py
new file mode 100644
index 000000000..20407411b
--- /dev/null
+++ b/youtube_dl/extractor/kusi.py
@@ -0,0 +1,61 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..compat import compat_urllib_parse_unquote_plus
+from ..utils import int_or_none
+
+
+class KUSIIE(InfoExtractor):
+    _VALID_URL = r'http://(?:www\.)?kusi\.com/(?P<path>story/.+|video\?clipId=(?P<clipId>\d+))'
+    _TEST = {
+        'url': 'http://www.kusi.com/story/31183873/turko-files-case-closed-put-on-hold',
+        'md5': 'f926e7684294cf8cb7bdf8858e1b3988',
+        'info_dict': {
+            'id': '12203019',
+            'ext': 'mp4',
+            'title': 'Turko Files: Case Closed! & Put On Hold!',
+            'duration': 231000,
+        }
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+
+        if mobj.group('clipId') is not None:
+            video_id = mobj.group('clipId')
+        else:
+            webpage = self._download_webpage(url, mobj.group('path'))
+            video_id = self._html_search_regex(r'"clipId", "(\d+)"', webpage,
+                                               'clipId')
+
+        xml_url = 'http://www.kusi.com/build.asp?buildtype=buildfeaturexml'\
+                  'request&featureType=Clip&featureid={0}&affiliateno=956&'\
+                  'clientgroupid=1&rnd=562461'.format(video_id)
+        doc = self._download_xml(xml_url, video_id,
+                                 note='Downloading video info',
+                                 errnote='Failed to download video info')
+
+        video_title = doc.find('HEADLINE').text
+        duration = int_or_none(doc.find('DURATION'), get_attr='text')
+        description = doc.find('ABSTRACT')
+
+        quality_options = doc.find('{http://search.yahoo.com/mrss/}group').findall('{http://search.yahoo.com/mrss/}content')
+        formats = []
+        for quality in quality_options:
+            if 'height' in quality.attrib:
+                formats.append({
+                    'url': compat_urllib_parse_unquote_plus(quality.attrib['url']),
+                    'height': quality.attrib['height'],
+                })
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': video_title,
+            'description': description,
+            'duration': duration,
+            'formats': formats,
+        }

From 0f425e65ec6d9ac47fa7f348fae7c06be2557f8e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 17 Feb 2016 21:47:18 +0600
Subject: [PATCH 4025/6219] [arte:+7] Add support for en and es URLs

---
 youtube_dl/extractor/arte.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index 6ed855a57..67194bd31 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -63,7 +63,7 @@ class ArteTvIE(InfoExtractor):
 
 class ArteTVPlus7IE(InfoExtractor):
     IE_NAME = 'arte.tv:+7'
-    _VALID_URL = r'https?://(?:www\.)?arte\.tv/guide/(?P<lang>fr|de)/(?:(?:sendungen|emissions)/)?(?P<id>.*?)/(?P<name>.*?)(\?.*)?'
+    _VALID_URL = r'https?://(?:www\.)?arte\.tv/guide/(?P<lang>fr|de|en|es)/(?:(?:sendungen|emissions)/)?(?P<id>.*?)/(?P<name>.*?)(\?.*)?'
 
     @classmethod
     def _extract_url_info(cls, url):

From 9654fc875bdcd055031670b640531375bc513012 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 17 Feb 2016 21:49:15 +0600
Subject: [PATCH 4026/6219] [arte:+7] Fix extraction for react-based layout

---
 youtube_dl/extractor/arte.py | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index 67194bd31..497899f8d 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -102,10 +102,20 @@ class ArteTVPlus7IE(InfoExtractor):
             iframe_url = find_iframe_url(webpage, None)
             if not iframe_url:
                 embed_url = self._html_search_regex(
-                    r'arte_vp_url_oembed=\'([^\']+?)\'', webpage, 'embed url')
-                player = self._download_json(
-                    embed_url, video_id, 'Downloading player page')
-                iframe_url = find_iframe_url(player['html'])
+                    r'arte_vp_url_oembed=\'([^\']+?)\'', webpage, 'embed url', default=None)
+                if embed_url:
+                    player = self._download_json(
+                        embed_url, video_id, 'Downloading player page')
+                    iframe_url = find_iframe_url(player['html'])
+            # en and es URLs produce react-based pages with different layout (e.g.
+            # http://www.arte.tv/guide/en/053330-002-A/carnival-italy?zone=world)
+            if not iframe_url:
+                embed_html = self._parse_json(
+                    self._search_regex(
+                        r'program\s*:\s*({.+?["\']embed_html["\'].+?}),?\s*\n',
+                        webpage, 'program'),
+                    video_id)['embed_html']
+                iframe_url= find_iframe_url(embed_html)
             json_url = compat_parse_qs(
                 compat_urllib_parse_urlparse(iframe_url).query)['json_url'][0]
         return self._extract_from_json_url(json_url, video_id, lang)

From 44b97450005dc973175e6897e3422a2a63983b36 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 17 Feb 2016 21:53:53 +0600
Subject: [PATCH 4027/6219] [arte] Extend more _VALID_URLs for en and es
 support

---
 youtube_dl/extractor/arte.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index 497899f8d..1f4e0e6a2 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -23,7 +23,7 @@ from ..utils import (
 
 
 class ArteTvIE(InfoExtractor):
-    _VALID_URL = r'http://videos\.arte\.tv/(?P<lang>fr|de)/.*-(?P<id>.*?)\.html'
+    _VALID_URL = r'http://videos\.arte\.tv/(?P<lang>fr|de|en|es)/.*-(?P<id>.*?)\.html'
     IE_NAME = 'arte.tv'
 
     def _real_extract(self, url):
@@ -195,7 +195,7 @@ class ArteTVPlus7IE(InfoExtractor):
 # It also uses the arte_vp_url url from the webpage to extract the information
 class ArteTVCreativeIE(ArteTVPlus7IE):
     IE_NAME = 'arte.tv:creative'
-    _VALID_URL = r'https?://creative\.arte\.tv/(?P<lang>fr|de)/(?:magazine?/)?(?P<id>[^?#]+)'
+    _VALID_URL = r'https?://creative\.arte\.tv/(?P<lang>fr|de|en|es)/(?:magazine?/)?(?P<id>[^?#]+)'
 
     _TESTS = [{
         'url': 'http://creative.arte.tv/de/magazin/agentur-amateur-corporate-design',
@@ -219,7 +219,7 @@ class ArteTVCreativeIE(ArteTVPlus7IE):
 
 class ArteTVFutureIE(ArteTVPlus7IE):
     IE_NAME = 'arte.tv:future'
-    _VALID_URL = r'https?://future\.arte\.tv/(?P<lang>fr|de)/(?P<id>.+)'
+    _VALID_URL = r'https?://future\.arte\.tv/(?P<lang>fr|de|en|es)/(?P<id>.+)'
 
     _TESTS = [{
         'url': 'http://future.arte.tv/fr/info-sciences/les-ecrevisses-aussi-sont-anxieuses',
@@ -254,7 +254,7 @@ class ArteTVDDCIE(ArteTVPlus7IE):
 
 class ArteTVConcertIE(ArteTVPlus7IE):
     IE_NAME = 'arte.tv:concert'
-    _VALID_URL = r'https?://concert\.arte\.tv/(?P<lang>de|fr)/(?P<id>.+)'
+    _VALID_URL = r'https?://concert\.arte\.tv/(?P<lang>fr|de|en|es)/(?P<id>.+)'
 
     _TEST = {
         'url': 'http://concert.arte.tv/de/notwist-im-pariser-konzertclub-divan-du-monde',
@@ -271,7 +271,7 @@ class ArteTVConcertIE(ArteTVPlus7IE):
 
 class ArteTVCinemaIE(ArteTVPlus7IE):
     IE_NAME = 'arte.tv:cinema'
-    _VALID_URL = r'https?://cinema\.arte\.tv/(?P<lang>de|fr)/(?P<id>.+)'
+    _VALID_URL = r'https?://cinema\.arte\.tv/(?P<lang>fr|de|en|es)/(?P<id>.+)'
 
     _TEST = {
         'url': 'http://cinema.arte.tv/de/node/38291',

From 08d65046f057272155cf6be4a8329ecf3274596f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 17 Feb 2016 22:37:05 +0600
Subject: [PATCH 4028/6219] [arte] Make sorting aware of en/es formats

---
 youtube_dl/extractor/arte.py | 27 +++++++++++++++------------
 1 file changed, 15 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index 1f4e0e6a2..4a82a106f 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -142,27 +142,30 @@ class ArteTVPlus7IE(InfoExtractor):
         }
         qfunc = qualities(['HQ', 'MQ', 'EQ', 'SQ'])
 
+        LANGS = {
+            'fr': 'F',
+            'de': 'A',
+            'en': 'E[ANG]',
+            'es': 'E[ESP]',
+        }
+
         formats = []
         for format_id, format_dict in player_info['VSR'].items():
             f = dict(format_dict)
             versionCode = f.get('versionCode')
-
-            langcode = {
-                'fr': 'F',
-                'de': 'A',
-            }.get(lang, lang)
-            lang_rexs = [r'VO?%s' % langcode, r'VO?.-ST%s' % langcode]
-            lang_pref = (
-                None if versionCode is None else (
-                    10 if any(re.match(r, versionCode) for r in lang_rexs)
-                    else -10))
+            langcode = LANGS.get(lang, lang)
+            lang_rexs = [r'VO?%s-' % re.escape(langcode), r'VO?.-ST%s$' % re.escape(langcode)]
+            lang_pref = None
+            if versionCode:
+                matched_lang_rexs = [r for r in lang_rexs if re.match(r, versionCode)]
+                lang_pref = -10 if not matched_lang_rexs else 10 * len(matched_lang_rexs)
             source_pref = 0
             if versionCode is not None:
                 # The original version with subtitles has lower relevance
-                if re.match(r'VO-ST(F|A)', versionCode):
+                if re.match(r'VO-ST(F|A|E)', versionCode):
                     source_pref -= 10
                 # The version with sourds/mal subtitles has also lower relevance
-                elif re.match(r'VO?(F|A)-STM\1', versionCode):
+                elif re.match(r'VO?(F|A|E)-STM\1', versionCode):
                     source_pref -= 9
             format = {
                 'format_id': format_id,

From e725e4bced92f4eb6c406d6ce01fa361c972fe30 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 17 Feb 2016 22:37:55 +0600
Subject: [PATCH 4029/6219] [arte] PEP 8

---
 youtube_dl/extractor/arte.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index 4a82a106f..5ec67bc9c 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -115,7 +115,7 @@ class ArteTVPlus7IE(InfoExtractor):
                         r'program\s*:\s*({.+?["\']embed_html["\'].+?}),?\s*\n',
                         webpage, 'program'),
                     video_id)['embed_html']
-                iframe_url= find_iframe_url(embed_html)
+                iframe_url = find_iframe_url(embed_html)
             json_url = compat_parse_qs(
                 compat_urllib_parse_urlparse(iframe_url).query)['json_url'][0]
         return self._extract_from_json_url(json_url, video_id, lang)

From 8bbd3d14765a836f72beb2d9dc53832872845669 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 17 Feb 2016 22:51:08 +0600
Subject: [PATCH 4030/6219] [arte] Fix upload date extraction (Closes #8581)

---
 youtube_dl/extractor/arte.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index 5ec67bc9c..c416f82ee 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -126,7 +126,7 @@ class ArteTVPlus7IE(InfoExtractor):
 
         upload_date_str = player_info.get('shootingDate')
         if not upload_date_str:
-            upload_date_str = player_info.get('VDA', '').split(' ')[0]
+            upload_date_str = (player_info.get('VRA') or player_info.get('VDA') or '').split(' ')[0]
 
         title = player_info['VTI'].strip()
         subtitle = player_info.get('VSU', '').strip()

From 80e59a0d5d397c46c73d52a4e2c1ec6af3b9b1e7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 17 Feb 2016 23:18:23 +0600
Subject: [PATCH 4031/6219] [vrt] Make formats extraction non fatal (Closes
 #8587)

---
 youtube_dl/extractor/vrt.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vrt.py b/youtube_dl/extractor/vrt.py
index 01891ac4c..522e2bbc8 100644
--- a/youtube_dl/extractor/vrt.py
+++ b/youtube_dl/extractor/vrt.py
@@ -73,11 +73,12 @@ class VRTIE(InfoExtractor):
         if mobj:
             formats.extend(self._extract_m3u8_formats(
                 '%s/%s' % (mobj.group('server'), mobj.group('path')),
-                video_id, 'mp4', m3u8_id='hls'))
+                video_id, 'mp4', m3u8_id='hls', fatal=False))
         mobj = re.search(r'data-video-src="(?P<src>[^"]+)"', webpage)
         if mobj:
             formats.extend(self._extract_f4m_formats(
-                '%s/manifest.f4m' % mobj.group('src'), video_id, f4m_id='hds'))
+                '%s/manifest.f4m' % mobj.group('src'),
+                video_id, f4m_id='hds', fatal=False))
         self._sort_formats(formats)
 
         title = self._og_search_title(webpage)

From 63bf4f0dc0502a87c19863c2509a4bc28c7c1abb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 17 Feb 2016 23:28:41 +0600
Subject: [PATCH 4032/6219] [vrt] Detect geo restriction

---
 youtube_dl/extractor/vrt.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/vrt.py b/youtube_dl/extractor/vrt.py
index 522e2bbc8..2b6bae89b 100644
--- a/youtube_dl/extractor/vrt.py
+++ b/youtube_dl/extractor/vrt.py
@@ -79,6 +79,10 @@ class VRTIE(InfoExtractor):
             formats.extend(self._extract_f4m_formats(
                 '%s/manifest.f4m' % mobj.group('src'),
                 video_id, f4m_id='hds', fatal=False))
+
+        if not formats and 'data-video-geoblocking="true"' in webpage:
+            self.raise_geo_restricted('This video is only available in Belgium')
+
         self._sort_formats(formats)
 
         title = self._og_search_title(webpage)

From 411cb8f4769215ea6835895206782d8ec00a5114 Mon Sep 17 00:00:00 2001
From: cazulu <jvlarapeinado@gmail.com>
Date: Tue, 16 Feb 2016 18:45:53 +0900
Subject: [PATCH 4033/6219] [dailymotion] Fix view count extraction

Fix view count parsing when the decimal marker is a whitespace, e.g. '101 101'
---
 youtube_dl/extractor/dailymotion.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index 7ae9f2359..2e6226ea0 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -122,10 +122,13 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
         description = self._og_search_description(webpage) or self._html_search_meta(
             'description', webpage, 'description')
 
-        view_count = str_to_int(self._search_regex(
-            [r'<meta[^>]+itemprop="interactionCount"[^>]+content="UserPlays:(\d+)"',
-             r'video_views_count[^>]+>\s+([\d\.,]+)'],
-            webpage, 'view count', fatal=False))
+        view_count_str = self._search_regex(
+            (r'<meta[^>]+itemprop="interactionCount"[^>]+content="UserPlays:([\s\d,.]+)"',
+             r'video_views_count[^>]+>\s+([\s\d\,.]+)'),
+            webpage, 'view count', fatal=False)
+        if view_count_str:
+            view_count_str = re.sub(r'\s', '', view_count_str)
+        view_count = str_to_int(view_count_str)
         comment_count = int_or_none(self._search_regex(
             r'<meta[^>]+itemprop="interactionCount"[^>]+content="UserComments:(\d+)"',
             webpage, 'comment count', fatal=False))

From 778433cb90064a3fbe21afb722c787841e85ddf4 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 18 Feb 2016 15:43:28 +0100
Subject: [PATCH 4034/6219] [cbsnews] extract subtitle url from theplatform
 SMIL manifest(fixes #8568)

---
 youtube_dl/extractor/cbsnews.py | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/cbsnews.py b/youtube_dl/extractor/cbsnews.py
index 8f864699f..f0de73aeb 100644
--- a/youtube_dl/extractor/cbsnews.py
+++ b/youtube_dl/extractor/cbsnews.py
@@ -46,6 +46,15 @@ class CBSNewsIE(ThePlatformIE):
         },
     ]
 
+    def _parse_smil_subtitles(self, smil, namespace=None, subtitles_lang='en'):
+        closed_caption_e = smil.find(self._xpath_ns('.//param[@name=\'ClosedCaptionURL\']', namespace))
+        return {
+            'en': [{
+                'ext': 'ttml',
+                'url': closed_caption_e.attrib['value'],
+            }]
+        } if closed_caption_e is not None and closed_caption_e.attrib.get('value') else []
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
@@ -61,12 +70,6 @@ class CBSNewsIE(ThePlatformIE):
         thumbnail = item.get('mediaImage') or item.get('thumbnail')
 
         subtitles = {}
-        if 'mpxRefId' in video_info:
-            subtitles['en'] = [{
-                'ext': 'ttml',
-                'url': 'http://www.cbsnews.com/videos/captions/%s.adb_xml' % video_info['mpxRefId'],
-            }]
-
         formats = []
         for format_id in ['RtmpMobileLow', 'RtmpMobileHigh', 'Hls', 'RtmpDesktop']:
             pid = item.get('media' + format_id)

From 8b809a079aa5b55d39029ee00c706e3c67ed311a Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 18 Feb 2016 16:10:09 +0100
Subject: [PATCH 4035/6219] [cbsnews] use find_xpath_attr

---
 youtube_dl/extractor/cbsnews.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/cbsnews.py b/youtube_dl/extractor/cbsnews.py
index f0de73aeb..7319ee1b7 100644
--- a/youtube_dl/extractor/cbsnews.py
+++ b/youtube_dl/extractor/cbsnews.py
@@ -3,7 +3,10 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from .theplatform import ThePlatformIE
-from ..utils import parse_duration
+from ..utils import (
+    parse_duration,
+    find_xpath_attr,
+)
 
 
 class CBSNewsIE(ThePlatformIE):
@@ -47,7 +50,7 @@ class CBSNewsIE(ThePlatformIE):
     ]
 
     def _parse_smil_subtitles(self, smil, namespace=None, subtitles_lang='en'):
-        closed_caption_e = smil.find(self._xpath_ns('.//param[@name=\'ClosedCaptionURL\']', namespace))
+        closed_caption_e = find_xpath_attr(smil, self._xpath_ns('.//param', namespace), 'name', 'ClosedCaptionURL')
         return {
             'en': [{
                 'ext': 'ttml',

From 49625662a94bec2d289ed831f7e024c4c43098fd Mon Sep 17 00:00:00 2001
From: Thomas Jost <schnouki@schnouki.net>
Date: Mon, 8 Feb 2016 22:17:21 +0100
Subject: [PATCH 4036/6219] [arte:magazine] Add extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/arte.py     | 23 +++++++++++++++++++++++
 2 files changed, 24 insertions(+)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 537d25777..4049141d9 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -45,6 +45,7 @@ from .arte import (
     ArteTVFutureIE,
     ArteTVCinemaIE,
     ArteTVDDCIE,
+    ArteTVMagazineIE,
     ArteTVEmbedIE,
 )
 from .atresplayer import AtresPlayerIE
diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index c416f82ee..b524866d9 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -289,6 +289,29 @@ class ArteTVCinemaIE(ArteTVPlus7IE):
     }
 
 
+class ArteTVMagazineIE(ArteTVPlus7IE):
+    IE_NAME = 'arte.tv:magazine'
+    _VALID_URL = r'https?://(?:www\.)?arte\.tv/magazine/(?P<name>.*?)/(?P<lang>fr|de)/(?P<id>.*?)'
+
+    _TESTS = [{
+        'url': 'http://www.arte.tv/magazine/trepalium/fr/entretien-avec-le-realisateur-vincent-lannoo-trepalium',
+        'md5': '66a093339c1278bb3719157ef07107b2',
+        'info_dict': {
+            'id': '065965-000-A',
+            'ext': 'mp4',
+            'title': 'Trepalium - Extrait Ep.01',
+        },
+    }, {
+        'url': 'http://www.arte.tv/magazine/metropolis/de/frank-woeste-german-paris-metropolis',
+        'md5': '53185a2b4876383552c037c21aeca8f7',
+        'info_dict': {
+            'id': '066149-000-A',
+            'ext': 'mp4',
+            'title': 'Bonus: Frank Woeste - ein deutscher Jazzmusiker in Paris',
+        },
+    }]
+
+
 class ArteTVEmbedIE(ArteTVPlus7IE):
     IE_NAME = 'arte.tv:embed'
     _VALID_URL = r'''(?x)

From 479ec54a8d9631215483f144c3d1b7ea830178b5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 18 Feb 2016 21:29:07 +0600
Subject: [PATCH 4037/6219] [arte:magazine] Improve (Closes #8473)

---
 youtube_dl/extractor/arte.py | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index b524866d9..3337d0085 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -291,7 +291,7 @@ class ArteTVCinemaIE(ArteTVPlus7IE):
 
 class ArteTVMagazineIE(ArteTVPlus7IE):
     IE_NAME = 'arte.tv:magazine'
-    _VALID_URL = r'https?://(?:www\.)?arte\.tv/magazine/(?P<name>.*?)/(?P<lang>fr|de)/(?P<id>.*?)'
+    _VALID_URL = r'https?://(?:www\.)?arte\.tv/magazine/[^/]+/(?P<lang>fr|de|en|es)/(?P<id>[^/?#&]+)'
 
     _TESTS = [{
         'url': 'http://www.arte.tv/magazine/trepalium/fr/entretien-avec-le-realisateur-vincent-lannoo-trepalium',
@@ -303,12 +303,7 @@ class ArteTVMagazineIE(ArteTVPlus7IE):
         },
     }, {
         'url': 'http://www.arte.tv/magazine/metropolis/de/frank-woeste-german-paris-metropolis',
-        'md5': '53185a2b4876383552c037c21aeca8f7',
-        'info_dict': {
-            'id': '066149-000-A',
-            'ext': 'mp4',
-            'title': 'Bonus: Frank Woeste - ein deutscher Jazzmusiker in Paris',
-        },
+        'only_matching': True,
     }]
 
 

From d48502b82a3466dc3f75dc35815bfe22a934aabf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 18 Feb 2016 21:29:52 +0600
Subject: [PATCH 4038/6219] [arte] Improve _VALID_URLs

---
 youtube_dl/extractor/arte.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index 3337d0085..793da2ee1 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -63,7 +63,7 @@ class ArteTvIE(InfoExtractor):
 
 class ArteTVPlus7IE(InfoExtractor):
     IE_NAME = 'arte.tv:+7'
-    _VALID_URL = r'https?://(?:www\.)?arte\.tv/guide/(?P<lang>fr|de|en|es)/(?:(?:sendungen|emissions)/)?(?P<id>.*?)/(?P<name>.*?)(\?.*)?'
+    _VALID_URL = r'https?://(?:www\.)?arte\.tv/guide/(?P<lang>fr|de|en|es)/(?:(?:sendungen|emissions)/)?(?P<id>[^/]+)/(?P<name>[^/?#&+])'
 
     @classmethod
     def _extract_url_info(cls, url):
@@ -198,7 +198,7 @@ class ArteTVPlus7IE(InfoExtractor):
 # It also uses the arte_vp_url url from the webpage to extract the information
 class ArteTVCreativeIE(ArteTVPlus7IE):
     IE_NAME = 'arte.tv:creative'
-    _VALID_URL = r'https?://creative\.arte\.tv/(?P<lang>fr|de|en|es)/(?:magazine?/)?(?P<id>[^?#]+)'
+    _VALID_URL = r'https?://creative\.arte\.tv/(?P<lang>fr|de|en|es)/(?:magazine?/)?(?P<id>[^/?#&]+)'
 
     _TESTS = [{
         'url': 'http://creative.arte.tv/de/magazin/agentur-amateur-corporate-design',
@@ -222,7 +222,7 @@ class ArteTVCreativeIE(ArteTVPlus7IE):
 
 class ArteTVFutureIE(ArteTVPlus7IE):
     IE_NAME = 'arte.tv:future'
-    _VALID_URL = r'https?://future\.arte\.tv/(?P<lang>fr|de|en|es)/(?P<id>.+)'
+    _VALID_URL = r'https?://future\.arte\.tv/(?P<lang>fr|de|en|es)/(?P<id>[^/?#&]+)'
 
     _TESTS = [{
         'url': 'http://future.arte.tv/fr/info-sciences/les-ecrevisses-aussi-sont-anxieuses',
@@ -239,7 +239,7 @@ class ArteTVFutureIE(ArteTVPlus7IE):
 
 class ArteTVDDCIE(ArteTVPlus7IE):
     IE_NAME = 'arte.tv:ddc'
-    _VALID_URL = r'https?://ddc\.arte\.tv/(?P<lang>emission|folge)/(?P<id>.+)'
+    _VALID_URL = r'https?://ddc\.arte\.tv/(?P<lang>emission|folge)/(?P<id>[^/?#&]+)'
 
     def _real_extract(self, url):
         video_id, lang = self._extract_url_info(url)
@@ -257,7 +257,7 @@ class ArteTVDDCIE(ArteTVPlus7IE):
 
 class ArteTVConcertIE(ArteTVPlus7IE):
     IE_NAME = 'arte.tv:concert'
-    _VALID_URL = r'https?://concert\.arte\.tv/(?P<lang>fr|de|en|es)/(?P<id>.+)'
+    _VALID_URL = r'https?://concert\.arte\.tv/(?P<lang>fr|de|en|es)/(?P<id>[^/?#&]+)'
 
     _TEST = {
         'url': 'http://concert.arte.tv/de/notwist-im-pariser-konzertclub-divan-du-monde',

From bc7a9cd8fb223481593840441d20c0f19f93c4c8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 18 Feb 2016 21:50:21 +0600
Subject: [PATCH 4039/6219] [youtube:watchlater] Improve _VALID_URL (Closes
 #8594)

---
 youtube_dl/extractor/youtube.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 45ad88152..408c8d52c 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -2026,9 +2026,15 @@ class YoutubeFeedsInfoExtractor(YoutubeBaseInfoExtractor):
 class YoutubeWatchLaterIE(YoutubePlaylistIE):
     IE_NAME = 'youtube:watchlater'
     IE_DESC = 'Youtube watch later list, ":ytwatchlater" for short (requires authentication)'
-    _VALID_URL = r'https?://www\.youtube\.com/(?:feed/watch_later|playlist\?list=WL)|:ytwatchlater'
+    _VALID_URL = r'https?://www\.youtube\.com/(?:feed/watch_later|(?:playlist|watch)\?(?:.+&)?list=WL)|:ytwatchlater'
 
-    _TESTS = []  # override PlaylistIE tests
+    _TESTS = [{
+        'url': 'https://www.youtube.com/playlist?list=WL',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.youtube.com/watch?v=bCNU9TrbiRk&index=1&list=WL',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         return self._extract_playlist('WL')

From ebf1b291d0fc16612fdac877c1242f1cb5761277 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 18 Feb 2016 22:03:46 +0600
Subject: [PATCH 4040/6219] [youtube:watchlater] Respect --no-playlist

---
 youtube_dl/extractor/youtube.py | 22 +++++++++++++++-------
 1 file changed, 15 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 408c8d52c..e24dd3e5b 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1701,13 +1701,7 @@ class YoutubePlaylistIE(YoutubePlaylistBaseInfoExtractor):
 
         return self.playlist_result(self._entries(page, playlist_id), playlist_id, playlist_title)
 
-    def _real_extract(self, url):
-        # Extract playlist id
-        mobj = re.match(self._VALID_URL, url)
-        if mobj is None:
-            raise ExtractorError('Invalid URL: %s' % url)
-        playlist_id = mobj.group(1) or mobj.group(2)
-
+    def _check_download_just_video(self, url, playlist_id):
         # Check if it's a video-specific URL
         query_dict = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
         if 'v' in query_dict:
@@ -1718,6 +1712,17 @@ class YoutubePlaylistIE(YoutubePlaylistBaseInfoExtractor):
             else:
                 self.to_screen('Downloading playlist %s - add --no-playlist to just download video %s' % (playlist_id, video_id))
 
+    def _real_extract(self, url):
+        # Extract playlist id
+        mobj = re.match(self._VALID_URL, url)
+        if mobj is None:
+            raise ExtractorError('Invalid URL: %s' % url)
+        playlist_id = mobj.group(1) or mobj.group(2)
+
+        video = self._check_download_just_video(url, playlist_id)
+        if video:
+            return video
+
         if playlist_id.startswith('RD') or playlist_id.startswith('UL'):
             # Mixes require a custom extraction process
             return self._extract_mix(playlist_id)
@@ -2037,6 +2042,9 @@ class YoutubeWatchLaterIE(YoutubePlaylistIE):
     }]
 
     def _real_extract(self, url):
+        video = self._check_download_just_video(url, 'WL')
+        if video:
+            return video
         return self._extract_playlist('WL')
 
 

From f3f9cd9234aada1d677517e3042bd9312610081d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 18 Feb 2016 22:09:21 +0600
Subject: [PATCH 4041/6219] [francetv] Improve video id regex (Closes #8563)

---
 youtube_dl/extractor/francetv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index 8e60cf60f..3f4ac3093 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -289,7 +289,7 @@ class FranceTVIE(FranceTVBaseInfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
         video_id, catalogue = self._html_search_regex(
-            r'href="http://videos?\.francetv\.fr/video/([^@]+@[^"]+)"',
+            r'(?:href=|player\.setVideo\(\s*)"http://videos?\.francetv\.fr/video/([^@]+@[^"]+)"',
             webpage, 'video ID').split('@')
         return self._extract_video(video_id, catalogue)
 

From 40e146aa1e1a8fd57d3f84b0a541174f56fa5dba Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 18 Feb 2016 22:29:17 +0600
Subject: [PATCH 4042/6219] [pornhub:user:videos] Add extractor (Closes #8548)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/pornhub.py  | 55 ++++++++++++++++++++++----------
 2 files changed, 40 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 4049141d9..1edbfbd28 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -555,6 +555,7 @@ from .pornhd import PornHdIE
 from .pornhub import (
     PornHubIE,
     PornHubPlaylistIE,
+    PornHubUserVideosIE,
 )
 from .pornotube import PornotubeIE
 from .pornovoisines import PornoVoisinesIE
diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index 91e574dc2..405dbf006 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -129,7 +129,31 @@ class PornHubIE(InfoExtractor):
         }
 
 
-class PornHubPlaylistIE(InfoExtractor):
+class PornHubPlaylistBaseIE(InfoExtractor):
+    def _extract_entries(self, webpage):
+        return [
+            self.url_result('http://www.pornhub.com/%s' % video_url, 'PornHub')
+            for video_url in set(re.findall(
+                r'href="/?(view_video\.php\?.*\bviewkey=[\da-z]+[^"]*)"', webpage))
+        ]
+
+    def _real_extract(self, url):
+        playlist_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, playlist_id)
+
+        entries = self._extract_entries(webpage)
+
+        playlist = self._parse_json(
+            self._search_regex(
+                r'playlistObject\s*=\s*({.+?});', webpage, 'playlist'),
+            playlist_id)
+
+        return self.playlist_result(
+            entries, playlist_id, playlist.get('title'), playlist.get('description'))
+
+
+class PornHubPlaylistIE(PornHubPlaylistBaseIE):
     _VALID_URL = r'https?://(?:www\.)?pornhub\.com/playlist/(?P<id>\d+)'
     _TESTS = [{
         'url': 'http://www.pornhub.com/playlist/6201671',
@@ -140,21 +164,20 @@ class PornHubPlaylistIE(InfoExtractor):
         'playlist_mincount': 35,
     }]
 
+
+class PornHubUserVideosIE(PornHubPlaylistBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?pornhub\.com/users/(?P<id>[^/]+)/videos'
+    _TESTS = [{
+        'url': 'http://www.pornhub.com/users/rushandlia/videos',
+        'info_dict': {
+            'id': 'rushandlia',
+        },
+        'playlist_mincount': 13,
+    }]
+
     def _real_extract(self, url):
-        playlist_id = self._match_id(url)
+        user_id = self._match_id(url)
 
-        webpage = self._download_webpage(url, playlist_id)
+        webpage = self._download_webpage(url, user_id)
 
-        entries = [
-            self.url_result('http://www.pornhub.com/%s' % video_url, 'PornHub')
-            for video_url in set(re.findall(
-                r'href="/?(view_video\.php\?.*\bviewkey=[\da-z]+[^"]*)"', webpage))
-        ]
-
-        playlist = self._parse_json(
-            self._search_regex(
-                r'playlistObject\s*=\s*({.+?});', webpage, 'playlist'),
-            playlist_id)
-
-        return self.playlist_result(
-            entries, playlist_id, playlist.get('title'), playlist.get('description'))
+        return self.playlist_result(self._extract_entries(webpage), user_id)

From 88641243abfdc5673359269574bf7c76191966bc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 18 Feb 2016 22:30:19 +0600
Subject: [PATCH 4043/6219] [pornhub:playlistbase] Improve extract entries

---
 youtube_dl/extractor/pornhub.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index 405dbf006..55af13324 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -132,7 +132,7 @@ class PornHubIE(InfoExtractor):
 class PornHubPlaylistBaseIE(InfoExtractor):
     def _extract_entries(self, webpage):
         return [
-            self.url_result('http://www.pornhub.com/%s' % video_url, 'PornHub')
+            self.url_result('http://www.pornhub.com/%s' % video_url, PornHubIE.ie_key())
             for video_url in set(re.findall(
                 r'href="/?(view_video\.php\?.*\bviewkey=[\da-z]+[^"]*)"', webpage))
         ]

From ed8648a322f8ec55957db0d4a959868807c38ebb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 19 Feb 2016 21:42:46 +0600
Subject: [PATCH 4044/6219] [pornhub] Fix thumbnail and duration extraction
 (Closes #8604)

---
 youtube_dl/extractor/pornhub.py | 31 ++++++++++++++++++++++++-------
 1 file changed, 24 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index 55af13324..5a55c25e7 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -11,6 +11,7 @@ from ..compat import (
 )
 from ..utils import (
     ExtractorError,
+    int_or_none,
     sanitized_Request,
     str_to_int,
 )
@@ -23,13 +24,18 @@ class PornHubIE(InfoExtractor):
     _VALID_URL = r'https?://(?:[a-z]+\.)?pornhub\.com/(?:view_video\.php\?viewkey=|embed/)(?P<id>[0-9a-z]+)'
     _TESTS = [{
         'url': 'http://www.pornhub.com/view_video.php?viewkey=648719015',
-        'md5': '882f488fa1f0026f023f33576004a2ed',
+        'md5': '1e19b41231a02eba417839222ac9d58e',
         'info_dict': {
             'id': '648719015',
             'ext': 'mp4',
-            'uploader': 'Babes',
             'title': 'Seductive Indian beauty strips down and fingers her pink pussy',
-            'age_limit': 18
+            'uploader': 'Babes',
+            'duration': 361,
+            'view_count': int,
+            'like_count': int,
+            'dislike_count': int,
+            'comment_count': int,
+            'age_limit': 18,
         }
     }, {
         'url': 'http://www.pornhub.com/view_video.php?viewkey=ph557bbb6676d2d',
@@ -67,13 +73,23 @@ class PornHubIE(InfoExtractor):
                 'PornHub said: %s' % error_msg,
                 expected=True, video_id=video_id)
 
-        video_title = self._html_search_regex(r'<h1 [^>]+>([^<]+)', webpage, 'title')
+        flashvars = self._parse_json(
+            self._search_regex(
+                r'var\s+flashv1ars_\d+\s*=\s*({.+?});', webpage, 'flashvars', default='{}'),
+            video_id)
+        if flashvars:
+            video_title = flashvars.get('video_title')
+            thumbnail = flashvars.get('image_url')
+            duration = int_or_none(flashvars.get('video_duration'))
+        else:
+            video_title, thumbnail, duration = [None] * 3
+
+        if not video_title:
+            video_title = self._html_search_regex(r'<h1 [^>]+>([^<]+)', webpage, 'title')
+
         video_uploader = self._html_search_regex(
             r'(?s)From:&nbsp;.+?<(?:a href="/users/|a href="/channels/|span class="username)[^>]+>(.+?)<',
             webpage, 'uploader', fatal=False)
-        thumbnail = self._html_search_regex(r'"image_url":"([^"]+)', webpage, 'thumbnail', fatal=False)
-        if thumbnail:
-            thumbnail = compat_urllib_parse_unquote(thumbnail)
 
         view_count = self._extract_count(
             r'<span class="count">([\d,\.]+)</span> views', webpage, 'view')
@@ -120,6 +136,7 @@ class PornHubIE(InfoExtractor):
             'uploader': video_uploader,
             'title': video_title,
             'thumbnail': thumbnail,
+            'duration': duration,
             'view_count': view_count,
             'like_count': like_count,
             'dislike_count': dislike_count,

From 8ec64ac68354cdf9428cd58506481ef4476c47c9 Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Fri, 19 Feb 2016 22:18:21 +0600
Subject: [PATCH 4045/6219] [README.md] Clarify verbose log

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 65e0ee963..db49ab6d5 100644
--- a/README.md
+++ b/README.md
@@ -935,9 +935,9 @@ with youtube_dl.YoutubeDL(ydl_opts) as ydl:
 
 Bugs and suggestions should be reported at: <https://github.com/rg3/youtube-dl/issues>. Unless you were prompted so or there is another pertinent reason (e.g. GitHub fails to accept the bug report), please do not send bug reports via personal email. For discussions, join us in the IRC channel [#youtube-dl](irc://chat.freenode.net/#youtube-dl) on freenode ([webchat](http://webchat.freenode.net/?randomnick=1&channels=youtube-dl)).
 
-**Please include the full output of youtube-dl when run with `-v`**, i.e. add `-v` flag to your command line, copy the **whole** output and post it in the issue body wrapped in \`\`\` for better formatting. It should look similar to this:
+**Please include the full output of youtube-dl when run with `-v`**, i.e. **add** `-v` flag to **your command line**, copy the **whole** output and post it in the issue body wrapped in \`\`\` for better formatting. It should look similar to this:
 ```
-$ youtube-dl -v http://www.youtube.com/watch?v=BaW_jenozKcj
+$ youtube-dl -v <your command line>
 [debug] System config: []
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']

From 12b84ac8c13754baeeead907d8c9d239141f8706 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 19 Feb 2016 19:29:24 +0100
Subject: [PATCH 4046/6219] [downloader/external] Add FFmpegFD(fixes #622)

- replace HlsFD and RtspFD
- add basic support for downloading part of the video or audio
---
 youtube_dl/downloader/__init__.py | 20 +++++----
 youtube_dl/downloader/external.py | 65 +++++++++++++++++++++++++++
 youtube_dl/downloader/hls.py      | 74 ++-----------------------------
 youtube_dl/downloader/rtsp.py     | 45 -------------------
 4 files changed, 80 insertions(+), 124 deletions(-)
 delete mode 100644 youtube_dl/downloader/rtsp.py

diff --git a/youtube_dl/downloader/__init__.py b/youtube_dl/downloader/__init__.py
index dccc59212..bb6afb1f8 100644
--- a/youtube_dl/downloader/__init__.py
+++ b/youtube_dl/downloader/__init__.py
@@ -1,14 +1,15 @@
 from __future__ import unicode_literals
 
 from .common import FileDownloader
-from .external import get_external_downloader
 from .f4m import F4mFD
 from .hls import HlsFD
-from .hls import NativeHlsFD
 from .http import HttpFD
-from .rtsp import RtspFD
 from .rtmp import RtmpFD
 from .dash import DashSegmentsFD
+from .external import (
+    get_external_downloader,
+    FFmpegFD,
+)
 
 from ..utils import (
     determine_protocol,
@@ -16,10 +17,10 @@ from ..utils import (
 
 PROTOCOL_MAP = {
     'rtmp': RtmpFD,
-    'm3u8_native': NativeHlsFD,
-    'm3u8': HlsFD,
-    'mms': RtspFD,
-    'rtsp': RtspFD,
+    'm3u8_native': HlsFD,
+    'm3u8': FFmpegFD,
+    'mms': FFmpegFD,
+    'rtsp': FFmpegFD,
     'f4m': F4mFD,
     'http_dash_segments': DashSegmentsFD,
 }
@@ -30,6 +31,9 @@ def get_suitable_downloader(info_dict, params={}):
     protocol = determine_protocol(info_dict)
     info_dict['protocol'] = protocol
 
+    if (info_dict.get('start_time') or info_dict.get('end_time')) and FFmpegFD.supports(info_dict):
+        return FFmpegFD
+
     external_downloader = params.get('external_downloader')
     if external_downloader is not None:
         ed = get_external_downloader(external_downloader)
@@ -37,7 +41,7 @@ def get_suitable_downloader(info_dict, params={}):
             return ed
 
     if protocol == 'm3u8' and params.get('hls_prefer_native'):
-        return NativeHlsFD
+        return HlsFD
 
     return PROTOCOL_MAP.get(protocol, HttpFD)
 
diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index 2bc011266..bb43677b7 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -2,8 +2,12 @@ from __future__ import unicode_literals
 
 import os.path
 import subprocess
+import sys
+import re
 
 from .common import FileDownloader
+from ..postprocessor.ffmpeg import FFmpegPostProcessor
+from ..compat import compat_str
 from ..utils import (
     cli_option,
     cli_valueless_option,
@@ -11,6 +15,7 @@ from ..utils import (
     cli_configuration_args,
     encodeFilename,
     encodeArgument,
+    handle_youtubedl_headers,
 )
 
 
@@ -136,6 +141,66 @@ class HttpieFD(ExternalFD):
             cmd += ['%s:%s' % (key, val)]
         return cmd
 
+
+class FFmpegFD(ExternalFD):
+    @classmethod
+    def supports(cls, info_dict):
+        return info_dict['protocol'] in ('http', 'https', 'ftp', 'ftps', 'm3u8', 'rtsp', 'rtmp', 'mms')
+
+    def _call_downloader(self, tmpfilename, info_dict):
+        url = info_dict['url']
+        ffpp = FFmpegPostProcessor(downloader=self)
+        ffpp.check_version()
+
+        args = [ffpp.executable, '-y']
+
+        start_time = info_dict.get('start_time', 0)
+        if start_time:
+            args += ['-ss', compat_str(start_time)]
+        end_time = info_dict.get('end_time')
+        if end_time:
+            args += ['-t', compat_str(end_time - start_time)]
+
+        if info_dict['http_headers'] and re.match(r'^https?://', url):
+            # Trailing \r\n after each HTTP header is important to prevent warning from ffmpeg/avconv:
+            # [http @ 00000000003d2fa0] No trailing CRLF found in HTTP header.
+            headers = handle_youtubedl_headers(info_dict['http_headers'])
+            args += [
+                '-headers',
+                ''.join('%s: %s\r\n' % (key, val) for key, val in headers.items())]
+
+        args += ['-i', url, '-c', 'copy']
+        if info_dict.get('protocol') == 'm3u8':
+            if self.params.get('hls_use_mpegts', False):
+                args += ['-f', 'mpegts']
+            else:
+                args += ['-f', 'mp4', '-bsf:a', 'aac_adtstoasc']
+        else:
+            args += ['-f', info_dict['ext']]
+
+        args = [encodeArgument(opt) for opt in args]
+        args.append(encodeFilename(ffpp._ffmpeg_filename_argument(tmpfilename), True))
+
+        self._debug_cmd(args)
+
+        proc = subprocess.Popen(args, stdin=subprocess.PIPE)
+        try:
+            retval = proc.wait()
+        except KeyboardInterrupt:
+            # subprocces.run would send the SIGKILL signal to ffmpeg and the
+            # mp4 file couldn't be played, but if we ask ffmpeg to quit it
+            # produces a file that is playable (this is mostly useful for live
+            # streams). Note that Windows is not affected and produces playable
+            # files (see https://github.com/rg3/youtube-dl/issues/8300).
+            if sys.platform != 'win32':
+                proc.communicate(b'q')
+            raise
+        return retval
+
+
+class AVconvFD(FFmpegFD):
+    pass
+
 _BY_NAME = dict(
     (klass.get_basename(), klass)
     for name, klass in globals().items()
diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index 2a775bf00..a01dac031 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -1,87 +1,19 @@
 from __future__ import unicode_literals
 
-import os
+import os.path
 import re
-import subprocess
-import sys
 
-from .common import FileDownloader
 from .fragment import FragmentFD
 
 from ..compat import compat_urlparse
-from ..postprocessor.ffmpeg import FFmpegPostProcessor
 from ..utils import (
-    encodeArgument,
     encodeFilename,
     sanitize_open,
-    handle_youtubedl_headers,
 )
 
 
-class HlsFD(FileDownloader):
-    def real_download(self, filename, info_dict):
-        url = info_dict['url']
-        self.report_destination(filename)
-        tmpfilename = self.temp_name(filename)
-
-        ffpp = FFmpegPostProcessor(downloader=self)
-        if not ffpp.available:
-            self.report_error('m3u8 download detected but ffmpeg or avconv could not be found. Please install one.')
-            return False
-        ffpp.check_version()
-
-        args = [ffpp.executable, '-y']
-
-        if info_dict['http_headers'] and re.match(r'^https?://', url):
-            # Trailing \r\n after each HTTP header is important to prevent warning from ffmpeg/avconv:
-            # [http @ 00000000003d2fa0] No trailing CRLF found in HTTP header.
-            headers = handle_youtubedl_headers(info_dict['http_headers'])
-            args += [
-                '-headers',
-                ''.join('%s: %s\r\n' % (key, val) for key, val in headers.items())]
-
-        args += ['-i', url, '-c', 'copy']
-        if self.params.get('hls_use_mpegts', False):
-            args += ['-f', 'mpegts']
-        else:
-            args += ['-f', 'mp4', '-bsf:a', 'aac_adtstoasc']
-
-        args = [encodeArgument(opt) for opt in args]
-        args.append(encodeFilename(ffpp._ffmpeg_filename_argument(tmpfilename), True))
-
-        self._debug_cmd(args)
-
-        proc = subprocess.Popen(args, stdin=subprocess.PIPE)
-        try:
-            retval = proc.wait()
-        except KeyboardInterrupt:
-            # subprocces.run would send the SIGKILL signal to ffmpeg and the
-            # mp4 file couldn't be played, but if we ask ffmpeg to quit it
-            # produces a file that is playable (this is mostly useful for live
-            # streams). Note that Windows is not affected and produces playable
-            # files (see https://github.com/rg3/youtube-dl/issues/8300).
-            if sys.platform != 'win32':
-                proc.communicate(b'q')
-            raise
-        if retval == 0:
-            fsize = os.path.getsize(encodeFilename(tmpfilename))
-            self.to_screen('\r[%s] %s bytes' % (args[0], fsize))
-            self.try_rename(tmpfilename, filename)
-            self._hook_progress({
-                'downloaded_bytes': fsize,
-                'total_bytes': fsize,
-                'filename': filename,
-                'status': 'finished',
-            })
-            return True
-        else:
-            self.to_stderr('\n')
-            self.report_error('%s exited with code %d' % (ffpp.basename, retval))
-            return False
-
-
-class NativeHlsFD(FragmentFD):
-    """ A more limited implementation that does not require ffmpeg """
+class HlsFD(FragmentFD):
+    """ A limited implementation that does not require ffmpeg """
 
     FD_NAME = 'hlsnative'
 
diff --git a/youtube_dl/downloader/rtsp.py b/youtube_dl/downloader/rtsp.py
deleted file mode 100644
index 3eb29526c..000000000
--- a/youtube_dl/downloader/rtsp.py
+++ /dev/null
@@ -1,45 +0,0 @@
-from __future__ import unicode_literals
-
-import os
-import subprocess
-
-from .common import FileDownloader
-from ..utils import (
-    check_executable,
-    encodeFilename,
-)
-
-
-class RtspFD(FileDownloader):
-    def real_download(self, filename, info_dict):
-        url = info_dict['url']
-        self.report_destination(filename)
-        tmpfilename = self.temp_name(filename)
-
-        if check_executable('mplayer', ['-h']):
-            args = [
-                'mplayer', '-really-quiet', '-vo', 'null', '-vc', 'dummy',
-                '-dumpstream', '-dumpfile', tmpfilename, url]
-        elif check_executable('mpv', ['-h']):
-            args = [
-                'mpv', '-really-quiet', '--vo=null', '--stream-dump=' + tmpfilename, url]
-        else:
-            self.report_error('MMS or RTSP download detected but neither "mplayer" nor "mpv" could be run. Please install any.')
-            return False
-
-        retval = subprocess.call(args)
-        if retval == 0:
-            fsize = os.path.getsize(encodeFilename(tmpfilename))
-            self.to_screen('\r[%s] %s bytes' % (args[0], fsize))
-            self.try_rename(tmpfilename, filename)
-            self._hook_progress({
-                'downloaded_bytes': fsize,
-                'total_bytes': fsize,
-                'filename': filename,
-                'status': 'finished',
-            })
-            return True
-        else:
-            self.to_stderr('\n')
-            self.report_error('%s exited with code %d' % (args[0], retval))
-            return False

From 8765151c8a026f1c57ed239bae15a443339572e3 Mon Sep 17 00:00:00 2001
From: RiCON <RiCON@nowhere>
Date: Fri, 19 Feb 2016 17:52:56 +0000
Subject: [PATCH 4047/6219] [mtv] Extract duration from each playlist item

RSS used instead of manifest files because it's exact to the millisecond
with the video I tested while in manifest it's only exact to the second.
---
 youtube_dl/extractor/mtv.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index e8bb527b8..56ddcbbf3 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -11,6 +11,7 @@ from ..utils import (
     ExtractorError,
     find_xpath_attr,
     fix_xml_ampersands,
+    float_or_none,
     HEADRequest,
     sanitized_Request,
     unescapeHTML,
@@ -158,6 +159,9 @@ class MTVServicesInfoExtractor(InfoExtractor):
         if mtvn_id_node is not None:
             mtvn_id = mtvn_id_node.text
 
+        content_el = find_xpath_attr(itemdoc, self._xpath_ns('.//content', 'http://search.yahoo.com/mrss/'), 'duration')
+        duration = float_or_none(content_el.attrib.get('duration')) if content_el is not None else None
+
         return {
             'title': title,
             'formats': self._extract_video_formats(mediagen_doc, mtvn_id),
@@ -165,6 +169,7 @@ class MTVServicesInfoExtractor(InfoExtractor):
             'id': video_id,
             'thumbnail': self._get_thumbnail_url(uri, itemdoc),
             'description': description,
+            'duration': duration,
         }
 
     def _get_feed_query(self, uri):

From 04cbc4980dcc35ce19c5b75222481f0839b74e98 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 19 Feb 2016 20:56:45 +0100
Subject: [PATCH 4048/6219] [mtv] imporove duration extraction

---
 youtube_dl/extractor/mtv.py | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index 56ddcbbf3..ed068365d 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -111,7 +111,8 @@ class MTVServicesInfoExtractor(InfoExtractor):
         uri = itemdoc.find('guid').text
         video_id = self._id_from_uri(uri)
         self.report_extraction(video_id)
-        mediagen_url = itemdoc.find('%s/%s' % (_media_xml_tag('group'), _media_xml_tag('content'))).attrib['url']
+        content_el = itemdoc.find('%s/%s' % (_media_xml_tag('group'), _media_xml_tag('content')))
+        mediagen_url = content_el.attrib['url']
         # Remove the templates, like &device={device}
         mediagen_url = re.sub(r'&[^=]*?={.*?}(?=(&|$))', '', mediagen_url)
         if 'acceptMethods' not in mediagen_url:
@@ -159,9 +160,6 @@ class MTVServicesInfoExtractor(InfoExtractor):
         if mtvn_id_node is not None:
             mtvn_id = mtvn_id_node.text
 
-        content_el = find_xpath_attr(itemdoc, self._xpath_ns('.//content', 'http://search.yahoo.com/mrss/'), 'duration')
-        duration = float_or_none(content_el.attrib.get('duration')) if content_el is not None else None
-
         return {
             'title': title,
             'formats': self._extract_video_formats(mediagen_doc, mtvn_id),
@@ -169,7 +167,7 @@ class MTVServicesInfoExtractor(InfoExtractor):
             'id': video_id,
             'thumbnail': self._get_thumbnail_url(uri, itemdoc),
             'description': description,
-            'duration': duration,
+            'duration': float_or_none(content_el.attrib.get('duration')),
         }
 
     def _get_feed_query(self, uri):

From 86bf29050e29d14dad894ee4f7ca68a05ebd29bc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 20 Feb 2016 03:36:03 +0600
Subject: [PATCH 4049/6219] [test_YoutubeDL] Make test pass until more
 intelligent sort formats (Closes #8462)

---
 test/test_YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 02caf5908..59f7ab49d 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -234,7 +234,7 @@ class TestFormatSelection(unittest.TestCase):
 
     def test_youtube_format_selection(self):
         order = [
-            '38', '37', '46', '22', '45', '35', '44', '18', '34', '43', '6', '5', '36', '17', '13',
+            '38', '37', '46', '22', '45', '35', '44', '18', '34', '43', '6', '5', '17', '36', '13',
             # Apple HTTP Live Streaming
             '96', '95', '94', '93', '92', '132', '151',
             # 3D

From 99cbe98ce8617c119c2fb6a567b0e6ef7eae8859 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 20 Feb 2016 07:58:25 +0100
Subject: [PATCH 4050/6219] [downloader/external] check for external
 downloaders availability

---
 youtube_dl/downloader/__init__.py |  4 ++--
 youtube_dl/downloader/external.py | 21 +++++++++++++++++++++
 2 files changed, 23 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/downloader/__init__.py b/youtube_dl/downloader/__init__.py
index bb6afb1f8..67c2840a5 100644
--- a/youtube_dl/downloader/__init__.py
+++ b/youtube_dl/downloader/__init__.py
@@ -31,13 +31,13 @@ def get_suitable_downloader(info_dict, params={}):
     protocol = determine_protocol(info_dict)
     info_dict['protocol'] = protocol
 
-    if (info_dict.get('start_time') or info_dict.get('end_time')) and FFmpegFD.supports(info_dict):
+    if (info_dict.get('start_time') or info_dict.get('end_time')) and FFmpegFD.available() and FFmpegFD.supports(info_dict):
         return FFmpegFD
 
     external_downloader = params.get('external_downloader')
     if external_downloader is not None:
         ed = get_external_downloader(external_downloader)
-        if ed.supports(info_dict):
+        if ed.available() and ed.supports(info_dict):
             return ed
 
     if protocol == 'm3u8' and params.get('hls_prefer_native'):
diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index bb43677b7..edf85483b 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -16,6 +16,7 @@ from ..utils import (
     encodeFilename,
     encodeArgument,
     handle_youtubedl_headers,
+    check_executable,
 )
 
 
@@ -50,6 +51,10 @@ class ExternalFD(FileDownloader):
     def exe(self):
         return self.params.get('external_downloader')
 
+    @classmethod
+    def available(cls):
+        return check_executable(cls.get_basename(), cls.available_opt)
+
     @classmethod
     def supports(cls, info_dict):
         return info_dict['protocol'] in ('http', 'https', 'ftp', 'ftps')
@@ -81,6 +86,8 @@ class ExternalFD(FileDownloader):
 
 
 class CurlFD(ExternalFD):
+    available_opt = ['-V']
+
     def _make_cmd(self, tmpfilename, info_dict):
         cmd = [self.exe, '--location', '-o', tmpfilename]
         for key, val in info_dict['http_headers'].items():
@@ -94,6 +101,8 @@ class CurlFD(ExternalFD):
 
 
 class AxelFD(ExternalFD):
+    available_opt = ['-V']
+
     def _make_cmd(self, tmpfilename, info_dict):
         cmd = [self.exe, '-o', tmpfilename]
         for key, val in info_dict['http_headers'].items():
@@ -104,6 +113,8 @@ class AxelFD(ExternalFD):
 
 
 class WgetFD(ExternalFD):
+    available_opt = ['--version']
+
     def _make_cmd(self, tmpfilename, info_dict):
         cmd = [self.exe, '-O', tmpfilename, '-nv', '--no-cookies']
         for key, val in info_dict['http_headers'].items():
@@ -117,6 +128,8 @@ class WgetFD(ExternalFD):
 
 
 class Aria2cFD(ExternalFD):
+    available_opt = ['-v']
+
     def _make_cmd(self, tmpfilename, info_dict):
         cmd = [self.exe, '-c']
         cmd += self._configuration_args([
@@ -135,6 +148,10 @@ class Aria2cFD(ExternalFD):
 
 
 class HttpieFD(ExternalFD):
+    @classmethod
+    def available(cls):
+        return check_executable('http', ['--version'])
+
     def _make_cmd(self, tmpfilename, info_dict):
         cmd = ['http', '--download', '--output', tmpfilename, info_dict['url']]
         for key, val in info_dict['http_headers'].items():
@@ -147,6 +164,10 @@ class FFmpegFD(ExternalFD):
     def supports(cls, info_dict):
         return info_dict['protocol'] in ('http', 'https', 'ftp', 'ftps', 'm3u8', 'rtsp', 'rtmp', 'mms')
 
+    @classmethod
+    def available(cls):
+        return FFmpegPostProcessor().available
+
     def _call_downloader(self, tmpfilename, info_dict):
         url = info_dict['url']
         ffpp = FFmpegPostProcessor(downloader=self)

From f34294fa0c0097cea7f6388d5d691d5a54950491 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 20 Feb 2016 08:06:12 +0100
Subject: [PATCH 4051/6219] [downloader/external:ffmpegfd] check for None value
 of start_time

---
 youtube_dl/downloader/external.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index edf85483b..a4fdf1af8 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -175,7 +175,7 @@ class FFmpegFD(ExternalFD):
 
         args = [ffpp.executable, '-y']
 
-        start_time = info_dict.get('start_time', 0)
+        start_time = info_dict.get('start_time') or 0
         if start_time:
             args += ['-ss', compat_str(start_time)]
         end_time = info_dict.get('end_time')

From 958759f44b3e7c37509394568d9764e2794f713a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 20 Feb 2016 15:54:00 +0800
Subject: [PATCH 4052/6219] [appletrailers] Extend _VALID_URL (#8524)

---
 youtube_dl/extractor/appletrailers.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/appletrailers.py b/youtube_dl/extractor/appletrailers.py
index 62ed0c918..be40f85b4 100644
--- a/youtube_dl/extractor/appletrailers.py
+++ b/youtube_dl/extractor/appletrailers.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 class AppleTrailersIE(InfoExtractor):
     IE_NAME = 'appletrailers'
-    _VALID_URL = r'https?://(?:www\.)?trailers\.apple\.com/(?:trailers|ca)/(?P<company>[^/]+)/(?P<movie>[^/]+)'
+    _VALID_URL = r'https?://(?:www\.|movie)?trailers\.apple\.com/(?:trailers|ca)/(?P<company>[^/]+)/(?P<movie>[^/]+)'
     _TESTS = [{
         'url': 'http://trailers.apple.com/trailers/wb/manofsteel/',
         'info_dict': {
@@ -73,6 +73,9 @@ class AppleTrailersIE(InfoExtractor):
     }, {
         'url': 'http://trailers.apple.com/ca/metropole/autrui/',
         'only_matching': True,
+    }, {
+        'url': 'http://movietrailers.apple.com/trailers/focus_features/kuboandthetwostrings/',
+        'only_matching': True,
     }]
 
     _JSON_RE = r'iTunes.playURL\((.*?)\);'

From 5bc880b9887c9689fef171683eae279adbbfc186 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 17 Feb 2016 06:01:44 +0800
Subject: [PATCH 4053/6219] [utils] Add OHDave's RSA encryption function

---
 test/test_utils.py  |  8 ++++++++
 youtube_dl/utils.py | 18 ++++++++++++++++++
 2 files changed, 26 insertions(+)

diff --git a/test/test_utils.py b/test/test_utils.py
index a1e416dd5..e6887be9f 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -35,6 +35,7 @@ from youtube_dl.utils import (
     is_html,
     js_to_json,
     limit_length,
+    ohdave_rsa_encrypt,
     OnDemandPagedList,
     orderedSet,
     parse_duration,
@@ -792,6 +793,13 @@ The first line
                 {'nocheckcertificate': False}, '--check-certificate', 'nocheckcertificate', 'false', 'true', '='),
             ['--check-certificate=true'])
 
+    def test_ohdave_rsa_encrypt(self):
+        N = 0xab86b6371b5318aaa1d3c9e612a9f1264f372323c8c0f19875b5fc3b3fd3afcc1e5bec527aa94bfa85bffc157e4245aebda05389a5357b75115ac94f074aefcd
+        e = 65537
+
+        self.assertEqual(
+            ohdave_rsa_encrypt(b'aa111222', e, N),
+            '726664bd9a23fd0c70f9f1b84aab5e3905ce1e45a584e9cbcf9bcc7510338fc1986d6c599ff990d923aa43c51c0d9013cd572e13bc58f4ae48f2ed8c0b0ba881')
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 672ce05ea..7ce661b09 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -4,6 +4,7 @@
 from __future__ import unicode_literals
 
 import base64
+import binascii
 import calendar
 import codecs
 import contextlib
@@ -2582,3 +2583,20 @@ class PerRequestProxyHandler(compat_urllib_request.ProxyHandler):
             return None  # No Proxy
         return compat_urllib_request.ProxyHandler.proxy_open(
             self, req, proxy, type)
+
+
+def ohdave_rsa_encrypt(data, exponent, modulus):
+    '''
+    Implement OHDave's RSA algorithm. See http://www.ohdave.com/rsa/
+
+    Input:
+        data: data to encrypt, bytes-like object
+        exponent, modulus: parameter e and N of RSA algorithm, both integer
+    Output: hex string of encrypted data
+
+    Limitation: supports one block encryption only
+    '''
+
+    payload = int(binascii.hexlify(data[::-1]), 16)
+    encrypted = pow(payload, exponent, modulus)
+    return '%x' % encrypted

From 99709cc3f121010421360ba94d75e16ce15f0be5 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 17 Feb 2016 06:26:27 +0800
Subject: [PATCH 4054/6219] [iqiyi] Implement _login()

Currently only email login supported
---
 youtube_dl/extractor/iqiyi.py | 236 +++++++++++++++++++++++++++++++++-
 1 file changed, 230 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index 9046705a5..541fe7ae3 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -5,29 +5,199 @@ import hashlib
 import math
 import os
 import random
+import re
 import time
 import uuid
 
 from .common import InfoExtractor
 from ..compat import (
     compat_parse_qs,
+    compat_str,
     compat_urllib_parse,
     compat_urllib_parse_urlparse,
 )
 from ..utils import (
     ExtractorError,
+    ohdave_rsa_encrypt,
     sanitized_Request,
     urlencode_postdata,
     url_basename,
 )
 
 
+def md5_text(text):
+    return hashlib.md5(text.encode('utf-8')).hexdigest()
+
+
+class IqiyiSDK(object):
+    def __init__(self, target, ip, timestamp):
+        self.target = target
+        self.ip = ip
+        self.timestamp = timestamp
+
+    @staticmethod
+    def split_sum(data):
+        return compat_str(sum(map(lambda p: int(p, 16), list(data))))
+
+    @staticmethod
+    def digit_sum(num):
+        if isinstance(num, int):
+            num = compat_str(num)
+        return compat_str(sum(map(int, num)))
+
+    def even_odd(self):
+        even = self.digit_sum(compat_str(self.timestamp)[::2])
+        odd = self.digit_sum(compat_str(self.timestamp)[1::2])
+        return even, odd
+
+    def preprocess(self, chunksize):
+        self.target = md5_text(self.target)
+        chunks = []
+        for i in range(32 // chunksize):
+            chunks.append(self.target[chunksize * i:chunksize * (i + 1)])
+        if 32 % chunksize:
+            chunks.append(self.target[32 - 32 % chunksize:])
+        return chunks, list(map(int, self.ip.split('.')))
+
+    def mod(self, modulus):
+        chunks, ip = self.preprocess(32)
+        self.target = chunks[0] + ''.join(map(lambda p: compat_str(p % modulus), ip))
+
+    def split(self, chunksize):
+        modulus_map = {
+            4: 256,
+            5: 10,
+            8: 100,
+        }
+
+        chunks, ip = self.preprocess(chunksize)
+        ret = ''
+        for i in range(len(chunks)):
+            ip_part = compat_str(ip[i] % modulus_map[chunksize]) if i < 4 else ''
+            if chunksize == 8:
+                ret += ip_part + chunks[i]
+            else:
+                ret += chunks[i] + ip_part
+        self.target = ret
+
+    def handle_input16(self):
+        self.target = md5_text(self.target)
+        self.target = self.split_sum(self.target[:16]) + self.target + self.split_sum(self.target[16:])
+
+    def handle_input8(self):
+        self.target = md5_text(self.target)
+        ret = ''
+        for i in range(4):
+            part = self.target[8 * i:8 * (i + 1)]
+            ret += self.split_sum(part) + part
+        self.target = ret
+
+    def handleSum(self):
+        self.target = md5_text(self.target)
+        self.target = self.split_sum(self.target) + self.target
+
+    def date(self, scheme):
+        self.target = md5_text(self.target)
+        d = time.localtime(self.timestamp)
+        strings = {
+            'y': compat_str(d.tm_year),
+            'm': '%02d' % d.tm_mon,
+            'd': '%02d' % d.tm_mday,
+        }
+        self.target += ''.join(map(lambda c: strings[c], list(scheme)))
+
+    def split_time_even_odd(self):
+        even, odd = self.even_odd()
+        self.target = odd + md5_text(self.target) + even
+
+    def split_time_odd_even(self):
+        even, odd = self.even_odd()
+        self.target = even + md5_text(self.target) + odd
+
+    def split_ip_time_sum(self):
+        chunks, ip = self.preprocess(32)
+        self.target = compat_str(sum(ip)) + chunks[0] + self.digit_sum(self.timestamp)
+
+    def split_time_ip_sum(self):
+        chunks, ip = self.preprocess(32)
+        self.target = self.digit_sum(self.timestamp) + chunks[0] + compat_str(sum(ip))
+
+
+class IqiyiSDKInterpreter(object):
+    BASE62_TABLE = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
+
+    def __init__(self, sdk_code):
+        self.sdk_code = sdk_code
+
+    @classmethod
+    def base62(cls, num):
+        if num == 0:
+            return '0'
+        ret = ''
+        while num:
+            ret = cls.BASE62_TABLE[num % 62] + ret
+            num = num // 62
+        return ret
+
+    def decode_eval_codes(self):
+        self.sdk_code = self.sdk_code[5:-3]
+
+        mobj = re.search(
+            r"'([^']+)',62,(\d+),'([^']+)'\.split\('\|'\),[^,]+,{}",
+            self.sdk_code)
+        obfucasted_code, count, symbols = mobj.groups()
+        count = int(count)
+        symbols = symbols.split('|')
+        symbol_table = {}
+
+        while count:
+            count -= 1
+            b62count = self.base62(count)
+            symbol_table[b62count] = symbols[count] or b62count
+
+        self.sdk_code = re.sub(
+            r'\b(\w+)\b', lambda mobj: symbol_table[mobj.group(0)],
+            obfucasted_code)
+
+    def run(self, target, ip, timestamp):
+        self.decode_eval_codes()
+
+        functions = re.findall(r'input=([a-zA-Z0-9]+)\(input', self.sdk_code)
+
+        sdk = IqiyiSDK(target, ip, timestamp)
+
+        other_functions = {
+            'handleSum': sdk.handleSum,
+            'handleInput8': sdk.handle_input8,
+            'handleInput16': sdk.handle_input16,
+            'splitTimeEvenOdd': sdk.split_time_even_odd,
+            'splitTimeOddEven': sdk.split_time_odd_even,
+            'splitIpTimeSum': sdk.split_ip_time_sum,
+            'splitTimeIpSum': sdk.split_time_ip_sum,
+        }
+        for function in functions:
+            if re.match(r'mod\d+', function):
+                sdk.mod(int(function[3:]))
+            elif re.match(r'date[ymd]{3}', function):
+                sdk.date(function[4:])
+            elif re.match(r'split\d+', function):
+                sdk.split(int(function[5:]))
+            elif function in other_functions:
+                other_functions[function]()
+            else:
+                raise ExtractorError('Unknown funcion %s' % function)
+
+        return sdk.target
+
+
 class IqiyiIE(InfoExtractor):
     IE_NAME = 'iqiyi'
     IE_DESC = '爱奇艺'
 
     _VALID_URL = r'http://(?:[^.]+\.)?iqiyi\.com/.+\.html'
 
+    _NETRC_MACHINE = 'iqiyi'
+
     _TESTS = [{
         'url': 'http://www.iqiyi.com/v_19rrojlavg.html',
         'md5': '2cb594dc2781e6c941a110d8f358118b',
@@ -136,9 +306,63 @@ class IqiyiIE(InfoExtractor):
         ('10', 'h1'),
     ]
 
+    def _real_initialize(self):
+        self._login()
+
     @staticmethod
-    def md5_text(text):
-        return hashlib.md5(text.encode('utf-8')).hexdigest()
+    def _rsa_fun(data):
+        # public key extracted from http://static.iqiyi.com/js/qiyiV2/20160129180840/jobs/i18n/i18nIndex.js
+        N = 0xab86b6371b5318aaa1d3c9e612a9f1264f372323c8c0f19875b5fc3b3fd3afcc1e5bec527aa94bfa85bffc157e4245aebda05389a5357b75115ac94f074aefcd
+        e = 65537
+
+        return ohdave_rsa_encrypt(data, e, N)
+
+    def _login(self):
+        (username, password) = self._get_login_info()
+
+        # No authentication to be performed
+        if not username:
+            return True
+
+        data = self._download_json(
+            'http://kylin.iqiyi.com/get_token', None,
+            note='Get token for logging', errnote='Unable to get token for logging')
+        sdk = data['sdk']
+        timestamp = int(time.time())
+        target = '/apis/reglogin/login.action?lang=zh_TW&area_code=null&email=%s&passwd=%s&agenttype=1&from=undefined&keeplogin=0&piccode=&fromurl=&_pos=1' % (
+            username, self._rsa_fun(password.encode('utf-8')))
+
+        interp = IqiyiSDKInterpreter(sdk)
+        sign = interp.run(target, data['ip'], timestamp)
+
+        validation_params = {
+            'target': target,
+            'server': 'BEA3AA1908656AABCCFF76582C4C6660',
+            'token': data['token'],
+            'bird_src': 'f8d91d57af224da7893dd397d52d811a',
+            'sign': sign,
+            'bird_t': timestamp,
+        }
+        validation_result = self._download_json(
+            'http://kylin.iqiyi.com/validate?' + compat_urllib_parse.urlencode(validation_params), None,
+            note='Validate credentials', errnote='Unable to validate credentials')
+
+        MSG_MAP = {
+            'P00107': 'please login via the web interface and enter the CAPTCHA code',
+            'P00117': 'bad username or password',
+        }
+
+        code = validation_result['code']
+        if code != 'A00000':
+            msg = MSG_MAP.get(code)
+            if not msg:
+                msg = 'error %s' % code
+                if validation_result.get('msg'):
+                    msg += ': ' + validation_result['msg']
+            self._downloader.report_warning('unable to log in: ' + msg)
+            return False
+
+        return True
 
     def _authenticate_vip_video(self, api_video_url, video_id, tvid, _uuid, do_report_warning):
         auth_params = {
@@ -199,7 +423,7 @@ class IqiyiIE(InfoExtractor):
                 note='Download path key of segment %d for format %s' % (segment_index + 1, format_id)
             )['t']
             t = str(int(math.floor(int(tm) / (600.0))))
-            return self.md5_text(t + mg + x)
+            return md5_text(t + mg + x)
 
         video_urls_dict = {}
         need_vip_warning_report = True
@@ -278,16 +502,16 @@ class IqiyiIE(InfoExtractor):
         tail = tm + tvid
         param = {
             'key': 'fvip',
-            'src': self.md5_text('youtube-dl'),
+            'src': md5_text('youtube-dl'),
             'tvId': tvid,
             'vid': video_id,
             'vinfo': 1,
             'tm': tm,
-            'enc': self.md5_text(enc_key + tail),
+            'enc': md5_text(enc_key + tail),
             'qyid': _uuid,
             'tn': random.random(),
             'um': 0,
-            'authkey': self.md5_text(self.md5_text('') + tail),
+            'authkey': md5_text(md5_text('') + tail),
             'k_tag': 1,
         }
 

From 1d14c75f55094484127dda641465b5333296d497 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 20 Feb 2016 20:49:30 +0800
Subject: [PATCH 4055/6219] [Makefile] iQiyi login test requires network

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index f826c1685..cb449b7e6 100644
--- a/Makefile
+++ b/Makefile
@@ -44,7 +44,7 @@ test:
 ot: offlinetest
 
 offlinetest: codetest
-	nosetests --verbose test --exclude test_download.py --exclude test_age_restriction.py --exclude test_subtitles.py --exclude test_write_annotations.py --exclude test_youtube_lists.py
+	nosetests --verbose test --exclude test_download.py --exclude test_age_restriction.py --exclude test_subtitles.py --exclude test_write_annotations.py --exclude test_youtube_lists.py --exclude test_iqiyi_sdk_interpreter.py
 
 tar: youtube-dl.tar.gz
 

From 9c86d509162fcf55eb328acb1c8d54b0de97e894 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 20 Feb 2016 14:11:44 +0100
Subject: [PATCH 4056/6219] [faz] Future-proof XML element check

---
 youtube_dl/extractor/faz.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/faz.py b/youtube_dl/extractor/faz.py
index 6f9b003c2..fd535457d 100644
--- a/youtube_dl/extractor/faz.py
+++ b/youtube_dl/extractor/faz.py
@@ -52,7 +52,7 @@ class FazIE(InfoExtractor):
         formats = []
         for pref, code in enumerate(['LOW', 'HIGH', 'HQ']):
             encoding = xpath_element(encodings, code)
-            if encoding:
+            if encoding is not None:
                 encoding_url = xpath_text(encoding, 'FILENAME')
                 if encoding_url:
                     formats.append({

From 73f9c2867df5048e78c6f60d2fc1d9691edb4b2d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 20 Feb 2016 22:44:04 +0800
Subject: [PATCH 4057/6219] [iqiyi] Support playlists (closes #8019)

---
 youtube_dl/extractor/iqiyi.py | 49 +++++++++++++++++++++++++++++++++++
 1 file changed, 49 insertions(+)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index 541fe7ae3..c3e33009a 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -2,6 +2,7 @@
 from __future__ import unicode_literals
 
 import hashlib
+import itertools
 import math
 import os
 import random
@@ -19,6 +20,7 @@ from ..compat import (
 from ..utils import (
     ExtractorError,
     ohdave_rsa_encrypt,
+    remove_start,
     sanitized_Request,
     urlencode_postdata,
     url_basename,
@@ -295,6 +297,13 @@ class IqiyiIE(InfoExtractor):
             },
         }],
         'expected_warnings': ['Needs a VIP account for full video'],
+    }, {
+        'url': 'http://www.iqiyi.com/a_19rrhb8ce1.html',
+        'info_dict': {
+            'id': '202918101',
+            'title': '灌篮高手 国语版',
+        },
+        'playlist_count': 101,
     }]
 
     _FORMATS_MAP = [
@@ -526,9 +535,49 @@ class IqiyiIE(InfoExtractor):
         enc_key = '6ab6d0280511493ba85594779759d4ed'
         return enc_key
 
+    def _extract_playlist(self, webpage):
+        PAGE_SIZE = 50
+
+        links = re.findall(
+            r'<a[^>]+class="site-piclist_pic_link"[^>]+href="(http://www\.iqiyi\.com/.+\.html)"',
+            webpage)
+        if not links:
+            return
+
+        album_id = self._search_regex(
+            r'albumId\s*:\s*(\d+),', webpage, 'album ID')
+        album_title = self._search_regex(
+            r'data-share-title="([^"]+)"', webpage, 'album title', fatal=False)
+
+        entries = list(map(self.url_result, links))
+
+        # Start from 2 because links in the first page are already on webpage
+        for page_num in itertools.count(2):
+            pagelist_page = self._download_webpage(
+                'http://cache.video.qiyi.com/jp/avlist/%s/%d/%d/' % (album_id, page_num, PAGE_SIZE),
+                album_id,
+                note='Download playlist page %d' % page_num,
+                errnote='Failed to download playlist page %d' % page_num)
+            pagelist = self._parse_json(
+                remove_start(pagelist_page, 'var tvInfoJs='), album_id)
+            vlist = pagelist['data']['vlist']
+            for item in vlist:
+                entries.append(self.url_result(item['vurl']))
+            if len(vlist) < PAGE_SIZE:
+                break
+
+        return self.playlist_result(entries, album_id, album_title)
+
     def _real_extract(self, url):
         webpage = self._download_webpage(
             url, 'temp_id', note='download video page')
+
+        # There's no simple way to determine whether an URL is a playlist or not
+        # So detect it
+        playlist_result = self._extract_playlist(webpage)
+        if playlist_result:
+            return playlist_result
+
         tvid = self._search_regex(
             r'data-player-tvid\s*=\s*[\'"](\d+)', webpage, 'tvid')
         video_id = self._search_regex(

From a4e4d7dfcd03598c8b0924d4973b84ad30bc944e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 20 Feb 2016 23:10:39 +0800
Subject: [PATCH 4058/6219] [test_iqiyi_sdk_interpreter] Add test for iQiyi
 login

---
 test/test_iqiyi_sdk_interpreter.py | 47 ++++++++++++++++++++++++++++++
 1 file changed, 47 insertions(+)
 create mode 100644 test/test_iqiyi_sdk_interpreter.py

diff --git a/test/test_iqiyi_sdk_interpreter.py b/test/test_iqiyi_sdk_interpreter.py
new file mode 100644
index 000000000..9d95cb606
--- /dev/null
+++ b/test/test_iqiyi_sdk_interpreter.py
@@ -0,0 +1,47 @@
+#!/usr/bin/env python
+
+from __future__ import unicode_literals
+
+# Allow direct execution
+import os
+import sys
+import unittest
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from test.helper import FakeYDL
+from youtube_dl.extractor import IqiyiIE
+
+
+class IqiyiIEWithCredentials(IqiyiIE):
+    def _get_login_info(self):
+        return 'foo', 'bar'
+
+
+class WarningLogger(object):
+    def __init__(self):
+        self.messages = []
+
+    def warning(self, msg):
+        self.messages.append(msg)
+
+    def debug(self, msg):
+        pass
+
+    def error(self, msg):
+        pass
+
+
+class TestIqiyiSDKInterpreter(unittest.TestCase):
+    def test_iqiyi_sdk_interpreter(self):
+        '''
+        Test the functionality of IqiyiSDKInterpreter by trying to log in
+
+        If `sign` is incorrect, /validate call throws an HTTP 556 error
+        '''
+        logger = WarningLogger()
+        ie = IqiyiIEWithCredentials(FakeYDL({'logger': logger}))
+        ie._login()
+        self.assertTrue('unable to log in:' in logger.messages[0])
+
+if __name__ == '__main__':
+    unittest.main()

From 765ac263dbf17442f41715a1cfce13fc61a4b315 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 20 Feb 2016 19:55:10 +0100
Subject: [PATCH 4059/6219] [utils] mimetype2ext: return 'm4a' for 'audio/mp4'
 (fixes #8620)

The youtube extractor was using 'mp4' for them, therefore filters like 'bestaudio[ext=m4a]' stopped working (94278f720272c5ad2cd5900f59f8e71f31d46633 broke it).
---
 youtube_dl/utils.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 7ce661b09..a2c6780ca 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1836,6 +1836,12 @@ def error_to_compat_str(err):
 
 
 def mimetype2ext(mt):
+    ext = {
+        'audio/mp4': 'm4a',
+    }.get(mt)
+    if ext is not None:
+        return ext
+
     _, _, res = mt.rpartition('/')
 
     return {

From 399a76e67bca0beb4849ea90c4f40803fbd06ed3 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 21 Feb 2016 03:28:25 +0800
Subject: [PATCH 4060/6219] [utils] Jython support: tolerate missing fcntl
 module

---
 youtube_dl/utils.py | 20 +++++++++++++++-----
 1 file changed, 15 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index a2c6780ca..17747be26 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1217,13 +1217,23 @@ if sys.platform == 'win32':
             raise OSError('Unlocking file failed: %r' % ctypes.FormatError())
 
 else:
-    import fcntl
+    # Some platforms, such as Jython, is missing fcntl
+    try:
+        import fcntl
 
-    def _lock_file(f, exclusive):
-        fcntl.flock(f, fcntl.LOCK_EX if exclusive else fcntl.LOCK_SH)
+        def _lock_file(f, exclusive):
+            fcntl.flock(f, fcntl.LOCK_EX if exclusive else fcntl.LOCK_SH)
 
-    def _unlock_file(f):
-        fcntl.flock(f, fcntl.LOCK_UN)
+        def _unlock_file(f):
+            fcntl.flock(f, fcntl.LOCK_UN)
+    except ImportError:
+        UNSUPPORTED_MSG = 'file locking is not supported on this platform'
+
+        def _lock_file(f, exclusive):
+            raise IOError(UNSUPPORTED_MSG)
+
+        def _unlock_file(f):
+            raise IOError(UNSUPPORTED_MSG)
 
 
 class locked_file(object):

From c1c05c67ea6087c3b0190c9f16cb9fdd8160e398 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 21 Feb 2016 03:29:02 +0800
Subject: [PATCH 4061/6219] [utils] Jython support - disable setproctitle()
 until ctypes is complete

---
 youtube_dl/utils.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 17747be26..16b4324a4 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1397,6 +1397,12 @@ def fix_xml_ampersands(xml_str):
 
 def setproctitle(title):
     assert isinstance(title, compat_str)
+
+    # ctypes in Jython is not complete
+    # http://bugs.jython.org/issue2148
+    if sys.platform.startswith('java'):
+        return
+
     try:
         libc = ctypes.cdll.LoadLibrary('libc.so.6')
     except OSError:

From 7360db05b43741c2dfa1fd024e9c2f013ed97c9e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 21 Feb 2016 03:25:44 +0800
Subject: [PATCH 4062/6219] [postprocessor/embedthumbnail] Allow mkv to embed
 thumbnails

Fixes #6046
---
 youtube_dl/postprocessor/embedthumbnail.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/postprocessor/embedthumbnail.py b/youtube_dl/postprocessor/embedthumbnail.py
index e19dbf73d..3bad5a266 100644
--- a/youtube_dl/postprocessor/embedthumbnail.py
+++ b/youtube_dl/postprocessor/embedthumbnail.py
@@ -40,7 +40,7 @@ class EmbedThumbnailPP(FFmpegPostProcessor):
                 'Skipping embedding the thumbnail because the file is missing.')
             return [], info
 
-        if info['ext'] == 'mp3':
+        if info['ext'] in ('mp3', 'mkv'):
             options = [
                 '-c', 'copy', '-map', '0', '-map', '1',
                 '-metadata:s:v', 'title="Album cover"', '-metadata:s:v', 'comment="Cover (Front)"']

From 101067de12e193c8ad42e1f474c9018eedadec9d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 21 Feb 2016 03:29:49 +0800
Subject: [PATCH 4063/6219] Jython support - handle *.class files

---
 .gitignore | 3 ++-
 Makefile   | 1 +
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/.gitignore b/.gitignore
index 0422adf44..26dbde73d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,6 @@
 *.pyc
 *.pyo
+*.class
 *~
 *.DS_Store
 wine-py2exe/
@@ -32,4 +33,4 @@ test/testdata
 .tox
 youtube-dl.zsh
 .idea
-.idea/*
\ No newline at end of file
+.idea/*
diff --git a/Makefile b/Makefile
index cb449b7e6..afd7f4032 100644
--- a/Makefile
+++ b/Makefile
@@ -3,6 +3,7 @@ all: youtube-dl README.md CONTRIBUTING.md README.txt youtube-dl.1 youtube-dl.bas
 clean:
 	rm -rf youtube-dl.1.temp.md youtube-dl.1 youtube-dl.bash-completion README.txt MANIFEST build/ dist/ .coverage cover/ youtube-dl.tar.gz youtube-dl.zsh youtube-dl.fish *.dump *.part *.info.json *.mp4 *.flv *.mp3 *.avi CONTRIBUTING.md.tmp youtube-dl youtube-dl.exe
 	find . -name "*.pyc" -delete
+	find . -name "*.class" -delete
 
 PREFIX ?= /usr/local
 BINDIR ?= $(PREFIX)/bin

From cafcf657a4e16f0bcf90195f8e35bfcce7faad8e Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 20 Feb 2016 22:02:03 +0100
Subject: [PATCH 4064/6219] add more subtitles mime types to mimetype2ext and
 fix the platform subtitle extraction

---
 youtube_dl/extractor/common.py      | 12 ++----------
 youtube_dl/extractor/theplatform.py |  3 ++-
 youtube_dl/utils.py                 |  4 ++++
 3 files changed, 8 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index f411ea763..c85dcda0c 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -46,6 +46,7 @@ from ..utils import (
     xpath_with_ns,
     determine_protocol,
     parse_duration,
+    mimetype2ext,
 )
 
 
@@ -1277,16 +1278,7 @@ class InfoExtractor(object):
             if not src or src in urls:
                 continue
             urls.append(src)
-            ext = textstream.get('ext') or determine_ext(src)
-            if not ext:
-                type_ = textstream.get('type')
-                SUBTITLES_TYPES = {
-                    'text/vtt': 'vtt',
-                    'text/srt': 'srt',
-                    'application/smptett+xml': 'tt',
-                }
-                if type_ in SUBTITLES_TYPES:
-                    ext = SUBTITLES_TYPES[type_]
+            ext = textstream.get('ext') or determine_ext(src) or mimetype2ext(textstream.get('type'))
             lang = textstream.get('systemLanguage') or textstream.get('systemLanguageName') or textstream.get('lang') or subtitles_lang
             subtitles.setdefault(lang, []).append({
                 'url': src,
diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index 755f816ff..93d871571 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -21,6 +21,7 @@ from ..utils import (
     sanitized_Request,
     unsmuggle_url,
     xpath_with_ns,
+    mimetype2ext,
 )
 
 default_ns = 'http://www.w3.org/2005/SMIL21/Language'
@@ -68,7 +69,7 @@ class ThePlatformBaseIE(InfoExtractor):
             for caption in captions:
                 lang, src, mime = caption.get('lang', 'en'), caption.get('src'), caption.get('type')
                 subtitles[lang] = [{
-                    'ext': 'srt' if mime == 'text/srt' else 'ttml',
+                    'ext': mimetype2ext(mime),
                     'url': src,
                 }]
 
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index a2c6780ca..6978a10e4 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1846,7 +1846,11 @@ def mimetype2ext(mt):
 
     return {
         '3gpp': '3gp',
+        'smptett+xml': 'tt',
+        'srt': 'srt',
+        'ttaf+xml': 'dfxp',
         'ttml+xml': 'ttml',
+        'vtt': 'vtt',
         'x-flv': 'flv',
         'x-mp4-fragmented': 'mp4',
         'x-ms-wmv': 'wmv',

From 33d4fdabfa0a2e008fe2765ae7b81f9f736c2163 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 21 Feb 2016 09:51:54 +0600
Subject: [PATCH 4065/6219] [extractor/generic] Add support for ok embeds
 (#8619)

---
 youtube_dl/extractor/generic.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 45adbb7a3..c6bf8d270 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1573,6 +1573,11 @@ class GenericIE(InfoExtractor):
         if mobj is not None:
             return self.url_result(mobj.group('url'), 'VK')
 
+        # Look for embedded Odnoklassniki player
+        mobj = re.search(r'<iframe[^>]+?src=(["\'])(?P<url>https?://(?:odnoklassniki|ok)\.ru/videoembed/.+?)\1', webpage)
+        if mobj is not None:
+            return self.url_result(mobj.group('url'), 'Odnoklassniki')
+
         # Look for embedded ivi player
         mobj = re.search(r'<embed[^>]+?src=(["\'])(?P<url>https?://(?:www\.)?ivi\.ru/video/player.+?)\1', webpage)
         if mobj is not None:

From b72f6163dc264218c4abefebefd1971875fff7d9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 21 Feb 2016 13:37:31 +0600
Subject: [PATCH 4066/6219] [arte:+7] Improve _VALID_URL

---
 youtube_dl/extractor/arte.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index 793da2ee1..f4d033e70 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -63,7 +63,7 @@ class ArteTvIE(InfoExtractor):
 
 class ArteTVPlus7IE(InfoExtractor):
     IE_NAME = 'arte.tv:+7'
-    _VALID_URL = r'https?://(?:www\.)?arte\.tv/guide/(?P<lang>fr|de|en|es)/(?:(?:sendungen|emissions)/)?(?P<id>[^/]+)/(?P<name>[^/?#&+])'
+    _VALID_URL = r'https?://(?:www\.)?arte\.tv/guide/(?P<lang>fr|de|en|es)/(?:(?:sendungen|emissions|embed)/)?(?P<id>[^/]+)/(?P<name>[^/?#&+])'
 
     @classmethod
     def _extract_url_info(cls, url):

From 305168ca3ee869aed5e95212ed529bf0e6e2f37c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 21 Feb 2016 13:55:25 +0600
Subject: [PATCH 4067/6219] [arte:+7] Detect more embeds (Closes #8613)

---
 youtube_dl/extractor/arte.py | 40 ++++++++++++++++++++++++++++--------
 1 file changed, 31 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index f4d033e70..a4c786d78 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -110,15 +110,25 @@ class ArteTVPlus7IE(InfoExtractor):
             # en and es URLs produce react-based pages with different layout (e.g.
             # http://www.arte.tv/guide/en/053330-002-A/carnival-italy?zone=world)
             if not iframe_url:
-                embed_html = self._parse_json(
-                    self._search_regex(
-                        r'program\s*:\s*({.+?["\']embed_html["\'].+?}),?\s*\n',
-                        webpage, 'program'),
-                    video_id)['embed_html']
-                iframe_url = find_iframe_url(embed_html)
-            json_url = compat_parse_qs(
-                compat_urllib_parse_urlparse(iframe_url).query)['json_url'][0]
-        return self._extract_from_json_url(json_url, video_id, lang)
+                program = self._search_regex(
+                    r'program\s*:\s*({.+?["\']embed_html["\'].+?}),?\s*\n',
+                    webpage, 'program', default=None)
+                if program:
+                    embed_html = self._parse_json(program,video_id)
+                    if embed_html:
+                        iframe_url = find_iframe_url(embed_html['embed_html'])
+            if iframe_url:
+                json_url = compat_parse_qs(
+                    compat_urllib_parse_urlparse(iframe_url).query)['json_url'][0]
+        if json_url:
+            return self._extract_from_json_url(json_url, video_id, lang)
+        # Differend kind of embed URL (e.g.
+        # http://www.arte.tv/magazine/trepalium/fr/episode-0406-replay-trepalium)
+        embed_url = self._search_regex(
+            r'<iframe[^>]+src=(["\'])(?P<url>.+?)\1',
+            webpage, 'embed url', group='url')
+        return self.url_result(embed_url)
+
 
     def _extract_from_json_url(self, json_url, video_id, lang):
         info = self._download_json(json_url, video_id)
@@ -294,6 +304,7 @@ class ArteTVMagazineIE(ArteTVPlus7IE):
     _VALID_URL = r'https?://(?:www\.)?arte\.tv/magazine/[^/]+/(?P<lang>fr|de|en|es)/(?P<id>[^/?#&]+)'
 
     _TESTS = [{
+        # Embedded via <iframe src="http://www.arte.tv/arte_vp/index.php?json_url=..."
         'url': 'http://www.arte.tv/magazine/trepalium/fr/entretien-avec-le-realisateur-vincent-lannoo-trepalium',
         'md5': '66a093339c1278bb3719157ef07107b2',
         'info_dict': {
@@ -301,6 +312,17 @@ class ArteTVMagazineIE(ArteTVPlus7IE):
             'ext': 'mp4',
             'title': 'Trepalium - Extrait Ep.01',
         },
+    }, {
+        # Embedded via <iframe src="http://www.arte.tv/guide/fr/embed/054813-004-A/medium"
+        'url': 'http://www.arte.tv/magazine/trepalium/fr/episode-0406-replay-trepalium',
+        'md5': 'fedc64fc7a946110fe311634e79782ca',
+        'info_dict': {
+            'id': '054813-004_PLUS7-F',
+            'ext': 'mp4',
+            'title': 'Trepalium (4/6)',
+            'description': 'md5:10057003c34d54e95350be4f9b05cb40',
+            'upload_date': '20160218',
+        },
     }, {
         'url': 'http://www.arte.tv/magazine/metropolis/de/frank-woeste-german-paris-metropolis',
         'only_matching': True,

From 0dd046c16c6829a545177fe4bb10828372c91699 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 21 Feb 2016 13:57:30 +0600
Subject: [PATCH 4068/6219] [arte:magazine] Fix test

---
 youtube_dl/extractor/arte.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index a4c786d78..9a6f49329 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -306,11 +306,12 @@ class ArteTVMagazineIE(ArteTVPlus7IE):
     _TESTS = [{
         # Embedded via <iframe src="http://www.arte.tv/arte_vp/index.php?json_url=..."
         'url': 'http://www.arte.tv/magazine/trepalium/fr/entretien-avec-le-realisateur-vincent-lannoo-trepalium',
-        'md5': '66a093339c1278bb3719157ef07107b2',
+        'md5': '2a9369bcccf847d1c741e51416299f25',
         'info_dict': {
             'id': '065965-000-A',
             'ext': 'mp4',
             'title': 'Trepalium - Extrait Ep.01',
+            'upload_date': '20160121',
         },
     }, {
         # Embedded via <iframe src="http://www.arte.tv/guide/fr/embed/054813-004-A/medium"

From 92c5e11b40c235caee69e92806001aa7899e10eb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 21 Feb 2016 14:23:58 +0600
Subject: [PATCH 4069/6219] [arte:future] Fix test

---
 youtube_dl/extractor/arte.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index 9a6f49329..47840cad0 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -240,6 +240,7 @@ class ArteTVFutureIE(ArteTVPlus7IE):
             'id': '050940-028-A',
             'ext': 'mp4',
             'title': 'Les écrevisses aussi peuvent être anxieuses',
+            'upload_date': '20140902',
         },
     }, {
         'url': 'http://future.arte.tv/fr/la-science-est-elle-responsable',

From 445d72b8b5cdce331833646ba856efe946253056 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 21 Feb 2016 16:41:24 +0800
Subject: [PATCH 4070/6219] [twitter:amplify] Add TwitterAmplifyIE for handling
 Twitter smart URLs

Closes #8075
---
 youtube_dl/extractor/__init__.py |  6 ++++-
 youtube_dl/extractor/twitter.py  | 41 ++++++++++++++++++++++++++++----
 2 files changed, 42 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 1edbfbd28..1ae606f1e 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -798,7 +798,11 @@ from .twitch import (
     TwitchBookmarksIE,
     TwitchStreamIE,
 )
-from .twitter import TwitterCardIE, TwitterIE
+from .twitter import (
+    TwitterCardIE,
+    TwitterIE,
+    TwitterAmplifyIE,
+)
 from .ubu import UbuIE
 from .udemy import (
     UdemyIE,
diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index a161f046b..77c9176e4 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -14,7 +14,13 @@ from ..utils import (
 )
 
 
-class TwitterCardIE(InfoExtractor):
+class TwitterBaseIE(InfoExtractor):
+    def _get_vmap_video_url(self, vmap_url, video_id):
+        vmap_data = self._download_xml(vmap_url, video_id)
+        return xpath_text(vmap_data, './/MediaFile').strip()
+
+
+class TwitterCardIE(TwitterBaseIE):
     IE_NAME = 'twitter:card'
     _VALID_URL = r'https?://(?:www\.)?twitter\.com/i/cards/tfw/v1/(?P<id>\d+)'
     _TESTS = [
@@ -96,10 +102,8 @@ class TwitterCardIE(InfoExtractor):
                 video_id)
             if 'playlist' not in config:
                 if 'vmapUrl' in config:
-                    vmap_data = self._download_xml(config['vmapUrl'], video_id)
-                    video_url = xpath_text(vmap_data, './/MediaFile').strip()
                     formats.append({
-                        'url': video_url,
+                        'url': self._get_vmap_video_url(config['vmapUrl'], video_id),
                     })
                     break   # same video regardless of UA
                 continue
@@ -226,3 +230,32 @@ class TwitterIE(InfoExtractor):
             return info
 
         raise ExtractorError('There\'s not video in this tweet.')
+
+
+class TwitterAmplifyIE(TwitterBaseIE):
+    IE_NAME = 'twitter:amplify'
+    _VALID_URL = 'https?://amp\.twimg\.com/v/(?P<id>[0-9a-f\-]{36})'
+
+    _TEST = {
+        'url': 'https://amp.twimg.com/v/0ba0c3c7-0af3-4c0a-bed5-7efd1ffa2951',
+        'md5': '7df102d0b9fd7066b86f3159f8e81bf6',
+        'info_dict': {
+            'id': '0ba0c3c7-0af3-4c0a-bed5-7efd1ffa2951',
+            'ext': 'mp4',
+            'title': 'Twitter Video',
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        vmap_url = self._html_search_meta(
+            'twitter:amplify:vmap', webpage, 'vmap url')
+        video_url = self._get_vmap_video_url(vmap_url, video_id)
+
+        return {
+            'id': video_id,
+            'title': 'Twitter Video',
+            'url': video_url,
+        }

From acb6e97e6a039b7bcce952e9fcf71a5ccf92aec3 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 21 Feb 2016 16:57:56 +0800
Subject: [PATCH 4071/6219] [twitter] Fix several failed tests

---
 youtube_dl/extractor/twitter.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 77c9176e4..64c429f02 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -26,7 +26,7 @@ class TwitterCardIE(TwitterBaseIE):
     _TESTS = [
         {
             'url': 'https://twitter.com/i/cards/tfw/v1/560070183650213889',
-            'md5': '4fa26a35f9d1bf4b646590ba8e84be19',
+            # MD5 checksums are different in different places
             'info_dict': {
                 'id': '560070183650213889',
                 'ext': 'mp4',
@@ -48,7 +48,7 @@ class TwitterCardIE(TwitterBaseIE):
         },
         {
             'url': 'https://twitter.com/i/cards/tfw/v1/654001591733886977',
-            'md5': 'b6f35e8b08a0bec6c8af77a2f4b3a814',
+            'md5': 'd4724ffe6d2437886d004fa5de1043b3',
             'info_dict': {
                 'id': 'dq4Oj5quskI',
                 'ext': 'mp4',
@@ -68,8 +68,8 @@ class TwitterCardIE(TwitterBaseIE):
                 'ext': 'mp4',
                 'upload_date': '20151113',
                 'uploader_id': '1189339351084113920',
-                'uploader': '@ArsenalTerje',
-                'title': 'Vine by @ArsenalTerje',
+                'uploader': 'ArsenalTerje',
+                'title': 'Vine by ArsenalTerje',
             },
             'add_ie': ['Vine'],
         }
@@ -142,7 +142,7 @@ class TwitterIE(InfoExtractor):
 
     _TESTS = [{
         'url': 'https://twitter.com/freethenipple/status/643211948184596480',
-        'md5': 'db6612ec5d03355953c3ca9250c97e5e',
+        # MD5 checksums are different in different places
         'info_dict': {
             'id': '643211948184596480',
             'ext': 'mp4',

From bdbf4ba40ebae986ddf65fd1da4741af9ddf3ec9 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 21 Feb 2016 17:16:35 +0800
Subject: [PATCH 4072/6219] [twitter:amplify] Extract more metadata

---
 youtube_dl/extractor/twitter.py | 30 +++++++++++++++++++++++++++++-
 1 file changed, 29 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 64c429f02..29ce84f5e 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -243,6 +243,7 @@ class TwitterAmplifyIE(TwitterBaseIE):
             'id': '0ba0c3c7-0af3-4c0a-bed5-7efd1ffa2951',
             'ext': 'mp4',
             'title': 'Twitter Video',
+            'thumbnail': 're:^https?://.*',
         },
     }
 
@@ -254,8 +255,35 @@ class TwitterAmplifyIE(TwitterBaseIE):
             'twitter:amplify:vmap', webpage, 'vmap url')
         video_url = self._get_vmap_video_url(vmap_url, video_id)
 
+        thumbnails = []
+        thumbnail = self._html_search_meta(
+            'twitter:image:src', webpage, 'thumbnail', fatal=False)
+
+        def _find_dimension(target):
+            w = int_or_none(self._html_search_meta(
+                'twitter:%s:width' % target, webpage, fatal=False))
+            h = int_or_none(self._html_search_meta(
+                'twitter:%s:height' % target, webpage, fatal=False))
+            return w, h
+
+        if thumbnail:
+            thumbnail_w, thumbnail_h = _find_dimension('image')
+            thumbnails.append({
+                'url': thumbnail,
+                'width': thumbnail_w,
+                'height': thumbnail_h,
+            })
+
+        video_w, video_h = _find_dimension('player')
+        formats = [{
+            'url': video_url,
+            'width': video_w,
+            'height': video_h,
+        }]
+
         return {
             'id': video_id,
             'title': 'Twitter Video',
-            'url': video_url,
+            'formats': formats,
+            'thumbnails': thumbnails,
         }

From 132e3b74bdcb73ccacbfeecdae5713dd65829af9 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 21 Feb 2016 17:21:37 +0800
Subject: [PATCH 4073/6219] [twitter] Fix a typo

---
 youtube_dl/extractor/twitter.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 29ce84f5e..8acd09f6a 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -229,7 +229,7 @@ class TwitterIE(InfoExtractor):
             })
             return info
 
-        raise ExtractorError('There\'s not video in this tweet.')
+        raise ExtractorError('There\'s no video in this tweet.')
 
 
 class TwitterAmplifyIE(TwitterBaseIE):

From 7efc1c2b493aa1d68e5f1b248980144bbacf943a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 21 Feb 2016 17:29:28 +0800
Subject: [PATCH 4074/6219] [twitter] Fix metadata extraction and
 test_Twitter_1

---
 youtube_dl/extractor/twitter.py | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 8acd09f6a..5d2b5ec35 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -165,6 +165,7 @@ class TwitterIE(InfoExtractor):
             'uploader': 'Gifs',
             'uploader_id': 'giphz',
         },
+        'expected_warnings': ['height', 'width'],
     }, {
         'url': 'https://twitter.com/starwars/status/665052190608723968',
         'md5': '39b7199856dee6cd4432e72c74bc69d4',
@@ -212,20 +213,24 @@ class TwitterIE(InfoExtractor):
             return info
 
         mobj = re.search(r'''(?x)
-            <video[^>]+class="animated-gif"[^>]+
-                (?:data-height="(?P<height>\d+)")?[^>]+
-                (?:data-width="(?P<width>\d+)")?[^>]+
-                (?:poster="(?P<poster>[^"]+)")?[^>]*>\s*
+            <video[^>]+class="animated-gif"(?P<more_info>[^>]+)>\s*
                 <source[^>]+video-src="(?P<url>[^"]+)"
         ''', webpage)
 
         if mobj:
+            more_info = mobj.group('more_info')
+            height = int_or_none(self._search_regex(
+                r'data-height="(\d+)"', more_info, 'height', fatal=False))
+            width = int_or_none(self._search_regex(
+                r'data-width="(\d+)"', more_info, 'width', fatal=False))
+            thumbnail = self._search_regex(
+                r'poster="([^"]+)"', more_info, 'poster', fatal=False)
             info.update({
                 'id': twid,
                 'url': mobj.group('url'),
-                'height': int_or_none(mobj.group('height')),
-                'width': int_or_none(mobj.group('width')),
-                'thumbnail': mobj.group('poster'),
+                'height': height,
+                'width': width,
+                'thumbnail': thumbnail,
             })
             return info
 

From f5bdb4444338f1462c2bc7f79b4ad7e260fe56a8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 22 Feb 2016 01:19:39 +0600
Subject: [PATCH 4075/6219] [extractor/common] Add _remove_duplicate_formats

---
 youtube_dl/extractor/common.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index c85dcda0c..14f575635 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -900,6 +900,16 @@ class InfoExtractor(object):
                     item='%s video format' % f.get('format_id') if f.get('format_id') else 'video'),
                 formats)
 
+    @staticmethod
+    def _remove_duplicate_formats(formats):
+        format_urls = set()
+        unique_formats = []
+        for f in formats:
+            if f['url'] not in format_urls:
+                format_urls.add(f['url'])
+                unique_formats.append(f)
+        formats[:] = unique_formats
+
     def _is_valid_url(self, url, video_id, item='video'):
         url = self._proto_relative_url(url, scheme='http:')
         # For now assume non HTTP(S) URLs always valid

From 5d6e1011df826b9ee0f440f800f2a69e42023b86 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 22 Feb 2016 01:23:27 +0600
Subject: [PATCH 4076/6219] [pbs] Extract all formats (Closes #8538)

---
 youtube_dl/extractor/pbs.py | 72 +++++++++++++++++++++++++++----------
 1 file changed, 54 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index cca012953..f43e3a146 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -337,6 +337,21 @@ class PBSIE(InfoExtractor):
                 'skip_download': True,  # requires ffmpeg
             },
         },
+        {
+            # Serves hd only via wigget/partnerplayer page
+            'url': 'http://www.pbs.org/video/2365641075/',
+            'info_dict': {
+                'id': '2365641075',
+                'ext': 'mp4',
+                'title': 'FRONTLINE - Netanyahu at War',
+                'duration': 6852,
+                'thumbnail': 're:^https?://.*\.jpg$',
+                'formats': 'mincount:8',
+            },
+            'params': {
+                'skip_download': True,  # requires ffmpeg
+            },
+        },
         {
             'url': 'http://player.pbs.org/widget/partnerplayer/2365297708/?start=0&end=0&chapterbar=false&endscreen=false&topbar=true',
             'only_matching': True,
@@ -437,34 +452,54 @@ class PBSIE(InfoExtractor):
                 for vid_id in video_id]
             return self.playlist_result(entries, display_id)
 
+        info = None
+        redirects = []
+        redirect_urls = set()
+
+        def extract_redirect_urls(info):
+            for encoding_name in ('recommended_encoding', 'alternate_encoding'):
+                redirect = info.get(encoding_name)
+                if not redirect:
+                    continue
+                redirect_url = redirect.get('url')
+                if redirect_url and redirect_url not in redirect_urls:
+                    redirects.append(redirect)
+                    redirect_urls.add(redirect_url)
+
         try:
-            info = self._download_json(
+            video_info = self._download_json(
                 'http://player.pbs.org/videoInfo/%s?format=json&type=partner' % video_id,
                 display_id, 'Downloading video info JSON')
+            extract_redirect_urls(video_info)
+            info = video_info
         except ExtractorError as e:
+            # videoInfo API may not work for some videos
             if not isinstance(e.cause, compat_HTTPError) or e.cause.code != 404:
                 raise
-            # videoInfo API may not work for some videos, fallback to portalplayer API
+
+        # Player pages may also serve different qualities
+        for page in ('widget/partnerplayer', 'portalplayer'):
             player = self._download_webpage(
-                'http://player.pbs.org/portalplayer/%s' % video_id, display_id)
-            info = self._parse_json(
-                self._search_regex(
-                    r'(?s)PBS\.videoData\s*=\s*({.+?});\n',
-                    player, 'video data', default='{}'),
-                display_id, transform_source=js_to_json, fatal=False)
+                'http://player.pbs.org/%s/%s' % (page, video_id),
+                display_id, 'Downloading %s page' % page, fatal=False)
+            if player:
+                video_info = self._parse_json(
+                    self._search_regex(
+                        r'(?s)PBS\.videoData\s*=\s*({.+?});\n',
+                        player, '%s video data' % page, default='{}'),
+                    display_id, transform_source=js_to_json, fatal=False)
+                if video_info:
+                    extract_redirect_urls(video_info)
+                    if not info:
+                        info = video_info
 
         formats = []
-        for encoding_name in ('recommended_encoding', 'alternate_encoding'):
-            redirect = info.get(encoding_name)
-            if not redirect:
-                continue
-            redirect_url = redirect.get('url')
-            if not redirect_url:
-                continue
+        for num, redirect in enumerate(redirects):
+            redirect_id = redirect.get('eeid')
 
             redirect_info = self._download_json(
-                redirect_url + '?format=json', display_id,
-                'Downloading %s video url info' % encoding_name)
+                '%s?format=json' % redirect['url'], display_id,
+                'Downloading %s video url info' % (redirect_id or num))
 
             if redirect_info['status'] == 'error':
                 raise ExtractorError(
@@ -483,8 +518,9 @@ class PBSIE(InfoExtractor):
             else:
                 formats.append({
                     'url': format_url,
-                    'format_id': redirect.get('eeid'),
+                    'format_id': redirect_id,
                 })
+        self._remove_duplicate_formats(formats)
         self._sort_formats(formats)
 
         rating_str = info.get('rating')

From 1d00a8823e930bfb092c65ec65aa37e32a1ecf5a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 22 Feb 2016 01:32:23 +0600
Subject: [PATCH 4077/6219] [arte] PEP 8

---
 youtube_dl/extractor/arte.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index 47840cad0..efde7e207 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -114,7 +114,7 @@ class ArteTVPlus7IE(InfoExtractor):
                     r'program\s*:\s*({.+?["\']embed_html["\'].+?}),?\s*\n',
                     webpage, 'program', default=None)
                 if program:
-                    embed_html = self._parse_json(program,video_id)
+                    embed_html = self._parse_json(program, video_id)
                     if embed_html:
                         iframe_url = find_iframe_url(embed_html['embed_html'])
             if iframe_url:
@@ -129,7 +129,6 @@ class ArteTVPlus7IE(InfoExtractor):
             webpage, 'embed url', group='url')
         return self.url_result(embed_url)
 
-
     def _extract_from_json_url(self, json_url, video_id, lang):
         info = self._download_json(json_url, video_id)
         player_info = info['videoJsonPlayer']

From d69abbd3f04b5f3fced2c6a53b4fce788e31906d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 22 Feb 2016 03:13:18 +0600
Subject: [PATCH 4078/6219] [googledrive] Make thumbnail optional (Closes
 #8629)

---
 youtube_dl/extractor/googledrive.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/googledrive.py b/youtube_dl/extractor/googledrive.py
index f354c9c7a..37be34091 100644
--- a/youtube_dl/extractor/googledrive.py
+++ b/youtube_dl/extractor/googledrive.py
@@ -82,7 +82,7 @@ class GoogleDriveIE(InfoExtractor):
         return {
             'id': video_id,
             'title': title,
-            'thumbnail': self._og_search_thumbnail(webpage),
+            'thumbnail': self._og_search_thumbnail(webpage, default=None),
             'duration': duration,
             'formats': formats,
         }

From 6212bcb19140da77d25fe17830930ebbd8a22c1d Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 22 Feb 2016 09:57:12 +0100
Subject: [PATCH 4079/6219] [tf1] fix info extraction(fixes #8599)

---
 youtube_dl/extractor/tf1.py |  4 +---
 youtube_dl/extractor/wat.py | 10 ++++++----
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/tf1.py b/youtube_dl/extractor/tf1.py
index 6890021cf..e1a64e284 100644
--- a/youtube_dl/extractor/tf1.py
+++ b/youtube_dl/extractor/tf1.py
@@ -50,6 +50,4 @@ class TF1IE(InfoExtractor):
         wat_id = self._html_search_regex(
             r'(["\'])(?:https?:)?//www\.wat\.tv/embedframe/.*?(?P<id>\d{8})\1',
             webpage, 'wat id', group='id')
-        wat_info = self._download_json(
-            'http://www.wat.tv/interface/contentv3/%s' % wat_id, video_id)
-        return self.url_result(wat_info['media']['url'], 'Wat')
+        return self.url_result('wat:%s' % wat_id, 'Wat')
diff --git a/youtube_dl/extractor/wat.py b/youtube_dl/extractor/wat.py
index affcc52f6..37cf3d309 100644
--- a/youtube_dl/extractor/wat.py
+++ b/youtube_dl/extractor/wat.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 
 class WatIE(InfoExtractor):
-    _VALID_URL = r'http://www\.wat\.tv/video/(?P<display_id>.*)-(?P<short_id>.*?)_.*?\.html'
+    _VALID_URL = r'(?:wat:(?P<real_id>\d{8})|http://www\.wat\.tv/video/(?P<display_id>.*)-(?P<short_id>.*?)_.*?\.html)'
     IE_NAME = 'wat.tv'
     _TESTS = [
         {
@@ -54,10 +54,12 @@ class WatIE(InfoExtractor):
         def real_id_for_chapter(chapter):
             return chapter['tc_start'].split('-')[0]
         mobj = re.match(self._VALID_URL, url)
-        short_id = mobj.group('short_id')
         display_id = mobj.group('display_id')
-        webpage = self._download_webpage(url, display_id or short_id)
-        real_id = self._search_regex(r'xtpage = ".*-(.*?)";', webpage, 'real id')
+        real_id = mobj.group('real_id')
+        if not real_id:
+            short_id = mobj.group('short_id')
+            webpage = self._download_webpage(url, display_id or short_id)
+            real_id = self._search_regex(r'xtpage = ".*-(.*?)";', webpage, 'real id')
 
         video_info = self.download_video_info(real_id)
 

From 92da3cd84800bf285b5d2125421d0c8d6a4aff09 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 22 Feb 2016 11:57:31 +0100
Subject: [PATCH 4080/6219] release 2016.02.22

---
 CONTRIBUTING.md        | 4 ++--
 docs/supportedsites.md | 8 +++++++-
 youtube_dl/version.py  | 2 +-
 3 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index d15267d7e..39472c554 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -1,6 +1,6 @@
-**Please include the full output of youtube-dl when run with `-v`**, i.e. add `-v` flag to your command line, copy the **whole** output and post it in the issue body wrapped in \`\`\` for better formatting. It should look similar to this:
+**Please include the full output of youtube-dl when run with `-v`**, i.e. **add** `-v` flag to **your command line**, copy the **whole** output and post it in the issue body wrapped in \`\`\` for better formatting. It should look similar to this:
 ```
-$ youtube-dl -v http://www.youtube.com/watch?v=BaW_jenozKcj
+$ youtube-dl -v <your command line>
 [debug] System config: []
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index b384a3165..74596155c 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -30,6 +30,7 @@
  - **AlJazeera**
  - **Allocine**
  - **AlphaPorno**
+ - **AnimeOnDemand**
  - **anitube.se**
  - **AnySex**
  - **Aparat**
@@ -49,6 +50,7 @@
  - **arte.tv:ddc**
  - **arte.tv:embed**
  - **arte.tv:future**
+ - **arte.tv:magazine**
  - **AtresPlayer**
  - **ATTTechChannel**
  - **AudiMedia**
@@ -360,7 +362,7 @@
  - **MySpace:album**
  - **MySpass**
  - **Myvi**
- - **myvideo**
+ - **myvideo** (Currently broken)
  - **MyVidster**
  - **n-tv.de**
  - **NationalGeographic**
@@ -410,6 +412,7 @@
  - **NowTV** (Currently broken)
  - **NowTVList**
  - **nowvideo**: NowVideo
+ - **Noz**
  - **npo**: npo.nl and ntr.nl
  - **npo.nl:live**
  - **npo.nl:radio**
@@ -460,6 +463,7 @@
  - **PornHd**
  - **PornHub**
  - **PornHubPlaylist**
+ - **PornHubUserVideos**
  - **Pornotube**
  - **PornoVoisines**
  - **PornoXO**
@@ -522,6 +526,7 @@
  - **screen.yahoo:search**: Yahoo screen search
  - **Screencast**
  - **ScreencastOMatic**
+ - **ScreenJunkies**
  - **ScreenwaveMedia**
  - **SenateISVP**
  - **ServingSys**
@@ -655,6 +660,7 @@
  - **twitch:video**
  - **twitch:vod**
  - **twitter**
+ - **twitter:amplify**
  - **twitter:card**
  - **Ubu**
  - **udemy**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 9aca8001a..7a3df6a26 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.02.13'
+__version__ = '2016.02.22'

From 1ad61430615b8552661ede21f2c3ce76c2e88d2d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 22 Feb 2016 17:37:00 +0600
Subject: [PATCH 4081/6219] [xfileshare] Add support for powerwatch (Closes
 #8628)

---
 youtube_dl/extractor/xfileshare.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/xfileshare.py b/youtube_dl/extractor/xfileshare.py
index a3236e66c..94abdb4f3 100644
--- a/youtube_dl/extractor/xfileshare.py
+++ b/youtube_dl/extractor/xfileshare.py
@@ -17,7 +17,7 @@ class XFileShareIE(InfoExtractor):
     IE_DESC = 'XFileShare based sites: GorillaVid.in, daclips.in, movpod.in, fastvideo.in, realvid.net, filehoot.com and vidto.me'
     _VALID_URL = r'''(?x)
         https?://(?P<host>(?:www\.)?
-            (?:daclips\.in|gorillavid\.in|movpod\.in|fastvideo\.in|realvid\.net|filehoot\.com|vidto\.me))/
+            (?:daclips\.in|gorillavid\.in|movpod\.in|fastvideo\.in|realvid\.net|filehoot\.com|vidto\.me|powerwatch\.pw))/
         (?:embed-)?(?P<id>[0-9a-zA-Z]+)(?:-[0-9]+x[0-9]+\.html)?
     '''
 
@@ -81,6 +81,13 @@ class XFileShareIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'test'
         }
+    }, {
+        'url': 'http://powerwatch.pw/duecjibvicbu',
+        'info_dict': {
+            'id': 'duecjibvicbu',
+            'ext': 'mp4',
+            'title': 'Big Buck Bunny trailer',
+        },
     }]
 
     def _real_extract(self, url):
@@ -112,6 +119,7 @@ class XFileShareIE(InfoExtractor):
         title = (self._search_regex(
             [r'style="z-index: [0-9]+;">([^<]+)</span>',
              r'<td nowrap>([^<]+)</td>',
+             r'h4-fine[^>]*>([^<]+)<',
              r'>Watch (.+) ',
              r'<h2 class="video-page-head">([^<]+)</h2>'],
             webpage, 'title', default=None) or self._og_search_title(webpage)).strip()

From 0fc871d2f0ae57b88f78c32e3f76bfc1ca609ef0 Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Tue, 23 Feb 2016 04:16:47 +0600
Subject: [PATCH 4082/6219] [README.md:output_template] Add example for
 channel/user playlists download

---
 README.md | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/README.md b/README.md
index db49ab6d5..2fbf0219b 100644
--- a/README.md
+++ b/README.md
@@ -525,6 +525,9 @@ youtube-dl_test_video_.mp4          # A simple file name
 # Download YouTube playlist videos in separate directory indexed by video order in a playlist
 $ youtube-dl -o '%(playlist)s/%(playlist_index)s - %(title)s.%(ext)s' https://www.youtube.com/playlist?list=PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re
 
+# Download all playlists of YouTube channel/user keeping each playlist in separate directory:
+$ youtube-dl -o '%(uploader)s/%(playlist)s/%(playlist_index)s - %(title)s.%(ext)s' https://www.youtube.com/user/TheLinuxFoundation/playlists
+
 # Download Udemy course keeping each chapter in separate directory under MyVideos directory in your home
 $ youtube-dl -u user -p password -o '~/MyVideos/%(playlist)s/%(chapter_number)s - %(chapter)s/%(title)s.%(ext)s' https://www.udemy.com/java-tutorial/
 

From 1610f770d7912bead235b2bc0fa0cc84a746be69 Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Tue, 23 Feb 2016 04:29:39 +0600
Subject: [PATCH 4083/6219] [README.md] Extract example subsections

---
 README.md | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 2fbf0219b..309de7782 100644
--- a/README.md
+++ b/README.md
@@ -513,7 +513,9 @@ The current default template is `%(title)s-%(id)s.%(ext)s`.
 
 In some cases, you don't want special characters such as 中, spaces, or &, such as when transferring the downloaded filename to a Windows system or the filename through an 8bit-unsafe channel. In these cases, add the `--restrict-filenames` flag to get a shorter title:
 
-Examples (note on Windows you may need to use double quotes instead of single):
+#### Examples
+
+Note on Windows you may need to use double quotes instead of single.
 
 ```bash
 $ youtube-dl --get-filename -o '%(title)s.%(ext)s' BaW_jenozKc
@@ -595,7 +597,10 @@ Since the end of April 2015 and version 2015.04.26 youtube-dl uses `-f bestvideo
 
 If you want to preserve the old format selection behavior (prior to youtube-dl 2015.04.26), i.e. you want to download the best available quality media served as a single file, you should explicitly specify your choice with `-f best`. You may want to add it to the [configuration file](#configuration) in order not to type it every time you run youtube-dl.
 
-Examples (note on Windows you may need to use double quotes instead of single):
+#### Examples
+
+Note on Windows you may need to use double quotes instead of single.
+
 ```bash
 # Download best mp4 format available or any other best if no mp4 available
 $ youtube-dl -f 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/best[ext=mp4]/best'

From 621bd0cda9b1d95e0b595d181b3576967abaa66a Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Tue, 23 Feb 2016 04:43:45 +0600
Subject: [PATCH 4084/6219] [README.md] Add tl;dr links to examples

---
 README.md | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index 309de7782..dba580390 100644
--- a/README.md
+++ b/README.md
@@ -440,7 +440,11 @@ On Windows you may also need to setup the `%HOME%` environment variable manually
 
 # OUTPUT TEMPLATE
 
-The `-o` option allows users to indicate a template for the output file names. The basic usage is not to set any template arguments when downloading a single file, like in `youtube-dl -o funny_video.flv "http://some/video"`. However, it may contain special sequences that will be replaced when downloading each video. The special sequences have the format `%(NAME)s`. To clarify, that is a percent symbol followed by a name in parentheses, followed by a lowercase S. Allowed names are:
+The `-o` option allows users to indicate a template for the output file names.
+
+**tl;dr:** [navigate me to examples](#output-template-examples).
+
+The basic usage is not to set any template arguments when downloading a single file, like in `youtube-dl -o funny_video.flv "http://some/video"`. However, it may contain special sequences that will be replaced when downloading each video. The special sequences have the format `%(NAME)s`. To clarify, that is a percent symbol followed by a name in parentheses, followed by a lowercase S. Allowed names are:
 
  - `id`: Video identifier
  - `title`: Video title
@@ -513,7 +517,7 @@ The current default template is `%(title)s-%(id)s.%(ext)s`.
 
 In some cases, you don't want special characters such as 中, spaces, or &, such as when transferring the downloaded filename to a Windows system or the filename through an 8bit-unsafe channel. In these cases, add the `--restrict-filenames` flag to get a shorter title:
 
-#### Examples
+#### Output template examples
 
 Note on Windows you may need to use double quotes instead of single.
 
@@ -548,6 +552,8 @@ But sometimes you may want to download in a different format, for example when y
 
 The general syntax for format selection is `--format FORMAT` or shorter `-f FORMAT` where `FORMAT` is a *selector expression*, i.e. an expression that describes format or formats you would like to download.
 
+**tl;dr:** [navigate me to examples](#format-selection-examples).
+
 The simplest case is requesting a specific format, for example with `-f 22` you can download the format with format code equal to 22. You can get the list of available format codes for particular video using `--list-formats` or `-F`. Note that these format codes are extractor specific. 
 
 You can also use a file extension (currently `3gp`, `aac`, `flv`, `m4a`, `mp3`, `mp4`, `ogg`, `wav`, `webm` are supported) to download best quality format of particular file extension served as a single file, e.g. `-f webm` will download best quality format with `webm` extension served as a single file.
@@ -597,7 +603,7 @@ Since the end of April 2015 and version 2015.04.26 youtube-dl uses `-f bestvideo
 
 If you want to preserve the old format selection behavior (prior to youtube-dl 2015.04.26), i.e. you want to download the best available quality media served as a single file, you should explicitly specify your choice with `-f best`. You may want to add it to the [configuration file](#configuration) in order not to type it every time you run youtube-dl.
 
-#### Examples
+#### Format selection examples
 
 Note on Windows you may need to use double quotes instead of single.
 

From 58525c94d547be1c8167d16c298bdd75506db328 Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Tue, 23 Feb 2016 04:58:51 +0600
Subject: [PATCH 4085/6219] [README.md] Emphasize copyright infringement
 aspects in add-new-site-support tutorial

---
 README.md | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index dba580390..60635d5df 100644
--- a/README.md
+++ b/README.md
@@ -830,7 +830,9 @@ If you want to create a build of youtube-dl yourself, you'll need
 
 ### Adding support for a new site
 
-If you want to add support for a new site, you can follow this quick list (assuming your service is called `yourextractor`):
+If you want to add support for a new site, first of all **make sure** this site is **not dedicated to [copyright infringement](#can-you-add-support-for-this-anime-video-site-or-site-which-shows-current-movies-for-free)**. youtube-dl does **not support** such sites thus pull requests adding support for them **will be rejected**.
+
+After you have ensured this site is distributing it's content legally, you can follow this quick list (assuming your service is called `yourextractor`):
 
 1. [Fork this repository](https://github.com/rg3/youtube-dl/fork)
 2. Check out the source code with `git clone git@github.com:YOUR_GITHUB_USERNAME/youtube-dl.git`

From 9b012728324737977e875ddd623fa4c6852e86a9 Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Tue, 23 Feb 2016 05:03:57 +0600
Subject: [PATCH 4086/6219] [README.md] Update link to extractor metafields

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 60635d5df..48b76b1ec 100644
--- a/README.md
+++ b/README.md
@@ -880,7 +880,7 @@ After you have ensured this site is distributing it's content legally, you can f
     ```
 5. Add an import in [`youtube_dl/extractor/__init__.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/__init__.py).
 6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc.
-7. Have a look at [`youtube_dl/extractor/common.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L62-L200). Add tests and code for as many as you want.
+7. Have a look at [`youtube_dl/extractor/common.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/rg3/youtube-dl/blob/58525c94d547be1c8167d16c298bdd75506db328/youtube_dl/extractor/common.py#L68-L226). Add tests and code for as many as you want.
 8. If you can, check the code with [flake8](https://pypi.python.org/pypi/flake8).
 9. When the tests pass, [add](http://git-scm.com/docs/git-add) the new files and [commit](http://git-scm.com/docs/git-commit) them and [push](http://git-scm.com/docs/git-push) the result, like this:
 

From cf89ba9eff2f8d76d3b4a54d07c7b4bb3e2cb76f Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Tue, 23 Feb 2016 05:35:19 +0600
Subject: [PATCH 4087/6219] [README.md] Clarify robustness and future-proof
 requirements for new extractors

---
 README.md | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 48b76b1ec..244ed2b59 100644
--- a/README.md
+++ b/README.md
@@ -881,8 +881,9 @@ After you have ensured this site is distributing it's content legally, you can f
 5. Add an import in [`youtube_dl/extractor/__init__.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/__init__.py).
 6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc.
 7. Have a look at [`youtube_dl/extractor/common.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/rg3/youtube-dl/blob/58525c94d547be1c8167d16c298bdd75506db328/youtube_dl/extractor/common.py#L68-L226). Add tests and code for as many as you want.
-8. If you can, check the code with [flake8](https://pypi.python.org/pypi/flake8).
-9. When the tests pass, [add](http://git-scm.com/docs/git-add) the new files and [commit](http://git-scm.com/docs/git-commit) them and [push](http://git-scm.com/docs/git-push) the result, like this:
+8. Keep in mind that the only mandatory fields in info dict for successful extraction process are `id`, `title` and either `url` or `formats`, e.g. these are the critical data the extraction does not make any sense without. This means that [any field](https://github.com/rg3/youtube-dl/blob/58525c94d547be1c8167d16c298bdd75506db328/youtube_dl/extractor/common.py#L138-L226) apart from aforementioned mandatory ones should be treated **as optional** and extraction should be **tolerate** to situations when sources for these fields can potentially be unavailable (even if they always available at the moment) and **future-proof** in order not to break the extraction of general purpose mandatory fields. For example, if you have some intermediate dict `meta` that is a source of metadata and it has a key `summary` that you want to extract and put into resulting info dict as `description`, you should be ready that this key may be missing from the `meta` dict, i.e. you should extract it as `meta.get('summary')` and not `meta['summary']`. Similarly, you should pass `fatal=False` when extracting data from a webpage with `_search_regex/_html_search_regex`.
+9. Check the code with [flake8](https://pypi.python.org/pypi/flake8).
+10. When the tests pass, [add](http://git-scm.com/docs/git-add) the new files and [commit](http://git-scm.com/docs/git-commit) them and [push](http://git-scm.com/docs/git-push) the result, like this:
 
         $ git add youtube_dl/extractor/__init__.py
         $ git add youtube_dl/extractor/yourextractor.py

From 6563d3171011312e4aa872e9e9fc66bb0a84b77f Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Tue, 23 Feb 2016 05:37:49 +0600
Subject: [PATCH 4088/6219] [README.md] Fix typo

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 244ed2b59..36923b15e 100644
--- a/README.md
+++ b/README.md
@@ -881,7 +881,7 @@ After you have ensured this site is distributing it's content legally, you can f
 5. Add an import in [`youtube_dl/extractor/__init__.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/__init__.py).
 6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc.
 7. Have a look at [`youtube_dl/extractor/common.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/rg3/youtube-dl/blob/58525c94d547be1c8167d16c298bdd75506db328/youtube_dl/extractor/common.py#L68-L226). Add tests and code for as many as you want.
-8. Keep in mind that the only mandatory fields in info dict for successful extraction process are `id`, `title` and either `url` or `formats`, e.g. these are the critical data the extraction does not make any sense without. This means that [any field](https://github.com/rg3/youtube-dl/blob/58525c94d547be1c8167d16c298bdd75506db328/youtube_dl/extractor/common.py#L138-L226) apart from aforementioned mandatory ones should be treated **as optional** and extraction should be **tolerate** to situations when sources for these fields can potentially be unavailable (even if they always available at the moment) and **future-proof** in order not to break the extraction of general purpose mandatory fields. For example, if you have some intermediate dict `meta` that is a source of metadata and it has a key `summary` that you want to extract and put into resulting info dict as `description`, you should be ready that this key may be missing from the `meta` dict, i.e. you should extract it as `meta.get('summary')` and not `meta['summary']`. Similarly, you should pass `fatal=False` when extracting data from a webpage with `_search_regex/_html_search_regex`.
+8. Keep in mind that the only mandatory fields in info dict for successful extraction process are `id`, `title` and either `url` or `formats`, i.e. these are the critical data the extraction does not make any sense without. This means that [any field](https://github.com/rg3/youtube-dl/blob/58525c94d547be1c8167d16c298bdd75506db328/youtube_dl/extractor/common.py#L138-L226) apart from aforementioned mandatory ones should be treated **as optional** and extraction should be **tolerate** to situations when sources for these fields can potentially be unavailable (even if they always available at the moment) and **future-proof** in order not to break the extraction of general purpose mandatory fields. For example, if you have some intermediate dict `meta` that is a source of metadata and it has a key `summary` that you want to extract and put into resulting info dict as `description`, you should be ready that this key may be missing from the `meta` dict, i.e. you should extract it as `meta.get('summary')` and not `meta['summary']`. Similarly, you should pass `fatal=False` when extracting data from a webpage with `_search_regex/_html_search_regex`.
 9. Check the code with [flake8](https://pypi.python.org/pypi/flake8).
 10. When the tests pass, [add](http://git-scm.com/docs/git-add) the new files and [commit](http://git-scm.com/docs/git-commit) them and [push](http://git-scm.com/docs/git-push) the result, like this:
 

From b43a7a92cdd7e85021f4e4e177b26d982195da31 Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Tue, 23 Feb 2016 05:41:09 +0600
Subject: [PATCH 4089/6219] [README.md] Fix typo

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 36923b15e..cf03e0d54 100644
--- a/README.md
+++ b/README.md
@@ -890,7 +890,7 @@ After you have ensured this site is distributing it's content legally, you can f
         $ git commit -m '[yourextractor] Add new extractor'
         $ git push origin yourextractor
 
-10. Finally, [create a pull request](https://help.github.com/articles/creating-a-pull-request). We'll then review and merge it.
+11. Finally, [create a pull request](https://help.github.com/articles/creating-a-pull-request). We'll then review and merge it.
 
 In any case, thank you very much for your contributions!
 

From 86a7dbe66e9aed051daa8c19a1ba8b860959a30a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 21 Feb 2016 18:20:55 +0800
Subject: [PATCH 4090/6219] [nba] Support non-video/ pages

Fixes #8589
---
 youtube_dl/extractor/nba.py | 30 +++++++++++++++++++++++++++---
 1 file changed, 27 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/nba.py b/youtube_dl/extractor/nba.py
index a071378b6..87b38c4c7 100644
--- a/youtube_dl/extractor/nba.py
+++ b/youtube_dl/extractor/nba.py
@@ -1,18 +1,20 @@
 from __future__ import unicode_literals
 
+import os.path
 import re
 
 from .common import InfoExtractor
 from ..utils import (
-    parse_duration,
     int_or_none,
+    parse_duration,
+    remove_start,
     xpath_text,
     xpath_attr,
 )
 
 
 class NBAIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:watch\.|www\.)?nba\.com/(?P<path>(?:[^/]+/)?video/(?P<id>[^?]*?))/?(?:/index\.html)?(?:\?.*)?$'
+    _VALID_URL = r'https?://(?:watch\.|www\.)?nba\.com/(?P<path>(?:[^/]+/)+(?P<id>[^?]*?))/?(?:/index\.html)?(?:\?.*)?$'
     _TESTS = [{
         'url': 'http://www.nba.com/video/games/nets/2012/12/04/0021200253-okc-bkn-recap.nba/index.html',
         'md5': '9e7729d3010a9c71506fd1248f74e4f4',
@@ -44,14 +46,36 @@ class NBAIE(InfoExtractor):
             'timestamp': 1432134543,
             'upload_date': '20150520',
         }
+    }, {
+        'url': 'http://www.nba.com/clippers/news/doc-rivers-were-not-trading-blake',
+        'info_dict': {
+            'id': '1455672027478-Doc_Feb16_720',
+            'ext': 'mp4',
+            'title': 'Practice: Doc Rivers - 2/16/16',
+            'description': 'Head Coach Doc Rivers addresses the media following practice.',
+            'upload_date': '20160217',
+            'timestamp': 1455672000,
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
     }]
 
     def _real_extract(self, url):
         path, video_id = re.match(self._VALID_URL, url).groups()
         if path.startswith('nba/'):
             path = path[3:]
+
+        if 'video/' not in path:
+            webpage = self._download_webpage(url, video_id)
+            path = remove_start(self._search_regex(r'data-videoid="([^"]+)"', webpage, 'video id'), '/')
+            # See prepareContentId() of pkgCvp.js
+            if path.startswith('video/teams'):
+                path = 'video/channels/proxy/' + path[6:]
+
         video_info = self._download_xml('http://www.nba.com/%s.xml' % path, video_id)
-        video_id = xpath_text(video_info, 'slug')
+        video_id = os.path.splitext(xpath_text(video_info, 'slug'))[0]
         title = xpath_text(video_info, 'headline')
         description = xpath_text(video_info, 'description')
         duration = parse_duration(xpath_text(video_info, 'length'))

From b95dc034ca96fbb19995c54a8c75b5d46116a1e1 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 23 Feb 2016 12:17:02 +0800
Subject: [PATCH 4091/6219] [utils] Implement cache for OnDemandPagedList

---
 youtube_dl/utils.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 6978a10e4..8ec70f93c 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1570,9 +1570,12 @@ class PagedList(object):
 
 
 class OnDemandPagedList(PagedList):
-    def __init__(self, pagefunc, pagesize):
+    def __init__(self, pagefunc, pagesize, use_cache=False):
         self._pagefunc = pagefunc
         self._pagesize = pagesize
+        self._use_cache = use_cache
+        if use_cache:
+            self._cache = {}
 
     def getslice(self, start=0, end=None):
         res = []
@@ -1582,7 +1585,13 @@ class OnDemandPagedList(PagedList):
             if start >= nextfirstid:
                 continue
 
-            page_results = list(self._pagefunc(pagenum))
+            page_results = None
+            if self._use_cache:
+                page_results = self._cache.get(pagenum)
+            if page_results is None:
+                page_results = list(self._pagefunc(pagenum))
+            if self._use_cache:
+                self._cache[pagenum] = page_results
 
             startv = (
                 start % self._pagesize

From 3cfeb1624a28b60cec24dbc0d21565f4afd924b3 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 23 Feb 2016 12:17:41 +0800
Subject: [PATCH 4092/6219] [nba] Support channels (#5362, #4167)

---
 youtube_dl/extractor/nba.py | 71 +++++++++++++++++++++++++++++++++++++
 1 file changed, 71 insertions(+)

diff --git a/youtube_dl/extractor/nba.py b/youtube_dl/extractor/nba.py
index 87b38c4c7..3e2b3e599 100644
--- a/youtube_dl/extractor/nba.py
+++ b/youtube_dl/extractor/nba.py
@@ -1,11 +1,17 @@
 from __future__ import unicode_literals
 
+import functools
 import os.path
 import re
 
 from .common import InfoExtractor
+from ..compat import (
+    compat_urllib_parse,
+    compat_urlparse,
+)
 from ..utils import (
     int_or_none,
+    OnDemandPagedList,
     parse_duration,
     remove_start,
     xpath_text,
@@ -60,16 +66,81 @@ class NBAIE(InfoExtractor):
             # m3u8 download
             'skip_download': True,
         },
+    }, {
+        'url': 'http://www.nba.com/timberwolves/wiggins-shootaround#',
+        'info_dict': {
+            'id': 'timberwolves',
+            'title': 'Shootaround Access - Dec. 12 | Andrew Wiggins',
+        },
+        'playlist_count': 30,
+        'params': {
+            # Download the whole playlist takes too long time
+            'playlist_items': '1-30',
+        },
+    }, {
+        'url': 'http://www.nba.com/timberwolves/wiggins-shootaround#',
+        'info_dict': {
+            'id': 'Wigginsmp4',
+            'ext': 'mp4',
+            'title': 'Shootaround Access - Dec. 12 | Andrew Wiggins',
+            'description': 'Wolves rookie Andrew Wiggins addresses the media after Friday\'s shootaround.',
+            'upload_date': '20141212',
+            'timestamp': 1418418600,
+        },
+        'params': {
+            'noplaylist': True,
+            # m3u8 download
+            'skip_download': True,
+        },
     }]
 
+    _PAGE_SIZE = 30
+
+    def _fetch_page(self, team, video_id, page):
+        search_url = 'http://searchapp2.nba.com/nba-search/query.jsp?' + compat_urllib_parse.urlencode({
+            'type': 'teamvideo',
+            'start': page * self._PAGE_SIZE + 1,
+            'npp': (page + 1) * self._PAGE_SIZE + 1,
+            'sort': 'recent',
+            'output': 'json',
+            'site': team,
+        })
+        results = self._download_json(
+            search_url, video_id, note='Download page %d of playlist data' % page)['results'][0]
+        for item in results:
+            yield self.url_result(compat_urlparse.urljoin('http://www.nba.com/', item['url']))
+
+    def _extract_playlist(self, orig_path, video_id, webpage):
+        team = orig_path.split('/')[0]
+
+        if self._downloader.params.get('noplaylist'):
+            self.to_screen('Downloading just video because of --no-playlist')
+            video_path = self._search_regex(
+                r'nbaVideoCore\.firstVideo\s*=\s*\'([^\']+)\';', webpage, 'video path')
+            video_url = 'http://www.nba.com/%s/video/%s' % (team, video_path)
+            return self.url_result(video_url)
+
+        self.to_screen('Downloading playlist - add --no-playlist to just download video')
+        playlist_title = self._og_search_title(webpage, fatal=False)
+        entries = OnDemandPagedList(
+            functools.partial(self._fetch_page, team, video_id),
+            self._PAGE_SIZE, use_cache=True)
+
+        return self.playlist_result(entries, team, playlist_title)
+
     def _real_extract(self, url):
         path, video_id = re.match(self._VALID_URL, url).groups()
+        orig_path = path
         if path.startswith('nba/'):
             path = path[3:]
 
         if 'video/' not in path:
             webpage = self._download_webpage(url, video_id)
             path = remove_start(self._search_regex(r'data-videoid="([^"]+)"', webpage, 'video id'), '/')
+
+            if path == '{{id}}':
+                return self._extract_playlist(orig_path, video_id, webpage)
+
             # See prepareContentId() of pkgCvp.js
             if path.startswith('video/teams'):
                 path = 'video/channels/proxy/' + path[6:]

From efbeddead35582126c622a74972af02db9680357 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 23 Feb 2016 13:17:24 +0800
Subject: [PATCH 4093/6219] [facebook] Support mobile URLs (closes #8638)

---
 youtube_dl/extractor/facebook.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index 0a9a5ca71..d2ff38140 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -34,8 +34,9 @@ class FacebookIE(InfoExtractor):
                                 video/video\.php|
                                 photo\.php|
                                 video\.php|
-                                video/embed
-                            )\?(?:.*?)(?:v|video_id)=|
+                                video/embed|
+                                story\.php
+                            )\?(?:.*?)(?:v|video_id|story_fbid)=|
                             [^/]+/videos/(?:[^/]+/)?
                         )|
                     facebook:
@@ -92,6 +93,9 @@ class FacebookIE(InfoExtractor):
     }, {
         'url': 'facebook:544765982287235',
         'only_matching': True,
+    }, {
+        'url': 'https://m.facebook.com/story.php?story_fbid=1035862816472149&id=116132035111903',
+        'only_matching': True,
     }]
 
     def _login(self):

From bf4b3b6bd91137b94d5ae8ac6dc421e64a647355 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 23 Feb 2016 18:47:13 +0600
Subject: [PATCH 4094/6219] [vk] Extract video URL from extra_data (Closes
 #8646)

---
 youtube_dl/extractor/vk.py | 31 +++++++++++++++++++++++++------
 1 file changed, 25 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 0805e3c08..670a438af 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -11,6 +11,7 @@ from ..compat import (
 )
 from ..utils import (
     ExtractorError,
+    int_or_none,
     orderedSet,
     sanitized_Request,
     str_to_int,
@@ -151,6 +152,19 @@ class VKIE(InfoExtractor):
                 'view_count': int,
             },
         },
+        {
+            # video key is extra_data not url\d+
+            'url': 'http://vk.com/video-110305615_171782105',
+            'md5': 'e13fcda136f99764872e739d13fac1d1',
+            'info_dict': {
+                'id': '171782105',
+                'ext': 'mp4',
+                'title': 'S-Dance, репетиции к The way show',
+                'uploader': 'THE WAY SHOW | 17 апреля',
+                'upload_date': '20160207',
+                'view_count': int,
+            },
+        },
         {
             # removed video, just testing that we match the pattern
             'url': 'http://vk.com/feed?z=video-43215063_166094326%2Fbb50cacd3177146d7a',
@@ -298,12 +312,17 @@ class VKIE(InfoExtractor):
             view_count = str_to_int(self._search_regex(
                 r'([\d,.]+)', views, 'view count', fatal=False))
 
-        formats = [{
-            'format_id': k,
-            'url': v,
-            'width': int(k[len('url'):]),
-        } for k, v in data.items()
-            if k.startswith('url')]
+        formats = []
+        for k, v in data.items():
+            if not k.startswith('url') and k != 'extra_data' or not v:
+                continue
+            height = int_or_none(self._search_regex(
+                r'^url(\d+)', k, 'height', default=None))
+            formats.append({
+                'format_id': k,
+                'url': v,
+                'height': height,
+            })
         self._sort_formats(formats)
 
         return {

From 1b77ee6248b2a9967d3d3e809e416bd26fd7342c Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 24 Feb 2016 03:32:12 +0800
Subject: [PATCH 4095/6219] [c56] Support videos hosted on Sohu (closes #8073)

---
 youtube_dl/extractor/c56.py | 22 ++++++++++++++++++++--
 1 file changed, 20 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/c56.py b/youtube_dl/extractor/c56.py
index cb96c3876..cac8fdcba 100644
--- a/youtube_dl/extractor/c56.py
+++ b/youtube_dl/extractor/c56.py
@@ -4,12 +4,13 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..utils import js_to_json
 
 
 class C56IE(InfoExtractor):
     _VALID_URL = r'https?://(?:(?:www|player)\.)?56\.com/(?:.+?/)?(?:v_|(?:play_album.+-))(?P<textid>.+?)\.(?:html|swf)'
     IE_NAME = '56.com'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.56.com/u39/v_OTM0NDA3MTY.html',
         'md5': 'e59995ac63d0457783ea05f93f12a866',
         'info_dict': {
@@ -18,12 +19,29 @@ class C56IE(InfoExtractor):
             'title': '网事知多少 第32期：车怒',
             'duration': 283.813,
         },
-    }
+    }, {
+        'url': 'http://www.56.com/u47/v_MTM5NjQ5ODc2.html',
+        'md5': '',
+        'info_dict': {
+            'id': '82247482',
+            'title': '爱的诅咒之杜鹃花开',
+        },
+        'playlist_count': 7,
+        'add_ie': ['Sohu'],
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url, flags=re.VERBOSE)
         text_id = mobj.group('textid')
 
+        webpage = self._download_webpage(url, text_id)
+        sohu_video_info_str = self._search_regex(
+            r'var\s+sohuVideoInfo\s*=\s*({[^}]+});', webpage, 'Sohu video info', default=None)
+        if sohu_video_info_str:
+            sohu_video_info = self._parse_json(
+                sohu_video_info_str, text_id, transform_source=js_to_json)
+            return self.url_result(sohu_video_info['url'], 'Sohu')
+
         page = self._download_json(
             'http://vxml.56.com/json/%s/' % text_id, text_id, 'Downloading video info')
 

From c24883a1c0e650a41b54e7cdec8427b046c27bc1 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 24 Feb 2016 03:43:24 +0800
Subject: [PATCH 4096/6219] [facebook] Fix format sorting

'hd' formats should have higher priorities
---
 youtube_dl/extractor/facebook.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index d2ff38140..6c6c3b1bd 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -212,10 +212,13 @@ class FacebookIE(InfoExtractor):
                 for src_type in ('src', 'src_no_ratelimit'):
                     src = f[0].get('%s_%s' % (quality, src_type))
                     if src:
+                        preference = -10 if format_id == 'progressive' else 0
+                        if quality == 'hd':
+                            preference += 5
                         formats.append({
                             'format_id': '%s_%s_%s' % (format_id, quality, src_type),
                             'url': src,
-                            'preference': -10 if format_id == 'progressive' else 0,
+                            'preference': preference,
                         })
             dash_manifest = f[0].get('dash_manifest')
             if dash_manifest:

From c0da50d2b2ba75e5d497caa8805ebac49845f168 Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Wed, 24 Feb 2016 23:05:23 +0600
Subject: [PATCH 4097/6219] [README.md] Turn references to issues to links

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index cf03e0d54..d50d4b950 100644
--- a/README.md
+++ b/README.md
@@ -599,7 +599,7 @@ You can merge the video and audio of two formats into a single file using `-f <v
 
 Format selectors can also be grouped using parentheses, for example if you want to download the best mp4 and webm formats with a height lower than 480 you can use `-f '(mp4,webm)[height<480]'`.
 
-Since the end of April 2015 and version 2015.04.26 youtube-dl uses `-f bestvideo+bestaudio/best` as default format selection (see #5447, #5456). If ffmpeg or avconv are installed this results in downloading `bestvideo` and `bestaudio` separately and muxing them together into a single file giving the best overall quality available. Otherwise it falls back to `best` and results in downloading the best available quality served as a single file. `best` is also needed for videos that don't come from YouTube because they don't provide the audio and video in two different files. If you want to only download some DASH formats (for example if you are not interested in getting videos with a resolution higher than 1080p), you can add `-f bestvideo[height<=?1080]+bestaudio/best` to your configuration file. Note that if you use youtube-dl to stream to `stdout` (and most likely to pipe it to your media player then), i.e. you explicitly specify output template as `-o -`, youtube-dl still uses `-f best` format selection in order to start content delivery immediately to your player and not to wait until `bestvideo` and `bestaudio` are downloaded and muxed.
+Since the end of April 2015 and version 2015.04.26 youtube-dl uses `-f bestvideo+bestaudio/best` as default format selection (see [#5447](https://github.com/rg3/youtube-dl/issues/5447), [#5456](https://github.com/rg3/youtube-dl/issues/5456)). If ffmpeg or avconv are installed this results in downloading `bestvideo` and `bestaudio` separately and muxing them together into a single file giving the best overall quality available. Otherwise it falls back to `best` and results in downloading the best available quality served as a single file. `best` is also needed for videos that don't come from YouTube because they don't provide the audio and video in two different files. If you want to only download some DASH formats (for example if you are not interested in getting videos with a resolution higher than 1080p), you can add `-f bestvideo[height<=?1080]+bestaudio/best` to your configuration file. Note that if you use youtube-dl to stream to `stdout` (and most likely to pipe it to your media player then), i.e. you explicitly specify output template as `-o -`, youtube-dl still uses `-f best` format selection in order to start content delivery immediately to your player and not to wait until `bestvideo` and `bestaudio` are downloaded and muxed.
 
 If you want to preserve the old format selection behavior (prior to youtube-dl 2015.04.26), i.e. you want to download the best available quality media served as a single file, you should explicitly specify your choice with `-f best`. You may want to add it to the [configuration file](#configuration) in order not to type it every time you run youtube-dl.
 
@@ -747,7 +747,7 @@ means you're using an outdated version of Python. Please update to Python 2.6 or
 
 ### What is this binary file? Where has the code gone?
 
-Since June 2012 (#342) youtube-dl is packed as an executable zipfile, simply unzip it (might need renaming to `youtube-dl.zip` first on some systems) or clone the git repository, as laid out above. If you modify the code, you can run it by executing the `__main__.py` file. To recompile the executable, run `make youtube-dl`.
+Since June 2012 ([#342](https://github.com/rg3/youtube-dl/issues/342)) youtube-dl is packed as an executable zipfile, simply unzip it (might need renaming to `youtube-dl.zip` first on some systems) or clone the git repository, as laid out above. If you modify the code, you can run it by executing the `__main__.py` file. To recompile the executable, run `make youtube-dl`.
 
 ### The exe throws a *Runtime error from Visual C++*
 

From 43479d9e9df3aa4a0c919bd1b5ff74d53095cac7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 25 Feb 2016 00:36:14 +0600
Subject: [PATCH 4098/6219] [motherless] Make categories optional (Closes
 #8654)

---
 youtube_dl/extractor/motherless.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/motherless.py b/youtube_dl/extractor/motherless.py
index 97d5da626..b3bfcac9e 100644
--- a/youtube_dl/extractor/motherless.py
+++ b/youtube_dl/extractor/motherless.py
@@ -54,6 +54,11 @@ class MotherlessIE(InfoExtractor):
                 'thumbnail': 're:http://.*\.jpg',
                 'age_limit': 18,
             }
+        },
+        {
+            # no keywords
+            'url': 'http://motherless.com/8B4BBC1',
+            'only_matching': True,
         }
     ]
 
@@ -86,7 +91,7 @@ class MotherlessIE(InfoExtractor):
             r'"thumb-member-username">\s+<a href="/m/([^"]+)"',
             webpage, 'uploader_id')
 
-        categories = self._html_search_meta('keywords', webpage)
+        categories = self._html_search_meta('keywords', webpage, default=None)
         if categories:
             categories = [cat.strip() for cat in categories.split(',')]
 

From 5c0a57185cf925e22b59004150d6b00cd7fd0417 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 25 Feb 2016 00:42:19 +0600
Subject: [PATCH 4099/6219] [motherless] Detect non-existing videos

---
 youtube_dl/extractor/motherless.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/motherless.py b/youtube_dl/extractor/motherless.py
index b3bfcac9e..30e686a4e 100644
--- a/youtube_dl/extractor/motherless.py
+++ b/youtube_dl/extractor/motherless.py
@@ -5,6 +5,7 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
+    ExtractorError,
     str_to_int,
     unified_strdate,
 )
@@ -66,6 +67,11 @@ class MotherlessIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
+        if any(p in webpage for p in (
+                '<title>404 - MOTHERLESS.COM<',
+                ">The page you're looking for cannot be found.<")):
+            raise ExtractorError('Video %s does not exist' % video_id, expected=True)
+
         title = self._html_search_regex(
             r'id="view-upload-title">\s+([^<]+)<', webpage, 'title')
         video_url = self._html_search_regex(

From f160785c5c5a99a2fdc4724f1f66f423cf8f6bf5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 25 Feb 2016 00:52:49 +0600
Subject: [PATCH 4100/6219] [utils] Remove AM/PM from unified_strdate patterns

---
 test/test_utils.py  | 1 +
 youtube_dl/utils.py | 6 +++---
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index e6887be9f..d0736f435 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -249,6 +249,7 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(
             unified_strdate('2/2/2015 6:47:40 PM', day_first=False),
             '20150202')
+        self.assertEqual(unified_strdate('Feb 14th 2016 5:45PM'), '20160214')
         self.assertEqual(unified_strdate('25-09-2014'), '20140925')
         self.assertEqual(unified_strdate('UNKNOWN DATE FORMAT'), None)
 
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 8ec70f93c..a3df90fad 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -905,9 +905,9 @@ def unified_strdate(date_str, day_first=True):
         '%d %b %Y',
         '%B %d %Y',
         '%b %d %Y',
-        '%b %dst %Y %I:%M%p',
-        '%b %dnd %Y %I:%M%p',
-        '%b %dth %Y %I:%M%p',
+        '%b %dst %Y %I:%M',
+        '%b %dnd %Y %I:%M',
+        '%b %dth %Y %I:%M',
         '%Y %m %d',
         '%Y-%m-%d',
         '%Y/%m/%d',

From d0459c530d3ce145fe89cb30eba813be6b8ed66a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 25 Feb 2016 00:54:41 +0600
Subject: [PATCH 4101/6219] [motherless] Update tests

---
 youtube_dl/extractor/motherless.py | 93 +++++++++++++++---------------
 1 file changed, 45 insertions(+), 48 deletions(-)

diff --git a/youtube_dl/extractor/motherless.py b/youtube_dl/extractor/motherless.py
index 30e686a4e..0b4787c1d 100644
--- a/youtube_dl/extractor/motherless.py
+++ b/youtube_dl/extractor/motherless.py
@@ -13,55 +13,52 @@ from ..utils import (
 
 class MotherlessIE(InfoExtractor):
     _VALID_URL = r'http://(?:www\.)?motherless\.com/(?:g/[a-z0-9_]+/)?(?P<id>[A-Z0-9]+)'
-    _TESTS = [
-        {
-            'url': 'http://motherless.com/AC3FFE1',
-            'md5': '310f62e325a9fafe64f68c0bccb6e75f',
-            'info_dict': {
-                'id': 'AC3FFE1',
-                'ext': 'mp4',
-                'title': 'Fucked in the ass while playing PS3',
-                'categories': ['Gaming', 'anal', 'reluctant', 'rough', 'Wife'],
-                'upload_date': '20100913',
-                'uploader_id': 'famouslyfuckedup',
-                'thumbnail': 're:http://.*\.jpg',
-                'age_limit': 18,
-            }
-        },
-        {
-            'url': 'http://motherless.com/532291B',
-            'md5': 'bc59a6b47d1f958e61fbd38a4d31b131',
-            'info_dict': {
-                'id': '532291B',
-                'ext': 'mp4',
-                'title': 'Amazing girl playing the omegle game, PERFECT!',
-                'categories': ['Amateur', 'webcam', 'omegle', 'pink', 'young', 'masturbate', 'teen', 'game', 'hairy'],
-                'upload_date': '20140622',
-                'uploader_id': 'Sulivana7x',
-                'thumbnail': 're:http://.*\.jpg',
-                'age_limit': 18,
-            }
-        },
-        {
-            'url': 'http://motherless.com/g/cosplay/633979F',
-            'md5': '0b2a43f447a49c3e649c93ad1fafa4a0',
-            'info_dict': {
-                'id': '633979F',
-                'ext': 'mp4',
-                'title': 'Turtlette',
-                'categories': ['superheroine heroine  superher'],
-                'upload_date': '20140827',
-                'uploader_id': 'shade0230',
-                'thumbnail': 're:http://.*\.jpg',
-                'age_limit': 18,
-            }
-        },
-        {
-            # no keywords
-            'url': 'http://motherless.com/8B4BBC1',
-            'only_matching': True,
+    _TESTS = [{
+        'url': 'http://motherless.com/AC3FFE1',
+        'md5': '310f62e325a9fafe64f68c0bccb6e75f',
+        'info_dict': {
+            'id': 'AC3FFE1',
+            'ext': 'mp4',
+            'title': 'Fucked in the ass while playing PS3',
+            'categories': ['Gaming', 'anal', 'reluctant', 'rough', 'Wife'],
+            'upload_date': '20100913',
+            'uploader_id': 'famouslyfuckedup',
+            'thumbnail': 're:http://.*\.jpg',
+            'age_limit': 18,
         }
-    ]
+    }, {
+        'url': 'http://motherless.com/532291B',
+        'md5': 'bc59a6b47d1f958e61fbd38a4d31b131',
+        'info_dict': {
+            'id': '532291B',
+            'ext': 'mp4',
+            'title': 'Amazing girl playing the omegle game, PERFECT!',
+            'categories': ['Amateur', 'webcam', 'omegle', 'pink', 'young', 'masturbate', 'teen',
+                           'game', 'hairy'],
+            'upload_date': '20140622',
+            'uploader_id': 'Sulivana7x',
+            'thumbnail': 're:http://.*\.jpg',
+            'age_limit': 18,
+        },
+        'skip': '404',
+    }, {
+        'url': 'http://motherless.com/g/cosplay/633979F',
+        'md5': '0b2a43f447a49c3e649c93ad1fafa4a0',
+        'info_dict': {
+            'id': '633979F',
+            'ext': 'mp4',
+            'title': 'Turtlette',
+            'categories': ['superheroine heroine  superher'],
+            'upload_date': '20140827',
+            'uploader_id': 'shade0230',
+            'thumbnail': 're:http://.*\.jpg',
+            'age_limit': 18,
+        }
+    }, {
+        # no keywords
+        'url': 'http://motherless.com/8B4BBC1',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From 9195ef745aba88a08e351af6405a87b5f355c026 Mon Sep 17 00:00:00 2001
From: mutantmonkey <mutantmonkey@mutantmonkey.in>
Date: Mon, 15 Feb 2016 17:07:13 -0800
Subject: [PATCH 4102/6219] [uStudio] Add new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/ustudio.py  | 58 ++++++++++++++++++++++++++++++++
 2 files changed, 59 insertions(+)
 create mode 100644 youtube_dl/extractor/ustudio.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 1ae606f1e..5386747f1 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -813,6 +813,7 @@ from .digiteka import DigitekaIE
 from .unistra import UnistraIE
 from .urort import UrortIE
 from .ustream import UstreamIE, UstreamChannelIE
+from .ustudio import UstudioIE
 from .varzesh3 import Varzesh3IE
 from .vbox7 import Vbox7IE
 from .veehd import VeeHDIE
diff --git a/youtube_dl/extractor/ustudio.py b/youtube_dl/extractor/ustudio.py
new file mode 100644
index 000000000..0f837ad73
--- /dev/null
+++ b/youtube_dl/extractor/ustudio.py
@@ -0,0 +1,58 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import int_or_none
+
+
+class UstudioIE(InfoExtractor):
+    IE_NAME = 'uStudio'
+    _VALID_URL = r'http://(?:www\.|v1\.)?ustudio.com/video/(?P<id>[\w\d]+)/.+'
+    _TESTS = [
+        {
+            'url': 'http://ustudio.com/video/Uxu2my9bgSph/san_francisco_golden_gate_bridge',
+            'md5': '58bbfca62125378742df01fc2abbdef6',
+            'info_dict': {
+                'id': 'Uxu2my9bgSph',
+                'ext': 'mp4',
+                'title': 'San Francisco: Golden Gate Bridge',
+                'thumbnail': 're:^https?://.*\.jpg$',
+                'description': 'md5:23925500697f2c6d4830e387ba51a9be',
+                'uploader': 'Tony Farley',
+            }
+        },
+    ]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        doc = self._download_xml(
+            'http://v1.ustudio.com/embed/{0}/ustudio/config.xml'.format(
+                video_id),
+            video_id,
+            note='Downloading video info',
+            errnote='Failed to download video info')
+
+        formats = [
+            {
+                'url': quality.attrib['url'],
+                'width': int_or_none(quality.attrib.get('width')),
+                'height': int_or_none(quality.attrib.get('height')),
+            } for quality in doc.findall('./qualities/quality/video')
+        ]
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': self._og_search_title(webpage),
+            'thumbnail': self._og_search_thumbnail(webpage),
+            'formats': formats,
+            'description': self._og_search_description(webpage),
+            'uploader': self._html_search_regex(
+                r'<a href=".*/user/.+">(.+)</a> on',
+                webpage,
+                'uploader',
+                fatal=False),
+        }

From 20108c6b9045f819580bf2d6ca687718bd7b0510 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 25 Feb 2016 21:30:19 +0600
Subject: [PATCH 4103/6219] [ustudio] Improve (Closes #8574)

---
 youtube_dl/extractor/ustudio.py | 91 ++++++++++++++++++---------------
 1 file changed, 50 insertions(+), 41 deletions(-)

diff --git a/youtube_dl/extractor/ustudio.py b/youtube_dl/extractor/ustudio.py
index 0f837ad73..cafc082b6 100644
--- a/youtube_dl/extractor/ustudio.py
+++ b/youtube_dl/extractor/ustudio.py
@@ -1,58 +1,67 @@
-# coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
-from ..utils import int_or_none
+from ..utils import (
+    int_or_none,
+    unified_strdate,
+)
 
 
 class UstudioIE(InfoExtractor):
-    IE_NAME = 'uStudio'
-    _VALID_URL = r'http://(?:www\.|v1\.)?ustudio.com/video/(?P<id>[\w\d]+)/.+'
-    _TESTS = [
-        {
-            'url': 'http://ustudio.com/video/Uxu2my9bgSph/san_francisco_golden_gate_bridge',
-            'md5': '58bbfca62125378742df01fc2abbdef6',
-            'info_dict': {
-                'id': 'Uxu2my9bgSph',
-                'ext': 'mp4',
-                'title': 'San Francisco: Golden Gate Bridge',
-                'thumbnail': 're:^https?://.*\.jpg$',
-                'description': 'md5:23925500697f2c6d4830e387ba51a9be',
-                'uploader': 'Tony Farley',
-            }
-        },
-    ]
+    _VALID_URL = r'https?://(?:(?:www|v1)\.)?ustudio\.com/video/(?P<id>[^/]+)/(?P<display_id>[^/?#&]+)'
+    _TEST = {
+        'url': 'http://ustudio.com/video/Uxu2my9bgSph/san_francisco_golden_gate_bridge',
+        'md5': '58bbfca62125378742df01fc2abbdef6',
+        'info_dict': {
+            'id': 'Uxu2my9bgSph',
+            'display_id': 'san_francisco_golden_gate_bridge',
+            'ext': 'mp4',
+            'title': 'San Francisco: Golden Gate Bridge',
+            'description': 'md5:23925500697f2c6d4830e387ba51a9be',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'upload_date': '20111107',
+            'uploader': 'Tony Farley',
+        }
+    }
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        display_id = mobj.group('display_id')
 
-        webpage = self._download_webpage(url, video_id)
+        config = self._download_xml(
+            'http://v1.ustudio.com/embed/%s/ustudio/config.xml' % video_id,
+            display_id)
 
-        doc = self._download_xml(
-            'http://v1.ustudio.com/embed/{0}/ustudio/config.xml'.format(
-                video_id),
-            video_id,
-            note='Downloading video info',
-            errnote='Failed to download video info')
+        def extract(kind):
+            return [{
+                'url': item.attrib['url'],
+                'width': int_or_none(item.get('width')),
+                'height': int_or_none(item.get('height')),
+            } for item in config.findall('./qualities/quality/%s' % kind) if item.get('url')]
 
-        formats = [
-            {
-                'url': quality.attrib['url'],
-                'width': int_or_none(quality.attrib.get('width')),
-                'height': int_or_none(quality.attrib.get('height')),
-            } for quality in doc.findall('./qualities/quality/video')
-        ]
+        formats = extract('video')
         self._sort_formats(formats)
 
+        webpage = self._download_webpage(url, display_id)
+
+        title = self._og_search_title(webpage)
+        upload_date = unified_strdate(self._search_regex(
+            r'(?s)Uploaded by\s*.+?\s*on\s*<span>([^<]+)</span>',
+            webpage, 'upload date', fatal=False))
+        uploader = self._search_regex(
+            r'Uploaded by\s*<a[^>]*>([^<]+)<',
+            webpage, 'uploader', fatal=False)
+
         return {
             'id': video_id,
-            'title': self._og_search_title(webpage),
-            'thumbnail': self._og_search_thumbnail(webpage),
-            'formats': formats,
+            'display_id': display_id,
+            'title': title,
             'description': self._og_search_description(webpage),
-            'uploader': self._html_search_regex(
-                r'<a href=".*/user/.+">(.+)</a> on',
-                webpage,
-                'uploader',
-                fatal=False),
+            'thumbnails': extract('image'),
+            'upload_date': upload_date,
+            'uploader': uploader,
+            'formats': formats,
         }

From e26cde092736c3356c1ab76c899a180fb3441d4b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 25 Feb 2016 21:46:43 +0600
Subject: [PATCH 4104/6219] [space] Remove extractor (Closes #8662)

Now uses ooyala embed
---
 youtube_dl/extractor/__init__.py |  1 -
 youtube_dl/extractor/space.py    | 38 --------------------------------
 2 files changed, 39 deletions(-)
 delete mode 100644 youtube_dl/extractor/space.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 5386747f1..77e2ae425 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -669,7 +669,6 @@ from .southpark import (
     SouthParkEsIE,
     SouthParkNlIE
 )
-from .space import SpaceIE
 from .spankbang import SpankBangIE
 from .spankwire import SpankwireIE
 from .spiegel import SpiegelIE, SpiegelArticleIE
diff --git a/youtube_dl/extractor/space.py b/youtube_dl/extractor/space.py
deleted file mode 100644
index ebb5d6ec0..000000000
--- a/youtube_dl/extractor/space.py
+++ /dev/null
@@ -1,38 +0,0 @@
-from __future__ import unicode_literals
-
-import re
-
-from .common import InfoExtractor
-from .brightcove import BrightcoveLegacyIE
-from ..utils import RegexNotFoundError, ExtractorError
-
-
-class SpaceIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:(?:www|m)\.)?space\.com/\d+-(?P<title>[^/\.\?]*?)-video\.html'
-    _TEST = {
-        'add_ie': ['BrightcoveLegacy'],
-        'url': 'http://www.space.com/23373-huge-martian-landforms-detail-revealed-by-european-probe-video.html',
-        'info_dict': {
-            'id': '2780937028001',
-            'ext': 'mp4',
-            'title': 'Huge Martian Landforms\' Detail Revealed By European Probe | Video',
-            'description': 'md5:db81cf7f3122f95ed234b631a6ea1e61',
-            'uploader': 'TechMedia Networks',
-        },
-    }
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        title = mobj.group('title')
-        webpage = self._download_webpage(url, title)
-        try:
-            # Some videos require the playerKey field, which isn't define in
-            # the BrightcoveExperience object
-            brightcove_url = self._og_search_video_url(webpage)
-        except RegexNotFoundError:
-            # Other videos works fine with the info from the object
-            brightcove_url = BrightcoveLegacyIE._extract_brightcove_url(webpage)
-        if brightcove_url is None:
-            raise ExtractorError(
-                'The webpage does not contain a video', expected=True)
-        return self.url_result(brightcove_url, BrightcoveLegacyIE.ie_key())

From e048d87fc9b5cfa19806649faf20fc1a7bdb82a6 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 24 Feb 2016 03:49:13 +0800
Subject: [PATCH 4105/6219] [kuwo] Fix a test

---
 youtube_dl/extractor/kuwo.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/kuwo.py b/youtube_dl/extractor/kuwo.py
index f641edef8..700e44b63 100644
--- a/youtube_dl/extractor/kuwo.py
+++ b/youtube_dl/extractor/kuwo.py
@@ -68,6 +68,7 @@ class KuwoIE(KuwoBaseIE):
             'id': '6446136',
             'ext': 'mp3',
             'title': '心',
+            'description': 'md5:b2ab6295d014005bfc607525bfc1e38a',
             'creator': 'IU',
             'upload_date': '20150518',
         },

From 81bdc8fdf6516b05bc3a26f82eacb1889f5e46d5 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 24 Feb 2016 22:08:40 +0800
Subject: [PATCH 4106/6219] [utils] Move base62 to utils

---
 youtube_dl/extractor/iqiyi.py | 15 ++-------------
 youtube_dl/utils.py           | 14 ++++++++++++++
 2 files changed, 16 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index c3e33009a..4f02b9f87 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -18,6 +18,7 @@ from ..compat import (
     compat_urllib_parse_urlparse,
 )
 from ..utils import (
+    base62,
     ExtractorError,
     ohdave_rsa_encrypt,
     remove_start,
@@ -126,21 +127,9 @@ class IqiyiSDK(object):
 
 
 class IqiyiSDKInterpreter(object):
-    BASE62_TABLE = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
-
     def __init__(self, sdk_code):
         self.sdk_code = sdk_code
 
-    @classmethod
-    def base62(cls, num):
-        if num == 0:
-            return '0'
-        ret = ''
-        while num:
-            ret = cls.BASE62_TABLE[num % 62] + ret
-            num = num // 62
-        return ret
-
     def decode_eval_codes(self):
         self.sdk_code = self.sdk_code[5:-3]
 
@@ -154,7 +143,7 @@ class IqiyiSDKInterpreter(object):
 
         while count:
             count -= 1
-            b62count = self.base62(count)
+            b62count = base62(count)
             symbol_table[b62count] = symbols[count] or b62count
 
         self.sdk_code = re.sub(
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index a3df90fad..d7a1586c0 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2619,3 +2619,17 @@ def ohdave_rsa_encrypt(data, exponent, modulus):
     payload = int(binascii.hexlify(data[::-1]), 16)
     encrypted = pow(payload, exponent, modulus)
     return '%x' % encrypted
+
+
+def base_n(num, n, table):
+    if num == 0:
+        return '0'
+    ret = ''
+    while num:
+        ret = table[num % n] + ret
+        num = num // n
+    return ret
+
+
+def base62(num):
+    return base_n(num, 62, '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ')

From d1e440a4a18522207a1a3e624bf801c8338f9146 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 26 Feb 2016 14:13:00 +0800
Subject: [PATCH 4107/6219] [jwplatform] Separate codes for for parsing
 jwplayer data

---
 youtube_dl/extractor/jwplatform.py | 59 ++++++++++++++++--------------
 1 file changed, 32 insertions(+), 27 deletions(-)

diff --git a/youtube_dl/extractor/jwplatform.py b/youtube_dl/extractor/jwplatform.py
index 8e90d5986..60a09044f 100644
--- a/youtube_dl/extractor/jwplatform.py
+++ b/youtube_dl/extractor/jwplatform.py
@@ -7,33 +7,9 @@ from .common import InfoExtractor
 from ..utils import int_or_none
 
 
-class JWPlatformIE(InfoExtractor):
-    _VALID_URL = r'(?:https?://content\.jwplatform\.com/(?:feeds|players|jw6)/|jwplatform:)(?P<id>[a-zA-Z0-9]{8})'
-    _TEST = {
-        'url': 'http://content.jwplatform.com/players/nPripu9l-ALJ3XQCI.js',
-        'md5': 'fa8899fa601eb7c83a64e9d568bdf325',
-        'info_dict': {
-            'id': 'nPripu9l',
-            'ext': 'mov',
-            'title': 'Big Buck Bunny Trailer',
-            'description': 'Big Buck Bunny is a short animated film by the Blender Institute. It is made using free and open source software.',
-            'upload_date': '20081127',
-            'timestamp': 1227796140,
-        }
-    }
-
-    @staticmethod
-    def _extract_url(webpage):
-        mobj = re.search(
-            r'<script[^>]+?src=["\'](?P<url>(?:https?:)?//content.jwplatform.com/players/[a-zA-Z0-9]{8})',
-            webpage)
-        if mobj:
-            return mobj.group('url')
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        json_data = self._download_json('http://content.jwplatform.com/feeds/%s.json' % video_id, video_id)
-        video_data = json_data['playlist'][0]
+class JWPlatformBaseIE(InfoExtractor):
+    def _parse_jwplayer_data(self, jwplayer_data, video_id):
+        video_data = jwplayer_data['playlist'][0]
         subtitles = {}
         for track in video_data['tracks']:
             if track['kind'] == 'captions':
@@ -68,3 +44,32 @@ class JWPlatformIE(InfoExtractor):
             'subtitles': subtitles,
             'formats': formats,
         }
+
+
+class JWPlatformIE(JWPlatformBaseIE):
+    _VALID_URL = r'(?:https?://content\.jwplatform\.com/(?:feeds|players|jw6)/|jwplatform:)(?P<id>[a-zA-Z0-9]{8})'
+    _TEST = {
+        'url': 'http://content.jwplatform.com/players/nPripu9l-ALJ3XQCI.js',
+        'md5': 'fa8899fa601eb7c83a64e9d568bdf325',
+        'info_dict': {
+            'id': 'nPripu9l',
+            'ext': 'mov',
+            'title': 'Big Buck Bunny Trailer',
+            'description': 'Big Buck Bunny is a short animated film by the Blender Institute. It is made using free and open source software.',
+            'upload_date': '20081127',
+            'timestamp': 1227796140,
+        }
+    }
+
+    @staticmethod
+    def _extract_url(webpage):
+        mobj = re.search(
+            r'<script[^>]+?src=["\'](?P<url>(?:https?:)?//content.jwplatform.com/players/[a-zA-Z0-9]{8})',
+            webpage)
+        if mobj:
+            return mobj.group('url')
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        json_data = self._download_json('http://content.jwplatform.com/feeds/%s.json' % video_id, video_id)
+        return self._parse_jwplayer_data(json_data, video_id)

From 481888294d19ef52075e531ce26588d97b3d16d2 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 26 Feb 2016 14:23:16 +0800
Subject: [PATCH 4108/6219] [utils] Add base36 for use in Vidzi

---
 youtube_dl/utils.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index d7a1586c0..be1f3b0d7 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2631,5 +2631,9 @@ def base_n(num, n, table):
     return ret
 
 
+def base36(num):
+    return base_n(num, 36, '0123456789abcdefghijklmnopqrstuvwxyz')
+
+
 def base62(num):
     return base_n(num, 62, '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ')

From 8f4a2124a914207912bf9fc37e593210e8dd423b Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 26 Feb 2016 14:26:07 +0800
Subject: [PATCH 4109/6219] [vidzi] Fix extraction

---
 youtube_dl/extractor/jwplatform.py |  6 ++---
 youtube_dl/extractor/vidzi.py      | 40 +++++++++++++++++++++---------
 2 files changed, 31 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/jwplatform.py b/youtube_dl/extractor/jwplatform.py
index 60a09044f..6770685d7 100644
--- a/youtube_dl/extractor/jwplatform.py
+++ b/youtube_dl/extractor/jwplatform.py
@@ -8,7 +8,7 @@ from ..utils import int_or_none
 
 
 class JWPlatformBaseIE(InfoExtractor):
-    def _parse_jwplayer_data(self, jwplayer_data, video_id):
+    def _parse_jwplayer_data(self, jwplayer_data, video_id, require_title=True):
         video_data = jwplayer_data['playlist'][0]
         subtitles = {}
         for track in video_data['tracks']:
@@ -19,7 +19,7 @@ class JWPlatformBaseIE(InfoExtractor):
         for source in video_data['sources']:
             source_url = self._proto_relative_url(source['file'])
             source_type = source.get('type') or ''
-            if source_type == 'application/vnd.apple.mpegurl':
+            if source_type in ('application/vnd.apple.mpegurl', 'hls'):
                 formats.extend(self._extract_m3u8_formats(
                     source_url, video_id, 'mp4', 'm3u8_native', fatal=False))
             elif source_type.startswith('audio'):
@@ -37,7 +37,7 @@ class JWPlatformBaseIE(InfoExtractor):
 
         return {
             'id': video_id,
-            'title': video_data['title'],
+            'title': video_data['title'] if require_title else video_data.get('title'),
             'description': video_data.get('description'),
             'thumbnail': self._proto_relative_url(video_data.get('image')),
             'timestamp': int_or_none(video_data.get('pubdate')),
diff --git a/youtube_dl/extractor/vidzi.py b/youtube_dl/extractor/vidzi.py
index 7c6e98026..4ec07db3a 100644
--- a/youtube_dl/extractor/vidzi.py
+++ b/youtube_dl/extractor/vidzi.py
@@ -1,11 +1,16 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-from .common import InfoExtractor
-from ..utils import smuggle_url
+import re
+
+from .jwplatform import JWPlatformBaseIE
+from ..utils import (
+    base36,
+    js_to_json,
+)
 
 
-class VidziIE(InfoExtractor):
+class VidziIE(JWPlatformBaseIE):
     _VALID_URL = r'https?://(?:www\.)?vidzi\.tv/(?P<id>\w+)'
     _TEST = {
         'url': 'http://vidzi.tv/cghql9yq6emu.html',
@@ -14,7 +19,6 @@ class VidziIE(InfoExtractor):
             'id': 'cghql9yq6emu',
             'ext': 'mp4',
             'title': 'youtube-dl test video  1\\\\2\'3/4<5\\\\6ä7↭',
-            'uploader': 'vidzi.tv',
         },
         'params': {
             # m3u8 download
@@ -29,11 +33,23 @@ class VidziIE(InfoExtractor):
         title = self._html_search_regex(
             r'(?s)<h2 class="video-title">(.*?)</h2>', webpage, 'title')
 
-        # Vidzi now uses jwplayer, which can be handled by GenericIE
-        return {
-            '_type': 'url_transparent',
-            'id': video_id,
-            'title': title,
-            'url': smuggle_url(url, {'to_generic': True}),
-            'ie_key': 'Generic',
-        }
+        mobj = re.search(r"}\('(.+)',36,(\d+),'([^']+)'\.split\('\|'\)", webpage)
+        code, count, symbols = mobj.groups()
+
+        count = int(count)
+        symbols = symbols.split('|')
+
+        while count:
+            count -= 1
+            if symbols[count]:
+                code = re.sub(r'\b%s\b' % base36(count), symbols[count], code)
+
+        code = code.replace('\\\'', '\'')
+        jwplayer_data = self._parse_json(
+            self._search_regex(r'setup\(([^)]+)\)', code, 'jwplayer data'),
+            video_id, transform_source=js_to_json)
+
+        info_dict = self._parse_jwplayer_data(jwplayer_data, video_id, require_title=False)
+        info_dict['title'] = title
+
+        return info_dict

From 59f898b7a72284efb994a8c6baee7771046226dd Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 26 Feb 2016 14:37:20 +0800
Subject: [PATCH 4110/6219] [utils] Merge base_n functions

---
 youtube_dl/extractor/iqiyi.py |  4 ++--
 youtube_dl/extractor/vidzi.py |  4 ++--
 youtube_dl/utils.py           | 16 +++++++---------
 3 files changed, 11 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index 4f02b9f87..76ecd55a4 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -18,7 +18,7 @@ from ..compat import (
     compat_urllib_parse_urlparse,
 )
 from ..utils import (
-    base62,
+    base_n,
     ExtractorError,
     ohdave_rsa_encrypt,
     remove_start,
@@ -143,7 +143,7 @@ class IqiyiSDKInterpreter(object):
 
         while count:
             count -= 1
-            b62count = base62(count)
+            b62count = base_n(count, 62)
             symbol_table[b62count] = symbols[count] or b62count
 
         self.sdk_code = re.sub(
diff --git a/youtube_dl/extractor/vidzi.py b/youtube_dl/extractor/vidzi.py
index 4ec07db3a..d671e27de 100644
--- a/youtube_dl/extractor/vidzi.py
+++ b/youtube_dl/extractor/vidzi.py
@@ -5,7 +5,7 @@ import re
 
 from .jwplatform import JWPlatformBaseIE
 from ..utils import (
-    base36,
+    base_n,
     js_to_json,
 )
 
@@ -42,7 +42,7 @@ class VidziIE(JWPlatformBaseIE):
         while count:
             count -= 1
             if symbols[count]:
-                code = re.sub(r'\b%s\b' % base36(count), symbols[count], code)
+                code = re.sub(r'\b%s\b' % base_n(count, 36), symbols[count], code)
 
         code = code.replace('\\\'', '\'')
         jwplayer_data = self._parse_json(
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index be1f3b0d7..900e07a8e 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2621,19 +2621,17 @@ def ohdave_rsa_encrypt(data, exponent, modulus):
     return '%x' % encrypted
 
 
-def base_n(num, n, table):
+def base_n(num, n, table=None):
     if num == 0:
         return '0'
+
+    FULL_TABLE = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
+    assert n <= len(FULL_TABLE)
+    if not table:
+        table = FULL_TABLE[:n]
+
     ret = ''
     while num:
         ret = table[num % n] + ret
         num = num // n
     return ret
-
-
-def base36(num):
-    return base_n(num, 36, '0123456789abcdefghijklmnopqrstuvwxyz')
-
-
-def base62(num):
-    return base_n(num, 62, '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ')

From f52354a88927eb66bcb1f603d2d91162b5bd2b5f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 26 Feb 2016 14:58:29 +0800
Subject: [PATCH 4111/6219] [utils] Move codes for handling eval() from
 iqiyi.py

---
 youtube_dl/extractor/iqiyi.py | 24 ++----------------------
 youtube_dl/utils.py           | 20 ++++++++++++++++++++
 2 files changed, 22 insertions(+), 22 deletions(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index 76ecd55a4..2b3952210 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -18,7 +18,7 @@ from ..compat import (
     compat_urllib_parse_urlparse,
 )
 from ..utils import (
-    base_n,
+    decode_packed_codes,
     ExtractorError,
     ohdave_rsa_encrypt,
     remove_start,
@@ -130,28 +130,8 @@ class IqiyiSDKInterpreter(object):
     def __init__(self, sdk_code):
         self.sdk_code = sdk_code
 
-    def decode_eval_codes(self):
-        self.sdk_code = self.sdk_code[5:-3]
-
-        mobj = re.search(
-            r"'([^']+)',62,(\d+),'([^']+)'\.split\('\|'\),[^,]+,{}",
-            self.sdk_code)
-        obfucasted_code, count, symbols = mobj.groups()
-        count = int(count)
-        symbols = symbols.split('|')
-        symbol_table = {}
-
-        while count:
-            count -= 1
-            b62count = base_n(count, 62)
-            symbol_table[b62count] = symbols[count] or b62count
-
-        self.sdk_code = re.sub(
-            r'\b(\w+)\b', lambda mobj: symbol_table[mobj.group(0)],
-            obfucasted_code)
-
     def run(self, target, ip, timestamp):
-        self.decode_eval_codes()
+        self.sdk_code = decode_packed_codes(self.sdk_code)
 
         functions = re.findall(r'input=([a-zA-Z0-9]+)\(input', self.sdk_code)
 
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 900e07a8e..fc7e2fb7f 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2635,3 +2635,23 @@ def base_n(num, n, table=None):
         ret = table[num % n] + ret
         num = num // n
     return ret
+
+
+def decode_packed_codes(code):
+    mobj = re.search(
+        r"'([^']+)',(\d+),(\d+),'([^']+)'\.split\('\|'\),[^,]+,{}",
+        code)
+    obfucasted_code, base, count, symbols = mobj.groups()
+    base = int(base)
+    count = int(count)
+    symbols = symbols.split('|')
+    symbol_table = {}
+
+    while count:
+        count -= 1
+        base_n_count = base_n(count, base)
+        symbol_table[base_n_count] = symbols[count] or base_n_count
+
+    return re.sub(
+        r'\b(\w+)\b', lambda mobj: symbol_table[mobj.group(0)],
+        obfucasted_code)

From e4fc8d2ebedfc6ea7f66ff9146940c73802a2edf Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 26 Feb 2016 15:00:48 +0800
Subject: [PATCH 4112/6219] [videomega] Fix extraction (closes #7606)

---
 youtube_dl/extractor/videomega.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/videomega.py b/youtube_dl/extractor/videomega.py
index 5e2e7cbac..4f0dcd18c 100644
--- a/youtube_dl/extractor/videomega.py
+++ b/youtube_dl/extractor/videomega.py
@@ -4,11 +4,13 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import sanitized_Request
+from ..utils import (
+    decode_packed_codes,
+    sanitized_Request,
+)
 
 
 class VideoMegaIE(InfoExtractor):
-    _WORKING = False
     _VALID_URL = r'(?:videomega:|https?://(?:www\.)?videomega\.tv/(?:(?:view|iframe|cdn)\.php)?\?ref=)(?P<id>[A-Za-z0-9]+)'
     _TESTS = [{
         'url': 'http://videomega.tv/cdn.php?ref=AOSQBJYKIDDIKYJBQSOA',
@@ -42,8 +44,10 @@ class VideoMegaIE(InfoExtractor):
             r'(?:^[Vv]ideo[Mm]ega\.tv\s-\s*|\s*-\svideomega\.tv$)', '', title)
         thumbnail = self._search_regex(
             r'<video[^>]+?poster="([^"]+)"', webpage, 'thumbnail', fatal=False)
+
+        real_codes = decode_packed_codes(webpage)
         video_url = self._search_regex(
-            r'<source[^>]+?src="([^"]+)"', webpage, 'video URL')
+            r'"src"\s*,\s*"([^"]+)"', real_codes, 'video URL')
 
         return {
             'id': video_id,

From 680079be39563a2ff810602413db5245d6cbf148 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 26 Feb 2016 15:13:03 +0800
Subject: [PATCH 4113/6219] [utils] Relaxing regex in decode_packed_codes for
 vidzi

---
 youtube_dl/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index fc7e2fb7f..756ad4fd1 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2639,7 +2639,7 @@ def base_n(num, n, table=None):
 
 def decode_packed_codes(code):
     mobj = re.search(
-        r"'([^']+)',(\d+),(\d+),'([^']+)'\.split\('\|'\),[^,]+,{}",
+        r"}\('(.+)',(\d+),(\d+),'([^']+)'\.split\('\|'\)",
         code)
     obfucasted_code, base, count, symbols = mobj.groups()
     base = int(base)

From efbd6fb8bb86c07e6f924a7ec2c4bd486face3a4 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 26 Feb 2016 15:13:35 +0800
Subject: [PATCH 4114/6219] [vidzi] Use decode_packed_codes

Javascript codes found on Vidzi are slightly different from those found
in VideoMega and iQiyi. Nevertheless, the difference has no effects on
the final result.
---
 youtube_dl/extractor/vidzi.py | 17 ++---------------
 1 file changed, 2 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/vidzi.py b/youtube_dl/extractor/vidzi.py
index d671e27de..3c78fb3d5 100644
--- a/youtube_dl/extractor/vidzi.py
+++ b/youtube_dl/extractor/vidzi.py
@@ -1,11 +1,9 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .jwplatform import JWPlatformBaseIE
 from ..utils import (
-    base_n,
+    decode_packed_codes,
     js_to_json,
 )
 
@@ -33,18 +31,7 @@ class VidziIE(JWPlatformBaseIE):
         title = self._html_search_regex(
             r'(?s)<h2 class="video-title">(.*?)</h2>', webpage, 'title')
 
-        mobj = re.search(r"}\('(.+)',36,(\d+),'([^']+)'\.split\('\|'\)", webpage)
-        code, count, symbols = mobj.groups()
-
-        count = int(count)
-        symbols = symbols.split('|')
-
-        while count:
-            count -= 1
-            if symbols[count]:
-                code = re.sub(r'\b%s\b' % base_n(count, 36), symbols[count], code)
-
-        code = code.replace('\\\'', '\'')
+        code = decode_packed_codes(webpage).replace('\\\'', '\'')
         jwplayer_data = self._parse_json(
             self._search_regex(r'setup\(([^)]+)\)', code, 'jwplayer data'),
             video_id, transform_source=js_to_json)

From b78b292f0c51323edaf3e18ae4f45927a55e9198 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 26 Feb 2016 22:21:47 +0600
Subject: [PATCH 4115/6219] [youtube] Add alternative automatic captions
 extraction approach (Closes #8667)

---
 youtube_dl/extractor/youtube.py | 83 ++++++++++++++++++++++-----------
 1 file changed, 55 insertions(+), 28 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index e24dd3e5b..ec90c2111 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -975,40 +975,67 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             return {}
         try:
             args = player_config['args']
-            caption_url = args['ttsurl']
-            if not caption_url:
-                self._downloader.report_warning(err_msg)
-                return {}
-            timestamp = args['timestamp']
-            # We get the available subtitles
-            list_params = compat_urllib_parse.urlencode({
-                'type': 'list',
-                'tlangs': 1,
-                'asrs': 1,
-            })
-            list_url = caption_url + '&' + list_params
-            caption_list = self._download_xml(list_url, video_id)
-            original_lang_node = caption_list.find('track')
-            if original_lang_node is None:
-                self._downloader.report_warning('Video doesn\'t have automatic captions')
-                return {}
-            original_lang = original_lang_node.attrib['lang_code']
-            caption_kind = original_lang_node.attrib.get('kind', '')
+            caption_url = args.get('ttsurl')
+            if caption_url:
+                timestamp = args['timestamp']
+                # We get the available subtitles
+                list_params = compat_urllib_parse.urlencode({
+                    'type': 'list',
+                    'tlangs': 1,
+                    'asrs': 1,
+                })
+                list_url = caption_url + '&' + list_params
+                caption_list = self._download_xml(list_url, video_id)
+                original_lang_node = caption_list.find('track')
+                if original_lang_node is None:
+                    self._downloader.report_warning('Video doesn\'t have automatic captions')
+                    return {}
+                original_lang = original_lang_node.attrib['lang_code']
+                caption_kind = original_lang_node.attrib.get('kind', '')
+
+                sub_lang_list = {}
+                for lang_node in caption_list.findall('target'):
+                    sub_lang = lang_node.attrib['lang_code']
+                    sub_formats = []
+                    for ext in self._SUBTITLE_FORMATS:
+                        params = compat_urllib_parse.urlencode({
+                            'lang': original_lang,
+                            'tlang': sub_lang,
+                            'fmt': ext,
+                            'ts': timestamp,
+                            'kind': caption_kind,
+                        })
+                        sub_formats.append({
+                            'url': caption_url + '&' + params,
+                            'ext': ext,
+                        })
+                    sub_lang_list[sub_lang] = sub_formats
+                return sub_lang_list
+
+            # Some videos don't provide ttsurl but rather caption_tracks and
+            # caption_translation_languages (e.g. 20LmZk1hakA)
+            caption_tracks = args['caption_tracks']
+            caption_translation_languages = args['caption_translation_languages']
+            caption_url = compat_parse_qs(caption_tracks.split(',')[0])['u'][0]
+            parsed_caption_url = compat_urlparse.urlparse(caption_url)
+            caption_qs = compat_parse_qs(parsed_caption_url.query)
 
             sub_lang_list = {}
-            for lang_node in caption_list.findall('target'):
-                sub_lang = lang_node.attrib['lang_code']
+            for lang in caption_translation_languages.split(','):
+                lang_qs = compat_parse_qs(compat_urllib_parse_unquote_plus(lang))
+                sub_lang = lang_qs.get('lc', [None])[0]
+                if not sub_lang:
+                    continue
                 sub_formats = []
                 for ext in self._SUBTITLE_FORMATS:
-                    params = compat_urllib_parse.urlencode({
-                        'lang': original_lang,
-                        'tlang': sub_lang,
-                        'fmt': ext,
-                        'ts': timestamp,
-                        'kind': caption_kind,
+                    caption_qs.update({
+                        'tlang': [sub_lang],
+                        'fmt': [ext],
                     })
+                    sub_url = compat_urlparse.urlunparse(parsed_caption_url._replace(
+                        query=compat_urllib_parse.urlencode(caption_qs, True)))
                     sub_formats.append({
-                        'url': caption_url + '&' + params,
+                        'url': sub_url,
                         'ext': ext,
                     })
                 sub_lang_list[sub_lang] = sub_formats

From 2ebd2eac880e21e0fe9751e6cef28ac009f69d79 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 27 Feb 2016 00:57:35 +0800
Subject: [PATCH 4116/6219] [letv] Speedup M3U8 decryption

---
 youtube_dl/extractor/letv.py | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/letv.py b/youtube_dl/extractor/letv.py
index 9665ece89..9fd494c29 100644
--- a/youtube_dl/extractor/letv.py
+++ b/youtube_dl/extractor/letv.py
@@ -94,17 +94,16 @@ class LetvIE(InfoExtractor):
             return encrypted_data
         encrypted_data = encrypted_data[5:]
 
-        _loc4_ = bytearray()
-        while encrypted_data:
-            b = compat_ord(encrypted_data[0])
-            _loc4_.extend([b // 16, b & 0x0f])
-            encrypted_data = encrypted_data[1:]
+        _loc4_ = bytearray(2 * len(encrypted_data))
+        for idx, val in enumerate(encrypted_data):
+            b = compat_ord(val)
+            _loc4_[2 * idx] = b // 16
+            _loc4_[2 * idx + 1] = b % 16
         idx = len(_loc4_) - 11
         _loc4_ = _loc4_[idx:] + _loc4_[:idx]
-        _loc7_ = bytearray()
-        while _loc4_:
-            _loc7_.append(_loc4_[0] * 16 + _loc4_[1])
-            _loc4_ = _loc4_[2:]
+        _loc7_ = bytearray(len(encrypted_data))
+        for i in range(len(encrypted_data)):
+            _loc7_[i] = _loc4_[2 * i] * 16 + _loc4_[2 * i + 1]
 
         return bytes(_loc7_)
 

From 4435c6e98eb44b10c306b53c85c32458fbfeac88 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 27 Feb 2016 02:54:43 +0800
Subject: [PATCH 4117/6219] [bokecc] Add new extractor (#2336)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/bokecc.py   | 60 ++++++++++++++++++++++++++++++++
 2 files changed, 61 insertions(+)
 create mode 100644 youtube_dl/extractor/bokecc.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 77e2ae425..c30679736 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -74,6 +74,7 @@ from .bleacherreport import (
 )
 from .blinkx import BlinkxIE
 from .bloomberg import BloombergIE
+from .bokecc import BokeCCIE
 from .bpb import BpbIE
 from .br import BRIE
 from .breakcom import BreakIE
diff --git a/youtube_dl/extractor/bokecc.py b/youtube_dl/extractor/bokecc.py
new file mode 100644
index 000000000..122a1cbb6
--- /dev/null
+++ b/youtube_dl/extractor/bokecc.py
@@ -0,0 +1,60 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..compat import compat_parse_qs
+from ..utils import ExtractorError
+
+
+class BokeCCBaseIE(InfoExtractor):
+    def _extract_bokecc_formats(self, webpage, video_id, format_id=None):
+        player_params_str = self._html_search_regex(
+            r'<(?:script|embed)[^>]+src="http://p\.bokecc\.com/player\?([^"]+)',
+            webpage, 'player params')
+
+        player_params = compat_parse_qs(player_params_str)
+
+        info_xml = self._download_xml(
+            'http://p.bokecc.com/servlet/playinfo?uid=%s&vid=%s&m=1' % (
+                player_params['siteid'][0], player_params['vid'][0]), video_id)
+
+        formats = [{
+            'format_id': format_id,
+            'url': quality.find('./copy').attrib['playurl'],
+            'preference': int(quality.attrib['value']),
+        } for quality in info_xml.findall('./video/quality')]
+
+        self._sort_formats(formats)
+
+        return formats
+
+
+class BokeCCIE(BokeCCBaseIE):
+    _IE_DESC = 'CC视频'
+    _VALID_URL = r'http://union\.bokecc\.com/playvideo\.bo\?(?P<query>.*)'
+
+    _TESTS = [{
+        'url': 'http://union.bokecc.com/playvideo.bo?vid=E44D40C15E65EA30&uid=CD0C5D3C8614B28B',
+        'info_dict': {
+            'id': 'CD0C5D3C8614B28B_E44D40C15E65EA30',
+            'ext': 'flv',
+            'title': 'BokeCC Video',
+        },
+    }]
+
+    def _real_extract(self, url):
+        qs = compat_parse_qs(re.match(self._VALID_URL, url).group('query'))
+        if not qs.get('vid') or not qs.get('uid'):
+            raise ExtractorError('Invalid URL', expected=True)
+
+        video_id = '%s_%s' % (qs['uid'][0], qs['vid'][0])
+
+        webpage = self._download_webpage(url, video_id)
+
+        return {
+            'id': video_id,
+            'title': 'BokeCC Video',  # no title provided in the webpage
+            'formats': self._extract_bokecc_formats(webpage, video_id),
+        }

From 5633b4d39d178402c6d89146c8c9c34e3bf58619 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 27 Feb 2016 02:55:11 +0800
Subject: [PATCH 4118/6219] [infoq] Use BokeCC extractor function

---
 youtube_dl/extractor/infoq.py | 31 ++++---------------------------
 1 file changed, 4 insertions(+), 27 deletions(-)

diff --git a/youtube_dl/extractor/infoq.py b/youtube_dl/extractor/infoq.py
index 016af2084..cca0b8a93 100644
--- a/youtube_dl/extractor/infoq.py
+++ b/youtube_dl/extractor/infoq.py
@@ -4,15 +4,12 @@ from __future__ import unicode_literals
 
 import base64
 
-from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse_unquote,
-    compat_parse_qs,
-)
+from ..compat import compat_urllib_parse_unquote
 from ..utils import determine_ext
+from .bokecc import BokeCCBaseIE
 
 
-class InfoQIE(InfoExtractor):
+class InfoQIE(BokeCCBaseIE):
     _VALID_URL = r'https?://(?:www\.)?infoq\.com/(?:[^/]+/)+(?P<id>[^/]+)'
 
     _TESTS = [{
@@ -38,26 +35,6 @@ class InfoQIE(InfoExtractor):
         },
     }]
 
-    def _extract_bokecc_videos(self, webpage, video_id):
-        # TODO: bokecc.com is a Chinese video cloud platform
-        # It should have an independent extractor but I don't have other
-        # examples using bokecc
-        player_params_str = self._html_search_regex(
-            r'<script[^>]+src="http://p\.bokecc\.com/player\?([^"]+)',
-            webpage, 'player params', default=None)
-
-        player_params = compat_parse_qs(player_params_str)
-
-        info_xml = self._download_xml(
-            'http://p.bokecc.com/servlet/playinfo?uid=%s&vid=%s&m=1' % (
-                player_params['siteid'][0], player_params['vid'][0]), video_id)
-
-        return [{
-            'format_id': 'bokecc',
-            'url': quality.find('./copy').attrib['playurl'],
-            'preference': int(quality.attrib['value']),
-        } for quality in info_xml.findall('./video/quality')]
-
     def _extract_rtmp_videos(self, webpage):
         # The server URL is hardcoded
         video_url = 'rtmpe://video.infoq.com/cfx/st/'
@@ -101,7 +78,7 @@ class InfoQIE(InfoExtractor):
 
         if '/cn/' in url:
             # for China videos, HTTP video URL exists but always fails with 403
-            formats = self._extract_bokecc_videos(webpage, video_id)
+            formats = self._extract_bokecc_formats(webpage, video_id)
         else:
             formats = self._extract_rtmp_videos(webpage) + self._extract_http_videos(webpage)
 

From 5eb6bdced4765cdeb70411c6aa93ecb4163a9ffe Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 27 Feb 2016 03:19:50 +0800
Subject: [PATCH 4119/6219] [utils] Multiple changes to base_n()

1. Renamed to encode_base_n()
2. Allow tables longer than 62 characters
3. Raise ValueError instead of AssertionError for invalid input data
4. Return the first character in the table instead of '0' for number 0
5. Add tests
---
 test/test_utils.py  | 12 ++++++++++++
 youtube_dl/utils.py | 14 ++++++++------
 2 files changed, 20 insertions(+), 6 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index d0736f435..97587ad2f 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -18,6 +18,7 @@ import xml.etree.ElementTree
 from youtube_dl.utils import (
     age_restricted,
     args_to_str,
+    encode_base_n,
     clean_html,
     DateRange,
     detect_exe_version,
@@ -802,5 +803,16 @@ The first line
             ohdave_rsa_encrypt(b'aa111222', e, N),
             '726664bd9a23fd0c70f9f1b84aab5e3905ce1e45a584e9cbcf9bcc7510338fc1986d6c599ff990d923aa43c51c0d9013cd572e13bc58f4ae48f2ed8c0b0ba881')
 
+    def test_encode_base_n(self):
+        self.assertEqual(encode_base_n(0, 30), '0')
+        self.assertEqual(encode_base_n(80, 30), '2k')
+
+        custom_table = '9876543210ZYXWVUTSRQPONMLKJIHGFEDCBA'
+        self.assertEqual(encode_base_n(0, 30, custom_table), '9')
+        self.assertEqual(encode_base_n(80, 30, custom_table), '7P')
+
+        self.assertRaises(ValueError, encode_base_n, 0, 70)
+        self.assertRaises(ValueError, encode_base_n, 0, 60, custom_table)
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 756ad4fd1..606977c58 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2621,15 +2621,17 @@ def ohdave_rsa_encrypt(data, exponent, modulus):
     return '%x' % encrypted
 
 
-def base_n(num, n, table=None):
-    if num == 0:
-        return '0'
-
+def encode_base_n(num, n, table=None):
     FULL_TABLE = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
-    assert n <= len(FULL_TABLE)
     if not table:
         table = FULL_TABLE[:n]
 
+    if n > len(table):
+        raise ValueError('base %d exceeds table length %d' % (n, len(table)))
+
+    if num == 0:
+        return table[0]
+
     ret = ''
     while num:
         ret = table[num % n] + ret
@@ -2649,7 +2651,7 @@ def decode_packed_codes(code):
 
     while count:
         count -= 1
-        base_n_count = base_n(count, base)
+        base_n_count = encode_base_n(count, base)
         symbol_table[base_n_count] = symbols[count] or base_n_count
 
     return re.sub(

From fbb6edd298a34cf63a5fd0bd900ca2337038810c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Feb 2016 06:48:13 +0600
Subject: [PATCH 4120/6219] [extractor/common] Properly extract audio only
 formats in master m3u8 playlists

---
 youtube_dl/extractor/common.py | 26 ++++++++++++++++++--------
 1 file changed, 18 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 14f575635..51351fb57 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1084,19 +1084,29 @@ class InfoExtractor(object):
                     'protocol': entry_protocol,
                     'preference': preference,
                 }
-                codecs = last_info.get('CODECS')
-                if codecs:
-                    # TODO: looks like video codec is not always necessarily goes first
-                    va_codecs = codecs.split(',')
-                    if va_codecs[0]:
-                        f['vcodec'] = va_codecs[0]
-                    if len(va_codecs) > 1 and va_codecs[1]:
-                        f['acodec'] = va_codecs[1]
                 resolution = last_info.get('RESOLUTION')
                 if resolution:
                     width_str, height_str = resolution.split('x')
                     f['width'] = int(width_str)
                     f['height'] = int(height_str)
+                codecs = last_info.get('CODECS')
+                if codecs:
+                    vcodec, acodec = [None] * 2
+                    va_codecs = codecs.split(',')
+                    if len(va_codecs) == 1:
+                        # Audio only entries usually come with single codec and
+                        # no resolution. For more robustness we also check it to
+                        # be mp4 audio.
+                        if not resolution and va_codecs[0].startswith('mp4a'):
+                            vcodec, acodec = 'none', va_codecs[0]
+                        else:
+                            vcodec = va_codecs[0]
+                    else:
+                        vcodec, acodec = va_codecs[:2]
+                    f.update({
+                        'acodec': acodec,
+                        'vcodec': vcodec,
+                    })
                 if last_media is not None:
                     f['m3u8_media'] = last_media
                     last_media = None

From 9cdffeeb3fbfa28407a8a519a58b3714b7cec874 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Feb 2016 07:01:11 +0600
Subject: [PATCH 4121/6219] [extractor/common] Clarify rationale on media
 playlist detection

---
 youtube_dl/extractor/common.py | 20 +++++++++++++++-----
 1 file changed, 15 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 51351fb57..3f16b1b9e 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1033,11 +1033,21 @@ class InfoExtractor(object):
             return []
         m3u8_doc, urlh = res
         m3u8_url = urlh.geturl()
-        # A Media Playlist Tag MUST NOT appear in a Master Playlist
-        # https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.3
-        # The EXT-X-TARGETDURATION tag is REQUIRED for every M3U8 Media Playlists
-        # https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.3.1
-        if '#EXT-X-TARGETDURATION' in m3u8_doc:
+
+        # We should try extracting formats only from master playlists [1], i.e.
+        # playlists that describe available qualities. On the other hand media
+        # playlists [2] should be returned as is since they contain just the media
+        # without qualities renditions.
+        # Fortunately, master playlist can be easily distinguished from media
+        # playlist based on particular tags availability. As of [1, 2] master
+        # playlist tags MUST NOT appear in a media playist and vice versa.
+        # As of [3] #EXT-X-TARGETDURATION tag is REQUIRED for every media playlist
+        # and MUST NOT appear in master playlist thus we can clearly detect media
+        # playlist with this criterion.
+        # 1. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.4
+        # 2. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.3
+        # 3. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.3.1
+        if '#EXT-X-TARGETDURATION' in m3u8_doc:  # media playlist, return as is
             return [{
                 'url': m3u8_url,
                 'format_id': m3u8_id,

From f7f2e53a0a25e8c99f6f1a50cb3843d8e6f5be6f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Feb 2016 15:51:25 +0600
Subject: [PATCH 4122/6219] [imdb] Recognize 1080p formats (Closes #8677)

---
 youtube_dl/extractor/imdb.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/imdb.py b/youtube_dl/extractor/imdb.py
index 02e1e428e..b61b2dc4e 100644
--- a/youtube_dl/extractor/imdb.py
+++ b/youtube_dl/extractor/imdb.py
@@ -42,7 +42,7 @@ class ImdbIE(InfoExtractor):
             for f_url, f_name in extra_formats]
         format_pages.append(player_page)
 
-        quality = qualities(['SD', '480p', '720p'])
+        quality = qualities(('SD', '480p', '720p', '1080p'))
         formats = []
         for format_page in format_pages:
             json_data = self._search_regex(

From d6e9c2706fd803b5963fba47b168b6bccc063667 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Feb 2016 16:58:11 +0600
Subject: [PATCH 4123/6219] [tnaflixnetwork:embed] Add extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/tnaflix.py  | 26 +++++++++++++++++++++++++-
 2 files changed, 26 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index c30679736..5817140c0 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -737,6 +737,7 @@ from .tmz import (
     TMZArticleIE,
 )
 from .tnaflix import (
+    TNAFlixNetworkEmbedIE,
     TNAFlixIE,
     EMPFlixIE,
     MovieFapIE,
diff --git a/youtube_dl/extractor/tnaflix.py b/youtube_dl/extractor/tnaflix.py
index 49516abca..547d83d09 100644
--- a/youtube_dl/extractor/tnaflix.py
+++ b/youtube_dl/extractor/tnaflix.py
@@ -71,7 +71,7 @@ class TNAFlixNetworkBaseIE(InfoExtractor):
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
-        display_id = mobj.group('display_id')
+        display_id = mobj.group('display_id') if 'display_id' in mobj.groupdict() else video_id
 
         webpage = self._download_webpage(url, display_id)
 
@@ -152,6 +152,30 @@ class TNAFlixNetworkBaseIE(InfoExtractor):
         }
 
 
+class TNAFlixNetworkEmbedIE(TNAFlixNetworkBaseIE):
+    _VALID_URL = r'https?://player\.(?:tna|emp)flix\.com/video/(?P<id>\d+)'
+
+    _TITLE_REGEX = r'<title>([^<]+)</title>'
+
+    _TESTS = [{
+        'url': 'https://player.tnaflix.com/video/6538',
+        'info_dict': {
+            'id': '6538',
+            'display_id': '6538',
+            'ext': 'mp4',
+            'title': 'Educational xxx video',
+            'thumbnail': 're:https?://.*\.jpg$',
+            'age_limit': 18,
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }, {
+        'url': 'https://player.empflix.com/video/33051',
+        'only_matching': True,
+    }]
+
+
 class TNAFlixIE(TNAFlixNetworkBaseIE):
     _VALID_URL = r'https?://(?:www\.)?tnaflix\.com/[^/]+/(?P<display_id>[^/]+)/video(?P<id>\d+)'
 

From 8fab62482abf8e75f2dd98fedbe95d81ce767382 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Feb 2016 16:59:10 +0600
Subject: [PATCH 4124/6219] [tnaflixnetwork] Fallback age limit to 18

---
 youtube_dl/extractor/tnaflix.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tnaflix.py b/youtube_dl/extractor/tnaflix.py
index 547d83d09..e8c272704 100644
--- a/youtube_dl/extractor/tnaflix.py
+++ b/youtube_dl/extractor/tnaflix.py
@@ -117,7 +117,7 @@ class TNAFlixNetworkBaseIE(InfoExtractor):
         title = self._html_search_regex(
             self._TITLE_REGEX, webpage, 'title') if self._TITLE_REGEX else self._og_search_title(webpage)
 
-        age_limit = self._rta_search(webpage)
+        age_limit = self._rta_search(webpage) or 18
 
         duration = parse_duration(self._html_search_meta(
             'duration', webpage, 'duration', default=None))

From 63719a8ac3054f702238632495e291dd6f81e6b1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Feb 2016 17:15:06 +0600
Subject: [PATCH 4125/6219] [tnaflixnetwork:embed] Add _extract_urls

---
 youtube_dl/extractor/tnaflix.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/tnaflix.py b/youtube_dl/extractor/tnaflix.py
index e8c272704..79f036fe4 100644
--- a/youtube_dl/extractor/tnaflix.py
+++ b/youtube_dl/extractor/tnaflix.py
@@ -175,6 +175,12 @@ class TNAFlixNetworkEmbedIE(TNAFlixNetworkBaseIE):
         'only_matching': True,
     }]
 
+    @staticmethod
+    def _extract_urls(webpage):
+        return [url for _, url in re.findall(
+            r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//player\.(?:tna|emp)flix\.com/video/\d+)\1',
+            webpage)]
+
 
 class TNAFlixIE(TNAFlixNetworkBaseIE):
     _VALID_URL = r'https?://(?:www\.)?tnaflix\.com/[^/]+/(?P<display_id>[^/]+)/video(?P<id>\d+)'

From 2c9ca78281f84abe194bdf23a5e06b747961c9eb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Feb 2016 17:15:49 +0600
Subject: [PATCH 4126/6219] [extractor/generic] Add support for tnaflix network
 embeds (Closes #7505)

---
 youtube_dl/extractor/generic.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index c6bf8d270..ca745ae41 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -47,6 +47,7 @@ from .senateisvp import SenateISVPIE
 from .svt import SVTIE
 from .pornhub import PornHubIE
 from .xhamster import XHamsterEmbedIE
+from .tnaflix import TNAFlixNetworkEmbedIE
 from .vimeo import VimeoIE
 from .dailymotion import DailymotionCloudIE
 from .onionstudios import OnionStudiosIE
@@ -1633,6 +1634,11 @@ class GenericIE(InfoExtractor):
         if xhamster_urls:
             return _playlist_from_matches(xhamster_urls, ie='XHamsterEmbed')
 
+        # Look for embedded TNAFlixNetwork player
+        tnaflix_urls = TNAFlixNetworkEmbedIE._extract_urls(webpage)
+        if tnaflix_urls:
+            return _playlist_from_matches(tnaflix_urls, ie=TNAFlixNetworkEmbedIE.ie_key())
+
         # Look for embedded Tvigle player
         mobj = re.search(
             r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//cloud\.tvigle\.ru/video/.+?)\1', webpage)

From 950505370416e79073acdeaa73a1023950363266 Mon Sep 17 00:00:00 2001
From: Aidan Rowe <aidanrowe@gmail.com>
Date: Mon, 8 Feb 2016 20:57:07 +1000
Subject: [PATCH 4127/6219] [dplay] add support for it.dplay.com and dplay.dk

---
 youtube_dl/extractor/dplay.py | 84 +++++++++++++++++++++++++++--------
 1 file changed, 65 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/extractor/dplay.py b/youtube_dl/extractor/dplay.py
index 6cda56a7f..adcfc1f0e 100644
--- a/youtube_dl/extractor/dplay.py
+++ b/youtube_dl/extractor/dplay.py
@@ -5,40 +5,83 @@ import time
 
 from .common import InfoExtractor
 from ..utils import int_or_none
-
+from ..compat import compat_urlparse
 
 class DPlayIE(InfoExtractor):
-    _VALID_URL = r'http://www\.dplay\.se/[^/]+/(?P<id>[^/?#]+)'
+    _VALID_URL = r'(?P<domain>http://(?:it|www)\.dplay\.(?:com|dk|se))/[^/]+/(?P<id>[^/?#]+)'
 
-    _TEST = {
-        'url': 'http://www.dplay.se/nugammalt-77-handelser-som-format-sverige/season-1-svensken-lar-sig-njuta-av-livet/',
-        'info_dict': {
-            'id': '3172',
-            'ext': 'mp4',
-            'display_id': 'season-1-svensken-lar-sig-njuta-av-livet',
-            'title': 'Svensken lär sig njuta av livet',
-            'duration': 2650,
+    _TESTS = [
+        {
+            'url': 'http://it.dplay.com/take-me-out/stagione-1-episodio-25/',
+            'info_dict': {
+                'id': '1255600',
+                'ext': 'mp4',
+                'display_id': 'stagione-1-episodio-25',
+                'title': 'Episodio 25',
+                'duration': 2761,
+                'description': "Gabriele Corsi conduce un nuovo provocante e divertente dating show. 30 ragazze single hanno l'opportunità di conoscere un ragazzo e decidere se tenerlo in gioco oppure no accendendo o spegnendo le luci.",
+                'season_number': 1,
+                'episode_number': 25,
+            },
         },
-    }
+        {
+            'url': 'http://www.dplay.se/nugammalt-77-handelser-som-format-sverige/season-1-svensken-lar-sig-njuta-av-livet/',
+            'info_dict': {
+                'id': '3172',
+                'ext': 'mp4',
+                'display_id': 'season-1-svensken-lar-sig-njuta-av-livet',
+                'title': 'Svensken lär sig njuta av livet',
+                'duration': 2650,
+                'description': "\"Svensken lär sig njuta av livet\". Införandet av systembolaget, industrisemestern och Skarastadgan. Med hjälp av arkivmaterial, experter och fakta ska händelserna dissekeras, analyseras och dras till sin absoluta underhållningsspets.",
+                'season_number': 1,
+                'episode_number': 1,
+            },
+        },
+        {
+            'url': 'http://www.dplay.dk/mig-og-min-mor/season-6-episode-12/',
+            'info_dict': {
+                'id': '70816',
+                'ext': 'mp4',
+                'display_id': 'season-6-episode-12',
+                'title': 'Episode 12',
+                'duration': 2563,
+                'description': " I sæsonafslutningen sker der store ting for mor og datter.\nDagen er endelig kommet for den højgravide Irina - hun skal føde! Men det bliver en lang og sej kamp for Irina, som selvfølgelig har mor Mila med som støtte hele vejen.\nMor Jette og Jessica er igen hjemme i Danmark efter deres store USA-eventyr. Og for at holde fast i den amerikanske ånd, tager pigerne i dag til gospel-undervisning. \nOg så skal Joy og mor Mia under kniven  - de skal nemlig have gjort lårene mindre og ballerne større. \n ",
+                'season_number': 6,
+                'episode_number': 12,
+            },
+        }
+    ]
 
     def _real_extract(self, url):
+        #this extrator works with it.dplay.com, www.dplay.se and www.dplay.dk
+        # so we need to determine the domain to send the requests to
+        domain = self._search_regex(self._VALID_URL, url, 'domain')
         display_id = self._match_id(url)
         webpage = self._download_webpage(url, display_id)
         video_id = self._search_regex(
             r'data-video-id="(\d+)"', webpage, 'video id')
 
-        info = self._download_json(
-            'http://www.dplay.se/api/v2/ajax/videos?video_id=' + video_id,
+        video_url = compat_urlparse.urljoin(domain, 'api/v2/ajax/videos?video_id=')
+
+        info = self._download_json(video_url + video_id,
             video_id)['data'][0]
 
-        self._set_cookie(
-            'secure.dplay.se', 'dsc-geo',
-            '{"countryCode":"NL","expiry":%d}' % ((time.time() + 20 * 60) * 1000))
         # TODO: consider adding support for 'stream_type=hds', it seems to
         # require setting some cookies
-        manifest_url = self._download_json(
-            'https://secure.dplay.se/secure/api/v2/user/authorization/stream/%s?stream_type=hls' % video_id,
-            video_id, 'Getting manifest url for hls stream')['hls']
+        # get url's TLD to determine which cookie and url to use
+        domain_tld = domain.split('.')[-1]
+        if domain_tld == 'se' or domain_tld == 'dk':
+            self._set_cookie(
+                'secure.dplay.%s' % domain_tld, 'dsc-geo',
+                '{"countryCode":"%s","expiry":%d}' % (domain_tld.upper(), ((time.time() + 20 * 60) * 1000)))
+
+            manifest_url = self._download_json(
+                'https://secure.dplay.%s/secure/api/v2/user/authorization/stream/%s?stream_type=hls' % (domain_tld, video_id),
+                video_id, 'Getting manifest url for hls stream')['hls']
+        else:
+            #.it requires no cookies at this point
+            manifest_url = info['hls']
+
         formats = self._extract_m3u8_formats(
             manifest_url, video_id, ext='mp4', entry_protocol='m3u8_native')
 
@@ -48,4 +91,7 @@ class DPlayIE(InfoExtractor):
             'title': info['title'],
             'formats': formats,
             'duration': int_or_none(info.get('video_metadata_length'), scale=1000),
+            'description': info.get('video_metadata_longDescription'),
+            'season_number': int_or_none(info.get('season')),
+            'episode_number': int_or_none(info.get('episode')),
         }

From 940b606a0743c0f23aa4313019d8af67d863f064 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Feb 2016 21:30:47 +0600
Subject: [PATCH 4128/6219] [dplay] Improve, extract all formats and metadata
 (Closes #8463)

---
 youtube_dl/extractor/dplay.py | 172 ++++++++++++++++++++--------------
 1 file changed, 103 insertions(+), 69 deletions(-)

diff --git a/youtube_dl/extractor/dplay.py b/youtube_dl/extractor/dplay.py
index adcfc1f0e..87071c4f8 100644
--- a/youtube_dl/extractor/dplay.py
+++ b/youtube_dl/extractor/dplay.py
@@ -1,97 +1,131 @@
-# encoding: utf-8
+# coding: utf-8
 from __future__ import unicode_literals
 
+import json
+import re
 import time
 
 from .common import InfoExtractor
 from ..utils import int_or_none
-from ..compat import compat_urlparse
+
 
 class DPlayIE(InfoExtractor):
-    _VALID_URL = r'(?P<domain>http://(?:it|www)\.dplay\.(?:com|dk|se))/[^/]+/(?P<id>[^/?#]+)'
+    _VALID_URL = r'http://(?P<domain>it\.dplay\.com|www\.dplay\.(?:dk|se))/[^/]+/(?P<id>[^/?#]+)'
 
-    _TESTS = [
-        {
-            'url': 'http://it.dplay.com/take-me-out/stagione-1-episodio-25/',
-            'info_dict': {
-                'id': '1255600',
-                'ext': 'mp4',
-                'display_id': 'stagione-1-episodio-25',
-                'title': 'Episodio 25',
-                'duration': 2761,
-                'description': "Gabriele Corsi conduce un nuovo provocante e divertente dating show. 30 ragazze single hanno l'opportunità di conoscere un ragazzo e decidere se tenerlo in gioco oppure no accendendo o spegnendo le luci.",
-                'season_number': 1,
-                'episode_number': 25,
-            },
+    _TESTS = [{
+        'url': 'http://it.dplay.com/take-me-out/stagione-1-episodio-25/',
+        'info_dict': {
+            'id': '1255600',
+            'display_id': 'stagione-1-episodio-25',
+            'ext': 'mp4',
+            'title': 'Episodio 25',
+            'description': 'md5:cae5f40ad988811b197d2d27a53227eb',
+            'duration': 2761,
+            'timestamp': 1454701800,
+            'upload_date': '20160205',
+            'creator': 'RTIT',
+            'series': 'Take me out',
+            'season_number': 1,
+            'episode_number': 25,
+            'age_limit': 0,
         },
-        {
-            'url': 'http://www.dplay.se/nugammalt-77-handelser-som-format-sverige/season-1-svensken-lar-sig-njuta-av-livet/',
-            'info_dict': {
-                'id': '3172',
-                'ext': 'mp4',
-                'display_id': 'season-1-svensken-lar-sig-njuta-av-livet',
-                'title': 'Svensken lär sig njuta av livet',
-                'duration': 2650,
-                'description': "\"Svensken lär sig njuta av livet\". Införandet av systembolaget, industrisemestern och Skarastadgan. Med hjälp av arkivmaterial, experter och fakta ska händelserna dissekeras, analyseras och dras till sin absoluta underhållningsspets.",
-                'season_number': 1,
-                'episode_number': 1,
-            },
+        'expected_warnings': ['Unable to download f4m manifest'],
+    }, {
+        'url': 'http://www.dplay.se/nugammalt-77-handelser-som-format-sverige/season-1-svensken-lar-sig-njuta-av-livet/',
+        'info_dict': {
+            'id': '3172',
+            'display_id': 'season-1-svensken-lar-sig-njuta-av-livet',
+            'ext': 'flv',
+            'title': 'Svensken lär sig njuta av livet',
+            'description': 'md5:d3819c9bccffd0fe458ca42451dd50d8',
+            'duration': 2650,
+            'timestamp': 1365454320,
+            'upload_date': '20130408',
+            'creator': 'Kanal 5 (Home)',
+            'series': 'Nugammalt - 77 händelser som format Sverige',
+            'season_number': 1,
+            'episode_number': 1,
+            'age_limit': 0,
         },
-        {
-            'url': 'http://www.dplay.dk/mig-og-min-mor/season-6-episode-12/',
-            'info_dict': {
-                'id': '70816',
-                'ext': 'mp4',
-                'display_id': 'season-6-episode-12',
-                'title': 'Episode 12',
-                'duration': 2563,
-                'description': " I sæsonafslutningen sker der store ting for mor og datter.\nDagen er endelig kommet for den højgravide Irina - hun skal føde! Men det bliver en lang og sej kamp for Irina, som selvfølgelig har mor Mila med som støtte hele vejen.\nMor Jette og Jessica er igen hjemme i Danmark efter deres store USA-eventyr. Og for at holde fast i den amerikanske ånd, tager pigerne i dag til gospel-undervisning. \nOg så skal Joy og mor Mia under kniven  - de skal nemlig have gjort lårene mindre og ballerne større. \n ",
-                'season_number': 6,
-                'episode_number': 12,
-            },
-        }
-    ]
+    }, {
+        'url': 'http://www.dplay.dk/mig-og-min-mor/season-6-episode-12/',
+        'info_dict': {
+            'id': '70816',
+            'display_id': 'season-6-episode-12',
+            'ext': 'flv',
+            'title': 'Episode 12',
+            'description': 'md5:9c86e51a93f8a4401fc9641ef9894c90',
+            'duration': 2563,
+            'timestamp': 1429696800,
+            'upload_date': '20150422',
+            'creator': 'Kanal 4',
+            'series': 'Mig og min mor',
+            'season_number': 6,
+            'episode_number': 12,
+            'age_limit': 0,
+        },
+    }]
 
     def _real_extract(self, url):
-        #this extrator works with it.dplay.com, www.dplay.se and www.dplay.dk
-        # so we need to determine the domain to send the requests to
-        domain = self._search_regex(self._VALID_URL, url, 'domain')
-        display_id = self._match_id(url)
+        mobj = re.match(self._VALID_URL, url)
+        display_id = mobj.group('id')
+        domain = mobj.group('domain')
+
         webpage = self._download_webpage(url, display_id)
+
         video_id = self._search_regex(
-            r'data-video-id="(\d+)"', webpage, 'video id')
+            r'data-video-id=["\'](\d+)', webpage, 'video id')
 
-        video_url = compat_urlparse.urljoin(domain, 'api/v2/ajax/videos?video_id=')
-
-        info = self._download_json(video_url + video_id,
+        info = self._download_json(
+            'http://%s/api/v2/ajax/videos?video_id=%s' % (domain, video_id),
             video_id)['data'][0]
 
-        # TODO: consider adding support for 'stream_type=hds', it seems to
-        # require setting some cookies
-        # get url's TLD to determine which cookie and url to use
+        title = info['title']
+
+        PROTOCOLS = ('hls', 'hds')
+        formats = []
+
+        def extract_formats(protocol, manifest_url):
+            if protocol == 'hls':
+                formats.extend(self._extract_m3u8_formats(
+                    manifest_url, video_id, ext='mp4',
+                    entry_protocol='m3u8_native', m3u8_id=protocol, fatal=False))
+            elif protocol == 'hds':
+                formats.extend(self._extract_f4m_formats(
+                    manifest_url + '&hdcore=3.8.0&plugin=flowplayer-3.8.0.0',
+                    video_id, f4m_id=protocol, fatal=False))
+
         domain_tld = domain.split('.')[-1]
-        if domain_tld == 'se' or domain_tld == 'dk':
-            self._set_cookie(
-                'secure.dplay.%s' % domain_tld, 'dsc-geo',
-                '{"countryCode":"%s","expiry":%d}' % (domain_tld.upper(), ((time.time() + 20 * 60) * 1000)))
-
-            manifest_url = self._download_json(
-                'https://secure.dplay.%s/secure/api/v2/user/authorization/stream/%s?stream_type=hls' % (domain_tld, video_id),
-                video_id, 'Getting manifest url for hls stream')['hls']
+        if domain_tld in ('se', 'dk'):
+            for protocol in PROTOCOLS:
+                self._set_cookie(
+                    'secure.dplay.%s' % domain_tld, 'dsc-geo',
+                    json.dumps({
+                        'countryCode': domain_tld.upper(),
+                        'expiry': (time.time() + 20 * 60) * 1000,
+                    }))
+                stream = self._download_json(
+                    'https://secure.dplay.%s/secure/api/v2/user/authorization/stream/%s?stream_type=%s'
+                    % (domain_tld, video_id, protocol), video_id,
+                    'Downloading %s stream JSON' % protocol, fatal=False)
+                if stream and stream.get(protocol):
+                    extract_formats(protocol, stream[protocol])
         else:
-            #.it requires no cookies at this point
-            manifest_url = info['hls']
-
-        formats = self._extract_m3u8_formats(
-            manifest_url, video_id, ext='mp4', entry_protocol='m3u8_native')
+            for protocol in PROTOCOLS:
+                if info.get(protocol):
+                    extract_formats(protocol, info[protocol])
 
         return {
             'id': video_id,
             'display_id': display_id,
-            'title': info['title'],
-            'formats': formats,
-            'duration': int_or_none(info.get('video_metadata_length'), scale=1000),
+            'title': title,
             'description': info.get('video_metadata_longDescription'),
+            'duration': int_or_none(info.get('video_metadata_length'), scale=1000),
+            'timestamp': int_or_none(info.get('video_publish_date')),
+            'creator': info.get('video_metadata_homeChannel'),
+            'series': info.get('video_metadata_show'),
             'season_number': int_or_none(info.get('season')),
             'episode_number': int_or_none(info.get('episode')),
+            'age_limit': int_or_none(info.get('minimum_age')),
+            'formats': formats,
         }

From 20afe8bd14c76af308d4fe2fce4cef05067b10c7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Feb 2016 21:31:43 +0600
Subject: [PATCH 4129/6219] Credit @aidan- for more dplay sites support (#8463)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 0e90a3ecb..b51e23f2d 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -160,3 +160,4 @@ Erwin de Haan
 Jens Wille
 Robin Houtevelts
 Patrick Griffis
+Aidan Rowe

From 5add979d918ceb2718026bdfa66f141078b09b19 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Feb 2016 21:42:08 +0600
Subject: [PATCH 4130/6219] [dplay] Add support for dplay.no

---
 youtube_dl/extractor/dplay.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/dplay.py b/youtube_dl/extractor/dplay.py
index 87071c4f8..a638c827c 100644
--- a/youtube_dl/extractor/dplay.py
+++ b/youtube_dl/extractor/dplay.py
@@ -10,7 +10,7 @@ from ..utils import int_or_none
 
 
 class DPlayIE(InfoExtractor):
-    _VALID_URL = r'http://(?P<domain>it\.dplay\.com|www\.dplay\.(?:dk|se))/[^/]+/(?P<id>[^/?#]+)'
+    _VALID_URL = r'http://(?P<domain>it\.dplay\.com|www\.dplay\.(?:dk|se|no))/[^/]+/(?P<id>[^/?#]+)'
 
     _TESTS = [{
         'url': 'http://it.dplay.com/take-me-out/stagione-1-episodio-25/',
@@ -64,6 +64,9 @@ class DPlayIE(InfoExtractor):
             'episode_number': 12,
             'age_limit': 0,
         },
+    }, {
+        'url': 'http://www.dplay.no/pga-tour/season-1-hoydepunkter-18-21-februar/',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From da665ddc2551147b7b38f919c9d5c719a93f9c5d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 27 Feb 2016 21:31:21 +0100
Subject: [PATCH 4131/6219] release 2016.02.27

---
 CONTRIBUTING.md        | 13 ++++++++-----
 docs/supportedsites.md |  6 ++++--
 youtube_dl/version.py  |  2 +-
 3 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 39472c554..c996f03ab 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -92,7 +92,9 @@ If you want to create a build of youtube-dl yourself, you'll need
 
 ### Adding support for a new site
 
-If you want to add support for a new site, you can follow this quick list (assuming your service is called `yourextractor`):
+If you want to add support for a new site, first of all **make sure** this site is **not dedicated to [copyright infringement](#can-you-add-support-for-this-anime-video-site-or-site-which-shows-current-movies-for-free)**. youtube-dl does **not support** such sites thus pull requests adding support for them **will be rejected**.
+
+After you have ensured this site is distributing it's content legally, you can follow this quick list (assuming your service is called `yourextractor`):
 
 1. [Fork this repository](https://github.com/rg3/youtube-dl/fork)
 2. Check out the source code with `git clone git@github.com:YOUR_GITHUB_USERNAME/youtube-dl.git`
@@ -140,16 +142,17 @@ If you want to add support for a new site, you can follow this quick list (assum
     ```
 5. Add an import in [`youtube_dl/extractor/__init__.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/__init__.py).
 6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc.
-7. Have a look at [`youtube_dl/extractor/common.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L62-L200). Add tests and code for as many as you want.
-8. If you can, check the code with [flake8](https://pypi.python.org/pypi/flake8).
-9. When the tests pass, [add](http://git-scm.com/docs/git-add) the new files and [commit](http://git-scm.com/docs/git-commit) them and [push](http://git-scm.com/docs/git-push) the result, like this:
+7. Have a look at [`youtube_dl/extractor/common.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/rg3/youtube-dl/blob/58525c94d547be1c8167d16c298bdd75506db328/youtube_dl/extractor/common.py#L68-L226). Add tests and code for as many as you want.
+8. Keep in mind that the only mandatory fields in info dict for successful extraction process are `id`, `title` and either `url` or `formats`, i.e. these are the critical data the extraction does not make any sense without. This means that [any field](https://github.com/rg3/youtube-dl/blob/58525c94d547be1c8167d16c298bdd75506db328/youtube_dl/extractor/common.py#L138-L226) apart from aforementioned mandatory ones should be treated **as optional** and extraction should be **tolerate** to situations when sources for these fields can potentially be unavailable (even if they always available at the moment) and **future-proof** in order not to break the extraction of general purpose mandatory fields. For example, if you have some intermediate dict `meta` that is a source of metadata and it has a key `summary` that you want to extract and put into resulting info dict as `description`, you should be ready that this key may be missing from the `meta` dict, i.e. you should extract it as `meta.get('summary')` and not `meta['summary']`. Similarly, you should pass `fatal=False` when extracting data from a webpage with `_search_regex/_html_search_regex`.
+9. Check the code with [flake8](https://pypi.python.org/pypi/flake8).
+10. When the tests pass, [add](http://git-scm.com/docs/git-add) the new files and [commit](http://git-scm.com/docs/git-commit) them and [push](http://git-scm.com/docs/git-push) the result, like this:
 
         $ git add youtube_dl/extractor/__init__.py
         $ git add youtube_dl/extractor/yourextractor.py
         $ git commit -m '[yourextractor] Add new extractor'
         $ git push origin yourextractor
 
-10. Finally, [create a pull request](https://help.github.com/articles/creating-a-pull-request). We'll then review and merge it.
+11. Finally, [create a pull request](https://help.github.com/articles/creating-a-pull-request). We'll then review and merge it.
 
 In any case, thank you very much for your contributions!
 
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 74596155c..43403233d 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -77,6 +77,7 @@
  - **BleacherReportCMS**
  - **blinkx**
  - **Bloomberg**
+ - **BokeCC**
  - **Bpb**: Bundeszentrale für politische Bildung
  - **BR**: Bayerischer Rundfunk Mediathek
  - **Break**
@@ -560,7 +561,6 @@
  - **southpark.de**
  - **southpark.nl**
  - **southparkstudios.dk**
- - **Space**
  - **SpankBang**
  - **Spankwire**
  - **Spiegel**
@@ -620,6 +620,7 @@
  - **TMZ**
  - **TMZArticle**
  - **TNAFlix**
+ - **TNAFlixNetworkEmbed**
  - **toggle**
  - **tou.tv**
  - **Toypics**: Toypics user profile
@@ -670,6 +671,7 @@
  - **Urort**: NRK P3 Urørt
  - **ustream**
  - **ustream:channel**
+ - **Ustudio**
  - **Varzesh3**
  - **Vbox7**
  - **VeeHD**
@@ -685,7 +687,7 @@
  - **video.mit.edu**
  - **VideoDetective**
  - **videofy.me**
- - **VideoMega** (Currently broken)
+ - **VideoMega**
  - **videomore**
  - **videomore:season**
  - **videomore:video**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 7a3df6a26..e2836357f 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.02.22'
+__version__ = '2016.02.27'

From 4587915b2aed8d2a8373614e5984fe6e5ccaeae2 Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Sun, 28 Feb 2016 02:56:09 +0600
Subject: [PATCH 4132/6219] [README.md] Make configuration file example more
 diverse

---
 README.md | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index d50d4b950..7d9d501d7 100644
--- a/README.md
+++ b/README.md
@@ -409,11 +409,14 @@ which means you can modify it, redistribute it or use it however you like.
 
 # CONFIGURATION
 
-You can configure youtube-dl by placing any supported command line option to a configuration file. On Linux, the system wide configuration file is located at `/etc/youtube-dl.conf` and the user wide configuration file at `~/.config/youtube-dl/config`. On Windows, the user wide configuration file locations are `%APPDATA%\youtube-dl\config.txt` or `C:\Users\<user name>\youtube-dl.conf`. For example, with the following configuration file youtube-dl will always extract the audio, not copy the mtime and use a proxy:
+You can configure youtube-dl by placing any supported command line option to a configuration file. On Linux, the system wide configuration file is located at `/etc/youtube-dl.conf` and the user wide configuration file at `~/.config/youtube-dl/config`. On Windows, the user wide configuration file locations are `%APPDATA%\youtube-dl\config.txt` or `C:\Users\<user name>\youtube-dl.conf`.
+
+For example, with the following configuration file youtube-dl will always extract the audio, not copy the mtime, use a proxy and save all videos under `Movies` directory in your `$HOME`:
 ```
---extract-audio
+-x
 --no-mtime
 --proxy 127.0.0.1:3128
+-o ~/Movies/%(title)s.%(ext)s
 ```
 
 You can use `--ignore-config` if you want to disable the configuration file for a particular youtube-dl run.

From a048ffc9b01a4c6a63603edbb7204479ddebbec6 Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Sun, 28 Feb 2016 03:04:06 +0600
Subject: [PATCH 4133/6219] [README.md] Clarify configuration file options
 syntax

---
 README.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/README.md b/README.md
index 7d9d501d7..ea4ad327b 100644
--- a/README.md
+++ b/README.md
@@ -419,6 +419,8 @@ For example, with the following configuration file youtube-dl will always extrac
 -o ~/Movies/%(title)s.%(ext)s
 ```
 
+Note that options in configuration file are just the same options aka switches used in regular command line calls thus there **must be no whitespace** after `-` or `--`, e.g. `-o` or `--proxy` but not `- o` or `-- proxy`.
+
 You can use `--ignore-config` if you want to disable the configuration file for a particular youtube-dl run.
 
 ### Authentication with `.netrc` file

From 7a0e7779fe571972b72edc6aab4b8c93db4b22e8 Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Sun, 28 Feb 2016 03:12:13 +0600
Subject: [PATCH 4134/6219] [README.md] Use simple wording instead of env
 variable for home

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index ea4ad327b..44b65da02 100644
--- a/README.md
+++ b/README.md
@@ -411,7 +411,7 @@ which means you can modify it, redistribute it or use it however you like.
 
 You can configure youtube-dl by placing any supported command line option to a configuration file. On Linux, the system wide configuration file is located at `/etc/youtube-dl.conf` and the user wide configuration file at `~/.config/youtube-dl/config`. On Windows, the user wide configuration file locations are `%APPDATA%\youtube-dl\config.txt` or `C:\Users\<user name>\youtube-dl.conf`.
 
-For example, with the following configuration file youtube-dl will always extract the audio, not copy the mtime, use a proxy and save all videos under `Movies` directory in your `$HOME`:
+For example, with the following configuration file youtube-dl will always extract the audio, not copy the mtime, use a proxy and save all videos under `Movies` directory in your home directory:
 ```
 -x
 --no-mtime

From 8870bb4653982a81c2ff332103499e12a825099c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 28 Feb 2016 03:37:48 +0600
Subject: [PATCH 4135/6219] [webofstories] Tolerate malforder og:title (Closes
 #8417)

---
 youtube_dl/extractor/webofstories.py | 64 +++++++++++++++++-----------
 1 file changed, 39 insertions(+), 25 deletions(-)

diff --git a/youtube_dl/extractor/webofstories.py b/youtube_dl/extractor/webofstories.py
index 2037d9b3d..7aea47ed5 100644
--- a/youtube_dl/extractor/webofstories.py
+++ b/youtube_dl/extractor/webofstories.py
@@ -12,38 +12,52 @@ class WebOfStoriesIE(InfoExtractor):
     _VIDEO_DOMAIN = 'http://eu-mobile.webofstories.com/'
     _GREAT_LIFE_STREAMER = 'rtmp://eu-cdn1.webofstories.com/cfx/st/'
     _USER_STREAMER = 'rtmp://eu-users.webofstories.com/cfx/st/'
-    _TESTS = [
-        {
-            'url': 'http://www.webofstories.com/play/hans.bethe/71',
-            'md5': '373e4dd915f60cfe3116322642ddf364',
-            'info_dict': {
-                'id': '4536',
-                'ext': 'mp4',
-                'title': 'The temperature of the sun',
-                'thumbnail': 're:^https?://.*\.jpg$',
-                'description': 'Hans Bethe talks about calculating the temperature of the sun',
-                'duration': 238,
-            }
+    _TESTS = [{
+        'url': 'http://www.webofstories.com/play/hans.bethe/71',
+        'md5': '373e4dd915f60cfe3116322642ddf364',
+        'info_dict': {
+            'id': '4536',
+            'ext': 'mp4',
+            'title': 'The temperature of the sun',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'description': 'Hans Bethe talks about calculating the temperature of the sun',
+            'duration': 238,
+        }
+    }, {
+        'url': 'http://www.webofstories.com/play/55908',
+        'md5': '2985a698e1fe3211022422c4b5ed962c',
+        'info_dict': {
+            'id': '55908',
+            'ext': 'mp4',
+            'title': 'The story of Gemmata obscuriglobus',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'description': 'Planctomycete talks about The story of Gemmata obscuriglobus',
+            'duration': 169,
         },
-        {
-            'url': 'http://www.webofstories.com/play/55908',
-            'md5': '2985a698e1fe3211022422c4b5ed962c',
-            'info_dict': {
-                'id': '55908',
-                'ext': 'mp4',
-                'title': 'The story of Gemmata obscuriglobus',
-                'thumbnail': 're:^https?://.*\.jpg$',
-                'description': 'Planctomycete talks about The story of Gemmata obscuriglobus',
-                'duration': 169,
-            }
+        'skip': 'notfound',
+    }, {
+        # malformed og:title meta
+        'url': 'http://www.webofstories.com/play/54215?o=MS',
+        'info_dict': {
+            'id': '54215',
+            'ext': 'mp4',
+            'title': '"A Leg to Stand On"',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'description': 'Oliver Sacks talks about the death and resurrection of a limb',
+            'duration': 97,
         },
-    ]
+        'params': {
+            'skip_download': True,
+        },
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id)
-        title = self._og_search_title(webpage)
+        # Sometimes og:title meta is malformed
+        title = self._og_search_title(webpage, default=None) or self._html_search_regex(
+            r'(?s)<strong>Title:\s*</strong>(.+?)<', webpage, 'title')
         description = self._html_search_meta('description', webpage)
         thumbnail = self._og_search_thumbnail(webpage)
 

From 9173202b84ea303c83c30b7305a1dcec5ccbe9e6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 28 Feb 2016 14:06:26 +0100
Subject: [PATCH 4136/6219] [zdf] Ignore hls manifests that use https (closes
 #8665)

The certificates are misconfigured, you get the following error mesage:

    ssl.CertificateError: hostname u'zdf-hdios-none-i.zdf.de' doesn't match either of 'a248.e.akamai.net', '*.akamaihd.net', '*.akamaihd-staging.net', '*.akamaized.net', '*.akamaized-staging.net'
---
 youtube_dl/extractor/zdf.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/zdf.py b/youtube_dl/extractor/zdf.py
index c619a75e2..81c22a627 100644
--- a/youtube_dl/extractor/zdf.py
+++ b/youtube_dl/extractor/zdf.py
@@ -137,6 +137,10 @@ class ZDFIE(InfoExtractor):
                 formats.extend(self._extract_smil_formats(
                     video_url, video_id, fatal=False))
             elif ext == 'm3u8':
+                # the certificates are misconfigured (see
+                # https://github.com/rg3/youtube-dl/issues/8665)
+                if video_url.startswith('https://'):
+                    continue
                 formats.extend(self._extract_m3u8_formats(
                     video_url, video_id, 'mp4', m3u8_id=format_id, fatal=False))
             elif ext == 'f4m':

From 6c10dbeae948b491b538f70481ee5b348a636067 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 28 Feb 2016 20:05:58 +0600
Subject: [PATCH 4137/6219] [screenwavemedia] Improve formats extraction

---
 youtube_dl/extractor/screenwavemedia.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/screenwavemedia.py b/youtube_dl/extractor/screenwavemedia.py
index 2cf210e0d..d2af26c1c 100644
--- a/youtube_dl/extractor/screenwavemedia.py
+++ b/youtube_dl/extractor/screenwavemedia.py
@@ -70,19 +70,19 @@ class ScreenwaveMediaIE(InfoExtractor):
 
         formats = []
         for source in sources:
-            if source['type'] == 'hls':
-                formats.extend(self._extract_m3u8_formats(source['file'], video_id, ext='mp4'))
+            file_ = source.get('file')
+            if not file_:
+                continue
+            if source.get('type') == 'hls':
+                formats.extend(self._extract_m3u8_formats(file_, video_id, ext='mp4'))
             else:
-                file_ = source.get('file')
-                if not file_:
-                    continue
                 format_label = source.get('label')
                 format_id = self._search_regex(
                     r'_(.+?)\.[^.]+$', file_, 'format id', default=None)
                 height = int_or_none(self._search_regex(
                     r'^(\d+)[pP]', format_label, 'height', default=None))
                 formats.append({
-                    'url': source['file'],
+                    'url': file_,
                     'format_id': format_id,
                     'format': format_label,
                     'ext': source.get('type'),

From 2b2dfae83ef80f86e9983ded3c4514b58b1eed60 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 28 Feb 2016 20:16:31 +0600
Subject: [PATCH 4138/6219] [screenwavemedia] Improve formats sorting

---
 youtube_dl/extractor/screenwavemedia.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/screenwavemedia.py b/youtube_dl/extractor/screenwavemedia.py
index d2af26c1c..8e77788ae 100644
--- a/youtube_dl/extractor/screenwavemedia.py
+++ b/youtube_dl/extractor/screenwavemedia.py
@@ -88,7 +88,7 @@ class ScreenwaveMediaIE(InfoExtractor):
                     'ext': source.get('type'),
                     'height': height,
                 })
-        self._sort_formats(formats)
+        self._sort_formats(formats, field_preference=('height', 'width', 'tbr', 'format_id'))
 
         return {
             'id': video_id,

From 6dae56384a3a50f832b647f798a4b4bbd770448f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 28 Feb 2016 21:46:36 +0600
Subject: [PATCH 4139/6219] [screenwavemedia] Check formats' URLs

---
 youtube_dl/extractor/screenwavemedia.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/screenwavemedia.py b/youtube_dl/extractor/screenwavemedia.py
index 8e77788ae..44b0bbee6 100644
--- a/youtube_dl/extractor/screenwavemedia.py
+++ b/youtube_dl/extractor/screenwavemedia.py
@@ -76,9 +76,11 @@ class ScreenwaveMediaIE(InfoExtractor):
             if source.get('type') == 'hls':
                 formats.extend(self._extract_m3u8_formats(file_, video_id, ext='mp4'))
             else:
-                format_label = source.get('label')
                 format_id = self._search_regex(
                     r'_(.+?)\.[^.]+$', file_, 'format id', default=None)
+                if not self._is_valid_url(file_, video_id, format_id or 'video'):
+                    continue
+                format_label = source.get('label')
                 height = int_or_none(self._search_regex(
                     r'^(\d+)[pP]', format_label, 'height', default=None))
                 formats.append({

From 4b3cd7316cbb95100f7fc4dd03d86e0fd7674996 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 29 Feb 2016 03:28:21 +0600
Subject: [PATCH 4140/6219] [tf1] Improve wat id regex (Closes #8691)

---
 youtube_dl/extractor/tf1.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tf1.py b/youtube_dl/extractor/tf1.py
index e1a64e284..9ee844684 100644
--- a/youtube_dl/extractor/tf1.py
+++ b/youtube_dl/extractor/tf1.py
@@ -48,6 +48,6 @@ class TF1IE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
         wat_id = self._html_search_regex(
-            r'(["\'])(?:https?:)?//www\.wat\.tv/embedframe/.*?(?P<id>\d{8})\1',
+            r'(["\'])(?:https?:)?//www\.wat\.tv/embedframe/.*?(?P<id>\d{8})(?:#.*?)?\1',
             webpage, 'wat id', group='id')
         return self.url_result('wat:%s' % wat_id, 'Wat')

From d77ab8e255e593d8534bdd47e84c0cc03c4e6efd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 1 Mar 2016 01:01:33 +0600
Subject: [PATCH 4141/6219] Add --mark-watched feature (Closes #5054)

---
 youtube_dl/__init__.py          |  1 +
 youtube_dl/extractor/common.py  |  9 +++++++++
 youtube_dl/extractor/youtube.py | 26 ++++++++++++++++++++++++++
 youtube_dl/options.py           |  4 ++++
 4 files changed, 40 insertions(+)

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index f5f064241..79b389840 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -355,6 +355,7 @@ def _real_main(argv=None):
         'youtube_include_dash_manifest': opts.youtube_include_dash_manifest,
         'encoding': opts.encoding,
         'extract_flat': opts.extract_flat,
+        'mark_watched': opts.mark_watched,
         'merge_output_format': opts.merge_output_format,
         'postprocessors': postprocessors,
         'fixup': opts.fixup,
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 3f16b1b9e..a7c700099 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1620,6 +1620,15 @@ class InfoExtractor(object):
     def _get_automatic_captions(self, *args, **kwargs):
         raise NotImplementedError('This method must be implemented by subclasses')
 
+    def mark_watched(self, *args, **kwargs):
+        if (self._downloader.params.get('mark_watched', False) and
+                (self._get_login_info()[0] is not None or
+                    self._downloader.params.get('cookiefile') is not None)):
+            self._mark_watched(*args, **kwargs)
+
+    def _mark_watched(self, *args, **kwargs):
+        raise NotImplementedError('This method must be implemented by subclasses')
+
 
 class SearchInfoExtractor(InfoExtractor):
     """
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index ec90c2111..ba339b67d 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -6,6 +6,7 @@ from __future__ import unicode_literals
 import itertools
 import json
 import os.path
+import random
 import re
 import time
 import traceback
@@ -1046,6 +1047,29 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             self._downloader.report_warning(err_msg)
             return {}
 
+    def _mark_watched(self, video_id, video_info):
+        playback_url = video_info.get('videostats_playback_base_url', [None])[0]
+        if not playback_url:
+            return
+        parsed_playback_url = compat_urlparse.urlparse(playback_url)
+        qs = compat_urlparse.parse_qs(parsed_playback_url.query)
+
+        # cpn generation algorithm is reverse engineered from base.js.
+        # In fact it works even with dummy cpn.
+        CPN_ALPHABET = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_'
+        cpn = ''.join((CPN_ALPHABET[random.randint(0, 256) & 63] for _ in range(0, 16)))
+
+        qs.update({
+            'ver': ['2'],
+            'cpn': [cpn],
+        })
+        playback_url = compat_urlparse.urlunparse(
+            parsed_playback_url._replace(query=compat_urllib_parse.urlencode(qs, True)))
+
+        self._download_webpage(
+            playback_url, video_id, 'Marking watched',
+            'Unable to mark watched', fatal=False)
+
     @classmethod
     def extract_id(cls, url):
         mobj = re.match(cls._VALID_URL, url, re.VERBOSE)
@@ -1555,6 +1579,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
 
         self._sort_formats(formats)
 
+        self.mark_watched(video_id, video_info)
+
         return {
             'id': video_id,
             'uploader': video_uploader,
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 3afa8bb6f..048dee881 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -170,6 +170,10 @@ def parseOpts(overrideArguments=None):
         action='store_const', dest='extract_flat', const='in_playlist',
         default=False,
         help='Do not extract the videos of a playlist, only list them.')
+    general.add_option(
+        '--mark-watched',
+        action='store_true', dest='mark_watched', default=False,
+        help='Mark videos watched (YouTube only)')
     general.add_option(
         '--no-color', '--no-colors',
         action='store_true', dest='no_color',

From 2812c24c167dfaeed62737f2b5ba2a5c7c0de97f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 1 Mar 2016 01:24:26 +0600
Subject: [PATCH 4142/6219] [mdr] Fix extraction (Closes #8702)

---
 youtube_dl/extractor/mdr.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/mdr.py b/youtube_dl/extractor/mdr.py
index 425fc9e2a..2338e7f96 100644
--- a/youtube_dl/extractor/mdr.py
+++ b/youtube_dl/extractor/mdr.py
@@ -14,7 +14,7 @@ from ..utils import (
 
 class MDRIE(InfoExtractor):
     IE_DESC = 'MDR.DE and KiKA'
-    _VALID_URL = r'https?://(?:www\.)?(?:mdr|kika)\.de/(?:.*)/[a-z]+(?P<id>\d+)(?:_.+?)?\.html'
+    _VALID_URL = r'https?://(?:www\.)?(?:mdr|kika)\.de/(?:.*)/[a-z]+-?(?P<id>\d+)(?:_.+?)?\.html'
 
     _TESTS = [{
         # MDR regularly deletes its videos
@@ -60,6 +60,9 @@ class MDRIE(InfoExtractor):
     }, {
         'url': 'http://www.kika.de/sendungen/einzelsendungen/weihnachtsprogramm/einzelsendung2534.html',
         'only_matching': True,
+    }, {
+        'url': 'http://www.mdr.de/mediathek/mdr-videos/a/video-1334.html',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
@@ -68,8 +71,8 @@ class MDRIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         data_url = self._search_regex(
-            r'dataURL\s*:\s*(["\'])(?P<url>/.+/(?:video|audio)[0-9]+-avCustom\.xml)\1',
-            webpage, 'data url', group='url')
+            r'(?:dataURL|playerXml(?:["\'])?)\s*:\s*(["\'])(?P<url>\\?/.+/(?:video|audio)-?[0-9]+-avCustom\.xml)\1',
+            webpage, 'data url', default=None, group='url').replace('\/', '/')
 
         doc = self._download_xml(
             compat_urlparse.urljoin(url, data_url), video_id)

From 3e7696822064c6ec7e7d62aab98093d89eeb7cad Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 29 Feb 2016 20:57:26 +0100
Subject: [PATCH 4143/6219] [rtve.es:live] Fix extraction

* Update _VALID_URL to match the current URLs
* Use the m3u8 manifest since I haven't figured out how to use the rtmp stream
---
 youtube_dl/extractor/rtve.py | 30 ++++++++++++++----------------
 1 file changed, 14 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/extractor/rtve.py b/youtube_dl/extractor/rtve.py
index 603d7bd00..8a8c5d2a0 100644
--- a/youtube_dl/extractor/rtve.py
+++ b/youtube_dl/extractor/rtve.py
@@ -10,6 +10,7 @@ from ..utils import (
     ExtractorError,
     float_or_none,
     remove_end,
+    remove_start,
     sanitized_Request,
     std_headers,
     struct_unpack,
@@ -178,14 +179,14 @@ class RTVEInfantilIE(InfoExtractor):
 class RTVELiveIE(InfoExtractor):
     IE_NAME = 'rtve.es:live'
     IE_DESC = 'RTVE.es live streams'
-    _VALID_URL = r'http://www\.rtve\.es/(?:deportes/directo|noticias|television)/(?P<id>[a-zA-Z0-9-]+)'
+    _VALID_URL = r'http://www\.rtve\.es/directo/(?P<id>[a-zA-Z0-9-]+)'
 
     _TESTS = [{
-        'url': 'http://www.rtve.es/noticias/directo-la-1/',
+        'url': 'http://www.rtve.es/directo/la-1/',
         'info_dict': {
-            'id': 'directo-la-1',
-            'ext': 'flv',
-            'title': 're:^La 1 de TVE [0-9]{4}-[0-9]{2}-[0-9]{2}Z[0-9]{6}$',
+            'id': 'la-1',
+            'ext': 'mp4',
+            'title': 're:^La 1 [0-9]{4}-[0-9]{2}-[0-9]{2}Z[0-9]{6}$',
         },
         'params': {
             'skip_download': 'live stream',
@@ -198,23 +199,20 @@ class RTVELiveIE(InfoExtractor):
         video_id = mobj.group('id')
 
         webpage = self._download_webpage(url, video_id)
-        player_url = self._search_regex(
-            r'<param name="movie" value="([^"]+)"/>', webpage, 'player URL')
-        title = remove_end(self._og_search_title(webpage), ' en directo')
+        title = remove_end(self._og_search_title(webpage), ' en directo en RTVE.es')
+        title = remove_start(title, 'Estoy viendo ')
         title += ' ' + time.strftime('%Y-%m-%dZ%H%M%S', start_time)
 
         vidplayer_id = self._search_regex(
-            r' id="vidplayer([0-9]+)"', webpage, 'internal video ID')
-        png_url = 'http://www.rtve.es/ztnr/movil/thumbnail/default/videos/%s.png' % vidplayer_id
+            r'playerId=player([0-9]+)', webpage, 'internal video ID')
+        png_url = 'http://www.rtve.es/ztnr/movil/thumbnail/amonet/videos/%s.png' % vidplayer_id
         png = self._download_webpage(png_url, video_id, 'Downloading url information')
-        video_url = _decrypt_url(png)
+        m3u8_url = _decrypt_url(png)
+        formats = self._extract_m3u8_formats(m3u8_url, video_id, ext='mp4')
 
         return {
             'id': video_id,
-            'ext': 'flv',
             'title': title,
-            'url': video_url,
-            'app': 'rtve-live-live?ovpfv=2.1.2',
-            'player_url': player_url,
-            'rtmp_live': True,
+            'formats': formats,
+            'is_live': True,
         }

From e781ab63dba6968f54adb612907c76f6f9896095 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 1 Mar 2016 00:05:39 +0100
Subject: [PATCH 4144/6219] release 2016.03.01

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index e2836357f..adafd601b 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.02.27'
+__version__ = '2016.03.01'

From 9fb556eef0c6e83594ca4a7fe4d924a6a2a43a60 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 1 Mar 2016 08:42:33 +0800
Subject: [PATCH 4145/6219] [iqiyi] SWF URLs are not used anymore

Since automatic detection of enc_key failed

Closes #8705
---
 youtube_dl/extractor/iqiyi.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index 2b3952210..d3bee3a19 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -498,7 +498,7 @@ class IqiyiIE(InfoExtractor):
         raw_data = self._download_json(api_url, video_id)
         return raw_data
 
-    def get_enc_key(self, swf_url, video_id):
+    def get_enc_key(self, video_id):
         # TODO: automatic key extraction
         # last update at 2016-01-22 for Zombie::bite
         enc_key = '6ab6d0280511493ba85594779759d4ed'
@@ -551,11 +551,9 @@ class IqiyiIE(InfoExtractor):
             r'data-player-tvid\s*=\s*[\'"](\d+)', webpage, 'tvid')
         video_id = self._search_regex(
             r'data-player-videoid\s*=\s*[\'"]([a-f\d]+)', webpage, 'video_id')
-        swf_url = self._search_regex(
-            r'(http://[^\'"]+MainPlayer[^.]+\.swf)', webpage, 'swf player URL')
         _uuid = uuid.uuid4().hex
 
-        enc_key = self.get_enc_key(swf_url, video_id)
+        enc_key = self.get_enc_key(video_id)
 
         raw_data = self.get_raw_data(tvid, video_id, enc_key, _uuid)
 

From e7998f59aa2e204ddfef2f0eea38df1075ec0191 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 1 Mar 2016 22:59:11 +0600
Subject: [PATCH 4146/6219] [lifenews] Fix extraction and improve (Closes
 #2482, closes #8714)

---
 youtube_dl/extractor/lifenews.py | 101 +++++++++++++++++++++----------
 1 file changed, 68 insertions(+), 33 deletions(-)

diff --git a/youtube_dl/extractor/lifenews.py b/youtube_dl/extractor/lifenews.py
index f8cbca7b3..a8fd639cc 100644
--- a/youtube_dl/extractor/lifenews.py
+++ b/youtube_dl/extractor/lifenews.py
@@ -20,18 +20,18 @@ class LifeNewsIE(InfoExtractor):
     _VALID_URL = r'http://lifenews\.ru/(?:mobile/)?(?P<section>news|video)/(?P<id>\d+)'
 
     _TESTS = [{
-        'url': 'http://lifenews.ru/news/126342',
-        'md5': 'e1b50a5c5fb98a6a544250f2e0db570a',
+        # single video embedded via video/source
+        'url': 'http://lifenews.ru/news/98736',
+        'md5': '77c95eaefaca216e32a76a343ad89d23',
         'info_dict': {
-            'id': '126342',
+            'id': '98736',
             'ext': 'mp4',
-            'title': 'МВД разыскивает мужчин, оставивших в IKEA сумку с автоматом',
-            'description': 'Камеры наблюдения гипермаркета зафиксировали троих мужчин, спрятавших оружейный арсенал в камере хранения.',
-            'thumbnail': 're:http://.*\.jpg',
-            'upload_date': '20140130',
+            'title': 'Мужчина нашел дома архив оборонного завода',
+            'description': 'md5:3b06b1b39b5e2bea548e403d99b8bf26',
+            'upload_date': '20120805',
         }
     }, {
-        # video in <iframe>
+        # single video embedded via iframe
         'url': 'http://lifenews.ru/news/152125',
         'md5': '77d19a6f0886cd76bdbf44b4d971a273',
         'info_dict': {
@@ -42,15 +42,33 @@ class LifeNewsIE(InfoExtractor):
             'upload_date': '20150402',
         }
     }, {
+        # two videos embedded via iframe
         'url': 'http://lifenews.ru/news/153461',
-        'md5': '9b6ef8bc0ffa25aebc8bdb40d89ab795',
         'info_dict': {
             'id': '153461',
-            'ext': 'mp4',
             'title': 'В Москве спасли потерявшегося медвежонка, который спрятался на дереве',
             'description': 'Маленький хищник не смог найти дорогу домой и обрел временное убежище на тополе недалеко от жилого массива, пока его не нашла соседская собака.',
             'upload_date': '20150505',
-        }
+        },
+        'playlist': [{
+            'md5': '9b6ef8bc0ffa25aebc8bdb40d89ab795',
+            'info_dict': {
+                'id': '153461-video1',
+                'ext': 'mp4',
+                'title': 'В Москве спасли потерявшегося медвежонка, который спрятался на дереве (Видео 1)',
+                'description': 'Маленький хищник не смог найти дорогу домой и обрел временное убежище на тополе недалеко от жилого массива, пока его не нашла соседская собака.',
+                'upload_date': '20150505',
+            },
+        }, {
+            'md5': 'ebb3bf3b1ce40e878d0d628e93eb0322',
+            'info_dict': {
+                'id': '153461-video2',
+                'ext': 'mp4',
+                'title': 'В Москве спасли потерявшегося медвежонка, который спрятался на дереве (Видео 2)',
+                'description': 'Маленький хищник не смог найти дорогу домой и обрел временное убежище на тополе недалеко от жилого массива, пока его не нашла соседская собака.',
+                'upload_date': '20150505',
+            },
+        }],
     }, {
         'url': 'http://lifenews.ru/video/13035',
         'only_matching': True,
@@ -65,10 +83,14 @@ class LifeNewsIE(InfoExtractor):
             'http://lifenews.ru/%s/%s' % (section, video_id),
             video_id, 'Downloading page')
 
-        videos = re.findall(r'<video.*?poster="(?P<poster>[^"]+)".*?src="(?P<video>[^"]+)".*?></video>', webpage)
-        iframe_link = self._html_search_regex(
-            '<iframe[^>]+src=["\']([^"\']+)["\']', webpage, 'iframe link', default=None)
-        if not videos and not iframe_link:
+        video_urls = re.findall(
+            r'<video[^>]+><source[^>]+src=["\'](.+?)["\']', webpage)
+
+        iframe_links = re.findall(
+            r'<iframe[^>]+src=["\']((?:https?:)?//embed\.life\.ru/embed/.+?)["\']',
+            webpage)
+
+        if not video_urls and not iframe_links:
             raise ExtractorError('No media links available for %s' % video_id)
 
         title = remove_end(
@@ -95,31 +117,44 @@ class LifeNewsIE(InfoExtractor):
             'upload_date': upload_date,
         }
 
-        def make_entry(video_id, media, video_number=None):
+        def make_entry(video_id, video_url, index=None):
             cur_info = dict(common_info)
             cur_info.update({
-                'id': video_id,
-                'url': media[1],
-                'thumbnail': media[0],
-                'title': title if video_number is None else '%s-video%s' % (title, video_number),
+                'id': video_id if not index else '%s-video%s' % (video_id, index),
+                'url': video_url,
+                'title': title if not index else '%s (Видео %s)' % (title, index),
             })
             return cur_info
 
-        if iframe_link:
-            iframe_link = self._proto_relative_url(iframe_link, 'http:')
-            cur_info = dict(common_info)
-            cur_info.update({
-                '_type': 'url_transparent',
-                'id': video_id,
-                'title': title,
-                'url': iframe_link,
-            })
+        def make_video_entry(video_id, video_url, index=None):
+            video_url = compat_urlparse.urljoin(url, video_url)
+            return make_entry(video_id, video_url, index)
+
+        def make_iframe_entry(video_id, video_url, index=None):
+            video_url = self._proto_relative_url(video_url, 'http:')
+            cur_info = make_entry(video_id, video_url, index)
+            cur_info['_type'] = 'url_transparent'
             return cur_info
 
-        if len(videos) == 1:
-            return make_entry(video_id, videos[0])
-        else:
-            return [make_entry(video_id, media, video_number + 1) for video_number, media in enumerate(videos)]
+        if len(video_urls) == 1 and not iframe_links:
+            return make_video_entry(video_id, video_urls[0])
+
+        if len(iframe_links) == 1 and not video_urls:
+            return make_iframe_entry(video_id, iframe_links[0])
+
+        entries = []
+
+        if video_urls:
+            for num, video_url in enumerate(video_urls, 1):
+                entries.append(make_video_entry(video_id, video_url, num))
+
+        if iframe_links:
+            for num, iframe_link in enumerate(iframe_links, len(video_urls) + 1):
+                entries.append(make_iframe_entry(video_id, iframe_link, num))
+
+        playlist = common_info.copy()
+        playlist.update(self.playlist_result(entries, video_id, title, description))
+        return playlist
 
 
 class LifeEmbedIE(InfoExtractor):

From 66ca2cfddd840e775f0b810c393a91c55a135c20 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 1 Mar 2016 23:26:53 +0600
Subject: [PATCH 4147/6219] [wistia] Fix extraction (Closes #8707)

---
 youtube_dl/extractor/wistia.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/wistia.py b/youtube_dl/extractor/wistia.py
index fdb16d91c..41061dd31 100644
--- a/youtube_dl/extractor/wistia.py
+++ b/youtube_dl/extractor/wistia.py
@@ -35,7 +35,8 @@ class WistiaIE(InfoExtractor):
 
         formats = []
         thumbnails = []
-        for atype, a in data['assets'].items():
+        for a in data['assets']:
+            atype = a.get('type')
             if atype == 'still':
                 thumbnails.append({
                     'url': a['url'],

From 90f794c6c3ec685ad1ec3b92fba3fc0be6322c10 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 1 Mar 2016 23:41:23 +0600
Subject: [PATCH 4148/6219] [options] Add --no-mark-watched (#5054)

---
 youtube_dl/options.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 048dee881..9dd7a8034 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -174,6 +174,10 @@ def parseOpts(overrideArguments=None):
         '--mark-watched',
         action='store_true', dest='mark_watched', default=False,
         help='Mark videos watched (YouTube only)')
+    general.add_option(
+        '--no-mark-watched',
+        action='store_false', dest='mark_watched', default=False,
+        help='Do not mark videos watched (YouTube only)')
     general.add_option(
         '--no-color', '--no-colors',
         action='store_true', dest='no_color',

From f17f86513eaa09c2945b4a48a28efe5152be7017 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 1 Mar 2016 21:08:50 +0100
Subject: [PATCH 4149/6219] Add fixup for media files produced by HlsNative
 downloader(fixes #4776)

---
 youtube_dl/YoutubeDL.py              | 17 +++++++++++++++++
 youtube_dl/postprocessor/__init__.py |  2 ++
 youtube_dl/postprocessor/ffmpeg.py   | 19 +++++++++++++++----
 3 files changed, 34 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index f4324039c..97cf31eb2 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -87,6 +87,7 @@ from .extractor import get_info_extractor, gen_extractors
 from .downloader import get_suitable_downloader
 from .downloader.rtmp import rtmpdump_version
 from .postprocessor import (
+    FFmpegFixupM3u8PP,
     FFmpegFixupM4aPP,
     FFmpegFixupStretchedPP,
     FFmpegMergerPP,
@@ -1670,6 +1671,22 @@ class YoutubeDL(object):
                     else:
                         assert fixup_policy in ('ignore', 'never')
 
+                if info_dict.get('protocol') == 'm3u8_native' or info_dict.get('protocol') == 'm3u8' and self._downloader.params.get('hls_prefer_native', False):
+                    if fixup_policy == 'warn':
+                        self.report_warning('%s: malformated aac bitstream.' % (
+                            info_dict['id']))
+                    elif fixup_policy == 'detect_or_warn':
+                        fixup_pp = FFmpegFixupM3u8PP(self)
+                        if fixup_pp.available:
+                            info_dict.setdefault('__postprocessors', [])
+                            info_dict['__postprocessors'].append(fixup_pp)
+                        else:
+                            self.report_warning(
+                                '%s: malformated aac bitstream. Install ffmpeg or avconv to fix this automatically.' % (
+                                    info_dict['id']))
+                    else:
+                        assert fixup_policy in ('ignore', 'never')
+
                 try:
                     self.post_process(filename, info_dict)
                 except (PostProcessingError) as err:
diff --git a/youtube_dl/postprocessor/__init__.py b/youtube_dl/postprocessor/__init__.py
index 0d8ef6ca2..3ea518399 100644
--- a/youtube_dl/postprocessor/__init__.py
+++ b/youtube_dl/postprocessor/__init__.py
@@ -6,6 +6,7 @@ from .ffmpeg import (
     FFmpegEmbedSubtitlePP,
     FFmpegExtractAudioPP,
     FFmpegFixupStretchedPP,
+    FFmpegFixupM3u8PP,
     FFmpegFixupM4aPP,
     FFmpegMergerPP,
     FFmpegMetadataPP,
@@ -26,6 +27,7 @@ __all__ = [
     'ExecAfterDownloadPP',
     'FFmpegEmbedSubtitlePP',
     'FFmpegExtractAudioPP',
+    'FFmpegFixupM3u8PP',
     'FFmpegFixupM4aPP',
     'FFmpegFixupStretchedPP',
     'FFmpegMergerPP',
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 380bc6f29..81102f9bb 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -391,10 +391,6 @@ class FFmpegMetadataPP(FFmpegPostProcessor):
         for (name, value) in metadata.items():
             options.extend(['-metadata', '%s=%s' % (name, value)])
 
-        # https://github.com/rg3/youtube-dl/issues/8350
-        if info.get('protocol') == 'm3u8_native' or info.get('protocol') == 'm3u8' and self._downloader.params.get('hls_prefer_native', False):
-            options.extend(['-bsf:a', 'aac_adtstoasc'])
-
         self._downloader.to_screen('[ffmpeg] Adding metadata to \'%s\'' % filename)
         self.run_ffmpeg(filename, temp_filename, options)
         os.remove(encodeFilename(filename))
@@ -467,6 +463,21 @@ class FFmpegFixupM4aPP(FFmpegPostProcessor):
         return [], info
 
 
+class FFmpegFixupM3u8PP(FFmpegPostProcessor):
+    def run(self, info):
+        filename = info['filepath']
+        temp_filename = prepend_extension(filename, 'temp')
+
+        options = ['-c', 'copy', '-f', 'mp4', '-bsf:a', 'aac_adtstoasc']
+        self._downloader.to_screen('[ffmpeg] Fixing malformated aac bitstream in "%s"' % filename)
+        self.run_ffmpeg(filename, temp_filename, options)
+
+        os.remove(encodeFilename(filename))
+        os.rename(encodeFilename(temp_filename), encodeFilename(filename))
+
+        return [], info
+
+
 class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor):
     def __init__(self, downloader=None, format=None):
         super(FFmpegSubtitlesConvertorPP, self).__init__(downloader)

From 3099b3127684d1386101e86005feb18c29136a81 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 2 Mar 2016 20:52:06 +0600
Subject: [PATCH 4150/6219] [nrk:skole] Add extractor (Closes #8728)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/nrk.py      | 30 +++++++++++++++++++++++++++++-
 2 files changed, 30 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 5817140c0..b3bf73963 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -506,6 +506,7 @@ from .npr import NprIE
 from .nrk import (
     NRKIE,
     NRKPlaylistIE,
+    NRKSkoleIE,
     NRKTVIE,
 )
 from .ntvde import NTVDeIE
diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index a126f5054..4ede1a362 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -87,7 +87,7 @@ class NRKIE(InfoExtractor):
 
 
 class NRKPlaylistIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?nrk\.no/(?!video)(?:[^/]+/)+(?P<id>[^/]+)'
+    _VALID_URL = r'https?://(?:www\.)?nrk\.no/(?!video|skole)(?:[^/]+/)+(?P<id>[^/]+)'
 
     _TESTS = [{
         'url': 'http://www.nrk.no/troms/gjenopplev-den-historiske-solformorkelsen-1.12270763',
@@ -126,6 +126,34 @@ class NRKPlaylistIE(InfoExtractor):
             entries, playlist_id, playlist_title, playlist_description)
 
 
+class NRKSkoleIE(InfoExtractor):
+    IE_DESC = 'NRK Skole'
+    _VALID_URL = r'https?://(?:www\.)?nrk\.no/skole/klippdetalj?.*\btopic=nrk(?::|%3[Aa])klipp(?:/|%2[Ff])(?P<id>\d+)'
+
+    _TESTS = [{
+        'url': 'http://nrk.no/skole/klippdetalj?topic=nrk:klipp/616532',
+        'md5': '04cd85877cc1913bce73c5d28a47e00f',
+        'info_dict': {
+            'id': '6021',
+            'ext': 'flv',
+            'title': 'Genetikk og eneggede tvillinger',
+            'description': 'md5:3aca25dcf38ec30f0363428d2b265f8d',
+            'duration': 399,
+        },
+    }, {
+        'url': 'http://www.nrk.no/skole/klippdetalj?topic=nrk%3Aklipp%2F616532#embed',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        nrk_id = self._search_regex(r'data-nrk-id=["\'](\d+)', webpage, 'nrk id')
+        return self.url_result('nrk:%s' % nrk_id)
+
+
 class NRKTVIE(InfoExtractor):
     IE_DESC = 'NRK TV and NRK Radio'
     _VALID_URL = r'(?P<baseurl>https?://(?:tv|radio)\.nrk(?:super)?\.no/)(?:serie/[^/]+|program)/(?P<id>[a-zA-Z]{4}\d{8})(?:/\d{2}-\d{2}-\d{4})?(?:#del=(?P<part_id>\d+))?'

From 61140904184f3fe84eea21e698c64d6b4fe7c889 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 2 Mar 2016 20:57:04 +0600
Subject: [PATCH 4151/6219] [nrk:skole] Relax _VALID_URL

---
 youtube_dl/extractor/nrk.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index 4ede1a362..3b21fbd4d 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -4,7 +4,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urlparse
+from ..compat import (
+    compat_urlparse,
+    compat_urllib_parse_unquote,
+)
 from ..utils import (
     determine_ext,
     ExtractorError,
@@ -128,7 +131,7 @@ class NRKPlaylistIE(InfoExtractor):
 
 class NRKSkoleIE(InfoExtractor):
     IE_DESC = 'NRK Skole'
-    _VALID_URL = r'https?://(?:www\.)?nrk\.no/skole/klippdetalj?.*\btopic=nrk(?::|%3[Aa])klipp(?:/|%2[Ff])(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?nrk\.no/skole/klippdetalj?.*\btopic=(?P<id>[^/?#&]+)'
 
     _TESTS = [{
         'url': 'http://nrk.no/skole/klippdetalj?topic=nrk:klipp/616532',
@@ -143,10 +146,13 @@ class NRKSkoleIE(InfoExtractor):
     }, {
         'url': 'http://www.nrk.no/skole/klippdetalj?topic=nrk%3Aklipp%2F616532#embed',
         'only_matching': True,
+    }, {
+        'url': 'http://www.nrk.no/skole/klippdetalj?topic=urn:x-mediadb:21379',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        video_id = compat_urllib_parse_unquote(self._match_id(url))
 
         webpage = self._download_webpage(url, video_id)
 

From 82156fdbf0913c75181484dcc813565713bf78e9 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 2 Mar 2016 14:23:17 +0100
Subject: [PATCH 4152/6219] [youtube] extract width and height from fmt_list

---
 youtube_dl/extractor/youtube.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index ba339b67d..56e6bd096 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1399,6 +1399,16 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             encoded_url_map = video_info.get('url_encoded_fmt_stream_map', [''])[0] + ',' + video_info.get('adaptive_fmts', [''])[0]
             if 'rtmpe%3Dyes' in encoded_url_map:
                 raise ExtractorError('rtmpe downloads are not supported, see https://github.com/rg3/youtube-dl/issues/343 for more information.', expected=True)
+            fmt_list = video_info.get('fmt_list', [''])[0]
+            if fmt_list:
+                for fmt in fmt_list.split(','):
+                    spec = fmt.split('/')
+                    width, height = spec[1].split('x')
+                    self._formats[spec[0]].update({
+                        'resolution': spec[1],
+                        'width': int_or_none(width),
+                        'height': int_or_none(height),
+                    })
             formats = []
             for url_data_str in encoded_url_map.split(','):
                 url_data = compat_parse_qs(url_data_str)

From 4a5ba28a87a1a1632e58a1de404eb1fa268118a3 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 2 Mar 2016 15:53:09 +0100
Subject: [PATCH 4153/6219] [youtube] add tbr to _formats extracted from
 watch_as3.swf

---
 youtube_dl/extractor/youtube.py | 71 +++++++++++++++++----------------
 1 file changed, 36 insertions(+), 35 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 56e6bd096..5356aaeb1 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -278,62 +278,63 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                      (?(1).+)?                                                # if we found the ID, everything can follow
                      $"""
     _NEXT_URL_RE = r'[\?&]next_url=([^&]+)'
+    # tbr was extracted from com/google/youtube/model/VideoFormat.as in watch_as3.swf and converted from Bytes/S to KBits/S
     _formats = {
-        '5': {'ext': 'flv', 'width': 400, 'height': 240, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
-        '6': {'ext': 'flv', 'width': 450, 'height': 270, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
-        '13': {'ext': '3gp', 'acodec': 'aac', 'vcodec': 'mp4v'},
-        '17': {'ext': '3gp', 'width': 176, 'height': 144, 'acodec': 'aac', 'abr': 24, 'vcodec': 'mp4v'},
-        '18': {'ext': 'mp4', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 96, 'vcodec': 'h264'},
-        '22': {'ext': 'mp4', 'width': 1280, 'height': 720, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
-        '34': {'ext': 'flv', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
-        '35': {'ext': 'flv', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
+        '5': {'ext': 'flv', 'width': 400, 'height': 240, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263', 'tbr': 320},
+        '6': {'ext': 'flv', 'width': 450, 'height': 270, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263', 'tbr': 896},
+        '13': {'ext': '3gp', 'acodec': 'aac', 'vcodec': 'mp4v', 'tbr': 60},
+        '17': {'ext': '3gp', 'width': 176, 'height': 144, 'acodec': 'aac', 'abr': 24, 'vcodec': 'mp4v', 'tbr': 80},
+        '18': {'ext': 'mp4', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 96, 'vcodec': 'h264', 'tbr': 736},
+        '22': {'ext': 'mp4', 'width': 1280, 'height': 720, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'tbr': 3192},
+        '34': {'ext': 'flv', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'tbr': 928},
+        '35': {'ext': 'flv', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'tbr': 1280},
         # itag 36 videos are either 320x180 (BaW_jenozKc) or 320x240 (__2ABJjxzNo), abr varies as well
-        '36': {'ext': '3gp', 'width': 320, 'acodec': 'aac', 'vcodec': 'mp4v'},
-        '37': {'ext': 'mp4', 'width': 1920, 'height': 1080, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
-        '38': {'ext': 'mp4', 'width': 4096, 'height': 3072, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
-        '43': {'ext': 'webm', 'width': 640, 'height': 360, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
-        '44': {'ext': 'webm', 'width': 854, 'height': 480, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
-        '45': {'ext': 'webm', 'width': 1280, 'height': 720, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
+        '36': {'ext': '3gp', 'width': 320, 'acodec': 'aac', 'vcodec': 'mp4v', 'tbr': 256},
+        '37': {'ext': 'mp4', 'width': 1920, 'height': 1080, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'tbr': 6192},
+        '38': {'ext': 'mp4', 'width': 4096, 'height': 3072, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'tbr': 10128},
+        '43': {'ext': 'webm', 'width': 640, 'height': 360, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8', 'tbr': 928},
+        '44': {'ext': 'webm', 'width': 854, 'height': 480, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8', 'tbr': 1280},
+        '45': {'ext': 'webm', 'width': 1280, 'height': 720, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'tbr': 3192},
         '46': {'ext': 'webm', 'width': 1920, 'height': 1080, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
-        '59': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
-        '78': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
+        '59': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'tbr': 1280},
+        '78': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'tbr': 1280},
 
 
         # 3D videos
-        '82': {'ext': 'mp4', 'height': 360, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
-        '83': {'ext': 'mp4', 'height': 480, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
-        '84': {'ext': 'mp4', 'height': 720, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
-        '85': {'ext': 'mp4', 'height': 1080, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
+        '82': {'ext': 'mp4', 'height': 360, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20, 'tbr': 800},
+        '83': {'ext': 'mp4', 'height': 480, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20, 'tbr': 1152},
+        '84': {'ext': 'mp4', 'height': 720, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20, 'tbr': 3000},
+        '85': {'ext': 'mp4', 'height': 1080, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20, 'tbr': 6000},
         '100': {'ext': 'webm', 'height': 360, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8', 'preference': -20},
         '101': {'ext': 'webm', 'height': 480, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
         '102': {'ext': 'webm', 'height': 720, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
 
         # Apple HTTP Live Streaming
-        '92': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
-        '93': {'ext': 'mp4', 'height': 360, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
-        '94': {'ext': 'mp4', 'height': 480, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
-        '95': {'ext': 'mp4', 'height': 720, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
-        '96': {'ext': 'mp4', 'height': 1080, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
+        '92': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10, 'tbr': 186.625},
+        '93': {'ext': 'mp4', 'height': 360, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10, 'tbr': 951.5625},
+        '94': {'ext': 'mp4', 'height': 480, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10, 'tbr': 1312.5},
+        '95': {'ext': 'mp4', 'height': 720, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10, 'tbr': 3207.421875},
+        '96': {'ext': 'mp4', 'height': 1080, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10, 'tbr': 6349.21875},
         '132': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
         '151': {'ext': 'mp4', 'height': 72, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 24, 'vcodec': 'h264', 'preference': -10},
 
         # DASH mp4 video
-        '133': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
-        '134': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
-        '135': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
-        '136': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
-        '137': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
-        '138': {'ext': 'mp4', 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},  # Height can vary (https://github.com/rg3/youtube-dl/issues/4559)
-        '160': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
+        '133': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40, 'tbr': 261.71875},
+        '134': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40, 'tbr': 951.5625},
+        '135': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40, 'tbr': 1312.5},
+        '136': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40, 'tbr': 3207.421875},
+        '137': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40, 'tbr': 6349.21875},
+        '138': {'ext': 'mp4', 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40, 'tbr': 10128.0},  # Height can vary (https://github.com/rg3/youtube-dl/issues/4559)
+        '160': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40, 'tbr': 91.796875},
         '264': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
         '298': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60, 'preference': -40},
         '299': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60, 'preference': -40},
         '266': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
 
         # Dash mp4 audio
-        '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 48, 'preference': -50, 'container': 'm4a_dash'},
-        '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 128, 'preference': -50, 'container': 'm4a_dash'},
-        '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 256, 'preference': -50, 'container': 'm4a_dash'},
+        '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 48, 'preference': -50, 'container': 'm4a_dash', 'tbr': 32},
+        '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 128, 'preference': -50, 'container': 'm4a_dash', 'tbr': 128},
+        '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 256, 'preference': -50, 'container': 'm4a_dash', 'tbr': 320},
 
         # Dash webm
         '167': {'ext': 'webm', 'height': 360, 'width': 640, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},

From 85ca019d9679d75c26b7ee4106dd7d98136f3c88 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 2 Mar 2016 16:02:48 +0100
Subject: [PATCH 4154/6219] [youtube] add basic info for some unknown formats
 extracted from watch_as3.swf

---
 youtube_dl/extractor/youtube.py | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 5356aaeb1..d42f5c17f 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -310,11 +310,13 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         '102': {'ext': 'webm', 'height': 720, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
 
         # Apple HTTP Live Streaming
+        '91': {'format_note': 'HLS', 'tbr': 98.4375},
         '92': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10, 'tbr': 186.625},
         '93': {'ext': 'mp4', 'height': 360, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10, 'tbr': 951.5625},
         '94': {'ext': 'mp4', 'height': 480, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10, 'tbr': 1312.5},
         '95': {'ext': 'mp4', 'height': 720, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10, 'tbr': 3207.421875},
         '96': {'ext': 'mp4', 'height': 1080, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10, 'tbr': 6349.21875},
+        '97': {'format_note': 'HLS', 'tbr': 10128},
         '132': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
         '151': {'ext': 'mp4', 'height': 72, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 24, 'vcodec': 'h264', 'preference': -10},
 
@@ -371,6 +373,25 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
 
         # RTMP (unnamed)
         '_rtmp': {'protocol': 'rtmp'},
+
+        # formats extracted from com/google/youtube/model/VideoFormat.as in watch_as3.swf
+        '20': {'tbr': 24000},
+        '23': {'tbr': 62.5},
+        '24': {'tbr': 125},
+        '25': {'tbr': 312.5},
+        '33': {'tbr': 132},
+        '40': {'tbr': 40},
+        '61': {'tbr': 928},
+        '62': {'tbr': 1280},
+        '63': {'tbr': 3192},
+        '64': {'tbr': 6192},
+        '65': {'tbr': 10128},
+        '81': {'ext': 'mp4', 'tbr': 928},
+        '88': {'tbr': 7192},
+        '98': {'tbr': 320},
+        '119': {'ext': 'mp4', 'tbr': 320},
+        '304': {'format_note': 'DASH'},
+        '305': {'format_note': 'DASH'},
     }
     _SUBTITLE_FORMATS = ('ttml', 'vtt')
 

From e38cafe986994d65230e6518752def8b53ad7308 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 2 Mar 2016 21:11:18 +0600
Subject: [PATCH 4155/6219] [YoutubeDL] Skip postprocessing and archive report
 when outputting to stdout (Closes #8729)

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index f4324039c..8f3a8b9e3 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1631,7 +1631,7 @@ class YoutubeDL(object):
                 self.report_error('content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
                 return
 
-            if success:
+            if success and filename != '-':
                 # Fixup content
                 fixup_policy = self.params.get('fixup')
                 if fixup_policy is None:

From c48dd4400f2aae391a640b13133ae2e24fb92fba Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 2 Mar 2016 17:34:56 +0100
Subject: [PATCH 4156/6219] Revert "[youtube] add basic info for some unknown
 formats extracted from watch_as3.swf"

This reverts commit 85ca019d9679d75c26b7ee4106dd7d98136f3c88.
---
 youtube_dl/extractor/youtube.py | 21 ---------------------
 1 file changed, 21 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index d42f5c17f..5356aaeb1 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -310,13 +310,11 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         '102': {'ext': 'webm', 'height': 720, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
 
         # Apple HTTP Live Streaming
-        '91': {'format_note': 'HLS', 'tbr': 98.4375},
         '92': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10, 'tbr': 186.625},
         '93': {'ext': 'mp4', 'height': 360, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10, 'tbr': 951.5625},
         '94': {'ext': 'mp4', 'height': 480, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10, 'tbr': 1312.5},
         '95': {'ext': 'mp4', 'height': 720, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10, 'tbr': 3207.421875},
         '96': {'ext': 'mp4', 'height': 1080, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10, 'tbr': 6349.21875},
-        '97': {'format_note': 'HLS', 'tbr': 10128},
         '132': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
         '151': {'ext': 'mp4', 'height': 72, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 24, 'vcodec': 'h264', 'preference': -10},
 
@@ -373,25 +371,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
 
         # RTMP (unnamed)
         '_rtmp': {'protocol': 'rtmp'},
-
-        # formats extracted from com/google/youtube/model/VideoFormat.as in watch_as3.swf
-        '20': {'tbr': 24000},
-        '23': {'tbr': 62.5},
-        '24': {'tbr': 125},
-        '25': {'tbr': 312.5},
-        '33': {'tbr': 132},
-        '40': {'tbr': 40},
-        '61': {'tbr': 928},
-        '62': {'tbr': 1280},
-        '63': {'tbr': 3192},
-        '64': {'tbr': 6192},
-        '65': {'tbr': 10128},
-        '81': {'ext': 'mp4', 'tbr': 928},
-        '88': {'tbr': 7192},
-        '98': {'tbr': 320},
-        '119': {'ext': 'mp4', 'tbr': 320},
-        '304': {'format_note': 'DASH'},
-        '305': {'format_note': 'DASH'},
     }
     _SUBTITLE_FORMATS = ('ttml', 'vtt')
 

From c2d3cb4c63495ef9f825efe67ee61d77d6fab9c0 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 2 Mar 2016 17:35:04 +0100
Subject: [PATCH 4157/6219] Revert "[youtube] add tbr to _formats extracted
 from watch_as3.swf"

This reverts commit 4a5ba28a87a1a1632e58a1de404eb1fa268118a3.
---
 youtube_dl/extractor/youtube.py | 71 ++++++++++++++++-----------------
 1 file changed, 35 insertions(+), 36 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 5356aaeb1..56e6bd096 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -278,63 +278,62 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                      (?(1).+)?                                                # if we found the ID, everything can follow
                      $"""
     _NEXT_URL_RE = r'[\?&]next_url=([^&]+)'
-    # tbr was extracted from com/google/youtube/model/VideoFormat.as in watch_as3.swf and converted from Bytes/S to KBits/S
     _formats = {
-        '5': {'ext': 'flv', 'width': 400, 'height': 240, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263', 'tbr': 320},
-        '6': {'ext': 'flv', 'width': 450, 'height': 270, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263', 'tbr': 896},
-        '13': {'ext': '3gp', 'acodec': 'aac', 'vcodec': 'mp4v', 'tbr': 60},
-        '17': {'ext': '3gp', 'width': 176, 'height': 144, 'acodec': 'aac', 'abr': 24, 'vcodec': 'mp4v', 'tbr': 80},
-        '18': {'ext': 'mp4', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 96, 'vcodec': 'h264', 'tbr': 736},
-        '22': {'ext': 'mp4', 'width': 1280, 'height': 720, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'tbr': 3192},
-        '34': {'ext': 'flv', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'tbr': 928},
-        '35': {'ext': 'flv', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'tbr': 1280},
+        '5': {'ext': 'flv', 'width': 400, 'height': 240, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
+        '6': {'ext': 'flv', 'width': 450, 'height': 270, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
+        '13': {'ext': '3gp', 'acodec': 'aac', 'vcodec': 'mp4v'},
+        '17': {'ext': '3gp', 'width': 176, 'height': 144, 'acodec': 'aac', 'abr': 24, 'vcodec': 'mp4v'},
+        '18': {'ext': 'mp4', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 96, 'vcodec': 'h264'},
+        '22': {'ext': 'mp4', 'width': 1280, 'height': 720, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
+        '34': {'ext': 'flv', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
+        '35': {'ext': 'flv', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
         # itag 36 videos are either 320x180 (BaW_jenozKc) or 320x240 (__2ABJjxzNo), abr varies as well
-        '36': {'ext': '3gp', 'width': 320, 'acodec': 'aac', 'vcodec': 'mp4v', 'tbr': 256},
-        '37': {'ext': 'mp4', 'width': 1920, 'height': 1080, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'tbr': 6192},
-        '38': {'ext': 'mp4', 'width': 4096, 'height': 3072, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'tbr': 10128},
-        '43': {'ext': 'webm', 'width': 640, 'height': 360, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8', 'tbr': 928},
-        '44': {'ext': 'webm', 'width': 854, 'height': 480, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8', 'tbr': 1280},
-        '45': {'ext': 'webm', 'width': 1280, 'height': 720, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'tbr': 3192},
+        '36': {'ext': '3gp', 'width': 320, 'acodec': 'aac', 'vcodec': 'mp4v'},
+        '37': {'ext': 'mp4', 'width': 1920, 'height': 1080, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
+        '38': {'ext': 'mp4', 'width': 4096, 'height': 3072, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
+        '43': {'ext': 'webm', 'width': 640, 'height': 360, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
+        '44': {'ext': 'webm', 'width': 854, 'height': 480, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
+        '45': {'ext': 'webm', 'width': 1280, 'height': 720, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
         '46': {'ext': 'webm', 'width': 1920, 'height': 1080, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
-        '59': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'tbr': 1280},
-        '78': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'tbr': 1280},
+        '59': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
+        '78': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
 
 
         # 3D videos
-        '82': {'ext': 'mp4', 'height': 360, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20, 'tbr': 800},
-        '83': {'ext': 'mp4', 'height': 480, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20, 'tbr': 1152},
-        '84': {'ext': 'mp4', 'height': 720, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20, 'tbr': 3000},
-        '85': {'ext': 'mp4', 'height': 1080, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20, 'tbr': 6000},
+        '82': {'ext': 'mp4', 'height': 360, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
+        '83': {'ext': 'mp4', 'height': 480, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
+        '84': {'ext': 'mp4', 'height': 720, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
+        '85': {'ext': 'mp4', 'height': 1080, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
         '100': {'ext': 'webm', 'height': 360, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8', 'preference': -20},
         '101': {'ext': 'webm', 'height': 480, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
         '102': {'ext': 'webm', 'height': 720, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
 
         # Apple HTTP Live Streaming
-        '92': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10, 'tbr': 186.625},
-        '93': {'ext': 'mp4', 'height': 360, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10, 'tbr': 951.5625},
-        '94': {'ext': 'mp4', 'height': 480, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10, 'tbr': 1312.5},
-        '95': {'ext': 'mp4', 'height': 720, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10, 'tbr': 3207.421875},
-        '96': {'ext': 'mp4', 'height': 1080, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10, 'tbr': 6349.21875},
+        '92': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
+        '93': {'ext': 'mp4', 'height': 360, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
+        '94': {'ext': 'mp4', 'height': 480, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
+        '95': {'ext': 'mp4', 'height': 720, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
+        '96': {'ext': 'mp4', 'height': 1080, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
         '132': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
         '151': {'ext': 'mp4', 'height': 72, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 24, 'vcodec': 'h264', 'preference': -10},
 
         # DASH mp4 video
-        '133': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40, 'tbr': 261.71875},
-        '134': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40, 'tbr': 951.5625},
-        '135': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40, 'tbr': 1312.5},
-        '136': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40, 'tbr': 3207.421875},
-        '137': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40, 'tbr': 6349.21875},
-        '138': {'ext': 'mp4', 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40, 'tbr': 10128.0},  # Height can vary (https://github.com/rg3/youtube-dl/issues/4559)
-        '160': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40, 'tbr': 91.796875},
+        '133': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
+        '134': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
+        '135': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
+        '136': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
+        '137': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
+        '138': {'ext': 'mp4', 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},  # Height can vary (https://github.com/rg3/youtube-dl/issues/4559)
+        '160': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
         '264': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
         '298': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60, 'preference': -40},
         '299': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60, 'preference': -40},
         '266': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
 
         # Dash mp4 audio
-        '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 48, 'preference': -50, 'container': 'm4a_dash', 'tbr': 32},
-        '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 128, 'preference': -50, 'container': 'm4a_dash', 'tbr': 128},
-        '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 256, 'preference': -50, 'container': 'm4a_dash', 'tbr': 320},
+        '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 48, 'preference': -50, 'container': 'm4a_dash'},
+        '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 128, 'preference': -50, 'container': 'm4a_dash'},
+        '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 256, 'preference': -50, 'container': 'm4a_dash'},
 
         # Dash webm
         '167': {'ext': 'webm', 'height': 360, 'width': 640, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},

From 3318832e9d42e85160ff2dab01d0f52ed739fbd6 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 2 Mar 2016 17:52:13 +0100
Subject: [PATCH 4158/6219] [youtube] improve width and height extraction from
 fmt_list

---
 youtube_dl/extractor/youtube.py | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 56e6bd096..e4bef8f2f 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1399,16 +1399,19 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             encoded_url_map = video_info.get('url_encoded_fmt_stream_map', [''])[0] + ',' + video_info.get('adaptive_fmts', [''])[0]
             if 'rtmpe%3Dyes' in encoded_url_map:
                 raise ExtractorError('rtmpe downloads are not supported, see https://github.com/rg3/youtube-dl/issues/343 for more information.', expected=True)
+            formats_spec = {}
             fmt_list = video_info.get('fmt_list', [''])[0]
             if fmt_list:
                 for fmt in fmt_list.split(','):
                     spec = fmt.split('/')
-                    width, height = spec[1].split('x')
-                    self._formats[spec[0]].update({
-                        'resolution': spec[1],
-                        'width': int_or_none(width),
-                        'height': int_or_none(height),
-                    })
+                    if len(spec) > 1:
+                        width_height = spec[1].split('x')
+                        if len(width_height) == 2:
+                            formats_spec[spec[0]] = {
+                                'resolution': spec[1],
+                                'width': int_or_none(width_height[0]),
+                                'height': int_or_none(width_height[1]),
+                            }
             formats = []
             for url_data_str in encoded_url_map.split(','):
                 url_data = compat_parse_qs(url_data_str)
@@ -1477,6 +1480,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 }
                 if format_id in self._formats:
                     dct.update(self._formats[format_id])
+                if format_id in formats_spec:
+                    dct.update(formats_spec[format_id])
 
                 # Some itags are not included in DASH manifest thus corresponding formats will
                 # lack metadata (see https://github.com/rg3/youtube-dl/pull/5993).

From 2bc0c46f989ce575e22502bb51f0fdcabd8baa15 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 2 Mar 2016 23:06:39 +0600
Subject: [PATCH 4159/6219] [extractor/common] Document license metafield

---
 youtube_dl/extractor/common.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index a7c700099..d84ce4525 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -157,6 +157,7 @@ class InfoExtractor(object):
     thumbnail:      Full URL to a video thumbnail image.
     description:    Full video description.
     uploader:       Full name of the video uploader.
+    license:        License name the video is licensed under.
     creator:        The main artist who created the video.
     release_date:   The date (YYYYMMDD) when the video was released.
     timestamp:      UNIX timestamp of the moment the video became available.

From 7caf9830b0ae80363d3b1817b322452474631cc1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 2 Mar 2016 23:07:25 +0600
Subject: [PATCH 4160/6219] [youtube] Extract license (Closes #8725)

---
 youtube_dl/extractor/youtube.py | 45 ++++++++++++++++++++++++++++++++-
 1 file changed, 44 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index e4bef8f2f..43fdbfab1 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -384,6 +384,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader': 'Philipp Hagemeister',
                 'uploader_id': 'phihag',
                 'upload_date': '20121002',
+                'license': 'Standard YouTube License',
                 'description': 'test chars:  "\'/\\ä↭𝕐\ntest URL: https://github.com/rg3/youtube-dl/issues/1892\n\nThis is a test video for youtube-dl.\n\nFor more information, contact phihag@phihag.de .',
                 'categories': ['Science & Technology'],
                 'tags': ['youtube-dl'],
@@ -402,12 +403,13 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'upload_date': '20120506',
                 'title': 'Icona Pop - I Love It (feat. Charli XCX) [OFFICIAL VIDEO]',
                 'alt_title': 'I Love It (feat. Charli XCX)',
-                'description': 'md5:782e8651347686cba06e58f71ab51773',
+                'description': 'md5:f3ceb5ef83a08d95b9d146f973157cc8',
                 'tags': ['Icona Pop i love it', 'sweden', 'pop music', 'big beat records', 'big beat', 'charli',
                          'xcx', 'charli xcx', 'girls', 'hbo', 'i love it', "i don't care", 'icona', 'pop',
                          'iconic ep', 'iconic', 'love', 'it'],
                 'uploader': 'Icona Pop',
                 'uploader_id': 'IconaPop',
+                'license': 'Standard YouTube License',
                 'creator': 'Icona Pop',
             }
         },
@@ -423,6 +425,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'description': 'md5:64249768eec3bc4276236606ea996373',
                 'uploader': 'justintimberlakeVEVO',
                 'uploader_id': 'justintimberlakeVEVO',
+                'license': 'Standard YouTube License',
                 'creator': 'Justin Timberlake',
                 'age_limit': 18,
             }
@@ -438,6 +441,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'description': 'md5:09b78bd971f1e3e289601dfba15ca4f7',
                 'uploader': 'SET India',
                 'uploader_id': 'setindia',
+                'license': 'Standard YouTube License',
                 'age_limit': 18,
             }
         },
@@ -451,6 +455,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader': 'Philipp Hagemeister',
                 'uploader_id': 'phihag',
                 'upload_date': '20121002',
+                'license': 'Standard YouTube License',
                 'description': 'test chars:  "\'/\\ä↭𝕐\ntest URL: https://github.com/rg3/youtube-dl/issues/1892\n\nThis is a test video for youtube-dl.\n\nFor more information, contact phihag@phihag.de .',
                 'categories': ['Science & Technology'],
                 'tags': ['youtube-dl'],
@@ -471,6 +476,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader_id': '8KVIDEO',
                 'description': '',
                 'uploader': '8KVIDEO',
+                'license': 'Standard YouTube License',
                 'title': 'UHDTV TEST 8K VIDEO.mp4'
             },
             'params': {
@@ -489,6 +495,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader': 'AfrojackVEVO',
                 'uploader_id': 'AfrojackVEVO',
                 'upload_date': '20131011',
+                'license': 'Standard YouTube License',
             },
             'params': {
                 'youtube_include_dash_manifest': True,
@@ -507,6 +514,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader': 'TaylorSwiftVEVO',
                 'uploader_id': 'TaylorSwiftVEVO',
                 'upload_date': '20140818',
+                'license': 'Standard YouTube License',
                 'creator': 'Taylor Swift',
             },
             'params': {
@@ -523,6 +531,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'upload_date': '20100909',
                 'uploader': 'The Amazing Atheist',
                 'uploader_id': 'TheAmazingAtheist',
+                'license': 'Standard YouTube License',
                 'title': 'Burning Everyone\'s Koran',
                 'description': 'SUBSCRIBE: http://www.youtube.com/saturninefilms\n\nEven Obama has taken a stand against freedom on this issue: http://www.huffingtonpost.com/2010/09/09/obama-gma-interview-quran_n_710282.html',
             }
@@ -538,6 +547,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader': 'The Witcher',
                 'uploader_id': 'WitcherGame',
                 'upload_date': '20140605',
+                'license': 'Standard YouTube License',
                 'age_limit': 18,
             },
         },
@@ -552,6 +562,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader': 'LloydVEVO',
                 'uploader_id': 'LloydVEVO',
                 'upload_date': '20110629',
+                'license': 'Standard YouTube License',
                 'age_limit': 18,
             },
         },
@@ -566,6 +577,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'creator': 'deadmau5',
                 'description': 'md5:12c56784b8032162bb936a5f76d55360',
                 'uploader': 'deadmau5',
+                'license': 'Standard YouTube License',
                 'title': 'Deadmau5 - Some Chords (HD)',
                 'alt_title': 'Some Chords',
             },
@@ -581,6 +593,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'ext': 'mp4',
                 'upload_date': '20150827',
                 'uploader_id': 'olympic',
+                'license': 'Standard YouTube License',
                 'description': 'HO09  - Women -  GER-AUS - Hockey - 31 July 2012 - London 2012 Olympic Games',
                 'uploader': 'Olympics',
                 'title': 'Hockey - Women -  GER-AUS - London 2012 Olympic Games',
@@ -600,6 +613,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader_id': 'AllenMeow',
                 'description': 'made by Wacom from Korea | 字幕&加油添醋 by TY\'s Allen | 感謝heylisa00cavey1001同學熱情提供梗及翻譯',
                 'uploader': '孫艾倫',
+                'license': 'Standard YouTube License',
                 'title': '[A-made] 變態妍字幕版 太妍 我就是這樣的人',
             },
         },
@@ -631,6 +645,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'upload_date': '20150625',
                 'uploader_id': 'dorappi2000',
                 'uploader': 'dorappi2000',
+                'license': 'Standard YouTube License',
                 'formats': 'mincount:33',
             },
         },
@@ -645,6 +660,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader': 'Airtek',
                 'description': 'Retransmisión en directo de la XVIII media maratón de Zaragoza.',
                 'uploader_id': 'UCzTzUmjXxxacNnL8I3m4LnQ',
+                'license': 'Standard YouTube License',
                 'title': 'Retransmisión XVIII Media maratón Zaragoza 2015',
             },
             'params': {
@@ -669,6 +685,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     'upload_date': '20150721',
                     'uploader': 'Beer Games Beer',
                     'uploader_id': 'beergamesbeer',
+                    'license': 'Standard YouTube License',
                 },
             }, {
                 'info_dict': {
@@ -679,6 +696,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     'upload_date': '20150721',
                     'uploader': 'Beer Games Beer',
                     'uploader_id': 'beergamesbeer',
+                    'license': 'Standard YouTube License',
                 },
             }, {
                 'info_dict': {
@@ -689,6 +707,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     'upload_date': '20150721',
                     'uploader': 'Beer Games Beer',
                     'uploader_id': 'beergamesbeer',
+                    'license': 'Standard YouTube License',
                 },
             }, {
                 'info_dict': {
@@ -699,6 +718,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     'upload_date': '20150721',
                     'uploader': 'Beer Games Beer',
                     'uploader_id': 'beergamesbeer',
+                    'license': 'Standard YouTube License',
                 },
             }],
             'params': {
@@ -733,6 +753,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'upload_date': '20151119',
                 'uploader_id': 'IronSoulElf',
                 'uploader': 'IronSoulElf',
+                'license': 'Standard YouTube License',
                 'creator': 'Todd Haberman, Daniel Law Heath & Aaron Kaplan',
             },
             'params': {
@@ -760,6 +781,23 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'skip_download': True,
             },
         },
+        {
+            # Video licensed under Creative Commons
+            'url': 'https://www.youtube.com/watch?v=M4gD1WSo5mA',
+            'info_dict': {
+                'id': 'M4gD1WSo5mA',
+                'ext': 'mp4',
+                'title': 'md5:e41008789470fc2533a3252216f1c1d1',
+                'description': 'md5:a677553cf0840649b731a3024aeff4cc',
+                'upload_date': '20150127',
+                'uploader_id': 'BerkmanCenter',
+                'uploader': 'BerkmanCenter',
+                'license': 'Creative Commons Attribution license (reuse allowed)',
+            },
+            'params': {
+                'skip_download': True,
+            },
+        },
         {
             'url': 'https://www.youtube.com/watch?feature=player_embedded&amp;amp;v=V36LpHqtcDY',
             'only_matching': True,
@@ -1326,6 +1364,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 upload_date = ' '.join(re.sub(r'[/,-]', r' ', mobj.group(1)).split())
         upload_date = unified_strdate(upload_date)
 
+        video_license = self._html_search_regex(
+            r'<h4[^>]+class="title"[^>]*>\s*License\s*</h4>\s*<ul[^>]*>\s*<li>(.+?)</li',
+            video_webpage, 'license', default=None)
+
         m_music = re.search(
             r'<h4[^>]+class="title"[^>]*>\s*Music\s*</h4>\s*<ul[^>]*>\s*<li>(?P<title>.+?) by (?P<creator>.+?)(?:\(.+?\))?</li',
             video_webpage)
@@ -1601,6 +1643,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             'uploader': video_uploader,
             'uploader_id': video_uploader_id,
             'upload_date': upload_date,
+            'license': video_license,
             'creator': video_creator,
             'title': video_title,
             'alt_title': video_alt_title,

From 47462a125bd47cb8adc06caeaf46e5692d841395 Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Wed, 2 Mar 2016 23:10:01 +0600
Subject: [PATCH 4161/6219] [README.md] Document license field for output
 template

---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index 44b65da02..d66804a7a 100644
--- a/README.md
+++ b/README.md
@@ -458,6 +458,7 @@ The basic usage is not to set any template arguments when downloading a single f
  - `alt_title`: A secondary title of the video
  - `display_id`: An alternative identifier for the video
  - `uploader`: Full name of the video uploader
+ - `license`: License name the video is licensed under
  - `creator`: The main artist who created the video
  - `release_date`: The date (YYYYMMDD) when the video was released
  - `timestamp`: UNIX timestamp of the moment the video became available

From 7bcd2830dd906c7a83b31d3ba8c8b10d4e7756af Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 2 Mar 2016 23:31:24 +0600
Subject: [PATCH 4162/6219] [extractor/common] Document uploader_url

---
 youtube_dl/extractor/common.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index d84ce4525..402f2f436 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -164,6 +164,7 @@ class InfoExtractor(object):
     upload_date:    Video upload date (YYYYMMDD).
                     If not explicitly set, calculated from timestamp.
     uploader_id:    Nickname or id of the video uploader.
+    uploader_url:   Full URL to a personal webpage of the video uploader.
     location:       Physical location where the video was filmed.
     subtitles:      The available subtitles as a dictionary in the format
                     {language: subformats}. "subformats" is a list sorted from

From fd050249afce1bcc9e7f4a127069375467007b55 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 2 Mar 2016 23:49:10 +0600
Subject: [PATCH 4163/6219] [youtube] Extract uploader_url (Closes #8724)

---
 youtube_dl/extractor/youtube.py | 46 +++++++++++++++++++++++++++++++--
 1 file changed, 44 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 43fdbfab1..27e67feb4 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -383,6 +383,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
                 'uploader': 'Philipp Hagemeister',
                 'uploader_id': 'phihag',
+                'uploader_url': 're:https?://(?:www\.)?youtube\.com/user/phihag',
                 'upload_date': '20121002',
                 'license': 'Standard YouTube License',
                 'description': 'test chars:  "\'/\\ä↭𝕐\ntest URL: https://github.com/rg3/youtube-dl/issues/1892\n\nThis is a test video for youtube-dl.\n\nFor more information, contact phihag@phihag.de .',
@@ -409,6 +410,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                          'iconic ep', 'iconic', 'love', 'it'],
                 'uploader': 'Icona Pop',
                 'uploader_id': 'IconaPop',
+                'uploader_url': 're:https?://(?:www\.)?youtube\.com/user/IconaPop',
                 'license': 'Standard YouTube License',
                 'creator': 'Icona Pop',
             }
@@ -425,6 +427,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'description': 'md5:64249768eec3bc4276236606ea996373',
                 'uploader': 'justintimberlakeVEVO',
                 'uploader_id': 'justintimberlakeVEVO',
+                'uploader_url': 're:https?://(?:www\.)?youtube\.com/user/justintimberlakeVEVO',
                 'license': 'Standard YouTube License',
                 'creator': 'Justin Timberlake',
                 'age_limit': 18,
@@ -441,6 +444,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'description': 'md5:09b78bd971f1e3e289601dfba15ca4f7',
                 'uploader': 'SET India',
                 'uploader_id': 'setindia',
+                'uploader_url': 're:https?://(?:www\.)?youtube\.com/user/setindia',
                 'license': 'Standard YouTube License',
                 'age_limit': 18,
             }
@@ -454,6 +458,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
                 'uploader': 'Philipp Hagemeister',
                 'uploader_id': 'phihag',
+                'uploader_url': 're:https?://(?:www\.)?youtube\.com/user/phihag',
                 'upload_date': '20121002',
                 'license': 'Standard YouTube License',
                 'description': 'test chars:  "\'/\\ä↭𝕐\ntest URL: https://github.com/rg3/youtube-dl/issues/1892\n\nThis is a test video for youtube-dl.\n\nFor more information, contact phihag@phihag.de .',
@@ -474,6 +479,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'ext': 'm4a',
                 'upload_date': '20121002',
                 'uploader_id': '8KVIDEO',
+                'uploader_url': 're:https?://(?:www\.)?youtube\.com/user/8KVIDEO',
                 'description': '',
                 'uploader': '8KVIDEO',
                 'license': 'Standard YouTube License',
@@ -531,6 +537,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'upload_date': '20100909',
                 'uploader': 'The Amazing Atheist',
                 'uploader_id': 'TheAmazingAtheist',
+                'uploader_url': 're:https?://(?:www\.)?youtube\.com/user/TheAmazingAtheist',
                 'license': 'Standard YouTube License',
                 'title': 'Burning Everyone\'s Koran',
                 'description': 'SUBSCRIBE: http://www.youtube.com/saturninefilms\n\nEven Obama has taken a stand against freedom on this issue: http://www.huffingtonpost.com/2010/09/09/obama-gma-interview-quran_n_710282.html',
@@ -546,6 +553,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'description': 're:(?s).{100,}About the Game\n.*?The Witcher 3: Wild Hunt.{100,}',
                 'uploader': 'The Witcher',
                 'uploader_id': 'WitcherGame',
+                'uploader_url': 're:https?://(?:www\.)?youtube\.com/user/WitcherGame',
                 'upload_date': '20140605',
                 'license': 'Standard YouTube License',
                 'age_limit': 18,
@@ -561,6 +569,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'description': 'md5:33765bb339e1b47e7e72b5490139bb41',
                 'uploader': 'LloydVEVO',
                 'uploader_id': 'LloydVEVO',
+                'uploader_url': 're:https?://(?:www\.)?youtube\.com/user/LloydVEVO',
                 'upload_date': '20110629',
                 'license': 'Standard YouTube License',
                 'age_limit': 18,
@@ -574,6 +583,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'ext': 'mp4',
                 'upload_date': '20100430',
                 'uploader_id': 'deadmau5',
+                'uploader_url': 're:https?://(?:www\.)?youtube\.com/user/deadmau5',
                 'creator': 'deadmau5',
                 'description': 'md5:12c56784b8032162bb936a5f76d55360',
                 'uploader': 'deadmau5',
@@ -593,6 +603,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'ext': 'mp4',
                 'upload_date': '20150827',
                 'uploader_id': 'olympic',
+                'uploader_url': 're:https?://(?:www\.)?youtube\.com/user/olympic',
                 'license': 'Standard YouTube License',
                 'description': 'HO09  - Women -  GER-AUS - Hockey - 31 July 2012 - London 2012 Olympic Games',
                 'uploader': 'Olympics',
@@ -611,6 +622,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'stretched_ratio': 16 / 9.,
                 'upload_date': '20110310',
                 'uploader_id': 'AllenMeow',
+                'uploader_url': 're:https?://(?:www\.)?youtube\.com/user/AllenMeow',
                 'description': 'made by Wacom from Korea | 字幕&加油添醋 by TY\'s Allen | 感謝heylisa00cavey1001同學熱情提供梗及翻譯',
                 'uploader': '孫艾倫',
                 'license': 'Standard YouTube License',
@@ -644,6 +656,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'description': 'md5:116377fd2963b81ec4ce64b542173306',
                 'upload_date': '20150625',
                 'uploader_id': 'dorappi2000',
+                'uploader_url': 're:https?://(?:www\.)?youtube\.com/user/dorappi2000',
                 'uploader': 'dorappi2000',
                 'license': 'Standard YouTube License',
                 'formats': 'mincount:33',
@@ -685,6 +698,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     'upload_date': '20150721',
                     'uploader': 'Beer Games Beer',
                     'uploader_id': 'beergamesbeer',
+                    'uploader_url': 're:https?://(?:www\.)?youtube\.com/user/beergamesbeer',
                     'license': 'Standard YouTube License',
                 },
             }, {
@@ -696,6 +710,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     'upload_date': '20150721',
                     'uploader': 'Beer Games Beer',
                     'uploader_id': 'beergamesbeer',
+                    'uploader_url': 're:https?://(?:www\.)?youtube\.com/user/beergamesbeer',
                     'license': 'Standard YouTube License',
                 },
             }, {
@@ -707,6 +722,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     'upload_date': '20150721',
                     'uploader': 'Beer Games Beer',
                     'uploader_id': 'beergamesbeer',
+                    'uploader_url': 're:https?://(?:www\.)?youtube\.com/user/beergamesbeer',
                     'license': 'Standard YouTube License',
                 },
             }, {
@@ -718,6 +734,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     'upload_date': '20150721',
                     'uploader': 'Beer Games Beer',
                     'uploader_id': 'beergamesbeer',
+                    'uploader_url': 're:https?://(?:www\.)?youtube\.com/user/beergamesbeer',
                     'license': 'Standard YouTube License',
                 },
             }],
@@ -752,6 +769,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'description': 'md5:8085699c11dc3f597ce0410b0dcbb34a',
                 'upload_date': '20151119',
                 'uploader_id': 'IronSoulElf',
+                'uploader_url': 're:https?://(?:www\.)?youtube\.com/user/IronSoulElf',
                 'uploader': 'IronSoulElf',
                 'license': 'Standard YouTube License',
                 'creator': 'Todd Haberman, Daniel Law Heath & Aaron Kaplan',
@@ -791,6 +809,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'description': 'md5:a677553cf0840649b731a3024aeff4cc',
                 'upload_date': '20150127',
                 'uploader_id': 'BerkmanCenter',
+                'uploader_url': 're:https?://(?:www\.)?youtube\.com/user/BerkmanCenter',
                 'uploader': 'BerkmanCenter',
                 'license': 'Creative Commons Attribution license (reuse allowed)',
             },
@@ -798,6 +817,24 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'skip_download': True,
             },
         },
+        {
+            # Channel-like uploader_url
+            'url': 'https://www.youtube.com/watch?v=eQcmzGIKrzg',
+            'info_dict': {
+                'id': 'eQcmzGIKrzg',
+                'ext': 'mp4',
+                'title': 'Democratic Socialism and Foreign Policy | Bernie Sanders',
+                'description': 'md5:dda0d780d5a6e120758d1711d062a867',
+                'upload_date': '20151119',
+                'uploader': 'Bernie 2016',
+                'uploader_id': 'UCH1dpzjCEiGAt8CXkryhkZg',
+                'uploader_url': 're:https?://(?:www\.)?youtube\.com/channel/UCH1dpzjCEiGAt8CXkryhkZg',
+                'license': 'Creative Commons Attribution license (reuse allowed)',
+            },
+            'params': {
+                'skip_download': True,
+            },
+        },
         {
             'url': 'https://www.youtube.com/watch?feature=player_embedded&amp;amp;v=V36LpHqtcDY',
             'only_matching': True,
@@ -1334,9 +1371,13 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
 
         # uploader_id
         video_uploader_id = None
-        mobj = re.search(r'<link itemprop="url" href="http://www.youtube.com/(?:user|channel)/([^"]+)">', video_webpage)
+        video_uploader_url = None
+        mobj = re.search(
+            r'<link itemprop="url" href="(?P<uploader_url>https?://www.youtube.com/(?:user|channel)/(?P<uploader_id>[^"]+))">',
+            video_webpage)
         if mobj is not None:
-            video_uploader_id = mobj.group(1)
+            video_uploader_id = mobj.group('uploader_id')
+            video_uploader_url = mobj.group('uploader_url')
         else:
             self._downloader.report_warning('unable to extract uploader nickname')
 
@@ -1642,6 +1683,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             'id': video_id,
             'uploader': video_uploader,
             'uploader_id': video_uploader_id,
+            'uploader_url': video_uploader_url,
             'upload_date': upload_date,
             'license': video_license,
             'creator': video_creator,

From a2d7797cee21e54866f11ce41863a92c992a56f8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 3 Mar 2016 00:00:11 +0600
Subject: [PATCH 4164/6219] [vimeo] Extract uploader_url (Closes #8727)

---
 youtube_dl/extractor/vimeo.py | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 3049dffb6..9f282a1da 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -93,6 +93,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
                 'title': "youtube-dl test video - \u2605 \" ' \u5e78 / \\ \u00e4 \u21ad \U0001d550",
                 'description': 'md5:2d3305bad981a06ff79f027f19865021',
                 'upload_date': '20121220',
+                'uploader_url': 're:https?://(?:www\.)?vimeo\.com/user7108434',
                 'uploader_id': 'user7108434',
                 'uploader': 'Filippo Valsorda',
                 'duration': 10,
@@ -105,6 +106,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
             'info_dict': {
                 'id': '68093876',
                 'ext': 'mp4',
+                'uploader_url': 're:https?://(?:www\.)?vimeo\.com/openstreetmapus',
                 'uploader_id': 'openstreetmapus',
                 'uploader': 'OpenStreetMap US',
                 'title': 'Andy Allan - Putting the Carto into OpenStreetMap Cartography',
@@ -121,6 +123,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
                 'ext': 'mp4',
                 'title': 'Kathy Sierra: Building the minimum Badass User, Business of Software 2012',
                 'uploader': 'The BLN & Business of Software',
+                'uploader_url': 're:https?://(?:www\.)?vimeo\.com/theblnbusinessofsoftware',
                 'uploader_id': 'theblnbusinessofsoftware',
                 'duration': 3610,
                 'description': None,
@@ -135,6 +138,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
                 'ext': 'mp4',
                 'title': 'youtube-dl password protected test video',
                 'upload_date': '20130614',
+                'uploader_url': 're:https?://(?:www\.)?vimeo\.com/user18948128',
                 'uploader_id': 'user18948128',
                 'uploader': 'Jaime Marquínez Ferrándiz',
                 'duration': 10,
@@ -154,6 +158,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
                 'ext': 'mp4',
                 'title': 'Key & Peele: Terrorist Interrogation',
                 'description': 'md5:8678b246399b070816b12313e8b4eb5c',
+                'uploader_url': 're:https?://(?:www\.)?vimeo\.com/atencio',
                 'uploader_id': 'atencio',
                 'uploader': 'Peter Atencio',
                 'upload_date': '20130927',
@@ -169,6 +174,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
                 'title': 'The New Vimeo Player (You Know, For Videos)',
                 'description': 'md5:2ec900bf97c3f389378a96aee11260ea',
                 'upload_date': '20131015',
+                'uploader_url': 're:https?://(?:www\.)?vimeo\.com/staff',
                 'uploader_id': 'staff',
                 'uploader': 'Vimeo Staff',
                 'duration': 62,
@@ -183,6 +189,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
                 'ext': 'mp4',
                 'title': 'Pier Solar OUYA Official Trailer',
                 'uploader': 'Tulio Gonçalves',
+                'uploader_url': 're:https?://(?:www\.)?vimeo\.com/user28849593',
                 'uploader_id': 'user28849593',
             },
         },
@@ -195,6 +202,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
                 'ext': 'mp4',
                 'title': 'FOX CLASSICS - Forever Classic ID - A Full Minute',
                 'uploader': 'The DMCI',
+                'uploader_url': 're:https?://(?:www\.)?vimeo\.com/dmci',
                 'uploader_id': 'dmci',
                 'upload_date': '20111220',
                 'description': 'md5:ae23671e82d05415868f7ad1aec21147',
@@ -370,9 +378,10 @@ class VimeoIE(VimeoBaseInfoExtractor):
         # Extract title
         video_title = config['video']['title']
 
-        # Extract uploader and uploader_id
-        video_uploader = config['video']['owner']['name']
-        video_uploader_id = config['video']['owner']['url'].split('/')[-1] if config['video']['owner']['url'] else None
+        # Extract uploader, uploader_url and uploader_id
+        video_uploader = config['video'].get('owner', {}).get('name')
+        video_uploader_url = config['video'].get('owner', {}).get('url')
+        video_uploader_id = video_uploader_url.split('/')[-1] if video_uploader_url else None
 
         # Extract video thumbnail
         video_thumbnail = config['video'].get('thumbnail')
@@ -473,6 +482,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
         return {
             'id': video_id,
             'uploader': video_uploader,
+            'uploader_url': video_uploader_url,
             'uploader_id': video_uploader_id,
             'upload_date': video_upload_date,
             'title': video_title,

From 2316dc2b9a1b31ea6845d31571d620c413884926 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 3 Mar 2016 00:41:36 +0600
Subject: [PATCH 4165/6219] [twitch:playlistbase] Mark broken

Twitch paging mechanism is completely broken on twitch side serving all videos all the time and making our travis builds stall.
---
 youtube_dl/extractor/twitch.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 69882da63..5b8586097 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -303,6 +303,7 @@ class TwitchPlaylistBaseIE(TwitchBaseIE):
 
 
 class TwitchProfileIE(TwitchPlaylistBaseIE):
+    _WORKING = False
     IE_NAME = 'twitch:profile'
     _VALID_URL = r'%s/(?P<id>[^/]+)/profile/?(?:\#.*)?$' % TwitchBaseIE._VALID_URL_BASE
     _PLAYLIST_TYPE = 'profile'
@@ -318,6 +319,7 @@ class TwitchProfileIE(TwitchPlaylistBaseIE):
 
 
 class TwitchPastBroadcastsIE(TwitchPlaylistBaseIE):
+    _WORKING = False
     IE_NAME = 'twitch:past_broadcasts'
     _VALID_URL = r'%s/(?P<id>[^/]+)/profile/past_broadcasts/?(?:\#.*)?$' % TwitchBaseIE._VALID_URL_BASE
     _PLAYLIST_URL = TwitchPlaylistBaseIE._PLAYLIST_URL + '&broadcasts=true'
@@ -334,6 +336,7 @@ class TwitchPastBroadcastsIE(TwitchPlaylistBaseIE):
 
 
 class TwitchBookmarksIE(TwitchPlaylistBaseIE):
+    _WORKING = False
     IE_NAME = 'twitch:bookmarks'
     _VALID_URL = r'%s/(?P<id>[^/]+)/profile/bookmarks/?(?:\#.*)?$' % TwitchBaseIE._VALID_URL_BASE
     _PLAYLIST_URL = '%s/api/bookmark/?user=%%s&offset=%%d&limit=%%d' % TwitchBaseIE._API_BASE

From 5e73a67d44bc25bd8660704cd437d35bd1e9541e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 3 Mar 2016 02:38:41 +0800
Subject: [PATCH 4166/6219] [letv] Domain name changed

---
 youtube_dl/extractor/letv.py | 33 ++++++++++++++++++---------------
 1 file changed, 18 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/letv.py b/youtube_dl/extractor/letv.py
index 9fd494c29..246556316 100644
--- a/youtube_dl/extractor/letv.py
+++ b/youtube_dl/extractor/letv.py
@@ -27,10 +27,12 @@ from ..utils import (
 
 class LetvIE(InfoExtractor):
     IE_DESC = '乐视网'
-    _VALID_URL = r'http://www\.letv\.com/ptv/vplay/(?P<id>\d+).html'
+    _VALID_URL = r'http://www\.le\.com/ptv/vplay/(?P<id>\d+).html'
+
+    _URL_TEMPLATE = r'http://www.le.com/ptv/vplay/%s.html'
 
     _TESTS = [{
-        'url': 'http://www.letv.com/ptv/vplay/22005890.html',
+        'url': 'http://www.le.com/ptv/vplay/22005890.html',
         'md5': 'edadcfe5406976f42f9f266057ee5e40',
         'info_dict': {
             'id': '22005890',
@@ -42,7 +44,7 @@ class LetvIE(InfoExtractor):
             'hls_prefer_native': True,
         },
     }, {
-        'url': 'http://www.letv.com/ptv/vplay/1415246.html',
+        'url': 'http://www.le.com/ptv/vplay/1415246.html',
         'info_dict': {
             'id': '1415246',
             'ext': 'mp4',
@@ -54,7 +56,7 @@ class LetvIE(InfoExtractor):
         },
     }, {
         'note': 'This video is available only in Mainland China, thus a proxy is needed',
-        'url': 'http://www.letv.com/ptv/vplay/1118082.html',
+        'url': 'http://www.le.com/ptv/vplay/1118082.html',
         'md5': '2424c74948a62e5f31988438979c5ad1',
         'info_dict': {
             'id': '1118082',
@@ -116,10 +118,10 @@ class LetvIE(InfoExtractor):
             'splatid': 101,
             'format': 1,
             'tkey': self.calc_time_key(int(time.time())),
-            'domain': 'www.letv.com'
+            'domain': 'www.le.com'
         }
         play_json_req = sanitized_Request(
-            'http://api.letv.com/mms/out/video/playJson?' + compat_urllib_parse.urlencode(params)
+            'http://api.le.com/mms/out/video/playJson?' + compat_urllib_parse.urlencode(params)
         )
         cn_verification_proxy = self._downloader.params.get('cn_verification_proxy')
         if cn_verification_proxy:
@@ -193,9 +195,9 @@ class LetvIE(InfoExtractor):
 
 
 class LetvTvIE(InfoExtractor):
-    _VALID_URL = r'http://www.letv.com/tv/(?P<id>\d+).html'
+    _VALID_URL = r'http://www.le.com/tv/(?P<id>\d+).html'
     _TESTS = [{
-        'url': 'http://www.letv.com/tv/46177.html',
+        'url': 'http://www.le.com/tv/46177.html',
         'info_dict': {
             'id': '46177',
             'title': '美人天下',
@@ -208,10 +210,11 @@ class LetvTvIE(InfoExtractor):
         playlist_id = self._match_id(url)
         page = self._download_webpage(url, playlist_id)
 
-        media_urls = list(set(re.findall(
-            r'http://www.letv.com/ptv/vplay/\d+.html', page)))
-        entries = [self.url_result(media_url, ie='Letv')
-                   for media_url in media_urls]
+        # Currently old domain names are still used in playlists
+        media_ids = list(set(re.findall(
+            r'http://www.letv.com/ptv/vplay/(\d+).html', page)))
+        entries = [self.url_result(LetvIE._URL_TEMPLATE % media_id, ie='Letv')
+                   for media_id in media_ids]
 
         title = self._html_search_meta('keywords', page,
                                        fatal=False).split('，')[0]
@@ -222,9 +225,9 @@ class LetvTvIE(InfoExtractor):
 
 
 class LetvPlaylistIE(LetvTvIE):
-    _VALID_URL = r'http://tv.letv.com/[a-z]+/(?P<id>[a-z]+)/index.s?html'
+    _VALID_URL = r'http://tv.le.com/[a-z]+/(?P<id>[a-z]+)/index.s?html'
     _TESTS = [{
-        'url': 'http://tv.letv.com/izt/wuzetian/index.html',
+        'url': 'http://tv.le.com/izt/wuzetian/index.html',
         'info_dict': {
             'id': 'wuzetian',
             'title': '武媚娘传奇',
@@ -233,7 +236,7 @@ class LetvPlaylistIE(LetvTvIE):
         # This playlist contains some extra videos other than the drama itself
         'playlist_mincount': 96
     }, {
-        'url': 'http://tv.letv.com/pzt/lswjzzjc/index.shtml',
+        'url': 'http://tv.le.com/pzt/lswjzzjc/index.shtml',
         'info_dict': {
             'id': 'lswjzzjc',
             # The title should be "劲舞青春", but I can't find a simple way to

From 7465e7e42d0a02196e8d8ecd49edcadca4823897 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 3 Mar 2016 02:40:51 +0800
Subject: [PATCH 4167/6219] [letv] Keep videos' order in playlists

---
 youtube_dl/extractor/letv.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/letv.py b/youtube_dl/extractor/letv.py
index 246556316..91172c096 100644
--- a/youtube_dl/extractor/letv.py
+++ b/youtube_dl/extractor/letv.py
@@ -22,6 +22,7 @@ from ..utils import (
     str_or_none,
     encode_data_uri,
     url_basename,
+    orderedSet,
 )
 
 
@@ -211,8 +212,8 @@ class LetvTvIE(InfoExtractor):
         page = self._download_webpage(url, playlist_id)
 
         # Currently old domain names are still used in playlists
-        media_ids = list(set(re.findall(
-            r'http://www.letv.com/ptv/vplay/(\d+).html', page)))
+        media_ids = orderedSet(re.findall(
+            r'http://www.letv.com/ptv/vplay/(\d+).html', page))
         entries = [self.url_result(LetvIE._URL_TEMPLATE % media_id, ie='Letv')
                    for media_id in media_ids]
 

From 7a8fef31730f0a9823c1379f67902dd95a9ea580 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 3 Mar 2016 02:42:19 +0800
Subject: [PATCH 4168/6219] [letv] Order imports alphabetically

---
 youtube_dl/extractor/letv.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/letv.py b/youtube_dl/extractor/letv.py
index 91172c096..2ece30839 100644
--- a/youtube_dl/extractor/letv.py
+++ b/youtube_dl/extractor/letv.py
@@ -1,28 +1,28 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import base64
 import datetime
+import hashlib
 import re
 import time
-import base64
-import hashlib
 
 from .common import InfoExtractor
 from ..compat import (
-    compat_urllib_parse,
     compat_ord,
     compat_str,
+    compat_urllib_parse,
 )
 from ..utils import (
     determine_ext,
+    encode_data_uri,
     ExtractorError,
+    int_or_none,
+    orderedSet,
     parse_iso8601,
     sanitized_Request,
-    int_or_none,
     str_or_none,
-    encode_data_uri,
     url_basename,
-    orderedSet,
 )
 
 

From f3bc19a989d9f6d6b21abd4271cb99a18897069f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 3 Mar 2016 02:50:12 +0800
Subject: [PATCH 4169/6219] [letv] Correct regular expressions and fix a typo

---
 youtube_dl/extractor/letv.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/letv.py b/youtube_dl/extractor/letv.py
index 2ece30839..ba8126f75 100644
--- a/youtube_dl/extractor/letv.py
+++ b/youtube_dl/extractor/letv.py
@@ -28,9 +28,9 @@ from ..utils import (
 
 class LetvIE(InfoExtractor):
     IE_DESC = '乐视网'
-    _VALID_URL = r'http://www\.le\.com/ptv/vplay/(?P<id>\d+).html'
+    _VALID_URL = r'http://www\.le\.com/ptv/vplay/(?P<id>\d+)\.html'
 
-    _URL_TEMPLATE = r'http://www.le.com/ptv/vplay/%s.html'
+    _URL_TEMPLATE = 'http://www.le.com/ptv/vplay/%s.html'
 
     _TESTS = [{
         'url': 'http://www.le.com/ptv/vplay/22005890.html',
@@ -196,7 +196,7 @@ class LetvIE(InfoExtractor):
 
 
 class LetvTvIE(InfoExtractor):
-    _VALID_URL = r'http://www.le.com/tv/(?P<id>\d+).html'
+    _VALID_URL = r'http://www\.le\.com/tv/(?P<id>\d+)\.html'
     _TESTS = [{
         'url': 'http://www.le.com/tv/46177.html',
         'info_dict': {
@@ -213,7 +213,7 @@ class LetvTvIE(InfoExtractor):
 
         # Currently old domain names are still used in playlists
         media_ids = orderedSet(re.findall(
-            r'http://www.letv.com/ptv/vplay/(\d+).html', page))
+            r'http://www\.letv\.com/ptv/vplay/(\d+)\.html', page))
         entries = [self.url_result(LetvIE._URL_TEMPLATE % media_id, ie='Letv')
                    for media_id in media_ids]
 
@@ -226,7 +226,7 @@ class LetvTvIE(InfoExtractor):
 
 
 class LetvPlaylistIE(LetvTvIE):
-    _VALID_URL = r'http://tv.le.com/[a-z]+/(?P<id>[a-z]+)/index.s?html'
+    _VALID_URL = r'http://tv\.le\.com/[a-z]+/(?P<id>[a-z]+)/index\.s?html'
     _TESTS = [{
         'url': 'http://tv.le.com/izt/wuzetian/index.html',
         'info_dict': {

From 2eb25b256bf38db9f35696822cee399099eb4777 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 3 Mar 2016 03:14:00 +0800
Subject: [PATCH 4170/6219] [letv] Merge LetvTvIE into LetvPlaylistIE

And
1. Add more URL examples
2. Improve the matching pattern
---
 youtube_dl/extractor/__init__.py |  1 -
 youtube_dl/extractor/letv.py     | 56 ++++++++++++++++++--------------
 2 files changed, 31 insertions(+), 26 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index b3bf73963..19c563172 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -354,7 +354,6 @@ from .lecture2go import Lecture2GoIE
 from .lemonde import LemondeIE
 from .letv import (
     LetvIE,
-    LetvTvIE,
     LetvPlaylistIE,
     LetvCloudIE,
 )
diff --git a/youtube_dl/extractor/letv.py b/youtube_dl/extractor/letv.py
index ba8126f75..ce25bb4db 100644
--- a/youtube_dl/extractor/letv.py
+++ b/youtube_dl/extractor/letv.py
@@ -195,8 +195,9 @@ class LetvIE(InfoExtractor):
         }
 
 
-class LetvTvIE(InfoExtractor):
-    _VALID_URL = r'http://www\.le\.com/tv/(?P<id>\d+)\.html'
+class LetvPlaylistIE(InfoExtractor):
+    _VALID_URL = r'http://[a-z]+\.le\.com/[a-z]+/(?P<id>[a-z0-9_]+)'
+
     _TESTS = [{
         'url': 'http://www.le.com/tv/46177.html',
         'info_dict': {
@@ -205,29 +206,7 @@ class LetvTvIE(InfoExtractor):
             'description': 'md5:395666ff41b44080396e59570dbac01c'
         },
         'playlist_count': 35
-    }]
-
-    def _real_extract(self, url):
-        playlist_id = self._match_id(url)
-        page = self._download_webpage(url, playlist_id)
-
-        # Currently old domain names are still used in playlists
-        media_ids = orderedSet(re.findall(
-            r'http://www\.letv\.com/ptv/vplay/(\d+)\.html', page))
-        entries = [self.url_result(LetvIE._URL_TEMPLATE % media_id, ie='Letv')
-                   for media_id in media_ids]
-
-        title = self._html_search_meta('keywords', page,
-                                       fatal=False).split('，')[0]
-        description = self._html_search_meta('description', page, fatal=False)
-
-        return self.playlist_result(entries, playlist_id, playlist_title=title,
-                                    playlist_description=description)
-
-
-class LetvPlaylistIE(LetvTvIE):
-    _VALID_URL = r'http://tv\.le\.com/[a-z]+/(?P<id>[a-z]+)/index\.s?html'
-    _TESTS = [{
+    }, {
         'url': 'http://tv.le.com/izt/wuzetian/index.html',
         'info_dict': {
             'id': 'wuzetian',
@@ -246,8 +225,35 @@ class LetvPlaylistIE(LetvTvIE):
             'description': 'md5:b1eef244f45589a7b5b1af9ff25a4489'
         },
         'playlist_mincount': 7
+    }, {
+        'url': 'http://www.le.com/comic/92063.html',
+        'only_matching': True,
+    }, {
+        'url': 'http://list.le.com/listn/c1009_sc532002_d2_p1_o1.html',
+        'only_matching': True,
     }]
 
+    @classmethod
+    def suitable(cls, url):
+        return False if LetvIE.suitable(url) else super(LetvPlaylistIE, cls).suitable(url)
+
+    def _real_extract(self, url):
+        playlist_id = self._match_id(url)
+        page = self._download_webpage(url, playlist_id)
+
+        # Currently old domain names are still used in playlists
+        media_ids = orderedSet(re.findall(
+            r'<a[^>]+href="http://www\.letv\.com/ptv/vplay/(\d+)\.html', page))
+        entries = [self.url_result(LetvIE._URL_TEMPLATE % media_id, ie='Letv')
+                   for media_id in media_ids]
+
+        title = self._html_search_meta('keywords', page,
+                                       fatal=False).split('，')[0]
+        description = self._html_search_meta('description', page, fatal=False)
+
+        return self.playlist_result(entries, playlist_id, playlist_title=title,
+                                    playlist_description=description)
+
 
 class LetvCloudIE(InfoExtractor):
     IE_DESC = '乐视云'

From 8c5a9944246d459f0eac876ae358dedb32fc890a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 3 Mar 2016 03:24:20 +0800
Subject: [PATCH 4171/6219] [leeco] Letv renamed to LeEco

LeEco is the company name and Le is the domain name.

For more information see the Chinese news post
http://www.techorz.com/company-news/letv-renamed-to-leeco-and-new-logo/
---
 youtube_dl/extractor/__init__.py           |  6 +++---
 youtube_dl/extractor/{letv.py => leeco.py} | 10 ++++++----
 2 files changed, 9 insertions(+), 7 deletions(-)
 rename youtube_dl/extractor/{letv.py => leeco.py} (97%)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 19c563172..037654a23 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -352,9 +352,9 @@ from .la7 import LA7IE
 from .laola1tv import Laola1TvIE
 from .lecture2go import Lecture2GoIE
 from .lemonde import LemondeIE
-from .letv import (
-    LetvIE,
-    LetvPlaylistIE,
+from .leeco import (
+    LeIE,
+    LePlaylistIE,
     LetvCloudIE,
 )
 from .libsyn import LibsynIE
diff --git a/youtube_dl/extractor/letv.py b/youtube_dl/extractor/leeco.py
similarity index 97%
rename from youtube_dl/extractor/letv.py
rename to youtube_dl/extractor/leeco.py
index ce25bb4db..d0cd3f591 100644
--- a/youtube_dl/extractor/letv.py
+++ b/youtube_dl/extractor/leeco.py
@@ -26,7 +26,7 @@ from ..utils import (
 )
 
 
-class LetvIE(InfoExtractor):
+class LeIE(InfoExtractor):
     IE_DESC = '乐视网'
     _VALID_URL = r'http://www\.le\.com/ptv/vplay/(?P<id>\d+)\.html'
 
@@ -195,7 +195,7 @@ class LetvIE(InfoExtractor):
         }
 
 
-class LetvPlaylistIE(InfoExtractor):
+class LePlaylistIE(InfoExtractor):
     _VALID_URL = r'http://[a-z]+\.le\.com/[a-z]+/(?P<id>[a-z0-9_]+)'
 
     _TESTS = [{
@@ -235,7 +235,7 @@ class LetvPlaylistIE(InfoExtractor):
 
     @classmethod
     def suitable(cls, url):
-        return False if LetvIE.suitable(url) else super(LetvPlaylistIE, cls).suitable(url)
+        return False if LeIE.suitable(url) else super(LePlaylistIE, cls).suitable(url)
 
     def _real_extract(self, url):
         playlist_id = self._match_id(url)
@@ -244,7 +244,7 @@ class LetvPlaylistIE(InfoExtractor):
         # Currently old domain names are still used in playlists
         media_ids = orderedSet(re.findall(
             r'<a[^>]+href="http://www\.letv\.com/ptv/vplay/(\d+)\.html', page))
-        entries = [self.url_result(LetvIE._URL_TEMPLATE % media_id, ie='Letv')
+        entries = [self.url_result(LeIE._URL_TEMPLATE % media_id, ie='Le')
                    for media_id in media_ids]
 
         title = self._html_search_meta('keywords', page,
@@ -256,6 +256,8 @@ class LetvPlaylistIE(InfoExtractor):
 
 
 class LetvCloudIE(InfoExtractor):
+    # Most of *.letv.com is changed to *.le.com on 2016/01/02
+    # but yuntv.letv.com is kept, so also keep the extractor name
     IE_DESC = '乐视云'
     _VALID_URL = r'https?://yuntv\.letv\.com/bcloud.html\?.+'
 

From e07237f6403fe2c18212b36e27bb630dd8e5cef1 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 2 Mar 2016 21:40:21 +0100
Subject: [PATCH 4172/6219] [utils] remove check for val from find_xpath_attr

---
 youtube_dl/utils.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 606977c58..210c47fce 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -160,8 +160,6 @@ if sys.version_info >= (2, 7):
     def find_xpath_attr(node, xpath, key, val=None):
         """ Find the xpath xpath[@key=val] """
         assert re.match(r'^[a-zA-Z_-]+$', key)
-        if val:
-            assert re.match(r'^[a-zA-Z0-9@\s:._-]*$', val)
         expr = xpath + ('[@%s]' % key if val is None else "[@%s='%s']" % (key, val))
         return node.find(expr)
 else:

From 4459bef20369584c46dc5550fa4e49b7858754bc Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 2 Mar 2016 21:41:29 +0100
Subject: [PATCH 4173/6219] [thepltform] detect other types of errors

---
 youtube_dl/extractor/theplatform.py | 15 ++++++---------
 1 file changed, 6 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index 93d871571..9a57b49df 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -22,6 +22,7 @@ from ..utils import (
     unsmuggle_url,
     xpath_with_ns,
     mimetype2ext,
+    find_xpath_attr,
 )
 
 default_ns = 'http://www.w3.org/2005/SMIL21/Language'
@@ -31,15 +32,11 @@ _x = lambda p: xpath_with_ns(p, {'smil': default_ns})
 class ThePlatformBaseIE(InfoExtractor):
     def _extract_theplatform_smil(self, smil_url, video_id, note='Downloading SMIL data'):
         meta = self._download_xml(smil_url, video_id, note=note)
-        try:
-            error_msg = next(
-                n.attrib['abstract']
-                for n in meta.findall(_x('.//smil:ref'))
-                if n.attrib.get('title') == 'Geographic Restriction' or n.attrib.get('title') == 'Expired')
-        except StopIteration:
-            pass
-        else:
-            raise ExtractorError(error_msg, expected=True)
+        error_element = find_xpath_attr(
+            meta, _x('.//smil:ref'), 'src',
+            'http://link.theplatform.com/s/errorFiles/Unavailable.mp4')
+        if error_element is not None:
+            raise ExtractorError(error_element.attrib['abstract'], expected=True)
 
         formats = self._parse_smil_formats(
             meta, smil_url, video_id, namespace=default_ns,

From 0ae937a798c2c9a2eae4c6dbadc68a1a4c5e101c Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 3 Mar 2016 13:39:04 +0800
Subject: [PATCH 4174/6219] [twitter] Support twitter.com/i/videos/tweet/ URLS

Closes #8737
---
 youtube_dl/extractor/twitter.py | 53 +++++++++++++++++++++++++++------
 1 file changed, 44 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 5d2b5ec35..5cbb536f9 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -22,7 +22,7 @@ class TwitterBaseIE(InfoExtractor):
 
 class TwitterCardIE(TwitterBaseIE):
     IE_NAME = 'twitter:card'
-    _VALID_URL = r'https?://(?:www\.)?twitter\.com/i/cards/tfw/v1/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?twitter\.com/i/(?:cards/tfw/v1|videos/tweet)/(?P<id>\d+)'
     _TESTS = [
         {
             'url': 'https://twitter.com/i/cards/tfw/v1/560070183650213889',
@@ -30,7 +30,7 @@ class TwitterCardIE(TwitterBaseIE):
             'info_dict': {
                 'id': '560070183650213889',
                 'ext': 'mp4',
-                'title': 'TwitterCard',
+                'title': 'Twitter Card',
                 'thumbnail': 're:^https?://.*\.jpg$',
                 'duration': 30.033,
             }
@@ -41,7 +41,7 @@ class TwitterCardIE(TwitterBaseIE):
             'info_dict': {
                 'id': '623160978427936768',
                 'ext': 'mp4',
-                'title': 'TwitterCard',
+                'title': 'Twitter Card',
                 'thumbnail': 're:^https?://.*\.jpg',
                 'duration': 80.155,
             },
@@ -72,7 +72,16 @@ class TwitterCardIE(TwitterBaseIE):
                 'title': 'Vine by ArsenalTerje',
             },
             'add_ie': ['Vine'],
-        }
+        }, {
+            'url': 'https://twitter.com/i/videos/tweet/705235433198714880',
+            'md5': '3846d0a07109b5ab622425449b59049d',
+            'info_dict': {
+                'id': '705235433198714880',
+                'ext': 'mp4',
+                'title': 'Twitter web player',
+                'thumbnail': 're:^https?://.*\.jpg',
+            },
+        },
     ]
 
     def _real_extract(self, url):
@@ -98,12 +107,13 @@ class TwitterCardIE(TwitterBaseIE):
                 return self.url_result(iframe_url)
 
             config = self._parse_json(self._html_search_regex(
-                r'data-player-config="([^"]+)"', webpage, 'data player config'),
+                r'data-(?:player-)?config="([^"]+)"', webpage, 'data player config'),
                 video_id)
             if 'playlist' not in config:
-                if 'vmapUrl' in config:
+                vmap_url = config.get('vmapUrl') or config.get('vmap_url')
+                if vmap_url:
                     formats.append({
-                        'url': self._get_vmap_video_url(config['vmapUrl'], video_id),
+                        'url': self._get_vmap_video_url(vmap_url, video_id),
                     })
                     break   # same video regardless of UA
                 continue
@@ -123,12 +133,13 @@ class TwitterCardIE(TwitterBaseIE):
             formats.append(f)
         self._sort_formats(formats)
 
-        thumbnail = config.get('posterImageUrl')
+        title = self._search_regex(r'<title>([^<]+)</title>', webpage, 'title')
+        thumbnail = config.get('posterImageUrl') or config.get('image_src')
         duration = float_or_none(config.get('duration'))
 
         return {
             'id': video_id,
-            'title': 'TwitterCard',
+            'title': title,
             'thumbnail': thumbnail,
             'duration': duration,
             'formats': formats,
@@ -177,6 +188,21 @@ class TwitterIE(InfoExtractor):
             'uploader_id': 'starwars',
             'uploader': 'Star Wars',
         },
+    }, {
+        'url': 'https://twitter.com/BTNBrentYarina/status/705235433198714880',
+        'info_dict': {
+            'id': '705235433198714880',
+            'ext': 'mp4',
+            'title': 'Brent Yarina - Khalil Iverson\'s missed highlight dunk. And made highlight dunk. In one highlight.',
+            'description': 'Brent Yarina on Twitter: "Khalil Iverson\'s missed highlight dunk. And made highlight dunk. In one highlight."',
+            'uploader_id': 'BTNBrentYarina',
+            'uploader': 'Brent Yarina',
+        },
+        'params': {
+            # The same video as https://twitter.com/i/videos/tweet/705235433198714880
+            # Test case of TwitterCardIE
+            'skip_download': True,
+        },
     }]
 
     def _real_extract(self, url):
@@ -234,6 +260,15 @@ class TwitterIE(InfoExtractor):
             })
             return info
 
+        if 'class="PlayableMedia' in webpage:
+            info.update({
+                '_type': 'url_transparent',
+                'ie_key': 'TwitterCard',
+                'url': '%s//twitter.com/i/videos/tweet/%s' % (self.http_scheme(), twid),
+            })
+
+            return info
+
         raise ExtractorError('There\'s no video in this tweet.')
 
 

From b8972bd69da48509333a8239fc0b755dccb1a681 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 3 Mar 2016 14:24:24 +0800
Subject: [PATCH 4175/6219] [twitter] Fix extraction of test_Twitter and
 test_Twitter_1

---
 youtube_dl/extractor/twitter.py | 71 +++++++++++++++++++++++----------
 1 file changed, 51 insertions(+), 20 deletions(-)

diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 5cbb536f9..3627d49f1 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -95,6 +95,8 @@ class TwitterCardIE(TwitterBaseIE):
 
         config = None
         formats = []
+        duration = None
+
         for user_agent in USER_AGENTS:
             request = sanitized_Request(url)
             request.add_header('User-Agent', user_agent)
@@ -109,33 +111,60 @@ class TwitterCardIE(TwitterBaseIE):
             config = self._parse_json(self._html_search_regex(
                 r'data-(?:player-)?config="([^"]+)"', webpage, 'data player config'),
                 video_id)
-            if 'playlist' not in config:
-                vmap_url = config.get('vmapUrl') or config.get('vmap_url')
-                if vmap_url:
-                    formats.append({
-                        'url': self._get_vmap_video_url(vmap_url, video_id),
+
+            playlist = config.get('playlist')
+            if playlist:
+                video_url = playlist[0]['source']
+
+                f = {
+                    'url': video_url,
+                }
+
+                m = re.search(r'/(?P<width>\d+)x(?P<height>\d+)/', video_url)
+                if m:
+                    f.update({
+                        'width': int(m.group('width')),
+                        'height': int(m.group('height')),
                     })
-                    break   # same video regardless of UA
+                formats.append(f)
                 continue
 
-            video_url = config['playlist'][0]['source']
-
-            f = {
-                'url': video_url,
-            }
-
-            m = re.search(r'/(?P<width>\d+)x(?P<height>\d+)/', video_url)
-            if m:
-                f.update({
-                    'width': int(m.group('width')),
-                    'height': int(m.group('height')),
+            vmap_url = config.get('vmapUrl') or config.get('vmap_url')
+            if vmap_url:
+                formats.append({
+                    'url': self._get_vmap_video_url(vmap_url, video_id),
                 })
-            formats.append(f)
+                break   # same video regardless of UA
+
+            media_info = config.get('status', {}).get('entities', [{}])[0].get('mediaInfo', {})
+            if media_info:
+                for media_variant in media_info['variants']:
+                    media_url = media_variant['url']
+                    if media_url.endswith('.m3u8'):
+                        formats.extend(self._extract_m3u8_formats(media_url, video_id, ext='mp4', m3u8_id='hls'))
+                    elif media_url.endswith('.mpd'):
+                        formats.extend(self._extract_mpd_formats(media_url, video_id, mpd_id='dash'))
+                    else:
+                        vbr = int_or_none(media_variant.get('bitRate'), scale=1000)
+                        a_format = {
+                            'url': media_url,
+                            'format_id': 'http-%d' % vbr if vbr else 'http',
+                            'vbr': vbr,
+                        }
+                        # Reported bitRate may be zero
+                        if not a_format['vbr']:
+                            del a_format['vbr']
+
+                        formats.append(a_format)
+
+                duration = float_or_none(media_info.get('duration', {}).get('nanos'), scale=1e9)
+                break   # same video regardless of UA
+
         self._sort_formats(formats)
 
         title = self._search_regex(r'<title>([^<]+)</title>', webpage, 'title')
         thumbnail = config.get('posterImageUrl') or config.get('image_src')
-        duration = float_or_none(config.get('duration'))
+        duration = float_or_none(config.get('duration')) or duration
 
         return {
             'id': video_id,
@@ -153,7 +182,6 @@ class TwitterIE(InfoExtractor):
 
     _TESTS = [{
         'url': 'https://twitter.com/freethenipple/status/643211948184596480',
-        # MD5 checksums are different in different places
         'info_dict': {
             'id': '643211948184596480',
             'ext': 'mp4',
@@ -164,6 +192,9 @@ class TwitterIE(InfoExtractor):
             'uploader': 'FREE THE NIPPLE',
             'uploader_id': 'freethenipple',
         },
+        'params': {
+            'skip_download': True,  # requires ffmpeg
+        },
     }, {
         'url': 'https://twitter.com/giphz/status/657991469417025536/photo/1',
         'md5': 'f36dcd5fb92bf7057f155e7d927eeb42',

From c8398a9b87dadf4cd8464e1ab77d2874d9ae484b Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 3 Mar 2016 14:27:27 +0800
Subject: [PATCH 4176/6219] [twitter] Now Twitter serves the same file for
 Firefox and Chrome

---
 youtube_dl/extractor/twitter.py | 107 ++++++++++++++------------------
 1 file changed, 47 insertions(+), 60 deletions(-)

diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 3627d49f1..97a4a5948 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -10,7 +10,6 @@ from ..utils import (
     remove_end,
     int_or_none,
     ExtractorError,
-    sanitized_Request,
 )
 
 
@@ -87,78 +86,66 @@ class TwitterCardIE(TwitterBaseIE):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        # Different formats served for different User-Agents
-        USER_AGENTS = [
-            'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20150101 Firefox/20.0 (Chrome)',  # mp4
-            'Mozilla/5.0 (Windows NT 5.2; WOW64; rv:38.0) Gecko/20100101 Firefox/38.0',  # webm
-        ]
-
         config = None
         formats = []
         duration = None
 
-        for user_agent in USER_AGENTS:
-            request = sanitized_Request(url)
-            request.add_header('User-Agent', user_agent)
-            webpage = self._download_webpage(request, video_id)
+        webpage = self._download_webpage(url, video_id)
 
-            iframe_url = self._html_search_regex(
-                r'<iframe[^>]+src="((?:https?:)?//(?:www.youtube.com/embed/[^"]+|(?:www\.)?vine\.co/v/\w+/card))"',
-                webpage, 'video iframe', default=None)
-            if iframe_url:
-                return self.url_result(iframe_url)
+        iframe_url = self._html_search_regex(
+            r'<iframe[^>]+src="((?:https?:)?//(?:www.youtube.com/embed/[^"]+|(?:www\.)?vine\.co/v/\w+/card))"',
+            webpage, 'video iframe', default=None)
+        if iframe_url:
+            return self.url_result(iframe_url)
 
-            config = self._parse_json(self._html_search_regex(
-                r'data-(?:player-)?config="([^"]+)"', webpage, 'data player config'),
-                video_id)
+        config = self._parse_json(self._html_search_regex(
+            r'data-(?:player-)?config="([^"]+)"', webpage, 'data player config'),
+            video_id)
 
-            playlist = config.get('playlist')
-            if playlist:
-                video_url = playlist[0]['source']
+        playlist = config.get('playlist')
+        if playlist:
+            video_url = playlist[0]['source']
 
-                f = {
-                    'url': video_url,
-                }
+            f = {
+                'url': video_url,
+            }
 
-                m = re.search(r'/(?P<width>\d+)x(?P<height>\d+)/', video_url)
-                if m:
-                    f.update({
-                        'width': int(m.group('width')),
-                        'height': int(m.group('height')),
-                    })
-                formats.append(f)
-                continue
-
-            vmap_url = config.get('vmapUrl') or config.get('vmap_url')
-            if vmap_url:
-                formats.append({
-                    'url': self._get_vmap_video_url(vmap_url, video_id),
+            m = re.search(r'/(?P<width>\d+)x(?P<height>\d+)/', video_url)
+            if m:
+                f.update({
+                    'width': int(m.group('width')),
+                    'height': int(m.group('height')),
                 })
-                break   # same video regardless of UA
+            formats.append(f)
 
-            media_info = config.get('status', {}).get('entities', [{}])[0].get('mediaInfo', {})
-            if media_info:
-                for media_variant in media_info['variants']:
-                    media_url = media_variant['url']
-                    if media_url.endswith('.m3u8'):
-                        formats.extend(self._extract_m3u8_formats(media_url, video_id, ext='mp4', m3u8_id='hls'))
-                    elif media_url.endswith('.mpd'):
-                        formats.extend(self._extract_mpd_formats(media_url, video_id, mpd_id='dash'))
-                    else:
-                        vbr = int_or_none(media_variant.get('bitRate'), scale=1000)
-                        a_format = {
-                            'url': media_url,
-                            'format_id': 'http-%d' % vbr if vbr else 'http',
-                            'vbr': vbr,
-                        }
-                        # Reported bitRate may be zero
-                        if not a_format['vbr']:
-                            del a_format['vbr']
+        vmap_url = config.get('vmapUrl') or config.get('vmap_url')
+        if vmap_url:
+            formats.append({
+                'url': self._get_vmap_video_url(vmap_url, video_id),
+            })
 
-                        formats.append(a_format)
+        media_info = config.get('status', {}).get('entities', [{}])[0].get('mediaInfo', {})
+        if media_info:
+            for media_variant in media_info['variants']:
+                media_url = media_variant['url']
+                if media_url.endswith('.m3u8'):
+                    formats.extend(self._extract_m3u8_formats(media_url, video_id, ext='mp4', m3u8_id='hls'))
+                elif media_url.endswith('.mpd'):
+                    formats.extend(self._extract_mpd_formats(media_url, video_id, mpd_id='dash'))
+                else:
+                    vbr = int_or_none(media_variant.get('bitRate'), scale=1000)
+                    a_format = {
+                        'url': media_url,
+                        'format_id': 'http-%d' % vbr if vbr else 'http',
+                        'vbr': vbr,
+                    }
+                    # Reported bitRate may be zero
+                    if not a_format['vbr']:
+                        del a_format['vbr']
 
-                duration = float_or_none(media_info.get('duration', {}).get('nanos'), scale=1e9)
-                break   # same video regardless of UA
+                    formats.append(a_format)
+
+            duration = float_or_none(media_info.get('duration', {}).get('nanos'), scale=1e9)
 
         self._sort_formats(formats)
 

From 03879ff0547b6d1b96c530075cd99f99b8c74a2b Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 3 Mar 2016 14:42:49 +0800
Subject: [PATCH 4177/6219] [twitter] Media info is not always in the first
 entity

Fixes #8704
---
 youtube_dl/extractor/twitter.py | 22 +++++++++++++++++++++-
 1 file changed, 21 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 97a4a5948..67762a003 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -124,7 +124,12 @@ class TwitterCardIE(TwitterBaseIE):
                 'url': self._get_vmap_video_url(vmap_url, video_id),
             })
 
-        media_info = config.get('status', {}).get('entities', [{}])[0].get('mediaInfo', {})
+        media_info = None
+
+        for entity in config.get('status', {}).get('entities', []):
+            if 'mediaInfo' in entity:
+                media_info = entity['mediaInfo']
+
         if media_info:
             for media_variant in media_info['variants']:
                 media_url = media_variant['url']
@@ -221,6 +226,21 @@ class TwitterIE(InfoExtractor):
             # Test case of TwitterCardIE
             'skip_download': True,
         },
+    }, {
+        'url': 'https://twitter.com/jaydingeer/status/700207533655363584',
+        'md5': '',
+        'info_dict': {
+            'id': '700207533655363584',
+            'ext': 'mp4',
+            'title': 'jay - BEAT PROD: @suhmeduh #Damndaniel',
+            'description': 'jay on Twitter: "BEAT PROD: @suhmeduh  https://t.co/HBrQ4AfpvZ #Damndaniel https://t.co/byBooq2ejZ"',
+            'thumbnail': 're:^https?://.*\.jpg',
+            'uploader': 'jay',
+            'uploader_id': 'jaydingeer',
+        },
+        'params': {
+            'skip_download': True,  # requires ffmpeg
+        },
     }]
 
     def _real_extract(self, url):

From f3bdae76de7265475277c265f9fff12cdc59825b Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 3 Mar 2016 10:27:22 +0100
Subject: [PATCH 4178/6219] [extractor/common] add update_url_params helper
 method to add or update query string params

---
 youtube_dl/extractor/common.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 402f2f436..282559597 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -517,6 +517,13 @@ class InfoExtractor(object):
             else:
                 self.report_warning(errmsg + str(ve))
 
+    def update_url_params(self, url, params):
+        parsed_url = compat_urlparse.urlparse(url)
+        qs = compat_urlparse.parse_qs(parsed_url.query)
+        qs.update(params)
+        return compat_urlparse.urlunparse(
+            parsed_url._replace(query=compat_urllib_parse.urlencode(qs, True)))
+
     def report_warning(self, msg, video_id=None):
         idstr = '' if video_id is None else '%s: ' % video_id
         self._downloader.report_warning(

From fa9e259fd91ce1bf31310330adc20ddef2b1d948 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 3 Mar 2016 10:54:39 +0100
Subject: [PATCH 4179/6219] [extractor/common] use compat_parse_qs in
 update_url_params

---
 youtube_dl/extractor/common.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 282559597..a95387cee 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -20,6 +20,7 @@ from ..compat import (
     compat_urllib_error,
     compat_urllib_parse,
     compat_urlparse,
+    compat_parse_qs,
     compat_str,
     compat_etree_fromstring,
 )
@@ -519,7 +520,7 @@ class InfoExtractor(object):
 
     def update_url_params(self, url, params):
         parsed_url = compat_urlparse.urlparse(url)
-        qs = compat_urlparse.parse_qs(parsed_url.query)
+        qs = compat_parse_qs(parsed_url.query)
         qs.update(params)
         return compat_urlparse.urlunparse(
             parsed_url._replace(query=compat_urllib_parse.urlencode(qs, True)))

From 8bb56eeeea8154f811076c0a9093203fab224003 Mon Sep 17 00:00:00 2001
From: Brian Foley <bpfoley@gmail.com>
Date: Sat, 2 Jan 2016 19:49:59 +0000
Subject: [PATCH 4180/6219] [utils] Add extract_attributes for extracting html
 tag attributes

This is much more robust than just using regexps, and handles all
the common scenarios, such as empty/no values, repeated attributes,
entity decoding, mixed case names, and the different possible value
quoting schemes.
---
 test/test_utils.py   | 40 ++++++++++++++++++++++++++++++++++++++++
 youtube_dl/compat.py |  6 ++++++
 youtube_dl/utils.py  | 30 ++++++++++++++++++++++++++++++
 3 files changed, 76 insertions(+)

diff --git a/test/test_utils.py b/test/test_utils.py
index 97587ad2f..cb85e18f0 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -28,6 +28,7 @@ from youtube_dl.utils import (
     encodeFilename,
     escape_rfc3986,
     escape_url,
+    extract_attributes,
     ExtractorError,
     find_xpath_attr,
     fix_xml_ampersands,
@@ -75,6 +76,7 @@ from youtube_dl.utils import (
     cli_bool_option,
 )
 from youtube_dl.compat import (
+    compat_chr,
     compat_etree_fromstring,
 )
 
@@ -591,6 +593,44 @@ class TestUtil(unittest.TestCase):
         on = js_to_json('{"abc": "def",}')
         self.assertEqual(json.loads(on), {'abc': 'def'})
 
+    def test_extract_attributes(self):
+        self.assertEqual(extract_attributes('<e x="y">'), {'x': 'y'})
+        self.assertEqual(extract_attributes("<e x='y'>"), {'x': 'y'})
+        self.assertEqual(extract_attributes('<e x=y>'), {'x': 'y'})
+        self.assertEqual(extract_attributes('<e x="a \'b\' c">'), {'x': "a 'b' c"})
+        self.assertEqual(extract_attributes('<e x=\'a "b" c\'>'), {'x': 'a "b" c'})
+        self.assertEqual(extract_attributes('<e x="&#121;">'), {'x': 'y'})
+        self.assertEqual(extract_attributes('<e x="&#x79;">'), {'x': 'y'})
+        self.assertEqual(extract_attributes('<e x="&amp;">'), {'x': '&'})  # XML
+        self.assertEqual(extract_attributes('<e x="&quot;">'), {'x': '"'})
+        self.assertEqual(extract_attributes('<e x="&pound;">'), {'x': '£'}) # HTML 3.2
+        self.assertEqual(extract_attributes('<e x="&lambda;">'), {'x': 'λ'}) # HTML 4.0
+        self.assertEqual(extract_attributes('<e x="&foo">'), {'x': '&foo'})
+        self.assertEqual(extract_attributes('<e x="\'">'), {'x': "'"})
+        self.assertEqual(extract_attributes('<e x=\'"\'>'), {'x': '"'})
+        self.assertEqual(extract_attributes('<e x >'), {'x': None})
+        self.assertEqual(extract_attributes('<e x=y a>'), {'x': 'y', 'a': None})
+        self.assertEqual(extract_attributes('<e x= y>'), {'x': 'y'})
+        self.assertEqual(extract_attributes('<e x=1 y=2 x=3>'), {'y': '2', 'x': '3'})
+        self.assertEqual(extract_attributes('<e \nx=\ny\n>'), {'x': 'y'})
+        self.assertEqual(extract_attributes('<e \nx=\n"y"\n>'), {'x': 'y'})
+        self.assertEqual(extract_attributes("<e \nx=\n'y'\n>"), {'x': 'y'})
+        self.assertEqual(extract_attributes('<e \nx="\ny\n">'), {'x': '\ny\n'})
+        self.assertEqual(extract_attributes('<e CAPS=x>'), {'caps': 'x'}) # Names lowercased
+        self.assertEqual(extract_attributes('<e x=1 X=2>'), {'x': '2'})
+        self.assertEqual(extract_attributes('<e X=1 x=2>'), {'x': '2'})
+        self.assertEqual(extract_attributes('<e _:funny-name1=1>'), {'_:funny-name1': '1'})
+        self.assertEqual(extract_attributes('<e x="Fáilte 世界 \U0001f600">'), {'x': 'Fáilte 世界 \U0001f600'})
+        self.assertEqual(extract_attributes('<e x="décompose&#769;">'), {'x': 'décompose\u0301'})
+        # "Narrow" Python builds don't support unicode code points outside BMP.
+        try:
+            compat_chr(0x10000)
+            supports_outside_bmp = True
+        except ValueError:
+            supports_outside_bmp = False
+        if supports_outside_bmp:
+            self.assertEqual(extract_attributes('<e x="Smile &#128512;!">'), {'x': 'Smile \U0001f600!'})
+
     def test_clean_html(self):
         self.assertEqual(clean_html('a:\nb'), 'a: b')
         self.assertEqual(clean_html('a:\n   "b"'), 'a:    "b"')
diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index b497da696..7b9afc36d 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -77,6 +77,11 @@ try:
 except ImportError:  # Python 2
     from urllib import urlretrieve as compat_urlretrieve
 
+try:
+    from html.parser import HTMLParser as compat_HTMLParser
+except ImportError:  # Python 2
+    from HTMLParser import HTMLParser as compat_HTMLParser
+
 
 try:
     from subprocess import DEVNULL
@@ -540,6 +545,7 @@ else:
     from tokenize import generate_tokens as compat_tokenize_tokenize
 
 __all__ = [
+    'compat_HTMLParser',
     'compat_HTTPError',
     'compat_basestring',
     'compat_chr',
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 210c47fce..a0234a3a8 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -35,6 +35,7 @@ import xml.etree.ElementTree
 import zlib
 
 from .compat import (
+    compat_HTMLParser,
     compat_basestring,
     compat_chr,
     compat_etree_fromstring,
@@ -272,6 +273,35 @@ def get_element_by_attribute(attribute, value, html):
 
     return unescapeHTML(res)
 
+class HTMLAttributeParser(compat_HTMLParser):
+    """Trivial HTML parser to gather the attributes for a single element"""
+    def __init__(self):
+        self.attrs = { }
+        compat_HTMLParser.__init__(self)
+
+    def handle_starttag(self, tag, attrs):
+        self.attrs = dict(attrs)
+
+def extract_attributes(html_element):
+    """Given a string for an HTML element such as
+    <el
+         a="foo" B="bar" c="&98;az" d=boz
+         empty= noval entity="&amp;"
+         sq='"' dq="'"
+    >
+    Decode and return a dictionary of attributes.
+    {
+        'a': 'foo', 'b': 'bar', c: 'baz', d: 'boz',
+        'empty': '', 'noval': None, 'entity': '&',
+        'sq': '"', 'dq': '\''
+    }.
+    NB HTMLParser is stricter in Python 2.6 & 3.2 than in later versions,
+    but the cases in the unit test will work for all of 2.6, 2.7, 3.2-3.5.
+    """
+    parser = HTMLAttributeParser()
+    parser.feed(html_element)
+    parser.close()
+    return parser.attrs
 
 def clean_html(html):
     """Clean an HTML snippet into a readable string"""

From 8ee239e9216f12eae38deb079090b677086e9de1 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 3 Mar 2016 18:47:54 +0800
Subject: [PATCH 4181/6219] [utils] Jython support - handle filenames correctly

Now test:youtube downloads
---
 youtube_dl/utils.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 16b4324a4..fc64a4186 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -467,6 +467,10 @@ def encodeFilename(s, for_subprocess=False):
     if not for_subprocess and sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
         return s
 
+    # Jython assumes filenames are Unicode strings though reported as Python 2.x compatible
+    if sys.platform.startswith('java'):
+        return s
+
     return s.encode(get_subprocess_encoding(), 'ignore')
 
 

From e9c0cdd3895436170de33324c1762364380c6a5a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 3 Mar 2016 19:24:24 +0800
Subject: [PATCH 4182/6219] [jython] Introduce compat_os_name

os.name is always 'java' on Jython
---
 test/helper.py                      |  7 +++++--
 youtube_dl/YoutubeDL.py             | 13 +++++++------
 youtube_dl/compat.py                |  8 ++++++--
 youtube_dl/downloader/common.py     |  3 ++-
 youtube_dl/extractor/common.py      |  9 +++++----
 youtube_dl/postprocessor/xattrpp.py |  5 +++--
 6 files changed, 28 insertions(+), 17 deletions(-)

diff --git a/test/helper.py b/test/helper.py
index bdd7acca4..f2d878212 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -11,8 +11,11 @@ import sys
 
 import youtube_dl.extractor
 from youtube_dl import YoutubeDL
-from youtube_dl.utils import (
+from youtube_dl.compat import (
+    compat_os_name,
     compat_str,
+)
+from youtube_dl.utils import (
     preferredencoding,
     write_string,
 )
@@ -42,7 +45,7 @@ def report_warning(message):
     Print the message to stderr, it will be prefixed with 'WARNING:'
     If stderr is a tty file the 'WARNING:' will be colored
     '''
-    if sys.stderr.isatty() and os.name != 'nt':
+    if sys.stderr.isatty() and compat_os_name != 'nt':
         _msg_header = '\033[0;33mWARNING:\033[0m'
     else:
         _msg_header = 'WARNING:'
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 8f3a8b9e3..dcc867e45 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -24,9 +24,6 @@ import time
 import tokenize
 import traceback
 
-if os.name == 'nt':
-    import ctypes
-
 from .compat import (
     compat_basestring,
     compat_cookiejar,
@@ -34,6 +31,7 @@ from .compat import (
     compat_get_terminal_size,
     compat_http_client,
     compat_kwargs,
+    compat_os_name,
     compat_str,
     compat_tokenize_tokenize,
     compat_urllib_error,
@@ -95,6 +93,9 @@ from .postprocessor import (
 )
 from .version import __version__
 
+if compat_os_name == 'nt':
+    import ctypes
+
 
 class YoutubeDL(object):
     """YoutubeDL class.
@@ -450,7 +451,7 @@ class YoutubeDL(object):
     def to_console_title(self, message):
         if not self.params.get('consoletitle', False):
             return
-        if os.name == 'nt' and ctypes.windll.kernel32.GetConsoleWindow():
+        if compat_os_name == 'nt' and ctypes.windll.kernel32.GetConsoleWindow():
             # c_wchar_p() might not be necessary if `message` is
             # already of type unicode()
             ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message))
@@ -521,7 +522,7 @@ class YoutubeDL(object):
         else:
             if self.params.get('no_warnings'):
                 return
-            if not self.params.get('no_color') and self._err_file.isatty() and os.name != 'nt':
+            if not self.params.get('no_color') and self._err_file.isatty() and compat_os_name != 'nt':
                 _msg_header = '\033[0;33mWARNING:\033[0m'
             else:
                 _msg_header = 'WARNING:'
@@ -533,7 +534,7 @@ class YoutubeDL(object):
         Do the same as trouble, but prefixes the message with 'ERROR:', colored
         in red if stderr is a tty file.
         '''
-        if not self.params.get('no_color') and self._err_file.isatty() and os.name != 'nt':
+        if not self.params.get('no_color') and self._err_file.isatty() and compat_os_name != 'nt':
             _msg_header = '\033[0;31mERROR:\033[0m'
         else:
             _msg_header = 'ERROR:'
diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index b497da696..2771fb5fa 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -326,6 +326,9 @@ def compat_ord(c):
         return ord(c)
 
 
+compat_os_name = os._name if os.name == 'java' else os.name
+
+
 if sys.version_info >= (3, 0):
     compat_getenv = os.getenv
     compat_expanduser = os.path.expanduser
@@ -346,7 +349,7 @@ else:
     # The following are os.path.expanduser implementations from cpython 2.7.8 stdlib
     # for different platforms with correct environment variables decoding.
 
-    if os.name == 'posix':
+    if compat_os_name == 'posix':
         def compat_expanduser(path):
             """Expand ~ and ~user constructions.  If user or $HOME is unknown,
             do nothing."""
@@ -370,7 +373,7 @@ else:
                 userhome = pwent.pw_dir
             userhome = userhome.rstrip('/')
             return (userhome + path[i:]) or '/'
-    elif os.name == 'nt' or os.name == 'ce':
+    elif compat_os_name == 'nt' or compat_os_name == 'ce':
         def compat_expanduser(path):
             """Expand ~ and ~user constructs.
 
@@ -556,6 +559,7 @@ __all__ = [
     'compat_itertools_count',
     'compat_kwargs',
     'compat_ord',
+    'compat_os_name',
     'compat_parse_qs',
     'compat_print',
     'compat_shlex_split',
diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index 2d5154051..f39db58f6 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -5,6 +5,7 @@ import re
 import sys
 import time
 
+from ..compat import compat_os_name
 from ..utils import (
     encodeFilename,
     error_to_compat_str,
@@ -219,7 +220,7 @@ class FileDownloader(object):
         if self.params.get('progress_with_newline', False):
             self.to_screen(fullmsg)
         else:
-            if os.name == 'nt':
+            if compat_os_name == 'nt':
                 prev_len = getattr(self, '_report_progress_prev_line_length',
                                    0)
                 if prev_len > len(fullmsg):
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 402f2f436..07bd2cbe2 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -15,13 +15,14 @@ import math
 from ..compat import (
     compat_cookiejar,
     compat_cookies,
+    compat_etree_fromstring,
     compat_getpass,
     compat_http_client,
+    compat_os_name,
+    compat_str,
     compat_urllib_error,
     compat_urllib_parse,
     compat_urlparse,
-    compat_str,
-    compat_etree_fromstring,
 )
 from ..utils import (
     NO_DEFAULT,
@@ -427,7 +428,7 @@ class InfoExtractor(object):
             self.to_screen('Saving request to ' + filename)
             # Working around MAX_PATH limitation on Windows (see
             # http://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx)
-            if os.name == 'nt':
+            if compat_os_name == 'nt':
                 absfilepath = os.path.abspath(filename)
                 if len(absfilepath) > 259:
                     filename = '\\\\?\\' + absfilepath
@@ -596,7 +597,7 @@ class InfoExtractor(object):
                 if mobj:
                     break
 
-        if not self._downloader.params.get('no_color') and os.name != 'nt' and sys.stderr.isatty():
+        if not self._downloader.params.get('no_color') and compat_os_name != 'nt' and sys.stderr.isatty():
             _name = '\033[0;34m%s\033[0m' % name
         else:
             _name = name
diff --git a/youtube_dl/postprocessor/xattrpp.py b/youtube_dl/postprocessor/xattrpp.py
index 480d48d05..e39ca60aa 100644
--- a/youtube_dl/postprocessor/xattrpp.py
+++ b/youtube_dl/postprocessor/xattrpp.py
@@ -6,6 +6,7 @@ import sys
 import errno
 
 from .common import PostProcessor
+from ..compat import compat_os_name
 from ..utils import (
     check_executable,
     hyphenate_date,
@@ -73,7 +74,7 @@ class XAttrMetadataPP(PostProcessor):
                     raise XAttrMetadataError(e.errno, e.strerror)
 
         except ImportError:
-            if os.name == 'nt':
+            if compat_os_name == 'nt':
                 # Write xattrs to NTFS Alternate Data Streams:
                 # http://en.wikipedia.org/wiki/NTFS#Alternate_data_streams_.28ADS.29
                 def write_xattr(path, key, value):
@@ -168,7 +169,7 @@ class XAttrMetadataPP(PostProcessor):
                     'Unable to write extended attributes due to too long values.')
             else:
                 msg = 'This filesystem doesn\'t support extended attributes. '
-                if os.name == 'nt':
+                if compat_os_name == 'nt':
                     msg += 'You need to use NTFS.'
                 else:
                     msg += '(You may have to enable them in your /etc/fstab)'

From 429491f531035f4fe9e958e6bf4108867107efec Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 3 Mar 2016 21:55:17 +0800
Subject: [PATCH 4183/6219] [test/http] Fix failure in Jython

make offlinetest passed on the latest Jython hg version with patched
lib-python/2.7/urllib2.py pulled from CPython 2.7.11
---
 test/test_http.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/test/test_http.py b/test/test_http.py
index f2e305b6f..fc59b1aed 100644
--- a/test/test_http.py
+++ b/test/test_http.py
@@ -52,7 +52,12 @@ class TestHTTP(unittest.TestCase):
             ('localhost', 0), HTTPTestRequestHandler)
         self.httpd.socket = ssl.wrap_socket(
             self.httpd.socket, certfile=certfn, server_side=True)
-        self.port = self.httpd.socket.getsockname()[1]
+        if os.name == 'java':
+            # In Jython SSLSocket is not a subclass of socket.socket
+            sock = self.httpd.socket.sock
+        else:
+            sock = self.httpd.socket
+        self.port = sock.getsockname()[1]
         self.server_thread = threading.Thread(target=self.httpd.serve_forever)
         self.server_thread.daemon = True
         self.server_thread.start()

From 1b0635aba3ded1142ff3f2b1c4bd67fb0cb3d6ae Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 3 Mar 2016 21:57:49 +0800
Subject: [PATCH 4184/6219] [Makefile] Allow specifying the Python version in
 offline tests

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index afd7f4032..e98806791 100644
--- a/Makefile
+++ b/Makefile
@@ -45,7 +45,7 @@ test:
 ot: offlinetest
 
 offlinetest: codetest
-	nosetests --verbose test --exclude test_download.py --exclude test_age_restriction.py --exclude test_subtitles.py --exclude test_write_annotations.py --exclude test_youtube_lists.py --exclude test_iqiyi_sdk_interpreter.py
+	$(PYTHON) -m nose --verbose test --exclude test_download.py --exclude test_age_restriction.py --exclude test_subtitles.py --exclude test_write_annotations.py --exclude test_youtube_lists.py --exclude test_iqiyi_sdk_interpreter.py
 
 tar: youtube-dl.tar.gz
 

From 539a1641c6effa3a5b2be2fecd2fd4a6f35cef62 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 3 Mar 2016 22:10:36 +0600
Subject: [PATCH 4185/6219] [twitch] Workaround broken paging (Closes #8740)

---
 youtube_dl/extractor/twitch.py | 27 ++++++++++++++++++++++-----
 1 file changed, 22 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 5b8586097..d92d2fd40 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -271,7 +271,7 @@ class TwitchVodIE(TwitchItemBaseIE):
 
 class TwitchPlaylistBaseIE(TwitchBaseIE):
     _PLAYLIST_URL = '%s/kraken/channels/%%s/videos/?offset=%%d&limit=%%d' % TwitchBaseIE._API_BASE
-    _PAGE_LIMIT = 100
+    _PAGE_LIMIT = 10
 
     def _extract_playlist(self, channel_id):
         info = self._download_json(
@@ -281,14 +281,34 @@ class TwitchPlaylistBaseIE(TwitchBaseIE):
         entries = []
         offset = 0
         limit = self._PAGE_LIMIT
+        broken_paging_detected = False
+        counter_override = None
         for counter in itertools.count(1):
             response = self._download_json(
                 self._PLAYLIST_URL % (channel_id, offset, limit),
-                channel_id, 'Downloading %s videos JSON page %d' % (self._PLAYLIST_TYPE, counter))
+                channel_id,
+                'Downloading %s videos JSON page %s'
+                % (self._PLAYLIST_TYPE, counter_override or counter))
             page_entries = self._extract_playlist_page(response)
             if not page_entries:
                 break
+            total = int_or_none(response.get('_total'))
+            # Since the beginning of March 2016 twitch's paging mechanism
+            # is completely broken on the twitch side. It simply ignores
+            # a limit and returns the whole offset number of videos.
+            # Working around by just requesting all videos at once.
+            if not broken_paging_detected and total and len(page_entries) > limit:
+                self.report_warning(
+                    'Twitch paging is broken on twitch side, requesting all videos at once',
+                    channel_id)
+                broken_paging_detected = True
+                limit = total
+                offset = 0
+                counter_override = '(all at once)'
+                continue
             entries.extend(page_entries)
+            if broken_paging_detected or total and len(page_entries) >= total:
+                break
             offset += limit
         return self.playlist_result(
             [self.url_result(entry) for entry in set(entries)],
@@ -303,7 +323,6 @@ class TwitchPlaylistBaseIE(TwitchBaseIE):
 
 
 class TwitchProfileIE(TwitchPlaylistBaseIE):
-    _WORKING = False
     IE_NAME = 'twitch:profile'
     _VALID_URL = r'%s/(?P<id>[^/]+)/profile/?(?:\#.*)?$' % TwitchBaseIE._VALID_URL_BASE
     _PLAYLIST_TYPE = 'profile'
@@ -319,7 +338,6 @@ class TwitchProfileIE(TwitchPlaylistBaseIE):
 
 
 class TwitchPastBroadcastsIE(TwitchPlaylistBaseIE):
-    _WORKING = False
     IE_NAME = 'twitch:past_broadcasts'
     _VALID_URL = r'%s/(?P<id>[^/]+)/profile/past_broadcasts/?(?:\#.*)?$' % TwitchBaseIE._VALID_URL_BASE
     _PLAYLIST_URL = TwitchPlaylistBaseIE._PLAYLIST_URL + '&broadcasts=true'
@@ -336,7 +354,6 @@ class TwitchPastBroadcastsIE(TwitchPlaylistBaseIE):
 
 
 class TwitchBookmarksIE(TwitchPlaylistBaseIE):
-    _WORKING = False
     IE_NAME = 'twitch:bookmarks'
     _VALID_URL = r'%s/(?P<id>[^/]+)/profile/bookmarks/?(?:\#.*)?$' % TwitchBaseIE._VALID_URL_BASE
     _PLAYLIST_URL = '%s/api/bookmark/?user=%%s&offset=%%d&limit=%%d' % TwitchBaseIE._API_BASE

From 8bbb4b56ee6c582f416ca1f8951a6821fccc1548 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 3 Mar 2016 22:11:26 +0600
Subject: [PATCH 4186/6219] [twitch:playlistsbase] Use orderedSet

---
 youtube_dl/extractor/twitch.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index d92d2fd40..a9f8a5b8b 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -17,6 +17,7 @@ from ..utils import (
     encode_dict,
     ExtractorError,
     int_or_none,
+    orderedSet,
     parse_duration,
     parse_iso8601,
     sanitized_Request,
@@ -311,7 +312,7 @@ class TwitchPlaylistBaseIE(TwitchBaseIE):
                 break
             offset += limit
         return self.playlist_result(
-            [self.url_result(entry) for entry in set(entries)],
+            [self.url_result(entry) for entry in orderedSet(entries)],
             channel_id, channel_name)
 
     def _extract_playlist_page(self, response):

From ececca6cde9f42f24d1d8b74a3c56f1149f70f5d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 3 Mar 2016 22:12:55 +0600
Subject: [PATCH 4187/6219] [twitch:playlistbase] Restore original _PAGE_LIMIT

---
 youtube_dl/extractor/twitch.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index a9f8a5b8b..0cf240e1e 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -272,7 +272,7 @@ class TwitchVodIE(TwitchItemBaseIE):
 
 class TwitchPlaylistBaseIE(TwitchBaseIE):
     _PLAYLIST_URL = '%s/kraken/channels/%%s/videos/?offset=%%d&limit=%%d' % TwitchBaseIE._API_BASE
-    _PAGE_LIMIT = 10
+    _PAGE_LIMIT = 100
 
     def _extract_playlist(self, channel_id):
         info = self._download_json(

From a8276b268074eb844f0d62ff1cc9fd163a91d970 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 3 Mar 2016 22:18:32 +0600
Subject: [PATCH 4188/6219] [twitch:playlistbase] Fix all at once fetch

---
 youtube_dl/extractor/twitch.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 0cf240e1e..8639293e3 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -303,8 +303,7 @@ class TwitchPlaylistBaseIE(TwitchBaseIE):
                     'Twitch paging is broken on twitch side, requesting all videos at once',
                     channel_id)
                 broken_paging_detected = True
-                limit = total
-                offset = 0
+                offset = total
                 counter_override = '(all at once)'
                 continue
             entries.extend(page_entries)

From 38f9ef31dc434a6702686844b421085955137c55 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 3 Mar 2016 18:34:52 +0100
Subject: [PATCH 4189/6219] [utils] add update_url_query function

---
 youtube_dl/extractor/common.py | 8 --------
 youtube_dl/utils.py            | 8 ++++++++
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index a95387cee..402f2f436 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -20,7 +20,6 @@ from ..compat import (
     compat_urllib_error,
     compat_urllib_parse,
     compat_urlparse,
-    compat_parse_qs,
     compat_str,
     compat_etree_fromstring,
 )
@@ -518,13 +517,6 @@ class InfoExtractor(object):
             else:
                 self.report_warning(errmsg + str(ve))
 
-    def update_url_params(self, url, params):
-        parsed_url = compat_urlparse.urlparse(url)
-        qs = compat_parse_qs(parsed_url.query)
-        qs.update(params)
-        return compat_urlparse.urlunparse(
-            parsed_url._replace(query=compat_urllib_parse.urlencode(qs, True)))
-
     def report_warning(self, msg, video_id=None):
         idstr = '' if video_id is None else '%s: ' % video_id
         self._downloader.report_warning(
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 210c47fce..31d60f323 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1719,6 +1719,14 @@ def urlencode_postdata(*args, **kargs):
     return compat_urllib_parse.urlencode(*args, **kargs).encode('ascii')
 
 
+def update_url_query(url, query):
+    parsed_url = compat_urlparse.urlparse(url)
+    qs = compat_parse_qs(parsed_url.query)
+    qs.update(query)
+    return compat_urlparse.urlunparse(parsed_url._replace(
+        query=compat_urllib_parse.urlencode(qs, True)))
+
+
 def encode_dict(d, encoding='utf-8'):
     def encode(v):
         return v.encode(encoding) if isinstance(v, compat_basestring) else v

From fb640d0a3d3af77e6835d9a6a2bf80920785511a Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 3 Mar 2016 18:40:05 +0100
Subject: [PATCH 4190/6219] [test/test_utils] add tests for update_url_query

---
 test/test_utils.py | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/test/test_utils.py b/test/test_utils.py
index 97587ad2f..19a546619 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -61,6 +61,7 @@ from youtube_dl.utils import (
     lowercase_escape,
     url_basename,
     urlencode_postdata,
+    update_url_query,
     version_tuple,
     xpath_with_ns,
     xpath_element,
@@ -76,6 +77,8 @@ from youtube_dl.utils import (
 )
 from youtube_dl.compat import (
     compat_etree_fromstring,
+    compat_urlparse,
+    compat_parse_qs,
 )
 
 
@@ -454,6 +457,31 @@ class TestUtil(unittest.TestCase):
         data = urlencode_postdata({'username': 'foo@bar.com', 'password': '1234'})
         self.assertTrue(isinstance(data, bytes))
 
+    def test_update_url_query(self):
+        def query_dict(url):
+            return compat_parse_qs(compat_urlparse.urlparse(url).query)
+        self.assertEqual(query_dict(update_url_query(
+            'http://example.com/path', {'quality': ['HD'], 'format': ['mp4']})),
+            query_dict('http://example.com/path?quality=HD&format=mp4'))
+        self.assertEqual(query_dict(update_url_query(
+            'http://example.com/path', {'system': ['LINUX', 'WINDOWS']})),
+            query_dict('http://example.com/path?system=LINUX&system=WINDOWS'))
+        self.assertEqual(query_dict(update_url_query(
+            'http://example.com/path', {'fields': 'id,formats,subtitles'})),
+            query_dict('http://example.com/path?fields=id,formats,subtitles'))
+        self.assertEqual(query_dict(update_url_query(
+            'http://example.com/path', {'fields': ('id,formats,subtitles', 'thumbnails')})),
+            query_dict('http://example.com/path?fields=id,formats,subtitles&fields=thumbnails'))
+        self.assertEqual(query_dict(update_url_query(
+            'http://example.com/path?manifest=f4m', {'manifest': []})),
+            query_dict('http://example.com/path'))
+        self.assertEqual(query_dict(update_url_query(
+            'http://example.com/path?system=LINUX&system=WINDOWS', {'system': 'LINUX'})),
+            query_dict('http://example.com/path?system=LINUX'))
+        self.assertEqual(query_dict(update_url_query(
+            'http://example.com/path', {'fields': b'id,formats,subtitles'})),
+            query_dict('http://example.com/path?fields=id,formats,subtitles'))
+
     def test_dict_get(self):
         FALSE_VALUES = {
             'none': None,

From d0ff690d68016f24d8cc90e69fac20847e7afa8e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 4 Mar 2016 00:09:40 +0600
Subject: [PATCH 4191/6219] [indavideo:embed] Fix tags extraction (Closes
 #8738)

---
 youtube_dl/extractor/indavideo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/indavideo.py b/youtube_dl/extractor/indavideo.py
index 12fb5e8e1..9622f198a 100644
--- a/youtube_dl/extractor/indavideo.py
+++ b/youtube_dl/extractor/indavideo.py
@@ -73,7 +73,7 @@ class IndavideoEmbedIE(InfoExtractor):
             'url': self._proto_relative_url(thumbnail)
         } for thumbnail in video.get('thumbnails', [])]
 
-        tags = [tag['title'] for tag in video.get('tags', [])]
+        tags = [tag['title'] for tag in video.get('tags') or []]
 
         return {
             'id': video.get('id') or video_id,

From 3201a67f618f3bb7b13b4df2a9c7f68f481940a4 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 3 Mar 2016 19:18:57 +0100
Subject: [PATCH 4192/6219] [test/test_utils] add more tests for
 update_url_query

---
 test/test_utils.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/test/test_utils.py b/test/test_utils.py
index 19a546619..2bcf8ecf0 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -481,6 +481,15 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(query_dict(update_url_query(
             'http://example.com/path', {'fields': b'id,formats,subtitles'})),
             query_dict('http://example.com/path?fields=id,formats,subtitles'))
+        self.assertEqual(query_dict(update_url_query(
+            'http://example.com/path', {'width': 1080, 'height': 720})),
+            query_dict('http://example.com/path?width=1080&height=720'))
+        self.assertEqual(query_dict(update_url_query(
+            'http://example.com/path', {'bitrate': 5020.43})),
+            query_dict('http://example.com/path?bitrate=5020.43'))
+        self.assertEqual(query_dict(update_url_query(
+            'http://example.com/path', {'test': '第二行тест'})),
+            query_dict('http://example.com/path?test=%E7%AC%AC%E4%BA%8C%E8%A1%8C%D1%82%D0%B5%D1%81%D1%82'))
 
     def test_dict_get(self):
         FALSE_VALUES = {

From e1dd521e4964eae3aa4207c00e1673ab1ea0d4c1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 4 Mar 2016 01:16:58 +0600
Subject: [PATCH 4193/6219] [livestream] Fix FutureWarning (Closes #8742)

---
 youtube_dl/extractor/livestream.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/livestream.py b/youtube_dl/extractor/livestream.py
index 38fb3d9e4..988436226 100644
--- a/youtube_dl/extractor/livestream.py
+++ b/youtube_dl/extractor/livestream.py
@@ -64,7 +64,7 @@ class LivestreamIE(InfoExtractor):
     def _parse_smil_formats(self, smil, smil_url, video_id, namespace=None, f4m_params=None, transform_rtmp_url=None):
         base_ele = find_xpath_attr(
             smil, self._xpath_ns('.//meta', namespace), 'name', 'httpBase')
-        base = base_ele.get('content') if base_ele else 'http://livestreamvod-f.akamaihd.net/'
+        base = base_ele.get('content') if base_ele is not None else 'http://livestreamvod-f.akamaihd.net/'
 
         formats = []
         video_nodes = smil.findall(self._xpath_ns('.//video', namespace))

From 24ca0e9c0b217c99a852682d9aa1357cd380eb06 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 4 Mar 2016 13:36:29 +0800
Subject: [PATCH 4194/6219] [douyutv] Fix tests

---
 youtube_dl/extractor/douyutv.py | 23 ++++++++++++++++++++---
 1 file changed, 20 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/douyutv.py b/youtube_dl/extractor/douyutv.py
index 373b3b4b4..bdc768c78 100644
--- a/youtube_dl/extractor/douyutv.py
+++ b/youtube_dl/extractor/douyutv.py
@@ -18,7 +18,7 @@ class DouyuTVIE(InfoExtractor):
             'display_id': 'iseven',
             'ext': 'flv',
             'title': 're:^清晨醒脑！T-ara根本停不下来！ [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
-            'description': 'md5:c93d6692dde6fe33809a46edcbecca44',
+            'description': 'md5:f34981259a03e980a3c6404190a3ed61',
             'thumbnail': 're:^https?://.*\.jpg$',
             'uploader': '7师傅',
             'uploader_id': '431925',
@@ -26,7 +26,7 @@ class DouyuTVIE(InfoExtractor):
         },
         'params': {
             'skip_download': True,
-        }
+        },
     }, {
         'url': 'http://www.douyutv.com/85982',
         'info_dict': {
@@ -42,7 +42,24 @@ class DouyuTVIE(InfoExtractor):
         },
         'params': {
             'skip_download': True,
-        }
+        },
+        'skip': 'Romm not found',
+    }, {
+        'url': 'http://www.douyutv.com/17732',
+        'info_dict': {
+            'id': '17732',
+            'display_id': '17732',
+            'ext': 'flv',
+            'title': 're:^清晨醒脑！T-ara根本停不下来！ [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
+            'description': 'md5:f34981259a03e980a3c6404190a3ed61',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'uploader': '7师傅',
+            'uploader_id': '431925',
+            'is_live': True,
+        },
+        'params': {
+            'skip_download': True,
+        },
     }]
 
     def _real_extract(self, url):

From 8a370aedace161f97b7bff572275f3e8dd59da2e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 4 Mar 2016 13:38:45 +0800
Subject: [PATCH 4195/6219] [leeco] format_id should be strings

---
 youtube_dl/extractor/leeco.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/leeco.py b/youtube_dl/extractor/leeco.py
index d0cd3f591..9c2bce036 100644
--- a/youtube_dl/extractor/leeco.py
+++ b/youtube_dl/extractor/leeco.py
@@ -338,7 +338,7 @@ class LetvCloudIE(InfoExtractor):
             formats.append({
                 'url': url,
                 'ext': determine_ext(decoded_url),
-                'format_id': int_or_none(play_url.get('vtype')),
+                'format_id': str_or_none(play_url.get('vtype')),
                 'format_note': str_or_none(play_url.get('definition')),
                 'width': int_or_none(play_url.get('vwidth')),
                 'height': int_or_none(play_url.get('vheight')),

From d58cb3ec7e115cf07a6cf6e035cbba2b5c6861fd Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 4 Mar 2016 13:46:38 +0800
Subject: [PATCH 4196/6219] [leeco] Skip an invalid test. test_LePlaylist_1 is
 sufficient

---
 youtube_dl/extractor/leeco.py | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/leeco.py b/youtube_dl/extractor/leeco.py
index 9c2bce036..df47e88ba 100644
--- a/youtube_dl/extractor/leeco.py
+++ b/youtube_dl/extractor/leeco.py
@@ -217,14 +217,8 @@ class LePlaylistIE(InfoExtractor):
         'playlist_mincount': 96
     }, {
         'url': 'http://tv.le.com/pzt/lswjzzjc/index.shtml',
-        'info_dict': {
-            'id': 'lswjzzjc',
-            # The title should be "劲舞青春", but I can't find a simple way to
-            # determine the playlist title
-            'title': '乐视午间自制剧场',
-            'description': 'md5:b1eef244f45589a7b5b1af9ff25a4489'
-        },
-        'playlist_mincount': 7
+        # This series is moved to http://www.le.com/tv/10005297.html
+        'only_matching': True,
     }, {
         'url': 'http://www.le.com/comic/92063.html',
         'only_matching': True,

From b6f94d81ea8d3243edf5a4378760281d9d9c2cba Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 4 Mar 2016 14:32:01 +0800
Subject: [PATCH 4197/6219] [kusi] Add a test for the alternative form of URL

---
 youtube_dl/extractor/kusi.py | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/kusi.py b/youtube_dl/extractor/kusi.py
index 20407411b..5e87c5f79 100644
--- a/youtube_dl/extractor/kusi.py
+++ b/youtube_dl/extractor/kusi.py
@@ -10,7 +10,7 @@ from ..utils import int_or_none
 
 class KUSIIE(InfoExtractor):
     _VALID_URL = r'http://(?:www\.)?kusi\.com/(?P<path>story/.+|video\?clipId=(?P<clipId>\d+))'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.kusi.com/story/31183873/turko-files-case-closed-put-on-hold',
         'md5': 'f926e7684294cf8cb7bdf8858e1b3988',
         'info_dict': {
@@ -18,8 +18,19 @@ class KUSIIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Turko Files: Case Closed! & Put On Hold!',
             'duration': 231000,
-        }
-    }
+        },
+    }, {
+        'url': 'http://kusi.com/video?clipId=12203019',
+        'info_dict': {
+            'id': '12203019',
+            'ext': 'mp4',
+            'title': 'Turko Files: Case Closed! & Put On Hold!',
+            'duration': 231000,
+        },
+        'params': {
+            'skip_download': True,  # Same as previous one
+        },
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)

From 6d6536acb27483087a8a4ff73f501eaca313ec6a Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 4 Mar 2016 10:25:16 +0100
Subject: [PATCH 4198/6219] [fivemin] improve extraction

- skip m3u8 formats(404 error)
- skip unavailable test
- download embed page only when it's needed
- update _VALID_URL regex(joystiq.com redirect to engadget.com)
---
 youtube_dl/extractor/fivemin.py | 51 +++++++++++++++++++--------------
 1 file changed, 30 insertions(+), 21 deletions(-)

diff --git a/youtube_dl/extractor/fivemin.py b/youtube_dl/extractor/fivemin.py
index 2955965d9..67d50a386 100644
--- a/youtube_dl/extractor/fivemin.py
+++ b/youtube_dl/extractor/fivemin.py
@@ -1,5 +1,7 @@
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse,
@@ -16,12 +18,7 @@ from ..utils import (
 
 class FiveMinIE(InfoExtractor):
     IE_NAME = '5min'
-    _VALID_URL = r'''(?x)
-        (?:https?://[^/]*?5min\.com/Scripts/PlayerSeed\.js\?(?:.*?&)?playList=|
-            https?://(?:(?:massively|www)\.)?joystiq\.com/video/|
-            5min:)
-        (?P<id>\d+)
-        '''
+    _VALID_URL = r'(?:5min:(?P<id>\d+)(?::(?P<sid>\d+))?|https?://[^/]*?5min\.com/Scripts/PlayerSeed\.js\?(?P<query>.*))'
 
     _TESTS = [
         {
@@ -45,6 +42,7 @@ class FiveMinIE(InfoExtractor):
                 'title': 'How to Make a Next-Level Fruit Salad',
                 'duration': 184,
             },
+            'skip': 'no longer available',
         },
     ]
     _ERRORS = {
@@ -91,20 +89,33 @@ class FiveMinIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        sid = mobj.group('sid')
+
+        if mobj.group('query'):
+            qs = compat_parse_qs(mobj.group('query'))
+            if not qs.get('playList'):
+                raise ExtractorError('Invalid URL', expected=True)
+            video_id = qs['playList'][0]
+            if qs.get('sid'):
+                sid = qs['sid'][0]
+
         embed_url = 'https://embed.5min.com/playerseed/?playList=%s' % video_id
-        embed_page = self._download_webpage(embed_url, video_id,
-                                            'Downloading embed page')
-        sid = self._search_regex(r'sid=(\d+)', embed_page, 'sid')
-        query = compat_urllib_parse.urlencode({
-            'func': 'GetResults',
-            'playlist': video_id,
-            'sid': sid,
-            'isPlayerSeed': 'true',
-            'url': embed_url,
-        })
+        if not sid:
+            embed_page = self._download_webpage(embed_url, video_id,
+                                                'Downloading embed page')
+            sid = self._search_regex(r'sid=(\d+)', embed_page, 'sid')
+
         response = self._download_json(
-            'https://syn.5min.com/handlers/SenseHandler.ashx?' + query,
+            'https://syn.5min.com/handlers/SenseHandler.ashx?' +
+            compat_urllib_parse.urlencode({
+                'func': 'GetResults',
+                'playlist': video_id,
+                'sid': sid,
+                'isPlayerSeed': 'true',
+                'url': embed_url,
+            }),
             video_id)
         if not response['success']:
             raise ExtractorError(
@@ -118,9 +129,7 @@ class FiveMinIE(InfoExtractor):
         parsed_video_url = compat_urllib_parse_urlparse(compat_parse_qs(
             compat_urllib_parse_urlparse(info['EmbededURL']).query)['videoUrl'][0])
         for rendition in info['Renditions']:
-            if rendition['RenditionType'] == 'm3u8':
-                formats.extend(self._extract_m3u8_formats(rendition['Url'], video_id, m3u8_id='hls'))
-            elif rendition['RenditionType'] == 'aac':
+            if rendition['RenditionType'] == 'aac' or rendition['RenditionType'] == 'm3u8':
                 continue
             else:
                 rendition_url = compat_urlparse.urlunparse(parsed_video_url._replace(path=replace_extension(parsed_video_url.path.replace('//', '/%s/' % rendition['ID']), rendition['RenditionType'])))

From d48352fb5d08208475c0eecf8d88d05d04747041 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 4 Mar 2016 10:40:39 +0100
Subject: [PATCH 4199/6219] [engadget] remove support for legacy urls

---
 youtube_dl/extractor/engadget.py | 25 +++----------------------
 1 file changed, 3 insertions(+), 22 deletions(-)

diff --git a/youtube_dl/extractor/engadget.py b/youtube_dl/extractor/engadget.py
index e4180701d..e5e57d485 100644
--- a/youtube_dl/extractor/engadget.py
+++ b/youtube_dl/extractor/engadget.py
@@ -1,21 +1,13 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
-from ..utils import (
-    url_basename,
-)
 
 
 class EngadgetIE(InfoExtractor):
-    _VALID_URL = r'''(?x)https?://www.engadget.com/
-        (?:video(?:/5min)?/(?P<id>\d+)|
-            [\d/]+/.*?)
-        '''
+    _VALID_URL = r'https?://www.engadget.com/video/(?P<id>\d+)'
 
     _TEST = {
-        'url': 'http://www.engadget.com/video/5min/518153925/',
+        'url': 'http://www.engadget.com/video/518153925/',
         'md5': 'c6820d4828a5064447a4d9fc73f312c9',
         'info_dict': {
             'id': '518153925',
@@ -27,15 +19,4 @@ class EngadgetIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-
-        if video_id is not None:
-            return self.url_result('5min:%s' % video_id)
-        else:
-            title = url_basename(url)
-            webpage = self._download_webpage(url, title)
-            ids = re.findall(r'<iframe[^>]+?playList=(\d+)', webpage)
-            return {
-                '_type': 'playlist',
-                'title': title,
-                'entries': [self.url_result('5min:%s' % vid) for vid in ids]
-            }
+        return self.url_result('5min:%s' % video_id)

From 1d4c9ed90ca8c01fcfdb838bafaa201c04bd599e Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 4 Mar 2016 10:42:58 +0100
Subject: [PATCH 4200/6219] [aol] imporve extraction

- add support for aol features
- remove support for legacy urls
---
 youtube_dl/extractor/__init__.py |  5 ++-
 youtube_dl/extractor/aol.py      | 70 +++++++++++---------------------
 2 files changed, 27 insertions(+), 48 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 037654a23..db1827666 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -23,7 +23,10 @@ from .alphaporno import AlphaPornoIE
 from .animeondemand import AnimeOnDemandIE
 from .anitube import AnitubeIE
 from .anysex import AnySexIE
-from .aol import AolIE
+from .aol import (
+    AolIE,
+    AolFeaturesIE,
+)
 from .allocine import AllocineIE
 from .aparat import AparatIE
 from .appleconnect import AppleConnectIE
diff --git a/youtube_dl/extractor/aol.py b/youtube_dl/extractor/aol.py
index b51eafc45..b761b2cc4 100644
--- a/youtube_dl/extractor/aol.py
+++ b/youtube_dl/extractor/aol.py
@@ -1,24 +1,11 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 
 
 class AolIE(InfoExtractor):
     IE_NAME = 'on.aol.com'
-    _VALID_URL = r'''(?x)
-        (?:
-            aol-video:|
-            http://on\.aol\.com/
-            (?:
-                video/.*-|
-                playlist/(?P<playlist_display_id>[^/?#]+?)-(?P<playlist_id>[0-9]+)[?#].*_videoid=
-            )
-        )
-        (?P<id>[0-9]+)
-        (?:$|\?)
-    '''
+    _VALID_URL = r'(?:aol-video:|http://on\.aol\.com/video/.*-)(?P<id>[0-9]+)(?:$|\?)'
 
     _TESTS = [{
         'url': 'http://on.aol.com/video/u-s--official-warns-of-largest-ever-irs-phone-scam-518167793?icid=OnHomepageC2Wide_MustSee_Img',
@@ -29,42 +16,31 @@ class AolIE(InfoExtractor):
             'title': 'U.S. Official Warns Of \'Largest Ever\' IRS Phone Scam',
         },
         'add_ie': ['FiveMin'],
-    }, {
-        'url': 'http://on.aol.com/playlist/brace-yourself---todays-weirdest-news-152147?icid=OnHomepageC4_Omg_Img#_videoid=518184316',
-        'info_dict': {
-            'id': '152147',
-            'title': 'Brace Yourself - Today\'s Weirdest News',
-        },
-        'playlist_mincount': 10,
     }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        playlist_id = mobj.group('playlist_id')
-        if not playlist_id or self._downloader.params.get('noplaylist'):
-            return self.url_result('5min:%s' % video_id)
+        video_id = self._match_id(url)
+        return self.url_result('5min:%s' % video_id)
 
-        self.to_screen('Downloading playlist %s - add --no-playlist to just download video %s' % (playlist_id, video_id))
 
-        webpage = self._download_webpage(url, playlist_id)
-        title = self._html_search_regex(
-            r'<h1 class="video-title[^"]*">(.+?)</h1>', webpage, 'title')
-        playlist_html = self._search_regex(
-            r"(?s)<ul\s+class='video-related[^']*'>(.*?)</ul>", webpage,
-            'playlist HTML')
-        entries = [{
-            '_type': 'url',
-            'url': 'aol-video:%s' % m.group('id'),
-            'ie_key': 'Aol',
-        } for m in re.finditer(
-            r"<a\s+href='.*videoid=(?P<id>[0-9]+)'\s+class='video-thumb'>",
-            playlist_html)]
+class AolFeaturesIE(InfoExtractor):
+    IE_NAME = 'features.aol.com'
+    _VALID_URL = r'http://features\.aol\.com/video/(?P<id>[^/?#]+)'
 
-        return {
-            '_type': 'playlist',
-            'id': playlist_id,
-            'display_id': mobj.group('playlist_display_id'),
-            'title': title,
-            'entries': entries,
-        }
+    _TESTS = [{
+        'url': 'http://features.aol.com/video/behind-secret-second-careers-late-night-talk-show-hosts',
+        'md5': '7db483bb0c09c85e241f84a34238cc75',
+        'info_dict': {
+            'id': '519507715',
+            'ext': 'mp4',
+            'title': 'What To Watch - February 17, 2016',
+        },
+        'add_ie': ['FiveMin'],
+    }]
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+        return self.url_result(self._search_regex(
+            r'<script type="text/javascript" src="(https?://[^/]*?5min\.com/Scripts/PlayerSeed\.js[^"]+)"',
+            webpage, '5min embed url'), 'FiveMin')

From 5f1688f27106b6501a2a97d7e6fc1d9f079f1f9f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 4 Mar 2016 23:08:47 +0800
Subject: [PATCH 4201/6219] [kusi] Simplify and improve

---
 youtube_dl/extractor/kusi.py | 72 ++++++++++++++++++++++++------------
 1 file changed, 49 insertions(+), 23 deletions(-)

diff --git a/youtube_dl/extractor/kusi.py b/youtube_dl/extractor/kusi.py
index 5e87c5f79..5d458e31e 100644
--- a/youtube_dl/extractor/kusi.py
+++ b/youtube_dl/extractor/kusi.py
@@ -1,11 +1,18 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import random
 import re
 
 from .common import InfoExtractor
 from ..compat import compat_urllib_parse_unquote_plus
-from ..utils import int_or_none
+from ..utils import (
+    int_or_none,
+    float_or_none,
+    timeconvert,
+    update_url_query,
+    xpath_text,
+)
 
 
 class KUSIIE(InfoExtractor):
@@ -17,7 +24,7 @@ class KUSIIE(InfoExtractor):
             'id': '12203019',
             'ext': 'mp4',
             'title': 'Turko Files: Case Closed! & Put On Hold!',
-            'duration': 231000,
+            'duration': 231,
         },
     }, {
         'url': 'http://kusi.com/video?clipId=12203019',
@@ -25,7 +32,10 @@ class KUSIIE(InfoExtractor):
             'id': '12203019',
             'ext': 'mp4',
             'title': 'Turko Files: Case Closed! & Put On Hold!',
-            'duration': 231000,
+            'duration': 231.0,
+            'upload_date': '20160210',
+            'timestamp': 1455087571,
+            'thumbnail': 're:^https?://.*\.jpg$'
         },
         'params': {
             'skip_download': True,  # Same as previous one
@@ -34,33 +44,47 @@ class KUSIIE(InfoExtractor):
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
+        clip_id = mobj.group('clipId')
+        video_id = clip_id or mobj.group('path')
 
-        if mobj.group('clipId') is not None:
-            video_id = mobj.group('clipId')
-        else:
-            webpage = self._download_webpage(url, mobj.group('path'))
-            video_id = self._html_search_regex(r'"clipId", "(\d+)"', webpage,
-                                               'clipId')
+        webpage = self._download_webpage(url, video_id)
 
-        xml_url = 'http://www.kusi.com/build.asp?buildtype=buildfeaturexml'\
-                  'request&featureType=Clip&featureid={0}&affiliateno=956&'\
-                  'clientgroupid=1&rnd=562461'.format(video_id)
-        doc = self._download_xml(xml_url, video_id,
-                                 note='Downloading video info',
-                                 errnote='Failed to download video info')
+        if clip_id is None:
+            video_id = clip_id = self._html_search_regex(
+                r'"clipId"\s*,\s*"(\d+)"', webpage, 'clip id')
 
-        video_title = doc.find('HEADLINE').text
-        duration = int_or_none(doc.find('DURATION'), get_attr='text')
-        description = doc.find('ABSTRACT')
+        affiliate_id = self._search_regex(
+            r'affiliateId\s*:\s*\'([^\']+)\'', webpage, 'affiliate id')
+
+        # See __Packages/worldnow/model/GalleryModel.as of WNGallery.swf
+        xml_url = update_url_query('http://www.kusi.com/build.asp', {
+            'buildtype': 'buildfeaturexmlrequest',
+            'featureType': 'Clip',
+            'featureid': clip_id,
+            'affiliateno': affiliate_id,
+            'clientgroupid': '1',
+            'rnd': int(round(random.random() * 1000000)),
+        })
+
+        doc = self._download_xml(xml_url, video_id)
+
+        video_title = xpath_text(doc, 'HEADLINE')
+        duration = float_or_none(
+            xpath_text(doc, 'DURATION', fatal=False), scale=1000)
+        description = xpath_text(doc, 'ABSTRACT', fatal=False)
+        thumbnail = xpath_text(doc, './THUMBNAILIMAGE/FILENAME', fatal=False)
+        createtion_time = timeconvert(
+            xpath_text(doc, 'rfc822creationdate', fatal=False))
 
         quality_options = doc.find('{http://search.yahoo.com/mrss/}group').findall('{http://search.yahoo.com/mrss/}content')
         formats = []
         for quality in quality_options:
-            if 'height' in quality.attrib:
-                formats.append({
-                    'url': compat_urllib_parse_unquote_plus(quality.attrib['url']),
-                    'height': quality.attrib['height'],
-                })
+            formats.append({
+                'url': compat_urllib_parse_unquote_plus(quality.attrib['url']),
+                'height': int_or_none(quality.attrib.get('height')),
+                'width': int_or_none(quality.attrib.get('width')),
+                'vbr': float_or_none(quality.attrib.get('bitratebits'), scale=1024),
+            })
         self._sort_formats(formats)
 
         return {
@@ -69,4 +93,6 @@ class KUSIIE(InfoExtractor):
             'description': description,
             'duration': duration,
             'formats': formats,
+            'thumbnail': thumbnail,
+            'timestamp': createtion_time,
         }

From ae7d31af1cd1b9c2f1cae86acb8a4156556dc748 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 4 Mar 2016 21:32:54 +0600
Subject: [PATCH 4202/6219] [yandexmusic] Capture and output API errors

---
 youtube_dl/extractor/yandexmusic.py | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/yandexmusic.py b/youtube_dl/extractor/yandexmusic.py
index d3cc1a29f..e699e663f 100644
--- a/youtube_dl/extractor/yandexmusic.py
+++ b/youtube_dl/extractor/yandexmusic.py
@@ -10,13 +10,27 @@ from ..compat import (
     compat_urllib_parse,
 )
 from ..utils import (
+    ExtractorError,
     int_or_none,
     float_or_none,
     sanitized_Request,
 )
 
 
-class YandexMusicTrackIE(InfoExtractor):
+class YandexMusicBaseIE(InfoExtractor):
+    @staticmethod
+    def _handle_error(response):
+        error = response.get('error')
+        if error:
+            raise ExtractorError(error, expected=True)
+
+    def _download_json(self, *args, **kwargs):
+        response = super(YandexMusicBaseIE, self)._download_json(*args, **kwargs)
+        self._handle_error(response)
+        return response
+
+
+class YandexMusicTrackIE(YandexMusicBaseIE):
     IE_NAME = 'yandexmusic:track'
     IE_DESC = 'Яндекс.Музыка - Трек'
     _VALID_URL = r'https?://music\.yandex\.(?:ru|kz|ua|by)/album/(?P<album_id>\d+)/track/(?P<id>\d+)'
@@ -73,7 +87,7 @@ class YandexMusicTrackIE(InfoExtractor):
         return self._get_track_info(track)
 
 
-class YandexMusicPlaylistBaseIE(InfoExtractor):
+class YandexMusicPlaylistBaseIE(YandexMusicBaseIE):
     def _build_playlist(self, tracks):
         return [
             self.url_result(

From fa880d20ad2a2185d9ea691f8a2fa5cfbfbbb1b2 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 4 Mar 2016 23:57:05 +0800
Subject: [PATCH 4203/6219] [kusi] Two fixes

Thanks @dstftw for pointing out those
---
 youtube_dl/extractor/kusi.py | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/kusi.py b/youtube_dl/extractor/kusi.py
index 5d458e31e..f74dbbf76 100644
--- a/youtube_dl/extractor/kusi.py
+++ b/youtube_dl/extractor/kusi.py
@@ -68,13 +68,11 @@ class KUSIIE(InfoExtractor):
 
         doc = self._download_xml(xml_url, video_id)
 
-        video_title = xpath_text(doc, 'HEADLINE')
-        duration = float_or_none(
-            xpath_text(doc, 'DURATION', fatal=False), scale=1000)
-        description = xpath_text(doc, 'ABSTRACT', fatal=False)
-        thumbnail = xpath_text(doc, './THUMBNAILIMAGE/FILENAME', fatal=False)
-        createtion_time = timeconvert(
-            xpath_text(doc, 'rfc822creationdate', fatal=False))
+        video_title = xpath_text(doc, 'HEADLINE', fatal=True)
+        duration = float_or_none(xpath_text(doc, 'DURATION'), scale=1000)
+        description = xpath_text(doc, 'ABSTRACT')
+        thumbnail = xpath_text(doc, './THUMBNAILIMAGE/FILENAME')
+        createtion_time = timeconvert(xpath_text(doc, 'rfc822creationdate'))
 
         quality_options = doc.find('{http://search.yahoo.com/mrss/}group').findall('{http://search.yahoo.com/mrss/}content')
         formats = []
@@ -83,7 +81,7 @@ class KUSIIE(InfoExtractor):
                 'url': compat_urllib_parse_unquote_plus(quality.attrib['url']),
                 'height': int_or_none(quality.attrib.get('height')),
                 'width': int_or_none(quality.attrib.get('width')),
-                'vbr': float_or_none(quality.attrib.get('bitratebits'), scale=1024),
+                'vbr': float_or_none(quality.attrib.get('bitratebits'), scale=1000),
             })
         self._sort_formats(formats)
 

From 28559564b20813c3e3b7206f493cc789a4f1c03d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 5 Mar 2016 00:04:29 +0800
Subject: [PATCH 4204/6219] [kusi] Correct test_KUSI

---
 youtube_dl/extractor/kusi.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/kusi.py b/youtube_dl/extractor/kusi.py
index f74dbbf76..931f34c9b 100644
--- a/youtube_dl/extractor/kusi.py
+++ b/youtube_dl/extractor/kusi.py
@@ -24,7 +24,10 @@ class KUSIIE(InfoExtractor):
             'id': '12203019',
             'ext': 'mp4',
             'title': 'Turko Files: Case Closed! & Put On Hold!',
-            'duration': 231,
+            'duration': 231.0,
+            'upload_date': '20160210',
+            'timestamp': 1455087571,
+            'thumbnail': 're:^https?://.*\.jpg$'
         },
     }, {
         'url': 'http://kusi.com/video?clipId=12203019',

From 7760b9ff4d5e864121811750c0bba78b1d9efdd5 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 4 Mar 2016 17:55:50 +0100
Subject: [PATCH 4205/6219] [audimedia] update _VALID_URL and video_id regex
 and improve http format_id

---
 youtube_dl/extractor/audimedia.py | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/audimedia.py b/youtube_dl/extractor/audimedia.py
index 3b2effa15..aa6925623 100644
--- a/youtube_dl/extractor/audimedia.py
+++ b/youtube_dl/extractor/audimedia.py
@@ -10,9 +10,9 @@ from ..utils import (
 
 
 class AudiMediaIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?audimedia\.tv/(?:en|de)/vid/(?P<id>[^/?#]+)'
+    _VALID_URL = r'https?://(?:www\.)?audi-mediacenter\.com/(?:en|de)/audimediatv/(?P<id>[^/?#]+)'
     _TEST = {
-        'url': 'https://audimedia.tv/en/vid/60-seconds-of-audi-sport-104-2015-wec-bahrain-rookie-test',
+        'url': 'https://www.audi-mediacenter.com/en/audimediatv/60-seconds-of-audi-sport-104-2015-wec-bahrain-rookie-test-1467',
         'md5': '79a8b71c46d49042609795ab59779b66',
         'info_dict': {
             'id': '1565',
@@ -32,7 +32,10 @@ class AudiMediaIE(InfoExtractor):
         display_id = self._match_id(url)
         webpage = self._download_webpage(url, display_id)
 
-        raw_payload = self._search_regex(r'<script[^>]+class="amtv-embed"[^>]+id="([^"]+)"', webpage, 'raw payload')
+        raw_payload = self._search_regex([
+            r'class="amtv-embed"[^>]+id="([^"]+)"',
+            r'class=\\"amtv-embed\\"[^>]+id=\\"([^"]+)\\"',
+        ], webpage, 'raw payload')
         _, stage_mode, video_id, lang = raw_payload.split('-')
 
         # TODO: handle s and e stage_mode (live streams and ended live streams)
@@ -59,13 +62,19 @@ class AudiMediaIE(InfoExtractor):
                 video_version_url = video_version.get('download_url') or video_version.get('stream_url')
                 if not video_version_url:
                     continue
-                formats.append({
+                f = {
                     'url': video_version_url,
                     'width': int_or_none(video_version.get('width')),
                     'height': int_or_none(video_version.get('height')),
                     'abr': int_or_none(video_version.get('audio_bitrate')),
                     'vbr': int_or_none(video_version.get('video_bitrate')),
-                })
+                }
+                bitrate = self._search_regex(r'(\d+)k', video_version_url, 'bitrate', default=None)
+                if bitrate:
+                    f.update({
+                        'format_id': 'http-%s' % bitrate,
+                    })
+                formats.append(f)
             self._sort_formats(formats)
 
             return {

From 1315224cbbf2dded21a53e880d58bd32f17ed3f0 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 4 Mar 2016 20:14:09 +0100
Subject: [PATCH 4206/6219] [bleacherreport] update tests

---
 youtube_dl/extractor/bleacherreport.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/bleacherreport.py b/youtube_dl/extractor/bleacherreport.py
index 38bda3af5..7a8e1f60b 100644
--- a/youtube_dl/extractor/bleacherreport.py
+++ b/youtube_dl/extractor/bleacherreport.py
@@ -28,10 +28,10 @@ class BleacherReportIE(InfoExtractor):
         'add_ie': ['Ooyala'],
     }, {
         'url': 'http://bleacherreport.com/articles/2586817-aussie-golfers-get-fright-of-their-lives-after-being-chased-by-angry-kangaroo',
-        'md5': 'af5f90dc9c7ba1c19d0a3eac806bbf50',
+        'md5': '6a5cd403418c7b01719248ca97fb0692',
         'info_dict': {
             'id': '2586817',
-            'ext': 'mp4',
+            'ext': 'webm',
             'title': 'Aussie Golfers Get Fright of Their Lives After Being Chased by Angry Kangaroo',
             'timestamp': 1446839961,
             'uploader': 'Sean Fay',
@@ -93,10 +93,14 @@ class BleacherReportCMSIE(AMPIE):
         'md5': '8c2c12e3af7805152675446c905d159b',
         'info_dict': {
             'id': '8fd44c2f-3dc5-4821-9118-2c825a98c0e1',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Cena vs. Rollins Would Expose the Heavyweight Division',
             'description': 'md5:984afb4ade2f9c0db35f3267ed88b36e',
         },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
     }]
 
     def _real_extract(self, url):

From d627cec60849d313ad7d39204ef7f59f695b6bca Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 5 Mar 2016 01:50:12 +0600
Subject: [PATCH 4207/6219] [youporn] Fix quality extraction (Closes #8758)

---
 youtube_dl/extractor/youporn.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youporn.py b/youtube_dl/extractor/youporn.py
index b29baafc4..567a8c5e6 100644
--- a/youtube_dl/extractor/youporn.py
+++ b/youtube_dl/extractor/youporn.py
@@ -101,8 +101,9 @@ class YouPornIE(InfoExtractor):
             }
             # Video URL's path looks like this:
             #  /201012/17/505835/720p_1500k_505835/YouPorn%20-%20Sex%20Ed%20Is%20It%20Safe%20To%20Masturbate%20Daily.mp4
+            #  /201012/17/505835/vl_240p_240k_505835/YouPorn%20-%20Sex%20Ed%20Is%20It%20Safe%20To%20Masturbate%20Daily.mp4
             # We will benefit from it by extracting some metadata
-            mobj = re.search(r'/(?P<height>\d{3,4})[pP]_(?P<bitrate>\d+)[kK]_\d+/', video_url)
+            mobj = re.search(r'(?P<height>\d{3,4})[pP]_(?P<bitrate>\d+)[kK]_\d+/', video_url)
             if mobj:
                 height = int(mobj.group('height'))
                 bitrate = int(mobj.group('bitrate'))

From 002c75524854033f02d03fa854a45d0fcd3ac74c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 5 Mar 2016 01:51:27 +0600
Subject: [PATCH 4208/6219] [youporn] Fix sources regex

---
 youtube_dl/extractor/youporn.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youporn.py b/youtube_dl/extractor/youporn.py
index 567a8c5e6..1124fe6c2 100644
--- a/youtube_dl/extractor/youporn.py
+++ b/youtube_dl/extractor/youporn.py
@@ -75,7 +75,7 @@ class YouPornIE(InfoExtractor):
         links = []
 
         sources = self._search_regex(
-            r'sources\s*:\s*({.+?})', webpage, 'sources', default=None)
+            r'(?s)sources\s*:\s*({.+?})', webpage, 'sources', default=None)
         if sources:
             for _, link in re.findall(r'[^:]+\s*:\s*(["\'])(http.+?)\1', sources):
                 links.append(link)

From cf074e5ddd3840071df78ec509bcd74520a8f425 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 4 Mar 2016 21:42:04 +0100
Subject: [PATCH 4209/6219] [foxnews] update test

---
 youtube_dl/extractor/foxnews.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/foxnews.py b/youtube_dl/extractor/foxnews.py
index 318ac013d..1dc50318c 100644
--- a/youtube_dl/extractor/foxnews.py
+++ b/youtube_dl/extractor/foxnews.py
@@ -36,6 +36,10 @@ class FoxNewsIE(AMPIE):
                 # 'upload_date': '20141204',
                 'thumbnail': 're:^https?://.*\.jpg$',
             },
+            'params': {
+                # m3u8 download
+                'skip_download': True,
+            },
         },
         {
             'url': 'http://video.foxnews.com/v/video-embed.html?video_id=3937480&d=video.foxnews.com',

From 3233a68fbb1cb608534db43f358f7203e59cd2de Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 4 Mar 2016 22:18:40 +0100
Subject: [PATCH 4210/6219] [utils] update_url_query: Encode the strings in the
 query dict
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The test case with {'test': '第二行тест'} was failing on python 2 (the non-ascii characters were replaced with '?').
---
 youtube_dl/utils.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index d431aa6b7..22a39a0ab 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1743,6 +1743,7 @@ def update_url_query(url, query):
     parsed_url = compat_urlparse.urlparse(url)
     qs = compat_parse_qs(parsed_url.query)
     qs.update(query)
+    qs = encode_dict(qs)
     return compat_urlparse.urlunparse(parsed_url._replace(
         query=compat_urllib_parse.urlencode(qs, True)))
 

From 845817aadfa5f811afe5834c0b34f50f6c3a2a67 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 5 Mar 2016 18:14:58 +0800
Subject: [PATCH 4211/6219] [twitter] Provide more metadata

---
 youtube_dl/extractor/twitter.py | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 67762a003..e70b2ab3c 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -102,6 +102,14 @@ class TwitterCardIE(TwitterBaseIE):
             r'data-(?:player-)?config="([^"]+)"', webpage, 'data player config'),
             video_id)
 
+        def _search_dimensions_in_video_url(a_format, video_url):
+            m = re.search(r'/(?P<width>\d+)x(?P<height>\d+)/', video_url)
+            if m:
+                a_format.update({
+                    'width': int(m.group('width')),
+                    'height': int(m.group('height')),
+                })
+
         playlist = config.get('playlist')
         if playlist:
             video_url = playlist[0]['source']
@@ -110,12 +118,8 @@ class TwitterCardIE(TwitterBaseIE):
                 'url': video_url,
             }
 
-            m = re.search(r'/(?P<width>\d+)x(?P<height>\d+)/', video_url)
-            if m:
-                f.update({
-                    'width': int(m.group('width')),
-                    'height': int(m.group('height')),
-                })
+            _search_dimensions_in_video_url(f, video_url)
+
             formats.append(f)
 
         vmap_url = config.get('vmapUrl') or config.get('vmap_url')
@@ -148,6 +152,8 @@ class TwitterCardIE(TwitterBaseIE):
                     if not a_format['vbr']:
                         del a_format['vbr']
 
+                    _search_dimensions_in_video_url(a_format, media_url)
+
                     formats.append(a_format)
 
             duration = float_or_none(media_info.get('duration', {}).get('nanos'), scale=1e9)

From 19a17d462388b5d10196352e5f9d1491d608783a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 5 Mar 2016 18:18:28 +0800
Subject: [PATCH 4212/6219] [utils] Add codec2ext

---
 youtube_dl/utils.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 22a39a0ab..dc0bf5627 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1893,6 +1893,19 @@ def mimetype2ext(mt):
     }.get(res, res)
 
 
+def codec2ext(codec):
+    codec_type = codec.split('.')[0]
+
+    # Leave the return value None for unknown values as codec_type
+    # is not a good fallback for file extensions
+    return {
+        'avc1': 'mp4',
+        'mp4a': 'm4a',
+        'vorbis': 'webm',
+        'vp9': 'webm',
+    }.get(codec_type)
+
+
 def urlhandle_detect_ext(url_handle):
     try:
         url_handle.headers

From 2def60c5f3426e4f6b7e5f0bc8a92811e5a592b9 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 5 Mar 2016 18:18:39 +0800
Subject: [PATCH 4213/6219] [common] Use codec2ext for DASH formats (#8764)

---
 youtube_dl/extractor/common.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 07bd2cbe2..f337caf20 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -29,6 +29,7 @@ from ..utils import (
     age_restricted,
     bug_reports_message,
     clean_html,
+    codec2ext,
     compiled_regex_type,
     determine_ext,
     error_to_compat_str,
@@ -1471,6 +1472,7 @@ class InfoExtractor(object):
                         f = {
                             'format_id': '%s-%s' % (mpd_id, representation_id) if mpd_id else representation_id,
                             'url': base_url,
+                            'ext': codec2ext(representation_attrib.get('codecs')),
                             'width': int_or_none(representation_attrib.get('width')),
                             'height': int_or_none(representation_attrib.get('height')),
                             'tbr': int_or_none(representation_attrib.get('bandwidth'), 1000),

From d1b29d13421fd91093fdf3699aea17d8ba4b636b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 5 Mar 2016 16:43:29 +0600
Subject: [PATCH 4214/6219] [elpais] Add support for alternative layout (Closes
 #8744)

---
 youtube_dl/extractor/elpais.py | 31 +++++++++++++++++++++----------
 1 file changed, 21 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/elpais.py b/youtube_dl/extractor/elpais.py
index 00a69e631..8c725a4e6 100644
--- a/youtube_dl/extractor/elpais.py
+++ b/youtube_dl/extractor/elpais.py
@@ -9,7 +9,7 @@ class ElPaisIE(InfoExtractor):
     _VALID_URL = r'https?://(?:[^.]+\.)?elpais\.com/.*/(?P<id>[^/#?]+)\.html(?:$|[?#])'
     IE_DESC = 'El País'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://blogs.elpais.com/la-voz-de-inaki/2014/02/tiempo-nuevo-recetas-viejas.html',
         'md5': '98406f301f19562170ec071b83433d55',
         'info_dict': {
@@ -19,30 +19,41 @@ class ElPaisIE(InfoExtractor):
             'description': 'De lunes a viernes, a partir de las ocho de la mañana, Iñaki Gabilondo nos cuenta su visión de la actualidad nacional e internacional.',
             'upload_date': '20140206',
         }
-    }
+    }, {
+        'url': 'http://elcomidista.elpais.com/elcomidista/2016/02/24/articulo/1456340311_668921.html#?id_externo_nwl=newsletter_diaria20160303t',
+        'md5': '3bd5b09509f3519d7d9e763179b013de',
+        'info_dict': {
+            'id': '1456340311_668921',
+            'ext': 'mp4',
+            'title': 'Cómo hacer el mejor café con cafetera italiana',
+            'description': 'Que sí, que las cápsulas son cómodas. Pero si le pides algo más a la vida, quizá deberías aprender a usar bien la cafetera italiana. No tienes más que ver este vídeo y seguir sus siete normas básicas.',
+            'upload_date': '20160303',
+        }
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
         prefix = self._html_search_regex(
-            r'var url_cache = "([^"]+)";', webpage, 'URL prefix')
+            r'var\s+url_cache\s*=\s*"([^"]+)";', webpage, 'URL prefix')
         video_suffix = self._search_regex(
-            r"URLMediaFile = url_cache \+ '([^']+)'", webpage, 'video URL')
+            r"(?:URLMediaFile|urlVideo_\d+)\s*=\s*url_cache\s*\+\s*'([^']+)'", webpage, 'video URL')
         video_url = prefix + video_suffix
         thumbnail_suffix = self._search_regex(
-            r"URLMediaStill = url_cache \+ '([^']+)'", webpage, 'thumbnail URL',
-            fatal=False)
+            r"(?:URLMediaStill|urlFotogramaFijo_\d+)\s*=\s*url_cache\s*\+\s*'([^']+)'",
+            webpage, 'thumbnail URL', fatal=False)
         thumbnail = (
             None if thumbnail_suffix is None
             else prefix + thumbnail_suffix)
         title = self._html_search_regex(
-            '<h2 class="entry-header entry-title.*?>(.*?)</h2>',
+            (r"tituloVideo\s*=\s*'([^']+)'", webpage, 'title',
+             r'<h2 class="entry-header entry-title.*?>(.*?)</h2>'),
             webpage, 'title')
-        date_str = self._search_regex(
+        upload_date = unified_strdate(self._search_regex(
             r'<p class="date-header date-int updated"\s+title="([^"]+)">',
-            webpage, 'upload date', fatal=False)
-        upload_date = (None if date_str is None else unified_strdate(date_str))
+            webpage, 'upload date', default=None) or self._html_search_meta(
+            'datePublished', webpage, 'timestamp'))
 
         return {
             'id': video_id,

From 18b0b23992b5ee4042967b63e366c937484e434e Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 5 Mar 2016 12:14:48 +0100
Subject: [PATCH 4215/6219] [revision3] add support pages of type embed

---
 youtube_dl/extractor/revision3.py | 66 +++++++++++++++++++++++--------
 1 file changed, 49 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/revision3.py b/youtube_dl/extractor/revision3.py
index b1b8800b9..fb2196ad6 100644
--- a/youtube_dl/extractor/revision3.py
+++ b/youtube_dl/extractor/revision3.py
@@ -52,6 +52,20 @@ class Revision3IE(InfoExtractor):
             'uploader': 'DNews',
             'uploader_id': 'dnews',
         },
+    }, {
+        'url': 'http://testtube.com/tt-editors-picks/the-israel-palestine-conflict-explained-in-ten-min',
+        'info_dict': {
+            'id': '73573',
+            'ext': 'mp4',
+            'display_id': 'tt-editors-picks/the-israel-palestine-conflict-explained-in-ten-min',
+            'title': 'The Israel-Palestine Conflict Explained in Ten Minutes',
+            'description': 'If you\'d like to learn about the struggle between Israelis and Palestinians, this video is a great place to start',
+            'uploader': 'Editors\' Picks',
+            'uploader_id': 'tt-editors-picks',
+            'timestamp': 1453309200,
+            'upload_date': '20160120',
+        },
+        'add_ie': ['Youtube'],
     }]
     _PAGE_DATA_TEMPLATE = 'http://www.%s/apiProxy/ddn/%s?domain=%s'
     _API_KEY = 'ba9c741bce1b9d8e3defcc22193f3651b8867e62'
@@ -61,9 +75,38 @@ class Revision3IE(InfoExtractor):
         page_info = self._download_json(
             self._PAGE_DATA_TEMPLATE % (domain, display_id, domain), display_id)
 
-        if page_info['data']['type'] == 'episode':
-            episode_data = page_info['data']
-            video_id = compat_str(episode_data['video']['data']['id'])
+        page_data = page_info['data']
+        page_type = page_data['type']
+        if page_type == 'episode' or page_type == 'embed':
+            show_data = page_data['show']['data']
+            video_id = compat_str(page_data['video']['data']['id'])
+
+            preference = qualities(['mini', 'small', 'medium', 'large'])
+            thumbnails = [{
+                'url': image_url,
+                'id': image_id,
+                'preference': preference(image_id)
+            } for image_id, image_url in page_data.get('images', {}).items()]
+
+            info = {
+                'id': video_id,
+                'display_id': display_id,
+                'title': unescapeHTML(page_data['name']),
+                'description': unescapeHTML(page_data.get('summary')),
+                'timestamp': parse_iso8601(page_data.get('publishTime'), ' '),
+                'author': page_data.get('author'),
+                'uploader': show_data.get('name'),
+                'uploader_id': show_data.get('slug'),
+                'thumbnails': thumbnails,
+            }
+
+            if page_type == 'embed':
+                info.update({
+                    '_type': 'url_transparent',
+                    'url': page_data['video']['data']['embed'],
+                })
+                return info
+
             video_data = self._download_json(
                 'http://revision3.com/api/getPlaylist.json?api_key=%s&codecs=h264,vp8,theora&video_id=%s' % (self._API_KEY, video_id),
                 video_id)['items'][0]
@@ -84,26 +127,15 @@ class Revision3IE(InfoExtractor):
                         })
             self._sort_formats(formats)
 
-            preference = qualities(['mini', 'small', 'medium', 'large'])
-            thumbnails = [{
-                'url': image_url,
-                'id': image_id,
-                'preference': preference(image_id)
-            } for image_id, image_url in video_data.get('images', {}).items()]
-
-            return {
-                'id': video_id,
-                'display_id': display_id,
+            info.update({
                 'title': unescapeHTML(video_data['title']),
                 'description': unescapeHTML(video_data.get('summary')),
-                'timestamp': parse_iso8601(episode_data.get('publishTime'), ' '),
-                'author': episode_data.get('author'),
                 'uploader': video_data.get('show', {}).get('name'),
                 'uploader_id': video_data.get('show', {}).get('slug'),
                 'duration': int_or_none(video_data.get('duration')),
-                'thumbnails': thumbnails,
                 'formats': formats,
-            }
+            })
+            return info
         else:
             show_data = page_info['show']['data']
             episodes_data = page_info['episodes']['data']

From 107d0c421a35083fd2412de5dce558fc16f337e7 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 5 Mar 2016 13:43:29 +0100
Subject: [PATCH 4216/6219] [revision3] add support for pages of type tag

---
 youtube_dl/extractor/revision3.py | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/revision3.py b/youtube_dl/extractor/revision3.py
index fb2196ad6..ffb84628e 100644
--- a/youtube_dl/extractor/revision3.py
+++ b/youtube_dl/extractor/revision3.py
@@ -31,6 +31,7 @@ class Revision3IE(InfoExtractor):
             'uploader_id': 'technobuffalo',
         }
     }, {
+        # Show
         'url': 'http://testtube.com/brainstuff',
         'info_dict': {
             'id': '251',
@@ -66,6 +67,14 @@ class Revision3IE(InfoExtractor):
             'upload_date': '20160120',
         },
         'add_ie': ['Youtube'],
+    }, {
+        # Tag
+        'url': 'http://testtube.com/tech-news',
+        'info_dict': {
+            'id': '21018',
+            'title': 'tech news',
+        },
+        'playlist_mincount': 9,
     }]
     _PAGE_DATA_TEMPLATE = 'http://www.%s/apiProxy/ddn/%s?domain=%s'
     _API_KEY = 'ba9c741bce1b9d8e3defcc22193f3651b8867e62'
@@ -77,7 +86,7 @@ class Revision3IE(InfoExtractor):
 
         page_data = page_info['data']
         page_type = page_data['type']
-        if page_type == 'episode' or page_type == 'embed':
+        if page_type in ('episode', 'embed'):
             show_data = page_data['show']['data']
             video_id = compat_str(page_data['video']['data']['id'])
 
@@ -137,7 +146,7 @@ class Revision3IE(InfoExtractor):
             })
             return info
         else:
-            show_data = page_info['show']['data']
+            list_data = page_info[page_type]['data']
             episodes_data = page_info['episodes']['data']
             num_episodes = page_info['meta']['totalEpisodes']
             processed_episodes = 0
@@ -145,7 +154,7 @@ class Revision3IE(InfoExtractor):
             page_num = 1
             while True:
                 entries.extend([self.url_result(
-                    'http://%s/%s/%s' % (domain, display_id, episode['slug'])) for episode in episodes_data])
+                    'http://%s%s' % (domain, episode['path'])) for episode in episodes_data])
                 processed_episodes += len(episodes_data)
                 if processed_episodes == num_episodes:
                     break
@@ -155,5 +164,5 @@ class Revision3IE(InfoExtractor):
                     display_id)['episodes']['data']
 
             return self.playlist_result(
-                entries, compat_str(show_data['id']),
-                show_data.get('name'), show_data.get('summary'))
+                entries, compat_str(list_data['id']),
+                list_data.get('name'), list_data.get('summary'))

From 51ce91174b9aefa5cb6b98bb0d7b03de09c30e00 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 5 Mar 2016 19:38:58 +0600
Subject: [PATCH 4217/6219] [YoutubeDL] Fix resolution with missing height in
 output template dict

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index dcc867e45..f91851df9 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -567,7 +567,7 @@ class YoutubeDL(object):
                 elif template_dict.get('height'):
                     template_dict['resolution'] = '%sp' % template_dict['height']
                 elif template_dict.get('width'):
-                    template_dict['resolution'] = '?x%d' % template_dict['width']
+                    template_dict['resolution'] = '%dx?' % template_dict['width']
 
             sanitize = lambda k, v: sanitize_filename(
                 compat_str(v),

From 2629a3802c019168340b9345b21b656f48e360fc Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 5 Mar 2016 15:42:15 +0100
Subject: [PATCH 4218/6219] [revison3] fix video_id for --download-archive

---
 youtube_dl/extractor/revision3.py | 20 ++++++++++++++------
 1 file changed, 14 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/revision3.py b/youtube_dl/extractor/revision3.py
index ffb84628e..99979ebe1 100644
--- a/youtube_dl/extractor/revision3.py
+++ b/youtube_dl/extractor/revision3.py
@@ -19,7 +19,7 @@ class Revision3IE(InfoExtractor):
         'url': 'http://www.revision3.com/technobuffalo/5-google-predictions-for-2016',
         'md5': 'd94a72d85d0a829766de4deb8daaf7df',
         'info_dict': {
-            'id': '73034',
+            'id': '71089',
             'display_id': 'technobuffalo/5-google-predictions-for-2016',
             'ext': 'webm',
             'title': '5 Google Predictions for 2016',
@@ -42,7 +42,7 @@ class Revision3IE(InfoExtractor):
     }, {
         'url': 'https://testtube.com/dnews/5-weird-ways-plants-can-eat-animals?utm_source=FB&utm_medium=DNews&utm_campaign=DNewsSocial',
         'info_dict': {
-            'id': '60163',
+            'id': '58227',
             'display_id': 'dnews/5-weird-ways-plants-can-eat-animals',
             'duration': 275,
             'ext': 'webm',
@@ -56,7 +56,7 @@ class Revision3IE(InfoExtractor):
     }, {
         'url': 'http://testtube.com/tt-editors-picks/the-israel-palestine-conflict-explained-in-ten-min',
         'info_dict': {
-            'id': '73573',
+            'id': '71618',
             'ext': 'mp4',
             'display_id': 'tt-editors-picks/the-israel-palestine-conflict-explained-in-ten-min',
             'title': 'The Israel-Palestine Conflict Explained in Ten Minutes',
@@ -81,6 +81,7 @@ class Revision3IE(InfoExtractor):
 
     def _real_extract(self, url):
         domain, display_id = re.match(self._VALID_URL, url).groups()
+        site = domain.split('.')[0]
         page_info = self._download_json(
             self._PAGE_DATA_TEMPLATE % (domain, display_id, domain), display_id)
 
@@ -88,6 +89,7 @@ class Revision3IE(InfoExtractor):
         page_type = page_data['type']
         if page_type in ('episode', 'embed'):
             show_data = page_data['show']['data']
+            page_id = compat_str(page_data['id'])
             video_id = compat_str(page_data['video']['data']['id'])
 
             preference = qualities(['mini', 'small', 'medium', 'large'])
@@ -98,7 +100,7 @@ class Revision3IE(InfoExtractor):
             } for image_id, image_url in page_data.get('images', {}).items()]
 
             info = {
-                'id': video_id,
+                'id': page_id,
                 'display_id': display_id,
                 'title': unescapeHTML(page_data['name']),
                 'description': unescapeHTML(page_data.get('summary')),
@@ -107,6 +109,7 @@ class Revision3IE(InfoExtractor):
                 'uploader': show_data.get('name'),
                 'uploader_id': show_data.get('slug'),
                 'thumbnails': thumbnails,
+                'extractor_key': site,
             }
 
             if page_type == 'embed':
@@ -153,8 +156,13 @@ class Revision3IE(InfoExtractor):
             entries = []
             page_num = 1
             while True:
-                entries.extend([self.url_result(
-                    'http://%s%s' % (domain, episode['path'])) for episode in episodes_data])
+                entries.extend([{
+                    '_type': 'url',
+                    'url': 'http://%s%s' % (domain, episode['path']),
+                    'id': compat_str(episode['id']),
+                    'ie_key': 'Revision3',
+                    'extractor_key': site,
+                } for episode in episodes_data])
                 processed_episodes += len(episodes_data)
                 if processed_episodes == num_episodes:
                     break

From 1e501f6c40cc5e1bd84c0432a323f87f49e18ad0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 5 Mar 2016 21:01:43 +0600
Subject: [PATCH 4219/6219] [jeuxvideo] Fix config URL extraction (Closes
 #8774)

---
 youtube_dl/extractor/jeuxvideo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/jeuxvideo.py b/youtube_dl/extractor/jeuxvideo.py
index eef7daa29..137db873c 100644
--- a/youtube_dl/extractor/jeuxvideo.py
+++ b/youtube_dl/extractor/jeuxvideo.py
@@ -30,7 +30,7 @@ class JeuxVideoIE(InfoExtractor):
         webpage = self._download_webpage(url, title)
         title = self._html_search_meta('name', webpage) or self._og_search_title(webpage)
         config_url = self._html_search_regex(
-            r'data-src="(/contenu/medias/video.php.*?)"',
+            r'data-src(?:set-video)?="(/contenu/medias/video.php.*?)"',
             webpage, 'config URL')
         config_url = 'http://www.jeuxvideo.com' + config_url
 

From 7177fd24f8ed262024501c1020e5791c811fbfe4 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 5 Mar 2016 17:51:46 +0100
Subject: [PATCH 4220/6219] [vgtv] support ap.vgtv.no and fix old videos
 extraction(fixes #8719)

---
 youtube_dl/extractor/vgtv.py | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/vgtv.py b/youtube_dl/extractor/vgtv.py
index 14e945d49..e148b1ef5 100644
--- a/youtube_dl/extractor/vgtv.py
+++ b/youtube_dl/extractor/vgtv.py
@@ -20,6 +20,7 @@ class VGTVIE(XstreamIE):
         'aftenbladet.no/tv': 'satv',
         'fvn.no/fvntv': 'fvntv',
         'aftenposten.no/webtv': 'aptv',
+        'ap.vgtv.no/webtv': 'aptv',
     }
 
     _APP_NAME_TO_VENDOR = {
@@ -35,7 +36,7 @@ class VGTVIE(XstreamIE):
                     (?P<host>
                         %s
                     )
-                    /
+                    /?
                     (?:
                         \#!/(?:video|live)/|
                         embed?.*id=
@@ -107,19 +108,27 @@ class VGTVIE(XstreamIE):
             'md5': 'fd828cd29774a729bf4d4425fe192972',
             'info_dict': {
                 'id': '21039',
-                'ext': 'mov',
+                'ext': 'mp4',
                 'title': 'TRAILER: «SWEATSHOP» - I can´t take any more',
                 'description': 'md5:21891f2b0dd7ec2f78d84a50e54f8238',
                 'duration': 66,
                 'timestamp': 1417002452,
                 'upload_date': '20141126',
                 'view_count': int,
-            }
+            },
+            'params': {
+                # m3u8 download
+                'skip_download': True,
+            },
         },
         {
             'url': 'http://www.bt.no/tv/#!/video/100250/norling-dette-er-forskjellen-paa-1-divisjon-og-eliteserien',
             'only_matching': True,
         },
+        {
+            'url': 'http://ap.vgtv.no/webtv#!/video/111084/de-nye-bysyklene-lettere-bedre-gir-stoerre-hjul-og-feste-til-mobil',
+            'only_matching': True,
+        },
     ]
 
     def _real_extract(self, url):
@@ -144,8 +153,6 @@ class VGTVIE(XstreamIE):
         if len(video_id) == 5:
             if appname == 'bttv':
                 info = self._extract_video_info('btno', video_id)
-            elif appname == 'aptv':
-                info = self._extract_video_info('ap', video_id)
 
         streams = data['streamUrls']
         stream_type = data.get('streamType')

From 75313f2baaec17062d60255976a8afd0fcf995d4 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 5 Mar 2016 21:10:00 +0100
Subject: [PATCH 4221/6219] [cnet] fix info extraction

---
 youtube_dl/extractor/cnet.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/cnet.py b/youtube_dl/extractor/cnet.py
index 5c3908f72..3cf0bf95b 100644
--- a/youtube_dl/extractor/cnet.py
+++ b/youtube_dl/extractor/cnet.py
@@ -51,9 +51,7 @@ class CNETIE(ThePlatformIE):
             uploader = None
             uploader_id = None
 
-        mpx_account = data['config']['uvpConfig']['default']['mpx_account']
-
-        metadata = self.get_metadata('%s/%s' % (mpx_account, list(vdata['files'].values())[0]), video_id)
+        metadata = self.get_metadata('kYEXFC/%s' % list(vdata['files'].values())[0], video_id)
         description = vdata.get('description') or metadata.get('description')
         duration = int_or_none(vdata.get('duration')) or metadata.get('duration')
 
@@ -62,7 +60,7 @@ class CNETIE(ThePlatformIE):
         for (fkey, vid) in vdata['files'].items():
             if fkey == 'hls_phone' and 'hls_tablet' in vdata['files']:
                 continue
-            release_url = 'http://link.theplatform.com/s/%s/%s?format=SMIL&mbr=true' % (mpx_account, vid)
+            release_url = 'http://link.theplatform.com/s/kYEXFC/%s?format=SMIL&mbr=true' % vid
             if fkey == 'hds':
                 release_url += '&manifest=f4m'
             tp_formats, tp_subtitles = self._extract_theplatform_smil(release_url, video_id, 'Downloading %s SMIL data' % fkey)

From 1db772673e6345d830e1db3470ade0d234515509 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 5 Mar 2016 21:34:34 +0100
Subject: [PATCH 4222/6219] [cinemassacre] update tests

---
 youtube_dl/extractor/cinemassacre.py | 24 ++++++++++++++++++------
 1 file changed, 18 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/cinemassacre.py b/youtube_dl/extractor/cinemassacre.py
index 6d9cd8abd..042c4f2f1 100644
--- a/youtube_dl/extractor/cinemassacre.py
+++ b/youtube_dl/extractor/cinemassacre.py
@@ -21,6 +21,10 @@ class CinemassacreIE(InfoExtractor):
                 'title': '“Angry Video Game Nerd: The Movie” – Trailer',
                 'description': 'md5:fb87405fcb42a331742a0dce2708560b',
             },
+            'params': {
+                # m3u8 download
+                'skip_download': True,
+            },
         },
         {
             'url': 'http://cinemassacre.com/2013/10/02/the-mummys-hand-1940',
@@ -31,14 +35,18 @@ class CinemassacreIE(InfoExtractor):
                 'upload_date': '20131002',
                 'title': 'The Mummy’s Hand (1940)',
             },
+            'params': {
+                # m3u8 download
+                'skip_download': True,
+            },
         },
         {
             # Youtube embedded video
             'url': 'http://cinemassacre.com/2006/12/07/chronologically-confused-about-bad-movie-and-video-game-sequel-titles/',
-            'md5': 'df4cf8a1dcedaec79a73d96d83b99023',
+            'md5': 'ec9838a5520ef5409b3e4e42fcb0a3b9',
             'info_dict': {
                 'id': 'OEVzPCY2T-g',
-                'ext': 'mp4',
+                'ext': 'webm',
                 'title': 'AVGN: Chronologically Confused about Bad Movie and Video Game Sequel Titles',
                 'upload_date': '20061207',
                 'uploader': 'Cinemassacre',
@@ -49,12 +57,12 @@ class CinemassacreIE(InfoExtractor):
         {
             # Youtube embedded video
             'url': 'http://cinemassacre.com/2006/09/01/mckids/',
-            'md5': '6eb30961fa795fedc750eac4881ad2e1',
+            'md5': '7393c4e0f54602ad110c793eb7a6513a',
             'info_dict': {
                 'id': 'FnxsNhuikpo',
-                'ext': 'mp4',
+                'ext': 'webm',
                 'upload_date': '20060901',
-                'uploader': 'Cinemassacre Extras',
+                'uploader': 'Cinemassacre Extra',
                 'description': 'md5:de9b751efa9e45fbaafd9c8a1123ed53',
                 'uploader_id': 'Cinemassacre',
                 'title': 'AVGN: McKids',
@@ -69,7 +77,11 @@ class CinemassacreIE(InfoExtractor):
                 'description': 'Let’s Play Mario Kart 64 !! Mario Kart 64 is a classic go-kart racing game released for the Nintendo 64 (N64). Today James & Mike do 4 player Battle Mode with Kyle and Bootsy!',
                 'title': 'Mario Kart 64 (Nintendo 64) James & Mike Mondays',
                 'upload_date': '20150525',
-            }
+            },
+            'params': {
+                # m3u8 download
+                'skip_download': True,
+            },
         }
     ]
 

From f22ba4bd60e9b4deeb1ca3e1f8162e607dc9648e Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 5 Mar 2016 21:52:24 +0100
Subject: [PATCH 4223/6219] update tests related to the change in youtube http
 format sorting

the change was done in 82156fdbf0913c75181484dcc813565713bf78e9
---
 youtube_dl/extractor/freespeech.py  | 2 +-
 youtube_dl/extractor/khanacademy.py | 4 ++--
 youtube_dl/extractor/mit.py         | 2 +-
 youtube_dl/extractor/pyvideo.py     | 6 +++---
 youtube_dl/extractor/ted.py         | 2 +-
 youtube_dl/extractor/viki.py        | 6 +++---
 youtube_dl/extractor/vk.py          | 4 ++--
 youtube_dl/extractor/wimp.py        | 2 +-
 8 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/freespeech.py b/youtube_dl/extractor/freespeech.py
index c210177f7..1477708bb 100644
--- a/youtube_dl/extractor/freespeech.py
+++ b/youtube_dl/extractor/freespeech.py
@@ -14,7 +14,7 @@ class FreespeechIE(InfoExtractor):
         'url': 'https://www.freespeech.org/video/obama-romney-campaign-colorado-ahead-debate-0',
         'info_dict': {
             'id': 'poKsVCZ64uU',
-            'ext': 'mp4',
+            'ext': 'webm',
             'title': 'Obama, Romney Campaign in Colorado Ahead of Debate',
             'description': 'Obama, Romney Campaign in Colorado Ahead of Debate',
             'uploader': 'freespeechtv',
diff --git a/youtube_dl/extractor/khanacademy.py b/youtube_dl/extractor/khanacademy.py
index 08a671fa8..61739efa7 100644
--- a/youtube_dl/extractor/khanacademy.py
+++ b/youtube_dl/extractor/khanacademy.py
@@ -14,10 +14,10 @@ class KhanAcademyIE(InfoExtractor):
 
     _TESTS = [{
         'url': 'http://www.khanacademy.org/video/one-time-pad',
-        'md5': '7021db7f2d47d4fff89b13177cb1e8f4',
+        'md5': '7b391cce85e758fb94f763ddc1bbb979',
         'info_dict': {
             'id': 'one-time-pad',
-            'ext': 'mp4',
+            'ext': 'webm',
             'title': 'The one-time pad',
             'description': 'The perfect cipher',
             'duration': 176,
diff --git a/youtube_dl/extractor/mit.py b/youtube_dl/extractor/mit.py
index 29ca45778..819c1b90b 100644
--- a/youtube_dl/extractor/mit.py
+++ b/youtube_dl/extractor/mit.py
@@ -99,7 +99,7 @@ class OCWMITIE(InfoExtractor):
             'url': 'http://ocw.mit.edu/courses/electrical-engineering-and-computer-science/6-041-probabilistic-systems-analysis-and-applied-probability-fall-2010/video-lectures/lecture-7-multiple-variables-expectations-independence/',
             'info_dict': {
                 'id': 'EObHWIEKGjA',
-                'ext': 'mp4',
+                'ext': 'webm',
                 'title': 'Lecture 7: Multiple Discrete Random Variables: Expectations, Conditioning, Independence',
                 'description': 'In this lecture, the professor discussed multiple random variables, expectations, and binomial distribution.',
                 'upload_date': '20121109',
diff --git a/youtube_dl/extractor/pyvideo.py b/youtube_dl/extractor/pyvideo.py
index 6d5732d45..30a5f2de4 100644
--- a/youtube_dl/extractor/pyvideo.py
+++ b/youtube_dl/extractor/pyvideo.py
@@ -12,14 +12,14 @@ class PyvideoIE(InfoExtractor):
     _TESTS = [
         {
             'url': 'http://pyvideo.org/video/1737/become-a-logging-expert-in-30-minutes',
-            'md5': 'de317418c8bc76b1fd8633e4f32acbc6',
+            'md5': '520915673e53a5c5d487c36e0c4d85b5',
             'info_dict': {
                 'id': '24_4WWkSmNo',
-                'ext': 'mp4',
+                'ext': 'webm',
                 'title': 'Become a logging expert in 30 minutes',
                 'description': 'md5:9665350d466c67fb5b1598de379021f7',
                 'upload_date': '20130320',
-                'uploader': 'NextDayVideo',
+                'uploader': 'Next Day Video',
                 'uploader_id': 'NextDayVideo',
             },
             'add_ie': ['Youtube'],
diff --git a/youtube_dl/extractor/ted.py b/youtube_dl/extractor/ted.py
index a48d77c30..cf8851438 100644
--- a/youtube_dl/extractor/ted.py
+++ b/youtube_dl/extractor/ted.py
@@ -73,7 +73,7 @@ class TEDIE(InfoExtractor):
         'add_ie': ['Youtube'],
         'info_dict': {
             'id': '_ZG8HBuDjgc',
-            'ext': 'mp4',
+            'ext': 'webm',
             'title': 'Douglas Adams: Parrots the Universe and Everything',
             'description': 'md5:01ad1e199c49ac640cb1196c0e9016af',
             'uploader': 'University of California Television (UCTV)',
diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index 433fc9914..e04b814c8 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -176,13 +176,13 @@ class VikiIE(VikiBaseIE):
     }, {
         # youtube external
         'url': 'http://www.viki.com/videos/50562v-poor-nastya-complete-episode-1',
-        'md5': '216d1afdc0c64d1febc1e9f2bd4b864b',
+        'md5': '63f8600c1da6f01b7640eee7eca4f1da',
         'info_dict': {
             'id': '50562v',
-            'ext': 'mp4',
+            'ext': 'webm',
             'title': 'Poor Nastya [COMPLETE] - Episode 1',
             'description': '',
-            'duration': 607,
+            'duration': 606,
             'timestamp': 1274949505,
             'upload_date': '20101213',
             'uploader': 'ad14065n',
diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 670a438af..d560a4b5e 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -142,10 +142,10 @@ class VKIE(InfoExtractor):
             'url': 'https://vk.com/video276849682_170681728',
             'info_dict': {
                 'id': 'V3K4mi0SYkc',
-                'ext': 'mp4',
+                'ext': 'webm',
                 'title': "DSWD Awards 'Children's Joy Foundation, Inc.' Certificate of Registration and License to Operate",
                 'description': 'md5:bf9c26cfa4acdfb146362682edd3827a',
-                'duration': 179,
+                'duration': 178,
                 'upload_date': '20130116',
                 'uploader': "Children's Joy Foundation",
                 'uploader_id': 'thecjf',
diff --git a/youtube_dl/extractor/wimp.py b/youtube_dl/extractor/wimp.py
index 041ff6c55..fb0accac7 100644
--- a/youtube_dl/extractor/wimp.py
+++ b/youtube_dl/extractor/wimp.py
@@ -20,7 +20,7 @@ class WimpIE(InfoExtractor):
         'md5': '4e2986c793694b55b37cf92521d12bb4',
         'info_dict': {
             'id': 'clowncar',
-            'ext': 'mp4',
+            'ext': 'webm',
             'title': 'It\'s like a clown car.',
             'description': 'md5:0e56db1370a6e49c5c1d19124c0d2fb2',
         },

From ff059017c0650e9635bbbc0565a7ec5f27f16615 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Mar 2016 04:30:19 +0600
Subject: [PATCH 4224/6219] [YoutubeDL] Fix typo in m3u8_native fixup

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 94e4ea432..72d7f0379 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1672,7 +1672,7 @@ class YoutubeDL(object):
                     else:
                         assert fixup_policy in ('ignore', 'never')
 
-                if info_dict.get('protocol') == 'm3u8_native' or info_dict.get('protocol') == 'm3u8' and self._downloader.params.get('hls_prefer_native', False):
+                if info_dict.get('protocol') == 'm3u8_native' or info_dict.get('protocol') == 'm3u8' and self.params.get('hls_prefer_native', False):
                     if fixup_policy == 'warn':
                         self.report_warning('%s: malformated aac bitstream.' % (
                             info_dict['id']))

From d1e4a464cd9f0efd6a9020c8577eb319847c0a68 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Mar 2016 04:32:18 +0600
Subject: [PATCH 4225/6219] [YoutubeDL] Carry long lines and improve
 readability

---
 youtube_dl/YoutubeDL.py | 28 ++++++++++++++++++----------
 1 file changed, 18 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 72d7f0379..2dfdea032 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1639,6 +1639,8 @@ class YoutubeDL(object):
                 if fixup_policy is None:
                     fixup_policy = 'detect_or_warn'
 
+                INSTALL_FFMPEG_MESSAGE = 'Install ffmpeg or avconv to fix this automatically.'
+
                 stretched_ratio = info_dict.get('stretched_ratio')
                 if stretched_ratio is not None and stretched_ratio != 1:
                     if fixup_policy == 'warn':
@@ -1651,15 +1653,18 @@ class YoutubeDL(object):
                             info_dict['__postprocessors'].append(stretched_pp)
                         else:
                             self.report_warning(
-                                '%s: Non-uniform pixel ratio (%s). Install ffmpeg or avconv to fix this automatically.' % (
-                                    info_dict['id'], stretched_ratio))
+                                '%s: Non-uniform pixel ratio (%s). %s'
+                                % (info_dict['id'], stretched_ratio, INSTALL_FFMPEG_MESSAGE))
                     else:
                         assert fixup_policy in ('ignore', 'never')
 
-                if info_dict.get('requested_formats') is None and info_dict.get('container') == 'm4a_dash':
+                if (info_dict.get('requested_formats') is None and
+                        info_dict.get('container') == 'm4a_dash'):
                     if fixup_policy == 'warn':
-                        self.report_warning('%s: writing DASH m4a. Only some players support this container.' % (
-                            info_dict['id']))
+                        self.report_warning(
+                            '%s: writing DASH m4a. '
+                            'Only some players support this container.'
+                            % info_dict['id'])
                     elif fixup_policy == 'detect_or_warn':
                         fixup_pp = FFmpegFixupM4aPP(self)
                         if fixup_pp.available:
@@ -1667,12 +1672,15 @@ class YoutubeDL(object):
                             info_dict['__postprocessors'].append(fixup_pp)
                         else:
                             self.report_warning(
-                                '%s: writing DASH m4a. Only some players support this container. Install ffmpeg or avconv to fix this automatically.' % (
-                                    info_dict['id']))
+                                '%s: writing DASH m4a. '
+                                'Only some players support this container. %s'
+                                % (info_dict['id'], INSTALL_FFMPEG_MESSAGE))
                     else:
                         assert fixup_policy in ('ignore', 'never')
 
-                if info_dict.get('protocol') == 'm3u8_native' or info_dict.get('protocol') == 'm3u8' and self.params.get('hls_prefer_native', False):
+                if (info_dict.get('protocol') == 'm3u8_native' or
+                        info_dict.get('protocol') == 'm3u8' and
+                        self.params.get('hls_prefer_native')):
                     if fixup_policy == 'warn':
                         self.report_warning('%s: malformated aac bitstream.' % (
                             info_dict['id']))
@@ -1683,8 +1691,8 @@ class YoutubeDL(object):
                             info_dict['__postprocessors'].append(fixup_pp)
                         else:
                             self.report_warning(
-                                '%s: malformated aac bitstream. Install ffmpeg or avconv to fix this automatically.' % (
-                                    info_dict['id']))
+                                '%s: malformated aac bitstream. %s'
+                                % (info_dict['id'], INSTALL_FFMPEG_MESSAGE))
                     else:
                         assert fixup_policy in ('ignore', 'never')
 

From 1b5284b13f1b579b2cbac5ce6ab9faa5b95800fb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Mar 2016 05:36:52 +0600
Subject: [PATCH 4226/6219] [downloader/fragment] Make speed more smooth

At the beginning of every segment there was a drop to Unknown speed due to timeslice being too small to calculate speed.
Now last speed from the previous fragment is used.
---
 youtube_dl/downloader/fragment.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
index 5bc99492b..a5bae9669 100644
--- a/youtube_dl/downloader/fragment.py
+++ b/youtube_dl/downloader/fragment.py
@@ -99,7 +99,8 @@ class FragmentFD(FileDownloader):
                     state['eta'] = self.calc_eta(
                         start, time_now, estimated_size,
                         state['downloaded_bytes'])
-                state['speed'] = s.get('speed')
+                state['speed'] = s.get('speed') or ctx.get('speed')
+                ctx['speed'] = state['speed']
                 ctx['prev_frag_downloaded_bytes'] = frag_downloaded_bytes
             self._hook_progress(state)
 

From 0f56a4b44366234dafbd4b56559c610d6a0d5b4c Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 6 Mar 2016 17:01:05 +0800
Subject: [PATCH 4227/6219] [vimeo] Don't pollute std_headers

Fixes #8778
---
 youtube_dl/extractor/vimeo.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 9f282a1da..560a80efd 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -277,9 +277,8 @@ class VimeoIE(VimeoBaseInfoExtractor):
 
     def _real_extract(self, url):
         url, data = unsmuggle_url(url, {})
-        headers = std_headers
+        headers = std_headers.copy()
         if 'http_headers' in data:
-            headers = headers.copy()
             headers.update(data['http_headers'])
         if 'Referer' not in headers:
             headers['Referer'] = url
@@ -294,7 +293,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
             url = 'https://vimeo.com/' + video_id
 
         # Retrieve video webpage to extract further information
-        request = sanitized_Request(url, None, headers)
+        request = sanitized_Request(url, headers=headers)
         try:
             webpage = self._download_webpage(request, video_id)
         except ExtractorError as ee:

From bdf7f13954f50b55cc38fa5f877ae261affbdc32 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 6 Mar 2016 10:08:02 +0100
Subject: [PATCH 4228/6219] release 2016.03.06

---
 README.md              | 2 ++
 docs/supportedsites.md | 8 +++++---
 youtube_dl/version.py  | 2 +-
 3 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index d66804a7a..a774e73e7 100644
--- a/README.md
+++ b/README.md
@@ -80,6 +80,8 @@ which means you can modify it, redistribute it or use it however you like.
                                      on Windows)
     --flat-playlist                  Do not extract the videos of a playlist,
                                      only list them.
+    --mark-watched                   Mark videos watched (YouTube only)
+    --no-mark-watched                Do not mark videos watched (YouTube only)
     --no-color                       Do not emit color codes in output
 
 ## Network Options:
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 43403233d..0682e4af0 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -194,6 +194,7 @@
  - **faz.net**
  - **fc2**
  - **Fczenit**
+ - **features.aol.com**
  - **fernsehkritik.tv**
  - **Firstpost**
  - **FiveTV**
@@ -293,6 +294,7 @@
  - **kontrtube**: KontrTube.ru - Труба зовёт
  - **KrasView**: Красвью
  - **Ku6**
+ - **KUSI**
  - **kuwo:album**: 酷我音乐 - 专辑
  - **kuwo:category**: 酷我音乐 - 分类
  - **kuwo:chart**: 酷我音乐 - 排行榜
@@ -301,12 +303,11 @@
  - **kuwo:song**: 酷我音乐
  - **la7.tv**
  - **Laola1Tv**
+ - **Le**: 乐视网
  - **Lecture2Go**
  - **Lemonde**
- - **Letv**: 乐视网
+ - **LePlaylist**
  - **LetvCloud**: 乐视云
- - **LetvPlaylist**
- - **LetvTv**
  - **Libsyn**
  - **life:embed**
  - **lifenews**: LIFE | NEWS
@@ -421,6 +422,7 @@
  - **Npr**
  - **NRK**
  - **NRKPlaylist**
+ - **NRKSkole**: NRK Skole
  - **NRKTV**: NRK TV and NRK Radio
  - **ntv.ru**
  - **Nuvid**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index adafd601b..246f5740d 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.03.01'
+__version__ = '2016.03.06'

From e41acb63647e4aad52dff25b505bdc0bb2a6b9e6 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 6 Mar 2016 17:38:39 +0800
Subject: [PATCH 4229/6219] [safari] Don't pollute std_headers (#8778)

---
 youtube_dl/extractor/safari.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/safari.py b/youtube_dl/extractor/safari.py
index 7de7b7273..a65fc8ed7 100644
--- a/youtube_dl/extractor/safari.py
+++ b/youtube_dl/extractor/safari.py
@@ -36,12 +36,13 @@ class SafariBaseIE(InfoExtractor):
         if username is None:
             self.raise_login_required('safaribooksonline.com account is required')
 
-        headers = std_headers
+        headers = std_headers.copy()
         if 'Referer' not in headers:
             headers['Referer'] = self._LOGIN_URL
+        login_page_request = sanitized_Request(self._LOGIN_URL, headers=headers)
 
         login_page = self._download_webpage(
-            self._LOGIN_URL, None,
+            login_page_request, None,
             'Downloading login form')
 
         csrf = self._html_search_regex(

From af7d5a63b2f127dda0923a68dfe05e8572a733c9 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 6 Mar 2016 17:47:07 +0800
Subject: [PATCH 4230/6219] [common] Document protocol http_dash_segments

---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 07bd2cbe2..bfa9c82f6 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -105,7 +105,7 @@ class InfoExtractor(object):
                     * protocol   The protocol that will be used for the actual
                                  download, lower-case.
                                  "http", "https", "rtsp", "rtmp", "rtmpe",
-                                 "m3u8", or "m3u8_native".
+                                 "m3u8", "m3u8_native" or "http_dash_segments".
                     * preference Order number of this format. If this field is
                                  present and not None, the formats get sorted
                                  by this field, regardless of all other values.

From 6d210f20905a5c31f72eff16f9f4dbbed7f52c70 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 6 Mar 2016 13:32:03 +0800
Subject: [PATCH 4231/6219] [utils] Add more codecs to codec2ext

BBC uses avc3. Here's an example (thanks to @remitamine for this example)

http://rdmedia.bbc.co.uk/dash/ondemand/bbb/2/client_manifest-common_init.mpd

See also https://trac.ffmpeg.org/ticket/5217
---
 youtube_dl/utils.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index dc0bf5627..0e04e91a4 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1900,6 +1900,9 @@ def codec2ext(codec):
     # is not a good fallback for file extensions
     return {
         'avc1': 'mp4',
+        'avc2': 'mp4',
+        'avc3': 'mp4',
+        'avc4': 'mp4',
         'mp4a': 'm4a',
         'vorbis': 'webm',
         'vp9': 'webm',

From c58e74062f58219a3843b795648eaa2fc160b9b4 Mon Sep 17 00:00:00 2001
From: "aystroganov@gmail.com" <aystroganov@gmail.com>
Date: Sun, 6 Mar 2016 21:57:41 +1100
Subject: [PATCH 4232/6219] [Sexu] fix extractor

---
 youtube_dl/extractor/sexu.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/sexu.py b/youtube_dl/extractor/sexu.py
index 6365a8779..474e9fdfa 100644
--- a/youtube_dl/extractor/sexu.py
+++ b/youtube_dl/extractor/sexu.py
@@ -14,7 +14,7 @@ class SexuIE(InfoExtractor):
             'id': '961791',
             'ext': 'mp4',
             'title': 'md5:4d05a19a5fc049a63dbbaf05fb71d91b',
-            'description': 'md5:c5ed8625eb386855d5a7967bd7b77a54',
+            'description': 'md5:2b75327061310a3afb3fbd7d09e2e403',
             'categories': list,  # NSFW
             'thumbnail': 're:https?://.*\.jpg$',
             'age_limit': 18,
@@ -26,7 +26,7 @@ class SexuIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         quality_arr = self._search_regex(
-            r'sources:\s*\[([^\]]+)\]', webpage, 'forrmat string')
+            r'"sources":\s*\[([^\]]+)\]', webpage, 'format string')
         formats = [{
             'url': fmt[0].replace('\\', ''),
             'format_id': fmt[1],
@@ -41,7 +41,7 @@ class SexuIE(InfoExtractor):
             'description', webpage, 'description')
 
         thumbnail = self._html_search_regex(
-            r'image:\s*"([^"]+)"',
+            r'"image":\s*"([^"]+)"',
             webpage, 'thumbnail', fatal=False)
 
         categories_str = self._html_search_meta(

From e7bd17373d90073792e6ca0dd7c5a650f14bf2e3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Mar 2016 18:08:53 +0600
Subject: [PATCH 4233/6219] [sexu] Improve extraction (Closes #8782)

---
 youtube_dl/extractor/sexu.py | 23 ++++++++++++-----------
 1 file changed, 12 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/sexu.py b/youtube_dl/extractor/sexu.py
index 474e9fdfa..a99b2a8e7 100644
--- a/youtube_dl/extractor/sexu.py
+++ b/youtube_dl/extractor/sexu.py
@@ -1,7 +1,5 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 
 
@@ -25,13 +23,18 @@ class SexuIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        quality_arr = self._search_regex(
-            r'"sources":\s*\[([^\]]+)\]', webpage, 'format string')
+        jwvideo = self._parse_json(
+            self._search_regex(r'\.setup\(\s*({.+?})\s*\);', webpage, 'jwvideo'),
+            video_id)
+
+        sources = jwvideo['sources']
+
         formats = [{
-            'url': fmt[0].replace('\\', ''),
-            'format_id': fmt[1],
-            'height': int(fmt[1][:3]),
-        } for fmt in re.findall(r'"file":"([^"]+)","label":"([^"]+)"', quality_arr)]
+            'url': source['file'].replace('\\', ''),
+            'format_id': source.get('label'),
+            'height': self._search_regex(
+                r'^(\d+)[pP]', source.get('label', ''), 'height', default=None),
+        } for source in sources if source.get('file')]
         self._sort_formats(formats)
 
         title = self._html_search_regex(
@@ -40,9 +43,7 @@ class SexuIE(InfoExtractor):
         description = self._html_search_meta(
             'description', webpage, 'description')
 
-        thumbnail = self._html_search_regex(
-            r'"image":\s*"([^"]+)"',
-            webpage, 'thumbnail', fatal=False)
+        thumbnail = jwvideo.get('image')
 
         categories_str = self._html_search_meta(
             'keywords', webpage, 'categories')

From 1818e4c2b4e0f55169d65131f257bb87d1ce52d5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 7 Mar 2016 02:10:16 +0600
Subject: [PATCH 4234/6219] [arte] Fix typo

---
 youtube_dl/extractor/arte.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index efde7e207..7f99effa9 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -122,7 +122,7 @@ class ArteTVPlus7IE(InfoExtractor):
                     compat_urllib_parse_urlparse(iframe_url).query)['json_url'][0]
         if json_url:
             return self._extract_from_json_url(json_url, video_id, lang)
-        # Differend kind of embed URL (e.g.
+        # Different kind of embed URL (e.g.
         # http://www.arte.tv/magazine/trepalium/fr/episode-0406-replay-trepalium)
         embed_url = self._search_regex(
             r'<iframe[^>]+src=(["\'])(?P<url>.+?)\1',

From 46c329d6f66368a7129e136ffe3fdf40dbb67b91 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 7 Mar 2016 02:19:54 +0600
Subject: [PATCH 4235/6219] [arte] Improve extraction (Closes #8768)

---
 youtube_dl/extractor/arte.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index 7f99effa9..3e119e21b 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -121,7 +121,10 @@ class ArteTVPlus7IE(InfoExtractor):
                 json_url = compat_parse_qs(
                     compat_urllib_parse_urlparse(iframe_url).query)['json_url'][0]
         if json_url:
-            return self._extract_from_json_url(json_url, video_id, lang)
+            title = self._search_regex(
+                r'<h3[^>]+title=(["\'])(?P<title>.+?)\1',
+                webpage, 'title', default=None, group='title')
+            return self._extract_from_json_url(json_url, video_id, lang, title=title)
         # Different kind of embed URL (e.g.
         # http://www.arte.tv/magazine/trepalium/fr/episode-0406-replay-trepalium)
         embed_url = self._search_regex(
@@ -129,7 +132,7 @@ class ArteTVPlus7IE(InfoExtractor):
             webpage, 'embed url', group='url')
         return self.url_result(embed_url)
 
-    def _extract_from_json_url(self, json_url, video_id, lang):
+    def _extract_from_json_url(self, json_url, video_id, lang, title=None):
         info = self._download_json(json_url, video_id)
         player_info = info['videoJsonPlayer']
 
@@ -137,7 +140,7 @@ class ArteTVPlus7IE(InfoExtractor):
         if not upload_date_str:
             upload_date_str = (player_info.get('VRA') or player_info.get('VDA') or '').split(' ')[0]
 
-        title = player_info['VTI'].strip()
+        title = (player_info.get('VTI') or title or player_info['VID']).strip()
         subtitle = player_info.get('VSU', '').strip()
         if subtitle:
             title += ' - %s' % subtitle

From 7404061141147d0346d6c27e7a76ff7285658219 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 7 Mar 2016 02:30:47 +0600
Subject: [PATCH 4236/6219] Credit @mutantmonkey for ustudio (#8574) and kusi
 (#8575)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index b51e23f2d..b6b47ac57 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -161,3 +161,4 @@ Jens Wille
 Robin Houtevelts
 Patrick Griffis
 Aidan Rowe
+mutantmonkey

From ac45505528a71e729a47153ce60053f861e644a9 Mon Sep 17 00:00:00 2001
From: Benjamin Congdon <bencongdon96@gmail.com>
Date: Mon, 7 Mar 2016 21:03:24 -0600
Subject: [PATCH 4237/6219] Added flag for 'allow_audio_only' format in Twitch
 queries

---
 youtube_dl/extractor/twitch.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 8639293e3..958bf8fff 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -252,6 +252,7 @@ class TwitchVodIE(TwitchItemBaseIE):
                 self._USHER_BASE, item_id,
                 compat_urllib_parse.urlencode({
                     'allow_source': 'true',
+                    'allow_audio_only': 'true',
                     'allow_spectre': 'true',
                     'player': 'twitchweb',
                     'nauth': access_token['token'],
@@ -431,6 +432,7 @@ class TwitchStreamIE(TwitchBaseIE):
 
         query = {
             'allow_source': 'true',
+            'allow_audio_only': 'true',
             'p': random.randint(1000000, 10000000),
             'player': 'twitchweb',
             'segment_preference': '4',

From e375a149e16139d0bd2d677d90d4ffe62c6116df Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 9 Mar 2016 23:11:09 +0600
Subject: [PATCH 4238/6219] [livestream] Properly build smil URLs (#8794)

---
 youtube_dl/extractor/livestream.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/livestream.py b/youtube_dl/extractor/livestream.py
index 988436226..eada7c299 100644
--- a/youtube_dl/extractor/livestream.py
+++ b/youtube_dl/extractor/livestream.py
@@ -14,6 +14,7 @@ from ..utils import (
     xpath_with_ns,
     xpath_text,
     orderedSet,
+    update_url_query,
     int_or_none,
     float_or_none,
     parse_iso8601,
@@ -72,7 +73,10 @@ class LivestreamIE(InfoExtractor):
         for vn in video_nodes:
             tbr = int_or_none(vn.attrib.get('system-bitrate'), 1000)
             furl = (
-                '%s%s?v=3.0.3&fp=WIN%%2014,0,0,145' % (base, vn.attrib['src']))
+                update_url_query(compat_urlparse.urljoin(base, vn.attrib['src']), {
+                    'v': '3.0.3',
+                    'fp': 'WIN% 14,0,0,145',
+                }))
             if 'clipBegin' in vn.attrib:
                 furl += '&ssek=' + vn.attrib['clipBegin']
             formats.append({

From 74278def2e69aacd022e00e2aea5f42ffcd9633a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 10 Mar 2016 00:51:07 +0600
Subject: [PATCH 4239/6219] [vimeo:ondemand] Separate ondemand extractor
 (Closes #8330, closes #8801)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/vimeo.py    | 60 +++++++++++++++++++++++++++-----
 2 files changed, 52 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 899bf8114..e07f724bf 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -859,6 +859,7 @@ from .vimeo import (
     VimeoChannelIE,
     VimeoGroupsIE,
     VimeoLikesIE,
+    VimeoOndemandIE,
     VimeoReviewIE,
     VimeoUserIE,
     VimeoWatchLaterIE,
diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 560a80efd..4bbb20c65 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -73,15 +73,26 @@ class VimeoIE(VimeoBaseInfoExtractor):
 
     # _VALID_URL matches Vimeo URLs
     _VALID_URL = r'''(?x)
-        https?://
-        (?:(?:www|(?P<player>player))\.)?
-        vimeo(?P<pro>pro)?\.com/
-        (?!channels/[^/?#]+/?(?:$|[?#])|album/)
-        (?:.*?/)?
-        (?:(?:play_redirect_hls|moogaloop\.swf)\?clip_id=)?
-        (?:videos?/)?
-        (?P<id>[0-9]+)
-        /?(?:[?&].*)?(?:[#].*)?$'''
+                    https?://
+                        (?:
+                            (?:
+                                www|
+                                (?P<player>player)
+                            )
+                            \.
+                        )?
+                        vimeo(?P<pro>pro)?\.com/
+                        (?!channels/[^/?#]+/?(?:$|[?#])|(?:album|ondemand)/)
+                        (?:.*?/)?
+                        (?:
+                            (?:
+                                play_redirect_hls|
+                                moogaloop\.swf)\?clip_id=
+                            )?
+                        (?:videos?/)?
+                        (?P<id>[0-9]+)
+                        /?(?:[?&].*)?(?:[#].*)?$
+                    '''
     IE_NAME = 'vimeo'
     _TESTS = [
         {
@@ -497,6 +508,37 @@ class VimeoIE(VimeoBaseInfoExtractor):
         }
 
 
+class VimeoOndemandIE(VimeoBaseInfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?vimeo\.com/ondemand/(?P<id>[^/?#&]+)'
+    _TESTS = [{
+        # ondemand video not available via https://vimeo.com/id
+        'url': 'https://vimeo.com/ondemand/20704',
+        'md5': 'c424deda8c7f73c1dfb3edd7630e2f35',
+        'info_dict': {
+            'id': '105442900',
+            'ext': 'mp4',
+            'title': 'המעבדה - במאי יותם פלדמן',
+            'uploader': 'גם סרטים',
+            'uploader_url': 're:https?://(?:www\.)?vimeo\.com/gumfilms',
+            'uploader_id': 'gumfilms',
+        },
+    }, {
+        'url': 'https://vimeo.com/ondemand/nazmaalik',
+        'only_matching': True,
+    }, {
+        'url': 'https://vimeo.com/ondemand/141692381',
+        'only_matching': True,
+    }, {
+        'url': 'https://vimeo.com/ondemand/thelastcolony/150274832',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        return self.url_result(self._og_search_video_url(webpage), VimeoIE.ie_key())
+
+
 class VimeoChannelIE(VimeoBaseInfoExtractor):
     IE_NAME = 'vimeo:channel'
     _VALID_URL = r'https://vimeo\.com/channels/(?P<id>[^/?#]+)/?(?:$|[?#])'

From 1e501364d5a7f5c99037e6791616ee48989bbaea Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 10 Mar 2016 00:52:52 +0600
Subject: [PATCH 4240/6219] [vimeo:ondemand] Clarify IE_NAME

---
 youtube_dl/extractor/vimeo.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 4bbb20c65..71c30d2cd 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -509,6 +509,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
 
 
 class VimeoOndemandIE(VimeoBaseInfoExtractor):
+    IE_NAME = 'vimeo:ondemand'
     _VALID_URL = r'https?://(?:www\.)?vimeo\.com/ondemand/(?P<id>[^/?#&]+)'
     _TESTS = [{
         # ondemand video not available via https://vimeo.com/id

From 5d583bdf6cff8161e2df87ba1a0c97a943596eab Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 10 Mar 2016 01:03:18 +0600
Subject: [PATCH 4241/6219] [YoutubeDL] Improve _format_note

---
 test/test_YoutubeDL.py  | 4 ++++
 youtube_dl/YoutubeDL.py | 4 +++-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 59f7ab49d..bf78b005f 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -502,6 +502,10 @@ class TestYoutubeDL(unittest.TestCase):
         assertRegexpMatches(self, ydl._format_note({
             'vbr': 10,
         }), '^\s*10k$')
+        assertRegexpMatches(self, ydl._format_note({
+            'fps': 30,
+        }), '^30fps$')
+
 
     def test_postprocessors(self):
         filename = 'post-processor-testfile.mp4'
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 2dfdea032..b3391088f 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1856,7 +1856,9 @@ class YoutubeDL(object):
         if fdict.get('vbr') is not None:
             res += '%4dk' % fdict['vbr']
         if fdict.get('fps') is not None:
-            res += ', %sfps' % fdict['fps']
+            if res:
+                res += ', '
+            res += '%sfps' % fdict['fps']
         if fdict.get('acodec') is not None:
             if res:
                 res += ', '

From e0dccdd39886c8cda1dddde0d5f5f5a2dcd33746 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 10 Mar 2016 09:04:48 +0600
Subject: [PATCH 4242/6219] [test_YoutubeDL] PEP 8

---
 test/test_YoutubeDL.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index bf78b005f..efbee3b71 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -506,7 +506,6 @@ class TestYoutubeDL(unittest.TestCase):
             'fps': 30,
         }), '^30fps$')
 
-
     def test_postprocessors(self):
         filename = 'post-processor-testfile.mp4'
         audiofile = filename + '.mp3'

From 9c7b509b2a3da1e23e81a2b9039185e09180328a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 10 Mar 2016 14:33:48 +0800
Subject: [PATCH 4243/6219] [facebook] Merge FacebookPostIE into FacebookIE

Fixes #8713
---
 youtube_dl/extractor/__init__.py |  5 +--
 youtube_dl/extractor/facebook.py | 68 ++++++++++++++++++--------------
 2 files changed, 40 insertions(+), 33 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index e07f724bf..98de5ddff 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -212,10 +212,7 @@ from .everyonesmixtape import EveryonesMixtapeIE
 from .exfm import ExfmIE
 from .expotv import ExpoTVIE
 from .extremetube import ExtremeTubeIE
-from .facebook import (
-    FacebookIE,
-    FacebookPostIE,
-)
+from .facebook import FacebookIE
 from .faz import FazIE
 from .fc2 import FC2IE
 from .fczenit import FczenitIE
diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index 6c6c3b1bd..23a6fcd88 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -37,7 +37,8 @@ class FacebookIE(InfoExtractor):
                                 video/embed|
                                 story\.php
                             )\?(?:.*?)(?:v|video_id|story_fbid)=|
-                            [^/]+/videos/(?:[^/]+/)?
+                            [^/]+/videos/(?:[^/]+/)?|
+                            [^/]+/posts/
                         )|
                     facebook:
                 )
@@ -50,6 +51,8 @@ class FacebookIE(InfoExtractor):
 
     _CHROME_USER_AGENT = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.97 Safari/537.36'
 
+    _VIDEO_PAGE_TEMPLATE = 'https://www.facebook.com/video/video.php?v=%s'
+
     _TESTS = [{
         'url': 'https://www.facebook.com/video.php?v=637842556329505&fref=nf',
         'md5': '6a40d33c0eccbb1af76cf0485a052659',
@@ -81,6 +84,15 @@ class FacebookIE(InfoExtractor):
             'title': 'When you post epic content on instagram.com/433 8 million followers, this is ...',
             'uploader': 'Demy de Zeeuw',
         },
+    }, {
+        'url': 'https://www.facebook.com/maxlayn/posts/10153807558977570',
+        'md5': '037b1fa7f3c2d02b7a0d7bc16031ecc6',
+        'info_dict': {
+            'id': '544765982287235',
+            'ext': 'mp4',
+            'title': '"What are you doing running in the snow?"',
+            'uploader': 'FailArmy',
+        }
     }, {
         'url': 'https://www.facebook.com/video.php?v=10204634152394104',
         'only_matching': True,
@@ -164,9 +176,8 @@ class FacebookIE(InfoExtractor):
     def _real_initialize(self):
         self._login()
 
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        req = sanitized_Request('https://www.facebook.com/video/video.php?v=%s' % video_id)
+    def _extract_from_url(self, url, video_id, fatal_if_no_video=True):
+        req = sanitized_Request(url)
         req.add_header('User-Agent', self._CHROME_USER_AGENT)
         webpage = self._download_webpage(req, video_id)
 
@@ -196,6 +207,8 @@ class FacebookIE(InfoExtractor):
                     break
 
         if not video_data:
+            if not fatal_if_no_video:
+                return webpage, False
             m_msg = re.search(r'class="[^"]*uiInterstitialContent[^"]*"><div>(.*?)</div>', webpage)
             if m_msg is not None:
                 raise ExtractorError(
@@ -241,39 +254,36 @@ class FacebookIE(InfoExtractor):
             video_title = 'Facebook video #%s' % video_id
         uploader = clean_html(get_element_by_id('fbPhotoPageAuthorName', webpage))
 
-        return {
+        info_dict = {
             'id': video_id,
             'title': video_title,
             'formats': formats,
             'uploader': uploader,
         }
 
-
-class FacebookPostIE(InfoExtractor):
-    IE_NAME = 'facebook:post'
-    _VALID_URL = r'https?://(?:\w+\.)?facebook\.com/[^/]+/posts/(?P<id>\d+)'
-    _TEST = {
-        'url': 'https://www.facebook.com/maxlayn/posts/10153807558977570',
-        'md5': '037b1fa7f3c2d02b7a0d7bc16031ecc6',
-        'info_dict': {
-            'id': '544765982287235',
-            'ext': 'mp4',
-            'title': '"What are you doing running in the snow?"',
-            'uploader': 'FailArmy',
-        }
-    }
+        return webpage, info_dict
 
     def _real_extract(self, url):
-        post_id = self._match_id(url)
+        video_id = self._match_id(url)
 
-        webpage = self._download_webpage(url, post_id)
+        real_url = self._VIDEO_PAGE_TEMPLATE % video_id if url.startswith('facebook:') else url
+        webpage, info_dict = self._extract_from_url(real_url, video_id, fatal_if_no_video=False)
 
-        entries = [
-            self.url_result('facebook:%s' % video_id, FacebookIE.ie_key())
-            for video_id in self._parse_json(
-                self._search_regex(
-                    r'(["\'])video_ids\1\s*:\s*(?P<ids>\[.+?\])',
-                    webpage, 'video ids', group='ids'),
-                post_id)]
+        if info_dict:
+            return info_dict
 
-        return self.playlist_result(entries, post_id)
+        if '/posts/' in url:
+            entries = [
+                self.url_result('facebook:%s' % video_id, FacebookIE.ie_key())
+                for video_id in self._parse_json(
+                    self._search_regex(
+                        r'(["\'])video_ids\1\s*:\s*(?P<ids>\[.+?\])',
+                        webpage, 'video ids', group='ids'),
+                    video_id)]
+
+            return self.playlist_result(entries, video_id)
+        else:
+            _, info_dict = self._extract_from_url(
+                self._VIDEO_PAGE_TEMPLATE % video_id,
+                video_id, fatal_if_no_video=True)
+            return info_dict

From 98801241965ff81afd68392df52e38a54d49ded2 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 10 Mar 2016 14:58:05 +0800
Subject: [PATCH 4244/6219] [facebook] Fix for m.facebook.com URLs

---
 youtube_dl/extractor/facebook.py | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index 23a6fcd88..b354c5521 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -93,6 +93,15 @@ class FacebookIE(InfoExtractor):
             'title': '"What are you doing running in the snow?"',
             'uploader': 'FailArmy',
         }
+    }, {
+        'url': 'https://m.facebook.com/story.php?story_fbid=1035862816472149&id=116132035111903',
+        'md5': '1deb90b6ac27f7efcf6d747c8a27f5e3',
+        'info_dict': {
+            'id': '1035862816472149',
+            'ext': 'mp4',
+            'title': 'What the Flock Is Going On In New Zealand  Credit: ViralHog',
+            'uploader': 'S. Saint',
+        },
     }, {
         'url': 'https://www.facebook.com/video.php?v=10204634152394104',
         'only_matching': True,
@@ -105,9 +114,6 @@ class FacebookIE(InfoExtractor):
     }, {
         'url': 'facebook:544765982287235',
         'only_matching': True,
-    }, {
-        'url': 'https://m.facebook.com/story.php?story_fbid=1035862816472149&id=116132035111903',
-        'only_matching': True,
     }]
 
     def _login(self):
@@ -200,7 +206,7 @@ class FacebookIE(InfoExtractor):
 
         if not video_data:
             server_js_data = self._parse_json(self._search_regex(
-                r'handleServerJS\(({.+})\);', webpage, 'server js data'), video_id)
+                r'handleServerJS\(({.+})\);', webpage, 'server js data', default='{}'), video_id)
             for item in server_js_data.get('instances', []):
                 if item[1][0] == 'VideoConfig':
                     video_data = video_data_list2dict(item[2][0]['videoData'])

From 5080cbf9fd26441256390f169d24ab3259c740f7 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 10 Mar 2016 15:26:32 +0800
Subject: [PATCH 4245/6219] [facebook] Handle escaped swf params

Fixes #8713
---
 youtube_dl/extractor/facebook.py | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index b354c5521..49231d3e9 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -102,6 +102,15 @@ class FacebookIE(InfoExtractor):
             'title': 'What the Flock Is Going On In New Zealand  Credit: ViralHog',
             'uploader': 'S. Saint',
         },
+    }, {
+        'note': 'swf params escaped',
+        'url': 'https://www.facebook.com/barackobama/posts/10153664894881749',
+        'md5': '97ba073838964d12c70566e0085c2b91',
+        'info_dict': {
+            'id': '10153664894881749',
+            'ext': 'mp4',
+            'title': 'Facebook video #10153664894881749',
+        },
     }, {
         'url': 'https://www.facebook.com/video.php?v=10204634152394104',
         'only_matching': True,
@@ -189,11 +198,12 @@ class FacebookIE(InfoExtractor):
 
         video_data = None
 
-        BEFORE = '{swf.addParam(param[0], param[1]);});\n'
+        BEFORE = '{swf.addParam(param[0], param[1]);});'
         AFTER = '.forEach(function(variable) {swf.addVariable(variable[0], variable[1]);});'
-        m = re.search(re.escape(BEFORE) + '(.*?)' + re.escape(AFTER), webpage)
+        m = re.search(re.escape(BEFORE) + '(?:\n|\\\\n)(.*?)' + re.escape(AFTER), webpage)
         if m:
-            data = dict(json.loads(m.group(1)))
+            swf_params = m.group(1).replace('\\\\', '\\').replace('\\"', '"')
+            data = dict(json.loads(swf_params))
             params_raw = compat_urllib_parse_unquote(data['params'])
             video_data = json.loads(params_raw)['video_data']
 

From 31db8709bff7209d215c00eb93bdb2cf178d7f58 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 10 Mar 2016 21:37:26 +0800
Subject: [PATCH 4246/6219] [iqiyi] Update enc_key

---
 youtube_dl/extractor/iqiyi.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index d3bee3a19..e7c0cb3f6 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -501,7 +501,7 @@ class IqiyiIE(InfoExtractor):
     def get_enc_key(self, video_id):
         # TODO: automatic key extraction
         # last update at 2016-01-22 for Zombie::bite
-        enc_key = '6ab6d0280511493ba85594779759d4ed'
+        enc_key = '8ed797d224d043e7ac23d95b70227d32'
         return enc_key
 
     def _extract_playlist(self, webpage):

From f6f6217a98adc07c41f7a45a9068aeaf706ad862 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 10 Mar 2016 15:17:04 +0100
Subject: [PATCH 4247/6219] [facebook] Don't override variable in list
 comprehension

---
 youtube_dl/extractor/facebook.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index 49231d3e9..5e8589479 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -290,8 +290,8 @@ class FacebookIE(InfoExtractor):
 
         if '/posts/' in url:
             entries = [
-                self.url_result('facebook:%s' % video_id, FacebookIE.ie_key())
-                for video_id in self._parse_json(
+                self.url_result('facebook:%s' % vid, FacebookIE.ie_key())
+                for vid in self._parse_json(
                     self._search_regex(
                         r'(["\'])video_ids\1\s*:\s*(?P<ids>\[.+?\])',
                         webpage, 'video ids', group='ids'),

From e738e43358a7f34dbbcb73b45b228d7b9b3360a7 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 11 Mar 2016 16:20:27 +0800
Subject: [PATCH 4248/6219] [facebook] Support videos in groups

Viewing/Downloading videos in groups requires logging in, even for
those in public groups.

Fixes #6951.
---
 youtube_dl/extractor/facebook.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index 5e8589479..f5bbd39d2 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -38,7 +38,8 @@ class FacebookIE(InfoExtractor):
                                 story\.php
                             )\?(?:.*?)(?:v|video_id|story_fbid)=|
                             [^/]+/videos/(?:[^/]+/)?|
-                            [^/]+/posts/
+                            [^/]+/posts/|
+                            groups/[^/]+/permalink/
                         )|
                     facebook:
                 )
@@ -123,6 +124,9 @@ class FacebookIE(InfoExtractor):
     }, {
         'url': 'facebook:544765982287235',
         'only_matching': True,
+    }, {
+        'url': 'https://www.facebook.com/groups/164828000315060/permalink/764967300301124/',
+        'only_matching': True,
     }]
 
     def _login(self):

From 61f317c24c040d051ec6652cb0d4c650c1fc0361 Mon Sep 17 00:00:00 2001
From: Benjamin Congdon <bencongdon96@gmail.com>
Date: Wed, 9 Mar 2016 21:16:21 -0600
Subject: [PATCH 4249/6219] Added extractor for AudioBoom.com

---
 youtube_dl/extractor/__init__.py  |  1 +
 youtube_dl/extractor/audioboom.py | 39 +++++++++++++++++++++++++++++++
 2 files changed, 40 insertions(+)
 create mode 100644 youtube_dl/extractor/audioboom.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 98de5ddff..1a7d689df 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -54,6 +54,7 @@ from .arte import (
 from .atresplayer import AtresPlayerIE
 from .atttechchannel import ATTTechChannelIE
 from .audimedia import AudiMediaIE
+from .audioboom import AudioBoomIE
 from .audiomack import AudiomackIE, AudiomackAlbumIE
 from .azubu import AzubuIE, AzubuLiveIE
 from .baidu import BaiduVideoIE
diff --git a/youtube_dl/extractor/audioboom.py b/youtube_dl/extractor/audioboom.py
new file mode 100644
index 000000000..84d4755f5
--- /dev/null
+++ b/youtube_dl/extractor/audioboom.py
@@ -0,0 +1,39 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import float_or_none
+
+
+class AudioBoomIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?audioboom\.com/boos/(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'https://audioboom.com/boos/4279833-3-09-2016-czaban-hour-3?t=0',
+        'md5': '63a8d73a055c6ed0f1e51921a10a5a76',
+        'info_dict': {
+            'id': '4279833',
+            'ext': 'mp3',
+            'title': '3/09/2016 Czaban Hour 3',
+            'description': 'Guest:   Nate Davis - NFL free agency,   Guest:   Stan Gans',
+            'duration': 2245.72
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        title = self._og_search_title(webpage)
+
+        download_url = self._og_search_property('audio', webpage, 'url')
+
+        duration = float_or_none(self._html_search_meta(
+            'weibo:audio:duration', webpage, fatal=False))
+
+        return {
+            'id': video_id,
+            'title': title,
+            'url': download_url,
+            'description': self._og_search_description(webpage),
+            'duration': duration,
+        }

From 883c05237869ceea3eabdf1e4ca53f4b45d9a04b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 11 Mar 2016 19:44:17 +0600
Subject: [PATCH 4250/6219] [audioboom] Improve robustness and extract uploader
 (Closes #8812)

---
 youtube_dl/extractor/audioboom.py | 41 +++++++++++++++++++++++++------
 1 file changed, 34 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/audioboom.py b/youtube_dl/extractor/audioboom.py
index 84d4755f5..2ec2d7092 100644
--- a/youtube_dl/extractor/audioboom.py
+++ b/youtube_dl/extractor/audioboom.py
@@ -15,25 +15,52 @@ class AudioBoomIE(InfoExtractor):
             'ext': 'mp3',
             'title': '3/09/2016 Czaban Hour 3',
             'description': 'Guest:   Nate Davis - NFL free agency,   Guest:   Stan Gans',
-            'duration': 2245.72
+            'duration': 2245.72,
+            'uploader': 'Steve Czaban',
+            'uploader_url': 're:https?://(?:www\.)?audioboom\.com/channel/steveczabanyahoosportsradio',
         }
     }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
+
         webpage = self._download_webpage(url, video_id)
 
-        title = self._og_search_title(webpage)
+        clip = None
 
-        download_url = self._og_search_property('audio', webpage, 'url')
+        clip_store = self._parse_json(
+            self._search_regex(
+                r'data-new-clip-store=(["\'])(?P<json>{.*?"clipId"\s*:\s*%s.*?})\1' % video_id,
+                webpage, 'clip store', default='{}', group='json'),
+            video_id, fatal=False)
+        if clip_store:
+            clips = clip_store.get('clips')
+            if clips and isinstance(clips, list) and isinstance(clips[0], dict):
+                clip = clips[0]
 
-        duration = float_or_none(self._html_search_meta(
-            'weibo:audio:duration', webpage, fatal=False))
+        def from_clip(field):
+            if clip:
+                clip.get(field)
+
+        audio_url = from_clip('clipURLPriorToLoading') or self._og_search_property(
+            'audio', webpage, 'audio url')
+        title = from_clip('title') or self._og_search_title(webpage)
+        description = from_clip('description') or self._og_search_description(webpage)
+
+        duration = float_or_none(from_clip('duration') or self._html_search_meta(
+            'weibo:audio:duration', webpage))
+
+        uploader = from_clip('author') or self._og_search_property(
+            'audio:artist', webpage, 'uploader', fatal=False)
+        uploader_url = from_clip('author_url') or self._html_search_meta(
+            'audioboo:channel', webpage, 'uploader url')
 
         return {
             'id': video_id,
+            'url': audio_url,
             'title': title,
-            'url': download_url,
-            'description': self._og_search_description(webpage),
+            'description': description,
             'duration': duration,
+            'uploader': uploader,
+            'uploader_url': uploader_url,
         }

From 6fa6d385490e913af48538a493384e7288d30c8f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 11 Mar 2016 19:46:06 +0600
Subject: [PATCH 4251/6219] Credit @benjamincongdon for audioboom (#8812)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index b6b47ac57..aa48cd5a6 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -162,3 +162,4 @@ Robin Houtevelts
 Patrick Griffis
 Aidan Rowe
 mutantmonkey
+Ben Congdon

From 25cb05bda9c8609c5e2f329f199735837b4d8c60 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 11 Mar 2016 23:48:05 +0800
Subject: [PATCH 4252/6219] [utils] Remove codec2ext

This function is orignally used for determining file extensions of DASH
formats. Now in DASH, ext is determined by mime_type. See #8766 for more
information.
---
 youtube_dl/utils.py | 16 ----------------
 1 file changed, 16 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 0e04e91a4..22a39a0ab 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1893,22 +1893,6 @@ def mimetype2ext(mt):
     }.get(res, res)
 
 
-def codec2ext(codec):
-    codec_type = codec.split('.')[0]
-
-    # Leave the return value None for unknown values as codec_type
-    # is not a good fallback for file extensions
-    return {
-        'avc1': 'mp4',
-        'avc2': 'mp4',
-        'avc3': 'mp4',
-        'avc4': 'mp4',
-        'mp4a': 'm4a',
-        'vorbis': 'webm',
-        'vp9': 'webm',
-    }.get(codec_type)
-
-
 def urlhandle_detect_ext(url_handle):
     try:
         url_handle.headers

From a6c8b75904f8c67c08708efe3a8d3dce94af9ee2 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 11 Mar 2016 23:49:55 +0800
Subject: [PATCH 4253/6219] [common] Use mimeType to determine file extensions
 (#8766)

---
 youtube_dl/extractor/common.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 393677251..52b4c125a 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -29,7 +29,6 @@ from ..utils import (
     age_restricted,
     bug_reports_message,
     clean_html,
-    codec2ext,
     compiled_regex_type,
     determine_ext,
     error_to_compat_str,
@@ -1448,8 +1447,9 @@ class InfoExtractor(object):
                         continue
                     representation_attrib = adaptation_set.attrib.copy()
                     representation_attrib.update(representation.attrib)
-                    mime_type = representation_attrib.get('mimeType')
-                    content_type = mime_type.split('/')[0] if mime_type else representation_attrib.get('contentType')
+                    # According to page 41 of ISO/IEC 29001-1:2014, @mimeType is mandatory
+                    mime_type = representation_attrib['mimeType']
+                    content_type = mime_type.split('/')[0]
                     if content_type == 'text':
                         # TODO implement WebVTT downloading
                         pass
@@ -1472,7 +1472,7 @@ class InfoExtractor(object):
                         f = {
                             'format_id': '%s-%s' % (mpd_id, representation_id) if mpd_id else representation_id,
                             'url': base_url,
-                            'ext': codec2ext(representation_attrib.get('codecs')),
+                            'ext': mimetype2ext(mime_type),
                             'width': int_or_none(representation_attrib.get('width')),
                             'height': int_or_none(representation_attrib.get('height')),
                             'tbr': int_or_none(representation_attrib.get('bandwidth'), 1000),

From fac7e79277942a182ba68d1ba01b8b3f7a9935b3 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 11 Mar 2016 16:52:07 +0100
Subject: [PATCH 4254/6219] [kaltura] add support for videos with reference id

---
 youtube_dl/extractor/kaltura.py | 54 +++++++++++++++++++++++++--------
 1 file changed, 42 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/kaltura.py b/youtube_dl/extractor/kaltura.py
index ccbc39c66..78e29f481 100644
--- a/youtube_dl/extractor/kaltura.py
+++ b/youtube_dl/extractor/kaltura.py
@@ -8,6 +8,7 @@ from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse,
     compat_urlparse,
+    compat_parse_qs,
 )
 from ..utils import (
     clean_html,
@@ -20,21 +21,17 @@ from ..utils import (
 class KalturaIE(InfoExtractor):
     _VALID_URL = r'''(?x)
                 (?:
-                    kaltura:(?P<partner_id_s>\d+):(?P<id_s>[0-9a-z_]+)|
+                    kaltura:(?P<partner_id>\d+):(?P<id>[0-9a-z_]+)|
                     https?://
                         (:?(?:www|cdnapi(?:sec)?)\.)?kaltura\.com/
                         (?:
                             (?:
                                 # flash player
-                                index\.php/kwidget/
-                                (?:[^/]+/)*?wid/_(?P<partner_id>\d+)/
-                                (?:[^/]+/)*?entry_id/(?P<id>[0-9a-z_]+)|
+                                index\.php/kwidget|
                                 # html5 player
-                                html5/html5lib/
-                                (?:[^/]+/)*?entry_id/(?P<id_html5>[0-9a-z_]+)
-                                .*\?.*\bwid=_(?P<partner_id_html5>\d+)
+                                html5/html5lib/[^/]+/mwEmbedFrame\.php
                             )
-                        )
+                        )(?:/(?P<path>[^?]+))?(?:\?(?P<query>.*))?
                 )
                 '''
     _API_BASE = 'http://cdnapi.kaltura.com/api_v3/index.php?'
@@ -127,10 +124,43 @@ class KalturaIE(InfoExtractor):
         url, smuggled_data = unsmuggle_url(url, {})
 
         mobj = re.match(self._VALID_URL, url)
-        partner_id = mobj.group('partner_id_s') or mobj.group('partner_id') or mobj.group('partner_id_html5')
-        entry_id = mobj.group('id_s') or mobj.group('id') or mobj.group('id_html5')
-
-        info, flavor_assets = self._get_video_info(entry_id, partner_id)
+        partner_id, entry_id = mobj.group('partner_id', 'id')
+        info, flavor_assets = None, None
+        if partner_id and entry_id:
+            info, flavor_assets = self._get_video_info(entry_id, partner_id)
+        else:
+            path, query = mobj.group('path', 'query')
+            if not path and not query:
+                raise ExtractorError('Invalid URL', expected=True)
+            params = {}
+            if query:
+                params = compat_parse_qs(query)
+            if path:
+                splitted_path = path.split('/')
+                if not splitted_path[-1]:
+                    splitted_path = splitted_path[:-1]
+                for i in range(0, len(splitted_path), 2):
+                    params[splitted_path[i]] = [splitted_path[i + 1]]
+            if 'wid' in params:
+                partner_id = params['wid'][0][1:]
+            elif 'p' in params:
+                partner_id = params['p'][0]
+            else:
+                raise ExtractorError('Invalid URL', expected=True)
+            if 'entry_id' in params:
+                entry_id = params['entry_id'][0]
+                info, flavor_assets = self._get_video_info(entry_id, partner_id)
+            elif 'uiconf_id' in params and 'flashvars[referenceId]' in params:
+                reference_id = params['flashvars[referenceId]'][0]
+                webpage = self._download_webpage(url, reference_id)
+                entry_data = self._parse_json(self._search_regex(
+                    r'window\.kalturaIframePackageData\s*=\s*({.*});',
+                    webpage, 'kalturaIframePackageData'),
+                    reference_id)['entryResult']
+                info, flavor_assets = entry_data['meta'], entry_data['contextData']['flavorAssets']
+                entry_id = info['id']
+            else:
+                raise ExtractorError('Invalid URL', expected=True)
 
         source_url = smuggled_data.get('source_url')
         if source_url:

From bcb668de189b92bd1dcad661d50ae981f25deca4 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 11 Mar 2016 16:57:06 +0100
Subject: [PATCH 4255/6219] [safari] extract free and preview videos(#7491)

---
 youtube_dl/extractor/safari.py | 29 +++++++++++++++++------------
 1 file changed, 17 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/safari.py b/youtube_dl/extractor/safari.py
index a65fc8ed7..ae619636c 100644
--- a/youtube_dl/extractor/safari.py
+++ b/youtube_dl/extractor/safari.py
@@ -12,6 +12,7 @@ from ..utils import (
     smuggle_url,
     std_headers,
     urlencode_postdata,
+    update_url_query,
 )
 
 
@@ -26,6 +27,7 @@ class SafariBaseIE(InfoExtractor):
     LOGGED_IN = False
 
     def _real_initialize(self):
+        return
         # We only need to log in once for courses or individual videos
         if not self.LOGGED_IN:
             self._login()
@@ -86,13 +88,15 @@ class SafariIE(SafariBaseIE):
 
     _TESTS = [{
         'url': 'https://www.safaribooksonline.com/library/view/hadoop-fundamentals-livelessons/9780133392838/part00.html',
-        'md5': '5b0c4cc1b3c1ba15dda7344085aa5592',
+        'md5': 'dcc5a425e79f2564148652616af1f2a3',
         'info_dict': {
-            'id': '2842601850001',
+            'id': '0_qbqx90ic',
             'ext': 'mp4',
-            'title': 'Introduction',
+            'title': 'Introduction to Hadoop Fundamentals LiveLessons',
+            'timestamp': 1437758058,
+            'upload_date': '20150724',
+            'uploader_id': 'stork',
         },
-        'skip': 'Requires safaribooksonline account credentials',
     }, {
         'url': 'https://www.safaribooksonline.com/api/v1/book/9780133392838/chapter/part00.html',
         'only_matching': True,
@@ -107,15 +111,16 @@ class SafariIE(SafariBaseIE):
         course_id = mobj.group('course_id')
         part = mobj.group('part')
 
-        webpage = self._download_webpage(
-            '%s/%s/chapter-content/%s.html' % (self._API_BASE, course_id, part),
-            part)
+        webpage = self._download_webpage(url, '%s/%s' % (course_id, part))
+        reference_id = self._search_regex(r'data-reference-id="([^"]+)"', webpage, 'kaltura reference id')
+        partner_id = self._search_regex(r'data-partner-id="([^"]+)"', webpage, 'kaltura widget id')
+        ui_id = self._search_regex(r'data-ui-id="([^"]+)"', webpage, 'kaltura uiconf id')
 
-        bc_url = BrightcoveLegacyIE._extract_brightcove_url(webpage)
-        if not bc_url:
-            raise ExtractorError('Could not extract Brightcove URL from %s' % url, expected=True)
-
-        return self.url_result(smuggle_url(bc_url, {'Referer': url}), 'BrightcoveLegacy')
+        return self.url_result(update_url_query('https://cdnapisec.kaltura.com/html5/html5lib/v2.37.1/mwEmbedFrame.php', {
+            'wid': '_%s' % partner_id,
+            'uiconf_id': ui_id,
+            'flashvars[referenceId]': reference_id,
+        }), 'Kaltura')
 
 
 class SafariCourseIE(SafariBaseIE):

From cdfee16818088d5442e9ea9fd9a2f85273bb59ed Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 10 Mar 2016 19:49:13 +0100
Subject: [PATCH 4256/6219] [extractor/common] add data, headers and query
 params to _request_webpage

---
 youtube_dl/extractor/common.py | 25 ++++++++++++++++---------
 1 file changed, 16 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index bfa9c82f6..d2443c93c 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -48,6 +48,7 @@ from ..utils import (
     determine_protocol,
     parse_duration,
     mimetype2ext,
+    update_url_query,
 )
 
 
@@ -345,7 +346,7 @@ class InfoExtractor(object):
     def IE_NAME(self):
         return compat_str(type(self).__name__[:-2])
 
-    def _request_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True):
+    def _request_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True, data=None, headers=None, query=None):
         """ Returns the response handle """
         if note is None:
             self.report_download_webpage(video_id)
@@ -354,6 +355,12 @@ class InfoExtractor(object):
                 self.to_screen('%s' % (note,))
             else:
                 self.to_screen('%s: %s' % (video_id, note))
+        # data, headers and query params will be ignored for `Request` objects
+        if isinstance(url_or_request, compat_str):
+            if query:
+                url_or_request = update_url_query(url_or_request, query)
+            if data or headers:
+                url_or_request = sanitized_Request(url_or_request, data, headers or {})
         try:
             return self._downloader.urlopen(url_or_request)
         except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
@@ -369,13 +376,13 @@ class InfoExtractor(object):
                 self._downloader.report_warning(errmsg)
                 return False
 
-    def _download_webpage_handle(self, url_or_request, video_id, note=None, errnote=None, fatal=True, encoding=None):
+    def _download_webpage_handle(self, url_or_request, video_id, note=None, errnote=None, fatal=True, encoding=None, data=None, headers=None, query=None):
         """ Returns a tuple (page content as string, URL handle) """
         # Strip hashes from the URL (#1038)
         if isinstance(url_or_request, (compat_str, str)):
             url_or_request = url_or_request.partition('#')[0]
 
-        urlh = self._request_webpage(url_or_request, video_id, note, errnote, fatal)
+        urlh = self._request_webpage(url_or_request, video_id, note, errnote, fatal, data=data, headers=headers, query=query)
         if urlh is False:
             assert not fatal
             return False
@@ -462,13 +469,13 @@ class InfoExtractor(object):
 
         return content
 
-    def _download_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True, tries=1, timeout=5, encoding=None):
+    def _download_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True, tries=1, timeout=5, encoding=None, data=None, headers=None, query=None):
         """ Returns the data of the page as a string """
         success = False
         try_count = 0
         while success is False:
             try:
-                res = self._download_webpage_handle(url_or_request, video_id, note, errnote, fatal, encoding=encoding)
+                res = self._download_webpage_handle(url_or_request, video_id, note, errnote, fatal, encoding=encoding, data=data, headers=headers, query=query)
                 success = True
             except compat_http_client.IncompleteRead as e:
                 try_count += 1
@@ -483,10 +490,10 @@ class InfoExtractor(object):
 
     def _download_xml(self, url_or_request, video_id,
                       note='Downloading XML', errnote='Unable to download XML',
-                      transform_source=None, fatal=True, encoding=None):
+                      transform_source=None, fatal=True, encoding=None, data=None, headers=None, query=None):
         """Return the xml as an xml.etree.ElementTree.Element"""
         xml_string = self._download_webpage(
-            url_or_request, video_id, note, errnote, fatal=fatal, encoding=encoding)
+            url_or_request, video_id, note, errnote, fatal=fatal, encoding=encoding, data=data, headers=headers, query=query)
         if xml_string is False:
             return xml_string
         if transform_source:
@@ -497,10 +504,10 @@ class InfoExtractor(object):
                        note='Downloading JSON metadata',
                        errnote='Unable to download JSON metadata',
                        transform_source=None,
-                       fatal=True, encoding=None):
+                       fatal=True, encoding=None, data=None, headers=None, query=None):
         json_string = self._download_webpage(
             url_or_request, video_id, note, errnote, fatal=fatal,
-            encoding=encoding)
+            encoding=encoding, data=data, headers=headers, query=query)
         if (not fatal) and json_string is False:
             return None
         return self._parse_json(

From 58e6d097d894416c0d5d1f19a059b3614f823a9d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 12 Mar 2016 00:36:39 +0600
Subject: [PATCH 4257/6219] [googledrive] Relax _VALID_URL (Closes #8829)

---
 youtube_dl/extractor/googledrive.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/googledrive.py b/youtube_dl/extractor/googledrive.py
index 37be34091..766fc26d0 100644
--- a/youtube_dl/extractor/googledrive.py
+++ b/youtube_dl/extractor/googledrive.py
@@ -10,8 +10,8 @@ from ..utils import (
 
 
 class GoogleDriveIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:(?:docs|drive)\.google\.com/(?:uc\?.*?id=|file/d/)|video\.google\.com/get_player\?.*?docid=)(?P<id>[a-zA-Z0-9_-]{28})'
-    _TEST = {
+    _VALID_URL = r'https?://(?:(?:docs|drive)\.google\.com/(?:uc\?.*?id=|file/d/)|video\.google\.com/get_player\?.*?docid=)(?P<id>[a-zA-Z0-9_-]{28,})'
+    _TESTS = [{
         'url': 'https://drive.google.com/file/d/0ByeS4oOUV-49Zzh4R1J6R09zazQ/edit?pli=1',
         'md5': '881f7700aec4f538571fa1e0eed4a7b6',
         'info_dict': {
@@ -20,7 +20,11 @@ class GoogleDriveIE(InfoExtractor):
             'title': 'Big Buck Bunny.mp4',
             'duration': 46,
         }
-    }
+    }, {
+        # video id is longer than 28 characters
+        'url': 'https://drive.google.com/file/d/1ENcQ_jeCuj7y19s66_Ou9dRP4GKGsodiDQ/edit',
+        'only_matching': True,
+    }]
     _FORMATS_EXT = {
         '5': 'flv',
         '6': 'flv',
@@ -43,7 +47,7 @@ class GoogleDriveIE(InfoExtractor):
     @staticmethod
     def _extract_url(webpage):
         mobj = re.search(
-            r'<iframe[^>]+src="https?://(?:video\.google\.com/get_player\?.*?docid=|(?:docs|drive)\.google\.com/file/d/)(?P<id>[a-zA-Z0-9_-]{28})',
+            r'<iframe[^>]+src="https?://(?:video\.google\.com/get_player\?.*?docid=|(?:docs|drive)\.google\.com/file/d/)(?P<id>[a-zA-Z0-9_-]{28,})',
             webpage)
         if mobj:
             return 'https://drive.google.com/file/d/%s' % mobj.group('id')

From 09f572fbc07e79cc3f62e65149237ee1952aa12b Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 11 Mar 2016 22:37:07 +0100
Subject: [PATCH 4258/6219] [extractor/common] add transform_source to
 _download_smil and _extract_smil_formats

---
 youtube_dl/extractor/common.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index aaca25a12..0b8b906ab 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1147,8 +1147,8 @@ class InfoExtractor(object):
                 out.append('{%s}%s' % (namespace, c))
         return '/'.join(out)
 
-    def _extract_smil_formats(self, smil_url, video_id, fatal=True, f4m_params=None):
-        smil = self._download_smil(smil_url, video_id, fatal=fatal)
+    def _extract_smil_formats(self, smil_url, video_id, fatal=True, f4m_params=None, transform_source=None):
+        smil = self._download_smil(smil_url, video_id, fatal=fatal, transform_source=transform_source)
 
         if smil is False:
             assert not fatal
@@ -1165,10 +1165,10 @@ class InfoExtractor(object):
             return {}
         return self._parse_smil(smil, smil_url, video_id, f4m_params=f4m_params)
 
-    def _download_smil(self, smil_url, video_id, fatal=True):
+    def _download_smil(self, smil_url, video_id, fatal=True, transform_source=None):
         return self._download_xml(
             smil_url, video_id, 'Downloading SMIL file',
-            'Unable to download SMIL file', fatal=fatal)
+            'Unable to download SMIL file', fatal=fatal, transform_source=transform_source)
 
     def _parse_smil(self, smil, smil_url, video_id, f4m_params=None):
         namespace = self._parse_smil_namespace(smil)

From c8868a9d83291c4502126ca375300026cc90acf9 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 11 Mar 2016 22:44:18 +0100
Subject: [PATCH 4259/6219] [dw] Add new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/dw.py       | 43 ++++++++++++++++++++++++++++++++
 2 files changed, 44 insertions(+)
 create mode 100644 youtube_dl/extractor/dw.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 1a7d689df..4bc7fbab0 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -189,6 +189,7 @@ from .dumpert import DumpertIE
 from .defense import DefenseGouvFrIE
 from .discovery import DiscoveryIE
 from .dropbox import DropboxIE
+from .dw import DWIE
 from .eagleplatform import EaglePlatformIE
 from .ebaumsworld import EbaumsWorldIE
 from .echomsk import EchoMskIE
diff --git a/youtube_dl/extractor/dw.py b/youtube_dl/extractor/dw.py
new file mode 100644
index 000000000..df901e1b2
--- /dev/null
+++ b/youtube_dl/extractor/dw.py
@@ -0,0 +1,43 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import int_or_none
+
+
+class DWIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?dw\.com/(?:[^/]+/)+av-(?P<id>\d+)'
+    _TEST = {
+        'url': 'http://www.dw.com/en/intelligent-light/av-19112290',
+        'md5': '7372046e1815c5a534b43f3c3c36e6e9',
+        'info_dict': {
+            'id': '19112290',
+            'ext': 'mp4',
+            'title': 'Intelligent light',
+            'description': 'md5:90e00d5881719f2a6a5827cb74985af1',
+            'upload_date': '20160311',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        formats = self._extract_smil_formats(
+            'http://www.dw.com/smil/v-%s' % video_id, video_id,
+            transform_source=lambda s: s.replace(
+                'rtmp://tv-od.dw.de/flash/',
+                'http://tv-download.dw.de/dwtv_video/flv/'))
+
+        webpage = self._download_webpage(url, video_id)
+        hidden_inputs = self._hidden_inputs(webpage)
+        title = hidden_inputs['media_title']
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': self._og_search_description(webpage),
+            'thumbnail': hidden_inputs.get('preview_image'),
+            'duration': int_or_none(hidden_inputs.get('file_duration')),
+            'upload_date': hidden_inputs.get('display_date'),
+            'formats': formats,
+        }

From 91d6aafb483b15c819c587357f7fd626f7a7d380 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 11 Mar 2016 23:55:26 +0100
Subject: [PATCH 4260/6219] [dw] add support for audio pages

---
 youtube_dl/extractor/dw.py | 32 ++++++++++++++++++++++++--------
 1 file changed, 24 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/dw.py b/youtube_dl/extractor/dw.py
index df901e1b2..dde66b41d 100644
--- a/youtube_dl/extractor/dw.py
+++ b/youtube_dl/extractor/dw.py
@@ -7,7 +7,8 @@ from ..utils import int_or_none
 
 class DWIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?dw\.com/(?:[^/]+/)+av-(?P<id>\d+)'
-    _TEST = {
+    _TESTS = [{
+        # video
         'url': 'http://www.dw.com/en/intelligent-light/av-19112290',
         'md5': '7372046e1815c5a534b43f3c3c36e6e9',
         'info_dict': {
@@ -17,21 +18,36 @@ class DWIE(InfoExtractor):
             'description': 'md5:90e00d5881719f2a6a5827cb74985af1',
             'upload_date': '20160311',
         }
-    }
+    }, {
+        # audio
+        'url': 'http://www.dw.com/en/worldlink-my-business/av-19111941',
+        'md5': '2814c9a1321c3a51f8a7aeb067a360dd',
+        'info_dict': {
+            'id': '19111941',
+            'ext': 'mp3',
+            'title': 'WorldLink: My business',
+            'description': 'md5:bc9ca6e4e063361e21c920c53af12405',
+            'upload_date': '20160311',
+        }
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        formats = self._extract_smil_formats(
-            'http://www.dw.com/smil/v-%s' % video_id, video_id,
-            transform_source=lambda s: s.replace(
-                'rtmp://tv-od.dw.de/flash/',
-                'http://tv-download.dw.de/dwtv_video/flv/'))
-
         webpage = self._download_webpage(url, video_id)
         hidden_inputs = self._hidden_inputs(webpage)
         title = hidden_inputs['media_title']
 
+        formats = []
+        if hidden_inputs.get('player_type') == 'video' and hidden_inputs.get('stream_file') == '1':
+            formats = self._extract_smil_formats(
+                'http://www.dw.com/smil/v-%s' % video_id, video_id,
+                transform_source=lambda s: s.replace(
+                    'rtmp://tv-od.dw.de/flash/',
+                    'http://tv-download.dw.de/dwtv_video/flv/'))
+        else:
+            formats = [{'url': hidden_inputs['file_name']}]
+
         return {
             'id': video_id,
             'title': title,

From 36bb63e0841f1d4fc19e76805739fff60abc283b Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 12 Mar 2016 08:33:22 +0100
Subject: [PATCH 4261/6219] [dw] add support for article pages(closes #8790)

---
 youtube_dl/extractor/__init__.py |  5 ++++-
 youtube_dl/extractor/dw.py       | 36 +++++++++++++++++++++++++++-----
 2 files changed, 35 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 4bc7fbab0..c5ca01ee7 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -189,7 +189,10 @@ from .dumpert import DumpertIE
 from .defense import DefenseGouvFrIE
 from .discovery import DiscoveryIE
 from .dropbox import DropboxIE
-from .dw import DWIE
+from .dw import (
+    DWIE,
+    DWArticleIE,
+)
 from .eagleplatform import EaglePlatformIE
 from .ebaumsworld import EbaumsWorldIE
 from .echomsk import EchoMskIE
diff --git a/youtube_dl/extractor/dw.py b/youtube_dl/extractor/dw.py
index dde66b41d..b6c985547 100644
--- a/youtube_dl/extractor/dw.py
+++ b/youtube_dl/extractor/dw.py
@@ -3,9 +3,11 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..utils import int_or_none
+from ..compat import compat_urlparse
 
 
 class DWIE(InfoExtractor):
+    IE_NAME = 'dw'
     _VALID_URL = r'https?://(?:www\.)?dw\.com/(?:[^/]+/)+av-(?P<id>\d+)'
     _TESTS = [{
         # video
@@ -32,16 +34,15 @@ class DWIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        webpage = self._download_webpage(url, video_id)
+        media_id = self._match_id(url)
+        webpage = self._download_webpage(url, media_id)
         hidden_inputs = self._hidden_inputs(webpage)
         title = hidden_inputs['media_title']
 
         formats = []
         if hidden_inputs.get('player_type') == 'video' and hidden_inputs.get('stream_file') == '1':
             formats = self._extract_smil_formats(
-                'http://www.dw.com/smil/v-%s' % video_id, video_id,
+                'http://www.dw.com/smil/v-%s' % media_id, media_id,
                 transform_source=lambda s: s.replace(
                     'rtmp://tv-od.dw.de/flash/',
                     'http://tv-download.dw.de/dwtv_video/flv/'))
@@ -49,7 +50,7 @@ class DWIE(InfoExtractor):
             formats = [{'url': hidden_inputs['file_name']}]
 
         return {
-            'id': video_id,
+            'id': media_id,
             'title': title,
             'description': self._og_search_description(webpage),
             'thumbnail': hidden_inputs.get('preview_image'),
@@ -57,3 +58,28 @@ class DWIE(InfoExtractor):
             'upload_date': hidden_inputs.get('display_date'),
             'formats': formats,
         }
+
+
+class DWArticleIE(InfoExtractor):
+    IE_NAME = 'dw:article'
+    _VALID_URL = r'https?://(?:www\.)?dw\.com/(?:[^/]+/)+a-(?P<id>\d+)'
+    _TEST = {
+        'url': 'http://www.dw.com/en/no-hope-limited-options-for-refugees-in-idomeni/a-19111009',
+        'md5': '8ca657f9d068bbef74d6fc38b97fc869',
+        'info_dict': {
+            'id': '19105868',
+            'ext': 'mp4',
+            'title': 'The harsh life of refugees in Idomeni',
+            'description': 'md5:196015cc7e48ebf474db9399420043c7',
+            'upload_date': '20160310',
+        }
+    }
+
+    def _real_extract(self, url):
+        article_id = self._match_id(url)
+        webpage = self._download_webpage(url, article_id)
+        hidden_inputs = self._hidden_inputs(webpage)
+        media_id = hidden_inputs['media_id']
+        media_path = self._search_regex(r'href="([^"]+av-%s)"\s+class="overlayLink"' % media_id, webpage, 'media url')
+        media_url = compat_urlparse.urljoin(url, media_path)
+        return self.url_result(media_url, 'DW', media_id)

From e5a2e17a9c65dd481cfbda55a778d6536a832a15 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 12 Mar 2016 18:43:45 +0100
Subject: [PATCH 4262/6219] [kaltura] optimize url info extraction

---
 youtube_dl/extractor/kaltura.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/kaltura.py b/youtube_dl/extractor/kaltura.py
index 78e29f481..79a72d45f 100644
--- a/youtube_dl/extractor/kaltura.py
+++ b/youtube_dl/extractor/kaltura.py
@@ -137,10 +137,7 @@ class KalturaIE(InfoExtractor):
                 params = compat_parse_qs(query)
             if path:
                 splitted_path = path.split('/')
-                if not splitted_path[-1]:
-                    splitted_path = splitted_path[:-1]
-                for i in range(0, len(splitted_path), 2):
-                    params[splitted_path[i]] = [splitted_path[i + 1]]
+                params.update(dict((zip(splitted_path[::2], [[v] for v in splitted_path[1::2]]))))
             if 'wid' in params:
                 partner_id = params['wid'][0][1:]
             elif 'p' in params:

From 9dce3c095b377104c0a05f6907d73a5b6fae5245 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 13 Mar 2016 02:01:10 +0600
Subject: [PATCH 4263/6219] [kaltura] Respect kaltura session

---
 youtube_dl/extractor/kaltura.py | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/kaltura.py b/youtube_dl/extractor/kaltura.py
index 79a72d45f..44d7c84a1 100644
--- a/youtube_dl/extractor/kaltura.py
+++ b/youtube_dl/extractor/kaltura.py
@@ -125,7 +125,7 @@ class KalturaIE(InfoExtractor):
 
         mobj = re.match(self._VALID_URL, url)
         partner_id, entry_id = mobj.group('partner_id', 'id')
-        info, flavor_assets = None, None
+        ks = None
         if partner_id and entry_id:
             info, flavor_assets = self._get_video_info(entry_id, partner_id)
         else:
@@ -158,6 +158,7 @@ class KalturaIE(InfoExtractor):
                 entry_id = info['id']
             else:
                 raise ExtractorError('Invalid URL', expected=True)
+            ks = params.get('flashvars[ks]', [None])[0]
 
         source_url = smuggled_data.get('source_url')
         if source_url:
@@ -167,14 +168,19 @@ class KalturaIE(InfoExtractor):
         else:
             referrer = None
 
+        def sign_url(unsigned_url):
+            if ks:
+                unsigned_url += '/ks/%s' % ks
+            if referrer:
+                unsigned_url += '?referrer=%s' % referrer
+            return unsigned_url
+
         formats = []
         for f in flavor_assets:
             # Continue if asset is not ready
             if f['status'] != 2:
                 continue
-            video_url = '%s/flavorId/%s' % (info['dataUrl'], f['id'])
-            if referrer:
-                video_url += '?referrer=%s' % referrer
+            video_url = sign_url('%s/flavorId/%s' % (info['dataUrl'], f['id']))
             formats.append({
                 'format_id': '%(fileExt)s-%(bitrate)s' % f,
                 'ext': f.get('fileExt'),
@@ -187,9 +193,7 @@ class KalturaIE(InfoExtractor):
                 'width': int_or_none(f.get('width')),
                 'url': video_url,
             })
-        m3u8_url = info['dataUrl'].replace('format/url', 'format/applehttp')
-        if referrer:
-            m3u8_url += '?referrer=%s' % referrer
+        m3u8_url = sign_url(info['dataUrl'].replace('format/url', 'format/applehttp'))
         formats.extend(self._extract_m3u8_formats(
             m3u8_url, entry_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
 

From 73cbd709f9fae8e27bd019bce4f5764db05634a1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 13 Mar 2016 02:03:07 +0600
Subject: [PATCH 4264/6219] [safari] Respect kaltura session (Closes #7491)

---
 youtube_dl/extractor/safari.py | 25 +++++++++++++++++++------
 1 file changed, 19 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/safari.py b/youtube_dl/extractor/safari.py
index ae619636c..56a7dfb92 100644
--- a/youtube_dl/extractor/safari.py
+++ b/youtube_dl/extractor/safari.py
@@ -21,13 +21,12 @@ class SafariBaseIE(InfoExtractor):
     _SUCCESSFUL_LOGIN_REGEX = r'<a href="/accounts/logout/"[^>]*>Sign Out</a>'
     _NETRC_MACHINE = 'safari'
 
-    _API_BASE = 'https://www.safaribooksonline.com/api/v1/book'
+    _API_BASE = 'https://www.safaribooksonline.com/api/v1'
     _API_FORMAT = 'json'
 
     LOGGED_IN = False
 
     def _real_initialize(self):
-        return
         # We only need to log in once for courses or individual videos
         if not self.LOGGED_IN:
             self._login()
@@ -36,7 +35,7 @@ class SafariBaseIE(InfoExtractor):
     def _login(self):
         (username, password) = self._get_login_info()
         if username is None:
-            self.raise_login_required('safaribooksonline.com account is required')
+            return
 
         headers = std_headers.copy()
         if 'Referer' not in headers:
@@ -116,11 +115,25 @@ class SafariIE(SafariBaseIE):
         partner_id = self._search_regex(r'data-partner-id="([^"]+)"', webpage, 'kaltura widget id')
         ui_id = self._search_regex(r'data-ui-id="([^"]+)"', webpage, 'kaltura uiconf id')
 
-        return self.url_result(update_url_query('https://cdnapisec.kaltura.com/html5/html5lib/v2.37.1/mwEmbedFrame.php', {
+        query = {
             'wid': '_%s' % partner_id,
             'uiconf_id': ui_id,
             'flashvars[referenceId]': reference_id,
-        }), 'Kaltura')
+        }
+
+        if self.LOGGED_IN:
+            kaltura_session = self._download_json(
+                '%s/player/kaltura_session/?reference_id=%s' % (self._API_BASE, reference_id),
+                course_id, 'Downloading kaltura session JSON',
+                'Unable to download kaltura session JSON', fatal=False)
+            if kaltura_session:
+                session = kaltura_session.get('session')
+                if session:
+                    query['flashvars[ks]'] = session
+
+        return self.url_result(update_url_query(
+            'https://cdnapisec.kaltura.com/html5/html5lib/v2.37.1/mwEmbedFrame.php', query),
+            'Kaltura')
 
 
 class SafariCourseIE(SafariBaseIE):
@@ -146,7 +159,7 @@ class SafariCourseIE(SafariBaseIE):
         course_id = self._match_id(url)
 
         course_json = self._download_json(
-            '%s/%s/?override_format=%s' % (self._API_BASE, course_id, self._API_FORMAT),
+            '%s/book/%s/?override_format=%s' % (self._API_BASE, course_id, self._API_FORMAT),
             course_id, 'Downloading course JSON')
 
         if 'chapters' not in course_json:

From e9c8999ede2e07c64f2b592b12158d763dc933c8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 13 Mar 2016 02:08:36 +0600
Subject: [PATCH 4265/6219] [safari] Fix authentication

---
 youtube_dl/extractor/safari.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/safari.py b/youtube_dl/extractor/safari.py
index 56a7dfb92..10c7f39b0 100644
--- a/youtube_dl/extractor/safari.py
+++ b/youtube_dl/extractor/safari.py
@@ -27,12 +27,13 @@ class SafariBaseIE(InfoExtractor):
     LOGGED_IN = False
 
     def _real_initialize(self):
-        # We only need to log in once for courses or individual videos
-        if not self.LOGGED_IN:
-            self._login()
-            SafariBaseIE.LOGGED_IN = True
+        self._login()
 
     def _login(self):
+        # We only need to log in once for courses or individual videos
+        if self.LOGGED_IN:
+            return
+
         (username, password) = self._get_login_info()
         if username is None:
             return
@@ -68,6 +69,8 @@ class SafariBaseIE(InfoExtractor):
                 'Login failed; make sure your credentials are correct and try again.',
                 expected=True)
 
+        SafariBaseIE.LOGGED_IN = True
+
         self.to_screen('Login successful')
 
 

From 0fdbb3322b263aeb66aa9a774fa59a9bad33321c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 13 Mar 2016 03:16:08 +0600
Subject: [PATCH 4266/6219] [extractor/common] Add _parse_f4m_formats routine

---
 youtube_dl/extractor/common.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 0b8b906ab..ecd7da767 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -973,6 +973,13 @@ class InfoExtractor(object):
         if manifest is False:
             return []
 
+        return self._parse_f4m_formats(
+            manifest, manifest_url, video_id, preference=preference, f4m_id=f4m_id,
+            transform_source=transform_source, fatal=fatal)
+
+    def _parse_f4m_formats(self, manifest, manifest_url, video_id, preference=None, f4m_id=None,
+                           transform_source=lambda s: fix_xml_ampersands(s).strip(),
+                           fatal=True):
         formats = []
         manifest_version = '1.0'
         media_nodes = manifest.findall('{http://ns.adobe.com/f4m/1.0}media')
@@ -998,7 +1005,8 @@ class InfoExtractor(object):
                 # bitrate in f4m downloader
                 if determine_ext(manifest_url) == 'f4m':
                     formats.extend(self._extract_f4m_formats(
-                        manifest_url, video_id, preference, f4m_id, fatal=fatal))
+                        manifest_url, video_id, preference=preference, f4m_id=f4m_id,
+                        transform_source=transform_source, fatal=fatal))
                     continue
             tbr = int_or_none(media_el.attrib.get('bitrate'))
             formats.append({

From f930e0c76e4c0dea36bf9af3aef77f08888fcdf9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 13 Mar 2016 03:17:25 +0600
Subject: [PATCH 4267/6219] [extractor/generic] Extract f4m formats and
 refactor common info

---
 youtube_dl/extractor/generic.py | 37 +++++++++++++++++++--------------
 1 file changed, 21 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index ca745ae41..36d3a7433 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1242,28 +1242,34 @@ class GenericIE(InfoExtractor):
             full_response = self._request_webpage(request, video_id)
             head_response = full_response
 
+        info_dict = {
+            'id': video_id,
+            'title': compat_urllib_parse_unquote(os.path.splitext(url_basename(url))[0]),
+        }
+
         # Check for direct link to a video
         content_type = head_response.headers.get('Content-Type', '')
         m = re.match(r'^(?P<type>audio|video|application(?=/(?:ogg$|(?:vnd\.apple\.|x-)?mpegurl)))/(?P<format_id>.+)$', content_type)
         if m:
             upload_date = unified_strdate(
                 head_response.headers.get('Last-Modified'))
-            formats = []
-            if m.group('format_id').endswith('mpegurl'):
+            format_id = m.group('format_id')
+            if format_id.endswith('mpegurl'):
                 formats = self._extract_m3u8_formats(url, video_id, 'mp4')
+            elif format_id == 'f4m':
+                formats = self._extract_f4m_formats(url, video_id)
             else:
                 formats = [{
                     'format_id': m.group('format_id'),
                     'url': url,
                     'vcodec': 'none' if m.group('type') == 'audio' else None
                 }]
-            return {
-                'id': video_id,
-                'title': compat_urllib_parse_unquote(os.path.splitext(url_basename(url))[0]),
+            info_dict.update({
                 'direct': True,
                 'formats': formats,
                 'upload_date': upload_date,
-            }
+            })
+            return info_dict
 
         if not self._downloader.params.get('test', False) and not is_intentional:
             force = self._downloader.params.get('force_generic_extractor', False)
@@ -1291,13 +1297,12 @@ class GenericIE(InfoExtractor):
                 'URL could be a direct video link, returning it as such.')
             upload_date = unified_strdate(
                 head_response.headers.get('Last-Modified'))
-            return {
-                'id': video_id,
-                'title': compat_urllib_parse_unquote(os.path.splitext(url_basename(url))[0]),
+            info_dict.update({
                 'direct': True,
                 'url': url,
                 'upload_date': upload_date,
-            }
+            })
+            return info_dict
 
         webpage = self._webpage_read_content(
             full_response, url, video_id, prefix=first_bytes)
@@ -1314,12 +1319,12 @@ class GenericIE(InfoExtractor):
             elif doc.tag == '{http://xspf.org/ns/0/}playlist':
                 return self.playlist_result(self._parse_xspf(doc, video_id), video_id)
             elif re.match(r'(?i)^(?:{[^}]+})?MPD$', doc.tag):
-                return {
-                    'id': video_id,
-                    'title': compat_urllib_parse_unquote(os.path.splitext(url_basename(url))[0]),
-                    'formats': self._parse_mpd_formats(
-                        doc, video_id, mpd_base_url=url.rpartition('/')[0]),
-                }
+                info_dict['formats'] = self._parse_mpd_formats(
+                    doc, video_id, mpd_base_url=url.rpartition('/')[0])
+                return info_dict
+            elif re.match(r'^{http://ns\.adobe\.com/f4m/[12]\.0}manifest$', doc.tag):
+                info_dict['formats'] = self._parse_f4m_formats(doc, url, video_id)
+                return info_dict
         except compat_xml_parse_error:
             pass
 

From 3f2f4a94aa63e156e8ad19ea493f23e2afec6b62 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 13 Mar 2016 03:38:20 +0600
Subject: [PATCH 4268/6219] [extractor/generic] Extract f4m formats from final
 URLs

---
 youtube_dl/extractor/generic.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 36d3a7433..8121f04a5 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1990,6 +1990,8 @@ class GenericIE(InfoExtractor):
                 entry_info_dict['formats'] = self._extract_m3u8_formats(video_url, video_id, ext='mp4')
             elif ext == 'mpd':
                 entry_info_dict['formats'] = self._extract_mpd_formats(video_url, video_id)
+            elif ext == 'f4m':
+                entry_info_dict['formats'] = self._extract_f4m_formats(video_url, video_id)
             else:
                 entry_info_dict['url'] = video_url
 

From 46a4ea827633c6590727f4688e8f4fbb2a545922 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 13 Mar 2016 03:48:38 +0600
Subject: [PATCH 4269/6219] [safari] Remove unused imports

---
 youtube_dl/extractor/safari.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/youtube_dl/extractor/safari.py b/youtube_dl/extractor/safari.py
index 10c7f39b0..256396bb8 100644
--- a/youtube_dl/extractor/safari.py
+++ b/youtube_dl/extractor/safari.py
@@ -4,12 +4,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from .brightcove import BrightcoveLegacyIE
 
 from ..utils import (
     ExtractorError,
     sanitized_Request,
-    smuggle_url,
     std_headers,
     urlencode_postdata,
     update_url_query,

From b970dfddaf6a086b62733c21550447ef4933364f Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 12 Mar 2016 22:13:25 +0100
Subject: [PATCH 4270/6219] [minoto] Add new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/minoto.py   | 56 ++++++++++++++++++++++++++++++++
 2 files changed, 57 insertions(+)
 create mode 100644 youtube_dl/extractor/minoto.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index c5ca01ee7..ba9c8b7a5 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -398,6 +398,7 @@ from .metacritic import MetacriticIE
 from .mgoon import MgoonIE
 from .minhateca import MinhatecaIE
 from .ministrygrid import MinistryGridIE
+from .minoto import MinotoIE
 from .miomio import MioMioIE
 from .mit import TechTVMITIE, MITIE, OCWMITIE
 from .mitele import MiTeleIE
diff --git a/youtube_dl/extractor/minoto.py b/youtube_dl/extractor/minoto.py
new file mode 100644
index 000000000..959a10589
--- /dev/null
+++ b/youtube_dl/extractor/minoto.py
@@ -0,0 +1,56 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import int_or_none
+
+
+class MinotoIE(InfoExtractor):
+    _VALID_URL = r'(?:minoto:|https?://(?:play|iframe|embed)\.minoto-video\.com/(?P<player_id>[0-9]+)/)(?P<id>[a-zA-Z0-9]+)'
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        player_id = mobj.group('player_id') or '1'
+        video_id = mobj.group('id')
+        video_data = self._download_json('http://play.minoto-video.com/%s/%s.js' % (player_id, video_id), video_id)
+        video_metadata = video_data['video-metadata']
+        formats = []
+        for fmt in video_data['video-files']:
+            fmt_url = fmt.get('url')
+            if not fmt_url:
+                continue
+            container = fmt.get('container')
+            if container == 'hls':
+                formats.extend(fmt_url, video_id, 'mp4', m3u8_id='hls', fatal=False)
+            else:
+                fmt_profile = fmt.get('profile') or {}
+                f = {
+                    'format_id': fmt_profile.get('name-short'),
+                    'format_note': fmt_profile.get('name'),
+                    'url': fmt_url,
+                    'container': container,
+                    'tbr': int_or_none(fmt.get('bitrate')),
+                    'filesize': int_or_none(fmt.get('filesize')),
+                    'width': int_or_none(fmt.get('width')),
+                    'height': int_or_none(fmt.get('height')),
+                }
+                codecs = fmt.get('codecs')
+                if codecs:
+                    codecs = codecs.split(',')
+                    if len(codecs) == 2:
+                        f.update({
+                            'vcodec': codecs[0],
+                            'acodec': codecs[1],
+                        })
+                formats.append(f)
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': video_metadata['title'],
+            'description': video_metadata.get('description'),
+            'thumbnail': video_metadata.get('video-poster', {}).get('url'),
+            'formats': formats,
+        }

From a42dfa629eccf4fc3c2a8d74b8d753a607c0fb25 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 12 Mar 2016 22:51:36 +0100
Subject: [PATCH 4271/6219] [makerschannel] Add new extractor(closes #8839)

---
 youtube_dl/extractor/__init__.py      |  1 +
 youtube_dl/extractor/makerschannel.py | 40 +++++++++++++++++++++++++++
 2 files changed, 41 insertions(+)
 create mode 100644 youtube_dl/extractor/makerschannel.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index ba9c8b7a5..dc5ecea30 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -389,6 +389,7 @@ from .lynda import (
 from .m6 import M6IE
 from .macgamestore import MacGameStoreIE
 from .mailru import MailRuIE
+from .makerschannel import MakersChannelIE
 from .makertv import MakerTVIE
 from .malemotion import MalemotionIE
 from .matchtv import MatchTVIE
diff --git a/youtube_dl/extractor/makerschannel.py b/youtube_dl/extractor/makerschannel.py
new file mode 100644
index 000000000..f5d00e61d
--- /dev/null
+++ b/youtube_dl/extractor/makerschannel.py
@@ -0,0 +1,40 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+
+class MakersChannelIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?makerschannel\.com/.*(?P<id_type>video|production)_id=(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'http://makerschannel.com/en/zoomin/community-highlights?video_id=849',
+        'md5': '624a512c6969236b5967bf9286345ad1',
+        'info_dict': {
+            'id': '849',
+            'ext': 'mp4',
+            'title': 'Landing a bus on a plane is an epic win',
+            'uploader': 'ZoomIn',
+            'description': 'md5:cd9cca2ea7b69b78be81d07020c97139',
+        }
+    }
+
+    def _real_extract(self, url):
+        id_type, url_id = re.match(self._VALID_URL, url).groups()
+        webpage = self._download_webpage(url, url_id)
+        video_data = self._html_search_regex(r'<div([^>]+data-%s-id="%s"[^>]+)>' % (id_type, url_id), webpage, 'video data')
+
+        def extract_data_val(attr, fatal=False):
+            return self._html_search_regex(r'data-%s\s*=\s*"([^"]+)"' % attr, video_data, attr, fatal=fatal)
+        minoto_id = self._search_regex(r'/id/([a-zA-Z0-9]+)', extract_data_val('video-src', True), 'minoto id')
+
+        return {
+            '_type': 'url_transparent',
+            'url': 'minoto:%s' % minoto_id,
+            'id': extract_data_val('video-id', True),
+            'title': extract_data_val('title', True),
+            'description': extract_data_val('description'),
+            'thumbnail': extract_data_val('image'),
+            'uploader': extract_data_val('channel'),
+        }

From 8e4aa7bf18af4403bf98742270483f3b9cfbdeb6 Mon Sep 17 00:00:00 2001
From: Benjamin Congdon <bencongdon96@gmail.com>
Date: Fri, 11 Mar 2016 09:31:35 -0600
Subject: [PATCH 4272/6219] [bbc] Fix BBC Extractor to work with 'School
 Report'

---
 youtube_dl/extractor/bbc.py | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 9d0dfb961..f4d8b4a2f 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -10,7 +10,6 @@ from ..utils import (
     int_or_none,
     parse_duration,
     parse_iso8601,
-    remove_end,
     unescapeHTML,
 )
 from ..compat import (
@@ -796,9 +795,15 @@ class BBCIE(BBCCoUkIE):
                             entries.append(self._extract_from_playlist_sxml(
                                 playlist.get('progressiveDownloadUrl'), playlist_id, timestamp))
 
+        playlist_title = self._og_search_title(webpage, default=None)
+        playlist_title = playlist_title or self._html_search_regex(
+            r'<title>(.*?)</title>', webpage, 'playlist title')
+
+        playlist_title = self._search_regex(r'(.+)\s*-\s*BBC', playlist_title, 'title', default=playlist_title)
+
+        playlist_description = self._og_search_description(webpage, default=None)
+
         if entries:
-            playlist_title = playlist_title or remove_end(self._og_search_title(webpage), ' - BBC News')
-            playlist_description = playlist_description or self._og_search_description(webpage, default=None)
             return self.playlist_result(entries, playlist_id, playlist_title, playlist_description)
 
         # single video story (e.g. http://www.bbc.com/travel/story/20150625-sri-lankas-spicy-secret)
@@ -829,10 +834,6 @@ class BBCIE(BBCCoUkIE):
                 'subtitles': subtitles,
             }
 
-        playlist_title = self._html_search_regex(
-            r'<title>(.*?)(?:\s*-\s*BBC [^ ]+)?</title>', webpage, 'playlist title')
-        playlist_description = self._og_search_description(webpage, default=None)
-
         def extract_all(pattern):
             return list(filter(None, map(
                 lambda s: self._parse_json(s, playlist_id, fatal=False),

From 0e832c2c97c62f67593ad356ea6d507778c56759 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 13 Mar 2016 15:54:56 +0600
Subject: [PATCH 4273/6219] [bbc] Improve title and description extraction
 (Closes #8826, closes #8822)

---
 youtube_dl/extractor/bbc.py | 27 ++++++++++++++++++---------
 1 file changed, 18 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index f4d8b4a2f..497ebfd72 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -563,6 +563,14 @@ class BBCIE(BBCCoUkIE):
             'title': 'BBC Blogs - Adam Curtis - BUGGER',
         },
         'playlist_count': 18,
+    }, {
+        # school report playlist with single video
+        'url': 'http://www.bbc.co.uk/schoolreport/35744779',
+        'info_dict': {
+            'id': '35744779',
+            'title': 'School which breaks down barriers in Jerusalem',
+        },
+        'playlist_count': 1,
     }, {
         # single video embedded with data-playable containing vpid
         'url': 'http://www.bbc.com/news/world-europe-32041533',
@@ -734,8 +742,17 @@ class BBCIE(BBCCoUkIE):
 
         json_ld_info = self._search_json_ld(webpage, playlist_id, default=None)
         timestamp = json_ld_info.get('timestamp')
+
         playlist_title = json_ld_info.get('title')
-        playlist_description = json_ld_info.get('description')
+        if not playlist_title:
+            playlist_title = self._og_search_title(
+                webpage, default=None) or self._html_search_regex(
+                r'<title>(.+?)</title>', webpage, 'playlist title', default=None)
+            if playlist_title:
+                playlist_title = re.sub(r'(.+)\s*-\s*BBC.*?$', r'\1', playlist_title).strip()
+
+        playlist_description = json_ld_info.get(
+            'description') or self._og_search_description(webpage, default=None)
 
         if not timestamp:
             timestamp = parse_iso8601(self._search_regex(
@@ -795,14 +812,6 @@ class BBCIE(BBCCoUkIE):
                             entries.append(self._extract_from_playlist_sxml(
                                 playlist.get('progressiveDownloadUrl'), playlist_id, timestamp))
 
-        playlist_title = self._og_search_title(webpage, default=None)
-        playlist_title = playlist_title or self._html_search_regex(
-            r'<title>(.*?)</title>', webpage, 'playlist title')
-
-        playlist_title = self._search_regex(r'(.+)\s*-\s*BBC', playlist_title, 'title', default=playlist_title)
-
-        playlist_description = self._og_search_description(webpage, default=None)
-
         if entries:
             return self.playlist_result(entries, playlist_id, playlist_title, playlist_description)
 

From b7d7674f1ead233bcf9d4237ab893e31ec9d5ec8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 13 Mar 2016 15:56:34 +0600
Subject: [PATCH 4274/6219] [bbc] Update test

---
 youtube_dl/extractor/bbc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 497ebfd72..b46bc4856 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -560,7 +560,7 @@ class BBCIE(BBCCoUkIE):
         'url': 'http://www.bbc.co.uk/blogs/adamcurtis/entries/3662a707-0af9-3149-963f-47bea720b460',
         'info_dict': {
             'id': '3662a707-0af9-3149-963f-47bea720b460',
-            'title': 'BBC Blogs - Adam Curtis - BUGGER',
+            'title': 'BUGGER',
         },
         'playlist_count': 18,
     }, {

From 450b233cc293e9a1fb63daf0ee0deb48a809dfd9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 13 Mar 2016 15:59:54 +0600
Subject: [PATCH 4275/6219] [bbc] Update test

---
 youtube_dl/extractor/bbc.py | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index b46bc4856..e62b3860e 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -563,14 +563,6 @@ class BBCIE(BBCCoUkIE):
             'title': 'BUGGER',
         },
         'playlist_count': 18,
-    }, {
-        # school report playlist with single video
-        'url': 'http://www.bbc.co.uk/schoolreport/35744779',
-        'info_dict': {
-            'id': '35744779',
-            'title': 'School which breaks down barriers in Jerusalem',
-        },
-        'playlist_count': 1,
     }, {
         # single video embedded with data-playable containing vpid
         'url': 'http://www.bbc.com/news/world-europe-32041533',
@@ -677,9 +669,17 @@ class BBCIE(BBCCoUkIE):
         'url': 'http://www.bbc.com/sport/0/football/34475836',
         'info_dict': {
             'id': '34475836',
-            'title': 'What Liverpool can expect from Klopp',
+            'title': 'Jurgen Klopp: Furious football from a witty and winning coach',
         },
         'playlist_count': 3,
+    }, {
+        # school report article with single video
+        'url': 'http://www.bbc.co.uk/schoolreport/35744779',
+        'info_dict': {
+            'id': '35744779',
+            'title': 'School which breaks down barriers in Jerusalem',
+        },
+        'playlist_count': 1,
     }, {
         # single video with playlist URL from weather section
         'url': 'http://www.bbc.com/weather/features/33601775',

From fb47597b09cf8474931a4509bebc510403b30e25 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 13 Mar 2016 16:27:20 +0600
Subject: [PATCH 4276/6219] [bbc] Generalize unit table lookup and add
 parse_count

---
 test/test_utils.py  | 10 ++++++++++
 youtube_dl/utils.py | 37 ++++++++++++++++++++++++++++++-------
 2 files changed, 40 insertions(+), 7 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 2bcf8ecf0..bc28ceb34 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -41,6 +41,7 @@ from youtube_dl.utils import (
     orderedSet,
     parse_duration,
     parse_filesize,
+    parse_count,
     parse_iso8601,
     read_batch_urls,
     sanitize_filename,
@@ -653,6 +654,15 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(parse_filesize('1.2Tb'), 1200000000000)
         self.assertEqual(parse_filesize('1,24 KB'), 1240)
 
+    def test_parse_count(self):
+        self.assertEqual(parse_count(None), None)
+        self.assertEqual(parse_count(''), None)
+        self.assertEqual(parse_count('0'), 0)
+        self.assertEqual(parse_count('1000'), 1000)
+        self.assertEqual(parse_count('1.000'), 1000)
+        self.assertEqual(parse_count('1.1k'), 1100)
+        self.assertEqual(parse_count('1.1kk'), 1100000)
+
     def test_version_tuple(self):
         self.assertEqual(version_tuple('1'), (1,))
         self.assertEqual(version_tuple('10.23.344'), (10, 23, 344))
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 22a39a0ab..1d6ee9964 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1316,6 +1316,17 @@ def format_bytes(bytes):
     return '%.2f%s' % (converted, suffix)
 
 
+def lookup_unit_table(unit_table, s):
+    units_re = '|'.join(re.escape(u) for u in unit_table)
+    m = re.match(
+        r'(?P<num>[0-9]+(?:[,.][0-9]*)?)\s*(?P<unit>%s)' % units_re, s)
+    if not m:
+        return None
+    num_str = m.group('num').replace(',', '.')
+    mult = unit_table[m.group('unit')]
+    return int(float(num_str) * mult)
+
+
 def parse_filesize(s):
     if s is None:
         return None
@@ -1359,16 +1370,28 @@ def parse_filesize(s):
         'Yb': 1000 ** 8,
     }
 
-    units_re = '|'.join(re.escape(u) for u in _UNIT_TABLE)
-    m = re.match(
-        r'(?P<num>[0-9]+(?:[,.][0-9]*)?)\s*(?P<unit>%s)' % units_re, s)
-    if not m:
+    return lookup_unit_table(_UNIT_TABLE, s)
+
+
+def parse_count(s):
+    if s is None:
         return None
 
-    num_str = m.group('num').replace(',', '.')
-    mult = _UNIT_TABLE[m.group('unit')]
-    return int(float(num_str) * mult)
+    s = s.strip()
 
+    if re.match(r'^[\d,.]+$', s):
+        return str_to_int(s)
+
+    _UNIT_TABLE = {
+        'k': 1000,
+        'K': 1000,
+        'm': 1000 ** 2,
+        'M': 1000 ** 2,
+        'kk': 1000 ** 2,
+        'KK': 1000 ** 2,
+    }
+
+    return lookup_unit_table(_UNIT_TABLE, s)
 
 def month_by_name(name):
     """ Return the number of a month by (locale-independently) English name """

From 7f4173ae7c0dae870250018cd4915df36ccf0f54 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 13 Mar 2016 16:27:58 +0600
Subject: [PATCH 4277/6219] [mixcloud] Fix view count extraction (Closes #8831,
 closes #8845)

---
 youtube_dl/extractor/mixcloud.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dl/extractor/mixcloud.py
index c2b7ed9ab..101497118 100644
--- a/youtube_dl/extractor/mixcloud.py
+++ b/youtube_dl/extractor/mixcloud.py
@@ -7,6 +7,7 @@ from ..compat import compat_urllib_parse_unquote
 from ..utils import (
     ExtractorError,
     HEADRequest,
+    parse_count,
     str_to_int,
 )
 
@@ -85,8 +86,8 @@ class MixcloudIE(InfoExtractor):
         uploader_id = self._search_regex(
             r'\s+"profile": "([^"]+)",', webpage, 'uploader id', fatal=False)
         description = self._og_search_description(webpage)
-        like_count = str_to_int(self._search_regex(
-            r'\bbutton-favorite\b[^>]+m-ajax-toggle-count="([^"]+)"',
+        like_count = parse_count(self._search_regex(
+            r'\bbutton-favorite[^>]+>.*?<span[^>]+class=["\']toggle-number[^>]+>\s*([^<]+)',
             webpage, 'like count', fatal=False))
         view_count = str_to_int(self._search_regex(
             [r'<meta itemprop="interactionCount" content="UserPlays:([0-9]+)"',

From a755f82549cb6bffd8ff9545e0b0a4883b7422d2 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 13 Mar 2016 12:15:29 +0100
Subject: [PATCH 4278/6219] [ffmpeg] convert format ext to ffmpeg output
 formats codes

---
 youtube_dl/downloader/external.py  |  4 ++--
 youtube_dl/postprocessor/ffmpeg.py | 13 +++++++++++++
 2 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index a4fdf1af8..fcb956e6c 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -6,7 +6,7 @@ import sys
 import re
 
 from .common import FileDownloader
-from ..postprocessor.ffmpeg import FFmpegPostProcessor
+from ..postprocessor.ffmpeg import FFmpegPostProcessor, EXT_TO_OUT_FORMATS
 from ..compat import compat_str
 from ..utils import (
     cli_option,
@@ -197,7 +197,7 @@ class FFmpegFD(ExternalFD):
             else:
                 args += ['-f', 'mp4', '-bsf:a', 'aac_adtstoasc']
         else:
-            args += ['-f', info_dict['ext']]
+            args += ['-f', EXT_TO_OUT_FORMATS.get(info_dict['ext'], info_dict['ext'])]
 
         args = [encodeArgument(opt) for opt in args]
         args.append(encodeFilename(ffpp._ffmpeg_filename_argument(tmpfilename), True))
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 380bc6f29..cc7aaeda3 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -25,6 +25,19 @@ from ..utils import (
 )
 
 
+EXT_TO_OUT_FORMATS = {
+    "aac": "adts",
+    "m4a": "ipod",
+    "mka": "matroska",
+    "mkv": "matroska",
+    "mpg": "mpeg",
+    "ogv": "ogg",
+    "ts": "mpegts",
+    "wma": "asf",
+    "wmv": "asf",
+}
+
+
 class FFmpegPostProcessorError(PostProcessingError):
     pass
 

From 0a477f873115a11dbaadae674c2aaabe15ec4249 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 13 Mar 2016 17:22:23 +0600
Subject: [PATCH 4279/6219] [vice:show] Add extractor (Closes #8847)

---
 youtube_dl/extractor/__init__.py |  5 ++-
 youtube_dl/extractor/vice.py     | 76 ++++++++++++++++++++++++--------
 2 files changed, 61 insertions(+), 20 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index dc5ecea30..c5b80f4aa 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -836,7 +836,10 @@ from .vgtv import (
     VGTVIE,
 )
 from .vh1 import VH1IE
-from .vice import ViceIE
+from .vice import (
+    ViceIE,
+    ViceShowIE,
+)
 from .viddler import ViddlerIE
 from .videodetective import VideoDetectiveIE
 from .videofyme import VideofyMeIE
diff --git a/youtube_dl/extractor/vice.py b/youtube_dl/extractor/vice.py
index 3db6286e4..46c785ae1 100644
--- a/youtube_dl/extractor/vice.py
+++ b/youtube_dl/extractor/vice.py
@@ -1,31 +1,37 @@
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from .ooyala import OoyalaIE
 from ..utils import ExtractorError
 
 
 class ViceIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:.+?\.)?vice\.com/(?:[^/]+/)+(?P<id>.+)'
+    _VALID_URL = r'https?://(?:.+?\.)?vice\.com/(?:[^/]+/)?videos?/(?P<id>[^/?#&]+)'
 
-    _TESTS = [
-        {
-            'url': 'http://www.vice.com/Fringes/cowboy-capitalists-part-1',
-            'info_dict': {
-                'id': '43cW1mYzpia9IlestBjVpd23Yu3afAfp',
-                'ext': 'mp4',
-                'title': 'VICE_COWBOYCAPITALISTS_PART01_v1_VICE_WM_1080p.mov',
-                'duration': 725.983,
-            },
-            'params': {
-                # Requires ffmpeg (m3u8 manifest)
-                'skip_download': True,
-            },
-        }, {
-            'url': 'https://news.vice.com/video/experimenting-on-animals-inside-the-monkey-lab',
-            'only_matching': True,
-        }
-    ]
+    _TESTS = [{
+        'url': 'http://www.vice.com/video/cowboy-capitalists-part-1',
+        'info_dict': {
+            'id': '43cW1mYzpia9IlestBjVpd23Yu3afAfp',
+            'ext': 'mp4',
+            'title': 'VICE_COWBOYCAPITALISTS_PART01_v1_VICE_WM_1080p.mov',
+            'duration': 725.983,
+        },
+        'params': {
+            # Requires ffmpeg (m3u8 manifest)
+            'skip_download': True,
+        },
+    }, {
+        'url': 'https://news.vice.com/video/experimenting-on-animals-inside-the-monkey-lab',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.vice.com/ru/video/big-night-out-ibiza-clive-martin-229',
+        'only_matching': True,
+    }, {
+        'url': 'https://munchies.vice.com/en/videos/watch-the-trailer-for-our-new-series-the-pizza-show',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -38,3 +44,35 @@ class ViceIE(InfoExtractor):
         except ExtractorError:
             raise ExtractorError('The page doesn\'t contain a video', expected=True)
         return self.url_result(ooyala_url, ie='Ooyala')
+
+
+class ViceShowIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:.+?\.)?vice\.com/(?:[^/]+/)?show/(?P<id>[^/?#&]+)'
+
+    _TEST = {
+        'url': 'https://munchies.vice.com/en/show/fuck-thats-delicious-2',
+        'info_dict': {
+            'id': 'fuck-thats-delicious-2',
+            'title': "Fuck, That's Delicious",
+            'description': 'Follow the culinary adventures of rapper Action Bronson during his ongoing world tour.',
+        },
+        'playlist_count': 17,
+    }
+
+    def _real_extract(self, url):
+        show_id = self._match_id(url)
+        webpage = self._download_webpage(url, show_id)
+
+        entries = [
+            self.url_result(video_url, ViceIE.ie_key())
+            for video_url, _ in re.findall(
+                r'<h2[^>]+class="article-title"[^>]+data-id="\d+"[^>]*>\s*<a[^>]+href="(%s.*?)"'
+                % ViceIE._VALID_URL, webpage)]
+
+        title = self._search_regex(
+            r'<title>(.+?)</title>', webpage, 'title', default=None)
+        if title:
+            title = re.sub(r'(.+)\s*\|\s*.+$', r'\1', title).strip()
+        description = self._html_search_meta('description', webpage, 'description')
+
+        return self.playlist_result(entries, show_id, title, description)

From 2f7ae819ac844f4052dcd4ef031fdae01daff3b8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 13 Mar 2016 17:23:08 +0600
Subject: [PATCH 4280/6219] [utils] PEP 8

---
 youtube_dl/utils.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 1d6ee9964..9fd0ec8d5 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1393,6 +1393,7 @@ def parse_count(s):
 
     return lookup_unit_table(_UNIT_TABLE, s)
 
+
 def month_by_name(name):
     """ Return the number of a month by (locale-independently) English name """
 

From 634415ca1791b22ddb8a272266084d8b22199d09 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 13 Mar 2016 12:23:10 +0100
Subject: [PATCH 4281/6219] [downloader/external] skip FFmpegFD when requesting
 multiple formats

---
 youtube_dl/downloader/external.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index fcb956e6c..4e299b4ee 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -162,7 +162,7 @@ class HttpieFD(ExternalFD):
 class FFmpegFD(ExternalFD):
     @classmethod
     def supports(cls, info_dict):
-        return info_dict['protocol'] in ('http', 'https', 'ftp', 'ftps', 'm3u8', 'rtsp', 'rtmp', 'mms')
+        return info_dict['protocol'] in ('http', 'https', 'ftp', 'ftps', 'm3u8', 'rtsp', 'rtmp', 'mms') and not info_dict.get('requested_formats')
 
     @classmethod
     def available(cls):

From b7b72db9ad248e16a7e26489fee93e52b70948d7 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 10 Mar 2016 20:17:35 +0100
Subject: [PATCH 4282/6219] [YoutubeDL] check for --list-thumbnails immediately
 after processing them

---
 youtube_dl/YoutubeDL.py | 16 ++++++----------
 1 file changed, 6 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index b3391088f..8c651cd52 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1234,6 +1234,10 @@ class YoutubeDL(object):
                 if t.get('id') is None:
                     t['id'] = '%d' % i
 
+        if self.params.get('list_thumbnails'):
+            self.list_thumbnails(info_dict)
+            return
+
         if thumbnails and 'thumbnail' not in info_dict:
             info_dict['thumbnail'] = thumbnails[-1]['url']
 
@@ -1335,9 +1339,6 @@ class YoutubeDL(object):
         if self.params.get('listformats'):
             self.list_formats(info_dict)
             return
-        if self.params.get('list_thumbnails'):
-            self.list_thumbnails(info_dict)
-            return
 
         req_format = self.params.get('format')
         if req_format is None:
@@ -1901,13 +1902,8 @@ class YoutubeDL(object):
     def list_thumbnails(self, info_dict):
         thumbnails = info_dict.get('thumbnails')
         if not thumbnails:
-            tn_url = info_dict.get('thumbnail')
-            if tn_url:
-                thumbnails = [{'id': '0', 'url': tn_url}]
-            else:
-                self.to_screen(
-                    '[info] No thumbnails present for %s' % info_dict['id'])
-                return
+            self.to_screen('[info] No thumbnails present for %s' % info_dict['id'])
+            return
 
         self.to_screen(
             '[info] Thumbnails for %s:' % info_dict['id'])

From 91ee320bfaa279571b27bf44730c8677ff2b3980 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 13 Mar 2016 14:37:45 +0100
Subject: [PATCH 4283/6219] [downloader/external] wrap available_opt in a list

---
 youtube_dl/downloader/external.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index 4e299b4ee..daedf66de 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -53,7 +53,7 @@ class ExternalFD(FileDownloader):
 
     @classmethod
     def available(cls):
-        return check_executable(cls.get_basename(), cls.available_opt)
+        return check_executable(cls.get_basename(), [cls.AVAILABLE_OPT])
 
     @classmethod
     def supports(cls, info_dict):
@@ -86,7 +86,7 @@ class ExternalFD(FileDownloader):
 
 
 class CurlFD(ExternalFD):
-    available_opt = ['-V']
+    AVAILABLE_OPT = '-V'
 
     def _make_cmd(self, tmpfilename, info_dict):
         cmd = [self.exe, '--location', '-o', tmpfilename]
@@ -101,7 +101,7 @@ class CurlFD(ExternalFD):
 
 
 class AxelFD(ExternalFD):
-    available_opt = ['-V']
+    AVAILABLE_OPT = '-V'
 
     def _make_cmd(self, tmpfilename, info_dict):
         cmd = [self.exe, '-o', tmpfilename]
@@ -113,7 +113,7 @@ class AxelFD(ExternalFD):
 
 
 class WgetFD(ExternalFD):
-    available_opt = ['--version']
+    AVAILABLE_OPT = '--version'
 
     def _make_cmd(self, tmpfilename, info_dict):
         cmd = [self.exe, '-O', tmpfilename, '-nv', '--no-cookies']
@@ -128,7 +128,7 @@ class WgetFD(ExternalFD):
 
 
 class Aria2cFD(ExternalFD):
-    available_opt = ['-v']
+    AVAILABLE_OPT = '-v'
 
     def _make_cmd(self, tmpfilename, info_dict):
         cmd = [self.exe, '-c']

From 2cb99ebbd0284f0de4bafd03179653bb9599b080 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 13 Mar 2016 14:53:17 +0100
Subject: [PATCH 4284/6219] [downloader/external] add can_download mathod for
 checking downloader availibilty and support

---
 youtube_dl/downloader/__init__.py | 4 ++--
 youtube_dl/downloader/external.py | 4 ++++
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/downloader/__init__.py b/youtube_dl/downloader/__init__.py
index 67c2840a5..daa5498d4 100644
--- a/youtube_dl/downloader/__init__.py
+++ b/youtube_dl/downloader/__init__.py
@@ -31,13 +31,13 @@ def get_suitable_downloader(info_dict, params={}):
     protocol = determine_protocol(info_dict)
     info_dict['protocol'] = protocol
 
-    if (info_dict.get('start_time') or info_dict.get('end_time')) and FFmpegFD.available() and FFmpegFD.supports(info_dict):
+    if (info_dict.get('start_time') or info_dict.get('end_time')) and FFmpegFD.can_download(info_dict):
         return FFmpegFD
 
     external_downloader = params.get('external_downloader')
     if external_downloader is not None:
         ed = get_external_downloader(external_downloader)
-        if ed.available() and ed.supports(info_dict):
+        if ed.can_download(info_dict):
             return ed
 
     if protocol == 'm3u8' and params.get('hls_prefer_native'):
diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index daedf66de..fe2a0198c 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -59,6 +59,10 @@ class ExternalFD(FileDownloader):
     def supports(cls, info_dict):
         return info_dict['protocol'] in ('http', 'https', 'ftp', 'ftps')
 
+    @classmethod
+    def can_download(cls, info_dict):
+        return cls.available() and cls.supports(info_dict)
+
     def _option(self, command_option, param):
         return cli_option(self.params, command_option, param)
 

From be24916a7f6af9540076a6bc39bc78a71d6264bb Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 13 Mar 2016 15:24:02 +0100
Subject: [PATCH 4285/6219] [downloader/rtsp] Add rtsp and mms downloader

---
 youtube_dl/downloader/__init__.py |  5 ++--
 youtube_dl/downloader/rtsp.py     | 45 +++++++++++++++++++++++++++++++
 2 files changed, 48 insertions(+), 2 deletions(-)
 create mode 100644 youtube_dl/downloader/rtsp.py

diff --git a/youtube_dl/downloader/__init__.py b/youtube_dl/downloader/__init__.py
index daa5498d4..0b65aa3e2 100644
--- a/youtube_dl/downloader/__init__.py
+++ b/youtube_dl/downloader/__init__.py
@@ -6,6 +6,7 @@ from .hls import HlsFD
 from .http import HttpFD
 from .rtmp import RtmpFD
 from .dash import DashSegmentsFD
+from .rtsp import RtspFD
 from .external import (
     get_external_downloader,
     FFmpegFD,
@@ -19,8 +20,8 @@ PROTOCOL_MAP = {
     'rtmp': RtmpFD,
     'm3u8_native': HlsFD,
     'm3u8': FFmpegFD,
-    'mms': FFmpegFD,
-    'rtsp': FFmpegFD,
+    'mms': RtspFD,
+    'rtsp': RtspFD,
     'f4m': F4mFD,
     'http_dash_segments': DashSegmentsFD,
 }
diff --git a/youtube_dl/downloader/rtsp.py b/youtube_dl/downloader/rtsp.py
new file mode 100644
index 000000000..3eb29526c
--- /dev/null
+++ b/youtube_dl/downloader/rtsp.py
@@ -0,0 +1,45 @@
+from __future__ import unicode_literals
+
+import os
+import subprocess
+
+from .common import FileDownloader
+from ..utils import (
+    check_executable,
+    encodeFilename,
+)
+
+
+class RtspFD(FileDownloader):
+    def real_download(self, filename, info_dict):
+        url = info_dict['url']
+        self.report_destination(filename)
+        tmpfilename = self.temp_name(filename)
+
+        if check_executable('mplayer', ['-h']):
+            args = [
+                'mplayer', '-really-quiet', '-vo', 'null', '-vc', 'dummy',
+                '-dumpstream', '-dumpfile', tmpfilename, url]
+        elif check_executable('mpv', ['-h']):
+            args = [
+                'mpv', '-really-quiet', '--vo=null', '--stream-dump=' + tmpfilename, url]
+        else:
+            self.report_error('MMS or RTSP download detected but neither "mplayer" nor "mpv" could be run. Please install any.')
+            return False
+
+        retval = subprocess.call(args)
+        if retval == 0:
+            fsize = os.path.getsize(encodeFilename(tmpfilename))
+            self.to_screen('\r[%s] %s bytes' % (args[0], fsize))
+            self.try_rename(tmpfilename, filename)
+            self._hook_progress({
+                'downloaded_bytes': fsize,
+                'total_bytes': fsize,
+                'filename': filename,
+                'status': 'finished',
+            })
+            return True
+        else:
+            self.to_stderr('\n')
+            self.report_error('%s exited with code %d' % (args[0], retval))
+            return False

From da1973a0383ddc8125702e74a88d3a35bbfcf06d Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 13 Mar 2016 16:16:26 +0100
Subject: [PATCH 4286/6219] [extractor/__init__] disable time range downloading

---
 youtube_dl/downloader/__init__.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/downloader/__init__.py b/youtube_dl/downloader/__init__.py
index 0b65aa3e2..19471a250 100644
--- a/youtube_dl/downloader/__init__.py
+++ b/youtube_dl/downloader/__init__.py
@@ -32,8 +32,8 @@ def get_suitable_downloader(info_dict, params={}):
     protocol = determine_protocol(info_dict)
     info_dict['protocol'] = protocol
 
-    if (info_dict.get('start_time') or info_dict.get('end_time')) and FFmpegFD.can_download(info_dict):
-        return FFmpegFD
+    # if (info_dict.get('start_time') or info_dict.get('end_time')) and FFmpegFD.can_download(info_dict):
+    #     return FFmpegFD
 
     external_downloader = params.get('external_downloader')
     if external_downloader is not None:

From 6ae27bed01111ddd3c533fb94ae187306e7cf940 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 13 Mar 2016 20:25:39 +0100
Subject: [PATCH 4287/6219] [download/external] move the check for multiple
 selected formats to get_suitable_downloader

---
 youtube_dl/downloader/__init__.py | 2 +-
 youtube_dl/downloader/external.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/downloader/__init__.py b/youtube_dl/downloader/__init__.py
index 19471a250..73b34fdae 100644
--- a/youtube_dl/downloader/__init__.py
+++ b/youtube_dl/downloader/__init__.py
@@ -32,7 +32,7 @@ def get_suitable_downloader(info_dict, params={}):
     protocol = determine_protocol(info_dict)
     info_dict['protocol'] = protocol
 
-    # if (info_dict.get('start_time') or info_dict.get('end_time')) and FFmpegFD.can_download(info_dict):
+    # if (info_dict.get('start_time') or info_dict.get('end_time')) and not info_dict.get('requested_formats') and FFmpegFD.can_download(info_dict):
     #     return FFmpegFD
 
     external_downloader = params.get('external_downloader')
diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index fe2a0198c..85cf834c7 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -166,7 +166,7 @@ class HttpieFD(ExternalFD):
 class FFmpegFD(ExternalFD):
     @classmethod
     def supports(cls, info_dict):
-        return info_dict['protocol'] in ('http', 'https', 'ftp', 'ftps', 'm3u8', 'rtsp', 'rtmp', 'mms') and not info_dict.get('requested_formats')
+        return info_dict['protocol'] in ('http', 'https', 'ftp', 'ftps', 'm3u8', 'rtsp', 'rtmp', 'mms')
 
     @classmethod
     def available(cls):

From 77dea16ac8bb38d735ac6f002eaba7570a67d361 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 13 Mar 2016 20:30:23 +0100
Subject: [PATCH 4288/6219] [downloader/external] check for ffmpeg availablity
 when it used for m3u8 download

---
 youtube_dl/downloader/external.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index 85cf834c7..845a63e93 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -175,6 +175,9 @@ class FFmpegFD(ExternalFD):
     def _call_downloader(self, tmpfilename, info_dict):
         url = info_dict['url']
         ffpp = FFmpegPostProcessor(downloader=self)
+        if not ffpp.available:
+            self.report_error('m3u8 download detected but ffmpeg or avconv could not be found. Please install one.')
+            return False
         ffpp.check_version()
 
         args = [ffpp.executable, '-y']

From 694c47b2619b9d3698e0bc7bba45bb937f97bd3c Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 13 Mar 2016 21:11:19 +0100
Subject: [PATCH 4289/6219] [external/downloader] don't pass -t and -ss to
 ffmpeg

---
 youtube_dl/downloader/external.py | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index 845a63e93..2ddd76882 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -7,7 +7,6 @@ import re
 
 from .common import FileDownloader
 from ..postprocessor.ffmpeg import FFmpegPostProcessor, EXT_TO_OUT_FORMATS
-from ..compat import compat_str
 from ..utils import (
     cli_option,
     cli_valueless_option,
@@ -182,12 +181,12 @@ class FFmpegFD(ExternalFD):
 
         args = [ffpp.executable, '-y']
 
-        start_time = info_dict.get('start_time') or 0
-        if start_time:
-            args += ['-ss', compat_str(start_time)]
-        end_time = info_dict.get('end_time')
-        if end_time:
-            args += ['-t', compat_str(end_time - start_time)]
+        # start_time = info_dict.get('start_time') or 0
+        # if start_time:
+        #     args += ['-ss', compat_str(start_time)]
+        # end_time = info_dict.get('end_time')
+        # if end_time:
+        #     args += ['-t', compat_str(end_time - start_time)]
 
         if info_dict['http_headers'] and re.match(r'^https?://', url):
             # Trailing \r\n after each HTTP header is important to prevent warning from ffmpeg/avconv:

From d8515fd41c504baea129de26d60ca55dd69ae3f8 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 13 Mar 2016 21:13:50 +0100
Subject: [PATCH 4290/6219] [downloader/external] pass configuration args to
 ffmpeg

---
 youtube_dl/downloader/external.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index 2ddd76882..697f81e3f 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -181,6 +181,8 @@ class FFmpegFD(ExternalFD):
 
         args = [ffpp.executable, '-y']
 
+        args += self._configuration_args()
+
         # start_time = info_dict.get('start_time') or 0
         # if start_time:
         #     args += ['-ss', compat_str(start_time)]

From 5d53c327015f3370c3af454d18d51f2f4401fad4 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 13 Mar 2016 22:36:15 +0100
Subject: [PATCH 4291/6219] [usatoday] Add new extractor(closes #8655)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/usatoday.py | 48 ++++++++++++++++++++++++++++++++
 2 files changed, 49 insertions(+)
 create mode 100644 youtube_dl/extractor/usatoday.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index c5b80f4aa..7874e7699 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -821,6 +821,7 @@ from .udn import UDNEmbedIE
 from .digiteka import DigitekaIE
 from .unistra import UnistraIE
 from .urort import UrortIE
+from .usatoday import USATodayIE
 from .ustream import UstreamIE, UstreamChannelIE
 from .ustudio import UstudioIE
 from .varzesh3 import Varzesh3IE
diff --git a/youtube_dl/extractor/usatoday.py b/youtube_dl/extractor/usatoday.py
new file mode 100644
index 000000000..e5678dc78
--- /dev/null
+++ b/youtube_dl/extractor/usatoday.py
@@ -0,0 +1,48 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    get_element_by_attribute,
+    parse_duration,
+    update_url_query,
+    ExtractorError,
+)
+from ..compat import compat_str
+
+
+class USATodayIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?usatoday\.com/(?:[^/]+/)*(?P<id>[^?/#]+)'
+    _TEST = {
+        'url': 'http://www.usatoday.com/media/cinematic/video/81729424/us-france-warn-syrian-regime-ahead-of-new-peace-talks/',
+        'md5': '4d40974481fa3475f8bccfd20c5361f8',
+        'info_dict': {
+            'id': '81729424',
+            'ext': 'mp4',
+            'title': 'US, France warn Syrian regime ahead of new peace talks',
+            'timestamp': 1457891045,
+            'description': 'md5:7e50464fdf2126b0f533748d3c78d58f',
+            'uploader_id': '29906170001',
+            'upload_date': '20160313',
+        }
+    }
+    BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/29906170001/38a9eecc-bdd8-42a3-ba14-95397e48b3f8_default/index.html?videoId=%s'
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(update_url_query(url, {'ajax': 'true'}), display_id)
+        ui_video_data = get_element_by_attribute('class', 'ui-video-data', webpage)
+        if not ui_video_data:
+            raise ExtractorError('no video on the webpage', expected=True)
+        video_data = self._parse_json(ui_video_data, display_id)
+
+        return {
+            '_type': 'url_transparent',
+            'url': self.BRIGHTCOVE_URL_TEMPLATE % video_data['brightcove_id'],
+            'id': compat_str(video_data['id']),
+            'title': video_data['title'],
+            'thumbnail': video_data.get('thumbnail'),
+            'description': video_data.get('description'),
+            'duration': parse_duration(video_data.get('length')),
+            'ie_key': 'BrightcoveNew',
+        }

From 6bb266693f4b8cd92e27ea0efd46356dc254115b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 14 Mar 2016 10:25:20 +0100
Subject: [PATCH 4292/6219] release 2016.03.14

---
 README.md              | 2 +-
 docs/supportedsites.md | 9 ++++++++-
 youtube_dl/version.py  | 2 +-
 3 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index a774e73e7..68db546ef 100644
--- a/README.md
+++ b/README.md
@@ -181,7 +181,7 @@ which means you can modify it, redistribute it or use it however you like.
                                      to play it)
     --external-downloader COMMAND    Use the specified external downloader.
                                      Currently supports
-                                     aria2c,axel,curl,httpie,wget
+                                     aria2c,avconv,axel,curl,ffmpeg,httpie,wget
     --external-downloader-args ARGS  Give these arguments to the external
                                      downloader
 
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 0682e4af0..a6dcc2576 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -54,6 +54,7 @@
  - **AtresPlayer**
  - **ATTTechChannel**
  - **AudiMedia**
+ - **AudioBoom**
  - **audiomack**
  - **audiomack:album**
  - **Azubu**
@@ -167,6 +168,8 @@
  - **Dump**
  - **Dumpert**
  - **dvtv**: http://video.aktualne.cz/
+ - **dw**
+ - **dw:article**
  - **EaglePlatform**
  - **EbaumsWorld**
  - **EchoMsk**
@@ -190,7 +193,6 @@
  - **ExpoTV**
  - **ExtremeTube**
  - **facebook**
- - **facebook:post**
  - **faz.net**
  - **fc2**
  - **Fczenit**
@@ -325,6 +327,7 @@
  - **m6**
  - **macgamestore**: MacGameStore trailers
  - **mailru**: Видео@Mail.Ru
+ - **MakersChannel**
  - **MakerTV**
  - **Malemotion**
  - **MatchTV**
@@ -335,6 +338,7 @@
  - **Mgoon**
  - **Minhateca**
  - **MinistryGrid**
+ - **Minoto**
  - **miomio.tv**
  - **MiTele**: mitele.es
  - **mixcloud**
@@ -671,6 +675,7 @@
  - **UDNEmbed**: 聯合影音
  - **Unistra**
  - **Urort**: NRK P3 Urørt
+ - **USAToday**
  - **ustream**
  - **ustream:channel**
  - **Ustudio**
@@ -684,6 +689,7 @@
  - **VGTV**: VGTV, BTTV, FTV, Aftenposten and Aftonbladet
  - **vh1.com**
  - **Vice**
+ - **ViceShow**
  - **Viddler**
  - **video.google:search**: Google Video search
  - **video.mit.edu**
@@ -711,6 +717,7 @@
  - **vimeo:channel**
  - **vimeo:group**
  - **vimeo:likes**: Vimeo user likes
+ - **vimeo:ondemand**
  - **vimeo:review**: Review pages on vimeo
  - **vimeo:user**
  - **vimeo:watchlater**: Vimeo watch later list, "vimeowatchlater" keyword (requires authentication)
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 246f5740d..9216fa547 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.03.06'
+__version__ = '2016.03.14'

From 4230c4894d0439387bcd09dea4aba697b16975ce Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 14 Mar 2016 16:49:16 +0100
Subject: [PATCH 4293/6219] [external/downloader] fix rtmp downloading using
 FFmpegFD

---
 youtube_dl/downloader/external.py | 29 ++++++++++++++++++++++++++++-
 1 file changed, 28 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index 697f81e3f..30277dc20 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -198,12 +198,39 @@ class FFmpegFD(ExternalFD):
                 '-headers',
                 ''.join('%s: %s\r\n' % (key, val) for key, val in headers.items())]
 
+        protocol = info_dict.get('protocol')
+
+        if protocol == 'rtmp':
+            player_url = info_dict.get('player_url')
+            page_url = info_dict.get('page_url')
+            app = info_dict.get('app')
+            play_path = info_dict.get('play_path')
+            tc_url = info_dict.get('tc_url')
+            flash_version = info_dict.get('flash_version')
+            live = info_dict.get('rtmp_live', False)
+            if player_url is not None:
+                args += ['-rtmp_swfverify', player_url]
+            if page_url is not None:
+                args += ['-rtmp_pageurl', page_url]
+            if app is not None:
+                args += ['-rtmp_app', app]
+            if play_path is not None:
+                args += ['-rtmp_playpath', play_path]
+            if tc_url is not None:
+                args += ['-rtmp_tcurl', tc_url]
+            if flash_version is not None:
+                args += ['-rtmp_flashver', flash_version]
+            if live:
+                args += ['-rtmp_live', 'live']
+
         args += ['-i', url, '-c', 'copy']
-        if info_dict.get('protocol') == 'm3u8':
+        if protocol == 'm3u8':
             if self.params.get('hls_use_mpegts', False):
                 args += ['-f', 'mpegts']
             else:
                 args += ['-f', 'mp4', '-bsf:a', 'aac_adtstoasc']
+        elif protocol == 'rtmp':
+            args += ['-f', 'flv']
         else:
             args += ['-f', EXT_TO_OUT_FORMATS.get(info_dict['ext'], info_dict['ext'])]
 

From 203f3d779aad18927962d01c8d1e9927835bd84f Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 14 Mar 2016 18:32:29 +0100
Subject: [PATCH 4294/6219] [gameinformer] update the extractor to use
 BrightcoveNewIE

---
 youtube_dl/extractor/gameinformer.py | 29 ++++++++--------------------
 1 file changed, 8 insertions(+), 21 deletions(-)

diff --git a/youtube_dl/extractor/gameinformer.py b/youtube_dl/extractor/gameinformer.py
index 25870c131..1a99e35ed 100644
--- a/youtube_dl/extractor/gameinformer.py
+++ b/youtube_dl/extractor/gameinformer.py
@@ -10,34 +10,21 @@ class GameInformerIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?gameinformer\.com/(?:[^/]+/)*(?P<id>.+)\.aspx'
     _TEST = {
         'url': 'http://www.gameinformer.com/b/features/archive/2015/09/26/replay-animal-crossing.aspx',
+        'md5': '292f26da1ab4beb4c9099f1304d2b071',
         'info_dict': {
             'id': '4515472681001',
-            'ext': 'm3u8',
+            'ext': 'mp4',
             'title': 'Replay - Animal Crossing',
             'description': 'md5:2e211891b215c85d061adc7a4dd2d930',
-            'timestamp': 1443457610706,
-        },
-        'params': {
-            # m3u8 download
-            'skip_download': True,
+            'timestamp': 1443457610,
+            'upload_date': '20150928',
+            'uploader_id': '694940074001',
         },
     }
+    BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/694940074001/default_default/index.html?videoId=%s'
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
         webpage = self._download_webpage(url, display_id)
-
-        bc_api_url = self._search_regex(r"getVideo\('([^']+)'", webpage, 'brightcove api url')
-        json_data = self._download_json(
-            bc_api_url + '&video_fields=id,name,shortDescription,publishedDate,videoStillURL,length,IOSRenditions',
-            display_id)
-
-        return {
-            'id': compat_str(json_data['id']),
-            'display_id': display_id,
-            'url': json_data['IOSRenditions'][0]['url'],
-            'title': json_data['name'],
-            'description': json_data.get('shortDescription'),
-            'timestamp': int_or_none(json_data.get('publishedDate')),
-            'duration': int_or_none(json_data.get('length')),
-        }
+        brightcove_id = self._search_regex(r"getVideo\('[^']+video_id=(\d+)", webpage, 'brightcove id')
+        return self.url_result(self.BRIGHTCOVE_URL_TEMPLATE % brightcove_id, 'BrightcoveNew', brightcove_id)

From bf475e1990f4f4e52cf36ee10a39106c25999b26 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 14 Mar 2016 21:53:00 +0100
Subject: [PATCH 4295/6219] [tlc] fix extraction and update extractor to use
 BrightcoveNewIE

---
 youtube_dl/extractor/tlc.py | 35 +++++++++++++----------------------
 1 file changed, 13 insertions(+), 22 deletions(-)

diff --git a/youtube_dl/extractor/tlc.py b/youtube_dl/extractor/tlc.py
index adc05ed5f..17add9543 100644
--- a/youtube_dl/extractor/tlc.py
+++ b/youtube_dl/extractor/tlc.py
@@ -4,12 +4,12 @@ import re
 
 from .common import InfoExtractor
 from .brightcove import BrightcoveLegacyIE
-from ..compat import compat_urlparse
+from ..compat import compat_parse_qs
 
 
 class TlcDeIE(InfoExtractor):
     IE_NAME = 'tlc.de'
-    _VALID_URL = r'http://www\.tlc\.de/sendungen/[^/]+/videos/(?P<title>[^/?]+)'
+    _VALID_URL = r'http://www\.tlc\.de/(?:[^/]+/)*videos/(?P<title>[^/?#]+)?(?:.*#(?P<id>\d+))?'
 
     _TEST = {
         'url': 'http://www.tlc.de/sendungen/breaking-amish/videos/#3235167922001',
@@ -17,32 +17,23 @@ class TlcDeIE(InfoExtractor):
             'id': '3235167922001',
             'ext': 'mp4',
             'title': 'Breaking Amish: Die Welt da draußen',
-            'uploader': 'Discovery Networks - Germany',
             'description': (
                 'Vier Amische und eine Mennonitin wagen in New York'
                 '  den Sprung in ein komplett anderes Leben. Begleitet sie auf'
                 ' ihrem spannenden Weg.'),
+            'timestamp': 1396598084,
+            'upload_date': '20140404',
+            'uploader_id': '1659832546',
         },
     }
+    BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/1659832546/default_default/index.html?videoId=%s'
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        title = mobj.group('title')
-        webpage = self._download_webpage(url, title)
-        iframe_url = self._search_regex(
-            '<iframe src="(http://www\.tlc\.de/wp-content/.+?)"', webpage,
-            'iframe url')
-        # Otherwise we don't get the correct 'BrightcoveExperience' element,
-        # example: http://www.tlc.de/sendungen/cake-boss/videos/cake-boss-cannoli-drama/
-        iframe_url = iframe_url.replace('.htm?', '.php?')
-        url_fragment = compat_urlparse.urlparse(url).fragment
-        if url_fragment:
-            # Since the fragment is not send to the server, we always get the same iframe
-            iframe_url = re.sub(r'playlist=(\d+)', 'playlist=%s' % url_fragment, iframe_url)
-        iframe = self._download_webpage(iframe_url, title)
-
-        return {
-            '_type': 'url',
-            'url': BrightcoveLegacyIE._extract_brightcove_url(iframe),
-            'ie': BrightcoveLegacyIE.ie_key(),
-        }
+        brightcove_id = mobj.group('id')
+        if not brightcove_id:
+            title = mobj.group('title')
+            webpage = self._download_webpage(url, title)
+            brightcove_legacy_url = BrightcoveLegacyIE._extract_brightcove_url(webpage)
+            brightcove_id = compat_parse_qs(brightcove_legacy_url)['@videoPlayer'][0]
+        return self.url_result(self.BRIGHTCOVE_URL_TEMPLATE % brightcove_id, 'BrightcoveNew', brightcove_id)

From 744daf941843bba00e47c37e85df91637861b596 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 14 Mar 2016 21:57:26 +0100
Subject: [PATCH 4296/6219] [gameinformer] remove unused imports

---
 youtube_dl/extractor/gameinformer.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/youtube_dl/extractor/gameinformer.py b/youtube_dl/extractor/gameinformer.py
index 1a99e35ed..a66e309de 100644
--- a/youtube_dl/extractor/gameinformer.py
+++ b/youtube_dl/extractor/gameinformer.py
@@ -2,8 +2,6 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import compat_str
-from ..utils import int_or_none
 
 
 class GameInformerIE(InfoExtractor):

From d13bdc38241b52ff942c02ac2c15def4520b39a5 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 26 Nov 2015 18:38:57 +0100
Subject: [PATCH 4297/6219] [brightcove] raise ExtractorError on 403 errors and
 fix regex to work with tenplay

---
 youtube_dl/extractor/brightcove.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index c947337f9..5258b907c 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -13,6 +13,7 @@ from ..compat import (
     compat_urllib_parse_urlparse,
     compat_urlparse,
     compat_xml_parse_error,
+    compat_HTTPError,
 )
 from ..utils import (
     determine_ext,
@@ -424,7 +425,7 @@ class BrightcoveNewIE(InfoExtractor):
                     </video>.*?
                     <script[^>]+
                         src=["\'](?:https?:)?//players\.brightcove\.net/
-                        (\d+)/([\da-f-]+)_([^/]+)/index\.min\.js
+                        (\d+)/([\da-f-]+)_([^/]+)/index(?:\.min)?\.js
                 ''', webpage):
             entries.append(
                 'http://players.brightcove.net/%s/%s_%s/index.html?videoId=%s'
@@ -458,7 +459,13 @@ class BrightcoveNewIE(InfoExtractor):
             'https://edge.api.brightcove.com/playback/v1/accounts/%s/videos/%s'
             % (account_id, video_id),
             headers={'Accept': 'application/json;pk=%s' % policy_key})
-        json_data = self._download_json(req, video_id)
+        try:
+            json_data = self._download_json(req, video_id)
+        except ExtractorError as e:
+            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
+                json_data = self._parse_json(e.cause.read().decode(), video_id)
+                raise ExtractorError(json_data[0]['message'], expected=True)
+            raise
 
         title = json_data['name']
 

From 877f440f7be1a4f83c230d6ec06aeb5242e41084 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 15 Mar 2016 00:49:23 +0100
Subject: [PATCH 4298/6219] [rice] Add new extractor(closes #1736)

---
 youtube_dl/extractor/__init__.py |   1 +
 youtube_dl/extractor/rice.py     | 116 +++++++++++++++++++++++++++++++
 2 files changed, 117 insertions(+)
 create mode 100644 youtube_dl/extractor/rice.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 7874e7699..179c11ffa 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -598,6 +598,7 @@ from .regiotv import RegioTVIE
 from .restudy import RestudyIE
 from .reverbnation import ReverbNationIE
 from .revision3 import Revision3IE
+from .rice import RICEIE
 from .ringtv import RingTVIE
 from .ro220 import Ro220IE
 from .rottentomatoes import RottenTomatoesIE
diff --git a/youtube_dl/extractor/rice.py b/youtube_dl/extractor/rice.py
new file mode 100644
index 000000000..f855719ac
--- /dev/null
+++ b/youtube_dl/extractor/rice.py
@@ -0,0 +1,116 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..compat import compat_parse_qs
+from ..utils import (
+    xpath_text,
+    xpath_element,
+    int_or_none,
+    parse_iso8601,
+    ExtractorError,
+)
+
+
+class RICEIE(InfoExtractor):
+    _VALID_URL = r'https?://mediahub\.rice\.edu/app/[Pp]ortal/video\.aspx\?(?P<query>.+)'
+    _TEST = {
+        'url': 'https://mediahub.rice.edu/app/Portal/video.aspx?PortalID=25ffd62c-3d01-4b29-8c70-7c94270efb3e&DestinationID=66bc9434-03bd-4725-b47e-c659d8d809db&ContentID=YEWIvbhb40aqdjMD1ALSqw',
+        'md5': '9b83b4a2eead4912dc3b7fac7c449b6a',
+        'info_dict': {
+            'id': 'YEWIvbhb40aqdjMD1ALSqw',
+            'ext': 'mp4',
+            'title': 'Active Learning in Archeology',
+            'upload_date': '20140616',
+            'timestamp': 1402926346,
+        }
+    }
+    _NS = 'http://schemas.datacontract.org/2004/07/ensembleVideo.Data.Service.Contracts.Models.Player.Config'
+
+    def _real_extract(self, url):
+        qs = compat_parse_qs(re.match(self._VALID_URL, url).group('query'))
+        if not qs.get('PortalID') or not qs.get('DestinationID') or not qs.get('ContentID'):
+            raise ExtractorError('Invalid URL', expected=True)
+
+        portal_id = qs['PortalID'][0]
+        playlist_id = qs['DestinationID'][0]
+        content_id = qs['ContentID'][0]
+
+        content_data = self._download_xml('https://mediahub.rice.edu/api/portal/GetContentTitle', content_id, query={
+            'portalId': portal_id,
+            'playlistId': playlist_id,
+            'contentId': content_id
+        })
+        metadata = xpath_element(content_data, './/metaData', fatal=True)
+        title = xpath_text(metadata, 'primaryTitle', fatal=True)
+        encodings = xpath_element(content_data, './/encodings', fatal=True)
+        player_data = self._download_xml('https://mediahub.rice.edu/api/player/GetPlayerConfig', content_id, query={
+            'temporaryLinkId': xpath_text(encodings, 'temporaryLinkId', fatal=True),
+            'contentId': content_id,
+        })
+
+        common_fmt = {}
+        dimensions = xpath_text(encodings, 'dimensions')
+        if dimensions:
+            wh = dimensions.split('x')
+            if len(wh) == 2:
+                common_fmt.update({
+                    'width': int_or_none(wh[0]),
+                    'height': int_or_none(wh[1]),
+                })
+
+        formats = []
+        rtsp_path = xpath_text(player_data, self._xpath_ns('RtspPath', self._NS))
+        if rtsp_path:
+            fmt = {
+                'url': rtsp_path,
+                'format_id': 'rtsp',
+            }
+            fmt.update(common_fmt)
+            formats.append(fmt)
+        for source in player_data.findall(self._xpath_ns('.//Source', self._NS)):
+            video_url = xpath_text(source, self._xpath_ns('File', self._NS))
+            if not video_url:
+                continue
+            if '.m3u8' in video_url:
+                formats.extend(self._extract_m3u8_formats(video_url, content_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
+            else:
+                fmt = {
+                    'url': video_url,
+                    'format_id': video_url.split(':')[0],
+                }
+                fmt.update(common_fmt)
+                rtmp = re.search(r'^(?P<url>rtmp://[^/]+/(?P<app>.+))/(?P<playpath>mp4:.+)$', video_url)
+                if rtmp:
+                    fmt.update({
+                        'url': rtmp.group('url'),
+                        'play_path': rtmp.group('playpath'),
+                        'app': rtmp.group('app'),
+                        'ext': 'flv',
+                    })
+                formats.append(fmt)
+        self._sort_formats(formats)
+
+        thumbnails = []
+        for content_asset in content_data.findall('.//contentAssets'):
+            asset_type = xpath_text(content_asset, 'type')
+            if asset_type == 'image':
+                image_url = xpath_text(content_asset, 'httpPath')
+                if not image_url:
+                    continue
+                thumbnails.append({
+                    'id': xpath_text(content_asset, 'ID'),
+                    'url': image_url,
+                })
+
+        return {
+            'id': content_id,
+            'title': title,
+            'description': xpath_text(metadata, 'abstract'),
+            'duration': int_or_none(xpath_text(metadata, 'duration')),
+            'timestamp': parse_iso8601(xpath_text(metadata, 'dateUpdated')),
+            'thumbnails': thumbnails,
+            'formats': formats,
+        }

From 8ac73bdbe44e14007fb3e630eb70e2a395133480 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 15 Mar 2016 10:03:08 +0100
Subject: [PATCH 4299/6219] [brightcove:new] Add support for non numeric ref:
 preffixed video ids

---
 youtube_dl/extractor/brightcove.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 5258b907c..9760e606f 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -356,7 +356,7 @@ class BrightcoveLegacyIE(InfoExtractor):
 
 class BrightcoveNewIE(InfoExtractor):
     IE_NAME = 'brightcove:new'
-    _VALID_URL = r'https?://players\.brightcove\.net/(?P<account_id>\d+)/(?P<player_id>[^/]+)_(?P<embed>[^/]+)/index\.html\?.*videoId=(?P<video_id>(?:ref:)?\d+)'
+    _VALID_URL = r'https?://players\.brightcove\.net/(?P<account_id>\d+)/(?P<player_id>[^/]+)_(?P<embed>[^/]+)/index\.html\?.*videoId=(?P<video_id>\d+|ref:[^&]+)'
     _TESTS = [{
         'url': 'http://players.brightcove.net/929656772001/e41d32dc-ec74-459e-a845-6c69f7b724ea_default/index.html?videoId=4463358922001',
         'md5': 'c8100925723840d4b0d243f7025703be',
@@ -392,6 +392,10 @@ class BrightcoveNewIE(InfoExtractor):
         # ref: prefixed video id
         'url': 'http://players.brightcove.net/3910869709001/21519b5c-4b3b-4363-accb-bdc8f358f823_default/index.html?videoId=ref:7069442',
         'only_matching': True,
+    }, {
+        # non numeric ref: prefixed video id
+        'url': 'http://players.brightcove.net/710858724001/default_default/index.html?videoId=ref:event-stream-356',
+        'only_matching': True,
     }]
 
     @staticmethod

From 975b9c9ab09306054d159d57085ed6ea1582b4de Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 15 Mar 2016 10:06:53 +0100
Subject: [PATCH 4300/6219] [brightcove:new] detect m3u8 manifests by M2TS
 container

---
 youtube_dl/extractor/brightcove.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 9760e606f..f8413d5f2 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -475,9 +475,10 @@ class BrightcoveNewIE(InfoExtractor):
 
         formats = []
         for source in json_data.get('sources', []):
+            container = source.get('container')
             source_type = source.get('type')
             src = source.get('src')
-            if source_type == 'application/x-mpegURL':
+            if source_type == 'application/x-mpegURL' or container == 'M2TS':
                 if not src:
                     continue
                 formats.extend(self._extract_m3u8_formats(
@@ -495,7 +496,7 @@ class BrightcoveNewIE(InfoExtractor):
                     'width': int_or_none(source.get('width')),
                     'height': height,
                     'filesize': int_or_none(source.get('size')),
-                    'container': source.get('container'),
+                    'container': container,
                     'vcodec': source.get('codec'),
                     'ext': source.get('container').lower(),
                 }

From cf6cd07396479940bbf9ca71778a72807208fabb Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 15 Mar 2016 15:24:12 +0100
Subject: [PATCH 4301/6219] [noz] extract f4m and m3u8 formats

---
 youtube_dl/extractor/noz.py | 35 ++++++++++++++++++++++++-----------
 1 file changed, 24 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/noz.py b/youtube_dl/extractor/noz.py
index 0ffb44b47..4e60b13a5 100644
--- a/youtube_dl/extractor/noz.py
+++ b/youtube_dl/extractor/noz.py
@@ -6,6 +6,7 @@ from ..compat import compat_urllib_parse_unquote
 from ..utils import (
     int_or_none,
     xpath_text,
+    update_url_query,
 )
 
 
@@ -46,17 +47,29 @@ class NozIE(InfoExtractor):
             doc, './/article/movie/file/duration'))
         formats = []
         for qnode in doc.findall('.//article/movie/file/qualities/qual'):
-            video_node = qnode.find('./html_urls/video_url[@format="video/mp4"]')
-            if video_node is None:
-                continue  # auto
-            formats.append({
-                'url': video_node.text,
-                'format_name': xpath_text(qnode, './name'),
-                'format_id': xpath_text(qnode, './id'),
-                'height': int_or_none(xpath_text(qnode, './height')),
-                'width': int_or_none(xpath_text(qnode, './width')),
-                'tbr': int_or_none(xpath_text(qnode, './bitrate'), scale=1000),
-            })
+            http_url = xpath_text(
+                qnode, './html_urls/video_url[@format="video/mp4"]')
+            if http_url:
+                formats.append({
+                    'url': http_url,
+                    'format_name': xpath_text(qnode, './name'),
+                    'format_id': '%s-%s' % ('http', xpath_text(qnode, './id')),
+                    'height': int_or_none(xpath_text(qnode, './height')),
+                    'width': int_or_none(xpath_text(qnode, './width')),
+                    'tbr': int_or_none(xpath_text(qnode, './bitrate'), scale=1000),
+                })
+            else:
+                f4m_url = xpath_text(qnode, 'url_hd2')
+                if f4m_url:
+                    formats.extend(self._extract_f4m_formats(
+                        update_url_query(f4m_url, {'hdcore': '3.4.0'}),
+                        video_id, f4m_id='hds', fatal=False))
+                m3u8_url = xpath_text(
+                    qnode, './html_urls/video_url[@format="application/vnd.apple.mpegurl"]')
+                if m3u8_url:
+                    formats.extend(self._extract_m3u8_formats(
+                        m3u8_url, video_id, 'mp4', 'm3u8_native',
+                        m3u8_id='hls', fatal=False))
         self._sort_formats(formats)
 
         return {

From dec2cae0a768d2eb1f7d28cfd267d0bf5383bcdd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 15 Mar 2016 21:45:43 +0600
Subject: [PATCH 4302/6219] [twitch:playlistbase] Clarify pagination bug

Pagination bug has been fixed by twitch on 15.03.2016.
---
 youtube_dl/extractor/twitch.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 958bf8fff..d4169ec6d 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -299,9 +299,10 @@ class TwitchPlaylistBaseIE(TwitchBaseIE):
             # is completely broken on the twitch side. It simply ignores
             # a limit and returns the whole offset number of videos.
             # Working around by just requesting all videos at once.
+            # Upd: pagination bug was fixed by twitch on 15.03.2016.
             if not broken_paging_detected and total and len(page_entries) > limit:
                 self.report_warning(
-                    'Twitch paging is broken on twitch side, requesting all videos at once',
+                    'Twitch pagination is broken on twitch side, requesting all videos at once',
                     channel_id)
                 broken_paging_detected = True
                 offset = total

From 89807d6a8293d7b028a204628aec32b6f39148dd Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 15 Mar 2016 18:48:21 +0100
Subject: [PATCH 4303/6219] [brightcove] extract dash formats and detect audio
 formats

---
 youtube_dl/extractor/brightcove.py | 20 ++++++++++++++++----
 1 file changed, 16 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index f8413d5f2..86de5b9ee 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -484,6 +484,10 @@ class BrightcoveNewIE(InfoExtractor):
                 formats.extend(self._extract_m3u8_formats(
                     src, video_id, 'mp4', entry_protocol='m3u8_native',
                     m3u8_id='hls', fatal=False))
+            elif source_type == 'application/dash+xml':
+                if not src:
+                    continue
+                formats.extend(self._extract_mpd_formats(src, video_id, 'dash', fatal=False))
             else:
                 streaming_src = source.get('streaming_src')
                 stream_name, app_name = source.get('stream_name'), source.get('app_name')
@@ -491,15 +495,23 @@ class BrightcoveNewIE(InfoExtractor):
                     continue
                 tbr = float_or_none(source.get('avg_bitrate'), 1000)
                 height = int_or_none(source.get('height'))
+                width = int_or_none(source.get('width'))
                 f = {
                     'tbr': tbr,
-                    'width': int_or_none(source.get('width')),
-                    'height': height,
                     'filesize': int_or_none(source.get('size')),
                     'container': container,
-                    'vcodec': source.get('codec'),
-                    'ext': source.get('container').lower(),
+                    'ext': container.lower(),
                 }
+                if width == 0 and height == 0:
+                    f.update({
+                        'vcodec': 'none',
+                    })
+                else:
+                    f.update({
+                        'width': width,
+                        'height': height,
+                        'vcodec': source.get('codec'),
+                    })
 
                 def build_format_id(kind):
                     format_id = kind

From edfd93518ea0a929dbdd51cc3472bddceaf96514 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 15 Mar 2016 19:33:09 +0100
Subject: [PATCH 4304/6219] [svt] extract dashhbbtv formats(#8867)

---
 youtube_dl/extractor/svt.py | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/svt.py b/youtube_dl/extractor/svt.py
index 399c3b8ee..2ab30e45f 100644
--- a/youtube_dl/extractor/svt.py
+++ b/youtube_dl/extractor/svt.py
@@ -19,20 +19,25 @@ class SVTBaseIE(InfoExtractor):
         video_info = info['video']
         formats = []
         for vr in video_info['videoReferences']:
+            player_type = vr.get('playerType')
             vurl = vr['url']
             ext = determine_ext(vurl)
             if ext == 'm3u8':
                 formats.extend(self._extract_m3u8_formats(
                     vurl, video_id,
                     ext='mp4', entry_protocol='m3u8_native',
-                    m3u8_id=vr.get('playerType')))
+                    m3u8_id=player_type, fatal=False))
             elif ext == 'f4m':
                 formats.extend(self._extract_f4m_formats(
                     vurl + '?hdcore=3.3.0', video_id,
-                    f4m_id=vr.get('playerType')))
+                    f4m_id=player_type, fatal=False))
+            elif ext == 'mpd':
+                if player_type == 'dashhbbtv':
+                    formats.extend(self._extract_mpd_formats(
+                        vurl, video_id, mpd_id=player_type, fatal=False))
             else:
                 formats.append({
-                    'format_id': vr.get('playerType'),
+                    'format_id': player_type,
                     'url': vurl,
                 })
         self._sort_formats(formats)

From cb6e477dfe09b68a810e587269958b1e56077b00 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 15 Mar 2016 19:38:10 +0100
Subject: [PATCH 4305/6219] [aljazeera] update the extractor to use
 BrightcoveNewIE

---
 youtube_dl/extractor/aljazeera.py | 18 ++++++------------
 1 file changed, 6 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/aljazeera.py b/youtube_dl/extractor/aljazeera.py
index 5b2c0dc9a..cddcaa489 100644
--- a/youtube_dl/extractor/aljazeera.py
+++ b/youtube_dl/extractor/aljazeera.py
@@ -13,24 +13,18 @@ class AlJazeeraIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'The Slum - Episode 1: Deliverance',
             'description': 'As a birth attendant advocating for family planning, Remy is on the frontline of Tondo\'s battle with overcrowding.',
-            'uploader': 'Al Jazeera English',
+            'uploader_id': '665003303001',
+            'timestamp': 1411116829,
+            'upload_date': '20140919',
         },
-        'add_ie': ['BrightcoveLegacy'],
+        'add_ie': ['BrightcoveNew'],
         'skip': 'Not accessible from Travis CI server',
     }
+    BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/665003303001/default_default/index.html?videoId=%s'
 
     def _real_extract(self, url):
         program_name = self._match_id(url)
         webpage = self._download_webpage(url, program_name)
         brightcove_id = self._search_regex(
             r'RenderPagesVideo\(\'(.+?)\'', webpage, 'brightcove id')
-
-        return {
-            '_type': 'url',
-            'url': (
-                'brightcove:'
-                'playerKey=AQ~~%2CAAAAmtVJIFk~%2CTVGOQ5ZTwJbeMWnq5d_H4MOM57xfzApc'
-                '&%40videoPlayer={0}'.format(brightcove_id)
-            ),
-            'ie_key': 'BrightcoveLegacy',
-        }
+        return self.url_result(self.BRIGHTCOVE_URL_TEMPLATE % brightcove_id, 'BrightcoveNew', brightcove_id)

From 3ff8279e80b4c057f12998dcca5164209bb71078 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 15 Mar 2016 02:20:37 +0800
Subject: [PATCH 4306/6219] [kuwo:mv] Fix the test and extraction of
 georestricted MVs

---
 youtube_dl/extractor/kuwo.py | 33 +++++++++++++++++++++++++++------
 1 file changed, 27 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/kuwo.py b/youtube_dl/extractor/kuwo.py
index 700e44b63..f94804d06 100644
--- a/youtube_dl/extractor/kuwo.py
+++ b/youtube_dl/extractor/kuwo.py
@@ -23,7 +23,7 @@ class KuwoBaseIE(InfoExtractor):
         {'format': 'aac', 'ext': 'aac', 'abr': 48, 'preference': 10}
     ]
 
-    def _get_formats(self, song_id):
+    def _get_formats(self, song_id, tolerate_ip_deny=False):
         formats = []
         for file_format in self._FORMATS:
             song_url = self._download_webpage(
@@ -32,7 +32,7 @@ class KuwoBaseIE(InfoExtractor):
                 song_id, note='Download %s url info' % file_format['format'],
             )
 
-            if song_url == 'IPDeny':
+            if song_url == 'IPDeny' and not tolerate_ip_deny:
                 raise ExtractorError('This song is blocked in this region', expected=True)
 
             if song_url.startswith('http://') or song_url.startswith('https://'):
@@ -43,7 +43,12 @@ class KuwoBaseIE(InfoExtractor):
                     'preference': file_format['preference'],
                     'abr': file_format.get('abr'),
                 })
-        self._sort_formats(formats)
+
+        # XXX _sort_formats fails if there are not formats, while it's not the
+        # desired behavior if 'IPDeny' is ignored
+        # This check can be removed if https://github.com/rg3/youtube-dl/pull/8051 is merged
+        if not tolerate_ip_deny:
+            self._sort_formats(formats)
         return formats
 
 
@@ -288,10 +293,16 @@ class KuwoMvIE(KuwoBaseIE):
         'url': 'http://www.kuwo.cn/mv/6480076/',
         'info_dict': {
             'id': '6480076',
-            'ext': 'mkv',
-            'title': '我们家MV',
+            'ext': 'mp4',
+            'title': 'My HouseMV',
             'creator': '2PM',
         },
+        # In this video, music URLs (anti.s) are blocked outside China and
+        # USA, while the MV URL (mvurl) is available globally, so force the MV
+        # URL for consistent results in different countries
+        'params': {
+            'format': 'mv',
+        },
     }
     _FORMATS = KuwoBaseIE._FORMATS + [
         {'format': 'mkv', 'ext': 'mkv', 'preference': 250},
@@ -313,7 +324,17 @@ class KuwoMvIE(KuwoBaseIE):
         else:
             raise ExtractorError('Unable to find song or singer names')
 
-        formats = self._get_formats(song_id)
+        formats = self._get_formats(song_id, tolerate_ip_deny=True)
+
+        mv_url = self._download_webpage(
+            'http://www.kuwo.cn/yy/st/mvurl?rid=MUSIC_%s' % song_id,
+            song_id, note='Download %s MV URL' % song_id)
+        formats.append({
+            'url': mv_url,
+            'format_id': 'mv',
+        })
+
+        self._sort_formats(formats)
 
         return {
             'id': song_id,

From 0c9ff2404119bc93fe9755fa5ecdf8e8dd53b146 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 15 Mar 2016 20:54:55 +0100
Subject: [PATCH 4307/6219] [noz] fix extraction in python 2.6

---
 youtube_dl/extractor/noz.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/noz.py b/youtube_dl/extractor/noz.py
index 4e60b13a5..656443c49 100644
--- a/youtube_dl/extractor/noz.py
+++ b/youtube_dl/extractor/noz.py
@@ -5,6 +5,7 @@ from .common import InfoExtractor
 from ..compat import compat_urllib_parse_unquote
 from ..utils import (
     int_or_none,
+    find_xpath_attr,
     xpath_text,
     update_url_query,
 )
@@ -47,8 +48,9 @@ class NozIE(InfoExtractor):
             doc, './/article/movie/file/duration'))
         formats = []
         for qnode in doc.findall('.//article/movie/file/qualities/qual'):
-            http_url = xpath_text(
-                qnode, './html_urls/video_url[@format="video/mp4"]')
+            http_url_ele = find_xpath_attr(
+                qnode, './html_urls/video_url', 'format', 'video/mp4')
+            http_url = http_url_ele.text if http_url_ele is not None else None
             if http_url:
                 formats.append({
                     'url': http_url,
@@ -64,8 +66,10 @@ class NozIE(InfoExtractor):
                     formats.extend(self._extract_f4m_formats(
                         update_url_query(f4m_url, {'hdcore': '3.4.0'}),
                         video_id, f4m_id='hds', fatal=False))
-                m3u8_url = xpath_text(
-                    qnode, './html_urls/video_url[@format="application/vnd.apple.mpegurl"]')
+                m3u8_url_ele = find_xpath_attr(
+                    qnode, './html_urls/video_url',
+                    'format', 'application/vnd.apple.mpegurl')
+                m3u8_url = m3u8_url_ele.text if m3u8_url_ele is not None else None
                 if m3u8_url:
                     formats.extend(self._extract_m3u8_formats(
                         m3u8_url, video_id, 'mp4', 'm3u8_native',

From 64d4f31d78d9555d7f79b8bebeabb7535a842090 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 15 Mar 2016 22:50:43 +0100
Subject: [PATCH 4308/6219] [brightcove:new] update embed_in_page embeds regex
 to match non numeric ref id

---
 youtube_dl/extractor/brightcove.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 86de5b9ee..b182edb26 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -425,7 +425,7 @@ class BrightcoveNewIE(InfoExtractor):
                 # According to [4] data-video-id may be prefixed with ref:
                 r'''(?sx)
                     <video[^>]+
-                        data-video-id=["\']((?:ref:)?\d+)["\'][^>]*>.*?
+                        data-video-id=["\'](\d+|ref:[^"\']+)["\'][^>]*>.*?
                     </video>.*?
                     <script[^>]+
                         src=["\'](?:https?:)?//players\.brightcove\.net/

From 2cab48704ced0d703f2b6de8575dc06e83207462 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 15 Mar 2016 22:57:59 +0100
Subject: [PATCH 4309/6219] [thestar] Add new extractor(closes #5955)

---
 youtube_dl/extractor/__init__.py   |  1 +
 youtube_dl/extractor/brightcove.py |  2 +-
 youtube_dl/extractor/thestar.py    | 31 ++++++++++++++++++++++++++++++
 3 files changed, 33 insertions(+), 1 deletion(-)
 create mode 100644 youtube_dl/extractor/thestar.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 179c11ffa..bf9fa17c9 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -737,6 +737,7 @@ from .theplatform import (
     ThePlatformFeedIE,
 )
 from .thesixtyone import TheSixtyOneIE
+from .thestar import TheStarIE
 from .thisamericanlife import ThisAmericanLifeIE
 from .thisav import ThisAVIE
 from .tinypic import TinyPicIE
diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index b182edb26..f56b642ab 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -157,7 +157,7 @@ class BrightcoveLegacyIE(InfoExtractor):
         if playerKey is not None:
             params['playerKey'] = playerKey
         # The three fields hold the id of the video
-        videoPlayer = find_param('@videoPlayer') or find_param('videoId') or find_param('videoID')
+        videoPlayer = find_param('@videoPlayer') or find_param('videoId') or find_param('videoID') or find_param('@videoList')
         if videoPlayer is not None:
             params['@videoPlayer'] = videoPlayer
         linkBase = find_param('linkBaseURL')
diff --git a/youtube_dl/extractor/thestar.py b/youtube_dl/extractor/thestar.py
new file mode 100644
index 000000000..b7e9af2af
--- /dev/null
+++ b/youtube_dl/extractor/thestar.py
@@ -0,0 +1,31 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from .brightcove import BrightcoveLegacyIE
+from ..compat import compat_parse_qs
+
+
+class TheStarIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?thestar\.com/(?:[^/]+/)*(?P<id>.+)\.html'
+    _TEST = {
+        'url': 'http://www.thestar.com/life/2016/02/01/mankind-why-this-woman-started-a-men-s-skincare-line.html',
+        'md5': '2c62dd4db2027e35579fefb97a8b6554',
+        'info_dict': {
+            'id': '4732393888001',
+            'ext': 'mp4',
+            'title': 'Mankind: Why this woman started a men\'s skin care line',
+            'description': 'Robert Cribb talks to Young Lee, the founder of Uncle Peter\'s MAN.',
+            'uploader_id': '794267642001',
+            'timestamp': 1454353482,
+            'upload_date': '20160201',
+        }
+    }
+    BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/794267642001/default_default/index.html?videoId=%s'
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+        brightcove_legacy_url = BrightcoveLegacyIE._extract_brightcove_url(webpage)
+        brightcove_id = compat_parse_qs(brightcove_legacy_url)['@videoPlayer'][0]
+        return self.url_result(self.BRIGHTCOVE_URL_TEMPLATE % brightcove_id, 'BrightcoveNew', brightcove_id)

From 48254c3f2cb315c4b9d2b679a6126f1e1208fbf8 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 16 Mar 2016 09:14:37 +0100
Subject: [PATCH 4310/6219] [brightcove] some improvements and fixes

- use FFmpeg downloader to download m3u8 formats extracted
from BrightcoveNew(some of the m3u8 media playlists use AES-128)
- update comment and update_url_query to handle url query
---
 youtube_dl/extractor/brightcove.py | 32 +++++++++++++-----------------
 1 file changed, 14 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index f56b642ab..304fb89e3 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -24,16 +24,16 @@ from ..utils import (
     js_to_json,
     int_or_none,
     parse_iso8601,
-    sanitized_Request,
     unescapeHTML,
     unsmuggle_url,
+    update_url_query,
 )
 
 
 class BrightcoveLegacyIE(InfoExtractor):
     IE_NAME = 'brightcove:legacy'
     _VALID_URL = r'(?:https?://.*brightcove\.com/(services|viewer).*?\?|brightcove:)(?P<query>.*)'
-    _FEDERATED_URL_TEMPLATE = 'http://c.brightcove.com/services/viewer/htmlFederated?%s'
+    _FEDERATED_URL = 'http://c.brightcove.com/services/viewer/htmlFederated'
 
     _TESTS = [
         {
@@ -156,7 +156,7 @@ class BrightcoveLegacyIE(InfoExtractor):
         # Not all pages define this value
         if playerKey is not None:
             params['playerKey'] = playerKey
-        # The three fields hold the id of the video
+        # These fields hold the id of the video
         videoPlayer = find_param('@videoPlayer') or find_param('videoId') or find_param('videoID') or find_param('@videoList')
         if videoPlayer is not None:
             params['@videoPlayer'] = videoPlayer
@@ -185,8 +185,7 @@ class BrightcoveLegacyIE(InfoExtractor):
 
     @classmethod
     def _make_brightcove_url(cls, params):
-        data = compat_urllib_parse.urlencode(params)
-        return cls._FEDERATED_URL_TEMPLATE % data
+        return update_url_query(cls._FEDERATED_URL, params)
 
     @classmethod
     def _extract_brightcove_url(cls, webpage):
@@ -240,7 +239,7 @@ class BrightcoveLegacyIE(InfoExtractor):
             # We set the original url as the default 'Referer' header
             referer = smuggled_data.get('Referer', url)
             return self._get_video_info(
-                videoPlayer[0], query_str, query, referer=referer)
+                videoPlayer[0], query, referer=referer)
         elif 'playerKey' in query:
             player_key = query['playerKey']
             return self._get_playlist_info(player_key[0])
@@ -249,15 +248,14 @@ class BrightcoveLegacyIE(InfoExtractor):
                 'Cannot find playerKey= variable. Did you forget quotes in a shell invocation?',
                 expected=True)
 
-    def _get_video_info(self, video_id, query_str, query, referer=None):
-        request_url = self._FEDERATED_URL_TEMPLATE % query_str
-        req = sanitized_Request(request_url)
+    def _get_video_info(self, video_id, query, referer=None):
+        headers = {}
         linkBase = query.get('linkBaseURL')
         if linkBase is not None:
             referer = linkBase[0]
         if referer is not None:
-            req.add_header('Referer', referer)
-        webpage = self._download_webpage(req, video_id)
+            headers['Referer'] = referer
+        webpage = self._download_webpage(self._FEDERATED_URL, video_id, headers=headers, query=query)
 
         error_msg = self._html_search_regex(
             r"<h1>We're sorry.</h1>([\s\n]*<p>.*?</p>)+", webpage,
@@ -459,12 +457,11 @@ class BrightcoveNewIE(InfoExtractor):
                 r'policyKey\s*:\s*(["\'])(?P<pk>.+?)\1',
                 webpage, 'policy key', group='pk')
 
-        req = sanitized_Request(
-            'https://edge.api.brightcove.com/playback/v1/accounts/%s/videos/%s'
-            % (account_id, video_id),
-            headers={'Accept': 'application/json;pk=%s' % policy_key})
+        api_url = 'https://edge.api.brightcove.com/playback/v1/accounts/%s/videos/%s' % (account_id, video_id)
         try:
-            json_data = self._download_json(req, video_id)
+            json_data = self._download_json(api_url, video_id, headers={
+                'Accept': 'application/json;pk=%s' % policy_key
+            })
         except ExtractorError as e:
             if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
                 json_data = self._parse_json(e.cause.read().decode(), video_id)
@@ -482,8 +479,7 @@ class BrightcoveNewIE(InfoExtractor):
                 if not src:
                     continue
                 formats.extend(self._extract_m3u8_formats(
-                    src, video_id, 'mp4', entry_protocol='m3u8_native',
-                    m3u8_id='hls', fatal=False))
+                    src, video_id, 'mp4', m3u8_id='hls', fatal=False))
             elif source_type == 'application/dash+xml':
                 if not src:
                     continue

From 23edc49509052e06afe7032802a0f4deb6710b47 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 16 Mar 2016 10:47:39 +0100
Subject: [PATCH 4311/6219] [tv3] Add new extractor(closes #8059)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/tv3.py      | 33 ++++++++++++++++++++++++++++++++
 2 files changed, 34 insertions(+)
 create mode 100644 youtube_dl/extractor/tv3.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index bf9fa17c9..9502d07a4 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -784,6 +784,7 @@ from .tv2 import (
     TV2IE,
     TV2ArticleIE,
 )
+from .tv3 import TV3IE
 from .tv4 import TV4IE
 from .tvc import (
     TVCIE,
diff --git a/youtube_dl/extractor/tv3.py b/youtube_dl/extractor/tv3.py
new file mode 100644
index 000000000..d3f690dc7
--- /dev/null
+++ b/youtube_dl/extractor/tv3.py
@@ -0,0 +1,33 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class TV3IE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?tv3\.co\.nz/(?P<id>[^/]+)/tabid/\d+/articleID/\d+/MCat/\d+/Default\.aspx'
+    _TEST = {
+        'url': 'http://www.tv3.co.nz/MOTORSPORT-SRS-SsangYong-Hampton-Downs-Round-3/tabid/3692/articleID/121615/MCat/2915/Default.aspx',
+        'info_dict': {
+            'id': '4659127992001',
+            'ext': 'mp4',
+            'title': 'CRC Motorsport: SRS SsangYong Hampton Downs Round 3 - S2015 Ep3',
+            'description': 'SsangYong Racing Series returns for Round 3 with drivers from New Zealand and Australia taking to the grid at Hampton Downs raceway.',
+            'uploader_id': '3812193411001',
+            'upload_date': '20151213',
+            'timestamp': 1449975272,
+        },
+        'expected_warnings': [
+            'Failed to download MPD manifest'
+        ],
+        'params': {
+            'skip_download': True,
+        },
+    }
+    BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/3812193411001/default_default/index.html?videoId=%s'
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+        brightcove_id = self._search_regex(r'<param\s*name="@videoPlayer"\s*value="(\d+)"', webpage, 'brightcove id')
+        return self.url_result(self.BRIGHTCOVE_URL_TEMPLATE % brightcove_id, 'BrightcoveNew', brightcove_id)

From 354dbbd8808dc5e835c7042f84c175eb56e0bcfc Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 16 Mar 2016 11:46:53 +0100
Subject: [PATCH 4312/6219] [brightcove:new] extract protocol-less embed
 URLs(closes #2914)

---
 youtube_dl/extractor/brightcove.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 304fb89e3..3ab383461 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -413,8 +413,8 @@ class BrightcoveNewIE(InfoExtractor):
 
         # Look for iframe embeds [1]
         for _, url in re.findall(
-                r'<iframe[^>]+src=(["\'])((?:https?:)//players\.brightcove\.net/\d+/[^/]+/index\.html.+?)\1', webpage):
-            entries.append(url)
+                r'<iframe[^>]+src=(["\'])((?:https?:)?//players\.brightcove\.net/\d+/[^/]+/index\.html.+?)\1', webpage):
+            entries.append(url if url.startswith('http') else 'http:' + url)
 
         # Look for embed_in_page embeds [2]
         for video_id, account_id, player_id, embed in re.findall(

From a7ba57dc176efaa50b5121a1f63963f4fc0111e7 Mon Sep 17 00:00:00 2001
From: Quan Hua <quanhua92@gmail.com>
Date: Wed, 16 Mar 2016 10:15:39 +0700
Subject: [PATCH 4313/6219] [udemy] Update course id regex to cover v4 layout
 (Closes #8753, closes #8868, closes #8870)

---
 youtube_dl/extractor/udemy.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index f5b5e7fd6..0fd2a0a0a 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -144,7 +144,8 @@ class UdemyIE(InfoExtractor):
         webpage = self._download_webpage(url, lecture_id)
 
         course_id = self._search_regex(
-            r'data-course-id=["\'](\d+)', webpage, 'course id')
+            (r'data-course-id=["\'](\d+)', r'&quot;id&quot;: (\d+)'),
+            webpage, 'course id')
 
         try:
             lecture = self._download_lecture(course_id, lecture_id)

From 70cab344c48598904fde657620156be62b70ee0d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 16 Mar 2016 21:46:09 +0600
Subject: [PATCH 4314/6219] [udemy] Improve course id v4 regex

---
 youtube_dl/extractor/udemy.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index 0fd2a0a0a..74cc36ece 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -144,7 +144,7 @@ class UdemyIE(InfoExtractor):
         webpage = self._download_webpage(url, lecture_id)
 
         course_id = self._search_regex(
-            (r'data-course-id=["\'](\d+)', r'&quot;id&quot;: (\d+)'),
+            (r'data-course-id=["\'](\d+)', r'&quot;id&quot;\s*:\s*(\d+)'),
             webpage, 'course id')
 
         try:

From 96f4f796fb02e3ef13fa6584b8f77ebafaabf59f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 16 Mar 2016 21:47:51 +0600
Subject: [PATCH 4315/6219] [brightcover] Remove unused import

---
 youtube_dl/extractor/brightcove.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 3ab383461..59e8008f9 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -9,7 +9,6 @@ from ..compat import (
     compat_etree_fromstring,
     compat_parse_qs,
     compat_str,
-    compat_urllib_parse,
     compat_urllib_parse_urlparse,
     compat_urlparse,
     compat_xml_parse_error,

From c5229f3926d64bce101d328fc5acf25bda83e0d8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 16 Mar 2016 21:50:04 +0600
Subject: [PATCH 4316/6219] [utils] PEP 8

---
 test/test_utils.py  | 6 +++---
 youtube_dl/utils.py | 5 ++++-
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 5a0109977..9a3a8ddff 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -641,8 +641,8 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(extract_attributes('<e x="&#x79;">'), {'x': 'y'})
         self.assertEqual(extract_attributes('<e x="&amp;">'), {'x': '&'})  # XML
         self.assertEqual(extract_attributes('<e x="&quot;">'), {'x': '"'})
-        self.assertEqual(extract_attributes('<e x="&pound;">'), {'x': '£'}) # HTML 3.2
-        self.assertEqual(extract_attributes('<e x="&lambda;">'), {'x': 'λ'}) # HTML 4.0
+        self.assertEqual(extract_attributes('<e x="&pound;">'), {'x': '£'})  # HTML 3.2
+        self.assertEqual(extract_attributes('<e x="&lambda;">'), {'x': 'λ'})  # HTML 4.0
         self.assertEqual(extract_attributes('<e x="&foo">'), {'x': '&foo'})
         self.assertEqual(extract_attributes('<e x="\'">'), {'x': "'"})
         self.assertEqual(extract_attributes('<e x=\'"\'>'), {'x': '"'})
@@ -654,7 +654,7 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(extract_attributes('<e \nx=\n"y"\n>'), {'x': 'y'})
         self.assertEqual(extract_attributes("<e \nx=\n'y'\n>"), {'x': 'y'})
         self.assertEqual(extract_attributes('<e \nx="\ny\n">'), {'x': '\ny\n'})
-        self.assertEqual(extract_attributes('<e CAPS=x>'), {'caps': 'x'}) # Names lowercased
+        self.assertEqual(extract_attributes('<e CAPS=x>'), {'caps': 'x'})  # Names lowercased
         self.assertEqual(extract_attributes('<e x=1 X=2>'), {'x': '2'})
         self.assertEqual(extract_attributes('<e X=1 x=2>'), {'x': '2'})
         self.assertEqual(extract_attributes('<e _:funny-name1=1>'), {'_:funny-name1': '1'})
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index ec186918c..8ec1bd469 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -273,15 +273,17 @@ def get_element_by_attribute(attribute, value, html):
 
     return unescapeHTML(res)
 
+
 class HTMLAttributeParser(compat_HTMLParser):
     """Trivial HTML parser to gather the attributes for a single element"""
     def __init__(self):
-        self.attrs = { }
+        self.attrs = {}
         compat_HTMLParser.__init__(self)
 
     def handle_starttag(self, tag, attrs):
         self.attrs = dict(attrs)
 
+
 def extract_attributes(html_element):
     """Given a string for an HTML element such as
     <el
@@ -303,6 +305,7 @@ def extract_attributes(html_element):
     parser.close()
     return parser.attrs
 
+
 def clean_html(html):
     """Clean an HTML snippet into a readable string"""
 

From 63f41d3821f28c7565618e725e6c1af618aa1602 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 16 Mar 2016 21:26:25 +0100
Subject: [PATCH 4317/6219] [bravotv] Add new extractor(#4657)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/bravotv.py  | 28 ++++++++++++++++++++++++++++
 2 files changed, 29 insertions(+)
 create mode 100644 youtube_dl/extractor/bravotv.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 9502d07a4..725ebec04 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -81,6 +81,7 @@ from .bloomberg import BloombergIE
 from .bokecc import BokeCCIE
 from .bpb import BpbIE
 from .br import BRIE
+from .bravotv import BravoTVIE
 from .breakcom import BreakIE
 from .brightcove import (
     BrightcoveLegacyIE,
diff --git a/youtube_dl/extractor/bravotv.py b/youtube_dl/extractor/bravotv.py
new file mode 100644
index 000000000..69d00b466
--- /dev/null
+++ b/youtube_dl/extractor/bravotv.py
@@ -0,0 +1,28 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import smuggle_url
+
+
+class BravoTVIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?bravotv\.com/(?:[^/]+/)+videos/(?P<id>[^/?]+)'
+    _TEST = {
+        'url': 'http://www.bravotv.com/last-chance-kitchen/season-5/videos/lck-ep-12-fishy-finale',
+        'md5': 'd60cdf68904e854fac669bd26cccf801',
+        'info_dict': {
+            'id': 'LitrBdX64qLn',
+            'ext': 'mp4',
+            'title': 'Last Chance Kitchen Returns',
+            'description': 'S13: Last Chance Kitchen Returns for Top Chef Season 13',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        account_pid = self._search_regex(r'"account_pid"\s*:\s*"([^"]+)"', webpage, 'account pid')
+        release_pid = self._search_regex(r'"release_pid"\s*:\s*"([^"]+)"', webpage, 'release pid')
+        return self.url_result(smuggle_url(
+            'http://link.theplatform.com/s/%s/%s?format=SMIL&mbr=true&switch=progressive' % (account_pid, release_pid),
+            {'force_smil_url': True}), 'ThePlatform', release_pid)

From a646a8cf980a946cfc15d2286fcec6ee3987886f Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 17 Mar 2016 02:02:18 +0100
Subject: [PATCH 4318/6219] [sbs] improve extraction(fixes #3811)

- extract error messages
- force the platform smil url(previously the manifest param
in the query is not respected which make theplatform return non working
mp4 files for some videos)
---
 youtube_dl/extractor/sbs.py | 31 +++++++++++++++++++++----------
 1 file changed, 21 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/sbs.py b/youtube_dl/extractor/sbs.py
index d6ee2d9e2..2f96477ca 100644
--- a/youtube_dl/extractor/sbs.py
+++ b/youtube_dl/extractor/sbs.py
@@ -2,6 +2,10 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..utils import (
+    smuggle_url,
+    ExtractorError,
+)
 
 
 class SBSIE(InfoExtractor):
@@ -31,21 +35,28 @@ class SBSIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
+        player_params = self._download_json(
+            'http://www.sbs.com.au/api/video_pdkvars/id/%s?form=json' % video_id, video_id)
 
-        webpage = self._download_webpage(
-            'http://www.sbs.com.au/ondemand/video/single/%s?context=web' % video_id, video_id)
-
-        player_params = self._parse_json(
-            self._search_regex(
-                r'(?s)var\s+playerParams\s*=\s*({.+?});', webpage, 'playerParams'),
-            video_id)
+        error = player_params.get('error')
+        if error:
+            error_message = 'Sorry, The video you are looking for does not exist.'
+            video_data = error.get('results') or {}
+            error_code = error.get('errorCode')
+            if error_code == 'ComingSoon':
+                error_message = '%s is not yet available.' % video_data.get('title', '')
+            elif error_code in ('Forbidden', 'intranetAccessOnly'):
+                error_message = 'Sorry, This video cannot be accessed via this website'
+            elif error_code == 'Expired':
+                error_message = 'Sorry, %s is no longer available.' % video_data.get('title', '')
+            raise ExtractorError('%s said: %s' % (self.IE_NAME, error_message), expected=True)
 
         urls = player_params['releaseUrls']
-        theplatform_url = (urls.get('progressive') or urls.get('standard') or
-                           urls.get('html') or player_params['relatedItemsURL'])
+        theplatform_url = (urls.get('progressive') or urls.get('html') or
+                           urls.get('standard') or player_params['relatedItemsURL'])
 
         return {
             '_type': 'url_transparent',
             'id': video_id,
-            'url': theplatform_url,
+            'url': smuggle_url(theplatform_url, {'force_smil_url': True}),
         }

From 11f12195af73a2b0a09de928247cb87aed6dd693 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 17 Mar 2016 19:25:37 +0800
Subject: [PATCH 4319/6219] [youtube] Added itag 91

Seen in https://www.youtube.com/watch?v=jMN4cxyhJjk
---
 youtube_dl/extractor/youtube.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 27e67feb4..466f5da2e 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -309,6 +309,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         '102': {'ext': 'webm', 'height': 720, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
 
         # Apple HTTP Live Streaming
+        '91': {'ext': 'mp4', 'height': 144, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
         '92': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
         '93': {'ext': 'mp4', 'height': 360, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
         '94': {'ext': 'mp4', 'height': 480, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},

From 0436ec0e7a4683539bc7844511ba76fbcab03f7b Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 17 Mar 2016 16:05:31 +0100
Subject: [PATCH 4320/6219] [once] Add new format extractor

---
 youtube_dl/extractor/once.py | 37 ++++++++++++++++++++++++++++++++++++
 1 file changed, 37 insertions(+)
 create mode 100644 youtube_dl/extractor/once.py

diff --git a/youtube_dl/extractor/once.py b/youtube_dl/extractor/once.py
new file mode 100644
index 000000000..403f8c0af
--- /dev/null
+++ b/youtube_dl/extractor/once.py
@@ -0,0 +1,37 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+
+class OnceIE(InfoExtractor):
+    _VALID_URL = r'https?://once\.unicornmedia\.com/now/[^/]+/[^/]+/(?P<domain_id>[^/]+)/(?P<application_id>[^/]+)/(?:[^/]+/)?(?P<media_item_id>[^/]+)/content\.(?:once|m3u8|mp4)'
+    ADAPTIVE_URL_TEMPLATE = 'http://once.unicornmedia.com/now/master/playlist/%s/%s/%s/content.m3u8'
+    PROGRESSIVE_URL_TEMPLATE = 'http://once.unicornmedia.com/now/media/progressive/%s/%s/%s/%s/content.mp4'
+
+    def _extract_once_formats(self, url):
+        domain_id, application_id, media_item_id = re.match(
+            OnceIE._VALID_URL, url).groups()
+        adaptive_formats = self._extract_m3u8_formats(
+            self.ADAPTIVE_URL_TEMPLATE % (
+                domain_id, application_id, media_item_id),
+            media_item_id, 'mp4', m3u8_id='hls', fatal=False)
+        formats = []
+        formats.extend(adaptive_formats)
+        for adaptive_format in adaptive_formats:
+            rendition_id = self._search_regex(
+                r'/now/media/playlist/[^/]+/[^/]+/([^/]+)',
+                adaptive_format['url'], 'redition id', default=None)
+            if rendition_id:
+                progressive_format = adaptive_format.copy()
+                progressive_format.update({
+                    'url': self.PROGRESSIVE_URL_TEMPLATE % (
+                        domain_id, application_id, rendition_id, media_item_id),
+                    'format_id': adaptive_format['format_id'].replace(
+                        'hls', 'http'),
+                    'protocol': 'http',
+                })
+                formats.append(progressive_format)
+        return formats

From 9f02ff537c6ddfd3f1ea3586f3e44f0ec07a2aea Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 17 Mar 2016 16:06:25 +0100
Subject: [PATCH 4321/6219] [theplatform] extract brightcove once formats

---
 youtube_dl/extractor/theplatform.py | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index 9a57b49df..ffe7c57ad 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -8,13 +8,12 @@ import binascii
 import hashlib
 
 
-from .common import InfoExtractor
+from .once import OnceIE
 from ..compat import (
     compat_parse_qs,
     compat_urllib_parse_urlparse,
 )
 from ..utils import (
-    determine_ext,
     ExtractorError,
     float_or_none,
     int_or_none,
@@ -29,7 +28,7 @@ default_ns = 'http://www.w3.org/2005/SMIL21/Language'
 _x = lambda p: xpath_with_ns(p, {'smil': default_ns})
 
 
-class ThePlatformBaseIE(InfoExtractor):
+class ThePlatformBaseIE(OnceIE):
     def _extract_theplatform_smil(self, smil_url, video_id, note='Downloading SMIL data'):
         meta = self._download_xml(smil_url, video_id, note=note)
         error_element = find_xpath_attr(
@@ -38,17 +37,19 @@ class ThePlatformBaseIE(InfoExtractor):
         if error_element is not None:
             raise ExtractorError(error_element.attrib['abstract'], expected=True)
 
-        formats = self._parse_smil_formats(
+        smil_formats = self._parse_smil_formats(
             meta, smil_url, video_id, namespace=default_ns,
             # the parameters are from syfy.com, other sites may use others,
             # they also work for nbc.com
             f4m_params={'g': 'UXWGVKRWHFSP', 'hdcore': '3.0.3'},
             transform_rtmp_url=lambda streamer, src: (streamer, 'mp4:' + src))
 
-        for _format in formats:
-            ext = determine_ext(_format['url'])
-            if ext == 'once':
-                _format['ext'] = 'mp4'
+        formats = []
+        for _format in smil_formats:
+            if OnceIE.suitable(_format['url']):
+                formats.extend(self._extract_once_formats(_format['url']))
+            else:
+                formats.append(_format)
 
         self._sort_formats(formats)
 
@@ -125,7 +126,7 @@ class ThePlatformIE(ThePlatformBaseIE):
         'only_matching': True,
     }, {
         'url': 'http://player.theplatform.com/p/2E2eJC/nbcNewsOffsite?guid=tdy_or_siri_150701',
-        'md5': '734f3790fb5fc4903da391beeebc4836',
+        'md5': 'fb96bb3d85118930a5b055783a3bd992',
         'info_dict': {
             'id': 'tdy_or_siri_150701',
             'ext': 'mp4',
@@ -135,7 +136,6 @@ class ThePlatformIE(ThePlatformBaseIE):
             'thumbnail': 're:^https?://.*\.jpg$',
             'timestamp': 1435752600,
             'upload_date': '20150701',
-            'categories': ['Today/Shows/Orange Room', 'Today/Sections/Money', 'Today/Topics/Tech', "Today/Topics/Editor's picks"],
         },
     }, {
         # From http://www.nbc.com/the-blacklist/video/sir-crispin-crandall/2928790?onid=137781#vc137781=1
@@ -250,7 +250,7 @@ class ThePlatformFeedIE(ThePlatformBaseIE):
     _TEST = {
         # From http://player.theplatform.com/p/7wvmTC/MSNBCEmbeddedOffSite?guid=n_hardball_5biden_140207
         'url': 'http://feed.theplatform.com/f/7wvmTC/msnbc_video-p-test?form=json&pretty=true&range=-40&byGuid=n_hardball_5biden_140207',
-        'md5': '22d2b84f058d3586efcd99e57d59d314',
+        'md5': '6e32495b5073ab414471b615c5ded394',
         'info_dict': {
             'id': 'n_hardball_5biden_140207',
             'ext': 'mp4',

From 574b2a7393ef389792d5010704e505ef0eaaa5e8 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 17 Mar 2016 16:07:36 +0100
Subject: [PATCH 4322/6219] [nbc:nbcnews] improve extraction(fixes #6922)

- extract more metadata and formats
- relax regex
---
 youtube_dl/extractor/nbc.py | 137 ++++++++++++++++++++++++------------
 1 file changed, 91 insertions(+), 46 deletions(-)

diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index 2202cfa33..bb0817e34 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -3,13 +3,16 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_HTTPError
+from .theplatform import ThePlatformIE
 from ..utils import (
-    ExtractorError,
     find_xpath_attr,
     lowercase_escape,
     smuggle_url,
     unescapeHTML,
+    update_url_query,
+    int_or_none,
+    HEADRequest,
+    parse_iso8601,
 )
 
 
@@ -131,10 +134,10 @@ class NBCSportsIE(InfoExtractor):
             NBCSportsVPlayerIE._extract_url(webpage), 'NBCSportsVPlayer')
 
 
-class NBCNewsIE(InfoExtractor):
+class NBCNewsIE(ThePlatformIE):
     _VALID_URL = r'''(?x)https?://(?:www\.)?nbcnews\.com/
         (?:video/.+?/(?P<id>\d+)|
-        (?:watch|feature|nightly-news)/[^/]+/(?P<title>.+))
+        ([^/]+/)*(?P<display_id>[^/?]+))
         '''
 
     _TESTS = [
@@ -149,15 +152,14 @@ class NBCNewsIE(InfoExtractor):
             },
         },
         {
-            'url': 'http://www.nbcnews.com/feature/edward-snowden-interview/how-twitter-reacted-snowden-interview-n117236',
-            'md5': 'b2421750c9f260783721d898f4c42063',
+            'url': 'http://www.nbcnews.com/watch/nbcnews-com/how-twitter-reacted-to-the-snowden-interview-269389891880',
+            'md5': 'af1adfa51312291a017720403826bb64',
             'info_dict': {
-                'id': 'I1wpAI_zmhsQ',
+                'id': '269389891880',
                 'ext': 'mp4',
                 'title': 'How Twitter Reacted To The Snowden Interview',
                 'description': 'md5:65a0bd5d76fe114f3c2727aa3a81fe64',
             },
-            'add_ie': ['ThePlatform'],
         },
         {
             'url': 'http://www.nbcnews.com/feature/dateline-full-episodes/full-episode-family-business-n285156',
@@ -168,17 +170,29 @@ class NBCNewsIE(InfoExtractor):
                 'title': 'FULL EPISODE: Family Business',
                 'description': 'md5:757988edbaae9d7be1d585eb5d55cc04',
             },
+            'skip': 'This page is unavailable.',
         },
         {
             'url': 'http://www.nbcnews.com/nightly-news/video/nightly-news-with-brian-williams-full-broadcast-february-4-394064451844',
-            'md5': 'b5dda8cddd8650baa0dcb616dd2cf60d',
+            'md5': '73135a2e0ef819107bbb55a5a9b2a802',
             'info_dict': {
-                'id': 'sekXqyTVnmN3',
+                'id': '394064451844',
                 'ext': 'mp4',
                 'title': 'Nightly News with Brian Williams Full Broadcast (February 4)',
                 'description': 'md5:1c10c1eccbe84a26e5debb4381e2d3c5',
             },
         },
+        {
+            'url': 'http://www.nbcnews.com/business/autos/volkswagen-11-million-vehicles-could-have-suspect-software-emissions-scandal-n431456',
+            'md5': 'a49e173825e5fcd15c13fc297fced39d',
+            'info_dict': {
+                'id': '529953347624',
+                'ext': 'mp4',
+                'title': 'Volkswagen U.S. Chief: We \'Totally Screwed Up\'',
+                'description': 'md5:d22d1281a24f22ea0880741bb4dd6301',
+            },
+            'expected_warnings': ['http-6000 is not available']
+        },
         {
             'url': 'http://www.nbcnews.com/watch/dateline/full-episode--deadly-betrayal-386250819952',
             'only_matching': True,
@@ -202,49 +216,80 @@ class NBCNewsIE(InfoExtractor):
             }
         else:
             # "feature" and "nightly-news" pages use theplatform.com
-            title = mobj.group('title')
-            webpage = self._download_webpage(url, title)
+            display_id = mobj.group('display_id')
+            webpage = self._download_webpage(url, display_id)
+            info = None
             bootstrap_json = self._search_regex(
-                r'var\s+(?:bootstrapJson|playlistData)\s*=\s*({.+});?\s*$',
-                webpage, 'bootstrap json', flags=re.MULTILINE)
-            bootstrap = self._parse_json(bootstrap_json, video_id)
-            info = bootstrap['results'][0]['video']
-            mpxid = info['mpxId']
+                r'(?m)var\s+(?:bootstrapJson|playlistData)\s*=\s*({.+});?\s*$',
+                webpage, 'bootstrap json', default=None)
+            if bootstrap_json:
+                bootstrap = self._parse_json(bootstrap_json, display_id)
+                info = bootstrap['results'][0]['video']
+            else:
+                player_instance_json = self._search_regex(
+                    r'videoObj\s*:\s*({.+})', webpage, 'player instance')
+                info = self._parse_json(player_instance_json, display_id)
+            video_id = info['mpxId']
+            title = info['title']
 
-            base_urls = [
-                info['fallbackPlaylistUrl'],
-                info['associatedPlaylistUrl'],
-            ]
+            subtitles = {}
+            caption_links = info.get('captionLinks')
+            if caption_links:
+                for (sub_key, sub_ext) in (('smpte-tt', 'ttml'), ('web-vtt', 'vtt'), ('srt', 'srt')):
+                    sub_url = caption_links.get(sub_key)
+                    if sub_url:
+                        subtitles.setdefault('en', []).append({
+                            'url': sub_url,
+                            'ext': sub_ext,
+                        })
 
-            for base_url in base_urls:
-                if not base_url:
+            formats = []
+            for video_asset in info['videoAssets']:
+                video_url = video_asset.get('publicUrl')
+                if not video_url:
                     continue
-                playlist_url = base_url + '?form=MPXNBCNewsAPI'
-
-                try:
-                    all_videos = self._download_json(playlist_url, title)
-                except ExtractorError as ee:
-                    if isinstance(ee.cause, compat_HTTPError):
-                        continue
-                    raise
-
-                if not all_videos or 'videos' not in all_videos:
+                container = video_asset.get('format')
+                asset_type = video_asset.get('assetType') or ''
+                if container == 'ISM' or asset_type == 'FireTV-Once':
                     continue
-
-                try:
-                    info = next(v for v in all_videos['videos'] if v['mpxId'] == mpxid)
-                    break
-                except StopIteration:
-                    continue
-
-            if info is None:
-                raise ExtractorError('Could not find video in playlists')
+                elif asset_type == 'OnceURL':
+                    tp_formats, tp_subtitles = self._extract_theplatform_smil(
+                        video_url, video_id)
+                    formats.extend(tp_formats)
+                    subtitles = self._merge_subtitles(subtitles, tp_subtitles)
+                else:
+                    tbr = int_or_none(video_asset.get('bitRate'), 1000)
+                    format_id = 'http%s' % ('-%d' % tbr if tbr else '')
+                    video_url = update_url_query(
+                        video_url, {'format': 'redirect'})
+                    # resolve the url so that we can check availability and detect the correct extension
+                    head = self._request_webpage(
+                        HEADRequest(video_url), video_id,
+                        'Checking %s url' % format_id,
+                        '%s is not available' % format_id,
+                        fatal=False)
+                    if head:
+                        video_url = head.geturl()
+                        formats.append({
+                            'format_id': format_id,
+                            'url': video_url,
+                            'width': int_or_none(video_asset.get('width')),
+                            'height': int_or_none(video_asset.get('height')),
+                            'tbr': tbr,
+                            'container': video_asset.get('format'),
+                        })
+            self._sort_formats(formats)
 
             return {
-                '_type': 'url',
-                # We get the best quality video
-                'url': info['videoAssets'][-1]['publicUrl'],
-                'ie_key': 'ThePlatform',
+                'id': video_id,
+                'title': title,
+                'description': info.get('description'),
+                'thumbnail': info.get('description'),
+                'thumbnail': info.get('thumbnail'),
+                'duration': int_or_none(info.get('duration')),
+                'timestamp': parse_iso8601(info.get('pubDate')),
+                'formats': formats,
+                'subtitles': subtitles,
             }
 
 

From cf45ed786e580999afe864724c3b7d16abadb4e1 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 17 Mar 2016 17:48:17 +0100
Subject: [PATCH 4323/6219] [wistia] extract more metadata

---
 youtube_dl/extractor/wistia.py | 42 ++++++++++++++++++++++------------
 1 file changed, 27 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/wistia.py b/youtube_dl/extractor/wistia.py
index 41061dd31..8b14840a2 100644
--- a/youtube_dl/extractor/wistia.py
+++ b/youtube_dl/extractor/wistia.py
@@ -4,6 +4,7 @@ from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
     sanitized_Request,
+    int_or_none,
 )
 
 
@@ -18,6 +19,9 @@ class WistiaIE(InfoExtractor):
             'id': 'sh7fpupwlt',
             'ext': 'mov',
             'title': 'Being Resourceful',
+            'description': 'a Clients From Hell Video Series video from worldwidewebhosting',
+            'upload_date': '20131204',
+            'timestamp': 1386185018,
             'duration': 117,
         },
     }
@@ -32,35 +36,43 @@ class WistiaIE(InfoExtractor):
             raise ExtractorError('Error while getting the playlist',
                                  expected=True)
         data = data_json['media']
+        title = data['name']
 
         formats = []
         thumbnails = []
         for a in data['assets']:
+            astatus = a.get('status')
             atype = a.get('type')
-            if atype == 'still':
+            if (astatus is not None and astatus != 2) or atype == 'preview':
+                continue
+            elif atype in ('still', 'still_image'):
                 thumbnails.append({
                     'url': a['url'],
                     'resolution': '%dx%d' % (a['width'], a['height']),
                 })
-                continue
-            if atype == 'preview':
-                continue
-            formats.append({
-                'format_id': atype,
-                'url': a['url'],
-                'width': a['width'],
-                'height': a['height'],
-                'filesize': a['size'],
-                'ext': a['ext'],
-                'preference': 1 if atype == 'original' else None,
-            })
+            else:
+                formats.append({
+                    'format_id': atype,
+                    'url': a['url'],
+                    'tbr': int_or_none(a.get('bitrate')),
+                    'vbr': int_or_none(a.get('opt_vbitrate')),
+                    'width': int_or_none(a.get('width')),
+                    'height': int_or_none(a.get('height')),
+                    'filesize': int_or_none(a.get('size')),
+                    'vcodec': a.get('codec'),
+                    'container': a.get('container'),
+                    'ext': a.get('ext'),
+                    'preference': 1 if atype == 'original' else None,
+                })
 
         self._sort_formats(formats)
 
         return {
             'id': video_id,
-            'title': data['name'],
+            'title': title,
+            'description': data.get('seoDescription'),
             'formats': formats,
             'thumbnails': thumbnails,
-            'duration': data.get('duration'),
+            'duration': int_or_none(data.get('duration')),
+            'timestamp': int_or_none(data.get('createdAt')),
         }

From cc162f6a0aa63a3e050c55cec9da728aa2cb9100 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 17 Mar 2016 22:55:04 +0600
Subject: [PATCH 4324/6219] [crunchyroll] Fix custom _download_webpage (Closes
 #8883)

---
 youtube_dl/extractor/crunchyroll.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index c7032ffa2..85fa7a725 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -54,7 +54,7 @@ class CrunchyrollBaseIE(InfoExtractor):
     def _real_initialize(self):
         self._login()
 
-    def _download_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True, tries=1, timeout=5, encoding=None):
+    def _download_webpage(self, url_or_request, *args, **kwargs):
         request = (url_or_request if isinstance(url_or_request, compat_urllib_request.Request)
                    else sanitized_Request(url_or_request))
         # Accept-Language must be set explicitly to accept any language to avoid issues
@@ -65,8 +65,7 @@ class CrunchyrollBaseIE(InfoExtractor):
         # Crunchyroll to not work in georestriction cases in some browsers that don't place
         # the locale lang first in header. However allowing any language seems to workaround the issue.
         request.add_header('Accept-Language', '*')
-        return super(CrunchyrollBaseIE, self)._download_webpage(
-            request, video_id, note, errnote, fatal, tries, timeout, encoding)
+        return super(CrunchyrollBaseIE, self)._download_webpage(request, *args, **kwargs)
 
     @staticmethod
     def _add_skip_wall(url):

From 8c97e7efb6ab273f0b7c91f0aa9ac6869c911bf1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 17 Mar 2016 23:43:14 +0600
Subject: [PATCH 4325/6219] [animeondemand] Expand episode title regex (Closes
 #8875)

---
 youtube_dl/extractor/animeondemand.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/animeondemand.py b/youtube_dl/extractor/animeondemand.py
index a7d8daf7b..3dbbe2a62 100644
--- a/youtube_dl/extractor/animeondemand.py
+++ b/youtube_dl/extractor/animeondemand.py
@@ -93,7 +93,7 @@ class AnimeOnDemandIE(InfoExtractor):
 
         for episode_html in re.findall(r'(?s)<h3[^>]+class="episodebox-title".+?>Episodeninhalt<', webpage):
             m = re.search(
-                r'class="episodebox-title"[^>]+title="Episode (?P<number>\d+) - (?P<title>.+?)"', episode_html)
+                r'class="episodebox-title"[^>]+title="(?:Episode|Film)\s*(?P<number>\d+)\s*-\s*(?P<title>.+?)"', episode_html)
             if not m:
                 continue
 

From b57fecfdddc78b5ef5cfd1c3302f7b79ab1bf64f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 17 Mar 2016 23:50:10 +0600
Subject: [PATCH 4326/6219] [animeondemand] Add test

---
 youtube_dl/extractor/animeondemand.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/animeondemand.py b/youtube_dl/extractor/animeondemand.py
index 3dbbe2a62..6cb3a84f9 100644
--- a/youtube_dl/extractor/animeondemand.py
+++ b/youtube_dl/extractor/animeondemand.py
@@ -18,7 +18,7 @@ class AnimeOnDemandIE(InfoExtractor):
     _LOGIN_URL = 'https://www.anime-on-demand.de/users/sign_in'
     _APPLY_HTML5_URL = 'https://www.anime-on-demand.de/html5apply'
     _NETRC_MACHINE = 'animeondemand'
-    _TEST = {
+    _TESTS = [{
         'url': 'https://www.anime-on-demand.de/anime/161',
         'info_dict': {
             'id': '161',
@@ -26,7 +26,11 @@ class AnimeOnDemandIE(InfoExtractor):
             'description': 'md5:6681ce3c07c7189d255ac6ab23812d31',
         },
         'playlist_mincount': 4,
-    }
+    }, {
+        # Film wording is used instead of Episode
+        'url': 'https://www.anime-on-demand.de/anime/39',
+        'only_matching': True,
+    }]
 
     def _login(self):
         (username, password) = self._get_login_info()

From 85e8f26b827e77cbed6a83268787d450ab2bea3b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 18 Mar 2016 00:02:34 +0600
Subject: [PATCH 4327/6219] [animeondemand] Improve extraction

---
 youtube_dl/extractor/animeondemand.py | 24 ++++++++++++++++++------
 1 file changed, 18 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/animeondemand.py b/youtube_dl/extractor/animeondemand.py
index 6cb3a84f9..3631c2451 100644
--- a/youtube_dl/extractor/animeondemand.py
+++ b/youtube_dl/extractor/animeondemand.py
@@ -30,6 +30,10 @@ class AnimeOnDemandIE(InfoExtractor):
         # Film wording is used instead of Episode
         'url': 'https://www.anime-on-demand.de/anime/39',
         'only_matching': True,
+    }, {
+        # Episodes without titles
+        'url': 'https://www.anime-on-demand.de/anime/162',
+        'only_matching': True,
     }]
 
     def _login(self):
@@ -95,14 +99,22 @@ class AnimeOnDemandIE(InfoExtractor):
 
         entries = []
 
-        for episode_html in re.findall(r'(?s)<h3[^>]+class="episodebox-title".+?>Episodeninhalt<', webpage):
-            m = re.search(
-                r'class="episodebox-title"[^>]+title="(?:Episode|Film)\s*(?P<number>\d+)\s*-\s*(?P<title>.+?)"', episode_html)
-            if not m:
+        for num, episode_html in enumerate(re.findall(
+                r'(?s)<h3[^>]+class="episodebox-title".+?>Episodeninhalt<', webpage)):
+            episodebox_title = self._search_regex(
+                (r'class="episodebox-title"[^>]+title="(.+?)"',
+                 r'class="episodebox-title"[^>]+>(.+?)<'),
+                webpage, 'episodebox title', default=None)
+            if not episodebox_title:
                 continue
 
-            episode_number = int(m.group('number'))
-            episode_title = m.group('title')
+            episode_number = int(self._search_regex(
+                r'^(?:Episode|Film)\s*(\d+)',
+                episodebox_title, 'episode number', default=num))
+            episode_title = self._search_regex(
+                r'(?:Episode|Film)\s*\d+\s*-\s*(?P<title>.+?)',
+                episodebox_title, 'episode title', default=None)
+
             video_id = 'episode-%d' % episode_number
 
             common_info = {

From 0d0e282912a7ade43a148518c742557c310a41a3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 18 Mar 2016 00:12:34 +0600
Subject: [PATCH 4328/6219] [animeondemand] Fix typo and improve

---
 youtube_dl/extractor/animeondemand.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/animeondemand.py b/youtube_dl/extractor/animeondemand.py
index 3631c2451..0158407f6 100644
--- a/youtube_dl/extractor/animeondemand.py
+++ b/youtube_dl/extractor/animeondemand.py
@@ -100,19 +100,19 @@ class AnimeOnDemandIE(InfoExtractor):
         entries = []
 
         for num, episode_html in enumerate(re.findall(
-                r'(?s)<h3[^>]+class="episodebox-title".+?>Episodeninhalt<', webpage)):
+                r'(?s)<h3[^>]+class="episodebox-title".+?>Episodeninhalt<', webpage), 1):
             episodebox_title = self._search_regex(
-                (r'class="episodebox-title"[^>]+title="(.+?)"',
-                 r'class="episodebox-title"[^>]+>(.+?)<'),
-                webpage, 'episodebox title', default=None)
+                (r'class="episodebox-title"[^>]+title=(["\'])(?P<title>.+?)\1',
+                 r'class="episodebox-title"[^>]+>(?P<title>.+?)<'),
+                episode_html, 'episodebox title', default=None, group='title')
             if not episodebox_title:
                 continue
 
             episode_number = int(self._search_regex(
-                r'^(?:Episode|Film)\s*(\d+)',
+                r'(?:Episode|Film)\s*(\d+)',
                 episodebox_title, 'episode number', default=num))
             episode_title = self._search_regex(
-                r'(?:Episode|Film)\s*\d+\s*-\s*(?P<title>.+?)',
+                r'(?:Episode|Film)\s*\d+\s*-\s*(.+)',
                 episodebox_title, 'episode title', default=None)
 
             video_id = 'episode-%d' % episode_number

From 57f7e3c62df187457a057be88fca43136f4c507f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 18 Mar 2016 02:51:38 +0600
Subject: [PATCH 4329/6219] [compat] Add compat_xpath

---
 youtube_dl/compat.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 74702786a..dbb91a6ef 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -256,6 +256,16 @@ else:
                 el.text = el.text.decode('utf-8')
         return doc
 
+if sys.version_info < (2, 7):
+    # Here comes the crazy part: In 2.6, if the xpath is a unicode,
+    # .//node does not match if a node is a direct child of . !
+    def compat_xpath(xpath):
+        if isinstance(xpath, compat_str):
+            xpath = xpath.encode('ascii')
+        return xpath
+else:
+    compat_xpath = lambda xpath: xpath
+
 try:
     from urllib.parse import parse_qs as compat_parse_qs
 except ImportError:  # Python 2
@@ -585,6 +595,7 @@ __all__ = [
     'compat_urlparse',
     'compat_urlretrieve',
     'compat_xml_parse_error',
+    'compat_xpath',
     'shlex_quote',
     'subprocess_check_output',
     'workaround_optparse_bug9161',

From 810c10baa1e0177a6a0ef39496f7e972db02d806 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 18 Mar 2016 02:52:23 +0600
Subject: [PATCH 4330/6219] [utils] Use compat_xpath

---
 youtube_dl/utils.py | 12 +++---------
 1 file changed, 3 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 8ec1bd469..ef6e7c7cb 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -50,6 +50,7 @@ from .compat import (
     compat_urllib_parse_urlparse,
     compat_urllib_request,
     compat_urlparse,
+    compat_xpath,
     shlex_quote,
 )
 
@@ -165,12 +166,7 @@ if sys.version_info >= (2, 7):
         return node.find(expr)
 else:
     def find_xpath_attr(node, xpath, key, val=None):
-        # Here comes the crazy part: In 2.6, if the xpath is a unicode,
-        # .//node does not match if a node is a direct child of . !
-        if isinstance(xpath, compat_str):
-            xpath = xpath.encode('ascii')
-
-        for f in node.findall(xpath):
+        for f in node.findall(compat_xpath(xpath)):
             if key not in f.attrib:
                 continue
             if val is None or f.attrib.get(key) == val:
@@ -195,9 +191,7 @@ def xpath_with_ns(path, ns_map):
 
 def xpath_element(node, xpath, name=None, fatal=False, default=NO_DEFAULT):
     def _find_xpath(xpath):
-        if sys.version_info < (2, 7):  # Crazy 2.6
-            xpath = xpath.encode('ascii')
-        return node.find(xpath)
+        return node.find(compat_xpath(xpath))
 
     if isinstance(xpath, (str, compat_str)):
         n = _find_xpath(xpath)

From e3d17b3c07c6d8bc7fd45af1e45523e8fde5fb58 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 18 Mar 2016 02:54:27 +0600
Subject: [PATCH 4331/6219] [noz] Fix extraction on python 2.6 by means of
 using compat_xpath

---
 youtube_dl/extractor/noz.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/noz.py b/youtube_dl/extractor/noz.py
index 656443c49..c47a33d15 100644
--- a/youtube_dl/extractor/noz.py
+++ b/youtube_dl/extractor/noz.py
@@ -2,7 +2,10 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse_unquote
+from ..compat import (
+    compat_urllib_parse_unquote,
+    compat_xpath,
+)
 from ..utils import (
     int_or_none,
     find_xpath_attr,
@@ -47,7 +50,7 @@ class NozIE(InfoExtractor):
         duration = int_or_none(xpath_text(
             doc, './/article/movie/file/duration'))
         formats = []
-        for qnode in doc.findall('.//article/movie/file/qualities/qual'):
+        for qnode in doc.findall(compat_xpath('.//article/movie/file/qualities/qual')):
             http_url_ele = find_xpath_attr(
                 qnode, './html_urls/video_url', 'format', 'video/mp4')
             http_url = http_url_ele.text if http_url_ele is not None else None

From 4c92fd2e835cde89866d3dfb1fc05d23196b19db Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 18 Mar 2016 09:21:21 +0100
Subject: [PATCH 4332/6219] [theplatform] always force theplatform to return a
 smil for _extract_theplatform_smil

---
 youtube_dl/extractor/bravotv.py            |  2 +-
 youtube_dl/extractor/cbsnews.py            |  2 +-
 youtube_dl/extractor/cnet.py               |  2 +-
 youtube_dl/extractor/nationalgeographic.py |  2 +-
 youtube_dl/extractor/theplatform.py        | 10 ++++++----
 5 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/bravotv.py b/youtube_dl/extractor/bravotv.py
index 69d00b466..34d451f38 100644
--- a/youtube_dl/extractor/bravotv.py
+++ b/youtube_dl/extractor/bravotv.py
@@ -24,5 +24,5 @@ class BravoTVIE(InfoExtractor):
         account_pid = self._search_regex(r'"account_pid"\s*:\s*"([^"]+)"', webpage, 'account pid')
         release_pid = self._search_regex(r'"release_pid"\s*:\s*"([^"]+)"', webpage, 'release pid')
         return self.url_result(smuggle_url(
-            'http://link.theplatform.com/s/%s/%s?format=SMIL&mbr=true&switch=progressive' % (account_pid, release_pid),
+            'http://link.theplatform.com/s/%s/%s?mbr=true&switch=progressive' % (account_pid, release_pid),
             {'force_smil_url': True}), 'ThePlatform', release_pid)
diff --git a/youtube_dl/extractor/cbsnews.py b/youtube_dl/extractor/cbsnews.py
index 7319ee1b7..8ddcc5097 100644
--- a/youtube_dl/extractor/cbsnews.py
+++ b/youtube_dl/extractor/cbsnews.py
@@ -78,7 +78,7 @@ class CBSNewsIE(ThePlatformIE):
             pid = item.get('media' + format_id)
             if not pid:
                 continue
-            release_url = 'http://link.theplatform.com/s/dJ5BDC/%s?format=SMIL&mbr=true' % pid
+            release_url = 'http://link.theplatform.com/s/dJ5BDC/%s?mbr=true' % pid
             tp_formats, tp_subtitles = self._extract_theplatform_smil(release_url, video_id, 'Downloading %s SMIL data' % pid)
             formats.extend(tp_formats)
             subtitles = self._merge_subtitles(subtitles, tp_subtitles)
diff --git a/youtube_dl/extractor/cnet.py b/youtube_dl/extractor/cnet.py
index 3cf0bf95b..c154b3e19 100644
--- a/youtube_dl/extractor/cnet.py
+++ b/youtube_dl/extractor/cnet.py
@@ -60,7 +60,7 @@ class CNETIE(ThePlatformIE):
         for (fkey, vid) in vdata['files'].items():
             if fkey == 'hls_phone' and 'hls_tablet' in vdata['files']:
                 continue
-            release_url = 'http://link.theplatform.com/s/kYEXFC/%s?format=SMIL&mbr=true' % vid
+            release_url = 'http://link.theplatform.com/s/kYEXFC/%s?mbr=true' % vid
             if fkey == 'hds':
                 release_url += '&manifest=f4m'
             tp_formats, tp_subtitles = self._extract_theplatform_smil(release_url, video_id, 'Downloading %s SMIL data' % fkey)
diff --git a/youtube_dl/extractor/nationalgeographic.py b/youtube_dl/extractor/nationalgeographic.py
index 6fc9e7b05..7ce8d9b18 100644
--- a/youtube_dl/extractor/nationalgeographic.py
+++ b/youtube_dl/extractor/nationalgeographic.py
@@ -48,7 +48,7 @@ class NationalGeographicIE(InfoExtractor):
         theplatform_id = url_basename(content.attrib.get('url'))
 
         return self.url_result(smuggle_url(
-            'http://link.theplatform.com/s/ngs/%s?format=SMIL&formats=MPEG4&manifest=f4m' % theplatform_id,
+            'http://link.theplatform.com/s/ngs/%s?formats=MPEG4&manifest=f4m' % theplatform_id,
             # For some reason, the normal links don't work and we must force
             # the use of f4m
             {'force_smil_url': True}))
diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index ffe7c57ad..a148f78ce 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -22,6 +22,7 @@ from ..utils import (
     xpath_with_ns,
     mimetype2ext,
     find_xpath_attr,
+    update_url_query,
 )
 
 default_ns = 'http://www.w3.org/2005/SMIL21/Language'
@@ -30,6 +31,7 @@ _x = lambda p: xpath_with_ns(p, {'smil': default_ns})
 
 class ThePlatformBaseIE(OnceIE):
     def _extract_theplatform_smil(self, smil_url, video_id, note='Downloading SMIL data'):
+        smil_url = update_url_query(smil_url, {'format': 'SMIL'})
         meta = self._download_xml(smil_url, video_id, note=note)
         error_element = find_xpath_attr(
             meta, _x('.//smil:ref'), 'src',
@@ -213,7 +215,7 @@ class ThePlatformIE(ThePlatformBaseIE):
                 webpage, 'smil url', group='url')
             path = self._search_regex(
                 r'link\.theplatform\.com/s/((?:[^/?#&]+/)+[^/?#&]+)', smil_url, 'path')
-            smil_url += '?' if '?' not in smil_url else '&' + 'formats=m3u,mpeg4&format=SMIL'
+            smil_url += '?' if '?' not in smil_url else '&' + 'formats=m3u,mpeg4'
         elif mobj.group('config'):
             config_url = url + '&form=json'
             config_url = config_url.replace('swf/', 'config/')
@@ -223,9 +225,9 @@ class ThePlatformIE(ThePlatformBaseIE):
                 release_url = config['releaseUrl']
             else:
                 release_url = 'http://link.theplatform.com/s/%s?mbr=true' % path
-            smil_url = release_url + '&format=SMIL&formats=MPEG4&manifest=f4m'
+            smil_url = release_url + '&formats=MPEG4&manifest=f4m'
         else:
-            smil_url = 'http://link.theplatform.com/s/%s/meta.smil?format=smil&mbr=true' % path
+            smil_url = 'http://link.theplatform.com/s/%s?mbr=true' % path
 
         sig = smuggled_data.get('sig')
         if sig:
@@ -280,7 +282,7 @@ class ThePlatformFeedIE(ThePlatformBaseIE):
         first_video_id = None
         duration = None
         for item in entry['media$content']:
-            smil_url = item['plfile$url'] + '&format=SMIL&mbr=true'
+            smil_url = item['plfile$url'] + '&mbr=true'
             cur_video_id = ThePlatformIE._match_id(smil_url)
             if first_video_id is None:
                 first_video_id = cur_video_id

From 87c03c6bd22e99d6410c907128ab872e79df1560 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 18 Mar 2016 09:43:28 +0100
Subject: [PATCH 4333/6219] [theplatform] remove unnecessary import

---
 youtube_dl/extractor/theplatform.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index a148f78ce..2230dfe02 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -22,7 +22,6 @@ from ..utils import (
     xpath_with_ns,
     mimetype2ext,
     find_xpath_attr,
-    update_url_query,
 )
 
 default_ns = 'http://www.w3.org/2005/SMIL21/Language'
@@ -31,8 +30,7 @@ _x = lambda p: xpath_with_ns(p, {'smil': default_ns})
 
 class ThePlatformBaseIE(OnceIE):
     def _extract_theplatform_smil(self, smil_url, video_id, note='Downloading SMIL data'):
-        smil_url = update_url_query(smil_url, {'format': 'SMIL'})
-        meta = self._download_xml(smil_url, video_id, note=note)
+        meta = self._download_xml(smil_url, video_id, note=note, query={'format': 'SMIL'})
         error_element = find_xpath_attr(
             meta, _x('.//smil:ref'), 'src',
             'http://link.theplatform.com/s/errorFiles/Unavailable.mp4')

From 0d33166ec586b9c75e20835adca927e923cb36e3 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 18 Mar 2016 11:43:48 +0100
Subject: [PATCH 4334/6219] release 2016.03.18

---
 docs/supportedsites.md | 4 ++++
 youtube_dl/version.py  | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index a6dcc2576..3415efc45 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -81,6 +81,7 @@
  - **BokeCC**
  - **Bpb**: Bundeszentrale für politische Bildung
  - **BR**: Bayerischer Rundfunk Mediathek
+ - **BravoTV**
  - **Break**
  - **brightcove:legacy**
  - **brightcove:new**
@@ -499,6 +500,7 @@
  - **Restudy**
  - **ReverbNation**
  - **Revision3**
+ - **RICE**
  - **RingTV**
  - **RottenTomatoes**
  - **Roxwel**
@@ -617,6 +619,7 @@
  - **ThePlatform**
  - **ThePlatformFeed**
  - **TheSixtyOne**
+ - **TheStar**
  - **ThisAmericanLife**
  - **ThisAV**
  - **THVideo**
@@ -650,6 +653,7 @@
  - **tv.dfb.de**
  - **TV2**
  - **TV2Article**
+ - **TV3**
  - **TV4**: tv4.se and tv4play.se
  - **TVC**
  - **TVCArticle**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 9216fa547..6b2c5fac9 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.03.14'
+__version__ = '2016.03.18'

From 61870915323abd126f5440282b1fd5734ee1ce6f Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 18 Mar 2016 11:50:04 +0100
Subject: [PATCH 4335/6219] [once] check http formats availability

---
 youtube_dl/extractor/once.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/once.py b/youtube_dl/extractor/once.py
index 403f8c0af..080045d4c 100644
--- a/youtube_dl/extractor/once.py
+++ b/youtube_dl/extractor/once.py
@@ -14,13 +14,12 @@ class OnceIE(InfoExtractor):
     def _extract_once_formats(self, url):
         domain_id, application_id, media_item_id = re.match(
             OnceIE._VALID_URL, url).groups()
-        adaptive_formats = self._extract_m3u8_formats(
+        formats = self._extract_m3u8_formats(
             self.ADAPTIVE_URL_TEMPLATE % (
                 domain_id, application_id, media_item_id),
             media_item_id, 'mp4', m3u8_id='hls', fatal=False)
-        formats = []
-        formats.extend(adaptive_formats)
-        for adaptive_format in adaptive_formats:
+        progressive_formats = []
+        for adaptive_format in formats:
             rendition_id = self._search_regex(
                 r'/now/media/playlist/[^/]+/[^/]+/([^/]+)',
                 adaptive_format['url'], 'redition id', default=None)
@@ -33,5 +32,7 @@ class OnceIE(InfoExtractor):
                         'hls', 'http'),
                     'protocol': 'http',
                 })
-                formats.append(progressive_format)
+                progressive_formats.append(progressive_format)
+        self._check_formats(progressive_formats, media_item_id)
+        formats.extend(progressive_formats)
         return formats

From cae21032ab38f404a9959e6b28984b960e579fb5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 18 Mar 2016 21:08:25 +0600
Subject: [PATCH 4336/6219] [theplatform] Improve geo restriction detection

---
 youtube_dl/extractor/theplatform.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index 2230dfe02..863914299 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -31,10 +31,9 @@ _x = lambda p: xpath_with_ns(p, {'smil': default_ns})
 class ThePlatformBaseIE(OnceIE):
     def _extract_theplatform_smil(self, smil_url, video_id, note='Downloading SMIL data'):
         meta = self._download_xml(smil_url, video_id, note=note, query={'format': 'SMIL'})
-        error_element = find_xpath_attr(
-            meta, _x('.//smil:ref'), 'src',
-            'http://link.theplatform.com/s/errorFiles/Unavailable.mp4')
-        if error_element is not None:
+        error_element = find_xpath_attr(meta, _x('.//smil:ref'), 'src')
+        if error_element is not None and error_element.attrib['src'].startswith(
+                'http://link.theplatform.com/s/errorFiles/Unavailable.'):
             raise ExtractorError(error_element.attrib['abstract'], expected=True)
 
         smil_formats = self._parse_smil_formats(

From 263eff9537c73caa9bff42b1e675043eaa124f9a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 18 Mar 2016 21:50:10 +0600
Subject: [PATCH 4337/6219] [extractor/generic] Properly extract format id from
 Content-Type

Fixes extraction for cases like: audio/x-mpegURL; charset=utf-8
---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 8121f04a5..b75db1252 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1249,7 +1249,7 @@ class GenericIE(InfoExtractor):
 
         # Check for direct link to a video
         content_type = head_response.headers.get('Content-Type', '')
-        m = re.match(r'^(?P<type>audio|video|application(?=/(?:ogg$|(?:vnd\.apple\.|x-)?mpegurl)))/(?P<format_id>.+)$', content_type)
+        m = re.match(r'^(?P<type>audio|video|application(?=/(?:ogg$|(?:vnd\.apple\.|x-)?mpegurl)))/(?P<format_id>[^;\s]+)', content_type)
         if m:
             upload_date = unified_strdate(
                 head_response.headers.get('Last-Modified'))

From 955737b2d40c0ce947c13659a27aae0c41077c65 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 18 Mar 2016 21:50:44 +0600
Subject: [PATCH 4338/6219] [extractor/generic] Force Content-Type to lowecase

---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index b75db1252..cce7799e2 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1248,7 +1248,7 @@ class GenericIE(InfoExtractor):
         }
 
         # Check for direct link to a video
-        content_type = head_response.headers.get('Content-Type', '')
+        content_type = head_response.headers.get('Content-Type', '').lower()
         m = re.match(r'^(?P<type>audio|video|application(?=/(?:ogg$|(?:vnd\.apple\.|x-)?mpegurl)))/(?P<format_id>[^;\s]+)', content_type)
         if m:
             upload_date = unified_strdate(

From 20938f768b16c945c6041ba3c0a7ae1a4e790881 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 18 Mar 2016 21:54:33 +0600
Subject: [PATCH 4339/6219] [extractor/generic] Add another test for generic
 m3u8

---
 youtube_dl/extractor/generic.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index cce7799e2..62b51e84e 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -239,6 +239,20 @@ class GenericIE(InfoExtractor):
                 'format': 'bestvideo',
             },
         },
+        # m3u8 served with Content-Type: audio/x-mpegURL; charset=utf-8
+        {
+            'url': 'http://once.unicornmedia.com/now/master/playlist/bb0b18ba-64f5-4b1b-a29f-0ac252f06b68/77a785f3-5188-4806-b788-0893a61634ed/93677179-2d99-4ef4-9e17-fe70d49abfbf/content.m3u8',
+            'info_dict': {
+                'id': 'content',
+                'ext': 'mp4',
+                'title': 'content',
+                'formats': 'mincount:8',
+            },
+            'params': {
+                # m3u8 downloads
+                'skip_download': True,
+            }
+        },
         # google redirect
         {
             'url': 'http://www.google.com/url?sa=t&rct=j&q=&esrc=s&source=web&cd=1&cad=rja&ved=0CCUQtwIwAA&url=http%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DcmQHVoWB5FY&ei=F-sNU-LLCaXk4QT52ICQBQ&usg=AFQjCNEw4hL29zgOohLXvpJ-Bdh2bils1Q&bvm=bv.61965928,d.bGE',

From 303dcdb99505b29ef4c499cc395ab9ec90c07ec1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 18 Mar 2016 22:41:16 +0600
Subject: [PATCH 4340/6219] [extractor/generic] Simplify upload_date extraction

---
 youtube_dl/extractor/generic.py | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 62b51e84e..a2e7ba5ad 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1259,14 +1259,13 @@ class GenericIE(InfoExtractor):
         info_dict = {
             'id': video_id,
             'title': compat_urllib_parse_unquote(os.path.splitext(url_basename(url))[0]),
+            'upload_date': unified_strdate(head_response.headers.get('Last-Modified'))
         }
 
         # Check for direct link to a video
         content_type = head_response.headers.get('Content-Type', '').lower()
         m = re.match(r'^(?P<type>audio|video|application(?=/(?:ogg$|(?:vnd\.apple\.|x-)?mpegurl)))/(?P<format_id>[^;\s]+)', content_type)
         if m:
-            upload_date = unified_strdate(
-                head_response.headers.get('Last-Modified'))
             format_id = m.group('format_id')
             if format_id.endswith('mpegurl'):
                 formats = self._extract_m3u8_formats(url, video_id, 'mp4')
@@ -1281,7 +1280,6 @@ class GenericIE(InfoExtractor):
             info_dict.update({
                 'direct': True,
                 'formats': formats,
-                'upload_date': upload_date,
             })
             return info_dict
 
@@ -1309,12 +1307,9 @@ class GenericIE(InfoExtractor):
         if not is_html(first_bytes):
             self._downloader.report_warning(
                 'URL could be a direct video link, returning it as such.')
-            upload_date = unified_strdate(
-                head_response.headers.get('Last-Modified'))
             info_dict.update({
                 'direct': True,
                 'url': url,
-                'upload_date': upload_date,
             })
             return info_dict
 

From de6c51e88eb61b49a95ccfcfa82547c2172eb52b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 18 Mar 2016 22:43:07 +0600
Subject: [PATCH 4341/6219] [extractor/generic] Fix direct link semantics

---
 youtube_dl/extractor/generic.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index a2e7ba5ad..5649e26da 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1277,10 +1277,8 @@ class GenericIE(InfoExtractor):
                     'url': url,
                     'vcodec': 'none' if m.group('type') == 'audio' else None
                 }]
-            info_dict.update({
-                'direct': True,
-                'formats': formats,
-            })
+                info_dict['direct'] = True
+            info_dict['formats'] = formats
             return info_dict
 
         if not self._downloader.params.get('test', False) and not is_intentional:

From 5940862d5a75ae45a640e0ce3104dd18c9864e26 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 18 Mar 2016 22:45:28 +0600
Subject: [PATCH 4342/6219] [extractor/generic] Detect m3u playlists served
 without proper Content-Type

---
 youtube_dl/extractor/generic.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 5649e26da..24d43a247 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1299,9 +1299,15 @@ class GenericIE(InfoExtractor):
             request.add_header('Accept-Encoding', '*')
             full_response = self._request_webpage(request, video_id)
 
+        first_bytes = full_response.read(512)
+
+        # Is it an M3U playlist?
+        if first_bytes.startswith('#EXTM3U'):
+            info_dict['formats'] = self._extract_m3u8_formats(url, video_id, 'mp4')
+            return info_dict
+
         # Maybe it's a direct link to a video?
         # Be careful not to download the whole thing!
-        first_bytes = full_response.read(512)
         if not is_html(first_bytes):
             self._downloader.report_warning(
                 'URL could be a direct video link, returning it as such.')

From edd9b71c2cca7e5a0df8799710d9ad410ec77d29 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 18 Mar 2016 22:49:11 +0600
Subject: [PATCH 4343/6219] [extractor/generic] Add a test for m3u playlist
 served without proper Content-Type

---
 youtube_dl/extractor/generic.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 24d43a247..f28a65d9b 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -253,6 +253,21 @@ class GenericIE(InfoExtractor):
                 'skip_download': True,
             }
         },
+        # m3u8 served with Content-Type: text/plain
+        {
+            'url': 'http://www.nacentapps.com/m3u8/index.m3u8',
+            'info_dict': {
+                'id': 'index',
+                'ext': 'mp4',
+                'title': 'index',
+                'upload_date': '20140720',
+                'formats': 'mincount:11',
+            },
+            'params': {
+                # m3u8 downloads
+                'skip_download': True,
+            }
+        },
         # google redirect
         {
             'url': 'http://www.google.com/url?sa=t&rct=j&q=&esrc=s&source=web&cd=1&cad=rja&ved=0CCUQtwIwAA&url=http%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DcmQHVoWB5FY&ei=F-sNU-LLCaXk4QT52ICQBQ&usg=AFQjCNEw4hL29zgOohLXvpJ-Bdh2bils1Q&bvm=bv.61965928,d.bGE',

From 19e2617a6fb614a84340757dacb2ea918c097a84 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 18 Mar 2016 23:42:15 +0600
Subject: [PATCH 4344/6219] [commonprotocols] Add generic support for rtmp URLs
 (Closes #8488)

---
 youtube_dl/extractor/__init__.py        |  1 +
 youtube_dl/extractor/commonprotocols.py | 36 +++++++++++++++++++++++++
 2 files changed, 37 insertions(+)
 create mode 100644 youtube_dl/extractor/commonprotocols.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 725ebec04..acc0b03bd 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -136,6 +136,7 @@ from .collegerama import CollegeRamaIE
 from .comedycentral import ComedyCentralIE, ComedyCentralShowsIE
 from .comcarcoff import ComCarCoffIE
 from .commonmistakes import CommonMistakesIE, UnicodeBOMIE
+from .commonprotocols import RtmpIE
 from .condenast import CondeNastIE
 from .cracked import CrackedIE
 from .crackle import CrackleIE
diff --git a/youtube_dl/extractor/commonprotocols.py b/youtube_dl/extractor/commonprotocols.py
new file mode 100644
index 000000000..5d130a170
--- /dev/null
+++ b/youtube_dl/extractor/commonprotocols.py
@@ -0,0 +1,36 @@
+from __future__ import unicode_literals
+
+import os
+
+from .common import InfoExtractor
+from ..compat import (
+    compat_urllib_parse_unquote,
+    compat_urlparse,
+)
+from ..utils import url_basename
+
+
+class RtmpIE(InfoExtractor):
+    IE_DESC = False  # Do not list
+    _VALID_URL = r'(?i)rtmp[est]?://.+'
+
+    _TESTS = [{
+        'url': 'rtmp://cp44293.edgefcs.net/ondemand?auth=daEcTdydfdqcsb8cZcDbAaCbhamacbbawaS-bw7dBb-bWG-GqpGFqCpNCnGoyL&aifp=v001&slist=public/unsecure/audio/2c97899446428e4301471a8cb72b4b97--audio--pmg-20110908-0900a_flv_aac_med_int.mp4',
+        'only_matching': True,
+    }, {
+        'url': 'rtmp://edge.live.hitbox.tv/live/dimak',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = compat_urllib_parse_unquote(os.path.splitext(url.rstrip('/').split('/')[-1])[0])
+        title = compat_urllib_parse_unquote(os.path.splitext(url_basename(url))[0])
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': [{
+                'url': url,
+                'ext': 'flv',
+                'format_id': compat_urlparse.urlparse(url).scheme,
+            }],
+        }

From d5aacf9a90e0855976401b6085ac56b66ca09d12 Mon Sep 17 00:00:00 2001
From: John Peel <john@dgby.org>
Date: Fri, 18 Mar 2016 00:33:03 -0700
Subject: [PATCH 4345/6219] Added format_id to the filers on -f.

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 8c651cd52..93b6ca54d 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -905,7 +905,7 @@ class YoutubeDL(object):
                 '*=': lambda attr, value: value in attr,
             }
             str_operator_rex = re.compile(r'''(?x)
-                \s*(?P<key>ext|acodec|vcodec|container|protocol)
+                \s*(?P<key>ext|acodec|vcodec|container|protocol|format_id)
                 \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?
                 \s*(?P<value>[a-zA-Z0-9._-]+)
                 \s*$

From 4c3b16d5d1bf4806693d2895928ac1b03585b2b2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 19 Mar 2016 00:04:26 +0600
Subject: [PATCH 4346/6219] [test_YoutubeDL] Add test for format_id format
 selection

---
 test/test_YoutubeDL.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index efbee3b71..ca25025e2 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -222,6 +222,11 @@ class TestFormatSelection(unittest.TestCase):
         downloaded = ydl.downloaded_info_dicts[0]
         self.assertEqual(downloaded['format_id'], 'dash-video-low')
 
+        ydl = YDL({'format': 'bestvideo[format_id^=dash][format_id$=low]'})
+        ydl.process_ie_result(info_dict.copy())
+        downloaded = ydl.downloaded_info_dicts[0]
+        self.assertEqual(downloaded['format_id'], 'dash-video-low')
+
         formats = [
             {'format_id': 'vid-vcodec-dot', 'ext': 'mp4', 'preference': 1, 'vcodec': 'avc1.123456', 'acodec': 'none', 'url': TEST_URL},
         ]

From 09fc33198a4cfc93a98ce1ba7d51d41c487e5f56 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 18 Mar 2016 19:18:55 +0100
Subject: [PATCH 4347/6219] utils: lookup_unit_table: Use a stricter regex

In parse_count multiple units start with the same letter, so it would match different units depending on the order they were sorted when iterating over them.
---
 youtube_dl/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index ef6e7c7cb..bad1c4ea8 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1346,7 +1346,7 @@ def format_bytes(bytes):
 def lookup_unit_table(unit_table, s):
     units_re = '|'.join(re.escape(u) for u in unit_table)
     m = re.match(
-        r'(?P<num>[0-9]+(?:[,.][0-9]*)?)\s*(?P<unit>%s)' % units_re, s)
+        r'(?P<num>[0-9]+(?:[,.][0-9]*)?)\s*(?P<unit>%s)$' % units_re, s)
     if not m:
         return None
     num_str = m.group('num').replace(',', '.')

From 4cd70099ea79a4a82b26694937ca46d31f7436ca Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 18 Mar 2016 21:17:45 +0100
Subject: [PATCH 4348/6219] [hbo] Add new extractor

---
 youtube_dl/extractor/__init__.py |   1 +
 youtube_dl/extractor/hbo.py      | 122 +++++++++++++++++++++++++++++++
 2 files changed, 123 insertions(+)
 create mode 100644 youtube_dl/extractor/hbo.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index acc0b03bd..529051a93 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -284,6 +284,7 @@ from .goshgay import GoshgayIE
 from .gputechconf import GPUTechConfIE
 from .groupon import GrouponIE
 from .hark import HarkIE
+from .hbo import HBOIE
 from .hearthisat import HearThisAtIE
 from .heise import HeiseIE
 from .hellporno import HellPornoIE
diff --git a/youtube_dl/extractor/hbo.py b/youtube_dl/extractor/hbo.py
new file mode 100644
index 000000000..dad0f3994
--- /dev/null
+++ b/youtube_dl/extractor/hbo.py
@@ -0,0 +1,122 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    xpath_text,
+    xpath_element,
+    int_or_none,
+    parse_duration,
+)
+
+
+class HBOIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?hbo\.com/video/video\.html\?.*vid=(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'http://www.hbo.com/video/video.html?autoplay=true&g=u&vid=1437839',
+        'md5': '1c33253f0c7782142c993c0ba62a8753',
+        'info_dict': {
+            'id': '1437839',
+            'ext': 'mp4',
+            'title': 'Ep. 64 Clip: Encryption',
+        }
+    }
+    _FORMATS_INFO = {
+        '1920': {
+            'width': 1280,
+            'height': 720,
+        },
+        '640': {
+            'width': 768,
+            'height': 432,
+        },
+        'highwifi': {
+            'width': 640,
+            'height': 360,
+        },
+        'high3g': {
+            'width': 640,
+            'height': 360,
+        },
+        'medwifi': {
+            'width': 400,
+            'height': 224,
+        },
+        'med3g': {
+            'width': 400,
+            'height': 224,
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        video_data = self._download_xml(
+            'http://render.lv3.hbo.com/data/content/global/videos/data/%s.xml' % video_id, video_id)
+        title = xpath_text(video_data, 'title', 'title', True)
+
+        formats = []
+        for source in xpath_element(video_data, 'videos', 'sources', True):
+            if source.tag == 'size':
+                path = xpath_text(source, './/path')
+                if not path:
+                    continue
+                width = source.attrib.get('width')
+                format_info = self._FORMATS_INFO.get(width, {})
+                height = format_info.get('height')
+                fmt = {
+                    'url': path,
+                    'format_id': 'http%s' % ('-%dp' % height if height else ''),
+                    'width': format_info.get('width'),
+                    'height': height,
+                }
+                rtmp = re.search(r'^(?P<url>rtmpe?://[^/]+/(?P<app>.+))/(?P<playpath>mp4:.+)$', path)
+                if rtmp:
+                    fmt.update({
+                        'url': rtmp.group('url'),
+                        'play_path': rtmp.group('playpath'),
+                        'app': rtmp.group('app'),
+                        'ext': 'flv',
+                        'format_id': fmt['format_id'].replace('http', 'rtmp'),
+                    })
+                formats.append(fmt)
+            else:
+                video_url = source.text
+                if not video_url:
+                    continue
+                if source.tag == 'tarball':
+                    formats.extend(self._extract_m3u8_formats(
+                        video_url.replace('.tar', '/base_index_w8.m3u8'),
+                        video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
+                else:
+                    format_info = self._FORMATS_INFO.get(source.tag, {})
+                    formats.append({
+                        'format_id': 'http-%s' % source.tag,
+                        'url': video_url,
+                        'width': format_info.get('width'),
+                        'height': format_info.get('height'),
+                    })
+        self._sort_formats(formats, ('width', 'height', 'tbr', 'format_id'))
+
+        thumbnails = []
+        card_sizes = xpath_element(video_data, 'titleCardSizes')
+        if card_sizes is not None:
+            for size in card_sizes:
+                path = xpath_text(size, 'path')
+                if not path:
+                    continue
+                width = int_or_none(size.get('width'))
+                thumbnails.append({
+                    'id': width,
+                    'url': path,
+                    'width': width,
+                })
+
+        return {
+            'id': video_id,
+            'title': title,
+            'duration': parse_duration(xpath_element(video_data, 'duration/tv14')),
+            'formats': formats,
+            'thumbnails': thumbnails,
+        }

From 0d769bcb781b46a00ddf958d6ea945560f2d6cd5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 19 Mar 2016 05:43:43 +0600
Subject: [PATCH 4349/6219] [extractor/generic] Fix missing byte literal prefix

---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index f28a65d9b..26de27a7e 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1317,7 +1317,7 @@ class GenericIE(InfoExtractor):
         first_bytes = full_response.read(512)
 
         # Is it an M3U playlist?
-        if first_bytes.startswith('#EXTM3U'):
+        if first_bytes.startswith(b'#EXTM3U'):
             info_dict['formats'] = self._extract_m3u8_formats(url, video_id, 'mp4')
             return info_dict
 

From 782b1b5bd1cdaaead6865dee5d300486e7dd8348 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 19 Mar 2016 11:42:35 +0100
Subject: [PATCH 4350/6219] [utils] lookup_unit_table: Match word boundary
 instead of end of string

---
 test/test_utils.py  | 2 ++
 youtube_dl/utils.py | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 9a3a8ddff..325b870cc 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -702,6 +702,8 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(parse_count('1.000'), 1000)
         self.assertEqual(parse_count('1.1k'), 1100)
         self.assertEqual(parse_count('1.1kk'), 1100000)
+        self.assertEqual(parse_count('1.1kk '), 1100000)
+        self.assertEqual(parse_count('1.1kk views'), 1100000)
 
     def test_version_tuple(self):
         self.assertEqual(version_tuple('1'), (1,))
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index bad1c4ea8..067b8a184 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1346,7 +1346,7 @@ def format_bytes(bytes):
 def lookup_unit_table(unit_table, s):
     units_re = '|'.join(re.escape(u) for u in unit_table)
     m = re.match(
-        r'(?P<num>[0-9]+(?:[,.][0-9]*)?)\s*(?P<unit>%s)$' % units_re, s)
+        r'(?P<num>[0-9]+(?:[,.][0-9]*)?)\s*(?P<unit>%s)\b' % units_re, s)
     if not m:
         return None
     num_str = m.group('num').replace(',', '.')

From 52bb437e412726a37d585cf782c88bc8c8a042a4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 19 Mar 2016 20:40:36 +0600
Subject: [PATCH 4351/6219] [options] Add --fragment-retries option

---
 youtube_dl/__init__.py | 17 ++++++++++++-----
 youtube_dl/options.py  |  4 ++++
 2 files changed, 16 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 79b389840..737f6545d 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -144,14 +144,20 @@ def _real_main(argv=None):
         if numeric_limit is None:
             parser.error('invalid max_filesize specified')
         opts.max_filesize = numeric_limit
-    if opts.retries is not None:
-        if opts.retries in ('inf', 'infinite'):
-            opts_retries = float('inf')
+
+    def parse_retries(retries):
+        if retries in ('inf', 'infinite'):
+            parsed_retries = float('inf')
         else:
             try:
-                opts_retries = int(opts.retries)
+                parsed_retries = int(retries)
             except (TypeError, ValueError):
                 parser.error('invalid retry count specified')
+        return parsed_retries
+    if opts.retries is not None:
+        opts.retries = parse_retries(opts.retries)
+    if opts.fragment_retries is not None:
+        opts.fragment_retries = parse_retries(opts.fragment_retries)
     if opts.buffersize is not None:
         numeric_buffersize = FileDownloader.parse_bytes(opts.buffersize)
         if numeric_buffersize is None:
@@ -299,7 +305,8 @@ def _real_main(argv=None):
         'force_generic_extractor': opts.force_generic_extractor,
         'ratelimit': opts.ratelimit,
         'nooverwrites': opts.nooverwrites,
-        'retries': opts_retries,
+        'retries': opts.retries,
+        'fragment_retries': opts.fragment_retries,
         'buffersize': opts.buffersize,
         'noresizebuffer': opts.noresizebuffer,
         'continuedl': opts.continue_dl,
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 9dd7a8034..822728afc 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -399,6 +399,10 @@ def parseOpts(overrideArguments=None):
         '-R', '--retries',
         dest='retries', metavar='RETRIES', default=10,
         help='Number of retries (default is %default), or "infinite".')
+    downloader.add_option(
+        '--fragment-retries',
+        dest='fragment_retries', metavar='RETRIES', default=10,
+        help='Number of retries for a fragment (default is %default), or "infinite" (DASH only)')
     downloader.add_option(
         '--buffer-size',
         dest='buffersize', metavar='SIZE', default='1024',

From 721f26b8211a22648523f51c80f9b81d1eaa3b32 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 19 Mar 2016 20:41:24 +0600
Subject: [PATCH 4352/6219] [downloader/fragment] Add report_retry_fragment

---
 youtube_dl/downloader/fragment.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
index a5bae9669..df66c35f0 100644
--- a/youtube_dl/downloader/fragment.py
+++ b/youtube_dl/downloader/fragment.py
@@ -21,6 +21,11 @@ class FragmentFD(FileDownloader):
     A base file downloader class for fragmented media (e.g. f4m/m3u8 manifests).
     """
 
+    def report_retry_fragment(self, fragment_name, count, retries):
+        self.to_screen(
+            '[download] Got server HTTP error. Retrying fragment %s (attempt %d of %.0f)...'
+            % (fragment_name, count, retries))
+
     def _prepare_and_start_frag_download(self, ctx):
         self._prepare_frag_download(ctx)
         self._start_frag_download(ctx)

From e33baba0dd6584475f75badec2186a7d86b88a5d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 19 Mar 2016 20:42:23 +0600
Subject: [PATCH 4353/6219] [downloader/dash] Add fragment retry capability

YouTube may often return 404 HTTP error for a fragment causing the
whole download to fail. However if the same fragment is immediately
retried with the same request data this usually succeeds (1-2 attemps
is usually enough) thus allowing to download the whole file successfully.
So, we will retry all fragments that fail with 404 HTTP error for now.
---
 youtube_dl/downloader/dash.py | 42 ++++++++++++++++++++++++++---------
 1 file changed, 32 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/downloader/dash.py b/youtube_dl/downloader/dash.py
index 8b1b17c6e..8bbab9dbc 100644
--- a/youtube_dl/downloader/dash.py
+++ b/youtube_dl/downloader/dash.py
@@ -4,6 +4,7 @@ import os
 import re
 
 from .fragment import FragmentFD
+from ..compat import compat_urllib_error
 from ..utils import (
     sanitize_open,
     encodeFilename,
@@ -36,20 +37,41 @@ class DashSegmentsFD(FragmentFD):
 
         segments_filenames = []
 
-        def append_url_to_file(target_url, target_filename):
-            success = ctx['dl'].download(target_filename, {'url': combine_url(base_url, target_url)})
-            if not success:
+        fragment_retries = self.params.get('fragment_retries', 0)
+
+        def append_url_to_file(target_url, tmp_filename, segment_name):
+            target_filename = '%s-%s' % (tmp_filename, segment_name)
+            count = 0
+            while count <= fragment_retries:
+                try:
+                    success = ctx['dl'].download(target_filename, {'url': combine_url(base_url, target_url)})
+                    if not success:
+                        return False
+                    down, target_sanitized = sanitize_open(target_filename, 'rb')
+                    ctx['dest_stream'].write(down.read())
+                    down.close()
+                    segments_filenames.append(target_sanitized)
+                    break
+                except (compat_urllib_error.HTTPError, ) as err:
+                    # YouTube may often return 404 HTTP error for a fragment causing the
+                    # whole download to fail. However if the same fragment is immediately
+                    # retried with the same request data this usually succeeds (1-2 attemps
+                    # is usually enough) thus allowing to download the whole file successfully.
+                    # So, we will retry all fragments that fail with 404 HTTP error for now.
+                    if err.code != 404:
+                        raise
+                    # Retry fragment
+                    count += 1
+                    if count <= fragment_retries:
+                        self.report_retry_fragment(segment_name, count, fragment_retries)
+            if count > fragment_retries:
+                self.report_error('giving up after %s fragment retries' % fragment_retries)
                 return False
-            down, target_sanitized = sanitize_open(target_filename, 'rb')
-            ctx['dest_stream'].write(down.read())
-            down.close()
-            segments_filenames.append(target_sanitized)
 
         if initialization_url:
-            append_url_to_file(initialization_url, ctx['tmpfilename'] + '-Init')
+            append_url_to_file(initialization_url, ctx['tmpfilename'], 'Init')
         for i, segment_url in enumerate(segment_urls):
-            segment_filename = '%s-Seg%d' % (ctx['tmpfilename'], i)
-            append_url_to_file(segment_url, segment_filename)
+            append_url_to_file(segment_url, ctx['tmpfilename'], 'Seg%d' % i)
 
         self._finish_frag_download(ctx)
 

From 617e58d85063b68fb9736355e8354b05e82b1147 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 19 Mar 2016 20:51:30 +0600
Subject: [PATCH 4354/6219] [downloader/{common,fragment}] Fix total retries
 reporting on python 2.6

---
 youtube_dl/downloader/common.py   | 8 +++++++-
 youtube_dl/downloader/fragment.py | 4 ++--
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index f39db58f6..1dba9f49a 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -115,6 +115,10 @@ class FileDownloader(object):
             return '%10s' % '---b/s'
         return '%10s' % ('%s/s' % format_bytes(speed))
 
+    @staticmethod
+    def format_retries(retries):
+        return 'inf' if retries == float('inf') else '%.0f' % retries
+
     @staticmethod
     def best_block_size(elapsed_time, bytes):
         new_min = max(bytes / 2.0, 1.0)
@@ -297,7 +301,9 @@ class FileDownloader(object):
 
     def report_retry(self, count, retries):
         """Report retry in case of HTTP error 5xx"""
-        self.to_screen('[download] Got server HTTP error. Retrying (attempt %d of %.0f)...' % (count, retries))
+        self.to_screen(
+            '[download] Got server HTTP error. Retrying (attempt %d of %s)...'
+            % (count, self.format_retries(retries)))
 
     def report_file_already_downloaded(self, file_name):
         """Report file has already been fully downloaded."""
diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
index df66c35f0..c2671e6d2 100644
--- a/youtube_dl/downloader/fragment.py
+++ b/youtube_dl/downloader/fragment.py
@@ -23,8 +23,8 @@ class FragmentFD(FileDownloader):
 
     def report_retry_fragment(self, fragment_name, count, retries):
         self.to_screen(
-            '[download] Got server HTTP error. Retrying fragment %s (attempt %d of %.0f)...'
-            % (fragment_name, count, retries))
+            '[download] Got server HTTP error. Retrying fragment %s (attempt %d of %s)...'
+            % (fragment_name, count, self.format_retries(retries)))
 
     def _prepare_and_start_frag_download(self, ctx):
         self._prepare_frag_download(ctx)

From 16a8b7986b88572aea12c0f80c499e6e8085f1cc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 19 Mar 2016 20:54:21 +0600
Subject: [PATCH 4355/6219] [downloader/fragment] Document fragment_retries

---
 youtube_dl/downloader/fragment.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
index c2671e6d2..ba903ae10 100644
--- a/youtube_dl/downloader/fragment.py
+++ b/youtube_dl/downloader/fragment.py
@@ -19,6 +19,10 @@ class HttpQuietDownloader(HttpFD):
 class FragmentFD(FileDownloader):
     """
     A base file downloader class for fragmented media (e.g. f4m/m3u8 manifests).
+
+    Available options:
+
+    fragment_retries:   Number of times to retry a fragment for HTTP error (DASH only)
     """
 
     def report_retry_fragment(self, fragment_name, count, retries):

From 3aec71766da38478740437c901514e666a39dbb0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 19 Mar 2016 22:30:48 +0600
Subject: [PATCH 4356/6219] [safari:api] Separate extractor (Closes #8871)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/safari.py   | 52 +++++++++++++++++++-------------
 2 files changed, 32 insertions(+), 21 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 529051a93..b3bc38916 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -628,6 +628,7 @@ from .ruutu import RuutuIE
 from .sandia import SandiaIE
 from .safari import (
     SafariIE,
+    SafariApiIE,
     SafariCourseIE,
 )
 from .sapo import SapoIE
diff --git a/youtube_dl/extractor/safari.py b/youtube_dl/extractor/safari.py
index 256396bb8..6ba91f202 100644
--- a/youtube_dl/extractor/safari.py
+++ b/youtube_dl/extractor/safari.py
@@ -75,16 +75,7 @@ class SafariBaseIE(InfoExtractor):
 class SafariIE(SafariBaseIE):
     IE_NAME = 'safari'
     IE_DESC = 'safaribooksonline.com online video'
-    _VALID_URL = r'''(?x)https?://
-                            (?:www\.)?safaribooksonline\.com/
-                                (?:
-                                    library/view/[^/]+|
-                                    api/v1/book
-                                )/
-                                (?P<course_id>[^/]+)/
-                                    (?:chapter(?:-content)?/)?
-                                (?P<part>part\d+)\.html
-    '''
+    _VALID_URL = r'https?://(?:www\.)?safaribooksonline\.com/library/view/[^/]+/(?P<course_id>[^/]+)/(?P<part>part\d+)\.html'
 
     _TESTS = [{
         'url': 'https://www.safaribooksonline.com/library/view/hadoop-fundamentals-livelessons/9780133392838/part00.html',
@@ -97,9 +88,6 @@ class SafariIE(SafariBaseIE):
             'upload_date': '20150724',
             'uploader_id': 'stork',
         },
-    }, {
-        'url': 'https://www.safaribooksonline.com/api/v1/book/9780133392838/chapter/part00.html',
-        'only_matching': True,
     }, {
         # non-digits in course id
         'url': 'https://www.safaribooksonline.com/library/view/create-a-nodejs/100000006A0210/part00.html',
@@ -108,13 +96,18 @@ class SafariIE(SafariBaseIE):
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        course_id = mobj.group('course_id')
-        part = mobj.group('part')
+        video_id = '%s/%s' % (mobj.group('course_id'), mobj.group('part'))
 
-        webpage = self._download_webpage(url, '%s/%s' % (course_id, part))
-        reference_id = self._search_regex(r'data-reference-id="([^"]+)"', webpage, 'kaltura reference id')
-        partner_id = self._search_regex(r'data-partner-id="([^"]+)"', webpage, 'kaltura widget id')
-        ui_id = self._search_regex(r'data-ui-id="([^"]+)"', webpage, 'kaltura uiconf id')
+        webpage = self._download_webpage(url, video_id)
+        reference_id = self._search_regex(
+            r'data-reference-id=(["\'])(?P<id>.+?)\1',
+            webpage, 'kaltura reference id', group='id')
+        partner_id = self._search_regex(
+            r'data-partner-id=(["\'])(?P<id>.+?)\1',
+            webpage, 'kaltura widget id', group='id')
+        ui_id = self._search_regex(
+            r'data-ui-id=(["\'])(?P<id>.+?)\1',
+            webpage, 'kaltura uiconf id', group='id')
 
         query = {
             'wid': '_%s' % partner_id,
@@ -125,7 +118,7 @@ class SafariIE(SafariBaseIE):
         if self.LOGGED_IN:
             kaltura_session = self._download_json(
                 '%s/player/kaltura_session/?reference_id=%s' % (self._API_BASE, reference_id),
-                course_id, 'Downloading kaltura session JSON',
+                video_id, 'Downloading kaltura session JSON',
                 'Unable to download kaltura session JSON', fatal=False)
             if kaltura_session:
                 session = kaltura_session.get('session')
@@ -137,6 +130,23 @@ class SafariIE(SafariBaseIE):
             'Kaltura')
 
 
+class SafariApiIE(SafariBaseIE):
+    IE_NAME = 'safari:api'
+    _VALID_URL = r'https?://(?:www\.)?safaribooksonline\.com/api/v1/book/(?P<course_id>[^/]+)/chapter(?:-content)?/(?P<part>part\d+)\.html'
+
+    _TEST = {
+        'url': 'https://www.safaribooksonline.com/api/v1/book/9780133392838/chapter/part00.html',
+        'only_matching': True,
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        part = self._download_json(
+            url, '%s/%s' % (mobj.group('course_id'), mobj.group('part')),
+            'Downloading part JSON')
+        return self.url_result(part['web_url'], SafariIE.ie_key())
+
+
 class SafariCourseIE(SafariBaseIE):
     IE_NAME = 'safari:course'
     IE_DESC = 'safaribooksonline.com online courses'
@@ -168,7 +178,7 @@ class SafariCourseIE(SafariBaseIE):
                 'No chapters found for course %s' % course_id, expected=True)
 
         entries = [
-            self.url_result(chapter, 'Safari')
+            self.url_result(chapter, SafariApiIE.ie_key())
             for chapter in course_json['chapters']]
 
         course_title = course_json['title']

From 8b0d7a66ef5451556bb8ae5b085c7bef4c992f8b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kacper=20Michaj=C5=82ow?= <kasper93@gmail.com>
Date: Wed, 9 Mar 2016 20:55:27 +0100
Subject: [PATCH 4357/6219] [cda] Add new extractor for cda.pl

Fixes #8760
---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/cda.py      | 96 ++++++++++++++++++++++++++++++++
 2 files changed, 97 insertions(+)
 create mode 100755 youtube_dl/extractor/cda.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index b3bc38916..5f5eca42b 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -108,6 +108,7 @@ from .cbsnews import (
 )
 from .cbssports import CBSSportsIE
 from .ccc import CCCIE
+from .cda import CDAIE
 from .ceskatelevize import CeskaTelevizeIE
 from .channel9 import Channel9IE
 from .chaturbate import ChaturbateIE
diff --git a/youtube_dl/extractor/cda.py b/youtube_dl/extractor/cda.py
new file mode 100755
index 000000000..4c53b8dda
--- /dev/null
+++ b/youtube_dl/extractor/cda.py
@@ -0,0 +1,96 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    decode_packed_codes,
+    ExtractorError,
+    parse_duration
+)
+
+
+class CDAIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:(?:www|ebd)\.)?cda\.pl/(?:video|[0-9]+x[0-9]+)/(?P<id>[0-9a-z]+)'
+    _TESTS = [
+        {
+            'url': 'http://www.cda.pl/video/5749950c',
+            'md5': '6f844bf51b15f31fae165365707ae970',
+            'info_dict': {
+                'id': '5749950c',
+                'ext': 'mp4',
+                'height': 720,
+                'title': 'Oto dlaczego przed zakrętem należy zwolnić.',
+                'duration': 39
+            }
+        },
+        {
+            'url': 'http://www.cda.pl/video/57413289',
+            'md5': 'a88828770a8310fc00be6c95faf7f4d5',
+            'info_dict': {
+                'id': '57413289',
+                'ext': 'mp4',
+                'title': 'Lądowanie na lotnisku na Maderze',
+                'duration': 137
+            }
+        }
+    ]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage('http://ebd.cda.pl/0x0/' + video_id, video_id)
+
+        if 'Ten film jest dostępny dla użytkowników premium' in webpage:
+            raise ExtractorError('This video is only available for premium users.', expected=True)
+
+        title = self._html_search_regex(r'<title>(.+?)</title>', webpage, 'title', fatal=False)
+
+        def _get_format(page, version=''):
+            unpacked = decode_packed_codes(page)
+            duration = self._search_regex(r"duration:\\'(.+?)\\'", unpacked, 'duration', fatal=False)
+            format_id = None
+            height = None
+
+            m = re.search(r'<a data-quality="(?P<format_id>[^"]+)" href="[^"]+" class="quality-btn quality-btn-active">(?P<height>[0-9]+)p<\/a>', page)
+            if m:
+                format_id = m.group('format_id')
+                height = int(m.group('height'))
+
+            url = self._search_regex(r"url:\\'(.+?)\\'", unpacked, version + ' url', fatal=False)
+            if url is None:
+                return None
+
+            return {
+                'format_id': format_id,
+                'height': height,
+                'url': url
+            }, parse_duration(duration)
+
+        formats = []
+
+        format_desc, duration = _get_format(webpage) or (None, None)
+        if format_desc is not None:
+            formats.append(format_desc)
+
+        pattern = re.compile(r'<a data-quality="[^"]+" href="([^"]+)" class="quality-btn">([0-9]+p)<\/a>')
+        for version in re.findall(pattern, webpage):
+            webpage = self._download_webpage(version[0], video_id, 'Downloading %s version information' % version[1], fatal=False)
+            if not webpage:
+                # Manually report warning because empty page is returned when invalid version is requested.
+                self.report_warning('Unable to download %s version information' % version[1])
+                continue
+
+            format_desc, duration_ = _get_format(webpage, version[1]) or (None, None)
+            duration = duration or duration_
+            if format_desc is not None:
+                formats.append(format_desc)
+
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': formats,
+            'duration': duration
+        }

From f1ced6df51e4d81523e9051cadb6e4f5ceac19f6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 19 Mar 2016 23:17:14 +0600
Subject: [PATCH 4358/6219] [cda] Improve and simplify (Closes #8805)

---
 youtube_dl/extractor/cda.py | 132 ++++++++++++++++++------------------
 1 file changed, 66 insertions(+), 66 deletions(-)

diff --git a/youtube_dl/extractor/cda.py b/youtube_dl/extractor/cda.py
index 4c53b8dda..498d2c0d8 100755
--- a/youtube_dl/extractor/cda.py
+++ b/youtube_dl/extractor/cda.py
@@ -12,30 +12,30 @@ from ..utils import (
 
 
 class CDAIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:(?:www|ebd)\.)?cda\.pl/(?:video|[0-9]+x[0-9]+)/(?P<id>[0-9a-z]+)'
-    _TESTS = [
-        {
-            'url': 'http://www.cda.pl/video/5749950c',
-            'md5': '6f844bf51b15f31fae165365707ae970',
-            'info_dict': {
-                'id': '5749950c',
-                'ext': 'mp4',
-                'height': 720,
-                'title': 'Oto dlaczego przed zakrętem należy zwolnić.',
-                'duration': 39
-            }
-        },
-        {
-            'url': 'http://www.cda.pl/video/57413289',
-            'md5': 'a88828770a8310fc00be6c95faf7f4d5',
-            'info_dict': {
-                'id': '57413289',
-                'ext': 'mp4',
-                'title': 'Lądowanie na lotnisku na Maderze',
-                'duration': 137
-            }
+    _VALID_URL = r'https?://(?:(?:www\.)?cda\.pl/video|ebd\.cda\.pl/[0-9]+x[0-9]+)/(?P<id>[0-9a-z]+)'
+    _TESTS = [{
+        'url': 'http://www.cda.pl/video/5749950c',
+        'md5': '6f844bf51b15f31fae165365707ae970',
+        'info_dict': {
+            'id': '5749950c',
+            'ext': 'mp4',
+            'height': 720,
+            'title': 'Oto dlaczego przed zakrętem należy zwolnić.',
+            'duration': 39
         }
-    ]
+    }, {
+        'url': 'http://www.cda.pl/video/57413289',
+        'md5': 'a88828770a8310fc00be6c95faf7f4d5',
+        'info_dict': {
+            'id': '57413289',
+            'ext': 'mp4',
+            'title': 'Lądowanie na lotnisku na Maderze',
+            'duration': 137
+        }
+    }, {
+        'url': 'http://ebd.cda.pl/0x0/5749950c',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -44,53 +44,53 @@ class CDAIE(InfoExtractor):
         if 'Ten film jest dostępny dla użytkowników premium' in webpage:
             raise ExtractorError('This video is only available for premium users.', expected=True)
 
-        title = self._html_search_regex(r'<title>(.+?)</title>', webpage, 'title', fatal=False)
-
-        def _get_format(page, version=''):
-            unpacked = decode_packed_codes(page)
-            duration = self._search_regex(r"duration:\\'(.+?)\\'", unpacked, 'duration', fatal=False)
-            format_id = None
-            height = None
-
-            m = re.search(r'<a data-quality="(?P<format_id>[^"]+)" href="[^"]+" class="quality-btn quality-btn-active">(?P<height>[0-9]+)p<\/a>', page)
-            if m:
-                format_id = m.group('format_id')
-                height = int(m.group('height'))
-
-            url = self._search_regex(r"url:\\'(.+?)\\'", unpacked, version + ' url', fatal=False)
-            if url is None:
-                return None
-
-            return {
-                'format_id': format_id,
-                'height': height,
-                'url': url
-            }, parse_duration(duration)
+        title = self._html_search_regex(r'<title>(.+?)</title>', webpage, 'title')
 
         formats = []
 
-        format_desc, duration = _get_format(webpage) or (None, None)
-        if format_desc is not None:
-            formats.append(format_desc)
-
-        pattern = re.compile(r'<a data-quality="[^"]+" href="([^"]+)" class="quality-btn">([0-9]+p)<\/a>')
-        for version in re.findall(pattern, webpage):
-            webpage = self._download_webpage(version[0], video_id, 'Downloading %s version information' % version[1], fatal=False)
-            if not webpage:
-                # Manually report warning because empty page is returned when invalid version is requested.
-                self.report_warning('Unable to download %s version information' % version[1])
-                continue
-
-            format_desc, duration_ = _get_format(webpage, version[1]) or (None, None)
-            duration = duration or duration_
-            if format_desc is not None:
-                formats.append(format_desc)
-
-        self._sort_formats(formats)
-
-        return {
+        info_dict = {
             'id': video_id,
             'title': title,
             'formats': formats,
-            'duration': duration
+            'duration': None,
         }
+
+        def extract_format(page, version):
+            unpacked = decode_packed_codes(page)
+            format_url = self._search_regex(
+                r"url:\\'(.+?)\\'", unpacked, '%s url' % version, fatal=False)
+            if not format_url:
+                return
+            f = {
+                'url': format_url,
+            }
+            m = re.search(
+                r'<a[^>]+data-quality="(?P<format_id>[^"]+)"[^>]+href="[^"]+"[^>]+class="[^"]*quality-btn-active[^"]*">(?P<height>[0-9]+)p',
+                page)
+            if m:
+                f.update({
+                    'format_id': m.group('format_id'),
+                    'height': int(m.group('height')),
+                })
+            info_dict['formats'].append(f)
+            if not info_dict['duration']:
+                info_dict['duration'] = parse_duration(self._search_regex(
+                    r"duration:\\'(.+?)\\'", unpacked, 'duration', fatal=False))
+
+        extract_format(webpage, 'default')
+
+        for href, resolution in re.findall(
+                r'<a[^>]+data-quality="[^"]+"[^>]+href="([^"]+)"[^>]+class="quality-btn"[^>]*>([0-9]+p)',
+                webpage):
+            webpage = self._download_webpage(
+                href, video_id, 'Downloading %s version information' % resolution, fatal=False)
+            if not webpage:
+                # Manually report warning because empty page is returned when
+                # invalid version is requested.
+                self.report_warning('Unable to download %s version information' % resolution)
+                continue
+            extract_format(webpage, resolution)
+
+        self._sort_formats(formats)
+
+        return info_dict

From 9261e347ccf63b31bd2035996279b0ad1a45247a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 19 Mar 2016 23:18:04 +0600
Subject: [PATCH 4359/6219] Credit @kasper93 for cda (#8805)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index aa48cd5a6..6c9747913 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -163,3 +163,4 @@ Patrick Griffis
 Aidan Rowe
 mutantmonkey
 Ben Congdon
+Kacper Michajłow

From e36f4aa72b01b3f6a322edc094cdf1c20b071367 Mon Sep 17 00:00:00 2001
From: jjatria <jjatria@gmail.com>
Date: Wed, 28 Oct 2015 18:31:52 +0000
Subject: [PATCH 4360/6219] [biobiotv] Add extractor

---
 youtube_dl/extractor/biobiotv.py | 75 ++++++++++++++++++++++++++++++++
 1 file changed, 75 insertions(+)
 create mode 100644 youtube_dl/extractor/biobiotv.py

diff --git a/youtube_dl/extractor/biobiotv.py b/youtube_dl/extractor/biobiotv.py
new file mode 100644
index 000000000..aae0588ef
--- /dev/null
+++ b/youtube_dl/extractor/biobiotv.py
@@ -0,0 +1,75 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+
+class BioBioTVIE(InfoExtractor):
+    _VALID_URL = r'https?://tv\.biobiochile\.cl/notas/(?P<year>\d{4})/\d{2}/\d{2}/(?P<id>[\w-]+)(?:\.shtml)?'
+
+    _TESTS = [{
+        'url': 'http://tv.biobiochile.cl/notas/2015/10/21/sobre-camaras-y-camarillas-parlamentarias.shtml',
+        'md5': '26f51f03cf580265defefb4518faec09',
+        'info_dict': {
+            'id': 'col_c266',
+            'display_id': 'sobre-camaras-y-camarillas-parlamentarias',
+            'ext': 'mp4',
+            'title': 'Sobre Cámaras y camarillas parlamentarias - BioBioChile TV',
+            'thumbnail': 'http://media.biobiochile.cl/wp-content/uploads/2015/10/atria-2010-730x350.jpg',
+            'url': 'http://unlimited2-cl.digitalproserver.com/bbtv/2015/col_c266.mp4',
+            'uploader': 'Fernando Atria',
+        }
+    }, {
+        'url': 'http://tv.biobiochile.cl/notas/2015/10/22/ninos-transexuales-de-quien-es-la-decision.shtml',
+        'md5': 'a8c868e6b5f6c17d56873d5633204f84',
+        'info_dict': {
+            'id': 'col_c270',
+            'display_id': 'ninos-transexuales-de-quien-es-la-decision',
+            'ext': 'mp4',
+            'title': 'Niños transexuales: ¿De quién es la decisión? - BioBioChile TV',
+            'thumbnail': 'http://media.biobiochile.cl/wp-content/uploads/2015/10/samantha-2210-730x350.jpg',
+            'url': 'http://unlimited2-cl.digitalproserver.com/bbtv/2015/col_c270.mp4',
+            'uploader': 'Samantha Morán',
+        }
+    }, {
+        'url': 'http://tv.biobiochile.cl/notas/2015/10/21/exclusivo-hector-pinto-formador-de-chupete-revela-version-del-ex-delantero-albo.shtml',
+        'md5': 'c8369b50d42ff0a4f6b969fbd1a7c32d',
+        'info_dict': {
+            'id': 'Keno_Pinto',
+            'display_id': 'exclusivo-hector-pinto-formador-de-chupete-revela-version-del-ex-delantero-albo',
+            'ext': 'mp4',
+            'title': 'Exclusivo: Héctor Pinto, formador de “Chupete”, revela versión del ex delantero albo - BioBioChile TV',
+            'thumbnail': 'http://media.biobiochile.cl/wp-content/uploads/2015/10/pinto-730x350.jpg',
+            'url': 'http://unlimited2-cl.digitalproserver.com/bbtv/2015/Keno_Pinto.mp4',
+            'uploader': 'Juan Pablo Echenique',
+        }
+    }]
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        display_id = mobj.group('id')
+        year = mobj.group('year')
+
+        webpage = self._download_webpage(url, display_id)
+
+        title = self._html_search_meta(
+            'og:title', webpage, 'title', fatal=True)
+
+        thumbnail = self._html_search_meta(
+            'og:image', webpage, 'thumbnail', fatal=True)
+
+        video_id = self._html_search_regex(
+            r'loadFWPlayerVideo\(\"player_0\", \"\d{4}/(.+)\.mp4\"\)', webpage, 'title')
+
+        url = 'http://unlimited2-cl.digitalproserver.com/bbtv/' + year + '/' + video_id + '.mp4'
+
+        return {
+            'id': video_id,
+            'title': title,
+            'url': url,
+            'display_id': display_id,
+            'thumbnail': thumbnail,
+            'uploader': self._search_regex(r'biobiochile\.cl/author[^"]+"[^>]*>([^<]+)<', webpage, 'uploader', fatal=False),
+        }

From fa023ccb2c00f393c78ae4cbbabec7a8ec7b3ac6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 20 Mar 2016 01:31:55 +0600
Subject: [PATCH 4361/6219] [biobiochiletv] Fix extraction, extract m3u8
 formats and overall improve (Closes #7314)

---
 youtube_dl/extractor/__init__.py      |  1 +
 youtube_dl/extractor/biobiochiletv.py | 86 +++++++++++++++++++++++++++
 youtube_dl/extractor/biobiotv.py      | 75 -----------------------
 3 files changed, 87 insertions(+), 75 deletions(-)
 create mode 100644 youtube_dl/extractor/biobiochiletv.py
 delete mode 100644 youtube_dl/extractor/biobiotv.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 5f5eca42b..b773edb3d 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -72,6 +72,7 @@ from .bet import BetIE
 from .bigflix import BigflixIE
 from .bild import BildIE
 from .bilibili import BiliBiliIE
+from .biobiochiletv import BioBioChileTVIE
 from .bleacherreport import (
     BleacherReportIE,
     BleacherReportCMSIE,
diff --git a/youtube_dl/extractor/biobiochiletv.py b/youtube_dl/extractor/biobiochiletv.py
new file mode 100644
index 000000000..133228133
--- /dev/null
+++ b/youtube_dl/extractor/biobiochiletv.py
@@ -0,0 +1,86 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import remove_end
+
+
+class BioBioChileTVIE(InfoExtractor):
+    _VALID_URL = r'https?://tv\.biobiochile\.cl/notas/(?:[^/]+/)+(?P<id>[^/]+)\.shtml'
+
+    _TESTS = [{
+        'url': 'http://tv.biobiochile.cl/notas/2015/10/21/sobre-camaras-y-camarillas-parlamentarias.shtml',
+        'md5': '26f51f03cf580265defefb4518faec09',
+        'info_dict': {
+            'id': 'sobre-camaras-y-camarillas-parlamentarias',
+            'ext': 'mp4',
+            'title': 'Sobre Cámaras y camarillas parlamentarias',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'uploader': 'Fernando Atria',
+        },
+    }, {
+        # different uploader layout
+        'url': 'http://tv.biobiochile.cl/notas/2016/03/18/natalia-valdebenito-repasa-a-diputado-hasbun-paso-a-la-categoria-de-hablar-brutalidades.shtml',
+        'md5': 'edc2e6b58974c46d5b047dea3c539ff3',
+        'info_dict': {
+            'id': 'natalia-valdebenito-repasa-a-diputado-hasbun-paso-a-la-categoria-de-hablar-brutalidades',
+            'ext': 'mp4',
+            'title': 'Natalia Valdebenito repasa a diputado Hasbún: Pasó a la categoría de hablar brutalidades',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'uploader': 'Piangella Obrador',
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }, {
+        'url': 'http://tv.biobiochile.cl/notas/2015/10/22/ninos-transexuales-de-quien-es-la-decision.shtml',
+        'only_matching': True,
+    }, {
+        'url': 'http://tv.biobiochile.cl/notas/2015/10/21/exclusivo-hector-pinto-formador-de-chupete-revela-version-del-ex-delantero-albo.shtml',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        title = remove_end(self._og_search_title(webpage), ' - BioBioChile TV')
+
+        file_url = self._search_regex(
+            r'loadFWPlayerVideo\([^,]+,\s*(["\'])(?P<url>.+?)\1',
+            webpage, 'file url', group='url')
+
+        base_url = self._search_regex(
+            r'file\s*:\s*(["\'])(?P<url>.+?)\1\s*\+\s*fileURL', webpage,
+            'base url', default='http://unlimited2-cl.digitalproserver.com/bbtv/',
+            group='url')
+
+        formats = self._extract_m3u8_formats(
+            '%s%s/playlist.m3u8' % (base_url, file_url), video_id, 'mp4',
+            entry_protocol='m3u8_native', m3u8_id='hls', fatal=False)
+        f = {
+            'url': '%s%s' % (base_url, file_url),
+            'format_id': 'http',
+            'protocol': 'http',
+            'preference': 1,
+        }
+        if formats:
+            f_copy = formats[-1].copy()
+            f_copy.update(f)
+            f = f_copy
+        formats.append(f)
+        self._sort_formats(formats)
+
+        thumbnail = self._og_search_thumbnail(webpage)
+        uploader = self._html_search_regex(
+            r'<a[^>]+href=["\']https?://busca\.biobiochile\.cl/author[^>]+>(.+?)</a>',
+            webpage, 'uploader', fatal=False)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'thumbnail': thumbnail,
+            'uploader': uploader,
+            'formats': formats,
+        }
diff --git a/youtube_dl/extractor/biobiotv.py b/youtube_dl/extractor/biobiotv.py
deleted file mode 100644
index aae0588ef..000000000
--- a/youtube_dl/extractor/biobiotv.py
+++ /dev/null
@@ -1,75 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-import re
-
-from .common import InfoExtractor
-
-
-class BioBioTVIE(InfoExtractor):
-    _VALID_URL = r'https?://tv\.biobiochile\.cl/notas/(?P<year>\d{4})/\d{2}/\d{2}/(?P<id>[\w-]+)(?:\.shtml)?'
-
-    _TESTS = [{
-        'url': 'http://tv.biobiochile.cl/notas/2015/10/21/sobre-camaras-y-camarillas-parlamentarias.shtml',
-        'md5': '26f51f03cf580265defefb4518faec09',
-        'info_dict': {
-            'id': 'col_c266',
-            'display_id': 'sobre-camaras-y-camarillas-parlamentarias',
-            'ext': 'mp4',
-            'title': 'Sobre Cámaras y camarillas parlamentarias - BioBioChile TV',
-            'thumbnail': 'http://media.biobiochile.cl/wp-content/uploads/2015/10/atria-2010-730x350.jpg',
-            'url': 'http://unlimited2-cl.digitalproserver.com/bbtv/2015/col_c266.mp4',
-            'uploader': 'Fernando Atria',
-        }
-    }, {
-        'url': 'http://tv.biobiochile.cl/notas/2015/10/22/ninos-transexuales-de-quien-es-la-decision.shtml',
-        'md5': 'a8c868e6b5f6c17d56873d5633204f84',
-        'info_dict': {
-            'id': 'col_c270',
-            'display_id': 'ninos-transexuales-de-quien-es-la-decision',
-            'ext': 'mp4',
-            'title': 'Niños transexuales: ¿De quién es la decisión? - BioBioChile TV',
-            'thumbnail': 'http://media.biobiochile.cl/wp-content/uploads/2015/10/samantha-2210-730x350.jpg',
-            'url': 'http://unlimited2-cl.digitalproserver.com/bbtv/2015/col_c270.mp4',
-            'uploader': 'Samantha Morán',
-        }
-    }, {
-        'url': 'http://tv.biobiochile.cl/notas/2015/10/21/exclusivo-hector-pinto-formador-de-chupete-revela-version-del-ex-delantero-albo.shtml',
-        'md5': 'c8369b50d42ff0a4f6b969fbd1a7c32d',
-        'info_dict': {
-            'id': 'Keno_Pinto',
-            'display_id': 'exclusivo-hector-pinto-formador-de-chupete-revela-version-del-ex-delantero-albo',
-            'ext': 'mp4',
-            'title': 'Exclusivo: Héctor Pinto, formador de “Chupete”, revela versión del ex delantero albo - BioBioChile TV',
-            'thumbnail': 'http://media.biobiochile.cl/wp-content/uploads/2015/10/pinto-730x350.jpg',
-            'url': 'http://unlimited2-cl.digitalproserver.com/bbtv/2015/Keno_Pinto.mp4',
-            'uploader': 'Juan Pablo Echenique',
-        }
-    }]
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        display_id = mobj.group('id')
-        year = mobj.group('year')
-
-        webpage = self._download_webpage(url, display_id)
-
-        title = self._html_search_meta(
-            'og:title', webpage, 'title', fatal=True)
-
-        thumbnail = self._html_search_meta(
-            'og:image', webpage, 'thumbnail', fatal=True)
-
-        video_id = self._html_search_regex(
-            r'loadFWPlayerVideo\(\"player_0\", \"\d{4}/(.+)\.mp4\"\)', webpage, 'title')
-
-        url = 'http://unlimited2-cl.digitalproserver.com/bbtv/' + year + '/' + video_id + '.mp4'
-
-        return {
-            'id': video_id,
-            'title': title,
-            'url': url,
-            'display_id': display_id,
-            'thumbnail': thumbnail,
-            'uploader': self._search_regex(r'biobiochile\.cl/author[^"]+"[^>]*>([^<]+)<', webpage, 'uploader', fatal=False),
-        }

From 94dcade8f892f27f1cdbee29be2e06c08442976e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 20 Mar 2016 01:36:20 +0600
Subject: [PATCH 4362/6219] Credit @jjatria for biobiochiletv (#7314)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 6c9747913..e507686f2 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -164,3 +164,4 @@ Aidan Rowe
 mutantmonkey
 Ben Congdon
 Kacper Michajłow
+José Joaquín Atria

From d95114dd8359d10c6a0ca5eaddbbd94806173957 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 20 Mar 2016 02:34:02 +0600
Subject: [PATCH 4363/6219] [91porn] Unquote final URL (Closes #8881)

---
 youtube_dl/extractor/porn91.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/porn91.py b/youtube_dl/extractor/porn91.py
index 3e15533e9..a6dd2694c 100644
--- a/youtube_dl/extractor/porn91.py
+++ b/youtube_dl/extractor/porn91.py
@@ -1,7 +1,10 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
-from ..compat import compat_urllib_parse
+from ..compat import (
+    compat_urllib_parse,
+    compat_urllib_parse_unquote,
+)
 from .common import InfoExtractor
 from ..utils import (
     parse_duration,
@@ -55,7 +58,8 @@ class Porn91IE(InfoExtractor):
         info_cn = self._download_webpage(
             'http://91porn.com/getfile.php?' + url_params, video_id,
             'get real video url')
-        video_url = self._search_regex(r'file=([^&]+)&', info_cn, 'url')
+        video_url = compat_urllib_parse_unquote(self._search_regex(
+            r'file=([^&]+)&', info_cn, 'url'))
 
         duration = parse_duration(self._search_regex(
             r'时长:\s*</span>\s*(\d+:\d+)', webpage, 'duration', fatal=False))

From 298c04b46497b924b1cbb2f031c5d73d09d2933d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 20 Mar 2016 02:35:48 +0600
Subject: [PATCH 4364/6219] [91porn] Use common messages' wording

---
 youtube_dl/extractor/porn91.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/porn91.py b/youtube_dl/extractor/porn91.py
index a6dd2694c..63ce87ee3 100644
--- a/youtube_dl/extractor/porn91.py
+++ b/youtube_dl/extractor/porn91.py
@@ -31,9 +31,10 @@ class Porn91IE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        url = 'http://91porn.com/view_video.php?viewkey=%s' % video_id
         self._set_cookie('91porn.com', 'language', 'cn_CN')
-        webpage = self._download_webpage(url, video_id, 'get HTML content')
+
+        webpage = self._download_webpage(
+            'http://91porn.com/view_video.php?viewkey=%s' % video_id, video_id)
 
         if '作为游客，你每天只可观看10个视频' in webpage:
             raise ExtractorError('91 Porn says: Daily limit 10 videos exceeded', expected=True)
@@ -57,7 +58,7 @@ class Porn91IE(InfoExtractor):
         })
         info_cn = self._download_webpage(
             'http://91porn.com/getfile.php?' + url_params, video_id,
-            'get real video url')
+            'Downloading real video url')
         video_url = compat_urllib_parse_unquote(self._search_regex(
             r'file=([^&]+)&', info_cn, 'url'))
 

From 5c7cd37ebd6dfb7d5809d2798d0188decce42914 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 19 Mar 2016 21:50:16 +0100
Subject: [PATCH 4365/6219] tox.ini: Exclude test_iqiyi_sdk_interpreter.py

---
 tox.ini | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tox.ini b/tox.ini
index 48504329f..2d7134005 100644
--- a/tox.ini
+++ b/tox.ini
@@ -8,6 +8,6 @@ deps =
 passenv = HOME
 defaultargs = test --exclude test_download.py --exclude test_age_restriction.py
     --exclude test_subtitles.py --exclude test_write_annotations.py
-    --exclude test_youtube_lists.py
+    --exclude test_youtube_lists.py --exclude test_iqiyi_sdk_interpreter.py
 commands = nosetests --verbose {posargs:{[testenv]defaultargs}}  # --with-coverage --cover-package=youtube_dl --cover-html
                                                # test.test_download:TestDownload.test_NowVideo

From 3ff63fb3657fee9e0c2df9d5bb96ae5827f257cc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 19 Mar 2016 21:51:13 +0100
Subject: [PATCH 4366/6219] Makefile: make it compatible with bmake

It's the portable version of BSD make: http://crufty.net/help/sjg/bmake.html
The syntax for conditionals is different in GNU make and BSD make, so we use the shell
---
 Makefile | 10 +---------
 1 file changed, 1 insertion(+), 9 deletions(-)

diff --git a/Makefile b/Makefile
index e98806791..6689ec06f 100644
--- a/Makefile
+++ b/Makefile
@@ -12,15 +12,7 @@ SHAREDIR ?= $(PREFIX)/share
 PYTHON ?= /usr/bin/env python
 
 # set SYSCONFDIR to /etc if PREFIX=/usr or PREFIX=/usr/local
-ifeq ($(PREFIX),/usr)
-	SYSCONFDIR=/etc
-else
-	ifeq ($(PREFIX),/usr/local)
-		SYSCONFDIR=/etc
-	else
-		SYSCONFDIR=$(PREFIX)/etc
-	endif
-endif
+SYSCONFDIR != if [ $(PREFIX) = /usr -o $(PREFIX) = /usr/local ]; then echo /etc; else echo $(PREFIX)/etc; fi
 
 install: youtube-dl youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtube-dl.fish
 	install -d $(DESTDIR)$(BINDIR)

From 40025ee2a339d1a357869fd8d8718a737d250d9e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 20 Mar 2016 04:12:34 +0600
Subject: [PATCH 4367/6219] [postprocessort/ffmpeg] Allow embedding webvtt into
 webm (Closes #8874)

---
 youtube_dl/options.py              |  2 +-
 youtube_dl/postprocessor/ffmpeg.py | 25 +++++++++++++++++++++----
 2 files changed, 22 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 9dd7a8034..755ed6540 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -720,7 +720,7 @@ def parseOpts(overrideArguments=None):
     postproc.add_option(
         '--embed-subs',
         action='store_true', dest='embedsubtitles', default=False,
-        help='Embed subtitles in the video (only for mkv and mp4 videos)')
+        help='Embed subtitles in the video (only for mp4, webm and mkv videos)')
     postproc.add_option(
         '--embed-thumbnail',
         action='store_true', dest='embedthumbnail', default=False,
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index a8819f258..06b8c0548 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -331,17 +331,34 @@ class FFmpegVideoConvertorPP(FFmpegPostProcessor):
 
 class FFmpegEmbedSubtitlePP(FFmpegPostProcessor):
     def run(self, information):
-        if information['ext'] not in ['mp4', 'mkv']:
-            self._downloader.to_screen('[ffmpeg] Subtitles can only be embedded in mp4 or mkv files')
+        if information['ext'] not in ('mp4', 'webm', 'mkv'):
+            self._downloader.to_screen('[ffmpeg] Subtitles can only be embedded in mp4, webm or mkv files')
             return [], information
         subtitles = information.get('requested_subtitles')
         if not subtitles:
             self._downloader.to_screen('[ffmpeg] There aren\'t any subtitles to embed')
             return [], information
 
-        sub_langs = list(subtitles.keys())
         filename = information['filepath']
-        sub_filenames = [subtitles_filename(filename, lang, sub_info['ext']) for lang, sub_info in subtitles.items()]
+
+        ext = information['ext']
+        sub_langs = []
+        sub_filenames = []
+        webm_vtt_warn = False
+
+        for lang, sub_info in subtitles.items():
+            sub_ext = sub_info['ext']
+            if ext != 'webm' or ext == 'webm' and sub_ext == 'vtt':
+                sub_langs.append(lang)
+                sub_filenames.append(subtitles_filename(filename, lang, sub_ext))
+            else:
+                if not webm_vtt_warn and ext == 'webm' and sub_ext != 'vtt':
+                    webm_vtt_warn = True
+                    self._downloader.to_screen('[ffmpeg] Only WebVTT subtitles can be embedded in webm files')
+
+        if not sub_langs:
+            return [], information
+
         input_files = [filename] + sub_filenames
 
         opts = [

From 96a9f22d983e414583f52eafece7902b1248377b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 20 Mar 2016 10:26:58 +0600
Subject: [PATCH 4368/6219] [discovery] Relax _VALID_URL (Closes #8903)

---
 youtube_dl/extractor/discovery.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/discovery.py b/youtube_dl/extractor/discovery.py
index ce680a9f3..fdce1429a 100644
--- a/youtube_dl/extractor/discovery.py
+++ b/youtube_dl/extractor/discovery.py
@@ -9,7 +9,7 @@ from ..compat import compat_str
 
 
 class DiscoveryIE(InfoExtractor):
-    _VALID_URL = r'''(?x)http://(?:www\.)?(?:
+    _VALID_URL = r'''(?x)https?://(?:www\.)?(?:
             discovery|
             investigationdiscovery|
             discoverylife|

From db264e3cc3bbab191972bbe1c4efb526ff8bfc26 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 20 Mar 2016 12:44:04 +0600
Subject: [PATCH 4369/6219] [francetvinfo] Add support for france3-regions and
 strip title (Closes #7673)

---
 youtube_dl/extractor/francetv.py | 22 ++++++++++++++++++++--
 1 file changed, 20 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index 3f4ac3093..7db5fb418 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -82,6 +82,7 @@ class FranceTVBaseInfoExtractor(InfoExtractor):
         subtitle = info.get('sous_titre')
         if subtitle:
             title += ' - %s' % subtitle
+        title = title.strip()
 
         subtitles = {}
         subtitles_list = [{
@@ -125,7 +126,7 @@ class PluzzIE(FranceTVBaseInfoExtractor):
 
 class FranceTvInfoIE(FranceTVBaseInfoExtractor):
     IE_NAME = 'francetvinfo.fr'
-    _VALID_URL = r'https?://(?:www|mobile)\.francetvinfo\.fr/.*/(?P<title>.+)\.html'
+    _VALID_URL = r'https?://(?:www|mobile|france3-regions)\.francetvinfo\.fr/.*/(?P<title>.+)\.html'
 
     _TESTS = [{
         'url': 'http://www.francetvinfo.fr/replay-jt/france-3/soir-3/jt-grand-soir-3-lundi-26-aout-2013_393427.html',
@@ -160,6 +161,21 @@ class FranceTvInfoIE(FranceTVBaseInfoExtractor):
             'title': 'Les entreprises familiales : le secret de la réussite',
             'thumbnail': 're:^https?://.*\.jpe?g$',
         }
+    }, {
+        'url': 'http://france3-regions.francetvinfo.fr/bretagne/cotes-d-armor/thalassa-echappee-breizh-ce-venredi-dans-les-cotes-d-armor-954961.html',
+        'md5': 'f485bda6e185e7d15dbc69b72bae993e',
+        'info_dict': {
+            'id': 'NI_657393',
+            'ext': 'flv',
+            'title': 'Olivier Monthus, réalisateur de "Bretagne, le choix de l’Armor"',
+            'description': 'md5:a3264114c9d29aeca11ced113c37b16c',
+            'thumbnail': 're:^https?://.*\.jpe?g$',
+            'timestamp': 1458300695,
+            'upload_date': '20160318',
+        },
+        'params': {
+            'skip_download': True,
+        },
     }]
 
     def _real_extract(self, url):
@@ -172,7 +188,9 @@ class FranceTvInfoIE(FranceTVBaseInfoExtractor):
             return self.url_result(dmcloud_url, 'DailymotionCloud')
 
         video_id, catalogue = self._search_regex(
-            r'id-video=([^@]+@[^"]+)', webpage, 'video id').split('@')
+            (r'id-video=([^@]+@[^"]+)',
+             r'<a[^>]+href="(?:https?:)?//videos\.francetv\.fr/video/([^@]+@[^"]+)"'),
+            webpage, 'video id').split('@')
         return self._extract_video(video_id, catalogue)
 
 

From 3c20208eff29fcd3e5d4b13f3d4ffa1be7c56309 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 20 Mar 2016 13:00:46 +0600
Subject: [PATCH 4370/6219] [francetv] Improve formats extraction

---
 youtube_dl/extractor/francetv.py | 36 +++++++++++++++++++++-----------
 1 file changed, 24 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index 7db5fb418..ad94e31f3 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -60,22 +60,24 @@ class FranceTVBaseInfoExtractor(InfoExtractor):
                     video_id, 'Downloading f4m manifest token', fatal=False)
                 if f4m_url:
                     formats.extend(self._extract_f4m_formats(
-                        f4m_url + '&hdcore=3.7.0&plugin=aasp-3.7.0.39.44', video_id, 1, format_id))
+                        f4m_url + '&hdcore=3.7.0&plugin=aasp-3.7.0.39.44',
+                        video_id, f4m_id=format_id, fatal=False))
             elif ext == 'm3u8':
-                formats.extend(self._extract_m3u8_formats(video_url, video_id, 'mp4', m3u8_id=format_id))
+                formats.extend(self._extract_m3u8_formats(
+                    video_url, video_id, 'mp4', entry_protocol='m3u8_native',
+                    m3u8_id=format_id, fatal=False))
             elif video_url.startswith('rtmp'):
                 formats.append({
                     'url': video_url,
                     'format_id': 'rtmp-%s' % format_id,
                     'ext': 'flv',
-                    'preference': 1,
                 })
             else:
-                formats.append({
-                    'url': video_url,
-                    'format_id': format_id,
-                    'preference': -1,
-                })
+                if self._is_valid_url(video_url, video_id, format_id):
+                    formats.append({
+                        'url': video_url,
+                        'format_id': format_id,
+                    })
         self._sort_formats(formats)
 
         title = info['titre']
@@ -132,7 +134,7 @@ class FranceTvInfoIE(FranceTVBaseInfoExtractor):
         'url': 'http://www.francetvinfo.fr/replay-jt/france-3/soir-3/jt-grand-soir-3-lundi-26-aout-2013_393427.html',
         'info_dict': {
             'id': '84981923',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Soir 3',
             'upload_date': '20130826',
             'timestamp': 1377548400,
@@ -140,6 +142,10 @@ class FranceTvInfoIE(FranceTVBaseInfoExtractor):
                 'fr': 'mincount:2',
             },
         },
+        'params': {
+            # m3u8 downloads
+            'skip_download': True,
+        },
     }, {
         'url': 'http://www.francetvinfo.fr/elections/europeennes/direct-europeennes-regardez-le-debat-entre-les-candidats-a-la-presidence-de-la-commission_600639.html',
         'info_dict': {
@@ -156,17 +162,23 @@ class FranceTvInfoIE(FranceTVBaseInfoExtractor):
         'url': 'http://www.francetvinfo.fr/economie/entreprises/les-entreprises-familiales-le-secret-de-la-reussite_933271.html',
         'md5': 'f485bda6e185e7d15dbc69b72bae993e',
         'info_dict': {
-            'id': '556e03339473995ee145930c',
+            'id': 'NI_173343',
             'ext': 'mp4',
             'title': 'Les entreprises familiales : le secret de la réussite',
             'thumbnail': 're:^https?://.*\.jpe?g$',
-        }
+            'timestamp': 1433273139,
+            'upload_date': '20150602',
+        },
+        'params': {
+            # m3u8 downloads
+            'skip_download': True,
+        },
     }, {
         'url': 'http://france3-regions.francetvinfo.fr/bretagne/cotes-d-armor/thalassa-echappee-breizh-ce-venredi-dans-les-cotes-d-armor-954961.html',
         'md5': 'f485bda6e185e7d15dbc69b72bae993e',
         'info_dict': {
             'id': 'NI_657393',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Olivier Monthus, réalisateur de "Bretagne, le choix de l’Armor"',
             'description': 'md5:a3264114c9d29aeca11ced113c37b16c',
             'thumbnail': 're:^https?://.*\.jpe?g$',

From 664bcd80b99ac84c3cc7a08e8284abc024a1e58c Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 20 Mar 2016 15:45:31 +0800
Subject: [PATCH 4371/6219] [tudou] Use InAdvancePagedList (closes #8884)

---
 youtube_dl/extractor/tudou.py | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/tudou.py b/youtube_dl/extractor/tudou.py
index f56b66d06..9892e8a62 100644
--- a/youtube_dl/extractor/tudou.py
+++ b/youtube_dl/extractor/tudou.py
@@ -6,6 +6,7 @@ from .common import InfoExtractor
 from ..compat import compat_str
 from ..utils import (
     int_or_none,
+    InAdvancePagedList,
     float_or_none,
     unescapeHTML,
 )
@@ -75,15 +76,16 @@ class TudouIE(InfoExtractor):
         quality = sorted(filter(lambda k: k.isdigit(), segments.keys()),
                          key=lambda k: int(k))[-1]
         parts = segments[quality]
-        result = []
         len_parts = len(parts)
         if len_parts > 1:
             self.to_screen('%s: found %s parts' % (video_id, len_parts))
-        for part in parts:
+
+        def part_func(partnum):
+            part = parts[partnum]
             part_id = part['k']
             final_url = self._url_for_id(part_id, quality)
             ext = (final_url.split('?')[0]).split('.')[-1]
-            part_info = {
+            return [{
                 'id': '%s' % part_id,
                 'url': final_url,
                 'ext': ext,
@@ -97,12 +99,13 @@ class TudouIE(InfoExtractor):
                 'http_headers': {
                     'Referer': self._PLAYER_URL,
                 },
-            }
-            result.append(part_info)
+            }]
+
+        entries = InAdvancePagedList(part_func, len_parts, 1)
 
         return {
             '_type': 'multi_video',
-            'entries': result,
+            'entries': entries,
             'id': video_id,
             'title': title,
         }

From 2bfeee69b976fe049761dd3012e30b637ee05a58 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 20 Mar 2016 15:54:58 +0800
Subject: [PATCH 4372/6219] [openload] Add new extractor (closes #8489)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/openload.py | 92 ++++++++++++++++++++++++++++++++
 2 files changed, 93 insertions(+)
 create mode 100644 youtube_dl/extractor/openload.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index b773edb3d..ee792bbe0 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -535,6 +535,7 @@ from .ooyala import (
     OoyalaIE,
     OoyalaExternalIE,
 )
+from .openload import OpenloadIE
 from .ora import OraTVIE
 from .orf import (
     ORFTVthekIE,
diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py
new file mode 100644
index 000000000..71021d573
--- /dev/null
+++ b/youtube_dl/extractor/openload.py
@@ -0,0 +1,92 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..compat import compat_chr
+from ..utils import encode_base_n
+
+
+class OpenloadIE(InfoExtractor):
+    _VALID_URL = r'https://openload.co/f/(?P<id>[a-zA-Z0-9]+)'
+
+    _TEST = {
+        'url': 'https://openload.co/f/kUEfGclsU9o',
+        'md5': 'bf1c059b004ebc7a256f89408e65c36e',
+        'info_dict': {
+            'id': 'kUEfGclsU9o',
+            'ext': 'mp4',
+            'title': 'skyrim_no-audio_1080.mp4',
+        },
+    }
+
+    @staticmethod
+    def openload_level2_debase(m):
+        radix, num = int(m.group(1)) + 27, int(m.group(2))
+        return '"' + encode_base_n(num, radix) + '"'
+
+    @classmethod
+    def openload_level2(cls, txt):
+        # The function name is ǃ \u01c3
+        # Using escaped unicode literals does not work in Python 3.2
+        return re.sub(r'ǃ\((\d+),(\d+)\)', cls.openload_level2_debase, txt, re.UNICODE).replace('"+"', '')
+
+    # Openload uses a variant of aadecode
+    # openload_decode and related functions are originally written by
+    # vitas@matfyz.cz and released with public domain
+    # See https://github.com/rg3/youtube-dl/issues/8489
+    @classmethod
+    def openload_decode(cls, txt):
+        symbol_table = [
+            ('_', '(ﾟДﾟ) [ﾟΘﾟ]'),
+            ('a', '(ﾟДﾟ) [ﾟωﾟﾉ]'),
+            ('b', '(ﾟДﾟ) [ﾟΘﾟﾉ]'),
+            ('c', '(ﾟДﾟ) [\'c\']'),
+            ('d', '(ﾟДﾟ) [ﾟｰﾟﾉ]'),
+            ('e', '(ﾟДﾟ) [ﾟДﾟﾉ]'),
+            ('f', '(ﾟДﾟ) [1]'),
+
+            ('o', '(ﾟДﾟ) [\'o\']'),
+            ('u', '(oﾟｰﾟo)'),
+            ('c', '(ﾟДﾟ) [\'c\']'),
+
+            ('7', '((ﾟｰﾟ) + (o^_^o))'),
+            ('6', '((o^_^o) +(o^_^o) +(c^_^o))'),
+            ('5', '((ﾟｰﾟ) + (ﾟΘﾟ))'),
+            ('4', '(-~3)'),
+            ('3', '(-~-~1)'),
+            ('2', '(-~1)'),
+            ('1', '(-~0)'),
+            ('0', '((c^_^o)-(c^_^o))'),
+        ]
+        delim = '(ﾟДﾟ)[ﾟεﾟ]+'
+        ret = ''
+        for aachar in txt.split(delim):
+            for val, pat in symbol_table:
+                aachar = aachar.replace(pat, val)
+            aachar = aachar.replace('+ ', '')
+            m = re.match(r'^\d+', aachar)
+            if m:
+                ret += compat_chr(int(m.group(0), 8))
+            else:
+                m = re.match(r'^u([\da-f]+)', aachar)
+                if m:
+                    ret += compat_chr(int(m.group(1), 16))
+        return cls.openload_level2(ret)
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        code = self._search_regex(
+            r'<video[^>]+>\s*<script[^>]+>([^<]+)</script>',
+            webpage, 'JS code')
+
+        video_url = self._search_regex(
+            r'return\s+"(https?://[^"]+)"', self.openload_decode(code), 'video URL')
+
+        return {
+            'id': video_id,
+            'title': self._og_search_title(webpage),
+            'url': video_url,
+        }

From 9e3c2f1d741acc4dd576f77c185e99cfd6bb2ea4 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 20 Mar 2016 16:49:44 +0800
Subject: [PATCH 4373/6219] [openload] Misc improvements

* Add thumbnail
* Detect errors (#6469)
* Match more (#6469, #8489)
---
 youtube_dl/extractor/openload.py | 23 +++++++++++++++++++----
 1 file changed, 19 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py
index 71021d573..4468f31fc 100644
--- a/youtube_dl/extractor/openload.py
+++ b/youtube_dl/extractor/openload.py
@@ -5,21 +5,31 @@ import re
 
 from .common import InfoExtractor
 from ..compat import compat_chr
-from ..utils import encode_base_n
+from ..utils import (
+    encode_base_n,
+    ExtractorError,
+)
 
 
 class OpenloadIE(InfoExtractor):
-    _VALID_URL = r'https://openload.co/f/(?P<id>[a-zA-Z0-9]+)'
+    _VALID_URL = r'https://openload.(?:co|io)/(?:f|embed)/(?P<id>[a-zA-Z0-9-]+)'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'https://openload.co/f/kUEfGclsU9o',
         'md5': 'bf1c059b004ebc7a256f89408e65c36e',
         'info_dict': {
             'id': 'kUEfGclsU9o',
             'ext': 'mp4',
             'title': 'skyrim_no-audio_1080.mp4',
+            'thumbnail': 're:^https?://.*\.jpg$',
         },
-    }
+    }, {
+        'url': 'https://openload.co/embed/kUEfGclsU9o/skyrim_no-audio_1080.mp4',
+        'only_matching': True,
+    }, {
+        'url': 'https://openload.io/f/ZAn6oz-VZGE/',
+        'only_matching': True,
+    }]
 
     @staticmethod
     def openload_level2_debase(m):
@@ -78,6 +88,10 @@ class OpenloadIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
+
+        if 'File not found' in webpage:
+            raise ExtractorError('File not found', expected=True)
+
         code = self._search_regex(
             r'<video[^>]+>\s*<script[^>]+>([^<]+)</script>',
             webpage, 'JS code')
@@ -88,5 +102,6 @@ class OpenloadIE(InfoExtractor):
         return {
             'id': video_id,
             'title': self._og_search_title(webpage),
+            'thumbnail': self._og_search_thumbnail(webpage),
             'url': video_url,
         }

From 920d318d3cf70ee0b80cc67e7d3f85b5d45a20a7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 20 Mar 2016 10:55:14 +0100
Subject: [PATCH 4374/6219] README: document that BSD make is also supported
 (#8902)

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 68db546ef..fcc12d2b3 100644
--- a/README.md
+++ b/README.md
@@ -831,7 +831,7 @@ To run the test, simply invoke your favorite test runner, or execute a test file
 If you want to create a build of youtube-dl yourself, you'll need
 
 * python
-* make
+* make (both GNU make and BSD make are supported)
 * pandoc
 * zip
 * nosetests

From 2648918c814773e746c6d26da834d32eac952ffa Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 20 Mar 2016 18:14:02 +0800
Subject: [PATCH 4375/6219] [vlive] Fix creator extraction (closes #8814)

---
 youtube_dl/extractor/vlive.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vlive.py b/youtube_dl/extractor/vlive.py
index 9e2aa58bd..bd5545173 100644
--- a/youtube_dl/extractor/vlive.py
+++ b/youtube_dl/extractor/vlive.py
@@ -64,7 +64,7 @@ class VLiveIE(InfoExtractor):
 
         thumbnail = self._og_search_thumbnail(webpage)
         creator = self._html_search_regex(
-            r'<div[^>]+class="info_area"[^>]*>\s*<strong[^>]+class="name"[^>]*>([^<]+)</strong>',
+            r'<div[^>]+class="info_area"[^>]*>\s*<a\s+[^>]*>([^<]+)',
             webpage, 'creator', fatal=False)
 
         view_count = int_or_none(playinfo.get('meta', {}).get('count'))

From 7caae128a72596e22cdfb538272c38a37d095db2 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 20 Mar 2016 19:11:02 +0800
Subject: [PATCH 4376/6219] Credit @vitstradal for the key algorithm in
 OpenloadIE (#8489)

[ci skip]
---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index e507686f2..51dfc8ddd 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -165,3 +165,4 @@ mutantmonkey
 Ben Congdon
 Kacper Michajłow
 José Joaquín Atria
+Viťas Strádal

From daef04a4e75ccd2ff5e2d2495baa0ac9bcf75724 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 20 Mar 2016 20:17:56 +0800
Subject: [PATCH 4377/6219] [kwuo] Fix KuwoChartIE and KuwoSingerIE and accept
 new URL forms

---
 youtube_dl/extractor/kuwo.py | 52 ++++++++++++++++++------------------
 1 file changed, 26 insertions(+), 26 deletions(-)

diff --git a/youtube_dl/extractor/kuwo.py b/youtube_dl/extractor/kuwo.py
index f94804d06..45d65e61f 100644
--- a/youtube_dl/extractor/kuwo.py
+++ b/youtube_dl/extractor/kuwo.py
@@ -2,13 +2,13 @@
 from __future__ import unicode_literals
 
 import re
-import itertools
 
 from .common import InfoExtractor
 from ..utils import (
     get_element_by_id,
     clean_html,
     ExtractorError,
+    InAdvancePagedList,
     remove_start,
 )
 
@@ -55,7 +55,7 @@ class KuwoBaseIE(InfoExtractor):
 class KuwoIE(KuwoBaseIE):
     IE_NAME = 'kuwo:song'
     IE_DESC = '酷我音乐'
-    _VALID_URL = r'http://www\.kuwo\.cn/yinyue/(?P<id>\d+?)/'
+    _VALID_URL = r'http://www\.kuwo\.cn/yinyue/(?P<id>\d+?)'
     _TESTS = [{
         'url': 'http://www.kuwo.cn/yinyue/635632/',
         'info_dict': {
@@ -80,6 +80,9 @@ class KuwoIE(KuwoBaseIE):
         'params': {
             'format': 'mp3-320'
         },
+    }, {
+        'url': 'http://www.kuwo.cn/yinyue/3197154?catalog=yueku2016',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
@@ -172,8 +175,6 @@ class KuwoChartIE(InfoExtractor):
         'url': 'http://yinyue.kuwo.cn/billboard_香港中文龙虎榜.htm',
         'info_dict': {
             'id': '香港中文龙虎榜',
-            'title': '香港中文龙虎榜',
-            'description': 're:\d{4}第\d{2}期',
         },
         'playlist_mincount': 10,
     }
@@ -184,17 +185,11 @@ class KuwoChartIE(InfoExtractor):
             url, chart_id, note='Download chart info',
             errnote='Unable to get chart info')
 
-        chart_name = self._html_search_regex(
-            r'<h1[^>]+class="unDis">([^<]+)</h1>', webpage, 'chart name')
-
-        chart_desc = self._html_search_regex(
-            r'<p[^>]+class="tabDef">(\d{4}第\d{2}期)</p>', webpage, 'chart desc')
-
         entries = [
             self.url_result(song_url, 'Kuwo') for song_url in re.findall(
-                r'<a[^>]+href="(http://www\.kuwo\.cn/yinyue/\d+)/"', webpage)
+                r'<a[^>]+href="(http://www\.kuwo\.cn/yinyue/\d+)', webpage)
         ]
-        return self.playlist_result(entries, chart_id, chart_name, chart_desc)
+        return self.playlist_result(entries, chart_id)
 
 
 class KuwoSingerIE(InfoExtractor):
@@ -207,7 +202,7 @@ class KuwoSingerIE(InfoExtractor):
             'id': 'bruno+mars',
             'title': 'Bruno Mars',
         },
-        'playlist_count': 10,
+        'playlist_mincount': 329,
     }, {
         'url': 'http://www.kuwo.cn/mingxing/Ali/music.htm',
         'info_dict': {
@@ -218,6 +213,8 @@ class KuwoSingerIE(InfoExtractor):
         'skip': 'Regularly stalls travis build',  # See https://travis-ci.org/rg3/youtube-dl/jobs/78878540
     }]
 
+    PAGE_SIZE = 15
+
     def _real_extract(self, url):
         singer_id = self._match_id(url)
         webpage = self._download_webpage(
@@ -225,25 +222,28 @@ class KuwoSingerIE(InfoExtractor):
             errnote='Unable to get singer info')
 
         singer_name = self._html_search_regex(
-            r'<div class="title clearfix">\s*<h1>([^<]+)<span', webpage, 'singer name'
-        )
+            r'<h1>([^<]+)</h1>', webpage, 'singer name')
 
-        entries = []
-        first_page_only = False if re.search(r'/music(?:_\d+)?\.htm', url) else True
-        for page_num in itertools.count(1):
+        artist_id = self._html_search_regex(
+            r'data-artistid="(\d+)"', webpage, 'artist id')
+
+        page_count = int(self._html_search_regex(
+            r'data-page="(\d+)"', webpage, 'page count'))
+
+        def page_func(page_num):
             webpage = self._download_webpage(
-                'http://www.kuwo.cn/mingxing/%s/music_%d.htm' % (singer_id, page_num),
-                singer_id, note='Download song list page #%d' % page_num,
-                errnote='Unable to get song list page #%d' % page_num)
+                'http://www.kuwo.cn/artist/contentMusicsAjax',
+                singer_id, note='Download song list page #%d' % (page_num + 1),
+                errnote='Unable to get song list page #%d' % (page_num + 1),
+                query={'artistId': artist_id, 'pn': page_num, 'rn': self.PAGE_SIZE})
 
-            entries.extend([
+            return [
                 self.url_result(song_url, 'Kuwo') for song_url in re.findall(
-                    r'<p[^>]+class="m_name"><a[^>]+href="(http://www\.kuwo\.cn/yinyue/\d+)/',
+                    r'<div[^>]+class="name"><a[^>]+href="(http://www\.kuwo\.cn/yinyue/\d+)',
                     webpage)
-            ][:10 if first_page_only else None])
+            ]
 
-            if first_page_only or not re.search(r'<a[^>]+href="[^"]+">下一页</a>', webpage):
-                break
+        entries = InAdvancePagedList(page_func, page_count, self.PAGE_SIZE)
 
         return self.playlist_result(entries, singer_id, singer_name)
 

From 3e8bb9a972a377442f5f433123ea70b332248f70 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 20 Mar 2016 20:39:00 +0600
Subject: [PATCH 4378/6219] [animeondemand] Detect geo restriction

---
 youtube_dl/extractor/animeondemand.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/animeondemand.py b/youtube_dl/extractor/animeondemand.py
index 0158407f6..85306a69c 100644
--- a/youtube_dl/extractor/animeondemand.py
+++ b/youtube_dl/extractor/animeondemand.py
@@ -44,6 +44,10 @@ class AnimeOnDemandIE(InfoExtractor):
         login_page = self._download_webpage(
             self._LOGIN_URL, None, 'Downloading login page')
 
+        if '>Our licensing terms allow the distribution of animes only to German-speaking countries of Europe' in login_page:
+            self.raise_geo_restricted(
+                '%s is only available in German-speaking countries of Europe' % self.IE_NAME)
+
         login_form = self._form_hidden_inputs('new_user', login_page)
 
         login_form.update({

From 3c5d183c19f29c1f52fe913ce7e7d47f6eebff2f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 20 Mar 2016 21:51:22 +0600
Subject: [PATCH 4379/6219] [animeondemand] Extract all formats (Closes #8906)

---
 youtube_dl/extractor/animeondemand.py | 85 ++++++++++++++++++++-------
 1 file changed, 65 insertions(+), 20 deletions(-)

diff --git a/youtube_dl/extractor/animeondemand.py b/youtube_dl/extractor/animeondemand.py
index 85306a69c..4352525e2 100644
--- a/youtube_dl/extractor/animeondemand.py
+++ b/youtube_dl/extractor/animeondemand.py
@@ -3,10 +3,14 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urlparse
+from ..compat import (
+    compat_urlparse,
+    compat_str,
+)
 from ..utils import (
     determine_ext,
     encode_dict,
+    extract_attributes,
     ExtractorError,
     sanitized_Request,
     urlencode_postdata,
@@ -34,6 +38,10 @@ class AnimeOnDemandIE(InfoExtractor):
         # Episodes without titles
         'url': 'https://www.anime-on-demand.de/anime/162',
         'only_matching': True,
+    }, {
+        # ger/jap, Dub/OmU, account required
+        'url': 'https://www.anime-on-demand.de/anime/169',
+        'only_matching': True,
     }]
 
     def _login(self):
@@ -130,33 +138,70 @@ class AnimeOnDemandIE(InfoExtractor):
 
             formats = []
 
-            playlist_url = self._search_regex(
-                r'data-playlist=(["\'])(?P<url>.+?)\1',
-                episode_html, 'data playlist', default=None, group='url')
-            if playlist_url:
-                request = sanitized_Request(
-                    compat_urlparse.urljoin(url, playlist_url),
-                    headers={
-                        'X-Requested-With': 'XMLHttpRequest',
-                        'X-CSRF-Token': csrf_token,
-                        'Referer': url,
-                        'Accept': 'application/json, text/javascript, */*; q=0.01',
-                    })
+            for input_ in re.findall(
+                    r'<input[^>]+class=["\'].*?streamstarter_html5[^>]+>', episode_html):
+                attributes = extract_attributes(input_)
+                playlist_urls = []
+                for playlist_key in ('data-playlist', 'data-otherplaylist'):
+                    playlist_url = attributes.get(playlist_key)
+                    if isinstance(playlist_url, compat_str) and re.match(
+                            r'/?[\da-zA-Z]+', playlist_url):
+                        playlist_urls.append(attributes[playlist_key])
+                if not playlist_urls:
+                    continue
 
-                playlist = self._download_json(
-                    request, video_id, 'Downloading playlist JSON', fatal=False)
-                if playlist:
-                    playlist = playlist['playlist'][0]
-                    title = playlist['title']
+                lang = attributes.get('data-lang')
+                lang_note = attributes.get('value')
+
+                for playlist_url in playlist_urls:
+                    kind = self._search_regex(
+                        r'videomaterialurl/\d+/([^/]+)/',
+                        playlist_url, 'media kind', default=None)
+                    format_id_list = []
+                    if lang:
+                        format_id_list.append(lang)
+                    if kind:
+                        format_id_list.append(kind)
+                    if not format_id_list:
+                        format_id_list.append('hls')
+                    format_id = '-'.join(format_id_list)
+                    format_note = ', '.join(filter(None, (kind, lang_note)))
+                    request = sanitized_Request(
+                        compat_urlparse.urljoin(url, playlist_url),
+                        headers={
+                            'X-Requested-With': 'XMLHttpRequest',
+                            'X-CSRF-Token': csrf_token,
+                            'Referer': url,
+                            'Accept': 'application/json, text/javascript, */*; q=0.01',
+                        })
+                    playlist = self._download_json(
+                        request, video_id, 'Downloading %s playlist JSON' % format_id,
+                        fatal=False)
+                    if not playlist:
+                        continue
+                    playlist = playlist.get('playlist')
+                    if not playlist or not isinstance(playlist, list):
+                        continue
+                    playlist = playlist[0]
+                    title = playlist.get('title')
+                    if not title:
+                        continue
                     description = playlist.get('description')
                     for source in playlist.get('sources', []):
                         file_ = source.get('file')
                         if file_ and determine_ext(file_) == 'm3u8':
-                            formats = self._extract_m3u8_formats(
+                            m3u8_formats = self._extract_m3u8_formats(
                                 file_, video_id, 'mp4',
-                                entry_protocol='m3u8_native', m3u8_id='hls')
+                                entry_protocol='m3u8_native', m3u8_id=format_id)
+                            for f in m3u8_formats:
+                                f.update({
+                                    'language': lang,
+                                    'format_note': format_note,
+                                })
+                            formats.extend(m3u8_formats)
 
             if formats:
+                self._sort_formats(formats)
                 f = common_info.copy()
                 f.update({
                     'title': title,

From 9016d76f71b30bd61d69f80dc88fa53f978cf99c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 20 Mar 2016 22:01:45 +0600
Subject: [PATCH 4380/6219] [YoutubeDL] Improve _format_note

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 93b6ca54d..29d7a3106 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1836,7 +1836,7 @@ class YoutubeDL(object):
         if fdict.get('language'):
             if res:
                 res += ' '
-            res += '[%s]' % fdict['language']
+            res += '[%s] ' % fdict['language']
         if fdict.get('format_note') is not None:
             res += fdict['format_note'] + ' '
         if fdict.get('tbr') is not None:

From 12af4beb3e28f986170ed00488b48e2e8bcd4e13 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 21 Mar 2016 21:17:29 +0600
Subject: [PATCH 4381/6219] [mailru] Add support for https (Closes #8920)

---
 youtube_dl/extractor/mailru.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mailru.py b/youtube_dl/extractor/mailru.py
index 71085f279..46eb00492 100644
--- a/youtube_dl/extractor/mailru.py
+++ b/youtube_dl/extractor/mailru.py
@@ -13,7 +13,7 @@ from ..utils import (
 class MailRuIE(InfoExtractor):
     IE_NAME = 'mailru'
     IE_DESC = 'Видео@Mail.Ru'
-    _VALID_URL = r'http://(?:www\.)?my\.mail\.ru/(?:video/.*#video=/?(?P<idv1>(?:[^/]+/){3}\d+)|(?:(?P<idv2prefix>(?:[^/]+/){2})video/(?P<idv2suffix>[^/]+/\d+))\.html)'
+    _VALID_URL = r'https?://(?:www\.)?my\.mail\.ru/(?:video/.*#video=/?(?P<idv1>(?:[^/]+/){3}\d+)|(?:(?P<idv2prefix>(?:[^/]+/){2})video/(?P<idv2suffix>[^/]+/\d+))\.html)'
 
     _TESTS = [
         {

From 0cef27ad255b5cb994b1fa0e80a04bd09514925a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 21 Mar 2016 21:22:37 +0600
Subject: [PATCH 4382/6219] Add missing r prefix for _VALID_URLs

---
 youtube_dl/extractor/bbc.py  | 2 +-
 youtube_dl/extractor/nova.py | 2 +-
 youtube_dl/extractor/tv2.py  | 4 ++--
 youtube_dl/extractor/vgtv.py | 4 ++--
 youtube_dl/extractor/wdr.py  | 2 +-
 5 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index e62b3860e..c3176700a 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -942,7 +942,7 @@ class BBCIE(BBCCoUkIE):
 
 
 class BBCCoUkArticleIE(InfoExtractor):
-    _VALID_URL = 'http://www.bbc.co.uk/programmes/articles/(?P<id>[a-zA-Z0-9]+)'
+    _VALID_URL = r'http://www.bbc.co.uk/programmes/articles/(?P<id>[a-zA-Z0-9]+)'
     IE_NAME = 'bbc.co.uk:article'
     IE_DESC = 'BBC articles'
 
diff --git a/youtube_dl/extractor/nova.py b/youtube_dl/extractor/nova.py
index 3f9c776ef..72f72b803 100644
--- a/youtube_dl/extractor/nova.py
+++ b/youtube_dl/extractor/nova.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 class NovaIE(InfoExtractor):
     IE_DESC = 'TN.cz, Prásk.tv, Nova.cz, Novaplus.cz, FANDA.tv, Krásná.cz and Doma.cz'
-    _VALID_URL = 'http://(?:[^.]+\.)?(?P<site>tv(?:noviny)?|tn|novaplus|vymena|fanda|krasna|doma|prask)\.nova\.cz/(?:[^/]+/)+(?P<id>[^/]+?)(?:\.html|/|$)'
+    _VALID_URL = r'http://(?:[^.]+\.)?(?P<site>tv(?:noviny)?|tn|novaplus|vymena|fanda|krasna|doma|prask)\.nova\.cz/(?:[^/]+/)+(?P<id>[^/]+?)(?:\.html|/|$)'
     _TESTS = [{
         'url': 'http://tvnoviny.nova.cz/clanek/novinky/co-na-sebe-sportaci-praskli-vime-jestli-pujde-hrdlicka-na-materskou.html?utm_source=tvnoviny&utm_medium=cpfooter&utm_campaign=novaplus',
         'info_dict': {
diff --git a/youtube_dl/extractor/tv2.py b/youtube_dl/extractor/tv2.py
index 1457e524e..535d0d361 100644
--- a/youtube_dl/extractor/tv2.py
+++ b/youtube_dl/extractor/tv2.py
@@ -14,7 +14,7 @@ from ..utils import (
 
 
 class TV2IE(InfoExtractor):
-    _VALID_URL = 'http://(?:www\.)?tv2\.no/v/(?P<id>\d+)'
+    _VALID_URL = r'http://(?:www\.)?tv2\.no/v/(?P<id>\d+)'
     _TEST = {
         'url': 'http://www.tv2.no/v/916509/',
         'info_dict': {
@@ -100,7 +100,7 @@ class TV2IE(InfoExtractor):
 
 
 class TV2ArticleIE(InfoExtractor):
-    _VALID_URL = 'http://(?:www\.)?tv2\.no/(?:a|\d{4}/\d{2}/\d{2}(/[^/]+)+)/(?P<id>\d+)'
+    _VALID_URL = r'http://(?:www\.)?tv2\.no/(?:a|\d{4}/\d{2}/\d{2}(/[^/]+)+)/(?P<id>\d+)'
     _TESTS = [{
         'url': 'http://www.tv2.no/2015/05/16/nyheter/alesund/krim/pingvin/6930542',
         'info_dict': {
diff --git a/youtube_dl/extractor/vgtv.py b/youtube_dl/extractor/vgtv.py
index e148b1ef5..77d8978d4 100644
--- a/youtube_dl/extractor/vgtv.py
+++ b/youtube_dl/extractor/vgtv.py
@@ -214,7 +214,7 @@ class VGTVIE(XstreamIE):
 class BTArticleIE(InfoExtractor):
     IE_NAME = 'bt:article'
     IE_DESC = 'Bergens Tidende Articles'
-    _VALID_URL = 'http://(?:www\.)?bt\.no/(?:[^/]+/)+(?P<id>[^/]+)-\d+\.html'
+    _VALID_URL = r'http://(?:www\.)?bt\.no/(?:[^/]+/)+(?P<id>[^/]+)-\d+\.html'
     _TEST = {
         'url': 'http://www.bt.no/nyheter/lokalt/Kjemper-for-internatet-1788214.html',
         'md5': '2acbe8ad129b3469d5ae51b1158878df',
@@ -241,7 +241,7 @@ class BTArticleIE(InfoExtractor):
 class BTVestlendingenIE(InfoExtractor):
     IE_NAME = 'bt:vestlendingen'
     IE_DESC = 'Bergens Tidende - Vestlendingen'
-    _VALID_URL = 'http://(?:www\.)?bt\.no/spesial/vestlendingen/#!/(?P<id>\d+)'
+    _VALID_URL = r'http://(?:www\.)?bt\.no/spesial/vestlendingen/#!/(?P<id>\d+)'
     _TESTS = [{
         'url': 'http://www.bt.no/spesial/vestlendingen/#!/86588',
         'md5': 'd7d17e3337dc80de6d3a540aefbe441b',
diff --git a/youtube_dl/extractor/wdr.py b/youtube_dl/extractor/wdr.py
index a851578e0..65cab4069 100644
--- a/youtube_dl/extractor/wdr.py
+++ b/youtube_dl/extractor/wdr.py
@@ -244,7 +244,7 @@ class WDRMobileIE(InfoExtractor):
 
 
 class WDRMausIE(InfoExtractor):
-    _VALID_URL = 'http://(?:www\.)?wdrmaus\.de/(?:[^/]+/){,2}(?P<id>[^/?#]+)(?:/index\.php5|(?<!index)\.php5|/(?:$|[?#]))'
+    _VALID_URL = r'http://(?:www\.)?wdrmaus\.de/(?:[^/]+/){,2}(?P<id>[^/?#]+)(?:/index\.php5|(?<!index)\.php5|/(?:$|[?#]))'
     IE_DESC = 'Sendung mit der Maus'
     _TESTS = [{
         'url': 'http://www.wdrmaus.de/aktuelle-sendung/index.php5',

From 5886b38d73c54239c85c3e0d8e7c1585d1bbb7da Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 21 Mar 2016 21:36:32 +0600
Subject: [PATCH 4383/6219] Add support for https for all extractors as
 preventive and future-proof measure

---
 youtube_dl/extractor/abc.py                 |  2 +-
 youtube_dl/extractor/addanime.py            |  2 +-
 youtube_dl/extractor/aftonbladet.py         |  2 +-
 youtube_dl/extractor/aljazeera.py           |  2 +-
 youtube_dl/extractor/aol.py                 |  4 ++--
 youtube_dl/extractor/arte.py                |  2 +-
 youtube_dl/extractor/azubu.py               |  2 +-
 youtube_dl/extractor/baidu.py               |  2 +-
 youtube_dl/extractor/bbc.py                 |  2 +-
 youtube_dl/extractor/behindkink.py          |  2 +-
 youtube_dl/extractor/bilibili.py            |  2 +-
 youtube_dl/extractor/bokecc.py              |  2 +-
 youtube_dl/extractor/bpb.py                 |  2 +-
 youtube_dl/extractor/breakcom.py            |  2 +-
 youtube_dl/extractor/camdemy.py             |  4 ++--
 youtube_dl/extractor/cbsnews.py             |  4 ++--
 youtube_dl/extractor/cbssports.py           |  2 +-
 youtube_dl/extractor/cliphunter.py          |  2 +-
 youtube_dl/extractor/clipsyndicate.py       |  2 +-
 youtube_dl/extractor/clubic.py              |  2 +-
 youtube_dl/extractor/comcarcoff.py          |  2 +-
 youtube_dl/extractor/condenast.py           |  2 +-
 youtube_dl/extractor/cspan.py               |  2 +-
 youtube_dl/extractor/ctsnews.py             |  2 +-
 youtube_dl/extractor/dctp.py                |  2 +-
 youtube_dl/extractor/defense.py             |  2 +-
 youtube_dl/extractor/douyutv.py             |  2 +-
 youtube_dl/extractor/dplay.py               |  2 +-
 youtube_dl/extractor/dreisat.py             |  2 +-
 youtube_dl/extractor/dvtv.py                |  2 +-
 youtube_dl/extractor/echomsk.py             |  2 +-
 youtube_dl/extractor/exfm.py                |  2 +-
 youtube_dl/extractor/fc2.py                 |  2 +-
 youtube_dl/extractor/firstpost.py           |  2 +-
 youtube_dl/extractor/firsttv.py             |  2 +-
 youtube_dl/extractor/fktv.py                |  2 +-
 youtube_dl/extractor/footyroom.py           |  2 +-
 youtube_dl/extractor/foxgay.py              |  2 +-
 youtube_dl/extractor/franceinter.py         |  2 +-
 youtube_dl/extractor/freevideo.py           |  2 +-
 youtube_dl/extractor/gamekings.py           |  2 +-
 youtube_dl/extractor/gamespot.py            |  2 +-
 youtube_dl/extractor/gamestar.py            |  2 +-
 youtube_dl/extractor/gametrailers.py        |  2 +-
 youtube_dl/extractor/hotnewhiphop.py        |  2 +-
 youtube_dl/extractor/hypem.py               |  2 +-
 youtube_dl/extractor/imdb.py                |  4 ++--
 youtube_dl/extractor/iqiyi.py               |  2 +-
 youtube_dl/extractor/jadorecettepub.py      |  2 +-
 youtube_dl/extractor/jeuxvideo.py           |  2 +-
 youtube_dl/extractor/karaoketv.py           |  2 +-
 youtube_dl/extractor/karrierevideos.py      |  2 +-
 youtube_dl/extractor/kontrtube.py           |  2 +-
 youtube_dl/extractor/ku6.py                 |  2 +-
 youtube_dl/extractor/kusi.py                |  2 +-
 youtube_dl/extractor/kuwo.py                | 12 ++++++------
 youtube_dl/extractor/leeco.py               |  4 ++--
 youtube_dl/extractor/lifenews.py            |  4 ++--
 youtube_dl/extractor/limelight.py           |  6 +++---
 youtube_dl/extractor/m6.py                  |  2 +-
 youtube_dl/extractor/metacafe.py            |  2 +-
 youtube_dl/extractor/mit.py                 |  2 +-
 youtube_dl/extractor/mitele.py              |  2 +-
 youtube_dl/extractor/mooshare.py            |  2 +-
 youtube_dl/extractor/motherless.py          |  2 +-
 youtube_dl/extractor/motorsport.py          |  2 +-
 youtube_dl/extractor/myspass.py             |  2 +-
 youtube_dl/extractor/myvideo.py             |  2 +-
 youtube_dl/extractor/myvidster.py           |  2 +-
 youtube_dl/extractor/nationalgeographic.py  |  2 +-
 youtube_dl/extractor/nbc.py                 |  4 ++--
 youtube_dl/extractor/nextmedia.py           |  6 +++---
 youtube_dl/extractor/noco.py                |  2 +-
 youtube_dl/extractor/normalboots.py         |  2 +-
 youtube_dl/extractor/nova.py                |  2 +-
 youtube_dl/extractor/npr.py                 |  2 +-
 youtube_dl/extractor/ntvru.py               |  2 +-
 youtube_dl/extractor/orf.py                 |  6 +++---
 youtube_dl/extractor/philharmoniedeparis.py |  2 +-
 youtube_dl/extractor/photobucket.py         |  2 +-
 youtube_dl/extractor/pornhd.py              |  2 +-
 youtube_dl/extractor/pornovoisines.py       |  2 +-
 youtube_dl/extractor/pyvideo.py             |  2 +-
 youtube_dl/extractor/qqmusic.py             | 10 +++++-----
 youtube_dl/extractor/rai.py                 |  4 ++--
 youtube_dl/extractor/redtube.py             |  2 +-
 youtube_dl/extractor/ringtv.py              |  2 +-
 youtube_dl/extractor/rtve.py                |  4 ++--
 youtube_dl/extractor/ruhd.py                |  2 +-
 youtube_dl/extractor/rutube.py              |  6 +++---
 youtube_dl/extractor/screenjunkies.py       |  2 +-
 youtube_dl/extractor/senateisvp.py          |  2 +-
 youtube_dl/extractor/shared.py              |  2 +-
 youtube_dl/extractor/sport5.py              |  2 +-
 youtube_dl/extractor/ssa.py                 |  2 +-
 youtube_dl/extractor/sztvhu.py              |  2 +-
 youtube_dl/extractor/teamcoco.py            |  2 +-
 youtube_dl/extractor/tele13.py              |  2 +-
 youtube_dl/extractor/tf1.py                 |  2 +-
 youtube_dl/extractor/thvideo.py             |  2 +-
 youtube_dl/extractor/tinypic.py             |  2 +-
 youtube_dl/extractor/tlc.py                 |  2 +-
 youtube_dl/extractor/toypics.py             |  2 +-
 youtube_dl/extractor/traileraddict.py       |  2 +-
 youtube_dl/extractor/trollvids.py           |  2 +-
 youtube_dl/extractor/tumblr.py              |  2 +-
 youtube_dl/extractor/tv2.py                 |  4 ++--
 youtube_dl/extractor/tvc.py                 |  4 ++--
 youtube_dl/extractor/tvplay.py              |  2 +-
 youtube_dl/extractor/ubu.py                 |  2 +-
 youtube_dl/extractor/unistra.py             |  2 +-
 youtube_dl/extractor/vbox7.py               |  2 +-
 youtube_dl/extractor/veoh.py                |  2 +-
 youtube_dl/extractor/vesti.py               |  2 +-
 youtube_dl/extractor/vgtv.py                |  4 ++--
 youtube_dl/extractor/videott.py             |  2 +-
 youtube_dl/extractor/viidea.py              |  2 +-
 youtube_dl/extractor/vube.py                |  2 +-
 youtube_dl/extractor/vuclip.py              |  2 +-
 youtube_dl/extractor/walla.py               |  2 +-
 youtube_dl/extractor/wat.py                 |  2 +-
 youtube_dl/extractor/wdr.py                 |  2 +-
 youtube_dl/extractor/weiqitv.py             |  2 +-
 youtube_dl/extractor/wimp.py                |  2 +-
 youtube_dl/extractor/xbef.py                |  2 +-
 youtube_dl/extractor/yam.py                 |  2 +-
 youtube_dl/extractor/ynet.py                |  2 +-
 127 files changed, 156 insertions(+), 156 deletions(-)

diff --git a/youtube_dl/extractor/abc.py b/youtube_dl/extractor/abc.py
index 6a29e587f..b584277be 100644
--- a/youtube_dl/extractor/abc.py
+++ b/youtube_dl/extractor/abc.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 class ABCIE(InfoExtractor):
     IE_NAME = 'abc.net.au'
-    _VALID_URL = r'http://www\.abc\.net\.au/news/(?:[^/]+/){1,2}(?P<id>\d+)'
+    _VALID_URL = r'https?://www\.abc\.net\.au/news/(?:[^/]+/){1,2}(?P<id>\d+)'
 
     _TESTS = [{
         'url': 'http://www.abc.net.au/news/2014-11-05/australia-to-staff-ebola-treatment-centre-in-sierra-leone/5868334',
diff --git a/youtube_dl/extractor/addanime.py b/youtube_dl/extractor/addanime.py
index e3e6d2113..fb1cc02e1 100644
--- a/youtube_dl/extractor/addanime.py
+++ b/youtube_dl/extractor/addanime.py
@@ -16,7 +16,7 @@ from ..utils import (
 
 
 class AddAnimeIE(InfoExtractor):
-    _VALID_URL = r'http://(?:\w+\.)?add-anime\.net/(?:watch_video\.php\?(?:.*?)v=|video/)(?P<id>[\w_]+)'
+    _VALID_URL = r'https?://(?:\w+\.)?add-anime\.net/(?:watch_video\.php\?(?:.*?)v=|video/)(?P<id>[\w_]+)'
     _TESTS = [{
         'url': 'http://www.add-anime.net/watch_video.php?v=24MR3YO5SAS9',
         'md5': '72954ea10bc979ab5e2eb288b21425a0',
diff --git a/youtube_dl/extractor/aftonbladet.py b/youtube_dl/extractor/aftonbladet.py
index e0518cf26..d548592fe 100644
--- a/youtube_dl/extractor/aftonbladet.py
+++ b/youtube_dl/extractor/aftonbladet.py
@@ -6,7 +6,7 @@ from ..utils import int_or_none
 
 
 class AftonbladetIE(InfoExtractor):
-    _VALID_URL = r'http://tv\.aftonbladet\.se/abtv/articles/(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://tv\.aftonbladet\.se/abtv/articles/(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://tv.aftonbladet.se/abtv/articles/36015',
         'info_dict': {
diff --git a/youtube_dl/extractor/aljazeera.py b/youtube_dl/extractor/aljazeera.py
index cddcaa489..b081695d8 100644
--- a/youtube_dl/extractor/aljazeera.py
+++ b/youtube_dl/extractor/aljazeera.py
@@ -4,7 +4,7 @@ from .common import InfoExtractor
 
 
 class AlJazeeraIE(InfoExtractor):
-    _VALID_URL = r'http://www\.aljazeera\.com/programmes/.*?/(?P<id>[^/]+)\.html'
+    _VALID_URL = r'https?://www\.aljazeera\.com/programmes/.*?/(?P<id>[^/]+)\.html'
 
     _TEST = {
         'url': 'http://www.aljazeera.com/programmes/the-slum/2014/08/deliverance-201482883754237240.html',
diff --git a/youtube_dl/extractor/aol.py b/youtube_dl/extractor/aol.py
index b761b2cc4..95a99c6b0 100644
--- a/youtube_dl/extractor/aol.py
+++ b/youtube_dl/extractor/aol.py
@@ -5,7 +5,7 @@ from .common import InfoExtractor
 
 class AolIE(InfoExtractor):
     IE_NAME = 'on.aol.com'
-    _VALID_URL = r'(?:aol-video:|http://on\.aol\.com/video/.*-)(?P<id>[0-9]+)(?:$|\?)'
+    _VALID_URL = r'(?:aol-video:|https?://on\.aol\.com/video/.*-)(?P<id>[0-9]+)(?:$|\?)'
 
     _TESTS = [{
         'url': 'http://on.aol.com/video/u-s--official-warns-of-largest-ever-irs-phone-scam-518167793?icid=OnHomepageC2Wide_MustSee_Img',
@@ -25,7 +25,7 @@ class AolIE(InfoExtractor):
 
 class AolFeaturesIE(InfoExtractor):
     IE_NAME = 'features.aol.com'
-    _VALID_URL = r'http://features\.aol\.com/video/(?P<id>[^/?#]+)'
+    _VALID_URL = r'https?://features\.aol\.com/video/(?P<id>[^/?#]+)'
 
     _TESTS = [{
         'url': 'http://features.aol.com/video/behind-secret-second-careers-late-night-talk-show-hosts',
diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index 3e119e21b..ae0f27dcb 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -23,7 +23,7 @@ from ..utils import (
 
 
 class ArteTvIE(InfoExtractor):
-    _VALID_URL = r'http://videos\.arte\.tv/(?P<lang>fr|de|en|es)/.*-(?P<id>.*?)\.html'
+    _VALID_URL = r'https?://videos\.arte\.tv/(?P<lang>fr|de|en|es)/.*-(?P<id>.*?)\.html'
     IE_NAME = 'arte.tv'
 
     def _real_extract(self, url):
diff --git a/youtube_dl/extractor/azubu.py b/youtube_dl/extractor/azubu.py
index 011edf128..1805b7312 100644
--- a/youtube_dl/extractor/azubu.py
+++ b/youtube_dl/extractor/azubu.py
@@ -98,7 +98,7 @@ class AzubuIE(InfoExtractor):
 
 
 class AzubuLiveIE(InfoExtractor):
-    _VALID_URL = r'http://www.azubu.tv/(?P<id>[^/]+)$'
+    _VALID_URL = r'https?://www.azubu.tv/(?P<id>[^/]+)$'
 
     _TEST = {
         'url': 'http://www.azubu.tv/MarsTVMDLen',
diff --git a/youtube_dl/extractor/baidu.py b/youtube_dl/extractor/baidu.py
index 76b21e596..234a661d3 100644
--- a/youtube_dl/extractor/baidu.py
+++ b/youtube_dl/extractor/baidu.py
@@ -9,7 +9,7 @@ from ..utils import unescapeHTML
 
 class BaiduVideoIE(InfoExtractor):
     IE_DESC = '百度视频'
-    _VALID_URL = r'http://v\.baidu\.com/(?P<type>[a-z]+)/(?P<id>\d+)\.htm'
+    _VALID_URL = r'https?://v\.baidu\.com/(?P<type>[a-z]+)/(?P<id>\d+)\.htm'
     _TESTS = [{
         'url': 'http://v.baidu.com/comic/1069.htm?frp=bdbrand&q=%E4%B8%AD%E5%8D%8E%E5%B0%8F%E5%BD%93%E5%AE%B6',
         'info_dict': {
diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index c3176700a..2dfcee98d 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -942,7 +942,7 @@ class BBCIE(BBCCoUkIE):
 
 
 class BBCCoUkArticleIE(InfoExtractor):
-    _VALID_URL = r'http://www.bbc.co.uk/programmes/articles/(?P<id>[a-zA-Z0-9]+)'
+    _VALID_URL = r'https?://www.bbc.co.uk/programmes/articles/(?P<id>[a-zA-Z0-9]+)'
     IE_NAME = 'bbc.co.uk:article'
     IE_DESC = 'BBC articles'
 
diff --git a/youtube_dl/extractor/behindkink.py b/youtube_dl/extractor/behindkink.py
index 1bdc25812..9bca853b3 100644
--- a/youtube_dl/extractor/behindkink.py
+++ b/youtube_dl/extractor/behindkink.py
@@ -8,7 +8,7 @@ from ..utils import url_basename
 
 
 class BehindKinkIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?behindkink\.com/(?P<year>[0-9]{4})/(?P<month>[0-9]{2})/(?P<day>[0-9]{2})/(?P<id>[^/#?_]+)'
+    _VALID_URL = r'https?://(?:www\.)?behindkink\.com/(?P<year>[0-9]{4})/(?P<month>[0-9]{2})/(?P<day>[0-9]{2})/(?P<id>[^/#?_]+)'
     _TEST = {
         'url': 'http://www.behindkink.com/2014/12/05/what-are-you-passionate-about-marley-blaze/',
         'md5': '507b57d8fdcd75a41a9a7bdb7989c762',
diff --git a/youtube_dl/extractor/bilibili.py b/youtube_dl/extractor/bilibili.py
index 59beb11bc..8baff2041 100644
--- a/youtube_dl/extractor/bilibili.py
+++ b/youtube_dl/extractor/bilibili.py
@@ -14,7 +14,7 @@ from ..utils import (
 
 
 class BiliBiliIE(InfoExtractor):
-    _VALID_URL = r'http://www\.bilibili\.(?:tv|com)/video/av(?P<id>\d+)(?:/index_(?P<page_num>\d+).html)?'
+    _VALID_URL = r'https?://www\.bilibili\.(?:tv|com)/video/av(?P<id>\d+)(?:/index_(?P<page_num>\d+).html)?'
 
     _TESTS = [{
         'url': 'http://www.bilibili.tv/video/av1074402/',
diff --git a/youtube_dl/extractor/bokecc.py b/youtube_dl/extractor/bokecc.py
index 122a1cbb6..86a7f4d7d 100644
--- a/youtube_dl/extractor/bokecc.py
+++ b/youtube_dl/extractor/bokecc.py
@@ -33,7 +33,7 @@ class BokeCCBaseIE(InfoExtractor):
 
 class BokeCCIE(BokeCCBaseIE):
     _IE_DESC = 'CC视频'
-    _VALID_URL = r'http://union\.bokecc\.com/playvideo\.bo\?(?P<query>.*)'
+    _VALID_URL = r'https?://union\.bokecc\.com/playvideo\.bo\?(?P<query>.*)'
 
     _TESTS = [{
         'url': 'http://union.bokecc.com/playvideo.bo?vid=E44D40C15E65EA30&uid=CD0C5D3C8614B28B',
diff --git a/youtube_dl/extractor/bpb.py b/youtube_dl/extractor/bpb.py
index c28e72927..6ad45a1e6 100644
--- a/youtube_dl/extractor/bpb.py
+++ b/youtube_dl/extractor/bpb.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 class BpbIE(InfoExtractor):
     IE_DESC = 'Bundeszentrale für politische Bildung'
-    _VALID_URL = r'http://www\.bpb\.de/mediathek/(?P<id>[0-9]+)/'
+    _VALID_URL = r'https?://www\.bpb\.de/mediathek/(?P<id>[0-9]+)/'
 
     _TEST = {
         'url': 'http://www.bpb.de/mediathek/297/joachim-gauck-zu-1989-und-die-erinnerung-an-die-ddr',
diff --git a/youtube_dl/extractor/breakcom.py b/youtube_dl/extractor/breakcom.py
index aa08051b1..725859b4d 100644
--- a/youtube_dl/extractor/breakcom.py
+++ b/youtube_dl/extractor/breakcom.py
@@ -11,7 +11,7 @@ from ..utils import (
 
 
 class BreakIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?break\.com/video/(?:[^/]+/)*.+-(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?break\.com/video/(?:[^/]+/)*.+-(?P<id>\d+)'
     _TESTS = [{
         'url': 'http://www.break.com/video/when-girls-act-like-guys-2468056',
         'info_dict': {
diff --git a/youtube_dl/extractor/camdemy.py b/youtube_dl/extractor/camdemy.py
index 897f3a104..dd4d96cec 100644
--- a/youtube_dl/extractor/camdemy.py
+++ b/youtube_dl/extractor/camdemy.py
@@ -16,7 +16,7 @@ from ..utils import (
 
 
 class CamdemyIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?camdemy\.com/media/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?camdemy\.com/media/(?P<id>\d+)'
     _TESTS = [{
         # single file
         'url': 'http://www.camdemy.com/media/5181/',
@@ -104,7 +104,7 @@ class CamdemyIE(InfoExtractor):
 
 
 class CamdemyFolderIE(InfoExtractor):
-    _VALID_URL = r'http://www.camdemy.com/folder/(?P<id>\d+)'
+    _VALID_URL = r'https?://www.camdemy.com/folder/(?P<id>\d+)'
     _TESTS = [{
         # links with trailing slash
         'url': 'http://www.camdemy.com/folder/450',
diff --git a/youtube_dl/extractor/cbsnews.py b/youtube_dl/extractor/cbsnews.py
index 8ddcc5097..f23bac9a1 100644
--- a/youtube_dl/extractor/cbsnews.py
+++ b/youtube_dl/extractor/cbsnews.py
@@ -11,7 +11,7 @@ from ..utils import (
 
 class CBSNewsIE(ThePlatformIE):
     IE_DESC = 'CBS News'
-    _VALID_URL = r'http://(?:www\.)?cbsnews\.com/(?:news|videos)/(?P<id>[\da-z_-]+)'
+    _VALID_URL = r'https?://(?:www\.)?cbsnews\.com/(?:news|videos)/(?P<id>[\da-z_-]+)'
 
     _TESTS = [
         {
@@ -96,7 +96,7 @@ class CBSNewsIE(ThePlatformIE):
 
 class CBSNewsLiveVideoIE(InfoExtractor):
     IE_DESC = 'CBS News Live Videos'
-    _VALID_URL = r'http://(?:www\.)?cbsnews\.com/live/video/(?P<id>[\da-z_-]+)'
+    _VALID_URL = r'https?://(?:www\.)?cbsnews\.com/live/video/(?P<id>[\da-z_-]+)'
 
     _TEST = {
         'url': 'http://www.cbsnews.com/live/video/clinton-sanders-prepare-to-face-off-in-nh/',
diff --git a/youtube_dl/extractor/cbssports.py b/youtube_dl/extractor/cbssports.py
index ae47e74cc..549ae32f3 100644
--- a/youtube_dl/extractor/cbssports.py
+++ b/youtube_dl/extractor/cbssports.py
@@ -6,7 +6,7 @@ from .common import InfoExtractor
 
 
 class CBSSportsIE(InfoExtractor):
-    _VALID_URL = r'http://www\.cbssports\.com/video/player/(?P<section>[^/]+)/(?P<id>[^/]+)'
+    _VALID_URL = r'https?://www\.cbssports\.com/video/player/(?P<section>[^/]+)/(?P<id>[^/]+)'
 
     _TEST = {
         'url': 'http://www.cbssports.com/video/player/tennis/318462531970/0/us-open-flashbacks-1990s',
diff --git a/youtube_dl/extractor/cliphunter.py b/youtube_dl/extractor/cliphunter.py
index 2996b6b09..19f8b397e 100644
--- a/youtube_dl/extractor/cliphunter.py
+++ b/youtube_dl/extractor/cliphunter.py
@@ -19,7 +19,7 @@ def _decode(s):
 class CliphunterIE(InfoExtractor):
     IE_NAME = 'cliphunter'
 
-    _VALID_URL = r'''(?x)http://(?:www\.)?cliphunter\.com/w/
+    _VALID_URL = r'''(?x)https?://(?:www\.)?cliphunter\.com/w/
         (?P<id>[0-9]+)/
         (?P<seo>.+?)(?:$|[#\?])
     '''
diff --git a/youtube_dl/extractor/clipsyndicate.py b/youtube_dl/extractor/clipsyndicate.py
index 8306d6fb7..0b6ad895f 100644
--- a/youtube_dl/extractor/clipsyndicate.py
+++ b/youtube_dl/extractor/clipsyndicate.py
@@ -8,7 +8,7 @@ from ..utils import (
 
 
 class ClipsyndicateIE(InfoExtractor):
-    _VALID_URL = r'http://(?:chic|www)\.clipsyndicate\.com/video/play(list/\d+)?/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:chic|www)\.clipsyndicate\.com/video/play(list/\d+)?/(?P<id>\d+)'
 
     _TESTS = [{
         'url': 'http://www.clipsyndicate.com/video/play/4629301/brick_briscoe',
diff --git a/youtube_dl/extractor/clubic.py b/youtube_dl/extractor/clubic.py
index 1dfa7c12e..2fba93543 100644
--- a/youtube_dl/extractor/clubic.py
+++ b/youtube_dl/extractor/clubic.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 
 class ClubicIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?clubic\.com/video/(?:[^/]+/)*video.*-(?P<id>[0-9]+)\.html'
+    _VALID_URL = r'https?://(?:www\.)?clubic\.com/video/(?:[^/]+/)*video.*-(?P<id>[0-9]+)\.html'
 
     _TESTS = [{
         'url': 'http://www.clubic.com/video/clubic-week/video-clubic-week-2-0-le-fbi-se-lance-dans-la-photo-d-identite-448474.html',
diff --git a/youtube_dl/extractor/comcarcoff.py b/youtube_dl/extractor/comcarcoff.py
index 7dff68492..e697d1410 100644
--- a/youtube_dl/extractor/comcarcoff.py
+++ b/youtube_dl/extractor/comcarcoff.py
@@ -11,7 +11,7 @@ from ..utils import (
 
 
 class ComCarCoffIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?comediansincarsgettingcoffee\.com/(?P<id>[a-z0-9\-]*)'
+    _VALID_URL = r'https?://(?:www\.)?comediansincarsgettingcoffee\.com/(?P<id>[a-z0-9\-]*)'
     _TESTS = [{
         'url': 'http://comediansincarsgettingcoffee.com/miranda-sings-happy-thanksgiving-miranda/',
         'info_dict': {
diff --git a/youtube_dl/extractor/condenast.py b/youtube_dl/extractor/condenast.py
index 6f92ae2ed..054978ff2 100644
--- a/youtube_dl/extractor/condenast.py
+++ b/youtube_dl/extractor/condenast.py
@@ -45,7 +45,7 @@ class CondeNastIE(InfoExtractor):
         'wmagazine': 'W Magazine',
     }
 
-    _VALID_URL = r'http://(?:video|www|player)\.(?P<site>%s)\.com/(?P<type>watch|series|video|embed(?:js)?)/(?P<id>[^/?#]+)' % '|'.join(_SITES.keys())
+    _VALID_URL = r'https?://(?:video|www|player)\.(?P<site>%s)\.com/(?P<type>watch|series|video|embed(?:js)?)/(?P<id>[^/?#]+)' % '|'.join(_SITES.keys())
     IE_DESC = 'Condé Nast media group: %s' % ', '.join(sorted(_SITES.values()))
 
     EMBED_URL = r'(?:https?:)?//player\.(?P<site>%s)\.com/(?P<type>embed(?:js)?)/.+?' % '|'.join(_SITES.keys())
diff --git a/youtube_dl/extractor/cspan.py b/youtube_dl/extractor/cspan.py
index b8b9d058d..84b36f44c 100644
--- a/youtube_dl/extractor/cspan.py
+++ b/youtube_dl/extractor/cspan.py
@@ -15,7 +15,7 @@ from .senateisvp import SenateISVPIE
 
 
 class CSpanIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?c-span\.org/video/\?(?P<id>[0-9a-f]+)'
+    _VALID_URL = r'https?://(?:www\.)?c-span\.org/video/\?(?P<id>[0-9a-f]+)'
     IE_DESC = 'C-SPAN'
     _TESTS = [{
         'url': 'http://www.c-span.org/video/?313572-1/HolderonV',
diff --git a/youtube_dl/extractor/ctsnews.py b/youtube_dl/extractor/ctsnews.py
index 45049bf37..1622fc844 100644
--- a/youtube_dl/extractor/ctsnews.py
+++ b/youtube_dl/extractor/ctsnews.py
@@ -8,7 +8,7 @@ from ..utils import parse_iso8601, ExtractorError
 class CtsNewsIE(InfoExtractor):
     IE_DESC = '華視新聞'
     # https connection failed (Connection reset)
-    _VALID_URL = r'http://news\.cts\.com\.tw/[a-z]+/[a-z]+/\d+/(?P<id>\d+)\.html'
+    _VALID_URL = r'https?://news\.cts\.com\.tw/[a-z]+/[a-z]+/\d+/(?P<id>\d+)\.html'
     _TESTS = [{
         'url': 'http://news.cts.com.tw/cts/international/201501/201501291578109.html',
         'md5': 'a9875cb790252b08431186d741beaabe',
diff --git a/youtube_dl/extractor/dctp.py b/youtube_dl/extractor/dctp.py
index aa2c09eb6..9099f5046 100644
--- a/youtube_dl/extractor/dctp.py
+++ b/youtube_dl/extractor/dctp.py
@@ -6,7 +6,7 @@ from ..compat import compat_str
 
 
 class DctpTvIE(InfoExtractor):
-    _VALID_URL = r'http://www.dctp.tv/(#/)?filme/(?P<id>.+?)/$'
+    _VALID_URL = r'https?://www.dctp.tv/(#/)?filme/(?P<id>.+?)/$'
     _TEST = {
         'url': 'http://www.dctp.tv/filme/videoinstallation-fuer-eine-kaufhausfassade/',
         'info_dict': {
diff --git a/youtube_dl/extractor/defense.py b/youtube_dl/extractor/defense.py
index 98e3aedfd..9fe144e14 100644
--- a/youtube_dl/extractor/defense.py
+++ b/youtube_dl/extractor/defense.py
@@ -5,7 +5,7 @@ from .common import InfoExtractor
 
 class DefenseGouvFrIE(InfoExtractor):
     IE_NAME = 'defense.gouv.fr'
-    _VALID_URL = r'http://.*?\.defense\.gouv\.fr/layout/set/ligthboxvideo/base-de-medias/webtv/(?P<id>[^/?#]*)'
+    _VALID_URL = r'https?://.*?\.defense\.gouv\.fr/layout/set/ligthboxvideo/base-de-medias/webtv/(?P<id>[^/?#]*)'
 
     _TEST = {
         'url': 'http://www.defense.gouv.fr/layout/set/ligthboxvideo/base-de-medias/webtv/attaque-chimique-syrienne-du-21-aout-2013-1',
diff --git a/youtube_dl/extractor/douyutv.py b/youtube_dl/extractor/douyutv.py
index bdc768c78..bcb670945 100644
--- a/youtube_dl/extractor/douyutv.py
+++ b/youtube_dl/extractor/douyutv.py
@@ -10,7 +10,7 @@ from ..compat import (compat_str, compat_basestring)
 
 class DouyuTVIE(InfoExtractor):
     IE_DESC = '斗鱼'
-    _VALID_URL = r'http://(?:www\.)?douyutv\.com/(?P<id>[A-Za-z0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?douyutv\.com/(?P<id>[A-Za-z0-9]+)'
     _TESTS = [{
         'url': 'http://www.douyutv.com/iseven',
         'info_dict': {
diff --git a/youtube_dl/extractor/dplay.py b/youtube_dl/extractor/dplay.py
index a638c827c..1e7dcada6 100644
--- a/youtube_dl/extractor/dplay.py
+++ b/youtube_dl/extractor/dplay.py
@@ -10,7 +10,7 @@ from ..utils import int_or_none
 
 
 class DPlayIE(InfoExtractor):
-    _VALID_URL = r'http://(?P<domain>it\.dplay\.com|www\.dplay\.(?:dk|se|no))/[^/]+/(?P<id>[^/?#]+)'
+    _VALID_URL = r'https?://(?P<domain>it\.dplay\.com|www\.dplay\.(?:dk|se|no))/[^/]+/(?P<id>[^/?#]+)'
 
     _TESTS = [{
         'url': 'http://it.dplay.com/take-me-out/stagione-1-episodio-25/',
diff --git a/youtube_dl/extractor/dreisat.py b/youtube_dl/extractor/dreisat.py
index 028144f20..0040e70d4 100644
--- a/youtube_dl/extractor/dreisat.py
+++ b/youtube_dl/extractor/dreisat.py
@@ -7,7 +7,7 @@ from .zdf import ZDFIE
 
 class DreiSatIE(ZDFIE):
     IE_NAME = '3sat'
-    _VALID_URL = r'(?:http://)?(?:www\.)?3sat\.de/mediathek/(?:index\.php|mediathek\.php)?\?(?:(?:mode|display)=[^&]+&)*obj=(?P<id>[0-9]+)$'
+    _VALID_URL = r'(?:https?://)?(?:www\.)?3sat\.de/mediathek/(?:index\.php|mediathek\.php)?\?(?:(?:mode|display)=[^&]+&)*obj=(?P<id>[0-9]+)$'
     _TESTS = [
         {
             'url': 'http://www.3sat.de/mediathek/index.php?mode=play&obj=45918',
diff --git a/youtube_dl/extractor/dvtv.py b/youtube_dl/extractor/dvtv.py
index c1a4bc757..974c69dbc 100644
--- a/youtube_dl/extractor/dvtv.py
+++ b/youtube_dl/extractor/dvtv.py
@@ -15,7 +15,7 @@ class DVTVIE(InfoExtractor):
     IE_NAME = 'dvtv'
     IE_DESC = 'http://video.aktualne.cz/'
 
-    _VALID_URL = r'http://video\.aktualne\.cz/(?:[^/]+/)+r~(?P<id>[0-9a-f]{32})'
+    _VALID_URL = r'https?://video\.aktualne\.cz/(?:[^/]+/)+r~(?P<id>[0-9a-f]{32})'
 
     _TESTS = [{
         'url': 'http://video.aktualne.cz/dvtv/vondra-o-ceskem-stoleti-pri-pohledu-na-havla-mi-bylo-trapne/r~e5efe9ca855511e4833a0025900fea04/',
diff --git a/youtube_dl/extractor/echomsk.py b/youtube_dl/extractor/echomsk.py
index d2d94049d..6b7cc652f 100644
--- a/youtube_dl/extractor/echomsk.py
+++ b/youtube_dl/extractor/echomsk.py
@@ -7,7 +7,7 @@ from .common import InfoExtractor
 
 
 class EchoMskIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?echo\.msk\.ru/sounds/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?echo\.msk\.ru/sounds/(?P<id>\d+)'
     _TEST = {
         'url': 'http://www.echo.msk.ru/sounds/1464134.html',
         'md5': '2e44b3b78daff5b458e4dbc37f191f7c',
diff --git a/youtube_dl/extractor/exfm.py b/youtube_dl/extractor/exfm.py
index 0c0fe6d65..09ed4f2b5 100644
--- a/youtube_dl/extractor/exfm.py
+++ b/youtube_dl/extractor/exfm.py
@@ -8,7 +8,7 @@ from .common import InfoExtractor
 class ExfmIE(InfoExtractor):
     IE_NAME = 'exfm'
     IE_DESC = 'ex.fm'
-    _VALID_URL = r'http://(?:www\.)?ex\.fm/song/(?P<id>[^/]+)'
+    _VALID_URL = r'https?://(?:www\.)?ex\.fm/song/(?P<id>[^/]+)'
     _SOUNDCLOUD_URL = r'http://(?:www\.)?api\.soundcloud\.com/tracks/([^/]+)/stream'
     _TESTS = [
         {
diff --git a/youtube_dl/extractor/fc2.py b/youtube_dl/extractor/fc2.py
index 9580f5c0c..508684d2e 100644
--- a/youtube_dl/extractor/fc2.py
+++ b/youtube_dl/extractor/fc2.py
@@ -17,7 +17,7 @@ from ..utils import (
 
 
 class FC2IE(InfoExtractor):
-    _VALID_URL = r'^http://video\.fc2\.com/(?:[^/]+/)*content/(?P<id>[^/]+)'
+    _VALID_URL = r'^https?://video\.fc2\.com/(?:[^/]+/)*content/(?P<id>[^/]+)'
     IE_NAME = 'fc2'
     _NETRC_MACHINE = 'fc2'
     _TESTS = [{
diff --git a/youtube_dl/extractor/firstpost.py b/youtube_dl/extractor/firstpost.py
index 298227d57..e8936cb24 100644
--- a/youtube_dl/extractor/firstpost.py
+++ b/youtube_dl/extractor/firstpost.py
@@ -4,7 +4,7 @@ from .common import InfoExtractor
 
 
 class FirstpostIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?firstpost\.com/[^/]+/.*-(?P<id>[0-9]+)\.html'
+    _VALID_URL = r'https?://(?:www\.)?firstpost\.com/[^/]+/.*-(?P<id>[0-9]+)\.html'
 
     _TEST = {
         'url': 'http://www.firstpost.com/india/india-to-launch-indigenous-aircraft-carrier-monday-1025403.html',
diff --git a/youtube_dl/extractor/firsttv.py b/youtube_dl/extractor/firsttv.py
index 510d4b108..98b165143 100644
--- a/youtube_dl/extractor/firsttv.py
+++ b/youtube_dl/extractor/firsttv.py
@@ -8,7 +8,7 @@ from ..utils import int_or_none
 class FirstTVIE(InfoExtractor):
     IE_NAME = '1tv'
     IE_DESC = 'Первый канал'
-    _VALID_URL = r'http://(?:www\.)?1tv\.ru/(?:[^/]+/)+(?P<id>.+)'
+    _VALID_URL = r'https?://(?:www\.)?1tv\.ru/(?:[^/]+/)+(?P<id>.+)'
 
     _TESTS = [{
         'url': 'http://www.1tv.ru/videoarchive/73390',
diff --git a/youtube_dl/extractor/fktv.py b/youtube_dl/extractor/fktv.py
index 5f6e65dae..a3a291599 100644
--- a/youtube_dl/extractor/fktv.py
+++ b/youtube_dl/extractor/fktv.py
@@ -10,7 +10,7 @@ from ..utils import (
 
 class FKTVIE(InfoExtractor):
     IE_NAME = 'fernsehkritik.tv'
-    _VALID_URL = r'http://(?:www\.)?fernsehkritik\.tv/folge-(?P<id>[0-9]+)(?:/.*)?'
+    _VALID_URL = r'https?://(?:www\.)?fernsehkritik\.tv/folge-(?P<id>[0-9]+)(?:/.*)?'
 
     _TEST = {
         'url': 'http://fernsehkritik.tv/folge-1',
diff --git a/youtube_dl/extractor/footyroom.py b/youtube_dl/extractor/footyroom.py
index 370fd006f..d2503ae2e 100644
--- a/youtube_dl/extractor/footyroom.py
+++ b/youtube_dl/extractor/footyroom.py
@@ -5,7 +5,7 @@ from .common import InfoExtractor
 
 
 class FootyRoomIE(InfoExtractor):
-    _VALID_URL = r'http://footyroom\.com/(?P<id>[^/]+)'
+    _VALID_URL = r'https?://footyroom\.com/(?P<id>[^/]+)'
     _TESTS = [{
         'url': 'http://footyroom.com/schalke-04-0-2-real-madrid-2015-02/',
         'info_dict': {
diff --git a/youtube_dl/extractor/foxgay.py b/youtube_dl/extractor/foxgay.py
index 08b8ea362..70c1a815d 100644
--- a/youtube_dl/extractor/foxgay.py
+++ b/youtube_dl/extractor/foxgay.py
@@ -4,7 +4,7 @@ from .common import InfoExtractor
 
 
 class FoxgayIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?foxgay\.com/videos/(?:\S+-)?(?P<id>\d+)\.shtml'
+    _VALID_URL = r'https?://(?:www\.)?foxgay\.com/videos/(?:\S+-)?(?P<id>\d+)\.shtml'
     _TEST = {
         'url': 'http://foxgay.com/videos/fuck-turkish-style-2582.shtml',
         'md5': '80d72beab5d04e1655a56ad37afe6841',
diff --git a/youtube_dl/extractor/franceinter.py b/youtube_dl/extractor/franceinter.py
index 0388ba00c..2369f868d 100644
--- a/youtube_dl/extractor/franceinter.py
+++ b/youtube_dl/extractor/franceinter.py
@@ -6,7 +6,7 @@ from ..utils import int_or_none
 
 
 class FranceInterIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?franceinter\.fr/player/reecouter\?play=(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?franceinter\.fr/player/reecouter\?play=(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://www.franceinter.fr/player/reecouter?play=793962',
         'md5': '4764932e466e6f6c79c317d2e74f6884',
diff --git a/youtube_dl/extractor/freevideo.py b/youtube_dl/extractor/freevideo.py
index c7bec027b..cd8423a6f 100644
--- a/youtube_dl/extractor/freevideo.py
+++ b/youtube_dl/extractor/freevideo.py
@@ -5,7 +5,7 @@ from ..utils import ExtractorError
 
 
 class FreeVideoIE(InfoExtractor):
-    _VALID_URL = r'^http://www.freevideo.cz/vase-videa/(?P<id>[^.]+)\.html(?:$|[?#])'
+    _VALID_URL = r'^https?://www.freevideo.cz/vase-videa/(?P<id>[^.]+)\.html(?:$|[?#])'
 
     _TEST = {
         'url': 'http://www.freevideo.cz/vase-videa/vysukany-zadecek-22033.html',
diff --git a/youtube_dl/extractor/gamekings.py b/youtube_dl/extractor/gamekings.py
index f6b9046f9..cbcddcb7c 100644
--- a/youtube_dl/extractor/gamekings.py
+++ b/youtube_dl/extractor/gamekings.py
@@ -10,7 +10,7 @@ from .youtube import YoutubeIE
 
 
 class GamekingsIE(InfoExtractor):
-    _VALID_URL = r'http://www\.gamekings\.nl/(?:videos|nieuws)/(?P<id>[^/]+)'
+    _VALID_URL = r'https?://www\.gamekings\.nl/(?:videos|nieuws)/(?P<id>[^/]+)'
     _TESTS = [{
         # YouTube embed video
         'url': 'http://www.gamekings.nl/videos/phoenix-wright-ace-attorney-dual-destinies-review/',
diff --git a/youtube_dl/extractor/gamespot.py b/youtube_dl/extractor/gamespot.py
index b3f1bafcc..4ffdd7515 100644
--- a/youtube_dl/extractor/gamespot.py
+++ b/youtube_dl/extractor/gamespot.py
@@ -14,7 +14,7 @@ from ..utils import (
 
 
 class GameSpotIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?gamespot\.com/.*-(?P<id>\d+)/?'
+    _VALID_URL = r'https?://(?:www\.)?gamespot\.com/.*-(?P<id>\d+)/?'
     _TESTS = [{
         'url': 'http://www.gamespot.com/videos/arma-3-community-guide-sitrep-i/2300-6410818/',
         'md5': 'b2a30deaa8654fcccd43713a6b6a4825',
diff --git a/youtube_dl/extractor/gamestar.py b/youtube_dl/extractor/gamestar.py
index 590ccf526..69058a583 100644
--- a/youtube_dl/extractor/gamestar.py
+++ b/youtube_dl/extractor/gamestar.py
@@ -13,7 +13,7 @@ from ..utils import (
 
 
 class GameStarIE(InfoExtractor):
-    _VALID_URL = r'http://www\.gamestar\.de/videos/.*,(?P<id>[0-9]+)\.html'
+    _VALID_URL = r'https?://www\.gamestar\.de/videos/.*,(?P<id>[0-9]+)\.html'
     _TEST = {
         'url': 'http://www.gamestar.de/videos/trailer,3/hobbit-3-die-schlacht-der-fuenf-heere,76110.html',
         'md5': '96974ecbb7fd8d0d20fca5a00810cea7',
diff --git a/youtube_dl/extractor/gametrailers.py b/youtube_dl/extractor/gametrailers.py
index c3f031d9c..1e7948ab8 100644
--- a/youtube_dl/extractor/gametrailers.py
+++ b/youtube_dl/extractor/gametrailers.py
@@ -9,7 +9,7 @@ from ..utils import (
 
 
 class GametrailersIE(InfoExtractor):
-    _VALID_URL = r'http://www\.gametrailers\.com/videos/view/[^/]+/(?P<id>.+)'
+    _VALID_URL = r'https?://www\.gametrailers\.com/videos/view/[^/]+/(?P<id>.+)'
 
     _TEST = {
         'url': 'http://www.gametrailers.com/videos/view/gametrailers-com/116437-Just-Cause-3-Review',
diff --git a/youtube_dl/extractor/hotnewhiphop.py b/youtube_dl/extractor/hotnewhiphop.py
index 31e219945..efc3e8429 100644
--- a/youtube_dl/extractor/hotnewhiphop.py
+++ b/youtube_dl/extractor/hotnewhiphop.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 
 class HotNewHipHopIE(InfoExtractor):
-    _VALID_URL = r'http://www\.hotnewhiphop\.com/.*\.(?P<id>.*)\.html'
+    _VALID_URL = r'https?://www\.hotnewhiphop\.com/.*\.(?P<id>.*)\.html'
     _TEST = {
         'url': 'http://www.hotnewhiphop.com/freddie-gibbs-lay-it-down-song.1435540.html',
         'md5': '2c2cd2f76ef11a9b3b581e8b232f3d96',
diff --git a/youtube_dl/extractor/hypem.py b/youtube_dl/extractor/hypem.py
index b3706fe6d..e0ab31802 100644
--- a/youtube_dl/extractor/hypem.py
+++ b/youtube_dl/extractor/hypem.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 
 class HypemIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?hypem\.com/track/(?P<id>[^/]+)/'
+    _VALID_URL = r'https?://(?:www\.)?hypem\.com/track/(?P<id>[^/]+)/'
     _TEST = {
         'url': 'http://hypem.com/track/1v6ga/BODYWORK+-+TAME',
         'md5': 'b9cc91b5af8995e9f0c1cee04c575828',
diff --git a/youtube_dl/extractor/imdb.py b/youtube_dl/extractor/imdb.py
index b61b2dc4e..8bed8ccd0 100644
--- a/youtube_dl/extractor/imdb.py
+++ b/youtube_dl/extractor/imdb.py
@@ -12,7 +12,7 @@ from ..utils import (
 class ImdbIE(InfoExtractor):
     IE_NAME = 'imdb'
     IE_DESC = 'Internet Movie Database trailers'
-    _VALID_URL = r'http://(?:www|m)\.imdb\.com/video/imdb/vi(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www|m)\.imdb\.com/video/imdb/vi(?P<id>\d+)'
 
     _TEST = {
         'url': 'http://www.imdb.com/video/imdb/vi2524815897',
@@ -70,7 +70,7 @@ class ImdbIE(InfoExtractor):
 class ImdbListIE(InfoExtractor):
     IE_NAME = 'imdb:list'
     IE_DESC = 'Internet Movie Database lists'
-    _VALID_URL = r'http://www\.imdb\.com/list/(?P<id>[\da-zA-Z_-]{11})'
+    _VALID_URL = r'https?://www\.imdb\.com/list/(?P<id>[\da-zA-Z_-]{11})'
     _TEST = {
         'url': 'http://www.imdb.com/list/JFs9NWw6XI0',
         'info_dict': {
diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index e7c0cb3f6..1a4c64713 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -165,7 +165,7 @@ class IqiyiIE(InfoExtractor):
     IE_NAME = 'iqiyi'
     IE_DESC = '爱奇艺'
 
-    _VALID_URL = r'http://(?:[^.]+\.)?iqiyi\.com/.+\.html'
+    _VALID_URL = r'https?://(?:[^.]+\.)?iqiyi\.com/.+\.html'
 
     _NETRC_MACHINE = 'iqiyi'
 
diff --git a/youtube_dl/extractor/jadorecettepub.py b/youtube_dl/extractor/jadorecettepub.py
index 063e86de4..158c09a33 100644
--- a/youtube_dl/extractor/jadorecettepub.py
+++ b/youtube_dl/extractor/jadorecettepub.py
@@ -9,7 +9,7 @@ from .youtube import YoutubeIE
 
 
 class JadoreCettePubIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?jadorecettepub\.com/[0-9]{4}/[0-9]{2}/(?P<id>.*?)\.html'
+    _VALID_URL = r'https?://(?:www\.)?jadorecettepub\.com/[0-9]{4}/[0-9]{2}/(?P<id>.*?)\.html'
 
     _TEST = {
         'url': 'http://www.jadorecettepub.com/2010/12/star-wars-massacre-par-les-japonais.html',
diff --git a/youtube_dl/extractor/jeuxvideo.py b/youtube_dl/extractor/jeuxvideo.py
index 137db873c..1a4227f6b 100644
--- a/youtube_dl/extractor/jeuxvideo.py
+++ b/youtube_dl/extractor/jeuxvideo.py
@@ -8,7 +8,7 @@ from .common import InfoExtractor
 
 
 class JeuxVideoIE(InfoExtractor):
-    _VALID_URL = r'http://.*?\.jeuxvideo\.com/.*/(.*?)\.htm'
+    _VALID_URL = r'https?://.*?\.jeuxvideo\.com/.*/(.*?)\.htm'
 
     _TESTS = [{
         'url': 'http://www.jeuxvideo.com/reportages-videos-jeux/0004/00046170/tearaway-playstation-vita-gc-2013-tearaway-nous-presente-ses-papiers-d-identite-00115182.htm',
diff --git a/youtube_dl/extractor/karaoketv.py b/youtube_dl/extractor/karaoketv.py
index 06daf5a89..b4c30b7f3 100644
--- a/youtube_dl/extractor/karaoketv.py
+++ b/youtube_dl/extractor/karaoketv.py
@@ -9,7 +9,7 @@ from ..utils import (
 
 
 class KaraoketvIE(InfoExtractor):
-    _VALID_URL = r'http://karaoketv\.co\.il/\?container=songs&id=(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://karaoketv\.co\.il/\?container=songs&id=(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://karaoketv.co.il/?container=songs&id=171568',
         'info_dict': {
diff --git a/youtube_dl/extractor/karrierevideos.py b/youtube_dl/extractor/karrierevideos.py
index bed94bc93..2cb04e533 100644
--- a/youtube_dl/extractor/karrierevideos.py
+++ b/youtube_dl/extractor/karrierevideos.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 
 class KarriereVideosIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?karrierevideos\.at(?:/[^/]+)+/(?P<id>[^/]+)'
+    _VALID_URL = r'https?://(?:www\.)?karrierevideos\.at(?:/[^/]+)+/(?P<id>[^/]+)'
     _TESTS = [{
         'url': 'http://www.karrierevideos.at/berufsvideos/mittlere-hoehere-schulen/altenpflegerin',
         'info_dict': {
diff --git a/youtube_dl/extractor/kontrtube.py b/youtube_dl/extractor/kontrtube.py
index a59c529f4..704bd7b34 100644
--- a/youtube_dl/extractor/kontrtube.py
+++ b/youtube_dl/extractor/kontrtube.py
@@ -13,7 +13,7 @@ from ..utils import (
 class KontrTubeIE(InfoExtractor):
     IE_NAME = 'kontrtube'
     IE_DESC = 'KontrTube.ru - Труба зовёт'
-    _VALID_URL = r'http://(?:www\.)?kontrtube\.ru/videos/(?P<id>\d+)/(?P<display_id>[^/]+)/'
+    _VALID_URL = r'https?://(?:www\.)?kontrtube\.ru/videos/(?P<id>\d+)/(?P<display_id>[^/]+)/'
 
     _TEST = {
         'url': 'http://www.kontrtube.ru/videos/2678/nad-olimpiyskoy-derevney-v-sochi-podnyat-rossiyskiy-flag/',
diff --git a/youtube_dl/extractor/ku6.py b/youtube_dl/extractor/ku6.py
index a602980a1..a574408e5 100644
--- a/youtube_dl/extractor/ku6.py
+++ b/youtube_dl/extractor/ku6.py
@@ -4,7 +4,7 @@ from .common import InfoExtractor
 
 
 class Ku6IE(InfoExtractor):
-    _VALID_URL = r'http://v\.ku6\.com/show/(?P<id>[a-zA-Z0-9\-\_]+)(?:\.)*html'
+    _VALID_URL = r'https?://v\.ku6\.com/show/(?P<id>[a-zA-Z0-9\-\_]+)(?:\.)*html'
     _TEST = {
         'url': 'http://v.ku6.com/show/JG-8yS14xzBr4bCn1pu0xw...html',
         'md5': '01203549b9efbb45f4b87d55bdea1ed1',
diff --git a/youtube_dl/extractor/kusi.py b/youtube_dl/extractor/kusi.py
index 931f34c9b..12cc56e44 100644
--- a/youtube_dl/extractor/kusi.py
+++ b/youtube_dl/extractor/kusi.py
@@ -16,7 +16,7 @@ from ..utils import (
 
 
 class KUSIIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?kusi\.com/(?P<path>story/.+|video\?clipId=(?P<clipId>\d+))'
+    _VALID_URL = r'https?://(?:www\.)?kusi\.com/(?P<path>story/.+|video\?clipId=(?P<clipId>\d+))'
     _TESTS = [{
         'url': 'http://www.kusi.com/story/31183873/turko-files-case-closed-put-on-hold',
         'md5': 'f926e7684294cf8cb7bdf8858e1b3988',
diff --git a/youtube_dl/extractor/kuwo.py b/youtube_dl/extractor/kuwo.py
index 45d65e61f..a586308b2 100644
--- a/youtube_dl/extractor/kuwo.py
+++ b/youtube_dl/extractor/kuwo.py
@@ -55,7 +55,7 @@ class KuwoBaseIE(InfoExtractor):
 class KuwoIE(KuwoBaseIE):
     IE_NAME = 'kuwo:song'
     IE_DESC = '酷我音乐'
-    _VALID_URL = r'http://www\.kuwo\.cn/yinyue/(?P<id>\d+?)'
+    _VALID_URL = r'https?://www\.kuwo\.cn/yinyue/(?P<id>\d+?)'
     _TESTS = [{
         'url': 'http://www.kuwo.cn/yinyue/635632/',
         'info_dict': {
@@ -134,7 +134,7 @@ class KuwoIE(KuwoBaseIE):
 class KuwoAlbumIE(InfoExtractor):
     IE_NAME = 'kuwo:album'
     IE_DESC = '酷我音乐 - 专辑'
-    _VALID_URL = r'http://www\.kuwo\.cn/album/(?P<id>\d+?)/'
+    _VALID_URL = r'https?://www\.kuwo\.cn/album/(?P<id>\d+?)/'
     _TEST = {
         'url': 'http://www.kuwo.cn/album/502294/',
         'info_dict': {
@@ -170,7 +170,7 @@ class KuwoAlbumIE(InfoExtractor):
 class KuwoChartIE(InfoExtractor):
     IE_NAME = 'kuwo:chart'
     IE_DESC = '酷我音乐 - 排行榜'
-    _VALID_URL = r'http://yinyue\.kuwo\.cn/billboard_(?P<id>[^.]+).htm'
+    _VALID_URL = r'https?://yinyue\.kuwo\.cn/billboard_(?P<id>[^.]+).htm'
     _TEST = {
         'url': 'http://yinyue.kuwo.cn/billboard_香港中文龙虎榜.htm',
         'info_dict': {
@@ -195,7 +195,7 @@ class KuwoChartIE(InfoExtractor):
 class KuwoSingerIE(InfoExtractor):
     IE_NAME = 'kuwo:singer'
     IE_DESC = '酷我音乐 - 歌手'
-    _VALID_URL = r'http://www\.kuwo\.cn/mingxing/(?P<id>[^/]+)'
+    _VALID_URL = r'https?://www\.kuwo\.cn/mingxing/(?P<id>[^/]+)'
     _TESTS = [{
         'url': 'http://www.kuwo.cn/mingxing/bruno+mars/',
         'info_dict': {
@@ -251,7 +251,7 @@ class KuwoSingerIE(InfoExtractor):
 class KuwoCategoryIE(InfoExtractor):
     IE_NAME = 'kuwo:category'
     IE_DESC = '酷我音乐 - 分类'
-    _VALID_URL = r'http://yinyue\.kuwo\.cn/yy/cinfo_(?P<id>\d+?).htm'
+    _VALID_URL = r'https?://yinyue\.kuwo\.cn/yy/cinfo_(?P<id>\d+?).htm'
     _TEST = {
         'url': 'http://yinyue.kuwo.cn/yy/cinfo_86375.htm',
         'info_dict': {
@@ -288,7 +288,7 @@ class KuwoCategoryIE(InfoExtractor):
 class KuwoMvIE(KuwoBaseIE):
     IE_NAME = 'kuwo:mv'
     IE_DESC = '酷我音乐 - MV'
-    _VALID_URL = r'http://www\.kuwo\.cn/mv/(?P<id>\d+?)/'
+    _VALID_URL = r'https?://www\.kuwo\.cn/mv/(?P<id>\d+?)/'
     _TEST = {
         'url': 'http://www.kuwo.cn/mv/6480076/',
         'info_dict': {
diff --git a/youtube_dl/extractor/leeco.py b/youtube_dl/extractor/leeco.py
index df47e88ba..462b752dd 100644
--- a/youtube_dl/extractor/leeco.py
+++ b/youtube_dl/extractor/leeco.py
@@ -28,7 +28,7 @@ from ..utils import (
 
 class LeIE(InfoExtractor):
     IE_DESC = '乐视网'
-    _VALID_URL = r'http://www\.le\.com/ptv/vplay/(?P<id>\d+)\.html'
+    _VALID_URL = r'https?://www\.le\.com/ptv/vplay/(?P<id>\d+)\.html'
 
     _URL_TEMPLATE = 'http://www.le.com/ptv/vplay/%s.html'
 
@@ -196,7 +196,7 @@ class LeIE(InfoExtractor):
 
 
 class LePlaylistIE(InfoExtractor):
-    _VALID_URL = r'http://[a-z]+\.le\.com/[a-z]+/(?P<id>[a-z0-9_]+)'
+    _VALID_URL = r'https?://[a-z]+\.le\.com/[a-z]+/(?P<id>[a-z0-9_]+)'
 
     _TESTS = [{
         'url': 'http://www.le.com/tv/46177.html',
diff --git a/youtube_dl/extractor/lifenews.py b/youtube_dl/extractor/lifenews.py
index a8fd639cc..ba2f80a75 100644
--- a/youtube_dl/extractor/lifenews.py
+++ b/youtube_dl/extractor/lifenews.py
@@ -17,7 +17,7 @@ from ..utils import (
 class LifeNewsIE(InfoExtractor):
     IE_NAME = 'lifenews'
     IE_DESC = 'LIFE | NEWS'
-    _VALID_URL = r'http://lifenews\.ru/(?:mobile/)?(?P<section>news|video)/(?P<id>\d+)'
+    _VALID_URL = r'https?://lifenews\.ru/(?:mobile/)?(?P<section>news|video)/(?P<id>\d+)'
 
     _TESTS = [{
         # single video embedded via video/source
@@ -159,7 +159,7 @@ class LifeNewsIE(InfoExtractor):
 
 class LifeEmbedIE(InfoExtractor):
     IE_NAME = 'life:embed'
-    _VALID_URL = r'http://embed\.life\.ru/embed/(?P<id>[\da-f]{32})'
+    _VALID_URL = r'https?://embed\.life\.ru/embed/(?P<id>[\da-f]{32})'
 
     _TEST = {
         'url': 'http://embed.life.ru/embed/e50c2dec2867350528e2574c899b8291',
diff --git a/youtube_dl/extractor/limelight.py b/youtube_dl/extractor/limelight.py
index 1a0625ac3..2599d45c3 100644
--- a/youtube_dl/extractor/limelight.py
+++ b/youtube_dl/extractor/limelight.py
@@ -123,7 +123,7 @@ class LimelightBaseIE(InfoExtractor):
 
 class LimelightMediaIE(LimelightBaseIE):
     IE_NAME = 'limelight'
-    _VALID_URL = r'(?:limelight:media:|http://link\.videoplatform\.limelight\.com/media/\??\bmediaId=)(?P<id>[a-z0-9]{32})'
+    _VALID_URL = r'(?:limelight:media:|https?://link\.videoplatform\.limelight\.com/media/\??\bmediaId=)(?P<id>[a-z0-9]{32})'
     _TESTS = [{
         'url': 'http://link.videoplatform.limelight.com/media/?mediaId=3ffd040b522b4485b6d84effc750cd86',
         'info_dict': {
@@ -176,7 +176,7 @@ class LimelightMediaIE(LimelightBaseIE):
 
 class LimelightChannelIE(LimelightBaseIE):
     IE_NAME = 'limelight:channel'
-    _VALID_URL = r'(?:limelight:channel:|http://link\.videoplatform\.limelight\.com/media/\??\bchannelId=)(?P<id>[a-z0-9]{32})'
+    _VALID_URL = r'(?:limelight:channel:|https?://link\.videoplatform\.limelight\.com/media/\??\bchannelId=)(?P<id>[a-z0-9]{32})'
     _TEST = {
         'url': 'http://link.videoplatform.limelight.com/media/?channelId=ab6a524c379342f9b23642917020c082',
         'info_dict': {
@@ -207,7 +207,7 @@ class LimelightChannelIE(LimelightBaseIE):
 
 class LimelightChannelListIE(LimelightBaseIE):
     IE_NAME = 'limelight:channel_list'
-    _VALID_URL = r'(?:limelight:channel_list:|http://link\.videoplatform\.limelight\.com/media/\?.*?\bchannelListId=)(?P<id>[a-z0-9]{32})'
+    _VALID_URL = r'(?:limelight:channel_list:|https?://link\.videoplatform\.limelight\.com/media/\?.*?\bchannelListId=)(?P<id>[a-z0-9]{32})'
     _TEST = {
         'url': 'http://link.videoplatform.limelight.com/media/?channelListId=301b117890c4465c8179ede21fd92e2b',
         'info_dict': {
diff --git a/youtube_dl/extractor/m6.py b/youtube_dl/extractor/m6.py
index 7e025831b..d5945ad66 100644
--- a/youtube_dl/extractor/m6.py
+++ b/youtube_dl/extractor/m6.py
@@ -8,7 +8,7 @@ from .common import InfoExtractor
 
 class M6IE(InfoExtractor):
     IE_NAME = 'm6'
-    _VALID_URL = r'http://(?:www\.)?m6\.fr/[^/]+/videos/(?P<id>\d+)-[^\.]+\.html'
+    _VALID_URL = r'https?://(?:www\.)?m6\.fr/[^/]+/videos/(?P<id>\d+)-[^\.]+\.html'
 
     _TEST = {
         'url': 'http://www.m6.fr/emission-les_reines_du_shopping/videos/11323908-emeline_est_la_reine_du_shopping_sur_le_theme_ma_fete_d_8217_anniversaire.html',
diff --git a/youtube_dl/extractor/metacafe.py b/youtube_dl/extractor/metacafe.py
index 67d6271e1..c31e8798a 100644
--- a/youtube_dl/extractor/metacafe.py
+++ b/youtube_dl/extractor/metacafe.py
@@ -17,7 +17,7 @@ from ..utils import (
 
 
 class MetacafeIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?metacafe\.com/watch/([^/]+)/([^/]+)/.*'
+    _VALID_URL = r'https?://(?:www\.)?metacafe\.com/watch/([^/]+)/([^/]+)/.*'
     _DISCLAIMER = 'http://www.metacafe.com/family_filter/'
     _FILTER_POST = 'http://www.metacafe.com/f/index.php?inputType=filter&controllerGroup=user'
     IE_NAME = 'metacafe'
diff --git a/youtube_dl/extractor/mit.py b/youtube_dl/extractor/mit.py
index 819c1b90b..1aea78d11 100644
--- a/youtube_dl/extractor/mit.py
+++ b/youtube_dl/extractor/mit.py
@@ -91,7 +91,7 @@ class MITIE(TechTVMITIE):
 
 class OCWMITIE(InfoExtractor):
     IE_NAME = 'ocw.mit.edu'
-    _VALID_URL = r'^http://ocw\.mit\.edu/courses/(?P<topic>[a-z0-9\-]+)'
+    _VALID_URL = r'^https?://ocw\.mit\.edu/courses/(?P<topic>[a-z0-9\-]+)'
     _BASE_URL = 'http://ocw.mit.edu/'
 
     _TESTS = [
diff --git a/youtube_dl/extractor/mitele.py b/youtube_dl/extractor/mitele.py
index c595f2077..9e584860a 100644
--- a/youtube_dl/extractor/mitele.py
+++ b/youtube_dl/extractor/mitele.py
@@ -14,7 +14,7 @@ from ..utils import (
 
 class MiTeleIE(InfoExtractor):
     IE_DESC = 'mitele.es'
-    _VALID_URL = r'http://www\.mitele\.es/[^/]+/[^/]+/[^/]+/(?P<id>[^/]+)/'
+    _VALID_URL = r'https?://www\.mitele\.es/[^/]+/[^/]+/[^/]+/(?P<id>[^/]+)/'
 
     _TESTS = [{
         'url': 'http://www.mitele.es/programas-tv/diario-de/la-redaccion/programa-144/',
diff --git a/youtube_dl/extractor/mooshare.py b/youtube_dl/extractor/mooshare.py
index 7cc7f054f..f010f52d5 100644
--- a/youtube_dl/extractor/mooshare.py
+++ b/youtube_dl/extractor/mooshare.py
@@ -13,7 +13,7 @@ from ..utils import (
 class MooshareIE(InfoExtractor):
     IE_NAME = 'mooshare'
     IE_DESC = 'Mooshare.biz'
-    _VALID_URL = r'http://(?:www\.)?mooshare\.biz/(?P<id>[\da-z]{12})'
+    _VALID_URL = r'https?://(?:www\.)?mooshare\.biz/(?P<id>[\da-z]{12})'
 
     _TESTS = [
         {
diff --git a/youtube_dl/extractor/motherless.py b/youtube_dl/extractor/motherless.py
index 0b4787c1d..ad04b12cd 100644
--- a/youtube_dl/extractor/motherless.py
+++ b/youtube_dl/extractor/motherless.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 
 class MotherlessIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?motherless\.com/(?:g/[a-z0-9_]+/)?(?P<id>[A-Z0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?motherless\.com/(?:g/[a-z0-9_]+/)?(?P<id>[A-Z0-9]+)'
     _TESTS = [{
         'url': 'http://motherless.com/AC3FFE1',
         'md5': '310f62e325a9fafe64f68c0bccb6e75f',
diff --git a/youtube_dl/extractor/motorsport.py b/youtube_dl/extractor/motorsport.py
index c1a482dba..370328b36 100644
--- a/youtube_dl/extractor/motorsport.py
+++ b/youtube_dl/extractor/motorsport.py
@@ -9,7 +9,7 @@ from ..compat import (
 
 class MotorsportIE(InfoExtractor):
     IE_DESC = 'motorsport.com'
-    _VALID_URL = r'http://www\.motorsport\.com/[^/?#]+/video/(?:[^/?#]+/)(?P<id>[^/]+)/?(?:$|[?#])'
+    _VALID_URL = r'https?://www\.motorsport\.com/[^/?#]+/video/(?:[^/?#]+/)(?P<id>[^/]+)/?(?:$|[?#])'
     _TEST = {
         'url': 'http://www.motorsport.com/f1/video/main-gallery/red-bull-racing-2014-rules-explained/',
         'info_dict': {
diff --git a/youtube_dl/extractor/myspass.py b/youtube_dl/extractor/myspass.py
index f936b92bb..1ca7b1a9e 100644
--- a/youtube_dl/extractor/myspass.py
+++ b/youtube_dl/extractor/myspass.py
@@ -11,7 +11,7 @@ from ..utils import (
 
 
 class MySpassIE(InfoExtractor):
-    _VALID_URL = r'http://www\.myspass\.de/.*'
+    _VALID_URL = r'https?://www\.myspass\.de/.*'
     _TEST = {
         'url': 'http://www.myspass.de/myspass/shows/tvshows/absolute-mehrheit/Absolute-Mehrheit-vom-17022013-Die-Highlights-Teil-2--/11741/',
         'md5': '0b49f4844a068f8b33f4b7c88405862b',
diff --git a/youtube_dl/extractor/myvideo.py b/youtube_dl/extractor/myvideo.py
index 1e21cf98a..c83a1eab5 100644
--- a/youtube_dl/extractor/myvideo.py
+++ b/youtube_dl/extractor/myvideo.py
@@ -20,7 +20,7 @@ from ..utils import (
 
 class MyVideoIE(InfoExtractor):
     _WORKING = False
-    _VALID_URL = r'http://(?:www\.)?myvideo\.de/(?:[^/]+/)?watch/(?P<id>[0-9]+)/[^?/]+.*'
+    _VALID_URL = r'https?://(?:www\.)?myvideo\.de/(?:[^/]+/)?watch/(?P<id>[0-9]+)/[^?/]+.*'
     IE_NAME = 'myvideo'
     _TEST = {
         'url': 'http://www.myvideo.de/watch/8229274/bowling_fail_or_win',
diff --git a/youtube_dl/extractor/myvidster.py b/youtube_dl/extractor/myvidster.py
index a94ab8358..731c24542 100644
--- a/youtube_dl/extractor/myvidster.py
+++ b/youtube_dl/extractor/myvidster.py
@@ -4,7 +4,7 @@ from .common import InfoExtractor
 
 
 class MyVidsterIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?myvidster\.com/video/(?P<id>\d+)/'
+    _VALID_URL = r'https?://(?:www\.)?myvidster\.com/video/(?P<id>\d+)/'
 
     _TEST = {
         'url': 'http://www.myvidster.com/video/32059805/Hot_chemistry_with_raw_love_making',
diff --git a/youtube_dl/extractor/nationalgeographic.py b/youtube_dl/extractor/nationalgeographic.py
index 7ce8d9b18..d5e53365c 100644
--- a/youtube_dl/extractor/nationalgeographic.py
+++ b/youtube_dl/extractor/nationalgeographic.py
@@ -8,7 +8,7 @@ from ..utils import (
 
 
 class NationalGeographicIE(InfoExtractor):
-    _VALID_URL = r'http://video\.nationalgeographic\.com/.*?'
+    _VALID_URL = r'https?://video\.nationalgeographic\.com/.*?'
 
     _TESTS = [
         {
diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index bb0817e34..a622f2212 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -115,7 +115,7 @@ class NBCSportsVPlayerIE(InfoExtractor):
 
 class NBCSportsIE(InfoExtractor):
     # Does not include https because its certificate is invalid
-    _VALID_URL = r'http://www\.nbcsports\.com//?(?:[^/]+/)+(?P<id>[0-9a-z-]+)'
+    _VALID_URL = r'https?://www\.nbcsports\.com//?(?:[^/]+/)+(?P<id>[0-9a-z-]+)'
 
     _TEST = {
         'url': 'http://www.nbcsports.com//college-basketball/ncaab/tom-izzo-michigan-st-has-so-much-respect-duke',
@@ -295,7 +295,7 @@ class NBCNewsIE(ThePlatformIE):
 
 class MSNBCIE(InfoExtractor):
     # https URLs redirect to corresponding http ones
-    _VALID_URL = r'http://www\.msnbc\.com/[^/]+/watch/(?P<id>[^/]+)'
+    _VALID_URL = r'https?://www\.msnbc\.com/[^/]+/watch/(?P<id>[^/]+)'
     _TEST = {
         'url': 'http://www.msnbc.com/all-in-with-chris-hayes/watch/the-chaotic-gop-immigration-vote-314487875924',
         'md5': '6d236bf4f3dddc226633ce6e2c3f814d',
diff --git a/youtube_dl/extractor/nextmedia.py b/youtube_dl/extractor/nextmedia.py
index d1688457f..aae7aeeeb 100644
--- a/youtube_dl/extractor/nextmedia.py
+++ b/youtube_dl/extractor/nextmedia.py
@@ -7,7 +7,7 @@ from ..utils import parse_iso8601
 
 class NextMediaIE(InfoExtractor):
     IE_DESC = '蘋果日報'
-    _VALID_URL = r'http://hk.apple.nextmedia.com/[^/]+/[^/]+/(?P<date>\d+)/(?P<id>\d+)'
+    _VALID_URL = r'https?://hk.apple.nextmedia.com/[^/]+/[^/]+/(?P<date>\d+)/(?P<id>\d+)'
     _TESTS = [{
         'url': 'http://hk.apple.nextmedia.com/realtime/news/20141108/53109199',
         'md5': 'dff9fad7009311c421176d1ac90bfe4f',
@@ -68,7 +68,7 @@ class NextMediaIE(InfoExtractor):
 
 class NextMediaActionNewsIE(NextMediaIE):
     IE_DESC = '蘋果日報 - 動新聞'
-    _VALID_URL = r'http://hk.dv.nextmedia.com/actionnews/[^/]+/(?P<date>\d+)/(?P<id>\d+)/\d+'
+    _VALID_URL = r'https?://hk.dv.nextmedia.com/actionnews/[^/]+/(?P<date>\d+)/(?P<id>\d+)/\d+'
     _TESTS = [{
         'url': 'http://hk.dv.nextmedia.com/actionnews/hit/20150121/19009428/20061460',
         'md5': '05fce8ffeed7a5e00665d4b7cf0f9201',
@@ -93,7 +93,7 @@ class NextMediaActionNewsIE(NextMediaIE):
 
 class AppleDailyIE(NextMediaIE):
     IE_DESC = '臺灣蘋果日報'
-    _VALID_URL = r'http://(www|ent).appledaily.com.tw/(?:animation|appledaily|enews|realtimenews)/[^/]+/[^/]+/(?P<date>\d+)/(?P<id>\d+)(/.*)?'
+    _VALID_URL = r'https?://(www|ent).appledaily.com.tw/(?:animation|appledaily|enews|realtimenews)/[^/]+/[^/]+/(?P<date>\d+)/(?P<id>\d+)(/.*)?'
     _TESTS = [{
         'url': 'http://ent.appledaily.com.tw/enews/article/entertainment/20150128/36354694',
         'md5': 'a843ab23d150977cc55ef94f1e2c1e4d',
diff --git a/youtube_dl/extractor/noco.py b/youtube_dl/extractor/noco.py
index d440313d5..ec7317a2f 100644
--- a/youtube_dl/extractor/noco.py
+++ b/youtube_dl/extractor/noco.py
@@ -22,7 +22,7 @@ from ..utils import (
 
 
 class NocoIE(InfoExtractor):
-    _VALID_URL = r'http://(?:(?:www\.)?noco\.tv/emission/|player\.noco\.tv/\?idvideo=)(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:(?:www\.)?noco\.tv/emission/|player\.noco\.tv/\?idvideo=)(?P<id>\d+)'
     _LOGIN_URL = 'http://noco.tv/do.php'
     _API_URL_TEMPLATE = 'https://api.noco.tv/1.1/%s?ts=%s&tk=%s'
     _SUB_LANG_TEMPLATE = '&sub_lang=%s'
diff --git a/youtube_dl/extractor/normalboots.py b/youtube_dl/extractor/normalboots.py
index 5952d136f..77e091072 100644
--- a/youtube_dl/extractor/normalboots.py
+++ b/youtube_dl/extractor/normalboots.py
@@ -9,7 +9,7 @@ from ..utils import (
 
 
 class NormalbootsIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?normalboots\.com/video/(?P<id>[0-9a-z-]*)/?$'
+    _VALID_URL = r'https?://(?:www\.)?normalboots\.com/video/(?P<id>[0-9a-z-]*)/?$'
     _TEST = {
         'url': 'http://normalboots.com/video/home-alone-games-jontron/',
         'md5': '8bf6de238915dd501105b44ef5f1e0f6',
diff --git a/youtube_dl/extractor/nova.py b/youtube_dl/extractor/nova.py
index 72f72b803..17671ad39 100644
--- a/youtube_dl/extractor/nova.py
+++ b/youtube_dl/extractor/nova.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 class NovaIE(InfoExtractor):
     IE_DESC = 'TN.cz, Prásk.tv, Nova.cz, Novaplus.cz, FANDA.tv, Krásná.cz and Doma.cz'
-    _VALID_URL = r'http://(?:[^.]+\.)?(?P<site>tv(?:noviny)?|tn|novaplus|vymena|fanda|krasna|doma|prask)\.nova\.cz/(?:[^/]+/)+(?P<id>[^/]+?)(?:\.html|/|$)'
+    _VALID_URL = r'https?://(?:[^.]+\.)?(?P<site>tv(?:noviny)?|tn|novaplus|vymena|fanda|krasna|doma|prask)\.nova\.cz/(?:[^/]+/)+(?P<id>[^/]+?)(?:\.html|/|$)'
     _TESTS = [{
         'url': 'http://tvnoviny.nova.cz/clanek/novinky/co-na-sebe-sportaci-praskli-vime-jestli-pujde-hrdlicka-na-materskou.html?utm_source=tvnoviny&utm_medium=cpfooter&utm_campaign=novaplus',
         'info_dict': {
diff --git a/youtube_dl/extractor/npr.py b/youtube_dl/extractor/npr.py
index 125c7010b..a3f0abb4e 100644
--- a/youtube_dl/extractor/npr.py
+++ b/youtube_dl/extractor/npr.py
@@ -9,7 +9,7 @@ from ..utils import (
 
 
 class NprIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?npr\.org/player/v2/mediaPlayer\.html\?.*\bid=(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?npr\.org/player/v2/mediaPlayer\.html\?.*\bid=(?P<id>\d+)'
     _TESTS = [{
         'url': 'http://www.npr.org/player/v2/mediaPlayer.html?id=449974205',
         'info_dict': {
diff --git a/youtube_dl/extractor/ntvru.py b/youtube_dl/extractor/ntvru.py
index 2cd924d05..0895d7ea4 100644
--- a/youtube_dl/extractor/ntvru.py
+++ b/youtube_dl/extractor/ntvru.py
@@ -11,7 +11,7 @@ from ..utils import (
 
 class NTVRuIE(InfoExtractor):
     IE_NAME = 'ntv.ru'
-    _VALID_URL = r'http://(?:www\.)?ntv\.ru/(?P<id>.+)'
+    _VALID_URL = r'https?://(?:www\.)?ntv\.ru/(?P<id>.+)'
 
     _TESTS = [
         {
diff --git a/youtube_dl/extractor/orf.py b/youtube_dl/extractor/orf.py
index 958eb398b..66c75f8b3 100644
--- a/youtube_dl/extractor/orf.py
+++ b/youtube_dl/extractor/orf.py
@@ -137,7 +137,7 @@ class ORFTVthekIE(InfoExtractor):
 class ORFOE1IE(InfoExtractor):
     IE_NAME = 'orf:oe1'
     IE_DESC = 'Radio Österreich 1'
-    _VALID_URL = r'http://oe1\.orf\.at/(?:programm/|konsole.*?#\?track_id=)(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://oe1\.orf\.at/(?:programm/|konsole.*?#\?track_id=)(?P<id>[0-9]+)'
 
     # Audios on ORF radio are only available for 7 days, so we can't add tests.
     _TEST = {
@@ -171,7 +171,7 @@ class ORFOE1IE(InfoExtractor):
 class ORFFM4IE(InfoExtractor):
     IE_NAME = 'orf:fm4'
     IE_DESC = 'radio FM4'
-    _VALID_URL = r'http://fm4\.orf\.at/(?:7tage/?#|player/)(?P<date>[0-9]+)/(?P<show>\w+)'
+    _VALID_URL = r'https?://fm4\.orf\.at/(?:7tage/?#|player/)(?P<date>[0-9]+)/(?P<show>\w+)'
 
     _TEST = {
         'url': 'http://fm4.orf.at/player/20160110/IS/',
@@ -222,7 +222,7 @@ class ORFFM4IE(InfoExtractor):
 class ORFIPTVIE(InfoExtractor):
     IE_NAME = 'orf:iptv'
     IE_DESC = 'iptv.ORF.at'
-    _VALID_URL = r'http://iptv\.orf\.at/(?:#/)?stories/(?P<id>\d+)'
+    _VALID_URL = r'https?://iptv\.orf\.at/(?:#/)?stories/(?P<id>\d+)'
 
     _TEST = {
         'url': 'http://iptv.orf.at/stories/2275236/',
diff --git a/youtube_dl/extractor/philharmoniedeparis.py b/youtube_dl/extractor/philharmoniedeparis.py
index 6e60e5fe9..f1008ae51 100644
--- a/youtube_dl/extractor/philharmoniedeparis.py
+++ b/youtube_dl/extractor/philharmoniedeparis.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 class PhilharmonieDeParisIE(InfoExtractor):
     IE_DESC = 'Philharmonie de Paris'
-    _VALID_URL = r'http://live\.philharmoniedeparis\.fr/(?:[Cc]oncert/|misc/Playlist\.ashx\?id=)(?P<id>\d+)'
+    _VALID_URL = r'https?://live\.philharmoniedeparis\.fr/(?:[Cc]oncert/|misc/Playlist\.ashx\?id=)(?P<id>\d+)'
     _TESTS = [{
         'url': 'http://live.philharmoniedeparis.fr/concert/1032066.html',
         'info_dict': {
diff --git a/youtube_dl/extractor/photobucket.py b/youtube_dl/extractor/photobucket.py
index 788411ccc..6c8bbe1d9 100644
--- a/youtube_dl/extractor/photobucket.py
+++ b/youtube_dl/extractor/photobucket.py
@@ -8,7 +8,7 @@ from ..compat import compat_urllib_parse_unquote
 
 
 class PhotobucketIE(InfoExtractor):
-    _VALID_URL = r'http://(?:[a-z0-9]+\.)?photobucket\.com/.*(([\?\&]current=)|_)(?P<id>.*)\.(?P<ext>(flv)|(mp4))'
+    _VALID_URL = r'https?://(?:[a-z0-9]+\.)?photobucket\.com/.*(([\?\&]current=)|_)(?P<id>.*)\.(?P<ext>(flv)|(mp4))'
     _TEST = {
         'url': 'http://media.photobucket.com/user/rachaneronas/media/TiredofLinkBuildingTryBacklinkMyDomaincom_zpsc0c3b9fa.mp4.html?filters[term]=search&filters[primary]=videos&filters[secondary]=images&sort=1&o=0',
         'md5': '7dabfb92b0a31f6c16cebc0f8e60ff99',
diff --git a/youtube_dl/extractor/pornhd.py b/youtube_dl/extractor/pornhd.py
index 57c78ba52..39b53ecf6 100644
--- a/youtube_dl/extractor/pornhd.py
+++ b/youtube_dl/extractor/pornhd.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 
 class PornHdIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?pornhd\.com/(?:[a-z]{2,4}/)?videos/(?P<id>\d+)(?:/(?P<display_id>.+))?'
+    _VALID_URL = r'https?://(?:www\.)?pornhd\.com/(?:[a-z]{2,4}/)?videos/(?P<id>\d+)(?:/(?P<display_id>.+))?'
     _TEST = {
         'url': 'http://www.pornhd.com/videos/1962/sierra-day-gets-his-cum-all-over-herself-hd-porn-video',
         'md5': '956b8ca569f7f4d8ec563e2c41598441',
diff --git a/youtube_dl/extractor/pornovoisines.py b/youtube_dl/extractor/pornovoisines.py
index 1a53fd71c..6b51e5c54 100644
--- a/youtube_dl/extractor/pornovoisines.py
+++ b/youtube_dl/extractor/pornovoisines.py
@@ -13,7 +13,7 @@ from ..utils import (
 
 
 class PornoVoisinesIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?pornovoisines\.com/showvideo/(?P<id>\d+)/(?P<display_id>[^/]+)'
+    _VALID_URL = r'https?://(?:www\.)?pornovoisines\.com/showvideo/(?P<id>\d+)/(?P<display_id>[^/]+)'
 
     _VIDEO_URL_TEMPLATE = 'http://stream%d.pornovoisines.com' \
         '/static/media/video/transcoded/%s-640x360-1000-trscded.mp4'
diff --git a/youtube_dl/extractor/pyvideo.py b/youtube_dl/extractor/pyvideo.py
index 30a5f2de4..cc0416cb8 100644
--- a/youtube_dl/extractor/pyvideo.py
+++ b/youtube_dl/extractor/pyvideo.py
@@ -7,7 +7,7 @@ from .common import InfoExtractor
 
 
 class PyvideoIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?pyvideo\.org/video/(?P<id>\d+)/(.*)'
+    _VALID_URL = r'https?://(?:www\.)?pyvideo\.org/video/(?P<id>\d+)/(.*)'
 
     _TESTS = [
         {
diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index 45a3c41c5..ff0af9543 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -18,7 +18,7 @@ from ..utils import (
 class QQMusicIE(InfoExtractor):
     IE_NAME = 'qqmusic'
     IE_DESC = 'QQ音乐'
-    _VALID_URL = r'http://y.qq.com/#type=song&mid=(?P<id>[0-9A-Za-z]+)'
+    _VALID_URL = r'https?://y.qq.com/#type=song&mid=(?P<id>[0-9A-Za-z]+)'
     _TESTS = [{
         'url': 'http://y.qq.com/#type=song&mid=004295Et37taLD',
         'md5': '9ce1c1c8445f561506d2e3cfb0255705',
@@ -172,7 +172,7 @@ class QQPlaylistBaseIE(InfoExtractor):
 class QQMusicSingerIE(QQPlaylistBaseIE):
     IE_NAME = 'qqmusic:singer'
     IE_DESC = 'QQ音乐 - 歌手'
-    _VALID_URL = r'http://y.qq.com/#type=singer&mid=(?P<id>[0-9A-Za-z]+)'
+    _VALID_URL = r'https?://y.qq.com/#type=singer&mid=(?P<id>[0-9A-Za-z]+)'
     _TEST = {
         'url': 'http://y.qq.com/#type=singer&mid=001BLpXF2DyJe2',
         'info_dict': {
@@ -217,7 +217,7 @@ class QQMusicSingerIE(QQPlaylistBaseIE):
 class QQMusicAlbumIE(QQPlaylistBaseIE):
     IE_NAME = 'qqmusic:album'
     IE_DESC = 'QQ音乐 - 专辑'
-    _VALID_URL = r'http://y.qq.com/#type=album&mid=(?P<id>[0-9A-Za-z]+)'
+    _VALID_URL = r'https?://y.qq.com/#type=album&mid=(?P<id>[0-9A-Za-z]+)'
 
     _TESTS = [{
         'url': 'http://y.qq.com/#type=album&mid=000gXCTb2AhRR1',
@@ -260,7 +260,7 @@ class QQMusicAlbumIE(QQPlaylistBaseIE):
 class QQMusicToplistIE(QQPlaylistBaseIE):
     IE_NAME = 'qqmusic:toplist'
     IE_DESC = 'QQ音乐 - 排行榜'
-    _VALID_URL = r'http://y\.qq\.com/#type=toplist&p=(?P<id>(top|global)_[0-9]+)'
+    _VALID_URL = r'https?://y\.qq\.com/#type=toplist&p=(?P<id>(top|global)_[0-9]+)'
 
     _TESTS = [{
         'url': 'http://y.qq.com/#type=toplist&p=global_123',
@@ -314,7 +314,7 @@ class QQMusicToplistIE(QQPlaylistBaseIE):
 class QQMusicPlaylistIE(QQPlaylistBaseIE):
     IE_NAME = 'qqmusic:playlist'
     IE_DESC = 'QQ音乐 - 歌单'
-    _VALID_URL = r'http://y\.qq\.com/#type=taoge&id=(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://y\.qq\.com/#type=taoge&id=(?P<id>[0-9]+)'
 
     _TESTS = [{
         'url': 'http://y.qq.com/#type=taoge&id=3462654915',
diff --git a/youtube_dl/extractor/rai.py b/youtube_dl/extractor/rai.py
index a4dc5c335..e36ce1aa1 100644
--- a/youtube_dl/extractor/rai.py
+++ b/youtube_dl/extractor/rai.py
@@ -18,7 +18,7 @@ from ..utils import (
 
 
 class RaiTVIE(InfoExtractor):
-    _VALID_URL = r'http://(?:.+?\.)?(?:rai\.it|rai\.tv|rainews\.it)/dl/(?:[^/]+/)+media/.+?-(?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})(?:-.+?)?\.html'
+    _VALID_URL = r'https?://(?:.+?\.)?(?:rai\.it|rai\.tv|rainews\.it)/dl/(?:[^/]+/)+media/.+?-(?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})(?:-.+?)?\.html'
     _TESTS = [
         {
             'url': 'http://www.rai.tv/dl/RaiTV/programmi/media/ContentItem-cb27157f-9dd0-4aee-b788-b1f67643a391.html',
@@ -175,7 +175,7 @@ class RaiTVIE(InfoExtractor):
 
 
 class RaiIE(InfoExtractor):
-    _VALID_URL = r'http://(?:.+?\.)?(?:rai\.it|rai\.tv|rainews\.it)/dl/.+?-(?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})(?:-.+?)?\.html'
+    _VALID_URL = r'https?://(?:.+?\.)?(?:rai\.it|rai\.tv|rainews\.it)/dl/.+?-(?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})(?:-.+?)?\.html'
     _TESTS = [
         {
             'url': 'http://www.report.rai.it/dl/Report/puntata/ContentItem-0c7a664b-d0f4-4b2c-8835-3f82e46f433e.html',
diff --git a/youtube_dl/extractor/redtube.py b/youtube_dl/extractor/redtube.py
index d6054d717..7ba41ba59 100644
--- a/youtube_dl/extractor/redtube.py
+++ b/youtube_dl/extractor/redtube.py
@@ -5,7 +5,7 @@ from ..utils import ExtractorError
 
 
 class RedTubeIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?redtube\.com/(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?redtube\.com/(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://www.redtube.com/66418',
         'md5': '7b8c22b5e7098a3e1c09709df1126d2d',
diff --git a/youtube_dl/extractor/ringtv.py b/youtube_dl/extractor/ringtv.py
index 508758075..2c2c707bd 100644
--- a/youtube_dl/extractor/ringtv.py
+++ b/youtube_dl/extractor/ringtv.py
@@ -6,7 +6,7 @@ from .common import InfoExtractor
 
 
 class RingTVIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?ringtv\.craveonline\.com/(?P<type>news|videos/video)/(?P<id>[^/?#]+)'
+    _VALID_URL = r'https?://(?:www\.)?ringtv\.craveonline\.com/(?P<type>news|videos/video)/(?P<id>[^/?#]+)'
     _TEST = {
         'url': 'http://ringtv.craveonline.com/news/310833-luis-collazo-says-victor-ortiz-better-not-quit-on-jan-30',
         'md5': 'd25945f5df41cdca2d2587165ac28720',
diff --git a/youtube_dl/extractor/rtve.py b/youtube_dl/extractor/rtve.py
index 8a8c5d2a0..08cd1ae6c 100644
--- a/youtube_dl/extractor/rtve.py
+++ b/youtube_dl/extractor/rtve.py
@@ -62,7 +62,7 @@ def _decrypt_url(png):
 class RTVEALaCartaIE(InfoExtractor):
     IE_NAME = 'rtve.es:alacarta'
     IE_DESC = 'RTVE a la carta'
-    _VALID_URL = r'http://www\.rtve\.es/(m/)?alacarta/videos/[^/]+/[^/]+/(?P<id>\d+)'
+    _VALID_URL = r'https?://www\.rtve\.es/(m/)?alacarta/videos/[^/]+/[^/]+/(?P<id>\d+)'
 
     _TESTS = [{
         'url': 'http://www.rtve.es/alacarta/videos/balonmano/o-swiss-cup-masculina-final-espana-suecia/2491869/',
@@ -179,7 +179,7 @@ class RTVEInfantilIE(InfoExtractor):
 class RTVELiveIE(InfoExtractor):
     IE_NAME = 'rtve.es:live'
     IE_DESC = 'RTVE.es live streams'
-    _VALID_URL = r'http://www\.rtve\.es/directo/(?P<id>[a-zA-Z0-9-]+)'
+    _VALID_URL = r'https?://www\.rtve\.es/directo/(?P<id>[a-zA-Z0-9-]+)'
 
     _TESTS = [{
         'url': 'http://www.rtve.es/directo/la-1/',
diff --git a/youtube_dl/extractor/ruhd.py b/youtube_dl/extractor/ruhd.py
index 0e470e73f..1f7c26299 100644
--- a/youtube_dl/extractor/ruhd.py
+++ b/youtube_dl/extractor/ruhd.py
@@ -5,7 +5,7 @@ from .common import InfoExtractor
 
 
 class RUHDIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?ruhd\.ru/play\.php\?vid=(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?ruhd\.ru/play\.php\?vid=(?P<id>\d+)'
     _TEST = {
         'url': 'http://www.ruhd.ru/play.php?vid=207',
         'md5': 'd1a9ec4edf8598e3fbd92bb16072ba83',
diff --git a/youtube_dl/extractor/rutube.py b/youtube_dl/extractor/rutube.py
index c5c47d01e..9ca4ae147 100644
--- a/youtube_dl/extractor/rutube.py
+++ b/youtube_dl/extractor/rutube.py
@@ -122,7 +122,7 @@ class RutubeEmbedIE(InfoExtractor):
 class RutubeChannelIE(InfoExtractor):
     IE_NAME = 'rutube:channel'
     IE_DESC = 'Rutube channels'
-    _VALID_URL = r'http://rutube\.ru/tags/video/(?P<id>\d+)'
+    _VALID_URL = r'https?://rutube\.ru/tags/video/(?P<id>\d+)'
     _TESTS = [{
         'url': 'http://rutube.ru/tags/video/1800/',
         'info_dict': {
@@ -156,7 +156,7 @@ class RutubeChannelIE(InfoExtractor):
 class RutubeMovieIE(RutubeChannelIE):
     IE_NAME = 'rutube:movie'
     IE_DESC = 'Rutube movies'
-    _VALID_URL = r'http://rutube\.ru/metainfo/tv/(?P<id>\d+)'
+    _VALID_URL = r'https?://rutube\.ru/metainfo/tv/(?P<id>\d+)'
     _TESTS = []
 
     _MOVIE_TEMPLATE = 'http://rutube.ru/api/metainfo/tv/%s/?format=json'
@@ -174,7 +174,7 @@ class RutubeMovieIE(RutubeChannelIE):
 class RutubePersonIE(RutubeChannelIE):
     IE_NAME = 'rutube:person'
     IE_DESC = 'Rutube person videos'
-    _VALID_URL = r'http://rutube\.ru/video/person/(?P<id>\d+)'
+    _VALID_URL = r'https?://rutube\.ru/video/person/(?P<id>\d+)'
     _TESTS = [{
         'url': 'http://rutube.ru/video/person/313878/',
         'info_dict': {
diff --git a/youtube_dl/extractor/screenjunkies.py b/youtube_dl/extractor/screenjunkies.py
index f2af15f6b..dd0a6ba19 100644
--- a/youtube_dl/extractor/screenjunkies.py
+++ b/youtube_dl/extractor/screenjunkies.py
@@ -11,7 +11,7 @@ from ..utils import (
 
 
 class ScreenJunkiesIE(InfoExtractor):
-    _VALID_URL = r'http://www.screenjunkies.com/video/(?P<display_id>[^/]+?)(?:-(?P<id>\d+))?(?:[/?#&]|$)'
+    _VALID_URL = r'https?://www.screenjunkies.com/video/(?P<display_id>[^/]+?)(?:-(?P<id>\d+))?(?:[/?#&]|$)'
     _TESTS = [{
         'url': 'http://www.screenjunkies.com/video/best-quentin-tarantino-movie-2841915',
         'md5': '5c2b686bec3d43de42bde9ec047536b0',
diff --git a/youtube_dl/extractor/senateisvp.py b/youtube_dl/extractor/senateisvp.py
index 4d3b58522..c5f474dd1 100644
--- a/youtube_dl/extractor/senateisvp.py
+++ b/youtube_dl/extractor/senateisvp.py
@@ -48,7 +48,7 @@ class SenateISVPIE(InfoExtractor):
         ['arch', '', 'http://ussenate-f.akamaihd.net/']
     ]
     _IE_NAME = 'senate.gov'
-    _VALID_URL = r'http://www\.senate\.gov/isvp/?\?(?P<qs>.+)'
+    _VALID_URL = r'https?://www\.senate\.gov/isvp/?\?(?P<qs>.+)'
     _TESTS = [{
         'url': 'http://www.senate.gov/isvp/?comm=judiciary&type=live&stt=&filename=judiciary031715&auto_play=false&wmode=transparent&poster=http%3A%2F%2Fwww.judiciary.senate.gov%2Fthemes%2Fjudiciary%2Fimages%2Fvideo-poster-flash-fit.png',
         'info_dict': {
diff --git a/youtube_dl/extractor/shared.py b/youtube_dl/extractor/shared.py
index 8eda3c864..96fe0b90d 100644
--- a/youtube_dl/extractor/shared.py
+++ b/youtube_dl/extractor/shared.py
@@ -13,7 +13,7 @@ from ..utils import (
 
 class SharedIE(InfoExtractor):
     IE_DESC = 'shared.sx and vivo.sx'
-    _VALID_URL = r'http://(?:shared|vivo)\.sx/(?P<id>[\da-z]{10})'
+    _VALID_URL = r'https?://(?:shared|vivo)\.sx/(?P<id>[\da-z]{10})'
 
     _TESTS = [{
         'url': 'http://shared.sx/0060718775',
diff --git a/youtube_dl/extractor/sport5.py b/youtube_dl/extractor/sport5.py
index dfe50ed45..7e6783306 100644
--- a/youtube_dl/extractor/sport5.py
+++ b/youtube_dl/extractor/sport5.py
@@ -8,7 +8,7 @@ from ..utils import ExtractorError
 
 
 class Sport5IE(InfoExtractor):
-    _VALID_URL = r'http://(?:www|vod)?\.sport5\.co\.il/.*\b(?:Vi|docID)=(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www|vod)?\.sport5\.co\.il/.*\b(?:Vi|docID)=(?P<id>\d+)'
     _TESTS = [
         {
             'url': 'http://vod.sport5.co.il/?Vc=147&Vi=176331&Page=1',
diff --git a/youtube_dl/extractor/ssa.py b/youtube_dl/extractor/ssa.py
index 13101c714..54d1843f2 100644
--- a/youtube_dl/extractor/ssa.py
+++ b/youtube_dl/extractor/ssa.py
@@ -8,7 +8,7 @@ from ..utils import (
 
 
 class SSAIE(InfoExtractor):
-    _VALID_URL = r'http://ssa\.nls\.uk/film/(?P<id>\d+)'
+    _VALID_URL = r'https?://ssa\.nls\.uk/film/(?P<id>\d+)'
     _TEST = {
         'url': 'http://ssa.nls.uk/film/3561',
         'info_dict': {
diff --git a/youtube_dl/extractor/sztvhu.py b/youtube_dl/extractor/sztvhu.py
index aa5964acb..f562aa6d3 100644
--- a/youtube_dl/extractor/sztvhu.py
+++ b/youtube_dl/extractor/sztvhu.py
@@ -5,7 +5,7 @@ from .common import InfoExtractor
 
 
 class SztvHuIE(InfoExtractor):
-    _VALID_URL = r'http://(?:(?:www\.)?sztv\.hu|www\.tvszombathely\.hu)/(?:[^/]+)/.+-(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://(?:(?:www\.)?sztv\.hu|www\.tvszombathely\.hu)/(?:[^/]+)/.+-(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://sztv.hu/hirek/cserkeszek-nepszerusitettek-a-kornyezettudatos-eletmodot-a-savaria-teren-20130909',
         'md5': 'a6df607b11fb07d0e9f2ad94613375cb',
diff --git a/youtube_dl/extractor/teamcoco.py b/youtube_dl/extractor/teamcoco.py
index d1b7264b4..b49ab5f5b 100644
--- a/youtube_dl/extractor/teamcoco.py
+++ b/youtube_dl/extractor/teamcoco.py
@@ -16,7 +16,7 @@ from ..compat import compat_ord
 
 
 class TeamcocoIE(InfoExtractor):
-    _VALID_URL = r'http://teamcoco\.com/video/(?P<video_id>[0-9]+)?/?(?P<display_id>.*)'
+    _VALID_URL = r'https?://teamcoco\.com/video/(?P<video_id>[0-9]+)?/?(?P<display_id>.*)'
     _TESTS = [
         {
             'url': 'http://teamcoco.com/video/80187/conan-becomes-a-mary-kay-beauty-consultant',
diff --git a/youtube_dl/extractor/tele13.py b/youtube_dl/extractor/tele13.py
index 4e860db0a..a29a64b6d 100644
--- a/youtube_dl/extractor/tele13.py
+++ b/youtube_dl/extractor/tele13.py
@@ -11,7 +11,7 @@ from ..utils import (
 
 
 class Tele13IE(InfoExtractor):
-    _VALID_URL = r'^http://(?:www\.)?t13\.cl/videos(?:/[^/]+)+/(?P<id>[\w-]+)'
+    _VALID_URL = r'^https?://(?:www\.)?t13\.cl/videos(?:/[^/]+)+/(?P<id>[\w-]+)'
     _TESTS = [
         {
             'url': 'http://www.t13.cl/videos/actualidad/el-circulo-de-hierro-de-michelle-bachelet-en-su-regreso-a-la-moneda',
diff --git a/youtube_dl/extractor/tf1.py b/youtube_dl/extractor/tf1.py
index 9ee844684..3f54b2744 100644
--- a/youtube_dl/extractor/tf1.py
+++ b/youtube_dl/extractor/tf1.py
@@ -6,7 +6,7 @@ from .common import InfoExtractor
 
 class TF1IE(InfoExtractor):
     """TF1 uses the wat.tv player."""
-    _VALID_URL = r'http://(?:(?:videos|www|lci)\.tf1|www\.tfou)\.fr/(?:[^/]+/)*(?P<id>.+?)\.html'
+    _VALID_URL = r'https?://(?:(?:videos|www|lci)\.tf1|www\.tfou)\.fr/(?:[^/]+/)*(?P<id>.+?)\.html'
     _TESTS = [{
         'url': 'http://videos.tf1.fr/auto-moto/citroen-grand-c4-picasso-2013-presentation-officielle-8062060.html',
         'info_dict': {
diff --git a/youtube_dl/extractor/thvideo.py b/youtube_dl/extractor/thvideo.py
index 496f15d80..406f4a826 100644
--- a/youtube_dl/extractor/thvideo.py
+++ b/youtube_dl/extractor/thvideo.py
@@ -10,7 +10,7 @@ from ..utils import (
 
 
 class THVideoIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?thvideo\.tv/(?:v/th|mobile\.php\?cid=)(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?thvideo\.tv/(?:v/th|mobile\.php\?cid=)(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://thvideo.tv/v/th1987/',
         'md5': 'fa107b1f73817e325e9433505a70db50',
diff --git a/youtube_dl/extractor/tinypic.py b/youtube_dl/extractor/tinypic.py
index e036b8cdf..c43cace24 100644
--- a/youtube_dl/extractor/tinypic.py
+++ b/youtube_dl/extractor/tinypic.py
@@ -9,7 +9,7 @@ from ..utils import ExtractorError
 class TinyPicIE(InfoExtractor):
     IE_NAME = 'tinypic'
     IE_DESC = 'tinypic.com videos'
-    _VALID_URL = r'http://(?:.+?\.)?tinypic\.com/player\.php\?v=(?P<id>[^&]+)&s=\d+'
+    _VALID_URL = r'https?://(?:.+?\.)?tinypic\.com/player\.php\?v=(?P<id>[^&]+)&s=\d+'
 
     _TESTS = [
         {
diff --git a/youtube_dl/extractor/tlc.py b/youtube_dl/extractor/tlc.py
index 17add9543..abad3ff64 100644
--- a/youtube_dl/extractor/tlc.py
+++ b/youtube_dl/extractor/tlc.py
@@ -9,7 +9,7 @@ from ..compat import compat_parse_qs
 
 class TlcDeIE(InfoExtractor):
     IE_NAME = 'tlc.de'
-    _VALID_URL = r'http://www\.tlc\.de/(?:[^/]+/)*videos/(?P<title>[^/?#]+)?(?:.*#(?P<id>\d+))?'
+    _VALID_URL = r'https?://www\.tlc\.de/(?:[^/]+/)*videos/(?P<title>[^/?#]+)?(?:.*#(?P<id>\d+))?'
 
     _TEST = {
         'url': 'http://www.tlc.de/sendungen/breaking-amish/videos/#3235167922001',
diff --git a/youtube_dl/extractor/toypics.py b/youtube_dl/extractor/toypics.py
index 2756f56d3..2579ba8c6 100644
--- a/youtube_dl/extractor/toypics.py
+++ b/youtube_dl/extractor/toypics.py
@@ -41,7 +41,7 @@ class ToypicsIE(InfoExtractor):
 
 class ToypicsUserIE(InfoExtractor):
     IE_DESC = 'Toypics user profile'
-    _VALID_URL = r'http://videos\.toypics\.net/(?P<username>[^/?]+)(?:$|[?#])'
+    _VALID_URL = r'https?://videos\.toypics\.net/(?P<username>[^/?]+)(?:$|[?#])'
     _TEST = {
         'url': 'http://videos.toypics.net/Mikey',
         'info_dict': {
diff --git a/youtube_dl/extractor/traileraddict.py b/youtube_dl/extractor/traileraddict.py
index 0e01b15fc..747370d12 100644
--- a/youtube_dl/extractor/traileraddict.py
+++ b/youtube_dl/extractor/traileraddict.py
@@ -7,7 +7,7 @@ from .common import InfoExtractor
 
 class TrailerAddictIE(InfoExtractor):
     _WORKING = False
-    _VALID_URL = r'(?:http://)?(?:www\.)?traileraddict\.com/(?:trailer|clip)/(?P<movie>.+?)/(?P<trailer_name>.+)'
+    _VALID_URL = r'(?:https?://)?(?:www\.)?traileraddict\.com/(?:trailer|clip)/(?P<movie>.+?)/(?P<trailer_name>.+)'
     _TEST = {
         'url': 'http://www.traileraddict.com/trailer/prince-avalanche/trailer',
         'md5': '41365557f3c8c397d091da510e73ceb4',
diff --git a/youtube_dl/extractor/trollvids.py b/youtube_dl/extractor/trollvids.py
index d239949a6..657705623 100644
--- a/youtube_dl/extractor/trollvids.py
+++ b/youtube_dl/extractor/trollvids.py
@@ -7,7 +7,7 @@ from .nuevo import NuevoBaseIE
 
 
 class TrollvidsIE(NuevoBaseIE):
-    _VALID_URL = r'http://(?:www\.)?trollvids\.com/video/(?P<id>\d+)/(?P<display_id>[^/?#&]+)'
+    _VALID_URL = r'https?://(?:www\.)?trollvids\.com/video/(?P<id>\d+)/(?P<display_id>[^/?#&]+)'
     IE_NAME = 'trollvids'
     _TEST = {
         'url': 'http://trollvids.com/video/2349002/%E3%80%90MMD-R-18%E3%80%91%E3%82%AC%E3%83%BC%E3%83%AB%E3%83%95%E3%83%AC%E3%83%B3%E3%83%89-carrymeoff',
diff --git a/youtube_dl/extractor/tumblr.py b/youtube_dl/extractor/tumblr.py
index 4f844706d..cea117c79 100644
--- a/youtube_dl/extractor/tumblr.py
+++ b/youtube_dl/extractor/tumblr.py
@@ -8,7 +8,7 @@ from ..utils import int_or_none
 
 
 class TumblrIE(InfoExtractor):
-    _VALID_URL = r'http://(?P<blog_name>.*?)\.tumblr\.com/(?:post|video)/(?P<id>[0-9]+)(?:$|[/?#])'
+    _VALID_URL = r'https?://(?P<blog_name>.*?)\.tumblr\.com/(?:post|video)/(?P<id>[0-9]+)(?:$|[/?#])'
     _TESTS = [{
         'url': 'http://tatianamaslanydaily.tumblr.com/post/54196191430/orphan-black-dvd-extra-behind-the-scenes',
         'md5': '479bb068e5b16462f5176a6828829767',
diff --git a/youtube_dl/extractor/tv2.py b/youtube_dl/extractor/tv2.py
index 535d0d361..86bb7915d 100644
--- a/youtube_dl/extractor/tv2.py
+++ b/youtube_dl/extractor/tv2.py
@@ -14,7 +14,7 @@ from ..utils import (
 
 
 class TV2IE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?tv2\.no/v/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?tv2\.no/v/(?P<id>\d+)'
     _TEST = {
         'url': 'http://www.tv2.no/v/916509/',
         'info_dict': {
@@ -100,7 +100,7 @@ class TV2IE(InfoExtractor):
 
 
 class TV2ArticleIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?tv2\.no/(?:a|\d{4}/\d{2}/\d{2}(/[^/]+)+)/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?tv2\.no/(?:a|\d{4}/\d{2}/\d{2}(/[^/]+)+)/(?P<id>\d+)'
     _TESTS = [{
         'url': 'http://www.tv2.no/2015/05/16/nyheter/alesund/krim/pingvin/6930542',
         'info_dict': {
diff --git a/youtube_dl/extractor/tvc.py b/youtube_dl/extractor/tvc.py
index 3a4f393fc..4065354dd 100644
--- a/youtube_dl/extractor/tvc.py
+++ b/youtube_dl/extractor/tvc.py
@@ -11,7 +11,7 @@ from ..utils import (
 
 
 class TVCIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?tvc\.ru/video/iframe/id/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?tvc\.ru/video/iframe/id/(?P<id>\d+)'
     _TEST = {
         'url': 'http://www.tvc.ru/video/iframe/id/74622/isPlay/false/id_stat/channel/?acc_video_id=/channel/brand/id/17/show/episodes/episode_id/39702',
         'md5': 'bbc5ff531d1e90e856f60fc4b3afd708',
@@ -64,7 +64,7 @@ class TVCIE(InfoExtractor):
 
 
 class TVCArticleIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?tvc\.ru/(?!video/iframe/id/)(?P<id>[^?#]+)'
+    _VALID_URL = r'https?://(?:www\.)?tvc\.ru/(?!video/iframe/id/)(?P<id>[^?#]+)'
     _TESTS = [{
         'url': 'http://www.tvc.ru/channel/brand/id/29/show/episodes/episode_id/39702/',
         'info_dict': {
diff --git a/youtube_dl/extractor/tvplay.py b/youtube_dl/extractor/tvplay.py
index b4683de54..df70a6b23 100644
--- a/youtube_dl/extractor/tvplay.py
+++ b/youtube_dl/extractor/tvplay.py
@@ -13,7 +13,7 @@ from ..utils import (
 
 class TVPlayIE(InfoExtractor):
     IE_DESC = 'TV3Play and related services'
-    _VALID_URL = r'''(?x)http://(?:www\.)?
+    _VALID_URL = r'''(?x)https?://(?:www\.)?
         (?:tvplay\.lv/parraides|
            tv3play\.lt/programos|
            play\.tv3\.lt/programos|
diff --git a/youtube_dl/extractor/ubu.py b/youtube_dl/extractor/ubu.py
index d50237758..1d52cbc98 100644
--- a/youtube_dl/extractor/ubu.py
+++ b/youtube_dl/extractor/ubu.py
@@ -10,7 +10,7 @@ from ..utils import (
 
 
 class UbuIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?ubu\.com/film/(?P<id>[\da-z_-]+)\.html'
+    _VALID_URL = r'https?://(?:www\.)?ubu\.com/film/(?P<id>[\da-z_-]+)\.html'
     _TEST = {
         'url': 'http://ubu.com/film/her_noise.html',
         'md5': '138d5652618bf0f03878978db9bef1ee',
diff --git a/youtube_dl/extractor/unistra.py b/youtube_dl/extractor/unistra.py
index 594bee4f9..66d9f1bf3 100644
--- a/youtube_dl/extractor/unistra.py
+++ b/youtube_dl/extractor/unistra.py
@@ -7,7 +7,7 @@ from ..utils import qualities
 
 
 class UnistraIE(InfoExtractor):
-    _VALID_URL = r'http://utv\.unistra\.fr/(?:index|video)\.php\?id_video\=(?P<id>\d+)'
+    _VALID_URL = r'https?://utv\.unistra\.fr/(?:index|video)\.php\?id_video\=(?P<id>\d+)'
 
     _TESTS = [
         {
diff --git a/youtube_dl/extractor/vbox7.py b/youtube_dl/extractor/vbox7.py
index 3794bcded..b755dda90 100644
--- a/youtube_dl/extractor/vbox7.py
+++ b/youtube_dl/extractor/vbox7.py
@@ -13,7 +13,7 @@ from ..utils import (
 
 
 class Vbox7IE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?vbox7\.com/play:(?P<id>[^/]+)'
+    _VALID_URL = r'https?://(?:www\.)?vbox7\.com/play:(?P<id>[^/]+)'
     _TEST = {
         'url': 'http://vbox7.com/play:249bb972c2',
         'md5': '99f65c0c9ef9b682b97313e052734c3f',
diff --git a/youtube_dl/extractor/veoh.py b/youtube_dl/extractor/veoh.py
index 9633f7ffe..23ce0a0d1 100644
--- a/youtube_dl/extractor/veoh.py
+++ b/youtube_dl/extractor/veoh.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 
 class VeohIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?veoh\.com/(?:watch|iphone/#_Watch)/(?P<id>(?:v|yapi-)[\da-zA-Z]+)'
+    _VALID_URL = r'https?://(?:www\.)?veoh\.com/(?:watch|iphone/#_Watch)/(?P<id>(?:v|yapi-)[\da-zA-Z]+)'
 
     _TESTS = [
         {
diff --git a/youtube_dl/extractor/vesti.py b/youtube_dl/extractor/vesti.py
index a0c59a2e0..cb64ae0bd 100644
--- a/youtube_dl/extractor/vesti.py
+++ b/youtube_dl/extractor/vesti.py
@@ -10,7 +10,7 @@ from .rutv import RUTVIE
 
 class VestiIE(InfoExtractor):
     IE_DESC = 'Вести.Ru'
-    _VALID_URL = r'http://(?:.+?\.)?vesti\.ru/(?P<id>.+)'
+    _VALID_URL = r'https?://(?:.+?\.)?vesti\.ru/(?P<id>.+)'
 
     _TESTS = [
         {
diff --git a/youtube_dl/extractor/vgtv.py b/youtube_dl/extractor/vgtv.py
index 77d8978d4..b11cd254c 100644
--- a/youtube_dl/extractor/vgtv.py
+++ b/youtube_dl/extractor/vgtv.py
@@ -214,7 +214,7 @@ class VGTVIE(XstreamIE):
 class BTArticleIE(InfoExtractor):
     IE_NAME = 'bt:article'
     IE_DESC = 'Bergens Tidende Articles'
-    _VALID_URL = r'http://(?:www\.)?bt\.no/(?:[^/]+/)+(?P<id>[^/]+)-\d+\.html'
+    _VALID_URL = r'https?://(?:www\.)?bt\.no/(?:[^/]+/)+(?P<id>[^/]+)-\d+\.html'
     _TEST = {
         'url': 'http://www.bt.no/nyheter/lokalt/Kjemper-for-internatet-1788214.html',
         'md5': '2acbe8ad129b3469d5ae51b1158878df',
@@ -241,7 +241,7 @@ class BTArticleIE(InfoExtractor):
 class BTVestlendingenIE(InfoExtractor):
     IE_NAME = 'bt:vestlendingen'
     IE_DESC = 'Bergens Tidende - Vestlendingen'
-    _VALID_URL = r'http://(?:www\.)?bt\.no/spesial/vestlendingen/#!/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?bt\.no/spesial/vestlendingen/#!/(?P<id>\d+)'
     _TESTS = [{
         'url': 'http://www.bt.no/spesial/vestlendingen/#!/86588',
         'md5': 'd7d17e3337dc80de6d3a540aefbe441b',
diff --git a/youtube_dl/extractor/videott.py b/youtube_dl/extractor/videott.py
index 2cd36508a..0f798711b 100644
--- a/youtube_dl/extractor/videott.py
+++ b/youtube_dl/extractor/videott.py
@@ -14,7 +14,7 @@ class VideoTtIE(InfoExtractor):
     _WORKING = False
     ID_NAME = 'video.tt'
     IE_DESC = 'video.tt - Your True Tube'
-    _VALID_URL = r'http://(?:www\.)?video\.tt/(?:(?:video|embed)/|watch_video\.php\?v=)(?P<id>[\da-zA-Z]{9})'
+    _VALID_URL = r'https?://(?:www\.)?video\.tt/(?:(?:video|embed)/|watch_video\.php\?v=)(?P<id>[\da-zA-Z]{9})'
 
     _TESTS = [{
         'url': 'http://www.video.tt/watch_video.php?v=amd5YujV8',
diff --git a/youtube_dl/extractor/viidea.py b/youtube_dl/extractor/viidea.py
index 315984bf9..03b9f1353 100644
--- a/youtube_dl/extractor/viidea.py
+++ b/youtube_dl/extractor/viidea.py
@@ -15,7 +15,7 @@ from ..utils import (
 
 
 class ViideaIE(InfoExtractor):
-    _VALID_URL = r'''(?x)http://(?:www\.)?(?:
+    _VALID_URL = r'''(?x)https?://(?:www\.)?(?:
             videolectures\.net|
             flexilearn\.viidea\.net|
             presentations\.ocwconsortium\.org|
diff --git a/youtube_dl/extractor/vube.py b/youtube_dl/extractor/vube.py
index 149e36467..10ca6acb1 100644
--- a/youtube_dl/extractor/vube.py
+++ b/youtube_dl/extractor/vube.py
@@ -15,7 +15,7 @@ from ..utils import (
 class VubeIE(InfoExtractor):
     IE_NAME = 'vube'
     IE_DESC = 'Vube.com'
-    _VALID_URL = r'http://vube\.com/(?:[^/]+/)+(?P<id>[\da-zA-Z]{10})\b'
+    _VALID_URL = r'https?://vube\.com/(?:[^/]+/)+(?P<id>[\da-zA-Z]{10})\b'
 
     _TESTS = [
         {
diff --git a/youtube_dl/extractor/vuclip.py b/youtube_dl/extractor/vuclip.py
index a6d9b5fee..eaa888f00 100644
--- a/youtube_dl/extractor/vuclip.py
+++ b/youtube_dl/extractor/vuclip.py
@@ -14,7 +14,7 @@ from ..utils import (
 
 
 class VuClipIE(InfoExtractor):
-    _VALID_URL = r'http://(?:m\.)?vuclip\.com/w\?.*?cid=(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://(?:m\.)?vuclip\.com/w\?.*?cid=(?P<id>[0-9]+)'
 
     _TEST = {
         'url': 'http://m.vuclip.com/w?cid=922692425&fid=70295&z=1010&nvar&frm=index.html',
diff --git a/youtube_dl/extractor/walla.py b/youtube_dl/extractor/walla.py
index 24efbd6e6..8b9488340 100644
--- a/youtube_dl/extractor/walla.py
+++ b/youtube_dl/extractor/walla.py
@@ -11,7 +11,7 @@ from ..utils import (
 
 
 class WallaIE(InfoExtractor):
-    _VALID_URL = r'http://vod\.walla\.co\.il/[^/]+/(?P<id>\d+)/(?P<display_id>.+)'
+    _VALID_URL = r'https?://vod\.walla\.co\.il/[^/]+/(?P<id>\d+)/(?P<display_id>.+)'
     _TEST = {
         'url': 'http://vod.walla.co.il/movie/2642630/one-direction-all-for-one',
         'info_dict': {
diff --git a/youtube_dl/extractor/wat.py b/youtube_dl/extractor/wat.py
index 37cf3d309..5227bb5ad 100644
--- a/youtube_dl/extractor/wat.py
+++ b/youtube_dl/extractor/wat.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 
 class WatIE(InfoExtractor):
-    _VALID_URL = r'(?:wat:(?P<real_id>\d{8})|http://www\.wat\.tv/video/(?P<display_id>.*)-(?P<short_id>.*?)_.*?\.html)'
+    _VALID_URL = r'(?:wat:(?P<real_id>\d{8})|https?://www\.wat\.tv/video/(?P<display_id>.*)-(?P<short_id>.*?)_.*?\.html)'
     IE_NAME = 'wat.tv'
     _TESTS = [
         {
diff --git a/youtube_dl/extractor/wdr.py b/youtube_dl/extractor/wdr.py
index 65cab4069..31c904303 100644
--- a/youtube_dl/extractor/wdr.py
+++ b/youtube_dl/extractor/wdr.py
@@ -244,7 +244,7 @@ class WDRMobileIE(InfoExtractor):
 
 
 class WDRMausIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?wdrmaus\.de/(?:[^/]+/){,2}(?P<id>[^/?#]+)(?:/index\.php5|(?<!index)\.php5|/(?:$|[?#]))'
+    _VALID_URL = r'https?://(?:www\.)?wdrmaus\.de/(?:[^/]+/){,2}(?P<id>[^/?#]+)(?:/index\.php5|(?<!index)\.php5|/(?:$|[?#]))'
     IE_DESC = 'Sendung mit der Maus'
     _TESTS = [{
         'url': 'http://www.wdrmaus.de/aktuelle-sendung/index.php5',
diff --git a/youtube_dl/extractor/weiqitv.py b/youtube_dl/extractor/weiqitv.py
index e333ae345..3dafbeec2 100644
--- a/youtube_dl/extractor/weiqitv.py
+++ b/youtube_dl/extractor/weiqitv.py
@@ -6,7 +6,7 @@ from .common import InfoExtractor
 
 class WeiqiTVIE(InfoExtractor):
     IE_DESC = 'WQTV'
-    _VALID_URL = r'http://www\.weiqitv\.com/index/video_play\?videoId=(?P<id>[A-Za-z0-9]+)'
+    _VALID_URL = r'https?://www\.weiqitv\.com/index/video_play\?videoId=(?P<id>[A-Za-z0-9]+)'
 
     _TESTS = [{
         'url': 'http://www.weiqitv.com/index/video_play?videoId=53c744f09874f0e76a8b46f3',
diff --git a/youtube_dl/extractor/wimp.py b/youtube_dl/extractor/wimp.py
index fb0accac7..828c03dc3 100644
--- a/youtube_dl/extractor/wimp.py
+++ b/youtube_dl/extractor/wimp.py
@@ -5,7 +5,7 @@ from .youtube import YoutubeIE
 
 
 class WimpIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?wimp\.com/(?P<id>[^/]+)'
+    _VALID_URL = r'https?://(?:www\.)?wimp\.com/(?P<id>[^/]+)'
     _TESTS = [{
         'url': 'http://www.wimp.com/maruexhausted/',
         'md5': 'ee21217ffd66d058e8b16be340b74883',
diff --git a/youtube_dl/extractor/xbef.py b/youtube_dl/extractor/xbef.py
index 4ff99e5ca..e4a2baad2 100644
--- a/youtube_dl/extractor/xbef.py
+++ b/youtube_dl/extractor/xbef.py
@@ -5,7 +5,7 @@ from ..compat import compat_urllib_parse_unquote
 
 
 class XBefIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?xbef\.com/video/(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?xbef\.com/video/(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://xbef.com/video/5119-glamourous-lesbians-smoking-drinking-and-fucking',
         'md5': 'a478b565baff61634a98f5e5338be995',
diff --git a/youtube_dl/extractor/yam.py b/youtube_dl/extractor/yam.py
index 001ee17b6..63bbc0634 100644
--- a/youtube_dl/extractor/yam.py
+++ b/youtube_dl/extractor/yam.py
@@ -15,7 +15,7 @@ from ..utils import (
 
 class YamIE(InfoExtractor):
     IE_DESC = '蕃薯藤yam天空部落'
-    _VALID_URL = r'http://mymedia.yam.com/m/(?P<id>\d+)'
+    _VALID_URL = r'https?://mymedia.yam.com/m/(?P<id>\d+)'
 
     _TESTS = [{
         # An audio hosted on Yam
diff --git a/youtube_dl/extractor/ynet.py b/youtube_dl/extractor/ynet.py
index 869f3e819..2522551dc 100644
--- a/youtube_dl/extractor/ynet.py
+++ b/youtube_dl/extractor/ynet.py
@@ -9,7 +9,7 @@ from ..compat import compat_urllib_parse_unquote_plus
 
 
 class YnetIE(InfoExtractor):
-    _VALID_URL = r'http://(?:.+?\.)?ynet\.co\.il/(?:.+?/)?0,7340,(?P<id>L(?:-[0-9]+)+),00\.html'
+    _VALID_URL = r'https?://(?:.+?\.)?ynet\.co\.il/(?:.+?/)?0,7340,(?P<id>L(?:-[0-9]+)+),00\.html'
     _TESTS = [
         {
             'url': 'http://hot.ynet.co.il/home/0,7340,L-11659-99244,00.html',

From 1600ed1ff9edb33ef901bda2c4b2732df7e0e4e7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 21 Mar 2016 21:46:49 +0600
Subject: [PATCH 4384/6219] [rutv] Improve flash version pattern (Closes #8911)

---
 youtube_dl/extractor/rutv.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/rutv.py b/youtube_dl/extractor/rutv.py
index f7fe1fece..a2379eb04 100644
--- a/youtube_dl/extractor/rutv.py
+++ b/youtube_dl/extractor/rutv.py
@@ -14,7 +14,7 @@ class RUTVIE(InfoExtractor):
     IE_DESC = 'RUTV.RU'
     _VALID_URL = r'''(?x)
         https?://player\.(?:rutv\.ru|vgtrk\.com)/
-            (?P<path>flash2v/container\.swf\?id=
+            (?P<path>flash\d+v/container\.swf\?id=
             |iframe/(?P<type>swf|video|live)/id/
             |index/iframe/cast_id/)
             (?P<id>\d+)'''
@@ -109,7 +109,7 @@ class RUTVIE(InfoExtractor):
             return mobj.group('url')
 
         mobj = re.search(
-            r'<meta[^>]+?property=(["\'])og:video\1[^>]+?content=(["\'])(?P<url>https?://player\.(?:rutv\.ru|vgtrk\.com)/flash2v/container\.swf\?id=.+?\2)',
+            r'<meta[^>]+?property=(["\'])og:video\1[^>]+?content=(["\'])(?P<url>https?://player\.(?:rutv\.ru|vgtrk\.com)/flash\d+v/container\.swf\?id=.+?\2)',
             webpage)
         if mobj:
             return mobj.group('url')
@@ -119,7 +119,7 @@ class RUTVIE(InfoExtractor):
         video_id = mobj.group('id')
         video_path = mobj.group('path')
 
-        if video_path.startswith('flash2v'):
+        if re.match(r'flash\d+v', video_path):
             video_type = 'video'
         elif video_path.startswith('iframe'):
             video_type = mobj.group('type')
@@ -168,7 +168,7 @@ class RUTVIE(InfoExtractor):
                         'play_path': mobj.group('playpath'),
                         'app': mobj.group('app'),
                         'page_url': 'http://player.rutv.ru',
-                        'player_url': 'http://player.rutv.ru/flash2v/osmf.swf?i=22',
+                        'player_url': 'http://player.rutv.ru/flash3v/osmf.swf?i=22',
                         'rtmp_live': True,
                         'ext': 'flv',
                         'vbr': int(quality),

From 065c4b27bfc4488758b357e023e9b6b1679c9641 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 21 Mar 2016 22:07:34 +0600
Subject: [PATCH 4385/6219] [xhamster:embed] Extract vars (Closes #8912)

---
 youtube_dl/extractor/xhamster.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index fd43e8854..b3547174d 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -4,6 +4,7 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
+    dict_get,
     float_or_none,
     int_or_none,
     unified_strdate,
@@ -170,6 +171,12 @@ class XHamsterEmbedIE(InfoExtractor):
 
         video_url = self._search_regex(
             r'href="(https?://xhamster\.com/movies/%s/[^"]+\.html[^"]*)"' % video_id,
-            webpage, 'xhamster url')
+            webpage, 'xhamster url', default=None)
+
+        if not video_url:
+            vars = self._parse_json(
+                self._search_regex(r'vars\s*:\s*({.+?})\s*,\s*\n', webpage, 'vars'),
+                video_id)
+            video_url = dict_get(vars, ('downloadLink', 'homepageLink', 'commentsLink', 'shareUrl'))
 
         return self.url_result(video_url, 'XHamster')

From ff5873b72de16854ae8d506d5648148a54828243 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 21 Mar 2016 22:24:42 +0600
Subject: [PATCH 4386/6219] [motherless] Detect friends only videos

---
 youtube_dl/extractor/motherless.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/motherless.py b/youtube_dl/extractor/motherless.py
index ad04b12cd..5e1a8a71a 100644
--- a/youtube_dl/extractor/motherless.py
+++ b/youtube_dl/extractor/motherless.py
@@ -69,6 +69,9 @@ class MotherlessIE(InfoExtractor):
                 ">The page you're looking for cannot be found.<")):
             raise ExtractorError('Video %s does not exist' % video_id, expected=True)
 
+        if '>The content you are trying to view is for friends only.' in webpage:
+            raise ExtractorError('Video %s is for friends only' % video_id, expected=True)
+
         title = self._html_search_regex(
             r'id="view-upload-title">\s+([^<]+)<', webpage, 'title')
         video_url = self._html_search_regex(

From 5c69f7a479936a8fc429228c8259dc5fcbf428b1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 21 Mar 2016 23:31:40 +0600
Subject: [PATCH 4387/6219] [animeondemand] Respect startvideo (Closes #8923)

---
 youtube_dl/extractor/animeondemand.py | 37 +++++++++++++++++++--------
 1 file changed, 26 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/animeondemand.py b/youtube_dl/extractor/animeondemand.py
index 4352525e2..81a843035 100644
--- a/youtube_dl/extractor/animeondemand.py
+++ b/youtube_dl/extractor/animeondemand.py
@@ -163,7 +163,7 @@ class AnimeOnDemandIE(InfoExtractor):
                     if kind:
                         format_id_list.append(kind)
                     if not format_id_list:
-                        format_id_list.append('hls')
+                        format_id_list.append(compat_str(num))
                     format_id = '-'.join(format_id_list)
                     format_note = ', '.join(filter(None, (kind, lang_note)))
                     request = sanitized_Request(
@@ -179,26 +179,41 @@ class AnimeOnDemandIE(InfoExtractor):
                         fatal=False)
                     if not playlist:
                         continue
+                    start_video = playlist.get('startvideo', 0)
                     playlist = playlist.get('playlist')
                     if not playlist or not isinstance(playlist, list):
                         continue
-                    playlist = playlist[0]
+                    playlist = playlist[start_video]
                     title = playlist.get('title')
                     if not title:
                         continue
                     description = playlist.get('description')
                     for source in playlist.get('sources', []):
                         file_ = source.get('file')
-                        if file_ and determine_ext(file_) == 'm3u8':
-                            m3u8_formats = self._extract_m3u8_formats(
+                        if not file_:
+                            continue
+                        ext = determine_ext(file_)
+                        format_id_list = [lang, kind]
+                        if ext == 'm3u8':
+                            format_id_list.append('hls')
+                        elif source.get('type') == 'video/dash' or ext == 'mpd':
+                            format_id_list.append('dash')
+                        format_id = '-'.join(filter(None, format_id_list))
+                        if ext == 'm3u8':
+                            file_formats = self._extract_m3u8_formats(
                                 file_, video_id, 'mp4',
-                                entry_protocol='m3u8_native', m3u8_id=format_id)
-                            for f in m3u8_formats:
-                                f.update({
-                                    'language': lang,
-                                    'format_note': format_note,
-                                })
-                            formats.extend(m3u8_formats)
+                                entry_protocol='m3u8_native', m3u8_id=format_id, fatal=False)
+                        elif source.get('type') == 'video/dash' or ext == 'mpd':
+                            file_formats = self._extract_mpd_formats(
+                                file_, video_id, mpd_id=format_id, fatal=False)
+                        else:
+                            continue
+                        for f in file_formats:
+                            f.update({
+                                'language': lang,
+                                'format_note': format_note,
+                            })
+                        formats.extend(file_formats)
 
             if formats:
                 self._sort_formats(formats)

From 85c637b7376f0426e5e0a6812da2a72b2ca28680 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 21 Mar 2016 23:35:50 +0600
Subject: [PATCH 4388/6219] [animeondemand] Extract teaser when no full episode
 available (#8923)

---
 youtube_dl/extractor/animeondemand.py | 24 +++++++++++++-----------
 1 file changed, 13 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/animeondemand.py b/youtube_dl/extractor/animeondemand.py
index 81a843035..a47697738 100644
--- a/youtube_dl/extractor/animeondemand.py
+++ b/youtube_dl/extractor/animeondemand.py
@@ -225,16 +225,18 @@ class AnimeOnDemandIE(InfoExtractor):
                 })
                 entries.append(f)
 
-            m = re.search(
-                r'data-dialog-header=(["\'])(?P<title>.+?)\1[^>]+href=(["\'])(?P<href>.+?)\3[^>]*>Teaser<',
-                episode_html)
-            if m:
-                f = common_info.copy()
-                f.update({
-                    'id': '%s-teaser' % f['id'],
-                    'title': m.group('title'),
-                    'url': compat_urlparse.urljoin(url, m.group('href')),
-                })
-                entries.append(f)
+            # Extract teaser only when full episode is not available
+            if not formats:
+                m = re.search(
+                    r'data-dialog-header=(["\'])(?P<title>.+?)\1[^>]+href=(["\'])(?P<href>.+?)\3[^>]*>Teaser<',
+                    episode_html)
+                if m:
+                    f = common_info.copy()
+                    f.update({
+                        'id': '%s-teaser' % f['id'],
+                        'title': m.group('title'),
+                        'url': compat_urlparse.urljoin(url, m.group('href')),
+                    })
+                    entries.append(f)
 
         return self.playlist_result(entries, anime_id, anime_title, anime_description)

From bc5d16b3023ab29216d868bbfe8838b5e92e72f6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 21 Mar 2016 23:37:39 +0600
Subject: [PATCH 4389/6219] [animeondemand] Skip dash for now

---
 youtube_dl/extractor/animeondemand.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/animeondemand.py b/youtube_dl/extractor/animeondemand.py
index a47697738..2cede55a7 100644
--- a/youtube_dl/extractor/animeondemand.py
+++ b/youtube_dl/extractor/animeondemand.py
@@ -204,6 +204,7 @@ class AnimeOnDemandIE(InfoExtractor):
                                 file_, video_id, 'mp4',
                                 entry_protocol='m3u8_native', m3u8_id=format_id, fatal=False)
                         elif source.get('type') == 'video/dash' or ext == 'mpd':
+                            continue
                             file_formats = self._extract_mpd_formats(
                                 file_, video_id, mpd_id=format_id, fatal=False)
                         else:

From cc7397b04d4a21b5ac680858ee2600e3b3bfb569 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 22 Mar 2016 21:12:29 +0600
Subject: [PATCH 4390/6219] [ceskatelevize] Make m3u8 formats extraction non
 fatal (Closes #8933)

---
 youtube_dl/extractor/ceskatelevize.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ceskatelevize.py b/youtube_dl/extractor/ceskatelevize.py
index b27b4e670..b355111cb 100644
--- a/youtube_dl/extractor/ceskatelevize.py
+++ b/youtube_dl/extractor/ceskatelevize.py
@@ -129,7 +129,8 @@ class CeskaTelevizeIE(InfoExtractor):
             formats = []
             for format_id, stream_url in item['streamUrls'].items():
                 formats.extend(self._extract_m3u8_formats(
-                    stream_url, playlist_id, 'mp4', entry_protocol='m3u8_native'))
+                    stream_url, playlist_id, 'mp4',
+                    entry_protocol='m3u8_native', fatal=False))
             self._sort_formats(formats)
 
             item_id = item.get('id') or item['assetId']

From 2beeb286e179a00bc0c76fc55ca5c8d19e74ca41 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 22 Mar 2016 22:32:59 +0600
Subject: [PATCH 4391/6219] [laola1tv] Add support for livestreams (Closes
 #8934)

---
 youtube_dl/extractor/laola1tv.py | 31 ++++++++++++++++++++++++++-----
 1 file changed, 26 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/laola1tv.py b/youtube_dl/extractor/laola1tv.py
index 5d8ebbeb3..41d80bc12 100644
--- a/youtube_dl/extractor/laola1tv.py
+++ b/youtube_dl/extractor/laola1tv.py
@@ -19,7 +19,7 @@ from ..utils import (
 
 
 class Laola1TvIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?laola1\.tv/(?P<lang>[a-z]+)-(?P<portal>[a-z]+)/[^/]+/(?P<slug>[^/?#&]+)'
+    _VALID_URL = r'https?://(?:www\.)?laola1\.tv/(?P<lang>[a-z]+)-(?P<portal>[a-z]+)/(?P<kind>[^/]+)/(?P<slug>[^/?#&]+)'
     _TESTS = [{
         'url': 'http://www.laola1.tv/de-de/video/straubing-tigers-koelner-haie/227883.html',
         'info_dict': {
@@ -33,7 +33,7 @@ class Laola1TvIE(InfoExtractor):
         },
         'params': {
             'skip_download': True,
-        }
+        },
     }, {
         'url': 'http://www.laola1.tv/de-de/video/straubing-tigers-koelner-haie',
         'info_dict': {
@@ -47,12 +47,28 @@ class Laola1TvIE(InfoExtractor):
         },
         'params': {
             'skip_download': True,
-        }
+        },
+    }, {
+        'url': 'http://www.laola1.tv/de-de/livestream/2016-03-22-belogorie-belgorod-trentino-diatec-lde',
+        'info_dict': {
+            'id': '487850',
+            'display_id': '2016-03-22-belogorie-belgorod-trentino-diatec-lde',
+            'ext': 'flv',
+            'title': 'Belogorie BELGOROD - TRENTINO Diatec',
+            'upload_date': '20160322',
+            'uploader': 'CEV - Europäischer Volleyball Verband',
+            'is_live': True,
+            'categories': ['Volleyball'],
+        },
+        'params': {
+            'skip_download': True,
+        },
     }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         display_id = mobj.group('slug')
+        kind = mobj.group('kind')
         lang = mobj.group('lang')
         portal = mobj.group('portal')
 
@@ -85,12 +101,17 @@ class Laola1TvIE(InfoExtractor):
         _v = lambda x, **k: xpath_text(hd_doc, './/video/' + x, **k)
         title = _v('title', fatal=True)
 
+        VS_TARGETS = {
+            'video': '2',
+            'livestream': '17',
+        }
+
         req = sanitized_Request(
             'https://club.laola1.tv/sp/laola1/api/v3/user/session/premium/player/stream-access?%s' %
             compat_urllib_parse.urlencode({
                 'videoId': video_id,
-                'target': '2',
-                'label': 'laola1tv',
+                'target': VS_TARGETS.get(kind, '2'),
+                'label': _v('label'),
                 'area': _v('area'),
             }),
             urlencode_postdata(

From c6ca11f1b31a292413cab03012654fd0021814a8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 22 Mar 2016 23:48:05 +0600
Subject: [PATCH 4392/6219] [once] Prevent ads from embedding into m3u8
 playlists (Closes #8893)

---
 youtube_dl/extractor/once.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/once.py b/youtube_dl/extractor/once.py
index 080045d4c..5db949b17 100644
--- a/youtube_dl/extractor/once.py
+++ b/youtube_dl/extractor/once.py
@@ -20,6 +20,10 @@ class OnceIE(InfoExtractor):
             media_item_id, 'mp4', m3u8_id='hls', fatal=False)
         progressive_formats = []
         for adaptive_format in formats:
+            # Prevent advertisement from embedding into m3u8 playlist (see
+            # https://github.com/rg3/youtube-dl/issues/8893#issuecomment-199912684)
+            adaptive_format['url'] = re.sub(
+                r'\badsegmentlength=\d+', r'adsegmentlength=0', adaptive_format['url'])
             rendition_id = self._search_regex(
                 r'/now/media/playlist/[^/]+/[^/]+/([^/]+)',
                 adaptive_format['url'], 'redition id', default=None)

From 7da2c87119db8beda1bdc979fad38c08fc1252e9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 22 Mar 2016 22:17:59 +0100
Subject: [PATCH 4393/6219] Add extractor for thescene.com (closes #8929)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/thescene.py | 48 ++++++++++++++++++++++++++++++++
 2 files changed, 49 insertions(+)
 create mode 100644 youtube_dl/extractor/thescene.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index ee792bbe0..8f7df4d12 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -743,6 +743,7 @@ from .theplatform import (
     ThePlatformIE,
     ThePlatformFeedIE,
 )
+from .thescene import TheSceneIE
 from .thesixtyone import TheSixtyOneIE
 from .thestar import TheStarIE
 from .thisamericanlife import ThisAmericanLifeIE
diff --git a/youtube_dl/extractor/thescene.py b/youtube_dl/extractor/thescene.py
new file mode 100644
index 000000000..08d666eaf
--- /dev/null
+++ b/youtube_dl/extractor/thescene.py
@@ -0,0 +1,48 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+from ..compat import compat_urllib_parse
+from ..utils import qualities
+
+
+class TheSceneIE(InfoExtractor):
+    _VALID_URL = r'https://thescene\.com/watch/[^/]+/(?P<id>[^/#?]+)'
+
+    _TEST = {
+        'url': 'https://thescene.com/watch/vogue/narciso-rodriguez-spring-2013-ready-to-wear',
+        'info_dict': {
+            'id': '520e8faac2b4c00e3c6e5f43',
+            'ext': 'mp4',
+            'title': 'Narciso Rodriguez: Spring 2013 Ready-to-Wear',
+            'display_id': 'narciso-rodriguez-spring-2013-ready-to-wear',
+        },
+    }
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+        player_url = compat_urllib_parse.urljoin(
+            url,
+            self._html_search_regex(
+                r'id=\'js-player-script\'[^>]+src=\'(.+?)\'', webpage, 'player url'))
+
+        self.to_screen(player_url)
+        player = self._download_webpage(player_url, player_url)
+        info = self._parse_json(self._search_regex(r'(?m)var\s+video\s+=\s+({.+?});$', player, 'info json'), display_id)
+
+        qualities_order = qualities(['low', 'high'])
+        formats = [{
+            'format_id': '{0}-{1}'.format(f['type'].split('/')[0], f['quality']),
+            'url': f['src'],
+            'quality': qualities_order(f['quality']),
+        } for f in info['sources'][0]]
+        self._sort_formats(formats)
+
+        return {
+            'id': info['id'],
+            'title': info['title'],
+            'formats': formats,
+            'thumbnail': info.get('poster_frame'),
+            'display_id': display_id,
+        }

From efbed08dc20c530fe428256e4dcbea4dc4423d0d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 23 Mar 2016 22:24:52 +0800
Subject: [PATCH 4394/6219] [utils] Encode hostnames before passing to urllib

With IDN (Internationalized Domain Name) and a proxy, non-ascii URLs
are passed down to urllib/urllib2, causing UnicodeEncodeError

Fixes #8890
---
 test/test_http.py   | 10 ++++++++++
 youtube_dl/utils.py |  1 +
 2 files changed, 11 insertions(+)

diff --git a/test/test_http.py b/test/test_http.py
index fc59b1aed..15e0ad369 100644
--- a/test/test_http.py
+++ b/test/test_http.py
@@ -1,4 +1,5 @@
 #!/usr/bin/env python
+# coding: utf-8
 from __future__ import unicode_literals
 
 # Allow direct execution
@@ -120,5 +121,14 @@ class TestProxy(unittest.TestCase):
         response = ydl.urlopen(req).read().decode('utf-8')
         self.assertEqual(response, 'cn: {0}'.format(url))
 
+    def test_proxy_with_idn(self):
+        ydl = YoutubeDL({
+            'proxy': 'localhost:{0}'.format(self.port),
+        })
+        url = 'http://中文.tw/'
+        response = ydl.urlopen(url).read().decode('utf-8')
+        # b'xn--fiq228c' is '中文'.encode('idna')
+        self.assertEqual(response, 'normal: http://xn--fiq228c.tw/')
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 067b8a184..03bb7782f 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1746,6 +1746,7 @@ def escape_url(url):
     """Escape URL as suggested by RFC 3986"""
     url_parsed = compat_urllib_parse_urlparse(url)
     return url_parsed._replace(
+        netloc=url_parsed.netloc.encode('idna').decode('ascii'),
         path=escape_rfc3986(url_parsed.path),
         params=escape_rfc3986(url_parsed.params),
         query=escape_rfc3986(url_parsed.query),

From 882c6992967914c245e086ddaacde9d595cd6ed9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 23 Mar 2016 20:45:39 +0600
Subject: [PATCH 4395/6219] [tunein] Fix stream data extraction (Closes #8899,
 closes #8924)

---
 youtube_dl/extractor/tunein.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/tunein.py b/youtube_dl/extractor/tunein.py
index 8322cc14d..ae4cfaec2 100644
--- a/youtube_dl/extractor/tunein.py
+++ b/youtube_dl/extractor/tunein.py
@@ -1,7 +1,7 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import json
+import re
 
 from .common import InfoExtractor
 from ..utils import ExtractorError
@@ -27,10 +27,9 @@ class TuneInBaseIE(InfoExtractor):
         if not streams_url.startswith('http://'):
             streams_url = compat_urlparse.urljoin(url, streams_url)
 
-        stream_data = self._download_webpage(
-            streams_url, content_id, note='Downloading stream data')
-        streams = json.loads(self._search_regex(
-            r'\((.*)\);', stream_data, 'stream info'))['Streams']
+        streams = self._download_json(
+            streams_url, content_id, note='Downloading stream data',
+            transform_source=lambda s: re.sub(r'^\s*\((.*)\);\s*$', r'\1', s))['Streams']
 
         is_live = None
         formats = []

From 2d60465e44c3290fa1ee4239fe18eb9d0c69d9e9 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 23 Mar 2016 23:20:28 +0800
Subject: [PATCH 4396/6219] [test/test_utils] Update for escape_url change

---
 test/test_utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 325b870cc..8ba531542 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -577,7 +577,7 @@ class TestUtil(unittest.TestCase):
         )
         self.assertEqual(
             escape_url('http://тест.рф/фрагмент'),
-            'http://тест.рф/%D1%84%D1%80%D0%B0%D0%B3%D0%BC%D0%B5%D0%BD%D1%82'
+            'http://xn--e1aybc.xn--p1ai/%D1%84%D1%80%D0%B0%D0%B3%D0%BC%D0%B5%D0%BD%D1%82'
         )
         self.assertEqual(
             escape_url('http://тест.рф/абв?абв=абв#абв'),

From 81f36eba886349475235dbacef29d6b23b40538f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 23 Mar 2016 23:23:26 +0800
Subject: [PATCH 4397/6219] [test/test_utils] Update for escape_url change
 (again)

---
 test/test_utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 8ba531542..a35debfe1 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -581,7 +581,7 @@ class TestUtil(unittest.TestCase):
         )
         self.assertEqual(
             escape_url('http://тест.рф/абв?абв=абв#абв'),
-            'http://тест.рф/%D0%B0%D0%B1%D0%B2?%D0%B0%D0%B1%D0%B2=%D0%B0%D0%B1%D0%B2#%D0%B0%D0%B1%D0%B2'
+            'http://xn--e1aybc.xn--p1ai/%D0%B0%D0%B1%D0%B2?%D0%B0%D0%B1%D0%B2=%D0%B0%D0%B1%D0%B2#%D0%B0%D0%B1%D0%B2'
         )
         self.assertEqual(escape_url('http://vimeo.com/56015672#at=0'), 'http://vimeo.com/56015672#at=0')
 

From 088e1aac5970ea2c24fa902873a5e0b984b37595 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 23 Mar 2016 23:55:08 +0800
Subject: [PATCH 4398/6219] [generic] Support Vine embeds (#8817)

---
 youtube_dl/extractor/generic.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 26de27a7e..93e0563b5 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1909,6 +1909,14 @@ class GenericIE(InfoExtractor):
                 self._proto_relative_url(unescapeHTML(mobj.group(1))),
                 'AdobeTVVideo')
 
+        # Look for Vine embeds
+        mobj = re.search(
+            r'<iframe[^>]+src=[\'"]((?:https?:)?//(?:www\.)?vine\.co/v/[^/]+/embed/(?:simple|postcard))',
+            webpage)
+        if mobj is not None:
+            return self.url_result(
+                self._proto_relative_url(unescapeHTML(mobj.group(1))), 'Vine')
+
         def check_video(vurl):
             if YoutubeIE.suitable(vurl):
                 return True

From fc27ea94642a8e2e9b0fcfdcc0c370ec7484c971 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 23 Mar 2016 23:55:52 +0800
Subject: [PATCH 4399/6219] [tumblr] Support Vine embeds (#8817)

---
 youtube_dl/extractor/tumblr.py | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/youtube_dl/extractor/tumblr.py b/youtube_dl/extractor/tumblr.py
index cea117c79..584716986 100644
--- a/youtube_dl/extractor/tumblr.py
+++ b/youtube_dl/extractor/tumblr.py
@@ -67,6 +67,22 @@ class TumblrIE(InfoExtractor):
             'uploader_id': 'user32021558',
         },
         'add_ie': ['Vimeo'],
+    }, {
+        'url': 'http://sutiblr.tumblr.com/post/139638707273',
+        'md5': '2dd184b3669e049ba40563a7d423f95c',
+        'info_dict': {
+            'id': 'ir7qBEIKqvq',
+            'ext': 'mp4',
+            'title': 'Vine by sutiblr',
+            'alt_title': 'Vine by sutiblr',
+            'uploader': 'sutiblr',
+            'uploader_id': '1198993975374495744',
+            'upload_date': '20160220',
+            'like_count': int,
+            'comment_count': int,
+            'repost_count': int,
+        },
+        'add_ie': ['Vine'],
     }]
 
     def _real_extract(self, url):

From c4096e8aeaa373159e350a3674b0ce18b6c519e2 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 24 Mar 2016 16:29:33 +0800
Subject: [PATCH 4400/6219] [instagram] Extract embed videos (#8817)

---
 youtube_dl/extractor/instagram.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/youtube_dl/extractor/instagram.py b/youtube_dl/extractor/instagram.py
index ed3e07118..e8b27b379 100644
--- a/youtube_dl/extractor/instagram.py
+++ b/youtube_dl/extractor/instagram.py
@@ -4,6 +4,7 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
+    get_element_by_attribute,
     int_or_none,
     limit_length,
 )
@@ -38,6 +39,18 @@ class InstagramIE(InfoExtractor):
         'only_matching': True,
     }]
 
+    @staticmethod
+    def _extract_embed_url(webpage):
+        blockquote_el = get_element_by_attribute(
+            'class', 'instagram-media', webpage)
+        if blockquote_el is None:
+            return
+
+        mobj = re.search(
+            r'<a[^>]+href=([\'"])(?P<link>[^\'"]+)\1', blockquote_el)
+        if mobj:
+            return mobj.group('link')
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
 

From 87696e78d7203cf47bdd27773d06ba15af7c819d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 24 Mar 2016 16:30:01 +0800
Subject: [PATCH 4401/6219] [instagram] Unescape description (#8817)

---
 youtube_dl/extractor/instagram.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/instagram.py b/youtube_dl/extractor/instagram.py
index e8b27b379..4e62098b0 100644
--- a/youtube_dl/extractor/instagram.py
+++ b/youtube_dl/extractor/instagram.py
@@ -7,6 +7,7 @@ from ..utils import (
     get_element_by_attribute,
     int_or_none,
     limit_length,
+    lowercase_escape,
 )
 
 
@@ -59,6 +60,8 @@ class InstagramIE(InfoExtractor):
                                          webpage, 'uploader id', fatal=False)
         desc = self._search_regex(
             r'"caption":"(.+?)"', webpage, 'description', default=None)
+        if desc is not None:
+            desc = lowercase_escape(desc)
 
         return {
             'id': video_id,

From 5a51775a58b901c63bb0b59e44a113ad16ceb236 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 24 Mar 2016 16:32:27 +0800
Subject: [PATCH 4402/6219] [generic] Extract Instagram embeds (#8817)

---
 youtube_dl/extractor/generic.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 93e0563b5..12f2309fc 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -59,6 +59,7 @@ from .videomore import VideomoreIE
 from .googledrive import GoogleDriveIE
 from .jwplatform import JWPlatformIE
 from .digiteka import DigitekaIE
+from .instagram import InstagramIE
 
 
 class GenericIE(InfoExtractor):
@@ -1917,6 +1918,11 @@ class GenericIE(InfoExtractor):
             return self.url_result(
                 self._proto_relative_url(unescapeHTML(mobj.group(1))), 'Vine')
 
+        # Look for Instagram embeds
+        instagram_embed_url = InstagramIE._extract_embed_url(webpage)
+        if instagram_embed_url is not None:
+            return self.url_result(instagram_embed_url, InstagramIE.ie_key())
+
         def check_video(vurl):
             if YoutubeIE.suitable(vurl):
                 return True

From 32d88410eb2ef0ca25ab770af8f2ca2326c0aca7 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 24 Mar 2016 16:32:53 +0800
Subject: [PATCH 4403/6219] [tumblr] Add a test with Instagram embed

Closes #8817
---
 youtube_dl/extractor/tumblr.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/youtube_dl/extractor/tumblr.py b/youtube_dl/extractor/tumblr.py
index 584716986..e5bcf7798 100644
--- a/youtube_dl/extractor/tumblr.py
+++ b/youtube_dl/extractor/tumblr.py
@@ -83,6 +83,18 @@ class TumblrIE(InfoExtractor):
             'repost_count': int,
         },
         'add_ie': ['Vine'],
+    }, {
+        'url': 'http://vitasidorkina.tumblr.com/post/134652425014/joskriver-victoriassecret-invisibility-or',
+        'md5': '01c12ceb82cbf6b2fe0703aa56b3ad72',
+        'info_dict': {
+            'id': '-7LnUPGlSo',
+            'ext': 'mp4',
+            'title': 'Video by victoriassecret',
+            'description': 'Invisibility or flight…which superpower would YOU choose? #VSFashionShow #ThisOrThat',
+            'uploader_id': 'victoriassecret',
+            'thumbnail': 're:^https?://.*\.jpg'
+        },
+        'add_ie': ['Instagram'],
     }]
 
     def _real_extract(self, url):

From 622d19160bd81161e18c6ce868c359549a4d0413 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 24 Mar 2016 18:06:15 +0800
Subject: [PATCH 4404/6219] [utils] Clarify Python versions affected by buggy
 struct module

---
 youtube_dl/utils.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 03bb7782f..b6e1dc809 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1756,7 +1756,8 @@ def escape_url(url):
 try:
     struct.pack('!I', 0)
 except TypeError:
-    # In Python 2.6 (and some 2.7 versions), struct requires a bytes argument
+    # In Python 2.6 and 2.7.x < 2.7.7, struct requires a bytes argument
+    # See https://bugs.python.org/issue19099
     def struct_pack(spec, *args):
         if isinstance(spec, compat_str):
             spec = spec.encode('ascii')

From 5767b4eeae997a4ef75e348b46489cbb55126414 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 24 Mar 2016 22:23:31 +0600
Subject: [PATCH 4405/6219] [mtv] Fix description extraction (Closes #8962)

---
 youtube_dl/extractor/mtv.py | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index ed068365d..824bbcb4e 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -17,6 +17,7 @@ from ..utils import (
     unescapeHTML,
     url_basename,
     RegexNotFoundError,
+    xpath_text,
 )
 
 
@@ -130,11 +131,7 @@ class MTVServicesInfoExtractor(InfoExtractor):
             message += item.text
             raise ExtractorError(message, expected=True)
 
-        description_node = itemdoc.find('description')
-        if description_node is not None:
-            description = description_node.text.strip()
-        else:
-            description = None
+        description = xpath_text(itemdoc, 'description')
 
         title_el = None
         if title_el is None:

From 369e7e3ff02201210864b4e20af2893c40894ddf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 24 Mar 2016 22:54:26 +0600
Subject: [PATCH 4406/6219] [iprima] Fix extraction (Closes #8953)

---
 youtube_dl/extractor/iprima.py | 44 ++++++++++++++++++++++++++++++----
 1 file changed, 39 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/iprima.py b/youtube_dl/extractor/iprima.py
index 61a0de472..788bbe0d5 100644
--- a/youtube_dl/extractor/iprima.py
+++ b/youtube_dl/extractor/iprima.py
@@ -1,4 +1,4 @@
-# -*- coding: utf-8 -*-
+# coding: utf-8
 from __future__ import unicode_literals
 
 import re
@@ -6,6 +6,8 @@ import time
 
 from .common import InfoExtractor
 from ..utils import (
+    determine_ext,
+    js_to_json,
     sanitized_Request,
 )
 
@@ -30,8 +32,7 @@ class IPrimaIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id)
 
@@ -43,9 +44,42 @@ class IPrimaIE(InfoExtractor):
         req.add_header('Referer', url)
         playerpage = self._download_webpage(req, video_id, note='Downloading player')
 
-        m3u8_url = self._search_regex(r"'src': '([^']+\.m3u8)'", playerpage, 'm3u8 url')
+        formats = []
 
-        formats = self._extract_m3u8_formats(m3u8_url, video_id, ext='mp4')
+        def extract_formats(format_url, format_key=None, lang=None):
+            ext = determine_ext(format_url)
+            new_formats = []
+            if format_key == 'hls' or ext == 'm3u8':
+                new_formats = self._extract_m3u8_formats(
+                    format_url, video_id, 'mp4', entry_protocol='m3u8_native',
+                    m3u8_id='hls', fatal=False)
+            elif format_key == 'dash' or ext == 'mpd':
+                return
+                new_formats = self._extract_mpd_formats(
+                    format_url, video_id, mpd_id='dash', fatal=False)
+            if lang:
+                for f in new_formats:
+                    if not f.get('language'):
+                        f['language'] = lang
+            formats.extend(new_formats)
+
+        options = self._parse_json(
+            self._search_regex(
+                r'(?s)var\s+playerOptions\s*=\s*({.+?});',
+                playerpage, 'player options', default='{}'),
+            video_id, transform_source=js_to_json, fatal=False)
+        if options:
+            for key, tracks in options.get('tracks', {}).items():
+                if not isinstance(tracks, list):
+                    continue
+                for track in tracks:
+                    src = track.get('src')
+                    if src:
+                        extract_formats(src, key.lower(), track.get('lang'))
+
+        if not formats:
+            for _, src in re.findall(r'src["\']\s*:\s*(["\'])(.+?)\1', playerpage):
+                extract_formats(src)
 
         self._sort_formats(formats)
 

From 993271da0a70d6d5c194a10e48d43f3aa2abc956 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 24 Mar 2016 23:28:24 +0600
Subject: [PATCH 4407/6219] [nytimes] Tolerate missing metadata (Closes #8952)

---
 youtube_dl/extractor/nytimes.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/nytimes.py b/youtube_dl/extractor/nytimes.py
index 7f254b867..681683e86 100644
--- a/youtube_dl/extractor/nytimes.py
+++ b/youtube_dl/extractor/nytimes.py
@@ -18,8 +18,9 @@ class NYTimesBaseIE(InfoExtractor):
         description = video_data.get('summary')
         duration = float_or_none(video_data.get('duration'), 1000)
 
-        uploader = video_data['byline']
-        timestamp = parse_iso8601(video_data['publication_date'][:-8])
+        uploader = video_data.get('byline')
+        publication_date = video_data.get('publication_date')
+        timestamp = parse_iso8601(publication_date[:-8]) if publication_date else None
 
         def get_file_size(file_size):
             if isinstance(file_size, int):
@@ -37,7 +38,7 @@ class NYTimesBaseIE(InfoExtractor):
                 'width': int_or_none(video.get('width')),
                 'height': int_or_none(video.get('height')),
                 'filesize': get_file_size(video.get('fileSize')),
-            } for video in video_data['renditions']
+            } for video in video_data['renditions'] if video.get('url')
         ]
         self._sort_formats(formats)
 
@@ -46,7 +47,7 @@ class NYTimesBaseIE(InfoExtractor):
                 'url': 'http://www.nytimes.com/%s' % image['url'],
                 'width': int_or_none(image.get('width')),
                 'height': int_or_none(image.get('height')),
-            } for image in video_data['images']
+            } for image in video_data.get('images', []) if image.get('url')
         ]
 
         return {

From f07e276a04292c3fa87f703931bad9b716e7ccdf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 25 Mar 2016 01:18:14 +0600
Subject: [PATCH 4408/6219] [youtube:live] Add extractor (Closes #8959)

---
 youtube_dl/extractor/youtube.py | 48 ++++++++++++++++++++++++++++++++-
 1 file changed, 47 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 466f5da2e..96fa3b5aa 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1911,7 +1911,8 @@ class YoutubeChannelIE(YoutubePlaylistBaseInfoExtractor):
 
     @classmethod
     def suitable(cls, url):
-        return False if YoutubePlaylistsIE.suitable(url) else super(YoutubeChannelIE, cls).suitable(url)
+        return (False if YoutubePlaylistsIE.suitable(url) or YoutubeLiveIE.suitable(url)
+                else super(YoutubeChannelIE, cls).suitable(url))
 
     def _real_extract(self, url):
         channel_id = self._match_id(url)
@@ -1986,6 +1987,51 @@ class YoutubeUserIE(YoutubeChannelIE):
             return super(YoutubeUserIE, cls).suitable(url)
 
 
+class YoutubeLiveIE(YoutubeBaseInfoExtractor):
+    IE_DESC = 'YouTube.com live streams'
+    _VALID_URL = r'(?P<base_url>https?://(?:\w+\.)?youtube\.com/(?:user|channel)/(?P<id>[^/]+))/live'
+    IE_NAME = 'youtube:live'
+
+    _TESTS = [{
+        'url': 'http://www.youtube.com/user/TheYoungTurks/live',
+        'info_dict': {
+            'id': 'a48o2S1cPoo',
+            'ext': 'mp4',
+            'title': 'The Young Turks - Live Main Show',
+            'uploader': 'The Young Turks',
+            'uploader_id': 'TheYoungTurks',
+            'uploader_url': 're:https?://(?:www\.)?youtube\.com/user/TheYoungTurks',
+            'upload_date': '20150715',
+            'license': 'Standard YouTube License',
+            'description': 'md5:438179573adcdff3c97ebb1ee632b891',
+            'categories': ['News & Politics'],
+            'tags': ['Cenk Uygur (TV Program Creator)', 'The Young Turks (Award-Winning Work)', 'Talk Show (TV Genre)'],
+            'like_count': int,
+            'dislike_count': int,
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }, {
+        'url': 'http://www.youtube.com/channel/UC1yBKRuGpC1tSM73A0ZjYjQ/live',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        channel_id = mobj.group('id')
+        base_url = mobj.group('base_url')
+        webpage = self._download_webpage(url, channel_id, fatal=False)
+        if webpage:
+            page_type = self._og_search_property(
+                'type', webpage, 'page type', default=None)
+            video_id = self._html_search_meta(
+                'videoId', webpage, 'video id', default=None)
+            if page_type == 'video' and video_id and re.match(r'^[0-9A-Za-z_-]{11}$', video_id):
+                return self.url_result(video_id, YoutubeIE.ie_key())
+        return self.url_result(base_url)
+
+
 class YoutubePlaylistsIE(YoutubePlaylistsBaseInfoExtractor):
     IE_DESC = 'YouTube.com user/channel playlists'
     _VALID_URL = r'https?://(?:\w+\.)?youtube\.com/(?:user|channel)/(?P<id>[^/]+)/playlists'

From d041a736741e37e75e94cddf9c8258de8f08b501 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 25 Mar 2016 01:39:25 +0600
Subject: [PATCH 4409/6219] [extractor/__init__] Add youtube:live and sort
 youtube extractors alphabetically

---
 youtube_dl/extractor/__init__.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 8f7df4d12..d99873419 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -960,7 +960,9 @@ from .youtube import (
     YoutubeChannelIE,
     YoutubeFavouritesIE,
     YoutubeHistoryIE,
+    YoutubeLiveIE,
     YoutubePlaylistIE,
+    YoutubePlaylistsIE,
     YoutubeRecommendedIE,
     YoutubeSearchDateIE,
     YoutubeSearchIE,
@@ -970,7 +972,6 @@ from .youtube import (
     YoutubeTruncatedIDIE,
     YoutubeTruncatedURLIE,
     YoutubeUserIE,
-    YoutubePlaylistsIE,
     YoutubeWatchLaterIE,
 )
 from .zapiks import ZapiksIE

From ff9d5d093854a974afdd0191d331d817e7e4c2a7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 25 Mar 2016 02:26:46 +0600
Subject: [PATCH 4410/6219] [udemy] Improve course enrolling

---
 youtube_dl/extractor/udemy.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index 74cc36ece..a5634ece9 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -5,6 +5,7 @@ from ..compat import (
     compat_HTTPError,
     compat_urllib_parse,
     compat_urllib_request,
+    compat_urlparse,
 )
 from ..utils import (
     ExtractorError,
@@ -35,7 +36,7 @@ class UdemyIE(InfoExtractor):
         'skip': 'Requires udemy account credentials',
     }]
 
-    def _enroll_course(self, webpage, course_id):
+    def _enroll_course(self, base_url, webpage, course_id):
         checkout_url = unescapeHTML(self._search_regex(
             r'href=(["\'])(?P<url>https?://(?:www\.)?udemy\.com/payment/checkout/.+?)\1',
             webpage, 'checkout url', group='url', default=None))
@@ -45,9 +46,11 @@ class UdemyIE(InfoExtractor):
                 'Use this URL to confirm purchase: %s' % (course_id, checkout_url), expected=True)
 
         enroll_url = unescapeHTML(self._search_regex(
-            r'href=(["\'])(?P<url>https?://(?:www\.)?udemy\.com/course/subscribe/.+?)\1',
+            r'href=(["\'])(?P<url>(?:https?://(?:www\.)?udemy\.com)?/course/subscribe/.+?)\1',
             webpage, 'enroll url', group='url', default=None))
         if enroll_url:
+            if not enroll_url.startswith('http'):
+                enroll_url = compat_urlparse.urljoin(base_url, enroll_url)
             webpage = self._download_webpage(enroll_url, course_id, 'Enrolling in the course')
             if '>You have enrolled in' in webpage:
                 self.to_screen('%s: Successfully enrolled in the course' % course_id)
@@ -152,7 +155,7 @@ class UdemyIE(InfoExtractor):
         except ExtractorError as e:
             # Error could possibly mean we are not enrolled in the course
             if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
-                self._enroll_course(webpage, course_id)
+                self._enroll_course(url, webpage, course_id)
                 lecture = self._download_lecture(course_id, lecture_id)
             else:
                 raise
@@ -244,7 +247,7 @@ class UdemyCourseIE(UdemyIE):
         course_id = response['id']
         course_title = response.get('title')
 
-        self._enroll_course(webpage, course_id)
+        self._enroll_course(url, webpage, course_id)
 
         response = self._download_json(
             'https://www.udemy.com/api-1.1/courses/%s/curriculum' % course_id,

From f0e83681d97db52af9dc73d1c8e51d671503a222 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 25 Mar 2016 02:27:13 +0600
Subject: [PATCH 4411/6219] [udemy] Extract formats from outputs

---
 youtube_dl/extractor/udemy.py | 78 +++++++++++++++++++++--------------
 1 file changed, 48 insertions(+), 30 deletions(-)

diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index a5634ece9..2b886d6c8 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -180,39 +180,57 @@ class UdemyIE(InfoExtractor):
         video_id = asset['id']
         thumbnail = asset.get('thumbnailUrl') or asset.get('thumbnail_url')
         duration = float_or_none(asset.get('data', {}).get('duration'))
-        outputs = asset.get('data', {}).get('outputs', {})
 
         formats = []
-        for format_ in asset.get('download_urls', {}).get('Video', []):
-            video_url = format_.get('file')
-            if not video_url:
-                continue
-            format_id = format_.get('label')
-            f = {
-                'url': format_['file'],
-                'height': int_or_none(format_id),
+
+        def extract_output_format(src):
+            return {
+                'url': src['url'],
+                'format_id': '%sp' % (src.get('label') or format_id),
+                'width': int_or_none(src.get('width')),
+                'height': int_or_none(src.get('height')),
+                'vbr': int_or_none(src.get('video_bitrate_in_kbps')),
+                'vcodec': src.get('video_codec'),
+                'fps': int_or_none(src.get('frame_rate')),
+                'abr': int_or_none(src.get('audio_bitrate_in_kbps')),
+                'acodec': src.get('audio_codec'),
+                'asr': int_or_none(src.get('audio_sample_rate')),
+                'tbr': int_or_none(src.get('total_bitrate_in_kbps')),
+                'filesize': int_or_none(src.get('file_size_in_bytes')),
             }
-            if format_id:
-                # Some videos contain additional metadata (e.g.
-                # https://www.udemy.com/ios9-swift/learn/#/lecture/3383208)
-                output = outputs.get(format_id)
-                if isinstance(output, dict):
-                    f.update({
-                        'format_id': '%sp' % (output.get('label') or format_id),
-                        'width': int_or_none(output.get('width')),
-                        'height': int_or_none(output.get('height')),
-                        'vbr': int_or_none(output.get('video_bitrate_in_kbps')),
-                        'vcodec': output.get('video_codec'),
-                        'fps': int_or_none(output.get('frame_rate')),
-                        'abr': int_or_none(output.get('audio_bitrate_in_kbps')),
-                        'acodec': output.get('audio_codec'),
-                        'asr': int_or_none(output.get('audio_sample_rate')),
-                        'tbr': int_or_none(output.get('total_bitrate_in_kbps')),
-                        'filesize': int_or_none(output.get('file_size_in_bytes')),
-                    })
-                else:
-                    f['format_id'] = '%sp' % format_id
-            formats.append(f)
+
+        outputs = asset.get('data', {}).get('outputs')
+        if not isinstance(outputs, dict):
+            outputs = {}
+
+        for format_id, output in outputs.items():
+            if isinstance(output, dict) and output.get('url'):
+                formats.append(extract_output_format(output))
+
+        download_urls = asset.get('download_urls')
+        if isinstance(download_urls, dict):
+            video = download_urls.get('Video')
+            if isinstance(video, list):
+                for format_ in video:
+                    video_url = format_.get('file')
+                    if not video_url:
+                        continue
+                    format_id = format_.get('label')
+                    f = {
+                        'url': format_['file'],
+                        'height': int_or_none(format_id),
+                    }
+                    if format_id:
+                        # Some videos contain additional metadata (e.g.
+                        # https://www.udemy.com/ios9-swift/learn/#/lecture/3383208)
+                        output = outputs.get(format_id)
+                        if isinstance(output, dict):
+                            output_format = extract_output_format(output)
+                            output_format.update(f)
+                            f = output_format
+                        else:
+                            f['format_id'] = '%sp' % format_id
+                    formats.append(f)
 
         self._sort_formats(formats)
 

From 5eb7db4ee96c97d98a87d8d7df46e8c6a607b682 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 25 Mar 2016 02:28:39 +0600
Subject: [PATCH 4412/6219] [udemy] Add support for new URL schema

---
 youtube_dl/extractor/udemy.py | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index 2b886d6c8..a9046b865 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -18,7 +18,16 @@ from ..utils import (
 
 class UdemyIE(InfoExtractor):
     IE_NAME = 'udemy'
-    _VALID_URL = r'https?://www\.udemy\.com/(?:[^#]+#/lecture/|lecture/view/?\?lectureId=)(?P<id>\d+)'
+    _VALID_URL = r'''(?x)
+                    https?://
+                        www\.udemy\.com/
+                        (?:
+                            [^#]+\#/lecture/|
+                            lecture/view/?\?lectureId=|
+                            [^/]+/learn/v4/t/lecture/
+                        )
+                        (?P<id>\d+)
+                    '''
     _LOGIN_URL = 'https://www.udemy.com/join/login-popup/?displayType=ajax&showSkipButton=1'
     _ORIGIN_URL = 'https://www.udemy.com'
     _NETRC_MACHINE = 'udemy'
@@ -34,6 +43,10 @@ class UdemyIE(InfoExtractor):
             'duration': 579.29,
         },
         'skip': 'Requires udemy account credentials',
+    }, {
+        # new URL schema
+        'url': 'https://www.udemy.com/electric-bass-right-from-the-start/learn/v4/t/lecture/4580906',
+        'only_matching': True,
     }]
 
     def _enroll_course(self, base_url, webpage, course_id):

From e0317686666f9de4a6eca3fc26ede32e664f2bec Mon Sep 17 00:00:00 2001
From: Kagami Hiiragi <kagami@genshiken.org>
Date: Thu, 24 Mar 2016 13:55:02 +0300
Subject: [PATCH 4413/6219] [mnet] Add new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/mnet.py     | 76 ++++++++++++++++++++++++++++++++
 2 files changed, 77 insertions(+)
 create mode 100644 youtube_dl/extractor/mnet.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index d99873419..1e4b078a4 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -410,6 +410,7 @@ from .mit import TechTVMITIE, MITIE, OCWMITIE
 from .mitele import MiTeleIE
 from .mixcloud import MixcloudIE
 from .mlb import MLBIE
+from .mnet import MnetIE
 from .mpora import MporaIE
 from .moevideo import MoeVideoIE
 from .mofosex import MofosexIE
diff --git a/youtube_dl/extractor/mnet.py b/youtube_dl/extractor/mnet.py
new file mode 100644
index 000000000..8e83b1fc3
--- /dev/null
+++ b/youtube_dl/extractor/mnet.py
@@ -0,0 +1,76 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    parse_duration,
+    parse_iso8601,
+)
+
+
+class MnetIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?mnet\.(?:com|interest\.me)/tv/vod/(?:.*?\bclip_id=)?(?P<id>[0-9]+)'
+    _TESTS = [
+        {
+            'url': 'http://www.mnet.com/tv/vod/171008',
+            'md5': '6abd7a837fa9fe56d22709a60b19bffb',
+            'info_dict': {
+                'id': '171008',
+                'title': 'SS_이해인@히든박스',
+                'description': 'md5:b9efa592c3918b615ba69fe9f8a05c55',
+                'duration': 88,
+                'upload_date': '20151231',
+                'timestamp': 1451564040,
+                'age_limit': 0,
+                'thumbnails': 'mincount:5',
+                'ext': 'flv',
+            },
+        },
+        {
+            'url': 'http://mnet.interest.me/tv/vod/172790',
+            'only_matching': True,
+        },
+        {
+            'url': 'http://www.mnet.com/tv/vod/vod_view.asp?clip_id=172790&tabMenu=',
+            'only_matching': True,
+        },
+    ]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        info_url = 'http://content.api.mnet.com/player/vodConfig?id=%s' % video_id
+        info = self._download_json(info_url, video_id)
+        info = info['data']['info']
+
+        title = info['title']
+        rtmp_info_url = info['cdn'] + 'CLIP'
+        rtmp_info = self._download_json(rtmp_info_url, video_id)
+        file_url = rtmp_info['serverurl'] + rtmp_info['fileurl']
+        description = info.get('ment')
+        duration = parse_duration(info.get('time'))
+        timestamp = parse_iso8601(info.get('date'), delimiter=' ')
+        age_limit = info.get('adult')
+        if age_limit is not None:
+            age_limit = 0 if age_limit == 'N' else 18
+        thumbnails = [
+            {
+                'id': thumb_format,
+                'url': thumb['url'],
+                'width': int_or_none(thumb.get('width')),
+                'height': int_or_none(thumb.get('height')),
+            }
+            for (thumb_format, thumb) in info.get('cover', {}).items()
+        ]
+
+        return {
+            'id': video_id,
+            'title': title,
+            'url': file_url,
+            'description': description,
+            'duration': duration,
+            'timestamp': timestamp,
+            'age_limit': age_limit,
+            'thumbnails': thumbnails,
+            'ext': 'flv',
+        }

From 98e68806fb8cfe2a81ee8a6ac6705bb3d61ed2d1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 25 Mar 2016 03:26:29 +0600
Subject: [PATCH 4414/6219] [mnet] Improve (Closes #8958)

---
 youtube_dl/extractor/mnet.py | 85 +++++++++++++++++++-----------------
 1 file changed, 45 insertions(+), 40 deletions(-)

diff --git a/youtube_dl/extractor/mnet.py b/youtube_dl/extractor/mnet.py
index 8e83b1fc3..e3f42e7bd 100644
--- a/youtube_dl/extractor/mnet.py
+++ b/youtube_dl/extractor/mnet.py
@@ -11,66 +11,71 @@ from ..utils import (
 
 class MnetIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?mnet\.(?:com|interest\.me)/tv/vod/(?:.*?\bclip_id=)?(?P<id>[0-9]+)'
-    _TESTS = [
-        {
-            'url': 'http://www.mnet.com/tv/vod/171008',
-            'md5': '6abd7a837fa9fe56d22709a60b19bffb',
-            'info_dict': {
-                'id': '171008',
-                'title': 'SS_이해인@히든박스',
-                'description': 'md5:b9efa592c3918b615ba69fe9f8a05c55',
-                'duration': 88,
-                'upload_date': '20151231',
-                'timestamp': 1451564040,
-                'age_limit': 0,
-                'thumbnails': 'mincount:5',
-                'ext': 'flv',
-            },
+    _TESTS = [{
+        'url': 'http://www.mnet.com/tv/vod/171008',
+        'info_dict': {
+            'id': '171008',
+            'title': 'SS_이해인@히든박스',
+            'description': 'md5:b9efa592c3918b615ba69fe9f8a05c55',
+            'duration': 88,
+            'upload_date': '20151231',
+            'timestamp': 1451564040,
+            'age_limit': 0,
+            'thumbnails': 'mincount:5',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'ext': 'flv',
         },
-        {
-            'url': 'http://mnet.interest.me/tv/vod/172790',
-            'only_matching': True,
+        'params': {
+            # rtmp download
+            'skip_download': True,
         },
-        {
-            'url': 'http://www.mnet.com/tv/vod/vod_view.asp?clip_id=172790&tabMenu=',
-            'only_matching': True,
-        },
-    ]
+    }, {
+        'url': 'http://mnet.interest.me/tv/vod/172790',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.mnet.com/tv/vod/vod_view.asp?clip_id=172790&tabMenu=',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        info_url = 'http://content.api.mnet.com/player/vodConfig?id=%s' % video_id
-        info = self._download_json(info_url, video_id)
-        info = info['data']['info']
+
+        info = self._download_json(
+            'http://content.api.mnet.com/player/vodConfig?id=%s&ctype=CLIP' % video_id,
+            video_id, 'Downloading vod config JSON')['data']['info']
 
         title = info['title']
-        rtmp_info_url = info['cdn'] + 'CLIP'
-        rtmp_info = self._download_json(rtmp_info_url, video_id)
-        file_url = rtmp_info['serverurl'] + rtmp_info['fileurl']
+
+        rtmp_info = self._download_json(
+            info['cdn'], video_id, 'Downloading vod cdn JSON')
+
+        formats = [{
+            'url': rtmp_info['serverurl'] + rtmp_info['fileurl'],
+            'ext': 'flv',
+            'page_url': url,
+            'player_url': 'http://flvfile.mnet.com/service/player/201602/cjem_player_tv.swf?v=201602191318',
+        }]
+
         description = info.get('ment')
         duration = parse_duration(info.get('time'))
         timestamp = parse_iso8601(info.get('date'), delimiter=' ')
         age_limit = info.get('adult')
         if age_limit is not None:
             age_limit = 0 if age_limit == 'N' else 18
-        thumbnails = [
-            {
-                'id': thumb_format,
-                'url': thumb['url'],
-                'width': int_or_none(thumb.get('width')),
-                'height': int_or_none(thumb.get('height')),
-            }
-            for (thumb_format, thumb) in info.get('cover', {}).items()
-        ]
+        thumbnails = [{
+            'id': thumb_format,
+            'url': thumb['url'],
+            'width': int_or_none(thumb.get('width')),
+            'height': int_or_none(thumb.get('height')),
+        } for thumb_format, thumb in info.get('cover', {}).items() if thumb.get('url')]
 
         return {
             'id': video_id,
             'title': title,
-            'url': file_url,
             'description': description,
             'duration': duration,
             'timestamp': timestamp,
             'age_limit': age_limit,
             'thumbnails': thumbnails,
-            'ext': 'flv',
+            'formats': formats,
         }

From 3f15fec1d13cf4b18c093271bee7ad7586c97f9c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 25 Mar 2016 03:56:27 +0600
Subject: [PATCH 4415/6219] Credit @Kagami for mnet (#8958)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 51dfc8ddd..ea8d39978 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -166,3 +166,4 @@ Ben Congdon
 Kacper Michajłow
 José Joaquín Atria
 Viťas Strádal
+Kagami Hiiragi

From 3bb33568121126809e965dfacf542828d3606c10 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 25 Mar 2016 15:43:29 +0800
Subject: [PATCH 4416/6219] [douyutv] Extend _VALID_URL

---
 youtube_dl/extractor/douyutv.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/douyutv.py b/youtube_dl/extractor/douyutv.py
index bcb670945..3915cb182 100644
--- a/youtube_dl/extractor/douyutv.py
+++ b/youtube_dl/extractor/douyutv.py
@@ -10,7 +10,7 @@ from ..compat import (compat_str, compat_basestring)
 
 class DouyuTVIE(InfoExtractor):
     IE_DESC = '斗鱼'
-    _VALID_URL = r'https?://(?:www\.)?douyutv\.com/(?P<id>[A-Za-z0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?douyu(?:tv)?\.com/(?P<id>[A-Za-z0-9]+)'
     _TESTS = [{
         'url': 'http://www.douyutv.com/iseven',
         'info_dict': {
@@ -60,6 +60,9 @@ class DouyuTVIE(InfoExtractor):
         'params': {
             'skip_download': True,
         },
+    }, {
+        'url': 'http://www.douyu.com/xiaocang',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From d7f62b049a07d59265f679d13d736f5f8b096ce4 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 25 Mar 2016 15:45:40 +0800
Subject: [PATCH 4417/6219] [iqiyi] Update enc_key

---
 youtube_dl/extractor/iqiyi.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index 1a4c64713..ffcea30ad 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -501,7 +501,7 @@ class IqiyiIE(InfoExtractor):
     def get_enc_key(self, video_id):
         # TODO: automatic key extraction
         # last update at 2016-01-22 for Zombie::bite
-        enc_key = '8ed797d224d043e7ac23d95b70227d32'
+        enc_key = '4a1caba4b4465345366f28da7c117d20'
         return enc_key
 
     def _extract_playlist(self, webpage):

From 0be8314dc86a2630863ea89c94ee827a4b97b846 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 25 Mar 2016 09:27:18 +0100
Subject: [PATCH 4418/6219] release 2016.03.25

---
 CONTRIBUTING.md        | 2 +-
 README.md              | 6 ++++--
 docs/supportedsites.md | 8 ++++++++
 youtube_dl/version.py  | 2 +-
 4 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index c996f03ab..0df6193fb 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -85,7 +85,7 @@ To run the test, simply invoke your favorite test runner, or execute a test file
 If you want to create a build of youtube-dl yourself, you'll need
 
 * python
-* make
+* make (both GNU make and BSD make are supported)
 * pandoc
 * zip
 * nosetests
diff --git a/README.md b/README.md
index fcc12d2b3..52b2a85a3 100644
--- a/README.md
+++ b/README.md
@@ -164,6 +164,8 @@ which means you can modify it, redistribute it or use it however you like.
                                      (e.g. 50K or 4.2M)
     -R, --retries RETRIES            Number of retries (default is 10), or
                                      "infinite".
+    --fragment-retries RETRIES       Number of retries for a fragment (default
+                                     is 10), or "infinite" (DASH only)
     --buffer-size SIZE               Size of download buffer (e.g. 1024 or 16K)
                                      (default is 1024)
     --no-resize-buffer               Do not automatically adjust the buffer
@@ -376,8 +378,8 @@ which means you can modify it, redistribute it or use it however you like.
     --no-post-overwrites             Do not overwrite post-processed files; the
                                      post-processed files are overwritten by
                                      default
-    --embed-subs                     Embed subtitles in the video (only for mkv
-                                     and mp4 videos)
+    --embed-subs                     Embed subtitles in the video (only for mp4,
+                                     webm and mkv videos)
     --embed-thumbnail                Embed thumbnail in the audio as cover art
     --add-metadata                   Write metadata to the video file
     --metadata-from-title FORMAT     Parse additional metadata like song title /
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 3415efc45..00b8c247c 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -74,6 +74,7 @@
  - **Bigflix**
  - **Bild**: Bild.de
  - **BiliBili**
+ - **BioBioChileTV**
  - **BleacherReport**
  - **BleacherReportCMS**
  - **blinkx**
@@ -100,6 +101,7 @@
  - **CBSNews**: CBS News
  - **CBSNewsLiveVideo**: CBS News Live Videos
  - **CBSSports**
+ - **CDA**
  - **CeskaTelevize**
  - **channel9**: Channel 9
  - **Chaturbate**
@@ -244,6 +246,7 @@
  - **GPUTechConf**
  - **Groupon**
  - **Hark**
+ - **HBO**
  - **HearThisAt**
  - **Heise**
  - **HellPorno**
@@ -344,6 +347,7 @@
  - **MiTele**: mitele.es
  - **mixcloud**
  - **MLB**
+ - **Mnet**
  - **MoeVideo**: LetitBit video services: moevideo.net, playreplay.net and videochart.net
  - **Mofosex**
  - **Mojvideo**
@@ -440,6 +444,7 @@
  - **OnionStudios**
  - **Ooyala**
  - **OoyalaExternal**
+ - **Openload**
  - **OraTV**
  - **orf:fm4**: radio FM4
  - **orf:iptv**: iptv.ORF.at
@@ -525,6 +530,7 @@
  - **RUTV**: RUTV.RU
  - **Ruutu**
  - **safari**: safaribooksonline.com online video
+ - **safari:api**
  - **safari:course**: safaribooksonline.com online courses
  - **Sandia**: Sandia National Laboratories
  - **Sapo**: SAPO Vídeos
@@ -618,6 +624,7 @@
  - **TheOnion**
  - **ThePlatform**
  - **ThePlatformFeed**
+ - **TheScene**
  - **TheSixtyOne**
  - **TheStar**
  - **ThisAmericanLife**
@@ -786,6 +793,7 @@
  - **youtube:channel**: YouTube.com channels
  - **youtube:favorites**: YouTube.com favourite videos, ":ytfav" for short (requires authentication)
  - **youtube:history**: Youtube watch history, ":ythistory" for short (requires authentication)
+ - **youtube:live**: YouTube.com live streams
  - **youtube:playlist**: YouTube.com playlists
  - **youtube:playlists**: YouTube.com user/channel playlists
  - **youtube:recommended**: YouTube.com recommended videos, ":ytrec" for short (requires authentication)
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 6b2c5fac9..2291ed783 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.03.18'
+__version__ = '2016.03.25'

From 4db441de72415f8262279e48d0b15ebc9e1da369 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 25 Mar 2016 19:51:28 +0600
Subject: [PATCH 4419/6219] [once] Relax _VALID_URL (Closes #8976)

---
 youtube_dl/extractor/once.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/once.py b/youtube_dl/extractor/once.py
index 5db949b17..1bf96ea56 100644
--- a/youtube_dl/extractor/once.py
+++ b/youtube_dl/extractor/once.py
@@ -7,7 +7,7 @@ from .common import InfoExtractor
 
 
 class OnceIE(InfoExtractor):
-    _VALID_URL = r'https?://once\.unicornmedia\.com/now/[^/]+/[^/]+/(?P<domain_id>[^/]+)/(?P<application_id>[^/]+)/(?:[^/]+/)?(?P<media_item_id>[^/]+)/content\.(?:once|m3u8|mp4)'
+    _VALID_URL = r'https?://.+?\.unicornmedia\.com/now/[^/]+/[^/]+/(?P<domain_id>[^/]+)/(?P<application_id>[^/]+)/(?:[^/]+/)?(?P<media_item_id>[^/]+)/content\.(?:once|m3u8|mp4)'
     ADAPTIVE_URL_TEMPLATE = 'http://once.unicornmedia.com/now/master/playlist/%s/%s/%s/content.m3u8'
     PROGRESSIVE_URL_TEMPLATE = 'http://once.unicornmedia.com/now/media/progressive/%s/%s/%s/%s/content.mp4'
 

From 2156f16ca7babde4c5fa813dbe4e7ac1a2f758d1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 25 Mar 2016 20:14:34 +0600
Subject: [PATCH 4420/6219] [thescene] Fix extraction and improve style (Closes
 #8978)

---
 youtube_dl/extractor/thescene.py | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/thescene.py b/youtube_dl/extractor/thescene.py
index 08d666eaf..3e4e14031 100644
--- a/youtube_dl/extractor/thescene.py
+++ b/youtube_dl/extractor/thescene.py
@@ -2,7 +2,7 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 
-from ..compat import compat_urllib_parse
+from ..compat import compat_urlparse
 from ..utils import qualities
 
 
@@ -21,17 +21,21 @@ class TheSceneIE(InfoExtractor):
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
+
         webpage = self._download_webpage(url, display_id)
-        player_url = compat_urllib_parse.urljoin(
+
+        player_url = compat_urlparse.urljoin(
             url,
             self._html_search_regex(
                 r'id=\'js-player-script\'[^>]+src=\'(.+?)\'', webpage, 'player url'))
 
-        self.to_screen(player_url)
-        player = self._download_webpage(player_url, player_url)
-        info = self._parse_json(self._search_regex(r'(?m)var\s+video\s+=\s+({.+?});$', player, 'info json'), display_id)
+        player = self._download_webpage(player_url, display_id)
+        info = self._parse_json(
+            self._search_regex(
+                r'(?m)var\s+video\s+=\s+({.+?});$', player, 'info json'),
+            display_id)
 
-        qualities_order = qualities(['low', 'high'])
+        qualities_order = qualities(('low', 'high'))
         formats = [{
             'format_id': '{0}-{1}'.format(f['type'].split('/')[0], f['quality']),
             'url': f['src'],
@@ -41,8 +45,8 @@ class TheSceneIE(InfoExtractor):
 
         return {
             'id': info['id'],
+            'display_id': display_id,
             'title': info['title'],
             'formats': formats,
             'thumbnail': info.get('poster_frame'),
-            'display_id': display_id,
         }

From 15707c7e024f1f29e7abd8ddaa362196ef2d4af6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 26 Mar 2016 01:46:57 +0600
Subject: [PATCH 4421/6219] [compat] Add compat_urllib_parse_urlencode and
 eliminate encode_dict

encode_dict functionality has been improved and moved directly into compat_urllib_parse_urlencode
All occurrences of compat_urllib_parse.urlencode throughout the codebase have been replaced by compat_urllib_parse_urlencode

Closes #8974
---
 youtube_dl/compat.py                         | 26 ++++++++++++++++++++
 youtube_dl/extractor/addanime.py             |  4 +--
 youtube_dl/extractor/animeondemand.py        |  3 +--
 youtube_dl/extractor/atresplayer.py          |  4 +--
 youtube_dl/extractor/bambuser.py             |  4 +--
 youtube_dl/extractor/camdemy.py              |  4 +--
 youtube_dl/extractor/ceskatelevize.py        |  4 +--
 youtube_dl/extractor/cloudy.py               |  4 +--
 youtube_dl/extractor/comedycentral.py        |  4 +--
 youtube_dl/extractor/common.py               |  4 +--
 youtube_dl/extractor/condenast.py            |  4 +--
 youtube_dl/extractor/crunchyroll.py          |  8 +++---
 youtube_dl/extractor/daum.py                 |  6 ++---
 youtube_dl/extractor/dcn.py                  |  8 +++---
 youtube_dl/extractor/dramafever.py           |  4 +--
 youtube_dl/extractor/eroprofile.py           |  4 +--
 youtube_dl/extractor/fc2.py                  |  5 ++--
 youtube_dl/extractor/fivemin.py              |  4 +--
 youtube_dl/extractor/flickr.py               |  4 +--
 youtube_dl/extractor/funimation.py           |  5 ++--
 youtube_dl/extractor/gdcvault.py             |  4 +--
 youtube_dl/extractor/hotnewhiphop.py         |  4 +--
 youtube_dl/extractor/hypem.py                |  4 +--
 youtube_dl/extractor/internetvideoarchive.py |  4 +--
 youtube_dl/extractor/iqiyi.py                |  8 +++---
 youtube_dl/extractor/ivideon.py              |  4 +--
 youtube_dl/extractor/kaltura.py              |  4 +--
 youtube_dl/extractor/laola1tv.py             |  6 ++---
 youtube_dl/extractor/leeco.py                |  8 +++---
 youtube_dl/extractor/lynda.py                |  6 ++---
 youtube_dl/extractor/matchtv.py              |  4 +--
 youtube_dl/extractor/metacafe.py             |  4 +--
 youtube_dl/extractor/minhateca.py            |  4 +--
 youtube_dl/extractor/mitele.py               |  5 ++--
 youtube_dl/extractor/moevideo.py             |  4 +--
 youtube_dl/extractor/moniker.py              |  4 +--
 youtube_dl/extractor/mooshare.py             |  4 +--
 youtube_dl/extractor/mtv.py                  |  4 +--
 youtube_dl/extractor/muzu.py                 |  8 +++---
 youtube_dl/extractor/myvideo.py              |  4 +--
 youtube_dl/extractor/naver.py                |  6 ++---
 youtube_dl/extractor/nba.py                  |  4 +--
 youtube_dl/extractor/neteasemusic.py         |  4 +--
 youtube_dl/extractor/nextmovie.py            |  4 +--
 youtube_dl/extractor/nfb.py                  |  4 +--
 youtube_dl/extractor/nhl.py                  |  6 ++---
 youtube_dl/extractor/nick.py                 |  4 +--
 youtube_dl/extractor/niconico.py             |  7 +++---
 youtube_dl/extractor/noco.py                 |  4 +--
 youtube_dl/extractor/novamov.py              |  3 +--
 youtube_dl/extractor/npr.py                  |  4 +--
 youtube_dl/extractor/ooyala.py               |  4 +--
 youtube_dl/extractor/patreon.py              |  2 +-
 youtube_dl/extractor/played.py               |  4 +--
 youtube_dl/extractor/playtvak.py             |  4 +--
 youtube_dl/extractor/pluralsight.py          |  4 +--
 youtube_dl/extractor/porn91.py               |  4 +--
 youtube_dl/extractor/primesharetv.py         |  4 +--
 youtube_dl/extractor/promptfile.py           |  4 +--
 youtube_dl/extractor/prosiebensat1.py        | 10 +++-----
 youtube_dl/extractor/shahid.py               |  4 +--
 youtube_dl/extractor/shared.py               |  4 +--
 youtube_dl/extractor/sharesix.py             |  4 +--
 youtube_dl/extractor/sina.py                 |  4 +--
 youtube_dl/extractor/smotri.py               |  6 ++---
 youtube_dl/extractor/sohu.py                 |  4 +--
 youtube_dl/extractor/soundcloud.py           | 12 ++++-----
 youtube_dl/extractor/streamcloud.py          |  4 +--
 youtube_dl/extractor/telecinco.py            |  4 +--
 youtube_dl/extractor/tubitv.py               |  4 +--
 youtube_dl/extractor/twitch.py               |  9 +++----
 youtube_dl/extractor/udemy.py                |  6 ++---
 youtube_dl/extractor/vbox7.py                |  4 +--
 youtube_dl/extractor/viddler.py              |  4 +--
 youtube_dl/extractor/vimeo.py                | 13 +++++-----
 youtube_dl/extractor/vk.py                   |  4 +--
 youtube_dl/extractor/vlive.py                |  4 +--
 youtube_dl/extractor/vodlocker.py            |  4 +--
 youtube_dl/extractor/xfileshare.py           |  5 ++--
 youtube_dl/extractor/yahoo.py                |  3 ++-
 youtube_dl/extractor/yandexmusic.py          |  4 +--
 youtube_dl/extractor/youku.py                |  4 +--
 youtube_dl/extractor/youtube.py              | 23 +++++++++--------
 youtube_dl/utils.py                          | 14 +++--------
 84 files changed, 229 insertions(+), 222 deletions(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index dbb91a6ef..76b6b0e38 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -169,6 +169,31 @@ except ImportError:  # Python 2
         string = string.replace('+', ' ')
         return compat_urllib_parse_unquote(string, encoding, errors)
 
+try:
+    from urllib.parse import urlencode as compat_urllib_parse_urlencode
+except ImportError:  # Python 2
+    # Python 2 will choke in urlencode on mixture of byte and unicode strings.
+    # Possible solutions are to either port it from python 3 with all
+    # the friends or manually ensure input query contains only byte strings.
+    # We will stick with latter thus recursively encoding the whole query.
+    def compat_urllib_parse_urlencode(query, doseq=0, encoding='utf-8'):
+        def encode_elem(e):
+            if isinstance(e, dict):
+                e = encode_dict(e)
+            elif isinstance(e, (list, tuple,)):
+                e = encode_list(e)
+            elif isinstance(e, compat_str):
+                e = e.encode(encoding)
+            return e
+
+        def encode_dict(d):
+            return dict((encode_elem(k), encode_elem(v)) for k, v in d.items())
+
+        def encode_list(l):
+            return [encode_elem(e) for e in l]
+
+        return compat_urllib_parse.urlencode(encode_elem(query), doseq=doseq)
+
 try:
     from urllib.request import DataHandler as compat_urllib_request_DataHandler
 except ImportError:  # Python < 3.4
@@ -588,6 +613,7 @@ __all__ = [
     'compat_urllib_parse_unquote',
     'compat_urllib_parse_unquote_plus',
     'compat_urllib_parse_unquote_to_bytes',
+    'compat_urllib_parse_urlencode',
     'compat_urllib_parse_urlparse',
     'compat_urllib_request',
     'compat_urllib_request_DataHandler',
diff --git a/youtube_dl/extractor/addanime.py b/youtube_dl/extractor/addanime.py
index fb1cc02e1..55a9322a7 100644
--- a/youtube_dl/extractor/addanime.py
+++ b/youtube_dl/extractor/addanime.py
@@ -6,7 +6,7 @@ from .common import InfoExtractor
 from ..compat import (
     compat_HTTPError,
     compat_str,
-    compat_urllib_parse,
+    compat_urllib_parse_urlencode,
     compat_urllib_parse_urlparse,
 )
 from ..utils import (
@@ -60,7 +60,7 @@ class AddAnimeIE(InfoExtractor):
             confirm_url = (
                 parsed_url.scheme + '://' + parsed_url.netloc +
                 action + '?' +
-                compat_urllib_parse.urlencode({
+                compat_urllib_parse_urlencode({
                     'jschl_vc': vc, 'jschl_answer': compat_str(av_val)}))
             self._download_webpage(
                 confirm_url, video_id,
diff --git a/youtube_dl/extractor/animeondemand.py b/youtube_dl/extractor/animeondemand.py
index 2cede55a7..9b01e38f5 100644
--- a/youtube_dl/extractor/animeondemand.py
+++ b/youtube_dl/extractor/animeondemand.py
@@ -9,7 +9,6 @@ from ..compat import (
 )
 from ..utils import (
     determine_ext,
-    encode_dict,
     extract_attributes,
     ExtractorError,
     sanitized_Request,
@@ -71,7 +70,7 @@ class AnimeOnDemandIE(InfoExtractor):
             post_url = compat_urlparse.urljoin(self._LOGIN_URL, post_url)
 
         request = sanitized_Request(
-            post_url, urlencode_postdata(encode_dict(login_form)))
+            post_url, urlencode_postdata(login_form))
         request.add_header('Referer', self._LOGIN_URL)
 
         response = self._download_webpage(
diff --git a/youtube_dl/extractor/atresplayer.py b/youtube_dl/extractor/atresplayer.py
index b8f9ae005..f9568cb5b 100644
--- a/youtube_dl/extractor/atresplayer.py
+++ b/youtube_dl/extractor/atresplayer.py
@@ -8,7 +8,7 @@ import re
 from .common import InfoExtractor
 from ..compat import (
     compat_str,
-    compat_urllib_parse,
+    compat_urllib_parse_urlencode,
 )
 from ..utils import (
     int_or_none,
@@ -86,7 +86,7 @@ class AtresPlayerIE(InfoExtractor):
         }
 
         request = sanitized_Request(
-            self._LOGIN_URL, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
+            self._LOGIN_URL, compat_urllib_parse_urlencode(login_form).encode('utf-8'))
         request.add_header('Content-Type', 'application/x-www-form-urlencoded')
         response = self._download_webpage(
             request, None, 'Logging in as %s' % username)
diff --git a/youtube_dl/extractor/bambuser.py b/youtube_dl/extractor/bambuser.py
index da986e063..1a2eef48d 100644
--- a/youtube_dl/extractor/bambuser.py
+++ b/youtube_dl/extractor/bambuser.py
@@ -5,7 +5,7 @@ import itertools
 
 from .common import InfoExtractor
 from ..compat import (
-    compat_urllib_parse,
+    compat_urllib_parse_urlencode,
     compat_str,
 )
 from ..utils import (
@@ -58,7 +58,7 @@ class BambuserIE(InfoExtractor):
         }
 
         request = sanitized_Request(
-            self._LOGIN_URL, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
+            self._LOGIN_URL, compat_urllib_parse_urlencode(login_form).encode('utf-8'))
         request.add_header('Referer', self._LOGIN_URL)
         response = self._download_webpage(
             request, None, 'Logging in as %s' % username)
diff --git a/youtube_dl/extractor/camdemy.py b/youtube_dl/extractor/camdemy.py
index dd4d96cec..6ffbeabd3 100644
--- a/youtube_dl/extractor/camdemy.py
+++ b/youtube_dl/extractor/camdemy.py
@@ -6,7 +6,7 @@ import re
 
 from .common import InfoExtractor
 from ..compat import (
-    compat_urllib_parse,
+    compat_urllib_parse_urlencode,
     compat_urlparse,
 )
 from ..utils import (
@@ -139,7 +139,7 @@ class CamdemyFolderIE(InfoExtractor):
         parsed_url = list(compat_urlparse.urlparse(url))
         query = dict(compat_urlparse.parse_qsl(parsed_url[4]))
         query.update({'displayMode': 'list'})
-        parsed_url[4] = compat_urllib_parse.urlencode(query)
+        parsed_url[4] = compat_urllib_parse_urlencode(query)
         final_url = compat_urlparse.urlunparse(parsed_url)
 
         page = self._download_webpage(final_url, folder_id)
diff --git a/youtube_dl/extractor/ceskatelevize.py b/youtube_dl/extractor/ceskatelevize.py
index b355111cb..d93108df5 100644
--- a/youtube_dl/extractor/ceskatelevize.py
+++ b/youtube_dl/extractor/ceskatelevize.py
@@ -5,8 +5,8 @@ import re
 
 from .common import InfoExtractor
 from ..compat import (
-    compat_urllib_parse,
     compat_urllib_parse_unquote,
+    compat_urllib_parse_urlencode,
     compat_urllib_parse_urlparse,
 )
 from ..utils import (
@@ -102,7 +102,7 @@ class CeskaTelevizeIE(InfoExtractor):
 
         req = sanitized_Request(
             'http://www.ceskatelevize.cz/ivysilani/ajax/get-client-playlist',
-            data=compat_urllib_parse.urlencode(data))
+            data=compat_urllib_parse_urlencode(data))
 
         req.add_header('Content-type', 'application/x-www-form-urlencoded')
         req.add_header('x-addr', '127.0.0.1')
diff --git a/youtube_dl/extractor/cloudy.py b/youtube_dl/extractor/cloudy.py
index 0fa720ee8..9e267e6c0 100644
--- a/youtube_dl/extractor/cloudy.py
+++ b/youtube_dl/extractor/cloudy.py
@@ -6,7 +6,7 @@ import re
 from .common import InfoExtractor
 from ..compat import (
     compat_parse_qs,
-    compat_urllib_parse,
+    compat_urllib_parse_urlencode,
     compat_HTTPError,
 )
 from ..utils import (
@@ -64,7 +64,7 @@ class CloudyIE(InfoExtractor):
                 'errorUrl': error_url,
             })
 
-        data_url = self._API_URL % (video_host, compat_urllib_parse.urlencode(form))
+        data_url = self._API_URL % (video_host, compat_urllib_parse_urlencode(form))
         player_data = self._download_webpage(
             data_url, video_id, 'Downloading player data')
         data = compat_parse_qs(player_data)
diff --git a/youtube_dl/extractor/comedycentral.py b/youtube_dl/extractor/comedycentral.py
index 5b1b99675..0c59102e0 100644
--- a/youtube_dl/extractor/comedycentral.py
+++ b/youtube_dl/extractor/comedycentral.py
@@ -5,7 +5,7 @@ import re
 from .mtv import MTVServicesInfoExtractor
 from ..compat import (
     compat_str,
-    compat_urllib_parse,
+    compat_urllib_parse_urlencode,
 )
 from ..utils import (
     ExtractorError,
@@ -201,7 +201,7 @@ class ComedyCentralShowsIE(MTVServicesInfoExtractor):
         # Correct cc.com in uri
         uri = re.sub(r'(episode:[^.]+)(\.cc)?\.com', r'\1.com', uri)
 
-        index_url = 'http://%s.cc.com/feeds/mrss?%s' % (show_name, compat_urllib_parse.urlencode({'uri': uri}))
+        index_url = 'http://%s.cc.com/feeds/mrss?%s' % (show_name, compat_urllib_parse_urlencode({'uri': uri}))
         idoc = self._download_xml(
             index_url, epTitle,
             'Downloading show index', 'Unable to download episode index')
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 770105a5b..b412fd030 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -21,7 +21,7 @@ from ..compat import (
     compat_os_name,
     compat_str,
     compat_urllib_error,
-    compat_urllib_parse,
+    compat_urllib_parse_urlencode,
     compat_urlparse,
 )
 from ..utils import (
@@ -1300,7 +1300,7 @@ class InfoExtractor(object):
                         'plugin': 'flowplayer-3.2.0.1',
                     }
                 f4m_url += '&' if '?' in f4m_url else '?'
-                f4m_url += compat_urllib_parse.urlencode(f4m_params)
+                f4m_url += compat_urllib_parse_urlencode(f4m_params)
                 formats.extend(self._extract_f4m_formats(f4m_url, video_id, f4m_id='hds', fatal=False))
                 continue
 
diff --git a/youtube_dl/extractor/condenast.py b/youtube_dl/extractor/condenast.py
index 054978ff2..e8f2b5a07 100644
--- a/youtube_dl/extractor/condenast.py
+++ b/youtube_dl/extractor/condenast.py
@@ -5,7 +5,7 @@ import re
 
 from .common import InfoExtractor
 from ..compat import (
-    compat_urllib_parse,
+    compat_urllib_parse_urlencode,
     compat_urllib_parse_urlparse,
     compat_urlparse,
 )
@@ -97,7 +97,7 @@ class CondeNastIE(InfoExtractor):
         video_id = self._search_regex(r'videoId: [\'"](.+?)[\'"]', params, 'video id')
         player_id = self._search_regex(r'playerId: [\'"](.+?)[\'"]', params, 'player id')
         target = self._search_regex(r'target: [\'"](.+?)[\'"]', params, 'target')
-        data = compat_urllib_parse.urlencode({'videoId': video_id,
+        data = compat_urllib_parse_urlencode({'videoId': video_id,
                                               'playerId': player_id,
                                               'target': target,
                                               })
diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 85fa7a725..7746f1be3 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -11,8 +11,8 @@ from math import pow, sqrt, floor
 from .common import InfoExtractor
 from ..compat import (
     compat_etree_fromstring,
-    compat_urllib_parse,
     compat_urllib_parse_unquote,
+    compat_urllib_parse_urlencode,
     compat_urllib_request,
     compat_urlparse,
 )
@@ -78,7 +78,7 @@ class CrunchyrollBaseIE(InfoExtractor):
         # See https://github.com/rg3/youtube-dl/issues/7202.
         qs['skip_wall'] = ['1']
         return compat_urlparse.urlunparse(
-            parsed_url._replace(query=compat_urllib_parse.urlencode(qs, True)))
+            parsed_url._replace(query=compat_urllib_parse_urlencode(qs, True)))
 
 
 class CrunchyrollIE(CrunchyrollBaseIE):
@@ -308,7 +308,7 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
 
         playerdata_url = compat_urllib_parse_unquote(self._html_search_regex(r'"config_url":"([^"]+)', webpage, 'playerdata_url'))
         playerdata_req = sanitized_Request(playerdata_url)
-        playerdata_req.data = compat_urllib_parse.urlencode({'current_page': webpage_url})
+        playerdata_req.data = compat_urllib_parse_urlencode({'current_page': webpage_url})
         playerdata_req.add_header('Content-Type', 'application/x-www-form-urlencoded')
         playerdata = self._download_webpage(playerdata_req, video_id, note='Downloading media info')
 
@@ -322,7 +322,7 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
             streamdata_req = sanitized_Request(
                 'http://www.crunchyroll.com/xml/?req=RpcApiVideoPlayer_GetStandardConfig&media_id=%s&video_format=%s&video_quality=%s'
                 % (stream_id, stream_format, stream_quality),
-                compat_urllib_parse.urlencode({'current_page': url}).encode('utf-8'))
+                compat_urllib_parse_urlencode({'current_page': url}).encode('utf-8'))
             streamdata_req.add_header('Content-Type', 'application/x-www-form-urlencoded')
             streamdata = self._download_xml(
                 streamdata_req, video_id,
diff --git a/youtube_dl/extractor/daum.py b/youtube_dl/extractor/daum.py
index c84c51058..86024a745 100644
--- a/youtube_dl/extractor/daum.py
+++ b/youtube_dl/extractor/daum.py
@@ -8,8 +8,8 @@ import itertools
 from .common import InfoExtractor
 from ..compat import (
     compat_parse_qs,
-    compat_urllib_parse,
     compat_urllib_parse_unquote,
+    compat_urllib_parse_urlencode,
     compat_urlparse,
 )
 from ..utils import (
@@ -70,7 +70,7 @@ class DaumIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = compat_urllib_parse_unquote(self._match_id(url))
-        query = compat_urllib_parse.urlencode({'vid': video_id})
+        query = compat_urllib_parse_urlencode({'vid': video_id})
         movie_data = self._download_json(
             'http://videofarm.daum.net/controller/api/closed/v1_2/IntegratedMovieData.json?' + query,
             video_id, 'Downloading video formats info')
@@ -86,7 +86,7 @@ class DaumIE(InfoExtractor):
         formats = []
         for format_el in movie_data['output_list']['output_list']:
             profile = format_el['profile']
-            format_query = compat_urllib_parse.urlencode({
+            format_query = compat_urllib_parse_urlencode({
                 'vid': video_id,
                 'profile': profile,
             })
diff --git a/youtube_dl/extractor/dcn.py b/youtube_dl/extractor/dcn.py
index 15a1c40f7..982ed94ea 100644
--- a/youtube_dl/extractor/dcn.py
+++ b/youtube_dl/extractor/dcn.py
@@ -6,7 +6,7 @@ import base64
 
 from .common import InfoExtractor
 from ..compat import (
-    compat_urllib_parse,
+    compat_urllib_parse_urlencode,
     compat_str,
 )
 from ..utils import (
@@ -106,7 +106,7 @@ class DCNVideoIE(DCNBaseIE):
 
         webpage = self._download_webpage(
             'http://admin.mangomolo.com/analytics/index.php/customers/embed/video?' +
-            compat_urllib_parse.urlencode({
+            compat_urllib_parse_urlencode({
                 'id': video_data['id'],
                 'user_id': video_data['user_id'],
                 'signature': video_data['signature'],
@@ -133,7 +133,7 @@ class DCNLiveIE(DCNBaseIE):
 
         webpage = self._download_webpage(
             'http://admin.mangomolo.com/analytics/index.php/customers/embed/index?' +
-            compat_urllib_parse.urlencode({
+            compat_urllib_parse_urlencode({
                 'id': base64.b64encode(channel_data['user_id'].encode()).decode(),
                 'channelid': base64.b64encode(channel_data['id'].encode()).decode(),
                 'signature': channel_data['signature'],
@@ -174,7 +174,7 @@ class DCNSeasonIE(InfoExtractor):
         data['show_id'] = show_id
         request = sanitized_Request(
             'http://admin.mangomolo.com/analytics/index.php/plus/show',
-            compat_urllib_parse.urlencode(data),
+            compat_urllib_parse_urlencode(data),
             {
                 'Origin': 'http://www.dcndigital.ae',
                 'Content-Type': 'application/x-www-form-urlencoded'
diff --git a/youtube_dl/extractor/dramafever.py b/youtube_dl/extractor/dramafever.py
index d35e88881..2101acaaf 100644
--- a/youtube_dl/extractor/dramafever.py
+++ b/youtube_dl/extractor/dramafever.py
@@ -6,7 +6,7 @@ import itertools
 from .amp import AMPIE
 from ..compat import (
     compat_HTTPError,
-    compat_urllib_parse,
+    compat_urllib_parse_urlencode,
     compat_urlparse,
 )
 from ..utils import (
@@ -50,7 +50,7 @@ class DramaFeverBaseIE(AMPIE):
         }
 
         request = sanitized_Request(
-            self._LOGIN_URL, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
+            self._LOGIN_URL, compat_urllib_parse_urlencode(login_form).encode('utf-8'))
         response = self._download_webpage(
             request, None, 'Logging in as %s' % username)
 
diff --git a/youtube_dl/extractor/eroprofile.py b/youtube_dl/extractor/eroprofile.py
index 7fcd0151d..297f8a6f5 100644
--- a/youtube_dl/extractor/eroprofile.py
+++ b/youtube_dl/extractor/eroprofile.py
@@ -3,7 +3,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse
+from ..compat import compat_urllib_parse_urlencode
 from ..utils import (
     ExtractorError,
     unescapeHTML
@@ -43,7 +43,7 @@ class EroProfileIE(InfoExtractor):
         if username is None:
             return
 
-        query = compat_urllib_parse.urlencode({
+        query = compat_urllib_parse_urlencode({
             'username': username,
             'password': password,
             'url': 'http://www.eroprofile.com/',
diff --git a/youtube_dl/extractor/fc2.py b/youtube_dl/extractor/fc2.py
index 508684d2e..cacf61973 100644
--- a/youtube_dl/extractor/fc2.py
+++ b/youtube_dl/extractor/fc2.py
@@ -5,12 +5,11 @@ import hashlib
 
 from .common import InfoExtractor
 from ..compat import (
-    compat_urllib_parse,
+    compat_urllib_parse_urlencode,
     compat_urllib_request,
     compat_urlparse,
 )
 from ..utils import (
-    encode_dict,
     ExtractorError,
     sanitized_Request,
 )
@@ -57,7 +56,7 @@ class FC2IE(InfoExtractor):
             'Submit': ' Login ',
         }
 
-        login_data = compat_urllib_parse.urlencode(encode_dict(login_form_strs)).encode('utf-8')
+        login_data = compat_urllib_parse_urlencode(login_form_strs).encode('utf-8')
         request = sanitized_Request(
             'https://secure.id.fc2.com/index.php?mode=login&switch_language=en', login_data)
 
diff --git a/youtube_dl/extractor/fivemin.py b/youtube_dl/extractor/fivemin.py
index 67d50a386..6b8345416 100644
--- a/youtube_dl/extractor/fivemin.py
+++ b/youtube_dl/extractor/fivemin.py
@@ -4,8 +4,8 @@ import re
 
 from .common import InfoExtractor
 from ..compat import (
-    compat_urllib_parse,
     compat_parse_qs,
+    compat_urllib_parse_urlencode,
     compat_urllib_parse_urlparse,
     compat_urlparse,
 )
@@ -109,7 +109,7 @@ class FiveMinIE(InfoExtractor):
 
         response = self._download_json(
             'https://syn.5min.com/handlers/SenseHandler.ashx?' +
-            compat_urllib_parse.urlencode({
+            compat_urllib_parse_urlencode({
                 'func': 'GetResults',
                 'playlist': video_id,
                 'sid': sid,
diff --git a/youtube_dl/extractor/flickr.py b/youtube_dl/extractor/flickr.py
index 18f439df9..0a3de1498 100644
--- a/youtube_dl/extractor/flickr.py
+++ b/youtube_dl/extractor/flickr.py
@@ -1,7 +1,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse
+from ..compat import compat_urllib_parse_urlencode
 from ..utils import (
     ExtractorError,
     int_or_none,
@@ -42,7 +42,7 @@ class FlickrIE(InfoExtractor):
         }
         if secret:
             query['secret'] = secret
-        data = self._download_json(self._API_BASE_URL + compat_urllib_parse.urlencode(query), video_id, note)
+        data = self._download_json(self._API_BASE_URL + compat_urllib_parse_urlencode(query), video_id, note)
         if data['stat'] != 'ok':
             raise ExtractorError(data['message'])
         return data
diff --git a/youtube_dl/extractor/funimation.py b/youtube_dl/extractor/funimation.py
index 0f37ed786..1eb528f31 100644
--- a/youtube_dl/extractor/funimation.py
+++ b/youtube_dl/extractor/funimation.py
@@ -5,7 +5,6 @@ from .common import InfoExtractor
 from ..utils import (
     clean_html,
     determine_ext,
-    encode_dict,
     int_or_none,
     sanitized_Request,
     ExtractorError,
@@ -54,10 +53,10 @@ class FunimationIE(InfoExtractor):
         (username, password) = self._get_login_info()
         if username is None:
             return
-        data = urlencode_postdata(encode_dict({
+        data = urlencode_postdata({
             'email_field': username,
             'password_field': password,
-        }))
+        })
         login_request = sanitized_Request('http://www.funimation.com/login', data, headers={
             'User-Agent': 'Mozilla/5.0 (Windows NT 5.2; WOW64; rv:42.0) Gecko/20100101 Firefox/42.0',
             'Content-Type': 'application/x-www-form-urlencoded'
diff --git a/youtube_dl/extractor/gdcvault.py b/youtube_dl/extractor/gdcvault.py
index 3befd3e7b..cc8fa45d2 100644
--- a/youtube_dl/extractor/gdcvault.py
+++ b/youtube_dl/extractor/gdcvault.py
@@ -3,7 +3,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse
+from ..compat import compat_urllib_parse_urlencode
 from ..utils import (
     remove_end,
     HEADRequest,
@@ -123,7 +123,7 @@ class GDCVaultIE(InfoExtractor):
             'password': password,
         }
 
-        request = sanitized_Request(login_url, compat_urllib_parse.urlencode(login_form))
+        request = sanitized_Request(login_url, compat_urllib_parse_urlencode(login_form))
         request.add_header('Content-Type', 'application/x-www-form-urlencoded')
         self._download_webpage(request, display_id, 'Logging in')
         start_page = self._download_webpage(webpage_url, display_id, 'Getting authenticated video page')
diff --git a/youtube_dl/extractor/hotnewhiphop.py b/youtube_dl/extractor/hotnewhiphop.py
index efc3e8429..152d2a98a 100644
--- a/youtube_dl/extractor/hotnewhiphop.py
+++ b/youtube_dl/extractor/hotnewhiphop.py
@@ -3,7 +3,7 @@ from __future__ import unicode_literals
 import base64
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse
+from ..compat import compat_urllib_parse_urlencode
 from ..utils import (
     ExtractorError,
     HEADRequest,
@@ -35,7 +35,7 @@ class HotNewHipHopIE(InfoExtractor):
                 r'"contentUrl" content="(.*?)"', webpage, 'content URL')
             return self.url_result(video_url, ie='Youtube')
 
-        reqdata = compat_urllib_parse.urlencode([
+        reqdata = compat_urllib_parse_urlencode([
             ('mediaType', 's'),
             ('mediaId', video_id),
         ])
diff --git a/youtube_dl/extractor/hypem.py b/youtube_dl/extractor/hypem.py
index e0ab31802..f7c913054 100644
--- a/youtube_dl/extractor/hypem.py
+++ b/youtube_dl/extractor/hypem.py
@@ -4,7 +4,7 @@ import json
 import time
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse
+from ..compat import compat_urllib_parse_urlencode
 from ..utils import (
     ExtractorError,
     sanitized_Request,
@@ -28,7 +28,7 @@ class HypemIE(InfoExtractor):
         track_id = self._match_id(url)
 
         data = {'ax': 1, 'ts': time.time()}
-        request = sanitized_Request(url + '?' + compat_urllib_parse.urlencode(data))
+        request = sanitized_Request(url + '?' + compat_urllib_parse_urlencode(data))
         response, urlh = self._download_webpage_handle(
             request, track_id, 'Downloading webpage with the url')
 
diff --git a/youtube_dl/extractor/internetvideoarchive.py b/youtube_dl/extractor/internetvideoarchive.py
index 483cc6f9e..e60145b3d 100644
--- a/youtube_dl/extractor/internetvideoarchive.py
+++ b/youtube_dl/extractor/internetvideoarchive.py
@@ -5,7 +5,7 @@ import re
 from .common import InfoExtractor
 from ..compat import (
     compat_urlparse,
-    compat_urllib_parse,
+    compat_urllib_parse_urlencode,
 )
 from ..utils import (
     xpath_with_ns,
@@ -38,7 +38,7 @@ class InternetVideoArchiveIE(InfoExtractor):
         # Other player ids return m3u8 urls
         cleaned_dic['playerid'] = '247'
         cleaned_dic['videokbrate'] = '100000'
-        return compat_urllib_parse.urlencode(cleaned_dic)
+        return compat_urllib_parse_urlencode(cleaned_dic)
 
     def _real_extract(self, url):
         query = compat_urlparse.urlparse(url).query
diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index ffcea30ad..9e8c9432a 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -14,7 +14,7 @@ from .common import InfoExtractor
 from ..compat import (
     compat_parse_qs,
     compat_str,
-    compat_urllib_parse,
+    compat_urllib_parse_urlencode,
     compat_urllib_parse_urlparse,
 )
 from ..utils import (
@@ -322,7 +322,7 @@ class IqiyiIE(InfoExtractor):
             'bird_t': timestamp,
         }
         validation_result = self._download_json(
-            'http://kylin.iqiyi.com/validate?' + compat_urllib_parse.urlencode(validation_params), None,
+            'http://kylin.iqiyi.com/validate?' + compat_urllib_parse_urlencode(validation_params), None,
             note='Validate credentials', errnote='Unable to validate credentials')
 
         MSG_MAP = {
@@ -456,7 +456,7 @@ class IqiyiIE(InfoExtractor):
                         'QY00001': auth_result['data']['u'],
                     })
                 api_video_url += '?' if '?' not in api_video_url else '&'
-                api_video_url += compat_urllib_parse.urlencode(param)
+                api_video_url += compat_urllib_parse_urlencode(param)
                 js = self._download_json(
                     api_video_url, video_id,
                     note='Download video info of segment %d for format %s' % (segment_index + 1, format_id))
@@ -494,7 +494,7 @@ class IqiyiIE(InfoExtractor):
         }
 
         api_url = 'http://cache.video.qiyi.com/vms' + '?' + \
-            compat_urllib_parse.urlencode(param)
+            compat_urllib_parse_urlencode(param)
         raw_data = self._download_json(api_url, video_id)
         return raw_data
 
diff --git a/youtube_dl/extractor/ivideon.py b/youtube_dl/extractor/ivideon.py
index 617dc8c07..3ca824f79 100644
--- a/youtube_dl/extractor/ivideon.py
+++ b/youtube_dl/extractor/ivideon.py
@@ -5,7 +5,7 @@ import re
 
 from .common import InfoExtractor
 from ..compat import (
-    compat_urllib_parse,
+    compat_urllib_parse_urlencode,
     compat_urlparse,
 )
 from ..utils import qualities
@@ -62,7 +62,7 @@ class IvideonIE(InfoExtractor):
         quality = qualities(self._QUALITIES)
 
         formats = [{
-            'url': 'https://streaming.ivideon.com/flv/live?%s' % compat_urllib_parse.urlencode({
+            'url': 'https://streaming.ivideon.com/flv/live?%s' % compat_urllib_parse_urlencode({
                 'server': server_id,
                 'camera': camera_id,
                 'sessionId': 'demo',
diff --git a/youtube_dl/extractor/kaltura.py b/youtube_dl/extractor/kaltura.py
index 44d7c84a1..a65697ff5 100644
--- a/youtube_dl/extractor/kaltura.py
+++ b/youtube_dl/extractor/kaltura.py
@@ -6,7 +6,7 @@ import base64
 
 from .common import InfoExtractor
 from ..compat import (
-    compat_urllib_parse,
+    compat_urllib_parse_urlencode,
     compat_urlparse,
     compat_parse_qs,
 )
@@ -71,7 +71,7 @@ class KalturaIE(InfoExtractor):
                 for k, v in a.items():
                     params['%d:%s' % (i, k)] = v
 
-        query = compat_urllib_parse.urlencode(params)
+        query = compat_urllib_parse_urlencode(params)
         url = self._API_BASE + query
         data = self._download_json(url, video_id, *args, **kwargs)
 
diff --git a/youtube_dl/extractor/laola1tv.py b/youtube_dl/extractor/laola1tv.py
index 41d80bc12..d9dc067d2 100644
--- a/youtube_dl/extractor/laola1tv.py
+++ b/youtube_dl/extractor/laola1tv.py
@@ -5,7 +5,7 @@ import re
 
 from .common import InfoExtractor
 from ..compat import (
-    compat_urllib_parse,
+    compat_urllib_parse_urlencode,
     compat_urlparse,
 )
 from ..utils import (
@@ -90,7 +90,7 @@ class Laola1TvIE(InfoExtractor):
 
         hd_doc = self._download_xml(
             'http://www.laola1.tv/server/hd_video.php?%s'
-            % compat_urllib_parse.urlencode({
+            % compat_urllib_parse_urlencode({
                 'play': video_id,
                 'partner': partner_id,
                 'portal': portal,
@@ -108,7 +108,7 @@ class Laola1TvIE(InfoExtractor):
 
         req = sanitized_Request(
             'https://club.laola1.tv/sp/laola1/api/v3/user/session/premium/player/stream-access?%s' %
-            compat_urllib_parse.urlencode({
+            compat_urllib_parse_urlencode({
                 'videoId': video_id,
                 'target': VS_TARGETS.get(kind, '2'),
                 'label': _v('label'),
diff --git a/youtube_dl/extractor/leeco.py b/youtube_dl/extractor/leeco.py
index 462b752dd..375fdaed1 100644
--- a/youtube_dl/extractor/leeco.py
+++ b/youtube_dl/extractor/leeco.py
@@ -11,7 +11,7 @@ from .common import InfoExtractor
 from ..compat import (
     compat_ord,
     compat_str,
-    compat_urllib_parse,
+    compat_urllib_parse_urlencode,
 )
 from ..utils import (
     determine_ext,
@@ -122,7 +122,7 @@ class LeIE(InfoExtractor):
             'domain': 'www.le.com'
         }
         play_json_req = sanitized_Request(
-            'http://api.le.com/mms/out/video/playJson?' + compat_urllib_parse.urlencode(params)
+            'http://api.le.com/mms/out/video/playJson?' + compat_urllib_parse_urlencode(params)
         )
         cn_verification_proxy = self._downloader.params.get('cn_verification_proxy')
         if cn_verification_proxy:
@@ -151,7 +151,7 @@ class LeIE(InfoExtractor):
         for format_id in formats:
             if format_id in dispatch:
                 media_url = playurl['domain'][0] + dispatch[format_id][0]
-                media_url += '&' + compat_urllib_parse.urlencode({
+                media_url += '&' + compat_urllib_parse_urlencode({
                     'm3v': 1,
                     'format': 1,
                     'expect': 3,
@@ -305,7 +305,7 @@ class LetvCloudIE(InfoExtractor):
             }
             self.sign_data(data)
             return self._download_json(
-                'http://api.letvcloud.com/gpc.php?' + compat_urllib_parse.urlencode(data),
+                'http://api.letvcloud.com/gpc.php?' + compat_urllib_parse_urlencode(data),
                 media_id, 'Downloading playJson data for type %s' % cf)
 
         play_json = get_play_json(cf, time.time())
diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index d4e1ae99d..df50cb655 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -6,7 +6,7 @@ import json
 from .common import InfoExtractor
 from ..compat import (
     compat_str,
-    compat_urllib_parse,
+    compat_urllib_parse_urlencode,
 )
 from ..utils import (
     ExtractorError,
@@ -36,7 +36,7 @@ class LyndaBaseIE(InfoExtractor):
             'stayPut': 'false'
         }
         request = sanitized_Request(
-            self._LOGIN_URL, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
+            self._LOGIN_URL, compat_urllib_parse_urlencode(login_form).encode('utf-8'))
         login_page = self._download_webpage(
             request, None, 'Logging in as %s' % username)
 
@@ -65,7 +65,7 @@ class LyndaBaseIE(InfoExtractor):
                     'stayPut': 'false',
                 }
                 request = sanitized_Request(
-                    self._LOGIN_URL, compat_urllib_parse.urlencode(confirm_form).encode('utf-8'))
+                    self._LOGIN_URL, compat_urllib_parse_urlencode(confirm_form).encode('utf-8'))
                 login_page = self._download_webpage(
                     request, None,
                     'Confirming log in and log out from another device')
diff --git a/youtube_dl/extractor/matchtv.py b/youtube_dl/extractor/matchtv.py
index 28e0dfe63..e33bfde3b 100644
--- a/youtube_dl/extractor/matchtv.py
+++ b/youtube_dl/extractor/matchtv.py
@@ -4,7 +4,7 @@ from __future__ import unicode_literals
 import random
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse
+from ..compat import compat_urllib_parse_urlencode
 from ..utils import (
     sanitized_Request,
     xpath_text,
@@ -29,7 +29,7 @@ class MatchTVIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = 'matchtv-live'
         request = sanitized_Request(
-            'http://player.matchtv.ntvplus.tv/player/smil?%s' % compat_urllib_parse.urlencode({
+            'http://player.matchtv.ntvplus.tv/player/smil?%s' % compat_urllib_parse_urlencode({
                 'ts': '',
                 'quality': 'SD',
                 'contentId': '561d2c0df7159b37178b4567',
diff --git a/youtube_dl/extractor/metacafe.py b/youtube_dl/extractor/metacafe.py
index c31e8798a..0e4865446 100644
--- a/youtube_dl/extractor/metacafe.py
+++ b/youtube_dl/extractor/metacafe.py
@@ -5,8 +5,8 @@ import re
 from .common import InfoExtractor
 from ..compat import (
     compat_parse_qs,
-    compat_urllib_parse,
     compat_urllib_parse_unquote,
+    compat_urllib_parse_urlencode,
 )
 from ..utils import (
     determine_ext,
@@ -117,7 +117,7 @@ class MetacafeIE(InfoExtractor):
             'filters': '0',
             'submit': "Continue - I'm over 18",
         }
-        request = sanitized_Request(self._FILTER_POST, compat_urllib_parse.urlencode(disclaimer_form))
+        request = sanitized_Request(self._FILTER_POST, compat_urllib_parse_urlencode(disclaimer_form))
         request.add_header('Content-Type', 'application/x-www-form-urlencoded')
         self.report_age_confirmation()
         self._download_webpage(request, None, False, 'Unable to confirm age')
diff --git a/youtube_dl/extractor/minhateca.py b/youtube_dl/extractor/minhateca.py
index e46b23a6f..6ec53c303 100644
--- a/youtube_dl/extractor/minhateca.py
+++ b/youtube_dl/extractor/minhateca.py
@@ -2,7 +2,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse
+from ..compat import compat_urllib_parse_urlencode
 from ..utils import (
     int_or_none,
     parse_duration,
@@ -39,7 +39,7 @@ class MinhatecaIE(InfoExtractor):
         ]
         req = sanitized_Request(
             'http://minhateca.com.br/action/License/Download',
-            data=compat_urllib_parse.urlencode(token_data))
+            data=compat_urllib_parse_urlencode(token_data))
         req.add_header('Content-Type', 'application/x-www-form-urlencoded')
         data = self._download_json(
             req, video_id, note='Downloading metadata')
diff --git a/youtube_dl/extractor/mitele.py b/youtube_dl/extractor/mitele.py
index 9e584860a..76ced7928 100644
--- a/youtube_dl/extractor/mitele.py
+++ b/youtube_dl/extractor/mitele.py
@@ -2,11 +2,10 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..compat import (
-    compat_urllib_parse,
+    compat_urllib_parse_urlencode,
     compat_urlparse,
 )
 from ..utils import (
-    encode_dict,
     get_element_by_attribute,
     int_or_none,
 )
@@ -60,7 +59,7 @@ class MiTeleIE(InfoExtractor):
                 'sta': '0',
             }
             media = self._download_json(
-                '%s/?%s' % (gat, compat_urllib_parse.urlencode(encode_dict(token_data))),
+                '%s/?%s' % (gat, compat_urllib_parse_urlencode(token_data)),
                 display_id, 'Downloading %s JSON' % location['loc'])
             file_ = media.get('file')
             if not file_:
diff --git a/youtube_dl/extractor/moevideo.py b/youtube_dl/extractor/moevideo.py
index d930b9634..89cdd4600 100644
--- a/youtube_dl/extractor/moevideo.py
+++ b/youtube_dl/extractor/moevideo.py
@@ -5,7 +5,7 @@ import json
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse
+from ..compat import compat_urllib_parse_urlencode
 from ..utils import (
     ExtractorError,
     int_or_none,
@@ -77,7 +77,7 @@ class MoeVideoIE(InfoExtractor):
             ],
         ]
         r_json = json.dumps(r)
-        post = compat_urllib_parse.urlencode({'r': r_json})
+        post = compat_urllib_parse_urlencode({'r': r_json})
         req = sanitized_Request(self._API_URL, post)
         req.add_header('Content-type', 'application/x-www-form-urlencoded')
 
diff --git a/youtube_dl/extractor/moniker.py b/youtube_dl/extractor/moniker.py
index f6bf94f2f..c5ce693f1 100644
--- a/youtube_dl/extractor/moniker.py
+++ b/youtube_dl/extractor/moniker.py
@@ -5,7 +5,7 @@ import os.path
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse
+from ..compat import compat_urllib_parse_urlencode
 from ..utils import (
     ExtractorError,
     remove_start,
@@ -88,7 +88,7 @@ class MonikerIE(InfoExtractor):
             fields = re.findall(r'type="hidden" name="(.+?)"\s* value="?(.+?)">', orig_webpage)
             data = dict(fields)
 
-            post = compat_urllib_parse.urlencode(data)
+            post = compat_urllib_parse_urlencode(data)
             headers = {
                 b'Content-Type': b'application/x-www-form-urlencoded',
             }
diff --git a/youtube_dl/extractor/mooshare.py b/youtube_dl/extractor/mooshare.py
index f010f52d5..ee3947f43 100644
--- a/youtube_dl/extractor/mooshare.py
+++ b/youtube_dl/extractor/mooshare.py
@@ -3,7 +3,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse
+from ..compat import compat_urllib_parse_urlencode
 from ..utils import (
     ExtractorError,
     sanitized_Request,
@@ -58,7 +58,7 @@ class MooshareIE(InfoExtractor):
         }
 
         request = sanitized_Request(
-            'http://mooshare.biz/%s' % video_id, compat_urllib_parse.urlencode(download_form))
+            'http://mooshare.biz/%s' % video_id, compat_urllib_parse_urlencode(download_form))
         request.add_header('Content-Type', 'application/x-www-form-urlencoded')
 
         self._sleep(5, video_id)
diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index 824bbcb4e..640ee3d93 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -4,7 +4,7 @@ import re
 
 from .common import InfoExtractor
 from ..compat import (
-    compat_urllib_parse,
+    compat_urllib_parse_urlencode,
     compat_str,
 )
 from ..utils import (
@@ -171,7 +171,7 @@ class MTVServicesInfoExtractor(InfoExtractor):
         data = {'uri': uri}
         if self._LANG:
             data['lang'] = self._LANG
-        return compat_urllib_parse.urlencode(data)
+        return compat_urllib_parse_urlencode(data)
 
     def _get_videos_info(self, uri):
         video_id = self._id_from_uri(uri)
diff --git a/youtube_dl/extractor/muzu.py b/youtube_dl/extractor/muzu.py
index 1e9cf8de9..cbc800481 100644
--- a/youtube_dl/extractor/muzu.py
+++ b/youtube_dl/extractor/muzu.py
@@ -1,9 +1,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse,
-)
+from ..compat import compat_urllib_parse_urlencode
 
 
 class MuzuTVIE(InfoExtractor):
@@ -25,7 +23,7 @@ class MuzuTVIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        info_data = compat_urllib_parse.urlencode({
+        info_data = compat_urllib_parse_urlencode({
             'format': 'json',
             'url': url,
         })
@@ -41,7 +39,7 @@ class MuzuTVIE(InfoExtractor):
             if video_info.get('v%s' % quality):
                 break
 
-        data = compat_urllib_parse.urlencode({
+        data = compat_urllib_parse_urlencode({
             'ai': video_id,
             # Even if each time you watch a video the hash changes,
             # it seems to work for different videos, and it will work
diff --git a/youtube_dl/extractor/myvideo.py b/youtube_dl/extractor/myvideo.py
index c83a1eab5..6d447a493 100644
--- a/youtube_dl/extractor/myvideo.py
+++ b/youtube_dl/extractor/myvideo.py
@@ -9,8 +9,8 @@ import json
 from .common import InfoExtractor
 from ..compat import (
     compat_ord,
-    compat_urllib_parse,
     compat_urllib_parse_unquote,
+    compat_urllib_parse_urlencode,
 )
 from ..utils import (
     ExtractorError,
@@ -112,7 +112,7 @@ class MyVideoIE(InfoExtractor):
                 encxml = compat_urllib_parse_unquote(b)
         if not params.get('domain'):
             params['domain'] = 'www.myvideo.de'
-        xmldata_url = '%s?%s' % (encxml, compat_urllib_parse.urlencode(params))
+        xmldata_url = '%s?%s' % (encxml, compat_urllib_parse_urlencode(params))
         if 'flash_playertype=MTV' in xmldata_url:
             self._downloader.report_warning('avoiding MTV player')
             xmldata_url = (
diff --git a/youtube_dl/extractor/naver.py b/youtube_dl/extractor/naver.py
index 1f5fc2145..6d6f69b44 100644
--- a/youtube_dl/extractor/naver.py
+++ b/youtube_dl/extractor/naver.py
@@ -5,7 +5,7 @@ import re
 
 from .common import InfoExtractor
 from ..compat import (
-    compat_urllib_parse,
+    compat_urllib_parse_urlencode,
     compat_urlparse,
 )
 from ..utils import (
@@ -53,8 +53,8 @@ class NaverIE(InfoExtractor):
             raise ExtractorError('couldn\'t extract vid and key')
         vid = m_id.group(1)
         key = m_id.group(2)
-        query = compat_urllib_parse.urlencode({'vid': vid, 'inKey': key, })
-        query_urls = compat_urllib_parse.urlencode({
+        query = compat_urllib_parse_urlencode({'vid': vid, 'inKey': key, })
+        query_urls = compat_urllib_parse_urlencode({
             'masterVid': vid,
             'protocol': 'p2p',
             'inKey': key,
diff --git a/youtube_dl/extractor/nba.py b/youtube_dl/extractor/nba.py
index 3e2b3e599..d896b0d04 100644
--- a/youtube_dl/extractor/nba.py
+++ b/youtube_dl/extractor/nba.py
@@ -6,7 +6,7 @@ import re
 
 from .common import InfoExtractor
 from ..compat import (
-    compat_urllib_parse,
+    compat_urllib_parse_urlencode,
     compat_urlparse,
 )
 from ..utils import (
@@ -97,7 +97,7 @@ class NBAIE(InfoExtractor):
     _PAGE_SIZE = 30
 
     def _fetch_page(self, team, video_id, page):
-        search_url = 'http://searchapp2.nba.com/nba-search/query.jsp?' + compat_urllib_parse.urlencode({
+        search_url = 'http://searchapp2.nba.com/nba-search/query.jsp?' + compat_urllib_parse_urlencode({
             'type': 'teamvideo',
             'start': page * self._PAGE_SIZE + 1,
             'npp': (page + 1) * self._PAGE_SIZE + 1,
diff --git a/youtube_dl/extractor/neteasemusic.py b/youtube_dl/extractor/neteasemusic.py
index 7830616f8..0d36474fa 100644
--- a/youtube_dl/extractor/neteasemusic.py
+++ b/youtube_dl/extractor/neteasemusic.py
@@ -8,7 +8,7 @@ import re
 
 from .common import InfoExtractor
 from ..compat import (
-    compat_urllib_parse,
+    compat_urllib_parse_urlencode,
     compat_str,
     compat_itertools_count,
 )
@@ -153,7 +153,7 @@ class NetEaseMusicIE(NetEaseMusicBaseIE):
             'ids': '[%s]' % song_id
         }
         info = self.query_api(
-            'song/detail?' + compat_urllib_parse.urlencode(params),
+            'song/detail?' + compat_urllib_parse_urlencode(params),
             song_id, 'Downloading song info')['songs'][0]
 
         formats = self.extract_formats(info)
diff --git a/youtube_dl/extractor/nextmovie.py b/youtube_dl/extractor/nextmovie.py
index 657ae77a0..9ccd7d774 100644
--- a/youtube_dl/extractor/nextmovie.py
+++ b/youtube_dl/extractor/nextmovie.py
@@ -2,7 +2,7 @@
 from __future__ import unicode_literals
 
 from .mtv import MTVServicesInfoExtractor
-from ..compat import compat_urllib_parse
+from ..compat import compat_urllib_parse_urlencode
 
 
 class NextMovieIE(MTVServicesInfoExtractor):
@@ -20,7 +20,7 @@ class NextMovieIE(MTVServicesInfoExtractor):
     }]
 
     def _get_feed_query(self, uri):
-        return compat_urllib_parse.urlencode({
+        return compat_urllib_parse_urlencode({
             'feed': '1505',
             'mgid': uri,
         })
diff --git a/youtube_dl/extractor/nfb.py b/youtube_dl/extractor/nfb.py
index 5bd15f7a7..ba1eefafc 100644
--- a/youtube_dl/extractor/nfb.py
+++ b/youtube_dl/extractor/nfb.py
@@ -1,7 +1,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse
+from ..compat import compat_urllib_parse_urlencode
 from ..utils import sanitized_Request
 
 
@@ -40,7 +40,7 @@ class NFBIE(InfoExtractor):
 
         request = sanitized_Request(
             'https://www.nfb.ca/film/%s/player_config' % video_id,
-            compat_urllib_parse.urlencode({'getConfig': 'true'}).encode('ascii'))
+            compat_urllib_parse_urlencode({'getConfig': 'true'}).encode('ascii'))
         request.add_header('Content-Type', 'application/x-www-form-urlencoded')
         request.add_header('X-NFB-Referer', 'http://www.nfb.ca/medias/flash/NFBVideoPlayer.swf')
 
diff --git a/youtube_dl/extractor/nhl.py b/youtube_dl/extractor/nhl.py
index 8d5ce46ad..c1dea8b6c 100644
--- a/youtube_dl/extractor/nhl.py
+++ b/youtube_dl/extractor/nhl.py
@@ -7,7 +7,7 @@ import os
 from .common import InfoExtractor
 from ..compat import (
     compat_urlparse,
-    compat_urllib_parse,
+    compat_urllib_parse_urlencode,
     compat_urllib_parse_urlparse
 )
 from ..utils import (
@@ -38,7 +38,7 @@ class NHLBaseInfoExtractor(InfoExtractor):
             parsed_url = compat_urllib_parse_urlparse(initial_video_url)
             filename, ext = os.path.splitext(parsed_url.path)
             path = '%s_sd%s' % (filename, ext)
-            data = compat_urllib_parse.urlencode({
+            data = compat_urllib_parse_urlencode({
                 'type': 'fvod',
                 'path': compat_urlparse.urlunparse(parsed_url[:2] + (path,) + parsed_url[3:])
             })
@@ -211,7 +211,7 @@ class NHLVideocenterIE(NHLBaseInfoExtractor):
             r'tab0"[^>]*?>(.*?)</td>',
             webpage, 'playlist title', flags=re.DOTALL).lower().capitalize()
 
-        data = compat_urllib_parse.urlencode({
+        data = compat_urllib_parse_urlencode({
             'cid': cat_id,
             # This is the default value
             'count': 12,
diff --git a/youtube_dl/extractor/nick.py b/youtube_dl/extractor/nick.py
index b62819ae5..ce065f2b0 100644
--- a/youtube_dl/extractor/nick.py
+++ b/youtube_dl/extractor/nick.py
@@ -2,7 +2,7 @@
 from __future__ import unicode_literals
 
 from .mtv import MTVServicesInfoExtractor
-from ..compat import compat_urllib_parse
+from ..compat import compat_urllib_parse_urlencode
 
 
 class NickIE(MTVServicesInfoExtractor):
@@ -54,7 +54,7 @@ class NickIE(MTVServicesInfoExtractor):
     }]
 
     def _get_feed_query(self, uri):
-        return compat_urllib_parse.urlencode({
+        return compat_urllib_parse_urlencode({
             'feed': 'nick_arc_player_prime',
             'mgid': uri,
         })
diff --git a/youtube_dl/extractor/niconico.py b/youtube_dl/extractor/niconico.py
index 586e52a4a..688f0a124 100644
--- a/youtube_dl/extractor/niconico.py
+++ b/youtube_dl/extractor/niconico.py
@@ -7,11 +7,10 @@ import datetime
 
 from .common import InfoExtractor
 from ..compat import (
-    compat_urllib_parse,
+    compat_urllib_parse_urlencode,
     compat_urlparse,
 )
 from ..utils import (
-    encode_dict,
     ExtractorError,
     int_or_none,
     parse_duration,
@@ -101,7 +100,7 @@ class NiconicoIE(InfoExtractor):
             'mail': username,
             'password': password,
         }
-        login_data = compat_urllib_parse.urlencode(encode_dict(login_form_strs)).encode('utf-8')
+        login_data = compat_urllib_parse_urlencode(login_form_strs).encode('utf-8')
         request = sanitized_Request(
             'https://secure.nicovideo.jp/secure/login', login_data)
         login_results = self._download_webpage(
@@ -141,7 +140,7 @@ class NiconicoIE(InfoExtractor):
                 r'\'thumbPlayKey\'\s*:\s*\'(.*?)\'', ext_player_info, 'thumbPlayKey')
 
             # Get flv info
-            flv_info_data = compat_urllib_parse.urlencode({
+            flv_info_data = compat_urllib_parse_urlencode({
                 'k': thumb_play_key,
                 'v': video_id
             })
diff --git a/youtube_dl/extractor/noco.py b/youtube_dl/extractor/noco.py
index ec7317a2f..8f4b69a6f 100644
--- a/youtube_dl/extractor/noco.py
+++ b/youtube_dl/extractor/noco.py
@@ -8,7 +8,7 @@ import hashlib
 from .common import InfoExtractor
 from ..compat import (
     compat_str,
-    compat_urllib_parse,
+    compat_urllib_parse_urlencode,
     compat_urlparse,
 )
 from ..utils import (
@@ -75,7 +75,7 @@ class NocoIE(InfoExtractor):
             'username': username,
             'password': password,
         }
-        request = sanitized_Request(self._LOGIN_URL, compat_urllib_parse.urlencode(login_form))
+        request = sanitized_Request(self._LOGIN_URL, compat_urllib_parse_urlencode(login_form))
         request.add_header('Content-Type', 'application/x-www-form-urlencoded; charset=UTF-8')
 
         login = self._download_json(request, None, 'Logging in as %s' % username)
diff --git a/youtube_dl/extractor/novamov.py b/youtube_dl/extractor/novamov.py
index d68c1ad79..a131f7dbd 100644
--- a/youtube_dl/extractor/novamov.py
+++ b/youtube_dl/extractor/novamov.py
@@ -7,7 +7,6 @@ from ..compat import compat_urlparse
 from ..utils import (
     ExtractorError,
     NO_DEFAULT,
-    encode_dict,
     sanitized_Request,
     urlencode_postdata,
 )
@@ -73,7 +72,7 @@ class NovaMovIE(InfoExtractor):
             if not post_url.startswith('http'):
                 post_url = compat_urlparse.urljoin(url, post_url)
             request = sanitized_Request(
-                post_url, urlencode_postdata(encode_dict(fields)))
+                post_url, urlencode_postdata(fields))
             request.add_header('Content-Type', 'application/x-www-form-urlencoded')
             request.add_header('Referer', post_url)
             webpage = self._download_webpage(
diff --git a/youtube_dl/extractor/npr.py b/youtube_dl/extractor/npr.py
index a3f0abb4e..1777aa10b 100644
--- a/youtube_dl/extractor/npr.py
+++ b/youtube_dl/extractor/npr.py
@@ -1,7 +1,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse
+from ..compat import compat_urllib_parse_urlencode
 from ..utils import (
     int_or_none,
     qualities,
@@ -38,7 +38,7 @@ class NprIE(InfoExtractor):
         playlist_id = self._match_id(url)
 
         config = self._download_json(
-            'http://api.npr.org/query?%s' % compat_urllib_parse.urlencode({
+            'http://api.npr.org/query?%s' % compat_urllib_parse_urlencode({
                 'id': playlist_id,
                 'fields': 'titles,audio,show',
                 'format': 'json',
diff --git a/youtube_dl/extractor/ooyala.py b/youtube_dl/extractor/ooyala.py
index 20b984288..16f040191 100644
--- a/youtube_dl/extractor/ooyala.py
+++ b/youtube_dl/extractor/ooyala.py
@@ -9,7 +9,7 @@ from ..utils import (
     ExtractorError,
     unsmuggle_url,
 )
-from ..compat import compat_urllib_parse
+from ..compat import compat_urllib_parse_urlencode
 
 
 class OoyalaBaseIE(InfoExtractor):
@@ -35,7 +35,7 @@ class OoyalaBaseIE(InfoExtractor):
         for supported_format in ('mp4', 'm3u8', 'hds', 'rtmp'):
             auth_data = self._download_json(
                 self._AUTHORIZATION_URL_TEMPLATE % (pcode, embed_code) +
-                compat_urllib_parse.urlencode({
+                compat_urllib_parse_urlencode({
                     'domain': domain,
                     'supportedFormats': supported_format
                 }),
diff --git a/youtube_dl/extractor/patreon.py b/youtube_dl/extractor/patreon.py
index ec8876c28..229750665 100644
--- a/youtube_dl/extractor/patreon.py
+++ b/youtube_dl/extractor/patreon.py
@@ -65,7 +65,7 @@ class PatreonIE(InfoExtractor):
 
         request = sanitized_Request(
             'https://www.patreon.com/processLogin',
-            compat_urllib_parse.urlencode(login_form).encode('utf-8')
+            compat_urllib_parse_urlencode(login_form).encode('utf-8')
         )
         login_page = self._download_webpage(request, None, note='Logging in as %s' % username)
 
diff --git a/youtube_dl/extractor/played.py b/youtube_dl/extractor/played.py
index 2856af96f..63065622b 100644
--- a/youtube_dl/extractor/played.py
+++ b/youtube_dl/extractor/played.py
@@ -5,7 +5,7 @@ import re
 import os.path
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse
+from ..compat import compat_urllib_parse_urlencode
 from ..utils import (
     ExtractorError,
     sanitized_Request,
@@ -40,7 +40,7 @@ class PlayedIE(InfoExtractor):
 
         self._sleep(2, video_id)
 
-        post = compat_urllib_parse.urlencode(data)
+        post = compat_urllib_parse_urlencode(data)
         headers = {
             b'Content-Type': b'application/x-www-form-urlencoded',
         }
diff --git a/youtube_dl/extractor/playtvak.py b/youtube_dl/extractor/playtvak.py
index e360404f7..1e8096a25 100644
--- a/youtube_dl/extractor/playtvak.py
+++ b/youtube_dl/extractor/playtvak.py
@@ -4,7 +4,7 @@ from __future__ import unicode_literals
 from .common import InfoExtractor
 from ..compat import (
     compat_urlparse,
-    compat_urllib_parse,
+    compat_urllib_parse_urlencode,
 )
 from ..utils import (
     ExtractorError,
@@ -106,7 +106,7 @@ class PlaytvakIE(InfoExtractor):
         })
 
         info_url = compat_urlparse.urlunparse(
-            parsed_url._replace(query=compat_urllib_parse.urlencode(qs, True)))
+            parsed_url._replace(query=compat_urllib_parse_urlencode(qs, True)))
 
         json_info = self._download_json(
             info_url, video_id,
diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index 12e1c2862..575775f09 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -8,7 +8,7 @@ import collections
 from .common import InfoExtractor
 from ..compat import (
     compat_str,
-    compat_urllib_parse,
+    compat_urllib_parse_urlencode,
     compat_urlparse,
 )
 from ..utils import (
@@ -76,7 +76,7 @@ class PluralsightIE(PluralsightBaseIE):
             post_url = compat_urlparse.urljoin(self._LOGIN_URL, post_url)
 
         request = sanitized_Request(
-            post_url, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
+            post_url, compat_urllib_parse_urlencode(login_form).encode('utf-8'))
         request.add_header('Content-Type', 'application/x-www-form-urlencoded')
 
         response = self._download_webpage(
diff --git a/youtube_dl/extractor/porn91.py b/youtube_dl/extractor/porn91.py
index 63ce87ee3..9894f3262 100644
--- a/youtube_dl/extractor/porn91.py
+++ b/youtube_dl/extractor/porn91.py
@@ -2,8 +2,8 @@
 from __future__ import unicode_literals
 
 from ..compat import (
-    compat_urllib_parse,
     compat_urllib_parse_unquote,
+    compat_urllib_parse_urlencode,
 )
 from .common import InfoExtractor
 from ..utils import (
@@ -50,7 +50,7 @@ class Porn91IE(InfoExtractor):
             r'so.addVariable\(\'seccode\',\'([^\']+)\'', webpage, 'sec code')
         max_vid = self._search_regex(
             r'so.addVariable\(\'max_vid\',\'(\d+)\'', webpage, 'max vid')
-        url_params = compat_urllib_parse.urlencode({
+        url_params = compat_urllib_parse_urlencode({
             'VID': file_id,
             'mp4': '1',
             'seccode': sec_code,
diff --git a/youtube_dl/extractor/primesharetv.py b/youtube_dl/extractor/primesharetv.py
index 85aae9576..188f08826 100644
--- a/youtube_dl/extractor/primesharetv.py
+++ b/youtube_dl/extractor/primesharetv.py
@@ -1,7 +1,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse
+from ..compat import compat_urllib_parse_urlencode
 from ..utils import (
     ExtractorError,
     sanitized_Request,
@@ -42,7 +42,7 @@ class PrimeShareTVIE(InfoExtractor):
         self._sleep(wait_time, video_id)
 
         req = sanitized_Request(
-            url, compat_urllib_parse.urlencode(fields), headers)
+            url, compat_urllib_parse_urlencode(fields), headers)
         video_page = self._download_webpage(
             req, video_id, 'Downloading video page')
 
diff --git a/youtube_dl/extractor/promptfile.py b/youtube_dl/extractor/promptfile.py
index d5357283a..67312016c 100644
--- a/youtube_dl/extractor/promptfile.py
+++ b/youtube_dl/extractor/promptfile.py
@@ -4,7 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse
+from ..compat import compat_urllib_parse_urlencode
 from ..utils import (
     determine_ext,
     ExtractorError,
@@ -34,7 +34,7 @@ class PromptFileIE(InfoExtractor):
                                  expected=True)
 
         fields = self._hidden_inputs(webpage)
-        post = compat_urllib_parse.urlencode(fields)
+        post = compat_urllib_parse_urlencode(fields)
         req = sanitized_Request(url, post)
         req.add_header('Content-type', 'application/x-www-form-urlencoded')
         webpage = self._download_webpage(
diff --git a/youtube_dl/extractor/prosiebensat1.py b/youtube_dl/extractor/prosiebensat1.py
index 670e6950f..07d49d489 100644
--- a/youtube_dl/extractor/prosiebensat1.py
+++ b/youtube_dl/extractor/prosiebensat1.py
@@ -5,9 +5,7 @@ import re
 
 from hashlib import sha1
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse,
-)
+from ..compat import compat_urllib_parse_urlencode
 from ..utils import (
     ExtractorError,
     determine_ext,
@@ -235,7 +233,7 @@ class ProSiebenSat1IE(InfoExtractor):
         client_name = 'kolibri-2.0.19-splec4'
         client_location = url
 
-        videos_api_url = 'http://vas.sim-technik.de/vas/live/v2/videos?%s' % compat_urllib_parse.urlencode({
+        videos_api_url = 'http://vas.sim-technik.de/vas/live/v2/videos?%s' % compat_urllib_parse_urlencode({
             'access_token': access_token,
             'client_location': client_location,
             'client_name': client_name,
@@ -256,7 +254,7 @@ class ProSiebenSat1IE(InfoExtractor):
         client_id = g[:2] + sha1(''.join([clip_id, g, access_token, client_location, g, client_name])
                                  .encode('utf-8')).hexdigest()
 
-        sources_api_url = 'http://vas.sim-technik.de/vas/live/v2/videos/%s/sources?%s' % (clip_id, compat_urllib_parse.urlencode({
+        sources_api_url = 'http://vas.sim-technik.de/vas/live/v2/videos/%s/sources?%s' % (clip_id, compat_urllib_parse_urlencode({
             'access_token': access_token,
             'client_id': client_id,
             'client_location': client_location,
@@ -270,7 +268,7 @@ class ProSiebenSat1IE(InfoExtractor):
                                           client_location, source_ids_str, g, client_name])
                                  .encode('utf-8')).hexdigest()
 
-        url_api_url = 'http://vas.sim-technik.de/vas/live/v2/videos/%s/sources/url?%s' % (clip_id, compat_urllib_parse.urlencode({
+        url_api_url = 'http://vas.sim-technik.de/vas/live/v2/videos/%s/sources/url?%s' % (clip_id, compat_urllib_parse_urlencode({
             'access_token': access_token,
             'client_id': client_id,
             'client_location': client_location,
diff --git a/youtube_dl/extractor/shahid.py b/youtube_dl/extractor/shahid.py
index 1178b7a27..b4433a689 100644
--- a/youtube_dl/extractor/shahid.py
+++ b/youtube_dl/extractor/shahid.py
@@ -2,7 +2,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse
+from ..compat import compat_urllib_parse_urlencode
 from ..utils import (
     ExtractorError,
     int_or_none,
@@ -81,7 +81,7 @@ class ShahidIE(InfoExtractor):
         video = self._download_json(
             '%s/%s/%s?%s' % (
                 api_vars['url'], api_vars['playerType'], api_vars['id'],
-                compat_urllib_parse.urlencode({
+                compat_urllib_parse_urlencode({
                     'apiKey': 'sh@hid0nlin3',
                     'hash': 'b2wMCTHpSmyxGqQjJFOycRmLSex+BpTK/ooxy6vHaqs=',
                 })),
diff --git a/youtube_dl/extractor/shared.py b/youtube_dl/extractor/shared.py
index 96fe0b90d..e66441997 100644
--- a/youtube_dl/extractor/shared.py
+++ b/youtube_dl/extractor/shared.py
@@ -3,7 +3,7 @@ from __future__ import unicode_literals
 import base64
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse
+from ..compat import compat_urllib_parse_urlencode
 from ..utils import (
     ExtractorError,
     int_or_none,
@@ -45,7 +45,7 @@ class SharedIE(InfoExtractor):
 
         download_form = self._hidden_inputs(webpage)
         request = sanitized_Request(
-            url, compat_urllib_parse.urlencode(download_form))
+            url, compat_urllib_parse_urlencode(download_form))
         request.add_header('Content-Type', 'application/x-www-form-urlencoded')
 
         video_page = self._download_webpage(
diff --git a/youtube_dl/extractor/sharesix.py b/youtube_dl/extractor/sharesix.py
index f1ea9bdb2..61dc1c235 100644
--- a/youtube_dl/extractor/sharesix.py
+++ b/youtube_dl/extractor/sharesix.py
@@ -4,7 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse
+from ..compat import compat_urllib_parse_urlencode
 from ..utils import (
     parse_duration,
     sanitized_Request,
@@ -47,7 +47,7 @@ class ShareSixIE(InfoExtractor):
         fields = {
             'method_free': 'Free'
         }
-        post = compat_urllib_parse.urlencode(fields)
+        post = compat_urllib_parse_urlencode(fields)
         req = sanitized_Request(url, post)
         req.add_header('Content-type', 'application/x-www-form-urlencoded')
 
diff --git a/youtube_dl/extractor/sina.py b/youtube_dl/extractor/sina.py
index b2258a0f6..d03f1b1d4 100644
--- a/youtube_dl/extractor/sina.py
+++ b/youtube_dl/extractor/sina.py
@@ -4,7 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse
+from ..compat import compat_urllib_parse_urlencode
 from ..utils import sanitized_Request
 
 
@@ -39,7 +39,7 @@ class SinaIE(InfoExtractor):
     ]
 
     def _extract_video(self, video_id):
-        data = compat_urllib_parse.urlencode({'vid': video_id})
+        data = compat_urllib_parse_urlencode({'vid': video_id})
         url_doc = self._download_xml('http://v.iask.com/v_play.php?%s' % data,
                                      video_id, 'Downloading video url')
         image_page = self._download_webpage(
diff --git a/youtube_dl/extractor/smotri.py b/youtube_dl/extractor/smotri.py
index 015ef75f3..b4c6d5bbf 100644
--- a/youtube_dl/extractor/smotri.py
+++ b/youtube_dl/extractor/smotri.py
@@ -7,7 +7,7 @@ import hashlib
 import uuid
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse
+from ..compat import compat_urllib_parse_urlencode
 from ..utils import (
     ExtractorError,
     int_or_none,
@@ -175,7 +175,7 @@ class SmotriIE(InfoExtractor):
             video_form['pass'] = hashlib.md5(video_password.encode('utf-8')).hexdigest()
 
         request = sanitized_Request(
-            'http://smotri.com/video/view/url/bot/', compat_urllib_parse.urlencode(video_form))
+            'http://smotri.com/video/view/url/bot/', compat_urllib_parse_urlencode(video_form))
         request.add_header('Content-Type', 'application/x-www-form-urlencoded')
 
         video = self._download_json(request, video_id, 'Downloading video JSON')
@@ -338,7 +338,7 @@ class SmotriBroadcastIE(InfoExtractor):
             }
 
             request = sanitized_Request(
-                broadcast_url + '/?no_redirect=1', compat_urllib_parse.urlencode(login_form))
+                broadcast_url + '/?no_redirect=1', compat_urllib_parse_urlencode(login_form))
             request.add_header('Content-Type', 'application/x-www-form-urlencoded')
             broadcast_page = self._download_webpage(
                 request, broadcast_id, 'Logging in and confirming age')
diff --git a/youtube_dl/extractor/sohu.py b/youtube_dl/extractor/sohu.py
index ea8fc258d..49e5d09ae 100644
--- a/youtube_dl/extractor/sohu.py
+++ b/youtube_dl/extractor/sohu.py
@@ -6,7 +6,7 @@ import re
 from .common import InfoExtractor
 from ..compat import (
     compat_str,
-    compat_urllib_parse,
+    compat_urllib_parse_urlencode,
 )
 from ..utils import (
     ExtractorError,
@@ -170,7 +170,7 @@ class SohuIE(InfoExtractor):
                     if retries > 0:
                         download_note += ' (retry #%d)' % retries
                     part_info = self._parse_json(self._download_webpage(
-                        'http://%s/?%s' % (allot, compat_urllib_parse.urlencode(params)),
+                        'http://%s/?%s' % (allot, compat_urllib_parse_urlencode(params)),
                         video_id, download_note), video_id)
 
                     video_url = part_info['url']
diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 1efb2b980..2bca8fa3a 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -11,10 +11,9 @@ from .common import (
 from ..compat import (
     compat_str,
     compat_urlparse,
-    compat_urllib_parse,
+    compat_urllib_parse_urlencode,
 )
 from ..utils import (
-    encode_dict,
     ExtractorError,
     int_or_none,
     unified_strdate,
@@ -393,7 +392,7 @@ class SoundcloudUserIE(SoundcloudIE):
         query = COMMON_QUERY.copy()
         query['offset'] = 0
 
-        next_href = base_url + '?' + compat_urllib_parse.urlencode(query)
+        next_href = base_url + '?' + compat_urllib_parse_urlencode(query)
 
         entries = []
         for i in itertools.count():
@@ -424,7 +423,7 @@ class SoundcloudUserIE(SoundcloudIE):
             qs = compat_urlparse.parse_qs(parsed_next_href.query)
             qs.update(COMMON_QUERY)
             next_href = compat_urlparse.urlunparse(
-                parsed_next_href._replace(query=compat_urllib_parse.urlencode(qs, True)))
+                parsed_next_href._replace(query=compat_urllib_parse_urlencode(qs, True)))
 
         return {
             '_type': 'playlist',
@@ -460,7 +459,7 @@ class SoundcloudPlaylistIE(SoundcloudIE):
         if token:
             data_dict['secret_token'] = token
 
-        data = compat_urllib_parse.urlencode(data_dict)
+        data = compat_urllib_parse_urlencode(data_dict)
         data = self._download_json(
             base_url + data, playlist_id, 'Downloading playlist')
 
@@ -500,7 +499,8 @@ class SoundcloudSearchIE(SearchInfoExtractor, SoundcloudIE):
         query['client_id'] = self._CLIENT_ID
         query['linked_partitioning'] = '1'
         query['offset'] = 0
-        data = compat_urllib_parse.urlencode(encode_dict(query))
+        data = compat_urllib_parse_urlencode(query)
+        data = compat_urllib_parse_urlencode(query)
         next_url = '{0}{1}?{2}'.format(self._API_V2_BASE, endpoint, data)
 
         collected_results = 0
diff --git a/youtube_dl/extractor/streamcloud.py b/youtube_dl/extractor/streamcloud.py
index 77841b946..b17779e4b 100644
--- a/youtube_dl/extractor/streamcloud.py
+++ b/youtube_dl/extractor/streamcloud.py
@@ -4,7 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse
+from ..compat import compat_urllib_parse_urlencode
 from ..utils import sanitized_Request
 
 
@@ -35,7 +35,7 @@ class StreamcloudIE(InfoExtractor):
             (?:id="[^"]+"\s+)?
             value="([^"]*)"
             ''', orig_webpage)
-        post = compat_urllib_parse.urlencode(fields)
+        post = compat_urllib_parse_urlencode(fields)
 
         self._sleep(12, video_id)
         headers = {
diff --git a/youtube_dl/extractor/telecinco.py b/youtube_dl/extractor/telecinco.py
index 2c8e9b941..d6b2560f8 100644
--- a/youtube_dl/extractor/telecinco.py
+++ b/youtube_dl/extractor/telecinco.py
@@ -5,8 +5,8 @@ import json
 
 from .common import InfoExtractor
 from ..compat import (
-    compat_urllib_parse,
     compat_urllib_parse_unquote,
+    compat_urllib_parse_urlencode,
     compat_urlparse,
 )
 from ..utils import (
@@ -74,7 +74,7 @@ class TelecincoIE(InfoExtractor):
         info_el = self._download_xml(info_url, episode).find('./video/info')
 
         video_link = info_el.find('videoUrl/link').text
-        token_query = compat_urllib_parse.urlencode({'id': video_link})
+        token_query = compat_urllib_parse_urlencode({'id': video_link})
         token_info = self._download_json(
             embed_data['flashvars']['ov_tk'] + '?' + token_query,
             episode,
diff --git a/youtube_dl/extractor/tubitv.py b/youtube_dl/extractor/tubitv.py
index 6d78b5dfe..50ed15163 100644
--- a/youtube_dl/extractor/tubitv.py
+++ b/youtube_dl/extractor/tubitv.py
@@ -5,7 +5,7 @@ import codecs
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse
+from ..compat import compat_urllib_parse_urlencode
 from ..utils import (
     ExtractorError,
     int_or_none,
@@ -41,7 +41,7 @@ class TubiTvIE(InfoExtractor):
             'username': username,
             'password': password,
         }
-        payload = compat_urllib_parse.urlencode(form_data).encode('utf-8')
+        payload = compat_urllib_parse_urlencode(form_data).encode('utf-8')
         request = sanitized_Request(self._LOGIN_URL, payload)
         request.add_header('Content-Type', 'application/x-www-form-urlencoded')
         login_page = self._download_webpage(
diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index d4169ec6d..c92dcc7b9 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -9,12 +9,11 @@ from .common import InfoExtractor
 from ..compat import (
     compat_parse_qs,
     compat_str,
-    compat_urllib_parse,
+    compat_urllib_parse_urlencode,
     compat_urllib_parse_urlparse,
     compat_urlparse,
 )
 from ..utils import (
-    encode_dict,
     ExtractorError,
     int_or_none,
     orderedSet,
@@ -82,7 +81,7 @@ class TwitchBaseIE(InfoExtractor):
             post_url = compat_urlparse.urljoin(redirect_url, post_url)
 
         request = sanitized_Request(
-            post_url, compat_urllib_parse.urlencode(encode_dict(login_form)).encode('utf-8'))
+            post_url, compat_urllib_parse_urlencode(login_form).encode('utf-8'))
         request.add_header('Referer', redirect_url)
         response = self._download_webpage(
             request, None, 'Logging in as %s' % username)
@@ -250,7 +249,7 @@ class TwitchVodIE(TwitchItemBaseIE):
         formats = self._extract_m3u8_formats(
             '%s/vod/%s?%s' % (
                 self._USHER_BASE, item_id,
-                compat_urllib_parse.urlencode({
+                compat_urllib_parse_urlencode({
                     'allow_source': 'true',
                     'allow_audio_only': 'true',
                     'allow_spectre': 'true',
@@ -442,7 +441,7 @@ class TwitchStreamIE(TwitchBaseIE):
         }
         formats = self._extract_m3u8_formats(
             '%s/api/channel/hls/%s.m3u8?%s'
-            % (self._USHER_BASE, channel_id, compat_urllib_parse.urlencode(query)),
+            % (self._USHER_BASE, channel_id, compat_urllib_parse_urlencode(query)),
             channel_id, 'mp4')
         self._prefer_source(formats)
 
diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index a9046b865..6adfb2cee 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -3,7 +3,7 @@ from __future__ import unicode_literals
 from .common import InfoExtractor
 from ..compat import (
     compat_HTTPError,
-    compat_urllib_parse,
+    compat_urllib_parse_urlencode,
     compat_urllib_request,
     compat_urlparse,
 )
@@ -71,7 +71,7 @@ class UdemyIE(InfoExtractor):
     def _download_lecture(self, course_id, lecture_id):
         return self._download_json(
             'https://www.udemy.com/api-2.0/users/me/subscribed-courses/%s/lectures/%s?%s' % (
-                course_id, lecture_id, compat_urllib_parse.urlencode({
+                course_id, lecture_id, compat_urllib_parse_urlencode({
                     'video_only': '',
                     'auto_play': '',
                     'fields[lecture]': 'title,description,asset',
@@ -139,7 +139,7 @@ class UdemyIE(InfoExtractor):
         })
 
         request = sanitized_Request(
-            self._LOGIN_URL, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
+            self._LOGIN_URL, compat_urllib_parse_urlencode(login_form).encode('utf-8'))
         request.add_header('Referer', self._ORIGIN_URL)
         request.add_header('Origin', self._ORIGIN_URL)
 
diff --git a/youtube_dl/extractor/vbox7.py b/youtube_dl/extractor/vbox7.py
index b755dda90..77bb200e9 100644
--- a/youtube_dl/extractor/vbox7.py
+++ b/youtube_dl/extractor/vbox7.py
@@ -3,7 +3,7 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..compat import (
-    compat_urllib_parse,
+    compat_urllib_parse_urlencode,
     compat_urlparse,
 )
 from ..utils import (
@@ -48,7 +48,7 @@ class Vbox7IE(InfoExtractor):
                                         webpage, 'title').split('/')[0].strip()
 
         info_url = 'http://vbox7.com/play/magare.do'
-        data = compat_urllib_parse.urlencode({'as3': '1', 'vid': video_id})
+        data = compat_urllib_parse_urlencode({'as3': '1', 'vid': video_id})
         info_request = sanitized_Request(info_url, data)
         info_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
         info_response = self._download_webpage(info_request, video_id, 'Downloading info webpage')
diff --git a/youtube_dl/extractor/viddler.py b/youtube_dl/extractor/viddler.py
index 6bfbd4d85..8d92aee87 100644
--- a/youtube_dl/extractor/viddler.py
+++ b/youtube_dl/extractor/viddler.py
@@ -2,7 +2,7 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..compat import (
-    compat_urllib_parse,
+    compat_urllib_parse_urlencode,
     compat_urlparse,
 )
 from ..utils import (
@@ -93,7 +93,7 @@ class ViddlerIE(InfoExtractor):
         headers = {'Referer': 'http://static.cdn-ec.viddler.com/js/arpeggio/v2/embed.html'}
         request = sanitized_Request(
             'http://api.viddler.com/api/v2/viddler.videos.getPlaybackDetails.json?%s'
-            % compat_urllib_parse.urlencode(query), None, headers)
+            % compat_urllib_parse_urlencode(query), None, headers)
         data = self._download_json(request, video_id)['video']
 
         formats = []
diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 71c30d2cd..707a5735a 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -12,7 +12,6 @@ from ..compat import (
 )
 from ..utils import (
     determine_ext,
-    encode_dict,
     ExtractorError,
     InAdvancePagedList,
     int_or_none,
@@ -42,13 +41,13 @@ class VimeoBaseInfoExtractor(InfoExtractor):
         self.report_login()
         webpage = self._download_webpage(self._LOGIN_URL, None, False)
         token, vuid = self._extract_xsrft_and_vuid(webpage)
-        data = urlencode_postdata(encode_dict({
+        data = urlencode_postdata({
             'action': 'login',
             'email': username,
             'password': password,
             'service': 'vimeo',
             'token': token,
-        }))
+        })
         login_request = sanitized_Request(self._LOGIN_URL, data)
         login_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
         login_request.add_header('Referer', self._LOGIN_URL)
@@ -255,10 +254,10 @@ class VimeoIE(VimeoBaseInfoExtractor):
         if password is None:
             raise ExtractorError('This video is protected by a password, use the --video-password option', expected=True)
         token, vuid = self._extract_xsrft_and_vuid(webpage)
-        data = urlencode_postdata(encode_dict({
+        data = urlencode_postdata({
             'password': password,
             'token': token,
-        }))
+        })
         if url.startswith('http://'):
             # vimeo only supports https now, but the user can give an http url
             url = url.replace('http://', 'https://')
@@ -274,7 +273,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
         password = self._downloader.params.get('videopassword')
         if password is None:
             raise ExtractorError('This video is protected by a password, use the --video-password option')
-        data = urlencode_postdata(encode_dict({'password': password}))
+        data = urlencode_postdata({'password': password})
         pass_url = url + '/check-password'
         password_request = sanitized_Request(pass_url, data)
         password_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
@@ -575,7 +574,7 @@ class VimeoChannelIE(VimeoBaseInfoExtractor):
         token, vuid = self._extract_xsrft_and_vuid(webpage)
         fields['token'] = token
         fields['password'] = password
-        post = urlencode_postdata(encode_dict(fields))
+        post = urlencode_postdata(fields)
         password_path = self._search_regex(
             r'action="([^"]+)"', login_form, 'password URL')
         password_url = compat_urlparse.urljoin(page_url, password_path)
diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index d560a4b5e..458099a4a 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -7,7 +7,7 @@ import json
 from .common import InfoExtractor
 from ..compat import (
     compat_str,
-    compat_urllib_parse,
+    compat_urllib_parse_urlencode,
 )
 from ..utils import (
     ExtractorError,
@@ -204,7 +204,7 @@ class VKIE(InfoExtractor):
 
         request = sanitized_Request(
             'https://login.vk.com/?act=login',
-            compat_urllib_parse.urlencode(login_form).encode('utf-8'))
+            compat_urllib_parse_urlencode(login_form).encode('utf-8'))
         login_page = self._download_webpage(
             request, None, note='Logging in as %s' % username)
 
diff --git a/youtube_dl/extractor/vlive.py b/youtube_dl/extractor/vlive.py
index bd5545173..baf39bb2c 100644
--- a/youtube_dl/extractor/vlive.py
+++ b/youtube_dl/extractor/vlive.py
@@ -7,7 +7,7 @@ from ..utils import (
     float_or_none,
     int_or_none,
 )
-from ..compat import compat_urllib_parse
+from ..compat import compat_urllib_parse_urlencode
 
 
 class VLiveIE(InfoExtractor):
@@ -43,7 +43,7 @@ class VLiveIE(InfoExtractor):
 
         playinfo = self._download_json(
             'http://global.apis.naver.com/rmcnmv/rmcnmv/vod_play_videoInfo.json?%s'
-            % compat_urllib_parse.urlencode({
+            % compat_urllib_parse_urlencode({
                 'videoId': long_video_id,
                 'key': key,
                 'ptc': 'http',
diff --git a/youtube_dl/extractor/vodlocker.py b/youtube_dl/extractor/vodlocker.py
index a97995a6d..f1abca4d9 100644
--- a/youtube_dl/extractor/vodlocker.py
+++ b/youtube_dl/extractor/vodlocker.py
@@ -2,7 +2,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse
+from ..compat import compat_urllib_parse_urlencode
 from ..utils import (
     ExtractorError,
     NO_DEFAULT,
@@ -38,7 +38,7 @@ class VodlockerIE(InfoExtractor):
 
         if fields['op'] == 'download1':
             self._sleep(3, video_id)  # they do detect when requests happen too fast!
-            post = compat_urllib_parse.urlencode(fields)
+            post = compat_urllib_parse_urlencode(fields)
             req = sanitized_Request(url, post)
             req.add_header('Content-type', 'application/x-www-form-urlencoded')
             webpage = self._download_webpage(
diff --git a/youtube_dl/extractor/xfileshare.py b/youtube_dl/extractor/xfileshare.py
index 94abdb4f3..4e35e1f44 100644
--- a/youtube_dl/extractor/xfileshare.py
+++ b/youtube_dl/extractor/xfileshare.py
@@ -4,10 +4,9 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse
+from ..compat import compat_urllib_parse_urlencode
 from ..utils import (
     ExtractorError,
-    encode_dict,
     int_or_none,
     sanitized_Request,
 )
@@ -109,7 +108,7 @@ class XFileShareIE(InfoExtractor):
             if countdown:
                 self._sleep(countdown, video_id)
 
-            post = compat_urllib_parse.urlencode(encode_dict(fields))
+            post = compat_urllib_parse_urlencode(fields)
 
             req = sanitized_Request(url, post)
             req.add_header('Content-type', 'application/x-www-form-urlencoded')
diff --git a/youtube_dl/extractor/yahoo.py b/youtube_dl/extractor/yahoo.py
index 4c6142927..b2d8f4b48 100644
--- a/youtube_dl/extractor/yahoo.py
+++ b/youtube_dl/extractor/yahoo.py
@@ -8,6 +8,7 @@ import re
 from .common import InfoExtractor, SearchInfoExtractor
 from ..compat import (
     compat_urllib_parse,
+    compat_urllib_parse_urlencode,
     compat_urlparse,
 )
 from ..utils import (
@@ -303,7 +304,7 @@ class YahooIE(InfoExtractor):
         region = self._search_regex(
             r'\\?"region\\?"\s*:\s*\\?"([^"]+?)\\?"',
             webpage, 'region', fatal=False, default='US')
-        data = compat_urllib_parse.urlencode({
+        data = compat_urllib_parse_urlencode({
             'protocol': 'http',
             'region': region,
         })
diff --git a/youtube_dl/extractor/yandexmusic.py b/youtube_dl/extractor/yandexmusic.py
index e699e663f..158f3ea68 100644
--- a/youtube_dl/extractor/yandexmusic.py
+++ b/youtube_dl/extractor/yandexmusic.py
@@ -7,7 +7,7 @@ import hashlib
 from .common import InfoExtractor
 from ..compat import (
     compat_str,
-    compat_urllib_parse,
+    compat_urllib_parse_urlencode,
 )
 from ..utils import (
     ExtractorError,
@@ -170,7 +170,7 @@ class YandexMusicPlaylistIE(YandexMusicPlaylistBaseIE):
             missing_track_ids = set(map(compat_str, track_ids)) - set(present_track_ids)
             request = sanitized_Request(
                 'https://music.yandex.ru/handlers/track-entries.jsx',
-                compat_urllib_parse.urlencode({
+                compat_urllib_parse_urlencode({
                     'entries': ','.join(missing_track_ids),
                     'lang': mu.get('settings', {}).get('lang', 'en'),
                     'external-domain': 'music.yandex.ru',
diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index 900eb2aba..fd7eb5a6d 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -8,7 +8,7 @@ import time
 
 from .common import InfoExtractor
 from ..compat import (
-    compat_urllib_parse,
+    compat_urllib_parse_urlencode,
     compat_ord,
 )
 from ..utils import (
@@ -138,7 +138,7 @@ class YoukuIE(InfoExtractor):
                     '_00' + \
                     '/st/' + self.parse_ext_l(format) + \
                     '/fileid/' + get_fileid(format, n) + '?' + \
-                    compat_urllib_parse.urlencode(param)
+                    compat_urllib_parse_urlencode(param)
                 video_urls.append(video_url)
             video_urls_dict[format] = video_urls
 
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 96fa3b5aa..83b5840f7 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -17,16 +17,15 @@ from ..swfinterp import SWFInterpreter
 from ..compat import (
     compat_chr,
     compat_parse_qs,
-    compat_urllib_parse,
     compat_urllib_parse_unquote,
     compat_urllib_parse_unquote_plus,
+    compat_urllib_parse_urlencode,
     compat_urllib_parse_urlparse,
     compat_urlparse,
     compat_str,
 )
 from ..utils import (
     clean_html,
-    encode_dict,
     error_to_compat_str,
     ExtractorError,
     float_or_none,
@@ -116,7 +115,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
             'hl': 'en_US',
         }
 
-        login_data = compat_urllib_parse.urlencode(encode_dict(login_form_strs)).encode('ascii')
+        login_data = compat_urllib_parse_urlencode(login_form_strs).encode('ascii')
 
         req = sanitized_Request(self._LOGIN_URL, login_data)
         login_results = self._download_webpage(
@@ -149,7 +148,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
                 'TrustDevice': 'on',
             })
 
-            tfa_data = compat_urllib_parse.urlencode(encode_dict(tfa_form_strs)).encode('ascii')
+            tfa_data = compat_urllib_parse_urlencode(tfa_form_strs).encode('ascii')
 
             tfa_req = sanitized_Request(self._TWOFACTOR_URL, tfa_data)
             tfa_results = self._download_webpage(
@@ -1007,7 +1006,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 continue
             sub_formats = []
             for ext in self._SUBTITLE_FORMATS:
-                params = compat_urllib_parse.urlencode({
+                params = compat_urllib_parse_urlencode({
                     'lang': lang,
                     'v': video_id,
                     'fmt': ext,
@@ -1056,7 +1055,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             if caption_url:
                 timestamp = args['timestamp']
                 # We get the available subtitles
-                list_params = compat_urllib_parse.urlencode({
+                list_params = compat_urllib_parse_urlencode({
                     'type': 'list',
                     'tlangs': 1,
                     'asrs': 1,
@@ -1075,7 +1074,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     sub_lang = lang_node.attrib['lang_code']
                     sub_formats = []
                     for ext in self._SUBTITLE_FORMATS:
-                        params = compat_urllib_parse.urlencode({
+                        params = compat_urllib_parse_urlencode({
                             'lang': original_lang,
                             'tlang': sub_lang,
                             'fmt': ext,
@@ -1094,7 +1093,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             caption_tracks = args['caption_tracks']
             caption_translation_languages = args['caption_translation_languages']
             caption_url = compat_parse_qs(caption_tracks.split(',')[0])['u'][0]
-            parsed_caption_url = compat_urlparse.urlparse(caption_url)
+            parsed_caption_url = compat_urllib_parse_urlparse(caption_url)
             caption_qs = compat_parse_qs(parsed_caption_url.query)
 
             sub_lang_list = {}
@@ -1110,7 +1109,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                         'fmt': [ext],
                     })
                     sub_url = compat_urlparse.urlunparse(parsed_caption_url._replace(
-                        query=compat_urllib_parse.urlencode(caption_qs, True)))
+                        query=compat_urllib_parse_urlencode(caption_qs, True)))
                     sub_formats.append({
                         'url': sub_url,
                         'ext': ext,
@@ -1140,7 +1139,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             'cpn': [cpn],
         })
         playback_url = compat_urlparse.urlunparse(
-            parsed_playback_url._replace(query=compat_urllib_parse.urlencode(qs, True)))
+            parsed_playback_url._replace(query=compat_urllib_parse_urlencode(qs, True)))
 
         self._download_webpage(
             playback_url, video_id, 'Marking watched',
@@ -1225,7 +1224,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             # this can be viewed without login into Youtube
             url = proto + '://www.youtube.com/embed/%s' % video_id
             embed_webpage = self._download_webpage(url, video_id, 'Downloading embed webpage')
-            data = compat_urllib_parse.urlencode({
+            data = compat_urllib_parse_urlencode({
                 'video_id': video_id,
                 'eurl': 'https://youtube.googleapis.com/v/' + video_id,
                 'sts': self._search_regex(
@@ -2085,7 +2084,7 @@ class YoutubeSearchIE(SearchInfoExtractor, YoutubePlaylistIE):
                 'spf': 'navigate',
             }
             url_query.update(self._EXTRA_QUERY_ARGS)
-            result_url = 'https://www.youtube.com/results?' + compat_urllib_parse.urlencode(url_query)
+            result_url = 'https://www.youtube.com/results?' + compat_urllib_parse_urlencode(url_query)
             data = self._download_json(
                 result_url, video_id='query "%s"' % query,
                 note='Downloading page %s' % pagenum,
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index b6e1dc809..eacd81bf9 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -47,6 +47,7 @@ from .compat import (
     compat_str,
     compat_urllib_error,
     compat_urllib_parse,
+    compat_urllib_parse_urlencode,
     compat_urllib_parse_urlparse,
     compat_urllib_request,
     compat_urlparse,
@@ -1315,7 +1316,7 @@ def shell_quote(args):
 def smuggle_url(url, data):
     """ Pass additional data in a URL for internal use. """
 
-    sdata = compat_urllib_parse.urlencode(
+    sdata = compat_urllib_parse_urlencode(
         {'__youtubedl_smuggle': json.dumps(data)})
     return url + '#' + sdata
 
@@ -1789,22 +1790,15 @@ def read_batch_urls(batch_fd):
 
 
 def urlencode_postdata(*args, **kargs):
-    return compat_urllib_parse.urlencode(*args, **kargs).encode('ascii')
+    return compat_urllib_parse_urlencode(*args, **kargs).encode('ascii')
 
 
 def update_url_query(url, query):
     parsed_url = compat_urlparse.urlparse(url)
     qs = compat_parse_qs(parsed_url.query)
     qs.update(query)
-    qs = encode_dict(qs)
     return compat_urlparse.urlunparse(parsed_url._replace(
-        query=compat_urllib_parse.urlencode(qs, True)))
-
-
-def encode_dict(d, encoding='utf-8'):
-    def encode(v):
-        return v.encode(encoding) if isinstance(v, compat_basestring) else v
-    return dict((encode(k), encode(v)) for k, v in d.items())
+        query=compat_urllib_parse_urlencode(qs, True)))
 
 
 def dict_get(d, key_or_keys, default=None, skip_false_values=True):

From 6e6bc8dae577c29c072ffc5c25078b5668435435 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 26 Mar 2016 02:19:24 +0600
Subject: [PATCH 4422/6219] Use urlencode_postdata across the codebase

---
 youtube_dl/extractor/atresplayer.py   | 16 +++++++---------
 youtube_dl/extractor/bambuser.py      | 10 ++++------
 youtube_dl/extractor/ceskatelevize.py |  4 ++--
 youtube_dl/extractor/crunchyroll.py   |  2 +-
 youtube_dl/extractor/dcn.py           |  3 ++-
 youtube_dl/extractor/dramafever.py    |  4 ++--
 youtube_dl/extractor/fc2.py           |  4 ++--
 youtube_dl/extractor/gdcvault.py      |  4 ++--
 youtube_dl/extractor/hotnewhiphop.py  |  4 ++--
 youtube_dl/extractor/lynda.py         | 10 ++++------
 youtube_dl/extractor/metacafe.py      |  4 ++--
 youtube_dl/extractor/minhateca.py     |  4 ++--
 youtube_dl/extractor/moevideo.py      |  4 ++--
 youtube_dl/extractor/moniker.py       |  4 ++--
 youtube_dl/extractor/mooshare.py      |  4 ++--
 youtube_dl/extractor/nfb.py           |  8 +++++---
 youtube_dl/extractor/niconico.py      |  3 ++-
 youtube_dl/extractor/noco.py          |  4 ++--
 youtube_dl/extractor/played.py        |  4 ++--
 youtube_dl/extractor/pluralsight.py   |  4 ++--
 youtube_dl/extractor/primesharetv.py  |  4 ++--
 youtube_dl/extractor/promptfile.py    |  4 ++--
 youtube_dl/extractor/shared.py        |  4 ++--
 youtube_dl/extractor/sharesix.py      |  4 ++--
 youtube_dl/extractor/smotri.py        |  6 +++---
 youtube_dl/extractor/soundcloud.py    |  1 -
 youtube_dl/extractor/streamcloud.py   |  8 +++++---
 youtube_dl/extractor/tubitv.py        |  4 ++--
 youtube_dl/extractor/twitch.py        |  3 ++-
 youtube_dl/extractor/udemy.py         |  3 ++-
 youtube_dl/extractor/vbox7.py         |  8 +++-----
 youtube_dl/extractor/vk.py            |  8 +++-----
 youtube_dl/extractor/vodlocker.py     |  4 ++--
 youtube_dl/extractor/xfileshare.py    |  4 ++--
 youtube_dl/extractor/yandexmusic.py   | 10 ++++------
 youtube_dl/extractor/youtube.py       |  5 +++--
 36 files changed, 90 insertions(+), 94 deletions(-)

diff --git a/youtube_dl/extractor/atresplayer.py b/youtube_dl/extractor/atresplayer.py
index f9568cb5b..d2f388964 100644
--- a/youtube_dl/extractor/atresplayer.py
+++ b/youtube_dl/extractor/atresplayer.py
@@ -6,16 +6,14 @@ import hashlib
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_str,
-    compat_urllib_parse_urlencode,
-)
+from ..compat import compat_str
 from ..utils import (
-    int_or_none,
-    float_or_none,
-    sanitized_Request,
-    xpath_text,
     ExtractorError,
+    float_or_none,
+    int_or_none,
+    sanitized_Request,
+    urlencode_postdata,
+    xpath_text,
 )
 
 
@@ -86,7 +84,7 @@ class AtresPlayerIE(InfoExtractor):
         }
 
         request = sanitized_Request(
-            self._LOGIN_URL, compat_urllib_parse_urlencode(login_form).encode('utf-8'))
+            self._LOGIN_URL, urlencode_postdata(login_form))
         request.add_header('Content-Type', 'application/x-www-form-urlencoded')
         response = self._download_webpage(
             request, None, 'Logging in as %s' % username)
diff --git a/youtube_dl/extractor/bambuser.py b/youtube_dl/extractor/bambuser.py
index 1a2eef48d..0eb1930c2 100644
--- a/youtube_dl/extractor/bambuser.py
+++ b/youtube_dl/extractor/bambuser.py
@@ -4,15 +4,13 @@ import re
 import itertools
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse_urlencode,
-    compat_str,
-)
+from ..compat import compat_str
 from ..utils import (
     ExtractorError,
-    int_or_none,
     float_or_none,
+    int_or_none,
     sanitized_Request,
+    urlencode_postdata,
 )
 
 
@@ -58,7 +56,7 @@ class BambuserIE(InfoExtractor):
         }
 
         request = sanitized_Request(
-            self._LOGIN_URL, compat_urllib_parse_urlencode(login_form).encode('utf-8'))
+            self._LOGIN_URL, urlencode_postdata(login_form))
         request.add_header('Referer', self._LOGIN_URL)
         response = self._download_webpage(
             request, None, 'Logging in as %s' % username)
diff --git a/youtube_dl/extractor/ceskatelevize.py b/youtube_dl/extractor/ceskatelevize.py
index d93108df5..6652c8e42 100644
--- a/youtube_dl/extractor/ceskatelevize.py
+++ b/youtube_dl/extractor/ceskatelevize.py
@@ -6,13 +6,13 @@ import re
 from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse_unquote,
-    compat_urllib_parse_urlencode,
     compat_urllib_parse_urlparse,
 )
 from ..utils import (
     ExtractorError,
     float_or_none,
     sanitized_Request,
+    urlencode_postdata,
 )
 
 
@@ -102,7 +102,7 @@ class CeskaTelevizeIE(InfoExtractor):
 
         req = sanitized_Request(
             'http://www.ceskatelevize.cz/ivysilani/ajax/get-client-playlist',
-            data=compat_urllib_parse_urlencode(data))
+            data=urlencode_postdata(data))
 
         req.add_header('Content-type', 'application/x-www-form-urlencoded')
         req.add_header('x-addr', '127.0.0.1')
diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 7746f1be3..8ae3f2890 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -308,7 +308,7 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
 
         playerdata_url = compat_urllib_parse_unquote(self._html_search_regex(r'"config_url":"([^"]+)', webpage, 'playerdata_url'))
         playerdata_req = sanitized_Request(playerdata_url)
-        playerdata_req.data = compat_urllib_parse_urlencode({'current_page': webpage_url})
+        playerdata_req.data = urlencode_postdata({'current_page': webpage_url})
         playerdata_req.add_header('Content-Type', 'application/x-www-form-urlencoded')
         playerdata = self._download_webpage(playerdata_req, video_id, note='Downloading media info')
 
diff --git a/youtube_dl/extractor/dcn.py b/youtube_dl/extractor/dcn.py
index 982ed94ea..5deff5f30 100644
--- a/youtube_dl/extractor/dcn.py
+++ b/youtube_dl/extractor/dcn.py
@@ -15,6 +15,7 @@ from ..utils import (
     sanitized_Request,
     smuggle_url,
     unsmuggle_url,
+    urlencode_postdata,
 )
 
 
@@ -174,7 +175,7 @@ class DCNSeasonIE(InfoExtractor):
         data['show_id'] = show_id
         request = sanitized_Request(
             'http://admin.mangomolo.com/analytics/index.php/plus/show',
-            compat_urllib_parse_urlencode(data),
+            urlencode_postdata(data),
             {
                 'Origin': 'http://www.dcndigital.ae',
                 'Content-Type': 'application/x-www-form-urlencoded'
diff --git a/youtube_dl/extractor/dramafever.py b/youtube_dl/extractor/dramafever.py
index 2101acaaf..3b6529f4b 100644
--- a/youtube_dl/extractor/dramafever.py
+++ b/youtube_dl/extractor/dramafever.py
@@ -6,7 +6,6 @@ import itertools
 from .amp import AMPIE
 from ..compat import (
     compat_HTTPError,
-    compat_urllib_parse_urlencode,
     compat_urlparse,
 )
 from ..utils import (
@@ -14,6 +13,7 @@ from ..utils import (
     clean_html,
     int_or_none,
     sanitized_Request,
+    urlencode_postdata
 )
 
 
@@ -50,7 +50,7 @@ class DramaFeverBaseIE(AMPIE):
         }
 
         request = sanitized_Request(
-            self._LOGIN_URL, compat_urllib_parse_urlencode(login_form).encode('utf-8'))
+            self._LOGIN_URL, urlencode_postdata(login_form))
         response = self._download_webpage(
             request, None, 'Logging in as %s' % username)
 
diff --git a/youtube_dl/extractor/fc2.py b/youtube_dl/extractor/fc2.py
index cacf61973..c7d69ff1f 100644
--- a/youtube_dl/extractor/fc2.py
+++ b/youtube_dl/extractor/fc2.py
@@ -5,13 +5,13 @@ import hashlib
 
 from .common import InfoExtractor
 from ..compat import (
-    compat_urllib_parse_urlencode,
     compat_urllib_request,
     compat_urlparse,
 )
 from ..utils import (
     ExtractorError,
     sanitized_Request,
+    urlencode_postdata,
 )
 
 
@@ -56,7 +56,7 @@ class FC2IE(InfoExtractor):
             'Submit': ' Login ',
         }
 
-        login_data = compat_urllib_parse_urlencode(login_form_strs).encode('utf-8')
+        login_data = urlencode_postdata(login_form_strs)
         request = sanitized_Request(
             'https://secure.id.fc2.com/index.php?mode=login&switch_language=en', login_data)
 
diff --git a/youtube_dl/extractor/gdcvault.py b/youtube_dl/extractor/gdcvault.py
index cc8fa45d2..59ed4c38f 100644
--- a/youtube_dl/extractor/gdcvault.py
+++ b/youtube_dl/extractor/gdcvault.py
@@ -3,11 +3,11 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse_urlencode
 from ..utils import (
     remove_end,
     HEADRequest,
     sanitized_Request,
+    urlencode_postdata,
 )
 
 
@@ -123,7 +123,7 @@ class GDCVaultIE(InfoExtractor):
             'password': password,
         }
 
-        request = sanitized_Request(login_url, compat_urllib_parse_urlencode(login_form))
+        request = sanitized_Request(login_url, urlencode_postdata(login_form))
         request.add_header('Content-Type', 'application/x-www-form-urlencoded')
         self._download_webpage(request, display_id, 'Logging in')
         start_page = self._download_webpage(webpage_url, display_id, 'Getting authenticated video page')
diff --git a/youtube_dl/extractor/hotnewhiphop.py b/youtube_dl/extractor/hotnewhiphop.py
index 152d2a98a..9db565209 100644
--- a/youtube_dl/extractor/hotnewhiphop.py
+++ b/youtube_dl/extractor/hotnewhiphop.py
@@ -3,11 +3,11 @@ from __future__ import unicode_literals
 import base64
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse_urlencode
 from ..utils import (
     ExtractorError,
     HEADRequest,
     sanitized_Request,
+    urlencode_postdata,
 )
 
 
@@ -35,7 +35,7 @@ class HotNewHipHopIE(InfoExtractor):
                 r'"contentUrl" content="(.*?)"', webpage, 'content URL')
             return self.url_result(video_url, ie='Youtube')
 
-        reqdata = compat_urllib_parse_urlencode([
+        reqdata = urlencode_postdata([
             ('mediaType', 's'),
             ('mediaId', video_id),
         ])
diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index df50cb655..71fd55ade 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -4,15 +4,13 @@ import re
 import json
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_str,
-    compat_urllib_parse_urlencode,
-)
+from ..compat import compat_str
 from ..utils import (
     ExtractorError,
     clean_html,
     int_or_none,
     sanitized_Request,
+    urlencode_postdata,
 )
 
 
@@ -36,7 +34,7 @@ class LyndaBaseIE(InfoExtractor):
             'stayPut': 'false'
         }
         request = sanitized_Request(
-            self._LOGIN_URL, compat_urllib_parse_urlencode(login_form).encode('utf-8'))
+            self._LOGIN_URL, urlencode_postdata(login_form))
         login_page = self._download_webpage(
             request, None, 'Logging in as %s' % username)
 
@@ -65,7 +63,7 @@ class LyndaBaseIE(InfoExtractor):
                     'stayPut': 'false',
                 }
                 request = sanitized_Request(
-                    self._LOGIN_URL, compat_urllib_parse_urlencode(confirm_form).encode('utf-8'))
+                    self._LOGIN_URL, urlencode_postdata(confirm_form))
                 login_page = self._download_webpage(
                     request, None,
                     'Confirming log in and log out from another device')
diff --git a/youtube_dl/extractor/metacafe.py b/youtube_dl/extractor/metacafe.py
index 0e4865446..61dadb7a7 100644
--- a/youtube_dl/extractor/metacafe.py
+++ b/youtube_dl/extractor/metacafe.py
@@ -6,13 +6,13 @@ from .common import InfoExtractor
 from ..compat import (
     compat_parse_qs,
     compat_urllib_parse_unquote,
-    compat_urllib_parse_urlencode,
 )
 from ..utils import (
     determine_ext,
     ExtractorError,
     int_or_none,
     sanitized_Request,
+    urlencode_postdata,
 )
 
 
@@ -117,7 +117,7 @@ class MetacafeIE(InfoExtractor):
             'filters': '0',
             'submit': "Continue - I'm over 18",
         }
-        request = sanitized_Request(self._FILTER_POST, compat_urllib_parse_urlencode(disclaimer_form))
+        request = sanitized_Request(self._FILTER_POST, urlencode_postdata(disclaimer_form))
         request.add_header('Content-Type', 'application/x-www-form-urlencoded')
         self.report_age_confirmation()
         self._download_webpage(request, None, False, 'Unable to confirm age')
diff --git a/youtube_dl/extractor/minhateca.py b/youtube_dl/extractor/minhateca.py
index 6ec53c303..e6730b75a 100644
--- a/youtube_dl/extractor/minhateca.py
+++ b/youtube_dl/extractor/minhateca.py
@@ -2,12 +2,12 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse_urlencode
 from ..utils import (
     int_or_none,
     parse_duration,
     parse_filesize,
     sanitized_Request,
+    urlencode_postdata,
 )
 
 
@@ -39,7 +39,7 @@ class MinhatecaIE(InfoExtractor):
         ]
         req = sanitized_Request(
             'http://minhateca.com.br/action/License/Download',
-            data=compat_urllib_parse_urlencode(token_data))
+            data=urlencode_postdata(token_data))
         req.add_header('Content-Type', 'application/x-www-form-urlencoded')
         data = self._download_json(
             req, video_id, note='Downloading metadata')
diff --git a/youtube_dl/extractor/moevideo.py b/youtube_dl/extractor/moevideo.py
index 89cdd4600..978d5d5bf 100644
--- a/youtube_dl/extractor/moevideo.py
+++ b/youtube_dl/extractor/moevideo.py
@@ -5,11 +5,11 @@ import json
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse_urlencode
 from ..utils import (
     ExtractorError,
     int_or_none,
     sanitized_Request,
+    urlencode_postdata,
 )
 
 
@@ -77,7 +77,7 @@ class MoeVideoIE(InfoExtractor):
             ],
         ]
         r_json = json.dumps(r)
-        post = compat_urllib_parse_urlencode({'r': r_json})
+        post = urlencode_postdata({'r': r_json})
         req = sanitized_Request(self._API_URL, post)
         req.add_header('Content-type', 'application/x-www-form-urlencoded')
 
diff --git a/youtube_dl/extractor/moniker.py b/youtube_dl/extractor/moniker.py
index c5ce693f1..b208820fe 100644
--- a/youtube_dl/extractor/moniker.py
+++ b/youtube_dl/extractor/moniker.py
@@ -5,11 +5,11 @@ import os.path
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse_urlencode
 from ..utils import (
     ExtractorError,
     remove_start,
     sanitized_Request,
+    urlencode_postdata,
 )
 
 
@@ -88,7 +88,7 @@ class MonikerIE(InfoExtractor):
             fields = re.findall(r'type="hidden" name="(.+?)"\s* value="?(.+?)">', orig_webpage)
             data = dict(fields)
 
-            post = compat_urllib_parse_urlencode(data)
+            post = urlencode_postdata(data)
             headers = {
                 b'Content-Type': b'application/x-www-form-urlencoded',
             }
diff --git a/youtube_dl/extractor/mooshare.py b/youtube_dl/extractor/mooshare.py
index ee3947f43..a85109a89 100644
--- a/youtube_dl/extractor/mooshare.py
+++ b/youtube_dl/extractor/mooshare.py
@@ -3,10 +3,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse_urlencode
 from ..utils import (
     ExtractorError,
     sanitized_Request,
+    urlencode_postdata,
 )
 
 
@@ -58,7 +58,7 @@ class MooshareIE(InfoExtractor):
         }
 
         request = sanitized_Request(
-            'http://mooshare.biz/%s' % video_id, compat_urllib_parse_urlencode(download_form))
+            'http://mooshare.biz/%s' % video_id, urlencode_postdata(download_form))
         request.add_header('Content-Type', 'application/x-www-form-urlencoded')
 
         self._sleep(5, video_id)
diff --git a/youtube_dl/extractor/nfb.py b/youtube_dl/extractor/nfb.py
index ba1eefafc..51e4a34f7 100644
--- a/youtube_dl/extractor/nfb.py
+++ b/youtube_dl/extractor/nfb.py
@@ -1,8 +1,10 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse_urlencode
-from ..utils import sanitized_Request
+from ..utils import (
+    sanitized_Request,
+    urlencode_postdata,
+)
 
 
 class NFBIE(InfoExtractor):
@@ -40,7 +42,7 @@ class NFBIE(InfoExtractor):
 
         request = sanitized_Request(
             'https://www.nfb.ca/film/%s/player_config' % video_id,
-            compat_urllib_parse_urlencode({'getConfig': 'true'}).encode('ascii'))
+            urlencode_postdata({'getConfig': 'true'}))
         request.add_header('Content-Type', 'application/x-www-form-urlencoded')
         request.add_header('X-NFB-Referer', 'http://www.nfb.ca/medias/flash/NFBVideoPlayer.swf')
 
diff --git a/youtube_dl/extractor/niconico.py b/youtube_dl/extractor/niconico.py
index 688f0a124..dd75a48af 100644
--- a/youtube_dl/extractor/niconico.py
+++ b/youtube_dl/extractor/niconico.py
@@ -18,6 +18,7 @@ from ..utils import (
     sanitized_Request,
     xpath_text,
     determine_ext,
+    urlencode_postdata,
 )
 
 
@@ -100,7 +101,7 @@ class NiconicoIE(InfoExtractor):
             'mail': username,
             'password': password,
         }
-        login_data = compat_urllib_parse_urlencode(login_form_strs).encode('utf-8')
+        login_data = urlencode_postdata(login_form_strs)
         request = sanitized_Request(
             'https://secure.nicovideo.jp/secure/login', login_data)
         login_results = self._download_webpage(
diff --git a/youtube_dl/extractor/noco.py b/youtube_dl/extractor/noco.py
index 8f4b69a6f..06f2bda07 100644
--- a/youtube_dl/extractor/noco.py
+++ b/youtube_dl/extractor/noco.py
@@ -8,7 +8,6 @@ import hashlib
 from .common import InfoExtractor
 from ..compat import (
     compat_str,
-    compat_urllib_parse_urlencode,
     compat_urlparse,
 )
 from ..utils import (
@@ -18,6 +17,7 @@ from ..utils import (
     float_or_none,
     parse_iso8601,
     sanitized_Request,
+    urlencode_postdata,
 )
 
 
@@ -75,7 +75,7 @@ class NocoIE(InfoExtractor):
             'username': username,
             'password': password,
         }
-        request = sanitized_Request(self._LOGIN_URL, compat_urllib_parse_urlencode(login_form))
+        request = sanitized_Request(self._LOGIN_URL, urlencode_postdata(login_form))
         request.add_header('Content-Type', 'application/x-www-form-urlencoded; charset=UTF-8')
 
         login = self._download_json(request, None, 'Logging in as %s' % username)
diff --git a/youtube_dl/extractor/played.py b/youtube_dl/extractor/played.py
index 63065622b..57c875ef0 100644
--- a/youtube_dl/extractor/played.py
+++ b/youtube_dl/extractor/played.py
@@ -5,10 +5,10 @@ import re
 import os.path
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse_urlencode
 from ..utils import (
     ExtractorError,
     sanitized_Request,
+    urlencode_postdata,
 )
 
 
@@ -40,7 +40,7 @@ class PlayedIE(InfoExtractor):
 
         self._sleep(2, video_id)
 
-        post = compat_urllib_parse_urlencode(data)
+        post = urlencode_postdata(data)
         headers = {
             b'Content-Type': b'application/x-www-form-urlencoded',
         }
diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index 575775f09..bc66f7a9d 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -8,7 +8,6 @@ import collections
 from .common import InfoExtractor
 from ..compat import (
     compat_str,
-    compat_urllib_parse_urlencode,
     compat_urlparse,
 )
 from ..utils import (
@@ -17,6 +16,7 @@ from ..utils import (
     parse_duration,
     qualities,
     sanitized_Request,
+    urlencode_postdata,
 )
 
 
@@ -76,7 +76,7 @@ class PluralsightIE(PluralsightBaseIE):
             post_url = compat_urlparse.urljoin(self._LOGIN_URL, post_url)
 
         request = sanitized_Request(
-            post_url, compat_urllib_parse_urlencode(login_form).encode('utf-8'))
+            post_url, urlencode_postdata(login_form))
         request.add_header('Content-Type', 'application/x-www-form-urlencoded')
 
         response = self._download_webpage(
diff --git a/youtube_dl/extractor/primesharetv.py b/youtube_dl/extractor/primesharetv.py
index 188f08826..0c1024772 100644
--- a/youtube_dl/extractor/primesharetv.py
+++ b/youtube_dl/extractor/primesharetv.py
@@ -1,10 +1,10 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse_urlencode
 from ..utils import (
     ExtractorError,
     sanitized_Request,
+    urlencode_postdata,
 )
 
 
@@ -42,7 +42,7 @@ class PrimeShareTVIE(InfoExtractor):
         self._sleep(wait_time, video_id)
 
         req = sanitized_Request(
-            url, compat_urllib_parse_urlencode(fields), headers)
+            url, urlencode_postdata(fields), headers)
         video_page = self._download_webpage(
             req, video_id, 'Downloading video page')
 
diff --git a/youtube_dl/extractor/promptfile.py b/youtube_dl/extractor/promptfile.py
index 67312016c..f93bd19ff 100644
--- a/youtube_dl/extractor/promptfile.py
+++ b/youtube_dl/extractor/promptfile.py
@@ -4,11 +4,11 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse_urlencode
 from ..utils import (
     determine_ext,
     ExtractorError,
     sanitized_Request,
+    urlencode_postdata,
 )
 
 
@@ -34,7 +34,7 @@ class PromptFileIE(InfoExtractor):
                                  expected=True)
 
         fields = self._hidden_inputs(webpage)
-        post = compat_urllib_parse_urlencode(fields)
+        post = urlencode_postdata(fields)
         req = sanitized_Request(url, post)
         req.add_header('Content-type', 'application/x-www-form-urlencoded')
         webpage = self._download_webpage(
diff --git a/youtube_dl/extractor/shared.py b/youtube_dl/extractor/shared.py
index e66441997..e7e5f653e 100644
--- a/youtube_dl/extractor/shared.py
+++ b/youtube_dl/extractor/shared.py
@@ -3,11 +3,11 @@ from __future__ import unicode_literals
 import base64
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse_urlencode
 from ..utils import (
     ExtractorError,
     int_or_none,
     sanitized_Request,
+    urlencode_postdata,
 )
 
 
@@ -45,7 +45,7 @@ class SharedIE(InfoExtractor):
 
         download_form = self._hidden_inputs(webpage)
         request = sanitized_Request(
-            url, compat_urllib_parse_urlencode(download_form))
+            url, urlencode_postdata(download_form))
         request.add_header('Content-Type', 'application/x-www-form-urlencoded')
 
         video_page = self._download_webpage(
diff --git a/youtube_dl/extractor/sharesix.py b/youtube_dl/extractor/sharesix.py
index 61dc1c235..9cce5ceb4 100644
--- a/youtube_dl/extractor/sharesix.py
+++ b/youtube_dl/extractor/sharesix.py
@@ -4,10 +4,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse_urlencode
 from ..utils import (
     parse_duration,
     sanitized_Request,
+    urlencode_postdata,
 )
 
 
@@ -47,7 +47,7 @@ class ShareSixIE(InfoExtractor):
         fields = {
             'method_free': 'Free'
         }
-        post = compat_urllib_parse_urlencode(fields)
+        post = urlencode_postdata(fields)
         req = sanitized_Request(url, post)
         req.add_header('Content-type', 'application/x-www-form-urlencoded')
 
diff --git a/youtube_dl/extractor/smotri.py b/youtube_dl/extractor/smotri.py
index b4c6d5bbf..5c3fd0fec 100644
--- a/youtube_dl/extractor/smotri.py
+++ b/youtube_dl/extractor/smotri.py
@@ -7,12 +7,12 @@ import hashlib
 import uuid
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse_urlencode
 from ..utils import (
     ExtractorError,
     int_or_none,
     sanitized_Request,
     unified_strdate,
+    urlencode_postdata,
 )
 
 
@@ -175,7 +175,7 @@ class SmotriIE(InfoExtractor):
             video_form['pass'] = hashlib.md5(video_password.encode('utf-8')).hexdigest()
 
         request = sanitized_Request(
-            'http://smotri.com/video/view/url/bot/', compat_urllib_parse_urlencode(video_form))
+            'http://smotri.com/video/view/url/bot/', urlencode_postdata(video_form))
         request.add_header('Content-Type', 'application/x-www-form-urlencoded')
 
         video = self._download_json(request, video_id, 'Downloading video JSON')
@@ -338,7 +338,7 @@ class SmotriBroadcastIE(InfoExtractor):
             }
 
             request = sanitized_Request(
-                broadcast_url + '/?no_redirect=1', compat_urllib_parse_urlencode(login_form))
+                broadcast_url + '/?no_redirect=1', urlencode_postdata(login_form))
             request.add_header('Content-Type', 'application/x-www-form-urlencoded')
             broadcast_page = self._download_webpage(
                 request, broadcast_id, 'Logging in and confirming age')
diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 2bca8fa3a..194dabc71 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -500,7 +500,6 @@ class SoundcloudSearchIE(SearchInfoExtractor, SoundcloudIE):
         query['linked_partitioning'] = '1'
         query['offset'] = 0
         data = compat_urllib_parse_urlencode(query)
-        data = compat_urllib_parse_urlencode(query)
         next_url = '{0}{1}?{2}'.format(self._API_V2_BASE, endpoint, data)
 
         collected_results = 0
diff --git a/youtube_dl/extractor/streamcloud.py b/youtube_dl/extractor/streamcloud.py
index b17779e4b..712359885 100644
--- a/youtube_dl/extractor/streamcloud.py
+++ b/youtube_dl/extractor/streamcloud.py
@@ -4,8 +4,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse_urlencode
-from ..utils import sanitized_Request
+from ..utils import (
+    sanitized_Request,
+    urlencode_postdata,
+)
 
 
 class StreamcloudIE(InfoExtractor):
@@ -35,7 +37,7 @@ class StreamcloudIE(InfoExtractor):
             (?:id="[^"]+"\s+)?
             value="([^"]*)"
             ''', orig_webpage)
-        post = compat_urllib_parse_urlencode(fields)
+        post = urlencode_postdata(fields)
 
         self._sleep(12, video_id)
         headers = {
diff --git a/youtube_dl/extractor/tubitv.py b/youtube_dl/extractor/tubitv.py
index 50ed15163..306ee4e15 100644
--- a/youtube_dl/extractor/tubitv.py
+++ b/youtube_dl/extractor/tubitv.py
@@ -5,11 +5,11 @@ import codecs
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse_urlencode
 from ..utils import (
     ExtractorError,
     int_or_none,
     sanitized_Request,
+    urlencode_postdata,
 )
 
 
@@ -41,7 +41,7 @@ class TubiTvIE(InfoExtractor):
             'username': username,
             'password': password,
         }
-        payload = compat_urllib_parse_urlencode(form_data).encode('utf-8')
+        payload = urlencode_postdata(form_data)
         request = sanitized_Request(self._LOGIN_URL, payload)
         request.add_header('Content-Type', 'application/x-www-form-urlencoded')
         login_page = self._download_webpage(
diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index c92dcc7b9..36ee1adff 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -20,6 +20,7 @@ from ..utils import (
     parse_duration,
     parse_iso8601,
     sanitized_Request,
+    urlencode_postdata,
 )
 
 
@@ -81,7 +82,7 @@ class TwitchBaseIE(InfoExtractor):
             post_url = compat_urlparse.urljoin(redirect_url, post_url)
 
         request = sanitized_Request(
-            post_url, compat_urllib_parse_urlencode(login_form).encode('utf-8'))
+            post_url, urlencode_postdata(login_form))
         request.add_header('Referer', redirect_url)
         response = self._download_webpage(
             request, None, 'Logging in as %s' % username)
diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index 6adfb2cee..be6f3be5e 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -13,6 +13,7 @@ from ..utils import (
     int_or_none,
     sanitized_Request,
     unescapeHTML,
+    urlencode_postdata,
 )
 
 
@@ -139,7 +140,7 @@ class UdemyIE(InfoExtractor):
         })
 
         request = sanitized_Request(
-            self._LOGIN_URL, compat_urllib_parse_urlencode(login_form).encode('utf-8'))
+            self._LOGIN_URL, urlencode_postdata(login_form))
         request.add_header('Referer', self._ORIGIN_URL)
         request.add_header('Origin', self._ORIGIN_URL)
 
diff --git a/youtube_dl/extractor/vbox7.py b/youtube_dl/extractor/vbox7.py
index 77bb200e9..dff1bb702 100644
--- a/youtube_dl/extractor/vbox7.py
+++ b/youtube_dl/extractor/vbox7.py
@@ -2,13 +2,11 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse_urlencode,
-    compat_urlparse,
-)
+from ..compat import compat_urlparse
 from ..utils import (
     ExtractorError,
     sanitized_Request,
+    urlencode_postdata,
 )
 
 
@@ -48,7 +46,7 @@ class Vbox7IE(InfoExtractor):
                                         webpage, 'title').split('/')[0].strip()
 
         info_url = 'http://vbox7.com/play/magare.do'
-        data = compat_urllib_parse_urlencode({'as3': '1', 'vid': video_id})
+        data = urlencode_postdata({'as3': '1', 'vid': video_id})
         info_request = sanitized_Request(info_url, data)
         info_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
         info_response = self._download_webpage(info_request, video_id, 'Downloading info webpage')
diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 458099a4a..67220f1b7 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -5,10 +5,7 @@ import re
 import json
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_str,
-    compat_urllib_parse_urlencode,
-)
+from ..compat import compat_str
 from ..utils import (
     ExtractorError,
     int_or_none,
@@ -17,6 +14,7 @@ from ..utils import (
     str_to_int,
     unescapeHTML,
     unified_strdate,
+    urlencode_postdata,
 )
 from .vimeo import VimeoIE
 from .pladform import PladformIE
@@ -204,7 +202,7 @@ class VKIE(InfoExtractor):
 
         request = sanitized_Request(
             'https://login.vk.com/?act=login',
-            compat_urllib_parse_urlencode(login_form).encode('utf-8'))
+            urlencode_postdata(login_form))
         login_page = self._download_webpage(
             request, None, note='Logging in as %s' % username)
 
diff --git a/youtube_dl/extractor/vodlocker.py b/youtube_dl/extractor/vodlocker.py
index f1abca4d9..a938a4007 100644
--- a/youtube_dl/extractor/vodlocker.py
+++ b/youtube_dl/extractor/vodlocker.py
@@ -2,11 +2,11 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse_urlencode
 from ..utils import (
     ExtractorError,
     NO_DEFAULT,
     sanitized_Request,
+    urlencode_postdata,
 )
 
 
@@ -38,7 +38,7 @@ class VodlockerIE(InfoExtractor):
 
         if fields['op'] == 'download1':
             self._sleep(3, video_id)  # they do detect when requests happen too fast!
-            post = compat_urllib_parse_urlencode(fields)
+            post = urlencode_postdata(fields)
             req = sanitized_Request(url, post)
             req.add_header('Content-type', 'application/x-www-form-urlencoded')
             webpage = self._download_webpage(
diff --git a/youtube_dl/extractor/xfileshare.py b/youtube_dl/extractor/xfileshare.py
index 4e35e1f44..2d1504eaa 100644
--- a/youtube_dl/extractor/xfileshare.py
+++ b/youtube_dl/extractor/xfileshare.py
@@ -4,11 +4,11 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse_urlencode
 from ..utils import (
     ExtractorError,
     int_or_none,
     sanitized_Request,
+    urlencode_postdata,
 )
 
 
@@ -108,7 +108,7 @@ class XFileShareIE(InfoExtractor):
             if countdown:
                 self._sleep(countdown, video_id)
 
-            post = compat_urllib_parse_urlencode(fields)
+            post = urlencode_postdata(fields)
 
             req = sanitized_Request(url, post)
             req.add_header('Content-type', 'application/x-www-form-urlencoded')
diff --git a/youtube_dl/extractor/yandexmusic.py b/youtube_dl/extractor/yandexmusic.py
index 158f3ea68..025716958 100644
--- a/youtube_dl/extractor/yandexmusic.py
+++ b/youtube_dl/extractor/yandexmusic.py
@@ -5,15 +5,13 @@ import re
 import hashlib
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_str,
-    compat_urllib_parse_urlencode,
-)
+from ..compat import compat_str
 from ..utils import (
     ExtractorError,
     int_or_none,
     float_or_none,
     sanitized_Request,
+    urlencode_postdata,
 )
 
 
@@ -170,14 +168,14 @@ class YandexMusicPlaylistIE(YandexMusicPlaylistBaseIE):
             missing_track_ids = set(map(compat_str, track_ids)) - set(present_track_ids)
             request = sanitized_Request(
                 'https://music.yandex.ru/handlers/track-entries.jsx',
-                compat_urllib_parse_urlencode({
+                urlencode_postdata({
                     'entries': ','.join(missing_track_ids),
                     'lang': mu.get('settings', {}).get('lang', 'en'),
                     'external-domain': 'music.yandex.ru',
                     'overembed': 'false',
                     'sign': mu.get('authData', {}).get('user', {}).get('sign'),
                     'strict': 'true',
-                }).encode('utf-8'))
+                }))
             request.add_header('Referer', url)
             request.add_header('X-Requested-With', 'XMLHttpRequest')
 
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 83b5840f7..8c321f1fc 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -44,6 +44,7 @@ from ..utils import (
     unified_strdate,
     unsmuggle_url,
     uppercase_escape,
+    urlencode_postdata,
     ISO3166Utils,
 )
 
@@ -115,7 +116,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
             'hl': 'en_US',
         }
 
-        login_data = compat_urllib_parse_urlencode(login_form_strs).encode('ascii')
+        login_data = urlencode_postdata(login_form_strs)
 
         req = sanitized_Request(self._LOGIN_URL, login_data)
         login_results = self._download_webpage(
@@ -148,7 +149,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
                 'TrustDevice': 'on',
             })
 
-            tfa_data = compat_urllib_parse_urlencode(tfa_form_strs).encode('ascii')
+            tfa_data = urlencode_postdata(tfa_form_strs)
 
             tfa_req = sanitized_Request(self._TWOFACTOR_URL, tfa_data)
             tfa_results = self._download_webpage(

From e289d6d62cae85ded46ad6e92b33385f221b8370 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 26 Mar 2016 02:38:33 +0600
Subject: [PATCH 4423/6219] [test_compat] Add tests for
 compat_urllib_parse_urlencode

---
 test/test_compat.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/test/test_compat.py b/test/test_compat.py
index b6bfad05e..cc105807a 100644
--- a/test/test_compat.py
+++ b/test/test_compat.py
@@ -19,6 +19,7 @@ from youtube_dl.compat import (
     compat_str,
     compat_urllib_parse_unquote,
     compat_urllib_parse_unquote_plus,
+    compat_urllib_parse_urlencode,
 )
 
 
@@ -70,6 +71,12 @@ class TestCompat(unittest.TestCase):
         self.assertEqual(compat_urllib_parse_unquote_plus('abc%20def'), 'abc def')
         self.assertEqual(compat_urllib_parse_unquote_plus('%7e/abc+def'), '~/abc def')
 
+    def test_compat_urllib_parse_urlencode(self):
+        self.assertEqual(compat_urllib_parse_urlencode({'abc': 'def'}), 'abc=def')
+        self.assertEqual(compat_urllib_parse_urlencode({'abc': b'def'}), 'abc=def')
+        self.assertEqual(compat_urllib_parse_urlencode({b'abc': 'def'}), 'abc=def')
+        self.assertEqual(compat_urllib_parse_urlencode({b'abc': b'def'}), 'abc=def')
+
     def test_compat_shlex_split(self):
         self.assertEqual(compat_shlex_split('-option "one two"'), ['-option', 'one two'])
 

From 62cdb96f51eca4226b4d499e292d1ea1f9babb72 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 26 Mar 2016 08:58:03 +0100
Subject: [PATCH 4424/6219] release 2016.03.26

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 2291ed783..d4cf099f5 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.03.25'
+__version__ = '2016.03.26'

From 5964b598ff536c32198181e5027610f3d9a474bb Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 25 Mar 2016 16:17:54 +0800
Subject: [PATCH 4425/6219] [brightcove] Support alternative
 BrightcoveExperience layout

The full URL lays in the `data` attribute of <object> (#8862)
---
 youtube_dl/extractor/brightcove.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 59e8008f9..afe081d82 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -136,13 +136,16 @@ class BrightcoveLegacyIE(InfoExtractor):
         else:
             flashvars = {}
 
+        data_url = object_doc.attrib.get('data', '')
+        data_url_params = compat_parse_qs(compat_urllib_parse_urlparse(data_url).query)
+
         def find_param(name):
             if name in flashvars:
                 return flashvars[name]
             node = find_xpath_attr(object_doc, './param', 'name', name)
             if node is not None:
                 return node.attrib['value']
-            return None
+            return data_url_params.get(name)
 
         params = {}
 

From d6c340cae5c1e5704d6e709eefb7009fcda6e213 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 26 Mar 2016 18:21:07 +0800
Subject: [PATCH 4426/6219] [brightcove] Extract more formats (#8862)

---
 youtube_dl/extractor/brightcove.py | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index afe081d82..c9e43a275 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -297,7 +297,7 @@ class BrightcoveLegacyIE(InfoExtractor):
             'uploader': video_info.get('publisherName'),
         }
 
-        renditions = video_info.get('renditions')
+        renditions = video_info.get('renditions', []) + video_info.get('IOSRenditions', [])
         if renditions:
             formats = []
             for rend in renditions:
@@ -319,13 +319,23 @@ class BrightcoveLegacyIE(InfoExtractor):
                 if ext is None:
                     ext = determine_ext(url)
                 size = rend.get('size')
-                formats.append({
+                a_format = {
                     'url': url,
                     'ext': ext,
                     'height': rend.get('frameHeight'),
                     'width': rend.get('frameWidth'),
                     'filesize': size if size != 0 else None,
-                })
+                }
+
+                # m3u8 manifests with remote == false are media playlists
+                # Not calling _extract_m3u8_formats here to save network traffic
+                if ext == 'm3u8':
+                    a_format.update({
+                        'ext': 'mp4',
+                        'protocol': 'm3u8',
+                    })
+
+                formats.append(a_format)
             self._sort_formats(formats)
             info['formats'] = formats
         elif video_info.get('FLVFullLengthURL') is not None:

From d10fe8358c064325349469a20be952ba794566d4 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 26 Mar 2016 18:30:43 +0800
Subject: [PATCH 4427/6219] [generic] Add a test case for brightcove embed

Closes #8862
---
 youtube_dl/extractor/generic.py | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 12f2309fc..ea4009b41 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1124,7 +1124,23 @@ class GenericIE(InfoExtractor):
                 # m3u8 downloads
                 'skip_download': True,
             }
-        }
+        },
+        # Brightcove embed, with no valid 'renditions' but valid 'IOSRenditions'
+        # This video can't be played in browsers if Flash disabled and UA set to iPhone, which is actually a false alarm
+        {
+            'url': 'https://dl.dropboxusercontent.com/u/29092637/interview.html',
+            'info_dict': {
+                'id': '4785848093001',
+                'ext': 'mp4',
+                'title': 'The Cardinal Pell Interview',
+                'description': 'Sky News Contributor Andrew Bolt interviews George Pell in Rome, following the Cardinal\'s evidence before the Royal Commission into Child Abuse. ',
+                'uploader': 'GlobeCast Australia - GlobeStream',
+            },
+            'params': {
+                # m3u8 downloads
+                'skip_download': True,
+            },
+        },
     ]
 
     def report_following_redirect(self, new_url):

From e68d3a010fcf34455c7922b28a05ccc012381729 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 26 Mar 2016 18:34:51 +0800
Subject: [PATCH 4428/6219] [twitter] Fix extraction (closes #8966)

HLS and DASH formats are no longer appeared in test cases. I keep them
for fear of triggering new errors.
---
 youtube_dl/extractor/twitter.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index e70b2ab3c..602538e5c 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -110,10 +110,9 @@ class TwitterCardIE(TwitterBaseIE):
                     'height': int(m.group('height')),
                 })
 
-        playlist = config.get('playlist')
-        if playlist:
-            video_url = playlist[0]['source']
+        video_url = config.get('video_url') or config.get('playlist', [{}])[0].get('source')
 
+        if video_url:
             f = {
                 'url': video_url,
             }
@@ -185,7 +184,6 @@ class TwitterIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'FREE THE NIPPLE - FTN supporters on Hollywood Blvd today!',
             'thumbnail': 're:^https?://.*\.jpg',
-            'duration': 12.922,
             'description': 'FREE THE NIPPLE on Twitter: "FTN supporters on Hollywood Blvd today! http://t.co/c7jHH749xJ"',
             'uploader': 'FREE THE NIPPLE',
             'uploader_id': 'freethenipple',

From b5a5bbf3764a3912a1d07816b6e91560fe1d8a10 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 26 Mar 2016 19:15:32 +0600
Subject: [PATCH 4429/6219] [mailru] Extend _VALID_URL (Closes #8990)

---
 youtube_dl/extractor/mailru.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mailru.py b/youtube_dl/extractor/mailru.py
index 46eb00492..9a7098c43 100644
--- a/youtube_dl/extractor/mailru.py
+++ b/youtube_dl/extractor/mailru.py
@@ -13,7 +13,7 @@ from ..utils import (
 class MailRuIE(InfoExtractor):
     IE_NAME = 'mailru'
     IE_DESC = 'Видео@Mail.Ru'
-    _VALID_URL = r'https?://(?:www\.)?my\.mail\.ru/(?:video/.*#video=/?(?P<idv1>(?:[^/]+/){3}\d+)|(?:(?P<idv2prefix>(?:[^/]+/){2})video/(?P<idv2suffix>[^/]+/\d+))\.html)'
+    _VALID_URL = r'https?://(?:(?:www|m)\.)?my\.mail\.ru/(?:video/.*#video=/?(?P<idv1>(?:[^/]+/){3}\d+)|(?:(?P<idv2prefix>(?:[^/]+/){2})video/(?P<idv2suffix>[^/]+/\d+))\.html)'
 
     _TESTS = [
         {
@@ -61,6 +61,10 @@ class MailRuIE(InfoExtractor):
                 'duration': 6001,
             },
             'skip': 'Not accessible from Travis CI server',
+        },
+        {
+            'url': 'http://m.my.mail.ru/mail/3sktvtr/video/_myvideo/138.html',
+            'only_matching': True,
         }
     ]
 

From 17bcc626bf67453cc5ab67e56684b6c6e33f4cb6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 26 Mar 2016 19:33:57 +0600
Subject: [PATCH 4430/6219] [utils] Extract sanitize_url routine

---
 youtube_dl/utils.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index eacd81bf9..6d27b80c0 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -417,9 +417,12 @@ def sanitize_path(s):
 
 # Prepend protocol-less URLs with `http:` scheme in order to mitigate the number of
 # unwanted failures due to missing protocol
+def sanitize_url(url):
+    return 'http:%s' % url if url.startswith('//') else url
+
+
 def sanitized_Request(url, *args, **kwargs):
-    return compat_urllib_request.Request(
-        'http:%s' % url if url.startswith('//') else url, *args, **kwargs)
+    return compat_urllib_request.Request(sanitize_url(url), *args, **kwargs)
 
 
 def orderedSet(iterable):

From dcf77cf1a74ebcf7def71aecf55b8641e4645835 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 26 Mar 2016 19:37:41 +0600
Subject: [PATCH 4431/6219] [YoutubeDL] Sanitize final URLs (Closes #8991)

---
 youtube_dl/YoutubeDL.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 29d7a3106..33c269f9c 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -65,6 +65,7 @@ from .utils import (
     SameFileError,
     sanitize_filename,
     sanitize_path,
+    sanitize_url,
     sanitized_Request,
     std_headers,
     subtitles_filename,
@@ -1229,6 +1230,7 @@ class YoutubeDL(object):
                 t.get('preference'), t.get('width'), t.get('height'),
                 t.get('id'), t.get('url')))
             for i, t in enumerate(thumbnails):
+                t['url'] = sanitize_url(t['url'])
                 if t.get('width') and t.get('height'):
                     t['resolution'] = '%dx%d' % (t['width'], t['height'])
                 if t.get('id') is None:
@@ -1263,6 +1265,7 @@ class YoutubeDL(object):
         if subtitles:
             for _, subtitle in subtitles.items():
                 for subtitle_format in subtitle:
+                    subtitle_format['url'] = sanitize_url(subtitle_format['url'])
                     if 'ext' not in subtitle_format:
                         subtitle_format['ext'] = determine_ext(subtitle_format['url']).lower()
 
@@ -1292,6 +1295,8 @@ class YoutubeDL(object):
             if 'url' not in format:
                 raise ExtractorError('Missing "url" key in result (index %d)' % i)
 
+            format['url'] = sanitize_url(format['url'])
+
             if format.get('format_id') is None:
                 format['format_id'] = compat_str(i)
             else:

From eedb7ba5364213b5f9dc773f70403ea028a44ab0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 26 Mar 2016 19:40:33 +0600
Subject: [PATCH 4432/6219] [YoutubeDL] Sort imports

---
 youtube_dl/YoutubeDL.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 33c269f9c..53a36c145 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -39,6 +39,8 @@ from .compat import (
     compat_urllib_request_DataHandler,
 )
 from .utils import (
+    age_restricted,
+    args_to_str,
     ContentTooShortError,
     date_from_str,
     DateRange,
@@ -58,10 +60,12 @@ from .utils import (
     PagedList,
     parse_filesize,
     PerRequestProxyHandler,
-    PostProcessingError,
     platform_name,
+    PostProcessingError,
     preferredencoding,
+    prepend_extension,
     render_table,
+    replace_extension,
     SameFileError,
     sanitize_filename,
     sanitize_path,
@@ -76,10 +80,6 @@ from .utils import (
     write_string,
     YoutubeDLCookieProcessor,
     YoutubeDLHandler,
-    prepend_extension,
-    replace_extension,
-    args_to_str,
-    age_restricted,
 )
 from .cache import Cache
 from .extractor import get_info_extractor, gen_extractors

From 6dee688e6d8992913bbdbcc65a413cd9897dd489 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 26 Mar 2016 20:42:18 +0600
Subject: [PATCH 4433/6219] [youtube:playlistsbase] Restrict playlist regex
 (Closes #8986)

---
 youtube_dl/extractor/youtube.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 8c321f1fc..28355bf46 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -234,7 +234,9 @@ class YoutubePlaylistBaseInfoExtractor(YoutubeEntryListBaseInfoExtractor):
 
 class YoutubePlaylistsBaseInfoExtractor(YoutubeEntryListBaseInfoExtractor):
     def _process_page(self, content):
-        for playlist_id in orderedSet(re.findall(r'href="/?playlist\?list=([0-9A-Za-z-_]{10,})"', content)):
+        for playlist_id in orderedSet(re.findall(
+                r'<h3[^>]+class="[^"]*yt-lockup-title[^"]*"[^>]*><a[^>]+href="/?playlist\?list=([0-9A-Za-z-_]{10,})"',
+                content)):
             yield self.url_result(
                 'https://www.youtube.com/playlist?list=%s' % playlist_id, 'YoutubePlaylist')
 

From efcba804f646dfc4eda2f9df2baf3ebed0f1bbe4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 26 Mar 2016 23:42:34 +0600
Subject: [PATCH 4434/6219] [udemy] Extract formats from view_html (Closes
 #8979)

---
 youtube_dl/extractor/udemy.py | 41 +++++++++++++++++++++++++++++++----
 1 file changed, 37 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index be6f3be5e..da2d542ec 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -1,5 +1,7 @@
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..compat import (
     compat_HTTPError,
@@ -8,6 +10,8 @@ from ..compat import (
     compat_urlparse,
 )
 from ..utils import (
+    determine_ext,
+    extract_attributes,
     ExtractorError,
     float_or_none,
     int_or_none,
@@ -73,11 +77,8 @@ class UdemyIE(InfoExtractor):
         return self._download_json(
             'https://www.udemy.com/api-2.0/users/me/subscribed-courses/%s/lectures/%s?%s' % (
                 course_id, lecture_id, compat_urllib_parse_urlencode({
-                    'video_only': '',
-                    'auto_play': '',
-                    'fields[lecture]': 'title,description,asset',
+                    'fields[lecture]': 'title,description,view_html,asset',
                     'fields[asset]': 'asset_type,stream_url,thumbnail_url,download_urls,data',
-                    'instructorPreviewMode': 'False',
                 })),
             lecture_id, 'Downloading lecture JSON')
 
@@ -246,6 +247,38 @@ class UdemyIE(InfoExtractor):
                             f['format_id'] = '%sp' % format_id
                     formats.append(f)
 
+        view_html = lecture.get('view_html')
+        if view_html:
+            view_html_urls = set()
+            for source in re.findall(r'<source[^>]+>', view_html):
+                attributes = extract_attributes(source)
+                src = attributes.get('src')
+                if not src:
+                    continue
+                res = attributes.get('data-res')
+                height = int_or_none(res)
+                if src in view_html_urls:
+                    continue
+                view_html_urls.add(src)
+                if attributes.get('type') == 'application/x-mpegURL' or determine_ext(src) == 'm3u8':
+                    m3u8_formats = self._extract_m3u8_formats(
+                        src, video_id, 'mp4', entry_protocol='m3u8_native',
+                        m3u8_id='hls', fatal=False)
+                    for f in m3u8_formats:
+                        m = re.search(r'/hls_(?P<height>\d{3,4})_(?P<tbr>\d{2,})/', f['url'])
+                        if m:
+                            if not f.get('height'):
+                                f['height'] = int(m.group('height'))
+                            if not f.get('tbr'):
+                                f['tbr'] = int(m.group('tbr'))
+                    formats.extend(m3u8_formats)
+                else:
+                    formats.append({
+                        'url': src,
+                        'format_id': '%dp' % height if height else None,
+                        'height': height,
+                    })
+
         self._sort_formats(formats)
 
         return {

From 48dce58ca907921f5013367a7b22235b3a3a05df Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 26 Mar 2016 23:42:46 +0600
Subject: [PATCH 4435/6219] [udemy] Use custom sorting

---
 youtube_dl/extractor/udemy.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index da2d542ec..100db4dd0 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -279,7 +279,7 @@ class UdemyIE(InfoExtractor):
                         'height': height,
                     })
 
-        self._sort_formats(formats)
+        self._sort_formats(formats, field_preference=('height', 'width', 'tbr', 'format_id'))
 
         return {
             'id': video_id,

From 02d7634d24b704a099e17224e3dc71906ccc92a9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 26 Mar 2016 23:43:25 +0600
Subject: [PATCH 4436/6219] [udemy] Fix outputs' formats format_id

---
 youtube_dl/extractor/udemy.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index 100db4dd0..89e713285 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -201,7 +201,7 @@ class UdemyIE(InfoExtractor):
         def extract_output_format(src):
             return {
                 'url': src['url'],
-                'format_id': '%sp' % (src.get('label') or format_id),
+                'format_id': '%sp' % (src.get('height') or format_id),
                 'width': int_or_none(src.get('width')),
                 'height': int_or_none(src.get('height')),
                 'vbr': int_or_none(src.get('video_bitrate_in_kbps')),

From 62f55aa68a5409c25457a14289cc859fdd73cc1e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 26 Mar 2016 23:54:12 +0600
Subject: [PATCH 4437/6219] [udemy] Add outputs metadata to view_html formats

---
 youtube_dl/extractor/udemy.py | 23 +++++++++++++----------
 1 file changed, 13 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index 89e713285..5a6de9982 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -218,6 +218,16 @@ class UdemyIE(InfoExtractor):
         if not isinstance(outputs, dict):
             outputs = {}
 
+        def add_output_format_meta(f, key, format_id):
+            output = outputs.get(key)
+            if isinstance(output, dict):
+                output_format = extract_output_format(output)
+                output_format.update(f)
+                return output_format
+            else:
+                f['format_id'] = format_id
+                return f
+
         for format_id, output in outputs.items():
             if isinstance(output, dict) and output.get('url'):
                 formats.append(extract_output_format(output))
@@ -238,13 +248,7 @@ class UdemyIE(InfoExtractor):
                     if format_id:
                         # Some videos contain additional metadata (e.g.
                         # https://www.udemy.com/ios9-swift/learn/#/lecture/3383208)
-                        output = outputs.get(format_id)
-                        if isinstance(output, dict):
-                            output_format = extract_output_format(output)
-                            output_format.update(f)
-                            f = output_format
-                        else:
-                            f['format_id'] = '%sp' % format_id
+                        f = add_output_format_meta(f, format_id, '%sp' % format_id)
                     formats.append(f)
 
         view_html = lecture.get('view_html')
@@ -273,11 +277,10 @@ class UdemyIE(InfoExtractor):
                                 f['tbr'] = int(m.group('tbr'))
                     formats.extend(m3u8_formats)
                 else:
-                    formats.append({
+                    formats.append(add_output_format_meta({
                         'url': src,
-                        'format_id': '%dp' % height if height else None,
                         'height': height,
-                    })
+                    }, res, '%dp' % height if height else None))
 
         self._sort_formats(formats, field_preference=('height', 'width', 'tbr', 'format_id'))
 

From f973e5d54e233c139d0b407b5772ff4966c8fa30 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 26 Mar 2016 23:55:07 +0600
Subject: [PATCH 4438/6219] [udemy] Drop outputs' formats

Always results in 403
---
 youtube_dl/extractor/udemy.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index 5a6de9982..6a3dcb8d6 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -228,10 +228,6 @@ class UdemyIE(InfoExtractor):
                 f['format_id'] = format_id
                 return f
 
-        for format_id, output in outputs.items():
-            if isinstance(output, dict) and output.get('url'):
-                formats.append(extract_output_format(output))
-
         download_urls = asset.get('download_urls')
         if isinstance(download_urls, dict):
             video = download_urls.get('Video')

From af4116f4f04a3fc8150fdb4a220ef31a0a2dd044 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 27 Mar 2016 00:02:52 +0600
Subject: [PATCH 4439/6219] [udemy] Improve format_id

---
 youtube_dl/extractor/udemy.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index 6a3dcb8d6..5a5e9fa9e 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -218,15 +218,13 @@ class UdemyIE(InfoExtractor):
         if not isinstance(outputs, dict):
             outputs = {}
 
-        def add_output_format_meta(f, key, format_id):
+        def add_output_format_meta(f, key):
             output = outputs.get(key)
             if isinstance(output, dict):
                 output_format = extract_output_format(output)
                 output_format.update(f)
                 return output_format
-            else:
-                f['format_id'] = format_id
-                return f
+            return f
 
         download_urls = asset.get('download_urls')
         if isinstance(download_urls, dict):
@@ -239,12 +237,13 @@ class UdemyIE(InfoExtractor):
                     format_id = format_.get('label')
                     f = {
                         'url': format_['file'],
+                        'format_id': '%sp' % format_id,
                         'height': int_or_none(format_id),
                     }
                     if format_id:
                         # Some videos contain additional metadata (e.g.
                         # https://www.udemy.com/ios9-swift/learn/#/lecture/3383208)
-                        f = add_output_format_meta(f, format_id, '%sp' % format_id)
+                        f = add_output_format_meta(f, format_id)
                     formats.append(f)
 
         view_html = lecture.get('view_html')
@@ -275,8 +274,9 @@ class UdemyIE(InfoExtractor):
                 else:
                     formats.append(add_output_format_meta({
                         'url': src,
+                        'format_id': '%dp' % height if height else None,
                         'height': height,
-                    }, res, '%dp' % height if height else None))
+                    }, res))
 
         self._sort_formats(formats, field_preference=('height', 'width', 'tbr', 'format_id'))
 

From b24ab3e341b9082774785332a1aa6405764f7202 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 27 Mar 2016 00:09:12 +0600
Subject: [PATCH 4440/6219] [udemy] Improve paid course detection

---
 youtube_dl/extractor/udemy.py | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index 5a5e9fa9e..71bea5363 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -55,21 +55,26 @@ class UdemyIE(InfoExtractor):
     }]
 
     def _enroll_course(self, base_url, webpage, course_id):
+        def combine_url(base_url, url):
+            return compat_urlparse.urljoin(base_url, url) if not url.startswith('http') else url
+
         checkout_url = unescapeHTML(self._search_regex(
-            r'href=(["\'])(?P<url>https?://(?:www\.)?udemy\.com/payment/checkout/.+?)\1',
+            r'href=(["\'])(?P<url>(?:https?://(?:www\.)?udemy\.com)?/payment/checkout/.+?)\1',
             webpage, 'checkout url', group='url', default=None))
         if checkout_url:
             raise ExtractorError(
                 'Course %s is not free. You have to pay for it before you can download. '
-                'Use this URL to confirm purchase: %s' % (course_id, checkout_url), expected=True)
+                'Use this URL to confirm purchase: %s'
+                % (course_id, combine_url(base_url, checkout_url)),
+                expected=True)
 
         enroll_url = unescapeHTML(self._search_regex(
             r'href=(["\'])(?P<url>(?:https?://(?:www\.)?udemy\.com)?/course/subscribe/.+?)\1',
             webpage, 'enroll url', group='url', default=None))
         if enroll_url:
-            if not enroll_url.startswith('http'):
-                enroll_url = compat_urlparse.urljoin(base_url, enroll_url)
-            webpage = self._download_webpage(enroll_url, course_id, 'Enrolling in the course')
+            webpage = self._download_webpage(
+                combine_url(base_url, enroll_url),
+                course_id, 'Enrolling in the course')
             if '>You have enrolled in' in webpage:
                 self.to_screen('%s: Successfully enrolled in the course' % course_id)
 

From 4cf3489c6e548aebe29534e496e7ccd638be6873 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 27 Mar 2016 01:11:11 +0600
Subject: [PATCH 4441/6219] [vevo] Update videoservice API URL (Closes #8900)

---
 youtube_dl/extractor/vevo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py
index 152fef42e..147480f64 100644
--- a/youtube_dl/extractor/vevo.py
+++ b/youtube_dl/extractor/vevo.py
@@ -152,7 +152,7 @@ class VevoIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        json_url = 'http://videoplayer.vevo.com/VideoService/AuthenticateVideo?isrc=%s' % video_id
+        json_url = 'http://api.vevo.com/VideoService/AuthenticateVideo?isrc=%s' % video_id
         response = self._download_json(
             json_url, video_id, 'Downloading video info', 'Unable to download info')
         video_info = response.get('video') or {}

From 00322ad4fda31864f249bce410ac3ba520e865ed Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 27 Mar 2016 02:00:36 +0600
Subject: [PATCH 4442/6219] [lynda] Extract chapter metadata (#8993)

---
 youtube_dl/extractor/lynda.py | 20 +++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index 71fd55ade..655627479 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -219,7 +219,7 @@ class LyndaCourseIE(LyndaBaseIE):
                 'Course %s does not exist' % course_id, expected=True)
 
         unaccessible_videos = 0
-        videos = []
+        entries = []
 
         # Might want to extract videos right here from video['Formats'] as it seems 'Formats' is not provided
         # by single video API anymore
@@ -229,20 +229,22 @@ class LyndaCourseIE(LyndaBaseIE):
                 if video.get('HasAccess') is False:
                     unaccessible_videos += 1
                     continue
-                if video.get('ID'):
-                    videos.append(video['ID'])
+                video_id = video.get('ID')
+                if video_id:
+                    entries.append({
+                        '_type': 'url_transparent',
+                        'url': 'http://www.lynda.com/%s/%s-4.html' % (course_path, video_id),
+                        'ie_key': LyndaIE.ie_key(),
+                        'chapter': chapter.get('Title'),
+                        'chapter_number': int_or_none(chapter.get('ChapterIndex')),
+                        'chapter_id': compat_str(chapter.get('ID')),
+                    })
 
         if unaccessible_videos > 0:
             self._downloader.report_warning(
                 '%s videos are only available for members (or paid members) and will not be downloaded. '
                 % unaccessible_videos + self._ACCOUNT_CREDENTIALS_HINT)
 
-        entries = [
-            self.url_result(
-                'http://www.lynda.com/%s/%s-4.html' % (course_path, video_id),
-                'Lynda')
-            for video_id in videos]
-
         course_title = course.get('Title')
 
         return self.playlist_result(entries, course_id, course_title)

From 8018028d0fabb00c32b19b04984c482c6b54d2fb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 27 Mar 2016 02:10:52 +0600
Subject: [PATCH 4443/6219] [pluralsight] Extract chapter metadata (Closes
 #8993)

---
 youtube_dl/extractor/pluralsight.py | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index bc66f7a9d..df03dd419 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -279,13 +279,18 @@ class PluralsightCourseIE(PluralsightBaseIE):
             course_id, 'Downloading course data JSON')
 
         entries = []
-        for module in course_data:
+        for num, module in enumerate(course_data, 1):
             for clip in module.get('clips', []):
                 player_parameters = clip.get('playerParameters')
                 if not player_parameters:
                     continue
-                entries.append(self.url_result(
-                    '%s/training/player?%s' % (self._API_BASE, player_parameters),
-                    'Pluralsight'))
+                entries.append({
+                    '_type': 'url_transparent',
+                    'url': '%s/training/player?%s' % (self._API_BASE, player_parameters),
+                    'ie_key': PluralsightIE.ie_key(),
+                    'chapter': module.get('title'),
+                    'chapter_number': num,
+                    'chapter_id': module.get('moduleRef'),
+                })
 
         return self.playlist_result(entries, course_id, title, description)

From 395fd4b08a4639f7e84754527e9facd83c8f782d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 27 Mar 2016 04:36:02 +0800
Subject: [PATCH 4444/6219] [twitter] Handle another form of embedded Vine

Fixes #8996
---
 youtube_dl/extractor/twitter.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 602538e5c..1f32ea2eb 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -102,6 +102,9 @@ class TwitterCardIE(TwitterBaseIE):
             r'data-(?:player-)?config="([^"]+)"', webpage, 'data player config'),
             video_id)
 
+        if config.get('source_type') == 'vine':
+            return self.url_result(config['player_url'], 'Vine')
+
         def _search_dimensions_in_video_url(a_format, video_url):
             m = re.search(r'/(?P<width>\d+)x(?P<height>\d+)/', video_url)
             if m:
@@ -245,6 +248,18 @@ class TwitterIE(InfoExtractor):
         'params': {
             'skip_download': True,  # requires ffmpeg
         },
+    }, {
+        'url': 'https://twitter.com/Filmdrunk/status/713801302971588609',
+        'md5': '89a15ed345d13b86e9a5a5e051fa308a',
+        'info_dict': {
+            'id': 'MIOxnrUteUd',
+            'ext': 'mp4',
+            'title': 'Dr.Pepperの飲み方 #japanese #バカ #ドクペ #電動ガン',
+            'uploader': 'TAKUMA',
+            'uploader_id': '1004126642786242560',
+            'upload_date': '20140615',
+        },
+        'add_ie': ['Vine'],
     }]
 
     def _real_extract(self, url):

From 19dbaeece321c51fa336ef142507adf440e22e22 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 27 Mar 2016 07:03:08 +0600
Subject: [PATCH 4445/6219] Remove _sort_formats from _extract_*_formats
 methods

Now _sort_formats should be called explicitly.
_sort_formats has been added to all the necessary places in code.

Closes #8051
---
 youtube_dl/extractor/abc7news.py   |  1 +
 youtube_dl/extractor/azubu.py      |  1 +
 youtube_dl/extractor/bet.py        |  1 +
 youtube_dl/extractor/cbsnews.py    |  1 +
 youtube_dl/extractor/chaturbate.py |  1 +
 youtube_dl/extractor/common.py     |  6 ------
 youtube_dl/extractor/cwtv.py       |  1 +
 youtube_dl/extractor/dfb.py        |  1 +
 youtube_dl/extractor/discovery.py  | 29 +++++++++++++++++------------
 youtube_dl/extractor/dplay.py      |  2 ++
 youtube_dl/extractor/dw.py         |  2 +-
 youtube_dl/extractor/generic.py    | 11 ++++++++++-
 youtube_dl/extractor/laola1tv.py   |  1 +
 youtube_dl/extractor/lrt.py        |  1 +
 youtube_dl/extractor/matchtv.py    |  1 +
 youtube_dl/extractor/mitele.py     |  1 +
 youtube_dl/extractor/nrk.py        |  1 +
 youtube_dl/extractor/restudy.py    |  1 +
 youtube_dl/extractor/rte.py        |  1 +
 youtube_dl/extractor/rtve.py       |  1 +
 youtube_dl/extractor/rtvnh.py      |  1 +
 youtube_dl/extractor/shahid.py     |  1 +
 youtube_dl/extractor/sportbox.py   |  1 +
 youtube_dl/extractor/telecinco.py  |  1 +
 youtube_dl/extractor/tubitv.py     |  1 +
 youtube_dl/extractor/videomore.py  |  1 +
 youtube_dl/extractor/vier.py       |  1 +
 youtube_dl/extractor/viidea.py     |  1 +
 youtube_dl/extractor/ynet.py       |  4 +++-
 29 files changed, 56 insertions(+), 21 deletions(-)

diff --git a/youtube_dl/extractor/abc7news.py b/youtube_dl/extractor/abc7news.py
index 122dc9099..c04949c21 100644
--- a/youtube_dl/extractor/abc7news.py
+++ b/youtube_dl/extractor/abc7news.py
@@ -44,6 +44,7 @@ class Abc7NewsIE(InfoExtractor):
             'contentURL', webpage, 'm3u8 url', fatal=True)
 
         formats = self._extract_m3u8_formats(m3u8, display_id, 'mp4')
+        self._sort_formats(formats)
 
         title = self._og_search_title(webpage).strip()
         description = self._og_search_description(webpage).strip()
diff --git a/youtube_dl/extractor/azubu.py b/youtube_dl/extractor/azubu.py
index 1805b7312..efa624de1 100644
--- a/youtube_dl/extractor/azubu.py
+++ b/youtube_dl/extractor/azubu.py
@@ -120,6 +120,7 @@ class AzubuLiveIE(InfoExtractor):
         bc_info = self._download_json(req, user)
         m3u8_url = next(source['src'] for source in bc_info['sources'] if source['container'] == 'M2TS')
         formats = self._extract_m3u8_formats(m3u8_url, user, ext='mp4')
+        self._sort_formats(formats)
 
         return {
             'id': info['id'],
diff --git a/youtube_dl/extractor/bet.py b/youtube_dl/extractor/bet.py
index 03dad4636..986245bf0 100644
--- a/youtube_dl/extractor/bet.py
+++ b/youtube_dl/extractor/bet.py
@@ -94,6 +94,7 @@ class BetIE(InfoExtractor):
             xpath_with_ns('./media:thumbnail', NS_MAP)).get('url')
 
         formats = self._extract_smil_formats(smil_url, display_id)
+        self._sort_formats(formats)
 
         return {
             'id': video_id,
diff --git a/youtube_dl/extractor/cbsnews.py b/youtube_dl/extractor/cbsnews.py
index f23bac9a1..e6b7f3584 100644
--- a/youtube_dl/extractor/cbsnews.py
+++ b/youtube_dl/extractor/cbsnews.py
@@ -122,6 +122,7 @@ class CBSNewsLiveVideoIE(InfoExtractor):
             for entry in f4m_formats:
                 # URLs without the extra param induce an 404 error
                 entry.update({'extra_param_to_segment_url': hdcore_sign})
+        self._sort_formats(f4m_formats)
 
         return {
             'id': video_id,
diff --git a/youtube_dl/extractor/chaturbate.py b/youtube_dl/extractor/chaturbate.py
index 242fba311..b2234549e 100644
--- a/youtube_dl/extractor/chaturbate.py
+++ b/youtube_dl/extractor/chaturbate.py
@@ -48,6 +48,7 @@ class ChaturbateIE(InfoExtractor):
             raise ExtractorError('Unable to find stream URL')
 
         formats = self._extract_m3u8_formats(m3u8_url, video_id, ext='mp4')
+        self._sort_formats(formats)
 
         return {
             'id': video_id,
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index b412fd030..40ddf175c 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1021,8 +1021,6 @@ class InfoExtractor(object):
                 'height': int_or_none(media_el.attrib.get('height')),
                 'preference': preference,
             })
-        self._sort_formats(formats)
-
         return formats
 
     def _extract_m3u8_formats(self, m3u8_url, video_id, ext=None,
@@ -1143,7 +1141,6 @@ class InfoExtractor(object):
                     last_media = None
                 formats.append(f)
                 last_info = {}
-        self._sort_formats(formats)
         return formats
 
     @staticmethod
@@ -1317,8 +1314,6 @@ class InfoExtractor(object):
                 })
                 continue
 
-        self._sort_formats(formats)
-
         return formats
 
     def _parse_smil_subtitles(self, smil, namespace=None, subtitles_lang='en'):
@@ -1536,7 +1531,6 @@ class InfoExtractor(object):
                             existing_format.update(f)
                     else:
                         self.report_warning('Unknown MIME type %s in DASH manifest' % mime_type)
-        self._sort_formats(formats)
         return formats
 
     def _live_title(self, name):
diff --git a/youtube_dl/extractor/cwtv.py b/youtube_dl/extractor/cwtv.py
index 36af67013..f5cefd966 100644
--- a/youtube_dl/extractor/cwtv.py
+++ b/youtube_dl/extractor/cwtv.py
@@ -57,6 +57,7 @@ class CWTVIE(InfoExtractor):
 
         formats = self._extract_m3u8_formats(
             video_data['videos']['variantplaylist']['uri'], video_id, 'mp4')
+        self._sort_formats(formats)
 
         thumbnails = [{
             'url': image['uri'],
diff --git a/youtube_dl/extractor/dfb.py b/youtube_dl/extractor/dfb.py
index 263532cc6..cdfeccacb 100644
--- a/youtube_dl/extractor/dfb.py
+++ b/youtube_dl/extractor/dfb.py
@@ -38,6 +38,7 @@ class DFBIE(InfoExtractor):
         token_el = f4m_info.find('token')
         manifest_url = token_el.attrib['url'] + '?' + 'hdnea=' + token_el.attrib['auth'] + '&hdcore=3.2.0'
         formats = self._extract_f4m_formats(manifest_url, display_id)
+        self._sort_formats(formats)
 
         return {
             'id': video_id,
diff --git a/youtube_dl/extractor/discovery.py b/youtube_dl/extractor/discovery.py
index fdce1429a..5f1275b39 100644
--- a/youtube_dl/extractor/discovery.py
+++ b/youtube_dl/extractor/discovery.py
@@ -63,18 +63,23 @@ class DiscoveryIE(InfoExtractor):
 
         video_title = info.get('playlist_title') or info.get('video_title')
 
-        entries = [{
-            'id': compat_str(video_info['id']),
-            'formats': self._extract_m3u8_formats(
+        entries = []
+
+        for idx, video_info in enumerate(info['playlist']):
+            formats = self._extract_m3u8_formats(
                 video_info['src'], display_id, 'mp4', 'm3u8_native', m3u8_id='hls',
-                note='Download m3u8 information for video %d' % (idx + 1)),
-            'title': video_info['title'],
-            'description': video_info.get('description'),
-            'duration': parse_duration(video_info.get('video_length')),
-            'webpage_url': video_info.get('href') or video_info.get('url'),
-            'thumbnail': video_info.get('thumbnailURL'),
-            'alt_title': video_info.get('secondary_title'),
-            'timestamp': parse_iso8601(video_info.get('publishedDate')),
-        } for idx, video_info in enumerate(info['playlist'])]
+                note='Download m3u8 information for video %d' % (idx + 1))
+            self._sort_formats(formats)
+            entries.append({
+                'id': compat_str(video_info['id']),
+                'formats': formats,
+                'title': video_info['title'],
+                'description': video_info.get('description'),
+                'duration': parse_duration(video_info.get('video_length')),
+                'webpage_url': video_info.get('href') or video_info.get('url'),
+                'thumbnail': video_info.get('thumbnailURL'),
+                'alt_title': video_info.get('secondary_title'),
+                'timestamp': parse_iso8601(video_info.get('publishedDate')),
+            })
 
         return self.playlist_result(entries, display_id, video_title)
diff --git a/youtube_dl/extractor/dplay.py b/youtube_dl/extractor/dplay.py
index 1e7dcada6..66bbfc6ca 100644
--- a/youtube_dl/extractor/dplay.py
+++ b/youtube_dl/extractor/dplay.py
@@ -118,6 +118,8 @@ class DPlayIE(InfoExtractor):
                 if info.get(protocol):
                     extract_formats(protocol, info[protocol])
 
+        self._sort_formats(formats)
+
         return {
             'id': video_id,
             'display_id': display_id,
diff --git a/youtube_dl/extractor/dw.py b/youtube_dl/extractor/dw.py
index b6c985547..ae7c571bd 100644
--- a/youtube_dl/extractor/dw.py
+++ b/youtube_dl/extractor/dw.py
@@ -39,13 +39,13 @@ class DWIE(InfoExtractor):
         hidden_inputs = self._hidden_inputs(webpage)
         title = hidden_inputs['media_title']
 
-        formats = []
         if hidden_inputs.get('player_type') == 'video' and hidden_inputs.get('stream_file') == '1':
             formats = self._extract_smil_formats(
                 'http://www.dw.com/smil/v-%s' % media_id, media_id,
                 transform_source=lambda s: s.replace(
                     'rtmp://tv-od.dw.de/flash/',
                     'http://tv-download.dw.de/dwtv_video/flv/'))
+            self._sort_formats(formats)
         else:
             formats = [{'url': hidden_inputs['file_name']}]
 
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index ea4009b41..f3de738f7 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1310,6 +1310,7 @@ class GenericIE(InfoExtractor):
                     'vcodec': 'none' if m.group('type') == 'audio' else None
                 }]
                 info_dict['direct'] = True
+            self._sort_formats(formats)
             info_dict['formats'] = formats
             return info_dict
 
@@ -1336,6 +1337,7 @@ class GenericIE(InfoExtractor):
         # Is it an M3U playlist?
         if first_bytes.startswith(b'#EXTM3U'):
             info_dict['formats'] = self._extract_m3u8_formats(url, video_id, 'mp4')
+            self._sort_formats(info_dict['formats'])
             return info_dict
 
         # Maybe it's a direct link to a video?
@@ -1360,15 +1362,19 @@ class GenericIE(InfoExtractor):
             if doc.tag == 'rss':
                 return self._extract_rss(url, video_id, doc)
             elif re.match(r'^(?:{[^}]+})?smil$', doc.tag):
-                return self._parse_smil(doc, url, video_id)
+                smil = self._parse_smil(doc, url, video_id)
+                self._sort_formats(smil['formats'])
+                return smil
             elif doc.tag == '{http://xspf.org/ns/0/}playlist':
                 return self.playlist_result(self._parse_xspf(doc, video_id), video_id)
             elif re.match(r'(?i)^(?:{[^}]+})?MPD$', doc.tag):
                 info_dict['formats'] = self._parse_mpd_formats(
                     doc, video_id, mpd_base_url=url.rpartition('/')[0])
+                self._sort_formats(info_dict['formats'])
                 return info_dict
             elif re.match(r'^{http://ns\.adobe\.com/f4m/[12]\.0}manifest$', doc.tag):
                 info_dict['formats'] = self._parse_f4m_formats(doc, url, video_id)
+                self._sort_formats(info_dict['formats'])
                 return info_dict
         except compat_xml_parse_error:
             pass
@@ -2053,6 +2059,9 @@ class GenericIE(InfoExtractor):
             else:
                 entry_info_dict['url'] = video_url
 
+            if entry_info_dict.get('formats'):
+                self._sort_formats(entry_info_dict['formats'])
+
             entries.append(entry_info_dict)
 
         if len(entries) == 1:
diff --git a/youtube_dl/extractor/laola1tv.py b/youtube_dl/extractor/laola1tv.py
index d9dc067d2..d4fbafece 100644
--- a/youtube_dl/extractor/laola1tv.py
+++ b/youtube_dl/extractor/laola1tv.py
@@ -130,6 +130,7 @@ class Laola1TvIE(InfoExtractor):
         formats = self._extract_f4m_formats(
             '%s?hdnea=%s&hdcore=3.2.0' % (token_attrib['url'], token_auth),
             video_id, f4m_id='hds')
+        self._sort_formats(formats)
 
         categories_str = _v('meta_sports')
         categories = categories_str.split(',') if categories_str else []
diff --git a/youtube_dl/extractor/lrt.py b/youtube_dl/extractor/lrt.py
index 863efd896..1072405b3 100644
--- a/youtube_dl/extractor/lrt.py
+++ b/youtube_dl/extractor/lrt.py
@@ -37,6 +37,7 @@ class LRTIE(InfoExtractor):
             r'file\s*:\s*(["\'])(?P<url>.+?)\1\s*\+\s*location\.hash\.substring\(1\)',
             webpage, 'm3u8 url', group='url')
         formats = self._extract_m3u8_formats(m3u8_url, video_id, 'mp4')
+        self._sort_formats(formats)
 
         thumbnail = self._og_search_thumbnail(webpage)
         description = self._og_search_description(webpage)
diff --git a/youtube_dl/extractor/matchtv.py b/youtube_dl/extractor/matchtv.py
index e33bfde3b..80a0d7013 100644
--- a/youtube_dl/extractor/matchtv.py
+++ b/youtube_dl/extractor/matchtv.py
@@ -47,6 +47,7 @@ class MatchTVIE(InfoExtractor):
         video_url = self._download_json(request, video_id)['data']['videoUrl']
         f4m_url = xpath_text(self._download_xml(video_url, video_id), './to')
         formats = self._extract_f4m_formats(f4m_url, video_id)
+        self._sort_formats(formats)
         return {
             'id': video_id,
             'title': self._live_title('Матч ТВ - Прямой эфир'),
diff --git a/youtube_dl/extractor/mitele.py b/youtube_dl/extractor/mitele.py
index 76ced7928..7b4581dc5 100644
--- a/youtube_dl/extractor/mitele.py
+++ b/youtube_dl/extractor/mitele.py
@@ -67,6 +67,7 @@ class MiTeleIE(InfoExtractor):
             formats.extend(self._extract_f4m_formats(
                 file_ + '&hdcore=3.2.0&plugin=aasp-3.2.0.77.18',
                 display_id, f4m_id=loc))
+        self._sort_formats(formats)
 
         title = self._search_regex(
             r'class="Destacado-text"[^>]*>\s*<strong>([^<]+)</strong>', webpage, 'title')
diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index 3b21fbd4d..9df200822 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -63,6 +63,7 @@ class NRKIE(InfoExtractor):
         if determine_ext(media_url) == 'f4m':
             formats = self._extract_f4m_formats(
                 media_url + '?hdcore=3.5.0&plugin=aasp-3.5.0.151.81', video_id, f4m_id='hds')
+            self._sort_formats(formats)
         else:
             formats = [{
                 'url': media_url,
diff --git a/youtube_dl/extractor/restudy.py b/youtube_dl/extractor/restudy.py
index b17c2bfc0..fd50065d4 100644
--- a/youtube_dl/extractor/restudy.py
+++ b/youtube_dl/extractor/restudy.py
@@ -31,6 +31,7 @@ class RestudyIE(InfoExtractor):
         formats = self._extract_smil_formats(
             'https://www.restudy.dk/awsmedia/SmilDirectory/video_%s.xml' % video_id,
             video_id)
+        self._sort_formats(formats)
 
         return {
             'id': video_id,
diff --git a/youtube_dl/extractor/rte.py b/youtube_dl/extractor/rte.py
index 042bc8dab..9c89974e7 100644
--- a/youtube_dl/extractor/rte.py
+++ b/youtube_dl/extractor/rte.py
@@ -49,6 +49,7 @@ class RteIE(InfoExtractor):
         # f4m_url = server + relative_url
         f4m_url = json_string['shows'][0]['media:group'][0]['rte:server'] + json_string['shows'][0]['media:group'][0]['url']
         f4m_formats = self._extract_f4m_formats(f4m_url, video_id)
+        self._sort_formats(f4m_formats)
 
         return {
             'id': video_id,
diff --git a/youtube_dl/extractor/rtve.py b/youtube_dl/extractor/rtve.py
index 08cd1ae6c..79af47715 100644
--- a/youtube_dl/extractor/rtve.py
+++ b/youtube_dl/extractor/rtve.py
@@ -209,6 +209,7 @@ class RTVELiveIE(InfoExtractor):
         png = self._download_webpage(png_url, video_id, 'Downloading url information')
         m3u8_url = _decrypt_url(png)
         formats = self._extract_m3u8_formats(m3u8_url, video_id, ext='mp4')
+        self._sort_formats(formats)
 
         return {
             'id': video_id,
diff --git a/youtube_dl/extractor/rtvnh.py b/youtube_dl/extractor/rtvnh.py
index 7c9d4b0cd..4896d09d6 100644
--- a/youtube_dl/extractor/rtvnh.py
+++ b/youtube_dl/extractor/rtvnh.py
@@ -38,6 +38,7 @@ class RTVNHIE(InfoExtractor):
                     item['file'], video_id, ext='mp4', entry_protocol='m3u8_native'))
             elif item.get('type') == '':
                 formats.append({'url': item['file']})
+        self._sort_formats(formats)
 
         return {
             'id': video_id,
diff --git a/youtube_dl/extractor/shahid.py b/youtube_dl/extractor/shahid.py
index b4433a689..d95ea06be 100644
--- a/youtube_dl/extractor/shahid.py
+++ b/youtube_dl/extractor/shahid.py
@@ -77,6 +77,7 @@ class ShahidIE(InfoExtractor):
             raise ExtractorError('This video is DRM protected.', expected=True)
 
         formats = self._extract_m3u8_formats(player['url'], video_id, 'mp4')
+        self._sort_formats(formats)
 
         video = self._download_json(
             '%s/%s/%s?%s' % (
diff --git a/youtube_dl/extractor/sportbox.py b/youtube_dl/extractor/sportbox.py
index 86d509ae5..4f0c66213 100644
--- a/youtube_dl/extractor/sportbox.py
+++ b/youtube_dl/extractor/sportbox.py
@@ -99,6 +99,7 @@ class SportBoxEmbedIE(InfoExtractor):
             webpage, 'hls file')
 
         formats = self._extract_m3u8_formats(hls, video_id, 'mp4')
+        self._sort_formats(formats)
 
         title = self._search_regex(
             r'sportboxPlayer\.node_title\s*=\s*"([^"]+)"', webpage, 'title')
diff --git a/youtube_dl/extractor/telecinco.py b/youtube_dl/extractor/telecinco.py
index d6b2560f8..4b4b740b4 100644
--- a/youtube_dl/extractor/telecinco.py
+++ b/youtube_dl/extractor/telecinco.py
@@ -82,6 +82,7 @@ class TelecincoIE(InfoExtractor):
         )
         formats = self._extract_m3u8_formats(
             token_info['tokenizedUrl'], episode, ext='mp4', entry_protocol='m3u8_native')
+        self._sort_formats(formats)
 
         return {
             'id': embed_data['videoId'],
diff --git a/youtube_dl/extractor/tubitv.py b/youtube_dl/extractor/tubitv.py
index 306ee4e15..7af233cd6 100644
--- a/youtube_dl/extractor/tubitv.py
+++ b/youtube_dl/extractor/tubitv.py
@@ -69,6 +69,7 @@ class TubiTvIE(InfoExtractor):
         apu = self._search_regex(r"apu='([^']+)'", webpage, 'apu')
         m3u8_url = codecs.decode(apu, 'rot_13')[::-1]
         formats = self._extract_m3u8_formats(m3u8_url, video_id, ext='mp4')
+        self._sort_formats(formats)
 
         return {
             'id': video_id,
diff --git a/youtube_dl/extractor/videomore.py b/youtube_dl/extractor/videomore.py
index 0bd1e1eec..04e95c66e 100644
--- a/youtube_dl/extractor/videomore.py
+++ b/youtube_dl/extractor/videomore.py
@@ -111,6 +111,7 @@ class VideomoreIE(InfoExtractor):
 
         video_url = xpath_text(video, './/video_url', 'video url', fatal=True)
         formats = self._extract_f4m_formats(video_url, video_id, f4m_id='hds')
+        self._sort_formats(formats)
 
         data = self._download_json(
             'http://videomore.ru/video/tracks/%s.json' % video_id,
diff --git a/youtube_dl/extractor/vier.py b/youtube_dl/extractor/vier.py
index c76c20614..6645c6186 100644
--- a/youtube_dl/extractor/vier.py
+++ b/youtube_dl/extractor/vier.py
@@ -50,6 +50,7 @@ class VierIE(InfoExtractor):
 
         playlist_url = 'http://vod.streamcloud.be/%s/mp4:_definst_/%s.mp4/playlist.m3u8' % (application, filename)
         formats = self._extract_m3u8_formats(playlist_url, display_id, 'mp4')
+        self._sort_formats(formats)
 
         title = self._og_search_title(webpage, default=display_id)
         description = self._og_search_description(webpage, default=None)
diff --git a/youtube_dl/extractor/viidea.py b/youtube_dl/extractor/viidea.py
index 03b9f1353..a4f914d14 100644
--- a/youtube_dl/extractor/viidea.py
+++ b/youtube_dl/extractor/viidea.py
@@ -151,6 +151,7 @@ class ViideaIE(InfoExtractor):
                 smil_url = '%s/%s/video/%s/smil.xml' % (base_url, lecture_slug, part_id)
                 smil = self._download_smil(smil_url, lecture_id)
                 info = self._parse_smil(smil, smil_url, lecture_id)
+                self._sort_formats(info['formats'])
                 info['id'] = lecture_id if not multipart else '%s_part%s' % (lecture_id, part_id)
                 info['display_id'] = lecture_slug if not multipart else '%s_part%s' % (lecture_slug, part_id)
                 if multipart:
diff --git a/youtube_dl/extractor/ynet.py b/youtube_dl/extractor/ynet.py
index 2522551dc..0d943c343 100644
--- a/youtube_dl/extractor/ynet.py
+++ b/youtube_dl/extractor/ynet.py
@@ -41,10 +41,12 @@ class YnetIE(InfoExtractor):
         m = re.search(r'ynet - HOT -- (["\']+)(?P<title>.+?)\1', title)
         if m:
             title = m.group('title')
+        formats = self._extract_f4m_formats(f4m_url, video_id)
+        self._sort_formats(formats)
 
         return {
             'id': video_id,
             'title': title,
-            'formats': self._extract_f4m_formats(f4m_url, video_id),
+            'formats': formats,
             'thumbnail': self._og_search_thumbnail(webpage),
         }

From f7df343b4a0223698f0a5320b850410d7a42be6f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 27 Mar 2016 07:41:19 +0600
Subject: [PATCH 4446/6219] [downloader/f4m] Extract routine for removing
 unsupported encrypted media

---
 youtube_dl/downloader/f4m.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index fc9642905..664d87543 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -223,6 +223,12 @@ def write_metadata_tag(stream, metadata):
         write_unsigned_int(stream, FLV_TAG_HEADER_LEN + len(metadata))
 
 
+def remove_encrypted_media(media):
+    return list(filter(lambda e: 'drmAdditionalHeaderId' not in e.attrib and
+                                 'drmAdditionalHeaderSetId' not in e.attrib,
+                       media))
+
+
 def _add_ns(prop):
     return '{http://ns.adobe.com/f4m/1.0}%s' % prop
 
@@ -244,9 +250,7 @@ class F4mFD(FragmentFD):
             # without drmAdditionalHeaderId or drmAdditionalHeaderSetId attribute
             if 'id' not in e.attrib:
                 self.report_error('Missing ID in f4m DRM')
-        media = list(filter(lambda e: 'drmAdditionalHeaderId' not in e.attrib and
-                                      'drmAdditionalHeaderSetId' not in e.attrib,
-                            media))
+        media = remove_encrypted_media(media)
         if not media:
             self.report_error('Unsupported DRM')
         return media

From b22ca76204e1a05e1c4b07d24cb6a0dbbc09d18e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 27 Mar 2016 07:42:38 +0600
Subject: [PATCH 4447/6219] [extractor/common] Filter out unsupported encrypted
 media for f4m formats (Closes #8573)

---
 youtube_dl/extractor/common.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 40ddf175c..9b7ab8924 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -24,6 +24,7 @@ from ..compat import (
     compat_urllib_parse_urlencode,
     compat_urlparse,
 )
+from ..downloader.f4m import remove_encrypted_media
 from ..utils import (
     NO_DEFAULT,
     age_restricted,
@@ -989,6 +990,11 @@ class InfoExtractor(object):
         if not media_nodes:
             manifest_version = '2.0'
             media_nodes = manifest.findall('{http://ns.adobe.com/f4m/2.0}media')
+        # Remove unsupported DRM protected media from final formats
+        # rendition (see https://github.com/rg3/youtube-dl/issues/8573).
+        media_nodes = remove_encrypted_media(media_nodes)
+        if not media_nodes:
+            return formats
         base_url = xpath_text(
             manifest, ['{http://ns.adobe.com/f4m/1.0}baseURL', '{http://ns.adobe.com/f4m/2.0}baseURL'],
             'base URL', default=None)

From a122e7080bbcc505d638eaef8ab4d1e4f5bd91ee Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 27 Mar 2016 16:56:33 +0200
Subject: [PATCH 4448/6219] release 2016.03.27

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index d4cf099f5..5daa7f4e8 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.03.26'
+__version__ = '2016.03.27'

From a1cf3e38a34caa333fd9703333ef55e0b3ac5a17 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 27 Mar 2016 23:22:51 +0600
Subject: [PATCH 4449/6219] [bbc] Extend vpid regex (Closes #9003)

---
 youtube_dl/extractor/bbc.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 2dfcee98d..dedf721bd 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -688,6 +688,10 @@ class BBCIE(BBCCoUkIE):
         # custom redirection to www.bbc.com
         'url': 'http://www.bbc.co.uk/news/science-environment-33661876',
         'only_matching': True,
+    }, {
+        # single video article embedded with data-media-vpid
+        'url': 'http://www.bbc.co.uk/sport/rowing/35908187',
+        'only_matching': True,
     }]
 
     @classmethod
@@ -817,7 +821,7 @@ class BBCIE(BBCCoUkIE):
 
         # single video story (e.g. http://www.bbc.com/travel/story/20150625-sri-lankas-spicy-secret)
         programme_id = self._search_regex(
-            [r'data-video-player-vpid="(%s)"' % self._ID_REGEX,
+            [r'data-(?:video-player|media)-vpid="(%s)"' % self._ID_REGEX,
              r'<param[^>]+name="externalIdentifier"[^>]+value="(%s)"' % self._ID_REGEX,
              r'videoId\s*:\s*["\'](%s)["\']' % self._ID_REGEX],
             webpage, 'vpid', default=None)

From 8f9a477e7f260d60836843fbe8f75629e3ae8892 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 28 Mar 2016 00:21:08 +0600
Subject: [PATCH 4450/6219] [pornhub:playlistbase] Use orderedSet

---
 youtube_dl/extractor/pornhub.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index 5a55c25e7..670e9294a 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -12,6 +12,7 @@ from ..compat import (
 from ..utils import (
     ExtractorError,
     int_or_none,
+    orderedSet,
     sanitized_Request,
     str_to_int,
 )
@@ -150,7 +151,7 @@ class PornHubPlaylistBaseIE(InfoExtractor):
     def _extract_entries(self, webpage):
         return [
             self.url_result('http://www.pornhub.com/%s' % video_url, PornHubIE.ie_key())
-            for video_url in set(re.findall(
+            for video_url in orderedSet(re.findall(
                 r'href="/?(view_video\.php\?.*\bviewkey=[\da-z]+[^"]*)"', webpage))
         ]
 

From 3a23bae9ccf11c9c114d2d27e4fbc09fb0bbeafe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 28 Mar 2016 00:32:57 +0600
Subject: [PATCH 4451/6219] [pornhub:playlistbase] Do not include videos not
 from playlist

---
 youtube_dl/extractor/pornhub.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index 670e9294a..b3bf81a13 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -150,9 +150,12 @@ class PornHubIE(InfoExtractor):
 class PornHubPlaylistBaseIE(InfoExtractor):
     def _extract_entries(self, webpage):
         return [
-            self.url_result('http://www.pornhub.com/%s' % video_url, PornHubIE.ie_key())
-            for video_url in orderedSet(re.findall(
-                r'href="/?(view_video\.php\?.*\bviewkey=[\da-z]+[^"]*)"', webpage))
+            self.url_result(
+                'http://www.pornhub.com/%s' % video_url,
+                PornHubIE.ie_key(), video_title=title)
+            for video_url, title in orderedSet(re.findall(
+                r'href="/?(view_video\.php\?.*\bviewkey=[\da-z]+[^"]*)"[^>]*\s+title="([^"]+)"',
+                webpage))
         ]
 
     def _real_extract(self, url):

From 3454139576ad98b62162ba0a9bca4b342c5d07ce Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 28 Mar 2016 00:50:46 +0600
Subject: [PATCH 4452/6219] [pornhub:uservideos] Add support for multipage
 videos (Closes #9006)

---
 youtube_dl/extractor/pornhub.py | 27 ++++++++++++++++++++++-----
 1 file changed, 22 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index b3bf81a13..ac298d0ce 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -1,10 +1,12 @@
 from __future__ import unicode_literals
 
+import itertools
 import os
 import re
 
 from .common import InfoExtractor
 from ..compat import (
+    compat_HTTPError,
     compat_urllib_parse_unquote,
     compat_urllib_parse_unquote_plus,
     compat_urllib_parse_urlparse,
@@ -189,16 +191,31 @@ class PornHubPlaylistIE(PornHubPlaylistBaseIE):
 class PornHubUserVideosIE(PornHubPlaylistBaseIE):
     _VALID_URL = r'https?://(?:www\.)?pornhub\.com/users/(?P<id>[^/]+)/videos'
     _TESTS = [{
-        'url': 'http://www.pornhub.com/users/rushandlia/videos',
+        'url': 'http://www.pornhub.com/users/zoe_ph/videos/public',
         'info_dict': {
-            'id': 'rushandlia',
+            'id': 'zoe_ph',
         },
-        'playlist_mincount': 13,
+        'playlist_mincount': 171,
+    }, {
+        'url': 'http://www.pornhub.com/users/rushandlia/videos',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
         user_id = self._match_id(url)
 
-        webpage = self._download_webpage(url, user_id)
+        entries = []
+        for page_num in itertools.count(1):
+            try:
+                webpage = self._download_webpage(
+                    url, user_id, 'Downloading page %d' % page_num,
+                    query={'page': page_num})
+            except ExtractorError as e:
+                if isinstance(e.cause, compat_HTTPError) and e.cause.code == 404:
+                    break
+            page_entries = self._extract_entries(webpage)
+            if not page_entries:
+                break
+            entries.extend(page_entries)
 
-        return self.playlist_result(self._extract_entries(webpage), user_id)
+        return self.playlist_result(entries, user_id)

From 87d105ac6c90b4dad519de7d013623923d74d570 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 28 Mar 2016 01:13:47 +0600
Subject: [PATCH 4453/6219] [amp] Fix upload timestamp extraction (Closes
 #9007)

---
 youtube_dl/extractor/amp.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/amp.py b/youtube_dl/extractor/amp.py
index 69e6baff7..138fa0808 100644
--- a/youtube_dl/extractor/amp.py
+++ b/youtube_dl/extractor/amp.py
@@ -69,12 +69,14 @@ class AMPIE(InfoExtractor):
 
         self._sort_formats(formats)
 
+        timestamp = parse_iso8601(item.get('pubDate'), ' ') or parse_iso8601(item.get('dc-date'))
+
         return {
             'id': video_id,
             'title': get_media_node('title'),
             'description': get_media_node('description'),
             'thumbnails': thumbnails,
-            'timestamp': parse_iso8601(item.get('pubDate'), ' '),
+            'timestamp': timestamp,
             'duration': int_or_none(media_content[0].get('@attributes', {}).get('duration')),
             'subtitles': subtitles,
             'formats': formats,

From c8b13fec025bcb7402656095df369ad3f3225ac6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 28 Mar 2016 01:14:12 +0600
Subject: [PATCH 4454/6219] [foxnews] Restore upload time fields in test

---
 youtube_dl/extractor/foxnews.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/foxnews.py b/youtube_dl/extractor/foxnews.py
index 1dc50318c..b04da2415 100644
--- a/youtube_dl/extractor/foxnews.py
+++ b/youtube_dl/extractor/foxnews.py
@@ -18,8 +18,8 @@ class FoxNewsIE(AMPIE):
                 'title': 'Frozen in Time',
                 'description': '16-year-old girl is size of toddler',
                 'duration': 265,
-                # 'timestamp': 1304411491,
-                # 'upload_date': '20110503',
+                'timestamp': 1304411491,
+                'upload_date': '20110503',
                 'thumbnail': 're:^https?://.*\.jpg$',
             },
         },
@@ -32,8 +32,8 @@ class FoxNewsIE(AMPIE):
                 'title': "Rep. Luis Gutierrez on if Obama's immigration plan is legal",
                 'description': "Congressman discusses president's plan",
                 'duration': 292,
-                # 'timestamp': 1417662047,
-                # 'upload_date': '20141204',
+                'timestamp': 1417662047,
+                'upload_date': '20141204',
                 'thumbnail': 're:^https?://.*\.jpg$',
             },
             'params': {

From 03442072c0890f10043d1de25dc3c3fcaf10f4eb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 28 Mar 2016 01:21:44 +0600
Subject: [PATCH 4455/6219] [pornhub] Fix typo (Closes #9008)

---
 youtube_dl/extractor/pornhub.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index ac298d0ce..407ea08d4 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -78,7 +78,7 @@ class PornHubIE(InfoExtractor):
 
         flashvars = self._parse_json(
             self._search_regex(
-                r'var\s+flashv1ars_\d+\s*=\s*({.+?});', webpage, 'flashvars', default='{}'),
+                r'var\s+flashvars_\d+\s*=\s*({.+?});', webpage, 'flashvars', default='{}'),
             video_id)
         if flashvars:
             video_title = flashvars.get('video_title')

From 33f3040a3e611f45ad920bd06030691910ddf815 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 28 Mar 2016 03:13:39 +0600
Subject: [PATCH 4456/6219] [YoutubeDL] Fix sanitizing subtitles' url

---
 youtube_dl/YoutubeDL.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 53a36c145..d7aa951ff 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1265,7 +1265,8 @@ class YoutubeDL(object):
         if subtitles:
             for _, subtitle in subtitles.items():
                 for subtitle_format in subtitle:
-                    subtitle_format['url'] = sanitize_url(subtitle_format['url'])
+                    if subtitle_format.get('url'):
+                        subtitle_format['url'] = sanitize_url(subtitle_format['url'])
                     if 'ext' not in subtitle_format:
                         subtitle_format['ext'] = determine_ext(subtitle_format['url']).lower()
 

From 8d9dd3c34bd38b2545af95f8ef670b07ae1fb6ff Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Mon, 28 Mar 2016 03:08:34 +0500
Subject: [PATCH 4457/6219] [README.md] Add format_id to the list of string
 meta fields available for use in format selection

---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index 52b2a85a3..e972bf69f 100644
--- a/README.md
+++ b/README.md
@@ -600,6 +600,7 @@ Also filtering work for comparisons `=` (equals), `!=` (not equals), `^=` (begin
  - `vcodec`: Name of the video codec in use
  - `container`: Name of the container format
  - `protocol`: The protocol that will be used for the actual download, lower-case. `http`, `https`, `rtsp`, `rtmp`, `rtmpe`, `m3u8`, or `m3u8_native`
+ - `format_id`: A short description of the format
 
 Note that none of the aforementioned meta fields are guaranteed to be present since this solely depends on the metadata obtained by particular extractor, i.e. the metadata offered by video hoster.
 

From 7710bdf4e813879dbd8c5857e13a2c64e0ce8837 Mon Sep 17 00:00:00 2001
From: Sander van den Oever <sandervdo@gmail.com>
Date: Sun, 6 Mar 2016 23:16:13 +0100
Subject: [PATCH 4458/6219] Add initial ISSUE_TEMPLATE

Add auto-updating of youtube-dl version in ISSUE_TEMPLATE

Move parts of template text and adopt makefile to new format

Moved the 'kind-of-issue' section and rephrased a bit

Rephrased and moved Example URL section upwards

Moved ISSUE_TEMPLATE inside .github folder.

Update makefile to match new folderstructure
---
 .github/ISSUE_TEMPLATE.md         | 37 +++++++++++++++++++++++++++++++
 Makefile                          |  5 ++++-
 devscripts/make_issue_template.py | 32 ++++++++++++++++++++++++++
 3 files changed, 73 insertions(+), 1 deletion(-)
 create mode 100644 .github/ISSUE_TEMPLATE.md
 create mode 100644 devscripts/make_issue_template.py

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
new file mode 100644
index 000000000..c34cbe743
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE.md
@@ -0,0 +1,37 @@
+Make sure you are using the **latest** version of youtube-dl. Latest youtube-dl version at this moment is **2016.03.06**. Read [our FAQ](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) if you have troubles updating.
+- [ ] I've verified that I'm running youtube-dl version **2016.03.06**
+
+**What is the purpose of this issue? Put an `x` to the relevant box**
+- [ ] Site support request (add support for a new website)
+- [ ] Feature request (request new functionality)
+- [ ] Bug report (encountered problems with youtube-dl)
+- [ ] Other, namely ...
+
+**If the purpose of this issues is a site support request please provide _at least_ one example URL of a video or a playlist you are trying to download.**
+
+- http://some.example.url/to-video
+
+*If the purpose of this issue is a bug report or you are unsure about its relevance please include a log as described below.*
+
+**Please include the full output of youtube-dl when run with `-v`**, i.e. **add** `-v` flag to **your command line**, copy the **whole** output and post it below wrapped in ``` for better formatting. It should look similar to this:
+```
+$ youtube-dl -v <your command line>
+[debug] System config: []
+[debug] User config: []
+[debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
+[debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
+[debug] youtube-dl version 2015.12.06
+[debug] Git HEAD: 135392e
+[debug] Python version 2.6.6 - Windows-2003Server-5.2.3790-SP2
+[debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
+[debug] Proxy map: {}
+...
+```
+
+**Brief description of the problem/request**
+
+*I am having a problem with ... I have tried to do ... and ... I expected that ... would happen, but instead ... happened. Example: I tried to download a file but the site was not supported. Please add support for site xyz. Another example: I encountered a bug when downloading a video from xyz. I have tried to do a and b.*
+
+**Suggested solution or other information**
+
+*In case you have suggestions for a solution or any other relevant information you can write it here*
diff --git a/Makefile b/Makefile
index 6689ec06f..bfbe5e6cb 100644
--- a/Makefile
+++ b/Makefile
@@ -1,4 +1,4 @@
-all: youtube-dl README.md CONTRIBUTING.md README.txt youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtube-dl.fish supportedsites
+all: youtube-dl README.md CONTRIBUTING.md issue_template README.txt youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtube-dl.fish supportedsites
 
 clean:
 	rm -rf youtube-dl.1.temp.md youtube-dl.1 youtube-dl.bash-completion README.txt MANIFEST build/ dist/ .coverage cover/ youtube-dl.tar.gz youtube-dl.zsh youtube-dl.fish *.dump *.part *.info.json *.mp4 *.flv *.mp3 *.avi CONTRIBUTING.md.tmp youtube-dl youtube-dl.exe
@@ -59,6 +59,9 @@ README.md: youtube_dl/*.py youtube_dl/*/*.py
 CONTRIBUTING.md: README.md
 	$(PYTHON) devscripts/make_contributing.py README.md CONTRIBUTING.md
 
+issue_template: .github/ISSUE_TEMPLATE.md youtube_dl/version.py
+	$(PYTHON) devscripts/make_issue_template.py .github/ISSUE_TEMPLATE.md
+
 supportedsites:
 	$(PYTHON) devscripts/make_supportedsites.py docs/supportedsites.md
 
diff --git a/devscripts/make_issue_template.py b/devscripts/make_issue_template.py
new file mode 100644
index 000000000..2fdd05035
--- /dev/null
+++ b/devscripts/make_issue_template.py
@@ -0,0 +1,32 @@
+#!/usr/bin/env python
+from __future__ import unicode_literals
+
+import io
+import optparse
+import re
+
+
+def main():
+    parser = optparse.OptionParser(usage='%prog FILE')
+    options, args = parser.parse_args()
+    if len(args) != 1:
+        parser.error('Expected an filename')
+
+    with io.open(args[0], encoding='utf-8') as inf:
+        issue_template_text = inf.read()
+
+    # Get the version from youtube_dl/version.py without importing the package
+    exec(compile(open('youtube_dl/version.py').read(),
+         'youtube_dl/version.py', 'exec'))
+
+    issue_template_text = re.sub(
+        r'(?<=\*\*)(?P<version>[0-9\.]+)(?=\*\*)',
+        __version__,
+        issue_template_text
+    )
+
+    with io.open(args[0], 'w', encoding='utf-8') as outf:
+         outf.write(issue_template_text)
+
+if __name__ == '__main__':
+    main()

From 3842a3e6524c8704d4295f65e6c0bce578d69e93 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 29 Mar 2016 02:15:26 +0600
Subject: [PATCH 4459/6219] Add ISSUE_TEMPLATE.tmpl as template for
 ISSUE_TEMPLATE.md

---
 .github/ISSUE_TEMPLATE.tmpl | 43 +++++++++++++++++++++++++++++++++++++
 1 file changed, 43 insertions(+)
 create mode 100644 .github/ISSUE_TEMPLATE.tmpl

diff --git a/.github/ISSUE_TEMPLATE.tmpl b/.github/ISSUE_TEMPLATE.tmpl
new file mode 100644
index 000000000..9bca5b531
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE.tmpl
@@ -0,0 +1,43 @@
+### Please follow the guide below, provide requested information and put an `x` into all the boxes [ ] relevant to your *issue*.
+
+#### :heavy_exclamation_mark: IMPORTANT :heavy_exclamation_mark: Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *%(version)s*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **%(version)s**
+
+#### Before submitting an *issue* make sure you have:
+- [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
+- [ ] [Searched](https://github.com/rg3/youtube-dl/search?type=Issues) the bugtracker for similar issues including closed ones
+
+#### What is the purpose of your *issue*?
+- [ ] Bug report (encountered problems with youtube-dl)
+- [ ] Site support request (request for adding support for a new site)
+- [ ] Feature request (request for a new functionality)
+- [ ] Question
+- [ ] Other
+
+#### The following sections concretize particular purposed issues, you can erase any section not applicable to your *issue*.
+
+#### :heavy_exclamation_mark: IMPORTANT :heavy_exclamation_mark: If the purpose of this *issue* is a *bug report*, *site support request* or you are not completely sure provide the full verbose output as follows:
+
+Add `-v` flag to **your command line** you run youtube-dl with, copy the **whole** output and insert it here. It should look similar to one below (replace it with **your** log inserted between triple ```):
+```
+$ youtube-dl -v <your command line>
+[debug] System config: []
+[debug] User config: []
+[debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
+[debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
+[debug] youtube-dl version %(version)s
+[debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
+[debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
+[debug] Proxy map: {}
+...
+<end of log>
+```
+
+#### If the purpose of this *issue* is a *site support request* please provide all kinds of example URLs support for which should be included (replace following example URLs by **yours**):
+- Single video: https://www.youtube.com/watch?v=BaW_jenozKc
+- Single video: https://youtu.be/BaW_jenozKc
+- Playlist: https://www.youtube.com/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc
+
+#### Description of your *issue*, suggested solution and other information
+
+Explanation of your *issue* in arbitrary form goes here. Please make sure the [description is worded well enough to be understood](https://github.com/rg3/youtube-dl#is-the-description-of-the-issue-itself-sufficient). Provide as much context and examples as possible.

From 3bf1df51fda4189eaa9164134b56393e2c4a7f72 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 29 Mar 2016 02:16:38 +0600
Subject: [PATCH 4460/6219] [devscripts/make_issue_template] Rework to use
 ISSUE_TEMPLATE.tmpl (Closes #8785)

---
 devscripts/make_issue_template.py | 25 +++++++++++--------------
 1 file changed, 11 insertions(+), 14 deletions(-)

diff --git a/devscripts/make_issue_template.py b/devscripts/make_issue_template.py
index 2fdd05035..e5564bac1 100644
--- a/devscripts/make_issue_template.py
+++ b/devscripts/make_issue_template.py
@@ -3,30 +3,27 @@ from __future__ import unicode_literals
 
 import io
 import optparse
-import re
 
 
 def main():
-    parser = optparse.OptionParser(usage='%prog FILE')
+    parser = optparse.OptionParser(usage='%prog INFILE OUTFILE')
     options, args = parser.parse_args()
-    if len(args) != 1:
-        parser.error('Expected an filename')
+    if len(args) != 2:
+        parser.error('Expected an input and an output filename')
 
-    with io.open(args[0], encoding='utf-8') as inf:
-        issue_template_text = inf.read()
+    infile, outfile = args
+
+    with io.open(infile, encoding='utf-8') as inf:
+        issue_template_tmpl = inf.read()
 
     # Get the version from youtube_dl/version.py without importing the package
     exec(compile(open('youtube_dl/version.py').read(),
-         'youtube_dl/version.py', 'exec'))
+                 'youtube_dl/version.py', 'exec'))
 
-    issue_template_text = re.sub(
-        r'(?<=\*\*)(?P<version>[0-9\.]+)(?=\*\*)',
-        __version__,
-        issue_template_text
-    )
+    out = issue_template_tmpl % {'version': __version__}
 
-    with io.open(args[0], 'w', encoding='utf-8') as outf:
-         outf.write(issue_template_text)
+    with io.open(outfile, 'w', encoding='utf-8') as outf:
+        outf.write(out)
 
 if __name__ == '__main__':
     main()

From 8751da85a7d21702132091986bc6224d3a3af319 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 29 Mar 2016 02:17:57 +0600
Subject: [PATCH 4461/6219] [Makefile] Fix ISSUE_TEMPLATE.md target

---
 Makefile | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/Makefile b/Makefile
index bfbe5e6cb..9e3ce78c8 100644
--- a/Makefile
+++ b/Makefile
@@ -1,7 +1,7 @@
-all: youtube-dl README.md CONTRIBUTING.md issue_template README.txt youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtube-dl.fish supportedsites
+all: youtube-dl README.md CONTRIBUTING.md ISSUE_TEMPLATE.md README.txt youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtube-dl.fish supportedsites
 
 clean:
-	rm -rf youtube-dl.1.temp.md youtube-dl.1 youtube-dl.bash-completion README.txt MANIFEST build/ dist/ .coverage cover/ youtube-dl.tar.gz youtube-dl.zsh youtube-dl.fish *.dump *.part *.info.json *.mp4 *.flv *.mp3 *.avi CONTRIBUTING.md.tmp youtube-dl youtube-dl.exe
+	rm -rf youtube-dl.1.temp.md youtube-dl.1 youtube-dl.bash-completion README.txt MANIFEST build/ dist/ .coverage cover/ youtube-dl.tar.gz youtube-dl.zsh youtube-dl.fish *.dump *.part *.info.json *.mp4 *.flv *.mp3 *.avi CONTRIBUTING.md.tmp ISSUE_TEMPLATE.md.tmp youtube-dl youtube-dl.exe
 	find . -name "*.pyc" -delete
 	find . -name "*.class" -delete
 
@@ -59,8 +59,8 @@ README.md: youtube_dl/*.py youtube_dl/*/*.py
 CONTRIBUTING.md: README.md
 	$(PYTHON) devscripts/make_contributing.py README.md CONTRIBUTING.md
 
-issue_template: .github/ISSUE_TEMPLATE.md youtube_dl/version.py
-	$(PYTHON) devscripts/make_issue_template.py .github/ISSUE_TEMPLATE.md
+ISSUE_TEMPLATE.md:
+	$(PYTHON) devscripts/make_issue_template.py .github/ISSUE_TEMPLATE.tmpl .github/ISSUE_TEMPLATE.md
 
 supportedsites:
 	$(PYTHON) devscripts/make_supportedsites.py docs/supportedsites.md

From 4a5a67ca2503165fc4b7c2ca9c881b79101245b5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 29 Mar 2016 02:18:52 +0600
Subject: [PATCH 4462/6219] [devscripts/release.sh] Make ISSUE_TEMPLATE.md and
 commit it

---
 devscripts/release.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/devscripts/release.sh b/devscripts/release.sh
index 61806961c..6718ce39b 100755
--- a/devscripts/release.sh
+++ b/devscripts/release.sh
@@ -45,9 +45,9 @@ fi
 /bin/echo -e "\n### Changing version in version.py..."
 sed -i "s/__version__ = '.*'/__version__ = '$version'/" youtube_dl/version.py
 
-/bin/echo -e "\n### Committing documentation and youtube_dl/version.py..."
-make README.md CONTRIBUTING.md supportedsites
-git add README.md CONTRIBUTING.md docs/supportedsites.md youtube_dl/version.py
+/bin/echo -e "\n### Committing documentation, templates and youtube_dl/version.py..."
+make README.md CONTRIBUTING.md ISSUE_TEMPLATE.md supportedsites
+git add README.md CONTRIBUTING.md .github/ISSUE_TEMPLATE.md docs/supportedsites.md youtube_dl/version.py
 git commit -m "release $version"
 
 /bin/echo -e "\n### Now tagging, signing and pushing..."

From a39c68f7e5a83d6bb3b1c4eb7f856b764a5488bf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 29 Mar 2016 02:19:24 +0600
Subject: [PATCH 4463/6219] Exclude make_issue_template.py from flake8

---
 setup.cfg | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.cfg b/setup.cfg
index 26857750c..5760112d4 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -2,5 +2,5 @@
 universal = True
 
 [flake8]
-exclude = youtube_dl/extractor/__init__.py,devscripts/buildserver.py,setup.py,build,.git
+exclude = youtube_dl/extractor/__init__.py,devscripts/buildserver.py,devscripts/make_issue_template.py,setup.py,build,.git
 ignore = E402,E501,E731

From 89924f8230987f9d6405d6487a7914197d42b12e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 29 Mar 2016 02:34:12 +0600
Subject: [PATCH 4464/6219] [devscripts/make_issue_template] Fix NameError
 under python3

---
 devscripts/make_issue_template.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/devscripts/make_issue_template.py b/devscripts/make_issue_template.py
index e5564bac1..eed4b3a00 100644
--- a/devscripts/make_issue_template.py
+++ b/devscripts/make_issue_template.py
@@ -16,6 +16,8 @@ def main():
     with io.open(infile, encoding='utf-8') as inf:
         issue_template_tmpl = inf.read()
 
+    __version__ = None
+
     # Get the version from youtube_dl/version.py without importing the package
     exec(compile(open('youtube_dl/version.py').read(),
                  'youtube_dl/version.py', 'exec'))

From 3c0de33ad758c2f16f0abb1bb594c79f4cb40593 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 29 Mar 2016 02:43:48 +0600
Subject: [PATCH 4465/6219] Remove ISSUE_TEMPLATE.md

---
 .github/ISSUE_TEMPLATE.md | 37 -------------------------------------
 1 file changed, 37 deletions(-)
 delete mode 100644 .github/ISSUE_TEMPLATE.md

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
deleted file mode 100644
index c34cbe743..000000000
--- a/.github/ISSUE_TEMPLATE.md
+++ /dev/null
@@ -1,37 +0,0 @@
-Make sure you are using the **latest** version of youtube-dl. Latest youtube-dl version at this moment is **2016.03.06**. Read [our FAQ](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) if you have troubles updating.
-- [ ] I've verified that I'm running youtube-dl version **2016.03.06**
-
-**What is the purpose of this issue? Put an `x` to the relevant box**
-- [ ] Site support request (add support for a new website)
-- [ ] Feature request (request new functionality)
-- [ ] Bug report (encountered problems with youtube-dl)
-- [ ] Other, namely ...
-
-**If the purpose of this issues is a site support request please provide _at least_ one example URL of a video or a playlist you are trying to download.**
-
-- http://some.example.url/to-video
-
-*If the purpose of this issue is a bug report or you are unsure about its relevance please include a log as described below.*
-
-**Please include the full output of youtube-dl when run with `-v`**, i.e. **add** `-v` flag to **your command line**, copy the **whole** output and post it below wrapped in ``` for better formatting. It should look similar to this:
-```
-$ youtube-dl -v <your command line>
-[debug] System config: []
-[debug] User config: []
-[debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
-[debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2015.12.06
-[debug] Git HEAD: 135392e
-[debug] Python version 2.6.6 - Windows-2003Server-5.2.3790-SP2
-[debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
-[debug] Proxy map: {}
-...
-```
-
-**Brief description of the problem/request**
-
-*I am having a problem with ... I have tried to do ... and ... I expected that ... would happen, but instead ... happened. Example: I tried to download a file but the site was not supported. Please add support for site xyz. Another example: I encountered a bug when downloading a video from xyz. I have tried to do a and b.*
-
-**Suggested solution or other information**
-
-*In case you have suggestions for a solution or any other relevant information you can write it here*

From 034947dd1eed9e7d61671c48844b3f77a4683e77 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 29 Mar 2016 02:48:04 +0600
Subject: [PATCH 4466/6219] Rename ISSUE_TEMPLATE.tmpl in order not to be
 picked up by github

---
 .github/{ISSUE_TEMPLATE.tmpl => ISSUE_TEMPLATE_tmpl.md} | 0
 Makefile                                                | 2 +-
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename .github/{ISSUE_TEMPLATE.tmpl => ISSUE_TEMPLATE_tmpl.md} (100%)

diff --git a/.github/ISSUE_TEMPLATE.tmpl b/.github/ISSUE_TEMPLATE_tmpl.md
similarity index 100%
rename from .github/ISSUE_TEMPLATE.tmpl
rename to .github/ISSUE_TEMPLATE_tmpl.md
diff --git a/Makefile b/Makefile
index 9e3ce78c8..3a6c37944 100644
--- a/Makefile
+++ b/Makefile
@@ -60,7 +60,7 @@ CONTRIBUTING.md: README.md
 	$(PYTHON) devscripts/make_contributing.py README.md CONTRIBUTING.md
 
 ISSUE_TEMPLATE.md:
-	$(PYTHON) devscripts/make_issue_template.py .github/ISSUE_TEMPLATE.tmpl .github/ISSUE_TEMPLATE.md
+	$(PYTHON) devscripts/make_issue_template.py .github/ISSUE_TEMPLATE_tmpl.md .github/ISSUE_TEMPLATE.md
 
 supportedsites:
 	$(PYTHON) devscripts/make_supportedsites.py docs/supportedsites.md

From 7168a6c874c1c0e4cffb6c1e29eeda322051def7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 29 Mar 2016 03:05:15 +0600
Subject: [PATCH 4467/6219] [devscripts/make_issue_template] Fix __version__
 again

---
 devscripts/make_issue_template.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/devscripts/make_issue_template.py b/devscripts/make_issue_template.py
index eed4b3a00..b7ad23d83 100644
--- a/devscripts/make_issue_template.py
+++ b/devscripts/make_issue_template.py
@@ -16,13 +16,11 @@ def main():
     with io.open(infile, encoding='utf-8') as inf:
         issue_template_tmpl = inf.read()
 
-    __version__ = None
-
     # Get the version from youtube_dl/version.py without importing the package
     exec(compile(open('youtube_dl/version.py').read(),
                  'youtube_dl/version.py', 'exec'))
 
-    out = issue_template_tmpl % {'version': __version__}
+    out = issue_template_tmpl % {'version': locals()['__version__']}
 
     with io.open(outfile, 'w', encoding='utf-8') as outf:
         outf.write(out)

From 0b7bfc94221bbdb79fd4602643891c8c9c59292f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 29 Mar 2016 22:02:42 +0600
Subject: [PATCH 4468/6219] Improve ISSUE_TEMPLATE_tmpl.md

---
 .github/ISSUE_TEMPLATE_tmpl.md | 31 +++++++++++++++++++++++--------
 1 file changed, 23 insertions(+), 8 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE_tmpl.md b/.github/ISSUE_TEMPLATE_tmpl.md
index 9bca5b531..a5e6a4233 100644
--- a/.github/ISSUE_TEMPLATE_tmpl.md
+++ b/.github/ISSUE_TEMPLATE_tmpl.md
@@ -1,22 +1,32 @@
-### Please follow the guide below, provide requested information and put an `x` into all the boxes [ ] relevant to your *issue*.
+## Please follow the guide below
 
-#### :heavy_exclamation_mark: IMPORTANT :heavy_exclamation_mark: Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *%(version)s*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- You will be asked some questions and requested to provide some information, please read them **carefully** and answer honestly
+- Put an `x` into all the boxes [ ] relevant to your *issue* (like that [x])
+- Use *Preview* tab to see how your issue will actually look like
+
+---
+
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *%(version)s*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
 - [ ] I've **verified** and **I assure** that I'm running youtube-dl **%(version)s**
 
-#### Before submitting an *issue* make sure you have:
+### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
 - [ ] [Searched](https://github.com/rg3/youtube-dl/search?type=Issues) the bugtracker for similar issues including closed ones
 
-#### What is the purpose of your *issue*?
+### What is the purpose of your *issue*?
 - [ ] Bug report (encountered problems with youtube-dl)
 - [ ] Site support request (request for adding support for a new site)
 - [ ] Feature request (request for a new functionality)
 - [ ] Question
 - [ ] Other
 
-#### The following sections concretize particular purposed issues, you can erase any section not applicable to your *issue*.
+---
 
-#### :heavy_exclamation_mark: IMPORTANT :heavy_exclamation_mark: If the purpose of this *issue* is a *bug report*, *site support request* or you are not completely sure provide the full verbose output as follows:
+### The following sections concretize particular purposed issues, you can erase any section (the contents between triple ---) not applicable to your *issue*
+
+---
+
+### If the purpose of this *issue* is a *bug report*, *site support request* or you are not completely sure provide the full verbose output as follows:
 
 Add `-v` flag to **your command line** you run youtube-dl with, copy the **whole** output and insert it here. It should look similar to one below (replace it with **your** log inserted between triple ```):
 ```
@@ -33,11 +43,16 @@ $ youtube-dl -v <your command line>
 <end of log>
 ```
 
-#### If the purpose of this *issue* is a *site support request* please provide all kinds of example URLs support for which should be included (replace following example URLs by **yours**):
+---
+
+### If the purpose of this *issue* is a *site support request* please provide all kinds of example URLs support for which should be included (replace following example URLs by **yours**):
 - Single video: https://www.youtube.com/watch?v=BaW_jenozKc
 - Single video: https://youtu.be/BaW_jenozKc
 - Playlist: https://www.youtube.com/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc
 
-#### Description of your *issue*, suggested solution and other information
+---
+
+### Description of your *issue*, suggested solution and other information
 
 Explanation of your *issue* in arbitrary form goes here. Please make sure the [description is worded well enough to be understood](https://github.com/rg3/youtube-dl#is-the-description-of-the-issue-itself-sufficient). Provide as much context and examples as possible.
+If work on your *issue* required an account credentials please provide them or explain how one can obtain them.

From 607619bc90e202b09028053d5eab9c03dc7cee4b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 29 Mar 2016 22:04:29 +0600
Subject: [PATCH 4469/6219] Add manually generated ISSUE_TEMPLATE.md

In order not to wait for the next release
---
 .github/ISSUE_TEMPLATE.md | 58 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 58 insertions(+)
 create mode 100644 .github/ISSUE_TEMPLATE.md

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
new file mode 100644
index 000000000..5b1f573e7
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE.md
@@ -0,0 +1,58 @@
+## Please follow the guide below
+
+- You will be asked some questions and requested to provide some information, please read them **carefully** and answer honestly
+- Put an `x` into all the boxes [ ] relevant to your *issue* (like that [x])
+- Use *Preview* tab to see how your issue will actually look like
+
+---
+
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.03.27*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.03.27**
+
+### Before submitting an *issue* make sure you have:
+- [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
+- [ ] [Searched](https://github.com/rg3/youtube-dl/search?type=Issues) the bugtracker for similar issues including closed ones
+
+### What is the purpose of your *issue*?
+- [ ] Bug report (encountered problems with youtube-dl)
+- [ ] Site support request (request for adding support for a new site)
+- [ ] Feature request (request for a new functionality)
+- [ ] Question
+- [ ] Other
+
+---
+
+### The following sections concretize particular purposed issues, you can erase any section (the contents between triple ---) not applicable to your *issue*
+
+---
+
+### If the purpose of this *issue* is a *bug report*, *site support request* or you are not completely sure provide the full verbose output as follows:
+
+Add `-v` flag to **your command line** you run youtube-dl with, copy the **whole** output and insert it here. It should look similar to one below (replace it with **your** log inserted between triple ```):
+```
+$ youtube-dl -v <your command line>
+[debug] System config: []
+[debug] User config: []
+[debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
+[debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
+[debug] youtube-dl version 2016.03.27
+[debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
+[debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
+[debug] Proxy map: {}
+...
+<end of log>
+```
+
+---
+
+### If the purpose of this *issue* is a *site support request* please provide all kinds of example URLs support for which should be included (replace following example URLs by **yours**):
+- Single video: https://www.youtube.com/watch?v=BaW_jenozKc
+- Single video: https://youtu.be/BaW_jenozKc
+- Playlist: https://www.youtube.com/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc
+
+---
+
+### Description of your *issue*, suggested solution and other information
+
+Explanation of your *issue* in arbitrary form goes here. Please make sure the [description is worded well enough to be understood](https://github.com/rg3/youtube-dl#is-the-description-of-the-issue-itself-sufficient). Provide as much context and examples as possible.
+If work on your *issue* required an account credentials please provide them or explain how one can obtain them.

From 6e359a1534e6a20acb53e1268ec77b6b92765e22 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 30 Mar 2016 12:27:00 +0100
Subject: [PATCH 4470/6219] [comcarcoff] don not depend on crackle
 extractor(closes #8995)

previously extraction has been delegated to crackle to extract more info
and subtitles #6106 but some of the episodes can't be extracted using
crackle #8995.
---
 youtube_dl/extractor/comcarcoff.py | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/comcarcoff.py b/youtube_dl/extractor/comcarcoff.py
index e697d1410..747c245c8 100644
--- a/youtube_dl/extractor/comcarcoff.py
+++ b/youtube_dl/extractor/comcarcoff.py
@@ -41,7 +41,13 @@ class ComCarCoffIE(InfoExtractor):
 
         display_id = full_data['activeVideo']['video']
         video_data = full_data.get('videos', {}).get(display_id) or full_data['singleshots'][display_id]
+
         video_id = compat_str(video_data['mediaId'])
+        title = video_data['title']
+        formats = self._extract_m3u8_formats(
+            video_data['mediaUrl'], video_id, 'mp4')
+        self._sort_formats(formats)
+
         thumbnails = [{
             'url': video_data['images']['thumb'],
         }, {
@@ -54,15 +60,14 @@ class ComCarCoffIE(InfoExtractor):
             video_data.get('duration'))
 
         return {
-            '_type': 'url_transparent',
-            'url': 'crackle:%s' % video_id,
             'id': video_id,
             'display_id': display_id,
-            'title': video_data['title'],
+            'title': title,
             'description': video_data.get('description'),
             'timestamp': timestamp,
             'duration': duration,
             'thumbnails': thumbnails,
+            'formats': formats,
             'season_number': int_or_none(video_data.get('season')),
             'episode_number': int_or_none(video_data.get('episode')),
             'webpage_url': 'http://comediansincarsgettingcoffee.com/%s' % (video_data.get('urlSlug', video_data.get('slug'))),

From afca767d196dee68b254720706a2ba191455e99b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 30 Mar 2016 22:26:43 +0600
Subject: [PATCH 4471/6219] [tumblr] Improve _VALID_URL (Closes #9027)

---
 youtube_dl/extractor/tumblr.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tumblr.py b/youtube_dl/extractor/tumblr.py
index e5bcf7798..4d8b57111 100644
--- a/youtube_dl/extractor/tumblr.py
+++ b/youtube_dl/extractor/tumblr.py
@@ -8,7 +8,7 @@ from ..utils import int_or_none
 
 
 class TumblrIE(InfoExtractor):
-    _VALID_URL = r'https?://(?P<blog_name>.*?)\.tumblr\.com/(?:post|video)/(?P<id>[0-9]+)(?:$|[/?#])'
+    _VALID_URL = r'https?://(?P<blog_name>[^/?#&]+)\.tumblr\.com/(?:post|video)/(?P<id>[0-9]+)(?:$|[/?#])'
     _TESTS = [{
         'url': 'http://tatianamaslanydaily.tumblr.com/post/54196191430/orphan-black-dvd-extra-behind-the-scenes',
         'md5': '479bb068e5b16462f5176a6828829767',

From 597d52fadbf32af4f2dcc7b9e236c318145f536b Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 31 Mar 2016 01:54:08 +0800
Subject: [PATCH 4472/6219] [kuwo:song] Correct song ID extraction (fixes
 #9033)

Bug introduced in daef04a4e75ccd2ff5e2d2495baa0ac9bcf75724.
---
 youtube_dl/extractor/kuwo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/kuwo.py b/youtube_dl/extractor/kuwo.py
index a586308b2..84c0f363d 100644
--- a/youtube_dl/extractor/kuwo.py
+++ b/youtube_dl/extractor/kuwo.py
@@ -55,7 +55,7 @@ class KuwoBaseIE(InfoExtractor):
 class KuwoIE(KuwoBaseIE):
     IE_NAME = 'kuwo:song'
     IE_DESC = '酷我音乐'
-    _VALID_URL = r'https?://www\.kuwo\.cn/yinyue/(?P<id>\d+?)'
+    _VALID_URL = r'https?://www\.kuwo\.cn/yinyue/(?P<id>\d+)'
     _TESTS = [{
         'url': 'http://www.kuwo.cn/yinyue/635632/',
         'info_dict': {

From 3ae6f8fec1381df41dc05272bfe3ab03654ac4af Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 31 Mar 2016 02:11:21 +0800
Subject: [PATCH 4473/6219] [kwuo] Remove _sort_formats() from
 KuwoBaseIE._get_formats()

Following the idea proposed in 19dbaeece321c51fa336ef142507adf440e22e22
---
 youtube_dl/extractor/kuwo.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/kuwo.py b/youtube_dl/extractor/kuwo.py
index 84c0f363d..f09436332 100644
--- a/youtube_dl/extractor/kuwo.py
+++ b/youtube_dl/extractor/kuwo.py
@@ -44,11 +44,6 @@ class KuwoBaseIE(InfoExtractor):
                     'abr': file_format.get('abr'),
                 })
 
-        # XXX _sort_formats fails if there are not formats, while it's not the
-        # desired behavior if 'IPDeny' is ignored
-        # This check can be removed if https://github.com/rg3/youtube-dl/pull/8051 is merged
-        if not tolerate_ip_deny:
-            self._sort_formats(formats)
         return formats
 
 
@@ -103,6 +98,7 @@ class KuwoIE(KuwoBaseIE):
             lrc_content = None
 
         formats = self._get_formats(song_id)
+        self._sort_formats(formats)
 
         album_id = self._html_search_regex(
             r'<p[^>]+class="album"[^<]+<a[^>]+href="http://www\.kuwo\.cn/album/(\d+)/"',

From e621a344e6ec3518420f0b13577726615c2f4485 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 31 Mar 2016 02:27:52 +0800
Subject: [PATCH 4474/6219] [kwuo] Port to new API and enable
 --cn-verification-proxy

---
 youtube_dl/extractor/kuwo.py | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/kuwo.py b/youtube_dl/extractor/kuwo.py
index f09436332..86c17c931 100644
--- a/youtube_dl/extractor/kuwo.py
+++ b/youtube_dl/extractor/kuwo.py
@@ -26,10 +26,23 @@ class KuwoBaseIE(InfoExtractor):
     def _get_formats(self, song_id, tolerate_ip_deny=False):
         formats = []
         for file_format in self._FORMATS:
+            headers = {}
+            cn_verification_proxy = self._downloader.params.get('cn_verification_proxy')
+            if cn_verification_proxy:
+                headers['Ytdl-request-proxy'] = cn_verification_proxy
+
+            query = {
+                'format': file_format['ext'],
+                'br': file_format.get('br', ''),
+                'rid': 'MUSIC_%s' % song_id,
+                'type': 'convert_url',
+                'response': 'url'
+            }
+
             song_url = self._download_webpage(
-                'http://antiserver.kuwo.cn/anti.s?format=%s&br=%s&rid=MUSIC_%s&type=convert_url&response=url' %
-                (file_format['ext'], file_format.get('br', ''), song_id),
+                'http://antiserver.kuwo.cn/anti.s',
                 song_id, note='Download %s url info' % file_format['format'],
+                query=query, headers=headers,
             )
 
             if song_url == 'IPDeny' and not tolerate_ip_deny:

From 6b820a2376a953657578f9a477ff7768d3633512 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 30 Mar 2016 21:18:07 +0100
Subject: [PATCH 4475/6219] [myspace] improve extraction

---
 youtube_dl/extractor/myspace.py | 80 ++++++++++++++++++++++-----------
 1 file changed, 53 insertions(+), 27 deletions(-)

diff --git a/youtube_dl/extractor/myspace.py b/youtube_dl/extractor/myspace.py
index 83414a232..0d5238d77 100644
--- a/youtube_dl/extractor/myspace.py
+++ b/youtube_dl/extractor/myspace.py
@@ -2,13 +2,13 @@
 from __future__ import unicode_literals
 
 import re
-import json
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_str,
+from ..utils import (
+    ExtractorError,
+    int_or_none,
+    parse_iso8601,
 )
-from ..utils import ExtractorError
 
 
 class MySpaceIE(InfoExtractor):
@@ -24,6 +24,8 @@ class MySpaceIE(InfoExtractor):
                 'description': 'This country quartet was all smiles while playing a sold out show at the Pacific Amphitheatre in Orange County, California.',
                 'uploader': 'Five Minutes to the Stage',
                 'uploader_id': 'fiveminutestothestage',
+                'timestamp': 1414108751,
+                'upload_date': '20141023',
             },
             'params': {
                 # rtmp download
@@ -64,7 +66,7 @@ class MySpaceIE(InfoExtractor):
                 'ext': 'mp4',
                 'title': 'Starset - First Light',
                 'description': 'md5:2d5db6c9d11d527683bcda818d332414',
-                'uploader': 'Jacob Soren',
+                'uploader': 'Yumi K',
                 'uploader_id': 'SorenPromotions',
                 'upload_date': '20140725',
             }
@@ -78,6 +80,19 @@ class MySpaceIE(InfoExtractor):
         player_url = self._search_regex(
             r'playerSwf":"([^"?]*)', webpage, 'player URL')
 
+        def rtmp_format_from_stream_url(stream_url, width=None, height=None):
+            rtmp_url, play_path = stream_url.split(';', 1)
+            return {
+                'format_id': 'rtmp',
+                'url': rtmp_url,
+                'play_path': play_path,
+                'player_url': player_url,
+                'protocol': 'rtmp',
+                'ext': 'flv',
+                'width': width,
+                'height': height,
+            }
+
         if mobj.group('mediatype').startswith('music/song'):
             # songs don't store any useful info in the 'context' variable
             song_data = self._search_regex(
@@ -93,8 +108,8 @@ class MySpaceIE(InfoExtractor):
                 return self._search_regex(
                     r'''data-%s=([\'"])(?P<data>.*?)\1''' % name,
                     song_data, name, default='', group='data')
-            streamUrl = search_data('stream-url')
-            if not streamUrl:
+            stream_url = search_data('stream-url')
+            if not stream_url:
                 vevo_id = search_data('vevo-id')
                 youtube_id = search_data('youtube-id')
                 if vevo_id:
@@ -106,36 +121,47 @@ class MySpaceIE(InfoExtractor):
                 else:
                     raise ExtractorError(
                         'Found song but don\'t know how to download it')
-            info = {
+            return {
                 'id': video_id,
                 'title': self._og_search_title(webpage),
                 'uploader': search_data('artist-name'),
                 'uploader_id': search_data('artist-username'),
                 'thumbnail': self._og_search_thumbnail(webpage),
+                'duration': int_or_none(search_data('duration')),
+                'formats': [rtmp_format_from_stream_url(stream_url)]
             }
         else:
-            context = json.loads(self._search_regex(
-                r'context = ({.*?});', webpage, 'context'))
-            video = context['video']
-            streamUrl = video['streamUrl']
-            info = {
-                'id': compat_str(video['mediaId']),
+            video = self._parse_json(self._search_regex(
+                r'context = ({.*?});', webpage, 'context'),
+                video_id)['video']
+            formats = []
+            hls_stream_url = video.get('hlsStreamUrl')
+            if hls_stream_url:
+                formats.append({
+                    'format_id': 'hls',
+                    'url': hls_stream_url,
+                    'protocol': 'm3u8_native',
+                    'ext': 'mp4',
+                })
+            stream_url = video.get('streamUrl')
+            if stream_url:
+                formats.append(rtmp_format_from_stream_url(
+                    stream_url,
+                    int_or_none(video.get('width')),
+                    int_or_none(video.get('height'))))
+            self._sort_formats(formats)
+            return {
+                'id': video_id,
                 'title': video['title'],
-                'description': video['description'],
-                'thumbnail': video['imageUrl'],
-                'uploader': video['artistName'],
-                'uploader_id': video['artistUsername'],
+                'description': video.get('description'),
+                'thumbnail': video.get('imageUrl'),
+                'uploader': video.get('artistName'),
+                'uploader_id': video.get('artistUsername'),
+                'duration': int_or_none(video.get('duration')),
+                'timestamp': parse_iso8601(video.get('dateAdded')),
+                'formats': formats,
             }
 
-        rtmp_url, play_path = streamUrl.split(';', 1)
-        info.update({
-            'url': rtmp_url,
-            'play_path': play_path,
-            'player_url': player_url,
-            'ext': 'flv',
-        })
-        return info
-
 
 class MySpaceAlbumIE(InfoExtractor):
     IE_NAME = 'MySpace:album'

From c02ec7d4300d3e2607f48fe73011fd8caa38f90c Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 30 Mar 2016 23:18:31 +0100
Subject: [PATCH 4476/6219] [cnbc] Add new extractor(closes #8012)

---
 youtube_dl/extractor/__init__.py    |  1 +
 youtube_dl/extractor/cnbc.py        | 29 +++++++++++++++++++++++++++++
 youtube_dl/extractor/theplatform.py |  8 ++++----
 3 files changed, 34 insertions(+), 4 deletions(-)
 create mode 100644 youtube_dl/extractor/cnbc.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 1e4b078a4..bd1f7d293 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -127,6 +127,7 @@ from .cloudy import CloudyIE
 from .clubic import ClubicIE
 from .clyp import ClypIE
 from .cmt import CMTIE
+from .cnbc import CNBCIE
 from .cnet import CNETIE
 from .cnn import (
     CNNIE,
diff --git a/youtube_dl/extractor/cnbc.py b/youtube_dl/extractor/cnbc.py
new file mode 100644
index 000000000..593e459aa
--- /dev/null
+++ b/youtube_dl/extractor/cnbc.py
@@ -0,0 +1,29 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import smuggle_url
+
+
+class CNBCIE(InfoExtractor):
+    _VALID_URL = r'https?://video\.cnbc\.com/gallery/\?video=(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'http://video.cnbc.com/gallery/?video=3000503714',
+        'md5': '',
+        'info_dict': {
+            'id': '3000503714',
+            'ext': 'mp4',
+            'title': 'Video title goes here',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        return {
+            '_type': 'url_transparent',
+            'ie_key': 'ThePlatform',
+            'url': smuggle_url(
+                'http://link.theplatform.com/s/gZWlPC/media/guid/2408950221/%s?mbr=true&manifest=m3u' % video_id,
+                {'force_smil_url': True}),
+            'id': video_id,
+        }
diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index 863914299..236c99972 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -82,7 +82,7 @@ class ThePlatformBaseIE(OnceIE):
 class ThePlatformIE(ThePlatformBaseIE):
     _VALID_URL = r'''(?x)
         (?:https?://(?:link|player)\.theplatform\.com/[sp]/(?P<provider_id>[^/]+)/
-           (?:(?P<media>(?:(?:[^/]+/)+select/)?media/)|(?P<config>(?:[^/\?]+/(?:swf|config)|onsite)/select/))?
+           (?:(?:(?:[^/]+/)+select/)?(?P<media>media/(?:guid/\d+/)?)|(?P<config>(?:[^/\?]+/(?:swf|config)|onsite)/select/))?
          |theplatform:)(?P<id>[^/\?&]+)'''
 
     _TESTS = [{
@@ -170,10 +170,10 @@ class ThePlatformIE(ThePlatformBaseIE):
         if not provider_id:
             provider_id = 'dJ5BDC'
 
-        path = provider_id
+        path = provider_id + '/'
         if mobj.group('media'):
-            path += '/media'
-        path += '/' + video_id
+            path += mobj.group('media')
+        path += video_id
 
         qs_dict = compat_parse_qs(compat_urllib_parse_urlparse(url).query)
         if 'guid' in qs_dict:

From ce548296fe8bde2756fd9915bd744c904231de8f Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 31 Mar 2016 00:25:11 +0100
Subject: [PATCH 4477/6219] [cnbc] fix test

---
 youtube_dl/extractor/cnbc.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/cnbc.py b/youtube_dl/extractor/cnbc.py
index 593e459aa..25b308752 100644
--- a/youtube_dl/extractor/cnbc.py
+++ b/youtube_dl/extractor/cnbc.py
@@ -9,12 +9,16 @@ class CNBCIE(InfoExtractor):
     _VALID_URL = r'https?://video\.cnbc\.com/gallery/\?video=(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://video.cnbc.com/gallery/?video=3000503714',
-        'md5': '',
         'info_dict': {
             'id': '3000503714',
             'ext': 'mp4',
-            'title': 'Video title goes here',
-        }
+            'title': 'Fighting zombies is big business',
+            'description': 'md5:0c100d8e1a7947bd2feec9a5550e519e',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
     }
 
     def _real_extract(self, url):

From 9cf01f7f30b698aee6d22052d8898b14a59d94bb Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 31 Mar 2016 00:26:42 +0100
Subject: [PATCH 4478/6219] [nbc] add new extractor for csnne.com(#5432)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/nbc.py      | 24 ++++++++++++++++++++++++
 2 files changed, 25 insertions(+)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index bd1f7d293..7b0f2b21a 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -442,6 +442,7 @@ from .nationalgeographic import NationalGeographicIE
 from .naver import NaverIE
 from .nba import NBAIE
 from .nbc import (
+    CSNNEIE,
     NBCIE,
     NBCNewsIE,
     NBCSportsIE,
diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index a622f2212..43d75d3ca 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -134,6 +134,30 @@ class NBCSportsIE(InfoExtractor):
             NBCSportsVPlayerIE._extract_url(webpage), 'NBCSportsVPlayer')
 
 
+class CSNNEIE(InfoExtractor):
+    _VALID_URL = r'https?://www\.csnne\.com/video/(?P<id>[0-9a-z-]+)'
+
+    _TEST = {
+        'url': 'http://www.csnne.com/video/snc-evening-update-wright-named-red-sox-no-5-starter',
+        'info_dict': {
+            'id': 'yvBLLUgQ8WU0',
+            'ext': 'mp4',
+            'title': 'SNC evening update: Wright named Red Sox\' No. 5 starter.',
+            'description': 'md5:1753cfee40d9352b19b4c9b3e589b9e3',
+        }
+    }
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+        return {
+            '_type': 'url_transparent',
+            'ie_key': 'ThePlatform',
+            'url': self._html_search_meta('twitter:player:stream', webpage),
+            'display_id': display_id,
+        }
+
+
 class NBCNewsIE(ThePlatformIE):
     _VALID_URL = r'''(?x)https?://(?:www\.)?nbcnews\.com/
         (?:video/.+?/(?P<id>\d+)|

From fea7295b1400f27218422cb37f70e7c4e2c66c29 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 31 Mar 2016 10:48:22 +0100
Subject: [PATCH 4479/6219] [brightcove] relax embed_in_page regex

---
 youtube_dl/extractor/brightcove.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index c9e43a275..0d162d337 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -439,7 +439,7 @@ class BrightcoveNewIE(InfoExtractor):
                     </video>.*?
                     <script[^>]+
                         src=["\'](?:https?:)?//players\.brightcove\.net/
-                        (\d+)/([\da-f-]+)_([^/]+)/index(?:\.min)?\.js
+                        (\d+)/([^/]+)_([^/]+)/index(?:\.min)?\.js
                 ''', webpage):
             entries.append(
                 'http://players.brightcove.net/%s/%s_%s/index.html?videoId=%s'

From dd17041c82169b862e7f91cee9c5a5ed86b68ca9 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 31 Mar 2016 11:59:40 +0100
Subject: [PATCH 4480/6219] [tenplay] remove extractor(fixes #6927)

---
 youtube_dl/extractor/__init__.py |  1 -
 youtube_dl/extractor/tenplay.py  | 90 --------------------------------
 2 files changed, 91 deletions(-)
 delete mode 100644 youtube_dl/extractor/tenplay.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 7b0f2b21a..0a2dee40a 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -737,7 +737,6 @@ from .telecinco import TelecincoIE
 from .telegraaf import TelegraafIE
 from .telemb import TeleMBIE
 from .teletask import TeleTaskIE
-from .tenplay import TenPlayIE
 from .testurl import TestURLIE
 from .tf1 import TF1IE
 from .theintercept import TheInterceptIE
diff --git a/youtube_dl/extractor/tenplay.py b/youtube_dl/extractor/tenplay.py
deleted file mode 100644
index 02a31a609..000000000
--- a/youtube_dl/extractor/tenplay.py
+++ /dev/null
@@ -1,90 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-from .common import InfoExtractor
-from ..utils import (
-    int_or_none,
-    float_or_none,
-)
-
-
-class TenPlayIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?ten(play)?\.com\.au/.+'
-    _TEST = {
-        'url': 'http://tenplay.com.au/ten-insider/extra/season-2013/tenplay-tv-your-way',
-        'info_dict': {
-            'id': '2695695426001',
-            'ext': 'flv',
-            'title': 'TENplay: TV your way',
-            'description': 'Welcome to a new TV experience. Enjoy a taste of the TENplay benefits.',
-            'timestamp': 1380150606.889,
-            'upload_date': '20130925',
-            'uploader': 'TENplay',
-        },
-        'params': {
-            'skip_download': True,  # Requires rtmpdump
-        }
-    }
-
-    _video_fields = [
-        'id', 'name', 'shortDescription', 'longDescription', 'creationDate',
-        'publishedDate', 'lastModifiedDate', 'customFields', 'videoStillURL',
-        'thumbnailURL', 'referenceId', 'length', 'playsTotal',
-        'playsTrailingWeek', 'renditions', 'captioning', 'startDate', 'endDate']
-
-    def _real_extract(self, url):
-        webpage = self._download_webpage(url, url)
-        video_id = self._html_search_regex(
-            r'videoID: "(\d+?)"', webpage, 'video_id')
-        api_token = self._html_search_regex(
-            r'apiToken: "([a-zA-Z0-9-_\.]+?)"', webpage, 'api_token')
-        title = self._html_search_regex(
-            r'<meta property="og:title" content="\s*(.*?)\s*"\s*/?\s*>',
-            webpage, 'title')
-
-        json = self._download_json('https://api.brightcove.com/services/library?command=find_video_by_id&video_id=%s&token=%s&video_fields=%s' % (video_id, api_token, ','.join(self._video_fields)), title)
-
-        formats = []
-        for rendition in json['renditions']:
-            url = rendition['remoteUrl'] or rendition['url']
-            protocol = 'rtmp' if url.startswith('rtmp') else 'http'
-            ext = 'flv' if protocol == 'rtmp' else rendition['videoContainer'].lower()
-
-            if protocol == 'rtmp':
-                url = url.replace('&mp4:', '')
-
-                tbr = int_or_none(rendition.get('encodingRate'), 1000)
-
-            formats.append({
-                'format_id': '_'.join(
-                    ['rtmp', rendition['videoContainer'].lower(),
-                     rendition['videoCodec'].lower(), '%sk' % tbr]),
-                'width': int_or_none(rendition['frameWidth']),
-                'height': int_or_none(rendition['frameHeight']),
-                'tbr': tbr,
-                'filesize': int_or_none(rendition['size']),
-                'protocol': protocol,
-                'ext': ext,
-                'vcodec': rendition['videoCodec'].lower(),
-                'container': rendition['videoContainer'].lower(),
-                'url': url,
-            })
-        self._sort_formats(formats)
-
-        return {
-            'id': video_id,
-            'display_id': json['referenceId'],
-            'title': json['name'],
-            'description': json['shortDescription'] or json['longDescription'],
-            'formats': formats,
-            'thumbnails': [{
-                'url': json['videoStillURL']
-            }, {
-                'url': json['thumbnailURL']
-            }],
-            'thumbnail': json['videoStillURL'],
-            'duration': float_or_none(json.get('length'), 1000),
-            'timestamp': float_or_none(json.get('creationDate'), 1000),
-            'uploader': json.get('customFields', {}).get('production_company_distributor') or 'TENplay',
-            'view_count': int_or_none(json.get('playsTotal')),
-        }

From d84b48e3f1d9c2099a2a8ba48df3a2bd5e591807 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 31 Mar 2016 13:44:55 +0100
Subject: [PATCH 4481/6219] [nationalgeographic] improve extraction

---
 youtube_dl/extractor/nationalgeographic.py | 30 +++++++++++-----------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/nationalgeographic.py b/youtube_dl/extractor/nationalgeographic.py
index d5e53365c..1560e3e81 100644
--- a/youtube_dl/extractor/nationalgeographic.py
+++ b/youtube_dl/extractor/nationalgeographic.py
@@ -8,14 +8,16 @@ from ..utils import (
 
 
 class NationalGeographicIE(InfoExtractor):
+    IE_NAME = 'natgeo'
     _VALID_URL = r'https?://video\.nationalgeographic\.com/.*?'
 
     _TESTS = [
         {
             'url': 'http://video.nationalgeographic.com/video/news/150210-news-crab-mating-vin?source=featuredvideo',
+            'md5': '730855d559abbad6b42c2be1fa584917',
             'info_dict': {
-                'id': '4DmDACA6Qtk_',
-                'ext': 'flv',
+                'id': '0000014b-70a1-dd8c-af7f-f7b559330001',
+                'ext': 'mp4',
                 'title': 'Mating Crabs Busted by Sharks',
                 'description': 'md5:16f25aeffdeba55aaa8ec37e093ad8b3',
             },
@@ -23,9 +25,10 @@ class NationalGeographicIE(InfoExtractor):
         },
         {
             'url': 'http://video.nationalgeographic.com/wild/when-sharks-attack/the-real-jaws',
+            'md5': '6a3105eb448c070503b3105fb9b320b5',
             'info_dict': {
-                'id': '_JeBD_D7PlS5',
-                'ext': 'flv',
+                'id': 'ngc-I0IauNSWznb_UV008GxSbwY35BZvgi2e',
+                'ext': 'mp4',
                 'title': 'The Real Jaws',
                 'description': 'md5:8d3e09d9d53a85cd397b4b21b2c77be6',
             },
@@ -37,18 +40,15 @@ class NationalGeographicIE(InfoExtractor):
         name = url_basename(url)
 
         webpage = self._download_webpage(url, name)
-        feed_url = self._search_regex(
-            r'data-feed-url="([^"]+)"', webpage, 'feed url')
         guid = self._search_regex(
             r'id="(?:videoPlayer|player-container)"[^>]+data-guid="([^"]+)"',
             webpage, 'guid')
 
-        feed = self._download_xml('%s?byGuid=%s' % (feed_url, guid), name)
-        content = feed.find('.//{http://search.yahoo.com/mrss/}content')
-        theplatform_id = url_basename(content.attrib.get('url'))
-
-        return self.url_result(smuggle_url(
-            'http://link.theplatform.com/s/ngs/%s?formats=MPEG4&manifest=f4m' % theplatform_id,
-            # For some reason, the normal links don't work and we must force
-            # the use of f4m
-            {'force_smil_url': True}))
+        return {
+            '_type': 'url_transparent',
+            'ie_key': 'ThePlatform',
+            'url': smuggle_url(
+                'http://link.theplatform.com/s/ngs/media/guid/2423130747/%s?mbr=true' % guid,
+                {'force_smil_url': True}),
+            'id': guid,
+        }

From c9c39c22c5740c1eedcc9ce7a10f5df199ea5c78 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 31 Mar 2016 13:47:38 +0100
Subject: [PATCH 4482/6219] [nationalgeographic] add support for
 channel.nationalgeographic.com urls

---
 youtube_dl/extractor/__init__.py           |  5 ++-
 youtube_dl/extractor/nationalgeographic.py | 47 ++++++++++++++++++++++
 2 files changed, 51 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 0a2dee40a..76354b67e 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -438,7 +438,10 @@ from .myspass import MySpassIE
 from .myvi import MyviIE
 from .myvideo import MyVideoIE
 from .myvidster import MyVidsterIE
-from .nationalgeographic import NationalGeographicIE
+from .nationalgeographic import (
+    NationalGeographicIE,
+    NationalGeographicChannelIE,
+)
 from .naver import NaverIE
 from .nba import NBAIE
 from .nbc import (
diff --git a/youtube_dl/extractor/nationalgeographic.py b/youtube_dl/extractor/nationalgeographic.py
index 1560e3e81..61b5c700e 100644
--- a/youtube_dl/extractor/nationalgeographic.py
+++ b/youtube_dl/extractor/nationalgeographic.py
@@ -4,6 +4,7 @@ from .common import InfoExtractor
 from ..utils import (
     smuggle_url,
     url_basename,
+    update_url_query,
 )
 
 
@@ -52,3 +53,49 @@ class NationalGeographicIE(InfoExtractor):
                 {'force_smil_url': True}),
             'id': guid,
         }
+
+
+class NationalGeographicChannelIE(InfoExtractor):
+    IE_NAME = 'natgeo:channel'
+    _VALID_URL = r'https?://channel\.nationalgeographic\.com/(?:wild/)?[^/]+/videos/(?P<id>[^/?]+)'
+
+    _TESTS = [
+        {
+            'url': 'http://channel.nationalgeographic.com/the-story-of-god-with-morgan-freeman/videos/uncovering-a-universal-knowledge/',
+            'md5': '518c9aa655686cf81493af5cc21e2a04',
+            'info_dict': {
+                'id': 'nB5vIAfmyllm',
+                'ext': 'mp4',
+                'title': 'Uncovering a Universal Knowledge',
+                'description': 'md5:1a89148475bf931b3661fcd6ddb2ae3a',
+            },
+            'add_ie': ['ThePlatform'],
+        },
+        {
+            'url': 'http://channel.nationalgeographic.com/wild/destination-wild/videos/the-stunning-red-bird-of-paradise/',
+            'md5': 'c4912f656b4cbe58f3e000c489360989',
+            'info_dict': {
+                'id': '3TmMv9OvGwIR',
+                'ext': 'mp4',
+                'title': 'The Stunning Red Bird of Paradise',
+                'description': 'md5:7bc8cd1da29686be4d17ad1230f0140c',
+            },
+            'add_ie': ['ThePlatform'],
+        },
+    ]
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+        release_url = self._search_regex(
+            r'video_auth_playlist_url\s*=\s*"([^"]+)"',
+            webpage, 'release url')
+
+        return {
+            '_type': 'url_transparent',
+            'ie_key': 'ThePlatform',
+            'url': smuggle_url(
+                update_url_query(release_url, {'mbr': 'true', 'switch': 'http'}),
+                {'force_smil_url': True}),
+            'display_id': display_id,
+        }

From 5299bc3f91ffbb784addaee002611a52232134a1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 31 Mar 2016 20:42:41 +0600
Subject: [PATCH 4483/6219] [beeg] Switch to api v6 (Closes #9036)

---
 youtube_dl/extractor/beeg.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/beeg.py b/youtube_dl/extractor/beeg.py
index 34c2a756f..9072949dd 100644
--- a/youtube_dl/extractor/beeg.py
+++ b/youtube_dl/extractor/beeg.py
@@ -34,7 +34,7 @@ class BeegIE(InfoExtractor):
         video_id = self._match_id(url)
 
         video = self._download_json(
-            'https://api.beeg.com/api/v5/video/%s' % video_id, video_id)
+            'https://api.beeg.com/api/v6/1738/video/%s' % video_id, video_id)
 
         def split(o, e):
             def cut(s, x):
@@ -50,8 +50,8 @@ class BeegIE(InfoExtractor):
             return n
 
         def decrypt_key(key):
-            # Reverse engineered from http://static.beeg.com/cpl/1105.js
-            a = '5ShMcIQlssOd7zChAIOlmeTZDaUxULbJRnywYaiB'
+            # Reverse engineered from http://static.beeg.com/cpl/1738.js
+            a = 'GUuyodcfS8FW8gQp4OKLMsZBcX0T7B'
             e = compat_urllib_parse_unquote(key)
             o = ''.join([
                 compat_chr(compat_ord(e[n]) - compat_ord(a[n % len(a)]) % 21)

From 81da8cbc4513df16d0d04dc2992d6de9ab0f4038 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 31 Mar 2016 22:05:25 +0600
Subject: [PATCH 4484/6219] [udemy] Switch to api 2.0 (Closes #9035)

---
 youtube_dl/extractor/udemy.py | 60 +++++++++++++++++++++--------------
 1 file changed, 36 insertions(+), 24 deletions(-)

diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index 71bea5363..2e54dbc11 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -17,6 +17,7 @@ from ..utils import (
     int_or_none,
     sanitized_Request,
     unescapeHTML,
+    update_url_query,
     urlencode_postdata,
 )
 
@@ -54,6 +55,16 @@ class UdemyIE(InfoExtractor):
         'only_matching': True,
     }]
 
+    def _extract_course_info(self, webpage, video_id):
+        course = self._parse_json(
+            unescapeHTML(self._search_regex(
+                r'ng-init=["\'].*\bcourse=({.+?});', webpage, 'course', default='{}')),
+            video_id, fatal=False) or {}
+        course_id = course.get('id') or self._search_regex(
+            (r'&quot;id&quot;\s*:\s*(\d+)', r'data-course-id=["\'](\d+)'),
+            webpage, 'course id')
+        return course_id, course.get('title')
+
     def _enroll_course(self, base_url, webpage, course_id):
         def combine_url(base_url, url):
             return compat_urlparse.urljoin(base_url, url) if not url.startswith('http') else url
@@ -98,7 +109,7 @@ class UdemyIE(InfoExtractor):
                 error_str += ' - %s' % error_data.get('formErrors')
             raise ExtractorError(error_str, expected=True)
 
-    def _download_json(self, url_or_request, video_id, note='Downloading JSON metadata'):
+    def _download_json(self, url_or_request, *args, **kwargs):
         headers = {
             'X-Udemy-Snail-Case': 'true',
             'X-Requested-With': 'XMLHttpRequest',
@@ -116,7 +127,7 @@ class UdemyIE(InfoExtractor):
         else:
             url_or_request = sanitized_Request(url_or_request, headers=headers)
 
-        response = super(UdemyIE, self)._download_json(url_or_request, video_id, note)
+        response = super(UdemyIE, self)._download_json(url_or_request, *args, **kwargs)
         self._handle_error(response)
         return response
 
@@ -166,9 +177,7 @@ class UdemyIE(InfoExtractor):
 
         webpage = self._download_webpage(url, lecture_id)
 
-        course_id = self._search_regex(
-            (r'data-course-id=["\'](\d+)', r'&quot;id&quot;\s*:\s*(\d+)'),
-            webpage, 'course id')
+        course_id, _ = self._extract_course_info(webpage, lecture_id)
 
         try:
             lecture = self._download_lecture(course_id, lecture_id)
@@ -309,29 +318,32 @@ class UdemyCourseIE(UdemyIE):
 
         webpage = self._download_webpage(url, course_path)
 
-        response = self._download_json(
-            'https://www.udemy.com/api-1.1/courses/%s' % course_path,
-            course_path, 'Downloading course JSON')
-
-        course_id = response['id']
-        course_title = response.get('title')
+        course_id, title = self._extract_course_info(webpage, course_path)
 
         self._enroll_course(url, webpage, course_id)
 
+        course_url = update_url_query(
+            'https://www.udemy.com/api-2.0/courses/%s/cached-subscriber-curriculum-items' % course_id,
+            {
+                'fields[chapter]': 'title,object_index',
+                'fields[lecture]': 'title',
+                'page_size': '1000',
+            })
+
         response = self._download_json(
-            'https://www.udemy.com/api-1.1/courses/%s/curriculum' % course_id,
-            course_id, 'Downloading course curriculum')
+            course_url, course_id, 'Downloading course curriculum')
 
         entries = []
-        chapter, chapter_number = None, None
-        for asset in response:
-            asset_type = asset.get('assetType') or asset.get('asset_type')
-            if asset_type == 'Video':
-                asset_id = asset.get('id')
-                if asset_id:
+        chapter, chapter_number = [None] * 2
+        for entry in response['results']:
+            clazz = entry.get('_class')
+            if clazz == 'lecture':
+                lecture_id = entry.get('id')
+                if lecture_id:
                     entry = {
                         '_type': 'url_transparent',
-                        'url': 'https://www.udemy.com/%s/#/lecture/%s' % (course_path, asset['id']),
+                        'url': 'https://www.udemy.com/%s/#/lecture/%s' % (course_path, entry['id']),
+                        'title': entry.get('title'),
                         'ie_key': UdemyIE.ie_key(),
                     }
                     if chapter_number:
@@ -339,8 +351,8 @@ class UdemyCourseIE(UdemyIE):
                     if chapter:
                         entry['chapter'] = chapter
                     entries.append(entry)
-            elif asset.get('type') == 'chapter':
-                chapter_number = asset.get('index') or asset.get('object_index')
-                chapter = asset.get('title')
+            elif clazz == 'chapter':
+                chapter_number = entry.get('object_index')
+                chapter = entry.get('title')
 
-        return self.playlist_result(entries, course_id, course_title)
+        return self.playlist_result(entries, course_id, title)

From ed0291d1533600b21903cb98f070791a20e47433 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 31 Mar 2016 22:55:01 +0600
Subject: [PATCH 4485/6219] [utils] Add update_Request

---
 youtube_dl/utils.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 6d27b80c0..4532b737b 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1804,6 +1804,20 @@ def update_url_query(url, query):
         query=compat_urllib_parse_urlencode(qs, True)))
 
 
+def update_Request(req, url=None, data=None, headers={}, query={}):
+    req_headers = req.headers.copy()
+    req_headers.update(headers)
+    req_data = data or req.data
+    req_url = update_url_query(url or req.get_full_url(), query)
+    req_type = HEADRequest if req.get_method() == 'HEAD' else compat_urllib_request.Request
+    new_req = req_type(
+        req_url, data=req_data, headers=req_headers,
+        origin_req_host=req.origin_req_host, unverifiable=req.unverifiable)
+    if hasattr(req, 'timeout'):
+        new_req.timeout = req.timeout
+    return new_req
+
+
 def dict_get(d, key_or_keys, default=None, skip_false_values=True):
     if isinstance(key_or_keys, (list, tuple)):
         for key in key_or_keys:

From 15d260ebaa48409112270685d306a5d9152260c4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 31 Mar 2016 22:55:49 +0600
Subject: [PATCH 4486/6219] [utils] Use update_Request in http_request

---
 youtube_dl/utils.py | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 4532b737b..5c4ab2748 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -778,12 +778,7 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
 
         # Substitute URL if any change after escaping
         if url != url_escaped:
-            req_type = HEADRequest if req.get_method() == 'HEAD' else compat_urllib_request.Request
-            new_req = req_type(
-                url_escaped, data=req.data, headers=req.headers,
-                origin_req_host=req.origin_req_host, unverifiable=req.unverifiable)
-            new_req.timeout = req.timeout
-            req = new_req
+            req = update_Request(req, url=url_escaped)
 
         for h, v in std_headers.items():
             # Capitalize is needed because of Python bug 2275: http://bugs.python.org/issue2275

From 41d06b042427aa019200bb80a1935d12110f50b1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 31 Mar 2016 22:58:38 +0600
Subject: [PATCH 4487/6219] [extractor/common] Improve _request_webpage

* Do not ignore data, headers and query for Requests
* Default values for headers and query switched to dicts since these are used by urllib itself
---
 youtube_dl/extractor/common.py | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 9b7ab8924..85ac0400c 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -22,6 +22,7 @@ from ..compat import (
     compat_str,
     compat_urllib_error,
     compat_urllib_parse_urlencode,
+    compat_urllib_request,
     compat_urlparse,
 )
 from ..downloader.f4m import remove_encrypted_media
@@ -49,6 +50,7 @@ from ..utils import (
     determine_protocol,
     parse_duration,
     mimetype2ext,
+    update_Request,
     update_url_query,
 )
 
@@ -347,7 +349,7 @@ class InfoExtractor(object):
     def IE_NAME(self):
         return compat_str(type(self).__name__[:-2])
 
-    def _request_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True, data=None, headers=None, query=None):
+    def _request_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True, data=None, headers={}, query={}):
         """ Returns the response handle """
         if note is None:
             self.report_download_webpage(video_id)
@@ -357,11 +359,14 @@ class InfoExtractor(object):
             else:
                 self.to_screen('%s: %s' % (video_id, note))
         # data, headers and query params will be ignored for `Request` objects
-        if isinstance(url_or_request, compat_str):
+        if isinstance(url_or_request, compat_urllib_request.Request):
+            url_or_request = update_Request(
+                url_or_request, data=data, headers=headers, query=query)
+        else:
             if query:
                 url_or_request = update_url_query(url_or_request, query)
             if data or headers:
-                url_or_request = sanitized_Request(url_or_request, data, headers or {})
+                url_or_request = sanitized_Request(url_or_request, data, headers)
         try:
             return self._downloader.urlopen(url_or_request)
         except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
@@ -377,7 +382,7 @@ class InfoExtractor(object):
                 self._downloader.report_warning(errmsg)
                 return False
 
-    def _download_webpage_handle(self, url_or_request, video_id, note=None, errnote=None, fatal=True, encoding=None, data=None, headers=None, query=None):
+    def _download_webpage_handle(self, url_or_request, video_id, note=None, errnote=None, fatal=True, encoding=None, data=None, headers={}, query={}):
         """ Returns a tuple (page content as string, URL handle) """
         # Strip hashes from the URL (#1038)
         if isinstance(url_or_request, (compat_str, str)):
@@ -470,7 +475,7 @@ class InfoExtractor(object):
 
         return content
 
-    def _download_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True, tries=1, timeout=5, encoding=None, data=None, headers=None, query=None):
+    def _download_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True, tries=1, timeout=5, encoding=None, data=None, headers={}, query={}):
         """ Returns the data of the page as a string """
         success = False
         try_count = 0
@@ -491,7 +496,7 @@ class InfoExtractor(object):
 
     def _download_xml(self, url_or_request, video_id,
                       note='Downloading XML', errnote='Unable to download XML',
-                      transform_source=None, fatal=True, encoding=None, data=None, headers=None, query=None):
+                      transform_source=None, fatal=True, encoding=None, data=None, headers={}, query={}):
         """Return the xml as an xml.etree.ElementTree.Element"""
         xml_string = self._download_webpage(
             url_or_request, video_id, note, errnote, fatal=fatal, encoding=encoding, data=data, headers=headers, query=query)
@@ -505,7 +510,7 @@ class InfoExtractor(object):
                        note='Downloading JSON metadata',
                        errnote='Unable to download JSON metadata',
                        transform_source=None,
-                       fatal=True, encoding=None, data=None, headers=None, query=None):
+                       fatal=True, encoding=None, data=None, headers={}, query={}):
         json_string = self._download_webpage(
             url_or_request, video_id, note, errnote, fatal=fatal,
             encoding=encoding, data=data, headers=headers, query=query)

From 6bb4600717cb5378d392845e5c9bebe236a1b224 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 31 Mar 2016 22:59:19 +0600
Subject: [PATCH 4488/6219] [udemy:course] Simplify course curriculum
 downloading

---
 youtube_dl/extractor/udemy.py | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index 2e54dbc11..1784599e9 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -322,17 +322,14 @@ class UdemyCourseIE(UdemyIE):
 
         self._enroll_course(url, webpage, course_id)
 
-        course_url = update_url_query(
+        response = self._download_json(
             'https://www.udemy.com/api-2.0/courses/%s/cached-subscriber-curriculum-items' % course_id,
-            {
+            course_id, 'Downloading course curriculum', query={
                 'fields[chapter]': 'title,object_index',
                 'fields[lecture]': 'title',
                 'page_size': '1000',
             })
 
-        response = self._download_json(
-            course_url, course_id, 'Downloading course curriculum')
-
         entries = []
         chapter, chapter_number = [None] * 2
         for entry in response['results']:

From 86f3b66cec3a1ba6d3b197018a954865ca14c323 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 31 Mar 2016 23:00:11 +0600
Subject: [PATCH 4489/6219] [udemy] Remove unused import

---
 youtube_dl/extractor/udemy.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index 1784599e9..e91cf44fe 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -17,7 +17,6 @@ from ..utils import (
     int_or_none,
     sanitized_Request,
     unescapeHTML,
-    update_url_query,
     urlencode_postdata,
 )
 

From 8369942773aec76a5b97582f77b3e67f701bcf47 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 31 Mar 2016 18:36:41 +0100
Subject: [PATCH 4490/6219] [voxmedia] Add new extractor(closes #3182)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/voxmedia.py | 99 ++++++++++++++++++++++++++++++++
 2 files changed, 100 insertions(+)
 create mode 100644 youtube_dl/extractor/voxmedia.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 76354b67e..19f802411 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -904,6 +904,7 @@ from .vk import (
 from .vlive import VLiveIE
 from .vodlocker import VodlockerIE
 from .voicerepublic import VoiceRepublicIE
+from .voxmedia import VoxMediaIE
 from .vporn import VpornIE
 from .vrt import VRTIE
 from .vube import VubeIE
diff --git a/youtube_dl/extractor/voxmedia.py b/youtube_dl/extractor/voxmedia.py
new file mode 100644
index 000000000..3b13d38a1
--- /dev/null
+++ b/youtube_dl/extractor/voxmedia.py
@@ -0,0 +1,99 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..compat import compat_urllib_parse_unquote
+
+
+class VoxMediaIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?(?:theverge|vox|sbnation|eater|polygon|curbed|racked)\.com/(?:[^/]+/)*(?P<id>[^/?]+)'
+    _TESTS = [{
+        'url': 'http://www.theverge.com/2014/6/27/5849272/material-world-how-google-discovered-what-software-is-made-of',
+        'md5': '73856edf3e89a711e70d5cf7cb280b37',
+        'info_dict': {
+            'id': '11eXZobjrG8DCSTgrNjVinU-YmmdYjhe',
+            'ext': 'mp4',
+            'title': 'Google\'s new material design direction',
+            'description': 'md5:2f44f74c4d14a1f800ea73e1c6832ad2',
+        }
+    }, {
+        # data-ooyala-id
+        'url': 'http://www.theverge.com/2014/10/21/7025853/google-nexus-6-hands-on-photos-video-android-phablet',
+        'md5': 'd744484ff127884cd2ba09e3fa604e4b',
+        'info_dict': {
+            'id': 'RkZXU4cTphOCPDMZg5oEounJyoFI0g-B',
+            'ext': 'mp4',
+            'title': 'The Nexus 6: hands-on with Google\'s phablet',
+            'description': 'md5:87a51fe95ff8cea8b5bdb9ac7ae6a6af',
+        }
+    }, {
+        # volume embed
+        'url': 'http://www.vox.com/2016/3/31/11336640/mississippi-lgbt-religious-freedom-bill',
+        'md5': '375c483c5080ab8cd85c9c84cfc2d1e4',
+        'info_dict': {
+            'id': 'wydzk3dDpmRz7PQoXRsTIX6XTkPjYL0b',
+            'ext': 'mp4',
+            'title': 'The new frontier of LGBTQ civil rights, explained',
+            'description': 'md5:0dc58e94a465cbe91d02950f770eb93f',
+        }
+    }, {
+        # youtube embed
+        'url': 'http://www.vox.com/2016/3/24/11291692/robot-dance',
+        'md5': '83b3080489fb103941e549352d3e0977',
+        'info_dict': {
+            'id': 'FcNHTJU1ufM',
+            'ext': 'mp4',
+            'title': 'How "the robot" became the greatest novelty dance of all time',
+            'description': 'md5:b081c0d588b8b2085870cda55e6da176',
+            'upload_date': '20160324',
+            'uploader_id': 'voxdotcom',
+            'uploader': 'Vox',
+        }
+    }]
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = compat_urllib_parse_unquote(self._download_webpage(url, display_id))
+
+        title = None
+        description = None
+        provider_video_id = None
+        provider_video_type = None
+
+        entry = self._search_regex([
+            r'Chorus\.VideoContext\.addVideo\(\[({.+})\]\);',
+            r'var\s+entry\s*=\s*({.+});'
+        ], webpage, 'video data', default=None)
+        if entry:
+            video_data = self._parse_json(entry, display_id)
+            provider_video_id = video_data.get('provider_video_id')
+            provider_video_type = video_data.get('provider_video_type')
+            if provider_video_id and provider_video_type:
+                title = video_data.get('title')
+                description = video_data.get('description')
+
+        if not provider_video_id or not provider_video_type:
+            provider_video_id = self._search_regex(
+                r'data-ooyala-id="([^"]+)"', webpage, 'ooyala id', default=None)
+            if provider_video_id:
+                provider_video_type = 'ooyala'
+            else:
+                volume_uuid = self._search_regex(r'data-volume-uuid="([^"]+)"', webpage, 'volume uuid')
+                volume_webpage = self._download_webpage(
+                    'http://volume.vox-cdn.com/embed/%s' % volume_uuid, volume_uuid)
+                video_data = self._parse_json(self._search_regex(
+                    r'Volume\.createVideo\(({.+})\s*,\s*{.*}\);', volume_webpage, 'video data'), volume_uuid)
+                title = video_data.get('title_short')
+                description = video_data.get('description_long') or video_data.get('description_short')
+                for pvtype in ('ooyala', 'youtube'):
+                    provider_video_id = video_data.get('%s_id' % pvtype)
+                    if provider_video_id:
+                        provider_video_type = pvtype
+                        break
+
+        return {
+            '_type': 'url_transparent',
+            'url': provider_video_id if provider_video_type == 'youtube' else '%s:%s' % (provider_video_type, provider_video_id),
+            'title': title or self._og_search_title(webpage),
+            'description': description or self._og_search_description(webpage),
+        }

From 5b012dfce87e4f7dd9ab4a2f5a01ab8f2e16adad Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 1 Apr 2016 01:42:16 +0800
Subject: [PATCH 4491/6219] [tudou] Improve error handling (closes #8988)

---
 youtube_dl/extractor/tudou.py | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/youtube_dl/extractor/tudou.py b/youtube_dl/extractor/tudou.py
index 9892e8a62..63b5d5924 100644
--- a/youtube_dl/extractor/tudou.py
+++ b/youtube_dl/extractor/tudou.py
@@ -5,6 +5,7 @@ from __future__ import unicode_literals
 from .common import InfoExtractor
 from ..compat import compat_str
 from ..utils import (
+    ExtractorError,
     int_or_none,
     InAdvancePagedList,
     float_or_none,
@@ -46,6 +47,19 @@ class TudouIE(InfoExtractor):
 
     _PLAYER_URL = 'http://js.tudouui.com/bin/lingtong/PortalPlayer_177.swf'
 
+    # Translated from tudou/tools/TVCHelper.as in PortalPlayer_193.swf
+    # 0001, 0002 and 4001 are not included as they indicate temporary issues
+    TVC_ERRORS = {
+        '0003': 'The video is deleted or does not exist',
+        '1001': 'This video is unavailable due to licensing issues',
+        '1002': 'This video is unavailable as it\'s under review',
+        '1003': 'This video is unavailable as it\'s under review',
+        '3001': 'Password required',
+        '5001': 'This video is available in Mainland China only due to licensing issues',
+        '7001': 'This video is unavailable',
+        '8001': 'This video is unavailable due to licensing issues',
+    }
+
     def _url_for_id(self, video_id, quality=None):
         info_url = 'http://v2.tudou.com/f?id=' + compat_str(video_id)
         if quality:
@@ -63,6 +77,15 @@ class TudouIE(InfoExtractor):
         if youku_vcode:
             return self.url_result('youku:' + youku_vcode, ie='Youku')
 
+        if not item_data.get('itemSegs'):
+            tvc_code = item_data.get('tvcCode')
+            if tvc_code:
+                err_msg = self.TVC_ERRORS.get(tvc_code)
+                if err_msg:
+                    raise ExtractorError('Tudou said: %s' % err_msg, expected=True)
+                raise ExtractorError('Unexpected error %s returned from Tudou' % tvc_code)
+            raise ExtractorError('Unxpected error returned from Tudou')
+
         title = unescapeHTML(item_data['kw'])
         description = item_data.get('desc')
         thumbnail_url = item_data.get('pic')

From 4ecc1fc6387d900b7d61d43a112becff9e293206 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 31 Mar 2016 21:09:27 +0100
Subject: [PATCH 4492/6219] [howstuffworks] improve extraction

---
 youtube_dl/extractor/howstuffworks.py | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/howstuffworks.py b/youtube_dl/extractor/howstuffworks.py
index 663e6632a..76b74c51d 100644
--- a/youtube_dl/extractor/howstuffworks.py
+++ b/youtube_dl/extractor/howstuffworks.py
@@ -6,6 +6,7 @@ from ..utils import (
     int_or_none,
     js_to_json,
     unescapeHTML,
+    determine_ext,
 )
 
 
@@ -39,7 +40,7 @@ class HowStuffWorksIE(InfoExtractor):
             'url': 'http://entertainment.howstuffworks.com/arts/2706-sword-swallowing-1-by-dan-meyer-video.htm',
             'info_dict': {
                 'id': '440011',
-                'ext': 'flv',
+                'ext': 'mp4',
                 'title': 'Sword Swallowing #1 by Dan Meyer',
                 'description': 'Video footage (1 of 3) used by permission of the owner Dan Meyer through Sword Swallowers Association International <www.swordswallow.org>',
                 'display_id': 'sword-swallowing-1-by-dan-meyer',
@@ -63,13 +64,19 @@ class HowStuffWorksIE(InfoExtractor):
         video_id = clip_info['content_id']
         formats = []
         m3u8_url = clip_info.get('m3u8')
-        if m3u8_url:
-            formats += self._extract_m3u8_formats(m3u8_url, video_id, 'mp4')
+        if m3u8_url and determine_ext(m3u8_url) == 'm3u8':
+            formats.extend(self._extract_m3u8_formats(m3u8_url, video_id, 'mp4', format_id='hls', fatal=True))
+        flv_url = clip_info.get('flv_url')
+        if flv_url:
+            formats.append({
+                'url': flv_url,
+                'format_id': 'flv',
+            })
         for video in clip_info.get('mp4', []):
             formats.append({
                 'url': video['src'],
-                'format_id': video['bitrate'],
-                'vbr': int(video['bitrate'].rstrip('k')),
+                'format_id': 'mp4-%s' % video['bitrate'],
+                'vbr': int_or_none(video['bitrate'].rstrip('k')),
             })
 
         if not formats:
@@ -102,6 +109,6 @@ class HowStuffWorksIE(InfoExtractor):
             'title': unescapeHTML(clip_info['clip_title']),
             'description': unescapeHTML(clip_info.get('caption')),
             'thumbnail': clip_info.get('video_still_url'),
-            'duration': clip_info.get('duration'),
+            'duration': int_or_none(clip_info.get('duration')),
             'formats': formats,
         }

From b53a06e3b9f2c4ad86b09d35051f5eff2ad1bff0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 1 Apr 2016 02:24:22 +0600
Subject: [PATCH 4493/6219] [udemy:course] Use new URL format

---
 youtube_dl/extractor/udemy.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index e91cf44fe..a788cdd77 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -305,7 +305,7 @@ class UdemyIE(InfoExtractor):
 
 class UdemyCourseIE(UdemyIE):
     IE_NAME = 'udemy:course'
-    _VALID_URL = r'https?://www\.udemy\.com/(?P<id>[\da-z-]+)'
+    _VALID_URL = r'https?://www\.udemy\.com/(?P<id>[^/?#&]+)'
     _TESTS = []
 
     @classmethod
@@ -338,7 +338,7 @@ class UdemyCourseIE(UdemyIE):
                 if lecture_id:
                     entry = {
                         '_type': 'url_transparent',
-                        'url': 'https://www.udemy.com/%s/#/lecture/%s' % (course_path, entry['id']),
+                        'url': 'https://www.udemy.com/%s/learn/v4/t/lecture/%s' % (course_path, entry['id']),
                         'title': entry.get('title'),
                         'ie_key': UdemyIE.ie_key(),
                     }

From 961fc024d218275221dce8de2015c3900f2557c8 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 31 Mar 2016 23:33:36 +0100
Subject: [PATCH 4494/6219] [voxmedia] improve sbnation support

---
 youtube_dl/extractor/voxmedia.py | 109 ++++++++++++++++++++-----------
 1 file changed, 71 insertions(+), 38 deletions(-)

diff --git a/youtube_dl/extractor/voxmedia.py b/youtube_dl/extractor/voxmedia.py
index 3b13d38a1..0c6b1f030 100644
--- a/youtube_dl/extractor/voxmedia.py
+++ b/youtube_dl/extractor/voxmedia.py
@@ -49,51 +49,84 @@ class VoxMediaIE(InfoExtractor):
             'uploader_id': 'voxdotcom',
             'uploader': 'Vox',
         }
+    }, {
+        # SBN.VideoLinkset.entryGroup multiple ooyala embeds
+        'url': 'http://www.sbnation.com/college-football-recruiting/2015/2/3/7970291/national-signing-day-rationalizations-itll-be-ok-itll-be-ok',
+        'info_dict': {
+            'id': 'national-signing-day-rationalizations-itll-be-ok-itll-be-ok',
+            'title': '25 lies you will tell yourself on National Signing Day',
+            'description': 'It\'s the most self-delusional time of the year, and everyone\'s gonna tell the same lies together!',
+        },
+        'playlist': [{
+            'md5': '721fededf2ab74ae4176c8c8cbfe092e',
+            'info_dict': {
+                'id': 'p3cThlMjE61VDi_SD9JlIteSNPWVDBB9',
+                'ext': 'mp4',
+                'title': 'Buddy Hield vs Steph Curry (and the world)',
+                'description': 'Let’s dissect only the most important Final Four storylines.',
+            },
+        }, {
+            'md5': 'bf0c5cc115636af028be1bab79217ea9',
+            'info_dict': {
+                'id': 'BmbmVjMjE6esPHxdALGubTrouQ0jYLHj',
+                'ext': 'mp4',
+                'title': 'Chasing Cinderella 2016: Syracuse basketball',
+                'description': 'md5:e02d56b026d51aa32c010676765a690d',
+            },
+        }],
     }]
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
         webpage = compat_urllib_parse_unquote(self._download_webpage(url, display_id))
 
-        title = None
-        description = None
-        provider_video_id = None
-        provider_video_type = None
+        def create_entry(provider_video_id, provider_video_type, title=None, description=None):
+            return {
+                '_type': 'url_transparent',
+                'url': provider_video_id if provider_video_type == 'youtube' else '%s:%s' % (provider_video_type, provider_video_id),
+                'title': title or self._og_search_title(webpage),
+                'description': description or self._og_search_description(webpage),
+            }
 
-        entry = self._search_regex([
-            r'Chorus\.VideoContext\.addVideo\(\[({.+})\]\);',
-            r'var\s+entry\s*=\s*({.+});'
+        entries = []
+        entries_data = self._search_regex([
+            r'Chorus\.VideoContext\.addVideo\((\[{.+}\])\);',
+            r'var\s+entry\s*=\s*({.+});',
+            r'SBN\.VideoLinkset\.entryGroup\(\s*(\[.+\])',
         ], webpage, 'video data', default=None)
-        if entry:
-            video_data = self._parse_json(entry, display_id)
-            provider_video_id = video_data.get('provider_video_id')
-            provider_video_type = video_data.get('provider_video_type')
-            if provider_video_id and provider_video_type:
-                title = video_data.get('title')
-                description = video_data.get('description')
+        if entries_data:
+            entries_data = self._parse_json(entries_data, display_id)
+            if isinstance(entries_data, dict):
+                entries_data = [entries_data]
+            for video_data in entries_data:
+                provider_video_id = video_data.get('provider_video_id')
+                provider_video_type = video_data.get('provider_video_type')
+                if provider_video_id and provider_video_type:
+                    entries.append(create_entry(
+                        provider_video_id, provider_video_type,
+                        video_data.get('title'), video_data.get('description')))
 
-        if not provider_video_id or not provider_video_type:
-            provider_video_id = self._search_regex(
-                r'data-ooyala-id="([^"]+)"', webpage, 'ooyala id', default=None)
-            if provider_video_id:
-                provider_video_type = 'ooyala'
-            else:
-                volume_uuid = self._search_regex(r'data-volume-uuid="([^"]+)"', webpage, 'volume uuid')
-                volume_webpage = self._download_webpage(
-                    'http://volume.vox-cdn.com/embed/%s' % volume_uuid, volume_uuid)
-                video_data = self._parse_json(self._search_regex(
-                    r'Volume\.createVideo\(({.+})\s*,\s*{.*}\);', volume_webpage, 'video data'), volume_uuid)
-                title = video_data.get('title_short')
-                description = video_data.get('description_long') or video_data.get('description_short')
-                for pvtype in ('ooyala', 'youtube'):
-                    provider_video_id = video_data.get('%s_id' % pvtype)
-                    if provider_video_id:
-                        provider_video_type = pvtype
-                        break
+        provider_video_id = self._search_regex(
+            r'data-ooyala-id="([^"]+)"', webpage, 'ooyala id', default=None)
+        if provider_video_id:
+            entries.append(create_entry(provider_video_id, 'ooyala'))
 
-        return {
-            '_type': 'url_transparent',
-            'url': provider_video_id if provider_video_type == 'youtube' else '%s:%s' % (provider_video_type, provider_video_id),
-            'title': title or self._og_search_title(webpage),
-            'description': description or self._og_search_description(webpage),
-        }
+        volume_uuid = self._search_regex(
+            r'data-volume-uuid="([^"]+)"', webpage, 'volume uuid', default=None)
+        if volume_uuid:
+            volume_webpage = self._download_webpage(
+                'http://volume.vox-cdn.com/embed/%s' % volume_uuid, volume_uuid)
+            video_data = self._parse_json(self._search_regex(
+                r'Volume\.createVideo\(({.+})\s*,\s*{.*}\);', volume_webpage, 'video data'), volume_uuid)
+            for provider_video_type in ('ooyala', 'youtube'):
+                provider_video_id = video_data.get('%s_id' % provider_video_type)
+                if provider_video_id:
+                    description = video_data.get('description_long') or video_data.get('description_short')
+                    entries.append(create_entry(
+                        provider_video_id, provider_video_type, video_data.get('title_short'), description))
+                    break
+
+        if len(entries) == 1:
+            return entries[0]
+        else:
+            return self.playlist_result(entries, display_id, self._og_search_title(webpage), self._og_search_description(webpage))

From f9b1529af8aec98bffd42edb5be15e1ada791a20 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 31 Mar 2016 23:50:45 +0100
Subject: [PATCH 4495/6219] [generic] remove sbnation test(handled by
 VoxMediaIE)

---
 youtube_dl/extractor/generic.py | 13 -------------
 1 file changed, 13 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index f3de738f7..589d1e152 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -406,19 +406,6 @@ class GenericIE(InfoExtractor):
                 'skip_download': True,
             },
         },
-        # multiple ooyala embeds on SBN network websites
-        {
-            'url': 'http://www.sbnation.com/college-football-recruiting/2015/2/3/7970291/national-signing-day-rationalizations-itll-be-ok-itll-be-ok',
-            'info_dict': {
-                'id': 'national-signing-day-rationalizations-itll-be-ok-itll-be-ok',
-                'title': '25 lies you will tell yourself on National Signing Day - SBNation.com',
-            },
-            'playlist_mincount': 3,
-            'params': {
-                'skip_download': True,
-            },
-            'add_ie': ['Ooyala'],
-        },
         # embed.ly video
         {
             'url': 'http://www.tested.com/science/weird/460206-tested-grinding-coffee-2000-frames-second/',

From 63c55e9f2245b24f437564e616ebff76994978cf Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 1 Apr 2016 07:33:37 +0100
Subject: [PATCH 4496/6219] [cbs] improve extraction(closes #6321)

---
 youtube_dl/extractor/cbs.py | 78 +++++++++++++++++++++++++++----------
 1 file changed, 57 insertions(+), 21 deletions(-)

diff --git a/youtube_dl/extractor/cbs.py b/youtube_dl/extractor/cbs.py
index 40d07ab18..0cf56b9c7 100644
--- a/youtube_dl/extractor/cbs.py
+++ b/youtube_dl/extractor/cbs.py
@@ -1,21 +1,24 @@
 from __future__ import unicode_literals
 
-from .common import InfoExtractor
+from .theplatform import ThePlatformIE
 from ..utils import (
-    sanitized_Request,
-    smuggle_url,
+    xpath_text,
+    xpath_element,
+    int_or_none,
+    ExtractorError,
+    find_xpath_attr,
 )
 
 
-class CBSIE(InfoExtractor):
+class CBSIE(ThePlatformIE):
     _VALID_URL = r'https?://(?:www\.)?(?:cbs\.com/shows/[^/]+/(?:video|artist)|colbertlateshow\.com/(?:video|podcasts))/[^/]+/(?P<id>[^/]+)'
 
     _TESTS = [{
         'url': 'http://www.cbs.com/shows/garth-brooks/video/_u7W953k6la293J7EPTd9oHkSPs6Xn6_/connect-chat-feat-garth-brooks/',
         'info_dict': {
-            'id': '4JUVEwq3wUT7',
+            'id': '_u7W953k6la293J7EPTd9oHkSPs6Xn6_',
             'display_id': 'connect-chat-feat-garth-brooks',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Connect Chat feat. Garth Brooks',
             'description': 'Connect with country music singer Garth Brooks, as he chats with fans on Wednesday November 27, 2013. Be sure to tune in to Garth Brooks: Live from Las Vegas, Friday November 29, at 9/8c on CBS!',
             'duration': 1495,
@@ -47,22 +50,55 @@ class CBSIE(InfoExtractor):
         'url': 'http://www.colbertlateshow.com/podcasts/dYSwjqPs_X1tvbV_P2FcPWRa_qT6akTC/in-the-bad-room-with-stephen/',
         'only_matching': True,
     }]
+    TP_RELEASE_URL_TEMPLATE = 'http://link.theplatform.com/s/dJ5BDC/%s?manifest=m3u&mbr=true'
+
+    def _parse_smil_subtitles(self, smil, namespace=None, subtitles_lang='en'):
+        closed_caption_e = find_xpath_attr(smil, self._xpath_ns('.//param', namespace), 'name', 'ClosedCaptionURL')
+        return {
+            'en': [{
+                'ext': 'ttml',
+                'url': closed_caption_e.attrib['value'],
+            }]
+        } if closed_caption_e is not None and closed_caption_e.attrib.get('value') else []
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
-        request = sanitized_Request(url)
-        # Android UA is served with higher quality (720p) streams (see
-        # https://github.com/rg3/youtube-dl/issues/7490)
-        request.add_header('User-Agent', 'Mozilla/5.0 (Linux; Android 4.4; Nexus 5)')
-        webpage = self._download_webpage(request, display_id)
-        real_id = self._search_regex(
-            [r"video\.settings\.pid\s*=\s*'([^']+)';", r"cbsplayer\.pid\s*=\s*'([^']+)';"],
-            webpage, 'real video ID')
-        return {
-            '_type': 'url_transparent',
-            'ie_key': 'ThePlatform',
-            'url': smuggle_url(
-                'http://link.theplatform.com/s/dJ5BDC/%s?mbr=true&manifest=m3u' % real_id,
-                {'force_smil_url': True}),
+        webpage = self._download_webpage(url, display_id)
+        content_id = self._search_regex(
+            [r"video\.settings\.content_id\s*=\s*'([^']+)';", r"cbsplayer\.contentId\s*=\s*'([^']+)';"],
+            webpage, 'content id')
+        items_data = self._download_xml(
+            'http://can.cbs.com/thunder/player/videoPlayerService.php',
+            content_id, query={'partner': 'cbs', 'contentId': content_id})
+        video_data = xpath_element(items_data, './/item')
+        title = xpath_text(video_data, 'videoTitle', 'title', True)
+
+        subtitles = {}
+        formats = []
+        for item in items_data.findall('.//item'):
+            pid = xpath_text(item, 'pid')
+            if not pid:
+                continue
+            try:
+                tp_formats, tp_subtitles = self._extract_theplatform_smil(
+                    self.TP_RELEASE_URL_TEMPLATE % pid, content_id, 'Downloading %s SMIL data' % pid)
+            except ExtractorError:
+                continue
+            formats.extend(tp_formats)
+            subtitles = self._merge_subtitles(subtitles, tp_subtitles)
+        self._sort_formats(formats)
+
+        info = self.get_metadata('dJ5BDC/media/guid/2198311517/%s' % content_id, content_id)
+        info.update({
+            'id': content_id,
             'display_id': display_id,
-        }
+            'title': title,
+            'series': xpath_text(video_data, 'seriesTitle'),
+            'season_number': int_or_none(xpath_text(video_data, 'seasonNumber')),
+            'episode_number': int_or_none(xpath_text(video_data, 'episodeNumber')),
+            'duration': int_or_none(xpath_text(video_data, 'videoLength'), 1000),
+            'thumbnail': xpath_text(video_data, 'previewImageURL'),
+            'formats': formats,
+            'subtitles': subtitles,
+        })
+        return info

From 1e02bc7ba28bfb5859dcd0c8d960b965d9c59f12 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 1 Apr 2016 09:07:40 +0200
Subject: [PATCH 4497/6219] release 2016.04.01

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 docs/supportedsites.md    | 7 +++++--
 youtube_dl/version.py     | 2 +-
 3 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 5b1f573e7..128ba2fc0 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.03.27*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.03.27**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.04.01*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.04.01**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.03.27
+[debug] youtube-dl version 2016.04.01
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 00b8c247c..b9bcf450c 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -118,6 +118,7 @@
  - **Clubic**
  - **Clyp**
  - **cmt.com**
+ - **CNBC**
  - **CNET**
  - **CNN**
  - **CNNArticle**
@@ -134,6 +135,7 @@
  - **CrooksAndLiars**
  - **Crunchyroll**
  - **crunchyroll:playlist**
+ - **CSNNE**
  - **CSpan**: C-SPAN
  - **CtsNews**: 華視新聞
  - **culturebox.francetvinfo.fr**
@@ -376,7 +378,8 @@
  - **myvideo** (Currently broken)
  - **MyVidster**
  - **n-tv.de**
- - **NationalGeographic**
+ - **natgeo**
+ - **natgeo:channel**
  - **Naver**
  - **NBA**
  - **NBC**
@@ -618,7 +621,6 @@
  - **Telegraaf**
  - **TeleMB**
  - **TeleTask**
- - **TenPlay**
  - **TF1**
  - **TheIntercept**
  - **TheOnion**
@@ -740,6 +742,7 @@
  - **vlive**
  - **Vodlocker**
  - **VoiceRepublic**
+ - **VoxMedia**
  - **Vporn**
  - **vpro**: npo.nl and ntr.nl
  - **VRT**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 5daa7f4e8..d9e1cb2a8 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.03.27'
+__version__ = '2016.04.01'

From db1c969da5dc22b36228b50ac2c09f3a50dd17ae Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 1 Apr 2016 09:55:46 +0100
Subject: [PATCH 4498/6219] [theplatform] sign https urls

---
 youtube_dl/extractor/theplatform.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index 236c99972..bf6f82f5a 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -154,7 +154,7 @@ class ThePlatformIE(ThePlatformBaseIE):
         def hex_to_str(hex):
             return binascii.a2b_hex(hex)
 
-        relative_path = url.split('http://link.theplatform.com/s/')[1].split('?')[0]
+        relative_path = re.match(r'https?://link.theplatform.com/s/([^?]+)', url).group(1)
         clear_text = hex_to_str(flags + expiration_date + str_to_hex(relative_path))
         checksum = hmac.new(sig_key.encode('ascii'), clear_text, hashlib.sha1).hexdigest()
         sig = flags + expiration_date + checksum + str_to_hex(sig_secret)

From d8873d4defdd527c82634bea8566370f5d385020 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 1 Apr 2016 09:56:18 +0100
Subject: [PATCH 4499/6219] [aenetworks] improve format extraction

---
 youtube_dl/extractor/aenetworks.py | 32 +++++++++++++++++++++---------
 1 file changed, 23 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/aenetworks.py b/youtube_dl/extractor/aenetworks.py
index 6018ae79a..b7232c904 100644
--- a/youtube_dl/extractor/aenetworks.py
+++ b/youtube_dl/extractor/aenetworks.py
@@ -1,13 +1,19 @@
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
-from ..utils import smuggle_url
+from ..utils import (
+    smuggle_url,
+    update_url_query,
+    unescapeHTML,
+)
 
 
 class AENetworksIE(InfoExtractor):
     IE_NAME = 'aenetworks'
     IE_DESC = 'A+E Networks: A&E, Lifetime, History.com, FYI Network'
-    _VALID_URL = r'https?://(?:www\.)?(?:(?:history|aetv|mylifetime)\.com|fyi\.tv)/(?:[^/]+/)+(?P<id>[^/]+?)(?:$|[?#])'
+    _VALID_URL = r'https?://(?:www\.)?(?:(?:history|aetv|mylifetime)\.com|fyi\.tv)/(?P<type>[^/]+)/(?:[^/]+/)+(?P<id>[^/]+?)(?:$|[?#])'
 
     _TESTS = [{
         'url': 'http://www.history.com/topics/valentines-day/history-of-valentines-day/videos/bet-you-didnt-know-valentines-day?m=528e394da93ae&s=undefined&f=1&free=false',
@@ -25,16 +31,13 @@ class AENetworksIE(InfoExtractor):
         'expected_warnings': ['JSON-LD'],
     }, {
         'url': 'http://www.history.com/shows/mountain-men/season-1/episode-1',
+        'md5': '8ff93eb073449f151d6b90c0ae1ef0c7',
         'info_dict': {
             'id': 'eg47EERs_JsZ',
             'ext': 'mp4',
             'title': 'Winter Is Coming',
             'description': 'md5:641f424b7a19d8e24f26dea22cf59d74',
         },
-        'params': {
-            # m3u8 download
-            'skip_download': True,
-        },
         'add_ie': ['ThePlatform'],
     }, {
         'url': 'http://www.aetv.com/shows/duck-dynasty/video/inlawful-entry',
@@ -48,7 +51,7 @@ class AENetworksIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        page_type, video_id = re.match(self._VALID_URL, url).groups()
 
         webpage = self._download_webpage(url, video_id)
 
@@ -56,11 +59,22 @@ class AENetworksIE(InfoExtractor):
             r'data-href="[^"]*/%s"[^>]+data-release-url="([^"]+)"' % video_id,
             r"media_url\s*=\s*'([^']+)'"
         ]
-        video_url = self._search_regex(video_url_re, webpage, 'video url')
+        video_url = unescapeHTML(self._search_regex(video_url_re, webpage, 'video url'))
+        query = {'mbr': 'true'}
+        if page_type == 'shows':
+            query['assetTypes'] = 'medium_video_s3'
+        if 'switch=hds' in video_url:
+            query['switch'] = 'hls'
 
         info = self._search_json_ld(webpage, video_id, fatal=False)
         info.update({
             '_type': 'url_transparent',
-            'url': smuggle_url(video_url, {'sig': {'key': 'crazyjava', 'secret': 's3cr3t'}}),
+            'url': smuggle_url(update_url_query(
+                video_url, query), {
+                    'sig': {
+                        'key': 'crazyjava',
+                        'secret': 's3cr3t'},
+                    'force_smil_url': True
+                }),
         })
         return info

From 3e0c3d14d9ab49f3a98324e91763c849ef52a1df Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 1 Apr 2016 10:12:29 +0100
Subject: [PATCH 4500/6219] [cbs] add base extractor

---
 youtube_dl/extractor/cbs.py     | 22 ++++++++++++----------
 youtube_dl/extractor/cbsnews.py | 13 ++-----------
 2 files changed, 14 insertions(+), 21 deletions(-)

diff --git a/youtube_dl/extractor/cbs.py b/youtube_dl/extractor/cbs.py
index 0cf56b9c7..6e4079ca3 100644
--- a/youtube_dl/extractor/cbs.py
+++ b/youtube_dl/extractor/cbs.py
@@ -10,7 +10,18 @@ from ..utils import (
 )
 
 
-class CBSIE(ThePlatformIE):
+class CBSBaseIE(ThePlatformIE):
+    def _parse_smil_subtitles(self, smil, namespace=None, subtitles_lang='en'):
+        closed_caption_e = find_xpath_attr(smil, self._xpath_ns('.//param', namespace), 'name', 'ClosedCaptionURL')
+        return {
+            'en': [{
+                'ext': 'ttml',
+                'url': closed_caption_e.attrib['value'],
+            }]
+        } if closed_caption_e is not None and closed_caption_e.attrib.get('value') else []
+
+
+class CBSIE(CBSBaseIE):
     _VALID_URL = r'https?://(?:www\.)?(?:cbs\.com/shows/[^/]+/(?:video|artist)|colbertlateshow\.com/(?:video|podcasts))/[^/]+/(?P<id>[^/]+)'
 
     _TESTS = [{
@@ -52,15 +63,6 @@ class CBSIE(ThePlatformIE):
     }]
     TP_RELEASE_URL_TEMPLATE = 'http://link.theplatform.com/s/dJ5BDC/%s?manifest=m3u&mbr=true'
 
-    def _parse_smil_subtitles(self, smil, namespace=None, subtitles_lang='en'):
-        closed_caption_e = find_xpath_attr(smil, self._xpath_ns('.//param', namespace), 'name', 'ClosedCaptionURL')
-        return {
-            'en': [{
-                'ext': 'ttml',
-                'url': closed_caption_e.attrib['value'],
-            }]
-        } if closed_caption_e is not None and closed_caption_e.attrib.get('value') else []
-
     def _real_extract(self, url):
         display_id = self._match_id(url)
         webpage = self._download_webpage(url, display_id)
diff --git a/youtube_dl/extractor/cbsnews.py b/youtube_dl/extractor/cbsnews.py
index e6b7f3584..b5e78a65d 100644
--- a/youtube_dl/extractor/cbsnews.py
+++ b/youtube_dl/extractor/cbsnews.py
@@ -2,14 +2,14 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from .theplatform import ThePlatformIE
+from .cbs import CBSBaseIE
 from ..utils import (
     parse_duration,
     find_xpath_attr,
 )
 
 
-class CBSNewsIE(ThePlatformIE):
+class CBSNewsIE(CBSBaseIE):
     IE_DESC = 'CBS News'
     _VALID_URL = r'https?://(?:www\.)?cbsnews\.com/(?:news|videos)/(?P<id>[\da-z_-]+)'
 
@@ -49,15 +49,6 @@ class CBSNewsIE(ThePlatformIE):
         },
     ]
 
-    def _parse_smil_subtitles(self, smil, namespace=None, subtitles_lang='en'):
-        closed_caption_e = find_xpath_attr(smil, self._xpath_ns('.//param', namespace), 'name', 'ClosedCaptionURL')
-        return {
-            'en': [{
-                'ext': 'ttml',
-                'url': closed_caption_e.attrib['value'],
-            }]
-        } if closed_caption_e is not None and closed_caption_e.attrib.get('value') else []
-
     def _real_extract(self, url):
         video_id = self._match_id(url)
 

From 3f64379eda3477306df013466045ab1a711533f4 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 1 Apr 2016 16:22:06 +0100
Subject: [PATCH 4501/6219] [movieclips] fix extraction

---
 youtube_dl/extractor/movieclips.py | 43 ++++++++++++++++++------------
 1 file changed, 26 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/movieclips.py b/youtube_dl/extractor/movieclips.py
index 1564cb71f..d0cb8278e 100644
--- a/youtube_dl/extractor/movieclips.py
+++ b/youtube_dl/extractor/movieclips.py
@@ -2,39 +2,48 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..utils import sanitized_Request
+from ..utils import (
+    smuggle_url,
+    float_or_none,
+    parse_iso8601,
+    update_url_query,
+)
 
 
 class MovieClipsIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www.)?movieclips\.com/videos/(?P<id>[^/?#]+)'
+    _VALID_URL = r'https?://(?:www.)?movieclips\.com/videos/.+-(?P<id>\d+)(?:\?|$)'
     _TEST = {
-        'url': 'http://www.movieclips.com/videos/warcraft-trailer-1-561180739597?autoPlay=true&playlistId=5',
+        'url': 'http://www.movieclips.com/videos/warcraft-trailer-1-561180739597',
+        'md5': '42b5a0352d4933a7bd54f2104f481244',
         'info_dict': {
             'id': 'pKIGmG83AqD9',
-            'display_id': 'warcraft-trailer-1-561180739597',
             'ext': 'mp4',
             'title': 'Warcraft Trailer 1',
             'description': 'Watch Trailer 1 from Warcraft (2016). Legendary’s WARCRAFT is a 3D epic adventure of world-colliding conflict based.',
             'thumbnail': 're:^https?://.*\.jpg$',
+            'timestamp': 1446843055,
+            'upload_date': '20151106',
+            'uploader': 'Movieclips',
         },
         'add_ie': ['ThePlatform'],
     }
 
     def _real_extract(self, url):
-        display_id = self._match_id(url)
-
-        req = sanitized_Request(url)
-        # it doesn't work if it thinks the browser it's too old
-        req.add_header('User-Agent', 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20150101 Firefox/43.0 (Chrome)')
-        webpage = self._download_webpage(req, display_id)
-        theplatform_link = self._html_search_regex(r'src="(http://player.theplatform.com/p/.*?)"', webpage, 'theplatform link')
-        title = self._html_search_regex(r'<title[^>]*>([^>]+)-\s*\d+\s*|\s*Movieclips.com</title>', webpage, 'title')
-        description = self._html_search_meta('description', webpage)
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        video = next(v for v in self._parse_json(self._search_regex(
+            r'var\s+__REACT_ENGINE__\s*=\s*({.+});',
+            webpage, 'react engine'), video_id)['playlist']['videos'] if v['id'] == video_id)
 
         return {
             '_type': 'url_transparent',
-            'url': theplatform_link,
-            'title': title,
-            'display_id': display_id,
-            'description': description,
+            'ie_key': 'ThePlatform',
+            'url': smuggle_url(update_url_query(
+                video['contentUrl'], {'mbr': 'true'}), {'force_smil_url': True}),
+            'title': self._og_search_title(webpage),
+            'description': self._html_search_meta('description', webpage),
+            'duration': float_or_none(video.get('duration')),
+            'timestamp': parse_iso8601(video.get('dateCreated')),
+            'thumbnail': video.get('defaultImage'),
+            'uploader': video.get('provider'),
         }

From 03caa463e73c2ae2f666b85febf25ddb03f961ca Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 1 Apr 2016 22:38:56 +0600
Subject: [PATCH 4502/6219] [udemy:course] Skip non-video lectures

---
 youtube_dl/extractor/udemy.py | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index a788cdd77..bc69e6e41 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -193,12 +193,12 @@ class UdemyIE(InfoExtractor):
 
         asset = lecture['asset']
 
-        asset_type = asset.get('assetType') or asset.get('asset_type')
+        asset_type = asset.get('asset_type') or asset.get('assetType')
         if asset_type != 'Video':
             raise ExtractorError(
                 'Lecture %s is not a video' % lecture_id, expected=True)
 
-        stream_url = asset.get('streamUrl') or asset.get('stream_url')
+        stream_url = asset.get('stream_url') or asset.get('streamUrl')
         if stream_url:
             youtube_url = self._search_regex(
                 r'(https?://www\.youtube\.com/watch\?v=.*)', stream_url, 'youtube URL', default=None)
@@ -206,7 +206,7 @@ class UdemyIE(InfoExtractor):
                 return self.url_result(youtube_url, 'Youtube')
 
         video_id = asset['id']
-        thumbnail = asset.get('thumbnailUrl') or asset.get('thumbnail_url')
+        thumbnail = asset.get('thumbnail_url') or asset.get('thumbnailUrl')
         duration = float_or_none(asset.get('data', {}).get('duration'))
 
         formats = []
@@ -325,7 +325,7 @@ class UdemyCourseIE(UdemyIE):
             'https://www.udemy.com/api-2.0/courses/%s/cached-subscriber-curriculum-items' % course_id,
             course_id, 'Downloading course curriculum', query={
                 'fields[chapter]': 'title,object_index',
-                'fields[lecture]': 'title',
+                'fields[lecture]': 'title,asset',
                 'page_size': '1000',
             })
 
@@ -334,6 +334,11 @@ class UdemyCourseIE(UdemyIE):
         for entry in response['results']:
             clazz = entry.get('_class')
             if clazz == 'lecture':
+                asset = entry.get('asset')
+                if isinstance(asset, dict):
+                    asset_type = asset.get('asset_type') or asset.get('assetType')
+                    if asset_type != 'Video':
+                        continue
                 lecture_id = entry.get('id')
                 if lecture_id:
                     entry = {

From a3373823e1bd0239e0f58d5dd16ef5a4ec6bceb3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 1 Apr 2016 22:42:09 +0600
Subject: [PATCH 4503/6219] [udemy] Remove unnecessary login/password encode

This is now covered by compat_urllib_parse_urlencode
---
 youtube_dl/extractor/udemy.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index bc69e6e41..d1e6f2703 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -151,8 +151,8 @@ class UdemyIE(InfoExtractor):
         login_form = self._form_hidden_inputs('login-form', login_popup)
 
         login_form.update({
-            'email': username.encode('utf-8'),
-            'password': password.encode('utf-8'),
+            'email': username,
+            'password': password,
         })
 
         request = sanitized_Request(

From fbdaced256f9d7d9b0adb97d093f0f381c9483f7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 1 Apr 2016 22:45:20 +0600
Subject: [PATCH 4504/6219] [lynda] Remove unnecessary login/password encode

---
 youtube_dl/extractor/lynda.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index 655627479..86d47266f 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -28,8 +28,8 @@ class LyndaBaseIE(InfoExtractor):
             return
 
         login_form = {
-            'username': username.encode('utf-8'),
-            'password': password.encode('utf-8'),
+            'username': username,
+            'password': password,
             'remember': 'false',
             'stayPut': 'false'
         }

From 244cd04237fe4a1e4d92421711f41de3c2566d5c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 1 Apr 2016 22:46:46 +0600
Subject: [PATCH 4505/6219] [pluralsight] Remove unnecessary login/password
 encode

---
 youtube_dl/extractor/pluralsight.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index df03dd419..9aab77645 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -64,8 +64,8 @@ class PluralsightIE(PluralsightBaseIE):
         login_form = self._hidden_inputs(login_page)
 
         login_form.update({
-            'Username': username.encode('utf-8'),
-            'Password': password.encode('utf-8'),
+            'Username': username,
+            'Password': password,
         })
 
         post_url = self._search_regex(

From 83cedc1cf224206adf513f5bdd5f5ce915d67933 Mon Sep 17 00:00:00 2001
From: Martin Trigaux <me@mart-e.be>
Date: Tue, 29 Mar 2016 14:18:44 +0200
Subject: [PATCH 4506/6219] screencast.com: support missing www

The "www." part of the URL is not mandatory
---
 youtube_dl/extractor/screencast.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/screencast.py b/youtube_dl/extractor/screencast.py
index dfd897ba3..d5111c629 100644
--- a/youtube_dl/extractor/screencast.py
+++ b/youtube_dl/extractor/screencast.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 
 class ScreencastIE(InfoExtractor):
-    _VALID_URL = r'https?://www\.screencast\.com/t/(?P<id>[a-zA-Z0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?screencast\.com/t/(?P<id>[a-zA-Z0-9]+)'
     _TESTS = [{
         'url': 'http://www.screencast.com/t/3ZEjQXlT',
         'md5': '917df1c13798a3e96211dd1561fded83',
@@ -34,7 +34,7 @@ class ScreencastIE(InfoExtractor):
             'thumbnail': 're:^https?://.*\.(?:gif|jpg)$',
         }
     }, {
-        'url': 'http://www.screencast.com/t/aAB3iowa',
+        'url': 'http://screencast.com/t/aAB3iowa',
         'md5': 'dedb2734ed00c9755761ccaee88527cd',
         'info_dict': {
             'id': 'aAB3iowa',

From 81de73e5b43e5009a14f569aed92fe73e61d4f03 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 1 Apr 2016 23:00:45 +0600
Subject: [PATCH 4507/6219] [screencast] Add test

---
 youtube_dl/extractor/screencast.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/screencast.py b/youtube_dl/extractor/screencast.py
index d5111c629..c69451151 100644
--- a/youtube_dl/extractor/screencast.py
+++ b/youtube_dl/extractor/screencast.py
@@ -34,7 +34,7 @@ class ScreencastIE(InfoExtractor):
             'thumbnail': 're:^https?://.*\.(?:gif|jpg)$',
         }
     }, {
-        'url': 'http://screencast.com/t/aAB3iowa',
+        'url': 'http://www.screencast.com/t/aAB3iowa',
         'md5': 'dedb2734ed00c9755761ccaee88527cd',
         'info_dict': {
             'id': 'aAB3iowa',
@@ -53,8 +53,10 @@ class ScreencastIE(InfoExtractor):
             'description': 'md5:7b9f393bc92af02326a5c5889639eab0',
             'thumbnail': 're:^https?://.*\.(?:gif|jpg)$',
         }
-    },
-    ]
+    }, {
+        'url': 'http://screencast.com/t/aAB3iowa',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From 791d6aaeccd2efae2c4c5fa1e72010be85eb89b8 Mon Sep 17 00:00:00 2001
From: Martin Trigaux <me@mart-e.be>
Date: Tue, 29 Mar 2016 14:34:58 +0200
Subject: [PATCH 4508/6219] screencast.com: fallback on page title

When determining the title of the page, use the <title> tag of the page
---
 youtube_dl/extractor/screencast.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/screencast.py b/youtube_dl/extractor/screencast.py
index c69451151..32f31fdd7 100644
--- a/youtube_dl/extractor/screencast.py
+++ b/youtube_dl/extractor/screencast.py
@@ -97,7 +97,8 @@ class ScreencastIE(InfoExtractor):
         if title is None:
             title = self._html_search_regex(
                 [r'<b>Title:</b> ([^<]*)</div>',
-                 r'class="tabSeperator">></span><span class="tabText">(.*?)<'],
+                 r'class="tabSeperator">></span><span class="tabText">(.*?)<',
+                 r'<title>([^<]*)</title>'],
                 webpage, 'title')
         thumbnail = self._og_search_thumbnail(webpage)
         description = self._og_search_description(webpage, default=None)

From 75d572e9fb8d3e26e4ab45e65cd5e23c6b1c6915 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 1 Apr 2016 23:01:55 +0600
Subject: [PATCH 4509/6219] [screencast] Improve title regexes (Closes #9025)

---
 youtube_dl/extractor/screencast.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/screencast.py b/youtube_dl/extractor/screencast.py
index 32f31fdd7..356631700 100644
--- a/youtube_dl/extractor/screencast.py
+++ b/youtube_dl/extractor/screencast.py
@@ -96,9 +96,9 @@ class ScreencastIE(InfoExtractor):
         title = self._og_search_title(webpage, default=None)
         if title is None:
             title = self._html_search_regex(
-                [r'<b>Title:</b> ([^<]*)</div>',
-                 r'class="tabSeperator">></span><span class="tabText">(.*?)<',
-                 r'<title>([^<]*)</title>'],
+                [r'<b>Title:</b> ([^<]+)</div>',
+                 r'class="tabSeperator">></span><span class="tabText">(.+?)<',
+                 r'<title>([^<]+)</title>'],
                 webpage, 'title')
         thumbnail = self._og_search_thumbnail(webpage)
         description = self._og_search_description(webpage, default=None)

From 79ba9140dc8fcf5883b7473596e8f20cba6b479f Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 1 Apr 2016 18:06:11 +0100
Subject: [PATCH 4510/6219] [theplatform] extract timestamp and uploader

---
 youtube_dl/extractor/aenetworks.py         |  6 ++++++
 youtube_dl/extractor/bravotv.py            |  3 +++
 youtube_dl/extractor/cbs.py                |  3 +++
 youtube_dl/extractor/cnbc.py               |  3 +++
 youtube_dl/extractor/fox.py                |  3 +++
 youtube_dl/extractor/nationalgeographic.py | 12 ++++++++++++
 youtube_dl/extractor/nbc.py                | 14 ++++++++++++++
 youtube_dl/extractor/sbs.py                |  6 +++++-
 youtube_dl/extractor/theplatform.py        | 10 ++++++++++
 9 files changed, 59 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/aenetworks.py b/youtube_dl/extractor/aenetworks.py
index b7232c904..3fddaba54 100644
--- a/youtube_dl/extractor/aenetworks.py
+++ b/youtube_dl/extractor/aenetworks.py
@@ -22,6 +22,9 @@ class AENetworksIE(InfoExtractor):
             'ext': 'mp4',
             'title': "Bet You Didn't Know: Valentine's Day",
             'description': 'md5:7b57ea4829b391995b405fa60bd7b5f7',
+            'timestamp': 1375819729,
+            'upload_date': '20130806',
+            'uploader': 'AENE-NEW',
         },
         'params': {
             # m3u8 download
@@ -37,6 +40,9 @@ class AENetworksIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Winter Is Coming',
             'description': 'md5:641f424b7a19d8e24f26dea22cf59d74',
+            'timestamp': 1338306241,
+            'upload_date': '20120529',
+            'uploader': 'AENE-NEW',
         },
         'add_ie': ['ThePlatform'],
     }, {
diff --git a/youtube_dl/extractor/bravotv.py b/youtube_dl/extractor/bravotv.py
index 34d451f38..541c76944 100644
--- a/youtube_dl/extractor/bravotv.py
+++ b/youtube_dl/extractor/bravotv.py
@@ -15,6 +15,9 @@ class BravoTVIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Last Chance Kitchen Returns',
             'description': 'S13: Last Chance Kitchen Returns for Top Chef Season 13',
+            'timestamp': 1448926740,
+            'upload_date': '20151130',
+            'uploader': 'NBCU-BRAV',
         }
     }
 
diff --git a/youtube_dl/extractor/cbs.py b/youtube_dl/extractor/cbs.py
index 6e4079ca3..c621a08d5 100644
--- a/youtube_dl/extractor/cbs.py
+++ b/youtube_dl/extractor/cbs.py
@@ -33,6 +33,9 @@ class CBSIE(CBSBaseIE):
             'title': 'Connect Chat feat. Garth Brooks',
             'description': 'Connect with country music singer Garth Brooks, as he chats with fans on Wednesday November 27, 2013. Be sure to tune in to Garth Brooks: Live from Las Vegas, Friday November 29, at 9/8c on CBS!',
             'duration': 1495,
+            'timestamp': 1385585425,
+            'upload_date': '20131127',
+            'uploader': 'CBSI-NEW',
         },
         'params': {
             # rtmp download
diff --git a/youtube_dl/extractor/cnbc.py b/youtube_dl/extractor/cnbc.py
index 25b308752..d354d9f95 100644
--- a/youtube_dl/extractor/cnbc.py
+++ b/youtube_dl/extractor/cnbc.py
@@ -14,6 +14,9 @@ class CNBCIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Fighting zombies is big business',
             'description': 'md5:0c100d8e1a7947bd2feec9a5550e519e',
+            'timestamp': 1459332000,
+            'upload_date': '20160330',
+            'uploader': 'NBCU-CNBC',
         },
         'params': {
             # m3u8 download
diff --git a/youtube_dl/extractor/fox.py b/youtube_dl/extractor/fox.py
index fa05af50d..95c1abf94 100644
--- a/youtube_dl/extractor/fox.py
+++ b/youtube_dl/extractor/fox.py
@@ -16,6 +16,9 @@ class FOXIE(InfoExtractor):
             'title': 'Official Trailer: Gotham',
             'description': 'Tracing the rise of the great DC Comics Super-Villains and vigilantes, Gotham reveals an entirely new chapter that has never been told.',
             'duration': 129,
+            'timestamp': 1400020798,
+            'upload_date': '20140513',
+            'uploader': 'NEWA-FNG-FOXCOM',
         },
         'add_ie': ['ThePlatform'],
     }
diff --git a/youtube_dl/extractor/nationalgeographic.py b/youtube_dl/extractor/nationalgeographic.py
index 61b5c700e..722518663 100644
--- a/youtube_dl/extractor/nationalgeographic.py
+++ b/youtube_dl/extractor/nationalgeographic.py
@@ -21,6 +21,9 @@ class NationalGeographicIE(InfoExtractor):
                 'ext': 'mp4',
                 'title': 'Mating Crabs Busted by Sharks',
                 'description': 'md5:16f25aeffdeba55aaa8ec37e093ad8b3',
+                'timestamp': 1423523799,
+                'upload_date': '20150209',
+                'uploader': 'NAGS',
             },
             'add_ie': ['ThePlatform'],
         },
@@ -32,6 +35,9 @@ class NationalGeographicIE(InfoExtractor):
                 'ext': 'mp4',
                 'title': 'The Real Jaws',
                 'description': 'md5:8d3e09d9d53a85cd397b4b21b2c77be6',
+                'timestamp': 1433772632,
+                'upload_date': '20150608',
+                'uploader': 'NAGS',
             },
             'add_ie': ['ThePlatform'],
         },
@@ -68,6 +74,9 @@ class NationalGeographicChannelIE(InfoExtractor):
                 'ext': 'mp4',
                 'title': 'Uncovering a Universal Knowledge',
                 'description': 'md5:1a89148475bf931b3661fcd6ddb2ae3a',
+                'timestamp': 1458680907,
+                'upload_date': '20160322',
+                'uploader': 'NEWA-FNG-NGTV',
             },
             'add_ie': ['ThePlatform'],
         },
@@ -79,6 +88,9 @@ class NationalGeographicChannelIE(InfoExtractor):
                 'ext': 'mp4',
                 'title': 'The Stunning Red Bird of Paradise',
                 'description': 'md5:7bc8cd1da29686be4d17ad1230f0140c',
+                'timestamp': 1459362152,
+                'upload_date': '20160330',
+                'uploader': 'NEWA-FNG-NGTV',
             },
             'add_ie': ['ThePlatform'],
         },
diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index 43d75d3ca..e67025ff6 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -27,6 +27,9 @@ class NBCIE(InfoExtractor):
                 'ext': 'mp4',
                 'title': 'Jimmy Fallon Surprises Fans at Ben & Jerry\'s',
                 'description': 'Jimmy gives out free scoops of his new "Tonight Dough" ice cream flavor by surprising customers at the Ben & Jerry\'s scoop shop.',
+                'timestamp': 1424246400,
+                'upload_date': '20150218',
+                'uploader': 'NBCU-COM',
             },
             'params': {
                 # m3u8 download
@@ -50,6 +53,9 @@ class NBCIE(InfoExtractor):
                 'ext': 'mp4',
                 'title': 'Star Wars Teaser',
                 'description': 'md5:0b40f9cbde5b671a7ff62fceccc4f442',
+                'timestamp': 1417852800,
+                'upload_date': '20141206',
+                'uploader': 'NBCU-COM',
             },
             'params': {
                 # m3u8 download
@@ -78,6 +84,7 @@ class NBCIE(InfoExtractor):
             theplatform_url = 'http:' + theplatform_url
         return {
             '_type': 'url_transparent',
+            'ie_key': 'ThePlatform',
             'url': smuggle_url(theplatform_url, {'source_url': url}),
             'id': video_id,
         }
@@ -93,6 +100,9 @@ class NBCSportsVPlayerIE(InfoExtractor):
             'ext': 'flv',
             'description': 'md5:df390f70a9ba7c95ff1daace988f0d8d',
             'title': 'Tyler Kalinoski hits buzzer-beater to lift Davidson',
+            'timestamp': 1426270238,
+            'upload_date': '20150313',
+            'uploader': 'NBCU-SPORTS',
         }
     }, {
         'url': 'http://vplayer.nbcsports.com/p/BxmELC/nbc_embedshare/select/_hqLjQ95yx8Z',
@@ -144,6 +154,9 @@ class CSNNEIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'SNC evening update: Wright named Red Sox\' No. 5 starter.',
             'description': 'md5:1753cfee40d9352b19b4c9b3e589b9e3',
+            'timestamp': 1459369979,
+            'upload_date': '20160330',
+            'uploader': 'NBCU-SPORTS',
         }
     }
 
@@ -331,6 +344,7 @@ class MSNBCIE(InfoExtractor):
             'thumbnail': 're:^https?://.*\.jpg$',
             'timestamp': 1406937606,
             'upload_date': '20140802',
+            'uploader': 'NBCU-NEWS',
             'categories': ['MSNBC/Topics/Franchise/Best of last night', 'MSNBC/Topics/General/Congress'],
         },
     }
diff --git a/youtube_dl/extractor/sbs.py b/youtube_dl/extractor/sbs.py
index 2f96477ca..96472fbc4 100644
--- a/youtube_dl/extractor/sbs.py
+++ b/youtube_dl/extractor/sbs.py
@@ -24,6 +24,9 @@ class SBSIE(InfoExtractor):
             'description': 'md5:f250a9856fca50d22dec0b5b8015f8a5',
             'thumbnail': 're:http://.*\.jpg',
             'duration': 308,
+            'timestamp': 1408613220,
+            'upload_date': '20140821',
+            'uploader': 'SBSC',
         },
     }, {
         'url': 'http://www.sbs.com.au/ondemand/video/320403011771/Dingo-Conservation-The-Feed',
@@ -57,6 +60,7 @@ class SBSIE(InfoExtractor):
 
         return {
             '_type': 'url_transparent',
+            'ie_key': 'ThePlatform',
             'id': video_id,
-            'url': smuggle_url(theplatform_url, {'force_smil_url': True}),
+            'url': smuggle_url(self._proto_relative_url(theplatform_url), {'force_smil_url': True}),
         }
diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index bf6f82f5a..6da701a39 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -76,6 +76,8 @@ class ThePlatformBaseIE(OnceIE):
             'description': info['description'],
             'thumbnail': info['defaultThumbnailUrl'],
             'duration': int_or_none(info.get('duration'), 1000),
+            'timestamp': int_or_none(info.get('pubDate'), 1000) or None,
+            'uploader': info.get('billingCode'),
         }
 
 
@@ -94,6 +96,9 @@ class ThePlatformIE(ThePlatformBaseIE):
             'title': 'Blackberry\'s big, bold Z30',
             'description': 'The Z30 is Blackberry\'s biggest, baddest mobile messaging device yet.',
             'duration': 247,
+            'timestamp': 1383239700,
+            'upload_date': '20131031',
+            'uploader': 'CBSI-NEW',
         },
         'params': {
             # rtmp download
@@ -107,6 +112,9 @@ class ThePlatformIE(ThePlatformBaseIE):
             'ext': 'flv',
             'description': 'md5:ac330c9258c04f9d7512cf26b9595409',
             'title': 'Tesla Model S: A second step towards a cleaner motoring future',
+            'timestamp': 1426176191,
+            'upload_date': '20150312',
+            'uploader': 'CBSI-NEW',
         },
         'params': {
             # rtmp download
@@ -119,6 +127,7 @@ class ThePlatformIE(ThePlatformBaseIE):
             'ext': 'mp4',
             'description': 'md5:644ad9188d655b742f942bf2e06b002d',
             'title': 'HIGHLIGHTS: USA bag first ever series Cup win',
+            'uploader': 'EGSM',
         }
     }, {
         'url': 'http://player.theplatform.com/p/NnzsPC/widget/select/media/4Y0TlYUr_ZT7',
@@ -135,6 +144,7 @@ class ThePlatformIE(ThePlatformBaseIE):
             'thumbnail': 're:^https?://.*\.jpg$',
             'timestamp': 1435752600,
             'upload_date': '20150701',
+            'uploader': 'NBCU-NEWS',
         },
     }, {
         # From http://www.nbc.com/the-blacklist/video/sir-crispin-crandall/2928790?onid=137781#vc137781=1

From 04819db58edfa7a169e7ba6fd2d5734500754571 Mon Sep 17 00:00:00 2001
From: theGeekPirate <thegeekpirate@gmail.com>
Date: Sat, 26 Mar 2016 05:37:40 -0700
Subject: [PATCH 4511/6219] [camwithher] Add extractor

Corrected unnecessary test

Sane variable naming

RTMP all .flv & url_id for _download_webpage()

Corrected all outstanding issues, next up is a squash!
---
 youtube_dl/extractor/__init__.py   |  1 +
 youtube_dl/extractor/camwithher.py | 55 ++++++++++++++++++++++++++++++
 2 files changed, 56 insertions(+)
 create mode 100644 youtube_dl/extractor/camwithher.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 19f802411..438e1cc63 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -95,6 +95,7 @@ from .camdemy import (
     CamdemyIE,
     CamdemyFolderIE
 )
+from .camwithher import CamWithHerIE
 from .canalplus import CanalplusIE
 from .canalc2 import Canalc2IE
 from .canvas import CanvasIE
diff --git a/youtube_dl/extractor/camwithher.py b/youtube_dl/extractor/camwithher.py
new file mode 100644
index 000000000..eb0a4ec56
--- /dev/null
+++ b/youtube_dl/extractor/camwithher.py
@@ -0,0 +1,55 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class CamWithHerIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?camwithher\.tv/view_video\.php\?.*viewkey=(?P<id>\w+)'
+
+    _TESTS = [
+        {
+            'url': 'http://camwithher.tv/view_video.php?viewkey=6e9a24e2c0e842e1f177&page=&viewtype=&category=',
+            'info_dict': {
+                'id': '5644',
+                'ext': 'flv',
+                'title': 'Periscope Tease',
+            },
+            'params': {
+                'skip_download': True,
+            }
+        },
+        {
+            'url': 'http://camwithher.tv/view_video.php?viewkey=6dfd8b7c97531a459937',
+            'only_matching': True,
+        },
+        {
+            'url': 'http://camwithher.tv/view_video.php?page=&viewkey=6e9a24e2c0e842e1f177&viewtype=&category=',
+            'only_matching': True,
+        },
+        {
+            'url': 'http://camwithher.tv/view_video.php?viewkey=b6c3b5bea9515d1a1fc4&page=&viewtype=&category=mv',
+            'only_matching': True,
+        }
+    ]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        flv_id = self._html_search_regex(r'<a href="/download/\?v=(\d+)', webpage, 'id')
+
+        # The number "2010" was reverse-engineered from cwhplayer.swf.
+        # It appears that they changed their video codec, and hence the RTMP URL
+        # scheme at that video's ID.
+        rtmp_url = 'rtmp://camwithher.tv/clipshare/%s' % (('mp4:%s.mp4' % flv_id) if int(flv_id) > 2010 else flv_id)
+
+        title = self._html_search_regex(r'<div style="float:left">\s+<h2>(.+?)</h2>', webpage, 'title')
+
+        return {
+            'id': flv_id,
+            'url': rtmp_url,
+            'no_resume': True,
+            'ext': 'flv',
+            'title': title,
+        }

From 9aaaf8e8e8ae12ed8fbc62461558a4cdb8640ad5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 1 Apr 2016 23:47:27 +0600
Subject: [PATCH 4512/6219] [camwithher] Improve extraction (Closes #8989)

---
 youtube_dl/extractor/camwithher.py | 95 ++++++++++++++++++++----------
 1 file changed, 64 insertions(+), 31 deletions(-)

diff --git a/youtube_dl/extractor/camwithher.py b/youtube_dl/extractor/camwithher.py
index eb0a4ec56..9809096ec 100644
--- a/youtube_dl/extractor/camwithher.py
+++ b/youtube_dl/extractor/camwithher.py
@@ -1,55 +1,88 @@
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    parse_duration,
+    unified_strdate,
+)
 
 
 class CamWithHerIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?camwithher\.tv/view_video\.php\?.*viewkey=(?P<id>\w+)'
+    _VALID_URL = r'https?://(?:www\.)?camwithher\.tv/view_video\.php\?.*\bviewkey=(?P<id>\w+)'
 
-    _TESTS = [
-        {
-            'url': 'http://camwithher.tv/view_video.php?viewkey=6e9a24e2c0e842e1f177&page=&viewtype=&category=',
-            'info_dict': {
-                'id': '5644',
-                'ext': 'flv',
-                'title': 'Periscope Tease',
-            },
-            'params': {
-                'skip_download': True,
-            }
+    _TESTS = [{
+        'url': 'http://camwithher.tv/view_video.php?viewkey=6e9a24e2c0e842e1f177&page=&viewtype=&category=',
+        'info_dict': {
+            'id': '5644',
+            'ext': 'flv',
+            'title': 'Periscope Tease',
+            'description': 'In the clouds teasing on periscope to my favorite song',
+            'duration': 240,
+            'view_count': int,
+            'comment_count': int,
+            'uploader': 'MileenaK',
+            'upload_date': '20160322',
         },
-        {
-            'url': 'http://camwithher.tv/view_video.php?viewkey=6dfd8b7c97531a459937',
-            'only_matching': True,
-        },
-        {
-            'url': 'http://camwithher.tv/view_video.php?page=&viewkey=6e9a24e2c0e842e1f177&viewtype=&category=',
-            'only_matching': True,
-        },
-        {
-            'url': 'http://camwithher.tv/view_video.php?viewkey=b6c3b5bea9515d1a1fc4&page=&viewtype=&category=mv',
-            'only_matching': True,
+        'params': {
+            'skip_download': True,
         }
-    ]
+    }, {
+        'url': 'http://camwithher.tv/view_video.php?viewkey=6dfd8b7c97531a459937',
+        'only_matching': True,
+    }, {
+        'url': 'http://camwithher.tv/view_video.php?page=&viewkey=6e9a24e2c0e842e1f177&viewtype=&category=',
+        'only_matching': True,
+    }, {
+        'url': 'http://camwithher.tv/view_video.php?viewkey=b6c3b5bea9515d1a1fc4&page=&viewtype=&category=mv',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id)
 
-        flv_id = self._html_search_regex(r'<a href="/download/\?v=(\d+)', webpage, 'id')
+        flv_id = self._html_search_regex(
+            r'<a[^>]+href=["\']/download/\?v=(\d+)', webpage, 'video id')
 
-        # The number "2010" was reverse-engineered from cwhplayer.swf.
-        # It appears that they changed their video codec, and hence the RTMP URL
-        # scheme at that video's ID.
-        rtmp_url = 'rtmp://camwithher.tv/clipshare/%s' % (('mp4:%s.mp4' % flv_id) if int(flv_id) > 2010 else flv_id)
+        # Video URL construction algorithm is reverse-engineered from cwhplayer.swf
+        rtmp_url = 'rtmp://camwithher.tv/clipshare/%s' % (
+            ('mp4:%s.mp4' % flv_id) if int(flv_id) > 2010 else flv_id)
+
+        title = self._html_search_regex(
+            r'<div[^>]+style="float:left"[^>]*>\s*<h2>(.+?)</h2>', webpage, 'title')
+        description = self._html_search_regex(
+            r'>Description:</span>(.+?)</div>', webpage, 'description', default=None)
+
+        runtime = self._search_regex(
+            r'Runtime\s*:\s*(.+?) \|', webpage, 'duration', default=None)
+        if runtime:
+            runtime = re.sub(r'[\s-]', '', runtime)
+        duration = parse_duration(runtime)
+        view_count = int_or_none(self._search_regex(
+            r'Views\s*:\s*(\d+)', webpage, 'view count', default=None))
+        comment_count = int_or_none(self._search_regex(
+            r'Comments\s*:\s*(\d+)', webpage, 'comment count', default=None))
+
+        uploader = self._search_regex(
+            r'Added by\s*:\s*<a[^>]+>([^<]+)</a>', webpage, 'uploader', default=None)
+        upload_date = unified_strdate(self._search_regex(
+            r'Added on\s*:\s*([\d-]+)', webpage, 'upload date', default=None))
 
-        title = self._html_search_regex(r'<div style="float:left">\s+<h2>(.+?)</h2>', webpage, 'title')
 
         return {
             'id': flv_id,
             'url': rtmp_url,
-            'no_resume': True,
             'ext': 'flv',
+            'no_resume': True,
             'title': title,
+            'description': description,
+            'duration': duration,
+            'view_count': view_count,
+            'comment_count': comment_count,
+            'uploader': uploader,
+            'upload_date': upload_date,
         }

From 329c1eae54bf71ae8602f79f71570eaf90ef7d2c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 1 Apr 2016 20:42:19 +0200
Subject: [PATCH 4513/6219] [aenetworks] Make pep8 happy

---
 youtube_dl/extractor/aenetworks.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/aenetworks.py b/youtube_dl/extractor/aenetworks.py
index 3fddaba54..1bbfe2641 100644
--- a/youtube_dl/extractor/aenetworks.py
+++ b/youtube_dl/extractor/aenetworks.py
@@ -75,8 +75,9 @@ class AENetworksIE(InfoExtractor):
         info = self._search_json_ld(webpage, video_id, fatal=False)
         info.update({
             '_type': 'url_transparent',
-            'url': smuggle_url(update_url_query(
-                video_url, query), {
+            'url': smuggle_url(
+                update_url_query(video_url, query),
+                {
                     'sig': {
                         'key': 'crazyjava',
                         'secret': 's3cr3t'},

From 0f28777f58b5c21226d8e02477834dbb08b170ba Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 1 Apr 2016 20:43:14 +0200
Subject: [PATCH 4514/6219] [cbsnews] Remove unused import

---
 youtube_dl/extractor/cbsnews.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/cbsnews.py b/youtube_dl/extractor/cbsnews.py
index b5e78a65d..79ddc20a0 100644
--- a/youtube_dl/extractor/cbsnews.py
+++ b/youtube_dl/extractor/cbsnews.py
@@ -5,7 +5,6 @@ from .common import InfoExtractor
 from .cbs import CBSBaseIE
 from ..utils import (
     parse_duration,
-    find_xpath_attr,
 )
 
 

From 6d628fafcadf6b9d2bc16c34c8cda8b53860e406 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 1 Apr 2016 20:45:21 +0200
Subject: [PATCH 4515/6219] [camwithher] Remove extra blank line

---
 youtube_dl/extractor/camwithher.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/camwithher.py b/youtube_dl/extractor/camwithher.py
index 9809096ec..afbc5ea26 100644
--- a/youtube_dl/extractor/camwithher.py
+++ b/youtube_dl/extractor/camwithher.py
@@ -72,7 +72,6 @@ class CamWithHerIE(InfoExtractor):
         upload_date = unified_strdate(self._search_regex(
             r'Added on\s*:\s*([\d-]+)', webpage, 'upload date', default=None))
 
-
         return {
             'id': flv_id,
             'url': rtmp_url,

From df634be2ed85b33968973a3e85935bb5d578ce42 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 1 Apr 2016 19:39:02 +0100
Subject: [PATCH 4516/6219] [common] prefer using mime type over ext for smil
 subtitle extraction

the subtitle ext for http://www.cnet.com/videos/download-amazon-prime-movies-and-tv/
is adb_xml while using the mime type it get tt(application/smptett+xml)
---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 85ac0400c..94a583891 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1335,7 +1335,7 @@ class InfoExtractor(object):
             if not src or src in urls:
                 continue
             urls.append(src)
-            ext = textstream.get('ext') or determine_ext(src) or mimetype2ext(textstream.get('type'))
+            ext = textstream.get('ext') or mimetype2ext(textstream.get('type')) or determine_ext(src)
             lang = textstream.get('systemLanguage') or textstream.get('systemLanguageName') or textstream.get('lang') or subtitles_lang
             subtitles.setdefault(lang, []).append({
                 'url': src,

From 0750b2491f5f14e51c2bf91584fd490944154393 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 1 Apr 2016 19:47:20 +0100
Subject: [PATCH 4517/6219] [ffmpeg] try to convert tt subtitles usng dfxp2srt

---
 youtube_dl/postprocessor/ffmpeg.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 06b8c0548..b64cd396b 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -536,7 +536,7 @@ class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor):
             sub_filenames.append(old_file)
             new_file = subtitles_filename(filename, lang, new_ext)
 
-            if ext == 'dfxp' or ext == 'ttml':
+            if ext == 'dfxp' or ext == 'ttml' or ext == 'tt':
                 self._downloader.report_warning(
                     'You have requested to convert dfxp (TTML) subtitles into another format, '
                     'which results in style information loss')

From 5f705baf5ecda6be678481ff9ab9c27a6cd54dc0 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 1 Apr 2016 20:42:15 +0100
Subject: [PATCH 4518/6219] [cnet] extract more formats

---
 youtube_dl/extractor/cnet.py | 25 +++++++++++++------------
 1 file changed, 13 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/cnet.py b/youtube_dl/extractor/cnet.py
index c154b3e19..58c26f20f 100644
--- a/youtube_dl/extractor/cnet.py
+++ b/youtube_dl/extractor/cnet.py
@@ -17,6 +17,8 @@ class CNETIE(ThePlatformIE):
             'uploader_id': '6085384d-619e-11e3-b231-14feb5ca9861',
             'uploader': 'Sarah Mitroff',
             'duration': 70,
+            'timestamp': 1396479627,
+            'upload_date': '20140402',
         },
     }, {
         'url': 'http://www.cnet.com/videos/whiny-pothole-tweets-at-local-government-when-hit-by-cars-tomorrow-daily-187/',
@@ -28,8 +30,11 @@ class CNETIE(ThePlatformIE):
             'uploader_id': 'b163284d-6b73-44fc-b3e6-3da66c392d40',
             'uploader': 'Ashley Esqueda',
             'duration': 1482,
+            'timestamp': 1433289889,
+            'upload_date': '20150603',
         },
     }]
+    TP_RELEASE_URL_TEMPLATE = 'http://link.theplatform.com/s/kYEXFC/%s?mbr=true'
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
@@ -51,16 +56,12 @@ class CNETIE(ThePlatformIE):
             uploader = None
             uploader_id = None
 
-        metadata = self.get_metadata('kYEXFC/%s' % list(vdata['files'].values())[0], video_id)
-        description = vdata.get('description') or metadata.get('description')
-        duration = int_or_none(vdata.get('duration')) or metadata.get('duration')
-
-        formats = []
-        subtitles = {}
+        media_guid_path = 'media/guid/2288573011/%s' % vdata['mpxRefId']
+        formats, subtitles = self._extract_theplatform_smil(self.TP_RELEASE_URL_TEMPLATE % media_guid_path, video_id)
         for (fkey, vid) in vdata['files'].items():
             if fkey == 'hls_phone' and 'hls_tablet' in vdata['files']:
                 continue
-            release_url = 'http://link.theplatform.com/s/kYEXFC/%s?mbr=true' % vid
+            release_url = self.TP_RELEASE_URL_TEMPLATE % vid
             if fkey == 'hds':
                 release_url += '&manifest=f4m'
             tp_formats, tp_subtitles = self._extract_theplatform_smil(release_url, video_id, 'Downloading %s SMIL data' % fkey)
@@ -68,15 +69,15 @@ class CNETIE(ThePlatformIE):
             subtitles = self._merge_subtitles(subtitles, tp_subtitles)
         self._sort_formats(formats)
 
-        return {
+        info = self.get_metadata('kYEXFC/%s' % media_guid_path, video_id)
+        info.update({
             'id': video_id,
             'display_id': display_id,
             'title': title,
-            'description': description,
-            'thumbnail': metadata.get('thumbnail'),
-            'duration': duration,
+            'duration': int_or_none(vdata.get('duration')),
             'uploader': uploader,
             'uploader_id': uploader_id,
             'subtitles': subtitles,
             'formats': formats,
-        }
+        })
+        return info

From fe7ef95e91cec1c1794692029561a68e2aaa7809 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 1 Apr 2016 23:53:32 +0100
Subject: [PATCH 4519/6219] [cbsinteractive] Add support for ZDNet videos

---
 youtube_dl/extractor/__init__.py              |  2 +-
 .../extractor/{cnet.py => cbsinteractive.py}  | 37 ++++++++++++++++---
 2 files changed, 32 insertions(+), 7 deletions(-)
 rename youtube_dl/extractor/{cnet.py => cbsinteractive.py} (69%)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 438e1cc63..ebf5ca3e9 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -104,6 +104,7 @@ from .cbc import (
     CBCPlayerIE,
 )
 from .cbs import CBSIE
+from .cbsinteractive import CBSInteractiveIE
 from .cbsnews import (
     CBSNewsIE,
     CBSNewsLiveVideoIE,
@@ -129,7 +130,6 @@ from .clubic import ClubicIE
 from .clyp import ClypIE
 from .cmt import CMTIE
 from .cnbc import CNBCIE
-from .cnet import CNETIE
 from .cnn import (
     CNNIE,
     CNNBlogsIE,
diff --git a/youtube_dl/extractor/cnet.py b/youtube_dl/extractor/cbsinteractive.py
similarity index 69%
rename from youtube_dl/extractor/cnet.py
rename to youtube_dl/extractor/cbsinteractive.py
index 58c26f20f..0011c3029 100644
--- a/youtube_dl/extractor/cnet.py
+++ b/youtube_dl/extractor/cbsinteractive.py
@@ -1,12 +1,14 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .theplatform import ThePlatformIE
 from ..utils import int_or_none
 
 
-class CNETIE(ThePlatformIE):
-    _VALID_URL = r'https?://(?:www\.)?cnet\.com/videos/(?P<id>[^/]+)/'
+class CBSInteractiveIE(ThePlatformIE):
+    _VALID_URL = r'https?://(?:www\.)?(?P<site>cnet|zdnet)\.com/(?:videos|video/share)/(?P<id>[^/?]+)'
     _TESTS = [{
         'url': 'http://www.cnet.com/videos/hands-on-with-microsofts-windows-8-1-update/',
         'info_dict': {
@@ -33,15 +35,35 @@ class CNETIE(ThePlatformIE):
             'timestamp': 1433289889,
             'upload_date': '20150603',
         },
+    }, {
+        'url': 'http://www.zdnet.com/video/share/video-keeping-android-smartphones-and-tablets-secure/',
+        'info_dict': {
+            'id': 'bc1af9f0-a2b5-4e54-880d-0d95525781c0',
+            'ext': 'mp4',
+            'title': 'Video: Keeping Android smartphones and tablets secure',
+            'description': 'Here\'s the best way to keep Android devices secure, and what you do when they\'ve come to the end of their lives.',
+            'uploader_id': 'f2d97ea2-8175-11e2-9d12-0018fe8a00b0',
+            'uploader': 'Adrian Kingsley-Hughes',
+            'timestamp': 1448961720,
+            'upload_date': '20151201',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        }
     }]
     TP_RELEASE_URL_TEMPLATE = 'http://link.theplatform.com/s/kYEXFC/%s?mbr=true'
+    MPX_ACCOUNTS = {
+        'cnet': 2288573011,
+        'zdnet': 2387448114,
+    }
 
     def _real_extract(self, url):
-        display_id = self._match_id(url)
+        site, display_id = re.match(self._VALID_URL, url).groups()
         webpage = self._download_webpage(url, display_id)
 
         data_json = self._html_search_regex(
-            r"data-cnet-video(?:-uvp)?-options='([^']+)'",
+            r"data-(?:cnet|zdnet)-video(?:-uvp)?-options='([^']+)'",
             webpage, 'data json')
         data = self._parse_json(data_json, display_id)
         vdata = data.get('video') or data['videos'][0]
@@ -56,8 +78,11 @@ class CNETIE(ThePlatformIE):
             uploader = None
             uploader_id = None
 
-        media_guid_path = 'media/guid/2288573011/%s' % vdata['mpxRefId']
-        formats, subtitles = self._extract_theplatform_smil(self.TP_RELEASE_URL_TEMPLATE % media_guid_path, video_id)
+        media_guid_path = 'media/guid/%d/%s' % (self.MPX_ACCOUNTS[site], vdata['mpxRefId'])
+        formats, subtitles = [], {}
+        if site == 'cnet':
+            formats, subtitles = self._extract_theplatform_smil(
+                self.TP_RELEASE_URL_TEMPLATE % media_guid_path, video_id)
         for (fkey, vid) in vdata['files'].items():
             if fkey == 'hls_phone' and 'hls_tablet' in vdata['files']:
                 continue

From 08136dc13805abb1832587e03e68066f07bd5776 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 2 Apr 2016 10:57:57 +0100
Subject: [PATCH 4520/6219] [brightcove] fix format sorting

---
 youtube_dl/extractor/brightcove.py | 3 ++-
 youtube_dl/extractor/thestar.py    | 4 ++++
 youtube_dl/extractor/tv3.py        | 1 +
 3 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 0d162d337..a8919001d 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -396,6 +396,7 @@ class BrightcoveNewIE(InfoExtractor):
             'formats': 'mincount:41',
         },
         'params': {
+            # m3u8 download
             'skip_download': True,
         }
     }, {
@@ -533,7 +534,7 @@ class BrightcoveNewIE(InfoExtractor):
                     f.update({
                         'url': src or streaming_src,
                         'format_id': build_format_id('http' if src else 'http-streaming'),
-                        'preference': 2 if src else 1,
+                        'source_preference': 0 if src else -1,
                     })
                 else:
                     f.update({
diff --git a/youtube_dl/extractor/thestar.py b/youtube_dl/extractor/thestar.py
index b7e9af2af..ba1380abc 100644
--- a/youtube_dl/extractor/thestar.py
+++ b/youtube_dl/extractor/thestar.py
@@ -19,6 +19,10 @@ class TheStarIE(InfoExtractor):
             'uploader_id': '794267642001',
             'timestamp': 1454353482,
             'upload_date': '20160201',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
         }
     }
     BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/794267642001/default_default/index.html?videoId=%s'
diff --git a/youtube_dl/extractor/tv3.py b/youtube_dl/extractor/tv3.py
index d3f690dc7..3867ec90d 100644
--- a/youtube_dl/extractor/tv3.py
+++ b/youtube_dl/extractor/tv3.py
@@ -21,6 +21,7 @@ class TV3IE(InfoExtractor):
             'Failed to download MPD manifest'
         ],
         'params': {
+            # m3u8 download
             'skip_download': True,
         },
     }

From db8ee7ec0598f8893e4259ac9373c44726e4f84f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 2 Apr 2016 18:48:05 +0600
Subject: [PATCH 4521/6219] [extractor/common] Fix numeric identifiers
 conversion in DASH URL templates

---
 youtube_dl/extractor/common.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 94a583891..011edcc0a 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1515,7 +1515,8 @@ class InfoExtractor(object):
                                 representation_ms_info['total_number'] = int(math.ceil(float(period_duration) / segment_duration))
                             media_template = representation_ms_info['media_template']
                             media_template = media_template.replace('$RepresentationID$', representation_id)
-                            media_template = re.sub(r'\$(Number|Bandwidth)(?:%(0\d+)d)?\$', r'%(\1)\2d', media_template)
+                            media_template = re.sub(r'\$(Number|Bandwidth)\$', r'%(\1)d', media_template)
+                            media_template = re.sub(r'\$(Number|Bandwidth)%(\d+)\$', r'%(\1)\2d', media_template)
                             media_template.replace('$$', '$')
                             representation_ms_info['segment_urls'] = [media_template % {'Number': segment_number, 'Bandwidth': representation_attrib.get('bandwidth')} for segment_number in range(representation_ms_info['start_number'], representation_ms_info['total_number'] + representation_ms_info['start_number'])]
                         if 'segment_urls' in representation_ms_info:

From b507cc925b8dbb37b0abce748ff73a7ad102494a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 2 Apr 2016 18:49:58 +0600
Subject: [PATCH 4522/6219] [extractor/common] Carry long line

---
 youtube_dl/extractor/common.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 011edcc0a..ec6625eea 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1518,7 +1518,13 @@ class InfoExtractor(object):
                             media_template = re.sub(r'\$(Number|Bandwidth)\$', r'%(\1)d', media_template)
                             media_template = re.sub(r'\$(Number|Bandwidth)%(\d+)\$', r'%(\1)\2d', media_template)
                             media_template.replace('$$', '$')
-                            representation_ms_info['segment_urls'] = [media_template % {'Number': segment_number, 'Bandwidth': representation_attrib.get('bandwidth')} for segment_number in range(representation_ms_info['start_number'], representation_ms_info['total_number'] + representation_ms_info['start_number'])]
+                            representation_ms_info['segment_urls'] = [
+                                media_template % {
+                                    'Number': segment_number,
+                                    'Bandwidth': representation_attrib.get('bandwidth')}
+                                for segment_number in range(
+                                    representation_ms_info['start_number'],
+                                    representation_ms_info['total_number'] + representation_ms_info['start_number'])]
                         if 'segment_urls' in representation_ms_info:
                             f.update({
                                 'segment_urls': representation_ms_info['segment_urls'],

From bbc26c8a012d215e98a98a671471cd75e7765286 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 2 Apr 2016 19:00:38 +0600
Subject: [PATCH 4523/6219] [bbc] Set vcodec to none for audio formats

---
 youtube_dl/extractor/bbc.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index dedf721bd..425f08f2b 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -328,6 +328,7 @@ class BBCCoUkIE(InfoExtractor):
                     'format_id': '%s_%s' % (service, format['format_id']),
                     'abr': abr,
                     'acodec': acodec,
+                    'vcodec': 'none',
                 })
             formats.extend(conn_formats)
         return formats

From 2defa7d75aa424c16ca76a25a05297daed0bb5bd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 2 Apr 2016 18:01:58 +0200
Subject: [PATCH 4524/6219] [instagram:user] Fix extraction (fixes #9059)

The URL for the next page was incorrect and we always got the same page, therefore it got trapped in an infinite loop.
---
 youtube_dl/extractor/instagram.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/instagram.py b/youtube_dl/extractor/instagram.py
index 4e62098b0..11bb58d8a 100644
--- a/youtube_dl/extractor/instagram.py
+++ b/youtube_dl/extractor/instagram.py
@@ -152,7 +152,7 @@ class InstagramUserIE(InfoExtractor):
 
             if not page['items']:
                 break
-            max_id = page['items'][-1]['id']
+            max_id = page['items'][-1]['id'].split('_')[0]
             media_url = (
                 'http://instagram.com/%s/media?max_id=%s' % (
                     uploader_id, max_id))

From 41f5492fbcddfcbae133dc27e8d94ece3755df2e Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 2 Apr 2016 18:54:40 +0100
Subject: [PATCH 4525/6219] [brightcove:legacy] improve format extraction and
 extract uploader_id, duration and timestamp

---
 youtube_dl/extractor/brightcove.py | 48 +++++++++++++++++++++++++-----
 1 file changed, 41 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index a8919001d..a5091238b 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -46,6 +46,9 @@ class BrightcoveLegacyIE(InfoExtractor):
                 'title': 'Xavier Sala i Martín: “Un banc que no presta és un banc zombi que no serveix per a res”',
                 'uploader': '8TV',
                 'description': 'md5:a950cc4285c43e44d763d036710cd9cd',
+                'timestamp': 1368213670,
+                'upload_date': '20130510',
+                'uploader_id': 1589608506001,
             }
         },
         {
@@ -57,6 +60,9 @@ class BrightcoveLegacyIE(InfoExtractor):
                 'title': 'JVMLS 2012: Arrays 2.0 - Opportunities and Challenges',
                 'description': 'John Rose speaks at the JVM Language Summit, August 1, 2012.',
                 'uploader': 'Oracle',
+                'timestamp': 1344975024,
+                'upload_date': '20120814',
+                'uploader_id': 1460825906,
             },
         },
         {
@@ -68,6 +74,9 @@ class BrightcoveLegacyIE(InfoExtractor):
                 'title': 'This Bracelet Acts as a Personal Thermostat',
                 'description': 'md5:547b78c64f4112766ccf4e151c20b6a0',
                 'uploader': 'Mashable',
+                'timestamp': 1382041798,
+                'upload_date': '20131017',
+                'uploader_id': 1130468786001,
             },
         },
         {
@@ -85,14 +94,17 @@ class BrightcoveLegacyIE(InfoExtractor):
         {
             # test flv videos served by akamaihd.net
             # From http://www.redbull.com/en/bike/stories/1331655643987/replay-uci-dh-world-cup-2014-from-fort-william
-            'url': 'http://c.brightcove.com/services/viewer/htmlFederated?%40videoPlayer=ref%3ABC2996102916001&linkBaseURL=http%3A%2F%2Fwww.redbull.com%2Fen%2Fbike%2Fvideos%2F1331655630249%2Freplay-uci-fort-william-2014-dh&playerKey=AQ%7E%7E%2CAAAApYJ7UqE%7E%2Cxqr_zXk0I-zzNndy8NlHogrCb5QdyZRf&playerID=1398061561001#__youtubedl_smuggle=%7B%22Referer%22%3A+%22http%3A%2F%2Fwww.redbull.com%2Fen%2Fbike%2Fstories%2F1331655643987%2Freplay-uci-dh-world-cup-2014-from-fort-william%22%7D',
+            'url': 'http://c.brightcove.com/services/viewer/htmlFederated?%40videoPlayer=ref%3Aevent-stream-356&linkBaseURL=http%3A%2F%2Fwww.redbull.com%2Fen%2Fbike%2Fvideos%2F1331655630249%2Freplay-uci-fort-william-2014-dh&playerKey=AQ%7E%7E%2CAAAApYJ7UqE%7E%2Cxqr_zXk0I-zzNndy8NlHogrCb5QdyZRf&playerID=1398061561001#__youtubedl_smuggle=%7B%22Referer%22%3A+%22http%3A%2F%2Fwww.redbull.com%2Fen%2Fbike%2Fstories%2F1331655643987%2Freplay-uci-dh-world-cup-2014-from-fort-william%22%7D',
             # The md5 checksum changes on each download
             'info_dict': {
-                'id': '2996102916001',
+                'id': '3750436379001',
                 'ext': 'flv',
                 'title': 'UCI MTB World Cup 2014: Fort William, UK - Downhill Finals',
-                'uploader': 'Red Bull TV',
+                'uploader': 'RBTV Old (do not use)',
                 'description': 'UCI MTB World Cup 2014: Fort William, UK - Downhill Finals',
+                'timestamp': 1409122195,
+                'upload_date': '20140827',
+                'uploader_id': 710858724001,
             },
         },
         {
@@ -106,6 +118,12 @@ class BrightcoveLegacyIE(InfoExtractor):
             'playlist_mincount': 7,
         },
     ]
+    FLV_VCODECS = {
+        1: 'SORENSON',
+        2: 'ON2',
+        3: 'H264',
+        4: 'VP8',
+    }
 
     @classmethod
     def _build_brighcove_url(cls, object_str):
@@ -295,6 +313,9 @@ class BrightcoveLegacyIE(InfoExtractor):
             'description': video_info.get('shortDescription'),
             'thumbnail': video_info.get('videoStillURL') or video_info.get('thumbnailURL'),
             'uploader': video_info.get('publisherName'),
+            'uploader_id': video_info.get('publisherId'),
+            'duration': float_or_none(video_info.get('length'), 1000),
+            'timestamp': int_or_none(video_info.get('creationDate'), 1000),
         }
 
         renditions = video_info.get('renditions', []) + video_info.get('IOSRenditions', [])
@@ -318,19 +339,30 @@ class BrightcoveLegacyIE(InfoExtractor):
                         ext = 'flv'
                 if ext is None:
                     ext = determine_ext(url)
-                size = rend.get('size')
+                tbr = int_or_none(rend.get('encodingRate'), 1000),
                 a_format = {
+                    'format_id': 'http%s' % ('-%s' % tbr if tbr else ''),
                     'url': url,
                     'ext': ext,
-                    'height': rend.get('frameHeight'),
-                    'width': rend.get('frameWidth'),
-                    'filesize': size if size != 0 else None,
+                    'filesize': int_or_none(rend.get('size')) or None,
+                    'tbr': tbr,
                 }
+                if rend.get('audioOnly'):
+                    a_format.update({
+                        'vcodec': 'none',
+                    })
+                else:
+                    a_format.update({
+                        'height': int_or_none(rend.get('frameHeight')),
+                        'width': int_or_none(rend.get('frameWidth')),
+                        'vcodec': rend.get('videoCodec'),
+                    })
 
                 # m3u8 manifests with remote == false are media playlists
                 # Not calling _extract_m3u8_formats here to save network traffic
                 if ext == 'm3u8':
                     a_format.update({
+                        'format_id': 'hls%s' % ('-%s' % tbr if tbr else ''),
                         'ext': 'mp4',
                         'protocol': 'm3u8',
                     })
@@ -341,6 +373,8 @@ class BrightcoveLegacyIE(InfoExtractor):
         elif video_info.get('FLVFullLengthURL') is not None:
             info.update({
                 'url': video_info['FLVFullLengthURL'],
+                'vcodec': self.FLV_VCODECS.get(video_info.get('FLVFullCodec')),
+                'filesize': int_or_none(video_info.get('FLVFullSize')),
             })
 
         if self._downloader.params.get('include_ads', False):

From e47d19e991456fe4afdab1a76a653f7821e99c3f Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 2 Apr 2016 18:56:01 +0100
Subject: [PATCH 4526/6219] [brightcove:new] extract subtitles and strip video
 title

---
 youtube_dl/extractor/brightcove.py | 24 +++++++++++++-----------
 1 file changed, 13 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index a5091238b..6128b6762 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -515,7 +515,7 @@ class BrightcoveNewIE(InfoExtractor):
                 raise ExtractorError(json_data[0]['message'], expected=True)
             raise
 
-        title = json_data['name']
+        title = json_data['name'].strip()
 
         formats = []
         for source in json_data.get('sources', []):
@@ -579,20 +579,22 @@ class BrightcoveNewIE(InfoExtractor):
                 formats.append(f)
         self._sort_formats(formats)
 
-        description = json_data.get('description')
-        thumbnail = json_data.get('thumbnail')
-        timestamp = parse_iso8601(json_data.get('published_at'))
-        duration = float_or_none(json_data.get('duration'), 1000)
-        tags = json_data.get('tags', [])
+        subtitles = {}
+        for text_track in json_data.get('text_tracks', []):
+            if text_track.get('src'):
+                subtitles.setdefault(text_track.get('srclang'), []).append({
+                    'url': text_track['src'],
+                })
 
         return {
             'id': video_id,
             'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
-            'duration': duration,
-            'timestamp': timestamp,
+            'description': json_data.get('description'),
+            'thumbnail': json_data.get('thumbnail') or json_data.get('poster'),
+            'duration': float_or_none(json_data.get('duration'), 1000),
+            'timestamp': parse_iso8601(json_data.get('published_at')),
             'uploader_id': account_id,
             'formats': formats,
-            'tags': tags,
+            'subtitles': subtitles,
+            'tags': json_data.get('tags', []),
         }

From 3aac9b2fb1a103f1d350ba10060e59bb04a6a2e8 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 2 Apr 2016 18:56:31 +0100
Subject: [PATCH 4527/6219] [nowness] update tests

---
 youtube_dl/extractor/nowness.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/nowness.py b/youtube_dl/extractor/nowness.py
index 446f5901c..74860eb20 100644
--- a/youtube_dl/extractor/nowness.py
+++ b/youtube_dl/extractor/nowness.py
@@ -63,8 +63,11 @@ class NownessIE(NownessBaseIE):
             'title': 'Candor: The Art of Gesticulation',
             'description': 'Candor: The Art of Gesticulation',
             'thumbnail': 're:^https?://.*\.jpg',
-            'uploader': 'Nowness',
+            'timestamp': 1446745676,
+            'upload_date': '20151105',
+            'uploader_id': '2385340575001',
         },
+        'add_ie': ['BrightcoveNew'],
     }, {
         'url': 'https://cn.nowness.com/story/kasper-bjorke-ft-jaakko-eino-kalevi-tnr',
         'md5': 'e79cf125e387216f86b2e0a5b5c63aa3',
@@ -74,8 +77,11 @@ class NownessIE(NownessBaseIE):
             'title': 'Kasper Bjørke ft. Jaakko Eino Kalevi: TNR',
             'description': 'Kasper Bjørke ft. Jaakko Eino Kalevi: TNR',
             'thumbnail': 're:^https?://.*\.jpg',
-            'uploader': 'Nowness',
+            'timestamp': 1407315371,
+            'upload_date': '20140806',
+            'uploader_id': '2385340575001',
         },
+        'add_ie': ['BrightcoveNew'],
     }, {
         # vimeo
         'url': 'https://www.nowness.com/series/nowness-picks/jean-luc-godard-supercut',
@@ -90,6 +96,7 @@ class NownessIE(NownessBaseIE):
             'uploader': 'Cinema Sem Lei',
             'uploader_id': 'cinemasemlei',
         },
+        'add_ie': ['Vimeo'],
     }]
 
     def _real_extract(self, url):

From 4d4cd35f485c652a39a631fbf3d40c9f4353e807 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 2 Apr 2016 20:55:44 +0100
Subject: [PATCH 4528/6219] [brightcove:legacy] extract uploader_id as a string

---
 youtube_dl/extractor/brightcove.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 6128b6762..f9056f514 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -48,7 +48,7 @@ class BrightcoveLegacyIE(InfoExtractor):
                 'description': 'md5:a950cc4285c43e44d763d036710cd9cd',
                 'timestamp': 1368213670,
                 'upload_date': '20130510',
-                'uploader_id': 1589608506001,
+                'uploader_id': '1589608506001',
             }
         },
         {
@@ -62,7 +62,7 @@ class BrightcoveLegacyIE(InfoExtractor):
                 'uploader': 'Oracle',
                 'timestamp': 1344975024,
                 'upload_date': '20120814',
-                'uploader_id': 1460825906,
+                'uploader_id': '1460825906',
             },
         },
         {
@@ -76,7 +76,7 @@ class BrightcoveLegacyIE(InfoExtractor):
                 'uploader': 'Mashable',
                 'timestamp': 1382041798,
                 'upload_date': '20131017',
-                'uploader_id': 1130468786001,
+                'uploader_id': '1130468786001',
             },
         },
         {
@@ -104,7 +104,7 @@ class BrightcoveLegacyIE(InfoExtractor):
                 'description': 'UCI MTB World Cup 2014: Fort William, UK - Downhill Finals',
                 'timestamp': 1409122195,
                 'upload_date': '20140827',
-                'uploader_id': 710858724001,
+                'uploader_id': '710858724001',
             },
         },
         {
@@ -313,7 +313,7 @@ class BrightcoveLegacyIE(InfoExtractor):
             'description': video_info.get('shortDescription'),
             'thumbnail': video_info.get('videoStillURL') or video_info.get('thumbnailURL'),
             'uploader': video_info.get('publisherName'),
-            'uploader_id': video_info.get('publisherId'),
+            'uploader_id': compat_str(video_info.get('publisherId')),
             'duration': float_or_none(video_info.get('length'), 1000),
             'timestamp': int_or_none(video_info.get('creationDate'), 1000),
         }

From 23576edbfcaa3d7f0283631516c82715a85c6856 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 2 Apr 2016 21:31:21 +0100
Subject: [PATCH 4529/6219] [brightcove:legacy] skip None value for uploader_id

---
 youtube_dl/extractor/brightcove.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index f9056f514..c718cf385 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -307,13 +307,14 @@ class BrightcoveLegacyIE(InfoExtractor):
                                     playlist_title=playlist_info['mediaCollectionDTO']['displayName'])
 
     def _extract_video_info(self, video_info):
+        publisher_id = video_info.get('publisherId')
         info = {
             'id': compat_str(video_info['id']),
             'title': video_info['displayName'].strip(),
             'description': video_info.get('shortDescription'),
             'thumbnail': video_info.get('videoStillURL') or video_info.get('thumbnailURL'),
             'uploader': video_info.get('publisherName'),
-            'uploader_id': compat_str(video_info.get('publisherId')),
+            'uploader_id': compat_str(publisher_id) if publisher_id else None,
             'duration': float_or_none(video_info.get('length'), 1000),
             'timestamp': int_or_none(video_info.get('creationDate'), 1000),
         }

From 6d4fc66bfc9bb3ed2a4f68366f372a9bedf6e708 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 4 Apr 2016 02:26:20 +0600
Subject: [PATCH 4530/6219] [youtube] Add support for zwearz (Closes #9062)

---
 youtube_dl/extractor/youtube.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 28355bf46..188066561 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -270,7 +270,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                          ))
                          |(?:
                             youtu\.be|                                        # just youtu.be/xxxx
-                            vid\.plus                                         # or vid.plus/xxxx
+                            vid\.plus|                                        # or vid.plus/xxxx
+                            zwearz\.com/watch|                                # or zwearz.com/watch/xxxx
                          )/
                          |(?:www\.)?cleanvideosearch\.com/media/action/yt/watch\?videoId=
                          )
@@ -758,6 +759,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             'url': 'http://vid.plus/FlRa-iH7PGw',
             'only_matching': True,
         },
+        {
+            'url': 'http://zwearz.com/watch/9lWxNJF-ufM/electra-woman-dyna-girl-official-trailer-grace-helbig.html',
+            'only_matching': True,
+        },
         {
             # Title with JS-like syntax "};" (see https://github.com/rg3/youtube-dl/issues/7468)
             # Also tests cut-off URL expansion in video description (see

From 8c7d6e8e2279beccf638cd0fae9d91876e0486b2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 4 Apr 2016 20:44:06 +0200
Subject: [PATCH 4531/6219] [zdf] Extract subtitles (closes #9081)

---
 youtube_dl/extractor/zdf.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/youtube_dl/extractor/zdf.py b/youtube_dl/extractor/zdf.py
index 81c22a627..2ef177275 100644
--- a/youtube_dl/extractor/zdf.py
+++ b/youtube_dl/extractor/zdf.py
@@ -85,6 +85,13 @@ class ZDFIE(InfoExtractor):
         uploader = xpath_text(doc, './/details/originChannelTitle', 'uploader')
         uploader_id = xpath_text(doc, './/details/originChannelId', 'uploader id')
         upload_date = unified_strdate(xpath_text(doc, './/details/airtime', 'upload date'))
+        subtitles = {}
+        captions_url = doc.find('.//caption/url')
+        if captions_url is not None:
+            subtitles['de'] = [{
+                'url': captions_url.text,
+                'ext': 'ttml',
+            }]
 
         def xml_to_thumbnails(fnode):
             thumbnails = []
@@ -190,6 +197,7 @@ class ZDFIE(InfoExtractor):
             'uploader_id': uploader_id,
             'upload_date': upload_date,
             'formats': formats,
+            'subtitles': subtitles,
         }
 
     def _real_extract(self, url):

From 5bf28d7864d83be98233b6d1e478d7911f99e2cb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 4 Apr 2016 20:46:35 +0200
Subject: [PATCH 4532/6219] [utils] dfxp2srt: add additional namespace

Used by the ZDF subtitles (#9081).
---
 youtube_dl/utils.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 5c4ab2748..8e53962c9 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2131,6 +2131,7 @@ def dfxp2srt(dfxp_data):
     _x = functools.partial(xpath_with_ns, ns_map={
         'ttml': 'http://www.w3.org/ns/ttml',
         'ttaf1': 'http://www.w3.org/2006/10/ttaf1',
+        'ttaf1_0604': 'http://www.w3.org/2006/04/ttaf1',
     })
 
     class TTMLPElementParser(object):
@@ -2157,7 +2158,7 @@ def dfxp2srt(dfxp_data):
 
     dfxp = compat_etree_fromstring(dfxp_data.encode('utf-8'))
     out = []
-    paras = dfxp.findall(_x('.//ttml:p')) or dfxp.findall(_x('.//ttaf1:p')) or dfxp.findall('.//p')
+    paras = dfxp.findall(_x('.//ttml:p')) or dfxp.findall(_x('.//ttaf1:p')) or dfxp.findall(_x('.//ttaf1_0604:p')) or dfxp.findall('.//p')
 
     if not paras:
         raise ValueError('Invalid dfxp/TTML subtitle')

From 995cf05c96e888f7f1a818f9886345ea25671aa4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 5 Apr 2016 21:40:43 +0600
Subject: [PATCH 4533/6219] [novamov] Make title fatal

---
 youtube_dl/extractor/novamov.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/novamov.py b/youtube_dl/extractor/novamov.py
index a131f7dbd..63a77b9bf 100644
--- a/youtube_dl/extractor/novamov.py
+++ b/youtube_dl/extractor/novamov.py
@@ -81,7 +81,7 @@ class NovaMovIE(InfoExtractor):
 
         filekey = extract_filekey()
 
-        title = self._html_search_regex(self._TITLE_REGEX, webpage, 'title', fatal=False)
+        title = self._html_search_regex(self._TITLE_REGEX, webpage, 'title')
         description = self._html_search_regex(self._DESCRIPTION_REGEX, webpage, 'description', default='', fatal=False)
 
         api_response = self._download_webpage(

From e7d77efb9dddc145b3d431047f2f98e19df4114e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 5 Apr 2016 21:52:07 +0600
Subject: [PATCH 4534/6219] [auroravid] Add extractor (Closes #9070)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/novamov.py  | 35 ++++++++++++++++++++++----------
 2 files changed, 25 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index ebf5ca3e9..69ea21a23 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -501,6 +501,7 @@ from .novamov import (
     NowVideoIE,
     VideoWeedIE,
     CloudTimeIE,
+    AuroraVidIE,
 )
 from .nowness import (
     NownessIE,
diff --git a/youtube_dl/extractor/novamov.py b/youtube_dl/extractor/novamov.py
index 63a77b9bf..5771a675d 100644
--- a/youtube_dl/extractor/novamov.py
+++ b/youtube_dl/extractor/novamov.py
@@ -27,17 +27,7 @@ class NovaMovIE(InfoExtractor):
     _DESCRIPTION_REGEX = r'(?s)<div class="v_tab blockborder rounded5" id="v_tab1">\s*<h3>[^<]+</h3><p>([^<]+)</p>'
     _URL_TEMPLATE = 'http://%s/video/%s'
 
-    _TEST = {
-        'url': 'http://www.novamov.com/video/4rurhn9x446jj',
-        'md5': '7205f346a52bbeba427603ba10d4b935',
-        'info_dict': {
-            'id': '4rurhn9x446jj',
-            'ext': 'flv',
-            'title': 'search engine optimization',
-            'description': 'search engine optimization is used to rank the web page in the google search engine'
-        },
-        'skip': '"Invalid token" errors abound (in web interface as well as youtube-dl, there is nothing we can do about it.)'
-    }
+    _TEST = None
 
     def _check_existence(self, webpage, video_id):
         if re.search(self._FILE_DELETED_REGEX, webpage) is not None:
@@ -187,3 +177,26 @@ class CloudTimeIE(NovaMovIE):
     _TITLE_REGEX = r'<div[^>]+class=["\']video_det["\'][^>]*>\s*<strong>([^<]+)</strong>'
 
     _TEST = None
+
+
+class AuroraVidIE(NovaMovIE):
+    IE_NAME = 'auroravid'
+    IE_DESC = 'AuroraVid'
+
+    _VALID_URL = NovaMovIE._VALID_URL_TEMPLATE % {'host': 'auroravid\.to'}
+
+    _HOST = 'www.auroravid.to'
+
+    _FILE_DELETED_REGEX = r'This file no longer exists on our servers!<'
+
+    _TEST = {
+        'url': 'http://www.auroravid.to/video/4rurhn9x446jj',
+        'md5': '7205f346a52bbeba427603ba10d4b935',
+        'info_dict': {
+            'id': '4rurhn9x446jj',
+            'ext': 'flv',
+            'title': 'search engine optimization',
+            'description': 'search engine optimization is used to rank the web page in the google search engine'
+        },
+        'skip': '"Invalid token" errors abound (in web interface as well as youtube-dl, there is nothing we can do about it.)'
+    }

From 40a056d85d7711e2f93bd5f7cc057672650386b8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 5 Apr 2016 21:54:09 +0600
Subject: [PATCH 4535/6219] [extractor/__init__] Remove novamov extractor and
 sort novamov based extractors alphabetically

---
 youtube_dl/extractor/__init__.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 69ea21a23..c3121d83c 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -496,12 +496,11 @@ from .normalboots import NormalbootsIE
 from .nosvideo import NosVideoIE
 from .nova import NovaIE
 from .novamov import (
-    NovaMovIE,
-    WholeCloudIE,
+    AuroraVidIE,
+    CloudTimeIE,
     NowVideoIE,
     VideoWeedIE,
-    CloudTimeIE,
-    AuroraVidIE,
+    WholeCloudIE,
 )
 from .nowness import (
     NownessIE,

From e42f413716de822918356d15b0cb3c66608681b5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 5 Apr 2016 22:23:20 +0600
Subject: [PATCH 4536/6219] [rte] Improve thumbnail extraction (Closes #9085)

---
 youtube_dl/extractor/rte.py | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/rte.py b/youtube_dl/extractor/rte.py
index 9c89974e7..ebe563ebb 100644
--- a/youtube_dl/extractor/rte.py
+++ b/youtube_dl/extractor/rte.py
@@ -39,9 +39,14 @@ class RteIE(InfoExtractor):
         duration = float_or_none(self._html_search_meta(
             'duration', webpage, 'duration', fatal=False), 1000)
 
-        thumbnail_id = self._search_regex(
-            r'<meta name="thumbnail" content="uri:irus:(.*?)" />', webpage, 'thumbnail')
-        thumbnail = 'http://img.rasset.ie/' + thumbnail_id + '.jpg'
+        thumbnail = None
+        thumbnail_meta = self._html_search_meta('thumbnail', webpage)
+        if thumbnail_meta:
+            thumbnail_id = self._search_regex(
+                r'uri:irus:(.+)', thumbnail_meta,
+                'thumbnail id', fatal=False)
+            if thumbnail_id:
+                thumbnail = 'http://img.rasset.ie/%s.jpg' % thumbnail_id
 
         feeds_url = self._html_search_meta('feeds-prefix', webpage, 'feeds url') + video_id
         json_string = self._download_json(feeds_url, video_id)

From 65150b41bb3055820938c3c572ccb2ffd9f312aa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 5 Apr 2016 22:27:33 +0600
Subject: [PATCH 4537/6219] [deezer] Fix extraction (Closes #9086)

---
 youtube_dl/extractor/deezer.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/deezer.py b/youtube_dl/extractor/deezer.py
index c3205ff5f..7a07f3267 100644
--- a/youtube_dl/extractor/deezer.py
+++ b/youtube_dl/extractor/deezer.py
@@ -41,7 +41,9 @@ class DeezerPlaylistIE(InfoExtractor):
                 'Deezer said: %s' % geoblocking_msg, expected=True)
 
         data_json = self._search_regex(
-            r'naboo\.display\(\'[^\']+\',\s*(.*?)\);\n', webpage, 'data JSON')
+            (r'__DZR_APP_STATE__\s*=\s*({.+?})\s*</script>',
+             r'naboo\.display\(\'[^\']+\',\s*(.*?)\);\n'),
+            webpage, 'data JSON')
         data = json.loads(data_json)
 
         playlist_title = data.get('DATA', {}).get('TITLE')

From 416930d45007cb1e24e4cd8638d9867dd84f3961 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 5 Apr 2016 18:36:24 +0200
Subject: [PATCH 4538/6219] release 2016.04.05

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 docs/supportedsites.md    | 5 +++--
 youtube_dl/version.py     | 2 +-
 3 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 128ba2fc0..e0a7d8282 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.04.01*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.04.01**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.04.05*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.04.05**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.04.01
+[debug] youtube-dl version 2016.04.05
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index b9bcf450c..d6ee8476b 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -57,6 +57,7 @@
  - **AudioBoom**
  - **audiomack**
  - **audiomack:album**
+ - **auroravid**: AuroraVid
  - **Azubu**
  - **AzubuLive**
  - **BaiduVideo**: 百度视频
@@ -92,12 +93,14 @@
  - **BYUtv**
  - **Camdemy**
  - **CamdemyFolder**
+ - **CamWithHer**
  - **canalc2.tv**
  - **Canalplus**: canalplus.fr, piwiplus.fr and d8.tv
  - **Canvas**
  - **CBC**
  - **CBCPlayer**
  - **CBS**
+ - **CBSInteractive**
  - **CBSNews**: CBS News
  - **CBSNewsLiveVideo**: CBS News Live Videos
  - **CBSSports**
@@ -119,7 +122,6 @@
  - **Clyp**
  - **cmt.com**
  - **CNBC**
- - **CNET**
  - **CNN**
  - **CNNArticle**
  - **CNNBlogs**
@@ -419,7 +421,6 @@
  - **Normalboots**
  - **NosVideo**
  - **Nova**: TN.cz, Prásk.tv, Nova.cz, Novaplus.cz, FANDA.tv, Krásná.cz and Doma.cz
- - **novamov**: NovaMov
  - **nowness**
  - **nowness:playlist**
  - **nowness:series**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index d9e1cb2a8..45d152902 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.04.01'
+__version__ = '2016.04.05'

From 8790249c6860374b4d7eeeffae9e7b30d346eaf7 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 6 Apr 2016 16:12:16 +0800
Subject: [PATCH 4539/6219] [iqiyi] Improve error detection for VIP-only videos

Closes #9071
---
 youtube_dl/extractor/iqiyi.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index 9e8c9432a..88570f261 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -368,7 +368,10 @@ class IqiyiIE(InfoExtractor):
             auth_req, video_id,
             note='Downloading video authentication JSON',
             errnote='Unable to download video authentication JSON')
-        if auth_result['code'] == 'Q00506':  # requires a VIP account
+
+        if auth_result['code'] == 'Q00505':  # No preview available (不允许试看鉴权失败)
+            raise ExtractorError('This video requires a VIP account', expected=True)
+        if auth_result['code'] == 'Q00506':  # End of preview time (试看结束鉴权失败)
             if do_report_warning:
                 self.report_warning('Needs a VIP account for full video')
             return False

From 92d5477d84c6663e71d6d22e261753a16b78000f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 6 Apr 2016 18:29:54 +0800
Subject: [PATCH 4540/6219] [compat] Handle tuples properly in urlencode()

Fixes #9055
---
 test/test_compat.py  | 4 ++++
 youtube_dl/compat.py | 3 ++-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/test/test_compat.py b/test/test_compat.py
index cc105807a..618668210 100644
--- a/test/test_compat.py
+++ b/test/test_compat.py
@@ -76,6 +76,10 @@ class TestCompat(unittest.TestCase):
         self.assertEqual(compat_urllib_parse_urlencode({'abc': b'def'}), 'abc=def')
         self.assertEqual(compat_urllib_parse_urlencode({b'abc': 'def'}), 'abc=def')
         self.assertEqual(compat_urllib_parse_urlencode({b'abc': b'def'}), 'abc=def')
+        self.assertEqual(compat_urllib_parse_urlencode([('abc', 'def')]), 'abc=def')
+        self.assertEqual(compat_urllib_parse_urlencode([('abc', b'def')]), 'abc=def')
+        self.assertEqual(compat_urllib_parse_urlencode([(b'abc', 'def')]), 'abc=def')
+        self.assertEqual(compat_urllib_parse_urlencode([(b'abc', b'def')]), 'abc=def')
 
     def test_compat_shlex_split(self):
         self.assertEqual(compat_shlex_split('-option "one two"'), ['-option', 'one two'])
diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 76b6b0e38..0b6c5ca7a 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -181,7 +181,8 @@ except ImportError:  # Python 2
             if isinstance(e, dict):
                 e = encode_dict(e)
             elif isinstance(e, (list, tuple,)):
-                e = encode_list(e)
+                list_e = encode_list(e)
+                e = tuple(list_e) if isinstance(e, tuple) else list_e
             elif isinstance(e, compat_str):
                 e = e.encode(encoding)
             return e

From be0c7009fb21ebbbe530ad594a9465dff9d72467 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 6 Apr 2016 14:09:31 +0200
Subject: [PATCH 4541/6219] Makefile: use full path for the ISSUE_TEMPLATE.md
 file

---
 Makefile              | 4 ++--
 devscripts/release.sh | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/Makefile b/Makefile
index 3a6c37944..1b08a4783 100644
--- a/Makefile
+++ b/Makefile
@@ -1,4 +1,4 @@
-all: youtube-dl README.md CONTRIBUTING.md ISSUE_TEMPLATE.md README.txt youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtube-dl.fish supportedsites
+all: youtube-dl README.md CONTRIBUTING.md .github/ISSUE_TEMPLATE.md README.txt youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtube-dl.fish supportedsites
 
 clean:
 	rm -rf youtube-dl.1.temp.md youtube-dl.1 youtube-dl.bash-completion README.txt MANIFEST build/ dist/ .coverage cover/ youtube-dl.tar.gz youtube-dl.zsh youtube-dl.fish *.dump *.part *.info.json *.mp4 *.flv *.mp3 *.avi CONTRIBUTING.md.tmp ISSUE_TEMPLATE.md.tmp youtube-dl youtube-dl.exe
@@ -59,7 +59,7 @@ README.md: youtube_dl/*.py youtube_dl/*/*.py
 CONTRIBUTING.md: README.md
 	$(PYTHON) devscripts/make_contributing.py README.md CONTRIBUTING.md
 
-ISSUE_TEMPLATE.md:
+.github/ISSUE_TEMPLATE.md: devscripts/make_issue_template.py .github/ISSUE_TEMPLATE_tmpl.md  youtube_dl/version.py
 	$(PYTHON) devscripts/make_issue_template.py .github/ISSUE_TEMPLATE_tmpl.md .github/ISSUE_TEMPLATE.md
 
 supportedsites:
diff --git a/devscripts/release.sh b/devscripts/release.sh
index 6718ce39b..8dea55dbb 100755
--- a/devscripts/release.sh
+++ b/devscripts/release.sh
@@ -46,7 +46,7 @@ fi
 sed -i "s/__version__ = '.*'/__version__ = '$version'/" youtube_dl/version.py
 
 /bin/echo -e "\n### Committing documentation, templates and youtube_dl/version.py..."
-make README.md CONTRIBUTING.md ISSUE_TEMPLATE.md supportedsites
+make README.md CONTRIBUTING.md .github/ISSUE_TEMPLATE.md supportedsites
 git add README.md CONTRIBUTING.md .github/ISSUE_TEMPLATE.md docs/supportedsites.md youtube_dl/version.py
 git commit -m "release $version"
 

From ec4a4c6fccebc1c8ae7a35129171b1181d8badb1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 6 Apr 2016 14:16:05 +0200
Subject: [PATCH 4542/6219] Makefile: remove ISSUE_TEMPLATE.md from the 'all'
 target (fixes #9088)

It isn't included in the tar file, causing build failures.
Since it's only used for GitHub, I think we don't need to store it in the tar file.
---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 1b08a4783..ba7f7ed36 100644
--- a/Makefile
+++ b/Makefile
@@ -1,4 +1,4 @@
-all: youtube-dl README.md CONTRIBUTING.md .github/ISSUE_TEMPLATE.md README.txt youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtube-dl.fish supportedsites
+all: youtube-dl README.md CONTRIBUTING.md README.txt youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtube-dl.fish supportedsites
 
 clean:
 	rm -rf youtube-dl.1.temp.md youtube-dl.1 youtube-dl.bash-completion README.txt MANIFEST build/ dist/ .coverage cover/ youtube-dl.tar.gz youtube-dl.zsh youtube-dl.fish *.dump *.part *.info.json *.mp4 *.flv *.mp3 *.avi CONTRIBUTING.md.tmp ISSUE_TEMPLATE.md.tmp youtube-dl youtube-dl.exe

From c41cf65d4ab41f08f98905082b7a137ac57927ca Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 6 Apr 2016 15:13:08 +0200
Subject: [PATCH 4543/6219] release 2016.04.06

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 youtube_dl/version.py     | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index e0a7d8282..bf9494646 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.04.05*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.04.05**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.04.06*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.04.06**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.04.05
+[debug] youtube-dl version 2016.04.06
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 45d152902..167b16e24 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.04.05'
+__version__ = '2016.04.06'

From 7a93ab5f3f1535efc948376869f61716ed2af1f0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 7 Apr 2016 02:53:53 +0600
Subject: [PATCH 4544/6219] [extractor/common] Introduce music album metafields

---
 youtube_dl/extractor/common.py | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index ec6625eea..2b40f3b7c 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -232,6 +232,24 @@ class InfoExtractor(object):
     episode_number: Number of the video episode within a season, as an integer.
     episode_id:     Id of the video episode, as a unicode string.
 
+    The following fields should only be used when the media is a track or a part of
+    a music album:
+
+    track:          Title of the track.
+    track_number:   Number of the track within an album or a disc, as an integer.
+    track_id:       Id of the track (useful in case of custom indexing, e.g. 6.iii),
+                    as a unicode string.
+    artist:         Artist(s) of the track.
+    genre:          Genre(s) of the track.
+    album:          Title of the album the track belongs to.
+    album_type:     Type of the album (e.g. "Demo", "Full-length", "Split", "Compilation", etc).
+    album_artist:   List of all artists appeared on the album (e.g.
+                    "Ash Borer / Fell Voices" or "Various Artists", useful for splits
+                    and compilations).
+    disc_number:    Number of the disc or other physical medium the track belongs to,
+                    as an integer.
+    release_year:   Year (YYYY) when the album was released.
+
     Unless mentioned otherwise, the fields should be Unicode strings.
 
     Unless mentioned otherwise, None is equivalent to absence of information.

From e90d175436e61e207e0b0cae7f699494dcf15922 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 7 Apr 2016 02:56:13 +0600
Subject: [PATCH 4545/6219] [yandexmusic] Extract music album metafields
 (Closes #7354)

---
 youtube_dl/extractor/yandexmusic.py | 40 ++++++++++++++++++++++++++---
 1 file changed, 37 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/yandexmusic.py b/youtube_dl/extractor/yandexmusic.py
index 025716958..7a90cc60c 100644
--- a/youtube_dl/extractor/yandexmusic.py
+++ b/youtube_dl/extractor/yandexmusic.py
@@ -39,9 +39,14 @@ class YandexMusicTrackIE(YandexMusicBaseIE):
         'info_dict': {
             'id': '4878838',
             'ext': 'mp3',
-            'title': 'Carlo Ambrosio - Gypsy Eyes 1',
+            'title': 'Carlo Ambrosio & Fabio Di Bari, Carlo Ambrosio - Gypsy Eyes 1',
             'filesize': 4628061,
             'duration': 193.04,
+            'track': 'Gypsy Eyes 1',
+            'album': 'Gypsy Soul',
+            'album_artist': 'Carlo Ambrosio',
+            'artist': 'Carlo Ambrosio & Fabio Di Bari, Carlo Ambrosio',
+            'release_year': '2009',
         }
     }
 
@@ -64,16 +69,45 @@ class YandexMusicTrackIE(YandexMusicBaseIE):
             thumbnail = cover_uri.replace('%%', 'orig')
             if not thumbnail.startswith('http'):
                 thumbnail = 'http://' + thumbnail
-        return {
+
+        track_title = track['title']
+        track_info = {
             'id': track['id'],
             'ext': 'mp3',
             'url': self._get_track_url(track['storageDir'], track['id']),
-            'title': '%s - %s' % (track['artists'][0]['name'], track['title']),
             'filesize': int_or_none(track.get('fileSize')),
             'duration': float_or_none(track.get('durationMs'), 1000),
             'thumbnail': thumbnail,
+            'track': track_title,
         }
 
+        def extract_artist(artist_list):
+            if artist_list and isinstance(artist_list, list):
+                artists_names = [a['name'] for a in artist_list if a.get('name')]
+                if artists_names:
+                    return ', '.join(artists_names)
+
+        albums = track.get('albums')
+        if albums and isinstance(albums, list):
+            album = albums[0]
+            if isinstance(album, dict):
+                year = album.get('year')
+                track_info.update({
+                    'album': album.get('title'),
+                    'album_artist': extract_artist(album.get('artists')),
+                    'release_year': compat_str(year) if year else None,
+                })
+
+        track_artist = extract_artist(track.get('artists'))
+        if track_artist:
+            track_info.update({
+                'artist': track_artist,
+                'title': '%s - %s' % (track_artist, track_title),
+            })
+        else:
+            track_info['title'] = track_title
+        return track_info
+
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         album_id, track_id = mobj.group('album_id'), mobj.group('id')

From 3afef2e3fc60a7baa2d923e9cfbaf521c7f5ca0d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 7 Apr 2016 22:40:35 +0600
Subject: [PATCH 4546/6219] [beeg] Improve extraction

---
 youtube_dl/extractor/beeg.py | 31 ++++++++++++++++++++++++++++---
 1 file changed, 28 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/beeg.py b/youtube_dl/extractor/beeg.py
index 9072949dd..956c7680e 100644
--- a/youtube_dl/extractor/beeg.py
+++ b/youtube_dl/extractor/beeg.py
@@ -33,8 +33,33 @@ class BeegIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
+        webpage = self._download_webpage(url, video_id)
+
+        cpl_url = self._search_regex(
+            r'<script[^>]+src=(["\'])(?P<url>(?:https?:)?//static\.beeg\.com/cpl/\d+\.js.*?)\1',
+            webpage, 'cpl', default=None, group='url')
+
+        beeg_version, beeg_salt = [None] * 2
+
+        if cpl_url:
+            cpl = self._download_webpage(
+                self._proto_relative_url(cpl_url), video_id,
+                'Downloading cpl JS', fatal=False)
+            if cpl:
+                beeg_version = self._search_regex(
+                    r'beeg_version\s*=\s*(\d+)', cpl,
+                    'beeg version', default=None) or self._search_regex(
+                    r'/(\d+)\.js', cpl_url, 'beeg version', default=None)
+                beeg_salt = self._search_regex(
+                    r'beeg_salt\s*=\s*(["\'])(?P<beeg_salt>.+?)\1', cpl, 'beeg beeg_salt',
+                    default=None, group='beeg_salt')
+
+        beeg_version = beeg_version or '1750'
+        beeg_salt = beeg_salt or 'MIDtGaw96f0N1kMMAM1DE46EC9pmFr'
+
         video = self._download_json(
-            'https://api.beeg.com/api/v6/1738/video/%s' % video_id, video_id)
+            'http://api.beeg.com/api/v6/%s/video/%s' % (beeg_version, video_id),
+            video_id)
 
         def split(o, e):
             def cut(s, x):
@@ -51,7 +76,7 @@ class BeegIE(InfoExtractor):
 
         def decrypt_key(key):
             # Reverse engineered from http://static.beeg.com/cpl/1738.js
-            a = 'GUuyodcfS8FW8gQp4OKLMsZBcX0T7B'
+            a = beeg_salt
             e = compat_urllib_parse_unquote(key)
             o = ''.join([
                 compat_chr(compat_ord(e[n]) - compat_ord(a[n % len(a)]) % 21)
@@ -101,5 +126,5 @@ class BeegIE(InfoExtractor):
             'duration': duration,
             'tags': tags,
             'formats': formats,
-            'age_limit': 18,
+            'age_limit': self._rta_search(webpage),
         }

From ed6fb8b804448724fcd1ba4abc3fa028b817efe2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 7 Apr 2016 23:22:43 +0600
Subject: [PATCH 4547/6219] [vrt] Add support for direct hls playlists and
 YouTube (Closes #9108)

---
 youtube_dl/extractor/vrt.py | 34 ++++++++++++++++++++++++++++------
 1 file changed, 28 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/vrt.py b/youtube_dl/extractor/vrt.py
index 2b6bae89b..8e35f24e8 100644
--- a/youtube_dl/extractor/vrt.py
+++ b/youtube_dl/extractor/vrt.py
@@ -4,7 +4,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import float_or_none
+from ..utils import (
+    determine_ext,
+    float_or_none,
+)
 
 
 class VRTIE(InfoExtractor):
@@ -52,6 +55,11 @@ class VRTIE(InfoExtractor):
                 'duration': 661,
             }
         },
+        {
+            # YouTube video
+            'url': 'http://deredactie.be/cm/vrtnieuws/videozone/nieuws/cultuurenmedia/1.2622957',
+            'only_matching': True,
+        },
         {
             'url': 'http://cobra.canvas.be/cm/cobra/videozone/rubriek/film-videozone/1.2377055',
             'only_matching': True,
@@ -66,7 +74,17 @@ class VRTIE(InfoExtractor):
         video_id = self._search_regex(
             r'data-video-id="([^"]+)_[^"]+"', webpage, 'video id', fatal=False)
 
+        src = self._search_regex(
+            r'data-video-src="([^"]+)"', webpage, 'video src', default=None)
+
+        video_type = self._search_regex(
+            r'data-video-type="([^"]+)"', webpage, 'video type', default=None)
+
+        if video_type == 'YouTubeVideo':
+            return self.url_result(src, 'Youtube')
+
         formats = []
+
         mobj = re.search(
             r'data-video-iphone-server="(?P<server>[^"]+)"\s+data-video-iphone-path="(?P<path>[^"]+)"',
             webpage)
@@ -74,11 +92,15 @@ class VRTIE(InfoExtractor):
             formats.extend(self._extract_m3u8_formats(
                 '%s/%s' % (mobj.group('server'), mobj.group('path')),
                 video_id, 'mp4', m3u8_id='hls', fatal=False))
-        mobj = re.search(r'data-video-src="(?P<src>[^"]+)"', webpage)
-        if mobj:
-            formats.extend(self._extract_f4m_formats(
-                '%s/manifest.f4m' % mobj.group('src'),
-                video_id, f4m_id='hds', fatal=False))
+
+        if src:
+            if determine_ext(src) == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    src, video_id, 'mp4', entry_protocol='m3u8_native',
+                    m3u8_id='hls', fatal=False))
+            else:
+                formats.extend(self._extract_f4m_formats(
+                    '%s/manifest.f4m' % src, video_id, f4m_id='hds', fatal=False))
 
         if not formats and 'data-video-geoblocking="true"' in webpage:
             self.raise_geo_restricted('This video is only available in Belgium')

From 536a55dabd7bcc2f34195beb84211028c934ed7a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 8 Apr 2016 00:17:47 +0600
Subject: [PATCH 4548/6219] [YoutubeDL] Sanitize single thumbnail URL

---
 youtube_dl/YoutubeDL.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index d7aa951ff..cd0805303 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1240,7 +1240,10 @@ class YoutubeDL(object):
             self.list_thumbnails(info_dict)
             return
 
-        if thumbnails and 'thumbnail' not in info_dict:
+        thumbnail = info_dict.get('thumbnail')
+        if thumbnail:
+            info_dict['thumbnail'] = sanitize_url(thumbnail)
+        elif thumbnails:
             info_dict['thumbnail'] = thumbnails[-1]['url']
 
         if 'display_id' not in info_dict and 'id' in info_dict:

From 9a32e80477f470b8d8d320db38b5de02e0e6bc92 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 8 Apr 2016 14:51:00 +0100
Subject: [PATCH 4549/6219] [acast] fix extraction(#9117)

---
 youtube_dl/extractor/acast.py | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/acast.py b/youtube_dl/extractor/acast.py
index 92eee8119..79a17e73a 100644
--- a/youtube_dl/extractor/acast.py
+++ b/youtube_dl/extractor/acast.py
@@ -26,13 +26,7 @@ class ACastIE(InfoExtractor):
 
     def _real_extract(self, url):
         channel, display_id = re.match(self._VALID_URL, url).groups()
-
-        embed_page = self._download_webpage(
-            re.sub('(?:www\.)?acast\.com', 'embedcdn.acast.com', url), display_id)
-        cast_data = self._parse_json(self._search_regex(
-            r'window\[\'acast/queries\'\]\s*=\s*([^;]+);', embed_page, 'acast data'),
-            display_id)['GetAcast/%s/%s' % (channel, display_id)]
-
+        cast_data = self._download_json('https://embed.acast.com/api/acasts/%s/%s' % (channel, display_id), display_id)
         return {
             'id': compat_str(cast_data['id']),
             'display_id': display_id,

From a1ff3cd5f98980e37b5bdb1fd24cdba56ed6e618 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 8 Apr 2016 15:15:34 +0100
Subject: [PATCH 4550/6219] [acast] fix channel extraction(closes #9117)

---
 youtube_dl/extractor/acast.py | 32 ++++++++++++++++++++++++--------
 1 file changed, 24 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/acast.py b/youtube_dl/extractor/acast.py
index 79a17e73a..94ce88c83 100644
--- a/youtube_dl/extractor/acast.py
+++ b/youtube_dl/extractor/acast.py
@@ -2,10 +2,14 @@
 from __future__ import unicode_literals
 
 import re
+import functools
 
 from .common import InfoExtractor
 from ..compat import compat_str
-from ..utils import int_or_none
+from ..utils import (
+    int_or_none,
+    OnDemandPagedList,
+)
 
 
 class ACastIE(InfoExtractor):
@@ -26,7 +30,8 @@ class ACastIE(InfoExtractor):
 
     def _real_extract(self, url):
         channel, display_id = re.match(self._VALID_URL, url).groups()
-        cast_data = self._download_json('https://embed.acast.com/api/acasts/%s/%s' % (channel, display_id), display_id)
+        cast_data = self._download_json(
+            'https://embed.acast.com/api/acasts/%s/%s' % (channel, display_id), display_id)
         return {
             'id': compat_str(cast_data['id']),
             'display_id': display_id,
@@ -52,15 +57,26 @@ class ACastChannelIE(InfoExtractor):
         'playlist_mincount': 20,
     }
     _API_BASE_URL = 'https://www.acast.com/api/'
+    _PAGE_SIZE = 10
 
     @classmethod
     def suitable(cls, url):
         return False if ACastIE.suitable(url) else super(ACastChannelIE, cls).suitable(url)
 
-    def _real_extract(self, url):
-        display_id = self._match_id(url)
-        channel_data = self._download_json(self._API_BASE_URL + 'channels/%s' % display_id, display_id)
-        casts = self._download_json(self._API_BASE_URL + 'channels/%s/acasts' % display_id, display_id)
-        entries = [self.url_result('https://www.acast.com/%s/%s' % (display_id, cast['url']), 'ACast') for cast in casts]
+    def _fetch_page(self, channel_slug, page):
+        casts = self._download_json(
+            self._API_BASE_URL + 'channels/%s/acasts?page=%s' % (channel_slug, page),
+            channel_slug, note='Download page %d of channel data' % page)
+        for cast in casts:
+            yield self.url_result(
+                'https://www.acast.com/%s/%s' % (channel_slug, cast['url']),
+                'ACast', cast['id'])
 
-        return self.playlist_result(entries, compat_str(channel_data['id']), channel_data['name'], channel_data.get('description'))
+    def _real_extract(self, url):
+        channel_slug = self._match_id(url)
+        channel_data = self._download_json(
+            self._API_BASE_URL + 'channels/%s' % channel_slug, channel_slug)
+        entries = OnDemandPagedList(functools.partial(
+            self._fetch_page, channel_slug), self._PAGE_SIZE)
+        return self.playlist_result(entries, compat_str(
+            channel_data['id']), channel_data['name'], channel_data.get('description'))

From 56019444cb2fe64f0937fb52aff9e36f30b3b343 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 8 Apr 2016 21:26:42 +0600
Subject: [PATCH 4551/6219] [novamov] Improve _VALID_URL template (Closes
 #9116)

---
 youtube_dl/extractor/novamov.py | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/novamov.py b/youtube_dl/extractor/novamov.py
index 5771a675d..3bbd47355 100644
--- a/youtube_dl/extractor/novamov.py
+++ b/youtube_dl/extractor/novamov.py
@@ -16,7 +16,14 @@ class NovaMovIE(InfoExtractor):
     IE_NAME = 'novamov'
     IE_DESC = 'NovaMov'
 
-    _VALID_URL_TEMPLATE = r'http://(?:(?:www\.)?%(host)s/(?:file|video|mobile/#/videos)/|(?:(?:embed|www)\.)%(host)s/embed\.php\?(?:.*?&)?v=)(?P<id>[a-z\d]{13})'
+    _VALID_URL_TEMPLATE = r'''(?x)
+                            http://
+                                (?:
+                                    (?:www\.)?%(host)s/(?:file|video|mobile/\#/videos)/|
+                                    (?:(?:embed|www)\.)%(host)s/embed(?:\.php|/)?\?(?:.*?&)?\bv=
+                                )
+                                (?P<id>[a-z\d]{13})
+                            '''
     _VALID_URL = _VALID_URL_TEMPLATE % {'host': 'novamov\.com'}
 
     _HOST = 'www.novamov.com'
@@ -189,7 +196,7 @@ class AuroraVidIE(NovaMovIE):
 
     _FILE_DELETED_REGEX = r'This file no longer exists on our servers!<'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.auroravid.to/video/4rurhn9x446jj',
         'md5': '7205f346a52bbeba427603ba10d4b935',
         'info_dict': {
@@ -199,4 +206,7 @@ class AuroraVidIE(NovaMovIE):
             'description': 'search engine optimization is used to rank the web page in the google search engine'
         },
         'skip': '"Invalid token" errors abound (in web interface as well as youtube-dl, there is nothing we can do about it.)'
-    }
+    }, {
+        'url': 'http://www.auroravid.to/embed/?v=4rurhn9x446jj',
+        'only_matching': True,
+    }]

From a64c0c9b06b24085b1332452b5859fe6ac0a26d5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 8 Apr 2016 22:15:36 +0600
Subject: [PATCH 4552/6219] [democracynow] Make description optional (Closes
 #9115)

---
 youtube_dl/extractor/democracynow.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/democracynow.py b/youtube_dl/extractor/democracynow.py
index 6cd395e11..4b6d2652a 100644
--- a/youtube_dl/extractor/democracynow.py
+++ b/youtube_dl/extractor/democracynow.py
@@ -38,7 +38,7 @@ class DemocracynowIE(InfoExtractor):
     def _real_extract(self, url):
         display_id = self._match_id(url)
         webpage = self._download_webpage(url, display_id)
-        description = self._og_search_description(webpage)
+        description = self._og_search_description(webpage, default=None)
 
         json_data = self._parse_json(self._search_regex(
             r'<script[^>]+type="text/json"[^>]*>\s*({[^>]+})', webpage, 'json'),

From a134426d619ac711f6adc24242b1e7d66d0b346a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 8 Apr 2016 22:19:16 +0600
Subject: [PATCH 4553/6219] [democracynow] Fix tests

---
 youtube_dl/extractor/democracynow.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/democracynow.py b/youtube_dl/extractor/democracynow.py
index 4b6d2652a..188f890ce 100644
--- a/youtube_dl/extractor/democracynow.py
+++ b/youtube_dl/extractor/democracynow.py
@@ -17,22 +17,23 @@ class DemocracynowIE(InfoExtractor):
     IE_NAME = 'democracynow'
     _TESTS = [{
         'url': 'http://www.democracynow.org/shows/2015/7/3',
-        'md5': 'fbb8fe3d7a56a5e12431ce2f9b2fab0d',
+        'md5': '3757c182d3d84da68f5c8f506c18c196',
         'info_dict': {
             'id': '2015-0703-001',
             'ext': 'mp4',
-            'title': 'July 03, 2015 - Democracy Now!',
-            'description': 'A daily independent global news hour with Amy Goodman & Juan González "What to the Slave is 4th of July?": James Earl Jones Reads Frederick Douglass\u2019 Historic Speech : "This Flag Comes Down Today": Bree Newsome Scales SC Capitol Flagpole, Takes Down Confederate Flag : "We Shall Overcome": Remembering Folk Icon, Activist Pete Seeger in His Own Words & Songs',
+            'title': 'Daily Show',
         },
     }, {
         'url': 'http://www.democracynow.org/2015/7/3/this_flag_comes_down_today_bree',
-        'md5': 'fbb8fe3d7a56a5e12431ce2f9b2fab0d',
         'info_dict': {
             'id': '2015-0703-001',
             'ext': 'mp4',
             'title': '"This Flag Comes Down Today": Bree Newsome Scales SC Capitol Flagpole, Takes Down Confederate Flag',
             'description': 'md5:4d2bc4f0d29f5553c2210a4bc7761a21',
         },
+        'params': {
+            'skip_download': True,
+        },
     }]
 
     def _real_extract(self, url):

From 18da24634c38ff6af4deaf606badfcbb9e6c3d68 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 8 Apr 2016 22:27:27 +0600
Subject: [PATCH 4554/6219] [democracynow] Improve extraction

---
 youtube_dl/extractor/democracynow.py | 36 ++++++++++++++++------------
 1 file changed, 21 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/democracynow.py b/youtube_dl/extractor/democracynow.py
index 188f890ce..65a98d789 100644
--- a/youtube_dl/extractor/democracynow.py
+++ b/youtube_dl/extractor/democracynow.py
@@ -38,17 +38,32 @@ class DemocracynowIE(InfoExtractor):
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
+
         webpage = self._download_webpage(url, display_id)
-        description = self._og_search_description(webpage, default=None)
 
         json_data = self._parse_json(self._search_regex(
             r'<script[^>]+type="text/json"[^>]*>\s*({[^>]+})', webpage, 'json'),
             display_id)
-        video_id = None
+
+        title = json_data['title']
         formats = []
 
-        default_lang = 'en'
+        video_id = None
 
+        for key in ('file', 'audio', 'video', 'high_res_video'):
+            media_url = json_data.get(key, '')
+            if not media_url:
+                continue
+            media_url = re.sub(r'\?.*', '', compat_urlparse.urljoin(url, media_url))
+            video_id = video_id or remove_start(os.path.splitext(url_basename(media_url))[0], 'dn')
+            formats.append({
+                'url': media_url,
+                'vcodec': 'none' if key == 'audio' else None,
+            })
+
+        self._sort_formats(formats)
+
+        default_lang = 'en'
         subtitles = {}
 
         def add_subtitle_item(lang, info_dict):
@@ -68,22 +83,13 @@ class DemocracynowIE(InfoExtractor):
                 'url': compat_urlparse.urljoin(url, subtitle_item['url']),
             })
 
-        for key in ('file', 'audio', 'video'):
-            media_url = json_data.get(key, '')
-            if not media_url:
-                continue
-            media_url = re.sub(r'\?.*', '', compat_urlparse.urljoin(url, media_url))
-            video_id = video_id or remove_start(os.path.splitext(url_basename(media_url))[0], 'dn')
-            formats.append({
-                'url': media_url,
-            })
-
-        self._sort_formats(formats)
+        description = self._og_search_description(webpage, default=None)
 
         return {
             'id': video_id or display_id,
-            'title': json_data['title'],
+            'title': title,
             'description': description,
+            'thumbnail': json_data.get('image'),
             'subtitles': subtitles,
             'formats': formats,
         }

From fb38aa8b53d25606d2582e1043d09ad1a077bf61 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 8 Apr 2016 22:48:08 +0600
Subject: [PATCH 4555/6219] [extractor/common] Support arbitrary format strings
 for template based identifiers in mpd manifests (Closes #9119, closes #9120)

---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 2b40f3b7c..a7324af5c 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1534,7 +1534,7 @@ class InfoExtractor(object):
                             media_template = representation_ms_info['media_template']
                             media_template = media_template.replace('$RepresentationID$', representation_id)
                             media_template = re.sub(r'\$(Number|Bandwidth)\$', r'%(\1)d', media_template)
-                            media_template = re.sub(r'\$(Number|Bandwidth)%(\d+)\$', r'%(\1)\2d', media_template)
+                            media_template = re.sub(r'\$(Number|Bandwidth)%([^$]+)\$', r'%(\1)\2', media_template)
                             media_template.replace('$$', '$')
                             representation_ms_info['segment_urls'] = [
                                 media_template % {

From 3c6c7e7d7e5a7cbf87385bf3c649342c3ee4327d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 8 Apr 2016 23:16:02 +0600
Subject: [PATCH 4556/6219] [gdcvault] Fix extraction (Closes #9107, closes
 #9114)

---
 youtube_dl/extractor/gdcvault.py | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/gdcvault.py b/youtube_dl/extractor/gdcvault.py
index 59ed4c38f..25e93c9a4 100644
--- a/youtube_dl/extractor/gdcvault.py
+++ b/youtube_dl/extractor/gdcvault.py
@@ -159,9 +159,10 @@ class GDCVaultIE(InfoExtractor):
                 'title': title,
             }
 
+        PLAYER_REGEX = r'<iframe src="(?P<xml_root>.+?)/player.*?\.html.*?".*?</iframe>'
+
         xml_root = self._html_search_regex(
-            r'<iframe src="(?P<xml_root>.*?)player.html.*?".*?</iframe>',
-            start_page, 'xml root', default=None)
+            PLAYER_REGEX, start_page, 'xml root', default=None)
         if xml_root is None:
             # Probably need to authenticate
             login_res = self._login(webpage_url, display_id)
@@ -171,18 +172,19 @@ class GDCVaultIE(InfoExtractor):
                 start_page = login_res
                 # Grab the url from the authenticated page
                 xml_root = self._html_search_regex(
-                    r'<iframe src="(.*?)player.html.*?".*?</iframe>',
-                    start_page, 'xml root')
+                    PLAYER_REGEX, start_page, 'xml root')
 
         xml_name = self._html_search_regex(
             r'<iframe src=".*?\?xml=(.+?\.xml).*?".*?</iframe>',
             start_page, 'xml filename', default=None)
         if xml_name is None:
             # Fallback to the older format
-            xml_name = self._html_search_regex(r'<iframe src=".*?\?xmlURL=xml/(?P<xml_file>.+?\.xml).*?".*?</iframe>', start_page, 'xml filename')
+            xml_name = self._html_search_regex(
+                r'<iframe src=".*?\?xmlURL=xml/(?P<xml_file>.+?\.xml).*?".*?</iframe>',
+                start_page, 'xml filename')
 
-        xml_description_url = xml_root + 'xml/' + xml_name
-        xml_description = self._download_xml(xml_description_url, display_id)
+        xml_description = self._download_xml(
+            '%s/xml/%s' % (xml_root, xml_name), display_id)
 
         video_title = xml_description.find('./metadata/title').text
         video_formats = self._parse_mp4(xml_description)

From bacec0397ff2abddac460148e8ceb49989fce6a2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 8 Apr 2016 23:33:45 +0600
Subject: [PATCH 4557/6219] [extractor/common] Relax _hidden_inputs

---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index a7324af5c..17d00721c 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -843,7 +843,7 @@ class InfoExtractor(object):
         for input in re.findall(r'(?i)<input([^>]+)>', html):
             if not re.search(r'type=(["\'])(?:hidden|submit)\1', input):
                 continue
-            name = re.search(r'name=(["\'])(?P<value>.+?)\1', input)
+            name = re.search(r'(?:name|id)=(["\'])(?P<value>.+?)\1', input)
             if not name:
                 continue
             value = re.search(r'value=(["\'])(?P<value>.*?)\1', input)

From 2f2fcf1a3304a7134d280726ae1a3d7010adbd59 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 8 Apr 2016 23:34:59 +0600
Subject: [PATCH 4558/6219] [tnaflix] Fix extraction (Closes #9074)

---
 youtube_dl/extractor/tnaflix.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tnaflix.py b/youtube_dl/extractor/tnaflix.py
index 79f036fe4..bc32c0926 100644
--- a/youtube_dl/extractor/tnaflix.py
+++ b/youtube_dl/extractor/tnaflix.py
@@ -76,7 +76,11 @@ class TNAFlixNetworkBaseIE(InfoExtractor):
         webpage = self._download_webpage(url, display_id)
 
         cfg_url = self._proto_relative_url(self._html_search_regex(
-            self._CONFIG_REGEX, webpage, 'flashvars.config'), 'http:')
+            self._CONFIG_REGEX, webpage, 'flashvars.config', default=None), 'http:')
+
+        if not cfg_url:
+            inputs = self._hidden_inputs(webpage)
+            cfg_url = 'https://cdn-fck.tnaflix.com/tnaflix/%s.fid?key=%s' % (inputs['vkey'], inputs['nkey'])
 
         cfg_xml = self._download_xml(
             cfg_url, display_id, 'Downloading metadata',

From 568d2f78d635c3993e95334b9f8f6d2b47ecee51 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 9 Apr 2016 00:27:24 +0600
Subject: [PATCH 4559/6219] [tnaflix] Fix metadata extraction

---
 youtube_dl/extractor/tnaflix.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/tnaflix.py b/youtube_dl/extractor/tnaflix.py
index bc32c0926..78174178e 100644
--- a/youtube_dl/extractor/tnaflix.py
+++ b/youtube_dl/extractor/tnaflix.py
@@ -136,7 +136,7 @@ class TNAFlixNetworkBaseIE(InfoExtractor):
         average_rating = float_or_none(extract_field(self._AVERAGE_RATING_REGEX, 'average rating'))
 
         categories_str = extract_field(self._CATEGORIES_REGEX, 'categories')
-        categories = categories_str.split(', ') if categories_str is not None else []
+        categories = [c.strip() for c in categories_str.split(',')] if categories_str is not None else []
 
         return {
             'id': video_id,
@@ -190,13 +190,14 @@ class TNAFlixIE(TNAFlixNetworkBaseIE):
     _VALID_URL = r'https?://(?:www\.)?tnaflix\.com/[^/]+/(?P<display_id>[^/]+)/video(?P<id>\d+)'
 
     _TITLE_REGEX = r'<title>(.+?) - TNAFlix Porn Videos</title>'
-    _DESCRIPTION_REGEX = r'<h3 itemprop="description">([^<]+)</h3>'
-    _UPLOADER_REGEX = r'(?s)<span[^>]+class="infoTitle"[^>]*>Uploaded By:</span>(.+?)<div'
+    _DESCRIPTION_REGEX = r'<meta[^>]+name="description"[^>]+content="([^"]+)"'
+    _UPLOADER_REGEX = r'<i>\s*Verified Member\s*</i>\s*<h1>(.+?)</h1>'
+    _CATEGORIES_REGEX = r'(?s)<span[^>]*>Categories:</span>(.+?)</div>'
 
     _TESTS = [{
         # anonymous uploader, no categories
         'url': 'http://www.tnaflix.com/porn-stars/Carmella-Decesare-striptease/video553878',
-        'md5': 'ecf3498417d09216374fc5907f9c6ec0',
+        'md5': '7e569419fe6d69543d01e6be22f5f7c4',
         'info_dict': {
             'id': '553878',
             'display_id': 'Carmella-Decesare-striptease',
@@ -205,17 +206,16 @@ class TNAFlixIE(TNAFlixNetworkBaseIE):
             'thumbnail': 're:https?://.*\.jpg$',
             'duration': 91,
             'age_limit': 18,
-            'uploader': 'Anonymous',
-            'categories': [],
+            'categories': ['Porn Stars'],
         }
     }, {
         # non-anonymous uploader, categories
         'url': 'https://www.tnaflix.com/teen-porn/Educational-xxx-video/video6538',
-        'md5': '0f5d4d490dbfd117b8607054248a07c0',
+        'md5': 'fcba2636572895aba116171a899a5658',
         'info_dict': {
             'id': '6538',
             'display_id': 'Educational-xxx-video',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': 'Educational xxx video',
             'description': 'md5:b4fab8f88a8621c8fabd361a173fe5b8',
             'thumbnail': 're:https?://.*\.jpg$',

From e52d7f85f25e806527d7b618d8c3ad16d27681f4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 10 Feb 2016 13:16:18 +0100
Subject: [PATCH 4560/6219] Delay initialization of InfoExtractors until they
 are needed

---
 youtube_dl/YoutubeDL.py          | 10 ++++++----
 youtube_dl/extractor/__init__.py |  9 ++++++++-
 2 files changed, 14 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index cd0805303..f18a8e840 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -82,7 +82,7 @@ from .utils import (
     YoutubeDLHandler,
 )
 from .cache import Cache
-from .extractor import get_info_extractor, gen_extractors
+from .extractor import get_info_extractor, gen_extractor_classes
 from .downloader import get_suitable_downloader
 from .downloader.rtmp import rtmpdump_version
 from .postprocessor import (
@@ -378,8 +378,9 @@ class YoutubeDL(object):
     def add_info_extractor(self, ie):
         """Add an InfoExtractor object to the end of the list."""
         self._ies.append(ie)
-        self._ies_instances[ie.ie_key()] = ie
-        ie.set_downloader(self)
+        if not isinstance(ie, type):
+            self._ies_instances[ie.ie_key()] = ie
+            ie.set_downloader(self)
 
     def get_info_extractor(self, ie_key):
         """
@@ -397,7 +398,7 @@ class YoutubeDL(object):
         """
         Add the InfoExtractors returned by gen_extractors to the end of the list
         """
-        for ie in gen_extractors():
+        for ie in gen_extractor_classes():
             self.add_info_extractor(ie)
 
     def add_post_processor(self, pp):
@@ -661,6 +662,7 @@ class YoutubeDL(object):
             if not ie.suitable(url):
                 continue
 
+            ie = self.get_info_extractor(ie.ie_key())
             if not ie.working():
                 self.report_warning('The program functionality for this site has been marked as broken, '
                                     'and will probably not work.')
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index c3121d83c..cd1f116e2 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -997,11 +997,18 @@ _ALL_CLASSES = [
 _ALL_CLASSES.append(GenericIE)
 
 
+def gen_extractor_classes():
+    """ Return a list of supported extractors.
+    The order does matter; the first extractor matched is the one handling the URL.
+    """
+    return _ALL_CLASSES
+
+
 def gen_extractors():
     """ Return a list of an instance of every supported extractor.
     The order does matter; the first extractor matched is the one handling the URL.
     """
-    return [klass() for klass in _ALL_CLASSES]
+    return [klass() for klass in gen_extractor_classes()]
 
 
 def list_extractors(age_limit):

From 1b3d5e05a824f880f1171eb840235e13cd8848dc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 10 Feb 2016 13:24:49 +0100
Subject: [PATCH 4561/6219] Move the extreactors import to
 youtube_dl/extractor/extractors.py

---
 README.md                          |   4 +-
 youtube_dl/extractor/__init__.py   | 989 +---------------------------
 youtube_dl/extractor/extractors.py | 991 +++++++++++++++++++++++++++++
 3 files changed, 994 insertions(+), 990 deletions(-)
 create mode 100644 youtube_dl/extractor/extractors.py

diff --git a/README.md b/README.md
index e972bf69f..cd18edd87 100644
--- a/README.md
+++ b/README.md
@@ -889,14 +889,14 @@ After you have ensured this site is distributing it's content legally, you can f
                 # TODO more properties (see youtube_dl/extractor/common.py)
             }
     ```
-5. Add an import in [`youtube_dl/extractor/__init__.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/__init__.py).
+5. Add an import in [`youtube_dl/extractor/extractors.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/extractors.py).
 6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc.
 7. Have a look at [`youtube_dl/extractor/common.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/rg3/youtube-dl/blob/58525c94d547be1c8167d16c298bdd75506db328/youtube_dl/extractor/common.py#L68-L226). Add tests and code for as many as you want.
 8. Keep in mind that the only mandatory fields in info dict for successful extraction process are `id`, `title` and either `url` or `formats`, i.e. these are the critical data the extraction does not make any sense without. This means that [any field](https://github.com/rg3/youtube-dl/blob/58525c94d547be1c8167d16c298bdd75506db328/youtube_dl/extractor/common.py#L138-L226) apart from aforementioned mandatory ones should be treated **as optional** and extraction should be **tolerate** to situations when sources for these fields can potentially be unavailable (even if they always available at the moment) and **future-proof** in order not to break the extraction of general purpose mandatory fields. For example, if you have some intermediate dict `meta` that is a source of metadata and it has a key `summary` that you want to extract and put into resulting info dict as `description`, you should be ready that this key may be missing from the `meta` dict, i.e. you should extract it as `meta.get('summary')` and not `meta['summary']`. Similarly, you should pass `fatal=False` when extracting data from a webpage with `_search_regex/_html_search_regex`.
 9. Check the code with [flake8](https://pypi.python.org/pypi/flake8).
 10. When the tests pass, [add](http://git-scm.com/docs/git-add) the new files and [commit](http://git-scm.com/docs/git-commit) them and [push](http://git-scm.com/docs/git-push) the result, like this:
 
-        $ git add youtube_dl/extractor/__init__.py
+        $ git add youtube_dl/extractor/extractors.py
         $ git add youtube_dl/extractor/yourextractor.py
         $ git commit -m '[yourextractor] Add new extractor'
         $ git push origin yourextractor
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index cd1f116e2..a0a53445a 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -1,993 +1,6 @@
 from __future__ import unicode_literals
 
-from .abc import ABCIE
-from .abc7news import Abc7NewsIE
-from .academicearth import AcademicEarthCourseIE
-from .acast import (
-    ACastIE,
-    ACastChannelIE,
-)
-from .addanime import AddAnimeIE
-from .adobetv import (
-    AdobeTVIE,
-    AdobeTVShowIE,
-    AdobeTVChannelIE,
-    AdobeTVVideoIE,
-)
-from .adultswim import AdultSwimIE
-from .aenetworks import AENetworksIE
-from .aftonbladet import AftonbladetIE
-from .airmozilla import AirMozillaIE
-from .aljazeera import AlJazeeraIE
-from .alphaporno import AlphaPornoIE
-from .animeondemand import AnimeOnDemandIE
-from .anitube import AnitubeIE
-from .anysex import AnySexIE
-from .aol import (
-    AolIE,
-    AolFeaturesIE,
-)
-from .allocine import AllocineIE
-from .aparat import AparatIE
-from .appleconnect import AppleConnectIE
-from .appletrailers import (
-    AppleTrailersIE,
-    AppleTrailersSectionIE,
-)
-from .archiveorg import ArchiveOrgIE
-from .ard import (
-    ARDIE,
-    ARDMediathekIE,
-    SportschauIE,
-)
-from .arte import (
-    ArteTvIE,
-    ArteTVPlus7IE,
-    ArteTVCreativeIE,
-    ArteTVConcertIE,
-    ArteTVFutureIE,
-    ArteTVCinemaIE,
-    ArteTVDDCIE,
-    ArteTVMagazineIE,
-    ArteTVEmbedIE,
-)
-from .atresplayer import AtresPlayerIE
-from .atttechchannel import ATTTechChannelIE
-from .audimedia import AudiMediaIE
-from .audioboom import AudioBoomIE
-from .audiomack import AudiomackIE, AudiomackAlbumIE
-from .azubu import AzubuIE, AzubuLiveIE
-from .baidu import BaiduVideoIE
-from .bambuser import BambuserIE, BambuserChannelIE
-from .bandcamp import BandcampIE, BandcampAlbumIE
-from .bbc import (
-    BBCCoUkIE,
-    BBCCoUkArticleIE,
-    BBCIE,
-)
-from .beeg import BeegIE
-from .behindkink import BehindKinkIE
-from .beatportpro import BeatportProIE
-from .bet import BetIE
-from .bigflix import BigflixIE
-from .bild import BildIE
-from .bilibili import BiliBiliIE
-from .biobiochiletv import BioBioChileTVIE
-from .bleacherreport import (
-    BleacherReportIE,
-    BleacherReportCMSIE,
-)
-from .blinkx import BlinkxIE
-from .bloomberg import BloombergIE
-from .bokecc import BokeCCIE
-from .bpb import BpbIE
-from .br import BRIE
-from .bravotv import BravoTVIE
-from .breakcom import BreakIE
-from .brightcove import (
-    BrightcoveLegacyIE,
-    BrightcoveNewIE,
-)
-from .buzzfeed import BuzzFeedIE
-from .byutv import BYUtvIE
-from .c56 import C56IE
-from .camdemy import (
-    CamdemyIE,
-    CamdemyFolderIE
-)
-from .camwithher import CamWithHerIE
-from .canalplus import CanalplusIE
-from .canalc2 import Canalc2IE
-from .canvas import CanvasIE
-from .cbc import (
-    CBCIE,
-    CBCPlayerIE,
-)
-from .cbs import CBSIE
-from .cbsinteractive import CBSInteractiveIE
-from .cbsnews import (
-    CBSNewsIE,
-    CBSNewsLiveVideoIE,
-)
-from .cbssports import CBSSportsIE
-from .ccc import CCCIE
-from .cda import CDAIE
-from .ceskatelevize import CeskaTelevizeIE
-from .channel9 import Channel9IE
-from .chaturbate import ChaturbateIE
-from .chilloutzone import ChilloutzoneIE
-from .chirbit import (
-    ChirbitIE,
-    ChirbitProfileIE,
-)
-from .cinchcast import CinchcastIE
-from .cinemassacre import CinemassacreIE
-from .clipfish import ClipfishIE
-from .cliphunter import CliphunterIE
-from .clipsyndicate import ClipsyndicateIE
-from .cloudy import CloudyIE
-from .clubic import ClubicIE
-from .clyp import ClypIE
-from .cmt import CMTIE
-from .cnbc import CNBCIE
-from .cnn import (
-    CNNIE,
-    CNNBlogsIE,
-    CNNArticleIE,
-)
-from .collegehumor import CollegeHumorIE
-from .collegerama import CollegeRamaIE
-from .comedycentral import ComedyCentralIE, ComedyCentralShowsIE
-from .comcarcoff import ComCarCoffIE
-from .commonmistakes import CommonMistakesIE, UnicodeBOMIE
-from .commonprotocols import RtmpIE
-from .condenast import CondeNastIE
-from .cracked import CrackedIE
-from .crackle import CrackleIE
-from .criterion import CriterionIE
-from .crooksandliars import CrooksAndLiarsIE
-from .crunchyroll import (
-    CrunchyrollIE,
-    CrunchyrollShowPlaylistIE
-)
-from .cspan import CSpanIE
-from .ctsnews import CtsNewsIE
-from .cultureunplugged import CultureUnpluggedIE
-from .cwtv import CWTVIE
-from .dailymotion import (
-    DailymotionIE,
-    DailymotionPlaylistIE,
-    DailymotionUserIE,
-    DailymotionCloudIE,
-)
-from .daum import (
-    DaumIE,
-    DaumClipIE,
-    DaumPlaylistIE,
-    DaumUserIE,
-)
-from .dbtv import DBTVIE
-from .dcn import (
-    DCNIE,
-    DCNVideoIE,
-    DCNLiveIE,
-    DCNSeasonIE,
-)
-from .dctp import DctpTvIE
-from .deezer import DeezerPlaylistIE
-from .democracynow import DemocracynowIE
-from .dfb import DFBIE
-from .dhm import DHMIE
-from .dotsub import DotsubIE
-from .douyutv import DouyuTVIE
-from .dplay import DPlayIE
-from .dramafever import (
-    DramaFeverIE,
-    DramaFeverSeriesIE,
-)
-from .dreisat import DreiSatIE
-from .drbonanza import DRBonanzaIE
-from .drtuber import DrTuberIE
-from .drtv import DRTVIE
-from .dvtv import DVTVIE
-from .dump import DumpIE
-from .dumpert import DumpertIE
-from .defense import DefenseGouvFrIE
-from .discovery import DiscoveryIE
-from .dropbox import DropboxIE
-from .dw import (
-    DWIE,
-    DWArticleIE,
-)
-from .eagleplatform import EaglePlatformIE
-from .ebaumsworld import EbaumsWorldIE
-from .echomsk import EchoMskIE
-from .ehow import EHowIE
-from .eighttracks import EightTracksIE
-from .einthusan import EinthusanIE
-from .eitb import EitbIE
-from .ellentv import (
-    EllenTVIE,
-    EllenTVClipsIE,
-)
-from .elpais import ElPaisIE
-from .embedly import EmbedlyIE
-from .engadget import EngadgetIE
-from .eporner import EpornerIE
-from .eroprofile import EroProfileIE
-from .escapist import EscapistIE
-from .espn import ESPNIE
-from .esri import EsriVideoIE
-from .europa import EuropaIE
-from .everyonesmixtape import EveryonesMixtapeIE
-from .exfm import ExfmIE
-from .expotv import ExpoTVIE
-from .extremetube import ExtremeTubeIE
-from .facebook import FacebookIE
-from .faz import FazIE
-from .fc2 import FC2IE
-from .fczenit import FczenitIE
-from .firstpost import FirstpostIE
-from .firsttv import FirstTVIE
-from .fivemin import FiveMinIE
-from .fivetv import FiveTVIE
-from .fktv import FKTVIE
-from .flickr import FlickrIE
-from .folketinget import FolketingetIE
-from .footyroom import FootyRoomIE
-from .fourtube import FourTubeIE
-from .fox import FOXIE
-from .foxgay import FoxgayIE
-from .foxnews import FoxNewsIE
-from .foxsports import FoxSportsIE
-from .franceculture import (
-    FranceCultureIE,
-    FranceCultureEmissionIE,
-)
-from .franceinter import FranceInterIE
-from .francetv import (
-    PluzzIE,
-    FranceTvInfoIE,
-    FranceTVIE,
-    GenerationQuoiIE,
-    CultureboxIE,
-)
-from .freesound import FreesoundIE
-from .freespeech import FreespeechIE
-from .freevideo import FreeVideoIE
-from .funimation import FunimationIE
-from .funnyordie import FunnyOrDieIE
-from .gameinformer import GameInformerIE
-from .gamekings import GamekingsIE
-from .gameone import (
-    GameOneIE,
-    GameOnePlaylistIE,
-)
-from .gamersyde import GamersydeIE
-from .gamespot import GameSpotIE
-from .gamestar import GameStarIE
-from .gametrailers import GametrailersIE
-from .gazeta import GazetaIE
-from .gdcvault import GDCVaultIE
-from .generic import GenericIE
-from .gfycat import GfycatIE
-from .giantbomb import GiantBombIE
-from .giga import GigaIE
-from .glide import GlideIE
-from .globo import (
-    GloboIE,
-    GloboArticleIE,
-)
-from .godtube import GodTubeIE
-from .goldenmoustache import GoldenMoustacheIE
-from .golem import GolemIE
-from .googledrive import GoogleDriveIE
-from .googleplus import GooglePlusIE
-from .googlesearch import GoogleSearchIE
-from .goshgay import GoshgayIE
-from .gputechconf import GPUTechConfIE
-from .groupon import GrouponIE
-from .hark import HarkIE
-from .hbo import HBOIE
-from .hearthisat import HearThisAtIE
-from .heise import HeiseIE
-from .hellporno import HellPornoIE
-from .helsinki import HelsinkiIE
-from .hentaistigma import HentaiStigmaIE
-from .historicfilms import HistoricFilmsIE
-from .hitbox import HitboxIE, HitboxLiveIE
-from .hornbunny import HornBunnyIE
-from .hotnewhiphop import HotNewHipHopIE
-from .hotstar import HotStarIE
-from .howcast import HowcastIE
-from .howstuffworks import HowStuffWorksIE
-from .huffpost import HuffPostIE
-from .hypem import HypemIE
-from .iconosquare import IconosquareIE
-from .ign import (
-    IGNIE,
-    OneUPIE,
-    PCMagIE,
-)
-from .imdb import (
-    ImdbIE,
-    ImdbListIE
-)
-from .imgur import (
-    ImgurIE,
-    ImgurAlbumIE,
-)
-from .ina import InaIE
-from .indavideo import (
-    IndavideoIE,
-    IndavideoEmbedIE,
-)
-from .infoq import InfoQIE
-from .instagram import InstagramIE, InstagramUserIE
-from .internetvideoarchive import InternetVideoArchiveIE
-from .iprima import IPrimaIE
-from .iqiyi import IqiyiIE
-from .ir90tv import Ir90TvIE
-from .ivi import (
-    IviIE,
-    IviCompilationIE
-)
-from .ivideon import IvideonIE
-from .izlesene import IzleseneIE
-from .jadorecettepub import JadoreCettePubIE
-from .jeuxvideo import JeuxVideoIE
-from .jove import JoveIE
-from .jwplatform import JWPlatformIE
-from .jpopsukitv import JpopsukiIE
-from .kaltura import KalturaIE
-from .kanalplay import KanalPlayIE
-from .kankan import KankanIE
-from .karaoketv import KaraoketvIE
-from .karrierevideos import KarriereVideosIE
-from .keezmovies import KeezMoviesIE
-from .khanacademy import KhanAcademyIE
-from .kickstarter import KickStarterIE
-from .keek import KeekIE
-from .konserthusetplay import KonserthusetPlayIE
-from .kontrtube import KontrTubeIE
-from .krasview import KrasViewIE
-from .ku6 import Ku6IE
-from .kusi import KUSIIE
-from .kuwo import (
-    KuwoIE,
-    KuwoAlbumIE,
-    KuwoChartIE,
-    KuwoSingerIE,
-    KuwoCategoryIE,
-    KuwoMvIE,
-)
-from .la7 import LA7IE
-from .laola1tv import Laola1TvIE
-from .lecture2go import Lecture2GoIE
-from .lemonde import LemondeIE
-from .leeco import (
-    LeIE,
-    LePlaylistIE,
-    LetvCloudIE,
-)
-from .libsyn import LibsynIE
-from .lifenews import (
-    LifeNewsIE,
-    LifeEmbedIE,
-)
-from .limelight import (
-    LimelightMediaIE,
-    LimelightChannelIE,
-    LimelightChannelListIE,
-)
-from .liveleak import LiveLeakIE
-from .livestream import (
-    LivestreamIE,
-    LivestreamOriginalIE,
-    LivestreamShortenerIE,
-)
-from .lnkgo import LnkGoIE
-from .lovehomeporn import LoveHomePornIE
-from .lrt import LRTIE
-from .lynda import (
-    LyndaIE,
-    LyndaCourseIE
-)
-from .m6 import M6IE
-from .macgamestore import MacGameStoreIE
-from .mailru import MailRuIE
-from .makerschannel import MakersChannelIE
-from .makertv import MakerTVIE
-from .malemotion import MalemotionIE
-from .matchtv import MatchTVIE
-from .mdr import MDRIE
-from .metacafe import MetacafeIE
-from .metacritic import MetacriticIE
-from .mgoon import MgoonIE
-from .minhateca import MinhatecaIE
-from .ministrygrid import MinistryGridIE
-from .minoto import MinotoIE
-from .miomio import MioMioIE
-from .mit import TechTVMITIE, MITIE, OCWMITIE
-from .mitele import MiTeleIE
-from .mixcloud import MixcloudIE
-from .mlb import MLBIE
-from .mnet import MnetIE
-from .mpora import MporaIE
-from .moevideo import MoeVideoIE
-from .mofosex import MofosexIE
-from .mojvideo import MojvideoIE
-from .moniker import MonikerIE
-from .mooshare import MooshareIE
-from .morningstar import MorningstarIE
-from .motherless import MotherlessIE
-from .motorsport import MotorsportIE
-from .movieclips import MovieClipsIE
-from .moviezine import MoviezineIE
-from .mtv import (
-    MTVIE,
-    MTVServicesEmbeddedIE,
-    MTVIggyIE,
-    MTVDEIE,
-)
-from .muenchentv import MuenchenTVIE
-from .musicplayon import MusicPlayOnIE
-from .muzu import MuzuTVIE
-from .mwave import MwaveIE
-from .myspace import MySpaceIE, MySpaceAlbumIE
-from .myspass import MySpassIE
-from .myvi import MyviIE
-from .myvideo import MyVideoIE
-from .myvidster import MyVidsterIE
-from .nationalgeographic import (
-    NationalGeographicIE,
-    NationalGeographicChannelIE,
-)
-from .naver import NaverIE
-from .nba import NBAIE
-from .nbc import (
-    CSNNEIE,
-    NBCIE,
-    NBCNewsIE,
-    NBCSportsIE,
-    NBCSportsVPlayerIE,
-    MSNBCIE,
-)
-from .ndr import (
-    NDRIE,
-    NJoyIE,
-    NDREmbedBaseIE,
-    NDREmbedIE,
-    NJoyEmbedIE,
-)
-from .ndtv import NDTVIE
-from .netzkino import NetzkinoIE
-from .nerdcubed import NerdCubedFeedIE
-from .nerdist import NerdistIE
-from .neteasemusic import (
-    NetEaseMusicIE,
-    NetEaseMusicAlbumIE,
-    NetEaseMusicSingerIE,
-    NetEaseMusicListIE,
-    NetEaseMusicMvIE,
-    NetEaseMusicProgramIE,
-    NetEaseMusicDjRadioIE,
-)
-from .newgrounds import NewgroundsIE
-from .newstube import NewstubeIE
-from .nextmedia import (
-    NextMediaIE,
-    NextMediaActionNewsIE,
-    AppleDailyIE,
-)
-from .nextmovie import NextMovieIE
-from .nfb import NFBIE
-from .nfl import NFLIE
-from .nhl import (
-    NHLIE,
-    NHLNewsIE,
-    NHLVideocenterIE,
-)
-from .nick import NickIE
-from .niconico import NiconicoIE, NiconicoPlaylistIE
-from .ninegag import NineGagIE
-from .noco import NocoIE
-from .normalboots import NormalbootsIE
-from .nosvideo import NosVideoIE
-from .nova import NovaIE
-from .novamov import (
-    AuroraVidIE,
-    CloudTimeIE,
-    NowVideoIE,
-    VideoWeedIE,
-    WholeCloudIE,
-)
-from .nowness import (
-    NownessIE,
-    NownessPlaylistIE,
-    NownessSeriesIE,
-)
-from .nowtv import (
-    NowTVIE,
-    NowTVListIE,
-)
-from .noz import NozIE
-from .npo import (
-    NPOIE,
-    NPOLiveIE,
-    NPORadioIE,
-    NPORadioFragmentIE,
-    SchoolTVIE,
-    VPROIE,
-    WNLIE
-)
-from .npr import NprIE
-from .nrk import (
-    NRKIE,
-    NRKPlaylistIE,
-    NRKSkoleIE,
-    NRKTVIE,
-)
-from .ntvde import NTVDeIE
-from .ntvru import NTVRuIE
-from .nytimes import (
-    NYTimesIE,
-    NYTimesArticleIE,
-)
-from .nuvid import NuvidIE
-from .odnoklassniki import OdnoklassnikiIE
-from .oktoberfesttv import OktoberfestTVIE
-from .onionstudios import OnionStudiosIE
-from .ooyala import (
-    OoyalaIE,
-    OoyalaExternalIE,
-)
-from .openload import OpenloadIE
-from .ora import OraTVIE
-from .orf import (
-    ORFTVthekIE,
-    ORFOE1IE,
-    ORFFM4IE,
-    ORFIPTVIE,
-)
-from .pandoratv import PandoraTVIE
-from .parliamentliveuk import ParliamentLiveUKIE
-from .patreon import PatreonIE
-from .pbs import PBSIE
-from .periscope import PeriscopeIE
-from .philharmoniedeparis import PhilharmonieDeParisIE
-from .phoenix import PhoenixIE
-from .photobucket import PhotobucketIE
-from .pinkbike import PinkbikeIE
-from .planetaplay import PlanetaPlayIE
-from .pladform import PladformIE
-from .played import PlayedIE
-from .playfm import PlayFMIE
-from .plays import PlaysTVIE
-from .playtvak import PlaytvakIE
-from .playvid import PlayvidIE
-from .playwire import PlaywireIE
-from .pluralsight import (
-    PluralsightIE,
-    PluralsightCourseIE,
-)
-from .podomatic import PodomaticIE
-from .porn91 import Porn91IE
-from .pornhd import PornHdIE
-from .pornhub import (
-    PornHubIE,
-    PornHubPlaylistIE,
-    PornHubUserVideosIE,
-)
-from .pornotube import PornotubeIE
-from .pornovoisines import PornoVoisinesIE
-from .pornoxo import PornoXOIE
-from .primesharetv import PrimeShareTVIE
-from .promptfile import PromptFileIE
-from .prosiebensat1 import ProSiebenSat1IE
-from .puls4 import Puls4IE
-from .pyvideo import PyvideoIE
-from .qqmusic import (
-    QQMusicIE,
-    QQMusicSingerIE,
-    QQMusicAlbumIE,
-    QQMusicToplistIE,
-    QQMusicPlaylistIE,
-)
-from .quickvid import QuickVidIE
-from .r7 import R7IE
-from .radiode import RadioDeIE
-from .radiojavan import RadioJavanIE
-from .radiobremen import RadioBremenIE
-from .radiofrance import RadioFranceIE
-from .rai import (
-    RaiTVIE,
-    RaiIE,
-)
-from .rbmaradio import RBMARadioIE
-from .rds import RDSIE
-from .redtube import RedTubeIE
-from .regiotv import RegioTVIE
-from .restudy import RestudyIE
-from .reverbnation import ReverbNationIE
-from .revision3 import Revision3IE
-from .rice import RICEIE
-from .ringtv import RingTVIE
-from .ro220 import Ro220IE
-from .rottentomatoes import RottenTomatoesIE
-from .roxwel import RoxwelIE
-from .rtbf import RTBFIE
-from .rte import RteIE, RteRadioIE
-from .rtlnl import RtlNlIE
-from .rtl2 import RTL2IE
-from .rtp import RTPIE
-from .rts import RTSIE
-from .rtve import RTVEALaCartaIE, RTVELiveIE, RTVEInfantilIE
-from .rtvnh import RTVNHIE
-from .ruhd import RUHDIE
-from .ruleporn import RulePornIE
-from .rutube import (
-    RutubeIE,
-    RutubeChannelIE,
-    RutubeEmbedIE,
-    RutubeMovieIE,
-    RutubePersonIE,
-)
-from .rutv import RUTVIE
-from .ruutu import RuutuIE
-from .sandia import SandiaIE
-from .safari import (
-    SafariIE,
-    SafariApiIE,
-    SafariCourseIE,
-)
-from .sapo import SapoIE
-from .savefrom import SaveFromIE
-from .sbs import SBSIE
-from .scivee import SciVeeIE
-from .screencast import ScreencastIE
-from .screencastomatic import ScreencastOMaticIE
-from .screenjunkies import ScreenJunkiesIE
-from .screenwavemedia import ScreenwaveMediaIE, TeamFourIE
-from .senateisvp import SenateISVPIE
-from .servingsys import ServingSysIE
-from .sexu import SexuIE
-from .sexykarma import SexyKarmaIE
-from .shahid import ShahidIE
-from .shared import SharedIE
-from .sharesix import ShareSixIE
-from .sina import SinaIE
-from .skynewsarabia import (
-    SkyNewsArabiaIE,
-    SkyNewsArabiaArticleIE,
-)
-from .slideshare import SlideshareIE
-from .slutload import SlutloadIE
-from .smotri import (
-    SmotriIE,
-    SmotriCommunityIE,
-    SmotriUserIE,
-    SmotriBroadcastIE,
-)
-from .snagfilms import (
-    SnagFilmsIE,
-    SnagFilmsEmbedIE,
-)
-from .snotr import SnotrIE
-from .sohu import SohuIE
-from .soundcloud import (
-    SoundcloudIE,
-    SoundcloudSetIE,
-    SoundcloudUserIE,
-    SoundcloudPlaylistIE,
-    SoundcloudSearchIE
-)
-from .soundgasm import (
-    SoundgasmIE,
-    SoundgasmProfileIE
-)
-from .southpark import (
-    SouthParkIE,
-    SouthParkDeIE,
-    SouthParkDkIE,
-    SouthParkEsIE,
-    SouthParkNlIE
-)
-from .spankbang import SpankBangIE
-from .spankwire import SpankwireIE
-from .spiegel import SpiegelIE, SpiegelArticleIE
-from .spiegeltv import SpiegeltvIE
-from .spike import SpikeIE
-from .stitcher import StitcherIE
-from .sport5 import Sport5IE
-from .sportbox import (
-    SportBoxIE,
-    SportBoxEmbedIE,
-)
-from .sportdeutschland import SportDeutschlandIE
-from .srgssr import (
-    SRGSSRIE,
-    SRGSSRPlayIE,
-)
-from .srmediathek import SRMediathekIE
-from .ssa import SSAIE
-from .stanfordoc import StanfordOpenClassroomIE
-from .steam import SteamIE
-from .streamcloud import StreamcloudIE
-from .streamcz import StreamCZIE
-from .streetvoice import StreetVoiceIE
-from .sunporno import SunPornoIE
-from .svt import (
-    SVTIE,
-    SVTPlayIE,
-)
-from .swrmediathek import SWRMediathekIE
-from .syfy import SyfyIE
-from .sztvhu import SztvHuIE
-from .tagesschau import TagesschauIE
-from .tapely import TapelyIE
-from .tass import TassIE
-from .teachertube import (
-    TeacherTubeIE,
-    TeacherTubeUserIE,
-)
-from .teachingchannel import TeachingChannelIE
-from .teamcoco import TeamcocoIE
-from .techtalks import TechTalksIE
-from .ted import TEDIE
-from .tele13 import Tele13IE
-from .telebruxelles import TeleBruxellesIE
-from .telecinco import TelecincoIE
-from .telegraaf import TelegraafIE
-from .telemb import TeleMBIE
-from .teletask import TeleTaskIE
-from .testurl import TestURLIE
-from .tf1 import TF1IE
-from .theintercept import TheInterceptIE
-from .theonion import TheOnionIE
-from .theplatform import (
-    ThePlatformIE,
-    ThePlatformFeedIE,
-)
-from .thescene import TheSceneIE
-from .thesixtyone import TheSixtyOneIE
-from .thestar import TheStarIE
-from .thisamericanlife import ThisAmericanLifeIE
-from .thisav import ThisAVIE
-from .tinypic import TinyPicIE
-from .tlc import TlcDeIE
-from .tmz import (
-    TMZIE,
-    TMZArticleIE,
-)
-from .tnaflix import (
-    TNAFlixNetworkEmbedIE,
-    TNAFlixIE,
-    EMPFlixIE,
-    MovieFapIE,
-)
-from .toggle import ToggleIE
-from .thvideo import (
-    THVideoIE,
-    THVideoPlaylistIE
-)
-from .toutv import TouTvIE
-from .toypics import ToypicsUserIE, ToypicsIE
-from .traileraddict import TrailerAddictIE
-from .trilulilu import TriluliluIE
-from .trollvids import TrollvidsIE
-from .trutube import TruTubeIE
-from .tube8 import Tube8IE
-from .tubitv import TubiTvIE
-from .tudou import (
-    TudouIE,
-    TudouPlaylistIE,
-    TudouAlbumIE,
-)
-from .tumblr import TumblrIE
-from .tunein import (
-    TuneInClipIE,
-    TuneInStationIE,
-    TuneInProgramIE,
-    TuneInTopicIE,
-    TuneInShortenerIE,
-)
-from .turbo import TurboIE
-from .tutv import TutvIE
-from .tv2 import (
-    TV2IE,
-    TV2ArticleIE,
-)
-from .tv3 import TV3IE
-from .tv4 import TV4IE
-from .tvc import (
-    TVCIE,
-    TVCArticleIE,
-)
-from .tvigle import TvigleIE
-from .tvland import TVLandIE
-from .tvp import TvpIE, TvpSeriesIE
-from .tvplay import TVPlayIE
-from .tweakers import TweakersIE
-from .twentyfourvideo import TwentyFourVideoIE
-from .twentymin import TwentyMinutenIE
-from .twentytwotracks import (
-    TwentyTwoTracksIE,
-    TwentyTwoTracksGenreIE
-)
-from .twitch import (
-    TwitchVideoIE,
-    TwitchChapterIE,
-    TwitchVodIE,
-    TwitchProfileIE,
-    TwitchPastBroadcastsIE,
-    TwitchBookmarksIE,
-    TwitchStreamIE,
-)
-from .twitter import (
-    TwitterCardIE,
-    TwitterIE,
-    TwitterAmplifyIE,
-)
-from .ubu import UbuIE
-from .udemy import (
-    UdemyIE,
-    UdemyCourseIE
-)
-from .udn import UDNEmbedIE
-from .digiteka import DigitekaIE
-from .unistra import UnistraIE
-from .urort import UrortIE
-from .usatoday import USATodayIE
-from .ustream import UstreamIE, UstreamChannelIE
-from .ustudio import UstudioIE
-from .varzesh3 import Varzesh3IE
-from .vbox7 import Vbox7IE
-from .veehd import VeeHDIE
-from .veoh import VeohIE
-from .vessel import VesselIE
-from .vesti import VestiIE
-from .vevo import VevoIE
-from .vgtv import (
-    BTArticleIE,
-    BTVestlendingenIE,
-    VGTVIE,
-)
-from .vh1 import VH1IE
-from .vice import (
-    ViceIE,
-    ViceShowIE,
-)
-from .viddler import ViddlerIE
-from .videodetective import VideoDetectiveIE
-from .videofyme import VideofyMeIE
-from .videomega import VideoMegaIE
-from .videomore import (
-    VideomoreIE,
-    VideomoreVideoIE,
-    VideomoreSeasonIE,
-)
-from .videopremium import VideoPremiumIE
-from .videott import VideoTtIE
-from .vidme import (
-    VidmeIE,
-    VidmeUserIE,
-    VidmeUserLikesIE,
-)
-from .vidzi import VidziIE
-from .vier import VierIE, VierVideosIE
-from .viewster import ViewsterIE
-from .viidea import ViideaIE
-from .vimeo import (
-    VimeoIE,
-    VimeoAlbumIE,
-    VimeoChannelIE,
-    VimeoGroupsIE,
-    VimeoLikesIE,
-    VimeoOndemandIE,
-    VimeoReviewIE,
-    VimeoUserIE,
-    VimeoWatchLaterIE,
-)
-from .vimple import VimpleIE
-from .vine import (
-    VineIE,
-    VineUserIE,
-)
-from .viki import (
-    VikiIE,
-    VikiChannelIE,
-)
-from .vk import (
-    VKIE,
-    VKUserVideosIE,
-)
-from .vlive import VLiveIE
-from .vodlocker import VodlockerIE
-from .voicerepublic import VoiceRepublicIE
-from .voxmedia import VoxMediaIE
-from .vporn import VpornIE
-from .vrt import VRTIE
-from .vube import VubeIE
-from .vuclip import VuClipIE
-from .vulture import VultureIE
-from .walla import WallaIE
-from .washingtonpost import WashingtonPostIE
-from .wat import WatIE
-from .wayofthemaster import WayOfTheMasterIE
-from .wdr import (
-    WDRIE,
-    WDRMobileIE,
-    WDRMausIE,
-)
-from .webofstories import (
-    WebOfStoriesIE,
-    WebOfStoriesPlaylistIE,
-)
-from .weibo import WeiboIE
-from .weiqitv import WeiqiTVIE
-from .wimp import WimpIE
-from .wistia import WistiaIE
-from .worldstarhiphop import WorldStarHipHopIE
-from .wrzuta import WrzutaIE
-from .wsj import WSJIE
-from .xbef import XBefIE
-from .xboxclips import XboxClipsIE
-from .xfileshare import XFileShareIE
-from .xhamster import (
-    XHamsterIE,
-    XHamsterEmbedIE,
-)
-from .xminus import XMinusIE
-from .xnxx import XNXXIE
-from .xstream import XstreamIE
-from .xtube import XTubeUserIE, XTubeIE
-from .xuite import XuiteIE
-from .xvideos import XVideosIE
-from .xxxymovies import XXXYMoviesIE
-from .yahoo import (
-    YahooIE,
-    YahooSearchIE,
-)
-from .yam import YamIE
-from .yandexmusic import (
-    YandexMusicTrackIE,
-    YandexMusicAlbumIE,
-    YandexMusicPlaylistIE,
-)
-from .yesjapan import YesJapanIE
-from .yinyuetai import YinYueTaiIE
-from .ynet import YnetIE
-from .youjizz import YouJizzIE
-from .youku import YoukuIE
-from .youporn import YouPornIE
-from .yourupload import YourUploadIE
-from .youtube import (
-    YoutubeIE,
-    YoutubeChannelIE,
-    YoutubeFavouritesIE,
-    YoutubeHistoryIE,
-    YoutubeLiveIE,
-    YoutubePlaylistIE,
-    YoutubePlaylistsIE,
-    YoutubeRecommendedIE,
-    YoutubeSearchDateIE,
-    YoutubeSearchIE,
-    YoutubeSearchURLIE,
-    YoutubeShowIE,
-    YoutubeSubscriptionsIE,
-    YoutubeTruncatedIDIE,
-    YoutubeTruncatedURLIE,
-    YoutubeUserIE,
-    YoutubeWatchLaterIE,
-)
-from .zapiks import ZapiksIE
-from .zdf import ZDFIE, ZDFChannelIE
-from .zingmp3 import (
-    ZingMp3SongIE,
-    ZingMp3AlbumIE,
-)
-from .zippcast import ZippCastIE
+from .extractors import *
 
 _ALL_CLASSES = [
     klass
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
new file mode 100644
index 000000000..de29c7956
--- /dev/null
+++ b/youtube_dl/extractor/extractors.py
@@ -0,0 +1,991 @@
+# flake8: noqa
+from __future__ import unicode_literals
+
+from .abc import ABCIE
+from .abc7news import Abc7NewsIE
+from .academicearth import AcademicEarthCourseIE
+from .acast import (
+    ACastIE,
+    ACastChannelIE,
+)
+from .addanime import AddAnimeIE
+from .adobetv import (
+    AdobeTVIE,
+    AdobeTVShowIE,
+    AdobeTVChannelIE,
+    AdobeTVVideoIE,
+)
+from .adultswim import AdultSwimIE
+from .aenetworks import AENetworksIE
+from .aftonbladet import AftonbladetIE
+from .airmozilla import AirMozillaIE
+from .aljazeera import AlJazeeraIE
+from .alphaporno import AlphaPornoIE
+from .animeondemand import AnimeOnDemandIE
+from .anitube import AnitubeIE
+from .anysex import AnySexIE
+from .aol import (
+    AolIE,
+    AolFeaturesIE,
+)
+from .allocine import AllocineIE
+from .aparat import AparatIE
+from .appleconnect import AppleConnectIE
+from .appletrailers import (
+    AppleTrailersIE,
+    AppleTrailersSectionIE,
+)
+from .archiveorg import ArchiveOrgIE
+from .ard import (
+    ARDIE,
+    ARDMediathekIE,
+    SportschauIE,
+)
+from .arte import (
+    ArteTvIE,
+    ArteTVPlus7IE,
+    ArteTVCreativeIE,
+    ArteTVConcertIE,
+    ArteTVFutureIE,
+    ArteTVCinemaIE,
+    ArteTVDDCIE,
+    ArteTVMagazineIE,
+    ArteTVEmbedIE,
+)
+from .atresplayer import AtresPlayerIE
+from .atttechchannel import ATTTechChannelIE
+from .audimedia import AudiMediaIE
+from .audioboom import AudioBoomIE
+from .audiomack import AudiomackIE, AudiomackAlbumIE
+from .azubu import AzubuIE, AzubuLiveIE
+from .baidu import BaiduVideoIE
+from .bambuser import BambuserIE, BambuserChannelIE
+from .bandcamp import BandcampIE, BandcampAlbumIE
+from .bbc import (
+    BBCCoUkIE,
+    BBCCoUkArticleIE,
+    BBCIE,
+)
+from .beeg import BeegIE
+from .behindkink import BehindKinkIE
+from .beatportpro import BeatportProIE
+from .bet import BetIE
+from .bigflix import BigflixIE
+from .bild import BildIE
+from .bilibili import BiliBiliIE
+from .biobiochiletv import BioBioChileTVIE
+from .bleacherreport import (
+    BleacherReportIE,
+    BleacherReportCMSIE,
+)
+from .blinkx import BlinkxIE
+from .bloomberg import BloombergIE
+from .bokecc import BokeCCIE
+from .bpb import BpbIE
+from .br import BRIE
+from .bravotv import BravoTVIE
+from .breakcom import BreakIE
+from .brightcove import (
+    BrightcoveLegacyIE,
+    BrightcoveNewIE,
+)
+from .buzzfeed import BuzzFeedIE
+from .byutv import BYUtvIE
+from .c56 import C56IE
+from .camdemy import (
+    CamdemyIE,
+    CamdemyFolderIE
+)
+from .camwithher import CamWithHerIE
+from .canalplus import CanalplusIE
+from .canalc2 import Canalc2IE
+from .canvas import CanvasIE
+from .cbc import (
+    CBCIE,
+    CBCPlayerIE,
+)
+from .cbs import CBSIE
+from .cbsinteractive import CBSInteractiveIE
+from .cbsnews import (
+    CBSNewsIE,
+    CBSNewsLiveVideoIE,
+)
+from .cbssports import CBSSportsIE
+from .ccc import CCCIE
+from .cda import CDAIE
+from .ceskatelevize import CeskaTelevizeIE
+from .channel9 import Channel9IE
+from .chaturbate import ChaturbateIE
+from .chilloutzone import ChilloutzoneIE
+from .chirbit import (
+    ChirbitIE,
+    ChirbitProfileIE,
+)
+from .cinchcast import CinchcastIE
+from .cinemassacre import CinemassacreIE
+from .clipfish import ClipfishIE
+from .cliphunter import CliphunterIE
+from .clipsyndicate import ClipsyndicateIE
+from .cloudy import CloudyIE
+from .clubic import ClubicIE
+from .clyp import ClypIE
+from .cmt import CMTIE
+from .cnbc import CNBCIE
+from .cnn import (
+    CNNIE,
+    CNNBlogsIE,
+    CNNArticleIE,
+)
+from .collegehumor import CollegeHumorIE
+from .collegerama import CollegeRamaIE
+from .comedycentral import ComedyCentralIE, ComedyCentralShowsIE
+from .comcarcoff import ComCarCoffIE
+from .commonmistakes import CommonMistakesIE, UnicodeBOMIE
+from .commonprotocols import RtmpIE
+from .condenast import CondeNastIE
+from .cracked import CrackedIE
+from .crackle import CrackleIE
+from .criterion import CriterionIE
+from .crooksandliars import CrooksAndLiarsIE
+from .crunchyroll import (
+    CrunchyrollIE,
+    CrunchyrollShowPlaylistIE
+)
+from .cspan import CSpanIE
+from .ctsnews import CtsNewsIE
+from .cultureunplugged import CultureUnpluggedIE
+from .cwtv import CWTVIE
+from .dailymotion import (
+    DailymotionIE,
+    DailymotionPlaylistIE,
+    DailymotionUserIE,
+    DailymotionCloudIE,
+)
+from .daum import (
+    DaumIE,
+    DaumClipIE,
+    DaumPlaylistIE,
+    DaumUserIE,
+)
+from .dbtv import DBTVIE
+from .dcn import (
+    DCNIE,
+    DCNVideoIE,
+    DCNLiveIE,
+    DCNSeasonIE,
+)
+from .dctp import DctpTvIE
+from .deezer import DeezerPlaylistIE
+from .democracynow import DemocracynowIE
+from .dfb import DFBIE
+from .dhm import DHMIE
+from .dotsub import DotsubIE
+from .douyutv import DouyuTVIE
+from .dplay import DPlayIE
+from .dramafever import (
+    DramaFeverIE,
+    DramaFeverSeriesIE,
+)
+from .dreisat import DreiSatIE
+from .drbonanza import DRBonanzaIE
+from .drtuber import DrTuberIE
+from .drtv import DRTVIE
+from .dvtv import DVTVIE
+from .dump import DumpIE
+from .dumpert import DumpertIE
+from .defense import DefenseGouvFrIE
+from .discovery import DiscoveryIE
+from .dropbox import DropboxIE
+from .dw import (
+    DWIE,
+    DWArticleIE,
+)
+from .eagleplatform import EaglePlatformIE
+from .ebaumsworld import EbaumsWorldIE
+from .echomsk import EchoMskIE
+from .ehow import EHowIE
+from .eighttracks import EightTracksIE
+from .einthusan import EinthusanIE
+from .eitb import EitbIE
+from .ellentv import (
+    EllenTVIE,
+    EllenTVClipsIE,
+)
+from .elpais import ElPaisIE
+from .embedly import EmbedlyIE
+from .engadget import EngadgetIE
+from .eporner import EpornerIE
+from .eroprofile import EroProfileIE
+from .escapist import EscapistIE
+from .espn import ESPNIE
+from .esri import EsriVideoIE
+from .europa import EuropaIE
+from .everyonesmixtape import EveryonesMixtapeIE
+from .exfm import ExfmIE
+from .expotv import ExpoTVIE
+from .extremetube import ExtremeTubeIE
+from .facebook import FacebookIE
+from .faz import FazIE
+from .fc2 import FC2IE
+from .fczenit import FczenitIE
+from .firstpost import FirstpostIE
+from .firsttv import FirstTVIE
+from .fivemin import FiveMinIE
+from .fivetv import FiveTVIE
+from .fktv import FKTVIE
+from .flickr import FlickrIE
+from .folketinget import FolketingetIE
+from .footyroom import FootyRoomIE
+from .fourtube import FourTubeIE
+from .fox import FOXIE
+from .foxgay import FoxgayIE
+from .foxnews import FoxNewsIE
+from .foxsports import FoxSportsIE
+from .franceculture import (
+    FranceCultureIE,
+    FranceCultureEmissionIE,
+)
+from .franceinter import FranceInterIE
+from .francetv import (
+    PluzzIE,
+    FranceTvInfoIE,
+    FranceTVIE,
+    GenerationQuoiIE,
+    CultureboxIE,
+)
+from .freesound import FreesoundIE
+from .freespeech import FreespeechIE
+from .freevideo import FreeVideoIE
+from .funimation import FunimationIE
+from .funnyordie import FunnyOrDieIE
+from .gameinformer import GameInformerIE
+from .gamekings import GamekingsIE
+from .gameone import (
+    GameOneIE,
+    GameOnePlaylistIE,
+)
+from .gamersyde import GamersydeIE
+from .gamespot import GameSpotIE
+from .gamestar import GameStarIE
+from .gametrailers import GametrailersIE
+from .gazeta import GazetaIE
+from .gdcvault import GDCVaultIE
+from .generic import GenericIE
+from .gfycat import GfycatIE
+from .giantbomb import GiantBombIE
+from .giga import GigaIE
+from .glide import GlideIE
+from .globo import (
+    GloboIE,
+    GloboArticleIE,
+)
+from .godtube import GodTubeIE
+from .goldenmoustache import GoldenMoustacheIE
+from .golem import GolemIE
+from .googledrive import GoogleDriveIE
+from .googleplus import GooglePlusIE
+from .googlesearch import GoogleSearchIE
+from .goshgay import GoshgayIE
+from .gputechconf import GPUTechConfIE
+from .groupon import GrouponIE
+from .hark import HarkIE
+from .hbo import HBOIE
+from .hearthisat import HearThisAtIE
+from .heise import HeiseIE
+from .hellporno import HellPornoIE
+from .helsinki import HelsinkiIE
+from .hentaistigma import HentaiStigmaIE
+from .historicfilms import HistoricFilmsIE
+from .hitbox import HitboxIE, HitboxLiveIE
+from .hornbunny import HornBunnyIE
+from .hotnewhiphop import HotNewHipHopIE
+from .hotstar import HotStarIE
+from .howcast import HowcastIE
+from .howstuffworks import HowStuffWorksIE
+from .huffpost import HuffPostIE
+from .hypem import HypemIE
+from .iconosquare import IconosquareIE
+from .ign import (
+    IGNIE,
+    OneUPIE,
+    PCMagIE,
+)
+from .imdb import (
+    ImdbIE,
+    ImdbListIE
+)
+from .imgur import (
+    ImgurIE,
+    ImgurAlbumIE,
+)
+from .ina import InaIE
+from .indavideo import (
+    IndavideoIE,
+    IndavideoEmbedIE,
+)
+from .infoq import InfoQIE
+from .instagram import InstagramIE, InstagramUserIE
+from .internetvideoarchive import InternetVideoArchiveIE
+from .iprima import IPrimaIE
+from .iqiyi import IqiyiIE
+from .ir90tv import Ir90TvIE
+from .ivi import (
+    IviIE,
+    IviCompilationIE
+)
+from .ivideon import IvideonIE
+from .izlesene import IzleseneIE
+from .jadorecettepub import JadoreCettePubIE
+from .jeuxvideo import JeuxVideoIE
+from .jove import JoveIE
+from .jwplatform import JWPlatformIE
+from .jpopsukitv import JpopsukiIE
+from .kaltura import KalturaIE
+from .kanalplay import KanalPlayIE
+from .kankan import KankanIE
+from .karaoketv import KaraoketvIE
+from .karrierevideos import KarriereVideosIE
+from .keezmovies import KeezMoviesIE
+from .khanacademy import KhanAcademyIE
+from .kickstarter import KickStarterIE
+from .keek import KeekIE
+from .konserthusetplay import KonserthusetPlayIE
+from .kontrtube import KontrTubeIE
+from .krasview import KrasViewIE
+from .ku6 import Ku6IE
+from .kusi import KUSIIE
+from .kuwo import (
+    KuwoIE,
+    KuwoAlbumIE,
+    KuwoChartIE,
+    KuwoSingerIE,
+    KuwoCategoryIE,
+    KuwoMvIE,
+)
+from .la7 import LA7IE
+from .laola1tv import Laola1TvIE
+from .lecture2go import Lecture2GoIE
+from .lemonde import LemondeIE
+from .leeco import (
+    LeIE,
+    LePlaylistIE,
+    LetvCloudIE,
+)
+from .libsyn import LibsynIE
+from .lifenews import (
+    LifeNewsIE,
+    LifeEmbedIE,
+)
+from .limelight import (
+    LimelightMediaIE,
+    LimelightChannelIE,
+    LimelightChannelListIE,
+)
+from .liveleak import LiveLeakIE
+from .livestream import (
+    LivestreamIE,
+    LivestreamOriginalIE,
+    LivestreamShortenerIE,
+)
+from .lnkgo import LnkGoIE
+from .lovehomeporn import LoveHomePornIE
+from .lrt import LRTIE
+from .lynda import (
+    LyndaIE,
+    LyndaCourseIE
+)
+from .m6 import M6IE
+from .macgamestore import MacGameStoreIE
+from .mailru import MailRuIE
+from .makerschannel import MakersChannelIE
+from .makertv import MakerTVIE
+from .malemotion import MalemotionIE
+from .matchtv import MatchTVIE
+from .mdr import MDRIE
+from .metacafe import MetacafeIE
+from .metacritic import MetacriticIE
+from .mgoon import MgoonIE
+from .minhateca import MinhatecaIE
+from .ministrygrid import MinistryGridIE
+from .minoto import MinotoIE
+from .miomio import MioMioIE
+from .mit import TechTVMITIE, MITIE, OCWMITIE
+from .mitele import MiTeleIE
+from .mixcloud import MixcloudIE
+from .mlb import MLBIE
+from .mnet import MnetIE
+from .mpora import MporaIE
+from .moevideo import MoeVideoIE
+from .mofosex import MofosexIE
+from .mojvideo import MojvideoIE
+from .moniker import MonikerIE
+from .mooshare import MooshareIE
+from .morningstar import MorningstarIE
+from .motherless import MotherlessIE
+from .motorsport import MotorsportIE
+from .movieclips import MovieClipsIE
+from .moviezine import MoviezineIE
+from .mtv import (
+    MTVIE,
+    MTVServicesEmbeddedIE,
+    MTVIggyIE,
+    MTVDEIE,
+)
+from .muenchentv import MuenchenTVIE
+from .musicplayon import MusicPlayOnIE
+from .muzu import MuzuTVIE
+from .mwave import MwaveIE
+from .myspace import MySpaceIE, MySpaceAlbumIE
+from .myspass import MySpassIE
+from .myvi import MyviIE
+from .myvideo import MyVideoIE
+from .myvidster import MyVidsterIE
+from .nationalgeographic import (
+    NationalGeographicIE,
+    NationalGeographicChannelIE,
+)
+from .naver import NaverIE
+from .nba import NBAIE
+from .nbc import (
+    CSNNEIE,
+    NBCIE,
+    NBCNewsIE,
+    NBCSportsIE,
+    NBCSportsVPlayerIE,
+    MSNBCIE,
+)
+from .ndr import (
+    NDRIE,
+    NJoyIE,
+    NDREmbedBaseIE,
+    NDREmbedIE,
+    NJoyEmbedIE,
+)
+from .ndtv import NDTVIE
+from .netzkino import NetzkinoIE
+from .nerdcubed import NerdCubedFeedIE
+from .nerdist import NerdistIE
+from .neteasemusic import (
+    NetEaseMusicIE,
+    NetEaseMusicAlbumIE,
+    NetEaseMusicSingerIE,
+    NetEaseMusicListIE,
+    NetEaseMusicMvIE,
+    NetEaseMusicProgramIE,
+    NetEaseMusicDjRadioIE,
+)
+from .newgrounds import NewgroundsIE
+from .newstube import NewstubeIE
+from .nextmedia import (
+    NextMediaIE,
+    NextMediaActionNewsIE,
+    AppleDailyIE,
+)
+from .nextmovie import NextMovieIE
+from .nfb import NFBIE
+from .nfl import NFLIE
+from .nhl import (
+    NHLIE,
+    NHLNewsIE,
+    NHLVideocenterIE,
+)
+from .nick import NickIE
+from .niconico import NiconicoIE, NiconicoPlaylistIE
+from .ninegag import NineGagIE
+from .noco import NocoIE
+from .normalboots import NormalbootsIE
+from .nosvideo import NosVideoIE
+from .nova import NovaIE
+from .novamov import (
+    AuroraVidIE,
+    CloudTimeIE,
+    NowVideoIE,
+    VideoWeedIE,
+    WholeCloudIE,
+)
+from .nowness import (
+    NownessIE,
+    NownessPlaylistIE,
+    NownessSeriesIE,
+)
+from .nowtv import (
+    NowTVIE,
+    NowTVListIE,
+)
+from .noz import NozIE
+from .npo import (
+    NPOIE,
+    NPOLiveIE,
+    NPORadioIE,
+    NPORadioFragmentIE,
+    SchoolTVIE,
+    VPROIE,
+    WNLIE
+)
+from .npr import NprIE
+from .nrk import (
+    NRKIE,
+    NRKPlaylistIE,
+    NRKSkoleIE,
+    NRKTVIE,
+)
+from .ntvde import NTVDeIE
+from .ntvru import NTVRuIE
+from .nytimes import (
+    NYTimesIE,
+    NYTimesArticleIE,
+)
+from .nuvid import NuvidIE
+from .odnoklassniki import OdnoklassnikiIE
+from .oktoberfesttv import OktoberfestTVIE
+from .onionstudios import OnionStudiosIE
+from .ooyala import (
+    OoyalaIE,
+    OoyalaExternalIE,
+)
+from .openload import OpenloadIE
+from .ora import OraTVIE
+from .orf import (
+    ORFTVthekIE,
+    ORFOE1IE,
+    ORFFM4IE,
+    ORFIPTVIE,
+)
+from .pandoratv import PandoraTVIE
+from .parliamentliveuk import ParliamentLiveUKIE
+from .patreon import PatreonIE
+from .pbs import PBSIE
+from .periscope import PeriscopeIE
+from .philharmoniedeparis import PhilharmonieDeParisIE
+from .phoenix import PhoenixIE
+from .photobucket import PhotobucketIE
+from .pinkbike import PinkbikeIE
+from .planetaplay import PlanetaPlayIE
+from .pladform import PladformIE
+from .played import PlayedIE
+from .playfm import PlayFMIE
+from .plays import PlaysTVIE
+from .playtvak import PlaytvakIE
+from .playvid import PlayvidIE
+from .playwire import PlaywireIE
+from .pluralsight import (
+    PluralsightIE,
+    PluralsightCourseIE,
+)
+from .podomatic import PodomaticIE
+from .porn91 import Porn91IE
+from .pornhd import PornHdIE
+from .pornhub import (
+    PornHubIE,
+    PornHubPlaylistIE,
+    PornHubUserVideosIE,
+)
+from .pornotube import PornotubeIE
+from .pornovoisines import PornoVoisinesIE
+from .pornoxo import PornoXOIE
+from .primesharetv import PrimeShareTVIE
+from .promptfile import PromptFileIE
+from .prosiebensat1 import ProSiebenSat1IE
+from .puls4 import Puls4IE
+from .pyvideo import PyvideoIE
+from .qqmusic import (
+    QQMusicIE,
+    QQMusicSingerIE,
+    QQMusicAlbumIE,
+    QQMusicToplistIE,
+    QQMusicPlaylistIE,
+)
+from .quickvid import QuickVidIE
+from .r7 import R7IE
+from .radiode import RadioDeIE
+from .radiojavan import RadioJavanIE
+from .radiobremen import RadioBremenIE
+from .radiofrance import RadioFranceIE
+from .rai import (
+    RaiTVIE,
+    RaiIE,
+)
+from .rbmaradio import RBMARadioIE
+from .rds import RDSIE
+from .redtube import RedTubeIE
+from .regiotv import RegioTVIE
+from .restudy import RestudyIE
+from .reverbnation import ReverbNationIE
+from .revision3 import Revision3IE
+from .rice import RICEIE
+from .ringtv import RingTVIE
+from .ro220 import Ro220IE
+from .rottentomatoes import RottenTomatoesIE
+from .roxwel import RoxwelIE
+from .rtbf import RTBFIE
+from .rte import RteIE, RteRadioIE
+from .rtlnl import RtlNlIE
+from .rtl2 import RTL2IE
+from .rtp import RTPIE
+from .rts import RTSIE
+from .rtve import RTVEALaCartaIE, RTVELiveIE, RTVEInfantilIE
+from .rtvnh import RTVNHIE
+from .ruhd import RUHDIE
+from .ruleporn import RulePornIE
+from .rutube import (
+    RutubeIE,
+    RutubeChannelIE,
+    RutubeEmbedIE,
+    RutubeMovieIE,
+    RutubePersonIE,
+)
+from .rutv import RUTVIE
+from .ruutu import RuutuIE
+from .sandia import SandiaIE
+from .safari import (
+    SafariIE,
+    SafariApiIE,
+    SafariCourseIE,
+)
+from .sapo import SapoIE
+from .savefrom import SaveFromIE
+from .sbs import SBSIE
+from .scivee import SciVeeIE
+from .screencast import ScreencastIE
+from .screencastomatic import ScreencastOMaticIE
+from .screenjunkies import ScreenJunkiesIE
+from .screenwavemedia import ScreenwaveMediaIE, TeamFourIE
+from .senateisvp import SenateISVPIE
+from .servingsys import ServingSysIE
+from .sexu import SexuIE
+from .sexykarma import SexyKarmaIE
+from .shahid import ShahidIE
+from .shared import SharedIE
+from .sharesix import ShareSixIE
+from .sina import SinaIE
+from .skynewsarabia import (
+    SkyNewsArabiaIE,
+    SkyNewsArabiaArticleIE,
+)
+from .slideshare import SlideshareIE
+from .slutload import SlutloadIE
+from .smotri import (
+    SmotriIE,
+    SmotriCommunityIE,
+    SmotriUserIE,
+    SmotriBroadcastIE,
+)
+from .snagfilms import (
+    SnagFilmsIE,
+    SnagFilmsEmbedIE,
+)
+from .snotr import SnotrIE
+from .sohu import SohuIE
+from .soundcloud import (
+    SoundcloudIE,
+    SoundcloudSetIE,
+    SoundcloudUserIE,
+    SoundcloudPlaylistIE,
+    SoundcloudSearchIE
+)
+from .soundgasm import (
+    SoundgasmIE,
+    SoundgasmProfileIE
+)
+from .southpark import (
+    SouthParkIE,
+    SouthParkDeIE,
+    SouthParkDkIE,
+    SouthParkEsIE,
+    SouthParkNlIE
+)
+from .spankbang import SpankBangIE
+from .spankwire import SpankwireIE
+from .spiegel import SpiegelIE, SpiegelArticleIE
+from .spiegeltv import SpiegeltvIE
+from .spike import SpikeIE
+from .stitcher import StitcherIE
+from .sport5 import Sport5IE
+from .sportbox import (
+    SportBoxIE,
+    SportBoxEmbedIE,
+)
+from .sportdeutschland import SportDeutschlandIE
+from .srgssr import (
+    SRGSSRIE,
+    SRGSSRPlayIE,
+)
+from .srmediathek import SRMediathekIE
+from .ssa import SSAIE
+from .stanfordoc import StanfordOpenClassroomIE
+from .steam import SteamIE
+from .streamcloud import StreamcloudIE
+from .streamcz import StreamCZIE
+from .streetvoice import StreetVoiceIE
+from .sunporno import SunPornoIE
+from .svt import (
+    SVTIE,
+    SVTPlayIE,
+)
+from .swrmediathek import SWRMediathekIE
+from .syfy import SyfyIE
+from .sztvhu import SztvHuIE
+from .tagesschau import TagesschauIE
+from .tapely import TapelyIE
+from .tass import TassIE
+from .teachertube import (
+    TeacherTubeIE,
+    TeacherTubeUserIE,
+)
+from .teachingchannel import TeachingChannelIE
+from .teamcoco import TeamcocoIE
+from .techtalks import TechTalksIE
+from .ted import TEDIE
+from .tele13 import Tele13IE
+from .telebruxelles import TeleBruxellesIE
+from .telecinco import TelecincoIE
+from .telegraaf import TelegraafIE
+from .telemb import TeleMBIE
+from .teletask import TeleTaskIE
+from .testurl import TestURLIE
+from .tf1 import TF1IE
+from .theintercept import TheInterceptIE
+from .theonion import TheOnionIE
+from .theplatform import (
+    ThePlatformIE,
+    ThePlatformFeedIE,
+)
+from .thescene import TheSceneIE
+from .thesixtyone import TheSixtyOneIE
+from .thestar import TheStarIE
+from .thisamericanlife import ThisAmericanLifeIE
+from .thisav import ThisAVIE
+from .tinypic import TinyPicIE
+from .tlc import TlcDeIE
+from .tmz import (
+    TMZIE,
+    TMZArticleIE,
+)
+from .tnaflix import (
+    TNAFlixNetworkEmbedIE,
+    TNAFlixIE,
+    EMPFlixIE,
+    MovieFapIE,
+)
+from .toggle import ToggleIE
+from .thvideo import (
+    THVideoIE,
+    THVideoPlaylistIE
+)
+from .toutv import TouTvIE
+from .toypics import ToypicsUserIE, ToypicsIE
+from .traileraddict import TrailerAddictIE
+from .trilulilu import TriluliluIE
+from .trollvids import TrollvidsIE
+from .trutube import TruTubeIE
+from .tube8 import Tube8IE
+from .tubitv import TubiTvIE
+from .tudou import (
+    TudouIE,
+    TudouPlaylistIE,
+    TudouAlbumIE,
+)
+from .tumblr import TumblrIE
+from .tunein import (
+    TuneInClipIE,
+    TuneInStationIE,
+    TuneInProgramIE,
+    TuneInTopicIE,
+    TuneInShortenerIE,
+)
+from .turbo import TurboIE
+from .tutv import TutvIE
+from .tv2 import (
+    TV2IE,
+    TV2ArticleIE,
+)
+from .tv3 import TV3IE
+from .tv4 import TV4IE
+from .tvc import (
+    TVCIE,
+    TVCArticleIE,
+)
+from .tvigle import TvigleIE
+from .tvland import TVLandIE
+from .tvp import TvpIE, TvpSeriesIE
+from .tvplay import TVPlayIE
+from .tweakers import TweakersIE
+from .twentyfourvideo import TwentyFourVideoIE
+from .twentymin import TwentyMinutenIE
+from .twentytwotracks import (
+    TwentyTwoTracksIE,
+    TwentyTwoTracksGenreIE
+)
+from .twitch import (
+    TwitchVideoIE,
+    TwitchChapterIE,
+    TwitchVodIE,
+    TwitchProfileIE,
+    TwitchPastBroadcastsIE,
+    TwitchBookmarksIE,
+    TwitchStreamIE,
+)
+from .twitter import (
+    TwitterCardIE,
+    TwitterIE,
+    TwitterAmplifyIE,
+)
+from .ubu import UbuIE
+from .udemy import (
+    UdemyIE,
+    UdemyCourseIE
+)
+from .udn import UDNEmbedIE
+from .digiteka import DigitekaIE
+from .unistra import UnistraIE
+from .urort import UrortIE
+from .usatoday import USATodayIE
+from .ustream import UstreamIE, UstreamChannelIE
+from .ustudio import UstudioIE
+from .varzesh3 import Varzesh3IE
+from .vbox7 import Vbox7IE
+from .veehd import VeeHDIE
+from .veoh import VeohIE
+from .vessel import VesselIE
+from .vesti import VestiIE
+from .vevo import VevoIE
+from .vgtv import (
+    BTArticleIE,
+    BTVestlendingenIE,
+    VGTVIE,
+)
+from .vh1 import VH1IE
+from .vice import (
+    ViceIE,
+    ViceShowIE,
+)
+from .viddler import ViddlerIE
+from .videodetective import VideoDetectiveIE
+from .videofyme import VideofyMeIE
+from .videomega import VideoMegaIE
+from .videomore import (
+    VideomoreIE,
+    VideomoreVideoIE,
+    VideomoreSeasonIE,
+)
+from .videopremium import VideoPremiumIE
+from .videott import VideoTtIE
+from .vidme import (
+    VidmeIE,
+    VidmeUserIE,
+    VidmeUserLikesIE,
+)
+from .vidzi import VidziIE
+from .vier import VierIE, VierVideosIE
+from .viewster import ViewsterIE
+from .viidea import ViideaIE
+from .vimeo import (
+    VimeoIE,
+    VimeoAlbumIE,
+    VimeoChannelIE,
+    VimeoGroupsIE,
+    VimeoLikesIE,
+    VimeoOndemandIE,
+    VimeoReviewIE,
+    VimeoUserIE,
+    VimeoWatchLaterIE,
+)
+from .vimple import VimpleIE
+from .vine import (
+    VineIE,
+    VineUserIE,
+)
+from .viki import (
+    VikiIE,
+    VikiChannelIE,
+)
+from .vk import (
+    VKIE,
+    VKUserVideosIE,
+)
+from .vlive import VLiveIE
+from .vodlocker import VodlockerIE
+from .voicerepublic import VoiceRepublicIE
+from .voxmedia import VoxMediaIE
+from .vporn import VpornIE
+from .vrt import VRTIE
+from .vube import VubeIE
+from .vuclip import VuClipIE
+from .vulture import VultureIE
+from .walla import WallaIE
+from .washingtonpost import WashingtonPostIE
+from .wat import WatIE
+from .wayofthemaster import WayOfTheMasterIE
+from .wdr import (
+    WDRIE,
+    WDRMobileIE,
+    WDRMausIE,
+)
+from .webofstories import (
+    WebOfStoriesIE,
+    WebOfStoriesPlaylistIE,
+)
+from .weibo import WeiboIE
+from .weiqitv import WeiqiTVIE
+from .wimp import WimpIE
+from .wistia import WistiaIE
+from .worldstarhiphop import WorldStarHipHopIE
+from .wrzuta import WrzutaIE
+from .wsj import WSJIE
+from .xbef import XBefIE
+from .xboxclips import XboxClipsIE
+from .xfileshare import XFileShareIE
+from .xhamster import (
+    XHamsterIE,
+    XHamsterEmbedIE,
+)
+from .xminus import XMinusIE
+from .xnxx import XNXXIE
+from .xstream import XstreamIE
+from .xtube import XTubeUserIE, XTubeIE
+from .xuite import XuiteIE
+from .xvideos import XVideosIE
+from .xxxymovies import XXXYMoviesIE
+from .yahoo import (
+    YahooIE,
+    YahooSearchIE,
+)
+from .yam import YamIE
+from .yandexmusic import (
+    YandexMusicTrackIE,
+    YandexMusicAlbumIE,
+    YandexMusicPlaylistIE,
+)
+from .yesjapan import YesJapanIE
+from .yinyuetai import YinYueTaiIE
+from .ynet import YnetIE
+from .youjizz import YouJizzIE
+from .youku import YoukuIE
+from .youporn import YouPornIE
+from .yourupload import YourUploadIE
+from .youtube import (
+    YoutubeIE,
+    YoutubeChannelIE,
+    YoutubeFavouritesIE,
+    YoutubeHistoryIE,
+    YoutubeLiveIE,
+    YoutubePlaylistIE,
+    YoutubePlaylistsIE,
+    YoutubeRecommendedIE,
+    YoutubeSearchDateIE,
+    YoutubeSearchIE,
+    YoutubeSearchURLIE,
+    YoutubeShowIE,
+    YoutubeSubscriptionsIE,
+    YoutubeTruncatedIDIE,
+    YoutubeTruncatedURLIE,
+    YoutubeUserIE,
+    YoutubeWatchLaterIE,
+)
+from .zapiks import ZapiksIE
+from .zdf import ZDFIE, ZDFChannelIE
+from .zingmp3 import (
+    ZingMp3SongIE,
+    ZingMp3AlbumIE,
+)
+from .zippcast import ZippCastIE

From 779822d945dc7ebba7062ac9a5e760d21a7f362a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 10 Feb 2016 14:01:31 +0100
Subject: [PATCH 4562/6219] Add experimental support for lazy loading the info
 extractors

'make lazy-extractors' creates the youtube_dl/extractor/lazy_extractors.py (imported by youtube_dl/extractor/__init__.py), which contains simplified classes that only have the 'suitable' class method and that load the appropiate class with the '__new__' method when a instance is created.
---
 .gitignore                         |  1 +
 Makefile                           |  8 +++-
 devscripts/lazy_load_template.py   | 17 ++++++++
 devscripts/make_lazy_extractors.py | 63 ++++++++++++++++++++++++++++++
 youtube_dl/extractor/__init__.py   | 18 +++++----
 5 files changed, 99 insertions(+), 8 deletions(-)
 create mode 100644 devscripts/lazy_load_template.py
 create mode 100644 devscripts/make_lazy_extractors.py

diff --git a/.gitignore b/.gitignore
index 26dbde73d..72c10425d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -13,6 +13,7 @@ README.txt
 youtube-dl.1
 youtube-dl.bash-completion
 youtube-dl.fish
+youtube_dl/extractor/lazy_extractors.py
 youtube-dl
 youtube-dl.exe
 youtube-dl.tar.gz
diff --git a/Makefile b/Makefile
index ba7f7ed36..06cffcb71 100644
--- a/Makefile
+++ b/Makefile
@@ -1,7 +1,7 @@
 all: youtube-dl README.md CONTRIBUTING.md README.txt youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtube-dl.fish supportedsites
 
 clean:
-	rm -rf youtube-dl.1.temp.md youtube-dl.1 youtube-dl.bash-completion README.txt MANIFEST build/ dist/ .coverage cover/ youtube-dl.tar.gz youtube-dl.zsh youtube-dl.fish *.dump *.part *.info.json *.mp4 *.flv *.mp3 *.avi CONTRIBUTING.md.tmp ISSUE_TEMPLATE.md.tmp youtube-dl youtube-dl.exe
+	rm -rf youtube-dl.1.temp.md youtube-dl.1 youtube-dl.bash-completion README.txt MANIFEST build/ dist/ .coverage cover/ youtube-dl.tar.gz youtube-dl.zsh youtube-dl.fish youtube_dl/extractor/lazy_extractors.py *.dump *.part *.info.json *.mp4 *.flv *.mp3 *.avi CONTRIBUTING.md.tmp ISSUE_TEMPLATE.md.tmp youtube-dl youtube-dl.exe
 	find . -name "*.pyc" -delete
 	find . -name "*.class" -delete
 
@@ -88,6 +88,12 @@ youtube-dl.fish: youtube_dl/*.py youtube_dl/*/*.py devscripts/fish-completion.in
 
 fish-completion: youtube-dl.fish
 
+lazy-extractors: youtube_dl/extractor/lazy_extractors.py
+
+_EXTRACTOR_FILES != find youtube_dl/extractor -iname '*.py' -and -not -iname 'lazy_extractors.py'
+youtube_dl/extractor/lazy_extractors.py: devscripts/make_lazy_extractors.py devscripts/lazy_load_template.py $(_EXTRACTOR_FILES)
+	$(PYTHON) devscripts/make_lazy_extractors.py $@
+
 youtube-dl.tar.gz: youtube-dl README.md README.txt youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtube-dl.fish
 	@tar -czf youtube-dl.tar.gz --transform "s|^|youtube-dl/|" --owner 0 --group 0 \
 		--exclude '*.DS_Store' \
diff --git a/devscripts/lazy_load_template.py b/devscripts/lazy_load_template.py
new file mode 100644
index 000000000..ae2bd2701
--- /dev/null
+++ b/devscripts/lazy_load_template.py
@@ -0,0 +1,17 @@
+# flake8: noqa
+from __future__ import unicode_literals
+
+import re
+
+
+class LazyLoadExtractor(object):
+    _module = None
+
+    @classmethod
+    def ie_key(cls):
+        return cls.__name__[:-2]
+
+    def __new__(cls):
+        mod = __import__(cls._module, fromlist=(cls.__name__,))
+        real_cls = getattr(mod, cls.__name__)
+        return real_cls.__new__(real_cls)
diff --git a/devscripts/make_lazy_extractors.py b/devscripts/make_lazy_extractors.py
new file mode 100644
index 000000000..8627d0b1c
--- /dev/null
+++ b/devscripts/make_lazy_extractors.py
@@ -0,0 +1,63 @@
+from __future__ import unicode_literals, print_function
+
+from inspect import getsource
+import os
+from os.path import dirname as dirn
+import sys
+
+print('WARNING: Lazy loading extractors is an experimental feature that may not always work', file=sys.stderr)
+
+sys.path.insert(0, dirn(dirn((os.path.abspath(__file__)))))
+
+lazy_extractors_filename = sys.argv[1]
+if os.path.exists(lazy_extractors_filename):
+    os.remove(lazy_extractors_filename)
+
+from youtube_dl.extractor import _ALL_CLASSES
+from youtube_dl.extractor.common import InfoExtractor
+
+with open('devscripts/lazy_load_template.py', 'rt') as f:
+    module_template = f.read()
+
+module_contents = [module_template + '\n' + getsource(InfoExtractor.suitable)]
+
+ie_template = '''
+class {name}(LazyLoadExtractor):
+    _VALID_URL = {valid_url!r}
+    _module = '{module}'
+'''
+
+make_valid_template = '''
+    @classmethod
+    def _make_valid_url(cls):
+        return {!r}
+'''
+
+
+def build_lazy_ie(ie, name):
+    valid_url = getattr(ie, '_VALID_URL', None)
+    s = ie_template.format(
+        name=name,
+        valid_url=valid_url,
+        module=ie.__module__)
+    if ie.suitable.__func__ is not InfoExtractor.suitable.__func__:
+        s += getsource(ie.suitable)
+    if hasattr(ie, '_make_valid_url'):
+        # search extractors
+        s += make_valid_template.format(ie._make_valid_url())
+    return s
+
+names = []
+for ie in _ALL_CLASSES:
+    name = ie.ie_key() + 'IE'
+    src = build_lazy_ie(ie, name)
+    module_contents.append(src)
+    names.append(name)
+
+module_contents.append(
+    '_ALL_CLASSES = [{}]'.format(', '.join(names)))
+
+module_src = '\n'.join(module_contents)
+
+with open(lazy_extractors_filename, 'wt') as f:
+    f.write(module_src)
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index a0a53445a..b0d4d156b 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -1,13 +1,17 @@
 from __future__ import unicode_literals
 
-from .extractors import *
+try:
+    from .lazy_extractors import *
+    from .lazy_extractors import _ALL_CLASSES
+except ImportError:
+    from .extractors import *
 
-_ALL_CLASSES = [
-    klass
-    for name, klass in globals().items()
-    if name.endswith('IE') and name != 'GenericIE'
-]
-_ALL_CLASSES.append(GenericIE)
+    _ALL_CLASSES = [
+        klass
+        for name, klass in globals().items()
+        if name.endswith('IE') and name != 'GenericIE'
+    ]
+    _ALL_CLASSES.append(GenericIE)
 
 
 def gen_extractor_classes():

From 0d778b1db909c8d096be4e199384fff96a722fc9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 11 Feb 2016 14:49:02 +0100
Subject: [PATCH 4563/6219] lazy extractors: specify the encoding

When building with python3 the unicode characters are not escaped, python2 needs to know the encoding.
---
 devscripts/lazy_load_template.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/devscripts/lazy_load_template.py b/devscripts/lazy_load_template.py
index ae2bd2701..563d629f8 100644
--- a/devscripts/lazy_load_template.py
+++ b/devscripts/lazy_load_template.py
@@ -1,3 +1,4 @@
+# encoding: utf-8
 # flake8: noqa
 from __future__ import unicode_literals
 

From c1ce6acdd73da7744f4bbe27698e96275467e14d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 21 Feb 2016 11:53:48 +0100
Subject: [PATCH 4564/6219] lazy extractors: Fix building with python2.6

---
 devscripts/make_lazy_extractors.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/devscripts/make_lazy_extractors.py b/devscripts/make_lazy_extractors.py
index 8627d0b1c..5d0ddb401 100644
--- a/devscripts/make_lazy_extractors.py
+++ b/devscripts/make_lazy_extractors.py
@@ -30,7 +30,7 @@ class {name}(LazyLoadExtractor):
 make_valid_template = '''
     @classmethod
     def _make_valid_url(cls):
-        return {!r}
+        return {valid_url!r}
 '''
 
 
@@ -44,7 +44,7 @@ def build_lazy_ie(ie, name):
         s += getsource(ie.suitable)
     if hasattr(ie, '_make_valid_url'):
         # search extractors
-        s += make_valid_template.format(ie._make_valid_url())
+        s += make_valid_template.format(valid_url=ie._make_valid_url())
     return s
 
 names = []
@@ -55,7 +55,7 @@ for ie in _ALL_CLASSES:
     names.append(name)
 
 module_contents.append(
-    '_ALL_CLASSES = [{}]'.format(', '.join(names)))
+    '_ALL_CLASSES = [{0}]'.format(', '.join(names)))
 
 module_src = '\n'.join(module_contents)
 

From 6b97ca96fc242c1d7639d080e2c8e3ee9f9d0bed Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 21 Feb 2016 12:22:12 +0100
Subject: [PATCH 4565/6219] lazy extractors: Style fixes

* Sort extractors alphabetically
* Add newlines when needed (youtube_dl/extractors/lazy_extractors.py pass the flake8 test now)
---
 devscripts/lazy_load_template.py   | 1 -
 devscripts/make_lazy_extractors.py | 6 +++---
 setup.cfg                          | 2 +-
 3 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/devscripts/lazy_load_template.py b/devscripts/lazy_load_template.py
index 563d629f8..b984aab9b 100644
--- a/devscripts/lazy_load_template.py
+++ b/devscripts/lazy_load_template.py
@@ -1,5 +1,4 @@
 # encoding: utf-8
-# flake8: noqa
 from __future__ import unicode_literals
 
 import re
diff --git a/devscripts/make_lazy_extractors.py b/devscripts/make_lazy_extractors.py
index 5d0ddb401..b5a8b9190 100644
--- a/devscripts/make_lazy_extractors.py
+++ b/devscripts/make_lazy_extractors.py
@@ -41,14 +41,14 @@ def build_lazy_ie(ie, name):
         valid_url=valid_url,
         module=ie.__module__)
     if ie.suitable.__func__ is not InfoExtractor.suitable.__func__:
-        s += getsource(ie.suitable)
+        s += '\n' + getsource(ie.suitable)
     if hasattr(ie, '_make_valid_url'):
         # search extractors
         s += make_valid_template.format(valid_url=ie._make_valid_url())
     return s
 
 names = []
-for ie in _ALL_CLASSES:
+for ie in list(sorted(_ALL_CLASSES[:-1], key=lambda cls: cls.ie_key())) + _ALL_CLASSES[-1:]:
     name = ie.ie_key() + 'IE'
     src = build_lazy_ie(ie, name)
     module_contents.append(src)
@@ -57,7 +57,7 @@ for ie in _ALL_CLASSES:
 module_contents.append(
     '_ALL_CLASSES = [{0}]'.format(', '.join(names)))
 
-module_src = '\n'.join(module_contents)
+module_src = '\n'.join(module_contents) + '\n'
 
 with open(lazy_extractors_filename, 'wt') as f:
     f.write(module_src)
diff --git a/setup.cfg b/setup.cfg
index 5760112d4..2dc06ffe4 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -2,5 +2,5 @@
 universal = True
 
 [flake8]
-exclude = youtube_dl/extractor/__init__.py,devscripts/buildserver.py,devscripts/make_issue_template.py,setup.py,build,.git
+exclude = youtube_dl/extractor/__init__.py,devscripts/buildserver.py,devscripts/lazy_load_template.py,devscripts/make_issue_template.py,setup.py,build,.git
 ignore = E402,E501,E731

From e0986e31cfd57392aaf3cc84b17fbf32c6134ff6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 21 Feb 2016 12:28:58 +0100
Subject: [PATCH 4566/6219] lazy extractors: Output if it's enabled in the
 verbose log

---
 youtube_dl/YoutubeDL.py          | 4 +++-
 youtube_dl/extractor/__init__.py | 2 ++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index f18a8e840..a89a71a25 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -82,7 +82,7 @@ from .utils import (
     YoutubeDLHandler,
 )
 from .cache import Cache
-from .extractor import get_info_extractor, gen_extractor_classes
+from .extractor import get_info_extractor, gen_extractor_classes, _LAZY_LOADER
 from .downloader import get_suitable_downloader
 from .downloader.rtmp import rtmpdump_version
 from .postprocessor import (
@@ -1959,6 +1959,8 @@ class YoutubeDL(object):
         write_string(encoding_str, encoding=None)
 
         self._write_string('[debug] youtube-dl version ' + __version__ + '\n')
+        if _LAZY_LOADER:
+            self._write_string('[debug] Lazy loading extractors enabled' + '\n')
         try:
             sp = subprocess.Popen(
                 ['git', 'rev-parse', '--short', 'HEAD'],
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index b0d4d156b..18d8dbcd6 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -3,7 +3,9 @@ from __future__ import unicode_literals
 try:
     from .lazy_extractors import *
     from .lazy_extractors import _ALL_CLASSES
+    _LAZY_LOADER = True
 except ImportError:
+    _LAZY_LOADER = False
     from .extractors import *
 
     _ALL_CLASSES = [

From 8a5dc1c1e14cc19f143c84702f4bbc29e4f91e47 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 21 Feb 2016 12:46:14 +0100
Subject: [PATCH 4567/6219] lazy extractors: Initialize the real info extractor

According to the docs '__init__' is only called automatically if '__new__' returns an instance of the original class.
---
 devscripts/lazy_load_template.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/devscripts/lazy_load_template.py b/devscripts/lazy_load_template.py
index b984aab9b..2e6e6641b 100644
--- a/devscripts/lazy_load_template.py
+++ b/devscripts/lazy_load_template.py
@@ -11,7 +11,9 @@ class LazyLoadExtractor(object):
     def ie_key(cls):
         return cls.__name__[:-2]
 
-    def __new__(cls):
+    def __new__(cls, *args, **kwargs):
         mod = __import__(cls._module, fromlist=(cls.__name__,))
         real_cls = getattr(mod, cls.__name__)
-        return real_cls.__new__(real_cls)
+        instance = real_cls.__new__(real_cls)
+        instance.__init__(*args, **kwargs)
+        return instance

From 5a9858bfa9aba01c9dec549b83f5a0b17a520f13 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 6 Mar 2016 19:36:39 +0100
Subject: [PATCH 4568/6219] setup.py: add command for building the
 lazy_extractors module

---
 setup.py | 22 ++++++++++++++++++++--
 1 file changed, 20 insertions(+), 2 deletions(-)

diff --git a/setup.py b/setup.py
index bfe931f5b..9444d403d 100644
--- a/setup.py
+++ b/setup.py
@@ -8,11 +8,12 @@ import warnings
 import sys
 
 try:
-    from setuptools import setup
+    from setuptools import setup, Command
     setuptools_available = True
 except ImportError:
-    from distutils.core import setup
+    from distutils.core import setup, Command
     setuptools_available = False
+from distutils.spawn import spawn
 
 try:
     # This will create an exe that needs Microsoft Visual C++ 2008
@@ -70,6 +71,22 @@ else:
     else:
         params['scripts'] = ['bin/youtube-dl']
 
+class build_lazy_extractors(Command):
+    description = "Build the extractor lazy loading module"
+    user_options = []
+
+    def initialize_options(self):
+        pass
+
+    def finalize_options(self):
+        pass
+
+    def run(self):
+        spawn(
+            [sys.executable, 'devscripts/make_lazy_extractors.py', 'youtube_dl/extractor/lazy_extractors.py'],
+            dry_run=self.dry_run,
+        )
+
 # Get the version from youtube_dl/version.py without importing the package
 exec(compile(open('youtube_dl/version.py').read(),
              'youtube_dl/version.py', 'exec'))
@@ -107,5 +124,6 @@ setup(
         "Programming Language :: Python :: 3.4",
     ],
 
+    cmdclass={'build_lazy_extractors': build_lazy_extractors},
     **params
 )

From bffb245a4882b10b5e66015fa89ef1cadf974415 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 9 Apr 2016 10:47:46 +0100
Subject: [PATCH 4569/6219] [aol] add support for videos with vidible
 IDs(closes #9124)

---
 youtube_dl/extractor/aol.py | 78 +++++++++++++++++++++++++++++++++++--
 1 file changed, 75 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/aol.py b/youtube_dl/extractor/aol.py
index 95a99c6b0..b729157d2 100644
--- a/youtube_dl/extractor/aol.py
+++ b/youtube_dl/extractor/aol.py
@@ -1,11 +1,17 @@
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    int_or_none,
+)
 
 
 class AolIE(InfoExtractor):
     IE_NAME = 'on.aol.com'
-    _VALID_URL = r'(?:aol-video:|https?://on\.aol\.com/video/.*-)(?P<id>[0-9]+)(?:$|\?)'
+    _VALID_URL = r'(?:aol-video:|https?://on\.aol\.com/video/.*-)(?P<id>[^/?-]+)'
 
     _TESTS = [{
         'url': 'http://on.aol.com/video/u-s--official-warns-of-largest-ever-irs-phone-scam-518167793?icid=OnHomepageC2Wide_MustSee_Img',
@@ -14,13 +20,79 @@ class AolIE(InfoExtractor):
             'id': '518167793',
             'ext': 'mp4',
             'title': 'U.S. Official Warns Of \'Largest Ever\' IRS Phone Scam',
+            'description': 'A major phone scam has cost thousands of taxpayers more than $1 million, with less than a month until income tax returns are due to the IRS.',
+            'timestamp': 1395405060,
+            'upload_date': '20140321',
+            'uploader': 'Newsy Studio',
         },
-        'add_ie': ['FiveMin'],
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        }
+    }, {
+        'url': 'http://on.aol.com/video/netflix-is-raising-rates-5707d6b8e4b090497b04f706?context=PC:homepage:PL1944:1460189336183',
+        'info_dict': {
+            'id': '5707d6b8e4b090497b04f706',
+            'ext': 'mp4',
+            'title': 'Netflix is Raising Rates',
+            'description': 'Netflix is rewarding millions of it’s long-standing members with an increase in cost. Veuer’s Carly Figueroa has more.',
+            'upload_date': '20160408',
+            'timestamp': 1460123280,
+            'uploader': 'Veuer',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        }
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        return self.url_result('5min:%s' % video_id)
+
+        response = self._download_json(
+            'https://feedapi.b2c.on.aol.com/v1.0/app/videos/aolon/%s/details' % video_id,
+            video_id)['response']
+        if response['statusText'] != 'Ok':
+            raise ExtractorError('%s said: %s' % (self.IE_NAME, response['statusText']), expected=True)
+
+        video_data = response['data']
+        formats = []
+        m3u8_url = video_data.get('videoMasterPlaylist')
+        if m3u8_url:
+            formats.extend(self._extract_m3u8_formats(
+                m3u8_url, video_id, 'mp4', m3u8_id='hls', fatal=False))
+        for rendition in video_data.get('renditions', []):
+            video_url = rendition.get('url')
+            if not video_url:
+                continue
+            ext = rendition.get('format')
+            if ext == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    video_url, video_id, 'mp4', m3u8_id='hls', fatal=False))
+            else:
+                f = {
+                    'url': video_url,
+                    'format_id': rendition.get('quality'),
+                }
+                mobj = re.search(r'(\d+)x(\d+)', video_url)
+                if mobj:
+                    f.update({
+                        'width': int(mobj.group(1)),
+                        'height': int(mobj.group(2)),
+                    })
+                formats.append(f)
+        self._sort_formats(formats, ('width', 'height', 'tbr', 'format_id'))
+
+        return {
+            'id': video_id,
+            'title': video_data['title'],
+            'duration': int_or_none(video_data.get('duration')),
+            'timestamp': int_or_none(video_data.get('publishDate')),
+            'view_count': int_or_none(video_data.get('views')),
+            'description': video_data.get('description'),
+            'uploader': video_data.get('videoOwner'),
+            'formats': formats,
+        }
 
 
 class AolFeaturesIE(InfoExtractor):

From cacd9966624883523b264fa9ac48138074597730 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 9 Apr 2016 19:27:54 +0800
Subject: [PATCH 4570/6219] [utils] Don't touch URLs if not necessary

Fix test_Generic_15 (Google redirect)
---
 youtube_dl/utils.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 8e53962c9..999dfabb5 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1792,6 +1792,8 @@ def urlencode_postdata(*args, **kargs):
 
 
 def update_url_query(url, query):
+    if not query:
+        return url
     parsed_url = compat_urlparse.urlparse(url)
     qs = compat_parse_qs(parsed_url.query)
     qs.update(query)

From 92c7f3157aad87096aa1fdd1a4daed3bdf262178 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 9 Apr 2016 17:32:23 +0600
Subject: [PATCH 4571/6219] [aol] Add coding cookie

---
 youtube_dl/extractor/aol.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/aol.py b/youtube_dl/extractor/aol.py
index b729157d2..d4801a25b 100644
--- a/youtube_dl/extractor/aol.py
+++ b/youtube_dl/extractor/aol.py
@@ -1,3 +1,4 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
 import re

From ab481b48e536dd2e03d6022abb7f4d1593294721 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 9 Apr 2016 20:12:11 +0800
Subject: [PATCH 4572/6219] [funnyordie] Relax M3U8 URL matching

Also, m3u8_url extraction should be fatal as all formats depends
directly or indirectly on it.

This change fixes test_Generic_26 and TestFunnyOrDieSubtitles
---
 youtube_dl/extractor/funnyordie.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/funnyordie.py b/youtube_dl/extractor/funnyordie.py
index 4c4a87e2a..8c5ffc9e8 100644
--- a/youtube_dl/extractor/funnyordie.py
+++ b/youtube_dl/extractor/funnyordie.py
@@ -46,8 +46,8 @@ class FunnyOrDieIE(InfoExtractor):
         links.sort(key=lambda link: 1 if link[1] == 'mp4' else 0)
 
         m3u8_url = self._search_regex(
-            r'<source[^>]+src=(["\'])(?P<url>.+?/master\.m3u8)\1',
-            webpage, 'm3u8 url', default=None, group='url')
+            r'<source[^>]+src=(["\'])(?P<url>.+?/master\.m3u8[^"\']*)\1',
+            webpage, 'm3u8 url', group='url')
 
         formats = []
 

From bfe96d7bea7c5227456bf1aecca51907c8f30c51 Mon Sep 17 00:00:00 2001
From: Philip Huppert <philip@zeilen-sprung.de>
Date: Fri, 9 Oct 2015 18:38:11 +0200
Subject: [PATCH 4573/6219] [presstv] Added extractor PressTV.

Fixes #7060
---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/presstv.py    | 80 ++++++++++++++++++++++++++++++
 2 files changed, 81 insertions(+)
 create mode 100644 youtube_dl/extractor/presstv.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index de29c7956..c2fa83918 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -583,6 +583,7 @@ from .pornhub import (
 from .pornotube import PornotubeIE
 from .pornovoisines import PornoVoisinesIE
 from .pornoxo import PornoXOIE
+from .presstv import PressTVIE
 from .primesharetv import PrimeShareTVIE
 from .promptfile import PromptFileIE
 from .prosiebensat1 import ProSiebenSat1IE
diff --git a/youtube_dl/extractor/presstv.py b/youtube_dl/extractor/presstv.py
new file mode 100644
index 000000000..724d8b1c4
--- /dev/null
+++ b/youtube_dl/extractor/presstv.py
@@ -0,0 +1,80 @@
+# coding: utf-8
+from __future__ import unicode_literals
+import re
+
+from .common import InfoExtractor
+from ..utils import str_to_int
+
+
+class PressTVIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?presstv\.ir/Video/(?P<y>[0-9]+)/(?P<m>[0-9]+)/(?P<d>[0-9]+)/(?P<id>[0-9]+)/'
+
+    _TEST = {
+        'url': 'http://www.presstv.ir/Video/2015/10/04/431915/Max-Igan-Press-TV-Face-to-Face',
+        'md5': 'e95736ac75088b5f1e5bbb68f248f90d',
+        'info_dict': {
+            'id': '431915',
+            'ext': 'mp4',
+            'title': 'Press TV’s full interview with Max Igan',
+            'upload_date': '20151004',
+            'thumbnail': 'http://217.218.67.233/photo/20151004/d5c333ad-98f9-4bd3-bc3e-a1ad6a192803.jpg',
+            'description': ('Watch Press TV’s full interview with Max Igan, a radio talk show host and political '
+                            'commentator.\nThe interview, conducted on Press TV’s Face '
+                            'to Face program, was aired on October 3, 2015.')
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        # extract video URL from webpage
+        video_url = self._html_search_regex(r'<input type="hidden" id="inpPlayback" value="([^"]+)" />', webpage,
+                                            'Video URL')
+
+        # build list of available formats
+        # specified in http://www.presstv.ir/Scripts/playback.js
+        base_url = 'http://192.99.219.222:82/presstv'
+        formats = [
+            {
+                'url': base_url + video_url,
+                'format': '1080p mp4',
+                'format_id': '1080p'
+            }, {
+                'url': base_url + video_url.replace(".mp4", "_low800.mp4"),
+                'format': '720p mp4',
+                'format_id': '720p'
+            }, {
+                'url': base_url + video_url.replace(".mp4", "_low400.mp4"),
+                'format': '360p mp4',
+                'format_id': '360p'
+            }, {
+                'url': base_url + video_url.replace(".mp4", "_low200.mp4"),
+                'format': '180p mp4',
+                'format_id': '180p'
+            }
+        ]
+        formats.reverse()
+
+        # extract video metadata
+        title = self._html_search_meta('title', webpage, 'Title', True)
+        title = title.partition(' - ')[2]
+
+        description = self._html_search_regex(r'<div class="media-text nano-content">(.*?)</div>', webpage,
+                                              'Description', flags=re.DOTALL)
+
+        thumbnail = self._html_search_meta('og:image', webpage, 'Thumbnail', True)
+
+        year = str_to_int(self._search_regex(PressTVIE._VALID_URL, url, 'Upload year', group='y'))
+        month = str_to_int(self._search_regex(PressTVIE._VALID_URL, url, 'Upload month', group='m'))
+        day = str_to_int(self._search_regex(PressTVIE._VALID_URL, url, 'Upload day', group='d'))
+        upload_date = '%04d%02d%02d' % (year, month, day)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': formats,
+            'thumbnail': thumbnail,
+            'upload_date': upload_date,
+            'description': description
+        }

From c05025fdd79993314e20a6074aed084889199e50 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 9 Apr 2016 21:46:51 +0800
Subject: [PATCH 4574/6219] [internetvideoarchive] Fix extraction and support
 json URLs

---
 youtube_dl/extractor/internetvideoarchive.py | 118 +++++++++----------
 1 file changed, 58 insertions(+), 60 deletions(-)

diff --git a/youtube_dl/extractor/internetvideoarchive.py b/youtube_dl/extractor/internetvideoarchive.py
index e60145b3d..45add007f 100644
--- a/youtube_dl/extractor/internetvideoarchive.py
+++ b/youtube_dl/extractor/internetvideoarchive.py
@@ -1,93 +1,91 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..compat import (
+    compat_parse_qs,
     compat_urlparse,
-    compat_urllib_parse_urlencode,
 )
 from ..utils import (
-    xpath_with_ns,
+    determine_ext,
+    int_or_none,
+    xpath_text,
 )
 
 
 class InternetVideoArchiveIE(InfoExtractor):
-    _VALID_URL = r'https?://video\.internetvideoarchive\.net/flash/players/.*?\?.*?publishedid.*?'
+    _VALID_URL = r'https?://video\.internetvideoarchive\.net/(?:player|flash/players)/.*?\?.*?publishedid.*?'
 
     _TEST = {
-        'url': 'http://video.internetvideoarchive.net/flash/players/flashconfiguration.aspx?customerid=69249&publishedid=452693&playerid=247',
+        'url': 'http://video.internetvideoarchive.net/player/6/configuration.ashx?customerid=69249&publishedid=194487&reporttag=vdbetatitle&playerid=641&autolist=0&domain=www.videodetective.com&maxrate=high&minrate=low&socialplayer=false',
         'info_dict': {
-            'id': '452693',
+            'id': '194487',
             'ext': 'mp4',
-            'title': 'SKYFALL',
-            'description': 'In SKYFALL, Bond\'s loyalty to M is tested as her past comes back to haunt her. As MI6 comes under attack, 007 must track down and destroy the threat, no matter how personal the cost.',
-            'duration': 152,
+            'title': 'KICK-ASS 2',
+            'description': 'md5:c189d5b7280400630a1d3dd17eaa8d8a',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
         },
     }
 
     @staticmethod
-    def _build_url(query):
-        return 'http://video.internetvideoarchive.net/flash/players/flashconfiguration.aspx?' + query
+    def _build_json_url(query):
+        return 'http://video.internetvideoarchive.net/player/6/configuration.ashx?' + query
 
     @staticmethod
-    def _clean_query(query):
-        NEEDED_ARGS = ['publishedid', 'customerid']
-        query_dic = compat_urlparse.parse_qs(query)
-        cleaned_dic = dict((k, v[0]) for (k, v) in query_dic.items() if k in NEEDED_ARGS)
-        # Other player ids return m3u8 urls
-        cleaned_dic['playerid'] = '247'
-        cleaned_dic['videokbrate'] = '100000'
-        return compat_urllib_parse_urlencode(cleaned_dic)
+    def _build_xml_url(query):
+        return 'http://video.internetvideoarchive.net/flash/players/flashconfiguration.aspx?' + query
 
     def _real_extract(self, url):
         query = compat_urlparse.urlparse(url).query
-        query_dic = compat_urlparse.parse_qs(query)
+        query_dic = compat_parse_qs(query)
         video_id = query_dic['publishedid'][0]
-        url = self._build_url(query)
 
-        flashconfiguration = self._download_xml(url, video_id,
-                                                'Downloading flash configuration')
-        file_url = flashconfiguration.find('file').text
-        file_url = file_url.replace('/playlist.aspx', '/mrssplaylist.aspx')
-        # Replace some of the parameters in the query to get the best quality
-        # and http links (no m3u8 manifests)
-        file_url = re.sub(r'(?<=\?)(.+)$',
-                          lambda m: self._clean_query(m.group()),
-                          file_url)
-        info = self._download_xml(file_url, video_id,
-                                  'Downloading video info')
-        item = info.find('channel/item')
+        if '/player/' in url:
+            configuration = self._download_json(url, video_id)
 
-        def _bp(p):
-            return xpath_with_ns(
-                p,
-                {
-                    'media': 'http://search.yahoo.com/mrss/',
-                    'jwplayer': 'http://developer.longtailvideo.com/trac/wiki/FlashFormats',
-                }
-            )
-        formats = []
-        for content in item.findall(_bp('media:group/media:content')):
-            attr = content.attrib
-            f_url = attr['url']
-            width = int(attr['width'])
-            bitrate = int(attr['bitrate'])
-            format_id = '%d-%dk' % (width, bitrate)
-            formats.append({
-                'format_id': format_id,
-                'url': f_url,
-                'width': width,
-                'tbr': bitrate,
-            })
+            # There are multiple videos in the playlist whlie only the first one
+            # matches the video played in browsers
+            video_info = configuration['playlist'][0]
 
-        self._sort_formats(formats)
+            formats = []
+            for source in video_info['sources']:
+                file_url = source['file']
+                if determine_ext(file_url) == 'm3u8':
+                    formats.extend(self._extract_m3u8_formats(
+                        file_url, video_id, ext='mp4', m3u8_id='hls'))
+                else:
+                    a_format = {
+                        'url': file_url,
+                    }
+
+                    if source.get('label') and source['label'][-4:] == ' kbs':
+                        tbr = int_or_none(source['label'][:-4])
+                        a_format.update({
+                            'tbr': tbr,
+                            'format_id': 'http-%d' % tbr,
+                        })
+                        formats.append(a_format)
+
+            self._sort_formats(formats)
+
+            title = video_info['title']
+            description = video_info.get('description')
+            thumbnail = video_info.get('image')
+        else:
+            configuration = self._download_xml(url, video_id)
+            formats = [{
+                'url': xpath_text(configuration, './file', 'file URL', fatal=True),
+            }]
+            thumbnail = xpath_text(configuration, './image', 'thumbnail')
+            title = 'InternetVideoArchive video %s' % video_id
+            description = None
 
         return {
             'id': video_id,
-            'title': item.find('title').text,
+            'title': title,
             'formats': formats,
-            'thumbnail': item.find(_bp('media:thumbnail')).attrib['url'],
-            'description': item.find('description').text,
-            'duration': int(attr['duration']),
+            'thumbnail': thumbnail,
+            'description': description,
         }

From dae2a058de81e42d73bdbe0041a598262703c352 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 9 Apr 2016 21:47:12 +0800
Subject: [PATCH 4575/6219] [rottentomatoes] Adapt to InternetVideoArchiveIE

---
 youtube_dl/extractor/rottentomatoes.py | 24 ++++++++++++++++++------
 1 file changed, 18 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/rottentomatoes.py b/youtube_dl/extractor/rottentomatoes.py
index e8bb20a08..f9cd48790 100644
--- a/youtube_dl/extractor/rottentomatoes.py
+++ b/youtube_dl/extractor/rottentomatoes.py
@@ -1,11 +1,11 @@
 from __future__ import unicode_literals
 
-from .videodetective import VideoDetectiveIE
+from .common import InfoExtractor
+from ..compat import compat_urlparse
+from .internetvideoarchive import InternetVideoArchiveIE
 
 
-# It just uses the same method as videodetective.com,
-# the internetvideoarchive.com is extracted from the og:video property
-class RottenTomatoesIE(VideoDetectiveIE):
+class RottenTomatoesIE(InfoExtractor):
     _VALID_URL = r'https?://www\.rottentomatoes\.com/m/[^/]+/trailers/(?P<id>\d+)'
 
     _TEST = {
@@ -13,7 +13,19 @@ class RottenTomatoesIE(VideoDetectiveIE):
         'info_dict': {
             'id': '613340',
             'ext': 'mp4',
-            'title': 'TOY STORY 3',
-            'description': 'From the creators of the beloved TOY STORY films, comes a story that will reunite the gang in a whole new way.',
+            'title': 'Toy Story 3',
         },
     }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        og_video = self._og_search_video_url(webpage)
+        query = compat_urlparse.urlparse(og_video).query
+
+        return {
+            '_type': 'url_transparent',
+            'url': InternetVideoArchiveIE._build_xml_url(query),
+            'ie_key': InternetVideoArchiveIE.ie_key(),
+            'title': self._og_search_title(webpage),
+        }

From c991106706c05401090bcba79e65feae5c7e3fda Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 9 Apr 2016 21:47:35 +0800
Subject: [PATCH 4576/6219] [videodetective] Adapt to InternetVideoArchiveIE

---
 youtube_dl/extractor/videodetective.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/videodetective.py b/youtube_dl/extractor/videodetective.py
index 0ffc7ff7d..2ed5d9643 100644
--- a/youtube_dl/extractor/videodetective.py
+++ b/youtube_dl/extractor/videodetective.py
@@ -14,8 +14,11 @@ class VideoDetectiveIE(InfoExtractor):
             'id': '194487',
             'ext': 'mp4',
             'title': 'KICK-ASS 2',
-            'description': 'md5:65ba37ad619165afac7d432eaded6013',
-            'duration': 138,
+            'description': 'md5:c189d5b7280400630a1d3dd17eaa8d8a',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
         },
     }
 
@@ -24,4 +27,4 @@ class VideoDetectiveIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
         og_video = self._og_search_video_url(webpage)
         query = compat_urlparse.urlparse(og_video).query
-        return self.url_result(InternetVideoArchiveIE._build_url(query), ie=InternetVideoArchiveIE.ie_key())
+        return self.url_result(InternetVideoArchiveIE._build_json_url(query), ie=InternetVideoArchiveIE.ie_key())

From 6c4c7539f222cd9e80dfae0b1c9dabbd45d1b3dc Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 9 Apr 2016 22:04:48 +0800
Subject: [PATCH 4577/6219] [test/helper] Check got values to be strings for
 md5: fields

Seen in PBSIE tests
---
 test/helper.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/test/helper.py b/test/helper.py
index f2d878212..b8e22c5cb 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -143,6 +143,9 @@ def expect_value(self, got, expected, field):
             expect_value(self, item_got, item_expected, field)
     else:
         if isinstance(expected, compat_str) and expected.startswith('md5:'):
+            self.assertTrue(
+                isinstance(got, compat_str),
+                'Expected field %s to be a unicode object, but got value %r of type %r' % (field, got, type(got)))
             got = 'md5:' + md5(got)
         elif isinstance(expected, compat_str) and expected.startswith('mincount:'):
             self.assertTrue(

From 95153a960d098d75e6100e38e77fdaa32f5267a2 Mon Sep 17 00:00:00 2001
From: Philip Huppert <philip@zeilen-sprung.de>
Date: Sat, 9 Apr 2016 16:14:05 +0200
Subject: [PATCH 4578/6219] [presstv] updated extractor and tests to work with
 current PressTV website

---
 youtube_dl/extractor/presstv.py | 26 ++++++++++++--------------
 1 file changed, 12 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/presstv.py b/youtube_dl/extractor/presstv.py
index 724d8b1c4..9af6780c1 100644
--- a/youtube_dl/extractor/presstv.py
+++ b/youtube_dl/extractor/presstv.py
@@ -7,20 +7,20 @@ from ..utils import str_to_int
 
 
 class PressTVIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?presstv\.ir/Video/(?P<y>[0-9]+)/(?P<m>[0-9]+)/(?P<d>[0-9]+)/(?P<id>[0-9]+)/'
+    _VALID_URL = r'https?://(?:www\.)?presstv\.ir/[^/]+/(?P<y>[0-9]+)/(?P<m>[0-9]+)/(?P<d>[0-9]+)/(?P<id>[0-9]+)/'
 
     _TEST = {
-        'url': 'http://www.presstv.ir/Video/2015/10/04/431915/Max-Igan-Press-TV-Face-to-Face',
-        'md5': 'e95736ac75088b5f1e5bbb68f248f90d',
+        'url': 'http://www.presstv.ir/Detail/2016/04/09/459911/Australian-sewerage-treatment-facility-/',
+        'md5': '5d7e3195a447cb13e9267e931d8dd5a5',
         'info_dict': {
-            'id': '431915',
+            'id': '459911',
             'ext': 'mp4',
-            'title': 'Press TV’s full interview with Max Igan',
-            'upload_date': '20151004',
-            'thumbnail': 'http://217.218.67.233/photo/20151004/d5c333ad-98f9-4bd3-bc3e-a1ad6a192803.jpg',
-            'description': ('Watch Press TV’s full interview with Max Igan, a radio talk show host and political '
-                            'commentator.\nThe interview, conducted on Press TV’s Face '
-                            'to Face program, was aired on October 3, 2015.')
+            'title': 'Organic mattresses used to clean waste water',
+            'upload_date': '20160409',
+            'thumbnail': 'http://media.presstv.com/photo/20160409/41719129-76fa-4372-a09d-bf348278eb5d.jpg',
+            'description': ('A trial program at an Australian sewerage treatment facility hopes to change '
+                            'the way waste water is treated by using plant mattresses to reduce chemical '
+                            'and electricity use.')
         }
     }
 
@@ -58,12 +58,10 @@ class PressTVIE(InfoExtractor):
 
         # extract video metadata
         title = self._html_search_meta('title', webpage, 'Title', True)
-        title = title.partition(' - ')[2]
-
-        description = self._html_search_regex(r'<div class="media-text nano-content">(.*?)</div>', webpage,
-                                              'Description', flags=re.DOTALL)
+        title = title.partition('-')[2].strip()
 
         thumbnail = self._html_search_meta('og:image', webpage, 'Thumbnail', True)
+        description = self._html_search_meta('og:description', webpage, 'Description', True)
 
         year = str_to_int(self._search_regex(PressTVIE._VALID_URL, url, 'Upload year', group='y'))
         month = str_to_int(self._search_regex(PressTVIE._VALID_URL, url, 'Upload month', group='m'))

From eb9c3edd5ec970abb349bd4c71040b75e9d19e0f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 9 Apr 2016 22:40:05 +0200
Subject: [PATCH 4579/6219] [test/utils] Add test for date_from_str

---
 test/test_utils.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/test/test_utils.py b/test/test_utils.py
index a35debfe1..0f36bb9f0 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -20,6 +20,7 @@ from youtube_dl.utils import (
     args_to_str,
     encode_base_n,
     clean_html,
+    date_from_str,
     DateRange,
     detect_exe_version,
     determine_ext,
@@ -234,6 +235,13 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(unescapeHTML('&eacute;'), 'é')
         self.assertEqual(unescapeHTML('&#2013266066;'), '&#2013266066;')
 
+    def test_date_from_str(self):
+        self.assertEqual(date_from_str('yesterday'), date_from_str('now-1day'))
+        self.assertEqual(date_from_str('now+7day'), date_from_str('now+1week'))
+        self.assertEqual(date_from_str('now+14day'), date_from_str('now+2week'))
+        self.assertEqual(date_from_str('now+365day'), date_from_str('now+1year'))
+        self.assertEqual(date_from_str('now+30day'), date_from_str('now+1month'))
+
     def test_daterange(self):
         _20century = DateRange("19000101", "20000101")
         self.assertFalse("17890714" in _20century)

From 61dd350a04a77abe86e46cfe8b7603514e8f2ca0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 10 Apr 2016 03:02:35 +0600
Subject: [PATCH 4580/6219] [1tv] Fix extraction (Closes #9103)

---
 youtube_dl/extractor/firsttv.py | 145 ++++++++++++++++++++++----------
 1 file changed, 100 insertions(+), 45 deletions(-)

diff --git a/youtube_dl/extractor/firsttv.py b/youtube_dl/extractor/firsttv.py
index 98b165143..88bca1007 100644
--- a/youtube_dl/extractor/firsttv.py
+++ b/youtube_dl/extractor/firsttv.py
@@ -2,78 +2,133 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..utils import int_or_none
+from ..compat import compat_xpath
+from ..utils import (
+    int_or_none,
+    qualities,
+    unified_strdate,
+    xpath_attr,
+    xpath_element,
+    xpath_text,
+    xpath_with_ns,
+)
 
 
 class FirstTVIE(InfoExtractor):
     IE_NAME = '1tv'
     IE_DESC = 'Первый канал'
-    _VALID_URL = r'https?://(?:www\.)?1tv\.ru/(?:[^/]+/)+(?P<id>.+)'
+    _VALID_URL = r'https?://(?:www\.)?1tv\.ru/(?:[^/]+/)+p?(?P<id>\d+)'
 
     _TESTS = [{
-        'url': 'http://www.1tv.ru/videoarchive/73390',
-        'md5': '777f525feeec4806130f4f764bc18a4f',
-        'info_dict': {
-            'id': '73390',
-            'ext': 'mp4',
-            'title': 'Олимпийские канатные дороги',
-            'description': 'md5:d41d8cd98f00b204e9800998ecf8427e',
-            'thumbnail': 're:^https?://.*\.(?:jpg|JPG)$',
-            'duration': 149,
-            'like_count': int,
-            'dislike_count': int,
-        },
-        'skip': 'Only works from Russia',
-    }, {
+        # single format via video_materials.json API
         'url': 'http://www.1tv.ru/prj/inprivate/vypusk/35930',
-        'md5': 'a1b6b60d530ebcf8daacf4565762bbaf',
+        'md5': '82a2777648acae812d58b3f5bd42882b',
         'info_dict': {
             'id': '35930',
             'ext': 'mp4',
-            'title': 'Наедине со всеми. Людмила Сенчина',
-            'description': 'md5:89553aed1d641416001fe8d450f06cb9',
+            'title': 'Гость Людмила Сенчина. Наедине со всеми. Выпуск от 12.02.2015',
+            'description': 'md5:357933adeede13b202c7c21f91b871b2',
             'thumbnail': 're:^https?://.*\.(?:jpg|JPG)$',
+            'upload_date': '20150212',
             'duration': 2694,
         },
-        'skip': 'Only works from Russia',
+    }, {
+        # multiple formats via video_materials.json API
+        'url': 'http://www.1tv.ru/video_archive/projects/dobroeutro/p113641',
+        'info_dict': {
+            'id': '113641',
+            'ext': 'mp4',
+            'title': 'Весенняя аллергия. Доброе утро. Фрагмент выпуска от 07.04.2016',
+            'description': 'md5:8dcebb3dded0ff20fade39087fd1fee2',
+            'thumbnail': 're:^https?://.*\.(?:jpg|JPG)$',
+            'upload_date': '20160407',
+            'duration': 179,
+            'formats': 'mincount:3',
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }, {
+        # single format only available via ONE_ONLINE_VIDEOS.archive_single_xml API
+        'url': 'http://www.1tv.ru/video_archive/series/f7552/p47038',
+        'md5': '519d306c5b5669761fd8906c39dbee23',
+        'info_dict': {
+            'id': '47038',
+            'ext': 'mp4',
+            'title': '"Побег". Второй сезон. 3 серия',
+            'description': 'md5:3abf8f6b9bce88201c33e9a3d794a00b',
+            'thumbnail': 're:^https?://.*\.(?:jpg|JPG)$',
+            'upload_date': '20120516',
+            'duration': 3080,
+        },
+    }, {
+        'url': 'http://www.1tv.ru/videoarchive/9967',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        webpage = self._download_webpage(url, video_id, 'Downloading page')
+        # Videos with multiple formats only available via this API
+        video = self._download_json(
+            'http://www.1tv.ru/video_materials.json?legacy_id=%s' % video_id,
+            video_id, fatal=False)
 
-        video_url = self._html_search_regex(
-            r'''(?s)(?:jwplayer\('flashvideoportal_1'\)\.setup\({|var\s+playlistObj\s*=).*?'file'\s*:\s*'([^']+)'.*?}\);''',
-            webpage, 'video URL')
+        description, thumbnail, upload_date, duration = [None] * 4
 
-        title = self._html_search_regex(
-            [r'<div class="tv_translation">\s*<h1><a href="[^"]+">([^<]*)</a>',
-             r"'title'\s*:\s*'([^']+)'"], webpage, 'title')
-        description = self._html_search_regex(
-            r'<div class="descr">\s*<div>&nbsp;</div>\s*<p>([^<]*)</p></div>',
-            webpage, 'description', default=None) or self._html_search_meta(
+        if video:
+            item = video[0]
+            title = item['title']
+            quality = qualities(('ld', 'sd', 'hd', ))
+            formats = [{
+                'url': f['src'],
+                'format_id': f.get('name'),
+                'quality': quality(f.get('name')),
+            } for f in item['mbr'] if f.get('src')]
+            thumbnail = item.get('poster')
+        else:
+            # Some videos are not available via video_materials.json
+            video = self._download_xml(
+                'http://www.1tv.ru/owa/win/ONE_ONLINE_VIDEOS.archive_single_xml?pid=%s' % video_id,
+                video_id)
+
+            NS_MAP = {
+                'media': 'http://search.yahoo.com/mrss/',
+            }
+
+            item = xpath_element(video, './channel/item', fatal=True)
+            title = xpath_text(item, './title', fatal=True)
+            formats = [{
+                'url': content.attrib['url'],
+            } for content in item.findall(
+                compat_xpath(xpath_with_ns('./media:content', NS_MAP))) if content.attrib.get('url')]
+            thumbnail = xpath_attr(
+                item, xpath_with_ns('./media:thumbnail', NS_MAP), 'url')
+
+        self._sort_formats(formats)
+
+        webpage = self._download_webpage(url, video_id, 'Downloading page', fatal=False)
+        if webpage:
+            title = self._html_search_regex(
+                (r'<div class="tv_translation">\s*<h1><a href="[^"]+">([^<]*)</a>',
+                 r"'title'\s*:\s*'([^']+)'"),
+                webpage, 'title', default=None) or title
+            description = self._html_search_regex(
+                r'<div class="descr">\s*<div>&nbsp;</div>\s*<p>([^<]*)</p></div>',
+                webpage, 'description', default=None) or self._html_search_meta(
                 'description', webpage, 'description')
-
-        thumbnail = self._og_search_thumbnail(webpage)
-        duration = self._og_search_property(
-            'video:duration', webpage,
-            'video duration', fatal=False)
-
-        like_count = self._html_search_regex(
-            r'title="Понравилось".*?/></label> \[(\d+)\]',
-            webpage, 'like count', default=None)
-        dislike_count = self._html_search_regex(
-            r'title="Не понравилось".*?/></label> \[(\d+)\]',
-            webpage, 'dislike count', default=None)
+            thumbnail = thumbnail or self._og_search_thumbnail(webpage)
+            duration = int_or_none(self._html_search_meta(
+                'video:duration', webpage, 'video duration', fatal=False))
+            upload_date = unified_strdate(self._html_search_meta(
+                'ya:ovs:upload_date', webpage, 'upload date', fatal=False))
 
         return {
             'id': video_id,
-            'url': video_url,
             'thumbnail': thumbnail,
             'title': title,
             'description': description,
+            'upload_date': upload_date,
             'duration': int_or_none(duration),
-            'like_count': int_or_none(like_count),
-            'dislike_count': int_or_none(dislike_count),
+            'formats': formats
         }

From 6a801f44704c3df49563852108c104c43a0551cc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 9 Apr 2016 23:18:41 +0200
Subject: [PATCH 4581/6219] [test/InfoExtractors] add test for _download_json

---
 test/test_InfoExtractor.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/test/test_InfoExtractor.py b/test/test_InfoExtractor.py
index 938466a80..6404ac89f 100644
--- a/test/test_InfoExtractor.py
+++ b/test/test_InfoExtractor.py
@@ -11,6 +11,7 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 from test.helper import FakeYDL
 from youtube_dl.extractor.common import InfoExtractor
 from youtube_dl.extractor import YoutubeIE, get_info_extractor
+from youtube_dl.utils import encode_data_uri, strip_jsonp, ExtractorError
 
 
 class TestIE(InfoExtractor):
@@ -66,5 +67,14 @@ class TestInfoExtractor(unittest.TestCase):
         self.assertEqual(ie._html_search_meta('e', html), '5')
         self.assertEqual(ie._html_search_meta('f', html), '6')
 
+    def test_download_json(self):
+        uri = encode_data_uri(b'{"foo": "blah"}', 'application/json')
+        self.assertEqual(self.ie._download_json(uri, None), {'foo': 'blah'})
+        uri = encode_data_uri(b'callback({"foo": "blah"})', 'application/javascript')
+        self.assertEqual(self.ie._download_json(uri, None, transform_source=strip_jsonp), {'foo': 'blah'})
+        uri = encode_data_uri(b'{"foo": invalid}', 'application/json')
+        self.assertRaises(ExtractorError, self.ie._download_json, uri, None)
+        self.assertEqual(self.ie._download_json(uri, None, fatal=False), None)
+
 if __name__ == '__main__':
     unittest.main()

From 49caf3307f1ae713acaeed651984a6338293b8d3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 10 Apr 2016 17:10:27 +0600
Subject: [PATCH 4582/6219] [extractor/common] Remove irrelevant comment

---
 youtube_dl/extractor/common.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 17d00721c..5269059d0 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -376,7 +376,6 @@ class InfoExtractor(object):
                 self.to_screen('%s' % (note,))
             else:
                 self.to_screen('%s: %s' % (video_id, note))
-        # data, headers and query params will be ignored for `Request` objects
         if isinstance(url_or_request, compat_urllib_request.Request):
             url_or_request = update_Request(
                 url_or_request, data=data, headers=headers, query=query)

From a1fa60a9340f61a8455a0cd85c18f63d9bdfe681 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 10 Apr 2016 18:43:40 +0600
Subject: [PATCH 4583/6219] [cliprs] Add extractor (Closes #9099)

---
 youtube_dl/extractor/cliprs.py     | 90 ++++++++++++++++++++++++++++++
 youtube_dl/extractor/extractors.py |  7 +++
 2 files changed, 97 insertions(+)
 create mode 100644 youtube_dl/extractor/cliprs.py

diff --git a/youtube_dl/extractor/cliprs.py b/youtube_dl/extractor/cliprs.py
new file mode 100644
index 000000000..4f9320ea5
--- /dev/null
+++ b/youtube_dl/extractor/cliprs.py
@@ -0,0 +1,90 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    float_or_none,
+    int_or_none,
+    parse_iso8601,
+)
+
+
+class ClipRsIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?clip\.rs/(?P<id>[^/]+)/\d+'
+    _TEST = {
+        'url': 'http://www.clip.rs/premijera-frajle-predstavljaju-novi-spot-za-pesmu-moli-me-moli/3732',
+        'md5': 'c412d57815ba07b56f9edc7b5d6a14e5',
+        'info_dict': {
+            'id': '1488842.1399140381',
+            'ext': 'mp4',
+            'title': 'PREMIJERA Frajle predstavljaju novi spot za pesmu Moli me, moli',
+            'description': 'md5:56ce2c3b4ab31c5a2e0b17cb9a453026',
+            'duration': 229,
+            'timestamp': 1459850243,
+            'upload_date': '20160405',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        video_id = self._search_regex(
+            r'id=(["\'])mvp:(?P<id>.+?)\1', webpage, 'mvp id', group='id')
+
+        response = self._download_json(
+            'http://qi.ckm.onetapi.pl/', video_id,
+            query={
+                'body[id]': video_id,
+                'body[jsonrpc]': '2.0',
+                'body[method]': 'get_asset_detail',
+                'body[params][ID_Publikacji]': video_id,
+                'body[params][Service]': 'www.onet.pl',
+                'content-type': 'application/jsonp',
+                'x-onet-app': 'player.front.onetapi.pl',
+            })
+
+        error = response.get('error')
+        if error:
+            raise ExtractorError(
+                '%s said: %s' % (self.IE_NAME, error['message']), expected=True)
+
+        video = response['result'].get('0')
+
+        formats = []
+        for _, formats_dict in video['formats'].items():
+            if not isinstance(formats_dict, dict):
+                continue
+            for format_id, format_list in formats_dict.items():
+                if not isinstance(format_list, list):
+                    continue
+                for f in format_list:
+                    if not f.get('url'):
+                        continue
+                    formats.append({
+                        'url': f['url'],
+                        'format_id': format_id,
+                        'height': int_or_none(f.get('vertical_resolution')),
+                        'width': int_or_none(f.get('horizontal_resolution')),
+                        'abr': float_or_none(f.get('audio_bitrate')),
+                        'vbr': float_or_none(f.get('video_bitrate')),
+                    })
+        self._sort_formats(formats)
+
+        meta = video.get('meta', {})
+
+        title = self._og_search_title(webpage, default=None) or meta['title']
+        description = self._og_search_description(webpage, default=None) or meta.get('description')
+        duration = meta.get('length') or meta.get('lenght')
+        timestamp = parse_iso8601(meta.get('addDate'), ' ')
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'duration': duration,
+            'timestamp': timestamp,
+            'formats': formats,
+        }
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index de29c7956..aefc4df01 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -123,6 +123,7 @@ from .chirbit import (
 )
 from .cinchcast import CinchcastIE
 from .cinemassacre import CinemassacreIE
+from .cliprs import ClipRsIE
 from .clipfish import ClipfishIE
 from .cliphunter import CliphunterIE
 from .clipsyndicate import ClipsyndicateIE
@@ -939,6 +940,12 @@ from .xhamster import (
     XHamsterIE,
     XHamsterEmbedIE,
 )
+from .xiami import (
+    XiamiIE,
+    XiamiAlbumIE,
+    XiamiArtistIE,
+    XiamiCollectionIE
+)
 from .xminus import XMinusIE
 from .xnxx import XNXXIE
 from .xstream import XstreamIE

From f44c2768421bc3b0ead3ccf86b5e499d498674c7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 10 Apr 2016 19:21:58 +0600
Subject: [PATCH 4584/6219] [extractor/extractors] Remove non-existant imports

---
 youtube_dl/extractor/extractors.py | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index aefc4df01..c1a13c982 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -940,12 +940,6 @@ from .xhamster import (
     XHamsterIE,
     XHamsterEmbedIE,
 )
-from .xiami import (
-    XiamiIE,
-    XiamiAlbumIE,
-    XiamiArtistIE,
-    XiamiCollectionIE
-)
 from .xminus import XMinusIE
 from .xnxx import XNXXIE
 from .xstream import XstreamIE

From de728757ad7218ce175649ec0d3f0b5723f2c580 Mon Sep 17 00:00:00 2001
From: Philip Huppert <philip@zeilen-sprung.de>
Date: Sun, 10 Apr 2016 16:36:44 +0200
Subject: [PATCH 4585/6219] [presstv] Refactored extractor.

---
 youtube_dl/extractor/presstv.py | 52 +++++++++++++++------------------
 1 file changed, 23 insertions(+), 29 deletions(-)

diff --git a/youtube_dl/extractor/presstv.py b/youtube_dl/extractor/presstv.py
index 9af6780c1..755e32528 100644
--- a/youtube_dl/extractor/presstv.py
+++ b/youtube_dl/extractor/presstv.py
@@ -17,10 +17,8 @@ class PressTVIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Organic mattresses used to clean waste water',
             'upload_date': '20160409',
-            'thumbnail': 'http://media.presstv.com/photo/20160409/41719129-76fa-4372-a09d-bf348278eb5d.jpg',
-            'description': ('A trial program at an Australian sewerage treatment facility hopes to change '
-                            'the way waste water is treated by using plant mattresses to reduce chemical '
-                            'and electricity use.')
+            'thumbnail': 're:^https?://.*\.jpg',
+            'description': 'md5:20002e654bbafb6908395a5c0cfcd125'
         }
     }
 
@@ -35,38 +33,34 @@ class PressTVIE(InfoExtractor):
         # build list of available formats
         # specified in http://www.presstv.ir/Scripts/playback.js
         base_url = 'http://192.99.219.222:82/presstv'
-        formats = [
-            {
-                'url': base_url + video_url,
-                'format': '1080p mp4',
-                'format_id': '1080p'
-            }, {
-                'url': base_url + video_url.replace(".mp4", "_low800.mp4"),
-                'format': '720p mp4',
-                'format_id': '720p'
-            }, {
-                'url': base_url + video_url.replace(".mp4", "_low400.mp4"),
-                'format': '360p mp4',
-                'format_id': '360p'
-            }, {
-                'url': base_url + video_url.replace(".mp4", "_low200.mp4"),
-                'format': '180p mp4',
-                'format_id': '180p'
-            }
+        _formats = [
+            ("180p", "_low200.mp4"),
+            ("360p", "_low400.mp4"),
+            ("720p", "_low800.mp4"),
+            ("1080p", ".mp4")
         ]
-        formats.reverse()
+
+        formats = []
+        for fmt in _formats:
+            format_id, extension = fmt
+            formats.append({
+                'url': base_url + video_url[:-4] + extension,
+                'format_id': format_id
+            })
 
         # extract video metadata
         title = self._html_search_meta('title', webpage, 'Title', True)
         title = title.partition('-')[2].strip()
 
-        thumbnail = self._html_search_meta('og:image', webpage, 'Thumbnail', True)
-        description = self._html_search_meta('og:description', webpage, 'Description', True)
+        thumbnail = self._og_search_thumbnail(webpage)
+        description = self._og_search_description(webpage)
 
-        year = str_to_int(self._search_regex(PressTVIE._VALID_URL, url, 'Upload year', group='y'))
-        month = str_to_int(self._search_regex(PressTVIE._VALID_URL, url, 'Upload month', group='m'))
-        day = str_to_int(self._search_regex(PressTVIE._VALID_URL, url, 'Upload day', group='d'))
-        upload_date = '%04d%02d%02d' % (year, month, day)
+        match = re.match(PressTVIE._VALID_URL, url)
+        upload_date = '%04d%02d%02d' % (
+            str_to_int(match.group('y')),
+            str_to_int(match.group('m')),
+            str_to_int(match.group('d'))
+        )
 
         return {
             'id': video_id,

From 443285aabef470f546f0b01b8e8194ca988bb315 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 10 Apr 2016 22:15:11 +0600
Subject: [PATCH 4586/6219] [ebaumsworlds] Update _VALID_URL (Closes #9135)

---
 youtube_dl/extractor/ebaumsworld.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/ebaumsworld.py b/youtube_dl/extractor/ebaumsworld.py
index b6bfd2b2d..c97682cd3 100644
--- a/youtube_dl/extractor/ebaumsworld.py
+++ b/youtube_dl/extractor/ebaumsworld.py
@@ -4,10 +4,10 @@ from .common import InfoExtractor
 
 
 class EbaumsWorldIE(InfoExtractor):
-    _VALID_URL = r'https?://www\.ebaumsworld\.com/video/watch/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?ebaumsworld\.com/videos/[^/]+/(?P<id>\d+)'
 
     _TEST = {
-        'url': 'http://www.ebaumsworld.com/video/watch/83367677/',
+        'url': 'http://www.ebaumsworld.com/videos/a-giant-python-opens-the-door/83367677/',
         'info_dict': {
             'id': '83367677',
             'ext': 'mp4',

From 66fa49586879418e357337ff82794fe851e71e7e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 10 Apr 2016 22:37:14 +0600
Subject: [PATCH 4587/6219] [screencastomatic] Fix extraction (Closes #9136)

---
 youtube_dl/extractor/screencastomatic.py | 35 ++++++++----------------
 1 file changed, 11 insertions(+), 24 deletions(-)

diff --git a/youtube_dl/extractor/screencastomatic.py b/youtube_dl/extractor/screencastomatic.py
index 05337421c..c08c89d94 100644
--- a/youtube_dl/extractor/screencastomatic.py
+++ b/youtube_dl/extractor/screencastomatic.py
@@ -1,15 +1,11 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-from .common import InfoExtractor
-from ..compat import compat_urlparse
-from ..utils import (
-    ExtractorError,
-    js_to_json,
-)
+from .jwplatform import JWPlatformBaseIE
+from ..utils import js_to_json
 
 
-class ScreencastOMaticIE(InfoExtractor):
+class ScreencastOMaticIE(JWPlatformBaseIE):
     _VALID_URL = r'https?://screencast-o-matic\.com/watch/(?P<id>[0-9a-zA-Z]+)'
     _TEST = {
         'url': 'http://screencast-o-matic.com/watch/c2lD3BeOPl',
@@ -27,23 +23,14 @@ class ScreencastOMaticIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        setup_js = self._search_regex(
-            r"(?s)jwplayer\('mp4Player'\).setup\((\{.*?\})\);",
-            webpage, 'setup code')
-        data = self._parse_json(setup_js, video_id, transform_source=js_to_json)
-        try:
-            video_data = next(
-                m for m in data['modes'] if m.get('type') == 'html5')
-        except StopIteration:
-            raise ExtractorError('Could not find any video entries!')
-        video_url = compat_urlparse.urljoin(url, video_data['config']['file'])
-        thumbnail = data.get('image')
+        jwplayer_data = self._parse_json(
+            self._search_regex(
+                r"(?s)jwplayer\('mp4Player'\).setup\((\{.*?\})\);", webpage, 'setup code'),
+            video_id, transform_source=js_to_json)
 
-        return {
-            'id': video_id,
+        info_dict = self._parse_jwplayer_data(jwplayer_data, video_id, require_title=False)
+        info_dict.update({
             'title': self._og_search_title(webpage),
             'description': self._og_search_description(webpage),
-            'url': video_url,
-            'ext': 'mp4',
-            'thumbnail': thumbnail,
-        }
+        })
+        return info_dict

From a6d6722c8fc2174ce72ed462e649d397d1448a0a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 10 Apr 2016 22:47:38 +0600
Subject: [PATCH 4588/6219] [jwplatform:base] Extract duration

---
 youtube_dl/extractor/jwplatform.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/jwplatform.py b/youtube_dl/extractor/jwplatform.py
index 6770685d7..01601c59e 100644
--- a/youtube_dl/extractor/jwplatform.py
+++ b/youtube_dl/extractor/jwplatform.py
@@ -4,7 +4,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import int_or_none
+from ..utils import (
+    float_or_none,
+    int_or_none,
+)
 
 
 class JWPlatformBaseIE(InfoExtractor):
@@ -41,6 +44,7 @@ class JWPlatformBaseIE(InfoExtractor):
             'description': video_data.get('description'),
             'thumbnail': self._proto_relative_url(video_data.get('image')),
             'timestamp': int_or_none(video_data.get('pubdate')),
+            'duration': float_or_none(jwplayer_data.get('duration')),
             'subtitles': subtitles,
             'formats': formats,
         }

From d7eb052fa2ab26839b050a7c3fa3f8874d508a02 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 10 Apr 2016 22:48:04 +0600
Subject: [PATCH 4589/6219] [screencastomatic] Add duration to test

---
 youtube_dl/extractor/screencastomatic.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/screencastomatic.py b/youtube_dl/extractor/screencastomatic.py
index c08c89d94..7a88a42cd 100644
--- a/youtube_dl/extractor/screencastomatic.py
+++ b/youtube_dl/extractor/screencastomatic.py
@@ -16,6 +16,7 @@ class ScreencastOMaticIE(JWPlatformBaseIE):
             'title': 'Welcome to 3-4 Philosophy @ DECV!',
             'thumbnail': 're:^https?://.*\.jpg$',
             'description': 'as the title says! also: some general info re 1) VCE philosophy and 2) distance learning.',
+            'duration': 369.163,
         }
     }
 

From 7ebc36900d15888321a45f04113eeda169469004 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 10 Apr 2016 22:55:07 +0600
Subject: [PATCH 4590/6219] [jwplatform:base] Improve subtitles extraction

---
 youtube_dl/extractor/jwplatform.py | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/jwplatform.py b/youtube_dl/extractor/jwplatform.py
index 01601c59e..8a5e562db 100644
--- a/youtube_dl/extractor/jwplatform.py
+++ b/youtube_dl/extractor/jwplatform.py
@@ -13,10 +13,6 @@ from ..utils import (
 class JWPlatformBaseIE(InfoExtractor):
     def _parse_jwplayer_data(self, jwplayer_data, video_id, require_title=True):
         video_data = jwplayer_data['playlist'][0]
-        subtitles = {}
-        for track in video_data['tracks']:
-            if track['kind'] == 'captions':
-                subtitles[track['label']] = [{'url': self._proto_relative_url(track['file'])}]
 
         formats = []
         for source in video_data['sources']:
@@ -38,6 +34,15 @@ class JWPlatformBaseIE(InfoExtractor):
                 })
         self._sort_formats(formats)
 
+        subtitles = {}
+        tracks = video_data.get('tracks')
+        if tracks and isinstance(tracks, list):
+            for track in tracks:
+                if track.get('file') and track.get('kind') == 'captions':
+                    subtitles.setdefault(track.get('label') or 'en', []).append({
+                        'url': self._proto_relative_url(track['file'])
+                    })
+
         return {
             'id': video_id,
             'title': video_data['title'] if require_title else video_data.get('title'),

From 4a121d29bb0700beb19e8b6edb5d479e9fe7ac1b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 10 Apr 2016 23:45:17 +0600
Subject: [PATCH 4591/6219] [glide] Fix extraction (Closes #9141)

---
 youtube_dl/extractor/glide.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/glide.py b/youtube_dl/extractor/glide.py
index 9561ed5fb..0ab23f766 100644
--- a/youtube_dl/extractor/glide.py
+++ b/youtube_dl/extractor/glide.py
@@ -23,8 +23,9 @@ class GlideIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
         title = self._html_search_regex(
             r'<title>(.*?)</title>', webpage, 'title')
-        video_url = self.http_scheme() + self._search_regex(
-            r'<source src="(.*?)" type="video/mp4">', webpage, 'video URL')
+        video_url = self._proto_relative_url(self._search_regex(
+            r'<source[^>]+src=(["\'])(?P<url>.+?)\1',
+            webpage, 'video URL', group='url'), self.http_scheme())
         thumbnail_url = self._search_regex(
             r'<img id="video-thumbnail" src="(.*?)"',
             webpage, 'thumbnail url', fatal=False)

From 5899e988d55f4c64500721716bb99c5ecf86afc6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 10 Apr 2016 23:56:23 +0600
Subject: [PATCH 4592/6219] [glide] Improve extraction and extract upload info

---
 youtube_dl/extractor/glide.py | 28 ++++++++++++++++++++--------
 1 file changed, 20 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/glide.py b/youtube_dl/extractor/glide.py
index 0ab23f766..62ff84835 100644
--- a/youtube_dl/extractor/glide.py
+++ b/youtube_dl/extractor/glide.py
@@ -2,6 +2,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..utils import unified_strdate
 
 
 class GlideIE(InfoExtractor):
@@ -15,27 +16,38 @@ class GlideIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Damon Timm\'s Glide message',
             'thumbnail': 're:^https?://.*?\.cloudfront\.net/.*\.jpg$',
+            'uploader': 'Damon Timm',
+            'upload_date': '20140919',
         }
     }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
+
         webpage = self._download_webpage(url, video_id)
+
         title = self._html_search_regex(
-            r'<title>(.*?)</title>', webpage, 'title')
+            r'<title>(.+?)</title>', webpage, 'title')
         video_url = self._proto_relative_url(self._search_regex(
             r'<source[^>]+src=(["\'])(?P<url>.+?)\1',
-            webpage, 'video URL', group='url'), self.http_scheme())
-        thumbnail_url = self._search_regex(
-            r'<img id="video-thumbnail" src="(.*?)"',
-            webpage, 'thumbnail url', fatal=False)
-        thumbnail = (
-            thumbnail_url if thumbnail_url is None
-            else self.http_scheme() + thumbnail_url)
+            webpage, 'video URL', default=None,
+            group='url')) or self._og_search_video_url(webpage)
+        thumbnail = self._proto_relative_url(self._search_regex(
+            r'<img[^>]+id=["\']video-thumbnail["\'][^>]+src=(["\'])(?P<url>.+?)\1',
+            webpage, 'thumbnail url', default=None,
+            group='url')) or self._og_search_thumbnail(webpage)
+        uploader = self._search_regex(
+            r'<div[^>]+class=["\']info-name["\'][^>]*>([^<]+)',
+            webpage, 'uploader', fatal=False)
+        upload_date = unified_strdate(self._search_regex(
+            r'<div[^>]+class="info-date"[^>]*>([^<]+)',
+            webpage, 'upload date', fatal=False))
 
         return {
             'id': video_id,
             'title': title,
             'url': video_url,
             'thumbnail': thumbnail,
+            'uploader': uploader,
+            'upload_date': upload_date,
         }

From 452908b257da1a5b228a2c0522c89fff87296622 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 11 Apr 2016 00:06:05 +0600
Subject: [PATCH 4593/6219] [telebruxelles] Fix extraction (Closes #9142)

---
 youtube_dl/extractor/telebruxelles.py | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/telebruxelles.py b/youtube_dl/extractor/telebruxelles.py
index a3d05f97d..eefecc490 100644
--- a/youtube_dl/extractor/telebruxelles.py
+++ b/youtube_dl/extractor/telebruxelles.py
@@ -1,11 +1,13 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 
 
 class TeleBruxellesIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?telebruxelles\.be/(news|sport|dernier-jt)/?(?P<id>[^/#?]+)'
+    _VALID_URL = r'https?://(?:www\.)?(?:telebruxelles|bx1)\.be/(news|sport|dernier-jt)/?(?P<id>[^/#?]+)'
     _TESTS = [{
         'url': 'http://www.telebruxelles.be/news/auditions-devant-parlement-francken-galant-tres-attendus/',
         'md5': '59439e568c9ee42fb77588b2096b214f',
@@ -39,18 +41,18 @@ class TeleBruxellesIE(InfoExtractor):
         webpage = self._download_webpage(url, display_id)
 
         article_id = self._html_search_regex(
-            r"<article id=\"post-(\d+)\"", webpage, 'article ID')
+            r"<article id=\"post-(\d+)\"", webpage, 'article ID', default=None)
         title = self._html_search_regex(
             r'<h1 class=\"entry-title\">(.*?)</h1>', webpage, 'title')
-        description = self._og_search_description(webpage)
+        description = self._og_search_description(webpage, default=None)
 
         rtmp_url = self._html_search_regex(
-            r"file: \"(rtmp://\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5}/vod/mp4:\" \+ \"\w+\" \+ \".mp4)\"",
+            r'file\s*:\s*"(rtmp://[^/]+/vod/mp4:"\s*\+\s*"[^"]+"\s*\+\s*".mp4)"',
             webpage, 'RTMP url')
-        rtmp_url = rtmp_url.replace("\" + \"", "")
+        rtmp_url = re.sub(r'"\s*\+\s*"', '', rtmp_url)
 
         return {
-            'id': article_id,
+            'id': article_id or display_id,
             'display_id': display_id,
             'title': title,
             'description': description,

From dfbc7f7f3f44ff7f9ed2beff76dc37edbb66af8d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 11 Apr 2016 16:14:07 +0800
Subject: [PATCH 4594/6219] [presstv] Improve and simplify

---
 youtube_dl/extractor/presstv.py | 48 +++++++++++++++++----------------
 1 file changed, 25 insertions(+), 23 deletions(-)

diff --git a/youtube_dl/extractor/presstv.py b/youtube_dl/extractor/presstv.py
index 755e32528..2da93ed34 100644
--- a/youtube_dl/extractor/presstv.py
+++ b/youtube_dl/extractor/presstv.py
@@ -1,19 +1,21 @@
 # coding: utf-8
 from __future__ import unicode_literals
+
 import re
 
 from .common import InfoExtractor
-from ..utils import str_to_int
+from ..utils import remove_start
 
 
 class PressTVIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?presstv\.ir/[^/]+/(?P<y>[0-9]+)/(?P<m>[0-9]+)/(?P<d>[0-9]+)/(?P<id>[0-9]+)/'
+    _VALID_URL = r'https?://(?:www\.)?presstv\.ir/[^/]+/(?P<y>\d+)/(?P<m>\d+)/(?P<d>\d+)/(?P<id>\d+)/(?P<display_id>[^/]+)?'
 
     _TEST = {
         'url': 'http://www.presstv.ir/Detail/2016/04/09/459911/Australian-sewerage-treatment-facility-/',
         'md5': '5d7e3195a447cb13e9267e931d8dd5a5',
         'info_dict': {
             'id': '459911',
+            'display_id': 'Australian-sewerage-treatment-facility-',
             'ext': 'mp4',
             'title': 'Organic mattresses used to clean waste water',
             'upload_date': '20160409',
@@ -23,47 +25,47 @@ class PressTVIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        display_id = mobj.group('display_id') or video_id
+
+        webpage = self._download_webpage(url, display_id)
 
         # extract video URL from webpage
-        video_url = self._html_search_regex(r'<input type="hidden" id="inpPlayback" value="([^"]+)" />', webpage,
-                                            'Video URL')
+        video_url = self._hidden_inputs(webpage)['inpPlayback']
 
         # build list of available formats
         # specified in http://www.presstv.ir/Scripts/playback.js
         base_url = 'http://192.99.219.222:82/presstv'
         _formats = [
-            ("180p", "_low200.mp4"),
-            ("360p", "_low400.mp4"),
-            ("720p", "_low800.mp4"),
-            ("1080p", ".mp4")
+            (180, '_low200.mp4'),
+            (360, '_low400.mp4'),
+            (720, '_low800.mp4'),
+            (1080, '.mp4')
         ]
 
-        formats = []
-        for fmt in _formats:
-            format_id, extension = fmt
-            formats.append({
-                'url': base_url + video_url[:-4] + extension,
-                'format_id': format_id
-            })
+        formats = [{
+            'url': base_url + video_url[:-4] + extension,
+            'format_id': '%dp' % height,
+            'height': height,
+        } for height, extension in _formats]
 
         # extract video metadata
-        title = self._html_search_meta('title', webpage, 'Title', True)
-        title = title.partition('-')[2].strip()
+        title = remove_start(
+            self._html_search_meta('title', webpage, fatal=True), 'PressTV-')
 
         thumbnail = self._og_search_thumbnail(webpage)
         description = self._og_search_description(webpage)
 
-        match = re.match(PressTVIE._VALID_URL, url)
         upload_date = '%04d%02d%02d' % (
-            str_to_int(match.group('y')),
-            str_to_int(match.group('m')),
-            str_to_int(match.group('d'))
+            int(mobj.group('y')),
+            int(mobj.group('m')),
+            int(mobj.group('d')),
         )
 
         return {
             'id': video_id,
+            'display_id': display_id,
             'title': title,
             'formats': formats,
             'thumbnail': thumbnail,

From 134c207e3faf1ad38a23e31d5067eafe0ef8e92a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 11 Apr 2016 19:17:11 +0800
Subject: [PATCH 4595/6219] [arte.tv:embed] Extended support (#2620)

---
 youtube_dl/extractor/arte.py    |  2 +-
 youtube_dl/extractor/generic.py | 14 +++++++++++++-
 2 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index ae0f27dcb..f042d9163 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -337,7 +337,7 @@ class ArteTVEmbedIE(ArteTVPlus7IE):
     IE_NAME = 'arte.tv:embed'
     _VALID_URL = r'''(?x)
         http://www\.arte\.tv
-        /playerv2/embed\.php\?json_url=
+        /(?:playerv2/embed|arte_vp/index)\.php\?json_url=
         (?P<json_url>
             http://arte\.tv/papi/tvguide/videos/stream/player/
             (?P<lang>[^/]+)/(?P<id>[^/]+)[^&]*
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 589d1e152..2aadd6a12 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1128,6 +1128,18 @@ class GenericIE(InfoExtractor):
                 'skip_download': True,
             },
         },
+        # Another form of arte.tv embed
+        {
+            'url': 'http://www.tv-replay.fr/redirection/09-04-16/arte-reportage-arte-11508975.html',
+            'md5': '850bfe45417ddf221288c88a0cffe2e2',
+            'info_dict': {
+                'id': '030273-562_PLUS7-F',
+                'ext': 'mp4',
+                'title': 'ARTE Reportage - Nulle part, en France',
+                'description': 'md5:e3a0e8868ed7303ed509b9e3af2b870d',
+                'upload_date': '20160409',
+            },
+        },
     ]
 
     def report_following_redirect(self, new_url):
@@ -1702,7 +1714,7 @@ class GenericIE(InfoExtractor):
 
         # Look for embedded arte.tv player
         mobj = re.search(
-            r'<script [^>]*?src="(?P<url>http://www\.arte\.tv/playerv2/embed[^"]+)"',
+            r'<(?:script|iframe) [^>]*?src="(?P<url>http://www\.arte\.tv/(?:playerv2/embed|arte_vp/index)[^"]+)"',
             webpage)
         if mobj is not None:
             return self.url_result(mobj.group('url'), 'ArteTVEmbed')

From 13267a2be37371bd3e8aea0bc7549f4ed2465c03 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 11 Apr 2016 14:23:55 +0100
Subject: [PATCH 4596/6219] [openclassroom] Add new extractor(closes #9147)

---
 youtube_dl/extractor/extractors.py    |  1 +
 youtube_dl/extractor/openclassroom.py | 42 +++++++++++++++++++++++++++
 2 files changed, 43 insertions(+)
 create mode 100644 youtube_dl/extractor/openclassroom.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index c234ff127..21159f5a1 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -544,6 +544,7 @@ from .ooyala import (
     OoyalaIE,
     OoyalaExternalIE,
 )
+from .openclassroom import OpenClassRoomIE
 from .openload import OpenloadIE
 from .ora import OraTVIE
 from .orf import (
diff --git a/youtube_dl/extractor/openclassroom.py b/youtube_dl/extractor/openclassroom.py
new file mode 100644
index 000000000..bbc1af7c9
--- /dev/null
+++ b/youtube_dl/extractor/openclassroom.py
@@ -0,0 +1,42 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..compat import compat_parse_qs
+from ..utils import (
+    ExtractorError,
+    xpath_text,
+    clean_html,
+)
+
+
+class OpenClassRoomIE(InfoExtractor):
+    _VALID_URL = r'https?://openclassroom\.stanford\.edu/MainFolder/VideoPage\.php\?(?P<query>.*)'
+    _TEST = {
+        'url': 'http://openclassroom.stanford.edu/MainFolder/VideoPage.php?course=PracticalUnix&video=intro-environment&speed=100',
+        'md5': '544a9468546059d4e80d76265b0443b8',
+        'info_dict': {
+            'id': 'intro-environment',
+            'ext': 'mp4',
+            'title': 'Intro Environment',
+            'description': 'md5:7d57306c8649f814ca00bb80dada600e',
+        }
+    }
+    _URL_TEMPLATE = 'http://openclassroom.stanford.edu/MainFolder/courses/%s/videos/%s'
+
+    def _real_extract(self, url):
+        qs = compat_parse_qs(re.match(self._VALID_URL, url).group('query'))
+        if not qs.get('course') or not qs.get('video'):
+            raise ExtractorError('Unsupported URL', expected=True)
+        video_id = qs['video'][0]
+        video_doc = self._download_xml(
+            self._URL_TEMPLATE % (qs['course'][0], video_id + '.xml'), video_id)
+        return {
+            'id': video_id,
+            'title': xpath_text(video_doc, 'title', 'title', True),
+            'url': self._URL_TEMPLATE % (qs['course'][0], xpath_text(
+                video_doc, 'videoFile', 'video url', True)),
+            'description': clean_html(xpath_text(video_doc, 'text')),
+        }

From 4a903b93a93eb49bb7942cfe62a338cb947f78bc Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 11 Apr 2016 14:44:35 +0100
Subject: [PATCH 4597/6219] Revert "[openclassroom] Add new extractor(closes
 #9147)"

This reverts commit 13267a2be37371bd3e8aea0bc7549f4ed2465c03.
---
 youtube_dl/extractor/extractors.py    |  1 -
 youtube_dl/extractor/openclassroom.py | 42 ---------------------------
 2 files changed, 43 deletions(-)
 delete mode 100644 youtube_dl/extractor/openclassroom.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 21159f5a1..c234ff127 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -544,7 +544,6 @@ from .ooyala import (
     OoyalaIE,
     OoyalaExternalIE,
 )
-from .openclassroom import OpenClassRoomIE
 from .openload import OpenloadIE
 from .ora import OraTVIE
 from .orf import (
diff --git a/youtube_dl/extractor/openclassroom.py b/youtube_dl/extractor/openclassroom.py
deleted file mode 100644
index bbc1af7c9..000000000
--- a/youtube_dl/extractor/openclassroom.py
+++ /dev/null
@@ -1,42 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-import re
-
-from .common import InfoExtractor
-from ..compat import compat_parse_qs
-from ..utils import (
-    ExtractorError,
-    xpath_text,
-    clean_html,
-)
-
-
-class OpenClassRoomIE(InfoExtractor):
-    _VALID_URL = r'https?://openclassroom\.stanford\.edu/MainFolder/VideoPage\.php\?(?P<query>.*)'
-    _TEST = {
-        'url': 'http://openclassroom.stanford.edu/MainFolder/VideoPage.php?course=PracticalUnix&video=intro-environment&speed=100',
-        'md5': '544a9468546059d4e80d76265b0443b8',
-        'info_dict': {
-            'id': 'intro-environment',
-            'ext': 'mp4',
-            'title': 'Intro Environment',
-            'description': 'md5:7d57306c8649f814ca00bb80dada600e',
-        }
-    }
-    _URL_TEMPLATE = 'http://openclassroom.stanford.edu/MainFolder/courses/%s/videos/%s'
-
-    def _real_extract(self, url):
-        qs = compat_parse_qs(re.match(self._VALID_URL, url).group('query'))
-        if not qs.get('course') or not qs.get('video'):
-            raise ExtractorError('Unsupported URL', expected=True)
-        video_id = qs['video'][0]
-        video_doc = self._download_xml(
-            self._URL_TEMPLATE % (qs['course'][0], video_id + '.xml'), video_id)
-        return {
-            'id': video_id,
-            'title': xpath_text(video_doc, 'title', 'title', True),
-            'url': self._URL_TEMPLATE % (qs['course'][0], xpath_text(
-                video_doc, 'videoFile', 'video url', True)),
-            'description': clean_html(xpath_text(video_doc, 'text')),
-        }

From 49f523ca506f3b57e8a54b97c6a29dc71624aeef Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 11 Apr 2016 20:45:58 +0600
Subject: [PATCH 4598/6219] [mixcloud] Capture error message (#9156)

---
 youtube_dl/extractor/mixcloud.py | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dl/extractor/mixcloud.py
index 101497118..9638cc9e6 100644
--- a/youtube_dl/extractor/mixcloud.py
+++ b/youtube_dl/extractor/mixcloud.py
@@ -7,6 +7,7 @@ from ..compat import compat_urllib_parse_unquote
 from ..utils import (
     ExtractorError,
     HEADRequest,
+    NO_DEFAULT,
     parse_count,
     str_to_int,
 )
@@ -63,8 +64,17 @@ class MixcloudIE(InfoExtractor):
 
         webpage = self._download_webpage(url, track_id)
 
+        message = self._html_search_regex(
+            r'(?s)<div[^>]+class="global-message cloudcast-disabled-notice-light"[^>]*>(.+?)<(?:a|/div)',
+            webpage, 'error message', default=None)
+
         preview_url = self._search_regex(
-            r'\s(?:data-preview-url|m-preview)="([^"]+)"', webpage, 'preview url')
+            r'\s(?:data-preview-url|m-preview)="([^"]+)"',
+            webpage, 'preview url', default=None if message else NO_DEFAULT)
+
+        if message:
+            raise ExtractorError('%s said: %s' % (self.IE_NAME, message), expected=True)
+
         song_url = re.sub(r'audiocdn(\d+)', r'stream\1', preview_url)
         song_url = song_url.replace('/previews/', '/c/originals/')
         if not self._check_url(song_url, track_id, 'mp3'):

From 12355ac4734507fbafba4ae0d876ff5fdecc08f9 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 12 Apr 2016 17:23:00 +0800
Subject: [PATCH 4599/6219] [goshgay] Fix extraction

isFamilyFriendly no longer exists in the webpage and I can't find
another indicator.
---
 youtube_dl/extractor/goshgay.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/goshgay.py b/youtube_dl/extractor/goshgay.py
index 1d9166455..0c015141f 100644
--- a/youtube_dl/extractor/goshgay.py
+++ b/youtube_dl/extractor/goshgay.py
@@ -14,13 +14,13 @@ class GoshgayIE(InfoExtractor):
     _VALID_URL = r'https?://www\.goshgay\.com/video(?P<id>\d+?)($|/)'
     _TEST = {
         'url': 'http://www.goshgay.com/video299069/diesel_sfw_xxx_video',
-        'md5': '027fcc54459dff0feb0bc06a7aeda680',
+        'md5': '4b6db9a0a333142eb9f15913142b0ed1',
         'info_dict': {
             'id': '299069',
             'ext': 'flv',
             'title': 'DIESEL SFW XXX Video',
             'thumbnail': 're:^http://.*\.jpg$',
-            'duration': 79,
+            'duration': 80,
             'age_limit': 18,
         }
     }
@@ -47,5 +47,5 @@ class GoshgayIE(InfoExtractor):
             'title': title,
             'thumbnail': thumbnail,
             'duration': duration,
-            'age_limit': self._family_friendly_search(webpage),
+            'age_limit': 18,
         }

From c77a0c01cb409d067770bb5ae6ef82064856e027 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 12 Apr 2016 17:26:09 +0800
Subject: [PATCH 4600/6219] [groupon] Fix extraction

---
 youtube_dl/extractor/groupon.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/groupon.py b/youtube_dl/extractor/groupon.py
index 63c05b6a6..f6b69662b 100644
--- a/youtube_dl/extractor/groupon.py
+++ b/youtube_dl/extractor/groupon.py
@@ -16,14 +16,14 @@ class GrouponIE(InfoExtractor):
         'playlist': [{
             'info_dict': {
                 'id': 'tubGNycTo_9Uxg82uESj4i61EYX8nyuf',
-                'ext': 'mp4',
+                'ext': 'flv',
                 'title': 'Bikram Yoga Huntington Beach | Orange County',
                 'description': 'md5:d41d8cd98f00b204e9800998ecf8427e',
                 'duration': 44.961,
             },
         }],
         'params': {
-            'skip_download': 'HLS',
+            'skip_download': 'HDS',
         }
     }
 
@@ -32,7 +32,7 @@ class GrouponIE(InfoExtractor):
         webpage = self._download_webpage(url, playlist_id)
 
         payload = self._parse_json(self._search_regex(
-            r'var\s+payload\s*=\s*(.*?);\n', webpage, 'payload'), playlist_id)
+            r'(?:var\s+|window\.)payload\s*=\s*(.*?);\n', webpage, 'payload'), playlist_id)
         videos = payload['carousel'].get('dealVideos', [])
         entries = []
         for v in videos:

From 8865b8abfd266dee688f0a4c022109eb0f20aaaf Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 12 Apr 2016 17:30:14 +0800
Subject: [PATCH 4601/6219] [howstuffworks] Skip a broken test case

---
 youtube_dl/extractor/howstuffworks.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/howstuffworks.py b/youtube_dl/extractor/howstuffworks.py
index 76b74c51d..65ba2a48b 100644
--- a/youtube_dl/extractor/howstuffworks.py
+++ b/youtube_dl/extractor/howstuffworks.py
@@ -24,6 +24,7 @@ class HowStuffWorksIE(InfoExtractor):
                 'thumbnail': 're:^https?://.*\.jpg$',
                 'duration': 161,
             },
+            'skip': 'Video broken',
         },
         {
             'url': 'http://adventure.howstuffworks.com/7199-survival-zone-food-and-water-in-the-savanna-video.htm',

From c9d448876f7380d7e37c67981d8cb96c5cf0a153 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 12 Apr 2016 18:29:28 +0800
Subject: [PATCH 4602/6219] [izlesene] Fix extraction

description may be absent
---
 youtube_dl/extractor/izlesene.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/izlesene.py b/youtube_dl/extractor/izlesene.py
index bc226fa67..aa0728abc 100644
--- a/youtube_dl/extractor/izlesene.py
+++ b/youtube_dl/extractor/izlesene.py
@@ -29,7 +29,7 @@ class IzleseneIE(InfoExtractor):
                 'ext': 'mp4',
                 'title': 'Sevinçten Çıldırtan Doğum Günü Hediyesi',
                 'description': 'md5:253753e2655dde93f59f74b572454f6d',
-                'thumbnail': 're:^http://.*\.jpg',
+                'thumbnail': 're:^https?://.*\.jpg',
                 'uploader_id': 'pelikzzle',
                 'timestamp': int,
                 'upload_date': '20140702',
@@ -44,8 +44,7 @@ class IzleseneIE(InfoExtractor):
                 'id': '17997',
                 'ext': 'mp4',
                 'title': 'Tarkan Dortmund 2006 Konseri',
-                'description': 'Tarkan Dortmund 2006 Konseri',
-                'thumbnail': 're:^http://.*\.jpg',
+                'thumbnail': 're:^https://.*\.jpg',
                 'uploader_id': 'parlayankiz',
                 'timestamp': int,
                 'upload_date': '20061112',
@@ -62,7 +61,7 @@ class IzleseneIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         title = self._og_search_title(webpage)
-        description = self._og_search_description(webpage)
+        description = self._og_search_description(webpage, default=None)
         thumbnail = self._proto_relative_url(
             self._og_search_thumbnail(webpage), scheme='http:')
 

From a8374160252e2bb7d52b85f7476e36b08e649d10 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 12 Apr 2016 18:30:53 +0800
Subject: [PATCH 4603/6219] [jadorecettepub] Remove extractor: website gone

---
 youtube_dl/extractor/extractors.py     |  1 -
 youtube_dl/extractor/jadorecettepub.py | 47 --------------------------
 2 files changed, 48 deletions(-)
 delete mode 100644 youtube_dl/extractor/jadorecettepub.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index c234ff127..04c6508f1 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -336,7 +336,6 @@ from .ivi import (
 )
 from .ivideon import IvideonIE
 from .izlesene import IzleseneIE
-from .jadorecettepub import JadoreCettePubIE
 from .jeuxvideo import JeuxVideoIE
 from .jove import JoveIE
 from .jwplatform import JWPlatformIE
diff --git a/youtube_dl/extractor/jadorecettepub.py b/youtube_dl/extractor/jadorecettepub.py
deleted file mode 100644
index 158c09a33..000000000
--- a/youtube_dl/extractor/jadorecettepub.py
+++ /dev/null
@@ -1,47 +0,0 @@
-# coding: utf-8
-
-from __future__ import unicode_literals
-
-import re
-
-from .common import InfoExtractor
-from .youtube import YoutubeIE
-
-
-class JadoreCettePubIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?jadorecettepub\.com/[0-9]{4}/[0-9]{2}/(?P<id>.*?)\.html'
-
-    _TEST = {
-        'url': 'http://www.jadorecettepub.com/2010/12/star-wars-massacre-par-les-japonais.html',
-        'md5': '401286a06067c70b44076044b66515de',
-        'info_dict': {
-            'id': 'jLMja3tr7a4',
-            'ext': 'mp4',
-            'title': 'La pire utilisation de Star Wars',
-            'description': "Jadorecettepub.com vous a gratifié de plusieurs pubs géniales utilisant Star Wars et Dark Vador plus particulièrement... Mais l'heure est venue de vous proposer une version totalement massacrée, venue du Japon.  Quand les Japonais détruisent l'image de Star Wars pour vendre du thon en boite, ça promet...",
-        },
-    }
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        display_id = mobj.group('id')
-
-        webpage = self._download_webpage(url, display_id)
-
-        title = self._html_search_regex(
-            r'<span style="font-size: x-large;"><b>(.*?)</b></span>',
-            webpage, 'title')
-        description = self._html_search_regex(
-            r'(?s)<div id="fb-root">(.*?)<script>', webpage, 'description',
-            fatal=False)
-        real_url = self._search_regex(
-            r'\[/postlink\](.*)endofvid', webpage, 'video URL')
-        video_id = YoutubeIE.extract_id(real_url)
-
-        return {
-            '_type': 'url_transparent',
-            'url': real_url,
-            'id': video_id,
-            'title': title,
-            'description': description,
-        }

From ffa2cecf7291a7b9aa40dba84a76c71fba20cc57 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 12 Apr 2016 21:20:31 +0600
Subject: [PATCH 4604/6219] [ard] Change subtitles extension to ttml (Closes
 #9169)

ttml is now served instead of srt
---
 youtube_dl/extractor/ard.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ard.py b/youtube_dl/extractor/ard.py
index 9fb84911a..26446c2fe 100644
--- a/youtube_dl/extractor/ard.py
+++ b/youtube_dl/extractor/ard.py
@@ -83,7 +83,7 @@ class ARDMediathekIE(InfoExtractor):
         subtitle_url = media_info.get('_subtitleUrl')
         if subtitle_url:
             subtitles['de'] = [{
-                'ext': 'srt',
+                'ext': 'ttml',
                 'url': subtitle_url,
             }]
 

From 466a6145372aa70f44a9b39c7fdeb05301a5485a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 12 Apr 2016 21:38:31 +0600
Subject: [PATCH 4605/6219] [youtube:playlist] Recognize popular uploads
 playlist as mix (Closes #9170)

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 188066561..5a102de51 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1884,7 +1884,7 @@ class YoutubePlaylistIE(YoutubePlaylistBaseInfoExtractor):
         if video:
             return video
 
-        if playlist_id.startswith('RD') or playlist_id.startswith('UL'):
+        if playlist_id.startswith(('RD', 'UL', 'PU')):
             # Mixes require a custom extraction process
             return self._extract_mix(playlist_id)
 

From c96eca426b6b4c963fafd3f3268ea1f39b3e2857 Mon Sep 17 00:00:00 2001
From: Philip Huppert <philip@zeilen-sprung.de>
Date: Mon, 5 Oct 2015 00:41:20 +0200
Subject: [PATCH 4606/6219] [mixcloud] Added support for user uploads,
 playlists, favorites and listens. Fixes #3750 and #5272

---
 youtube_dl/extractor/extractors.py |   6 +-
 youtube_dl/extractor/mixcloud.py   | 200 ++++++++++++++++++++++++++++-
 2 files changed, 203 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 04c6508f1..b06b717f5 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -411,7 +411,11 @@ from .minoto import MinotoIE
 from .miomio import MioMioIE
 from .mit import TechTVMITIE, MITIE, OCWMITIE
 from .mitele import MiTeleIE
-from .mixcloud import MixcloudIE
+from .mixcloud import (
+    MixcloudIE,
+    MixcloudUserIE,
+    MixcloudPlaylistIE
+)
 from .mlb import MLBIE
 from .mnet import MnetIE
 from .mpora import MporaIE
diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dl/extractor/mixcloud.py
index 9638cc9e6..20e64bab5 100644
--- a/youtube_dl/extractor/mixcloud.py
+++ b/youtube_dl/extractor/mixcloud.py
@@ -3,18 +3,22 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse_unquote
+from ..compat import (
+    compat_urllib_parse_unquote,
+    compat_urllib_request
+)
 from ..utils import (
     ExtractorError,
     HEADRequest,
     NO_DEFAULT,
     parse_count,
     str_to_int,
+    clean_html
 )
 
 
 class MixcloudIE(InfoExtractor):
-    _VALID_URL = r'^(?:https?://)?(?:www\.)?mixcloud\.com/([^/]+)/([^/]+)'
+    _VALID_URL = r'^(?:https?://)?(?:www\.)?mixcloud\.com/([^/]+)/(?!stream|uploads|favorites|listens|playlists)([^/]+)'
     IE_NAME = 'mixcloud'
 
     _TESTS = [{
@@ -115,3 +119,195 @@ class MixcloudIE(InfoExtractor):
             'view_count': view_count,
             'like_count': like_count,
         }
+
+
+class MixcloudUserIE(InfoExtractor):
+    """
+    Information extractor for Mixcloud users.
+    It can retrieve a list of a user's uploads, favorites or listens.
+    """
+
+    _VALID_URL = r'^(?:https?://)?(?:www\.)?mixcloud\.com/(?P<user>[^/]+)/(?P<type>uploads|favorites|listens)?/?$'
+    IE_NAME = 'mixcloud:user'
+
+    _TESTS = [{
+        'url': 'http://www.mixcloud.com/dholbach/',
+        'info_dict': {
+            'id': 'dholbach/uploads',
+            'title': 'Daniel Holbach (uploads)',
+            'description': 'md5:327af72d1efeb404a8216c27240d1370',
+        },
+        'playlist_mincount': 11
+    }, {
+        'url': 'http://www.mixcloud.com/dholbach/uploads/',
+        'info_dict': {
+            'id': 'dholbach/uploads',
+            'title': 'Daniel Holbach (uploads)',
+            'description': 'md5:327af72d1efeb404a8216c27240d1370',
+        },
+        'playlist_mincount': 11
+    }, {
+        'url': 'http://www.mixcloud.com/dholbach/favorites/',
+        'info_dict': {
+            'id': 'dholbach/favorites',
+            'title': 'Daniel Holbach (favorites)',
+            'description': 'md5:327af72d1efeb404a8216c27240d1370',
+        },
+        'playlist_mincount': 244
+    }, {
+        'url': 'http://www.mixcloud.com/dholbach/listens/',
+        'info_dict': {
+            'id': 'dholbach/listens',
+            'title': 'Daniel Holbach (listens)',
+            'description': 'md5:327af72d1efeb404a8216c27240d1370',
+        },
+        'playlist_mincount': 846
+    }]
+
+    def _fetch_tracks(self, base_url, video_id, dl_note=None, dl_errnote=None):
+        # retrieve all fragments of a list of tracks with fake AJAX calls
+        track_urls = []
+        current_page = 1
+        while True:
+            # fake a AJAX request to retrieve a list fragment
+            page_url = base_url + "?page=%d&list=main&_ajax=1" % current_page
+            req = compat_urllib_request.Request(page_url, headers={"X-Requested-With": "XMLHttpRequest"}, method="GET")
+            resp = self._download_webpage(req, video_id, note=dl_note + " (page %d)" % current_page, errnote=dl_errnote)
+
+            # extract all track URLs from fragment
+            urls = re.findall(r'm-play-button m-url="(?P<url>[^"]+)"', resp)
+            # clean up URLs
+            urls = map(clean_html, urls)
+            # create absolute URLs
+            urls = map(lambda u: "https://www.mixcloud.com" + u, urls)
+            track_urls.extend(urls)
+
+            # advance to next fragment, if any
+            if " m-next-page-url=" in resp:
+                current_page += 1
+            else:
+                break
+
+        return track_urls
+
+    def _handle_track_urls(self, urls):
+        return map(lambda u: self.url_result(u, "Mixcloud"), urls)
+
+    def _get_user_description(self, page_content):
+        return self._html_search_regex(
+            r'<div class="description-text">.*?<p>(?P<description>.*?)</p></div></div></div>',
+            page_content,
+            "user description",
+            group="description",
+            fatal=False,
+            default="")
+
+    def _get_username(self, page_content):
+        return self._og_search_title(page_content)
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        user_id = mobj.group("user")
+        list_type = mobj.group("type")
+
+        # if only a profile URL was supplied, default to download all uploads
+        if list_type is None:
+            list_type = "uploads"
+
+        video_id = "%s/%s" % (user_id, list_type)
+
+        # download the user's profile to retrieve some metadata
+        profile = self._download_webpage("https://www.mixcloud.com/%s/" % user_id,
+                                         video_id,
+                                         note="Downloading user profile",
+                                         errnote="Unable to download user profile")
+
+        username = self._get_username(profile)
+        description = self._get_user_description(profile)
+
+        # retrieve all page fragments of uploads, favorites or listens
+        track_urls = self._fetch_tracks(
+            "https://www.mixcloud.com/%s/%s/" % (user_id, list_type),
+            video_id,
+            dl_note="Downloading list of %s" % list_type,
+            dl_errnote="Unable to download list of %s" % list_type)
+
+        # let MixcloudIE handle each track URL
+        entries = self._handle_track_urls(track_urls)
+
+        return {
+            '_type': 'playlist',
+            'entries': entries,
+            'title': "%s (%s)" % (username, list_type),
+            'id': video_id,
+            "description": description
+        }
+
+
+class MixcloudPlaylistIE(MixcloudUserIE):
+    """
+    Information extractor for Mixcloud playlists.
+    """
+
+    _VALID_URL = r'^(?:https?://)?(?:www\.)?mixcloud\.com/(?P<user>[^/]+)/playlists/(?P<playlist>[^/]+)/?$'
+    IE_NAME = 'mixcloud:playlist'
+
+    _TESTS = [{
+        'url': 'https://www.mixcloud.com/RedBullThre3style/playlists/tokyo-finalists-2015/',
+        'info_dict': {
+            'id': 'RedBullThre3style/playlists/tokyo-finalists-2015',
+            'title': 'National Champions 2015',
+            'description': 'md5:6ff5fb01ac76a31abc9b3939c16243a3',
+        },
+        'playlist_mincount': 16
+    }, {
+        'url': 'https://www.mixcloud.com/maxvibes/playlists/jazzcat-on-ness-radio/',
+        'info_dict': {
+            'id': 'maxvibes/playlists/jazzcat-on-ness-radio',
+            'title': 'Jazzcat on Ness Radio',
+            'description': 'md5:c2c51a1f1b8bb5442f2ca67c3dc4af27',
+        },
+        'playlist_mincount': 23
+    }]
+
+    def _get_playlist_title(self, page_content):
+        return self._html_search_regex(
+            r'<span class="main-list-title list-playlist-title ">(?P<title>.*?)</span>',
+            page_content,
+            "playlist title",
+            group="title",
+            fatal=True
+        )
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        user_id = mobj.group("user")
+        playlist_id = mobj.group("playlist")
+        video_id = "%s/playlists/%s" % (user_id, playlist_id)
+
+        # download the playlist page to retrieve some metadata
+        profile = self._download_webpage(url,
+                                         user_id,
+                                         note="Downloading playlist page",
+                                         errnote="Unable to download playlist page")
+
+        description = self._get_user_description(profile)
+        playlist_title = self._get_playlist_title(profile)
+
+        # retrieve all page fragments of playlist
+        track_urls = self._fetch_tracks(
+            "https://www.mixcloud.com/%s/playlists/%s/" % (user_id, playlist_id),
+            video_id,
+            dl_note="Downloading tracklist of %s" % playlist_title,
+            dl_errnote="Unable to tracklist of %s" % playlist_title)
+
+        # let MixcloudIE handle each track
+        entries = self._handle_track_urls(track_urls)
+
+        return {
+            '_type': 'playlist',
+            'entries': entries,
+            'title': playlist_title,
+            'id': video_id,
+            "description": description
+        }

From f896e1ccefc5d946c83f91bf517801882f4184dc Mon Sep 17 00:00:00 2001
From: Philip Huppert <philip@zeilen-sprung.de>
Date: Sat, 9 Apr 2016 14:30:01 +0200
Subject: [PATCH 4607/6219] [mixcloud] fixed some tests

---
 youtube_dl/extractor/mixcloud.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dl/extractor/mixcloud.py
index 20e64bab5..dcc4ddf25 100644
--- a/youtube_dl/extractor/mixcloud.py
+++ b/youtube_dl/extractor/mixcloud.py
@@ -25,7 +25,7 @@ class MixcloudIE(InfoExtractor):
         'url': 'http://www.mixcloud.com/dholbach/cryptkeeper/',
         'info_dict': {
             'id': 'dholbach-cryptkeeper',
-            'ext': 'mp3',
+            'ext': 'm4a',
             'title': 'Cryptkeeper',
             'description': 'After quite a long silence from myself, finally another Drum\'n\'Bass mix with my favourite current dance floor bangers.',
             'uploader': 'Daniel Holbach',
@@ -265,7 +265,7 @@ class MixcloudPlaylistIE(MixcloudUserIE):
         'info_dict': {
             'id': 'maxvibes/playlists/jazzcat-on-ness-radio',
             'title': 'Jazzcat on Ness Radio',
-            'description': 'md5:c2c51a1f1b8bb5442f2ca67c3dc4af27',
+            'description': 'md5:7bbbf0d6359a0b8cda85224be0f8f263',
         },
         'playlist_mincount': 23
     }]

From dcaf00fb3eb716ea146a8e8870c7753b9acf67b8 Mon Sep 17 00:00:00 2001
From: Philip Huppert <philip@zeilen-sprung.de>
Date: Sat, 9 Apr 2016 22:20:16 +0200
Subject: [PATCH 4608/6219] [mixcloud] support older urllib versions

---
 youtube_dl/extractor/mixcloud.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dl/extractor/mixcloud.py
index dcc4ddf25..769b68dc3 100644
--- a/youtube_dl/extractor/mixcloud.py
+++ b/youtube_dl/extractor/mixcloud.py
@@ -171,7 +171,7 @@ class MixcloudUserIE(InfoExtractor):
         while True:
             # fake a AJAX request to retrieve a list fragment
             page_url = base_url + "?page=%d&list=main&_ajax=1" % current_page
-            req = compat_urllib_request.Request(page_url, headers={"X-Requested-With": "XMLHttpRequest"}, method="GET")
+            req = compat_urllib_request.Request(page_url, headers={"X-Requested-With": "XMLHttpRequest"})
             resp = self._download_webpage(req, video_id, note=dl_note + " (page %d)" % current_page, errnote=dl_errnote)
 
             # extract all track URLs from fragment

From 6d671695092236af75c49e8a74d19b771e1b4d79 Mon Sep 17 00:00:00 2001
From: Philip Huppert <philip@zeilen-sprung.de>
Date: Sun, 10 Apr 2016 15:53:17 +0200
Subject: [PATCH 4609/6219] [mixcloud] improved extraction of user description

---
 youtube_dl/extractor/mixcloud.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dl/extractor/mixcloud.py
index 769b68dc3..30b33e7e9 100644
--- a/youtube_dl/extractor/mixcloud.py
+++ b/youtube_dl/extractor/mixcloud.py
@@ -195,12 +195,10 @@ class MixcloudUserIE(InfoExtractor):
 
     def _get_user_description(self, page_content):
         return self._html_search_regex(
-            r'<div class="description-text">.*?<p>(?P<description>.*?)</p></div></div></div>',
+            r'<div class="description-text">.*?<p>(.*?)</p></div></div></div>',
             page_content,
             "user description",
-            group="description",
-            fatal=False,
-            default="")
+            fatal=False)
 
     def _get_username(self, page_content):
         return self._og_search_title(page_content)

From b969d12490a4c618ee65b5731084db4c95209af8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 13 Apr 2016 01:52:50 +0600
Subject: [PATCH 4610/6219] Credit @Phaeilo for presstv (#7113)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index ea8d39978..07cade723 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -167,3 +167,4 @@ Kacper Michajłow
 José Joaquín Atria
 Viťas Strádal
 Kagami Hiiragi
+Philip Huppert

From 75af5d59ae5c4c6549cce5357a8a8d060c3f462c Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 13 Apr 2016 04:50:44 +0800
Subject: [PATCH 4611/6219] [netease] Skip all tests: completely georestricted

---
 youtube_dl/extractor/neteasemusic.py | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/neteasemusic.py b/youtube_dl/extractor/neteasemusic.py
index 0d36474fa..978a05841 100644
--- a/youtube_dl/extractor/neteasemusic.py
+++ b/youtube_dl/extractor/neteasemusic.py
@@ -89,6 +89,7 @@ class NetEaseMusicIE(NetEaseMusicBaseIE):
             'timestamp': 1431878400,
             'description': 'md5:a10a54589c2860300d02e1de821eb2ef',
         },
+        'skip': 'Blocked outside Mainland China',
     }, {
         'note': 'No lyrics translation.',
         'url': 'http://music.163.com/#/song?id=29822014',
@@ -101,6 +102,7 @@ class NetEaseMusicIE(NetEaseMusicBaseIE):
             'timestamp': 1419523200,
             'description': 'md5:a4d8d89f44656af206b7b2555c0bce6c',
         },
+        'skip': 'Blocked outside Mainland China',
     }, {
         'note': 'No lyrics.',
         'url': 'http://music.163.com/song?id=17241424',
@@ -112,6 +114,7 @@ class NetEaseMusicIE(NetEaseMusicBaseIE):
             'upload_date': '20080211',
             'timestamp': 1202745600,
         },
+        'skip': 'Blocked outside Mainland China',
     }, {
         'note': 'Has translated name.',
         'url': 'http://music.163.com/#/song?id=22735043',
@@ -124,7 +127,8 @@ class NetEaseMusicIE(NetEaseMusicBaseIE):
             'upload_date': '20100127',
             'timestamp': 1264608000,
             'alt_title': '说出愿望吧(Genie)',
-        }
+        },
+        'skip': 'Blocked outside Mainland China',
     }]
 
     def _process_lyrics(self, lyrics_info):
@@ -192,6 +196,7 @@ class NetEaseMusicAlbumIE(NetEaseMusicBaseIE):
             'title': 'B\'day',
         },
         'playlist_count': 23,
+        'skip': 'Blocked outside Mainland China',
     }
 
     def _real_extract(self, url):
@@ -223,6 +228,7 @@ class NetEaseMusicSingerIE(NetEaseMusicBaseIE):
             'title': '张惠妹 - aMEI;阿密特',
         },
         'playlist_count': 50,
+        'skip': 'Blocked outside Mainland China',
     }, {
         'note': 'Singer has translated name.',
         'url': 'http://music.163.com/#/artist?id=124098',
@@ -231,6 +237,7 @@ class NetEaseMusicSingerIE(NetEaseMusicBaseIE):
             'title': '李昇基 - 이승기',
         },
         'playlist_count': 50,
+        'skip': 'Blocked outside Mainland China',
     }]
 
     def _real_extract(self, url):
@@ -266,6 +273,7 @@ class NetEaseMusicListIE(NetEaseMusicBaseIE):
             'description': 'md5:12fd0819cab2965b9583ace0f8b7b022'
         },
         'playlist_count': 99,
+        'skip': 'Blocked outside Mainland China',
     }, {
         'note': 'Toplist/Charts sample',
         'url': 'http://music.163.com/#/discover/toplist?id=3733003',
@@ -275,6 +283,7 @@ class NetEaseMusicListIE(NetEaseMusicBaseIE):
             'description': 'md5:73ec782a612711cadc7872d9c1e134fc',
         },
         'playlist_count': 50,
+        'skip': 'Blocked outside Mainland China',
     }]
 
     def _real_extract(self, url):
@@ -314,6 +323,7 @@ class NetEaseMusicMvIE(NetEaseMusicBaseIE):
             'creator': '白雅言',
             'upload_date': '20150520',
         },
+        'skip': 'Blocked outside Mainland China',
     }
 
     def _real_extract(self, url):
@@ -357,6 +367,7 @@ class NetEaseMusicProgramIE(NetEaseMusicBaseIE):
             'upload_date': '20150613',
             'duration': 900,
         },
+        'skip': 'Blocked outside Mainland China',
     }, {
         'note': 'This program has accompanying songs.',
         'url': 'http://music.163.com/#/program?id=10141022',
@@ -366,6 +377,7 @@ class NetEaseMusicProgramIE(NetEaseMusicBaseIE):
             'description': 'md5:8d594db46cc3e6509107ede70a4aaa3b',
         },
         'playlist_count': 4,
+        'skip': 'Blocked outside Mainland China',
     }, {
         'note': 'This program has accompanying songs.',
         'url': 'http://music.163.com/#/program?id=10141022',
@@ -379,7 +391,8 @@ class NetEaseMusicProgramIE(NetEaseMusicBaseIE):
         },
         'params': {
             'noplaylist': True
-        }
+        },
+        'skip': 'Blocked outside Mainland China',
     }]
 
     def _real_extract(self, url):
@@ -438,6 +451,7 @@ class NetEaseMusicDjRadioIE(NetEaseMusicBaseIE):
             'description': 'md5:766220985cbd16fdd552f64c578a6b15'
         },
         'playlist_mincount': 40,
+        'skip': 'Blocked outside Mainland China',
     }
     _PAGE_SIZE = 1000
 

From b8f67449ecafa76d4d925c7ad72f9c8f8338aba7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kacper=20Michaj=C5=82ow?= <kasper93@gmail.com>
Date: Thu, 31 Mar 2016 20:42:55 +0200
Subject: [PATCH 4612/6219] [generic] Add support for LiveLeak embeds

---
 youtube_dl/extractor/generic.py  | 18 ++++++++++++++++++
 youtube_dl/extractor/liveleak.py |  8 ++++++++
 2 files changed, 26 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 2aadd6a12..5b22b6b5e 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -60,6 +60,7 @@ from .googledrive import GoogleDriveIE
 from .jwplatform import JWPlatformIE
 from .digiteka import DigitekaIE
 from .instagram import InstagramIE
+from .liveleak import LiveLeakIE
 
 
 class GenericIE(InfoExtractor):
@@ -1140,6 +1141,18 @@ class GenericIE(InfoExtractor):
                 'upload_date': '20160409',
             },
         },
+        # LiveLeak embed
+        {
+            'url': 'http://www.wykop.pl/link/3088787/',
+            'md5': 'ace83b9ed19b21f68e1b50e844fdf95d',
+            'info_dict': {
+                'id': '874_1459135191',
+                'ext': 'mp4',
+                'title': 'Man shows poor quality of new apartment building',
+                'description': 'The wall is like a sand pile.',
+                'uploader': 'Lake8737',
+            }
+        },
     ]
 
     def report_following_redirect(self, new_url):
@@ -1944,6 +1957,11 @@ class GenericIE(InfoExtractor):
         if instagram_embed_url is not None:
             return self.url_result(instagram_embed_url, InstagramIE.ie_key())
 
+        # Look for LiveLeak embeds
+        liveleak_url = LiveLeakIE._extract_url(webpage)
+        if liveleak_url:
+            return self.url_result(liveleak_url, 'LiveLeak')
+
         def check_video(vurl):
             if YoutubeIE.suitable(vurl):
                 return True
diff --git a/youtube_dl/extractor/liveleak.py b/youtube_dl/extractor/liveleak.py
index 4684994e1..29fba5f30 100644
--- a/youtube_dl/extractor/liveleak.py
+++ b/youtube_dl/extractor/liveleak.py
@@ -53,6 +53,14 @@ class LiveLeakIE(InfoExtractor):
         }
     }]
 
+    @staticmethod
+    def _extract_url(webpage):
+        mobj = re.search(
+            r'<iframe[^>]+src="https?://(?:\w+\.)?liveleak\.com/ll_embed\?(?:.*?)i=(?P<id>[\w_]+)(?:.*)',
+            webpage)
+        if mobj:
+            return 'http://www.liveleak.com/view?i=%s' % mobj.group('id')
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)

From b0ba11cc64656c09f349117fffa9739dcbb2541e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 13 Apr 2016 08:02:03 +0200
Subject: [PATCH 4613/6219] release 2016.04.13

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 CONTRIBUTING.md           | 4 ++--
 docs/supportedsites.md    | 3 ++-
 youtube_dl/version.py     | 2 +-
 4 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index bf9494646..caed64e38 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.04.06*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.04.06**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.04.13*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.04.13**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.04.06
+[debug] youtube-dl version 2016.04.13
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 0df6193fb..c83b8655a 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -140,14 +140,14 @@ After you have ensured this site is distributing it's content legally, you can f
                 # TODO more properties (see youtube_dl/extractor/common.py)
             }
     ```
-5. Add an import in [`youtube_dl/extractor/__init__.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/__init__.py).
+5. Add an import in [`youtube_dl/extractor/extractors.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/extractors.py).
 6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc.
 7. Have a look at [`youtube_dl/extractor/common.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/rg3/youtube-dl/blob/58525c94d547be1c8167d16c298bdd75506db328/youtube_dl/extractor/common.py#L68-L226). Add tests and code for as many as you want.
 8. Keep in mind that the only mandatory fields in info dict for successful extraction process are `id`, `title` and either `url` or `formats`, i.e. these are the critical data the extraction does not make any sense without. This means that [any field](https://github.com/rg3/youtube-dl/blob/58525c94d547be1c8167d16c298bdd75506db328/youtube_dl/extractor/common.py#L138-L226) apart from aforementioned mandatory ones should be treated **as optional** and extraction should be **tolerate** to situations when sources for these fields can potentially be unavailable (even if they always available at the moment) and **future-proof** in order not to break the extraction of general purpose mandatory fields. For example, if you have some intermediate dict `meta` that is a source of metadata and it has a key `summary` that you want to extract and put into resulting info dict as `description`, you should be ready that this key may be missing from the `meta` dict, i.e. you should extract it as `meta.get('summary')` and not `meta['summary']`. Similarly, you should pass `fatal=False` when extracting data from a webpage with `_search_regex/_html_search_regex`.
 9. Check the code with [flake8](https://pypi.python.org/pypi/flake8).
 10. When the tests pass, [add](http://git-scm.com/docs/git-add) the new files and [commit](http://git-scm.com/docs/git-commit) them and [push](http://git-scm.com/docs/git-push) the result, like this:
 
-        $ git add youtube_dl/extractor/__init__.py
+        $ git add youtube_dl/extractor/extractors.py
         $ git add youtube_dl/extractor/yourextractor.py
         $ git commit -m '[yourextractor] Add new extractor'
         $ git push origin yourextractor
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index d6ee8476b..51a6b5609 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -115,6 +115,7 @@
  - **Cinemassacre**
  - **Clipfish**
  - **cliphunter**
+ - **ClipRs**
  - **Clipsyndicate**
  - **cloudtime**: CloudTime
  - **Cloudy**
@@ -286,7 +287,6 @@
  - **ivi:compilation**: ivi.ru compilations
  - **ivideon**: Ivideon TV
  - **Izlesene**
- - **JadoreCettePub**
  - **JeuxVideo**
  - **Jove**
  - **jpopsuki.tv**
@@ -484,6 +484,7 @@
  - **Pornotube**
  - **PornoVoisines**
  - **PornoXO**
+ - **PressTV**
  - **PrimeShareTV**
  - **PromptFile**
  - **prosiebensat1**: ProSiebenSat.1 Digital
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 167b16e24..0618d9a4f 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.04.06'
+__version__ = '2016.04.13'

From 8334637f4ac4a1aeddc6f05131be1c42ea6761e1 Mon Sep 17 00:00:00 2001
From: "aystroganov@gmail.com" <aystroganov@gmail.com>
Date: Wed, 13 Apr 2016 21:30:32 +1000
Subject: [PATCH 4614/6219] Make tbr field 'int' rather than 'tuple'

Closes #9180.
---
 youtube_dl/extractor/brightcove.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index c718cf385..f0781fc27 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -340,7 +340,7 @@ class BrightcoveLegacyIE(InfoExtractor):
                         ext = 'flv'
                 if ext is None:
                     ext = determine_ext(url)
-                tbr = int_or_none(rend.get('encodingRate'), 1000),
+                tbr = int_or_none(rend.get('encodingRate'), 1000)
                 a_format = {
                     'format_id': 'http%s' % ('-%s' % tbr if tbr else ''),
                     'url': url,

From f141fefab73d96a6dd2c927f56d0b86253991c02 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 14 Apr 2016 14:06:05 +0800
Subject: [PATCH 4615/6219] [karrierevideos] Fix extraction

The server serves malformed header "Content Type: text/xml" for the XML
request (it should be Content-Type but not Content Type). Python 3.x,
which uses email.feedparser rejects such headers. As a result,
Content-Encoding header is not parsed, so the returned content is kept
not decompressed, and thus XML parsing error.
---
 youtube_dl/extractor/karrierevideos.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/karrierevideos.py b/youtube_dl/extractor/karrierevideos.py
index 2cb04e533..c05263e61 100644
--- a/youtube_dl/extractor/karrierevideos.py
+++ b/youtube_dl/extractor/karrierevideos.py
@@ -52,9 +52,12 @@ class KarriereVideosIE(InfoExtractor):
 
         video_id = self._search_regex(
             r'/config/video/(.+?)\.xml', webpage, 'video id')
+        # Server returns malformed headers
+        # Force Accept-Encoding: * to prevent gzipped results
         playlist = self._download_xml(
             'http://www.karrierevideos.at/player-playlist.xml.php?p=%s' % video_id,
-            video_id, transform_source=fix_xml_ampersands)
+            video_id, transform_source=fix_xml_ampersands,
+            headers={'Accept-Encoding': '*'})
 
         NS_MAP = {
             'jwplayer': 'http://developer.longtailvideo.com/trac/wiki/FlashFormats'

From d1c4e4ba150562fcf7aedef26646dff7425ccd73 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 14 Apr 2016 14:11:28 +0800
Subject: [PATCH 4616/6219] [laola1tv] Improve error detection and skip an
 invalid test

---
 youtube_dl/extractor/laola1tv.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/laola1tv.py b/youtube_dl/extractor/laola1tv.py
index d4fbafece..2fab38079 100644
--- a/youtube_dl/extractor/laola1tv.py
+++ b/youtube_dl/extractor/laola1tv.py
@@ -63,6 +63,7 @@ class Laola1TvIE(InfoExtractor):
         'params': {
             'skip_download': True,
         },
+        'skip': 'This live stream has already finished.',
     }]
 
     def _real_extract(self, url):
@@ -74,6 +75,9 @@ class Laola1TvIE(InfoExtractor):
 
         webpage = self._download_webpage(url, display_id)
 
+        if 'Dieser Livestream ist bereits beendet.' in webpage:
+            raise ExtractorError('This live stream has already finished.', expected=True)
+
         iframe_url = self._search_regex(
             r'<iframe[^>]*?id="videoplayer"[^>]*?src="([^"]+)"',
             webpage, 'iframe url')

From f5d8743e0a1fdcbfed2bea4fb87bf5aaf40c1dfa Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 14 Apr 2016 15:07:31 +0800
Subject: [PATCH 4617/6219] [downloader/rtsp] Print the command

---
 youtube_dl/downloader/rtsp.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/downloader/rtsp.py b/youtube_dl/downloader/rtsp.py
index 3eb29526c..939358b2a 100644
--- a/youtube_dl/downloader/rtsp.py
+++ b/youtube_dl/downloader/rtsp.py
@@ -27,6 +27,8 @@ class RtspFD(FileDownloader):
             self.report_error('MMS or RTSP download detected but neither "mplayer" nor "mpv" could be run. Please install any.')
             return False
 
+        self._debug_cmd(args)
+
         retval = subprocess.call(args)
         if retval == 0:
             fsize = os.path.getsize(encodeFilename(tmpfilename))

From 73d93f948ee71b2a07d46fdc4d446255d8ddcc9f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 14 Apr 2016 15:08:01 +0800
Subject: [PATCH 4618/6219] [lecture2go] Fix extraction

RTSP stream fails to download. Seems it's a mpv bug as direct playback
works well:

$ mpv --ytdl-format rtsp https://lecture2go.uni-hamburg.de/veranstaltungen/-/v/17473
---
 youtube_dl/extractor/lecture2go.py | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/lecture2go.py b/youtube_dl/extractor/lecture2go.py
index 40a3d2346..81b5d41be 100644
--- a/youtube_dl/extractor/lecture2go.py
+++ b/youtube_dl/extractor/lecture2go.py
@@ -6,6 +6,7 @@ import re
 from .common import InfoExtractor
 from ..utils import (
     determine_ext,
+    determine_protocol,
     parse_duration,
     int_or_none,
 )
@@ -18,10 +19,14 @@ class Lecture2GoIE(InfoExtractor):
         'md5': 'ac02b570883020d208d405d5a3fd2f7f',
         'info_dict': {
             'id': '17473',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': '2 - Endliche Automaten und reguläre Sprachen',
             'creator': 'Frank Heitmann',
             'duration': 5220,
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
         }
     }
 
@@ -32,14 +37,18 @@ class Lecture2GoIE(InfoExtractor):
         title = self._html_search_regex(r'<em[^>]+class="title">(.+)</em>', webpage, 'title')
 
         formats = []
-        for url in set(re.findall(r'"src","([^"]+)"', webpage)):
+        for url in set(re.findall(r'var\s+playerUri\d+\s*=\s*"([^"]+)"', webpage)):
             ext = determine_ext(url)
+            protocol = determine_protocol({'url': url})
             if ext == 'f4m':
-                formats.extend(self._extract_f4m_formats(url, video_id))
+                formats.extend(self._extract_f4m_formats(url, video_id, f4m_id='hds'))
             elif ext == 'm3u8':
-                formats.extend(self._extract_m3u8_formats(url, video_id))
+                formats.extend(self._extract_m3u8_formats(url, video_id, ext='mp4', m3u8_id='hls'))
             else:
+                if protocol == 'rtmp':
+                    continue  # XXX: currently broken
                 formats.append({
+                    'format_id': protocol,
                     'url': url,
                 })
 

From 86475d59b1a6892dca5a8eb1ef3f05639ee3ab6a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 14 Apr 2016 15:12:59 +0800
Subject: [PATCH 4619/6219] [metacritic] Add a new valid test case

---
 youtube_dl/extractor/metacritic.py | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/metacritic.py b/youtube_dl/extractor/metacritic.py
index e30320569..444ec0310 100644
--- a/youtube_dl/extractor/metacritic.py
+++ b/youtube_dl/extractor/metacritic.py
@@ -11,7 +11,7 @@ from ..utils import (
 class MetacriticIE(InfoExtractor):
     _VALID_URL = r'https?://www\.metacritic\.com/.+?/trailers/(?P<id>\d+)'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.metacritic.com/game/playstation-4/infamous-second-son/trailers/3698222',
         'info_dict': {
             'id': '3698222',
@@ -20,7 +20,17 @@ class MetacriticIE(InfoExtractor):
             'description': 'Take a peak behind-the-scenes to see how Sucker Punch brings smoke into the universe of inFAMOUS Second Son on the PS4.',
             'duration': 221,
         },
-    }
+        'skip': 'Not providing trailers anymore',
+    }, {
+        'url': 'http://www.metacritic.com/game/playstation-4/tales-from-the-borderlands-a-telltale-game-series/trailers/5740315',
+        'info_dict': {
+            'id': '5740315',
+            'ext': 'mp4',
+            'title': 'Tales from the Borderlands - Finale: The Vault of the Traveler',
+            'description': 'In the final episode of the season, all hell breaks loose. Jack is now in control of Helios\' systems, and he\'s ready to reclaim his rightful place as king of Hyperion (with or without you).',
+            'duration': 114,
+        },
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)

From 5565be9dd9bfce361dd2c77cc0b1fc735a908b4e Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 14 Apr 2016 08:47:55 +0100
Subject: [PATCH 4620/6219] [aol] relex _VALID_URL regex

---
 youtube_dl/extractor/aol.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/aol.py b/youtube_dl/extractor/aol.py
index d4801a25b..24df8fe93 100644
--- a/youtube_dl/extractor/aol.py
+++ b/youtube_dl/extractor/aol.py
@@ -12,9 +12,10 @@ from ..utils import (
 
 class AolIE(InfoExtractor):
     IE_NAME = 'on.aol.com'
-    _VALID_URL = r'(?:aol-video:|https?://on\.aol\.com/video/.*-)(?P<id>[^/?-]+)'
+    _VALID_URL = r'(?:aol-video:|https?://on\.aol\.com/.*-)(?P<id>[^/?-]+)'
 
     _TESTS = [{
+        # video with 5min ID
         'url': 'http://on.aol.com/video/u-s--official-warns-of-largest-ever-irs-phone-scam-518167793?icid=OnHomepageC2Wide_MustSee_Img',
         'md5': '18ef68f48740e86ae94b98da815eec42',
         'info_dict': {
@@ -31,6 +32,7 @@ class AolIE(InfoExtractor):
             'skip_download': True,
         }
     }, {
+        # video with vidible ID
         'url': 'http://on.aol.com/video/netflix-is-raising-rates-5707d6b8e4b090497b04f706?context=PC:homepage:PL1944:1460189336183',
         'info_dict': {
             'id': '5707d6b8e4b090497b04f706',
@@ -45,6 +47,12 @@ class AolIE(InfoExtractor):
             # m3u8 download
             'skip_download': True,
         }
+    }, {
+        'url': 'http://on.aol.com/partners/abc-551438d309eab105804dbfe8/sneak-peek-was-haley-really-framed-570eaebee4b0448640a5c944',
+        'only_matching': True,
+    }, {
+        'url': 'http://on.aol.com/shows/park-bench-shw518173474-559a1b9be4b0c3bfad3357a7?context=SH:SHW518173474:PL4327:1460619712763',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 404284132c7877767a5b751d2ada2e064f75bd10 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 14 Apr 2016 21:52:05 +0600
Subject: [PATCH 4621/6219] [arte:info] Add extractor (Closes #9182)

---
 youtube_dl/extractor/arte.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index f042d9163..662230a90 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -232,6 +232,21 @@ class ArteTVCreativeIE(ArteTVPlus7IE):
     }]
 
 
+class ArteTVInfoIE(ArteTVPlus7IE):
+    IE_NAME = 'arte.tv:info'
+    _VALID_URL = r'https?://info\.arte\.tv/(?P<lang>fr|de|en|es)/(?:[^/]+/)*(?P<id>[^/?#&]+)'
+
+    _TEST = {
+        'url': 'http://info.arte.tv/fr/service-civique-un-cache-misere',
+        'info_dict': {
+            'id': '067528-000-A',
+            'ext': 'mp4',
+            'title': 'Service civique, un cache misère ?',
+            'upload_date': '20160403',
+        },
+    }
+
+
 class ArteTVFutureIE(ArteTVPlus7IE):
     IE_NAME = 'arte.tv:future'
     _VALID_URL = r'https?://future\.arte\.tv/(?P<lang>fr|de|en|es)/(?P<id>[^/?#&]+)'

From 9e285387260a019d7471c3bdbd52cc764c0e8700 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 14 Apr 2016 21:54:41 +0600
Subject: [PATCH 4622/6219] [arte:creative] Improve _VALID_URL

---
 youtube_dl/extractor/arte.py       | 5 ++++-
 youtube_dl/extractor/extractors.py | 1 +
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index 662230a90..a9e3266dc 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -210,7 +210,7 @@ class ArteTVPlus7IE(InfoExtractor):
 # It also uses the arte_vp_url url from the webpage to extract the information
 class ArteTVCreativeIE(ArteTVPlus7IE):
     IE_NAME = 'arte.tv:creative'
-    _VALID_URL = r'https?://creative\.arte\.tv/(?P<lang>fr|de|en|es)/(?:magazine?/)?(?P<id>[^/?#&]+)'
+    _VALID_URL = r'https?://creative\.arte\.tv/(?P<lang>fr|de|en|es)/(?:[^/]+/)*(?P<id>[^/?#&]+)'
 
     _TESTS = [{
         'url': 'http://creative.arte.tv/de/magazin/agentur-amateur-corporate-design',
@@ -229,6 +229,9 @@ class ArteTVCreativeIE(ArteTVPlus7IE):
             'description': 'Événement ! Quarante-cinq ans après leurs premiers succès, les légendaires Monty Python remontent sur scène.\n',
             'upload_date': '20140805',
         }
+    }, {
+        'url': 'http://creative.arte.tv/de/episode/agentur-amateur-4-der-erste-kunde',
+        'only_matching': True,
     }]
 
 
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 04c6508f1..2ae9bc9a8 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -46,6 +46,7 @@ from .arte import (
     ArteTVPlus7IE,
     ArteTVCreativeIE,
     ArteTVConcertIE,
+    ArteTVInfoIE,
     ArteTVFutureIE,
     ArteTVCinemaIE,
     ArteTVDDCIE,

From ce599d5a7edf2161d0881ccd263131d51c656093 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 14 Apr 2016 18:48:00 +0100
Subject: [PATCH 4623/6219] [downloader/external] enable piping for
 FFmpegFD(closes #2124)

---
 youtube_dl/downloader/external.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index 30277dc20..cda39fe97 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -225,7 +225,7 @@ class FFmpegFD(ExternalFD):
 
         args += ['-i', url, '-c', 'copy']
         if protocol == 'm3u8':
-            if self.params.get('hls_use_mpegts', False):
+            if self.params.get('hls_use_mpegts', False) or tmpfilename == '-':
                 args += ['-f', 'mpegts']
             else:
                 args += ['-f', 'mp4', '-bsf:a', 'aac_adtstoasc']
@@ -235,7 +235,10 @@ class FFmpegFD(ExternalFD):
             args += ['-f', EXT_TO_OUT_FORMATS.get(info_dict['ext'], info_dict['ext'])]
 
         args = [encodeArgument(opt) for opt in args]
-        args.append(encodeFilename(ffpp._ffmpeg_filename_argument(tmpfilename), True))
+        if tmpfilename == '-':
+            args.append('pipe:1')
+        else:
+            args.append(encodeFilename(ffpp._ffmpeg_filename_argument(tmpfilename), True))
 
         self._debug_cmd(args)
 

From e1bf277e19ff41d0d899f544749b8d9505fb6689 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 15 Apr 2016 02:41:02 +0800
Subject: [PATCH 4624/6219] [tdslifeway] Add TDSLifewayIE

Used by MinistryGridIE
---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/tdslifeway.py | 56 ++++++++++++++++++++++++++++++
 2 files changed, 57 insertions(+)
 create mode 100644 youtube_dl/extractor/tdslifeway.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 2ae9bc9a8..0a75a56c5 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -730,6 +730,7 @@ from .sztvhu import SztvHuIE
 from .tagesschau import TagesschauIE
 from .tapely import TapelyIE
 from .tass import TassIE
+from .tdslifeway import TDSLifewayIE
 from .teachertube import (
     TeacherTubeIE,
     TeacherTubeUserIE,
diff --git a/youtube_dl/extractor/tdslifeway.py b/youtube_dl/extractor/tdslifeway.py
new file mode 100644
index 000000000..34c7a13fd
--- /dev/null
+++ b/youtube_dl/extractor/tdslifeway.py
@@ -0,0 +1,56 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class TDSLifewayIE(InfoExtractor):
+    _VALID_URL = r'https?://tds\.lifeway\.com/v1/trainingdeliverysystem/courses/(?P<id>\d+)/index\.html'
+
+    _TEST = {
+        # From http://www.ministrygrid.com/training-viewer/-/training/t4g-2014-conference/the-gospel-by-numbers-4/the-gospel-by-numbers
+        'url': 'http://tds.lifeway.com/v1/trainingdeliverysystem/courses/3453494717001/index.html?externalRegistration=AssetId%7C34F466F1-78F3-4619-B2AB-A8EFFA55E9E9%21InstanceId%7C0%21UserId%7Caaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa&grouping=http%3A%2F%2Flifeway.com%2Fvideo%2F3453494717001&activity_id=http%3A%2F%2Flifeway.com%2Fvideo%2F3453494717001&content_endpoint=http%3A%2F%2Ftds.lifeway.com%2Fv1%2Ftrainingdeliverysystem%2FScormEngineInterface%2FTCAPI%2Fcontent%2F&actor=%7B%22name%22%3A%5B%22Guest%20Guest%22%5D%2C%22account%22%3A%5B%7B%22accountServiceHomePage%22%3A%22http%3A%2F%2Fscorm.lifeway.com%2F%22%2C%22accountName%22%3A%22aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa%22%7D%5D%2C%22objectType%22%3A%22Agent%22%7D&content_token=462a50b2-b6f9-4970-99b1-930882c499fb&registration=93d6ec8e-7f7b-4ed3-bbc8-a857913c0b2a&externalConfiguration=access%7CFREE%21adLength%7C-1%21assignOrgId%7C4AE36F78-299A-425D-91EF-E14A899B725F%21assignOrgParentId%7C%21courseId%7C%21isAnonymous%7Cfalse%21previewAsset%7Cfalse%21previewLength%7C-1%21previewMode%7Cfalse%21royalty%7CFREE%21sessionId%7C671422F9-8E79-48D4-9C2C-4EE6111EA1CD%21trackId%7C&auth=Basic%20OjhmZjk5MDBmLTBlYTMtNDJhYS04YjFlLWE4MWQ3NGNkOGRjYw%3D%3D&endpoint=http%3A%2F%2Ftds.lifeway.com%2Fv1%2Ftrainingdeliverysystem%2FScormEngineInterface%2FTCAPI%2F',
+        'info_dict': {
+            'id': '3453494717001',
+            'ext': 'mp4',
+            'title': 'The Gospel by Numbers',
+            'thumbnail': 're:^https?://.*\.jpg',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        # XXX: A generic brightcove function?
+        json_data = self._download_json(
+            'http://api.brightcove.com/services/library', video_id,
+            query={
+                'command': 'find_video_by_id',
+                'video_id': video_id,
+                'video_fields': 'id,name,videoStillURL,HLSURL,FLVURL',
+                'media_delivery': 'http',
+                # token extracted from http://tds.lifeway.com/v1/trainingdeliverysystem/courses/player_test.js
+                'token': 'MrrNjVSP15NGY3R0gipp-lvclofucPXKD3skFouJMjZXM3KOS2ch0g..',
+            })
+
+        formats = []
+
+        if 'HLSURL' in json_data:
+            formats.extend(self._extract_m3u8_formats(
+                json_data['HLSURL'], video_id, ext='mp4', m3u8_id='hls', fatal=False))
+        if 'FLVURL' in json_data:
+            formats.append({
+                'url': json_data['FLVURL'],
+            })
+
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': json_data['name'],
+            'thumbnail': json_data.get('videoStillURL'),
+            'formats': formats,
+        }

From 8cb57bab8e98c667c13e1da22b96d97bc37cfe78 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 15 Apr 2016 02:47:07 +0800
Subject: [PATCH 4625/6219] [ministrygrid] Fix extraction and modernize

---
 youtube_dl/extractor/ministrygrid.py | 30 +++++++++++-----------------
 1 file changed, 12 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/ministrygrid.py b/youtube_dl/extractor/ministrygrid.py
index 949ad11db..000989873 100644
--- a/youtube_dl/extractor/ministrygrid.py
+++ b/youtube_dl/extractor/ministrygrid.py
@@ -1,8 +1,5 @@
 from __future__ import unicode_literals
 
-import json
-import re
-
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
@@ -20,21 +17,22 @@ class MinistryGridIE(InfoExtractor):
             'id': '3453494717001',
             'ext': 'mp4',
             'title': 'The Gospel by Numbers',
-            'description': 'Coming soon from T4G 2014!',
-            'uploader': 'LifeWay Christian Resources (MG)',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
         },
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id)
-        portlets_json = self._search_regex(
-            r'Liferay\.Portlet\.list=(\[.+?\])', webpage, 'portlet list')
-        portlets = json.loads(portlets_json)
+        portlets = self._parse_json(self._search_regex(
+            r'Liferay\.Portlet\.list=(\[.+?\])', webpage, 'portlet list'),
+            video_id)
         pl_id = self._search_regex(
-            r'<!--\s*p_l_id - ([0-9]+)<br>', webpage, 'p_l_id')
+            r'getPlid:function\(\){return"(\d+)"}', webpage, 'p_l_id')
 
         for i, portlet in enumerate(portlets):
             portlet_url = 'http://www.ministrygrid.com/c/portal/render_portlet?p_l_id=%s&p_p_id=%s' % (pl_id, portlet)
@@ -46,12 +44,8 @@ class MinistryGridIE(InfoExtractor):
                 r'<iframe.*?src="([^"]+)"', portlet_code, 'video iframe',
                 default=None)
             if video_iframe_url:
-                surl = smuggle_url(
-                    video_iframe_url, {'force_videoid': video_id})
-                return {
-                    '_type': 'url',
-                    'id': video_id,
-                    'url': surl,
-                }
+                return self.url_result(
+                    smuggle_url(video_iframe_url, {'force_videoid': video_id}),
+                    video_id=video_id)
 
         raise ExtractorError('Could not find video iframe in any portlets')

From 74b47d00c3d807f91b0c24781077cb9100403bd5 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 15 Apr 2016 03:30:38 +0800
Subject: [PATCH 4626/6219] [xboxclips] Use http:// URL

xboxclips has misconfigured certificates
---
 youtube_dl/extractor/xboxclips.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/xboxclips.py b/youtube_dl/extractor/xboxclips.py
index 236ff403b..b113ab1c4 100644
--- a/youtube_dl/extractor/xboxclips.py
+++ b/youtube_dl/extractor/xboxclips.py
@@ -12,7 +12,7 @@ from ..utils import (
 class XboxClipsIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?xboxclips\.com/(?:video\.php\?.*vid=|[^/]+/)(?P<id>[\w-]{36})'
     _TEST = {
-        'url': 'https://xboxclips.com/video.php?uid=2533274823424419&gamertag=Iabdulelah&vid=074a69a9-5faf-46aa-b93b-9909c1720325',
+        'url': 'http://xboxclips.com/video.php?uid=2533274823424419&gamertag=Iabdulelah&vid=074a69a9-5faf-46aa-b93b-9909c1720325',
         'md5': 'fbe1ec805e920aeb8eced3c3e657df5d',
         'info_dict': {
             'id': '074a69a9-5faf-46aa-b93b-9909c1720325',

From b0cf2e7c1b844e533c447572b6979ae64f7e2870 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 15 Apr 2016 03:48:23 +0800
Subject: [PATCH 4627/6219] [ubu] Remove extractor

1. Videos on ubu.com are now hosted on Vimeo
2. The duration is far from correct, and may not exist on other videos
   (For example http://ubu.com/film/hammons_king.html)
---
 youtube_dl/extractor/extractors.py |  1 -
 youtube_dl/extractor/ubu.py        | 57 ------------------------------
 2 files changed, 58 deletions(-)
 delete mode 100644 youtube_dl/extractor/ubu.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 0a75a56c5..d9193349d 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -833,7 +833,6 @@ from .twitter import (
     TwitterIE,
     TwitterAmplifyIE,
 )
-from .ubu import UbuIE
 from .udemy import (
     UdemyIE,
     UdemyCourseIE
diff --git a/youtube_dl/extractor/ubu.py b/youtube_dl/extractor/ubu.py
deleted file mode 100644
index 1d52cbc98..000000000
--- a/youtube_dl/extractor/ubu.py
+++ /dev/null
@@ -1,57 +0,0 @@
-from __future__ import unicode_literals
-
-import re
-
-from .common import InfoExtractor
-from ..utils import (
-    int_or_none,
-    qualities,
-)
-
-
-class UbuIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?ubu\.com/film/(?P<id>[\da-z_-]+)\.html'
-    _TEST = {
-        'url': 'http://ubu.com/film/her_noise.html',
-        'md5': '138d5652618bf0f03878978db9bef1ee',
-        'info_dict': {
-            'id': 'her_noise',
-            'ext': 'm4v',
-            'title': 'Her Noise - The Making Of (2007)',
-            'duration': 3600,
-        },
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-
-        title = self._html_search_regex(
-            r'<title>.+?Film &amp; Video: ([^<]+)</title>', webpage, 'title')
-
-        duration = int_or_none(self._html_search_regex(
-            r'Duration: (\d+) minutes', webpage, 'duration', fatal=False),
-            invscale=60)
-
-        formats = []
-        FORMAT_REGEXES = [
-            ('sq', r"'flashvars'\s*,\s*'file=([^']+)'"),
-            ('hq', r'href="(http://ubumexico\.centro\.org\.mx/video/[^"]+)"'),
-        ]
-        preference = qualities([fid for fid, _ in FORMAT_REGEXES])
-        for format_id, format_regex in FORMAT_REGEXES:
-            m = re.search(format_regex, webpage)
-            if m:
-                formats.append({
-                    'url': m.group(1),
-                    'format_id': format_id,
-                    'preference': preference(format_id),
-                })
-        self._sort_formats(formats)
-
-        return {
-            'id': video_id,
-            'title': title,
-            'duration': duration,
-            'formats': formats,
-        }

From f2159c9815fa056ca1d4ef4a6d1c31c4847b3d47 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 15 Apr 2016 04:02:23 +0800
Subject: [PATCH 4628/6219] [wayofthemaster] Remove extractor

Now it's using YouTube embeds.
---
 youtube_dl/extractor/extractors.py     |  1 -
 youtube_dl/extractor/wayofthemaster.py | 52 --------------------------
 2 files changed, 53 deletions(-)
 delete mode 100644 youtube_dl/extractor/wayofthemaster.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index d9193349d..3148869d9 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -917,7 +917,6 @@ from .vulture import VultureIE
 from .walla import WallaIE
 from .washingtonpost import WashingtonPostIE
 from .wat import WatIE
-from .wayofthemaster import WayOfTheMasterIE
 from .wdr import (
     WDRIE,
     WDRMobileIE,
diff --git a/youtube_dl/extractor/wayofthemaster.py b/youtube_dl/extractor/wayofthemaster.py
deleted file mode 100644
index af7bb8b49..000000000
--- a/youtube_dl/extractor/wayofthemaster.py
+++ /dev/null
@@ -1,52 +0,0 @@
-from __future__ import unicode_literals
-
-import re
-
-from .common import InfoExtractor
-
-
-class WayOfTheMasterIE(InfoExtractor):
-    _VALID_URL = r'https?://www\.wayofthemaster\.com/([^/?#]*/)*(?P<id>[^/?#]+)\.s?html(?:$|[?#])'
-
-    _TEST = {
-        'url': 'http://www.wayofthemaster.com/hbks.shtml',
-        'md5': '5316b57487ada8480606a93cb3d18d24',
-        'info_dict': {
-            'id': 'hbks',
-            'ext': 'mp4',
-            'title': 'Intelligent Design vs. Evolution',
-        },
-    }
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
-        webpage = self._download_webpage(url, video_id)
-
-        title = self._search_regex(
-            r'<img src="images/title_[^"]+".*?alt="([^"]+)"',
-            webpage, 'title', default=None)
-        if title is None:
-            title = self._html_search_regex(
-                r'<title>(.*?)</title>', webpage, 'page title')
-
-        url_base = self._search_regex(
-            r'<param\s+name="?movie"?\s+value=".*?/wotm_videoplayer_highlow[0-9]*\.swf\?vid=([^"]+)"',
-            webpage, 'URL base')
-        formats = [{
-            'format_id': 'low',
-            'quality': 1,
-            'url': url_base + '_low.mp4',
-        }, {
-            'format_id': 'high',
-            'quality': 2,
-            'url': url_base + '_high.mp4',
-        }]
-        self._sort_formats(formats)
-
-        return {
-            'id': video_id,
-            'title': title,
-            'formats': formats,
-        }

From 56f1750049ee04d78d1349617124349bb1309cba Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 15 Apr 2016 04:28:54 +0800
Subject: [PATCH 4629/6219] [tdslifeway] Use the new Brightcove API

Thanks for @remitamine's suggestion.
---
 youtube_dl/extractor/ministrygrid.py |  6 ++++
 youtube_dl/extractor/tdslifeway.py   | 41 ++++++----------------------
 2 files changed, 15 insertions(+), 32 deletions(-)

diff --git a/youtube_dl/extractor/ministrygrid.py b/youtube_dl/extractor/ministrygrid.py
index 000989873..e48eba3fa 100644
--- a/youtube_dl/extractor/ministrygrid.py
+++ b/youtube_dl/extractor/ministrygrid.py
@@ -17,11 +17,17 @@ class MinistryGridIE(InfoExtractor):
             'id': '3453494717001',
             'ext': 'mp4',
             'title': 'The Gospel by Numbers',
+            'thumbnail': 're:^https?://.*\.jpg',
+            'upload_date': '20140410',
+            'description': 'Coming soon from T4G 2014!',
+            'uploader_id': '2034960640001',
+            'timestamp': 1397145591,
         },
         'params': {
             # m3u8 download
             'skip_download': True,
         },
+        'add_ie': ['TDSLifeway'],
     }
 
     def _real_extract(self, url):
diff --git a/youtube_dl/extractor/tdslifeway.py b/youtube_dl/extractor/tdslifeway.py
index 34c7a13fd..4d1f5c801 100644
--- a/youtube_dl/extractor/tdslifeway.py
+++ b/youtube_dl/extractor/tdslifeway.py
@@ -14,43 +14,20 @@ class TDSLifewayIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'The Gospel by Numbers',
             'thumbnail': 're:^https?://.*\.jpg',
+            'upload_date': '20140410',
+            'description': 'Coming soon from T4G 2014!',
+            'uploader_id': '2034960640001',
+            'timestamp': 1397145591,
         },
         'params': {
             # m3u8 download
             'skip_download': True,
         },
+        'add_ie': ['BrightcoveNew'],
     }
 
+    BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/2034960640001/default_default/index.html?videoId=%s'
+
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        # XXX: A generic brightcove function?
-        json_data = self._download_json(
-            'http://api.brightcove.com/services/library', video_id,
-            query={
-                'command': 'find_video_by_id',
-                'video_id': video_id,
-                'video_fields': 'id,name,videoStillURL,HLSURL,FLVURL',
-                'media_delivery': 'http',
-                # token extracted from http://tds.lifeway.com/v1/trainingdeliverysystem/courses/player_test.js
-                'token': 'MrrNjVSP15NGY3R0gipp-lvclofucPXKD3skFouJMjZXM3KOS2ch0g..',
-            })
-
-        formats = []
-
-        if 'HLSURL' in json_data:
-            formats.extend(self._extract_m3u8_formats(
-                json_data['HLSURL'], video_id, ext='mp4', m3u8_id='hls', fatal=False))
-        if 'FLVURL' in json_data:
-            formats.append({
-                'url': json_data['FLVURL'],
-            })
-
-        self._sort_formats(formats)
-
-        return {
-            'id': video_id,
-            'title': json_data['name'],
-            'thumbnail': json_data.get('videoStillURL'),
-            'formats': formats,
-        }
+        brightcove_id = self._match_id(url)
+        return self.url_result(self.BRIGHTCOVE_URL_TEMPLATE % brightcove_id, 'BrightcoveNew', brightcove_id)

From 9c250931f5e1e68a835065c0cc5fa58e3f1e4734 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 14 Apr 2016 22:19:52 +0800
Subject: [PATCH 4630/6219] [mixcloud] Improve and simplify mixcloud:user and
 mixcloud:playlist

---
 youtube_dl/extractor/mixcloud.py | 198 ++++++++++++-------------------
 1 file changed, 76 insertions(+), 122 deletions(-)

diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dl/extractor/mixcloud.py
index 30b33e7e9..5cf42198d 100644
--- a/youtube_dl/extractor/mixcloud.py
+++ b/youtube_dl/extractor/mixcloud.py
@@ -1,19 +1,21 @@
 from __future__ import unicode_literals
 
+import functools
 import re
 
 from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse_unquote,
-    compat_urllib_request
+    compat_urlparse,
 )
 from ..utils import (
+    clean_html,
     ExtractorError,
     HEADRequest,
+    OnDemandPagedList,
     NO_DEFAULT,
     parse_count,
     str_to_int,
-    clean_html
 )
 
 
@@ -121,191 +123,143 @@ class MixcloudIE(InfoExtractor):
         }
 
 
-class MixcloudUserIE(InfoExtractor):
-    """
-    Information extractor for Mixcloud users.
-    It can retrieve a list of a user's uploads, favorites or listens.
-    """
+class MixcloudPlaylistBaseIE(InfoExtractor):
+    _PAGE_SIZE = 24
 
+    def _fetch_tracks_page(self, path, video_id, page_name, current_page):
+        resp = self._download_webpage(
+            'https://www.mixcloud.com/%s/' % path, video_id,
+            note='Download %s (page %d)' % (page_name, current_page + 1),
+            errnote='Unable to download %s' % page_name,
+            query={'page': (current_page + 1), 'list': 'main', '_ajax': '1'},
+            headers={'X-Requested-With': 'XMLHttpRequest'})
+
+        for url in re.findall(r'm-play-button m-url="(?P<url>[^"]+)"', resp):
+            yield self.url_result(
+                compat_urlparse.urljoin('https://www.mixcloud.com', clean_html(url)),
+                MixcloudIE.ie_key())
+
+    def _get_user_description(self, page_content):
+        return self._html_search_regex(
+            r'<div[^>]+class="description-text"[^>]*>(.+?)</div>',
+            page_content, 'user description', fatal=False)
+
+
+class MixcloudUserIE(MixcloudPlaylistBaseIE):
     _VALID_URL = r'^(?:https?://)?(?:www\.)?mixcloud\.com/(?P<user>[^/]+)/(?P<type>uploads|favorites|listens)?/?$'
     IE_NAME = 'mixcloud:user'
 
     _TESTS = [{
         'url': 'http://www.mixcloud.com/dholbach/',
         'info_dict': {
-            'id': 'dholbach/uploads',
+            'id': 'dholbach_uploads',
             'title': 'Daniel Holbach (uploads)',
             'description': 'md5:327af72d1efeb404a8216c27240d1370',
         },
-        'playlist_mincount': 11
+        'playlist_mincount': 11,
     }, {
         'url': 'http://www.mixcloud.com/dholbach/uploads/',
         'info_dict': {
-            'id': 'dholbach/uploads',
+            'id': 'dholbach_uploads',
             'title': 'Daniel Holbach (uploads)',
             'description': 'md5:327af72d1efeb404a8216c27240d1370',
         },
-        'playlist_mincount': 11
+        'playlist_mincount': 11,
     }, {
         'url': 'http://www.mixcloud.com/dholbach/favorites/',
         'info_dict': {
-            'id': 'dholbach/favorites',
+            'id': 'dholbach_favorites',
             'title': 'Daniel Holbach (favorites)',
             'description': 'md5:327af72d1efeb404a8216c27240d1370',
         },
-        'playlist_mincount': 244
+        'params': {
+            'playlist_items': '1-100',
+        },
+        'playlist_mincount': 100,
     }, {
         'url': 'http://www.mixcloud.com/dholbach/listens/',
         'info_dict': {
-            'id': 'dholbach/listens',
+            'id': 'dholbach_listens',
             'title': 'Daniel Holbach (listens)',
             'description': 'md5:327af72d1efeb404a8216c27240d1370',
         },
-        'playlist_mincount': 846
+        'params': {
+            'playlist_items': '1-100',
+        },
+        'playlist_mincount': 100,
     }]
 
-    def _fetch_tracks(self, base_url, video_id, dl_note=None, dl_errnote=None):
-        # retrieve all fragments of a list of tracks with fake AJAX calls
-        track_urls = []
-        current_page = 1
-        while True:
-            # fake a AJAX request to retrieve a list fragment
-            page_url = base_url + "?page=%d&list=main&_ajax=1" % current_page
-            req = compat_urllib_request.Request(page_url, headers={"X-Requested-With": "XMLHttpRequest"})
-            resp = self._download_webpage(req, video_id, note=dl_note + " (page %d)" % current_page, errnote=dl_errnote)
-
-            # extract all track URLs from fragment
-            urls = re.findall(r'm-play-button m-url="(?P<url>[^"]+)"', resp)
-            # clean up URLs
-            urls = map(clean_html, urls)
-            # create absolute URLs
-            urls = map(lambda u: "https://www.mixcloud.com" + u, urls)
-            track_urls.extend(urls)
-
-            # advance to next fragment, if any
-            if " m-next-page-url=" in resp:
-                current_page += 1
-            else:
-                break
-
-        return track_urls
-
-    def _handle_track_urls(self, urls):
-        return map(lambda u: self.url_result(u, "Mixcloud"), urls)
-
-    def _get_user_description(self, page_content):
-        return self._html_search_regex(
-            r'<div class="description-text">.*?<p>(.*?)</p></div></div></div>',
-            page_content,
-            "user description",
-            fatal=False)
-
-    def _get_username(self, page_content):
-        return self._og_search_title(page_content)
-
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        user_id = mobj.group("user")
-        list_type = mobj.group("type")
+        user_id = mobj.group('user')
+        list_type = mobj.group('type')
 
         # if only a profile URL was supplied, default to download all uploads
         if list_type is None:
-            list_type = "uploads"
+            list_type = 'uploads'
 
-        video_id = "%s/%s" % (user_id, list_type)
+        video_id = '%s_%s' % (user_id, list_type)
 
-        # download the user's profile to retrieve some metadata
-        profile = self._download_webpage("https://www.mixcloud.com/%s/" % user_id,
-                                         video_id,
-                                         note="Downloading user profile",
-                                         errnote="Unable to download user profile")
+        profile = self._download_webpage(
+            'https://www.mixcloud.com/%s/' % user_id, video_id,
+            note='Downloading user profile',
+            errnote='Unable to download user profile')
 
-        username = self._get_username(profile)
+        username = self._og_search_title(profile)
         description = self._get_user_description(profile)
 
-        # retrieve all page fragments of uploads, favorites or listens
-        track_urls = self._fetch_tracks(
-            "https://www.mixcloud.com/%s/%s/" % (user_id, list_type),
-            video_id,
-            dl_note="Downloading list of %s" % list_type,
-            dl_errnote="Unable to download list of %s" % list_type)
+        entries = OnDemandPagedList(
+            functools.partial(
+                self._fetch_tracks_page,
+                '%s/%s' % (user_id, list_type), video_id, 'list of %s' % list_type),
+            self._PAGE_SIZE, use_cache=True)
 
-        # let MixcloudIE handle each track URL
-        entries = self._handle_track_urls(track_urls)
-
-        return {
-            '_type': 'playlist',
-            'entries': entries,
-            'title': "%s (%s)" % (username, list_type),
-            'id': video_id,
-            "description": description
-        }
+        return self.playlist_result(
+            entries, video_id, '%s (%s)' % (username, list_type), description)
 
 
-class MixcloudPlaylistIE(MixcloudUserIE):
-    """
-    Information extractor for Mixcloud playlists.
-    """
-
+class MixcloudPlaylistIE(MixcloudPlaylistBaseIE):
     _VALID_URL = r'^(?:https?://)?(?:www\.)?mixcloud\.com/(?P<user>[^/]+)/playlists/(?P<playlist>[^/]+)/?$'
     IE_NAME = 'mixcloud:playlist'
 
     _TESTS = [{
         'url': 'https://www.mixcloud.com/RedBullThre3style/playlists/tokyo-finalists-2015/',
         'info_dict': {
-            'id': 'RedBullThre3style/playlists/tokyo-finalists-2015',
+            'id': 'RedBullThre3style_tokyo-finalists-2015',
             'title': 'National Champions 2015',
             'description': 'md5:6ff5fb01ac76a31abc9b3939c16243a3',
         },
-        'playlist_mincount': 16
+        'playlist_mincount': 16,
     }, {
         'url': 'https://www.mixcloud.com/maxvibes/playlists/jazzcat-on-ness-radio/',
         'info_dict': {
-            'id': 'maxvibes/playlists/jazzcat-on-ness-radio',
+            'id': 'maxvibes_jazzcat-on-ness-radio',
             'title': 'Jazzcat on Ness Radio',
             'description': 'md5:7bbbf0d6359a0b8cda85224be0f8f263',
         },
         'playlist_mincount': 23
     }]
 
-    def _get_playlist_title(self, page_content):
-        return self._html_search_regex(
-            r'<span class="main-list-title list-playlist-title ">(?P<title>.*?)</span>',
-            page_content,
-            "playlist title",
-            group="title",
-            fatal=True
-        )
-
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        user_id = mobj.group("user")
-        playlist_id = mobj.group("playlist")
-        video_id = "%s/playlists/%s" % (user_id, playlist_id)
+        user_id = mobj.group('user')
+        playlist_id = mobj.group('playlist')
+        video_id = '%s_%s' % (user_id, playlist_id)
 
-        # download the playlist page to retrieve some metadata
-        profile = self._download_webpage(url,
-                                         user_id,
-                                         note="Downloading playlist page",
-                                         errnote="Unable to download playlist page")
+        profile = self._download_webpage(
+            url, user_id,
+            note='Downloading playlist page',
+            errnote='Unable to download playlist page')
 
         description = self._get_user_description(profile)
-        playlist_title = self._get_playlist_title(profile)
+        playlist_title = self._html_search_regex(
+            r'<span[^>]+class="[^"]*list-playlist-title[^"]*"[^>]*>(.*?)</span>',
+            profile, 'playlist title')
 
-        # retrieve all page fragments of playlist
-        track_urls = self._fetch_tracks(
-            "https://www.mixcloud.com/%s/playlists/%s/" % (user_id, playlist_id),
-            video_id,
-            dl_note="Downloading tracklist of %s" % playlist_title,
-            dl_errnote="Unable to tracklist of %s" % playlist_title)
+        entries = OnDemandPagedList(
+            functools.partial(
+                self._fetch_tracks_page,
+                '%s/playlists/%s' % (user_id, playlist_id), video_id, 'tracklist'),
+            self._PAGE_SIZE)
 
-        # let MixcloudIE handle each track
-        entries = self._handle_track_urls(track_urls)
-
-        return {
-            '_type': 'playlist',
-            'entries': entries,
-            'title': playlist_title,
-            'id': video_id,
-            "description": description
-        }
+        return self.playlist_result(entries, video_id, playlist_title, description)

From dd91dfcd67a0d1db25836f734579742ff73b0c66 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 15 Apr 2016 15:42:00 +0800
Subject: [PATCH 4631/6219] [mixcloud] Fix extraction by decrypting play info

Fixes #7521
---
 youtube_dl/extractor/mixcloud.py | 43 +++++++++++++++-----------------
 1 file changed, 20 insertions(+), 23 deletions(-)

diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dl/extractor/mixcloud.py
index 5cf42198d..d4a4963ee 100644
--- a/youtube_dl/extractor/mixcloud.py
+++ b/youtube_dl/extractor/mixcloud.py
@@ -1,19 +1,20 @@
 from __future__ import unicode_literals
 
+import base64
 import functools
 import re
 
 from .common import InfoExtractor
 from ..compat import (
+    compat_chr,
+    compat_ord,
     compat_urllib_parse_unquote,
     compat_urlparse,
 )
 from ..utils import (
     clean_html,
     ExtractorError,
-    HEADRequest,
     OnDemandPagedList,
-    NO_DEFAULT,
     parse_count,
     str_to_int,
 )
@@ -45,22 +46,22 @@ class MixcloudIE(InfoExtractor):
             'description': 'md5:2b8aec6adce69f9d41724647c65875e8',
             'uploader': 'Gilles Peterson Worldwide',
             'uploader_id': 'gillespeterson',
-            'thumbnail': 're:https?://.*/images/',
+            'thumbnail': 're:https?://.*',
             'view_count': int,
             'like_count': int,
         },
     }]
 
-    def _check_url(self, url, track_id, ext):
-        try:
-            # We only want to know if the request succeed
-            # don't download the whole file
-            self._request_webpage(
-                HEADRequest(url), track_id,
-                'Trying %s URL' % ext)
-            return True
-        except ExtractorError:
-            return False
+    # See https://www.mixcloud.com/media/js2/www_js_2.9e23256562c080482435196ca3975ab5.js
+    @staticmethod
+    def _decrypt_play_info(play_info):
+        KEY = 'pleasedontdownloadourmusictheartistswontgetpaid'
+
+        play_info = base64.b64decode(play_info.encode('ascii'))
+
+        return ''.join([
+            compat_chr(compat_ord(ch) ^ compat_ord(KEY[idx % len(KEY)]))
+            for idx, ch in enumerate(play_info)])
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
@@ -74,19 +75,15 @@ class MixcloudIE(InfoExtractor):
             r'(?s)<div[^>]+class="global-message cloudcast-disabled-notice-light"[^>]*>(.+?)<(?:a|/div)',
             webpage, 'error message', default=None)
 
-        preview_url = self._search_regex(
-            r'\s(?:data-preview-url|m-preview)="([^"]+)"',
-            webpage, 'preview url', default=None if message else NO_DEFAULT)
+        encrypted_play_info = self._search_regex(
+            r'm-play-info="([^"]+)"', webpage, 'play info')
+        play_info = self._parse_json(
+            self._decrypt_play_info(encrypted_play_info), track_id)
 
-        if message:
+        if message and 'stream_url' not in play_info:
             raise ExtractorError('%s said: %s' % (self.IE_NAME, message), expected=True)
 
-        song_url = re.sub(r'audiocdn(\d+)', r'stream\1', preview_url)
-        song_url = song_url.replace('/previews/', '/c/originals/')
-        if not self._check_url(song_url, track_id, 'mp3'):
-            song_url = song_url.replace('.mp3', '.m4a').replace('originals/', 'm4a/64/')
-            if not self._check_url(song_url, track_id, 'm4a'):
-                raise ExtractorError('Unable to extract track url')
+        song_url = play_info['stream_url']
 
         PREFIX = (
             r'm-play-on-spacebar[^>]+'

From e6da9240d44774495a7ae0f2780bd42e4d2628f5 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 15 Apr 2016 17:14:17 +0800
Subject: [PATCH 4632/6219] [mixcloud:stream] Add new extractor

Closes #7633
---
 youtube_dl/extractor/extractors.py |  3 +-
 youtube_dl/extractor/mixcloud.py   | 79 +++++++++++++++++++++++++-----
 2 files changed, 70 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 51c9a4719..d00445b3c 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -415,7 +415,8 @@ from .mitele import MiTeleIE
 from .mixcloud import (
     MixcloudIE,
     MixcloudUserIE,
-    MixcloudPlaylistIE
+    MixcloudPlaylistIE,
+    MixcloudStreamIE,
 )
 from .mlb import MLBIE
 from .mnet import MnetIE
diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dl/extractor/mixcloud.py
index d4a4963ee..483f6925f 100644
--- a/youtube_dl/extractor/mixcloud.py
+++ b/youtube_dl/extractor/mixcloud.py
@@ -2,6 +2,7 @@ from __future__ import unicode_literals
 
 import base64
 import functools
+import itertools
 import re
 
 from .common import InfoExtractor
@@ -123,19 +124,27 @@ class MixcloudIE(InfoExtractor):
 class MixcloudPlaylistBaseIE(InfoExtractor):
     _PAGE_SIZE = 24
 
-    def _fetch_tracks_page(self, path, video_id, page_name, current_page):
-        resp = self._download_webpage(
-            'https://www.mixcloud.com/%s/' % path, video_id,
-            note='Download %s (page %d)' % (page_name, current_page + 1),
-            errnote='Unable to download %s' % page_name,
-            query={'page': (current_page + 1), 'list': 'main', '_ajax': '1'},
-            headers={'X-Requested-With': 'XMLHttpRequest'})
-
-        for url in re.findall(r'm-play-button m-url="(?P<url>[^"]+)"', resp):
+    def _find_urls_in_page(self, page):
+        for url in re.findall(r'm-play-button m-url="(?P<url>[^"]+)"', page):
             yield self.url_result(
                 compat_urlparse.urljoin('https://www.mixcloud.com', clean_html(url)),
                 MixcloudIE.ie_key())
 
+    def _fetch_tracks_page(self, path, video_id, page_name, current_page, real_page_number=None):
+        real_page_number = real_page_number or current_page + 1
+        return self._download_webpage(
+            'https://www.mixcloud.com/%s/' % path, video_id,
+            note='Download %s (page %d)' % (page_name, current_page + 1),
+            errnote='Unable to download %s' % page_name,
+            query={'page': real_page_number, 'list': 'main', '_ajax': '1'},
+            headers={'X-Requested-With': 'XMLHttpRequest'})
+
+    def _tracks_page_func(self, page, video_id, page_name, current_page):
+        resp = self._fetch_tracks_page(page, video_id, page_name, current_page)
+
+        for item in self._find_urls_in_page(resp):
+            yield item
+
     def _get_user_description(self, page_content):
         return self._html_search_regex(
             r'<div[^>]+class="description-text"[^>]*>(.+?)</div>',
@@ -207,7 +216,7 @@ class MixcloudUserIE(MixcloudPlaylistBaseIE):
 
         entries = OnDemandPagedList(
             functools.partial(
-                self._fetch_tracks_page,
+                self._tracks_page_func,
                 '%s/%s' % (user_id, list_type), video_id, 'list of %s' % list_type),
             self._PAGE_SIZE, use_cache=True)
 
@@ -255,8 +264,56 @@ class MixcloudPlaylistIE(MixcloudPlaylistBaseIE):
 
         entries = OnDemandPagedList(
             functools.partial(
-                self._fetch_tracks_page,
+                self._tracks_page_func,
                 '%s/playlists/%s' % (user_id, playlist_id), video_id, 'tracklist'),
             self._PAGE_SIZE)
 
         return self.playlist_result(entries, video_id, playlist_title, description)
+
+
+class MixcloudStreamIE(MixcloudPlaylistBaseIE):
+    _VALID_URL = r'^(?:https?://)?(?:www\.)?mixcloud\.com/(?P<id>[^/]+)/stream/?$'
+    IE_NAME = 'mixcloud:stream'
+
+    _TEST = {
+        'url': 'https://www.mixcloud.com/FirstEar/stream/',
+        'info_dict': {
+            'id': 'FirstEar',
+            'title': 'First Ear',
+            'description': 'Curators of good music\nfirstearmusic.com',
+        },
+        'playlist_mincount': 192,
+    }
+
+    def _real_extract(self, url):
+        user_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, user_id)
+
+        entries = []
+        prev_page_url = None
+
+        def _handle_page(page):
+            entries.extend(self._find_urls_in_page(page))
+            return self._search_regex(
+                r'm-next-page-url="([^"]+)"', page,
+                'next page URL', default=None)
+
+        next_page_url = _handle_page(webpage)
+
+        for idx in itertools.count(0):
+            if not next_page_url or prev_page_url == next_page_url:
+                break
+
+            prev_page_url = next_page_url
+            current_page = int(self._search_regex(
+                r'\?page=(\d+)', next_page_url, 'next page number'))
+
+            next_page_url = _handle_page(self._fetch_tracks_page(
+                '%s/stream' % user_id, user_id, 'stream', idx,
+                real_page_number=current_page))
+
+        username = self._og_search_title(webpage)
+        description = self._get_user_description(webpage)
+
+        return self.playlist_result(entries, user_id, username, description)

From f0ec61b52559dc6e25f66895a55e1b73e9e9f58b Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 15 Apr 2016 20:54:12 +0800
Subject: [PATCH 4633/6219] [huffpost] Fix extraction

---
 youtube_dl/extractor/huffpost.py | 36 ++++++++++++++++++++++++--------
 1 file changed, 27 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/huffpost.py b/youtube_dl/extractor/huffpost.py
index a38eae421..1dc5701b2 100644
--- a/youtube_dl/extractor/huffpost.py
+++ b/youtube_dl/extractor/huffpost.py
@@ -4,6 +4,7 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
+    determine_ext,
     parse_duration,
     unified_strdate,
 )
@@ -29,7 +30,12 @@ class HuffPostIE(InfoExtractor):
             'description': 'This week on Legalese It, Mike talks to David Bosco about his new book on the ICC, "Rough Justice," he also discusses the Virginia AG\'s historic stance on gay marriage, the execution of Edgar Tamayo, the ICC\'s delay of Kenya\'s President and more.  ',
             'duration': 1549,
             'upload_date': '20140124',
-        }
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+        'expected_warnings': ['HTTP Error 404: Not Found'],
     }
 
     def _real_extract(self, url):
@@ -45,7 +51,7 @@ class HuffPostIE(InfoExtractor):
         description = data.get('description')
 
         thumbnails = []
-        for url in data['images'].values():
+        for url in filter(None, data['images'].values()):
             m = re.match('.*-([0-9]+x[0-9]+)\.', url)
             if not m:
                 continue
@@ -54,13 +60,25 @@ class HuffPostIE(InfoExtractor):
                 'resolution': m.group(1),
             })
 
-        formats = [{
-            'format': key,
-            'format_id': key.replace('/', '.'),
-            'ext': 'mp4',
-            'url': url,
-            'vcodec': 'none' if key.startswith('audio/') else None,
-        } for key, url in data.get('sources', {}).get('live', {}).items()]
+        formats = []
+        sources = data.get('sources', {})
+        live_sources = list(sources.get('live', {}).items()) + list(sources.get('live_again', {}).items())
+        for key, url in live_sources:
+            ext = determine_ext(url)
+            if ext == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    url, video_id, ext='mp4', m3u8_id='hls', fatal=False))
+            elif ext == 'f4m':
+                formats.extend(self._extract_f4m_formatsa(
+                    url + '?hdcore=2.9.5', video_id, f4m_id='hds', fatal=False))
+            else:
+                formats.append({
+                    'format': key,
+                    'format_id': key.replace('/', '.'),
+                    'ext': 'mp4',
+                    'url': url,
+                    'vcodec': 'none' if key.startswith('audio/') else None,
+                })
 
         if not formats and data.get('fivemin_id'):
             return self.url_result('5min:%s' % data['fivemin_id'])

From 4a252cc2d2c84b61f611bdb64b59f049ff44c955 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 15 Apr 2016 21:49:17 +0800
Subject: [PATCH 4634/6219] [karaoketv] Update and mark as not _WORKING

---
 youtube_dl/extractor/karaoketv.py | 35 ++++++++++++++-----------------
 1 file changed, 16 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/extractor/karaoketv.py b/youtube_dl/extractor/karaoketv.py
index b4c30b7f3..dbc83cb6c 100644
--- a/youtube_dl/extractor/karaoketv.py
+++ b/youtube_dl/extractor/karaoketv.py
@@ -2,20 +2,22 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse_unquote_plus
-from ..utils import (
-    js_to_json,
-)
 
 
 class KaraoketvIE(InfoExtractor):
-    _VALID_URL = r'https?://karaoketv\.co\.il/\?container=songs&id=(?P<id>[0-9]+)'
+    '''
+    In api_play.php there's a video-cdn.com <iframe>. The latter plays an
+    unencrypted RTMP stream. However I can't download it with rtmpdump.
+    '''
+    _WORKING = False
+
+    _VALID_URL = r'http://www.karaoketv.co.il/[^/]+/(?P<id>\d+)'
     _TEST = {
-        'url': 'http://karaoketv.co.il/?container=songs&id=171568',
+        'url': 'http://www.karaoketv.co.il/%D7%A9%D7%99%D7%A8%D7%99_%D7%A7%D7%A8%D7%99%D7%95%D7%A7%D7%99/58356/%D7%90%D7%99%D7%96%D7%95%D7%9F',
         'info_dict': {
-            'id': '171568',
-            'ext': 'mp4',
-            'title': 'אל העולם שלך - רותם כהן - שרים קריוקי',
+            'id': '58356',
+            'ext': 'flv',
+            'title': 'קריוקי של איזון',
         }
     }
 
@@ -23,18 +25,13 @@ class KaraoketvIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        page_video_url = self._og_search_video_url(webpage, video_id)
-        config_json = compat_urllib_parse_unquote_plus(self._search_regex(
-            r'config=(.*)', page_video_url, 'configuration'))
-
-        urls_info_json = self._download_json(
-            config_json, video_id, 'Downloading configuration',
-            transform_source=js_to_json)
-
-        url = urls_info_json['playlist'][0]['url']
+        api_page_url = self._html_search_regex(
+            r'<iframe[^>]+src="(http://www.karaoke.co.il/api_play.php?[^"]+)"',
+            webpage, 'API play URL')
 
         return {
+            '_type': 'url_transparent',
             'id': video_id,
             'title': self._og_search_title(webpage),
-            'url': url,
+            'url': api_page_url,
         }

From 7e176effb24ea2d654e44a8bedc23248d3291bbd Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 15 Apr 2016 22:39:18 +0800
Subject: [PATCH 4635/6219] [iqiyi] Also suuport pps.tv URLs

PPS is acquired by Baidu and merged with iQiyi in 2013 [1]. Now they
have the same page layouts.

[1] http://www.chinanews.com/it/2013/05-07/4792526.shtml
---
 youtube_dl/extractor/iqiyi.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index 88570f261..ea8fbb329 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -165,7 +165,7 @@ class IqiyiIE(InfoExtractor):
     IE_NAME = 'iqiyi'
     IE_DESC = '爱奇艺'
 
-    _VALID_URL = r'https?://(?:[^.]+\.)?iqiyi\.com/.+\.html'
+    _VALID_URL = r'https?://(?:(?:[^.]+\.)?iqiyi\.com|www\.pps\.tv)/.+\.html'
 
     _NETRC_MACHINE = 'iqiyi'
 
@@ -273,6 +273,9 @@ class IqiyiIE(InfoExtractor):
             'title': '灌篮高手 国语版',
         },
         'playlist_count': 101,
+    }, {
+        'url': 'http://www.pps.tv/w_19rrbav0ph.html',
+        'only_matching': True,
     }]
 
     _FORMATS_MAP = [

From b6612c9b11888e268874b4863b5cc2ef6ec2208a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 15 Apr 2016 21:26:54 +0600
Subject: [PATCH 4636/6219] [karaoketv] Fix extraction

---
 youtube_dl/extractor/karaoketv.py | 51 +++++++++++++++++++++++--------
 1 file changed, 39 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/karaoketv.py b/youtube_dl/extractor/karaoketv.py
index dbc83cb6c..a6050c4de 100644
--- a/youtube_dl/extractor/karaoketv.py
+++ b/youtube_dl/extractor/karaoketv.py
@@ -5,12 +5,6 @@ from .common import InfoExtractor
 
 
 class KaraoketvIE(InfoExtractor):
-    '''
-    In api_play.php there's a video-cdn.com <iframe>. The latter plays an
-    unencrypted RTMP stream. However I can't download it with rtmpdump.
-    '''
-    _WORKING = False
-
     _VALID_URL = r'http://www.karaoketv.co.il/[^/]+/(?P<id>\d+)'
     _TEST = {
         'url': 'http://www.karaoketv.co.il/%D7%A9%D7%99%D7%A8%D7%99_%D7%A7%D7%A8%D7%99%D7%95%D7%A7%D7%99/58356/%D7%90%D7%99%D7%96%D7%95%D7%9F',
@@ -18,20 +12,53 @@ class KaraoketvIE(InfoExtractor):
             'id': '58356',
             'ext': 'flv',
             'title': 'קריוקי של איזון',
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
         }
     }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
 
-        api_page_url = self._html_search_regex(
-            r'<iframe[^>]+src="(http://www.karaoke.co.il/api_play.php?[^"]+)"',
-            webpage, 'API play URL')
+        webpage = self._download_webpage(url, video_id)
+        api_page_url = self._search_regex(
+            r'<iframe[^>]+src=(["\'])(?P<url>https?://www\.karaoke\.co\.il/api_play\.php\?.+?)\1',
+            webpage, 'API play URL', group='url')
+
+        api_page = self._download_webpage(api_page_url, video_id)
+        video_cdn_url = self._search_regex(
+            r'<iframe[^>]+src=(["\'])(?P<url>https?://www\.video-cdn\.com/embed/iframe/.+?)\1',
+            api_page, 'video cdn URL', group='url')
+
+        video_cdn = self._download_webpage(video_cdn_url, video_id)
+        play_path = self._parse_json(
+            self._search_regex(
+                r'var\s+options\s*=\s*({.+?});', video_cdn, 'options'),
+            video_id)['clip']['url']
+
+        settings = self._parse_json(
+            self._search_regex(
+                r'var\s+settings\s*=\s*({.+?});', video_cdn, 'servers', default='{}'),
+            video_id, fatal=False) or {}
+
+        servers = settings.get('servers')
+        if not servers or not isinstance(servers, list):
+            servers = ('wowzail.video-cdn.com:80/vodcdn', )
+
+        formats = [{
+            'url': 'rtmp://%s' % server if not server.startswith('rtmp') else server,
+            'play_path': play_path,
+            'app': 'vodcdn',
+            'page_url': video_cdn_url,
+            'player_url': 'http://www.video-cdn.com/assets/flowplayer/flowplayer.commercial-3.2.18.swf',
+            'rtmp_real_time': True,
+            'ext': 'flv',
+        } for server in servers]
 
         return {
-            '_type': 'url_transparent',
             'id': video_id,
             'title': self._og_search_title(webpage),
-            'url': api_page_url,
+            'formats': formats,
         }

From f3a58d46bf969b33910479a3a6096635e793a553 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 15 Apr 2016 19:04:37 +0100
Subject: [PATCH 4637/6219] [youtube:user] check if the url didn't match only
 the other youtube extractors

---
 youtube_dl/extractor/youtube.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 5a102de51..44c1191bd 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1987,8 +1987,8 @@ class YoutubeUserIE(YoutubeChannelIE):
     def suitable(cls, url):
         # Don't return True if the url can be extracted with other youtube
         # extractor, the regex would is too permissive and it would match.
-        other_ies = iter(klass for (name, klass) in globals().items() if name.endswith('IE') and klass is not cls)
-        if any(ie.suitable(url) for ie in other_ies):
+        other_yt_ies = iter(klass for (name, klass) in globals().items() if name.startswith('Youtube') and name.endswith('IE') and klass is not cls)
+        if any(ie.suitable(url) for ie in other_yt_ies):
             return False
         else:
             return super(YoutubeUserIE, cls).suitable(url)

From f74197a0746ac24fc6003166fbc7b36362ce8ba8 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 15 Apr 2016 22:38:37 +0100
Subject: [PATCH 4638/6219] [cbs] extract rtmp formats

---
 youtube_dl/extractor/cbs.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/cbs.py b/youtube_dl/extractor/cbs.py
index c621a08d5..62f52ec8e 100644
--- a/youtube_dl/extractor/cbs.py
+++ b/youtube_dl/extractor/cbs.py
@@ -64,7 +64,7 @@ class CBSIE(CBSBaseIE):
         'url': 'http://www.colbertlateshow.com/podcasts/dYSwjqPs_X1tvbV_P2FcPWRa_qT6akTC/in-the-bad-room-with-stephen/',
         'only_matching': True,
     }]
-    TP_RELEASE_URL_TEMPLATE = 'http://link.theplatform.com/s/dJ5BDC/%s?manifest=m3u&mbr=true'
+    TP_RELEASE_URL_TEMPLATE = 'http://link.theplatform.com/s/dJ5BDC/%s?mbr=true'
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
@@ -85,8 +85,11 @@ class CBSIE(CBSBaseIE):
             if not pid:
                 continue
             try:
+                tp_release_url = self.TP_RELEASE_URL_TEMPLATE % pid
+                if '.m3u8' in xpath_text(item, 'contentUrl', default=''):
+                    tp_release_url += '&manifest=m3u'
                 tp_formats, tp_subtitles = self._extract_theplatform_smil(
-                    self.TP_RELEASE_URL_TEMPLATE % pid, content_id, 'Downloading %s SMIL data' % pid)
+                    tp_release_url, content_id, 'Downloading %s SMIL data' % pid)
             except ExtractorError:
                 continue
             formats.extend(tp_formats)

From 260c7036bab996a6364ed8578819fdfa034d6bd1 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 16 Apr 2016 16:13:14 +0800
Subject: [PATCH 4639/6219] [sportbox] Fix SportBoxEmbedIE

Also fixes test_Generic_29 (http://www.vestifinance.ru/articles/25753)
---
 youtube_dl/extractor/sportbox.py | 34 ++++++++++++++++++++++----------
 1 file changed, 24 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/sportbox.py b/youtube_dl/extractor/sportbox.py
index 4f0c66213..e5c28ae89 100644
--- a/youtube_dl/extractor/sportbox.py
+++ b/youtube_dl/extractor/sportbox.py
@@ -6,6 +6,7 @@ import re
 from .common import InfoExtractor
 from ..compat import compat_urlparse
 from ..utils import (
+    js_to_json,
     unified_strdate,
 )
 
@@ -94,19 +95,32 @@ class SportBoxEmbedIE(InfoExtractor):
 
         webpage = self._download_webpage(url, video_id)
 
-        hls = self._search_regex(
-            r"sportboxPlayer\.jwplayer_common_params\.file\s*=\s*['\"]([^'\"]+)['\"]",
-            webpage, 'hls file')
+        formats = []
+
+        def cleanup_js(code):
+            # desktop_advert_config contains complex Javascripts and we don't need it
+            return js_to_json(re.sub(r'desktop_advert_config.*', '', code))
+
+        jwplayer_data = self._parse_json(self._search_regex(
+            r'(?s)player\.setup\(({.+?})\);', webpage, 'jwplayer settings'), video_id,
+            transform_source=cleanup_js)
+
+        hls_url = jwplayer_data.get('hls_url')
+        if hls_url:
+            formats.extend(self._extract_m3u8_formats(
+                hls_url, video_id, ext='mp4', m3u8_id='hls'))
+
+        rtsp_url = jwplayer_data.get('rtsp_url')
+        if rtsp_url:
+            formats.append({
+                'url': rtsp_url,
+                'format_id': 'rtsp',
+            })
 
-        formats = self._extract_m3u8_formats(hls, video_id, 'mp4')
         self._sort_formats(formats)
 
-        title = self._search_regex(
-            r'sportboxPlayer\.node_title\s*=\s*"([^"]+)"', webpage, 'title')
-
-        thumbnail = self._search_regex(
-            r'sportboxPlayer\.jwplayer_common_params\.image\s*=\s*"([^"]+)"',
-            webpage, 'thumbnail', default=None)
+        title = jwplayer_data['node_title']
+        thumbnail = jwplayer_data.get('image_url')
 
         return {
             'id': video_id,

From c394b4f4cb6d6071613a1733d0c3188e5882f476 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 16 Apr 2016 16:22:44 +0800
Subject: [PATCH 4640/6219] [puls4] Fix error detection (#9194)

---
 youtube_dl/extractor/puls4.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/puls4.py b/youtube_dl/extractor/puls4.py
index cce84b9e4..fca30e1aa 100644
--- a/youtube_dl/extractor/puls4.py
+++ b/youtube_dl/extractor/puls4.py
@@ -40,7 +40,7 @@ class Puls4IE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         error_message = self._html_search_regex(
-            r'<div class="message-error">(.+?)</div>',
+            r'<div[^>]+class="message-error"[^>]*>(.+?)</div>',
             webpage, 'error message', default=None)
         if error_message:
             raise ExtractorError(

From ae65567102336f45a211a91e39d21f6d8aaecbbd Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 16 Apr 2016 16:47:16 +0800
Subject: [PATCH 4641/6219] [eagleplatform] Fix error handling

---
 youtube_dl/extractor/eagleplatform.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/eagleplatform.py b/youtube_dl/extractor/eagleplatform.py
index 7bbf617d4..fa3cb7023 100644
--- a/youtube_dl/extractor/eagleplatform.py
+++ b/youtube_dl/extractor/eagleplatform.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import compat_HTTPError
 from ..utils import (
     ExtractorError,
     int_or_none,
@@ -55,8 +56,13 @@ class EaglePlatformIE(InfoExtractor):
             raise ExtractorError(' '.join(response['errors']), expected=True)
 
     def _download_json(self, url_or_request, video_id, note='Downloading JSON metadata'):
-        response = super(EaglePlatformIE, self)._download_json(url_or_request, video_id, note)
-        self._handle_error(response)
+        try:
+            response = super(EaglePlatformIE, self)._download_json(url_or_request, video_id, note)
+        except ExtractorError as ee:
+            if isinstance(ee.cause, compat_HTTPError):
+                response = self._parse_json(ee.cause.read().decode('utf-8'), video_id)
+                self._handle_error(response)
+            raise
         return response
 
     def _get_video_url(self, url_or_request, video_id, note='Downloading JSON metadata'):

From 759e37c9e661d962c2318d173f9c99264c1e7e2b Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 16 Apr 2016 16:48:47 +0800
Subject: [PATCH 4642/6219] [gazeta] Relax _VALID_URL and update tests

Closes #9196
---
 youtube_dl/extractor/gazeta.py | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/gazeta.py b/youtube_dl/extractor/gazeta.py
index ea32b621c..ba1c15414 100644
--- a/youtube_dl/extractor/gazeta.py
+++ b/youtube_dl/extractor/gazeta.py
@@ -7,7 +7,7 @@ from .common import InfoExtractor
 
 
 class GazetaIE(InfoExtractor):
-    _VALID_URL = r'(?P<url>https?://(?:www\.)?gazeta\.ru/(?:[^/]+/)?video/(?:(?:main|\d{4}/\d{2}/\d{2})/)?(?P<id>[A-Za-z0-9-_.]+)\.s?html)'
+    _VALID_URL = r'(?P<url>https?://(?:www\.)?gazeta\.ru/(?:[^/]+/)?video/(?:main/)*(?:\d{4}/\d{2}/\d{2}/)?(?P<id>[A-Za-z0-9-_.]+)\.s?html)'
     _TESTS = [{
         'url': 'http://www.gazeta.ru/video/main/zadaite_vopros_vladislavu_yurevichu.shtml',
         'md5': 'd49c9bdc6e5a7888f27475dc215ee789',
@@ -18,9 +18,22 @@ class GazetaIE(InfoExtractor):
             'description': 'md5:38617526050bd17b234728e7f9620a71',
             'thumbnail': 're:^https?://.*\.jpg',
         },
+        'skip': 'video not found',
     }, {
         'url': 'http://www.gazeta.ru/lifestyle/video/2015/03/08/master-klass_krasivoi_byt._delaem_vesennii_makiyazh.shtml',
         'only_matching': True,
+    }, {
+        'url': 'http://www.gazeta.ru/video/main/main/2015/06/22/platit_ili_ne_platit_po_isku_yukosa.shtml',
+        'info_dict': {
+            'id': '252048',
+            'ext': 'mp4',
+            'title': '"Если по иску ЮКОСа придется платить, это будет большой удар по бюджету"',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+        'add_ie': ['EaglePlatform'],
     }]
 
     def _real_extract(self, url):

From ee94e7e66d8c715f0df29c22642e51cb56c612b9 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 16 Apr 2016 17:13:22 +0800
Subject: [PATCH 4643/6219] [varzesh3] Fix metadata extraction (closes #9197)

---
 youtube_dl/extractor/varzesh3.py | 47 +++++++++++++++++++++++++++-----
 1 file changed, 40 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/varzesh3.py b/youtube_dl/extractor/varzesh3.py
index 9369abaf8..438bb580c 100644
--- a/youtube_dl/extractor/varzesh3.py
+++ b/youtube_dl/extractor/varzesh3.py
@@ -2,11 +2,19 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..compat import (
+    compat_urllib_parse_urlparse,
+    compat_parse_qs,
+)
+from ..utils import (
+    clean_html,
+    remove_start,
+)
 
 
 class Varzesh3IE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?video\.varzesh3\.com/(?:[^/]+/)+(?P<id>[^/]+)/?'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://video.varzesh3.com/germany/bundesliga/5-%D9%88%D8%A7%DA%A9%D9%86%D8%B4-%D8%A8%D8%B1%D8%AA%D8%B1-%D8%AF%D8%B1%D9%88%D8%A7%D8%B2%D9%87%E2%80%8C%D8%A8%D8%A7%D9%86%D8%A7%D9%86%D8%9B%D9%87%D9%81%D8%AA%D9%87-26-%D8%A8%D9%88%D9%86%D8%AF%D8%B3/',
         'md5': '2a933874cb7dce4366075281eb49e855',
         'info_dict': {
@@ -15,8 +23,18 @@ class Varzesh3IE(InfoExtractor):
             'title': '۵ واکنش برتر دروازه‌بانان؛هفته ۲۶ بوندسلیگا',
             'description': 'فصل ۲۰۱۵-۲۰۱۴',
             'thumbnail': 're:^https?://.*\.jpg$',
-        }
-    }
+        },
+        'skip': 'HTTP 404 Error',
+    }, {
+        'url': 'http://video.varzesh3.com/video/112785/%D8%AF%D9%84%D9%87-%D8%B9%D9%84%DB%8C%D8%9B-%D8%B3%D8%AA%D8%A7%D8%B1%D9%87-%D9%86%D9%88%D8%B8%D9%87%D9%88%D8%B1-%D9%84%DB%8C%DA%AF-%D8%A8%D8%B1%D8%AA%D8%B1-%D8%AC%D8%B2%DB%8C%D8%B1%D9%87',
+        'info_dict': {
+            'id': '112785',
+            'ext': 'mp4',
+            'title': 'دله علی؛ ستاره نوظهور لیگ برتر جزیره',
+            'description': 'فوتبال 120',
+        },
+        'expected_warnings': ['description'],
+    }]
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
@@ -26,15 +44,30 @@ class Varzesh3IE(InfoExtractor):
         video_url = self._search_regex(
             r'<source[^>]+src="([^"]+)"', webpage, 'video url')
 
-        title = self._og_search_title(webpage)
+        title = remove_start(self._html_search_regex(
+            r'<title>([^<]+)</title>', webpage, 'title'), 'ویدیو ورزش 3 | ')
+
         description = self._html_search_regex(
             r'(?s)<div class="matn">(.+?)</div>',
-            webpage, 'description', fatal=False)
-        thumbnail = self._og_search_thumbnail(webpage)
+            webpage, 'description', default=None)
+        if description is None:
+            description = clean_html(self._html_search_meta('description', webpage))
+
+        thumbnail = self._og_search_thumbnail(webpage, default=None)
+        if thumbnail is None:
+            fb_sharer_url = self._search_regex(
+                r'<a[^>]+href="(https?://www\.facebook\.com/sharer/sharer\.php?[^"]+)"',
+                webpage, 'facebook sharer URL', fatal=False)
+            sharer_params = compat_parse_qs(compat_urllib_parse_urlparse(fb_sharer_url).query)
+            thumbnail = sharer_params.get('p[images][0]', [None])[0]
 
         video_id = self._search_regex(
             r"<link[^>]+rel='(?:canonical|shortlink)'[^>]+href='/\?p=([^']+)'",
-            webpage, display_id, default=display_id)
+            webpage, display_id, default=None)
+        if video_id is None:
+            video_id = self._search_regex(
+                'var\s+VideoId\s*=\s*(\d+);', webpage, 'video id',
+                default=display_id)
 
         return {
             'url': video_url,

From a71fca8577facddf05a5a1c44691bd257047a3c3 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 16 Apr 2016 10:23:56 +0100
Subject: [PATCH 4644/6219] [theplatform] remove _sort_formats from
 _extract_theplatform_smil

---
 youtube_dl/extractor/theplatform.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index 6da701a39..7a5a533b7 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -50,8 +50,6 @@ class ThePlatformBaseIE(OnceIE):
             else:
                 formats.append(_format)
 
-        self._sort_formats(formats)
-
         subtitles = self._parse_smil_subtitles(meta, default_ns)
 
         return formats, subtitles
@@ -241,6 +239,7 @@ class ThePlatformIE(ThePlatformBaseIE):
             smil_url = self._sign_url(smil_url, sig['key'], sig['secret'])
 
         formats, subtitles = self._extract_theplatform_smil(smil_url, video_id)
+        self._sort_formats(formats)
 
         ret = self.get_metadata(path, video_id)
         combined_subtitles = self._merge_subtitles(ret.get('subtitles', {}), subtitles)

From 0841d5013c21457932b34af6a121f26619ba5aca Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 16 Apr 2016 10:25:59 +0100
Subject: [PATCH 4645/6219] [cbs] do not catch Exceptions raised by by
 _extract_theplatform_smil

---
 youtube_dl/extractor/cbs.py | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/cbs.py b/youtube_dl/extractor/cbs.py
index 62f52ec8e..653d09e34 100644
--- a/youtube_dl/extractor/cbs.py
+++ b/youtube_dl/extractor/cbs.py
@@ -84,14 +84,11 @@ class CBSIE(CBSBaseIE):
             pid = xpath_text(item, 'pid')
             if not pid:
                 continue
-            try:
-                tp_release_url = self.TP_RELEASE_URL_TEMPLATE % pid
-                if '.m3u8' in xpath_text(item, 'contentUrl', default=''):
-                    tp_release_url += '&manifest=m3u'
-                tp_formats, tp_subtitles = self._extract_theplatform_smil(
-                    tp_release_url, content_id, 'Downloading %s SMIL data' % pid)
-            except ExtractorError:
-                continue
+            tp_release_url = self.TP_RELEASE_URL_TEMPLATE % pid
+            if '.m3u8' in xpath_text(item, 'contentUrl', default=''):
+                tp_release_url += '&manifest=m3u'
+            tp_formats, tp_subtitles = self._extract_theplatform_smil(
+                tp_release_url, content_id, 'Downloading %s SMIL data' % pid)
             formats.extend(tp_formats)
             subtitles = self._merge_subtitles(subtitles, tp_subtitles)
         self._sort_formats(formats)

From 0de968b58408a5584c9ad51f646f07e0fe161def Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 16 Apr 2016 17:36:30 +0800
Subject: [PATCH 4646/6219] [newgrounds] Support videos (closes #9138)

---
 youtube_dl/extractor/newgrounds.py | 21 ++++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/newgrounds.py b/youtube_dl/extractor/newgrounds.py
index cd117b04e..705940323 100644
--- a/youtube_dl/extractor/newgrounds.py
+++ b/youtube_dl/extractor/newgrounds.py
@@ -7,8 +7,8 @@ from .common import InfoExtractor
 
 
 class NewgroundsIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?newgrounds\.com/audio/listen/(?P<id>[0-9]+)'
-    _TEST = {
+    _VALID_URL = r'https?://(?:www\.)?newgrounds\.com/(?:audio/listen|portal/view)/(?P<id>[0-9]+)'
+    _TESTS = [{
         'url': 'http://www.newgrounds.com/audio/listen/549479',
         'md5': 'fe6033d297591288fa1c1f780386f07a',
         'info_dict': {
@@ -17,7 +17,16 @@ class NewgroundsIE(InfoExtractor):
             'title': 'B7 - BusMode',
             'uploader': 'Burn7',
         }
-    }
+    }, {
+        'url': 'http://www.newgrounds.com/portal/view/673111',
+        'md5': '3394735822aab2478c31b1004fe5e5bc',
+        'info_dict': {
+            'id': '673111',
+            'ext': 'mp4',
+            'title': 'Dancin',
+            'uploader': 'Squirrelman82',
+        },
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
@@ -25,9 +34,11 @@ class NewgroundsIE(InfoExtractor):
         webpage = self._download_webpage(url, music_id)
 
         title = self._html_search_regex(
-            r',"name":"([^"]+)",', webpage, 'music title')
+            r'<title>([^>]+)</title>', webpage, 'title')
+
         uploader = self._html_search_regex(
-            r',"artist":"([^"]+)",', webpage, 'music uploader')
+            [r',"artist":"([^"]+)",', r'[\'"]owner[\'"]\s*:\s*[\'"]([^\'"]+)[\'"],'],
+            webpage, 'uploader')
 
         music_url_json_string = self._html_search_regex(
             r'({"url":"[^"]+"),', webpage, 'music url') + '}'

From 00b426d66dd8ecf671666837866a4da373619de8 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 16 Apr 2016 17:37:44 +0800
Subject: [PATCH 4647/6219] [varzesh3] Add md5 to the test

---
 youtube_dl/extractor/varzesh3.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/varzesh3.py b/youtube_dl/extractor/varzesh3.py
index 438bb580c..84698371a 100644
--- a/youtube_dl/extractor/varzesh3.py
+++ b/youtube_dl/extractor/varzesh3.py
@@ -27,6 +27,7 @@ class Varzesh3IE(InfoExtractor):
         'skip': 'HTTP 404 Error',
     }, {
         'url': 'http://video.varzesh3.com/video/112785/%D8%AF%D9%84%D9%87-%D8%B9%D9%84%DB%8C%D8%9B-%D8%B3%D8%AA%D8%A7%D8%B1%D9%87-%D9%86%D9%88%D8%B8%D9%87%D9%88%D8%B1-%D9%84%DB%8C%DA%AF-%D8%A8%D8%B1%D8%AA%D8%B1-%D8%AC%D8%B2%DB%8C%D8%B1%D9%87',
+        'md5': '841b7cd3afbc76e61708d94e53a4a4e7',
         'info_dict': {
             'id': '112785',
             'ext': 'mp4',

From 6ad2ef8b7c23008ac654e90396ce2459f45a7629 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 16 Apr 2016 17:54:39 +0800
Subject: [PATCH 4648/6219] [audiomack] Update the test

The original test raises 404
---
 youtube_dl/extractor/audiomack.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/audiomack.py b/youtube_dl/extractor/audiomack.py
index 3eed91279..a52d26cec 100644
--- a/youtube_dl/extractor/audiomack.py
+++ b/youtube_dl/extractor/audiomack.py
@@ -30,14 +30,14 @@ class AudiomackIE(InfoExtractor):
         # audiomack wrapper around soundcloud song
         {
             'add_ie': ['Soundcloud'],
-            'url': 'http://www.audiomack.com/song/xclusiveszone/take-kare',
+            'url': 'http://www.audiomack.com/song/hip-hop-daily/black-mamba-freestyle',
             'info_dict': {
-                'id': '172419696',
+                'id': '258901379',
                 'ext': 'mp3',
-                'description': 'md5:1fc3272ed7a635cce5be1568c2822997',
-                'title': 'Young Thug ft Lil Wayne - Take Kare',
-                'uploader': 'Young Thug World',
-                'upload_date': '20141016',
+                'description': 'mamba day freestyle for the legend Kobe Bryant ',
+                'title': 'Black Mamba Freestyle [Prod. By Danny Wolf]',
+                'uploader': 'ILOVEMAKONNEN',
+                'upload_date': '20160414',
             }
         },
     ]

From 8c65e4a52707413607f5f70725ed71053dd257f1 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 16 Apr 2016 18:00:19 +0800
Subject: [PATCH 4649/6219] [bbc] Fix a test

---
 youtube_dl/extractor/bbc.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 425f08f2b..74c4510f9 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -671,6 +671,7 @@ class BBCIE(BBCCoUkIE):
         'info_dict': {
             'id': '34475836',
             'title': 'Jurgen Klopp: Furious football from a witty and winning coach',
+            'description': 'Fast-paced football, wit, wisdom and a ready smile - why Liverpool fans should come to love new boss Jurgen Klopp.',
         },
         'playlist_count': 3,
     }, {

From 36b7d9dbfaf43c357034e8b8792cdd5fb1363fb9 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 16 Apr 2016 18:57:50 +0800
Subject: [PATCH 4650/6219] [twitter] Don't check /cards/ URLs

Fixes #9181

In this tweet, there are two cards:
1. https://twitter.com/i/cards/tfw/v1/719944006306701313
   This shows #TeamCap vs. #TeamIronMan
2. https://twitter.com/i/videos/tweet/719944021058060289
   This is the real video and can be handled by TwitterCardIE

In all current test_Twitter* tests, /videos/tweet/ approach works fine.
---
 youtube_dl/extractor/twitter.py | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 1f32ea2eb..ea673054f 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -260,6 +260,17 @@ class TwitterIE(InfoExtractor):
             'upload_date': '20140615',
         },
         'add_ie': ['Vine'],
+    }, {
+        'url': 'https://twitter.com/captainamerica/status/719944021058060289',
+        # md5 constantly changes
+        'info_dict': {
+            'id': '719944021058060289',
+            'ext': 'mp4',
+            'title': 'Captain America - @King0fNerd Are you sure you made the right choice? Find out in theaters.',
+            'description': 'Captain America on Twitter: "@King0fNerd Are you sure you made the right choice? Find out in theaters. https://t.co/GpgYi9xMJI"',
+            'uploader_id': 'captainamerica',
+            'uploader': 'Captain America',
+        },
     }]
 
     def _real_extract(self, url):
@@ -284,17 +295,6 @@ class TwitterIE(InfoExtractor):
             'title': username + ' - ' + title,
         }
 
-        card_id = self._search_regex(
-            r'["\']/i/cards/tfw/v1/(\d+)', webpage, 'twitter card url', default=None)
-        if card_id:
-            card_url = 'https://twitter.com/i/cards/tfw/v1/' + card_id
-            info.update({
-                '_type': 'url_transparent',
-                'ie_key': 'TwitterCard',
-                'url': card_url,
-            })
-            return info
-
         mobj = re.search(r'''(?x)
             <video[^>]+class="animated-gif"(?P<more_info>[^>]+)>\s*
                 <source[^>]+video-src="(?P<url>[^"]+)"

From bec47a0748e11cfd2e92f6c9c3f71347dc6ed2f8 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 16 Apr 2016 19:11:25 +0800
Subject: [PATCH 4651/6219] [tudou] Improve error detection (closes #9175)

---
 youtube_dl/extractor/tudou.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/tudou.py b/youtube_dl/extractor/tudou.py
index 63b5d5924..bb8b8e234 100644
--- a/youtube_dl/extractor/tudou.py
+++ b/youtube_dl/extractor/tudou.py
@@ -65,6 +65,9 @@ class TudouIE(InfoExtractor):
         if quality:
             info_url += '&hd' + quality
         xml_data = self._download_xml(info_url, video_id, 'Opening the info XML page')
+        error = xml_data.attrib.get('error')
+        if error is not None:
+            raise ExtractorError('Tudou said: %s' % error, expected=True)
         final_url = xml_data.text
         return final_url
 

From 67d46a3f90ffebbdd1e83ec7b760f89e4bf32ce1 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 16 Apr 2016 19:39:25 +0800
Subject: [PATCH 4652/6219] [ustream] Fix /embed/ URLs and add a test

---
 youtube_dl/extractor/ustream.py | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/ustream.py b/youtube_dl/extractor/ustream.py
index b5fe753d7..54605d863 100644
--- a/youtube_dl/extractor/ustream.py
+++ b/youtube_dl/extractor/ustream.py
@@ -41,6 +41,12 @@ class UstreamIE(InfoExtractor):
             'uploader': 'sportscanadatv',
         },
         'skip': 'This Pro Broadcaster has chosen to remove this video from the ustream.tv site.',
+    }, {
+        'url': 'http://www.ustream.tv/embed/10299409',
+        'info_dict': {
+            'id': '10299409',
+        },
+        'playlist_count': 3,
     }]
 
     def _real_extract(self, url):
@@ -55,10 +61,12 @@ class UstreamIE(InfoExtractor):
         if m.group('type') == 'embed':
             video_id = m.group('id')
             webpage = self._download_webpage(url, video_id)
-            desktop_video_id = self._html_search_regex(
-                r'ContentVideoIds=\["([^"]*?)"\]', webpage, 'desktop_video_id')
-            desktop_url = 'http://www.ustream.tv/recorded/' + desktop_video_id
-            return self.url_result(desktop_url, 'Ustream')
+            content_video_ids = self._parse_json(self._search_regex(
+                r'ustream\.vars\.offAirContentVideoIds=([^;]+);', webpage,
+                'content video IDs'), video_id)
+            return self.playlist_result(
+                map(lambda u: self.url_result('http://www.ustream.tv/recorded/' + u, 'Ustream'), content_video_ids),
+                video_id)
 
         params = self._download_json(
             'https://api.ustream.tv/videos/%s.json' % video_id, video_id)

From ef0e4e7bc03669fbef0cb7923a21637a71993dad Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 16 Apr 2016 19:44:12 +0800
Subject: [PATCH 4653/6219] [generic] Fix test_Generic_2

Now a HEAD request returns 400 Bad Request
---
 youtube_dl/extractor/generic.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 5b22b6b5e..518b4f9de 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -105,7 +105,8 @@ class GenericIE(InfoExtractor):
                 'skip_download': True,  # infinite live stream
             },
             'expected_warnings': [
-                r'501.*Not Implemented'
+                r'501.*Not Implemented',
+                r'400.*Bad Request',
             ],
         },
         # Direct link with incorrect MIME type

From 9154c87fc4583b7bad2ee851b08f17c57df755cc Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 16 Apr 2016 21:41:22 +0800
Subject: [PATCH 4654/6219] [huffpost] Fix a typo

---
 youtube_dl/extractor/huffpost.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/huffpost.py b/youtube_dl/extractor/huffpost.py
index 1dc5701b2..059073749 100644
--- a/youtube_dl/extractor/huffpost.py
+++ b/youtube_dl/extractor/huffpost.py
@@ -69,7 +69,7 @@ class HuffPostIE(InfoExtractor):
                 formats.extend(self._extract_m3u8_formats(
                     url, video_id, ext='mp4', m3u8_id='hls', fatal=False))
             elif ext == 'f4m':
-                formats.extend(self._extract_f4m_formatsa(
+                formats.extend(self._extract_f4m_formats(
                     url + '?hdcore=2.9.5', video_id, f4m_id='hds', fatal=False))
             else:
                 formats.append({

From 5e5c30c3fd6d85d36bf169fddbfd58760420b5e3 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 16 Apr 2016 21:57:28 +0800
Subject: [PATCH 4655/6219] [mdr] Fix extraction and update tests

It's strange that the date is changed. Anyway, new data matches what the
webpage says.
---
 youtube_dl/extractor/mdr.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/mdr.py b/youtube_dl/extractor/mdr.py
index 2338e7f96..2100583df 100644
--- a/youtube_dl/extractor/mdr.py
+++ b/youtube_dl/extractor/mdr.py
@@ -49,8 +49,8 @@ class MDRIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Beutolomäus und der geheime Weihnachtswunsch',
             'description': 'md5:b69d32d7b2c55cbe86945ab309d39bbd',
-            'timestamp': 1419047100,
-            'upload_date': '20141220',
+            'timestamp': 1450950000,
+            'upload_date': '20151224',
             'duration': 4628,
             'uploader': 'KIKA',
         },
@@ -71,8 +71,8 @@ class MDRIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         data_url = self._search_regex(
-            r'(?:dataURL|playerXml(?:["\'])?)\s*:\s*(["\'])(?P<url>\\?/.+/(?:video|audio)-?[0-9]+-avCustom\.xml)\1',
-            webpage, 'data url', default=None, group='url').replace('\/', '/')
+            r'(?:dataURL|playerXml(?:["\'])?)\s*:\s*(["\'])(?P<url>.+/(?:video|audio)-?[0-9]+-avCustom\.xml)\1',
+            webpage, 'data url', group='url').replace('\/', '/')
 
         doc = self._download_xml(
             compat_urlparse.urljoin(url, data_url), video_id)

From 5d6360c3b720f74946fec6b7f0596965473d8d1d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 16 Apr 2016 21:31:50 +0600
Subject: [PATCH 4656/6219] [mooshare] Remove extractor

---
 youtube_dl/extractor/extractors.py |   1 -
 youtube_dl/extractor/mooshare.py   | 110 -----------------------------
 2 files changed, 111 deletions(-)
 delete mode 100644 youtube_dl/extractor/mooshare.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index d00445b3c..bbc0a219a 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -425,7 +425,6 @@ from .moevideo import MoeVideoIE
 from .mofosex import MofosexIE
 from .mojvideo import MojvideoIE
 from .moniker import MonikerIE
-from .mooshare import MooshareIE
 from .morningstar import MorningstarIE
 from .motherless import MotherlessIE
 from .motorsport import MotorsportIE
diff --git a/youtube_dl/extractor/mooshare.py b/youtube_dl/extractor/mooshare.py
deleted file mode 100644
index a85109a89..000000000
--- a/youtube_dl/extractor/mooshare.py
+++ /dev/null
@@ -1,110 +0,0 @@
-from __future__ import unicode_literals
-
-import re
-
-from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
-    sanitized_Request,
-    urlencode_postdata,
-)
-
-
-class MooshareIE(InfoExtractor):
-    IE_NAME = 'mooshare'
-    IE_DESC = 'Mooshare.biz'
-    _VALID_URL = r'https?://(?:www\.)?mooshare\.biz/(?P<id>[\da-z]{12})'
-
-    _TESTS = [
-        {
-            'url': 'http://mooshare.biz/8dqtk4bjbp8g',
-            'md5': '4e14f9562928aecd2e42c6f341c8feba',
-            'info_dict': {
-                'id': '8dqtk4bjbp8g',
-                'ext': 'mp4',
-                'title': 'Comedy Football 2011 - (part 1-2)',
-                'duration': 893,
-            },
-        },
-        {
-            'url': 'http://mooshare.biz/aipjtoc4g95j',
-            'info_dict': {
-                'id': 'aipjtoc4g95j',
-                'ext': 'mp4',
-                'title': 'Orange Caramel  Dashing Through the Snow',
-                'duration': 212,
-            },
-            'params': {
-                # rtmp download
-                'skip_download': True,
-            }
-        }
-    ]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        page = self._download_webpage(url, video_id, 'Downloading page')
-
-        if re.search(r'>Video Not Found or Deleted<', page) is not None:
-            raise ExtractorError('Video %s does not exist' % video_id, expected=True)
-
-        hash_key = self._html_search_regex(r'<input type="hidden" name="hash" value="([^"]+)">', page, 'hash')
-        title = self._html_search_regex(r'(?m)<div class="blockTitle">\s*<h2>Watch ([^<]+)</h2>', page, 'title')
-
-        download_form = {
-            'op': 'download1',
-            'id': video_id,
-            'hash': hash_key,
-        }
-
-        request = sanitized_Request(
-            'http://mooshare.biz/%s' % video_id, urlencode_postdata(download_form))
-        request.add_header('Content-Type', 'application/x-www-form-urlencoded')
-
-        self._sleep(5, video_id)
-
-        video_page = self._download_webpage(request, video_id, 'Downloading video page')
-
-        thumbnail = self._html_search_regex(r'image:\s*"([^"]+)",', video_page, 'thumbnail', fatal=False)
-        duration_str = self._html_search_regex(r'duration:\s*"(\d+)",', video_page, 'duration', fatal=False)
-        duration = int(duration_str) if duration_str is not None else None
-
-        formats = []
-
-        # SD video
-        mobj = re.search(r'(?m)file:\s*"(?P<url>[^"]+)",\s*provider:', video_page)
-        if mobj is not None:
-            formats.append({
-                'url': mobj.group('url'),
-                'format_id': 'sd',
-                'format': 'SD',
-            })
-
-        # HD video
-        mobj = re.search(r'\'hd-2\': { file: \'(?P<url>[^\']+)\' },', video_page)
-        if mobj is not None:
-            formats.append({
-                'url': mobj.group('url'),
-                'format_id': 'hd',
-                'format': 'HD',
-            })
-
-        # rtmp video
-        mobj = re.search(r'(?m)file: "(?P<playpath>[^"]+)",\s*streamer: "(?P<rtmpurl>rtmp://[^"]+)",', video_page)
-        if mobj is not None:
-            formats.append({
-                'url': mobj.group('rtmpurl'),
-                'play_path': mobj.group('playpath'),
-                'rtmp_live': False,
-                'ext': 'mp4',
-                'format_id': 'rtmp',
-                'format': 'HD',
-            })
-
-        return {
-            'id': video_id,
-            'title': title,
-            'thumbnail': thumbnail,
-            'duration': duration,
-            'formats': formats,
-        }

From 0dafea02e6644053b6db6603238827176f6c6f3e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 16 Apr 2016 22:23:08 +0600
Subject: [PATCH 4657/6219] [instagram] Add support for embed URLs

---
 youtube_dl/extractor/instagram.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/instagram.py b/youtube_dl/extractor/instagram.py
index 11bb58d8a..3a5dd14e6 100644
--- a/youtube_dl/extractor/instagram.py
+++ b/youtube_dl/extractor/instagram.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 
 class InstagramIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?instagram\.com/p/(?P<id>[^/?#&]+)'
+    _VALID_URL = r'(?P<url>https?://(?:www\.)?instagram\.com/p/(?P<id>[^/?#&]+))'
     _TESTS = [{
         'url': 'https://instagram.com/p/aye83DjauH/?foo=bar#abc',
         'md5': '0d2da106a9d2631273e192b372806516',
@@ -38,6 +38,9 @@ class InstagramIE(InfoExtractor):
     }, {
         'url': 'https://instagram.com/p/-Cmh1cukG2/',
         'only_matching': True,
+    }, {
+        'url': 'http://instagram.com/p/9o6LshA7zy/embed/',
+        'only_matching': True,
     }]
 
     @staticmethod
@@ -53,7 +56,9 @@ class InstagramIE(InfoExtractor):
             return mobj.group('link')
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        url = mobj.group('url')
 
         webpage = self._download_webpage(url, video_id)
         uploader_id = self._search_regex(r'"owner":{"username":"(.+?)"',

From c23533a100eb0a383b60ed2e5990602e9318fe4b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 16 Apr 2016 22:31:05 +0600
Subject: [PATCH 4658/6219] [instagram] Add support for iframe embeds

---
 youtube_dl/extractor/instagram.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/instagram.py b/youtube_dl/extractor/instagram.py
index 3a5dd14e6..3cbe77ad8 100644
--- a/youtube_dl/extractor/instagram.py
+++ b/youtube_dl/extractor/instagram.py
@@ -45,6 +45,12 @@ class InstagramIE(InfoExtractor):
 
     @staticmethod
     def _extract_embed_url(webpage):
+        mobj = re.search(
+            r'<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?instagram\.com/p/[^/]+/embed.*?)\1',
+            webpage)
+        if mobj:
+            return mobj.group('url')
+
         blockquote_el = get_element_by_attribute(
             'class', 'instagram-media', webpage)
         if blockquote_el is None:

From 11e60fcad83725cd74fb5b94946e2b49456435eb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 16 Apr 2016 22:39:20 +0600
Subject: [PATCH 4659/6219] [extractor/generic] Improve instagram embeds
 (Closes #9213)

---
 youtube_dl/extractor/generic.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 518b4f9de..95d233259 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1956,7 +1956,8 @@ class GenericIE(InfoExtractor):
         # Look for Instagram embeds
         instagram_embed_url = InstagramIE._extract_embed_url(webpage)
         if instagram_embed_url is not None:
-            return self.url_result(instagram_embed_url, InstagramIE.ie_key())
+            return self.url_result(
+                self._proto_relative_url(instagram_embed_url), InstagramIE.ie_key())
 
         # Look for LiveLeak embeds
         liveleak_url = LiveLeakIE._extract_url(webpage)

From b6b950bf5895617e0da1ffe8ddb1e5809ba26222 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 16 Apr 2016 22:47:10 +0600
Subject: [PATCH 4660/6219] [cbs] Remove unused import

---
 youtube_dl/extractor/cbs.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/cbs.py b/youtube_dl/extractor/cbs.py
index 653d09e34..051d783a2 100644
--- a/youtube_dl/extractor/cbs.py
+++ b/youtube_dl/extractor/cbs.py
@@ -5,7 +5,6 @@ from ..utils import (
     xpath_text,
     xpath_element,
     int_or_none,
-    ExtractorError,
     find_xpath_attr,
 )
 

From 663ee5f0a99011b19b2e7a4f23dc709fb15c6aed Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 16 Apr 2016 17:48:34 +0100
Subject: [PATCH 4661/6219] [vice] extract youtube embed

---
 youtube_dl/extractor/vice.py | 26 +++++++++++++++++++-------
 1 file changed, 19 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/vice.py b/youtube_dl/extractor/vice.py
index 46c785ae1..ec1245b0e 100644
--- a/youtube_dl/extractor/vice.py
+++ b/youtube_dl/extractor/vice.py
@@ -14,13 +14,21 @@ class ViceIE(InfoExtractor):
         'url': 'http://www.vice.com/video/cowboy-capitalists-part-1',
         'info_dict': {
             'id': '43cW1mYzpia9IlestBjVpd23Yu3afAfp',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': 'VICE_COWBOYCAPITALISTS_PART01_v1_VICE_WM_1080p.mov',
             'duration': 725.983,
         },
-        'params': {
-            # Requires ffmpeg (m3u8 manifest)
-            'skip_download': True,
+    }, {
+        'url': 'http://www.vice.com/video/how-to-hack-a-car',
+        'md5': '6fb2989a3fed069fb8eab3401fc2d3c9',
+        'info_dict': {
+            'id': '3jstaBeXgAs',
+            'ext': 'mp4',
+            'title': 'How to Hack a Car: Phreaked Out (Episode 2)',
+            'description': 'md5:ee95453f7ff495db8efe14ae8bf56f30',
+            'uploader_id': 'MotherboardTV',
+            'uploader': 'Motherboard',
+            'upload_date': '20140529',
         },
     }, {
         'url': 'https://news.vice.com/video/experimenting-on-animals-inside-the-monkey-lab',
@@ -39,11 +47,15 @@ class ViceIE(InfoExtractor):
         try:
             embed_code = self._search_regex(
                 r'embedCode=([^&\'"]+)', webpage,
-                'ooyala embed code')
-            ooyala_url = OoyalaIE._url_for_embed_code(embed_code)
+                'ooyala embed code', default=None)
+            if embed_code:
+                ooyala_url = OoyalaIE._url_for_embed_code(embed_code)
+                return self.url_result('ooyala:%s' % embed_code, 'Ooyala')
+            youtube_id = self._search_regex(
+                r'data-youtube-id="([^"]+)"', webpage, 'youtube id')
+            return self.url_result(youtube_id, 'Youtube')
         except ExtractorError:
             raise ExtractorError('The page doesn\'t contain a video', expected=True)
-        return self.url_result(ooyala_url, ie='Ooyala')
 
 
 class ViceShowIE(InfoExtractor):

From d8f103159f38a4ef18da27cd3a4378c4b9a29dc8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 17 Apr 2016 00:16:31 +0600
Subject: [PATCH 4662/6219] [nerdist] Remove extractor

It now uses brightcove
---
 youtube_dl/extractor/extractors.py |  1 -
 youtube_dl/extractor/nerdist.py    | 80 ------------------------------
 2 files changed, 81 deletions(-)
 delete mode 100644 youtube_dl/extractor/nerdist.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index bbc0a219a..06b3d5e24 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -469,7 +469,6 @@ from .ndr import (
 from .ndtv import NDTVIE
 from .netzkino import NetzkinoIE
 from .nerdcubed import NerdCubedFeedIE
-from .nerdist import NerdistIE
 from .neteasemusic import (
     NetEaseMusicIE,
     NetEaseMusicAlbumIE,
diff --git a/youtube_dl/extractor/nerdist.py b/youtube_dl/extractor/nerdist.py
deleted file mode 100644
index c6dc34be4..000000000
--- a/youtube_dl/extractor/nerdist.py
+++ /dev/null
@@ -1,80 +0,0 @@
-# encoding: utf-8
-from __future__ import unicode_literals
-
-from .common import InfoExtractor
-
-from ..utils import (
-    determine_ext,
-    parse_iso8601,
-    xpath_text,
-)
-
-
-class NerdistIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?nerdist\.com/vepisode/(?P<id>[^/?#]+)'
-    _TEST = {
-        'url': 'http://www.nerdist.com/vepisode/exclusive-which-dc-characters-w',
-        'md5': '3698ed582931b90d9e81e02e26e89f23',
-        'info_dict': {
-            'display_id': 'exclusive-which-dc-characters-w',
-            'id': 'RPHpvJyr',
-            'ext': 'mp4',
-            'title': 'Your TEEN TITANS Revealed! Who\'s on the show?',
-            'thumbnail': 're:^https?://.*/thumbs/.*\.jpg$',
-            'description': 'Exclusive: Find out which DC Comics superheroes will star in TEEN TITANS Live-Action TV Show on Nerdist News with Jessica Chobot!',
-            'uploader': 'Eric Diaz',
-            'upload_date': '20150202',
-            'timestamp': 1422892808,
-        }
-    }
-
-    def _real_extract(self, url):
-        display_id = self._match_id(url)
-        webpage = self._download_webpage(url, display_id)
-
-        video_id = self._search_regex(
-            r'''(?x)<script\s+(?:type="text/javascript"\s+)?
-                src="https?://content\.nerdist\.com/players/([a-zA-Z0-9_]+)-''',
-            webpage, 'video ID')
-        timestamp = parse_iso8601(self._html_search_meta(
-            'shareaholic:article_published_time', webpage, 'upload date'))
-        uploader = self._html_search_meta(
-            'shareaholic:article_author_name', webpage, 'article author')
-
-        doc = self._download_xml(
-            'http://content.nerdist.com/jw6/%s.xml' % video_id, video_id)
-        video_info = doc.find('.//item')
-        title = xpath_text(video_info, './title', fatal=True)
-        description = xpath_text(video_info, './description')
-        thumbnail = xpath_text(
-            video_info, './{http://rss.jwpcdn.com/}image', 'thumbnail')
-
-        formats = []
-        for source in video_info.findall('./{http://rss.jwpcdn.com/}source'):
-            vurl = source.attrib['file']
-            ext = determine_ext(vurl)
-            if ext == 'm3u8':
-                formats.extend(self._extract_m3u8_formats(
-                    vurl, video_id, entry_protocol='m3u8_native', ext='mp4',
-                    preference=0))
-            elif ext == 'smil':
-                formats.extend(self._extract_smil_formats(
-                    vurl, video_id, fatal=False
-                ))
-            else:
-                formats.append({
-                    'format_id': ext,
-                    'url': vurl,
-                })
-        self._sort_formats(formats)
-
-        return {
-            'id': video_id,
-            'display_id': display_id,
-            'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
-            'timestamp': timestamp,
-            'formats': formats,
-            'uploader': uploader,
-        }

From 21525bb8ca0e7229b3877acc162addcf5d5bcdfd Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 17 Apr 2016 02:38:05 +0800
Subject: [PATCH 4663/6219] [kuwo:category] Update the test

Now the webpage says there are 24 songs.
---
 youtube_dl/extractor/kuwo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/kuwo.py b/youtube_dl/extractor/kuwo.py
index 86c17c931..c0ece5113 100644
--- a/youtube_dl/extractor/kuwo.py
+++ b/youtube_dl/extractor/kuwo.py
@@ -268,7 +268,7 @@ class KuwoCategoryIE(InfoExtractor):
             'title': '八十年代精选',
             'description': '这些都是属于八十年代的回忆！',
         },
-        'playlist_count': 30,
+        'playlist_count': 24,
     }
 
     def _real_extract(self, url):

From d868f43c588b9929208823e86c12669d69e4e2cd Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 16 Apr 2016 19:45:56 +0100
Subject: [PATCH 4664/6219] [ffmpeg] check for - file name in
 _ffmpeg_filename_argument

---
 youtube_dl/downloader/external.py  | 5 +----
 youtube_dl/postprocessor/ffmpeg.py | 2 +-
 2 files changed, 2 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index cda39fe97..8d642fc3e 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -235,10 +235,7 @@ class FFmpegFD(ExternalFD):
             args += ['-f', EXT_TO_OUT_FORMATS.get(info_dict['ext'], info_dict['ext'])]
 
         args = [encodeArgument(opt) for opt in args]
-        if tmpfilename == '-':
-            args.append('pipe:1')
-        else:
-            args.append(encodeFilename(ffpp._ffmpeg_filename_argument(tmpfilename), True))
+        args.append(encodeFilename(ffpp._ffmpeg_filename_argument(tmpfilename), True))
 
         self._debug_cmd(args)
 
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index b64cd396b..30af6eb5a 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -175,7 +175,7 @@ class FFmpegPostProcessor(PostProcessor):
         # Always use 'file:' because the filename may contain ':' (ffmpeg
         # interprets that as a protocol) or can start with '-' (-- is broken in
         # ffmpeg, see https://ffmpeg.org/trac/ffmpeg/ticket/2127 for details)
-        return 'file:' + fn
+        return 'file:' + fn if fn != '-' else fn
 
 
 class FFmpegExtractAudioPP(FFmpegPostProcessor):

From bbb3f730bb729d258675dbc132f7daf6099edbba Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 16 Apr 2016 20:53:13 +0100
Subject: [PATCH 4665/6219] [onionstudios] extract m3u8 formats

---
 youtube_dl/extractor/onionstudios.py | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/onionstudios.py b/youtube_dl/extractor/onionstudios.py
index 0f1f448fe..6e843c327 100644
--- a/youtube_dl/extractor/onionstudios.py
+++ b/youtube_dl/extractor/onionstudios.py
@@ -4,7 +4,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import determine_ext
+from ..utils import (
+    determine_ext,
+    int_or_none,
+)
 
 
 class OnionStudiosIE(InfoExtractor):
@@ -17,7 +20,7 @@ class OnionStudiosIE(InfoExtractor):
             'id': '2937',
             'ext': 'mp4',
             'title': 'Hannibal charges forward, stops for a cocktail',
-            'description': 'md5:545299bda6abf87e5ec666548c6a9448',
+            'description': 'md5:e786add7f280b7f0fe237b64cc73df76',
             'thumbnail': 're:^https?://.*\.jpg$',
             'uploader': 'The A.V. Club',
             'uploader_id': 'TheAVClub',
@@ -42,9 +45,19 @@ class OnionStudiosIE(InfoExtractor):
 
         formats = []
         for src in re.findall(r'<source[^>]+src="([^"]+)"', webpage):
-            if determine_ext(src) != 'm3u8':  # m3u8 always results in 403
+            ext = determine_ext(src)
+            if ext == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    src, video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
+            else:
+                height = int_or_none(self._search_regex(
+                    r'/(\d+)\.%s' % ext, src, 'height', default=None))
                 formats.append({
+                    'format_id': ext + ('-%sp' % height if height else ''),
                     'url': src,
+                    'height': height,
+                    'ext': ext,
+                    'preference': 1,
                 })
         self._sort_formats(formats)
 

From b9f2fdd37fe2085deb09710a2084c940e9920304 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 16 Apr 2016 21:49:13 +0100
Subject: [PATCH 4666/6219] [ffmpeg] Clarify rationale for pipe(-) exclusion in
 _ffmpeg_filename_argument

---
 youtube_dl/postprocessor/ffmpeg.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 30af6eb5a..1793a878c 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -175,6 +175,7 @@ class FFmpegPostProcessor(PostProcessor):
         # Always use 'file:' because the filename may contain ':' (ffmpeg
         # interprets that as a protocol) or can start with '-' (-- is broken in
         # ffmpeg, see https://ffmpeg.org/trac/ffmpeg/ticket/2127 for details)
+        # Also leave '-' intact in order not to break streaming to stdout.
         return 'file:' + fn if fn != '-' else fn
 
 

From b1cf58f48ff1b748d0bfe73c75b20371a9a3629c Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 17 Apr 2016 15:08:51 +0800
Subject: [PATCH 4667/6219] [musicplayon] Fix extraction (closes #9222)

---
 youtube_dl/extractor/musicplayon.py | 47 ++++++++++-------------------
 1 file changed, 16 insertions(+), 31 deletions(-)

diff --git a/youtube_dl/extractor/musicplayon.py b/youtube_dl/extractor/musicplayon.py
index 50d92b50a..d74b276c1 100644
--- a/youtube_dl/extractor/musicplayon.py
+++ b/youtube_dl/extractor/musicplayon.py
@@ -1,10 +1,13 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
-from ..utils import int_or_none
+from ..compat import compat_urlparse
+from ..utils import (
+    int_or_none,
+    js_to_json,
+    mimetype2ext,
+)
 
 
 class MusicPlayOnIE(InfoExtractor):
@@ -12,6 +15,7 @@ class MusicPlayOnIE(InfoExtractor):
 
     _TEST = {
         'url': 'http://en.musicplayon.com/play?v=433377',
+        'md5': '00cdcdea1726abdf500d1e7fd6dd59bb',
         'info_dict': {
             'id': '433377',
             'ext': 'mp4',
@@ -20,15 +24,10 @@ class MusicPlayOnIE(InfoExtractor):
             'duration': 342,
             'uploader': 'ultrafish',
         },
-        'params': {
-            # m3u8 download
-            'skip_download': True,
-        },
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         page = self._download_webpage(url, video_id)
 
@@ -40,28 +39,14 @@ class MusicPlayOnIE(InfoExtractor):
         uploader = self._html_search_regex(
             r'<div>by&nbsp;<a href="[^"]+" class="purple">([^<]+)</a></div>', page, 'uploader', fatal=False)
 
-        formats = [
-            {
-                'url': 'http://media0-eu-nl.musicplayon.com/stream-mobile?id=%s&type=.mp4' % video_id,
-                'ext': 'mp4',
-            }
-        ]
-
-        manifest = self._download_webpage(
-            'http://en.musicplayon.com/manifest.m3u8?v=%s' % video_id, video_id, 'Downloading manifest')
-
-        for entry in manifest.split('#')[1:]:
-            if entry.startswith('EXT-X-STREAM-INF:'):
-                meta, url, _ = entry.split('\n')
-                params = dict(param.split('=') for param in meta.split(',')[1:])
-                formats.append({
-                    'url': url,
-                    'ext': 'mp4',
-                    'tbr': int(params['BANDWIDTH']),
-                    'width': int(params['RESOLUTION'].split('x')[1]),
-                    'height': int(params['RESOLUTION'].split('x')[-1]),
-                    'format_note': params['NAME'].replace('"', '').strip(),
-                })
+        sources = self._parse_json(
+            self._search_regex(r'setup\[\'_sources\'\]\s*=\s*([^;]+);', page, 'video sources'),
+            video_id, transform_source=js_to_json)
+        formats = [{
+            'url': compat_urlparse.urljoin(url, source['src']),
+            'ext': mimetype2ext(source.get('type')),
+            'format_note': source.get('data-res'),
+        } for source in sources]
 
         return {
             'id': video_id,

From 0f97fb4d00fa6353c9aebf38e632f559fe89b636 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 17 Apr 2016 17:22:59 +0800
Subject: [PATCH 4668/6219] [musicplayon] Relax _VALID_URL and improve metadata
 extraction

In r'pl=\d+&play=\d+' pages, several metadata items are missing

Closes #9222.
---
 youtube_dl/extractor/musicplayon.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/musicplayon.py b/youtube_dl/extractor/musicplayon.py
index d74b276c1..2174e5665 100644
--- a/youtube_dl/extractor/musicplayon.py
+++ b/youtube_dl/extractor/musicplayon.py
@@ -11,9 +11,9 @@ from ..utils import (
 
 
 class MusicPlayOnIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:.+?\.)?musicplayon\.com/play(?:-touch)?\?(?:v|pl=100&play)=(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:.+?\.)?musicplayon\.com/play(?:-touch)?\?(?:v|pl=\d+&play)=(?P<id>\d+)'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://en.musicplayon.com/play?v=433377',
         'md5': '00cdcdea1726abdf500d1e7fd6dd59bb',
         'info_dict': {
@@ -24,10 +24,16 @@ class MusicPlayOnIE(InfoExtractor):
             'duration': 342,
             'uploader': 'ultrafish',
         },
-    }
+    }, {
+        'url': 'http://en.musicplayon.com/play?pl=102&play=442629',
+        'only_matching': True,
+    }]
+
+    _URL_TEMPLATE = 'http://en.musicplayon.com/play?v=%s'
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
+        url = self._URL_TEMPLATE % video_id
 
         page = self._download_webpage(url, video_id)
 

From 7bab22a4029feb651d08f4b631359a0d1aaffd71 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 17 Apr 2016 14:06:19 +0100
Subject: [PATCH 4669/6219] [vice] remove unused import and variable

---
 youtube_dl/extractor/vice.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/youtube_dl/extractor/vice.py b/youtube_dl/extractor/vice.py
index ec1245b0e..95daf4dfd 100644
--- a/youtube_dl/extractor/vice.py
+++ b/youtube_dl/extractor/vice.py
@@ -3,7 +3,6 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from .ooyala import OoyalaIE
 from ..utils import ExtractorError
 
 
@@ -49,7 +48,6 @@ class ViceIE(InfoExtractor):
                 r'embedCode=([^&\'"]+)', webpage,
                 'ooyala embed code', default=None)
             if embed_code:
-                ooyala_url = OoyalaIE._url_for_embed_code(embed_code)
                 return self.url_result('ooyala:%s' % embed_code, 'Ooyala')
             youtube_id = self._search_regex(
                 r'data-youtube-id="([^"]+)"', webpage, 'youtube id')

From 1b6182d8f759546b54a675123901348361bad979 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 17 Apr 2016 17:07:57 +0200
Subject: [PATCH 4670/6219] [youtube:playlist] Fetch all the videos in a mix
 (fixes #3837)

Since there doesn't seem to be any indication, it stops when there aren't new videos in the webpage.
---
 test/test_youtube_lists.py      |  2 +-
 youtube_dl/extractor/youtube.py | 28 ++++++++++++++++++++--------
 2 files changed, 21 insertions(+), 9 deletions(-)

diff --git a/test/test_youtube_lists.py b/test/test_youtube_lists.py
index 47df0f348..af1c45421 100644
--- a/test/test_youtube_lists.py
+++ b/test/test_youtube_lists.py
@@ -44,7 +44,7 @@ class TestYoutubeLists(unittest.TestCase):
         ie = YoutubePlaylistIE(dl)
         result = ie.extract('https://www.youtube.com/watch?v=W01L70IGBgE&index=2&list=RDOQpdSVF_k_w')
         entries = result['entries']
-        self.assertTrue(len(entries) >= 20)
+        self.assertTrue(len(entries) >= 50)
         original_video = entries[0]
         self.assertEqual(original_video['id'], 'OQpdSVF_k_w')
 
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 44c1191bd..a4dd628a1 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1818,20 +1818,32 @@ class YoutubePlaylistIE(YoutubePlaylistBaseInfoExtractor):
     def _extract_mix(self, playlist_id):
         # The mixes are generated from a single video
         # the id of the playlist is just 'RD' + video_id
-        url = 'https://youtube.com/watch?v=%s&list=%s' % (playlist_id[-11:], playlist_id)
-        webpage = self._download_webpage(
-            url, playlist_id, 'Downloading Youtube mix')
+        ids = []
+        last_id = playlist_id[-11:]
+        for n in itertools.count(1):
+            url = 'https://youtube.com/watch?v=%s&list=%s' % (last_id, playlist_id)
+            webpage = self._download_webpage(
+                url, playlist_id, 'Downloading page {0} of Youtube mix'.format(n))
+            new_ids = orderedSet(re.findall(
+                r'''(?xs)data-video-username=".*?".*?
+                           href="/watch\?v=([0-9A-Za-z_-]{11})&amp;[^"]*?list=%s''' % re.escape(playlist_id),
+                webpage))
+            # Fetch new pages until all the videos are repeated, it seems that
+            # there are always 51 unique videos.
+            new_ids = [_id for _id in new_ids if _id not in ids]
+            if not new_ids:
+                break
+            ids.extend(new_ids)
+            last_id = ids[-1]
+
+        url_results = self._ids_to_results(ids)
+
         search_title = lambda class_name: get_element_by_attribute('class', class_name, webpage)
         title_span = (
             search_title('playlist-title') or
             search_title('title long-title') or
             search_title('title'))
         title = clean_html(title_span)
-        ids = orderedSet(re.findall(
-            r'''(?xs)data-video-username=".*?".*?
-                       href="/watch\?v=([0-9A-Za-z_-]{11})&amp;[^"]*?list=%s''' % re.escape(playlist_id),
-            webpage))
-        url_results = self._ids_to_results(ids)
 
         return self.playlist_result(url_results, playlist_id, title)
 

From 4fdf0823750b594d5408640829ad235bd210e989 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 17 Apr 2016 23:12:23 +0600
Subject: [PATCH 4671/6219] [theonion] Remove extractor (Closes #9220)

It now uses generic onionstudios embed
---
 youtube_dl/extractor/extractors.py |  1 -
 youtube_dl/extractor/theonion.py   | 63 ------------------------------
 2 files changed, 64 deletions(-)
 delete mode 100644 youtube_dl/extractor/theonion.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 06b3d5e24..aa83daa02 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -751,7 +751,6 @@ from .teletask import TeleTaskIE
 from .testurl import TestURLIE
 from .tf1 import TF1IE
 from .theintercept import TheInterceptIE
-from .theonion import TheOnionIE
 from .theplatform import (
     ThePlatformIE,
     ThePlatformFeedIE,
diff --git a/youtube_dl/extractor/theonion.py b/youtube_dl/extractor/theonion.py
deleted file mode 100644
index 10239c906..000000000
--- a/youtube_dl/extractor/theonion.py
+++ /dev/null
@@ -1,63 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-import re
-
-from .common import InfoExtractor
-
-
-class TheOnionIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?theonion\.com/video/[^,]+,(?P<id>[0-9]+)/?'
-    _TEST = {
-        'url': 'http://www.theonion.com/video/man-wearing-mm-jacket-gods-image,36918/',
-        'md5': '19eaa9a39cf9b9804d982e654dc791ee',
-        'info_dict': {
-            'id': '2133',
-            'ext': 'mp4',
-            'title': 'Man Wearing M&M Jacket Apparently Made In God\'s Image',
-            'description': 'md5:cc12448686b5600baae9261d3e180910',
-            'thumbnail': 're:^https?://.*\.jpg\?\d+$',
-        }
-    }
-
-    def _real_extract(self, url):
-        display_id = self._match_id(url)
-        webpage = self._download_webpage(url, display_id)
-
-        video_id = self._search_regex(
-            r'"videoId":\s(\d+),', webpage, 'video ID')
-        title = self._og_search_title(webpage)
-        description = self._og_search_description(webpage)
-        thumbnail = self._og_search_thumbnail(webpage)
-
-        sources = re.findall(r'<source src="([^"]+)" type="([^"]+)"', webpage)
-        formats = []
-        for src, type_ in sources:
-            if type_ == 'video/mp4':
-                formats.append({
-                    'format_id': 'mp4_sd',
-                    'preference': 1,
-                    'url': src,
-                })
-            elif type_ == 'video/webm':
-                formats.append({
-                    'format_id': 'webm_sd',
-                    'preference': 0,
-                    'url': src,
-                })
-            elif type_ == 'application/x-mpegURL':
-                formats.extend(
-                    self._extract_m3u8_formats(src, display_id, preference=-1))
-            else:
-                self.report_warning(
-                    'Encountered unexpected format: %s' % type_)
-        self._sort_formats(formats)
-
-        return {
-            'id': video_id,
-            'display_id': display_id,
-            'title': title,
-            'formats': formats,
-            'thumbnail': thumbnail,
-            'description': description,
-        }

From 241a318f2766ec879346fc9763916700c20d65d7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 18 Apr 2016 21:40:28 +0600
Subject: [PATCH 4672/6219] [vimeo] Improve _VALID_URL (Closes #9229)

---
 youtube_dl/extractor/vimeo.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 707a5735a..a4833a858 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -81,7 +81,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
                             \.
                         )?
                         vimeo(?P<pro>pro)?\.com/
-                        (?!channels/[^/?#]+/?(?:$|[?#])|(?:album|ondemand)/)
+                        (?!channels/[^/?#]+/?(?:$|[?#])|[^/]+/review/|(?:album|ondemand)/)
                         (?:.*?/)?
                         (?:
                             (?:
@@ -90,6 +90,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
                             )?
                         (?:videos?/)?
                         (?P<id>[0-9]+)
+                        (?:/[\da-f]+)?
                         /?(?:[?&].*)?(?:[#].*)?$
                     '''
     IE_NAME = 'vimeo'
@@ -232,6 +233,10 @@ class VimeoIE(VimeoBaseInfoExtractor):
             'url': 'https://vimeo.com/7809605',
             'only_matching': True,
         },
+        {
+            'url': 'https://vimeo.com/160743502/abd0e13fb4',
+            'only_matching': True,
+        }
     ]
 
     @staticmethod

From 81215d56524e2b4d26898e5cc3ecd92a856df344 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 19 Apr 2016 03:03:52 +0200
Subject: [PATCH 4673/6219] release 2016.04.19

---
 .github/ISSUE_TEMPLATE.md |  6 +++---
 docs/supportedsites.md    | 10 +++++-----
 youtube_dl/version.py     |  2 +-
 3 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index caed64e38..35f8e6863 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.04.13*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.04.13**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.04.19*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.04.19**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.04.13
+[debug] youtube-dl version 2016.04.19
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 51a6b5609..e12a7d182 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -50,6 +50,7 @@
  - **arte.tv:ddc**
  - **arte.tv:embed**
  - **arte.tv:future**
+ - **arte.tv:info**
  - **arte.tv:magazine**
  - **AtresPlayer**
  - **ATTTechChannel**
@@ -350,13 +351,15 @@
  - **miomio.tv**
  - **MiTele**: mitele.es
  - **mixcloud**
+ - **mixcloud:playlist**
+ - **mixcloud:stream**
+ - **mixcloud:user**
  - **MLB**
  - **Mnet**
  - **MoeVideo**: LetitBit video services: moevideo.net, playreplay.net and videochart.net
  - **Mofosex**
  - **Mojvideo**
  - **Moniker**: allmyvideos.net and vidspot.net
- - **mooshare**: Mooshare.biz
  - **Morningstar**: morningstar.com
  - **Motherless**
  - **Motorsport**: motorsport.com
@@ -393,7 +396,6 @@
  - **ndr:embed:base**
  - **NDTV**
  - **NerdCubedFeed**
- - **Nerdist**
  - **netease:album**: 网易云音乐 - 专辑
  - **netease:djradio**: 网易云音乐 - 电台
  - **netease:mv**: 网易云音乐 - MV
@@ -609,6 +611,7 @@
  - **Tagesschau**
  - **Tapely**
  - **Tass**
+ - **TDSLifeway**
  - **teachertube**: teachertube.com videos
  - **teachertube:user:collection**: teachertube.com user and collection videos
  - **TeachingChannel**
@@ -625,7 +628,6 @@
  - **TeleTask**
  - **TF1**
  - **TheIntercept**
- - **TheOnion**
  - **ThePlatform**
  - **ThePlatformFeed**
  - **TheScene**
@@ -684,7 +686,6 @@
  - **twitter**
  - **twitter:amplify**
  - **twitter:card**
- - **Ubu**
  - **udemy**
  - **udemy:course**
  - **UDNEmbed**: 聯合影音
@@ -754,7 +755,6 @@
  - **Walla**
  - **WashingtonPost**
  - **wat.tv**
- - **WayOfTheMaster**
  - **WDR**
  - **wdr:mobile**
  - **WDRMaus**: Sendung mit der Maus
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 0618d9a4f..140a67847 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.04.13'
+__version__ = '2016.04.19'

From f77970765adb0ffe837706ef8eb041ea448c34b3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 20 Apr 2016 03:37:23 +0600
Subject: [PATCH 4674/6219] [people] Add extractor

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/people.py     | 33 ++++++++++++++++++++++++++++++
 2 files changed, 34 insertions(+)
 create mode 100644 youtube_dl/extractor/people.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index aa83daa02..9b8e1ab25 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -559,6 +559,7 @@ from .pandoratv import PandoraTVIE
 from .parliamentliveuk import ParliamentLiveUKIE
 from .patreon import PatreonIE
 from .pbs import PBSIE
+from .people import PeopleIE
 from .periscope import PeriscopeIE
 from .philharmoniedeparis import PhilharmonieDeParisIE
 from .phoenix import PhoenixIE
diff --git a/youtube_dl/extractor/people.py b/youtube_dl/extractor/people.py
new file mode 100644
index 000000000..f25459045
--- /dev/null
+++ b/youtube_dl/extractor/people.py
@@ -0,0 +1,33 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class PeopleIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?people\.com/people/videos/0,,(?P<id>\d+),00\.html'
+
+    _TEST = {
+        'url': 'http://www.people.com/people/videos/0,,20995451,00.html',
+        'info_dict': {
+            'id': 'ref:20995451',
+            'ext': 'mp4',
+            'title': 'Astronaut Love Triangle Victim Speaks Out: “The Crime in 2007 Hasn’t Defined Us”',
+            'description': 'Colleen Shipman speaks to PEOPLE for the first time about life after the attack',
+            'thumbnail': 're:^https?://.*\.jpg',
+            'duration': 246.318,
+            'timestamp': 1458720585,
+            'upload_date': '20160323',
+            'uploader_id': '416418724',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+        'add_ie': ['BrightcoveNew'],
+    }
+
+    def _real_extract(self, url):
+        return self.url_result(
+            'http://players.brightcove.net/416418724/default_default/index.html?videoId=ref:%s'
+            % self._match_id(url), 'BrightcoveNew')

From 107701fcfced36af62465f4a56a0a9723f5253bf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 20 Apr 2016 03:40:02 +0600
Subject: [PATCH 4675/6219] [people] Remove bogus comment

---
 youtube_dl/extractor/people.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/people.py b/youtube_dl/extractor/people.py
index f25459045..9ecdbc13b 100644
--- a/youtube_dl/extractor/people.py
+++ b/youtube_dl/extractor/people.py
@@ -21,7 +21,6 @@ class PeopleIE(InfoExtractor):
             'uploader_id': '416418724',
         },
         'params': {
-            # m3u8 download
             'skip_download': True,
         },
         'add_ie': ['BrightcoveNew'],

From 494ab6db7394b39126d775efe03eb6dab428eff9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 20 Apr 2016 22:14:32 +0600
Subject: [PATCH 4676/6219] [youtube] Capture and output login error message

---
 youtube_dl/extractor/youtube.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index a4dd628a1..44f98d294 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -125,6 +125,12 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
         if login_results is False:
             return False
 
+        error_msg = self._html_search_regex(
+            r'<[^>]+id="errormsg_0_Passwd"[^>]*>([^<]+)<',
+            login_results, 'error message', default=None)
+        if error_msg:
+            raise ExtractorError('Unable to login: %s' % error_msg, expected=True)
+
         if re.search(r'id="errormsg_0_Passwd"', login_results) is not None:
             raise ExtractorError('Please use your account password and a two-factor code instead of an application-specific password.', expected=True)
 

From 16e633a5d715a36fa4dbff1dbc8bfbf01ad081db Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 20 Apr 2016 23:29:02 +0600
Subject: [PATCH 4677/6219] [quickvid] Remove extractor (Closes #9258)

---
 youtube_dl/extractor/extractors.py |  1 -
 youtube_dl/extractor/quickvid.py   | 54 ------------------------------
 2 files changed, 55 deletions(-)
 delete mode 100644 youtube_dl/extractor/quickvid.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 9b8e1ab25..1f3172119 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -601,7 +601,6 @@ from .qqmusic import (
     QQMusicToplistIE,
     QQMusicPlaylistIE,
 )
-from .quickvid import QuickVidIE
 from .r7 import R7IE
 from .radiode import RadioDeIE
 from .radiojavan import RadioJavanIE
diff --git a/youtube_dl/extractor/quickvid.py b/youtube_dl/extractor/quickvid.py
deleted file mode 100644
index f414e2384..000000000
--- a/youtube_dl/extractor/quickvid.py
+++ /dev/null
@@ -1,54 +0,0 @@
-from __future__ import unicode_literals
-
-import re
-
-from .common import InfoExtractor
-from ..compat import (
-    compat_urlparse,
-)
-from ..utils import (
-    determine_ext,
-    int_or_none,
-)
-
-
-class QuickVidIE(InfoExtractor):
-    _VALID_URL = r'https?://(www\.)?quickvid\.org/watch\.php\?v=(?P<id>[a-zA-Z_0-9-]+)'
-    _TEST = {
-        'url': 'http://quickvid.org/watch.php?v=sUQT3RCG8dx',
-        'md5': 'c0c72dd473f260c06c808a05d19acdc5',
-        'info_dict': {
-            'id': 'sUQT3RCG8dx',
-            'ext': 'mp4',
-            'title': 'Nick Offerman\'s Summer Reading Recap',
-            'thumbnail': 're:^https?://.*\.(?:png|jpg|gif)$',
-            'view_count': int,
-        },
-        'skip': 'Not accessible from Travis CI server',
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-
-        title = self._html_search_regex(r'<h2>(.*?)</h2>', webpage, 'title')
-        view_count = int_or_none(self._html_search_regex(
-            r'(?s)<div id="views">(.*?)</div>',
-            webpage, 'view count', fatal=False))
-        video_code = self._search_regex(
-            r'(?s)<video id="video"[^>]*>(.*?)</video>', webpage, 'video code')
-        formats = [
-            {
-                'url': compat_urlparse.urljoin(url, src),
-                'format_id': determine_ext(src, None),
-            } for src in re.findall('<source\s+src="([^"]+)"', video_code)
-        ]
-        self._sort_formats(formats)
-
-        return {
-            'id': video_id,
-            'title': title,
-            'formats': formats,
-            'thumbnail': self._og_search_thumbnail(webpage),
-            'view_count': view_count,
-        }

From 12a51345962bc032de45952ec60989f46544101c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 20 Apr 2016 23:52:41 +0600
Subject: [PATCH 4678/6219] [tvigle] Fix extraction (Closes #9259)

---
 youtube_dl/extractor/tvigle.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tvigle.py b/youtube_dl/extractor/tvigle.py
index dc3a8334a..f3aba3530 100644
--- a/youtube_dl/extractor/tvigle.py
+++ b/youtube_dl/extractor/tvigle.py
@@ -58,7 +58,9 @@ class TvigleIE(InfoExtractor):
         if not video_id:
             webpage = self._download_webpage(url, display_id)
             video_id = self._html_search_regex(
-                r'class="video-preview current_playing" id="(\d+)">',
+                (r'<div[^>]+class=["\']player["\'][^>]+id=["\'](\d+)',
+                 r'var\s+cloudId\s*=\s*["\'](\d+)',
+                 r'class="video-preview current_playing" id="(\d+)"'),
                 webpage, 'video id')
 
         video_data = self._download_json(

From 2c0d9c6217f994b2742ff4e8e3fba7c7469b9f81 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 21 Apr 2016 13:06:06 +0800
Subject: [PATCH 4679/6219] [extractor/common] Allow empty post data

---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 5269059d0..02cd2c003 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -382,7 +382,7 @@ class InfoExtractor(object):
         else:
             if query:
                 url_or_request = update_url_query(url_or_request, query)
-            if data or headers:
+            if data is not None or headers:
                 url_or_request = sanitized_Request(url_or_request, data, headers)
         try:
             return self._downloader.urlopen(url_or_request)

From 4dccea8ad01d51dcdfd4ecc79442bbd30e29871e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 21 Apr 2016 13:07:53 +0800
Subject: [PATCH 4680/6219] [streetvoice] Fix extraction

The old API results in URLs with HTTP 403 from time to time.

Hopefully fixes #9219.
---
 youtube_dl/extractor/streetvoice.py | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/streetvoice.py b/youtube_dl/extractor/streetvoice.py
index 6a57fa60a..e529051d1 100644
--- a/youtube_dl/extractor/streetvoice.py
+++ b/youtube_dl/extractor/streetvoice.py
@@ -14,7 +14,6 @@ class StreetVoiceIE(InfoExtractor):
         'info_dict': {
             'id': '94440',
             'ext': 'mp3',
-            'filesize': 4167053,
             'title': '輸',
             'description': 'Crispy脆樂團 - 輸',
             'thumbnail': 're:^https?://.*\.jpg$',
@@ -32,20 +31,19 @@ class StreetVoiceIE(InfoExtractor):
         song_id = self._match_id(url)
 
         song = self._download_json(
-            'http://streetvoice.com/music/api/song/%s' % song_id, song_id)
+            'https://streetvoice.com/api/v1/public/song/%s/' % song_id, song_id, data=b'')
 
         title = song['name']
-        author = song['musician']['name']
+        author = song['user']['nickname']
 
         return {
             'id': song_id,
             'url': song['file'],
-            'filesize': song.get('size'),
             'title': title,
             'description': '%s - %s' % (author, title),
             'thumbnail': self._proto_relative_url(song.get('image'), 'http:'),
             'duration': song.get('length'),
             'upload_date': unified_strdate(song.get('created_at')),
             'uploader': author,
-            'uploader_id': compat_str(song['musician']['id']),
+            'uploader_id': compat_str(song['user']['id']),
         }

From 99ef96f84c2c9fa1267d6edb836c8f1945295424 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 21 Apr 2016 18:07:03 +0800
Subject: [PATCH 4681/6219] [gdcvault] Fix for videos with hard-coded hostnames

Fixes #9248
---
 youtube_dl/extractor/gdcvault.py | 28 ++++++++++++++++++++++------
 1 file changed, 22 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/gdcvault.py b/youtube_dl/extractor/gdcvault.py
index 25e93c9a4..3ebcaf733 100644
--- a/youtube_dl/extractor/gdcvault.py
+++ b/youtube_dl/extractor/gdcvault.py
@@ -51,18 +51,34 @@ class GDCVaultIE(InfoExtractor):
         {
             'url': 'http://gdcvault.com/play/1020791/',
             'only_matching': True,
-        }
+        },
+        {
+            'url': 'http://gdcvault.com/play/1023460/Tenacious-Design-and-The-Interface',
+            'md5': 'a8efb6c31ed06ca8739294960b2dbabd',
+            'info_dict': {
+                'id': '1023460',
+                'ext': 'mp4',
+                'display_id': 'Tenacious-Design-and-The-Interface',
+                'title': 'Tenacious Design and The Interface of \'Destiny\'',
+            },
+        },
     ]
 
     def _parse_mp4(self, xml_description):
         video_formats = []
-        mp4_video = xml_description.find('./metadata/mp4video')
-        if mp4_video is None:
-            return None
+        video_root = None
+
+        mp4_video = xml_description.find('./metadata/mp4video')
+        if mp4_video is not None:
+            mobj = re.match(r'(?P<root>https?://.*?/).*', mp4_video.text)
+            video_root = mobj.group('root')
+        if video_root is None:
+            # Hard-coded in http://evt.dispeak.com/ubm/gdc/sf16/custom/player2.js
+            video_root = 'http://s3-2u.digitallyspeaking.com/'
 
-        mobj = re.match(r'(?P<root>https?://.*?/).*', mp4_video.text)
-        video_root = mobj.group('root')
         formats = xml_description.findall('./metadata/MBRVideos/MBRVideo')
+        if not formats:
+            return None
         for format in formats:
             mobj = re.match(r'mp4\:(?P<path>.*)', format.find('streamName').text)
             url = video_root + mobj.group('path')

From ec59d657e7d898cce8f3a1b6556a79fd9495fc9d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 21 Apr 2016 19:36:33 +0800
Subject: [PATCH 4682/6219] [dispeak] Add new extractor

Both GDCVault and GPUTechConf uses the service of DigitalSpeaking.
---
 youtube_dl/extractor/dispeak.py     | 111 ++++++++++++++++++++++++++++
 youtube_dl/extractor/extractors.py  |   1 +
 youtube_dl/extractor/gdcvault.py    |  74 +------------------
 youtube_dl/extractor/gputechconf.py |  36 ++-------
 4 files changed, 123 insertions(+), 99 deletions(-)
 create mode 100644 youtube_dl/extractor/dispeak.py

diff --git a/youtube_dl/extractor/dispeak.py b/youtube_dl/extractor/dispeak.py
new file mode 100644
index 000000000..6ebc3255a
--- /dev/null
+++ b/youtube_dl/extractor/dispeak.py
@@ -0,0 +1,111 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    parse_duration,
+    remove_end,
+    xpath_element,
+    xpath_text,
+)
+
+
+class DigitalSpeakingIE(InfoExtractor):
+    _VALID_URL = r'http://evt.dispeak.com/([^/]+/)+xml/(?P<id>[^.]+).xml'
+
+    _TEST = {
+        # From http://evt.dispeak.com/ubm/gdc/sf16/xml/840376_BQRC.xml
+        'url': 'http://evt.dispeak.com/ubm/gdc/sf16/xml/840376_BQRC.xml',
+        'md5': 'a8efb6c31ed06ca8739294960b2dbabd',
+        'info_dict': {
+            'id': '840376_BQRC',
+            'ext': 'mp4',
+            'title': 'Tenacious Design and The Interface of \'Destiny\'',
+        },
+    }
+
+    def _parse_mp4(self, metadata):
+        video_formats = []
+        video_root = None
+
+        mp4_video = xpath_text(metadata, './mp4video', default=None)
+        if mp4_video is not None:
+            mobj = re.match(r'(?P<root>https?://.*?/).*', mp4_video)
+            video_root = mobj.group('root')
+        if video_root is None:
+            http_host = xpath_text(metadata, 'httpHost', default=None)
+            if http_host:
+                video_root = 'http://%s/' % http_host
+        if video_root is None:
+            # Hard-coded in http://evt.dispeak.com/ubm/gdc/sf16/custom/player2.js
+            # Works for GPUTechConf, too
+            video_root = 'http://s3-2u.digitallyspeaking.com/'
+
+        formats = metadata.findall('./MBRVideos/MBRVideo')
+        if not formats:
+            return None
+        for a_format in formats:
+            stream_name = xpath_text(a_format, 'streamName', fatal=True)
+            video_path = re.match(r'mp4\:(?P<path>.*)', stream_name).group('path')
+            url = video_root + video_path
+            vbr = xpath_text(a_format, 'bitrate')
+            video_formats.append({
+                'url': url,
+                'vbr': int_or_none(vbr),
+            })
+        return video_formats
+
+    def _parse_flv(self, metadata):
+        formats = []
+        akamai_url = xpath_text(metadata, './akamaiHost', fatal=True)
+        audios = metadata.find('./audios')
+        if audios is not None:
+            for audio in audios:
+                formats.append({
+                    'url': 'rtmp://%s/ondemand?ovpfv=1.1' % akamai_url,
+                    'play_path': remove_end(audio.get('url'), '.flv'),
+                    'ext': 'flv',
+                    'vcodec': 'none',
+                    'format_id': audio.get('code'),
+                })
+        slide_video_path = xpath_text(metadata, './slideVideo', fatal=True)
+        formats.append({
+            'url': 'rtmp://%s/ondemand?ovpfv=1.1' % akamai_url,
+            'play_path': remove_end(slide_video_path, '.flv'),
+            'ext': 'flv',
+            'format_note': 'slide deck video',
+            'quality': -2,
+            'preference': -2,
+            'format_id': 'slides',
+        })
+        speaker_video_path = xpath_text(metadata, './speakerVideo', fatal=True)
+        formats.append({
+            'url': 'rtmp://%s/ondemand?ovpfv=1.1' % akamai_url,
+            'play_path': remove_end(speaker_video_path, '.flv'),
+            'ext': 'flv',
+            'format_note': 'speaker video',
+            'quality': -1,
+            'preference': -1,
+            'format_id': 'speaker',
+        })
+        return formats
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        xml_description = self._download_xml(url, video_id)
+        metadata = xpath_element(xml_description, 'metadata')
+
+        video_formats = self._parse_mp4(metadata)
+        if video_formats is None:
+            video_formats = self._parse_flv(metadata)
+
+        return {
+            'id': video_id,
+            'formats': video_formats,
+            'title': xpath_text(metadata, 'title', fatal=True),
+            'duration': parse_duration(xpath_text(metadata, 'endTime')),
+            'creator': xpath_text(metadata, 'speaker'),
+        }
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 1f3172119..84bdf5e97 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -197,6 +197,7 @@ from .dump import DumpIE
 from .dumpert import DumpertIE
 from .defense import DefenseGouvFrIE
 from .discovery import DiscoveryIE
+from .dispeak import DigitalSpeakingIE
 from .dropbox import DropboxIE
 from .dw import (
     DWIE,
diff --git a/youtube_dl/extractor/gdcvault.py b/youtube_dl/extractor/gdcvault.py
index 3ebcaf733..01e1ceec8 100644
--- a/youtube_dl/extractor/gdcvault.py
+++ b/youtube_dl/extractor/gdcvault.py
@@ -4,7 +4,6 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
-    remove_end,
     HEADRequest,
     sanitized_Request,
     urlencode_postdata,
@@ -64,66 +63,6 @@ class GDCVaultIE(InfoExtractor):
         },
     ]
 
-    def _parse_mp4(self, xml_description):
-        video_formats = []
-        video_root = None
-
-        mp4_video = xml_description.find('./metadata/mp4video')
-        if mp4_video is not None:
-            mobj = re.match(r'(?P<root>https?://.*?/).*', mp4_video.text)
-            video_root = mobj.group('root')
-        if video_root is None:
-            # Hard-coded in http://evt.dispeak.com/ubm/gdc/sf16/custom/player2.js
-            video_root = 'http://s3-2u.digitallyspeaking.com/'
-
-        formats = xml_description.findall('./metadata/MBRVideos/MBRVideo')
-        if not formats:
-            return None
-        for format in formats:
-            mobj = re.match(r'mp4\:(?P<path>.*)', format.find('streamName').text)
-            url = video_root + mobj.group('path')
-            vbr = format.find('bitrate').text
-            video_formats.append({
-                'url': url,
-                'vbr': int(vbr),
-            })
-        return video_formats
-
-    def _parse_flv(self, xml_description):
-        formats = []
-        akamai_url = xml_description.find('./metadata/akamaiHost').text
-        audios = xml_description.find('./metadata/audios')
-        if audios is not None:
-            for audio in audios:
-                formats.append({
-                    'url': 'rtmp://%s/ondemand?ovpfv=1.1' % akamai_url,
-                    'play_path': remove_end(audio.get('url'), '.flv'),
-                    'ext': 'flv',
-                    'vcodec': 'none',
-                    'format_id': audio.get('code'),
-                })
-        slide_video_path = xml_description.find('./metadata/slideVideo').text
-        formats.append({
-            'url': 'rtmp://%s/ondemand?ovpfv=1.1' % akamai_url,
-            'play_path': remove_end(slide_video_path, '.flv'),
-            'ext': 'flv',
-            'format_note': 'slide deck video',
-            'quality': -2,
-            'preference': -2,
-            'format_id': 'slides',
-        })
-        speaker_video_path = xml_description.find('./metadata/speakerVideo').text
-        formats.append({
-            'url': 'rtmp://%s/ondemand?ovpfv=1.1' % akamai_url,
-            'play_path': remove_end(speaker_video_path, '.flv'),
-            'ext': 'flv',
-            'format_note': 'speaker video',
-            'quality': -1,
-            'preference': -1,
-            'format_id': 'speaker',
-        })
-        return formats
-
     def _login(self, webpage_url, display_id):
         (username, password) = self._get_login_info()
         if username is None or password is None:
@@ -199,17 +138,10 @@ class GDCVaultIE(InfoExtractor):
                 r'<iframe src=".*?\?xmlURL=xml/(?P<xml_file>.+?\.xml).*?".*?</iframe>',
                 start_page, 'xml filename')
 
-        xml_description = self._download_xml(
-            '%s/xml/%s' % (xml_root, xml_name), display_id)
-
-        video_title = xml_description.find('./metadata/title').text
-        video_formats = self._parse_mp4(xml_description)
-        if video_formats is None:
-            video_formats = self._parse_flv(xml_description)
-
         return {
+            '_type': 'url_transparent',
             'id': video_id,
             'display_id': display_id,
-            'title': video_title,
-            'formats': video_formats,
+            'url': '%s/xml/%s' % (xml_root, xml_name),
+            'ie': 'DigitalSpeaking',
         }
diff --git a/youtube_dl/extractor/gputechconf.py b/youtube_dl/extractor/gputechconf.py
index 145b55bf3..359b348e4 100644
--- a/youtube_dl/extractor/gputechconf.py
+++ b/youtube_dl/extractor/gputechconf.py
@@ -2,12 +2,6 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..utils import (
-    xpath_element,
-    xpath_text,
-    int_or_none,
-    parse_duration,
-)
 
 
 class GPUTechConfIE(InfoExtractor):
@@ -27,29 +21,15 @@ class GPUTechConfIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        root_path = self._search_regex(r'var\s+rootPath\s*=\s*"([^"]+)', webpage, 'root path', 'http://evt.dispeak.com/nvidia/events/gtc15/')
-        xml_file_id = self._search_regex(r'var\s+xmlFileId\s*=\s*"([^"]+)', webpage, 'xml file id')
-
-        doc = self._download_xml('%sxml/%s.xml' % (root_path, xml_file_id), video_id)
-
-        metadata = xpath_element(doc, 'metadata')
-        http_host = xpath_text(metadata, 'httpHost', 'http host', True)
-        mbr_videos = xpath_element(metadata, 'MBRVideos')
-
-        formats = []
-        for mbr_video in mbr_videos.findall('MBRVideo'):
-            stream_name = xpath_text(mbr_video, 'streamName')
-            if stream_name:
-                formats.append({
-                    'url': 'http://%s/%s' % (http_host, stream_name.replace('mp4:', '')),
-                    'tbr': int_or_none(xpath_text(mbr_video, 'bitrate')),
-                })
-        self._sort_formats(formats)
+        root_path = self._search_regex(
+            r'var\s+rootPath\s*=\s*"([^"]+)', webpage, 'root path',
+            default='http://evt.dispeak.com/nvidia/events/gtc15/')
+        xml_file_id = self._search_regex(
+            r'var\s+xmlFileId\s*=\s*"([^"]+)', webpage, 'xml file id')
 
         return {
+            '_type': 'url_transparent',
             'id': video_id,
-            'title': xpath_text(metadata, 'title'),
-            'duration': parse_duration(xpath_text(metadata, 'endTime')),
-            'creator': xpath_text(metadata, 'speaker'),
-            'formats': formats,
+            'url': '%sxml/%s.xml' % (root_path, xml_file_id),
+            'ie': 'DigitalSpeaking',
         }

From 442c4d361f309f64176f2c1b053ea8775701634c Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 21 Apr 2016 19:46:32 +0800
Subject: [PATCH 4683/6219] [dispeak/gdcvault] Add the test case from #5784

---
 youtube_dl/extractor/dispeak.py  | 29 ++++++++++++++++-------------
 youtube_dl/extractor/gdcvault.py | 14 ++++++++++++++
 2 files changed, 30 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/dispeak.py b/youtube_dl/extractor/dispeak.py
index 6ebc3255a..c74fa7e07 100644
--- a/youtube_dl/extractor/dispeak.py
+++ b/youtube_dl/extractor/dispeak.py
@@ -13,9 +13,9 @@ from ..utils import (
 
 
 class DigitalSpeakingIE(InfoExtractor):
-    _VALID_URL = r'http://evt.dispeak.com/([^/]+/)+xml/(?P<id>[^.]+).xml'
+    _VALID_URL = r'http://(?:evt\.dispeak|events\.digitallyspeaking)\.com/([^/]+/)+xml/(?P<id>[^.]+).xml'
 
-    _TEST = {
+    _TESTS = [{
         # From http://evt.dispeak.com/ubm/gdc/sf16/xml/840376_BQRC.xml
         'url': 'http://evt.dispeak.com/ubm/gdc/sf16/xml/840376_BQRC.xml',
         'md5': 'a8efb6c31ed06ca8739294960b2dbabd',
@@ -24,7 +24,11 @@ class DigitalSpeakingIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Tenacious Design and The Interface of \'Destiny\'',
         },
-    }
+    }, {
+        # From http://www.gdcvault.com/play/1014631/Classic-Game-Postmortem-PAC
+        'url': 'http://events.digitallyspeaking.com/gdc/sf11/xml/12396_1299111843500GMPX.xml',
+        'only_matching': True,
+    }]
 
     def _parse_mp4(self, metadata):
         video_formats = []
@@ -60,16 +64,15 @@ class DigitalSpeakingIE(InfoExtractor):
     def _parse_flv(self, metadata):
         formats = []
         akamai_url = xpath_text(metadata, './akamaiHost', fatal=True)
-        audios = metadata.find('./audios')
-        if audios is not None:
-            for audio in audios:
-                formats.append({
-                    'url': 'rtmp://%s/ondemand?ovpfv=1.1' % akamai_url,
-                    'play_path': remove_end(audio.get('url'), '.flv'),
-                    'ext': 'flv',
-                    'vcodec': 'none',
-                    'format_id': audio.get('code'),
-                })
+        audios = metadata.findall('./audios/audio')
+        for audio in audios:
+            formats.append({
+                'url': 'rtmp://%s/ondemand?ovpfv=1.1' % akamai_url,
+                'play_path': remove_end(audio.get('url'), '.flv'),
+                'ext': 'flv',
+                'vcodec': 'none',
+                'format_id': audio.get('code'),
+            })
         slide_video_path = xpath_text(metadata, './slideVideo', fatal=True)
         formats.append({
             'url': 'rtmp://%s/ondemand?ovpfv=1.1' % akamai_url,
diff --git a/youtube_dl/extractor/gdcvault.py b/youtube_dl/extractor/gdcvault.py
index 01e1ceec8..5d45faf85 100644
--- a/youtube_dl/extractor/gdcvault.py
+++ b/youtube_dl/extractor/gdcvault.py
@@ -52,6 +52,7 @@ class GDCVaultIE(InfoExtractor):
             'only_matching': True,
         },
         {
+            # Hard-coded hostname
             'url': 'http://gdcvault.com/play/1023460/Tenacious-Design-and-The-Interface',
             'md5': 'a8efb6c31ed06ca8739294960b2dbabd',
             'info_dict': {
@@ -61,6 +62,19 @@ class GDCVaultIE(InfoExtractor):
                 'title': 'Tenacious Design and The Interface of \'Destiny\'',
             },
         },
+        {
+            # Multiple audios
+            'url': 'http://www.gdcvault.com/play/1014631/Classic-Game-Postmortem-PAC',
+            'info_dict': {
+                'id': '1014631',
+                'ext': 'flv',
+                'title': 'How to Create a Good Game - From My Experience of Designing Pac-Man',
+            },
+            'params': {
+                'skip_download': True,  # Requires rtmpdump
+                'format': 'jp',  # The japanese audio
+            }
+        },
     ]
 
     def _login(self, webpage_url, display_id):

From 1da19488f906841cbfa285ad5ea378003c68ddf2 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 21 Apr 2016 23:29:51 +0800
Subject: [PATCH 4684/6219] [mgtv] Add new extractor (closes #9212)

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/mgtv.py       | 62 ++++++++++++++++++++++++++++++
 2 files changed, 63 insertions(+)
 create mode 100644 youtube_dl/extractor/mgtv.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 84bdf5e97..d28f9e863 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -407,6 +407,7 @@ from .mdr import MDRIE
 from .metacafe import MetacafeIE
 from .metacritic import MetacriticIE
 from .mgoon import MgoonIE
+from .mgtv import MGTVIE
 from .minhateca import MinhatecaIE
 from .ministrygrid import MinistryGridIE
 from .minoto import MinotoIE
diff --git a/youtube_dl/extractor/mgtv.py b/youtube_dl/extractor/mgtv.py
new file mode 100644
index 000000000..8f16a8f4c
--- /dev/null
+++ b/youtube_dl/extractor/mgtv.py
@@ -0,0 +1,62 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import int_or_none
+
+
+class MGTVIE(InfoExtractor):
+    _VALID_URL = r'https?://www\.mgtv\.com/v/(?:[^/]+/)*(?P<id>\d+).html'
+
+    _TEST = {
+        'url': 'http://www.mgtv.com/v/1/290525/f/3116640.html',
+        'md5': '',
+        'info_dict': {
+            'id': '3116640',
+            'ext': 'mp4',
+            'title': '我是歌手第四季双年巅峰会：韩红李玟“双王”领军对抗',
+            'description': '我是歌手第四季双年巅峰会',
+            'duration': 7461,
+            'thumbnail': 're:^https?://.*\.jpg$',
+        },
+        'params': {
+            'skip_download': True,  # m3u8 download
+        },
+    }
+
+    _FORMAT_MAP = {
+        '标清': ('Standard', 0),
+        '高清': ('High', 1),
+        '超清': ('SuperHigh', 2),
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        api_data = self._download_json(
+            'http://v.api.mgtv.com/player/video', video_id,
+            query={'video_id': video_id})['data']
+        info = api_data['info']
+
+        formats = []
+        for idx, stream in enumerate(api_data['stream']):
+            format_name = stream.get('name')
+            format_id, preference = self._FORMAT_MAP.get(format_name, (None, None))
+            format_info = self._download_json(
+                stream['url'], video_id,
+                note='Download video info for format %s' % format_id or '#%d' % idx)
+            formats.append({
+                'format_id': format_id,
+                'url': format_info['info'],
+                'ext': 'mp4',  # These are m3u8 playlists
+                'preference': preference,
+            })
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': info['title'].strip(),
+            'formats': formats,
+            'description': info.get('desc'),
+            'duration': int_or_none(info.get('duration')),
+            'thumbnail': info.get('thumb'),
+        }

From acaff49575f9f0c2ea893c4dbbbf57ba8baf656a Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 7 Apr 2016 19:30:47 +0100
Subject: [PATCH 4685/6219] [utils] imporove parse_duration to handle more
 formats

---
 test/test_utils.py  |  1 +
 youtube_dl/utils.py | 74 +++++++++++++++++++++++----------------------
 2 files changed, 39 insertions(+), 36 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index a35debfe1..e0323a5c6 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -405,6 +405,7 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(parse_duration('01:02:03:04'), 93784)
         self.assertEqual(parse_duration('1 hour 3 minutes'), 3780)
         self.assertEqual(parse_duration('87 Min.'), 5220)
+        self.assertEqual(parse_duration('PT1H0.040S'), 3600.04)
 
     def test_fix_xml_ampersands(self):
         self.assertEqual(
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 8e53962c9..c91aa0682 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1540,44 +1540,46 @@ def parse_duration(s):
 
     s = s.strip()
 
-    m = re.match(
-        r'''(?ix)(?:P?T)?
-        (?:
-            (?P<only_mins>[0-9.]+)\s*(?:mins?\.?|minutes?)\s*|
-            (?P<only_hours>[0-9.]+)\s*(?:hours?)|
-
-            \s*(?P<hours_reversed>[0-9]+)\s*(?:[:h]|hours?)\s*(?P<mins_reversed>[0-9]+)\s*(?:[:m]|mins?\.?|minutes?)\s*|
-            (?:
+    days, hours, mins, secs, ms = [None] * 5
+    m = re.match(r'(?:(?:(?:(?P<days>[0-9]+):)?(?P<hours>[0-9]+):)?(?P<mins>[0-9]+):)?(?P<secs>[0-9]+)(?P<ms>\.[0-9]+)?$', s)
+    if m:
+        days, hours, mins, secs, ms = m.groups()
+    else:
+        m = re.match(
+            r'''(?ix)(?:P?T)?
                 (?:
-                    (?:(?P<days>[0-9]+)\s*(?:[:d]|days?)\s*)?
-                    (?P<hours>[0-9]+)\s*(?:[:h]|hours?)\s*
+                    (?P<days>[0-9]+)\s*d(?:ays?)?\s*
                 )?
-                (?P<mins>[0-9]+)\s*(?:[:m]|mins?|minutes?)\s*
-            )?
-            (?P<secs>[0-9]+)(?P<ms>\.[0-9]+)?\s*(?:s|secs?|seconds?)?
-        )$''', s)
-    if not m:
-        return None
-    res = 0
-    if m.group('only_mins'):
-        return float_or_none(m.group('only_mins'), invscale=60)
-    if m.group('only_hours'):
-        return float_or_none(m.group('only_hours'), invscale=60 * 60)
-    if m.group('secs'):
-        res += int(m.group('secs'))
-    if m.group('mins_reversed'):
-        res += int(m.group('mins_reversed')) * 60
-    if m.group('mins'):
-        res += int(m.group('mins')) * 60
-    if m.group('hours'):
-        res += int(m.group('hours')) * 60 * 60
-    if m.group('hours_reversed'):
-        res += int(m.group('hours_reversed')) * 60 * 60
-    if m.group('days'):
-        res += int(m.group('days')) * 24 * 60 * 60
-    if m.group('ms'):
-        res += float(m.group('ms'))
-    return res
+                (?:
+                    (?P<hours>[0-9]+)\s*h(?:ours?)?\s*
+                )?
+                (?:
+                    (?P<mins>[0-9]+)\s*m(?:in(?:ute)?s?)?\s*
+                )?
+                (?:
+                    (?P<secs>[0-9]+)(?P<ms>\.[0-9]+)?\s*s(?:ec(?:ond)?s?)?\s*
+                )?$''', s)
+        if m:
+            days, hours, mins, secs, ms = m.groups()
+        else:
+            m = re.match(r'(?i)(?:(?P<hours>[0-9.]+)\s*(?:hours?)|(?P<mins>[0-9.]+)\s*(?:mins?\.?|minutes?)\s*)$', s)
+            if m:
+                hours, mins = m.groups()
+            else:
+                return None
+
+    duration = 0
+    if secs:
+        duration += float(secs)
+    if mins:
+        duration += float(mins) * 60
+    if hours:
+        duration += float(hours) * 60 * 60
+    if days:
+        duration += float(days) * 24 * 60 * 60
+    if ms:
+        duration += float(ms)
+    return duration
 
 
 def prepend_extension(filename, ext, expected_real_ext=None):

From 748e7300991ba9ea5b9ede7134110be135743b66 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 21 Apr 2016 23:58:36 +0800
Subject: [PATCH 4686/6219] [dispeak] Several fixes

---
 youtube_dl/extractor/dispeak.py     | 4 ++--
 youtube_dl/extractor/gdcvault.py    | 2 +-
 youtube_dl/extractor/gputechconf.py | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/dispeak.py b/youtube_dl/extractor/dispeak.py
index c74fa7e07..aaa2f8cd1 100644
--- a/youtube_dl/extractor/dispeak.py
+++ b/youtube_dl/extractor/dispeak.py
@@ -13,10 +13,10 @@ from ..utils import (
 
 
 class DigitalSpeakingIE(InfoExtractor):
-    _VALID_URL = r'http://(?:evt\.dispeak|events\.digitallyspeaking)\.com/([^/]+/)+xml/(?P<id>[^.]+).xml'
+    _VALID_URL = r'https?://(?:evt\.dispeak|events\.digitallyspeaking)\.com/(?:[^/]+/)+xml/(?P<id>[^.]+)\.xml'
 
     _TESTS = [{
-        # From http://evt.dispeak.com/ubm/gdc/sf16/xml/840376_BQRC.xml
+        # From http://gdcvault.com/play/1023460/Tenacious-Design-and-The-Interface
         'url': 'http://evt.dispeak.com/ubm/gdc/sf16/xml/840376_BQRC.xml',
         'md5': 'a8efb6c31ed06ca8739294960b2dbabd',
         'info_dict': {
diff --git a/youtube_dl/extractor/gdcvault.py b/youtube_dl/extractor/gdcvault.py
index 5d45faf85..373053a66 100644
--- a/youtube_dl/extractor/gdcvault.py
+++ b/youtube_dl/extractor/gdcvault.py
@@ -157,5 +157,5 @@ class GDCVaultIE(InfoExtractor):
             'id': video_id,
             'display_id': display_id,
             'url': '%s/xml/%s' % (xml_root, xml_name),
-            'ie': 'DigitalSpeaking',
+            'ie_key': 'DigitalSpeaking',
         }
diff --git a/youtube_dl/extractor/gputechconf.py b/youtube_dl/extractor/gputechconf.py
index 359b348e4..14daef7a8 100644
--- a/youtube_dl/extractor/gputechconf.py
+++ b/youtube_dl/extractor/gputechconf.py
@@ -31,5 +31,5 @@ class GPUTechConfIE(InfoExtractor):
             '_type': 'url_transparent',
             'id': video_id,
             'url': '%sxml/%s.xml' % (root_path, xml_file_id),
-            'ie': 'DigitalSpeaking',
+            'ie_key': 'DigitalSpeaking',
         }

From 982e518a96be4659ae621b5bcafe3bbb1b9ca6fc Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 22 Apr 2016 00:02:58 +0800
Subject: [PATCH 4687/6219] [dispeak] Rename DigitalSpeaking to
 DigitallySpeaking

---
 youtube_dl/extractor/dispeak.py     | 2 +-
 youtube_dl/extractor/extractors.py  | 2 +-
 youtube_dl/extractor/gdcvault.py    | 2 +-
 youtube_dl/extractor/gputechconf.py | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/dispeak.py b/youtube_dl/extractor/dispeak.py
index aaa2f8cd1..a78cb8a2a 100644
--- a/youtube_dl/extractor/dispeak.py
+++ b/youtube_dl/extractor/dispeak.py
@@ -12,7 +12,7 @@ from ..utils import (
 )
 
 
-class DigitalSpeakingIE(InfoExtractor):
+class DigitallySpeakingIE(InfoExtractor):
     _VALID_URL = r'https?://(?:evt\.dispeak|events\.digitallyspeaking)\.com/(?:[^/]+/)+xml/(?P<id>[^.]+)\.xml'
 
     _TESTS = [{
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index d28f9e863..92fbdbb64 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -197,7 +197,7 @@ from .dump import DumpIE
 from .dumpert import DumpertIE
 from .defense import DefenseGouvFrIE
 from .discovery import DiscoveryIE
-from .dispeak import DigitalSpeakingIE
+from .dispeak import DigitallySpeakingIE
 from .dropbox import DropboxIE
 from .dw import (
     DWIE,
diff --git a/youtube_dl/extractor/gdcvault.py b/youtube_dl/extractor/gdcvault.py
index 373053a66..3136427db 100644
--- a/youtube_dl/extractor/gdcvault.py
+++ b/youtube_dl/extractor/gdcvault.py
@@ -157,5 +157,5 @@ class GDCVaultIE(InfoExtractor):
             'id': video_id,
             'display_id': display_id,
             'url': '%s/xml/%s' % (xml_root, xml_name),
-            'ie_key': 'DigitalSpeaking',
+            'ie_key': 'DigitallySpeaking',
         }
diff --git a/youtube_dl/extractor/gputechconf.py b/youtube_dl/extractor/gputechconf.py
index 14daef7a8..73dc62c49 100644
--- a/youtube_dl/extractor/gputechconf.py
+++ b/youtube_dl/extractor/gputechconf.py
@@ -31,5 +31,5 @@ class GPUTechConfIE(InfoExtractor):
             '_type': 'url_transparent',
             'id': video_id,
             'url': '%sxml/%s.xml' % (root_path, xml_file_id),
-            'ie_key': 'DigitalSpeaking',
+            'ie_key': 'DigitallySpeaking',
         }

From a292cba256c1adca354d4ed59ef30a62aeb8606e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 22 Apr 2016 00:05:48 +0800
Subject: [PATCH 4688/6219] [mgtv] Fix _VALID_URL and add localized name

---
 youtube_dl/extractor/mgtv.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mgtv.py b/youtube_dl/extractor/mgtv.py
index 8f16a8f4c..a14d176a5 100644
--- a/youtube_dl/extractor/mgtv.py
+++ b/youtube_dl/extractor/mgtv.py
@@ -6,7 +6,8 @@ from ..utils import int_or_none
 
 
 class MGTVIE(InfoExtractor):
-    _VALID_URL = r'https?://www\.mgtv\.com/v/(?:[^/]+/)*(?P<id>\d+).html'
+    _VALID_URL = r'https?://www\.mgtv\.com/v/(?:[^/]+/)*(?P<id>\d+)\.html'
+    IE_DESC = '芒果TV'
 
     _TEST = {
         'url': 'http://www.mgtv.com/v/1/290525/f/3116640.html',

From 1988647dda86546de87c3f146ba68a17a0a5e247 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 21 Apr 2016 22:15:20 +0600
Subject: [PATCH 4689/6219] [tvigle] Skip hls completely (#9259)

---
 youtube_dl/extractor/tvigle.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/tvigle.py b/youtube_dl/extractor/tvigle.py
index f3aba3530..ead4c00c7 100644
--- a/youtube_dl/extractor/tvigle.py
+++ b/youtube_dl/extractor/tvigle.py
@@ -83,10 +83,10 @@ class TvigleIE(InfoExtractor):
 
         formats = []
         for vcodec, fmts in item['videos'].items():
+            if vcodec == 'hls':
+                continue
             for format_id, video_url in fmts.items():
                 if format_id == 'm3u8':
-                    formats.extend(self._extract_m3u8_formats(
-                        video_url, video_id, 'mp4', m3u8_id=vcodec))
                     continue
                 height = self._search_regex(
                     r'^(\d+)[pP]$', format_id, 'height', default=None)

From 65771128902a4a43a9340fba337aa323a3cf2db0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 21 Apr 2016 22:33:54 +0600
Subject: [PATCH 4690/6219] [planetaplay] Remove extractor (Closes #9256)

---
 youtube_dl/extractor/planetaplay.py | 61 -----------------------------
 1 file changed, 61 deletions(-)
 delete mode 100644 youtube_dl/extractor/planetaplay.py

diff --git a/youtube_dl/extractor/planetaplay.py b/youtube_dl/extractor/planetaplay.py
deleted file mode 100644
index 06505e96f..000000000
--- a/youtube_dl/extractor/planetaplay.py
+++ /dev/null
@@ -1,61 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-import re
-
-from .common import InfoExtractor
-from ..utils import ExtractorError
-
-
-class PlanetaPlayIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?planetaplay\.com/\?sng=(?P<id>[0-9]+)'
-    _API_URL = 'http://planetaplay.com/action/playlist/?sng={0:}'
-    _THUMBNAIL_URL = 'http://planetaplay.com/img/thumb/{thumb:}'
-    _TEST = {
-        'url': 'http://planetaplay.com/?sng=3586',
-        'md5': '9d569dceb7251a4e01355d5aea60f9db',
-        'info_dict': {
-            'id': '3586',
-            'ext': 'flv',
-            'title': 'md5:e829428ee28b1deed00de90de49d1da1',
-        },
-        'skip': 'Not accessible from Travis CI server',
-    }
-
-    _SONG_FORMATS = {
-        'lq': (0, 'http://www.planetaplay.com/videoplayback/{med_hash:}'),
-        'hq': (1, 'http://www.planetaplay.com/videoplayback/hi/{med_hash:}'),
-    }
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
-        response = self._download_json(
-            self._API_URL.format(video_id), video_id)['response']
-        try:
-            data = response.get('data')[0]
-        except IndexError:
-            raise ExtractorError(
-                '%s: failed to get the playlist' % self.IE_NAME, expected=True)
-
-        title = '{song_artists:} - {sng_name:}'.format(**data)
-        thumbnail = self._THUMBNAIL_URL.format(**data)
-
-        formats = []
-        for format_id, (quality, url_template) in self._SONG_FORMATS.items():
-            formats.append({
-                'format_id': format_id,
-                'url': url_template.format(**data),
-                'quality': quality,
-                'ext': 'flv',
-            })
-
-        self._sort_formats(formats)
-
-        return {
-            'id': video_id,
-            'title': title,
-            'formats': formats,
-            'thumbnail': thumbnail,
-        }

From 870d525848d2535be23ea04fcd3fbf826d03cae1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 21 Apr 2016 22:44:01 +0600
Subject: [PATCH 4691/6219] [options] Remove experimental mark for
 --hls-prefer-native

---
 youtube_dl/options.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 7819f14ab..b60a723c5 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -426,7 +426,7 @@ def parseOpts(overrideArguments=None):
     downloader.add_option(
         '--hls-prefer-native',
         dest='hls_prefer_native', action='store_true',
-        help='Use the native HLS downloader instead of ffmpeg (experimental)')
+        help='Use the native HLS downloader instead of ffmpeg')
     downloader.add_option(
         '--hls-use-mpegts',
         dest='hls_use_mpegts', action='store_true',

From 88296ac326694b8da0f6815efbab988ff6134405 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 21 Apr 2016 22:57:38 +0600
Subject: [PATCH 4692/6219] [planetaplay] Remove remainings of extractor

---
 youtube_dl/extractor/extractors.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 92fbdbb64..4f4e3f5cb 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -567,7 +567,6 @@ from .philharmoniedeparis import PhilharmonieDeParisIE
 from .phoenix import PhoenixIE
 from .photobucket import PhotobucketIE
 from .pinkbike import PinkbikeIE
-from .planetaplay import PlanetaPlayIE
 from .pladform import PladformIE
 from .played import PlayedIE
 from .playfm import PlayFMIE

From bf09af3acbafd37e8327b8bab118066bd4c23d31 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 21 Apr 2016 23:02:17 +0600
Subject: [PATCH 4693/6219] Add --hls-prefer-ffmpeg

---
 youtube_dl/YoutubeDL.py           | 4 +++-
 youtube_dl/downloader/__init__.py | 5 ++++-
 youtube_dl/options.py             | 6 +++++-
 3 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index a89a71a25..055433362 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -260,7 +260,9 @@ class YoutubeDL(object):
     The following options determine which downloader is picked:
     external_downloader: Executable of the external downloader to call.
                        None or unset for standard (built-in) downloader.
-    hls_prefer_native: Use the native HLS downloader instead of ffmpeg/avconv.
+    hls_prefer_native: Use the native HLS downloader instead of ffmpeg/avconv
+                       if True, otherwise use ffmpeg/avconv if False, otherwise
+                       use downloader suggested by extractor if None.
 
     The following parameters are not used by YoutubeDL itself, they are used by
     the downloader (see youtube_dl/downloader/common.py):
diff --git a/youtube_dl/downloader/__init__.py b/youtube_dl/downloader/__init__.py
index 73b34fdae..817591d97 100644
--- a/youtube_dl/downloader/__init__.py
+++ b/youtube_dl/downloader/__init__.py
@@ -41,9 +41,12 @@ def get_suitable_downloader(info_dict, params={}):
         if ed.can_download(info_dict):
             return ed
 
-    if protocol == 'm3u8' and params.get('hls_prefer_native'):
+    if protocol == 'm3u8' and params.get('hls_prefer_native') is True:
         return HlsFD
 
+    if protocol == 'm3u8_native' and params.get('hls_prefer_native') is False:
+        return FFmpegFD
+
     return PROTOCOL_MAP.get(protocol, HttpFD)
 
 
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index b60a723c5..d1f8d1331 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -425,8 +425,12 @@ def parseOpts(overrideArguments=None):
         help='Set file xattribute ytdl.filesize with expected filesize (experimental)')
     downloader.add_option(
         '--hls-prefer-native',
-        dest='hls_prefer_native', action='store_true',
+        dest='hls_prefer_native', action='store_true', default=None,
         help='Use the native HLS downloader instead of ffmpeg')
+    downloader.add_option(
+        '--hls-prefer-ffmpeg',
+        dest='hls_prefer_native', action='store_false', default=None,
+        help='Use ffmpeg instead of the native HLS downloader')
     downloader.add_option(
         '--hls-use-mpegts',
         dest='hls_use_mpegts', action='store_true',

From 09a9fadb843d55f70724382c6ad6c5a387f52bc3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 21 Apr 2016 23:31:34 +0600
Subject: [PATCH 4694/6219] [dump] Remove extractor

---
 youtube_dl/extractor/dump.py       | 39 ------------------------------
 youtube_dl/extractor/extractors.py |  1 -
 2 files changed, 40 deletions(-)
 delete mode 100644 youtube_dl/extractor/dump.py

diff --git a/youtube_dl/extractor/dump.py b/youtube_dl/extractor/dump.py
deleted file mode 100644
index ff78d4fd2..000000000
--- a/youtube_dl/extractor/dump.py
+++ /dev/null
@@ -1,39 +0,0 @@
-# encoding: utf-8
-from __future__ import unicode_literals
-
-import re
-
-from .common import InfoExtractor
-
-
-class DumpIE(InfoExtractor):
-    _VALID_URL = r'^https?://(?:www\.)?dump\.com/(?P<id>[a-zA-Z0-9]+)/'
-
-    _TEST = {
-        'url': 'http://www.dump.com/oneus/',
-        'md5': 'ad71704d1e67dfd9e81e3e8b42d69d99',
-        'info_dict': {
-            'id': 'oneus',
-            'ext': 'flv',
-            'title': "He's one of us.",
-            'thumbnail': 're:^https?://.*\.jpg$',
-        },
-    }
-
-    def _real_extract(self, url):
-        m = re.match(self._VALID_URL, url)
-        video_id = m.group('id')
-
-        webpage = self._download_webpage(url, video_id)
-        video_url = self._search_regex(
-            r's1.addVariable\("file",\s*"([^"]+)"', webpage, 'video URL')
-
-        title = self._og_search_title(webpage)
-        thumbnail = self._og_search_thumbnail(webpage)
-
-        return {
-            'id': video_id,
-            'title': title,
-            'url': video_url,
-            'thumbnail': thumbnail,
-        }
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 4f4e3f5cb..27a7e89a4 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -193,7 +193,6 @@ from .drbonanza import DRBonanzaIE
 from .drtuber import DrTuberIE
 from .drtv import DRTVIE
 from .dvtv import DVTVIE
-from .dump import DumpIE
 from .dumpert import DumpertIE
 from .defense import DefenseGouvFrIE
 from .discovery import DiscoveryIE

From bdbb8530c7103a47be19b2dd2dfad8bfe9f8fd67 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 22 Apr 2016 00:02:39 +0600
Subject: [PATCH 4695/6219] [vimeo] Pass Referer for check-password request

---
 youtube_dl/extractor/vimeo.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index a4833a858..59f9cb1ae 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -282,10 +282,10 @@ class VimeoIE(VimeoBaseInfoExtractor):
         pass_url = url + '/check-password'
         password_request = sanitized_Request(pass_url, data)
         password_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
+        password_request.add_header('Referer', url)
         return self._download_json(
             password_request, video_id,
-            'Verifying the password',
-            'Wrong password')
+            'Verifying the password', 'Wrong password')
 
     def _real_initialize(self):
         self._login()

From 9260cf1d97ef86bd864793041f05e3ad4ba1e72f Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 21 Apr 2016 20:29:42 +0100
Subject: [PATCH 4696/6219] [tubitv] fix extraction(closes #8741)

---
 youtube_dl/extractor/tubitv.py | 53 +++++++++++++++++++---------------
 1 file changed, 29 insertions(+), 24 deletions(-)

diff --git a/youtube_dl/extractor/tubitv.py b/youtube_dl/extractor/tubitv.py
index 7af233cd6..c6572defb 100644
--- a/youtube_dl/extractor/tubitv.py
+++ b/youtube_dl/extractor/tubitv.py
@@ -1,7 +1,6 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import codecs
 import re
 
 from .common import InfoExtractor
@@ -10,22 +9,24 @@ from ..utils import (
     int_or_none,
     sanitized_Request,
     urlencode_postdata,
+    parse_iso8601,
 )
 
 
 class TubiTvIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?tubitv\.com/video\?id=(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?tubitv\.com/video/(?P<id>[0-9]+)'
     _LOGIN_URL = 'http://tubitv.com/login'
     _NETRC_MACHINE = 'tubitv'
     _TEST = {
-        'url': 'http://tubitv.com/video?id=54411&title=The_Kitchen_Musical_-_EP01',
+        'url': 'http://tubitv.com/video/283829/the_comedian_at_the_friday',
         'info_dict': {
-            'id': '54411',
+            'id': '283829',
             'ext': 'mp4',
-            'title': 'The Kitchen Musical - EP01',
-            'thumbnail': 're:^https?://.*\.png$',
-            'description': 'md5:37532716166069b353e8866e71fefae7',
-            'duration': 2407,
+            'title': 'The Comedian at The Friday',
+            'description': 'A stand up comedian is forced to look at the decisions in his life while on a one week trip to the west coast.',
+            'uploader': 'Indie Rights Films',
+            'upload_date': '20160111',
+            'timestamp': 1452555979,
         },
         'params': {
             'skip_download': 'HLS download',
@@ -55,27 +56,31 @@ class TubiTvIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
+        video_data = self._download_json(
+            'http://tubitv.com/oz/videos/%s/content' % video_id, video_id)
+        title = video_data['n']
 
-        webpage = self._download_webpage(url, video_id)
-        if re.search(r"<(?:DIV|div) class='login-required-screen'>", webpage):
-            self.raise_login_required('This video requires login')
-
-        title = self._og_search_title(webpage)
-        description = self._og_search_description(webpage)
-        thumbnail = self._og_search_thumbnail(webpage)
-        duration = int_or_none(self._html_search_meta(
-            'video:duration', webpage, 'duration'))
-
-        apu = self._search_regex(r"apu='([^']+)'", webpage, 'apu')
-        m3u8_url = codecs.decode(apu, 'rot_13')[::-1]
-        formats = self._extract_m3u8_formats(m3u8_url, video_id, ext='mp4')
+        formats = self._extract_m3u8_formats(
+            video_data['mh'], video_id, 'mp4', 'm3u8_native')
         self._sort_formats(formats)
 
+        subtitles = {}
+        for sub in video_data.get('sb', []):
+            sub_url = sub.get('u')
+            if not sub_url:
+                continue
+            subtitles.setdefault(sub.get('l', 'en'), []).append({
+                'url': sub_url,
+            })
+
         return {
             'id': video_id,
             'title': title,
             'formats': formats,
-            'thumbnail': thumbnail,
-            'description': description,
-            'duration': duration,
+            'subtitles': subtitles,
+            'thumbnail': video_data.get('ph'),
+            'description': video_data.get('d'),
+            'duration': int_or_none(video_data.get('s')),
+            'timestamp': parse_iso8601(video_data.get('u')),
+            'uploader': video_data.get('on'),
         }

From 7c36ea7d540a8a34c6868493928c3bac4987a540 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 21 Apr 2016 22:52:49 +0100
Subject: [PATCH 4697/6219] [rtbf] improve extraction(fixes #9267)

---
 youtube_dl/extractor/rtbf.py | 62 ++++++++++++++++++++++++++----------
 1 file changed, 45 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/rtbf.py b/youtube_dl/extractor/rtbf.py
index e42b319a3..28cc5522d 100644
--- a/youtube_dl/extractor/rtbf.py
+++ b/youtube_dl/extractor/rtbf.py
@@ -4,12 +4,18 @@ from __future__ import unicode_literals
 from .common import InfoExtractor
 from ..utils import (
     int_or_none,
-    unescapeHTML,
+    ExtractorError,
 )
 
 
 class RTBFIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?rtbf\.be/(?:video/[^?]+\?.*\bid=|ouftivi/(?:[^/]+/)*[^?]+\?.*\bvideoId=)(?P<id>\d+)'
+    _VALID_URL = r'''(?x)
+        https?://(?:www\.)?rtbf\.be/
+        (?:
+            video/[^?]+\?.*\bid=|
+            ouftivi/(?:[^/]+/)*[^?]+\?.*\bvideoId=|
+            auvio/[^/]+\?.*id=
+        )(?P<id>\d+)'''
     _TESTS = [{
         'url': 'https://www.rtbf.be/video/detail_les-diables-au-coeur-episode-2?id=1921274',
         'md5': '799f334ddf2c0a582ba80c44655be570',
@@ -17,7 +23,11 @@ class RTBFIE(InfoExtractor):
             'id': '1921274',
             'ext': 'mp4',
             'title': 'Les Diables au coeur (épisode 2)',
+            'description': 'Football - Diables Rouges',
             'duration': 3099,
+            'upload_date': '20140425',
+            'timestamp': 1398456336,
+            'uploader': 'rtbfsport',
         }
     }, {
         # geo restricted
@@ -26,45 +36,63 @@ class RTBFIE(InfoExtractor):
     }, {
         'url': 'http://www.rtbf.be/ouftivi/niouzz?videoId=2055858',
         'only_matching': True,
+    }, {
+        'url': 'http://www.rtbf.be/auvio/detail_jeudi-en-prime-siegfried-bracke?id=2102996',
+        'only_matching': True,
     }]
-
+    _IMAGE_HOST = 'http://ds1.ds.static.rtbf.be'
+    _PROVIDERS = {
+        'YOUTUBE': 'Youtube',
+        'DAILYMOTION': 'Dailymotion',
+        'VIMEO': 'Vimeo',
+    }
     _QUALITIES = [
-        ('mobile', 'mobile'),
-        ('web', 'SD'),
-        ('url', 'MD'),
+        ('mobile', 'SD'),
+        ('web', 'MD'),
         ('high', 'HD'),
     ]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
+        data = self._download_json(
+            'http://www.rtbf.be/api/media/video?method=getVideoDetail&args[]=%s' % video_id, video_id)
 
-        webpage = self._download_webpage(
-            'http://www.rtbf.be/video/embed?id=%s' % video_id, video_id)
+        error = data.get('error')
+        if error:
+            raise ExtractorError('%s said: %s' % (self.IE_NAME, error), expected=True)
 
-        data = self._parse_json(
-            unescapeHTML(self._search_regex(
-                r'data-media="([^"]+)"', webpage, 'data video')),
-            video_id)
+        data = data['data']
+
+        provider = data.get('provider')
+        if provider in self._PROVIDERS:
+            return self.url_result(data['url'], self._PROVIDERS[provider])
 
-        if data.get('provider').lower() == 'youtube':
-            video_url = data.get('downloadUrl') or data.get('url')
-            return self.url_result(video_url, 'Youtube')
         formats = []
         for key, format_id in self._QUALITIES:
-            format_url = data['sources'].get(key)
+            format_url = data.get(key + 'Url')
             if format_url:
                 formats.append({
                     'format_id': format_id,
                     'url': format_url,
                 })
 
+        thumbnails = []
+        for thumbnail_id, thumbnail_url in data.get('thumbnail', {}).items():
+            if thumbnail_id != 'default':
+                thumbnails.append({
+                    'url': self._IMAGE_HOST + thumbnail_url,
+                    'id': thumbnail_id,
+                })
+
         return {
             'id': video_id,
             'formats': formats,
             'title': data['title'],
             'description': data.get('description') or data.get('subtitle'),
-            'thumbnail': data.get('thumbnail'),
+            'thumbnails': thumbnails,
             'duration': data.get('duration') or data.get('realDuration'),
             'timestamp': int_or_none(data.get('created')),
             'view_count': int_or_none(data.get('viewCount')),
+            'uploader': data.get('channel'),
+            'tags': data.get('tags'),
         }

From e962ae15d39d9c351735663573fcd1964730faa1 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 22 Apr 2016 11:26:43 +0100
Subject: [PATCH 4698/6219] [newstube] extract http formats(closes #9253)

---
 youtube_dl/extractor/newstube.py | 43 +++++++++++++++++++++++++-------
 1 file changed, 34 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/newstube.py b/youtube_dl/extractor/newstube.py
index 5a9e73cd6..0092b85ce 100644
--- a/youtube_dl/extractor/newstube.py
+++ b/youtube_dl/extractor/newstube.py
@@ -4,24 +4,24 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import ExtractorError
+from ..utils import (
+    ExtractorError,
+    int_or_none,
+)
 
 
 class NewstubeIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?newstube\.ru/media/(?P<id>.+)'
     _TEST = {
         'url': 'http://www.newstube.ru/media/telekanal-cnn-peremestil-gorod-slavyansk-v-krym',
+        'md5': '801eef0c2a9f4089fa04e4fe3533abdc',
         'info_dict': {
             'id': '728e0ef2-e187-4012-bac0-5a081fdcb1f6',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Телеканал CNN переместил город Славянск в Крым',
             'description': 'md5:419a8c9f03442bc0b0a794d689360335',
             'duration': 31.05,
         },
-        'params': {
-            # rtmp download
-            'skip_download': True,
-        },
     }
 
     def _real_extract(self, url):
@@ -62,7 +62,6 @@ class NewstubeIE(InfoExtractor):
             server = media_location.find(ns('./Server')).text
             app = media_location.find(ns('./App')).text
             media_id = stream_info.find(ns('./Id')).text
-            quality_id = stream_info.find(ns('./QualityId')).text
             name = stream_info.find(ns('./Name')).text
             width = int(stream_info.find(ns('./Width')).text)
             height = int(stream_info.find(ns('./Height')).text)
@@ -74,12 +73,38 @@ class NewstubeIE(InfoExtractor):
                 'rtmp_conn': ['S:%s' % session_id, 'S:%s' % media_id, 'S:n2'],
                 'page_url': url,
                 'ext': 'flv',
-                'format_id': quality_id,
-                'format_note': name,
+                'format_id': 'rtmp' + ('-%s' % name if name else ''),
                 'width': width,
                 'height': height,
             })
 
+        sources_data = self._download_json(
+            'http://www.newstube.ru/player2/getsources?guid=%s' % video_guid,
+            video_guid, fatal=False)
+        if sources_data:
+            for source in sources_data.get('Sources', []):
+                source_url = source.get('Src')
+                if not source_url:
+                    continue
+                height = int_or_none(source.get('Height'))
+                f = {
+                    'format_id': 'http' + ('-%dp' % height if height else ''),
+                    'url': source_url,
+                    'width': int_or_none(source.get('Width')),
+                    'height': height,
+                }
+                source_type = source.get('Type')
+                if source_type:
+                    mobj = re.search(r'codecs="([^,]+),\s*([^"]+)"', source_type)
+                    if mobj:
+                        vcodec, acodec = mobj.groups()
+                        f.update({
+                            'vcodec': vcodec,
+                            'acodec': acodec,
+                        })
+                formats.append(f)
+
+        self._check_formats(formats, video_guid)
         self._sort_formats(formats)
 
         return {

From 237a41108ae2be9eb76364bb92471489b2eedb7b Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 22 Apr 2016 14:32:38 +0100
Subject: [PATCH 4699/6219] [eagleplatform] extract all http formats

---
 youtube_dl/extractor/eagleplatform.py | 22 ++++++++++++++++++----
 youtube_dl/extractor/gazeta.py        |  5 +----
 2 files changed, 19 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/eagleplatform.py b/youtube_dl/extractor/eagleplatform.py
index fa3cb7023..0f8c73fd7 100644
--- a/youtube_dl/extractor/eagleplatform.py
+++ b/youtube_dl/extractor/eagleplatform.py
@@ -8,6 +8,7 @@ from ..compat import compat_HTTPError
 from ..utils import (
     ExtractorError,
     int_or_none,
+    url_basename,
 )
 
 
@@ -22,7 +23,7 @@ class EaglePlatformIE(InfoExtractor):
     _TESTS = [{
         # http://lenta.ru/news/2015/03/06/navalny/
         'url': 'http://lentaru.media.eagleplatform.com/index/player?player=new&record_id=227304&player_template_id=5201',
-        'md5': '70f5187fb620f2c1d503b3b22fd4efe3',
+        'md5': '881ee8460e1b7735a8be938e2ffb362b',
         'info_dict': {
             'id': '227304',
             'ext': 'mp4',
@@ -37,7 +38,7 @@ class EaglePlatformIE(InfoExtractor):
         # http://muz-tv.ru/play/7129/
         # http://media.clipyou.ru/index/player?record_id=12820&width=730&height=415&autoplay=true
         'url': 'eagleplatform:media.clipyou.ru:12820',
-        'md5': '90b26344ba442c8e44aa4cf8f301164a',
+        'md5': '358597369cf8ba56675c1df15e7af624',
         'info_dict': {
             'id': '12820',
             'ext': 'mp4',
@@ -90,17 +91,30 @@ class EaglePlatformIE(InfoExtractor):
 
         secure_m3u8 = self._proto_relative_url(media['sources']['secure_m3u8']['auto'], 'http:')
 
+        formats = []
+
         m3u8_url = self._get_video_url(secure_m3u8, video_id, 'Downloading m3u8 JSON')
-        formats = self._extract_m3u8_formats(
+        m3u8_formats = self._extract_m3u8_formats(
             m3u8_url, video_id,
             'mp4', entry_protocol='m3u8_native', m3u8_id='hls')
+        formats.extend(m3u8_formats)
 
         mp4_url = self._get_video_url(
             # Secure mp4 URL is constructed according to Player.prototype.mp4 from
             # http://lentaru.media.eagleplatform.com/player/player.js
             re.sub(r'm3u8|hlsvod|hls|f4m', 'mp4', secure_m3u8),
             video_id, 'Downloading mp4 JSON')
-        formats.append({'url': mp4_url, 'format_id': 'mp4'})
+        mp4_url_basename = url_basename(mp4_url)
+        for m3u8_format in m3u8_formats:
+            mobj = re.search('/([^/]+)/index\.m3u8', m3u8_format['url'])
+            if mobj:
+                http_format = m3u8_format.copy()
+                http_format.update({
+                    'url': mp4_url.replace(mp4_url_basename, mobj.group(1)),
+                    'format_id': m3u8_format['format_id'].replace('hls', 'http'),
+                    'protocol': 'http',
+                })
+                formats.append(http_format)
 
         self._sort_formats(formats)
 
diff --git a/youtube_dl/extractor/gazeta.py b/youtube_dl/extractor/gazeta.py
index ba1c15414..18ef5c252 100644
--- a/youtube_dl/extractor/gazeta.py
+++ b/youtube_dl/extractor/gazeta.py
@@ -24,15 +24,12 @@ class GazetaIE(InfoExtractor):
         'only_matching': True,
     }, {
         'url': 'http://www.gazeta.ru/video/main/main/2015/06/22/platit_ili_ne_platit_po_isku_yukosa.shtml',
+        'md5': '37f19f78355eb2f4256ee1688359f24c',
         'info_dict': {
             'id': '252048',
             'ext': 'mp4',
             'title': '"Если по иску ЮКОСа придется платить, это будет большой удар по бюджету"',
         },
-        'params': {
-            # m3u8 download
-            'skip_download': True,
-        },
         'add_ie': ['EaglePlatform'],
     }]
 

From c80037918b86cf79c1542bb7bd7dda94d81c3efb Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 23 Apr 2016 00:06:49 +0800
Subject: [PATCH 4700/6219] [iqiyi] Improve error detection (#9276)

---
 youtube_dl/extractor/iqiyi.py | 25 ++++++++++++++++++-------
 1 file changed, 18 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index ea8fbb329..ffb8008ce 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -287,6 +287,13 @@ class IqiyiIE(InfoExtractor):
         ('10', 'h1'),
     ]
 
+    AUTH_API_ERRORS = {
+        # No preview available (不允许试看鉴权失败)
+        'Q00505': 'This video requires a VIP account',
+        # End of preview time (试看结束鉴权失败)
+        'Q00506': 'Needs a VIP account for full video',
+    }
+
     def _real_initialize(self):
         self._login()
 
@@ -372,14 +379,18 @@ class IqiyiIE(InfoExtractor):
             note='Downloading video authentication JSON',
             errnote='Unable to download video authentication JSON')
 
-        if auth_result['code'] == 'Q00505':  # No preview available (不允许试看鉴权失败)
-            raise ExtractorError('This video requires a VIP account', expected=True)
-        if auth_result['code'] == 'Q00506':  # End of preview time (试看结束鉴权失败)
+        code = auth_result.get('code')
+        msg = self.AUTH_API_ERRORS.get(code) or auth_result.get('msg') or code
+        if code == 'Q00506':
             if do_report_warning:
-                self.report_warning('Needs a VIP account for full video')
+                self.report_warning(msg)
             return False
+        if 'data' not in auth_result:
+            if msg is not None:
+                raise ExtractorError('%s said: %s' % (self.IE_NAME, msg), expected=True)
+            raise ExtractorError('Unexpected error from Iqiyi auth API')
 
-        return auth_result
+        return auth_result['data']
 
     def construct_video_urls(self, data, video_id, _uuid, tvid):
         def do_xor(x, y):
@@ -455,11 +466,11 @@ class IqiyiIE(InfoExtractor):
                         need_vip_warning_report = False
                         break
                     param.update({
-                        't': auth_result['data']['t'],
+                        't': auth_result['t'],
                         # cid is hard-coded in com/qiyi/player/core/player/RuntimeData.as
                         'cid': 'afbe8fd3d73448c9',
                         'vid': video_id,
-                        'QY00001': auth_result['data']['u'],
+                        'QY00001': auth_result['u'],
                     })
                 api_video_url += '?' if '?' not in api_video_url else '&'
                 api_video_url += compat_urllib_parse_urlencode(param)

From cb7d4d0efd8c58485e5269895e128b649d8c6c0c Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 22 Apr 2016 18:07:40 +0100
Subject: [PATCH 4701/6219] [nbc] add support for today.com(closes #2909)

---
 youtube_dl/extractor/nbc.py | 24 +++++++++++++++++++-----
 1 file changed, 19 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index e67025ff6..f9d42d07a 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -172,7 +172,7 @@ class CSNNEIE(InfoExtractor):
 
 
 class NBCNewsIE(ThePlatformIE):
-    _VALID_URL = r'''(?x)https?://(?:www\.)?nbcnews\.com/
+    _VALID_URL = r'''(?x)https?://(?:www\.)?(?:nbcnews|today)\.com/
         (?:video/.+?/(?P<id>\d+)|
         ([^/]+/)*(?P<display_id>[^/?]+))
         '''
@@ -230,6 +230,18 @@ class NBCNewsIE(ThePlatformIE):
             },
             'expected_warnings': ['http-6000 is not available']
         },
+        {
+            'url': 'http://www.today.com/video/see-the-aurora-borealis-from-space-in-stunning-new-nasa-video-669831235788',
+            'md5': '118d7ca3f0bea6534f119c68ef539f71',
+            'info_dict': {
+                'id': '669831235788',
+                'ext': 'mp4',
+                'title': 'See the aurora borealis from space in stunning new NASA video',
+                'description': 'md5:74752b7358afb99939c5f8bb2d1d04b1',
+                'upload_date': '20160420',
+                'timestamp': 1461152093,
+            },
+        },
         {
             'url': 'http://www.nbcnews.com/watch/dateline/full-episode--deadly-betrayal-386250819952',
             'only_matching': True,
@@ -264,7 +276,10 @@ class NBCNewsIE(ThePlatformIE):
                 info = bootstrap['results'][0]['video']
             else:
                 player_instance_json = self._search_regex(
-                    r'videoObj\s*:\s*({.+})', webpage, 'player instance')
+                    r'videoObj\s*:\s*({.+})', webpage, 'player instance', default=None)
+                if not player_instance_json:
+                    player_instance_json = self._html_search_regex(
+                        r'data-video="([^"]+)"', webpage, 'video json')
                 info = self._parse_json(player_instance_json, display_id)
             video_id = info['mpxId']
             title = info['title']
@@ -295,7 +310,7 @@ class NBCNewsIE(ThePlatformIE):
                     formats.extend(tp_formats)
                     subtitles = self._merge_subtitles(subtitles, tp_subtitles)
                 else:
-                    tbr = int_or_none(video_asset.get('bitRate'), 1000)
+                    tbr = int_or_none(video_asset.get('bitRate') or video_asset.get('bitrate'), 1000)
                     format_id = 'http%s' % ('-%d' % tbr if tbr else '')
                     video_url = update_url_query(
                         video_url, {'format': 'redirect'})
@@ -321,10 +336,9 @@ class NBCNewsIE(ThePlatformIE):
                 'id': video_id,
                 'title': title,
                 'description': info.get('description'),
-                'thumbnail': info.get('description'),
                 'thumbnail': info.get('thumbnail'),
                 'duration': int_or_none(info.get('duration')),
-                'timestamp': parse_iso8601(info.get('pubDate')),
+                'timestamp': parse_iso8601(info.get('pubDate') or info.get('pub_date')),
                 'formats': formats,
                 'subtitles': subtitles,
             }

From eb01e97e10a5c1eb096695fbfd91ff7f69a4aa9e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 23 Apr 2016 02:51:17 +0800
Subject: [PATCH 4702/6219] [youku] Skip streams with channel_type=tail

Fixes #9275

These video segments look like ads and they don't appear in the web
player.
---
 youtube_dl/extractor/youku.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index fd7eb5a6d..349ce0941 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -64,6 +64,14 @@ class YoukuIE(InfoExtractor):
         'params': {
             'videopassword': '100600',
         },
+    }, {
+        # /play/get.json contains streams with "channel_type":"tail"
+        'url': 'http://v.youku.com/v_show/id_XOTUxMzg4NDMy.html',
+        'info_dict': {
+            'id': 'XOTUxMzg4NDMy',
+            'title': '我的世界☆明月庄主☆车震猎杀☆杀人艺术Minecraft',
+        },
+        'playlist_count': 6,
     }]
 
     def construct_video_urls(self, data):
@@ -92,6 +100,8 @@ class YoukuIE(InfoExtractor):
 
         fileid_dict = {}
         for stream in data['stream']:
+            if stream.get('channel_type') == 'tail':
+                continue
             format = stream.get('stream_type')
             fileid = stream['stream_fileid']
             fileid_dict[format] = fileid
@@ -117,6 +127,8 @@ class YoukuIE(InfoExtractor):
         # generate video_urls
         video_urls_dict = {}
         for stream in data['stream']:
+            if stream.get('channel_type') == 'tail':
+                continue
             format = stream.get('stream_type')
             video_urls = []
             for dt in stream['segs']:
@@ -253,6 +265,8 @@ class YoukuIE(InfoExtractor):
             # which one has all
         } for i in range(max(len(v.get('segs')) for v in data['stream']))]
         for stream in data['stream']:
+            if stream.get('channel_type') == 'tail':
+                continue
             fm = stream.get('stream_type')
             video_urls = video_urls_dict[fm]
             for video_url, seg, entry in zip(video_urls, stream['segs'], entries):

From 29a7e8f6f88e9a4c522e67aae5badd26ee226fde Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 22 Apr 2016 20:17:20 +0100
Subject: [PATCH 4703/6219] [nhl] Add new extractor(closes #8419)(closes #8798)

---
 youtube_dl/extractor/extractors.py |  5 +-
 youtube_dl/extractor/nhl.py        | 98 ++++++++++++++++++++++++++++--
 2 files changed, 96 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 27a7e89a4..6de3438fc 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -490,9 +490,10 @@ from .nextmovie import NextMovieIE
 from .nfb import NFBIE
 from .nfl import NFLIE
 from .nhl import (
-    NHLIE,
-    NHLNewsIE,
     NHLVideocenterIE,
+    NHLNewsIE,
+    NHLVideocenterCategoryIE,
+    NHLIE,
 )
 from .nick import NickIE
 from .niconico import NiconicoIE, NiconicoPlaylistIE
diff --git a/youtube_dl/extractor/nhl.py b/youtube_dl/extractor/nhl.py
index c1dea8b6c..b04d21113 100644
--- a/youtube_dl/extractor/nhl.py
+++ b/youtube_dl/extractor/nhl.py
@@ -8,10 +8,15 @@ from .common import InfoExtractor
 from ..compat import (
     compat_urlparse,
     compat_urllib_parse_urlencode,
-    compat_urllib_parse_urlparse
+    compat_urllib_parse_urlparse,
+    compat_str,
 )
 from ..utils import (
     unified_strdate,
+    determine_ext,
+    int_or_none,
+    parse_iso8601,
+    parse_duration,
 )
 
 
@@ -70,8 +75,8 @@ class NHLBaseInfoExtractor(InfoExtractor):
         return ret
 
 
-class NHLIE(NHLBaseInfoExtractor):
-    IE_NAME = 'nhl.com'
+class NHLVideocenterIE(NHLBaseInfoExtractor):
+    IE_NAME = 'nhl.com:videocenter'
     _VALID_URL = r'https?://video(?P<team>\.[^.]*)?\.nhl\.com/videocenter/(?:console|embed)?(?:\?(?:.*?[?&])?)(?:id|hlg|playlist)=(?P<id>[-0-9a-zA-Z,]+)'
 
     _TESTS = [{
@@ -186,8 +191,8 @@ class NHLNewsIE(NHLBaseInfoExtractor):
         return self._real_extract_video(video_id)
 
 
-class NHLVideocenterIE(NHLBaseInfoExtractor):
-    IE_NAME = 'nhl.com:videocenter'
+class NHLVideocenterCategoryIE(NHLBaseInfoExtractor):
+    IE_NAME = 'nhl.com:videocenter:category'
     IE_DESC = 'NHL videocenter category'
     _VALID_URL = r'https?://video\.(?P<team>[^.]*)\.nhl\.com/videocenter/(console\?[^(id=)]*catid=(?P<catid>[0-9]+)(?![&?]id=).*?)?$'
     _TEST = {
@@ -236,3 +241,86 @@ class NHLVideocenterIE(NHLBaseInfoExtractor):
             'id': cat_id,
             'entries': [self._extract_video(v) for v in videos],
         }
+
+
+class NHLIE(InfoExtractor):
+    IE_NAME = 'nhl.com'
+    _VALID_URL = r'https?://(?:www\.)?nhl\.com/([^/]+/)*c-(?P<id>\d+)'
+    _TESTS = [{
+        # type=video
+        'url': 'https://www.nhl.com/video/anisimov-cleans-up-mess/t-277752844/c-43663503',
+        'md5': '0f7b9a8f986fb4b4eeeece9a56416eaf',
+        'info_dict': {
+            'id': '43663503',
+            'ext': 'mp4',
+            'title': 'Anisimov cleans up mess',
+            'description': 'md5:a02354acdfe900e940ce40706939ca63',
+            'timestamp': 1461288600,
+            'upload_date': '20160422',
+        },
+    }, {
+        # type=article
+        'url': 'https://www.nhl.com/news/dennis-wideman-suspended/c-278258934',
+        'md5': '1f39f4ea74c1394dea110699a25b366c',
+        'info_dict': {
+            'id': '40784403',
+            'ext': 'mp4',
+            'title': 'Wideman suspended by NHL',
+            'description': 'Flames defenseman Dennis Wideman was banned 20 games for violation of Rule 40 (Physical Abuse of Officials)',
+            'upload_date': '20160204',
+            'timestamp': 1454544904,
+        },
+    }]
+
+    def _real_extract(self, url):
+        tmp_id = self._match_id(url)
+        video_data = self._download_json(
+            'https://nhl.bamcontent.com/nhl/id/v1/%s/details/web-v1.json' % tmp_id,
+            tmp_id)
+        if video_data.get('type') == 'article':
+            video_data = video_data['media']
+
+        video_id = compat_str(video_data['id'])
+        title = video_data['title']
+
+        formats = []
+        for playback in video_data.get('playbacks', []):
+            playback_url = playback.get('url')
+            if not playback_url:
+                continue
+            ext = determine_ext(playback_url)
+            if ext == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    playback_url, video_id, 'mp4', 'm3u8_native',
+                    m3u8_id=playback.get('name', 'hls'), fatal=False))
+            else:
+                height = int_or_none(playback.get('height'))
+                formats.append({
+                    'format_id': playback.get('name', 'http' + ('-%dp' % height if height else '')),
+                    'url': playback_url,
+                    'width': int_or_none(playback.get('width')),
+                    'height': height,
+                })
+        self._sort_formats(formats, ('preference', 'width', 'height', 'tbr', 'format_id'))
+
+        thumbnails = []
+        for thumbnail_id, thumbnail_data in video_data.get('image', {}).get('cuts', {}).items():
+            thumbnail_url = thumbnail_data.get('src')
+            if not thumbnail_url:
+                continue
+            thumbnails.append({
+                'id': thumbnail_id,
+                'url': thumbnail_url,
+                'width': int_or_none(thumbnail_data.get('width')),
+                'height': int_or_none(thumbnail_data.get('height')),
+            })
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': video_data.get('description'),
+            'timestamp': parse_iso8601(video_data.get('date')),
+            'duration': parse_duration(video_data.get('duration')),
+            'thumbnails': thumbnails,
+            'formats': formats,
+        }

From c0837a12c8a64c682a01e4bfdee6f22615568d69 Mon Sep 17 00:00:00 2001
From: Boris Wachtmeister <boris-code@gmx.com>
Date: Sat, 12 Mar 2016 18:00:26 +0100
Subject: [PATCH 4704/6219] [WDR] complete overhaul after relaunch of the site

The WDR relaunched their site on 2016-02-23 which not only changed the
URL-schema completely but also the layout of their pages.

Apparently the whole "mediathek" now runs on the wdr-domain, so no
separate URL for funkhauseuropa anymore.
There seems to be no explicit handling of video-sizes on the page or in
the URLs anymore. There seems to be only one size for HTML5, but still
several sizes for flash. The extractor adds all to the list of formats.

There is no metadata for the HTML5-stream, so that the best flash-stream
will always be considered as the "best" format. At least in my tests
this seemed to be true anyway.
---
 youtube_dl/extractor/wdr.py | 251 +++++++++++++++---------------------
 1 file changed, 101 insertions(+), 150 deletions(-)

diff --git a/youtube_dl/extractor/wdr.py b/youtube_dl/extractor/wdr.py
index 31c904303..f881b7300 100644
--- a/youtube_dl/extractor/wdr.py
+++ b/youtube_dl/extractor/wdr.py
@@ -1,7 +1,6 @@
 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
-import itertools
 import re
 
 from .common import InfoExtractor
@@ -11,204 +10,156 @@ from ..compat import (
 )
 from ..utils import (
     unified_strdate,
-    qualities,
+    ExtractorError,
 )
 
 
 class WDRIE(InfoExtractor):
-    _PLAYER_REGEX = '-(?:video|audio)player(?:_size-[LMS])?'
-    _VALID_URL = r'(?P<url>https?://www\d?\.(?:wdr\d?|funkhauseuropa)\.de/)(?P<id>.+?)(?P<player>%s)?\.html' % _PLAYER_REGEX
+    _PAGE_REGEX = r'/mediathek/(?P<media_type>[^/]+)/(?P<type>[^/]+)/(?P<display_id>.+)\.html'
+    _VALID_URL = r'(?P<page_url>https?://(?:www\d\.)?wdr\d?\.de)' + _PAGE_REGEX
+
+    _JS_URL_REGEX = r'(https?://deviceids-medp.wdr.de/ondemand/\d+/\d+\.js)'
 
     _TESTS = [
         {
-            'url': 'http://www1.wdr.de/mediathek/video/sendungen/servicezeit/videoservicezeit560-videoplayer_size-L.html',
+            'url': 'http://www1.wdr.de/mediathek/video/sendungen/doku-am-freitag/video-geheimnis-aachener-dom-100.html',
+            'md5': 'e58c39c3e30077141d258bf588700a7b',
             'info_dict': {
-                'id': 'mdb-362427',
+                'id': 'mdb-1058683',
                 'ext': 'flv',
-                'title': 'Servicezeit',
-                'description': 'md5:c8f43e5e815eeb54d0b96df2fba906cb',
-                'upload_date': '20140310',
-                'is_live': False
-            },
-            'params': {
-                'skip_download': True,
+                'display_id': 'doku-am-freitag/video-geheimnis-aachener-dom-100',
+                'title': 'Geheimnis Aachener Dom',
+                'alt_title': 'Doku am Freitag',
+                'upload_date': '20160304',
+                'description': 'md5:87be8ff14d8dfd7a7ee46f0299b52318',
+                'is_live': False,
+                'subtitles': {'de': [{
+                    'url': 'http://ondemand-ww.wdr.de/medp/fsk0/105/1058683/1058683_12220974.xml'
+                }]},
             },
             'skip': 'Page Not Found',
         },
         {
-            'url': 'http://www1.wdr.de/themen/av/videomargaspiegelisttot101-videoplayer.html',
+            'url': 'http://www1.wdr.de/mediathek/audio/wdr3/wdr3-gespraech-am-samstag/audio-schriftstellerin-juli-zeh-100.html',
+            'md5': 'f4c1f96d01cf285240f53ea4309663d8',
             'info_dict': {
-                'id': 'mdb-363194',
+                'id': 'mdb-1072000',
+                'ext': 'mp3',
+                'display_id': 'wdr3-gespraech-am-samstag/audio-schriftstellerin-juli-zeh-100',
+                'title': 'Schriftstellerin Juli Zeh',
+                'alt_title': 'WDR 3 Gespräch am Samstag',
+                'upload_date': '20160312',
+                'description': 'md5:e127d320bc2b1f149be697ce044a3dd7',
+                'is_live': False,
+                'subtitles': {}
+            },
+            'skip': 'Page Not Found',
+        },
+        {
+            'url': 'http://www1.wdr.de/mediathek/video/live/index.html',
+            'info_dict': {
+                'id': 'mdb-103364',
                 'ext': 'flv',
-                'title': 'Marga Spiegel ist tot',
-                'description': 'md5:2309992a6716c347891c045be50992e4',
-                'upload_date': '20140311',
-                'is_live': False
-            },
-            'params': {
-                'skip_download': True,
-            },
-            'skip': 'Page Not Found',
-        },
-        {
-            'url': 'http://www1.wdr.de/themen/kultur/audioerlebtegeschichtenmargaspiegel100-audioplayer.html',
-            'md5': '83e9e8fefad36f357278759870805898',
-            'info_dict': {
-                'id': 'mdb-194332',
-                'ext': 'mp3',
-                'title': 'Erlebte Geschichten: Marga Spiegel (29.11.2009)',
-                'description': 'md5:2309992a6716c347891c045be50992e4',
-                'upload_date': '20091129',
-                'is_live': False
-            },
-        },
-        {
-            'url': 'http://www.funkhauseuropa.de/av/audioflaviacoelhoamaramar100-audioplayer.html',
-            'md5': '99a1443ff29af19f6c52cf6f4dc1f4aa',
-            'info_dict': {
-                'id': 'mdb-478135',
-                'ext': 'mp3',
-                'title': 'Flavia Coelho: Amar é Amar',
-                'description': 'md5:7b29e97e10dfb6e265238b32fa35b23a',
-                'upload_date': '20140717',
-                'is_live': False
-            },
-            'skip': 'Page Not Found',
-        },
-        {
-            'url': 'http://www1.wdr.de/mediathek/video/sendungen/quarks_und_co/filterseite-quarks-und-co100.html',
-            'playlist_mincount': 146,
-            'info_dict': {
-                'id': 'mediathek/video/sendungen/quarks_und_co/filterseite-quarks-und-co100',
+                'display_id': 'index',
+                'title': r're:^WDR Fernsehen im Livestream [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
+                'alt_title': 'WDR Fernsehen Live',
+                'upload_date': None,
+                'description': 'md5:ae2ff888510623bf8d4b115f95a9b7c9',
+                'is_live': True,
+                'subtitles': {}
             }
         },
         {
-            'url': 'http://www1.wdr.de/mediathek/video/livestream/index.html',
+            'url': 'http://www1.wdr.de/mediathek/video/sendungen/aktuelle-stunde/aktuelle-stunde-120.html',
+            'playlist_mincount': 10,
             'info_dict': {
-                'id': 'mdb-103364',
-                'title': 're:^WDR Fernsehen Live [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
-                'description': 'md5:ae2ff888510623bf8d4b115f95a9b7c9',
-                'ext': 'flv',
-                'upload_date': '20150101',
-                'is_live': True
-            },
-            'params': {
-                'skip_download': True,
+                'id': 'aktuelle-stunde/aktuelle-stunde-120',
             },
         }
     ]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        page_url = mobj.group('url')
-        page_id = mobj.group('id')
+        url_type = mobj.group('type')
+        page_url = mobj.group('page_url')
+        display_id = mobj.group('display_id')
+        webpage = self._download_webpage(url, display_id)
 
-        webpage = self._download_webpage(url, page_id)
+        js_url = self._search_regex(self._JS_URL_REGEX, webpage, 'js_url', default=None)
 
-        if mobj.group('player') is None:
+        if not js_url:
             entries = [
-                self.url_result(page_url + href, 'WDR')
+                self.url_result(page_url + href[0], 'WDR')
                 for href in re.findall(
-                    r'<a href="/?(.+?%s\.html)" rel="nofollow"' % self._PLAYER_REGEX,
+                    r'<a href="(%s)"' % self._PAGE_REGEX,
                     webpage)
             ]
 
             if entries:  # Playlist page
-                return self.playlist_result(entries, page_id)
+                return self.playlist_result(entries, playlist_id=display_id)
 
-            # Overview page
-            entries = []
-            for page_num in itertools.count(2):
-                hrefs = re.findall(
-                    r'<li class="mediathekvideo"\s*>\s*<img[^>]*>\s*<a href="(/mediathek/video/[^"]+)"',
-                    webpage)
-                entries.extend(
-                    self.url_result(page_url + href, 'WDR')
-                    for href in hrefs)
-                next_url_m = re.search(
-                    r'<li class="nextToLast">\s*<a href="([^"]+)"', webpage)
-                if not next_url_m:
-                    break
-                next_url = page_url + next_url_m.group(1)
-                webpage = self._download_webpage(
-                    next_url, page_id,
-                    note='Downloading playlist page %d' % page_num)
-            return self.playlist_result(entries, page_id)
+            raise ExtractorError('No downloadable streams found', expected=True)
 
-        flashvars = compat_parse_qs(self._html_search_regex(
-            r'<param name="flashvars" value="([^"]+)"', webpage, 'flashvars'))
+        js_data = self._download_webpage(js_url, 'metadata')
+        json_data = self._search_regex(r'\(({.*})\)', js_data, 'json')
+        metadata = self._parse_json(json_data, display_id)
 
-        page_id = flashvars['trackerClipId'][0]
-        video_url = flashvars['dslSrc'][0]
-        title = flashvars['trackerClipTitle'][0]
-        thumbnail = flashvars['startPicture'][0] if 'startPicture' in flashvars else None
-        is_live = flashvars.get('isLive', ['0'])[0] == '1'
+        metadata_tracker_data = metadata["trackerData"]
+        metadata_media_resource = metadata["mediaResource"]
+
+        formats = []
+
+        # check if the metadata contains a direct URL to a file
+        metadata_media_alt = metadata_media_resource.get("alt")
+        if metadata_media_alt:
+            for tag_name in ["videoURL", 'audioURL']:
+                if tag_name in metadata_media_alt:
+                    formats.append({
+                        'url': metadata_media_alt[tag_name]
+                    })
+
+        # check if there are flash-streams for this video
+        if "dflt" in metadata_media_resource and "videoURL" in metadata_media_resource["dflt"]:
+            video_url = metadata_media_resource["dflt"]["videoURL"]
+            if video_url.endswith('.f4m'):
+                full_video_url = video_url + '?hdcore=3.2.0&plugin=aasp-3.2.0.77.18'
+                formats.extend(self._extract_f4m_formats(full_video_url, display_id, f4m_id='hds', fatal=False))
+            elif video_url.endswith('.smil'):
+                formats.extend(self._extract_smil_formats(video_url, 'stream', fatal=False))
+
+        subtitles = {}
+        caption_url = metadata_media_resource.get("captionURL")
+        if caption_url:
+            subtitles['de'] = [{
+                'url': caption_url
+            }]
+
+        title = metadata_tracker_data.get("trackerClipTitle")
+        is_live = url_type == 'live'
 
         if is_live:
             title = self._live_title(title)
-
-        if 'trackerClipAirTime' in flashvars:
-            upload_date = flashvars['trackerClipAirTime'][0]
+            upload_date = None
+        elif 'trackerClipAirTime' in metadata_tracker_data:
+            upload_date = metadata_tracker_data['trackerClipAirTime']
         else:
-            upload_date = self._html_search_meta(
-                'DC.Date', webpage, 'upload date')
+            upload_date = self._html_search_meta('DC.Date', webpage, 'upload date')
 
         if upload_date:
             upload_date = unified_strdate(upload_date)
 
-        formats = []
-        preference = qualities(['S', 'M', 'L', 'XL'])
-
-        if video_url.endswith('.f4m'):
-            formats.extend(self._extract_f4m_formats(
-                video_url + '?hdcore=3.2.0&plugin=aasp-3.2.0.77.18', page_id,
-                f4m_id='hds', fatal=False))
-        elif video_url.endswith('.smil'):
-            formats.extend(self._extract_smil_formats(
-                video_url, page_id, False, {
-                    'hdcore': '3.3.0',
-                    'plugin': 'aasp-3.3.0.99.43',
-                }))
-        else:
-            formats.append({
-                'url': video_url,
-                'http_headers': {
-                    'User-Agent': 'mobile',
-                },
-            })
-
-        m3u8_url = self._search_regex(
-            r'rel="adaptiv"[^>]+href="([^"]+)"',
-            webpage, 'm3u8 url', default=None)
-        if m3u8_url:
-            formats.extend(self._extract_m3u8_formats(
-                m3u8_url, page_id, 'mp4', 'm3u8_native',
-                m3u8_id='hls', fatal=False))
-
-        direct_urls = re.findall(
-            r'rel="web(S|M|L|XL)"[^>]+href="([^"]+)"', webpage)
-        if direct_urls:
-            for quality, video_url in direct_urls:
-                formats.append({
-                    'url': video_url,
-                    'preference': preference(quality),
-                    'http_headers': {
-                        'User-Agent': 'mobile',
-                    },
-                })
-
         self._sort_formats(formats)
 
-        description = self._html_search_meta('Description', webpage, 'description')
-
         return {
-            'id': page_id,
-            'formats': formats,
+            'id': metadata_tracker_data.get("trackerClipId", display_id),
+            'display_id': display_id,
             'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
+            'alt_title': metadata_tracker_data.get("trackerClipSubcategory"),
+            'formats': formats,
             'upload_date': upload_date,
-            'is_live': is_live
+            'description': self._html_search_meta("Description", webpage),
+            'is_live': is_live,
+            'subtitles': subtitles,
         }
 
 

From 14f7a2b8af17d1f490c46a0a9028ba9d97cf7df2 Mon Sep 17 00:00:00 2001
From: Boris Wachtmeister <boris-code@gmx.com>
Date: Sat, 12 Mar 2016 20:14:46 +0100
Subject: [PATCH 4705/6219] [WDRMaus] switch current show to new WDR extractor
 (fixes #8562)

It seems that the "current show" already uses the new WDR video-player,
while all the others videos still use the old player.

I just added the current show URL to the normal WDR-extractor, which
works fine. This commit needs my changes from PR #8842 that fix the
support for WDR.
---
 youtube_dl/extractor/wdr.py | 37 ++++++++++++++++++++-----------------
 1 file changed, 20 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/wdr.py b/youtube_dl/extractor/wdr.py
index f881b7300..ec81f1a28 100644
--- a/youtube_dl/extractor/wdr.py
+++ b/youtube_dl/extractor/wdr.py
@@ -15,8 +15,9 @@ from ..utils import (
 
 
 class WDRIE(InfoExtractor):
+    _CURRENT_MAUS_URL = r'https?://www.wdrmaus.de/aktuelle-sendung/(wdr|index).php5'
     _PAGE_REGEX = r'/mediathek/(?P<media_type>[^/]+)/(?P<type>[^/]+)/(?P<display_id>.+)\.html'
-    _VALID_URL = r'(?P<page_url>https?://(?:www\d\.)?wdr\d?\.de)' + _PAGE_REGEX
+    _VALID_URL = r'(?P<page_url>https?://(?:www\d\.)?wdr\d?\.de)' + _PAGE_REGEX + "|" + _CURRENT_MAUS_URL
 
     _JS_URL_REGEX = r'(https?://deviceids-medp.wdr.de/ondemand/\d+/\d+\.js)'
 
@@ -75,7 +76,18 @@ class WDRIE(InfoExtractor):
             'info_dict': {
                 'id': 'aktuelle-stunde/aktuelle-stunde-120',
             },
-        }
+        },
+        {
+            'url': 'http://www.wdrmaus.de/aktuelle-sendung/index.php5',
+            'info_dict': {
+                'id': 'mdb-1096487',
+                'ext': 'flv',
+                'upload_date': 're:^[0-9]{8}$',
+                'title': 're:^Die Sendung mit der Maus vom [0-9.]{10}$',
+                'description': '- Die Sendung mit der Maus -',
+            },
+            'skip': 'The id changes from week to week because of the new episode'
+        },
     ]
 
     def _real_extract(self, url):
@@ -195,26 +207,17 @@ class WDRMobileIE(InfoExtractor):
 
 
 class WDRMausIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?wdrmaus\.de/(?:[^/]+/){,2}(?P<id>[^/?#]+)(?:/index\.php5|(?<!index)\.php5|/(?:$|[?#]))'
+    _VALID_URL = 'https?://(?:www\.)?wdrmaus\.de/(?:[^/]+/){,2}(?P<id>[^/?#]+)((?<!index)\.php5|/(?:$|[?#]))'
     IE_DESC = 'Sendung mit der Maus'
     _TESTS = [{
-        'url': 'http://www.wdrmaus.de/aktuelle-sendung/index.php5',
+        'url': 'http://www.wdrmaus.de/sachgeschichten/sachgeschichten/achterbahn.php5',
+        'md5': '178b432d002162a14ccb3e0876741095',
         'info_dict': {
-            'id': 'aktuelle-sendung',
+            'id': 'achterbahn',
             'ext': 'mp4',
             'thumbnail': 're:^http://.+\.jpg',
-            'upload_date': 're:^[0-9]{8}$',
-            'title': 're:^[0-9.]{10} - Aktuelle Sendung$',
-        }
-    }, {
-        'url': 'http://www.wdrmaus.de/sachgeschichten/sachgeschichten/40_jahre_maus.php5',
-        'md5': '3b1227ca3ed28d73ec5737c65743b2a3',
-        'info_dict': {
-            'id': '40_jahre_maus',
-            'ext': 'mp4',
-            'thumbnail': 're:^http://.+\.jpg',
-            'upload_date': '20131007',
-            'title': '12.03.2011 - 40 Jahre Maus',
+            'upload_date': '20131001',
+            'title': '19.09.2013 - Achterbahn',
         }
     }]
 

From 397ec446f3816a0b13bb71068c10ab8122192cbd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 Apr 2016 15:59:30 +0600
Subject: [PATCH 4706/6219] [dplay] Try secure api for no tld (Closes #9282)

---
 youtube_dl/extractor/dplay.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/dplay.py b/youtube_dl/extractor/dplay.py
index 66bbfc6ca..ef68ef087 100644
--- a/youtube_dl/extractor/dplay.py
+++ b/youtube_dl/extractor/dplay.py
@@ -99,7 +99,7 @@ class DPlayIE(InfoExtractor):
                     video_id, f4m_id=protocol, fatal=False))
 
         domain_tld = domain.split('.')[-1]
-        if domain_tld in ('se', 'dk'):
+        if domain_tld in ('se', 'dk', 'no'):
             for protocol in PROTOCOLS:
                 self._set_cookie(
                     'secure.dplay.%s' % domain_tld, 'dsc-geo',

From fd0ff8bad85595fc7bb7fdafb09be63c502927ad Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 Apr 2016 16:36:17 +0600
Subject: [PATCH 4707/6219] [dplay] Improve extraction and document workarounds
 and tests

---
 youtube_dl/extractor/dplay.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/dplay.py b/youtube_dl/extractor/dplay.py
index ef68ef087..b2f654ca5 100644
--- a/youtube_dl/extractor/dplay.py
+++ b/youtube_dl/extractor/dplay.py
@@ -13,6 +13,7 @@ class DPlayIE(InfoExtractor):
     _VALID_URL = r'https?://(?P<domain>it\.dplay\.com|www\.dplay\.(?:dk|se|no))/[^/]+/(?P<id>[^/?#]+)'
 
     _TESTS = [{
+        # geo restricted, via direct unsigned hls URL
         'url': 'http://it.dplay.com/take-me-out/stagione-1-episodio-25/',
         'info_dict': {
             'id': '1255600',
@@ -31,6 +32,7 @@ class DPlayIE(InfoExtractor):
         },
         'expected_warnings': ['Unable to download f4m manifest'],
     }, {
+        # non geo restricted, via secure api
         'url': 'http://www.dplay.se/nugammalt-77-handelser-som-format-sverige/season-1-svensken-lar-sig-njuta-av-livet/',
         'info_dict': {
             'id': '3172',
@@ -48,6 +50,7 @@ class DPlayIE(InfoExtractor):
             'age_limit': 0,
         },
     }, {
+        # geo restricted, via secure api
         'url': 'http://www.dplay.dk/mig-og-min-mor/season-6-episode-12/',
         'info_dict': {
             'id': '70816',
@@ -65,6 +68,7 @@ class DPlayIE(InfoExtractor):
             'age_limit': 0,
         },
     }, {
+        # geo restricted, via direct unsigned hls URL
         'url': 'http://www.dplay.no/pga-tour/season-1-hoydepunkter-18-21-februar/',
         'only_matching': True,
     }]
@@ -101,6 +105,7 @@ class DPlayIE(InfoExtractor):
         domain_tld = domain.split('.')[-1]
         if domain_tld in ('se', 'dk', 'no'):
             for protocol in PROTOCOLS:
+                # Providing dsc-geo allows to bypass geo restriction in some cases
                 self._set_cookie(
                     'secure.dplay.%s' % domain_tld, 'dsc-geo',
                     json.dumps({
@@ -113,7 +118,11 @@ class DPlayIE(InfoExtractor):
                     'Downloading %s stream JSON' % protocol, fatal=False)
                 if stream and stream.get(protocol):
                     extract_formats(protocol, stream[protocol])
-        else:
+
+        # The last resort is to try direct unsigned hls/hds URLs from info dictionary.
+        # Sometimes this does work even when secure API with dsc-geo has failed (e.g.
+        # http://www.dplay.no/pga-tour/season-1-hoydepunkter-18-21-februar/).
+        if not formats:
             for protocol in PROTOCOLS:
                 if info.get(protocol):
                     extract_formats(protocol, info[protocol])

From e239413fbc9f37996a5fb1ed5b5d211f5a0e613b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 Apr 2016 16:50:31 +0600
Subject: [PATCH 4708/6219] [dplay] Extract subtitles (Closes #9284)

---
 youtube_dl/extractor/dplay.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/youtube_dl/extractor/dplay.py b/youtube_dl/extractor/dplay.py
index b2f654ca5..a7994697c 100644
--- a/youtube_dl/extractor/dplay.py
+++ b/youtube_dl/extractor/dplay.py
@@ -129,6 +129,13 @@ class DPlayIE(InfoExtractor):
 
         self._sort_formats(formats)
 
+        subtitles = {}
+        for lang in ('se', 'sv', 'da', 'nl', 'no'):
+            for format_id in ('web_vtt', 'vtt', 'srt'):
+                subtitle_url = info.get('subtitles_%s_%s' % (lang, format_id))
+                if subtitle_url:
+                    subtitles.setdefault(lang, []).append({'url': subtitle_url})
+
         return {
             'id': video_id,
             'display_id': display_id,
@@ -142,4 +149,5 @@ class DPlayIE(InfoExtractor):
             'episode_number': int_or_none(info.get('episode')),
             'age_limit': int_or_none(info.get('minimum_age')),
             'formats': formats,
+            'subtitles': subtitles,
         }

From 5448b781f697af2c0ea2efe8e553e2049d4de047 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 Apr 2016 17:28:45 +0600
Subject: [PATCH 4709/6219] [dplay] Sign unsigned final download hls URLs

---
 youtube_dl/extractor/dplay.py | 26 ++++++++++++++++++--------
 1 file changed, 18 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/dplay.py b/youtube_dl/extractor/dplay.py
index a7994697c..5790553f3 100644
--- a/youtube_dl/extractor/dplay.py
+++ b/youtube_dl/extractor/dplay.py
@@ -6,7 +6,11 @@ import re
 import time
 
 from .common import InfoExtractor
-from ..utils import int_or_none
+from ..compat import compat_urlparse
+from ..utils import (
+    int_or_none,
+    update_url_query,
+)
 
 
 class DPlayIE(InfoExtractor):
@@ -32,12 +36,12 @@ class DPlayIE(InfoExtractor):
         },
         'expected_warnings': ['Unable to download f4m manifest'],
     }, {
-        # non geo restricted, via secure api
+        # non geo restricted, via secure api, unsigned download hls URL
         'url': 'http://www.dplay.se/nugammalt-77-handelser-som-format-sverige/season-1-svensken-lar-sig-njuta-av-livet/',
         'info_dict': {
             'id': '3172',
             'display_id': 'season-1-svensken-lar-sig-njuta-av-livet',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Svensken lär sig njuta av livet',
             'description': 'md5:d3819c9bccffd0fe458ca42451dd50d8',
             'duration': 2650,
@@ -50,18 +54,18 @@ class DPlayIE(InfoExtractor):
             'age_limit': 0,
         },
     }, {
-        # geo restricted, via secure api
+        # geo restricted, via secure api, unsigned download hls URL
         'url': 'http://www.dplay.dk/mig-og-min-mor/season-6-episode-12/',
         'info_dict': {
             'id': '70816',
             'display_id': 'season-6-episode-12',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Episode 12',
             'description': 'md5:9c86e51a93f8a4401fc9641ef9894c90',
             'duration': 2563,
             'timestamp': 1429696800,
             'upload_date': '20150422',
-            'creator': 'Kanal 4',
+            'creator': 'Kanal 4 (Home)',
             'series': 'Mig og min mor',
             'season_number': 6,
             'episode_number': 12,
@@ -94,9 +98,15 @@ class DPlayIE(InfoExtractor):
 
         def extract_formats(protocol, manifest_url):
             if protocol == 'hls':
-                formats.extend(self._extract_m3u8_formats(
+                m3u8_formats = self._extract_m3u8_formats(
                     manifest_url, video_id, ext='mp4',
-                    entry_protocol='m3u8_native', m3u8_id=protocol, fatal=False))
+                    entry_protocol='m3u8_native', m3u8_id=protocol, fatal=False)
+                # Sometimes final URLs inside m3u8 are unsigned, let's fix this
+                # ourselves
+                query = compat_urlparse.parse_qs(compat_urlparse.urlparse(manifest_url).query)
+                for m3u8_format in m3u8_formats:
+                    m3u8_format['url'] = update_url_query(m3u8_format['url'], query)
+                formats.extend(m3u8_formats)
             elif protocol == 'hds':
                 formats.extend(self._extract_f4m_formats(
                     manifest_url + '&hdcore=3.8.0&plugin=flowplayer-3.8.0.0',

From ccff2c404d7ea9f5b21ede8ae57bb79feec7eb94 Mon Sep 17 00:00:00 2001
From: Wang Jun Tham <bouncerox@gmail.com>
Date: Sun, 24 Apr 2016 00:08:02 +0800
Subject: [PATCH 4710/6219] [ffmpeg] Fix embedding subtitles (#9063)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Changed command line parameters for ffmpeg when embedding subtitles.
Changed to ‘-map 0:v -c:v copy -map 0:a -c:a copy’
---
 youtube_dl/postprocessor/ffmpeg.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 1793a878c..ca2d401f8 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -363,8 +363,10 @@ class FFmpegEmbedSubtitlePP(FFmpegPostProcessor):
         input_files = [filename] + sub_filenames
 
         opts = [
-            '-map', '0',
-            '-c', 'copy',
+            '-map', '0:v',
+            '-c:v', 'copy',
+            '-map', '0:a',
+            '-c:a', 'copy',
             # Don't copy the existing subtitles, we may be running the
             # postprocessor a second time
             '-map', '-0:s',

From 864d5e7231c931d10d3a556224d152c8228270c3 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 24 Apr 2016 02:32:56 +0100
Subject: [PATCH 4711/6219] [viewster] extract all http formats

---
 youtube_dl/extractor/viewster.py | 36 +++++++++++++++++++++-----------
 1 file changed, 24 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/viewster.py b/youtube_dl/extractor/viewster.py
index fe94a4793..1fbf9d794 100644
--- a/youtube_dl/extractor/viewster.py
+++ b/youtube_dl/extractor/viewster.py
@@ -1,6 +1,8 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..compat import (
     compat_HTTPError,
@@ -14,6 +16,7 @@ from ..utils import (
     parse_iso8601,
     sanitized_Request,
     HEADRequest,
+    url_basename,
 )
 
 
@@ -114,6 +117,7 @@ class ViewsterIE(InfoExtractor):
             return self.playlist_result(entries, video_id, title, description)
 
         formats = []
+        manifest_url = None
         for media_type in ('application/f4m+xml', 'application/x-mpegURL', 'video/mp4'):
             media = self._download_json(
                 'https://public-api.viewster.com/movies/%s/video?mediaType=%s'
@@ -126,29 +130,37 @@ class ViewsterIE(InfoExtractor):
                 continue
             ext = determine_ext(video_url)
             if ext == 'f4m':
+                manifest_url = video_url
                 video_url += '&' if '?' in video_url else '?'
                 video_url += 'hdcore=3.2.0&plugin=flowplayer-3.2.0.1'
                 formats.extend(self._extract_f4m_formats(
                     video_url, video_id, f4m_id='hds'))
             elif ext == 'm3u8':
+                manifest_url = video_url
                 m3u8_formats = self._extract_m3u8_formats(
                     video_url, video_id, 'mp4', m3u8_id='hls',
                     fatal=False)  # m3u8 sometimes fail
                 if m3u8_formats:
                     formats.extend(m3u8_formats)
             else:
-                format_id = media.get('Bitrate')
-                f = {
-                    'url': video_url,
-                    'format_id': 'mp4-%s' % format_id,
-                    'height': int_or_none(media.get('Height')),
-                    'width': int_or_none(media.get('Width')),
-                    'preference': 1,
-                }
-                if format_id and not f['height']:
-                    f['height'] = int_or_none(self._search_regex(
-                        r'^(\d+)[pP]$', format_id, 'height', default=None))
-                formats.append(f)
+                qualities_basename = self._search_regex(
+                    '/([^/]+)(?:.csmil/manifest.f4m|.csmil/master.m3u8)',
+                    manifest_url, 'qualities basename', default=None)
+                if qualities_basename:
+                    QUALITIES_RE = r'((,\d+k)+,?)'
+                    qualities = self._search_regex(
+                        QUALITIES_RE, qualities_basename,
+                        'qualities').strip(',').split(',')
+                    http_template = re.sub(QUALITIES_RE, r'%s', qualities_basename)
+                    http_url_basename = url_basename(video_url)
+                    for q in qualities:
+                        formats.append({
+                            'url': video_url.replace(http_url_basename, http_template % q),
+                            'ext': 'mp4',
+                            'format_id': 'http-%s' % q,
+                            'tbr': int_or_none(self._search_regex(
+                                r'(\d+)k', q, 'bitrate', default=None)),
+                        })
 
         if not formats and not info.get('LanguageSets') and not info.get('VODSettings'):
             self.raise_geo_restricted()

From fda9a1ca9e469b97bb28edc00bc6b9974e1fab7a Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 24 Apr 2016 03:06:46 +0100
Subject: [PATCH 4712/6219] [viewster] simplify qualities_basename regex

---
 youtube_dl/extractor/viewster.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/viewster.py b/youtube_dl/extractor/viewster.py
index 1fbf9d794..4289a2f31 100644
--- a/youtube_dl/extractor/viewster.py
+++ b/youtube_dl/extractor/viewster.py
@@ -144,7 +144,7 @@ class ViewsterIE(InfoExtractor):
                     formats.extend(m3u8_formats)
             else:
                 qualities_basename = self._search_regex(
-                    '/([^/]+)(?:.csmil/manifest.f4m|.csmil/master.m3u8)',
+                    '/([^/]+)\.csmil/',
                     manifest_url, 'qualities basename', default=None)
                 if qualities_basename:
                     QUALITIES_RE = r'((,\d+k)+,?)'

From 2c21152ca7af4f96fccb225f161862046280a12a Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Sun, 24 Apr 2016 12:22:18 +0600
Subject: [PATCH 4713/6219] [README.md] Document track metafields in output
 template

---
 README.md | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/README.md b/README.md
index cd18edd87..7a54d651d 100644
--- a/README.md
+++ b/README.md
@@ -515,6 +515,18 @@ Available for the video that is an episode of some series or programme:
  - `episode_number`: Number of the video episode within a season
  - `episode_id`: Id of the video episode
 
+Available for the media that is a track or a part of a music album:
+ - `track`: Title of the track
+ - `track_number`: Number of the track within an album or a disc
+ - `track_id`: Id of the track
+ - `artist`: Artist(s) of the track
+ - `genre`: Genre(s) of the track
+ - `album`: Title of the album the track belongs to
+ - `album_type`: Type of the album
+ - `album_artist`: List of all artists appeared on the album
+ - `disc_number`: Number of the disc or other physical medium the track belongs to
+ - `release_year`: Year (YYYY) when the album was released
+
 Each aforementioned sequence when referenced in output template will be replaced by the actual value corresponding to the sequence name. Note that some of the sequences are not guaranteed to be present since they depend on the metadata obtained by particular extractor, such sequences will be replaced with `NA`.
 
 For example for `-o %(title)s-%(id)s.%(ext)s` and mp4 video with title `youtube-dl test video` and id `BaW_jenozKcj` this will result in a `youtube-dl test video-BaW_jenozKcj.mp4` file created in the current directory.

From 949b6497cc2be1f21cd439c6fc6e4047eddb0d66 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 24 Apr 2016 16:23:21 +0800
Subject: [PATCH 4714/6219] [generic] Unescape the video URL

Fixes #9279
---
 youtube_dl/extractor/generic.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 95d233259..16c2c60d7 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -2045,6 +2045,7 @@ class GenericIE(InfoExtractor):
 
         entries = []
         for video_url in found:
+            video_url = unescapeHTML(video_url)
             video_url = video_url.replace('\\/', '/')
             video_url = compat_urlparse.urljoin(url, video_url)
             video_id = compat_urllib_parse_unquote(os.path.basename(video_url))

From 2a7c38831cc8f789cdf4ee63f8d4450a46f45017 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 24 Apr 2016 17:01:18 +0800
Subject: [PATCH 4715/6219] [yahoo] Extend _VALID_URL and fix extraction

Closes #9271
---
 youtube_dl/extractor/yahoo.py | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/yahoo.py b/youtube_dl/extractor/yahoo.py
index b2d8f4b48..e2613659c 100644
--- a/youtube_dl/extractor/yahoo.py
+++ b/youtube_dl/extractor/yahoo.py
@@ -24,7 +24,7 @@ from .nbc import NBCSportsVPlayerIE
 
 class YahooIE(InfoExtractor):
     IE_DESC = 'Yahoo screen and movies'
-    _VALID_URL = r'(?P<url>(?P<host>https?://(?:[a-zA-Z]{2}\.)?[\da-zA-Z_-]+\.yahoo\.com)/(?:[^/]+/)*(?P<display_id>.+)?-(?P<id>[0-9]+)(?:-[a-z]+)?\.html)'
+    _VALID_URL = r'(?P<url>(?P<host>https?://(?:[a-zA-Z]{2}\.)?[\da-zA-Z_-]+\.yahoo\.com)/(?:[^/]+/)*(?P<display_id>.+)?-(?P<id>[0-9]+)(?:-[a-z]+)?(?:\.html)?)'
     _TESTS = [
         {
             'url': 'http://screen.yahoo.com/julian-smith-travis-legg-watch-214727115.html',
@@ -166,6 +166,17 @@ class YahooIE(InfoExtractor):
                 'description': 'While they play feuding fathers in \'Daddy\'s Home,\' star Will Ferrell & Mark Wahlberg share their true feelings on parenthood.',
             },
         },
+        {
+            # config['models']['applet_model']['data']['sapi'] has no query
+            'url': 'https://www.yahoo.com/music/livenation/event/galactic-2016',
+            'md5': 'dac0c72d502bc5facda80c9e6d5c98db',
+            'info_dict': {
+                'id': 'a6015640-e9e5-3efb-bb60-05589a183919',
+                'ext': 'mp4',
+                'description': 'Galactic',
+                'title': 'Dolla Diva (feat. Maggie Koerner)',
+            },
+        },
     ]
 
     def _real_extract(self, url):
@@ -202,7 +213,7 @@ class YahooIE(InfoExtractor):
             config = self._parse_json(config_json, display_id, fatal=False)
             if config:
                 sapi = config.get('models', {}).get('applet_model', {}).get('data', {}).get('sapi')
-                if sapi:
+                if sapi and 'query' in sapi:
                     return self._extract_info(display_id, sapi, webpage)
 
         items_json = self._search_regex(

From 4f549580977ab94364fd404cdebba22575c74b91 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 24 Apr 2016 17:28:18 +0800
Subject: [PATCH 4716/6219] [yahoo] Update some tests

One has new fields as ThePlatformIE changed, and others have changed
files.
---
 youtube_dl/extractor/yahoo.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/yahoo.py b/youtube_dl/extractor/yahoo.py
index e2613659c..e4f3d8937 100644
--- a/youtube_dl/extractor/yahoo.py
+++ b/youtube_dl/extractor/yahoo.py
@@ -38,7 +38,7 @@ class YahooIE(InfoExtractor):
         },
         {
             'url': 'http://screen.yahoo.com/wired/codefellas-s1-ep12-cougar-lies-103000935.html',
-            'md5': 'd6e6fc6e1313c608f316ddad7b82b306',
+            'md5': 'c3466d2b6d5dd6b9f41ba9ed04c24b23',
             'info_dict': {
                 'id': 'd1dedf8c-d58c-38c3-8963-e899929ae0a9',
                 'ext': 'mp4',
@@ -49,7 +49,7 @@ class YahooIE(InfoExtractor):
         },
         {
             'url': 'https://screen.yahoo.com/community/community-sizzle-reel-203225340.html?format=embed',
-            'md5': '60e8ac193d8fb71997caa8fce54c6460',
+            'md5': '75ffabdb87c16d4ffe8c036dc4d1c136',
             'info_dict': {
                 'id': '4fe78544-8d48-39d8-97cd-13f205d9fcdb',
                 'ext': 'mp4',
@@ -122,7 +122,7 @@ class YahooIE(InfoExtractor):
             }
         }, {
             'url': 'https://www.yahoo.com/movies/v/true-story-trailer-173000497.html',
-            'md5': '989396ae73d20c6f057746fb226aa215',
+            'md5': 'b17ac378b1134fa44370fb27db09a744',
             'info_dict': {
                 'id': '071c4013-ce30-3a93-a5b2-e0413cd4a9d1',
                 'ext': 'mp4',
@@ -141,6 +141,9 @@ class YahooIE(InfoExtractor):
                 'ext': 'flv',
                 'description': 'md5:df390f70a9ba7c95ff1daace988f0d8d',
                 'title': 'Tyler Kalinoski hits buzzer-beater to lift Davidson',
+                'upload_date': '20150313',
+                'uploader': 'NBCU-SPORTS',
+                'timestamp': 1426270238,
             }
         }, {
             'url': 'https://tw.news.yahoo.com/-100120367.html',
@@ -148,7 +151,7 @@ class YahooIE(InfoExtractor):
         }, {
             # Query result is embedded in webpage, but explicit request to video API fails with geo restriction
             'url': 'https://screen.yahoo.com/community/communitary-community-episode-1-ladders-154501237.html',
-            'md5': '4fbafb9c9b6f07aa8f870629f6671b35',
+            'md5': '1ddbf7c850777548438e5c4f147c7b8c',
             'info_dict': {
                 'id': '1f32853c-a271-3eef-8cb6-f6d6872cb504',
                 'ext': 'mp4',

From d9ed362116969362e1c404aea63d9f6f3e833478 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 24 Apr 2016 17:46:25 +0800
Subject: [PATCH 4717/6219] [yahoo] Extract all <iframe>s

Fixes test_yahoo_6

(https://ca.finance.yahoo.com/news/hackers-sony-more-trouble-well-154609075.html)
---
 youtube_dl/extractor/yahoo.py | 41 +++++++++++++++++++++++++----------
 1 file changed, 30 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/yahoo.py b/youtube_dl/extractor/yahoo.py
index e4f3d8937..73396f9f8 100644
--- a/youtube_dl/extractor/yahoo.py
+++ b/youtube_dl/extractor/yahoo.py
@@ -92,14 +92,28 @@ class YahooIE(InfoExtractor):
             }
         }, {
             'url': 'https://ca.finance.yahoo.com/news/hackers-sony-more-trouble-well-154609075.html',
-            'md5': '226a895aae7e21b0129e2a2006fe9690',
             'info_dict': {
-                'id': 'e624c4bc-3389-34de-9dfc-025f74943409',
-                'ext': 'mp4',
-                'title': '\'The Interview\' TV Spot: War',
-                'description': 'The Interview',
-                'duration': 30,
-            }
+                'id': '154609075',
+            },
+            'playlist': [{
+                'md5': 'f8e336c6b66f503282e5f719641d6565',
+                'info_dict': {
+                    'id': 'e624c4bc-3389-34de-9dfc-025f74943409',
+                    'ext': 'mp4',
+                    'title': '\'The Interview\' TV Spot: War',
+                    'description': 'The Interview',
+                    'duration': 30,
+                },
+            }, {
+                'md5': '958bcb90b4d6df71c56312137ee1cd5a',
+                'info_dict': {
+                    'id': '1fc8ada0-718e-3abe-a450-bf31f246d1a9',
+                    'ext': 'mp4',
+                    'title': '\'The Interview\' TV Spot: Guys',
+                    'description': 'The Interview',
+                    'duration': 30,
+                },
+            }],
         }, {
             'url': 'http://news.yahoo.com/video/china-moses-crazy-blues-104538833.html',
             'md5': '88e209b417f173d86186bef6e4d1f160',
@@ -191,16 +205,21 @@ class YahooIE(InfoExtractor):
         webpage = self._download_webpage(url, display_id)
 
         # Look for iframed media first
-        iframe_m = re.search(r'<iframe[^>]+src="(/video/.+?-\d+\.html\?format=embed.*?)"', webpage)
-        if iframe_m:
+        entries = []
+        iframe_urls = re.findall(r'<iframe[^>]+src="(/video/.+?-\d+\.html\?format=embed.*?)"', webpage)
+        for idx, iframe_url in enumerate(iframe_urls):
             iframepage = self._download_webpage(
-                host + iframe_m.group(1), display_id, 'Downloading iframe webpage')
+                host + iframe_url, display_id,
+                note='Downloading iframe webpage for video #%d' % idx)
             items_json = self._search_regex(
                 r'mediaItems: (\[.+?\])$', iframepage, 'items', flags=re.MULTILINE, default=None)
             if items_json:
                 items = json.loads(items_json)
                 video_id = items[0]['id']
-                return self._get_info(video_id, display_id, webpage)
+                entries.append(self._get_info(video_id, display_id, webpage))
+        if entries:
+            return self.playlist_result(entries, page_id)
+
         # Look for NBCSports iframes
         nbc_sports_url = NBCSportsVPlayerIE._extract_url(webpage)
         if nbc_sports_url:

From 2a7dee8cc5afac1036bd2f071c8cca8ac9815d8f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 24 Apr 2016 18:11:51 +0800
Subject: [PATCH 4718/6219] [yahoo] Improve error detection and update tests

---
 youtube_dl/extractor/yahoo.py | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/yahoo.py b/youtube_dl/extractor/yahoo.py
index 73396f9f8..b376f2b93 100644
--- a/youtube_dl/extractor/yahoo.py
+++ b/youtube_dl/extractor/yahoo.py
@@ -59,15 +59,15 @@ class YahooIE(InfoExtractor):
             }
         },
         {
-            'url': 'https://tw.screen.yahoo.com/election-2014-askmayor/敢問市長-黃秀霜批賴清德-非常高傲-033009720.html',
-            'md5': '3a09cf59349cfaddae1797acc3c087fc',
+            'url': 'https://tw.news.yahoo.com/%E6%95%A2%E5%95%8F%E5%B8%82%E9%95%B7%20%E9%BB%83%E7%A7%80%E9%9C%9C%E6%89%B9%E8%B3%B4%E6%B8%85%E5%BE%B7%20%E9%9D%9E%E5%B8%B8%E9%AB%98%E5%82%B2-034024051.html',
+            'md5': '9035d38f88b1782682a3e89f985be5bb',
             'info_dict': {
                 'id': 'cac903b3-fcf4-3c14-b632-643ab541712f',
                 'ext': 'mp4',
                 'title': '敢問市長／黃秀霜批賴清德「非常高傲」',
                 'description': '直言台南沒捷運 交通居五都之末',
                 'duration': 396,
-            }
+            },
         },
         {
             'url': 'https://uk.screen.yahoo.com/editor-picks/cute-raccoon-freed-drain-using-091756545.html',
@@ -89,7 +89,8 @@ class YahooIE(InfoExtractor):
                 'title': 'Program that makes hockey more affordable not offered in Manitoba',
                 'description': 'md5:c54a609f4c078d92b74ffb9bf1f496f4',
                 'duration': 121,
-            }
+            },
+            'skip': 'Video gone',
         }, {
             'url': 'https://ca.finance.yahoo.com/news/hackers-sony-more-trouble-well-154609075.html',
             'info_dict': {
@@ -133,7 +134,8 @@ class YahooIE(InfoExtractor):
                 'title': 'Connect the Dots: Dark Side of Virgo',
                 'description': 'md5:1428185051cfd1949807ad4ff6d3686a',
                 'duration': 201,
-            }
+            },
+            'skip': 'Domain name in.lifestyle.yahoo.com gone',
         }, {
             'url': 'https://www.yahoo.com/movies/v/true-story-trailer-173000497.html',
             'md5': 'b17ac378b1134fa44370fb27db09a744',
@@ -202,7 +204,9 @@ class YahooIE(InfoExtractor):
         page_id = mobj.group('id')
         url = mobj.group('url')
         host = mobj.group('host')
-        webpage = self._download_webpage(url, display_id)
+        webpage, urlh = self._download_webpage_handle(url, display_id)
+        if 'err=404' in urlh.geturl():
+            raise ExtractorError('Video gone', expected=True)
 
         # Look for iframed media first
         entries = []

From a8062eabcd850278d5545e2a8e6d39392e540571 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 24 Apr 2016 20:04:21 +0800
Subject: [PATCH 4719/6219] [mwave] Skip checking unstable MD5

On my PC the checksum is 02eda6d09fb63131a17a8d44e6237463, while a
recent Travis CI build
(https://travis-ci.org/rg3/youtube-dl/jobs/125341081) shows it's
c930e27b7720aaa3c9d0018dfc8ff6cc
---
 youtube_dl/extractor/mwave.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mwave.py b/youtube_dl/extractor/mwave.py
index 66b523197..5c3c8d464 100644
--- a/youtube_dl/extractor/mwave.py
+++ b/youtube_dl/extractor/mwave.py
@@ -12,7 +12,7 @@ class MwaveIE(InfoExtractor):
     _VALID_URL = r'https?://mwave\.interest\.me/mnettv/videodetail\.m\?searchVideoDetailVO\.clip_id=(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://mwave.interest.me/mnettv/videodetail.m?searchVideoDetailVO.clip_id=168859',
-        'md5': 'c930e27b7720aaa3c9d0018dfc8ff6cc',
+        # md5 is unstable
         'info_dict': {
             'id': '168859',
             'ext': 'flv',

From a956cb63064f02121da73c298f3b500a9a49265e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 24 Apr 2016 20:41:17 +0800
Subject: [PATCH 4720/6219] [onionstudios] Fix description extraction

\1 does not work in []. Fixes test_Generic_75
(http://www.clickhole.com/video/dont-understand-bitcoin-man-will-mumble-explanatio-2537)
---
 youtube_dl/extractor/onionstudios.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/onionstudios.py b/youtube_dl/extractor/onionstudios.py
index 6e843c327..d7b13a0f1 100644
--- a/youtube_dl/extractor/onionstudios.py
+++ b/youtube_dl/extractor/onionstudios.py
@@ -65,7 +65,7 @@ class OnionStudiosIE(InfoExtractor):
             r'share_title\s*=\s*(["\'])(?P<title>[^\1]+?)\1',
             webpage, 'title', group='title')
         description = self._search_regex(
-            r'share_description\s*=\s*(["\'])(?P<description>[^\1]+?)\1',
+            r'share_description\s*=\s*(["\'])(?P<description>[^\'"]+?)\1',
             webpage, 'description', default=None, group='description')
         thumbnail = self._search_regex(
             r'poster\s*=\s*(["\'])(?P<thumbnail>[^\1]+?)\1',

From 0738187f9b66e7cfb3d0b15ca4b08834a9e076df Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 24 Apr 2016 20:44:52 +0800
Subject: [PATCH 4721/6219] [ThePlatform] Fix tests failed since
 79ba9140dc8fcf5883b7473596e8f20cba6b479f

---
 youtube_dl/extractor/cbc.py         |  1 +
 youtube_dl/extractor/generic.py     | 17 +++++++++++++++++
 youtube_dl/extractor/metacafe.py    |  3 +++
 youtube_dl/extractor/nbc.py         |  3 +++
 youtube_dl/extractor/theplatform.py |  1 +
 5 files changed, 25 insertions(+)

diff --git a/youtube_dl/extractor/cbc.py b/youtube_dl/extractor/cbc.py
index d8aa31038..68a0633b6 100644
--- a/youtube_dl/extractor/cbc.py
+++ b/youtube_dl/extractor/cbc.py
@@ -33,6 +33,7 @@ class CBCIE(InfoExtractor):
             'title': 'Robin Williams freestyles on 90 Minutes Live',
             'description': 'Wacky American comedian Robin Williams shows off his infamous "freestyle" comedic talents while being interviewed on CBC\'s 90 Minutes Live.',
             'upload_date': '19700101',
+            'uploader': 'CBCC-NEW',
         },
         'params': {
             # rtmp download
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 16c2c60d7..c63bdbd08 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -237,6 +237,7 @@ class GenericIE(InfoExtractor):
                 'ext': 'mp4',
                 'title': 'car-20120827-manifest',
                 'formats': 'mincount:9',
+                'upload_date': '20130904',
             },
             'params': {
                 'format': 'bestvideo',
@@ -596,7 +597,11 @@ class GenericIE(InfoExtractor):
                 'id': 'k2mm4bCdJ6CQ2i7c8o2',
                 'ext': 'mp4',
                 'title': 'Le Zap de Spi0n n°216 - Zapping du Web',
+                'description': 'md5:faf028e48a461b8b7fad38f1e104b119',
                 'uploader': 'Spi0n',
+                'uploader_id': 'xgditw',
+                'upload_date': '20140425',
+                'timestamp': 1398441542,
             },
             'add_ie': ['Dailymotion'],
         },
@@ -729,8 +734,11 @@ class GenericIE(InfoExtractor):
                 'id': 'uxjb0lwrcz',
                 'ext': 'mp4',
                 'title': 'Conversation about Hexagonal Rails Part 1 - ThoughtWorks',
+                'description': 'a Martin Fowler video from ThoughtWorks',
                 'duration': 1715.0,
                 'uploader': 'thoughtworks.wistia.com',
+                'upload_date': '20140603',
+                'timestamp': 1401832161,
             },
         },
         # Soundcloud embed
@@ -981,6 +989,9 @@ class GenericIE(InfoExtractor):
                 'ext': 'flv',
                 'title': "PFT Live: New leader in the 'new-look' defense",
                 'description': 'md5:65a19b4bbfb3b0c0c5768bed1dfad74e',
+                'uploader': 'NBCU-SPORTS',
+                'upload_date': '20140107',
+                'timestamp': 1389118457,
             },
         },
         # UDN embed
@@ -1033,6 +1044,9 @@ class GenericIE(InfoExtractor):
                 'title': 'SN Presents: Russell Martin, World Citizen',
                 'description': 'To understand why he was the Toronto Blue Jays’ top off-season priority is to appreciate his background and upbringing in Montreal, where he first developed his baseball skills. Written and narrated by Stephen Brunt.',
                 'uploader': 'Rogers Sportsnet',
+                'uploader_id': '1704050871',
+                'upload_date': '20150525',
+                'timestamp': 1432570283,
             },
         },
         # Dailymotion Cloud video
@@ -1124,6 +1138,9 @@ class GenericIE(InfoExtractor):
                 'title': 'The Cardinal Pell Interview',
                 'description': 'Sky News Contributor Andrew Bolt interviews George Pell in Rome, following the Cardinal\'s evidence before the Royal Commission into Child Abuse. ',
                 'uploader': 'GlobeCast Australia - GlobeStream',
+                'uploader_id': '2733773828001',
+                'upload_date': '20160304',
+                'timestamp': 1457083087,
             },
             'params': {
                 # m3u8 downloads
diff --git a/youtube_dl/extractor/metacafe.py b/youtube_dl/extractor/metacafe.py
index 61dadb7a7..b6f00cc25 100644
--- a/youtube_dl/extractor/metacafe.py
+++ b/youtube_dl/extractor/metacafe.py
@@ -81,6 +81,9 @@ class MetacafeIE(InfoExtractor):
                 'title': 'Open: This is Face the Nation, February 9',
                 'description': 'md5:8a9ceec26d1f7ed6eab610834cc1a476',
                 'duration': 96,
+                'uploader': 'CBSI-NEW',
+                'upload_date': '20140209',
+                'timestamp': 1391959800,
             },
             'params': {
                 # rtmp download
diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index f9d42d07a..46504cd5f 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -134,6 +134,9 @@ class NBCSportsIE(InfoExtractor):
             'ext': 'flv',
             'title': 'Tom Izzo, Michigan St. has \'so much respect\' for Duke',
             'description': 'md5:ecb459c9d59e0766ac9c7d5d0eda8113',
+            'uploader': 'NBCU-SPORTS',
+            'upload_date': '20150330',
+            'timestamp': 1427726529,
         }
     }
 
diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index 7a5a533b7..8272dd969 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -269,6 +269,7 @@ class ThePlatformFeedIE(ThePlatformBaseIE):
             'timestamp': 1391824260,
             'duration': 467.0,
             'categories': ['MSNBC/Issues/Democrats', 'MSNBC/Issues/Elections/Election 2016'],
+            'uploader': 'NBCU-NEWS',
         },
     }
 

From 03dd60ca41ddb2dec3723ac6446bd291a94cad18 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 24 Apr 2016 21:16:06 +0800
Subject: [PATCH 4722/6219] [kuwo:category] Fix the test

Sometimes there are 24 songs and sometimes 30 lol
---
 youtube_dl/extractor/kuwo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/kuwo.py b/youtube_dl/extractor/kuwo.py
index c0ece5113..a02638665 100644
--- a/youtube_dl/extractor/kuwo.py
+++ b/youtube_dl/extractor/kuwo.py
@@ -268,7 +268,7 @@ class KuwoCategoryIE(InfoExtractor):
             'title': '八十年代精选',
             'description': '这些都是属于八十年代的回忆！',
         },
-        'playlist_count': 24,
+        'playlist_mincount': 24,
     }
 
     def _real_extract(self, url):

From c14dc00df33f29f681f1cc3424783acaee7efbb7 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 24 Apr 2016 14:33:47 +0100
Subject: [PATCH 4723/6219] [viewster] improve http formats extraction

---
 youtube_dl/extractor/viewster.py | 35 ++++++++++++++++++--------------
 1 file changed, 20 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/viewster.py b/youtube_dl/extractor/viewster.py
index 4289a2f31..7839225d4 100644
--- a/youtube_dl/extractor/viewster.py
+++ b/youtube_dl/extractor/viewster.py
@@ -146,21 +146,26 @@ class ViewsterIE(InfoExtractor):
                 qualities_basename = self._search_regex(
                     '/([^/]+)\.csmil/',
                     manifest_url, 'qualities basename', default=None)
-                if qualities_basename:
-                    QUALITIES_RE = r'((,\d+k)+,?)'
-                    qualities = self._search_regex(
-                        QUALITIES_RE, qualities_basename,
-                        'qualities').strip(',').split(',')
-                    http_template = re.sub(QUALITIES_RE, r'%s', qualities_basename)
-                    http_url_basename = url_basename(video_url)
-                    for q in qualities:
-                        formats.append({
-                            'url': video_url.replace(http_url_basename, http_template % q),
-                            'ext': 'mp4',
-                            'format_id': 'http-%s' % q,
-                            'tbr': int_or_none(self._search_regex(
-                                r'(\d+)k', q, 'bitrate', default=None)),
-                        })
+                if not qualities_basename:
+                    continue
+                QUALITIES_RE = r'((,\d+k)+,?)'
+                qualities = self._search_regex(
+                    QUALITIES_RE, qualities_basename,
+                    'qualities', default=None)
+                if not qualities:
+                    continue
+                qualities = qualities.strip(',').split(',')
+                http_template = re.sub(QUALITIES_RE, r'%s', qualities_basename)
+                http_url_basename = url_basename(video_url)
+                for q in qualities:
+                    tbr = int_or_none(self._search_regex(
+                        r'(\d+)k', q, 'bitrate', default=None))
+                    formats.append({
+                        'url': video_url.replace(http_url_basename, http_template % q),
+                        'ext': 'mp4',
+                        'format_id': 'http' + ('-%d' % tbr if tbr else ''),
+                        'tbr': tbr,
+                    })
 
         if not formats and not info.get('LanguageSets') and not info.get('VODSettings'):
             self.raise_geo_restricted()

From a82398bd72272987133c4b81cad46139ec4b5359 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 24 Apr 2016 22:19:26 +0800
Subject: [PATCH 4724/6219] [kwuo:song] Fix extraction and update the test

---
 youtube_dl/extractor/kuwo.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/kuwo.py b/youtube_dl/extractor/kuwo.py
index a02638665..3740869c7 100644
--- a/youtube_dl/extractor/kuwo.py
+++ b/youtube_dl/extractor/kuwo.py
@@ -81,7 +81,7 @@ class KuwoIE(KuwoBaseIE):
             'id': '6446136',
             'ext': 'mp3',
             'title': '心',
-            'description': 'md5:b2ab6295d014005bfc607525bfc1e38a',
+            'description': 'md5:5d0e947b242c35dc0eb1d2fce9fbf02c',
             'creator': 'IU',
             'upload_date': '20150518',
         },
@@ -102,10 +102,10 @@ class KuwoIE(KuwoBaseIE):
             raise ExtractorError('this song has been offline because of copyright issues', expected=True)
 
         song_name = self._html_search_regex(
-            r'(?s)class="(?:[^"\s]+\s+)*title(?:\s+[^"\s]+)*".*?<h1[^>]+title="([^"]+)"', webpage, 'song name')
-        singer_name = self._html_search_regex(
-            r'<div[^>]+class="s_img">\s*<a[^>]+title="([^>]+)"',
-            webpage, 'singer name', fatal=False)
+            r'<p[^>]+id="lrcName">([^<]+)</p>', webpage, 'song name')
+        singer_name = remove_start(self._html_search_regex(
+            r'<a[^>]+href="http://www\.kuwo\.cn/artist/content\?name=([^"]+)">',
+            webpage, 'singer name', fatal=False), '歌手')
         lrc_content = clean_html(get_element_by_id('lrcContent', webpage))
         if lrc_content == '暂无':     # indicates no lyrics
             lrc_content = None
@@ -114,7 +114,7 @@ class KuwoIE(KuwoBaseIE):
         self._sort_formats(formats)
 
         album_id = self._html_search_regex(
-            r'<p[^>]+class="album"[^<]+<a[^>]+href="http://www\.kuwo\.cn/album/(\d+)/"',
+            r'<a[^>]+href="http://www\.kuwo\.cn/album/(\d+)/"',
             webpage, 'album id', fatal=False)
 
         publish_time = None

From 8b38f2ac40f30743b87fcf92c7570d669923a796 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 24 Apr 2016 17:06:46 +0200
Subject: [PATCH 4725/6219] release 2016.04.24

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 README.md                 | 4 +++-
 docs/supportedsites.md    | 9 +++++----
 youtube_dl/version.py     | 2 +-
 4 files changed, 12 insertions(+), 9 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 35f8e6863..c208eb689 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.04.19*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.04.19**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.04.24*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.04.24**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.04.19
+[debug] youtube-dl version 2016.04.24
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/README.md b/README.md
index 7a54d651d..e062444b3 100644
--- a/README.md
+++ b/README.md
@@ -176,7 +176,9 @@ which means you can modify it, redistribute it or use it however you like.
     --xattr-set-filesize             Set file xattribute ytdl.filesize with
                                      expected filesize (experimental)
     --hls-prefer-native              Use the native HLS downloader instead of
-                                     ffmpeg (experimental)
+                                     ffmpeg
+    --hls-prefer-ffmpeg              Use ffmpeg instead of the native HLS
+                                     downloader
     --hls-use-mpegts                 Use the mpegts container for HLS videos,
                                      allowing to play the video while
                                      downloading (some players may not be able
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index e12a7d182..03875b8db 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -163,6 +163,7 @@
  - **defense.gouv.fr**
  - **democracynow**
  - **DHM**: Filmarchiv - Deutsches Historisches Museum
+ - **DigitallySpeaking**
  - **Digiteka**
  - **Discovery**
  - **Dotsub**
@@ -174,7 +175,6 @@
  - **Dropbox**
  - **DrTuber**
  - **DRTV**
- - **Dump**
  - **Dumpert**
  - **dvtv**: http://video.aktualne.cz/
  - **dw**
@@ -345,6 +345,7 @@
  - **metacafe**
  - **Metacritic**
  - **Mgoon**
+ - **MGTV**: 芒果TV
  - **Minhateca**
  - **MinistryGrid**
  - **Minoto**
@@ -413,7 +414,8 @@
  - **nfl.com**
  - **nhl.com**
  - **nhl.com:news**: NHL news
- - **nhl.com:videocenter**: NHL videocenter category
+ - **nhl.com:videocenter**
+ - **nhl.com:videocenter:category**: NHL videocenter category
  - **nick.com**
  - **niconico**: ニコニコ動画
  - **NiconicoPlaylist**
@@ -461,13 +463,13 @@
  - **Patreon**
  - **pbs**: Public Broadcasting Service (PBS) and member stations: PBS: Public Broadcasting Service, APT - Alabama Public Television (WBIQ), GPB/Georgia Public Broadcasting (WGTV), Mississippi Public Broadcasting (WMPN), Nashville Public Television (WNPT), WFSU-TV (WFSU), WSRE (WSRE), WTCI (WTCI), WPBA/Channel 30 (WPBA), Alaska Public Media (KAKM), Arizona PBS (KAET), KNME-TV/Channel 5 (KNME), Vegas PBS (KLVX), AETN/ARKANSAS ETV NETWORK (KETS), KET (WKLE), WKNO/Channel 10 (WKNO), LPB/LOUISIANA PUBLIC BROADCASTING (WLPB), OETA (KETA), Ozarks Public Television (KOZK), WSIU Public Broadcasting (WSIU), KEET TV (KEET), KIXE/Channel 9 (KIXE), KPBS San Diego (KPBS), KQED (KQED), KVIE Public Television (KVIE), PBS SoCal/KOCE (KOCE), ValleyPBS (KVPT), CONNECTICUT PUBLIC TELEVISION (WEDH), KNPB Channel 5 (KNPB), SOPTV (KSYS), Rocky Mountain PBS (KRMA), KENW-TV3 (KENW), KUED Channel 7 (KUED), Wyoming PBS (KCWC), Colorado Public Television / KBDI 12 (KBDI), KBYU-TV (KBYU), Thirteen/WNET New York (WNET), WGBH/Channel 2 (WGBH), WGBY (WGBY), NJTV Public Media NJ (WNJT), WLIW21 (WLIW), mpt/Maryland Public Television (WMPB), WETA Television and Radio (WETA), WHYY (WHYY), PBS 39 (WLVT), WVPT - Your Source for PBS and More! (WVPT), Howard University Television (WHUT), WEDU PBS (WEDU), WGCU Public Media (WGCU), WPBT2 (WPBT), WUCF TV (WUCF), WUFT/Channel 5 (WUFT), WXEL/Channel 42 (WXEL), WLRN/Channel 17 (WLRN), WUSF Public Broadcasting (WUSF), ETV (WRLK), UNC-TV (WUNC), PBS Hawaii - Oceanic Cable Channel 10 (KHET), Idaho Public Television (KAID), KSPS (KSPS), OPB (KOPB), KWSU/Channel 10 & KTNW/Channel 31 (KWSU), WILL-TV (WILL), Network Knowledge - WSEC/Springfield (WSEC), WTTW11 (WTTW), Iowa Public Television/IPTV (KDIN), Nine Network (KETC), PBS39 Fort Wayne (WFWA), WFYI Indianapolis (WFYI), Milwaukee Public Television (WMVS), WNIN (WNIN), WNIT Public Television (WNIT), WPT (WPNE), WVUT/Channel 22 (WVUT), WEIU/Channel 51 (WEIU), WQPT-TV (WQPT), WYCC PBS Chicago (WYCC), WIPB-TV (WIPB), WTIU (WTIU), CET  (WCET), ThinkTVNetwork (WPTD), WBGU-TV (WBGU), WGVU TV (WGVU), NET1 (KUON), Pioneer Public Television (KWCM), SDPB Television (KUSD), TPT (KTCA), KSMQ (KSMQ), KPTS/Channel 8 (KPTS), KTWU/Channel 11 (KTWU), East Tennessee PBS (WSJK), WCTE-TV (WCTE), WLJT, Channel 11 (WLJT), WOSU TV (WOSU), WOUB/WOUC (WOUB), WVPB (WVPB), WKYU-PBS (WKYU), KERA 13 (KERA), MPBN (WCBB), Mountain Lake PBS (WCFE), NHPTV (WENH), Vermont PBS (WETK), witf (WITF), WQED Multimedia (WQED), WMHT Educational Telecommunications (WMHT), Q-TV (WDCQ), WTVS Detroit Public TV (WTVS), CMU Public Television (WCMU), WKAR-TV (WKAR), WNMU-TV Public TV 13 (WNMU), WDSE - WRPT (WDSE), WGTE TV (WGTE), Lakeland Public Television (KAWE), KMOS-TV - Channels 6.1, 6.2 and 6.3 (KMOS), MontanaPBS (KUSM), KRWG/Channel 22 (KRWG), KACV (KACV), KCOS/Channel 13 (KCOS), WCNY/Channel 24 (WCNY), WNED (WNED), WPBS (WPBS), WSKG Public TV (WSKG), WXXI (WXXI), WPSU (WPSU), WVIA Public Media Studios (WVIA), WTVI (WTVI), Western Reserve PBS (WNEO), WVIZ/PBS ideastream (WVIZ), KCTS 9 (KCTS), Basin PBS (KPBT), KUHT / Channel 8 (KUHT), KLRN (KLRN), KLRU (KLRU), WTJX Channel 12 (WTJX), WCVE PBS (WCVE), KBTC Public Television (KBTC)
  - **pcmag**
+ - **People**
  - **Periscope**: Periscope
  - **PhilharmonieDeParis**: Philharmonie de Paris
  - **phoenix.de**
  - **Photobucket**
  - **Pinkbike**
  - **Pladform**
- - **PlanetaPlay**
  - **play.fm**
  - **played.to**
  - **PlaysTV**
@@ -497,7 +499,6 @@
  - **qqmusic:playlist**: QQ音乐 - 歌单
  - **qqmusic:singer**: QQ音乐 - 歌手
  - **qqmusic:toplist**: QQ音乐 - 排行榜
- - **QuickVid**
  - **R7**
  - **radio.de**
  - **radiobremen**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 140a67847..8befd9607 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.04.19'
+__version__ = '2016.04.24'

From 51762e1a31a58d441cd8e3f1fb9374dd6572a3d0 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 24 Apr 2016 23:18:34 +0800
Subject: [PATCH 4726/6219] [xminus] Fix extraction (closes #9228)

---
 youtube_dl/extractor/xminus.py | 45 ++++++++++++++++++----------------
 1 file changed, 24 insertions(+), 21 deletions(-)

diff --git a/youtube_dl/extractor/xminus.py b/youtube_dl/extractor/xminus.py
index 7c9d8af6f..36e5ead1e 100644
--- a/youtube_dl/extractor/xminus.py
+++ b/youtube_dl/extractor/xminus.py
@@ -2,15 +2,15 @@
 from __future__ import unicode_literals
 
 import re
+import time
 
 from .common import InfoExtractor
 from ..compat import (
-    compat_chr,
     compat_ord,
 )
 from ..utils import (
     int_or_none,
-    parse_filesize,
+    parse_duration,
 )
 
 
@@ -22,7 +22,7 @@ class XMinusIE(InfoExtractor):
         'info_dict': {
             'id': '4542',
             'ext': 'mp3',
-            'title': 'Леонид Агутин-Песенка шофера',
+            'title': 'Леонид Агутин-Песенка шофёра',
             'duration': 156,
             'tbr': 320,
             'filesize_approx': 5900000,
@@ -36,38 +36,41 @@ class XMinusIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         artist = self._html_search_regex(
-            r'minus_track\.artist="(.+?)"', webpage, 'artist')
+            r'<a[^>]+href="/artist/\d+">([^<]+)</a>', webpage, 'artist')
         title = artist + '-' + self._html_search_regex(
-            r'minus_track\.title="(.+?)"', webpage, 'title')
-        duration = int_or_none(self._html_search_regex(
-            r'minus_track\.dur_sec=\'([0-9]*?)\'',
+            r'<span[^>]+class="minustrack-full-title(?:\s+[^"]+)?"[^>]*>([^<]+)', webpage, 'title')
+        duration = parse_duration(self._html_search_regex(
+            r'<span[^>]+class="player-duration(?:\s+[^"]+)?"[^>]*>([^<]+)',
             webpage, 'duration', fatal=False))
-        filesize_approx = parse_filesize(self._html_search_regex(
-            r'<div id="finfo"[^>]*>\s*↓\s*([0-9.]+\s*[a-zA-Z][bB])',
-            webpage, 'approximate filesize', fatal=False))
-        tbr = int_or_none(self._html_search_regex(
-            r'<div class="quality[^"]*"></div>\s*([0-9]+)\s*kbps',
-            webpage, 'bitrate', fatal=False))
+        mobj = re.search(
+            r'<div[^>]+class="dw-info(?:\s+[^"]+)?"[^>]*>(?P<tbr>\d+)\s*кбит/c\s+(?P<filesize>[0-9.]+)\s*мб</div>',
+            webpage)
+        tbr = filesize_approx = None
+        if mobj:
+            filesize_approx = float(mobj.group('filesize')) * 1000000
+            tbr = float(mobj.group('tbr'))
         view_count = int_or_none(self._html_search_regex(
-            r'<div class="quality.*?► ([0-9]+)',
+            r'<span><[^>]+class="icon-chart-bar".*?>(\d+)</span>',
             webpage, 'view count', fatal=False))
         description = self._html_search_regex(
-            r'(?s)<div id="song_texts">(.*?)</div><br',
+            r'(?s)<pre[^>]+id="lyrics-original"[^>]*>(.*?)</pre>',
             webpage, 'song lyrics', fatal=False)
         if description:
             description = re.sub(' *\r *', '\n', description)
 
-        enc_token = self._html_search_regex(
-            r'minus_track\.s?tkn="(.+?)"', webpage, 'enc_token')
-        token = ''.join(
-            c if pos == 3 else compat_chr(compat_ord(c) - 1)
-            for pos, c in enumerate(reversed(enc_token)))
-        video_url = 'http://x-minus.org/dwlf/%s/%s.mp3' % (video_id, token)
+        k = self._search_regex(
+            r'<div[^>]+id="player-bottom"[^>]+data-k="([^"]+)">', webpage,
+            'encoded data')
+        h = time.time() / 3600
+        a = sum(map(int, [compat_ord(c) for c in k])) + int(video_id) + h
+        video_url = 'http://x-minus.me/dl/minus?id=%s&tkn2=%df%d' % (video_id, a, h)
 
         return {
             'id': video_id,
             'title': title,
             'url': video_url,
+            # The extension is unknown until actual downloading
+            'ext': 'mp3',
             'duration': duration,
             'filesize_approx': filesize_approx,
             'tbr': tbr,

From aa9dc24f5a84265b66a25b9fa70c06e4fae299c6 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 24 Apr 2016 23:52:17 +0800
Subject: [PATCH 4727/6219] [douyutv] Improve extraction and update tests

The JSON API sometimes return HTML pages with errors
---
 youtube_dl/extractor/douyutv.py | 33 ++++++++++++++++++++++++---------
 1 file changed, 24 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/douyutv.py b/youtube_dl/extractor/douyutv.py
index 3915cb182..ce6962755 100644
--- a/youtube_dl/extractor/douyutv.py
+++ b/youtube_dl/extractor/douyutv.py
@@ -18,7 +18,7 @@ class DouyuTVIE(InfoExtractor):
             'display_id': 'iseven',
             'ext': 'flv',
             'title': 're:^清晨醒脑！T-ara根本停不下来！ [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
-            'description': 'md5:f34981259a03e980a3c6404190a3ed61',
+            'description': 're:.*m7show@163\.com.*',
             'thumbnail': 're:^https?://.*\.jpg$',
             'uploader': '7师傅',
             'uploader_id': '431925',
@@ -43,7 +43,7 @@ class DouyuTVIE(InfoExtractor):
         'params': {
             'skip_download': True,
         },
-        'skip': 'Romm not found',
+        'skip': 'Room not found',
     }, {
         'url': 'http://www.douyutv.com/17732',
         'info_dict': {
@@ -51,7 +51,7 @@ class DouyuTVIE(InfoExtractor):
             'display_id': '17732',
             'ext': 'flv',
             'title': 're:^清晨醒脑！T-ara根本停不下来！ [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
-            'description': 'md5:f34981259a03e980a3c6404190a3ed61',
+            'description': 're:.*m7show@163\.com.*',
             'thumbnail': 're:^https?://.*\.jpg$',
             'uploader': '7师傅',
             'uploader_id': '431925',
@@ -75,13 +75,28 @@ class DouyuTVIE(InfoExtractor):
             room_id = self._html_search_regex(
                 r'"room_id"\s*:\s*(\d+),', page, 'room id')
 
-        prefix = 'room/%s?aid=android&client_sys=android&time=%d' % (
-            room_id, int(time.time()))
+        config = None
+        # Douyu API sometimes returns error "Unable to load the requested class: eticket_redis_cache"
+        # Retry with different parameters - same parameters cause same errors
+        for i in range(5):
+            prefix = 'room/%s?aid=android&client_sys=android&time=%d' % (
+                room_id, int(time.time()))
+            auth = hashlib.md5((prefix + '1231').encode('ascii')).hexdigest()
 
-        auth = hashlib.md5((prefix + '1231').encode('ascii')).hexdigest()
-        config = self._download_json(
-            'http://www.douyutv.com/api/v1/%s&auth=%s' % (prefix, auth),
-            video_id)
+            config_page = self._download_webpage(
+                'http://www.douyutv.com/api/v1/%s&auth=%s' % (prefix, auth),
+                video_id)
+            try:
+                config = self._parse_json(config_page, video_id, fatal=False)
+            except ExtractorError:
+                # Wait some time before retrying to get a different time() value
+                self._sleep(1, video_id, msg_template='%(video_id)s: Error occurs. '
+                                                      'Waiting for %(timeout)s seconds before retrying')
+                continue
+            else:
+                break
+        if config is None:
+            raise ExtractorError('Unable to fetch API result')
 
         data = config['data']
 

From a1394b820d1f9004a9e48322bb1a3d99f0377704 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 25 Apr 2016 00:01:37 +0600
Subject: [PATCH 4728/6219] [openload] Fix title extraction (Closes #9298)

---
 youtube_dl/extractor/openload.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py
index 4468f31fc..216a40745 100644
--- a/youtube_dl/extractor/openload.py
+++ b/youtube_dl/extractor/openload.py
@@ -99,9 +99,14 @@ class OpenloadIE(InfoExtractor):
         video_url = self._search_regex(
             r'return\s+"(https?://[^"]+)"', self.openload_decode(code), 'video URL')
 
+        title = self._og_search_title(webpage, default=None) or self._search_regex(
+            r'<span[^>]+class=["\']title["\'][^>]*>([^<]+)', webpage,
+            'title', default=None) or self._html_search_meta(
+            'description', webpage, 'title', fatal=True)
+
         return {
             'id': video_id,
-            'title': self._og_search_title(webpage),
+            'title': title,
             'thumbnail': self._og_search_thumbnail(webpage),
             'url': video_url,
         }

From eb9ee194221e2fb0c260ead7573280fe06d875e8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 25 Apr 2016 00:03:12 +0600
Subject: [PATCH 4729/6219] [utils] Allow None mimetypes in mimetype2ext

---
 youtube_dl/utils.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index f333e4712..7bcc85e2b 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1935,6 +1935,9 @@ def error_to_compat_str(err):
 
 
 def mimetype2ext(mt):
+    if mt is None:
+        return None
+
     ext = {
         'audio/mp4': 'm4a',
     }.get(mt)

From 594b0c4c69da504906eb772e2bcd9bdbdc3f3a56 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 25 Apr 2016 00:03:29 +0600
Subject: [PATCH 4730/6219] [openload] Fix ext extraction

---
 youtube_dl/extractor/openload.py | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py
index 216a40745..697f312c3 100644
--- a/youtube_dl/extractor/openload.py
+++ b/youtube_dl/extractor/openload.py
@@ -6,8 +6,10 @@ import re
 from .common import InfoExtractor
 from ..compat import compat_chr
 from ..utils import (
+    determine_ext,
     encode_base_n,
     ExtractorError,
+    mimetype2ext,
 )
 
 
@@ -96,17 +98,25 @@ class OpenloadIE(InfoExtractor):
             r'<video[^>]+>\s*<script[^>]+>([^<]+)</script>',
             webpage, 'JS code')
 
+        decoded = self.openload_decode(code)
+
         video_url = self._search_regex(
-            r'return\s+"(https?://[^"]+)"', self.openload_decode(code), 'video URL')
+            r'return\s+"(https?://[^"]+)"', decoded, 'video URL')
 
         title = self._og_search_title(webpage, default=None) or self._search_regex(
             r'<span[^>]+class=["\']title["\'][^>]*>([^<]+)', webpage,
             'title', default=None) or self._html_search_meta(
             'description', webpage, 'title', fatal=True)
 
+        ext = mimetype2ext(self._search_regex(
+            r'window\.vt\s*=\s*(["\'])(?P<mimetype>.+?)\1', decoded,
+            'mimetype', default=None, group='mimetype')) or determine_ext(
+            video_url, 'mp4')
+
         return {
             'id': video_id,
             'title': title,
+            'ext': ext,
             'thumbnail': self._og_search_thumbnail(webpage),
             'url': video_url,
         }

From e9063b5de9a1118842185768f5e615b76ec8692c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 25 Apr 2016 00:22:55 +0600
Subject: [PATCH 4731/6219] [openload] Add test

---
 youtube_dl/extractor/openload.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py
index 697f312c3..9704f2e9d 100644
--- a/youtube_dl/extractor/openload.py
+++ b/youtube_dl/extractor/openload.py
@@ -31,6 +31,11 @@ class OpenloadIE(InfoExtractor):
     }, {
         'url': 'https://openload.io/f/ZAn6oz-VZGE/',
         'only_matching': True,
+    }, {
+        # unavailable via https://openload.co/f/Sxz5sADo82g/, different layout
+        # for title and ext
+        'url': 'https://openload.co/embed/Sxz5sADo82g/',
+        'only_matching': True,
     }]
 
     @staticmethod

From c83a352227401d7ca7eac045b58043ed576c0cdc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 25 Apr 2016 00:26:06 +0600
Subject: [PATCH 4732/6219] [openload] Make thumbnail optional

---
 youtube_dl/extractor/openload.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py
index 9704f2e9d..456561bcc 100644
--- a/youtube_dl/extractor/openload.py
+++ b/youtube_dl/extractor/openload.py
@@ -122,6 +122,6 @@ class OpenloadIE(InfoExtractor):
             'id': video_id,
             'title': title,
             'ext': ext,
-            'thumbnail': self._og_search_thumbnail(webpage),
+            'thumbnail': self._og_search_thumbnail(webpage, default=None),
             'url': video_url,
         }

From fb72ec58ae2612590d661c9943fe6b2fa0864401 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 30 Jul 2015 17:34:38 +0100
Subject: [PATCH 4733/6219] [extractor/common] do not process f4m manifest that
 contain akamai playerVerificationChallenge

---
 youtube_dl/extractor/common.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index dc5080504..e3d1dd076 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -856,6 +856,13 @@ class InfoExtractor(object):
             # (see https://github.com/rg3/youtube-dl/issues/6215#issuecomment-121704244)
             transform_source=transform_source)
 
+        # currently youtube-dl cannot decode the playerVerificationChallenge as Akamai uses Adobe Alchemy
+        akamai_pv = manifest.find('{http://ns.adobe.com/f4m/1.0}pv-2.0')
+        if akamai_pv is not None and ';' in akamai_pv.text:
+            playerVerificationChallenge = akamai_pv.text.split(';')[0]
+            if playerVerificationChallenge.strip() != '':
+                return []
+
         formats = []
         manifest_version = '1.0'
         media_nodes = manifest.findall('{http://ns.adobe.com/f4m/1.0}media')

From abc1723edd03d38b256e012d465e3343064f5682 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 25 Apr 2016 22:24:40 +0800
Subject: [PATCH 4734/6219] [unistra] Sort formats

Originally URLs are passed to set() and not sorted, so the result is not
deterministic, causing occasional FAILs on Travis CI.
---
 youtube_dl/extractor/unistra.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/unistra.py b/youtube_dl/extractor/unistra.py
index 66d9f1bf3..a724cdbef 100644
--- a/youtube_dl/extractor/unistra.py
+++ b/youtube_dl/extractor/unistra.py
@@ -49,6 +49,7 @@ class UnistraIE(InfoExtractor):
                 'format_id': format_id,
                 'quality': quality(format_id)
             })
+        self._sort_formats(formats)
 
         title = self._html_search_regex(
             r'<title>UTV - (.*?)</', webpage, 'title')

From 2beff95da5fb28440d26a3dee5de575c792d133c Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 25 Apr 2016 22:26:19 +0800
Subject: [PATCH 4735/6219] [nrk] Comment out unstable MD5 checksums

Both are Akamai f4f fragments.
---
 youtube_dl/extractor/nrk.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index 9df200822..51dfc27ac 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -23,7 +23,7 @@ class NRKIE(InfoExtractor):
     _TESTS = [
         {
             'url': 'http://www.nrk.no/video/PS*150533',
-            'md5': 'bccd850baebefe23b56d708a113229c2',
+            # MD5 is unstable
             'info_dict': {
                 'id': '150533',
                 'ext': 'flv',
@@ -34,7 +34,7 @@ class NRKIE(InfoExtractor):
         },
         {
             'url': 'http://www.nrk.no/video/PS*154915',
-            'md5': '0b1493ba1aae7d9579a5ad5531bc395a',
+            # MD5 is unstable
             'info_dict': {
                 'id': '154915',
                 'ext': 'flv',

From 6bdc2d5358c2843e3be4d073b2005e5196519664 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 25 Apr 2016 22:27:25 +0800
Subject: [PATCH 4736/6219] [mitele] Comment out unstable MD5

Also Akamai f4f fragments
---
 youtube_dl/extractor/mitele.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/mitele.py b/youtube_dl/extractor/mitele.py
index 7b4581dc5..3589c223d 100644
--- a/youtube_dl/extractor/mitele.py
+++ b/youtube_dl/extractor/mitele.py
@@ -15,9 +15,9 @@ class MiTeleIE(InfoExtractor):
     IE_DESC = 'mitele.es'
     _VALID_URL = r'https?://www\.mitele\.es/[^/]+/[^/]+/[^/]+/(?P<id>[^/]+)/'
 
-    _TESTS = [{
+    _TEST = {
         'url': 'http://www.mitele.es/programas-tv/diario-de/la-redaccion/programa-144/',
-        'md5': '0ff1a13aebb35d9bc14081ff633dd324',
+        # MD5 is unstable
         'info_dict': {
             'id': '0NF1jJnxS1Wu3pHrmvFyw2',
             'display_id': 'programa-144',
@@ -27,7 +27,7 @@ class MiTeleIE(InfoExtractor):
             'thumbnail': 're:(?i)^https?://.*\.jpg$',
             'duration': 2913,
         },
-    }]
+    }
 
     def _real_extract(self, url):
         display_id = self._match_id(url)

From 4645432d7a92bfb950571dde5dd690110e0f2284 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 25 Apr 2016 22:48:17 +0800
Subject: [PATCH 4737/6219] [eagleplatform] Checking direct HTTP links

Sometimes they fail with 404
---
 youtube_dl/extractor/eagleplatform.py | 7 +++++--
 youtube_dl/extractor/generic.py       | 2 ++
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/eagleplatform.py b/youtube_dl/extractor/eagleplatform.py
index 0f8c73fd7..113a4966f 100644
--- a/youtube_dl/extractor/eagleplatform.py
+++ b/youtube_dl/extractor/eagleplatform.py
@@ -23,7 +23,7 @@ class EaglePlatformIE(InfoExtractor):
     _TESTS = [{
         # http://lenta.ru/news/2015/03/06/navalny/
         'url': 'http://lentaru.media.eagleplatform.com/index/player?player=new&record_id=227304&player_template_id=5201',
-        'md5': '881ee8460e1b7735a8be938e2ffb362b',
+        # Not checking MD5 as sometimes the direct HTTP link results in 404 and HLS is used
         'info_dict': {
             'id': '227304',
             'ext': 'mp4',
@@ -109,8 +109,11 @@ class EaglePlatformIE(InfoExtractor):
             mobj = re.search('/([^/]+)/index\.m3u8', m3u8_format['url'])
             if mobj:
                 http_format = m3u8_format.copy()
+                video_url = mp4_url.replace(mp4_url_basename, mobj.group(1))
+                if not self._is_valid_url(video_url, video_id):
+                    continue
                 http_format.update({
-                    'url': mp4_url.replace(mp4_url_basename, mobj.group(1)),
+                    'url': video_url,
                     'format_id': m3u8_format['format_id'].replace('hls', 'http'),
                     'protocol': 'http',
                 })
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index c63bdbd08..a95501d86 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -887,6 +887,7 @@ class GenericIE(InfoExtractor):
         # Eagle.Platform embed (generic URL)
         {
             'url': 'http://lenta.ru/news/2015/03/06/navalny/',
+            # Not checking MD5 as sometimes the direct HTTP link results in 404 and HLS is used
             'info_dict': {
                 'id': '227304',
                 'ext': 'mp4',
@@ -901,6 +902,7 @@ class GenericIE(InfoExtractor):
         # ClipYou (Eagle.Platform) embed (custom URL)
         {
             'url': 'http://muz-tv.ru/play/7129/',
+            # Not checking MD5 as sometimes the direct HTTP link results in 404 and HLS is used
             'info_dict': {
                 'id': '12820',
                 'ext': 'mp4',

From ad58942d57996f7f43601f22c85b8c6a9afe1b09 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 25 Apr 2016 23:35:05 +0800
Subject: [PATCH 4738/6219] [muzu] Remove extractor

MUZU is shutting down in October 2015. [1]

[1] http://www.musicbusinessworldwide.com/youtube-rival-muzu-is-heading-into-liquidation/
---
 youtube_dl/extractor/extractors.py |  1 -
 youtube_dl/extractor/muzu.py       | 63 ------------------------------
 2 files changed, 64 deletions(-)
 delete mode 100644 youtube_dl/extractor/muzu.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 6de3438fc..8b215c5ab 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -439,7 +439,6 @@ from .mtv import (
 )
 from .muenchentv import MuenchenTVIE
 from .musicplayon import MusicPlayOnIE
-from .muzu import MuzuTVIE
 from .mwave import MwaveIE
 from .myspace import MySpaceIE, MySpaceAlbumIE
 from .myspass import MySpassIE
diff --git a/youtube_dl/extractor/muzu.py b/youtube_dl/extractor/muzu.py
deleted file mode 100644
index cbc800481..000000000
--- a/youtube_dl/extractor/muzu.py
+++ /dev/null
@@ -1,63 +0,0 @@
-from __future__ import unicode_literals
-
-from .common import InfoExtractor
-from ..compat import compat_urllib_parse_urlencode
-
-
-class MuzuTVIE(InfoExtractor):
-    _VALID_URL = r'https?://www\.muzu\.tv/(.+?)/(.+?)/(?P<id>\d+)'
-    IE_NAME = 'muzu.tv'
-
-    _TEST = {
-        'url': 'http://www.muzu.tv/defected/marcashken-featuring-sos-cat-walk-original-mix-music-video/1981454/',
-        'md5': '98f8b2c7bc50578d6a0364fff2bfb000',
-        'info_dict': {
-            'id': '1981454',
-            'ext': 'mp4',
-            'title': 'Cat Walk (Original Mix)',
-            'description': 'md5:90e868994de201b2570e4e5854e19420',
-            'uploader': 'MarcAshken featuring SOS',
-        },
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        info_data = compat_urllib_parse_urlencode({
-            'format': 'json',
-            'url': url,
-        })
-        info = self._download_json(
-            'http://www.muzu.tv/api/oembed/?%s' % info_data,
-            video_id, 'Downloading video info')
-
-        player_info = self._download_json(
-            'http://player.muzu.tv/player/playerInit?ai=%s' % video_id,
-            video_id, 'Downloading player info')
-        video_info = player_info['videos'][0]
-        for quality in ['1080', '720', '480', '360']:
-            if video_info.get('v%s' % quality):
-                break
-
-        data = compat_urllib_parse_urlencode({
-            'ai': video_id,
-            # Even if each time you watch a video the hash changes,
-            # it seems to work for different videos, and it will work
-            # even if you use any non empty string as a hash
-            'viewhash': 'VBNff6djeV4HV5TRPW5kOHub2k',
-            'device': 'web',
-            'qv': quality,
-        })
-        video_url_info = self._download_json(
-            'http://player.muzu.tv/player/requestVideo?%s' % data,
-            video_id, 'Downloading video url')
-        video_url = video_url_info['url']
-
-        return {
-            'id': video_id,
-            'title': info['title'],
-            'url': video_url,
-            'thumbnail': info['thumbnail_url'],
-            'description': info['description'],
-            'uploader': info['author_name'],
-        }

From e3de3d6f2f9c82683e76b6bc12697aa7264372ca Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 25 Apr 2016 23:49:12 +0800
Subject: [PATCH 4739/6219] [normalboots] Fix extraction

Now it's using ScreenwaveMedia
---
 youtube_dl/extractor/normalboots.py     | 18 +++++++++---------
 youtube_dl/extractor/screenwavemedia.py |  2 +-
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/normalboots.py b/youtube_dl/extractor/normalboots.py
index 77e091072..af44c3bb5 100644
--- a/youtube_dl/extractor/normalboots.py
+++ b/youtube_dl/extractor/normalboots.py
@@ -2,6 +2,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from .screenwavemedia import ScreenwaveMediaIE
 
 from ..utils import (
     unified_strdate,
@@ -12,7 +13,6 @@ class NormalbootsIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?normalboots\.com/video/(?P<id>[0-9a-z-]*)/?$'
     _TEST = {
         'url': 'http://normalboots.com/video/home-alone-games-jontron/',
-        'md5': '8bf6de238915dd501105b44ef5f1e0f6',
         'info_dict': {
             'id': 'home-alone-games-jontron',
             'ext': 'mp4',
@@ -22,9 +22,10 @@ class NormalbootsIE(InfoExtractor):
             'upload_date': '20140125',
         },
         'params': {
-            # rtmp download
+            # m3u8 download
             'skip_download': True,
         },
+        'add_ie': ['ScreenwaveMedia'],
     }
 
     def _real_extract(self, url):
@@ -38,16 +39,15 @@ class NormalbootsIE(InfoExtractor):
             r'<span style="text-transform:uppercase; font-size:inherit;">[A-Za-z]+, (?P<date>.*)</span>',
             webpage, 'date', fatal=False))
 
-        player_url = self._html_search_regex(
-            r'<iframe\swidth="[0-9]+"\sheight="[0-9]+"\ssrc="(?P<url>[\S]+)"',
-            webpage, 'player url')
-        player_page = self._download_webpage(player_url, video_id)
-        video_url = self._html_search_regex(
-            r"file:\s'(?P<file>[^']+\.mp4)'", player_page, 'file')
+        screenwavemedia_url = self._html_search_regex(
+            ScreenwaveMediaIE.EMBED_PATTERN, webpage, 'screenwave URL',
+            group='url')
 
         return {
+            '_type': 'url_transparent',
             'id': video_id,
-            'url': video_url,
+            'url': screenwavemedia_url,
+            'ie_key': ScreenwaveMediaIE.ie_key(),
             'title': self._og_search_title(webpage),
             'description': self._og_search_description(webpage),
             'thumbnail': self._og_search_thumbnail(webpage),
diff --git a/youtube_dl/extractor/screenwavemedia.py b/youtube_dl/extractor/screenwavemedia.py
index 44b0bbee6..40333c825 100644
--- a/youtube_dl/extractor/screenwavemedia.py
+++ b/youtube_dl/extractor/screenwavemedia.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 
 class ScreenwaveMediaIE(InfoExtractor):
-    _VALID_URL = r'https?://player\d?\.screenwavemedia\.com/(?:play/)?[a-zA-Z]+\.php\?.*\bid=(?P<id>[A-Za-z0-9-]+)'
+    _VALID_URL = r'(?:https?:)?//player\d?\.screenwavemedia\.com/(?:play/)?[a-zA-Z]+\.php\?.*\bid=(?P<id>[A-Za-z0-9-]+)'
     EMBED_PATTERN = r'src=(["\'])(?P<url>(?:https?:)?//player\d?\.screenwavemedia\.com/(?:play/)?[a-zA-Z]+\.php\?.*\bid=.+?)\1'
     _TESTS = [{
         'url': 'http://player.screenwavemedia.com/play/play.php?playerdiv=videoarea&companiondiv=squareAd&id=Cinemassacre-19911',

From 749b0046a8664d023ff622dd38844f5c8632f3f2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 25 Apr 2016 22:05:47 +0600
Subject: [PATCH 4740/6219] [ok] Allow embeds without title (Closes #9303)

---
 youtube_dl/extractor/odnoklassniki.py | 27 +++++++++++++++++++++++++--
 1 file changed, 25 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/odnoklassniki.py b/youtube_dl/extractor/odnoklassniki.py
index f9e064a60..cd614f427 100644
--- a/youtube_dl/extractor/odnoklassniki.py
+++ b/youtube_dl/extractor/odnoklassniki.py
@@ -60,6 +60,22 @@ class OdnoklassnikiIE(InfoExtractor):
             'uploader': 'Алина П',
             'age_limit': 0,
         },
+    }, {
+        # YouTube embed (metadata, provider == USER_YOUTUBE, no metadata.movie.title field)
+        'url': 'http://ok.ru/video/62036049272859-0',
+        'info_dict': {
+            'id': '62036049272859-0',
+            'ext': 'mp4',
+            'title': 'МУЗЫКА     ДОЖДЯ .',
+            'description': 'md5:6f1867132bd96e33bf53eda1091e8ed0',
+            'upload_date': '20120106',
+            'uploader_id': '473534735899',
+            'uploader': 'МARINA D',
+            'age_limit': 0,
+        },
+        'params': {
+            'skip_download': True,
+        },
     }, {
         'url': 'http://ok.ru/web-api/video/moviePlayer/20079905452',
         'only_matching': True,
@@ -106,7 +122,14 @@ class OdnoklassnikiIE(InfoExtractor):
                 video_id, 'Downloading metadata JSON')
 
         movie = metadata['movie']
-        title = movie['title']
+
+        # Some embedded videos may not contain title in movie dict (e.g.
+        # http://ok.ru/video/62036049272859-0) thus we allow missing title
+        # here and it's going to be extracted later by an extractor that
+        # will process the actual embed.
+        provider = metadata.get('provider')
+        title = movie['title'] if provider == 'UPLOADED_ODKL' else movie.get('title')
+
         thumbnail = movie.get('poster')
         duration = int_or_none(movie.get('duration'))
 
@@ -137,7 +160,7 @@ class OdnoklassnikiIE(InfoExtractor):
             'age_limit': age_limit,
         }
 
-        if metadata.get('provider') == 'USER_YOUTUBE':
+        if provider == 'USER_YOUTUBE':
             info.update({
                 '_type': 'url_transparent',
                 'url': movie['contentId'],

From c9fd5306709d0c03487a3b0163b7a33cab6774aa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 25 Apr 2016 22:15:15 +0600
Subject: [PATCH 4741/6219] [ok] Extract start time

---
 youtube_dl/extractor/odnoklassniki.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/odnoklassniki.py b/youtube_dl/extractor/odnoklassniki.py
index cd614f427..986708e75 100644
--- a/youtube_dl/extractor/odnoklassniki.py
+++ b/youtube_dl/extractor/odnoklassniki.py
@@ -2,7 +2,11 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse_unquote
+from ..compat import (
+    compat_parse_qs,
+    compat_urllib_parse_unquote,
+    compat_urllib_parse_urlparse,
+)
 from ..utils import (
     ExtractorError,
     unified_strdate,
@@ -32,7 +36,7 @@ class OdnoklassnikiIE(InfoExtractor):
         'skip': 'Video has been blocked',
     }, {
         # metadataUrl
-        'url': 'http://ok.ru/video/63567059965189-0',
+        'url': 'http://ok.ru/video/63567059965189-0?fromTime=5',
         'md5': '9676cf86eff5391d35dea675d224e131',
         'info_dict': {
             'id': '63567059965189-0',
@@ -44,6 +48,7 @@ class OdnoklassnikiIE(InfoExtractor):
             'uploader': '☭ Андрей Мещанинов ☭',
             'like_count': int,
             'age_limit': 0,
+            'start_time': 5,
         },
     }, {
         # YouTube embed (metadataUrl, provider == USER_YOUTUBE)
@@ -94,6 +99,9 @@ class OdnoklassnikiIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
+        start_time = int_or_none(compat_parse_qs(
+            compat_urllib_parse_urlparse(url).query).get('fromTime', [None])[0])
+
         video_id = self._match_id(url)
 
         webpage = self._download_webpage(
@@ -158,6 +166,7 @@ class OdnoklassnikiIE(InfoExtractor):
             'uploader_id': uploader_id,
             'like_count': like_count,
             'age_limit': age_limit,
+            'start_time': start_time,
         }
 
         if provider == 'USER_YOUTUBE':

From f1f879098a38c786d78927df8915b547f7ac3569 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 26 Apr 2016 13:39:53 +0100
Subject: [PATCH 4742/6219] [viewster] extract more metadata for http formats

---
 youtube_dl/extractor/viewster.py | 37 ++++++++++++++++++++++----------
 1 file changed, 26 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/viewster.py b/youtube_dl/extractor/viewster.py
index 7839225d4..6edc2c44e 100644
--- a/youtube_dl/extractor/viewster.py
+++ b/youtube_dl/extractor/viewster.py
@@ -118,6 +118,7 @@ class ViewsterIE(InfoExtractor):
 
         formats = []
         manifest_url = None
+        m3u8_formats = []
         for media_type in ('application/f4m+xml', 'application/x-mpegURL', 'video/mp4'):
             media = self._download_json(
                 'https://public-api.viewster.com/movies/%s/video?mediaType=%s'
@@ -154,18 +155,32 @@ class ViewsterIE(InfoExtractor):
                     'qualities', default=None)
                 if not qualities:
                     continue
-                qualities = qualities.strip(',').split(',')
-                http_template = re.sub(QUALITIES_RE, r'%s', qualities_basename)
+                qualities = list(map(lambda q: int(q[:-1]), qualities.strip(',').split(',')))
+                qualities.sort()
+                http_template = re.sub(QUALITIES_RE, r'%dk', qualities_basename)
                 http_url_basename = url_basename(video_url)
-                for q in qualities:
-                    tbr = int_or_none(self._search_regex(
-                        r'(\d+)k', q, 'bitrate', default=None))
-                    formats.append({
-                        'url': video_url.replace(http_url_basename, http_template % q),
-                        'ext': 'mp4',
-                        'format_id': 'http' + ('-%d' % tbr if tbr else ''),
-                        'tbr': tbr,
-                    })
+                if m3u8_formats:
+                    self._sort_formats(m3u8_formats)
+                    m3u8_formats = list(filter(
+                        lambda f: f.get('vcodec') != 'none' and f.get('resolution') != 'multiple',
+                        m3u8_formats))
+                if len(qualities) == len(m3u8_formats):
+                    for q, m3u8_format in zip(qualities, m3u8_formats):
+                        f = m3u8_format.copy()
+                        f.update({
+                            'url': video_url.replace(http_url_basename, http_template % q),
+                            'format_id': f['format_id'].replace('hls', 'http'),
+                            'protocol': 'http',
+                        })
+                        formats.append(f)
+                else:
+                    for q in qualities:
+                        formats.append({
+                            'url': video_url.replace(http_url_basename, http_template % q),
+                            'ext': 'mp4',
+                            'format_id': 'http-%d' % q,
+                            'tbr': q,
+                        })
 
         if not formats and not info.get('LanguageSets') and not info.get('VODSettings'):
             self.raise_geo_restricted()

From 175c2e9ec326f9ef820413837608eb4f5c8c5961 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 26 Apr 2016 22:29:29 +0600
Subject: [PATCH 4743/6219] [youtube:search_url] Reimplement in terms of
 youtube:playlistbase

---
 youtube_dl/extractor/youtube.py | 29 +++--------------------------
 1 file changed, 3 insertions(+), 26 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 44f98d294..b7c3cb63f 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -2139,10 +2139,11 @@ class YoutubeSearchDateIE(YoutubeSearchIE):
     _EXTRA_QUERY_ARGS = {'search_sort': 'video_date_uploaded'}
 
 
-class YoutubeSearchURLIE(InfoExtractor):
+class YoutubeSearchURLIE(YoutubePlaylistBaseInfoExtractor):
     IE_DESC = 'YouTube.com search URLs'
     IE_NAME = 'youtube:search_url'
     _VALID_URL = r'https?://(?:www\.)?youtube\.com/results\?(.*?&)?(?:search_query|q)=(?P<query>[^&]+)(?:[&]|$)'
+    _VIDEO_RE = r'href="\s*/watch\?v=(?P<id>[0-9A-Za-z_-]{11})(?:[^"]*"[^>]+\btitle="(?P<title>[^"]+))?'
     _TESTS = [{
         'url': 'https://www.youtube.com/results?baz=bar&search_query=youtube-dl+test+video&filters=video&lclk=video',
         'playlist_mincount': 5,
@@ -2157,32 +2158,8 @@ class YoutubeSearchURLIE(InfoExtractor):
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         query = compat_urllib_parse_unquote_plus(mobj.group('query'))
-
         webpage = self._download_webpage(url, query)
-        result_code = self._search_regex(
-            r'(?s)<ol[^>]+class="item-section"(.*?)</ol>', webpage, 'result HTML')
-
-        part_codes = re.findall(
-            r'(?s)<h3[^>]+class="[^"]*yt-lockup-title[^"]*"[^>]*>(.*?)</h3>', result_code)
-        entries = []
-        for part_code in part_codes:
-            part_title = self._html_search_regex(
-                [r'(?s)title="([^"]+)"', r'>([^<]+)</a>'], part_code, 'item title', fatal=False)
-            part_url_snippet = self._html_search_regex(
-                r'(?s)href="([^"]+)"', part_code, 'item URL')
-            part_url = compat_urlparse.urljoin(
-                'https://www.youtube.com/', part_url_snippet)
-            entries.append({
-                '_type': 'url',
-                'url': part_url,
-                'title': part_title,
-            })
-
-        return {
-            '_type': 'playlist',
-            'entries': entries,
-            'title': query,
-        }
+        return self.playlist_result(self._process_page(webpage), playlist_title=query)
 
 
 class YoutubeShowIE(YoutubePlaylistsBaseInfoExtractor):

From 7464360379a1a3fc6ba3228f54dd4853df349142 Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Wed, 27 Apr 2016 00:16:48 +0600
Subject: [PATCH 4744/6219] [README.md] Add FAQ entry on output template
 conflicts

---
 README.md | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/README.md b/README.md
index e062444b3..fb57b0323 100644
--- a/README.md
+++ b/README.md
@@ -697,6 +697,10 @@ YouTube changed their playlist format in March 2014 and later on, so you'll need
 
 If you have installed youtube-dl with a package manager, pip, setup.py or a tarball, please use that to update. Note that Ubuntu packages do not seem to get updated anymore. Since we are not affiliated with Ubuntu, there is little we can do. Feel free to [report bugs](https://bugs.launchpad.net/ubuntu/+source/youtube-dl/+filebug) to the [Ubuntu packaging guys](mailto:ubuntu-motu@lists.ubuntu.com?subject=outdated%20version%20of%20youtube-dl) - all they have to do is update the package to a somewhat recent version. See above for a way to update.
 
+### I'm getting an error when trying to use output template: `error: using output template conflicts with using title, video ID or auto number`
+
+Make sure you are not using `-o` with any of these options `-t`, `--title`, `--id`, `-A` or `--auto-number` set in command line or in a configuration file. Remove the latter if any.
+
 ### Do I always have to pass `-citw`?
 
 By default, youtube-dl intends to have the best options (incidentally, if you have a convincing case that these should be different, [please file an issue where you explain that](https://yt-dl.org/bug)). Therefore, it is unnecessary and sometimes harmful to copy long option strings from webpages. In particular, the only option out of `-citw` that is regularly useful is `-i`.

From 046ea04a7d8601a85007430a7a3da3ce236549f7 Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Wed, 27 Apr 2016 00:22:08 +0600
Subject: [PATCH 4745/6219] [README.md] Mention mpv

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index fb57b0323..ecf737047 100644
--- a/README.md
+++ b/README.md
@@ -721,7 +721,7 @@ Videos or video formats streamed via RTMP protocol can only be downloaded when [
 
 ### I have downloaded a video but how can I play it?
 
-Once the video is fully downloaded, use any video player, such as [vlc](http://www.videolan.org) or [mplayer](http://www.mplayerhq.hu/).
+Once the video is fully downloaded, use any video player, such as [mpv](https://mpv.io/), [vlc](http://www.videolan.org) or [mplayer](http://www.mplayerhq.hu/).
 
 ### I extracted a video URL with `-g`, but it does not play on another machine / in my webbrowser.
 

From a7e03861e8d0ce18ad698e0e38ffac40a09cef8b Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 27 Apr 2016 13:52:04 +0800
Subject: [PATCH 4746/6219] [scivee] Skip the test

Not accessible from either Travis CI or my machine.

Closes #9315
---
 youtube_dl/extractor/scivee.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/scivee.py b/youtube_dl/extractor/scivee.py
index 3bf93c870..b1ca12fde 100644
--- a/youtube_dl/extractor/scivee.py
+++ b/youtube_dl/extractor/scivee.py
@@ -18,6 +18,7 @@ class SciVeeIE(InfoExtractor):
             'title': 'Adam Arkin at the 2014 DOE JGI Genomics of Energy & Environment Meeting',
             'description': 'md5:81f1710638e11a481358fab1b11059d7',
         },
+        'skip': 'Not accessible from Travis CI server',
     }
 
     def _real_extract(self, url):

From 2ac2cbc0a351785e0c6d034bd1bab77973ec7a41 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 27 Apr 2016 13:55:32 +0800
Subject: [PATCH 4747/6219] [malemotion] Remove the extractor

Announcement from their homepage:

```
MaleMotion is closed

After another system crash, I'm forced to close the site

This week all content will be erased

Don't forget to cancel your subscription if any !
```

Closes #9311.
---
 youtube_dl/extractor/extractors.py |  1 -
 youtube_dl/extractor/malemotion.py | 46 ------------------------------
 2 files changed, 47 deletions(-)
 delete mode 100644 youtube_dl/extractor/malemotion.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 8b215c5ab..00f8a7a85 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -400,7 +400,6 @@ from .macgamestore import MacGameStoreIE
 from .mailru import MailRuIE
 from .makerschannel import MakersChannelIE
 from .makertv import MakerTVIE
-from .malemotion import MalemotionIE
 from .matchtv import MatchTVIE
 from .mdr import MDRIE
 from .metacafe import MetacafeIE
diff --git a/youtube_dl/extractor/malemotion.py b/youtube_dl/extractor/malemotion.py
deleted file mode 100644
index 92511a671..000000000
--- a/youtube_dl/extractor/malemotion.py
+++ /dev/null
@@ -1,46 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-from .common import InfoExtractor
-from ..compat import compat_urllib_parse_unquote
-
-
-class MalemotionIE(InfoExtractor):
-    _VALID_URL = r'https?://malemotion\.com/video/(.+?)\.(?P<id>.+?)(#|$)'
-    _TEST = {
-        'url': 'http://malemotion.com/video/bete-de-concours.ltc',
-        'md5': '3013e53a0afbde2878bc39998c33e8a5',
-        'info_dict': {
-            'id': 'ltc',
-            'ext': 'mp4',
-            'title': 'Bête de Concours',
-            'age_limit': 18,
-        },
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-
-        video_url = compat_urllib_parse_unquote(self._search_regex(
-            r'<source type="video/mp4" src="(.+?)"', webpage, 'video URL'))
-        video_title = self._html_search_regex(
-            r'<title>(.*?)</title', webpage, 'title')
-        video_thumbnail = self._search_regex(
-            r'<video .+?poster="(.+?)"', webpage, 'thumbnail', fatal=False)
-
-        formats = [{
-            'url': video_url,
-            'ext': 'mp4',
-            'format_id': 'mp4',
-            'preference': 1,
-        }]
-        self._sort_formats(formats)
-
-        return {
-            'id': video_id,
-            'formats': formats,
-            'title': video_title,
-            'thumbnail': video_thumbnail,
-            'age_limit': 18,
-        }

From 5b5d7cc11e3037408aeedf8d6dc57ac228b02496 Mon Sep 17 00:00:00 2001
From: Peter Rowlands <peter@pmrowla.com>
Date: Wed, 27 Apr 2016 15:57:17 +0900
Subject: [PATCH 4748/6219] [mwave] Add Mwave Meet & Greet extractor

---
 youtube_dl/extractor/extractors.py |  2 +-
 youtube_dl/extractor/mwave.py      | 23 +++++++++++++++++++++++
 2 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 8b215c5ab..9d1992721 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -439,7 +439,7 @@ from .mtv import (
 )
 from .muenchentv import MuenchenTVIE
 from .musicplayon import MusicPlayOnIE
-from .mwave import MwaveIE
+from .mwave import MwaveIE, MwaveMeetGreetIE
 from .myspace import MySpaceIE, MySpaceAlbumIE
 from .myspass import MySpassIE
 from .myvi import MyviIE
diff --git a/youtube_dl/extractor/mwave.py b/youtube_dl/extractor/mwave.py
index 5c3c8d464..6485c6928 100644
--- a/youtube_dl/extractor/mwave.py
+++ b/youtube_dl/extractor/mwave.py
@@ -56,3 +56,26 @@ class MwaveIE(InfoExtractor):
             'view_count': int_or_none(vod_info.get('hit')),
             'formats': formats,
         }
+
+
+class MwaveMeetGreetIE(InfoExtractor):
+    _VALID_URL = r'https?://mwave\.interest\.me/meetgreet/view/(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'http://mwave.interest.me/meetgreet/view/256',
+        'info_dict': {
+            'id': '173294',
+            'ext': 'flv',
+            'title': '[MEET&GREET] Park BoRam',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'uploader': 'Mwave',
+            'duration': 3634,
+            'view_count': int,
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        clip_id = self._html_search_regex(r'<iframe src="/mnettv/ifr_clip\.m\?searchVideoDetailVO\.clip_id=(?P<id>[0-9]+)', webpage, 'clip ID')
+        clip_url = 'http://mwave.interest.me/mnettv/videodetail.m?searchVideoDetailVO.clip_id={0}'.format(clip_id)
+        return self.url_result(clip_url, 'Mwave', clip_id)

From dcf094d62699f8ad06ceaf3fba55e453980fac91 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 27 Apr 2016 18:08:23 +0800
Subject: [PATCH 4749/6219] [theplatform] Fix for Python 3.2

test_AENetworks{,_1} fails as in Python < 3.3, binascii.a2b_* functions
accepts only bytes-like objects
---
 youtube_dl/extractor/theplatform.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index 8272dd969..a25417f94 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -159,11 +159,11 @@ class ThePlatformIE(ThePlatformBaseIE):
         def str_to_hex(str):
             return binascii.b2a_hex(str.encode('ascii')).decode('ascii')
 
-        def hex_to_str(hex):
-            return binascii.a2b_hex(hex)
+        def hex_to_bytes(hex):
+            return binascii.a2b_hex(hex.encode('ascii'))
 
         relative_path = re.match(r'https?://link.theplatform.com/s/([^?]+)', url).group(1)
-        clear_text = hex_to_str(flags + expiration_date + str_to_hex(relative_path))
+        clear_text = hex_to_bytes(flags + expiration_date + str_to_hex(relative_path))
         checksum = hmac.new(sig_key.encode('ascii'), clear_text, hashlib.sha1).hexdigest()
         sig = flags + expiration_date + checksum + str_to_hex(sig_secret)
         return '%s&sig=%s' % (url, sig)

From 3cc8649c9d42bab8c7b665115ebdc569bf44a762 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 28 Apr 2016 02:58:11 +0800
Subject: [PATCH 4750/6219] [20min] Detect embedded YouTube videos

Fixes #9331
---
 youtube_dl/extractor/twentymin.py | 23 ++++++++++++++++++++++-
 1 file changed, 22 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/twentymin.py b/youtube_dl/extractor/twentymin.py
index ca7d953b8..b721ecb0a 100644
--- a/youtube_dl/extractor/twentymin.py
+++ b/youtube_dl/extractor/twentymin.py
@@ -32,7 +32,22 @@ class TwentyMinutenIE(InfoExtractor):
             'title': '«Wir müssen mutig nach vorne schauen»',
             'description': 'Kein Land sei innovativer als die Schweiz, sagte Johann Schneider-Ammann in seiner Neujahrsansprache. Das Land müsse aber seine Hausaufgaben machen.',
             'thumbnail': 'http://www.20min.ch/images/content/2/2/0/22050469/10/teaserbreit.jpg'
-        }
+        },
+        'skip': '"This video is no longer available" is shown both on the web page and in the downloaded file.',
+    }, {
+        # YouTube embed
+        'url': 'http://www.20min.ch/ro/sports/football/story/Il-marque-une-bicyclette-de-plus-de-30-metres--21115184',
+        'md5': 'cec64d59aa01c0ed9dbba9cf639dd82f',
+        'info_dict': {
+            'id': 'ivM7A7SpDOs',
+            'ext': 'mp4',
+            'title': 'GOLAZO DE CHILENA DE JAVI GÓMEZ, FINALISTA AL BALÓN DE CLM 2016',
+            'description': 'md5:903c92fbf2b2f66c09de514bc25e9f5a',
+            'upload_date': '20160424',
+            'uploader': 'RTVCM Castilla-La Mancha',
+            'uploader_id': 'RTVCM',
+        },
+        'add_ie': ['Youtube'],
     }, {
         'url': 'http://www.20min.ch/videotv/?cid=44&vid=468738',
         'only_matching': True,
@@ -48,6 +63,12 @@ class TwentyMinutenIE(InfoExtractor):
 
         webpage = self._download_webpage(url, display_id)
 
+        youtube_url = self._html_search_regex(
+            r'<iframe[^>]+src="((?:https?:)?//www\.youtube\.com/embed/[^"]+)"',
+            webpage, 'YouTube embed URL', default=None)
+        if youtube_url is not None:
+            return self.url_result(youtube_url, 'Youtube')
+
         title = self._html_search_regex(
             r'<h1>.*?<span>(.+?)</span></h1>',
             webpage, 'title', default=None)

From 52af8f222bc4f067b4c5e7a977a64345d35ae4fe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 28 Apr 2016 04:01:21 +0600
Subject: [PATCH 4751/6219] [cwtv] Relax _VALID_URL (Closes #9327)

---
 youtube_dl/extractor/cwtv.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/cwtv.py b/youtube_dl/extractor/cwtv.py
index f5cefd966..ebd14cb16 100644
--- a/youtube_dl/extractor/cwtv.py
+++ b/youtube_dl/extractor/cwtv.py
@@ -9,7 +9,7 @@ from ..utils import (
 
 
 class CWTVIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?cw(?:tv|seed)\.com/shows/(?:[^/]+/){2}\?play=(?P<id>[a-z0-9]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12})'
+    _VALID_URL = r'https?://(?:www\.)?cw(?:tv|seed)\.com/(?:shows/)?(?:[^/]+/){2}\?.*\bplay=(?P<id>[a-z0-9]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12})'
     _TESTS = [{
         'url': 'http://cwtv.com/shows/arrow/legends-of-yesterday/?play=6b15e985-9345-4f60-baf8-56e96be57c63',
         'info_dict': {
@@ -48,6 +48,9 @@ class CWTVIE(InfoExtractor):
             # m3u8 download
             'skip_download': True,
         }
+    }, {
+        'url': 'http://cwtv.com/thecw/chroniclesofcisco/?play=8adebe35-f447-465f-ab52-e863506ff6d6',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 618c71dc64086f751b6ae87d5f32687e02a54e58 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 28 Apr 2016 15:00:02 +0800
Subject: [PATCH 4752/6219] [cloudy] New domain name for the test_cloudy_1

I'm sure whether videoraj.ch still works or not, so keep it.
---
 youtube_dl/extractor/cloudy.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/cloudy.py b/youtube_dl/extractor/cloudy.py
index 9e267e6c0..9a28ef354 100644
--- a/youtube_dl/extractor/cloudy.py
+++ b/youtube_dl/extractor/cloudy.py
@@ -19,7 +19,7 @@ from ..utils import (
 class CloudyIE(InfoExtractor):
     _IE_DESC = 'cloudy.ec and videoraj.ch'
     _VALID_URL = r'''(?x)
-        https?://(?:www\.)?(?P<host>cloudy\.ec|videoraj\.ch)/
+        https?://(?:www\.)?(?P<host>cloudy\.ec|videoraj\.(?:ch|to))/
         (?:v/|embed\.php\?id=)
         (?P<id>[A-Za-z0-9]+)
         '''
@@ -37,7 +37,7 @@ class CloudyIE(InfoExtractor):
             }
         },
         {
-            'url': 'http://www.videoraj.ch/v/47f399fd8bb60',
+            'url': 'http://www.videoraj.to/v/47f399fd8bb60',
             'md5': '7d0f8799d91efd4eda26587421c3c3b0',
             'info_dict': {
                 'id': '47f399fd8bb60',

From a5941305b6ba0921ea4f34641dd9095372dd1c1d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 28 Apr 2016 16:03:08 +0800
Subject: [PATCH 4753/6219] [mwave] Coding style

---
 youtube_dl/extractor/mwave.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/mwave.py b/youtube_dl/extractor/mwave.py
index 6485c6928..a103e0323 100644
--- a/youtube_dl/extractor/mwave.py
+++ b/youtube_dl/extractor/mwave.py
@@ -10,6 +10,7 @@ from ..utils import (
 
 class MwaveIE(InfoExtractor):
     _VALID_URL = r'https?://mwave\.interest\.me/mnettv/videodetail\.m\?searchVideoDetailVO\.clip_id=(?P<id>[0-9]+)'
+    _URL_TEMPLATE = 'http://mwave.interest.me/mnettv/videodetail.m?searchVideoDetailVO.clip_id=%s'
     _TEST = {
         'url': 'http://mwave.interest.me/mnettv/videodetail.m?searchVideoDetailVO.clip_id=168859',
         # md5 is unstable
@@ -59,7 +60,7 @@ class MwaveIE(InfoExtractor):
 
 
 class MwaveMeetGreetIE(InfoExtractor):
-    _VALID_URL = r'https?://mwave\.interest\.me/meetgreet/view/(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://mwave\.interest\.me/meetgreet/view/(?P<id>\d+)'
     _TEST = {
         'url': 'http://mwave.interest.me/meetgreet/view/256',
         'info_dict': {
@@ -76,6 +77,8 @@ class MwaveMeetGreetIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
-        clip_id = self._html_search_regex(r'<iframe src="/mnettv/ifr_clip\.m\?searchVideoDetailVO\.clip_id=(?P<id>[0-9]+)', webpage, 'clip ID')
-        clip_url = 'http://mwave.interest.me/mnettv/videodetail.m?searchVideoDetailVO.clip_id={0}'.format(clip_id)
+        clip_id = self._html_search_regex(
+            r'<iframe[^>]+src="/mnettv/ifr_clip\.m\?searchVideoDetailVO\.clip_id=(\d+)',
+            webpage, 'clip ID')
+        clip_url = MwaveIE._URL_TEMPLATE % clip_id
         return self.url_result(clip_url, 'Mwave', clip_id)

From 7f776fa4b510b7973e08f06de556fa39cb5946e5 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 28 Apr 2016 17:08:41 +0800
Subject: [PATCH 4754/6219] [yandexmusic] Skip tests as Travis CI blocked

---
 youtube_dl/extractor/yandexmusic.py | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/yandexmusic.py b/youtube_dl/extractor/yandexmusic.py
index 7a90cc60c..0d32a612f 100644
--- a/youtube_dl/extractor/yandexmusic.py
+++ b/youtube_dl/extractor/yandexmusic.py
@@ -22,6 +22,12 @@ class YandexMusicBaseIE(InfoExtractor):
         if error:
             raise ExtractorError(error, expected=True)
 
+    def _download_webpage(self, *args, **kwargs):
+        webpage = super(YandexMusicBaseIE, self)._download_webpage(*args, **kwargs)
+        if 'Нам очень жаль, но&nbsp;запросы, поступившие с&nbsp;вашего IP-адреса, похожи на&nbsp;автоматические.' in webpage:
+            raise ExtractorError('Blocked by YandexMusic', expected=True)
+        return webpage
+
     def _download_json(self, *args, **kwargs):
         response = super(YandexMusicBaseIE, self)._download_json(*args, **kwargs)
         self._handle_error(response)
@@ -47,7 +53,8 @@ class YandexMusicTrackIE(YandexMusicBaseIE):
             'album_artist': 'Carlo Ambrosio',
             'artist': 'Carlo Ambrosio & Fabio Di Bari, Carlo Ambrosio',
             'release_year': '2009',
-        }
+        },
+        'skip': 'Travis CI servers blocked by YandexMusic',
     }
 
     def _get_track_url(self, storage_dir, track_id):
@@ -139,6 +146,7 @@ class YandexMusicAlbumIE(YandexMusicPlaylistBaseIE):
             'title': 'Carlo Ambrosio - Gypsy Soul (2009)',
         },
         'playlist_count': 50,
+        'skip': 'Travis CI servers blocked by YandexMusic',
     }
 
     def _real_extract(self, url):
@@ -171,6 +179,7 @@ class YandexMusicPlaylistIE(YandexMusicPlaylistBaseIE):
             'description': 'md5:3b9f27b0efbe53f2ee1e844d07155cc9',
         },
         'playlist_count': 6,
+        'skip': 'Travis CI servers blocked by YandexMusic',
     }, {
         # playlist exceeding the limit of 150 tracks shipped with webpage (see
         # https://github.com/rg3/youtube-dl/issues/6666)
@@ -180,6 +189,7 @@ class YandexMusicPlaylistIE(YandexMusicPlaylistBaseIE):
             'title': 'Музыка 90-х',
         },
         'playlist_count': 310,
+        'skip': 'Travis CI servers blocked by YandexMusic',
     }]
 
     def _real_extract(self, url):

From 0cbcbdd89dbc3573ecfcf68496c54bd84804967d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 28 Apr 2016 17:51:20 +0800
Subject: [PATCH 4755/6219] [nuvid] Fix extraction

Closes #7620
---
 youtube_dl/extractor/nuvid.py | 44 +++++++++++++++++------------------
 1 file changed, 21 insertions(+), 23 deletions(-)

diff --git a/youtube_dl/extractor/nuvid.py b/youtube_dl/extractor/nuvid.py
index 9fa7cefad..ab6bfcd7f 100644
--- a/youtube_dl/extractor/nuvid.py
+++ b/youtube_dl/extractor/nuvid.py
@@ -5,8 +5,6 @@ import re
 from .common import InfoExtractor
 from ..utils import (
     parse_duration,
-    sanitized_Request,
-    unified_strdate,
 )
 
 
@@ -20,7 +18,6 @@ class NuvidIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Horny babes show their awesome bodeis and',
             'duration': 129,
-            'upload_date': '20140508',
             'age_limit': 18,
         }
     }
@@ -28,28 +25,31 @@ class NuvidIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        formats = []
+        page_url = 'http://m.nuvid.com/video/%s' % video_id
+        webpage = self._download_webpage(
+            page_url, video_id, 'Downloading video page')
+        # When dwnld_speed exists and has a value larger than the MP4 file's
+        # bitrate, Nuvid returns the MP4 URL
+        # It's unit is 100bytes/millisecond, see mobile-nuvid-min.js for the algorithm
+        self._set_cookie('nuvid.com', 'dwnld_speed', '10.0')
+        mp4_webpage = self._download_webpage(
+            page_url, video_id, 'Downloading video page for MP4 format')
 
-        for dwnld_speed, format_id in [(0, '3gp'), (5, 'mp4')]:
-            request = sanitized_Request(
-                'http://m.nuvid.com/play/%s' % video_id)
-            request.add_header('Cookie', 'skip_download_page=1; dwnld_speed=%d; adv_show=1' % dwnld_speed)
-            webpage = self._download_webpage(
-                request, video_id, 'Downloading %s page' % format_id)
-            video_url = self._html_search_regex(
-                r'<a\s+href="([^"]+)"\s+class="b_link">', webpage, '%s video URL' % format_id, fatal=False)
-            if not video_url:
-                continue
+        html5_video_re = r'(?s)<(?:video|audio)[^<]*(?:>.*?<source[^>]*)?\s+src=["\'](.*?)["\']',
+        video_url = self._html_search_regex(html5_video_re, webpage, video_id)
+        mp4_video_url = self._html_search_regex(html5_video_re, mp4_webpage, video_id)
+        formats = [{
+            'url': video_url,
+        }]
+        if mp4_video_url != video_url:
             formats.append({
-                'url': video_url,
-                'format_id': format_id,
+                'url': mp4_video_url,
             })
 
-        webpage = self._download_webpage(
-            'http://m.nuvid.com/video/%s' % video_id, video_id, 'Downloading video page')
         title = self._html_search_regex(
             [r'<span title="([^"]+)">',
-             r'<div class="thumb-holder video">\s*<h5[^>]*>([^<]+)</h5>'], webpage, 'title').strip()
+             r'<div class="thumb-holder video">\s*<h5[^>]*>([^<]+)</h5>',
+             r'<span[^>]+class="title_thumb">([^<]+)</span>'], webpage, 'title').strip()
         thumbnails = [
             {
                 'url': thumb_url,
@@ -57,9 +57,8 @@ class NuvidIE(InfoExtractor):
         ]
         thumbnail = thumbnails[0]['url'] if thumbnails else None
         duration = parse_duration(self._html_search_regex(
-            r'<i class="fa fa-clock-o"></i>\s*(\d{2}:\d{2})', webpage, 'duration', fatal=False))
-        upload_date = unified_strdate(self._html_search_regex(
-            r'<i class="fa fa-user"></i>\s*(\d{4}-\d{2}-\d{2})', webpage, 'upload date', fatal=False))
+            [r'<i class="fa fa-clock-o"></i>\s*(\d{2}:\d{2})',
+             r'<span[^>]+class="view_time">([^<]+)</span>'], webpage, 'duration', fatal=False))
 
         return {
             'id': video_id,
@@ -67,7 +66,6 @@ class NuvidIE(InfoExtractor):
             'thumbnails': thumbnails,
             'thumbnail': thumbnail,
             'duration': duration,
-            'upload_date': upload_date,
             'age_limit': 18,
             'formats': formats,
         }

From eebe6b382eb6bd9e8118b616f3dde48c294e3b0d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 28 Apr 2016 21:37:34 +0600
Subject: [PATCH 4756/6219] [yandexmusic] Improve error handling

---
 youtube_dl/extractor/yandexmusic.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/yandexmusic.py b/youtube_dl/extractor/yandexmusic.py
index 0d32a612f..b0e68a087 100644
--- a/youtube_dl/extractor/yandexmusic.py
+++ b/youtube_dl/extractor/yandexmusic.py
@@ -18,9 +18,10 @@ from ..utils import (
 class YandexMusicBaseIE(InfoExtractor):
     @staticmethod
     def _handle_error(response):
-        error = response.get('error')
-        if error:
-            raise ExtractorError(error, expected=True)
+        if isinstance(response, dict):
+            error = response.get('error')
+            if error:
+                raise ExtractorError(error, expected=True)
 
     def _download_webpage(self, *args, **kwargs):
         webpage = super(YandexMusicBaseIE, self)._download_webpage(*args, **kwargs)

From 4b537629143c8f51c5814c650227971c438b12e1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 28 Apr 2016 21:45:33 +0600
Subject: [PATCH 4757/6219] [yandexmusic] Clarify blockage

---
 youtube_dl/extractor/yandexmusic.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/yandexmusic.py b/youtube_dl/extractor/yandexmusic.py
index b0e68a087..a33fe3d83 100644
--- a/youtube_dl/extractor/yandexmusic.py
+++ b/youtube_dl/extractor/yandexmusic.py
@@ -26,7 +26,11 @@ class YandexMusicBaseIE(InfoExtractor):
     def _download_webpage(self, *args, **kwargs):
         webpage = super(YandexMusicBaseIE, self)._download_webpage(*args, **kwargs)
         if 'Нам очень жаль, но&nbsp;запросы, поступившие с&nbsp;вашего IP-адреса, похожи на&nbsp;автоматические.' in webpage:
-            raise ExtractorError('Blocked by YandexMusic', expected=True)
+            raise ExtractorError(
+                'YandexMusic asks you to solve a CAPTCHA: go to '
+                'https://music.yandex.ru/ and solve it, then export '
+                'cookies and pass cookie file to youtube-dl with --cookies',
+                expected=True)
         return webpage
 
     def _download_json(self, *args, **kwargs):

From 0ba9e3ca2233d018d695bac4eebe0e34043a7ec9 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 28 Apr 2016 17:44:33 +0100
Subject: [PATCH 4758/6219] [viewster] extract formats for videos with multiple
 audios/subtitles

---
 youtube_dl/extractor/viewster.py | 147 +++++++++++++++++--------------
 1 file changed, 80 insertions(+), 67 deletions(-)

diff --git a/youtube_dl/extractor/viewster.py b/youtube_dl/extractor/viewster.py
index 6edc2c44e..1813b81d6 100644
--- a/youtube_dl/extractor/viewster.py
+++ b/youtube_dl/extractor/viewster.py
@@ -78,11 +78,11 @@ class ViewsterIE(InfoExtractor):
 
     _ACCEPT_HEADER = 'application/json, text/javascript, */*; q=0.01'
 
-    def _download_json(self, url, video_id, note='Downloading JSON metadata', fatal=True):
+    def _download_json(self, url, video_id, note='Downloading JSON metadata', fatal=True, query={}):
         request = sanitized_Request(url)
         request.add_header('Accept', self._ACCEPT_HEADER)
         request.add_header('Auth-token', self._AUTH_TOKEN)
-        return super(ViewsterIE, self)._download_json(request, video_id, note, fatal=fatal)
+        return super(ViewsterIE, self)._download_json(request, video_id, note, fatal=fatal, query=query)
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -117,72 +117,85 @@ class ViewsterIE(InfoExtractor):
             return self.playlist_result(entries, video_id, title, description)
 
         formats = []
-        manifest_url = None
-        m3u8_formats = []
-        for media_type in ('application/f4m+xml', 'application/x-mpegURL', 'video/mp4'):
-            media = self._download_json(
-                'https://public-api.viewster.com/movies/%s/video?mediaType=%s'
-                % (entry_id, compat_urllib_parse.quote(media_type)),
-                video_id, 'Downloading %s JSON' % media_type, fatal=False)
-            if not media:
-                continue
-            video_url = media.get('Uri')
-            if not video_url:
-                continue
-            ext = determine_ext(video_url)
-            if ext == 'f4m':
-                manifest_url = video_url
-                video_url += '&' if '?' in video_url else '?'
-                video_url += 'hdcore=3.2.0&plugin=flowplayer-3.2.0.1'
-                formats.extend(self._extract_f4m_formats(
-                    video_url, video_id, f4m_id='hds'))
-            elif ext == 'm3u8':
-                manifest_url = video_url
-                m3u8_formats = self._extract_m3u8_formats(
-                    video_url, video_id, 'mp4', m3u8_id='hls',
-                    fatal=False)  # m3u8 sometimes fail
-                if m3u8_formats:
-                    formats.extend(m3u8_formats)
-            else:
-                qualities_basename = self._search_regex(
-                    '/([^/]+)\.csmil/',
-                    manifest_url, 'qualities basename', default=None)
-                if not qualities_basename:
-                    continue
-                QUALITIES_RE = r'((,\d+k)+,?)'
-                qualities = self._search_regex(
-                    QUALITIES_RE, qualities_basename,
-                    'qualities', default=None)
-                if not qualities:
-                    continue
-                qualities = list(map(lambda q: int(q[:-1]), qualities.strip(',').split(',')))
-                qualities.sort()
-                http_template = re.sub(QUALITIES_RE, r'%dk', qualities_basename)
-                http_url_basename = url_basename(video_url)
-                if m3u8_formats:
-                    self._sort_formats(m3u8_formats)
-                    m3u8_formats = list(filter(
-                        lambda f: f.get('vcodec') != 'none' and f.get('resolution') != 'multiple',
-                        m3u8_formats))
-                if len(qualities) == len(m3u8_formats):
-                    for q, m3u8_format in zip(qualities, m3u8_formats):
-                        f = m3u8_format.copy()
-                        f.update({
-                            'url': video_url.replace(http_url_basename, http_template % q),
-                            'format_id': f['format_id'].replace('hls', 'http'),
-                            'protocol': 'http',
-                        })
-                        formats.append(f)
-                else:
-                    for q in qualities:
-                        formats.append({
-                            'url': video_url.replace(http_url_basename, http_template % q),
-                            'ext': 'mp4',
-                            'format_id': 'http-%d' % q,
-                            'tbr': q,
-                        })
+        for language_set in info.get('LanguageSets', []):
+            manifest_url = None
+            m3u8_formats = []
+            audio = language_set.get('Audio') or ''
+            subtitle = language_set.get('Subtitle') or ''
+            base_format_id = audio
+            if subtitle:
+                base_format_id += '-%s' % subtitle
 
-        if not formats and not info.get('LanguageSets') and not info.get('VODSettings'):
+            def concat(suffix, sep='-'):
+                return (base_format_id + '%s%s' % (sep, suffix)) if base_format_id else suffix
+
+            for media_type in ('application/f4m+xml', 'application/x-mpegURL', 'video/mp4'):
+                media = self._download_json(
+                    'https://public-api.viewster.com/movies/%s/video' % entry_id,
+                    video_id, 'Downloading %s JSON' % concat(media_type, ' '), fatal=False, query={
+                        'mediaType': media_type,
+                        'language': audio,
+                        'subtitle': subtitle,
+                    })
+                if not media:
+                    continue
+                video_url = media.get('Uri')
+                if not video_url:
+                    continue
+                ext = determine_ext(video_url)
+                if ext == 'f4m':
+                    manifest_url = video_url
+                    video_url += '&' if '?' in video_url else '?'
+                    video_url += 'hdcore=3.2.0&plugin=flowplayer-3.2.0.1'
+                    formats.extend(self._extract_f4m_formats(
+                        video_url, video_id, f4m_id=concat('hds')))
+                elif ext == 'm3u8':
+                    manifest_url = video_url
+                    m3u8_formats = self._extract_m3u8_formats(
+                        video_url, video_id, 'mp4', m3u8_id=concat('hls'),
+                        fatal=False)  # m3u8 sometimes fail
+                    if m3u8_formats:
+                        formats.extend(m3u8_formats)
+                else:
+                    qualities_basename = self._search_regex(
+                        '/([^/]+)\.csmil/',
+                        manifest_url, 'qualities basename', default=None)
+                    if not qualities_basename:
+                        continue
+                    QUALITIES_RE = r'((,\d+k)+,?)'
+                    qualities = self._search_regex(
+                        QUALITIES_RE, qualities_basename,
+                        'qualities', default=None)
+                    if not qualities:
+                        continue
+                    qualities = list(map(lambda q: int(q[:-1]), qualities.strip(',').split(',')))
+                    qualities.sort()
+                    http_template = re.sub(QUALITIES_RE, r'%dk', qualities_basename)
+                    http_url_basename = url_basename(video_url)
+                    if m3u8_formats:
+                        self._sort_formats(m3u8_formats)
+                        m3u8_formats = list(filter(
+                            lambda f: f.get('vcodec') != 'none' and f.get('resolution') != 'multiple',
+                            m3u8_formats))
+                    if len(qualities) == len(m3u8_formats):
+                        for q, m3u8_format in zip(qualities, m3u8_formats):
+                            f = m3u8_format.copy()
+                            f.update({
+                                'url': video_url.replace(http_url_basename, http_template % q),
+                                'format_id': f['format_id'].replace('hls', 'http'),
+                                'protocol': 'http',
+                            })
+                            formats.append(f)
+                    else:
+                        for q in qualities:
+                            formats.append({
+                                'url': video_url.replace(http_url_basename, http_template % q),
+                                'ext': 'mp4',
+                                'format_id': 'http-%d' % q,
+                                'tbr': q,
+                            })
+
+        if not formats and not info.get('VODSettings'):
             self.raise_geo_restricted()
 
         self._sort_formats(formats)

From e757fb3d053a195da4084c08a59a7b17b08ba598 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 28 Apr 2016 18:42:20 +0100
Subject: [PATCH 4759/6219] [crunchyroll] improve extraction

- extract more metadata(series, episode, episode_number)
- reduce duplicate requests for extracting formats
- remove duplicate formats
---
 youtube_dl/extractor/crunchyroll.py | 31 +++++++++++++++++------------
 1 file changed, 18 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 8ae3f2890..dd753c7c3 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -11,7 +11,6 @@ from math import pow, sqrt, floor
 from .common import InfoExtractor
 from ..compat import (
     compat_etree_fromstring,
-    compat_urllib_parse_unquote,
     compat_urllib_parse_urlencode,
     compat_urllib_request,
     compat_urlparse,
@@ -306,28 +305,24 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
             r'<a[^>]+href="/publisher/[^"]+"[^>]*>([^<]+)</a>', webpage,
             'video_uploader', fatal=False)
 
-        playerdata_url = compat_urllib_parse_unquote(self._html_search_regex(r'"config_url":"([^"]+)', webpage, 'playerdata_url'))
-        playerdata_req = sanitized_Request(playerdata_url)
-        playerdata_req.data = urlencode_postdata({'current_page': webpage_url})
-        playerdata_req.add_header('Content-Type', 'application/x-www-form-urlencoded')
-        playerdata = self._download_webpage(playerdata_req, video_id, note='Downloading media info')
-
-        stream_id = self._search_regex(r'<media_id>([^<]+)', playerdata, 'stream_id')
-        video_thumbnail = self._search_regex(r'<episode_image_url>([^<]+)', playerdata, 'thumbnail', fatal=False)
-
         formats = []
-        for fmt in re.findall(r'showmedia\.([0-9]{3,4})p', webpage):
+        video_encode_ids = []
+        for fmt in re.findall(r'token="showmedia\.([0-9]{3,4})p"', webpage):
             stream_quality, stream_format = self._FORMAT_IDS[fmt]
             video_format = fmt + 'p'
             streamdata_req = sanitized_Request(
                 'http://www.crunchyroll.com/xml/?req=RpcApiVideoPlayer_GetStandardConfig&media_id=%s&video_format=%s&video_quality=%s'
-                % (stream_id, stream_format, stream_quality),
+                % (video_id, stream_format, stream_quality),
                 compat_urllib_parse_urlencode({'current_page': url}).encode('utf-8'))
             streamdata_req.add_header('Content-Type', 'application/x-www-form-urlencoded')
             streamdata = self._download_xml(
                 streamdata_req, video_id,
                 note='Downloading media info for %s' % video_format)
             stream_info = streamdata.find('./{default}preload/stream_info')
+            video_encode_id = xpath_text(stream_info, './video_encode_id')
+            if video_encode_id in video_encode_ids:
+                continue
+            video_encode_ids.append(video_encode_id)
             video_url = xpath_text(stream_info, './host')
             video_play_path = xpath_text(stream_info, './file')
             if not video_url or not video_play_path:
@@ -360,15 +355,25 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
             })
             formats.append(format_info)
 
+        metadata = self._download_xml(
+            'http://www.crunchyroll.com/xml', video_id,
+            note='Downloading media info', query={
+                'req': 'RpcApiVideoPlayer_GetMediaMetadata',
+                'media_id': video_id,
+            })
+
         subtitles = self.extract_subtitles(video_id, webpage)
 
         return {
             'id': video_id,
             'title': video_title,
             'description': video_description,
-            'thumbnail': video_thumbnail,
+            'thumbnail': xpath_text(metadata, 'episode_image_url'),
             'uploader': video_uploader,
             'upload_date': video_upload_date,
+            'series': xpath_text(metadata, 'series_title'),
+            'episode': xpath_text(metadata, 'episode_title'),
+            'episode_number': int_or_none(xpath_text(metadata, 'episode_number')),
             'subtitles': subtitles,
             'formats': formats,
         }

From 497971cd4a8407651debfb2fd4b10fc4009b0f15 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 29 Apr 2016 01:28:07 +0600
Subject: [PATCH 4760/6219] [yandexmusic] Clarify blockage even more

---
 youtube_dl/extractor/yandexmusic.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/yandexmusic.py b/youtube_dl/extractor/yandexmusic.py
index a33fe3d83..ce3723b55 100644
--- a/youtube_dl/extractor/yandexmusic.py
+++ b/youtube_dl/extractor/yandexmusic.py
@@ -27,9 +27,12 @@ class YandexMusicBaseIE(InfoExtractor):
         webpage = super(YandexMusicBaseIE, self)._download_webpage(*args, **kwargs)
         if 'Нам очень жаль, но&nbsp;запросы, поступившие с&nbsp;вашего IP-адреса, похожи на&nbsp;автоматические.' in webpage:
             raise ExtractorError(
-                'YandexMusic asks you to solve a CAPTCHA: go to '
-                'https://music.yandex.ru/ and solve it, then export '
-                'cookies and pass cookie file to youtube-dl with --cookies',
+                'YandexMusic has considered youtube-dl requests automated and '
+                'asks you to solve a CAPTCHA. You can either wait for some '
+                'time until unblocked and optionally use --sleep-interval '
+                'in future or alternatively you can go to https://music.yandex.ru/ '
+                'solve CAPTCHA, then export cookies and pass cookie file to '
+                'youtube-dl with --cookies',
                 expected=True)
         return webpage
 

From 683d892bf9332df1a255c673bca56a8f5487292a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 29 Apr 2016 01:30:53 +0600
Subject: [PATCH 4761/6219] [viewster] Remove unused import

---
 youtube_dl/extractor/viewster.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/viewster.py b/youtube_dl/extractor/viewster.py
index 1813b81d6..a93196a07 100644
--- a/youtube_dl/extractor/viewster.py
+++ b/youtube_dl/extractor/viewster.py
@@ -6,7 +6,6 @@ import re
 from .common import InfoExtractor
 from ..compat import (
     compat_HTTPError,
-    compat_urllib_parse,
     compat_urllib_parse_unquote,
 )
 from ..utils import (

From 72670c39decc296a3ee757301dc70389674d19c4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 29 Apr 2016 04:46:23 +0600
Subject: [PATCH 4762/6219] [arte:+7] Fix typo in _VALID_URL

---
 youtube_dl/extractor/arte.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index a9e3266dc..881cacfab 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -63,7 +63,7 @@ class ArteTvIE(InfoExtractor):
 
 class ArteTVPlus7IE(InfoExtractor):
     IE_NAME = 'arte.tv:+7'
-    _VALID_URL = r'https?://(?:www\.)?arte\.tv/guide/(?P<lang>fr|de|en|es)/(?:(?:sendungen|emissions|embed)/)?(?P<id>[^/]+)/(?P<name>[^/?#&+])'
+    _VALID_URL = r'https?://(?:www\.)?arte\.tv/guide/(?P<lang>fr|de|en|es)/(?:(?:sendungen|emissions|embed)/)?(?P<id>[^/]+)/(?P<name>[^/?#&]+)'
 
     @classmethod
     def _extract_url_info(cls, url):

From 31ff3c074eddf4078b6eb49281830875eb4e65a1 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 29 Apr 2016 13:36:52 +0800
Subject: [PATCH 4763/6219] [sexykarma] Remove the extractor

Its domain name is on sale.

Closes #9317
---
 youtube_dl/extractor/extractors.py |   1 -
 youtube_dl/extractor/sexykarma.py  | 121 -----------------------------
 2 files changed, 122 deletions(-)
 delete mode 100644 youtube_dl/extractor/sexykarma.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 88405f070..41ff1e7a5 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -657,7 +657,6 @@ from .screenwavemedia import ScreenwaveMediaIE, TeamFourIE
 from .senateisvp import SenateISVPIE
 from .servingsys import ServingSysIE
 from .sexu import SexuIE
-from .sexykarma import SexyKarmaIE
 from .shahid import ShahidIE
 from .shared import SharedIE
 from .sharesix import ShareSixIE
diff --git a/youtube_dl/extractor/sexykarma.py b/youtube_dl/extractor/sexykarma.py
deleted file mode 100644
index e33483674..000000000
--- a/youtube_dl/extractor/sexykarma.py
+++ /dev/null
@@ -1,121 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-import re
-
-from .common import InfoExtractor
-from ..utils import (
-    unified_strdate,
-    parse_duration,
-    int_or_none,
-)
-
-
-class SexyKarmaIE(InfoExtractor):
-    IE_DESC = 'Sexy Karma and Watch Indian Porn'
-    _VALID_URL = r'https?://(?:www\.)?(?:sexykarma\.com|watchindianporn\.net)/(?:[^/]+/)*video/(?P<display_id>[^/]+)-(?P<id>[a-zA-Z0-9]+)\.html'
-    _TESTS = [{
-        'url': 'http://www.sexykarma.com/gonewild/video/taking-a-quick-pee-yHI70cOyIHt.html',
-        'md5': 'b9798e7d1ef1765116a8f516c8091dbd',
-        'info_dict': {
-            'id': 'yHI70cOyIHt',
-            'display_id': 'taking-a-quick-pee',
-            'ext': 'mp4',
-            'title': 'Taking a quick pee.',
-            'thumbnail': 're:^https?://.*\.jpg$',
-            'uploader': 'wildginger7',
-            'upload_date': '20141008',
-            'duration': 22,
-            'view_count': int,
-            'comment_count': int,
-            'categories': list,
-            'age_limit': 18,
-        }
-    }, {
-        'url': 'http://www.sexykarma.com/gonewild/video/pot-pixie-tribute-8Id6EZPbuHf.html',
-        'md5': 'dd216c68d29b49b12842b9babe762a5d',
-        'info_dict': {
-            'id': '8Id6EZPbuHf',
-            'display_id': 'pot-pixie-tribute',
-            'ext': 'mp4',
-            'title': 'pot_pixie tribute',
-            'thumbnail': 're:^https?://.*\.jpg$',
-            'uploader': 'banffite',
-            'upload_date': '20141013',
-            'duration': 16,
-            'view_count': int,
-            'comment_count': int,
-            'categories': list,
-            'age_limit': 18,
-        }
-    }, {
-        'url': 'http://www.watchindianporn.net/video/desi-dancer-namrata-stripping-completely-nude-and-dancing-on-a-hot-number-dW2mtctxJfs.html',
-        'md5': '9afb80675550406ed9a63ac2819ef69d',
-        'info_dict': {
-            'id': 'dW2mtctxJfs',
-            'display_id': 'desi-dancer-namrata-stripping-completely-nude-and-dancing-on-a-hot-number',
-            'ext': 'mp4',
-            'title': 'Desi dancer namrata stripping completely nude and dancing on a hot number',
-            'thumbnail': 're:^https?://.*\.jpg$',
-            'uploader': 'Don',
-            'upload_date': '20140213',
-            'duration': 83,
-            'view_count': int,
-            'comment_count': int,
-            'categories': list,
-            'age_limit': 18,
-        }
-    }]
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        display_id = mobj.group('display_id')
-
-        webpage = self._download_webpage(url, display_id)
-
-        video_url = self._html_search_regex(
-            r"url: escape\('([^']+)'\)", webpage, 'url')
-
-        title = self._html_search_regex(
-            r'<h2 class="he2"><span>(.*?)</span>',
-            webpage, 'title')
-        thumbnail = self._html_search_regex(
-            r'<span id="container"><img\s+src="([^"]+)"',
-            webpage, 'thumbnail', fatal=False)
-
-        uploader = self._html_search_regex(
-            r'class="aupa">\s*(.*?)</a>',
-            webpage, 'uploader')
-        upload_date = unified_strdate(self._html_search_regex(
-            r'Added: <strong>(.+?)</strong>', webpage, 'upload date', fatal=False))
-
-        duration = parse_duration(self._search_regex(
-            r'<td>Time:\s*</td>\s*<td align="right"><span>\s*(.+?)\s*</span>',
-            webpage, 'duration', fatal=False))
-
-        view_count = int_or_none(self._search_regex(
-            r'<td>Views:\s*</td>\s*<td align="right"><span>\s*(\d+)\s*</span>',
-            webpage, 'view count', fatal=False))
-        comment_count = int_or_none(self._search_regex(
-            r'<td>Comments:\s*</td>\s*<td align="right"><span>\s*(\d+)\s*</span>',
-            webpage, 'comment count', fatal=False))
-
-        categories = re.findall(
-            r'<a href="[^"]+/search/video/desi"><span>([^<]+)</span></a>',
-            webpage)
-
-        return {
-            'id': video_id,
-            'display_id': display_id,
-            'url': video_url,
-            'title': title,
-            'thumbnail': thumbnail,
-            'uploader': uploader,
-            'upload_date': upload_date,
-            'duration': duration,
-            'view_count': view_count,
-            'comment_count': comment_count,
-            'categories': categories,
-            'age_limit': 18,
-        }

From f5535ed0e3537acee90820c98d6ca474d437d7d0 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 29 Apr 2016 14:24:07 +0800
Subject: [PATCH 4764/6219] [orf] Skip the expired test

---
 youtube_dl/extractor/orf.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/orf.py b/youtube_dl/extractor/orf.py
index 66c75f8b3..4e3864f0d 100644
--- a/youtube_dl/extractor/orf.py
+++ b/youtube_dl/extractor/orf.py
@@ -185,6 +185,7 @@ class ORFFM4IE(InfoExtractor):
             'timestamp': 1452456073,
             'upload_date': '20160110',
         },
+        'skip': 'Live streams on FM4 got deleted soon',
     }
 
     def _real_extract(self, url):

From 5819edef034819b76b8eec6a0cdf7b29cc9ddff3 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 29 Apr 2016 14:27:15 +0800
Subject: [PATCH 4765/6219] [ooyala] Skip an invalid test

Ooyala is used by lots of extractors and its correctness can be verified
by these websites.
---
 youtube_dl/extractor/ooyala.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/ooyala.py b/youtube_dl/extractor/ooyala.py
index 16f040191..95e982897 100644
--- a/youtube_dl/extractor/ooyala.py
+++ b/youtube_dl/extractor/ooyala.py
@@ -96,6 +96,8 @@ class OoyalaIE(OoyalaBaseIE):
                 'description': 'How badly damaged does a drive have to be to defeat Russell and his crew? Apparently, smashed to bits.',
                 'duration': 853.386,
             },
+            # The video in the original webpage now uses PlayWire
+            'skip': 'Ooyala said: movie expired',
         }, {
             # Only available for ipad
             'url': 'http://player.ooyala.com/player.js?embedCode=x1b3lqZDq9y_7kMyC2Op5qo-p077tXD0',

From 1910077ed77a270fea8e368c3815b23cee254f85 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 29 Apr 2016 17:59:23 +0800
Subject: [PATCH 4766/6219] Revert "[sexykarma] Remove the extractor"

This reverts commit 31ff3c074eddf4078b6eb49281830875eb4e65a1.
---
 youtube_dl/extractor/extractors.py |   1 +
 youtube_dl/extractor/sexykarma.py  | 121 +++++++++++++++++++++++++++++
 2 files changed, 122 insertions(+)
 create mode 100644 youtube_dl/extractor/sexykarma.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 41ff1e7a5..88405f070 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -657,6 +657,7 @@ from .screenwavemedia import ScreenwaveMediaIE, TeamFourIE
 from .senateisvp import SenateISVPIE
 from .servingsys import ServingSysIE
 from .sexu import SexuIE
+from .sexykarma import SexyKarmaIE
 from .shahid import ShahidIE
 from .shared import SharedIE
 from .sharesix import ShareSixIE
diff --git a/youtube_dl/extractor/sexykarma.py b/youtube_dl/extractor/sexykarma.py
new file mode 100644
index 000000000..e33483674
--- /dev/null
+++ b/youtube_dl/extractor/sexykarma.py
@@ -0,0 +1,121 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    unified_strdate,
+    parse_duration,
+    int_or_none,
+)
+
+
+class SexyKarmaIE(InfoExtractor):
+    IE_DESC = 'Sexy Karma and Watch Indian Porn'
+    _VALID_URL = r'https?://(?:www\.)?(?:sexykarma\.com|watchindianporn\.net)/(?:[^/]+/)*video/(?P<display_id>[^/]+)-(?P<id>[a-zA-Z0-9]+)\.html'
+    _TESTS = [{
+        'url': 'http://www.sexykarma.com/gonewild/video/taking-a-quick-pee-yHI70cOyIHt.html',
+        'md5': 'b9798e7d1ef1765116a8f516c8091dbd',
+        'info_dict': {
+            'id': 'yHI70cOyIHt',
+            'display_id': 'taking-a-quick-pee',
+            'ext': 'mp4',
+            'title': 'Taking a quick pee.',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'uploader': 'wildginger7',
+            'upload_date': '20141008',
+            'duration': 22,
+            'view_count': int,
+            'comment_count': int,
+            'categories': list,
+            'age_limit': 18,
+        }
+    }, {
+        'url': 'http://www.sexykarma.com/gonewild/video/pot-pixie-tribute-8Id6EZPbuHf.html',
+        'md5': 'dd216c68d29b49b12842b9babe762a5d',
+        'info_dict': {
+            'id': '8Id6EZPbuHf',
+            'display_id': 'pot-pixie-tribute',
+            'ext': 'mp4',
+            'title': 'pot_pixie tribute',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'uploader': 'banffite',
+            'upload_date': '20141013',
+            'duration': 16,
+            'view_count': int,
+            'comment_count': int,
+            'categories': list,
+            'age_limit': 18,
+        }
+    }, {
+        'url': 'http://www.watchindianporn.net/video/desi-dancer-namrata-stripping-completely-nude-and-dancing-on-a-hot-number-dW2mtctxJfs.html',
+        'md5': '9afb80675550406ed9a63ac2819ef69d',
+        'info_dict': {
+            'id': 'dW2mtctxJfs',
+            'display_id': 'desi-dancer-namrata-stripping-completely-nude-and-dancing-on-a-hot-number',
+            'ext': 'mp4',
+            'title': 'Desi dancer namrata stripping completely nude and dancing on a hot number',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'uploader': 'Don',
+            'upload_date': '20140213',
+            'duration': 83,
+            'view_count': int,
+            'comment_count': int,
+            'categories': list,
+            'age_limit': 18,
+        }
+    }]
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        display_id = mobj.group('display_id')
+
+        webpage = self._download_webpage(url, display_id)
+
+        video_url = self._html_search_regex(
+            r"url: escape\('([^']+)'\)", webpage, 'url')
+
+        title = self._html_search_regex(
+            r'<h2 class="he2"><span>(.*?)</span>',
+            webpage, 'title')
+        thumbnail = self._html_search_regex(
+            r'<span id="container"><img\s+src="([^"]+)"',
+            webpage, 'thumbnail', fatal=False)
+
+        uploader = self._html_search_regex(
+            r'class="aupa">\s*(.*?)</a>',
+            webpage, 'uploader')
+        upload_date = unified_strdate(self._html_search_regex(
+            r'Added: <strong>(.+?)</strong>', webpage, 'upload date', fatal=False))
+
+        duration = parse_duration(self._search_regex(
+            r'<td>Time:\s*</td>\s*<td align="right"><span>\s*(.+?)\s*</span>',
+            webpage, 'duration', fatal=False))
+
+        view_count = int_or_none(self._search_regex(
+            r'<td>Views:\s*</td>\s*<td align="right"><span>\s*(\d+)\s*</span>',
+            webpage, 'view count', fatal=False))
+        comment_count = int_or_none(self._search_regex(
+            r'<td>Comments:\s*</td>\s*<td align="right"><span>\s*(\d+)\s*</span>',
+            webpage, 'comment count', fatal=False))
+
+        categories = re.findall(
+            r'<a href="[^"]+/search/video/desi"><span>([^<]+)</span></a>',
+            webpage)
+
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            'url': video_url,
+            'title': title,
+            'thumbnail': thumbnail,
+            'uploader': uploader,
+            'upload_date': upload_date,
+            'duration': duration,
+            'view_count': view_count,
+            'comment_count': comment_count,
+            'categories': categories,
+            'age_limit': 18,
+        }

From 14638e291511c3305b70dce64e9bd97686e9da93 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 29 Apr 2016 18:17:08 +0800
Subject: [PATCH 4767/6219] [sexykarma] Rename to WatchIndianPornIE and fix
 extraction

---
 youtube_dl/extractor/extractors.py            |  2 +-
 .../{sexykarma.py => watchindianporn.py}      | 63 +++++--------------
 2 files changed, 17 insertions(+), 48 deletions(-)
 rename youtube_dl/extractor/{sexykarma.py => watchindianporn.py} (54%)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 88405f070..3adcd41c4 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -657,7 +657,6 @@ from .screenwavemedia import ScreenwaveMediaIE, TeamFourIE
 from .senateisvp import SenateISVPIE
 from .servingsys import ServingSysIE
 from .sexu import SexuIE
-from .sexykarma import SexyKarmaIE
 from .shahid import ShahidIE
 from .shared import SharedIE
 from .sharesix import ShareSixIE
@@ -918,6 +917,7 @@ from .vulture import VultureIE
 from .walla import WallaIE
 from .washingtonpost import WashingtonPostIE
 from .wat import WatIE
+from .watchindianporn import WatchIndianPornIE
 from .wdr import (
     WDRIE,
     WDRMobileIE,
diff --git a/youtube_dl/extractor/sexykarma.py b/youtube_dl/extractor/watchindianporn.py
similarity index 54%
rename from youtube_dl/extractor/sexykarma.py
rename to youtube_dl/extractor/watchindianporn.py
index e33483674..5d3b5bdb4 100644
--- a/youtube_dl/extractor/sexykarma.py
+++ b/youtube_dl/extractor/watchindianporn.py
@@ -11,61 +11,27 @@ from ..utils import (
 )
 
 
-class SexyKarmaIE(InfoExtractor):
-    IE_DESC = 'Sexy Karma and Watch Indian Porn'
-    _VALID_URL = r'https?://(?:www\.)?(?:sexykarma\.com|watchindianporn\.net)/(?:[^/]+/)*video/(?P<display_id>[^/]+)-(?P<id>[a-zA-Z0-9]+)\.html'
-    _TESTS = [{
-        'url': 'http://www.sexykarma.com/gonewild/video/taking-a-quick-pee-yHI70cOyIHt.html',
-        'md5': 'b9798e7d1ef1765116a8f516c8091dbd',
+class WatchIndianPornIE(InfoExtractor):
+    IE_DESC = 'Watch Indian Porn'
+    _VALID_URL = r'https?://(?:www\.)?watchindianporn\.net/(?:[^/]+/)*video/(?P<display_id>[^/]+)-(?P<id>[a-zA-Z0-9]+)\.html'
+    _TEST = {
+        'url': 'http://www.watchindianporn.net/video/hot-milf-from-kerala-shows-off-her-gorgeous-large-breasts-on-camera-RZa2avywNPa.html',
+        'md5': '249589a164dde236ec65832bfce17440',
         'info_dict': {
-            'id': 'yHI70cOyIHt',
-            'display_id': 'taking-a-quick-pee',
+            'id': 'RZa2avywNPa',
+            'display_id': 'hot-milf-from-kerala-shows-off-her-gorgeous-large-breasts-on-camera',
             'ext': 'mp4',
-            'title': 'Taking a quick pee.',
+            'title': 'Hot milf from kerala shows off her gorgeous large breasts on camera',
             'thumbnail': 're:^https?://.*\.jpg$',
-            'uploader': 'wildginger7',
-            'upload_date': '20141008',
-            'duration': 22,
+            'uploader': 'LoveJay',
+            'upload_date': '20160428',
+            'duration': 226,
             'view_count': int,
             'comment_count': int,
             'categories': list,
             'age_limit': 18,
         }
-    }, {
-        'url': 'http://www.sexykarma.com/gonewild/video/pot-pixie-tribute-8Id6EZPbuHf.html',
-        'md5': 'dd216c68d29b49b12842b9babe762a5d',
-        'info_dict': {
-            'id': '8Id6EZPbuHf',
-            'display_id': 'pot-pixie-tribute',
-            'ext': 'mp4',
-            'title': 'pot_pixie tribute',
-            'thumbnail': 're:^https?://.*\.jpg$',
-            'uploader': 'banffite',
-            'upload_date': '20141013',
-            'duration': 16,
-            'view_count': int,
-            'comment_count': int,
-            'categories': list,
-            'age_limit': 18,
-        }
-    }, {
-        'url': 'http://www.watchindianporn.net/video/desi-dancer-namrata-stripping-completely-nude-and-dancing-on-a-hot-number-dW2mtctxJfs.html',
-        'md5': '9afb80675550406ed9a63ac2819ef69d',
-        'info_dict': {
-            'id': 'dW2mtctxJfs',
-            'display_id': 'desi-dancer-namrata-stripping-completely-nude-and-dancing-on-a-hot-number',
-            'ext': 'mp4',
-            'title': 'Desi dancer namrata stripping completely nude and dancing on a hot number',
-            'thumbnail': 're:^https?://.*\.jpg$',
-            'uploader': 'Don',
-            'upload_date': '20140213',
-            'duration': 83,
-            'view_count': int,
-            'comment_count': int,
-            'categories': list,
-            'age_limit': 18,
-        }
-    }]
+    }
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
@@ -109,6 +75,9 @@ class SexyKarmaIE(InfoExtractor):
             'id': video_id,
             'display_id': display_id,
             'url': video_url,
+            'http_headers': {
+                'Referer': url,
+            },
             'title': title,
             'thumbnail': thumbnail,
             'uploader': uploader,

From 67167920db50e818c9fca20579c8a05eb2218f86 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 29 Apr 2016 11:14:42 +0100
Subject: [PATCH 4768/6219] [viewlift] replace SnagFilms extractors

- add support for other sites that use the same logic
- improve format extraction and sorting
---
 youtube_dl/extractor/extractors.py            |  8 ++---
 youtube_dl/extractor/generic.py               | 10 +++---
 .../extractor/{snagfilms.py => viewlift.py}   | 35 +++++++++++++------
 3 files changed, 34 insertions(+), 19 deletions(-)
 rename youtube_dl/extractor/{snagfilms.py => viewlift.py} (81%)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 3adcd41c4..b1b7f9b42 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -673,10 +673,6 @@ from .smotri import (
     SmotriUserIE,
     SmotriBroadcastIE,
 )
-from .snagfilms import (
-    SnagFilmsIE,
-    SnagFilmsEmbedIE,
-)
 from .snotr import SnotrIE
 from .sohu import SohuIE
 from .soundcloud import (
@@ -879,6 +875,10 @@ from .vidme import (
 )
 from .vidzi import VidziIE
 from .vier import VierIE, VierVideosIE
+from .viewlift import (
+    ViewLiftIE,
+    ViewLiftEmbedIE,
+)
 from .viewster import ViewsterIE
 from .viidea import ViideaIE
 from .vimeo import (
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index a95501d86..0f1eb7fa6 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -51,7 +51,7 @@ from .tnaflix import TNAFlixNetworkEmbedIE
 from .vimeo import VimeoIE
 from .dailymotion import DailymotionCloudIE
 from .onionstudios import OnionStudiosIE
-from .snagfilms import SnagFilmsEmbedIE
+from .viewlift import ViewLiftEmbedIE
 from .screenwavemedia import ScreenwaveMediaIE
 from .mtv import MTVServicesEmbeddedIE
 from .pladform import PladformIE
@@ -1924,10 +1924,10 @@ class GenericIE(InfoExtractor):
         if onionstudios_url:
             return self.url_result(onionstudios_url)
 
-        # Look for SnagFilms embeds
-        snagfilms_url = SnagFilmsEmbedIE._extract_url(webpage)
-        if snagfilms_url:
-            return self.url_result(snagfilms_url)
+        # Look for ViewLift embeds
+        viewlift_url = ViewLiftEmbedIE._extract_url(webpage)
+        if viewlift_url:
+            return self.url_result(viewlift_url)
 
         # Look for JWPlatform embeds
         jwplatform_url = JWPlatformIE._extract_url(webpage)
diff --git a/youtube_dl/extractor/snagfilms.py b/youtube_dl/extractor/viewlift.py
similarity index 81%
rename from youtube_dl/extractor/snagfilms.py
rename to youtube_dl/extractor/viewlift.py
index 6977afb27..dd4a13a4a 100644
--- a/youtube_dl/extractor/snagfilms.py
+++ b/youtube_dl/extractor/viewlift.py
@@ -13,8 +13,12 @@ from ..utils import (
 )
 
 
-class SnagFilmsEmbedIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:(?:www|embed)\.)?snagfilms\.com/embed/player\?.*\bfilmId=(?P<id>[\da-f-]{36})'
+class ViewLiftBaseIE(InfoExtractor):
+    _DOMAINS_REGEX = '(?:snagfilms|snagxtreme|funnyforfree|kiddovid|winnersview|monumentalsportsnetwork|vayafilm)\.com|kesari\.tv'
+
+
+class ViewLiftEmbedIE(ViewLiftBaseIE):
+    _VALID_URL = r'https?://(?:(?:www|embed)\.)?(?:%s)/embed/player\?.*\bfilmId=(?P<id>[\da-f-]{36})' % ViewLiftBaseIE._DOMAINS_REGEX
     _TESTS = [{
         'url': 'http://embed.snagfilms.com/embed/player?filmId=74849a00-85a9-11e1-9660-123139220831&w=500',
         'md5': '2924e9215c6eff7a55ed35b72276bd93',
@@ -40,7 +44,7 @@ class SnagFilmsEmbedIE(InfoExtractor):
     @staticmethod
     def _extract_url(webpage):
         mobj = re.search(
-            r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:embed\.)?snagfilms\.com/embed/player.+?)\1',
+            r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:embed\.)?(?:%s)/embed/player.+?)\1' % ViewLiftBaseIE._DOMAINS_REGEX,
             webpage)
         if mobj:
             return mobj.group('url')
@@ -55,6 +59,7 @@ class SnagFilmsEmbedIE(InfoExtractor):
                 'Film %s is not playable in your area.' % video_id, expected=True)
 
         formats = []
+        has_bitrate = False
         for source in self._parse_json(js_to_json(self._search_regex(
                 r'(?s)sources:\s*(\[.+?\]),', webpage, 'json')), video_id):
             file_ = source.get('file')
@@ -63,22 +68,25 @@ class SnagFilmsEmbedIE(InfoExtractor):
             type_ = source.get('type')
             ext = determine_ext(file_)
             format_id = source.get('label') or ext
-            if all(v == 'm3u8' for v in (type_, ext)):
+            if all(v == 'm3u8' or v == 'hls' for v in (type_, ext)):
                 formats.extend(self._extract_m3u8_formats(
                     file_, video_id, 'mp4', m3u8_id='hls'))
             else:
                 bitrate = int_or_none(self._search_regex(
                     [r'(\d+)kbps', r'_\d{1,2}x\d{1,2}_(\d{3,})\.%s' % ext],
                     file_, 'bitrate', default=None))
+                if not has_bitrate and bitrate:
+                    has_bitrate = True
                 height = int_or_none(self._search_regex(
                     r'^(\d+)[pP]$', format_id, 'height', default=None))
                 formats.append({
                     'url': file_,
-                    'format_id': format_id,
+                    'format_id': 'http-%s%s' % (format_id, ('-%dk' % bitrate if bitrate else '')),
                     'tbr': bitrate,
                     'height': height,
                 })
-        self._sort_formats(formats)
+        field_preference = None if has_bitrate else ('height', 'tbr', 'format_id')
+        self._sort_formats(formats, field_preference)
 
         title = self._search_regex(
             [r"title\s*:\s*'([^']+)'", r'<title>([^<]+)</title>'],
@@ -91,8 +99,8 @@ class SnagFilmsEmbedIE(InfoExtractor):
         }
 
 
-class SnagFilmsIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?snagfilms\.com/(?:films/title|show)/(?P<id>[^?#]+)'
+class ViewLiftIE(ViewLiftBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?(?P<domain>%s)/(?:films/title|show|(?:news/)?videos?)/(?P<id>[^?#]+)' % ViewLiftBaseIE._DOMAINS_REGEX
     _TESTS = [{
         'url': 'http://www.snagfilms.com/films/title/lost_for_life',
         'md5': '19844f897b35af219773fd63bdec2942',
@@ -127,10 +135,16 @@ class SnagFilmsIE(InfoExtractor):
         # Film is not available.
         'url': 'http://www.snagfilms.com/show/augie_alone/flirting',
         'only_matching': True,
+    }, {
+        'url': 'http://www.winnersview.com/videos/the-good-son',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.kesari.tv/news/video/1461919076414',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
-        display_id = self._match_id(url)
+        domain, display_id = re.match(self._VALID_URL, url).groups()
 
         webpage = self._download_webpage(url, display_id)
 
@@ -170,7 +184,7 @@ class SnagFilmsIE(InfoExtractor):
 
         return {
             '_type': 'url_transparent',
-            'url': 'http://embed.snagfilms.com/embed/player?filmId=%s' % film_id,
+            'url': 'http://%s/embed/player?filmId=%s' % (domain, film_id),
             'id': film_id,
             'display_id': display_id,
             'title': title,
@@ -178,4 +192,5 @@ class SnagFilmsIE(InfoExtractor):
             'thumbnail': thumbnail,
             'duration': duration,
             'categories': categories,
+            'ie_key': 'ViewLiftEmbed',
         }

From 065216d94f59953a228d2683d3bafe4241fd1e29 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 29 Apr 2016 11:46:42 +0100
Subject: [PATCH 4769/6219] [crunchyroll] reduce requests for formats
 extraction

---
 youtube_dl/extractor/crunchyroll.py | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index dd753c7c3..184ba6896 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -26,6 +26,7 @@ from ..utils import (
     unified_strdate,
     urlencode_postdata,
     xpath_text,
+    extract_attributes,
 )
 from ..aes import (
     aes_cbc_decrypt,
@@ -305,9 +306,18 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
             r'<a[^>]+href="/publisher/[^"]+"[^>]*>([^<]+)</a>', webpage,
             'video_uploader', fatal=False)
 
-        formats = []
+        available_fmts = []
+        for a, fmt in re.findall(r'(<a[^>]+token="showmedia\.([0-9]{3,4})p"[^>]+>.*?</a>)', webpage):
+            attrs = extract_attributes(a)
+            href = attrs.get('href')
+            if href and '/freetrial' in href:
+                continue
+            available_fmts.append(fmt)
+        if not available_fmts:
+            available_fmts = re.findall(r'token="showmedia\.([0-9]{3,4})p"', webpage)
         video_encode_ids = []
-        for fmt in re.findall(r'token="showmedia\.([0-9]{3,4})p"', webpage):
+        formats = []
+        for fmt in available_fmts:
             stream_quality, stream_format = self._FORMAT_IDS[fmt]
             video_format = fmt + 'p'
             streamdata_req = sanitized_Request(

From b24d6336a797b99339c12a0aa1b431755e22e8cf Mon Sep 17 00:00:00 2001
From: Kagami Hiiragi <kagami@genshiken.org>
Date: Tue, 26 Apr 2016 17:30:24 +0300
Subject: [PATCH 4770/6219] [vlive] Add support for live videos

---
 youtube_dl/extractor/common.py |  8 ++-
 youtube_dl/extractor/vlive.py  | 98 ++++++++++++++++++++++++++--------
 2 files changed, 83 insertions(+), 23 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index a285ee7d8..2763d2ffe 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1061,7 +1061,7 @@ class InfoExtractor(object):
     def _extract_m3u8_formats(self, m3u8_url, video_id, ext=None,
                               entry_protocol='m3u8', preference=None,
                               m3u8_id=None, note=None, errnote=None,
-                              fatal=True):
+                              fatal=True, live=False):
 
         formats = [{
             'format_id': '-'.join(filter(None, [m3u8_id, 'meta'])),
@@ -1139,7 +1139,11 @@ class InfoExtractor(object):
                 if m3u8_id:
                     format_id.append(m3u8_id)
                 last_media_name = last_media.get('NAME') if last_media and last_media.get('TYPE') != 'SUBTITLES' else None
-                format_id.append(last_media_name if last_media_name else '%d' % (tbr if tbr else len(formats)))
+                # Bandwidth of live streams may differ over time thus making
+                # format_id unpredictable. So it's better to keep provided
+                # format_id intact.
+                if last_media_name and not live:
+                    format_id.append(last_media_name if last_media_name else '%d' % (tbr if tbr else len(formats)))
                 f = {
                     'format_id': '-'.join(format_id),
                     'url': format_url(line.strip()),
diff --git a/youtube_dl/extractor/vlive.py b/youtube_dl/extractor/vlive.py
index baf39bb2c..2151696ea 100644
--- a/youtube_dl/extractor/vlive.py
+++ b/youtube_dl/extractor/vlive.py
@@ -1,8 +1,11 @@
 # coding: utf-8
-from __future__ import unicode_literals
+from __future__ import division, unicode_literals
 
+import re
+import time
 from .common import InfoExtractor
 from ..utils import (
+    ExtractorError,
     dict_get,
     float_or_none,
     int_or_none,
@@ -31,16 +34,77 @@ class VLiveIE(InfoExtractor):
         webpage = self._download_webpage(
             'http://www.vlive.tv/video/%s' % video_id, video_id)
 
-        long_video_id = self._search_regex(
-            r'vlive\.tv\.video\.ajax\.request\.handler\.init\(\s*"[0-9]+"\s*,\s*"[^"]*"\s*,\s*"([^"]+)"',
-            webpage, 'long video id')
+        # UTC+x - UTC+9 (KST)
+        tz = time.altzone if time.localtime().tm_isdst == 1 else time.timezone
+        tz_offset = -tz // 60 - 9 * 60
+        self._set_cookie('vlive.tv', 'timezoneOffset', '%d' % tz_offset)
 
-        key = self._search_regex(
-            r'vlive\.tv\.video\.ajax\.request\.handler\.init\(\s*"[0-9]+"\s*,\s*"[^"]*"\s*,\s*"[^"]+"\s*,\s*"([^"]+)"',
-            webpage, 'key')
+        status_params = self._download_json(
+            'http://www.vlive.tv/video/status?videoSeq=%s' % video_id,
+            video_id, 'Downloading JSON status',
+            headers={'Referer': url})
+        status = status_params.get('status')
+        air_start = status_params.get('onAirStartAt', '')
+        is_live = status_params.get('isLive')
 
+        video_params = self._search_regex(
+            r'vlive\.tv\.video\.ajax\.request\.handler\.init\((.+)\)',
+            webpage, 'video params')
+        live_params, long_video_id, key = re.split(
+            r'"\s*,\s*"', video_params)[1:4]
+
+        if status == 'LIVE_ON_AIR' or status == 'BIG_EVENT_ON_AIR':
+            live_params = self._parse_json('"%s"' % live_params, video_id)
+            live_params = self._parse_json(live_params, video_id)
+            return self._live(video_id, webpage, live_params)
+        elif status == 'VOD_ON_AIR' or status == 'BIG_EVENT_INTRO':
+            if long_video_id and key:
+                return self._replay(video_id, webpage, long_video_id, key)
+            elif is_live:
+                status = 'LIVE_END'
+            else:
+                status = 'COMING_SOON'
+
+        if status == 'LIVE_END':
+            raise ExtractorError('Uploading for replay. Please wait...',
+                                 expected=True)
+        elif status == 'COMING_SOON':
+            raise ExtractorError('Coming soon! %s' % air_start, expected=True)
+        elif status == 'CANCELED':
+            raise ExtractorError('We are sorry, '
+                                 'but the live broadcast has been canceled.',
+                                 expected=True)
+        else:
+            raise ExtractorError('Unknown status %s' % status)
+
+    def _get_common_fields(self, webpage):
         title = self._og_search_title(webpage)
+        creator = self._html_search_regex(
+            r'<div[^>]+class="info_area"[^>]*>\s*<a\s+[^>]*>([^<]+)',
+            webpage, 'creator', fatal=False)
+        thumbnail = self._og_search_thumbnail(webpage)
+        return {
+            'title': title,
+            'creator': creator,
+            'thumbnail': thumbnail,
+        }
 
+    def _live(self, video_id, webpage, live_params):
+        formats = []
+        for vid in live_params.get('resolutions', []):
+            formats.extend(self._extract_m3u8_formats(
+                vid['cdnUrl'], video_id, 'mp4',
+                m3u8_id=vid.get('name'),
+                fatal=False, live=True))
+        self._sort_formats(formats)
+
+        return dict(self._get_common_fields(webpage),
+            id=video_id,
+            formats=formats,
+            is_live=True,
+        )
+
+    def _replay(self, video_id, webpage, long_video_id, key):
         playinfo = self._download_json(
             'http://global.apis.naver.com/rmcnmv/rmcnmv/vod_play_videoInfo.json?%s'
             % compat_urllib_parse_urlencode({
@@ -62,11 +126,6 @@ class VLiveIE(InfoExtractor):
         } for vid in playinfo.get('videos', {}).get('list', []) if vid.get('source')]
         self._sort_formats(formats)
 
-        thumbnail = self._og_search_thumbnail(webpage)
-        creator = self._html_search_regex(
-            r'<div[^>]+class="info_area"[^>]*>\s*<a\s+[^>]*>([^<]+)',
-            webpage, 'creator', fatal=False)
-
         view_count = int_or_none(playinfo.get('meta', {}).get('count'))
 
         subtitles = {}
@@ -77,12 +136,9 @@ class VLiveIE(InfoExtractor):
                     'ext': 'vtt',
                     'url': caption['source']}]
 
-        return {
-            'id': video_id,
-            'title': title,
-            'creator': creator,
-            'thumbnail': thumbnail,
-            'view_count': view_count,
-            'formats': formats,
-            'subtitles': subtitles,
-        }
+        return dict(self._get_common_fields(webpage),
+            id=video_id,
+            formats=formats,
+            view_count=view_count,
+            subtitles=subtitles,
+        )

From 9d186afac818645490122aa7457f247c31c601bf Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 29 Apr 2016 19:29:00 +0800
Subject: [PATCH 4771/6219] [vlive] Coding style and PEP8

---
 youtube_dl/extractor/vlive.py | 19 +++++++++----------
 1 file changed, 9 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/vlive.py b/youtube_dl/extractor/vlive.py
index 2151696ea..7f9e99ec2 100644
--- a/youtube_dl/extractor/vlive.py
+++ b/youtube_dl/extractor/vlive.py
@@ -3,10 +3,11 @@ from __future__ import division, unicode_literals
 
 import re
 import time
+
 from .common import InfoExtractor
 from ..utils import (
-    ExtractorError,
     dict_get,
+    ExtractorError,
     float_or_none,
     int_or_none,
 )
@@ -99,10 +100,9 @@ class VLiveIE(InfoExtractor):
         self._sort_formats(formats)
 
         return dict(self._get_common_fields(webpage),
-            id=video_id,
-            formats=formats,
-            is_live=True,
-        )
+                    id=video_id,
+                    formats=formats,
+                    is_live=True)
 
     def _replay(self, video_id, webpage, long_video_id, key):
         playinfo = self._download_json(
@@ -137,8 +137,7 @@ class VLiveIE(InfoExtractor):
                     'url': caption['source']}]
 
         return dict(self._get_common_fields(webpage),
-            id=video_id,
-            formats=formats,
-            view_count=view_count,
-            subtitles=subtitles,
-        )
+                    id=video_id,
+                    formats=formats,
+                    view_count=view_count,
+                    subtitles=subtitles)

From 6ff4469528d642bd678df9b1fa83545a0942e333 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 29 Apr 2016 19:39:27 +0600
Subject: [PATCH 4772/6219] [crunchyroll] Relax fmt regex

---
 youtube_dl/extractor/crunchyroll.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 184ba6896..4a7664296 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -307,7 +307,7 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
             'video_uploader', fatal=False)
 
         available_fmts = []
-        for a, fmt in re.findall(r'(<a[^>]+token="showmedia\.([0-9]{3,4})p"[^>]+>.*?</a>)', webpage):
+        for a, fmt in re.findall(r'(<a[^>]+token=["\']showmedia\.([0-9]{3,4})p["\'][^>]+>)', webpage):
             attrs = extract_attributes(a)
             href = attrs.get('href')
             if href and '/freetrial' in href:

From 8312b1a3d1dc07d80d33e31f9b2b6facf13fa744 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 29 Apr 2016 19:43:53 +0600
Subject: [PATCH 4773/6219] [crunchyroll] Add even more relaxed fmt fallback

---
 youtube_dl/extractor/crunchyroll.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 4a7664296..58960b2f8 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -314,7 +314,10 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
                 continue
             available_fmts.append(fmt)
         if not available_fmts:
-            available_fmts = re.findall(r'token="showmedia\.([0-9]{3,4})p"', webpage)
+            for p in (r'token=["\']showmedia\.([0-9]{3,4})p"', r'showmedia\.([0-9]{3,4})p'):
+                available_fmts = re.findall(p, webpage)
+                if available_fmts:
+                    break
         video_encode_ids = []
         formats = []
         for fmt in available_fmts:

From 00a17a9e1234ecc868a15b5759472a0f9215f797 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 29 Apr 2016 19:44:10 +0600
Subject: [PATCH 4774/6219] [crunchyroll] Sort formats

---
 youtube_dl/extractor/crunchyroll.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 58960b2f8..90a64303d 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -367,6 +367,7 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
                 'ext': 'flv',
             })
             formats.append(format_info)
+        self._sort_formats(formats)
 
         metadata = self._download_xml(
             'http://www.crunchyroll.com/xml', video_id,

From e9c6cdf4a103d1ebdb6927bdab429c370cbe66b2 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 29 Apr 2016 22:49:04 +0800
Subject: [PATCH 4775/6219] [common] Fix format_id construction for HLS

---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 2763d2ffe..61a5d124c 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1142,7 +1142,7 @@ class InfoExtractor(object):
                 # Bandwidth of live streams may differ over time thus making
                 # format_id unpredictable. So it's better to keep provided
                 # format_id intact.
-                if last_media_name and not live:
+                if not live:
                     format_id.append(last_media_name if last_media_name else '%d' % (tbr if tbr else len(formats)))
                 f = {
                     'format_id': '-'.join(format_id),

From cef3f3011f9d3a67de3ff064a5185a1a4bcf40e7 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 30 Apr 2016 00:17:09 +0800
Subject: [PATCH 4776/6219] [funimation] Detect blocking and support CloudFlare
 cookies

---
 youtube_dl/extractor/funimation.py | 48 ++++++++++++++++++++++++++++--
 1 file changed, 45 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/funimation.py b/youtube_dl/extractor/funimation.py
index 1eb528f31..0ad0d9b6a 100644
--- a/youtube_dl/extractor/funimation.py
+++ b/youtube_dl/extractor/funimation.py
@@ -2,6 +2,10 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..compat import (
+    compat_HTTPError,
+    compat_urllib_parse_unquote_plus,
+)
 from ..utils import (
     clean_html,
     determine_ext,
@@ -27,6 +31,7 @@ class FunimationIE(InfoExtractor):
             'description': 'md5:1769f43cd5fc130ace8fd87232207892',
             'thumbnail': 're:https?://.*\.jpg',
         },
+        'skip': 'Access without user interaction is forbidden by CloudFlare, and video removed',
     }, {
         'url': 'http://www.funimation.com/shows/hacksign/videos/official/role-play',
         'info_dict': {
@@ -37,6 +42,7 @@ class FunimationIE(InfoExtractor):
             'description': 'md5:b602bdc15eef4c9bbb201bb6e6a4a2dd',
             'thumbnail': 're:https?://.*\.jpg',
         },
+        'skip': 'Access without user interaction is forbidden by CloudFlare',
     }, {
         'url': 'http://www.funimation.com/shows/attack-on-titan-junior-high/videos/promotional/broadcast-dub-preview',
         'info_dict': {
@@ -47,8 +53,36 @@ class FunimationIE(InfoExtractor):
             'description': 'md5:f8ec49c0aff702a7832cd81b8a44f803',
             'thumbnail': 're:https?://.*\.(?:jpg|png)',
         },
+        'skip': 'Access without user interaction is forbidden by CloudFlare',
     }]
 
+    _LOGIN_URL = 'http://www.funimation.com/login'
+
+    def _download_webpage(self, *args, **kwargs):
+        try:
+            return super(FunimationIE, self)._download_webpage(*args, **kwargs)
+        except ExtractorError as ee:
+            if isinstance(ee.cause, compat_HTTPError) and ee.cause.code == 403:
+                response = ee.cause.read()
+                if b'>Please complete the security check to access<' in response:
+                    raise ExtractorError(
+                        'Access to funimation.com is blocked by CloudFlare. '
+                        'Please browse to http://www.funimation.com/, solve '
+                        'the reCAPTCHA, export browser cookies to a text file,'
+                        ' and then try again with --cookies YOUR_COOKIE_FILE.',
+                        expected=True)
+            raise
+
+    def _extract_cloudflare_session_ua(self, url):
+        ci_session_cookie = self._get_cookies(url).get('ci_session')
+        if ci_session_cookie:
+            ci_session = compat_urllib_parse_unquote_plus(ci_session_cookie.value)
+            # ci_session is a string serialized by PHP function serialize()
+            # This case is simple enough to use regular expressions only
+            return self._search_regex(
+                r'"user_agent";s:\d+:"([^"]+)"', ci_session, 'user agent',
+                default=None)
+
     def _login(self):
         (username, password) = self._get_login_info()
         if username is None:
@@ -57,8 +91,11 @@ class FunimationIE(InfoExtractor):
             'email_field': username,
             'password_field': password,
         })
-        login_request = sanitized_Request('http://www.funimation.com/login', data, headers={
-            'User-Agent': 'Mozilla/5.0 (Windows NT 5.2; WOW64; rv:42.0) Gecko/20100101 Firefox/42.0',
+        user_agent = self._extract_cloudflare_session_ua(self._LOGIN_URL)
+        if not user_agent:
+            user_agent = 'Mozilla/5.0 (Windows NT 5.2; WOW64; rv:42.0) Gecko/20100101 Firefox/42.0'
+        login_request = sanitized_Request(self._LOGIN_URL, data, headers={
+            'User-Agent': user_agent,
             'Content-Type': 'application/x-www-form-urlencoded'
         })
         login_page = self._download_webpage(
@@ -103,11 +140,16 @@ class FunimationIE(InfoExtractor):
             ('mobile', 'Mozilla/5.0 (Linux; Android 4.4.2; Nexus 4 Build/KOT49H) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.114 Mobile Safari/537.36'),
         )
 
+        user_agent = self._extract_cloudflare_session_ua(url)
+        if user_agent:
+            USER_AGENTS = ((None, user_agent),)
+
         for kind, user_agent in USER_AGENTS:
             request = sanitized_Request(url)
             request.add_header('User-Agent', user_agent)
             webpage = self._download_webpage(
-                request, display_id, 'Downloading %s webpage' % kind)
+                request, display_id,
+                'Downloading %s webpage' % kind if kind else 'Downloading webpage')
 
             playlist = self._parse_json(
                 self._search_regex(

From 65a3bfb379c9d5e53cac874af097d2071ee4ac4d Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 29 Apr 2016 19:21:17 +0100
Subject: [PATCH 4777/6219] [dfb] extract m3u8 formats

---
 youtube_dl/extractor/dfb.py | 31 +++++++++++++++++++------------
 1 file changed, 19 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/dfb.py b/youtube_dl/extractor/dfb.py
index cdfeccacb..a4d0448c2 100644
--- a/youtube_dl/extractor/dfb.py
+++ b/youtube_dl/extractor/dfb.py
@@ -12,39 +12,46 @@ class DFBIE(InfoExtractor):
 
     _TEST = {
         'url': 'http://tv.dfb.de/video/u-19-em-stimmen-zum-spiel-gegen-russland/11633/',
-        # The md5 is different each time
+        'md5': 'ac0f98a52a330f700b4b3034ad240649',
         'info_dict': {
             'id': '11633',
             'display_id': 'u-19-em-stimmen-zum-spiel-gegen-russland',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'U 19-EM: Stimmen zum Spiel gegen Russland',
             'upload_date': '20150714',
         },
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        display_id = mobj.group('display_id')
+        display_id, video_id = re.match(self._VALID_URL, url).groups()
 
-        webpage = self._download_webpage(url, display_id)
         player_info = self._download_xml(
             'http://tv.dfb.de/server/hd_video.php?play=%s' % video_id,
             display_id)
         video_info = player_info.find('video')
+        stream_access_url = self._proto_relative_url(video_info.find('url').text.strip())
 
-        f4m_info = self._download_xml(
-            self._proto_relative_url(video_info.find('url').text.strip()), display_id)
-        token_el = f4m_info.find('token')
-        manifest_url = token_el.attrib['url'] + '?' + 'hdnea=' + token_el.attrib['auth'] + '&hdcore=3.2.0'
-        formats = self._extract_f4m_formats(manifest_url, display_id)
+        formats = []
+        # see http://tv.dfb.de/player/js/ajax.js for the method to extract m3u8 formats
+        for sa_url in (stream_access_url, stream_access_url + '&area=&format=iphone'):
+            stream_access_info = self._download_xml(sa_url, display_id)
+            token_el = stream_access_info.find('token')
+            manifest_url = token_el.attrib['url'] + '?' + 'hdnea=' + token_el.attrib['auth']
+            if '.f4m' in manifest_url:
+                formats.extend(self._extract_f4m_formats(
+                    manifest_url + '&hdcore=3.2.0',
+                    display_id, f4m_id='hds', fatal=False))
+            else:
+                formats.extend(self._extract_m3u8_formats(
+                    manifest_url, display_id, 'mp4',
+                    'm3u8_native', m3u8_id='hls', fatal=False))
         self._sort_formats(formats)
 
         return {
             'id': video_id,
             'display_id': display_id,
             'title': video_info.find('title').text,
-            'thumbnail': self._og_search_thumbnail(webpage),
+            'thumbnail': 'http://tv.dfb.de/images/%s_640x360.jpg' % video_id,
             'upload_date': unified_strdate(video_info.find('time_date').text),
             'formats': formats,
         }

From 5556047465e0601d2bdee0e5a436cee64b745851 Mon Sep 17 00:00:00 2001
From: Reino17 <reino@degeelebosch.nl>
Date: Wed, 27 Apr 2016 13:11:38 +0200
Subject: [PATCH 4778/6219] [rtlnl] Update 720p PG_URL_TEMPLATE

- Fixed the format_id for the 720p progressive videostream and added the video's resolution.
- The adaptive videostreams have the m3u8-extension, so I removed the confusing mp4-extension in order to make a better distinction between the these and the progressive videostreams.
---
 youtube_dl/extractor/rtlnl.py | 21 ++++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/rtlnl.py b/youtube_dl/extractor/rtlnl.py
index 543d94417..e8b55ea25 100644
--- a/youtube_dl/extractor/rtlnl.py
+++ b/youtube_dl/extractor/rtlnl.py
@@ -94,19 +94,30 @@ class RtlNlIE(InfoExtractor):
         videopath = material['videopath']
         m3u8_url = meta.get('videohost', 'http://manifest.us.rtl.nl') + videopath
 
-        formats = self._extract_m3u8_formats(m3u8_url, uuid, ext='mp4')
+        formats = self._extract_m3u8_formats(m3u8_url, uuid)
 
         video_urlpart = videopath.split('/adaptive/')[1][:-5]
         PG_URL_TEMPLATE = 'http://pg.us.rtl.nl/rtlxl/network/%s/progressive/%s.mp4'
 
         formats.extend([
             {
-                'url': PG_URL_TEMPLATE % ('a2m', video_urlpart),
-                'format_id': 'pg-sd',
+                'url': PG_URL_TEMPLATE % ('a2t', video_urlpart),
+                'format_id': 'a2t',
+                'width': 512,
+                'height': 288,
             },
             {
-                'url': PG_URL_TEMPLATE % ('a3m', video_urlpart),
-                'format_id': 'pg-hd',
+                'url': PG_URL_TEMPLATE % ('a3t', video_urlpart),
+                'format_id': 'a3t',
+                'width': 704,
+                'height': 400,
+                'quality': 0,
+            },
+            {
+                'url': PG_URL_TEMPLATE % ('nettv', video_urlpart),
+                'format_id': 'nettv',
+                'width': 1280,
+                'height': 720,
                 'quality': 0,
             }
         ])

From 0571ffda7dd12fc1067c0344f3ce4ce47b39edb0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 30 Apr 2016 01:43:39 +0600
Subject: [PATCH 4779/6219] [rtlnl] Improve extraction (Closes #9329)

* Make hls extraction non fatal and revert ext
* Extract progressive formats' metadata from corresponding hls formats
---
 youtube_dl/extractor/rtlnl.py | 55 +++++++++++++++++++++--------------
 1 file changed, 33 insertions(+), 22 deletions(-)

diff --git a/youtube_dl/extractor/rtlnl.py b/youtube_dl/extractor/rtlnl.py
index e8b55ea25..c95bcf035 100644
--- a/youtube_dl/extractor/rtlnl.py
+++ b/youtube_dl/extractor/rtlnl.py
@@ -94,33 +94,44 @@ class RtlNlIE(InfoExtractor):
         videopath = material['videopath']
         m3u8_url = meta.get('videohost', 'http://manifest.us.rtl.nl') + videopath
 
-        formats = self._extract_m3u8_formats(m3u8_url, uuid)
+        formats = self._extract_m3u8_formats(
+            m3u8_url, uuid, 'mp4', m3u8_id='hls', fatal=False)
 
         video_urlpart = videopath.split('/adaptive/')[1][:-5]
         PG_URL_TEMPLATE = 'http://pg.us.rtl.nl/rtlxl/network/%s/progressive/%s.mp4'
 
-        formats.extend([
-            {
-                'url': PG_URL_TEMPLATE % ('a2t', video_urlpart),
-                'format_id': 'a2t',
-                'width': 512,
-                'height': 288,
-            },
-            {
-                'url': PG_URL_TEMPLATE % ('a3t', video_urlpart),
-                'format_id': 'a3t',
-                'width': 704,
-                'height': 400,
-                'quality': 0,
-            },
-            {
-                'url': PG_URL_TEMPLATE % ('nettv', video_urlpart),
-                'format_id': 'nettv',
-                'width': 1280,
-                'height': 720,
-                'quality': 0,
+        PG_FORMATS = (
+            ('a2t', 512, 288),
+            ('a3t', 704, 400),
+            ('nettv', 1280, 720),
+        )
+
+        def pg_format(format_id, width, height):
+            return {
+                'url': PG_URL_TEMPLATE % (format_id, video_urlpart),
+                'format_id': 'pg-%s' % format_id,
+                'protocol': 'http',
+                'width': width,
+                'height': height,
             }
-        ])
+
+        if not formats:
+            formats = [pg_format(*pg_tuple) for pg_tuple in PG_FORMATS]
+        else:
+            pg_formats = []
+            for format_id, width, height in PG_FORMATS:
+                try:
+                    # Find hls format with the same width and height corresponding
+                    # to progressive format and copy metadata from it.
+                    f = next(f for f in formats
+                             if f.get('width') == width and f.get('height') == height).copy()
+                    f.update(pg_format(format_id, width, height))
+                    pg_formats.append(f)
+                except StopIteration:
+                    # Missing hls format does mean that no progressive format with
+                    # such width and height exists either.
+                    pass
+            formats.extend(pg_formats)
         self._sort_formats(formats)
 
         thumbnails = []

From cd63d091cecd8a85a2080035051205b00f3454d3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 30 Apr 2016 01:48:14 +0600
Subject: [PATCH 4780/6219] [rtlnl] Fix tests

---
 youtube_dl/extractor/rtlnl.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/rtlnl.py b/youtube_dl/extractor/rtlnl.py
index c95bcf035..e4411054a 100644
--- a/youtube_dl/extractor/rtlnl.py
+++ b/youtube_dl/extractor/rtlnl.py
@@ -39,7 +39,7 @@ class RtlNlIE(InfoExtractor):
             'ext': 'mp4',
             'timestamp': 1424039400,
             'title': 'RTL Nieuws - Nieuwe beelden Kopenhagen: chaos direct na aanslag',
-            'thumbnail': 're:^https?://screenshots\.rtl\.nl/system/thumb/sz=[0-9]+x[0-9]+/uuid=84ae5571-ac25-4225-ae0c-ef8d9efb2aed$',
+            'thumbnail': 're:^https?://screenshots\.rtl\.nl/(?:[^/]+/)*sz=[0-9]+x[0-9]+/uuid=84ae5571-ac25-4225-ae0c-ef8d9efb2aed$',
             'upload_date': '20150215',
             'description': 'Er zijn nieuwe beelden vrijgegeven die vlak na de aanslag in Kopenhagen zijn gemaakt. Op de video is goed te zien hoe omstanders zich bekommeren om één van de slachtoffers, terwijl de eerste agenten ter plaatse komen.',
         }
@@ -50,7 +50,7 @@ class RtlNlIE(InfoExtractor):
             'id': 'f536aac0-1dc3-4314-920e-3bd1c5b3811a',
             'ext': 'mp4',
             'title': 'RTL Nieuws - Meer beelden van overval juwelier',
-            'thumbnail': 're:^https?://screenshots\.rtl\.nl/system/thumb/sz=[0-9]+x[0-9]+/uuid=f536aac0-1dc3-4314-920e-3bd1c5b3811a$',
+            'thumbnail': 're:^https?://screenshots\.rtl\.nl/(?:[^/]+/)*sz=[0-9]+x[0-9]+/uuid=f536aac0-1dc3-4314-920e-3bd1c5b3811a$',
             'timestamp': 1437233400,
             'upload_date': '20150718',
             'duration': 30.474,

From 373e1230e4a3b934ddc59c212773d36a7e998dec Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 30 Apr 2016 01:50:26 +0600
Subject: [PATCH 4781/6219] [rtlnl] Clarify tests

---
 youtube_dl/extractor/rtlnl.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/rtlnl.py b/youtube_dl/extractor/rtlnl.py
index e4411054a..5e916c4ab 100644
--- a/youtube_dl/extractor/rtlnl.py
+++ b/youtube_dl/extractor/rtlnl.py
@@ -32,6 +32,7 @@ class RtlNlIE(InfoExtractor):
             'duration': 576.880,
         },
     }, {
+        # best format avaialble a3t
         'url': 'http://www.rtl.nl/system/videoplayer/derden/rtlnieuws/video_embed.html#uuid=84ae5571-ac25-4225-ae0c-ef8d9efb2aed/autoplay=false',
         'md5': 'dea7474214af1271d91ef332fb8be7ea',
         'info_dict': {
@@ -45,6 +46,7 @@ class RtlNlIE(InfoExtractor):
         }
     }, {
         # empty synopsis and missing episodes (see https://github.com/rg3/youtube-dl/issues/6275)
+        # best format available nettv
         'url': 'http://www.rtl.nl/system/videoplayer/derden/rtlnieuws/video_embed.html#uuid=f536aac0-1dc3-4314-920e-3bd1c5b3811a/autoplay=false',
         'info_dict': {
             'id': 'f536aac0-1dc3-4314-920e-3bd1c5b3811a',

From ca278a182b9331201e058f9f4d46b3b6114a1518 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 30 Apr 2016 02:07:29 +0600
Subject: [PATCH 4782/6219] [rtlnl] Replace test

---
 youtube_dl/extractor/rtlnl.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/rtlnl.py b/youtube_dl/extractor/rtlnl.py
index 5e916c4ab..8598b5840 100644
--- a/youtube_dl/extractor/rtlnl.py
+++ b/youtube_dl/extractor/rtlnl.py
@@ -20,16 +20,16 @@ class RtlNlIE(InfoExtractor):
         (?P<id>[0-9a-f-]+)'''
 
     _TESTS = [{
-        'url': 'http://www.rtlxl.nl/#!/rtl-nieuws-132237/6e4203a6-0a5e-3596-8424-c599a59e0677',
-        'md5': 'cc16baa36a6c169391f0764fa6b16654',
+        'url': 'http://www.rtlxl.nl/#!/rtl-nieuws-132237/82b1aad1-4a14-3d7b-b554-b0aed1b2c416',
+        'md5': '473d1946c1fdd050b2c0161a4b13c373',
         'info_dict': {
-            'id': '6e4203a6-0a5e-3596-8424-c599a59e0677',
+            'id': '82b1aad1-4a14-3d7b-b554-b0aed1b2c416',
             'ext': 'mp4',
-            'title': 'RTL Nieuws - Laat',
-            'description': 'md5:6b61f66510c8889923b11f2778c72dc5',
-            'timestamp': 1408051800,
-            'upload_date': '20140814',
-            'duration': 576.880,
+            'title': 'RTL Nieuws',
+            'description': 'md5:d41d8cd98f00b204e9800998ecf8427e',
+            'timestamp': 1461951000,
+            'upload_date': '20160429',
+            'duration': 1167.96,
         },
     }, {
         # best format avaialble a3t

From 69c4cde4ba6a4c7dfb8a46d1713cbb46d6f1d623 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 29 Apr 2016 21:35:09 +0100
Subject: [PATCH 4783/6219] [wsj] improve extraction

---
 youtube_dl/extractor/wsj.py | 95 +++++++++++++++++++------------------
 1 file changed, 48 insertions(+), 47 deletions(-)

diff --git a/youtube_dl/extractor/wsj.py b/youtube_dl/extractor/wsj.py
index 5a897371d..a83e68b17 100644
--- a/youtube_dl/extractor/wsj.py
+++ b/youtube_dl/extractor/wsj.py
@@ -4,16 +4,22 @@ from __future__ import unicode_literals
 from .common import InfoExtractor
 from ..utils import (
     int_or_none,
+    float_or_none,
     unified_strdate,
 )
 
 
 class WSJIE(InfoExtractor):
-    _VALID_URL = r'https?://video-api\.wsj\.com/api-video/player/iframe\.html\?guid=(?P<id>[a-zA-Z0-9-]+)'
+    _VALID_URL = r'''(?x)https?://
+        (?:
+            video-api\.wsj\.com/api-video/player/iframe\.html\?guid=|
+            (?:www\.)?wsj\.com/video/[^/]+/
+        )
+        (?P<id>[a-zA-Z0-9-]+)'''
     IE_DESC = 'Wall Street Journal'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://video-api.wsj.com/api-video/player/iframe.html?guid=1BD01A4C-BFE8-40A5-A42F-8A8AF9898B1A',
-        'md5': '9747d7a6ebc2f4df64b981e1dde9efa9',
+        'md5': 'e230a5bb249075e40793b655a54a02e4',
         'info_dict': {
             'id': '1BD01A4C-BFE8-40A5-A42F-8A8AF9898B1A',
             'ext': 'mp4',
@@ -24,65 +30,60 @@ class WSJIE(InfoExtractor):
             'duration': 90,
             'title': 'Bills Coach Rex Ryan Updates His Old Jets Tattoo',
         },
-    }
+    }, {
+        'url': 'http://www.wsj.com/video/can-alphabet-build-a-smarter-city/359DDAA8-9AC1-489C-82E6-0429C1E430E0.html',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        bitrates = [128, 174, 264, 320, 464, 664, 1264]
         api_url = (
             'http://video-api.wsj.com/api-video/find_all_videos.asp?'
-            'type=guid&count=1&query=%s&'
-            'fields=hls,adZone,thumbnailList,guid,state,secondsUntilStartTime,'
-            'author,description,name,linkURL,videoStillURL,duration,videoURL,'
-            'adCategory,catastrophic,linkShortURL,doctypeID,youtubeID,'
-            'titletag,rssURL,wsj-section,wsj-subsection,allthingsd-section,'
-            'allthingsd-subsection,sm-section,sm-subsection,provider,'
-            'formattedCreationDate,keywords,keywordsOmniture,column,editor,'
-            'emailURL,emailPartnerID,showName,omnitureProgramName,'
-            'omnitureVideoFormat,linkRelativeURL,touchCastID,'
-            'omniturePublishDate,%s') % (
-                video_id, ','.join('video%dkMP4Url' % br for br in bitrates))
+            'type=guid&count=1&query=%s&fields=type,hls,videoMP4List,'
+            'thumbnailList,author,description,name,duration,videoURL,'
+            'titletag,formattedCreationDate,keywords,editor' % video_id)
         info = self._download_json(api_url, video_id)['items'][0]
-
-        # Thumbnails are conveniently in the correct format already
-        thumbnails = info.get('thumbnailList')
-        creator = info.get('author')
-        uploader_id = info.get('editor')
-        categories = info.get('keywords')
-        duration = int_or_none(info.get('duration'))
-        upload_date = unified_strdate(
-            info.get('formattedCreationDate'), day_first=False)
         title = info.get('name', info.get('titletag'))
 
-        formats = [{
-            'format_id': 'f4m',
-            'format_note': 'f4m (meta URL)',
-            'url': info['videoURL'],
-        }]
-        if info.get('hls'):
+        formats = []
+
+        f4m_url = info.get('videoURL')
+        if f4m_url:
+            formats.extend(self._extract_f4m_formats(
+                f4m_url, video_id, f4m_id='hds', fatal=False))
+
+        m3u8_url = info.get('hls')
+        if m3u8_url:
             formats.extend(self._extract_m3u8_formats(
                 info['hls'], video_id, ext='mp4',
-                preference=0, entry_protocol='m3u8_native'))
-        for br in bitrates:
-            field = 'video%dkMP4Url' % br
-            if info.get(field):
-                formats.append({
-                    'format_id': 'mp4-%d' % br,
-                    'container': 'mp4',
-                    'tbr': br,
-                    'url': info[field],
-                })
+                entry_protocol='m3u8_native', m3u8_id='hls', fatal=False))
+
+        for v in info.get('videoMP4List', []):
+            mp4_url = v.get('url')
+            if not mp4_url:
+                continue
+            tbr = int_or_none(v.get('bitrate'))
+            formats.append({
+                'url': mp4_url,
+                'format_id': 'http' + ('-%d' % tbr if tbr else ''),
+                'tbr': tbr,
+                'width': int_or_none(v.get('width')),
+                'height': int_or_none(v.get('height')),
+                'fps': float_or_none(v.get('fps')),
+            })
         self._sort_formats(formats)
 
         return {
             'id': video_id,
             'formats': formats,
-            'thumbnails': thumbnails,
-            'creator': creator,
-            'uploader_id': uploader_id,
-            'duration': duration,
-            'upload_date': upload_date,
+            # Thumbnails are conveniently in the correct format already
+            'thumbnails': info.get('thumbnailList'),
+            'creator': info.get('author'),
+            'uploader_id': info.get('editor'),
+            'duration': int_or_none(info.get('duration')),
+            'upload_date': unified_strdate(info.get(
+                'formattedCreationDate'), day_first=False),
             'title': title,
-            'categories': categories,
+            'categories': info.get('keywords'),
         }

From cbc032c8b70a038a69259378c92b4ba97b42d491 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 30 Apr 2016 01:24:36 +0100
Subject: [PATCH 4784/6219] [pbs] extract all http formats

---
 youtube_dl/extractor/pbs.py | 48 ++++++++++++++++++-------------------
 1 file changed, 24 insertions(+), 24 deletions(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index f43e3a146..38cdb9975 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -196,7 +196,7 @@ class PBSIE(InfoExtractor):
     _TESTS = [
         {
             'url': 'http://www.pbs.org/tpt/constitution-usa-peter-sagal/watch/a-more-perfect-union/',
-            'md5': 'ce1888486f0908d555a8093cac9a7362',
+            'md5': '173dc391afd361fa72eab5d3d918968d',
             'info_dict': {
                 'id': '2365006249',
                 'ext': 'mp4',
@@ -204,13 +204,10 @@ class PBSIE(InfoExtractor):
                 'description': 'md5:36f341ae62e251b8f5bd2b754b95a071',
                 'duration': 3190,
             },
-            'params': {
-                'skip_download': True,  # requires ffmpeg
-            },
         },
         {
             'url': 'http://www.pbs.org/wgbh/pages/frontline/losing-iraq/',
-            'md5': '143c98aa54a346738a3d78f54c925321',
+            'md5': '6f722cb3c3982186d34b0f13374499c7',
             'info_dict': {
                 'id': '2365297690',
                 'ext': 'mp4',
@@ -218,9 +215,6 @@ class PBSIE(InfoExtractor):
                 'description': 'md5:4d3eaa01f94e61b3e73704735f1196d9',
                 'duration': 5050,
             },
-            'params': {
-                'skip_download': True,  # requires ffmpeg
-            }
         },
         {
             'url': 'http://www.pbs.org/newshour/bb/education-jan-june12-cyberschools_02-23/',
@@ -244,9 +238,6 @@ class PBSIE(InfoExtractor):
                 'duration': 6559,
                 'thumbnail': 're:^https?://.*\.jpg$',
             },
-            'params': {
-                'skip_download': True,  # requires ffmpeg
-            },
         },
         {
             'url': 'http://www.pbs.org/wgbh/nova/earth/killer-typhoon.html',
@@ -262,9 +253,6 @@ class PBSIE(InfoExtractor):
                 'upload_date': '20140122',
                 'age_limit': 10,
             },
-            'params': {
-                'skip_download': True,  # requires ffmpeg
-            },
         },
         {
             'url': 'http://www.pbs.org/wgbh/pages/frontline/united-states-of-secrets/',
@@ -290,6 +278,7 @@ class PBSIE(InfoExtractor):
         },
         {
             'url': 'http://www.pbs.org/video/2365245528/',
+            'md5': '115223d41bd55cda8ae5cd5ed4e11497',
             'info_dict': {
                 'id': '2365245528',
                 'display_id': '2365245528',
@@ -299,15 +288,13 @@ class PBSIE(InfoExtractor):
                 'duration': 6851,
                 'thumbnail': 're:^https?://.*\.jpg$',
             },
-            'params': {
-                'skip_download': True,  # requires ffmpeg
-            },
         },
         {
             # Video embedded in iframe containing angle brackets as attribute's value (e.g.
             # "<iframe style='position: absolute;<br />\ntop: 0; left: 0;' ...", see
             # https://github.com/rg3/youtube-dl/issues/7059)
             'url': 'http://www.pbs.org/food/features/a-chefs-life-season-3-episode-5-prickly-business/',
+            'md5': '84ced42850d78f1d4650297356e95e6f',
             'info_dict': {
                 'id': '2365546844',
                 'display_id': 'a-chefs-life-season-3-episode-5-prickly-business',
@@ -317,9 +304,6 @@ class PBSIE(InfoExtractor):
                 'duration': 1480,
                 'thumbnail': 're:^https?://.*\.jpg$',
             },
-            'params': {
-                'skip_download': True,  # requires ffmpeg
-            },
         },
         {
             # Frontline video embedded via flp2012.js
@@ -340,6 +324,7 @@ class PBSIE(InfoExtractor):
         {
             # Serves hd only via wigget/partnerplayer page
             'url': 'http://www.pbs.org/video/2365641075/',
+            'md5': 'acfd4c400b48149a44861cb16dd305cf',
             'info_dict': {
                 'id': '2365641075',
                 'ext': 'mp4',
@@ -348,9 +333,6 @@ class PBSIE(InfoExtractor):
                 'thumbnail': 're:^https?://.*\.jpg$',
                 'formats': 'mincount:8',
             },
-            'params': {
-                'skip_download': True,  # requires ffmpeg
-            },
         },
         {
             'url': 'http://player.pbs.org/widget/partnerplayer/2365297708/?start=0&end=0&chapterbar=false&endscreen=false&topbar=true',
@@ -494,6 +476,7 @@ class PBSIE(InfoExtractor):
                         info = video_info
 
         formats = []
+        http_url = None
         for num, redirect in enumerate(redirects):
             redirect_id = redirect.get('eeid')
 
@@ -514,13 +497,30 @@ class PBSIE(InfoExtractor):
 
             if determine_ext(format_url) == 'm3u8':
                 formats.extend(self._extract_m3u8_formats(
-                    format_url, display_id, 'mp4', preference=1, m3u8_id='hls'))
+                    format_url, display_id, 'mp4', m3u8_id='hls', fatal=False))
             else:
                 formats.append({
                     'url': format_url,
                     'format_id': redirect_id,
                 })
+                if re.search(r'^https?://.*(?:\d+k|baseline)', format_url):
+                    http_url = format_url
         self._remove_duplicate_formats(formats)
+        m3u8_formats = list(filter(
+            lambda f: f.get('protocol') == 'm3u8' and f.get('vcodec') != 'none' and f.get('resolution') != 'multiple',
+            formats))
+        if http_url:
+            for m3u8_format in m3u8_formats:
+                bitrate = self._search_regex(r'(\d+k)', m3u8_format['url'], 'bitrate', default=None)
+                if not bitrate:
+                    continue
+                f = m3u8_format.copy()
+                f.update({
+                    'url': re.sub(r'\d+k|baseline', bitrate, http_url),
+                    'format_id': m3u8_format['format_id'].replace('hls', 'http'),
+                    'protocol': 'http',
+                })
+                formats.append(f)
         self._sort_formats(formats)
 
         rating_str = info.get('rating')

From 350d7963db671884acd43f56f41bd499efd8e74a Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 30 Apr 2016 11:12:11 +0100
Subject: [PATCH 4785/6219] [pbs] fix the least bitrate http url construction

---
 youtube_dl/extractor/pbs.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 38cdb9975..75c36a621 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -514,6 +514,8 @@ class PBSIE(InfoExtractor):
                 bitrate = self._search_regex(r'(\d+k)', m3u8_format['url'], 'bitrate', default=None)
                 if not bitrate:
                     continue
+                if bitrate == '192k':
+                    bitrate = 'baseline'
                 f = m3u8_format.copy()
                 f.update({
                     'url': re.sub(r'\d+k|baseline', bitrate, http_url),

From 35cd2f4c253fa9d37b6a253f9f63bfe258d8f334 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 30 Apr 2016 11:31:09 +0100
Subject: [PATCH 4786/6219] [pbs] extract only the formats that we know that
 they will be available as http format

https://projects.pbs.org/confluence/display/coveapi/COVE+Video+Specifications
---
 youtube_dl/extractor/pbs.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 75c36a621..17c85dd7b 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -512,7 +512,9 @@ class PBSIE(InfoExtractor):
         if http_url:
             for m3u8_format in m3u8_formats:
                 bitrate = self._search_regex(r'(\d+k)', m3u8_format['url'], 'bitrate', default=None)
-                if not bitrate:
+                # extract only the formats that we know that they will be available as http format.
+                # https://projects.pbs.org/confluence/display/coveapi/COVE+Video+Specifications
+                if not bitrate or bitrate not in ('192k', '400k', '800k', '1200k', '2500k'):
                     continue
                 if bitrate == '192k':
                     bitrate = 'baseline'

From 7691184a3128bd46544ff49e264322d5e9187fdc Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 30 Apr 2016 12:57:30 +0100
Subject: [PATCH 4787/6219] [pbs] remove duplicate format

---
 youtube_dl/extractor/pbs.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 17c85dd7b..35fb1798d 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -514,10 +514,8 @@ class PBSIE(InfoExtractor):
                 bitrate = self._search_regex(r'(\d+k)', m3u8_format['url'], 'bitrate', default=None)
                 # extract only the formats that we know that they will be available as http format.
                 # https://projects.pbs.org/confluence/display/coveapi/COVE+Video+Specifications
-                if not bitrate or bitrate not in ('192k', '400k', '800k', '1200k', '2500k'):
+                if not bitrate or bitrate not in ('400k', '800k', '1200k', '2500k'):
                     continue
-                if bitrate == '192k':
-                    bitrate = 'baseline'
                 f = m3u8_format.copy()
                 f.update({
                     'url': re.sub(r'\d+k|baseline', bitrate, http_url),

From e0e9bbb0e9dd92f526b04584c47e6509a73fed04 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 30 Apr 2016 14:02:17 +0100
Subject: [PATCH 4788/6219] [pbs] extract srt and vtt subtitles

---
 youtube_dl/extractor/pbs.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 35fb1798d..81918ac6e 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -537,6 +537,19 @@ class PBSIE(InfoExtractor):
                 'ext': 'ttml',
                 'url': closed_captions_url,
             }]
+            mobj = re.search(r'/(\d+)_Encoded\.dfxp', closed_captions_url)
+            if mobj:
+                ttml_caption_suffix, ttml_caption_id = mobj.group(0, 1)
+                ttml_caption_id = int(ttml_caption_id)
+                subtitles['en'].extend([{
+                    'url': closed_captions_url.replace(
+                        ttml_caption_suffix, '/%d_Encoded.srt' % (ttml_caption_id + 1)),
+                    'ext': 'srt',
+                }, {
+                    'url': closed_captions_url.replace(
+                        ttml_caption_suffix, '/%d_Encoded.vtt' % (ttml_caption_id + 2)),
+                    'ext': 'vtt',
+                }])
 
         # info['title'] is often incomplete (e.g. 'Full Episode', 'Episode 5', etc)
         # Try turning it to 'program - title' naming scheme if possible

From d41ee7b7745d59d398f37b435146d4036e4a7448 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 30 Apr 2016 19:22:42 +0600
Subject: [PATCH 4789/6219] [vlive] Pass Referer as bytestring (Closes #9352)

---
 youtube_dl/extractor/vlive.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vlive.py b/youtube_dl/extractor/vlive.py
index 7f9e99ec2..a672ea9c5 100644
--- a/youtube_dl/extractor/vlive.py
+++ b/youtube_dl/extractor/vlive.py
@@ -43,7 +43,7 @@ class VLiveIE(InfoExtractor):
         status_params = self._download_json(
             'http://www.vlive.tv/video/status?videoSeq=%s' % video_id,
             video_id, 'Downloading JSON status',
-            headers={'Referer': url})
+            headers={'Referer': url.encode('utf-8')})
         status = status_params.get('status')
         air_start = status_params.get('onAirStartAt', '')
         is_live = status_params.get('isLive')

From 11fa3d7f997019dfce8b670e1b10042ac2004f69 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 30 Apr 2016 15:41:22 +0100
Subject: [PATCH 4790/6219] [ted] extract all http formats

---
 youtube_dl/extractor/ted.py | 45 ++++++++++++++++++++++++-------------
 1 file changed, 29 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/extractor/ted.py b/youtube_dl/extractor/ted.py
index cf8851438..aea6a02a7 100644
--- a/youtube_dl/extractor/ted.py
+++ b/youtube_dl/extractor/ted.py
@@ -102,9 +102,9 @@ class TEDIE(InfoExtractor):
     }]
 
     _NATIVE_FORMATS = {
-        'low': {'preference': 1, 'width': 320, 'height': 180},
-        'medium': {'preference': 2, 'width': 512, 'height': 288},
-        'high': {'preference': 3, 'width': 854, 'height': 480},
+        'low': {'width': 320, 'height': 180},
+        'medium': {'width': 512, 'height': 288},
+        'high': {'width': 854, 'height': 480},
     }
 
     def _extract_info(self, webpage):
@@ -171,15 +171,21 @@ class TEDIE(InfoExtractor):
                 if finfo:
                     f.update(finfo)
 
+        http_url = None
         for format_id, resources in talk_info['resources'].items():
             if format_id == 'h264':
                 for resource in resources:
+                    h264_url = resource.get('file')
+                    if not h264_url:
+                        continue
                     bitrate = int_or_none(resource.get('bitrate'))
                     formats.append({
-                        'url': resource['file'],
+                        'url': h264_url,
                         'format_id': '%s-%sk' % (format_id, bitrate),
                         'tbr': bitrate,
                     })
+                    if re.search('\d+k', h264_url):
+                        http_url = h264_url
             elif format_id == 'rtmp':
                 streamer = talk_info.get('streamer')
                 if not streamer:
@@ -195,16 +201,24 @@ class TEDIE(InfoExtractor):
                         'tbr': int_or_none(resource.get('bitrate')),
                     })
             elif format_id == 'hls':
-                hls_formats = self._extract_m3u8_formats(
-                    resources.get('stream'), video_name, 'mp4', m3u8_id=format_id)
-                for f in hls_formats:
-                    if f.get('format_id') == 'hls-meta':
-                        continue
-                    if not f.get('height'):
-                        f['vcodec'] = 'none'
-                    else:
-                        f['acodec'] = 'none'
-                formats.extend(hls_formats)
+                formats.extend(self._extract_m3u8_formats(
+                    resources.get('stream'), video_name, 'mp4', m3u8_id=format_id, fatal=False))
+
+        m3u8_formats = list(filter(
+            lambda f: f.get('protocol') == 'm3u8' and f.get('vcodec') != 'none' and f.get('resolution') != 'multiple',
+            formats))
+        if http_url:
+            for m3u8_format in m3u8_formats:
+                bitrate = self._search_regex(r'(\d+k)', m3u8_format['url'], 'bitrate', default=None)
+                if not bitrate:
+                    continue
+                f = m3u8_format.copy()
+                f.update({
+                    'url': re.sub(r'\d+k', bitrate, http_url),
+                    'format_id': m3u8_format['format_id'].replace('hls', 'http'),
+                    'protocol': 'http',
+                })
+                formats.append(f)
 
         audio_download = talk_info.get('audioDownload')
         if audio_download:
@@ -212,10 +226,9 @@ class TEDIE(InfoExtractor):
                 'url': audio_download,
                 'format_id': 'audio',
                 'vcodec': 'none',
-                'preference': -0.5,
             })
 
-        self._sort_formats(formats)
+        self._sort_formats(formats, ('width', 'height', 'tbr', 'format_id'))
 
         video_id = compat_str(talk_info['id'])
 

From f628d800fbaefe180bd354a0ff8a9009bc64da41 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 30 Apr 2016 16:34:57 +0100
Subject: [PATCH 4791/6219] [ted] add support for youtube embeds and update
 tests

---
 youtube_dl/extractor/ted.py | 25 +++++++++++++++++--------
 1 file changed, 17 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/ted.py b/youtube_dl/extractor/ted.py
index aea6a02a7..451cde76d 100644
--- a/youtube_dl/extractor/ted.py
+++ b/youtube_dl/extractor/ted.py
@@ -27,7 +27,7 @@ class TEDIE(InfoExtractor):
         '''
     _TESTS = [{
         'url': 'http://www.ted.com/talks/dan_dennett_on_our_consciousness.html',
-        'md5': 'fc94ac279feebbce69f21c0c6ee82810',
+        'md5': '0de43ac406aa3e4ea74b66c9c7789b13',
         'info_dict': {
             'id': '102',
             'ext': 'mp4',
@@ -37,21 +37,26 @@ class TEDIE(InfoExtractor):
                             'consciousness, but that half the time our brains are '
                             'actively fooling us.'),
             'uploader': 'Dan Dennett',
-            'width': 854,
+            'width': 853,
             'duration': 1308,
         }
     }, {
         'url': 'http://www.ted.com/watch/ted-institute/ted-bcg/vishal-sikka-the-beauty-and-power-of-algorithms',
-        'md5': '226f4fb9c62380d11b7995efa4c87994',
+        'md5': 'b899ac15e345fb39534d913f7606082b',
         'info_dict': {
-            'id': 'vishal-sikka-the-beauty-and-power-of-algorithms',
+            'id': 'tSVI8ta_P4w',
             'ext': 'mp4',
             'title': 'Vishal Sikka: The beauty and power of algorithms',
             'thumbnail': 're:^https?://.+\.jpg',
-            'description': 'Adaptive, intelligent, and consistent, algorithms are emerging as the ultimate app for everything from matching consumers to products to assessing medical diagnoses. Vishal Sikka shares his appreciation for the algorithm, charting both its inherent beauty and its growing power.',
-        }
+            'description': 'md5:6261fdfe3e02f4f579cbbfc00aff73f4',
+            'upload_date': '20140122',
+            'uploader_id': 'TEDInstitute',
+            'uploader': 'TED Institute',
+        },
+        'add_ie': ['Youtube'],
     }, {
         'url': 'http://www.ted.com/talks/gabby_giffords_and_mark_kelly_be_passionate_be_courageous_be_your_best',
+        'md5': '71b3ab2f4233012dce09d515c9c39ce2',
         'info_dict': {
             'id': '1972',
             'ext': 'mp4',
@@ -228,7 +233,7 @@ class TEDIE(InfoExtractor):
                 'vcodec': 'none',
             })
 
-        self._sort_formats(formats, ('width', 'height', 'tbr', 'format_id'))
+        self._sort_formats(formats)
 
         video_id = compat_str(talk_info['id'])
 
@@ -267,7 +272,11 @@ class TEDIE(InfoExtractor):
 
         config_json = self._html_search_regex(
             r'"pages\.jwplayer"\s*,\s*({.+?})\s*\)\s*</script>',
-            webpage, 'config')
+            webpage, 'config', default=None)
+        if not config_json:
+            embed_url = self._search_regex(
+                r"<iframe[^>]+class='pages-video-embed__video__object'[^>]+src='([^']+)'", webpage, 'embed url')
+            return self.url_result(self._proto_relative_url(embed_url))
         config = json.loads(config_json)['config']
         video_url = config['video']['url']
         thumbnail = config.get('image', {}).get('url')

From 89c0dc9a5fadc3927f7c03f5829e4f2ef8555888 Mon Sep 17 00:00:00 2001
From: BlahGeek <i@BlahGeek.com>
Date: Sat, 30 Apr 2016 21:32:54 +0800
Subject: [PATCH 4792/6219] [xiami] Add xiami extractor

---
 youtube_dl/extractor/extractors.py |   6 ++
 youtube_dl/extractor/xiami.py      | 161 +++++++++++++++++++++++++++++
 2 files changed, 167 insertions(+)
 create mode 100644 youtube_dl/extractor/xiami.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index b1b7f9b42..14ca9eaee 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -941,6 +941,12 @@ from .xhamster import (
     XHamsterIE,
     XHamsterEmbedIE,
 )
+from .xiami import (
+    XiamiIE,
+    XiamiAlbumIE,
+    XiamiArtistIE,
+    XiamiCollectionIE
+)
 from .xminus import XMinusIE
 from .xnxx import XNXXIE
 from .xstream import XstreamIE
diff --git a/youtube_dl/extractor/xiami.py b/youtube_dl/extractor/xiami.py
new file mode 100644
index 000000000..a28d63c48
--- /dev/null
+++ b/youtube_dl/extractor/xiami.py
@@ -0,0 +1,161 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    xpath_element,
+    xpath_text,
+    xpath_with_ns,
+    int_or_none,
+    ExtractorError
+)
+from ..compat import compat_urllib_parse_unquote
+
+
+class XiamiBaseIE(InfoExtractor):
+
+    _XML_BASE_URL = 'http://www.xiami.com/song/playlist/id'
+    _NS_MAP = {'xm': 'http://xspf.org/ns/0/'}
+
+    def _extract_track(self, track):
+        artist = xpath_text(track, xpath_with_ns('xm:artist', self._NS_MAP), default='')
+        artist = artist.split(';')
+
+        ret = {
+            'id': xpath_text(track, xpath_with_ns('xm:song_id', self._NS_MAP)),
+            'title': xpath_text(track, xpath_with_ns('xm:title', self._NS_MAP)),
+            'album': xpath_text(track, xpath_with_ns('xm:album_name', self._NS_MAP)),
+            'artist': ';'.join(artist) if artist else None,
+            'creator': artist[0] if artist else None,
+            'url': self._decrypt(xpath_text(track, xpath_with_ns('xm:location', self._NS_MAP))),
+            'thumbnail': xpath_text(track, xpath_with_ns('xm:pic', self._NS_MAP), default=None),
+            'duration': int_or_none(xpath_text(track, xpath_with_ns('xm:length', self._NS_MAP))),
+        }
+
+        lyrics_url = xpath_text(track, xpath_with_ns('xm:lyric', self._NS_MAP))
+        if lyrics_url and lyrics_url.endswith('.lrc'):
+            ret['description'] = self._download_webpage(lyrics_url, ret['id'])
+        return ret
+
+    def _extract_xml(self, _id, typ=''):
+        playlist = self._download_xml('%s/%s%s' % (self._XML_BASE_URL, _id, typ), _id)
+        tracklist = xpath_element(playlist, xpath_with_ns('./xm:trackList', self._NS_MAP))
+
+        if not len(tracklist):
+            raise ExtractorError('No track found')
+        return [self._extract_track(track) for track in tracklist]
+
+    @staticmethod
+    def _decrypt(origin):
+        n = int(origin[0])
+        origin = origin[1:]
+        short_lenth = len(origin) // n
+        long_num = len(origin) - short_lenth * n
+        l = tuple()
+        for i in range(0, n):
+            length = short_lenth
+            if i < long_num:
+                length += 1
+            l += (origin[0:length], )
+            origin = origin[length:]
+        ans = ''
+        for i in range(0, short_lenth + 1):
+            for j in range(0, n):
+                if len(l[j])>i:
+                    ans += l[j][i]
+        return compat_urllib_parse_unquote(ans).replace('^', '0')
+
+
+class XiamiIE(XiamiBaseIE):
+    IE_NAME = 'xiami:song'
+    IE_DESC = '虾米音乐'
+    _VALID_URL = r'http://www\.xiami\.com/song/(?P<id>[0-9]+)'
+    _TESTS = [
+        {
+            'url': 'http://www.xiami.com/song/1775610518',
+            'md5': '521dd6bea40fd5c9c69f913c232cb57e',
+            'info_dict': {
+                'id': '1775610518',
+                'ext': 'mp3',
+                'title': 'Woman',
+                'creator': 'HONNE',
+                'album': 'Woman',
+                'thumbnail': r're:http://img\.xiami\.net/images/album/.*\.jpg',
+                'description': 'md5:052ec7de41ca19f67e7fd70a1bfc4e0b',
+            }
+        },
+        {
+            'url': 'http://www.xiami.com/song/1775256504',
+            'md5': '932a3abd45c6aa2b1fdbe028fcb4c4fc',
+            'info_dict': {
+                'id': '1775256504',
+                'ext': 'mp3',
+                'title': '悟空',
+                'creator': '戴荃',
+                'album': '悟空',
+                'description': 'md5:206e67e84f9bed1d473d04196a00b990',
+            }
+        },
+    ]
+
+    def _real_extract(self, url):
+        _id = self._match_id(url)
+        return self._extract_xml(_id)[0]
+
+
+class XiamiAlbumIE(XiamiBaseIE):
+    IE_NAME = 'xiami:album'
+    IE_DESC = '虾米音乐 - 专辑'
+    _VALID_URL = r'http://www\.xiami\.com/album/(?P<id>[0-9]+)'
+    _TESTS = [
+        {
+            'url': 'http://www.xiami.com/album/2100300444',
+            'info_dict': {
+                'id': '2100300444',
+            },
+            'playlist_count': 10,
+        },
+        {
+            'url': 'http://www.xiami.com/album/512288?spm=a1z1s.6843761.1110925389.6.hhE9p9',
+            'only_matching': True,
+        }
+    ]
+
+    def _real_extract(self, url):
+        _id = self._match_id(url)
+        return self.playlist_result(self._extract_xml(_id, '/type/1'), _id)
+
+
+class XiamiArtistIE(XiamiBaseIE):
+    IE_NAME = 'xiami:artist'
+    IE_DESC = '虾米音乐 - 歌手'
+    _VALID_URL = r'http://www\.xiami\.com/artist/(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'http://www.xiami.com/artist/2132?spm=0.0.0.0.dKaScp',
+        'info_dict': {
+            'id': '2132',
+        },
+        'playlist_count': 20,
+    }
+
+    def _real_extract(self, url):
+        _id = self._match_id(url)
+        return self.playlist_result(self._extract_xml(_id, '/type/2'), _id)
+
+
+class XiamiCollectionIE(XiamiBaseIE):
+    IE_NAME = 'xiami:collection'
+    IE_DESC = '虾米音乐 - 精选集'
+    _VALID_URL = r'http://www\.xiami\.com/collect/(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'http://www.xiami.com/collect/156527391?spm=a1z1s.2943601.6856193.12.4jpBnr',
+        'info_dict': {
+            'id': '156527391',
+        },
+        'playlist_count': 26,
+    }
+
+    def _real_extract(self, url):
+        _id = self._match_id(url)
+        return self.playlist_result(self._extract_xml(_id, '/type/3'), _id)

From 4e0c0c1508810eb494cd32ef00fb75d03d03ce6f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 30 Apr 2016 21:50:23 +0600
Subject: [PATCH 4793/6219] [xiami] Improve extraction (Closes #9079)

* Switch to JSON source
* Add abstract IE for playlists
* Extract more track related metadata
---
 youtube_dl/extractor/extractors.py |   2 +-
 youtube_dl/extractor/xiami.py      | 193 ++++++++++++++---------------
 2 files changed, 96 insertions(+), 99 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 14ca9eaee..737960a01 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -942,7 +942,7 @@ from .xhamster import (
     XHamsterEmbedIE,
 )
 from .xiami import (
-    XiamiIE,
+    XiamiSongIE,
     XiamiAlbumIE,
     XiamiArtistIE,
     XiamiCollectionIE
diff --git a/youtube_dl/extractor/xiami.py b/youtube_dl/extractor/xiami.py
index a28d63c48..e4ed306b4 100644
--- a/youtube_dl/extractor/xiami.py
+++ b/youtube_dl/extractor/xiami.py
@@ -1,50 +1,42 @@
-# -*- coding: utf-8 -*-
-
+# coding: utf-8
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..utils import (
-    xpath_element,
-    xpath_text,
-    xpath_with_ns,
-    int_or_none,
-    ExtractorError
-)
 from ..compat import compat_urllib_parse_unquote
+from ..utils import int_or_none
 
 
 class XiamiBaseIE(InfoExtractor):
+    _API_BASE_URL = 'http://www.xiami.com/song/playlist/cat/json/id'
 
-    _XML_BASE_URL = 'http://www.xiami.com/song/playlist/id'
-    _NS_MAP = {'xm': 'http://xspf.org/ns/0/'}
+    def _extract_track(self, track, track_id=None):
+        title = track['title']
+        track_url = self._decrypt(track['location'])
 
-    def _extract_track(self, track):
-        artist = xpath_text(track, xpath_with_ns('xm:artist', self._NS_MAP), default='')
-        artist = artist.split(';')
+        subtitles = {}
+        lyrics_url = track.get('lyric_url') or track.get('lyric')
+        if lyrics_url and lyrics_url.startswith('http'):
+            subtitles['origin'] = [{'url': lyrics_url}]
 
-        ret = {
-            'id': xpath_text(track, xpath_with_ns('xm:song_id', self._NS_MAP)),
-            'title': xpath_text(track, xpath_with_ns('xm:title', self._NS_MAP)),
-            'album': xpath_text(track, xpath_with_ns('xm:album_name', self._NS_MAP)),
-            'artist': ';'.join(artist) if artist else None,
-            'creator': artist[0] if artist else None,
-            'url': self._decrypt(xpath_text(track, xpath_with_ns('xm:location', self._NS_MAP))),
-            'thumbnail': xpath_text(track, xpath_with_ns('xm:pic', self._NS_MAP), default=None),
-            'duration': int_or_none(xpath_text(track, xpath_with_ns('xm:length', self._NS_MAP))),
+        return {
+            'id': track.get('song_id') or track_id,
+            'url': track_url,
+            'title': title,
+            'thumbnail': track.get('pic') or track.get('album_pic'),
+            'duration': int_or_none(track.get('length')),
+            'creator': track.get('artist', '').split(';')[0],
+            'track': title,
+            'album': track.get('album_name'),
+            'artist': track.get('artist'),
+            'subtitles': subtitles,
         }
 
-        lyrics_url = xpath_text(track, xpath_with_ns('xm:lyric', self._NS_MAP))
-        if lyrics_url and lyrics_url.endswith('.lrc'):
-            ret['description'] = self._download_webpage(lyrics_url, ret['id'])
-        return ret
-
-    def _extract_xml(self, _id, typ=''):
-        playlist = self._download_xml('%s/%s%s' % (self._XML_BASE_URL, _id, typ), _id)
-        tracklist = xpath_element(playlist, xpath_with_ns('./xm:trackList', self._NS_MAP))
-
-        if not len(tracklist):
-            raise ExtractorError('No track found')
-        return [self._extract_track(track) for track in tracklist]
+    def _extract_tracks(self, item_id, typ=None):
+        playlist = self._download_json(
+            '%s/%s%s' % (self._API_BASE_URL, item_id, '/type/%s' % typ if typ else ''), item_id)
+        return [
+            self._extract_track(track, item_id)
+            for track in playlist['data']['trackList']]
 
     @staticmethod
     def _decrypt(origin):
@@ -62,75 +54,87 @@ class XiamiBaseIE(InfoExtractor):
         ans = ''
         for i in range(0, short_lenth + 1):
             for j in range(0, n):
-                if len(l[j])>i:
+                if len(l[j]) > i:
                     ans += l[j][i]
         return compat_urllib_parse_unquote(ans).replace('^', '0')
 
 
-class XiamiIE(XiamiBaseIE):
+class XiamiSongIE(XiamiBaseIE):
     IE_NAME = 'xiami:song'
     IE_DESC = '虾米音乐'
-    _VALID_URL = r'http://www\.xiami\.com/song/(?P<id>[0-9]+)'
-    _TESTS = [
-        {
-            'url': 'http://www.xiami.com/song/1775610518',
-            'md5': '521dd6bea40fd5c9c69f913c232cb57e',
-            'info_dict': {
-                'id': '1775610518',
-                'ext': 'mp3',
-                'title': 'Woman',
-                'creator': 'HONNE',
-                'album': 'Woman',
-                'thumbnail': r're:http://img\.xiami\.net/images/album/.*\.jpg',
-                'description': 'md5:052ec7de41ca19f67e7fd70a1bfc4e0b',
-            }
-        },
-        {
-            'url': 'http://www.xiami.com/song/1775256504',
-            'md5': '932a3abd45c6aa2b1fdbe028fcb4c4fc',
-            'info_dict': {
-                'id': '1775256504',
-                'ext': 'mp3',
-                'title': '悟空',
-                'creator': '戴荃',
-                'album': '悟空',
-                'description': 'md5:206e67e84f9bed1d473d04196a00b990',
-            }
-        },
-    ]
+    _VALID_URL = r'https?://(?:www\.)?xiami\.com/song/(?P<id>[0-9]+)'
+    _TESTS = [{
+        'url': 'http://www.xiami.com/song/1775610518',
+        'md5': '521dd6bea40fd5c9c69f913c232cb57e',
+        'info_dict': {
+            'id': '1775610518',
+            'ext': 'mp3',
+            'title': 'Woman',
+            'thumbnail': r're:http://img\.xiami\.net/images/album/.*\.jpg',
+            'duration': 265,
+            'creator': 'HONNE',
+            'track': 'Woman',
+            'album': 'Woman',
+            'artist': 'HONNE',
+            'subtitles': {
+                'origin': [{
+                    'ext': 'lrc',
+                }],
+            },
+        }
+    }, {
+        'url': 'http://www.xiami.com/song/1775256504',
+        'md5': '932a3abd45c6aa2b1fdbe028fcb4c4fc',
+        'info_dict': {
+            'id': '1775256504',
+            'ext': 'mp3',
+            'title': '悟空',
+            'thumbnail': r're:http://img\.xiami\.net/images/album/.*\.jpg',
+            'duration': 200,
+            'creator': '戴荃',
+            'track': '悟空',
+            'album': '悟空',
+            'artist': '戴荃',
+            'subtitles': {
+                'origin': [{
+                    'ext': 'lrc',
+                }],
+            },
+        }
+    }]
 
     def _real_extract(self, url):
-        _id = self._match_id(url)
-        return self._extract_xml(_id)[0]
+        return self._extract_tracks(self._match_id(url))[0]
 
 
-class XiamiAlbumIE(XiamiBaseIE):
+class XiamiPlaylistBaseIE(XiamiBaseIE):
+    def _real_extract(self, url):
+        item_id = self._match_id(url)
+        return self.playlist_result(self._extract_tracks(item_id, self._TYPE), item_id)
+
+
+class XiamiAlbumIE(XiamiPlaylistBaseIE):
     IE_NAME = 'xiami:album'
     IE_DESC = '虾米音乐 - 专辑'
-    _VALID_URL = r'http://www\.xiami\.com/album/(?P<id>[0-9]+)'
-    _TESTS = [
-        {
-            'url': 'http://www.xiami.com/album/2100300444',
-            'info_dict': {
-                'id': '2100300444',
-            },
-            'playlist_count': 10,
+    _VALID_URL = r'https?://(?:www\.)?xiami\.com/album/(?P<id>[0-9]+)'
+    _TYPE = '1'
+    _TESTS = [{
+        'url': 'http://www.xiami.com/album/2100300444',
+        'info_dict': {
+            'id': '2100300444',
         },
-        {
-            'url': 'http://www.xiami.com/album/512288?spm=a1z1s.6843761.1110925389.6.hhE9p9',
-            'only_matching': True,
-        }
-    ]
-
-    def _real_extract(self, url):
-        _id = self._match_id(url)
-        return self.playlist_result(self._extract_xml(_id, '/type/1'), _id)
+        'playlist_count': 10,
+    }, {
+        'url': 'http://www.xiami.com/album/512288?spm=a1z1s.6843761.1110925389.6.hhE9p9',
+        'only_matching': True,
+    }]
 
 
-class XiamiArtistIE(XiamiBaseIE):
+class XiamiArtistIE(XiamiPlaylistBaseIE):
     IE_NAME = 'xiami:artist'
     IE_DESC = '虾米音乐 - 歌手'
-    _VALID_URL = r'http://www\.xiami\.com/artist/(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?xiami\.com/artist/(?P<id>[0-9]+)'
+    _TYPE = '2'
     _TEST = {
         'url': 'http://www.xiami.com/artist/2132?spm=0.0.0.0.dKaScp',
         'info_dict': {
@@ -139,23 +143,16 @@ class XiamiArtistIE(XiamiBaseIE):
         'playlist_count': 20,
     }
 
-    def _real_extract(self, url):
-        _id = self._match_id(url)
-        return self.playlist_result(self._extract_xml(_id, '/type/2'), _id)
 
-
-class XiamiCollectionIE(XiamiBaseIE):
+class XiamiCollectionIE(XiamiPlaylistBaseIE):
     IE_NAME = 'xiami:collection'
     IE_DESC = '虾米音乐 - 精选集'
-    _VALID_URL = r'http://www\.xiami\.com/collect/(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?xiami\.com/collect/(?P<id>[0-9]+)'
+    _TYPE = '3'
     _TEST = {
         'url': 'http://www.xiami.com/collect/156527391?spm=a1z1s.2943601.6856193.12.4jpBnr',
         'info_dict': {
             'id': '156527391',
         },
-        'playlist_count': 26,
+        'playlist_mincount': 29,
     }
-
-    def _real_extract(self, url):
-        _id = self._match_id(url)
-        return self.playlist_result(self._extract_xml(_id, '/type/3'), _id)

From 33a1ec950c97b03e742926f3d37e9aa4ce642633 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 30 Apr 2016 20:38:45 +0100
Subject: [PATCH 4794/6219] [discovery] extract http formats

---
 youtube_dl/extractor/discovery.py | 50 +++++++++++++++++++++++++++++--
 1 file changed, 47 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/discovery.py b/youtube_dl/extractor/discovery.py
index 5f1275b39..6d1f8e670 100644
--- a/youtube_dl/extractor/discovery.py
+++ b/youtube_dl/extractor/discovery.py
@@ -1,11 +1,16 @@
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..utils import (
     parse_duration,
     parse_iso8601,
 )
-from ..compat import compat_str
+from ..compat import (
+    compat_str,
+    compat_urlparse,
+)
 
 
 class DiscoveryIE(InfoExtractor):
@@ -66,9 +71,48 @@ class DiscoveryIE(InfoExtractor):
         entries = []
 
         for idx, video_info in enumerate(info['playlist']):
-            formats = self._extract_m3u8_formats(
-                video_info['src'], display_id, 'mp4', 'm3u8_native', m3u8_id='hls',
+            m3u8_url = video_info['src']
+            formats = m3u8_formats = self._extract_m3u8_formats(
+                m3u8_url, display_id, 'mp4', 'm3u8_native', m3u8_id='hls',
                 note='Download m3u8 information for video %d' % (idx + 1))
+            qualities_basename = self._search_regex(
+                '/([^/]+)\.csmil/', m3u8_url, 'qualities basename', default=None)
+            if qualities_basename:
+                m3u8_path = compat_urlparse.urlparse(m3u8_url).path
+                QUALITIES_RE = r'((,\d+k)+,?)'
+                qualities = self._search_regex(
+                    QUALITIES_RE, qualities_basename,
+                    'qualities', default=None)
+                if qualities:
+                    qualities = list(map(lambda q: int(q[:-1]), qualities.strip(',').split(',')))
+                    qualities.sort()
+                    http_path = m3u8_path[1:].split('/', 1)[1]
+                    http_template = re.sub(QUALITIES_RE, r'%dk', http_path)
+                    http_template = http_template.replace('.csmil/master.m3u8', '')
+                    http_template = compat_urlparse.urljoin(
+                        'http://discsmil.edgesuite.net/', http_template)
+                    if m3u8_formats:
+                        self._sort_formats(m3u8_formats)
+                        m3u8_formats = list(filter(
+                            lambda f: f.get('vcodec') != 'none' and f.get('resolution') != 'multiple',
+                            m3u8_formats))
+                    if len(qualities) == len(m3u8_formats):
+                        for q, m3u8_format in zip(qualities, m3u8_formats):
+                            f = m3u8_format.copy()
+                            f.update({
+                                'url': http_template % q,
+                                'format_id': f['format_id'].replace('hls', 'http'),
+                                'protocol': 'http',
+                            })
+                            formats.append(f)
+                    else:
+                        for q in qualities:
+                            formats.append({
+                                'url': http_template % q,
+                                'ext': 'mp4',
+                                'format_id': 'http-%d' % q,
+                                'tbr': q,
+                            })
             self._sort_formats(formats)
             entries.append({
                 'id': compat_str(video_info['id']),

From 93f7a31bf3b572a98982a380ae3167ce954adc04 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 30 Apr 2016 20:49:09 +0100
Subject: [PATCH 4795/6219] [discovery] extract subtitle

---
 youtube_dl/extractor/discovery.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/youtube_dl/extractor/discovery.py b/youtube_dl/extractor/discovery.py
index 6d1f8e670..87fb29b02 100644
--- a/youtube_dl/extractor/discovery.py
+++ b/youtube_dl/extractor/discovery.py
@@ -114,6 +114,16 @@ class DiscoveryIE(InfoExtractor):
                                 'tbr': q,
                             })
             self._sort_formats(formats)
+
+            subtitles = []
+            caption_url = video_info.get('captionsUrl')
+            if caption_url:
+                subtitles = {
+                    'en': [{
+                        'url': caption_url,
+                    }]
+                }
+
             entries.append({
                 'id': compat_str(video_info['id']),
                 'formats': formats,
@@ -124,6 +134,7 @@ class DiscoveryIE(InfoExtractor):
                 'thumbnail': video_info.get('thumbnailURL'),
                 'alt_title': video_info.get('secondary_title'),
                 'timestamp': parse_iso8601(video_info.get('publishedDate')),
+                'subtitles': subtitles,
             })
 
         return self.playlist_result(entries, display_id, video_title)

From d00b93d58c2b78ce02fbf6c8e7ea556cd06ac3b6 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 30 Apr 2016 21:49:32 +0100
Subject: [PATCH 4796/6219] [discovery] extract more info using BrightcoveNewIE

---
 youtube_dl/extractor/discovery.py | 52 +++++--------------------------
 1 file changed, 7 insertions(+), 45 deletions(-)

diff --git a/youtube_dl/extractor/discovery.py b/youtube_dl/extractor/discovery.py
index 87fb29b02..9dd631752 100644
--- a/youtube_dl/extractor/discovery.py
+++ b/youtube_dl/extractor/discovery.py
@@ -38,6 +38,7 @@ class DiscoveryIE(InfoExtractor):
             'duration': 156,
             'timestamp': 1302032462,
             'upload_date': '20110405',
+            'uploader_id': '103207',
         },
         'params': {
             'skip_download': True,  # requires ffmpeg
@@ -59,7 +60,11 @@ class DiscoveryIE(InfoExtractor):
             'upload_date': '20140725',
             'timestamp': 1406246400,
             'duration': 116,
+            'uploader_id': '103207',
         },
+        'params': {
+            'skip_download': True,  # requires ffmpeg
+        }
     }]
 
     def _real_extract(self, url):
@@ -71,50 +76,6 @@ class DiscoveryIE(InfoExtractor):
         entries = []
 
         for idx, video_info in enumerate(info['playlist']):
-            m3u8_url = video_info['src']
-            formats = m3u8_formats = self._extract_m3u8_formats(
-                m3u8_url, display_id, 'mp4', 'm3u8_native', m3u8_id='hls',
-                note='Download m3u8 information for video %d' % (idx + 1))
-            qualities_basename = self._search_regex(
-                '/([^/]+)\.csmil/', m3u8_url, 'qualities basename', default=None)
-            if qualities_basename:
-                m3u8_path = compat_urlparse.urlparse(m3u8_url).path
-                QUALITIES_RE = r'((,\d+k)+,?)'
-                qualities = self._search_regex(
-                    QUALITIES_RE, qualities_basename,
-                    'qualities', default=None)
-                if qualities:
-                    qualities = list(map(lambda q: int(q[:-1]), qualities.strip(',').split(',')))
-                    qualities.sort()
-                    http_path = m3u8_path[1:].split('/', 1)[1]
-                    http_template = re.sub(QUALITIES_RE, r'%dk', http_path)
-                    http_template = http_template.replace('.csmil/master.m3u8', '')
-                    http_template = compat_urlparse.urljoin(
-                        'http://discsmil.edgesuite.net/', http_template)
-                    if m3u8_formats:
-                        self._sort_formats(m3u8_formats)
-                        m3u8_formats = list(filter(
-                            lambda f: f.get('vcodec') != 'none' and f.get('resolution') != 'multiple',
-                            m3u8_formats))
-                    if len(qualities) == len(m3u8_formats):
-                        for q, m3u8_format in zip(qualities, m3u8_formats):
-                            f = m3u8_format.copy()
-                            f.update({
-                                'url': http_template % q,
-                                'format_id': f['format_id'].replace('hls', 'http'),
-                                'protocol': 'http',
-                            })
-                            formats.append(f)
-                    else:
-                        for q in qualities:
-                            formats.append({
-                                'url': http_template % q,
-                                'ext': 'mp4',
-                                'format_id': 'http-%d' % q,
-                                'tbr': q,
-                            })
-            self._sort_formats(formats)
-
             subtitles = []
             caption_url = video_info.get('captionsUrl')
             if caption_url:
@@ -125,8 +86,9 @@ class DiscoveryIE(InfoExtractor):
                 }
 
             entries.append({
+                '_type': 'url_transparent',
+                'url': 'http://players.brightcove.net/103207/default_default/index.html?videoId=ref:%s' % video_info['referenceId'],
                 'id': compat_str(video_info['id']),
-                'formats': formats,
                 'title': video_info['title'],
                 'description': video_info.get('description'),
                 'duration': parse_duration(video_info.get('video_length')),

From ea7e7fecbd5da6866be003ea1ce5072dbe0118ae Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 30 Apr 2016 21:55:28 +0100
Subject: [PATCH 4797/6219] [discovery] remove unused imports

---
 youtube_dl/extractor/discovery.py | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/discovery.py b/youtube_dl/extractor/discovery.py
index 9dd631752..7c554ec14 100644
--- a/youtube_dl/extractor/discovery.py
+++ b/youtube_dl/extractor/discovery.py
@@ -1,16 +1,11 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..utils import (
     parse_duration,
     parse_iso8601,
 )
-from ..compat import (
-    compat_str,
-    compat_urlparse,
-)
+from ..compat import compat_str
 
 
 class DiscoveryIE(InfoExtractor):

From 339fe7228ae149db9fc163c94bef168f65a0a775 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 1 May 2016 02:53:42 +0600
Subject: [PATCH 4798/6219] [tagesschau] Update _FORMATS map

---
 youtube_dl/extractor/tagesschau.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/tagesschau.py b/youtube_dl/extractor/tagesschau.py
index 73e7657d4..ebd81eadc 100644
--- a/youtube_dl/extractor/tagesschau.py
+++ b/youtube_dl/extractor/tagesschau.py
@@ -64,9 +64,12 @@ class TagesschauIE(InfoExtractor):
     }]
 
     _FORMATS = {
-        's': {'width': 256, 'height': 144, 'quality': 1},
+        'xs': {'quality': 0},
+        's': {'width': 320, 'height': 180, 'quality': 1},
         'm': {'width': 512, 'height': 288, 'quality': 2},
-        'l': {'width': 960, 'height': 544, 'quality': 3},
+        'l': {'width': 960, 'height': 540, 'quality': 3},
+        'xl': {'width': 1280, 'height': 720, 'quality': 4},
+        'xxl': {'quality': 5},
     }
 
     def _real_extract(self, url):

From fc35cd9e0c7ec88fedd90880dea23d593fed85ee Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 1 May 2016 02:54:39 +0600
Subject: [PATCH 4799/6219] [tagesschau] Relax _VALID_URL

---
 youtube_dl/extractor/tagesschau.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tagesschau.py b/youtube_dl/extractor/tagesschau.py
index ebd81eadc..fcccb230c 100644
--- a/youtube_dl/extractor/tagesschau.py
+++ b/youtube_dl/extractor/tagesschau.py
@@ -8,7 +8,7 @@ from ..utils import parse_filesize
 
 
 class TagesschauIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?tagesschau\.de/multimedia/(?:[^/]+/)*?[^/#?]+?(?P<id>-?[0-9]+)(?:~_[^/#?]+?)?\.html'
+    _VALID_URL = r'https?://(?:www\.)?tagesschau\.de/multimedia/(?:[^/]+/)*?[^/#?]+?(?P<id>-?[0-9]+)(?:~_?[^/#?]+?)?\.html'
 
     _TESTS = [{
         'url': 'http://www.tagesschau.de/multimedia/video/video-102143.html',
@@ -61,6 +61,9 @@ class TagesschauIE(InfoExtractor):
     }, {
         'url': 'http://www.tagesschau.de/multimedia/video/video-102303~_bab-sendung-211.html',
         'only_matching': True,
+    }, {
+        'url': 'http://www.tagesschau.de/multimedia/video/video-179517~player.html',
+        'only_matching': True,
     }]
 
     _FORMATS = {

From 9e1b96ae400c70b1ecfc2d8917510def2ed23a6e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 1 May 2016 03:20:36 +0600
Subject: [PATCH 4800/6219] [rtlnl] Match formats only by height

---
 youtube_dl/extractor/rtlnl.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/rtlnl.py b/youtube_dl/extractor/rtlnl.py
index 8598b5840..4d612b5e3 100644
--- a/youtube_dl/extractor/rtlnl.py
+++ b/youtube_dl/extractor/rtlnl.py
@@ -125,10 +125,12 @@ class RtlNlIE(InfoExtractor):
                 try:
                     # Find hls format with the same width and height corresponding
                     # to progressive format and copy metadata from it.
-                    f = next(f for f in formats
-                             if f.get('width') == width and f.get('height') == height).copy()
-                    f.update(pg_format(format_id, width, height))
-                    pg_formats.append(f)
+                    f = next(f for f in formats if f.get('height') == height)
+                    # hls formats may have invalid width
+                    f['width'] = width
+                    f_copy = f.copy()
+                    f_copy.update(pg_format(format_id, width, height))
+                    pg_formats.append(f_copy)
                 except StopIteration:
                     # Missing hls format does mean that no progressive format with
                     # such width and height exists either.

From 4c1b2e5c0ea6a041bfd773efd7c4ac78ac8f3b4b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 1 May 2016 04:18:56 +0600
Subject: [PATCH 4801/6219] [tagesschau] Add support for playlists

---
 youtube_dl/extractor/tagesschau.py | 110 ++++++++++++++++++-----------
 1 file changed, 68 insertions(+), 42 deletions(-)

diff --git a/youtube_dl/extractor/tagesschau.py b/youtube_dl/extractor/tagesschau.py
index fcccb230c..e58385c57 100644
--- a/youtube_dl/extractor/tagesschau.py
+++ b/youtube_dl/extractor/tagesschau.py
@@ -8,11 +8,11 @@ from ..utils import parse_filesize
 
 
 class TagesschauIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?tagesschau\.de/multimedia/(?:[^/]+/)*?[^/#?]+?(?P<id>-?[0-9]+)(?:~_?[^/#?]+?)?\.html'
+    _VALID_URL = r'https?://(?:www\.)?tagesschau\.de/[^/]+/(?:[^/]+/)*?[^/#?]+?(?P<id>-?[0-9]+)(?:~_?[^/#?]+?)?\.html'
 
     _TESTS = [{
         'url': 'http://www.tagesschau.de/multimedia/video/video-102143.html',
-        'md5': '917a228bc7df7850783bc47979673a09',
+        'md5': 'f7c27a0eff3bfe8c7727e65f8fe1b1e6',
         'info_dict': {
             'id': '102143',
             'ext': 'mp4',
@@ -40,6 +40,13 @@ class TagesschauIE(InfoExtractor):
             'description': 'Flüchtlingsdebatte: Hitzig, aber wenig hilfreich',
             'thumbnail': 're:^https?:.*\.jpg$',
         },
+    }, {
+        'url': 'http://www.tagesschau.de/inland/afd-parteitag-135.html',
+        'info_dict': {
+            'id': '135',
+            'title': 'Möchtegern-Underdog mit Machtanspruch',
+        },
+        'playlist_count': 2,
     }, {
         'url': 'http://www.tagesschau.de/multimedia/sendung/tsg-3771.html',
         'only_matching': True,
@@ -75,6 +82,41 @@ class TagesschauIE(InfoExtractor):
         'xxl': {'quality': 5},
     }
 
+    def _extract_formats(self, download_text):
+        links = re.finditer(
+            r'<div class="button" title="(?P<title>[^"]*)"><a href="(?P<url>[^"]+)">(?P<name>.+?)</a></div>',
+            download_text)
+        formats = []
+        for l in links:
+            format_id = self._search_regex(
+                r'.*/[^/.]+\.([^/]+)\.[^/.]+', l.group('url'), 'format ID')
+            format = {
+                'format_id': format_id,
+                'url': l.group('url'),
+                'format_name': l.group('name'),
+            }
+            m = re.match(
+                r'''(?x)
+                    Video:\s*(?P<vcodec>[a-zA-Z0-9/._-]+)\s*&\#10;
+                    (?P<width>[0-9]+)x(?P<height>[0-9]+)px&\#10;
+                    (?P<vbr>[0-9]+)kbps&\#10;
+                    Audio:\s*(?P<abr>[0-9]+)kbps,\s*(?P<audio_desc>[A-Za-z\.0-9]+)&\#10;
+                    Gr&ouml;&szlig;e:\s*(?P<filesize_approx>[0-9.,]+\s+[a-zA-Z]*B)''',
+                l.group('title'))
+            if m:
+                format.update({
+                    'format_note': m.group('audio_desc'),
+                    'vcodec': m.group('vcodec'),
+                    'width': int(m.group('width')),
+                    'height': int(m.group('height')),
+                    'abr': int(m.group('abr')),
+                    'vbr': int(m.group('vbr')),
+                    'filesize_approx': parse_filesize(m.group('filesize_approx')),
+                })
+            formats.append(format)
+        self._sort_formats(formats)
+        return formats
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
         display_id = video_id.lstrip('-')
@@ -94,14 +136,14 @@ class TagesschauIE(InfoExtractor):
                         (?:,\s*quality:(?P<q_quality>["\'])(?P<quality>.+?)(?P=q_quality))?
                     ''', playerpage):
                 url = media.group('url')
-                type_ = media.group('type')
+                webpage_type = media.group('type')
                 ext = media.group('ext')
                 res = media.group('quality')
                 f = {
                     'format_id': '%s_%s' % (res, ext) if res else ext,
                     'url': url,
                     'ext': ext,
-                    'vcodec': 'none' if type_ == 'audio' else None,
+                    'vcodec': 'none' if webpage_type == 'audio' else None,
                 }
                 f.update(self._FORMATS.get(res, {}))
                 formats.append(f)
@@ -109,47 +151,31 @@ class TagesschauIE(InfoExtractor):
             title = self._og_search_title(webpage).strip()
             description = self._og_search_description(webpage).strip()
         else:
-            download_text = self._search_regex(
-                r'(?s)<p>Wir bieten dieses Video in folgenden Formaten zum Download an:</p>\s*<div class="controls">(.*?)</div>\s*<p>',
-                webpage, 'download links')
-            links = re.finditer(
-                r'<div class="button" title="(?P<title>[^"]*)"><a href="(?P<url>[^"]+)">(?P<name>.+?)</a></div>',
-                download_text)
-            formats = []
-            for l in links:
-                format_id = self._search_regex(
-                    r'.*/[^/.]+\.([^/]+)\.[^/.]+', l.group('url'), 'format ID')
-                format = {
-                    'format_id': format_id,
-                    'url': l.group('url'),
-                    'format_name': l.group('name'),
-                }
-                m = re.match(
-                    r'''(?x)
-                        Video:\s*(?P<vcodec>[a-zA-Z0-9/._-]+)\s*&\#10;
-                        (?P<width>[0-9]+)x(?P<height>[0-9]+)px&\#10;
-                        (?P<vbr>[0-9]+)kbps&\#10;
-                        Audio:\s*(?P<abr>[0-9]+)kbps,\s*(?P<audio_desc>[A-Za-z\.0-9]+)&\#10;
-                        Gr&ouml;&szlig;e:\s*(?P<filesize_approx>[0-9.,]+\s+[a-zA-Z]*B)''',
-                    l.group('title'))
-                if m:
-                    format.update({
-                        'format_note': m.group('audio_desc'),
-                        'vcodec': m.group('vcodec'),
-                        'width': int(m.group('width')),
-                        'height': int(m.group('height')),
-                        'abr': int(m.group('abr')),
-                        'vbr': int(m.group('vbr')),
-                        'filesize_approx': parse_filesize(m.group('filesize_approx')),
-                    })
-                formats.append(format)
-            thumbnail = self._og_search_thumbnail(webpage)
-            description = self._html_search_regex(
-                r'(?s)<p class="teasertext">(.*?)</p>',
-                webpage, 'description', default=None)
             title = self._html_search_regex(
                 r'<span class="headline".*?>(.*?)</span>', webpage, 'title')
 
+            DOWNLOAD_REGEX = r'(?s)<p>Wir bieten dieses Video in folgenden Formaten zum Download an:</p>\s*<div class="controls">(.*?)</div>\s*<p>'
+
+            webpage_type = self._og_search_property('type', webpage, default=None)
+            if webpage_type == 'website':  # Article
+                entries = []
+                for num, (entry_title, download_text) in enumerate(re.findall(
+                        r'(?s)<p[^>]+class="infotext"[^>]*>.*?<strong>(.+?)</strong>.*?</p>.*?%s' % DOWNLOAD_REGEX,
+                        webpage)):
+                    entries.append({
+                        'id': display_id,
+                        'title': '%s-%d' % (entry_title, num),
+                        'formats': self._extract_formats(download_text),
+                    })
+                return self.playlist_result(entries, display_id, title)
+            else:  # Assume single video
+                download_text = self._search_regex(DOWNLOAD_REGEX, webpage, 'download links')
+                formats = self._extract_formats(download_text)
+                thumbnail = self._og_search_thumbnail(webpage)
+                description = self._html_search_regex(
+                    r'(?s)<p class="teasertext">(.*?)</p>',
+                    webpage, 'description', default=None)
+
         self._sort_formats(formats)
 
         return {

From 1a2b377cc2fa9546fa08a7777a6fc5fc545cc441 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 1 May 2016 04:38:46 +0600
Subject: [PATCH 4802/6219] [tagesschau] Fix audio support

---
 youtube_dl/extractor/tagesschau.py | 75 +++++++++++++++++++-----------
 1 file changed, 49 insertions(+), 26 deletions(-)

diff --git a/youtube_dl/extractor/tagesschau.py b/youtube_dl/extractor/tagesschau.py
index e58385c57..ccc2d476d 100644
--- a/youtube_dl/extractor/tagesschau.py
+++ b/youtube_dl/extractor/tagesschau.py
@@ -4,7 +4,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import parse_filesize
+from ..utils import (
+    determine_ext,
+    parse_filesize,
+)
 
 
 class TagesschauIE(InfoExtractor):
@@ -82,37 +85,54 @@ class TagesschauIE(InfoExtractor):
         'xxl': {'quality': 5},
     }
 
-    def _extract_formats(self, download_text):
+    def _extract_formats(self, download_text, media_kind):
         links = re.finditer(
             r'<div class="button" title="(?P<title>[^"]*)"><a href="(?P<url>[^"]+)">(?P<name>.+?)</a></div>',
             download_text)
         formats = []
         for l in links:
+            link_url = l.group('url')
+            if not link_url:
+                continue
             format_id = self._search_regex(
-                r'.*/[^/.]+\.([^/]+)\.[^/.]+', l.group('url'), 'format ID')
+                r'.*/[^/.]+\.([^/]+)\.[^/.]+$', link_url, 'format ID',
+                default=determine_ext(link_url))
             format = {
                 'format_id': format_id,
                 'url': l.group('url'),
                 'format_name': l.group('name'),
             }
-            m = re.match(
-                r'''(?x)
-                    Video:\s*(?P<vcodec>[a-zA-Z0-9/._-]+)\s*&\#10;
-                    (?P<width>[0-9]+)x(?P<height>[0-9]+)px&\#10;
-                    (?P<vbr>[0-9]+)kbps&\#10;
-                    Audio:\s*(?P<abr>[0-9]+)kbps,\s*(?P<audio_desc>[A-Za-z\.0-9]+)&\#10;
-                    Gr&ouml;&szlig;e:\s*(?P<filesize_approx>[0-9.,]+\s+[a-zA-Z]*B)''',
-                l.group('title'))
-            if m:
-                format.update({
-                    'format_note': m.group('audio_desc'),
-                    'vcodec': m.group('vcodec'),
-                    'width': int(m.group('width')),
-                    'height': int(m.group('height')),
-                    'abr': int(m.group('abr')),
-                    'vbr': int(m.group('vbr')),
-                    'filesize_approx': parse_filesize(m.group('filesize_approx')),
-                })
+            title = l.group('title')
+            if title:
+                if media_kind.lower() == 'video':
+                    m = re.match(
+                        r'''(?x)
+                            Video:\s*(?P<vcodec>[a-zA-Z0-9/._-]+)\s*&\#10;
+                            (?P<width>[0-9]+)x(?P<height>[0-9]+)px&\#10;
+                            (?P<vbr>[0-9]+)kbps&\#10;
+                            Audio:\s*(?P<abr>[0-9]+)kbps,\s*(?P<audio_desc>[A-Za-z\.0-9]+)&\#10;
+                            Gr&ouml;&szlig;e:\s*(?P<filesize_approx>[0-9.,]+\s+[a-zA-Z]*B)''',
+                        title)
+                    if m:
+                        format.update({
+                            'format_note': m.group('audio_desc'),
+                            'vcodec': m.group('vcodec'),
+                            'width': int(m.group('width')),
+                            'height': int(m.group('height')),
+                            'abr': int(m.group('abr')),
+                            'vbr': int(m.group('vbr')),
+                            'filesize_approx': parse_filesize(m.group('filesize_approx')),
+                        })
+                else:
+                    m = re.match(
+                        r'(?P<format>.+?)-Format\s*:\s*(?P<abr>\d+)kbps\s*,\s*(?P<note>.+)',
+                        title)
+                    if m:
+                        format.update({
+                            'format_note': '%s, %s' % (m.group('format'), m.group('note')),
+                            'vcodec': 'none',
+                            'abr': int(m.group('abr')),
+                        })
             formats.append(format)
         self._sort_formats(formats)
         return formats
@@ -154,23 +174,26 @@ class TagesschauIE(InfoExtractor):
             title = self._html_search_regex(
                 r'<span class="headline".*?>(.*?)</span>', webpage, 'title')
 
-            DOWNLOAD_REGEX = r'(?s)<p>Wir bieten dieses Video in folgenden Formaten zum Download an:</p>\s*<div class="controls">(.*?)</div>\s*<p>'
+            DOWNLOAD_REGEX = r'(?s)<p>Wir bieten dieses (?P<kind>Video|Audio) in folgenden Formaten zum Download an:</p>\s*<div class="controls">(?P<links>.*?)</div>\s*<p>'
 
             webpage_type = self._og_search_property('type', webpage, default=None)
             if webpage_type == 'website':  # Article
                 entries = []
-                for num, (entry_title, download_text) in enumerate(re.findall(
+                for num, (entry_title, media_kind, download_text) in enumerate(re.findall(
                         r'(?s)<p[^>]+class="infotext"[^>]*>.*?<strong>(.+?)</strong>.*?</p>.*?%s' % DOWNLOAD_REGEX,
                         webpage)):
                     entries.append({
                         'id': display_id,
                         'title': '%s-%d' % (entry_title, num),
-                        'formats': self._extract_formats(download_text),
+                        'formats': self._extract_formats(download_text, media_kind),
                     })
                 return self.playlist_result(entries, display_id, title)
             else:  # Assume single video
-                download_text = self._search_regex(DOWNLOAD_REGEX, webpage, 'download links')
-                formats = self._extract_formats(download_text)
+                download_text = self._search_regex(
+                    DOWNLOAD_REGEX, webpage, 'download links', group='links')
+                media_kind = self._search_regex(
+                    DOWNLOAD_REGEX, webpage, 'media kind', default='Video', group='links')
+                formats = self._extract_formats(download_text, media_kind)
                 thumbnail = self._og_search_thumbnail(webpage)
                 description = self._html_search_regex(
                     r'(?s)<p class="teasertext">(.*?)</p>',

From 2844b093360cf53829e1c127aba0bbc4a6a279a8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 1 May 2016 04:42:05 +0600
Subject: [PATCH 4803/6219] [tagesschau] Fix article media ids

---
 youtube_dl/extractor/tagesschau.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/tagesschau.py b/youtube_dl/extractor/tagesschau.py
index ccc2d476d..6b71c8f81 100644
--- a/youtube_dl/extractor/tagesschau.py
+++ b/youtube_dl/extractor/tagesschau.py
@@ -181,10 +181,10 @@ class TagesschauIE(InfoExtractor):
                 entries = []
                 for num, (entry_title, media_kind, download_text) in enumerate(re.findall(
                         r'(?s)<p[^>]+class="infotext"[^>]*>.*?<strong>(.+?)</strong>.*?</p>.*?%s' % DOWNLOAD_REGEX,
-                        webpage)):
+                        webpage), 1):
                     entries.append({
-                        'id': display_id,
-                        'title': '%s-%d' % (entry_title, num),
+                        'id': '%s-%d' % (display_id, num),
+                        'title': '%s' % entry_title,
                         'formats': self._extract_formats(download_text, media_kind),
                     })
                 return self.playlist_result(entries, display_id, title)

From 9cf79e8f4bd7e0dfdb2ea9d29cf3ba7d3c6ab647 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 1 May 2016 01:43:58 +0100
Subject: [PATCH 4804/6219] [ccc] improve extraction

---
 youtube_dl/extractor/ccc.py | 111 ++++++++++++------------------------
 1 file changed, 38 insertions(+), 73 deletions(-)

diff --git a/youtube_dl/extractor/ccc.py b/youtube_dl/extractor/ccc.py
index dda2c0959..8f7f09e22 100644
--- a/youtube_dl/extractor/ccc.py
+++ b/youtube_dl/extractor/ccc.py
@@ -1,13 +1,9 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..utils import (
     int_or_none,
-    parse_duration,
-    qualities,
-    unified_strdate,
+    parse_iso8601,
 )
 
 
@@ -19,14 +15,14 @@ class CCCIE(InfoExtractor):
         'url': 'https://media.ccc.de/v/30C3_-_5443_-_en_-_saal_g_-_201312281830_-_introduction_to_processor_design_-_byterazor#video',
         'md5': '3a1eda8f3a29515d27f5adb967d7e740',
         'info_dict': {
-            'id': '30C3_-_5443_-_en_-_saal_g_-_201312281830_-_introduction_to_processor_design_-_byterazor',
+            'id': '1839',
             'ext': 'mp4',
             'title': 'Introduction to Processor Design',
-            'description': 'md5:80be298773966f66d56cb11260b879af',
+            'description': 'md5:df55f6d073d4ceae55aae6f2fd98a0ac',
             'thumbnail': 're:^https?://.*\.jpg$',
-            'view_count': int,
             'upload_date': '20131228',
-            'duration': 3660,
+            'timestamp': 1388188800,
+            'duration': 3710,
         }
     }, {
         'url': 'https://media.ccc.de/v/32c3-7368-shopshifting#download',
@@ -34,79 +30,48 @@ class CCCIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+        event_id = self._search_regex("data-id='(\d+)'", webpage, 'event id')
+        event_data = self._download_json('https://media.ccc.de/public/events/%s' % event_id, event_id)
 
-        if self._downloader.params.get('prefer_free_formats'):
-            preference = qualities(['mp3', 'opus', 'mp4-lq', 'webm-lq', 'h264-sd', 'mp4-sd', 'webm-sd', 'mp4', 'webm', 'mp4-hd', 'h264-hd', 'webm-hd'])
-        else:
-            preference = qualities(['opus', 'mp3', 'webm-lq', 'mp4-lq', 'webm-sd', 'h264-sd', 'mp4-sd', 'webm', 'mp4', 'webm-hd', 'mp4-hd', 'h264-hd'])
-
-        title = self._html_search_regex(
-            r'(?s)<h1>(.*?)</h1>', webpage, 'title')
-        description = self._html_search_regex(
-            r'(?s)<h3>About</h3>(.+?)<h3>',
-            webpage, 'description', fatal=False)
-        upload_date = unified_strdate(self._html_search_regex(
-            r"(?s)<span[^>]+class='[^']*fa-calendar-o'[^>]*>(.+?)</span>",
-            webpage, 'upload date', fatal=False))
-        view_count = int_or_none(self._html_search_regex(
-            r"(?s)<span class='[^']*fa-eye'></span>(.*?)</li>",
-            webpage, 'view count', fatal=False))
-        duration = parse_duration(self._html_search_regex(
-            r'(?s)<span[^>]+class=(["\']).*?fa-clock-o.*?\1[^>]*></span>(?P<duration>.+?)</li',
-            webpage, 'duration', fatal=False, group='duration'))
-
-        matches = re.finditer(r'''(?xs)
-            <(?:span|div)\s+class='label\s+filetype'>(?P<format>[^<]*)</(?:span|div)>\s*
-            <(?:span|div)\s+class='label\s+filetype'>(?P<lang>[^<]*)</(?:span|div)>\s*
-            <a\s+download\s+href='(?P<http_url>[^']+)'>\s*
-            (?:
-                .*?
-                <a\s+(?:download\s+)?href='(?P<torrent_url>[^']+\.torrent)'
-            )?''', webpage)
         formats = []
-        for m in matches:
-            format = m.group('format')
-            format_id = self._search_regex(
-                r'.*/([a-z0-9_-]+)/[^/]*$',
-                m.group('http_url'), 'format id', default=None)
-            if format_id:
-                format_id = m.group('lang') + '-' + format_id
-            vcodec = 'h264' if 'h264' in format_id else (
-                'none' if format_id in ('mp3', 'opus') else None
+        for recording in event_data.get('recordings', []):
+            recording_url = recording.get('recording_url')
+            if not recording_url:
+                continue
+            language = recording.get('language')
+            folder = recording.get('folder')
+            format_id = None
+            if language:
+                format_id = language
+            if folder:
+                if language:
+                    format_id += '-' + folder
+                else:
+                    format_id = folder
+            vcodec = 'h264' if 'h264' in folder else (
+                'none' if folder in ('mp3', 'opus') else None
             )
             formats.append({
                 'format_id': format_id,
-                'format': format,
-                'language': m.group('lang'),
-                'url': m.group('http_url'),
+                'url': recording_url,
+                'width': int_or_none(recording.get('width')),
+                'height': int_or_none(recording.get('height')),
+                'filesize': int_or_none(recording.get('size'), invscale=1024 * 1024),
+                'language': language,
                 'vcodec': vcodec,
-                'preference': preference(format_id),
             })
-
-            if m.group('torrent_url'):
-                formats.append({
-                    'format_id': 'torrent-%s' % (format if format_id is None else format_id),
-                    'format': '%s (torrent)' % format,
-                    'proto': 'torrent',
-                    'format_note': '(unsupported; will just download the .torrent file)',
-                    'vcodec': vcodec,
-                    'preference': -100 + preference(format_id),
-                    'url': m.group('torrent_url'),
-                })
         self._sort_formats(formats)
 
-        thumbnail = self._html_search_regex(
-            r"<video.*?poster='([^']+)'", webpage, 'thumbnail', fatal=False)
-
         return {
-            'id': video_id,
-            'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
-            'view_count': view_count,
-            'upload_date': upload_date,
-            'duration': duration,
+            'id': event_id,
+            'display_id': display_id,
+            'title': event_data['title'],
+            'description': event_data.get('description'),
+            'thumbnail': event_data.get('thumb_url'),
+            'timestamp': parse_iso8601(event_data.get('date')),
+            'duration': int_or_none(event_data.get('length')),
+            'tags': event_data.get('tags'),
             'formats': formats,
         }

From 6a0f9a24d0b8d979579276a7648df444634a5a50 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 1 May 2016 06:44:59 +0600
Subject: [PATCH 4805/6219] [tagesschau] Separate player extractor

---
 youtube_dl/extractor/extractors.py |   5 +-
 youtube_dl/extractor/tagesschau.py | 235 ++++++++++++++++++++---------
 2 files changed, 168 insertions(+), 72 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 737960a01..4aee53d6a 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -724,7 +724,10 @@ from .svt import (
 from .swrmediathek import SWRMediathekIE
 from .syfy import SyfyIE
 from .sztvhu import SztvHuIE
-from .tagesschau import TagesschauIE
+from .tagesschau import (
+    TagesschauPlayerIE,
+    TagesschauIE,
+)
 from .tapely import TapelyIE
 from .tass import TassIE
 from .tdslifeway import TDSLifewayIE
diff --git a/youtube_dl/extractor/tagesschau.py b/youtube_dl/extractor/tagesschau.py
index 6b71c8f81..a71fbad7d 100644
--- a/youtube_dl/extractor/tagesschau.py
+++ b/youtube_dl/extractor/tagesschau.py
@@ -6,10 +6,124 @@ import re
 from .common import InfoExtractor
 from ..utils import (
     determine_ext,
+    js_to_json,
+    parse_iso8601,
     parse_filesize,
 )
 
 
+class TagesschauPlayerIE(InfoExtractor):
+    IE_NAME = 'tagesschau:player'
+    _VALID_URL = r'https?://(?:www\.)?tagesschau\.de/multimedia/(?P<kind>audio|video)/(?P=kind)-(?P<id>\d+)~player(?:_[^/?#&]+)?\.html'
+
+    _TESTS = [{
+        'url': 'http://www.tagesschau.de/multimedia/video/video-179517~player.html',
+        'md5': '8d09548d5c15debad38bee3a4d15ca21',
+        'info_dict': {
+            'id': '179517',
+            'ext': 'mp4',
+            'title': 'Marie Kristin Boese, ARD Berlin, über den zukünftigen Kurs der AfD',
+            'thumbnail': 're:^https?:.*\.jpg$',
+            'formats': 'mincount:6',
+        },
+    }, {
+        'url': 'https://www.tagesschau.de/multimedia/audio/audio-29417~player.html',
+        'md5': '76e6eec6ebd40740671cf0a2c88617e5',
+        'info_dict': {
+            'id': '29417',
+            'ext': 'mp3',
+            'title': 'Trabi - Bye, bye Rennpappe',
+            'thumbnail': 're:^https?:.*\.jpg$',
+            'formats': 'mincount:2',
+        },
+    }, {
+        'url': 'http://www.tagesschau.de/multimedia/audio/audio-29417~player_autoplay-true.html',
+        'only_matching': True,
+    }]
+
+    _FORMATS = {
+        'xs': {'quality': 0},
+        's': {'width': 320, 'height': 180, 'quality': 1},
+        'm': {'width': 512, 'height': 288, 'quality': 2},
+        'l': {'width': 960, 'height': 540, 'quality': 3},
+        'xl': {'width': 1280, 'height': 720, 'quality': 4},
+        'xxl': {'quality': 5},
+    }
+
+    def _extract_via_api(self, kind, video_id):
+        info = self._download_json(
+            'https://www.tagesschau.de/api/multimedia/{0}/{0}-{1}.json'.format(kind, video_id),
+            video_id)
+        title = info['headline']
+        formats = []
+        for media in info['mediadata']:
+            for format_id, format_url in media.items():
+                if determine_ext(format_url) == 'm3u8':
+                    formats.extend(self._extract_m3u8_formats(
+                        format_url, video_id, 'mp4',
+                        entry_protocol='m3u8_native', m3u8_id='hls'))
+                else:
+                    formats.append({
+                        'url': format_url,
+                        'format_id': format_id,
+                        'vcodec': 'none' if kind == 'audio' else None,
+                    })
+        self._sort_formats(formats)
+        timestamp = parse_iso8601(info.get('date'))
+        return {
+            'id': video_id,
+            'title': title,
+            'timestamp': timestamp,
+            'formats': formats,
+        }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+
+        # kind = mobj.group('kind').lower()
+        # if kind == 'video':
+        #     return self._extract_via_api(kind, video_id)
+
+        # JSON api does not provide some audio formats (e.g. ogg) thus
+        # extractiong audio via webpage
+
+        webpage = self._download_webpage(url, video_id)
+
+        title = self._og_search_title(webpage).strip()
+        formats = []
+
+        for media_json in re.findall(r'({src\s*:\s*["\']http[^}]+type\s*:[^}]+})', webpage):
+            media = self._parse_json(js_to_json(media_json), video_id, fatal=False)
+            if not media:
+                continue
+            src = media.get('src')
+            if not src:
+                return
+            quality = media.get('quality')
+            kind = media.get('type', '').split('/')[0]
+            ext = determine_ext(src)
+            f = {
+                'url': src,
+                'format_id': '%s_%s' % (quality, ext) if quality else ext,
+                'ext': ext,
+                'vcodec': 'none' if kind == 'audio' else None,
+            }
+            f.update(self._FORMATS.get(quality, {}))
+            formats.append(f)
+
+        self._sort_formats(formats)
+
+        thumbnail = self._og_search_thumbnail(webpage)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'thumbnail': thumbnail,
+            'formats': formats,
+        }
+
+
 class TagesschauIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?tagesschau\.de/[^/]+/(?:[^/]+/)*?[^/#?]+?(?P<id>-?[0-9]+)(?:~_?[^/#?]+?)?\.html'
 
@@ -20,7 +134,7 @@ class TagesschauIE(InfoExtractor):
             'id': '102143',
             'ext': 'mp4',
             'title': 'Regierungsumbildung in Athen: Neue Minister in Griechenland vereidigt',
-            'description': 'md5:171feccd9d9b3dd54d05d501568f6359',
+            'description': '18.07.2015 20:10 Uhr',
             'thumbnail': 're:^https?:.*\.jpg$',
         },
     }, {
@@ -29,18 +143,30 @@ class TagesschauIE(InfoExtractor):
         'info_dict': {
             'id': '5727',
             'ext': 'mp4',
-            'description': 'md5:695c01bfd98b7e313c501386327aea59',
             'title': 'Sendung: tagesschau \t04.12.2014 20:00 Uhr',
+            'description': 'md5:695c01bfd98b7e313c501386327aea59',
             'thumbnail': 're:^https?:.*\.jpg$',
         },
     }, {
-        'url': 'http://www.tagesschau.de/multimedia/politikimradio/audio-18407.html',
-        'md5': 'aef45de271c4bf0a5db834aa40bf774c',
+        # exclusive audio
+        'url': 'http://www.tagesschau.de/multimedia/audio/audio-29417.html',
+        'md5': '76e6eec6ebd40740671cf0a2c88617e5',
         'info_dict': {
-            'id': '18407',
+            'id': '29417',
             'ext': 'mp3',
-            'title': 'Flüchtlingsdebatte: Hitzig, aber wenig hilfreich',
-            'description': 'Flüchtlingsdebatte: Hitzig, aber wenig hilfreich',
+            'title': 'Trabi - Bye, bye Rennpappe',
+            'description': 'md5:8687dda862cbbe2cfb2df09b56341317',
+            'thumbnail': 're:^https?:.*\.jpg$',
+        },
+    }, {
+        # audio in article
+        'url': 'http://www.tagesschau.de/inland/bnd-303.html',
+        'md5': 'e0916c623e85fc1d2b26b78f299d3958',
+        'info_dict': {
+            'id': '303',
+            'ext': 'mp3',
+            'title': 'Viele Baustellen für neuen BND-Chef',
+            'description': 'md5:1e69a54be3e1255b2b07cdbce5bcd8b4',
             'thumbnail': 're:^https?:.*\.jpg$',
         },
     }, {
@@ -71,19 +197,11 @@ class TagesschauIE(InfoExtractor):
     }, {
         'url': 'http://www.tagesschau.de/multimedia/video/video-102303~_bab-sendung-211.html',
         'only_matching': True,
-    }, {
-        'url': 'http://www.tagesschau.de/multimedia/video/video-179517~player.html',
-        'only_matching': True,
     }]
 
-    _FORMATS = {
-        'xs': {'quality': 0},
-        's': {'width': 320, 'height': 180, 'quality': 1},
-        'm': {'width': 512, 'height': 288, 'quality': 2},
-        'l': {'width': 960, 'height': 540, 'quality': 3},
-        'xl': {'width': 1280, 'height': 720, 'quality': 4},
-        'xxl': {'quality': 5},
-    }
+    @classmethod
+    def suitable(cls, url):
+        return False if TagesschauPlayerIE.suitable(url) else super(TagesschauIE, cls).suitable(url)
 
     def _extract_formats(self, download_text, media_kind):
         links = re.finditer(
@@ -140,64 +258,39 @@ class TagesschauIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
         display_id = video_id.lstrip('-')
+
         webpage = self._download_webpage(url, display_id)
 
-        player_url = self._html_search_meta(
-            'twitter:player', webpage, 'player URL', default=None)
-        if player_url:
-            playerpage = self._download_webpage(
-                player_url, display_id, 'Downloading player page')
+        title = self._html_search_regex(
+            r'<span[^>]*class="headline"[^>]*>(.+?)</span>',
+            webpage, 'title', default=None) or self._og_search_title(webpage)
 
-            formats = []
-            for media in re.finditer(
-                    r'''(?x)
-                        (?P<q_url>["\'])(?P<url>http://media.+?)(?P=q_url)
-                        ,\s*type:(?P<q_type>["\'])(?P<type>video|audio)/(?P<ext>.+?)(?P=q_type)
-                        (?:,\s*quality:(?P<q_quality>["\'])(?P<quality>.+?)(?P=q_quality))?
-                    ''', playerpage):
-                url = media.group('url')
-                webpage_type = media.group('type')
-                ext = media.group('ext')
-                res = media.group('quality')
-                f = {
-                    'format_id': '%s_%s' % (res, ext) if res else ext,
-                    'url': url,
-                    'ext': ext,
-                    'vcodec': 'none' if webpage_type == 'audio' else None,
-                }
-                f.update(self._FORMATS.get(res, {}))
-                formats.append(f)
-            thumbnail = self._og_search_thumbnail(playerpage)
-            title = self._og_search_title(webpage).strip()
-            description = self._og_search_description(webpage).strip()
-        else:
-            title = self._html_search_regex(
-                r'<span class="headline".*?>(.*?)</span>', webpage, 'title')
+        DOWNLOAD_REGEX = r'(?s)<p>Wir bieten dieses (?P<kind>Video|Audio) in folgenden Formaten zum Download an:</p>\s*<div class="controls">(?P<links>.*?)</div>\s*<p>'
 
-            DOWNLOAD_REGEX = r'(?s)<p>Wir bieten dieses (?P<kind>Video|Audio) in folgenden Formaten zum Download an:</p>\s*<div class="controls">(?P<links>.*?)</div>\s*<p>'
-
-            webpage_type = self._og_search_property('type', webpage, default=None)
-            if webpage_type == 'website':  # Article
-                entries = []
-                for num, (entry_title, media_kind, download_text) in enumerate(re.findall(
-                        r'(?s)<p[^>]+class="infotext"[^>]*>.*?<strong>(.+?)</strong>.*?</p>.*?%s' % DOWNLOAD_REGEX,
-                        webpage), 1):
-                    entries.append({
-                        'id': '%s-%d' % (display_id, num),
-                        'title': '%s' % entry_title,
-                        'formats': self._extract_formats(download_text, media_kind),
-                    })
+        webpage_type = self._og_search_property('type', webpage, default=None)
+        if webpage_type == 'website':  # Article
+            entries = []
+            for num, (entry_title, media_kind, download_text) in enumerate(re.findall(
+                    r'(?s)<p[^>]+class="infotext"[^>]*>.*?<strong>(.+?)</strong>.*?</p>.*?%s' % DOWNLOAD_REGEX,
+                    webpage), 1):
+                entries.append({
+                    'id': '%s-%d' % (display_id, num),
+                    'title': '%s' % entry_title,
+                    'formats': self._extract_formats(download_text, media_kind),
+                })
+            if len(entries) > 1:
                 return self.playlist_result(entries, display_id, title)
-            else:  # Assume single video
-                download_text = self._search_regex(
-                    DOWNLOAD_REGEX, webpage, 'download links', group='links')
-                media_kind = self._search_regex(
-                    DOWNLOAD_REGEX, webpage, 'media kind', default='Video', group='links')
-                formats = self._extract_formats(download_text, media_kind)
-                thumbnail = self._og_search_thumbnail(webpage)
-                description = self._html_search_regex(
-                    r'(?s)<p class="teasertext">(.*?)</p>',
-                    webpage, 'description', default=None)
+            formats = entries[0]['formats']
+        else:  # Assume single video
+            download_text = self._search_regex(
+                DOWNLOAD_REGEX, webpage, 'download links', group='links')
+            media_kind = self._search_regex(
+                DOWNLOAD_REGEX, webpage, 'media kind', default='Video', group='kind')
+            formats = self._extract_formats(download_text, media_kind)
+        thumbnail = self._og_search_thumbnail(webpage)
+        description = self._html_search_regex(
+            r'(?s)<p class="teasertext">(.*?)</p>',
+            webpage, 'description', default=None)
 
         self._sort_formats(formats)
 

From 651ad35ce0f0ee9d04db085c50c29441b47bc825 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 1 May 2016 06:57:19 +0600
Subject: [PATCH 4806/6219] [tagesschau] Relax _VALID_URL

---
 youtube_dl/extractor/tagesschau.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/tagesschau.py b/youtube_dl/extractor/tagesschau.py
index a71fbad7d..f6102c224 100644
--- a/youtube_dl/extractor/tagesschau.py
+++ b/youtube_dl/extractor/tagesschau.py
@@ -125,7 +125,7 @@ class TagesschauPlayerIE(InfoExtractor):
 
 
 class TagesschauIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?tagesschau\.de/[^/]+/(?:[^/]+/)*?[^/#?]+?(?P<id>-?[0-9]+)(?:~_?[^/#?]+?)?\.html'
+    _VALID_URL = r'https?://(?:www\.)?tagesschau\.de/(?P<path>[^/]+/(?:[^/]+/)*?[^/#?]+?(?P<id>-?[0-9]+)?)(?:~_?[^/#?]+?)?\.html'
 
     _TESTS = [{
         'url': 'http://www.tagesschau.de/multimedia/video/video-102143.html',
@@ -197,6 +197,9 @@ class TagesschauIE(InfoExtractor):
     }, {
         'url': 'http://www.tagesschau.de/multimedia/video/video-102303~_bab-sendung-211.html',
         'only_matching': True,
+    }, {
+        'url': 'http://www.tagesschau.de/100sekunden/index.html',
+        'only_matching': True,
     }]
 
     @classmethod
@@ -256,7 +259,8 @@ class TagesschauIE(InfoExtractor):
         return formats
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id') or mobj.group('path')
         display_id = video_id.lstrip('-')
 
         webpage = self._download_webpage(url, display_id)

From 854cc54bc1d0488d8fa88bd5dfed6f7f8981847e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 1 May 2016 07:01:55 +0600
Subject: [PATCH 4807/6219] [tagesschau] Expand video id

---
 youtube_dl/extractor/tagesschau.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/tagesschau.py b/youtube_dl/extractor/tagesschau.py
index f6102c224..499bd260b 100644
--- a/youtube_dl/extractor/tagesschau.py
+++ b/youtube_dl/extractor/tagesschau.py
@@ -125,13 +125,13 @@ class TagesschauPlayerIE(InfoExtractor):
 
 
 class TagesschauIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?tagesschau\.de/(?P<path>[^/]+/(?:[^/]+/)*?[^/#?]+?(?P<id>-?[0-9]+)?)(?:~_?[^/#?]+?)?\.html'
+    _VALID_URL = r'https?://(?:www\.)?tagesschau\.de/(?P<path>[^/]+/(?:[^/]+/)*?(?P<id>[^/#?]+?(?:-?[0-9]+)?))(?:~_?[^/#?]+?)?\.html'
 
     _TESTS = [{
         'url': 'http://www.tagesschau.de/multimedia/video/video-102143.html',
         'md5': 'f7c27a0eff3bfe8c7727e65f8fe1b1e6',
         'info_dict': {
-            'id': '102143',
+            'id': 'video-102143',
             'ext': 'mp4',
             'title': 'Regierungsumbildung in Athen: Neue Minister in Griechenland vereidigt',
             'description': '18.07.2015 20:10 Uhr',
@@ -141,7 +141,7 @@ class TagesschauIE(InfoExtractor):
         'url': 'http://www.tagesschau.de/multimedia/sendung/ts-5727.html',
         'md5': '3c54c1f6243d279b706bde660ceec633',
         'info_dict': {
-            'id': '5727',
+            'id': 'ts-5727',
             'ext': 'mp4',
             'title': 'Sendung: tagesschau \t04.12.2014 20:00 Uhr',
             'description': 'md5:695c01bfd98b7e313c501386327aea59',
@@ -152,7 +152,7 @@ class TagesschauIE(InfoExtractor):
         'url': 'http://www.tagesschau.de/multimedia/audio/audio-29417.html',
         'md5': '76e6eec6ebd40740671cf0a2c88617e5',
         'info_dict': {
-            'id': '29417',
+            'id': 'audio-29417',
             'ext': 'mp3',
             'title': 'Trabi - Bye, bye Rennpappe',
             'description': 'md5:8687dda862cbbe2cfb2df09b56341317',
@@ -163,7 +163,7 @@ class TagesschauIE(InfoExtractor):
         'url': 'http://www.tagesschau.de/inland/bnd-303.html',
         'md5': 'e0916c623e85fc1d2b26b78f299d3958',
         'info_dict': {
-            'id': '303',
+            'id': 'bnd-303',
             'ext': 'mp3',
             'title': 'Viele Baustellen für neuen BND-Chef',
             'description': 'md5:1e69a54be3e1255b2b07cdbce5bcd8b4',
@@ -172,7 +172,7 @@ class TagesschauIE(InfoExtractor):
     }, {
         'url': 'http://www.tagesschau.de/inland/afd-parteitag-135.html',
         'info_dict': {
-            'id': '135',
+            'id': 'afd-parteitag-135',
             'title': 'Möchtegern-Underdog mit Machtanspruch',
         },
         'playlist_count': 2,

From 68bb2fef9565159eba4a47f464b6b420cf2d5cda Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 1 May 2016 07:15:23 +0600
Subject: [PATCH 4808/6219] [tagesschau] Restrict playlist entry regex

---
 youtube_dl/extractor/tagesschau.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tagesschau.py b/youtube_dl/extractor/tagesschau.py
index 499bd260b..136e18f96 100644
--- a/youtube_dl/extractor/tagesschau.py
+++ b/youtube_dl/extractor/tagesschau.py
@@ -200,6 +200,10 @@ class TagesschauIE(InfoExtractor):
     }, {
         'url': 'http://www.tagesschau.de/100sekunden/index.html',
         'only_matching': True,
+    }, {
+        # playlist article with collapsing sections
+        'url': 'http://www.tagesschau.de/wirtschaft/faq-freihandelszone-eu-usa-101.html',
+        'only_matching': True,
     }]
 
     @classmethod
@@ -275,7 +279,7 @@ class TagesschauIE(InfoExtractor):
         if webpage_type == 'website':  # Article
             entries = []
             for num, (entry_title, media_kind, download_text) in enumerate(re.findall(
-                    r'(?s)<p[^>]+class="infotext"[^>]*>.*?<strong>(.+?)</strong>.*?</p>.*?%s' % DOWNLOAD_REGEX,
+                    r'(?s)<p[^>]+class="infotext"[^>]*>\s*(?:<a[^>]+>)?\s*<strong>(.+?)</strong>.*?</p>.*?%s' % DOWNLOAD_REGEX,
                     webpage), 1):
                 entries.append({
                     'id': '%s-%d' % (display_id, num),

From 6f27bf1c7425d97eb07aee9f7e15d0066b0a74bf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 1 May 2016 08:08:51 +0600
Subject: [PATCH 4809/6219] Credit @blahgeek for xiami (#9079)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 07cade723..814fe9ec3 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -168,3 +168,4 @@ José Joaquín Atria
 Viťas Strádal
 Kagami Hiiragi
 Philip Huppert
+blahgeek

From 4bd143a3a06264fcda5fa254709d404ccab6601c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 1 May 2016 10:56:54 +0600
Subject: [PATCH 4810/6219] [postprocessor/ffmpeg] Simplify metadata
 preparation and add track related metafields (Closes #9357)

---
 youtube_dl/postprocessor/ffmpeg.py | 41 +++++++++++++++++-------------
 1 file changed, 24 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 1793a878c..fa99b0c2a 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -389,23 +389,30 @@ class FFmpegEmbedSubtitlePP(FFmpegPostProcessor):
 class FFmpegMetadataPP(FFmpegPostProcessor):
     def run(self, info):
         metadata = {}
-        if info.get('title') is not None:
-            metadata['title'] = info['title']
-        if info.get('upload_date') is not None:
-            metadata['date'] = info['upload_date']
-        if info.get('artist') is not None:
-            metadata['artist'] = info['artist']
-        elif info.get('uploader') is not None:
-            metadata['artist'] = info['uploader']
-        elif info.get('uploader_id') is not None:
-            metadata['artist'] = info['uploader_id']
-        if info.get('description') is not None:
-            metadata['description'] = info['description']
-            metadata['comment'] = info['description']
-        if info.get('webpage_url') is not None:
-            metadata['purl'] = info['webpage_url']
-        if info.get('album') is not None:
-            metadata['album'] = info['album']
+
+        def add(meta_list, info_list=None):
+            if not info_list:
+                info_list = meta_list
+            if not isinstance(meta_list, (list, tuple)):
+                meta_list = (meta_list,)
+            if not isinstance(info_list, (list, tuple)):
+                info_list = (info_list,)
+            for info_f in info_list:
+                if info.get(info_f) is not None:
+                    for meta_f in meta_list:
+                        metadata[meta_f] = info[info_f]
+                    break
+
+        add('title', ('track', 'title'))
+        add('date', 'upload_date')
+        add(('description', 'comment'), 'description')
+        add('purl', 'webpage_url')
+        add('track', 'track_number')
+        add('artist', ('artist', 'creator', 'uploader', 'uploader_id'))
+        add('genre')
+        add('album')
+        add('album_artist')
+        add('disc', 'disc_number')
 
         if not metadata:
             self._downloader.to_screen('[ffmpeg] There isn\'t any metadata to add')

From 0d66bd0eab436f7215f5da168b378127898ccd66 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 1 May 2016 13:56:51 +0600
Subject: [PATCH 4811/6219] [downloader/hls] Delegate extraction to ffmpeg when
 unsupported features detected

---
 youtube_dl/downloader/hls.py | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index a01dac031..d7b34bde3 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -4,6 +4,7 @@ import os.path
 import re
 
 from .fragment import FragmentFD
+from .external import FFmpegFD
 
 from ..compat import compat_urlparse
 from ..utils import (
@@ -17,12 +18,34 @@ class HlsFD(FragmentFD):
 
     FD_NAME = 'hlsnative'
 
+    @staticmethod
+    def can_download(manifest):
+        UNSUPPORTED_FEATURES = (
+            r'#EXT-X-KEY:METHOD=(?!NONE)',  # encrypted streams [1]
+            r'#EXT-X-BYTERANGE',  # playlists composed of byte ranges of media files [2]
+            r'#EXT-X-MEDIA-SEQUENCE:(?!0$)',  # live streams [3]
+            # 1. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.2.4
+            # 2. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.2.2
+            # 3. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.3.2
+        )
+        return all(not re.search(feature, manifest) for feature in UNSUPPORTED_FEATURES)
+
     def real_download(self, filename, info_dict):
         man_url = info_dict['url']
         self.to_screen('[%s] Downloading m3u8 manifest' % self.FD_NAME)
         manifest = self.ydl.urlopen(man_url).read()
 
         s = manifest.decode('utf-8', 'ignore')
+
+        if not self.can_download(s):
+            self.report_warning(
+                'hlsnative has detected features it does not support, '
+                'extraction will be delegated to ffmpeg')
+            fd = FFmpegFD(self.ydl, self.params)
+            for ph in self._progress_hooks:
+                fd.add_progress_hook(ph)
+            return fd.real_download(filename, info_dict)
+
         fragment_urls = []
         for line in s.splitlines():
             line = line.strip()

From 174aba3223d0b01ec30cc0a814c6c0813ce13f15 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 May 2016 10:19:14 +0200
Subject: [PATCH 4812/6219] release 2016.05.01

---
 .github/ISSUE_TEMPLATE.md |  6 +++---
 docs/supportedsites.md    | 14 +++++++++-----
 youtube_dl/version.py     |  2 +-
 3 files changed, 13 insertions(+), 9 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index c208eb689..a26ff1de4 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.04.24*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.04.24**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.05.01*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.05.01**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.04.24
+[debug] youtube-dl version 2016.05.01
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 03875b8db..9fb43671f 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -338,7 +338,6 @@
  - **mailru**: Видео@Mail.Ru
  - **MakersChannel**
  - **MakerTV**
- - **Malemotion**
  - **MatchTV**
  - **MDR**: MDR.DE and KiKA
  - **media.ccc.de**
@@ -375,8 +374,8 @@
  - **mtvservices:embedded**
  - **MuenchenTV**: münchen.tv
  - **MusicPlayOn**
- - **muzu.tv**
  - **Mwave**
+ - **MwaveMeetGreet**
  - **MySpace**
  - **MySpace:album**
  - **MySpass**
@@ -554,7 +553,6 @@
  - **SenateISVP**
  - **ServingSys**
  - **Sexu**
- - **SexyKarma**: Sexy Karma and Watch Indian Porn
  - **Shahid**
  - **Shared**: shared.sx and vivo.sx
  - **ShareSix**
@@ -567,8 +565,6 @@
  - **smotri:broadcast**: Smotri.com broadcasts
  - **smotri:community**: Smotri.com community videos
  - **smotri:user**: Smotri.com user videos
- - **SnagFilms**
- - **SnagFilmsEmbed**
  - **Snotr**
  - **Sohu**
  - **soundcloud**
@@ -610,6 +606,7 @@
  - **Syfy**
  - **SztvHu**
  - **Tagesschau**
+ - **tagesschau:player**
  - **Tapely**
  - **Tass**
  - **TDSLifeway**
@@ -725,6 +722,8 @@
  - **Vidzi**
  - **vier**
  - **vier:videos**
+ - **ViewLift**
+ - **ViewLiftEmbed**
  - **Viewster**
  - **Viidea**
  - **viki**
@@ -756,6 +755,7 @@
  - **Walla**
  - **WashingtonPost**
  - **wat.tv**
+ - **WatchIndianPorn**: Watch Indian Porn
  - **WDR**
  - **wdr:mobile**
  - **WDRMaus**: Sendung mit der Maus
@@ -775,6 +775,10 @@
  - **XFileShare**: XFileShare based sites: GorillaVid.in, daclips.in, movpod.in, fastvideo.in, realvid.net, filehoot.com and vidto.me
  - **XHamster**
  - **XHamsterEmbed**
+ - **xiami:album**: 虾米音乐 - 专辑
+ - **xiami:artist**: 虾米音乐 - 歌手
+ - **xiami:collection**: 虾米音乐 - 精选集
+ - **xiami:song**: 虾米音乐
  - **XMinus**
  - **XNXX**
  - **Xstream**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 8befd9607..551160897 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.04.24'
+__version__ = '2016.05.01'

From e0da32df6e924535dbbe1e1c8ad05386dcf93029 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 2 May 2016 00:48:26 +0600
Subject: [PATCH 4813/6219] [vevo:playlist] Add extractor (Closes #9334, closes
 #9364)

---
 youtube_dl/extractor/extractors.py |  5 +-
 youtube_dl/extractor/vevo.py       | 74 +++++++++++++++++++++++++++++-
 2 files changed, 76 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 4aee53d6a..ef4431364 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -849,7 +849,10 @@ from .veehd import VeeHDIE
 from .veoh import VeohIE
 from .vessel import VesselIE
 from .vesti import VestiIE
-from .vevo import VevoIE
+from .vevo import (
+    VevoIE,
+    VevoPlaylistIE,
+)
 from .vgtv import (
     BTArticleIE,
     BTVestlendingenIE,
diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py
index 147480f64..4ad1e87e4 100644
--- a/youtube_dl/extractor/vevo.py
+++ b/youtube_dl/extractor/vevo.py
@@ -3,7 +3,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_etree_fromstring
+from ..compat import (
+    compat_etree_fromstring,
+    compat_urlparse,
+)
 from ..utils import (
     ExtractorError,
     int_or_none,
@@ -18,7 +21,7 @@ class VevoIE(InfoExtractor):
     (currently used by MTVIE and MySpaceIE)
     '''
     _VALID_URL = r'''(?x)
-        (?:https?://www\.vevo\.com/watch/(?:[^/]+/(?:[^/]+/)?)?|
+        (?:https?://www\.vevo\.com/watch/(?!playlist|genre)(?:[^/]+/(?:[^/]+/)?)?|
            https?://cache\.vevo\.com/m/html/embed\.html\?video=|
            https?://videoplayer\.vevo\.com/embed/embedded\?videoId=|
            vevo:)
@@ -301,3 +304,70 @@ class VevoIE(InfoExtractor):
             'view_count': view_count,
             'age_limit': age_limit,
         }
+
+
+class VevoPlaylistIE(InfoExtractor):
+    _VALID_URL = r'https?://www\.vevo\.com/watch/(?:playlist|genre)/(?P<id>[^/?#&]+)'
+
+    _TESTS = [{
+        'url': 'http://www.vevo.com/watch/playlist/dadbf4e7-b99f-4184-9670-6f0e547b6a29',
+        'info_dict': {
+            'id': 'dadbf4e7-b99f-4184-9670-6f0e547b6a29',
+            'title': 'Best-Of: Birdman',
+        },
+        'playlist_count': 10,
+        'params': {
+            'proxy': '52.53.186.253:8083',
+            'no_check_certificate': True,
+        },
+    }, {
+        'url': 'http://www.vevo.com/watch/playlist/dadbf4e7-b99f-4184-9670-6f0e547b6a29?index=0',
+        'md5': '32dcdfddddf9ec6917fc88ca26d36282',
+        'info_dict': {
+            'id': 'USCMV1100073',
+            'ext': 'mp4',
+            'title': 'Y.U. MAD',
+            'timestamp': 1323417600,
+            'upload_date': '20111209',
+            'uploader': 'Birdman',
+        },
+        'expected_warnings': ['Unable to download SMIL file'],
+        'params': {
+            'proxy': '52.53.186.253:8083',
+            'no_check_certificate': True,
+        },
+    }, {
+        'url': 'http://www.vevo.com/watch/genre/rock?index=0',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        playlist_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, playlist_id)
+
+        qs = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
+        index = qs.get('index', [None])[0]
+
+        if index:
+            video_id = self._search_regex(
+                r'<meta[^>]+content=(["\'])vevo://video/(?P<id>.+?)\1[^>]*>',
+                webpage, 'video id', default=None, group='id')
+            if video_id:
+                return self.url_result('vevo:%s' % video_id, VevoIE.ie_key())
+
+        playlists = self._parse_json(
+            self._search_regex(
+                r'window\.__INITIAL_STORE__\s*=\s*({.+?});\s*</script>',
+                webpage, 'initial store'),
+            playlist_id)['default']['playlists']
+
+        playlist = list(playlists.values())[0]
+
+        entries = [
+            self.url_result('vevo:%s' % src, VevoIE.ie_key())
+            for src in playlist['isrcs']]
+
+        return self.playlist_result(
+            entries, playlist.get('playlistId'),
+            playlist.get('name'), playlist.get('description'))

From 0c9d288ba0db3c1309bc6090457e11375073f3eb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 2 May 2016 00:50:31 +0600
Subject: [PATCH 4814/6219] [vevo:playlist] Remove debug params

---
 youtube_dl/extractor/vevo.py | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py
index 4ad1e87e4..8431077ad 100644
--- a/youtube_dl/extractor/vevo.py
+++ b/youtube_dl/extractor/vevo.py
@@ -316,10 +316,6 @@ class VevoPlaylistIE(InfoExtractor):
             'title': 'Best-Of: Birdman',
         },
         'playlist_count': 10,
-        'params': {
-            'proxy': '52.53.186.253:8083',
-            'no_check_certificate': True,
-        },
     }, {
         'url': 'http://www.vevo.com/watch/playlist/dadbf4e7-b99f-4184-9670-6f0e547b6a29?index=0',
         'md5': '32dcdfddddf9ec6917fc88ca26d36282',
@@ -332,10 +328,6 @@ class VevoPlaylistIE(InfoExtractor):
             'uploader': 'Birdman',
         },
         'expected_warnings': ['Unable to download SMIL file'],
-        'params': {
-            'proxy': '52.53.186.253:8083',
-            'no_check_certificate': True,
-        },
     }, {
         'url': 'http://www.vevo.com/watch/genre/rock?index=0',
         'only_matching': True,

From e2bd301ce7795597a7e3ef7f5a5446f9ec987883 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 2 May 2016 01:00:42 +0600
Subject: [PATCH 4815/6219] [vevo:playlist] Fix genre playlists

---
 youtube_dl/extractor/vevo.py | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py
index 8431077ad..2d1ff05e1 100644
--- a/youtube_dl/extractor/vevo.py
+++ b/youtube_dl/extractor/vevo.py
@@ -307,7 +307,7 @@ class VevoIE(InfoExtractor):
 
 
 class VevoPlaylistIE(InfoExtractor):
-    _VALID_URL = r'https?://www\.vevo\.com/watch/(?:playlist|genre)/(?P<id>[^/?#&]+)'
+    _VALID_URL = r'https?://www\.vevo\.com/watch/(?P<kind>playlist|genre)/(?P<id>[^/?#&]+)'
 
     _TESTS = [{
         'url': 'http://www.vevo.com/watch/playlist/dadbf4e7-b99f-4184-9670-6f0e547b6a29',
@@ -316,6 +316,13 @@ class VevoPlaylistIE(InfoExtractor):
             'title': 'Best-Of: Birdman',
         },
         'playlist_count': 10,
+    }, {
+        'url': 'http://www.vevo.com/watch/genre/rock',
+        'info_dict': {
+            'id': 'rock',
+            'title': 'Rock',
+        },
+        'playlist_count': 20,
     }, {
         'url': 'http://www.vevo.com/watch/playlist/dadbf4e7-b99f-4184-9670-6f0e547b6a29?index=0',
         'md5': '32dcdfddddf9ec6917fc88ca26d36282',
@@ -334,7 +341,9 @@ class VevoPlaylistIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        playlist_id = self._match_id(url)
+        mobj = re.match(self._VALID_URL, url)
+        playlist_id = mobj.group('id')
+        playlist_kind = mobj.group('kind')
 
         webpage = self._download_webpage(url, playlist_id)
 
@@ -352,9 +361,10 @@ class VevoPlaylistIE(InfoExtractor):
             self._search_regex(
                 r'window\.__INITIAL_STORE__\s*=\s*({.+?});\s*</script>',
                 webpage, 'initial store'),
-            playlist_id)['default']['playlists']
+            playlist_id)['default']['%ss' % playlist_kind]
 
-        playlist = list(playlists.values())[0]
+        playlist = (list(playlists.values())[0]
+                    if playlist_kind == 'playlist' else playlists[playlist_id])
 
         entries = [
             self.url_result('vevo:%s' % src, VevoIE.ie_key())

From 516ea41a7dd5a350e93ea7cc5ca2c1fcbd0cb43b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 2 May 2016 02:54:50 +0600
Subject: [PATCH 4816/6219] [vevo] Fix _call_api

---
 youtube_dl/extractor/vevo.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py
index 2d1ff05e1..35f974c4e 100644
--- a/youtube_dl/extractor/vevo.py
+++ b/youtube_dl/extractor/vevo.py
@@ -149,8 +149,8 @@ class VevoIE(InfoExtractor):
         auth_info = self._parse_json(webpage, video_id)
         self._api_url_template = self.http_scheme() + '//apiv2.vevo.com/%s?token=' + auth_info['access_token']
 
-    def _call_api(self, path, video_id, note, errnote, fatal=True):
-        return self._download_json(self._api_url_template % path, video_id, note, errnote)
+    def _call_api(self, path, *args, **kwargs):
+        return self._download_json(self._api_url_template % path, *args, **kwargs)
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From 9618c448247a6aa528b4bf2f289d3dd164c11417 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 2 May 2016 02:58:20 +0600
Subject: [PATCH 4817/6219] [vevo] Extract video versions from webpage as a
 last resort (Closes #8426, closes #9366)

---
 youtube_dl/extractor/vevo.py | 28 ++++++++++++++++++++--------
 1 file changed, 20 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py
index 35f974c4e..b6c6ba89f 100644
--- a/youtube_dl/extractor/vevo.py
+++ b/youtube_dl/extractor/vevo.py
@@ -15,7 +15,16 @@ from ..utils import (
 )
 
 
-class VevoIE(InfoExtractor):
+class VevoBaseIE(InfoExtractor):
+    def _extract_json(self, webpage, video_id, item):
+        return self._parse_json(
+            self._search_regex(
+                r'window\.__INITIAL_STORE__\s*=\s*({.+?});\s*</script>',
+                webpage, 'initial store'),
+            video_id)['default'][item]
+
+
+class VevoIE(VevoBaseIE):
     '''
     Accepts urls from vevo.com or in the format 'vevo:{id}'
     (currently used by MTVIE and MySpaceIE)
@@ -186,7 +195,14 @@ class VevoIE(InfoExtractor):
             video_versions = self._call_api(
                 'video/%s/streams' % video_id, video_id,
                 'Downloading video versions info',
-                'Failed to download video versions info')
+                'Failed to download video versions info',
+                fatal=False)
+
+            # Some videos are only available via webpage (e.g.
+            # https://github.com/rg3/youtube-dl/issues/9366)
+            if not video_versions:
+                webpage = self._download_webpage(url, video_id)
+                video_versions = self._extract_json(webpage, video_id, 'streams')[video_id][0]
 
             timestamp = parse_iso8601(video_info.get('releaseDate'))
             artists = video_info.get('artists')
@@ -306,7 +322,7 @@ class VevoIE(InfoExtractor):
         }
 
 
-class VevoPlaylistIE(InfoExtractor):
+class VevoPlaylistIE(VevoBaseIE):
     _VALID_URL = r'https?://www\.vevo\.com/watch/(?P<kind>playlist|genre)/(?P<id>[^/?#&]+)'
 
     _TESTS = [{
@@ -357,11 +373,7 @@ class VevoPlaylistIE(InfoExtractor):
             if video_id:
                 return self.url_result('vevo:%s' % video_id, VevoIE.ie_key())
 
-        playlists = self._parse_json(
-            self._search_regex(
-                r'window\.__INITIAL_STORE__\s*=\s*({.+?});\s*</script>',
-                webpage, 'initial store'),
-            playlist_id)['default']['%ss' % playlist_kind]
+        playlists = self._extract_json(webpage, playlist_id, '%ss' % playlist_kind)
 
         playlist = (list(playlists.values())[0]
                     if playlist_kind == 'playlist' else playlists[playlist_id])

From 8e7d0048886f374a58f0fe8ba021644d7074d02c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 2 May 2016 03:06:48 +0600
Subject: [PATCH 4818/6219] [vevo] Add test for video only available via
 webpage

---
 youtube_dl/extractor/vevo.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py
index b6c6ba89f..4eb98e50d 100644
--- a/youtube_dl/extractor/vevo.py
+++ b/youtube_dl/extractor/vevo.py
@@ -84,6 +84,20 @@ class VevoIE(VevoBaseIE):
             'uploader': 'K Camp',
             'timestamp': 1449468000,
         },
+    }, {
+        'note': 'Only available via webpage',
+        'url': 'http://www.vevo.com/watch/GBUV71600656',
+        'md5': '67e79210613865b66a47c33baa5e37fe',
+        'info_dict': {
+            'id': 'GBUV71600656',
+            'ext': 'mp4',
+            'title': 'Viva Love',
+            'upload_date': '20160428',
+            'age_limit': 0,
+            'uploader': 'ABC',
+            'timestamp': 1461830400,
+        },
+        'expected_warnings': ['Failed to download video versions info'],
     }]
     _SMIL_BASE_URL = 'http://smil.lvl3.vevo.com'
     _SOURCE_TYPES = {

From 881dbc86c4e70252a5b8e5c726a6f2a32ff878c1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 2 May 2016 03:28:58 +0600
Subject: [PATCH 4819/6219] [vevo] Extract track related metafields and add
 artists to title (Closes #1684)

---
 youtube_dl/extractor/vevo.py | 58 ++++++++++++++++++++++++++----------
 1 file changed, 42 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py
index 4eb98e50d..49cb3f479 100644
--- a/youtube_dl/extractor/vevo.py
+++ b/youtube_dl/extractor/vevo.py
@@ -42,11 +42,15 @@ class VevoIE(VevoBaseIE):
         'info_dict': {
             'id': 'GB1101300280',
             'ext': 'mp4',
-            'title': 'Somebody to Die For',
+            'title': 'Hurts - Somebody to Die For',
+            'timestamp': 1372057200,
             'upload_date': '20130624',
             'uploader': 'Hurts',
-            'timestamp': 1372057200,
+            'track': 'Somebody to Die For',
+            'artist': 'Hurts',
+            'genre': 'Pop',
         },
+        'expected_warnings': ['Unable to download SMIL file'],
     }, {
         'note': 'v3 SMIL format',
         'url': 'http://www.vevo.com/watch/cassadee-pope/i-wish-i-could-break-your-heart/USUV71302923',
@@ -54,23 +58,31 @@ class VevoIE(VevoBaseIE):
         'info_dict': {
             'id': 'USUV71302923',
             'ext': 'mp4',
-            'title': 'I Wish I Could Break Your Heart',
+            'title': 'Cassadee Pope - I Wish I Could Break Your Heart',
+            'timestamp': 1392796919,
             'upload_date': '20140219',
             'uploader': 'Cassadee Pope',
-            'timestamp': 1392796919,
+            'track': 'I Wish I Could Break Your Heart',
+            'artist': 'Cassadee Pope',
+            'genre': 'Country',
         },
+        'expected_warnings': ['Unable to download SMIL file'],
     }, {
         'note': 'Age-limited video',
         'url': 'https://www.vevo.com/watch/justin-timberlake/tunnel-vision-explicit/USRV81300282',
         'info_dict': {
             'id': 'USRV81300282',
             'ext': 'mp4',
-            'title': 'Tunnel Vision (Explicit)',
-            'upload_date': '20130703',
+            'title': 'Justin Timberlake - Tunnel Vision (Explicit)',
             'age_limit': 18,
-            'uploader': 'Justin Timberlake',
             'timestamp': 1372888800,
+            'upload_date': '20130703',
+            'uploader': 'Justin Timberlake',
+            'track': 'Tunnel Vision (Explicit)',
+            'artist': 'Justin Timberlake',
+            'genre': 'Pop',
         },
+        'expected_warnings': ['Unable to download SMIL file'],
     }, {
         'note': 'No video_info',
         'url': 'http://www.vevo.com/watch/k-camp-1/Till-I-Die/USUV71503000',
@@ -78,11 +90,14 @@ class VevoIE(VevoBaseIE):
         'info_dict': {
             'id': 'USUV71503000',
             'ext': 'mp4',
-            'title': 'Till I Die',
-            'upload_date': '20151207',
+            'title': 'K Camp - Till I Die',
             'age_limit': 18,
-            'uploader': 'K Camp',
             'timestamp': 1449468000,
+            'upload_date': '20151207',
+            'uploader': 'K Camp',
+            'track': 'Till I Die',
+            'artist': 'K Camp',
+            'genre': 'Rap/Hip-Hop',
         },
     }, {
         'note': 'Only available via webpage',
@@ -91,11 +106,14 @@ class VevoIE(VevoBaseIE):
         'info_dict': {
             'id': 'GBUV71600656',
             'ext': 'mp4',
-            'title': 'Viva Love',
-            'upload_date': '20160428',
+            'title': 'ABC - Viva Love',
             'age_limit': 0,
-            'uploader': 'ABC',
             'timestamp': 1461830400,
+            'upload_date': '20160428',
+            'uploader': 'ABC',
+            'track': 'Viva Love',
+            'artist': 'ABC',
+            'genre': 'Pop',
         },
         'expected_warnings': ['Failed to download video versions info'],
     }]
@@ -184,8 +202,8 @@ class VevoIE(VevoBaseIE):
         video_info = response.get('video') or {}
         video_versions = video_info.get('videoVersions')
         uploader = None
-        timestamp = None
         view_count = None
+        timestamp = None
         formats = []
 
         if not video_info:
@@ -311,7 +329,9 @@ class VevoIE(VevoBaseIE):
                         smil_parsed = True
         self._sort_formats(formats)
 
-        title = video_info['title']
+        track = video_info['title']
+        title = '%s - %s' % (uploader, track) if uploader else track
+        genre = video_info.get('genres', [None])[0]
 
         is_explicit = video_info.get('isExplicit')
         if is_explicit is True:
@@ -333,6 +353,9 @@ class VevoIE(VevoBaseIE):
             'duration': duration,
             'view_count': view_count,
             'age_limit': age_limit,
+            'track': track,
+            'artist': uploader,
+            'genre': genre,
         }
 
 
@@ -359,10 +382,13 @@ class VevoPlaylistIE(VevoBaseIE):
         'info_dict': {
             'id': 'USCMV1100073',
             'ext': 'mp4',
-            'title': 'Y.U. MAD',
+            'title': 'Birdman - Y.U. MAD',
             'timestamp': 1323417600,
             'upload_date': '20111209',
             'uploader': 'Birdman',
+            'track': 'Y.U. MAD',
+            'artist': 'Birdman',
+            'genre': 'Rap/Hip-Hop',
         },
         'expected_warnings': ['Unable to download SMIL file'],
     }, {

From 78a3ff33ab686bc6fc75735e2b2a5935b80311be Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 2 May 2016 03:29:48 +0600
Subject: [PATCH 4820/6219] [vevo:playlist] Add fallback for playlist id

---
 youtube_dl/extractor/vevo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py
index 49cb3f479..bbe29fc51 100644
--- a/youtube_dl/extractor/vevo.py
+++ b/youtube_dl/extractor/vevo.py
@@ -423,5 +423,5 @@ class VevoPlaylistIE(VevoBaseIE):
             for src in playlist['isrcs']]
 
         return self.playlist_result(
-            entries, playlist.get('playlistId'),
+            entries, playlist.get('playlistId') or playlist_id,
             playlist.get('name'), playlist.get('description'))

From 9508738f9a9d6fd3de2e60cd7ccb4c8631bea6f4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 2 May 2016 03:36:40 +0600
Subject: [PATCH 4821/6219] [vevo] Extract featured artist

---
 youtube_dl/extractor/vevo.py | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py
index bbe29fc51..63eab4148 100644
--- a/youtube_dl/extractor/vevo.py
+++ b/youtube_dl/extractor/vevo.py
@@ -201,6 +201,8 @@ class VevoIE(VevoBaseIE):
             json_url, video_id, 'Downloading video info', 'Unable to download info')
         video_info = response.get('video') or {}
         video_versions = video_info.get('videoVersions')
+        artist = None
+        featured_artist = None
         uploader = None
         view_count = None
         timestamp = None
@@ -239,7 +241,7 @@ class VevoIE(VevoBaseIE):
             timestamp = parse_iso8601(video_info.get('releaseDate'))
             artists = video_info.get('artists')
             if artists:
-                uploader = artists[0]['name']
+                artist = uploader = artists[0]['name']
             view_count = int_or_none(video_info.get('views', {}).get('total'))
 
             for video_version in video_versions:
@@ -292,7 +294,11 @@ class VevoIE(VevoBaseIE):
                 scale=1000)
             artists = video_info.get('mainArtists')
             if artists:
-                uploader = artists[0]['artistName']
+                artist = uploader = artists[0]['artistName']
+
+            featured_artists = video_info.get('featuredArtists')
+            if featured_artists:
+                featured_artist = featured_artists[0]['artistName']
 
             smil_parsed = False
             for video_version in video_info['videoVersions']:
@@ -330,7 +336,9 @@ class VevoIE(VevoBaseIE):
         self._sort_formats(formats)
 
         track = video_info['title']
-        title = '%s - %s' % (uploader, track) if uploader else track
+        if featured_artist:
+            artist = '%s ft. %s' % (artist, featured_artist)
+        title = '%s - %s' % (artist, track) if artist else track
         genre = video_info.get('genres', [None])[0]
 
         is_explicit = video_info.get('isExplicit')

From 686cc8963441c37105c0447f31c5ea21405be05a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 2 May 2016 07:07:35 +0600
Subject: [PATCH 4822/6219] [discovery] Fix typo

---
 youtube_dl/extractor/discovery.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/discovery.py b/youtube_dl/extractor/discovery.py
index 7c554ec14..55853f76f 100644
--- a/youtube_dl/extractor/discovery.py
+++ b/youtube_dl/extractor/discovery.py
@@ -71,7 +71,7 @@ class DiscoveryIE(InfoExtractor):
         entries = []
 
         for idx, video_info in enumerate(info['playlist']):
-            subtitles = []
+            subtitles = {}
             caption_url = video_info.get('captionsUrl')
             if caption_url:
                 subtitles = {

From 79a2e94e79e65cdf4898bc2dedb6a1bb4ca9af3c Mon Sep 17 00:00:00 2001
From: Adam Thalhammer <s3544305@student.rmit.edu.au>
Date: Mon, 2 May 2016 13:21:39 +1000
Subject: [PATCH 4823/6219] Instead of replacing accented characters with an
 underscore when sanitizing file names in restricted mode, replace them with
 their non-accented equivalents fixes #9347

---
 test/test_utils.py  | 9 +++++++--
 youtube_dl/utils.py | 9 +++++++--
 2 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index e16a6761b..0072ba241 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -139,8 +139,8 @@ class TestUtil(unittest.TestCase):
         self.assertEqual('yes_no', sanitize_filename('yes? no', restricted=True))
         self.assertEqual('this_-_that', sanitize_filename('this: that', restricted=True))
 
-        tests = 'a\xe4b\u4e2d\u56fd\u7684c'
-        self.assertEqual(sanitize_filename(tests, restricted=True), 'a_b_c')
+        tests = 'aäb\u4e2d\u56fd\u7684c'
+        self.assertEqual(sanitize_filename(tests, restricted=True), 'aab_c')
         self.assertTrue(sanitize_filename('\xf6', restricted=True) != '')  # No empty filename
 
         forbidden = '"\0\\/&!: \'\t\n()[]{}$;`^,#'
@@ -155,6 +155,11 @@ class TestUtil(unittest.TestCase):
         self.assertTrue(sanitize_filename('-', restricted=True) != '')
         self.assertTrue(sanitize_filename(':', restricted=True) != '')
 
+        self.assertEqual(sanitize_filename(
+            'ÂÃÄÀÁÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖØÙÚÛÜÝÞßàáâãäåæçèéêëìíîïðñòóôõöøùúûüýþÿ', restricted=True),
+            'AAAAAAAECEEEEIIIIDNOOOOOOUUUUYPssaaaaaaaeceeeeiiiionoooooouuuuypy')
+        pass
+
     def test_sanitize_ids(self):
         self.assertEqual(sanitize_filename('_n_cd26wFpw', is_id=True), '_n_cd26wFpw')
         self.assertEqual(sanitize_filename('_BD_eEpuzXw', is_id=True), '_BD_eEpuzXw')
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 7bcc85e2b..f74f62268 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -14,8 +14,8 @@ import email.utils
 import errno
 import functools
 import gzip
-import itertools
 import io
+import itertools
 import json
 import locale
 import math
@@ -24,8 +24,8 @@ import os
 import pipes
 import platform
 import re
-import ssl
 import socket
+import ssl
 import struct
 import subprocess
 import sys
@@ -365,6 +365,11 @@ def sanitize_filename(s, restricted=False, is_id=False):
     Set is_id if this is not an arbitrary string, but an ID that should be kept if possible
     """
     def replace_insane(char):
+        accents = dict(zip('ÂÃÄÀÁÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖØÙÚÛÜÝÞßàáâãäåæçèéêëìíîïðñòóôõöøùúûüýþÿ',
+                           itertools.chain('AAAAAA', ['AE'], 'CEEEEIIIIDNOOOOOOUUUUYP', ['ss'],
+                                           'aaaaaa', ['ae'], 'ceeeeiiiionoooooouuuuypy')))
+        if restricted and char in accents:
+            return accents[char]
         if char == '?' or ord(char) < 32 or ord(char) == 127:
             return ''
         elif char == '"':

From 31c4448f6ea8ec4236e0b67af33b53db1a93ef13 Mon Sep 17 00:00:00 2001
From: Adam Thalhammer <s3544305@student.rmit.edu.au>
Date: Mon, 2 May 2016 13:25:12 +1000
Subject: [PATCH 4824/6219] Instead of replacing accented characters with an
 underscore when sanitizing file names in restricted mode, replace them with
 their non-accented equivalents fixes #9347

---
 test/test_utils.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 0072ba241..00ada95ec 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -158,7 +158,6 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(sanitize_filename(
             'ÂÃÄÀÁÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖØÙÚÛÜÝÞßàáâãäåæçèéêëìíîïðñòóôõöøùúûüýþÿ', restricted=True),
             'AAAAAAAECEEEEIIIIDNOOOOOOUUUUYPssaaaaaaaeceeeeiiiionoooooouuuuypy')
-        pass
 
     def test_sanitize_ids(self):
         self.assertEqual(sanitize_filename('_n_cd26wFpw', is_id=True), '_n_cd26wFpw')

From 5c9ced9504bd2ceb8e55a929124aad2091b23403 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 2 May 2016 18:19:00 +0600
Subject: [PATCH 4825/6219] [vevo] Improve genre extraction

---
 youtube_dl/extractor/vevo.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py
index 63eab4148..3cb0343e6 100644
--- a/youtube_dl/extractor/vevo.py
+++ b/youtube_dl/extractor/vevo.py
@@ -5,6 +5,7 @@ import re
 from .common import InfoExtractor
 from ..compat import (
     compat_etree_fromstring,
+    compat_str,
     compat_urlparse,
 )
 from ..utils import (
@@ -116,6 +117,10 @@ class VevoIE(VevoBaseIE):
             'genre': 'Pop',
         },
         'expected_warnings': ['Failed to download video versions info'],
+    }, {
+        # no genres available
+        'url': 'http://www.vevo.com/watch/INS171400764',
+        'only_matching': True,
     }]
     _SMIL_BASE_URL = 'http://smil.lvl3.vevo.com'
     _SOURCE_TYPES = {
@@ -339,7 +344,11 @@ class VevoIE(VevoBaseIE):
         if featured_artist:
             artist = '%s ft. %s' % (artist, featured_artist)
         title = '%s - %s' % (artist, track) if artist else track
-        genre = video_info.get('genres', [None])[0]
+
+        genres = video_info.get('genres')
+        genre = (
+            genres[0] if genres and isinstance(genres, list) and
+            isinstance(genres[0], compat_str) else None)
 
         is_explicit = video_info.get('isExplicit')
         if is_explicit is True:

From 7960b0563b957d418ddd36555275d98ba4668c03 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 2 May 2016 18:35:50 +0600
Subject: [PATCH 4826/6219] [YoutubeDL] Properly process
 unable-to-download-error on python2

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 055433362..4e57c9687 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1639,7 +1639,7 @@ class YoutubeDL(object):
                     # Just a single file
                     success = dl(filename, info_dict)
             except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
-                self.report_error('unable to download video data: %s' % str(err))
+                self.report_error('unable to download video data: %s' % error_to_compat_str(err))
                 return
             except (OSError, IOError) as err:
                 raise UnavailableVideoError(err)

From df5f4e8888bc02f6064b9b92fbf4cfc4eedd4c1f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 2 May 2016 18:47:35 +0600
Subject: [PATCH 4827/6219] [vevo] Remove superfluous code

---
 youtube_dl/extractor/vevo.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py
index 3cb0343e6..a6177f2cb 100644
--- a/youtube_dl/extractor/vevo.py
+++ b/youtube_dl/extractor/vevo.py
@@ -205,12 +205,10 @@ class VevoIE(VevoBaseIE):
         response = self._download_json(
             json_url, video_id, 'Downloading video info', 'Unable to download info')
         video_info = response.get('video') or {}
-        video_versions = video_info.get('videoVersions')
         artist = None
         featured_artist = None
         uploader = None
         view_count = None
-        timestamp = None
         formats = []
 
         if not video_info:

From f0e14fdd43bf8e86c5385220430eef842a10ccab Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 2 May 2016 20:05:06 +0600
Subject: [PATCH 4828/6219] [YoutubeDL] Skip non-relevant field types when
 building output template

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 4e57c9687..2187dcc8f 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -580,7 +580,7 @@ class YoutubeDL(object):
                 is_id=(k == 'id'))
             template_dict = dict((k, sanitize(k, v))
                                  for k, v in template_dict.items()
-                                 if v is not None)
+                                 if v is not None and not isinstance(v, (list, tuple, dict)))
             template_dict = collections.defaultdict(lambda: 'NA', template_dict)
 
             outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)

From 8a92e51c60fd122e676e4619e7e132b692292801 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 2 May 2016 21:31:35 +0600
Subject: [PATCH 4829/6219] [extractor/common] Relax wording for creator
 metafield

---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 61a5d124c..0843d89af 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -163,7 +163,7 @@ class InfoExtractor(object):
     description:    Full video description.
     uploader:       Full name of the video uploader.
     license:        License name the video is licensed under.
-    creator:        The main artist who created the video.
+    creator:        The creator of the video.
     release_date:   The date (YYYYMMDD) when the video was released.
     timestamp:      UNIX timestamp of the moment the video became available.
     upload_date:    Video upload date (YYYYMMDD).

From 6c52a86f54b230a3f08dd10a89f55b8af4d98ee3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 2 May 2016 21:32:57 +0600
Subject: [PATCH 4830/6219] [README.md] Update creator description

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index ecf737047..50acb26a0 100644
--- a/README.md
+++ b/README.md
@@ -465,7 +465,7 @@ The basic usage is not to set any template arguments when downloading a single f
  - `display_id`: An alternative identifier for the video
  - `uploader`: Full name of the video uploader
  - `license`: License name the video is licensed under
- - `creator`: The main artist who created the video
+ - `creator`: The creator of the video
  - `release_date`: The date (YYYYMMDD) when the video was released
  - `timestamp`: UNIX timestamp of the moment the video became available
  - `upload_date`: Video upload date (YYYYMMDD)

From c587cbb793a6eda4fc7b1c7b4163e236abec1a00 Mon Sep 17 00:00:00 2001
From: Adam Thalhammer <s3544305@student.rmit.edu.au>
Date: Tue, 3 May 2016 10:40:30 +1000
Subject: [PATCH 4831/6219] improved performance by extracting accented chars
 to top level

---
 youtube_dl/utils.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index f74f62268..a5922b2b5 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -89,6 +89,11 @@ KNOWN_EXTENSIONS = (
     'wav',
     'f4f', 'f4m', 'm3u8', 'smil')
 
+# needed for sanitizing filenames in restricted mode
+ACCENT_CHARS = dict(zip('ÂÃÄÀÁÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖØÙÚÛÜÝÞßàáâãäåæçèéêëìíîïðñòóôõöøùúûüýþÿ',
+                        itertools.chain('AAAAAA', ['AE'], 'CEEEEIIIIDNOOOOOOUUUUYP', ['ss'],
+                                        'aaaaaa', ['ae'], 'ceeeeiiiionoooooouuuuypy')))
+
 
 def preferredencoding():
     """Get preferred encoding.
@@ -365,11 +370,8 @@ def sanitize_filename(s, restricted=False, is_id=False):
     Set is_id if this is not an arbitrary string, but an ID that should be kept if possible
     """
     def replace_insane(char):
-        accents = dict(zip('ÂÃÄÀÁÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖØÙÚÛÜÝÞßàáâãäåæçèéêëìíîïðñòóôõöøùúûüýþÿ',
-                           itertools.chain('AAAAAA', ['AE'], 'CEEEEIIIIDNOOOOOOUUUUYP', ['ss'],
-                                           'aaaaaa', ['ae'], 'ceeeeiiiionoooooouuuuypy')))
-        if restricted and char in accents:
-            return accents[char]
+        if restricted and char in ACCENT_CHARS:
+            return ACCENT_CHARS[char]
         if char == '?' or ord(char) < 32 or ord(char) == 127:
             return ''
         elif char == '"':

From a0a309b9734ea3d5f66d4a52e42f5cc24df7f808 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 3 May 2016 16:06:28 +0800
Subject: [PATCH 4832/6219] [kuwo:category] Fix description and update test

---
 youtube_dl/extractor/kuwo.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/kuwo.py b/youtube_dl/extractor/kuwo.py
index 3740869c7..616ed19e1 100644
--- a/youtube_dl/extractor/kuwo.py
+++ b/youtube_dl/extractor/kuwo.py
@@ -266,7 +266,6 @@ class KuwoCategoryIE(InfoExtractor):
         'info_dict': {
             'id': '86375',
             'title': '八十年代精选',
-            'description': '这些都是属于八十年代的回忆！',
         },
         'playlist_mincount': 24,
     }
@@ -283,6 +282,8 @@ class KuwoCategoryIE(InfoExtractor):
         category_desc = remove_start(
             get_element_by_id('intro', webpage).strip(),
             '%s简介：' % category_name)
+        if category_desc == '暂无':
+            category_desc = None
 
         jsonm = self._parse_json(self._html_search_regex(
             r'var\s+jsonm\s*=\s*([^;]+);', webpage, 'category songs'), category_id)

From 7759be38da1b5ad99a8ef04d9a1df21b989e3b8c Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 3 May 2016 16:19:20 +0800
Subject: [PATCH 4833/6219] [xiami] Detect georestriction and skip tests

---
 youtube_dl/extractor/xiami.py | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/xiami.py b/youtube_dl/extractor/xiami.py
index e4ed306b4..a6dfc4af9 100644
--- a/youtube_dl/extractor/xiami.py
+++ b/youtube_dl/extractor/xiami.py
@@ -9,6 +9,11 @@ from ..utils import int_or_none
 class XiamiBaseIE(InfoExtractor):
     _API_BASE_URL = 'http://www.xiami.com/song/playlist/cat/json/id'
 
+    def _download_webpage(self, *args, **kwargs):
+        webpage = super(XiamiBaseIE, self)._download_webpage(*args, **kwargs)
+        if '>Xiami is currently not available in your country.<' in webpage:
+            self.raise_geo_restricted('Xiami is currently not available in your country')
+
     def _extract_track(self, track, track_id=None):
         title = track['title']
         track_url = self._decrypt(track['location'])
@@ -81,7 +86,8 @@ class XiamiSongIE(XiamiBaseIE):
                     'ext': 'lrc',
                 }],
             },
-        }
+        },
+        'skip': 'Georestricted',
     }, {
         'url': 'http://www.xiami.com/song/1775256504',
         'md5': '932a3abd45c6aa2b1fdbe028fcb4c4fc',
@@ -100,7 +106,8 @@ class XiamiSongIE(XiamiBaseIE):
                     'ext': 'lrc',
                 }],
             },
-        }
+        },
+        'skip': 'Georestricted',
     }]
 
     def _real_extract(self, url):
@@ -124,6 +131,7 @@ class XiamiAlbumIE(XiamiPlaylistBaseIE):
             'id': '2100300444',
         },
         'playlist_count': 10,
+        'skip': 'Georestricted',
     }, {
         'url': 'http://www.xiami.com/album/512288?spm=a1z1s.6843761.1110925389.6.hhE9p9',
         'only_matching': True,
@@ -141,6 +149,7 @@ class XiamiArtistIE(XiamiPlaylistBaseIE):
             'id': '2132',
         },
         'playlist_count': 20,
+        'skip': 'Georestricted',
     }
 
 
@@ -155,4 +164,5 @@ class XiamiCollectionIE(XiamiPlaylistBaseIE):
             'id': '156527391',
         },
         'playlist_mincount': 29,
+        'skip': 'Georestricted',
     }

From 80bc4106aff21c46cf65c7310f1ed988518b3df5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 3 May 2016 15:09:23 +0600
Subject: [PATCH 4834/6219] [xfileshare] Add support for thevideobee.to (Closes
 #9374)

---
 youtube_dl/extractor/xfileshare.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/xfileshare.py b/youtube_dl/extractor/xfileshare.py
index 2d1504eaa..472ed1b3f 100644
--- a/youtube_dl/extractor/xfileshare.py
+++ b/youtube_dl/extractor/xfileshare.py
@@ -16,7 +16,7 @@ class XFileShareIE(InfoExtractor):
     IE_DESC = 'XFileShare based sites: GorillaVid.in, daclips.in, movpod.in, fastvideo.in, realvid.net, filehoot.com and vidto.me'
     _VALID_URL = r'''(?x)
         https?://(?P<host>(?:www\.)?
-            (?:daclips\.in|gorillavid\.in|movpod\.in|fastvideo\.in|realvid\.net|filehoot\.com|vidto\.me|powerwatch\.pw))/
+            (?:daclips\.in|gorillavid\.in|movpod\.in|fastvideo\.in|realvid\.net|filehoot\.com|vidto\.me|powerwatch\.pw|thevideobee\.to))/
         (?:embed-)?(?P<id>[0-9a-zA-Z]+)(?:-[0-9]+x[0-9]+\.html)?
     '''
 

From 41745523918dee174a3e642b629bcbc585931c4a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 3 May 2016 15:35:32 +0600
Subject: [PATCH 4835/6219] [xfileshare] Refactor _VALID_URL and remove ded
 sites

---
 youtube_dl/extractor/xfileshare.py | 39 +++++++++++-------------------
 1 file changed, 14 insertions(+), 25 deletions(-)

diff --git a/youtube_dl/extractor/xfileshare.py b/youtube_dl/extractor/xfileshare.py
index 472ed1b3f..4ab42d24e 100644
--- a/youtube_dl/extractor/xfileshare.py
+++ b/youtube_dl/extractor/xfileshare.py
@@ -13,12 +13,20 @@ from ..utils import (
 
 
 class XFileShareIE(InfoExtractor):
-    IE_DESC = 'XFileShare based sites: GorillaVid.in, daclips.in, movpod.in, fastvideo.in, realvid.net, filehoot.com and vidto.me'
-    _VALID_URL = r'''(?x)
-        https?://(?P<host>(?:www\.)?
-            (?:daclips\.in|gorillavid\.in|movpod\.in|fastvideo\.in|realvid\.net|filehoot\.com|vidto\.me|powerwatch\.pw|thevideobee\.to))/
-        (?:embed-)?(?P<id>[0-9a-zA-Z]+)(?:-[0-9]+x[0-9]+\.html)?
-    '''
+    _SITES = (
+        ('daclips.in', 'DaClips'),
+        ('filehoot.com', 'FileHoot'),
+        ('gorillavid.in', 'GorillaVid'),
+        ('movpod.in', 'MovPod'),
+        ('powerwatch.pw', 'PowerWatch'),
+        ('rapidvideo.ws', 'Rapidvideo.ws'),
+        ('thevideobee.to', 'TheVideoBee'),
+        ('vidto.me', 'Vidto'),
+    )
+
+    IE_DESC = 'XFileShare based sites: %s' % ', '.join(list(zip(*_SITES))[1])
+    _VALID_URL = (r'https?://(?P<host>(?:www\.)?(?:%s))/(?:embed-)?(?P<id>[0-9a-zA-Z]+)'
+                  % '|'.join(re.escape(site) for site in list(zip(*_SITES))[0]))
 
     _FILE_NOT_FOUND_REGEX = r'>(?:404 - )?File Not Found<'
 
@@ -43,25 +51,6 @@ class XFileShareIE(InfoExtractor):
             'title': 'Micro Pig piglets ready on 16th July 2009-bG0PdrCdxUc',
             'thumbnail': 're:http://.*\.jpg',
         }
-    }, {
-        # video with countdown timeout
-        'url': 'http://fastvideo.in/1qmdn1lmsmbw',
-        'md5': '8b87ec3f6564a3108a0e8e66594842ba',
-        'info_dict': {
-            'id': '1qmdn1lmsmbw',
-            'ext': 'mp4',
-            'title': 'Man of Steel - Trailer',
-            'thumbnail': 're:http://.*\.jpg',
-        },
-    }, {
-        'url': 'http://realvid.net/ctn2y6p2eviw',
-        'md5': 'b2166d2cf192efd6b6d764c18fd3710e',
-        'info_dict': {
-            'id': 'ctn2y6p2eviw',
-            'ext': 'flv',
-            'title': 'rdx 1955',
-            'thumbnail': 're:http://.*\.jpg',
-        },
     }, {
         'url': 'http://movpod.in/0wguyyxi1yca',
         'only_matching': True,

From 57d8e32a3ec7fe70522edad6fd0c2847b4e00944 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 3 May 2016 16:58:11 +0600
Subject: [PATCH 4836/6219] [xfileshare] Add support for streamin.to

---
 youtube_dl/extractor/xfileshare.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/xfileshare.py b/youtube_dl/extractor/xfileshare.py
index 4ab42d24e..769003735 100644
--- a/youtube_dl/extractor/xfileshare.py
+++ b/youtube_dl/extractor/xfileshare.py
@@ -22,6 +22,7 @@ class XFileShareIE(InfoExtractor):
         ('rapidvideo.ws', 'Rapidvideo.ws'),
         ('thevideobee.to', 'TheVideoBee'),
         ('vidto.me', 'Vidto'),
+        ('streamin.to', 'Streamin.To'),
     )
 
     IE_DESC = 'XFileShare based sites: %s' % ', '.join(list(zip(*_SITES))[1])

From 964f49336fcf94b3a5399a026db3914b27a2a445 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 3 May 2016 21:24:51 +0600
Subject: [PATCH 4837/6219] [aol] Improve _VALID_URL (Closes #9381)

---
 youtube_dl/extractor/aol.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/aol.py b/youtube_dl/extractor/aol.py
index 24df8fe93..42c21bf41 100644
--- a/youtube_dl/extractor/aol.py
+++ b/youtube_dl/extractor/aol.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 class AolIE(InfoExtractor):
     IE_NAME = 'on.aol.com'
-    _VALID_URL = r'(?:aol-video:|https?://on\.aol\.com/.*-)(?P<id>[^/?-]+)'
+    _VALID_URL = r'(?:aol-video:|https?://on\.aol\.com/(?:[^/]+/)*(?:[^/?#&]+-)?)(?P<id>[^/?#&]+)'
 
     _TESTS = [{
         # video with 5min ID
@@ -53,6 +53,12 @@ class AolIE(InfoExtractor):
     }, {
         'url': 'http://on.aol.com/shows/park-bench-shw518173474-559a1b9be4b0c3bfad3357a7?context=SH:SHW518173474:PL4327:1460619712763',
         'only_matching': True,
+    }, {
+        'url': 'http://on.aol.com/video/519442220',
+        'only_matching': True,
+    }, {
+        'url': 'aol-video:5707d6b8e4b090497b04f706',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From bc7e77a04be6094e64263f9c622cff3cd1fc13cb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 3 May 2016 23:18:36 +0600
Subject: [PATCH 4838/6219] [vevo] Use raise_geo_restricted

---
 youtube_dl/extractor/vevo.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py
index a6177f2cb..c0ef08c02 100644
--- a/youtube_dl/extractor/vevo.py
+++ b/youtube_dl/extractor/vevo.py
@@ -189,8 +189,8 @@ class VevoIE(VevoBaseIE):
             errnote='Unable to retrieve oauth token')
 
         if 'THIS PAGE IS CURRENTLY UNAVAILABLE IN YOUR REGION' in webpage:
-            raise ExtractorError(
-                '%s said: This page is currently unavailable in your region.' % self.IE_NAME, expected=True)
+            self.raise_geo_restricted(
+                '%s said: This page is currently unavailable in your region' % self.IE_NAME)
 
         auth_info = self._parse_json(webpage, video_id)
         self._api_url_template = self.http_scheme() + '//apiv2.vevo.com/%s?token=' + auth_info['access_token']

From e960c3c223acadb2fac81fb68595d902cf21e349 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 4 May 2016 22:25:39 +0600
Subject: [PATCH 4839/6219] [yandexmusic:playlist] Improve extraction (Closes
 #6801)

---
 youtube_dl/extractor/yandexmusic.py | 38 +++++++++++++++++++----------
 1 file changed, 25 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/yandexmusic.py b/youtube_dl/extractor/yandexmusic.py
index ce3723b55..22050add3 100644
--- a/youtube_dl/extractor/yandexmusic.py
+++ b/youtube_dl/extractor/yandexmusic.py
@@ -177,7 +177,7 @@ class YandexMusicAlbumIE(YandexMusicPlaylistBaseIE):
 class YandexMusicPlaylistIE(YandexMusicPlaylistBaseIE):
     IE_NAME = 'yandexmusic:playlist'
     IE_DESC = 'Яндекс.Музыка - Плейлист'
-    _VALID_URL = r'https?://music\.yandex\.(?:ru|kz|ua|by)/users/[^/]+/playlists/(?P<id>\d+)'
+    _VALID_URL = r'https?://music\.yandex\.(?P<tld>ru|kz|ua|by)/users/(?P<user>[^/]+)/playlists/(?P<id>\d+)'
 
     _TESTS = [{
         'url': 'http://music.yandex.ru/users/music.partners/playlists/1245',
@@ -201,19 +201,32 @@ class YandexMusicPlaylistIE(YandexMusicPlaylistBaseIE):
     }]
 
     def _real_extract(self, url):
-        playlist_id = self._match_id(url)
+        mobj = re.match(self._VALID_URL, url)
+        tld = mobj.group('tld')
+        user = mobj.group('user')
+        playlist_id = mobj.group('id')
 
-        webpage = self._download_webpage(url, playlist_id)
+        playlist = self._download_json(
+            'https://music.yandex.%s/handlers/playlist.jsx' % tld,
+            playlist_id, 'Downloading missing tracks JSON',
+            fatal=False,
+            headers={
+                'Referer': url,
+                'X-Requested-With': 'XMLHttpRequest',
+                'X-Retpath-Y': url,
+            },
+            query={
+                'owner': user,
+                'kinds': playlist_id,
+                'light': 'true',
+                'lang': tld,
+                'external-domain': 'music.yandex.%s' % tld,
+                'overembed': 'false',
+            })['playlist']
 
-        mu = self._parse_json(
-            self._search_regex(
-                r'var\s+Mu\s*=\s*({.+?});\s*</script>', webpage, 'player'),
-            playlist_id)
-
-        playlist = mu['pageData']['playlist']
         tracks, track_ids = playlist['tracks'], playlist['trackIds']
 
-        # tracks dictionary shipped with webpage is limited to 150 tracks,
+        # tracks dictionary shipped with playlist.jsx API is limited to 150 tracks,
         # missing tracks should be retrieved manually.
         if len(tracks) < len(track_ids):
             present_track_ids = set([compat_str(track['id']) for track in tracks if track.get('id')])
@@ -222,10 +235,9 @@ class YandexMusicPlaylistIE(YandexMusicPlaylistBaseIE):
                 'https://music.yandex.ru/handlers/track-entries.jsx',
                 urlencode_postdata({
                     'entries': ','.join(missing_track_ids),
-                    'lang': mu.get('settings', {}).get('lang', 'en'),
-                    'external-domain': 'music.yandex.ru',
+                    'lang': tld,
+                    'external-domain': 'music.yandex.%s' % tld,
                     'overembed': 'false',
-                    'sign': mu.get('authData', {}).get('user', {}).get('sign'),
                     'strict': 'true',
                 }))
             request.add_header('Referer', url)

From 15fc0658f75403e76f9cb29dd4ce5d1e514d4bf4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 4 May 2016 22:33:29 +0600
Subject: [PATCH 4840/6219] [yandexmusic:playlist] Modernize

---
 youtube_dl/extractor/yandexmusic.py | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/yandexmusic.py b/youtube_dl/extractor/yandexmusic.py
index 22050add3..0dda901a8 100644
--- a/youtube_dl/extractor/yandexmusic.py
+++ b/youtube_dl/extractor/yandexmusic.py
@@ -231,20 +231,21 @@ class YandexMusicPlaylistIE(YandexMusicPlaylistBaseIE):
         if len(tracks) < len(track_ids):
             present_track_ids = set([compat_str(track['id']) for track in tracks if track.get('id')])
             missing_track_ids = set(map(compat_str, track_ids)) - set(present_track_ids)
-            request = sanitized_Request(
-                'https://music.yandex.ru/handlers/track-entries.jsx',
-                urlencode_postdata({
+            missing_tracks = self._download_json(
+                'https://music.yandex.%s/handlers/track-entries.jsx' % tld,
+                playlist_id, 'Downloading missing tracks JSON',
+                fatal=False,
+                headers={
+                    'Referer': url,
+                    'X-Requested-With': 'XMLHttpRequest',
+                },
+                query={
                     'entries': ','.join(missing_track_ids),
                     'lang': tld,
                     'external-domain': 'music.yandex.%s' % tld,
                     'overembed': 'false',
                     'strict': 'true',
-                }))
-            request.add_header('Referer', url)
-            request.add_header('X-Requested-With', 'XMLHttpRequest')
-
-            missing_tracks = self._download_json(
-                request, playlist_id, 'Downloading missing tracks JSON', fatal=False)
+                })
             if missing_tracks:
                 tracks.extend(missing_tracks)
 

From d36724cca47c0dbbe691d5467c0c88e28a6627db Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 4 May 2016 22:34:37 +0600
Subject: [PATCH 4841/6219] [yandexmusic:playlist] Remove unused imports

---
 youtube_dl/extractor/yandexmusic.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/youtube_dl/extractor/yandexmusic.py b/youtube_dl/extractor/yandexmusic.py
index 0dda901a8..a9ebbbc04 100644
--- a/youtube_dl/extractor/yandexmusic.py
+++ b/youtube_dl/extractor/yandexmusic.py
@@ -10,8 +10,6 @@ from ..utils import (
     ExtractorError,
     int_or_none,
     float_or_none,
-    sanitized_Request,
-    urlencode_postdata,
 )
 
 

From 203a3c0e6a589627f9b0f0c4c6ec56e24c27272c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 4 May 2016 22:35:28 +0600
Subject: [PATCH 4842/6219] [yandexmusic:playlist] Make title optional

---
 youtube_dl/extractor/yandexmusic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/yandexmusic.py b/youtube_dl/extractor/yandexmusic.py
index a9ebbbc04..6e7a23782 100644
--- a/youtube_dl/extractor/yandexmusic.py
+++ b/youtube_dl/extractor/yandexmusic.py
@@ -250,4 +250,4 @@ class YandexMusicPlaylistIE(YandexMusicPlaylistBaseIE):
         return self.playlist_result(
             self._build_playlist(tracks),
             compat_str(playlist_id),
-            playlist['title'], playlist.get('description'))
+            playlist.get('title'), playlist.get('description'))

From 2a48e6f01a20545cd117cacb78463dba87d97b21 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 4 May 2016 22:45:01 +0600
Subject: [PATCH 4843/6219] [yandexmusic:playlist] Respect track order for long
 (>150) playlists

---
 youtube_dl/extractor/yandexmusic.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/yandexmusic.py b/youtube_dl/extractor/yandexmusic.py
index 6e7a23782..283b55a91 100644
--- a/youtube_dl/extractor/yandexmusic.py
+++ b/youtube_dl/extractor/yandexmusic.py
@@ -222,13 +222,17 @@ class YandexMusicPlaylistIE(YandexMusicPlaylistBaseIE):
                 'overembed': 'false',
             })['playlist']
 
-        tracks, track_ids = playlist['tracks'], playlist['trackIds']
+        tracks, track_ids = playlist['tracks'], map(compat_str, playlist['trackIds'])
 
         # tracks dictionary shipped with playlist.jsx API is limited to 150 tracks,
         # missing tracks should be retrieved manually.
         if len(tracks) < len(track_ids):
-            present_track_ids = set([compat_str(track['id']) for track in tracks if track.get('id')])
-            missing_track_ids = set(map(compat_str, track_ids)) - set(present_track_ids)
+            present_track_ids = set([
+                compat_str(track['id'])
+                for track in tracks if track.get('id')])
+            missing_track_ids = [
+                track_id for track_id in track_ids
+                if track_id not in present_track_ids]
             missing_tracks = self._download_json(
                 'https://music.yandex.%s/handlers/track-entries.jsx' % tld,
                 playlist_id, 'Downloading missing tracks JSON',

From aabdc83d6e9ea1e6a28c48cb228a644c55fc11b3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 4 May 2016 23:03:44 +0600
Subject: [PATCH 4844/6219] [udemy] Fix course enroll (Closes #9393)

---
 youtube_dl/extractor/udemy.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index d1e6f2703..7ebd231f0 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -84,7 +84,8 @@ class UdemyIE(InfoExtractor):
         if enroll_url:
             webpage = self._download_webpage(
                 combine_url(base_url, enroll_url),
-                course_id, 'Enrolling in the course')
+                course_id, 'Enrolling in the course',
+                headers={'Referer': base_url})
             if '>You have enrolled in' in webpage:
                 self.to_screen('%s: Successfully enrolled in the course' % course_id)
 

From 75b81df3af31bdbe794b55924dd9ab85e1314d04 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 4 May 2016 23:14:12 +0600
Subject: [PATCH 4845/6219] [udemy] Modernize

---
 youtube_dl/extractor/udemy.py | 25 ++++++++++++-------------
 1 file changed, 12 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index 7ebd231f0..13e0cd237 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -5,7 +5,6 @@ import re
 from .common import InfoExtractor
 from ..compat import (
     compat_HTTPError,
-    compat_urllib_parse_urlencode,
     compat_urllib_request,
     compat_urlparse,
 )
@@ -91,12 +90,12 @@ class UdemyIE(InfoExtractor):
 
     def _download_lecture(self, course_id, lecture_id):
         return self._download_json(
-            'https://www.udemy.com/api-2.0/users/me/subscribed-courses/%s/lectures/%s?%s' % (
-                course_id, lecture_id, compat_urllib_parse_urlencode({
-                    'fields[lecture]': 'title,description,view_html,asset',
-                    'fields[asset]': 'asset_type,stream_url,thumbnail_url,download_urls,data',
-                })),
-            lecture_id, 'Downloading lecture JSON')
+            'https://www.udemy.com/api-2.0/users/me/subscribed-courses/%s/lectures/%s?'
+            % (course_id, lecture_id),
+            lecture_id, 'Downloading lecture JSON', query={
+                'fields[lecture]': 'title,description,view_html,asset',
+                'fields[asset]': 'asset_type,stream_url,thumbnail_url,download_urls,data',
+            })
 
     def _handle_error(self, response):
         if not isinstance(response, dict):
@@ -156,13 +155,13 @@ class UdemyIE(InfoExtractor):
             'password': password,
         })
 
-        request = sanitized_Request(
-            self._LOGIN_URL, urlencode_postdata(login_form))
-        request.add_header('Referer', self._ORIGIN_URL)
-        request.add_header('Origin', self._ORIGIN_URL)
-
         response = self._download_webpage(
-            request, None, 'Logging in as %s' % username)
+            self._LOGIN_URL, None, 'Logging in as %s' % username,
+            data=urlencode_postdata(login_form),
+            headers={
+                'Referer': self._ORIGIN_URL,
+                'Origin': self._ORIGIN_URL,
+            })
 
         if not is_logged(response):
             error = self._html_search_regex(

From 9da526aae75fc7bbc07e791b9f4eab78c007636c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 4 May 2016 23:18:48 +0600
Subject: [PATCH 4846/6219] [yandexmusic:playlist] Update test

---
 youtube_dl/extractor/yandexmusic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/yandexmusic.py b/youtube_dl/extractor/yandexmusic.py
index 283b55a91..0f78466e6 100644
--- a/youtube_dl/extractor/yandexmusic.py
+++ b/youtube_dl/extractor/yandexmusic.py
@@ -194,7 +194,7 @@ class YandexMusicPlaylistIE(YandexMusicPlaylistBaseIE):
             'id': '1036',
             'title': 'Музыка 90-х',
         },
-        'playlist_count': 310,
+        'playlist_mincount': 300,
         'skip': 'Travis CI servers blocked by YandexMusic',
     }]
 

From 57cf9b7f0689ed7b643ce863427c3211e407e5bf Mon Sep 17 00:00:00 2001
From: Peter Rowlands <peter@pmrowla.com>
Date: Thu, 5 May 2016 03:11:04 +0900
Subject: [PATCH 4847/6219] [afreecatv] Add new extractor for afreecatv.com
 VODs

---
 youtube_dl/extractor/afreecatv.py  | 84 ++++++++++++++++++++++++++++++
 youtube_dl/extractor/extractors.py |  1 +
 2 files changed, 85 insertions(+)
 create mode 100644 youtube_dl/extractor/afreecatv.py

diff --git a/youtube_dl/extractor/afreecatv.py b/youtube_dl/extractor/afreecatv.py
new file mode 100644
index 000000000..d57546e49
--- /dev/null
+++ b/youtube_dl/extractor/afreecatv.py
@@ -0,0 +1,84 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..compat import (
+    compat_urllib_parse_urlparse,
+    compat_urlparse,
+)
+from ..utils import (
+    ExtractorError,
+    int_or_none,
+)
+
+
+class AfreecaTVIE(InfoExtractor):
+    IE_DESC = 'afreecatv.com'
+    _VALID_URL = r'''(?x)^
+        https?://(?:(live|afbbs|www)\.)?afreeca(?:tv)?\.com(?::\d+)?
+        (?:
+            /app/(?:index|read_ucc_bbs)\.cgi|
+            /player/[Pp]layer\.(?:swf|html))
+        \?.*?\bnTitleNo=(?P<id>\d+)'''
+    _TEST = {
+        'url': 'http://live.afreecatv.com:8079/app/index.cgi?szType=read_ucc_bbs&szBjId=dailyapril&nStationNo=16711924&nBbsNo=18605867&nTitleNo=36164052&szSkin=',
+        'md5': 'f72c89fe7ecc14c1b5ce506c4996046e',
+        'info_dict': {
+            'id': '36164052',
+            'ext': 'mp4',
+            'title': '데일리 에이프릴 요정들의 시상식!',
+            'thumbnail': 're:^https?://videoimg.afreecatv.com/.*$',
+            'uploader': 'dailyapril',
+            'uploader_id': 'dailyapril',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        parsed_url = compat_urllib_parse_urlparse(url)
+        info_url = compat_urlparse.urlunparse(parsed_url._replace(
+            netloc='afbbs.afreecatv.com:8080',
+            path='/api/video/get_video_info.php'))
+        video_xml = self._download_xml(info_url, video_id)
+
+        track = video_xml.find('track')
+        if track.find('flag').text != 'SUCCEED':
+            raise ExtractorError('Specified AfreecaTV video does not exist',
+                                 expected=True)
+        title = track.find('title').text
+        uploader = track.find('nickname').text
+        uploader_id = track.find('bj_id').text
+        duration = int_or_none(track.find('duration').text)
+        thumbnail = track.find('titleImage').text
+
+        entries = []
+        for video in track.findall('video'):
+            for video_file in video.findall('file'):
+                entries.append({
+                    'id': video_file.get('key'),
+                    'title': title,
+                    'duration': int_or_none(video_file.get('duration')),
+                    'formats': [{'url': video_file.text}]
+                })
+
+        info = {
+            'id': video_id,
+            'title': title,
+            'uploader': uploader,
+            'uploader_id': uploader_id,
+            'duration': duration,
+            'thumbnail': thumbnail,
+        }
+
+        if len(entries) > 1:
+            info['_type'] = 'multi_video'
+            info['entries'] = entries
+        elif len(entries) == 1:
+            info['formats'] = entries[0]['formats']
+        else:
+            raise ExtractorError(
+                'No files found for the specified AfreecaTV video, either'
+                ' the URL is incorrect or the video has been made private.',
+                expected=True)
+
+        return info
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index ef4431364..f85d75933 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -16,6 +16,7 @@ from .adobetv import (
     AdobeTVVideoIE,
 )
 from .adultswim import AdultSwimIE
+from .afreecatv import AfreecaTVIE
 from .aenetworks import AENetworksIE
 from .aftonbladet import AftonbladetIE
 from .airmozilla import AirMozillaIE

From 4f8c56eb4e52d0a61a5facc6f22e36a2e420f4c9 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 5 May 2016 17:55:37 +0800
Subject: [PATCH 4848/6219] [fczenit] Fix extraction and update test

Closes #9359
---
 youtube_dl/extractor/fczenit.py | 33 ++++++++++++++++++++-------------
 1 file changed, 20 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/fczenit.py b/youtube_dl/extractor/fczenit.py
index f1f150ef2..8d1010b88 100644
--- a/youtube_dl/extractor/fczenit.py
+++ b/youtube_dl/extractor/fczenit.py
@@ -1,20 +1,19 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
+from ..compat import compat_urlparse
 
 
 class FczenitIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?fc-zenit\.ru/video/gl(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?fc-zenit\.ru/video/(?P<id>[0-9]+)'
     _TEST = {
-        'url': 'http://fc-zenit.ru/video/gl6785/',
-        'md5': '458bacc24549173fe5a5aa29174a5606',
+        'url': 'http://fc-zenit.ru/video/41044/',
+        'md5': '0e3fab421b455e970fa1aa3891e57df0',
         'info_dict': {
-            'id': '6785',
+            'id': '41044',
             'ext': 'mp4',
-            'title': '«Зенит-ТВ»: как Олег Шатов играл против «Урала»',
+            'title': 'Так пишется история: казанский разгром ЦСКА на «Зенит-ТВ»',
         },
     }
 
@@ -22,15 +21,23 @@ class FczenitIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        video_title = self._html_search_regex(r'<div class=\"photoalbum__title\">([^<]+)', webpage, 'title')
+        video_title = self._html_search_regex(
+            r'<[^>]+class=\"photoalbum__title\">([^<]+)', webpage, 'title')
 
-        bitrates_raw = self._html_search_regex(r'bitrates:.*\n(.*)\]', webpage, 'video URL')
-        bitrates = re.findall(r'url:.?\'(.+?)\'.*?bitrate:.?([0-9]{3}?)', bitrates_raw)
+        video_items = self._parse_json(self._search_regex(
+            r'arrPath\s*=\s*JSON\.parse\(\'(.+)\'\)', webpage, 'video items'),
+            video_id)
+
+        def merge_dicts(*dicts):
+            ret = {}
+            for a_dict in dicts:
+                ret.update(a_dict)
+            return ret
 
         formats = [{
-            'url': furl,
-            'tbr': tbr,
-        } for furl, tbr in bitrates]
+            'url': compat_urlparse.urljoin(url, video_url),
+            'tbr': int(tbr),
+        } for tbr, video_url in merge_dicts(*video_items).items()]
 
         self._sort_formats(formats)
 

From 758a05924180ad2d1197ee1a293f28363ace54aa Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 5 May 2016 13:12:28 +0100
Subject: [PATCH 4849/6219] [dailymail] Add new extractor(closes #2667)

---
 youtube_dl/extractor/dailymail.py  | 61 ++++++++++++++++++++++++++++++
 youtube_dl/extractor/extractors.py |  1 +
 2 files changed, 62 insertions(+)
 create mode 100644 youtube_dl/extractor/dailymail.py

diff --git a/youtube_dl/extractor/dailymail.py b/youtube_dl/extractor/dailymail.py
new file mode 100644
index 000000000..b60a1d813
--- /dev/null
+++ b/youtube_dl/extractor/dailymail.py
@@ -0,0 +1,61 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    determine_protocol,
+)
+
+
+class DailyMailIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?dailymail\.co\.uk/video/[^/]+/video-(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'http://www.dailymail.co.uk/video/sciencetech/video-1288527/Turn-video-impressionist-masterpiece.html',
+        'md5': '2f639d446394f53f3a33658b518b6615',
+        'info_dict': {
+            'id': '1288527',
+            'ext': 'mp4',
+            'title': 'Turn any video into an impressionist masterpiece',
+            'description': 'md5:88ddbcb504367987b2708bb38677c9d2',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        video_data = self._parse_json(self._search_regex(
+            r"data-opts='({.+?})'", webpage, 'video data'), video_id)
+        title = video_data['title']
+        video_sources = self._download_json(video_data.get(
+            'sources', {}).get('url') or 'http://www.dailymail.co.uk/api/player/%s/video-sources.json' % video_id, video_id)
+
+        formats = []
+        for rendition in video_sources['renditions']:
+            rendition_url = rendition.get('url')
+            if not rendition_url:
+                continue
+            tbr = int_or_none(rendition.get('encodingRate'), 1000)
+            container = rendition.get('videoContainer')
+            is_hls = container == 'M2TS'
+            protocol = 'm3u8_native' if is_hls else determine_protocol({'url': rendition_url})
+            formats.append({
+                'format_id': ('hls' if is_hls else protocol) + ('-%d' % tbr if tbr else ''),
+                'url': rendition_url,
+                'width': int_or_none(rendition.get('frameWidth')),
+                'height': int_or_none(rendition.get('frameHeight')),
+                'tbr': tbr,
+                'vcodec': rendition.get('videoCodec'),
+                'container': container,
+                'protocol': protocol,
+                'ext': 'mp4' if is_hls else None,
+            })
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': video_data.get('descr'),
+            'thumbnail': video_data.get('poster') or video_data.get('thumbnail'),
+            'formats': formats,
+        }
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index ef4431364..aac85066f 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -157,6 +157,7 @@ from .cspan import CSpanIE
 from .ctsnews import CtsNewsIE
 from .cultureunplugged import CultureUnpluggedIE
 from .cwtv import CWTVIE
+from .dailymail import DailyMailIE
 from .dailymotion import (
     DailymotionIE,
     DailymotionPlaylistIE,

From 7d08f6073d42c8623031615c4b7e0bb26136c128 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 5 May 2016 20:06:59 +0800
Subject: [PATCH 4850/6219] [kuwo:category] Update test

---
 youtube_dl/extractor/kuwo.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/kuwo.py b/youtube_dl/extractor/kuwo.py
index 616ed19e1..11b31a699 100644
--- a/youtube_dl/extractor/kuwo.py
+++ b/youtube_dl/extractor/kuwo.py
@@ -266,6 +266,7 @@ class KuwoCategoryIE(InfoExtractor):
         'info_dict': {
             'id': '86375',
             'title': '八十年代精选',
+            'description': '这些都是属于八十年代的回忆！',
         },
         'playlist_mincount': 24,
     }

From b1c6a5bac8e3daf233cc79f0172d907905290f24 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 5 May 2016 20:50:39 +0800
Subject: [PATCH 4851/6219] [Makefile] Remove more media files in `make clean`

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 06cffcb71..c9ce216d1 100644
--- a/Makefile
+++ b/Makefile
@@ -1,7 +1,7 @@
 all: youtube-dl README.md CONTRIBUTING.md README.txt youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtube-dl.fish supportedsites
 
 clean:
-	rm -rf youtube-dl.1.temp.md youtube-dl.1 youtube-dl.bash-completion README.txt MANIFEST build/ dist/ .coverage cover/ youtube-dl.tar.gz youtube-dl.zsh youtube-dl.fish youtube_dl/extractor/lazy_extractors.py *.dump *.part *.info.json *.mp4 *.flv *.mp3 *.avi CONTRIBUTING.md.tmp ISSUE_TEMPLATE.md.tmp youtube-dl youtube-dl.exe
+	rm -rf youtube-dl.1.temp.md youtube-dl.1 youtube-dl.bash-completion README.txt MANIFEST build/ dist/ .coverage cover/ youtube-dl.tar.gz youtube-dl.zsh youtube-dl.fish youtube_dl/extractor/lazy_extractors.py *.dump *.part *.info.json *.mp4 *.flv *.mp3 *.avi *.mkv *.webm CONTRIBUTING.md.tmp ISSUE_TEMPLATE.md.tmp youtube-dl youtube-dl.exe
 	find . -name "*.pyc" -delete
 	find . -name "*.class" -delete
 

From ac12e888f9d76c75666822a79c125db8577e5fa5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 5 May 2016 21:02:54 +0600
Subject: [PATCH 4852/6219] [redtube] Extract all formats, duration, upload
 date and view count (Closes #9397)

---
 youtube_dl/extractor/redtube.py | 59 +++++++++++++++++++++++++++------
 1 file changed, 49 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/redtube.py b/youtube_dl/extractor/redtube.py
index 7ba41ba59..1e532286c 100644
--- a/youtube_dl/extractor/redtube.py
+++ b/youtube_dl/extractor/redtube.py
@@ -1,7 +1,12 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..utils import ExtractorError
+from ..utils import (
+    ExtractorError,
+    int_or_none,
+    str_to_int,
+    unified_strdate,
+)
 
 
 class RedTubeIE(InfoExtractor):
@@ -13,6 +18,9 @@ class RedTubeIE(InfoExtractor):
             'id': '66418',
             'ext': 'mp4',
             'title': 'Sucked on a toilet',
+            'upload_date': '20120831',
+            'duration': 596,
+            'view_count': int,
             'age_limit': 18,
         }
     }
@@ -24,12 +32,40 @@ class RedTubeIE(InfoExtractor):
         if any(s in webpage for s in ['video-deleted-info', '>This video has been removed']):
             raise ExtractorError('Video %s has been removed' % video_id, expected=True)
 
-        video_url = self._html_search_regex(
-            r'<source src="(.+?)" type="video/mp4">', webpage, 'video URL')
-        video_title = self._html_search_regex(
-            r'<h1 class="videoTitle[^"]*">(.+?)</h1>',
-            webpage, 'title')
-        video_thumbnail = self._og_search_thumbnail(webpage)
+        title = self._html_search_regex(
+            (r'<h1 class="videoTitle[^"]*">(?P<title>.+?)</h1>',
+             r'videoTitle\s*:\s*(["\'])(?P<title>)\1'),
+            webpage, 'title', group='title')
+
+        formats = []
+        sources = self._parse_json(
+            self._search_regex(
+                r'sources\s*:\s*({.+?})', webpage, 'source', default='{}'),
+            video_id, fatal=False)
+        if sources and isinstance(sources, dict):
+            for format_id, format_url in sources.items():
+                if format_url:
+                    formats.append({
+                        'url': format_url,
+                        'format_id': format_id,
+                        'height': int_or_none(format_id),
+                    })
+        else:
+            video_url = self._html_search_regex(
+                r'<source src="(.+?)" type="video/mp4">', webpage, 'video URL')
+            formats.append({'url': video_url})
+        self._sort_formats(formats)
+
+        thumbnail = self._og_search_thumbnail(webpage)
+        upload_date = unified_strdate(self._search_regex(
+            r'<span[^>]+class="added-time"[^>]*>ADDED ([^<]+)<',
+            webpage, 'upload date', fatal=False))
+        duration = int_or_none(self._search_regex(
+            r'videoDuration\s*:\s*(\d+)', webpage, 'duration', fatal=False))
+        view_count = str_to_int(self._search_regex(
+            r'<span[^>]*>VIEWS</span></td>\s*<td>([\d,.]+)',
+            webpage, 'view count', fatal=False))
+
 
         # No self-labeling, but they describe themselves as
         # "Home of Videos Porno"
@@ -37,9 +73,12 @@ class RedTubeIE(InfoExtractor):
 
         return {
             'id': video_id,
-            'url': video_url,
             'ext': 'mp4',
-            'title': video_title,
-            'thumbnail': video_thumbnail,
+            'title': title,
+            'thumbnail': thumbnail,
+            'upload_date': upload_date,
+            'duration': duration,
+            'view_count': view_count,
             'age_limit': age_limit,
+            'formats': formats,
         }

From 915620fd6894d92f89f9e5c9362d20f94e787e57 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 5 May 2016 21:34:06 +0600
Subject: [PATCH 4853/6219] [redtube] PEP 8

---
 youtube_dl/extractor/redtube.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/redtube.py b/youtube_dl/extractor/redtube.py
index 1e532286c..721fc3a9e 100644
--- a/youtube_dl/extractor/redtube.py
+++ b/youtube_dl/extractor/redtube.py
@@ -66,7 +66,6 @@ class RedTubeIE(InfoExtractor):
             r'<span[^>]*>VIEWS</span></td>\s*<td>([\d,.]+)',
             webpage, 'view count', fatal=False))
 
-
         # No self-labeling, but they describe themselves as
         # "Home of Videos Porno"
         age_limit = 18

From 833b644fffae4d4cb0807591006e34ef963d9bc0 Mon Sep 17 00:00:00 2001
From: Peter Rowlands <peter@pmrowla.com>
Date: Fri, 6 May 2016 01:24:02 +0900
Subject: [PATCH 4854/6219] use xpath_text

---
 youtube_dl/extractor/afreecatv.py | 30 +++++++++++++++---------------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/afreecatv.py b/youtube_dl/extractor/afreecatv.py
index d57546e49..9f9399edc 100644
--- a/youtube_dl/extractor/afreecatv.py
+++ b/youtube_dl/extractor/afreecatv.py
@@ -9,6 +9,7 @@ from ..compat import (
 from ..utils import (
     ExtractorError,
     int_or_none,
+    xpath_text,
 )
 
 
@@ -41,25 +42,24 @@ class AfreecaTVIE(InfoExtractor):
             path='/api/video/get_video_info.php'))
         video_xml = self._download_xml(info_url, video_id)
 
-        track = video_xml.find('track')
-        if track.find('flag').text != 'SUCCEED':
+        if xpath_text(video_xml, './track/flag', default='FAIL') != 'SUCCEED':
             raise ExtractorError('Specified AfreecaTV video does not exist',
                                  expected=True)
-        title = track.find('title').text
-        uploader = track.find('nickname').text
-        uploader_id = track.find('bj_id').text
-        duration = int_or_none(track.find('duration').text)
-        thumbnail = track.find('titleImage').text
+        title = xpath_text(video_xml, './track/title', 'title')
+        uploader = xpath_text(video_xml, './track/nickname', 'uploader')
+        uploader_id = xpath_text(video_xml, './track/bj_id', 'uploader id')
+        duration = int_or_none(xpath_text(video_xml, './track/duration',
+                                          'duration'))
+        thumbnail = xpath_text(video_xml, './track/titleImage', 'thumbnail')
 
         entries = []
-        for video in track.findall('video'):
-            for video_file in video.findall('file'):
-                entries.append({
-                    'id': video_file.get('key'),
-                    'title': title,
-                    'duration': int_or_none(video_file.get('duration')),
-                    'formats': [{'url': video_file.text}]
-                })
+        for video_file in video_xml.findall('./track/video/file'):
+            entries.append({
+                'id': video_file.get('key'),
+                'title': title,
+                'duration': int_or_none(video_file.get('duration')),
+                'formats': [{'url': video_file.text}]
+            })
 
         info = {
             'id': video_id,

From 6f59aa934bd5842a57d4c27ace0f3735be18ac27 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 6 May 2016 02:14:39 +0800
Subject: [PATCH 4855/6219] [periscope:user] Add new extractor for user pages

Closes #9388
---
 youtube_dl/extractor/extractors.py |  5 +++-
 youtube_dl/extractor/periscope.py  | 37 ++++++++++++++++++++++++++++++
 2 files changed, 41 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index aac85066f..c9d1422e5 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -561,7 +561,10 @@ from .parliamentliveuk import ParliamentLiveUKIE
 from .patreon import PatreonIE
 from .pbs import PBSIE
 from .people import PeopleIE
-from .periscope import PeriscopeIE
+from .periscope import (
+    PeriscopeIE,
+    PeriscopeUserIE,
+)
 from .philharmoniedeparis import PhilharmonieDeParisIE
 from .phoenix import PhoenixIE
 from .photobucket import PhotobucketIE
diff --git a/youtube_dl/extractor/periscope.py b/youtube_dl/extractor/periscope.py
index 514e9b433..0a4bc761d 100644
--- a/youtube_dl/extractor/periscope.py
+++ b/youtube_dl/extractor/periscope.py
@@ -7,6 +7,7 @@ from ..utils import parse_iso8601
 
 class PeriscopeIE(InfoExtractor):
     IE_DESC = 'Periscope'
+    IE_NAME = 'periscope'
     _VALID_URL = r'https?://(?:www\.)?periscope\.tv/[^/]+/(?P<id>[^/?#]+)'
     # Alive example URLs can be found here http://onperiscope.com/
     _TESTS = [{
@@ -79,3 +80,39 @@ class PeriscopeIE(InfoExtractor):
             'thumbnails': thumbnails,
             'formats': formats,
         }
+
+
+class PeriscopeUserIE(InfoExtractor):
+    _VALID_URL = r'https?://www\.periscope\.tv/(?P<id>[^/]+)/?$'
+    IE_DESC = 'Periscope user videos'
+    IE_NAME = 'periscope:user'
+
+    _TEST = {
+        'url': 'https://www.periscope.tv/LularoeHusbandMike/',
+        'info_dict': {
+            'id': 'LularoeHusbandMike',
+            'title': 'LULAROE HUSBAND MIKE',
+        },
+        # Periscope only shows videos in the last 24 hours, so it's possible to
+        # get 0 videos
+        'playlist_mincount': 0,
+    }
+
+    def _real_extract(self, url):
+        user_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, user_id)
+
+        broadcast_data = self._parse_json(self._html_search_meta(
+            'broadcast-data', webpage, default='{}'), user_id)
+        username = broadcast_data.get('user', {}).get('display_name')
+        user_broadcasts = self._parse_json(
+            self._html_search_meta('user-broadcasts', webpage, default='{}'),
+            user_id)
+
+        entries = [
+            self.url_result(
+                'https://www.periscope.tv/%s/%s' % (user_id, broadcast['id']))
+            for broadcast in user_broadcasts.get('broadcasts', [])]
+
+        return self.playlist_result(entries, user_id, username)

From 22e35adefdcea7dccade852ad7dff0953060381d Mon Sep 17 00:00:00 2001
From: Peter Rowlands <peter@pmrowla.com>
Date: Fri, 6 May 2016 10:41:30 +0900
Subject: [PATCH 4856/6219] use url instead of single formats entry

---
 youtube_dl/extractor/afreecatv.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/afreecatv.py b/youtube_dl/extractor/afreecatv.py
index 9f9399edc..c9a4b7311 100644
--- a/youtube_dl/extractor/afreecatv.py
+++ b/youtube_dl/extractor/afreecatv.py
@@ -58,7 +58,7 @@ class AfreecaTVIE(InfoExtractor):
                 'id': video_file.get('key'),
                 'title': title,
                 'duration': int_or_none(video_file.get('duration')),
-                'formats': [{'url': video_file.text}]
+                'url': video_file.text,
             })
 
         info = {
@@ -74,7 +74,7 @@ class AfreecaTVIE(InfoExtractor):
             info['_type'] = 'multi_video'
             info['entries'] = entries
         elif len(entries) == 1:
-            info['formats'] = entries[0]['formats']
+            info['url'] = entries[0]['url']
         else:
             raise ExtractorError(
                 'No files found for the specified AfreecaTV video, either'

From 1dbfd7875497398e8c92c67b8307bafef20e8113 Mon Sep 17 00:00:00 2001
From: Peter Rowlands <peter@pmrowla.com>
Date: Fri, 6 May 2016 12:07:29 +0900
Subject: [PATCH 4857/6219] fix multi_video part naming, add upload_date field

---
 youtube_dl/extractor/afreecatv.py | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/afreecatv.py b/youtube_dl/extractor/afreecatv.py
index c9a4b7311..e927e4a48 100644
--- a/youtube_dl/extractor/afreecatv.py
+++ b/youtube_dl/extractor/afreecatv.py
@@ -1,6 +1,8 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse_urlparse,
@@ -34,6 +36,15 @@ class AfreecaTVIE(InfoExtractor):
         }
     }
 
+    @staticmethod
+    def parse_video_key(key):
+        video_key = {'upload_date': None, 'part': '0'}
+        m = re.match(r'^(?P<upload_date>\d{8})_\w+_(?P<part>\d+)$', key)
+        if m:
+            video_key['upload_date'] = m.group('upload_date')
+            video_key['part'] = m.group('part')
+        return video_key
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
         parsed_url = compat_urllib_parse_urlparse(url)
@@ -54,9 +65,11 @@ class AfreecaTVIE(InfoExtractor):
 
         entries = []
         for video_file in video_xml.findall('./track/video/file'):
+            video_key = self.parse_video_key(video_file.get('key'))
             entries.append({
-                'id': video_file.get('key'),
+                'id': '%s_%s' % (video_id, video_key['part']),
                 'title': title,
+                'upload_date': video_key['upload_date'],
                 'duration': int_or_none(video_file.get('duration')),
                 'url': video_file.text,
             })
@@ -75,6 +88,7 @@ class AfreecaTVIE(InfoExtractor):
             info['entries'] = entries
         elif len(entries) == 1:
             info['url'] = entries[0]['url']
+            info['upload_date'] = entries[0]['upload_date']
         else:
             raise ExtractorError(
                 'No files found for the specified AfreecaTV video, either'

From 8d93c214664e5442320a20e899123b8bfd51cd08 Mon Sep 17 00:00:00 2001
From: Peter Rowlands <peter@pmrowla.com>
Date: Fri, 6 May 2016 12:08:43 +0900
Subject: [PATCH 4858/6219] add multi_video test case

---
 youtube_dl/extractor/afreecatv.py | 32 +++++++++++++++++++++++++++++--
 1 file changed, 30 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/afreecatv.py b/youtube_dl/extractor/afreecatv.py
index e927e4a48..aa5847677 100644
--- a/youtube_dl/extractor/afreecatv.py
+++ b/youtube_dl/extractor/afreecatv.py
@@ -23,7 +23,7 @@ class AfreecaTVIE(InfoExtractor):
             /app/(?:index|read_ucc_bbs)\.cgi|
             /player/[Pp]layer\.(?:swf|html))
         \?.*?\bnTitleNo=(?P<id>\d+)'''
-    _TEST = {
+    _TESTS = [{
         'url': 'http://live.afreecatv.com:8079/app/index.cgi?szType=read_ucc_bbs&szBjId=dailyapril&nStationNo=16711924&nBbsNo=18605867&nTitleNo=36164052&szSkin=',
         'md5': 'f72c89fe7ecc14c1b5ce506c4996046e',
         'info_dict': {
@@ -33,8 +33,36 @@ class AfreecaTVIE(InfoExtractor):
             'thumbnail': 're:^https?://videoimg.afreecatv.com/.*$',
             'uploader': 'dailyapril',
             'uploader_id': 'dailyapril',
+            'upload_date': '20160503',
         }
-    }
+    }, {
+        'url': 'http://afbbs.afreecatv.com:8080/app/read_ucc_bbs.cgi?nStationNo=16711924&nTitleNo=36153164&szBjId=dailyapril&nBbsNo=18605867',
+        'info_dict': {
+            'id': '36153164',
+            'title': "BJ유트루와 함께하는 '팅커벨 메이크업!'",
+            'thumbnail': 're:^https?://videoimg.afreecatv.com/.*$',
+            'uploader': 'dailyapril',
+            'uploader_id': 'dailyapril',
+        },
+        'playlist_count': 2,
+        'playlist': [{
+            'md5': 'd8b7c174568da61d774ef0203159bf97',
+            'info_dict': {
+                'id': '36153164_1',
+                'ext': 'mp4',
+                'title': "BJ유트루와 함께하는 '팅커벨 메이크업!'",
+                'upload_date': '20160502',
+            },
+        }, {
+            'md5': '58f2ce7f6044e34439ab2d50612ab02b',
+            'info_dict': {
+                'id': '36153164_2',
+                'ext': 'mp4',
+                'title': "BJ유트루와 함께하는 '팅커벨 메이크업!'",
+                'upload_date': '20160502',
+            },
+        }],
+    }]
 
     @staticmethod
     def parse_video_key(key):

From 04e88ca2cac8f97e78cdee2825946e94b6173023 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 6 May 2016 15:02:40 +0100
Subject: [PATCH 4859/6219] [vk] improve extraction(fixes #7976)

---
 youtube_dl/extractor/biqle.py      | 39 +++++++++++++++++++++++++
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/vk.py         | 46 ++++++++++++++++--------------
 3 files changed, 64 insertions(+), 22 deletions(-)
 create mode 100644 youtube_dl/extractor/biqle.py

diff --git a/youtube_dl/extractor/biqle.py b/youtube_dl/extractor/biqle.py
new file mode 100644
index 000000000..ae4579b33
--- /dev/null
+++ b/youtube_dl/extractor/biqle.py
@@ -0,0 +1,39 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class BIQLEIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?biqle\.(?:com|org|ru)/watch/(?P<id>-?\d+_\d+)'
+    _TESTS = [{
+        'url': 'http://www.biqle.ru/watch/847655_160197695',
+        'md5': 'ad5f746a874ccded7b8f211aeea96637',
+        'info_dict': {
+            'id': '160197695',
+            'ext': 'mp4',
+            'title': 'Foo Fighters - The Pretender (Live at Wembley Stadium)',
+            'uploader': 'Andrey Rogozin',
+            'upload_date': '20110605',
+        }
+    }, {
+        'url': 'https://biqle.org/watch/-44781847_168547604',
+        'md5': '7f24e72af1db0edf7c1aaba513174f97',
+        'info_dict': {
+            'id': '168547604',
+            'ext': 'mp4',
+            'title': 'Ребенок в шоке от автоматической мойки',
+            'uploader': 'Dmitry Kotov',
+        }
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        embed_url = self._proto_relative_url(self._search_regex(
+            r'<iframe.+?src="((?:http:)?//daxab\.com/[^"]+)".*?></iframe>', webpage, 'embed url'))
+
+        return {
+            '_type': 'url_transparent',
+            'url': embed_url,
+        }
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index c9d1422e5..14b4f245f 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -75,6 +75,7 @@ from .bigflix import BigflixIE
 from .bild import BildIE
 from .bilibili import BiliBiliIE
 from .biobiochiletv import BioBioChileTVIE
+from .biqle import BIQLEIE
 from .bleacherreport import (
     BleacherReportIE,
     BleacherReportCMSIE,
diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 67220f1b7..041d93629 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -26,12 +26,16 @@ class VKIE(InfoExtractor):
     _VALID_URL = r'''(?x)
                     https?://
                         (?:
-                            (?:m\.)?vk\.com/video_ext\.php\?.*?\boid=(?P<oid>-?\d+).*?\bid=(?P<id>\d+)|
+                            (?:
+                                (?:m\.)?vk\.com/video_|
+                                (?:www\.)?daxab.com/
+                            )
+                            ext\.php\?(?P<embed_query>.*?\boid=(?P<oid>-?\d+).*?\bid=(?P<id>\d+).*)|
                             (?:
                                 (?:m\.)?vk\.com/(?:.+?\?.*?z=)?video|
-                                (?:www\.)?biqle\.ru/watch/
+                                (?:www\.)?daxab.com/embed/
                             )
-                            (?P<videoid>[^s].*?)(?:\?(?:.*\blist=(?P<list_id>[\da-f]+))?|%2F|$)
+                            (?P<videoid>-?\d+_\d+)(?:.*\blist=(?P<list_id>[\da-f]+))?
                         )
                     '''
     _NETRC_MACHINE = 'vk'
@@ -75,7 +79,8 @@ class VKIE(InfoExtractor):
                 'duration': 101,
                 'upload_date': '20120730',
                 'view_count': int,
-            }
+            },
+            'skip': 'This video has been removed from public access.',
         },
         {
             # VIDEO NOW REMOVED
@@ -142,7 +147,7 @@ class VKIE(InfoExtractor):
                 'id': 'V3K4mi0SYkc',
                 'ext': 'webm',
                 'title': "DSWD Awards 'Children's Joy Foundation, Inc.' Certificate of Registration and License to Operate",
-                'description': 'md5:bf9c26cfa4acdfb146362682edd3827a',
+                'description': 'md5:d9903938abdc74c738af77f527ca0596',
                 'duration': 178,
                 'upload_date': '20130116',
                 'uploader': "Children's Joy Foundation",
@@ -173,11 +178,6 @@ class VKIE(InfoExtractor):
             'url': 'https://vk.com/video205387401_164765225',
             'only_matching': True,
         },
-        {
-            # vk wrapper
-            'url': 'http://www.biqle.ru/watch/847655_160197695',
-            'only_matching': True,
-        },
         {
             # pladform embed
             'url': 'https://vk.com/video-76116461_171554880',
@@ -217,20 +217,22 @@ class VKIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('videoid')
 
-        if not video_id:
+        info_url = url
+        if video_id:
+            info_url = 'https://vk.com/al_video.php?act=show&al=1&module=video&video=%s' % video_id
+            # Some videos (removed?) can only be downloaded with list id specified
+            list_id = mobj.group('list_id')
+            if list_id:
+                info_url += '&list=%s' % list_id
+        else:
+            info_url = 'http://vk.com/video_ext.php?' + mobj.group('embed_query')
             video_id = '%s_%s' % (mobj.group('oid'), mobj.group('id'))
 
-        info_url = 'https://vk.com/al_video.php?act=show&al=1&module=video&video=%s' % video_id
-
-        # Some videos (removed?) can only be downloaded with list id specified
-        list_id = mobj.group('list_id')
-        if list_id:
-            info_url += '&list=%s' % list_id
-
         info_page = self._download_webpage(info_url, video_id)
 
         error_message = self._html_search_regex(
-            r'(?s)<!><div[^>]+class="video_layer_message"[^>]*>(.+?)</div>',
+            [r'(?s)<!><div[^>]+class="video_layer_message"[^>]*>(.+?)</div>',
+                r'(?s)<div[^>]+id="video_ext_msg"[^>]*>(.+?)</div>'],
             info_page, 'error message', default=None)
         if error_message:
             raise ExtractorError(error_message, expected=True)
@@ -305,17 +307,17 @@ class VKIE(InfoExtractor):
         view_count = None
         views = self._html_search_regex(
             r'"mv_views_count_number"[^>]*>(.+?\bviews?)<',
-            info_page, 'view count', fatal=False)
+            info_page, 'view count', default=None)
         if views:
             view_count = str_to_int(self._search_regex(
                 r'([\d,.]+)', views, 'view count', fatal=False))
 
         formats = []
         for k, v in data.items():
-            if not k.startswith('url') and k != 'extra_data' or not v:
+            if not k.startswith('url') and not k.startswith('cache') and k != 'extra_data' or not v:
                 continue
             height = int_or_none(self._search_regex(
-                r'^url(\d+)', k, 'height', default=None))
+                r'^(?:url|cache)(\d+)', k, 'height', default=None))
             formats.append({
                 'format_id': k,
                 'url': v,

From abc97b5eda4ed4b36cec29e9966eb1bb7bcd97ce Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 6 May 2016 22:07:30 +0600
Subject: [PATCH 4860/6219] [utils] Allow empty attribute values in
 get_element_by_attribute (Closes #9415)

---
 youtube_dl/utils.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index a5922b2b5..6e4573784 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -256,9 +256,9 @@ def get_element_by_attribute(attribute, value, html):
 
     m = re.search(r'''(?xs)
         <([a-zA-Z0-9:._-]+)
-         (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]+|="[^"]+"|='[^']+'))*?
+         (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]*|="[^"]*"|='[^']*'))*?
          \s+%s=['"]?%s['"]?
-         (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]+|="[^"]+"|='[^']+'))*?
+         (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]*|="[^"]*"|='[^']*'))*?
         \s*>
         (?P<content>.*?)
         </\1>

From 25cb7a0eebae0093a81fa1c930480fafa13feb25 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 6 May 2016 22:11:18 +0600
Subject: [PATCH 4861/6219] [youtube] Allow empty attribute values in
 description regex

---
 youtube_dl/extractor/youtube.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index b7c3cb63f..f3f102c30 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1326,9 +1326,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         if video_description:
             video_description = re.sub(r'''(?x)
                 <a\s+
-                    (?:[a-zA-Z-]+="[^"]+"\s+)*?
+                    (?:[a-zA-Z-]+="[^"]*"\s+)*?
                     (?:title|href)="([^"]+)"\s+
-                    (?:[a-zA-Z-]+="[^"]+"\s+)*?
+                    (?:[a-zA-Z-]+="[^"]*"\s+)*?
                     class="(?:yt-uix-redirect-link|yt-uix-sessionlink[^"]*)"[^>]*>
                 [^<]+\.{3}\s*
                 </a>

From 3e80e6f40d6ef76142340a2292ef2445dc79594b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 6 May 2016 23:35:58 +0600
Subject: [PATCH 4862/6219] [vevo] Allow request to api.vevo.com to fail
 (Closes #9417)

I don't know whether this it's tempopary or api has just gone
---
 youtube_dl/extractor/vevo.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py
index c0ef08c02..30b3a9e7e 100644
--- a/youtube_dl/extractor/vevo.py
+++ b/youtube_dl/extractor/vevo.py
@@ -201,9 +201,10 @@ class VevoIE(VevoBaseIE):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        json_url = 'http://api.vevo.com/VideoService/AuthenticateVideo?isrc=%s' % video_id
+        json_url = 'http://videoplayer.vevo.com/VideoService/AuthenticateVideo?isrc=%s' % video_id
         response = self._download_json(
-            json_url, video_id, 'Downloading video info', 'Unable to download info')
+            json_url, video_id, 'Downloading video info',
+            'Unable to download info', fatal=False) or {}
         video_info = response.get('video') or {}
         artist = None
         featured_artist = None
@@ -212,7 +213,7 @@ class VevoIE(VevoBaseIE):
         formats = []
 
         if not video_info:
-            if response.get('statusCode') != 909:
+            if response and response.get('statusCode') != 909:
                 ytid = response.get('errorInfo', {}).get('ytid')
                 if ytid:
                     self.report_warning(

From f745403b5b448c170710256a61b8505e09e77674 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 6 May 2016 23:37:17 +0600
Subject: [PATCH 4863/6219] [vevo] Revert videoplayer.vevo.com to api.vevo.com

---
 youtube_dl/extractor/vevo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py
index 30b3a9e7e..c0632cd6a 100644
--- a/youtube_dl/extractor/vevo.py
+++ b/youtube_dl/extractor/vevo.py
@@ -201,7 +201,7 @@ class VevoIE(VevoBaseIE):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        json_url = 'http://videoplayer.vevo.com/VideoService/AuthenticateVideo?isrc=%s' % video_id
+        json_url = 'http://api.vevo.com/VideoService/AuthenticateVideo?isrc=%s' % video_id
         response = self._download_json(
             json_url, video_id, 'Downloading video info',
             'Unable to download info', fatal=False) or {}

From e2ee97dcd5c55e1c2aceae0d93fbfd64d0cc5ba3 Mon Sep 17 00:00:00 2001
From: inondle <qfulsher@gmai.com>
Date: Fri, 6 May 2016 12:05:37 -0700
Subject: [PATCH 4864/6219] [liveleak] Adds support for thumbnails, updates
 tests

---
 youtube_dl/extractor/liveleak.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/liveleak.py b/youtube_dl/extractor/liveleak.py
index 29fba5f30..ea0565ac0 100644
--- a/youtube_dl/extractor/liveleak.py
+++ b/youtube_dl/extractor/liveleak.py
@@ -17,7 +17,8 @@ class LiveLeakIE(InfoExtractor):
             'ext': 'flv',
             'description': 'extremely bad day for this guy..!',
             'uploader': 'ljfriel2',
-            'title': 'Most unlucky car accident'
+            'title': 'Most unlucky car accident',
+            'thumbnail': 're:^https?://.*\.jpg$'
         }
     }, {
         'url': 'http://www.liveleak.com/view?i=f93_1390833151',
@@ -28,6 +29,7 @@ class LiveLeakIE(InfoExtractor):
             'description': 'German Television Channel NDR does an exclusive interview with Edward Snowden.\r\nUploaded on LiveLeak cause German Television thinks the rest of the world isn\'t intereseted in Edward Snowden.',
             'uploader': 'ARD_Stinkt',
             'title': 'German Television does first Edward Snowden Interview (ENGLISH)',
+            'thumbnail': 're:^https?://.*\.jpg$'
         }
     }, {
         'url': 'http://www.liveleak.com/view?i=4f7_1392687779',
@@ -49,7 +51,8 @@ class LiveLeakIE(InfoExtractor):
             'ext': 'mp4',
             'description': 'Happened on 27.7.2014. \r\nAt 0:53 you can see people still swimming at near beach.',
             'uploader': 'bony333',
-            'title': 'Crazy Hungarian tourist films close call waterspout in Croatia'
+            'title': 'Crazy Hungarian tourist films close call waterspout in Croatia',
+            'thumbnail': 're:^https?://.*\.jpg$'
         }
     }]
 
@@ -72,6 +75,7 @@ class LiveLeakIE(InfoExtractor):
         age_limit = int_or_none(self._search_regex(
             r'you confirm that you are ([0-9]+) years and over.',
             webpage, 'age limit', default=None))
+        video_thumbnail = self._og_search_thumbnail(webpage)
 
         sources_raw = self._search_regex(
             r'(?s)sources:\s*(\[.*?\]),', webpage, 'video URLs', default=None)
@@ -124,4 +128,5 @@ class LiveLeakIE(InfoExtractor):
             'uploader': video_uploader,
             'formats': formats,
             'age_limit': age_limit,
+            'thumbnail': video_thumbnail,
         }

From 3fd6332c056115e5de37b0789d907e9344c2ff5c Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 7 May 2016 15:12:20 +0100
Subject: [PATCH 4865/6219] [flickr] extract license field(closes #9425)

---
 youtube_dl/extractor/flickr.py | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/flickr.py b/youtube_dl/extractor/flickr.py
index 0a3de1498..73ae3adee 100644
--- a/youtube_dl/extractor/flickr.py
+++ b/youtube_dl/extractor/flickr.py
@@ -27,10 +27,24 @@ class FlickrIE(InfoExtractor):
             'comment_count': int,
             'view_count': int,
             'tags': list,
+            'license': 'Attribution-ShareAlike',
         }
     }
-
     _API_BASE_URL = 'https://api.flickr.com/services/rest?'
+    # https://help.yahoo.com/kb/flickr/SLN25525.html
+    _LICENSES = {
+        '0': 'All Rights Reserved',
+        '1': 'Attribution-NonCommercial-ShareAlike',
+        '2': 'Attribution-NonCommercial',
+        '3': 'Attribution-NonCommercial-NoDerivs',
+        '4': 'Attribution',
+        '5': 'Attribution-ShareAlike',
+        '6': 'Attribution-NoDerivs',
+        '7': 'No known copyright restrictions',
+        '8': 'United States government work',
+        '9': 'Public Domain Dedication (CC0)',
+        '10': 'Public Domain Work',
+    }
 
     def _call_api(self, method, video_id, api_key, note, secret=None):
         query = {
@@ -87,7 +101,8 @@ class FlickrIE(InfoExtractor):
                 'uploader': owner.get('realname'),
                 'comment_count': int_or_none(video_info.get('comments', {}).get('_content')),
                 'view_count': int_or_none(video_info.get('views')),
-                'tags': [tag.get('_content') for tag in video_info.get('tags', {}).get('tag', [])]
+                'tags': [tag.get('_content') for tag in video_info.get('tags', {}).get('tag', [])],
+                'license': self._LICENSES.get(video_info.get('license')),
             }
         else:
             raise ExtractorError('not a video', expected=True)

From cb1fa5881315ed998a366f47511b7a4b4ea067b0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 7 May 2016 20:15:40 +0600
Subject: [PATCH 4866/6219] [flickr] Extract uploader URL (Closes #9426)

---
 youtube_dl/extractor/flickr.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/flickr.py b/youtube_dl/extractor/flickr.py
index 73ae3adee..a8e1bf42a 100644
--- a/youtube_dl/extractor/flickr.py
+++ b/youtube_dl/extractor/flickr.py
@@ -24,6 +24,7 @@ class FlickrIE(InfoExtractor):
             'upload_date': '20110423',
             'uploader_id': '10922353@N03',
             'uploader': 'Forest Wander',
+            'uploader_url': 'https://www.flickr.com/photos/forestwander-nature-pictures/',
             'comment_count': int,
             'view_count': int,
             'tags': list,
@@ -89,6 +90,9 @@ class FlickrIE(InfoExtractor):
             self._sort_formats(formats)
 
             owner = video_info.get('owner', {})
+            uploader_id = owner.get('nsid')
+            uploader_path = owner.get('path_alias') or uploader_id
+            uploader_url = 'https://www.flickr.com/photos/%s/' % uploader_path if uploader_path else None
 
             return {
                 'id': video_id,
@@ -97,8 +101,9 @@ class FlickrIE(InfoExtractor):
                 'formats': formats,
                 'timestamp': int_or_none(video_info.get('dateuploaded')),
                 'duration': int_or_none(video_info.get('video', {}).get('duration')),
-                'uploader_id': owner.get('nsid'),
+                'uploader_id': uploader_id,
                 'uploader': owner.get('realname'),
+                'uploader_url': uploader_url,
                 'comment_count': int_or_none(video_info.get('comments', {}).get('_content')),
                 'view_count': int_or_none(video_info.get('views')),
                 'tags': [tag.get('_content') for tag in video_info.get('tags', {}).get('tag', [])],

From a0904c5d8024c12b7f95b1126a6b8152a4e1021f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 8 May 2016 00:56:31 +0800
Subject: [PATCH 4867/6219] [telegraaf] Fix extractor (closes #9318)

---
 youtube_dl/extractor/telegraaf.py | 58 +++++++++++++++++++++++++++----
 1 file changed, 51 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/telegraaf.py b/youtube_dl/extractor/telegraaf.py
index 6f8333cfc..9092e9b85 100644
--- a/youtube_dl/extractor/telegraaf.py
+++ b/youtube_dl/extractor/telegraaf.py
@@ -2,14 +2,16 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..utils import remove_end
+from ..utils import (
+    determine_ext,
+    remove_end,
+)
 
 
 class TelegraafIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?telegraaf\.nl/tv/(?:[^/]+/)+(?P<id>\d+)/[^/]+\.html'
     _TEST = {
         'url': 'http://www.telegraaf.nl/tv/nieuws/binnenland/24353229/__Tikibad_ontruimd_wegens_brand__.html',
-        'md5': '83245a9779bcc4a24454bfd53c65b6dc',
         'info_dict': {
             'id': '24353229',
             'ext': 'mp4',
@@ -18,18 +20,60 @@ class TelegraafIE(InfoExtractor):
             'thumbnail': 're:^https?://.*\.jpg$',
             'duration': 33,
         },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
     }
 
     def _real_extract(self, url):
-        playlist_id = self._match_id(url)
+        video_id = self._match_id(url)
 
-        webpage = self._download_webpage(url, playlist_id)
+        webpage = self._download_webpage(url, video_id)
 
+        player_url = self._html_search_regex(
+            r'<iframe[^>]+src="([^"]+")', webpage, 'player URL')
+        player_page = self._download_webpage(
+            player_url, video_id, note='Download player webpage')
         playlist_url = self._search_regex(
-            r"iframe\.loadPlayer\('([^']+)'", webpage, 'player')
+            r'playlist\s*:\s*"([^"]+)"', player_page, 'playlist URL')
+        playlist_data = self._download_json(playlist_url, video_id)
+
+        item = playlist_data['items'][0]
+        formats = []
+        locations = item['locations']
+        for location in locations.get('adaptive', []):
+            manifest_url = location['src']
+            ext = determine_ext(manifest_url)
+            if ext == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    manifest_url, video_id, ext='mp4', m3u8_id='hls'))
+            elif ext == 'mpd':
+                # TODO: Current DASH formats are broken - $Time$ pattern in
+                # <SegmentTemplate> not implemented yet
+                continue
+            else:
+                self.report_warning('Unknown adaptive format %s' % ext)
+        for location in locations.get('progressive', []):
+            formats.append({
+                'url': location['sources'][0]['src'],
+                'width': location.get('width'),
+                'height': location.get('height'),
+                'format_id': 'http-%s' % location['label'],
+            })
+
+        self._sort_formats(formats)
 
-        entries = self._extract_xspf_playlist(playlist_url, playlist_id)
         title = remove_end(self._og_search_title(webpage), ' - VIDEO')
         description = self._og_search_description(webpage)
+        duration = item.get('duration')
+        thumbnail = item.get('poster')
 
-        return self.playlist_result(entries, playlist_id, title, description)
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'formats': formats,
+            'duration': duration,
+            'thumbnail': thumbnail,
+        }

From e2eca6f65e9969c31b3374bd3688321f3e471cd7 Mon Sep 17 00:00:00 2001
From: Kevin Deldycke <kevin@deldycke.com>
Date: Sat, 7 May 2016 20:03:25 +0200
Subject: [PATCH 4868/6219] Expand user's home in batch file path.

---
 youtube_dl/__init__.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 737f6545d..7a0466077 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -86,7 +86,9 @@ def _real_main(argv=None):
             if opts.batchfile == '-':
                 batchfd = sys.stdin
             else:
-                batchfd = io.open(opts.batchfile, 'r', encoding='utf-8', errors='ignore')
+                batchfd = io.open(
+                    compat_expanduser(opts.batchfile),
+                    'r', encoding='utf-8', errors='ignore')
             batch_urls = read_batch_urls(batchfd)
             if opts.verbose:
                 write_string('[debug] Batch file urls: ' + repr(batch_urls) + '\n')

From 00c21c225decf648199013f2fa3385a1332037bb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 8 May 2016 00:11:44 +0600
Subject: [PATCH 4869/6219] Credit @kdeldycke for #9430

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 814fe9ec3..5f668338b 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -169,3 +169,4 @@ Viťas Strádal
 Kagami Hiiragi
 Philip Huppert
 blahgeek
+Kevin Deldycke

From 5c24873a9e6a47e58b10eb0c0825e165604796f2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 8 May 2016 02:04:34 +0600
Subject: [PATCH 4870/6219] Credit @inondle for #9400

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 5f668338b..bf860b7f7 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -170,3 +170,4 @@ Kagami Hiiragi
 Philip Huppert
 blahgeek
 Kevin Deldycke
+inondle

From f5436c5d9e4e65790440ada40476712ff430651b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 8 May 2016 02:29:26 +0600
Subject: [PATCH 4871/6219] [downloader/external] Add temp fix ffmpeg m3u8
 downloads (Closes #9394)

---
 youtube_dl/downloader/external.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index 8d642fc3e..45f49c350 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -224,7 +224,7 @@ class FFmpegFD(ExternalFD):
                 args += ['-rtmp_live', 'live']
 
         args += ['-i', url, '-c', 'copy']
-        if protocol == 'm3u8':
+        if protocol in ('m3u8', 'm3u8_native'):
             if self.params.get('hls_use_mpegts', False) or tmpfilename == '-':
                 args += ['-f', 'mpegts']
             else:

From 3e169233daf76cd7585ebac12504f8e624b7693b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 8 May 2016 04:36:57 +0600
Subject: [PATCH 4872/6219] Expanduser for more options with input files

---
 youtube_dl/YoutubeDL.py | 1 +
 youtube_dl/__init__.py  | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 2187dcc8f..a96482e68 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -2018,6 +2018,7 @@ class YoutubeDL(object):
         if opts_cookiefile is None:
             self.cookiejar = compat_cookiejar.CookieJar()
         else:
+            opts_cookiefile = compat_expanduser(opts_cookiefile)
             self.cookiejar = compat_cookiejar.MozillaCookieJar(
                 opts_cookiefile)
             if os.access(opts_cookiefile, os.R_OK):
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 7a0466077..cbd84c3af 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -406,7 +406,7 @@ def _real_main(argv=None):
 
         try:
             if opts.load_info_filename is not None:
-                retcode = ydl.download_with_info_file(opts.load_info_filename)
+                retcode = ydl.download_with_info_file(compat_expanduser(opts.load_info_filename))
             else:
                 retcode = ydl.download(all_urls)
         except MaxDownloadsReached:

From 0fdbe3146c2b3825cc26aca7e918df041b0f9adf Mon Sep 17 00:00:00 2001
From: Peter Rowlands <peter@pmrowla.com>
Date: Sun, 8 May 2016 08:56:22 +0900
Subject: [PATCH 4873/6219] use dict.get in case upload_date does not exist

---
 youtube_dl/extractor/afreecatv.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/afreecatv.py b/youtube_dl/extractor/afreecatv.py
index aa5847677..4ebc61bae 100644
--- a/youtube_dl/extractor/afreecatv.py
+++ b/youtube_dl/extractor/afreecatv.py
@@ -66,7 +66,7 @@ class AfreecaTVIE(InfoExtractor):
 
     @staticmethod
     def parse_video_key(key):
-        video_key = {'upload_date': None, 'part': '0'}
+        video_key = {}
         m = re.match(r'^(?P<upload_date>\d{8})_\w+_(?P<part>\d+)$', key)
         if m:
             video_key['upload_date'] = m.group('upload_date')
@@ -92,12 +92,12 @@ class AfreecaTVIE(InfoExtractor):
         thumbnail = xpath_text(video_xml, './track/titleImage', 'thumbnail')
 
         entries = []
-        for video_file in video_xml.findall('./track/video/file'):
+        for i, video_file in enumerate(video_xml.findall('./track/video/file')):
             video_key = self.parse_video_key(video_file.get('key'))
             entries.append({
-                'id': '%s_%s' % (video_id, video_key['part']),
+                'id': '%s_%s' % (video_id, video_key.get('part', i + 1)),
                 'title': title,
-                'upload_date': video_key['upload_date'],
+                'upload_date': video_key.get('upload_date'),
                 'duration': int_or_none(video_file.get('duration')),
                 'url': video_file.text,
             })

From 81f35fee2fd2b58d909887aaa7667310a4d65759 Mon Sep 17 00:00:00 2001
From: Peter Rowlands <peter@pmrowla.com>
Date: Sun, 8 May 2016 08:56:44 +0900
Subject: [PATCH 4874/6219] fix extractors.py import order

---
 youtube_dl/extractor/extractors.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index f85d75933..1f95530a5 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -16,8 +16,8 @@ from .adobetv import (
     AdobeTVVideoIE,
 )
 from .adultswim import AdultSwimIE
-from .afreecatv import AfreecaTVIE
 from .aenetworks import AENetworksIE
+from .afreecatv import AfreecaTVIE
 from .aftonbladet import AftonbladetIE
 from .airmozilla import AirMozillaIE
 from .aljazeera import AlJazeeraIE

From 9c072d38c6b0361d91e92c50cd0c753dc8ce3101 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 8 May 2016 06:52:42 +0600
Subject: [PATCH 4875/6219] [arte] Improve language preference (Closes #9401,
 closes #9162)

---
 youtube_dl/extractor/arte.py | 58 ++++++++++++++++++++++++++----------
 1 file changed, 43 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index 881cacfab..e37fdae13 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -161,24 +161,53 @@ class ArteTVPlus7IE(InfoExtractor):
             'es': 'E[ESP]',
         }
 
+        langcode = LANGS.get(lang, lang)
+
         formats = []
         for format_id, format_dict in player_info['VSR'].items():
             f = dict(format_dict)
             versionCode = f.get('versionCode')
-            langcode = LANGS.get(lang, lang)
-            lang_rexs = [r'VO?%s-' % re.escape(langcode), r'VO?.-ST%s$' % re.escape(langcode)]
-            lang_pref = None
-            if versionCode:
-                matched_lang_rexs = [r for r in lang_rexs if re.match(r, versionCode)]
-                lang_pref = -10 if not matched_lang_rexs else 10 * len(matched_lang_rexs)
-            source_pref = 0
-            if versionCode is not None:
-                # The original version with subtitles has lower relevance
-                if re.match(r'VO-ST(F|A|E)', versionCode):
-                    source_pref -= 10
-                # The version with sourds/mal subtitles has also lower relevance
-                elif re.match(r'VO?(F|A|E)-STM\1', versionCode):
-                    source_pref -= 9
+            l = re.escape(langcode)
+
+            # Language preference from most to least priority
+            # Reference: section 5.6.3 of
+            # http://www.arte.tv/sites/en/corporate/files/complete-technical-guidelines-arte-geie-v1-05.pdf
+            PREFERENCES = (
+                # original version in requested language, without subtitles
+                r'VO{0}$'.format(l),
+                # original version in requested language, with partial subtitles in requested language
+                r'VO{0}-ST{0}$'.format(l),
+                # original version in requested language, with subtitles for the deaf and hard-of-hearing in requested language
+                r'VO{0}-STM{0}$'.format(l),
+                # non-original (dubbed) version in requested language, without subtitles
+                r'V{0}$'.format(l),
+                # non-original (dubbed) version in requested language, with subtitles partial subtitles in requested language
+                r'V{0}-ST{0}$'.format(l),
+                # non-original (dubbed) version in requested language, with subtitles for the deaf and hard-of-hearing in requested language
+                r'V{0}-STM{0}$'.format(l),
+                # original version in requested language, with partial subtitles in different language
+                r'VO{0}-ST(?!{0}).+?$'.format(l),
+                # original version in requested language, with subtitles for the deaf and hard-of-hearing in different language
+                r'VO{0}-STM(?!{0}).+?$'.format(l),
+                # original version in different language, with partial subtitles in requested language
+                r'VO(?:(?!{0}).+?)?-ST{0}$'.format(l),
+                # original version in different language, with subtitles for the deaf and hard-of-hearing in requested language
+                r'VO(?:(?!{0}).+?)?-STM{0}$'.format(l),
+                # original version in different language, without subtitles
+                r'VO(?:(?!{0}))?$'.format(l),
+                # original version in different language, with partial subtitles in different language
+                r'VO(?:(?!{0}).+?)?-ST(?!{0}).+?$'.format(l),
+                # original version in different language, with subtitles for the deaf and hard-of-hearing in different language
+                r'VO(?:(?!{0}).+?)?-STM(?!{0}).+?$'.format(l),
+            )
+
+            for pref, p in enumerate(PREFERENCES):
+                if re.match(p, versionCode):
+                    lang_pref = len(PREFERENCES) - pref
+                    break
+            else:
+                lang_pref = -1
+
             format = {
                 'format_id': format_id,
                 'preference': -10 if f.get('videoFormat') == 'M3U8' else None,
@@ -188,7 +217,6 @@ class ArteTVPlus7IE(InfoExtractor):
                 'height': int_or_none(f.get('height')),
                 'tbr': int_or_none(f.get('bitrate')),
                 'quality': qfunc(f.get('quality')),
-                'source_preference': source_pref,
             }
 
             if f.get('mediaType') == 'rtmp':

From 3452c3a27c2bfd278746314cda4247c2226a35f3 Mon Sep 17 00:00:00 2001
From: Peter Rowlands <peter@pmrowla.com>
Date: Sun, 8 May 2016 10:02:19 +0900
Subject: [PATCH 4876/6219] update tests

---
 youtube_dl/extractor/afreecatv.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/afreecatv.py b/youtube_dl/extractor/afreecatv.py
index 4ebc61bae..b90095881 100644
--- a/youtube_dl/extractor/afreecatv.py
+++ b/youtube_dl/extractor/afreecatv.py
@@ -30,7 +30,7 @@ class AfreecaTVIE(InfoExtractor):
             'id': '36164052',
             'ext': 'mp4',
             'title': '데일리 에이프릴 요정들의 시상식!',
-            'thumbnail': 're:^https?://videoimg.afreecatv.com/.*$',
+            'thumbnail': 're:^https?://(?:video|st)img.afreecatv.com/.*$',
             'uploader': 'dailyapril',
             'uploader_id': 'dailyapril',
             'upload_date': '20160503',
@@ -40,7 +40,7 @@ class AfreecaTVIE(InfoExtractor):
         'info_dict': {
             'id': '36153164',
             'title': "BJ유트루와 함께하는 '팅커벨 메이크업!'",
-            'thumbnail': 're:^https?://videoimg.afreecatv.com/.*$',
+            'thumbnail': 're:^https?://(?:video|st)img.afreecatv.com/.*$',
             'uploader': 'dailyapril',
             'uploader_id': 'dailyapril',
         },
@@ -62,6 +62,9 @@ class AfreecaTVIE(InfoExtractor):
                 'upload_date': '20160502',
             },
         }],
+    }, {
+        'url': 'http://www.afreecatv.com/player/Player.swf?szType=szBjId=djleegoon&nStationNo=11273158&nBbsNo=13161095&nTitleNo=36327652',
+        'only_matching': True,
     }]
 
     @staticmethod

From 370d4eb8ad3d9d092fc5eb116509eaf4a3e83177 Mon Sep 17 00:00:00 2001
From: Peter Rowlands <peter@pmrowla.com>
Date: Sun, 8 May 2016 10:02:48 +0900
Subject: [PATCH 4877/6219] use stricter file selector

in case of empty in case of empty ./track/video/file entries
---
 youtube_dl/extractor/afreecatv.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/afreecatv.py b/youtube_dl/extractor/afreecatv.py
index b90095881..527386be3 100644
--- a/youtube_dl/extractor/afreecatv.py
+++ b/youtube_dl/extractor/afreecatv.py
@@ -95,7 +95,7 @@ class AfreecaTVIE(InfoExtractor):
         thumbnail = xpath_text(video_xml, './track/titleImage', 'thumbnail')
 
         entries = []
-        for i, video_file in enumerate(video_xml.findall('./track/video/file')):
+        for i, video_file in enumerate(video_xml.findall('./track/video/file[@key]')):
             video_key = self.parse_video_key(video_file.get('key'))
             entries.append({
                 'id': '%s_%s' % (video_id, video_key.get('part', i + 1)),
@@ -119,7 +119,7 @@ class AfreecaTVIE(InfoExtractor):
             info['entries'] = entries
         elif len(entries) == 1:
             info['url'] = entries[0]['url']
-            info['upload_date'] = entries[0]['upload_date']
+            info['upload_date'] = entries[0].get('upload_date')
         else:
             raise ExtractorError(
                 'No files found for the specified AfreecaTV video, either'

From 93fdb1417766015ddadcd13a709cdfae4de5e246 Mon Sep 17 00:00:00 2001
From: Peter Rowlands <peter@pmrowla.com>
Date: Sun, 8 May 2016 10:33:17 +0900
Subject: [PATCH 4878/6219] don't use selection by attribute

---
 youtube_dl/extractor/afreecatv.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/afreecatv.py b/youtube_dl/extractor/afreecatv.py
index 527386be3..0fcbea0d1 100644
--- a/youtube_dl/extractor/afreecatv.py
+++ b/youtube_dl/extractor/afreecatv.py
@@ -95,8 +95,10 @@ class AfreecaTVIE(InfoExtractor):
         thumbnail = xpath_text(video_xml, './track/titleImage', 'thumbnail')
 
         entries = []
-        for i, video_file in enumerate(video_xml.findall('./track/video/file[@key]')):
-            video_key = self.parse_video_key(video_file.get('key'))
+        for i, video_file in enumerate(video_xml.findall('./track/video/file')):
+            video_key = self.parse_video_key(video_file.get('key', ''))
+            if not video_key:
+                continue
             entries.append({
                 'id': '%s_%s' % (video_id, video_key.get('part', i + 1)),
                 'title': title,

From 3b01a9fbb63e33325fa979db8a846d3e655e79e6 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 8 May 2016 14:34:38 +0800
Subject: [PATCH 4879/6219] [litv] Add new extractor

LiTV is a streaming platform providing free and paid legal contents in
Taiwan.
---
 youtube_dl/extractor/extractors.py |   1 +
 youtube_dl/extractor/litv.py       | 137 +++++++++++++++++++++++++++++
 2 files changed, 138 insertions(+)
 create mode 100644 youtube_dl/extractor/litv.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 14b4f245f..7bacef184 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -384,6 +384,7 @@ from .limelight import (
     LimelightChannelIE,
     LimelightChannelListIE,
 )
+from .litv import LiTVIE
 from .liveleak import LiveLeakIE
 from .livestream import (
     LivestreamIE,
diff --git a/youtube_dl/extractor/litv.py b/youtube_dl/extractor/litv.py
new file mode 100644
index 000000000..3356d015d
--- /dev/null
+++ b/youtube_dl/extractor/litv.py
@@ -0,0 +1,137 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import json
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    int_or_none,
+    smuggle_url,
+    unsmuggle_url,
+)
+
+
+class LiTVIE(InfoExtractor):
+    _VALID_URL = r'https?://www\.litv\.tv/vod/[^/]+/content\.do\?.*?\bid=(?P<id>[^&]+)'
+
+    _URL_TEMPLATE = 'https://www.litv.tv/vod/%s/content.do?id=%s'
+
+    _TESTS = [{
+        'url': 'https://www.litv.tv/vod/drama/content.do?brc_id=root&id=VOD00041610&isUHEnabled=true&autoPlay=1',
+        'info_dict': {
+            'id': 'VOD00041606',
+            'title': '花千骨',
+        },
+        'playlist_count': 50,
+    }, {
+        'url': 'https://www.litv.tv/vod/drama/content.do?brc_id=root&id=VOD00041610&isUHEnabled=true&autoPlay=1',
+        'info_dict': {
+            'id': 'VOD00041610',
+            'ext': 'mp4',
+            'title': '花千骨第1集',
+            'thumbnail': 're:https?://.*\.jpg$',
+            'description': 'md5:c7017aa144c87467c4fb2909c4b05d6f',
+            'episode_number': 1,
+        },
+        'params': {
+            'noplaylist': True,
+            'skip_download': True,  # m3u8 download
+        },
+        'skip': 'Georestricted to Taiwan',
+    }]
+
+    def _extract_playlist(self, season_list, video_id, vod_data, view_data, prompt=True):
+        episode_title = view_data['title']
+        content_id = season_list['contentId']
+
+        if prompt:
+            self.to_screen('Downloading playlist %s - add --no-playlist to just download video %s' % (content_id, video_id))
+
+        all_episodes = [
+            self.url_result(smuggle_url(
+                self._URL_TEMPLATE % (view_data['contentType'], episode['contentId']),
+                {'force_noplaylist': True}))  # To prevent infinite recursion
+            for episode in season_list['episode']]
+
+        return self.playlist_result(all_episodes, content_id, episode_title)
+
+    def _real_extract(self, url):
+        url, data = unsmuggle_url(url, {})
+
+        video_id = self._match_id(url)
+
+        noplaylist = self._downloader.params.get('noplaylist')
+        noplaylist_prompt = True
+        if 'force_noplaylist' in data:
+            noplaylist = data['force_noplaylist']
+            noplaylist_prompt = False
+
+        webpage = self._download_webpage(url, video_id)
+
+        view_data = dict(map(lambda t: (t[0], t[2]), re.findall(
+            r'viewData\.([a-zA-Z]+)\s*=\s*(["\'])([^"\']+)\2',
+            webpage)))
+
+        vod_data = self._parse_json(self._search_regex(
+            'var\s+vod\s*=\s*([^;]+)', webpage, 'VOD data', default='{}'),
+            video_id)
+
+        season_list = list(vod_data.get('seasonList', {}).values())
+        if season_list:
+            if not noplaylist:
+                return self._extract_playlist(
+                    season_list[0], video_id, vod_data, view_data,
+                    prompt=noplaylist_prompt)
+
+            if noplaylist_prompt:
+                self.to_screen('Downloading just video %s because of --no-playlist' % video_id)
+
+        # In browsers `getMainUrl` request is always issued. Usually this
+        # endpoint gives the same result as the data embedded in the webpage.
+        # If georestricted, there are no embedded data, so an extra request is
+        # necessary to get the error code
+        video_data = self._parse_json(self._search_regex(
+            r'uiHlsUrl\s*=\s*testBackendData\(([^;]+)\);',
+            webpage, 'video data', default='{}'), video_id)
+        if not video_data:
+            payload = {
+                'assetId': view_data['assetId'],
+                'watchDevices': vod_data['watchDevices'],
+                'contentType': view_data['contentType'],
+            }
+            video_data = self._download_json(
+                'https://www.litv.tv/vod/getMainUrl', video_id,
+                data=json.dumps(payload).encode('utf-8'),
+                headers={'Content-Type': 'application/json'})
+
+        if not video_data.get('fullpath'):
+            error_msg = video_data.get('errorMessage')
+            if error_msg == 'vod.error.outsideregionerror':
+                self.raise_geo_restricted('This video is available in Taiwan only')
+            if error_msg:
+                raise ExtractorError('%s said: %s' % (self.IE_NAME, error_msg), expected=True)
+            raise ExtractorError('Unexpected result from %s' % self.IE_NAME)
+
+        formats = self._extract_m3u8_formats(
+            video_data['fullpath'], video_id, ext='mp4', m3u8_id='hls')
+        for a_format in formats:
+            # LiTV HLS segments doesn't like compressions
+            a_format.setdefault('http_headers', {})['Youtubedl-no-compression'] = True
+
+        title = view_data['title'] + view_data.get('secondaryMark', '')
+        description = view_data.get('description')
+        thumbnail = view_data.get('imageFile')
+        categories = [item['name'] for item in vod_data.get('category', [])]
+        episode = int_or_none(view_data.get('episode'))
+
+        return {
+            'id': video_id,
+            'formats': formats,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'categories': categories,
+            'episode_number': episode,
+        }

From f23a92a0cecac0d4db60e086e429793556347271 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 8 May 2016 20:02:54 +0600
Subject: [PATCH 4880/6219] [mva] Add extractor (Closes #6667)

---
 youtube_dl/extractor/extractors.py            |   4 +
 .../extractor/microsoftvirtualacademy.py      | 192 ++++++++++++++++++
 2 files changed, 196 insertions(+)
 create mode 100644 youtube_dl/extractor/microsoftvirtualacademy.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 7bacef184..a0bb3d4c2 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -409,6 +409,10 @@ from .metacafe import MetacafeIE
 from .metacritic import MetacriticIE
 from .mgoon import MgoonIE
 from .mgtv import MGTVIE
+from .microsoftvirtualacademy import (
+    MicrosoftVirtualAcademyIE,
+    MicrosoftVirtualAcademyCourseIE,
+)
 from .minhateca import MinhatecaIE
 from .ministrygrid import MinistryGridIE
 from .minoto import MinotoIE
diff --git a/youtube_dl/extractor/microsoftvirtualacademy.py b/youtube_dl/extractor/microsoftvirtualacademy.py
new file mode 100644
index 000000000..b7fea47ee
--- /dev/null
+++ b/youtube_dl/extractor/microsoftvirtualacademy.py
@@ -0,0 +1,192 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..compat import (
+    compat_xpath,
+)
+from ..utils import (
+    int_or_none,
+    parse_duration,
+    smuggle_url,
+    unsmuggle_url,
+    xpath_text,
+)
+
+
+class MicrosoftVirtualAcademyBaseIE(InfoExtractor):
+    def _extract_base_url(self, course_id, display_id):
+        return self._download_json(
+            'https://api-mlxprod.microsoft.com/services/products/anonymous/%s' % course_id,
+            display_id, 'Downloading course base URL')
+
+    def _extract_chapter_and_title(self, title):
+        if not title:
+            return None, None
+        m = re.search(r'(?P<chapter>\d+)\s*\|\s*(?P<title>.+)', title)
+        return (int(m.group('chapter')), m.group('title')) if m else (None, title)
+
+
+class MicrosoftVirtualAcademyIE(MicrosoftVirtualAcademyBaseIE):
+    IE_NAME = 'mva'
+    IE_DESC = 'Microsoft Virtual Academy videos'
+    _VALID_URL = r'(?:%s:|https?://(?:mva\.microsoft|microsoftvirtualacademy)\.com/[^/]+/training-courses/[^/?#&]+-)(?P<course_id>\d+)(?::|\?l=)(?P<id>[\da-zA-Z]+_\d+)' % IE_NAME
+
+    _TESTS = [{
+        'url': 'https://mva.microsoft.com/en-US/training-courses/microsoft-azure-fundamentals-virtual-machines-11788?l=gfVXISmEB_6804984382',
+        'md5': '7826c44fc31678b12ad8db11f6b5abb9',
+        'info_dict': {
+            'id': 'gfVXISmEB_6804984382',
+            'ext': 'mp4',
+            'title': 'Course Introduction',
+            'formats': 'mincount:3',
+            'subtitles': {
+                'en': [{
+                    'ext': 'ttml',
+                }],
+            },
+        }
+    }, {
+        'url': 'mva:11788:gfVXISmEB_6804984382',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        url, smuggled_data = unsmuggle_url(url, {})
+
+        mobj = re.match(self._VALID_URL, url)
+        course_id = mobj.group('course_id')
+        video_id = mobj.group('id')
+
+        base_url = smuggled_data.get('base_url') or self._extract_base_url(course_id, video_id)
+
+        settings = self._download_xml(
+            '%s/content/content_%s/videosettings.xml?v=1' % (base_url, video_id),
+            video_id, 'Downloading video settings XML')
+
+        _, title = self._extract_chapter_and_title(xpath_text(
+            settings, './/Title', 'title', fatal=True))
+
+        formats = []
+
+        for sources in settings.findall(compat_xpath('.//MediaSources')):
+            if sources.get('videoType') == 'smoothstreaming':
+                continue
+            for source in sources.findall(compat_xpath('./MediaSource')):
+                video_url = source.text
+                if not video_url or not video_url.startswith('http'):
+                    continue
+                video_mode = source.get('videoMode')
+                height = int_or_none(self._search_regex(
+                    r'^(\d+)[pP]$', video_mode or '', 'height', default=None))
+                codec = source.get('codec')
+                acodec, vcodec = [None] * 2
+                if codec:
+                    codecs = codec.split(',')
+                    if len(codecs) == 2:
+                        acodec, vcodec = codecs
+                    elif len(codecs) == 1:
+                        vcodec = codecs[0]
+                formats.append({
+                    'url': video_url,
+                    'format_id': video_mode,
+                    'height': height,
+                    'acodec': acodec,
+                    'vcodec': vcodec,
+                })
+        self._sort_formats(formats)
+
+        subtitles = {}
+        for source in settings.findall(compat_xpath('.//MarkerResourceSource')):
+            subtitle_url = source.text
+            if not subtitle_url:
+                continue
+            subtitles.setdefault('en', []).append({
+                'url': '%s/%s' % (base_url, subtitle_url),
+                'ext': source.get('type'),
+            })
+
+        return {
+            'id': video_id,
+            'title': title,
+            'subtitles': subtitles,
+            'formats': formats
+        }
+
+
+class MicrosoftVirtualAcademyCourseIE(MicrosoftVirtualAcademyBaseIE):
+    IE_NAME = 'mva:course'
+    IE_DESC = 'Microsoft Virtual Academy courses'
+    _VALID_URL = r'(?:%s:|https?://(?:mva\.microsoft|microsoftvirtualacademy)\.com/[^/]+/training-courses/(?P<display_id>[^/?#&]+)-)(?P<id>\d+)' % IE_NAME
+
+    _TESTS = [{
+        'url': 'https://mva.microsoft.com/en-US/training-courses/microsoft-azure-fundamentals-virtual-machines-11788',
+        'info_dict': {
+            'id': '11788',
+            'title': 'Microsoft Azure Fundamentals: Virtual Machines',
+        },
+        'playlist_count': 36,
+    }, {
+        # with emphasized chapters
+        'url': 'https://mva.microsoft.com/en-US/training-courses/developing-windows-10-games-with-construct-2-16335',
+        'info_dict': {
+            'id': '16335',
+            'title': 'Developing Windows 10 Games with Construct 2',
+        },
+        'playlist_count': 10,
+    }, {
+        'url': 'https://www.microsoftvirtualacademy.com/en-US/training-courses/microsoft-azure-fundamentals-virtual-machines-11788',
+        'only_matching': True,
+    }, {
+        'url': 'mva:course:11788',
+        'only_matching': True,
+    }]
+
+    @classmethod
+    def suitable(cls, url):
+        return False if MicrosoftVirtualAcademyIE.suitable(url) else super(
+            MicrosoftVirtualAcademyCourseIE, cls).suitable(url)
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        course_id = mobj.group('id')
+        display_id = mobj.group('display_id')
+
+        base_url = self._extract_base_url(course_id, display_id)
+
+        manifest = self._download_json(
+            '%s/imsmanifestlite.json' % base_url,
+            display_id, 'Downloading course manifest JSON')['manifest']
+
+        organization = manifest['organizations']['organization'][0]
+
+        entries = []
+        for chapter in organization['item']:
+            chapter_number, chapter_title = self._extract_chapter_and_title(chapter.get('title'))
+            chapter_id = chapter.get('@identifier')
+            for item in chapter.get('item', []):
+                item_id = item.get('@identifier')
+                if not item_id:
+                    continue
+                metadata = item.get('resource', {}).get('metadata') or {}
+                if metadata.get('learningresourcetype') != 'Video':
+                    continue
+                _, title = self._extract_chapter_and_title(item.get('title'))
+                duration = parse_duration(metadata.get('duration'))
+                description = metadata.get('description')
+                entries.append({
+                    '_type': 'url_transparent',
+                    'url': smuggle_url(
+                        'mva:%s:%s' % (course_id, item_id), {'base_url': base_url}),
+                    'title': title,
+                    'description': description,
+                    'duration': duration,
+                    'chapter': chapter_title,
+                    'chapter_number': chapter_number,
+                    'chapter_id': chapter_id,
+                })
+
+        title = organization.get('title') or manifest.get('metadata', {}).get('title')
+
+        return self.playlist_result(entries, course_id, title)

From c52f4efaee2386a72c3f6b694fb4f4c3132ced55 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 8 May 2016 20:10:20 +0600
Subject: [PATCH 4881/6219] [mva] Improve _VALID_URLs

---
 youtube_dl/extractor/microsoftvirtualacademy.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/microsoftvirtualacademy.py b/youtube_dl/extractor/microsoftvirtualacademy.py
index b7fea47ee..afd3e98ec 100644
--- a/youtube_dl/extractor/microsoftvirtualacademy.py
+++ b/youtube_dl/extractor/microsoftvirtualacademy.py
@@ -31,7 +31,7 @@ class MicrosoftVirtualAcademyBaseIE(InfoExtractor):
 class MicrosoftVirtualAcademyIE(MicrosoftVirtualAcademyBaseIE):
     IE_NAME = 'mva'
     IE_DESC = 'Microsoft Virtual Academy videos'
-    _VALID_URL = r'(?:%s:|https?://(?:mva\.microsoft|microsoftvirtualacademy)\.com/[^/]+/training-courses/[^/?#&]+-)(?P<course_id>\d+)(?::|\?l=)(?P<id>[\da-zA-Z]+_\d+)' % IE_NAME
+    _VALID_URL = r'(?:%s:|https?://(?:mva\.microsoft|(?:www\.)?microsoftvirtualacademy)\.com/[^/]+/training-courses/[^/?#&]+-)(?P<course_id>\d+)(?::|\?l=)(?P<id>[\da-zA-Z]+_\d+)' % IE_NAME
 
     _TESTS = [{
         'url': 'https://mva.microsoft.com/en-US/training-courses/microsoft-azure-fundamentals-virtual-machines-11788?l=gfVXISmEB_6804984382',
@@ -118,7 +118,7 @@ class MicrosoftVirtualAcademyIE(MicrosoftVirtualAcademyBaseIE):
 class MicrosoftVirtualAcademyCourseIE(MicrosoftVirtualAcademyBaseIE):
     IE_NAME = 'mva:course'
     IE_DESC = 'Microsoft Virtual Academy courses'
-    _VALID_URL = r'(?:%s:|https?://(?:mva\.microsoft|microsoftvirtualacademy)\.com/[^/]+/training-courses/(?P<display_id>[^/?#&]+)-)(?P<id>\d+)' % IE_NAME
+    _VALID_URL = r'(?:%s:|https?://(?:mva\.microsoft|(?:www\.)?microsoftvirtualacademy)\.com/[^/]+/training-courses/(?P<display_id>[^/?#&]+)-)(?P<id>\d+)' % IE_NAME
 
     _TESTS = [{
         'url': 'https://mva.microsoft.com/en-US/training-courses/microsoft-azure-fundamentals-virtual-machines-11788',

From f1f6f5aa5e2a6d66fa54d35bf3e8b3626e85ee73 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tom=C3=A1=C5=A1=20=C4=8Cech?= <sleep_walker@suse.cz>
Date: Sat, 7 May 2016 20:15:49 +0200
Subject: [PATCH 4882/6219] [ceskatelevize] Add support for live streams

Live streams has no playlist title, use title of the stream containing
TV channel name. Internal m3u8 handler doesn't seem to handle well
continuous streams. Add test for live stream. Remove no longer
reachable test.
---
 youtube_dl/extractor/ceskatelevize.py | 35 +++++++++++++++++++--------
 1 file changed, 25 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/ceskatelevize.py b/youtube_dl/extractor/ceskatelevize.py
index 6652c8e42..b41888531 100644
--- a/youtube_dl/extractor/ceskatelevize.py
+++ b/youtube_dl/extractor/ceskatelevize.py
@@ -33,14 +33,13 @@ class CeskaTelevizeIE(InfoExtractor):
             'skip_download': True,
         },
     }, {
-        'url': 'http://www.ceskatelevize.cz/ivysilani/10532695142-prvni-republika/bonus/14716-zpevacka-z-duparny-bobina',
+        # live stream
+        'url': 'http://www.ceskatelevize.cz/ivysilani/zive/ct4/',
         'info_dict': {
-            'id': '61924494876844374',
+            'id': 402,
             'ext': 'mp4',
-            'title': 'První republika: Zpěvačka z Dupárny Bobina',
-            'description': 'Sága mapující atmosféru první republiky od r. 1918 do r. 1945.',
-            'thumbnail': 're:^https?://.*\.jpg',
-            'duration': 88.4,
+            'title': 're:ČT Sport.*',
+            'is_live': True,
         },
         'params': {
             # m3u8 download
@@ -118,19 +117,21 @@ class CeskaTelevizeIE(InfoExtractor):
         req = sanitized_Request(compat_urllib_parse_unquote(playlist_url))
         req.add_header('Referer', url)
 
-        playlist_title = self._og_search_title(webpage)
-        playlist_description = self._og_search_description(webpage)
+        playlist_title = self._og_search_title(webpage, default=None)
+        playlist_description = self._og_search_description(webpage, default=None)
 
         playlist = self._download_json(req, playlist_id)['playlist']
         playlist_len = len(playlist)
 
         entries = []
         for item in playlist:
+            is_live = item['type'] == 'LIVE'
             formats = []
             for format_id, stream_url in item['streamUrls'].items():
                 formats.extend(self._extract_m3u8_formats(
                     stream_url, playlist_id, 'mp4',
-                    entry_protocol='m3u8_native', fatal=False))
+                    entry_protocol='m3u8' if is_live else 'm3u8_native',
+                    fatal=False))
             self._sort_formats(formats)
 
             item_id = item.get('id') or item['assetId']
@@ -145,14 +146,28 @@ class CeskaTelevizeIE(InfoExtractor):
                 if subs:
                     subtitles = self.extract_subtitles(episode_id, subs)
 
+            if playlist_len == 1:
+                if is_live:
+                    # live streams has channel name in title
+                    final_title = self._live_title(title)
+                elif playlist_title:
+                    # title is always set (no KeyError caught)
+                    # and gives good fallback
+                    final_title = title
+                else:
+                    final_title = playlist_title
+            else:
+                final_title = '%s (%s)' % (playlist_title, title)
+
             entries.append({
                 'id': item_id,
-                'title': playlist_title if playlist_len == 1 else '%s (%s)' % (playlist_title, title),
+                'title': final_title,
                 'description': playlist_description if playlist_len == 1 else None,
                 'thumbnail': thumbnail,
                 'duration': duration,
                 'formats': formats,
                 'subtitles': subtitles,
+                'is_live': is_live,
             })
 
         return self.playlist_result(entries, playlist_id, playlist_title, playlist_description)

From 3951e7eb9305448aab6395f4303ed7ab19248c52 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 9 May 2016 20:37:20 +0600
Subject: [PATCH 4883/6219] [ceskatelevize] Simplify, restore bonus video test
 and skip georestricted test (Closes #9431)

---
 youtube_dl/extractor/ceskatelevize.py | 29 ++++++++++++++++++---------
 1 file changed, 19 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/ceskatelevize.py b/youtube_dl/extractor/ceskatelevize.py
index b41888531..5a58d1777 100644
--- a/youtube_dl/extractor/ceskatelevize.py
+++ b/youtube_dl/extractor/ceskatelevize.py
@@ -32,19 +32,34 @@ class CeskaTelevizeIE(InfoExtractor):
             # m3u8 download
             'skip_download': True,
         },
+    }, {
+        'url': 'http://www.ceskatelevize.cz/ivysilani/10441294653-hyde-park-civilizace/215411058090502/bonus/20641-bonus-01-en',
+        'info_dict': {
+            'id': '61924494877028507',
+            'ext': 'mp4',
+            'title': 'Hyde Park Civilizace: Bonus 01 - En',
+            'description': 'English Subtittles',
+            'thumbnail': 're:^https?://.*\.jpg',
+            'duration': 81.3,
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
     }, {
         # live stream
         'url': 'http://www.ceskatelevize.cz/ivysilani/zive/ct4/',
         'info_dict': {
             'id': 402,
             'ext': 'mp4',
-            'title': 're:ČT Sport.*',
+            'title': 're:^ČT Sport \d{4}-\d{2}-\d{2} \d{2}:\d{2}$',
             'is_live': True,
         },
         'params': {
             # m3u8 download
             'skip_download': True,
         },
+        'skip': 'Georestricted to Czech Republic',
     }, {
         # video with 18+ caution trailer
         'url': 'http://www.ceskatelevize.cz/porady/10520528904-queer/215562210900007-bogotart/',
@@ -125,7 +140,7 @@ class CeskaTelevizeIE(InfoExtractor):
 
         entries = []
         for item in playlist:
-            is_live = item['type'] == 'LIVE'
+            is_live = item.get('type') == 'LIVE'
             formats = []
             for format_id, stream_url in item['streamUrls'].items():
                 formats.extend(self._extract_m3u8_formats(
@@ -147,15 +162,9 @@ class CeskaTelevizeIE(InfoExtractor):
                     subtitles = self.extract_subtitles(episode_id, subs)
 
             if playlist_len == 1:
+                final_title = playlist_title or title
                 if is_live:
-                    # live streams has channel name in title
-                    final_title = self._live_title(title)
-                elif playlist_title:
-                    # title is always set (no KeyError caught)
-                    # and gives good fallback
-                    final_title = title
-                else:
-                    final_title = playlist_title
+                    final_title = self._live_title(final_title)
             else:
                 final_title = '%s (%s)' % (playlist_title, title)
 

From 965fefdcd879405c3e4b5604513719353ba8474a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 9 May 2016 20:38:33 +0600
Subject: [PATCH 4884/6219] Credit @sleep-walker for #9431

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index bf860b7f7..5ca71ace7 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -171,3 +171,4 @@ Philip Huppert
 blahgeek
 Kevin Deldycke
 inondle
+Tomáš Čech

From c15c47d19bfeeacd42f44dd7736f175711a91346 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 9 May 2016 20:45:03 +0600
Subject: [PATCH 4885/6219] [downloader/hls] Remove EXT-X-MEDIA-SEQUENCE from
 unsupported features for hlsnative

---
 youtube_dl/downloader/hls.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index d7b34bde3..dcedc9a64 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -23,7 +23,9 @@ class HlsFD(FragmentFD):
         UNSUPPORTED_FEATURES = (
             r'#EXT-X-KEY:METHOD=(?!NONE)',  # encrypted streams [1]
             r'#EXT-X-BYTERANGE',  # playlists composed of byte ranges of media files [2]
-            r'#EXT-X-MEDIA-SEQUENCE:(?!0$)',  # live streams [3]
+            # Live streams heuristic does not always work (e.g. geo restricted to Germany
+            # http://hls-geo.daserste.de/i/videoportal/Film/c_620000/622873/format,716451,716457,716450,716458,716459,.mp4.csmil/index_4_av.m3u8?null=0)
+            #r'#EXT-X-MEDIA-SEQUENCE:(?!0$)',  # live streams [3]
             # 1. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.2.4
             # 2. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.2.2
             # 3. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.3.2

From 6104cc2985c36e996df1aae7cfcc686f3bae0b82 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 9 May 2016 20:55:37 +0600
Subject: [PATCH 4886/6219] [downloader/hls] Add event media playlists to
 unsupported features of hlsnative

---
 youtube_dl/downloader/hls.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index dcedc9a64..a8279718b 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -26,9 +26,12 @@ class HlsFD(FragmentFD):
             # Live streams heuristic does not always work (e.g. geo restricted to Germany
             # http://hls-geo.daserste.de/i/videoportal/Film/c_620000/622873/format,716451,716457,716450,716458,716459,.mp4.csmil/index_4_av.m3u8?null=0)
             #r'#EXT-X-MEDIA-SEQUENCE:(?!0$)',  # live streams [3]
+            r'#EXT-X-PLAYLIST-TYPE:EVENT',  # media segments may be appended to the end of
+                                            # event media playlists [4]
             # 1. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.2.4
             # 2. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.2.2
             # 3. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.3.2
+            # 4. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.3.5
         )
         return all(not re.search(feature, manifest) for feature in UNSUPPORTED_FEATURES)
 

From fe40f9eef2483748ed83c9749f35220143d8cc9b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 9 May 2016 21:55:03 +0600
Subject: [PATCH 4887/6219] [compat] Add compat_setenv

---
 test/test_compat.py  |  8 ++++++++
 youtube_dl/compat.py | 10 ++++++++++
 2 files changed, 18 insertions(+)

diff --git a/test/test_compat.py b/test/test_compat.py
index 618668210..0d751a856 100644
--- a/test/test_compat.py
+++ b/test/test_compat.py
@@ -13,6 +13,7 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 from youtube_dl.utils import get_filesystem_encoding
 from youtube_dl.compat import (
     compat_getenv,
+    compat_setenv,
     compat_etree_fromstring,
     compat_expanduser,
     compat_shlex_split,
@@ -31,6 +32,13 @@ class TestCompat(unittest.TestCase):
             else test_str.encode(get_filesystem_encoding()))
         self.assertEqual(compat_getenv('YOUTUBE-DL-TEST'), test_str)
 
+    def test_compat_setenv(self):
+        test_var = 'YOUTUBE-DL-TEST'
+        test_str = 'тест'
+        compat_setenv(test_var, test_str)
+        compat_getenv(test_var)
+        self.assertEqual(compat_getenv(test_var), test_str)
+
     def test_compat_expanduser(self):
         old_home = os.environ.get('HOME')
         test_str = 'C:\Documents and Settings\тест\Application Data'
diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 0b6c5ca7a..12b53cdc8 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -373,6 +373,9 @@ compat_os_name = os._name if os.name == 'java' else os.name
 if sys.version_info >= (3, 0):
     compat_getenv = os.getenv
     compat_expanduser = os.path.expanduser
+
+    def compat_setenv(key, value, env=os.environ):
+        env[key] = value
 else:
     # Environment variables should be decoded with filesystem encoding.
     # Otherwise it will fail if any non-ASCII characters present (see #3854 #3217 #2918)
@@ -384,6 +387,12 @@ else:
             env = env.decode(get_filesystem_encoding())
         return env
 
+    def compat_setenv(key, value, env=os.environ):
+        def encode(v):
+            from .utils import get_filesystem_encoding
+            return v.encode(get_filesystem_encoding()) if isinstance(v, compat_str) else v
+        env[encode(key)] = encode(value)
+
     # HACK: The default implementations of os.path.expanduser from cpython do not decode
     # environment variables with filesystem encoding. We will work around this by
     # providing adjusted implementations.
@@ -604,6 +613,7 @@ __all__ = [
     'compat_os_name',
     'compat_parse_qs',
     'compat_print',
+    'compat_setenv',
     'compat_shlex_split',
     'compat_socket_create_connection',
     'compat_str',

From 129263875403841da485ac74b09960d862d23f63 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 9 May 2016 21:58:38 +0600
Subject: [PATCH 4888/6219] [test_compat] Use compat_setenv

---
 test/test_compat.py | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/test/test_compat.py b/test/test_compat.py
index 0d751a856..afe6bd528 100644
--- a/test/test_compat.py
+++ b/test/test_compat.py
@@ -27,9 +27,7 @@ from youtube_dl.compat import (
 class TestCompat(unittest.TestCase):
     def test_compat_getenv(self):
         test_str = 'тест'
-        os.environ['YOUTUBE-DL-TEST'] = (
-            test_str if sys.version_info >= (3, 0)
-            else test_str.encode(get_filesystem_encoding()))
+        compat_setenv('YOUTUBE-DL-TEST', test_str)
         self.assertEqual(compat_getenv('YOUTUBE-DL-TEST'), test_str)
 
     def test_compat_setenv(self):
@@ -42,11 +40,9 @@ class TestCompat(unittest.TestCase):
     def test_compat_expanduser(self):
         old_home = os.environ.get('HOME')
         test_str = 'C:\Documents and Settings\тест\Application Data'
-        os.environ['HOME'] = (
-            test_str if sys.version_info >= (3, 0)
-            else test_str.encode(get_filesystem_encoding()))
+        compat_setenv('HOME', test_str)
         self.assertEqual(compat_expanduser('~'), test_str)
-        os.environ['HOME'] = old_home
+        compat_setenv('HOME', old_home)
 
     def test_all_present(self):
         import youtube_dl.compat

From 20cfdcc910d0bc2ee4b0ee38bdf5e6ecb67e5731 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 9 May 2016 22:00:14 +0600
Subject: [PATCH 4889/6219] [test_compat] Avoid None values for compat_setenv

---
 test/test_compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/test_compat.py b/test/test_compat.py
index afe6bd528..b20814249 100644
--- a/test/test_compat.py
+++ b/test/test_compat.py
@@ -42,7 +42,7 @@ class TestCompat(unittest.TestCase):
         test_str = 'C:\Documents and Settings\тест\Application Data'
         compat_setenv('HOME', test_str)
         self.assertEqual(compat_expanduser('~'), test_str)
-        compat_setenv('HOME', old_home)
+        compat_setenv('HOME', old_home or '')
 
     def test_all_present(self):
         import youtube_dl.compat

From e62d9c5caaa972ef4b1ed5d6ab5ee4a087a4ba95 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 9 May 2016 22:05:12 +0600
Subject: [PATCH 4890/6219] [downloader/external] Call ffmpeg with with
 HTTP_PROXY env variable set (#9437)

---
 youtube_dl/downloader/external.py | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index 45f49c350..3a73cee1c 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -6,6 +6,7 @@ import sys
 import re
 
 from .common import FileDownloader
+from ..compat import compat_setenv
 from ..postprocessor.ffmpeg import FFmpegPostProcessor, EXT_TO_OUT_FORMATS
 from ..utils import (
     cli_option,
@@ -198,6 +199,18 @@ class FFmpegFD(ExternalFD):
                 '-headers',
                 ''.join('%s: %s\r\n' % (key, val) for key, val in headers.items())]
 
+        env = None
+        proxy = self.params.get('proxy')
+        if proxy:
+            if not re.match(r'^[\da-zA-Z]+://', proxy):
+                proxy = 'http://%s' % proxy
+            # Since December 2015 ffmpeg supports -http_proxy option (see
+            # http://git.videolan.org/?p=ffmpeg.git;a=commit;h=b4eb1f29ebddd60c41a2eb39f5af701e38e0d3fd)
+            # We could switch to the following code if we are able to detect version properly
+            # args += ['-http_proxy', proxy]
+            env = os.environ.copy()
+            compat_setenv('HTTP_PROXY', proxy, env=env)
+
         protocol = info_dict.get('protocol')
 
         if protocol == 'rtmp':
@@ -239,7 +252,7 @@ class FFmpegFD(ExternalFD):
 
         self._debug_cmd(args)
 
-        proc = subprocess.Popen(args, stdin=subprocess.PIPE)
+        proc = subprocess.Popen(args, stdin=subprocess.PIPE, env=env)
         try:
             retval = proc.wait()
         except KeyboardInterrupt:

From fad7bbec3a1fb62964c8e6637dfd535fabe9c133 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 9 May 2016 22:15:55 +0600
Subject: [PATCH 4891/6219] [test_compat] Remove unused import

---
 test/test_compat.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/test/test_compat.py b/test/test_compat.py
index b20814249..9adf75763 100644
--- a/test/test_compat.py
+++ b/test/test_compat.py
@@ -10,7 +10,6 @@ import unittest
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 
-from youtube_dl.utils import get_filesystem_encoding
 from youtube_dl.compat import (
     compat_getenv,
     compat_setenv,

From 2937590e8b70384ef91bdadbb56a55897aab0837 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 9 May 2016 22:16:33 +0600
Subject: [PATCH 4892/6219] [downloader/hls] PEP 8

---
 youtube_dl/downloader/hls.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index a8279718b..62136ee54 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -25,7 +25,7 @@ class HlsFD(FragmentFD):
             r'#EXT-X-BYTERANGE',  # playlists composed of byte ranges of media files [2]
             # Live streams heuristic does not always work (e.g. geo restricted to Germany
             # http://hls-geo.daserste.de/i/videoportal/Film/c_620000/622873/format,716451,716457,716450,716458,716459,.mp4.csmil/index_4_av.m3u8?null=0)
-            #r'#EXT-X-MEDIA-SEQUENCE:(?!0$)',  # live streams [3]
+            # r'#EXT-X-MEDIA-SEQUENCE:(?!0$)',  # live streams [3]
             r'#EXT-X-PLAYLIST-TYPE:EVENT',  # media segments may be appended to the end of
                                             # event media playlists [4]
             # 1. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.2.4

From 4350b74545ea3d3ce072444655613bc6974d5516 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 23 Apr 2016 15:44:34 +0800
Subject: [PATCH 4893/6219] [socks] Add socks.py from @bluec0re's public domain
 implementation

https://gist.github.com/bluec0re/cafd3764412967417fd3
---
 youtube_dl/socks.py | 336 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 336 insertions(+)
 create mode 100644 youtube_dl/socks.py

diff --git a/youtube_dl/socks.py b/youtube_dl/socks.py
new file mode 100644
index 000000000..b0c36a189
--- /dev/null
+++ b/youtube_dl/socks.py
@@ -0,0 +1,336 @@
+# This is free and unencumbered software released into the public domain.
+# 
+# Anyone is free to copy, modify, publish, use, compile, sell, or
+# distribute this software, either in source code form or as a compiled
+# binary, for any purpose, commercial or non-commercial, and by any
+# means.
+# 
+# In jurisdictions that recognize copyright laws, the author or authors
+# of this software dedicate any and all copyright interest in the
+# software to the public domain. We make this dedication for the benefit
+# of the public at large and to the detriment of our heirs and
+# successors. We intend this dedication to be an overt act of
+# relinquishment in perpetuity of all present and future rights to this
+# software under copyright law.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+# OTHER DEALINGS IN THE SOFTWARE.
+# 
+# For more information, please refer to <http://unlicense.org/>
+#
+# Example:
+# import socks
+# import ftplib
+# import socket
+#
+# socks.patch_socket()
+#
+# f = ftplib.FTP('ftp.kernel.org')
+# f.login()
+# print f.retrlines('LIST')
+# f.quit()
+# 
+# s = socket.create_connection(('www.google.com', 80))
+# s.sendall('HEAD / HTTP/1.0\r\n\r\n')
+# print s.recv(1024)
+# s.close()
+from __future__ import unicode_literals
+import os
+import struct
+import socket
+import time
+
+__author__ = 'Timo Schmid <coding@timoschmid.de>'
+
+_orig_socket = socket.socket
+
+try:
+    from collections import namedtuple
+except ImportError:
+    from Collections import namedtuple
+
+try:
+    from urllib.parse import urlparse
+except:
+    from urlparse import urlparse
+
+try:
+    from enum import Enum
+except ImportError:
+    Enum = object
+
+
+class ProxyError(IOError): pass
+class Socks4Error(ProxyError):
+    CODES = {
+        0x5B: 'request rejected or failed',
+        0x5C: 'request rejected becasue SOCKS server cannot connect to identd on the client',
+        0x5D: 'request rejected because the client program and identd report different user-ids'
+    }
+    def __init__(self, code=None, msg=None):
+        if code is not None and msg is None:
+            msg = self.CODES.get(code)
+            if msg is None:
+                msg = 'unknown error'
+        super(Socks4Error, self).__init__(code, msg)
+
+class Socks5Error(Socks4Error):
+    CODES = {
+        0x01: 'general SOCKS server failure',
+        0x02: 'connection not allowed by ruleset',
+        0x03: 'Network unreachable',
+        0x04: 'Host unreachable',
+        0x05: 'Connection refused',
+        0x06: 'TTL expired',
+        0x07: 'Command not supported',
+        0x08: 'Address type not supported',
+        0xFE: 'unknown username or invalid password',
+        0xFF: 'all offered authentication methods were rejected'
+    }
+
+class ProxyType(Enum):
+    SOCKS4  = 0
+    SOCKS4A = 1
+    SOCKS5  = 2
+
+Proxy = namedtuple('Proxy', ('type', 'host', 'port', 'username', 'password', 'remote_dns'))
+
+_default_proxy = None
+
+def setdefaultproxy(proxytype=None, addr=None, port=None, rdns=True, username=None, password=None, allow_env_override=True):
+    global _default_proxy
+    if allow_env_override:
+        all_proxy = os.environ.get('ALL_PROXY', os.environ.get('all_proxy'))
+        if all_proxy:
+            all_proxy = urlparse(all_proxy)
+            if all_proxy.scheme.startswith('socks'):
+                if all_proxy.scheme == 'socks' or all_proxy.scheme == 'socks4':
+                    proxytype = ProxyType.SOCKS4
+                elif all_proxy.scheme == 'socks4a':
+                    proxytype = ProxyType.SOCKS4A
+                elif all_proxy.scheme == 'socks5':
+                    proxytype = ProxyType.SOCKS5
+                addr = all_proxy.hostname
+                port = all_proxy.port
+                username = all_proxy.username
+                password = all_proxy.password
+
+    if proxytype is not None:
+        _default_proxy = Proxy(proxytype, addr, port, username, password, rdns)
+
+
+def wrap_socket(sock):
+    return socksocket(_sock=sock._sock)
+
+def wrap_module(module):
+    if hasattr(module, 'socket'):
+        sock = module.socket
+        if isinstance(sock, socket.socket):
+            module.socket = sockssocket
+        elif hasattr(socket, 'socket'):
+            socket.socket = sockssocket
+
+def patch_socket():
+    import sys
+    if 'socket' not in sys.modules:
+        import socket
+    sys.modules['socket'].socket = sockssocket
+
+
+class sockssocket(socket.socket):
+    def __init__(self, *args, **kwargs):
+        self.__proxy = None
+        if 'proxy' in kwargs:
+            self.__proxy = kwargs['proxy']
+            del kwargs['proxy']
+        super(sockssocket, self).__init__(*args, **kwargs)
+
+    @property
+    def _proxy(self):
+        if self.__proxy:
+            return self.__proxy
+        return _default_proxy
+
+    @property
+    def _proxy_port(self):
+        if self._proxy:
+            if self._proxy.port:
+                return self._proxy.port
+            return 1080
+        return None
+
+    def setproxy(self, proxytype=None, addr=None, port=None, rdns=True, username=None, password=None):
+        if proxytype is None:
+            self.__proxy = None
+        else:
+            self.__proxy = Proxy(proxytype, addr, port, username, password, rdns)
+
+    def recvall(self, cnt):
+        data = b''
+        while len(data) < cnt:
+            cur = self.recv(cnt - len(data))
+            if not cur:
+                raise IOError("{0} bytes missing".format(cnt-len(data)))
+            data += cur
+        return data
+
+    def _setup_socks4(self, address, is_4a=False):
+        destaddr, port = address
+
+        try:
+            ipaddr = socket.inet_aton(destaddr)
+        except socket.error:
+            if is_4a and self._proxy.remote_dns:
+                ipaddr = struct.pack('!BBBB', 0, 0, 0, 0xFF)
+            else:
+                ipaddr = socket.inet_aton(socket.gethostbyname(destaddr))
+
+        packet = struct.pack('!BBH', 0x4, 0x1, port) + ipaddr
+        if self._proxy.username:
+            username = self._proxy.username
+            if hasattr(username, 'encode'):
+                username = username.encode()
+            packet += struct.pack('!{0}s'.format(len(username)+1), username)
+        else:
+            packet += b'\x00'
+
+        if is_4a and self._proxy.remote_dns:
+            if hasattr(destaddr, 'encode'):
+                destaddr = destaddr.encode()
+            packet += struct.pack('!{0}s'.format(len(destaddr)+1), destaddr)
+
+        self.sendall(packet)
+
+        packet = self.recvall(8)
+        nbyte, resp_code, dstport, dsthost = struct.unpack('!BBHI', packet)
+
+        # check valid response
+        if nbyte != 0x00:
+            self.close()
+            raise ProxyError(0, "Invalid response from server. Expected {0:02x} got {1:02x}".format(0, nbyte))
+
+        # access granted
+        if resp_code != 0x5a:
+            self.close()
+            raise Socks4Error(resp_code)
+
+    def _setup_socks5(self, address):
+        destaddr, port = address
+
+        try:
+            ipaddr = socket.inet_aton(destaddr)
+        except socket.error:
+            if self._proxy.remote_dns:
+                ipaddr = None
+            else:
+                ipaddr = socket.inet_aton(socket.gethostbyname(destaddr))
+
+        auth_methods = 1
+        if self._proxy.username and self._proxy.password:
+            # two auth methods available
+            auth_methods = 2
+        packet = struct.pack('!BBB', 0x5, auth_methods, 0x00) # no auth
+        if self._proxy.username and self._proxy.password:
+            packet += struct.pack('!B', 0x02) # user/pass auth
+
+        self.sendall(packet)
+
+        packet = self.recvall(2)
+        version, method = struct.unpack('!BB', packet)
+
+        # check valid response
+        if version != 0x05:
+            self.close()
+            raise ProxyError(0, "Invalid response from server. Expected {0:02x} got {1:02x}".format(5, version))
+
+        # no auth methods
+        if method == 0xFF:
+            self.close()
+            raise Socks5Error(method)
+
+        # user/pass auth
+        if method == 0x01:
+            username = self._proxy.username
+            if hasattr(username, 'encode'):
+                username = username.encode()
+            password = self._proxy.password
+            if hasattr(password, 'encode'):
+                password = password.encode()
+            packet = struct.pack('!BB', 1, len(username)) + username
+            packet += struct.pack('!B', len(password)) + password
+            self.sendall(packet)
+
+            packet = self.recvall(2)
+            version, status = struct.unpack('!BB', packet)
+
+            if version != 0x01:
+                self.close()
+                raise ProxyError(0, "Invalid response from server. Expected {0:02x} got {1:02x}".format(1, version))
+
+            if status != 0x00:
+                self.close()
+                raise Socks5Error(1)
+        elif method == 0x00: # no auth
+            pass
+
+
+        packet = struct.pack('!BBB', 5, 1, 0)
+        if ipaddr is None:
+            if hasattr(destaddr, 'encode'):
+                destaddr = destaddr.encode()
+            packet += struct.pack('!BB', 3, len(destaddr)) + destaddr
+        else:
+            packet += struct.pack('!B', 1) + ipaddr
+        packet += struct.pack('!H', port)
+
+        self.sendall(packet)
+
+        packet = self.recvall(4)
+        version, status, _, atype = struct.unpack('!BBBB', packet)
+
+        if version != 0x05:
+            self.close()
+            raise ProxyError(0, "Invalid response from server. Expected {0:02x} got {1:02x}".format(5, version))
+
+        if status != 0x00:
+            self.close()
+            raise Socks5Error(status)
+
+        if atype == 0x01:
+            destaddr = self.recvall(4)
+        elif atype == 0x03:
+            alen = struct.unpack('!B', self.recv(1))[0]
+            destaddr = self.recvall(alen)
+        elif atype == 0x04:
+            destaddr = self.recvall(16)
+        destport = struct.unpack('!H', self.recvall(2))[0]
+
+    def _make_proxy(self, connect_func, address):
+        if self._proxy.type == ProxyType.SOCKS4:
+            result = connect_func(self, (self._proxy.host, self._proxy_port))
+            if result != 0 and result is not None:
+                return result
+            self._setup_socks4(address)
+        elif self._proxy.type == ProxyType.SOCKS4A:
+            result = connect_func(self, (self._proxy.host, self._proxy_port))
+            if result != 0 and result is not None:
+                return result
+            self._setup_socks4(address, is_4a=True)
+        elif self._proxy.type == ProxyType.SOCKS5:
+            result = connect_func(self, (self._proxy.host, self._proxy_port))
+            if result != 0 and result is not None:
+                return result
+            self._setup_socks5(address)
+        else:
+            return connect_func(self, address)
+
+    def connect(self, address):
+        self._make_proxy(_orig_socket.connect, address)
+
+    def connect_ex(self, address):
+        return self._make_proxy(_orig_socket.connect_ex, address)

From dab0daeeb0929b9b560d2b9a5f39c1e2e6dfa449 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 23 Apr 2016 18:28:49 +0800
Subject: [PATCH 4894/6219] [utils,compat] Move struct_pack and struct_unpack
 to compat.py

---
 test/test_compat.py          |  5 +++++
 test/test_utils.py           |  4 ----
 youtube_dl/compat.py         | 23 +++++++++++++++++++++++
 youtube_dl/downloader/f4m.py |  4 ++--
 youtube_dl/extractor/rtve.py |  4 +++-
 youtube_dl/swfinterp.py      |  6 ++++--
 youtube_dl/utils.py          | 20 +-------------------
 7 files changed, 38 insertions(+), 28 deletions(-)

diff --git a/test/test_compat.py b/test/test_compat.py
index 9adf75763..dd62a5d6b 100644
--- a/test/test_compat.py
+++ b/test/test_compat.py
@@ -20,6 +20,7 @@ from youtube_dl.compat import (
     compat_urllib_parse_unquote,
     compat_urllib_parse_unquote_plus,
     compat_urllib_parse_urlencode,
+    struct_unpack,
 )
 
 
@@ -102,5 +103,9 @@ class TestCompat(unittest.TestCase):
         self.assertTrue(isinstance(doc.find('chinese').text, compat_str))
         self.assertTrue(isinstance(doc.find('foo/bar').text, compat_str))
 
+    def test_struct_unpack(self):
+        self.assertEqual(struct_unpack('!B', b'\x00'), (0,))
+
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/test/test_utils.py b/test/test_utils.py
index 00ada95ec..5702ffa97 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -55,7 +55,6 @@ from youtube_dl.utils import (
     smuggle_url,
     str_to_int,
     strip_jsonp,
-    struct_unpack,
     timeconvert,
     unescapeHTML,
     unified_strdate,
@@ -457,9 +456,6 @@ class TestUtil(unittest.TestCase):
         testPL(5, 2, (2, 99), [2, 3, 4])
         testPL(5, 2, (20, 99), [])
 
-    def test_struct_unpack(self):
-        self.assertEqual(struct_unpack('!B', b'\x00'), (0,))
-
     def test_read_batch_urls(self):
         f = io.StringIO('''\xef\xbb\xbf foo
             bar\r
diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 12b53cdc8..f697bee7e 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -11,6 +11,7 @@ import re
 import shlex
 import shutil
 import socket
+import struct
 import subprocess
 import sys
 import itertools
@@ -592,6 +593,26 @@ if sys.version_info >= (3, 0):
 else:
     from tokenize import generate_tokens as compat_tokenize_tokenize
 
+
+try:
+    struct.pack('!I', 0)
+except TypeError:
+    # In Python 2.6 and 2.7.x < 2.7.7, struct requires a bytes argument
+    # See https://bugs.python.org/issue19099
+    def struct_pack(spec, *args):
+        if isinstance(spec, compat_str):
+            spec = spec.encode('ascii')
+        return struct.pack(spec, *args)
+
+    def struct_unpack(spec, *args):
+        if isinstance(spec, compat_str):
+            spec = spec.encode('ascii')
+        return struct.unpack(spec, *args)
+else:
+    struct_pack = struct.pack
+    struct_unpack = struct.unpack
+
+
 __all__ = [
     'compat_HTMLParser',
     'compat_HTTPError',
@@ -634,6 +655,8 @@ __all__ = [
     'compat_xml_parse_error',
     'compat_xpath',
     'shlex_quote',
+    'struct_pack',
+    'struct_unpack',
     'subprocess_check_output',
     'workaround_optparse_bug9161',
 ]
diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index 664d87543..b282fe3d6 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -12,13 +12,13 @@ from ..compat import (
     compat_urlparse,
     compat_urllib_error,
     compat_urllib_parse_urlparse,
+    struct_pack,
+    struct_unpack,
 )
 from ..utils import (
     encodeFilename,
     fix_xml_ampersands,
     sanitize_open,
-    struct_pack,
-    struct_unpack,
     xpath_text,
 )
 
diff --git a/youtube_dl/extractor/rtve.py b/youtube_dl/extractor/rtve.py
index 79af47715..f59040877 100644
--- a/youtube_dl/extractor/rtve.py
+++ b/youtube_dl/extractor/rtve.py
@@ -6,6 +6,9 @@ import re
 import time
 
 from .common import InfoExtractor
+from ..compat import (
+    struct_unpack,
+)
 from ..utils import (
     ExtractorError,
     float_or_none,
@@ -13,7 +16,6 @@ from ..utils import (
     remove_start,
     sanitized_Request,
     std_headers,
-    struct_unpack,
 )
 
 
diff --git a/youtube_dl/swfinterp.py b/youtube_dl/swfinterp.py
index 06c1d6cc1..86b28716c 100644
--- a/youtube_dl/swfinterp.py
+++ b/youtube_dl/swfinterp.py
@@ -4,10 +4,12 @@ import collections
 import io
 import zlib
 
-from .compat import compat_str
+from .compat import (
+    compat_str,
+    struct_unpack,
+)
 from .utils import (
     ExtractorError,
-    struct_unpack,
 )
 
 
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 6e4573784..fa16a42ad 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -26,7 +26,6 @@ import platform
 import re
 import socket
 import ssl
-import struct
 import subprocess
 import sys
 import tempfile
@@ -53,6 +52,7 @@ from .compat import (
     compat_urlparse,
     compat_xpath,
     shlex_quote,
+    struct_pack,
 )
 
 
@@ -1761,24 +1761,6 @@ def escape_url(url):
         fragment=escape_rfc3986(url_parsed.fragment)
     ).geturl()
 
-try:
-    struct.pack('!I', 0)
-except TypeError:
-    # In Python 2.6 and 2.7.x < 2.7.7, struct requires a bytes argument
-    # See https://bugs.python.org/issue19099
-    def struct_pack(spec, *args):
-        if isinstance(spec, compat_str):
-            spec = spec.encode('ascii')
-        return struct.pack(spec, *args)
-
-    def struct_unpack(spec, *args):
-        if isinstance(spec, compat_str):
-            spec = spec.encode('ascii')
-        return struct.unpack(spec, *args)
-else:
-    struct_pack = struct.pack
-    struct_unpack = struct.unpack
-
 
 def read_batch_urls(batch_fd):
     def fixup(url):

From 71aff18809a70b7fa32d8fd07f4fb2f64641aea5 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 23 Apr 2016 21:30:06 +0800
Subject: [PATCH 4895/6219] [socks] Support SOCKS proxies

---
 youtube_dl/socks.py | 196 ++++++++++++--------------------------------
 youtube_dl/utils.py |  63 +++++++++++++-
 2 files changed, 115 insertions(+), 144 deletions(-)

diff --git a/youtube_dl/socks.py b/youtube_dl/socks.py
index b0c36a189..95795b5a9 100644
--- a/youtube_dl/socks.py
+++ b/youtube_dl/socks.py
@@ -1,77 +1,30 @@
-# This is free and unencumbered software released into the public domain.
-# 
-# Anyone is free to copy, modify, publish, use, compile, sell, or
-# distribute this software, either in source code form or as a compiled
-# binary, for any purpose, commercial or non-commercial, and by any
-# means.
-# 
-# In jurisdictions that recognize copyright laws, the author or authors
-# of this software dedicate any and all copyright interest in the
-# software to the public domain. We make this dedication for the benefit
-# of the public at large and to the detriment of our heirs and
-# successors. We intend this dedication to be an overt act of
-# relinquishment in perpetuity of all present and future rights to this
-# software under copyright law.
-# 
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
-# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
-# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
-# OTHER DEALINGS IN THE SOFTWARE.
-# 
-# For more information, please refer to <http://unlicense.org/>
-#
-# Example:
-# import socks
-# import ftplib
-# import socket
-#
-# socks.patch_socket()
-#
-# f = ftplib.FTP('ftp.kernel.org')
-# f.login()
-# print f.retrlines('LIST')
-# f.quit()
-# 
-# s = socket.create_connection(('www.google.com', 80))
-# s.sendall('HEAD / HTTP/1.0\r\n\r\n')
-# print s.recv(1024)
-# s.close()
+# Public Domain SOCKS proxy protocol implementation
+# Adapted from https://gist.github.com/bluec0re/cafd3764412967417fd3
+
 from __future__ import unicode_literals
-import os
-import struct
+
+import collections
 import socket
-import time
+
+from .compat import (
+    struct_pack,
+    struct_unpack,
+)
 
 __author__ = 'Timo Schmid <coding@timoschmid.de>'
 
-_orig_socket = socket.socket
 
-try:
-    from collections import namedtuple
-except ImportError:
-    from Collections import namedtuple
-
-try:
-    from urllib.parse import urlparse
-except:
-    from urlparse import urlparse
-
-try:
-    from enum import Enum
-except ImportError:
-    Enum = object
+class ProxyError(IOError):
+    pass
 
 
-class ProxyError(IOError): pass
 class Socks4Error(ProxyError):
     CODES = {
         0x5B: 'request rejected or failed',
         0x5C: 'request rejected becasue SOCKS server cannot connect to identd on the client',
         0x5D: 'request rejected because the client program and identd report different user-ids'
     }
+
     def __init__(self, code=None, msg=None):
         if code is not None and msg is None:
             msg = self.CODES.get(code)
@@ -79,6 +32,7 @@ class Socks4Error(ProxyError):
                 msg = 'unknown error'
         super(Socks4Error, self).__init__(code, msg)
 
+
 class Socks5Error(Socks4Error):
     CODES = {
         0x01: 'general SOCKS server failure',
@@ -93,68 +47,19 @@ class Socks5Error(Socks4Error):
         0xFF: 'all offered authentication methods were rejected'
     }
 
-class ProxyType(Enum):
-    SOCKS4  = 0
+
+class ProxyType(object):
+    SOCKS4 = 0
     SOCKS4A = 1
-    SOCKS5  = 2
+    SOCKS5 = 2
 
-Proxy = namedtuple('Proxy', ('type', 'host', 'port', 'username', 'password', 'remote_dns'))
-
-_default_proxy = None
-
-def setdefaultproxy(proxytype=None, addr=None, port=None, rdns=True, username=None, password=None, allow_env_override=True):
-    global _default_proxy
-    if allow_env_override:
-        all_proxy = os.environ.get('ALL_PROXY', os.environ.get('all_proxy'))
-        if all_proxy:
-            all_proxy = urlparse(all_proxy)
-            if all_proxy.scheme.startswith('socks'):
-                if all_proxy.scheme == 'socks' or all_proxy.scheme == 'socks4':
-                    proxytype = ProxyType.SOCKS4
-                elif all_proxy.scheme == 'socks4a':
-                    proxytype = ProxyType.SOCKS4A
-                elif all_proxy.scheme == 'socks5':
-                    proxytype = ProxyType.SOCKS5
-                addr = all_proxy.hostname
-                port = all_proxy.port
-                username = all_proxy.username
-                password = all_proxy.password
-
-    if proxytype is not None:
-        _default_proxy = Proxy(proxytype, addr, port, username, password, rdns)
-
-
-def wrap_socket(sock):
-    return socksocket(_sock=sock._sock)
-
-def wrap_module(module):
-    if hasattr(module, 'socket'):
-        sock = module.socket
-        if isinstance(sock, socket.socket):
-            module.socket = sockssocket
-        elif hasattr(socket, 'socket'):
-            socket.socket = sockssocket
-
-def patch_socket():
-    import sys
-    if 'socket' not in sys.modules:
-        import socket
-    sys.modules['socket'].socket = sockssocket
+Proxy = collections.namedtuple('Proxy', ('type', 'host', 'port', 'username', 'password', 'remote_dns'))
 
 
 class sockssocket(socket.socket):
-    def __init__(self, *args, **kwargs):
-        self.__proxy = None
-        if 'proxy' in kwargs:
-            self.__proxy = kwargs['proxy']
-            del kwargs['proxy']
-        super(sockssocket, self).__init__(*args, **kwargs)
-
     @property
     def _proxy(self):
-        if self.__proxy:
-            return self.__proxy
-        return _default_proxy
+        return self.__proxy
 
     @property
     def _proxy_port(self):
@@ -175,7 +80,7 @@ class sockssocket(socket.socket):
         while len(data) < cnt:
             cur = self.recv(cnt - len(data))
             if not cur:
-                raise IOError("{0} bytes missing".format(cnt-len(data)))
+                raise IOError('{0} bytes missing'.format(cnt - len(data)))
             data += cur
         return data
 
@@ -186,39 +91,42 @@ class sockssocket(socket.socket):
             ipaddr = socket.inet_aton(destaddr)
         except socket.error:
             if is_4a and self._proxy.remote_dns:
-                ipaddr = struct.pack('!BBBB', 0, 0, 0, 0xFF)
+                ipaddr = struct_pack('!BBBB', 0, 0, 0, 0xFF)
             else:
                 ipaddr = socket.inet_aton(socket.gethostbyname(destaddr))
 
-        packet = struct.pack('!BBH', 0x4, 0x1, port) + ipaddr
+        packet = struct_pack('!BBH', 0x4, 0x1, port) + ipaddr
         if self._proxy.username:
             username = self._proxy.username
             if hasattr(username, 'encode'):
                 username = username.encode()
-            packet += struct.pack('!{0}s'.format(len(username)+1), username)
+            packet += struct_pack('!{0}s'.format(len(username) + 1), username)
         else:
             packet += b'\x00'
 
         if is_4a and self._proxy.remote_dns:
             if hasattr(destaddr, 'encode'):
                 destaddr = destaddr.encode()
-            packet += struct.pack('!{0}s'.format(len(destaddr)+1), destaddr)
+            packet += struct_pack('!{0}s'.format(len(destaddr) + 1), destaddr)
 
         self.sendall(packet)
 
         packet = self.recvall(8)
-        nbyte, resp_code, dstport, dsthost = struct.unpack('!BBHI', packet)
+        nbyte, resp_code, dstport, dsthost = struct_unpack('!BBHI', packet)
 
         # check valid response
         if nbyte != 0x00:
             self.close()
-            raise ProxyError(0, "Invalid response from server. Expected {0:02x} got {1:02x}".format(0, nbyte))
+            raise ProxyError(
+                0, 'Invalid response from server. Expected {0:02x} got {1:02x}'.format(0, nbyte))
 
         # access granted
         if resp_code != 0x5a:
             self.close()
             raise Socks4Error(resp_code)
 
+        return (dsthost, dstport)
+
     def _setup_socks5(self, address):
         destaddr, port = address
 
@@ -234,19 +142,20 @@ class sockssocket(socket.socket):
         if self._proxy.username and self._proxy.password:
             # two auth methods available
             auth_methods = 2
-        packet = struct.pack('!BBB', 0x5, auth_methods, 0x00) # no auth
+        packet = struct_pack('!BBB', 0x5, auth_methods, 0x00)  # no auth
         if self._proxy.username and self._proxy.password:
-            packet += struct.pack('!B', 0x02) # user/pass auth
+            packet += struct_pack('!B', 0x02)  # user/pass auth
 
         self.sendall(packet)
 
         packet = self.recvall(2)
-        version, method = struct.unpack('!BB', packet)
+        version, method = struct_unpack('!BB', packet)
 
         # check valid response
         if version != 0x05:
             self.close()
-            raise ProxyError(0, "Invalid response from server. Expected {0:02x} got {1:02x}".format(5, version))
+            raise ProxyError(
+                0, 'Invalid response from server. Expected {0:02x} got {1:02x}'.format(5, version))
 
         # no auth methods
         if method == 0xFF:
@@ -261,41 +170,42 @@ class sockssocket(socket.socket):
             password = self._proxy.password
             if hasattr(password, 'encode'):
                 password = password.encode()
-            packet = struct.pack('!BB', 1, len(username)) + username
-            packet += struct.pack('!B', len(password)) + password
+            packet = struct_pack('!BB', 1, len(username)) + username
+            packet += struct_pack('!B', len(password)) + password
             self.sendall(packet)
 
             packet = self.recvall(2)
-            version, status = struct.unpack('!BB', packet)
+            version, status = struct_unpack('!BB', packet)
 
             if version != 0x01:
                 self.close()
-                raise ProxyError(0, "Invalid response from server. Expected {0:02x} got {1:02x}".format(1, version))
+                raise ProxyError(
+                    0, 'Invalid response from server. Expected {0:02x} got {1:02x}'.format(1, version))
 
             if status != 0x00:
                 self.close()
                 raise Socks5Error(1)
-        elif method == 0x00: # no auth
+        elif method == 0x00:  # no auth
             pass
 
-
-        packet = struct.pack('!BBB', 5, 1, 0)
+        packet = struct_pack('!BBB', 5, 1, 0)
         if ipaddr is None:
             if hasattr(destaddr, 'encode'):
                 destaddr = destaddr.encode()
-            packet += struct.pack('!BB', 3, len(destaddr)) + destaddr
+            packet += struct_pack('!BB', 3, len(destaddr)) + destaddr
         else:
-            packet += struct.pack('!B', 1) + ipaddr
-        packet += struct.pack('!H', port)
+            packet += struct_pack('!B', 1) + ipaddr
+        packet += struct_pack('!H', port)
 
         self.sendall(packet)
 
         packet = self.recvall(4)
-        version, status, _, atype = struct.unpack('!BBBB', packet)
+        version, status, _, atype = struct_unpack('!BBBB', packet)
 
         if version != 0x05:
             self.close()
-            raise ProxyError(0, "Invalid response from server. Expected {0:02x} got {1:02x}".format(5, version))
+            raise ProxyError(
+                0, 'Invalid response from server. Expected {0:02x} got {1:02x}'.format(5, version))
 
         if status != 0x00:
             self.close()
@@ -304,11 +214,13 @@ class sockssocket(socket.socket):
         if atype == 0x01:
             destaddr = self.recvall(4)
         elif atype == 0x03:
-            alen = struct.unpack('!B', self.recv(1))[0]
+            alen = struct_unpack('!B', self.recv(1))[0]
             destaddr = self.recvall(alen)
         elif atype == 0x04:
             destaddr = self.recvall(16)
-        destport = struct.unpack('!H', self.recvall(2))[0]
+        destport = struct_unpack('!H', self.recvall(2))[0]
+
+        return (destaddr, destport)
 
     def _make_proxy(self, connect_func, address):
         if self._proxy.type == ProxyType.SOCKS4:
@@ -330,7 +242,7 @@ class sockssocket(socket.socket):
             return connect_func(self, address)
 
     def connect(self, address):
-        self._make_proxy(_orig_socket.connect, address)
+        self._make_proxy(socket.socket.connect, address)
 
     def connect_ex(self, address):
-        return self._make_proxy(_orig_socket.connect_ex, address)
+        return self._make_proxy(socket.socket.connect_ex, address)
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index fa16a42ad..b2e4a2dfb 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -55,6 +55,11 @@ from .compat import (
     struct_pack,
 )
 
+from .socks import (
+    ProxyType,
+    sockssocket,
+)
+
 
 # This is not clearly defined otherwise
 compiled_regex_type = type(re.compile(''))
@@ -752,8 +757,15 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
         self._params = params
 
     def http_open(self, req):
+        conn_class = compat_http_client.HTTPConnection
+
+        socks_proxy = req.headers.get('Ytdl-socks-proxy')
+        if socks_proxy:
+            conn_class = make_socks_conn_class(conn_class, socks_proxy)
+            del req.headers['Ytdl-socks-proxy']
+
         return self.do_open(functools.partial(
-            _create_http_connection, self, compat_http_client.HTTPConnection, False),
+            _create_http_connection, self, conn_class, False),
             req)
 
     @staticmethod
@@ -849,6 +861,41 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
     https_response = http_response
 
 
+def make_socks_conn_class(base_class, socks_proxy):
+    assert issubclass(base_class, (
+        compat_http_client.HTTPConnection, compat_http_client.HTTPSConnection))
+
+    url_components = compat_urlparse.urlparse(socks_proxy)
+    if url_components.scheme.lower() == 'socks5':
+        socks_type = ProxyType.SOCKS5
+    elif url_components.scheme.lower() in ('socks', 'socks4'):
+        socks_type = ProxyType.SOCKS4
+
+    proxy_args = (
+        socks_type,
+        url_components.hostname, url_components.port or 1080,
+        True,  # Remote DNS
+        url_components.username, url_components.password
+    )
+
+    class SocksConnection(base_class):
+        def connect(self):
+            self.sock = sockssocket()
+            self.sock.setproxy(*proxy_args)
+            if type(self.timeout) in (int, float):
+                self.sock.settimeout(self.timeout)
+            self.sock.connect((self.host, self.port))
+
+            if isinstance(self, compat_http_client.HTTPSConnection):
+                if hasattr(self, '_context'):  # Python > 2.6
+                    self.sock = self._context.wrap_socket(
+                        self.sock, server_hostname=self.host)
+                else:
+                    self.sock = ssl.wrap_socket(self.sock)
+
+    return SocksConnection
+
+
 class YoutubeDLHTTPSHandler(compat_urllib_request.HTTPSHandler):
     def __init__(self, params, https_conn_class=None, *args, **kwargs):
         compat_urllib_request.HTTPSHandler.__init__(self, *args, **kwargs)
@@ -857,12 +904,20 @@ class YoutubeDLHTTPSHandler(compat_urllib_request.HTTPSHandler):
 
     def https_open(self, req):
         kwargs = {}
+        conn_class = self._https_conn_class
+
         if hasattr(self, '_context'):  # python > 2.6
             kwargs['context'] = self._context
         if hasattr(self, '_check_hostname'):  # python 3.x
             kwargs['check_hostname'] = self._check_hostname
+
+        socks_proxy = req.headers.get('Ytdl-socks-proxy')
+        if socks_proxy:
+            conn_class = make_socks_conn_class(conn_class, socks_proxy)
+            del req.headers['Ytdl-socks-proxy']
+
         return self.do_open(functools.partial(
-            _create_http_connection, self, self._https_conn_class, True),
+            _create_http_connection, self, conn_class, True),
             req, **kwargs)
 
 
@@ -2683,6 +2738,10 @@ class PerRequestProxyHandler(compat_urllib_request.ProxyHandler):
 
         if proxy == '__noproxy__':
             return None  # No Proxy
+        if compat_urlparse.urlparse(proxy).scheme.lower() in ('socks', 'socks4', 'socks5'):
+            req.add_header('Ytdl-socks-proxy', proxy)
+            # youtube-dl's http/https handlers do wrapping the socket with socks
+            return None
         return compat_urllib_request.ProxyHandler.proxy_open(
             self, req, proxy, type)
 

From 72f3289ac48d8dbfe1ee3fd2d82a23f1bff045df Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 23 Apr 2016 21:30:44 +0800
Subject: [PATCH 4896/6219] [test/test_socks] Add tests for SOCKS proxies

---
 .gitignore         |  1 +
 Makefile           |  2 +-
 test/helper.py     |  5 ++++
 test/test_socks.py | 71 ++++++++++++++++++++++++++++++++++++++++++++++
 tox.ini            |  1 +
 5 files changed, 79 insertions(+), 1 deletion(-)
 create mode 100644 test/test_socks.py

diff --git a/.gitignore b/.gitignore
index 72c10425d..0e7128551 100644
--- a/.gitignore
+++ b/.gitignore
@@ -31,6 +31,7 @@ updates_key.pem
 *.part
 *.swp
 test/testdata
+test/local_parameters.json
 .tox
 youtube-dl.zsh
 .idea
diff --git a/Makefile b/Makefile
index c9ce216d1..5d7cd5a7e 100644
--- a/Makefile
+++ b/Makefile
@@ -37,7 +37,7 @@ test:
 ot: offlinetest
 
 offlinetest: codetest
-	$(PYTHON) -m nose --verbose test --exclude test_download.py --exclude test_age_restriction.py --exclude test_subtitles.py --exclude test_write_annotations.py --exclude test_youtube_lists.py --exclude test_iqiyi_sdk_interpreter.py
+	$(PYTHON) -m nose --verbose test --exclude test_download.py --exclude test_age_restriction.py --exclude test_subtitles.py --exclude test_write_annotations.py --exclude test_youtube_lists.py --exclude test_iqiyi_sdk_interpreter.py --exclude test_socks.py
 
 tar: youtube-dl.tar.gz
 
diff --git a/test/helper.py b/test/helper.py
index b8e22c5cb..dfee217a9 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -24,8 +24,13 @@ from youtube_dl.utils import (
 def get_params(override=None):
     PARAMETERS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)),
                                    "parameters.json")
+    LOCAL_PARAMETERS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)),
+                                         "local_parameters.json")
     with io.open(PARAMETERS_FILE, encoding='utf-8') as pf:
         parameters = json.load(pf)
+    if os.path.exists(LOCAL_PARAMETERS_FILE):
+        with io.open(LOCAL_PARAMETERS_FILE, encoding='utf-8') as pf:
+            parameters.update(json.load(pf))
     if override:
         parameters.update(override)
     return parameters
diff --git a/test/test_socks.py b/test/test_socks.py
new file mode 100644
index 000000000..92574c6fd
--- /dev/null
+++ b/test/test_socks.py
@@ -0,0 +1,71 @@
+#!/usr/bin/env python
+# coding: utf-8
+from __future__ import unicode_literals
+
+# Allow direct execution
+import os
+import sys
+import unittest
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from test.helper import (FakeYDL, get_params)
+from youtube_dl.compat import compat_urllib_request
+
+
+class TestSocks(unittest.TestCase):
+    @staticmethod
+    def _check_params(attrs):
+        params = get_params()
+        for attr in attrs:
+            if attr not in params:
+                print('Missing %s. Skipping.' % attr)
+                return
+        return params
+
+    def test_proxy_http(self):
+        params = self._check_params(['primary_proxy', 'primary_server_ip'])
+        if params is None:
+            return
+        ydl = FakeYDL({
+            'proxy': params['primary_proxy']
+        })
+        self.assertEqual(
+            ydl.urlopen('http://yt-dl.org/ip').read().decode('utf-8'),
+            params['primary_server_ip'])
+
+    def test_proxy_https(self):
+        params = self._check_params(['primary_proxy', 'primary_server_ip'])
+        if params is None:
+            return
+        ydl = FakeYDL({
+            'proxy': params['primary_proxy']
+        })
+        self.assertEqual(
+            ydl.urlopen('https://yt-dl.org/ip').read().decode('utf-8'),
+            params['primary_server_ip'])
+
+    def test_secondary_proxy_http(self):
+        params = self._check_params(['secondary_proxy', 'secondary_server_ip'])
+        if params is None:
+            return
+        ydl = FakeYDL()
+        req = compat_urllib_request.Request('http://yt-dl.org/ip')
+        req.add_header('Ytdl-request-proxy', params['secondary_proxy'])
+        self.assertEqual(
+            ydl.urlopen(req).read().decode('utf-8'),
+            params['secondary_server_ip'])
+
+    def test_secondary_proxy_https(self):
+        params = self._check_params(['secondary_proxy', 'secondary_server_ip'])
+        if params is None:
+            return
+        ydl = FakeYDL()
+        req = compat_urllib_request.Request('https://yt-dl.org/ip')
+        req.add_header('Ytdl-request-proxy', params['secondary_proxy'])
+        self.assertEqual(
+            ydl.urlopen(req).read().decode('utf-8'),
+            params['secondary_server_ip'])
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/tox.ini b/tox.ini
index 2d7134005..9c4e4a3d1 100644
--- a/tox.ini
+++ b/tox.ini
@@ -9,5 +9,6 @@ passenv = HOME
 defaultargs = test --exclude test_download.py --exclude test_age_restriction.py
     --exclude test_subtitles.py --exclude test_write_annotations.py
     --exclude test_youtube_lists.py --exclude test_iqiyi_sdk_interpreter.py
+    --exclude test_socks.py
 commands = nosetests --verbose {posargs:{[testenv]defaultargs}}  # --with-coverage --cover-package=youtube_dl --cover-html
                                                # test.test_download:TestDownload.test_NowVideo

From 9e9cd7248d387954d1009087ac300ee3ff6a9766 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 3 May 2016 15:11:05 +0800
Subject: [PATCH 4897/6219] [socks] Eliminate magic constants and improve

---
 youtube_dl/socks.py | 289 ++++++++++++++++++++++++--------------------
 1 file changed, 157 insertions(+), 132 deletions(-)

diff --git a/youtube_dl/socks.py b/youtube_dl/socks.py
index 95795b5a9..0e3dd7893 100644
--- a/youtube_dl/socks.py
+++ b/youtube_dl/socks.py
@@ -3,37 +3,87 @@
 
 from __future__ import unicode_literals
 
+# References:
+# SOCKS4 protocol http://www.openssh.com/txt/socks4.protocol
+# SOCKS4A protocol http://www.openssh.com/txt/socks4a.protocol
+# SOCKS5 protocol https://tools.ietf.org/html/rfc1928
+# SOCKS5 username/password authentication https://tools.ietf.org/html/rfc1929
+
 import collections
 import socket
 
 from .compat import (
+    compat_ord,
     struct_pack,
     struct_unpack,
 )
 
 __author__ = 'Timo Schmid <coding@timoschmid.de>'
 
+SOCKS4_VERSION = 4
+SOCKS4_REPLY_VERSION = 0x00
+# Excerpt from SOCKS4A protocol:
+# if the client cannot resolve the destination host's domain name to find its
+# IP address, it should set the first three bytes of DSTIP to NULL and the last
+# byte to a non-zero value.
+SOCKS4_DEFAULT_DSTIP = struct_pack('!BBBB', 0, 0, 0, 0xFF)
+
+SOCKS5_VERSION = 5
+SOCKS5_USER_AUTH_VERSION = 0x01
+SOCKS5_USER_AUTH_SUCCESS = 0x00
+
+
+class Socks4Command(object):
+    CMD_CONNECT = 0x01
+    CMD_BIND = 0x02
+
+
+class Socks5Command(Socks4Command):
+    CMD_UDP_ASSOCIATE = 0x03
+
+
+class Socks5Auth(object):
+    AUTH_NONE = 0x00
+    AUTH_GSSAPI = 0x01
+    AUTH_USER_PASS = 0x02
+    AUTH_NO_ACCEPTABLE = 0xFF  # For server response
+
+
+class Socks5AddressType(object):
+    ATYP_IPV4 = 0x01
+    ATYP_DOMAINNAME = 0x03
+    ATYP_IPV6 = 0x04
+
 
 class ProxyError(IOError):
-    pass
-
-
-class Socks4Error(ProxyError):
-    CODES = {
-        0x5B: 'request rejected or failed',
-        0x5C: 'request rejected becasue SOCKS server cannot connect to identd on the client',
-        0x5D: 'request rejected because the client program and identd report different user-ids'
-    }
+    ERR_SUCCESS = 0x00
 
     def __init__(self, code=None, msg=None):
         if code is not None and msg is None:
-            msg = self.CODES.get(code)
-            if msg is None:
-                msg = 'unknown error'
-        super(Socks4Error, self).__init__(code, msg)
+            msg = self.CODES.get(code) and 'unknown error'
+        super(ProxyError, self).__init__(code, msg)
 
 
-class Socks5Error(Socks4Error):
+class InvalidVersionError(ProxyError):
+    def __init__(self, expected_version, got_version):
+        msg = ('Invalid response version from server. Expected {0:02x} got '
+               '{1:02x}'.format(expected_version, got_version))
+        super(InvalidVersionError, self).__init__(0, msg)
+
+
+class Socks4Error(ProxyError):
+    ERR_SUCCESS = 90
+
+    CODES = {
+        91: 'request rejected or failed',
+        92: 'request rejected becasue SOCKS server cannot connect to identd on the client',
+        93: 'request rejected because the client program and identd report different user-ids'
+    }
+
+
+class Socks5Error(ProxyError):
+    ERR_GENERAL_FAILURE = 0x01
+
     CODES = {
         0x01: 'general SOCKS server failure',
         0x02: 'connection not allowed by ruleset',
@@ -53,27 +103,19 @@ class ProxyType(object):
     SOCKS4A = 1
     SOCKS5 = 2
 
-Proxy = collections.namedtuple('Proxy', ('type', 'host', 'port', 'username', 'password', 'remote_dns'))
+Proxy = collections.namedtuple('Proxy', (
+    'type', 'host', 'port', 'username', 'password', 'remote_dns'))
 
 
 class sockssocket(socket.socket):
-    @property
-    def _proxy(self):
-        return self.__proxy
+    def __init__(self, *args, **kwargs):
+        self._proxy = None
+        super(sockssocket, self).__init__(*args, **kwargs)
 
-    @property
-    def _proxy_port(self):
-        if self._proxy:
-            if self._proxy.port:
-                return self._proxy.port
-            return 1080
-        return None
+    def setproxy(self, proxytype, addr, port, rdns=True, username=None, password=None):
+        assert proxytype in (ProxyType.SOCKS4, ProxyType.SOCKS4A, ProxyType.SOCKS5)
 
-    def setproxy(self, proxytype=None, addr=None, port=None, rdns=True, username=None, password=None):
-        if proxytype is None:
-            self.__proxy = None
-        else:
-            self.__proxy = Proxy(proxytype, addr, port, username, password, rdns)
+        self._proxy = Proxy(proxytype, addr, port, username, password, rdns)
 
     def recvall(self, cnt):
         data = b''
@@ -84,163 +126,146 @@ class sockssocket(socket.socket):
             data += cur
         return data
 
+    def _recv_bytes(self, cnt):
+        data = self.recvall(cnt)
+        return struct_unpack('!{0}B'.format(cnt), data)
+
+    @staticmethod
+    def _len_and_data(data):
+        return struct_pack('!B', len(data)) + data
+
+    def _check_response_version(self, expected_version, got_version):
+        if got_version != expected_version:
+            self.close()
+            raise InvalidVersionError(expected_version, got_version)
+
+    def _resolve_address(self, destaddr, default, use_remote_dns):
+        try:
+            return socket.inet_aton(destaddr)
+        except socket.error:
+            if use_remote_dns and self._proxy.remote_dns:
+                return default
+            else:
+                return socket.inet_aton(socket.gethostbyname(destaddr))
+
     def _setup_socks4(self, address, is_4a=False):
         destaddr, port = address
 
-        try:
-            ipaddr = socket.inet_aton(destaddr)
-        except socket.error:
-            if is_4a and self._proxy.remote_dns:
-                ipaddr = struct_pack('!BBBB', 0, 0, 0, 0xFF)
-            else:
-                ipaddr = socket.inet_aton(socket.gethostbyname(destaddr))
+        ipaddr = self._resolve_address(destaddr, SOCKS4_DEFAULT_DSTIP, use_remote_dns=is_4a)
 
-        packet = struct_pack('!BBH', 0x4, 0x1, port) + ipaddr
-        if self._proxy.username:
-            username = self._proxy.username
-            if hasattr(username, 'encode'):
-                username = username.encode()
-            packet += struct_pack('!{0}s'.format(len(username) + 1), username)
-        else:
-            packet += b'\x00'
+        packet = struct_pack('!BBH', SOCKS4_VERSION, Socks4Command.CMD_CONNECT, port) + ipaddr
+
+        username = (self._proxy.username or '').encode('utf-8')
+        packet += username + b'\x00'
 
         if is_4a and self._proxy.remote_dns:
-            if hasattr(destaddr, 'encode'):
-                destaddr = destaddr.encode()
-            packet += struct_pack('!{0}s'.format(len(destaddr) + 1), destaddr)
+            packet += destaddr.encode('utf-8') + b'\x00'
 
         self.sendall(packet)
 
-        packet = self.recvall(8)
-        nbyte, resp_code, dstport, dsthost = struct_unpack('!BBHI', packet)
+        version, resp_code, dstport, dsthost = struct_unpack('!BBHI', self.recvall(8))
 
-        # check valid response
-        if nbyte != 0x00:
-            self.close()
-            raise ProxyError(
-                0, 'Invalid response from server. Expected {0:02x} got {1:02x}'.format(0, nbyte))
+        self._check_response_version(SOCKS4_REPLY_VERSION, version)
 
-        # access granted
-        if resp_code != 0x5a:
+        if resp_code != Socks4Error.ERR_SUCCESS:
             self.close()
             raise Socks4Error(resp_code)
 
         return (dsthost, dstport)
 
-    def _setup_socks5(self, address):
-        destaddr, port = address
+    def _setup_socks4a(self, address):
+        self._setup_socks4(address, is_4a=True)
 
-        try:
-            ipaddr = socket.inet_aton(destaddr)
-        except socket.error:
-            if self._proxy.remote_dns:
-                ipaddr = None
-            else:
-                ipaddr = socket.inet_aton(socket.gethostbyname(destaddr))
+    def _socks5_auth(self):
+        packet = struct_pack('!B', SOCKS5_VERSION)
 
-        auth_methods = 1
+        auth_methods = [Socks5Auth.AUTH_NONE]
         if self._proxy.username and self._proxy.password:
-            # two auth methods available
-            auth_methods = 2
-        packet = struct_pack('!BBB', 0x5, auth_methods, 0x00)  # no auth
-        if self._proxy.username and self._proxy.password:
-            packet += struct_pack('!B', 0x02)  # user/pass auth
+            auth_methods.append(Socks5Auth.AUTH_USER_PASS)
+
+        packet += struct_pack('!B', len(auth_methods))
+        packet += struct_pack('!{0}B'.format(len(auth_methods)), *auth_methods)
 
         self.sendall(packet)
 
-        packet = self.recvall(2)
-        version, method = struct_unpack('!BB', packet)
+        version, method = self._recv_bytes(2)
 
-        # check valid response
-        if version != 0x05:
-            self.close()
-            raise ProxyError(
-                0, 'Invalid response from server. Expected {0:02x} got {1:02x}'.format(5, version))
+        self._check_response_version(SOCKS5_VERSION, version)
 
-        # no auth methods
-        if method == 0xFF:
+        if method == Socks5Auth.AUTH_NO_ACCEPTABLE:
             self.close()
             raise Socks5Error(method)
 
-        # user/pass auth
-        if method == 0x01:
-            username = self._proxy.username
-            if hasattr(username, 'encode'):
-                username = username.encode()
-            password = self._proxy.password
-            if hasattr(password, 'encode'):
-                password = password.encode()
-            packet = struct_pack('!BB', 1, len(username)) + username
-            packet += struct_pack('!B', len(password)) + password
+        if method == Socks5Auth.AUTH_USER_PASS:
+            username = self._proxy.username.encode('utf-8')
+            password = self._proxy.password.encode('utf-8')
+            packet = struct_pack('!B', SOCKS5_USER_AUTH_VERSION)
+            packet += self._len_and_data(username) + self._len_and_data(password)
             self.sendall(packet)
 
-            packet = self.recvall(2)
-            version, status = struct_unpack('!BB', packet)
+            version, status = self._recv_bytes(2)
 
-            if version != 0x01:
-                self.close()
-                raise ProxyError(
-                    0, 'Invalid response from server. Expected {0:02x} got {1:02x}'.format(1, version))
+            self._check_response_version(SOCKS5_USER_AUTH_VERSION, version)
 
-            if status != 0x00:
+            if status != SOCKS5_USER_AUTH_SUCCESS:
                 self.close()
-                raise Socks5Error(1)
-        elif method == 0x00:  # no auth
+                raise Socks5Error(Socks5Error.ERR_GENERAL_FAILURE)
+        elif method == Socks5Auth.AUTH_NONE:
             pass
 
-        packet = struct_pack('!BBB', 5, 1, 0)
+    def _setup_socks5(self, address):
+        destaddr, port = address
+
+        ipaddr = self._resolve_address(destaddr, None, use_remote_dns=True)
+
+        self._socks5_auth()
+
+        reserved = 0
+        packet = struct_pack('!BBB', SOCKS5_VERSION, Socks5Command.CMD_CONNECT, reserved)
         if ipaddr is None:
-            if hasattr(destaddr, 'encode'):
-                destaddr = destaddr.encode()
-            packet += struct_pack('!BB', 3, len(destaddr)) + destaddr
+            destaddr = destaddr.encode('utf-8')
+            packet += struct_pack('!B', Socks5AddressType.ATYP_DOMAINNAME)
+            packet += self._len_and_data(destaddr)
         else:
-            packet += struct_pack('!B', 1) + ipaddr
+            packet += struct_pack('!B', Socks5AddressType.ATYP_IPV4) + ipaddr
         packet += struct_pack('!H', port)
 
         self.sendall(packet)
 
-        packet = self.recvall(4)
-        version, status, _, atype = struct_unpack('!BBBB', packet)
+        version, status, reserved, atype = self._recv_bytes(4)
 
-        if version != 0x05:
-            self.close()
-            raise ProxyError(
-                0, 'Invalid response from server. Expected {0:02x} got {1:02x}'.format(5, version))
+        self._check_response_version(SOCKS5_VERSION, version)
 
-        if status != 0x00:
+        if status != Socks5Error.ERR_SUCCESS:
             self.close()
             raise Socks5Error(status)
 
-        if atype == 0x01:
+        if atype == Socks5AddressType.ATYP_IPV4:
             destaddr = self.recvall(4)
-        elif atype == 0x03:
-            alen = struct_unpack('!B', self.recv(1))[0]
+        elif atype == Socks5AddressType.ATYP_DOMAINNAME:
+            alen = compat_ord(self.recv(1))
             destaddr = self.recvall(alen)
-        elif atype == 0x04:
+        elif atype == Socks5AddressType.ATYP_IPV6:
             destaddr = self.recvall(16)
         destport = struct_unpack('!H', self.recvall(2))[0]
 
         return (destaddr, destport)
 
     def _make_proxy(self, connect_func, address):
-        if self._proxy.type == ProxyType.SOCKS4:
-            result = connect_func(self, (self._proxy.host, self._proxy_port))
-            if result != 0 and result is not None:
-                return result
-            self._setup_socks4(address)
-        elif self._proxy.type == ProxyType.SOCKS4A:
-            result = connect_func(self, (self._proxy.host, self._proxy_port))
-            if result != 0 and result is not None:
-                return result
-            self._setup_socks4(address, is_4a=True)
-        elif self._proxy.type == ProxyType.SOCKS5:
-            result = connect_func(self, (self._proxy.host, self._proxy_port))
-            if result != 0 and result is not None:
-                return result
-            self._setup_socks5(address)
-        else:
+        if not self._proxy:
             return connect_func(self, address)
 
+        result = connect_func(self, (self._proxy.host, self._proxy.port))
+        if result != 0 and result is not None:
+            return result
+        setup_funcs = {
+            ProxyType.SOCKS4: self._setup_socks4,
+            ProxyType.SOCKS4A: self._setup_socks4a,
+            ProxyType.SOCKS5: self._setup_socks5,
+        }
+        setup_funcs[self._proxy.type](address)
+        return result
+
     def connect(self, address):
         self._make_proxy(socket.socket.connect, address)
 

From 51fb4995a5242c0edca09167cf8c4b050cf5a186 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 3 May 2016 15:15:32 +0800
Subject: [PATCH 4898/6219] [utils] Register SOCKS protocols in urllib and
 support SOCKS4A

---
 youtube_dl/YoutubeDL.py |  3 +++
 youtube_dl/utils.py     | 11 ++++++++++-
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index a96482e68..34eeb77c5 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -64,6 +64,7 @@ from .utils import (
     PostProcessingError,
     preferredencoding,
     prepend_extension,
+    register_socks_protocols,
     render_table,
     replace_extension,
     SameFileError,
@@ -361,6 +362,8 @@ class YoutubeDL(object):
         for ph in self.params.get('progress_hooks', []):
             self.add_progress_hook(ph)
 
+        register_socks_protocols()
+
     def warn_if_short_id(self, argv):
         # short YouTube ID starting with dash?
         idxs = [
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index b2e4a2dfb..c9702fd93 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -61,6 +61,13 @@ from .socks import (
 )
 
 
+def register_socks_protocols():
+    # "Register" SOCKS protocols
+    for scheme in ('socks', 'socks4', 'socks4a', 'socks5'):
+        if scheme not in compat_urlparse.uses_netloc:
+            compat_urlparse.uses_netloc.append(scheme)
+
+
 # This is not clearly defined otherwise
 compiled_regex_type = type(re.compile(''))
 
@@ -870,6 +877,8 @@ def make_socks_conn_class(base_class, socks_proxy):
         socks_type = ProxyType.SOCKS5
     elif url_components.scheme.lower() in ('socks', 'socks4'):
         socks_type = ProxyType.SOCKS4
+    elif url_components.scheme.lower() == 'socks4a':
+        socks_type = ProxyType.SOCKS4A
 
     proxy_args = (
         socks_type,
@@ -2738,7 +2747,7 @@ class PerRequestProxyHandler(compat_urllib_request.ProxyHandler):
 
         if proxy == '__noproxy__':
             return None  # No Proxy
-        if compat_urlparse.urlparse(proxy).scheme.lower() in ('socks', 'socks4', 'socks5'):
+        if compat_urlparse.urlparse(proxy).scheme.lower() in ('socks', 'socks4', 'socks4a', 'socks5'):
             req.add_header('Ytdl-socks-proxy', proxy)
             # youtube-dl's http/https handlers do wrapping the socket with socks
             return None

From d5ae6bb50124f8320f2b492380480038c487a6d2 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 3 May 2016 15:37:30 +0800
Subject: [PATCH 4899/6219] [utils] Add rationale for register_socks_protocols

---
 youtube_dl/utils.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index c9702fd93..dc73f3407 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -63,6 +63,8 @@ from .socks import (
 
 def register_socks_protocols():
     # "Register" SOCKS protocols
+    # In Python < 2.6.5, urlsplit() suffers from bug https://bugs.python.org/issue7904
+    # URLs with protocols not in urlparse.uses_netloc are not handled correctly
     for scheme in ('socks', 'socks4', 'socks4a', 'socks5'):
         if scheme not in compat_urlparse.uses_netloc:
             compat_urlparse.uses_netloc.append(scheme)

From edaa23f822a1e4a62771422fb598c7bd8ae0a152 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 3 May 2016 16:50:16 +0800
Subject: [PATCH 4900/6219] [compat] Rename struct_(un)pack to
 compat_struct_(un)pack

---
 test/test_compat.py          |  4 ++--
 youtube_dl/compat.py         | 12 ++++++------
 youtube_dl/downloader/f4m.py | 14 +++++++-------
 youtube_dl/extractor/rtve.py |  4 ++--
 youtube_dl/socks.py          | 32 ++++++++++++++++----------------
 youtube_dl/swfinterp.py      | 14 +++++++-------
 youtube_dl/utils.py          |  4 ++--
 7 files changed, 42 insertions(+), 42 deletions(-)

diff --git a/test/test_compat.py b/test/test_compat.py
index dd62a5d6b..539b30540 100644
--- a/test/test_compat.py
+++ b/test/test_compat.py
@@ -17,10 +17,10 @@ from youtube_dl.compat import (
     compat_expanduser,
     compat_shlex_split,
     compat_str,
+    compat_struct_unpack,
     compat_urllib_parse_unquote,
     compat_urllib_parse_unquote_plus,
     compat_urllib_parse_urlencode,
-    struct_unpack,
 )
 
 
@@ -104,7 +104,7 @@ class TestCompat(unittest.TestCase):
         self.assertTrue(isinstance(doc.find('foo/bar').text, compat_str))
 
     def test_struct_unpack(self):
-        self.assertEqual(struct_unpack('!B', b'\x00'), (0,))
+        self.assertEqual(compat_struct_unpack('!B', b'\x00'), (0,))
 
 
 if __name__ == '__main__':
diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index f697bee7e..e48c761a6 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -599,18 +599,18 @@ try:
 except TypeError:
     # In Python 2.6 and 2.7.x < 2.7.7, struct requires a bytes argument
     # See https://bugs.python.org/issue19099
-    def struct_pack(spec, *args):
+    def compat_struct_pack(spec, *args):
         if isinstance(spec, compat_str):
             spec = spec.encode('ascii')
         return struct.pack(spec, *args)
 
-    def struct_unpack(spec, *args):
+    def compat_struct_unpack(spec, *args):
         if isinstance(spec, compat_str):
             spec = spec.encode('ascii')
         return struct.unpack(spec, *args)
 else:
-    struct_pack = struct.pack
-    struct_unpack = struct.unpack
+    compat_struct_pack = struct.pack
+    compat_struct_unpack = struct.unpack
 
 
 __all__ = [
@@ -638,6 +638,8 @@ __all__ = [
     'compat_shlex_split',
     'compat_socket_create_connection',
     'compat_str',
+    'compat_struct_pack',
+    'compat_struct_unpack',
     'compat_subprocess_get_DEVNULL',
     'compat_tokenize_tokenize',
     'compat_urllib_error',
@@ -655,8 +657,6 @@ __all__ = [
     'compat_xml_parse_error',
     'compat_xpath',
     'shlex_quote',
-    'struct_pack',
-    'struct_unpack',
     'subprocess_check_output',
     'workaround_optparse_bug9161',
 ]
diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index b282fe3d6..3d9337afa 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -12,8 +12,8 @@ from ..compat import (
     compat_urlparse,
     compat_urllib_error,
     compat_urllib_parse_urlparse,
-    struct_pack,
-    struct_unpack,
+    compat_struct_pack,
+    compat_struct_unpack,
 )
 from ..utils import (
     encodeFilename,
@@ -31,13 +31,13 @@ class FlvReader(io.BytesIO):
 
     # Utility functions for reading numbers and strings
     def read_unsigned_long_long(self):
-        return struct_unpack('!Q', self.read(8))[0]
+        return compat_struct_unpack('!Q', self.read(8))[0]
 
     def read_unsigned_int(self):
-        return struct_unpack('!I', self.read(4))[0]
+        return compat_struct_unpack('!I', self.read(4))[0]
 
     def read_unsigned_char(self):
-        return struct_unpack('!B', self.read(1))[0]
+        return compat_struct_unpack('!B', self.read(1))[0]
 
     def read_string(self):
         res = b''
@@ -194,11 +194,11 @@ def build_fragments_list(boot_info):
 
 
 def write_unsigned_int(stream, val):
-    stream.write(struct_pack('!I', val))
+    stream.write(compat_struct_pack('!I', val))
 
 
 def write_unsigned_int_24(stream, val):
-    stream.write(struct_pack('!I', val)[1:])
+    stream.write(compat_struct_pack('!I', val)[1:])
 
 
 def write_flv_header(stream):
diff --git a/youtube_dl/extractor/rtve.py b/youtube_dl/extractor/rtve.py
index f59040877..edd0d108e 100644
--- a/youtube_dl/extractor/rtve.py
+++ b/youtube_dl/extractor/rtve.py
@@ -7,7 +7,7 @@ import time
 
 from .common import InfoExtractor
 from ..compat import (
-    struct_unpack,
+    compat_struct_unpack,
 )
 from ..utils import (
     ExtractorError,
@@ -23,7 +23,7 @@ def _decrypt_url(png):
     encrypted_data = base64.b64decode(png.encode('utf-8'))
     text_index = encrypted_data.find(b'tEXt')
     text_chunk = encrypted_data[text_index - 4:]
-    length = struct_unpack('!I', text_chunk[:4])[0]
+    length = compat_struct_unpack('!I', text_chunk[:4])[0]
     # Use bytearray to get integers when iterating in both python 2.x and 3.x
     data = bytearray(text_chunk[8:8 + length])
     data = [chr(b) for b in data if b != 0]
diff --git a/youtube_dl/socks.py b/youtube_dl/socks.py
index 0e3dd7893..a5b27fea7 100644
--- a/youtube_dl/socks.py
+++ b/youtube_dl/socks.py
@@ -14,8 +14,8 @@ import socket
 
 from .compat import (
     compat_ord,
-    struct_pack,
-    struct_unpack,
+    compat_struct_pack,
+    compat_struct_unpack,
 )
 
 __author__ = 'Timo Schmid <coding@timoschmid.de>'
@@ -26,7 +26,7 @@ SOCKS4_REPLY_VERSION = 0x00
 # if the client cannot resolve the destination host's domain name to find its
 # IP address, it should set the first three bytes of DSTIP to NULL and the last
 # byte to a non-zero value.
-SOCKS4_DEFAULT_DSTIP = struct_pack('!BBBB', 0, 0, 0, 0xFF)
+SOCKS4_DEFAULT_DSTIP = compat_struct_pack('!BBBB', 0, 0, 0, 0xFF)
 
 SOCKS5_VERSION = 5
 SOCKS5_USER_AUTH_VERSION = 0x01
@@ -128,11 +128,11 @@ class sockssocket(socket.socket):
 
     def _recv_bytes(self, cnt):
         data = self.recvall(cnt)
-        return struct_unpack('!{0}B'.format(cnt), data)
+        return compat_struct_unpack('!{0}B'.format(cnt), data)
 
     @staticmethod
     def _len_and_data(data):
-        return struct_pack('!B', len(data)) + data
+        return compat_struct_pack('!B', len(data)) + data
 
     def _check_response_version(self, expected_version, got_version):
         if got_version != expected_version:
@@ -153,7 +153,7 @@ class sockssocket(socket.socket):
 
         ipaddr = self._resolve_address(destaddr, SOCKS4_DEFAULT_DSTIP, use_remote_dns=is_4a)
 
-        packet = struct_pack('!BBH', SOCKS4_VERSION, Socks4Command.CMD_CONNECT, port) + ipaddr
+        packet = compat_struct_pack('!BBH', SOCKS4_VERSION, Socks4Command.CMD_CONNECT, port) + ipaddr
 
         username = (self._proxy.username or '').encode('utf-8')
         packet += username + b'\x00'
@@ -163,7 +163,7 @@ class sockssocket(socket.socket):
 
         self.sendall(packet)
 
-        version, resp_code, dstport, dsthost = struct_unpack('!BBHI', self.recvall(8))
+        version, resp_code, dstport, dsthost = compat_struct_unpack('!BBHI', self.recvall(8))
 
         self._check_response_version(SOCKS4_REPLY_VERSION, version)
 
@@ -177,14 +177,14 @@ class sockssocket(socket.socket):
         self._setup_socks4(address, is_4a=True)
 
     def _socks5_auth(self):
-        packet = struct_pack('!B', SOCKS5_VERSION)
+        packet = compat_struct_pack('!B', SOCKS5_VERSION)
 
         auth_methods = [Socks5Auth.AUTH_NONE]
         if self._proxy.username and self._proxy.password:
             auth_methods.append(Socks5Auth.AUTH_USER_PASS)
 
-        packet += struct_pack('!B', len(auth_methods))
-        packet += struct_pack('!{0}B'.format(len(auth_methods)), *auth_methods)
+        packet += compat_struct_pack('!B', len(auth_methods))
+        packet += compat_struct_pack('!{0}B'.format(len(auth_methods)), *auth_methods)
 
         self.sendall(packet)
 
@@ -199,7 +199,7 @@ class sockssocket(socket.socket):
         if method == Socks5Auth.AUTH_USER_PASS:
             username = self._proxy.username.encode('utf-8')
             password = self._proxy.password.encode('utf-8')
-            packet = struct_pack('!B', SOCKS5_USER_AUTH_VERSION)
+            packet = compat_struct_pack('!B', SOCKS5_USER_AUTH_VERSION)
             packet += self._len_and_data(username) + self._len_and_data(password)
             self.sendall(packet)
 
@@ -221,14 +221,14 @@ class sockssocket(socket.socket):
         self._socks5_auth()
 
         reserved = 0
-        packet = struct_pack('!BBB', SOCKS5_VERSION, Socks5Command.CMD_CONNECT, reserved)
+        packet = compat_struct_pack('!BBB', SOCKS5_VERSION, Socks5Command.CMD_CONNECT, reserved)
         if ipaddr is None:
             destaddr = destaddr.encode('utf-8')
-            packet += struct_pack('!B', Socks5AddressType.ATYP_DOMAINNAME)
+            packet += compat_struct_pack('!B', Socks5AddressType.ATYP_DOMAINNAME)
             packet += self._len_and_data(destaddr)
         else:
-            packet += struct_pack('!B', Socks5AddressType.ATYP_IPV4) + ipaddr
-        packet += struct_pack('!H', port)
+            packet += compat_struct_pack('!B', Socks5AddressType.ATYP_IPV4) + ipaddr
+        packet += compat_struct_pack('!H', port)
 
         self.sendall(packet)
 
@@ -247,7 +247,7 @@ class sockssocket(socket.socket):
             destaddr = self.recvall(alen)
         elif atype == Socks5AddressType.ATYP_IPV6:
             destaddr = self.recvall(16)
-        destport = struct_unpack('!H', self.recvall(2))[0]
+        destport = compat_struct_unpack('!H', self.recvall(2))[0]
 
         return (destaddr, destport)
 
diff --git a/youtube_dl/swfinterp.py b/youtube_dl/swfinterp.py
index 86b28716c..7cf490aa4 100644
--- a/youtube_dl/swfinterp.py
+++ b/youtube_dl/swfinterp.py
@@ -6,7 +6,7 @@ import zlib
 
 from .compat import (
     compat_str,
-    struct_unpack,
+    compat_struct_unpack,
 )
 from .utils import (
     ExtractorError,
@@ -25,17 +25,17 @@ def _extract_tags(file_contents):
             file_contents[:1])
 
     # Determine number of bits in framesize rectangle
-    framesize_nbits = struct_unpack('!B', content[:1])[0] >> 3
+    framesize_nbits = compat_struct_unpack('!B', content[:1])[0] >> 3
     framesize_len = (5 + 4 * framesize_nbits + 7) // 8
 
     pos = framesize_len + 2 + 2
     while pos < len(content):
-        header16 = struct_unpack('<H', content[pos:pos + 2])[0]
+        header16 = compat_struct_unpack('<H', content[pos:pos + 2])[0]
         pos += 2
         tag_code = header16 >> 6
         tag_len = header16 & 0x3f
         if tag_len == 0x3f:
-            tag_len = struct_unpack('<I', content[pos:pos + 4])[0]
+            tag_len = compat_struct_unpack('<I', content[pos:pos + 4])[0]
             pos += 4
         assert pos + tag_len <= len(content), \
             ('Tag %d ends at %d+%d - that\'s longer than the file (%d)'
@@ -103,7 +103,7 @@ def _read_int(reader):
     for _ in range(5):
         buf = reader.read(1)
         assert len(buf) == 1
-        b = struct_unpack('<B', buf)[0]
+        b = compat_struct_unpack('<B', buf)[0]
         res = res | ((b & 0x7f) << shift)
         if b & 0x80 == 0:
             break
@@ -129,7 +129,7 @@ def _s24(reader):
     bs = reader.read(3)
     assert len(bs) == 3
     last_byte = b'\xff' if (ord(bs[2:3]) >= 0x80) else b'\x00'
-    return struct_unpack('<i', bs + last_byte)[0]
+    return compat_struct_unpack('<i', bs + last_byte)[0]
 
 
 def _read_string(reader):
@@ -148,7 +148,7 @@ def _read_bytes(count, reader):
 
 def _read_byte(reader):
     resb = _read_bytes(1, reader=reader)
-    res = struct_unpack('<B', resb)[0]
+    res = compat_struct_unpack('<B', resb)[0]
     return res
 
 
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index dc73f3407..dbac38b55 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -44,6 +44,7 @@ from .compat import (
     compat_parse_qs,
     compat_socket_create_connection,
     compat_str,
+    compat_struct_pack,
     compat_urllib_error,
     compat_urllib_parse,
     compat_urllib_parse_urlencode,
@@ -52,7 +53,6 @@ from .compat import (
     compat_urlparse,
     compat_xpath,
     shlex_quote,
-    struct_pack,
 )
 
 from .socks import (
@@ -1259,7 +1259,7 @@ def bytes_to_intlist(bs):
 def intlist_to_bytes(xs):
     if not xs:
         return b''
-    return struct_pack('%dB' % len(xs), *xs)
+    return compat_struct_pack('%dB' % len(xs), *xs)
 
 
 # Cross-platform file locking

From e21f17fc86aab0ac7f1f4cee28f64e7b9b954f71 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 5 May 2016 17:09:13 +0800
Subject: [PATCH 4901/6219] [test/test_socks] Test with local SOCKS servers

---
 .gitignore                   |  1 +
 .travis.yml                  |  3 +++
 devscripts/install_srelay.sh |  8 +++++++
 test/test_socks.py           | 42 +++++++++++++++++++++++++++++++++---
 4 files changed, 51 insertions(+), 3 deletions(-)
 create mode 100755 devscripts/install_srelay.sh

diff --git a/.gitignore b/.gitignore
index 0e7128551..d5f216b5f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -36,3 +36,4 @@ test/local_parameters.json
 youtube-dl.zsh
 .idea
 .idea/*
+tmp/
diff --git a/.travis.yml b/.travis.yml
index cc21fae8f..998995845 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -7,6 +7,9 @@ python:
   - "3.4"
   - "3.5"
 sudo: false
+install:
+  - bash ./devscripts/install_srelay.sh
+  - export PATH=$PATH:$(pwd)/tmp/srelay-0.4.8b6
 script: nosetests test --verbose
 notifications:
   email:
diff --git a/devscripts/install_srelay.sh b/devscripts/install_srelay.sh
new file mode 100755
index 000000000..33ce8a3f7
--- /dev/null
+++ b/devscripts/install_srelay.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+mkdir -p tmp && cd tmp
+wget -N http://downloads.sourceforge.net/project/socks-relay/socks-relay/srelay-0.4.8/srelay-0.4.8b6.tar.gz
+tar zxvf srelay-0.4.8b6.tar.gz
+cd srelay-0.4.8b6
+./configure
+make
diff --git a/test/test_socks.py b/test/test_socks.py
index 92574c6fd..dc9b8d276 100644
--- a/test/test_socks.py
+++ b/test/test_socks.py
@@ -8,11 +8,20 @@ import sys
 import unittest
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-from test.helper import (FakeYDL, get_params)
-from youtube_dl.compat import compat_urllib_request
+import random
+import subprocess
+
+from test.helper import (
+    FakeYDL,
+    get_params,
+)
+from youtube_dl.compat import (
+    compat_str,
+    compat_urllib_request,
+)
 
 
-class TestSocks(unittest.TestCase):
+class TestMultipleSocks(unittest.TestCase):
     @staticmethod
     def _check_params(attrs):
         params = get_params()
@@ -67,5 +76,32 @@ class TestSocks(unittest.TestCase):
             params['secondary_server_ip'])
 
 
+class TestSocks(unittest.TestCase):
+    def setUp(self):
+        self.port = random.randint(49152, 65535)
+        self.server_process = subprocess.Popen([
+            'srelay', '-f', '-i', '127.0.0.1:%d' % self.port],
+            stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+    def tearDown(self):
+        self.server_process.terminate()
+        self.server_process.communicate()
+
+    def _get_ip(self, protocol):
+        ydl = FakeYDL({
+            'proxy': '%s://127.0.0.1:%d' % (protocol, self.port),
+        })
+        return ydl.urlopen('http://yt-dl.org/ip').read().decode('utf-8')
+
+    def test_socks4(self):
+        self.assertTrue(isinstance(self._get_ip('socks4'), compat_str))
+
+    def test_socks4a(self):
+        self.assertTrue(isinstance(self._get_ip('socks4a'), compat_str))
+
+    def test_socks5(self):
+        self.assertTrue(isinstance(self._get_ip('socks5'), compat_str))
+
+
 if __name__ == '__main__':
     unittest.main()

From fa5cb8d0212918657cb58b4d5791ed3de831bd74 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 8 May 2016 15:14:56 +0800
Subject: [PATCH 4902/6219] [socks] Remove a superfluous clause

---
 youtube_dl/socks.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/youtube_dl/socks.py b/youtube_dl/socks.py
index a5b27fea7..fd49d7435 100644
--- a/youtube_dl/socks.py
+++ b/youtube_dl/socks.py
@@ -210,8 +210,6 @@ class sockssocket(socket.socket):
             if status != SOCKS5_USER_AUTH_SUCCESS:
                 self.close()
                 raise Socks5Error(Socks5Error.ERR_GENERAL_FAILURE)
-        elif method == Socks5Auth.AUTH_NONE:
-            pass
 
     def _setup_socks5(self, address):
         destaddr, port = address

From 6ddb4888d2610df3bbb5024440caddde50fe9ad8 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 8 May 2016 15:15:58 +0800
Subject: [PATCH 4903/6219] [options] Update --proxy description for SOCKS
 proxies

---
 youtube_dl/options.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index d1f8d1331..38efd292d 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -188,7 +188,10 @@ def parseOpts(overrideArguments=None):
     network.add_option(
         '--proxy', dest='proxy',
         default=None, metavar='URL',
-        help='Use the specified HTTP/HTTPS proxy. Pass in an empty string (--proxy "") for direct connection')
+        help='Use the specified HTTP/HTTPS/SOCKS proxy. To enable experimental '
+             'SOCKS proxy, specify a proper scheme. For example '
+             'socks5://127.0.0.1:1080/. Pass in an empty string (--proxy "") '
+             'for direct connection')
     network.add_option(
         '--socket-timeout',
         dest='socket_timeout', type=float, default=None, metavar='SECONDS',

From c2876afafef392220cdb2baebace1d6d533f8d63 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 8 May 2016 15:16:32 +0800
Subject: [PATCH 4904/6219] [test/test_socks] Use a different port range

Seems on Travis CI, ports in the original range are often used.
---
 test/test_socks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/test_socks.py b/test/test_socks.py
index dc9b8d276..d07003ceb 100644
--- a/test/test_socks.py
+++ b/test/test_socks.py
@@ -78,7 +78,7 @@ class TestMultipleSocks(unittest.TestCase):
 
 class TestSocks(unittest.TestCase):
     def setUp(self):
-        self.port = random.randint(49152, 65535)
+        self.port = random.randint(20000, 30000)
         self.server_process = subprocess.Popen([
             'srelay', '-f', '-i', '127.0.0.1:%d' % self.port],
             stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

From 28b4f73620c82e7007b3154e4d5f437cf6fb2608 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 10 May 2016 09:08:08 +0200
Subject: [PATCH 4905/6219] release 2016.05.10

---
 .github/ISSUE_TEMPLATE.md |  6 +++---
 README.md                 |  8 +++++---
 docs/supportedsites.md    | 11 +++++++++--
 youtube_dl/version.py     |  2 +-
 4 files changed, 18 insertions(+), 9 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index a26ff1de4..1fb878b59 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.05.01*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.05.01**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.05.10*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.05.10**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.05.01
+[debug] youtube-dl version 2016.05.10
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/README.md b/README.md
index 50acb26a0..4ef6b6d5a 100644
--- a/README.md
+++ b/README.md
@@ -85,9 +85,11 @@ which means you can modify it, redistribute it or use it however you like.
     --no-color                       Do not emit color codes in output
 
 ## Network Options:
-    --proxy URL                      Use the specified HTTP/HTTPS proxy. Pass in
-                                     an empty string (--proxy "") for direct
-                                     connection
+    --proxy URL                      Use the specified HTTP/HTTPS/SOCKS proxy.
+                                     To enable experimental SOCKS proxy, specify
+                                     a proper scheme. For example
+                                     socks5://127.0.0.1:1080/. Pass in an empty
+                                     string (--proxy "") for direct connection
     --socket-timeout SECONDS         Time to wait before giving up, in seconds
     --source-address IP              Client-side IP address to bind to
                                      (experimental)
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 9fb43671f..de84e5c84 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -77,6 +77,7 @@
  - **Bild**: Bild.de
  - **BiliBili**
  - **BioBioChileTV**
+ - **BIQLE**
  - **BleacherReport**
  - **BleacherReportCMS**
  - **blinkx**
@@ -145,6 +146,7 @@
  - **culturebox.francetvinfo.fr**
  - **CultureUnplugged**
  - **CWTV**
+ - **DailyMail**
  - **dailymotion**
  - **dailymotion:playlist**
  - **dailymotion:user**
@@ -325,6 +327,7 @@
  - **limelight**
  - **limelight:channel**
  - **limelight:channel_list**
+ - **LiTV**
  - **LiveLeak**
  - **livestream**
  - **livestream:original**
@@ -374,6 +377,8 @@
  - **mtvservices:embedded**
  - **MuenchenTV**: münchen.tv
  - **MusicPlayOn**
+ - **mva**: Microsoft Virtual Academy videos
+ - **mva:course**: Microsoft Virtual Academy courses
  - **Mwave**
  - **MwaveMeetGreet**
  - **MySpace**
@@ -463,7 +468,8 @@
  - **pbs**: Public Broadcasting Service (PBS) and member stations: PBS: Public Broadcasting Service, APT - Alabama Public Television (WBIQ), GPB/Georgia Public Broadcasting (WGTV), Mississippi Public Broadcasting (WMPN), Nashville Public Television (WNPT), WFSU-TV (WFSU), WSRE (WSRE), WTCI (WTCI), WPBA/Channel 30 (WPBA), Alaska Public Media (KAKM), Arizona PBS (KAET), KNME-TV/Channel 5 (KNME), Vegas PBS (KLVX), AETN/ARKANSAS ETV NETWORK (KETS), KET (WKLE), WKNO/Channel 10 (WKNO), LPB/LOUISIANA PUBLIC BROADCASTING (WLPB), OETA (KETA), Ozarks Public Television (KOZK), WSIU Public Broadcasting (WSIU), KEET TV (KEET), KIXE/Channel 9 (KIXE), KPBS San Diego (KPBS), KQED (KQED), KVIE Public Television (KVIE), PBS SoCal/KOCE (KOCE), ValleyPBS (KVPT), CONNECTICUT PUBLIC TELEVISION (WEDH), KNPB Channel 5 (KNPB), SOPTV (KSYS), Rocky Mountain PBS (KRMA), KENW-TV3 (KENW), KUED Channel 7 (KUED), Wyoming PBS (KCWC), Colorado Public Television / KBDI 12 (KBDI), KBYU-TV (KBYU), Thirteen/WNET New York (WNET), WGBH/Channel 2 (WGBH), WGBY (WGBY), NJTV Public Media NJ (WNJT), WLIW21 (WLIW), mpt/Maryland Public Television (WMPB), WETA Television and Radio (WETA), WHYY (WHYY), PBS 39 (WLVT), WVPT - Your Source for PBS and More! (WVPT), Howard University Television (WHUT), WEDU PBS (WEDU), WGCU Public Media (WGCU), WPBT2 (WPBT), WUCF TV (WUCF), WUFT/Channel 5 (WUFT), WXEL/Channel 42 (WXEL), WLRN/Channel 17 (WLRN), WUSF Public Broadcasting (WUSF), ETV (WRLK), UNC-TV (WUNC), PBS Hawaii - Oceanic Cable Channel 10 (KHET), Idaho Public Television (KAID), KSPS (KSPS), OPB (KOPB), KWSU/Channel 10 & KTNW/Channel 31 (KWSU), WILL-TV (WILL), Network Knowledge - WSEC/Springfield (WSEC), WTTW11 (WTTW), Iowa Public Television/IPTV (KDIN), Nine Network (KETC), PBS39 Fort Wayne (WFWA), WFYI Indianapolis (WFYI), Milwaukee Public Television (WMVS), WNIN (WNIN), WNIT Public Television (WNIT), WPT (WPNE), WVUT/Channel 22 (WVUT), WEIU/Channel 51 (WEIU), WQPT-TV (WQPT), WYCC PBS Chicago (WYCC), WIPB-TV (WIPB), WTIU (WTIU), CET  (WCET), ThinkTVNetwork (WPTD), WBGU-TV (WBGU), WGVU TV (WGVU), NET1 (KUON), Pioneer Public Television (KWCM), SDPB Television (KUSD), TPT (KTCA), KSMQ (KSMQ), KPTS/Channel 8 (KPTS), KTWU/Channel 11 (KTWU), East Tennessee PBS (WSJK), WCTE-TV (WCTE), WLJT, Channel 11 (WLJT), WOSU TV (WOSU), WOUB/WOUC (WOUB), WVPB (WVPB), WKYU-PBS (WKYU), KERA 13 (KERA), MPBN (WCBB), Mountain Lake PBS (WCFE), NHPTV (WENH), Vermont PBS (WETK), witf (WITF), WQED Multimedia (WQED), WMHT Educational Telecommunications (WMHT), Q-TV (WDCQ), WTVS Detroit Public TV (WTVS), CMU Public Television (WCMU), WKAR-TV (WKAR), WNMU-TV Public TV 13 (WNMU), WDSE - WRPT (WDSE), WGTE TV (WGTE), Lakeland Public Television (KAWE), KMOS-TV - Channels 6.1, 6.2 and 6.3 (KMOS), MontanaPBS (KUSM), KRWG/Channel 22 (KRWG), KACV (KACV), KCOS/Channel 13 (KCOS), WCNY/Channel 24 (WCNY), WNED (WNED), WPBS (WPBS), WSKG Public TV (WSKG), WXXI (WXXI), WPSU (WPSU), WVIA Public Media Studios (WVIA), WTVI (WTVI), Western Reserve PBS (WNEO), WVIZ/PBS ideastream (WVIZ), KCTS 9 (KCTS), Basin PBS (KPBT), KUHT / Channel 8 (KUHT), KLRN (KLRN), KLRU (KLRU), WTJX Channel 12 (WTJX), WCVE PBS (WCVE), KBTC Public Television (KBTC)
  - **pcmag**
  - **People**
- - **Periscope**: Periscope
+ - **periscope**: Periscope
+ - **periscope:user**: Periscope user videos
  - **PhilharmonieDeParis**: Philharmonie de Paris
  - **phoenix.de**
  - **Photobucket**
@@ -700,6 +706,7 @@
  - **Vessel**
  - **Vesti**: Вести.Ru
  - **Vevo**
+ - **VevoPlaylist**
  - **VGTV**: VGTV, BTTV, FTV, Aftenposten and Aftonbladet
  - **vh1.com**
  - **Vice**
@@ -772,7 +779,7 @@
  - **WSJ**: Wall Street Journal
  - **XBef**
  - **XboxClips**
- - **XFileShare**: XFileShare based sites: GorillaVid.in, daclips.in, movpod.in, fastvideo.in, realvid.net, filehoot.com and vidto.me
+ - **XFileShare**: XFileShare based sites: DaClips, FileHoot, GorillaVid, MovPod, PowerWatch, Rapidvideo.ws, TheVideoBee, Vidto, Streamin.To
  - **XHamster**
  - **XHamsterEmbed**
  - **xiami:album**: 虾米音乐 - 专辑
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 551160897..45e40c0d1 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.05.01'
+__version__ = '2016.05.10'

From 702ccf2dc08603fed98d2672f86af1a0e300d83e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 10 May 2016 15:58:25 +0800
Subject: [PATCH 4906/6219] [compat] Rename shlex_quote and remove unused
 subprocess_check_output

---
 youtube_dl/compat.py                          | 19 +++----------------
 youtube_dl/postprocessor/execafterdownload.py |  4 ++--
 youtube_dl/utils.py                           |  4 ++--
 3 files changed, 7 insertions(+), 20 deletions(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index e48c761a6..1392361a1 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -341,9 +341,9 @@ except ImportError:  # Python 2
         return parsed_result
 
 try:
-    from shlex import quote as shlex_quote
+    from shlex import quote as compat_shlex_quote
 except ImportError:  # Python < 3.3
-    def shlex_quote(s):
+    def compat_shlex_quote(s):
         if re.match(r'^[-_\w./]+$', s):
             return s
         else:
@@ -466,18 +466,6 @@ else:
         print(s)
 
 
-try:
-    subprocess_check_output = subprocess.check_output
-except AttributeError:
-    def subprocess_check_output(*args, **kwargs):
-        assert 'input' not in kwargs
-        p = subprocess.Popen(*args, stdout=subprocess.PIPE, **kwargs)
-        output, _ = p.communicate()
-        ret = p.poll()
-        if ret:
-            raise subprocess.CalledProcessError(ret, p.args, output=output)
-        return output
-
 if sys.version_info < (3, 0) and sys.platform == 'win32':
     def compat_getpass(prompt, *args, **kwargs):
         if isinstance(prompt, compat_str):
@@ -635,6 +623,7 @@ __all__ = [
     'compat_parse_qs',
     'compat_print',
     'compat_setenv',
+    'compat_shlex_quote',
     'compat_shlex_split',
     'compat_socket_create_connection',
     'compat_str',
@@ -656,7 +645,5 @@ __all__ = [
     'compat_urlretrieve',
     'compat_xml_parse_error',
     'compat_xpath',
-    'shlex_quote',
-    'subprocess_check_output',
     'workaround_optparse_bug9161',
 ]
diff --git a/youtube_dl/postprocessor/execafterdownload.py b/youtube_dl/postprocessor/execafterdownload.py
index 74f66d669..90630c2d7 100644
--- a/youtube_dl/postprocessor/execafterdownload.py
+++ b/youtube_dl/postprocessor/execafterdownload.py
@@ -3,7 +3,7 @@ from __future__ import unicode_literals
 import subprocess
 
 from .common import PostProcessor
-from ..compat import shlex_quote
+from ..compat import compat_shlex_quote
 from ..utils import PostProcessingError
 
 
@@ -17,7 +17,7 @@ class ExecAfterDownloadPP(PostProcessor):
         if '{}' not in cmd:
             cmd += ' {}'
 
-        cmd = cmd.replace('{}', shlex_quote(information['filepath']))
+        cmd = cmd.replace('{}', compat_shlex_quote(information['filepath']))
 
         self._downloader.to_screen('[exec] Executing command: %s' % cmd)
         retCode = subprocess.call(cmd, shell=True)
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index dbac38b55..e8b09e9db 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -42,6 +42,7 @@ from .compat import (
     compat_http_client,
     compat_kwargs,
     compat_parse_qs,
+    compat_shlex_quote,
     compat_socket_create_connection,
     compat_str,
     compat_struct_pack,
@@ -52,7 +53,6 @@ from .compat import (
     compat_urllib_request,
     compat_urlparse,
     compat_xpath,
-    shlex_quote,
 )
 
 from .socks import (
@@ -1977,7 +1977,7 @@ def ytdl_is_updateable():
 
 def args_to_str(args):
     # Get a short string representation for a subprocess command
-    return ' '.join(shlex_quote(a) for a in args)
+    return ' '.join(compat_shlex_quote(a) for a in args)
 
 
 def error_to_compat_str(err):

From e73b9c65e279f283b28d14be5b7173eae46d4364 Mon Sep 17 00:00:00 2001
From: teemuy <z0rs4m37tAlL>
Date: Wed, 11 May 2016 18:10:30 +0300
Subject: [PATCH 4907/6219] Bugfix: Allow colons in custom HTTP header values.

---
 youtube_dl/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index cbd84c3af..740a1904b 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -69,7 +69,7 @@ def _real_main(argv=None):
         for h in opts.headers:
             if h.find(':', 1) < 0:
                 parser.error('wrong header formatting, it should be key:value, not "%s"' % h)
-            key, value = h.split(':', 2)
+            key, value = h.split(':', 1)
             if opts.verbose:
                 write_string('[debug] Adding header from command line option %s:%s\n' % (key, value))
             std_headers[key] = value

From e0741fd4496c85ef447e72df935cb6edd1af53ed Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 11 May 2016 22:03:30 +0600
Subject: [PATCH 4908/6219] [__init__] Simplify colon presence check

---
 youtube_dl/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 740a1904b..5df965191 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -67,7 +67,7 @@ def _real_main(argv=None):
     # Custom HTTP headers
     if opts.headers is not None:
         for h in opts.headers:
-            if h.find(':', 1) < 0:
+            if ':' not in h:
                 parser.error('wrong header formatting, it should be key:value, not "%s"' % h)
             key, value = h.split(':', 1)
             if opts.verbose:

From 4540515cb3daa0716fa94e54cacb566ef1461ab3 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 12 May 2016 18:48:27 +0800
Subject: [PATCH 4909/6219] [iqiyi] Fix 1080P extraction (closes #9446)

---
 youtube_dl/extractor/iqiyi.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index ffb8008ce..ddcb3c916 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -505,7 +505,10 @@ class IqiyiIE(InfoExtractor):
             'enc': md5_text(enc_key + tail),
             'qyid': _uuid,
             'tn': random.random(),
-            'um': 0,
+            # In iQiyi's flash player, um is set to 1 if there's a logged user
+            # Some 1080P formats are only available with a logged user.
+            # Here force um=1 to trick the iQiyi server
+            'um': 1,
             'authkey': md5_text(md5_text('') + tail),
             'k_tag': 1,
         }

From 778a1ccca7d6cce06faf17867f20b87883d84e98 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 12 May 2016 19:48:48 +0800
Subject: [PATCH 4910/6219] =?UTF-8?q?[utils]=20Add=20=C5=92=20and=20=C5=93?=
 =?UTF-8?q?=20found=20in=20French=20to=20ACCENT=5FCHARS?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Fixes #9463
---
 test/test_utils.py  | 4 ++--
 youtube_dl/utils.py | 6 +++---
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 5702ffa97..ca254779f 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -155,8 +155,8 @@ class TestUtil(unittest.TestCase):
         self.assertTrue(sanitize_filename(':', restricted=True) != '')
 
         self.assertEqual(sanitize_filename(
-            'ÂÃÄÀÁÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖØÙÚÛÜÝÞßàáâãäåæçèéêëìíîïðñòóôõöøùúûüýþÿ', restricted=True),
-            'AAAAAAAECEEEEIIIIDNOOOOOOUUUUYPssaaaaaaaeceeeeiiiionoooooouuuuypy')
+            'ÂÃÄÀÁÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖØŒÙÚÛÜÝÞßàáâãäåæçèéêëìíîïðñòóôõöøœùúûüýþÿ', restricted=True),
+            'AAAAAAAECEEEEIIIIDNOOOOOOOEUUUUYPssaaaaaaaeceeeeiiiionoooooooeuuuuypy')
 
     def test_sanitize_ids(self):
         self.assertEqual(sanitize_filename('_n_cd26wFpw', is_id=True), '_n_cd26wFpw')
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index e8b09e9db..6592c8ec2 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -104,9 +104,9 @@ KNOWN_EXTENSIONS = (
     'f4f', 'f4m', 'm3u8', 'smil')
 
 # needed for sanitizing filenames in restricted mode
-ACCENT_CHARS = dict(zip('ÂÃÄÀÁÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖØÙÚÛÜÝÞßàáâãäåæçèéêëìíîïðñòóôõöøùúûüýþÿ',
-                        itertools.chain('AAAAAA', ['AE'], 'CEEEEIIIIDNOOOOOOUUUUYP', ['ss'],
-                                        'aaaaaa', ['ae'], 'ceeeeiiiionoooooouuuuypy')))
+ACCENT_CHARS = dict(zip('ÂÃÄÀÁÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖØŒÙÚÛÜÝÞßàáâãäåæçèéêëìíîïðñòóôõöøœùúûüýþÿ',
+                        itertools.chain('AAAAAA', ['AE'], 'CEEEEIIIIDNOOOOOO', ['OE'], 'UUUUYP', ['ss'],
+                                        'aaaaaa', ['ae'], 'ceeeeiiiionoooooo', ['oe'], 'uuuuypy')))
 
 
 def preferredencoding():

From 7e8ddca1bb10068356d1ec43cf66e7627b76fce7 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 12 May 2016 19:56:58 +0800
Subject: [PATCH 4911/6219] [vevo] Delay the georestriction check to prevent
 false alerts

Fixes #9408
---
 youtube_dl/extractor/vevo.py | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py
index c0632cd6a..388b4debe 100644
--- a/youtube_dl/extractor/vevo.py
+++ b/youtube_dl/extractor/vevo.py
@@ -213,19 +213,17 @@ class VevoIE(VevoBaseIE):
         formats = []
 
         if not video_info:
-            if response and response.get('statusCode') != 909:
+            try:
+                self._initialize_api(video_id)
+            except ExtractorError:
                 ytid = response.get('errorInfo', {}).get('ytid')
                 if ytid:
                     self.report_warning(
                         'Video is geoblocked, trying with the YouTube video %s' % ytid)
                     return self.url_result(ytid, 'Youtube', ytid)
 
-                if 'statusMessage' in response:
-                    raise ExtractorError('%s said: %s' % (
-                        self.IE_NAME, response['statusMessage']), expected=True)
-                raise ExtractorError('Unable to extract videos')
+                raise
 
-            self._initialize_api(video_id)
             video_info = self._call_api(
                 'video/%s' % video_id, video_id, 'Downloading api video info',
                 'Failed to download video info')

From 1b405bb47d91119cc612a90d26f27f2b93f7c7b4 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 3 May 2016 18:06:50 +0800
Subject: [PATCH 4912/6219] [downloader/f4m] Tolerate truncate segments when
 testing

Replaces #9216

Fixes #9214 and test_Bloomberg partially
---
 youtube_dl/downloader/f4m.py | 42 +++++++++++++++++++++++++++---------
 1 file changed, 32 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index 3d9337afa..314def4cb 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -23,26 +23,38 @@ from ..utils import (
 )
 
 
+class DataTruncatedError(Exception):
+    pass
+
+
 class FlvReader(io.BytesIO):
     """
     Reader for Flv files
     The file format is documented in https://www.adobe.com/devnet/f4v.html
     """
 
+    def read_bytes(self, n):
+        data = self.read(n)
+        if len(data) < n:
+            raise DataTruncatedError(
+                'FlvReader error: need %d bytes while only %d bytes got' % (
+                    n, len(data)))
+        return data
+
     # Utility functions for reading numbers and strings
     def read_unsigned_long_long(self):
-        return compat_struct_unpack('!Q', self.read(8))[0]
+        return compat_struct_unpack('!Q', self.read_bytes(8))[0]
 
     def read_unsigned_int(self):
-        return compat_struct_unpack('!I', self.read(4))[0]
+        return compat_struct_unpack('!I', self.read_bytes(4))[0]
 
     def read_unsigned_char(self):
-        return compat_struct_unpack('!B', self.read(1))[0]
+        return compat_struct_unpack('!B', self.read_bytes(1))[0]
 
     def read_string(self):
         res = b''
         while True:
-            char = self.read(1)
+            char = self.read_bytes(1)
             if char == b'\x00':
                 break
             res += char
@@ -53,18 +65,18 @@ class FlvReader(io.BytesIO):
         Read a box and return the info as a tuple: (box_size, box_type, box_data)
         """
         real_size = size = self.read_unsigned_int()
-        box_type = self.read(4)
+        box_type = self.read_bytes(4)
         header_end = 8
         if size == 1:
             real_size = self.read_unsigned_long_long()
             header_end = 16
-        return real_size, box_type, self.read(real_size - header_end)
+        return real_size, box_type, self.read_bytes(real_size - header_end)
 
     def read_asrt(self):
         # version
         self.read_unsigned_char()
         # flags
-        self.read(3)
+        self.read_bytes(3)
         quality_entry_count = self.read_unsigned_char()
         # QualityEntryCount
         for i in range(quality_entry_count):
@@ -85,7 +97,7 @@ class FlvReader(io.BytesIO):
         # version
         self.read_unsigned_char()
         # flags
-        self.read(3)
+        self.read_bytes(3)
         # time scale
         self.read_unsigned_int()
 
@@ -119,7 +131,7 @@ class FlvReader(io.BytesIO):
         # version
         self.read_unsigned_char()
         # flags
-        self.read(3)
+        self.read_bytes(3)
 
         self.read_unsigned_int()  # BootstrapinfoVersion
         # Profile,Live,Update,Reserved
@@ -374,7 +386,17 @@ class F4mFD(FragmentFD):
                 down.close()
                 reader = FlvReader(down_data)
                 while True:
-                    _, box_type, box_data = reader.read_box_info()
+                    try:
+                        _, box_type, box_data = reader.read_box_info()
+                    except DataTruncatedError:
+                        if test:
+                            # In tests, segments may be truncated, and thus
+                            # FlvReader may not be able to parse the whole
+                            # chunk. If so, write the segment as is
+                            # See https://github.com/rg3/youtube-dl/issues/9214
+                            dest_stream.write(down_data)
+                            break
+                        raise
                     if box_type == b'mdat':
                         dest_stream.write(box_data)
                         break

From a3fa6024d676ec20a06fe618f5c3d6e064f49336 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 12 May 2016 20:05:43 +0800
Subject: [PATCH 4913/6219] [bloomberg] Fix test_Bloomberg

In this test case, sometimes HLS is the best format while sometimes HDS
is. To prevent occasional test failures, force HDS to be the best
format. In the past, testing against HDS formats causes the same error
as #9214, which is fixed as #9377 landed.
---
 youtube_dl/extractor/bloomberg.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/bloomberg.py b/youtube_dl/extractor/bloomberg.py
index 13343bc25..bd538be50 100644
--- a/youtube_dl/extractor/bloomberg.py
+++ b/youtube_dl/extractor/bloomberg.py
@@ -17,6 +17,9 @@ class BloombergIE(InfoExtractor):
             'title': 'Shah\'s Presentation on Foreign-Exchange Strategies',
             'description': 'md5:a8ba0302912d03d246979735c17d2761',
         },
+        'params': {
+            'format': 'best[format_id^=hds]',
+        },
     }, {
         'url': 'http://www.bloomberg.com/news/articles/2015-11-12/five-strange-things-that-have-been-happening-in-financial-markets',
         'only_matching': True,

From f388f616c1f2ad9d2c906c4183cf996c845b2858 Mon Sep 17 00:00:00 2001
From: TRox1972 <archcr8@gmail.com>
Date: Thu, 12 May 2016 16:48:12 +0200
Subject: [PATCH 4914/6219] Update README.md

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 4ef6b6d5a..a2febab2c 100644
--- a/README.md
+++ b/README.md
@@ -417,7 +417,7 @@ which means you can modify it, redistribute it or use it however you like.
 
 # CONFIGURATION
 
-You can configure youtube-dl by placing any supported command line option to a configuration file. On Linux, the system wide configuration file is located at `/etc/youtube-dl.conf` and the user wide configuration file at `~/.config/youtube-dl/config`. On Windows, the user wide configuration file locations are `%APPDATA%\youtube-dl\config.txt` or `C:\Users\<user name>\youtube-dl.conf`.
+You can configure youtube-dl by placing any supported command line option to a configuration file. On Linux and OS X, the system wide configuration file is located at `/etc/youtube-dl.conf` and the user wide configuration file at `~/.config/youtube-dl/config`. On Windows, the user wide configuration file locations are `%APPDATA%\youtube-dl\config.txt` or `C:\Users\<user name>\youtube-dl.conf`.
 
 For example, with the following configuration file youtube-dl will always extract the audio, not copy the mtime, use a proxy and save all videos under `Movies` directory in your home directory:
 ```

From 7581bfc958c8de77adbf8a502564d2263d17479d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 12 May 2016 18:57:53 +0800
Subject: [PATCH 4915/6219] [utils] Unquote crendentials passed to SOCKS
 proxies

Fixes #9450
---
 youtube_dl/utils.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 6592c8ec2..d6f94f8cd 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -50,6 +50,7 @@ from .compat import (
     compat_urllib_parse,
     compat_urllib_parse_urlencode,
     compat_urllib_parse_urlparse,
+    compat_urllib_parse_unquote_plus,
     compat_urllib_request,
     compat_urlparse,
     compat_xpath,
@@ -886,7 +887,8 @@ def make_socks_conn_class(base_class, socks_proxy):
         socks_type,
         url_components.hostname, url_components.port or 1080,
         True,  # Remote DNS
-        url_components.username, url_components.password
+        compat_urllib_parse_unquote_plus(url_components.username),
+        compat_urllib_parse_unquote_plus(url_components.password),
     )
 
     class SocksConnection(base_class):

From 0db3a66162cf1059dbfccd60db350596f7c5b469 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 12 May 2016 23:57:52 +0600
Subject: [PATCH 4916/6219] [twitch] Skip dead tests

---
 youtube_dl/extractor/twitch.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 36ee1adff..68f50487b 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -171,6 +171,7 @@ class TwitchVideoIE(TwitchItemBaseIE):
             'title': 'Worlds Semifinals - Star Horn Royal Club vs. OMG',
         },
         'playlist_mincount': 12,
+        'skip': 'HTTP Error 404: Not Found',
     }
 
 
@@ -187,6 +188,7 @@ class TwitchChapterIE(TwitchItemBaseIE):
             'title': 'ACRL Off Season - Sports Cars @ Nordschleife',
         },
         'playlist_mincount': 3,
+        'skip': 'HTTP Error 404: Not Found',
     }, {
         'url': 'http://www.twitch.tv/tsm_theoddone/c/2349361',
         'only_matching': True,
@@ -368,6 +370,7 @@ class TwitchBookmarksIE(TwitchPlaylistBaseIE):
             'title': 'Ognos',
         },
         'playlist_mincount': 3,
+        'skip': 'HTTP Error 404: Not Found',
     }
 
     def _extract_playlist_page(self, response):

From 0df79d552a6d528ac5bb1a9cce99199aafe79144 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 13 May 2016 00:14:30 +0600
Subject: [PATCH 4917/6219] [twitch:bookmarks] Remove extractor

Bookmarks no longer available
---
 youtube_dl/extractor/extractors.py |  1 -
 youtube_dl/extractor/twitch.py     | 26 --------------------------
 2 files changed, 27 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index a0bb3d4c2..f2bd4fe97 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -834,7 +834,6 @@ from .twitch import (
     TwitchVodIE,
     TwitchProfileIE,
     TwitchPastBroadcastsIE,
-    TwitchBookmarksIE,
     TwitchStreamIE,
 )
 from .twitter import (
diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 68f50487b..f7b98e190 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -357,32 +357,6 @@ class TwitchPastBroadcastsIE(TwitchPlaylistBaseIE):
     }
 
 
-class TwitchBookmarksIE(TwitchPlaylistBaseIE):
-    IE_NAME = 'twitch:bookmarks'
-    _VALID_URL = r'%s/(?P<id>[^/]+)/profile/bookmarks/?(?:\#.*)?$' % TwitchBaseIE._VALID_URL_BASE
-    _PLAYLIST_URL = '%s/api/bookmark/?user=%%s&offset=%%d&limit=%%d' % TwitchBaseIE._API_BASE
-    _PLAYLIST_TYPE = 'bookmarks'
-
-    _TEST = {
-        'url': 'http://www.twitch.tv/ognos/profile/bookmarks',
-        'info_dict': {
-            'id': 'ognos',
-            'title': 'Ognos',
-        },
-        'playlist_mincount': 3,
-        'skip': 'HTTP Error 404: Not Found',
-    }
-
-    def _extract_playlist_page(self, response):
-        entries = []
-        for bookmark in response.get('bookmarks', []):
-            video = bookmark.get('video')
-            if not video:
-                continue
-            entries.append(video['url'])
-        return entries
-
-
 class TwitchStreamIE(TwitchBaseIE):
     IE_NAME = 'twitch:stream'
     _VALID_URL = r'%s/(?P<id>[^/#?]+)/?(?:\#.*)?$' % TwitchBaseIE._VALID_URL_BASE

From d8d540cf0d11dbf7b3d9de611470fc7114c8d1ca Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 13 May 2016 02:07:12 +0600
Subject: [PATCH 4918/6219] [nrk] Rework extractor (Closes #9470)

---
 youtube_dl/extractor/nrk.py | 435 ++++++++++++++++--------------------
 1 file changed, 196 insertions(+), 239 deletions(-)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index 51dfc27ac..f0fbdd8be 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -4,91 +4,224 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urlparse,
-    compat_urllib_parse_unquote,
-)
+from ..compat import compat_urllib_parse_unquote
 from ..utils import (
-    determine_ext,
     ExtractorError,
-    float_or_none,
+    int_or_none,
+    parse_age_limit,
     parse_duration,
-    unified_strdate,
 )
 
 
-class NRKIE(InfoExtractor):
-    _VALID_URL = r'(?:nrk:|https?://(?:www\.)?nrk\.no/video/PS\*)(?P<id>\d+)'
-
-    _TESTS = [
-        {
-            'url': 'http://www.nrk.no/video/PS*150533',
-            # MD5 is unstable
-            'info_dict': {
-                'id': '150533',
-                'ext': 'flv',
-                'title': 'Dompap og andre fugler i Piip-Show',
-                'description': 'md5:d9261ba34c43b61c812cb6b0269a5c8f',
-                'duration': 263,
-            }
-        },
-        {
-            'url': 'http://www.nrk.no/video/PS*154915',
-            # MD5 is unstable
-            'info_dict': {
-                'id': '154915',
-                'ext': 'flv',
-                'title': 'Slik høres internett ut når du er blind',
-                'description': 'md5:a621f5cc1bd75c8d5104cb048c6b8568',
-                'duration': 20,
-            }
-        },
-    ]
+class NRKBaseIE(InfoExtractor):
+    def _extract_formats(self, manifest_url, video_id, fatal=True):
+        return self._extract_f4m_formats(
+            manifest_url + '?hdcore=3.5.0&plugin=aasp-3.5.0.151.81',
+            video_id, f4m_id='hds', fatal=fatal)
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
         data = self._download_json(
-            'http://v8.psapi.nrk.no/mediaelement/%s' % video_id,
-            video_id, 'Downloading media JSON')
+            'http://%s/mediaelement/%s' % (self._API_HOST, video_id),
+            video_id, 'Downloading mediaelement JSON')
 
-        media_url = data.get('mediaUrl')
+        title = data.get('fullTitle') or data.get('mainTitle') or data['title']
+        video_id = data.get('id') or video_id
 
-        if not media_url:
-            if data['usageRights']['isGeoBlocked']:
+        entries = []
+
+        media_assets = data.get('mediaAssets')
+        if media_assets and isinstance(media_assets, list):
+            def video_id_and_title(idx):
+                return ((video_id, title) if len(media_assets) == 1
+                        else ('%s-%d' % (video_id, idx), '%s (Part %d)' % (title, idx)))
+            for num, asset in enumerate(media_assets, 1):
+                asset_url = asset.get('url')
+                if not asset_url:
+                    continue
+                formats = self._extract_formats(asset_url, video_id, fatal=False)
+                if not formats:
+                    continue
+                self._sort_formats(formats)
+                entry_id, entry_title = video_id_and_title(num)
+                duration = parse_duration(asset.get('duration'))
+                subtitles = {}
+                for subtitle in ('webVtt', 'timedText'):
+                    subtitle_url = asset.get('%sSubtitlesUrl' % subtitle)
+                    if subtitle_url:
+                        subtitles.setdefault('no', []).append({'url': subtitle_url})
+                entries.append({
+                    'id': asset.get('carrierId') or entry_id,
+                    'title': entry_title,
+                    'duration': duration,
+                    'subtitles': subtitles,
+                    'formats': formats,
+                })
+
+        if not entries:
+            media_url = data.get('mediaUrl')
+            if media_url:
+                formats = self._extract_formats(media_url, video_id)
+                self._sort_formats(formats)
+                duration = parse_duration(data.get('duration'))
+                entries = [{
+                    'id': video_id,
+                    'title': title,
+                    'duration': duration,
+                    'formats': formats,
+                }]
+
+        if not entries:
+            if data.get('usageRights', {}).get('isGeoBlocked'):
                 raise ExtractorError(
                     'NRK har ikke rettigheter til å vise dette programmet utenfor Norge',
                     expected=True)
 
-        if determine_ext(media_url) == 'f4m':
-            formats = self._extract_f4m_formats(
-                media_url + '?hdcore=3.5.0&plugin=aasp-3.5.0.151.81', video_id, f4m_id='hds')
-            self._sort_formats(formats)
-        else:
-            formats = [{
-                'url': media_url,
-                'ext': 'flv',
-            }]
-
-        duration = parse_duration(data.get('duration'))
+        conviva = data.get('convivaStatistics') or {}
+        series = conviva.get('seriesName') or data.get('seriesTitle')
+        episode = conviva.get('episodeName') or data.get('episodeNumberOrDate')
 
+        thumbnails = None
         images = data.get('images')
-        if images:
-            thumbnails = images['webImages']
-            thumbnails.sort(key=lambda image: image['pixelWidth'])
-            thumbnail = thumbnails[-1]['imageUrl']
-        else:
-            thumbnail = None
+        if images and isinstance(images, dict):
+            web_images = images.get('webImages')
+            if isinstance(web_images, list):
+                thumbnails = [{
+                    'url': image['imageUrl'],
+                    'width': int_or_none(image.get('width')),
+                    'height': int_or_none(image.get('height')),
+                } for image in web_images if image.get('imageUrl')]
 
-        return {
-            'id': video_id,
-            'title': data['title'],
-            'description': data['description'],
-            'duration': duration,
-            'thumbnail': thumbnail,
-            'formats': formats,
+        description = data.get('description')
+
+        common_info = {
+            'description': description,
+            'series': series,
+            'episode': episode,
+            'age_limit': parse_age_limit(data.get('legalAge')),
+            'thumbnails': thumbnails,
         }
 
+        vcodec = 'none' if data.get('mediaType') == 'Audio' else None
+
+        # TODO: extract chapters when https://github.com/rg3/youtube-dl/pull/9409 is merged
+
+        for entry in entries:
+            entry.update(common_info)
+            for f in entry['formats']:
+                f['vcodec'] = vcodec
+
+        return self.playlist_result(entries, video_id, title, description)
+
+
+class NRKIE(NRKBaseIE):
+    _VALID_URL = r'(?:nrk:|https?://(?:www\.)?nrk\.no/video/PS\*)(?P<id>\d+)'
+    _API_HOST = 'v8.psapi.nrk.no'
+    _TESTS = [{
+        # video
+        'url': 'http://www.nrk.no/video/PS*150533',
+        # MD5 is unstable
+        'info_dict': {
+            'id': '150533',
+            'ext': 'flv',
+            'title': 'Dompap og andre fugler i Piip-Show',
+            'description': 'md5:d9261ba34c43b61c812cb6b0269a5c8f',
+            'duration': 263,
+        }
+    }, {
+        # audio
+        'url': 'http://www.nrk.no/video/PS*154915',
+        # MD5 is unstable
+        'info_dict': {
+            'id': '154915',
+            'ext': 'flv',
+            'title': 'Slik høres internett ut når du er blind',
+            'description': 'md5:a621f5cc1bd75c8d5104cb048c6b8568',
+            'duration': 20,
+        }
+    }]
+
+
+class NRKTVIE(NRKBaseIE):
+    IE_DESC = 'NRK TV and NRK Radio'
+    _VALID_URL = r'https?://(?:tv|radio)\.nrk(?:super)?\.no/(?:serie/[^/]+|program)/(?P<id>[a-zA-Z]{4}\d{8})(?:/\d{2}-\d{2}-\d{4})?(?:#del=(?P<part_id>\d+))?'
+    _API_HOST = 'psapi-we.nrk.no'
+
+    _TESTS = [{
+        'url': 'https://tv.nrk.no/serie/20-spoersmaal-tv/MUHH48000314/23-05-2014',
+        'info_dict': {
+            'id': 'MUHH48000314',
+            'ext': 'mp4',
+            'title': '20 spørsmål',
+            'description': 'md5:bdea103bc35494c143c6a9acdd84887a',
+            'upload_date': '20140523',
+            'duration': 1741.52,
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }, {
+        'url': 'https://tv.nrk.no/program/mdfp15000514',
+        'info_dict': {
+            'id': 'mdfp15000514',
+            'ext': 'mp4',
+            'title': 'Grunnlovsjubiléet - Stor ståhei for ingenting',
+            'description': 'md5:654c12511f035aed1e42bdf5db3b206a',
+            'upload_date': '20140524',
+            'duration': 4605.08,
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }, {
+        # single playlist video
+        'url': 'https://tv.nrk.no/serie/tour-de-ski/MSPO40010515/06-01-2015#del=2',
+        'md5': 'adbd1dbd813edaf532b0a253780719c2',
+        'info_dict': {
+            'id': 'MSPO40010515-part2',
+            'ext': 'flv',
+            'title': 'Tour de Ski: Sprint fri teknikk, kvinner og menn 06.01.2015 (del 2:2)',
+            'description': 'md5:238b67b97a4ac7d7b4bf0edf8cc57d26',
+            'upload_date': '20150106',
+        },
+        'skip': 'Only works from Norway',
+    }, {
+        'url': 'https://tv.nrk.no/serie/tour-de-ski/MSPO40010515/06-01-2015',
+        'playlist': [{
+            'md5': '9480285eff92d64f06e02a5367970a7a',
+            'info_dict': {
+                'id': 'MSPO40010515-part1',
+                'ext': 'flv',
+                'title': 'Tour de Ski: Sprint fri teknikk, kvinner og menn 06.01.2015 (del 1:2)',
+                'description': 'md5:238b67b97a4ac7d7b4bf0edf8cc57d26',
+                'upload_date': '20150106',
+            },
+        }, {
+            'md5': 'adbd1dbd813edaf532b0a253780719c2',
+            'info_dict': {
+                'id': 'MSPO40010515-part2',
+                'ext': 'flv',
+                'title': 'Tour de Ski: Sprint fri teknikk, kvinner og menn 06.01.2015 (del 2:2)',
+                'description': 'md5:238b67b97a4ac7d7b4bf0edf8cc57d26',
+                'upload_date': '20150106',
+            },
+        }],
+        'info_dict': {
+            'id': 'MSPO40010515',
+            'title': 'Tour de Ski: Sprint fri teknikk, kvinner og menn',
+            'description': 'md5:238b67b97a4ac7d7b4bf0edf8cc57d26',
+            'upload_date': '20150106',
+            'duration': 6947.52,
+        },
+        'skip': 'Only works from Norway',
+    }, {
+        'url': 'https://radio.nrk.no/serie/dagsnytt/NPUB21019315/12-07-2015#',
+        'only_matching': True,
+    }]
+
 
 class NRKPlaylistIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?nrk\.no/(?!video|skole)(?:[^/]+/)+(?P<id>[^/]+)'
@@ -159,179 +292,3 @@ class NRKSkoleIE(InfoExtractor):
 
         nrk_id = self._search_regex(r'data-nrk-id=["\'](\d+)', webpage, 'nrk id')
         return self.url_result('nrk:%s' % nrk_id)
-
-
-class NRKTVIE(InfoExtractor):
-    IE_DESC = 'NRK TV and NRK Radio'
-    _VALID_URL = r'(?P<baseurl>https?://(?:tv|radio)\.nrk(?:super)?\.no/)(?:serie/[^/]+|program)/(?P<id>[a-zA-Z]{4}\d{8})(?:/\d{2}-\d{2}-\d{4})?(?:#del=(?P<part_id>\d+))?'
-
-    _TESTS = [
-        {
-            'url': 'https://tv.nrk.no/serie/20-spoersmaal-tv/MUHH48000314/23-05-2014',
-            'info_dict': {
-                'id': 'MUHH48000314',
-                'ext': 'mp4',
-                'title': '20 spørsmål',
-                'description': 'md5:bdea103bc35494c143c6a9acdd84887a',
-                'upload_date': '20140523',
-                'duration': 1741.52,
-            },
-            'params': {
-                # m3u8 download
-                'skip_download': True,
-            },
-        },
-        {
-            'url': 'https://tv.nrk.no/program/mdfp15000514',
-            'info_dict': {
-                'id': 'mdfp15000514',
-                'ext': 'mp4',
-                'title': 'Grunnlovsjubiléet - Stor ståhei for ingenting',
-                'description': 'md5:654c12511f035aed1e42bdf5db3b206a',
-                'upload_date': '20140524',
-                'duration': 4605.08,
-            },
-            'params': {
-                # m3u8 download
-                'skip_download': True,
-            },
-        },
-        {
-            # single playlist video
-            'url': 'https://tv.nrk.no/serie/tour-de-ski/MSPO40010515/06-01-2015#del=2',
-            'md5': 'adbd1dbd813edaf532b0a253780719c2',
-            'info_dict': {
-                'id': 'MSPO40010515-part2',
-                'ext': 'flv',
-                'title': 'Tour de Ski: Sprint fri teknikk, kvinner og menn 06.01.2015 (del 2:2)',
-                'description': 'md5:238b67b97a4ac7d7b4bf0edf8cc57d26',
-                'upload_date': '20150106',
-            },
-            'skip': 'Only works from Norway',
-        },
-        {
-            'url': 'https://tv.nrk.no/serie/tour-de-ski/MSPO40010515/06-01-2015',
-            'playlist': [
-                {
-                    'md5': '9480285eff92d64f06e02a5367970a7a',
-                    'info_dict': {
-                        'id': 'MSPO40010515-part1',
-                        'ext': 'flv',
-                        'title': 'Tour de Ski: Sprint fri teknikk, kvinner og menn 06.01.2015 (del 1:2)',
-                        'description': 'md5:238b67b97a4ac7d7b4bf0edf8cc57d26',
-                        'upload_date': '20150106',
-                    },
-                },
-                {
-                    'md5': 'adbd1dbd813edaf532b0a253780719c2',
-                    'info_dict': {
-                        'id': 'MSPO40010515-part2',
-                        'ext': 'flv',
-                        'title': 'Tour de Ski: Sprint fri teknikk, kvinner og menn 06.01.2015 (del 2:2)',
-                        'description': 'md5:238b67b97a4ac7d7b4bf0edf8cc57d26',
-                        'upload_date': '20150106',
-                    },
-                },
-            ],
-            'info_dict': {
-                'id': 'MSPO40010515',
-                'title': 'Tour de Ski: Sprint fri teknikk, kvinner og menn',
-                'description': 'md5:238b67b97a4ac7d7b4bf0edf8cc57d26',
-                'upload_date': '20150106',
-                'duration': 6947.5199999999995,
-            },
-            'skip': 'Only works from Norway',
-        },
-        {
-            'url': 'https://radio.nrk.no/serie/dagsnytt/NPUB21019315/12-07-2015#',
-            'only_matching': True,
-        }
-    ]
-
-    def _extract_f4m(self, manifest_url, video_id):
-        return self._extract_f4m_formats(
-            manifest_url + '?hdcore=3.1.1&plugin=aasp-3.1.1.69.124', video_id, f4m_id='hds')
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        part_id = mobj.group('part_id')
-        base_url = mobj.group('baseurl')
-
-        webpage = self._download_webpage(url, video_id)
-
-        title = self._html_search_meta(
-            'title', webpage, 'title')
-        description = self._html_search_meta(
-            'description', webpage, 'description')
-
-        thumbnail = self._html_search_regex(
-            r'data-posterimage="([^"]+)"',
-            webpage, 'thumbnail', fatal=False)
-        upload_date = unified_strdate(self._html_search_meta(
-            'rightsfrom', webpage, 'upload date', fatal=False))
-        duration = float_or_none(self._html_search_regex(
-            r'data-duration="([^"]+)"',
-            webpage, 'duration', fatal=False))
-
-        # playlist
-        parts = re.findall(
-            r'<a href="#del=(\d+)"[^>]+data-argument="([^"]+)">([^<]+)</a>', webpage)
-        if parts:
-            entries = []
-            for current_part_id, stream_url, part_title in parts:
-                if part_id and current_part_id != part_id:
-                    continue
-                video_part_id = '%s-part%s' % (video_id, current_part_id)
-                formats = self._extract_f4m(stream_url, video_part_id)
-                entries.append({
-                    'id': video_part_id,
-                    'title': part_title,
-                    'description': description,
-                    'thumbnail': thumbnail,
-                    'upload_date': upload_date,
-                    'formats': formats,
-                })
-            if part_id:
-                if entries:
-                    return entries[0]
-            else:
-                playlist = self.playlist_result(entries, video_id, title, description)
-                playlist.update({
-                    'thumbnail': thumbnail,
-                    'upload_date': upload_date,
-                    'duration': duration,
-                })
-                return playlist
-
-        formats = []
-
-        f4m_url = re.search(r'data-media="([^"]+)"', webpage)
-        if f4m_url:
-            formats.extend(self._extract_f4m(f4m_url.group(1), video_id))
-
-        m3u8_url = re.search(r'data-hls-media="([^"]+)"', webpage)
-        if m3u8_url:
-            formats.extend(self._extract_m3u8_formats(m3u8_url.group(1), video_id, 'mp4', m3u8_id='hls'))
-        self._sort_formats(formats)
-
-        subtitles_url = self._html_search_regex(
-            r'data-subtitlesurl\s*=\s*(["\'])(?P<url>.+?)\1',
-            webpage, 'subtitle URL', default=None, group='url')
-        subtitles = {}
-        if subtitles_url:
-            subtitles['no'] = [{
-                'ext': 'ttml',
-                'url': compat_urlparse.urljoin(base_url, subtitles_url),
-            }]
-
-        return {
-            'id': video_id,
-            'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
-            'upload_date': upload_date,
-            'duration': duration,
-            'formats': formats,
-            'subtitles': subtitles,
-        }

From b9e7bc55da1c1275737b356efadc06435b8bfa2c Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 12 May 2016 22:45:54 +0100
Subject: [PATCH 4919/6219] [mgtv] extract http formats

---
 youtube_dl/extractor/mgtv.py | 43 ++++++++++++++++++------------------
 1 file changed, 22 insertions(+), 21 deletions(-)

diff --git a/youtube_dl/extractor/mgtv.py b/youtube_dl/extractor/mgtv.py
index a14d176a5..9fbc74f5d 100644
--- a/youtube_dl/extractor/mgtv.py
+++ b/youtube_dl/extractor/mgtv.py
@@ -11,7 +11,7 @@ class MGTVIE(InfoExtractor):
 
     _TEST = {
         'url': 'http://www.mgtv.com/v/1/290525/f/3116640.html',
-        'md5': '',
+        'md5': '1bdadcf760a0b90946ca68ee9a2db41a',
         'info_dict': {
             'id': '3116640',
             'ext': 'mp4',
@@ -20,15 +20,6 @@ class MGTVIE(InfoExtractor):
             'duration': 7461,
             'thumbnail': 're:^https?://.*\.jpg$',
         },
-        'params': {
-            'skip_download': True,  # m3u8 download
-        },
-    }
-
-    _FORMAT_MAP = {
-        '标清': ('Standard', 0),
-        '高清': ('High', 1),
-        '超清': ('SuperHigh', 2),
     }
 
     def _real_extract(self, url):
@@ -40,17 +31,27 @@ class MGTVIE(InfoExtractor):
 
         formats = []
         for idx, stream in enumerate(api_data['stream']):
-            format_name = stream.get('name')
-            format_id, preference = self._FORMAT_MAP.get(format_name, (None, None))
-            format_info = self._download_json(
-                stream['url'], video_id,
-                note='Download video info for format %s' % format_id or '#%d' % idx)
-            formats.append({
-                'format_id': format_id,
-                'url': format_info['info'],
-                'ext': 'mp4',  # These are m3u8 playlists
-                'preference': preference,
-            })
+            stream_url = stream.get('url')
+            if not stream_url:
+                continue
+            tbr = int_or_none(self._search_regex(
+                r'(\d+)\.mp4', stream_url, 'tbr', default=None))
+
+            def extract_format(stream_url, format_id, idx, query={}):
+                format_info = self._download_json(
+                    stream_url, video_id,
+                    note='Download video info for format %s' % format_id or '#%d' % idx, query=query)
+                return {
+                    'format_id': format_id,
+                    'url': format_info['info'],
+                    'ext': 'mp4',
+                    'tbr': tbr,
+                }
+
+            formats.append(extract_format(
+                stream_url, 'hls-%d' % tbr if tbr else None, idx * 2))
+            formats.append(extract_format(stream_url.replace(
+                '/playlist.m3u8', ''), 'http-%d' % tbr if tbr else None, idx * 2 + 1, {'pno': 1031}))
         self._sort_formats(formats)
 
         return {

From 99d79b8692ae8981aff91cf5b1475516b60eb765 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 13 May 2016 05:21:45 +0100
Subject: [PATCH 4920/6219] [ustudio] add support ustudio app/embed urls

---
 youtube_dl/extractor/ustudio.py | 66 +++++++++++++++++++++++++++++++--
 1 file changed, 62 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/ustudio.py b/youtube_dl/extractor/ustudio.py
index cafc082b6..3484a2046 100644
--- a/youtube_dl/extractor/ustudio.py
+++ b/youtube_dl/extractor/ustudio.py
@@ -6,10 +6,12 @@ from .common import InfoExtractor
 from ..utils import (
     int_or_none,
     unified_strdate,
+    unescapeHTML,
 )
 
 
 class UstudioIE(InfoExtractor):
+    IE_NAME = 'ustudio'
     _VALID_URL = r'https?://(?:(?:www|v1)\.)?ustudio\.com/video/(?P<id>[^/]+)/(?P<display_id>[^/?#&]+)'
     _TEST = {
         'url': 'http://ustudio.com/video/Uxu2my9bgSph/san_francisco_golden_gate_bridge',
@@ -27,9 +29,7 @@ class UstudioIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        display_id = mobj.group('display_id')
+        video_id, display_id = re.match(self._VALID_URL, url).groups()
 
         config = self._download_xml(
             'http://v1.ustudio.com/embed/%s/ustudio/config.xml' % video_id,
@@ -37,7 +37,7 @@ class UstudioIE(InfoExtractor):
 
         def extract(kind):
             return [{
-                'url': item.attrib['url'],
+                'url': unescapeHTML(item.attrib['url']),
                 'width': int_or_none(item.get('width')),
                 'height': int_or_none(item.get('height')),
             } for item in config.findall('./qualities/quality/%s' % kind) if item.get('url')]
@@ -65,3 +65,61 @@ class UstudioIE(InfoExtractor):
             'uploader': uploader,
             'formats': formats,
         }
+
+
+class UstudioEmbedIE(InfoExtractor):
+    IE_NAME = 'ustudio:embed'
+    _VALID_URL = r'https?://(?:(?:app|embed)\.)?ustudio\.com/embed/(?P<uid>[^/]+)/(?P<id>[^/]+)'
+    _TEST = {
+        'url': 'http://app.ustudio.com/embed/DeN7VdYRDKhP/Uw7G1kMCe65T',
+        'md5': '47c0be52a09b23a7f40de9469cec58f4',
+        'info_dict': {
+            'id': 'Uw7G1kMCe65T',
+            'ext': 'mp4',
+            'title': '5 Things IT Should Know About Video',
+            'description': 'md5:93d32650884b500115e158c5677d25ad',
+            'uploader_id': 'DeN7VdYRDKhP',
+        }
+    }
+
+    def _real_extract(self, url):
+        uploader_id, video_id = re.match(self._VALID_URL, url).groups()
+        video_data = self._download_json(
+            'http://app.ustudio.com/embed/%s/%s/config.json' % (uploader_id, video_id),
+            video_id)['videos'][0]
+        title = video_data['name']
+
+        formats = []
+        for ext, qualities in video_data.get('transcodes', {}).items():
+            for quality in qualities:
+                quality_url = quality.get('url')
+                if not quality_url:
+                    continue
+                height = int_or_none(quality.get('height'))
+                formats.append({
+                    'format_id': '%s-%dp' % (ext, height) if height else ext,
+                    'url': quality_url,
+                    'width': int_or_none(quality.get('width')),
+                    'height': height,
+                })
+        self._sort_formats(formats)
+
+        thumbnails = []
+        for image in video_data.get('images', []):
+            image_url = image.get('url')
+            if not image_url:
+                continue
+            thumbnails.append({
+                'url': image_url,
+            })
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': video_data.get('description'),
+            'duration': int_or_none(video_data.get('duration')),
+            'uploader_id': uploader_id,
+            'tags': video_data.get('keywords'),
+            'thumbnails': thumbnails,
+            'formats': formats,
+        }

From cdf32ff15d6fc9d1902bfb3ed10a582070d20cd9 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 13 May 2016 05:25:32 +0100
Subject: [PATCH 4921/6219] [extractors] add import for UstudioEmbedIE

---
 youtube_dl/extractor/extractors.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index f2bd4fe97..50d2204f2 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -851,7 +851,10 @@ from .unistra import UnistraIE
 from .urort import UrortIE
 from .usatoday import USATodayIE
 from .ustream import UstreamIE, UstreamChannelIE
-from .ustudio import UstudioIE
+from .ustudio import (
+    UstudioIE,
+    UstudioEmbedIE,
+)
 from .varzesh3 import Varzesh3IE
 from .vbox7 import Vbox7IE
 from .veehd import VeeHDIE

From 18cf6381f6b140431f3a747fc2d222be08ab2e23 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 13 May 2016 08:05:28 +0100
Subject: [PATCH 4922/6219] [nrk] extract m3u8 formats

---
 youtube_dl/extractor/nrk.py | 39 +++++++++++++++----------------------
 1 file changed, 16 insertions(+), 23 deletions(-)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index f0fbdd8be..7532f40c1 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -15,9 +15,14 @@ from ..utils import (
 
 class NRKBaseIE(InfoExtractor):
     def _extract_formats(self, manifest_url, video_id, fatal=True):
-        return self._extract_f4m_formats(
+        formats = []
+        formats.extend(self._extract_f4m_formats(
             manifest_url + '?hdcore=3.5.0&plugin=aasp-3.5.0.151.81',
-            video_id, f4m_id='hds', fatal=fatal)
+            video_id, f4m_id='hds', fatal=fatal))
+        formats.extend(self._extract_m3u8_formats(manifest_url.replace(
+            'akamaihd.net/z/', 'akamaihd.net/i/').replace('/manifest.f4m', '/master.m3u8'),
+            video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=fatal))
+        return formats
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -121,10 +126,10 @@ class NRKIE(NRKBaseIE):
     _TESTS = [{
         # video
         'url': 'http://www.nrk.no/video/PS*150533',
-        # MD5 is unstable
+        'md5': '2f7f6eeb2aacdd99885f355428715cfa',
         'info_dict': {
             'id': '150533',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Dompap og andre fugler i Piip-Show',
             'description': 'md5:d9261ba34c43b61c812cb6b0269a5c8f',
             'duration': 263,
@@ -150,32 +155,24 @@ class NRKTVIE(NRKBaseIE):
 
     _TESTS = [{
         'url': 'https://tv.nrk.no/serie/20-spoersmaal-tv/MUHH48000314/23-05-2014',
+        'md5': '4e9ca6629f09e588ed240fb11619922a',
         'info_dict': {
-            'id': 'MUHH48000314',
+            'id': 'MUHH48000314AA',
             'ext': 'mp4',
-            'title': '20 spørsmål',
+            'title': '20 spørsmål 23.05.2014',
             'description': 'md5:bdea103bc35494c143c6a9acdd84887a',
-            'upload_date': '20140523',
             'duration': 1741.52,
         },
-        'params': {
-            # m3u8 download
-            'skip_download': True,
-        },
     }, {
         'url': 'https://tv.nrk.no/program/mdfp15000514',
+        'md5': '43d0be26663d380603a9cf0c24366531',
         'info_dict': {
-            'id': 'mdfp15000514',
+            'id': 'MDFP15000514CA',
             'ext': 'mp4',
-            'title': 'Grunnlovsjubiléet - Stor ståhei for ingenting',
-            'description': 'md5:654c12511f035aed1e42bdf5db3b206a',
-            'upload_date': '20140524',
+            'title': 'Grunnlovsjubiléet - Stor ståhei for ingenting 24.05.2014',
+            'description': 'md5:89290c5ccde1b3a24bb8050ab67fe1db',
             'duration': 4605.08,
         },
-        'params': {
-            # m3u8 download
-            'skip_download': True,
-        },
     }, {
         # single playlist video
         'url': 'https://tv.nrk.no/serie/tour-de-ski/MSPO40010515/06-01-2015#del=2',
@@ -185,7 +182,6 @@ class NRKTVIE(NRKBaseIE):
             'ext': 'flv',
             'title': 'Tour de Ski: Sprint fri teknikk, kvinner og menn 06.01.2015 (del 2:2)',
             'description': 'md5:238b67b97a4ac7d7b4bf0edf8cc57d26',
-            'upload_date': '20150106',
         },
         'skip': 'Only works from Norway',
     }, {
@@ -197,7 +193,6 @@ class NRKTVIE(NRKBaseIE):
                 'ext': 'flv',
                 'title': 'Tour de Ski: Sprint fri teknikk, kvinner og menn 06.01.2015 (del 1:2)',
                 'description': 'md5:238b67b97a4ac7d7b4bf0edf8cc57d26',
-                'upload_date': '20150106',
             },
         }, {
             'md5': 'adbd1dbd813edaf532b0a253780719c2',
@@ -206,14 +201,12 @@ class NRKTVIE(NRKBaseIE):
                 'ext': 'flv',
                 'title': 'Tour de Ski: Sprint fri teknikk, kvinner og menn 06.01.2015 (del 2:2)',
                 'description': 'md5:238b67b97a4ac7d7b4bf0edf8cc57d26',
-                'upload_date': '20150106',
             },
         }],
         'info_dict': {
             'id': 'MSPO40010515',
             'title': 'Tour de Ski: Sprint fri teknikk, kvinner og menn',
             'description': 'md5:238b67b97a4ac7d7b4bf0edf8cc57d26',
-            'upload_date': '20150106',
             'duration': 6947.52,
         },
         'skip': 'Only works from Norway',

From ad55e101651edc732acac22cfb25d276d6c8bdca Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 13 May 2016 08:35:38 +0100
Subject: [PATCH 4923/6219] [brightcove] change the protocol for m3u8 formats
 to m3u8_native

---
 youtube_dl/extractor/brightcove.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index f0781fc27..fc7fc5b16 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -307,9 +307,10 @@ class BrightcoveLegacyIE(InfoExtractor):
                                     playlist_title=playlist_info['mediaCollectionDTO']['displayName'])
 
     def _extract_video_info(self, video_info):
+        video_id = compat_str(video_info['id'])
         publisher_id = video_info.get('publisherId')
         info = {
-            'id': compat_str(video_info['id']),
+            'id': video_id,
             'title': video_info['displayName'].strip(),
             'description': video_info.get('shortDescription'),
             'thumbnail': video_info.get('videoStillURL') or video_info.get('thumbnailURL'),
@@ -331,7 +332,8 @@ class BrightcoveLegacyIE(InfoExtractor):
                     url_comp = compat_urllib_parse_urlparse(url)
                     if url_comp.path.endswith('.m3u8'):
                         formats.extend(
-                            self._extract_m3u8_formats(url, info['id'], 'mp4'))
+                            self._extract_m3u8_formats(
+                                url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
                         continue
                     elif 'akamaihd.net' in url_comp.netloc:
                         # This type of renditions are served through
@@ -365,7 +367,7 @@ class BrightcoveLegacyIE(InfoExtractor):
                     a_format.update({
                         'format_id': 'hls%s' % ('-%s' % tbr if tbr else ''),
                         'ext': 'mp4',
-                        'protocol': 'm3u8',
+                        'protocol': 'm3u8_native',
                     })
 
                 formats.append(a_format)
@@ -395,7 +397,7 @@ class BrightcoveLegacyIE(InfoExtractor):
                     return ad_info
 
         if 'url' not in info and not info.get('formats'):
-            raise ExtractorError('Unable to extract video url for %s' % info['id'])
+            raise ExtractorError('Unable to extract video url for %s' % video_id)
         return info
 
 
@@ -527,7 +529,7 @@ class BrightcoveNewIE(InfoExtractor):
                 if not src:
                     continue
                 formats.extend(self._extract_m3u8_formats(
-                    src, video_id, 'mp4', m3u8_id='hls', fatal=False))
+                    src, video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
             elif source_type == 'application/dash+xml':
                 if not src:
                     continue

From cc1028aa6d27aeec39617d1ff8d2edcf1ee989d7 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 13 May 2016 18:11:08 +0800
Subject: [PATCH 4924/6219] [openload] Fix extraction (closes #9472)

---
 youtube_dl/extractor/openload.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py
index 456561bcc..5049b870e 100644
--- a/youtube_dl/extractor/openload.py
+++ b/youtube_dl/extractor/openload.py
@@ -100,7 +100,7 @@ class OpenloadIE(InfoExtractor):
             raise ExtractorError('File not found', expected=True)
 
         code = self._search_regex(
-            r'<video[^>]+>\s*<script[^>]+>([^<]+)</script>',
+            r'</video>\s*</div>\s*<script[^>]+>([^<]+)</script>',
             webpage, 'JS code')
 
         decoded = self.openload_decode(code)

From f196508f7b872963d13bcff94c0105d743322f71 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 13 May 2016 22:19:00 +0600
Subject: [PATCH 4925/6219] [imdb] Relax _VALID_URL (Closes #9481)

---
 youtube_dl/extractor/imdb.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/imdb.py b/youtube_dl/extractor/imdb.py
index 8bed8ccd0..203156229 100644
--- a/youtube_dl/extractor/imdb.py
+++ b/youtube_dl/extractor/imdb.py
@@ -12,9 +12,9 @@ from ..utils import (
 class ImdbIE(InfoExtractor):
     IE_NAME = 'imdb'
     IE_DESC = 'Internet Movie Database trailers'
-    _VALID_URL = r'https?://(?:www|m)\.imdb\.com/video/imdb/vi(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www|m)\.imdb\.com/video/[^/]+/vi(?P<id>\d+)'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.imdb.com/video/imdb/vi2524815897',
         'info_dict': {
             'id': '2524815897',
@@ -22,7 +22,10 @@ class ImdbIE(InfoExtractor):
             'title': 'Ice Age: Continental Drift Trailer (No. 2) - IMDb',
             'description': 'md5:9061c2219254e5d14e03c25c98e96a81',
         }
-    }
+    }, {
+        'url': 'http://www.imdb.com/video/_/vi2524815897',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From 96c2e3e909171d103beafd1fd88e9d6e215681c6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 13 May 2016 23:25:05 +0600
Subject: [PATCH 4926/6219] [imdb] Improve extraction

---
 youtube_dl/extractor/imdb.py | 28 +++++++++++++++++++++-------
 1 file changed, 21 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/imdb.py b/youtube_dl/extractor/imdb.py
index 203156229..3a2b7cec5 100644
--- a/youtube_dl/extractor/imdb.py
+++ b/youtube_dl/extractor/imdb.py
@@ -1,10 +1,10 @@
 from __future__ import unicode_literals
 
 import re
-import json
 
 from .common import InfoExtractor
 from ..utils import (
+    mimetype2ext,
     qualities,
 )
 
@@ -51,13 +51,27 @@ class ImdbIE(InfoExtractor):
             json_data = self._search_regex(
                 r'<script[^>]+class="imdb-player-data"[^>]*?>(.*?)</script>',
                 format_page, 'json data', flags=re.DOTALL)
-            info = json.loads(json_data)
-            format_info = info['videoPlayerObject']['video']
-            f_id = format_info['ffname']
+            info = self._parse_json(json_data, video_id, fatal=False)
+            if not info:
+                continue
+            format_info = info.get('videoPlayerObject', {}).get('video', {})
+            if not format_info:
+                continue
+            video_info_list = format_info.get('videoInfoList')
+            if not video_info_list or not isinstance(video_info_list, list):
+                continue
+            video_info = video_info_list[0]
+            if not video_info or not isinstance(video_info, dict):
+                continue
+            video_url = video_info.get('videoUrl')
+            if not video_url:
+                continue
+            format_id = format_info.get('ffname')
             formats.append({
-                'format_id': f_id,
-                'url': format_info['videoInfoList'][0]['videoUrl'],
-                'quality': quality(f_id),
+                'format_id': format_id,
+                'url': video_url,
+                'ext': mimetype2ext(video_info.get('videoMimeType')),
+                'quality': quality(format_id),
             })
         self._sort_formats(formats)
 

From 0730be9022b415738e917c4cf72c2347ff0008e0 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 13 May 2016 20:24:36 +0100
Subject: [PATCH 4927/6219] [sina] fix extraction(fixes #1146)

---
 youtube_dl/extractor/sina.py | 124 ++++++++++++++++++++++++-----------
 1 file changed, 84 insertions(+), 40 deletions(-)

diff --git a/youtube_dl/extractor/sina.py b/youtube_dl/extractor/sina.py
index d03f1b1d4..8fc66732a 100644
--- a/youtube_dl/extractor/sina.py
+++ b/youtube_dl/extractor/sina.py
@@ -4,28 +4,35 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse_urlencode
-from ..utils import sanitized_Request
+from ..utils import (
+    HEADRequest,
+    ExtractorError,
+    int_or_none,
+    update_url_query,
+    qualities,
+    get_element_by_attribute,
+    clean_html,
+)
 
 
 class SinaIE(InfoExtractor):
-    _VALID_URL = r'''(?x)https?://(.*?\.)?video\.sina\.com\.cn/
-                        (
-                            (.+?/(((?P<pseudo_id>\d+).html)|(.*?(\#|(vid=)|b/)(?P<id>\d+?)($|&|\-))))
-                            |
+    _VALID_URL = r'''(?x)https?://(?:.*?\.)?video\.sina\.com\.cn/
+                        (?:
+                            (?:view/|.*\#)(?P<video_id>\d+)|
+                            .+?/(?P<pseudo_id>[^/?#]+)(?:\.s?html)|
                             # This is used by external sites like Weibo
-                            (api/sinawebApi/outplay.php/(?P<token>.+?)\.swf)
+                            api/sinawebApi/outplay.php/(?P<token>.+?)\.swf
                         )
                   '''
 
     _TESTS = [
         {
-            'url': 'http://video.sina.com.cn/news/vlist/zt/chczlj2013/?opsubject_id=top12#110028898',
-            'md5': 'd65dd22ddcf44e38ce2bf58a10c3e71f',
+            'url': 'http://video.sina.com.cn/news/spj/topvideoes20160504/?opsubject_id=top1#250576622',
+            'md5': 'd38433e2fc886007729735650ae4b3e9',
             'info_dict': {
-                'id': '110028898',
-                'ext': 'flv',
-                'title': '《中国新闻》 朝鲜要求巴拿马立即释放被扣船员',
+                'id': '250576622',
+                'ext': 'mp4',
+                'title': '现场:克鲁兹宣布退选 特朗普将稳获提名',
             }
         },
         {
@@ -35,37 +42,74 @@ class SinaIE(InfoExtractor):
                 'ext': 'flv',
                 'title': '军方提高对朝情报监视级别',
             },
+            'skip': 'the page does not exist or has been deleted',
+        },
+        {
+            'url': 'http://video.sina.com.cn/view/250587748.html',
+            'md5': '3d1807a25c775092aab3bc157fff49b4',
+            'info_dict': {
+                'id': '250587748',
+                'ext': 'mp4',
+                'title': '瞬间泪目：8年前汶川地震珍贵视频首曝光',
+            },
         },
     ]
 
-    def _extract_video(self, video_id):
-        data = compat_urllib_parse_urlencode({'vid': video_id})
-        url_doc = self._download_xml('http://v.iask.com/v_play.php?%s' % data,
-                                     video_id, 'Downloading video url')
-        image_page = self._download_webpage(
-            'http://interface.video.sina.com.cn/interface/common/getVideoImage.php?%s' % data,
-            video_id, 'Downloading thumbnail info')
-
-        return {'id': video_id,
-                'url': url_doc.find('./durl/url').text,
-                'ext': 'flv',
-                'title': url_doc.find('./vname').text,
-                'thumbnail': image_page.split('=')[1],
-                }
-
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        if mobj.group('token') is not None:
-            # The video id is in the redirected url
-            self.to_screen('Getting video id')
-            request = sanitized_Request(url)
-            request.get_method = lambda: 'HEAD'
-            (_, urlh) = self._download_webpage_handle(request, 'NA', False)
-            return self._real_extract(urlh.geturl())
-        elif video_id is None:
-            pseudo_id = mobj.group('pseudo_id')
-            webpage = self._download_webpage(url, pseudo_id)
-            video_id = self._search_regex(r'vid:\'(\d+?)\'', webpage, 'video id')
 
-        return self._extract_video(video_id)
+        video_id = mobj.group('video_id')
+        if not video_id:
+            if mobj.group('token') is not None:
+                # The video id is in the redirected url
+                self.to_screen('Getting video id')
+                request = HEADRequest(url)
+                (_, urlh) = self._download_webpage_handle(request, 'NA', False)
+                return self._real_extract(urlh.geturl())
+            else:
+                pseudo_id = mobj.group('pseudo_id')
+                webpage = self._download_webpage(url, pseudo_id)
+                error = get_element_by_attribute('class', 'errtitle', webpage)
+                if error:
+                    raise ExtractorError('%s said: %s' % (
+                        self.IE_NAME, clean_html(error)), expected=True)
+                video_id = self._search_regex(
+                    r"video_id\s*:\s*'(\d+)'", webpage, 'video id')
+
+        video_data = self._download_json(
+            'http://s.video.sina.com.cn/video/h5play',
+            video_id, query={'video_id': video_id})
+        if video_data['code'] != 1:
+            raise ExtractorError('%s said: %s' % (
+                self.IE_NAME, video_data['message']), expected=True)
+        else:
+            video_data = video_data['data']
+            title = video_data['title']
+            description = video_data.get('description')
+            if description:
+                description = description.strip()
+
+            preference = qualities(['cif', 'sd', 'hd', 'fhd', 'ffd'])
+            formats = []
+            for quality_id, quality in video_data.get('videos', {}).get('mp4', {}).items():
+                file_api = quality.get('file_api')
+                file_id = quality.get('file_id')
+                if not file_api or not file_id:
+                    continue
+                formats.append({
+                    'format_id': quality_id,
+                    'url': update_url_query(file_api, {'vid': file_id}),
+                    'preference': preference(quality_id),
+                    'ext': 'mp4',
+                })
+            self._sort_formats(formats)
+
+            return {
+                'id': video_id,
+                'title': title,
+                'description': description,
+                'thumbnail': video_data.get('image'),
+                'duration': int_or_none(video_data.get('length')),
+                'timestamp': int_or_none(video_data.get('create_time')),
+                'formats': formats,
+            }

From 134c6ea856be472f253bffbe99b88546fe417806 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 May 2016 04:46:38 +0600
Subject: [PATCH 4928/6219] [YoutubeDL] Sanitize url for url and
 url_transparent extraction results

---
 youtube_dl/YoutubeDL.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 34eeb77c5..03a6a1890 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -720,6 +720,7 @@ class YoutubeDL(object):
         result_type = ie_result.get('_type', 'video')
 
         if result_type in ('url', 'url_transparent'):
+            ie_result['url'] = sanitize_url(ie_result['url'])
             extract_flat = self.params.get('extract_flat', False)
             if ((extract_flat == 'in_playlist' and 'playlist' in extra_info) or
                     extract_flat is True):

From b5abf8614898cc728488d7ecc7a55a4c5c92758f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 May 2016 04:53:14 +0600
Subject: [PATCH 4929/6219] [cinemassacre] Remove extractor (Closes #9457)

It now uses jwplatform
---
 youtube_dl/extractor/cinemassacre.py | 119 ---------------------------
 youtube_dl/extractor/extractors.py   |   1 -
 2 files changed, 120 deletions(-)
 delete mode 100644 youtube_dl/extractor/cinemassacre.py

diff --git a/youtube_dl/extractor/cinemassacre.py b/youtube_dl/extractor/cinemassacre.py
deleted file mode 100644
index 042c4f2f1..000000000
--- a/youtube_dl/extractor/cinemassacre.py
+++ /dev/null
@@ -1,119 +0,0 @@
-# encoding: utf-8
-from __future__ import unicode_literals
-
-import re
-
-from .common import InfoExtractor
-from ..utils import ExtractorError
-from .screenwavemedia import ScreenwaveMediaIE
-
-
-class CinemassacreIE(InfoExtractor):
-    _VALID_URL = 'https?://(?:www\.)?cinemassacre\.com/(?P<date_y>[0-9]{4})/(?P<date_m>[0-9]{2})/(?P<date_d>[0-9]{2})/(?P<display_id>[^?#/]+)'
-    _TESTS = [
-        {
-            'url': 'http://cinemassacre.com/2012/11/10/avgn-the-movie-trailer/',
-            'md5': 'fde81fbafaee331785f58cd6c0d46190',
-            'info_dict': {
-                'id': 'Cinemassacre-19911',
-                'ext': 'mp4',
-                'upload_date': '20121110',
-                'title': '“Angry Video Game Nerd: The Movie” – Trailer',
-                'description': 'md5:fb87405fcb42a331742a0dce2708560b',
-            },
-            'params': {
-                # m3u8 download
-                'skip_download': True,
-            },
-        },
-        {
-            'url': 'http://cinemassacre.com/2013/10/02/the-mummys-hand-1940',
-            'md5': 'd72f10cd39eac4215048f62ab477a511',
-            'info_dict': {
-                'id': 'Cinemassacre-521be8ef82b16',
-                'ext': 'mp4',
-                'upload_date': '20131002',
-                'title': 'The Mummy’s Hand (1940)',
-            },
-            'params': {
-                # m3u8 download
-                'skip_download': True,
-            },
-        },
-        {
-            # Youtube embedded video
-            'url': 'http://cinemassacre.com/2006/12/07/chronologically-confused-about-bad-movie-and-video-game-sequel-titles/',
-            'md5': 'ec9838a5520ef5409b3e4e42fcb0a3b9',
-            'info_dict': {
-                'id': 'OEVzPCY2T-g',
-                'ext': 'webm',
-                'title': 'AVGN: Chronologically Confused about Bad Movie and Video Game Sequel Titles',
-                'upload_date': '20061207',
-                'uploader': 'Cinemassacre',
-                'uploader_id': 'JamesNintendoNerd',
-                'description': 'md5:784734696c2b8b7f4b8625cc799e07f6',
-            }
-        },
-        {
-            # Youtube embedded video
-            'url': 'http://cinemassacre.com/2006/09/01/mckids/',
-            'md5': '7393c4e0f54602ad110c793eb7a6513a',
-            'info_dict': {
-                'id': 'FnxsNhuikpo',
-                'ext': 'webm',
-                'upload_date': '20060901',
-                'uploader': 'Cinemassacre Extra',
-                'description': 'md5:de9b751efa9e45fbaafd9c8a1123ed53',
-                'uploader_id': 'Cinemassacre',
-                'title': 'AVGN: McKids',
-            }
-        },
-        {
-            'url': 'http://cinemassacre.com/2015/05/25/mario-kart-64-nintendo-64-james-mike-mondays/',
-            'md5': '1376908e49572389e7b06251a53cdd08',
-            'info_dict': {
-                'id': 'Cinemassacre-555779690c440',
-                'ext': 'mp4',
-                'description': 'Let’s Play Mario Kart 64 !! Mario Kart 64 is a classic go-kart racing game released for the Nintendo 64 (N64). Today James & Mike do 4 player Battle Mode with Kyle and Bootsy!',
-                'title': 'Mario Kart 64 (Nintendo 64) James & Mike Mondays',
-                'upload_date': '20150525',
-            },
-            'params': {
-                # m3u8 download
-                'skip_download': True,
-            },
-        }
-    ]
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        display_id = mobj.group('display_id')
-        video_date = mobj.group('date_y') + mobj.group('date_m') + mobj.group('date_d')
-
-        webpage = self._download_webpage(url, display_id)
-
-        playerdata_url = self._search_regex(
-            [
-                ScreenwaveMediaIE.EMBED_PATTERN,
-                r'<iframe[^>]+src="(?P<url>(?:https?:)?//(?:[^.]+\.)?youtube\.com/.+?)"',
-            ],
-            webpage, 'player data URL', default=None, group='url')
-        if not playerdata_url:
-            raise ExtractorError('Unable to find player data')
-
-        video_title = self._html_search_regex(
-            r'<title>(?P<title>.+?)\|', webpage, 'title')
-        video_description = self._html_search_regex(
-            r'<div class="entry-content">(?P<description>.+?)</div>',
-            webpage, 'description', flags=re.DOTALL, fatal=False)
-        video_thumbnail = self._og_search_thumbnail(webpage)
-
-        return {
-            '_type': 'url_transparent',
-            'display_id': display_id,
-            'title': video_title,
-            'description': video_description,
-            'upload_date': video_date,
-            'thumbnail': video_thumbnail,
-            'url': playerdata_url,
-        }
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 50d2204f2..b6f4ccc5d 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -124,7 +124,6 @@ from .chirbit import (
     ChirbitProfileIE,
 )
 from .cinchcast import CinchcastIE
-from .cinemassacre import CinemassacreIE
 from .cliprs import ClipRsIE
 from .clipfish import ClipfishIE
 from .cliphunter import CliphunterIE

From 98d560f205e6aeddc767844d142b00525a9eaff9 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 14 May 2016 18:48:36 +0800
Subject: [PATCH 4930/6219] [test/test_socks] Skip SOCKS tests

They occasional trigger errors or blocks
(https://travis-ci.org/rg3/youtube-dl/jobs/130184883)
---
 test/test_socks.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/test/test_socks.py b/test/test_socks.py
index d07003ceb..1e68eb0da 100644
--- a/test/test_socks.py
+++ b/test/test_socks.py
@@ -77,17 +77,28 @@ class TestMultipleSocks(unittest.TestCase):
 
 
 class TestSocks(unittest.TestCase):
+    _SKIP_SOCKS_TEST = True
+
     def setUp(self):
+        if self._SKIP_SOCKS_TEST:
+            return
+
         self.port = random.randint(20000, 30000)
         self.server_process = subprocess.Popen([
             'srelay', '-f', '-i', '127.0.0.1:%d' % self.port],
             stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 
     def tearDown(self):
+        if self._SKIP_SOCKS_TEST:
+            return
+
         self.server_process.terminate()
         self.server_process.communicate()
 
     def _get_ip(self, protocol):
+        if self._SKIP_SOCKS_TEST:
+            return '127.0.0.1'
+
         ydl = FakeYDL({
             'proxy': '%s://127.0.0.1:%d' % (protocol, self.port),
         })

From 791ff52f753ee123426766aaa5320eb63a874b7b Mon Sep 17 00:00:00 2001
From: Jakub Wilk <jwilk@jwilk.net>
Date: Sat, 14 May 2016 13:19:54 +0200
Subject: [PATCH 4931/6219] [teamcoco] Fix base64 regexp

---
 youtube_dl/extractor/teamcoco.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/teamcoco.py b/youtube_dl/extractor/teamcoco.py
index b49ab5f5b..79a778920 100644
--- a/youtube_dl/extractor/teamcoco.py
+++ b/youtube_dl/extractor/teamcoco.py
@@ -88,7 +88,7 @@ class TeamcocoIE(InfoExtractor):
         preload_codes = self._html_search_regex(
             r'(function.+)setTimeout\(function\(\)\{playlist',
             webpage, 'preload codes')
-        base64_fragments = re.findall(r'"([a-zA-z0-9+/=]+)"', preload_codes)
+        base64_fragments = re.findall(r'"([a-zA-Z0-9+/=]+)"', preload_codes)
         base64_fragments.remove('init')
 
         def _check_sequence(cur_fragments):

From 66e7ace17a36ed0f761ae620801e9e27d5c3cb3f Mon Sep 17 00:00:00 2001
From: Jakub Wilk <jwilk@jwilk.net>
Date: Sat, 14 May 2016 13:41:41 +0200
Subject: [PATCH 4932/6219] Don't hardcode errno constant

The value of ENOENT is architecture-dependent, so don't assume it's
always 2.
---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 03a6a1890..3917ca9dc 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -326,7 +326,7 @@ class YoutubeDL(object):
                         ['fribidi', '-c', 'UTF-8'] + width_args, **sp_kwargs)
                 self._output_channel = os.fdopen(master, 'rb')
             except OSError as ose:
-                if ose.errno == 2:
+                if ose.errno == errno.ENOENT:
                     self.report_warning('Could not find fribidi executable, ignoring --bidi-workaround . Make sure that  fribidi  is an executable file in one of the directories in your $PATH.')
                 else:
                     raise

From bd1e484448c84904ce0d99fe05c3721053aa3c00 Mon Sep 17 00:00:00 2001
From: felix <felix.von.s@posteo.de>
Date: Sun, 13 Mar 2016 12:29:15 +0100
Subject: [PATCH 4933/6219] [utils] js_to_json: various improvements

now JS object literals like { /* " */ 0: ",]\xaa<\/p>", } will be correctly converted to JSON.
---
 test/test_utils.py  | 12 ++++++++++++
 youtube_dl/utils.py | 30 ++++++++++++++++--------------
 2 files changed, 28 insertions(+), 14 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index ca254779f..ab2842f3b 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -640,6 +640,18 @@ class TestUtil(unittest.TestCase):
         on = js_to_json('{"abc": "def",}')
         self.assertEqual(json.loads(on), {'abc': 'def'})
 
+        on = js_to_json('{ 0: /* " \n */ ",]" , }')
+        self.assertEqual(json.loads(on), {'0': ',]'})
+
+        on = js_to_json(r'["<p>x<\/p>"]')
+        self.assertEqual(json.loads(on), ['<p>x</p>'])
+
+        on = js_to_json(r'["\xaa"]')
+        self.assertEqual(json.loads(on), ['\u00aa'])
+
+        on = js_to_json("['a\\\nb']")
+        self.assertEqual(json.loads(on), ['ab'])
+
     def test_extract_attributes(self):
         self.assertEqual(extract_attributes('<e x="y">'), {'x': 'y'})
         self.assertEqual(extract_attributes("<e x='y'>"), {'x': 'y'})
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index d6f94f8cd..52a20632f 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1914,24 +1914,26 @@ def js_to_json(code):
         v = m.group(0)
         if v in ('true', 'false', 'null'):
             return v
-        if v.startswith('"'):
-            v = re.sub(r"\\'", "'", v[1:-1])
-        elif v.startswith("'"):
-            v = v[1:-1]
-            v = re.sub(r"\\\\|\\'|\"", lambda m: {
-                '\\\\': '\\\\',
-                "\\'": "'",
+        elif v.startswith('/*') or v == ',':
+            return ""
+
+        if v[0] in ("'", '"'):
+            v = re.sub(r'(?s)\\.|"', lambda m: {
                 '"': '\\"',
-            }[m.group(0)], v)
+                "\\'": "'",
+                '\\\n': '',
+                '\\x': '\\u00',
+            }.get(m.group(0), m.group(0)), v[1:-1])
+
         return '"%s"' % v
 
-    res = re.sub(r'''(?x)
-        "(?:[^"\\]*(?:\\\\|\\['"nu]))*[^"\\]*"|
-        '(?:[^'\\]*(?:\\\\|\\['"nu]))*[^'\\]*'|
-        [a-zA-Z_][.a-zA-Z_0-9]*
+    return re.sub(r'''(?sx)
+        "(?:[^"\\]*(?:\\\\|\\['"nurtbfx/\n]))*[^"\\]*"|
+        '(?:[^'\\]*(?:\\\\|\\['"nurtbfx/\n]))*[^'\\]*'|
+        /\*.*?\*/|,(?=\s*[\]}])|
+        [a-zA-Z_][.a-zA-Z_0-9]*|
+        [0-9]+(?=\s*:)
         ''', fix_kv, code)
-    res = re.sub(r',(\s*[\]}])', lambda m: m.group(1), res)
-    return res
 
 
 def qualities(quality_ids):

From 640eea0a0cf7ae589126f7762e1cfc7bdd2250d9 Mon Sep 17 00:00:00 2001
From: felix <felix.von.s@posteo.de>
Date: Sun, 20 Mar 2016 12:17:57 +0100
Subject: [PATCH 4934/6219] [ora] minimise fragile regex shenanigans; recognise
 unsafespeech.com URLs

---
 youtube_dl/extractor/ora.py | 20 +++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/ora.py b/youtube_dl/extractor/ora.py
index 8545fb1b8..cfae71bcc 100644
--- a/youtube_dl/extractor/ora.py
+++ b/youtube_dl/extractor/ora.py
@@ -6,13 +6,14 @@ from .common import InfoExtractor
 from ..compat import compat_urlparse
 from ..utils import (
     get_element_by_attribute,
+    js_to_json,
     qualities,
     unescapeHTML,
 )
 
 
 class OraTVIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?ora\.tv/([^/]+/)*(?P<id>[^/\?#]+)'
+    _VALID_URL = r'https?://(?:www\.)?(ora\.tv|unsafespeech\.com)/([^/]+/)*(?P<id>[^/\?#]+)'
     _TEST = {
         'url': 'https://www.ora.tv/larrykingnow/2015/12/16/vine-youtube-stars-zach-king-king-bach-on-their-viral-videos-0_36jupg6090pq',
         'md5': 'fa33717591c631ec93b04b0e330df786',
@@ -28,10 +29,13 @@ class OraTVIE(InfoExtractor):
         display_id = self._match_id(url)
         webpage = self._download_webpage(url, display_id)
 
-        video_data = self._search_regex(
-            r'"(?:video|current)"\s*:\s*({[^}]+?})', webpage, 'current video')
-        m3u8_url = self._search_regex(
-            r'hls_stream"?\s*:\s*"([^"]+)', video_data, 'm3u8 url', None)
+        ora_meta = self._parse_json(self._search_regex(
+            r'(?s);\s*ora_meta = ({.*?});</script>', webpage, 'ora_meta'), display_id,
+            transform_source=lambda data: js_to_json(re.sub('":(document|\().*?(:false|\(\)),', '":null,', data)))
+
+        video_data = ora_meta.get('video', ora_meta.get('current'))
+        m3u8_url = video_data['hls_stream']
+
         if m3u8_url:
             formats = self._extract_m3u8_formats(
                 m3u8_url, display_id, 'mp4', 'm3u8_native',
@@ -60,13 +64,11 @@ class OraTVIE(InfoExtractor):
                 r'"youtube_id"\s*:\s*"([^"]+)', webpage, 'youtube id'), 'Youtube')
 
         return {
-            'id': self._search_regex(
-                r'"id"\s*:\s*(\d+)', video_data, 'video id', default=display_id),
+            'id': video_data.get('id', display_id),
             'display_id': display_id,
             'title': unescapeHTML(self._og_search_title(webpage)),
             'description': get_element_by_attribute(
                 'class', 'video_txt_decription', webpage),
-            'thumbnail': self._proto_relative_url(self._search_regex(
-                r'"thumb"\s*:\s*"([^"]+)', video_data, 'thumbnail', None)),
+            'thumbnail': self._proto_relative_url(video_data.get('thumb')),
             'formats': formats,
         }

From 89ac4a19e658203db85c6a1d4b267a2eeb47a38e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 May 2016 20:39:58 +0600
Subject: [PATCH 4935/6219] [utils] Process non-base 10 integers in js_to_json

---
 test/test_utils.py  | 19 +++++++++++++++++++
 youtube_dl/utils.py | 12 ++++++++++++
 2 files changed, 31 insertions(+)

diff --git a/test/test_utils.py b/test/test_utils.py
index ab2842f3b..26f66bff6 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -617,6 +617,15 @@ class TestUtil(unittest.TestCase):
         json_code = js_to_json(inp)
         self.assertEqual(json.loads(json_code), json.loads(inp))
 
+        inp = '''{
+            0:{src:'skipped', type: 'application/dash+xml'},
+            1:{src:'skipped', type: 'application/vnd.apple.mpegURL'},
+        }'''
+        self.assertEqual(js_to_json(inp), '''{
+            "0":{"src":"skipped", "type": "application/dash+xml"},
+            "1":{"src":"skipped", "type": "application/vnd.apple.mpegURL"}
+        }''')
+
     def test_js_to_json_edgecases(self):
         on = js_to_json("{abc_def:'1\\'\\\\2\\\\\\'3\"4'}")
         self.assertEqual(json.loads(on), {"abc_def": "1'\\2\\'3\"4"})
@@ -652,6 +661,16 @@ class TestUtil(unittest.TestCase):
         on = js_to_json("['a\\\nb']")
         self.assertEqual(json.loads(on), ['ab'])
 
+        on = js_to_json('{0xff:0xff}')
+        self.assertEqual(json.loads(on), {'255': 255})
+
+        on = js_to_json('{077:077}')
+        self.assertEqual(json.loads(on), {'63': 63})
+
+        on = js_to_json('{42:42}')
+        self.assertEqual(json.loads(on), {'42': 42})
+
+
     def test_extract_attributes(self):
         self.assertEqual(extract_attributes('<e x="y">'), {'x': 'y'})
         self.assertEqual(extract_attributes("<e x='y'>"), {'x': 'y'})
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 52a20632f..25a9f33c0 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1925,6 +1925,17 @@ def js_to_json(code):
                 '\\x': '\\u00',
             }.get(m.group(0), m.group(0)), v[1:-1])
 
+        INTEGER_TABLE = (
+            (r'^(0[xX][0-9a-fA-F]+)', 16),
+            (r'^(0+[0-7]+)', 8),
+        )
+
+        for regex, base in INTEGER_TABLE:
+            im = re.match(regex, v)
+            if im:
+                i = int(im.group(1), base)
+                return '"%d":' % i if v.endswith(':') else '%d' % i
+
         return '"%s"' % v
 
     return re.sub(r'''(?sx)
@@ -1932,6 +1943,7 @@ def js_to_json(code):
         '(?:[^'\\]*(?:\\\\|\\['"nurtbfx/\n]))*[^'\\]*'|
         /\*.*?\*/|,(?=\s*[\]}])|
         [a-zA-Z_][.a-zA-Z_0-9]*|
+        (?:0[xX][0-9a-fA-F]+|0+[0-7]+)(?:\s*:)?|
         [0-9]+(?=\s*:)
         ''', fix_kv, code)
 

From ca950f49e909baf6672034ffc2c1c2ee7133cf23 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 May 2016 20:45:18 +0600
Subject: [PATCH 4936/6219] [ora] Revert extraction to regexes

It's less fragile than using js_to_json with ora js
---
 youtube_dl/extractor/ora.py | 27 ++++++++++++++-------------
 1 file changed, 14 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/ora.py b/youtube_dl/extractor/ora.py
index cfae71bcc..1d42be39b 100644
--- a/youtube_dl/extractor/ora.py
+++ b/youtube_dl/extractor/ora.py
@@ -6,15 +6,14 @@ from .common import InfoExtractor
 from ..compat import compat_urlparse
 from ..utils import (
     get_element_by_attribute,
-    js_to_json,
     qualities,
     unescapeHTML,
 )
 
 
 class OraTVIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?(ora\.tv|unsafespeech\.com)/([^/]+/)*(?P<id>[^/\?#]+)'
-    _TEST = {
+    _VALID_URL = r'https?://(?:www\.)?(?:ora\.tv|unsafespeech\.com)/([^/]+/)*(?P<id>[^/\?#]+)'
+    _TESTS = [{
         'url': 'https://www.ora.tv/larrykingnow/2015/12/16/vine-youtube-stars-zach-king-king-bach-on-their-viral-videos-0_36jupg6090pq',
         'md5': 'fa33717591c631ec93b04b0e330df786',
         'info_dict': {
@@ -23,19 +22,19 @@ class OraTVIE(InfoExtractor):
             'title': 'Vine & YouTube Stars Zach King & King Bach On Their Viral Videos!',
             'description': 'md5:ebbc5b1424dd5dba7be7538148287ac1',
         }
-    }
+    }, {
+        'url': 'http://www.unsafespeech.com/video/2016/5/10/student-self-censorship-and-the-thought-police-on-university-campuses-0_6622bnkppw4d',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
         webpage = self._download_webpage(url, display_id)
 
-        ora_meta = self._parse_json(self._search_regex(
-            r'(?s);\s*ora_meta = ({.*?});</script>', webpage, 'ora_meta'), display_id,
-            transform_source=lambda data: js_to_json(re.sub('":(document|\().*?(:false|\(\)),', '":null,', data)))
-
-        video_data = ora_meta.get('video', ora_meta.get('current'))
-        m3u8_url = video_data['hls_stream']
-
+        video_data = self._search_regex(
+            r'"(?:video|current)"\s*:\s*({[^}]+?})', webpage, 'current video')
+        m3u8_url = self._search_regex(
+            r'hls_stream"?\s*:\s*"([^"]+)', video_data, 'm3u8 url', None)
         if m3u8_url:
             formats = self._extract_m3u8_formats(
                 m3u8_url, display_id, 'mp4', 'm3u8_native',
@@ -64,11 +63,13 @@ class OraTVIE(InfoExtractor):
                 r'"youtube_id"\s*:\s*"([^"]+)', webpage, 'youtube id'), 'Youtube')
 
         return {
-            'id': video_data.get('id', display_id),
+            'id': self._search_regex(
+                r'"id"\s*:\s*(\d+)', video_data, 'video id', default=display_id),
             'display_id': display_id,
             'title': unescapeHTML(self._og_search_title(webpage)),
             'description': get_element_by_attribute(
                 'class', 'video_txt_decription', webpage),
-            'thumbnail': self._proto_relative_url(video_data.get('thumb')),
+            'thumbnail': self._proto_relative_url(self._search_regex(
+                r'"thumb"\s*:\s*"([^"]+)', video_data, 'thumbnail', None)),
             'formats': formats,
         }

From 364cf465dd53e8006f5523c348f127f8df657bc3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 May 2016 20:46:33 +0600
Subject: [PATCH 4937/6219] [test_utils] PEP 8

---
 test/test_utils.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 26f66bff6..520d32ff5 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -670,7 +670,6 @@ class TestUtil(unittest.TestCase):
         on = js_to_json('{42:42}')
         self.assertEqual(json.loads(on), {'42': 42})
 
-
     def test_extract_attributes(self):
         self.assertEqual(extract_attributes('<e x="y">'), {'x': 'y'})
         self.assertEqual(extract_attributes("<e x='y'>"), {'x': 'y'})

From 5c86bfe70ff0048e59c6e890af14a055522fd3fd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 May 2016 23:35:03 +0600
Subject: [PATCH 4938/6219] [3qsdn] Add extractor

---
 youtube_dl/extractor/extractors.py |   1 +
 youtube_dl/extractor/threeqsdn.py  | 132 +++++++++++++++++++++++++++++
 2 files changed, 133 insertions(+)
 create mode 100644 youtube_dl/extractor/threeqsdn.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index b6f4ccc5d..2db3b3c3f 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -766,6 +766,7 @@ from .thesixtyone import TheSixtyOneIE
 from .thestar import TheStarIE
 from .thisamericanlife import ThisAmericanLifeIE
 from .thisav import ThisAVIE
+from .threeqsdn import ThreeQSDNIE
 from .tinypic import TinyPicIE
 from .tlc import TlcDeIE
 from .tmz import (
diff --git a/youtube_dl/extractor/threeqsdn.py b/youtube_dl/extractor/threeqsdn.py
new file mode 100644
index 000000000..27a3de5c4
--- /dev/null
+++ b/youtube_dl/extractor/threeqsdn.py
@@ -0,0 +1,132 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    determine_ext,
+    js_to_json,
+    mimetype2ext,
+)
+
+
+class ThreeQSDNIE(InfoExtractor):
+    IE_NAME = '3qsdn'
+    IE_DESC = '3Q SDN'
+    _VALID_URL = r'https?://playout\.3qsdn\.com/(?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})'
+    _TESTS = [{
+        # ondemand from http://www.philharmonie.tv/veranstaltung/26/
+        'url': 'http://playout.3qsdn.com/0280d6b9-1215-11e6-b427-0cc47a188158?protocol=http',
+        'md5': 'ab040e37bcfa2e0c079f92cb1dd7f6cd',
+        'info_dict': {
+            'id': '0280d6b9-1215-11e6-b427-0cc47a188158',
+            'ext': 'mp4',
+            'title': '0280d6b9-1215-11e6-b427-0cc47a188158',
+            'is_live': False,
+        },
+        'expected_warnings': ['Failed to download MPD manifest'],
+    }, {
+        # live video stream
+        'url': 'https://playout.3qsdn.com/d755d94b-4ab9-11e3-9162-0025907ad44f?js=true',
+        'info_dict': {
+            'id': 'd755d94b-4ab9-11e3-9162-0025907ad44f',
+            'ext': 'mp4',
+            'title': 'd755d94b-4ab9-11e3-9162-0025907ad44f',
+            'is_live': False,
+        },
+    }, {
+        # live audio stream
+        'url': 'http://playout.3qsdn.com/9edf36e0-6bf2-11e2-a16a-9acf09e2db48',
+        'only_matching': True,
+    }, {
+        # live audio stream with some 404 URLs
+        'url': 'http://playout.3qsdn.com/ac5c3186-777a-11e2-9c30-9acf09e2db48',
+        'only_matching': True,
+    }, {
+        # geo restricted with 'This content is not available in your country'
+        'url': 'http://playout.3qsdn.com/d63a3ffe-75e8-11e2-9c30-9acf09e2db48',
+        'only_matching': True,
+    }, {
+        # geo restricted with 'playout.3qsdn.com/forbidden'
+        'url': 'http://playout.3qsdn.com/8e330f26-6ae2-11e2-a16a-9acf09e2db48',
+        'only_matching': True,
+    }, {
+        # live video with rtmp link
+        'url': 'https://playout.3qsdn.com/6092bb9e-8f72-11e4-a173-002590c750be',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        js = self._download_webpage(
+            'http://playout.3qsdn.com/%s' % video_id, video_id,
+            query={'js': 'true'})
+
+        if any(p in js for p in (
+                '>This content is not available in your country',
+                'playout.3qsdn.com/forbidden')):
+            self.raise_geo_restricted()
+
+        stream_content = self._search_regex(
+            r'streamContent\s*:\s*(["\'])(?P<content>.+?)\1', js,
+            'stream content', default='demand', group='content')
+
+        live = stream_content == 'live'
+
+        stream_type = self._search_regex(
+            r'streamType\s*:\s*(["\'])(?P<type>audio|video)\1', js,
+            'stream type', default='video', group='type')
+
+        formats = []
+        urls = set()
+
+        def extract_formats(item_url, item={}):
+            if not item_url or item_url in urls:
+                return
+            urls.add(item_url)
+            type_ = item.get('type')
+            ext = determine_ext(item_url, default_ext=None)
+            if type_ == 'application/dash+xml' or ext == 'mpd':
+                formats.extend(self._extract_mpd_formats(
+                    item_url, video_id, mpd_id='mpd', fatal=False))
+            elif type_ in ('application/vnd.apple.mpegURL', 'application/x-mpegurl') or ext == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    item_url, video_id, 'mp4',
+                    entry_protocol='m3u8' if live else 'm3u8_native',
+                    m3u8_id='hls', fatal=False))
+            elif ext == 'f4m':
+                formats.extend(self._extract_f4m_formats(
+                    item_url, video_id, f4m_id='hds', fatal=False))
+            else:
+                if not self._is_valid_url(item_url, video_id):
+                    return
+                formats.append({
+                    'url': item_url,
+                    'format_id': item.get('quality'),
+                    'ext': 'mp4' if item_url.startswith('rtsp') else mimetype2ext(type_) or ext,
+                    'vcodec': 'none' if stream_type == 'audio' else None,
+                })
+
+        for item_js in re.findall(r'({.*?\b(?:src|source)\s*:\s*["\'].+?})', js):
+            f = self._parse_json(
+                item_js, video_id, transform_source=js_to_json, fatal=False)
+            if not f:
+                continue
+            extract_formats(f.get('src'), f)
+
+        # More relaxed version to collect additional URLs and acting
+        # as a future-proof fallback
+        for _, src in re.findall(r'\b(?:src|source)\s*:\s*(["\'])((?:https?|rtsp)://.+?)\1', js):
+            extract_formats(src)
+
+        self._sort_formats(formats)
+
+        title = self._live_title(video_id) if live else video_id
+
+        return {
+            'id': video_id,
+            'title': title,
+            'is_live': live,
+            'formats': formats,
+        }

From 5d39176f6de8bab1e019ead7cd497659f3fc1a94 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 May 2016 23:40:34 +0600
Subject: [PATCH 4939/6219] [extractor/generic:3qsdn] Add support for embeds

---
 youtube_dl/extractor/generic.py   | 6 ++++++
 youtube_dl/extractor/threeqsdn.py | 7 +++++++
 2 files changed, 13 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 0f1eb7fa6..b48ccfc97 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -61,6 +61,7 @@ from .jwplatform import JWPlatformIE
 from .digiteka import DigitekaIE
 from .instagram import InstagramIE
 from .liveleak import LiveLeakIE
+from .threeqsdn import ThreeQSDNIE
 
 
 class GenericIE(InfoExtractor):
@@ -1983,6 +1984,11 @@ class GenericIE(InfoExtractor):
         if liveleak_url:
             return self.url_result(liveleak_url, 'LiveLeak')
 
+        # Look for 3Q SDN embeds
+        threeqsdn_url = ThreeQSDNIE._extract_url(webpage)
+        if threeqsdn_url:
+            return self.url_result(self._proto_relative_url(threeqsdn_url), ThreeQSDNIE.ie_key())
+
         def check_video(vurl):
             if YoutubeIE.suitable(vurl):
                 return True
diff --git a/youtube_dl/extractor/threeqsdn.py b/youtube_dl/extractor/threeqsdn.py
index 27a3de5c4..c77a07989 100644
--- a/youtube_dl/extractor/threeqsdn.py
+++ b/youtube_dl/extractor/threeqsdn.py
@@ -56,6 +56,13 @@ class ThreeQSDNIE(InfoExtractor):
         'only_matching': True,
     }]
 
+    @staticmethod
+    def _extract_url(webpage):
+        mobj = re.search(
+            r'<iframe[^>]+\b(?:data-)?src=(["\'])(?P<url>%s.*?)\1' % ThreeQSDNIE._VALID_URL, webpage)
+        if mobj:
+            return mobj.group('url')
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
 

From cda6d47aad106a825f837c7a583fffc783c4b63b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 May 2016 23:41:57 +0600
Subject: [PATCH 4940/6219] [utils] Simplify integer conversion in js_to_json

---
 youtube_dl/utils.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 25a9f33c0..a637563cb 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1926,14 +1926,14 @@ def js_to_json(code):
             }.get(m.group(0), m.group(0)), v[1:-1])
 
         INTEGER_TABLE = (
-            (r'^(0[xX][0-9a-fA-F]+)', 16),
-            (r'^(0+[0-7]+)', 8),
+            (r'^0[xX][0-9a-fA-F]+', 16),
+            (r'^0+[0-7]+', 8),
         )
 
         for regex, base in INTEGER_TABLE:
             im = re.match(regex, v)
             if im:
-                i = int(im.group(1), base)
+                i = int(im.group(0), base)
                 return '"%d":' % i if v.endswith(':') else '%d' % i
 
         return '"%s"' % v

From 6f41b2bcf16899f8c3f0ea705b2914cf1ae668a3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 May 2016 23:58:25 +0600
Subject: [PATCH 4941/6219] [extractor/generic] Improve 3qsdn embeds support
 (Closes #9453)

---
 youtube_dl/extractor/generic.py | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index b48ccfc97..a6b1e23e3 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1428,7 +1428,8 @@ class GenericIE(InfoExtractor):
         #   Site Name | Video Title
         #   Video Title - Tagline | Site Name
         # and so on and so forth; it's just not practical
-        video_title = self._html_search_regex(
+        video_title = self._og_search_title(
+            webpage, default=None) or self._html_search_regex(
             r'(?s)<title>(.*?)</title>', webpage, 'video title',
             default='video')
 
@@ -1446,6 +1447,9 @@ class GenericIE(InfoExtractor):
         video_uploader = self._search_regex(
             r'^(?:https?://)?([^/]*)/.*', url, 'video uploader')
 
+        video_description = self._og_search_description(webpage, default=None)
+        video_thumbnail = self._og_search_thumbnail(webpage, default=None)
+
         # Helper method
         def _playlist_from_matches(matches, getter=None, ie=None):
             urlrs = orderedSet(
@@ -1987,7 +1991,15 @@ class GenericIE(InfoExtractor):
         # Look for 3Q SDN embeds
         threeqsdn_url = ThreeQSDNIE._extract_url(webpage)
         if threeqsdn_url:
-            return self.url_result(self._proto_relative_url(threeqsdn_url), ThreeQSDNIE.ie_key())
+            return {
+                '_type': 'url_transparent',
+                'ie_key': ThreeQSDNIE.ie_key(),
+                'url': self._proto_relative_url(threeqsdn_url),
+                'title': video_title,
+                'description': video_description,
+                'thumbnail': video_thumbnail,
+                'uploader': video_uploader,
+            }
 
         def check_video(vurl):
             if YoutubeIE.suitable(vurl):

From ed56f260399728f1975dd30f4c8ee110cf106d84 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 15 May 2016 03:34:35 +0600
Subject: [PATCH 4942/6219] [extractor/common] Improve name extraction for m3u8
 formats

---
 youtube_dl/extractor/common.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 0843d89af..8a8c07226 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1139,11 +1139,14 @@ class InfoExtractor(object):
                 if m3u8_id:
                     format_id.append(m3u8_id)
                 last_media_name = last_media.get('NAME') if last_media and last_media.get('TYPE') != 'SUBTITLES' else None
+                # Despite specification does not mention NAME attribute for
+                # EXT-X-STREAM-INF it still sometimes may be present
+                stream_name = last_info.get('NAME') or last_media_name
                 # Bandwidth of live streams may differ over time thus making
                 # format_id unpredictable. So it's better to keep provided
                 # format_id intact.
                 if not live:
-                    format_id.append(last_media_name if last_media_name else '%d' % (tbr if tbr else len(formats)))
+                    format_id.append(stream_name if stream_name else '%d' % (tbr if tbr else len(formats)))
                 f = {
                     'format_id': '-'.join(format_id),
                     'url': format_url(line.strip()),

From 69c9cc2716a4d076b023096c23b6f7646627824a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 15 May 2016 03:38:04 +0600
Subject: [PATCH 4943/6219] [xvideos] Extract html5 player formats (Closes
 #9495)

---
 youtube_dl/extractor/xvideos.py | 43 ++++++++++++++++-----------------
 1 file changed, 21 insertions(+), 22 deletions(-)

diff --git a/youtube_dl/extractor/xvideos.py b/youtube_dl/extractor/xvideos.py
index 710ad5041..1dfe031ca 100644
--- a/youtube_dl/extractor/xvideos.py
+++ b/youtube_dl/extractor/xvideos.py
@@ -8,7 +8,6 @@ from ..utils import (
     clean_html,
     ExtractorError,
     determine_ext,
-    sanitized_Request,
 )
 
 
@@ -25,8 +24,6 @@ class XVideosIE(InfoExtractor):
         }
     }
 
-    _ANDROID_USER_AGENT = 'Mozilla/5.0 (Linux; Android 4.0.4; Galaxy Nexus Build/IMM76B) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.133 Mobile Safari/535.19'
-
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
@@ -35,31 +32,34 @@ class XVideosIE(InfoExtractor):
         if mobj:
             raise ExtractorError('%s said: %s' % (self.IE_NAME, clean_html(mobj.group(1))), expected=True)
 
-        video_url = compat_urllib_parse_unquote(
-            self._search_regex(r'flv_url=(.+?)&', webpage, 'video URL'))
         video_title = self._html_search_regex(
             r'<title>(.*?)\s+-\s+XVID', webpage, 'title')
         video_thumbnail = self._search_regex(
             r'url_bigthumb=(.+?)&amp', webpage, 'thumbnail', fatal=False)
 
-        formats = [{
-            'url': video_url,
-        }]
+        formats = []
 
-        android_req = sanitized_Request(url)
-        android_req.add_header('User-Agent', self._ANDROID_USER_AGENT)
-        android_webpage = self._download_webpage(android_req, video_id, fatal=False)
+        video_url = compat_urllib_parse_unquote(self._search_regex(
+            r'flv_url=(.+?)&', webpage, 'video URL', default=''))
+        if video_url:
+            formats.append({'url': video_url})
 
-        if android_webpage is not None:
-            player_params_str = self._search_regex(
-                'mobileReplacePlayerDivTwoQual\(([^)]+)\)',
-                android_webpage, 'player parameters', default='')
-            player_params = list(map(lambda s: s.strip(' \''), player_params_str.split(',')))
-            if player_params:
-                formats.extend([{
-                    'url': param,
-                    'preference': -10,
-                } for param in player_params if determine_ext(param) == 'mp4'])
+        player_args = self._search_regex(
+            r'(?s)new\s+HTML5Player\((.+?)\)', webpage, ' html5 player', default=None)
+        if player_args:
+            for arg in player_args.split(','):
+                format_url = self._search_regex(
+                    r'(["\'])(?P<url>https?://.+?)\1', arg, 'url',
+                    default=None, group='url')
+                if not format_url:
+                    continue
+                ext = determine_ext(format_url)
+                if ext == 'mp4':
+                    formats.append({'url': format_url})
+                elif ext == 'm3u8':
+                    formats.extend(self._extract_m3u8_formats(
+                        format_url, video_id, 'mp4',
+                        entry_protocol='m3u8_native', m3u8_id='hls', fatal=False))
 
         self._sort_formats(formats)
 
@@ -67,7 +67,6 @@ class XVideosIE(InfoExtractor):
             'id': video_id,
             'formats': formats,
             'title': video_title,
-            'ext': 'flv',
             'thumbnail': video_thumbnail,
             'age_limit': 18,
         }

From 79298173c5a957456cb17b2b26338a657f1aae1e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 15 May 2016 15:32:54 +0800
Subject: [PATCH 4944/6219] [utils] Fix getheader in urlhandle_detect_ext

Fixes #7049, related to #9440
---
 youtube_dl/utils.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index a637563cb..24e74428b 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2031,11 +2031,7 @@ def mimetype2ext(mt):
 
 
 def urlhandle_detect_ext(url_handle):
-    try:
-        url_handle.headers
-        getheader = lambda h: url_handle.headers[h]
-    except AttributeError:  # Python < 3
-        getheader = url_handle.info().getheader
+    getheader = url_handle.headers.get
 
     cd = getheader('Content-Disposition')
     if cd:

From cec9727c7f6a0dad8b10a51f0a6581ac5a1dbe86 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 15 May 2016 15:35:31 +0800
Subject: [PATCH 4945/6219] [hearthisat] Detect invalid download links (fixes
 #9440)

---
 youtube_dl/extractor/hearthisat.py | 38 +++++++++++++++++++++++-------
 1 file changed, 29 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/hearthisat.py b/youtube_dl/extractor/hearthisat.py
index 7d8698655..ac42ef414 100644
--- a/youtube_dl/extractor/hearthisat.py
+++ b/youtube_dl/extractor/hearthisat.py
@@ -7,6 +7,7 @@ from .common import InfoExtractor
 from ..compat import compat_urlparse
 from ..utils import (
     HEADRequest,
+    KNOWN_EXTENSIONS,
     sanitized_Request,
     str_to_int,
     urlencode_postdata,
@@ -17,7 +18,7 @@ from ..utils import (
 class HearThisAtIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?hearthis\.at/(?P<artist>[^/]+)/(?P<title>[A-Za-z0-9\-]+)/?$'
     _PLAYLIST_URL = 'https://hearthis.at/playlist.php'
-    _TEST = {
+    _TESTS = [{
         'url': 'https://hearthis.at/moofi/dr-kreep',
         'md5': 'ab6ec33c8fed6556029337c7885eb4e0',
         'info_dict': {
@@ -34,7 +35,25 @@ class HearThisAtIE(InfoExtractor):
             'duration': 71,
             'categories': ['Experimental'],
         }
-    }
+    }, {
+        # 'download' link redirects to the original webpage
+        'url': 'https://hearthis.at/twitchsf/dj-jim-hopkins-totally-bitchin-80s-dance-mix/',
+        'md5': '5980ceb7c461605d30f1f039df160c6e',
+        'info_dict': {
+            'id': '811296',
+            'ext': 'mp3',
+            'title': 'TwitchSF - DJ Jim Hopkins -  Totally Bitchin\' 80\'s Dance Mix!',
+            'description': 'Listen to DJ Jim Hopkins -  Totally Bitchin\' 80\'s Dance Mix! by TwitchSF on hearthis.at - Dance',
+            'upload_date': '20160328',
+            'timestamp': 1459186146,
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'comment_count': int,
+            'view_count': int,
+            'like_count': int,
+            'duration': 4360,
+            'categories': ['Dance'],
+        },
+    }]
 
     def _real_extract(self, url):
         m = re.match(self._VALID_URL, url)
@@ -90,13 +109,14 @@ class HearThisAtIE(InfoExtractor):
             ext_handle = self._request_webpage(
                 ext_req, display_id, note='Determining extension')
             ext = urlhandle_detect_ext(ext_handle)
-            formats.append({
-                'format_id': 'download',
-                'vcodec': 'none',
-                'ext': ext,
-                'url': download_url,
-                'preference': 2,  # Usually better quality
-            })
+            if ext in KNOWN_EXTENSIONS:
+                formats.append({
+                    'format_id': 'download',
+                    'vcodec': 'none',
+                    'ext': ext,
+                    'url': download_url,
+                    'preference': 2,  # Usually better quality
+                })
         self._sort_formats(formats)
 
         return {

From 5572d598a537998615c760ca06bd8d3894150c6a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 15 May 2016 15:44:04 +0800
Subject: [PATCH 4946/6219] [hearthisat] Update the first test

---
 youtube_dl/extractor/hearthisat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/hearthisat.py b/youtube_dl/extractor/hearthisat.py
index ac42ef414..256453882 100644
--- a/youtube_dl/extractor/hearthisat.py
+++ b/youtube_dl/extractor/hearthisat.py
@@ -27,7 +27,7 @@ class HearThisAtIE(InfoExtractor):
             'title': 'Moofi - Dr. Kreep',
             'thumbnail': 're:^https?://.*\.jpg$',
             'timestamp': 1421564134,
-            'description': 'Creepy Patch. Mutable Instruments Braids Vowel + Formant Mode.',
+            'description': 'Listen to Dr. Kreep by Moofi on hearthis.at - Modular, Eurorack, Mutable Intruments Braids, Valhalla-DSP',
             'upload_date': '20150118',
             'comment_count': int,
             'view_count': int,

From a0a81918f18252805b161e4f7d0dc4924b672948 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 15 May 2016 22:07:51 +0600
Subject: [PATCH 4947/6219] [collegehumor] Remove extractor

It now uses brightcove
---
 youtube_dl/extractor/collegehumor.py | 101 ---------------------------
 youtube_dl/extractor/extractors.py   |   1 -
 2 files changed, 102 deletions(-)
 delete mode 100644 youtube_dl/extractor/collegehumor.py

diff --git a/youtube_dl/extractor/collegehumor.py b/youtube_dl/extractor/collegehumor.py
deleted file mode 100644
index 002b24037..000000000
--- a/youtube_dl/extractor/collegehumor.py
+++ /dev/null
@@ -1,101 +0,0 @@
-from __future__ import unicode_literals
-
-import json
-import re
-
-from .common import InfoExtractor
-from ..utils import int_or_none
-
-
-class CollegeHumorIE(InfoExtractor):
-    _VALID_URL = r'^(?:https?://)?(?:www\.)?collegehumor\.com/(video|embed|e)/(?P<videoid>[0-9]+)/?(?P<shorttitle>.*)$'
-
-    _TESTS = [
-        {
-            'url': 'http://www.collegehumor.com/video/6902724/comic-con-cosplay-catastrophe',
-            'md5': 'dcc0f5c1c8be98dc33889a191f4c26bd',
-            'info_dict': {
-                'id': '6902724',
-                'ext': 'mp4',
-                'title': 'Comic-Con Cosplay Catastrophe',
-                'description': "Fans get creative this year at San Diego.  Too creative.  And yes, that's really Joss Whedon.",
-                'age_limit': 13,
-                'duration': 187,
-            },
-        }, {
-            'url': 'http://www.collegehumor.com/video/3505939/font-conference',
-            'md5': '72fa701d8ef38664a4dbb9e2ab721816',
-            'info_dict': {
-                'id': '3505939',
-                'ext': 'mp4',
-                'title': 'Font Conference',
-                'description': "This video wasn't long enough, so we made it double-spaced.",
-                'age_limit': 10,
-                'duration': 179,
-            },
-        }, {
-            # embedded youtube video
-            'url': 'http://www.collegehumor.com/embed/6950306',
-            'info_dict': {
-                'id': 'Z-bao9fg6Yc',
-                'ext': 'mp4',
-                'title': 'Young Americans Think President John F. Kennedy Died THIS MORNING IN A CAR ACCIDENT!!!',
-                'uploader': 'Mark Dice',
-                'uploader_id': 'MarkDice',
-                'description': 'md5:62c3dab9351fac7bb44b53b69511d87f',
-                'upload_date': '20140127',
-            },
-            'params': {
-                'skip_download': True,
-            },
-            'add_ie': ['Youtube'],
-        },
-    ]
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('videoid')
-
-        jsonUrl = 'http://www.collegehumor.com/moogaloop/video/' + video_id + '.json'
-        data = json.loads(self._download_webpage(
-            jsonUrl, video_id, 'Downloading info JSON'))
-        vdata = data['video']
-        if vdata.get('youtubeId') is not None:
-            return {
-                '_type': 'url',
-                'url': vdata['youtubeId'],
-                'ie_key': 'Youtube',
-            }
-
-        AGE_LIMITS = {'nc17': 18, 'r': 18, 'pg13': 13, 'pg': 10, 'g': 0}
-        rating = vdata.get('rating')
-        if rating:
-            age_limit = AGE_LIMITS.get(rating.lower())
-        else:
-            age_limit = None  # None = No idea
-
-        PREFS = {'high_quality': 2, 'low_quality': 0}
-        formats = []
-        for format_key in ('mp4', 'webm'):
-            for qname, qurl in vdata.get(format_key, {}).items():
-                formats.append({
-                    'format_id': format_key + '_' + qname,
-                    'url': qurl,
-                    'format': format_key,
-                    'preference': PREFS.get(qname),
-                })
-        self._sort_formats(formats)
-
-        duration = int_or_none(vdata.get('duration'), 1000)
-        like_count = int_or_none(vdata.get('likes'))
-
-        return {
-            'id': video_id,
-            'title': vdata['title'],
-            'description': vdata.get('description'),
-            'thumbnail': vdata.get('thumbnail'),
-            'formats': formats,
-            'age_limit': age_limit,
-            'duration': duration,
-            'like_count': like_count,
-        }
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 2db3b3c3f..ca9d85e33 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -138,7 +138,6 @@ from .cnn import (
     CNNBlogsIE,
     CNNArticleIE,
 )
-from .collegehumor import CollegeHumorIE
 from .collegerama import CollegeRamaIE
 from .comedycentral import ComedyCentralIE, ComedyCentralShowsIE
 from .comcarcoff import ComCarCoffIE

From f7199423e542580cf8c30991d122673276113497 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 16 May 2016 00:30:13 +0600
Subject: [PATCH 4948/6219] [groupon] Add support for Youtube embeds (Closes
 #9508)

---
 youtube_dl/extractor/groupon.py | 31 ++++++++++++++++++++++---------
 1 file changed, 22 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/groupon.py b/youtube_dl/extractor/groupon.py
index f6b69662b..1dd0a81cc 100644
--- a/youtube_dl/extractor/groupon.py
+++ b/youtube_dl/extractor/groupon.py
@@ -4,7 +4,7 @@ from .common import InfoExtractor
 
 
 class GrouponIE(InfoExtractor):
-    _VALID_URL = r'https?://www\.groupon\.com/deals/(?P<id>[^?#]+)'
+    _VALID_URL = r'https?://(?:www\.)?groupon\.com/deals/(?P<id>[^/?#&]+)'
 
     _TEST = {
         'url': 'https://www.groupon.com/deals/bikram-yoga-huntington-beach-2#ooid=tubGNycTo_9Uxg82uESj4i61EYX8nyuf',
@@ -15,18 +15,26 @@ class GrouponIE(InfoExtractor):
         },
         'playlist': [{
             'info_dict': {
-                'id': 'tubGNycTo_9Uxg82uESj4i61EYX8nyuf',
-                'ext': 'flv',
-                'title': 'Bikram Yoga Huntington Beach | Orange County',
+                'id': 'fk6OhWpXgIQ',
+                'ext': 'mp4',
+                'title': 'Bikram Yoga Huntington Beach | Orange County !tubGNycTo@9Uxg82uESj4i61EYX8nyuf',
                 'description': 'md5:d41d8cd98f00b204e9800998ecf8427e',
-                'duration': 44.961,
+                'duration': 45,
+                'upload_date': '20160405',
+                'uploader_id': 'groupon',
+                'uploader': 'Groupon',
             },
         }],
         'params': {
-            'skip_download': 'HDS',
+            'skip_download': True,
         }
     }
 
+    _PROVIDERS = {
+        'ooyala': ('ooyala:%s', 'Ooyala'),
+        'youtube': ('%s', 'Youtube'),
+    }
+
     def _real_extract(self, url):
         playlist_id = self._match_id(url)
         webpage = self._download_webpage(url, playlist_id)
@@ -36,12 +44,17 @@ class GrouponIE(InfoExtractor):
         videos = payload['carousel'].get('dealVideos', [])
         entries = []
         for v in videos:
-            if v.get('provider') != 'OOYALA':
+            provider = v.get('provider')
+            video_id = v.get('media') or v.get('id') or v.get('baseURL')
+            if not provider or not video_id:
+                continue
+            url_pattern, ie_key = self._PROVIDERS.get(provider.lower())
+            if not url_pattern:
                 self.report_warning(
                     '%s: Unsupported video provider %s, skipping video' %
-                    (playlist_id, v.get('provider')))
+                    (playlist_id, provider))
                 continue
-            entries.append(self.url_result('ooyala:%s' % v['media']))
+            entries.append(self.url_result(url_pattern % video_id, ie_key))
 
         return {
             '_type': 'playlist',

From 36755d9d694f818ce8f367ce7eb41374f194893d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 16 May 2016 17:25:47 +0200
Subject: [PATCH 4949/6219] release 2016.05.16

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 docs/supportedsites.md    | 7 +++----
 youtube_dl/version.py     | 2 +-
 3 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 1fb878b59..7024fc729 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.05.10*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.05.10**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.05.16*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.05.16**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.05.10
+[debug] youtube-dl version 2016.05.16
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index de84e5c84..29db13883 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -6,6 +6,7 @@
  - **22tracks:genre**
  - **22tracks:track**
  - **24video**
+ - **3qsdn**: 3Q SDN
  - **3sat**
  - **4tube**
  - **56.com**
@@ -114,7 +115,6 @@
  - **chirbit**
  - **chirbit:profile**
  - **Cinchcast**
- - **Cinemassacre**
  - **Clipfish**
  - **cliphunter**
  - **ClipRs**
@@ -128,7 +128,6 @@
  - **CNN**
  - **CNNArticle**
  - **CNNBlogs**
- - **CollegeHumor**
  - **CollegeRama**
  - **ComCarCoff**
  - **ComedyCentral**
@@ -680,7 +679,6 @@
  - **tvp.pl:Series**
  - **TVPlay**: TV3Play and related services
  - **Tweakers**
- - **twitch:bookmarks**
  - **twitch:chapter**
  - **twitch:past_broadcasts**
  - **twitch:profile**
@@ -698,7 +696,8 @@
  - **USAToday**
  - **ustream**
  - **ustream:channel**
- - **Ustudio**
+ - **ustudio**
+ - **ustudio:embed**
  - **Varzesh3**
  - **Vbox7**
  - **VeeHD**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 45e40c0d1..5a0fdd6ce 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.05.10'
+__version__ = '2016.05.16'

From cdd94c2eae6c6f0a627d457c3a73894a62eb86c5 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 17 May 2016 14:38:15 +0800
Subject: [PATCH 4950/6219] [utils] Check for None values in SOCKS proxy

Originally reported at
https://github.com/rg3/youtube-dl/pull/9287#issuecomment-219617864
---
 youtube_dl/utils.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 24e74428b..ac60ba18c 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -883,12 +883,17 @@ def make_socks_conn_class(base_class, socks_proxy):
     elif url_components.scheme.lower() == 'socks4a':
         socks_type = ProxyType.SOCKS4A
 
+    def unquote_if_non_empty(s):
+        if not s:
+            return s
+        return compat_urllib_parse_unquote_plus(s)
+
     proxy_args = (
         socks_type,
         url_components.hostname, url_components.port or 1080,
         True,  # Remote DNS
-        compat_urllib_parse_unquote_plus(url_components.username),
-        compat_urllib_parse_unquote_plus(url_components.password),
+        unquote_if_non_empty(url_components.username),
+        unquote_if_non_empty(url_components.password),
     )
 
     class SocksConnection(base_class):

From 055f0d3d0636e343354a19cd558a3aac3cf31399 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 17 May 2016 15:38:57 +0800
Subject: [PATCH 4951/6219] [abcnews] Added a new extractor (closes #3992)

Related: #6108, #8664, #9459
---
 youtube_dl/extractor/abcnews.py    | 135 +++++++++++++++++++++++++++++
 youtube_dl/extractor/amp.py        |   4 +-
 youtube_dl/extractor/extractors.py |   4 +
 3 files changed, 141 insertions(+), 2 deletions(-)
 create mode 100644 youtube_dl/extractor/abcnews.py

diff --git a/youtube_dl/extractor/abcnews.py b/youtube_dl/extractor/abcnews.py
new file mode 100644
index 000000000..b61a6327c
--- /dev/null
+++ b/youtube_dl/extractor/abcnews.py
@@ -0,0 +1,135 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import calendar
+import re
+import time
+
+from .amp import AMPIE
+from .common import InfoExtractor
+from ..compat import compat_urlparse
+
+
+class AbcNewsVideoIE(AMPIE):
+    IE_NAME = 'abcnews:video'
+    _VALID_URL = 'http://abcnews.go.com/[^/]+/video/(?P<display_id>[0-9a-z-]+)-(?P<id>\d+)'
+
+    _TESTS = [{
+        'url': 'http://abcnews.go.com/ThisWeek/video/week-exclusive-irans-foreign-minister-zarif-20411932',
+        'info_dict': {
+            'id': '20411932',
+            'ext': 'mp4',
+            'display_id': 'week-exclusive-irans-foreign-minister-zarif',
+            'title': '\'This Week\' Exclusive: Iran\'s Foreign Minister Zarif',
+            'description': 'George Stephanopoulos goes one-on-one with Iranian Foreign Minister Dr. Javad Zarif.',
+            'duration': 180,
+            'thumbnail': 're:^https?://.*\.jpg$',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }, {
+        'url': 'http://abcnews.go.com/2020/video/2020-husband-stands-teacher-jail-student-affairs-26119478',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        display_id = mobj.group('display_id')
+        video_id = mobj.group('id')
+        info_dict = self._extract_feed_info(
+            'http://abcnews.go.com/video/itemfeed?id=%s' % video_id)
+        info_dict.update({
+            'id': video_id,
+            'display_id': display_id,
+        })
+        return info_dict
+
+
+class AbcNewsIE(InfoExtractor):
+    IE_NAME = 'abcnews'
+    _VALID_URL = 'https?://abcnews\.go\.com/(?:[^/]+/)+(?P<display_id>[0-9a-z-]+)/story\?id=(?P<id>\d+)'
+
+    _TESTS = [{
+        'url': 'http://abcnews.go.com/Blotter/News/dramatic-video-rare-death-job-america/story?id=10498713#.UIhwosWHLjY',
+        'info_dict': {
+            'id': '10498713',
+            'ext': 'flv',
+            'display_id': 'dramatic-video-rare-death-job-america',
+            'title': 'Occupational Hazards',
+            'description': 'Nightline investigates the dangers that lurk at various jobs.',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'upload_date': '20100428',
+            'timestamp': 1272412800,
+        },
+        'add_ie': ['AbcNewsVideo'],
+    }, {
+        'url': 'http://abcnews.go.com/Entertainment/justin-timberlake-performs-stop-feeling-eurovision-2016/story?id=39125818',
+        'info_dict': {
+            'id': '39125818',
+            'ext': 'mp4',
+            'display_id': 'justin-timberlake-performs-stop-feeling-eurovision-2016',
+            'title': 'Justin Timberlake Drops Hints For Secret Single',
+            'description': 'Lara Spencer reports the buzziest stories of the day in "GMA" Pop News.',
+            'upload_date': '20160515',
+            'timestamp': 1463329500,
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+            # The embedded YouTube video is blocked due to copyright issues
+            'playlist_items': '1',
+        },
+        'add_ie': ['AbcNewsVideo'],
+    }, {
+        'url': 'http://abcnews.go.com/Technology/exclusive-apple-ceo-tim-cook-iphone-cracking-software/story?id=37173343',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        display_id = mobj.group('display_id')
+        video_id = mobj.group('id')
+
+        webpage = self._download_webpage(url, video_id)
+        video_url = self._search_regex(
+            r'window\.abcnvideo\.url\s*=\s*"([^"]+)"', webpage, 'video URL')
+        full_video_url = compat_urlparse.urljoin(url, video_url)
+
+        youtube_url = self._html_search_regex(
+            r'<iframe[^>]+src="(https://www\.youtube\.com/embed/[^"]+)"',
+            webpage, 'YouTube URL', default=None)
+
+        timestamp = None
+        date_str = self._html_search_regex(
+            r'<span[^>]+class="timestamp">([^<]+)</span>',
+            webpage, 'timestamp', fatal=False)
+        if date_str:
+            tz_offset = 0
+            if date_str.endswith(' ET'):  # Eastern Time
+                tz_offset = -5
+                date_str = date_str[:-3]
+            date_formats = ['%b. %d, %Y', '%b %d, %Y, %I:%M %p']
+            for date_format in date_formats:
+                try:
+                    timestamp = calendar.timegm(time.strptime(date_str.strip(), date_format))
+                except ValueError:
+                    continue
+            if timestamp is not None:
+                timestamp -= tz_offset * 3600
+
+        entry = {
+            '_type': 'url_transparent',
+            'ie_key': AbcNewsVideoIE.ie_key(),
+            'url': full_video_url,
+            'id': video_id,
+            'display_id': display_id,
+            'timestamp': timestamp,
+        }
+
+        if youtube_url:
+            entries = [entry, self.url_result(youtube_url, 'Youtube')]
+            return self.playlist_result(entries)
+
+        return entry
diff --git a/youtube_dl/extractor/amp.py b/youtube_dl/extractor/amp.py
index 138fa0808..8545681be 100644
--- a/youtube_dl/extractor/amp.py
+++ b/youtube_dl/extractor/amp.py
@@ -52,7 +52,7 @@ class AMPIE(InfoExtractor):
         for media_data in media_content:
             media = media_data['@attributes']
             media_type = media['type']
-            if media_type == 'video/f4m':
+            if media_type in ('video/f4m', 'application/f4m+xml'):
                 formats.extend(self._extract_f4m_formats(
                     media['url'] + '?hdcore=3.4.0&plugin=aasp-3.4.0.132.124',
                     video_id, f4m_id='hds', fatal=False))
@@ -61,7 +61,7 @@ class AMPIE(InfoExtractor):
                     media['url'], video_id, 'mp4', m3u8_id='hls', fatal=False))
             else:
                 formats.append({
-                    'format_id': media_data['media-category']['@attributes']['label'],
+                    'format_id': media_data.get('media-category', {}).get('@attributes', {}).get('label'),
                     'url': media['url'],
                     'tbr': int_or_none(media.get('bitrate')),
                     'filesize': int_or_none(media.get('fileSize')),
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index ca9d85e33..861701f4c 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -3,6 +3,10 @@ from __future__ import unicode_literals
 
 from .abc import ABCIE
 from .abc7news import Abc7NewsIE
+from .abcnews import (
+    AbcNewsIE,
+    AbcNewsVideoIE,
+)
 from .academicearth import AcademicEarthCourseIE
 from .acast import (
     ACastIE,

From 15cda1ef774e9dbc538765f59dff5b10a492eca5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 17 May 2016 23:46:47 +0600
Subject: [PATCH 4952/6219] [nfb] Fix uploader extraction

---
 youtube_dl/extractor/nfb.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/nfb.py b/youtube_dl/extractor/nfb.py
index 51e4a34f7..234e49047 100644
--- a/youtube_dl/extractor/nfb.py
+++ b/youtube_dl/extractor/nfb.py
@@ -37,8 +37,7 @@ class NFBIE(InfoExtractor):
 
         uploader_id = self._html_search_regex(r'<a class="director-link" href="/explore-all-directors/([^/]+)/"',
                                               page, 'director id', fatal=False)
-        uploader = self._html_search_regex(r'<em class="director-name" itemprop="name">([^<]+)</em>',
-                                           page, 'director name', fatal=False)
+        uploader = self._og_search_property('video:director', page, 'director name')
 
         request = sanitized_Request(
             'https://www.nfb.ca/film/%s/player_config' % video_id,

From 11e6a0b64130f9b4aea1a6115a3ebaad73f2f5e0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 18 May 2016 00:25:15 +0600
Subject: [PATCH 4953/6219] [nfb] Modernize and extract subtitles

---
 youtube_dl/extractor/nfb.py | 110 +++++++++++++++++++++---------------
 1 file changed, 64 insertions(+), 46 deletions(-)

diff --git a/youtube_dl/extractor/nfb.py b/youtube_dl/extractor/nfb.py
index 234e49047..adcc636bc 100644
--- a/youtube_dl/extractor/nfb.py
+++ b/youtube_dl/extractor/nfb.py
@@ -2,8 +2,12 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..utils import (
-    sanitized_Request,
+    clean_html,
+    determine_ext,
+    int_or_none,
+    qualities,
     urlencode_postdata,
+    xpath_text,
 )
 
 
@@ -16,12 +20,12 @@ class NFBIE(InfoExtractor):
         'url': 'https://www.nfb.ca/film/qallunaat_why_white_people_are_funny',
         'info_dict': {
             'id': 'qallunaat_why_white_people_are_funny',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': 'Qallunaat! Why White People Are Funny ',
-            'description': 'md5:836d8aff55e087d04d9f6df554d4e038',
+            'description': 'md5:6b8e32dde3abf91e58857b174916620c',
             'duration': 3128,
+            'creator': 'Mark Sandiford',
             'uploader': 'Mark Sandiford',
-            'uploader_id': 'mark-sandiford',
         },
         'params': {
             # rtmp download
@@ -31,64 +35,78 @@ class NFBIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        page = self._download_webpage(
-            'https://www.nfb.ca/film/%s' % video_id, video_id,
-            'Downloading film page')
 
-        uploader_id = self._html_search_regex(r'<a class="director-link" href="/explore-all-directors/([^/]+)/"',
-                                              page, 'director id', fatal=False)
-        uploader = self._og_search_property('video:director', page, 'director name')
-
-        request = sanitized_Request(
+        config = self._download_xml(
             'https://www.nfb.ca/film/%s/player_config' % video_id,
-            urlencode_postdata({'getConfig': 'true'}))
-        request.add_header('Content-Type', 'application/x-www-form-urlencoded')
-        request.add_header('X-NFB-Referer', 'http://www.nfb.ca/medias/flash/NFBVideoPlayer.swf')
+            video_id, 'Downloading player config XML',
+            data=urlencode_postdata({'getConfig': 'true'}),
+            headers={
+                'Content-Type': 'application/x-www-form-urlencoded',
+                'X-NFB-Referer': 'http://www.nfb.ca/medias/flash/NFBVideoPlayer.swf'
+            })
 
-        config = self._download_xml(request, video_id, 'Downloading player config XML')
-
-        title = None
-        description = None
-        thumbnail = None
-        duration = None
-        formats = []
-
-        def extract_thumbnail(media):
-            thumbnails = {}
-            for asset in media.findall('assets/asset'):
-                thumbnails[asset.get('quality')] = asset.find('default/url').text
-            if not thumbnails:
-                return None
-            if 'high' in thumbnails:
-                return thumbnails['high']
-            return list(thumbnails.values())[0]
+        title, description, thumbnail, duration, uploader, author = [None] * 6
+        thumbnails, formats = [[]] * 2
+        subtitles = {}
 
         for media in config.findall('./player/stream/media'):
             if media.get('type') == 'posterImage':
-                thumbnail = extract_thumbnail(media)
-            elif media.get('type') == 'video':
-                duration = int(media.get('duration'))
-                title = media.find('title').text
-                description = media.find('description').text
-                # It seems assets always go from lower to better quality, so no need to sort
+                quality_key = qualities(('low', 'high'))
+                thumbnails = []
                 for asset in media.findall('assets/asset'):
-                    for x in asset:
+                    asset_url = xpath_text(asset, 'default/url', default=None)
+                    if not asset_url:
+                        continue
+                    quality = asset.get('quality')
+                    thumbnails.append({
+                        'url': asset_url,
+                        'id': quality,
+                        'preference': quality_key(quality),
+                    })
+            elif media.get('type') == 'video':
+                title = xpath_text(media, 'title', fatal=True)
+                for asset in media.findall('assets/asset'):
+                    quality = asset.get('quality')
+                    height = int_or_none(self._search_regex(
+                        r'^(\d+)[pP]$', quality or '', 'height', default=None))
+                    for node in asset:
+                        streamer = xpath_text(node, 'streamerURI', default=None)
+                        if not streamer:
+                            continue
+                        play_path = xpath_text(node, 'url', default=None)
+                        if not play_path:
+                            continue
                         formats.append({
-                            'url': x.find('streamerURI').text,
-                            'app': x.find('streamerURI').text.split('/', 3)[3],
-                            'play_path': x.find('url').text,
+                            'url': streamer,
+                            'app': streamer.split('/', 3)[3],
+                            'play_path': play_path,
                             'rtmp_live': False,
-                            'ext': 'mp4',
-                            'format_id': '%s-%s' % (x.tag, asset.get('quality')),
+                            'ext': 'flv',
+                            'format_id': '%s-%s' % (node.tag, quality) if quality else node.tag,
+                            'height': height,
                         })
+                self._sort_formats(formats)
+                description = clean_html(xpath_text(media, 'description'))
+                uploader = xpath_text(media, 'author')
+                duration = int_or_none(media.get('duration'))
+                for subtitle in media.findall('./subtitles/subtitle'):
+                    subtitle_url = xpath_text(subtitle, 'url', default=None)
+                    if not subtitle_url:
+                        continue
+                    lang = xpath_text(subtitle, 'lang', default='en')
+                    subtitles.setdefault(lang, []).append({
+                        'url': subtitle_url,
+                        'ext': (subtitle.get('format') or determine_ext(subtitle_url)).lower(),
+                    })
 
         return {
             'id': video_id,
             'title': title,
             'description': description,
-            'thumbnail': thumbnail,
+            'thumbnails': thumbnails,
             'duration': duration,
+            'creator': uploader,
             'uploader': uploader,
-            'uploader_id': uploader_id,
             'formats': formats,
+            'subtitles': subtitles,
         }

From b78531a36abd765aa9c9df1dba1cf82dc23f8fec Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 18 May 2016 22:24:46 +0100
Subject: [PATCH 4954/6219] [formula1] Add new extractor(closes #3617)

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/formula1.py   | 25 +++++++++++++++++++++++++
 2 files changed, 26 insertions(+)
 create mode 100644 youtube_dl/extractor/formula1.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 861701f4c..efbe970fe 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -242,6 +242,7 @@ from .fktv import FKTVIE
 from .flickr import FlickrIE
 from .folketinget import FolketingetIE
 from .footyroom import FootyRoomIE
+from .formula1 import Formula1IE
 from .fourtube import FourTubeIE
 from .fox import FOXIE
 from .foxgay import FoxgayIE
diff --git a/youtube_dl/extractor/formula1.py b/youtube_dl/extractor/formula1.py
new file mode 100644
index 000000000..726393fcc
--- /dev/null
+++ b/youtube_dl/extractor/formula1.py
@@ -0,0 +1,25 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class Formula1IE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?formula1\.com/content/fom-website/en/video/\d{4}/\d{1,2}/(?P<id>.+?)\.html'
+    _TEST = {
+        'url': 'http://www.formula1.com/content/fom-website/en/video/2016/5/Race_highlights_-_Spain_2016.html',
+        'md5': '8c79e54be72078b26b89e0e111c0502b',
+        'info_dict': {
+            'id': 'JvYXJpMzE6pArfHWm5ARp5AiUmD-gibV',
+            'ext': 'flv',
+            'title': 'Race highlights - Spain 2016',
+        }
+    }
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+        ooyala_embed_code = self._search_regex(
+            r'data-videoid="([^"]+)"', webpage, 'ooyala embed code')
+        return self.url_result(
+            'ooyala:%s' % ooyala_embed_code, 'Ooyala', ooyala_embed_code)

From 46bc9b7d7cea2e161670e65abe42ef01d39e8957 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 19 May 2016 04:31:30 +0600
Subject: [PATCH 4955/6219] [utils] Allow None in remove_{start,end}

---
 test/test_utils.py  | 12 ++++++++++++
 youtube_dl/utils.py |  8 ++------
 2 files changed, 14 insertions(+), 6 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 520d32ff5..a697232a8 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -50,6 +50,8 @@ from youtube_dl.utils import (
     sanitize_path,
     prepend_extension,
     replace_extension,
+    remove_start,
+    remove_end,
     remove_quotes,
     shell_quote,
     smuggle_url,
@@ -215,6 +217,16 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(replace_extension('.abc', 'temp'), '.abc.temp')
         self.assertEqual(replace_extension('.abc.ext', 'temp'), '.abc.temp')
 
+    def test_remove_start(self):
+        self.assertEqual(remove_start(None, 'A - '), None)
+        self.assertEqual(remove_start('A - B', 'A - '), 'B')
+        self.assertEqual(remove_start('B - A', 'A - '), 'B - A')
+
+    def test_remove_end(self):
+        self.assertEqual(remove_end(None, ' - B'), None)
+        self.assertEqual(remove_end('A - B', ' - B'), 'A')
+        self.assertEqual(remove_end('B - A', ' - B'), 'B - A')
+
     def test_remove_quotes(self):
         self.assertEqual(remove_quotes(None), None)
         self.assertEqual(remove_quotes('"'), '"')
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index ac60ba18c..5301d0740 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1549,15 +1549,11 @@ def setproctitle(title):
 
 
 def remove_start(s, start):
-    if s.startswith(start):
-        return s[len(start):]
-    return s
+    return s[len(start):] if s is not None and s.startswith(start) else s
 
 
 def remove_end(s, end):
-    if s.endswith(end):
-        return s[:-len(end)]
-    return s
+    return s[:-len(end)] if s is not None and s.endswith(end) else s
 
 
 def remove_quotes(s):

From dd81769c62661d168fb87b896ffb8a80dacbe45b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 19 May 2016 04:34:19 +0600
Subject: [PATCH 4956/6219] [ndtv] Fix extraction

---
 youtube_dl/extractor/ndtv.py | 40 ++++++++++--------------------------
 1 file changed, 11 insertions(+), 29 deletions(-)

diff --git a/youtube_dl/extractor/ndtv.py b/youtube_dl/extractor/ndtv.py
index 2a1ca80df..96528f649 100644
--- a/youtube_dl/extractor/ndtv.py
+++ b/youtube_dl/extractor/ndtv.py
@@ -1,19 +1,18 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..utils import (
-    month_by_name,
     int_or_none,
+    remove_end,
+    unified_strdate,
 )
 
 
 class NDTVIE(InfoExtractor):
-    _VALID_URL = r'^https?://(?:www\.)?ndtv\.com/video/player/[^/]*/[^/]*/(?P<id>[a-z0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?ndtv\.com/video/(?:[^/]+/)+[^/?^&]+-(?P<id>\d+)'
 
     _TEST = {
-        'url': 'http://www.ndtv.com/video/player/news/ndtv-exclusive-don-t-need-character-certificate-from-rahul-gandhi-says-arvind-kejriwal/300710',
+        'url': 'http://www.ndtv.com/video/news/news/ndtv-exclusive-don-t-need-character-certificate-from-rahul-gandhi-says-arvind-kejriwal-300710',
         'md5': '39f992dbe5fb531c395d8bbedb1e5e88',
         'info_dict': {
             'id': '300710',
@@ -22,7 +21,7 @@ class NDTVIE(InfoExtractor):
             'description': 'md5:ab2d4b4a6056c5cb4caa6d729deabf02',
             'upload_date': '20131208',
             'duration': 1327,
-            'thumbnail': 'http://i.ndtvimg.com/video/images/vod/medium/2013-12/big_300710_1386518307.jpg',
+            'thumbnail': 're:https?://.*\.jpg',
         },
     }
 
@@ -30,36 +29,19 @@ class NDTVIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
+        title = remove_end(self._og_search_title(webpage), ' - NDTV')
+
         filename = self._search_regex(
             r"__filename='([^']+)'", webpage, 'video filename')
-        video_url = ('http://bitcast-b.bitgravity.com/ndtvod/23372/ndtv/%s' %
-                     filename)
+        video_url = 'http://bitcast-b.bitgravity.com/ndtvod/23372/ndtv/%s' % filename
 
         duration = int_or_none(self._search_regex(
             r"__duration='([^']+)'", webpage, 'duration', fatal=False))
 
-        date_m = re.search(r'''(?x)
-            <p\s+class="vod_dateline">\s*
-                Published\s+On:\s*
-                (?P<monthname>[A-Za-z]+)\s+(?P<day>[0-9]+),\s*(?P<year>[0-9]+)
-            ''', webpage)
-        upload_date = None
+        upload_date = unified_strdate(self._html_search_meta(
+            'publish-date', webpage, 'upload date', fatal=False))
 
-        if date_m is not None:
-            month = month_by_name(date_m.group('monthname'))
-            if month is not None:
-                upload_date = '%s%02d%02d' % (
-                    date_m.group('year'), month, int(date_m.group('day')))
-
-        description = self._og_search_description(webpage)
-        READ_MORE = ' (Read more)'
-        if description.endswith(READ_MORE):
-            description = description[:-len(READ_MORE)]
-
-        title = self._og_search_title(webpage)
-        TITLE_SUFFIX = ' - NDTV'
-        if title.endswith(TITLE_SUFFIX):
-            title = title[:-len(TITLE_SUFFIX)]
+        description = remove_end(self._og_search_description(webpage), ' (Read more)')
 
         return {
             'id': video_id,

From 8585dc4cdc735eb8a077dffb68affa81e1a98693 Mon Sep 17 00:00:00 2001
From: TRox1972 <TRox1972@users.noreply.github.com>
Date: Thu, 19 May 2016 01:18:01 +0200
Subject: [PATCH 4957/6219] [Makefile] delete thumbnails

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 5d7cd5a7e..d760e4576 100644
--- a/Makefile
+++ b/Makefile
@@ -1,7 +1,7 @@
 all: youtube-dl README.md CONTRIBUTING.md README.txt youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtube-dl.fish supportedsites
 
 clean:
-	rm -rf youtube-dl.1.temp.md youtube-dl.1 youtube-dl.bash-completion README.txt MANIFEST build/ dist/ .coverage cover/ youtube-dl.tar.gz youtube-dl.zsh youtube-dl.fish youtube_dl/extractor/lazy_extractors.py *.dump *.part *.info.json *.mp4 *.flv *.mp3 *.avi *.mkv *.webm CONTRIBUTING.md.tmp ISSUE_TEMPLATE.md.tmp youtube-dl youtube-dl.exe
+	rm -rf youtube-dl.1.temp.md youtube-dl.1 youtube-dl.bash-completion README.txt MANIFEST build/ dist/ .coverage cover/ youtube-dl.tar.gz youtube-dl.zsh youtube-dl.fish youtube_dl/extractor/lazy_extractors.py *.dump *.part *.info.json *.mp4 *.flv *.mp3 *.avi *.mkv *.webm *.jpg *.png CONTRIBUTING.md.tmp ISSUE_TEMPLATE.md.tmp youtube-dl youtube-dl.exe
 	find . -name "*.pyc" -delete
 	find . -name "*.class" -delete
 

From a00129670390c241d097afd873b4ee226ca7d550 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 19 May 2016 18:18:03 +0100
Subject: [PATCH 4958/6219] [learnr] Add new extractor(closes #4284)

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/learnr.py     | 33 ++++++++++++++++++++++++++++++
 2 files changed, 34 insertions(+)
 create mode 100644 youtube_dl/extractor/learnr.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index efbe970fe..74aba2d5c 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -370,6 +370,7 @@ from .kuwo import (
 )
 from .la7 import LA7IE
 from .laola1tv import Laola1TvIE
+from .learnr import LearnrIE
 from .lecture2go import Lecture2GoIE
 from .lemonde import LemondeIE
 from .leeco import (
diff --git a/youtube_dl/extractor/learnr.py b/youtube_dl/extractor/learnr.py
new file mode 100644
index 000000000..1435e090e
--- /dev/null
+++ b/youtube_dl/extractor/learnr.py
@@ -0,0 +1,33 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class LearnrIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?learnr\.pro/view/video/(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'http://www.learnr.pro/view/video/51624-web-development-tutorial-for-beginners-1-how-to-build-webpages-with-html-css-javascript',
+        'md5': '3719fdf0a68397f49899e82c308a89de',
+        'info_dict': {
+            'id': '51624',
+            'ext': 'mp4',
+            'title': 'Web Development Tutorial for Beginners (#1) - How to build webpages with HTML, CSS, Javascript',
+            'description': 'md5:b36dbfa92350176cdf12b4d388485503',
+            'uploader': 'LearnCode.academy',
+            'uploader_id': 'learncodeacademy',
+            'upload_date': '20131021',
+        },
+        'add_ie': ['Youtube'],
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        return {
+            '_type': 'url_transparent',
+            'url': self._search_regex(
+                r"videoId\s*:\s*'([^']+)'", webpage, 'youtube id'),
+            'id': video_id,
+        }

From f6e588afc0b12ebec2bc65551e882e6d99467499 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 20 May 2016 08:53:04 +0600
Subject: [PATCH 4959/6219] [24video] Fix description extraction

---
 youtube_dl/extractor/twentyfourvideo.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/twentyfourvideo.py b/youtube_dl/extractor/twentyfourvideo.py
index e03e2dbaa..4025edf02 100644
--- a/youtube_dl/extractor/twentyfourvideo.py
+++ b/youtube_dl/extractor/twentyfourvideo.py
@@ -47,7 +47,8 @@ class TwentyFourVideoIE(InfoExtractor):
 
         title = self._og_search_title(webpage)
         description = self._html_search_regex(
-            r'<span itemprop="description">([^<]+)</span>', webpage, 'description', fatal=False)
+            r'<(p|span)[^>]+itemprop="description"[^>]*>(?P<description>[^<]+)</\1>',
+            webpage, 'description', fatal=False, group='description')
         thumbnail = self._og_search_thumbnail(webpage)
         duration = int_or_none(self._og_search_property(
             'duration', webpage, 'duration', fatal=False))

From 52f7c75cff3d7f7923deda469f9d2a551742c193 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 20 May 2016 06:53:14 +0100
Subject: [PATCH 4960/6219] [cbc] extract http formats and update tests

---
 youtube_dl/extractor/cbc.py | 63 +++++++++++++++++++------------------
 1 file changed, 33 insertions(+), 30 deletions(-)

diff --git a/youtube_dl/extractor/cbc.py b/youtube_dl/extractor/cbc.py
index 68a0633b6..581928f7d 100644
--- a/youtube_dl/extractor/cbc.py
+++ b/youtube_dl/extractor/cbc.py
@@ -4,7 +4,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import js_to_json
+from ..utils import (
+    js_to_json,
+    smuggle_url,
+)
 
 
 class CBCIE(InfoExtractor):
@@ -12,57 +15,54 @@ class CBCIE(InfoExtractor):
     _TESTS = [{
         # with mediaId
         'url': 'http://www.cbc.ca/22minutes/videos/clips-season-23/don-cherry-play-offs',
+        'md5': '97e24d09672fc4cf56256d6faa6c25bc',
         'info_dict': {
             'id': '2682904050',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Don Cherry – All-Stars',
             'description': 'Don Cherry has a bee in his bonnet about AHL player John Scott because that guy’s got heart.',
-            'timestamp': 1454475540,
+            'timestamp': 1454463000,
             'upload_date': '20160203',
-        },
-        'params': {
-            # rtmp download
-            'skip_download': True,
+            'uploader': 'CBCC-NEW',
         },
     }, {
         # with clipId
         'url': 'http://www.cbc.ca/archives/entry/1978-robin-williams-freestyles-on-90-minutes-live',
         'info_dict': {
             'id': '2487345465',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Robin Williams freestyles on 90 Minutes Live',
             'description': 'Wacky American comedian Robin Williams shows off his infamous "freestyle" comedic talents while being interviewed on CBC\'s 90 Minutes Live.',
-            'upload_date': '19700101',
+            'upload_date': '19780210',
             'uploader': 'CBCC-NEW',
-        },
-        'params': {
-            # rtmp download
-            'skip_download': True,
+            'timestamp': 255977160,
         },
     }, {
         # multiple iframes
         'url': 'http://www.cbc.ca/natureofthings/blog/birds-eye-view-from-vancouvers-burrard-street-bridge-how-we-got-the-shot',
         'playlist': [{
+            'md5': '377572d0b49c4ce0c9ad77470e0b96b4',
             'info_dict': {
                 'id': '2680832926',
-                'ext': 'flv',
+                'ext': 'mp4',
                 'title': 'An Eagle\'s-Eye View Off Burrard Bridge',
                 'description': 'Hercules the eagle flies from Vancouver\'s Burrard Bridge down to a nearby park with a mini-camera strapped to his back.',
-                'upload_date': '19700101',
+                'upload_date': '20160201',
+                'timestamp': 1454342820,
+                'uploader': 'CBCC-NEW',
             },
         }, {
+            'md5': '415a0e3f586113894174dfb31aa5bb1a',
             'info_dict': {
                 'id': '2658915080',
-                'ext': 'flv',
+                'ext': 'mp4',
                 'title': 'Fly like an eagle!',
                 'description': 'Eagle equipped with a mini camera flies from the world\'s tallest tower',
-                'upload_date': '19700101',
+                'upload_date': '20150315',
+                'timestamp': 1426443984,
+                'uploader': 'CBCC-NEW',
             },
         }],
-        'params': {
-            # rtmp download
-            'skip_download': True,
-        },
     }]
 
     @classmethod
@@ -95,20 +95,23 @@ class CBCPlayerIE(InfoExtractor):
         'url': 'http://www.cbc.ca/player/play/2683190193',
         'info_dict': {
             'id': '2683190193',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Gerry Runs a Sweat Shop',
             'description': 'md5:b457e1c01e8ff408d9d801c1c2cd29b0',
-            'timestamp': 1455067800,
+            'timestamp': 1455071400,
             'upload_date': '20160210',
-        },
-        'params': {
-            # rtmp download
-            'skip_download': True,
+            'uploader': 'CBCC-NEW',
         },
     }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        return self.url_result(
-            'http://feed.theplatform.com/f/ExhSPC/vms_5akSXx4Ng_Zn?byGuid=%s' % video_id,
-            'ThePlatformFeed', video_id)
+        return {
+            '_type': 'url_transparent',
+            'ie_key': 'ThePlatform',
+            'url': smuggle_url(
+                'http://link.theplatform.com/s/ExhSPC/media/guid/2655402169/%s?mbr=true' % video_id, {
+                    'force_smil_url': True
+                }),
+            'id': video_id,
+        }

From 043dc9d36fea85a964bad3ec13f77d32c462115b Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 20 May 2016 18:39:54 +0800
Subject: [PATCH 4961/6219] [cbc] Fix for old-styled URLs

The URL http://www.cbc.ca/player/News/ID/2672225049/ (#6342) redirects
to http://www.cbc.ca/player/play/2672224672, while youtube-dl wasn't
able to handle it correctly.
---
 youtube_dl/extractor/cbc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/cbc.py b/youtube_dl/extractor/cbc.py
index 581928f7d..daf237ca8 100644
--- a/youtube_dl/extractor/cbc.py
+++ b/youtube_dl/extractor/cbc.py
@@ -11,7 +11,7 @@ from ..utils import (
 
 
 class CBCIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?cbc\.ca/(?:[^/]+/)+(?P<id>[^/?#]+)'
+    _VALID_URL = r'https?://(?:www\.)?cbc\.ca/(?!player/)(?:[^/]+/)+(?P<id>[^/?#]+)'
     _TESTS = [{
         # with mediaId
         'url': 'http://www.cbc.ca/22minutes/videos/clips-season-23/don-cherry-play-offs',

From ad96b4c8f56ba9873c62a2ce9916253f9b8a49ee Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 20 May 2016 19:02:53 +0800
Subject: [PATCH 4962/6219] [common] Extract audio formats in SMIL

Found in http://www.cbc.ca/player/play/2657631896

Closes #5156
---
 youtube_dl/extractor/common.py | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 8a8c07226..9f22ee930 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1278,21 +1278,21 @@ class InfoExtractor(object):
         m3u8_count = 0
 
         srcs = []
-        videos = smil.findall(self._xpath_ns('.//video', namespace))
-        for video in videos:
-            src = video.get('src')
+        media = smil.findall(self._xpath_ns('.//video', namespace)) + smil.findall(self._xpath_ns('.//audio', namespace))
+        for medium in media:
+            src = medium.get('src')
             if not src or src in srcs:
                 continue
             srcs.append(src)
 
-            bitrate = float_or_none(video.get('system-bitrate') or video.get('systemBitrate'), 1000)
-            filesize = int_or_none(video.get('size') or video.get('fileSize'))
-            width = int_or_none(video.get('width'))
-            height = int_or_none(video.get('height'))
-            proto = video.get('proto')
-            ext = video.get('ext')
+            bitrate = float_or_none(medium.get('system-bitrate') or medium.get('systemBitrate'), 1000)
+            filesize = int_or_none(medium.get('size') or medium.get('fileSize'))
+            width = int_or_none(medium.get('width'))
+            height = int_or_none(medium.get('height'))
+            proto = medium.get('proto')
+            ext = medium.get('ext')
             src_ext = determine_ext(src)
-            streamer = video.get('streamer') or base
+            streamer = medium.get('streamer') or base
 
             if proto == 'rtmp' or streamer.startswith('rtmp'):
                 rtmp_count += 1

From 31a70191e730a2a963c8b2e4d19921cad573ad8a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 20 May 2016 19:04:50 +0800
Subject: [PATCH 4963/6219] [cbc] Add the test case from #5156

---
 youtube_dl/extractor/cbc.py | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/cbc.py b/youtube_dl/extractor/cbc.py
index daf237ca8..22d5e72d5 100644
--- a/youtube_dl/extractor/cbc.py
+++ b/youtube_dl/extractor/cbc.py
@@ -91,7 +91,7 @@ class CBCIE(InfoExtractor):
 
 class CBCPlayerIE(InfoExtractor):
     _VALID_URL = r'(?:cbcplayer:|https?://(?:www\.)?cbc\.ca/(?:player/play/|i/caffeine/syndicate/\?mediaId=))(?P<id>\d+)'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.cbc.ca/player/play/2683190193',
         'info_dict': {
             'id': '2683190193',
@@ -102,7 +102,20 @@ class CBCPlayerIE(InfoExtractor):
             'upload_date': '20160210',
             'uploader': 'CBCC-NEW',
         },
-    }
+    }, {
+        # Redirected from http://www.cbc.ca/player/AudioMobile/All%20in%20a%20Weekend%20Montreal/ID/2657632011/
+        'url': 'http://www.cbc.ca/player/play/2657631896',
+        'md5': 'e5e708c34ae6fca156aafe17c43e8b75',
+        'info_dict': {
+            'id': '2657631896',
+            'ext': 'mp3',
+            'title': 'CBC Montreal is organizing its first ever community hackathon!',
+            'description': 'The modern technology we tend to depend on so heavily, is never without it\'s share of hiccups and headaches. Next weekend - CBC Montreal will be getting members of the public for its first Hackathon.',
+            'timestamp': 1425704400,
+            'upload_date': '20150307',
+            'uploader': 'CBCC-NEW',
+        },
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From f0c96af9cb0edc69f9ba73d39e6e191994e31256 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 20 May 2016 20:55:10 +0600
Subject: [PATCH 4964/6219] [wistia] Add alias and modernize

---
 youtube_dl/extractor/wistia.py | 28 ++++++++++++++++++----------
 1 file changed, 18 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/wistia.py b/youtube_dl/extractor/wistia.py
index 8b14840a2..478c42833 100644
--- a/youtube_dl/extractor/wistia.py
+++ b/youtube_dl/extractor/wistia.py
@@ -3,16 +3,16 @@ from __future__ import unicode_literals
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
-    sanitized_Request,
     int_or_none,
 )
 
 
 class WistiaIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:fast\.)?wistia\.net/embed/iframe/(?P<id>[a-z0-9]+)'
-    _API_URL = 'http://fast.wistia.com/embed/medias/{0:}.json'
+    _VALID_URL = r'(?:wistia:|https?://(?:fast\.)?wistia\.net/embed/iframe/)(?P<id>[a-z0-9]+)'
+    _API_URL = 'http://fast.wistia.com/embed/medias/%s.json'
+    _IFRAME_URL = 'http://fast.wistia.net/embed/iframe/%s'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://fast.wistia.net/embed/iframe/sh7fpupwlt',
         'md5': 'cafeb56ec0c53c18c97405eecb3133df',
         'info_dict': {
@@ -24,17 +24,25 @@ class WistiaIE(InfoExtractor):
             'timestamp': 1386185018,
             'duration': 117,
         },
-    }
+    }, {
+        'url': 'wistia:sh7fpupwlt',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        request = sanitized_Request(self._API_URL.format(video_id))
-        request.add_header('Referer', url)  # Some videos require this.
-        data_json = self._download_json(request, video_id)
+        data_json = self._download_json(
+            self._API_URL % video_id, video_id,
+            # Some videos require this.
+            headers={
+                'Referer': url if url.startswith('http') else self._IFRAME_URL % video_id,
+            })
+
         if data_json.get('error'):
-            raise ExtractorError('Error while getting the playlist',
-                                 expected=True)
+            raise ExtractorError(
+                'Error while getting the playlist', expected=True)
+
         data = data_json['media']
         title = data['name']
 

From 36ca2c55db7939aff2dc700523843a9a0f82ae2b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 20 May 2016 21:04:01 +0600
Subject: [PATCH 4965/6219] [wistia] Skip storyboard and improve extraction

---
 youtube_dl/extractor/wistia.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/wistia.py b/youtube_dl/extractor/wistia.py
index 478c42833..6eb94fcab 100644
--- a/youtube_dl/extractor/wistia.py
+++ b/youtube_dl/extractor/wistia.py
@@ -49,19 +49,23 @@ class WistiaIE(InfoExtractor):
         formats = []
         thumbnails = []
         for a in data['assets']:
+            aurl = a.get('url')
+            if not aurl:
+                continue
             astatus = a.get('status')
             atype = a.get('type')
-            if (astatus is not None and astatus != 2) or atype == 'preview':
+            if (astatus is not None and astatus != 2) or atype in ('preview', 'storyboard'):
                 continue
             elif atype in ('still', 'still_image'):
                 thumbnails.append({
-                    'url': a['url'],
-                    'resolution': '%dx%d' % (a['width'], a['height']),
+                    'url': aurl,
+                    'width': int_or_none(a.get('width')),
+                    'height': int_or_none(a.get('height')),
                 })
             else:
                 formats.append({
                     'format_id': atype,
-                    'url': a['url'],
+                    'url': aurl,
                     'tbr': int_or_none(a.get('bitrate')),
                     'vbr': int_or_none(a.get('opt_vbitrate')),
                     'width': int_or_none(a.get('width')),

From 45f160a43c5f103af7a843f1159a1f6e8f498f0f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 20 May 2016 21:16:08 +0600
Subject: [PATCH 4966/6219] [wistia] Improve hls support

---
 youtube_dl/extractor/wistia.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/wistia.py b/youtube_dl/extractor/wistia.py
index 6eb94fcab..97139a35a 100644
--- a/youtube_dl/extractor/wistia.py
+++ b/youtube_dl/extractor/wistia.py
@@ -27,6 +27,10 @@ class WistiaIE(InfoExtractor):
     }, {
         'url': 'wistia:sh7fpupwlt',
         'only_matching': True,
+    }, {
+        # with hls video
+        'url': 'wistia:807fafadvk',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
@@ -63,6 +67,8 @@ class WistiaIE(InfoExtractor):
                     'height': int_or_none(a.get('height')),
                 })
             else:
+                aext = a.get('ext')
+                is_m3u8 = a.get('container') == 'm3u8' or aext == 'm3u8'
                 formats.append({
                     'format_id': atype,
                     'url': aurl,
@@ -73,7 +79,8 @@ class WistiaIE(InfoExtractor):
                     'filesize': int_or_none(a.get('size')),
                     'vcodec': a.get('codec'),
                     'container': a.get('container'),
-                    'ext': a.get('ext'),
+                    'ext': 'mp4' if is_m3u8 else aext,
+                    'protocol': 'm3u8' if is_m3u8 else None,
                     'preference': 1 if atype == 'original' else None,
                 })
 

From 64413f7563eb7a89e06ede91fc135de73bc57db4 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 20 May 2016 16:20:05 +0100
Subject: [PATCH 4967/6219] [cbc] fix extraction for flv only videos(fixes
 #5309)

---
 youtube_dl/extractor/cbc.py | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/cbc.py b/youtube_dl/extractor/cbc.py
index 22d5e72d5..ff663d079 100644
--- a/youtube_dl/extractor/cbc.py
+++ b/youtube_dl/extractor/cbc.py
@@ -28,6 +28,7 @@ class CBCIE(InfoExtractor):
     }, {
         # with clipId
         'url': 'http://www.cbc.ca/archives/entry/1978-robin-williams-freestyles-on-90-minutes-live',
+        'md5': '0274a90b51a9b4971fe005c63f592f12',
         'info_dict': {
             'id': '2487345465',
             'ext': 'mp4',
@@ -93,6 +94,7 @@ class CBCPlayerIE(InfoExtractor):
     _VALID_URL = r'(?:cbcplayer:|https?://(?:www\.)?cbc\.ca/(?:player/play/|i/caffeine/syndicate/\?mediaId=))(?P<id>\d+)'
     _TESTS = [{
         'url': 'http://www.cbc.ca/player/play/2683190193',
+        'md5': '64d25f841ddf4ddb28a235338af32e2c',
         'info_dict': {
             'id': '2683190193',
             'ext': 'mp4',
@@ -115,6 +117,19 @@ class CBCPlayerIE(InfoExtractor):
             'upload_date': '20150307',
             'uploader': 'CBCC-NEW',
         },
+    }, {
+        # available only when we add `formats=MPEG4,FLV,MP3` to theplatform url
+        'url': 'http://www.cbc.ca/player/play/2164402062',
+        'md5': '17a61eb813539abea40618d6323a7f82',
+        'info_dict': {
+            'id': '2164402062',
+            'ext': 'flv',
+            'title': 'Cancer survivor four times over',
+            'description': 'Tim Mayer has beaten three different forms of cancer four times in five years.',
+            'timestamp': 1320410746,
+            'upload_date': '20111104',
+            'uploader': 'CBCC-NEW',
+        },
     }]
 
     def _real_extract(self, url):
@@ -123,7 +138,7 @@ class CBCPlayerIE(InfoExtractor):
             '_type': 'url_transparent',
             'ie_key': 'ThePlatform',
             'url': smuggle_url(
-                'http://link.theplatform.com/s/ExhSPC/media/guid/2655402169/%s?mbr=true' % video_id, {
+                'http://link.theplatform.com/s/ExhSPC/media/guid/2655402169/%s?mbr=true&formats=MPEG4,FLV,MP3' % video_id, {
                     'force_smil_url': True
                 }),
             'id': video_id,

From aa5957ac49aad5165ce9ab5b9403539d61a09dcf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 20 May 2016 21:33:31 +0600
Subject: [PATCH 4968/6219] [extractor/generic] Add support for async wistia
 embeds (Closes #9549)

---
 youtube_dl/extractor/generic.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index a6b1e23e3..632d7b5f0 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1563,6 +1563,15 @@ class GenericIE(InfoExtractor):
                 'id': match.group('id')
             }
 
+        match = re.search(
+            r'''(?sx)
+                <script[^>]+src=(["'])(?:https?:)?//fast\.wistia\.com/assets/external/E-v1\.js\1[^>]*>.*?
+                <div[^>]+class=(["']).*?\bwistia_async_(?P<id>[a-z0-9]+)\b.*?\2
+            ''', webpage)
+        if match:
+            return self.url_result(self._proto_relative_url(
+                'wistia:%s' % match.group('id')), 'Wistia')
+
         # Look for SVT player
         svt_url = SVTIE._extract_url(webpage)
         if svt_url:

From 7ded6545edb18bb008e8277b42a21d60fb6cd653 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 20 May 2016 21:43:36 +0600
Subject: [PATCH 4969/6219] [extractor/generic] Add test for wistia standard
 embed

---
 youtube_dl/extractor/generic.py | 16 ++++++++++++++++
 youtube_dl/extractor/wistia.py  |  3 ++-
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 632d7b5f0..9883cde61 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -742,6 +742,22 @@ class GenericIE(InfoExtractor):
                 'timestamp': 1401832161,
             },
         },
+        # Wistia standard embed (async)
+        {
+            'url': 'https://www.getdrip.com/university/brennan-dunn-drip-workshop/',
+            'info_dict': {
+                'id': '807fafadvk',
+                'ext': 'mp4',
+                'title': 'Drip Brennan Dunn Workshop',
+                'description': 'a JV Webinars video from getdrip-1',
+                'duration': 4986.95,
+                'upload_date': '20160518',
+                'timestamp': 1463607249,
+            },
+            'params': {
+                'skip_download': True,
+            }
+        },
         # Soundcloud embed
         {
             'url': 'http://nakedsecurity.sophos.com/2014/10/29/sscc-171-are-you-sure-that-1234-is-a-bad-password-podcast/',
diff --git a/youtube_dl/extractor/wistia.py b/youtube_dl/extractor/wistia.py
index 97139a35a..c634b8dec 100644
--- a/youtube_dl/extractor/wistia.py
+++ b/youtube_dl/extractor/wistia.py
@@ -4,6 +4,7 @@ from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
     int_or_none,
+    float_or_none,
 )
 
 
@@ -92,6 +93,6 @@ class WistiaIE(InfoExtractor):
             'description': data.get('seoDescription'),
             'formats': formats,
             'thumbnails': thumbnails,
-            'duration': int_or_none(data.get('duration')),
+            'duration': float_or_none(data.get('duration')),
             'timestamp': int_or_none(data.get('createdAt')),
         }

From 6c114b12104e8c9d0713d1cb2cd6c4ddc7872b7f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 20 May 2016 21:55:35 +0600
Subject: [PATCH 4970/6219] [extractor/generic] Remove generic id and title
 from wistia extractionand update tests

---
 youtube_dl/extractor/generic.py | 29 ++++++++++++++---------------
 1 file changed, 14 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 9883cde61..c368f08e1 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -717,15 +717,18 @@ class GenericIE(InfoExtractor):
         },
         # Wistia embed
         {
-            'url': 'http://education-portal.com/academy/lesson/north-american-exploration-failed-colonies-of-spain-france-england.html#lesson',
-            'md5': '8788b683c777a5cf25621eaf286d0c23',
+            'url': 'http://study.com/academy/lesson/north-american-exploration-failed-colonies-of-spain-france-england.html#lesson',
+            'md5': '1953f3a698ab51cfc948ed3992a0b7ff',
             'info_dict': {
-                'id': '1cfaf6b7ea',
+                'id': '6e2wtrbdaf',
                 'ext': 'mov',
-                'title': 'md5:51364a8d3d009997ba99656004b5e20d',
-                'duration': 643.0,
-                'filesize': 182808282,
-                'uploader': 'education-portal.com',
+                'title': 'paywall_north-american-exploration-failed-colonies-of-spain-france-england',
+                'description': 'a Paywall Videos video from Remilon',
+                'duration': 644.072,
+                'uploader': 'study.com',
+                'timestamp': 1459678540,
+                'upload_date': '20160403',
+                'filesize': 24687186,
             },
         },
         {
@@ -734,12 +737,12 @@ class GenericIE(InfoExtractor):
             'info_dict': {
                 'id': 'uxjb0lwrcz',
                 'ext': 'mp4',
-                'title': 'Conversation about Hexagonal Rails Part 1 - ThoughtWorks',
+                'title': 'Conversation about Hexagonal Rails Part 1',
                 'description': 'a Martin Fowler video from ThoughtWorks',
                 'duration': 1715.0,
                 'uploader': 'thoughtworks.wistia.com',
-                'upload_date': '20140603',
                 'timestamp': 1401832161,
+                'upload_date': '20140603',
             },
         },
         # Wistia standard embed (async)
@@ -751,8 +754,8 @@ class GenericIE(InfoExtractor):
                 'title': 'Drip Brennan Dunn Workshop',
                 'description': 'a JV Webinars video from getdrip-1',
                 'duration': 4986.95,
-                'upload_date': '20160518',
                 'timestamp': 1463607249,
+                'upload_date': '20160518',
             },
             'params': {
                 'skip_download': True,
@@ -1564,19 +1567,15 @@ class GenericIE(InfoExtractor):
                 'url': embed_url,
                 'ie_key': 'Wistia',
                 'uploader': video_uploader,
-                'title': video_title,
-                'id': video_id,
             }
 
         match = re.search(r'(?:id=["\']wistia_|data-wistia-?id=["\']|Wistia\.embed\(["\'])(?P<id>[^"\']+)', webpage)
         if match:
             return {
                 '_type': 'url_transparent',
-                'url': 'http://fast.wistia.net/embed/iframe/{0:}'.format(match.group('id')),
+                'url': 'wistia:%s' % match.group('id'),
                 'ie_key': 'Wistia',
                 'uploader': video_uploader,
-                'title': video_title,
-                'id': match.group('id')
             }
 
         match = re.search(

From 6756602be6b59c7bff57ccaeb33844cdc5636910 Mon Sep 17 00:00:00 2001
From: TRox1972 <TRox1972@users.noreply.github.com>
Date: Thu, 19 May 2016 03:42:09 +0200
Subject: [PATCH 4971/6219] [LocalNews8] add extractor (Closes #9200)

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/localnews8.py | 29 +++++++++++++++++++++++++++++
 2 files changed, 30 insertions(+)
 create mode 100644 youtube_dl/extractor/localnews8.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 74aba2d5c..5b96a086d 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -396,6 +396,7 @@ from .livestream import (
     LivestreamShortenerIE,
 )
 from .lnkgo import LnkGoIE
+from .localnews8 import LocalNews8IE
 from .lovehomeporn import LoveHomePornIE
 from .lrt import LRTIE
 from .lynda import (
diff --git a/youtube_dl/extractor/localnews8.py b/youtube_dl/extractor/localnews8.py
new file mode 100644
index 000000000..b38d1d58a
--- /dev/null
+++ b/youtube_dl/extractor/localnews8.py
@@ -0,0 +1,29 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class LocalNews8IE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?localnews8\.com/.+?/(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'http://www.localnews8.com/news/rexburg-business-turns-carbon-fiber-scraps-into-wedding-rings/35183304',
+        'md5': '477bdb188f177788c65db27ecb56649b',
+        'info_dict': {
+            'id': '35183304',
+            'ext': 'mp4',
+            'title': 'Rexburg business turns carbon fiber scraps into wedding ring',
+            'description': 'The process was first invented by Lamborghini and less than a dozen companies around the world use it.',
+            'duration': '153',
+            'timestamp': '1441844822',
+            'uploader_id': 'api',
+        }}
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        partner_id = self._search_regex(r'partnerId\s*:\s*"(\d+)"', webpage, video_id)
+        kaltura_id = self._search_regex(r'var\s+videoIdString\s*=\s*"kaltura:(.+)";', webpage, video_id)
+
+        return self.url_result('kaltura:%s:%s' % (partner_id, kaltura_id), 'Kaltura')

From 1846e9ade0fb9508459282a992539c700aa26f9c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 20 May 2016 22:31:08 +0600
Subject: [PATCH 4972/6219] [localnews8] Fix extractor (Closes #9539)

---
 youtube_dl/extractor/localnews8.py | 38 ++++++++++++++++++++++--------
 1 file changed, 28 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/localnews8.py b/youtube_dl/extractor/localnews8.py
index b38d1d58a..aad396135 100644
--- a/youtube_dl/extractor/localnews8.py
+++ b/youtube_dl/extractor/localnews8.py
@@ -1,29 +1,47 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 
 
 class LocalNews8IE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?localnews8\.com/.+?/(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?localnews8\.com/(?:[^/]+/)*(?P<display_id>[^/]+)/(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://www.localnews8.com/news/rexburg-business-turns-carbon-fiber-scraps-into-wedding-rings/35183304',
-        'md5': '477bdb188f177788c65db27ecb56649b',
+        'md5': 'be4d48aea61aa2bde7be2ee47691ad20',
         'info_dict': {
             'id': '35183304',
+            'display_id': 'rexburg-business-turns-carbon-fiber-scraps-into-wedding-rings',
             'ext': 'mp4',
             'title': 'Rexburg business turns carbon fiber scraps into wedding ring',
             'description': 'The process was first invented by Lamborghini and less than a dozen companies around the world use it.',
-            'duration': '153',
-            'timestamp': '1441844822',
+            'duration': 153,
+            'timestamp': 1441844822,
+            'upload_date': '20150910',
             'uploader_id': 'api',
-        }}
+        }
+    }
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        display_id = mobj.group('display_id')
 
-        partner_id = self._search_regex(r'partnerId\s*:\s*"(\d+)"', webpage, video_id)
-        kaltura_id = self._search_regex(r'var\s+videoIdString\s*=\s*"kaltura:(.+)";', webpage, video_id)
+        webpage = self._download_webpage(url, display_id)
 
-        return self.url_result('kaltura:%s:%s' % (partner_id, kaltura_id), 'Kaltura')
+        partner_id = self._search_regex(
+            r'partnerId\s*[:=]\s*(["\'])(?P<id>\d+)\1',
+            webpage, 'partner id', group='id')
+        kaltura_id = self._search_regex(
+            r'videoIdString\s*[:=]\s*(["\'])kaltura:(?P<id>[0-9a-z_]+)\1',
+            webpage, 'videl id', group='id')
+
+        return {
+            '_type': 'url_transparent',
+            'url': 'kaltura:%s:%s' % (partner_id, kaltura_id),
+            'ie_key': 'Kaltura',
+            'id': video_id,
+            'display_id': display_id,
+        }

From b219f5e51be520b2e23acd1ec08735fc733f9619 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 May 2016 00:59:06 +0600
Subject: [PATCH 4973/6219] [brightcove:new] Improve error reporting

---
 youtube_dl/extractor/brightcove.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index fc7fc5b16..ef560b592 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -444,6 +444,10 @@ class BrightcoveNewIE(InfoExtractor):
         # non numeric ref: prefixed video id
         'url': 'http://players.brightcove.net/710858724001/default_default/index.html?videoId=ref:event-stream-356',
         'only_matching': True,
+    }, {
+        # unavailable video without message but with error_code
+        'url': 'http://players.brightcove.net/1305187701/c832abfb-641b-44eb-9da0-2fe76786505f_default/index.html?videoId=4377407326001',
+        'only_matching': True,
     }]
 
     @staticmethod
@@ -514,8 +518,9 @@ class BrightcoveNewIE(InfoExtractor):
             })
         except ExtractorError as e:
             if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
-                json_data = self._parse_json(e.cause.read().decode(), video_id)
-                raise ExtractorError(json_data[0]['message'], expected=True)
+                json_data = self._parse_json(e.cause.read().decode(), video_id)[0]
+                raise ExtractorError(
+                    json_data.get('message') or json_data['error_code'], expected=True)
             raise
 
         title = json_data['name'].strip()

From c8602b2f9bcdda00398b2c54db4c1be85b75ce39 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 May 2016 05:09:16 +0600
Subject: [PATCH 4974/6219] [nrk] Unquote subtitles' URLs

---
 youtube_dl/extractor/nrk.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index 7532f40c1..486e086bb 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -55,7 +55,9 @@ class NRKBaseIE(InfoExtractor):
                 for subtitle in ('webVtt', 'timedText'):
                     subtitle_url = asset.get('%sSubtitlesUrl' % subtitle)
                     if subtitle_url:
-                        subtitles.setdefault('no', []).append({'url': subtitle_url})
+                        subtitles.setdefault('no', []).append({
+                            'url': compat_urllib_parse_unquote(subtitle_url)
+                        })
                 entries.append({
                     'id': asset.get('carrierId') or entry_id,
                     'title': entry_title,

From 16da9bbc29b76b6e6e1a6134a17e9f25d91296c8 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 21 May 2016 13:15:28 +0800
Subject: [PATCH 4975/6219] [common] Add _m3u8_meta_format() template

For extractors who handle m3u8 manifests by themselves. (eg., AnvatoIE)

Part of #9522
---
 youtube_dl/extractor/common.py | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 9f22ee930..17e866f91 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1058,12 +1058,8 @@ class InfoExtractor(object):
             })
         return formats
 
-    def _extract_m3u8_formats(self, m3u8_url, video_id, ext=None,
-                              entry_protocol='m3u8', preference=None,
-                              m3u8_id=None, note=None, errnote=None,
-                              fatal=True, live=False):
-
-        formats = [{
+    def _m3u8_meta_format(self, m3u8_url, ext=None, preference=None, m3u8_id=None):
+        return {
             'format_id': '-'.join(filter(None, [m3u8_id, 'meta'])),
             'url': m3u8_url,
             'ext': ext,
@@ -1071,7 +1067,14 @@ class InfoExtractor(object):
             'preference': preference - 1 if preference else -1,
             'resolution': 'multiple',
             'format_note': 'Quality selection URL',
-        }]
+        }
+
+    def _extract_m3u8_formats(self, m3u8_url, video_id, ext=None,
+                              entry_protocol='m3u8', preference=None,
+                              m3u8_id=None, note=None, errnote=None,
+                              fatal=True, live=False):
+
+        formats = [self._m3u8_meta_format(m3u8_url, ext, preference, m3u8_id)]
 
         format_url = lambda u: (
             u

From 7b2fcbfd4ea34e6d29484f5987a36665117aefaa Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 21 May 2016 13:16:28 +0800
Subject: [PATCH 4976/6219] [common] Skip TYPE=CLOSED-CAPTIONS lines in m3u8
 manifests

According to [1], valid values for TYPE are AUDIO, VIDEO, SUBTITLES
and CLOSED-CAPTIONS. Such a value is found in Anvato master playlists,
though I don't use _extract_m3u8_formats() in the end.

Part of #9522.

[1] https://tools.ietf.org/html/draft-pantos-http-live-streaming-19#section-4.3.4.1
---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 17e866f91..4bfa610c1 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1141,7 +1141,7 @@ class InfoExtractor(object):
                 format_id = []
                 if m3u8_id:
                     format_id.append(m3u8_id)
-                last_media_name = last_media.get('NAME') if last_media and last_media.get('TYPE') != 'SUBTITLES' else None
+                last_media_name = last_media.get('NAME') if last_media and last_media.get('TYPE') not in ('SUBTITLES', 'CLOSED-CAPTIONS') else None
                 # Despite specification does not mention NAME attribute for
                 # EXT-X-STREAM-INF it still sometimes may be present
                 stream_name = last_info.get('NAME') or last_media_name

From 9f54e692d2de2d52f147f2d714d0312dbe21a5ed Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 21 May 2016 13:18:29 +0800
Subject: [PATCH 4977/6219] [anvato] Add new extractor

Used in CBSLocal (#9522)
---
 youtube_dl/extractor/anvato.py | 224 +++++++++++++++++++++++++++++++++
 1 file changed, 224 insertions(+)
 create mode 100644 youtube_dl/extractor/anvato.py

diff --git a/youtube_dl/extractor/anvato.py b/youtube_dl/extractor/anvato.py
new file mode 100644
index 000000000..cb29cf111
--- /dev/null
+++ b/youtube_dl/extractor/anvato.py
@@ -0,0 +1,224 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import base64
+import hashlib
+import json
+import random
+import time
+
+from .common import InfoExtractor
+from ..aes import aes_encrypt
+from ..compat import compat_str
+from ..utils import (
+    bytes_to_intlist,
+    determine_ext,
+    intlist_to_bytes,
+    int_or_none,
+    strip_jsonp,
+)
+
+
+def md5_text(s):
+    if not isinstance(s, compat_str):
+        s = compat_str(s)
+    return hashlib.md5(s.encode('utf-8')).hexdigest()
+
+
+class AnvatoIE(InfoExtractor):
+    # Copied from anvplayer.min.js
+    _ANVACK_TABLE = {
+        'nbcu_nbcd_desktop_web_prod_93d8ead38ce2024f8f544b78306fbd15895ae5e6': 'NNemUkySjxLyPTKvZRiGntBIjEyK8uqicjMakIaQ',
+        'nbcu_nbcd_desktop_web_qa_1a6f01bdd0dc45a439043b694c8a031d': 'eSxJUbA2UUKBTXryyQ2d6NuM8oEqaPySvaPzfKNA',
+        'nbcu_nbcd_desktop_web_acc_eb2ff240a5d4ae9a63d4c297c32716b6c523a129': '89JR3RtUGbvKuuJIiKOMK0SoarLb5MUx8v89RcbP',
+        'nbcu_nbcd_watchvod_web_prod_e61107507180976724ec8e8319fe24ba5b4b60e1': 'Uc7dFt7MJ9GsBWB5T7iPvLaMSOt8BBxv4hAXk5vv',
+        'nbcu_nbcd_watchvod_web_qa_42afedba88a36203db5a4c09a5ba29d045302232': 'T12oDYVFP2IaFvxkmYMy5dKxswpLHtGZa4ZAXEi7',
+        'nbcu_nbcd_watchvod_web_acc_9193214448e2e636b0ffb78abacfd9c4f937c6ca': 'MmobcxUxMedUpohNWwXaOnMjlbiyTOBLL6d46ZpR',
+        'nbcu_local_monitor_web_acc_f998ad54eaf26acd8ee033eb36f39a7b791c6335': 'QvfIoPYrwsjUCcASiw3AIkVtQob2LtJHfidp9iWg',
+        'nbcu_cable_monitor_web_acc_a413759603e8bedfcd3c61b14767796e17834077': 'uwVPJLShvJWSs6sWEIuVem7MTF8A4IknMMzIlFto',
+        'nbcu_nbcd_mcpstage_web_qa_4c43a8f6e95a88dbb40276c0630ba9f693a63a4e': 'PxVYZVwjhgd5TeoPRxL3whssb5OUPnM3zyAzq8GY',
+        'nbcu_comcast_comcast_web_prod_074080762ad4ce956b26b43fb22abf153443a8c4': 'afnaRZfDyg1Z3WZHdupKfy6xrbAG2MHqe3VfuSwh',
+        'nbcu_comcast_comcast_web_qa_706103bb93ead3ef70b1de12a0e95e3c4481ade0': 'DcjsVbX9b3uoPlhdriIiovgFQZVxpISZwz0cx1ZK',
+        'nbcu_comcast_comcastcable_web_prod_669f04817536743563d7331c9293e59fbdbe3d07': '0RwMN2cWy10qhAhOscq3eK7aEe0wqnKt3vJ0WS4D',
+        'nbcu_comcast_comcastcable_web_qa_3d9d2d66219094127f0f6b09cc3c7bb076e3e1ca': '2r8G9DEya7PCqBceKZgrn2XkXgASjwLMuaFE1Aad',
+        'hearst_hearst_demo_web_stage_960726dfef3337059a01a78816e43b29ec04dfc7': 'cuZBPXTR6kSdoTCVXwk5KGA8rk3NrgGn4H6e9Dsp',
+        'anvato_mcpqa_demo_web_stage_18b55e00db5a13faa8d03ae6e41f6f5bcb15b922': 'IOaaLQ8ymqVyem14QuAvE5SndQynTcH5CrLkU2Ih',
+        'anvato_nextmedia_demo_web_stage_9787d56a02ff6b9f43e9a2b0920d8ca88beb5818': 'Pqu9zVzI1ApiIzbVA3VkGBEQHvdKSUuKpD6s2uaR',
+        'anvato_scripps_app_web_prod_0837996dbe373629133857ae9eb72e740424d80a': 'du1ccmn7RxzgizwbWU7hyUaGodNlJn7HtXI0WgXW',
+        'anvato_scripps_app_web_stage_360797e00fe2826be142155c4618cc52fce6c26c': '2PMrQ0BRoqCWl7nzphj0GouIMEh2mZYivAT0S1Su',
+        'fs2go_fs2go_go_all_prod_21934911ccfafc03a075894ead2260d11e2ddd24': 'RcuHlKikW2IJw6HvVoEkqq2UsuEJlbEl11pWXs4Q',
+        'fs2go_fs2go_go_web_prod_ead4b0eec7460c1a07783808db21b49cf1f2f9a7': '4K0HTT2u1zkQA2MaGaZmkLa1BthGSBdr7jllrhk5',
+        'fs2go_fs2go_go_web_stage_407585454a4400355d4391691c67f361': 'ftnc37VKRJBmHfoGGi3kT05bHyeJzilEzhKJCyl3',
+        'fs2go_fs2go_go_android_stage_44b714db6f8477f29afcba15a41e1d30': 'CtxpPvVpo6AbZGomYUhkKs7juHZwNml9b9J0J2gI',
+        'anvato_cbslocal_app_web_prod_547f3e49241ef0e5d30c79b2efbca5d92c698f67': 'Pw0XX5KBDsyRnPS0R2JrSrXftsy8Jnz5pAjaYC8s',
+        'anvato_cbslocal_app_web_stage_547a5f096594cd3e00620c6f825cad1096d28c80': '37OBUhX2uwNyKhhrNzSSNHSRPZpApC3trdqDBpuz',
+        'fs2go_att_att_web_prod_1042dddd089a05438b6a08f972941176f699ffd8': 'JLcF20JwYvpv6uAGcLWIaV12jKwaL1R8us4b6Zkg',
+        'fs2go_att_att_web_stage_807c5001955fc114a3331fe027ddc76e': 'gbu1oO1y0JiOFh4SUipt86P288JHpyjSqolrrT1x',
+        'fs2go_fs2go_tudor_web_prod_a7dd8e5a7cdc830cae55eae6f3e9fee5ee49eb9b': 'ipcp87VCEZXPPe868j3orLqzc03oTy7DXsGkAXXH',
+        'anvato_mhz_app_web_prod_b808218b30de7fdf60340cbd9831512bc1bf6d37': 'Stlm5Gs6BEhJLRTZHcNquyzxGqr23EuFmE5DCgjX',
+        'fs2go_charter_charter_web_stage_c2c6e5a68375a1bf00fff213d3ff8f61a835a54c': 'Lz4hbJp1fwL6jlcz4M2PMzghM4jp4aAmybtT5dPc',
+        'fs2go_charter_charter_web_prod_ebfe3b10f1af215a7321cd3d629e0b81dfa6fa8c': 'vUJsK345A1bVmyYDRhZX0lqFIgVXuqhmuyp1EtPK',
+        'anvato_epfox_app_web_prod_b3373168e12f423f41504f207000188daf88251b': 'GDKq1ixvX3MoBNdU5IOYmYa2DTUXYOozPjrCJnW7',
+        'anvato_epfox_app_web_stage_a3c2ce60f8f83ef374a88b68ee73a950f8ab87ce': '2jz2NH4BsXMaDsoJ5qkHMbcczAfIReo2eFYuVC1C',
+        'fs2go_verizon_verizon_web_stage_08e6df0354a4803f1b1f2428b5a9a382e8dbcd62': 'rKTVapNaAcmnUbGL4ZcuOoY4SE7VmZSQsblPFr7e',
+        'fs2go_verizon_verizon_web_prod_f909564cb606eff1f731b5e22e0928676732c445': 'qLSUuHerM3u9eNPzaHyUK52obai5MvE4XDJfqYe1',
+        'fs2go_foxcom_synd_web_stage_f7b9091f00ea25a4fdaaae77fca5b54cdc7e7043': '96VKF2vLd24fFiDfwPFpzM5llFN4TiIGAlodE0Re',
+        'fs2go_foxcom_synd_web_prod_0f2cdd64d87e4ab6a1d54aada0ff7a7c8387a064': 'agiPjbXEyEZUkbuhcnmVPhe9NNVbDjCFq2xkcx51',
+        'anvato_own_app_web_stage_1214ade5d28422c4dae9d03c1243aba0563c4dba': 'mzhamNac3swG4WsJAiUTacnGIODi6SWeVWk5D7ho',
+        'anvato_own_app_web_prod_944e162ed927ec3e9ed13eb68ed2f1008ee7565e': '9TSxh6G2TXOLBoYm9ro3LdNjjvnXpKb8UR8KoIP9',
+        'anvato_scripps_app_ftv_prod_a10a10468edd5afb16fb48171c03b956176afad1': 'COJ2i2UIPK7xZqIWswxe7FaVBOVgRkP1F6O6qGoH',
+        'anvato_scripps_app_ftv_stage_77d3ad2bdb021ec37ca2e35eb09acd396a974c9a': 'Q7nnopNLe2PPfGLOTYBqxSaRpl209IhqaEuDZi1F',
+        'anvato_univision_app_web_stage_551236ef07a0e17718c3995c35586b5ed8cb5031': 'D92PoLS6UitwxDRA191HUGT9OYcOjV6mPMa5wNyo',
+        'anvato_univision_app_web_prod_039a5c0a6009e637ae8ac906718a79911e0e65e1': '5mVS5u4SQjtw6NGw2uhMbKEIONIiLqRKck5RwQLR',
+        'nbcu_cnbc_springfield_ios_prod_670207fae43d6e9a94c351688851a2ce': 'M7fqCCIP9lW53oJbHs19OlJlpDrVyc2OL8gNeuTa',
+        'nbcu_cnbc_springfieldvod_ios_prod_7a5f04b1ceceb0e9c9e2264a44aa236e08e034c2': 'Yia6QbJahW0S7K1I0drksimhZb4UFq92xLBmmMvk',
+        'anvato_cox_app_web_prod_ce45cda237969f93e7130f50ee8bb6280c1484ab': 'cc0miZexpFtdoqZGvdhfXsLy7FXjRAOgb9V0f5fZ',
+        'anvato_cox_app_web_stage_c23dbe016a8e9d8c7101d10172b92434f6088bf9': 'yivU3MYHd2eDZcOfmLbINVtqxyecKTOp8OjOuoGJ',
+        'anvato_chnzero_app_web_stage_b1164d1352b579e792e542fddf13ee34c0eeb46b': 'A76QkXMmVH8lTCfU15xva1mZnSVcqeY4Xb22Kp7m',
+        'anvato_chnzero_app_web_prod_253d358928dc08ec161eda2389d53707288a730c': 'OA5QI3ZWZZkdtUEDqh28AH8GedsF6FqzJI32596b',
+        'anvato_discovery_vodpoc_web_stage_9fa7077b5e8af1f8355f65d4fb8d2e0e9d54e2b7': 'q3oT191tTQ5g3JCP67PkjLASI9s16DuWZ6fYmry3',
+        'anvato_discovery_vodpoc_web_prod_688614983167a1af6cdf6d76343fda10a65223c1': 'qRvRQCTVHd0VVOHsMvvfidyWmlYVrTbjby7WqIuK',
+        'nbcu_cnbc_springfieldvod_ftv_stage_826040aad1925a46ac5dfb4b3c5143e648c6a30d': 'JQaSb5a8Tz0PT4ti329DNmzDO30TnngTHmvX8Vua',
+        'nbcu_cnbc_springfield_ftv_stage_826040aad1925a46ac5dfb4b3c5143e648c6a30d': 'JQaSb5a8Tz0PT4ti329DNmzDO30TnngTHmvX8Vua',
+        'nbcu_nbcd_capture_web_stage_4dd9d585bfb984ebf856dee35db027b2465cc4ae': '0j1Ov4Vopyi2HpBZJYdL2m8ERJVGYh3nNpzPiO8F',
+        'nbcu_nbcd_watch3_android_prod_7712ca5fcf1c22f19ec1870a9650f9c37db22dcf': '3LN2UB3rPUAMu7ZriWkHky9vpLMXYha8JbSnxBlx',
+        'nbcu_nbcd_watchvod3_android_prod_0910a3a4692d57c0b5ff4316075bc5d096be45b9': 'mJagcQ2II30vUOAauOXne7ERwbf5S9nlB3IP17lQ',
+        'anvato_scripps_app_atv_prod_790deda22e16e71e83df58f880cd389908a45d52': 'CB6trI1mpoDIM5o54DNTsji90NDBQPZ4z4RqBNSH',
+        'nbcu_nbcd_watchv4_android_prod_ff67cef9cb409158c6f8c3533edddadd0b750507': 'j8CHQCUWjlYERj4NFRmUYOND85QNbHViH09UwuKm',
+        'nbcu_nbcd_watchvodv4_android_prod_a814d781609989dea6a629d50ae4c7ad8cc8e907': 'rkVnUXxdA9rawVLUlDQtMue9Y4Q7lFEaIotcUhjt',
+        'rvVKpA50qlOPLFxMjrCGf5pdkdQDm7qn': '1J7ZkY5Qz5lMLi93QOH9IveE7EYB3rLl',
+        'nbcu_dtv_local_web_prod_b266cf49defe255fd4426a97e27c09e513e9f82f': 'HuLnJDqzLa4saCzYMJ79zDRSQpEduw1TzjMNQu2b',
+        'nbcu_att_local_web_prod_4cef038b2d969a6b7d700a56a599040b6a619f67': 'Q0Em5VDc2KpydUrVwzWRXAwoNBulWUxCq2faK0AV',
+        'nbcu_dish_local_web_prod_c56dcaf2da2e9157a4266c82a78195f1dd570f6b': 'bC1LWmRz9ayj2AlzizeJ1HuhTfIaJGsDBnZNgoRg',
+        'nbcu_verizon_local_web_prod_88bebd2ce006d4ed980de8133496f9a74cb9b3e1': 'wzhDKJZpgvUSS1EQvpCQP8Q59qVzcPixqDGJefSk',
+        'nbcu_charter_local_web_prod_9ad90f7fc4023643bb718f0fe0fd5beea2382a50': 'PyNbxNhEWLzy1ZvWEQelRuIQY88Eub7xbSVRMdfT',
+        'nbcu_suddenlink_local_web_prod_20fb711725cac224baa1c1cb0b1c324d25e97178': '0Rph41lPXZbb3fqeXtHjjbxfSrNbtZp1Ygq7Jypa',
+        'nbcu_wow_local_web_prod_652d9ce4f552d9c2e7b5b1ed37b8cb48155174ad': 'qayIBZ70w1dItm2zS42AptXnxW15mkjRrwnBjMPv',
+        'nbcu_centurylink_local_web_prod_2034402b029bf3e837ad46814d9e4b1d1345ccd5': 'StePcPMkjsX51PcizLdLRMzxMEl5k2FlsMLUNV4k',
+        'nbcu_atlanticbrd_local_web_prod_8d5f5ecbf7f7b2f5e6d908dd75d90ae3565f682e': 'NtYLb4TFUS0pRs3XTkyO5sbVGYjVf17bVbjaGscI',
+        'nbcu_nbcd_watchvod_web_dev_08bc05699be47c4f31d5080263a8cfadc16d0f7c': 'hwxi2dgDoSWgfmVVXOYZm14uuvku4QfopstXckhr',
+        'anvato_nextmedia_app_web_prod_a4fa8c7204aa65e71044b57aaf63711980cfe5a0': 'tQN1oGPYY1nM85rJYePWGcIb92TG0gSqoVpQTWOw',
+        'anvato_mcp_lin_web_prod_4c36fbfd4d8d8ecae6488656e21ac6d1ac972749': 'GUXNf5ZDX2jFUpu4WT2Go4DJ5nhUCzpnwDRRUx1K',
+        'anvato_mcp_univision_web_prod_37fe34850c99a3b5cdb71dab10a417dd5cdecafa': 'bLDYF8JqfG42b7bwKEgQiU9E2LTIAtnKzSgYpFUH',
+        'anvato_mcp_fs2go_web_prod_c7b90a93e171469cdca00a931211a2f556370d0a': 'icgGoYGipQMMSEvhplZX1pwbN69srwKYWksz3xWK',
+        'anvato_mcp_sps_web_prod_54bdc90dd6ba21710e9f7074338365bba28da336': 'fA2iQdI7RDpynqzQYIpXALVS83NTPr8LLFK4LFsu',
+        'anvato_mcp_anv_web_prod_791407490f4c1ef2a4bcb21103e0cb1bcb3352b3': 'rMOUZqe9lwcGq2mNgG3EDusm6lKgsUnczoOX3mbg',
+        'anvato_mcp_gray_web_prod_4c10f067c393ed8fc453d3930f8ab2b159973900': 'rMOUZqe9lwcGq2mNgG3EDusm6lKgsUnczoOX3mbg',
+        'anvato_mcp_hearst_web_prod_5356c3de0fc7c90a3727b4863ca7fec3a4524a99': 'P3uXJ0fXXditBPCGkfvlnVScpPEfKmc64Zv7ZgbK',
+        'anvato_mcp_cbs_web_prod_02f26581ff80e5bda7aad28226a8d369037f2cbe': 'mGPvo5ZA5SgjOFAPEPXv7AnOpFUICX8hvFQVz69n',
+        'anvato_mcp_telemundo_web_prod_c5278d51ad46fda4b6ca3d0ea44a7846a054f582': 'qyT6PXXLjVNCrHaRVj0ugAhalNRS7Ee9BP7LUokD',
+        'nbcu_nbcd_watchvodv4_web_stage_4108362fba2d4ede21f262fea3c4162cbafd66c7': 'DhaU5lj0W2gEdcSSsnxURq8t7KIWtJfD966crVDk',
+        'anvato_scripps_app_ios_prod_409c41960c60b308db43c3cc1da79cab9f1c3d93': 'WPxj5GraLTkYCyj3M7RozLqIycjrXOEcDGFMIJPn',
+        'EZqvRyKBJLrgpClDPDF8I7Xpdp40Vx73': '4OxGd2dEakylntVKjKF0UK9PDPYB6A9W',
+        'M2v78QkpleXm9hPp9jUXI63x5vA6BogR': 'ka6K32k7ZALmpINkjJUGUo0OE42Md1BQ',
+        'nbcu_nbcd_desktop_web_prod_93d8ead38ce2024f8f544b78306fbd15895ae5e6_secure': 'NNemUkySjxLyPTKvZRiGntBIjEyK8uqicjMakIaQ'
+    }
+
+    _AUTH_KEY = b'\x31\xc2\x42\x84\x9e\x73\xa0\xce'
+
+    def __init__(self, *args, **kwargs):
+        super(AnvatoIE, self).__init__(*args, **kwargs)
+        self.__server_time = None
+
+    def _server_time(self, access_key, video_id):
+        if self.__server_time is not None:
+            return self.__server_time
+
+        self.__server_time = int(self._download_json(
+            self._api_prefix(access_key) + 'server_time?anvack=' + access_key, video_id,
+            note='Fetching server time')['server_time'])
+
+        return self.__server_time
+
+    def _api_prefix(self, access_key):
+        return 'https://tkx2-%s.anvato.net/rest/v2/' % ('prod' if 'prod' in access_key else 'stage')
+
+    def _get_video_json(self, access_key, video_id):
+        # See et() in anvplayer.min.js, which is an alias of getVideoJSON()
+        video_data_url = self._api_prefix(access_key) + 'mcp/video/%s?anvack=%s' % (video_id, access_key)
+        server_time = self._server_time(access_key, video_id)
+        input_data = '%d~%s~%s' % (server_time, md5_text(video_data_url), md5_text(server_time))
+
+        auth_secret = intlist_to_bytes(aes_encrypt(
+            bytes_to_intlist(input_data[:64]), bytes_to_intlist(self._AUTH_KEY)))
+
+        video_data_url += '&X-Anvato-Adst-Auth=' + base64.b64encode(auth_secret).decode('ascii')
+        anvrid = md5_text(time.time() * 1000 * random.random())[:30]
+        payload = {
+            'api': {
+                'anvrid': anvrid,
+                'anvstk': md5_text('%s|%s|%d|%s' % (
+                    access_key, anvrid, server_time, self._ANVACK_TABLE[access_key])),
+                'anvts': server_time,
+            },
+        }
+
+        return self._download_json(
+            video_data_url, video_id, transform_source=strip_jsonp,
+            data=json.dumps(payload).encode('utf-8'))
+
+    def _extract_anvato_videos(self, webpage, video_id):
+        anvplayer_data = self._parse_json(self._html_search_regex(
+            r'<script[^>]+data-anvp=\'([^\']+)\'', webpage,
+            'Anvato player data'), video_id)
+
+        video_id = anvplayer_data['video']
+        access_key = anvplayer_data['accessKey']
+
+        video_data = self._get_video_json(access_key, video_id)
+
+        formats = []
+        for published_url in video_data['published_urls']:
+            video_url = published_url['embed_url']
+            ext = determine_ext(video_url)
+
+            if ext == 'smil':
+                formats.extend(self._extract_smil_formats(video_url, video_id))
+                continue
+
+            tbr = int_or_none(published_url.get('kbps'))
+            a_format = {
+                'url': video_url,
+                'format_id': ('-'.join(filter(None, ['http', published_url.get('cdn_name')]))).lower(),
+                'tbr': tbr if tbr != 0 else None,
+            }
+
+            if ext == 'm3u8':
+                # Not using _extract_m3u8_formats here as individual media
+                # playlists are also included in published_urls.
+                if tbr is None:
+                    formats.append(self._m3u8_meta_format(video_url, ext='mp4', m3u8_id='hls'))
+                    continue
+                else:
+                    a_format.update({
+                        'format_id': '-'.join(filter(None, ['hls', compat_str(tbr)])),
+                        'ext': 'mp4',
+                    })
+            elif ext == 'mp3':
+                a_format['vcodec'] = 'none'
+            else:
+                a_format.update({
+                    'width': int_or_none(published_url.get('width')),
+                    'height': int_or_none(published_url.get('height')),
+                })
+            formats.append(a_format)
+
+        self._sort_formats(formats)
+
+        subtitles = {}
+        for caption in video_data.get('captions', []):
+            a_caption = {
+                'url': caption['url'],
+                'ext': 'tt' if caption.get('format') == 'SMPTE-TT' else None
+            }
+            subtitles.setdefault(caption['language'], []).append(a_caption)
+
+        return {
+            'id': video_id,
+            'formats': formats,
+            'title': video_data.get('def_title'),
+            'description': video_data.get('def_description'),
+            'categories': video_data.get('categories'),
+            'thumbnail': video_data.get('thumbnail'),
+            'subtitles': subtitles,
+        }

From 612b5f403e33d5c164b5c0bbad9f01ef6d38d050 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 21 May 2016 13:38:01 +0800
Subject: [PATCH 4978/6219] [jwplatform] Improved m3u8 and rtmp support

Changes made for SendtoNewsIE. Part of #9522
---
 youtube_dl/extractor/jwplatform.py | 24 +++++++++++++++++++-----
 1 file changed, 19 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/jwplatform.py b/youtube_dl/extractor/jwplatform.py
index 8a5e562db..0aa6fc750 100644
--- a/youtube_dl/extractor/jwplatform.py
+++ b/youtube_dl/extractor/jwplatform.py
@@ -5,33 +5,47 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
+    determine_ext,
     float_or_none,
     int_or_none,
 )
 
 
 class JWPlatformBaseIE(InfoExtractor):
-    def _parse_jwplayer_data(self, jwplayer_data, video_id, require_title=True):
+    def _parse_jwplayer_data(self, jwplayer_data, video_id, require_title=True, m3u8_id=None, rtmp_params=None):
         video_data = jwplayer_data['playlist'][0]
 
         formats = []
         for source in video_data['sources']:
             source_url = self._proto_relative_url(source['file'])
             source_type = source.get('type') or ''
-            if source_type in ('application/vnd.apple.mpegurl', 'hls'):
+            if source_type in ('application/vnd.apple.mpegurl', 'hls') or determine_ext(source_url) == 'm3u8':
                 formats.extend(self._extract_m3u8_formats(
-                    source_url, video_id, 'mp4', 'm3u8_native', fatal=False))
+                    source_url, video_id, 'mp4', 'm3u8_native', m3u8_id=m3u8_id, fatal=False))
             elif source_type.startswith('audio'):
                 formats.append({
                     'url': source_url,
                     'vcodec': 'none',
                 })
             else:
-                formats.append({
+                a_format = {
                     'url': source_url,
                     'width': int_or_none(source.get('width')),
                     'height': int_or_none(source.get('height')),
-                })
+                }
+                if source_url.startswith('rtmp'):
+                    # See com/longtailvideo/jwplayer/media/RTMPMediaProvider.as
+                    # of jwplayer.flash.swf
+                    rtmp_url, prefix, play_path = re.split(
+                        r'((?:mp4|mp3|flv):)', source_url, 1)
+                    a_format.update({
+                        'url': rtmp_url,
+                        'ext': 'flv',
+                        'play_path': prefix + play_path,
+                    })
+                    if rtmp_params:
+                        a_format.update(rtmp_params)
+                formats.append(a_format)
         self._sort_formats(formats)
 
         subtitles = {}

From 5ce3d5bd1b0933a26a4224643cf8d3ad14330e17 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 21 May 2016 13:39:42 +0800
Subject: [PATCH 4979/6219] [sendtonews] Add new extractor

Used in CBSLocal. Part of #9522
---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/sendtonews.py | 86 ++++++++++++++++++++++++++++++
 2 files changed, 87 insertions(+)
 create mode 100644 youtube_dl/extractor/sendtonews.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 5b96a086d..8352b3c3a 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -670,6 +670,7 @@ from .screencastomatic import ScreencastOMaticIE
 from .screenjunkies import ScreenJunkiesIE
 from .screenwavemedia import ScreenwaveMediaIE, TeamFourIE
 from .senateisvp import SenateISVPIE
+from .sendtonews import SendtoNewsIE
 from .servingsys import ServingSysIE
 from .sexu import SexuIE
 from .shahid import ShahidIE
diff --git a/youtube_dl/extractor/sendtonews.py b/youtube_dl/extractor/sendtonews.py
new file mode 100644
index 000000000..1c636f672
--- /dev/null
+++ b/youtube_dl/extractor/sendtonews.py
@@ -0,0 +1,86 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .jwplatform import JWPlatformBaseIE
+from ..compat import compat_parse_qs
+from ..utils import (
+    ExtractorError,
+    parse_duration,
+)
+
+
+class SendtoNewsIE(JWPlatformBaseIE):
+    _VALID_URL = r'https?://embed\.sendtonews\.com/player/embed\.php\?(?P<query>[^#]+)'
+
+    _TEST = {
+        # From http://cleveland.cbslocal.com/2016/05/16/indians-score-season-high-15-runs-in-blowout-win-over-reds-rapid-reaction/
+        'url': 'http://embed.sendtonews.com/player/embed.php?SK=GxfCe0Zo7D&MK=175909&PK=5588&autoplay=on&sound=yes',
+        'info_dict': {
+            'id': 'GxfCe0Zo7D-175909-5588',
+            'ext': 'mp4',
+            'title': 'Recap: CLE 15, CIN 6',
+            'description': '5/16/16: Indians\' bats explode for 15 runs in a win',
+            'duration': 49,
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }
+
+    _URL_TEMPLATE = '//embed.sendtonews.com/player/embed.php?SK=%s&MK=%s&PK=%s'
+
+    @classmethod
+    def _extract_url(cls, webpage):
+        mobj = re.search(r'''(?x)<script[^>]+src=([\'"])
+            (?:https?:)?//embed\.sendtonews\.com/player/responsiveembed\.php\?
+                .*\bSC=(?P<SC>[0-9a-zA-Z-]+).*
+            \1>''', webpage)
+        if mobj:
+            sk, mk, pk = mobj.group('SC').split('-')
+            return cls._URL_TEMPLATE % (sk, mk, pk)
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        params = compat_parse_qs(mobj.group('query'))
+
+        if 'SK' not in params or 'MK' not in params or 'PK' not in params:
+            raise ExtractorError('Invalid URL', expected=True)
+
+        video_id = '-'.join([params['SK'][0], params['MK'][0], params['PK'][0]])
+
+        webpage = self._download_webpage(url, video_id)
+
+        jwplayer_data_str = self._search_regex(
+            r'jwplayer\("[^"]+"\)\.setup\((.+?)\);', webpage, 'JWPlayer data')
+        js_vars = {
+            'w': 1024,
+            'h': 768,
+            'modeVar': 'html5',
+        }
+        for name, val in js_vars.items():
+            js_val = '%d' % val if isinstance(val, int) else '"%s"' % val
+            jwplayer_data_str = jwplayer_data_str.replace(':%s,' % name, ':%s,' % js_val)
+
+        info_dict = self._parse_jwplayer_data(
+            self._parse_json(jwplayer_data_str, video_id),
+            video_id, require_title=False, rtmp_params={'no_resume': True})
+
+        title = self._html_search_regex(
+            r'<div[^>]+class="embedTitle">([^<]+)</div>', webpage, 'title')
+        description = self._html_search_regex(
+            r'<div[^>]+class="embedSubTitle">([^<]+)</div>', webpage,
+            'description', fatal=False)
+        duration = parse_duration(self._html_search_regex(
+            r'<div[^>]+class="embedDetails">([0-9:]+)', webpage,
+            'duration', fatal=False))
+
+        info_dict.update({
+            'title': title,
+            'description': description,
+            'duration': duration,
+        })
+
+        return info_dict

From 661d46b28f6de2772fc642c36b505a3c7b9a3b10 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 21 May 2016 13:40:45 +0800
Subject: [PATCH 4980/6219] [cbslocal] Add new extractor (closes #9522)

---
 youtube_dl/extractor/cbslocal.py   | 84 ++++++++++++++++++++++++++++++
 youtube_dl/extractor/extractors.py |  1 +
 2 files changed, 85 insertions(+)
 create mode 100644 youtube_dl/extractor/cbslocal.py

diff --git a/youtube_dl/extractor/cbslocal.py b/youtube_dl/extractor/cbslocal.py
new file mode 100644
index 000000000..74adb38a6
--- /dev/null
+++ b/youtube_dl/extractor/cbslocal.py
@@ -0,0 +1,84 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import calendar
+import datetime
+
+from .anvato import AnvatoIE
+from .sendtonews import SendtoNewsIE
+from ..compat import compat_urlparse
+
+
+class CBSLocalIE(AnvatoIE):
+    _VALID_URL = r'https?://[a-z]+\.cbslocal\.com/\d+/\d+/\d+/(?P<id>[0-9a-z-]+)'
+
+    _TESTS = [{
+        # Anvato backend
+        'url': 'http://losangeles.cbslocal.com/2016/05/16/safety-advocates-say-fatal-car-seat-failures-are-public-health-crisis',
+        'md5': 'f0ee3081e3843f575fccef901199b212',
+        'info_dict': {
+            'id': '3401037',
+            'ext': 'mp4',
+            'title': 'Safety Advocates Say Fatal Car Seat Failures Are \'Public Health Crisis\'',
+            'description': 'Collapsing seats have been the focus of scrutiny for decades, though experts say remarkably little has been done to address the issue. Randy Paige reports.',
+            'thumbnail': 're:^https?://.*',
+            'timestamp': 1463440500,
+            'upload_date': '20160516',
+            'subtitles': {
+                'en': 'mincount:5',
+            },
+            'categories': [
+                'Stations\\Spoken Word\\KCBSTV',
+                'Syndication\\MSN',
+                'Syndication\\NDN',
+                'Syndication\\AOL',
+                'Syndication\\Yahoo',
+                'Syndication\\Tribune',
+                'Syndication\\Curb.tv',
+                'Content\\News'
+            ],
+        },
+    }, {
+        # SendtoNews embed
+        'url': 'http://cleveland.cbslocal.com/2016/05/16/indians-score-season-high-15-runs-in-blowout-win-over-reds-rapid-reaction/',
+        'info_dict': {
+            'id': 'GxfCe0Zo7D-175909-5588',
+            'ext': 'mp4',
+            'title': 'Recap: CLE 15, CIN 6',
+            'description': '5/16/16: Indians\' bats explode for 15 runs in a win',
+            'upload_date': '20160516',
+            'timestamp': 1463433840,
+            'duration': 49,
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }]
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+
+        sendtonews_url = SendtoNewsIE._extract_url(webpage)
+        if sendtonews_url:
+            info_dict = {
+                '_type': 'url_transparent',
+                'url': compat_urlparse.urljoin(url, sendtonews_url),
+            }
+        else:
+            info_dict = self._extract_anvato_videos(webpage, display_id)
+
+        time_str = self._html_search_regex(
+            r'class="entry-date">([^<]+)<', webpage, 'released date', fatal=False)
+        timestamp = None
+        if time_str:
+            timestamp = calendar.timegm(datetime.datetime.strptime(
+                time_str, '%b %d, %Y %I:%M %p').timetuple())
+
+        info_dict.update({
+            'display_id': display_id,
+            'timestamp': timestamp,
+        })
+
+        return info_dict
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 8352b3c3a..c93cd2765 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -111,6 +111,7 @@ from .cbc import (
     CBCPlayerIE,
 )
 from .cbs import CBSIE
+from .cbslocal import CBSLocalIE
 from .cbsinteractive import CBSInteractiveIE
 from .cbsnews import (
     CBSNewsIE,

From 115c65793af4c56c8f1986d2640105fc7e760c13 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 21 May 2016 13:50:38 +0800
Subject: [PATCH 4981/6219] [jwplatform] Don't fail with RTMP URLs without
 mp4:, mp3: or flv:

---
 youtube_dl/extractor/jwplatform.py | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/jwplatform.py b/youtube_dl/extractor/jwplatform.py
index 0aa6fc750..fa6f335e1 100644
--- a/youtube_dl/extractor/jwplatform.py
+++ b/youtube_dl/extractor/jwplatform.py
@@ -34,15 +34,18 @@ class JWPlatformBaseIE(InfoExtractor):
                     'height': int_or_none(source.get('height')),
                 }
                 if source_url.startswith('rtmp'):
+                    a_format['ext'] = 'flv',
+
                     # See com/longtailvideo/jwplayer/media/RTMPMediaProvider.as
                     # of jwplayer.flash.swf
-                    rtmp_url, prefix, play_path = re.split(
+                    rtmp_url_parts = re.split(
                         r'((?:mp4|mp3|flv):)', source_url, 1)
-                    a_format.update({
-                        'url': rtmp_url,
-                        'ext': 'flv',
-                        'play_path': prefix + play_path,
-                    })
+                    if len(rtmp_url_parts) == 3:
+                        rtmp_url, prefix, play_path = rtmp_url_parts
+                        a_format.update({
+                            'url': rtmp_url,
+                            'play_path': prefix + play_path,
+                        })
                     if rtmp_params:
                         a_format.update(rtmp_params)
                 formats.append(a_format)

From 4c718d3c50b8d80bf07e44c73a5bdcd98544388f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 21 May 2016 17:37:35 +0200
Subject: [PATCH 4982/6219] [rtve] Recognize 'filmoteca' URLs

---
 youtube_dl/extractor/rtve.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/rtve.py b/youtube_dl/extractor/rtve.py
index edd0d108e..f11e3588b 100644
--- a/youtube_dl/extractor/rtve.py
+++ b/youtube_dl/extractor/rtve.py
@@ -64,7 +64,7 @@ def _decrypt_url(png):
 class RTVEALaCartaIE(InfoExtractor):
     IE_NAME = 'rtve.es:alacarta'
     IE_DESC = 'RTVE a la carta'
-    _VALID_URL = r'https?://www\.rtve\.es/(m/)?alacarta/videos/[^/]+/[^/]+/(?P<id>\d+)'
+    _VALID_URL = r'https?://www\.rtve\.es/(m/)?(alacarta/videos|filmoteca)/[^/]+/[^/]+/(?P<id>\d+)'
 
     _TESTS = [{
         'url': 'http://www.rtve.es/alacarta/videos/balonmano/o-swiss-cup-masculina-final-espana-suecia/2491869/',
@@ -87,6 +87,9 @@ class RTVEALaCartaIE(InfoExtractor):
     }, {
         'url': 'http://www.rtve.es/m/alacarta/videos/cuentame-como-paso/cuentame-como-paso-t16-ultimo-minuto-nuestra-vida-capitulo-276/2969138/?media=tve',
         'only_matching': True,
+    }, {
+        'url': 'http://www.rtve.es/filmoteca/no-do/not-1-introduccion-primer-noticiario-espanol/1465256/',
+        'only_matching': True,
     }]
 
     def _real_initialize(self):

From c8cc3745fbb34d39f4dfb0c3facb6fa9278af93c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 21 May 2016 21:18:59 +0200
Subject: [PATCH 4983/6219] release 2016.05.21

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 docs/supportedsites.md    | 7 +++++++
 youtube_dl/version.py     | 2 +-
 3 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 7024fc729..00cc634e3 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.05.16*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.05.16**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.05.21*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.05.21**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.05.16
+[debug] youtube-dl version 2016.05.21
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 29db13883..cd6bfa51c 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -16,6 +16,8 @@
  - **9gag**
  - **abc.net.au**
  - **Abc7News**
+ - **abcnews**
+ - **abcnews:video**
  - **AcademicEarth:Course**
  - **acast**
  - **acast:channel**
@@ -104,6 +106,7 @@
  - **CBCPlayer**
  - **CBS**
  - **CBSInteractive**
+ - **CBSLocal**
  - **CBSNews**: CBS News
  - **CBSNewsLiveVideo**: CBS News Live Videos
  - **CBSSports**
@@ -213,6 +216,7 @@
  - **Flickr**
  - **Folketinget**: Folketinget (ft.dk; Danish parliament)
  - **FootyRoom**
+ - **Formula1**
  - **FOX**
  - **Foxgay**
  - **FoxNews**: Fox News and Fox Business Video
@@ -316,6 +320,7 @@
  - **la7.tv**
  - **Laola1Tv**
  - **Le**: 乐视网
+ - **Learnr**
  - **Lecture2Go**
  - **Lemonde**
  - **LePlaylist**
@@ -331,6 +336,7 @@
  - **livestream**
  - **livestream:original**
  - **LnkGo**
+ - **LocalNews8**
  - **LoveHomePorn**
  - **lrt.lt**
  - **lynda**: lynda.com videos
@@ -556,6 +562,7 @@
  - **ScreenJunkies**
  - **ScreenwaveMedia**
  - **SenateISVP**
+ - **SendtoNews**
  - **ServingSys**
  - **Sexu**
  - **Shahid**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 5a0fdd6ce..4bdb5f352 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.05.16'
+__version__ = '2016.05.21'

From 7e642e4fd68f9418ecdcb852aa34a4e49c41e58b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 21 May 2016 21:24:53 +0200
Subject: [PATCH 4984/6219] release: check for pandoc

Abort releaseing if pandoc is missing.
(pandoc was not included in my essential app database, and thus missing on my new machine.)
---
 devscripts/release.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/devscripts/release.sh b/devscripts/release.sh
index 8dea55dbb..8b37152a7 100755
--- a/devscripts/release.sh
+++ b/devscripts/release.sh
@@ -33,6 +33,7 @@ if [ ! -z "`git status --porcelain | grep -v CHANGELOG`" ]; then echo 'ERROR: th
 useless_files=$(find youtube_dl -type f -not -name '*.py')
 if [ ! -z "$useless_files" ]; then echo "ERROR: Non-.py files in youtube_dl: $useless_files"; exit 1; fi
 if [ ! -f "updates_key.pem" ]; then echo 'ERROR: updates_key.pem missing'; exit 1; fi
+if ! type pandoc 2>/dev/null; then echo 'ERROR: pandoc is missing'; exit 1; fi
 
 /bin/echo -e "\n### First of all, testing..."
 make clean

From 598869afb139707c7064a6c8397bbcf09b2b43f5 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 21 May 2016 21:27:00 +0200
Subject: [PATCH 4985/6219] release 2016.05.21.1

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 youtube_dl/version.py     | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 00cc634e3..7f8650553 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.05.21*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.05.21**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.05.21.1*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.05.21.1**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.05.21
+[debug] youtube-dl version 2016.05.21.1
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 4bdb5f352..0a2e43d05 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.05.21'
+__version__ = '2016.05.21.1'

From d2fee3c99e9d1c8eba5bd55aa3a9dd5702b23b34 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 21 May 2016 21:46:42 +0200
Subject: [PATCH 4986/6219] release.sh: also check for python3 rsa module

---
 devscripts/release.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/devscripts/release.sh b/devscripts/release.sh
index 8b37152a7..7dd391b38 100755
--- a/devscripts/release.sh
+++ b/devscripts/release.sh
@@ -33,7 +33,8 @@ if [ ! -z "`git status --porcelain | grep -v CHANGELOG`" ]; then echo 'ERROR: th
 useless_files=$(find youtube_dl -type f -not -name '*.py')
 if [ ! -z "$useless_files" ]; then echo "ERROR: Non-.py files in youtube_dl: $useless_files"; exit 1; fi
 if [ ! -f "updates_key.pem" ]; then echo 'ERROR: updates_key.pem missing'; exit 1; fi
-if ! type pandoc 2>/dev/null; then echo 'ERROR: pandoc is missing'; exit 1; fi
+if ! type pandoc >/dev/null 2>/dev/null; then echo 'ERROR: pandoc is missing'; exit 1; fi
+if ! python3 -c 'import rsa' 2>/dev/null; then echo 'ERROR: python3-rsa is missing'; exit 1; fi
 
 /bin/echo -e "\n### First of all, testing..."
 make clean

From e03b35b8f998692aa853c6dbd498655fc831f9e7 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 21 May 2016 21:47:39 +0200
Subject: [PATCH 4987/6219] release 2016.05.21.2

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 youtube_dl/version.py     | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 7f8650553..2d80d45b6 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.05.21.1*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.05.21.1**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.05.21.2*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.05.21.2**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.05.21.1
+[debug] youtube-dl version 2016.05.21.2
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 0a2e43d05..522a56669 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.05.21.1'
+__version__ = '2016.05.21.2'

From 0db9a05f88cbbe6709da3875b798634dc536536b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 May 2016 02:15:56 +0600
Subject: [PATCH 4988/6219] [periscope:user] Adapt to layout changes (Closes
 #9563)

---
 youtube_dl/extractor/periscope.py | 23 +++++++++++++++--------
 1 file changed, 15 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/periscope.py b/youtube_dl/extractor/periscope.py
index 0a4bc761d..b2008decc 100644
--- a/youtube_dl/extractor/periscope.py
+++ b/youtube_dl/extractor/periscope.py
@@ -2,7 +2,10 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..utils import parse_iso8601
+from ..utils import (
+    parse_iso8601,
+    unescapeHTML,
+)
 
 
 class PeriscopeIE(InfoExtractor):
@@ -92,6 +95,7 @@ class PeriscopeUserIE(InfoExtractor):
         'info_dict': {
             'id': 'LularoeHusbandMike',
             'title': 'LULAROE HUSBAND MIKE',
+            'description': 'md5:6cf4ec8047768098da58e446e82c82f0',
         },
         # Periscope only shows videos in the last 24 hours, so it's possible to
         # get 0 videos
@@ -103,16 +107,19 @@ class PeriscopeUserIE(InfoExtractor):
 
         webpage = self._download_webpage(url, user_id)
 
-        broadcast_data = self._parse_json(self._html_search_meta(
-            'broadcast-data', webpage, default='{}'), user_id)
-        username = broadcast_data.get('user', {}).get('display_name')
-        user_broadcasts = self._parse_json(
-            self._html_search_meta('user-broadcasts', webpage, default='{}'),
+        data_store = self._parse_json(
+            unescapeHTML(self._search_regex(
+                r'data-store=(["\'])(?P<data>.+?)\1',
+                webpage, 'data store', default='{}', group='data')),
             user_id)
 
+        user = data_store.get('User', {}).get('user', {})
+        title = user.get('display_name') or user.get('username')
+        description = user.get('description')
+
         entries = [
             self.url_result(
                 'https://www.periscope.tv/%s/%s' % (user_id, broadcast['id']))
-            for broadcast in user_broadcasts.get('broadcasts', [])]
+            for broadcast in data_store.get('UserBroadcastHistory', {}).get('broadcasts', [])]
 
-        return self.playlist_result(entries, user_id, username)
+        return self.playlist_result(entries, user_id, title, description)

From 92d221ad4858a62143ce5645c56261b26023308e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 May 2016 02:39:15 +0600
Subject: [PATCH 4989/6219] [periscope] Update uploader_id (Closes #9565)

---
 youtube_dl/extractor/periscope.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/periscope.py b/youtube_dl/extractor/periscope.py
index b2008decc..c23b314e7 100644
--- a/youtube_dl/extractor/periscope.py
+++ b/youtube_dl/extractor/periscope.py
@@ -45,8 +45,11 @@ class PeriscopeIE(InfoExtractor):
         broadcast = broadcast_data['broadcast']
         status = broadcast['status']
 
-        uploader = broadcast.get('user_display_name') or broadcast_data.get('user', {}).get('display_name')
-        uploader_id = broadcast.get('user_id') or broadcast_data.get('user', {}).get('id')
+        user = broadcast_data.get('user', {})
+
+        uploader = broadcast.get('user_display_name') or user.get('display_name')
+        uploader_id = (broadcast.get('username') or user.get('username') or
+                       broadcast.get('user_id') or user.get('id'))
 
         title = '%s - %s' % (uploader, status) if uploader else status
         state = broadcast.get('state').lower()

From c8831015f41879e0d8788c228acf52579e6cf12b Mon Sep 17 00:00:00 2001
From: TRox1972 <TRox1972@users.noreply.github.com>
Date: Sat, 21 May 2016 18:51:34 +0200
Subject: [PATCH 4990/6219] [ComedyCentral] Add support for tosh.cc.com and
 cc.com/video-clips

---
 youtube_dl/extractor/comedycentral.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/comedycentral.py b/youtube_dl/extractor/comedycentral.py
index 0c59102e0..830073834 100644
--- a/youtube_dl/extractor/comedycentral.py
+++ b/youtube_dl/extractor/comedycentral.py
@@ -44,10 +44,10 @@ class ComedyCentralShowsIE(MTVServicesInfoExtractor):
     #                     or: http://www.colbertnation.com/the-colbert-report-collections/422008/festival-of-lights/79524
     _VALID_URL = r'''(?x)^(:(?P<shortname>tds|thedailyshow)
                       |https?://(:www\.)?
-                          (?P<showname>thedailyshow|thecolbertreport)\.(?:cc\.)?com/
+                          (?P<showname>thedailyshow|thecolbertreport|tosh)\.(?:cc\.)?com/
                          ((?:full-)?episodes/(?:[0-9a-z]{6}/)?(?P<episode>.*)|
                           (?P<clip>
-                              (?:(?:guests/[^/]+|videos|video-playlists|special-editions|news-team/[^/]+)/[^/]+/(?P<videotitle>[^/?#]+))
+                              (?:(?:guests/[^/]+|videos|video-clips|video-playlists|special-editions|news-team/[^/]+)/[^/]+/(?P<videotitle>[^/?#]+))
                               |(the-colbert-report-(videos|collections)/(?P<clipID>[0-9]+)/[^/]*/(?P<cntitle>.*?))
                               |(watch/(?P<date>[^/]*)/(?P<tdstitle>.*))
                           )|

From 0150a00f333371b366ff10871458e0b071f20ee3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 May 2016 02:58:41 +0600
Subject: [PATCH 4991/6219] [cc] Add test for tosh.cc (Closes #9566)

---
 youtube_dl/extractor/comedycentral.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/comedycentral.py b/youtube_dl/extractor/comedycentral.py
index 830073834..2b6aaa3aa 100644
--- a/youtube_dl/extractor/comedycentral.py
+++ b/youtube_dl/extractor/comedycentral.py
@@ -47,7 +47,7 @@ class ComedyCentralShowsIE(MTVServicesInfoExtractor):
                           (?P<showname>thedailyshow|thecolbertreport|tosh)\.(?:cc\.)?com/
                          ((?:full-)?episodes/(?:[0-9a-z]{6}/)?(?P<episode>.*)|
                           (?P<clip>
-                              (?:(?:guests/[^/]+|videos|video-clips|video-playlists|special-editions|news-team/[^/]+)/[^/]+/(?P<videotitle>[^/?#]+))
+                              (?:(?:guests/[^/]+|videos|video-(?:clips|playlists)|special-editions|news-team/[^/]+)/[^/]+/(?P<videotitle>[^/?#]+))
                               |(the-colbert-report-(videos|collections)/(?P<clipID>[0-9]+)/[^/]*/(?P<cntitle>.*?))
                               |(watch/(?P<date>[^/]*)/(?P<tdstitle>.*))
                           )|
@@ -129,6 +129,9 @@ class ComedyCentralShowsIE(MTVServicesInfoExtractor):
     }, {
         'url': 'http://thedailyshow.cc.com/news-team/michael-che/7wnfel/we-need-to-talk-about-israel',
         'only_matching': True,
+    }, {
+        'url': 'http://tosh.cc.com/video-clips/68g93d/twitter-users-share-summer-plans',
+        'only_matching': True,
     }]
 
     _available_formats = ['3500', '2200', '1700', '1200', '750', '400']

From 898f4b49ccc828f86a075d656aa9a1e1428e538c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 May 2016 06:47:22 +0600
Subject: [PATCH 4992/6219] [theplatform] Add _extract_urls

---
 youtube_dl/extractor/theplatform.py | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index a25417f94..02dbef913 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -151,6 +151,22 @@ class ThePlatformIE(ThePlatformBaseIE):
         'only_matching': True,
     }]
 
+    @classmethod
+    def _extract_urls(cls, webpage):
+        m = re.search(
+            r'''(?x)
+                    <meta\s+
+                        property=(["'])(?:og:video(?::(?:secure_)?url)?|twitter:player)\1\s+
+                        content=(["'])(?P<url>https?://player\.theplatform\.com/p/.+?)\2
+            ''', webpage)
+        if m:
+            return [m.group('url')]
+
+        matches = re.findall(
+            r'<(?:iframe|script)[^>]+src=(["\'])((?:https?:)?//player\.theplatform\.com/p/.+?)\1', webpage)
+        if matches:
+            return list(zip(*matches))[1]
+
     @staticmethod
     def _sign_url(url, sig_key, sig_secret, life=600, include_qs=False):
         flags = '10' if include_qs else '00'

From 4d8819d2492e10f10bd09490f8f203d2f5e2cac4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 May 2016 06:52:39 +0600
Subject: [PATCH 4993/6219] [extractor/generic] Add support for theplatform
 embeds (Closes #8636, closes #9476)

---
 youtube_dl/extractor/generic.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index c368f08e1..ad6a40730 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -62,6 +62,7 @@ from .digiteka import DigitekaIE
 from .instagram import InstagramIE
 from .liveleak import LiveLeakIE
 from .threeqsdn import ThreeQSDNIE
+from .theplatform import ThePlatformIE
 
 
 class GenericIE(InfoExtractor):
@@ -1499,6 +1500,11 @@ class GenericIE(InfoExtractor):
         if bc_urls:
             return _playlist_from_matches(bc_urls, ie='BrightcoveNew')
 
+        # Look for ThePlatform embeds
+        tp_urls = ThePlatformIE._extract_urls(webpage)
+        if tp_urls:
+            return _playlist_from_matches(tp_urls, ie='ThePlatform')
+
         # Look for embedded rtl.nl player
         matches = re.findall(
             r'<iframe[^>]+?src="((?:https?:)?//(?:www\.)?rtl\.nl/system/videoplayer/[^"]+(?:video_)?embed[^"]+)"',

From c6b9cf05e1dbd5e2534607fd3319ac73791d1c89 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 May 2016 08:28:41 +0600
Subject: [PATCH 4994/6219] [utils] Do not fail on unknown date formats in
 unified_strdate

---
 youtube_dl/utils.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 5301d0740..d65f5e833 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1055,7 +1055,10 @@ def unified_strdate(date_str, day_first=True):
     if upload_date is None:
         timetuple = email.utils.parsedate_tz(date_str)
         if timetuple:
-            upload_date = datetime.datetime(*timetuple[:6]).strftime('%Y%m%d')
+            try:
+                upload_date = datetime.datetime(*timetuple[:6]).strftime('%Y%m%d')
+            except ValueError:
+                pass
     if upload_date is not None:
         return compat_str(upload_date)
 

From 21a19aa94d7a650d90ab258bd277a8648378c135 Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Sun, 22 May 2016 08:59:28 +0600
Subject: [PATCH 4995/6219] [README.md] Clarify location for youtube-dl.exe

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index a2febab2c..96cefb548 100644
--- a/README.md
+++ b/README.md
@@ -25,7 +25,7 @@ If you do not have curl, you can alternatively use a recent wget:
     sudo wget https://yt-dl.org/downloads/latest/youtube-dl -O /usr/local/bin/youtube-dl
     sudo chmod a+rx /usr/local/bin/youtube-dl
 
-Windows users can [download a .exe file](https://yt-dl.org/latest/youtube-dl.exe) and place it in their home directory or any other location on their [PATH](http://en.wikipedia.org/wiki/PATH_%28variable%29).
+Windows users can [download an .exe file](https://yt-dl.org/latest/youtube-dl.exe) and place it in their home directory (`%USERPROFILE%`, for example `C:\Users\<user name>\` or `C:\Documents and Settings\<user name>\`) or any other location on their [PATH](http://en.wikipedia.org/wiki/PATH_%28variable%29) except for `%SYSTEMROOT%\system32` (e.g. **do not** put in `C:\Windows\System32`).
 
 OS X users can install **youtube-dl** with [Homebrew](http://brew.sh/).
 

From 4f3a25c2b413977bf0ea6f7bd16d3d20259470bb Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Sun, 22 May 2016 09:00:08 +0600
Subject: [PATCH 4996/6219] [README.md] Fix typo

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 96cefb548..759d2bb01 100644
--- a/README.md
+++ b/README.md
@@ -433,7 +433,7 @@ You can use `--ignore-config` if you want to disable the configuration file for
 
 ### Authentication with `.netrc` file
 
-You may also want to configure automatic credentials storage for extractors that support authentication (by providing login and password with `--username` and `--password`) in order not to pass credentials as command line arguments on every youtube-dl execution and prevent tracking plain text passwords in the shell command history. You can achieve this using a [`.netrc` file](http://stackoverflow.com/tags/.netrc/info) on per extractor basis. For that you will need to create a`.netrc` file in your `$HOME` and restrict permissions to read/write by you only:
+You may also want to configure automatic credentials storage for extractors that support authentication (by providing login and password with `--username` and `--password`) in order not to pass credentials as command line arguments on every youtube-dl execution and prevent tracking plain text passwords in the shell command history. You can achieve this using a [`.netrc` file](http://stackoverflow.com/tags/.netrc/info) on per extractor basis. For that you will need to create a `.netrc` file in your `$HOME` and restrict permissions to read/write by you only:
 ```
 touch $HOME/.netrc
 chmod a-rwx,u+rw $HOME/.netrc

From 9b06b0fb9297efe47a8de71142e926dda5031b65 Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Sun, 22 May 2016 09:26:06 +0600
Subject: [PATCH 4997/6219] [README.md] Clarify updating on Windows

---
 README.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/README.md b/README.md
index 759d2bb01..649e78186 100644
--- a/README.md
+++ b/README.md
@@ -675,6 +675,8 @@ $ youtube-dl --dateafter 20000101 --datebefore 20091231
 
 If you've followed [our manual installation instructions](http://rg3.github.io/youtube-dl/download.html), you can simply run `youtube-dl -U` (or, on Linux, `sudo youtube-dl -U`).
 
+Note that on Windows before running the update command in command prompt you should either `cd` to the directory where `youtube-dl.exe` is located or use the full path to `youtube-dl.exe` (e.g. `C:\Program Files (x86)\youtube-dl\youtube-dl.exe -U`).
+
 If you have used pip, a simple `sudo pip install -U youtube-dl` is sufficient to update.
 
 If you have installed youtube-dl using a package manager like *apt-get* or *yum*, use the standard system update mechanism to update. Note that distribution packages are often outdated. As a rule of thumb, youtube-dl releases at least once a month, and often weekly or even daily. Simply go to http://yt-dl.org/ to find out the current version. Unfortunately, there is nothing we youtube-dl developers can do if your distribution serves a really outdated version. You can (and should) complain to your distribution in their bugtracker or support forum.

From e5871c672b32d30fe4a943ad1375a7000829f03c Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Sun, 22 May 2016 09:36:07 +0600
Subject: [PATCH 4998/6219] [README.md] Clarify location for youtube-dl.exe
 even more

%USERPROFILE% not in %PATH% by default.
---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 649e78186..185df5e76 100644
--- a/README.md
+++ b/README.md
@@ -25,7 +25,7 @@ If you do not have curl, you can alternatively use a recent wget:
     sudo wget https://yt-dl.org/downloads/latest/youtube-dl -O /usr/local/bin/youtube-dl
     sudo chmod a+rx /usr/local/bin/youtube-dl
 
-Windows users can [download an .exe file](https://yt-dl.org/latest/youtube-dl.exe) and place it in their home directory (`%USERPROFILE%`, for example `C:\Users\<user name>\` or `C:\Documents and Settings\<user name>\`) or any other location on their [PATH](http://en.wikipedia.org/wiki/PATH_%28variable%29) except for `%SYSTEMROOT%\system32` (e.g. **do not** put in `C:\Windows\System32`).
+Windows users can [download an .exe file](https://yt-dl.org/latest/youtube-dl.exe) and place it in any location on their [PATH](http://en.wikipedia.org/wiki/PATH_%28variable%29) except for `%SYSTEMROOT%\System32` (e.g. **do not** put in `C:\Windows\System32`).
 
 OS X users can install **youtube-dl** with [Homebrew](http://brew.sh/).
 

From e9297256d405651428d5d52f0bb6b32ca66ea15a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 May 2016 10:06:45 +0600
Subject: [PATCH 4999/6219] [update] Fix youtube-dl.exe updating from arbitrary
 directory (Closes #2718)

---
 youtube_dl/update.py | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/update.py b/youtube_dl/update.py
index 676ebe1c4..ebce9666a 100644
--- a/youtube_dl/update.py
+++ b/youtube_dl/update.py
@@ -83,11 +83,8 @@ def update_self(to_screen, verbose, opener):
 
     print_notes(to_screen, versions_info['versions'])
 
-    filename = sys.argv[0]
-    # Py2EXE: Filename could be different
-    if hasattr(sys, 'frozen') and not os.path.isfile(filename):
-        if os.path.isfile(filename + '.exe'):
-            filename += '.exe'
+    # sys.executable is set to the full pathname of the exe-file for py2exe
+    filename = sys.executable if hasattr(sys, 'frozen') else sys.argv[0]
 
     if not os.access(filename, os.W_OK):
         to_screen('ERROR: no write permissions on %s' % filename)
@@ -95,7 +92,7 @@ def update_self(to_screen, verbose, opener):
 
     # Py2EXE
     if hasattr(sys, 'frozen'):
-        exe = os.path.abspath(filename)
+        exe = filename
         directory = os.path.dirname(exe)
         if not os.access(directory, os.W_OK):
             to_screen('ERROR: no write permissions on %s' % directory)

From c776b99691e5fdec75cc7d5c268c260f23bd2ac7 Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Sun, 22 May 2016 10:14:02 +0600
Subject: [PATCH 5000/6219] [README.md] Remove Windows updating trickery

Windows updating fixed in e9297256d405651428d5d52f0bb6b32ca66ea15a.
---
 README.md | 2 --
 1 file changed, 2 deletions(-)

diff --git a/README.md b/README.md
index 185df5e76..00f42e056 100644
--- a/README.md
+++ b/README.md
@@ -675,8 +675,6 @@ $ youtube-dl --dateafter 20000101 --datebefore 20091231
 
 If you've followed [our manual installation instructions](http://rg3.github.io/youtube-dl/download.html), you can simply run `youtube-dl -U` (or, on Linux, `sudo youtube-dl -U`).
 
-Note that on Windows before running the update command in command prompt you should either `cd` to the directory where `youtube-dl.exe` is located or use the full path to `youtube-dl.exe` (e.g. `C:\Program Files (x86)\youtube-dl\youtube-dl.exe -U`).
-
 If you have used pip, a simple `sudo pip install -U youtube-dl` is sufficient to update.
 
 If you have installed youtube-dl using a package manager like *apt-get* or *yum*, use the standard system update mechanism to update. Note that distribution packages are often outdated. As a rule of thumb, youtube-dl releases at least once a month, and often weekly or even daily. Simply go to http://yt-dl.org/ to find out the current version. Unfortunately, there is nothing we youtube-dl developers can do if your distribution serves a really outdated version. You can (and should) complain to your distribution in their bugtracker or support forum.

From 70346165fe9348b54e8d71fb40654d135af945f8 Mon Sep 17 00:00:00 2001
From: Thor77 <xXThor77Xx@gmail.com>
Date: Sun, 22 May 2016 08:15:39 +0200
Subject: [PATCH 5001/6219] [bandcamp] raise ExtractorError when track not
 streamable (#9465)

* [bandcamp] raise ExtractorError when track not streamable

* [bandcamp] update md5 for second test

* don't rely on json-data, but just check for 'file'

* don't rely on presence of 'file'
---
 youtube_dl/extractor/bandcamp.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/bandcamp.py b/youtube_dl/extractor/bandcamp.py
index c1ef8051d..991ab0676 100644
--- a/youtube_dl/extractor/bandcamp.py
+++ b/youtube_dl/extractor/bandcamp.py
@@ -29,7 +29,7 @@ class BandcampIE(InfoExtractor):
         '_skip': 'There is a limit of 200 free downloads / month for the test song'
     }, {
         'url': 'http://benprunty.bandcamp.com/track/lanius-battle',
-        'md5': '2b68e5851514c20efdff2afc5603b8b4',
+        'md5': '73d0b3171568232574e45652f8720b5c',
         'info_dict': {
             'id': '2650410135',
             'ext': 'mp3',
@@ -48,6 +48,10 @@ class BandcampIE(InfoExtractor):
             if m_trackinfo:
                 json_code = m_trackinfo.group(1)
                 data = json.loads(json_code)[0]
+                track_id = compat_str(data['id'])
+
+                if not data.get('file'):
+                    raise ExtractorError('Not streamable', video_id=track_id, expected=True)
 
                 formats = []
                 for format_id, format_url in data['file'].items():
@@ -64,7 +68,7 @@ class BandcampIE(InfoExtractor):
                 self._sort_formats(formats)
 
                 return {
-                    'id': compat_str(data['id']),
+                    'id': track_id,
                     'title': data['title'],
                     'formats': formats,
                     'duration': float_or_none(data.get('duration')),

From a4a7c44bd337cdda534ad879c516d5b33e25a893 Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Sun, 22 May 2016 15:04:51 +0600
Subject: [PATCH 5002/6219] [README.md] Document solution for extremely slow
 start on Windows

---
 README.md | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/README.md b/README.md
index 00f42e056..ef0e265c8 100644
--- a/README.md
+++ b/README.md
@@ -693,6 +693,10 @@ hash -r
 
 Again, from then on you'll be able to update with `sudo youtube-dl -U`.
 
+### youtube-dl is extremely slow to start on Windows
+
+Add a file exclusion for `youtube-dl.exe` in Windows Defender settings.
+
 ### I'm getting an error `Unable to extract OpenGraph title` on YouTube playlists
 
 YouTube changed their playlist format in March 2014 and later on, so you'll need at least youtube-dl 2014.07.25 to download all YouTube videos.

From 4a12077855026a0ca9cf31868c13d2d029f7a723 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 22 May 2016 22:22:27 +0800
Subject: [PATCH 5003/6219] [genric] Eliminate duplicated video URLs (closes
 #6562)

---
 youtube_dl/extractor/generic.py | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index ad6a40730..bb96e7231 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1194,6 +1194,16 @@ class GenericIE(InfoExtractor):
                 'uploader': 'Lake8737',
             }
         },
+        # Duplicated embedded video URLs
+        {
+            'url': 'http://www.hudl.com/athlete/2538180/highlights/149298443',
+            'info_dict': {
+                'id': '149298443_480_16c25b74_2',
+                'ext': 'mp4',
+                'title': 'vs. Blue Orange Spring Game',
+                'uploader': 'www.hudl.com',
+            },
+        },
     ]
 
     def report_following_redirect(self, new_url):
@@ -2111,7 +2121,7 @@ class GenericIE(InfoExtractor):
             raise UnsupportedError(url)
 
         entries = []
-        for video_url in found:
+        for video_url in orderedSet(found):
             video_url = unescapeHTML(video_url)
             video_url = video_url.replace('\\/', '/')
             video_url = compat_urlparse.urljoin(url, video_url)

From c5f51551006c9d9ad7263cb3d3d90d1c91c8c648 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 22 Apr 2016 09:36:14 +0100
Subject: [PATCH 5004/6219] [wat] extract all formats

---
 youtube_dl/extractor/wat.py | 129 +++++++++++++++++-------------------
 1 file changed, 59 insertions(+), 70 deletions(-)

diff --git a/youtube_dl/extractor/wat.py b/youtube_dl/extractor/wat.py
index 5227bb5ad..de7d6b559 100644
--- a/youtube_dl/extractor/wat.py
+++ b/youtube_dl/extractor/wat.py
@@ -2,25 +2,26 @@
 from __future__ import unicode_literals
 
 import re
-import hashlib
 
 from .common import InfoExtractor
+from ..compat import compat_str
 from ..utils import (
     ExtractorError,
     unified_strdate,
+    HEADRequest,
+    float_or_none,
 )
 
 
 class WatIE(InfoExtractor):
-    _VALID_URL = r'(?:wat:(?P<real_id>\d{8})|https?://www\.wat\.tv/video/(?P<display_id>.*)-(?P<short_id>.*?)_.*?\.html)'
+    _VALID_URL = r'(?:wat:|https?://(?:www\.)?wat\.tv/video/.*-)(?P<id>[0-9a-z]+)'
     IE_NAME = 'wat.tv'
     _TESTS = [
         {
             'url': 'http://www.wat.tv/video/soupe-figues-l-orange-aux-epices-6z1uz_2hvf7_.html',
-            'md5': 'ce70e9223945ed26a8056d413ca55dc9',
+            'md5': '83d882d9de5c9d97f0bb2c6273cde56a',
             'info_dict': {
                 'id': '11713067',
-                'display_id': 'soupe-figues-l-orange-aux-epices',
                 'ext': 'mp4',
                 'title': 'Soupe de figues à l\'orange et aux épices',
                 'description': 'Retrouvez l\'émission "Petits plats en équilibre", diffusée le 18 août 2014.',
@@ -33,7 +34,6 @@ class WatIE(InfoExtractor):
             'md5': 'fbc84e4378165278e743956d9c1bf16b',
             'info_dict': {
                 'id': '11713075',
-                'display_id': 'gregory-lemarchal-voix-ange',
                 'ext': 'mp4',
                 'title': 'Grégory Lemarchal, une voix d\'ange depuis 10 ans (1/3)',
                 'description': 'md5:b7a849cf16a2b733d9cd10c52906dee3',
@@ -44,96 +44,85 @@ class WatIE(InfoExtractor):
         },
     ]
 
-    def download_video_info(self, real_id):
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        video_id = video_id if video_id.isdigit() and len(video_id) > 6 else compat_str(int(video_id, 36))
+
         # 'contentv4' is used in the website, but it also returns the related
         # videos, we don't need them
-        info = self._download_json('http://www.wat.tv/interface/contentv3/' + real_id, real_id)
-        return info['media']
-
-    def _real_extract(self, url):
-        def real_id_for_chapter(chapter):
-            return chapter['tc_start'].split('-')[0]
-        mobj = re.match(self._VALID_URL, url)
-        display_id = mobj.group('display_id')
-        real_id = mobj.group('real_id')
-        if not real_id:
-            short_id = mobj.group('short_id')
-            webpage = self._download_webpage(url, display_id or short_id)
-            real_id = self._search_regex(r'xtpage = ".*-(.*?)";', webpage, 'real id')
-
-        video_info = self.download_video_info(real_id)
+        video_info = self._download_json(
+            'http://www.wat.tv/interface/contentv3/' + video_id, video_id)['media']
 
         error_desc = video_info.get('error_desc')
         if error_desc:
             raise ExtractorError(
                 '%s returned error: %s' % (self.IE_NAME, error_desc), expected=True)
 
-        geo_list = video_info.get('geoList')
-        country = geo_list[0] if geo_list else ''
-
         chapters = video_info['chapters']
         first_chapter = chapters[0]
-        files = video_info['files']
-        first_file = files[0]
 
-        if real_id_for_chapter(first_chapter) != real_id:
+        def video_id_for_chapter(chapter):
+            return chapter['tc_start'].split('-')[0]
+
+        if video_id_for_chapter(first_chapter) != video_id:
             self.to_screen('Multipart video detected')
-            chapter_urls = []
-            for chapter in chapters:
-                chapter_id = real_id_for_chapter(chapter)
-                # Yes, when we this chapter is processed by WatIE,
-                # it will download the info again
-                chapter_info = self.download_video_info(chapter_id)
-                chapter_urls.append(chapter_info['url'])
-            entries = [self.url_result(chapter_url) for chapter_url in chapter_urls]
-            return self.playlist_result(entries, real_id, video_info['title'])
-
-        upload_date = None
-        if 'date_diffusion' in first_chapter:
-            upload_date = unified_strdate(first_chapter['date_diffusion'])
+            entries = [self.url_result('wat:%s' % video_id_for_chapter(chapter)) for chapter in chapters]
+            return self.playlist_result(entries, video_id, video_info['title'])
         # Otherwise we can continue and extract just one part, we have to use
-        # the short id for getting the video url
+        # the video id for getting the video url
 
-        formats = [{
-            'url': 'http://wat.tv/get/android5/%s.mp4' % real_id,
-            'format_id': 'Mobile',
-        }]
+        date_diffusion = first_chapter.get('date_diffusion')
+        upload_date = unified_strdate(date_diffusion) if date_diffusion else None
 
-        fmts = [('SD', 'web')]
-        if first_file.get('hasHD'):
-            fmts.append(('HD', 'webhd'))
+        def extract_url(path_template, url_type):
+            req_url = 'http://www.wat.tv/get/%s' % (path_template % video_id)
+            head = self._request_webpage(HEADRequest(req_url), video_id, 'Extracting %s url' % url_type)
+            red_url = head.geturl()
+            if req_url == red_url:
+                raise ExtractorError(
+                    '%s said: Sorry, this video is not available from your country.' % self.IE_NAME,
+                    expected=True)
+            return red_url
 
-        def compute_token(param):
-            timestamp = '%08x' % int(self._download_webpage(
-                'http://www.wat.tv/servertime', real_id,
-                'Downloading server time').split('|')[0])
-            magic = '9b673b13fa4682ed14c3cfa5af5310274b514c4133e9b3a81e6e3aba009l2564'
-            return '%s/%s' % (hashlib.md5((magic + param + timestamp).encode('ascii')).hexdigest(), timestamp)
+        m3u8_url = extract_url('ipad/%s.m3u8', 'm3u8')
+        http_url = extract_url('android5/%s.mp4', 'http')
 
-        for fmt in fmts:
-            webid = '/%s/%s' % (fmt[1], real_id)
-            video_url = self._download_webpage(
-                'http://www.wat.tv/get%s?token=%s&getURL=1&country=%s' % (webid, compute_token(webid), country),
-                real_id,
-                'Downloading %s video URL' % fmt[0],
-                'Failed to download %s video URL' % fmt[0],
-                False)
-            if not video_url:
+        formats = []
+        m3u8_formats = self._extract_m3u8_formats(
+            m3u8_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls')
+        formats.extend(m3u8_formats)
+        formats.extend(self._extract_f4m_formats(
+            m3u8_url.replace('ios.', 'web.').replace('.m3u8', '.f4m'),
+            video_id, f4m_id='hds', fatal=False))
+        for m3u8_format in m3u8_formats:
+            mobj = re.search(
+                r'audio.*?%3D(\d+)(?:-video.*?%3D(\d+))?', m3u8_format['url'])
+            if not mobj:
                 continue
-            formats.append({
-                'url': video_url,
-                'ext': 'mp4',
-                'format_id': fmt[0],
+            abr, vbr = mobj.groups()
+            abr, vbr = float_or_none(abr, 1000), float_or_none(vbr, 1000)
+            m3u8_format.update({
+                'vbr': vbr,
+                'abr': abr,
             })
+            if not vbr or not abr:
+                continue
+            f = m3u8_format.copy()
+            f.update({
+                'url': re.sub(r'%s-\d+00-\d+' % video_id, '%s-%d00-%d' % (video_id, round(vbr / 100), round(abr)), http_url),
+                'format_id': f['format_id'].replace('hls', 'http'),
+                'protocol': 'http',
+            })
+            formats.append(f)
+        self._sort_formats(formats)
 
         return {
-            'id': real_id,
-            'display_id': display_id,
+            'id': video_id,
             'title': first_chapter['title'],
             'thumbnail': first_chapter['preview'],
             'description': first_chapter['description'],
             'view_count': video_info['views'],
             'upload_date': upload_date,
-            'duration': first_file['duration'],
+            'duration': video_info['files'][0]['duration'],
             'formats': formats,
         }

From db3b8b2103099a8859402f2167d7ad1a8fa66829 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 22 May 2016 16:54:41 +0100
Subject: [PATCH 5005/6219] [tf1] add support for more related web sites

---
 youtube_dl/extractor/tf1.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/tf1.py b/youtube_dl/extractor/tf1.py
index 3f54b2744..aff5121b9 100644
--- a/youtube_dl/extractor/tf1.py
+++ b/youtube_dl/extractor/tf1.py
@@ -6,7 +6,7 @@ from .common import InfoExtractor
 
 class TF1IE(InfoExtractor):
     """TF1 uses the wat.tv player."""
-    _VALID_URL = r'https?://(?:(?:videos|www|lci)\.tf1|www\.tfou)\.fr/(?:[^/]+/)*(?P<id>.+?)\.html'
+    _VALID_URL = r'https?://(?:(?:videos|www|lci)\.tf1|(?:www\.)?(?:tfou|ushuaiatv|histoire|tvbreizh))\.fr/(?:[^/]+/)*(?P<id>[^/?#.]+)'
     _TESTS = [{
         'url': 'http://videos.tf1.fr/auto-moto/citroen-grand-c4-picasso-2013-presentation-officielle-8062060.html',
         'info_dict': {
@@ -48,6 +48,6 @@ class TF1IE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
         wat_id = self._html_search_regex(
-            r'(["\'])(?:https?:)?//www\.wat\.tv/embedframe/.*?(?P<id>\d{8})(?:#.*?)?\1',
+            r'(["\'])(?:https?:)?//www\.wat\.tv/embedframe/.*?(?P<id>\d{8})(?:.*?)?\1',
             webpage, 'wat id', group='id')
         return self.url_result('wat:%s' % wat_id, 'Wat')

From eb7941e3e6e92bac26f5d21525fc8ac89c934abe Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 23 May 2016 01:34:08 +0800
Subject: [PATCH 5006/6219] [compat] Fix for XML with <!DOCTYPE> in Python 2.7
 and 3.2

Such XML documents cause DeprecationWarning if python is run
with `-W error`
---
 test/test_compat.py  |  6 ++++++
 youtube_dl/compat.py | 17 ++++++++++++-----
 2 files changed, 18 insertions(+), 5 deletions(-)

diff --git a/test/test_compat.py b/test/test_compat.py
index 539b30540..f5317ac3e 100644
--- a/test/test_compat.py
+++ b/test/test_compat.py
@@ -103,6 +103,12 @@ class TestCompat(unittest.TestCase):
         self.assertTrue(isinstance(doc.find('chinese').text, compat_str))
         self.assertTrue(isinstance(doc.find('foo/bar').text, compat_str))
 
+    def test_compat_etree_fromstring_doctype(self):
+        xml = '''<?xml version="1.0"?>
+<!DOCTYPE smil PUBLIC "-//W3C//DTD SMIL 2.0//EN" "http://www.w3.org/2001/SMIL20/SMIL20.dtd">
+<smil xmlns="http://www.w3.org/2001/SMIL20/Language"></smil>'''
+        compat_etree_fromstring(xml)
+
     def test_struct_unpack(self):
         self.assertEqual(compat_struct_unpack('!B', b'\x00'), (0,))
 
diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 1392361a1..06e5f3ff6 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -245,13 +245,20 @@ try:
 except ImportError:  # Python 2.6
     from xml.parsers.expat import ExpatError as compat_xml_parse_error
 
+
+etree = xml.etree.ElementTree
+
+
+class _TreeBuilder(etree.TreeBuilder):
+    def doctype(self, name, pubid, system):
+        pass
+
 if sys.version_info[0] >= 3:
-    compat_etree_fromstring = xml.etree.ElementTree.fromstring
+    def compat_etree_fromstring(text):
+        return etree.XML(text, parser=etree.XMLParser(target=_TreeBuilder()))
 else:
     # python 2.x tries to encode unicode strings with ascii (see the
     # XMLParser._fixtext method)
-    etree = xml.etree.ElementTree
-
     try:
         _etree_iter = etree.Element.iter
     except AttributeError:  # Python <=2.6
@@ -265,7 +272,7 @@ else:
     # 2.7 source
     def _XML(text, parser=None):
         if not parser:
-            parser = etree.XMLParser(target=etree.TreeBuilder())
+            parser = etree.XMLParser(target=_TreeBuilder())
         parser.feed(text)
         return parser.close()
 
@@ -277,7 +284,7 @@ else:
         return el
 
     def compat_etree_fromstring(text):
-        doc = _XML(text, parser=etree.XMLParser(target=etree.TreeBuilder(element_factory=_element_factory)))
+        doc = _XML(text, parser=etree.XMLParser(target=_TreeBuilder(element_factory=_element_factory)))
         for el in _etree_iter(doc):
             if el.text is not None and isinstance(el.text, bytes):
                 el.text = el.text.decode('utf-8')

From 7a46542f97c99e47ad86707bf21628630c8d871e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 23 May 2016 01:38:00 +0800
Subject: [PATCH 5007/6219] [livestream] Video IDs should always be strings
 (#2234)

---
 youtube_dl/extractor/livestream.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/livestream.py b/youtube_dl/extractor/livestream.py
index eada7c299..0edc06c43 100644
--- a/youtube_dl/extractor/livestream.py
+++ b/youtube_dl/extractor/livestream.py
@@ -150,7 +150,7 @@ class LivestreamIE(InfoExtractor):
         }
 
     def _extract_stream_info(self, stream_info):
-        broadcast_id = stream_info['broadcast_id']
+        broadcast_id = compat_str(stream_info['broadcast_id'])
         is_live = stream_info.get('is_live')
 
         formats = []

From 78d3b3e2137f6be75b64e9bbfdec88cb420a91d1 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 23 May 2016 01:39:09 +0800
Subject: [PATCH 5008/6219] [generic] Improve Livestream detection (closes
 #2234)

---
 youtube_dl/extractor/generic.py | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index bb96e7231..303e112d2 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -784,6 +784,19 @@ class GenericIE(InfoExtractor):
                 'title': 'Rosetta #CometLanding webcast HL 10',
             }
         },
+        # Another Livestream embed, without 'new.' in URL
+        {
+            'url': 'https://www.freespeech.org/',
+            'info_dict': {
+                'id': '123537347',
+                'ext': 'mp4',
+                'title': 're:^FSTV [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
+            },
+            'params': {
+                # Live stream
+                'skip_download': True,
+            },
+        },
         # LazyYT
         {
             'url': 'http://discourse.ubuntu.com/t/unity-8-desktop-mode-windows-on-mir/1986',
@@ -1878,7 +1891,7 @@ class GenericIE(InfoExtractor):
             return self.url_result(self._proto_relative_url(mobj.group('url'), scheme='http:'), 'CondeNast')
 
         mobj = re.search(
-            r'<iframe[^>]+src="(?P<url>https?://new\.livestream\.com/[^"]+/player[^"]+)"',
+            r'<iframe[^>]+src="(?P<url>https?://(?:new\.)?livestream\.com/[^"]+/player[^"]+)"',
             webpage)
         if mobj is not None:
             return self.url_result(mobj.group('url'), 'Livestream')

From 102810ef0402834bd5d43e70a5e397f2a581a5dc Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 22 May 2016 20:36:23 +0100
Subject: [PATCH 5009/6219] [voxmedia] fix volume embed extraction

---
 youtube_dl/extractor/voxmedia.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/voxmedia.py b/youtube_dl/extractor/voxmedia.py
index 0c6b1f030..9d73600aa 100644
--- a/youtube_dl/extractor/voxmedia.py
+++ b/youtube_dl/extractor/voxmedia.py
@@ -117,7 +117,7 @@ class VoxMediaIE(InfoExtractor):
             volume_webpage = self._download_webpage(
                 'http://volume.vox-cdn.com/embed/%s' % volume_uuid, volume_uuid)
             video_data = self._parse_json(self._search_regex(
-                r'Volume\.createVideo\(({.+})\s*,\s*{.*}\);', volume_webpage, 'video data'), volume_uuid)
+                r'Volume\.createVideo\(({.+})\s*,\s*{.*}\s*,\s*\[.*\]\s*,\s*{.*}\);', volume_webpage, 'video data'), volume_uuid)
             for provider_video_type in ('ooyala', 'youtube'):
                 provider_video_id = video_data.get('%s_id' % provider_video_type)
                 if provider_video_id:

From e54373204ab6c5be36823695a571680d9a641ba0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 23 May 2016 03:44:04 +0600
Subject: [PATCH 5010/6219] [lifenews] Fix metadata extraction

---
 youtube_dl/extractor/lifenews.py | 31 +++++++++++++++++--------------
 1 file changed, 17 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/lifenews.py b/youtube_dl/extractor/lifenews.py
index ba2f80a75..4b1fb9772 100644
--- a/youtube_dl/extractor/lifenews.py
+++ b/youtube_dl/extractor/lifenews.py
@@ -7,10 +7,10 @@ from .common import InfoExtractor
 from ..compat import compat_urlparse
 from ..utils import (
     determine_ext,
-    int_or_none,
-    remove_end,
-    unified_strdate,
     ExtractorError,
+    int_or_none,
+    parse_iso8601,
+    remove_end,
 )
 
 
@@ -28,7 +28,9 @@ class LifeNewsIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Мужчина нашел дома архив оборонного завода',
             'description': 'md5:3b06b1b39b5e2bea548e403d99b8bf26',
+            'timestamp': 1344154740,
             'upload_date': '20120805',
+            'view_count': int,
         }
     }, {
         # single video embedded via iframe
@@ -39,7 +41,9 @@ class LifeNewsIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'В Сети появилось видео захвата «Правым сектором» колхозных полей ',
             'description': 'Жители двух поселков Днепропетровской области не простили радикалам угрозу лишения плодородных земель и пошли в лобовую. ',
+            'timestamp': 1427961840,
             'upload_date': '20150402',
+            'view_count': int,
         }
     }, {
         # two videos embedded via iframe
@@ -48,7 +52,8 @@ class LifeNewsIE(InfoExtractor):
             'id': '153461',
             'title': 'В Москве спасли потерявшегося медвежонка, который спрятался на дереве',
             'description': 'Маленький хищник не смог найти дорогу домой и обрел временное убежище на тополе недалеко от жилого массива, пока его не нашла соседская собака.',
-            'upload_date': '20150505',
+            'timestamp': 1430825520,
+            'view_count': int,
         },
         'playlist': [{
             'md5': '9b6ef8bc0ffa25aebc8bdb40d89ab795',
@@ -57,6 +62,7 @@ class LifeNewsIE(InfoExtractor):
                 'ext': 'mp4',
                 'title': 'В Москве спасли потерявшегося медвежонка, который спрятался на дереве (Видео 1)',
                 'description': 'Маленький хищник не смог найти дорогу домой и обрел временное убежище на тополе недалеко от жилого массива, пока его не нашла соседская собака.',
+                'timestamp': 1430825520,
                 'upload_date': '20150505',
             },
         }, {
@@ -66,6 +72,7 @@ class LifeNewsIE(InfoExtractor):
                 'ext': 'mp4',
                 'title': 'В Москве спасли потерявшегося медвежонка, который спрятался на дереве (Видео 2)',
                 'description': 'Маленький хищник не смог найти дорогу домой и обрел временное убежище на тополе недалеко от жилого массива, пока его не нашла соседская собака.',
+                'timestamp': 1430825520,
                 'upload_date': '20150505',
             },
         }],
@@ -100,21 +107,17 @@ class LifeNewsIE(InfoExtractor):
         description = self._og_search_description(webpage)
 
         view_count = self._html_search_regex(
-            r'<div class=\'views\'>\s*(\d+)\s*</div>', webpage, 'view count', fatal=False)
-        comment_count = self._html_search_regex(
-            r'=\'commentCount\'[^>]*>\s*(\d+)\s*<',
-            webpage, 'comment count', fatal=False)
+            r'<div[^>]+class=(["\']).*?\bhits-count\b.*?\1[^>]*>\s*(?P<value>\d+)\s*</div>',
+            webpage, 'view count', fatal=False, group='value')
 
-        upload_date = self._html_search_regex(
-            r'<time[^>]*datetime=\'([^\']+)\'', webpage, 'upload date', fatal=False)
-        if upload_date is not None:
-            upload_date = unified_strdate(upload_date)
+        timestamp = parse_iso8601(self._search_regex(
+            r'<time[^>]+datetime=(["\'])(?P<value>.+?)\1',
+            webpage, 'upload date', fatal=False, group='value'))
 
         common_info = {
             'description': description,
             'view_count': int_or_none(view_count),
-            'comment_count': int_or_none(comment_count),
-            'upload_date': upload_date,
+            'timestamp': timestamp,
         }
 
         def make_entry(video_id, video_url, index=None):

From 5181759c0d488f9fc30175f6aff4b8d4a236352d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 23 May 2016 04:00:08 +0600
Subject: [PATCH 5011/6219] [life] Update _VALID_URL

---
 youtube_dl/extractor/lifenews.py | 30 ++++++++++++++++--------------
 1 file changed, 16 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/lifenews.py b/youtube_dl/extractor/lifenews.py
index 4b1fb9772..d5d528a36 100644
--- a/youtube_dl/extractor/lifenews.py
+++ b/youtube_dl/extractor/lifenews.py
@@ -15,13 +15,13 @@ from ..utils import (
 
 
 class LifeNewsIE(InfoExtractor):
-    IE_NAME = 'lifenews'
-    IE_DESC = 'LIFE | NEWS'
-    _VALID_URL = r'https?://lifenews\.ru/(?:mobile/)?(?P<section>news|video)/(?P<id>\d+)'
+    IE_NAME = 'life'
+    IE_DESC = 'Life.ru'
+    _VALID_URL = r'https?://life\.ru/t/[^/]+/(?P<id>\d+)'
 
     _TESTS = [{
         # single video embedded via video/source
-        'url': 'http://lifenews.ru/news/98736',
+        'url': 'https://life.ru/t/новости/98736',
         'md5': '77c95eaefaca216e32a76a343ad89d23',
         'info_dict': {
             'id': '98736',
@@ -34,7 +34,7 @@ class LifeNewsIE(InfoExtractor):
         }
     }, {
         # single video embedded via iframe
-        'url': 'http://lifenews.ru/news/152125',
+        'url': 'https://life.ru/t/новости/152125',
         'md5': '77d19a6f0886cd76bdbf44b4d971a273',
         'info_dict': {
             'id': '152125',
@@ -47,7 +47,7 @@ class LifeNewsIE(InfoExtractor):
         }
     }, {
         # two videos embedded via iframe
-        'url': 'http://lifenews.ru/news/153461',
+        'url': 'https://life.ru/t/новости/153461',
         'info_dict': {
             'id': '153461',
             'title': 'В Москве спасли потерявшегося медвежонка, который спрятался на дереве',
@@ -77,18 +77,20 @@ class LifeNewsIE(InfoExtractor):
             },
         }],
     }, {
-        'url': 'http://lifenews.ru/video/13035',
+        'url': 'https://life.ru/t/новости/213035',
+        'only_matching': True,
+    }, {
+        'url': 'https://life.ru/t/%D0%BD%D0%BE%D0%B2%D0%BE%D1%81%D1%82%D0%B8/153461',
+        'only_matching': True,
+    }, {
+        'url': 'https://life.ru/t/новости/411489/manuel_vals_nazval_frantsiiu_tsieliu_nomier_odin_dlia_ighil',
         'only_matching': True,
     }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        section = mobj.group('section')
+        video_id = self._match_id(url)
 
-        webpage = self._download_webpage(
-            'http://lifenews.ru/%s/%s' % (section, video_id),
-            video_id, 'Downloading page')
+        webpage = self._download_webpage(url, video_id)
 
         video_urls = re.findall(
             r'<video[^>]+><source[^>]+src=["\'](.+?)["\']', webpage)
@@ -102,7 +104,7 @@ class LifeNewsIE(InfoExtractor):
 
         title = remove_end(
             self._og_search_title(webpage),
-            ' - Первый по срочным новостям — LIFE | NEWS')
+            ' - Life.ru')
 
         description = self._og_search_description(webpage)
 

From 5db9df622fb45ba6fbb57ef4a2ad5f2da0236a56 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 23 May 2016 04:22:09 +0600
Subject: [PATCH 5012/6219] [life:embed] Use native hls

---
 youtube_dl/extractor/lifenews.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/lifenews.py b/youtube_dl/extractor/lifenews.py
index d5d528a36..c2b4490c4 100644
--- a/youtube_dl/extractor/lifenews.py
+++ b/youtube_dl/extractor/lifenews.py
@@ -188,7 +188,8 @@ class LifeEmbedIE(InfoExtractor):
             ext = determine_ext(video_url)
             if ext == 'm3u8':
                 formats.extend(self._extract_m3u8_formats(
-                    video_url, video_id, 'mp4', m3u8_id='m3u8'))
+                    video_url, video_id, 'mp4',
+                    entry_protocol='m3u8_native', m3u8_id='m3u8'))
             else:
                 formats.append({
                     'url': video_url,

From 4b464a6a78749dfdc7c71fa932146403f18f6cb5 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 23 May 2016 00:47:22 +0100
Subject: [PATCH 5013/6219] [washingtonpost] improve format extraction and add
 support for video pages extraction

---
 youtube_dl/extractor/extractors.py     |   5 +-
 youtube_dl/extractor/washingtonpost.py | 148 +++++++++++++++++--------
 2 files changed, 103 insertions(+), 50 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index c93cd2765..d0346714c 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -941,7 +941,10 @@ from .vube import VubeIE
 from .vuclip import VuClipIE
 from .vulture import VultureIE
 from .walla import WallaIE
-from .washingtonpost import WashingtonPostIE
+from .washingtonpost import (
+    WashingtonPostIE,
+    WashingtonPostArticleIE,
+)
 from .wat import WatIE
 from .watchindianporn import WatchIndianPornIE
 from .wdr import (
diff --git a/youtube_dl/extractor/washingtonpost.py b/youtube_dl/extractor/washingtonpost.py
index ec8b99998..71349d487 100644
--- a/youtube_dl/extractor/washingtonpost.py
+++ b/youtube_dl/extractor/washingtonpost.py
@@ -11,7 +11,100 @@ from ..utils import (
 
 
 class WashingtonPostIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?washingtonpost\.com/.*?/(?P<id>[^/]+)/(?:$|[?#])'
+    IE_NAME = 'washingtonpost'
+    _VALID_URL = r'(?:washingtonpost:|https?://(?:www\.)?washingtonpost\.com/video/(?:[^/]+/)*)(?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})'
+    _TEST = {
+        'url': 'https://www.washingtonpost.com/video/c/video/480ba4ee-1ec7-11e6-82c2-a7dcb313287d',
+        'md5': '6f537e1334b714eb15f9563bd4b9cdfa',
+        'info_dict': {
+            'id': '480ba4ee-1ec7-11e6-82c2-a7dcb313287d',
+            'ext': 'mp4',
+            'title': 'Egypt finds belongings, debris from plane crash',
+            'description': 'md5:a17ceee432f215a5371388c1f680bd86',
+            'upload_date': '20160520',
+            'uploader': 'Reuters',
+            'timestamp': 1463778452,
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        video_data = self._download_json(
+            'http://www.washingtonpost.com/posttv/c/videojson/%s?resType=jsonp' % video_id,
+            video_id, transform_source=strip_jsonp)[0]['contentConfig']
+        title = video_data['title']
+
+        urls = []
+        formats = []
+        for s in video_data.get('streams', []):
+            s_url = s.get('url')
+            if not s_url or s_url in urls:
+                continue
+            urls.append(s_url)
+            video_type = s.get('type')
+            if video_type == 'smil':
+                continue
+            elif video_type in ('ts', 'hls'):
+                m3u8_formats = self._extract_m3u8_formats(
+                    s_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False)
+                for m3u8_format in m3u8_formats:
+                    width = m3u8_format.get('width')
+                    if not width:
+                        continue
+                    vbr = self._search_regex(
+                        r'%d_%d_(\d+)' % (width, m3u8_format['height']), m3u8_format['url'], 'vbr', default=None)
+                    if vbr:
+                        m3u8_format.update({
+                            'vbr': int_or_none(vbr),
+                        })
+                formats.extend(m3u8_formats)
+            else:
+                width = int_or_none(s.get('width'))
+                vbr = int_or_none(s.get('bitrate'))
+                has_width = width != 0
+                formats.append({
+                    'format_id': (
+                        '%s-%d-%d' % (video_type, width, vbr)
+                        if width
+                        else video_type),
+                    'vbr': vbr if has_width else None,
+                    'width': width,
+                    'height': int_or_none(s.get('height')),
+                    'acodec': s.get('audioCodec'),
+                    'vcodec': s.get('videoCodec') if has_width else 'none',
+                    'filesize': int_or_none(s.get('fileSize')),
+                    'url': s_url,
+                    'ext': 'mp4',
+                    'protocol': {
+                        'mp4': 'http',
+                        'ts': 'm3u8_native',
+                        'hls': 'm3u8_native',
+                    }.get(s.get('type')),
+                })
+        source_media_url = video_data.get('sourceMediaURL')
+        if source_media_url:
+            formats.append({
+                'format_id': 'source_media',
+                'url': source_media_url,
+            })
+        self._sort_formats(
+            formats, ('width', 'height', 'vbr', 'filesize', 'tbr', 'format_id'))
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': video_data.get('blurb'),
+            'uploader': video_data.get('credits', {}).get('source'),
+            'formats': formats,
+            'duration': int_or_none(video_data.get('videoDuration'), 100),
+            'timestamp': int_or_none(
+                video_data.get('dateConfig', {}).get('dateFirstPublished'), 1000),
+        }
+
+
+class WashingtonPostArticleIE(InfoExtractor):
+    IE_NAME = 'washingtonpost:article'
+    _VALID_URL = r'https?://(?:www\.)?washingtonpost\.com/(?:[^/]+/)*(?P<id>[^/?#]+)'
     _TESTS = [{
         'url': 'http://www.washingtonpost.com/sf/national/2014/03/22/sinkhole-of-bureaucracy/',
         'info_dict': {
@@ -63,6 +156,10 @@ class WashingtonPostIE(InfoExtractor):
         }]
     }]
 
+    @classmethod
+    def suitable(cls, url):
+        return False if WashingtonPostIE.suitable(url) else super(WashingtonPostArticleIE, cls).suitable(url)
+
     def _real_extract(self, url):
         page_id = self._match_id(url)
         webpage = self._download_webpage(url, page_id)
@@ -74,54 +171,7 @@ class WashingtonPostIE(InfoExtractor):
                 <div\s+class="posttv-video-embed[^>]*?data-uuid=|
                 data-video-uuid=
             )"([^"]+)"''', webpage)
-        entries = []
-        for i, uuid in enumerate(uuids, start=1):
-            vinfo_all = self._download_json(
-                'http://www.washingtonpost.com/posttv/c/videojson/%s?resType=jsonp' % uuid,
-                page_id,
-                transform_source=strip_jsonp,
-                note='Downloading information of video %d/%d' % (i, len(uuids))
-            )
-            vinfo = vinfo_all[0]['contentConfig']
-            uploader = vinfo.get('credits', {}).get('source')
-            timestamp = int_or_none(
-                vinfo.get('dateConfig', {}).get('dateFirstPublished'), 1000)
-
-            formats = [{
-                'format_id': (
-                    '%s-%s-%s' % (s.get('type'), s.get('width'), s.get('bitrate'))
-                    if s.get('width')
-                    else s.get('type')),
-                'vbr': s.get('bitrate') if s.get('width') != 0 else None,
-                'width': s.get('width'),
-                'height': s.get('height'),
-                'acodec': s.get('audioCodec'),
-                'vcodec': s.get('videoCodec') if s.get('width') != 0 else 'none',
-                'filesize': s.get('fileSize'),
-                'url': s.get('url'),
-                'ext': 'mp4',
-                'preference': -100 if s.get('type') == 'smil' else None,
-                'protocol': {
-                    'MP4': 'http',
-                    'F4F': 'f4m',
-                }.get(s.get('type')),
-            } for s in vinfo.get('streams', [])]
-            source_media_url = vinfo.get('sourceMediaURL')
-            if source_media_url:
-                formats.append({
-                    'format_id': 'source_media',
-                    'url': source_media_url,
-                })
-            self._sort_formats(formats)
-            entries.append({
-                'id': uuid,
-                'title': vinfo['title'],
-                'description': vinfo.get('blurb'),
-                'uploader': uploader,
-                'formats': formats,
-                'duration': int_or_none(vinfo.get('videoDuration'), 100),
-                'timestamp': timestamp,
-            })
+        entries = [self.url_result('washingtonpost:%s' % uuid, 'WashingtonPost', uuid) for uuid in uuids]
 
         return {
             '_type': 'playlist',

From 0c50eeb9870ec7d940c35c9cec52bfd35d009420 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 23 May 2016 02:27:31 +0100
Subject: [PATCH 5014/6219] [reuters] Add new extractor

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/reuters.py    | 69 ++++++++++++++++++++++++++++++
 2 files changed, 70 insertions(+)
 create mode 100644 youtube_dl/extractor/reuters.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index d0346714c..d8b3170ba 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -630,6 +630,7 @@ from .rds import RDSIE
 from .redtube import RedTubeIE
 from .regiotv import RegioTVIE
 from .restudy import RestudyIE
+from .reuters import ReutersIE
 from .reverbnation import ReverbNationIE
 from .revision3 import Revision3IE
 from .rice import RICEIE
diff --git a/youtube_dl/extractor/reuters.py b/youtube_dl/extractor/reuters.py
new file mode 100644
index 000000000..961d504eb
--- /dev/null
+++ b/youtube_dl/extractor/reuters.py
@@ -0,0 +1,69 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    js_to_json,
+    int_or_none,
+    unescapeHTML,
+)
+
+
+class ReutersIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?reuters\.com/.*?\?.*?videoId=(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'http://www.reuters.com/video/2016/05/20/san-francisco-police-chief-resigns?videoId=368575562',
+        'md5': '8015113643a0b12838f160b0b81cc2ee',
+        'info_dict': {
+            'id': '368575562',
+            'ext': 'mp4',
+            'title': 'San Francisco police chief resigns',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(
+            'http://www.reuters.com/assets/iframe/yovideo?videoId=%s' % video_id, video_id)
+        video_data = js_to_json(self._search_regex(
+            r'(?s)Reuters\.yovideo\.drawPlayer\(({.*?})\);',
+            webpage, 'video data'))
+
+        def get_json_value(key, fatal=False):
+            return self._search_regex('"%s"\s*:\s*"([^"]+)"' % key, video_data, key, fatal=fatal)
+
+        title = unescapeHTML(get_json_value('title', fatal=True))
+        mmid, fid = re.search(r',/(\d+)\?f=(\d+)', get_json_value('flv', fatal=True)).groups()
+
+        mas_data = self._download_json(
+            'http://mas-e.cds1.yospace.com/mas/%s/%s?trans=json' % (mmid, fid),
+            video_id, transform_source=js_to_json)
+        formats = []
+        for f in mas_data:
+            f_url = f.get('url')
+            if not f_url:
+                continue
+            method = f.get('method')
+            if method == 'hls':
+                formats.extend(self._extract_m3u8_formats(
+                    f_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
+            else:
+                container = f.get('container')
+                ext = '3gp' if method == 'mobile' else container
+                formats.append({
+                    'format_id': ext,
+                    'url': f_url,
+                    'ext': ext,
+                    'container': container if method != 'mobile' else None,
+                })
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'thumbnail': get_json_value('thumb'),
+            'duration': int_or_none(get_json_value('seconds')),
+            'formats': formats,
+        }

From b1e9ebd08087c7e591b55451551d51120b7eec9d Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 23 May 2016 02:30:12 +0100
Subject: [PATCH 5015/6219] [washingtonpost] remove unnecessary code

---
 youtube_dl/extractor/washingtonpost.py | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/youtube_dl/extractor/washingtonpost.py b/youtube_dl/extractor/washingtonpost.py
index 71349d487..c2c5bae05 100644
--- a/youtube_dl/extractor/washingtonpost.py
+++ b/youtube_dl/extractor/washingtonpost.py
@@ -74,12 +74,6 @@ class WashingtonPostIE(InfoExtractor):
                     'vcodec': s.get('videoCodec') if has_width else 'none',
                     'filesize': int_or_none(s.get('fileSize')),
                     'url': s_url,
-                    'ext': 'mp4',
-                    'protocol': {
-                        'mp4': 'http',
-                        'ts': 'm3u8_native',
-                        'hls': 'm3u8_native',
-                    }.get(s.get('type')),
                 })
         source_media_url = video_data.get('sourceMediaURL')
         if source_media_url:

From 42a7439717610530b0f7c630ef0eecf1b0638475 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 23 May 2016 09:30:26 +0100
Subject: [PATCH 5016/6219] [cbs] allow to pass content id to the
 extractor(closes #9589)

---
 youtube_dl/extractor/cbs.py | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/cbs.py b/youtube_dl/extractor/cbs.py
index 051d783a2..ac2c7dced 100644
--- a/youtube_dl/extractor/cbs.py
+++ b/youtube_dl/extractor/cbs.py
@@ -1,5 +1,7 @@
 from __future__ import unicode_literals
 
+import re
+
 from .theplatform import ThePlatformIE
 from ..utils import (
     xpath_text,
@@ -21,7 +23,7 @@ class CBSBaseIE(ThePlatformIE):
 
 
 class CBSIE(CBSBaseIE):
-    _VALID_URL = r'https?://(?:www\.)?(?:cbs\.com/shows/[^/]+/(?:video|artist)|colbertlateshow\.com/(?:video|podcasts))/[^/]+/(?P<id>[^/]+)'
+    _VALID_URL = r'(?:cbs:(?P<content_id>\w+)|https?://(?:www\.)?(?:cbs\.com/shows/[^/]+/(?:video|artist)|colbertlateshow\.com/(?:video|podcasts))/[^/]+/(?P<display_id>[^/]+))'
 
     _TESTS = [{
         'url': 'http://www.cbs.com/shows/garth-brooks/video/_u7W953k6la293J7EPTd9oHkSPs6Xn6_/connect-chat-feat-garth-brooks/',
@@ -66,11 +68,12 @@ class CBSIE(CBSBaseIE):
     TP_RELEASE_URL_TEMPLATE = 'http://link.theplatform.com/s/dJ5BDC/%s?mbr=true'
 
     def _real_extract(self, url):
-        display_id = self._match_id(url)
-        webpage = self._download_webpage(url, display_id)
-        content_id = self._search_regex(
-            [r"video\.settings\.content_id\s*=\s*'([^']+)';", r"cbsplayer\.contentId\s*=\s*'([^']+)';"],
-            webpage, 'content id')
+        content_id, display_id = re.match(self._VALID_URL, url).groups()
+        if not content_id:
+            webpage = self._download_webpage(url, display_id)
+            content_id = self._search_regex(
+                [r"video\.settings\.content_id\s*=\s*'([^']+)';", r"cbsplayer\.contentId\s*=\s*'([^']+)';"],
+                webpage, 'content id')
         items_data = self._download_xml(
             'http://can.cbs.com/thunder/player/videoPlayerService.php',
             content_id, query={'partner': 'cbs', 'contentId': content_id})

From 05b651e3a58081492eb35d896c80dd1bdb87081c Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 23 May 2016 13:04:50 +0100
Subject: [PATCH 5017/6219] [washingtonpost] reduce requests for m3u8 manifests

---
 youtube_dl/extractor/washingtonpost.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/washingtonpost.py b/youtube_dl/extractor/washingtonpost.py
index c2c5bae05..839cad986 100644
--- a/youtube_dl/extractor/washingtonpost.py
+++ b/youtube_dl/extractor/washingtonpost.py
@@ -44,7 +44,7 @@ class WashingtonPostIE(InfoExtractor):
             video_type = s.get('type')
             if video_type == 'smil':
                 continue
-            elif video_type in ('ts', 'hls'):
+            elif video_type in ('ts', 'hls') and ('_master.m3u8' in s_url or '_mobile.m3u8' in s_url):
                 m3u8_formats = self._extract_m3u8_formats(
                     s_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False)
                 for m3u8_format in m3u8_formats:
@@ -74,6 +74,8 @@ class WashingtonPostIE(InfoExtractor):
                     'vcodec': s.get('videoCodec') if has_width else 'none',
                     'filesize': int_or_none(s.get('fileSize')),
                     'url': s_url,
+                    'ext': 'mp4',
+                    'protocol': 'm3u8_native' if video_type in ('ts', 'hls') else None,
                 })
         source_media_url = video_data.get('sourceMediaURL')
         if source_media_url:

From e8593f346a4b1236d2a023eb3070610bf180459c Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 23 May 2016 23:58:16 +0100
Subject: [PATCH 5018/6219] [ooyala] extract subtitles

---
 youtube_dl/extractor/ooyala.py | 28 +++++++++++++++++++---------
 1 file changed, 19 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/ooyala.py b/youtube_dl/extractor/ooyala.py
index 95e982897..4c119071d 100644
--- a/youtube_dl/extractor/ooyala.py
+++ b/youtube_dl/extractor/ooyala.py
@@ -22,13 +22,7 @@ class OoyalaBaseIE(InfoExtractor):
         metadata = content_tree[list(content_tree)[0]]
         embed_code = metadata['embed_code']
         pcode = metadata.get('asset_pcode') or embed_code
-        video_info = {
-            'id': embed_code,
-            'title': metadata['title'],
-            'description': metadata.get('description'),
-            'thumbnail': metadata.get('thumbnail_image') or metadata.get('promo_image'),
-            'duration': float_or_none(metadata.get('duration'), 1000),
-        }
+        title = metadata['title']
 
         urls = []
         formats = []
@@ -78,8 +72,24 @@ class OoyalaBaseIE(InfoExtractor):
                     self.IE_NAME, cur_auth_data['message']), expected=True)
         self._sort_formats(formats)
 
-        video_info['formats'] = formats
-        return video_info
+        subtitles = {}
+        for lang, sub in metadata.get('closed_captions_vtt', {}).get('captions', {}).items():
+            sub_url = sub.get('url')
+            if not sub_url:
+                continue
+            subtitles[lang] = [{
+                'url': sub_url,
+            }]
+
+        return {
+            'id': embed_code,
+            'title': title,
+            'description': metadata.get('description'),
+            'thumbnail': metadata.get('thumbnail_image') or metadata.get('promo_image'),
+            'duration': float_or_none(metadata.get('duration'), 1000),
+            'subtitles': subtitles,
+            'formats': formats,
+        }
 
 
 class OoyalaIE(OoyalaBaseIE):

From a4760d204fe4cd7592bdfc91cbf550eb985374ac Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 24 May 2016 00:22:29 +0100
Subject: [PATCH 5019/6219] [ooyala] use api v2 to reduce requests for format
 extraction

---
 youtube_dl/extractor/ooyala.py | 88 +++++++++++++++++-----------------
 1 file changed, 43 insertions(+), 45 deletions(-)

diff --git a/youtube_dl/extractor/ooyala.py b/youtube_dl/extractor/ooyala.py
index 4c119071d..09bc291f0 100644
--- a/youtube_dl/extractor/ooyala.py
+++ b/youtube_dl/extractor/ooyala.py
@@ -15,7 +15,7 @@ from ..compat import compat_urllib_parse_urlencode
 class OoyalaBaseIE(InfoExtractor):
     _PLAYER_BASE = 'http://player.ooyala.com/'
     _CONTENT_TREE_BASE = _PLAYER_BASE + 'player_api/v1/content_tree/'
-    _AUTHORIZATION_URL_TEMPLATE = _PLAYER_BASE + 'sas/player_api/v1/authorization/embed_code/%s/%s?'
+    _AUTHORIZATION_URL_TEMPLATE = _PLAYER_BASE + 'sas/player_api/v2/authorization/embed_code/%s/%s?'
 
     def _extract(self, content_tree_url, video_id, domain='example.org'):
         content_tree = self._download_json(content_tree_url, video_id)['content_tree']
@@ -24,52 +24,50 @@ class OoyalaBaseIE(InfoExtractor):
         pcode = metadata.get('asset_pcode') or embed_code
         title = metadata['title']
 
+        auth_data = self._download_json(
+            self._AUTHORIZATION_URL_TEMPLATE % (pcode, embed_code) +
+            compat_urllib_parse_urlencode({
+                'domain': domain,
+                'supportedFormats': 'mp4,rtmp,m3u8,hds',
+            }), video_id)
+
+        cur_auth_data = auth_data['authorization_data'][embed_code]
+
         urls = []
         formats = []
-        for supported_format in ('mp4', 'm3u8', 'hds', 'rtmp'):
-            auth_data = self._download_json(
-                self._AUTHORIZATION_URL_TEMPLATE % (pcode, embed_code) +
-                compat_urllib_parse_urlencode({
-                    'domain': domain,
-                    'supportedFormats': supported_format
-                }),
-                video_id, 'Downloading %s JSON' % supported_format)
-
-            cur_auth_data = auth_data['authorization_data'][embed_code]
-
-            if cur_auth_data['authorized']:
-                for stream in cur_auth_data['streams']:
-                    url = base64.b64decode(
-                        stream['url']['data'].encode('ascii')).decode('utf-8')
-                    if url in urls:
-                        continue
-                    urls.append(url)
-                    delivery_type = stream['delivery_type']
-                    if delivery_type == 'hls' or '.m3u8' in url:
-                        formats.extend(self._extract_m3u8_formats(
-                            url, embed_code, 'mp4', 'm3u8_native',
-                            m3u8_id='hls', fatal=False))
-                    elif delivery_type == 'hds' or '.f4m' in url:
-                        formats.extend(self._extract_f4m_formats(
-                            url + '?hdcore=3.7.0', embed_code, f4m_id='hds', fatal=False))
-                    elif '.smil' in url:
-                        formats.extend(self._extract_smil_formats(
-                            url, embed_code, fatal=False))
-                    else:
-                        formats.append({
-                            'url': url,
-                            'ext': stream.get('delivery_type'),
-                            'vcodec': stream.get('video_codec'),
-                            'format_id': delivery_type,
-                            'width': int_or_none(stream.get('width')),
-                            'height': int_or_none(stream.get('height')),
-                            'abr': int_or_none(stream.get('audio_bitrate')),
-                            'vbr': int_or_none(stream.get('video_bitrate')),
-                            'fps': float_or_none(stream.get('framerate')),
-                        })
-            else:
-                raise ExtractorError('%s said: %s' % (
-                    self.IE_NAME, cur_auth_data['message']), expected=True)
+        if cur_auth_data['authorized']:
+            for stream in cur_auth_data['streams']:
+                url = base64.b64decode(
+                    stream['url']['data'].encode('ascii')).decode('utf-8')
+                if url in urls:
+                    continue
+                urls.append(url)
+                delivery_type = stream['delivery_type']
+                if delivery_type == 'hls' or '.m3u8' in url:
+                    formats.extend(self._extract_m3u8_formats(
+                        url, embed_code, 'mp4', 'm3u8_native',
+                        m3u8_id='hls', fatal=False))
+                elif delivery_type == 'hds' or '.f4m' in url:
+                    formats.extend(self._extract_f4m_formats(
+                        url + '?hdcore=3.7.0', embed_code, f4m_id='hds', fatal=False))
+                elif '.smil' in url:
+                    formats.extend(self._extract_smil_formats(
+                        url, embed_code, fatal=False))
+                else:
+                    formats.append({
+                        'url': url,
+                        'ext': stream.get('delivery_type'),
+                        'vcodec': stream.get('video_codec'),
+                        'format_id': delivery_type,
+                        'width': int_or_none(stream.get('width')),
+                        'height': int_or_none(stream.get('height')),
+                        'abr': int_or_none(stream.get('audio_bitrate')),
+                        'vbr': int_or_none(stream.get('video_bitrate')),
+                        'fps': float_or_none(stream.get('framerate')),
+                    })
+        else:
+            raise ExtractorError('%s said: %s' % (
+                self.IE_NAME, cur_auth_data['message']), expected=True)
         self._sort_formats(formats)
 
         subtitles = {}

From 25bcd3550ee67bb521173d7a43dbc91178a11cfc Mon Sep 17 00:00:00 2001
From: Kagami Hiiragi <kagami@genshiken.org>
Date: Tue, 24 May 2016 12:13:05 +0300
Subject: [PATCH 5020/6219] [vlive] Address site update

Changes:
* Fix video params extraction
* Don't make status request since status info now available on the page
* Remove unneeded code
* Fix test
---
 youtube_dl/extractor/vlive.py | 28 ++++++----------------------
 1 file changed, 6 insertions(+), 22 deletions(-)

diff --git a/youtube_dl/extractor/vlive.py b/youtube_dl/extractor/vlive.py
index a672ea9c5..147f52d45 100644
--- a/youtube_dl/extractor/vlive.py
+++ b/youtube_dl/extractor/vlive.py
@@ -1,8 +1,7 @@
 # coding: utf-8
-from __future__ import division, unicode_literals
+from __future__ import unicode_literals
 
 import re
-import time
 
 from .common import InfoExtractor
 from ..utils import (
@@ -23,7 +22,7 @@ class VLiveIE(InfoExtractor):
         'info_dict': {
             'id': '1326',
             'ext': 'mp4',
-            'title': "[V] Girl's Day's Broadcast",
+            'title': "[V LIVE] Girl's Day's Broadcast",
             'creator': "Girl's Day",
             'view_count': int,
         },
@@ -35,24 +34,11 @@ class VLiveIE(InfoExtractor):
         webpage = self._download_webpage(
             'http://www.vlive.tv/video/%s' % video_id, video_id)
 
-        # UTC+x - UTC+9 (KST)
-        tz = time.altzone if time.localtime().tm_isdst == 1 else time.timezone
-        tz_offset = -tz // 60 - 9 * 60
-        self._set_cookie('vlive.tv', 'timezoneOffset', '%d' % tz_offset)
-
-        status_params = self._download_json(
-            'http://www.vlive.tv/video/status?videoSeq=%s' % video_id,
-            video_id, 'Downloading JSON status',
-            headers={'Referer': url.encode('utf-8')})
-        status = status_params.get('status')
-        air_start = status_params.get('onAirStartAt', '')
-        is_live = status_params.get('isLive')
-
         video_params = self._search_regex(
-            r'vlive\.tv\.video\.ajax\.request\.handler\.init\((.+)\)',
+            r'\bvlive\.video\.init\(([^)]+)\)',
             webpage, 'video params')
-        live_params, long_video_id, key = re.split(
-            r'"\s*,\s*"', video_params)[1:4]
+        status, _, _, live_params, long_video_id, key = re.split(
+            r'"\s*,\s*"', video_params)[2:8]
 
         if status == 'LIVE_ON_AIR' or status == 'BIG_EVENT_ON_AIR':
             live_params = self._parse_json('"%s"' % live_params, video_id)
@@ -61,8 +47,6 @@ class VLiveIE(InfoExtractor):
         elif status == 'VOD_ON_AIR' or status == 'BIG_EVENT_INTRO':
             if long_video_id and key:
                 return self._replay(video_id, webpage, long_video_id, key)
-            elif is_live:
-                status = 'LIVE_END'
             else:
                 status = 'COMING_SOON'
 
@@ -70,7 +54,7 @@ class VLiveIE(InfoExtractor):
             raise ExtractorError('Uploading for replay. Please wait...',
                                  expected=True)
         elif status == 'COMING_SOON':
-            raise ExtractorError('Coming soon! %s' % air_start, expected=True)
+            raise ExtractorError('Coming soon!', expected=True)
         elif status == 'CANCELED':
             raise ExtractorError('We are sorry, '
                                  'but the live broadcast has been canceled.',

From 277c7465f58e0ac50de0dd9ebc2083f6142e9a94 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 24 May 2016 11:24:29 +0100
Subject: [PATCH 5021/6219] [ooyala] check manifest ext with determine_ext and
 update tests for related extractors

---
 youtube_dl/extractor/byutv.py           |  5 ++---
 youtube_dl/extractor/espn.py            | 12 ++++--------
 youtube_dl/extractor/formula1.py        |  3 ++-
 youtube_dl/extractor/groupon.py         |  7 +++----
 youtube_dl/extractor/howcast.py         |  7 ++-----
 youtube_dl/extractor/ooyala.py          | 24 +++++++++++++-----------
 youtube_dl/extractor/teachingchannel.py |  6 ++----
 youtube_dl/extractor/veoh.py            |  1 +
 youtube_dl/extractor/vice.py            |  3 +++
 youtube_dl/extractor/voxmedia.py        | 12 ++++++++----
 10 files changed, 40 insertions(+), 40 deletions(-)

diff --git a/youtube_dl/extractor/byutv.py b/youtube_dl/extractor/byutv.py
index dda98059e..54eb57b46 100644
--- a/youtube_dl/extractor/byutv.py
+++ b/youtube_dl/extractor/byutv.py
@@ -11,6 +11,7 @@ class BYUtvIE(InfoExtractor):
     _VALID_URL = r'^https?://(?:www\.)?byutv.org/watch/[0-9a-f-]+/(?P<video_id>[^/?#]+)'
     _TEST = {
         'url': 'http://www.byutv.org/watch/6587b9a3-89d2-42a6-a7f7-fd2f81840a7d/studio-c-season-5-episode-5',
+        'md5': '05850eb8c749e2ee05ad5a1c34668493',
         'info_dict': {
             'id': 'studio-c-season-5-episode-5',
             'ext': 'mp4',
@@ -19,9 +20,7 @@ class BYUtvIE(InfoExtractor):
             'thumbnail': 're:^https?://.*\.jpg$',
             'duration': 1486.486,
         },
-        'params': {
-            'skip_download': True,
-        }
+        'add_ie': ['Ooyala'],
     }
 
     def _real_extract(self, url):
diff --git a/youtube_dl/extractor/espn.py b/youtube_dl/extractor/espn.py
index db4b263bc..e3575aed1 100644
--- a/youtube_dl/extractor/espn.py
+++ b/youtube_dl/extractor/espn.py
@@ -8,28 +8,24 @@ class ESPNIE(InfoExtractor):
     _VALID_URL = r'https?://espn\.go\.com/(?:[^/]+/)*(?P<id>[^/]+)'
     _TESTS = [{
         'url': 'http://espn.go.com/video/clip?id=10365079',
+        'md5': '60e5d097a523e767d06479335d1bdc58',
         'info_dict': {
             'id': 'FkYWtmazr6Ed8xmvILvKLWjd4QvYZpzG',
             'ext': 'mp4',
             'title': '30 for 30 Shorts: Judging Jewell',
             'description': None,
         },
-        'params': {
-            # m3u8 download
-            'skip_download': True,
-        },
+        'add_ie': ['OoyalaExternal'],
     }, {
         # intl video, from http://www.espnfc.us/video/mls-highlights/150/video/2743663/must-see-moments-best-of-the-mls-season
         'url': 'http://espn.go.com/video/clip?id=2743663',
+        'md5': 'f4ac89b59afc7e2d7dbb049523df6768',
         'info_dict': {
             'id': '50NDFkeTqRHB0nXBOK-RGdSG5YQPuxHg',
             'ext': 'mp4',
             'title': 'Must-See Moments: Best of the MLS season',
         },
-        'params': {
-            # m3u8 download
-            'skip_download': True,
-        },
+        'add_ie': ['OoyalaExternal'],
     }, {
         'url': 'https://espn.go.com/video/iframe/twitter/?cms=espn&id=10365079',
         'only_matching': True,
diff --git a/youtube_dl/extractor/formula1.py b/youtube_dl/extractor/formula1.py
index 726393fcc..322c41e5a 100644
--- a/youtube_dl/extractor/formula1.py
+++ b/youtube_dl/extractor/formula1.py
@@ -13,7 +13,8 @@ class Formula1IE(InfoExtractor):
             'id': 'JvYXJpMzE6pArfHWm5ARp5AiUmD-gibV',
             'ext': 'flv',
             'title': 'Race highlights - Spain 2016',
-        }
+        },
+        'add_ie': ['Ooyala'],
     }
 
     def _real_extract(self, url):
diff --git a/youtube_dl/extractor/groupon.py b/youtube_dl/extractor/groupon.py
index 1dd0a81cc..7bbb669c7 100644
--- a/youtube_dl/extractor/groupon.py
+++ b/youtube_dl/extractor/groupon.py
@@ -14,6 +14,7 @@ class GrouponIE(InfoExtractor):
             'description': 'Studio kept at 105 degrees and 40% humidity with anti-microbial and anti-slip Flotex flooring; certified instructors',
         },
         'playlist': [{
+            'md5': '42428ce8a00585f9bc36e49226eae7a1',
             'info_dict': {
                 'id': 'fk6OhWpXgIQ',
                 'ext': 'mp4',
@@ -24,10 +25,8 @@ class GrouponIE(InfoExtractor):
                 'uploader_id': 'groupon',
                 'uploader': 'Groupon',
             },
-        }],
-        'params': {
-            'skip_download': True,
-        }
+            'add_ie': ['Youtube'],
+        }]
     }
 
     _PROVIDERS = {
diff --git a/youtube_dl/extractor/howcast.py b/youtube_dl/extractor/howcast.py
index e8f51e545..92caeb8f9 100644
--- a/youtube_dl/extractor/howcast.py
+++ b/youtube_dl/extractor/howcast.py
@@ -8,7 +8,7 @@ class HowcastIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?howcast\.com/videos/(?P<id>\d+)'
     _TEST = {
         'url': 'http://www.howcast.com/videos/390161-How-to-Tie-a-Square-Knot-Properly',
-        'md5': '8b743df908c42f60cf6496586c7f12c3',
+        'md5': '7d45932269a288149483144f01b99789',
         'info_dict': {
             'id': '390161',
             'ext': 'mp4',
@@ -18,10 +18,7 @@ class HowcastIE(InfoExtractor):
             'upload_date': '20100609',
             'duration': 56.823,
         },
-        'params': {
-            # m3u8 download
-            'skip_download': True,
-        },
+        'add_ie': ['Ooyala'],
     }
 
     def _real_extract(self, url):
diff --git a/youtube_dl/extractor/ooyala.py b/youtube_dl/extractor/ooyala.py
index 09bc291f0..2038a6ba5 100644
--- a/youtube_dl/extractor/ooyala.py
+++ b/youtube_dl/extractor/ooyala.py
@@ -8,6 +8,7 @@ from ..utils import (
     float_or_none,
     ExtractorError,
     unsmuggle_url,
+    determine_ext,
 )
 from ..compat import compat_urllib_parse_urlencode
 
@@ -37,26 +38,27 @@ class OoyalaBaseIE(InfoExtractor):
         formats = []
         if cur_auth_data['authorized']:
             for stream in cur_auth_data['streams']:
-                url = base64.b64decode(
+                s_url = base64.b64decode(
                     stream['url']['data'].encode('ascii')).decode('utf-8')
-                if url in urls:
+                if s_url in urls:
                     continue
-                urls.append(url)
+                urls.append(s_url)
+                ext = determine_ext(s_url, None)
                 delivery_type = stream['delivery_type']
-                if delivery_type == 'hls' or '.m3u8' in url:
+                if delivery_type == 'hls' or ext == 'm3u8':
                     formats.extend(self._extract_m3u8_formats(
-                        url, embed_code, 'mp4', 'm3u8_native',
+                        s_url, embed_code, 'mp4', 'm3u8_native',
                         m3u8_id='hls', fatal=False))
-                elif delivery_type == 'hds' or '.f4m' in url:
+                elif delivery_type == 'hds' or ext == 'f4m':
                     formats.extend(self._extract_f4m_formats(
-                        url + '?hdcore=3.7.0', embed_code, f4m_id='hds', fatal=False))
-                elif '.smil' in url:
+                        s_url + '?hdcore=3.7.0', embed_code, f4m_id='hds', fatal=False))
+                elif ext == 'smil':
                     formats.extend(self._extract_smil_formats(
-                        url, embed_code, fatal=False))
+                        s_url, embed_code, fatal=False))
                 else:
                     formats.append({
-                        'url': url,
-                        'ext': stream.get('delivery_type'),
+                        'url': s_url,
+                        'ext': ext or stream.get('delivery_type'),
                         'vcodec': stream.get('video_codec'),
                         'format_id': delivery_type,
                         'width': int_or_none(stream.get('width')),
diff --git a/youtube_dl/extractor/teachingchannel.py b/youtube_dl/extractor/teachingchannel.py
index e0477382c..e279280e9 100644
--- a/youtube_dl/extractor/teachingchannel.py
+++ b/youtube_dl/extractor/teachingchannel.py
@@ -11,6 +11,7 @@ class TeachingChannelIE(InfoExtractor):
 
     _TEST = {
         'url': 'https://www.teachingchannel.org/videos/teacher-teaming-evolution',
+        'md5': '3d6361864d7cac20b57c8784da17166f',
         'info_dict': {
             'id': 'F3bnlzbToeI6pLEfRyrlfooIILUjz4nM',
             'ext': 'mp4',
@@ -18,10 +19,7 @@ class TeachingChannelIE(InfoExtractor):
             'description': 'md5:2a9033db8da81f2edffa4c99888140b3',
             'duration': 422.255,
         },
-        'params': {
-            # m3u8 download
-            'skip_download': True,
-        },
+        'add_ie': ['Ooyala'],
     }
 
     def _real_extract(self, url):
diff --git a/youtube_dl/extractor/veoh.py b/youtube_dl/extractor/veoh.py
index 23ce0a0d1..0f5d68738 100644
--- a/youtube_dl/extractor/veoh.py
+++ b/youtube_dl/extractor/veoh.py
@@ -37,6 +37,7 @@ class VeohIE(InfoExtractor):
                 'uploader': 'afp-news',
                 'duration': 123,
             },
+            'skip': 'This video has been deleted.',
         },
         {
             'url': 'http://www.veoh.com/watch/v69525809F6Nc4frX',
diff --git a/youtube_dl/extractor/vice.py b/youtube_dl/extractor/vice.py
index 95daf4dfd..e2b2ce098 100644
--- a/youtube_dl/extractor/vice.py
+++ b/youtube_dl/extractor/vice.py
@@ -11,12 +11,14 @@ class ViceIE(InfoExtractor):
 
     _TESTS = [{
         'url': 'http://www.vice.com/video/cowboy-capitalists-part-1',
+        'md5': 'e9d77741f9e42ba583e683cd170660f7',
         'info_dict': {
             'id': '43cW1mYzpia9IlestBjVpd23Yu3afAfp',
             'ext': 'flv',
             'title': 'VICE_COWBOYCAPITALISTS_PART01_v1_VICE_WM_1080p.mov',
             'duration': 725.983,
         },
+        'add_ie': ['Ooyala'],
     }, {
         'url': 'http://www.vice.com/video/how-to-hack-a-car',
         'md5': '6fb2989a3fed069fb8eab3401fc2d3c9',
@@ -29,6 +31,7 @@ class ViceIE(InfoExtractor):
             'uploader': 'Motherboard',
             'upload_date': '20140529',
         },
+        'add_ie': ['Youtube'],
     }, {
         'url': 'https://news.vice.com/video/experimenting-on-animals-inside-the-monkey-lab',
         'only_matching': True,
diff --git a/youtube_dl/extractor/voxmedia.py b/youtube_dl/extractor/voxmedia.py
index 9d73600aa..b1b32ad44 100644
--- a/youtube_dl/extractor/voxmedia.py
+++ b/youtube_dl/extractor/voxmedia.py
@@ -15,7 +15,8 @@ class VoxMediaIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Google\'s new material design direction',
             'description': 'md5:2f44f74c4d14a1f800ea73e1c6832ad2',
-        }
+        },
+        'add_ie': ['Ooyala'],
     }, {
         # data-ooyala-id
         'url': 'http://www.theverge.com/2014/10/21/7025853/google-nexus-6-hands-on-photos-video-android-phablet',
@@ -25,7 +26,8 @@ class VoxMediaIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'The Nexus 6: hands-on with Google\'s phablet',
             'description': 'md5:87a51fe95ff8cea8b5bdb9ac7ae6a6af',
-        }
+        },
+        'add_ie': ['Ooyala'],
     }, {
         # volume embed
         'url': 'http://www.vox.com/2016/3/31/11336640/mississippi-lgbt-religious-freedom-bill',
@@ -35,7 +37,8 @@ class VoxMediaIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'The new frontier of LGBTQ civil rights, explained',
             'description': 'md5:0dc58e94a465cbe91d02950f770eb93f',
-        }
+        },
+        'add_ie': ['Ooyala'],
     }, {
         # youtube embed
         'url': 'http://www.vox.com/2016/3/24/11291692/robot-dance',
@@ -48,7 +51,8 @@ class VoxMediaIE(InfoExtractor):
             'upload_date': '20160324',
             'uploader_id': 'voxdotcom',
             'uploader': 'Vox',
-        }
+        },
+        'add_ie': ['Youtube'],
     }, {
         # SBN.VideoLinkset.entryGroup multiple ooyala embeds
         'url': 'http://www.sbnation.com/college-football-recruiting/2015/2/3/7970291/national-signing-day-rationalizations-itll-be-ok-itll-be-ok',

From 444417edb55a5bf471697a3b2353fdbfb6f7e26d Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 24 May 2016 15:58:27 +0100
Subject: [PATCH 5022/6219] [radiocanada] Add new extractor(#4020)

---
 youtube_dl/extractor/extractors.py  |   4 +
 youtube_dl/extractor/radiocanada.py | 130 ++++++++++++++++++++++++++++
 2 files changed, 134 insertions(+)
 create mode 100644 youtube_dl/extractor/radiocanada.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index d8b3170ba..f9fed18f6 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -617,6 +617,10 @@ from .qqmusic import (
     QQMusicPlaylistIE,
 )
 from .r7 import R7IE
+from .radiocanada import (
+    RadioCanadaIE,
+    RadioCanadaAudioVideoIE,
+)
 from .radiode import RadioDeIE
 from .radiojavan import RadioJavanIE
 from .radiobremen import RadioBremenIE
diff --git a/youtube_dl/extractor/radiocanada.py b/youtube_dl/extractor/radiocanada.py
new file mode 100644
index 000000000..4f05bbddc
--- /dev/null
+++ b/youtube_dl/extractor/radiocanada.py
@@ -0,0 +1,130 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    xpath_text,
+    find_xpath_attr,
+    determine_ext,
+    int_or_none,
+    unified_strdate,
+    xpath_element,
+    ExtractorError,
+)
+
+
+class RadioCanadaIE(InfoExtractor):
+    IE_NAME = 'radiocanada'
+    _VALID_URL = r'(?:radiocanada:|https?://ici\.radio-canada\.ca/widgets/mediaconsole/)(?P<app_code>[^:/]+)[:/](?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'http://ici.radio-canada.ca/widgets/mediaconsole/medianet/7184272',
+        'info_dict': {
+            'id': '7184272',
+            'ext': 'flv',
+            'title': 'Le parcours du tireur capté sur vidéo',
+            'description': 'Images des caméras de surveillance fournies par la GRC montrant le parcours du tireur d\'Ottawa',
+            'upload_date': '20141023',
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        },
+    }
+
+    def _real_extract(self, url):
+        app_code, video_id = re.match(self._VALID_URL, url).groups()
+
+        formats = []
+        # TODO: extract m3u8 and f4m formats
+        # m3u8 formats can be extracted using ipad device_type return 403 error code when ffmpeg try to download segements
+        # f4m formats can be extracted using flashhd device_type but they produce unplayable file
+        for device_type in ('flash',):
+            v_data = self._download_xml(
+                'http://api.radio-canada.ca/validationMedia/v1/Validation.ashx',
+                video_id, note='Downloading %s XML' % device_type, query={
+                    'appCode': app_code,
+                    'idMedia': video_id,
+                    'connectionType': 'broadband',
+                    'multibitrate': 'true',
+                    'deviceType': device_type,
+                    # paysJ391wsHjbOJwvCs26toz and bypasslock are used to bypass geo-restriction
+                    'paysJ391wsHjbOJwvCs26toz': 'CA',
+                    'bypasslock': 'NZt5K62gRqfc',
+                })
+            v_url = xpath_text(v_data, 'url')
+            if not v_url:
+                continue
+            if v_url == 'null':
+                raise ExtractorError('%s said: %s' % (
+                    self.IE_NAME, xpath_text(v_data, 'message')), expected=True)
+            ext = determine_ext(v_url)
+            if ext == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    v_url, video_id, 'mp4', m3u8_id='hls', fatal=False))
+            elif ext == 'f4m':
+                formats.extend(self._extract_f4m_formats(v_url, video_id, f4m_id='hds', fatal=False))
+            else:
+                ext = determine_ext(v_url)
+                bitrates = xpath_element(v_data, 'bitrates')
+                for url_e in bitrates.findall('url'):
+                    tbr = int_or_none(url_e.get('bitrate'))
+                    if not tbr:
+                        continue
+                    formats.append({
+                        'format_id': 'rtmp-%d' % tbr,
+                        'url': re.sub(r'\d+\.%s' % ext, '%d.%s' % (tbr, ext), v_url),
+                        'ext': 'flv',
+                        'protocol': 'rtmp',
+                        'width': int_or_none(url_e.get('width')),
+                        'height': int_or_none(url_e.get('height')),
+                        'tbr': tbr,
+                    })
+        self._sort_formats(formats)
+
+        metadata = self._download_xml(
+            'http://api.radio-canada.ca/metaMedia/v1/index.ashx',
+            video_id, note='Downloading metadata XML', query={
+                'appCode': app_code,
+                'idMedia': video_id,
+            })
+
+        def get_meta(name):
+            el = find_xpath_attr(metadata, './/Meta', 'name', name)
+            return el.text if el is not None else None
+
+        return {
+            'id': video_id,
+            'title': get_meta('Title'),
+            'description': get_meta('Description') or get_meta('ShortDescription'),
+            'thumbnail': get_meta('imageHR') or get_meta('imageMR') or get_meta('imageBR'),
+            'duration': int_or_none(get_meta('length')),
+            'series': get_meta('Emission'),
+            'season_number': int_or_none('SrcSaison'),
+            'episode_number': int_or_none('SrcEpisode'),
+            'upload_date': unified_strdate(get_meta('Date')),
+            'formats': formats,
+        }
+
+
+class RadioCanadaAudioVideoIE(InfoExtractor):
+    'radiocanada:audiovideo'
+    _VALID_URL = r'https?://ici\.radio-canada\.ca/audio-video/media-(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'http://ici.radio-canada.ca/audio-video/media-7527184/barack-obama-au-vietnam',
+        'info_dict': {
+            'id': '7527184',
+            'ext': 'flv',
+            'title': 'Barack Obama au Vietnam',
+            'description': 'Les États-Unis lèvent l\'embargo sur la vente d\'armes qui datait de la guerre du Vietnam',
+            'upload_date': '20160523',
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        },
+    }
+
+    def _real_extract(self, url):
+        return self.url_result('radiocanada:medianet:%s' % self._match_id(url))

From a4690b3244a42a833146c406e622c96045b23df5 Mon Sep 17 00:00:00 2001
From: mexican porn commits <xyz71412@laoeq.com>
Date: Mon, 23 May 2016 16:32:39 -0500
Subject: [PATCH 5023/6219] [xhamster] url regex fix for videos with empty
 title.

---
 youtube_dl/extractor/xhamster.py | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index b3547174d..314e5020d 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 
 class XHamsterIE(InfoExtractor):
-    _VALID_URL = r'(?P<proto>https?)://(?:.+?\.)?xhamster\.com/movies/(?P<id>[0-9]+)/(?P<seo>.+?)\.html(?:\?.*)?'
+    _VALID_URL = r'(?P<proto>https?)://(?:.+?\.)?xhamster\.com/movies/(?P<id>[0-9]+)/(?P<seo>.*?)\.html(?:\?.*)?'
     _TESTS = [
         {
             'url': 'http://xhamster.com/movies/1509445/femaleagent_shy_beauty_takes_the_bait.html',
@@ -38,6 +38,18 @@ class XHamsterIE(InfoExtractor):
                 'age_limit': 18,
             }
         },
+        {
+            'url': 'http://xhamster.com/movies/5667973/.html',
+            'info_dict': {
+                'id': '5667973',
+                'ext': 'mp4',
+                'title': '....',
+                'upload_date': '20160208',
+                'uploader': 'parejafree',
+                'duration': 72.0,
+                'age_limit': 18,
+            }
+        },
         {
             'url': 'https://xhamster.com/movies/2272726/amber_slayed_by_the_knight.html',
             'only_matching': True,
@@ -170,7 +182,7 @@ class XHamsterEmbedIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         video_url = self._search_regex(
-            r'href="(https?://xhamster\.com/movies/%s/[^"]+\.html[^"]*)"' % video_id,
+            r'href="(https?://xhamster\.com/movies/%s/[^"]*\.html[^"]*)"' % video_id,
             webpage, 'xhamster url', default=None)
 
         if not video_url:

From 6b43132ce9ec7477d69d8ad9d5b868060679de95 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 24 May 2016 21:38:27 +0600
Subject: [PATCH 5024/6219] [xhamster] Update tests

---
 youtube_dl/extractor/xhamster.py | 79 +++++++++++++++++---------------
 1 file changed, 41 insertions(+), 38 deletions(-)

diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index 314e5020d..bd8e1af2e 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -13,48 +13,51 @@ from ..utils import (
 
 class XHamsterIE(InfoExtractor):
     _VALID_URL = r'(?P<proto>https?)://(?:.+?\.)?xhamster\.com/movies/(?P<id>[0-9]+)/(?P<seo>.*?)\.html(?:\?.*)?'
-    _TESTS = [
-        {
-            'url': 'http://xhamster.com/movies/1509445/femaleagent_shy_beauty_takes_the_bait.html',
-            'info_dict': {
-                'id': '1509445',
-                'ext': 'mp4',
-                'title': 'FemaleAgent Shy beauty takes the bait',
-                'upload_date': '20121014',
-                'uploader': 'Ruseful2011',
-                'duration': 893.52,
-                'age_limit': 18,
-            }
+    _TESTS = [{
+        'url': 'http://xhamster.com/movies/1509445/femaleagent_shy_beauty_takes_the_bait.html',
+        'md5': '8281348b8d3c53d39fffb377d24eac4e',
+        'info_dict': {
+            'id': '1509445',
+            'ext': 'mp4',
+            'title': 'FemaleAgent Shy beauty takes the bait',
+            'upload_date': '20121014',
+            'uploader': 'Ruseful2011',
+            'duration': 893.52,
+            'age_limit': 18,
         },
-        {
-            'url': 'http://xhamster.com/movies/2221348/britney_spears_sexy_booty.html?hd',
-            'info_dict': {
-                'id': '2221348',
-                'ext': 'mp4',
-                'title': 'Britney Spears  Sexy Booty',
-                'upload_date': '20130914',
-                'uploader': 'jojo747400',
-                'duration': 200.48,
-                'age_limit': 18,
-            }
+    }, {
+        'url': 'http://xhamster.com/movies/2221348/britney_spears_sexy_booty.html?hd',
+        'info_dict': {
+            'id': '2221348',
+            'ext': 'mp4',
+            'title': 'Britney Spears  Sexy Booty',
+            'upload_date': '20130914',
+            'uploader': 'jojo747400',
+            'duration': 200.48,
+            'age_limit': 18,
         },
-        {
-            'url': 'http://xhamster.com/movies/5667973/.html',
-            'info_dict': {
-                'id': '5667973',
-                'ext': 'mp4',
-                'title': '....',
-                'upload_date': '20160208',
-                'uploader': 'parejafree',
-                'duration': 72.0,
-                'age_limit': 18,
-            }
+        'params': {
+            'skip_download': True,
         },
-        {
-            'url': 'https://xhamster.com/movies/2272726/amber_slayed_by_the_knight.html',
-            'only_matching': True,
+    }, {
+        # empty seo
+        'url': 'http://xhamster.com/movies/5667973/.html',
+        'info_dict': {
+            'id': '5667973',
+            'ext': 'mp4',
+            'title': '....',
+            'upload_date': '20160208',
+            'uploader': 'parejafree',
+            'duration': 72.0,
+            'age_limit': 18,
         },
-    ]
+        'params': {
+            'skip_download': True,
+        },
+    }, {
+        'url': 'https://xhamster.com/movies/2272726/amber_slayed_by_the_knight.html',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         def extract_video_url(webpage, name):

From 0d6ee9750801045e45157f38d98ef2be0c6da4f5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 24 May 2016 21:42:47 +0600
Subject: [PATCH 5025/6219] Credit @TRox1972 for tosh.cc (#9566) and localnews8
 (#9539)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 5ca71ace7..3272fc6ea 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -172,3 +172,4 @@ blahgeek
 Kevin Deldycke
 inondle
 Tomáš Čech
+Déstin Reed

From 688c634b7d95a20c6081b202427a9e5fd7f36422 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 24 May 2016 16:42:22 +0100
Subject: [PATCH 5026/6219] skip some tests to reduce test time

---
 youtube_dl/extractor/byutv.py           | 3 +++
 youtube_dl/extractor/espn.py            | 6 ++++++
 youtube_dl/extractor/groupon.py         | 5 ++++-
 youtube_dl/extractor/howcast.py         | 3 +++
 youtube_dl/extractor/teachingchannel.py | 3 +++
 5 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/byutv.py b/youtube_dl/extractor/byutv.py
index 54eb57b46..3aec601f8 100644
--- a/youtube_dl/extractor/byutv.py
+++ b/youtube_dl/extractor/byutv.py
@@ -20,6 +20,9 @@ class BYUtvIE(InfoExtractor):
             'thumbnail': 're:^https?://.*\.jpg$',
             'duration': 1486.486,
         },
+        'params': {
+            'skip_download': True,
+        },
         'add_ie': ['Ooyala'],
     }
 
diff --git a/youtube_dl/extractor/espn.py b/youtube_dl/extractor/espn.py
index e3575aed1..66c08bec4 100644
--- a/youtube_dl/extractor/espn.py
+++ b/youtube_dl/extractor/espn.py
@@ -15,6 +15,9 @@ class ESPNIE(InfoExtractor):
             'title': '30 for 30 Shorts: Judging Jewell',
             'description': None,
         },
+        'params': {
+            'skip_download': True,
+        },
         'add_ie': ['OoyalaExternal'],
     }, {
         # intl video, from http://www.espnfc.us/video/mls-highlights/150/video/2743663/must-see-moments-best-of-the-mls-season
@@ -25,6 +28,9 @@ class ESPNIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Must-See Moments: Best of the MLS season',
         },
+        'params': {
+            'skip_download': True,
+        },
         'add_ie': ['OoyalaExternal'],
     }, {
         'url': 'https://espn.go.com/video/iframe/twitter/?cms=espn&id=10365079',
diff --git a/youtube_dl/extractor/groupon.py b/youtube_dl/extractor/groupon.py
index 7bbb669c7..a6da90931 100644
--- a/youtube_dl/extractor/groupon.py
+++ b/youtube_dl/extractor/groupon.py
@@ -26,7 +26,10 @@ class GrouponIE(InfoExtractor):
                 'uploader': 'Groupon',
             },
             'add_ie': ['Youtube'],
-        }]
+        }],
+        'params': {
+            'skip_download': True,
+        },
     }
 
     _PROVIDERS = {
diff --git a/youtube_dl/extractor/howcast.py b/youtube_dl/extractor/howcast.py
index 92caeb8f9..7e36b85ad 100644
--- a/youtube_dl/extractor/howcast.py
+++ b/youtube_dl/extractor/howcast.py
@@ -18,6 +18,9 @@ class HowcastIE(InfoExtractor):
             'upload_date': '20100609',
             'duration': 56.823,
         },
+        'params': {
+            'skip_download': True,
+        },
         'add_ie': ['Ooyala'],
     }
 
diff --git a/youtube_dl/extractor/teachingchannel.py b/youtube_dl/extractor/teachingchannel.py
index e279280e9..d14d93e3a 100644
--- a/youtube_dl/extractor/teachingchannel.py
+++ b/youtube_dl/extractor/teachingchannel.py
@@ -19,6 +19,9 @@ class TeachingChannelIE(InfoExtractor):
             'description': 'md5:2a9033db8da81f2edffa4c99888140b3',
             'duration': 422.255,
         },
+        'params': {
+            'skip_download': True,
+        },
         'add_ie': ['Ooyala'],
     }
 

From 1de32771e1d3f89ef2738883b304ce52a5ecf303 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 24 May 2016 20:10:12 +0100
Subject: [PATCH 5027/6219] [eyedotv] Add new extractor(closes #9582)

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/eyedotv.py    | 64 ++++++++++++++++++++++++++++++
 2 files changed, 65 insertions(+)
 create mode 100644 youtube_dl/extractor/eyedotv.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index f9fed18f6..05561149a 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -231,6 +231,7 @@ from .everyonesmixtape import EveryonesMixtapeIE
 from .exfm import ExfmIE
 from .expotv import ExpoTVIE
 from .extremetube import ExtremeTubeIE
+from .eyedotv import EyedoTVIE
 from .facebook import FacebookIE
 from .faz import FazIE
 from .fc2 import FC2IE
diff --git a/youtube_dl/extractor/eyedotv.py b/youtube_dl/extractor/eyedotv.py
new file mode 100644
index 000000000..2f3035147
--- /dev/null
+++ b/youtube_dl/extractor/eyedotv.py
@@ -0,0 +1,64 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    xpath_text,
+    parse_duration,
+    ExtractorError,
+)
+
+
+class EyedoTVIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?eyedo\.tv/[^/]+/(?:#!/)?Live/Detail/(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'https://www.eyedo.tv/en-US/#!/Live/Detail/16301',
+        'md5': 'ba14f17995cdfc20c36ba40e21bf73f7',
+        'info_dict': {
+            'id': '16301',
+            'ext': 'mp4',
+            'title': 'Journée du conseil scientifique de l\'Afnic 2015',
+            'description': 'md5:4abe07293b2f73efc6e1c37028d58c98',
+            'uploader': 'Afnic Live',
+            'uploader_id': '8023',
+        }
+    }
+    _ROOT_URL = 'http://live.eyedo.net:1935/'
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        video_data = self._download_xml('http://eyedo.tv/api/live/GetLive/%s' % video_id, video_id)
+
+        def _add_ns(path):
+            return self._xpath_ns(path, 'http://schemas.datacontract.org/2004/07/EyeDo.Core.Implementation.Web.ViewModels.Api')
+
+        title = xpath_text(video_data, _add_ns('Titre'), 'title', True)
+        state_live_code = xpath_text(video_data, _add_ns('StateLiveCode'), 'title', True)
+        if state_live_code == 'avenir':
+            raise ExtractorError(
+                '%s said: We\'re sorry, but this video is not yet available.' % self.IE_NAME,
+                expected=True)
+
+        is_live = state_live_code == 'live'
+        m3u8_url = None
+        # http://eyedo.tv/Content/Html5/Scripts/html5view.js
+        if is_live:
+            if xpath_text(video_data, 'Cdn') == 'true':
+                m3u8_url = 'http://rrr.sz.xlcdn.com/?account=eyedo&file=A%s&type=live&service=wowza&protocol=http&output=playlist.m3u8' % video_id
+            else:
+                m3u8_url = self._ROOT_URL + 'w/%s/eyedo_720p/playlist.m3u8' % video_id
+        else:
+            m3u8_url = self._ROOT_URL + 'replay-w/%s/mp4:%s.mp4/playlist.m3u8' % (video_id, video_id)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': self._extract_m3u8_formats(
+                m3u8_url, video_id, 'mp4', 'm3u8' if is_live else 'm3u8_native'),
+            'description': xpath_text(video_data, _add_ns('Description')),
+            'duration': parse_duration(xpath_text(video_data, _add_ns('Duration'))),
+            'uploader': xpath_text(video_data, _add_ns('Createur')),
+            'uploader_id': xpath_text(video_data, _add_ns('CreateurId')),
+            'chapter': xpath_text(video_data, _add_ns('ChapitreTitre')),
+            'chapter_id': xpath_text(video_data, _add_ns('ChapitreId')),
+        }

From 4ee0b8afdb384ad3e2d65b6b0159a801ee73d26d Mon Sep 17 00:00:00 2001
From: wankerer <git@wanker.33mail.com>
Date: Tue, 24 May 2016 10:18:36 -0700
Subject: [PATCH 5028/6219] [eporner] fix for the new URL layout

Recently eporner slightly changed the URL layout, the ID that used to be
digits only are now digits and letters, so youtube-dl falls back to
the generic extractor that doesn't work.

Fix the matching regex to allow letters in ID.

[v2: added a test case]
---
 youtube_dl/extractor/eporner.py | 22 ++++++++++++++++++----
 1 file changed, 18 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/eporner.py b/youtube_dl/extractor/eporner.py
index e006921ec..581276694 100644
--- a/youtube_dl/extractor/eporner.py
+++ b/youtube_dl/extractor/eporner.py
@@ -11,8 +11,8 @@ from ..utils import (
 
 
 class EpornerIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?eporner\.com/hd-porn/(?P<id>\d+)/(?P<display_id>[\w-]+)'
-    _TEST = {
+    _VALID_URL = r'https?://(?:www\.)?eporner\.com/hd-porn/(?P<id>\w+)/(?P<display_id>[\w-]+)'
+    _TESTS = [{
         'url': 'http://www.eporner.com/hd-porn/95008/Infamous-Tiffany-Teen-Strip-Tease-Video/',
         'md5': '39d486f046212d8e1b911c52ab4691f8',
         'info_dict': {
@@ -23,8 +23,22 @@ class EpornerIE(InfoExtractor):
             'duration': 1838,
             'view_count': int,
             'age_limit': 18,
-        }
-    }
+        },
+    },
+    # New (May 2016) URL layout
+    {
+        'url': 'http://www.eporner.com/hd-porn/3YRUtzMcWn0/Star-Wars-XXX-Parody/',
+        'md5': '3469eeaa93b6967a34cdbdbb9d064b33',
+        'info_dict': {
+            'id': '3YRUtzMcWn0',
+            'display_id': 'Star-Wars-XXX-Parody',
+            'ext': 'mp4',
+            'title': 'Star Wars XXX Parody',
+            'duration': 361.0,
+            'view_count': int,
+            'age_limit': 18,
+        },
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)

From 6f748df43ff3476e4dbd29c7464837ea63d78b2a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 25 May 2016 20:51:17 +0600
Subject: [PATCH 5029/6219] [eporner] Make test only_matching

---
 youtube_dl/extractor/eporner.py | 16 +++-------------
 1 file changed, 3 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/eporner.py b/youtube_dl/extractor/eporner.py
index 581276694..ac5d0fe24 100644
--- a/youtube_dl/extractor/eporner.py
+++ b/youtube_dl/extractor/eporner.py
@@ -24,20 +24,10 @@ class EpornerIE(InfoExtractor):
             'view_count': int,
             'age_limit': 18,
         },
-    },
-    # New (May 2016) URL layout
-    {
+    }, {
+        # New (May 2016) URL layout
         'url': 'http://www.eporner.com/hd-porn/3YRUtzMcWn0/Star-Wars-XXX-Parody/',
-        'md5': '3469eeaa93b6967a34cdbdbb9d064b33',
-        'info_dict': {
-            'id': '3YRUtzMcWn0',
-            'display_id': 'Star-Wars-XXX-Parody',
-            'ext': 'mp4',
-            'title': 'Star Wars XXX Parody',
-            'duration': 361.0,
-            'view_count': int,
-            'age_limit': 18,
-        },
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 0a5685b26fae0940f14cb063a6e4fc6986f9c124 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 26 May 2016 21:41:47 +0800
Subject: [PATCH 5030/6219] [common] Support non-bootstraped streams in f4m
 manifests

Related: #9531
---
 youtube_dl/extractor/common.py | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 4bfa610c1..7eb7464ec 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -987,7 +987,7 @@ class InfoExtractor(object):
 
     def _extract_f4m_formats(self, manifest_url, video_id, preference=None, f4m_id=None,
                              transform_source=lambda s: fix_xml_ampersands(s).strip(),
-                             fatal=True):
+                             fatal=True, assume_f4mv2=False):
         manifest = self._download_xml(
             manifest_url, video_id, 'Downloading f4m manifest',
             'Unable to download f4m manifest',
@@ -1001,11 +1001,11 @@ class InfoExtractor(object):
 
         return self._parse_f4m_formats(
             manifest, manifest_url, video_id, preference=preference, f4m_id=f4m_id,
-            transform_source=transform_source, fatal=fatal)
+            transform_source=transform_source, fatal=fatal, assume_f4mv2=assume_f4mv2)
 
     def _parse_f4m_formats(self, manifest, manifest_url, video_id, preference=None, f4m_id=None,
                            transform_source=lambda s: fix_xml_ampersands(s).strip(),
-                           fatal=True):
+                           fatal=True, assume_f4mv2=False):
         # currently youtube-dl cannot decode the playerVerificationChallenge as Akamai uses Adobe Alchemy
         akamai_pv = manifest.find('{http://ns.adobe.com/f4m/1.0}pv-2.0')
         if akamai_pv is not None and ';' in akamai_pv.text:
@@ -1029,8 +1029,13 @@ class InfoExtractor(object):
             'base URL', default=None)
         if base_url:
             base_url = base_url.strip()
+
+        bootstrap_info = xpath_text(
+            manifest, ['{http://ns.adobe.com/f4m/1.0}bootstrapInfo', '{http://ns.adobe.com/f4m/2.0}bootstrapInfo'],
+            'bootstrap info', default=None)
+
         for i, media_el in enumerate(media_nodes):
-            if manifest_version == '2.0':
+            if manifest_version == '2.0' or assume_f4mv2:
                 media_url = media_el.attrib.get('href') or media_el.attrib.get('url')
                 if not media_url:
                     continue
@@ -1050,7 +1055,7 @@ class InfoExtractor(object):
             formats.append({
                 'format_id': '-'.join(filter(None, [f4m_id, compat_str(i if tbr is None else tbr)])),
                 'url': manifest_url,
-                'ext': 'flv',
+                'ext': 'flv' if bootstrap_info else None,
                 'tbr': tbr,
                 'width': int_or_none(media_el.attrib.get('width')),
                 'height': int_or_none(media_el.attrib.get('height')),

From 85b0fe7d6442d4ddb056fb5a5d15e51e8a625ae7 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 26 May 2016 21:43:35 +0800
Subject: [PATCH 5031/6219] [playwire] Use _extract_f4m_formats

Related: #9531
---
 youtube_dl/extractor/playwire.py | 27 ++++++---------------------
 1 file changed, 6 insertions(+), 21 deletions(-)

diff --git a/youtube_dl/extractor/playwire.py b/youtube_dl/extractor/playwire.py
index 6d138ef25..7580e4a85 100644
--- a/youtube_dl/extractor/playwire.py
+++ b/youtube_dl/extractor/playwire.py
@@ -4,9 +4,8 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
-    xpath_text,
+    dict_get,
     float_or_none,
-    int_or_none,
 )
 
 
@@ -23,6 +22,7 @@ class PlaywireIE(InfoExtractor):
             'duration': 145.94,
         },
     }, {
+        # Multiple resolutions while bitrates missing
         'url': 'http://cdn.playwire.com/11625/embed/85228.html',
         'only_matching': True,
     }, {
@@ -48,25 +48,10 @@ class PlaywireIE(InfoExtractor):
         thumbnail = content.get('poster')
         src = content['media']['f4m']
 
-        f4m = self._download_xml(src, video_id)
-        base_url = xpath_text(f4m, './{http://ns.adobe.com/f4m/1.0}baseURL', 'base url', fatal=True)
-        formats = []
-        for media in f4m.findall('./{http://ns.adobe.com/f4m/1.0}media'):
-            media_url = media.get('url')
-            if not media_url:
-                continue
-            tbr = int_or_none(media.get('bitrate'))
-            width = int_or_none(media.get('width'))
-            height = int_or_none(media.get('height'))
-            f = {
-                'url': '%s/%s' % (base_url, media.attrib['url']),
-                'tbr': tbr,
-                'width': width,
-                'height': height,
-            }
-            if not (tbr or width or height):
-                f['quality'] = 1 if '-hd.' in media_url else 0
-            formats.append(f)
+        formats = self._extract_f4m_formats(src, video_id, assume_f4mv2=True)
+        for a_format in formats:
+            if not dict_get(a_format, ['tbr', 'width', 'height']):
+                a_format['quality'] = 1 if '-hd.' in a_format['url'] else 0
         self._sort_formats(formats)
 
         return {

From 240b60453e1237473dfd8deff40c9dc54661668c Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 26 May 2016 21:55:43 +0800
Subject: [PATCH 5032/6219] [common] Support m3u8 in f4m manifests

Related: #9531
---
 youtube_dl/extractor/common.py | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 7eb7464ec..b5bea5904 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -987,7 +987,7 @@ class InfoExtractor(object):
 
     def _extract_f4m_formats(self, manifest_url, video_id, preference=None, f4m_id=None,
                              transform_source=lambda s: fix_xml_ampersands(s).strip(),
-                             fatal=True, assume_f4mv2=False):
+                             fatal=True, assume_f4mv2=False, m3u8_id=None):
         manifest = self._download_xml(
             manifest_url, video_id, 'Downloading f4m manifest',
             'Unable to download f4m manifest',
@@ -1001,11 +1001,12 @@ class InfoExtractor(object):
 
         return self._parse_f4m_formats(
             manifest, manifest_url, video_id, preference=preference, f4m_id=f4m_id,
-            transform_source=transform_source, fatal=fatal, assume_f4mv2=assume_f4mv2)
+            transform_source=transform_source, fatal=fatal, assume_f4mv2=assume_f4mv2,
+            m3u8_id=m3u8_id)
 
     def _parse_f4m_formats(self, manifest, manifest_url, video_id, preference=None, f4m_id=None,
                            transform_source=lambda s: fix_xml_ampersands(s).strip(),
-                           fatal=True, assume_f4mv2=False):
+                           fatal=True, assume_f4mv2=False, m3u8_id=None):
         # currently youtube-dl cannot decode the playerVerificationChallenge as Akamai uses Adobe Alchemy
         akamai_pv = manifest.find('{http://ns.adobe.com/f4m/1.0}pv-2.0')
         if akamai_pv is not None and ';' in akamai_pv.text:
@@ -1046,11 +1047,17 @@ class InfoExtractor(object):
                 # since bitrates in parent manifest (this one) and media_url manifest
                 # may differ leading to inability to resolve the format by requested
                 # bitrate in f4m downloader
-                if determine_ext(manifest_url) == 'f4m':
+                ext = determine_ext(manifest_url)
+                if ext == 'f4m':
                     formats.extend(self._extract_f4m_formats(
                         manifest_url, video_id, preference=preference, f4m_id=f4m_id,
                         transform_source=transform_source, fatal=fatal))
                     continue
+                elif ext == 'm3u8':
+                    formats.extend(self._extract_m3u8_formats(
+                        manifest_url, video_id, 'mp4', preference=preference,
+                        m3u8_id=m3u8_id, fatal=False))
+                    continue
             tbr = int_or_none(media_el.attrib.get('bitrate'))
             formats.append({
                 'format_id': '-'.join(filter(None, [f4m_id, compat_str(i if tbr is None else tbr)])),

From 761052db922a525d6ccaf250f9914841c9d3d66f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 26 May 2016 21:57:06 +0800
Subject: [PATCH 5033/6219] [playwire] Add the test (closed #9531)

---
 youtube_dl/extractor/playwire.py | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/playwire.py b/youtube_dl/extractor/playwire.py
index 7580e4a85..2ee5c5aa3 100644
--- a/youtube_dl/extractor/playwire.py
+++ b/youtube_dl/extractor/playwire.py
@@ -21,6 +21,18 @@ class PlaywireIE(InfoExtractor):
             'thumbnail': 're:^https?://.*\.png$',
             'duration': 145.94,
         },
+    }, {
+        # m3u8 in f4m
+        'url': 'http://config.playwire.com/21772/videos/v2/4840492/zeus.json',
+        'info_dict': {
+            'id': '4840492',
+            'ext': 'mp4',
+            'title': 'ITV EL SHOW FULL',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
     }, {
         # Multiple resolutions while bitrates missing
         'url': 'http://cdn.playwire.com/11625/embed/85228.html',
@@ -48,7 +60,7 @@ class PlaywireIE(InfoExtractor):
         thumbnail = content.get('poster')
         src = content['media']['f4m']
 
-        formats = self._extract_f4m_formats(src, video_id, assume_f4mv2=True)
+        formats = self._extract_f4m_formats(src, video_id, assume_f4mv2=True, m3u8_id='hls')
         for a_format in formats:
             if not dict_get(a_format, ['tbr', 'width', 'height']):
                 a_format['quality'] = 1 if '-hd.' in a_format['url'] else 0

From 5950cb1d6d8d27f7a7272895100da9652212fad6 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 26 May 2016 22:44:00 +0800
Subject: [PATCH 5034/6219] [utils] Support a new form of date

Found in dw.com (#9475)
---
 youtube_dl/utils.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index d65f5e833..316a307e0 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1035,6 +1035,7 @@ def unified_strdate(date_str, day_first=True):
         format_expressions.extend([
             '%d-%m-%Y',
             '%d.%m.%Y',
+            '%d.%m.%y',
             '%d/%m/%Y',
             '%d/%m/%y',
             '%d/%m/%Y %H:%M:%S',
@@ -1049,6 +1050,8 @@ def unified_strdate(date_str, day_first=True):
         ])
     for expression in format_expressions:
         try:
+            print(expression)
+            print(date_str)
             upload_date = datetime.datetime.strptime(date_str, expression).strftime('%Y%m%d')
         except ValueError:
             pass
@@ -1910,7 +1913,7 @@ def parse_age_limit(s):
 
 def strip_jsonp(code):
     return re.sub(
-        r'(?s)^[a-zA-Z0-9_.]+\s*\(\s*(.*)\);?\s*?(?://[^\n]*)*$', r'\1', code)
+        r'(?s)^[a-zA-Z0-9_.$]+\s*\(\s*(.*)\);?\s*?(?://[^\n]*)*$', r'\1', code)
 
 
 def js_to_json(code):

From ac88d2316ebef5b00cf5c94d94f01c9f7e17ce51 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 26 May 2016 22:48:47 +0800
Subject: [PATCH 5035/6219] [dw] Support documentaries (closes #9475)

---
 youtube_dl/extractor/dw.py | 27 ++++++++++++++++++++++++---
 1 file changed, 24 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/dw.py b/youtube_dl/extractor/dw.py
index ae7c571bd..0f0f0b8d3 100644
--- a/youtube_dl/extractor/dw.py
+++ b/youtube_dl/extractor/dw.py
@@ -2,13 +2,16 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..utils import int_or_none
+from ..utils import (
+    int_or_none,
+    unified_strdate,
+)
 from ..compat import compat_urlparse
 
 
 class DWIE(InfoExtractor):
     IE_NAME = 'dw'
-    _VALID_URL = r'https?://(?:www\.)?dw\.com/(?:[^/]+/)+av-(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?dw\.com/(?:[^/]+/)+(?:av|e)-(?P<id>\d+)'
     _TESTS = [{
         # video
         'url': 'http://www.dw.com/en/intelligent-light/av-19112290',
@@ -31,6 +34,16 @@ class DWIE(InfoExtractor):
             'description': 'md5:bc9ca6e4e063361e21c920c53af12405',
             'upload_date': '20160311',
         }
+    }, {
+        'url': 'http://www.dw.com/en/documentaries-welcome-to-the-90s-2016-05-21/e-19220158-9798',
+        'md5': '56b6214ef463bfb9a3b71aeb886f3cf1',
+        'info_dict': {
+            'id': '19274438',
+            'ext': 'mp4',
+            'title': 'Welcome to the 90s – Hip Hop',
+            'description': 'Welcome to the 90s - The Golden Decade of Hip Hop',
+            'upload_date': '20160521',
+        },
     }]
 
     def _real_extract(self, url):
@@ -38,6 +51,7 @@ class DWIE(InfoExtractor):
         webpage = self._download_webpage(url, media_id)
         hidden_inputs = self._hidden_inputs(webpage)
         title = hidden_inputs['media_title']
+        media_id = hidden_inputs.get('media_id') or media_id
 
         if hidden_inputs.get('player_type') == 'video' and hidden_inputs.get('stream_file') == '1':
             formats = self._extract_smil_formats(
@@ -49,13 +63,20 @@ class DWIE(InfoExtractor):
         else:
             formats = [{'url': hidden_inputs['file_name']}]
 
+        upload_date = hidden_inputs.get('display_date')
+        if not upload_date:
+            upload_date = self._html_search_regex(
+                r'<span[^>]+class="date">([0-9.]+)\s*\|', webpage,
+                'upload date', default=None)
+            upload_date = unified_strdate(upload_date)
+
         return {
             'id': media_id,
             'title': title,
             'description': self._og_search_description(webpage),
             'thumbnail': hidden_inputs.get('preview_image'),
             'duration': int_or_none(hidden_inputs.get('file_duration')),
-            'upload_date': hidden_inputs.get('display_date'),
+            'upload_date': upload_date,
             'formats': formats,
         }
 

From 293c2556886c34d11919eb0af6760c52bd6a2632 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 26 May 2016 22:54:16 +0800
Subject: [PATCH 5036/6219] [utils] Remove debugging codes

---
 youtube_dl/utils.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 316a307e0..cfb2d1bf5 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1050,8 +1050,6 @@ def unified_strdate(date_str, day_first=True):
         ])
     for expression in format_expressions:
         try:
-            print(expression)
-            print(date_str)
             upload_date = datetime.datetime.strptime(date_str, expression).strftime('%Y%m%d')
         except ValueError:
             pass

From 448bb5f333c6c4c8084e479e1035ff674e4f8fd4 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 27 May 2016 00:03:03 +0800
Subject: [PATCH 5037/6219] [common] Fix non-bootstrapped support in f4m

---
 youtube_dl/extractor/common.py   | 19 +++++++++++++------
 youtube_dl/extractor/playwire.py |  2 +-
 2 files changed, 14 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index b5bea5904..e53b7ad64 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -987,7 +987,7 @@ class InfoExtractor(object):
 
     def _extract_f4m_formats(self, manifest_url, video_id, preference=None, f4m_id=None,
                              transform_source=lambda s: fix_xml_ampersands(s).strip(),
-                             fatal=True, assume_f4mv2=False, m3u8_id=None):
+                             fatal=True, m3u8_id=None):
         manifest = self._download_xml(
             manifest_url, video_id, 'Downloading f4m manifest',
             'Unable to download f4m manifest',
@@ -1001,12 +1001,11 @@ class InfoExtractor(object):
 
         return self._parse_f4m_formats(
             manifest, manifest_url, video_id, preference=preference, f4m_id=f4m_id,
-            transform_source=transform_source, fatal=fatal, assume_f4mv2=assume_f4mv2,
-            m3u8_id=m3u8_id)
+            transform_source=transform_source, fatal=fatal, m3u8_id=m3u8_id)
 
     def _parse_f4m_formats(self, manifest, manifest_url, video_id, preference=None, f4m_id=None,
                            transform_source=lambda s: fix_xml_ampersands(s).strip(),
-                           fatal=True, assume_f4mv2=False, m3u8_id=None):
+                           fatal=True, m3u8_id=None):
         # currently youtube-dl cannot decode the playerVerificationChallenge as Akamai uses Adobe Alchemy
         akamai_pv = manifest.find('{http://ns.adobe.com/f4m/1.0}pv-2.0')
         if akamai_pv is not None and ';' in akamai_pv.text:
@@ -1036,8 +1035,16 @@ class InfoExtractor(object):
             'bootstrap info', default=None)
 
         for i, media_el in enumerate(media_nodes):
-            if manifest_version == '2.0' or assume_f4mv2:
-                media_url = media_el.attrib.get('href') or media_el.attrib.get('url')
+            # If <bootstrapInfo> is present, the specified f4m is a
+            # stream-level manifest, and only set-level manifests may refer to
+            # external resources.  See section 11.4 and section 4 of F4M spec
+            if bootstrap_info is None:
+                media_url = None
+                # @href is introduced in 2.0, see section 11.6 of F4M spec
+                if manifest_version == '2.0':
+                    media_url = media_el.attrib.get('href')
+                if media_url is None:
+                    media_url = media_el.attrib.get('url')
                 if not media_url:
                     continue
                 manifest_url = (
diff --git a/youtube_dl/extractor/playwire.py b/youtube_dl/extractor/playwire.py
index 2ee5c5aa3..0bc743118 100644
--- a/youtube_dl/extractor/playwire.py
+++ b/youtube_dl/extractor/playwire.py
@@ -60,7 +60,7 @@ class PlaywireIE(InfoExtractor):
         thumbnail = content.get('poster')
         src = content['media']['f4m']
 
-        formats = self._extract_f4m_formats(src, video_id, assume_f4mv2=True, m3u8_id='hls')
+        formats = self._extract_f4m_formats(src, video_id, m3u8_id='hls')
         for a_format in formats:
             if not dict_get(a_format, ['tbr', 'width', 'height']):
                 a_format['quality'] = 1 if '-hd.' in a_format['url'] else 0

From 6f8cb2421948fd128b3004fde7eebaa2463f5f06 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 26 May 2016 22:21:55 +0600
Subject: [PATCH 5038/6219] [tvp] Expand _VALID_URL and improve naming (Closes
 #9602)

---
 youtube_dl/extractor/extractors.py |  5 +++-
 youtube_dl/extractor/tvp.py        | 47 ++++++++++++++++--------------
 2 files changed, 29 insertions(+), 23 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 05561149a..ddf62139e 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -833,7 +833,10 @@ from .tvc import (
 )
 from .tvigle import TvigleIE
 from .tvland import TVLandIE
-from .tvp import TvpIE, TvpSeriesIE
+from .tvp import (
+    TVPIE,
+    TVPSeriesIE,
+)
 from .tvplay import TVPlayIE
 from .tweakers import TweakersIE
 from .twentyfourvideo import TwentyFourVideoIE
diff --git a/youtube_dl/extractor/tvp.py b/youtube_dl/extractor/tvp.py
index f57d609d4..a4997cb89 100644
--- a/youtube_dl/extractor/tvp.py
+++ b/youtube_dl/extractor/tvp.py
@@ -1,4 +1,4 @@
-# -*- coding: utf-8 -*-
+# coding: utf-8
 from __future__ import unicode_literals
 
 import re
@@ -6,20 +6,13 @@ import re
 from .common import InfoExtractor
 
 
-class TvpIE(InfoExtractor):
-    IE_NAME = 'tvp.pl'
-    _VALID_URL = r'https?://(?:vod|www)\.tvp\.pl/.*/(?P<id>\d+)$'
+class TVPIE(InfoExtractor):
+    IE_NAME = 'tvp'
+    IE_DESC = 'Telewizja Polska'
+    _VALID_URL = r'https?://[^/]+\.tvp\.(?:pl|info)/(?:(?!\d+/)[^/]+/)*(?P<id>\d+)'
 
     _TESTS = [{
-        'url': 'http://vod.tvp.pl/filmy-fabularne/filmy-za-darmo/ogniem-i-mieczem/wideo/odc-2/4278035',
-        'md5': 'cdd98303338b8a7f7abab5cd14092bf2',
-        'info_dict': {
-            'id': '4278035',
-            'ext': 'wmv',
-            'title': 'Ogniem i mieczem, odc. 2',
-        },
-    }, {
-        'url': 'http://vod.tvp.pl/seriale/obyczajowe/czas-honoru/sezon-1-1-13/i-seria-odc-13/194536',
+        'url': 'http://vod.tvp.pl/194536/i-seria-odc-13',
         'md5': '8aa518c15e5cc32dfe8db400dc921fbb',
         'info_dict': {
             'id': '194536',
@@ -36,12 +29,22 @@ class TvpIE(InfoExtractor):
         },
     }, {
         'url': 'http://vod.tvp.pl/seriale/obyczajowe/na-sygnale/sezon-2-27-/odc-39/17834272',
-        'md5': 'c3b15ed1af288131115ff17a17c19dda',
-        'info_dict': {
-            'id': '17834272',
-            'ext': 'mp4',
-            'title': 'Na sygnale, odc. 39',
-        },
+        'only_matching': True,
+    }, {
+        'url': 'http://wiadomosci.tvp.pl/25169746/24052016-1200',
+        'only_matching': True,
+    }, {
+        'url': 'http://krakow.tvp.pl/25511623/25lecie-mck-wyjatkowe-miejsce-na-mapie-krakowa',
+        'only_matching': True,
+    }, {
+        'url': 'http://teleexpress.tvp.pl/25522307/wierni-wzieli-udzial-w-procesjach',
+        'only_matching': True,
+    }, {
+        'url': 'http://sport.tvp.pl/25522165/krychowiak-uspokaja-w-sprawie-kontuzji-dwa-tygodnie-to-maksimum',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.tvp.info/25511919/trwa-rewolucja-wladza-zdecydowala-sie-na-pogwalcenie-konstytucji',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
@@ -92,8 +95,8 @@ class TvpIE(InfoExtractor):
         }
 
 
-class TvpSeriesIE(InfoExtractor):
-    IE_NAME = 'tvp.pl:Series'
+class TVPSeriesIE(InfoExtractor):
+    IE_NAME = 'tvp:series'
     _VALID_URL = r'https?://vod\.tvp\.pl/(?:[^/]+/){2}(?P<id>[^/]+)/?$'
 
     _TESTS = [{
@@ -127,7 +130,7 @@ class TvpSeriesIE(InfoExtractor):
         videos_paths = re.findall(
             '(?s)class="shortTitle">.*?href="(/[^"]+)', playlist)
         entries = [
-            self.url_result('http://vod.tvp.pl%s' % v_path, ie=TvpIE.ie_key())
+            self.url_result('http://vod.tvp.pl%s' % v_path, ie=TVPIE.ie_key())
             for v_path in videos_paths]
 
         return {

From fac2af3c51c92b7f9abc4f229bc9351e8a301b29 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 27 May 2016 01:41:27 +0800
Subject: [PATCH 5039/6219] [common] Fix m3u8 extraction in f4m manifests

---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index e53b7ad64..0029c3694 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1063,7 +1063,7 @@ class InfoExtractor(object):
                 elif ext == 'm3u8':
                     formats.extend(self._extract_m3u8_formats(
                         manifest_url, video_id, 'mp4', preference=preference,
-                        m3u8_id=m3u8_id, fatal=False))
+                        m3u8_id=m3u8_id, fatal=fatal))
                     continue
             tbr = int_or_none(media_el.attrib.get('bitrate'))
             formats.append({

From 3874e6ea66c738910c6a1065b2d781e04a8143ae Mon Sep 17 00:00:00 2001
From: Boris Wachtmeister <boris-code@gmx.com>
Date: Thu, 26 May 2016 16:45:14 +0200
Subject: [PATCH 5040/6219] [WDR] use single quotes for strings

---
 youtube_dl/extractor/wdr.py | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/wdr.py b/youtube_dl/extractor/wdr.py
index ec81f1a28..05bfe7deb 100644
--- a/youtube_dl/extractor/wdr.py
+++ b/youtube_dl/extractor/wdr.py
@@ -17,7 +17,7 @@ from ..utils import (
 class WDRIE(InfoExtractor):
     _CURRENT_MAUS_URL = r'https?://www.wdrmaus.de/aktuelle-sendung/(wdr|index).php5'
     _PAGE_REGEX = r'/mediathek/(?P<media_type>[^/]+)/(?P<type>[^/]+)/(?P<display_id>.+)\.html'
-    _VALID_URL = r'(?P<page_url>https?://(?:www\d\.)?wdr\d?\.de)' + _PAGE_REGEX + "|" + _CURRENT_MAUS_URL
+    _VALID_URL = r'(?P<page_url>https?://(?:www\d\.)?wdr\d?\.de)' + _PAGE_REGEX + '|' + _CURRENT_MAUS_URL
 
     _JS_URL_REGEX = r'(https?://deviceids-medp.wdr.de/ondemand/\d+/\d+\.js)'
 
@@ -116,23 +116,23 @@ class WDRIE(InfoExtractor):
         json_data = self._search_regex(r'\(({.*})\)', js_data, 'json')
         metadata = self._parse_json(json_data, display_id)
 
-        metadata_tracker_data = metadata["trackerData"]
-        metadata_media_resource = metadata["mediaResource"]
+        metadata_tracker_data = metadata['trackerData']
+        metadata_media_resource = metadata['mediaResource']
 
         formats = []
 
         # check if the metadata contains a direct URL to a file
-        metadata_media_alt = metadata_media_resource.get("alt")
+        metadata_media_alt = metadata_media_resource.get('alt')
         if metadata_media_alt:
-            for tag_name in ["videoURL", 'audioURL']:
+            for tag_name in ['videoURL', 'audioURL']:
                 if tag_name in metadata_media_alt:
                     formats.append({
                         'url': metadata_media_alt[tag_name]
                     })
 
         # check if there are flash-streams for this video
-        if "dflt" in metadata_media_resource and "videoURL" in metadata_media_resource["dflt"]:
-            video_url = metadata_media_resource["dflt"]["videoURL"]
+        if 'dflt' in metadata_media_resource and 'videoURL' in metadata_media_resource['dflt']:
+            video_url = metadata_media_resource['dflt']['videoURL']
             if video_url.endswith('.f4m'):
                 full_video_url = video_url + '?hdcore=3.2.0&plugin=aasp-3.2.0.77.18'
                 formats.extend(self._extract_f4m_formats(full_video_url, display_id, f4m_id='hds', fatal=False))
@@ -140,13 +140,13 @@ class WDRIE(InfoExtractor):
                 formats.extend(self._extract_smil_formats(video_url, 'stream', fatal=False))
 
         subtitles = {}
-        caption_url = metadata_media_resource.get("captionURL")
+        caption_url = metadata_media_resource.get('captionURL')
         if caption_url:
             subtitles['de'] = [{
                 'url': caption_url
             }]
 
-        title = metadata_tracker_data.get("trackerClipTitle")
+        title = metadata_tracker_data.get('trackerClipTitle')
         is_live = url_type == 'live'
 
         if is_live:
@@ -163,13 +163,13 @@ class WDRIE(InfoExtractor):
         self._sort_formats(formats)
 
         return {
-            'id': metadata_tracker_data.get("trackerClipId", display_id),
+            'id': metadata_tracker_data.get('trackerClipId', display_id),
             'display_id': display_id,
             'title': title,
-            'alt_title': metadata_tracker_data.get("trackerClipSubcategory"),
+            'alt_title': metadata_tracker_data.get('trackerClipSubcategory'),
             'formats': formats,
             'upload_date': upload_date,
-            'description': self._html_search_meta("Description", webpage),
+            'description': self._html_search_meta('Description', webpage),
             'is_live': is_live,
             'subtitles': subtitles,
         }

From 37f972954da0d0f1f0c5e97da8357c4baf687ee6 Mon Sep 17 00:00:00 2001
From: Boris Wachtmeister <boris-code@gmx.com>
Date: Thu, 26 May 2016 16:59:45 +0200
Subject: [PATCH 5041/6219] [WDR] use _download_json with a strip_jsonp

---
 youtube_dl/extractor/wdr.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/wdr.py b/youtube_dl/extractor/wdr.py
index 05bfe7deb..73a343c69 100644
--- a/youtube_dl/extractor/wdr.py
+++ b/youtube_dl/extractor/wdr.py
@@ -9,6 +9,7 @@ from ..compat import (
     compat_urlparse,
 )
 from ..utils import (
+    strip_jsonp,
     unified_strdate,
     ExtractorError,
 )
@@ -112,9 +113,8 @@ class WDRIE(InfoExtractor):
 
             raise ExtractorError('No downloadable streams found', expected=True)
 
-        js_data = self._download_webpage(js_url, 'metadata')
-        json_data = self._search_regex(r'\(({.*})\)', js_data, 'json')
-        metadata = self._parse_json(json_data, display_id)
+        metadata = self._download_json(
+            js_url, 'metadata', transform_source=strip_jsonp)
 
         metadata_tracker_data = metadata['trackerData']
         metadata_media_resource = metadata['mediaResource']

From bec2c14f2cf4f06f1b99e04d59779d8d103d726a Mon Sep 17 00:00:00 2001
From: Boris Wachtmeister <boris-code@gmx.com>
Date: Thu, 26 May 2016 17:30:38 +0200
Subject: [PATCH 5042/6219] [WDR] add special handling if alt-url is a m3u8

---
 youtube_dl/extractor/wdr.py | 21 ++++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/wdr.py b/youtube_dl/extractor/wdr.py
index 73a343c69..fddcbf190 100644
--- a/youtube_dl/extractor/wdr.py
+++ b/youtube_dl/extractor/wdr.py
@@ -9,6 +9,7 @@ from ..compat import (
     compat_urlparse,
 )
 from ..utils import (
+    determine_ext,
     strip_jsonp,
     unified_strdate,
     ExtractorError,
@@ -61,7 +62,7 @@ class WDRIE(InfoExtractor):
             'url': 'http://www1.wdr.de/mediathek/video/live/index.html',
             'info_dict': {
                 'id': 'mdb-103364',
-                'ext': 'flv',
+                'ext': 'mp4',
                 'display_id': 'index',
                 'title': r're:^WDR Fernsehen im Livestream [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
                 'alt_title': 'WDR Fernsehen Live',
@@ -69,7 +70,10 @@ class WDRIE(InfoExtractor):
                 'description': 'md5:ae2ff888510623bf8d4b115f95a9b7c9',
                 'is_live': True,
                 'subtitles': {}
-            }
+            },
+            'params': {
+                'skip_download': True,  # m3u8 download
+            },
         },
         {
             'url': 'http://www1.wdr.de/mediathek/video/sendungen/aktuelle-stunde/aktuelle-stunde-120.html',
@@ -126,9 +130,16 @@ class WDRIE(InfoExtractor):
         if metadata_media_alt:
             for tag_name in ['videoURL', 'audioURL']:
                 if tag_name in metadata_media_alt:
-                    formats.append({
-                        'url': metadata_media_alt[tag_name]
-                    })
+                    alt_url = metadata_media_alt[tag_name]
+                    if determine_ext(alt_url) == 'm3u8':
+                        m3u_fmt = self._extract_m3u8_formats(
+                            alt_url, display_id, 'mp4', 'm3u8_native',
+                            m3u8_id='hls')
+                        formats.extend(m3u_fmt)
+                    else:
+                        formats.append({
+                            'url': alt_url
+                        })
 
         # check if there are flash-streams for this video
         if 'dflt' in metadata_media_resource and 'videoURL' in metadata_media_resource['dflt']:

From 33a1ff7113d9dd656b3c56cb404de85646caa559 Mon Sep 17 00:00:00 2001
From: Boris Wachtmeister <boris-code@gmx.com>
Date: Thu, 26 May 2016 19:08:12 +0200
Subject: [PATCH 5043/6219] [WDR] extract jsonp-url by parsing data-extension
 of mediaLink

---
 youtube_dl/extractor/wdr.py | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/wdr.py b/youtube_dl/extractor/wdr.py
index fddcbf190..dd107ef8a 100644
--- a/youtube_dl/extractor/wdr.py
+++ b/youtube_dl/extractor/wdr.py
@@ -10,6 +10,7 @@ from ..compat import (
 )
 from ..utils import (
     determine_ext,
+    js_to_json,
     strip_jsonp,
     unified_strdate,
     ExtractorError,
@@ -21,8 +22,6 @@ class WDRIE(InfoExtractor):
     _PAGE_REGEX = r'/mediathek/(?P<media_type>[^/]+)/(?P<type>[^/]+)/(?P<display_id>.+)\.html'
     _VALID_URL = r'(?P<page_url>https?://(?:www\d\.)?wdr\d?\.de)' + _PAGE_REGEX + '|' + _CURRENT_MAUS_URL
 
-    _JS_URL_REGEX = r'(https?://deviceids-medp.wdr.de/ondemand/\d+/\d+\.js)'
-
     _TESTS = [
         {
             'url': 'http://www1.wdr.de/mediathek/video/sendungen/doku-am-freitag/video-geheimnis-aachener-dom-100.html',
@@ -102,9 +101,13 @@ class WDRIE(InfoExtractor):
         display_id = mobj.group('display_id')
         webpage = self._download_webpage(url, display_id)
 
-        js_url = self._search_regex(self._JS_URL_REGEX, webpage, 'js_url', default=None)
+        # for wdr.de the data-extension is in a tag with the class "mediaLink"
+        # for wdrmaus its in a link to the page in a multiline "videoLink"-tag
+        json_metadata = self._html_search_regex(
+            r'class=(?:"mediaLink\b[^"]*"[^>]+|"videoLink\b[^"]*"[\s]*>\n[^\n]*)data-extension="([^"]+)"',
+            webpage, 'media link', default=None, flags=re.MULTILINE)
 
-        if not js_url:
+        if not json_metadata:
             entries = [
                 self.url_result(page_url + href[0], 'WDR')
                 for href in re.findall(
@@ -117,8 +120,12 @@ class WDRIE(InfoExtractor):
 
             raise ExtractorError('No downloadable streams found', expected=True)
 
+        media_link_obj = self._parse_json(json_metadata, display_id,
+                                          transform_source=js_to_json)
+        jsonp_url = media_link_obj['mediaObj']['url']
+
         metadata = self._download_json(
-            js_url, 'metadata', transform_source=strip_jsonp)
+            jsonp_url, 'metadata', transform_source=strip_jsonp)
 
         metadata_tracker_data = metadata['trackerData']
         metadata_media_resource = metadata['mediaResource']

From 949fc42e009aed5414caad280d0dc551ffcd9c14 Mon Sep 17 00:00:00 2001
From: Boris Wachtmeister <boris-code@gmx.com>
Date: Thu, 26 May 2016 19:58:55 +0200
Subject: [PATCH 5044/6219] [WDR] the other wdrmaus.de pages also changed to
 the new player

---
 youtube_dl/extractor/extractors.py |  1 -
 youtube_dl/extractor/wdr.py        | 89 +++++-------------------------
 2 files changed, 15 insertions(+), 75 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 6de3438fc..023598130 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -923,7 +923,6 @@ from .wat import WatIE
 from .wdr import (
     WDRIE,
     WDRMobileIE,
-    WDRMausIE,
 )
 from .webofstories import (
     WebOfStoriesIE,
diff --git a/youtube_dl/extractor/wdr.py b/youtube_dl/extractor/wdr.py
index dd107ef8a..1af1e996d 100644
--- a/youtube_dl/extractor/wdr.py
+++ b/youtube_dl/extractor/wdr.py
@@ -4,10 +4,6 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_parse_qs,
-    compat_urlparse,
-)
 from ..utils import (
     determine_ext,
     js_to_json,
@@ -18,7 +14,7 @@ from ..utils import (
 
 
 class WDRIE(InfoExtractor):
-    _CURRENT_MAUS_URL = r'https?://www.wdrmaus.de/aktuelle-sendung/(wdr|index).php5'
+    _CURRENT_MAUS_URL = r'https?://(?:www\.)wdrmaus.de/(?:[^/]+/){1,2}[^/?#]+\.php5'
     _PAGE_REGEX = r'/mediathek/(?P<media_type>[^/]+)/(?P<type>[^/]+)/(?P<display_id>.+)\.html'
     _VALID_URL = r'(?P<page_url>https?://(?:www\d\.)?wdr\d?\.de)' + _PAGE_REGEX + '|' + _CURRENT_MAUS_URL
 
@@ -92,6 +88,20 @@ class WDRIE(InfoExtractor):
             },
             'skip': 'The id changes from week to week because of the new episode'
         },
+        {
+            'url': 'http://www.wdrmaus.de/sachgeschichten/sachgeschichten/achterbahn.php5',
+            'md5': 'ca365705551e4bd5217490f3b0591290',
+            'info_dict': {
+                'id': 'mdb-186083',
+                'ext': 'flv',
+                'upload_date': '20130919',
+                'title': 'Sachgeschichte - Achterbahn ',
+                'description': '- Die Sendung mit der Maus -',
+            },
+            'params': {
+                'skip_download': True,  # the file has different versions :(
+            },
+        },
     ]
 
     def _real_extract(self, url):
@@ -222,72 +232,3 @@ class WDRMobileIE(InfoExtractor):
                 'User-Agent': 'mobile',
             },
         }
-
-
-class WDRMausIE(InfoExtractor):
-    _VALID_URL = 'https?://(?:www\.)?wdrmaus\.de/(?:[^/]+/){,2}(?P<id>[^/?#]+)((?<!index)\.php5|/(?:$|[?#]))'
-    IE_DESC = 'Sendung mit der Maus'
-    _TESTS = [{
-        'url': 'http://www.wdrmaus.de/sachgeschichten/sachgeschichten/achterbahn.php5',
-        'md5': '178b432d002162a14ccb3e0876741095',
-        'info_dict': {
-            'id': 'achterbahn',
-            'ext': 'mp4',
-            'thumbnail': 're:^http://.+\.jpg',
-            'upload_date': '20131001',
-            'title': '19.09.2013 - Achterbahn',
-        }
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        webpage = self._download_webpage(url, video_id)
-        param_code = self._html_search_regex(
-            r'<a href="\?startVideo=1&amp;([^"]+)"', webpage, 'parameters')
-
-        title_date = self._search_regex(
-            r'<div class="sendedatum"><p>Sendedatum:\s*([0-9\.]+)</p>',
-            webpage, 'air date')
-        title_str = self._html_search_regex(
-            r'<h1>(.*?)</h1>', webpage, 'title')
-        title = '%s - %s' % (title_date, title_str)
-        upload_date = unified_strdate(
-            self._html_search_meta('dc.date', webpage))
-
-        fields = compat_parse_qs(param_code)
-        video_url = fields['firstVideo'][0]
-        thumbnail = compat_urlparse.urljoin(url, fields['startPicture'][0])
-
-        formats = [{
-            'format_id': 'rtmp',
-            'url': video_url,
-        }]
-
-        jscode = self._download_webpage(
-            'http://www.wdrmaus.de/codebase/js/extended-medien.min.js',
-            video_id, fatal=False,
-            note='Downloading URL translation table',
-            errnote='Could not download URL translation table')
-        if jscode:
-            for m in re.finditer(
-                    r"stream:\s*'dslSrc=(?P<stream>[^']+)',\s*download:\s*'(?P<dl>[^']+)'\s*\}",
-                    jscode):
-                if video_url.startswith(m.group('stream')):
-                    http_url = video_url.replace(
-                        m.group('stream'), m.group('dl'))
-                    formats.append({
-                        'format_id': 'http',
-                        'url': http_url,
-                    })
-                    break
-
-        self._sort_formats(formats)
-
-        return {
-            'id': video_id,
-            'title': title,
-            'formats': formats,
-            'thumbnail': thumbnail,
-            'upload_date': upload_date,
-        }

From 3a686853e1739dfc26548cdc09fe89e693e76a9f Mon Sep 17 00:00:00 2001
From: Boris Wachtmeister <boris-code@gmx.com>
Date: Thu, 26 May 2016 20:16:33 +0200
Subject: [PATCH 5045/6219] [WDR] fixed parsing of playlists

---
 youtube_dl/extractor/wdr.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/wdr.py b/youtube_dl/extractor/wdr.py
index 1af1e996d..1e729cb7c 100644
--- a/youtube_dl/extractor/wdr.py
+++ b/youtube_dl/extractor/wdr.py
@@ -72,7 +72,7 @@ class WDRIE(InfoExtractor):
         },
         {
             'url': 'http://www1.wdr.de/mediathek/video/sendungen/aktuelle-stunde/aktuelle-stunde-120.html',
-            'playlist_mincount': 10,
+            'playlist_mincount': 8,
             'info_dict': {
                 'id': 'aktuelle-stunde/aktuelle-stunde-120',
             },
@@ -121,7 +121,7 @@ class WDRIE(InfoExtractor):
             entries = [
                 self.url_result(page_url + href[0], 'WDR')
                 for href in re.findall(
-                    r'<a href="(%s)"' % self._PAGE_REGEX,
+                    r'<a href="(%s)"[^>]+data-extension=' % self._PAGE_REGEX,
                     webpage)
             ]
 

From 2615fa758422deaaf11049e71f0c183e655c0b76 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 27 May 2016 01:46:12 +0600
Subject: [PATCH 5046/6219] [downloader/f4m] Simply select format when it's the
 only one

---
 youtube_dl/downloader/f4m.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index 314def4cb..8f88b0241 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -319,7 +319,7 @@ class F4mFD(FragmentFD):
         doc = compat_etree_fromstring(manifest)
         formats = [(int(f.attrib.get('bitrate', -1)), f)
                    for f in self._get_unencrypted_media(doc)]
-        if requested_bitrate is None:
+        if requested_bitrate is None or len(formats) == 1:
             # get the best format
             formats = sorted(formats, key=lambda f: f[0])
             rate, media = formats[-1]

From 77b8b4e696dd5ffb1330a2de328eb9c3ecd09a15 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 27 May 2016 01:47:44 +0600
Subject: [PATCH 5047/6219] [extractor/common] Borrow quality metadata from
 parent set-level manifest for f4m

---
 youtube_dl/extractor/common.py | 28 ++++++++++++++++++++++------
 1 file changed, 22 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 0029c3694..57793537b 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1035,6 +1035,10 @@ class InfoExtractor(object):
             'bootstrap info', default=None)
 
         for i, media_el in enumerate(media_nodes):
+            tbr = int_or_none(media_el.attrib.get('bitrate'))
+            width = int_or_none(media_el.attrib.get('width'))
+            height = int_or_none(media_el.attrib.get('height'))
+            format_id = '-'.join(filter(None, [f4m_id, compat_str(i if tbr is None else tbr)]))
             # If <bootstrapInfo> is present, the specified f4m is a
             # stream-level manifest, and only set-level manifests may refer to
             # external resources.  See section 11.4 and section 4 of F4M spec
@@ -1056,23 +1060,35 @@ class InfoExtractor(object):
                 # bitrate in f4m downloader
                 ext = determine_ext(manifest_url)
                 if ext == 'f4m':
-                    formats.extend(self._extract_f4m_formats(
+                    f4m_formats = self._extract_f4m_formats(
                         manifest_url, video_id, preference=preference, f4m_id=f4m_id,
-                        transform_source=transform_source, fatal=fatal))
+                        transform_source=transform_source, fatal=fatal)
+                    # Sometimes stream-level manifest contains single media entry that
+                    # does not contain any quality metadata (e.g. http://matchtv.ru/#live-player).
+                    # At the same time parent's media entry in set-level manifest may
+                    # contain it. We will copy it from parent in such cases.
+                    if len(f4m_formats) == 1:
+                        f = f4m_formats[0]
+                        f.update({
+                            'tbr': f.get('tbr') or tbr,
+                            'width': f.get('width') or width,
+                            'height': f.get('height') or height,
+                            'format_id': f.get('format_id') if not tbr else format_id,
+                        })
+                    formats.extend(f4m_formats)
                     continue
                 elif ext == 'm3u8':
                     formats.extend(self._extract_m3u8_formats(
                         manifest_url, video_id, 'mp4', preference=preference,
                         m3u8_id=m3u8_id, fatal=fatal))
                     continue
-            tbr = int_or_none(media_el.attrib.get('bitrate'))
             formats.append({
-                'format_id': '-'.join(filter(None, [f4m_id, compat_str(i if tbr is None else tbr)])),
+                'format_id': format_id,
                 'url': manifest_url,
                 'ext': 'flv' if bootstrap_info else None,
                 'tbr': tbr,
-                'width': int_or_none(media_el.attrib.get('width')),
-                'height': int_or_none(media_el.attrib.get('height')),
+                'width': width,
+                'height': height,
                 'preference': preference,
             })
         return formats

From f36532404dedb08f103083fba931864927de369d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 27 May 2016 22:19:10 +0600
Subject: [PATCH 5048/6219] [vk] Remove superfluous code

---
 youtube_dl/extractor/vk.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 041d93629..79c819bc3 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -217,7 +217,6 @@ class VKIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('videoid')
 
-        info_url = url
         if video_id:
             info_url = 'https://vk.com/al_video.php?act=show&al=1&module=video&video=%s' % video_id
             # Some videos (removed?) can only be downloaded with list id specified

From 11c70deba792de58c64c82d96ffcfdf295483b84 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 27 May 2016 23:34:58 +0600
Subject: [PATCH 5049/6219] [coub] Add extractor (Closes #9609)

---
 youtube_dl/extractor/coub.py       | 139 +++++++++++++++++++++++++++++
 youtube_dl/extractor/extractors.py |   1 +
 2 files changed, 140 insertions(+)
 create mode 100644 youtube_dl/extractor/coub.py

diff --git a/youtube_dl/extractor/coub.py b/youtube_dl/extractor/coub.py
new file mode 100644
index 000000000..c3b09b177
--- /dev/null
+++ b/youtube_dl/extractor/coub.py
@@ -0,0 +1,139 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    float_or_none,
+    int_or_none,
+    parse_iso8601,
+    qualities,
+)
+
+
+class CoubIE(InfoExtractor):
+    _VALID_URL = r'(?:coub:|https?://(?:coub\.com/(?:view|embed|coubs)/|c-cdn\.coub\.com/fb-player\.swf\?.*\bcoub(?:ID|id)=))(?P<id>[\da-z]+)'
+
+    _TESTS = [{
+        'url': 'http://coub.com/view/5u5n1',
+        'info_dict': {
+            'id': '5u5n1',
+            'ext': 'mp4',
+            'title': 'The Matrix Moonwalk',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 4.6,
+            'timestamp': 1428527772,
+            'upload_date': '20150408',
+            'uploader': 'Артём Лоскутников',
+            'uploader_id': 'artyom.loskutnikov',
+            'view_count': int,
+            'like_count': int,
+            'repost_count': int,
+            'comment_count': int,
+            'age_limit': 0,
+        },
+    }, {
+        'url': 'http://c-cdn.coub.com/fb-player.swf?bot_type=vk&coubID=7w5a4',
+        'only_matching': True,
+    }, {
+        'url': 'coub:5u5n1',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        coub = self._download_json(
+            'http://coub.com/api/v2/coubs/%s.json' % video_id, video_id)
+
+        if coub.get('error'):
+            raise ExtractorError(
+                '%s said: %s' % (self.IE_NAME, coub['error']), expected=True)
+
+        title = coub['title']
+
+        file_versions = coub['file_versions']
+
+        QUALITIES = ('low', 'med', 'high')
+
+        MOBILE = 'mobile'
+        IPHONE = 'iphone'
+        HTML5 = 'html5'
+
+        SOURCE_PREFERENCE = (MOBILE, IPHONE, HTML5)
+
+        quality_key = qualities(QUALITIES)
+        preference_key = qualities(SOURCE_PREFERENCE)
+
+        formats = []
+
+        for kind, items in file_versions.get(HTML5, {}).items():
+            if kind not in ('video', 'audio'):
+                continue
+            if not isinstance(items, dict):
+                continue
+            for quality, item in items.items():
+                if not isinstance(item, dict):
+                    continue
+                item_url = item.get('url')
+                if not item_url:
+                    continue
+                formats.append({
+                    'url': item_url,
+                    'format_id': '%s-%s-%s' % (HTML5, kind, quality),
+                    'filesize': int_or_none(item.get('size')),
+                    'vcodec': 'none' if kind == 'audio' else None,
+                    'quality': quality_key(quality),
+                    'preference': preference_key(HTML5),
+                })
+
+        iphone_url = file_versions.get(IPHONE, {}).get('url')
+        if iphone_url:
+            formats.append({
+                'url': iphone_url,
+                'format_id': IPHONE,
+                'preference': preference_key(IPHONE),
+            })
+
+        mobile_url = file_versions.get(MOBILE, {}).get('audio_url')
+        if mobile_url:
+            formats.append({
+                'url': mobile_url,
+                'format_id': '%s-audio' % MOBILE,
+                'preference': preference_key(MOBILE),
+            })
+
+        self._sort_formats(formats)
+
+        thumbnail = coub.get('picture')
+        duration = float_or_none(coub.get('duration'))
+        timestamp = parse_iso8601(coub.get('published_at') or coub.get('created_at'))
+        uploader = coub.get('channel', {}).get('title')
+        uploader_id = coub.get('channel', {}).get('permalink')
+
+        view_count = int_or_none(coub.get('views_count') or coub.get('views_increase_count'))
+        like_count = int_or_none(coub.get('likes_count'))
+        repost_count = int_or_none(coub.get('recoubs_count'))
+        comment_count = int_or_none(coub.get('comments_count'))
+
+        age_restricted = coub.get('age_restricted', coub.get('age_restricted_by_admin'))
+        if age_restricted is not None:
+            age_limit = 18 if age_restricted is True else 0
+        else:
+            age_limit = None
+
+        return {
+            'id': video_id,
+            'title': title,
+            'thumbnail': thumbnail,
+            'duration': duration,
+            'timestamp': timestamp,
+            'uploader': uploader,
+            'uploader_id': uploader_id,
+            'view_count': view_count,
+            'like_count': like_count,
+            'repost_count': repost_count,
+            'comment_count': comment_count,
+            'age_limit': age_limit,
+            'formats': formats,
+        }
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index ddf62139e..dd4b2b838 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -143,6 +143,7 @@ from .cnn import (
     CNNBlogsIE,
     CNNArticleIE,
 )
+from .coub import CoubIE
 from .collegerama import CollegeRamaIE
 from .comedycentral import ComedyCentralIE, ComedyCentralShowsIE
 from .comcarcoff import ComCarCoffIE

From de7d76af52c6cb462dfab967d57f5fa7cd17df50 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 27 May 2016 23:38:17 +0600
Subject: [PATCH 5050/6219] [coub] Add another test

---
 youtube_dl/extractor/coub.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/coub.py b/youtube_dl/extractor/coub.py
index c3b09b177..a901b8d22 100644
--- a/youtube_dl/extractor/coub.py
+++ b/youtube_dl/extractor/coub.py
@@ -38,6 +38,10 @@ class CoubIE(InfoExtractor):
     }, {
         'url': 'coub:5u5n1',
         'only_matching': True,
+    }, {
+        # longer video id
+        'url': 'http://coub.com/view/237d5l5h',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 807cf7b07f6ac1299f5578ea1264b43fc30d8301 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 28 May 2016 21:18:24 +0600
Subject: [PATCH 5051/6219] [udemy] Fix authentication for localized layout
 (Closes #9594)

---
 youtube_dl/extractor/udemy.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index 13e0cd237..89b869559 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -142,7 +142,9 @@ class UdemyIE(InfoExtractor):
             self._LOGIN_URL, None, 'Downloading login popup')
 
         def is_logged(webpage):
-            return any(p in webpage for p in ['href="https://www.udemy.com/user/logout/', '>Logout<'])
+            return any(re.search(p, webpage) for p in (
+                r'href=["\'](?:https://www\.udemy\.com)?/user/logout/',
+                r'>Logout<'))
 
         # already logged in
         if is_logged(login_popup):

From 6461f2b7ec5fa0114d4bb38ca27c0f72edff8e23 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 29 May 2016 01:26:00 +0800
Subject: [PATCH 5052/6219] [bilibili] Fix extraction, improve and cleanup

---
 youtube_dl/extractor/bilibili.py | 129 ++++++++++++++++++++-----------
 1 file changed, 86 insertions(+), 43 deletions(-)

diff --git a/youtube_dl/extractor/bilibili.py b/youtube_dl/extractor/bilibili.py
index 8baff2041..71a54b4f4 100644
--- a/youtube_dl/extractor/bilibili.py
+++ b/youtube_dl/extractor/bilibili.py
@@ -1,34 +1,42 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import calendar
+import datetime
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_str
+from ..compat import (
+    compat_etree_fromstring,
+    compat_str,
+    compat_parse_qs,
+    compat_xml_parse_error,
+)
 from ..utils import (
-    int_or_none,
-    unescapeHTML,
     ExtractorError,
+    int_or_none,
+    float_or_none,
     xpath_text,
 )
 
 
 class BiliBiliIE(InfoExtractor):
-    _VALID_URL = r'https?://www\.bilibili\.(?:tv|com)/video/av(?P<id>\d+)(?:/index_(?P<page_num>\d+).html)?'
+    _VALID_URL = r'https?://www\.bilibili\.(?:tv|com)/video/av(?P<id>\d+)'
 
     _TESTS = [{
         'url': 'http://www.bilibili.tv/video/av1074402/',
-        'md5': '2c301e4dab317596e837c3e7633e7d86',
+        'md5': '5f7d29e1a2872f3df0cf76b1f87d3788',
         'info_dict': {
             'id': '1554319',
             'ext': 'flv',
             'title': '【金坷垃】金泡沫',
-            'duration': 308313,
+            'description': 'md5:ce18c2a2d2193f0df2917d270f2e5923',
+            'duration': 308.067,
+            'timestamp': 1398012660,
             'upload_date': '20140420',
             'thumbnail': 're:^https?://.+\.jpg',
-            'description': 'md5:ce18c2a2d2193f0df2917d270f2e5923',
-            'timestamp': 1397983878,
             'uploader': '菊子桑',
+            'uploader_id': '156160',
         },
     }, {
         'url': 'http://www.bilibili.com/video/av1041170/',
@@ -36,75 +44,110 @@ class BiliBiliIE(InfoExtractor):
             'id': '1041170',
             'title': '【BD1080P】刀语【诸神&异域】',
             'description': '这是个神奇的故事~每个人不留弹幕不给走哦~切利哦！~',
-            'uploader': '枫叶逝去',
-            'timestamp': 1396501299,
         },
         'playlist_count': 9,
     }]
 
+    # BiliBili blocks keys from time to time. The current key is extracted from
+    # the Android client
+    # TODO: find the sign algorithm used in the flash player
+    _APP_KEY = '86385cdc024c0f6c'
+
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
-        page_num = mobj.group('page_num') or '1'
 
-        view_data = self._download_json(
-            'http://api.bilibili.com/view?type=json&appkey=8e9fc618fbd41e28&id=%s&page=%s' % (video_id, page_num),
-            video_id)
-        if 'error' in view_data:
-            raise ExtractorError('%s said: %s' % (self.IE_NAME, view_data['error']), expected=True)
+        webpage = self._download_webpage(url, video_id)
 
-        cid = view_data['cid']
-        title = unescapeHTML(view_data['title'])
+        params = compat_parse_qs(self._search_regex(
+            [r'EmbedPlayer\([^)]+,\s*"([^"]+)"\)',
+             r'<iframe[^>]+src="https://secure\.bilibili\.com/secure,([^"]+)"'],
+            webpage, 'player parameters'))
+        cid = params['cid'][0]
 
-        doc = self._download_xml(
-            'http://interface.bilibili.com/v_cdn_play?appkey=8e9fc618fbd41e28&cid=%s' % cid,
-            cid,
-            'Downloading page %s/%s' % (page_num, view_data['pages'])
-        )
+        info_xml_str = self._download_webpage(
+            'http://interface.bilibili.com/v_cdn_play',
+            cid, query={'appkey': self._APP_KEY, 'cid': cid},
+            note='Downloading video info page')
 
-        if xpath_text(doc, './result') == 'error':
-            raise ExtractorError('%s said: %s' % (self.IE_NAME, xpath_text(doc, './message')), expected=True)
+        err_msg = None
+        durls = None
+        info_xml = None
+        try:
+            info_xml = compat_etree_fromstring(info_xml_str.encode('utf-8'))
+        except compat_xml_parse_error:
+            info_json = self._parse_json(info_xml_str, video_id, fatal=False)
+            err_msg = (info_json or {}).get('error_text')
+        else:
+            err_msg = xpath_text(info_xml, './message')
+
+        if info_xml is not None:
+            durls = info_xml.findall('./durl')
+        if not durls:
+            if err_msg:
+                raise ExtractorError('%s said: %s' % (self.IE_NAME, err_msg), expected=True)
+            else:
+                raise ExtractorError('No videos found!')
 
         entries = []
 
-        for durl in doc.findall('./durl'):
+        for durl in durls:
             size = xpath_text(durl, ['./filesize', './size'])
             formats = [{
                 'url': durl.find('./url').text,
                 'filesize': int_or_none(size),
-                'ext': 'flv',
             }]
-            backup_urls = durl.find('./backup_url')
-            if backup_urls is not None:
-                for backup_url in backup_urls.findall('./url'):
-                    formats.append({'url': backup_url.text})
-            formats.reverse()
+            for backup_url in durl.findall('./backup_url/url'):
+                formats.append({
+                    'url': backup_url.text,
+                    # backup URLs have lower priorities
+                    'preference': -2 if 'hd.mp4' in backup_url.text else -3,
+                })
+
+            self._sort_formats(formats)
 
             entries.append({
                 'id': '%s_part%s' % (cid, xpath_text(durl, './order')),
-                'title': title,
                 'duration': int_or_none(xpath_text(durl, './length'), 1000),
                 'formats': formats,
             })
 
+        title = self._html_search_regex('<h1[^>]+title="([^"]+)">', webpage, 'title')
+        description = self._html_search_meta('description', webpage)
+        datetime_str = self._html_search_regex(
+            r'<time[^>]+datetime="([^"]+)"', webpage, 'upload time', fatal=False)
+        if datetime_str:
+            timestamp = calendar.timegm(datetime.datetime.strptime(datetime_str, '%Y-%m-%dT%H:%M').timetuple())
+
+        # TODO 'view_count' requires deobfuscating Javascript
         info = {
             'id': compat_str(cid),
             'title': title,
-            'description': view_data.get('description'),
-            'thumbnail': view_data.get('pic'),
-            'uploader': view_data.get('author'),
-            'timestamp': int_or_none(view_data.get('created')),
-            'view_count': int_or_none(view_data.get('play')),
-            'duration': int_or_none(xpath_text(doc, './timelength')),
+            'description': description,
+            'timestamp': timestamp,
+            'thumbnail': self._html_search_meta('thumbnailUrl', webpage),
+            'duration': float_or_none(xpath_text(info_xml, './timelength'), scale=1000),
         }
 
+        uploader_mobj = re.search(
+            r'<a[^>]+href="https?://space\.bilibili\.com/(?P<id>\d+)"[^>]+title="(?P<name>[^"]+)"',
+            webpage)
+        if uploader_mobj:
+            info.update({
+                'uploader': uploader_mobj.group('name'),
+                'uploader_id': uploader_mobj.group('id'),
+            })
+
+        for entry in entries:
+            entry.update(info)
+
         if len(entries) == 1:
-            entries[0].update(info)
             return entries[0]
         else:
-            info.update({
+            return {
                 '_type': 'multi_video',
                 'id': video_id,
+                'title': title,
+                'description': description,
                 'entries': entries,
-            })
-            return info
+            }

From 92cf872a4870482e797bcd54316a4b8dc024fcc5 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 29 May 2016 01:58:27 +0800
Subject: [PATCH 5053/6219] [.gitignore] Ignore mp3 files

[ci skip]
---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index d5f216b5f..d13551274 100644
--- a/.gitignore
+++ b/.gitignore
@@ -28,6 +28,7 @@ updates_key.pem
 *.mp4
 *.m4a
 *.m4v
+*.mp3
 *.part
 *.swp
 test/testdata

From 2bee7b25f39471c1ac5641b714e003bcf8335d15 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 29 May 2016 01:59:09 +0800
Subject: [PATCH 5054/6219] [Makefile] Cleanup m4a files

[ci skip]
---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index d760e4576..3861b23d5 100644
--- a/Makefile
+++ b/Makefile
@@ -1,7 +1,7 @@
 all: youtube-dl README.md CONTRIBUTING.md README.txt youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtube-dl.fish supportedsites
 
 clean:
-	rm -rf youtube-dl.1.temp.md youtube-dl.1 youtube-dl.bash-completion README.txt MANIFEST build/ dist/ .coverage cover/ youtube-dl.tar.gz youtube-dl.zsh youtube-dl.fish youtube_dl/extractor/lazy_extractors.py *.dump *.part *.info.json *.mp4 *.flv *.mp3 *.avi *.mkv *.webm *.jpg *.png CONTRIBUTING.md.tmp ISSUE_TEMPLATE.md.tmp youtube-dl youtube-dl.exe
+	rm -rf youtube-dl.1.temp.md youtube-dl.1 youtube-dl.bash-completion README.txt MANIFEST build/ dist/ .coverage cover/ youtube-dl.tar.gz youtube-dl.zsh youtube-dl.fish youtube_dl/extractor/lazy_extractors.py *.dump *.part *.info.json *.mp4 *.m4a *.flv *.mp3 *.avi *.mkv *.webm *.jpg *.png CONTRIBUTING.md.tmp ISSUE_TEMPLATE.md.tmp youtube-dl youtube-dl.exe
 	find . -name "*.pyc" -delete
 	find . -name "*.class" -delete
 

From 2a329110b90102720bf5de00355473586674040f Mon Sep 17 00:00:00 2001
From: venth <artur.krysiak.warszawa@gmail.com>
Date: Tue, 19 Apr 2016 11:17:46 +0200
Subject: [PATCH 5055/6219] ignored intellij related files

---
 .gitignore | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/.gitignore b/.gitignore
index d13551274..a802c75a1 100644
--- a/.gitignore
+++ b/.gitignore
@@ -35,6 +35,9 @@ test/testdata
 test/local_parameters.json
 .tox
 youtube-dl.zsh
+
+# IntelliJ related files
 .idea
-.idea/*
+*.iml
+
 tmp/

From f574103d7ca08a63e0dc58fdd7efde0871b9b395 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 29 May 2016 09:03:17 +0600
Subject: [PATCH 5056/6219] [buildserver] Fix buildserver and make python2
 compatible

---
 devscripts/buildserver.py | 62 +++++++++++++++++++++++++++------------
 1 file changed, 43 insertions(+), 19 deletions(-)

diff --git a/devscripts/buildserver.py b/devscripts/buildserver.py
index 7c2f49f8b..dada6bfc7 100644
--- a/devscripts/buildserver.py
+++ b/devscripts/buildserver.py
@@ -1,17 +1,42 @@
 #!/usr/bin/python3
 
-from http.server import HTTPServer, BaseHTTPRequestHandler
-from socketserver import ThreadingMixIn
 import argparse
 import ctypes
 import functools
+import shutil
+import subprocess
 import sys
+import tempfile
 import threading
 import traceback
 import os.path
 
+sys.path.insert(0, os.path.dirname(os.path.dirname((os.path.abspath(__file__)))))
+from youtube_dl.compat import (
+    compat_http_server,
+    compat_str,
+    compat_urlparse,
+)
 
-class BuildHTTPServer(ThreadingMixIn, HTTPServer):
+# These are not used outside of buildserver.py thus not in compat.py
+
+try:
+    import winreg as compat_winreg
+except ImportError:  # Python 2
+    import _winreg as compat_winreg
+
+try:
+    import socketserver as compat_socketserver
+except ImportError:  # Python 2
+    import SocketServer as compat_socketserver
+
+try:
+    compat_input = raw_input
+except NameError:  # Python 3
+    compat_input = input
+
+
+class BuildHTTPServer(compat_socketserver.ThreadingMixIn, compat_http_server.HTTPServer):
     allow_reuse_address = True
 
 
@@ -216,7 +241,7 @@ def main(args=None):
     srv = BuildHTTPServer((host, port), BuildHTTPRequestHandler)
     thr = threading.Thread(target=srv.serve_forever)
     thr.start()
-    input('Press ENTER to shut down')
+    compat_input('Press ENTER to shut down')
     srv.shutdown()
     thr.join()
 
@@ -231,8 +256,6 @@ def rmtree(path):
             os.remove(fname)
     os.rmdir(path)
 
-#==============================================================================
-
 
 class BuildError(Exception):
     def __init__(self, output, code=500):
@@ -249,15 +272,16 @@ class HTTPError(BuildError):
 
 class PythonBuilder(object):
     def __init__(self, **kwargs):
-        pythonVersion = kwargs.pop('python', '2.7')
+        python_version = kwargs.pop('python', '3.4')
         try:
-            key = _winreg.OpenKey(_winreg.HKEY_LOCAL_MACHINE, r'SOFTWARE\Python\PythonCore\%s\InstallPath' % pythonVersion)
+            key = compat_winreg.OpenKey(
+                compat_winreg.HKEY_LOCAL_MACHINE, r'SOFTWARE\Python\PythonCore\%s\InstallPath' % python_version)
             try:
-                self.pythonPath, _ = _winreg.QueryValueEx(key, '')
+                self.pythonPath, _ = compat_winreg.QueryValueEx(key, '')
             finally:
-                _winreg.CloseKey(key)
+                compat_winreg.CloseKey(key)
         except Exception:
-            raise BuildError('No such Python version: %s' % pythonVersion)
+            raise BuildError('No such Python version: %s' % python_version)
 
         super(PythonBuilder, self).__init__(**kwargs)
 
@@ -305,8 +329,10 @@ class YoutubeDLBuilder(object):
 
     def build(self):
         try:
-            subprocess.check_output([os.path.join(self.pythonPath, 'python.exe'), 'setup.py', 'py2exe'],
-                                    cwd=self.buildPath)
+            proc = subprocess.Popen([os.path.join(self.pythonPath, 'python.exe'), 'setup.py', 'py2exe'], stdin=subprocess.PIPE, cwd=self.buildPath)
+            proc.wait()
+            #subprocess.check_output([os.path.join(self.pythonPath, 'python.exe'), 'setup.py', 'py2exe'],
+            #                        cwd=self.buildPath)
         except subprocess.CalledProcessError as e:
             raise BuildError(e.output)
 
@@ -369,12 +395,12 @@ class Builder(PythonBuilder, GITBuilder, YoutubeDLBuilder, DownloadBuilder, Clea
     pass
 
 
-class BuildHTTPRequestHandler(BaseHTTPRequestHandler):
+class BuildHTTPRequestHandler(compat_http_server.BaseHTTPRequestHandler):
     actionDict = {'build': Builder, 'download': Builder}  # They're the same, no more caching.
 
     def do_GET(self):
-        path = urlparse.urlparse(self.path)
-        paramDict = dict([(key, value[0]) for key, value in urlparse.parse_qs(path.query).items()])
+        path = compat_urlparse.urlparse(self.path)
+        paramDict = dict([(key, value[0]) for key, value in compat_urlparse.parse_qs(path.query).items()])
         action, _, path = path.path.strip('/').partition('/')
         if path:
             path = path.split('/')
@@ -388,7 +414,7 @@ class BuildHTTPRequestHandler(BaseHTTPRequestHandler):
                         builder.close()
                 except BuildError as e:
                     self.send_response(e.code)
-                    msg = unicode(e).encode('UTF-8')
+                    msg = compat_str(e).encode('UTF-8')
                     self.send_header('Content-Type', 'text/plain; charset=UTF-8')
                     self.send_header('Content-Length', len(msg))
                     self.end_headers()
@@ -400,7 +426,5 @@ class BuildHTTPRequestHandler(BaseHTTPRequestHandler):
         else:
             self.send_response(500, 'Malformed URL')
 
-#==============================================================================
-
 if __name__ == '__main__':
     main()

From 44c88923696d383bb1a74d9890e7e3126b846625 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 29 May 2016 09:06:10 +0600
Subject: [PATCH 5057/6219] [devscripts/prepare_manpage] Fix manpage generation
 on Windows

---
 devscripts/prepare_manpage.py | 61 +++++++++++++++++++++--------------
 1 file changed, 36 insertions(+), 25 deletions(-)

diff --git a/devscripts/prepare_manpage.py b/devscripts/prepare_manpage.py
index 776e6556e..e3f6339b5 100644
--- a/devscripts/prepare_manpage.py
+++ b/devscripts/prepare_manpage.py
@@ -1,13 +1,46 @@
 from __future__ import unicode_literals
 
 import io
+import optparse
 import os.path
-import sys
 import re
 
 ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 README_FILE = os.path.join(ROOT_DIR, 'README.md')
 
+PREFIX = '''%YOUTUBE-DL(1)
+
+# NAME
+
+youtube\-dl \- download videos from youtube.com or other video platforms
+
+# SYNOPSIS
+
+**youtube-dl** \[OPTIONS\] URL [URL...]
+
+'''
+
+
+def main():
+    parser = optparse.OptionParser(usage='%prog OUTFILE.md')
+    options, args = parser.parse_args()
+    if len(args) != 1:
+        parser.error('Expected an output filename')
+
+    outfile, = args
+
+    with io.open(README_FILE, encoding='utf-8') as f:
+        readme = f.read()
+
+    readme = re.sub(r'(?s)^.*?(?=# DESCRIPTION)', '', readme)
+    readme = re.sub(r'\s+youtube-dl \[OPTIONS\] URL \[URL\.\.\.\]', '', readme)
+    readme = PREFIX + readme
+
+    readme = filter_options(readme)
+
+    with io.open(outfile, 'w', encoding='utf-8') as outf:
+        outf.write(readme)
+
 
 def filter_options(readme):
     ret = ''
@@ -37,27 +70,5 @@ def filter_options(readme):
 
     return ret
 
-with io.open(README_FILE, encoding='utf-8') as f:
-    readme = f.read()
-
-PREFIX = '''%YOUTUBE-DL(1)
-
-# NAME
-
-youtube\-dl \- download videos from youtube.com or other video platforms
-
-# SYNOPSIS
-
-**youtube-dl** \[OPTIONS\] URL [URL...]
-
-'''
-readme = re.sub(r'(?s)^.*?(?=# DESCRIPTION)', '', readme)
-readme = re.sub(r'\s+youtube-dl \[OPTIONS\] URL \[URL\.\.\.\]', '', readme)
-readme = PREFIX + readme
-
-readme = filter_options(readme)
-
-if sys.version_info < (3, 0):
-    print(readme.encode('utf-8'))
-else:
-    print(readme)
+if __name__ == '__main__':
+    main()

From 27f17c0eabde55cbaab613280f60c01f5ee01025 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 29 May 2016 09:11:16 +0600
Subject: [PATCH 5058/6219] [Makefile] Fix youtube-dl.1 target

Now it accepts output filename as argument
---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 3861b23d5..6ee4ba4eb 100644
--- a/Makefile
+++ b/Makefile
@@ -69,7 +69,7 @@ README.txt: README.md
 	pandoc -f markdown -t plain README.md -o README.txt
 
 youtube-dl.1: README.md
-	$(PYTHON) devscripts/prepare_manpage.py >youtube-dl.1.temp.md
+	$(PYTHON) devscripts/prepare_manpage.py youtube-dl.1.temp.md
 	pandoc -s -f markdown -t man youtube-dl.1.temp.md -o youtube-dl.1
 	rm -f youtube-dl.1.temp.md
 

From 165e3561e9ec8f8a1a1037e4fdebe880cdbd92fb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 29 May 2016 10:02:00 +0600
Subject: [PATCH 5059/6219] [devscripts/buildserver] Check Wow6432Node first
 when searching for python

This allows building releases from 64bit OS
---
 devscripts/buildserver.py | 23 ++++++++++++++++-------
 1 file changed, 16 insertions(+), 7 deletions(-)

diff --git a/devscripts/buildserver.py b/devscripts/buildserver.py
index dada6bfc7..2bd12da50 100644
--- a/devscripts/buildserver.py
+++ b/devscripts/buildserver.py
@@ -273,16 +273,25 @@ class HTTPError(BuildError):
 class PythonBuilder(object):
     def __init__(self, **kwargs):
         python_version = kwargs.pop('python', '3.4')
-        try:
-            key = compat_winreg.OpenKey(
-                compat_winreg.HKEY_LOCAL_MACHINE, r'SOFTWARE\Python\PythonCore\%s\InstallPath' % python_version)
+        python_path = None
+        for node in ('Wow6432Node\\', ''):
             try:
-                self.pythonPath, _ = compat_winreg.QueryValueEx(key, '')
-            finally:
-                compat_winreg.CloseKey(key)
-        except Exception:
+                key = compat_winreg.OpenKey(
+                    compat_winreg.HKEY_LOCAL_MACHINE,
+                    r'SOFTWARE\%sPython\PythonCore\%s\InstallPath' % (node, python_version))
+                try:
+                    python_path, _ = compat_winreg.QueryValueEx(key, '')
+                finally:
+                    compat_winreg.CloseKey(key)
+                break
+            except Exception:
+                pass
+
+        if not python_path:
             raise BuildError('No such Python version: %s' % python_version)
 
+        self.pythonPath = python_path
+
         super(PythonBuilder, self).__init__(**kwargs)
 
 

From f3fb420b827ce04dff101b64d81f8658fa2e5c73 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 29 May 2016 11:49:14 +0600
Subject: [PATCH 5060/6219] [devscripts/release.sh] Check for wheel

---
 devscripts/release.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/devscripts/release.sh b/devscripts/release.sh
index 7dd391b38..c34567f4c 100755
--- a/devscripts/release.sh
+++ b/devscripts/release.sh
@@ -6,7 +6,7 @@
 # * the git config user.signingkey is properly set
 
 # You will need
-# pip install coverage nose rsa
+# pip install coverage nose rsa wheel
 
 # TODO
 # release notes
@@ -35,6 +35,7 @@ if [ ! -z "$useless_files" ]; then echo "ERROR: Non-.py files in youtube_dl: $us
 if [ ! -f "updates_key.pem" ]; then echo 'ERROR: updates_key.pem missing'; exit 1; fi
 if ! type pandoc >/dev/null 2>/dev/null; then echo 'ERROR: pandoc is missing'; exit 1; fi
 if ! python3 -c 'import rsa' 2>/dev/null; then echo 'ERROR: python3-rsa is missing'; exit 1; fi
+if ! python3 -c 'import wheel' 2>/dev/null; then echo 'ERROR: wheel is missing'; exit 1; fi
 
 /bin/echo -e "\n### First of all, testing..."
 make clean

From 9ed6d8c6c5b0c7a411d6b97d269a3e786875d66a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 29 May 2016 13:54:05 +0800
Subject: [PATCH 5061/6219] [youku] Extract resolution

---
 youtube_dl/extractor/youku.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index 349ce0941..dbccbe228 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -275,6 +275,8 @@ class YoukuIE(InfoExtractor):
                     'format_id': self.get_format_name(fm),
                     'ext': self.parse_ext_l(fm),
                     'filesize': int(seg['size']),
+                    'width': stream.get('width'),
+                    'height': stream.get('height'),
                 })
 
         return {

From 681b923b5ca04338dfacd4154f627255d6e27d3a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 29 May 2016 23:36:42 +0700
Subject: [PATCH 5062/6219] [devscripts/release.sh] Allow passing buildserver
 address as cli option

---
 devscripts/release.sh | 28 +++++++++++++++++++++++-----
 1 file changed, 23 insertions(+), 5 deletions(-)

diff --git a/devscripts/release.sh b/devscripts/release.sh
index c34567f4c..cde4d0a39 100755
--- a/devscripts/release.sh
+++ b/devscripts/release.sh
@@ -15,10 +15,28 @@
 set -e
 
 skip_tests=true
-if [ "$1" = '--run-tests' ]; then
-    skip_tests=false
-    shift
-fi
+buildserver='localhost:8142'
+
+while true
+do
+case "$1" in
+    --run-tests)
+        skip_tests=false
+        shift
+    ;;
+    --buildserver)
+        buildserver="$2"
+        shift 2
+    ;;
+    --*)
+        echo "ERROR: unknown option $1"
+        exit 1
+    ;;
+    *)
+        break
+    ;;
+esac
+done
 
 if [ -z "$1" ]; then echo "ERROR: specify version number like this: $0 1994.09.06"; exit 1; fi
 version="$1"
@@ -67,7 +85,7 @@ git push origin "$version"
 REV=$(git rev-parse HEAD)
 make youtube-dl youtube-dl.tar.gz
 read -p "VM running? (y/n) " -n 1
-wget "http://localhost:8142/build/rg3/youtube-dl/youtube-dl.exe?rev=$REV" -O youtube-dl.exe
+wget "http://$buildserver/build/rg3/youtube-dl/youtube-dl.exe?rev=$REV" -O youtube-dl.exe
 mkdir -p "build/$version"
 mv youtube-dl youtube-dl.exe "build/$version"
 mv youtube-dl.tar.gz "build/$version/youtube-dl-$version.tar.gz"

From 56bd028a0f4b3809403d887012bb93bbc06296a5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 30 May 2016 00:21:18 +0700
Subject: [PATCH 5063/6219] [devscripts/buildserver] Listen on all interfaces

---
 devscripts/buildserver.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/devscripts/buildserver.py b/devscripts/buildserver.py
index 2bd12da50..f7979c43e 100644
--- a/devscripts/buildserver.py
+++ b/devscripts/buildserver.py
@@ -216,7 +216,7 @@ def main(args=None):
                         action='store_const', dest='action', const='service',
                         help='Run as a Windows service')
     parser.add_argument('-b', '--bind', metavar='<host:port>',
-                        action='store', default='localhost:8142',
+                        action='store', default='0.0.0.0:8142',
                         help='Bind to host:port (default %default)')
     options = parser.parse_args(args=args)
 

From 917a3196f862ff785d15e595e03363c94e9d2e5b Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Mon, 30 May 2016 01:03:40 +0700
Subject: [PATCH 5064/6219] [README.md] Update c runtime dependency FAQ entry

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index ef0e265c8..2b8a0fb85 100644
--- a/README.md
+++ b/README.md
@@ -784,9 +784,9 @@ means you're using an outdated version of Python. Please update to Python 2.6 or
 
 Since June 2012 ([#342](https://github.com/rg3/youtube-dl/issues/342)) youtube-dl is packed as an executable zipfile, simply unzip it (might need renaming to `youtube-dl.zip` first on some systems) or clone the git repository, as laid out above. If you modify the code, you can run it by executing the `__main__.py` file. To recompile the executable, run `make youtube-dl`.
 
-### The exe throws a *Runtime error from Visual C++*
+### The exe throws an error due to missing `MSVCR100.dll`
 
-To run the exe you need to install first the [Microsoft Visual C++ 2008 Redistributable Package](http://www.microsoft.com/en-us/download/details.aspx?id=29).
+To run the exe you need to install first the [Microsoft Visual C++ 2010 Redistributable Package (x86)](https://www.microsoft.com/en-US/download/details.aspx?id=5555).
 
 ### On Windows, how should I set up ffmpeg and youtube-dl? Where should I put the exe files?
 

From 82674236520b55893f6767362a32f74a831362ca Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 30 May 2016 01:18:23 +0700
Subject: [PATCH 5065/6219] release 2016.05.30

---
 .github/ISSUE_TEMPLATE.md |  6 +++---
 README.md                 | 15 ++++++++-------
 docs/supportedsites.md    | 16 +++++++++++-----
 youtube_dl/version.py     |  2 +-
 4 files changed, 23 insertions(+), 16 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 2d80d45b6..03dac8244 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.05.21.2*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.05.21.2**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.05.30*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.05.30**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.05.21.2
+[debug] youtube-dl version 2016.05.30
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/README.md b/README.md
index 2b8a0fb85..7e18112de 100644
--- a/README.md
+++ b/README.md
@@ -73,8 +73,8 @@ which means you can modify it, redistribute it or use it however you like.
                                      repairs broken URLs, but emits an error if
                                      this is not possible instead of searching.
     --ignore-config                  Do not read configuration files. When given
-                                     in the global configuration file /etc
-                                     /youtube-dl.conf: Do not read the user
+                                     in the global configuration file
+                                     /etc/youtube-dl.conf: Do not read the user
                                      configuration in ~/.config/youtube-
                                      dl/config (%APPDATA%/youtube-dl/config.txt
                                      on Windows)
@@ -256,11 +256,12 @@ which means you can modify it, redistribute it or use it however you like.
                                      jar in
     --cache-dir DIR                  Location in the filesystem where youtube-dl
                                      can store some downloaded information
-                                     permanently. By default $XDG_CACHE_HOME
-                                     /youtube-dl or ~/.cache/youtube-dl . At the
-                                     moment, only YouTube player files (for
-                                     videos with obfuscated signatures) are
-                                     cached, but that may change.
+                                     permanently. By default
+                                     $XDG_CACHE_HOME/youtube-dl or
+                                     ~/.cache/youtube-dl . At the moment, only
+                                     YouTube player files (for videos with
+                                     obfuscated signatures) are cached, but that
+                                     may change.
     --no-cache-dir                   Disable filesystem caching
     --rm-cache-dir                   Delete all filesystem cache files
 
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index cd6bfa51c..bbc647030 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -43,8 +43,8 @@
  - **appletrailers:section**
  - **archive.org**: archive.org videos
  - **ARD**
- - **ARD:mediathek**: Saarländischer Rundfunk
  - **ARD:mediathek**
+ - **ARD:mediathek**: Saarländischer Rundfunk
  - **arte.tv**
  - **arte.tv:+7**
  - **arte.tv:cinema**
@@ -136,6 +136,7 @@
  - **ComedyCentral**
  - **ComedyCentralShows**: The Daily Show / The Colbert Report
  - **CondeNast**: Condé Nast media group: Allure, Architectural Digest, Ars Technica, Bon Appétit, Brides, Condé Nast, Condé Nast Traveler, Details, Epicurious, GQ, Glamour, Golf Digest, SELF, Teen Vogue, The New Yorker, Vanity Fair, Vogue, W Magazine, WIRED
+ - **Coub**
  - **Cracked**
  - **Crackle**
  - **Criterion**
@@ -205,6 +206,7 @@
  - **exfm**: ex.fm
  - **ExpoTV**
  - **ExtremeTube**
+ - **EyedoTV**
  - **facebook**
  - **faz.net**
  - **fc2**
@@ -326,8 +328,8 @@
  - **LePlaylist**
  - **LetvCloud**: 乐视云
  - **Libsyn**
+ - **life**: Life.ru
  - **life:embed**
- - **lifenews**: LIFE | NEWS
  - **limelight**
  - **limelight:channel**
  - **limelight:channel_list**
@@ -512,6 +514,8 @@
  - **R7**
  - **radio.de**
  - **radiobremen**
+ - **radiocanada**
+ - **RadioCanadaAudioVideo**
  - **radiofrance**
  - **RadioJavan**
  - **Rai**
@@ -521,6 +525,7 @@
  - **RedTube**
  - **RegioTV**
  - **Restudy**
+ - **Reuters**
  - **ReverbNation**
  - **Revision3**
  - **RICE**
@@ -682,8 +687,8 @@
  - **TVCArticle**
  - **tvigle**: Интернет-телевидение Tvigle.ru
  - **tvland.com**
- - **tvp.pl**
- - **tvp.pl:Series**
+ - **tvp**: Telewizja Polska
+ - **tvp:series**
  - **TVPlay**: TV3Play and related services
  - **Tweakers**
  - **twitch:chapter**
@@ -766,7 +771,8 @@
  - **VuClip**
  - **vulture.com**
  - **Walla**
- - **WashingtonPost**
+ - **washingtonpost**
+ - **washingtonpost:article**
  - **wat.tv**
  - **WatchIndianPorn**: Watch Indian Porn
  - **WDR**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 522a56669..0c38ec4cf 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.05.21.2'
+__version__ = '2016.05.30'

From 86a52881c6211dd58503480dc62b4f2404cc0f6b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 29 May 2016 21:29:38 +0200
Subject: [PATCH 5066/6219] [travis] unsubscribe @phihag

---
 .travis.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 998995845..136c339f0 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -14,7 +14,6 @@ script: nosetests test --verbose
 notifications:
   email:
     - filippo.valsorda@gmail.com
-    - phihag@phihag.de
     - yasoob.khld@gmail.com
 #  irc:
 #    channels:

From f657b1a5f29e9f5eac7ca41b6e98c38cb3128183 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 30 May 2016 03:03:06 +0700
Subject: [PATCH 5067/6219] release 2016.05.30.1

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 youtube_dl/version.py     | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 03dac8244..1099b0c92 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.05.30*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.05.30**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.05.30.1*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.05.30.1**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.05.30
+[debug] youtube-dl version 2016.05.30.1
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 0c38ec4cf..5f8542f8e 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.05.30'
+__version__ = '2016.05.30.1'

From abbb2938fa08733e3a08f6d1917aa7687633b971 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 30 May 2016 03:12:12 +0700
Subject: [PATCH 5068/6219] release 2016.05.30.2

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 youtube_dl/version.py     | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 1099b0c92..e3de48eb5 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.05.30.1*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.05.30.1**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.05.30.2*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.05.30.2**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.05.30.1
+[debug] youtube-dl version 2016.05.30.2
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 5f8542f8e..ad6fb26c6 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.05.30.1'
+__version__ = '2016.05.30.2'

From 197a5da1d01179f6a2d60e3c2017b0070e5abc8b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 30 May 2016 03:26:26 +0700
Subject: [PATCH 5069/6219] [yandexmusic] Improve captcha detection

---
 youtube_dl/extractor/yandexmusic.py | 22 ++++++++++++++--------
 1 file changed, 14 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/yandexmusic.py b/youtube_dl/extractor/yandexmusic.py
index 0f78466e6..b37d0eab6 100644
--- a/youtube_dl/extractor/yandexmusic.py
+++ b/youtube_dl/extractor/yandexmusic.py
@@ -20,18 +20,24 @@ class YandexMusicBaseIE(InfoExtractor):
             error = response.get('error')
             if error:
                 raise ExtractorError(error, expected=True)
+            if response.get('type') == 'captcha' or 'captcha' in response:
+                YandexMusicBaseIE._raise_captcha()
+
+    @staticmethod
+    def _raise_captcha():
+        raise ExtractorError(
+            'YandexMusic has considered youtube-dl requests automated and '
+            'asks you to solve a CAPTCHA. You can either wait for some '
+            'time until unblocked and optionally use --sleep-interval '
+            'in future or alternatively you can go to https://music.yandex.ru/ '
+            'solve CAPTCHA, then export cookies and pass cookie file to '
+            'youtube-dl with --cookies',
+            expected=True)
 
     def _download_webpage(self, *args, **kwargs):
         webpage = super(YandexMusicBaseIE, self)._download_webpage(*args, **kwargs)
         if 'Нам очень жаль, но&nbsp;запросы, поступившие с&nbsp;вашего IP-адреса, похожи на&nbsp;автоматические.' in webpage:
-            raise ExtractorError(
-                'YandexMusic has considered youtube-dl requests automated and '
-                'asks you to solve a CAPTCHA. You can either wait for some '
-                'time until unblocked and optionally use --sleep-interval '
-                'in future or alternatively you can go to https://music.yandex.ru/ '
-                'solve CAPTCHA, then export cookies and pass cookie file to '
-                'youtube-dl with --cookies',
-                expected=True)
+            self._raise_captcha()
         return webpage
 
     def _download_json(self, *args, **kwargs):

From 8ec2b2c41c7f3952ad9097085993d1f24f6b6776 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 30 May 2016 21:48:35 +0700
Subject: [PATCH 5070/6219] [options] Add --limit-rate alias for rate limiting
 option

Closes #9644
In order to follow regular --verb-noun pattern and better conformity with wget and curl
---
 youtube_dl/options.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 38efd292d..14051b714 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -395,8 +395,8 @@ def parseOpts(overrideArguments=None):
 
     downloader = optparse.OptionGroup(parser, 'Download Options')
     downloader.add_option(
-        '-r', '--rate-limit',
-        dest='ratelimit', metavar='LIMIT',
+        '-r', '--limit-rate', '--rate-limit',
+        dest='ratelimit', metavar='RATE',
         help='Maximum download rate in bytes per second (e.g. 50K or 4.2M)')
     downloader.add_option(
         '-R', '--retries',

From e7d85c4ef7d2c74058d41ded1e2a6d6aa527dc9a Mon Sep 17 00:00:00 2001
From: Peter Rowlands <peter@pmrowla.com>
Date: Tue, 31 May 2016 17:28:49 +0900
Subject: [PATCH 5071/6219] use /track/video/file to determine if video exists

---
 youtube_dl/extractor/afreecatv.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/afreecatv.py b/youtube_dl/extractor/afreecatv.py
index 0fcbea0d1..518c61f67 100644
--- a/youtube_dl/extractor/afreecatv.py
+++ b/youtube_dl/extractor/afreecatv.py
@@ -11,6 +11,7 @@ from ..compat import (
 from ..utils import (
     ExtractorError,
     int_or_none,
+    xpath_element,
     xpath_text,
 )
 
@@ -84,9 +85,10 @@ class AfreecaTVIE(InfoExtractor):
             path='/api/video/get_video_info.php'))
         video_xml = self._download_xml(info_url, video_id)
 
-        if xpath_text(video_xml, './track/flag', default='FAIL') != 'SUCCEED':
+        if xpath_element(video_xml, './track/video/file') is None:
             raise ExtractorError('Specified AfreecaTV video does not exist',
                                  expected=True)
+
         title = xpath_text(video_xml, './track/title', 'title')
         uploader = xpath_text(video_xml, './track/nickname', 'uploader')
         uploader_id = xpath_text(video_xml, './track/bj_id', 'uploader id')

From 877032314fdf2d9b391325f96e3bc53a60ea067c Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 1 Jun 2016 18:37:34 +0800
Subject: [PATCH 5072/6219] [generic] Improve Kaltura detection

Closes #4004
---
 youtube_dl/extractor/generic.py | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 303e112d2..5cb188b20 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -918,6 +918,19 @@ class GenericIE(InfoExtractor):
                 'uploader_id': 'echojecka',
             },
         },
+        # Kaltura embed with single quotes
+        {
+            'url': 'http://fod.infobase.com/p_ViewPlaylist.aspx?AssignmentID=NUN8ZY',
+            'info_dict': {
+                'id': '0_izeg5utt',
+                'ext': 'mp4',
+                'title': '35871',
+                'timestamp': 1355743100,
+                'upload_date': '20121217',
+                'uploader_id': 'batchUser',
+            },
+            'add_ie': ['Kaltura'],
+        },
         # Eagle.Platform embed (generic URL)
         {
             'url': 'http://lenta.ru/news/2015/03/06/navalny/',
@@ -1903,7 +1916,7 @@ class GenericIE(InfoExtractor):
             return self.url_result(mobj.group('url'), 'Zapiks')
 
         # Look for Kaltura embeds
-        mobj = (re.search(r"(?s)kWidget\.(?:thumb)?[Ee]mbed\(\{.*?'wid'\s*:\s*'_?(?P<partner_id>[^']+)',.*?'entry_?[Ii]d'\s*:\s*'(?P<id>[^']+)',", webpage) or
+        mobj = (re.search(r"(?s)kWidget\.(?:thumb)?[Ee]mbed\(\{.*?(?P<q1>['\"])wid(?P=q1)\s*:\s*(?P<q2>['\"])_?(?P<partner_id>[^'\"]+)(?P=q2),.*?(?P<q3>['\"])entry_?[Ii]d(?P=q3)\s*:\s*(?P<q4>['\"])(?P<id>[^'\"]+)(?P=q4),", webpage) or
                 re.search(r'(?s)(?P<q1>["\'])(?:https?:)?//cdnapi(?:sec)?\.kaltura\.com/.*?(?:p|partner_id)/(?P<partner_id>\d+).*?(?P=q1).*?entry_?[Ii]d\s*:\s*(?P<q2>["\'])(?P<id>.+?)(?P=q2)', webpage))
         if mobj is not None:
             return self.url_result(smuggle_url(

From 28bab13348f84ac75e4d1362ce5828429bb7993f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 1 Jun 2016 19:18:01 +0800
Subject: [PATCH 5073/6219] [generic,viewlift] Move a test case to the
 specialized extractor

---
 youtube_dl/extractor/generic.py  | 12 ------------
 youtube_dl/extractor/viewlift.py |  4 ++++
 2 files changed, 4 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 5cb188b20..e478f86a8 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -881,18 +881,6 @@ class GenericIE(InfoExtractor):
                 'title': 'EP3S5 - Bon Appétit - Baqueira Mi Corazon !',
             }
         },
-        # Kaltura embed
-        {
-            'url': 'http://www.monumentalnetwork.com/videos/john-carlson-postgame-2-25-15',
-            'info_dict': {
-                'id': '1_eergr3h1',
-                'ext': 'mp4',
-                'upload_date': '20150226',
-                'uploader_id': 'MonumentalSports-Kaltura@perfectsensedigital.com',
-                'timestamp': int,
-                'title': 'John Carlson Postgame 2/25/15',
-            },
-        },
         # Kaltura embed (different embed code)
         {
             'url': 'http://www.premierchristianradio.com/Shows/Saturday/Unbelievable/Conference-Videos/Os-Guinness-Is-It-Fools-Talk-Unbelievable-Conference-2014',
diff --git a/youtube_dl/extractor/viewlift.py b/youtube_dl/extractor/viewlift.py
index dd4a13a4a..19500eba8 100644
--- a/youtube_dl/extractor/viewlift.py
+++ b/youtube_dl/extractor/viewlift.py
@@ -141,6 +141,10 @@ class ViewLiftIE(ViewLiftBaseIE):
     }, {
         'url': 'http://www.kesari.tv/news/video/1461919076414',
         'only_matching': True,
+    }, {
+        # Was once Kaltura embed
+        'url': 'https://www.monumentalsportsnetwork.com/videos/john-carlson-postgame-2-25-15',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 0ff3749bfe6d149dd7250ea8df83387d3af40e0f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 1 Jun 2016 19:23:09 +0800
Subject: [PATCH 5074/6219] [udn] Fix m3u8 and f4m extraction as well as
 improve

---
 youtube_dl/extractor/udn.py | 62 ++++++++++++++++++++++++-------------
 1 file changed, 41 insertions(+), 21 deletions(-)

diff --git a/youtube_dl/extractor/udn.py b/youtube_dl/extractor/udn.py
index ee35b7227..57dd73aef 100644
--- a/youtube_dl/extractor/udn.py
+++ b/youtube_dl/extractor/udn.py
@@ -2,10 +2,13 @@
 from __future__ import unicode_literals
 
 import json
+import re
+
 from .common import InfoExtractor
 from ..utils import (
+    determine_ext,
+    int_or_none,
     js_to_json,
-    ExtractorError,
 )
 from ..compat import compat_urlparse
 
@@ -16,13 +19,16 @@ class UDNEmbedIE(InfoExtractor):
     _VALID_URL = r'https?:' + _PROTOCOL_RELATIVE_VALID_URL
     _TESTS = [{
         'url': 'http://video.udn.com/embed/news/300040',
-        'md5': 'de06b4c90b042c128395a88f0384817e',
         'info_dict': {
             'id': '300040',
             'ext': 'mp4',
             'title': '生物老師男變女 全校挺"做自己"',
             'thumbnail': 're:^https?://.*\.jpg$',
-        }
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
     }, {
         'url': 'https://video.udn.com/embed/news/300040',
         'only_matching': True,
@@ -38,39 +44,53 @@ class UDNEmbedIE(InfoExtractor):
         page = self._download_webpage(url, video_id)
 
         options = json.loads(js_to_json(self._html_search_regex(
-            r'var options\s*=\s*([^;]+);', page, 'video urls dictionary')))
+            r'var\s+options\s*=\s*([^;]+);', page, 'video urls dictionary')))
 
         video_urls = options['video']
 
         if video_urls.get('youtube'):
             return self.url_result(video_urls.get('youtube'), 'Youtube')
 
-        try:
-            del video_urls['youtube']
-        except KeyError:
-            pass
+        formats = []
+        for video_type, api_url in video_urls.items():
+            if not api_url:
+                continue
 
-        formats = [{
-            'url': self._download_webpage(
+            video_url = self._download_webpage(
                 compat_urlparse.urljoin(url, api_url), video_id,
-                'retrieve url for %s video' % video_type),
-            'format_id': video_type,
-            'preference': 0 if video_type == 'mp4' else -1,
-        } for video_type, api_url in video_urls.items() if api_url]
+                note='retrieve url for %s video' % video_type)
 
-        if not formats:
-            raise ExtractorError('No videos found', expected=True)
+            ext = determine_ext(video_url)
+            if ext == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    video_url, video_id, ext='mp4', m3u8_id='hls'))
+            elif ext == 'f4m':
+                formats.extend(self._extract_f4m_formats(
+                    video_url, video_id, f4m_id='hds'))
+            else:
+                mobj = re.search(r'_(?P<height>\d+)p_(?P<tbr>\d+).mp4', video_url)
+                a_format = {
+                    'url': video_url,
+                    # video_type may be 'mp4', which confuses YoutubeDL
+                    'format_id': 'http-' + video_type,
+                }
+                if mobj:
+                    a_format.update({
+                        'height': int_or_none(mobj.group('height')),
+                        'tbr': int_or_none(mobj.group('tbr')),
+                    })
+                formats.append(a_format)
 
         self._sort_formats(formats)
 
-        thumbnail = None
-
-        if options.get('gallery') and len(options['gallery']):
-            thumbnail = options['gallery'][0].get('original')
+        thumbnails = [{
+            'url': img_url,
+            'id': img_type,
+        } for img_type, img_url in options.get('gallery', [{}])[0].items() if img_url]
 
         return {
             'id': video_id,
             'formats': formats,
             'title': options['title'],
-            'thumbnail': thumbnail
+            'thumbnails': thumbnails,
         }

From 811586ebcfb04878ad3347706bfee020d0e3652b Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 1 Jun 2016 19:23:44 +0800
Subject: [PATCH 5075/6219] [generic] Update the UDNEmbed test case

---
 youtube_dl/extractor/generic.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index e478f86a8..b4138381d 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1033,14 +1033,18 @@ class GenericIE(InfoExtractor):
         },
         # UDN embed
         {
-            'url': 'http://www.udn.com/news/story/7314/822787',
+            'url': 'https://video.udn.com/news/300346',
             'md5': 'fd2060e988c326991037b9aff9df21a6',
             'info_dict': {
                 'id': '300346',
                 'ext': 'mp4',
                 'title': '中一中男師變性 全校師生力挺',
                 'thumbnail': 're:^https?://.*\.jpg$',
-            }
+            },
+            'params': {
+                # m3u8 download
+                'skip_download': True,
+            },
         },
         # Ooyala embed
         {

From dde1ce7c061cae123264eb555f1da98956923301 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 1 Jun 2016 20:04:43 +0800
Subject: [PATCH 5076/6219] [tf1] Fix a regular expression (closes #9656)

This is a Python bug fixed in 2.7.6 [1]

[1] https://github.com/rg3/youtube-dl/issues/9656#issuecomment-222968594
---
 youtube_dl/extractor/tf1.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tf1.py b/youtube_dl/extractor/tf1.py
index aff5121b9..6c848dc6f 100644
--- a/youtube_dl/extractor/tf1.py
+++ b/youtube_dl/extractor/tf1.py
@@ -48,6 +48,6 @@ class TF1IE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
         wat_id = self._html_search_regex(
-            r'(["\'])(?:https?:)?//www\.wat\.tv/embedframe/.*?(?P<id>\d{8})(?:.*?)?\1',
+            r'(["\'])(?:https?:)?//www\.wat\.tv/embedframe/.*?(?P<id>\d{8}).*?\1',
             webpage, 'wat id', group='id')
         return self.url_result('wat:%s' % wat_id, 'Wat')

From 6a1df4fb5fb76710457b59195e8b530ba269f09f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 1 Jun 2016 21:23:58 +0700
Subject: [PATCH 5077/6219] [spankwire] Add support for new URL format (Closes
 #9657)

---
 youtube_dl/extractor/spankwire.py | 24 +++++++++++-------------
 1 file changed, 11 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/spankwire.py b/youtube_dl/extractor/spankwire.py
index 692fd78e8..92a7120a3 100644
--- a/youtube_dl/extractor/spankwire.py
+++ b/youtube_dl/extractor/spankwire.py
@@ -96,20 +96,18 @@ class SpankwireIE(InfoExtractor):
         formats = []
         for height, video_url in zip(heights, video_urls):
             path = compat_urllib_parse_urlparse(video_url).path
-            _, quality = path.split('/')[4].split('_')[:2]
-            f = {
-                'url': video_url,
-                'height': height,
-            }
-            tbr = self._search_regex(r'^(\d+)[Kk]$', quality, 'tbr', default=None)
-            if tbr:
-                f.update({
-                    'tbr': int(tbr),
-                    'format_id': '%dp' % height,
-                })
+            m = re.search(r'/(?P<height>\d+)[pP]_(?P<tbr>\d+)[kK]', path)
+            if m:
+                tbr = int(m.group('tbr'))
+                height = int(m.group('height'))
             else:
-                f['format_id'] = quality
-            formats.append(f)
+                tbr = None
+            formats.append({
+                'url': video_url,
+                'format_id': '%dp' % height,
+                'height': height,
+                'tbr': tbr,
+            })
         self._sort_formats(formats)
 
         age_limit = self._rta_search(webpage)

From 6e6b9f600f2f447604f6108fb6486b73cc25def1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 2 Jun 2016 01:10:23 +0700
Subject: [PATCH 5078/6219] [arte] Add support for playlists and rework tests
 (Closes #9632)

---
 youtube_dl/extractor/arte.py       | 173 ++++++++++++++++++-----------
 youtube_dl/extractor/extractors.py |   1 +
 2 files changed, 110 insertions(+), 64 deletions(-)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index e37fdae13..f40532929 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -61,10 +61,7 @@ class ArteTvIE(InfoExtractor):
         }
 
 
-class ArteTVPlus7IE(InfoExtractor):
-    IE_NAME = 'arte.tv:+7'
-    _VALID_URL = r'https?://(?:www\.)?arte\.tv/guide/(?P<lang>fr|de|en|es)/(?:(?:sendungen|emissions|embed)/)?(?P<id>[^/]+)/(?P<name>[^/?#&]+)'
-
+class ArteTVBaseIE(InfoExtractor):
     @classmethod
     def _extract_url_info(cls, url):
         mobj = re.match(cls._VALID_URL, url)
@@ -78,60 +75,6 @@ class ArteTVPlus7IE(InfoExtractor):
             video_id = mobj.group('id')
         return video_id, lang
 
-    def _real_extract(self, url):
-        video_id, lang = self._extract_url_info(url)
-        webpage = self._download_webpage(url, video_id)
-        return self._extract_from_webpage(webpage, video_id, lang)
-
-    def _extract_from_webpage(self, webpage, video_id, lang):
-        patterns_templates = (r'arte_vp_url=["\'](.*?%s.*?)["\']', r'data-url=["\']([^"]+%s[^"]+)["\']')
-        ids = (video_id, '')
-        # some pages contain multiple videos (like
-        # http://www.arte.tv/guide/de/sendungen/XEN/xenius/?vid=055918-015_PLUS7-D),
-        # so we first try to look for json URLs that contain the video id from
-        # the 'vid' parameter.
-        patterns = [t % re.escape(_id) for _id in ids for t in patterns_templates]
-        json_url = self._html_search_regex(
-            patterns, webpage, 'json vp url', default=None)
-        if not json_url:
-            def find_iframe_url(webpage, default=NO_DEFAULT):
-                return self._html_search_regex(
-                    r'<iframe[^>]+src=(["\'])(?P<url>.+\bjson_url=.+?)\1',
-                    webpage, 'iframe url', group='url', default=default)
-
-            iframe_url = find_iframe_url(webpage, None)
-            if not iframe_url:
-                embed_url = self._html_search_regex(
-                    r'arte_vp_url_oembed=\'([^\']+?)\'', webpage, 'embed url', default=None)
-                if embed_url:
-                    player = self._download_json(
-                        embed_url, video_id, 'Downloading player page')
-                    iframe_url = find_iframe_url(player['html'])
-            # en and es URLs produce react-based pages with different layout (e.g.
-            # http://www.arte.tv/guide/en/053330-002-A/carnival-italy?zone=world)
-            if not iframe_url:
-                program = self._search_regex(
-                    r'program\s*:\s*({.+?["\']embed_html["\'].+?}),?\s*\n',
-                    webpage, 'program', default=None)
-                if program:
-                    embed_html = self._parse_json(program, video_id)
-                    if embed_html:
-                        iframe_url = find_iframe_url(embed_html['embed_html'])
-            if iframe_url:
-                json_url = compat_parse_qs(
-                    compat_urllib_parse_urlparse(iframe_url).query)['json_url'][0]
-        if json_url:
-            title = self._search_regex(
-                r'<h3[^>]+title=(["\'])(?P<title>.+?)\1',
-                webpage, 'title', default=None, group='title')
-            return self._extract_from_json_url(json_url, video_id, lang, title=title)
-        # Different kind of embed URL (e.g.
-        # http://www.arte.tv/magazine/trepalium/fr/episode-0406-replay-trepalium)
-        embed_url = self._search_regex(
-            r'<iframe[^>]+src=(["\'])(?P<url>.+?)\1',
-            webpage, 'embed url', group='url')
-        return self.url_result(embed_url)
-
     def _extract_from_json_url(self, json_url, video_id, lang, title=None):
         info = self._download_json(json_url, video_id)
         player_info = info['videoJsonPlayer']
@@ -235,6 +178,74 @@ class ArteTVPlus7IE(InfoExtractor):
         return info_dict
 
 
+class ArteTVPlus7IE(ArteTVBaseIE):
+    IE_NAME = 'arte.tv:+7'
+    _VALID_URL = r'https?://(?:www\.)?arte\.tv/guide/(?P<lang>fr|de|en|es)/(?:(?:sendungen|emissions|embed)/)?(?P<id>[^/]+)/(?P<name>[^/?#&]+)'
+
+    _TESTS = [{
+        'url': 'http://www.arte.tv/guide/de/sendungen/XEN/xenius/?vid=055918-015_PLUS7-D',
+        'only_matching': True,
+    }]
+
+    @classmethod
+    def suitable(cls, url):
+        return False if ArteTVPlaylistIE.suitable(url) else super(ArteTVPlus7IE, cls).suitable(url)
+
+    def _real_extract(self, url):
+        video_id, lang = self._extract_url_info(url)
+        webpage = self._download_webpage(url, video_id)
+        return self._extract_from_webpage(webpage, video_id, lang)
+
+    def _extract_from_webpage(self, webpage, video_id, lang):
+        patterns_templates = (r'arte_vp_url=["\'](.*?%s.*?)["\']', r'data-url=["\']([^"]+%s[^"]+)["\']')
+        ids = (video_id, '')
+        # some pages contain multiple videos (like
+        # http://www.arte.tv/guide/de/sendungen/XEN/xenius/?vid=055918-015_PLUS7-D),
+        # so we first try to look for json URLs that contain the video id from
+        # the 'vid' parameter.
+        patterns = [t % re.escape(_id) for _id in ids for t in patterns_templates]
+        json_url = self._html_search_regex(
+            patterns, webpage, 'json vp url', default=None)
+        if not json_url:
+            def find_iframe_url(webpage, default=NO_DEFAULT):
+                return self._html_search_regex(
+                    r'<iframe[^>]+src=(["\'])(?P<url>.+\bjson_url=.+?)\1',
+                    webpage, 'iframe url', group='url', default=default)
+
+            iframe_url = find_iframe_url(webpage, None)
+            if not iframe_url:
+                embed_url = self._html_search_regex(
+                    r'arte_vp_url_oembed=\'([^\']+?)\'', webpage, 'embed url', default=None)
+                if embed_url:
+                    player = self._download_json(
+                        embed_url, video_id, 'Downloading player page')
+                    iframe_url = find_iframe_url(player['html'])
+            # en and es URLs produce react-based pages with different layout (e.g.
+            # http://www.arte.tv/guide/en/053330-002-A/carnival-italy?zone=world)
+            if not iframe_url:
+                program = self._search_regex(
+                    r'program\s*:\s*({.+?["\']embed_html["\'].+?}),?\s*\n',
+                    webpage, 'program', default=None)
+                if program:
+                    embed_html = self._parse_json(program, video_id)
+                    if embed_html:
+                        iframe_url = find_iframe_url(embed_html['embed_html'])
+            if iframe_url:
+                json_url = compat_parse_qs(
+                    compat_urllib_parse_urlparse(iframe_url).query)['json_url'][0]
+        if json_url:
+            title = self._search_regex(
+                r'<h3[^>]+title=(["\'])(?P<title>.+?)\1',
+                webpage, 'title', default=None, group='title')
+            return self._extract_from_json_url(json_url, video_id, lang, title=title)
+        # Different kind of embed URL (e.g.
+        # http://www.arte.tv/magazine/trepalium/fr/episode-0406-replay-trepalium)
+        embed_url = self._search_regex(
+            r'<iframe[^>]+src=(["\'])(?P<url>.+?)\1',
+            webpage, 'embed url', group='url')
+        return self.url_result(embed_url)
+
+
 # It also uses the arte_vp_url url from the webpage to extract the information
 class ArteTVCreativeIE(ArteTVPlus7IE):
     IE_NAME = 'arte.tv:creative'
@@ -267,7 +278,7 @@ class ArteTVInfoIE(ArteTVPlus7IE):
     IE_NAME = 'arte.tv:info'
     _VALID_URL = r'https?://info\.arte\.tv/(?P<lang>fr|de|en|es)/(?:[^/]+/)*(?P<id>[^/?#&]+)'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://info.arte.tv/fr/service-civique-un-cache-misere',
         'info_dict': {
             'id': '067528-000-A',
@@ -275,7 +286,7 @@ class ArteTVInfoIE(ArteTVPlus7IE):
             'title': 'Service civique, un cache misère ?',
             'upload_date': '20160403',
         },
-    }
+    }]
 
 
 class ArteTVFutureIE(ArteTVPlus7IE):
@@ -300,6 +311,8 @@ class ArteTVDDCIE(ArteTVPlus7IE):
     IE_NAME = 'arte.tv:ddc'
     _VALID_URL = r'https?://ddc\.arte\.tv/(?P<lang>emission|folge)/(?P<id>[^/?#&]+)'
 
+    _TESTS = []
+
     def _real_extract(self, url):
         video_id, lang = self._extract_url_info(url)
         if lang == 'folge':
@@ -318,7 +331,7 @@ class ArteTVConcertIE(ArteTVPlus7IE):
     IE_NAME = 'arte.tv:concert'
     _VALID_URL = r'https?://concert\.arte\.tv/(?P<lang>fr|de|en|es)/(?P<id>[^/?#&]+)'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://concert.arte.tv/de/notwist-im-pariser-konzertclub-divan-du-monde',
         'md5': '9ea035b7bd69696b67aa2ccaaa218161',
         'info_dict': {
@@ -328,14 +341,14 @@ class ArteTVConcertIE(ArteTVPlus7IE):
             'upload_date': '20140128',
             'description': 'md5:486eb08f991552ade77439fe6d82c305',
         },
-    }
+    }]
 
 
 class ArteTVCinemaIE(ArteTVPlus7IE):
     IE_NAME = 'arte.tv:cinema'
     _VALID_URL = r'https?://cinema\.arte\.tv/(?P<lang>fr|de|en|es)/(?P<id>.+)'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://cinema.arte.tv/de/node/38291',
         'md5': '6b275511a5107c60bacbeeda368c3aa1',
         'info_dict': {
@@ -345,7 +358,7 @@ class ArteTVCinemaIE(ArteTVPlus7IE):
             'upload_date': '20160122',
             'description': 'md5:7f749bbb77d800ef2be11d54529b96bc',
         },
-    }
+    }]
 
 
 class ArteTVMagazineIE(ArteTVPlus7IE):
@@ -390,9 +403,41 @@ class ArteTVEmbedIE(ArteTVPlus7IE):
         )
     '''
 
+    _TESTS = []
+
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
         lang = mobj.group('lang')
         json_url = mobj.group('json_url')
         return self._extract_from_json_url(json_url, video_id, lang)
+
+
+class ArteTVPlaylistIE(ArteTVBaseIE):
+    IE_NAME = 'arte.tv:playlist'
+    _VALID_URL = r'https?://(?:www\.)?arte\.tv/guide/(?P<lang>fr|de|en|es)/[^#]*#collection/(?P<id>PL-\d+)'
+
+    _TESTS = [{
+        'url': 'http://www.arte.tv/guide/de/plus7/?country=DE#collection/PL-013263/ARTETV',
+        'info_dict': {
+            'id': 'PL-013263',
+            'title': 'Areva & Uramin',
+        },
+        'playlist_mincount': 6,
+    }, {
+        'url': 'http://www.arte.tv/guide/de/playlists?country=DE#collection/PL-013190/ARTETV',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        playlist_id, lang = self._extract_url_info(url)
+        collection = self._download_json(
+            'https://api.arte.tv/api/player/v1/collectionData/%s/%s?source=videos'
+            % (lang, playlist_id), playlist_id)
+        title = collection.get('title')
+        description = collection.get('shortDescription') or collection.get('teaserText')
+        entries = [
+            self._extract_from_json_url(
+                video['jsonUrl'], video.get('programId') or playlist_id, lang)
+            for video in collection['videos'] if video.get('jsonUrl')]
+        return self.playlist_result(entries, playlist_id, title, description)
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index dd4b2b838..dc21cfed9 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -56,6 +56,7 @@ from .arte import (
     ArteTVDDCIE,
     ArteTVMagazineIE,
     ArteTVEmbedIE,
+    ArteTVPlaylistIE,
 )
 from .atresplayer import AtresPlayerIE
 from .atttechchannel import ATTTechChannelIE

From 5e3856a2c5af0a622b74921c0d60acde53a664ff Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 2 Jun 2016 01:19:57 +0700
Subject: [PATCH 5079/6219] release 2016.06.02

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 README.md                 | 2 +-
 docs/supportedsites.md    | 1 +
 youtube_dl/version.py     | 2 +-
 4 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index e3de48eb5..ae98e0626 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.05.30.2*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.05.30.2**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.06.02*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.06.02**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.05.30.2
+[debug] youtube-dl version 2016.06.02
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/README.md b/README.md
index 7e18112de..253d51bc8 100644
--- a/README.md
+++ b/README.md
@@ -162,7 +162,7 @@ which means you can modify it, redistribute it or use it however you like.
                                      (experimental)
 
 ## Download Options:
-    -r, --rate-limit LIMIT           Maximum download rate in bytes per second
+    -r, --limit-rate RATE            Maximum download rate in bytes per second
                                      (e.g. 50K or 4.2M)
     -R, --retries RETRIES            Number of retries (default is 10), or
                                      "infinite".
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index bbc647030..dcbc632a1 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -55,6 +55,7 @@
  - **arte.tv:future**
  - **arte.tv:info**
  - **arte.tv:magazine**
+ - **arte.tv:playlist**
  - **AtresPlayer**
  - **ATTTechChannel**
  - **AudiMedia**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index ad6fb26c6..fba427dde 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.05.30.2'
+__version__ = '2016.06.02'

From f4e4aa9b6b7057af400ad404efcca51669012b73 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 1 Jun 2016 21:18:57 +0100
Subject: [PATCH 5080/6219] [revision3:embed] Add new extractor

---
 youtube_dl/extractor/revision3.py | 132 ++++++++++++++----------------
 1 file changed, 63 insertions(+), 69 deletions(-)

diff --git a/youtube_dl/extractor/revision3.py b/youtube_dl/extractor/revision3.py
index 99979ebe1..833d8a2f0 100644
--- a/youtube_dl/extractor/revision3.py
+++ b/youtube_dl/extractor/revision3.py
@@ -13,8 +13,64 @@ from ..utils import (
 )
 
 
+class Revision3EmbedIE(InfoExtractor):
+    IE_NAME = 'revision3:embed'
+    _VALID_URL = r'(?:revision3:(?:(?P<playlist_type>[^:]+):)?|https?://(?:(?:(?:www|embed)\.)?(?:revision3|animalist)|(?:(?:api|embed)\.)?seekernetwork)\.com/player/embed\?videoId=)(?P<playlist_id>\d+)'
+    _TEST = {
+        'url': 'http://api.seekernetwork.com/player/embed?videoId=67558',
+        'md5': '83bcd157cab89ad7318dd7b8c9cf1306',
+        'info_dict': {
+            'id': '67558',
+            'ext': 'mp4',
+            'title': 'The Pros & Cons Of Zoos',
+            'description': 'Zoos are often depicted as a terrible place for animals to live, but is there any truth to this?',
+            'uploader_id': 'dnews',
+            'uploader': 'DNews',
+        }
+    }
+    _API_KEY = 'ba9c741bce1b9d8e3defcc22193f3651b8867e62'
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        playlist_id = mobj.group('playlist_id')
+        playlist_type = mobj.group('playlist_type') or 'video_id'
+        video_data = self._download_json(
+            'http://revision3.com/api/getPlaylist.json', playlist_id, query={
+                'api_key': self._API_KEY,
+                'codecs': 'h264,vp8,theora',
+                playlist_type: playlist_id,
+            })['items'][0]
+
+        formats = []
+        for vcodec, media in video_data['media'].items():
+            for quality_id, quality in media.items():
+                if quality_id == 'hls':
+                    formats.extend(self._extract_m3u8_formats(
+                        quality['url'], playlist_id, 'mp4',
+                        'm3u8_native', m3u8_id='hls', fatal=False))
+                else:
+                    formats.append({
+                        'url': quality['url'],
+                        'format_id': '%s-%s' % (vcodec, quality_id),
+                        'tbr': int_or_none(quality.get('bitrate')),
+                        'vcodec': vcodec,
+                    })
+        self._sort_formats(formats)
+
+        return {
+            'id': playlist_id,
+            'title': unescapeHTML(video_data['title']),
+            'description': unescapeHTML(video_data.get('summary')),
+            'uploader': video_data.get('show', {}).get('name'),
+            'uploader_id': video_data.get('show', {}).get('slug'),
+            'duration': int_or_none(video_data.get('duration')),
+            'formats': formats,
+        }
+
+
 class Revision3IE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?(?P<domain>(?:revision3|testtube|animalist)\.com)/(?P<id>[^/]+(?:/[^/?#]+)?)'
+    IE_NAME = 'revision'
+    _VALID_URL = r'https?://(?:www\.)?(?P<domain>(?:revision3|animalist)\.com)/(?P<id>[^/]+(?:/[^/?#]+)?)'
     _TESTS = [{
         'url': 'http://www.revision3.com/technobuffalo/5-google-predictions-for-2016',
         'md5': 'd94a72d85d0a829766de4deb8daaf7df',
@@ -32,52 +88,14 @@ class Revision3IE(InfoExtractor):
         }
     }, {
         # Show
-        'url': 'http://testtube.com/brainstuff',
-        'info_dict': {
-            'id': '251',
-            'title': 'BrainStuff',
-            'description': 'Whether the topic is popcorn or particle physics, you can count on the HowStuffWorks team to explore-and explain-the everyday science in the world around us on BrainStuff.',
-        },
-        'playlist_mincount': 93,
-    }, {
-        'url': 'https://testtube.com/dnews/5-weird-ways-plants-can-eat-animals?utm_source=FB&utm_medium=DNews&utm_campaign=DNewsSocial',
-        'info_dict': {
-            'id': '58227',
-            'display_id': 'dnews/5-weird-ways-plants-can-eat-animals',
-            'duration': 275,
-            'ext': 'webm',
-            'title': '5 Weird Ways Plants Can Eat Animals',
-            'description': 'Why have some plants evolved to eat meat?',
-            'upload_date': '20150120',
-            'timestamp': 1421763300,
-            'uploader': 'DNews',
-            'uploader_id': 'dnews',
-        },
-    }, {
-        'url': 'http://testtube.com/tt-editors-picks/the-israel-palestine-conflict-explained-in-ten-min',
-        'info_dict': {
-            'id': '71618',
-            'ext': 'mp4',
-            'display_id': 'tt-editors-picks/the-israel-palestine-conflict-explained-in-ten-min',
-            'title': 'The Israel-Palestine Conflict Explained in Ten Minutes',
-            'description': 'If you\'d like to learn about the struggle between Israelis and Palestinians, this video is a great place to start',
-            'uploader': 'Editors\' Picks',
-            'uploader_id': 'tt-editors-picks',
-            'timestamp': 1453309200,
-            'upload_date': '20160120',
-        },
-        'add_ie': ['Youtube'],
+        'url': 'http://revision3.com/variant',
+        'only_matching': True,
     }, {
         # Tag
-        'url': 'http://testtube.com/tech-news',
-        'info_dict': {
-            'id': '21018',
-            'title': 'tech news',
-        },
-        'playlist_mincount': 9,
+        'url': 'http://revision3.com/vr',
+        'only_matching': True,
     }]
     _PAGE_DATA_TEMPLATE = 'http://www.%s/apiProxy/ddn/%s?domain=%s'
-    _API_KEY = 'ba9c741bce1b9d8e3defcc22193f3651b8867e62'
 
     def _real_extract(self, url):
         domain, display_id = re.match(self._VALID_URL, url).groups()
@@ -119,33 +137,9 @@ class Revision3IE(InfoExtractor):
                 })
                 return info
 
-            video_data = self._download_json(
-                'http://revision3.com/api/getPlaylist.json?api_key=%s&codecs=h264,vp8,theora&video_id=%s' % (self._API_KEY, video_id),
-                video_id)['items'][0]
-
-            formats = []
-            for vcodec, media in video_data['media'].items():
-                for quality_id, quality in media.items():
-                    if quality_id == 'hls':
-                        formats.extend(self._extract_m3u8_formats(
-                            quality['url'], video_id, 'mp4',
-                            'm3u8_native', m3u8_id='hls', fatal=False))
-                    else:
-                        formats.append({
-                            'url': quality['url'],
-                            'format_id': '%s-%s' % (vcodec, quality_id),
-                            'tbr': int_or_none(quality.get('bitrate')),
-                            'vcodec': vcodec,
-                        })
-            self._sort_formats(formats)
-
             info.update({
-                'title': unescapeHTML(video_data['title']),
-                'description': unescapeHTML(video_data.get('summary')),
-                'uploader': video_data.get('show', {}).get('name'),
-                'uploader_id': video_data.get('show', {}).get('slug'),
-                'duration': int_or_none(video_data.get('duration')),
-                'formats': formats,
+                '_type': 'url_transparent',
+                'url': 'revision3:%s' % video_id,
             })
             return info
         else:

From 4a684895c0227bf18896eae36e693d7046aacaf4 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 1 Jun 2016 21:20:02 +0100
Subject: [PATCH 5081/6219] [seeker] Add new extractor(closes #9619)

---
 youtube_dl/extractor/extractors.py |  6 +++-
 youtube_dl/extractor/seeker.py     | 57 ++++++++++++++++++++++++++++++
 2 files changed, 62 insertions(+), 1 deletion(-)
 create mode 100644 youtube_dl/extractor/seeker.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index dc21cfed9..9dd55bd70 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -639,7 +639,10 @@ from .regiotv import RegioTVIE
 from .restudy import RestudyIE
 from .reuters import ReutersIE
 from .reverbnation import ReverbNationIE
-from .revision3 import Revision3IE
+from .revision3 import (
+    Revision3EmbedIE,
+    Revision3IE,
+)
 from .rice import RICEIE
 from .ringtv import RingTVIE
 from .ro220 import Ro220IE
@@ -678,6 +681,7 @@ from .screencast import ScreencastIE
 from .screencastomatic import ScreencastOMaticIE
 from .screenjunkies import ScreenJunkiesIE
 from .screenwavemedia import ScreenwaveMediaIE, TeamFourIE
+from .seeker import SeekerIE
 from .senateisvp import SenateISVPIE
 from .sendtonews import SendtoNewsIE
 from .servingsys import ServingSysIE
diff --git a/youtube_dl/extractor/seeker.py b/youtube_dl/extractor/seeker.py
new file mode 100644
index 000000000..3b9c65e7e
--- /dev/null
+++ b/youtube_dl/extractor/seeker.py
@@ -0,0 +1,57 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+
+class SeekerIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?seeker\.com/(?P<display_id>.*)-(?P<article_id>\d+)\.html'
+    _TESTS = [{
+        # player.loadRevision3Item
+        'url': 'http://www.seeker.com/should-trump-be-required-to-release-his-tax-returns-1833805621.html',
+        'md5': '30c1dc4030cc715cf05b423d0947ac18',
+        'info_dict': {
+            'id': '76243',
+            'ext': 'webm',
+            'title': 'Should Trump Be Required To Release His Tax Returns?',
+            'description': 'Donald Trump has been secretive about his "big," "beautiful" tax returns. So what can we learn if he decides to release them?',
+            'uploader': 'Seeker Daily',
+            'uploader_id': 'seekerdaily',
+        }
+    }, {
+        'url': 'http://www.seeker.com/changes-expected-at-zoos-following-recent-gorilla-lion-shootings-1834116536.html',
+        'playlist': [
+            {
+                'md5': '83bcd157cab89ad7318dd7b8c9cf1306',
+                'info_dict': {
+                    'id': '67558',
+                    'ext': 'mp4',
+                    'title': 'The Pros & Cons Of Zoos',
+                    'description': 'Zoos are often depicted as a terrible place for animals to live, but is there any truth to this?',
+                    'uploader': 'DNews',
+                    'uploader_id': 'dnews',
+                },
+            }
+        ],
+        'info_dict': {
+            'id': '1834116536',
+            'title': 'After Gorilla Killing, Changes Ahead for Zoos',
+            'description': 'The largest association of zoos and others are hoping to learn from recent incidents that led to the shooting deaths of a gorilla and two lions.',
+        },
+    }]
+
+    def _real_extract(self, url):
+        display_id, article_id = re.match(self._VALID_URL, url).groups()
+        webpage = self._download_webpage(url, display_id)
+        mobj = re.search(r"player\.loadRevision3Item\('([^']+)'\s*,\s*(\d+)\);", webpage)
+        if mobj:
+            playlist_type, playlist_id = mobj.groups()
+            return self.url_result(
+                'revision3:%s:%s' % (playlist_type, playlist_id), 'Revision3Embed', playlist_id)
+        else:
+            entries = [self.url_result('revision3:video_id:%s' % video_id, 'Revision3Embed', video_id) for video_id in re.findall(
+                r'<iframe[^>]+src=[\'"](?:https?:)?//api\.seekernetwork\.com/player/embed\?videoId=(\d+)', webpage)]
+            return self.playlist_result(
+                entries, article_id, self._og_search_title(webpage), self._og_search_description(webpage))

From 0ea590076fd3b714b6a3345a28ac61d8be5e2afd Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 26 May 2016 17:22:40 +0800
Subject: [PATCH 5082/6219] [utils] Always decode Location header

escape_url is broken for bytes-like objects
---
 youtube_dl/utils.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index cfb2d1bf5..6ab1747b3 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -861,6 +861,8 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
                 # As of RFC 2616 default charset is iso-8859-1 that is respected by python 3
                 if sys.version_info >= (3, 0):
                     location = location.encode('iso-8859-1').decode('utf-8')
+                else:
+                    location = location.decode('utf-8')
                 location_escaped = escape_url(location)
                 if location != location_escaped:
                     del resp.headers['Location']

From 8c32e5dc32f401ae22fa7087f3f3b1c9d6563835 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 26 May 2016 17:24:40 +0800
Subject: [PATCH 5083/6219] [test/test_utils] Add test for #9588

---
 test/test_http.py | 55 ++++++++++++++++++++++++++++++++++++++++-------
 1 file changed, 47 insertions(+), 8 deletions(-)

diff --git a/test/test_http.py b/test/test_http.py
index 15e0ad369..6b8493e5e 100644
--- a/test/test_http.py
+++ b/test/test_http.py
@@ -16,6 +16,15 @@ import threading
 TEST_DIR = os.path.dirname(os.path.abspath(__file__))
 
 
+def http_server_port(httpd):
+    if os.name == 'java':
+        # In Jython SSLSocket is not a subclass of socket.socket
+        sock = httpd.socket.sock
+    else:
+        sock = httpd.socket
+    return sock.getsockname()[1]
+
+
 class HTTPTestRequestHandler(compat_http_server.BaseHTTPRequestHandler):
     def log_message(self, format, *args):
         pass
@@ -31,6 +40,22 @@ class HTTPTestRequestHandler(compat_http_server.BaseHTTPRequestHandler):
             self.send_header('Content-Type', 'video/mp4')
             self.end_headers()
             self.wfile.write(b'\x00\x00\x00\x00\x20\x66\x74[video]')
+        elif self.path == '/302':
+            if sys.version_info[0] == 3:
+                # XXX: Python 3 http server does not allow non-ASCII header values
+                self.send_response(404)
+                self.end_headers()
+                return
+
+            new_url = 'http://localhost:%d/中文.html' % http_server_port(self.server)
+            self.send_response(302)
+            self.send_header(b'Location', new_url.encode('utf-8'))
+            self.end_headers()
+        elif self.path == '/%E4%B8%AD%E6%96%87.html':
+            self.send_response(200)
+            self.send_header('Content-Type', 'text/html; charset=utf-8')
+            self.end_headers()
+            self.wfile.write(b'<html><video src="/vid.mp4" /></html>')
         else:
             assert False
 
@@ -47,18 +72,32 @@ class FakeLogger(object):
 
 
 class TestHTTP(unittest.TestCase):
+    def setUp(self):
+        self.httpd = compat_http_server.HTTPServer(
+            ('localhost', 0), HTTPTestRequestHandler)
+        self.port = http_server_port(self.httpd)
+        self.server_thread = threading.Thread(target=self.httpd.serve_forever)
+        self.server_thread.daemon = True
+        self.server_thread.start()
+
+    def test_unicode_path_redirection(self):
+        # XXX: Python 3 http server does not allow non-ASCII header values
+        if sys.version_info[0] == 3:
+            return
+
+        ydl = YoutubeDL({'logger': FakeLogger()})
+        r = ydl.extract_info('http://localhost:%d/302' % self.port)
+        self.assertEqual(r['url'], 'http://localhost:%d/vid.mp4' % self.port)
+
+
+class TestHTTPS(unittest.TestCase):
     def setUp(self):
         certfn = os.path.join(TEST_DIR, 'testcert.pem')
         self.httpd = compat_http_server.HTTPServer(
             ('localhost', 0), HTTPTestRequestHandler)
         self.httpd.socket = ssl.wrap_socket(
             self.httpd.socket, certfile=certfn, server_side=True)
-        if os.name == 'java':
-            # In Jython SSLSocket is not a subclass of socket.socket
-            sock = self.httpd.socket.sock
-        else:
-            sock = self.httpd.socket
-        self.port = sock.getsockname()[1]
+        self.port = http_server_port(self.httpd)
         self.server_thread = threading.Thread(target=self.httpd.serve_forever)
         self.server_thread.daemon = True
         self.server_thread.start()
@@ -94,14 +133,14 @@ class TestProxy(unittest.TestCase):
     def setUp(self):
         self.proxy = compat_http_server.HTTPServer(
             ('localhost', 0), _build_proxy_handler('normal'))
-        self.port = self.proxy.socket.getsockname()[1]
+        self.port = http_server_port(self.proxy)
         self.proxy_thread = threading.Thread(target=self.proxy.serve_forever)
         self.proxy_thread.daemon = True
         self.proxy_thread.start()
 
         self.cn_proxy = compat_http_server.HTTPServer(
             ('localhost', 0), _build_proxy_handler('cn'))
-        self.cn_port = self.cn_proxy.socket.getsockname()[1]
+        self.cn_port = http_server_port(self.cn_proxy)
         self.cn_proxy_thread = threading.Thread(target=self.cn_proxy.serve_forever)
         self.cn_proxy_thread.daemon = True
         self.cn_proxy_thread.start()

From 54fb1996812fa09f0f81ac28f42647e7706212b2 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 29 May 2016 19:24:28 +0800
Subject: [PATCH 5084/6219] [test/test_http] Fix getsockname() on Jython

---
 test/test_http.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/test_http.py b/test/test_http.py
index 6b8493e5e..5076ced51 100644
--- a/test/test_http.py
+++ b/test/test_http.py
@@ -17,7 +17,7 @@ TEST_DIR = os.path.dirname(os.path.abspath(__file__))
 
 
 def http_server_port(httpd):
-    if os.name == 'java':
+    if os.name == 'java' and isinstance(httpd.socket, ssl.SSLSocket):
         # In Jython SSLSocket is not a subclass of socket.socket
         sock = httpd.socket.sock
     else:

From 9a4aec8b7ea2c0863bc03ba8f3d3e69a61e77c80 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 29 May 2016 19:25:25 +0800
Subject: [PATCH 5085/6219] [utils] Use bytes-like objects as header values on
 Python 2

---
 youtube_dl/utils.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 6ab1747b3..26f21602c 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -866,6 +866,8 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
                 location_escaped = escape_url(location)
                 if location != location_escaped:
                     del resp.headers['Location']
+                    if sys.version_info < (3, 0):
+                        location_escaped = location_escaped.encode('utf-8')
                     resp.headers['Location'] = location_escaped
         return resp
 

From b96f007eeb432cdd118fb4cfa027dfa36b0ea0f2 Mon Sep 17 00:00:00 2001
From: bzc6p <bzc6p@users.noreply.github.com>
Date: Thu, 2 Jun 2016 11:39:32 +0200
Subject: [PATCH 5086/6219] =?UTF-8?q?Added=20sanitization=20support=20for?=
 =?UTF-8?q?=20Hungarian=20letters=20=C5=90=20and=20=C5=B0?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 test/test_utils.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index a697232a8..feef80465 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -157,8 +157,8 @@ class TestUtil(unittest.TestCase):
         self.assertTrue(sanitize_filename(':', restricted=True) != '')
 
         self.assertEqual(sanitize_filename(
-            'ÂÃÄÀÁÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖØŒÙÚÛÜÝÞßàáâãäåæçèéêëìíîïðñòóôõöøœùúûüýþÿ', restricted=True),
-            'AAAAAAAECEEEEIIIIDNOOOOOOOEUUUUYPssaaaaaaaeceeeeiiiionoooooooeuuuuypy')
+            'ÂÃÄÀÁÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖŐØŒÙÚÛÜŰÝÞßàáâãäåæçèéêëìíîïðñòóôõöőøœùúûüűýþÿ', restricted=True),
+            'AAAAAAAECEEEEIIIIDNOOOOOOOOEUUUUUYPssaaaaaaaeceeeeiiiionooooooooeuuuuuypy')
 
     def test_sanitize_ids(self):
         self.assertEqual(sanitize_filename('_n_cd26wFpw', is_id=True), '_n_cd26wFpw')

From c88270271e65be591f7e194b71728256644b8684 Mon Sep 17 00:00:00 2001
From: bzc6p <bzc6p@users.noreply.github.com>
Date: Thu, 2 Jun 2016 11:51:48 +0200
Subject: [PATCH 5087/6219] =?UTF-8?q?Added=20sanitization=20support=20for?=
 =?UTF-8?q?=20Hungarian=20letters=20=C5=90=20and=20=C5=B0?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 youtube_dl/utils.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 26f21602c..89234b39d 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -105,9 +105,9 @@ KNOWN_EXTENSIONS = (
     'f4f', 'f4m', 'm3u8', 'smil')
 
 # needed for sanitizing filenames in restricted mode
-ACCENT_CHARS = dict(zip('ÂÃÄÀÁÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖØŒÙÚÛÜÝÞßàáâãäåæçèéêëìíîïðñòóôõöøœùúûüýþÿ',
-                        itertools.chain('AAAAAA', ['AE'], 'CEEEEIIIIDNOOOOOO', ['OE'], 'UUUUYP', ['ss'],
-                                        'aaaaaa', ['ae'], 'ceeeeiiiionoooooo', ['oe'], 'uuuuypy')))
+ACCENT_CHARS = dict(zip('ÂÃÄÀÁÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖŐØŒÙÚÛÜŰÝÞßàáâãäåæçèéêëìíîïðñòóôõöőøœùúûüűýþÿ',
+                        itertools.chain('AAAAAA', ['AE'], 'CEEEEIIIIDNOOOOOOO', ['OE'], 'UUUUUYP', ['ss'],
+                                        'aaaaaa', ['ae'], 'ceeeeiiiionooooooo', ['oe'], 'uuuuuypy')))
 
 
 def preferredencoding():

From ad73083ff05eafa64e07500fd42306ac349bd76b Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 2 Jun 2016 19:27:57 +0800
Subject: [PATCH 5088/6219] [bilibili] Add _part%d suffixes back (closes #9660)

---
 youtube_dl/extractor/bilibili.py | 59 ++++++++++++++++++++++++++++++++
 1 file changed, 59 insertions(+)

diff --git a/youtube_dl/extractor/bilibili.py b/youtube_dl/extractor/bilibili.py
index 71a54b4f4..910e539e4 100644
--- a/youtube_dl/extractor/bilibili.py
+++ b/youtube_dl/extractor/bilibili.py
@@ -46,6 +46,62 @@ class BiliBiliIE(InfoExtractor):
             'description': '这是个神奇的故事~每个人不留弹幕不给走哦~切利哦！~',
         },
         'playlist_count': 9,
+    }, {
+        'url': 'http://www.bilibili.com/video/av4808130/',
+        'info_dict': {
+            'id': '4808130',
+            'title': '【长篇】哆啦A梦443【钉铛】',
+            'description': '(2016.05.27)来组合客人的脸吧&amp;amp;寻母六千里锭 抱歉，又轮到周日上班现在才到家 封面www.pixiv.net/member_illust.php?mode=medium&amp;amp;illust_id=56912929',
+        },
+        'playlist': [{
+            'md5': '55cdadedf3254caaa0d5d27cf20a8f9c',
+            'info_dict': {
+                'id': '4808130_part1',
+                'ext': 'flv',
+                'title': '【长篇】哆啦A梦443【钉铛】',
+                'description': '(2016.05.27)来组合客人的脸吧&amp;amp;寻母六千里锭 抱歉，又轮到周日上班现在才到家 封面www.pixiv.net/member_illust.php?mode=medium&amp;amp;illust_id=56912929',
+                'timestamp': 1464564180,
+                'upload_date': '20160529',
+                'uploader': '喜欢拉面',
+                'uploader_id': '151066',
+            },
+        }, {
+            'md5': '926f9f67d0c482091872fbd8eca7ea3d',
+            'info_dict': {
+                'id': '4808130_part2',
+                'ext': 'flv',
+                'title': '【长篇】哆啦A梦443【钉铛】',
+                'description': '(2016.05.27)来组合客人的脸吧&amp;amp;寻母六千里锭 抱歉，又轮到周日上班现在才到家 封面www.pixiv.net/member_illust.php?mode=medium&amp;amp;illust_id=56912929',
+                'timestamp': 1464564180,
+                'upload_date': '20160529',
+                'uploader': '喜欢拉面',
+                'uploader_id': '151066',
+            },
+        }, {
+            'md5': '4b7b225b968402d7c32348c646f1fd83',
+            'info_dict': {
+                'id': '4808130_part3',
+                'ext': 'flv',
+                'title': '【长篇】哆啦A梦443【钉铛】',
+                'description': '(2016.05.27)来组合客人的脸吧&amp;amp;寻母六千里锭 抱歉，又轮到周日上班现在才到家 封面www.pixiv.net/member_illust.php?mode=medium&amp;amp;illust_id=56912929',
+                'timestamp': 1464564180,
+                'upload_date': '20160529',
+                'uploader': '喜欢拉面',
+                'uploader_id': '151066',
+            },
+        }, {
+            'md5': '7b795e214166501e9141139eea236e91',
+            'info_dict': {
+                'id': '4808130_part4',
+                'ext': 'flv',
+                'title': '【长篇】哆啦A梦443【钉铛】',
+                'description': '(2016.05.27)来组合客人的脸吧&amp;amp;寻母六千里锭 抱歉，又轮到周日上班现在才到家 封面www.pixiv.net/member_illust.php?mode=medium&amp;amp;illust_id=56912929',
+                'timestamp': 1464564180,
+                'upload_date': '20160529',
+                'uploader': '喜欢拉面',
+                'uploader_id': '151066',
+            },
+        }],
     }]
 
     # BiliBili blocks keys from time to time. The current key is extracted from
@@ -144,6 +200,9 @@ class BiliBiliIE(InfoExtractor):
         if len(entries) == 1:
             return entries[0]
         else:
+            for idx, entry in enumerate(entries):
+                entry['id'] = '%s_part%d' % (video_id, (idx + 1))
+
             return {
                 '_type': 'multi_video',
                 'id': video_id,

From 9c3c447eb389726d98189d972a2d772ef729132d Mon Sep 17 00:00:00 2001
From: TRox1972 <archcr8@gmail.com>
Date: Tue, 17 May 2016 16:21:52 +0200
Subject: [PATCH 5089/6219] [loc] Add extractor (Closes #3188)

Added extractor of loc.gov, which closes #3188. I am not an experienced programmer, so I am sure I did a bunch of mistakes, but the extractor works (for me at least).

[LibraryOfCongress] don't use video_id for _search_regex()

[LibraryOfCongress] Improvements
---
 youtube_dl/extractor/extractors.py        |  1 +
 youtube_dl/extractor/libraryofcongress.py | 65 +++++++++++++++++++++++
 2 files changed, 66 insertions(+)
 create mode 100644 youtube_dl/extractor/libraryofcongress.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 9dd55bd70..3b5143ace 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -382,6 +382,7 @@ from .leeco import (
     LePlaylistIE,
     LetvCloudIE,
 )
+from .libraryofcongress import LibraryOfCongressIE
 from .libsyn import LibsynIE
 from .lifenews import (
     LifeNewsIE,
diff --git a/youtube_dl/extractor/libraryofcongress.py b/youtube_dl/extractor/libraryofcongress.py
new file mode 100644
index 000000000..0c34dbce3
--- /dev/null
+++ b/youtube_dl/extractor/libraryofcongress.py
@@ -0,0 +1,65 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+from ..utils import determine_ext
+
+
+class LibraryOfCongressIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?loc\.gov/item/(?P<id>[0-9]+)'
+    _TESTS = [{
+    'url': 'http://loc.gov/item/90716351/',
+        'info_dict': {
+            'id': '90716351',
+            'ext': 'mp4',
+            'title': 'Pa\'s trip to Mars /'
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        }
+    }, {
+        'url': 'https://www.loc.gov/item/97516576/',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        self.report_extraction(video_id)
+        json_id = self._search_regex('media-player-([0-9A-Z]{32})', webpage, 'json id')
+
+        data = self._parse_json(self._download_webpage(
+            'https://media.loc.gov/services/v1/media?id=%s' % json_id,
+            video_id), video_id)
+        data = data['mediaObject']
+
+        media_url = data['derivatives'][0]['derivativeUrl']
+        media_url = media_url.replace('rtmp', 'https')
+
+        is_video = data['mediaType'].lower() == 'v'
+        if not determine_ext(media_url) in ('mp4', 'mp3'):
+            media_url += '.mp4' if is_video else '.mp3'
+
+        if media_url.index('vod/mp4:') > -1:
+            media_url = media_url.replace('vod/mp4:', 'hls-vod/media/') + '.m3u8'
+        elif url.index('vod/mp3:') > -1:
+            media_url = media_url.replace('vod/mp3:', '')
+
+        formats = []
+        if determine_ext(media_url) == 'm3u8':
+            formats = self._extract_m3u8_formats(media_url, video_id, ext='mp4')
+        elif determine_ext(media_url) is 'mp3':
+            formats.append({
+                'url': media_url,
+                'ext': 'mp3',
+            })
+
+        return {
+            'id': video_id,
+            'thumbnail': self._og_search_thumbnail(webpage),
+            'title': self._og_search_title(webpage),
+            'formats': formats,
+        }

From 7f3c3dfa52769d1f44c1f1031449118c564a92bf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 3 Jun 2016 23:19:11 +0700
Subject: [PATCH 5090/6219] [loc] Improve (Closes #9521)

---
 youtube_dl/extractor/libraryofcongress.py | 87 ++++++++++++++---------
 1 file changed, 53 insertions(+), 34 deletions(-)

diff --git a/youtube_dl/extractor/libraryofcongress.py b/youtube_dl/extractor/libraryofcongress.py
index 0c34dbce3..d311f9946 100644
--- a/youtube_dl/extractor/libraryofcongress.py
+++ b/youtube_dl/extractor/libraryofcongress.py
@@ -3,63 +3,82 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 
-from ..utils import determine_ext
+from ..utils import (
+    determine_ext,
+    float_or_none,
+    int_or_none,
+)
 
 
 class LibraryOfCongressIE(InfoExtractor):
+    IE_NAME = 'loc'
+    IE_DESC = 'Library of Congress'
     _VALID_URL = r'https?://(?:www\.)?loc\.gov/item/(?P<id>[0-9]+)'
-    _TESTS = [{
-    'url': 'http://loc.gov/item/90716351/',
+    _TEST = {
+        'url': 'http://loc.gov/item/90716351/',
+        'md5': '353917ff7f0255aa6d4b80a034833de8',
         'info_dict': {
             'id': '90716351',
             'ext': 'mp4',
-            'title': 'Pa\'s trip to Mars /'
+            'title': "Pa's trip to Mars",
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 0,
+            'view_count': int,
         },
-        'params': {
-            # m3u8 download
-            'skip_download': True,
-        }
-    }, {
-        'url': 'https://www.loc.gov/item/97516576/',
-        'only_matching': True,
-    }]
+    }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        self.report_extraction(video_id)
-        json_id = self._search_regex('media-player-([0-9A-Z]{32})', webpage, 'json id')
+        media_id = self._search_regex(
+            (r'id=(["\'])media-player-(?P<id>.+?)\1',
+             r'<video[^>]+id=(["\'])uuid-(?P<id>.+?)\1',
+             r'<video[^>]+data-uuid=(["\'])(?P<id>.+?)\1'),
+            webpage, 'media id', group='id')
 
-        data = self._parse_json(self._download_webpage(
-            'https://media.loc.gov/services/v1/media?id=%s' % json_id,
-            video_id), video_id)
-        data = data['mediaObject']
+        data = self._parse_json(
+            self._download_webpage(
+                'https://media.loc.gov/services/v1/media?id=%s&context=json' % media_id,
+                video_id),
+            video_id)['mediaObject']
 
-        media_url = data['derivatives'][0]['derivativeUrl']
+        derivative = data['derivatives'][0]
+        media_url = derivative['derivativeUrl']
+
+        # Following algorithm was extracted from setAVSource js function
+        # found in webpage
         media_url = media_url.replace('rtmp', 'https')
 
-        is_video = data['mediaType'].lower() == 'v'
-        if not determine_ext(media_url) in ('mp4', 'mp3'):
+        is_video = data.get('mediaType', 'v').lower() == 'v'
+        ext = determine_ext(media_url)
+        if ext not in ('mp4', 'mp3'):
             media_url += '.mp4' if is_video else '.mp3'
 
-        if media_url.index('vod/mp4:') > -1:
-            media_url = media_url.replace('vod/mp4:', 'hls-vod/media/') + '.m3u8'
-        elif url.index('vod/mp3:') > -1:
-            media_url = media_url.replace('vod/mp3:', '')
+        if 'vod/mp4:' in media_url:
+            formats = [{
+                'url': media_url.replace('vod/mp4:', 'hls-vod/media/') + '.m3u8',
+                'format_id': 'hls',
+                'ext': 'mp4',
+                'protocol': 'm3u8_native',
+            }]
+        elif 'vod/mp3:' in media_url:
+            formats = [{
+                'url': media_url.replace('vod/mp3:', ''),
+                'vcodec': 'none',
+            }]
 
-        formats = []
-        if determine_ext(media_url) == 'm3u8':
-            formats = self._extract_m3u8_formats(media_url, video_id, ext='mp4')
-        elif determine_ext(media_url) is 'mp3':
-            formats.append({
-                'url': media_url,
-                'ext': 'mp3',
-            })
+        self._sort_formats(formats)
+
+        title = derivative.get('shortName') or data.get('shortName') or self._og_search_title(webpage)
+        duration = float_or_none(data.get('duration'))
+        view_count = int_or_none(data.get('viewCount'))
 
         return {
             'id': video_id,
+            'title': title,
             'thumbnail': self._og_search_thumbnail(webpage),
-            'title': self._og_search_title(webpage),
+            'duration': duration,
+            'view_count': view_count,
             'formats': formats,
         }

From bf4c6a38e1a98606b269d70ccc65c7ec5d47ec07 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 3 Jun 2016 23:25:24 +0700
Subject: [PATCH 5091/6219] release 2016.06.03

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 docs/supportedsites.md    | 7 +++++--
 youtube_dl/version.py     | 2 +-
 3 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index ae98e0626..e593ee78a 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.06.02*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.06.02**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.06.03*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.06.03**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.06.02
+[debug] youtube-dl version 2016.06.03
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index dcbc632a1..619bd0825 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -43,8 +43,8 @@
  - **appletrailers:section**
  - **archive.org**: archive.org videos
  - **ARD**
- - **ARD:mediathek**
  - **ARD:mediathek**: Saarländischer Rundfunk
+ - **ARD:mediathek**
  - **arte.tv**
  - **arte.tv:+7**
  - **arte.tv:cinema**
@@ -339,6 +339,7 @@
  - **livestream**
  - **livestream:original**
  - **LnkGo**
+ - **loc**: Library of Congress
  - **LocalNews8**
  - **LoveHomePorn**
  - **lrt.lt**
@@ -528,7 +529,8 @@
  - **Restudy**
  - **Reuters**
  - **ReverbNation**
- - **Revision3**
+ - **revision**
+ - **revision3:embed**
  - **RICE**
  - **RingTV**
  - **RottenTomatoes**
@@ -567,6 +569,7 @@
  - **ScreencastOMatic**
  - **ScreenJunkies**
  - **ScreenwaveMedia**
+ - **Seeker**
  - **SenateISVP**
  - **SendtoNews**
  - **ServingSys**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index fba427dde..d24d06f4a 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.06.02'
+__version__ = '2016.06.03'

From 76e9cd7f24f6b175e4cce85082647403266ed233 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 3 Jun 2016 23:43:34 +0700
Subject: [PATCH 5092/6219] [loc] Add support for another URL schema and
 simplify

---
 youtube_dl/extractor/libraryofcongress.py | 20 +++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/libraryofcongress.py b/youtube_dl/extractor/libraryofcongress.py
index d311f9946..a5f22b204 100644
--- a/youtube_dl/extractor/libraryofcongress.py
+++ b/youtube_dl/extractor/libraryofcongress.py
@@ -13,8 +13,8 @@ from ..utils import (
 class LibraryOfCongressIE(InfoExtractor):
     IE_NAME = 'loc'
     IE_DESC = 'Library of Congress'
-    _VALID_URL = r'https?://(?:www\.)?loc\.gov/item/(?P<id>[0-9]+)'
-    _TEST = {
+    _VALID_URL = r'https?://(?:www\.)?loc\.gov/(?:item/|today/cyberlc/feature_wdesc\.php\?.*\brec=)(?P<id>[0-9]+)'
+    _TESTS = [{
         'url': 'http://loc.gov/item/90716351/',
         'md5': '353917ff7f0255aa6d4b80a034833de8',
         'info_dict': {
@@ -25,7 +25,10 @@ class LibraryOfCongressIE(InfoExtractor):
             'duration': 0,
             'view_count': int,
         },
-    }
+    }, {
+        'url': 'https://www.loc.gov/today/cyberlc/feature_wdesc.php?rec=5578',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -34,13 +37,12 @@ class LibraryOfCongressIE(InfoExtractor):
         media_id = self._search_regex(
             (r'id=(["\'])media-player-(?P<id>.+?)\1',
              r'<video[^>]+id=(["\'])uuid-(?P<id>.+?)\1',
-             r'<video[^>]+data-uuid=(["\'])(?P<id>.+?)\1'),
+             r'<video[^>]+data-uuid=(["\'])(?P<id>.+?)\1',
+             r'mediaObjectId\s*:\s*(["\'])(?P<id>.+?)\1'),
             webpage, 'media id', group='id')
 
-        data = self._parse_json(
-            self._download_webpage(
-                'https://media.loc.gov/services/v1/media?id=%s&context=json' % media_id,
-                video_id),
+        data = self._download_json(
+            'https://media.loc.gov/services/v1/media?id=%s&context=json' % media_id,
             video_id)['mediaObject']
 
         derivative = data['derivatives'][0]
@@ -77,7 +79,7 @@ class LibraryOfCongressIE(InfoExtractor):
         return {
             'id': video_id,
             'title': title,
-            'thumbnail': self._og_search_thumbnail(webpage),
+            'thumbnail': self._og_search_thumbnail(webpage, default=None),
             'duration': duration,
             'view_count': view_count,
             'formats': formats,

From c917106be4d6d98ce7504d71a32b58ddca2bc03d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 3 Jun 2016 23:55:22 +0700
Subject: [PATCH 5093/6219] [loc] Extract subtites

---
 youtube_dl/extractor/libraryofcongress.py | 23 ++++++++++++++++++++++-
 1 file changed, 22 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/libraryofcongress.py b/youtube_dl/extractor/libraryofcongress.py
index a5f22b204..49351759e 100644
--- a/youtube_dl/extractor/libraryofcongress.py
+++ b/youtube_dl/extractor/libraryofcongress.py
@@ -15,6 +15,7 @@ class LibraryOfCongressIE(InfoExtractor):
     IE_DESC = 'Library of Congress'
     _VALID_URL = r'https?://(?:www\.)?loc\.gov/(?:item/|today/cyberlc/feature_wdesc\.php\?.*\brec=)(?P<id>[0-9]+)'
     _TESTS = [{
+        # embedded via <div class="media-player"
         'url': 'http://loc.gov/item/90716351/',
         'md5': '353917ff7f0255aa6d4b80a034833de8',
         'info_dict': {
@@ -26,8 +27,19 @@ class LibraryOfCongressIE(InfoExtractor):
             'view_count': int,
         },
     }, {
+        # webcast embedded via mediaObjectId
         'url': 'https://www.loc.gov/today/cyberlc/feature_wdesc.php?rec=5578',
-        'only_matching': True,
+        'info_dict': {
+            'id': '5578',
+            'ext': 'mp4',
+            'title': 'Help! Preservation Training Needs Here, There & Everywhere',
+            'duration': 3765,
+            'view_count': int,
+            'subtitles': 'mincount:1',
+        },
+        'params': {
+            'skip_download': True,
+        },
     }]
 
     def _real_extract(self, url):
@@ -76,6 +88,14 @@ class LibraryOfCongressIE(InfoExtractor):
         duration = float_or_none(data.get('duration'))
         view_count = int_or_none(data.get('viewCount'))
 
+        subtitles = {}
+        cc_url = data.get('ccUrl')
+        if cc_url:
+            subtitles.setdefault('en', []).append({
+                'url': cc_url,
+                'ext': 'ttml',
+            })
+
         return {
             'id': video_id,
             'title': title,
@@ -83,4 +103,5 @@ class LibraryOfCongressIE(InfoExtractor):
             'duration': duration,
             'view_count': view_count,
             'formats': formats,
+            'subtitles': subtitles,
         }

From 4d8856d511aef11b5dbeb9f6523c2a117bdbb85d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Jun 2016 00:26:03 +0700
Subject: [PATCH 5094/6219] [loc] Extract direct download links

---
 youtube_dl/extractor/libraryofcongress.py | 38 ++++++++++++++++++++++-
 1 file changed, 37 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/libraryofcongress.py b/youtube_dl/extractor/libraryofcongress.py
index 49351759e..0a94366fd 100644
--- a/youtube_dl/extractor/libraryofcongress.py
+++ b/youtube_dl/extractor/libraryofcongress.py
@@ -1,12 +1,15 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 
 from ..utils import (
     determine_ext,
     float_or_none,
     int_or_none,
+    parse_filesize,
 )
 
 
@@ -40,6 +43,20 @@ class LibraryOfCongressIE(InfoExtractor):
         'params': {
             'skip_download': True,
         },
+    }, {
+        # with direct download links
+        'url': 'https://www.loc.gov/item/78710669/',
+        'info_dict': {
+            'id': '78710669',
+            'ext': 'mp4',
+            'title': 'La vie et la passion de Jesus-Christ',
+            'duration': 0,
+            'view_count': int,
+            'formats': 'mincount:4',
+        },
+        'params': {
+            'skip_download': True,
+        },
     }]
 
     def _real_extract(self, url):
@@ -60,6 +77,9 @@ class LibraryOfCongressIE(InfoExtractor):
         derivative = data['derivatives'][0]
         media_url = derivative['derivativeUrl']
 
+        title = derivative.get('shortName') or data.get('shortName') or self._og_search_title(
+            webpage)
+
         # Following algorithm was extracted from setAVSource js function
         # found in webpage
         media_url = media_url.replace('rtmp', 'https')
@@ -75,6 +95,7 @@ class LibraryOfCongressIE(InfoExtractor):
                 'format_id': 'hls',
                 'ext': 'mp4',
                 'protocol': 'm3u8_native',
+                'quality': 1,
             }]
         elif 'vod/mp3:' in media_url:
             formats = [{
@@ -82,9 +103,24 @@ class LibraryOfCongressIE(InfoExtractor):
                 'vcodec': 'none',
             }]
 
+        download_urls = set()
+        for m in re.finditer(
+                r'<option[^>]+value=(["\'])(?P<url>.+?)\1[^>]+data-file-download=[^>]+>\s*(?P<id>.+?)(?:(?:&nbsp;|\s+)\((?P<size>.+?)\))?\s*<', webpage):
+            format_id = m.group('id').lower()
+            if format_id == 'gif':
+                continue
+            download_url = m.group('url')
+            if download_url in download_urls:
+                continue
+            download_urls.add(download_url)
+            formats.append({
+                'url': download_url,
+                'format_id': format_id,
+                'filesize_approx': parse_filesize(m.group('size')),
+            })
+
         self._sort_formats(formats)
 
-        title = derivative.get('shortName') or data.get('shortName') or self._og_search_title(webpage)
         duration = float_or_none(data.get('duration'))
         view_count = int_or_none(data.get('viewCount'))
 

From 762d44c9567af424b2731cb643429ddd8e76d704 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Jun 2016 04:57:16 +0700
Subject: [PATCH 5095/6219] [channel9] Add support for rss links (Closes #9673)

---
 youtube_dl/extractor/channel9.py | 123 ++++++++++++++++++-------------
 1 file changed, 70 insertions(+), 53 deletions(-)

diff --git a/youtube_dl/extractor/channel9.py b/youtube_dl/extractor/channel9.py
index c74553dcf..34d4e6156 100644
--- a/youtube_dl/extractor/channel9.py
+++ b/youtube_dl/extractor/channel9.py
@@ -20,54 +20,64 @@ class Channel9IE(InfoExtractor):
     '''
     IE_DESC = 'Channel 9'
     IE_NAME = 'channel9'
-    _VALID_URL = r'https?://(?:www\.)?channel9\.msdn\.com/(?P<contentpath>.+)/?'
+    _VALID_URL = r'https?://(?:www\.)?channel9\.msdn\.com/(?P<contentpath>.+?)(?P<rss>/RSS)?/?(?:[?#&]|$)'
 
-    _TESTS = [
-        {
-            'url': 'http://channel9.msdn.com/Events/TechEd/Australia/2013/KOS002',
-            'md5': 'bbd75296ba47916b754e73c3a4bbdf10',
-            'info_dict': {
-                'id': 'Events/TechEd/Australia/2013/KOS002',
-                'ext': 'mp4',
-                'title': 'Developer Kick-Off Session: Stuff We Love',
-                'description': 'md5:c08d72240b7c87fcecafe2692f80e35f',
-                'duration': 4576,
-                'thumbnail': 're:http://.*\.jpg',
-                'session_code': 'KOS002',
-                'session_day': 'Day 1',
-                'session_room': 'Arena 1A',
-                'session_speakers': ['Ed Blankenship', 'Andrew Coates', 'Brady Gaster', 'Patrick Klug', 'Mads Kristensen'],
-            },
+    _TESTS = [{
+        'url': 'http://channel9.msdn.com/Events/TechEd/Australia/2013/KOS002',
+        'md5': 'bbd75296ba47916b754e73c3a4bbdf10',
+        'info_dict': {
+            'id': 'Events/TechEd/Australia/2013/KOS002',
+            'ext': 'mp4',
+            'title': 'Developer Kick-Off Session: Stuff We Love',
+            'description': 'md5:c08d72240b7c87fcecafe2692f80e35f',
+            'duration': 4576,
+            'thumbnail': 're:http://.*\.jpg',
+            'session_code': 'KOS002',
+            'session_day': 'Day 1',
+            'session_room': 'Arena 1A',
+            'session_speakers': ['Ed Blankenship', 'Andrew Coates', 'Brady Gaster', 'Patrick Klug',
+                                 'Mads Kristensen'],
         },
-        {
-            'url': 'http://channel9.msdn.com/posts/Self-service-BI-with-Power-BI-nuclear-testing',
-            'md5': 'b43ee4529d111bc37ba7ee4f34813e68',
-            'info_dict': {
-                'id': 'posts/Self-service-BI-with-Power-BI-nuclear-testing',
-                'ext': 'mp4',
-                'title': 'Self-service BI with Power BI - nuclear testing',
-                'description': 'md5:d1e6ecaafa7fb52a2cacdf9599829f5b',
-                'duration': 1540,
-                'thumbnail': 're:http://.*\.jpg',
-                'authors': ['Mike Wilmot'],
-            },
+    }, {
+        'url': 'http://channel9.msdn.com/posts/Self-service-BI-with-Power-BI-nuclear-testing',
+        'md5': 'b43ee4529d111bc37ba7ee4f34813e68',
+        'info_dict': {
+            'id': 'posts/Self-service-BI-with-Power-BI-nuclear-testing',
+            'ext': 'mp4',
+            'title': 'Self-service BI with Power BI - nuclear testing',
+            'description': 'md5:d1e6ecaafa7fb52a2cacdf9599829f5b',
+            'duration': 1540,
+            'thumbnail': 're:http://.*\.jpg',
+            'authors': ['Mike Wilmot'],
         },
-        {
-            # low quality mp4 is best
-            'url': 'https://channel9.msdn.com/Events/CPP/CppCon-2015/Ranges-for-the-Standard-Library',
-            'info_dict': {
-                'id': 'Events/CPP/CppCon-2015/Ranges-for-the-Standard-Library',
-                'ext': 'mp4',
-                'title': 'Ranges for the Standard Library',
-                'description': 'md5:2e6b4917677af3728c5f6d63784c4c5d',
-                'duration': 5646,
-                'thumbnail': 're:http://.*\.jpg',
-            },
-            'params': {
-                'skip_download': True,
-            },
-        }
-    ]
+    }, {
+        # low quality mp4 is best
+        'url': 'https://channel9.msdn.com/Events/CPP/CppCon-2015/Ranges-for-the-Standard-Library',
+        'info_dict': {
+            'id': 'Events/CPP/CppCon-2015/Ranges-for-the-Standard-Library',
+            'ext': 'mp4',
+            'title': 'Ranges for the Standard Library',
+            'description': 'md5:2e6b4917677af3728c5f6d63784c4c5d',
+            'duration': 5646,
+            'thumbnail': 're:http://.*\.jpg',
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }, {
+        'url': 'https://channel9.msdn.com/Niners/Splendid22/Queue/76acff796e8f411184b008028e0d492b/RSS',
+        'info_dict': {
+            'id': 'Niners/Splendid22/Queue/76acff796e8f411184b008028e0d492b',
+            'title': 'Channel 9',
+        },
+        'playlist_count': 2,
+    }, {
+        'url': 'https://channel9.msdn.com/Events/DEVintersection/DEVintersection-2016/RSS',
+        'only_matching': True,
+    }, {
+        'url': 'https://channel9.msdn.com/Events/Speakers/scott-hanselman/RSS?UrlSafeName=scott-hanselman',
+        'only_matching': True,
+    }]
 
     _RSS_URL = 'http://channel9.msdn.com/%s/RSS'
 
@@ -254,22 +264,30 @@ class Channel9IE(InfoExtractor):
 
         return self.playlist_result(contents)
 
-    def _extract_list(self, content_path):
-        rss = self._download_xml(self._RSS_URL % content_path, content_path, 'Downloading RSS')
+    def _extract_list(self, video_id, rss_url=None):
+        if not rss_url:
+            rss_url = self._RSS_URL % video_id
+        rss = self._download_xml(rss_url, video_id, 'Downloading RSS')
         entries = [self.url_result(session_url.text, 'Channel9')
                    for session_url in rss.findall('./channel/item/link')]
         title_text = rss.find('./channel/title').text
-        return self.playlist_result(entries, content_path, title_text)
+        return self.playlist_result(entries, video_id, title_text)
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         content_path = mobj.group('contentpath')
+        rss = mobj.group('rss')
 
-        webpage = self._download_webpage(url, content_path, 'Downloading web page')
+        if rss:
+            return self._extract_list(content_path, url)
 
-        page_type_m = re.search(r'<meta name="WT.entryid" content="(?P<pagetype>[^:]+)[^"]+"/>', webpage)
-        if page_type_m is not None:
-            page_type = page_type_m.group('pagetype')
+        webpage = self._download_webpage(
+            url, content_path, 'Downloading web page')
+
+        page_type = self._search_regex(
+            r'<meta[^>]+name=(["\'])WT\.entryid\1[^>]+content=(["\'])(?P<pagetype>[^:]+).+?\2',
+            webpage, 'page type', default=None, group='pagetype')
+        if page_type:
             if page_type == 'Entry':      # Any 'item'-like page, may contain downloadable content
                 return self._extract_entry_item(webpage, content_path)
             elif page_type == 'Session':  # Event session page, may contain downloadable content
@@ -278,6 +296,5 @@ class Channel9IE(InfoExtractor):
                 return self._extract_list(content_path)
             else:
                 raise ExtractorError('Unexpected WT.entryid %s' % page_type, expected=True)
-
         else:  # Assuming list
             return self._extract_list(content_path)

From cad88f96dc8eaa845a458f0b80e92c1ba36c5491 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 4 Jun 2016 11:42:52 +0200
Subject: [PATCH 5096/6219] disable uploading to yt-dl.org for now

---
 devscripts/release.sh | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/devscripts/release.sh b/devscripts/release.sh
index cde4d0a39..1a7b1e054 100755
--- a/devscripts/release.sh
+++ b/devscripts/release.sh
@@ -97,8 +97,10 @@ RELEASE_FILES="youtube-dl youtube-dl.exe youtube-dl-$version.tar.gz"
 
 /bin/echo -e "\n### Signing and uploading the new binaries to yt-dl.org ..."
 for f in $RELEASE_FILES; do gpg --passphrase-repeat 5 --detach-sig "build/$version/$f"; done
-scp -r "build/$version" ytdl@yt-dl.org:html/tmp/
-ssh ytdl@yt-dl.org "mv html/tmp/$version html/downloads/"
+
+echo 'TODO: upload on GitHub'
+exit 1
+
 ssh ytdl@yt-dl.org "sh html/update_latest.sh $version"
 
 /bin/echo -e "\n### Now switching to gh-pages..."

From 7def35712a7047578643f18eaf6dda79fd8c9291 Mon Sep 17 00:00:00 2001
From: TRox1972 <TRox1972@users.noreply.github.com>
Date: Sat, 21 May 2016 17:48:17 +0200
Subject: [PATCH 5097/6219] [vidio] Add extractor (Closes #7195)

[Vidio] fix fallback value and wrap duration in int_or_none

[Vidio] don't use video_id for _html_search_regex()
---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/vidio.py      | 48 ++++++++++++++++++++++++++++++
 2 files changed, 49 insertions(+)
 create mode 100644 youtube_dl/extractor/vidio.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 3b5143ace..ed4e39574 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -910,6 +910,7 @@ from .videomore import (
 )
 from .videopremium import VideoPremiumIE
 from .videott import VideoTtIE
+from .vidio import VidioIE
 from .vidme import (
     VidmeIE,
     VidmeUserIE,
diff --git a/youtube_dl/extractor/vidio.py b/youtube_dl/extractor/vidio.py
new file mode 100644
index 000000000..d17c663fd
--- /dev/null
+++ b/youtube_dl/extractor/vidio.py
@@ -0,0 +1,48 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+from .common import InfoExtractor
+
+from ..utils import int_or_none
+
+
+class VidioIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?vidio\.com/watch/(?P<id>\d{6})-(?P<display_id>[^/?]+)'
+    _TEST = {
+        'url': 'http://www.vidio.com/watch/165683-dj_ambred-booyah-live-2015',
+        'info_dict': {
+            'id': '165683',
+            'title': 'DJ_AMBRED - Booyah (Live 2015)',
+            'ext': 'mp4',
+            'thumbnail': 'https://cdn0-a.production.vidio.static6.com/uploads/video/image/165683/dj_ambred-booyah-live-2015-bfb2ba.jpg',
+            'description': 'md5:27dc15f819b6a78a626490881adbadf8',
+            'duration': 149, 
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True
+        }
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id, display_id = mobj.group('id', 'display_id')
+
+        webpage = self._download_webpage(url, display_id)
+
+        video_data = self._parse_json(self._html_search_regex(
+            r'data-json-clips\s*=\s*"\[(.+)\]"', webpage, 'video data'), display_id)
+
+        formats = self._extract_m3u8_formats(
+            video_data['sources'][0]['file'],
+            display_id, ext='mp4')
+
+        return {
+            'id': video_id,
+            'title': self._og_search_title(webpage),
+            'formats': formats,
+            'thumbnail': video_data.get('image'),
+            'description': self._og_search_description(webpage),
+            'duration': int_or_none(video_data.get('clip_duration')),
+        }

From 0fc832e1b2c8f48298e135d42818a16bfba4d3ed Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Jun 2016 16:47:43 +0700
Subject: [PATCH 5098/6219] [vidio] Improve (Closes #9562)

---
 youtube_dl/extractor/vidio.py | 65 ++++++++++++++++++++++++-----------
 1 file changed, 45 insertions(+), 20 deletions(-)

diff --git a/youtube_dl/extractor/vidio.py b/youtube_dl/extractor/vidio.py
index d17c663fd..6898042de 100644
--- a/youtube_dl/extractor/vidio.py
+++ b/youtube_dl/extractor/vidio.py
@@ -2,28 +2,30 @@
 from __future__ import unicode_literals
 
 import re
-from .common import InfoExtractor
 
+from .common import InfoExtractor
 from ..utils import int_or_none
 
 
 class VidioIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?vidio\.com/watch/(?P<id>\d{6})-(?P<display_id>[^/?]+)'
-    _TEST = {
+    _VALID_URL = r'https?://(?:www\.)?vidio\.com/watch/(?P<id>\d+)-(?P<display_id>[^/?#&]+)'
+    _TESTS = [{
         'url': 'http://www.vidio.com/watch/165683-dj_ambred-booyah-live-2015',
+        'md5': 'cd2801394afc164e9775db6a140b91fe',
         'info_dict': {
             'id': '165683',
-            'title': 'DJ_AMBRED - Booyah (Live 2015)',
+            'display_id': 'dj_ambred-booyah-live-2015',
             'ext': 'mp4',
-            'thumbnail': 'https://cdn0-a.production.vidio.static6.com/uploads/video/image/165683/dj_ambred-booyah-live-2015-bfb2ba.jpg',
+            'title': 'DJ_AMBRED - Booyah (Live 2015)',
             'description': 'md5:27dc15f819b6a78a626490881adbadf8',
-            'duration': 149, 
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 149,
+            'like_count': int,
         },
-        'params': {
-            # m3u8 download
-            'skip_download': True
-        }
-    }
+    }, {
+        'url': 'https://www.vidio.com/watch/77949-south-korea-test-fires-missile-that-can-strike-all-of-the-north',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
@@ -31,18 +33,41 @@ class VidioIE(InfoExtractor):
 
         webpage = self._download_webpage(url, display_id)
 
-        video_data = self._parse_json(self._html_search_regex(
-            r'data-json-clips\s*=\s*"\[(.+)\]"', webpage, 'video data'), display_id)
+        title = self._og_search_title(webpage)
 
-        formats = self._extract_m3u8_formats(
-            video_data['sources'][0]['file'],
-            display_id, ext='mp4')
+        m3u8_url, duration, thumbnail = [None] * 3
+
+        clips = self._parse_json(
+            self._html_search_regex(
+                r'data-json-clips\s*=\s*(["\'])(?P<data>\[.+?\])\1',
+                webpage, 'video data', default='[]', group='data'),
+            display_id, fatal=False)
+        if clips:
+            clip = clips[0]
+            m3u8_url = clip.get('sources', [{}])[0].get('file')
+            duration = clip.get('clip_duration')
+            thumbnail = clip.get('image')
+
+        m3u8_url = m3u8_url or self._search_regex(
+            r'data(?:-vjs)?-clip-hls-url=(["\'])(?P<url>.+?)\1', webpage, 'hls url')
+        formats = self._extract_m3u8_formats(m3u8_url, display_id, 'mp4', entry_protocol='m3u8_native')
+
+        duration = int_or_none(duration or self._search_regex(
+            r'data-video-duration=(["\'])(?P<duartion>\d+)\1', webpage, 'duration'))
+        thumbnail = thumbnail or self._og_search_thumbnail(webpage)
+
+        like_count = int_or_none(self._search_regex(
+            (r'<span[^>]+data-comment-vote-count=["\'](\d+)',
+             r'<span[^>]+class=["\'].*?\blike(?:__|-)count\b.*?["\'][^>]*>\s*(\d+)'),
+            webpage, 'like count', fatal=False))
 
         return {
             'id': video_id,
-            'title': self._og_search_title(webpage),
-            'formats': formats,
-            'thumbnail': video_data.get('image'),
+            'display_id': display_id,
+            'title': title,
             'description': self._og_search_description(webpage),
-            'duration': int_or_none(video_data.get('clip_duration')),
+            'thumbnail': thumbnail,
+            'duration': duration,
+            'like_count': like_count,
+            'formats': formats,
         }

From 1ae6c83bceb6dbc7093fe35ddafcde08dd0151a6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 5 Jun 2016 00:43:55 +0700
Subject: [PATCH 5099/6219] [compat] Add compat_input

---
 youtube_dl/compat.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 06e5f3ff6..fabac9fd2 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -482,6 +482,11 @@ if sys.version_info < (3, 0) and sys.platform == 'win32':
 else:
     compat_getpass = getpass.getpass
 
+try:
+    compat_input = raw_input
+except NameError:  # Python 3
+    compat_input = input
+
 # Python < 2.6.5 require kwargs to be bytes
 try:
     def _testfunc(x):

From e92b552a102f509066a605b26d6df38eb73764b1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 5 Jun 2016 00:44:51 +0700
Subject: [PATCH 5100/6219] [devscripts/buildserver] Use compat_input from
 compat

---
 devscripts/buildserver.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/devscripts/buildserver.py b/devscripts/buildserver.py
index f7979c43e..fc99c3213 100644
--- a/devscripts/buildserver.py
+++ b/devscripts/buildserver.py
@@ -13,6 +13,7 @@ import os.path
 
 sys.path.insert(0, os.path.dirname(os.path.dirname((os.path.abspath(__file__)))))
 from youtube_dl.compat import (
+    compat_input,
     compat_http_server,
     compat_str,
     compat_urlparse,
@@ -30,11 +31,6 @@ try:
 except ImportError:  # Python 2
     import SocketServer as compat_socketserver
 
-try:
-    compat_input = raw_input
-except NameError:  # Python 3
-    compat_input = input
-
 
 class BuildHTTPServer(compat_socketserver.ThreadingMixIn, compat_http_server.HTTPServer):
     allow_reuse_address = True

From db56f281d9c5d57cb2c44a2ea356a9a0a12b3b4d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 5 Jun 2016 00:47:26 +0700
Subject: [PATCH 5101/6219] [devscripts/create-github-release] Add script for
 releasing on GitHub

Yet only Basic authentication is supported either via .netrc or by manual input
---
 devscripts/create-github-release.py | 112 ++++++++++++++++++++++++++++
 1 file changed, 112 insertions(+)
 create mode 100644 devscripts/create-github-release.py

diff --git a/devscripts/create-github-release.py b/devscripts/create-github-release.py
new file mode 100644
index 000000000..f74d39490
--- /dev/null
+++ b/devscripts/create-github-release.py
@@ -0,0 +1,112 @@
+#!/usr/bin/env python
+from __future__ import unicode_literals
+
+import base64
+import json
+import mimetypes
+import netrc
+import optparse
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from youtube_dl.compat import (
+    compat_basestring,
+    compat_input,
+    compat_getpass,
+    compat_print,
+    compat_urllib_request,
+)
+from youtube_dl.utils import (
+    make_HTTPS_handler,
+    sanitized_Request,
+)
+
+
+class GitHubReleaser(object):
+    _API_URL = 'https://api.github.com/repos/rg3/youtube-dl/releases'
+    _UPLOADS_URL = 'https://uploads.github.com/repos/rg3/youtube-dl/releases/%s/assets?name=%s'
+    _NETRC_MACHINE = 'github.com'
+
+    def __init__(self, debuglevel=0):
+        self._init_github_account()
+        https_handler = make_HTTPS_handler({}, debuglevel=debuglevel)
+        self._opener = compat_urllib_request.build_opener(https_handler)
+
+    def _init_github_account(self):
+        try:
+            info = netrc.netrc().authenticators(self._NETRC_MACHINE)
+            if info is not None:
+                self._username = info[0]
+                self._password = info[2]
+                compat_print('Using GitHub credentials found in .netrc...')
+                return
+            else:
+                compat_print('No GitHub credentials found in .netrc')
+        except (IOError, netrc.NetrcParseError):
+            compat_print('Unable to parse .netrc')
+        self._username = compat_input(
+            'Type your GitHub username or email address and press [Return]: ')
+        self._password = compat_getpass(
+            'Type your GitHub password and press [Return]: ')
+
+    def _call(self, req):
+        if isinstance(req, compat_basestring):
+            req = sanitized_Request(req)
+        # Authorizing manually since GitHub does not response with 401 with
+        # WWW-Authenticate header set (see
+        # https://developer.github.com/v3/#basic-authentication)
+        b64 = base64.b64encode(
+            ('%s:%s' % (self._username, self._password)).encode('utf-8')).decode('ascii')
+        req.add_header('Authorization', 'Basic %s' % b64)
+        response = self._opener.open(req).read().decode('utf-8')
+        return json.loads(response)
+
+    def list_releases(self):
+        return self._call(self._API_URL)
+
+    def create_release(self, tag_name, name=None, body='', draft=False, prerelease=False):
+        data = {
+            'tag_name': tag_name,
+            'target_commitish': 'master',
+            'name': name,
+            'body': body,
+            'draft': draft,
+            'prerelease': prerelease,
+        }
+        req = sanitized_Request(self._API_URL, json.dumps(data).encode('utf-8'))
+        return self._call(req)
+
+    def create_asset(self, release_id, asset):
+        asset_name = os.path.basename(asset)
+        url = self._UPLOADS_URL % (release_id, asset_name)
+        # Our files are small enough to be loaded directly into memory.
+        data = open(asset, 'rb').read()
+        req = sanitized_Request(url, data)
+        mime_type, _ = mimetypes.guess_type(asset_name)
+        req.add_header('Content-Type', mime_type or 'application/octet-stream')
+        return self._call(req)
+
+
+def main():
+    parser = optparse.OptionParser(usage='%prog VERSION BUILDPATH')
+    options, args = parser.parse_args()
+    if len(args) != 2:
+        parser.error('Expected a version and a build directory')
+
+    version, build_path = args
+
+    releaser = GitHubReleaser(debuglevel=0)
+
+    new_release = releaser.create_release(
+        version, name='youtube-dl %s' % version, draft=True, prerelease=True)
+    release_id = new_release['id']
+
+    for asset in os.listdir(build_path):
+        compat_print('Uploading %s...' % asset)
+        releaser.create_asset(release_id, os.path.join(build_path, asset))
+
+
+if __name__ == '__main__':
+    main()

From 39b32571df802ef869db1067454aa654f3f66235 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 5 Jun 2016 00:48:33 +0700
Subject: [PATCH 5102/6219] [devscripts/release.sh] Release to GitHub

---
 devscripts/release.sh | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/devscripts/release.sh b/devscripts/release.sh
index 1a7b1e054..87e8eda50 100755
--- a/devscripts/release.sh
+++ b/devscripts/release.sh
@@ -95,17 +95,16 @@ RELEASE_FILES="youtube-dl youtube-dl.exe youtube-dl-$version.tar.gz"
 (cd build/$version/ && sha256sum $RELEASE_FILES > SHA2-256SUMS)
 (cd build/$version/ && sha512sum $RELEASE_FILES > SHA2-512SUMS)
 
-/bin/echo -e "\n### Signing and uploading the new binaries to yt-dl.org ..."
+/bin/echo -e "\n### Signing and uploading the new binaries to GitHub..."
 for f in $RELEASE_FILES; do gpg --passphrase-repeat 5 --detach-sig "build/$version/$f"; done
 
-echo 'TODO: upload on GitHub'
-exit 1
+ROOT=$(pwd)
+python devscripts/create-github-release.py $version "$ROOT/build/$version"
 
 ssh ytdl@yt-dl.org "sh html/update_latest.sh $version"
 
 /bin/echo -e "\n### Now switching to gh-pages..."
 git clone --branch gh-pages --single-branch . build/gh-pages
-ROOT=$(pwd)
 (
     set -e
     ORIGIN_URL=$(git config --get remote.origin.url)

From 2c347352677f023678ffd488a51b19f54b97fa36 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 5 Jun 2016 01:44:13 +0700
Subject: [PATCH 5103/6219] [youtube] Add itags 256 and 258

---
 youtube_dl/extractor/youtube.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index f3f102c30..6c9f77d95 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -344,6 +344,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 48, 'preference': -50, 'container': 'm4a_dash'},
         '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 128, 'preference': -50, 'container': 'm4a_dash'},
         '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 256, 'preference': -50, 'container': 'm4a_dash'},
+        '256': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'preference': -50, 'container': 'm4a_dash'},
+        '258': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'preference': -50, 'container': 'm4a_dash'},
 
         # Dash webm
         '167': {'ext': 'webm', 'height': 360, 'width': 640, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},

From 1e236d7e2350e055bbe230b12490e4369aaa0956 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 5 Jun 2016 03:16:05 +0700
Subject: [PATCH 5104/6219] [downloader/hls] Do not rely on
 EXT-X-PLAYLIST-TYPE:EVENT

---
 youtube_dl/downloader/hls.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index 62136ee54..049fb78ce 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -23,11 +23,17 @@ class HlsFD(FragmentFD):
         UNSUPPORTED_FEATURES = (
             r'#EXT-X-KEY:METHOD=(?!NONE)',  # encrypted streams [1]
             r'#EXT-X-BYTERANGE',  # playlists composed of byte ranges of media files [2]
+
             # Live streams heuristic does not always work (e.g. geo restricted to Germany
             # http://hls-geo.daserste.de/i/videoportal/Film/c_620000/622873/format,716451,716457,716450,716458,716459,.mp4.csmil/index_4_av.m3u8?null=0)
             # r'#EXT-X-MEDIA-SEQUENCE:(?!0$)',  # live streams [3]
-            r'#EXT-X-PLAYLIST-TYPE:EVENT',  # media segments may be appended to the end of
-                                            # event media playlists [4]
+
+            # This heuristic also is not correct since segments may not be appended as well.
+            # Twitch vods have EXT-X-PLAYLIST-TYPE:EVENT despite no segments will definitely
+            # be appended to the end of the playlist.
+            # r'#EXT-X-PLAYLIST-TYPE:EVENT',  # media segments may be appended to the end of
+                                              # event media playlists [4]
+
             # 1. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.2.4
             # 2. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.2.2
             # 3. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.3.2

From 631d4c87ee84183917fcdf5db59e1cd1bb48d9a9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 5 Jun 2016 03:19:44 +0700
Subject: [PATCH 5105/6219] [twitch:vod] Use native hls

---
 youtube_dl/extractor/twitch.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index f7b98e190..d898f14c3 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -260,7 +260,7 @@ class TwitchVodIE(TwitchItemBaseIE):
                     'nauth': access_token['token'],
                     'nauthsig': access_token['sig'],
                 })),
-            item_id, 'mp4')
+            item_id, 'mp4', entry_protocol='m3u8_native')
 
         self._prefer_source(formats)
         info['formats'] = formats

From 51c4d85ce788497584bd056d571ed9b7b24c9651 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 5 Jun 2016 03:21:43 +0700
Subject: [PATCH 5106/6219] [downloader/hls] PEP 8

---
 youtube_dl/downloader/hls.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index 049fb78ce..8e4a7189a 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -32,7 +32,7 @@ class HlsFD(FragmentFD):
             # Twitch vods have EXT-X-PLAYLIST-TYPE:EVENT despite no segments will definitely
             # be appended to the end of the playlist.
             # r'#EXT-X-PLAYLIST-TYPE:EVENT',  # media segments may be appended to the end of
-                                              # event media playlists [4]
+            #                                 # event media playlists [4]
 
             # 1. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.2.4
             # 2. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.2.2

From 633b444fd29aa9d8b3ba722285ae2475ae66595f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 5 Jun 2016 03:31:10 +0700
Subject: [PATCH 5107/6219] [downloader/hls] Correct comment on twitch vods

---
 youtube_dl/downloader/hls.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index 8e4a7189a..54f2108e9 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -29,8 +29,8 @@ class HlsFD(FragmentFD):
             # r'#EXT-X-MEDIA-SEQUENCE:(?!0$)',  # live streams [3]
 
             # This heuristic also is not correct since segments may not be appended as well.
-            # Twitch vods have EXT-X-PLAYLIST-TYPE:EVENT despite no segments will definitely
-            # be appended to the end of the playlist.
+            # Twitch vods of finished streams have EXT-X-PLAYLIST-TYPE:EVENT despite
+            # no segments will definitely be appended to the end of the playlist.
             # r'#EXT-X-PLAYLIST-TYPE:EVENT',  # media segments may be appended to the end of
             #                                 # event media playlists [4]
 

From 71b9cb3107e156c7f17ec4cdf1d09421cb4dd4b1 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 4 Jun 2016 22:55:15 +0200
Subject: [PATCH 5108/6219] extend FAQ (#9696)

---
 README.md | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/README.md b/README.md
index 253d51bc8..91078eee8 100644
--- a/README.md
+++ b/README.md
@@ -842,6 +842,12 @@ It is *not* possible to detect whether a URL is supported or not. That's because
 
 If you want to find out whether a given URL is supported, simply call youtube-dl with it. If you get no videos back, chances are the URL is either not referring to a video or unsupported. You can find out which by examining the output (if you run youtube-dl on the console) or catching an `UnsupportedError` exception if you run it from a Python program.
 
+# Why do I need to go through that much red tape when filing bugs?
+
+Before we had the issue template, despite our extensive [bug reporting instructions](#bugs), about 80% of the issue reports we got were useless, for instance because people used ancient versions hundreds of releases old, because of simple syntactic errors (not in youtube-dl but in general shell usage), because the problem was alrady reported multiple times before, because people did not actually read an error message, even if it said "please install ffmpeg", because people did not mention the URL they were trying to download and many more simple, easy-to-avoid problems, many of whom were totally unrelated to youtube-dl.
+
+youtube-dl is an open-source project manned by too few volunteers, so we'd rather spend time fixing bugs where we are certain none of those simple problems apply, and where we can be reasonably confident to be able to reproduce the issue without asking the reporter repeatedly. As such, the output of `youtube-dl -v YOUR_URL_HERE` is really all that's required to file an issue. The issue template also guides you through some basic steps you can do, such as checking that your version of youtube-dl is current.
+
 # DEVELOPER INSTRUCTIONS
 
 Most users do not need to build youtube-dl and can [download the builds](http://rg3.github.io/youtube-dl/download.html) or get them from their distribution.

From bc270284b56b8ce7623b22b5c5cbf8d0d390c09e Mon Sep 17 00:00:00 2001
From: Ryan Schmidt <github@ryandesign.com>
Date: Sat, 4 Jun 2016 21:30:22 -0500
Subject: [PATCH 5109/6219] Update README.md to mention MacPorts

---
 README.md | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 91078eee8..f60e7ce33 100644
--- a/README.md
+++ b/README.md
@@ -27,10 +27,14 @@ If you do not have curl, you can alternatively use a recent wget:
 
 Windows users can [download an .exe file](https://yt-dl.org/latest/youtube-dl.exe) and place it in any location on their [PATH](http://en.wikipedia.org/wiki/PATH_%28variable%29) except for `%SYSTEMROOT%\System32` (e.g. **do not** put in `C:\Windows\System32`).
 
-OS X users can install **youtube-dl** with [Homebrew](http://brew.sh/).
+OS X users can install **youtube-dl** with [Homebrew](http://brew.sh/):
 
     brew install youtube-dl
 
+Or with [MacPorts](https://www.macports.org/):
+
+    sudo port install youtube-dl
+
 You can also use pip:
 
     sudo pip install youtube-dl

From 8f1aaa97a1e3eb60749f8046f2f0b1a0749d007c Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Sun, 5 Jun 2016 11:19:44 +0700
Subject: [PATCH 5110/6219] [README.md] Update pypi instructions

---
 README.md | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/README.md b/README.md
index f60e7ce33..e7240f41a 100644
--- a/README.md
+++ b/README.md
@@ -27,7 +27,13 @@ If you do not have curl, you can alternatively use a recent wget:
 
 Windows users can [download an .exe file](https://yt-dl.org/latest/youtube-dl.exe) and place it in any location on their [PATH](http://en.wikipedia.org/wiki/PATH_%28variable%29) except for `%SYSTEMROOT%\System32` (e.g. **do not** put in `C:\Windows\System32`).
 
-OS X users can install **youtube-dl** with [Homebrew](http://brew.sh/):
+You can also use pip:
+
+    sudo pip install --upgrade youtube-dl
+    
+This command will update youtube-dl if you have already installed it. See the [pypi page](https://pypi.python.org/pypi/youtube_dl) for more information.
+
+OS X users can install youtube-dl with [Homebrew](http://brew.sh/):
 
     brew install youtube-dl
 
@@ -35,10 +41,6 @@ Or with [MacPorts](https://www.macports.org/):
 
     sudo port install youtube-dl
 
-You can also use pip:
-
-    sudo pip install youtube-dl
-
 Alternatively, refer to the [developer instructions](#developer-instructions) for how to check out and work with the git repository. For further options, including PGP signatures, see the [youtube-dl Download Page](https://rg3.github.io/youtube-dl/download.html).
 
 # DESCRIPTION

From 47f12ad3e39ebe714abec7e7588e8e411e2841b5 Mon Sep 17 00:00:00 2001
From: Tobias Salzmann <eun@su.am>
Date: Sun, 5 Jun 2016 11:04:55 +0200
Subject: [PATCH 5111/6219] curl: follow redirect

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index e7240f41a..205c485d0 100644
--- a/README.md
+++ b/README.md
@@ -17,7 +17,7 @@ youtube-dl - download videos from youtube.com or other video platforms
 
 To install it right away for all UNIX users (Linux, OS X, etc.), type:
 
-    sudo curl https://yt-dl.org/latest/youtube-dl -o /usr/local/bin/youtube-dl
+    sudo curl -L https://yt-dl.org/latest/youtube-dl -o /usr/local/bin/youtube-dl
     sudo chmod a+rx /usr/local/bin/youtube-dl
 
 If you do not have curl, you can alternatively use a recent wget:

From 7b0d1c28597bd38567e5b4e853f669a5a601c6e8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 5 Jun 2016 21:01:20 +0700
Subject: [PATCH 5112/6219] [__init__] Use write_string instead of
 compat_string (Closes #9689)

---
 youtube_dl/__init__.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 5df965191..4905674ad 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -18,7 +18,6 @@ from .options import (
 from .compat import (
     compat_expanduser,
     compat_getpass,
-    compat_print,
     compat_shlex_split,
     workaround_optparse_bug9161,
 )
@@ -76,7 +75,7 @@ def _real_main(argv=None):
 
     # Dump user agent
     if opts.dump_user_agent:
-        compat_print(std_headers['User-Agent'])
+        write_string(std_headers['User-Agent'] + '\n', out=sys.stdout)
         sys.exit(0)
 
     # Batch file verification
@@ -101,10 +100,10 @@ def _real_main(argv=None):
 
     if opts.list_extractors:
         for ie in list_extractors(opts.age_limit):
-            compat_print(ie.IE_NAME + (' (CURRENTLY BROKEN)' if not ie._WORKING else ''))
+            write_string(ie.IE_NAME + (' (CURRENTLY BROKEN)' if not ie._WORKING else '') + '\n', out=sys.stdout)
             matchedUrls = [url for url in all_urls if ie.suitable(url)]
             for mu in matchedUrls:
-                compat_print('  ' + mu)
+                write_string('  ' + mu + '\n', out=sys.stdout)
         sys.exit(0)
     if opts.list_extractor_descriptions:
         for ie in list_extractors(opts.age_limit):
@@ -117,7 +116,7 @@ def _real_main(argv=None):
                 _SEARCHES = ('cute kittens', 'slithering pythons', 'falling cat', 'angry poodle', 'purple fish', 'running tortoise', 'sleeping bunny', 'burping cow')
                 _COUNTS = ('', '5', '10', 'all')
                 desc += ' (Example: "%s%s:%s" )' % (ie.SEARCH_KEY, random.choice(_COUNTS), random.choice(_SEARCHES))
-            compat_print(desc)
+            write_string(desc + '\n', out=sys.stdout)
         sys.exit(0)
 
     # Conflicting, missing and erroneous options

From 244fe977fec880f1bce55683437a711e12075b72 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 6 Jun 2016 02:52:58 +0700
Subject: [PATCH 5113/6219] [options] Add --load-info-json alias for symmetry
 with --write-info-json

---
 youtube_dl/options.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 14051b714..99ce4131f 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -668,7 +668,7 @@ def parseOpts(overrideArguments=None):
         action='store_true', dest='writeannotations', default=False,
         help='Write video annotations to a .annotations.xml file')
     filesystem.add_option(
-        '--load-info',
+        '--load-info-json', '--load-info',
         dest='load_info_filename', metavar='FILE',
         help='JSON file containing the video information (created with the "--write-info-json" option)')
     filesystem.add_option(

From db59b37d0bb2bbb4894f28b6b65d1d7f5496444d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 6 Jun 2016 03:02:11 +0700
Subject: [PATCH 5114/6219] [devscripts/create-github-release] Make full
 published releases by default

---
 devscripts/create-github-release.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/devscripts/create-github-release.py b/devscripts/create-github-release.py
index f74d39490..3b8021e74 100644
--- a/devscripts/create-github-release.py
+++ b/devscripts/create-github-release.py
@@ -97,10 +97,9 @@ def main():
 
     version, build_path = args
 
-    releaser = GitHubReleaser(debuglevel=0)
+    releaser = GitHubReleaser()
 
-    new_release = releaser.create_release(
-        version, name='youtube-dl %s' % version, draft=True, prerelease=True)
+    new_release = releaser.create_release(version, name='youtube-dl %s' % version)
     release_id = new_release['id']
 
     for asset in os.listdir(build_path):

From e67f6880257068c395d38e24a5e13f69902e1e4c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 5 Jun 2016 23:16:08 +0200
Subject: [PATCH 5115/6219] [compat] Add 'compat_input' to __all__

---
 youtube_dl/compat.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index fabac9fd2..e3cab4dd0 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -628,6 +628,7 @@ __all__ = [
     'compat_html_entities',
     'compat_http_client',
     'compat_http_server',
+    'compat_input',
     'compat_itertools_count',
     'compat_kwargs',
     'compat_ord',

From 345dec937fcc2b9ae106e91f4c01568c8c7e41f8 Mon Sep 17 00:00:00 2001
From: Kagami Hiiragi <kagami@genshiken.org>
Date: Tue, 7 Jun 2016 14:39:21 +0300
Subject: [PATCH 5116/6219] [vlive] Acknowledge vlive+ streams statuses

Same as common statuses just with "PRODUCT_" prefix:
PRODUCE_LIVE_END, PRODUCT_COMING_SOON, etc.
---
 youtube_dl/extractor/vlive.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/vlive.py b/youtube_dl/extractor/vlive.py
index 147f52d45..8d671cca7 100644
--- a/youtube_dl/extractor/vlive.py
+++ b/youtube_dl/extractor/vlive.py
@@ -9,6 +9,7 @@ from ..utils import (
     ExtractorError,
     float_or_none,
     int_or_none,
+    remove_start,
 )
 from ..compat import compat_urllib_parse_urlencode
 
@@ -39,6 +40,7 @@ class VLiveIE(InfoExtractor):
             webpage, 'video params')
         status, _, _, live_params, long_video_id, key = re.split(
             r'"\s*,\s*"', video_params)[2:8]
+        status = remove_start(status, 'PRODUCT_')
 
         if status == 'LIVE_ON_AIR' or status == 'BIG_EVENT_ON_AIR':
             live_params = self._parse_json('"%s"' % live_params, video_id)

From 74193838f71addcb08a9f56a7fad8c2e7df298ec Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 7 Jun 2016 22:12:20 +0700
Subject: [PATCH 5117/6219] [canal+] Improve extraction (Closes #9718)

---
 youtube_dl/extractor/canalplus.py | 33 ++++++++++++++++++++++---------
 1 file changed, 24 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/canalplus.py b/youtube_dl/extractor/canalplus.py
index 25b2d4efe..8d0f91158 100644
--- a/youtube_dl/extractor/canalplus.py
+++ b/youtube_dl/extractor/canalplus.py
@@ -4,11 +4,11 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import compat_urllib_parse_urlparse
 from ..utils import (
     ExtractorError,
     HEADRequest,
     unified_strdate,
-    url_basename,
     qualities,
     int_or_none,
 )
@@ -16,13 +16,25 @@ from ..utils import (
 
 class CanalplusIE(InfoExtractor):
     IE_DESC = 'canalplus.fr, piwiplus.fr and d8.tv'
-    _VALID_URL = r'https?://(?:www\.(?P<site>canalplus\.fr|piwiplus\.fr|d8\.tv|itele\.fr)/.*?/(?P<path>.*)|player\.canalplus\.fr/#/(?P<id>[0-9]+))'
+    _VALID_URL = r'''(?x)
+                        https?://
+                            (?:
+                                (?:
+                                    (?:(?:www|m)\.)?canalplus\.fr|
+                                    (?:www\.)?piwiplus\.fr|
+                                    (?:www\.)?d8\.tv|
+                                    (?:www\.)?itele\.fr
+                                )/(?:(?:[^/]+/)*(?P<display_id>[^/?#&]+))?(?:\?.*\bvid=(?P<vid>\d+))?|
+                                player\.canalplus\.fr/#/(?P<id>\d+)
+                            )
+
+                    '''
     _VIDEO_INFO_TEMPLATE = 'http://service.canal-plus.com/video/rest/getVideosLiees/%s/%s?format=json'
     _SITE_ID_MAP = {
-        'canalplus.fr': 'cplus',
-        'piwiplus.fr': 'teletoon',
-        'd8.tv': 'd8',
-        'itele.fr': 'itele',
+        'canalplus': 'cplus',
+        'piwiplus': 'teletoon',
+        'd8': 'd8',
+        'itele': 'itele',
     }
 
     _TESTS = [{
@@ -65,16 +77,19 @@ class CanalplusIE(InfoExtractor):
             'description': 'md5:8216206ec53426ea6321321f3b3c16db',
             'upload_date': '20150211',
         },
+    }, {
+        'url': 'http://m.canalplus.fr/?vid=1398231',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.groupdict().get('id')
+        video_id = mobj.groupdict().get('id') or mobj.groupdict().get('vid')
 
-        site_id = self._SITE_ID_MAP[mobj.group('site') or 'canal']
+        site_id = self._SITE_ID_MAP[compat_urllib_parse_urlparse(url).netloc.rsplit('.', 2)[-2]]
 
         # Beware, some subclasses do not define an id group
-        display_id = url_basename(mobj.group('path'))
+        display_id = mobj.group('display_id') or video_id
 
         if video_id is None:
             webpage = self._download_webpage(url, display_id)

From 3d9b3605a35eb48bd20e569ed9ce9d706e457ec6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 7 Jun 2016 22:26:18 +0700
Subject: [PATCH 5118/6219] [canal+] Update tests

---
 youtube_dl/extractor/canalplus.py | 48 +++++++++++++++++--------------
 1 file changed, 26 insertions(+), 22 deletions(-)

diff --git a/youtube_dl/extractor/canalplus.py b/youtube_dl/extractor/canalplus.py
index 8d0f91158..605c5e957 100644
--- a/youtube_dl/extractor/canalplus.py
+++ b/youtube_dl/extractor/canalplus.py
@@ -38,14 +38,14 @@ class CanalplusIE(InfoExtractor):
     }
 
     _TESTS = [{
-        'url': 'http://www.canalplus.fr/c-emissions/pid1830-c-zapping.html?vid=1263092',
-        'md5': '12164a6f14ff6df8bd628e8ba9b10b78',
+        'url': 'http://www.canalplus.fr/c-emissions/pid1830-c-zapping.html?vid=1192814',
+        'md5': '41f438a4904f7664b91b4ed0dec969dc',
         'info_dict': {
-            'id': '1263092',
+            'id': '1192814',
             'ext': 'mp4',
-            'title': 'Le Zapping - 13/05/15',
-            'description': 'md5:09738c0d06be4b5d06a0940edb0da73f',
-            'upload_date': '20150513',
+            'title': "L'Année du Zapping 2014 - L'Année du Zapping 2014",
+            'description': "Toute l'année 2014 dans un Zapping exceptionnel !",
+            'upload_date': '20150105',
         },
     }, {
         'url': 'http://www.piwiplus.fr/videos-piwi/pid1405-le-labyrinthe-boing-super-ranger.html?vid=1108190',
@@ -58,24 +58,28 @@ class CanalplusIE(InfoExtractor):
         },
         'skip': 'Only works from France',
     }, {
-        'url': 'http://www.d8.tv/d8-docs-mags/pid6589-d8-campagne-intime.html',
+        'url': 'http://www.d8.tv/d8-docs-mags/pid5198-d8-en-quete-d-actualite.html?vid=1390231',
         'info_dict': {
-            'id': '966289',
-            'ext': 'flv',
-            'title': 'Campagne intime - Documentaire exceptionnel',
-            'description': 'md5:d2643b799fb190846ae09c61e59a859f',
-            'upload_date': '20131108',
-        },
-        'skip': 'videos get deleted after a while',
-    }, {
-        'url': 'http://www.itele.fr/france/video/aubervilliers-un-lycee-en-colere-111559',
-        'md5': '38b8f7934def74f0d6f3ba6c036a5f82',
-        'info_dict': {
-            'id': '1213714',
+            'id': '1390231',
             'ext': 'mp4',
-            'title': 'Aubervilliers : un lycée en colère - Le 11/02/2015 à 06h45',
-            'description': 'md5:8216206ec53426ea6321321f3b3c16db',
-            'upload_date': '20150211',
+            'title': "Vacances pas chères : prix discount ou grosses dépenses ? - En quête d'actualité",
+            'description': 'md5:edb6cf1cb4a1e807b5dd089e1ac8bfc6',
+            'upload_date': '20160512',
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }, {
+        'url': 'http://www.itele.fr/chroniques/invite-bruce-toussaint/thierry-solere-nicolas-sarkozy-officialisera-sa-candidature-a-la-primaire-quand-il-le-voudra-167224',
+        'info_dict': {
+            'id': '1398334',
+            'ext': 'mp4',
+            'title': "L'invité de Bruce Toussaint du 07/06/2016 - ",
+            'description': 'md5:40ac7c9ad0feaeb6f605bad986f61324',
+            'upload_date': '20160607',
+        },
+        'params': {
+            'skip_download': True,
         },
     }, {
         'url': 'http://m.canalplus.fr/?vid=1398231',

From 57b6e9652e27aa46395dab6238e54d63746f9a0e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 7 Jun 2016 22:32:08 +0700
Subject: [PATCH 5119/6219] [canal+] Add support for d17.tv

---
 youtube_dl/extractor/canalplus.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/canalplus.py b/youtube_dl/extractor/canalplus.py
index 605c5e957..61463f249 100644
--- a/youtube_dl/extractor/canalplus.py
+++ b/youtube_dl/extractor/canalplus.py
@@ -23,6 +23,7 @@ class CanalplusIE(InfoExtractor):
                                     (?:(?:www|m)\.)?canalplus\.fr|
                                     (?:www\.)?piwiplus\.fr|
                                     (?:www\.)?d8\.tv|
+                                    (?:www\.)?d17\.tv|
                                     (?:www\.)?itele\.fr
                                 )/(?:(?:[^/]+/)*(?P<display_id>[^/?#&]+))?(?:\?.*\bvid=(?P<vid>\d+))?|
                                 player\.canalplus\.fr/#/(?P<id>\d+)
@@ -34,6 +35,7 @@ class CanalplusIE(InfoExtractor):
         'canalplus': 'cplus',
         'piwiplus': 'teletoon',
         'd8': 'd8',
+        'd17': 'd17',
         'itele': 'itele',
     }
 
@@ -84,6 +86,9 @@ class CanalplusIE(InfoExtractor):
     }, {
         'url': 'http://m.canalplus.fr/?vid=1398231',
         'only_matching': True,
+    }, {
+        'url': 'http://www.d17.tv/emissions/pid8303-lolywood.html?vid=1397061',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From a6571f1073eab6c9a4cc9800a0bff31cf12fe09f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 8 Jun 2016 00:19:33 +0800
Subject: [PATCH 5120/6219] [common] Fix <bootstrapInfo> detection in F4M
 manifests

Regression since 0a5685b26fae0940f14cb063a6e4fc6986f9c124
---
 youtube_dl/extractor/common.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 57793537b..bfd432160 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -45,6 +45,7 @@ from ..utils import (
     unescapeHTML,
     unified_strdate,
     url_basename,
+    xpath_element,
     xpath_text,
     xpath_with_ns,
     determine_protocol,
@@ -1030,7 +1031,7 @@ class InfoExtractor(object):
         if base_url:
             base_url = base_url.strip()
 
-        bootstrap_info = xpath_text(
+        bootstrap_info = xpath_element(
             manifest, ['{http://ns.adobe.com/f4m/1.0}bootstrapInfo', '{http://ns.adobe.com/f4m/2.0}bootstrapInfo'],
             'bootstrap info', default=None)
 
@@ -1085,7 +1086,7 @@ class InfoExtractor(object):
             formats.append({
                 'format_id': format_id,
                 'url': manifest_url,
-                'ext': 'flv' if bootstrap_info else None,
+                'ext': 'flv' if bootstrap_info is not None else None,
                 'tbr': tbr,
                 'width': width,
                 'height': height,

From a4a8201c02d06bff384ecb66a257dbec0652ff52 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 8 Jun 2016 00:25:51 +0800
Subject: [PATCH 5121/6219] [wdr] Update _TESTS

---
 youtube_dl/extractor/wdr.py | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/wdr.py b/youtube_dl/extractor/wdr.py
index 1e729cb7c..6174eb19f 100644
--- a/youtube_dl/extractor/wdr.py
+++ b/youtube_dl/extractor/wdr.py
@@ -21,7 +21,7 @@ class WDRIE(InfoExtractor):
     _TESTS = [
         {
             'url': 'http://www1.wdr.de/mediathek/video/sendungen/doku-am-freitag/video-geheimnis-aachener-dom-100.html',
-            'md5': 'e58c39c3e30077141d258bf588700a7b',
+            # HDS download, MD5 is unstable
             'info_dict': {
                 'id': 'mdb-1058683',
                 'ext': 'flv',
@@ -35,7 +35,6 @@ class WDRIE(InfoExtractor):
                     'url': 'http://ondemand-ww.wdr.de/medp/fsk0/105/1058683/1058683_12220974.xml'
                 }]},
             },
-            'skip': 'Page Not Found',
         },
         {
             'url': 'http://www1.wdr.de/mediathek/audio/wdr3/wdr3-gespraech-am-samstag/audio-schriftstellerin-juli-zeh-100.html',
@@ -51,7 +50,6 @@ class WDRIE(InfoExtractor):
                 'is_live': False,
                 'subtitles': {}
             },
-            'skip': 'Page Not Found',
         },
         {
             'url': 'http://www1.wdr.de/mediathek/video/live/index.html',
@@ -90,7 +88,7 @@ class WDRIE(InfoExtractor):
         },
         {
             'url': 'http://www.wdrmaus.de/sachgeschichten/sachgeschichten/achterbahn.php5',
-            'md5': 'ca365705551e4bd5217490f3b0591290',
+            # HDS download, MD5 is unstable
             'info_dict': {
                 'id': 'mdb-186083',
                 'ext': 'flv',
@@ -98,9 +96,6 @@ class WDRIE(InfoExtractor):
                 'title': 'Sachgeschichte - Achterbahn ',
                 'description': '- Die Sendung mit der Maus -',
             },
-            'params': {
-                'skip_download': True,  # the file has different versions :(
-            },
         },
     ]
 

From a26a9d62396641364690974de9c859cf26f9acf6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 7 Jun 2016 23:53:08 +0700
Subject: [PATCH 5122/6219] [livestream:event] Ensure video id is string
 (Closes #9721)

---
 youtube_dl/extractor/livestream.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/livestream.py b/youtube_dl/extractor/livestream.py
index 0edc06c43..bc7894bf1 100644
--- a/youtube_dl/extractor/livestream.py
+++ b/youtube_dl/extractor/livestream.py
@@ -203,9 +203,10 @@ class LivestreamIE(InfoExtractor):
             if not videos_info:
                 break
             for v in videos_info:
+                v_id = compat_str(v['id'])
                 entries.append(self.url_result(
-                    'http://livestream.com/accounts/%s/events/%s/videos/%s' % (account_id, event_id, v['id']),
-                    'Livestream', v['id'], v['caption']))
+                    'http://livestream.com/accounts/%s/events/%s/videos/%s' % (account_id, event_id, v_id),
+                    'Livestream', v_id, v.get('caption')))
             last_video = videos_info[-1]['id']
         return self.playlist_result(entries, event_id, event_data['full_name'])
 

From 33d9f3707ccccfe8d73c1b398f198792e80a259f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 8 Jun 2016 02:22:04 +0700
Subject: [PATCH 5123/6219] [thesixtyone] Relax _VALID_URL (Closes #9714)

---
 youtube_dl/extractor/thesixtyone.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/thesixtyone.py b/youtube_dl/extractor/thesixtyone.py
index d8b1fd281..d63aef5de 100644
--- a/youtube_dl/extractor/thesixtyone.py
+++ b/youtube_dl/extractor/thesixtyone.py
@@ -12,7 +12,7 @@ class TheSixtyOneIE(InfoExtractor):
             s|
             song/comments/list|
             song
-        )/(?P<id>[A-Za-z0-9]+)/?$'''
+        )/(?:[^/]+/)?(?P<id>[A-Za-z0-9]+)/?$'''
     _SONG_URL_TEMPLATE = 'http://thesixtyone.com/s/{0:}'
     _SONG_FILE_URL_TEMPLATE = 'http://{audio_server:}/thesixtyone_production/audio/{0:}_stream'
     _THUMBNAIL_URL_TEMPLATE = '{photo_base_url:}_desktop'
@@ -45,6 +45,10 @@ class TheSixtyOneIE(InfoExtractor):
             'url': 'http://www.thesixtyone.com/song/SrE3zD7s1jt/',
             'only_matching': True,
         },
+        {
+            'url': 'http://www.thesixtyone.com/maryatmidnight/song/StrawberriesandCream/yvWtLp0c4GQ/',
+            'only_matching': True,
+        },
     ]
 
     _DECODE_MAP = {

From 7264e385912951167c27b40df5fd22010d594b12 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 8 Jun 2016 14:29:53 +0800
Subject: [PATCH 5124/6219] [bilibili] Fix for videos without upload time
 (closes #9710)

---
 youtube_dl/extractor/bilibili.py | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/youtube_dl/extractor/bilibili.py b/youtube_dl/extractor/bilibili.py
index 910e539e4..b17047b39 100644
--- a/youtube_dl/extractor/bilibili.py
+++ b/youtube_dl/extractor/bilibili.py
@@ -102,6 +102,22 @@ class BiliBiliIE(InfoExtractor):
                 'uploader_id': '151066',
             },
         }],
+    }, {
+        # Missing upload time
+        'url': 'http://www.bilibili.com/video/av1867637/',
+        'info_dict': {
+            'id': '2880301',
+            'ext': 'flv',
+            'title': '【HDTV】【喜剧】岳父岳母真难当 （2014）【法国票房冠军】',
+            'description': '一个信奉天主教的法国旧式传统资产阶级家庭中有四个女儿。三个女儿却分别找了阿拉伯、犹太、中国丈夫，老夫老妻唯独期盼剩下未嫁的小女儿能找一个信奉天主教的法国白人，结果没想到小女儿找了一位非裔黑人……【这次应该不会跳帧了】',
+            'uploader': '黑夜为猫',
+            'uploader_id': '610729',
+        },
+        'params': {
+            # Just to test metadata extraction
+            'skip_download': True,
+        },
+        'expected_warnings': ['upload time'],
     }]
 
     # BiliBili blocks keys from time to time. The current key is extracted from
@@ -172,6 +188,7 @@ class BiliBiliIE(InfoExtractor):
         description = self._html_search_meta('description', webpage)
         datetime_str = self._html_search_regex(
             r'<time[^>]+datetime="([^"]+)"', webpage, 'upload time', fatal=False)
+        timestamp = None
         if datetime_str:
             timestamp = calendar.timegm(datetime.datetime.strptime(datetime_str, '%Y-%m-%dT%H:%M').timetuple())
 

From 50ce1c331c736d8219f3bf631ff069b9aecc48e3 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 8 Jun 2016 14:43:52 +0800
Subject: [PATCH 5125/6219] [downloader/external] Add another env for proxies
 in ffmpeg/avconv

Related sources:
https://git.libav.org/?p=libav.git;a=blob;f=libavformat/http.c;h=8fe8d11e1edfdbb04a8726db2c49cfef3f572aac;hb=HEAD#l152
https://git.libav.org/?p=libav.git;a=blob;f=libavformat/tls.c;h=fab243e93e20034e88e619188c13a44a5d8ccdb9;hb=HEAD#l63
https://github.com/FFmpeg/FFmpeg/blob/f8e89d8/libavformat/http.c#L191
https://github.com/FFmpeg/FFmpeg/blob/f8e89d8/libavformat/tls.c#L92
---
 youtube_dl/downloader/external.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index 3a73cee1c..3ff1f9ed4 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -210,6 +210,7 @@ class FFmpegFD(ExternalFD):
             # args += ['-http_proxy', proxy]
             env = os.environ.copy()
             compat_setenv('HTTP_PROXY', proxy, env=env)
+            compat_setenv('http_proxy', proxy, env=env)
 
         protocol = info_dict.get('protocol')
 

From 22a0a95247c30b346592b6e3d464776bceb3b934 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 8 Jun 2016 20:47:39 +0800
Subject: [PATCH 5126/6219] [theplatform] Some NBC videos require an additional
 cookie

Related: #9578
---
 youtube_dl/extractor/theplatform.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index 02dbef913..5793ec6ef 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -14,11 +14,13 @@ from ..compat import (
     compat_urllib_parse_urlparse,
 )
 from ..utils import (
+    determine_ext,
     ExtractorError,
     float_or_none,
     int_or_none,
     sanitized_Request,
     unsmuggle_url,
+    update_url_query,
     xpath_with_ns,
     mimetype2ext,
     find_xpath_attr,
@@ -48,6 +50,12 @@ class ThePlatformBaseIE(OnceIE):
             if OnceIE.suitable(_format['url']):
                 formats.extend(self._extract_once_formats(_format['url']))
             else:
+                media_url = _format['url']
+                if determine_ext(media_url) == 'm3u8':
+                    hdnea2 = self._get_cookies(media_url).get('hdnea2')
+                    if hdnea2:
+                        _format['url'] = update_url_query(media_url, {'hdnea3': hdnea2.value})
+
                 formats.append(_format)
 
         subtitles = self._parse_smil_subtitles(meta, default_ns)

From e6e90515db983ca447cf7a59bbc153907d4fff4a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 8 Jun 2016 20:50:01 +0800
Subject: [PATCH 5127/6219] [nbc] Add the test case from #9578

Closes #9578
---
 youtube_dl/extractor/nbc.py | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index 46504cd5f..f27c7f139 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -67,6 +67,23 @@ class NBCIE(InfoExtractor):
             # This video has expired but with an escaped embedURL
             'url': 'http://www.nbc.com/parenthood/episode-guide/season-5/just-like-at-home/515',
             'only_matching': True,
+        },
+        {
+            # HLS streams requires the 'hdnea3' cookie
+            'url': 'http://www.nbc.com/Kings/video/goliath/n1806',
+            'info_dict': {
+                'id': 'n1806',
+                'ext': 'mp4',
+                'title': 'Goliath',
+                'description': 'When an unknown soldier saves the life of the King\'s son in battle, he\'s thrust into the limelight and politics of the kingdom.',
+                'timestamp': 1237100400,
+                'upload_date': '20090315',
+                'uploader': 'NBCU-COM',
+            },
+            'params': {
+                'skip_download': True,
+            },
+            'skip': 'Only works from US',
         }
     ]
 

From fc0a45fa416ad3e3ecf5936061efbb0328afa6b5 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 8 Jun 2016 21:12:14 +0800
Subject: [PATCH 5128/6219] [twitter] Detect suspended accounts and update
 _TESTS

---
 youtube_dl/extractor/twitter.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index ea673054f..129103c64 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -207,6 +207,7 @@ class TwitterIE(InfoExtractor):
             'uploader_id': 'giphz',
         },
         'expected_warnings': ['height', 'width'],
+        'skip': 'Account suspended',
     }, {
         'url': 'https://twitter.com/starwars/status/665052190608723968',
         'md5': '39b7199856dee6cd4432e72c74bc69d4',
@@ -278,7 +279,11 @@ class TwitterIE(InfoExtractor):
         user_id = mobj.group('user_id')
         twid = mobj.group('id')
 
-        webpage = self._download_webpage(self._TEMPLATE_URL % (user_id, twid), twid)
+        webpage, urlh = self._download_webpage_handle(
+            self._TEMPLATE_URL % (user_id, twid), twid)
+
+        if 'twitter.com/account/suspended' in urlh.geturl():
+            raise ExtractorError('Account suspended by Twitter.', expected=True)
 
         username = remove_end(self._og_search_title(webpage), ' on Twitter')
 

From c6308b3153acc57300f750f0061c63ffcba4d150 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 8 Jun 2016 21:28:10 +0800
Subject: [PATCH 5129/6219] [twitter] Fix extraction for videos with HLS
 streams

Closes #9623
---
 youtube_dl/extractor/twitter.py | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 129103c64..76421e533 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -5,6 +5,7 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
+    determine_ext,
     float_or_none,
     xpath_text,
     remove_end,
@@ -116,13 +117,16 @@ class TwitterCardIE(TwitterBaseIE):
         video_url = config.get('video_url') or config.get('playlist', [{}])[0].get('source')
 
         if video_url:
-            f = {
-                'url': video_url,
-            }
+            if determine_ext(video_url) == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(video_url, video_id, ext='mp4', m3u8_id='hls'))
+            else:
+                f = {
+                    'url': video_url,
+                }
 
-            _search_dimensions_in_video_url(f, video_url)
+                _search_dimensions_in_video_url(f, video_url)
 
-            formats.append(f)
+                formats.append(f)
 
         vmap_url = config.get('vmapUrl') or config.get('vmap_url')
         if vmap_url:
@@ -263,7 +267,6 @@ class TwitterIE(InfoExtractor):
         'add_ie': ['Vine'],
     }, {
         'url': 'https://twitter.com/captainamerica/status/719944021058060289',
-        # md5 constantly changes
         'info_dict': {
             'id': '719944021058060289',
             'ext': 'mp4',
@@ -272,6 +275,9 @@ class TwitterIE(InfoExtractor):
             'uploader_id': 'captainamerica',
             'uploader': 'Captain America',
         },
+        'params': {
+            'skip_download': True,  # requires ffmpeg
+        },
     }]
 
     def _real_extract(self, url):

From 6da8d7de69af144a96e9e50168e66f66af54129f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 8 Jun 2016 21:48:12 +0800
Subject: [PATCH 5130/6219] [twitter] Update _TESTS

---
 youtube_dl/extractor/twitter.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 76421e533..b73842986 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -53,7 +53,7 @@ class TwitterCardIE(TwitterBaseIE):
                 'id': 'dq4Oj5quskI',
                 'ext': 'mp4',
                 'title': 'Ubuntu 11.10 Overview',
-                'description': 'Take a quick peek at what\'s new and improved in Ubuntu 11.10.\n\nOnce installed take a look at 10 Things to Do After Installing: http://www.omgubuntu.co.uk/2011/10/10-things-to-do-after-installing-ubuntu-11-10/',
+                'description': 'Take a quick peek at what\'s new and improved in Ubuntu 11.10.\n\nOnce installed take a look at 10 Things to Do After Installing: http://www.omgubuntu.co.uk/2011/10/10...',
                 'upload_date': '20111013',
                 'uploader': 'OMG! Ubuntu!',
                 'uploader_id': 'omgubuntu',
@@ -244,10 +244,10 @@ class TwitterIE(InfoExtractor):
         'info_dict': {
             'id': '700207533655363584',
             'ext': 'mp4',
-            'title': 'jay - BEAT PROD: @suhmeduh #Damndaniel',
-            'description': 'jay on Twitter: "BEAT PROD: @suhmeduh  https://t.co/HBrQ4AfpvZ #Damndaniel https://t.co/byBooq2ejZ"',
+            'title': 'Donte The Dumbass - BEAT PROD: @suhmeduh #Damndaniel',
+            'description': 'Donte The Dumbass on Twitter: "BEAT PROD: @suhmeduh  https://t.co/HBrQ4AfpvZ #Damndaniel https://t.co/byBooq2ejZ"',
             'thumbnail': 're:^https?://.*\.jpg',
-            'uploader': 'jay',
+            'uploader': 'Donte The Dumbass',
             'uploader_id': 'jaydingeer',
         },
         'params': {

From 411c590a1f997f9efd71be8f434821acbf33a35f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 8 Jun 2016 23:45:46 +0800
Subject: [PATCH 5131/6219] [youku:show] Add new extractor

---
 youtube_dl/extractor/extractors.py |  5 ++-
 youtube_dl/extractor/youku.py      | 52 ++++++++++++++++++++++++++++++
 2 files changed, 56 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index d107080f5..676a0400c 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1013,7 +1013,10 @@ from .yesjapan import YesJapanIE
 from .yinyuetai import YinYueTaiIE
 from .ynet import YnetIE
 from .youjizz import YouJizzIE
-from .youku import YoukuIE
+from .youku import (
+    YoukuIE,
+    YoukuShowIE,
+)
 from .youporn import YouPornIE
 from .yourupload import YourUploadIE
 from .youtube import (
diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index dbccbe228..147608ebe 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -2,7 +2,9 @@
 from __future__ import unicode_literals
 
 import base64
+import itertools
 import random
+import re
 import string
 import time
 
@@ -13,6 +15,7 @@ from ..compat import (
 )
 from ..utils import (
     ExtractorError,
+    get_element_by_attribute,
     sanitized_Request,
 )
 
@@ -285,3 +288,52 @@ class YoukuIE(InfoExtractor):
             'title': title,
             'entries': entries,
         }
+
+
+class YoukuShowIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?youku\.com/show_page/id_(?P<id>[0-9a-z]+)\.html'
+    IE_NAME = 'youku:show'
+
+    _TEST = {
+        'url': 'http://www.youku.com/show_page/id_zc7c670be07ff11e48b3f.html',
+        'info_dict': {
+            'id': 'zc7c670be07ff11e48b3f',
+            'title': '花千骨 未删减版',
+            'description': 'md5:578d4f2145ae3f9128d9d4d863312910',
+        },
+        'playlist_count': 50,
+    }
+
+    _PAGE_SIZE = 40
+
+    def _find_videos_in_page(self, webpage):
+        videos = re.findall(
+            r'<li><a[^>]+href="(?P<url>https?://v\.youku\.com/[^"]+)"[^>]+title="(?P<title>[^"]+)"', webpage)
+        return [
+            self.url_result(video_url, YoukuIE.ie_key(), title)
+            for video_url, title in videos]
+
+    def _real_extract(self, url):
+        show_id = self._match_id(url)
+        webpage = self._download_webpage(url, show_id)
+
+        entries = self._find_videos_in_page(webpage)
+
+        playlist_title = self._html_search_regex(
+            r'<span[^>]+class="name">([^<]+)</span>', webpage, 'playlist title', fatal=False)
+        detail_div = get_element_by_attribute('class', 'detail', webpage) or ''
+        playlist_description = self._html_search_regex(
+            r'<span[^>]+style="display:none"[^>]*>([^<]+)</span>',
+            detail_div, 'playlist description', fatal=False)
+
+        for idx in itertools.count(1):
+            episodes_page = self._download_webpage(
+                'http://www.youku.com/show_episode/id_%s.html' % show_id,
+                show_id, query={'divid': 'reload_%d' % (idx * self._PAGE_SIZE + 1)},
+                note='Downloading episodes page %d' % idx)
+            new_entries = self._find_videos_in_page(episodes_page)
+            entries.extend(new_entries)
+            if len(new_entries) < self._PAGE_SIZE:
+                break
+
+        return self.playlist_result(entries, show_id, playlist_title, playlist_description)

From 11380753b5aa9d8128ef28a968ab325973276fa5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 9 Jun 2016 04:00:47 +0700
Subject: [PATCH 5132/6219] [vessel] Add support for embed urls and improve
 extraction

---
 youtube_dl/extractor/vessel.py | 27 ++++++++++++++++++++-------
 1 file changed, 20 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/vessel.py b/youtube_dl/extractor/vessel.py
index 1a0ff3395..e027c018b 100644
--- a/youtube_dl/extractor/vessel.py
+++ b/youtube_dl/extractor/vessel.py
@@ -2,6 +2,7 @@
 from __future__ import unicode_literals
 
 import json
+import re
 
 from .common import InfoExtractor
 from ..utils import (
@@ -12,11 +13,11 @@ from ..utils import (
 
 
 class VesselIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?vessel\.com/videos/(?P<id>[0-9a-zA-Z]+)'
+    _VALID_URL = r'https?://(?:www\.)?vessel\.com/(?:videos|embed)/(?P<id>[0-9a-zA-Z]+)'
     _API_URL_TEMPLATE = 'https://www.vessel.com/api/view/items/%s'
     _LOGIN_URL = 'https://www.vessel.com/api/account/login'
     _NETRC_MACHINE = 'vessel'
-    _TEST = {
+    _TESTS = [{
         'url': 'https://www.vessel.com/videos/HDN7G5UMs',
         'md5': '455cdf8beb71c6dd797fd2f3818d05c4',
         'info_dict': {
@@ -28,7 +29,16 @@ class VesselIE(InfoExtractor):
             'description': 'Did Nvidia pull out all the stops on the Titan X, or does its performance leave something to be desired?',
             'timestamp': int,
         },
-    }
+    }, {
+        'url': 'https://www.vessel.com/embed/G4U7gUJ6a?w=615&h=346',
+        'only_matching': True,
+    }]
+
+    @staticmethod
+    def _extract_urls(webpage):
+        return [url for _, url in re.findall(
+            r'<iframe[^>]+src=(["\'])((?:https?:)?//(?:www\.)?vessel\.com/embed/[0-9a-zA-Z]+.*?)\1',
+            webpage)]
 
     @staticmethod
     def make_json_request(url, data):
@@ -98,16 +108,19 @@ class VesselIE(InfoExtractor):
 
         formats = []
         for f in video_asset.get('sources', []):
-            if f['name'] == 'hls-index':
+            location = f.get('location')
+            if not location:
+                continue
+            if f.get('name') == 'hls-index':
                 formats.extend(self._extract_m3u8_formats(
-                    f['location'], video_id, ext='mp4', m3u8_id='m3u8'))
+                    location, video_id, ext='mp4', m3u8_id='m3u8'))
             else:
                 formats.append({
-                    'format_id': f['name'],
+                    'format_id': f.get('name'),
                     'tbr': f.get('bitrate'),
                     'height': f.get('height'),
                     'width': f.get('width'),
-                    'url': f['location'],
+                    'url': location,
                 })
         self._sort_formats(formats)
 

From 48a5eabc487058ccaa1076b74ad9106fc6019955 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 9 Jun 2016 04:02:27 +0700
Subject: [PATCH 5133/6219] [extractor/generic] Add support vessel embeds
 (Closes #7083)

---
 youtube_dl/extractor/generic.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index b4138381d..90575ab0e 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -63,6 +63,7 @@ from .instagram import InstagramIE
 from .liveleak import LiveLeakIE
 from .threeqsdn import ThreeQSDNIE
 from .theplatform import ThePlatformIE
+from .vessel import VesselIE
 
 
 class GenericIE(InfoExtractor):
@@ -1533,6 +1534,11 @@ class GenericIE(InfoExtractor):
         if tp_urls:
             return _playlist_from_matches(tp_urls, ie='ThePlatform')
 
+        # Look for Vessel embeds
+        vessel_urls = VesselIE._extract_urls(webpage)
+        if vessel_urls:
+            return _playlist_from_matches(vessel_urls, ie=VesselIE.ie_key())
+
         # Look for embedded rtl.nl player
         matches = re.findall(
             r'<iframe[^>]+?src="((?:https?:)?//(?:www\.)?rtl\.nl/system/videoplayer/[^"]+(?:video_)?embed[^"]+)"',

From a479b8f687245a9cb1b5c25ed9ece28c4710981f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 9 Jun 2016 04:09:32 +0700
Subject: [PATCH 5134/6219] [vessel] Use native hls by default

---
 youtube_dl/extractor/vessel.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vessel.py b/youtube_dl/extractor/vessel.py
index e027c018b..59f2b4ba4 100644
--- a/youtube_dl/extractor/vessel.py
+++ b/youtube_dl/extractor/vessel.py
@@ -113,7 +113,8 @@ class VesselIE(InfoExtractor):
                 continue
             if f.get('name') == 'hls-index':
                 formats.extend(self._extract_m3u8_formats(
-                    location, video_id, ext='mp4', m3u8_id='m3u8'))
+                    location, video_id, ext='mp4',
+                    entry_protocol='m3u8_native', m3u8_id='m3u8'))
             else:
                 formats.append({
                     'format_id': f.get('name'),

From 39da509f6712b6b0e9d52a9c9e990a5b5cd6c2ac Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 9 Jun 2016 04:12:48 +0700
Subject: [PATCH 5135/6219] [vessel] Extract DASH formats

---
 youtube_dl/extractor/vessel.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vessel.py b/youtube_dl/extractor/vessel.py
index 59f2b4ba4..c53f44584 100644
--- a/youtube_dl/extractor/vessel.py
+++ b/youtube_dl/extractor/vessel.py
@@ -111,13 +111,17 @@ class VesselIE(InfoExtractor):
             location = f.get('location')
             if not location:
                 continue
-            if f.get('name') == 'hls-index':
+            name = f.get('name')
+            if name == 'hls-index':
                 formats.extend(self._extract_m3u8_formats(
                     location, video_id, ext='mp4',
                     entry_protocol='m3u8_native', m3u8_id='m3u8'))
+            elif name == 'dash-index':
+                formats.extend(self._extract_mpd_formats(
+                    location, video_id, mpd_id='dash', fatal=False))
             else:
                 formats.append({
-                    'format_id': f.get('name'),
+                    'format_id': name,
                     'tbr': f.get('bitrate'),
                     'height': f.get('height'),
                     'width': f.get('width'),

From 9d51a0a9a19f07997cfb3ff1bb9fc9c1669a455c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 9 Jun 2016 04:13:38 +0700
Subject: [PATCH 5136/6219] [vessel] Make hls formats non fatal

---
 youtube_dl/extractor/vessel.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vessel.py b/youtube_dl/extractor/vessel.py
index c53f44584..2cd617b91 100644
--- a/youtube_dl/extractor/vessel.py
+++ b/youtube_dl/extractor/vessel.py
@@ -115,7 +115,7 @@ class VesselIE(InfoExtractor):
             if name == 'hls-index':
                 formats.extend(self._extract_m3u8_formats(
                     location, video_id, ext='mp4',
-                    entry_protocol='m3u8_native', m3u8_id='m3u8'))
+                    entry_protocol='m3u8_native', m3u8_id='m3u8', fatal=False))
             elif name == 'dash-index':
                 formats.extend(self._extract_mpd_formats(
                     location, video_id, mpd_id='dash', fatal=False))

From be6217b26142491232fb697b125015d45437832d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 9 Jun 2016 05:34:19 +0700
Subject: [PATCH 5137/6219] [YoutubeDL] Force string conversion on non string
 video ids

---
 youtube_dl/YoutubeDL.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 3917ca9dc..5036289b0 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1223,6 +1223,10 @@ class YoutubeDL(object):
         if 'title' not in info_dict:
             raise ExtractorError('Missing "title" field in extractor result')
 
+        if not isinstance(info_dict['id'], compat_str):
+            self.report_warning('"id" field is not a string - forcing string conversion')
+            info_dict['id'] = compat_str(info_dict['id'])
+
         if 'playlist' not in info_dict:
             # It isn't part of a playlist
             info_dict['playlist'] = None

From 6c33d24b46ecfb1f2ce790e21f2410149fdfb095 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 9 Jun 2016 12:58:24 +0800
Subject: [PATCH 5138/6219] [utils] Add audio/mpeg to mimetype2ext()

Used in WDR live radios (#6147)
---
 youtube_dl/utils.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 89234b39d..229de4b39 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2020,6 +2020,9 @@ def mimetype2ext(mt):
 
     ext = {
         'audio/mp4': 'm4a',
+        # Per RFC 3003, audio/mpeg can be .mp1, .mp2 or .mp3. Here use .mp3 as
+        # it's the most popular one
+        'audio/mpeg': 'mp3',
     }.get(mt)
     if ext is not None:
         return ext

From 50918c4ee01be6c1218a72bef35838216b2bf8d1 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 9 Jun 2016 13:04:30 +0800
Subject: [PATCH 5139/6219] [wdr] Support radio players (closes #6147)

---
 youtube_dl/extractor/wdr.py | 29 ++++++++++++++++++++++++-----
 1 file changed, 24 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/wdr.py b/youtube_dl/extractor/wdr.py
index 6174eb19f..059e2aa08 100644
--- a/youtube_dl/extractor/wdr.py
+++ b/youtube_dl/extractor/wdr.py
@@ -10,12 +10,13 @@ from ..utils import (
     strip_jsonp,
     unified_strdate,
     ExtractorError,
+    urlhandle_detect_ext,
 )
 
 
 class WDRIE(InfoExtractor):
     _CURRENT_MAUS_URL = r'https?://(?:www\.)wdrmaus.de/(?:[^/]+/){1,2}[^/?#]+\.php5'
-    _PAGE_REGEX = r'/mediathek/(?P<media_type>[^/]+)/(?P<type>[^/]+)/(?P<display_id>.+)\.html'
+    _PAGE_REGEX = r'/(?:mediathek/)?(?P<media_type>[^/]+)/(?P<type>[^/]+)/(?P<display_id>.+)\.html'
     _VALID_URL = r'(?P<page_url>https?://(?:www\d\.)?wdr\d?\.de)' + _PAGE_REGEX + '|' + _CURRENT_MAUS_URL
 
     _TESTS = [
@@ -97,6 +98,16 @@ class WDRIE(InfoExtractor):
                 'description': '- Die Sendung mit der Maus -',
             },
         },
+        {
+            'url': 'http://www1.wdr.de/radio/player/radioplayer116~_layout-popupVersion.html',
+            'info_dict': {
+                'id': 'mdb-869971',
+                'ext': 'mp3',
+                'title': 'Funkhaus Europa Livestream',
+                'description': 'md5:2309992a6716c347891c045be50992e4',
+                'upload_date': '20160101',
+            },
+        }
     ]
 
     def _real_extract(self, url):
@@ -107,9 +118,10 @@ class WDRIE(InfoExtractor):
         webpage = self._download_webpage(url, display_id)
 
         # for wdr.de the data-extension is in a tag with the class "mediaLink"
+        # for wdr.de radio players, in a tag with the class "wdrrPlayerPlayBtn"
         # for wdrmaus its in a link to the page in a multiline "videoLink"-tag
         json_metadata = self._html_search_regex(
-            r'class=(?:"mediaLink\b[^"]*"[^>]+|"videoLink\b[^"]*"[\s]*>\n[^\n]*)data-extension="([^"]+)"',
+            r'class=(?:"(?:mediaLink|wdrrPlayerPlayBtn)\b[^"]*"[^>]+|"videoLink\b[^"]*"[\s]*>\n[^\n]*)data-extension="([^"]+)"',
             webpage, 'media link', default=None, flags=re.MULTILINE)
 
         if not json_metadata:
@@ -143,15 +155,22 @@ class WDRIE(InfoExtractor):
             for tag_name in ['videoURL', 'audioURL']:
                 if tag_name in metadata_media_alt:
                     alt_url = metadata_media_alt[tag_name]
-                    if determine_ext(alt_url) == 'm3u8':
+                    ext = determine_ext(alt_url)
+                    if ext == 'm3u8':
                         m3u_fmt = self._extract_m3u8_formats(
                             alt_url, display_id, 'mp4', 'm3u8_native',
                             m3u8_id='hls')
                         formats.extend(m3u_fmt)
                     else:
-                        formats.append({
+                        a_format = {
                             'url': alt_url
-                        })
+                        }
+                        if ext == 'unknown_video':
+                            urlh = self._request_webpage(
+                                alt_url, display_id, note='Determining extension')
+                            ext = urlhandle_detect_ext(urlh)
+                            a_format['ext'] = ext
+                        formats.append(a_format)
 
         # check if there are flash-streams for this video
         if 'dflt' in metadata_media_resource and 'videoURL' in metadata_media_resource['dflt']:

From 6869d634c6d7482dd53034dec8a8f2f0b8e1f9b0 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 9 Jun 2016 13:41:12 +0800
Subject: [PATCH 5140/6219] [wdr] Simplify extraction

---
 youtube_dl/extractor/wdr.py | 64 ++++++++++++++++++++-----------------
 1 file changed, 34 insertions(+), 30 deletions(-)

diff --git a/youtube_dl/extractor/wdr.py b/youtube_dl/extractor/wdr.py
index 059e2aa08..88369d3f2 100644
--- a/youtube_dl/extractor/wdr.py
+++ b/youtube_dl/extractor/wdr.py
@@ -10,6 +10,7 @@ from ..utils import (
     strip_jsonp,
     unified_strdate,
     ExtractorError,
+    update_url_query,
     urlhandle_detect_ext,
 )
 
@@ -100,9 +101,10 @@ class WDRIE(InfoExtractor):
         },
         {
             'url': 'http://www1.wdr.de/radio/player/radioplayer116~_layout-popupVersion.html',
+            # Live stream, MD5 unstable
             'info_dict': {
                 'id': 'mdb-869971',
-                'ext': 'mp3',
+                'ext': 'flv',
                 'title': 'Funkhaus Europa Livestream',
                 'description': 'md5:2309992a6716c347891c045be50992e4',
                 'upload_date': '20160101',
@@ -150,36 +152,38 @@ class WDRIE(InfoExtractor):
         formats = []
 
         # check if the metadata contains a direct URL to a file
-        metadata_media_alt = metadata_media_resource.get('alt')
-        if metadata_media_alt:
-            for tag_name in ['videoURL', 'audioURL']:
-                if tag_name in metadata_media_alt:
-                    alt_url = metadata_media_alt[tag_name]
-                    ext = determine_ext(alt_url)
-                    if ext == 'm3u8':
-                        m3u_fmt = self._extract_m3u8_formats(
-                            alt_url, display_id, 'mp4', 'm3u8_native',
-                            m3u8_id='hls')
-                        formats.extend(m3u_fmt)
-                    else:
-                        a_format = {
-                            'url': alt_url
-                        }
-                        if ext == 'unknown_video':
-                            urlh = self._request_webpage(
-                                alt_url, display_id, note='Determining extension')
-                            ext = urlhandle_detect_ext(urlh)
-                            a_format['ext'] = ext
-                        formats.append(a_format)
+        for kind, media_resource in metadata_media_resource.items():
+            if kind not in ('dflt', 'alt'):
+                continue
 
-        # check if there are flash-streams for this video
-        if 'dflt' in metadata_media_resource and 'videoURL' in metadata_media_resource['dflt']:
-            video_url = metadata_media_resource['dflt']['videoURL']
-            if video_url.endswith('.f4m'):
-                full_video_url = video_url + '?hdcore=3.2.0&plugin=aasp-3.2.0.77.18'
-                formats.extend(self._extract_f4m_formats(full_video_url, display_id, f4m_id='hds', fatal=False))
-            elif video_url.endswith('.smil'):
-                formats.extend(self._extract_smil_formats(video_url, 'stream', fatal=False))
+            for tag_name, medium_url in media_resource.items():
+                if tag_name not in ('videoURL', 'audioURL'):
+                    continue
+
+                ext = determine_ext(medium_url)
+                if ext == 'm3u8':
+                    m3u_fmt = self._extract_m3u8_formats(
+                        medium_url, display_id, 'mp4', 'm3u8_native',
+                        m3u8_id='hls')
+                    formats.extend(m3u_fmt)
+                elif ext == 'f4m':
+                    manifest_url = update_url_query(
+                        medium_url, {'hdcore': '3.2.0', 'plugin': 'aasp-3.2.0.77.18'})
+                    formats.extend(self._extract_f4m_formats(
+                        manifest_url, display_id, f4m_id='hds', fatal=False))
+                elif ext == 'smil':
+                    formats.extend(self._extract_smil_formats(
+                        medium_url, 'stream', fatal=False))
+                else:
+                    a_format = {
+                        'url': medium_url
+                    }
+                    if ext == 'unknown_video':
+                        urlh = self._request_webpage(
+                            medium_url, display_id, note='Determining extension')
+                        ext = urlhandle_detect_ext(urlh)
+                        a_format['ext'] = ext
+                    formats.append(a_format)
 
         subtitles = {}
         caption_url = metadata_media_resource.get('captionURL')

From 1594a4932f7e94287c32b5d4d63a60b57ffee96a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 9 Jun 2016 13:49:35 +0800
Subject: [PATCH 5141/6219] [wdr] Misc changes

---
 youtube_dl/extractor/wdr.py | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/wdr.py b/youtube_dl/extractor/wdr.py
index 88369d3f2..a9238cbeb 100644
--- a/youtube_dl/extractor/wdr.py
+++ b/youtube_dl/extractor/wdr.py
@@ -6,10 +6,10 @@ import re
 from .common import InfoExtractor
 from ..utils import (
     determine_ext,
+    ExtractorError,
     js_to_json,
     strip_jsonp,
     unified_strdate,
-    ExtractorError,
     update_url_query,
     urlhandle_detect_ext,
 )
@@ -17,7 +17,7 @@ from ..utils import (
 
 class WDRIE(InfoExtractor):
     _CURRENT_MAUS_URL = r'https?://(?:www\.)wdrmaus.de/(?:[^/]+/){1,2}[^/?#]+\.php5'
-    _PAGE_REGEX = r'/(?:mediathek/)?(?P<media_type>[^/]+)/(?P<type>[^/]+)/(?P<display_id>.+)\.html'
+    _PAGE_REGEX = r'/(?:mediathek/)?[^/]+/(?P<type>[^/]+)/(?P<display_id>.+)\.html'
     _VALID_URL = r'(?P<page_url>https?://(?:www\d\.)?wdr\d?\.de)' + _PAGE_REGEX + '|' + _CURRENT_MAUS_URL
 
     _TESTS = [
@@ -162,10 +162,9 @@ class WDRIE(InfoExtractor):
 
                 ext = determine_ext(medium_url)
                 if ext == 'm3u8':
-                    m3u_fmt = self._extract_m3u8_formats(
+                    formats.extend(self._extract_m3u8_formats(
                         medium_url, display_id, 'mp4', 'm3u8_native',
-                        m3u8_id='hls')
-                    formats.extend(m3u_fmt)
+                        m3u8_id='hls'))
                 elif ext == 'f4m':
                     manifest_url = update_url_query(
                         medium_url, {'hdcore': '3.2.0', 'plugin': 'aasp-3.2.0.77.18'})
@@ -185,6 +184,8 @@ class WDRIE(InfoExtractor):
                         a_format['ext'] = ext
                     formats.append(a_format)
 
+        self._sort_formats(formats)
+
         subtitles = {}
         caption_url = metadata_media_resource.get('captionURL')
         if caption_url:
@@ -206,8 +207,6 @@ class WDRIE(InfoExtractor):
         if upload_date:
             upload_date = unified_strdate(upload_date)
 
-        self._sort_formats(formats)
-
         return {
             'id': metadata_tracker_data.get('trackerClipId', display_id),
             'display_id': display_id,

From e2713d32f49f1bfa830cc755a96691c39da88290 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 9 Jun 2016 19:00:13 +0800
Subject: [PATCH 5142/6219] [openload] Fix extraction. Thanks @perron375 for
 the solution

Closes #9706
---
 youtube_dl/extractor/openload.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py
index 5049b870e..1b57462b5 100644
--- a/youtube_dl/extractor/openload.py
+++ b/youtube_dl/extractor/openload.py
@@ -100,7 +100,7 @@ class OpenloadIE(InfoExtractor):
             raise ExtractorError('File not found', expected=True)
 
         code = self._search_regex(
-            r'</video>\s*</div>\s*<script[^>]+>([^<]+)</script>',
+            r'</video>\s*</div>\s*<script[^>]+>[^>]+</script>\s*<script[^>]+>([^<]+)</script>',
             webpage, 'JS code')
 
         decoded = self.openload_decode(code)

From 21efee5f8bc8daf0cbb5fc3408a1fc5b9d5eadcb Mon Sep 17 00:00:00 2001
From: N1k145 <N1k145@users.noreply.github.com>
Date: Thu, 9 Jun 2016 12:13:15 +0200
Subject: [PATCH 5143/6219] [openload] Relax _VALID_URL

[openload] added to _TESTS, removed escape
---
 youtube_dl/extractor/openload.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py
index 1b57462b5..6415b8fdc 100644
--- a/youtube_dl/extractor/openload.py
+++ b/youtube_dl/extractor/openload.py
@@ -14,7 +14,7 @@ from ..utils import (
 
 
 class OpenloadIE(InfoExtractor):
-    _VALID_URL = r'https://openload.(?:co|io)/(?:f|embed)/(?P<id>[a-zA-Z0-9-]+)'
+    _VALID_URL = r'https://openload.(?:co|io)/(?:f|embed)/(?P<id>[a-zA-Z0-9-_]+)'
 
     _TESTS = [{
         'url': 'https://openload.co/f/kUEfGclsU9o',
@@ -31,6 +31,9 @@ class OpenloadIE(InfoExtractor):
     }, {
         'url': 'https://openload.io/f/ZAn6oz-VZGE/',
         'only_matching': True,
+    }, {
+        'url': 'https://openload.co/f/_-ztPaZtMhM/',
+        'only_matching': True,
     }, {
         # unavailable via https://openload.co/f/Sxz5sADo82g/, different layout
         # for title and ext

From bb1e44cc8ee7937422fb5635f3431feb6d5fd918 Mon Sep 17 00:00:00 2001
From: TRox1972 <TRox1972@users.noreply.github.com>
Date: Fri, 27 May 2016 13:37:40 +0200
Subject: [PATCH 5144/6219] [godtv] Add extractor

[GodTV] Improvements
---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/godtv.py      | 29 +++++++++++++++++++++++++++++
 2 files changed, 30 insertions(+)
 create mode 100644 youtube_dl/extractor/godtv.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index aa98782a5..40dcfcde3 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -292,6 +292,7 @@ from .globo import (
     GloboArticleIE,
 )
 from .godtube import GodTubeIE
+from .godtv import GodTVIE
 from .goldenmoustache import GoldenMoustacheIE
 from .golem import GolemIE
 from .googledrive import GoogleDriveIE
diff --git a/youtube_dl/extractor/godtv.py b/youtube_dl/extractor/godtv.py
new file mode 100644
index 000000000..50f093ace
--- /dev/null
+++ b/youtube_dl/extractor/godtv.py
@@ -0,0 +1,29 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from .ooyala import OoyalaIE
+
+
+class GodTVIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?god\.tv(?:/[^/]+)+/(?P<id>[^/?#&]+)'
+    _TEST = {
+        'url': 'http://god.tv/jesus-image/video/jesus-conference-2016/randy-needham',
+        'info_dict': {
+            'id': 'lpd3g2MzE6D1g8zFAKz8AGpxWcpu6o_3',
+            'ext': 'mp4',
+            'title': 'Randy Needham',
+            'duration': 3615.08,
+        },
+        'params': {
+            'skip_download': True,
+        }
+    }
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, display_id)
+        ooyala_id = self._search_regex(r'"content_id"\s*:\s*"([\w-]{32})"', webpage, display_id)
+
+        return OoyalaIE._build_url_result(ooyala_id)

From c0fed3bda50f77d063f3817cfbc3d8b81c18afa6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 9 Jun 2016 21:29:41 +0700
Subject: [PATCH 5145/6219] [godtv] Improve and add support for playlists
 (Closes #9608)

---
 youtube_dl/extractor/godtv.py | 36 +++++++++++++++++++++++++++++++----
 1 file changed, 32 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/godtv.py b/youtube_dl/extractor/godtv.py
index 50f093ace..78d638cf0 100644
--- a/youtube_dl/extractor/godtv.py
+++ b/youtube_dl/extractor/godtv.py
@@ -1,13 +1,13 @@
-# coding: utf-8
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from .ooyala import OoyalaIE
+from ..utils import js_to_json
 
 
 class GodTVIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?god\.tv(?:/[^/]+)+/(?P<id>[^/?#&]+)'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://god.tv/jesus-image/video/jesus-conference-2016/randy-needham',
         'info_dict': {
             'id': 'lpd3g2MzE6D1g8zFAKz8AGpxWcpu6o_3',
@@ -18,12 +18,40 @@ class GodTVIE(InfoExtractor):
         'params': {
             'skip_download': True,
         }
-    }
+    }, {
+        'url': 'http://god.tv/playlist/bible-study',
+        'info_dict': {
+            'id': 'bible-study',
+        },
+        'playlist_mincount': 37,
+    }]
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
 
         webpage = self._download_webpage(url, display_id)
-        ooyala_id = self._search_regex(r'"content_id"\s*:\s*"([\w-]{32})"', webpage, display_id)
+
+        settings = self._parse_json(
+            self._search_regex(
+                r'jQuery\.extend\(Drupal\.settings\s*,\s*({.+?})\);',
+                webpage, 'settings', default='{}'),
+            display_id, transform_source=js_to_json, fatal=False)
+
+        ooyala_id = None
+
+        if settings:
+            playlist = settings.get('playlist')
+            if playlist and isinstance(playlist, list):
+                entries = [
+                    OoyalaIE._build_url_result(video['content_id'])
+                    for video in playlist if video.get('content_id')]
+                if entries:
+                    return self.playlist_result(entries, display_id)
+            ooyala_id = settings.get('ooyala', {}).get('content_id')
+
+        if not ooyala_id:
+            ooyala_id = self._search_regex(
+                r'["\']content_id["\']\s*:\s*(["\'])(?P<id>[\w-]+)\1',
+                webpage, 'ooyala id', group='id')
 
         return OoyalaIE._build_url_result(ooyala_id)

From 416878f41f3b33cf1b10b0b30093dcd7a90bdbfd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 9 Jun 2016 21:33:51 +0700
Subject: [PATCH 5146/6219] [godtv] Add more tests

---
 youtube_dl/extractor/godtv.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/godtv.py b/youtube_dl/extractor/godtv.py
index 78d638cf0..7deca00aa 100644
--- a/youtube_dl/extractor/godtv.py
+++ b/youtube_dl/extractor/godtv.py
@@ -24,6 +24,12 @@ class GodTVIE(InfoExtractor):
             'id': 'bible-study',
         },
         'playlist_mincount': 37,
+    }, {
+        'url': 'http://god.tv/node/15097',
+        'only_matching': True,
+    }, {
+        'url': 'http://god.tv/live/africa',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From b0aebe702c538010fd92cd0807963293f112adcd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 9 Jun 2016 21:34:47 +0700
Subject: [PATCH 5147/6219] [godtv] Relax _VALID_URL

---
 youtube_dl/extractor/godtv.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/godtv.py b/youtube_dl/extractor/godtv.py
index 7deca00aa..c5d3b4e6a 100644
--- a/youtube_dl/extractor/godtv.py
+++ b/youtube_dl/extractor/godtv.py
@@ -6,7 +6,7 @@ from ..utils import js_to_json
 
 
 class GodTVIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?god\.tv(?:/[^/]+)+/(?P<id>[^/?#&]+)'
+    _VALID_URL = r'https?://(?:www\.)?god\.tv(?:/[^/]+)*/(?P<id>[^/?#&]+)'
     _TESTS = [{
         'url': 'http://god.tv/jesus-image/video/jesus-conference-2016/randy-needham',
         'info_dict': {
@@ -30,6 +30,9 @@ class GodTVIE(InfoExtractor):
     }, {
         'url': 'http://god.tv/live/africa',
         'only_matching': True,
+    }, {
+        'url': 'http://god.tv/liveevents',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From bc7e7adf5154f15b74b2df3e2989f630667778ce Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 9 Jun 2016 21:40:16 +0800
Subject: [PATCH 5148/6219] [wdr] Subtitles are TTML

---
 youtube_dl/extractor/wdr.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/wdr.py b/youtube_dl/extractor/wdr.py
index a9238cbeb..6b83a2a04 100644
--- a/youtube_dl/extractor/wdr.py
+++ b/youtube_dl/extractor/wdr.py
@@ -34,7 +34,8 @@ class WDRIE(InfoExtractor):
                 'description': 'md5:87be8ff14d8dfd7a7ee46f0299b52318',
                 'is_live': False,
                 'subtitles': {'de': [{
-                    'url': 'http://ondemand-ww.wdr.de/medp/fsk0/105/1058683/1058683_12220974.xml'
+                    'url': 'http://ondemand-ww.wdr.de/medp/fsk0/105/1058683/1058683_12220974.xml',
+                    'ext': 'ttml',
                 }]},
             },
         },
@@ -190,7 +191,8 @@ class WDRIE(InfoExtractor):
         caption_url = metadata_media_resource.get('captionURL')
         if caption_url:
             subtitles['de'] = [{
-                'url': caption_url
+                'url': caption_url,
+                'ext': 'ttml',
             }]
 
         title = metadata_tracker_data.get('trackerClipTitle')

From 55290788d352168844c8e64d64428a76baa63eea Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 10 Jun 2016 12:28:09 +0800
Subject: [PATCH 5149/6219] [yahoo] Yahoo doesn't like region names in lower
 cases

Fix test_Yahoo_7
---
 youtube_dl/extractor/yahoo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/yahoo.py b/youtube_dl/extractor/yahoo.py
index b376f2b93..927a964a4 100644
--- a/youtube_dl/extractor/yahoo.py
+++ b/youtube_dl/extractor/yahoo.py
@@ -343,7 +343,7 @@ class YahooIE(InfoExtractor):
             webpage, 'region', fatal=False, default='US')
         data = compat_urllib_parse_urlencode({
             'protocol': 'http',
-            'region': region,
+            'region': region.upper(),
         })
         query_url = (
             'https://video.media.yql.yahoo.com/v1/video/sapi/streams/'

From 506d0e96936f84c2b21c7ed37f4a7fca2eec86a2 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 10 Jun 2016 12:29:58 +0800
Subject: [PATCH 5150/6219] [xuite] Skip the invalid test

---
 youtube_dl/extractor/xuite.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/xuite.py b/youtube_dl/extractor/xuite.py
index 2466410fa..0be8932ad 100644
--- a/youtube_dl/extractor/xuite.py
+++ b/youtube_dl/extractor/xuite.py
@@ -66,6 +66,7 @@ class XuiteIE(InfoExtractor):
             'uploader_id': '242127761',
             'categories': ['電玩動漫'],
         },
+        'skip': 'Video removed',
     }, {
         'url': 'http://vlog.xuite.net/play/S1dDUjdyLTMyOTc3NjcuZmx2/%E5%AD%AB%E7%87%95%E5%A7%BF-%E7%9C%BC%E6%B7%9A%E6%88%90%E8%A9%A9',
         'only_matching': True,

From 436214baf70c1a50fbaf1fbfca4b48f33695590c Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 10 Jun 2016 12:31:06 +0800
Subject: [PATCH 5151/6219] [xfileshare] Skip an invalid test

---
 youtube_dl/extractor/xfileshare.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/xfileshare.py b/youtube_dl/extractor/xfileshare.py
index 769003735..ee4d04c20 100644
--- a/youtube_dl/extractor/xfileshare.py
+++ b/youtube_dl/extractor/xfileshare.py
@@ -62,7 +62,8 @@ class XFileShareIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'youtube-dl test video \'äBaW_jenozKc.mp4.mp4',
             'thumbnail': 're:http://.*\.jpg',
-        }
+        },
+        'skip': 'Video removed',
     }, {
         'url': 'http://vidto.me/ku5glz52nqe1.html',
         'info_dict': {

From e1e0a10c567e8457bf83f6b54e65963447e17a8f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 10 Jun 2016 12:33:31 +0800
Subject: [PATCH 5152/6219] [weibo] Remove the extractor
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The Weibo weishipin (微視頻, tiny videos) service is dead and now all
videos are hosted on Sina videos, which is covered by sina.py
---
 youtube_dl/extractor/extractors.py |  1 -
 youtube_dl/extractor/weibo.py      | 49 ------------------------------
 2 files changed, 50 deletions(-)
 delete mode 100644 youtube_dl/extractor/weibo.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 40dcfcde3..0789e4a6e 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -974,7 +974,6 @@ from .webofstories import (
     WebOfStoriesIE,
     WebOfStoriesPlaylistIE,
 )
-from .weibo import WeiboIE
 from .weiqitv import WeiqiTVIE
 from .wimp import WimpIE
 from .wistia import WistiaIE
diff --git a/youtube_dl/extractor/weibo.py b/youtube_dl/extractor/weibo.py
deleted file mode 100644
index 20bb039d3..000000000
--- a/youtube_dl/extractor/weibo.py
+++ /dev/null
@@ -1,49 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-import re
-
-from .common import InfoExtractor
-
-
-class WeiboIE(InfoExtractor):
-    """
-    The videos in Weibo come from different sites, this IE just finds the link
-    to the external video and returns it.
-    """
-    _VALID_URL = r'https?://video\.weibo\.com/v/weishipin/t_(?P<id>.+?)\.htm'
-
-    _TEST = {
-        'url': 'http://video.weibo.com/v/weishipin/t_zjUw2kZ.htm',
-        'info_dict': {
-            'id': '98322879',
-            'ext': 'flv',
-            'title': '魔声耳机最新广告“All Eyes On Us”',
-        },
-        'params': {
-            'skip_download': True,
-        },
-        'add_ie': ['Sina'],
-    }
-
-    # Additional example videos from different sites
-    # Youku: http://video.weibo.com/v/weishipin/t_zQGDWQ8.htm
-    # 56.com: http://video.weibo.com/v/weishipin/t_zQ44HxN.htm
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url, flags=re.VERBOSE)
-        video_id = mobj.group('id')
-        info_url = 'http://video.weibo.com/?s=v&a=play_list&format=json&mix_video_id=t_%s' % video_id
-        info = self._download_json(info_url, video_id)
-
-        videos_urls = map(lambda v: v['play_page_url'], info['result']['data'])
-        # Prefer sina video since they have thumbnails
-        videos_urls = sorted(videos_urls, key=lambda u: 'video.sina.com' in u)
-        player_url = videos_urls[-1]
-        m_sina = re.match(r'https?://video\.sina\.com\.cn/v/b/(\d+)-\d+\.html',
-                          player_url)
-        if m_sina is not None:
-            self.to_screen('Sina video detected')
-            sina_id = m_sina.group(1)
-            player_url = 'http://you.video.sina.com.cn/swf/quotePlayer.swf?vid=%s' % sina_id
-        return self.url_result(player_url)

From 3e74b444e7324fdda956aa816240b938eabf9c93 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 10 Jun 2016 13:13:59 +0800
Subject: [PATCH 5153/6219] [vulture] Remove the extractor

The first 10 URLs in google search "site:http://video.vulture.com/video"
is dead. I guess Vulture does not host videos on their own anymore.
---
 youtube_dl/extractor/extractors.py |  1 -
 youtube_dl/extractor/vulture.py    | 69 ------------------------------
 2 files changed, 70 deletions(-)
 delete mode 100644 youtube_dl/extractor/vulture.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 0789e4a6e..38708294a 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -958,7 +958,6 @@ from .vporn import VpornIE
 from .vrt import VRTIE
 from .vube import VubeIE
 from .vuclip import VuClipIE
-from .vulture import VultureIE
 from .walla import WallaIE
 from .washingtonpost import (
     WashingtonPostIE,
diff --git a/youtube_dl/extractor/vulture.py b/youtube_dl/extractor/vulture.py
deleted file mode 100644
index faa167e65..000000000
--- a/youtube_dl/extractor/vulture.py
+++ /dev/null
@@ -1,69 +0,0 @@
-from __future__ import unicode_literals
-
-import json
-import os.path
-import re
-
-from .common import InfoExtractor
-from ..utils import (
-    int_or_none,
-    parse_iso8601,
-)
-
-
-class VultureIE(InfoExtractor):
-    IE_NAME = 'vulture.com'
-    _VALID_URL = r'https?://video\.vulture\.com/video/(?P<display_id>[^/]+)/'
-    _TEST = {
-        'url': 'http://video.vulture.com/video/Mindy-Kaling-s-Harvard-Speech/player?layout=compact&read_more=1',
-        'md5': '8d997845642a2b5152820f7257871bc8',
-        'info_dict': {
-            'id': '6GHRQL3RV7MSD1H4',
-            'ext': 'mp4',
-            'title': 'kaling-speech-2-MAGNIFY STANDARD CONTAINER REVISED',
-            'uploader_id': 'Sarah',
-            'thumbnail': 're:^http://.*\.jpg$',
-            'timestamp': 1401288564,
-            'upload_date': '20140528',
-            'description': 'Uplifting and witty, as predicted.',
-            'duration': 1015,
-        }
-    }
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        display_id = mobj.group('display_id')
-
-        webpage = self._download_webpage(url, display_id)
-        query_string = self._search_regex(
-            r"queryString\s*=\s*'([^']+)'", webpage, 'query string')
-        video_id = self._search_regex(
-            r'content=([^&]+)', query_string, 'video ID')
-        query_url = 'http://video.vulture.com/embed/player/container/1000/1000/?%s' % query_string
-
-        query_webpage = self._download_webpage(
-            query_url, display_id, note='Downloading query page')
-        params_json = self._search_regex(
-            r'(?sm)new MagnifyEmbeddablePlayer\({.*?contentItem:\s*(\{.*?\})\n?,\n',
-            query_webpage,
-            'player params')
-        params = json.loads(params_json)
-
-        upload_timestamp = parse_iso8601(params['posted'].replace(' ', 'T'))
-        uploader_id = params.get('user', {}).get('handle')
-
-        media_item = params['media_item']
-        title = os.path.splitext(media_item['title'])[0]
-        duration = int_or_none(media_item.get('duration_seconds'))
-
-        return {
-            'id': video_id,
-            'display_id': display_id,
-            'url': media_item['pipeline_xid'],
-            'title': title,
-            'timestamp': upload_timestamp,
-            'thumbnail': params.get('thumbnail_url'),
-            'uploader_id': uploader_id,
-            'description': params.get('description'),
-            'duration': duration,
-        }

From 5de008e8c3e4058c20956d19f69ac3347a2722e0 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 10 Jun 2016 13:31:55 +0800
Subject: [PATCH 5154/6219] [nbcnews] Support embed widgets

Used in some Vulture videos
---
 youtube_dl/extractor/nbc.py | 22 +++++++++++++---------
 1 file changed, 13 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index f27c7f139..6b7da1149 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -266,6 +266,11 @@ class NBCNewsIE(ThePlatformIE):
             'url': 'http://www.nbcnews.com/watch/dateline/full-episode--deadly-betrayal-386250819952',
             'only_matching': True,
         },
+        {
+            # From http://www.vulture.com/2016/06/letterman-couldnt-care-less-about-late-night.html
+            'url': 'http://www.nbcnews.com/widget/video-embed/701714499682',
+            'only_matching': True,
+        },
     ]
 
     def _real_extract(self, url):
@@ -289,18 +294,17 @@ class NBCNewsIE(ThePlatformIE):
             webpage = self._download_webpage(url, display_id)
             info = None
             bootstrap_json = self._search_regex(
-                r'(?m)var\s+(?:bootstrapJson|playlistData)\s*=\s*({.+});?\s*$',
+                [r'(?m)(?:var\s+(?:bootstrapJson|playlistData)|NEWS\.videoObj)\s*=\s*({.+});?\s*$',
+                 r'videoObj\s*:\s*({.+})', r'data-video="([^"]+)"'],
                 webpage, 'bootstrap json', default=None)
-            if bootstrap_json:
-                bootstrap = self._parse_json(bootstrap_json, display_id)
+            bootstrap = self._parse_json(
+                bootstrap_json, display_id, transform_source=unescapeHTML)
+            if 'results' in bootstrap:
                 info = bootstrap['results'][0]['video']
+            elif 'video' in bootstrap:
+                info = bootstrap['video']
             else:
-                player_instance_json = self._search_regex(
-                    r'videoObj\s*:\s*({.+})', webpage, 'player instance', default=None)
-                if not player_instance_json:
-                    player_instance_json = self._html_search_regex(
-                        r'data-video="([^"]+)"', webpage, 'video json')
-                info = self._parse_json(player_instance_json, display_id)
+                info = bootstrap
             video_id = info['mpxId']
             title = info['title']
 

From de3eb07ed64e3d50164a6db59385a94f2675b0b4 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 10 Jun 2016 13:32:59 +0800
Subject: [PATCH 5155/6219] [generic] Detect NBC News embeds

---
 youtube_dl/extractor/generic.py | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 90575ab0e..36a3d91fc 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1032,6 +1032,17 @@ class GenericIE(InfoExtractor):
                 'timestamp': 1389118457,
             },
         },
+        # NBC News embed
+        {
+            'url': 'http://www.vulture.com/2016/06/letterman-couldnt-care-less-about-late-night.html',
+            'md5': '1aa589c675898ae6d37a17913cf68d66',
+            'info_dict': {
+                'id': '701714499682',
+                'ext': 'mp4',
+                'title': 'PREVIEW: On Assignment: David Letterman',
+                'description': 'A preview of Tom Brokaw\'s interview with David Letterman as part of the On Assignment series powered by Dateline. Airs Sunday June 12 at 7/6c.',
+            },
+        },
         # UDN embed
         {
             'url': 'https://video.udn.com/news/300346',
@@ -1966,6 +1977,12 @@ class GenericIE(InfoExtractor):
         if nbc_sports_url:
             return self.url_result(nbc_sports_url, 'NBCSportsVPlayer')
 
+        # Look for NBC News embeds
+        nbc_news_embed_url = re.search(
+            r'<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//www\.nbcnews\.com/widget/video-embed/[^"\']+)\1', webpage)
+        if nbc_news_embed_url:
+            return self.url_result(nbc_news_embed_url.group('url'), 'NBCNews')
+
         # Look for Google Drive embeds
         google_drive_url = GoogleDriveIE._extract_url(webpage)
         if google_drive_url:

From cc4444662c54c24f6f82efd3ba5e60e9556d88b8 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 10 Jun 2016 13:33:59 +0800
Subject: [PATCH 5156/6219] [generic] Remove Vulture embed detection

Vulture.com videos now hosts on YouTube, Vimeo, MTV, NBC News or Hulu.
Here's an example of Hulu:
http://www.vulture.com/2016/06/kimmel-interviews-mariah-carey-in-a-bathtub.html
---
 youtube_dl/extractor/generic.py | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 36a3d91fc..798c109c6 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1857,14 +1857,6 @@ class GenericIE(InfoExtractor):
             url = unescapeHTML(mobj.group('url'))
             return self.url_result(url)
 
-        # Look for embedded vulture.com player
-        mobj = re.search(
-            r'<iframe src="(?P<url>https?://video\.vulture\.com/[^"]+)"',
-            webpage)
-        if mobj is not None:
-            url = unescapeHTML(mobj.group('url'))
-            return self.url_result(url, ie='Vulture')
-
         # Look for embedded mtvservices player
         mtvservices_url = MTVServicesEmbeddedIE._extract_url(webpage)
         if mtvservices_url:

From 9631a94fb5e5ee9b92135f938df00866535fc6c6 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 10 Jun 2016 15:05:24 +0800
Subject: [PATCH 5157/6219] [compat] Add compat_html_entities_html5

Used in tset_Vporn_1. Also Related to #9270
---
 youtube_dl/compat.py | 2240 +++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 2239 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index e3cab4dd0..0243949a4 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -63,6 +63,2244 @@ try:
 except ImportError:  # Python 2
     import htmlentitydefs as compat_html_entities
 
+try:  # Python >= 3.3
+    from compat_html_entities import html as compat_html_entities_html5
+except ImportError:
+    # Copied from CPython 3.5.1 html/entities.py
+    compat_html_entities_html5 = {
+        'Aacute': '\xc1',
+        'aacute': '\xe1',
+        'Aacute;': '\xc1',
+        'aacute;': '\xe1',
+        'Abreve;': '\u0102',
+        'abreve;': '\u0103',
+        'ac;': '\u223e',
+        'acd;': '\u223f',
+        'acE;': '\u223e\u0333',
+        'Acirc': '\xc2',
+        'acirc': '\xe2',
+        'Acirc;': '\xc2',
+        'acirc;': '\xe2',
+        'acute': '\xb4',
+        'acute;': '\xb4',
+        'Acy;': '\u0410',
+        'acy;': '\u0430',
+        'AElig': '\xc6',
+        'aelig': '\xe6',
+        'AElig;': '\xc6',
+        'aelig;': '\xe6',
+        'af;': '\u2061',
+        'Afr;': '\U0001d504',
+        'afr;': '\U0001d51e',
+        'Agrave': '\xc0',
+        'agrave': '\xe0',
+        'Agrave;': '\xc0',
+        'agrave;': '\xe0',
+        'alefsym;': '\u2135',
+        'aleph;': '\u2135',
+        'Alpha;': '\u0391',
+        'alpha;': '\u03b1',
+        'Amacr;': '\u0100',
+        'amacr;': '\u0101',
+        'amalg;': '\u2a3f',
+        'AMP': '&',
+        'amp': '&',
+        'AMP;': '&',
+        'amp;': '&',
+        'And;': '\u2a53',
+        'and;': '\u2227',
+        'andand;': '\u2a55',
+        'andd;': '\u2a5c',
+        'andslope;': '\u2a58',
+        'andv;': '\u2a5a',
+        'ang;': '\u2220',
+        'ange;': '\u29a4',
+        'angle;': '\u2220',
+        'angmsd;': '\u2221',
+        'angmsdaa;': '\u29a8',
+        'angmsdab;': '\u29a9',
+        'angmsdac;': '\u29aa',
+        'angmsdad;': '\u29ab',
+        'angmsdae;': '\u29ac',
+        'angmsdaf;': '\u29ad',
+        'angmsdag;': '\u29ae',
+        'angmsdah;': '\u29af',
+        'angrt;': '\u221f',
+        'angrtvb;': '\u22be',
+        'angrtvbd;': '\u299d',
+        'angsph;': '\u2222',
+        'angst;': '\xc5',
+        'angzarr;': '\u237c',
+        'Aogon;': '\u0104',
+        'aogon;': '\u0105',
+        'Aopf;': '\U0001d538',
+        'aopf;': '\U0001d552',
+        'ap;': '\u2248',
+        'apacir;': '\u2a6f',
+        'apE;': '\u2a70',
+        'ape;': '\u224a',
+        'apid;': '\u224b',
+        'apos;': "'",
+        'ApplyFunction;': '\u2061',
+        'approx;': '\u2248',
+        'approxeq;': '\u224a',
+        'Aring': '\xc5',
+        'aring': '\xe5',
+        'Aring;': '\xc5',
+        'aring;': '\xe5',
+        'Ascr;': '\U0001d49c',
+        'ascr;': '\U0001d4b6',
+        'Assign;': '\u2254',
+        'ast;': '*',
+        'asymp;': '\u2248',
+        'asympeq;': '\u224d',
+        'Atilde': '\xc3',
+        'atilde': '\xe3',
+        'Atilde;': '\xc3',
+        'atilde;': '\xe3',
+        'Auml': '\xc4',
+        'auml': '\xe4',
+        'Auml;': '\xc4',
+        'auml;': '\xe4',
+        'awconint;': '\u2233',
+        'awint;': '\u2a11',
+        'backcong;': '\u224c',
+        'backepsilon;': '\u03f6',
+        'backprime;': '\u2035',
+        'backsim;': '\u223d',
+        'backsimeq;': '\u22cd',
+        'Backslash;': '\u2216',
+        'Barv;': '\u2ae7',
+        'barvee;': '\u22bd',
+        'Barwed;': '\u2306',
+        'barwed;': '\u2305',
+        'barwedge;': '\u2305',
+        'bbrk;': '\u23b5',
+        'bbrktbrk;': '\u23b6',
+        'bcong;': '\u224c',
+        'Bcy;': '\u0411',
+        'bcy;': '\u0431',
+        'bdquo;': '\u201e',
+        'becaus;': '\u2235',
+        'Because;': '\u2235',
+        'because;': '\u2235',
+        'bemptyv;': '\u29b0',
+        'bepsi;': '\u03f6',
+        'bernou;': '\u212c',
+        'Bernoullis;': '\u212c',
+        'Beta;': '\u0392',
+        'beta;': '\u03b2',
+        'beth;': '\u2136',
+        'between;': '\u226c',
+        'Bfr;': '\U0001d505',
+        'bfr;': '\U0001d51f',
+        'bigcap;': '\u22c2',
+        'bigcirc;': '\u25ef',
+        'bigcup;': '\u22c3',
+        'bigodot;': '\u2a00',
+        'bigoplus;': '\u2a01',
+        'bigotimes;': '\u2a02',
+        'bigsqcup;': '\u2a06',
+        'bigstar;': '\u2605',
+        'bigtriangledown;': '\u25bd',
+        'bigtriangleup;': '\u25b3',
+        'biguplus;': '\u2a04',
+        'bigvee;': '\u22c1',
+        'bigwedge;': '\u22c0',
+        'bkarow;': '\u290d',
+        'blacklozenge;': '\u29eb',
+        'blacksquare;': '\u25aa',
+        'blacktriangle;': '\u25b4',
+        'blacktriangledown;': '\u25be',
+        'blacktriangleleft;': '\u25c2',
+        'blacktriangleright;': '\u25b8',
+        'blank;': '\u2423',
+        'blk12;': '\u2592',
+        'blk14;': '\u2591',
+        'blk34;': '\u2593',
+        'block;': '\u2588',
+        'bne;': '=\u20e5',
+        'bnequiv;': '\u2261\u20e5',
+        'bNot;': '\u2aed',
+        'bnot;': '\u2310',
+        'Bopf;': '\U0001d539',
+        'bopf;': '\U0001d553',
+        'bot;': '\u22a5',
+        'bottom;': '\u22a5',
+        'bowtie;': '\u22c8',
+        'boxbox;': '\u29c9',
+        'boxDL;': '\u2557',
+        'boxDl;': '\u2556',
+        'boxdL;': '\u2555',
+        'boxdl;': '\u2510',
+        'boxDR;': '\u2554',
+        'boxDr;': '\u2553',
+        'boxdR;': '\u2552',
+        'boxdr;': '\u250c',
+        'boxH;': '\u2550',
+        'boxh;': '\u2500',
+        'boxHD;': '\u2566',
+        'boxHd;': '\u2564',
+        'boxhD;': '\u2565',
+        'boxhd;': '\u252c',
+        'boxHU;': '\u2569',
+        'boxHu;': '\u2567',
+        'boxhU;': '\u2568',
+        'boxhu;': '\u2534',
+        'boxminus;': '\u229f',
+        'boxplus;': '\u229e',
+        'boxtimes;': '\u22a0',
+        'boxUL;': '\u255d',
+        'boxUl;': '\u255c',
+        'boxuL;': '\u255b',
+        'boxul;': '\u2518',
+        'boxUR;': '\u255a',
+        'boxUr;': '\u2559',
+        'boxuR;': '\u2558',
+        'boxur;': '\u2514',
+        'boxV;': '\u2551',
+        'boxv;': '\u2502',
+        'boxVH;': '\u256c',
+        'boxVh;': '\u256b',
+        'boxvH;': '\u256a',
+        'boxvh;': '\u253c',
+        'boxVL;': '\u2563',
+        'boxVl;': '\u2562',
+        'boxvL;': '\u2561',
+        'boxvl;': '\u2524',
+        'boxVR;': '\u2560',
+        'boxVr;': '\u255f',
+        'boxvR;': '\u255e',
+        'boxvr;': '\u251c',
+        'bprime;': '\u2035',
+        'Breve;': '\u02d8',
+        'breve;': '\u02d8',
+        'brvbar': '\xa6',
+        'brvbar;': '\xa6',
+        'Bscr;': '\u212c',
+        'bscr;': '\U0001d4b7',
+        'bsemi;': '\u204f',
+        'bsim;': '\u223d',
+        'bsime;': '\u22cd',
+        'bsol;': '\\',
+        'bsolb;': '\u29c5',
+        'bsolhsub;': '\u27c8',
+        'bull;': '\u2022',
+        'bullet;': '\u2022',
+        'bump;': '\u224e',
+        'bumpE;': '\u2aae',
+        'bumpe;': '\u224f',
+        'Bumpeq;': '\u224e',
+        'bumpeq;': '\u224f',
+        'Cacute;': '\u0106',
+        'cacute;': '\u0107',
+        'Cap;': '\u22d2',
+        'cap;': '\u2229',
+        'capand;': '\u2a44',
+        'capbrcup;': '\u2a49',
+        'capcap;': '\u2a4b',
+        'capcup;': '\u2a47',
+        'capdot;': '\u2a40',
+        'CapitalDifferentialD;': '\u2145',
+        'caps;': '\u2229\ufe00',
+        'caret;': '\u2041',
+        'caron;': '\u02c7',
+        'Cayleys;': '\u212d',
+        'ccaps;': '\u2a4d',
+        'Ccaron;': '\u010c',
+        'ccaron;': '\u010d',
+        'Ccedil': '\xc7',
+        'ccedil': '\xe7',
+        'Ccedil;': '\xc7',
+        'ccedil;': '\xe7',
+        'Ccirc;': '\u0108',
+        'ccirc;': '\u0109',
+        'Cconint;': '\u2230',
+        'ccups;': '\u2a4c',
+        'ccupssm;': '\u2a50',
+        'Cdot;': '\u010a',
+        'cdot;': '\u010b',
+        'cedil': '\xb8',
+        'cedil;': '\xb8',
+        'Cedilla;': '\xb8',
+        'cemptyv;': '\u29b2',
+        'cent': '\xa2',
+        'cent;': '\xa2',
+        'CenterDot;': '\xb7',
+        'centerdot;': '\xb7',
+        'Cfr;': '\u212d',
+        'cfr;': '\U0001d520',
+        'CHcy;': '\u0427',
+        'chcy;': '\u0447',
+        'check;': '\u2713',
+        'checkmark;': '\u2713',
+        'Chi;': '\u03a7',
+        'chi;': '\u03c7',
+        'cir;': '\u25cb',
+        'circ;': '\u02c6',
+        'circeq;': '\u2257',
+        'circlearrowleft;': '\u21ba',
+        'circlearrowright;': '\u21bb',
+        'circledast;': '\u229b',
+        'circledcirc;': '\u229a',
+        'circleddash;': '\u229d',
+        'CircleDot;': '\u2299',
+        'circledR;': '\xae',
+        'circledS;': '\u24c8',
+        'CircleMinus;': '\u2296',
+        'CirclePlus;': '\u2295',
+        'CircleTimes;': '\u2297',
+        'cirE;': '\u29c3',
+        'cire;': '\u2257',
+        'cirfnint;': '\u2a10',
+        'cirmid;': '\u2aef',
+        'cirscir;': '\u29c2',
+        'ClockwiseContourIntegral;': '\u2232',
+        'CloseCurlyDoubleQuote;': '\u201d',
+        'CloseCurlyQuote;': '\u2019',
+        'clubs;': '\u2663',
+        'clubsuit;': '\u2663',
+        'Colon;': '\u2237',
+        'colon;': ':',
+        'Colone;': '\u2a74',
+        'colone;': '\u2254',
+        'coloneq;': '\u2254',
+        'comma;': ',',
+        'commat;': '@',
+        'comp;': '\u2201',
+        'compfn;': '\u2218',
+        'complement;': '\u2201',
+        'complexes;': '\u2102',
+        'cong;': '\u2245',
+        'congdot;': '\u2a6d',
+        'Congruent;': '\u2261',
+        'Conint;': '\u222f',
+        'conint;': '\u222e',
+        'ContourIntegral;': '\u222e',
+        'Copf;': '\u2102',
+        'copf;': '\U0001d554',
+        'coprod;': '\u2210',
+        'Coproduct;': '\u2210',
+        'COPY': '\xa9',
+        'copy': '\xa9',
+        'COPY;': '\xa9',
+        'copy;': '\xa9',
+        'copysr;': '\u2117',
+        'CounterClockwiseContourIntegral;': '\u2233',
+        'crarr;': '\u21b5',
+        'Cross;': '\u2a2f',
+        'cross;': '\u2717',
+        'Cscr;': '\U0001d49e',
+        'cscr;': '\U0001d4b8',
+        'csub;': '\u2acf',
+        'csube;': '\u2ad1',
+        'csup;': '\u2ad0',
+        'csupe;': '\u2ad2',
+        'ctdot;': '\u22ef',
+        'cudarrl;': '\u2938',
+        'cudarrr;': '\u2935',
+        'cuepr;': '\u22de',
+        'cuesc;': '\u22df',
+        'cularr;': '\u21b6',
+        'cularrp;': '\u293d',
+        'Cup;': '\u22d3',
+        'cup;': '\u222a',
+        'cupbrcap;': '\u2a48',
+        'CupCap;': '\u224d',
+        'cupcap;': '\u2a46',
+        'cupcup;': '\u2a4a',
+        'cupdot;': '\u228d',
+        'cupor;': '\u2a45',
+        'cups;': '\u222a\ufe00',
+        'curarr;': '\u21b7',
+        'curarrm;': '\u293c',
+        'curlyeqprec;': '\u22de',
+        'curlyeqsucc;': '\u22df',
+        'curlyvee;': '\u22ce',
+        'curlywedge;': '\u22cf',
+        'curren': '\xa4',
+        'curren;': '\xa4',
+        'curvearrowleft;': '\u21b6',
+        'curvearrowright;': '\u21b7',
+        'cuvee;': '\u22ce',
+        'cuwed;': '\u22cf',
+        'cwconint;': '\u2232',
+        'cwint;': '\u2231',
+        'cylcty;': '\u232d',
+        'Dagger;': '\u2021',
+        'dagger;': '\u2020',
+        'daleth;': '\u2138',
+        'Darr;': '\u21a1',
+        'dArr;': '\u21d3',
+        'darr;': '\u2193',
+        'dash;': '\u2010',
+        'Dashv;': '\u2ae4',
+        'dashv;': '\u22a3',
+        'dbkarow;': '\u290f',
+        'dblac;': '\u02dd',
+        'Dcaron;': '\u010e',
+        'dcaron;': '\u010f',
+        'Dcy;': '\u0414',
+        'dcy;': '\u0434',
+        'DD;': '\u2145',
+        'dd;': '\u2146',
+        'ddagger;': '\u2021',
+        'ddarr;': '\u21ca',
+        'DDotrahd;': '\u2911',
+        'ddotseq;': '\u2a77',
+        'deg': '\xb0',
+        'deg;': '\xb0',
+        'Del;': '\u2207',
+        'Delta;': '\u0394',
+        'delta;': '\u03b4',
+        'demptyv;': '\u29b1',
+        'dfisht;': '\u297f',
+        'Dfr;': '\U0001d507',
+        'dfr;': '\U0001d521',
+        'dHar;': '\u2965',
+        'dharl;': '\u21c3',
+        'dharr;': '\u21c2',
+        'DiacriticalAcute;': '\xb4',
+        'DiacriticalDot;': '\u02d9',
+        'DiacriticalDoubleAcute;': '\u02dd',
+        'DiacriticalGrave;': '`',
+        'DiacriticalTilde;': '\u02dc',
+        'diam;': '\u22c4',
+        'Diamond;': '\u22c4',
+        'diamond;': '\u22c4',
+        'diamondsuit;': '\u2666',
+        'diams;': '\u2666',
+        'die;': '\xa8',
+        'DifferentialD;': '\u2146',
+        'digamma;': '\u03dd',
+        'disin;': '\u22f2',
+        'div;': '\xf7',
+        'divide': '\xf7',
+        'divide;': '\xf7',
+        'divideontimes;': '\u22c7',
+        'divonx;': '\u22c7',
+        'DJcy;': '\u0402',
+        'djcy;': '\u0452',
+        'dlcorn;': '\u231e',
+        'dlcrop;': '\u230d',
+        'dollar;': '$',
+        'Dopf;': '\U0001d53b',
+        'dopf;': '\U0001d555',
+        'Dot;': '\xa8',
+        'dot;': '\u02d9',
+        'DotDot;': '\u20dc',
+        'doteq;': '\u2250',
+        'doteqdot;': '\u2251',
+        'DotEqual;': '\u2250',
+        'dotminus;': '\u2238',
+        'dotplus;': '\u2214',
+        'dotsquare;': '\u22a1',
+        'doublebarwedge;': '\u2306',
+        'DoubleContourIntegral;': '\u222f',
+        'DoubleDot;': '\xa8',
+        'DoubleDownArrow;': '\u21d3',
+        'DoubleLeftArrow;': '\u21d0',
+        'DoubleLeftRightArrow;': '\u21d4',
+        'DoubleLeftTee;': '\u2ae4',
+        'DoubleLongLeftArrow;': '\u27f8',
+        'DoubleLongLeftRightArrow;': '\u27fa',
+        'DoubleLongRightArrow;': '\u27f9',
+        'DoubleRightArrow;': '\u21d2',
+        'DoubleRightTee;': '\u22a8',
+        'DoubleUpArrow;': '\u21d1',
+        'DoubleUpDownArrow;': '\u21d5',
+        'DoubleVerticalBar;': '\u2225',
+        'DownArrow;': '\u2193',
+        'Downarrow;': '\u21d3',
+        'downarrow;': '\u2193',
+        'DownArrowBar;': '\u2913',
+        'DownArrowUpArrow;': '\u21f5',
+        'DownBreve;': '\u0311',
+        'downdownarrows;': '\u21ca',
+        'downharpoonleft;': '\u21c3',
+        'downharpoonright;': '\u21c2',
+        'DownLeftRightVector;': '\u2950',
+        'DownLeftTeeVector;': '\u295e',
+        'DownLeftVector;': '\u21bd',
+        'DownLeftVectorBar;': '\u2956',
+        'DownRightTeeVector;': '\u295f',
+        'DownRightVector;': '\u21c1',
+        'DownRightVectorBar;': '\u2957',
+        'DownTee;': '\u22a4',
+        'DownTeeArrow;': '\u21a7',
+        'drbkarow;': '\u2910',
+        'drcorn;': '\u231f',
+        'drcrop;': '\u230c',
+        'Dscr;': '\U0001d49f',
+        'dscr;': '\U0001d4b9',
+        'DScy;': '\u0405',
+        'dscy;': '\u0455',
+        'dsol;': '\u29f6',
+        'Dstrok;': '\u0110',
+        'dstrok;': '\u0111',
+        'dtdot;': '\u22f1',
+        'dtri;': '\u25bf',
+        'dtrif;': '\u25be',
+        'duarr;': '\u21f5',
+        'duhar;': '\u296f',
+        'dwangle;': '\u29a6',
+        'DZcy;': '\u040f',
+        'dzcy;': '\u045f',
+        'dzigrarr;': '\u27ff',
+        'Eacute': '\xc9',
+        'eacute': '\xe9',
+        'Eacute;': '\xc9',
+        'eacute;': '\xe9',
+        'easter;': '\u2a6e',
+        'Ecaron;': '\u011a',
+        'ecaron;': '\u011b',
+        'ecir;': '\u2256',
+        'Ecirc': '\xca',
+        'ecirc': '\xea',
+        'Ecirc;': '\xca',
+        'ecirc;': '\xea',
+        'ecolon;': '\u2255',
+        'Ecy;': '\u042d',
+        'ecy;': '\u044d',
+        'eDDot;': '\u2a77',
+        'Edot;': '\u0116',
+        'eDot;': '\u2251',
+        'edot;': '\u0117',
+        'ee;': '\u2147',
+        'efDot;': '\u2252',
+        'Efr;': '\U0001d508',
+        'efr;': '\U0001d522',
+        'eg;': '\u2a9a',
+        'Egrave': '\xc8',
+        'egrave': '\xe8',
+        'Egrave;': '\xc8',
+        'egrave;': '\xe8',
+        'egs;': '\u2a96',
+        'egsdot;': '\u2a98',
+        'el;': '\u2a99',
+        'Element;': '\u2208',
+        'elinters;': '\u23e7',
+        'ell;': '\u2113',
+        'els;': '\u2a95',
+        'elsdot;': '\u2a97',
+        'Emacr;': '\u0112',
+        'emacr;': '\u0113',
+        'empty;': '\u2205',
+        'emptyset;': '\u2205',
+        'EmptySmallSquare;': '\u25fb',
+        'emptyv;': '\u2205',
+        'EmptyVerySmallSquare;': '\u25ab',
+        'emsp13;': '\u2004',
+        'emsp14;': '\u2005',
+        'emsp;': '\u2003',
+        'ENG;': '\u014a',
+        'eng;': '\u014b',
+        'ensp;': '\u2002',
+        'Eogon;': '\u0118',
+        'eogon;': '\u0119',
+        'Eopf;': '\U0001d53c',
+        'eopf;': '\U0001d556',
+        'epar;': '\u22d5',
+        'eparsl;': '\u29e3',
+        'eplus;': '\u2a71',
+        'epsi;': '\u03b5',
+        'Epsilon;': '\u0395',
+        'epsilon;': '\u03b5',
+        'epsiv;': '\u03f5',
+        'eqcirc;': '\u2256',
+        'eqcolon;': '\u2255',
+        'eqsim;': '\u2242',
+        'eqslantgtr;': '\u2a96',
+        'eqslantless;': '\u2a95',
+        'Equal;': '\u2a75',
+        'equals;': '=',
+        'EqualTilde;': '\u2242',
+        'equest;': '\u225f',
+        'Equilibrium;': '\u21cc',
+        'equiv;': '\u2261',
+        'equivDD;': '\u2a78',
+        'eqvparsl;': '\u29e5',
+        'erarr;': '\u2971',
+        'erDot;': '\u2253',
+        'Escr;': '\u2130',
+        'escr;': '\u212f',
+        'esdot;': '\u2250',
+        'Esim;': '\u2a73',
+        'esim;': '\u2242',
+        'Eta;': '\u0397',
+        'eta;': '\u03b7',
+        'ETH': '\xd0',
+        'eth': '\xf0',
+        'ETH;': '\xd0',
+        'eth;': '\xf0',
+        'Euml': '\xcb',
+        'euml': '\xeb',
+        'Euml;': '\xcb',
+        'euml;': '\xeb',
+        'euro;': '\u20ac',
+        'excl;': '!',
+        'exist;': '\u2203',
+        'Exists;': '\u2203',
+        'expectation;': '\u2130',
+        'ExponentialE;': '\u2147',
+        'exponentiale;': '\u2147',
+        'fallingdotseq;': '\u2252',
+        'Fcy;': '\u0424',
+        'fcy;': '\u0444',
+        'female;': '\u2640',
+        'ffilig;': '\ufb03',
+        'fflig;': '\ufb00',
+        'ffllig;': '\ufb04',
+        'Ffr;': '\U0001d509',
+        'ffr;': '\U0001d523',
+        'filig;': '\ufb01',
+        'FilledSmallSquare;': '\u25fc',
+        'FilledVerySmallSquare;': '\u25aa',
+        'fjlig;': 'fj',
+        'flat;': '\u266d',
+        'fllig;': '\ufb02',
+        'fltns;': '\u25b1',
+        'fnof;': '\u0192',
+        'Fopf;': '\U0001d53d',
+        'fopf;': '\U0001d557',
+        'ForAll;': '\u2200',
+        'forall;': '\u2200',
+        'fork;': '\u22d4',
+        'forkv;': '\u2ad9',
+        'Fouriertrf;': '\u2131',
+        'fpartint;': '\u2a0d',
+        'frac12': '\xbd',
+        'frac12;': '\xbd',
+        'frac13;': '\u2153',
+        'frac14': '\xbc',
+        'frac14;': '\xbc',
+        'frac15;': '\u2155',
+        'frac16;': '\u2159',
+        'frac18;': '\u215b',
+        'frac23;': '\u2154',
+        'frac25;': '\u2156',
+        'frac34': '\xbe',
+        'frac34;': '\xbe',
+        'frac35;': '\u2157',
+        'frac38;': '\u215c',
+        'frac45;': '\u2158',
+        'frac56;': '\u215a',
+        'frac58;': '\u215d',
+        'frac78;': '\u215e',
+        'frasl;': '\u2044',
+        'frown;': '\u2322',
+        'Fscr;': '\u2131',
+        'fscr;': '\U0001d4bb',
+        'gacute;': '\u01f5',
+        'Gamma;': '\u0393',
+        'gamma;': '\u03b3',
+        'Gammad;': '\u03dc',
+        'gammad;': '\u03dd',
+        'gap;': '\u2a86',
+        'Gbreve;': '\u011e',
+        'gbreve;': '\u011f',
+        'Gcedil;': '\u0122',
+        'Gcirc;': '\u011c',
+        'gcirc;': '\u011d',
+        'Gcy;': '\u0413',
+        'gcy;': '\u0433',
+        'Gdot;': '\u0120',
+        'gdot;': '\u0121',
+        'gE;': '\u2267',
+        'ge;': '\u2265',
+        'gEl;': '\u2a8c',
+        'gel;': '\u22db',
+        'geq;': '\u2265',
+        'geqq;': '\u2267',
+        'geqslant;': '\u2a7e',
+        'ges;': '\u2a7e',
+        'gescc;': '\u2aa9',
+        'gesdot;': '\u2a80',
+        'gesdoto;': '\u2a82',
+        'gesdotol;': '\u2a84',
+        'gesl;': '\u22db\ufe00',
+        'gesles;': '\u2a94',
+        'Gfr;': '\U0001d50a',
+        'gfr;': '\U0001d524',
+        'Gg;': '\u22d9',
+        'gg;': '\u226b',
+        'ggg;': '\u22d9',
+        'gimel;': '\u2137',
+        'GJcy;': '\u0403',
+        'gjcy;': '\u0453',
+        'gl;': '\u2277',
+        'gla;': '\u2aa5',
+        'glE;': '\u2a92',
+        'glj;': '\u2aa4',
+        'gnap;': '\u2a8a',
+        'gnapprox;': '\u2a8a',
+        'gnE;': '\u2269',
+        'gne;': '\u2a88',
+        'gneq;': '\u2a88',
+        'gneqq;': '\u2269',
+        'gnsim;': '\u22e7',
+        'Gopf;': '\U0001d53e',
+        'gopf;': '\U0001d558',
+        'grave;': '`',
+        'GreaterEqual;': '\u2265',
+        'GreaterEqualLess;': '\u22db',
+        'GreaterFullEqual;': '\u2267',
+        'GreaterGreater;': '\u2aa2',
+        'GreaterLess;': '\u2277',
+        'GreaterSlantEqual;': '\u2a7e',
+        'GreaterTilde;': '\u2273',
+        'Gscr;': '\U0001d4a2',
+        'gscr;': '\u210a',
+        'gsim;': '\u2273',
+        'gsime;': '\u2a8e',
+        'gsiml;': '\u2a90',
+        'GT': '>',
+        'gt': '>',
+        'GT;': '>',
+        'Gt;': '\u226b',
+        'gt;': '>',
+        'gtcc;': '\u2aa7',
+        'gtcir;': '\u2a7a',
+        'gtdot;': '\u22d7',
+        'gtlPar;': '\u2995',
+        'gtquest;': '\u2a7c',
+        'gtrapprox;': '\u2a86',
+        'gtrarr;': '\u2978',
+        'gtrdot;': '\u22d7',
+        'gtreqless;': '\u22db',
+        'gtreqqless;': '\u2a8c',
+        'gtrless;': '\u2277',
+        'gtrsim;': '\u2273',
+        'gvertneqq;': '\u2269\ufe00',
+        'gvnE;': '\u2269\ufe00',
+        'Hacek;': '\u02c7',
+        'hairsp;': '\u200a',
+        'half;': '\xbd',
+        'hamilt;': '\u210b',
+        'HARDcy;': '\u042a',
+        'hardcy;': '\u044a',
+        'hArr;': '\u21d4',
+        'harr;': '\u2194',
+        'harrcir;': '\u2948',
+        'harrw;': '\u21ad',
+        'Hat;': '^',
+        'hbar;': '\u210f',
+        'Hcirc;': '\u0124',
+        'hcirc;': '\u0125',
+        'hearts;': '\u2665',
+        'heartsuit;': '\u2665',
+        'hellip;': '\u2026',
+        'hercon;': '\u22b9',
+        'Hfr;': '\u210c',
+        'hfr;': '\U0001d525',
+        'HilbertSpace;': '\u210b',
+        'hksearow;': '\u2925',
+        'hkswarow;': '\u2926',
+        'hoarr;': '\u21ff',
+        'homtht;': '\u223b',
+        'hookleftarrow;': '\u21a9',
+        'hookrightarrow;': '\u21aa',
+        'Hopf;': '\u210d',
+        'hopf;': '\U0001d559',
+        'horbar;': '\u2015',
+        'HorizontalLine;': '\u2500',
+        'Hscr;': '\u210b',
+        'hscr;': '\U0001d4bd',
+        'hslash;': '\u210f',
+        'Hstrok;': '\u0126',
+        'hstrok;': '\u0127',
+        'HumpDownHump;': '\u224e',
+        'HumpEqual;': '\u224f',
+        'hybull;': '\u2043',
+        'hyphen;': '\u2010',
+        'Iacute': '\xcd',
+        'iacute': '\xed',
+        'Iacute;': '\xcd',
+        'iacute;': '\xed',
+        'ic;': '\u2063',
+        'Icirc': '\xce',
+        'icirc': '\xee',
+        'Icirc;': '\xce',
+        'icirc;': '\xee',
+        'Icy;': '\u0418',
+        'icy;': '\u0438',
+        'Idot;': '\u0130',
+        'IEcy;': '\u0415',
+        'iecy;': '\u0435',
+        'iexcl': '\xa1',
+        'iexcl;': '\xa1',
+        'iff;': '\u21d4',
+        'Ifr;': '\u2111',
+        'ifr;': '\U0001d526',
+        'Igrave': '\xcc',
+        'igrave': '\xec',
+        'Igrave;': '\xcc',
+        'igrave;': '\xec',
+        'ii;': '\u2148',
+        'iiiint;': '\u2a0c',
+        'iiint;': '\u222d',
+        'iinfin;': '\u29dc',
+        'iiota;': '\u2129',
+        'IJlig;': '\u0132',
+        'ijlig;': '\u0133',
+        'Im;': '\u2111',
+        'Imacr;': '\u012a',
+        'imacr;': '\u012b',
+        'image;': '\u2111',
+        'ImaginaryI;': '\u2148',
+        'imagline;': '\u2110',
+        'imagpart;': '\u2111',
+        'imath;': '\u0131',
+        'imof;': '\u22b7',
+        'imped;': '\u01b5',
+        'Implies;': '\u21d2',
+        'in;': '\u2208',
+        'incare;': '\u2105',
+        'infin;': '\u221e',
+        'infintie;': '\u29dd',
+        'inodot;': '\u0131',
+        'Int;': '\u222c',
+        'int;': '\u222b',
+        'intcal;': '\u22ba',
+        'integers;': '\u2124',
+        'Integral;': '\u222b',
+        'intercal;': '\u22ba',
+        'Intersection;': '\u22c2',
+        'intlarhk;': '\u2a17',
+        'intprod;': '\u2a3c',
+        'InvisibleComma;': '\u2063',
+        'InvisibleTimes;': '\u2062',
+        'IOcy;': '\u0401',
+        'iocy;': '\u0451',
+        'Iogon;': '\u012e',
+        'iogon;': '\u012f',
+        'Iopf;': '\U0001d540',
+        'iopf;': '\U0001d55a',
+        'Iota;': '\u0399',
+        'iota;': '\u03b9',
+        'iprod;': '\u2a3c',
+        'iquest': '\xbf',
+        'iquest;': '\xbf',
+        'Iscr;': '\u2110',
+        'iscr;': '\U0001d4be',
+        'isin;': '\u2208',
+        'isindot;': '\u22f5',
+        'isinE;': '\u22f9',
+        'isins;': '\u22f4',
+        'isinsv;': '\u22f3',
+        'isinv;': '\u2208',
+        'it;': '\u2062',
+        'Itilde;': '\u0128',
+        'itilde;': '\u0129',
+        'Iukcy;': '\u0406',
+        'iukcy;': '\u0456',
+        'Iuml': '\xcf',
+        'iuml': '\xef',
+        'Iuml;': '\xcf',
+        'iuml;': '\xef',
+        'Jcirc;': '\u0134',
+        'jcirc;': '\u0135',
+        'Jcy;': '\u0419',
+        'jcy;': '\u0439',
+        'Jfr;': '\U0001d50d',
+        'jfr;': '\U0001d527',
+        'jmath;': '\u0237',
+        'Jopf;': '\U0001d541',
+        'jopf;': '\U0001d55b',
+        'Jscr;': '\U0001d4a5',
+        'jscr;': '\U0001d4bf',
+        'Jsercy;': '\u0408',
+        'jsercy;': '\u0458',
+        'Jukcy;': '\u0404',
+        'jukcy;': '\u0454',
+        'Kappa;': '\u039a',
+        'kappa;': '\u03ba',
+        'kappav;': '\u03f0',
+        'Kcedil;': '\u0136',
+        'kcedil;': '\u0137',
+        'Kcy;': '\u041a',
+        'kcy;': '\u043a',
+        'Kfr;': '\U0001d50e',
+        'kfr;': '\U0001d528',
+        'kgreen;': '\u0138',
+        'KHcy;': '\u0425',
+        'khcy;': '\u0445',
+        'KJcy;': '\u040c',
+        'kjcy;': '\u045c',
+        'Kopf;': '\U0001d542',
+        'kopf;': '\U0001d55c',
+        'Kscr;': '\U0001d4a6',
+        'kscr;': '\U0001d4c0',
+        'lAarr;': '\u21da',
+        'Lacute;': '\u0139',
+        'lacute;': '\u013a',
+        'laemptyv;': '\u29b4',
+        'lagran;': '\u2112',
+        'Lambda;': '\u039b',
+        'lambda;': '\u03bb',
+        'Lang;': '\u27ea',
+        'lang;': '\u27e8',
+        'langd;': '\u2991',
+        'langle;': '\u27e8',
+        'lap;': '\u2a85',
+        'Laplacetrf;': '\u2112',
+        'laquo': '\xab',
+        'laquo;': '\xab',
+        'Larr;': '\u219e',
+        'lArr;': '\u21d0',
+        'larr;': '\u2190',
+        'larrb;': '\u21e4',
+        'larrbfs;': '\u291f',
+        'larrfs;': '\u291d',
+        'larrhk;': '\u21a9',
+        'larrlp;': '\u21ab',
+        'larrpl;': '\u2939',
+        'larrsim;': '\u2973',
+        'larrtl;': '\u21a2',
+        'lat;': '\u2aab',
+        'lAtail;': '\u291b',
+        'latail;': '\u2919',
+        'late;': '\u2aad',
+        'lates;': '\u2aad\ufe00',
+        'lBarr;': '\u290e',
+        'lbarr;': '\u290c',
+        'lbbrk;': '\u2772',
+        'lbrace;': '{',
+        'lbrack;': '[',
+        'lbrke;': '\u298b',
+        'lbrksld;': '\u298f',
+        'lbrkslu;': '\u298d',
+        'Lcaron;': '\u013d',
+        'lcaron;': '\u013e',
+        'Lcedil;': '\u013b',
+        'lcedil;': '\u013c',
+        'lceil;': '\u2308',
+        'lcub;': '{',
+        'Lcy;': '\u041b',
+        'lcy;': '\u043b',
+        'ldca;': '\u2936',
+        'ldquo;': '\u201c',
+        'ldquor;': '\u201e',
+        'ldrdhar;': '\u2967',
+        'ldrushar;': '\u294b',
+        'ldsh;': '\u21b2',
+        'lE;': '\u2266',
+        'le;': '\u2264',
+        'LeftAngleBracket;': '\u27e8',
+        'LeftArrow;': '\u2190',
+        'Leftarrow;': '\u21d0',
+        'leftarrow;': '\u2190',
+        'LeftArrowBar;': '\u21e4',
+        'LeftArrowRightArrow;': '\u21c6',
+        'leftarrowtail;': '\u21a2',
+        'LeftCeiling;': '\u2308',
+        'LeftDoubleBracket;': '\u27e6',
+        'LeftDownTeeVector;': '\u2961',
+        'LeftDownVector;': '\u21c3',
+        'LeftDownVectorBar;': '\u2959',
+        'LeftFloor;': '\u230a',
+        'leftharpoondown;': '\u21bd',
+        'leftharpoonup;': '\u21bc',
+        'leftleftarrows;': '\u21c7',
+        'LeftRightArrow;': '\u2194',
+        'Leftrightarrow;': '\u21d4',
+        'leftrightarrow;': '\u2194',
+        'leftrightarrows;': '\u21c6',
+        'leftrightharpoons;': '\u21cb',
+        'leftrightsquigarrow;': '\u21ad',
+        'LeftRightVector;': '\u294e',
+        'LeftTee;': '\u22a3',
+        'LeftTeeArrow;': '\u21a4',
+        'LeftTeeVector;': '\u295a',
+        'leftthreetimes;': '\u22cb',
+        'LeftTriangle;': '\u22b2',
+        'LeftTriangleBar;': '\u29cf',
+        'LeftTriangleEqual;': '\u22b4',
+        'LeftUpDownVector;': '\u2951',
+        'LeftUpTeeVector;': '\u2960',
+        'LeftUpVector;': '\u21bf',
+        'LeftUpVectorBar;': '\u2958',
+        'LeftVector;': '\u21bc',
+        'LeftVectorBar;': '\u2952',
+        'lEg;': '\u2a8b',
+        'leg;': '\u22da',
+        'leq;': '\u2264',
+        'leqq;': '\u2266',
+        'leqslant;': '\u2a7d',
+        'les;': '\u2a7d',
+        'lescc;': '\u2aa8',
+        'lesdot;': '\u2a7f',
+        'lesdoto;': '\u2a81',
+        'lesdotor;': '\u2a83',
+        'lesg;': '\u22da\ufe00',
+        'lesges;': '\u2a93',
+        'lessapprox;': '\u2a85',
+        'lessdot;': '\u22d6',
+        'lesseqgtr;': '\u22da',
+        'lesseqqgtr;': '\u2a8b',
+        'LessEqualGreater;': '\u22da',
+        'LessFullEqual;': '\u2266',
+        'LessGreater;': '\u2276',
+        'lessgtr;': '\u2276',
+        'LessLess;': '\u2aa1',
+        'lesssim;': '\u2272',
+        'LessSlantEqual;': '\u2a7d',
+        'LessTilde;': '\u2272',
+        'lfisht;': '\u297c',
+        'lfloor;': '\u230a',
+        'Lfr;': '\U0001d50f',
+        'lfr;': '\U0001d529',
+        'lg;': '\u2276',
+        'lgE;': '\u2a91',
+        'lHar;': '\u2962',
+        'lhard;': '\u21bd',
+        'lharu;': '\u21bc',
+        'lharul;': '\u296a',
+        'lhblk;': '\u2584',
+        'LJcy;': '\u0409',
+        'ljcy;': '\u0459',
+        'Ll;': '\u22d8',
+        'll;': '\u226a',
+        'llarr;': '\u21c7',
+        'llcorner;': '\u231e',
+        'Lleftarrow;': '\u21da',
+        'llhard;': '\u296b',
+        'lltri;': '\u25fa',
+        'Lmidot;': '\u013f',
+        'lmidot;': '\u0140',
+        'lmoust;': '\u23b0',
+        'lmoustache;': '\u23b0',
+        'lnap;': '\u2a89',
+        'lnapprox;': '\u2a89',
+        'lnE;': '\u2268',
+        'lne;': '\u2a87',
+        'lneq;': '\u2a87',
+        'lneqq;': '\u2268',
+        'lnsim;': '\u22e6',
+        'loang;': '\u27ec',
+        'loarr;': '\u21fd',
+        'lobrk;': '\u27e6',
+        'LongLeftArrow;': '\u27f5',
+        'Longleftarrow;': '\u27f8',
+        'longleftarrow;': '\u27f5',
+        'LongLeftRightArrow;': '\u27f7',
+        'Longleftrightarrow;': '\u27fa',
+        'longleftrightarrow;': '\u27f7',
+        'longmapsto;': '\u27fc',
+        'LongRightArrow;': '\u27f6',
+        'Longrightarrow;': '\u27f9',
+        'longrightarrow;': '\u27f6',
+        'looparrowleft;': '\u21ab',
+        'looparrowright;': '\u21ac',
+        'lopar;': '\u2985',
+        'Lopf;': '\U0001d543',
+        'lopf;': '\U0001d55d',
+        'loplus;': '\u2a2d',
+        'lotimes;': '\u2a34',
+        'lowast;': '\u2217',
+        'lowbar;': '_',
+        'LowerLeftArrow;': '\u2199',
+        'LowerRightArrow;': '\u2198',
+        'loz;': '\u25ca',
+        'lozenge;': '\u25ca',
+        'lozf;': '\u29eb',
+        'lpar;': '(',
+        'lparlt;': '\u2993',
+        'lrarr;': '\u21c6',
+        'lrcorner;': '\u231f',
+        'lrhar;': '\u21cb',
+        'lrhard;': '\u296d',
+        'lrm;': '\u200e',
+        'lrtri;': '\u22bf',
+        'lsaquo;': '\u2039',
+        'Lscr;': '\u2112',
+        'lscr;': '\U0001d4c1',
+        'Lsh;': '\u21b0',
+        'lsh;': '\u21b0',
+        'lsim;': '\u2272',
+        'lsime;': '\u2a8d',
+        'lsimg;': '\u2a8f',
+        'lsqb;': '[',
+        'lsquo;': '\u2018',
+        'lsquor;': '\u201a',
+        'Lstrok;': '\u0141',
+        'lstrok;': '\u0142',
+        'LT': '<',
+        'lt': '<',
+        'LT;': '<',
+        'Lt;': '\u226a',
+        'lt;': '<',
+        'ltcc;': '\u2aa6',
+        'ltcir;': '\u2a79',
+        'ltdot;': '\u22d6',
+        'lthree;': '\u22cb',
+        'ltimes;': '\u22c9',
+        'ltlarr;': '\u2976',
+        'ltquest;': '\u2a7b',
+        'ltri;': '\u25c3',
+        'ltrie;': '\u22b4',
+        'ltrif;': '\u25c2',
+        'ltrPar;': '\u2996',
+        'lurdshar;': '\u294a',
+        'luruhar;': '\u2966',
+        'lvertneqq;': '\u2268\ufe00',
+        'lvnE;': '\u2268\ufe00',
+        'macr': '\xaf',
+        'macr;': '\xaf',
+        'male;': '\u2642',
+        'malt;': '\u2720',
+        'maltese;': '\u2720',
+        'Map;': '\u2905',
+        'map;': '\u21a6',
+        'mapsto;': '\u21a6',
+        'mapstodown;': '\u21a7',
+        'mapstoleft;': '\u21a4',
+        'mapstoup;': '\u21a5',
+        'marker;': '\u25ae',
+        'mcomma;': '\u2a29',
+        'Mcy;': '\u041c',
+        'mcy;': '\u043c',
+        'mdash;': '\u2014',
+        'mDDot;': '\u223a',
+        'measuredangle;': '\u2221',
+        'MediumSpace;': '\u205f',
+        'Mellintrf;': '\u2133',
+        'Mfr;': '\U0001d510',
+        'mfr;': '\U0001d52a',
+        'mho;': '\u2127',
+        'micro': '\xb5',
+        'micro;': '\xb5',
+        'mid;': '\u2223',
+        'midast;': '*',
+        'midcir;': '\u2af0',
+        'middot': '\xb7',
+        'middot;': '\xb7',
+        'minus;': '\u2212',
+        'minusb;': '\u229f',
+        'minusd;': '\u2238',
+        'minusdu;': '\u2a2a',
+        'MinusPlus;': '\u2213',
+        'mlcp;': '\u2adb',
+        'mldr;': '\u2026',
+        'mnplus;': '\u2213',
+        'models;': '\u22a7',
+        'Mopf;': '\U0001d544',
+        'mopf;': '\U0001d55e',
+        'mp;': '\u2213',
+        'Mscr;': '\u2133',
+        'mscr;': '\U0001d4c2',
+        'mstpos;': '\u223e',
+        'Mu;': '\u039c',
+        'mu;': '\u03bc',
+        'multimap;': '\u22b8',
+        'mumap;': '\u22b8',
+        'nabla;': '\u2207',
+        'Nacute;': '\u0143',
+        'nacute;': '\u0144',
+        'nang;': '\u2220\u20d2',
+        'nap;': '\u2249',
+        'napE;': '\u2a70\u0338',
+        'napid;': '\u224b\u0338',
+        'napos;': '\u0149',
+        'napprox;': '\u2249',
+        'natur;': '\u266e',
+        'natural;': '\u266e',
+        'naturals;': '\u2115',
+        'nbsp': '\xa0',
+        'nbsp;': '\xa0',
+        'nbump;': '\u224e\u0338',
+        'nbumpe;': '\u224f\u0338',
+        'ncap;': '\u2a43',
+        'Ncaron;': '\u0147',
+        'ncaron;': '\u0148',
+        'Ncedil;': '\u0145',
+        'ncedil;': '\u0146',
+        'ncong;': '\u2247',
+        'ncongdot;': '\u2a6d\u0338',
+        'ncup;': '\u2a42',
+        'Ncy;': '\u041d',
+        'ncy;': '\u043d',
+        'ndash;': '\u2013',
+        'ne;': '\u2260',
+        'nearhk;': '\u2924',
+        'neArr;': '\u21d7',
+        'nearr;': '\u2197',
+        'nearrow;': '\u2197',
+        'nedot;': '\u2250\u0338',
+        'NegativeMediumSpace;': '\u200b',
+        'NegativeThickSpace;': '\u200b',
+        'NegativeThinSpace;': '\u200b',
+        'NegativeVeryThinSpace;': '\u200b',
+        'nequiv;': '\u2262',
+        'nesear;': '\u2928',
+        'nesim;': '\u2242\u0338',
+        'NestedGreaterGreater;': '\u226b',
+        'NestedLessLess;': '\u226a',
+        'NewLine;': '\n',
+        'nexist;': '\u2204',
+        'nexists;': '\u2204',
+        'Nfr;': '\U0001d511',
+        'nfr;': '\U0001d52b',
+        'ngE;': '\u2267\u0338',
+        'nge;': '\u2271',
+        'ngeq;': '\u2271',
+        'ngeqq;': '\u2267\u0338',
+        'ngeqslant;': '\u2a7e\u0338',
+        'nges;': '\u2a7e\u0338',
+        'nGg;': '\u22d9\u0338',
+        'ngsim;': '\u2275',
+        'nGt;': '\u226b\u20d2',
+        'ngt;': '\u226f',
+        'ngtr;': '\u226f',
+        'nGtv;': '\u226b\u0338',
+        'nhArr;': '\u21ce',
+        'nharr;': '\u21ae',
+        'nhpar;': '\u2af2',
+        'ni;': '\u220b',
+        'nis;': '\u22fc',
+        'nisd;': '\u22fa',
+        'niv;': '\u220b',
+        'NJcy;': '\u040a',
+        'njcy;': '\u045a',
+        'nlArr;': '\u21cd',
+        'nlarr;': '\u219a',
+        'nldr;': '\u2025',
+        'nlE;': '\u2266\u0338',
+        'nle;': '\u2270',
+        'nLeftarrow;': '\u21cd',
+        'nleftarrow;': '\u219a',
+        'nLeftrightarrow;': '\u21ce',
+        'nleftrightarrow;': '\u21ae',
+        'nleq;': '\u2270',
+        'nleqq;': '\u2266\u0338',
+        'nleqslant;': '\u2a7d\u0338',
+        'nles;': '\u2a7d\u0338',
+        'nless;': '\u226e',
+        'nLl;': '\u22d8\u0338',
+        'nlsim;': '\u2274',
+        'nLt;': '\u226a\u20d2',
+        'nlt;': '\u226e',
+        'nltri;': '\u22ea',
+        'nltrie;': '\u22ec',
+        'nLtv;': '\u226a\u0338',
+        'nmid;': '\u2224',
+        'NoBreak;': '\u2060',
+        'NonBreakingSpace;': '\xa0',
+        'Nopf;': '\u2115',
+        'nopf;': '\U0001d55f',
+        'not': '\xac',
+        'Not;': '\u2aec',
+        'not;': '\xac',
+        'NotCongruent;': '\u2262',
+        'NotCupCap;': '\u226d',
+        'NotDoubleVerticalBar;': '\u2226',
+        'NotElement;': '\u2209',
+        'NotEqual;': '\u2260',
+        'NotEqualTilde;': '\u2242\u0338',
+        'NotExists;': '\u2204',
+        'NotGreater;': '\u226f',
+        'NotGreaterEqual;': '\u2271',
+        'NotGreaterFullEqual;': '\u2267\u0338',
+        'NotGreaterGreater;': '\u226b\u0338',
+        'NotGreaterLess;': '\u2279',
+        'NotGreaterSlantEqual;': '\u2a7e\u0338',
+        'NotGreaterTilde;': '\u2275',
+        'NotHumpDownHump;': '\u224e\u0338',
+        'NotHumpEqual;': '\u224f\u0338',
+        'notin;': '\u2209',
+        'notindot;': '\u22f5\u0338',
+        'notinE;': '\u22f9\u0338',
+        'notinva;': '\u2209',
+        'notinvb;': '\u22f7',
+        'notinvc;': '\u22f6',
+        'NotLeftTriangle;': '\u22ea',
+        'NotLeftTriangleBar;': '\u29cf\u0338',
+        'NotLeftTriangleEqual;': '\u22ec',
+        'NotLess;': '\u226e',
+        'NotLessEqual;': '\u2270',
+        'NotLessGreater;': '\u2278',
+        'NotLessLess;': '\u226a\u0338',
+        'NotLessSlantEqual;': '\u2a7d\u0338',
+        'NotLessTilde;': '\u2274',
+        'NotNestedGreaterGreater;': '\u2aa2\u0338',
+        'NotNestedLessLess;': '\u2aa1\u0338',
+        'notni;': '\u220c',
+        'notniva;': '\u220c',
+        'notnivb;': '\u22fe',
+        'notnivc;': '\u22fd',
+        'NotPrecedes;': '\u2280',
+        'NotPrecedesEqual;': '\u2aaf\u0338',
+        'NotPrecedesSlantEqual;': '\u22e0',
+        'NotReverseElement;': '\u220c',
+        'NotRightTriangle;': '\u22eb',
+        'NotRightTriangleBar;': '\u29d0\u0338',
+        'NotRightTriangleEqual;': '\u22ed',
+        'NotSquareSubset;': '\u228f\u0338',
+        'NotSquareSubsetEqual;': '\u22e2',
+        'NotSquareSuperset;': '\u2290\u0338',
+        'NotSquareSupersetEqual;': '\u22e3',
+        'NotSubset;': '\u2282\u20d2',
+        'NotSubsetEqual;': '\u2288',
+        'NotSucceeds;': '\u2281',
+        'NotSucceedsEqual;': '\u2ab0\u0338',
+        'NotSucceedsSlantEqual;': '\u22e1',
+        'NotSucceedsTilde;': '\u227f\u0338',
+        'NotSuperset;': '\u2283\u20d2',
+        'NotSupersetEqual;': '\u2289',
+        'NotTilde;': '\u2241',
+        'NotTildeEqual;': '\u2244',
+        'NotTildeFullEqual;': '\u2247',
+        'NotTildeTilde;': '\u2249',
+        'NotVerticalBar;': '\u2224',
+        'npar;': '\u2226',
+        'nparallel;': '\u2226',
+        'nparsl;': '\u2afd\u20e5',
+        'npart;': '\u2202\u0338',
+        'npolint;': '\u2a14',
+        'npr;': '\u2280',
+        'nprcue;': '\u22e0',
+        'npre;': '\u2aaf\u0338',
+        'nprec;': '\u2280',
+        'npreceq;': '\u2aaf\u0338',
+        'nrArr;': '\u21cf',
+        'nrarr;': '\u219b',
+        'nrarrc;': '\u2933\u0338',
+        'nrarrw;': '\u219d\u0338',
+        'nRightarrow;': '\u21cf',
+        'nrightarrow;': '\u219b',
+        'nrtri;': '\u22eb',
+        'nrtrie;': '\u22ed',
+        'nsc;': '\u2281',
+        'nsccue;': '\u22e1',
+        'nsce;': '\u2ab0\u0338',
+        'Nscr;': '\U0001d4a9',
+        'nscr;': '\U0001d4c3',
+        'nshortmid;': '\u2224',
+        'nshortparallel;': '\u2226',
+        'nsim;': '\u2241',
+        'nsime;': '\u2244',
+        'nsimeq;': '\u2244',
+        'nsmid;': '\u2224',
+        'nspar;': '\u2226',
+        'nsqsube;': '\u22e2',
+        'nsqsupe;': '\u22e3',
+        'nsub;': '\u2284',
+        'nsubE;': '\u2ac5\u0338',
+        'nsube;': '\u2288',
+        'nsubset;': '\u2282\u20d2',
+        'nsubseteq;': '\u2288',
+        'nsubseteqq;': '\u2ac5\u0338',
+        'nsucc;': '\u2281',
+        'nsucceq;': '\u2ab0\u0338',
+        'nsup;': '\u2285',
+        'nsupE;': '\u2ac6\u0338',
+        'nsupe;': '\u2289',
+        'nsupset;': '\u2283\u20d2',
+        'nsupseteq;': '\u2289',
+        'nsupseteqq;': '\u2ac6\u0338',
+        'ntgl;': '\u2279',
+        'Ntilde': '\xd1',
+        'ntilde': '\xf1',
+        'Ntilde;': '\xd1',
+        'ntilde;': '\xf1',
+        'ntlg;': '\u2278',
+        'ntriangleleft;': '\u22ea',
+        'ntrianglelefteq;': '\u22ec',
+        'ntriangleright;': '\u22eb',
+        'ntrianglerighteq;': '\u22ed',
+        'Nu;': '\u039d',
+        'nu;': '\u03bd',
+        'num;': '#',
+        'numero;': '\u2116',
+        'numsp;': '\u2007',
+        'nvap;': '\u224d\u20d2',
+        'nVDash;': '\u22af',
+        'nVdash;': '\u22ae',
+        'nvDash;': '\u22ad',
+        'nvdash;': '\u22ac',
+        'nvge;': '\u2265\u20d2',
+        'nvgt;': '>\u20d2',
+        'nvHarr;': '\u2904',
+        'nvinfin;': '\u29de',
+        'nvlArr;': '\u2902',
+        'nvle;': '\u2264\u20d2',
+        'nvlt;': '<\u20d2',
+        'nvltrie;': '\u22b4\u20d2',
+        'nvrArr;': '\u2903',
+        'nvrtrie;': '\u22b5\u20d2',
+        'nvsim;': '\u223c\u20d2',
+        'nwarhk;': '\u2923',
+        'nwArr;': '\u21d6',
+        'nwarr;': '\u2196',
+        'nwarrow;': '\u2196',
+        'nwnear;': '\u2927',
+        'Oacute': '\xd3',
+        'oacute': '\xf3',
+        'Oacute;': '\xd3',
+        'oacute;': '\xf3',
+        'oast;': '\u229b',
+        'ocir;': '\u229a',
+        'Ocirc': '\xd4',
+        'ocirc': '\xf4',
+        'Ocirc;': '\xd4',
+        'ocirc;': '\xf4',
+        'Ocy;': '\u041e',
+        'ocy;': '\u043e',
+        'odash;': '\u229d',
+        'Odblac;': '\u0150',
+        'odblac;': '\u0151',
+        'odiv;': '\u2a38',
+        'odot;': '\u2299',
+        'odsold;': '\u29bc',
+        'OElig;': '\u0152',
+        'oelig;': '\u0153',
+        'ofcir;': '\u29bf',
+        'Ofr;': '\U0001d512',
+        'ofr;': '\U0001d52c',
+        'ogon;': '\u02db',
+        'Ograve': '\xd2',
+        'ograve': '\xf2',
+        'Ograve;': '\xd2',
+        'ograve;': '\xf2',
+        'ogt;': '\u29c1',
+        'ohbar;': '\u29b5',
+        'ohm;': '\u03a9',
+        'oint;': '\u222e',
+        'olarr;': '\u21ba',
+        'olcir;': '\u29be',
+        'olcross;': '\u29bb',
+        'oline;': '\u203e',
+        'olt;': '\u29c0',
+        'Omacr;': '\u014c',
+        'omacr;': '\u014d',
+        'Omega;': '\u03a9',
+        'omega;': '\u03c9',
+        'Omicron;': '\u039f',
+        'omicron;': '\u03bf',
+        'omid;': '\u29b6',
+        'ominus;': '\u2296',
+        'Oopf;': '\U0001d546',
+        'oopf;': '\U0001d560',
+        'opar;': '\u29b7',
+        'OpenCurlyDoubleQuote;': '\u201c',
+        'OpenCurlyQuote;': '\u2018',
+        'operp;': '\u29b9',
+        'oplus;': '\u2295',
+        'Or;': '\u2a54',
+        'or;': '\u2228',
+        'orarr;': '\u21bb',
+        'ord;': '\u2a5d',
+        'order;': '\u2134',
+        'orderof;': '\u2134',
+        'ordf': '\xaa',
+        'ordf;': '\xaa',
+        'ordm': '\xba',
+        'ordm;': '\xba',
+        'origof;': '\u22b6',
+        'oror;': '\u2a56',
+        'orslope;': '\u2a57',
+        'orv;': '\u2a5b',
+        'oS;': '\u24c8',
+        'Oscr;': '\U0001d4aa',
+        'oscr;': '\u2134',
+        'Oslash': '\xd8',
+        'oslash': '\xf8',
+        'Oslash;': '\xd8',
+        'oslash;': '\xf8',
+        'osol;': '\u2298',
+        'Otilde': '\xd5',
+        'otilde': '\xf5',
+        'Otilde;': '\xd5',
+        'otilde;': '\xf5',
+        'Otimes;': '\u2a37',
+        'otimes;': '\u2297',
+        'otimesas;': '\u2a36',
+        'Ouml': '\xd6',
+        'ouml': '\xf6',
+        'Ouml;': '\xd6',
+        'ouml;': '\xf6',
+        'ovbar;': '\u233d',
+        'OverBar;': '\u203e',
+        'OverBrace;': '\u23de',
+        'OverBracket;': '\u23b4',
+        'OverParenthesis;': '\u23dc',
+        'par;': '\u2225',
+        'para': '\xb6',
+        'para;': '\xb6',
+        'parallel;': '\u2225',
+        'parsim;': '\u2af3',
+        'parsl;': '\u2afd',
+        'part;': '\u2202',
+        'PartialD;': '\u2202',
+        'Pcy;': '\u041f',
+        'pcy;': '\u043f',
+        'percnt;': '%',
+        'period;': '.',
+        'permil;': '\u2030',
+        'perp;': '\u22a5',
+        'pertenk;': '\u2031',
+        'Pfr;': '\U0001d513',
+        'pfr;': '\U0001d52d',
+        'Phi;': '\u03a6',
+        'phi;': '\u03c6',
+        'phiv;': '\u03d5',
+        'phmmat;': '\u2133',
+        'phone;': '\u260e',
+        'Pi;': '\u03a0',
+        'pi;': '\u03c0',
+        'pitchfork;': '\u22d4',
+        'piv;': '\u03d6',
+        'planck;': '\u210f',
+        'planckh;': '\u210e',
+        'plankv;': '\u210f',
+        'plus;': '+',
+        'plusacir;': '\u2a23',
+        'plusb;': '\u229e',
+        'pluscir;': '\u2a22',
+        'plusdo;': '\u2214',
+        'plusdu;': '\u2a25',
+        'pluse;': '\u2a72',
+        'PlusMinus;': '\xb1',
+        'plusmn': '\xb1',
+        'plusmn;': '\xb1',
+        'plussim;': '\u2a26',
+        'plustwo;': '\u2a27',
+        'pm;': '\xb1',
+        'Poincareplane;': '\u210c',
+        'pointint;': '\u2a15',
+        'Popf;': '\u2119',
+        'popf;': '\U0001d561',
+        'pound': '\xa3',
+        'pound;': '\xa3',
+        'Pr;': '\u2abb',
+        'pr;': '\u227a',
+        'prap;': '\u2ab7',
+        'prcue;': '\u227c',
+        'prE;': '\u2ab3',
+        'pre;': '\u2aaf',
+        'prec;': '\u227a',
+        'precapprox;': '\u2ab7',
+        'preccurlyeq;': '\u227c',
+        'Precedes;': '\u227a',
+        'PrecedesEqual;': '\u2aaf',
+        'PrecedesSlantEqual;': '\u227c',
+        'PrecedesTilde;': '\u227e',
+        'preceq;': '\u2aaf',
+        'precnapprox;': '\u2ab9',
+        'precneqq;': '\u2ab5',
+        'precnsim;': '\u22e8',
+        'precsim;': '\u227e',
+        'Prime;': '\u2033',
+        'prime;': '\u2032',
+        'primes;': '\u2119',
+        'prnap;': '\u2ab9',
+        'prnE;': '\u2ab5',
+        'prnsim;': '\u22e8',
+        'prod;': '\u220f',
+        'Product;': '\u220f',
+        'profalar;': '\u232e',
+        'profline;': '\u2312',
+        'profsurf;': '\u2313',
+        'prop;': '\u221d',
+        'Proportion;': '\u2237',
+        'Proportional;': '\u221d',
+        'propto;': '\u221d',
+        'prsim;': '\u227e',
+        'prurel;': '\u22b0',
+        'Pscr;': '\U0001d4ab',
+        'pscr;': '\U0001d4c5',
+        'Psi;': '\u03a8',
+        'psi;': '\u03c8',
+        'puncsp;': '\u2008',
+        'Qfr;': '\U0001d514',
+        'qfr;': '\U0001d52e',
+        'qint;': '\u2a0c',
+        'Qopf;': '\u211a',
+        'qopf;': '\U0001d562',
+        'qprime;': '\u2057',
+        'Qscr;': '\U0001d4ac',
+        'qscr;': '\U0001d4c6',
+        'quaternions;': '\u210d',
+        'quatint;': '\u2a16',
+        'quest;': '?',
+        'questeq;': '\u225f',
+        'QUOT': '"',
+        'quot': '"',
+        'QUOT;': '"',
+        'quot;': '"',
+        'rAarr;': '\u21db',
+        'race;': '\u223d\u0331',
+        'Racute;': '\u0154',
+        'racute;': '\u0155',
+        'radic;': '\u221a',
+        'raemptyv;': '\u29b3',
+        'Rang;': '\u27eb',
+        'rang;': '\u27e9',
+        'rangd;': '\u2992',
+        'range;': '\u29a5',
+        'rangle;': '\u27e9',
+        'raquo': '\xbb',
+        'raquo;': '\xbb',
+        'Rarr;': '\u21a0',
+        'rArr;': '\u21d2',
+        'rarr;': '\u2192',
+        'rarrap;': '\u2975',
+        'rarrb;': '\u21e5',
+        'rarrbfs;': '\u2920',
+        'rarrc;': '\u2933',
+        'rarrfs;': '\u291e',
+        'rarrhk;': '\u21aa',
+        'rarrlp;': '\u21ac',
+        'rarrpl;': '\u2945',
+        'rarrsim;': '\u2974',
+        'Rarrtl;': '\u2916',
+        'rarrtl;': '\u21a3',
+        'rarrw;': '\u219d',
+        'rAtail;': '\u291c',
+        'ratail;': '\u291a',
+        'ratio;': '\u2236',
+        'rationals;': '\u211a',
+        'RBarr;': '\u2910',
+        'rBarr;': '\u290f',
+        'rbarr;': '\u290d',
+        'rbbrk;': '\u2773',
+        'rbrace;': '}',
+        'rbrack;': ']',
+        'rbrke;': '\u298c',
+        'rbrksld;': '\u298e',
+        'rbrkslu;': '\u2990',
+        'Rcaron;': '\u0158',
+        'rcaron;': '\u0159',
+        'Rcedil;': '\u0156',
+        'rcedil;': '\u0157',
+        'rceil;': '\u2309',
+        'rcub;': '}',
+        'Rcy;': '\u0420',
+        'rcy;': '\u0440',
+        'rdca;': '\u2937',
+        'rdldhar;': '\u2969',
+        'rdquo;': '\u201d',
+        'rdquor;': '\u201d',
+        'rdsh;': '\u21b3',
+        'Re;': '\u211c',
+        'real;': '\u211c',
+        'realine;': '\u211b',
+        'realpart;': '\u211c',
+        'reals;': '\u211d',
+        'rect;': '\u25ad',
+        'REG': '\xae',
+        'reg': '\xae',
+        'REG;': '\xae',
+        'reg;': '\xae',
+        'ReverseElement;': '\u220b',
+        'ReverseEquilibrium;': '\u21cb',
+        'ReverseUpEquilibrium;': '\u296f',
+        'rfisht;': '\u297d',
+        'rfloor;': '\u230b',
+        'Rfr;': '\u211c',
+        'rfr;': '\U0001d52f',
+        'rHar;': '\u2964',
+        'rhard;': '\u21c1',
+        'rharu;': '\u21c0',
+        'rharul;': '\u296c',
+        'Rho;': '\u03a1',
+        'rho;': '\u03c1',
+        'rhov;': '\u03f1',
+        'RightAngleBracket;': '\u27e9',
+        'RightArrow;': '\u2192',
+        'Rightarrow;': '\u21d2',
+        'rightarrow;': '\u2192',
+        'RightArrowBar;': '\u21e5',
+        'RightArrowLeftArrow;': '\u21c4',
+        'rightarrowtail;': '\u21a3',
+        'RightCeiling;': '\u2309',
+        'RightDoubleBracket;': '\u27e7',
+        'RightDownTeeVector;': '\u295d',
+        'RightDownVector;': '\u21c2',
+        'RightDownVectorBar;': '\u2955',
+        'RightFloor;': '\u230b',
+        'rightharpoondown;': '\u21c1',
+        'rightharpoonup;': '\u21c0',
+        'rightleftarrows;': '\u21c4',
+        'rightleftharpoons;': '\u21cc',
+        'rightrightarrows;': '\u21c9',
+        'rightsquigarrow;': '\u219d',
+        'RightTee;': '\u22a2',
+        'RightTeeArrow;': '\u21a6',
+        'RightTeeVector;': '\u295b',
+        'rightthreetimes;': '\u22cc',
+        'RightTriangle;': '\u22b3',
+        'RightTriangleBar;': '\u29d0',
+        'RightTriangleEqual;': '\u22b5',
+        'RightUpDownVector;': '\u294f',
+        'RightUpTeeVector;': '\u295c',
+        'RightUpVector;': '\u21be',
+        'RightUpVectorBar;': '\u2954',
+        'RightVector;': '\u21c0',
+        'RightVectorBar;': '\u2953',
+        'ring;': '\u02da',
+        'risingdotseq;': '\u2253',
+        'rlarr;': '\u21c4',
+        'rlhar;': '\u21cc',
+        'rlm;': '\u200f',
+        'rmoust;': '\u23b1',
+        'rmoustache;': '\u23b1',
+        'rnmid;': '\u2aee',
+        'roang;': '\u27ed',
+        'roarr;': '\u21fe',
+        'robrk;': '\u27e7',
+        'ropar;': '\u2986',
+        'Ropf;': '\u211d',
+        'ropf;': '\U0001d563',
+        'roplus;': '\u2a2e',
+        'rotimes;': '\u2a35',
+        'RoundImplies;': '\u2970',
+        'rpar;': ')',
+        'rpargt;': '\u2994',
+        'rppolint;': '\u2a12',
+        'rrarr;': '\u21c9',
+        'Rrightarrow;': '\u21db',
+        'rsaquo;': '\u203a',
+        'Rscr;': '\u211b',
+        'rscr;': '\U0001d4c7',
+        'Rsh;': '\u21b1',
+        'rsh;': '\u21b1',
+        'rsqb;': ']',
+        'rsquo;': '\u2019',
+        'rsquor;': '\u2019',
+        'rthree;': '\u22cc',
+        'rtimes;': '\u22ca',
+        'rtri;': '\u25b9',
+        'rtrie;': '\u22b5',
+        'rtrif;': '\u25b8',
+        'rtriltri;': '\u29ce',
+        'RuleDelayed;': '\u29f4',
+        'ruluhar;': '\u2968',
+        'rx;': '\u211e',
+        'Sacute;': '\u015a',
+        'sacute;': '\u015b',
+        'sbquo;': '\u201a',
+        'Sc;': '\u2abc',
+        'sc;': '\u227b',
+        'scap;': '\u2ab8',
+        'Scaron;': '\u0160',
+        'scaron;': '\u0161',
+        'sccue;': '\u227d',
+        'scE;': '\u2ab4',
+        'sce;': '\u2ab0',
+        'Scedil;': '\u015e',
+        'scedil;': '\u015f',
+        'Scirc;': '\u015c',
+        'scirc;': '\u015d',
+        'scnap;': '\u2aba',
+        'scnE;': '\u2ab6',
+        'scnsim;': '\u22e9',
+        'scpolint;': '\u2a13',
+        'scsim;': '\u227f',
+        'Scy;': '\u0421',
+        'scy;': '\u0441',
+        'sdot;': '\u22c5',
+        'sdotb;': '\u22a1',
+        'sdote;': '\u2a66',
+        'searhk;': '\u2925',
+        'seArr;': '\u21d8',
+        'searr;': '\u2198',
+        'searrow;': '\u2198',
+        'sect': '\xa7',
+        'sect;': '\xa7',
+        'semi;': ';',
+        'seswar;': '\u2929',
+        'setminus;': '\u2216',
+        'setmn;': '\u2216',
+        'sext;': '\u2736',
+        'Sfr;': '\U0001d516',
+        'sfr;': '\U0001d530',
+        'sfrown;': '\u2322',
+        'sharp;': '\u266f',
+        'SHCHcy;': '\u0429',
+        'shchcy;': '\u0449',
+        'SHcy;': '\u0428',
+        'shcy;': '\u0448',
+        'ShortDownArrow;': '\u2193',
+        'ShortLeftArrow;': '\u2190',
+        'shortmid;': '\u2223',
+        'shortparallel;': '\u2225',
+        'ShortRightArrow;': '\u2192',
+        'ShortUpArrow;': '\u2191',
+        'shy': '\xad',
+        'shy;': '\xad',
+        'Sigma;': '\u03a3',
+        'sigma;': '\u03c3',
+        'sigmaf;': '\u03c2',
+        'sigmav;': '\u03c2',
+        'sim;': '\u223c',
+        'simdot;': '\u2a6a',
+        'sime;': '\u2243',
+        'simeq;': '\u2243',
+        'simg;': '\u2a9e',
+        'simgE;': '\u2aa0',
+        'siml;': '\u2a9d',
+        'simlE;': '\u2a9f',
+        'simne;': '\u2246',
+        'simplus;': '\u2a24',
+        'simrarr;': '\u2972',
+        'slarr;': '\u2190',
+        'SmallCircle;': '\u2218',
+        'smallsetminus;': '\u2216',
+        'smashp;': '\u2a33',
+        'smeparsl;': '\u29e4',
+        'smid;': '\u2223',
+        'smile;': '\u2323',
+        'smt;': '\u2aaa',
+        'smte;': '\u2aac',
+        'smtes;': '\u2aac\ufe00',
+        'SOFTcy;': '\u042c',
+        'softcy;': '\u044c',
+        'sol;': '/',
+        'solb;': '\u29c4',
+        'solbar;': '\u233f',
+        'Sopf;': '\U0001d54a',
+        'sopf;': '\U0001d564',
+        'spades;': '\u2660',
+        'spadesuit;': '\u2660',
+        'spar;': '\u2225',
+        'sqcap;': '\u2293',
+        'sqcaps;': '\u2293\ufe00',
+        'sqcup;': '\u2294',
+        'sqcups;': '\u2294\ufe00',
+        'Sqrt;': '\u221a',
+        'sqsub;': '\u228f',
+        'sqsube;': '\u2291',
+        'sqsubset;': '\u228f',
+        'sqsubseteq;': '\u2291',
+        'sqsup;': '\u2290',
+        'sqsupe;': '\u2292',
+        'sqsupset;': '\u2290',
+        'sqsupseteq;': '\u2292',
+        'squ;': '\u25a1',
+        'Square;': '\u25a1',
+        'square;': '\u25a1',
+        'SquareIntersection;': '\u2293',
+        'SquareSubset;': '\u228f',
+        'SquareSubsetEqual;': '\u2291',
+        'SquareSuperset;': '\u2290',
+        'SquareSupersetEqual;': '\u2292',
+        'SquareUnion;': '\u2294',
+        'squarf;': '\u25aa',
+        'squf;': '\u25aa',
+        'srarr;': '\u2192',
+        'Sscr;': '\U0001d4ae',
+        'sscr;': '\U0001d4c8',
+        'ssetmn;': '\u2216',
+        'ssmile;': '\u2323',
+        'sstarf;': '\u22c6',
+        'Star;': '\u22c6',
+        'star;': '\u2606',
+        'starf;': '\u2605',
+        'straightepsilon;': '\u03f5',
+        'straightphi;': '\u03d5',
+        'strns;': '\xaf',
+        'Sub;': '\u22d0',
+        'sub;': '\u2282',
+        'subdot;': '\u2abd',
+        'subE;': '\u2ac5',
+        'sube;': '\u2286',
+        'subedot;': '\u2ac3',
+        'submult;': '\u2ac1',
+        'subnE;': '\u2acb',
+        'subne;': '\u228a',
+        'subplus;': '\u2abf',
+        'subrarr;': '\u2979',
+        'Subset;': '\u22d0',
+        'subset;': '\u2282',
+        'subseteq;': '\u2286',
+        'subseteqq;': '\u2ac5',
+        'SubsetEqual;': '\u2286',
+        'subsetneq;': '\u228a',
+        'subsetneqq;': '\u2acb',
+        'subsim;': '\u2ac7',
+        'subsub;': '\u2ad5',
+        'subsup;': '\u2ad3',
+        'succ;': '\u227b',
+        'succapprox;': '\u2ab8',
+        'succcurlyeq;': '\u227d',
+        'Succeeds;': '\u227b',
+        'SucceedsEqual;': '\u2ab0',
+        'SucceedsSlantEqual;': '\u227d',
+        'SucceedsTilde;': '\u227f',
+        'succeq;': '\u2ab0',
+        'succnapprox;': '\u2aba',
+        'succneqq;': '\u2ab6',
+        'succnsim;': '\u22e9',
+        'succsim;': '\u227f',
+        'SuchThat;': '\u220b',
+        'Sum;': '\u2211',
+        'sum;': '\u2211',
+        'sung;': '\u266a',
+        'sup1': '\xb9',
+        'sup1;': '\xb9',
+        'sup2': '\xb2',
+        'sup2;': '\xb2',
+        'sup3': '\xb3',
+        'sup3;': '\xb3',
+        'Sup;': '\u22d1',
+        'sup;': '\u2283',
+        'supdot;': '\u2abe',
+        'supdsub;': '\u2ad8',
+        'supE;': '\u2ac6',
+        'supe;': '\u2287',
+        'supedot;': '\u2ac4',
+        'Superset;': '\u2283',
+        'SupersetEqual;': '\u2287',
+        'suphsol;': '\u27c9',
+        'suphsub;': '\u2ad7',
+        'suplarr;': '\u297b',
+        'supmult;': '\u2ac2',
+        'supnE;': '\u2acc',
+        'supne;': '\u228b',
+        'supplus;': '\u2ac0',
+        'Supset;': '\u22d1',
+        'supset;': '\u2283',
+        'supseteq;': '\u2287',
+        'supseteqq;': '\u2ac6',
+        'supsetneq;': '\u228b',
+        'supsetneqq;': '\u2acc',
+        'supsim;': '\u2ac8',
+        'supsub;': '\u2ad4',
+        'supsup;': '\u2ad6',
+        'swarhk;': '\u2926',
+        'swArr;': '\u21d9',
+        'swarr;': '\u2199',
+        'swarrow;': '\u2199',
+        'swnwar;': '\u292a',
+        'szlig': '\xdf',
+        'szlig;': '\xdf',
+        'Tab;': '\t',
+        'target;': '\u2316',
+        'Tau;': '\u03a4',
+        'tau;': '\u03c4',
+        'tbrk;': '\u23b4',
+        'Tcaron;': '\u0164',
+        'tcaron;': '\u0165',
+        'Tcedil;': '\u0162',
+        'tcedil;': '\u0163',
+        'Tcy;': '\u0422',
+        'tcy;': '\u0442',
+        'tdot;': '\u20db',
+        'telrec;': '\u2315',
+        'Tfr;': '\U0001d517',
+        'tfr;': '\U0001d531',
+        'there4;': '\u2234',
+        'Therefore;': '\u2234',
+        'therefore;': '\u2234',
+        'Theta;': '\u0398',
+        'theta;': '\u03b8',
+        'thetasym;': '\u03d1',
+        'thetav;': '\u03d1',
+        'thickapprox;': '\u2248',
+        'thicksim;': '\u223c',
+        'ThickSpace;': '\u205f\u200a',
+        'thinsp;': '\u2009',
+        'ThinSpace;': '\u2009',
+        'thkap;': '\u2248',
+        'thksim;': '\u223c',
+        'THORN': '\xde',
+        'thorn': '\xfe',
+        'THORN;': '\xde',
+        'thorn;': '\xfe',
+        'Tilde;': '\u223c',
+        'tilde;': '\u02dc',
+        'TildeEqual;': '\u2243',
+        'TildeFullEqual;': '\u2245',
+        'TildeTilde;': '\u2248',
+        'times': '\xd7',
+        'times;': '\xd7',
+        'timesb;': '\u22a0',
+        'timesbar;': '\u2a31',
+        'timesd;': '\u2a30',
+        'tint;': '\u222d',
+        'toea;': '\u2928',
+        'top;': '\u22a4',
+        'topbot;': '\u2336',
+        'topcir;': '\u2af1',
+        'Topf;': '\U0001d54b',
+        'topf;': '\U0001d565',
+        'topfork;': '\u2ada',
+        'tosa;': '\u2929',
+        'tprime;': '\u2034',
+        'TRADE;': '\u2122',
+        'trade;': '\u2122',
+        'triangle;': '\u25b5',
+        'triangledown;': '\u25bf',
+        'triangleleft;': '\u25c3',
+        'trianglelefteq;': '\u22b4',
+        'triangleq;': '\u225c',
+        'triangleright;': '\u25b9',
+        'trianglerighteq;': '\u22b5',
+        'tridot;': '\u25ec',
+        'trie;': '\u225c',
+        'triminus;': '\u2a3a',
+        'TripleDot;': '\u20db',
+        'triplus;': '\u2a39',
+        'trisb;': '\u29cd',
+        'tritime;': '\u2a3b',
+        'trpezium;': '\u23e2',
+        'Tscr;': '\U0001d4af',
+        'tscr;': '\U0001d4c9',
+        'TScy;': '\u0426',
+        'tscy;': '\u0446',
+        'TSHcy;': '\u040b',
+        'tshcy;': '\u045b',
+        'Tstrok;': '\u0166',
+        'tstrok;': '\u0167',
+        'twixt;': '\u226c',
+        'twoheadleftarrow;': '\u219e',
+        'twoheadrightarrow;': '\u21a0',
+        'Uacute': '\xda',
+        'uacute': '\xfa',
+        'Uacute;': '\xda',
+        'uacute;': '\xfa',
+        'Uarr;': '\u219f',
+        'uArr;': '\u21d1',
+        'uarr;': '\u2191',
+        'Uarrocir;': '\u2949',
+        'Ubrcy;': '\u040e',
+        'ubrcy;': '\u045e',
+        'Ubreve;': '\u016c',
+        'ubreve;': '\u016d',
+        'Ucirc': '\xdb',
+        'ucirc': '\xfb',
+        'Ucirc;': '\xdb',
+        'ucirc;': '\xfb',
+        'Ucy;': '\u0423',
+        'ucy;': '\u0443',
+        'udarr;': '\u21c5',
+        'Udblac;': '\u0170',
+        'udblac;': '\u0171',
+        'udhar;': '\u296e',
+        'ufisht;': '\u297e',
+        'Ufr;': '\U0001d518',
+        'ufr;': '\U0001d532',
+        'Ugrave': '\xd9',
+        'ugrave': '\xf9',
+        'Ugrave;': '\xd9',
+        'ugrave;': '\xf9',
+        'uHar;': '\u2963',
+        'uharl;': '\u21bf',
+        'uharr;': '\u21be',
+        'uhblk;': '\u2580',
+        'ulcorn;': '\u231c',
+        'ulcorner;': '\u231c',
+        'ulcrop;': '\u230f',
+        'ultri;': '\u25f8',
+        'Umacr;': '\u016a',
+        'umacr;': '\u016b',
+        'uml': '\xa8',
+        'uml;': '\xa8',
+        'UnderBar;': '_',
+        'UnderBrace;': '\u23df',
+        'UnderBracket;': '\u23b5',
+        'UnderParenthesis;': '\u23dd',
+        'Union;': '\u22c3',
+        'UnionPlus;': '\u228e',
+        'Uogon;': '\u0172',
+        'uogon;': '\u0173',
+        'Uopf;': '\U0001d54c',
+        'uopf;': '\U0001d566',
+        'UpArrow;': '\u2191',
+        'Uparrow;': '\u21d1',
+        'uparrow;': '\u2191',
+        'UpArrowBar;': '\u2912',
+        'UpArrowDownArrow;': '\u21c5',
+        'UpDownArrow;': '\u2195',
+        'Updownarrow;': '\u21d5',
+        'updownarrow;': '\u2195',
+        'UpEquilibrium;': '\u296e',
+        'upharpoonleft;': '\u21bf',
+        'upharpoonright;': '\u21be',
+        'uplus;': '\u228e',
+        'UpperLeftArrow;': '\u2196',
+        'UpperRightArrow;': '\u2197',
+        'Upsi;': '\u03d2',
+        'upsi;': '\u03c5',
+        'upsih;': '\u03d2',
+        'Upsilon;': '\u03a5',
+        'upsilon;': '\u03c5',
+        'UpTee;': '\u22a5',
+        'UpTeeArrow;': '\u21a5',
+        'upuparrows;': '\u21c8',
+        'urcorn;': '\u231d',
+        'urcorner;': '\u231d',
+        'urcrop;': '\u230e',
+        'Uring;': '\u016e',
+        'uring;': '\u016f',
+        'urtri;': '\u25f9',
+        'Uscr;': '\U0001d4b0',
+        'uscr;': '\U0001d4ca',
+        'utdot;': '\u22f0',
+        'Utilde;': '\u0168',
+        'utilde;': '\u0169',
+        'utri;': '\u25b5',
+        'utrif;': '\u25b4',
+        'uuarr;': '\u21c8',
+        'Uuml': '\xdc',
+        'uuml': '\xfc',
+        'Uuml;': '\xdc',
+        'uuml;': '\xfc',
+        'uwangle;': '\u29a7',
+        'vangrt;': '\u299c',
+        'varepsilon;': '\u03f5',
+        'varkappa;': '\u03f0',
+        'varnothing;': '\u2205',
+        'varphi;': '\u03d5',
+        'varpi;': '\u03d6',
+        'varpropto;': '\u221d',
+        'vArr;': '\u21d5',
+        'varr;': '\u2195',
+        'varrho;': '\u03f1',
+        'varsigma;': '\u03c2',
+        'varsubsetneq;': '\u228a\ufe00',
+        'varsubsetneqq;': '\u2acb\ufe00',
+        'varsupsetneq;': '\u228b\ufe00',
+        'varsupsetneqq;': '\u2acc\ufe00',
+        'vartheta;': '\u03d1',
+        'vartriangleleft;': '\u22b2',
+        'vartriangleright;': '\u22b3',
+        'Vbar;': '\u2aeb',
+        'vBar;': '\u2ae8',
+        'vBarv;': '\u2ae9',
+        'Vcy;': '\u0412',
+        'vcy;': '\u0432',
+        'VDash;': '\u22ab',
+        'Vdash;': '\u22a9',
+        'vDash;': '\u22a8',
+        'vdash;': '\u22a2',
+        'Vdashl;': '\u2ae6',
+        'Vee;': '\u22c1',
+        'vee;': '\u2228',
+        'veebar;': '\u22bb',
+        'veeeq;': '\u225a',
+        'vellip;': '\u22ee',
+        'Verbar;': '\u2016',
+        'verbar;': '|',
+        'Vert;': '\u2016',
+        'vert;': '|',
+        'VerticalBar;': '\u2223',
+        'VerticalLine;': '|',
+        'VerticalSeparator;': '\u2758',
+        'VerticalTilde;': '\u2240',
+        'VeryThinSpace;': '\u200a',
+        'Vfr;': '\U0001d519',
+        'vfr;': '\U0001d533',
+        'vltri;': '\u22b2',
+        'vnsub;': '\u2282\u20d2',
+        'vnsup;': '\u2283\u20d2',
+        'Vopf;': '\U0001d54d',
+        'vopf;': '\U0001d567',
+        'vprop;': '\u221d',
+        'vrtri;': '\u22b3',
+        'Vscr;': '\U0001d4b1',
+        'vscr;': '\U0001d4cb',
+        'vsubnE;': '\u2acb\ufe00',
+        'vsubne;': '\u228a\ufe00',
+        'vsupnE;': '\u2acc\ufe00',
+        'vsupne;': '\u228b\ufe00',
+        'Vvdash;': '\u22aa',
+        'vzigzag;': '\u299a',
+        'Wcirc;': '\u0174',
+        'wcirc;': '\u0175',
+        'wedbar;': '\u2a5f',
+        'Wedge;': '\u22c0',
+        'wedge;': '\u2227',
+        'wedgeq;': '\u2259',
+        'weierp;': '\u2118',
+        'Wfr;': '\U0001d51a',
+        'wfr;': '\U0001d534',
+        'Wopf;': '\U0001d54e',
+        'wopf;': '\U0001d568',
+        'wp;': '\u2118',
+        'wr;': '\u2240',
+        'wreath;': '\u2240',
+        'Wscr;': '\U0001d4b2',
+        'wscr;': '\U0001d4cc',
+        'xcap;': '\u22c2',
+        'xcirc;': '\u25ef',
+        'xcup;': '\u22c3',
+        'xdtri;': '\u25bd',
+        'Xfr;': '\U0001d51b',
+        'xfr;': '\U0001d535',
+        'xhArr;': '\u27fa',
+        'xharr;': '\u27f7',
+        'Xi;': '\u039e',
+        'xi;': '\u03be',
+        'xlArr;': '\u27f8',
+        'xlarr;': '\u27f5',
+        'xmap;': '\u27fc',
+        'xnis;': '\u22fb',
+        'xodot;': '\u2a00',
+        'Xopf;': '\U0001d54f',
+        'xopf;': '\U0001d569',
+        'xoplus;': '\u2a01',
+        'xotime;': '\u2a02',
+        'xrArr;': '\u27f9',
+        'xrarr;': '\u27f6',
+        'Xscr;': '\U0001d4b3',
+        'xscr;': '\U0001d4cd',
+        'xsqcup;': '\u2a06',
+        'xuplus;': '\u2a04',
+        'xutri;': '\u25b3',
+        'xvee;': '\u22c1',
+        'xwedge;': '\u22c0',
+        'Yacute': '\xdd',
+        'yacute': '\xfd',
+        'Yacute;': '\xdd',
+        'yacute;': '\xfd',
+        'YAcy;': '\u042f',
+        'yacy;': '\u044f',
+        'Ycirc;': '\u0176',
+        'ycirc;': '\u0177',
+        'Ycy;': '\u042b',
+        'ycy;': '\u044b',
+        'yen': '\xa5',
+        'yen;': '\xa5',
+        'Yfr;': '\U0001d51c',
+        'yfr;': '\U0001d536',
+        'YIcy;': '\u0407',
+        'yicy;': '\u0457',
+        'Yopf;': '\U0001d550',
+        'yopf;': '\U0001d56a',
+        'Yscr;': '\U0001d4b4',
+        'yscr;': '\U0001d4ce',
+        'YUcy;': '\u042e',
+        'yucy;': '\u044e',
+        'yuml': '\xff',
+        'Yuml;': '\u0178',
+        'yuml;': '\xff',
+        'Zacute;': '\u0179',
+        'zacute;': '\u017a',
+        'Zcaron;': '\u017d',
+        'zcaron;': '\u017e',
+        'Zcy;': '\u0417',
+        'zcy;': '\u0437',
+        'Zdot;': '\u017b',
+        'zdot;': '\u017c',
+        'zeetrf;': '\u2128',
+        'ZeroWidthSpace;': '\u200b',
+        'Zeta;': '\u0396',
+        'zeta;': '\u03b6',
+        'Zfr;': '\u2128',
+        'zfr;': '\U0001d537',
+        'ZHcy;': '\u0416',
+        'zhcy;': '\u0436',
+        'zigrarr;': '\u21dd',
+        'Zopf;': '\u2124',
+        'zopf;': '\U0001d56b',
+        'Zscr;': '\U0001d4b5',
+        'zscr;': '\U0001d4cf',
+        'zwj;': '\u200d',
+        'zwnj;': '\u200c',
+    }
+
 try:
     import http.client as compat_http_client
 except ImportError:  # Python 2
@@ -83,7 +2321,6 @@ try:
 except ImportError:  # Python 2
     from HTMLParser import HTMLParser as compat_HTMLParser
 
-
 try:
     from subprocess import DEVNULL
     compat_subprocess_get_DEVNULL = lambda: DEVNULL
@@ -626,6 +2863,7 @@ __all__ = [
     'compat_getenv',
     'compat_getpass',
     'compat_html_entities',
+    'compat_html_entities_html5',
     'compat_http_client',
     'compat_http_server',
     'compat_input',

From 55b2f099c0c820d6c4b46609b175a44a6d7f97bf Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 10 Jun 2016 15:11:55 +0800
Subject: [PATCH 5158/6219] [utils] Decode HTML5 entities

Used in test_Vporn_1. Also related to #9270
---
 test/test_utils.py  |  2 ++
 youtube_dl/utils.py | 12 ++++++++++--
 2 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index feef80465..0e25de6b7 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -249,6 +249,8 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(unescapeHTML('&#47;'), '/')
         self.assertEqual(unescapeHTML('&eacute;'), 'é')
         self.assertEqual(unescapeHTML('&#2013266066;'), '&#2013266066;')
+        # HTML5 entities
+        self.assertEqual(unescapeHTML('&period;&apos;'), '.\'')
 
     def test_date_from_str(self):
         self.assertEqual(date_from_str('yesterday'), date_from_str('now-1day'))
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 229de4b39..f77ab8650 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -39,6 +39,7 @@ from .compat import (
     compat_chr,
     compat_etree_fromstring,
     compat_html_entities,
+    compat_html_entities_html5,
     compat_http_client,
     compat_kwargs,
     compat_parse_qs,
@@ -456,12 +457,19 @@ def orderedSet(iterable):
     return res
 
 
-def _htmlentity_transform(entity):
+def _htmlentity_transform(entity_with_semicolon):
     """Transforms an HTML entity to a character."""
+    entity = entity_with_semicolon[:-1]
+
     # Known non-numeric HTML entity
     if entity in compat_html_entities.name2codepoint:
         return compat_chr(compat_html_entities.name2codepoint[entity])
 
+    # TODO: HTML5 allows entities without a semicolon. For example,
+    # '&Eacuteric' should be decoded as 'Éric'.
+    if entity_with_semicolon in compat_html_entities_html5:
+        return compat_html_entities_html5[entity_with_semicolon]
+
     mobj = re.match(r'#(x[0-9a-fA-F]+|[0-9]+)', entity)
     if mobj is not None:
         numstr = mobj.group(1)
@@ -486,7 +494,7 @@ def unescapeHTML(s):
     assert type(s) == compat_str
 
     return re.sub(
-        r'&([^;]+);', lambda m: _htmlentity_transform(m.group(1)), s)
+        r'&([^;]+;)', lambda m: _htmlentity_transform(m.group(1)), s)
 
 
 def get_subprocess_encoding():

From a2252385308898074f5006ed737aeb98bb8b0402 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 10 Jun 2016 15:12:53 +0800
Subject: [PATCH 5159/6219] [vporn] Improve error detection and update _TESTS

---
 youtube_dl/extractor/vporn.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vporn.py b/youtube_dl/extractor/vporn.py
index 92c90e517..1557a0e04 100644
--- a/youtube_dl/extractor/vporn.py
+++ b/youtube_dl/extractor/vporn.py
@@ -4,6 +4,7 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
+    ExtractorError,
     parse_duration,
     str_to_int,
 )
@@ -27,7 +28,8 @@ class VpornIE(InfoExtractor):
                 'duration': 393,
                 'age_limit': 18,
                 'view_count': int,
-            }
+            },
+            'skip': 'video removed',
         },
         {
             'url': 'http://www.vporn.com/female/hana-shower/523564/',
@@ -40,7 +42,7 @@ class VpornIE(InfoExtractor):
                 'description': 'Hana showers at the bathroom.',
                 'thumbnail': 're:^https?://.*\.jpg$',
                 'uploader': 'Hmmmmm',
-                'categories': ['Big Boobs', 'Erotic', 'Teen', 'Female'],
+                'categories': ['Big Boobs', 'Erotic', 'Teen', 'Female', '720p'],
                 'duration': 588,
                 'age_limit': 18,
                 'view_count': int,
@@ -55,6 +57,10 @@ class VpornIE(InfoExtractor):
 
         webpage = self._download_webpage(url, display_id)
 
+        errmsg = 'This video has been deleted due to Copyright Infringement or by the account owner!'
+        if errmsg in webpage:
+            raise ExtractorError('%s said: %s' % (self.IE_NAME, errmsg), expected=True)
+
         title = self._html_search_regex(
             r'videoname\s*=\s*\'([^\']+)\'', webpage, 'title').strip()
         description = self._html_search_regex(

From c16f8a4659566fd7421226b0d5ddb871425b392b Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 10 Jun 2016 16:04:28 +0800
Subject: [PATCH 5160/6219] [voicerepublic] Force video_id to be strings

Related: be6217b26142491232fb697b125015d45437832d
---
 youtube_dl/extractor/voicerepublic.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/voicerepublic.py b/youtube_dl/extractor/voicerepublic.py
index 93d15a556..4f1a99a89 100644
--- a/youtube_dl/extractor/voicerepublic.py
+++ b/youtube_dl/extractor/voicerepublic.py
@@ -3,7 +3,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urlparse
+from ..compat import (
+    compat_str,
+    compat_urlparse,
+)
 from ..utils import (
     ExtractorError,
     determine_ext,
@@ -16,13 +19,13 @@ class VoiceRepublicIE(InfoExtractor):
     _VALID_URL = r'https?://voicerepublic\.com/(?:talks|embed)/(?P<id>[0-9a-z-]+)'
     _TESTS = [{
         'url': 'http://voicerepublic.com/talks/watching-the-watchers-building-a-sousveillance-state',
-        'md5': '0554a24d1657915aa8e8f84e15dc9353',
+        'md5': 'b9174d651323f17783000876347116e3',
         'info_dict': {
             'id': '2296',
             'display_id': 'watching-the-watchers-building-a-sousveillance-state',
             'ext': 'm4a',
             'title': 'Watching the Watchers: Building a Sousveillance State',
-            'description': 'md5:715ba964958afa2398df615809cfecb1',
+            'description': 'Secret surveillance programs have metadata too. The people and companies that operate secret surveillance programs can be surveilled.',
             'thumbnail': 're:^https?://.*\.(?:png|jpg)$',
             'duration': 1800,
             'view_count': int,
@@ -52,7 +55,7 @@ class VoiceRepublicIE(InfoExtractor):
         if data:
             title = data['title']
             description = data.get('teaser')
-            talk_id = data.get('talk_id') or display_id
+            talk_id = compat_str(data.get('talk_id') or display_id)
             talk = data['talk']
             duration = int_or_none(talk.get('duration'))
             formats = [{

From 09728d5fbc93c769b3f8971c06e9ed0bfb168b37 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 10 Jun 2016 16:11:28 +0800
Subject: [PATCH 5161/6219] [audiomack:album] Force video_id to be strings

Related: be6217b26142491232fb697b125015d45437832d
---
 youtube_dl/extractor/audiomack.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/audiomack.py b/youtube_dl/extractor/audiomack.py
index a52d26cec..f3bd4d444 100644
--- a/youtube_dl/extractor/audiomack.py
+++ b/youtube_dl/extractor/audiomack.py
@@ -6,6 +6,7 @@ import time
 
 from .common import InfoExtractor
 from .soundcloud import SoundcloudIE
+from ..compat import compat_str
 from ..utils import (
     ExtractorError,
     url_basename,
@@ -136,7 +137,7 @@ class AudiomackAlbumIE(InfoExtractor):
                         result[resultkey] = api_response[apikey]
                 song_id = url_basename(api_response['url']).rpartition('.')[0]
                 result['entries'].append({
-                    'id': api_response.get('id', song_id),
+                    'id': compat_str(api_response.get('id', song_id)),
                     'uploader': api_response.get('artist'),
                     'title': api_response.get('title', song_id),
                     'url': api_response['url'],

From daa0df9e8beac1325e5fb55d828e7a3a38e74bf6 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 10 Jun 2016 16:37:12 +0800
Subject: [PATCH 5162/6219] [youtube:user] Support another URL form

Such an URL comes from http://www.gametrailers.com/. This is originally
a test case in GenericIE, but now seems all GameTrailers videos are on
YouTube.
---
 youtube_dl/extractor/youtube.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 6c9f77d95..00dd602ff 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1988,7 +1988,7 @@ class YoutubeChannelIE(YoutubePlaylistBaseInfoExtractor):
 
 class YoutubeUserIE(YoutubeChannelIE):
     IE_DESC = 'YouTube.com user videos (URL or "ytuser" keyword)'
-    _VALID_URL = r'(?:(?:https?://(?:\w+\.)?youtube\.com/(?:user/)?(?!(?:attribution_link|watch|results)(?:$|[^a-z_A-Z0-9-])))|ytuser:)(?!feed/)(?P<id>[A-Za-z0-9_-]+)'
+    _VALID_URL = r'(?:(?:https?://(?:\w+\.)?youtube\.com/(?:user/|c/)?(?!(?:attribution_link|watch|results)(?:$|[^a-z_A-Z0-9-])))|ytuser:)(?!feed/)(?P<id>[A-Za-z0-9_-]+)'
     _TEMPLATE_URL = 'https://www.youtube.com/user/%s/videos'
     IE_NAME = 'youtube:user'
 
@@ -2001,6 +2001,9 @@ class YoutubeUserIE(YoutubeChannelIE):
     }, {
         'url': 'ytuser:phihag',
         'only_matching': True,
+    }, {
+        'url': 'https://www.youtube.com/c/gametrailers',
+        'only_matching': True,
     }]
 
     @classmethod

From 1fa309da40bfc5e7e72639e80cf6556b3839fc81 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 10 Jun 2016 16:39:31 +0800
Subject: [PATCH 5163/6219] [generic] Update test_Generic_40

The original link now redirects to an YouTube user channel.
---
 youtube_dl/extractor/generic.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 798c109c6..ef18ce3dc 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -627,13 +627,13 @@ class GenericIE(InfoExtractor):
         },
         # MTVSercices embed
         {
-            'url': 'http://www.gametrailers.com/news-post/76093/north-america-europe-is-getting-that-mario-kart-8-mercedes-dlc-too',
-            'md5': '35727f82f58c76d996fc188f9755b0d5',
+            'url': 'http://www.vulture.com/2016/06/new-key-peele-sketches-released.html',
+            'md5': 'ca1aef97695ef2c1d6973256a57e5252',
             'info_dict': {
-                'id': '0306a69b-8adf-4fb5-aace-75f8e8cbfca9',
+                'id': '769f7ec0-0692-4d62-9b45-0d88074bffc1',
                 'ext': 'mp4',
-                'title': 'Review',
-                'description': 'Mario\'s life in the fast lane has never looked so good.',
+                'title': 'Key and Peele|October 10, 2012|2|203|Liam Neesons - Uncensored',
+                'description': 'Two valets share their love for movie star Liam Neesons.',
             },
         },
         # YouTube embed via <data-embed-url="">

From 6c0376fe4f16f53fd87f5e6a56531fc153922980 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 10 Jun 2016 16:53:40 +0800
Subject: [PATCH 5164/6219] [dw] Skip an invalid test

DW documentaries only last for one or two weeks. See #9475
---
 youtube_dl/extractor/dw.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/dw.py b/youtube_dl/extractor/dw.py
index 0f0f0b8d3..d740652f1 100644
--- a/youtube_dl/extractor/dw.py
+++ b/youtube_dl/extractor/dw.py
@@ -35,6 +35,7 @@ class DWIE(InfoExtractor):
             'upload_date': '20160311',
         }
     }, {
+        # DW documentaries, only last for one or two weeks
         'url': 'http://www.dw.com/en/documentaries-welcome-to-the-90s-2016-05-21/e-19220158-9798',
         'md5': '56b6214ef463bfb9a3b71aeb886f3cf1',
         'info_dict': {
@@ -44,6 +45,7 @@ class DWIE(InfoExtractor):
             'description': 'Welcome to the 90s - The Golden Decade of Hip Hop',
             'upload_date': '20160521',
         },
+        'skip': 'Video removed',
     }]
 
     def _real_extract(self, url):

From 836ab0c554f13751adff02d3987f6f3f79e2db09 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 10 Jun 2016 18:12:57 +0800
Subject: [PATCH 5165/6219] [compat] Import html5 entities correctly

---
 youtube_dl/compat.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 0243949a4..67db1c7c6 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -64,8 +64,8 @@ except ImportError:  # Python 2
     import htmlentitydefs as compat_html_entities
 
 try:  # Python >= 3.3
-    from compat_html_entities import html as compat_html_entities_html5
-except ImportError:
+    compat_html_entities_html5 = compat_html_entities.html5
+except AttributeError:
     # Copied from CPython 3.5.1 html/entities.py
     compat_html_entities_html5 = {
         'Aacute': '\xc1',

From bdf16f81403c036a0f40d10a136a46aa7d2f6f0d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Jun 2016 22:40:18 +0700
Subject: [PATCH 5166/6219] [lynda] Add support for new authentication (Closes
 #9740)

---
 youtube_dl/extractor/lynda.py | 115 ++++++++++++++++++----------------
 1 file changed, 60 insertions(+), 55 deletions(-)

diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index 86d47266f..c2678652e 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -1,84 +1,89 @@
 from __future__ import unicode_literals
 
 import re
-import json
 
 from .common import InfoExtractor
-from ..compat import compat_str
+from ..compat import (
+    compat_HTTPError,
+    compat_str,
+    compat_urlparse,
+)
 from ..utils import (
     ExtractorError,
-    clean_html,
     int_or_none,
-    sanitized_Request,
     urlencode_postdata,
 )
 
 
 class LyndaBaseIE(InfoExtractor):
-    _LOGIN_URL = 'https://www.lynda.com/login/login.aspx'
+    _SIGNIN_URL = 'https://www.lynda.com/signin'
+    _PASSWORD_URL = 'https://www.lynda.com/signin/password'
+    _USER_URL = 'https://www.lynda.com/signin/user'
     _ACCOUNT_CREDENTIALS_HINT = 'Use --username and --password options to provide lynda.com account credentials.'
     _NETRC_MACHINE = 'lynda'
 
     def _real_initialize(self):
         self._login()
 
+    @staticmethod
+    def _check_error(json_string, key_or_keys):
+        keys = [key_or_keys] if isinstance(key_or_keys, compat_str) else key_or_keys
+        for key in keys:
+            error = json_string.get(key)
+            if error:
+                raise ExtractorError('Unable to login: %s' % error, expected=True)
+
+    def _login_step(self, form_html, fallback_action_url, extra_form_data, note, referrer_url):
+        action_url = self._search_regex(
+            r'<form[^>]+action=(["\'])(?P<url>.+?)\1', form_html,
+            'post url', default=fallback_action_url, group='url')
+
+        if not action_url.startswith('http'):
+            action_url = compat_urlparse.urljoin(self._SIGNIN_URL, action_url)
+
+        form_data = self._hidden_inputs(form_html)
+        form_data.update(extra_form_data)
+
+        try:
+            response = self._download_json(
+                action_url, None, note,
+                data=urlencode_postdata(form_data),
+                headers={
+                    'Referer': referrer_url,
+                    'X-Requested-With': 'XMLHttpRequest',
+                })
+        except ExtractorError as e:
+            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 500:
+                response = self._parse_json(e.cause.read().decode('utf-8'), None)
+                self._check_error(response, ('email', 'password'))
+            raise
+
+        self._check_error(response, 'ErrorMessage')
+
+        return response, action_url
+
     def _login(self):
         username, password = self._get_login_info()
         if username is None:
             return
 
-        login_form = {
-            'username': username,
-            'password': password,
-            'remember': 'false',
-            'stayPut': 'false'
-        }
-        request = sanitized_Request(
-            self._LOGIN_URL, urlencode_postdata(login_form))
-        login_page = self._download_webpage(
-            request, None, 'Logging in as %s' % username)
+        # Step 1: download signin page
+        signin_page = self._download_webpage(
+            self._SIGNIN_URL, None, 'Downloading signin page')
 
-        # Not (yet) logged in
-        m = re.search(r'loginResultJson\s*=\s*\'(?P<json>[^\']+)\';', login_page)
-        if m is not None:
-            response = m.group('json')
-            response_json = json.loads(response)
-            state = response_json['state']
+        # Step 2: submit email
+        signin_form = self._search_regex(
+            r'(?s)(<form[^>]+data-form-name=["\']signin["\'][^>]*>.+?</form>)',
+            signin_page, 'signin form')
+        signin_page, signin_url = self._login_step(
+            signin_form, self._PASSWORD_URL, {'email': username},
+            'Submitting email', self._SIGNIN_URL)
 
-            if state == 'notlogged':
-                raise ExtractorError(
-                    'Unable to login, incorrect username and/or password',
-                    expected=True)
-
-            # This is when we get popup:
-            # > You're already logged in to lynda.com on two devices.
-            # > If you log in here, we'll log you out of another device.
-            # So, we need to confirm this.
-            if state == 'conflicted':
-                confirm_form = {
-                    'username': '',
-                    'password': '',
-                    'resolve': 'true',
-                    'remember': 'false',
-                    'stayPut': 'false',
-                }
-                request = sanitized_Request(
-                    self._LOGIN_URL, urlencode_postdata(confirm_form))
-                login_page = self._download_webpage(
-                    request, None,
-                    'Confirming log in and log out from another device')
-
-        if all(not re.search(p, login_page) for p in ('isLoggedIn\s*:\s*true', r'logout\.aspx', r'>Log out<')):
-            if 'login error' in login_page:
-                mobj = re.search(
-                    r'(?s)<h1[^>]+class="topmost">(?P<title>[^<]+)</h1>\s*<div>(?P<description>.+?)</div>',
-                    login_page)
-                if mobj:
-                    raise ExtractorError(
-                        'lynda returned error: %s - %s'
-                        % (mobj.group('title'), clean_html(mobj.group('description'))),
-                        expected=True)
-            raise ExtractorError('Unable to log in')
+        # Step 3: submit password
+        password_form = signin_page['body']
+        self._login_step(
+            password_form, self._USER_URL, {'email': username, 'password': password},
+            'Submitting password', signin_url)
 
     def _logout(self):
         username, _ = self._get_login_info()

From 3841256c2c5fd35229cd8f2c2c8a8e2401f7016b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Jun 2016 23:01:52 +0700
Subject: [PATCH 5167/6219] [lynda] Skip login if already logged in

---
 youtube_dl/extractor/lynda.py | 16 +++++-----------
 1 file changed, 5 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index c2678652e..7610985b4 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -71,6 +71,11 @@ class LyndaBaseIE(InfoExtractor):
         signin_page = self._download_webpage(
             self._SIGNIN_URL, None, 'Downloading signin page')
 
+        # Already logged in
+        if any(re.search(p, signin_page) for p in (
+                'isLoggedIn\s*:\s*true', r'logout\.aspx', r'>Log out<')):
+            return
+
         # Step 2: submit email
         signin_form = self._search_regex(
             r'(?s)(<form[^>]+data-form-name=["\']signin["\'][^>]*>.+?</form>)',
@@ -85,15 +90,6 @@ class LyndaBaseIE(InfoExtractor):
             password_form, self._USER_URL, {'email': username, 'password': password},
             'Submitting password', signin_url)
 
-    def _logout(self):
-        username, _ = self._get_login_info()
-        if username is None:
-            return
-
-        self._download_webpage(
-            'http://www.lynda.com/ajax/logout.aspx', None,
-            'Logging out', 'Unable to log out', fatal=False)
-
 
 class LyndaIE(LyndaBaseIE):
     IE_NAME = 'lynda'
@@ -217,8 +213,6 @@ class LyndaCourseIE(LyndaBaseIE):
             'http://www.lynda.com/ajax/player?courseId=%s&type=course' % course_id,
             course_id, 'Downloading course JSON')
 
-        self._logout()
-
         if course.get('Status') == 'NotFound':
             raise ExtractorError(
                 'Course %s does not exist' % course_id, expected=True)

From 0434358823a9b7da7656f3e6d8de28d1b42036f5 Mon Sep 17 00:00:00 2001
From: TRox1972 <TRox1972@users.noreply.github.com>
Date: Fri, 10 Jun 2016 19:17:58 +0200
Subject: [PATCH 5168/6219] [Lynda] Extract course description

---
 youtube_dl/extractor/lynda.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index 86d47266f..c1bca5678 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -246,5 +246,6 @@ class LyndaCourseIE(LyndaBaseIE):
                 % unaccessible_videos + self._ACCOUNT_CREDENTIALS_HINT)
 
         course_title = course.get('Title')
+        course_description = course.get('Description')
 
-        return self.playlist_result(entries, course_id, course_title)
+        return self.playlist_result(entries, course_id, course_title, course_description)

From d845622b2e09ebac28e21f76f6d5c2795aa9bb50 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 11 Jun 2016 02:41:48 +0700
Subject: [PATCH 5169/6219] release 2016.06.11

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 README.md                 | 2 +-
 docs/supportedsites.md    | 9 +++++----
 youtube_dl/version.py     | 2 +-
 4 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index e593ee78a..16ef23066 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.06.03*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.06.03**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.06.11*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.06.11**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.06.03
+[debug] youtube-dl version 2016.06.11
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/README.md b/README.md
index 205c485d0..2ea8acb30 100644
--- a/README.md
+++ b/README.md
@@ -255,7 +255,7 @@ which means you can modify it, redistribute it or use it however you like.
     --write-info-json                Write video metadata to a .info.json file
     --write-annotations              Write video annotations to a
                                      .annotations.xml file
-    --load-info FILE                 JSON file containing the video information
+    --load-info-json FILE            JSON file containing the video information
                                      (created with the "--write-info-json"
                                      option)
     --cookies FILE                   File to read cookies from and dump cookie
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 619bd0825..f89c2d1f2 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -28,6 +28,7 @@
  - **AdobeTVVideo**
  - **AdultSwim**
  - **aenetworks**: A+E Networks: A&E, Lifetime, History.com, FYI Network
+ - **AfreecaTV**: afreecatv.com
  - **Aftonbladet**
  - **AirMozilla**
  - **AlJazeera**
@@ -43,8 +44,8 @@
  - **appletrailers:section**
  - **archive.org**: archive.org videos
  - **ARD**
- - **ARD:mediathek**: Saarländischer Rundfunk
  - **ARD:mediathek**
+ - **ARD:mediathek**: Saarländischer Rundfunk
  - **arte.tv**
  - **arte.tv:+7**
  - **arte.tv:cinema**
@@ -253,6 +254,7 @@
  - **Globo**
  - **GloboArticle**
  - **GodTube**
+ - **GodTV**
  - **GoldenMoustache**
  - **Golem**
  - **GoogleDrive**
@@ -738,6 +740,7 @@
  - **VideoPremium**
  - **VideoTt**: video.tt - Your True Tube (Currently broken)
  - **videoweed**: VideoWeed
+ - **Vidio**
  - **vidme**
  - **vidme:user**
  - **vidme:user:likes**
@@ -773,7 +776,6 @@
  - **VRT**
  - **vube**: Vube.com
  - **VuClip**
- - **vulture.com**
  - **Walla**
  - **washingtonpost**
  - **washingtonpost:article**
@@ -781,10 +783,8 @@
  - **WatchIndianPorn**: Watch Indian Porn
  - **WDR**
  - **wdr:mobile**
- - **WDRMaus**: Sendung mit der Maus
  - **WebOfStories**
  - **WebOfStoriesPlaylist**
- - **Weibo**
  - **WeiqiTV**: WQTV
  - **wholecloud**: WholeCloud
  - **Wimp**
@@ -820,6 +820,7 @@
  - **Ynet**
  - **YouJizz**
  - **youku**: 优酷
+ - **youku:show**
  - **YouPorn**
  - **YourUpload**
  - **youtube**: YouTube.com
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index d24d06f4a..dafb6513a 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.06.03'
+__version__ = '2016.06.11'

From 6626c214e1e0fa422d68b875cbb69dfb5aad8745 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 11 Jun 2016 03:00:08 +0700
Subject: [PATCH 5170/6219] release 2016.06.11.1

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 youtube_dl/version.py     | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 16ef23066..564cffae7 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.06.11*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.06.11**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.06.11.1*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.06.11.1**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.06.11
+[debug] youtube-dl version 2016.06.11.1
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index dafb6513a..5bcb6a7b3 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.06.11'
+__version__ = '2016.06.11.1'

From 9ddc289f88542f4b0bf7ad5e9c725caf8889f71b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 11 Jun 2016 04:59:47 +0700
Subject: [PATCH 5171/6219] [README.md] Document missing playlist fields in
 output template

---
 README.md | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/README.md b/README.md
index 2ea8acb30..3ff33c156 100644
--- a/README.md
+++ b/README.md
@@ -511,6 +511,9 @@ The basic usage is not to set any template arguments when downloading a single f
  - `autonumber`: Five-digit number that will be increased with each download, starting at zero
  - `playlist`: Name or id of the playlist that contains the video
  - `playlist_index`: Index of the video in the playlist padded with leading zeros according to the total length of the playlist
+ - `playlist_id`: Playlist identifier
+ - `playlist_title`: Playlist title
+
 
 Available for the video that belongs to some logical chapter or section:
  - `chapter`: Name or title of the chapter the video belongs to

From 62666af99fb55e3ba535ce630e8ce0aed1b5b0e8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 11 Jun 2016 05:13:05 +0700
Subject: [PATCH 5172/6219] [indavideo] Fix formats' height (Closes #9744)

---
 youtube_dl/extractor/indavideo.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/indavideo.py b/youtube_dl/extractor/indavideo.py
index 9622f198a..c6f080484 100644
--- a/youtube_dl/extractor/indavideo.py
+++ b/youtube_dl/extractor/indavideo.py
@@ -60,7 +60,8 @@ class IndavideoEmbedIE(InfoExtractor):
 
         formats = [{
             'url': video_url,
-            'height': self._search_regex(r'\.(\d{3,4})\.mp4$', video_url, 'height', default=None),
+            'height': int_or_none(self._search_regex(
+                r'\.(\d{3,4})\.mp4(?:\?|$)', video_url, 'height', default=None)),
         } for video_url in video_urls]
         self._sort_formats(formats)
 

From 4cad2929cd7e90be174ae6b0ad0c7d9f47795374 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 11 Jun 2016 05:30:44 +0700
Subject: [PATCH 5173/6219] [limelight] Fix _VALID_URLs

---
 youtube_dl/extractor/limelight.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/limelight.py b/youtube_dl/extractor/limelight.py
index 2599d45c3..8dbc940a7 100644
--- a/youtube_dl/extractor/limelight.py
+++ b/youtube_dl/extractor/limelight.py
@@ -123,7 +123,7 @@ class LimelightBaseIE(InfoExtractor):
 
 class LimelightMediaIE(LimelightBaseIE):
     IE_NAME = 'limelight'
-    _VALID_URL = r'(?:limelight:media:|https?://link\.videoplatform\.limelight\.com/media/\??\bmediaId=)(?P<id>[a-z0-9]{32})'
+    _VALID_URL = r'(?:limelight:media:|https?://link\.videoplatform\.limelight\.com/media/\?.*?\bmediaId=)(?P<id>[a-z0-9]{32})'
     _TESTS = [{
         'url': 'http://link.videoplatform.limelight.com/media/?mediaId=3ffd040b522b4485b6d84effc750cd86',
         'info_dict': {
@@ -176,7 +176,7 @@ class LimelightMediaIE(LimelightBaseIE):
 
 class LimelightChannelIE(LimelightBaseIE):
     IE_NAME = 'limelight:channel'
-    _VALID_URL = r'(?:limelight:channel:|https?://link\.videoplatform\.limelight\.com/media/\??\bchannelId=)(?P<id>[a-z0-9]{32})'
+    _VALID_URL = r'(?:limelight:channel:|https?://link\.videoplatform\.limelight\.com/media/\?.*?\bchannelId=)(?P<id>[a-z0-9]{32})'
     _TEST = {
         'url': 'http://link.videoplatform.limelight.com/media/?channelId=ab6a524c379342f9b23642917020c082',
         'info_dict': {

From 79027c0ea02d4f296aefe6ca6e5af393c2a4a209 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 11 Jun 2016 05:40:02 +0700
Subject: [PATCH 5174/6219] [limelight] Improve _VALID_URLs

---
 youtube_dl/extractor/limelight.py | 56 +++++++++++++++++++++++++++----
 1 file changed, 49 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/limelight.py b/youtube_dl/extractor/limelight.py
index 8dbc940a7..da5d198b9 100644
--- a/youtube_dl/extractor/limelight.py
+++ b/youtube_dl/extractor/limelight.py
@@ -123,7 +123,18 @@ class LimelightBaseIE(InfoExtractor):
 
 class LimelightMediaIE(LimelightBaseIE):
     IE_NAME = 'limelight'
-    _VALID_URL = r'(?:limelight:media:|https?://link\.videoplatform\.limelight\.com/media/\?.*?\bmediaId=)(?P<id>[a-z0-9]{32})'
+    _VALID_URL = r'''(?x)
+                        (?:
+                            limelight:media:|
+                            https?://
+                                (?:
+                                    link\.videoplatform\.limelight\.com/media/|
+                                    assets\.delvenetworks\.com/player/loader\.swf
+                                )
+                                \?.*?\bmediaId=
+                        )
+                        (?P<id>[a-z0-9]{32})
+                    '''
     _TESTS = [{
         'url': 'http://link.videoplatform.limelight.com/media/?mediaId=3ffd040b522b4485b6d84effc750cd86',
         'info_dict': {
@@ -158,6 +169,9 @@ class LimelightMediaIE(LimelightBaseIE):
             # rtmp download
             'skip_download': True,
         },
+    }, {
+        'url': 'https://assets.delvenetworks.com/player/loader.swf?mediaId=8018a574f08d416e95ceaccae4ba0452',
+        'only_matching': True,
     }]
     _PLAYLIST_SERVICE_PATH = 'media'
     _API_PATH = 'media'
@@ -176,15 +190,29 @@ class LimelightMediaIE(LimelightBaseIE):
 
 class LimelightChannelIE(LimelightBaseIE):
     IE_NAME = 'limelight:channel'
-    _VALID_URL = r'(?:limelight:channel:|https?://link\.videoplatform\.limelight\.com/media/\?.*?\bchannelId=)(?P<id>[a-z0-9]{32})'
-    _TEST = {
+    _VALID_URL = r'''(?x)
+                        (?:
+                            limelight:channel:|
+                            https?://
+                                (?:
+                                    link\.videoplatform\.limelight\.com/media/|
+                                    assets\.delvenetworks\.com/player/loader\.swf
+                                )
+                                \?.*?\bchannelId=
+                        )
+                        (?P<id>[a-z0-9]{32})
+                    '''
+    _TESTS = [{
         'url': 'http://link.videoplatform.limelight.com/media/?channelId=ab6a524c379342f9b23642917020c082',
         'info_dict': {
             'id': 'ab6a524c379342f9b23642917020c082',
             'title': 'Javascript Sample Code',
         },
         'playlist_mincount': 3,
-    }
+    }, {
+        'url': 'http://assets.delvenetworks.com/player/loader.swf?channelId=ab6a524c379342f9b23642917020c082',
+        'only_matching': True,
+    }]
     _PLAYLIST_SERVICE_PATH = 'channel'
     _API_PATH = 'channels'
 
@@ -207,15 +235,29 @@ class LimelightChannelIE(LimelightBaseIE):
 
 class LimelightChannelListIE(LimelightBaseIE):
     IE_NAME = 'limelight:channel_list'
-    _VALID_URL = r'(?:limelight:channel_list:|https?://link\.videoplatform\.limelight\.com/media/\?.*?\bchannelListId=)(?P<id>[a-z0-9]{32})'
-    _TEST = {
+    _VALID_URL = r'''(?x)
+                        (?:
+                            limelight:channel_list:|
+                            https?://
+                                (?:
+                                    link\.videoplatform\.limelight\.com/media/|
+                                    assets\.delvenetworks\.com/player/loader\.swf
+                                )
+                                \?.*?\bchannelListId=
+                        )
+                        (?P<id>[a-z0-9]{32})
+                    '''
+    _TESTS = [{
         'url': 'http://link.videoplatform.limelight.com/media/?channelListId=301b117890c4465c8179ede21fd92e2b',
         'info_dict': {
             'id': '301b117890c4465c8179ede21fd92e2b',
             'title': 'Website - Hero Player',
         },
         'playlist_mincount': 2,
-    }
+    }, {
+        'url': 'https://assets.delvenetworks.com/player/loader.swf?channelListId=301b117890c4465c8179ede21fd92e2b',
+        'only_matching': True,
+    }]
     _PLAYLIST_SERVICE_PATH = 'channel_list'
 
     def _real_extract(self, url):

From 21ac1a8ac3f2a3c301ad8c08730166a8fd82c287 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 11 Jun 2016 05:52:50 +0700
Subject: [PATCH 5175/6219] [limelight] Fix typo

---
 youtube_dl/extractor/limelight.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/limelight.py b/youtube_dl/extractor/limelight.py
index da5d198b9..a25fb8e2c 100644
--- a/youtube_dl/extractor/limelight.py
+++ b/youtube_dl/extractor/limelight.py
@@ -98,7 +98,7 @@ class LimelightBaseIE(InfoExtractor):
         } for thumbnail in properties.get('thumbnails', []) if thumbnail.get('url')]
 
         subtitles = {}
-        for caption in properties.get('captions', {}):
+        for caption in properties.get('captions', []):
             lang = caption.get('language_code')
             subtitles_url = caption.get('url')
             if lang and subtitles_url:

From fe458b65965e5a847a24d00138b723ce67b274e2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 11 Jun 2016 05:57:27 +0700
Subject: [PATCH 5176/6219] [limelight] Extract ttml subtitles (Closes #9739)

---
 youtube_dl/extractor/limelight.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/limelight.py b/youtube_dl/extractor/limelight.py
index a25fb8e2c..5d2c3e256 100644
--- a/youtube_dl/extractor/limelight.py
+++ b/youtube_dl/extractor/limelight.py
@@ -102,9 +102,15 @@ class LimelightBaseIE(InfoExtractor):
             lang = caption.get('language_code')
             subtitles_url = caption.get('url')
             if lang and subtitles_url:
-                subtitles[lang] = [{
+                subtitles.setdefault(lang, []).append({
                     'url': subtitles_url,
-                }]
+                })
+        closed_captions_url = properties.get('closed_captions_url')
+        if closed_captions_url:
+            subtitles.setdefault('en', []).append({
+                'url': closed_captions_url,
+                'ext': 'ttml',
+            })
 
         return {
             'id': video_id,

From 698f127c1a9dd460c8dede59df6a0e2ce69f913a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 11 Jun 2016 06:14:22 +0700
Subject: [PATCH 5177/6219] [setup.py] Add python 3.5 classifier

---
 setup.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/setup.py b/setup.py
index 9444d403d..c1e923f71 100644
--- a/setup.py
+++ b/setup.py
@@ -122,6 +122,7 @@ setup(
         "Programming Language :: Python :: 3.2",
         "Programming Language :: Python :: 3.3",
         "Programming Language :: Python :: 3.4",
+        "Programming Language :: Python :: 3.5",
     ],
 
     cmdclass={'build_lazy_extractors': build_lazy_extractors},

From 33751818d3e31270304db519849d85bec43e9c95 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 11 Jun 2016 08:28:51 +0700
Subject: [PATCH 5178/6219] release 2016.06.11.2

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 youtube_dl/version.py     | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 564cffae7..8fa97ee87 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.06.11.1*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.06.11.1**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.06.11.2*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.06.11.2**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.06.11.1
+[debug] youtube-dl version 2016.06.11.2
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 5bcb6a7b3..f6cc8b79e 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.06.11.1'
+__version__ = '2016.06.11.2'

From 4a420119a6e0b7363f9d31e37d3e7af818bedfd9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 11 Jun 2016 08:34:30 +0700
Subject: [PATCH 5179/6219] release 2016.06.11.3

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 youtube_dl/version.py     | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 8fa97ee87..a46b75fd8 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.06.11.2*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.06.11.2**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.06.11.3*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.06.11.3**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.06.11.2
+[debug] youtube-dl version 2016.06.11.3
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index f6cc8b79e..9932b1e62 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.06.11.2'
+__version__ = '2016.06.11.3'

From 47787efa2b6bd5dc1b6f6cb7027586bac2de4c6c Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 11 Jun 2016 13:13:16 +0800
Subject: [PATCH 5180/6219] [leeco] Recognize Le Sports URLs (fixes #9750)

---
 youtube_dl/extractor/leeco.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/leeco.py b/youtube_dl/extractor/leeco.py
index 375fdaed1..63f581cd9 100644
--- a/youtube_dl/extractor/leeco.py
+++ b/youtube_dl/extractor/leeco.py
@@ -28,7 +28,7 @@ from ..utils import (
 
 class LeIE(InfoExtractor):
     IE_DESC = '乐视网'
-    _VALID_URL = r'https?://www\.le\.com/ptv/vplay/(?P<id>\d+)\.html'
+    _VALID_URL = r'https?://(?:www\.le\.com/ptv/vplay|sports\.le\.com/video)/(?P<id>\d+)\.html'
 
     _URL_TEMPLATE = 'http://www.le.com/ptv/vplay/%s.html'
 
@@ -69,6 +69,9 @@ class LeIE(InfoExtractor):
             'hls_prefer_native': True,
         },
         'skip': 'Only available in China',
+    }, {
+        'url': 'http://sports.le.com/video/25737697.html',
+        'only_matching': True,
     }]
 
     @staticmethod
@@ -196,7 +199,7 @@ class LeIE(InfoExtractor):
 
 
 class LePlaylistIE(InfoExtractor):
-    _VALID_URL = r'https?://[a-z]+\.le\.com/[a-z]+/(?P<id>[a-z0-9_]+)'
+    _VALID_URL = r'https?://[a-z]+\.le\.com/(?!video)[a-z]+/(?P<id>[a-z0-9_]+)'
 
     _TESTS = [{
         'url': 'http://www.le.com/tv/46177.html',

From 7aab3696dd02ca45feba523b4194d6430939dd1c Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 11 Jun 2016 15:37:04 +0800
Subject: [PATCH 5181/6219] [kuwo] Update _TESTS

---
 youtube_dl/extractor/kuwo.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/kuwo.py b/youtube_dl/extractor/kuwo.py
index 11b31a699..0221fb919 100644
--- a/youtube_dl/extractor/kuwo.py
+++ b/youtube_dl/extractor/kuwo.py
@@ -148,8 +148,8 @@ class KuwoAlbumIE(InfoExtractor):
         'url': 'http://www.kuwo.cn/album/502294/',
         'info_dict': {
             'id': '502294',
-            'title': 'M',
-            'description': 'md5:6a7235a84cc6400ec3b38a7bdaf1d60c',
+            'title': 'Made\xa0Series\xa0《M》',
+            'description': 'md5:d463f0d8a0ff3c3ea3d6ed7452a9483f',
         },
         'playlist_count': 2,
     }
@@ -209,7 +209,7 @@ class KuwoSingerIE(InfoExtractor):
         'url': 'http://www.kuwo.cn/mingxing/bruno+mars/',
         'info_dict': {
             'id': 'bruno+mars',
-            'title': 'Bruno Mars',
+            'title': 'Bruno\xa0Mars',
         },
         'playlist_mincount': 329,
     }, {
@@ -306,7 +306,7 @@ class KuwoMvIE(KuwoBaseIE):
             'id': '6480076',
             'ext': 'mp4',
             'title': 'My HouseMV',
-            'creator': '2PM',
+            'creator': 'PM02:00',
         },
         # In this video, music URLs (anti.s) are blocked outside China and
         # USA, while the MV URL (mvurl) is available globally, so force the MV

From 15d106787e8c21e4d4df95957062bd07c873d203 Mon Sep 17 00:00:00 2001
From: Paul Henning <vxbinaca@users.noreply.github.com>
Date: Sat, 11 Jun 2016 05:36:31 -0400
Subject: [PATCH 5182/6219] [utils] Change Firefox 44 to 47

See commit title.
---
 youtube_dl/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index f77ab8650..0acbd67de 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -76,7 +76,7 @@ def register_socks_protocols():
 compiled_regex_type = type(re.compile(''))
 
 std_headers = {
-    'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20150101 Firefox/44.0 (Chrome)',
+    'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20150101 Firefox/47.0 (Chrome)',
     'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
     'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
     'Accept-Encoding': 'gzip, deflate',

From 856150d05647904a5cf6c519c6e276ce3536bd20 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 11 Jun 2016 18:22:26 +0800
Subject: [PATCH 5183/6219] [telewebion] Add new extractor (closes #5135)

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/telewebion.py | 55 ++++++++++++++++++++++++++++++
 2 files changed, 56 insertions(+)
 create mode 100644 youtube_dl/extractor/telewebion.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 38708294a..36ddc1f73 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -777,6 +777,7 @@ from .telecinco import TelecincoIE
 from .telegraaf import TelegraafIE
 from .telemb import TeleMBIE
 from .teletask import TeleTaskIE
+from .telewebion import TelewebionIE
 from .testurl import TestURLIE
 from .tf1 import TF1IE
 from .theintercept import TheInterceptIE
diff --git a/youtube_dl/extractor/telewebion.py b/youtube_dl/extractor/telewebion.py
new file mode 100644
index 000000000..77916c601
--- /dev/null
+++ b/youtube_dl/extractor/telewebion.py
@@ -0,0 +1,55 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class TelewebionIE(InfoExtractor):
+    _VALID_URL = r'https?://www\.telewebion\.com/#!/episode/(?P<id>\d+)'
+
+    _TEST = {
+        'url': 'http://www.telewebion.com/#!/episode/1263668/',
+        'info_dict': {
+            'id': '1263668',
+            'ext': 'mp4',
+            'title': 'قرعه\u200cکشی لیگ قهرمانان اروپا',
+            'thumbnail': 're:^https?://.*\.jpg',
+            'view_count': int,
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        secure_token = self._download_webpage(
+            'http://m.s2.telewebion.com/op/op?action=getSecurityToken', video_id)
+        episode_details = self._download_json(
+            'http://m.s2.telewebion.com/op/op', video_id,
+            query={'action': 'getEpisodeDetails', 'episode_id': video_id})
+
+        m3u8_url = 'http://m.s1.telewebion.com/smil/%s.m3u8?filepath=%s&m3u8=1&secure_token=%s' % (
+            video_id, episode_details['file_path'], secure_token)
+        formats = self._extract_m3u8_formats(
+            m3u8_url, video_id, ext='mp4', m3u8_id='hls')
+
+        picture_paths = [
+            episode_details.get('picture_path'),
+            episode_details.get('large_picture_path'),
+        ]
+
+        thumbnails = [{
+            'url': picture_path,
+            'preference': idx,
+        } for idx, picture_path in enumerate(picture_paths) if picture_path is not None]
+
+        return {
+            'id': video_id,
+            'title': episode_details['title'],
+            'formats': formats,
+            'thumbnails': thumbnails,
+            'view_count': episode_details.get('view_count'),
+        }

From c5edd147d1d2cf0502f5ef48652c88a75ef62529 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 11 Jun 2016 18:33:37 +0800
Subject: [PATCH 5184/6219] [generic] Remove an invalid test

Now handled by telewebion.py
---
 youtube_dl/extractor/generic.py | 14 --------------
 1 file changed, 14 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index ef18ce3dc..4aa24061c 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1073,20 +1073,6 @@ class GenericIE(InfoExtractor):
                 'skip_download': True,
             }
         },
-        # Contains a SMIL manifest
-        {
-            'url': 'http://www.telewebion.com/fa/1263668/%D9%82%D8%B1%D8%B9%D9%87%E2%80%8C%DA%A9%D8%B4%DB%8C-%D9%84%DB%8C%DA%AF-%D9%82%D9%87%D8%B1%D9%85%D8%A7%D9%86%D8%A7%D9%86-%D8%A7%D8%B1%D9%88%D9%BE%D8%A7/%2B-%D9%81%D9%88%D8%AA%D8%A8%D8%A7%D9%84.html',
-            'info_dict': {
-                'id': 'file',
-                'ext': 'flv',
-                'title': '+ Football: Lottery Champions League Europe',
-                'uploader': 'www.telewebion.com',
-            },
-            'params': {
-                # rtmpe downloads
-                'skip_download': True,
-            }
-        },
         # Brightcove URL in single quotes
         {
             'url': 'http://www.sportsnet.ca/baseball/mlb/sn-presents-russell-martin-world-citizen/',

From 531a74968c24416cb2e4a79c9bfbcc9d02368e44 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 11 Jun 2016 21:35:08 +0800
Subject: [PATCH 5185/6219] [vimeo] Fix extraction for VimeoReview videos

---
 youtube_dl/extractor/vimeo.py | 147 +++++++++++++++++++---------------
 1 file changed, 83 insertions(+), 64 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 59f9cb1ae..0fd2c18a0 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -66,6 +66,69 @@ class VimeoBaseInfoExtractor(InfoExtractor):
     def _set_vimeo_cookie(self, name, value):
         self._set_cookie('vimeo.com', name, value)
 
+    def _vimeo_sort_formats(self, formats):
+        # Bitrates are completely broken. Single m3u8 may contain entries in kbps and bps
+        # at the same time without actual units specified. This lead to wrong sorting.
+        self._sort_formats(formats, field_preference=('preference', 'height', 'width', 'fps', 'format_id'))
+
+    def _parse_config(self, config, video_id):
+        # Extract title
+        video_title = config['video']['title']
+
+        # Extract uploader, uploader_url and uploader_id
+        video_uploader = config['video'].get('owner', {}).get('name')
+        video_uploader_url = config['video'].get('owner', {}).get('url')
+        video_uploader_id = video_uploader_url.split('/')[-1] if video_uploader_url else None
+
+        # Extract video thumbnail
+        video_thumbnail = config['video'].get('thumbnail')
+        if video_thumbnail is None:
+            video_thumbs = config['video'].get('thumbs')
+            if video_thumbs and isinstance(video_thumbs, dict):
+                _, video_thumbnail = sorted((int(width if width.isdigit() else 0), t_url) for (width, t_url) in video_thumbs.items())[-1]
+
+        # Extract video duration
+        video_duration = int_or_none(config['video'].get('duration'))
+
+        formats = []
+        config_files = config['video'].get('files') or config['request'].get('files', {})
+        for f in config_files.get('progressive', []):
+            video_url = f.get('url')
+            if not video_url:
+                continue
+            formats.append({
+                'url': video_url,
+                'format_id': 'http-%s' % f.get('quality'),
+                'width': int_or_none(f.get('width')),
+                'height': int_or_none(f.get('height')),
+                'fps': int_or_none(f.get('fps')),
+                'tbr': int_or_none(f.get('bitrate')),
+            })
+        m3u8_url = config_files.get('hls', {}).get('url')
+        if m3u8_url:
+            formats.extend(self._extract_m3u8_formats(
+                m3u8_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
+
+        subtitles = {}
+        text_tracks = config['request'].get('text_tracks')
+        if text_tracks:
+            for tt in text_tracks:
+                subtitles[tt['lang']] = [{
+                    'ext': 'vtt',
+                    'url': 'https://vimeo.com' + tt['url'],
+                }]
+
+        return {
+            'title': video_title,
+            'uploader': video_uploader,
+            'uploader_id': video_uploader_id,
+            'uploader_url': video_uploader_url,
+            'thumbnail': video_thumbnail,
+            'duration': video_duration,
+            'formats': formats,
+            'subtitles': subtitles,
+        }
+
 
 class VimeoIE(VimeoBaseInfoExtractor):
     """Information extractor for vimeo.com."""
@@ -153,7 +216,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
                 'uploader_id': 'user18948128',
                 'uploader': 'Jaime Marquínez Ferrándiz',
                 'duration': 10,
-                'description': 'This is "youtube-dl password protected test video" by Jaime Marquínez Ferrándiz on Vimeo, the home for high quality videos and the people\u2026',
+                'description': 'This is "youtube-dl password protected test video" by  on Vimeo, the home for high quality videos and the people who love them.',
             },
             'params': {
                 'videopassword': 'youtube-dl',
@@ -389,21 +452,6 @@ class VimeoIE(VimeoBaseInfoExtractor):
                     'https://player.vimeo.com/player/%s' % feature_id,
                     {'force_feature_id': True}), 'Vimeo')
 
-        # Extract title
-        video_title = config['video']['title']
-
-        # Extract uploader, uploader_url and uploader_id
-        video_uploader = config['video'].get('owner', {}).get('name')
-        video_uploader_url = config['video'].get('owner', {}).get('url')
-        video_uploader_id = video_uploader_url.split('/')[-1] if video_uploader_url else None
-
-        # Extract video thumbnail
-        video_thumbnail = config['video'].get('thumbnail')
-        if video_thumbnail is None:
-            video_thumbs = config['video'].get('thumbs')
-            if video_thumbs and isinstance(video_thumbs, dict):
-                _, video_thumbnail = sorted((int(width if width.isdigit() else 0), t_url) for (width, t_url) in video_thumbs.items())[-1]
-
         # Extract video description
 
         video_description = self._html_search_regex(
@@ -423,9 +471,6 @@ class VimeoIE(VimeoBaseInfoExtractor):
         if not video_description and not mobj.group('player'):
             self._downloader.report_warning('Cannot find video description')
 
-        # Extract video duration
-        video_duration = int_or_none(config['video'].get('duration'))
-
         # Extract upload date
         video_upload_date = None
         mobj = re.search(r'<time[^>]+datetime="([^"]+)"', webpage)
@@ -463,53 +508,22 @@ class VimeoIE(VimeoBaseInfoExtractor):
                             'format_id': source_name,
                             'preference': 1,
                         })
-        config_files = config['video'].get('files') or config['request'].get('files', {})
-        for f in config_files.get('progressive', []):
-            video_url = f.get('url')
-            if not video_url:
-                continue
-            formats.append({
-                'url': video_url,
-                'format_id': 'http-%s' % f.get('quality'),
-                'width': int_or_none(f.get('width')),
-                'height': int_or_none(f.get('height')),
-                'fps': int_or_none(f.get('fps')),
-                'tbr': int_or_none(f.get('bitrate')),
-            })
-        m3u8_url = config_files.get('hls', {}).get('url')
-        if m3u8_url:
-            formats.extend(self._extract_m3u8_formats(
-                m3u8_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
-        # Bitrates are completely broken. Single m3u8 may contain entries in kbps and bps
-        # at the same time without actual units specified. This lead to wrong sorting.
-        self._sort_formats(formats, field_preference=('preference', 'height', 'width', 'fps', 'format_id'))
 
-        subtitles = {}
-        text_tracks = config['request'].get('text_tracks')
-        if text_tracks:
-            for tt in text_tracks:
-                subtitles[tt['lang']] = [{
-                    'ext': 'vtt',
-                    'url': 'https://vimeo.com' + tt['url'],
-                }]
-
-        return {
+        info_dict = self._parse_config(config, video_id)
+        formats.extend(info_dict['formats'])
+        self._vimeo_sort_formats(formats)
+        info_dict.update({
             'id': video_id,
-            'uploader': video_uploader,
-            'uploader_url': video_uploader_url,
-            'uploader_id': video_uploader_id,
-            'upload_date': video_upload_date,
-            'title': video_title,
-            'thumbnail': video_thumbnail,
-            'description': video_description,
-            'duration': video_duration,
             'formats': formats,
+            'upload_date': video_upload_date,
+            'description': video_description,
             'webpage_url': url,
             'view_count': view_count,
             'like_count': like_count,
             'comment_count': comment_count,
-            'subtitles': subtitles,
-        }
+        })
+
+        return info_dict
 
 
 class VimeoOndemandIE(VimeoBaseInfoExtractor):
@@ -692,7 +706,7 @@ class VimeoGroupsIE(VimeoAlbumIE):
         return self._extract_videos(name, 'https://vimeo.com/groups/%s' % name)
 
 
-class VimeoReviewIE(InfoExtractor):
+class VimeoReviewIE(VimeoBaseInfoExtractor):
     IE_NAME = 'vimeo:review'
     IE_DESC = 'Review pages on vimeo'
     _VALID_URL = r'https://vimeo\.com/[^/]+/review/(?P<id>[^/]+)'
@@ -704,6 +718,7 @@ class VimeoReviewIE(InfoExtractor):
             'ext': 'mp4',
             'title': "DICK HARDWICK 'Comedian'",
             'uploader': 'Richard Hardwick',
+            'uploader_id': 'user21297594',
         }
     }, {
         'note': 'video player needs Referer',
@@ -716,14 +731,18 @@ class VimeoReviewIE(InfoExtractor):
             'uploader': 'DevWeek Events',
             'duration': 2773,
             'thumbnail': 're:^https?://.*\.jpg$',
+            'uploader_id': 'user22258446',
         }
     }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        player_url = 'https://player.vimeo.com/player/' + video_id
-        return self.url_result(player_url, 'Vimeo', video_id)
+        video_id = self._match_id(url)
+        config = self._download_json(
+            'https://player.vimeo.com/video/%s/config' % video_id, video_id)
+        info_dict = self._parse_config(config, video_id)
+        self._vimeo_sort_formats(info_dict['formats'])
+        info_dict['id'] = video_id
+        return info_dict
 
 
 class VimeoWatchLaterIE(VimeoChannelIE):

From 94e5d6aedb5b509601d29dd8ea352afa925d3b22 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 11 Jun 2016 21:49:01 +0800
Subject: [PATCH 5186/6219] [viki] Skip a geo-restricted test

---
 youtube_dl/extractor/viki.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index e04b814c8..0c0cd622a 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -156,7 +156,8 @@ class VikiIE(VikiBaseIE):
         'params': {
             # m3u8 download
             'skip_download': True,
-        }
+        },
+        'skip': 'Blocked in the US',
     }, {
         # episode
         'url': 'http://www.viki.com/videos/44699v-boys-over-flowers-episode-1',

From c83b35d4aa4cec98ac171cca94ec515500076926 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 11 Jun 2016 22:39:13 +0800
Subject: [PATCH 5187/6219] [viki] Update _TESTS

---
 youtube_dl/extractor/viki.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index 0c0cd622a..70ce5de0e 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -127,7 +127,7 @@ class VikiIE(VikiBaseIE):
     }, {
         # clip
         'url': 'http://www.viki.com/videos/1067139v-the-avengers-age-of-ultron-press-conference',
-        'md5': '86c0b5dbd4d83a6611a79987cc7a1989',
+        'md5': 'feea2b1d7b3957f70886e6dfd8b8be84',
         'info_dict': {
             'id': '1067139v',
             'ext': 'mp4',
@@ -161,13 +161,13 @@ class VikiIE(VikiBaseIE):
     }, {
         # episode
         'url': 'http://www.viki.com/videos/44699v-boys-over-flowers-episode-1',
-        'md5': '190f3ef426005ba3a080a63325955bc3',
+        'md5': '1f54697dabc8f13f31bf06bb2e4de6db',
         'info_dict': {
             'id': '44699v',
             'ext': 'mp4',
             'title': 'Boys Over Flowers - Episode 1',
-            'description': 'md5:52617e4f729c7d03bfd4bcbbb6e946f2',
-            'duration': 4155,
+            'description': 'md5:b89cf50038b480b88b5b3c93589a9076',
+            'duration': 4204,
             'timestamp': 1270496524,
             'upload_date': '20100405',
             'uploader': 'group8',
@@ -197,7 +197,7 @@ class VikiIE(VikiBaseIE):
     }, {
         # non-English description
         'url': 'http://www.viki.com/videos/158036v-love-in-magic',
-        'md5': '1713ae35df5a521b31f6dc40730e7c9c',
+        'md5': '013dc282714e22acf9447cad14ff1208',
         'info_dict': {
             'id': '158036v',
             'ext': 'mp4',
@@ -303,7 +303,7 @@ class VikiChannelIE(VikiBaseIE):
             'title': 'Boys Over Flowers',
             'description': 'md5:ecd3cff47967fe193cff37c0bec52790',
         },
-        'playlist_count': 70,
+        'playlist_mincount': 71,
     }, {
         'url': 'http://www.viki.com/tv/1354c-poor-nastya-complete',
         'info_dict': {

From 6d28c408cfb0ce42f591cc6e2bb67522c0812c72 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 11 Jun 2016 23:00:44 +0800
Subject: [PATCH 5188/6219] [viki] Do not use a fallback language for title in
 the first try

In test_Viki_3, 'titles' gives a Hebrew title.
---
 youtube_dl/extractor/viki.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index 70ce5de0e..efa15e0b6 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -101,10 +101,13 @@ class VikiBaseIE(InfoExtractor):
             self.report_warning('Unable to get session token, login has probably failed')
 
     @staticmethod
-    def dict_selection(dict_obj, preferred_key):
+    def dict_selection(dict_obj, preferred_key, allow_fallback=True):
         if preferred_key in dict_obj:
             return dict_obj.get(preferred_key)
 
+        if not allow_fallback:
+            return
+
         filtered_dict = list(filter(None, [dict_obj.get(k) for k in dict_obj.keys()]))
         return filtered_dict[0] if filtered_dict else None
 
@@ -218,7 +221,7 @@ class VikiIE(VikiBaseIE):
 
         self._check_errors(video)
 
-        title = self.dict_selection(video.get('titles', {}), 'en')
+        title = self.dict_selection(video.get('titles', {}), 'en', allow_fallback=False)
         if not title:
             title = 'Episode %d' % video.get('number') if video.get('type') == 'episode' else video.get('id') or video_id
             container_titles = video.get('container', {}).get('titles', {})

From 80ae228b344ce36a07fb91c7e968fc5249c03161 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 12 Jun 2016 01:57:23 +0700
Subject: [PATCH 5189/6219] [matchtv] Modernize

---
 youtube_dl/extractor/matchtv.py | 25 ++++++++++++-------------
 1 file changed, 12 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/matchtv.py b/youtube_dl/extractor/matchtv.py
index 80a0d7013..33b0b539f 100644
--- a/youtube_dl/extractor/matchtv.py
+++ b/youtube_dl/extractor/matchtv.py
@@ -4,16 +4,12 @@ from __future__ import unicode_literals
 import random
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse_urlencode
-from ..utils import (
-    sanitized_Request,
-    xpath_text,
-)
+from ..utils import xpath_text
 
 
 class MatchTVIE(InfoExtractor):
-    _VALID_URL = r'https?://matchtv\.ru/?#live-player'
-    _TEST = {
+    _VALID_URL = r'https?://matchtv\.ru(?:/on-air|/?#live-player)'
+    _TESTS = [{
         'url': 'http://matchtv.ru/#live-player',
         'info_dict': {
             'id': 'matchtv-live',
@@ -24,12 +20,16 @@ class MatchTVIE(InfoExtractor):
         'params': {
             'skip_download': True,
         },
-    }
+    }, {
+        'url': 'http://matchtv.ru/on-air/',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = 'matchtv-live'
-        request = sanitized_Request(
-            'http://player.matchtv.ntvplus.tv/player/smil?%s' % compat_urllib_parse_urlencode({
+        video_url = self._download_json(
+            'http://player.matchtv.ntvplus.tv/player/smil', video_id,
+            query={
                 'ts': '',
                 'quality': 'SD',
                 'contentId': '561d2c0df7159b37178b4567',
@@ -40,11 +40,10 @@ class MatchTVIE(InfoExtractor):
                 'contentType': 'channel',
                 'timeShift': '0',
                 'platform': 'portal',
-            }),
+            },
             headers={
                 'Referer': 'http://player.matchtv.ntvplus.tv/embed-player/NTVEmbedPlayer.swf',
-            })
-        video_url = self._download_json(request, video_id)['data']['videoUrl']
+            })['data']['videoUrl']
         f4m_url = xpath_text(self._download_xml(video_url, video_id), './to')
         formats = self._extract_f4m_formats(f4m_url, video_id)
         self._sort_formats(formats)

From 2c3322e36ef23eb0566b820dd8e8711de20ed963 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 12 Jun 2016 04:49:37 +0700
Subject: [PATCH 5190/6219] [youporn] Fix metadata extraction

---
 youtube_dl/extractor/youporn.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/youporn.py b/youtube_dl/extractor/youporn.py
index 1124fe6c2..0df2d76ee 100644
--- a/youtube_dl/extractor/youporn.py
+++ b/youtube_dl/extractor/youporn.py
@@ -17,7 +17,7 @@ class YouPornIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?youporn\.com/watch/(?P<id>\d+)/(?P<display_id>[^/?#&]+)'
     _TESTS = [{
         'url': 'http://www.youporn.com/watch/505835/sex-ed-is-it-safe-to-masturbate-daily/',
-        'md5': '71ec5fcfddacf80f495efa8b6a8d9a89',
+        'md5': '3744d24c50438cf5b6f6d59feb5055c2',
         'info_dict': {
             'id': '505835',
             'display_id': 'sex-ed-is-it-safe-to-masturbate-daily',
@@ -121,21 +121,21 @@ class YouPornIE(InfoExtractor):
             webpage, 'thumbnail', fatal=False, group='thumbnail')
 
         uploader = self._html_search_regex(
-            r'(?s)<div[^>]+class=["\']videoInfoBy(?:\s+[^"\']+)?["\'][^>]*>\s*By:\s*</div>(.+?)</(?:a|div)>',
+            r'(?s)<div[^>]+class=["\']submitByLink["\'][^>]*>(.+?)</div>',
             webpage, 'uploader', fatal=False)
         upload_date = unified_strdate(self._html_search_regex(
-            r'(?s)<div[^>]+class=["\']videoInfoTime["\'][^>]*>(.+?)</div>',
+            r'(?s)<div[^>]+class=["\']videoInfo(?:Date|Time)["\'][^>]*>(.+?)</div>',
             webpage, 'upload date', fatal=False))
 
         age_limit = self._rta_search(webpage)
 
         average_rating = int_or_none(self._search_regex(
-            r'<div[^>]+class=["\']videoInfoRating["\'][^>]*>\s*<div[^>]+class=["\']videoRatingPercentage["\'][^>]*>(\d+)%</div>',
+            r'<div[^>]+class=["\']videoRatingPercentage["\'][^>]*>(\d+)%</div>',
             webpage, 'average rating', fatal=False))
 
         view_count = str_to_int(self._search_regex(
-            r'(?s)<div[^>]+class=["\']videoInfoViews["\'][^>]*>.*?([\d,.]+)\s*</div>',
-            webpage, 'view count', fatal=False))
+            r'(?s)<div[^>]+class=(["\']).*?\bvideoInfoViews\b.*?\1[^>]*>.*?(?P<count>[\d,.]+)<',
+            webpage, 'view count', fatal=False, group='count'))
         comment_count = str_to_int(self._search_regex(
             r'>All [Cc]omments? \(([\d,.]+)\)',
             webpage, 'comment count', fatal=False))

From 329ca3bef695bff011ed9b2d5f03e1331bf5bf0f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 12 Jun 2016 06:05:34 +0700
Subject: [PATCH 5191/6219] [utils] Add try_get

To reduce boilerplate when accessing JSON
---
 youtube_dl/utils.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 0acbd67de..c8308ba3a 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1901,6 +1901,16 @@ def dict_get(d, key_or_keys, default=None, skip_false_values=True):
     return d.get(key_or_keys, default)
 
 
+def try_get(src, getter, expected_type=None):
+    try:
+        v = getter(src)
+    except (AttributeError, KeyError, TypeError, IndexError):
+        pass
+    else:
+        if expected_type is None or isinstance(v, expected_type):
+            return v
+
+
 def encode_compat_str(string, encoding=preferredencoding(), errors='strict'):
     return string if isinstance(string, compat_str) else compat_str(string, encoding, errors)
 

From 98960c911c9bacc0c366dd11b194963a82606850 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 12 Jun 2016 06:06:04 +0700
Subject: [PATCH 5192/6219] [instagram] Extract metadata from JSON

---
 youtube_dl/extractor/instagram.py | 72 ++++++++++++++++++++++++++-----
 1 file changed, 61 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/instagram.py b/youtube_dl/extractor/instagram.py
index 3cbe77ad8..fc0197ae1 100644
--- a/youtube_dl/extractor/instagram.py
+++ b/youtube_dl/extractor/instagram.py
@@ -8,6 +8,7 @@ from ..utils import (
     int_or_none,
     limit_length,
     lowercase_escape,
+    try_get,
 )
 
 
@@ -19,10 +20,16 @@ class InstagramIE(InfoExtractor):
         'info_dict': {
             'id': 'aye83DjauH',
             'ext': 'mp4',
-            'uploader_id': 'naomipq',
             'title': 'Video by naomipq',
             'description': 'md5:1f17f0ab29bd6fe2bfad705f58de3cb8',
-        }
+            'thumbnail': 're:^https?://.*\.jpg',
+            'timestamp': 1371748545,
+            'upload_date': '20130620',
+            'uploader_id': 'naomipq',
+            'uploader': 'Naomi Leonor Phan-Quang',
+            'like_count': int,
+            'comment_count': int,
+        },
     }, {
         # missing description
         'url': 'https://www.instagram.com/p/BA-pQFBG8HZ/?taken-by=britneyspears',
@@ -31,6 +38,13 @@ class InstagramIE(InfoExtractor):
             'ext': 'mp4',
             'uploader_id': 'britneyspears',
             'title': 'Video by britneyspears',
+            'thumbnail': 're:^https?://.*\.jpg',
+            'timestamp': 1453760977,
+            'upload_date': '20160125',
+            'uploader_id': 'britneyspears',
+            'uploader': 'Britney Spears',
+            'like_count': int,
+            'comment_count': int,
         },
         'params': {
             'skip_download': True,
@@ -67,21 +81,57 @@ class InstagramIE(InfoExtractor):
         url = mobj.group('url')
 
         webpage = self._download_webpage(url, video_id)
-        uploader_id = self._search_regex(r'"owner":{"username":"(.+?)"',
-                                         webpage, 'uploader id', fatal=False)
-        desc = self._search_regex(
-            r'"caption":"(.+?)"', webpage, 'description', default=None)
-        if desc is not None:
-            desc = lowercase_escape(desc)
+
+        (video_url, description, thumbnail, timestamp, uploader,
+         uploader_id, like_count, comment_count) = [None] * 8
+
+        shared_data = self._parse_json(
+            self._search_regex(
+                r'window\._sharedData\s*=\s*({.+?});',
+                webpage, 'shared data', default='{}'),
+            video_id, fatal=False)
+        if shared_data:
+            media = try_get(
+                shared_data, lambda x: x['entry_data']['PostPage'][0]['media'], dict)
+            if media:
+                video_url = media.get('video_url')
+                description = media.get('caption')
+                thumbnail = media.get('display_src')
+                timestamp = int_or_none(media.get('date'))
+                uploader = media.get('owner', {}).get('full_name')
+                uploader_id = media.get('owner', {}).get('username')
+                like_count = int_or_none(media.get('likes', {}).get('count'))
+                comment_count = int_or_none(media.get('comments', {}).get('count'))
+
+        if not video_url:
+            video_url = self._og_search_video_url(webpage, secure=False)
+
+        if not uploader_id:
+            uploader_id = self._search_regex(
+                r'"owner"\s*:\s*{\s*"username"\s*:\s*"(.+?)"',
+                webpage, 'uploader id', fatal=False)
+
+        if not description:
+            description = self._search_regex(
+                r'"caption"\s*:\s*"(.+?)"', webpage, 'description', default=None)
+            if description is not None:
+                description = lowercase_escape(description)
+
+        if not thumbnail:
+            thumbnail = self._og_search_thumbnail(webpage)
 
         return {
             'id': video_id,
-            'url': self._og_search_video_url(webpage, secure=False),
+            'url': video_url,
             'ext': 'mp4',
             'title': 'Video by %s' % uploader_id,
-            'thumbnail': self._og_search_thumbnail(webpage),
+            'description': description,
+            'thumbnail': thumbnail,
+            'timestamp': timestamp,
             'uploader_id': uploader_id,
-            'description': desc,
+            'uploader': uploader,
+            'like_count': like_count,
+            'comment_count': comment_count,
         }
 
 

From a936ac321c5c0cee8e9769334945e744cdc60ae2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 12 Jun 2016 06:39:31 +0700
Subject: [PATCH 5193/6219] [README.md] Document using output template in batch
 files (Closes #9717)

---
 README.md | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/README.md b/README.md
index 3ff33c156..43e5114ea 100644
--- a/README.md
+++ b/README.md
@@ -553,6 +553,10 @@ The current default template is `%(title)s-%(id)s.%(ext)s`.
 
 In some cases, you don't want special characters such as 中, spaces, or &, such as when transferring the downloaded filename to a Windows system or the filename through an 8bit-unsafe channel. In these cases, add the `--restrict-filenames` flag to get a shorter title:
 
+#### Output template and Windows batch files
+
+If you are using output template inside a Windows batch file then you must escape plain percent characters (`%`) by doubling, so that `-o "%(title)s-%(id)s.%(ext)s"` should become `-o "%%(title)s-%%(id)s.%%(ext)s"`. However you should not touch `%`'s that are not plain characters, e.g. environment variables for expansion should stay intact: `-o "C:\%HOMEPATH%\Desktop\%%(title)s.%%(ext)s"`.
+
 #### Output template examples
 
 Note on Windows you may need to use double quotes instead of single.

From 4e790117292d060a3e449c9edfffe14d231aee96 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 12 Jun 2016 06:57:04 +0700
Subject: [PATCH 5194/6219] [nrktv] Fix tests

---
 youtube_dl/extractor/nrk.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index 486e086bb..4a790da7b 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -163,7 +163,7 @@ class NRKTVIE(NRKBaseIE):
             'ext': 'mp4',
             'title': '20 spørsmål 23.05.2014',
             'description': 'md5:bdea103bc35494c143c6a9acdd84887a',
-            'duration': 1741.52,
+            'duration': 1741,
         },
     }, {
         'url': 'https://tv.nrk.no/program/mdfp15000514',
@@ -173,7 +173,7 @@ class NRKTVIE(NRKBaseIE):
             'ext': 'mp4',
             'title': 'Grunnlovsjubiléet - Stor ståhei for ingenting 24.05.2014',
             'description': 'md5:89290c5ccde1b3a24bb8050ab67fe1db',
-            'duration': 4605.08,
+            'duration': 4605,
         },
     }, {
         # single playlist video

From 971e3b7520563936f6e6946f5c08d64f65ab6f42 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 12 Jun 2016 07:20:37 +0700
Subject: [PATCH 5195/6219] [nrk:skole] Fix extraction

---
 youtube_dl/extractor/nrk.py | 26 +++++++++++++++-----------
 1 file changed, 15 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index 4a790da7b..6ded5bd45 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -260,30 +260,34 @@ class NRKPlaylistIE(InfoExtractor):
 
 class NRKSkoleIE(InfoExtractor):
     IE_DESC = 'NRK Skole'
-    _VALID_URL = r'https?://(?:www\.)?nrk\.no/skole/klippdetalj?.*\btopic=(?P<id>[^/?#&]+)'
+    _VALID_URL = r'https?://(?:www\.)?nrk\.no/skole/?\?.*\bmediaId=(?P<id>\d+)'
 
     _TESTS = [{
-        'url': 'http://nrk.no/skole/klippdetalj?topic=nrk:klipp/616532',
-        'md5': '04cd85877cc1913bce73c5d28a47e00f',
+        'url': 'https://www.nrk.no/skole/?page=search&q=&mediaId=14099',
+        'md5': '6bc936b01f9dd8ed45bc58b252b2d9b6',
         'info_dict': {
             'id': '6021',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Genetikk og eneggede tvillinger',
             'description': 'md5:3aca25dcf38ec30f0363428d2b265f8d',
             'duration': 399,
         },
     }, {
-        'url': 'http://www.nrk.no/skole/klippdetalj?topic=nrk%3Aklipp%2F616532#embed',
-        'only_matching': True,
-    }, {
-        'url': 'http://www.nrk.no/skole/klippdetalj?topic=urn:x-mediadb:21379',
+        'url': 'https://www.nrk.no/skole/?page=objectives&subject=naturfag&objective=K15114&mediaId=19355',
         'only_matching': True,
     }]
 
     def _real_extract(self, url):
-        video_id = compat_urllib_parse_unquote(self._match_id(url))
+        video_id = self._match_id(url)
 
-        webpage = self._download_webpage(url, video_id)
+        webpage = self._download_webpage(
+            'https://mimir.nrk.no/plugin/1.0/static?mediaId=%s' % video_id,
+            video_id)
+
+        nrk_id = self._parse_json(
+            self._search_regex(
+                r'<script[^>]+type=["\']application/json["\'][^>]*>({.+?})</script>',
+                webpage, 'application json'),
+            video_id)['activeMedia']['psId']
 
-        nrk_id = self._search_regex(r'data-nrk-id=["\'](\d+)', webpage, 'nrk id')
         return self.url_result('nrk:%s' % nrk_id)

From 84dcd1c4e47f2a5a84a4658f42c66f7546588001 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 12 Jun 2016 11:08:39 +0700
Subject: [PATCH 5196/6219] [streamcloud] Detect removed videos (Closes #3768)

---
 youtube_dl/extractor/streamcloud.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/streamcloud.py b/youtube_dl/extractor/streamcloud.py
index 712359885..58560ec64 100644
--- a/youtube_dl/extractor/streamcloud.py
+++ b/youtube_dl/extractor/streamcloud.py
@@ -5,6 +5,7 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
+    ExtractorError,
     sanitized_Request,
     urlencode_postdata,
 )
@@ -14,7 +15,7 @@ class StreamcloudIE(InfoExtractor):
     IE_NAME = 'streamcloud.eu'
     _VALID_URL = r'https?://streamcloud\.eu/(?P<id>[a-zA-Z0-9_-]+)(?:/(?P<fname>[^#?]*)\.html)?'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://streamcloud.eu/skp9j99s4bpz/youtube-dl_test_video_____________-BaW_jenozKc.mp4.html',
         'md5': '6bea4c7fa5daaacc2a946b7146286686',
         'info_dict': {
@@ -23,7 +24,10 @@ class StreamcloudIE(InfoExtractor):
             'title': 'youtube-dl test video  \'/\\ ä ↭',
         },
         'skip': 'Only available from the EU'
-    }
+    }, {
+        'url': 'http://streamcloud.eu/ua8cmfh1nbe6/NSHIP-148--KUC-NG--H264-.mp4.html',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -31,6 +35,10 @@ class StreamcloudIE(InfoExtractor):
 
         orig_webpage = self._download_webpage(url, video_id)
 
+        if '>File Not Found<' in orig_webpage:
+            raise ExtractorError(
+                'Video %s does not exist' % video_id, expected=True)
+
         fields = re.findall(r'''(?x)<input\s+
             type="(?:hidden|submit)"\s+
             name="([^"]+)"\s+

From 77a9a9c295c753c4de4c96def6a9a15de1025f0f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 12 Jun 2016 12:06:48 +0700
Subject: [PATCH 5197/6219] release 2016.06.12

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 docs/supportedsites.md    | 3 ++-
 youtube_dl/version.py     | 2 +-
 3 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index a46b75fd8..243f2de5d 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.06.11.3*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.06.11.3**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.06.12*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.06.12**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.06.11.3
+[debug] youtube-dl version 2016.06.12
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index f89c2d1f2..e8c0a5d24 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -44,8 +44,8 @@
  - **appletrailers:section**
  - **archive.org**: archive.org videos
  - **ARD**
- - **ARD:mediathek**
  - **ARD:mediathek**: Saarländischer Rundfunk
+ - **ARD:mediathek**
  - **arte.tv**
  - **arte.tv:+7**
  - **arte.tv:cinema**
@@ -647,6 +647,7 @@
  - **Telegraaf**
  - **TeleMB**
  - **TeleTask**
+ - **Telewebion**
  - **TF1**
  - **TheIntercept**
  - **ThePlatform**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 9932b1e62..5e9c14398 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.06.11.3'
+__version__ = '2016.06.12'

From e69f9f5d68aed32cc27ca188b0f51925d949c365 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 12 Jun 2016 16:45:07 +0700
Subject: [PATCH 5198/6219] [downloader/external] Decode error string before
 writing to stderr

---
 youtube_dl/downloader/external.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index 3ff1f9ed4..fae245024 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -85,7 +85,7 @@ class ExternalFD(FileDownloader):
             cmd, stderr=subprocess.PIPE)
         _, stderr = p.communicate()
         if p.returncode != 0:
-            self.to_stderr(stderr)
+            self.to_stderr(stderr.decode('utf-8', 'replace'))
         return p.returncode
 
 

From bccdac68749e7a39a47dd0e1ad0ec9c177657de6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 13 Jun 2016 01:11:04 +0700
Subject: [PATCH 5199/6219] [xfileshare:xvidstage] Add support for videos with
 packed codes (Closes #4335)

---
 youtube_dl/extractor/xfileshare.py | 31 ++++++++++++++++++++++++++----
 1 file changed, 27 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/xfileshare.py b/youtube_dl/extractor/xfileshare.py
index ee4d04c20..fe0ab6300 100644
--- a/youtube_dl/extractor/xfileshare.py
+++ b/youtube_dl/extractor/xfileshare.py
@@ -5,8 +5,10 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
+    decode_packed_codes,
     ExtractorError,
     int_or_none,
+    NO_DEFAULT,
     sanitized_Request,
     urlencode_postdata,
 )
@@ -23,6 +25,7 @@ class XFileShareIE(InfoExtractor):
         ('thevideobee.to', 'TheVideoBee'),
         ('vidto.me', 'Vidto'),
         ('streamin.to', 'Streamin.To'),
+        ('xvidstage.com', 'XVIDSTAGE'),
     )
 
     IE_DESC = 'XFileShare based sites: %s' % ', '.join(list(zip(*_SITES))[1])
@@ -78,6 +81,13 @@ class XFileShareIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Big Buck Bunny trailer',
         },
+    }, {
+        'url': 'http://xvidstage.com/e0qcnl03co6z',
+        'info_dict': {
+            'id': 'e0qcnl03co6z',
+            'ext': 'mp4',
+            'title': 'Chucky Prank 2015.mp4',
+        },
     }]
 
     def _real_extract(self, url):
@@ -113,10 +123,23 @@ class XFileShareIE(InfoExtractor):
              r'>Watch (.+) ',
              r'<h2 class="video-page-head">([^<]+)</h2>'],
             webpage, 'title', default=None) or self._og_search_title(webpage)).strip()
-        video_url = self._search_regex(
-            [r'file\s*:\s*["\'](http[^"\']+)["\'],',
-             r'file_link\s*=\s*\'(https?:\/\/[0-9a-zA-z.\/\-_]+)'],
-            webpage, 'file url')
+
+        def extract_video_url(default=NO_DEFAULT):
+            return self._search_regex(
+                (r'file\s*:\s*(["\'])(?P<url>http.+?)\1,',
+                 r'file_link\s*=\s*(["\'])(?P<url>http.+?)\1',
+                 r'addVariable\((\\?["\'])file\1\s*,\s*(\\?["\'])(?P<url>http.+?)\2\)',
+                 r'<embed[^>]+src=(["\'])(?P<url>http.+?)\1'),
+                webpage, 'file url', default=default, group='url')
+
+        video_url = extract_video_url(default=None)
+
+        if not video_url:
+            webpage = decode_packed_codes(self._search_regex(
+                r"(}\('(.+)',(\d+),(\d+),'[^']*\b(?:file|embed)\b[^']*'\.split\('\|'\))",
+                webpage, 'packed code'))
+            video_url = extract_video_url()
+
         thumbnail = self._search_regex(
             r'image\s*:\s*["\'](http[^"\']+)["\'],', webpage, 'thumbnail', default=None)
 

From cf2bf840bac1742cb422549a5491a30f70d1abb0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 13 Jun 2016 01:11:14 +0700
Subject: [PATCH 5200/6219] [xfileshare] Fix test

---
 youtube_dl/extractor/xfileshare.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/xfileshare.py b/youtube_dl/extractor/xfileshare.py
index fe0ab6300..0f8ccf430 100644
--- a/youtube_dl/extractor/xfileshare.py
+++ b/youtube_dl/extractor/xfileshare.py
@@ -39,7 +39,7 @@ class XFileShareIE(InfoExtractor):
         'md5': '5ae4a3580620380619678ee4875893ba',
         'info_dict': {
             'id': '06y9juieqpmi',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Rebecca Black My Moment Official Music Video Reaction-6GK87Rc8bzQ',
             'thumbnail': 're:http://.*\.jpg',
         },

From 33b72ce64e8705a71f8ab0e6a322e5f9f3b99276 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 13 Jun 2016 01:19:54 +0700
Subject: [PATCH 5201/6219] [xfileshare] Improve removed videos detection

---
 youtube_dl/extractor/xfileshare.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/xfileshare.py b/youtube_dl/extractor/xfileshare.py
index 0f8ccf430..995aada0d 100644
--- a/youtube_dl/extractor/xfileshare.py
+++ b/youtube_dl/extractor/xfileshare.py
@@ -32,7 +32,10 @@ class XFileShareIE(InfoExtractor):
     _VALID_URL = (r'https?://(?P<host>(?:www\.)?(?:%s))/(?:embed-)?(?P<id>[0-9a-zA-Z]+)'
                   % '|'.join(re.escape(site) for site in list(zip(*_SITES))[0]))
 
-    _FILE_NOT_FOUND_REGEX = r'>(?:404 - )?File Not Found<'
+    _FILE_NOT_FOUND_REGEXES = (
+        r'>(?:404 - )?File Not Found<',
+        r'>The file was removed by administrator<',
+    )
 
     _TESTS = [{
         'url': 'http://gorillavid.in/06y9juieqpmi',
@@ -88,6 +91,10 @@ class XFileShareIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Chucky Prank 2015.mp4',
         },
+    }, {
+        # removed by administrator
+        'url': 'http://xvidstage.com/amfy7atlkx25',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
@@ -97,7 +104,7 @@ class XFileShareIE(InfoExtractor):
         url = 'http://%s/%s' % (mobj.group('host'), video_id)
         webpage = self._download_webpage(url, video_id)
 
-        if re.search(self._FILE_NOT_FOUND_REGEX, webpage) is not None:
+        if any(re.search(p, webpage) for p in self._FILE_NOT_FOUND_REGEXES):
             raise ExtractorError('Video %s does not exist' % video_id, expected=True)
 
         fields = self._hidden_inputs(webpage)

From b50e02c1e4c9ea70e88ab115b17cfa109b0c9617 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 13 Jun 2016 07:05:32 +0700
Subject: [PATCH 5202/6219] [README.md] Update links to options available for
 YoutubeDL

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 43e5114ea..b5cbaced7 100644
--- a/README.md
+++ b/README.md
@@ -964,7 +964,7 @@ with youtube_dl.YoutubeDL(ydl_opts) as ydl:
     ydl.download(['http://www.youtube.com/watch?v=BaW_jenozKc'])
 ```
 
-Most likely, you'll want to use various options. For a list of what can be done, have a look at [`youtube_dl/YoutubeDL.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/YoutubeDL.py#L121-L269). For a start, if you want to intercept youtube-dl's output, set a `logger` object.
+Most likely, you'll want to use various options. For a list of options available, have a look at [`youtube_dl/YoutubeDL.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/YoutubeDL.py#L128-L278). For a start, if you want to intercept youtube-dl's output, set a `logger` object.
 
 Here's a more complete example of a program that outputs only errors (and a short message after the download is finished), and downloads/converts the video to an mp3 file:
 

From b4663f12b1c872f4e731f1940831ec017bc86959 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 13 Jun 2016 07:16:35 +0700
Subject: [PATCH 5203/6219] [README.md] Update links to info dict metafields

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index b5cbaced7..5a9768161 100644
--- a/README.md
+++ b/README.md
@@ -935,8 +935,8 @@ After you have ensured this site is distributing it's content legally, you can f
     ```
 5. Add an import in [`youtube_dl/extractor/extractors.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/extractors.py).
 6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc.
-7. Have a look at [`youtube_dl/extractor/common.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/rg3/youtube-dl/blob/58525c94d547be1c8167d16c298bdd75506db328/youtube_dl/extractor/common.py#L68-L226). Add tests and code for as many as you want.
-8. Keep in mind that the only mandatory fields in info dict for successful extraction process are `id`, `title` and either `url` or `formats`, i.e. these are the critical data the extraction does not make any sense without. This means that [any field](https://github.com/rg3/youtube-dl/blob/58525c94d547be1c8167d16c298bdd75506db328/youtube_dl/extractor/common.py#L138-L226) apart from aforementioned mandatory ones should be treated **as optional** and extraction should be **tolerate** to situations when sources for these fields can potentially be unavailable (even if they always available at the moment) and **future-proof** in order not to break the extraction of general purpose mandatory fields. For example, if you have some intermediate dict `meta` that is a source of metadata and it has a key `summary` that you want to extract and put into resulting info dict as `description`, you should be ready that this key may be missing from the `meta` dict, i.e. you should extract it as `meta.get('summary')` and not `meta['summary']`. Similarly, you should pass `fatal=False` when extracting data from a webpage with `_search_regex/_html_search_regex`.
+7. Have a look at [`youtube_dl/extractor/common.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L74-L252). Add tests and code for as many as you want.
+8. Keep in mind that the only mandatory fields in info dict for successful extraction process are `id`, `title` and either `url` or `formats`, i.e. these are the critical data the extraction does not make any sense without. This means that [any field](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L148-L252) apart from aforementioned mandatory ones should be treated **as optional** and extraction should be **tolerate** to situations when sources for these fields can potentially be unavailable (even if they always available at the moment) and **future-proof** in order not to break the extraction of general purpose mandatory fields. For example, if you have some intermediate dict `meta` that is a source of metadata and it has a key `summary` that you want to extract and put into resulting info dict as `description`, you should be ready that this key may be missing from the `meta` dict, i.e. you should extract it as `meta.get('summary')` and not `meta['summary']`. Similarly, you should pass `fatal=False` when extracting data from a webpage with `_search_regex/_html_search_regex`.
 9. Check the code with [flake8](https://pypi.python.org/pypi/flake8).
 10. When the tests pass, [add](http://git-scm.com/docs/git-add) the new files and [commit](http://git-scm.com/docs/git-commit) them and [push](http://git-scm.com/docs/git-push) the result, like this:
 

From 79cd8b3d8acee7845260d5bd60698155a0d81d33 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 13 Jun 2016 10:04:04 +0700
Subject: [PATCH 5204/6219] [README.md] Suggest checking extractor code under
 all Python versions

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 5a9768161..f1e59542d 100644
--- a/README.md
+++ b/README.md
@@ -937,7 +937,7 @@ After you have ensured this site is distributing it's content legally, you can f
 6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc.
 7. Have a look at [`youtube_dl/extractor/common.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L74-L252). Add tests and code for as many as you want.
 8. Keep in mind that the only mandatory fields in info dict for successful extraction process are `id`, `title` and either `url` or `formats`, i.e. these are the critical data the extraction does not make any sense without. This means that [any field](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L148-L252) apart from aforementioned mandatory ones should be treated **as optional** and extraction should be **tolerate** to situations when sources for these fields can potentially be unavailable (even if they always available at the moment) and **future-proof** in order not to break the extraction of general purpose mandatory fields. For example, if you have some intermediate dict `meta` that is a source of metadata and it has a key `summary` that you want to extract and put into resulting info dict as `description`, you should be ready that this key may be missing from the `meta` dict, i.e. you should extract it as `meta.get('summary')` and not `meta['summary']`. Similarly, you should pass `fatal=False` when extracting data from a webpage with `_search_regex/_html_search_regex`.
-9. Check the code with [flake8](https://pypi.python.org/pypi/flake8).
+9. Check the code with [flake8](https://pypi.python.org/pypi/flake8). Also make sure your code works under all [Python](http://www.python.org/) versions claimed supported by youtube-dl, namely 2.6, 2.7, and 3.2+.
 10. When the tests pass, [add](http://git-scm.com/docs/git-add) the new files and [commit](http://git-scm.com/docs/git-commit) them and [push](http://git-scm.com/docs/git-push) the result, like this:
 
         $ git add youtube_dl/extractor/extractors.py

From 778f96944785f814a97964be1d6fb3bb78bc13f0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 14 Jun 2016 00:06:31 +0700
Subject: [PATCH 5205/6219] [twitch:clips] Add extractor (Closes #9767)

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/twitch.py     | 43 ++++++++++++++++++++++++++++++
 2 files changed, 44 insertions(+)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 36ddc1f73..d2db4d803 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -862,6 +862,7 @@ from .twitch import (
     TwitchProfileIE,
     TwitchPastBroadcastsIE,
     TwitchStreamIE,
+    TwitchClipsIE,
 )
 from .twitter import (
     TwitterCardIE,
diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index d898f14c3..20919774d 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -16,6 +16,7 @@ from ..compat import (
 from ..utils import (
     ExtractorError,
     int_or_none,
+    js_to_json,
     orderedSet,
     parse_duration,
     parse_iso8601,
@@ -454,3 +455,45 @@ class TwitchStreamIE(TwitchBaseIE):
             'formats': formats,
             'is_live': True,
         }
+
+
+class TwitchClipsIE(InfoExtractor):
+    IE_NAME = 'twitch:clips'
+    _VALID_URL = r'https?://clips\.twitch\.tv/(?:[^/]+/)*(?P<id>[^/?#&]+)'
+
+    _TEST = {
+        'url': 'https://clips.twitch.tv/ea/AggressiveCobraPoooound',
+        'md5': '761769e1eafce0ffebfb4089cb3847cd',
+        'info_dict': {
+            'id': 'AggressiveCobraPoooound',
+            'ext': 'mp4',
+            'title': 'EA Play 2016 Live from the Novo Theatre',
+            'thumbnail': 're:^https?://.*\.jpg',
+            'creator': 'EA',
+            'uploader': 'stereotype_',
+            'uploader_id': 'stereotype_',
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        clip = self._parse_json(
+            self._search_regex(
+                r'(?s)clipInfo\s*=\s*({.+?});', webpage, 'clip info'),
+            video_id, transform_source=js_to_json)
+
+        video_url = clip['clip_video_url']
+        title = clip['channel_title']
+
+        return {
+            'id': video_id,
+            'url': video_url,
+            'title': title,
+            'thumbnail': self._og_search_thumbnail(webpage),
+            'creator': clip.get('broadcaster_display_name') or clip.get('broadcaster_login'),
+            'uploader': clip.get('curator_login'),
+            'uploader_id': clip.get('curator_display_name'),
+        }

From 14d0f4e0f3e1b6a467b6302eb60644535aff4292 Mon Sep 17 00:00:00 2001
From: Dracony <draconyster@gmail.com>
Date: Thu, 9 Jun 2016 13:31:22 +0200
Subject: [PATCH 5206/6219] Added extractor for rockstargames.com

---
 youtube_dl/extractor/extractors.py    |  1 +
 youtube_dl/extractor/rockstargames.py | 54 +++++++++++++++++++++++++++
 2 files changed, 55 insertions(+)
 create mode 100644 youtube_dl/extractor/rockstargames.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index d2db4d803..8a6c54b97 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -649,6 +649,7 @@ from .revision3 import (
 from .rice import RICEIE
 from .ringtv import RingTVIE
 from .ro220 import Ro220IE
+from .rockstargames import RockstarGamesIE
 from .rottentomatoes import RottenTomatoesIE
 from .roxwel import RoxwelIE
 from .rtbf import RTBFIE
diff --git a/youtube_dl/extractor/rockstargames.py b/youtube_dl/extractor/rockstargames.py
new file mode 100644
index 000000000..427ab153a
--- /dev/null
+++ b/youtube_dl/extractor/rockstargames.py
@@ -0,0 +1,54 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    qualities,
+    parse_iso8601
+)
+
+
+class RockstarGamesIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?rockstargames\.com/videos/video/(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'https://www.rockstargames.com/videos/video/11544/',
+        'md5': '03b5caa6e357a4bd50e3143fc03e5733',
+        'info_dict': {
+            'id': '11544',
+            'ext': 'mp4',
+            'title': 'Further Adventures in Finance and Felony Trailer',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'description': 'md5:6d31f55f30cb101b5476c4a379e324a3',
+            'upload_date': '20160602',
+            'timestamp': 1464876000
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        json_data = self._download_json(
+            'https://www.rockstargames.com/videoplayer/videos/get-video.json?id=%s&locale=en_us' % video_id,
+            video_id
+        )['video']
+
+        formats = []
+
+        for video in json_data['files_processed']['video/mp4']:
+            if not video.get('src'):
+                continue
+            height = video.get('resolution', '').replace('p', '')
+            
+            formats.append({
+                'url': self._proto_relative_url(video['src']),
+                'height': int(height) if height.isdigit() else -1,
+            })
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': json_data['title'],
+            'description': json_data.get('description'),
+            'formats': formats,
+            'thumbnail': self._proto_relative_url(json_data.get('screencap')),
+            'timestamp': parse_iso8601(json_data.get('created'))
+        }

From 16b6bd01d238c2c58e3ac7ba91c706261d5810e8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 14 Jun 2016 01:11:24 +0700
Subject: [PATCH 5207/6219] [rockstargames] Improve and add Youtube fallback
 (Closes #9737)

---
 youtube_dl/extractor/rockstargames.py | 55 +++++++++++++++++----------
 1 file changed, 35 insertions(+), 20 deletions(-)

diff --git a/youtube_dl/extractor/rockstargames.py b/youtube_dl/extractor/rockstargames.py
index 427ab153a..48128e219 100644
--- a/youtube_dl/extractor/rockstargames.py
+++ b/youtube_dl/extractor/rockstargames.py
@@ -3,52 +3,67 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..utils import (
-    qualities,
-    parse_iso8601
+    int_or_none,
+    parse_iso8601,
 )
 
 
 class RockstarGamesIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?rockstargames\.com/videos/video/(?P<id>[0-9]+)'
-    _TEST = {
+    _VALID_URL = r'https?://(?:www\.)?rockstargames\.com/videos(?:/video/|#?/?\?.*\bvideo=)(?P<id>\d+)'
+    _TESTS = [{
         'url': 'https://www.rockstargames.com/videos/video/11544/',
         'md5': '03b5caa6e357a4bd50e3143fc03e5733',
         'info_dict': {
             'id': '11544',
             'ext': 'mp4',
             'title': 'Further Adventures in Finance and Felony Trailer',
-            'thumbnail': 're:^https?://.*\.jpg$',
             'description': 'md5:6d31f55f30cb101b5476c4a379e324a3',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'timestamp': 1464876000,
             'upload_date': '20160602',
-            'timestamp': 1464876000
         }
-    }
+    }, {
+        'url': 'http://www.rockstargames.com/videos#/?video=48',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        json_data = self._download_json(
-            'https://www.rockstargames.com/videoplayer/videos/get-video.json?id=%s&locale=en_us' % video_id,
-            video_id
-        )['video']
+
+        video = self._download_json(
+            'https://www.rockstargames.com/videoplayer/videos/get-video.json',
+            video_id, query={
+                'id': video_id,
+                'locale': 'en_us',
+            })['video']
+
+        title = video['title']
 
         formats = []
-
-        for video in json_data['files_processed']['video/mp4']:
+        for video in video['files_processed']['video/mp4']:
             if not video.get('src'):
                 continue
-            height = video.get('resolution', '').replace('p', '')
-            
+            resolution = video.get('resolution')
+            height = int_or_none(self._search_regex(
+                r'^(\d+)[pP]$', resolution or '', 'height', default=None))
             formats.append({
                 'url': self._proto_relative_url(video['src']),
-                'height': int(height) if height.isdigit() else -1,
+                'format_id': resolution,
+                'height': height,
             })
+
+        if not formats:
+            youtube_id = video.get('youtube_id')
+            if youtube_id:
+                return self.url_result(youtube_id, 'Youtube')
+
         self._sort_formats(formats)
 
         return {
             'id': video_id,
-            'title': json_data['title'],
-            'description': json_data.get('description'),
+            'title': title,
+            'description': video.get('description'),
+            'thumbnail': self._proto_relative_url(video.get('screencap')),
+            'timestamp': parse_iso8601(video.get('created')),
             'formats': formats,
-            'thumbnail': self._proto_relative_url(json_data.get('screencap')),
-            'timestamp': parse_iso8601(json_data.get('created'))
         }

From fea55ef4a95d226668bd63742c4731832de93a79 Mon Sep 17 00:00:00 2001
From: venth <artur.krysiak.warszawa@gmail.com>
Date: Sun, 8 May 2016 22:26:08 +0200
Subject: [PATCH 5208/6219] [wrzuta.pl:playlist] Added playlist extraction from
 wrzuta.pl

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/wrzuta.py     | 74 ++++++++++++++++++++++++++++++
 2 files changed, 75 insertions(+)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 8a6c54b97..5a93fec6a 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -981,6 +981,7 @@ from .wimp import WimpIE
 from .wistia import WistiaIE
 from .worldstarhiphop import WorldStarHipHopIE
 from .wrzuta import WrzutaIE
+from .wrzuta import WrzutaPlaylistIE
 from .wsj import WSJIE
 from .xbef import XBefIE
 from .xboxclips import XboxClipsIE
diff --git a/youtube_dl/extractor/wrzuta.py b/youtube_dl/extractor/wrzuta.py
index c42764921..830649214 100644
--- a/youtube_dl/extractor/wrzuta.py
+++ b/youtube_dl/extractor/wrzuta.py
@@ -80,3 +80,77 @@ class WrzutaIE(InfoExtractor):
             'description': self._og_search_description(webpage),
             'age_limit': embedpage.get('minimalAge', 0),
         }
+
+
+_ENTRY_PATTERN = r'<a href="(?P<playlist_entry_url>[^"]+)" target="_blank" class="playlist\-file\-page">'
+_PLAYLIST_SIZE_PATTERN = r'<div class="playlist-counter">[0-9]+/([0-9]+)</div>'
+
+
+class WrzutaPlaylistIE(InfoExtractor):
+    """
+        this class covers extraction of wrzuta playlist entries
+        the extraction process bases on following steps:
+        * collect information of playlist size
+        * download all entries provided on
+          the playlist webpage (the playlist is split
+          on two pages: first directly reached from webpage
+          second: downloaded on demand by ajax call and rendered
+          using the ajax call response)
+        * in case size of extracted entries not reached total number of entries
+          use the ajax call to collect the remaining entries
+    """
+
+    IE_NAME = 'wrzuta.pl:playlist'
+
+    _VALID_URL = r'https?://(?P<uploader>[0-9a-zA-Z]+)\.wrzuta\.pl/playlista/' \
+                 '(?P<id>[0-9a-zA-Z]+)/.*'
+
+    _TESTS = [{
+        'url': 'http://miromak71.wrzuta.pl/playlista/7XfO4vE84iR/moja_muza',
+        'playlist_mincount': 14,
+        'info_dict': {
+            'id': '7XfO4vE84iR',
+            'title': 'Moja muza',
+        },
+    }, {
+        'url': 'http://heroesf70.wrzuta.pl/playlista/6Nj3wQHx756/lipiec_-_lato_2015_muzyka_swiata',
+        'playlist_mincount': 144,
+        'info_dict': {
+            'id': '6Nj3wQHx756',
+            'title': 'Lipiec - Lato 2015 Muzyka Świata',
+        },
+    }]
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        playlist_id = mobj.group('id')
+        uploader = mobj.group('uploader')
+
+        entries = []
+
+        webpage = self._download_webpage(url, playlist_id)
+
+        playlist_size = self._html_search_regex(_PLAYLIST_SIZE_PATTERN, webpage, 'Size of the playlist')
+        playlist_size = int(playlist_size) if playlist_size else 0
+
+        playlist_title = self._og_search_title(webpage).replace('Playlista: ', '', 1)
+
+        if playlist_size:
+            entries = list(map(
+                lambda entry_url: self.url_result(entry_url),
+                re.findall(_ENTRY_PATTERN, webpage)
+            ))
+
+            if playlist_size > len(entries):
+                playlist_content = self._download_json(
+                    'http://{uploader_id}.wrzuta.pl/xhr/get_playlist_offset/{playlist_id}'.format(
+                        uploader_id=uploader,
+                        playlist_id=playlist_id,
+                    ),
+                    playlist_id,
+                    'Downloading playlist content as JSON metadata',
+                    'Unable to download playlist content as JSON metadata',
+                )
+                entries += [self.url_result(entry['filelink']) for entry in playlist_content['files']]
+
+        return self.playlist_result(entries, playlist_id, playlist_title)

From 1759672eede27be0a3d473c4b2925a0b10dce547 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 14 Jun 2016 02:13:54 +0700
Subject: [PATCH 5209/6219] [wrzuta:playlist] Improve and simplify (Closes
 #9341)

---
 youtube_dl/extractor/extractors.py |  6 ++--
 youtube_dl/extractor/wrzuta.py     | 49 ++++++++++++++----------------
 2 files changed, 27 insertions(+), 28 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 5a93fec6a..5fce9f47a 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -980,8 +980,10 @@ from .weiqitv import WeiqiTVIE
 from .wimp import WimpIE
 from .wistia import WistiaIE
 from .worldstarhiphop import WorldStarHipHopIE
-from .wrzuta import WrzutaIE
-from .wrzuta import WrzutaPlaylistIE
+from .wrzuta import (
+    WrzutaIE,
+    WrzutaPlaylistIE,
+)
 from .wsj import WSJIE
 from .xbef import XBefIE
 from .xboxclips import XboxClipsIE
diff --git a/youtube_dl/extractor/wrzuta.py b/youtube_dl/extractor/wrzuta.py
index 830649214..b811f57fb 100644
--- a/youtube_dl/extractor/wrzuta.py
+++ b/youtube_dl/extractor/wrzuta.py
@@ -7,6 +7,7 @@ from .common import InfoExtractor
 from ..utils import (
     int_or_none,
     qualities,
+    remove_start,
 )
 
 
@@ -82,10 +83,6 @@ class WrzutaIE(InfoExtractor):
         }
 
 
-_ENTRY_PATTERN = r'<a href="(?P<playlist_entry_url>[^"]+)" target="_blank" class="playlist\-file\-page">'
-_PLAYLIST_SIZE_PATTERN = r'<div class="playlist-counter">[0-9]+/([0-9]+)</div>'
-
-
 class WrzutaPlaylistIE(InfoExtractor):
     """
         this class covers extraction of wrzuta playlist entries
@@ -101,10 +98,7 @@ class WrzutaPlaylistIE(InfoExtractor):
     """
 
     IE_NAME = 'wrzuta.pl:playlist'
-
-    _VALID_URL = r'https?://(?P<uploader>[0-9a-zA-Z]+)\.wrzuta\.pl/playlista/' \
-                 '(?P<id>[0-9a-zA-Z]+)/.*'
-
+    _VALID_URL = r'https?://(?P<uploader>[0-9a-zA-Z]+)\.wrzuta\.pl/playlista/(?P<id>[0-9a-zA-Z]+)'
     _TESTS = [{
         'url': 'http://miromak71.wrzuta.pl/playlista/7XfO4vE84iR/moja_muza',
         'playlist_mincount': 14,
@@ -119,6 +113,9 @@ class WrzutaPlaylistIE(InfoExtractor):
             'id': '6Nj3wQHx756',
             'title': 'Lipiec - Lato 2015 Muzyka Świata',
         },
+    }, {
+        'url': 'http://miromak71.wrzuta.pl/playlista/7XfO4vE84iR',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
@@ -126,31 +123,31 @@ class WrzutaPlaylistIE(InfoExtractor):
         playlist_id = mobj.group('id')
         uploader = mobj.group('uploader')
 
-        entries = []
-
         webpage = self._download_webpage(url, playlist_id)
 
-        playlist_size = self._html_search_regex(_PLAYLIST_SIZE_PATTERN, webpage, 'Size of the playlist')
-        playlist_size = int(playlist_size) if playlist_size else 0
+        playlist_size = int_or_none(self._html_search_regex(
+            (r'<div[^>]+class=["\']playlist-counter["\'][^>]*>\d+/(\d+)',
+             r'<div[^>]+class=["\']all-counter["\'][^>]*>(.+?)</div>'),
+            webpage, 'playlist size', default=None))
 
-        playlist_title = self._og_search_title(webpage).replace('Playlista: ', '', 1)
+        playlist_title = remove_start(
+            self._og_search_title(webpage), 'Playlista: ')
 
+        entries = []
         if playlist_size:
-            entries = list(map(
-                lambda entry_url: self.url_result(entry_url),
-                re.findall(_ENTRY_PATTERN, webpage)
-            ))
-
+            entries = [
+                self.url_result(entry_url)
+                for _, entry_url in re.findall(
+                    r'<a[^>]+href=(["\'])(http.+?)\1[^>]+class=["\']playlist-file-page',
+                    webpage)]
             if playlist_size > len(entries):
                 playlist_content = self._download_json(
-                    'http://{uploader_id}.wrzuta.pl/xhr/get_playlist_offset/{playlist_id}'.format(
-                        uploader_id=uploader,
-                        playlist_id=playlist_id,
-                    ),
+                    'http://%s.wrzuta.pl/xhr/get_playlist_offset/%s' % (uploader, playlist_id),
                     playlist_id,
-                    'Downloading playlist content as JSON metadata',
-                    'Unable to download playlist content as JSON metadata',
-                )
-                entries += [self.url_result(entry['filelink']) for entry in playlist_content['files']]
+                    'Downloading playlist JSON',
+                    'Unable to download playlist JSON')
+                entries.extend([
+                    self.url_result(entry['filelink'])
+                    for entry in playlist_content.get('files', []) if entry.get('filelink')])
 
         return self.playlist_result(entries, playlist_id, playlist_title)

From bc2a871f3eb5f2fce7fc1097787e829106d11f4e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 14 Jun 2016 02:15:09 +0700
Subject: [PATCH 5210/6219] Credit @dracony for rockstargames (#9737)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 3272fc6ea..4f77de3c7 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -173,3 +173,4 @@ Kevin Deldycke
 inondle
 Tomáš Čech
 Déstin Reed
+Roman Tsiupa

From a4ea28eee6c89756ac5bddfd0c6ef11dd490a191 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 14 Jun 2016 02:15:47 +0700
Subject: [PATCH 5211/6219] Credit @venth for wrzuta:playlist (#9341)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 4f77de3c7..cdf655c39 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -174,3 +174,4 @@ inondle
 Tomáš Čech
 Déstin Reed
 Roman Tsiupa
+Artur Krysiak

From d01fb21d4c58650a3ccd2a6fe2877cc9a53dd942 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 14 Jun 2016 02:19:42 +0700
Subject: [PATCH 5212/6219] release 2016.06.14

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 CONTRIBUTING.md           | 6 +++---
 docs/supportedsites.md    | 5 ++++-
 youtube_dl/version.py     | 2 +-
 4 files changed, 11 insertions(+), 8 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 243f2de5d..4c52c5933 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.06.12*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.06.12**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.06.14*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.06.14**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.06.12
+[debug] youtube-dl version 2016.06.14
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index c83b8655a..a59fac9b2 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -142,9 +142,9 @@ After you have ensured this site is distributing it's content legally, you can f
     ```
 5. Add an import in [`youtube_dl/extractor/extractors.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/extractors.py).
 6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc.
-7. Have a look at [`youtube_dl/extractor/common.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/rg3/youtube-dl/blob/58525c94d547be1c8167d16c298bdd75506db328/youtube_dl/extractor/common.py#L68-L226). Add tests and code for as many as you want.
-8. Keep in mind that the only mandatory fields in info dict for successful extraction process are `id`, `title` and either `url` or `formats`, i.e. these are the critical data the extraction does not make any sense without. This means that [any field](https://github.com/rg3/youtube-dl/blob/58525c94d547be1c8167d16c298bdd75506db328/youtube_dl/extractor/common.py#L138-L226) apart from aforementioned mandatory ones should be treated **as optional** and extraction should be **tolerate** to situations when sources for these fields can potentially be unavailable (even if they always available at the moment) and **future-proof** in order not to break the extraction of general purpose mandatory fields. For example, if you have some intermediate dict `meta` that is a source of metadata and it has a key `summary` that you want to extract and put into resulting info dict as `description`, you should be ready that this key may be missing from the `meta` dict, i.e. you should extract it as `meta.get('summary')` and not `meta['summary']`. Similarly, you should pass `fatal=False` when extracting data from a webpage with `_search_regex/_html_search_regex`.
-9. Check the code with [flake8](https://pypi.python.org/pypi/flake8).
+7. Have a look at [`youtube_dl/extractor/common.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L74-L252). Add tests and code for as many as you want.
+8. Keep in mind that the only mandatory fields in info dict for successful extraction process are `id`, `title` and either `url` or `formats`, i.e. these are the critical data the extraction does not make any sense without. This means that [any field](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L148-L252) apart from aforementioned mandatory ones should be treated **as optional** and extraction should be **tolerate** to situations when sources for these fields can potentially be unavailable (even if they always available at the moment) and **future-proof** in order not to break the extraction of general purpose mandatory fields. For example, if you have some intermediate dict `meta` that is a source of metadata and it has a key `summary` that you want to extract and put into resulting info dict as `description`, you should be ready that this key may be missing from the `meta` dict, i.e. you should extract it as `meta.get('summary')` and not `meta['summary']`. Similarly, you should pass `fatal=False` when extracting data from a webpage with `_search_regex/_html_search_regex`.
+9. Check the code with [flake8](https://pypi.python.org/pypi/flake8). Also make sure your code works under all [Python](http://www.python.org/) versions claimed supported by youtube-dl, namely 2.6, 2.7, and 3.2+.
 10. When the tests pass, [add](http://git-scm.com/docs/git-add) the new files and [commit](http://git-scm.com/docs/git-commit) them and [push](http://git-scm.com/docs/git-push) the result, like this:
 
         $ git add youtube_dl/extractor/extractors.py
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index e8c0a5d24..152552dee 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -535,6 +535,7 @@
  - **revision3:embed**
  - **RICE**
  - **RingTV**
+ - **RockstarGames**
  - **RottenTomatoes**
  - **Roxwel**
  - **RTBF**
@@ -699,6 +700,7 @@
  - **TVPlay**: TV3Play and related services
  - **Tweakers**
  - **twitch:chapter**
+ - **twitch:clips**
  - **twitch:past_broadcasts**
  - **twitch:profile**
  - **twitch:stream**
@@ -793,10 +795,11 @@
  - **WNL**
  - **WorldStarHipHop**
  - **wrzuta.pl**
+ - **wrzuta.pl:playlist**
  - **WSJ**: Wall Street Journal
  - **XBef**
  - **XboxClips**
- - **XFileShare**: XFileShare based sites: DaClips, FileHoot, GorillaVid, MovPod, PowerWatch, Rapidvideo.ws, TheVideoBee, Vidto, Streamin.To
+ - **XFileShare**: XFileShare based sites: DaClips, FileHoot, GorillaVid, MovPod, PowerWatch, Rapidvideo.ws, TheVideoBee, Vidto, Streamin.To, XVIDSTAGE
  - **XHamster**
  - **XHamsterEmbed**
  - **xiami:album**: 虾米音乐 - 专辑
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 5e9c14398..e441a5dc4 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.06.12'
+__version__ = '2016.06.14'

From ff4af6ec5903dd922c7bb6ec632b11830c44d04a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 14 Jun 2016 02:49:33 +0700
Subject: [PATCH 5213/6219] [lynda] Remove superfluous _NETRC_MACHINE

---
 youtube_dl/extractor/lynda.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index 5b458d9bc..2d5040032 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -95,7 +95,6 @@ class LyndaIE(LyndaBaseIE):
     IE_NAME = 'lynda'
     IE_DESC = 'lynda.com videos'
     _VALID_URL = r'https?://www\.lynda\.com/(?:[^/]+/[^/]+/\d+|player/embed)/(?P<id>\d+)'
-    _NETRC_MACHINE = 'lynda'
 
     _TIMECODE_REGEX = r'\[(?P<timecode>\d+:\d+:\d+[\.,]\d+)\]'
 

From 4cef70db6c3c3dfd4f45fdc5a85f98bef3ec67a2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 14 Jun 2016 03:16:56 +0700
Subject: [PATCH 5214/6219] [devscripts/release.sh] Add flag for gpg-sign
 commits

---
 devscripts/release.sh | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/devscripts/release.sh b/devscripts/release.sh
index 87e8eda50..f8d466ba8 100755
--- a/devscripts/release.sh
+++ b/devscripts/release.sh
@@ -15,6 +15,7 @@
 set -e
 
 skip_tests=true
+gpg_sign_commits=""
 buildserver='localhost:8142'
 
 while true
@@ -24,6 +25,10 @@ case "$1" in
         skip_tests=false
         shift
     ;;
+    --gpg-sign-commits|-S)
+        gpg_sign_commits="-S"
+        shift
+    ;;
     --buildserver)
         buildserver="$2"
         shift 2
@@ -69,7 +74,7 @@ sed -i "s/__version__ = '.*'/__version__ = '$version'/" youtube_dl/version.py
 /bin/echo -e "\n### Committing documentation, templates and youtube_dl/version.py..."
 make README.md CONTRIBUTING.md .github/ISSUE_TEMPLATE.md supportedsites
 git add README.md CONTRIBUTING.md .github/ISSUE_TEMPLATE.md docs/supportedsites.md youtube_dl/version.py
-git commit -m "release $version"
+git commit $gpg_sign_commits -m "release $version"
 
 /bin/echo -e "\n### Now tagging, signing and pushing..."
 git tag -s -m "Release $version" "$version"
@@ -116,7 +121,7 @@ git clone --branch gh-pages --single-branch . build/gh-pages
     "$ROOT/devscripts/gh-pages/update-copyright.py"
     "$ROOT/devscripts/gh-pages/update-sites.py"
     git add *.html *.html.in update
-    git commit -m "release $version"
+    git commit $gpg_sign_commits -m "release $version"
     git push "$ROOT" gh-pages
     git push "$ORIGIN_URL" gh-pages
 )

From 6c3760292c9d20f891395111bea97f401270d86b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 14 Jun 2016 04:57:59 +0700
Subject: [PATCH 5215/6219] [pornhub] Improve title extraction (Closes #9777)

---
 youtube_dl/extractor/pornhub.py | 39 +++++++++++++++++++++++++++------
 1 file changed, 32 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index 407ea08d4..6d57e1d35 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -1,3 +1,4 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
 import itertools
@@ -39,7 +40,25 @@ class PornHubIE(InfoExtractor):
             'dislike_count': int,
             'comment_count': int,
             'age_limit': 18,
-        }
+        },
+    }, {
+        # non-ASCII title
+        'url': 'http://www.pornhub.com/view_video.php?viewkey=1331683002',
+        'info_dict': {
+            'id': '1331683002',
+            'ext': 'mp4',
+            'title': '重庆婷婷女王足交',
+            'uploader': 'cj397186295',
+            'duration': 1753,
+            'view_count': int,
+            'like_count': int,
+            'dislike_count': int,
+            'comment_count': int,
+            'age_limit': 18,
+        },
+        'params': {
+            'skip_download': True,
+        },
     }, {
         'url': 'http://www.pornhub.com/view_video.php?viewkey=ph557bbb6676d2d',
         'only_matching': True,
@@ -76,19 +95,25 @@ class PornHubIE(InfoExtractor):
                 'PornHub said: %s' % error_msg,
                 expected=True, video_id=video_id)
 
+        # video_title from flashvars contains whitespace instead of non-ASCII (see
+        # http://www.pornhub.com/view_video.php?viewkey=1331683002), not relying
+        # on that anymore.
+        title = self._html_search_meta(
+            'twitter:title', webpage, default=None) or self._search_regex(
+            (r'<h1[^>]+class=["\']title["\'][^>]*>(?P<title>[^<]+)',
+             r'<div[^>]+data-video-title=(["\'])(?P<title>.+?)\1',
+             r'shareTitle\s*=\s*(["\'])(?P<title>.+?)\1'),
+            webpage, 'title', group='title')
+
         flashvars = self._parse_json(
             self._search_regex(
                 r'var\s+flashvars_\d+\s*=\s*({.+?});', webpage, 'flashvars', default='{}'),
             video_id)
         if flashvars:
-            video_title = flashvars.get('video_title')
             thumbnail = flashvars.get('image_url')
             duration = int_or_none(flashvars.get('video_duration'))
         else:
-            video_title, thumbnail, duration = [None] * 3
-
-        if not video_title:
-            video_title = self._html_search_regex(r'<h1 [^>]+>([^<]+)', webpage, 'title')
+            title, thumbnail, duration = [None] * 3
 
         video_uploader = self._html_search_regex(
             r'(?s)From:&nbsp;.+?<(?:a href="/users/|a href="/channels/|span class="username)[^>]+>(.+?)<',
@@ -137,7 +162,7 @@ class PornHubIE(InfoExtractor):
         return {
             'id': video_id,
             'uploader': video_uploader,
-            'title': video_title,
+            'title': title,
             'thumbnail': thumbnail,
             'duration': duration,
             'view_count': view_count,

From 8bc4dbb1af1573f26685b0e609af319dcf34a48e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 14 Jun 2016 11:14:59 +0800
Subject: [PATCH 5216/6219] [wrzuta.pl] Detect error and update _TESTS

---
 youtube_dl/extractor/wrzuta.py | 23 ++++++++++++++---------
 1 file changed, 14 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/wrzuta.py b/youtube_dl/extractor/wrzuta.py
index b811f57fb..bdd7097ba 100644
--- a/youtube_dl/extractor/wrzuta.py
+++ b/youtube_dl/extractor/wrzuta.py
@@ -5,6 +5,7 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
+    ExtractorError,
     int_or_none,
     qualities,
     remove_start,
@@ -27,16 +28,17 @@ class WrzutaIE(InfoExtractor):
             'uploader_id': 'laboratoriumdextera',
             'description': 'md5:7fb5ef3c21c5893375fda51d9b15d9cd',
         },
+        'skip': 'Redirected to wrzuta.pl',
     }, {
-        'url': 'http://jolka85.wrzuta.pl/audio/063jOPX5ue2/liber_natalia_szroeder_-_teraz_ty',
-        'md5': 'bc78077859bea7bcfe4295d7d7fc9025',
+        'url': 'http://vexling.wrzuta.pl/audio/01xBFabGXu6/james_horner_-_into_the_na_39_vi_world_bonus',
+        'md5': 'f80564fb5a2ec6ec59705ae2bf2ba56d',
         'info_dict': {
-            'id': '063jOPX5ue2',
-            'ext': 'ogg',
-            'title': 'Liber & Natalia Szroeder - Teraz Ty',
-            'duration': 203,
-            'uploader_id': 'jolka85',
-            'description': 'md5:2d2b6340f9188c8c4cd891580e481096',
+            'id': '01xBFabGXu6',
+            'ext': 'mp3',
+            'title': 'James Horner - Into The Na\'vi World [Bonus]',
+            'description': 'md5:30a70718b2cd9df3120fce4445b0263b',
+            'duration': 95,
+            'uploader_id': 'vexling',
         },
     }]
 
@@ -46,7 +48,10 @@ class WrzutaIE(InfoExtractor):
         typ = mobj.group('typ')
         uploader = mobj.group('uploader')
 
-        webpage = self._download_webpage(url, video_id)
+        webpage, urlh = self._download_webpage_handle(url, video_id)
+
+        if urlh.geturl() == 'http://www.wrzuta.pl/':
+            raise ExtractorError('Video removed', expected=True)
 
         quality = qualities(['SD', 'MQ', 'HQ', 'HD'])
 

From 4c93ee8d14dc081d413304d2d2eb694cb62cc71a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 15 Jun 2016 22:34:55 +0700
Subject: [PATCH 5217/6219] [imdb] Improve _VALID_URL (Closes #9788)

---
 youtube_dl/extractor/imdb.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/imdb.py b/youtube_dl/extractor/imdb.py
index 3a2b7cec5..0acce9f4c 100644
--- a/youtube_dl/extractor/imdb.py
+++ b/youtube_dl/extractor/imdb.py
@@ -12,7 +12,7 @@ from ..utils import (
 class ImdbIE(InfoExtractor):
     IE_NAME = 'imdb'
     IE_DESC = 'Internet Movie Database trailers'
-    _VALID_URL = r'https?://(?:www|m)\.imdb\.com/video/[^/]+/vi(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www|m)\.imdb\.com/(?:video/[^/]+/|title/tt\d+.*?#lb-)vi(?P<id>\d+)'
 
     _TESTS = [{
         'url': 'http://www.imdb.com/video/imdb/vi2524815897',
@@ -25,6 +25,12 @@ class ImdbIE(InfoExtractor):
     }, {
         'url': 'http://www.imdb.com/video/_/vi2524815897',
         'only_matching': True,
+    }, {
+        'url': 'http://www.imdb.com/title/tt1667889/?ref_=ext_shr_eml_vi#lb-vi2524815897',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.imdb.com/title/tt1667889/#lb-vi2524815897',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 47212f7bcbd59af40f91796562a6b72ba0439ac4 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 16 Jun 2016 11:00:54 +0800
Subject: [PATCH 5218/6219] [utils] Don't transform numbers not starting with a
 zero

Fix test_Viidea and maybe others
---
 test/test_utils.py  | 3 +++
 youtube_dl/utils.py | 2 +-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 0e25de6b7..b7ef51f8d 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -640,6 +640,9 @@ class TestUtil(unittest.TestCase):
             "1":{"src":"skipped", "type": "application/vnd.apple.mpegURL"}
         }''')
 
+        inp = '''{"foo":101}'''
+        self.assertEqual(js_to_json(inp), '''{"foo":101}''')
+
     def test_js_to_json_edgecases(self):
         on = js_to_json("{abc_def:'1\\'\\\\2\\\\\\'3\"4'}")
         self.assertEqual(json.loads(on), {"abc_def": "1'\\2\\'3\"4"})
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index c8308ba3a..82f67f6cd 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1970,7 +1970,7 @@ def js_to_json(code):
         '(?:[^'\\]*(?:\\\\|\\['"nurtbfx/\n]))*[^'\\]*'|
         /\*.*?\*/|,(?=\s*[\]}])|
         [a-zA-Z_][.a-zA-Z_0-9]*|
-        (?:0[xX][0-9a-fA-F]+|0+[0-7]+)(?:\s*:)?|
+        \b(?:0[xX][0-9a-fA-F]+|0+[0-7]+)(?:\s*:)?|
         [0-9]+(?=\s*:)
         ''', fix_kv, code)
 

From 369ff750812ff874a0f4b4ceebb15a024e2f0a9d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 16 Jun 2016 12:26:45 +0800
Subject: [PATCH 5219/6219] [jwplatform] Improved JWPlayer support

---
 youtube_dl/extractor/jwplatform.py | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/youtube_dl/extractor/jwplatform.py b/youtube_dl/extractor/jwplatform.py
index fa6f335e1..e44e31104 100644
--- a/youtube_dl/extractor/jwplatform.py
+++ b/youtube_dl/extractor/jwplatform.py
@@ -12,9 +12,35 @@ from ..utils import (
 
 
 class JWPlatformBaseIE(InfoExtractor):
+    @staticmethod
+    def _find_jwplayer_data(webpage):
+        # TODO: Merge this with JWPlayer-related codes in generic.py
+
+        mobj = re.search(
+            'jwplayer\((?P<quote>[\'"])[^\'" ]+(?P=quote)\)\.setup\((?P<options>[^)]+)\)',
+            webpage)
+        if mobj:
+            return mobj.group('options')
+
+    def _extract_jwplayer_data(self, webpage, video_id, *args, **kwargs):
+        jwplayer_data = self._parse_json(
+            self._find_jwplayer_data(webpage), video_id)
+        return self._parse_jwplayer_data(
+            jwplayer_data, video_id, *args, **kwargs)
+
     def _parse_jwplayer_data(self, jwplayer_data, video_id, require_title=True, m3u8_id=None, rtmp_params=None):
+        # JWPlayer backward compatibility: flattened playlists
+        # https://github.com/jwplayer/jwplayer/blob/v7.4.3/src/js/api/config.js#L81-L96
+        if 'playlist' not in jwplayer_data:
+            jwplayer_data = {'playlist': [jwplayer_data]}
+
         video_data = jwplayer_data['playlist'][0]
 
+        # JWPlayer backward compatibility: flattened sources
+        # https://github.com/jwplayer/jwplayer/blob/v7.4.3/src/js/playlist/item.js#L29-L35
+        if 'sources' not in video_data:
+            video_data['sources'] = [video_data]
+
         formats = []
         for source in video_data['sources']:
             source_url = self._proto_relative_url(source['file'])

From efbd1eb51a9e940d01a2c02fd02c3778dd88b14b Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 16 Jun 2016 12:27:21 +0800
Subject: [PATCH 5220/6219] [wimp] Fix extraction and update _TESTS

---
 youtube_dl/extractor/wimp.py | 33 ++++++++++++++++++---------------
 1 file changed, 18 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/wimp.py b/youtube_dl/extractor/wimp.py
index 828c03dc3..54eb51427 100644
--- a/youtube_dl/extractor/wimp.py
+++ b/youtube_dl/extractor/wimp.py
@@ -1,29 +1,33 @@
 from __future__ import unicode_literals
 
-from .common import InfoExtractor
 from .youtube import YoutubeIE
+from .jwplatform import JWPlatformBaseIE
 
 
-class WimpIE(InfoExtractor):
+class WimpIE(JWPlatformBaseIE):
     _VALID_URL = r'https?://(?:www\.)?wimp\.com/(?P<id>[^/]+)'
     _TESTS = [{
-        'url': 'http://www.wimp.com/maruexhausted/',
+        'url': 'http://www.wimp.com/maru-is-exhausted/',
         'md5': 'ee21217ffd66d058e8b16be340b74883',
         'info_dict': {
-            'id': 'maruexhausted',
+            'id': 'maru-is-exhausted',
             'ext': 'mp4',
             'title': 'Maru is exhausted.',
             'description': 'md5:57e099e857c0a4ea312542b684a869b8',
         }
     }, {
         'url': 'http://www.wimp.com/clowncar/',
-        'md5': '4e2986c793694b55b37cf92521d12bb4',
+        'md5': '5c31ad862a90dc5b1f023956faec13fe',
         'info_dict': {
-            'id': 'clowncar',
+            'id': 'cG4CEr2aiSg',
             'ext': 'webm',
-            'title': 'It\'s like a clown car.',
-            'description': 'md5:0e56db1370a6e49c5c1d19124c0d2fb2',
+            'title': 'Basset hound clown car...incredible!',
+            'description': '5 of my Bassets crawled in this dog loo! www.bellinghambassets.com\n\nFor licensing/usage please contact: licensing(at)jukinmediadotcom',
+            'upload_date': '20140303',
+            'uploader': 'Gretchen Hoey',
+            'uploader_id': 'gretchenandjeff1',
         },
+        'add_ie': ['Youtube'],
     }]
 
     def _real_extract(self, url):
@@ -41,14 +45,13 @@ class WimpIE(InfoExtractor):
                 'ie_key': YoutubeIE.ie_key(),
             }
 
-        video_url = self._search_regex(
-            r'<video[^>]+>\s*<source[^>]+src=(["\'])(?P<url>.+?)\1',
-            webpage, 'video URL', group='url')
+        info_dict = self._extract_jwplayer_data(
+            webpage, video_id, require_title=False)
 
-        return {
+        info_dict.update({
             'id': video_id,
-            'url': video_url,
             'title': self._og_search_title(webpage),
-            'thumbnail': self._og_search_thumbnail(webpage),
             'description': self._og_search_description(webpage),
-        }
+        })
+
+        return info_dict

From 27e5fa8198a187c62cbe7275ebb474d3821bfaa4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 16 Jun 2016 22:33:12 +0700
Subject: [PATCH 5221/6219] [cda] Fix extraction (Closes #9803)

---
 youtube_dl/extractor/cda.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/cda.py b/youtube_dl/extractor/cda.py
index 498d2c0d8..8af318703 100755
--- a/youtube_dl/extractor/cda.py
+++ b/youtube_dl/extractor/cda.py
@@ -58,7 +58,8 @@ class CDAIE(InfoExtractor):
         def extract_format(page, version):
             unpacked = decode_packed_codes(page)
             format_url = self._search_regex(
-                r"url:\\'(.+?)\\'", unpacked, '%s url' % version, fatal=False)
+                r"(?:file|url)\s*:\s*(\\?[\"'])(?P<url>http.+?)\1", unpacked,
+                '%s url' % version, fatal=False, group='url')
             if not format_url:
                 return
             f = {
@@ -75,7 +76,8 @@ class CDAIE(InfoExtractor):
             info_dict['formats'].append(f)
             if not info_dict['duration']:
                 info_dict['duration'] = parse_duration(self._search_regex(
-                    r"duration:\\'(.+?)\\'", unpacked, 'duration', fatal=False))
+                    r"duration\s*:\s*(\\?[\"'])(?P<duration>.+?)\1",
+                    unpacked, 'duration', fatal=False, group='duration'))
 
         extract_format(webpage, 'default')
 

From d2161cade56353e7bee2a0f3eeb2db9f095c2129 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 16 Jun 2016 22:40:55 +0700
Subject: [PATCH 5222/6219] release 2016.06.16

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 docs/supportedsites.md    | 2 +-
 youtube_dl/version.py     | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 4c52c5933..eeac09d5d 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.06.14*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.06.14**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.06.16*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.06.16**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.06.14
+[debug] youtube-dl version 2016.06.16
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 152552dee..13315f4f4 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -44,8 +44,8 @@
  - **appletrailers:section**
  - **archive.org**: archive.org videos
  - **ARD**
- - **ARD:mediathek**: Saarländischer Rundfunk
  - **ARD:mediathek**
+ - **ARD:mediathek**: Saarländischer Rundfunk
  - **arte.tv**
  - **arte.tv:+7**
  - **arte.tv:cinema**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index e441a5dc4..52619cae8 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.06.14'
+__version__ = '2016.06.16'

From ded7511a700cb6962f8a5922b1e3b4ef480a4c81 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Jun 2016 23:42:52 +0700
Subject: [PATCH 5223/6219] [bbccouk] Add support for playlists (Closes #9812)

---
 youtube_dl/extractor/bbc.py        | 75 +++++++++++++++++++++++++++++-
 youtube_dl/extractor/extractors.py |  2 +
 2 files changed, 75 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 74c4510f9..de236fbde 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -31,7 +31,7 @@ class BBCCoUkIE(InfoExtractor):
                             music/clips[/#]|
                             radio/player/
                         )
-                        (?P<id>%s)
+                        (?P<id>%s)(?!/(?:episodes|broadcasts|clips))
                     ''' % _ID_REGEX
 
     _MEDIASELECTOR_URLS = [
@@ -698,7 +698,9 @@ class BBCIE(BBCCoUkIE):
 
     @classmethod
     def suitable(cls, url):
-        return False if BBCCoUkIE.suitable(url) or BBCCoUkArticleIE.suitable(url) else super(BBCIE, cls).suitable(url)
+        EXCLUDE_IE = (BBCCoUkIE, BBCCoUkArticleIE, BBCCoUkIPlayerPlaylistIE, BBCCoUkPlaylistIE)
+        return (False if any(ie.suitable(url) for ie in EXCLUDE_IE)
+                else super(BBCIE, cls).suitable(url))
 
     def _extract_from_media_meta(self, media_meta, video_id):
         # Direct links to media in media metadata (e.g.
@@ -975,3 +977,72 @@ class BBCCoUkArticleIE(InfoExtractor):
             r'<div[^>]+typeof="Clip"[^>]+resource="([^"]+)"', webpage)]
 
         return self.playlist_result(entries, playlist_id, title, description)
+
+
+class BBCCoUkPlaylistBaseIE(InfoExtractor):
+    def _real_extract(self, url):
+        playlist_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, playlist_id)
+
+        entries = [
+            self.url_result(self._URL_TEMPLATE % video_id, BBCCoUkIE.ie_key())
+            for video_id in re.findall(
+                self._VIDEO_ID_TEMPLATE % BBCCoUkIE._ID_REGEX, webpage)]
+
+        title, description = self._extract_title_and_description(webpage)
+
+        return self.playlist_result(entries, playlist_id, title, description)
+
+
+class BBCCoUkIPlayerPlaylistIE(BBCCoUkPlaylistBaseIE):
+    IE_NAME = 'bbc.co.uk:iplayer:playlist'
+    _VALID_URL = r'https?://(?:www\.)?bbc\.co\.uk/iplayer/episodes/(?P<id>%s)' % BBCCoUkIE._ID_REGEX
+    _URL_TEMPLATE = 'http://www.bbc.co.uk/iplayer/episode/%s'
+    _VIDEO_ID_TEMPLATE = r'data-ip-id=["\'](%s)'
+    _TEST = {
+        'url': 'http://www.bbc.co.uk/iplayer/episodes/b05rcz9v',
+        'info_dict': {
+            'id': 'b05rcz9v',
+            'title': 'The Disappearance',
+            'description': 'French thriller serial about a missing teenager.',
+        },
+        'playlist_mincount': 6,
+    }
+
+    def _extract_title_and_description(self, webpage):
+        title = self._search_regex(r'<h1>([^<]+)</h1>', webpage, 'title', fatal=False)
+        description = self._search_regex(
+            r'<p[^>]+class=(["\'])subtitle\1[^>]*>(?P<value>[^<]+)</p>',
+            webpage, 'description', fatal=False, group='value')
+        return title, description
+
+
+class BBCCoUkPlaylistIE(BBCCoUkPlaylistBaseIE):
+    IE_NAME = 'bbc.co.uk:playlist'
+    _VALID_URL = r'https?://(?:www\.)?bbc\.co\.uk/programmes/(?P<id>%s)/(?:episodes|broadcasts|clips)' % BBCCoUkIE._ID_REGEX
+    _URL_TEMPLATE = 'http://www.bbc.co.uk/programmes/%s'
+    _VIDEO_ID_TEMPLATE = r'data-pid=["\'](%s)'
+    _TESTS = [{
+        'url': 'http://www.bbc.co.uk/programmes/b05rcz9v/clips',
+        'info_dict': {
+            'id': 'b05rcz9v',
+            'title': 'The Disappearance - Clips - BBC Four',
+            'description': 'French thriller serial about a missing teenager.',
+        },
+        'playlist_mincount': 7,
+    }, {
+        'url': 'http://www.bbc.co.uk/programmes/b05rcz9v/broadcasts/2016/06',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.bbc.co.uk/programmes/b05rcz9v/clips',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.bbc.co.uk/programmes/b055jkys/episodes/player',
+        'only_matching': True,
+    }]
+
+    def _extract_title_and_description(self, webpage):
+        title = self._og_search_title(webpage, fatal=False)
+        description = self._og_search_description(webpage)
+        return title, description
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 5fce9f47a..ed51dfdaa 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -71,6 +71,8 @@ from .bandcamp import BandcampIE, BandcampAlbumIE
 from .bbc import (
     BBCCoUkIE,
     BBCCoUkArticleIE,
+    BBCCoUkIPlayerPlaylistIE,
+    BBCCoUkPlaylistIE,
     BBCIE,
 )
 from .beeg import BeegIE

From eb451890da79e686a218e42c9761df2a6f5b6ee3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jun 2016 03:04:14 +0700
Subject: [PATCH 5224/6219] [carambatv] Add extractor (Closes #9815)

---
 youtube_dl/extractor/carambatv.py  | 88 ++++++++++++++++++++++++++++++
 youtube_dl/extractor/extractors.py |  4 ++
 2 files changed, 92 insertions(+)
 create mode 100644 youtube_dl/extractor/carambatv.py

diff --git a/youtube_dl/extractor/carambatv.py b/youtube_dl/extractor/carambatv.py
new file mode 100644
index 000000000..5797fb951
--- /dev/null
+++ b/youtube_dl/extractor/carambatv.py
@@ -0,0 +1,88 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..compat import compat_str
+from ..utils import (
+    float_or_none,
+    int_or_none,
+    try_get,
+)
+
+
+class CarambaTVIE(InfoExtractor):
+    _VALID_URL = r'(?:carambatv:|https?://video1\.carambatv\.ru/v/)(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'http://video1.carambatv.ru/v/191910501',
+        'md5': '2f4a81b7cfd5ab866ee2d7270cb34a2a',
+        'info_dict': {
+            'id': '191910501',
+            'ext': 'mp4',
+            'title': '[BadComedian] - Разборка в Маниле (Абсолютный обзор)',
+            'thumbnail': 're:^https?://.*\.jpg',
+            'duration': 2678.31,
+        },
+    }, {
+        'url': 'carambatv:191910501',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        video = self._download_json(
+            'http://video1.carambatv.ru/v/%s/videoinfo.js' % video_id,
+            video_id)
+
+        title = video['title']
+
+        base_url = video.get('video') or 'http://video1.carambatv.ru/v/%s/' % video_id
+
+        formats = [{
+            'url': base_url + f['fn'],
+            'height': int_or_none(f.get('height')),
+            'format_id': '%sp' % f['height'] if f.get('height') else None,
+        } for f in video['qualities'] if f.get('fn')]
+        self._sort_formats(formats)
+
+        thumbnail = video.get('splash')
+        duration = float_or_none(try_get(
+            video, lambda x: x['annotations'][0]['end_time'], compat_str))
+
+        return {
+            'id': video_id,
+            'title': title,
+            'thumbnail': thumbnail,
+            'duration': duration,
+            'formats': formats,
+        }
+
+
+class CarambaTVPageIE(InfoExtractor):
+    _VALID_URL = r'https?://carambatv\.ru/(?:[^/]+/)+(?P<id>[^/?#&]+)'
+    _TEST = {
+        'url': 'http://carambatv.ru/movie/bad-comedian/razborka-v-manile/',
+        'md5': '',
+        'info_dict': {
+            'id': '191910501',
+            'ext': 'mp4',
+            'title': '[BadComedian] - Разборка в Маниле (Абсолютный обзор)',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 2678.31,
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        video_url = self._og_search_property('video:iframe', webpage, default=None)
+
+        if not video_url:
+            video_id = self._search_regex(
+                r'(?:video_id|crmb_vuid)\s*[:=]\s*["\']?(\d+)',
+                webpage, 'video id')
+            video_url = 'carambatv:%s' % video_id
+
+        return self.url_result(video_url, CarambaTVIE.ie_key())
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index ed51dfdaa..23320229b 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -110,6 +110,10 @@ from .camwithher import CamWithHerIE
 from .canalplus import CanalplusIE
 from .canalc2 import Canalc2IE
 from .canvas import CanvasIE
+from .carambatv import (
+    CarambaTVIE,
+    CarambaTVPageIE,
+)
 from .cbc import (
     CBCIE,
     CBCPlayerIE,

From 6929569403aeade9aced5c4103db652e9c16bdca Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jun 2016 04:06:19 +0700
Subject: [PATCH 5225/6219] [mitele] Extract series metadata and make title
 more robust (Closes #9758)

---
 youtube_dl/extractor/mitele.py | 49 +++++++++++++++++++++++++++++++---
 1 file changed, 46 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/mitele.py b/youtube_dl/extractor/mitele.py
index 3589c223d..5a00cd397 100644
--- a/youtube_dl/extractor/mitele.py
+++ b/youtube_dl/extractor/mitele.py
@@ -1,5 +1,8 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse_urlencode,
@@ -8,6 +11,7 @@ from ..compat import (
 from ..utils import (
     get_element_by_attribute,
     int_or_none,
+    remove_start,
 )
 
 
@@ -15,7 +19,7 @@ class MiTeleIE(InfoExtractor):
     IE_DESC = 'mitele.es'
     _VALID_URL = r'https?://www\.mitele\.es/[^/]+/[^/]+/[^/]+/(?P<id>[^/]+)/'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.mitele.es/programas-tv/diario-de/la-redaccion/programa-144/',
         # MD5 is unstable
         'info_dict': {
@@ -24,10 +28,31 @@ class MiTeleIE(InfoExtractor):
             'ext': 'flv',
             'title': 'Tor, la web invisible',
             'description': 'md5:3b6fce7eaa41b2d97358726378d9369f',
+            'series': 'Diario de',
+            'season': 'La redacción',
+            'episode': 'Programa 144',
             'thumbnail': 're:(?i)^https?://.*\.jpg$',
             'duration': 2913,
         },
-    }
+    }, {
+        # no explicit title
+        'url': 'http://www.mitele.es/programas-tv/cuarto-milenio/temporada-6/programa-226/',
+        'info_dict': {
+            'id': 'eLZSwoEd1S3pVyUm8lc6F',
+            'display_id': 'programa-226',
+            'ext': 'flv',
+            'title': 'Cuarto Milenio - Temporada 6 - Programa 226',
+            'description': 'md5:50daf9fadefa4e62d9fc866d0c015701',
+            'series': 'Cuarto Milenio',
+            'season': 'Temporada 6',
+            'episode': 'Programa 226',
+            'thumbnail': 're:(?i)^https?://.*\.jpg$',
+            'duration': 7312,
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }]
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
@@ -70,7 +95,22 @@ class MiTeleIE(InfoExtractor):
         self._sort_formats(formats)
 
         title = self._search_regex(
-            r'class="Destacado-text"[^>]*>\s*<strong>([^<]+)</strong>', webpage, 'title')
+            r'class="Destacado-text"[^>]*>\s*<strong>([^<]+)</strong>',
+            webpage, 'title', default=None)
+
+        mobj = re.search(r'''(?sx)
+                            class="Destacado-text"[^>]*>.*?<h1>\s*
+                            <span>(?P<series>[^<]+)</span>\s*
+                            <span>(?P<season>[^<]+)</span>\s*
+                            <span>(?P<episode>[^<]+)</span>''', webpage)
+        series, season, episode = mobj.groups() if mobj else [None] * 3
+
+        if not title:
+            if mobj:
+                title = '%s - %s - %s' % (series, season, episode)
+            else:
+                title = remove_start(self._search_regex(
+                    r'<title>([^<]+)</title>', webpage, 'title'), 'Ver online ')
 
         video_id = self._search_regex(
             r'data-media-id\s*=\s*"([^"]+)"', webpage,
@@ -83,6 +123,9 @@ class MiTeleIE(InfoExtractor):
             'display_id': display_id,
             'title': title,
             'description': get_element_by_attribute('class', 'text', webpage),
+            'series': series,
+            'season': season,
+            'episode': episode,
             'thumbnail': thumbnail,
             'duration': duration,
             'formats': formats,

From f011876076a9fc4ee3fcb8b17f8bc2bcf5c8b8b0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jun 2016 04:40:48 +0700
Subject: [PATCH 5226/6219] [nickde] Add extractor (Closes #9778)

---
 youtube_dl/extractor/extractors.py |  5 ++++-
 youtube_dl/extractor/nick.py       | 24 ++++++++++++++++++++++++
 2 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 23320229b..efa58f70f 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -518,7 +518,10 @@ from .nhl import (
     NHLVideocenterCategoryIE,
     NHLIE,
 )
-from .nick import NickIE
+from .nick import (
+    NickIE,
+    NickDeIE,
+)
 from .niconico import NiconicoIE, NiconicoPlaylistIE
 from .ninegag import NineGagIE
 from .noco import NocoIE
diff --git a/youtube_dl/extractor/nick.py b/youtube_dl/extractor/nick.py
index ce065f2b0..e96013791 100644
--- a/youtube_dl/extractor/nick.py
+++ b/youtube_dl/extractor/nick.py
@@ -3,6 +3,7 @@ from __future__ import unicode_literals
 
 from .mtv import MTVServicesInfoExtractor
 from ..compat import compat_urllib_parse_urlencode
+from ..utils import update_url_query
 
 
 class NickIE(MTVServicesInfoExtractor):
@@ -61,3 +62,26 @@ class NickIE(MTVServicesInfoExtractor):
 
     def _extract_mgid(self, webpage):
         return self._search_regex(r'data-contenturi="([^"]+)', webpage, 'mgid')
+
+
+class NickDeIE(MTVServicesInfoExtractor):
+    IE_NAME = 'nick.de'
+    _VALID_URL = r'https?://(?:www\.)?nick\.de/(?:playlist|shows)/(?:[^/]+/)*(?P<id>[^/?#&]+)'
+    _TESTS = [{
+        'url': 'http://www.nick.de/playlist/3773-top-videos/videos/episode/17306-zu-wasser-und-zu-land-rauchende-erdnusse',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.nick.de/shows/342-icarly',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        mrss_url = update_url_query(self._search_regex(
+            r'data-mrss=(["\'])(?P<url>http.+?)\1', webpage, 'mrss url', group='url'),
+            {'siteKey': 'nick.de'})
+
+        return self._get_videos_info_from_url(mrss_url, video_id)

From 20a6a154fe8e6a5d246ad1326c4082ddd6091718 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jun 2016 04:46:26 +0700
Subject: [PATCH 5227/6219] [mtv] Use compat_xpath and fix FutureWarning

---
 youtube_dl/extractor/mtv.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index 640ee3d93..8a638a47c 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -6,6 +6,7 @@ from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse_urlencode,
     compat_str,
+    compat_xpath,
 )
 from ..utils import (
     ExtractorError,
@@ -139,9 +140,9 @@ class MTVServicesInfoExtractor(InfoExtractor):
                 itemdoc, './/{http://search.yahoo.com/mrss/}category',
                 'scheme', 'urn:mtvn:video_title')
         if title_el is None:
-            title_el = itemdoc.find('.//{http://search.yahoo.com/mrss/}title')
+            title_el = itemdoc.find(compat_xpath('.//{http://search.yahoo.com/mrss/}title'))
         if title_el is None:
-            title_el = itemdoc.find('.//title') or itemdoc.find('./title')
+            title_el = itemdoc.find(compat_xpath('.//title'))
             if title_el.text is None:
                 title_el = None
 

From d0d93f76ea0dd1dae15bdba6059815d9cc467b05 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jun 2016 05:30:46 +0700
Subject: [PATCH 5228/6219] [pornhd] Fix metadata extraction

---
 youtube_dl/extractor/pornhd.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/pornhd.py b/youtube_dl/extractor/pornhd.py
index 39b53ecf6..33faf5e58 100644
--- a/youtube_dl/extractor/pornhd.py
+++ b/youtube_dl/extractor/pornhd.py
@@ -39,9 +39,10 @@ class PornHdIE(InfoExtractor):
             [r'<span[^>]+class=["\']video-name["\'][^>]*>([^<]+)',
              r'<title>(.+?) - .*?[Pp]ornHD.*?</title>'], webpage, 'title')
         description = self._html_search_regex(
-            r'<div class="description">([^<]+)</div>', webpage, 'description', fatal=False)
+            r'<(div|p)[^>]+class="description"[^>]*>(?P<value>[^<]+)</\1',
+            webpage, 'description', fatal=False, group='value')
         view_count = int_or_none(self._html_search_regex(
-            r'(\d+) views\s*</span>', webpage, 'view count', fatal=False))
+            r'(\d+) views\s*<', webpage, 'view count', fatal=False))
         thumbnail = self._search_regex(
             r"'poster'\s*:\s*'([^']+)'", webpage, 'thumbnail', fatal=False)
 

From e6fe993c318738fee5a4a2ce7a86c4512e42653a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jun 2016 05:37:53 +0700
Subject: [PATCH 5229/6219] [pornhd] Improve formats extraction

---
 youtube_dl/extractor/pornhd.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/pornhd.py b/youtube_dl/extractor/pornhd.py
index 33faf5e58..e7721b013 100644
--- a/youtube_dl/extractor/pornhd.py
+++ b/youtube_dl/extractor/pornhd.py
@@ -7,7 +7,6 @@ from .common import InfoExtractor
 from ..utils import (
     int_or_none,
     js_to_json,
-    qualities,
 )
 
 
@@ -46,18 +45,19 @@ class PornHdIE(InfoExtractor):
         thumbnail = self._search_regex(
             r"'poster'\s*:\s*'([^']+)'", webpage, 'thumbnail', fatal=False)
 
-        quality = qualities(['sd', 'hd'])
         sources = json.loads(js_to_json(self._search_regex(
             r"(?s)'sources'\s*:\s*(\{.+?\})\s*\}[;,)]",
             webpage, 'sources')))
         formats = []
-        for qname, video_url in sources.items():
+        for format_id, video_url in sources.items():
             if not video_url:
                 continue
+            height = int_or_none(self._search_regex(
+                r'^(\d+)[pP]', format_id, 'height', default=None))
             formats.append({
                 'url': video_url,
-                'format_id': qname,
-                'quality': quality(qname),
+                'format_id': format_id,
+                'height': height,
             })
         self._sort_formats(formats)
 

From 667d96480b4f9c78ceace063415c3424d4d562ea Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jun 2016 05:42:20 +0700
Subject: [PATCH 5230/6219] [pornhd] Detect removed videos and modernize

---
 youtube_dl/extractor/pornhd.py | 28 ++++++++++++++++++----------
 1 file changed, 18 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/pornhd.py b/youtube_dl/extractor/pornhd.py
index e7721b013..7a5f00fe0 100644
--- a/youtube_dl/extractor/pornhd.py
+++ b/youtube_dl/extractor/pornhd.py
@@ -1,10 +1,10 @@
 from __future__ import unicode_literals
 
 import re
-import json
 
 from .common import InfoExtractor
 from ..utils import (
+    ExtractorError,
     int_or_none,
     js_to_json,
 )
@@ -37,17 +37,17 @@ class PornHdIE(InfoExtractor):
         title = self._html_search_regex(
             [r'<span[^>]+class=["\']video-name["\'][^>]*>([^<]+)',
              r'<title>(.+?) - .*?[Pp]ornHD.*?</title>'], webpage, 'title')
-        description = self._html_search_regex(
-            r'<(div|p)[^>]+class="description"[^>]*>(?P<value>[^<]+)</\1',
-            webpage, 'description', fatal=False, group='value')
-        view_count = int_or_none(self._html_search_regex(
-            r'(\d+) views\s*<', webpage, 'view count', fatal=False))
-        thumbnail = self._search_regex(
-            r"'poster'\s*:\s*'([^']+)'", webpage, 'thumbnail', fatal=False)
 
-        sources = json.loads(js_to_json(self._search_regex(
+        sources = self._parse_json(js_to_json(self._search_regex(
             r"(?s)'sources'\s*:\s*(\{.+?\})\s*\}[;,)]",
-            webpage, 'sources')))
+            webpage, 'sources', default='{}')), video_id)
+
+        if not sources:
+            message = self._html_search_regex(
+                r'(?s)<(div|p)[^>]+class="no-video"[^>]*>(?P<value>.+?)</\1',
+                webpage, 'error message', group='value')
+            raise ExtractorError('%s said: %s' % (self.IE_NAME, message), expected=True)
+
         formats = []
         for format_id, video_url in sources.items():
             if not video_url:
@@ -61,6 +61,14 @@ class PornHdIE(InfoExtractor):
             })
         self._sort_formats(formats)
 
+        description = self._html_search_regex(
+            r'<(div|p)[^>]+class="description"[^>]*>(?P<value>[^<]+)</\1',
+            webpage, 'description', fatal=False, group='value')
+        view_count = int_or_none(self._html_search_regex(
+            r'(\d+) views\s*<', webpage, 'view count', fatal=False))
+        thumbnail = self._search_regex(
+            r"'poster'\s*:\s*'([^']+)'", webpage, 'thumbnail', fatal=False)
+
         return {
             'id': video_id,
             'display_id': display_id,

From e3944c2621aa46db8623556493dacb87a030b365 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jun 2016 05:50:17 +0700
Subject: [PATCH 5231/6219] [pornhd] Add working test

---
 youtube_dl/extractor/pornhd.py | 21 ++++++++++++++++++---
 1 file changed, 18 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/pornhd.py b/youtube_dl/extractor/pornhd.py
index 7a5f00fe0..8df12eec0 100644
--- a/youtube_dl/extractor/pornhd.py
+++ b/youtube_dl/extractor/pornhd.py
@@ -12,7 +12,21 @@ from ..utils import (
 
 class PornHdIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?pornhd\.com/(?:[a-z]{2,4}/)?videos/(?P<id>\d+)(?:/(?P<display_id>.+))?'
-    _TEST = {
+    _TESTS = [{
+        'url': 'http://www.pornhd.com/videos/9864/selfie-restroom-masturbation-fun-with-chubby-cutie-hd-porn-video',
+        'md5': 'c8b964b1f0a4b5f7f28ae3a5c9f86ad5',
+        'info_dict': {
+            'id': '9864',
+            'display_id': 'selfie-restroom-masturbation-fun-with-chubby-cutie-hd-porn-video',
+            'ext': 'mp4',
+            'title': 'Restroom selfie masturbation',
+            'description': 'md5:3748420395e03e31ac96857a8f125b2b',
+            'thumbnail': 're:^https?://.*\.jpg',
+            'view_count': int,
+            'age_limit': 18,
+        }
+    }, {
+        # removed video
         'url': 'http://www.pornhd.com/videos/1962/sierra-day-gets-his-cum-all-over-herself-hd-porn-video',
         'md5': '956b8ca569f7f4d8ec563e2c41598441',
         'info_dict': {
@@ -24,8 +38,9 @@ class PornHdIE(InfoExtractor):
             'thumbnail': 're:^https?://.*\.jpg',
             'view_count': int,
             'age_limit': 18,
-        }
-    }
+        },
+        'skip': 'Not available anymore',
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)

From 5fc2757682483b4b0277df2e2454dab882237882 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jun 2016 06:00:05 +0700
Subject: [PATCH 5232/6219] release 2016.06.18

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 docs/supportedsites.md    | 5 +++++
 youtube_dl/version.py     | 2 +-
 3 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index eeac09d5d..4b281e649 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.06.16*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.06.16**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.06.18*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.06.18**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.06.16
+[debug] youtube-dl version 2016.06.18
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 13315f4f4..6a7a4bf2d 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -74,6 +74,8 @@
  - **bbc**: BBC
  - **bbc.co.uk**: BBC iPlayer
  - **bbc.co.uk:article**: BBC articles
+ - **bbc.co.uk:iplayer:playlist**
+ - **bbc.co.uk:playlist**
  - **BeatportPro**
  - **Beeg**
  - **BehindKink**
@@ -104,6 +106,8 @@
  - **canalc2.tv**
  - **Canalplus**: canalplus.fr, piwiplus.fr and d8.tv
  - **Canvas**
+ - **CarambaTV**
+ - **CarambaTVPage**
  - **CBC**
  - **CBCPlayer**
  - **CBS**
@@ -432,6 +436,7 @@
  - **nhl.com:videocenter**
  - **nhl.com:videocenter:category**: NHL videocenter category
  - **nick.com**
+ - **nick.de**
  - **niconico**: ニコニコ動画
  - **NiconicoPlaylist**
  - **njoy**: N-JOY
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 52619cae8..92b7badc9 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.06.16'
+__version__ = '2016.06.18'

From 0f47cc2e925014afef4339a8213d52797a710eb3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jun 2016 06:20:34 +0700
Subject: [PATCH 5233/6219] release 2016.06.18.1

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 docs/supportedsites.md    | 2 +-
 youtube_dl/version.py     | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 4b281e649..2736bb23b 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.06.18*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.06.18**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.06.18.1*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.06.18.1**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.06.18
+[debug] youtube-dl version 2016.06.18.1
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 6a7a4bf2d..c79798d86 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -44,8 +44,8 @@
  - **appletrailers:section**
  - **archive.org**: archive.org videos
  - **ARD**
- - **ARD:mediathek**
  - **ARD:mediathek**: Saarländischer Rundfunk
+ - **ARD:mediathek**
  - **arte.tv**
  - **arte.tv:+7**
  - **arte.tv:cinema**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 92b7badc9..f71bc8c2a 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.06.18'
+__version__ = '2016.06.18.1'

From c878e635de1563ded54b2504104e94cd59c70b55 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 18 Jun 2016 12:17:24 +0800
Subject: [PATCH 5234/6219] [bet] Moved to MTVServices

---
 youtube_dl/extractor/bet.py | 96 +++++++++++++------------------------
 1 file changed, 34 insertions(+), 62 deletions(-)

diff --git a/youtube_dl/extractor/bet.py b/youtube_dl/extractor/bet.py
index 986245bf0..bd3ee2e2e 100644
--- a/youtube_dl/extractor/bet.py
+++ b/youtube_dl/extractor/bet.py
@@ -1,31 +1,27 @@
 from __future__ import unicode_literals
 
-from .common import InfoExtractor
-from ..compat import compat_urllib_parse_unquote
-from ..utils import (
-    xpath_text,
-    xpath_with_ns,
-    int_or_none,
-    parse_iso8601,
-)
+from .mtv import MTVServicesInfoExtractor
+from ..utils import unified_strdate
+from ..compat import compat_urllib_parse_urlencode
 
 
-class BetIE(InfoExtractor):
+class BetIE(MTVServicesInfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?bet\.com/(?:[^/]+/)+(?P<id>.+?)\.html'
     _TESTS = [
         {
             'url': 'http://www.bet.com/news/politics/2014/12/08/in-bet-exclusive-obama-talks-race-and-racism.html',
             'info_dict': {
-                'id': 'news/national/2014/a-conversation-with-president-obama',
+                'id': '07e96bd3-8850-3051-b856-271b457f0ab8',
                 'display_id': 'in-bet-exclusive-obama-talks-race-and-racism',
                 'ext': 'flv',
                 'title': 'A Conversation With President Obama',
-                'description': 'md5:699d0652a350cf3e491cd15cc745b5da',
+                'description': 'President Obama urges persistence in confronting racism and bias.',
                 'duration': 1534,
-                'timestamp': 1418075340,
                 'upload_date': '20141208',
-                'uploader': 'admin',
                 'thumbnail': 're:(?i)^https?://.*\.jpg$',
+                'subtitles': {
+                    'en': 'mincount:2',
+                }
             },
             'params': {
                 # rtmp download
@@ -35,16 +31,17 @@ class BetIE(InfoExtractor):
         {
             'url': 'http://www.bet.com/video/news/national/2014/justice-for-ferguson-a-community-reacts.html',
             'info_dict': {
-                'id': 'news/national/2014/justice-for-ferguson-a-community-reacts',
+                'id': '9f516bf1-7543-39c4-8076-dd441b459ba9',
                 'display_id': 'justice-for-ferguson-a-community-reacts',
                 'ext': 'flv',
                 'title': 'Justice for Ferguson: A Community Reacts',
                 'description': 'A BET News special.',
                 'duration': 1696,
-                'timestamp': 1416942360,
                 'upload_date': '20141125',
-                'uploader': 'admin',
                 'thumbnail': 're:(?i)^https?://.*\.jpg$',
+                'subtitles': {
+                    'en': 'mincount:2',
+                }
             },
             'params': {
                 # rtmp download
@@ -53,57 +50,32 @@ class BetIE(InfoExtractor):
         }
     ]
 
+    _FEED_URL = "http://feeds.mtvnservices.com/od/feed/bet-mrss-player"
+
+    def _get_feed_query(self, uri):
+        return compat_urllib_parse_urlencode({
+            'uuid': uri,
+        })
+
+    def _extract_mgid(self, webpage):
+        return self._search_regex(r'data-uri="([^"]+)', webpage, 'mgid')
+
     def _real_extract(self, url):
         display_id = self._match_id(url)
+
         webpage = self._download_webpage(url, display_id)
+        mgid = self._extract_mgid(webpage)
+        videos_info = self._get_videos_info(mgid)
 
-        media_url = compat_urllib_parse_unquote(self._search_regex(
-            [r'mediaURL\s*:\s*"([^"]+)"', r"var\s+mrssMediaUrl\s*=\s*'([^']+)'"],
-            webpage, 'media URL'))
+        info_dict = videos_info['entries'][0]
 
-        video_id = self._search_regex(
-            r'/video/(.*)/_jcr_content/', media_url, 'video id')
+        upload_date = unified_strdate(self._html_search_meta('date', webpage))
+        description = self._html_search_meta('description', webpage)
 
-        mrss = self._download_xml(media_url, display_id)
-
-        item = mrss.find('./channel/item')
-
-        NS_MAP = {
-            'dc': 'http://purl.org/dc/elements/1.1/',
-            'media': 'http://search.yahoo.com/mrss/',
-            'ka': 'http://kickapps.com/karss',
-        }
-
-        title = xpath_text(item, './title', 'title')
-        description = xpath_text(
-            item, './description', 'description', fatal=False)
-
-        timestamp = parse_iso8601(xpath_text(
-            item, xpath_with_ns('./dc:date', NS_MAP),
-            'upload date', fatal=False))
-        uploader = xpath_text(
-            item, xpath_with_ns('./dc:creator', NS_MAP),
-            'uploader', fatal=False)
-
-        media_content = item.find(
-            xpath_with_ns('./media:content', NS_MAP))
-        duration = int_or_none(media_content.get('duration'))
-        smil_url = media_content.get('url')
-
-        thumbnail = media_content.find(
-            xpath_with_ns('./media:thumbnail', NS_MAP)).get('url')
-
-        formats = self._extract_smil_formats(smil_url, display_id)
-        self._sort_formats(formats)
-
-        return {
-            'id': video_id,
+        info_dict.update({
             'display_id': display_id,
-            'title': title,
             'description': description,
-            'thumbnail': thumbnail,
-            'timestamp': timestamp,
-            'uploader': uploader,
-            'duration': duration,
-            'formats': formats,
-        }
+            'upload_date': upload_date,
+        })
+
+        return info_dict

From a72df5f36feddaffbcfa35e3415562509a9f67b4 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 18 Jun 2016 12:19:06 +0800
Subject: [PATCH 5235/6219] [mtvservices] Fix ext for RTMP streams

---
 youtube_dl/extractor/mtv.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index 8a638a47c..dd0639589 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -85,9 +85,10 @@ class MTVServicesInfoExtractor(InfoExtractor):
                 rtmp_video_url = rendition.find('./src').text
                 if rtmp_video_url.endswith('siteunavail.png'):
                     continue
+                new_url = self._transform_rtmp_url(rtmp_video_url)
                 formats.append({
-                    'ext': ext,
-                    'url': self._transform_rtmp_url(rtmp_video_url),
+                    'ext': 'flv' if new_url.startswith('rtmp') else ext,
+                    'url': new_url,
                     'format_id': rendition.get('bitrate'),
                     'width': int(rendition.get('width')),
                     'height': int(rendition.get('height')),

From 26264cb0566309bcdc7f0f2a8ee376bf205fc675 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 18 Jun 2016 12:21:40 +0800
Subject: [PATCH 5236/6219] [adobetv] Use embedded data in the webpage

Sometimes the HTML webpage is returned even with '?format=json'
---
 youtube_dl/extractor/adobetv.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/adobetv.py b/youtube_dl/extractor/adobetv.py
index 8753ee2cf..5ae16fa16 100644
--- a/youtube_dl/extractor/adobetv.py
+++ b/youtube_dl/extractor/adobetv.py
@@ -156,7 +156,10 @@ class AdobeTVVideoIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        video_data = self._download_json(url + '?format=json', video_id)
+        webpage = self._download_webpage(url, video_id)
+
+        video_data = self._parse_json(self._search_regex(
+            r'var\s+bridge\s*=\s*([^;]+);', webpage, 'bridged data'), video_id)
 
         formats = [{
             'format_id': '%s-%s' % (determine_ext(source['src']), source.get('height')),

From 1b6cf16be7e8a771ca79e7eb161db9ff59238dab Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 18 Jun 2016 12:27:39 +0800
Subject: [PATCH 5237/6219] [aftonbladet] Fix extraction

---
 youtube_dl/extractor/aftonbladet.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/aftonbladet.py b/youtube_dl/extractor/aftonbladet.py
index d548592fe..5766b4fe8 100644
--- a/youtube_dl/extractor/aftonbladet.py
+++ b/youtube_dl/extractor/aftonbladet.py
@@ -24,10 +24,10 @@ class AftonbladetIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         # find internal video meta data
-        meta_url = 'http://aftonbladet-play.drlib.aptoma.no/video/%s.json'
+        meta_url = 'http://aftonbladet-play-metadata.cdn.drvideo.aptoma.no/video/%s.json'
         player_config = self._parse_json(self._html_search_regex(
             r'data-player-config="([^"]+)"', webpage, 'player config'), video_id)
-        internal_meta_id = player_config['videoId']
+        internal_meta_id = player_config['aptomaVideoId']
         internal_meta_url = meta_url % internal_meta_id
         internal_meta_json = self._download_json(
             internal_meta_url, video_id, 'Downloading video meta data')

From 09e3f91e408eb357929abad3710c799376004138 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 18 Jun 2016 12:34:58 +0800
Subject: [PATCH 5238/6219] [arte] Update _TESTS and fix for pages with
 multiple YouTube videos

Some tests are from #6895 and #6613
---
 youtube_dl/extractor/arte.py | 36 +++++++++++++++---------------------
 1 file changed, 15 insertions(+), 21 deletions(-)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index f40532929..9e39faf47 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -240,10 +240,10 @@ class ArteTVPlus7IE(ArteTVBaseIE):
             return self._extract_from_json_url(json_url, video_id, lang, title=title)
         # Different kind of embed URL (e.g.
         # http://www.arte.tv/magazine/trepalium/fr/episode-0406-replay-trepalium)
-        embed_url = self._search_regex(
-            r'<iframe[^>]+src=(["\'])(?P<url>.+?)\1',
-            webpage, 'embed url', group='url')
-        return self.url_result(embed_url)
+        entries = [
+            self.url_result(url)
+            for _, url in re.findall(r'<iframe[^>]+src=(["\'])(?P<url>.+?)\1', webpage)]
+        return self.playlist_result(entries)
 
 
 # It also uses the arte_vp_url url from the webpage to extract the information
@@ -252,22 +252,17 @@ class ArteTVCreativeIE(ArteTVPlus7IE):
     _VALID_URL = r'https?://creative\.arte\.tv/(?P<lang>fr|de|en|es)/(?:[^/]+/)*(?P<id>[^/?#&]+)'
 
     _TESTS = [{
-        'url': 'http://creative.arte.tv/de/magazin/agentur-amateur-corporate-design',
+        'url': 'http://creative.arte.tv/fr/episode/osmosis-episode-1',
         'info_dict': {
-            'id': '72176',
+            'id': '057405-001-A',
             'ext': 'mp4',
-            'title': 'Folge 2 - Corporate Design',
-            'upload_date': '20131004',
+            'title': 'OSMOSIS - N\'AYEZ PLUS PEUR D\'AIMER (1)',
+            'upload_date': '20150716',
         },
     }, {
         'url': 'http://creative.arte.tv/fr/Monty-Python-Reunion',
-        'info_dict': {
-            'id': '160676',
-            'ext': 'mp4',
-            'title': 'Monty Python live (mostly)',
-            'description': 'Événement ! Quarante-cinq ans après leurs premiers succès, les légendaires Monty Python remontent sur scène.\n',
-            'upload_date': '20140805',
-        }
+        'playlist_count': 11,
+        'add_ie': ['Youtube'],
     }, {
         'url': 'http://creative.arte.tv/de/episode/agentur-amateur-4-der-erste-kunde',
         'only_matching': True,
@@ -349,14 +344,13 @@ class ArteTVCinemaIE(ArteTVPlus7IE):
     _VALID_URL = r'https?://cinema\.arte\.tv/(?P<lang>fr|de|en|es)/(?P<id>.+)'
 
     _TESTS = [{
-        'url': 'http://cinema.arte.tv/de/node/38291',
-        'md5': '6b275511a5107c60bacbeeda368c3aa1',
+        'url': 'http://cinema.arte.tv/fr/article/les-ailes-du-desir-de-julia-reck',
+        'md5': 'a5b9dd5575a11d93daf0e3f404f45438',
         'info_dict': {
-            'id': '055876-000_PWA12025-D',
+            'id': '062494-000-A',
             'ext': 'mp4',
-            'title': 'Tod auf dem Nil',
-            'upload_date': '20160122',
-            'description': 'md5:7f749bbb77d800ef2be11d54529b96bc',
+            'title': 'Film lauréat du concours web - "Les ailes du désir" de Julia Reck',
+            'upload_date': '20150807',
         },
     }]
 

From 573c35272f7a1973e44109614c8639e0d3e21fdd Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 18 Jun 2016 12:35:55 +0800
Subject: [PATCH 5239/6219] [bbc] Skip a geo-restricted test case

---
 youtube_dl/extractor/bbc.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index de236fbde..4b3cd8c65 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -192,6 +192,7 @@ class BBCCoUkIE(InfoExtractor):
                 # rtmp download
                 'skip_download': True,
             },
+            'skip': 'Now it\'s really geo-restricted',
         }, {
             # compact player (https://github.com/rg3/youtube-dl/issues/8147)
             'url': 'http://www.bbc.co.uk/programmes/p028bfkf/player',

From 1f3574575851eb34b6c6a983e276fa77a0dc3da1 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 18 Jun 2016 12:39:08 +0800
Subject: [PATCH 5240/6219] [azubu] Don't fail on optional fields

---
 youtube_dl/extractor/azubu.py | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/azubu.py b/youtube_dl/extractor/azubu.py
index efa624de1..a813eb429 100644
--- a/youtube_dl/extractor/azubu.py
+++ b/youtube_dl/extractor/azubu.py
@@ -46,6 +46,7 @@ class AzubuIE(InfoExtractor):
                 'uploader_id': 272749,
                 'view_count': int,
             },
+            'skip': 'Channel offline',
         },
     ]
 
@@ -56,22 +57,26 @@ class AzubuIE(InfoExtractor):
             'http://www.azubu.tv/api/video/%s' % video_id, video_id)['data']
 
         title = data['title'].strip()
-        description = data['description']
-        thumbnail = data['thumbnail']
-        view_count = data['view_count']
-        uploader = data['user']['username']
-        uploader_id = data['user']['id']
+        description = data.get('description')
+        thumbnail = data.get('thumbnail')
+        view_count = data.get('view_count')
+        user = data.get('user', {})
+        uploader = user.get('username')
+        uploader_id = user.get('id')
 
         stream_params = json.loads(data['stream_params'])
 
-        timestamp = float_or_none(stream_params['creationDate'], 1000)
-        duration = float_or_none(stream_params['length'], 1000)
+        timestamp = float_or_none(stream_params.get('creationDate'), 1000)
+        duration = float_or_none(stream_params.get('length'), 1000)
 
         renditions = stream_params.get('renditions') or []
         video = stream_params.get('FLVFullLength') or stream_params.get('videoFullLength')
         if video:
             renditions.append(video)
 
+        if not renditions and not user.get('channel', {}).get('is_live', True):
+            raise ExtractorError('%s said: channel is offline.' % self.IE_NAME, expected=True)
+
         formats = [{
             'url': fmt['url'],
             'width': fmt['frameWidth'],

From 0278aa443f7b4cc0886f40d6704925dc2488921d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 18 Jun 2016 12:53:48 +0800
Subject: [PATCH 5241/6219] [br] Skip invalid tests

---
 youtube_dl/extractor/br.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/br.py b/youtube_dl/extractor/br.py
index 11cf49851..ff0aa11b1 100644
--- a/youtube_dl/extractor/br.py
+++ b/youtube_dl/extractor/br.py
@@ -29,7 +29,8 @@ class BRIE(InfoExtractor):
                 'duration': 180,
                 'uploader': 'Reinhard Weber',
                 'upload_date': '20150422',
-            }
+            },
+            'skip': '404 not found',
         },
         {
             'url': 'http://www.br.de/nachrichten/oberbayern/inhalt/muenchner-polizeipraesident-schreiber-gestorben-100.html',
@@ -40,7 +41,8 @@ class BRIE(InfoExtractor):
                 'title': 'Manfred Schreiber ist tot',
                 'description': 'md5:b454d867f2a9fc524ebe88c3f5092d97',
                 'duration': 26,
-            }
+            },
+            'skip': '404 not found',
         },
         {
             'url': 'https://www.br-klassik.de/audio/peeping-tom-premierenkritik-dance-festival-muenchen-100.html',
@@ -51,7 +53,8 @@ class BRIE(InfoExtractor):
                 'title': 'Kurzweilig und sehr bewegend',
                 'description': 'md5:0351996e3283d64adeb38ede91fac54e',
                 'duration': 296,
-            }
+            },
+            'skip': '404 not found',
         },
         {
             'url': 'http://www.br.de/radio/bayern1/service/team/videos/team-video-erdelt100.html',

From 6d0d4fc26d45c55ef6e99b31892047b0bdfed0e0 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 18 Jun 2016 13:40:55 +0800
Subject: [PATCH 5242/6219] [wdr] Add WDRBaseIE, for Sportschau (#9799)

---
 youtube_dl/extractor/wdr.py | 179 +++++++++++++++++++-----------------
 1 file changed, 95 insertions(+), 84 deletions(-)

diff --git a/youtube_dl/extractor/wdr.py b/youtube_dl/extractor/wdr.py
index 6b83a2a04..390f9e830 100644
--- a/youtube_dl/extractor/wdr.py
+++ b/youtube_dl/extractor/wdr.py
@@ -15,7 +15,87 @@ from ..utils import (
 )
 
 
-class WDRIE(InfoExtractor):
+class WDRBaseIE(InfoExtractor):
+    def _extract_wdr_video(self, webpage, display_id):
+        # for wdr.de the data-extension is in a tag with the class "mediaLink"
+        # for wdr.de radio players, in a tag with the class "wdrrPlayerPlayBtn"
+        # for wdrmaus its in a link to the page in a multiline "videoLink"-tag
+        json_metadata = self._html_search_regex(
+            r'class=(?:"(?:mediaLink|wdrrPlayerPlayBtn)\b[^"]*"[^>]+|"videoLink\b[^"]*"[\s]*>\n[^\n]*)data-extension="([^"]+)"',
+            webpage, 'media link', default=None, flags=re.MULTILINE)
+
+        if not json_metadata:
+            return
+
+        media_link_obj = self._parse_json(json_metadata, display_id,
+                                          transform_source=js_to_json)
+        jsonp_url = media_link_obj['mediaObj']['url']
+
+        metadata = self._download_json(
+            jsonp_url, 'metadata', transform_source=strip_jsonp)
+
+        metadata_tracker_data = metadata['trackerData']
+        metadata_media_resource = metadata['mediaResource']
+
+        formats = []
+
+        # check if the metadata contains a direct URL to a file
+        for kind, media_resource in metadata_media_resource.items():
+            if kind not in ('dflt', 'alt'):
+                continue
+
+            for tag_name, medium_url in media_resource.items():
+                if tag_name not in ('videoURL', 'audioURL'):
+                    continue
+
+                ext = determine_ext(medium_url)
+                if ext == 'm3u8':
+                    formats.extend(self._extract_m3u8_formats(
+                        medium_url, display_id, 'mp4', 'm3u8_native',
+                        m3u8_id='hls'))
+                elif ext == 'f4m':
+                    manifest_url = update_url_query(
+                        medium_url, {'hdcore': '3.2.0', 'plugin': 'aasp-3.2.0.77.18'})
+                    formats.extend(self._extract_f4m_formats(
+                        manifest_url, display_id, f4m_id='hds', fatal=False))
+                elif ext == 'smil':
+                    formats.extend(self._extract_smil_formats(
+                        medium_url, 'stream', fatal=False))
+                else:
+                    a_format = {
+                        'url': medium_url
+                    }
+                    if ext == 'unknown_video':
+                        urlh = self._request_webpage(
+                            medium_url, display_id, note='Determining extension')
+                        ext = urlhandle_detect_ext(urlh)
+                        a_format['ext'] = ext
+                    formats.append(a_format)
+
+        self._sort_formats(formats)
+
+        subtitles = {}
+        caption_url = metadata_media_resource.get('captionURL')
+        if caption_url:
+            subtitles['de'] = [{
+                'url': caption_url,
+                'ext': 'ttml',
+            }]
+
+        title = metadata_tracker_data['trackerClipTitle']
+
+        return {
+            'id': metadata_tracker_data.get('trackerClipId', display_id),
+            'display_id': display_id,
+            'title': title,
+            'alt_title': metadata_tracker_data.get('trackerClipSubcategory'),
+            'formats': formats,
+            'subtitles': subtitles,
+            'upload_date': unified_strdate(metadata_tracker_data.get('trackerClipAirTime')),
+        }
+
+
+class WDRIE(WDRBaseIE):
     _CURRENT_MAUS_URL = r'https?://(?:www\.)wdrmaus.de/(?:[^/]+/){1,2}[^/?#]+\.php5'
     _PAGE_REGEX = r'/(?:mediathek/)?[^/]+/(?P<type>[^/]+)/(?P<display_id>.+)\.html'
     _VALID_URL = r'(?P<page_url>https?://(?:www\d\.)?wdr\d?\.de)' + _PAGE_REGEX + '|' + _CURRENT_MAUS_URL
@@ -91,10 +171,10 @@ class WDRIE(InfoExtractor):
         },
         {
             'url': 'http://www.wdrmaus.de/sachgeschichten/sachgeschichten/achterbahn.php5',
-            # HDS download, MD5 is unstable
+            'md5': '803138901f6368ee497b4d195bb164f2',
             'info_dict': {
                 'id': 'mdb-186083',
-                'ext': 'flv',
+                'ext': 'mp4',
                 'upload_date': '20130919',
                 'title': 'Sachgeschichte - Achterbahn ',
                 'description': '- Die Sendung mit der Maus -',
@@ -120,14 +200,9 @@ class WDRIE(InfoExtractor):
         display_id = mobj.group('display_id')
         webpage = self._download_webpage(url, display_id)
 
-        # for wdr.de the data-extension is in a tag with the class "mediaLink"
-        # for wdr.de radio players, in a tag with the class "wdrrPlayerPlayBtn"
-        # for wdrmaus its in a link to the page in a multiline "videoLink"-tag
-        json_metadata = self._html_search_regex(
-            r'class=(?:"(?:mediaLink|wdrrPlayerPlayBtn)\b[^"]*"[^>]+|"videoLink\b[^"]*"[\s]*>\n[^\n]*)data-extension="([^"]+)"',
-            webpage, 'media link', default=None, flags=re.MULTILINE)
+        info_dict = self._extract_wdr_video(webpage, display_id)
 
-        if not json_metadata:
+        if not info_dict:
             entries = [
                 self.url_result(page_url + href[0], 'WDR')
                 for href in re.findall(
@@ -140,86 +215,22 @@ class WDRIE(InfoExtractor):
 
             raise ExtractorError('No downloadable streams found', expected=True)
 
-        media_link_obj = self._parse_json(json_metadata, display_id,
-                                          transform_source=js_to_json)
-        jsonp_url = media_link_obj['mediaObj']['url']
-
-        metadata = self._download_json(
-            jsonp_url, 'metadata', transform_source=strip_jsonp)
-
-        metadata_tracker_data = metadata['trackerData']
-        metadata_media_resource = metadata['mediaResource']
-
-        formats = []
-
-        # check if the metadata contains a direct URL to a file
-        for kind, media_resource in metadata_media_resource.items():
-            if kind not in ('dflt', 'alt'):
-                continue
-
-            for tag_name, medium_url in media_resource.items():
-                if tag_name not in ('videoURL', 'audioURL'):
-                    continue
-
-                ext = determine_ext(medium_url)
-                if ext == 'm3u8':
-                    formats.extend(self._extract_m3u8_formats(
-                        medium_url, display_id, 'mp4', 'm3u8_native',
-                        m3u8_id='hls'))
-                elif ext == 'f4m':
-                    manifest_url = update_url_query(
-                        medium_url, {'hdcore': '3.2.0', 'plugin': 'aasp-3.2.0.77.18'})
-                    formats.extend(self._extract_f4m_formats(
-                        manifest_url, display_id, f4m_id='hds', fatal=False))
-                elif ext == 'smil':
-                    formats.extend(self._extract_smil_formats(
-                        medium_url, 'stream', fatal=False))
-                else:
-                    a_format = {
-                        'url': medium_url
-                    }
-                    if ext == 'unknown_video':
-                        urlh = self._request_webpage(
-                            medium_url, display_id, note='Determining extension')
-                        ext = urlhandle_detect_ext(urlh)
-                        a_format['ext'] = ext
-                    formats.append(a_format)
-
-        self._sort_formats(formats)
-
-        subtitles = {}
-        caption_url = metadata_media_resource.get('captionURL')
-        if caption_url:
-            subtitles['de'] = [{
-                'url': caption_url,
-                'ext': 'ttml',
-            }]
-
-        title = metadata_tracker_data.get('trackerClipTitle')
         is_live = url_type == 'live'
 
         if is_live:
-            title = self._live_title(title)
-            upload_date = None
-        elif 'trackerClipAirTime' in metadata_tracker_data:
-            upload_date = metadata_tracker_data['trackerClipAirTime']
-        else:
-            upload_date = self._html_search_meta('DC.Date', webpage, 'upload date')
+            info_dict.update({
+                'title': self._live_title(info_dict['title']),
+                'upload_date': None,
+            })
+        elif 'upload_date' not in info_dict:
+            info_dict['upload_date'] = unified_strdate(self._html_search_meta('DC.Date', webpage, 'upload date'))
 
-        if upload_date:
-            upload_date = unified_strdate(upload_date)
-
-        return {
-            'id': metadata_tracker_data.get('trackerClipId', display_id),
-            'display_id': display_id,
-            'title': title,
-            'alt_title': metadata_tracker_data.get('trackerClipSubcategory'),
-            'formats': formats,
-            'upload_date': upload_date,
+        info_dict.update({
             'description': self._html_search_meta('Description', webpage),
             'is_live': is_live,
-            'subtitles': subtitles,
-        }
+        })
+
+        return info_dict
 
 
 class WDRMobileIE(InfoExtractor):

From b5aad37f6bdc72acaca198202dc9f7eaa3185e51 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 18 Jun 2016 13:41:50 +0800
Subject: [PATCH 5243/6219] [ard] Remove SportschauIE, which is now based on
 WDR (#9799)

---
 youtube_dl/extractor/ard.py | 39 -------------------------------------
 1 file changed, 39 deletions(-)

diff --git a/youtube_dl/extractor/ard.py b/youtube_dl/extractor/ard.py
index 26446c2fe..fd45b3e42 100644
--- a/youtube_dl/extractor/ard.py
+++ b/youtube_dl/extractor/ard.py
@@ -8,7 +8,6 @@ from .generic import GenericIE
 from ..utils import (
     determine_ext,
     ExtractorError,
-    get_element_by_attribute,
     qualities,
     int_or_none,
     parse_duration,
@@ -274,41 +273,3 @@ class ARDIE(InfoExtractor):
             'upload_date': upload_date,
             'thumbnail': thumbnail,
         }
-
-
-class SportschauIE(ARDMediathekIE):
-    IE_NAME = 'Sportschau'
-    _VALID_URL = r'(?P<baseurl>https?://(?:www\.)?sportschau\.de/(?:[^/]+/)+video(?P<id>[^/#?]+))\.html'
-    _TESTS = [{
-        'url': 'http://www.sportschau.de/tourdefrance/videoseppeltkokainhatnichtsmitklassischemdopingzutun100.html',
-        'info_dict': {
-            'id': 'seppeltkokainhatnichtsmitklassischemdopingzutun100',
-            'ext': 'mp4',
-            'title': 'Seppelt: "Kokain hat nichts mit klassischem Doping zu tun"',
-            'thumbnail': 're:^https?://.*\.jpg$',
-            'description': 'Der ARD-Doping Experte Hajo Seppelt gibt seine Einschätzung zum ersten Dopingfall der diesjährigen Tour de France um den Italiener Luca Paolini ab.',
-        },
-        'params': {
-            # m3u8 download
-            'skip_download': True,
-        },
-    }]
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        base_url = mobj.group('baseurl')
-
-        webpage = self._download_webpage(url, video_id)
-        title = get_element_by_attribute('class', 'headline', webpage)
-        description = self._html_search_meta('description', webpage, 'description')
-
-        info = self._extract_media_info(
-            base_url + '-mc_defaultQuality-h.json', webpage, video_id)
-
-        info.update({
-            'title': title,
-            'description': description,
-        })
-
-        return info

From e8f13f2637fd33b20ac2682dbbdaef63b6288bf4 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 18 Jun 2016 13:42:58 +0800
Subject: [PATCH 5244/6219] [sportschau.de] Fix extraction and moved to its own
 file (closes #9799)

---
 youtube_dl/extractor/sportschau.py | 38 ++++++++++++++++++++++++++++++
 1 file changed, 38 insertions(+)
 create mode 100644 youtube_dl/extractor/sportschau.py

diff --git a/youtube_dl/extractor/sportschau.py b/youtube_dl/extractor/sportschau.py
new file mode 100644
index 000000000..0d7925a08
--- /dev/null
+++ b/youtube_dl/extractor/sportschau.py
@@ -0,0 +1,38 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .wdr import WDRBaseIE
+from ..utils import get_element_by_attribute
+
+
+class SportschauIE(WDRBaseIE):
+    IE_NAME = 'Sportschau'
+    _VALID_URL = r'https?://(?:www\.)?sportschau\.de/(?:[^/]+/)+video-?(?P<id>[^/#?]+)\.html'
+    _TEST = {
+        'url': 'http://www.sportschau.de/uefaeuro2016/videos/video-dfb-team-geht-gut-gelaunt-ins-spiel-gegen-polen-100.html',
+        'info_dict': {
+            'id': 'mdb-1140188',
+            'display_id': 'dfb-team-geht-gut-gelaunt-ins-spiel-gegen-polen-100',
+            'ext': 'mp4',
+            'title': 'DFB-Team geht gut gelaunt ins Spiel gegen Polen',
+            'description': 'Vor dem zweiten Gruppenspiel gegen Polen herrscht gute Stimmung im deutschen Team. Insbesondere Bastian Schweinsteiger strotzt vor Optimismus nach seinem Tor gegen die Ukraine.',
+            'upload_date': '20160615',
+        },
+        'skip': 'Geo-restricted to Germany',
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+        title = get_element_by_attribute('class', 'headline', webpage)
+        description = self._html_search_meta('description', webpage, 'description')
+
+        info = self._extract_wdr_video(webpage, video_id)
+
+        info.update({
+            'title': title,
+            'description': description,
+        })
+
+        return info

From b0b128049a9180e58698e74bdc8079d80086dbef Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 18 Jun 2016 13:43:47 +0800
Subject: [PATCH 5245/6219] [extractors] Update references to sportschau
 (#9799)

---
 youtube_dl/extractor/extractors.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index efa58f70f..6dc5904b3 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -44,7 +44,6 @@ from .archiveorg import ArchiveOrgIE
 from .ard import (
     ARDIE,
     ARDMediathekIE,
-    SportschauIE,
 )
 from .arte import (
     ArteTvIE,
@@ -747,6 +746,7 @@ from .sportbox import (
     SportBoxEmbedIE,
 )
 from .sportdeutschland import SportDeutschlandIE
+from .sportschau import SportschauIE
 from .srgssr import (
     SRGSSRIE,
     SRGSSRPlayIE,

From d7c6c656c5c2fa64a1f8a4365a5fe62861b1dceb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jun 2016 21:42:17 +0700
Subject: [PATCH 5246/6219] [arte:+7] Expand _VALID_URL (Closes #9820)

---
 youtube_dl/extractor/arte.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index 9e39faf47..e602d3673 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -180,11 +180,14 @@ class ArteTVBaseIE(InfoExtractor):
 
 class ArteTVPlus7IE(ArteTVBaseIE):
     IE_NAME = 'arte.tv:+7'
-    _VALID_URL = r'https?://(?:www\.)?arte\.tv/guide/(?P<lang>fr|de|en|es)/(?:(?:sendungen|emissions|embed)/)?(?P<id>[^/]+)/(?P<name>[^/?#&]+)'
+    _VALID_URL = r'https?://(?:(?:www|sites)\.)?arte\.tv/[^/]+/(?P<lang>fr|de|en|es)/(?:(?:sendungen|emissions|embed)/)?(?P<id>[^/]+)/(?P<name>[^/?#&]+)'
 
     _TESTS = [{
         'url': 'http://www.arte.tv/guide/de/sendungen/XEN/xenius/?vid=055918-015_PLUS7-D',
         'only_matching': True,
+    }. {
+        'url': 'http://sites.arte.tv/karambolage/de/video/karambolage-22',
+        'only_matching': True,
     }]
 
     @classmethod

From c1823c8ad9ca49f61ef15bbc126b0f95e4825d14 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jun 2016 22:08:48 +0700
Subject: [PATCH 5247/6219] [README.md] Remove 'small' from description (#9814)

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index f1e59542d..c6feef116 100644
--- a/README.md
+++ b/README.md
@@ -44,7 +44,7 @@ Or with [MacPorts](https://www.macports.org/):
 Alternatively, refer to the [developer instructions](#developer-instructions) for how to check out and work with the git repository. For further options, including PGP signatures, see the [youtube-dl Download Page](https://rg3.github.io/youtube-dl/download.html).
 
 # DESCRIPTION
-**youtube-dl** is a small command-line program to download videos from
+**youtube-dl** is a command-line program to download videos from
 YouTube.com and a few more sites. It requires the Python interpreter, version
 2.6, 2.7, or 3.2+, and it is not platform specific. It should work on
 your Unix box, on Windows or on Mac OS X. It is released to the public domain,

From 90b6288cce3e5a433a521bc862d98d31be9624c2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Jun 2016 22:23:48 +0700
Subject: [PATCH 5248/6219] [arte:+7] Simplify _VALID_URL

---
 youtube_dl/extractor/arte.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index e602d3673..049f1fa9e 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -180,12 +180,12 @@ class ArteTVBaseIE(InfoExtractor):
 
 class ArteTVPlus7IE(ArteTVBaseIE):
     IE_NAME = 'arte.tv:+7'
-    _VALID_URL = r'https?://(?:(?:www|sites)\.)?arte\.tv/[^/]+/(?P<lang>fr|de|en|es)/(?:(?:sendungen|emissions|embed)/)?(?P<id>[^/]+)/(?P<name>[^/?#&]+)'
+    _VALID_URL = r'https?://(?:(?:www|sites)\.)?arte\.tv/[^/]+/(?P<lang>fr|de|en|es)/(?:[^/]+/)*(?P<id>[^/?#&]+)'
 
     _TESTS = [{
         'url': 'http://www.arte.tv/guide/de/sendungen/XEN/xenius/?vid=055918-015_PLUS7-D',
         'only_matching': True,
-    }. {
+    }, {
         'url': 'http://sites.arte.tv/karambolage/de/video/karambolage-22',
         'only_matching': True,
     }]

From 41c1023300596f62dff93d9275f5e4d7a6762e66 Mon Sep 17 00:00:00 2001
From: Steven Gosseling <steven@stevengosseling.nl>
Date: Fri, 26 Feb 2016 13:31:52 +0100
Subject: [PATCH 5249/6219] [closertotruth] Add extractor

Removed print statement from code.

Replaced two regex searches with the corret ones.

Removed some unnecessary semicolumns

fixed title extraction

refactored everything to search_regex

processed comments on commit 5650b0d, fixed feedback from flake8

Improved regexes and returns info dict now.

Added support for closertotruth interview URL

Added support for episodes page
---
 youtube_dl/extractor/closertotruth.py | 69 +++++++++++++++++++++++++++
 1 file changed, 69 insertions(+)
 create mode 100644 youtube_dl/extractor/closertotruth.py

diff --git a/youtube_dl/extractor/closertotruth.py b/youtube_dl/extractor/closertotruth.py
new file mode 100644
index 000000000..d04ff5e4f
--- /dev/null
+++ b/youtube_dl/extractor/closertotruth.py
@@ -0,0 +1,69 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class CloserToTruthIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?closertotruth\.com/(episodes/|(series|interviews)/(?:[^#]+#video-)?(?P<id>\d+))'
+    _TESTS = [
+        {
+            'url': 'http://closertotruth.com/series/solutions-the-mind-body-problem#video-3688',
+            'md5': '5c548bde260a9247ddfdc07c7458ed29',
+            'info_dict': {
+                'id': '0_zof1ktre',
+                'ext': 'mov',
+                'title': 'Solutions to the Mind-Body Problem?',
+                'upload_date': '20140221',
+                'timestamp': 1392956007,
+                'uploader_id': 'CTTXML'
+            }
+        },
+        {
+            'url': 'http://closertotruth.com/interviews/1725',
+            'md5': 'b00598fd6a38372edb976408f72c5792',
+            'info_dict': {
+                'id': '0_19qv5rn1',
+                'ext': 'mov',
+                'title': 'AyaFr-002 - Francisco J. Ayala',
+                'upload_date': '20140307',
+                'timestamp': 1394236431,
+                'uploader_id': 'CTTXML'
+            }
+        },
+        {
+            'url': 'http://closertotruth.com/episodes/how-do-brains-work',
+            'md5': '4dd96aa0a5c296afa5c0bd24895c2f16',
+            'info_dict': {
+                'id': '0_iuxai6g6',
+                'ext': 'mov',
+                'title': 'How do Brains Work?',
+                'upload_date': '20140221',
+                'timestamp': 1392956024,
+                'uploader_id': 'CTTXML'
+            }
+        },
+    ]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        video_title = self._search_regex(r'<title>(.+) \|.+</title>', webpage, 'video title')
+
+        entry_id = self._search_regex(r'<a[^>]+id="(?:video-%s|embed-kaltura)"[^>]+data-kaltura="([^"]+)' % video_id, webpage, "video entry_id")
+
+        interviewee_name = self._search_regex(r'<div id="(?:node_interview_full_group_white_wrapper|node_interview_series_full_group_ajax_content)"(?:.|\n)*<h3>(.*)</h3>.+', webpage, "video interviewee_name", False)
+
+        if interviewee_name:
+            video_title = video_title + ' - ' + interviewee_name
+
+        p_id = self._search_regex(r'<script[^>]+src=["\'].+?partner_id/(\d+)', webpage, "kaltura partner_id")
+
+        return {
+            '_type': 'url_transparent',
+            'id': entry_id,
+            'url': 'kaltura:%s:%s' % (p_id, entry_id),
+            'ie_key': 'Kaltura',
+            'title': video_title
+        }

From cb23192bc4c56d80229a7a5f70cb61d0879db6c5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 19 Jun 2016 00:35:29 +0700
Subject: [PATCH 5250/6219] [closertotruth] Update and improve (Closes #8680)

---
 youtube_dl/extractor/closertotruth.py | 117 +++++++++++++++-----------
 youtube_dl/extractor/extractors.py    |   1 +
 2 files changed, 71 insertions(+), 47 deletions(-)

diff --git a/youtube_dl/extractor/closertotruth.py b/youtube_dl/extractor/closertotruth.py
index d04ff5e4f..26243d52d 100644
--- a/youtube_dl/extractor/closertotruth.py
+++ b/youtube_dl/extractor/closertotruth.py
@@ -1,69 +1,92 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 
 
 class CloserToTruthIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?closertotruth\.com/(episodes/|(series|interviews)/(?:[^#]+#video-)?(?P<id>\d+))'
-    _TESTS = [
-        {
-            'url': 'http://closertotruth.com/series/solutions-the-mind-body-problem#video-3688',
-            'md5': '5c548bde260a9247ddfdc07c7458ed29',
-            'info_dict': {
-                'id': '0_zof1ktre',
-                'ext': 'mov',
-                'title': 'Solutions to the Mind-Body Problem?',
-                'upload_date': '20140221',
-                'timestamp': 1392956007,
-                'uploader_id': 'CTTXML'
-            }
+    _VALID_URL = r'https?://(?:www\.)?closertotruth\.com/(?:[^/]+/)*(?P<id>[^/?#&]+)'
+    _TESTS = [{
+        'url': 'http://closertotruth.com/series/solutions-the-mind-body-problem#video-3688',
+        'info_dict': {
+            'id': '0_zof1ktre',
+            'display_id': 'solutions-the-mind-body-problem',
+            'ext': 'mov',
+            'title': 'Solutions to the Mind-Body Problem?',
+            'upload_date': '20140221',
+            'timestamp': 1392956007,
+            'uploader_id': 'CTTXML'
         },
-        {
-            'url': 'http://closertotruth.com/interviews/1725',
-            'md5': 'b00598fd6a38372edb976408f72c5792',
-            'info_dict': {
-                'id': '0_19qv5rn1',
-                'ext': 'mov',
-                'title': 'AyaFr-002 - Francisco J. Ayala',
-                'upload_date': '20140307',
-                'timestamp': 1394236431,
-                'uploader_id': 'CTTXML'
-            }
+        'params': {
+            'skip_download': True,
         },
-        {
-            'url': 'http://closertotruth.com/episodes/how-do-brains-work',
-            'md5': '4dd96aa0a5c296afa5c0bd24895c2f16',
-            'info_dict': {
-                'id': '0_iuxai6g6',
-                'ext': 'mov',
-                'title': 'How do Brains Work?',
-                'upload_date': '20140221',
-                'timestamp': 1392956024,
-                'uploader_id': 'CTTXML'
-            }
+    }, {
+        'url': 'http://closertotruth.com/episodes/how-do-brains-work',
+        'info_dict': {
+            'id': '0_iuxai6g6',
+            'display_id': 'how-do-brains-work',
+            'ext': 'mov',
+            'title': 'How do Brains Work?',
+            'upload_date': '20140221',
+            'timestamp': 1392956024,
+            'uploader_id': 'CTTXML'
         },
-    ]
+        'params': {
+            'skip_download': True,
+        },
+    }, {
+        'url': 'http://closertotruth.com/interviews/1725',
+        'info_dict': {
+            'id': '1725',
+            'title': 'AyaFr-002',
+        },
+        'playlist_mincount': 2,
+    }]
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
+        display_id = self._match_id(url)
 
-        video_title = self._search_regex(r'<title>(.+) \|.+</title>', webpage, 'video title')
+        webpage = self._download_webpage(url, display_id)
 
-        entry_id = self._search_regex(r'<a[^>]+id="(?:video-%s|embed-kaltura)"[^>]+data-kaltura="([^"]+)' % video_id, webpage, "video entry_id")
+        partner_id = self._search_regex(
+            r'<script[^>]+src=["\'].*?\b(?:partner_id|p)/(\d+)',
+            webpage, 'kaltura partner_id')
 
-        interviewee_name = self._search_regex(r'<div id="(?:node_interview_full_group_white_wrapper|node_interview_series_full_group_ajax_content)"(?:.|\n)*<h3>(.*)</h3>.+', webpage, "video interviewee_name", False)
+        title = self._search_regex(
+            r'<title>(.+?)\s*\|\s*.+?</title>', webpage, 'video title')
 
-        if interviewee_name:
-            video_title = video_title + ' - ' + interviewee_name
+        select = self._search_regex(
+            r'(?s)<select[^>]+id="select-version"[^>]*>(.+?)</select>',
+            webpage, 'select version', default=None)
+        if select:
+            entry_ids = set()
+            entries = []
+            for mobj in re.finditer(
+                    r'<option[^>]+value=(["\'])(?P<id>[0-9a-z_]+)(?:#.+?)?\1[^>]*>(?P<title>[^<]+)',
+                    webpage):
+                entry_id = mobj.group('id')
+                if entry_id in entry_ids:
+                    continue
+                entry_ids.add(entry_id)
+                entries.append({
+                    '_type': 'url_transparent',
+                    'url': 'kaltura:%s:%s' % (partner_id, entry_id),
+                    'ie_key': 'Kaltura',
+                    'title': mobj.group('title'),
+                })
+            if entries:
+                return self.playlist_result(entries, display_id, title)
 
-        p_id = self._search_regex(r'<script[^>]+src=["\'].+?partner_id/(\d+)', webpage, "kaltura partner_id")
+        entry_id = self._search_regex(
+            r'<a[^>]+id=(["\'])embed-kaltura\1[^>]+data-kaltura=(["\'])(?P<id>[0-9a-z_]+)\2',
+            webpage, 'kaltura entry_id', group='id')
 
         return {
             '_type': 'url_transparent',
-            'id': entry_id,
-            'url': 'kaltura:%s:%s' % (p_id, entry_id),
+            'display_id': display_id,
+            'url': 'kaltura:%s:%s' % (partner_id, entry_id),
             'ie_key': 'Kaltura',
-            'title': video_title
+            'title': title
         }
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 6dc5904b3..2ff867651 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -140,6 +140,7 @@ from .cliprs import ClipRsIE
 from .clipfish import ClipfishIE
 from .cliphunter import CliphunterIE
 from .clipsyndicate import ClipsyndicateIE
+from .closertotruth import CloserToTruthIE
 from .cloudy import CloudyIE
 from .clubic import ClubicIE
 from .clyp import ClypIE

From 7577d849a62ecdcc52ede6dcf73edf2a717fc646 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 19 Jun 2016 02:25:34 +0700
Subject: [PATCH 5251/6219] [r7] Fix extraction and add support for articles
 (Closes #9826)

---
 youtube_dl/extractor/extractors.py |  5 +-
 youtube_dl/extractor/r7.py         | 95 +++++++++++++++++++-----------
 2 files changed, 64 insertions(+), 36 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 2ff867651..b1b04f2fc 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -631,7 +631,10 @@ from .qqmusic import (
     QQMusicToplistIE,
     QQMusicPlaylistIE,
 )
-from .r7 import R7IE
+from .r7 import (
+    R7IE,
+    R7ArticleIE,
+)
 from .radiocanada import (
     RadioCanadaIE,
     RadioCanadaAudioVideoIE,
diff --git a/youtube_dl/extractor/r7.py b/youtube_dl/extractor/r7.py
index 976c8feec..069dbfaed 100644
--- a/youtube_dl/extractor/r7.py
+++ b/youtube_dl/extractor/r7.py
@@ -2,22 +2,19 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..utils import (
-    js_to_json,
-    unescapeHTML,
-    int_or_none,
-)
+from ..utils import int_or_none
 
 
 class R7IE(InfoExtractor):
-    _VALID_URL = r'''(?x)https?://
+    _VALID_URL = r'''(?x)
+                        https?://
                         (?:
                             (?:[a-zA-Z]+)\.r7\.com(?:/[^/]+)+/idmedia/|
                             noticias\.r7\.com(?:/[^/]+)+/[^/]+-|
                             player\.r7\.com/video/i/
                         )
                         (?P<id>[\da-f]{24})
-                        '''
+                    '''
     _TESTS = [{
         'url': 'http://videos.r7.com/policiais-humilham-suspeito-a-beira-da-morte-morre-com-dignidade-/idmedia/54e7050b0cf2ff57e0279389.html',
         'md5': '403c4e393617e8e8ddc748978ee8efde',
@@ -25,6 +22,7 @@ class R7IE(InfoExtractor):
             'id': '54e7050b0cf2ff57e0279389',
             'ext': 'mp4',
             'title': 'Policiais humilham suspeito à beira da morte: "Morre com dignidade"',
+            'description': 'md5:01812008664be76a6479aa58ec865b72',
             'thumbnail': 're:^https?://.*\.jpg$',
             'duration': 98,
             'like_count': int,
@@ -44,45 +42,72 @@ class R7IE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        webpage = self._download_webpage(
-            'http://player.r7.com/video/i/%s' % video_id, video_id)
+        video = self._download_json(
+            'http://player-api.r7.com/video/i/%s' % video_id, video_id)
 
-        item = self._parse_json(js_to_json(self._search_regex(
-            r'(?s)var\s+item\s*=\s*({.+?});', webpage, 'player')), video_id)
-
-        title = unescapeHTML(item['title'])
-        thumbnail = item.get('init', {}).get('thumbUri')
-        duration = None
-
-        statistics = item.get('statistics', {})
-        like_count = int_or_none(statistics.get('likes'))
-        view_count = int_or_none(statistics.get('views'))
+        title = video['title']
 
         formats = []
-        for format_key, format_dict in item['playlist'][0].items():
-            src = format_dict.get('src')
-            if not src:
-                continue
-            format_id = format_dict.get('format') or format_key
-            if duration is None:
-                duration = format_dict.get('duration')
-            if '.f4m' in src:
-                formats.extend(self._extract_f4m_formats(src, video_id, preference=-1))
-            elif src.endswith('.m3u8'):
-                formats.extend(self._extract_m3u8_formats(src, video_id, 'mp4', preference=-2))
-            else:
-                formats.append({
-                    'url': src,
-                    'format_id': format_id,
-                })
+        media_url_hls = video.get('media_url_hls')
+        if media_url_hls:
+            formats.extend(self._extract_m3u8_formats(
+                media_url_hls, video_id, 'mp4', entry_protocol='m3u8_native',
+                m3u8_id='hls', fatal=False))
+        media_url = video.get('media_url')
+        if media_url:
+            f = {
+                'url': media_url,
+                'format_id': 'http',
+            }
+            # m3u8 format always matches the http format, let's copy metadata from
+            # one to another
+            m3u8_formats = list(filter(
+                lambda f: f.get('vcodec') != 'none' and f.get('resolution') != 'multiple',
+                formats))
+            if len(m3u8_formats) == 1:
+                f_copy = m3u8_formats[0].copy()
+                f_copy.update(f)
+                f_copy['protocol'] = 'http'
+                f = f_copy
+            formats.append(f)
         self._sort_formats(formats)
 
+        description = video.get('description')
+        thumbnail = video.get('thumb')
+        duration = int_or_none(video.get('media_duration'))
+        like_count = int_or_none(video.get('likes'))
+        view_count = int_or_none(video.get('views'))
+
         return {
             'id': video_id,
             'title': title,
+            'description': description,
             'thumbnail': thumbnail,
             'duration': duration,
             'like_count': like_count,
             'view_count': view_count,
             'formats': formats,
         }
+
+
+class R7ArticleIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:[a-zA-Z]+)\.r7\.com/(?:[^/]+/)+[^/?#&]+-(?P<id>\d+)'
+    _TEST = {
+        'url': 'http://tv.r7.com/record-play/balanco-geral/videos/policiais-humilham-suspeito-a-beira-da-morte-morre-com-dignidade-16102015',
+        'only_matching': True,
+    }
+
+    @classmethod
+    def suitable(cls, url):
+        return False if R7IE.suitable(url) else super(R7ArticleIE, cls).suitable(url)
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, display_id)
+
+        video_id = self._search_regex(
+            r'<div[^>]+(?:id=["\']player-|class=["\']embed["\'][^>]+id=["\'])([\da-f]{24})',
+            webpage, 'video id')
+
+        return self.url_result('http://player.r7.com/video/i/%s' % video_id, R7IE.ie_key())

From 589568789f500b7a515355a07efec4bcec0f3243 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 19 Jun 2016 02:30:29 +0700
Subject: [PATCH 5252/6219] release 2016.06.19

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 docs/supportedsites.md    | 4 +++-
 youtube_dl/version.py     | 2 +-
 3 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 2736bb23b..52e04aa74 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.06.18.1*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.06.18.1**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.06.19*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.06.19**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.06.18.1
+[debug] youtube-dl version 2016.06.19
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index c79798d86..7c90940c7 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -44,8 +44,8 @@
  - **appletrailers:section**
  - **archive.org**: archive.org videos
  - **ARD**
- - **ARD:mediathek**: Saarländischer Rundfunk
  - **ARD:mediathek**
+ - **ARD:mediathek**: Saarländischer Rundfunk
  - **arte.tv**
  - **arte.tv:+7**
  - **arte.tv:cinema**
@@ -128,6 +128,7 @@
  - **cliphunter**
  - **ClipRs**
  - **Clipsyndicate**
+ - **CloserToTruth**
  - **cloudtime**: CloudTime
  - **Cloudy**
  - **Clubic**
@@ -521,6 +522,7 @@
  - **qqmusic:singer**: QQ音乐 - 歌手
  - **qqmusic:toplist**: QQ音乐 - 排行榜
  - **R7**
+ - **R7Article**
  - **radio.de**
  - **radiobremen**
  - **radiocanada**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index f71bc8c2a..417e86ed6 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.06.18.1'
+__version__ = '2016.06.19'

From 7c05097633138459e9bdf7e10738e021b04689a7 Mon Sep 17 00:00:00 2001
From: Lucas Moura <lucas.moura128@gmail.com>
Date: Sat, 18 Jun 2016 17:01:47 -0300
Subject: [PATCH 5253/6219] [jsinterp] Avoid double key lookup for setting new
 key

In order to add a new key to both __objects and __functions dicts on jsinterp.py, it is
necessary to first verify if a key was present and if not, create the key and
assign it to a value.

However, this can be done with a single step using dict setdefault method.
---
 youtube_dl/jsinterp.py | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index a7440c582..4a5a0dbc3 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -131,9 +131,8 @@ class JSInterpreter(object):
             if variable in local_vars:
                 obj = local_vars[variable]
             else:
-                if variable not in self._objects:
-                    self._objects[variable] = self.extract_object(variable)
-                obj = self._objects[variable]
+                obj = self._objects.setdefault(
+                    variable, self.extract_object(variable))
 
             if arg_str is None:
                 # Member access
@@ -204,8 +203,7 @@ class JSInterpreter(object):
             argvals = tuple([
                 int(v) if v.isdigit() else local_vars[v]
                 for v in m.group('args').split(',')])
-            if fname not in self._functions:
-                self._functions[fname] = self.extract_function(fname)
+            self._functions.setdefault(fname, self.extract_function(fname))
             return self._functions[fname](argvals)
 
         raise ExtractorError('Unsupported JS expression %r' % expr)

From 6a55bb66ee4367a8445c8df2d803090e68c42fe9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 19 Jun 2016 03:56:01 +0700
Subject: [PATCH 5254/6219] [vimeo] Fix rented videos (Closes #9830)

---
 youtube_dl/extractor/vimeo.py | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 0fd2c18a0..c52986af6 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -8,6 +8,7 @@ import itertools
 from .common import InfoExtractor
 from ..compat import (
     compat_HTTPError,
+    compat_str,
     compat_urlparse,
 )
 from ..utils import (
@@ -24,6 +25,7 @@ from ..utils import (
     urlencode_postdata,
     unescapeHTML,
     parse_filesize,
+    try_get,
 )
 
 
@@ -445,7 +447,18 @@ class VimeoIE(VimeoBaseInfoExtractor):
             if config.get('view') == 4:
                 config = self._verify_player_video_password(url, video_id)
 
-        if '>You rented this title.<' in webpage:
+        def is_rented():
+            if '>You rented this title.<' in webpage:
+                return True
+            if config.get('user', {}).get('purchased'):
+                return True
+            label = try_get(
+                config, lambda x: x['video']['vod']['purchase_options'][0]['label_string'], compat_str)
+            if label and label.startswith('You rented this'):
+                return True
+            return False
+
+        if is_rented():
             feature_id = config.get('video', {}).get('vod', {}).get('feature_id')
             if feature_id and not data.get('force_feature_id', False):
                 return self.url_result(smuggle_url(

From a50fd6e0263b6e5d97a13a0f781a2325c0ab7efa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 19 Jun 2016 03:57:14 +0700
Subject: [PATCH 5255/6219] release 2016.06.19.1

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 youtube_dl/version.py     | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 52e04aa74..1b25628bf 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.06.19*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.06.19**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.06.19.1*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.06.19.1**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.06.19
+[debug] youtube-dl version 2016.06.19.1
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 417e86ed6..a3e2c3079 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.06.19'
+__version__ = '2016.06.19.1'

From e154c65128305fcaf6c6b2d5fd41ecd83e4ec154 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 10 Jan 2016 20:09:53 +0100
Subject: [PATCH 5256/6219] [downloader/hls] Add support for AES-128 encrypted
 segments in hlsnative downloader

---
 youtube_dl/downloader/hls.py   | 78 ++++++++++++++++++++++++----------
 youtube_dl/extractor/common.py | 17 ++------
 youtube_dl/utils.py            |  9 ++++
 3 files changed, 68 insertions(+), 36 deletions(-)

diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index 54f2108e9..1d5f178a0 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -2,14 +2,24 @@ from __future__ import unicode_literals
 
 import os.path
 import re
+import binascii
+try:
+    from Crypto.Cipher import AES
+    can_decrypt_frag = True
+except ImportError:
+    can_decrypt_frag = False
 
 from .fragment import FragmentFD
 from .external import FFmpegFD
 
-from ..compat import compat_urlparse
+from ..compat import (
+    compat_urlparse,
+    compat_struct_pack,
+)
 from ..utils import (
     encodeFilename,
     sanitize_open,
+    parse_m3u8_attributes,
 )
 
 
@@ -21,7 +31,7 @@ class HlsFD(FragmentFD):
     @staticmethod
     def can_download(manifest):
         UNSUPPORTED_FEATURES = (
-            r'#EXT-X-KEY:METHOD=(?!NONE)',  # encrypted streams [1]
+            r'#EXT-X-KEY:METHOD=(?!NONE|AES-128)',  # encrypted streams [1]
             r'#EXT-X-BYTERANGE',  # playlists composed of byte ranges of media files [2]
 
             # Live streams heuristic does not always work (e.g. geo restricted to Germany
@@ -39,7 +49,9 @@ class HlsFD(FragmentFD):
             # 3. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.3.2
             # 4. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.3.5
         )
-        return all(not re.search(feature, manifest) for feature in UNSUPPORTED_FEATURES)
+        check_results = [not re.search(feature, manifest) for feature in UNSUPPORTED_FEATURES]
+        check_results.append(not (re.search(r'#EXT-X-KEY:METHOD=AES-128', manifest) and not can_decrypt_frag))
+        return all(check_results)
 
     def real_download(self, filename, info_dict):
         man_url = info_dict['url']
@@ -57,36 +69,58 @@ class HlsFD(FragmentFD):
                 fd.add_progress_hook(ph)
             return fd.real_download(filename, info_dict)
 
-        fragment_urls = []
+        total_frags = 0
         for line in s.splitlines():
             line = line.strip()
             if line and not line.startswith('#'):
-                segment_url = (
-                    line
-                    if re.match(r'^https?://', line)
-                    else compat_urlparse.urljoin(man_url, line))
-                fragment_urls.append(segment_url)
-                # We only download the first fragment during the test
-                if self.params.get('test', False):
-                    break
+                total_frags += 1
 
         ctx = {
             'filename': filename,
-            'total_frags': len(fragment_urls),
+            'total_frags': total_frags,
         }
 
         self._prepare_and_start_frag_download(ctx)
 
+        i = 0
+        media_sequence = 0
+        decrypt_info = {'METHOD': 'NONE'}
         frags_filenames = []
-        for i, frag_url in enumerate(fragment_urls):
-            frag_filename = '%s-Frag%d' % (ctx['tmpfilename'], i)
-            success = ctx['dl'].download(frag_filename, {'url': frag_url})
-            if not success:
-                return False
-            down, frag_sanitized = sanitize_open(frag_filename, 'rb')
-            ctx['dest_stream'].write(down.read())
-            down.close()
-            frags_filenames.append(frag_sanitized)
+        for line in s.splitlines():
+            line = line.strip()
+            if line:
+                if not line.startswith('#'):
+                    frag_url = (
+                        line
+                        if re.match(r'^https?://', line)
+                        else compat_urlparse.urljoin(man_url, line))
+                    frag_filename = '%s-Frag%d' % (ctx['tmpfilename'], i)
+                    success = ctx['dl'].download(frag_filename, {'url': frag_url})
+                    if not success:
+                        return False
+                    down, frag_sanitized = sanitize_open(frag_filename, 'rb')
+                    frag_content = down.read()
+                    down.close()
+                    if decrypt_info['METHOD'] == 'AES-128':
+                        iv = decrypt_info.get('IV') or compat_struct_pack(">8xq", media_sequence)
+                        frag_content = AES.new(decrypt_info['KEY'], AES.MODE_CBC, iv).decrypt(frag_content)
+                    ctx['dest_stream'].write(frag_content)
+                    frags_filenames.append(frag_sanitized)
+                    # We only download the first fragment during the test
+                    if self.params.get('test', False):
+                        break
+                    i += 1
+                    media_sequence += 1
+                elif line.startswith('#EXT-X-KEY'):
+                    decrypt_info = parse_m3u8_attributes(line[11:])
+                    if decrypt_info['METHOD'] == 'AES-128':
+                        if 'IV' in decrypt_info:
+                            decrypt_info['IV'] = binascii.unhexlify(decrypt_info['IV'][2:])
+                        if not re.match(r'^https?://', decrypt_info['URI']):
+                            decrypt_info['URI'] = compat_urlparse.urljoin(man_url, decrypt_info['URI'])
+                        decrypt_info['KEY'] = self.ydl.urlopen(decrypt_info['URI']).read()
+                elif line.startswith('#EXT-X-MEDIA-SEQUENCE'):
+                    media_sequence = int(line[22:])
 
         self._finish_frag_download(ctx)
 
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index bfd432160..5a2603b50 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -53,6 +53,7 @@ from ..utils import (
     mimetype2ext,
     update_Request,
     update_url_query,
+    parse_m3u8_attributes,
 )
 
 
@@ -1150,23 +1151,11 @@ class InfoExtractor(object):
             }]
         last_info = None
         last_media = None
-        kv_rex = re.compile(
-            r'(?P<key>[a-zA-Z_-]+)=(?P<val>"[^"]+"|[^",]+)(?:,|$)')
         for line in m3u8_doc.splitlines():
             if line.startswith('#EXT-X-STREAM-INF:'):
-                last_info = {}
-                for m in kv_rex.finditer(line):
-                    v = m.group('val')
-                    if v.startswith('"'):
-                        v = v[1:-1]
-                    last_info[m.group('key')] = v
+                last_info = parse_m3u8_attributes(line)
             elif line.startswith('#EXT-X-MEDIA:'):
-                last_media = {}
-                for m in kv_rex.finditer(line):
-                    v = m.group('val')
-                    if v.startswith('"'):
-                        v = v[1:-1]
-                    last_media[m.group('key')] = v
+                last_media = parse_m3u8_attributes(line)
             elif line.startswith('#') or not line.strip():
                 continue
             else:
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 82f67f6cd..562031fe1 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2852,3 +2852,12 @@ def decode_packed_codes(code):
     return re.sub(
         r'\b(\w+)\b', lambda mobj: symbol_table[mobj.group(0)],
         obfucasted_code)
+
+
+def parse_m3u8_attributes(attrib):
+    info = {}
+    for (key, val) in re.findall(r'(?P<key>[A-Z0-9-]+)=(?P<val>"[^"]+"|[^",]+)(?:,|$)', attrib):
+        if val.startswith('"'):
+            val = val[1:-1]
+        info[key] = val
+    return info

From 6cd64b6806e92b7246aebd89448189180d88db82 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 19 Jun 2016 05:45:48 +0100
Subject: [PATCH 5257/6219] [foxsports] extract http formats

---
 youtube_dl/extractor/foxsports.py | 20 +++++++++++++++-----
 1 file changed, 15 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/foxsports.py b/youtube_dl/extractor/foxsports.py
index df7665176..a3bb98377 100644
--- a/youtube_dl/extractor/foxsports.py
+++ b/youtube_dl/extractor/foxsports.py
@@ -1,7 +1,10 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..utils import smuggle_url
+from ..utils import (
+    smuggle_url,
+    update_url_query,
+)
 
 
 class FoxSportsIE(InfoExtractor):
@@ -9,11 +12,15 @@ class FoxSportsIE(InfoExtractor):
 
     _TEST = {
         'url': 'http://www.foxsports.com/video?vid=432609859715',
+        'md5': 'b49050e955bebe32c301972e4012ac17',
         'info_dict': {
-            'id': 'gA0bHB3Ladz3',
-            'ext': 'flv',
+            'id': 'i0qKWsk3qJaM',
+            'ext': 'mp4',
             'title': 'Courtney Lee on going up 2-0 in series vs. Blazers',
             'description': 'Courtney Lee talks about Memphis being focused.',
+            'upload_date': '20150423',
+            'timestamp': 1429761109,
+            'uploader': 'NEWA-FNG-FOXSPORTS',
         },
         'add_ie': ['ThePlatform'],
     }
@@ -28,5 +35,8 @@ class FoxSportsIE(InfoExtractor):
                 r"data-player-config='([^']+)'", webpage, 'data player config'),
             video_id)
 
-        return self.url_result(smuggle_url(
-            config['releaseURL'] + '&manifest=f4m', {'force_smil_url': True}))
+        return self.url_result(smuggle_url(update_url_query(
+            config['releaseURL'], {
+                'mbr': 'true',
+                'switch': 'http',
+            }), {'force_smil_url': True}))

From 6c83e583b334226965bdf45583c09dbe8bfe9dab Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 19 Jun 2016 13:32:08 +0800
Subject: [PATCH 5258/6219] [radiojavan] PEP8

E275 is added in pycodestyle 2.6

See https://github.com/PyCQA/pycodestyle/pull/491
---
 youtube_dl/extractor/radiojavan.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/radiojavan.py b/youtube_dl/extractor/radiojavan.py
index 884c28420..ec4fa6e60 100644
--- a/youtube_dl/extractor/radiojavan.py
+++ b/youtube_dl/extractor/radiojavan.py
@@ -3,7 +3,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import(
+from ..utils import (
     unified_strdate,
     str_to_int,
 )

From 5839d556e4d491ce940324965eaeecfb843306cc Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 19 Jun 2016 23:37:05 +0100
Subject: [PATCH 5259/6219] [theplatform] reduce requests for theplatform feed
 info extraction

---
 youtube_dl/extractor/theplatform.py | 54 +++++++++++++++++++----------
 1 file changed, 36 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index 5793ec6ef..07d222ae3 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -277,9 +277,9 @@ class ThePlatformIE(ThePlatformBaseIE):
 
 
 class ThePlatformFeedIE(ThePlatformBaseIE):
-    _URL_TEMPLATE = '%s//feed.theplatform.com/f/%s/%s?form=json&byGuid=%s'
-    _VALID_URL = r'https?://feed\.theplatform\.com/f/(?P<provider_id>[^/]+)/(?P<feed_id>[^?/]+)\?(?:[^&]+&)*byGuid=(?P<id>[a-zA-Z0-9_]+)'
-    _TEST = {
+    _URL_TEMPLATE = '%s//feed.theplatform.com/f/%s/%s?form=json&%s'
+    _VALID_URL = r'https?://feed\.theplatform\.com/f/(?P<provider_id>[^/]+)/(?P<feed_id>[^?/]+)\?(?:[^&]+&)*(?P<filter>by(?:Gui|I)d=(?P<id>[\w-]+))'
+    _TESTS = [{
         # From http://player.theplatform.com/p/7wvmTC/MSNBCEmbeddedOffSite?guid=n_hardball_5biden_140207
         'url': 'http://feed.theplatform.com/f/7wvmTC/msnbc_video-p-test?form=json&pretty=true&range=-40&byGuid=n_hardball_5biden_140207',
         'md5': '6e32495b5073ab414471b615c5ded394',
@@ -295,32 +295,38 @@ class ThePlatformFeedIE(ThePlatformBaseIE):
             'categories': ['MSNBC/Issues/Democrats', 'MSNBC/Issues/Elections/Election 2016'],
             'uploader': 'NBCU-NEWS',
         },
-    }
+    }]
 
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-
-        video_id = mobj.group('id')
-        provider_id = mobj.group('provider_id')
-        feed_id = mobj.group('feed_id')
-
-        real_url = self._URL_TEMPLATE % (self.http_scheme(), provider_id, feed_id, video_id)
-        feed = self._download_json(real_url, video_id)
-        entry = feed['entries'][0]
+    def _extract_feed_info(self, provider_id, feed_id, filter_query, video_id, custom_fields=None, asset_types_query={}):
+        real_url = self._URL_TEMPLATE % (self.http_scheme(), provider_id, feed_id, filter_query)
+        entry = self._download_json(real_url, video_id)['entries'][0]
 
         formats = []
         subtitles = {}
         first_video_id = None
         duration = None
+        asset_types = []
         for item in entry['media$content']:
-            smil_url = item['plfile$url'] + '&mbr=true'
+            smil_url = item['plfile$url']
             cur_video_id = ThePlatformIE._match_id(smil_url)
             if first_video_id is None:
                 first_video_id = cur_video_id
                 duration = float_or_none(item.get('plfile$duration'))
-            cur_formats, cur_subtitles = self._extract_theplatform_smil(smil_url, video_id, 'Downloading SMIL data for %s' % cur_video_id)
-            formats.extend(cur_formats)
-            subtitles = self._merge_subtitles(subtitles, cur_subtitles)
+            for asset_type in item['plfile$assetTypes']:
+                if asset_type in asset_types:
+                    continue
+                asset_types.append(asset_type)
+                query = {
+                    'mbr': 'true',
+                    'formats': item['plfile$format'],
+                    'assetTypes': asset_type,
+                }
+                if asset_type in asset_types_query:
+                    query.update(asset_types_query[asset_type])
+                cur_formats, cur_subtitles = self._extract_theplatform_smil(update_url_query(
+                    smil_url, query), video_id, 'Downloading SMIL data for %s' % asset_type)
+                formats.extend(cur_formats)
+                subtitles = self._merge_subtitles(subtitles, cur_subtitles)
 
         self._sort_formats(formats)
 
@@ -344,5 +350,17 @@ class ThePlatformFeedIE(ThePlatformBaseIE):
             'timestamp': timestamp,
             'categories': categories,
         })
+        if custom_fields:
+            ret.update(custom_fields(entry))
 
         return ret
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+
+        video_id = mobj.group('id')
+        provider_id = mobj.group('provider_id')
+        feed_id = mobj.group('feed_id')
+        filter_query = mobj.group('filter')
+
+        return self._extract_feed_info(provider_id, feed_id, filter_query, video_id)

From 43518503a66d670330a8406829fc30b431420c9c Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 19 Jun 2016 23:40:00 +0100
Subject: [PATCH 5260/6219] [cbs,cbsnews,cbssports] reduce requests while
 extracting all formats

---
 youtube_dl/extractor/cbs.py       | 84 +++++++------------------------
 youtube_dl/extractor/cbsnews.py   | 31 +++---------
 youtube_dl/extractor/cbssports.py | 40 +++++++--------
 3 files changed, 44 insertions(+), 111 deletions(-)

diff --git a/youtube_dl/extractor/cbs.py b/youtube_dl/extractor/cbs.py
index ac2c7dced..030eeaa65 100644
--- a/youtube_dl/extractor/cbs.py
+++ b/youtube_dl/extractor/cbs.py
@@ -2,16 +2,14 @@ from __future__ import unicode_literals
 
 import re
 
-from .theplatform import ThePlatformIE
+from .theplatform import ThePlatformFeedIE
 from ..utils import (
-    xpath_text,
-    xpath_element,
     int_or_none,
     find_xpath_attr,
 )
 
 
-class CBSBaseIE(ThePlatformIE):
+class CBSBaseIE(ThePlatformFeedIE):
     def _parse_smil_subtitles(self, smil, namespace=None, subtitles_lang='en'):
         closed_caption_e = find_xpath_attr(smil, self._xpath_ns('.//param', namespace), 'name', 'ClosedCaptionURL')
         return {
@@ -21,9 +19,22 @@ class CBSBaseIE(ThePlatformIE):
             }]
         } if closed_caption_e is not None and closed_caption_e.attrib.get('value') else []
 
+    def _extract_video_info(self, filter_query, video_id):
+        return self._extract_feed_info(
+            'dJ5BDC', 'VxxJg8Ymh8sE', filter_query, video_id, lambda entry: {
+                'series': entry.get('cbs$SeriesTitle'),
+                'season_number': int_or_none(entry.get('cbs$SeasonNumber')),
+                'episode': entry.get('cbs$EpisodeTitle'),
+                'episode_number': int_or_none(entry.get('cbs$EpisodeNumber')),
+            }, {
+                'StreamPack': {
+                    'manifest': 'm3u',
+                }
+            })
+
 
 class CBSIE(CBSBaseIE):
-    _VALID_URL = r'(?:cbs:(?P<content_id>\w+)|https?://(?:www\.)?(?:cbs\.com/shows/[^/]+/(?:video|artist)|colbertlateshow\.com/(?:video|podcasts))/[^/]+/(?P<display_id>[^/]+))'
+    _VALID_URL = r'(?:cbs|https?://(?:www\.)?(?:cbs\.com/shows/[^/]+/video|colbertlateshow\.com/(?:video|podcasts))/)(?P<id>[\w-]+)'
 
     _TESTS = [{
         'url': 'http://www.cbs.com/shows/garth-brooks/video/_u7W953k6la293J7EPTd9oHkSPs6Xn6_/connect-chat-feat-garth-brooks/',
@@ -38,25 +49,7 @@ class CBSIE(CBSBaseIE):
             'upload_date': '20131127',
             'uploader': 'CBSI-NEW',
         },
-        'params': {
-            # rtmp download
-            'skip_download': True,
-        },
-        '_skip': 'Blocked outside the US',
-    }, {
-        'url': 'http://www.cbs.com/shows/liveonletterman/artist/221752/st-vincent/',
-        'info_dict': {
-            'id': 'WWF_5KqY3PK1',
-            'display_id': 'st-vincent',
-            'ext': 'flv',
-            'title': 'Live on Letterman - St. Vincent',
-            'description': 'Live On Letterman: St. Vincent in concert from New York\'s Ed Sullivan Theater on Tuesday, July 16, 2014.',
-            'duration': 3221,
-        },
-        'params': {
-            # rtmp download
-            'skip_download': True,
-        },
+        'expected_warnings': ['Failed to download m3u8 information'],
         '_skip': 'Blocked outside the US',
     }, {
         'url': 'http://colbertlateshow.com/video/8GmB0oY0McANFvp2aEffk9jZZZ2YyXxy/the-colbeard/',
@@ -68,44 +61,5 @@ class CBSIE(CBSBaseIE):
     TP_RELEASE_URL_TEMPLATE = 'http://link.theplatform.com/s/dJ5BDC/%s?mbr=true'
 
     def _real_extract(self, url):
-        content_id, display_id = re.match(self._VALID_URL, url).groups()
-        if not content_id:
-            webpage = self._download_webpage(url, display_id)
-            content_id = self._search_regex(
-                [r"video\.settings\.content_id\s*=\s*'([^']+)';", r"cbsplayer\.contentId\s*=\s*'([^']+)';"],
-                webpage, 'content id')
-        items_data = self._download_xml(
-            'http://can.cbs.com/thunder/player/videoPlayerService.php',
-            content_id, query={'partner': 'cbs', 'contentId': content_id})
-        video_data = xpath_element(items_data, './/item')
-        title = xpath_text(video_data, 'videoTitle', 'title', True)
-
-        subtitles = {}
-        formats = []
-        for item in items_data.findall('.//item'):
-            pid = xpath_text(item, 'pid')
-            if not pid:
-                continue
-            tp_release_url = self.TP_RELEASE_URL_TEMPLATE % pid
-            if '.m3u8' in xpath_text(item, 'contentUrl', default=''):
-                tp_release_url += '&manifest=m3u'
-            tp_formats, tp_subtitles = self._extract_theplatform_smil(
-                tp_release_url, content_id, 'Downloading %s SMIL data' % pid)
-            formats.extend(tp_formats)
-            subtitles = self._merge_subtitles(subtitles, tp_subtitles)
-        self._sort_formats(formats)
-
-        info = self.get_metadata('dJ5BDC/media/guid/2198311517/%s' % content_id, content_id)
-        info.update({
-            'id': content_id,
-            'display_id': display_id,
-            'title': title,
-            'series': xpath_text(video_data, 'seriesTitle'),
-            'season_number': int_or_none(xpath_text(video_data, 'seasonNumber')),
-            'episode_number': int_or_none(xpath_text(video_data, 'episodeNumber')),
-            'duration': int_or_none(xpath_text(video_data, 'videoLength'), 1000),
-            'thumbnail': xpath_text(video_data, 'previewImageURL'),
-            'formats': formats,
-            'subtitles': subtitles,
-        })
-        return info
+        content_id = self._match_id(url)
+        return self._extract_video_info('byGuid=%s' % content_id, content_id)
diff --git a/youtube_dl/extractor/cbsnews.py b/youtube_dl/extractor/cbsnews.py
index 79ddc20a0..387537e76 100644
--- a/youtube_dl/extractor/cbsnews.py
+++ b/youtube_dl/extractor/cbsnews.py
@@ -30,9 +30,12 @@ class CBSNewsIE(CBSBaseIE):
         {
             'url': 'http://www.cbsnews.com/videos/fort-hood-shooting-army-downplays-mental-illness-as-cause-of-attack/',
             'info_dict': {
-                'id': 'fort-hood-shooting-army-downplays-mental-illness-as-cause-of-attack',
+                'id': 'SNJBOYzXiWBOvaLsdzwH8fmtP1SCd91Y',
                 'ext': 'mp4',
                 'title': 'Fort Hood shooting: Army downplays mental illness as cause of attack',
+                'description': 'md5:4a6983e480542d8b333a947bfc64ddc7',
+                'upload_date': '19700101',
+                'uploader': 'CBSI-NEW',
                 'thumbnail': 're:^https?://.*\.jpg$',
                 'duration': 205,
                 'subtitles': {
@@ -58,30 +61,8 @@ class CBSNewsIE(CBSBaseIE):
             webpage, 'video JSON info'), video_id)
 
         item = video_info['item'] if 'item' in video_info else video_info
-        title = item.get('articleTitle') or item.get('hed')
-        duration = item.get('duration')
-        thumbnail = item.get('mediaImage') or item.get('thumbnail')
-
-        subtitles = {}
-        formats = []
-        for format_id in ['RtmpMobileLow', 'RtmpMobileHigh', 'Hls', 'RtmpDesktop']:
-            pid = item.get('media' + format_id)
-            if not pid:
-                continue
-            release_url = 'http://link.theplatform.com/s/dJ5BDC/%s?mbr=true' % pid
-            tp_formats, tp_subtitles = self._extract_theplatform_smil(release_url, video_id, 'Downloading %s SMIL data' % pid)
-            formats.extend(tp_formats)
-            subtitles = self._merge_subtitles(subtitles, tp_subtitles)
-        self._sort_formats(formats)
-
-        return {
-            'id': video_id,
-            'title': title,
-            'thumbnail': thumbnail,
-            'duration': duration,
-            'formats': formats,
-            'subtitles': subtitles,
-        }
+        guid = item['mpxRefId']
+        return self._extract_video_info('byGuid=%s' % guid, guid)
 
 
 class CBSNewsLiveVideoIE(InfoExtractor):
diff --git a/youtube_dl/extractor/cbssports.py b/youtube_dl/extractor/cbssports.py
index 549ae32f3..78ca44b02 100644
--- a/youtube_dl/extractor/cbssports.py
+++ b/youtube_dl/extractor/cbssports.py
@@ -1,30 +1,28 @@
 from __future__ import unicode_literals
 
-import re
-
-from .common import InfoExtractor
+from .cbs import CBSBaseIE
 
 
-class CBSSportsIE(InfoExtractor):
-    _VALID_URL = r'https?://www\.cbssports\.com/video/player/(?P<section>[^/]+)/(?P<id>[^/]+)'
+class CBSSportsIE(CBSBaseIE):
+    _VALID_URL = r'https?://www\.cbssports\.com/video/player/[^/]+/(?P<id>\d+)'
 
-    _TEST = {
-        'url': 'http://www.cbssports.com/video/player/tennis/318462531970/0/us-open-flashbacks-1990s',
+    _TESTS = [{
+        'url': 'http://www.cbssports.com/video/player/videos/708337219968/0/ben-simmons-the-next-lebron?-not-so-fast',
         'info_dict': {
-            'id': '_d5_GbO8p1sT',
-            'ext': 'flv',
-            'title': 'US Open flashbacks: 1990s',
-            'description': 'Bill Macatee relives the best moments in US Open history from the 1990s.',
+            'id': '708337219968',
+            'ext': 'mp4',
+            'title': 'Ben Simmons the next LeBron? Not so fast',
+            'description': 'md5:854294f627921baba1f4b9a990d87197',
+            'timestamp': 1466293740,
+            'upload_date': '20160618',
+            'uploader': 'CBSI-NEW',
         },
-    }
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        }
+    }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        section = mobj.group('section')
-        video_id = mobj.group('id')
-        all_videos = self._download_json(
-            'http://www.cbssports.com/data/video/player/getVideos/%s?as=json' % section,
-            video_id)
-        # The json file contains the info of all the videos in the section
-        video_info = next(v for v in all_videos if v['pcid'] == video_id)
-        return self.url_result('theplatform:%s' % video_info['pid'], 'ThePlatform')
+        video_id = self._match_id(url)
+        return self._extract_video_info('byId=%s' % video_id, video_id)

From 819707920a63946ea1e4f0ae2bf842425d22c2e9 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 19 Jun 2016 23:55:19 +0100
Subject: [PATCH 5261/6219] [cbs] fix _VALID_URL

---
 youtube_dl/extractor/cbs.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/cbs.py b/youtube_dl/extractor/cbs.py
index 030eeaa65..21720f084 100644
--- a/youtube_dl/extractor/cbs.py
+++ b/youtube_dl/extractor/cbs.py
@@ -34,7 +34,7 @@ class CBSBaseIE(ThePlatformFeedIE):
 
 
 class CBSIE(CBSBaseIE):
-    _VALID_URL = r'(?:cbs|https?://(?:www\.)?(?:cbs\.com/shows/[^/]+/video|colbertlateshow\.com/(?:video|podcasts))/)(?P<id>[\w-]+)'
+    _VALID_URL = r'(?:cbs:|https?://(?:www\.)?(?:cbs\.com/shows/[^/]+/video|colbertlateshow\.com/(?:video|podcasts))/)(?P<id>[\w-]+)'
 
     _TESTS = [{
         'url': 'http://www.cbs.com/shows/garth-brooks/video/_u7W953k6la293J7EPTd9oHkSPs6Xn6_/connect-chat-feat-garth-brooks/',

From 1f749b6658439049b952fdb979acb6c4422a358a Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 20 Jun 2016 13:29:13 +0200
Subject: [PATCH 5262/6219] Revert "[jsinterp] Avoid double key lookup for
 setting new key"

This reverts commit 7c05097633138459e9bdf7e10738e021b04689a7.
---
 youtube_dl/jsinterp.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index 4a5a0dbc3..a7440c582 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -131,8 +131,9 @@ class JSInterpreter(object):
             if variable in local_vars:
                 obj = local_vars[variable]
             else:
-                obj = self._objects.setdefault(
-                    variable, self.extract_object(variable))
+                if variable not in self._objects:
+                    self._objects[variable] = self.extract_object(variable)
+                obj = self._objects[variable]
 
             if arg_str is None:
                 # Member access
@@ -203,7 +204,8 @@ class JSInterpreter(object):
             argvals = tuple([
                 int(v) if v.isdigit() else local_vars[v]
                 for v in m.group('args').split(',')])
-            self._functions.setdefault(fname, self.extract_function(fname))
+            if fname not in self._functions:
+                self._functions[fname] = self.extract_function(fname)
             return self._functions[fname](argvals)
 
         raise ExtractorError('Unsupported JS expression %r' % expr)

From 8369a4fe768b1838f640ad984fbc923037b06c3a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 20 Jun 2016 21:55:17 +0700
Subject: [PATCH 5263/6219] [downloader/hls] Simplify and carry long lines

---
 youtube_dl/downloader/hls.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index 1d5f178a0..3b7bb3508 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -50,7 +50,7 @@ class HlsFD(FragmentFD):
             # 4. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.3.5
         )
         check_results = [not re.search(feature, manifest) for feature in UNSUPPORTED_FEATURES]
-        check_results.append(not (re.search(r'#EXT-X-KEY:METHOD=AES-128', manifest) and not can_decrypt_frag))
+        check_results.append(can_decrypt_frag or '#EXT-X-KEY:METHOD=AES-128' not in manifest)
         return all(check_results)
 
     def real_download(self, filename, info_dict):
@@ -102,8 +102,9 @@ class HlsFD(FragmentFD):
                     frag_content = down.read()
                     down.close()
                     if decrypt_info['METHOD'] == 'AES-128':
-                        iv = decrypt_info.get('IV') or compat_struct_pack(">8xq", media_sequence)
-                        frag_content = AES.new(decrypt_info['KEY'], AES.MODE_CBC, iv).decrypt(frag_content)
+                        iv = decrypt_info.get('IV') or compat_struct_pack('>8xq', media_sequence)
+                        frag_content = AES.new(
+                            decrypt_info['KEY'], AES.MODE_CBC, iv).decrypt(frag_content)
                     ctx['dest_stream'].write(frag_content)
                     frags_filenames.append(frag_sanitized)
                     # We only download the first fragment during the test
@@ -117,7 +118,8 @@ class HlsFD(FragmentFD):
                         if 'IV' in decrypt_info:
                             decrypt_info['IV'] = binascii.unhexlify(decrypt_info['IV'][2:])
                         if not re.match(r'^https?://', decrypt_info['URI']):
-                            decrypt_info['URI'] = compat_urlparse.urljoin(man_url, decrypt_info['URI'])
+                            decrypt_info['URI'] = compat_urlparse.urljoin(
+                                man_url, decrypt_info['URI'])
                         decrypt_info['KEY'] = self.ydl.urlopen(decrypt_info['URI']).read()
                 elif line.startswith('#EXT-X-MEDIA-SEQUENCE'):
                     media_sequence = int(line[22:])

From 19e2d1cdeaf36805d72206a6309a6f7421f3c9ed Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 20 Jun 2016 20:50:01 +0700
Subject: [PATCH 5264/6219] release 2016.06.20

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 docs/supportedsites.md    | 2 +-
 youtube_dl/version.py     | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 1b25628bf..64ddb891e 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.06.19.1*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.06.19.1**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.06.20*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.06.20**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.06.19.1
+[debug] youtube-dl version 2016.06.20
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 7c90940c7..5be8238c0 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -44,8 +44,8 @@
  - **appletrailers:section**
  - **archive.org**: archive.org videos
  - **ARD**
- - **ARD:mediathek**
  - **ARD:mediathek**: Saarländischer Rundfunk
+ - **ARD:mediathek**
  - **arte.tv**
  - **arte.tv:+7**
  - **arte.tv:cinema**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index a3e2c3079..4a9f162c1 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.06.19.1'
+__version__ = '2016.06.20'

From feef925f49c80fc125ff24f61a144af902a648d2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 20 Jun 2016 22:40:22 +0700
Subject: [PATCH 5265/6219] [streamcloud] Capture error message (#9840)

---
 youtube_dl/extractor/streamcloud.py | 27 ++++++++++++++++-----------
 1 file changed, 16 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/streamcloud.py b/youtube_dl/extractor/streamcloud.py
index 58560ec64..6a6bb90c4 100644
--- a/youtube_dl/extractor/streamcloud.py
+++ b/youtube_dl/extractor/streamcloud.py
@@ -6,7 +6,6 @@ import re
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
-    sanitized_Request,
     urlencode_postdata,
 )
 
@@ -45,20 +44,26 @@ class StreamcloudIE(InfoExtractor):
             (?:id="[^"]+"\s+)?
             value="([^"]*)"
             ''', orig_webpage)
-        post = urlencode_postdata(fields)
 
         self._sleep(12, video_id)
-        headers = {
-            b'Content-Type': b'application/x-www-form-urlencoded',
-        }
-        req = sanitized_Request(url, post, headers)
 
         webpage = self._download_webpage(
-            req, video_id, note='Downloading video page ...')
-        title = self._html_search_regex(
-            r'<h1[^>]*>([^<]+)<', webpage, 'title')
-        video_url = self._search_regex(
-            r'file:\s*"([^"]+)"', webpage, 'video URL')
+            url, video_id, data=urlencode_postdata(fields), headers={
+                b'Content-Type': b'application/x-www-form-urlencoded',
+            })
+
+        try:
+            title = self._html_search_regex(
+                r'<h1[^>]*>([^<]+)<', webpage, 'title')
+            video_url = self._search_regex(
+                r'file:\s*"([^"]+)"', webpage, 'video URL')
+        except ExtractorError:
+            message = self._html_search_regex(
+                r'(?s)<div[^>]+class=(["\']).*?msgboxinfo.*?\1[^>]*>(?P<message>.+?)</div>',
+                webpage, 'message', default=None, group='message')
+            if message:
+                raise ExtractorError('%s said: %s' % (self.IE_NAME, message), expected=True)
+            raise
         thumbnail = self._search_regex(
             r'image:\s*"([^"]+)"', webpage, 'thumbnail URL', fatal=False)
 

From cdfc187cd5bd163e7e67ca0c02108380cc06c180 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 20 Jun 2016 22:40:33 +0700
Subject: [PATCH 5266/6219] [cbs] Remove unused import

---
 youtube_dl/extractor/cbs.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/youtube_dl/extractor/cbs.py b/youtube_dl/extractor/cbs.py
index 21720f084..a23173d6f 100644
--- a/youtube_dl/extractor/cbs.py
+++ b/youtube_dl/extractor/cbs.py
@@ -1,7 +1,5 @@
 from __future__ import unicode_literals
 
-import re
-
 from .theplatform import ThePlatformFeedIE
 from ..utils import (
     int_or_none,

From e4f90ea0a72711f6577d4cde1dd145f03ab34803 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 21 Jun 2016 17:55:53 +0800
Subject: [PATCH 5267/6219] [svt] Fix extraction for SVTPlay (closes #9809)

---
 youtube_dl/extractor/svt.py | 63 ++++++++++++++++++++++++-------------
 1 file changed, 42 insertions(+), 21 deletions(-)

diff --git a/youtube_dl/extractor/svt.py b/youtube_dl/extractor/svt.py
index 2ab30e45f..6526a6345 100644
--- a/youtube_dl/extractor/svt.py
+++ b/youtube_dl/extractor/svt.py
@@ -6,17 +6,14 @@ import re
 from .common import InfoExtractor
 from ..utils import (
     determine_ext,
+    dict_get,
 )
 
 
 class SVTBaseIE(InfoExtractor):
-    def _extract_video(self, url, video_id):
-        info = self._download_json(url, video_id)
+    def _extract_video(self, info, video_id):
+        video_info = self._get_video_info(info)
 
-        title = info['context']['title']
-        thumbnail = info['context'].get('thumbnailImage')
-
-        video_info = info['video']
         formats = []
         for vr in video_info['videoReferences']:
             player_type = vr.get('playerType')
@@ -43,22 +40,25 @@ class SVTBaseIE(InfoExtractor):
         self._sort_formats(formats)
 
         subtitles = {}
-        subtitle_references = video_info.get('subtitleReferences')
+        subtitle_references = dict_get(video_info, ('subtitles', 'subtitleReferences'))
         if isinstance(subtitle_references, list):
             for sr in subtitle_references:
                 subtitle_url = sr.get('url')
+                subtitle_lang = sr.get('language', 'sv')
                 if subtitle_url:
-                    subtitles.setdefault('sv', []).append({'url': subtitle_url})
+                    if determine_ext(subtitle_url) == 'm3u8':
+                        # TODO(yan12125): handle WebVTT in m3u8 manifests
+                        continue
+
+                    subtitles.setdefault(subtitle_lang, []).append({'url': subtitle_url})
 
         duration = video_info.get('materialLength')
         age_limit = 18 if video_info.get('inappropriateForChildren') else 0
 
         return {
             'id': video_id,
-            'title': title,
             'formats': formats,
             'subtitles': subtitles,
-            'thumbnail': thumbnail,
             'duration': duration,
             'age_limit': age_limit,
         }
@@ -68,11 +68,11 @@ class SVTIE(SVTBaseIE):
     _VALID_URL = r'https?://(?:www\.)?svt\.se/wd\?(?:.*?&)?widgetId=(?P<widget_id>\d+)&.*?\barticleId=(?P<id>\d+)'
     _TEST = {
         'url': 'http://www.svt.se/wd?widgetId=23991&sectionId=541&articleId=2900353&type=embed&contextSectionId=123&autostart=false',
-        'md5': '9648197555fc1b49e3dc22db4af51d46',
+        'md5': '33e9a5d8f646523ce0868ecfb0eed77d',
         'info_dict': {
             'id': '2900353',
-            'ext': 'flv',
-            'title': 'Här trycker Jagr till Giroux (under SVT-intervjun)',
+            'ext': 'mp4',
+            'title': 'Stjärnorna skojar till det - under SVT-intervjun',
             'duration': 27,
             'age_limit': 0,
         },
@@ -85,18 +85,26 @@ class SVTIE(SVTBaseIE):
         if mobj:
             return mobj.group('url')
 
+    def _get_video_info(self, info):
+        return info['video']
+
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         widget_id = mobj.group('widget_id')
         article_id = mobj.group('id')
-        return self._extract_video(
+
+        info = self._download_json(
             'http://www.svt.se/wd?widgetId=%s&articleId=%s&format=json&type=embed&output=json' % (widget_id, article_id),
             article_id)
 
+        info_dict = self._extract_video(info, article_id)
+        info_dict['title'] = info['context']['title']
+        return info_dict
+
 
 class SVTPlayIE(SVTBaseIE):
     IE_DESC = 'SVT Play and Öppet arkiv'
-    _VALID_URL = r'https?://(?:www\.)?(?P<host>svtplay|oppetarkiv)\.se/video/(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?(?:svtplay|oppetarkiv)\.se/video/(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://www.svtplay.se/video/5996901/flygplan-till-haile-selassie/flygplan-till-haile-selassie-2',
         'md5': '2b6704fe4a28801e1a098bbf3c5ac611',
@@ -115,10 +123,23 @@ class SVTPlayIE(SVTBaseIE):
         },
     }
 
+    def _get_video_info(self, info):
+        return info['context']['dispatcher']['stores']['VideoTitlePageStore']['data']['video']
+
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        host = mobj.group('host')
-        return self._extract_video(
-            'http://www.%s.se/video/%s?output=json' % (host, video_id),
-            video_id)
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        data = self._parse_json(self._search_regex(
+            r'root\["__svtplay"\]\s*=\s*([^;]+);', webpage, 'embedded data'), video_id)
+
+        thumbnail = self._og_search_thumbnail(webpage)
+
+        info_dict = self._extract_video(data, video_id)
+        info_dict.update({
+            'title': data['context']['dispatcher']['stores']['MetaStore']['title'],
+            'thumbnail': thumbnail,
+        })
+
+        return info_dict

From 1ac5705f62aa3f6fdb6f2a97fbd24594010b7598 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 21 Jun 2016 13:36:56 +0100
Subject: [PATCH 5268/6219] [gamespot] extract all formats

---
 youtube_dl/extractor/gamespot.py | 91 ++++++++++++++++++++++++--------
 1 file changed, 68 insertions(+), 23 deletions(-)

diff --git a/youtube_dl/extractor/gamespot.py b/youtube_dl/extractor/gamespot.py
index 4ffdd7515..621257c9f 100644
--- a/youtube_dl/extractor/gamespot.py
+++ b/youtube_dl/extractor/gamespot.py
@@ -1,19 +1,19 @@
 from __future__ import unicode_literals
 
 import re
-import json
 
-from .common import InfoExtractor
+from .once import OnceIE
 from ..compat import (
     compat_urllib_parse_unquote,
-    compat_urlparse,
 )
 from ..utils import (
     unescapeHTML,
+    url_basename,
+    dict_get,
 )
 
 
-class GameSpotIE(InfoExtractor):
+class GameSpotIE(OnceIE):
     _VALID_URL = r'https?://(?:www\.)?gamespot\.com/.*-(?P<id>\d+)/?'
     _TESTS = [{
         'url': 'http://www.gamespot.com/videos/arma-3-community-guide-sitrep-i/2300-6410818/',
@@ -39,29 +39,73 @@ class GameSpotIE(InfoExtractor):
         webpage = self._download_webpage(url, page_id)
         data_video_json = self._search_regex(
             r'data-video=["\'](.*?)["\']', webpage, 'data video')
-        data_video = json.loads(unescapeHTML(data_video_json))
+        data_video = self._parse_json(unescapeHTML(data_video_json), page_id)
         streams = data_video['videoStreams']
 
+        manifest_url = None
         formats = []
         f4m_url = streams.get('f4m_stream')
-        if f4m_url is not None:
-            # Transform the manifest url to a link to the mp4 files
-            # they are used in mobile devices.
-            f4m_path = compat_urlparse.urlparse(f4m_url).path
-            QUALITIES_RE = r'((,\d+)+,?)'
-            qualities = self._search_regex(QUALITIES_RE, f4m_path, 'qualities').strip(',').split(',')
-            http_path = f4m_path[1:].split('/', 1)[1]
-            http_template = re.sub(QUALITIES_RE, r'%s', http_path)
-            http_template = http_template.replace('.csmil/manifest.f4m', '')
-            http_template = compat_urlparse.urljoin(
-                'http://video.gamespotcdn.com/', http_template)
-            for q in qualities:
-                formats.append({
-                    'url': http_template % q,
-                    'ext': 'mp4',
-                    'format_id': q,
-                })
-        else:
+        if f4m_url:
+            manifest_url = f4m_url
+            formats.extend(self._extract_f4m_formats(
+                f4m_url + '?hdcore=3.7.0', page_id, f4m_id='hds', fatal=False))
+        m3u8_url = streams.get('m3u8_stream')
+        if m3u8_url:
+            manifest_url = m3u8_url
+            m3u8_formats = self._extract_m3u8_formats(
+                m3u8_url, page_id, 'mp4', 'm3u8_native',
+                m3u8_id='hls', fatal=False)
+            formats.extend(m3u8_formats)
+        progressive_url = dict_get(
+            streams, ('progressive_hd', 'progressive_high', 'progressive_low'))
+        if progressive_url and manifest_url:
+            qualities_basename = self._search_regex(
+                '/([^/]+)\.csmil/',
+                manifest_url, 'qualities basename', default=None)
+            if qualities_basename:
+                QUALITIES_RE = r'((,\d+)+,?)'
+                qualities = self._search_regex(
+                    QUALITIES_RE, qualities_basename,
+                    'qualities', default=None)
+                if qualities:
+                    qualities = list(map(lambda q: int(q), qualities.strip(',').split(',')))
+                    qualities.sort()
+                    http_template = re.sub(QUALITIES_RE, r'%d', qualities_basename)
+                    http_url_basename = url_basename(progressive_url)
+                    if m3u8_formats:
+                        self._sort_formats(m3u8_formats)
+                        m3u8_formats = list(filter(
+                            lambda f: f.get('vcodec') != 'none' and f.get('resolution') != 'multiple',
+                            m3u8_formats))
+                    if len(qualities) == len(m3u8_formats):
+                        for q, m3u8_format in zip(qualities, m3u8_formats):
+                            f = m3u8_format.copy()
+                            f.update({
+                                'url': progressive_url.replace(
+                                    http_url_basename, http_template % q),
+                                'format_id': f['format_id'].replace('hls', 'http'),
+                                'protocol': 'http',
+                            })
+                            formats.append(f)
+                    else:
+                        for q in qualities:
+                            formats.append({
+                                'url': progressive_url.replace(
+                                    http_url_basename, http_template % q),
+                                'ext': 'mp4',
+                                'format_id': 'http-%d' % q,
+                                'tbr': q,
+                            })
+
+        onceux_json = self._search_regex(
+            r'data-onceux-options=["\'](.*?)["\']', webpage, 'data video', default=None)
+        if onceux_json:
+            onceux_url = self._parse_json(unescapeHTML(onceux_json), page_id).get('metadataUri')
+            if onceux_url:
+                formats.extend(self._extract_once_formats(re.sub(
+                    r'https?://[^/]+', 'http://once.unicornmedia.com', onceux_url).replace('ads/vmap/', '')))
+
+        if not formats:
             for quality in ['sd', 'hd']:
                 # It's actually a link to a flv file
                 flv_url = streams.get('f4m_{0}'.format(quality))
@@ -71,6 +115,7 @@ class GameSpotIE(InfoExtractor):
                         'ext': 'flv',
                         'format_id': quality,
                     })
+        self._sort_formats(formats)
 
         return {
             'id': data_video['guid'],

From 7cfc1e2a104977c41f6008885b36b96bcb2b146e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 21 Jun 2016 22:31:41 +0700
Subject: [PATCH 5269/6219] [gametrailers] Remove extractor

gametrailers closed (see http://www.polygon.com/2016/2/8/10944452/gametrailers-shuts-down-after-13-year-run)
---
 youtube_dl/extractor/extractors.py   |  1 -
 youtube_dl/extractor/gametrailers.py | 62 ----------------------------
 2 files changed, 63 deletions(-)
 delete mode 100644 youtube_dl/extractor/gametrailers.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index b1b04f2fc..4e2a2f2e9 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -285,7 +285,6 @@ from .gameone import (
 from .gamersyde import GamersydeIE
 from .gamespot import GameSpotIE
 from .gamestar import GameStarIE
-from .gametrailers import GametrailersIE
 from .gazeta import GazetaIE
 from .gdcvault import GDCVaultIE
 from .generic import GenericIE
diff --git a/youtube_dl/extractor/gametrailers.py b/youtube_dl/extractor/gametrailers.py
deleted file mode 100644
index 1e7948ab8..000000000
--- a/youtube_dl/extractor/gametrailers.py
+++ /dev/null
@@ -1,62 +0,0 @@
-from __future__ import unicode_literals
-
-from .common import InfoExtractor
-from ..utils import (
-    int_or_none,
-    parse_age_limit,
-    url_basename,
-)
-
-
-class GametrailersIE(InfoExtractor):
-    _VALID_URL = r'https?://www\.gametrailers\.com/videos/view/[^/]+/(?P<id>.+)'
-
-    _TEST = {
-        'url': 'http://www.gametrailers.com/videos/view/gametrailers-com/116437-Just-Cause-3-Review',
-        'md5': 'f28c4efa0bdfaf9b760f6507955b6a6a',
-        'info_dict': {
-            'id': '2983958',
-            'ext': 'mp4',
-            'display_id': '116437-Just-Cause-3-Review',
-            'title': 'Just Cause 3 - Review',
-            'description': 'It\'s a lot of fun to shoot at things and then watch them explode in Just Cause 3, but should there be more to the experience than that?',
-        },
-    }
-
-    def _real_extract(self, url):
-        display_id = self._match_id(url)
-        webpage = self._download_webpage(url, display_id)
-        title = self._html_search_regex(
-            r'<title>(.+?)\|', webpage, 'title').strip()
-        embed_url = self._proto_relative_url(
-            self._search_regex(
-                r'src=\'(//embed.gametrailers.com/embed/[^\']+)\'', webpage,
-                'embed url'),
-            scheme='http:')
-        video_id = url_basename(embed_url)
-        embed_page = self._download_webpage(embed_url, video_id)
-        embed_vars_json = self._search_regex(
-            r'(?s)var embedVars = (\{.*?\})\s*</script>', embed_page,
-            'embed vars')
-        info = self._parse_json(embed_vars_json, video_id)
-
-        formats = []
-        for media in info['media']:
-            if media['mediaPurpose'] == 'play':
-                formats.append({
-                    'url': media['uri'],
-                    'height': media['height'],
-                    'width:': media['width'],
-                })
-        self._sort_formats(formats)
-
-        return {
-            'id': video_id,
-            'display_id': display_id,
-            'title': title,
-            'formats': formats,
-            'thumbnail': info.get('thumbUri'),
-            'description': self._og_search_description(webpage),
-            'duration': int_or_none(info.get('videoLengthInSeconds')),
-            'age_limit': parse_age_limit(info.get('audienceRating')),
-        }

From ca74c90bf55cfb16f5eadc3a63e9389202ee80f5 Mon Sep 17 00:00:00 2001
From: Shai Coleman <shai.coleman@storyful.com>
Date: Wed, 22 Jun 2016 12:52:15 +0100
Subject: [PATCH 5270/6219] Fix issue downloading facebook videos

youtube-dl expects the format items to be returned as a list,
but when there's only one item Facebook returns a dict instead,
this wraps the dict in a list if necessary
---
 youtube_dl/extractor/facebook.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index f5bbd39d2..9b87b37ae 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -239,6 +239,8 @@ class FacebookIE(InfoExtractor):
 
         formats = []
         for format_id, f in video_data.items():
+            if f and isinstance(f, dict):
+                f = [f]
             if not f or not isinstance(f, list):
                 continue
             for quality in ('sd', 'hd'):

From 23bdae0955ae5e0adaf6212bb7aa6cec77ae4d1d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 22 Jun 2016 23:36:07 +0700
Subject: [PATCH 5271/6219] [svt] Various improvements

+ [svt:play] Add fallback path looking for video id and fix extraction for oppetarkiv
* [svt:base] Detect geo restriction
* [svt:base] Extract series related metadata
---
 youtube_dl/extractor/svt.py | 82 +++++++++++++++++++++++++++----------
 1 file changed, 60 insertions(+), 22 deletions(-)

diff --git a/youtube_dl/extractor/svt.py b/youtube_dl/extractor/svt.py
index 6526a6345..67f56fab8 100644
--- a/youtube_dl/extractor/svt.py
+++ b/youtube_dl/extractor/svt.py
@@ -7,13 +7,13 @@ from .common import InfoExtractor
 from ..utils import (
     determine_ext,
     dict_get,
+    int_or_none,
+    try_get,
 )
 
 
 class SVTBaseIE(InfoExtractor):
-    def _extract_video(self, info, video_id):
-        video_info = self._get_video_info(info)
-
+    def _extract_video(self, video_info, video_id):
         formats = []
         for vr in video_info['videoReferences']:
             player_type = vr.get('playerType')
@@ -37,6 +37,8 @@ class SVTBaseIE(InfoExtractor):
                     'format_id': player_type,
                     'url': vurl,
                 })
+        if not formats and video_info.get('rights', {}).get('geoBlockedSweden'):
+            self.raise_geo_restricted('This video is only available in Sweden')
         self._sort_formats(formats)
 
         subtitles = {}
@@ -52,15 +54,32 @@ class SVTBaseIE(InfoExtractor):
 
                     subtitles.setdefault(subtitle_lang, []).append({'url': subtitle_url})
 
-        duration = video_info.get('materialLength')
-        age_limit = 18 if video_info.get('inappropriateForChildren') else 0
+        title = video_info.get('title')
+
+        series = video_info.get('programTitle')
+        season_number = int_or_none(video_info.get('season'))
+        episode = video_info.get('episodeTitle')
+        episode_number = int_or_none(video_info.get('episodeNumber'))
+
+        duration = int_or_none(dict_get(video_info, ('materialLength', 'contentDuration')))
+        age_limit = None
+        adult = dict_get(
+            video_info, ('inappropriateForChildren', 'blockedForChildren'),
+            skip_false_values=False)
+        if adult is not None:
+            age_limit = 18 if adult else 0
 
         return {
             'id': video_id,
+            'title': title,
             'formats': formats,
             'subtitles': subtitles,
             'duration': duration,
             'age_limit': age_limit,
+            'series': series,
+            'season_number': season_number,
+            'episode': episode,
+            'episode_number': episode_number,
         }
 
 
@@ -85,9 +104,6 @@ class SVTIE(SVTBaseIE):
         if mobj:
             return mobj.group('url')
 
-    def _get_video_info(self, info):
-        return info['video']
-
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         widget_id = mobj.group('widget_id')
@@ -97,7 +113,7 @@ class SVTIE(SVTBaseIE):
             'http://www.svt.se/wd?widgetId=%s&articleId=%s&format=json&type=embed&output=json' % (widget_id, article_id),
             article_id)
 
-        info_dict = self._extract_video(info, article_id)
+        info_dict = self._extract_video(info['video'], article_id)
         info_dict['title'] = info['context']['title']
         return info_dict
 
@@ -105,7 +121,7 @@ class SVTIE(SVTBaseIE):
 class SVTPlayIE(SVTBaseIE):
     IE_DESC = 'SVT Play and Öppet arkiv'
     _VALID_URL = r'https?://(?:www\.)?(?:svtplay|oppetarkiv)\.se/video/(?P<id>[0-9]+)'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.svtplay.se/video/5996901/flygplan-till-haile-selassie/flygplan-till-haile-selassie-2',
         'md5': '2b6704fe4a28801e1a098bbf3c5ac611',
         'info_dict': {
@@ -121,25 +137,47 @@ class SVTPlayIE(SVTBaseIE):
                 }]
             },
         },
-    }
-
-    def _get_video_info(self, info):
-        return info['context']['dispatcher']['stores']['VideoTitlePageStore']['data']['video']
+    }, {
+        # geo restricted to Sweden
+        'url': 'http://www.oppetarkiv.se/video/5219710/trollflojten',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id)
 
-        data = self._parse_json(self._search_regex(
-            r'root\["__svtplay"\]\s*=\s*([^;]+);', webpage, 'embedded data'), video_id)
+        data = self._parse_json(
+            self._search_regex(
+                r'root\["__svtplay"\]\s*=\s*([^;]+);',
+                webpage, 'embedded data', default='{}'),
+            video_id, fatal=False)
 
         thumbnail = self._og_search_thumbnail(webpage)
 
-        info_dict = self._extract_video(data, video_id)
-        info_dict.update({
-            'title': data['context']['dispatcher']['stores']['MetaStore']['title'],
-            'thumbnail': thumbnail,
-        })
+        if data:
+            video_info = try_get(
+                data, lambda x: x['context']['dispatcher']['stores']['VideoTitlePageStore']['data']['video'],
+                dict)
+            if video_info:
+                info_dict = self._extract_video(video_info, video_id)
+                info_dict.update({
+                    'title': data['context']['dispatcher']['stores']['MetaStore']['title'],
+                    'thumbnail': thumbnail,
+                })
+                return info_dict
 
-        return info_dict
+        video_id = self._search_regex(
+            r'<video[^>]+data-video-id=["\']([\da-zA-Z-]+)',
+            webpage, 'video id', default=None)
+
+        if video_id:
+            data = self._download_json(
+                'http://www.svt.se/videoplayer-api/video/%s' % video_id, video_id)
+            info_dict = self._extract_video(data, video_id)
+            if not info_dict.get('title'):
+                info_dict['title'] = re.sub(
+                    r'\s*\|\s*.+?$', '',
+                    info_dict.get('episode') or self._og_search_title(webpage))
+            return info_dict

From cf40fdf5c1da33180b5a1b333784c529bc504b6f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 22 Jun 2016 23:43:24 +0700
Subject: [PATCH 5272/6219] release 2016.06.22

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 docs/supportedsites.md    | 1 -
 youtube_dl/version.py     | 2 +-
 3 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 64ddb891e..e17625f21 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.06.20*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.06.20**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.06.22*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.06.22**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.06.20
+[debug] youtube-dl version 2016.06.22
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 5be8238c0..96cc407db 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -248,7 +248,6 @@
  - **Gamersyde**
  - **GameSpot**
  - **GameStar**
- - **Gametrailers**
  - **Gazeta**
  - **GDCVault**
  - **generic**: Generic downloader that works on some sites
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 4a9f162c1..d2152b2f1 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.06.20'
+__version__ = '2016.06.22'

From 6ae938b295eaca06944f66faa7c6d668c6c5866c Mon Sep 17 00:00:00 2001
From: TRox1972 <TRox1972@users.noreply.github.com>
Date: Tue, 21 Jun 2016 11:36:54 +0200
Subject: [PATCH 5273/6219] [Vine] Extract view count

---
 youtube_dl/extractor/vine.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/vine.py b/youtube_dl/extractor/vine.py
index a6a6cc479..5b801849c 100644
--- a/youtube_dl/extractor/vine.py
+++ b/youtube_dl/extractor/vine.py
@@ -24,6 +24,7 @@ class VineIE(InfoExtractor):
             'upload_date': '20130519',
             'uploader': 'Jack Dorsey',
             'uploader_id': '76',
+            'view_count': int,
             'like_count': int,
             'comment_count': int,
             'repost_count': int,
@@ -39,6 +40,7 @@ class VineIE(InfoExtractor):
             'upload_date': '20140815',
             'uploader': 'Mars Ruiz',
             'uploader_id': '1102363502380728320',
+            'view_count': int,
             'like_count': int,
             'comment_count': int,
             'repost_count': int,
@@ -54,6 +56,7 @@ class VineIE(InfoExtractor):
             'upload_date': '20130430',
             'uploader': 'Z3k3',
             'uploader_id': '936470460173008896',
+            'view_count': int,
             'like_count': int,
             'comment_count': int,
             'repost_count': int,
@@ -71,6 +74,7 @@ class VineIE(InfoExtractor):
             'upload_date': '20150705',
             'uploader': 'Pimry_zaa',
             'uploader_id': '1135760698325307392',
+            'view_count': int,
             'like_count': int,
             'comment_count': int,
             'repost_count': int,
@@ -109,6 +113,7 @@ class VineIE(InfoExtractor):
             'upload_date': unified_strdate(data.get('created')),
             'uploader': username,
             'uploader_id': data.get('userIdStr'),
+            'view_count': int_or_none(data.get('loops', {}).get('count')),
             'like_count': int_or_none(data.get('likes', {}).get('count')),
             'comment_count': int_or_none(data.get('comments', {}).get('count')),
             'repost_count': int_or_none(data.get('reposts', {}).get('count')),

From 169d836feb9d796205a02713db33eafcbb49f1e6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 22 Jun 2016 19:13:46 +0200
Subject: [PATCH 5274/6219] lazy-extractors: Fix after commit
 6e6b9f600f2f447604f6108fb6486b73cc25def1

The problem was in the following code:

    class ArteTVPlus7IE(ArteTVBaseIE):

        ...

        @classmethod
        def suitable(cls, url):
            return False if ArteTVPlaylistIE.suitable(url) else super(ArteTVPlus7IE, cls).suitable(url)

And its sublcasses like ArteTVCinemaIE.

Since in the lazy_extractors.py file ArteTVCinemaIE was not a subclass of ArteTVPlus7IE, super(ArteTVPlus7IE, cls) failed.

To fix it we have to make it a subclass. Since the order of _ALL_CLASSES is arbitrary we must sort them so that the base classes are defined first. We also must add base classes like YoutubeBaseInfoExtractor.
---
 devscripts/make_lazy_extractors.py | 47 ++++++++++++++++++++++++++----
 1 file changed, 41 insertions(+), 6 deletions(-)

diff --git a/devscripts/make_lazy_extractors.py b/devscripts/make_lazy_extractors.py
index b5a8b9190..9a79c2bc5 100644
--- a/devscripts/make_lazy_extractors.py
+++ b/devscripts/make_lazy_extractors.py
@@ -14,15 +14,17 @@ if os.path.exists(lazy_extractors_filename):
     os.remove(lazy_extractors_filename)
 
 from youtube_dl.extractor import _ALL_CLASSES
-from youtube_dl.extractor.common import InfoExtractor
+from youtube_dl.extractor.common import InfoExtractor, SearchInfoExtractor
 
 with open('devscripts/lazy_load_template.py', 'rt') as f:
     module_template = f.read()
 
-module_contents = [module_template + '\n' + getsource(InfoExtractor.suitable)]
+module_contents = [
+    module_template + '\n' + getsource(InfoExtractor.suitable) + '\n',
+    'class LazyLoadSearchExtractor(LazyLoadExtractor):\n    pass\n']
 
 ie_template = '''
-class {name}(LazyLoadExtractor):
+class {name}({bases}):
     _VALID_URL = {valid_url!r}
     _module = '{module}'
 '''
@@ -34,10 +36,20 @@ make_valid_template = '''
 '''
 
 
+def get_base_name(base):
+    if base is InfoExtractor:
+        return 'LazyLoadExtractor'
+    elif base is SearchInfoExtractor:
+        return 'LazyLoadSearchExtractor'
+    else:
+        return base.__name__
+
+
 def build_lazy_ie(ie, name):
     valid_url = getattr(ie, '_VALID_URL', None)
     s = ie_template.format(
         name=name,
+        bases=', '.join(map(get_base_name, ie.__bases__)),
         valid_url=valid_url,
         module=ie.__module__)
     if ie.suitable.__func__ is not InfoExtractor.suitable.__func__:
@@ -47,12 +59,35 @@ def build_lazy_ie(ie, name):
         s += make_valid_template.format(valid_url=ie._make_valid_url())
     return s
 
+# find the correct sorting and add the required base classes so that sublcasses
+# can be correctly created
+classes = _ALL_CLASSES[:-1]
+ordered_cls = []
+while classes:
+    for c in classes[:]:
+        bases = set(c.__bases__) - set((object, InfoExtractor, SearchInfoExtractor))
+        stop = False
+        for b in bases:
+            if b not in classes and b not in ordered_cls:
+                if b.__name__ == 'GenericIE':
+                    exit()
+                classes.insert(0, b)
+                stop = True
+        if stop:
+            break
+        if all(b in ordered_cls for b in bases):
+            ordered_cls.append(c)
+            classes.remove(c)
+            break
+ordered_cls.append(_ALL_CLASSES[-1])
+
 names = []
-for ie in list(sorted(_ALL_CLASSES[:-1], key=lambda cls: cls.ie_key())) + _ALL_CLASSES[-1:]:
-    name = ie.ie_key() + 'IE'
+for ie in ordered_cls:
+    name = ie.__name__
     src = build_lazy_ie(ie, name)
     module_contents.append(src)
-    names.append(name)
+    if ie in _ALL_CLASSES:
+        names.append(name)
 
 module_contents.append(
     '_ALL_CLASSES = [{0}]'.format(', '.join(names)))

From c143ddce5d1e24697f891292ea865e6ed499f162 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 23 Jun 2016 00:51:36 +0700
Subject: [PATCH 5275/6219] [vimeo] Override original URL only when necessary

---
 youtube_dl/extractor/vimeo.py | 29 +++++++++++++++++++++++++----
 1 file changed, 25 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index c52986af6..1f163d6a4 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -227,8 +227,6 @@ class VimeoIE(VimeoBaseInfoExtractor):
         {
             'url': 'http://vimeo.com/channels/keypeele/75629013',
             'md5': '2f86a05afe9d7abc0b9126d229bbe15d',
-            'note': 'Video is freely available via original URL '
-                    'and protected with password when accessed via http://vimeo.com/75629013',
             'info_dict': {
                 'id': '75629013',
                 'ext': 'mp4',
@@ -272,7 +270,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
         {
             # contains original format
             'url': 'https://vimeo.com/33951933',
-            'md5': '53c688fa95a55bf4b7293d37a89c5c53',
+            'md5': '2d9f5475e0537f013d0073e812ab89e6',
             'info_dict': {
                 'id': '33951933',
                 'ext': 'mp4',
@@ -284,6 +282,29 @@ class VimeoIE(VimeoBaseInfoExtractor):
                 'description': 'md5:ae23671e82d05415868f7ad1aec21147',
             },
         },
+        {
+            # only available via https://vimeo.com/channels/tributes/6213729 and
+            # not via https://vimeo.com/6213729
+            'url': 'https://vimeo.com/channels/tributes/6213729',
+            'info_dict': {
+                'id': '6213729',
+                'ext': 'mp4',
+                'title': 'Vimeo Tribute: The Shining',
+                'uploader': 'Casey Donahue',
+                'uploader_url': 're:https?://(?:www\.)?vimeo\.com/caseydonahue',
+                'uploader_id': 'caseydonahue',
+                'upload_date': '20090821',
+                'description': 'md5:bdbf314014e58713e6e5b66eb252f4a6',
+            },
+            'params': {
+                'skip_download': True,
+            },
+            'expected_warnings': ['Unable to download JSON metadata'],
+        },
+        {
+            'url': 'http://vimeo.com/moogaloop.swf?clip_id=2539741',
+            'only_matching': True,
+        },
         {
             'url': 'https://vimeo.com/109815029',
             'note': 'Video not completely processed, "failed" seed status',
@@ -369,7 +390,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
         orig_url = url
         if mobj.group('pro') or mobj.group('player'):
             url = 'https://player.vimeo.com/video/' + video_id
-        else:
+        elif any(p in url for p in ('play_redirect_hls', 'moogaloop.swf')):
             url = 'https://vimeo.com/' + video_id
 
         # Retrieve video webpage to extract further information

From dfc8f46e1c0e47a3b080d2e38d7d6da279f18fd2 Mon Sep 17 00:00:00 2001
From: Purdea Andrei <andrei@purdea.ro>
Date: Wed, 22 Jun 2016 00:34:57 +0300
Subject: [PATCH 5276/6219] [vimeo:channel] Add video id to url_result

This will allow us to decide much faster that we don't want an already archived video,
and will allow having to download webpages for each video that has already been downloaded,
thus significantly speeding up the archival of channels that have no new content.
---
 youtube_dl/extractor/vimeo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 1f163d6a4..32490a8ed 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -652,7 +652,7 @@ class VimeoChannelIE(VimeoBaseInfoExtractor):
                 yield self._extract_list_title(webpage)
 
             for video_id in re.findall(r'id="clip_(\d+?)"', webpage):
-                yield self.url_result('https://vimeo.com/%s' % video_id, 'Vimeo')
+                yield self.url_result('https://vimeo.com/%s' % video_id, 'Vimeo', video_id=video_id)
 
             if re.search(self._MORE_PAGES_INDICATOR, webpage, re.DOTALL) is None:
                 break

From c8e3e0974b4ffb6792694336664f90eff38fc762 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 23 Jun 2016 01:28:36 +0700
Subject: [PATCH 5277/6219] [vimeo:channel] Improve playlist extraction

---
 youtube_dl/extractor/vimeo.py | 25 +++++++++++++++++++++----
 1 file changed, 21 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 32490a8ed..26a3d9931 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -146,7 +146,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
                             \.
                         )?
                         vimeo(?P<pro>pro)?\.com/
-                        (?!channels/[^/?#]+/?(?:$|[?#])|[^/]+/review/|(?:album|ondemand)/)
+                        (?!(?:channels|album)/[^/?#]+/?(?:$|[?#])|[^/]+/review/|ondemand/)
                         (?:.*?/)?
                         (?:
                             (?:
@@ -314,6 +314,10 @@ class VimeoIE(VimeoBaseInfoExtractor):
             'url': 'https://vimeo.com/groups/travelhd/videos/22439234',
             'only_matching': True,
         },
+        {
+            'url': 'https://vimeo.com/album/2632481/video/79010983',
+            'only_matching': True,
+        },
         {
             # source file returns 403: Forbidden
             'url': 'https://vimeo.com/7809605',
@@ -651,8 +655,21 @@ class VimeoChannelIE(VimeoBaseInfoExtractor):
                 webpage = self._login_list_password(page_url, list_id, webpage)
                 yield self._extract_list_title(webpage)
 
-            for video_id in re.findall(r'id="clip_(\d+?)"', webpage):
-                yield self.url_result('https://vimeo.com/%s' % video_id, 'Vimeo', video_id=video_id)
+            # Try extracting href first since not all videos are available via
+            # short https://vimeo.com/id URL (e.g. https://vimeo.com/channels/tributes/6213729)
+            clips = re.findall(
+                r'id="clip_(\d+)"[^>]*>\s*<a[^>]+href="(/(?:[^/]+/)*\1)', webpage)
+            if clips:
+                for video_id, video_url in clips:
+                    yield self.url_result(
+                        compat_urlparse.urljoin(base_url, video_url),
+                        VimeoIE.ie_key(), video_id=video_id)
+            # More relaxed fallback
+            else:
+                for video_id in re.findall(r'id=["\']clip_(\d+)', webpage):
+                    yield self.url_result(
+                        'https://vimeo.com/%s' % video_id,
+                        VimeoIE.ie_key(), video_id=video_id)
 
             if re.search(self._MORE_PAGES_INDICATOR, webpage, re.DOTALL) is None:
                 break
@@ -689,7 +706,7 @@ class VimeoUserIE(VimeoChannelIE):
 
 class VimeoAlbumIE(VimeoChannelIE):
     IE_NAME = 'vimeo:album'
-    _VALID_URL = r'https://vimeo\.com/album/(?P<id>\d+)'
+    _VALID_URL = r'https://vimeo\.com/album/(?P<id>\d+)/?(?:$|[?#])'
     _TITLE_RE = r'<header id="page_header">\n\s*<h1>(.*?)</h1>'
     _TESTS = [{
         'url': 'https://vimeo.com/album/2632481',

From b5eab86c2424ec04d17fac5de9d15574320ea8f1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 23 Jun 2016 01:56:58 +0700
Subject: [PATCH 5278/6219] [vimeo:album] Impove _VALID_URL

---
 youtube_dl/extractor/vimeo.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 26a3d9931..8ba3f55f4 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -706,7 +706,7 @@ class VimeoUserIE(VimeoChannelIE):
 
 class VimeoAlbumIE(VimeoChannelIE):
     IE_NAME = 'vimeo:album'
-    _VALID_URL = r'https://vimeo\.com/album/(?P<id>\d+)/?(?:$|[?#])'
+    _VALID_URL = r'https://vimeo\.com/album/(?P<id>\d+)(?:$|[?#]|/(?!video))'
     _TITLE_RE = r'<header id="page_header">\n\s*<h1>(.*?)</h1>'
     _TESTS = [{
         'url': 'https://vimeo.com/album/2632481',
@@ -726,6 +726,9 @@ class VimeoAlbumIE(VimeoChannelIE):
         'params': {
             'videopassword': 'youtube-dl',
         }
+    }, {
+        'url': 'https://vimeo.com/album/2632481/sort:plays/format:thumbnail',
+        'only_matching': True,
     }]
 
     def _page_url(self, base_url, pagenum):

From 089657ed1f6edcdb10a958a8cd7d91b4888e41eb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 23 Jun 2016 02:00:03 +0700
Subject: [PATCH 5279/6219] [vimeo:album] Add paged example URL

---
 youtube_dl/extractor/vimeo.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 8ba3f55f4..4bdeb1187 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -729,6 +729,10 @@ class VimeoAlbumIE(VimeoChannelIE):
     }, {
         'url': 'https://vimeo.com/album/2632481/sort:plays/format:thumbnail',
         'only_matching': True,
+    }, {
+        # TODO: respect page number
+        'url': 'https://vimeo.com/album/2632481/page:2/sort:plays/format:thumbnail',
+        'only_matching': True,
     }]
 
     def _page_url(self, base_url, pagenum):

From 75ca6bcee2466cb9ca3dc4d1ca35b56a59f6cc4d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 23 Jun 2016 04:17:13 +0700
Subject: [PATCH 5280/6219] [vk] Workaround buggy new.vk.com Set-Cookie headers

---
 youtube_dl/extractor/vk.py | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 79c819bc3..4e8ec0f86 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -3,6 +3,7 @@ from __future__ import unicode_literals
 
 import re
 import json
+import sys
 
 from .common import InfoExtractor
 from ..compat import compat_str
@@ -190,7 +191,7 @@ class VKIE(InfoExtractor):
         if username is None:
             return
 
-        login_page = self._download_webpage(
+        login_page, url_handle = self._download_webpage_handle(
             'https://vk.com', None, 'Downloading login page')
 
         login_form = self._hidden_inputs(login_page)
@@ -200,6 +201,22 @@ class VKIE(InfoExtractor):
             'pass': password.encode('cp1251'),
         })
 
+        # https://new.vk.com/ serves two same remixlhk cookies in Set-Cookie header
+        # and expects the first one to be set rather than second (see
+        # https://github.com/rg3/youtube-dl/issues/9841#issuecomment-227871201).
+        # As of RFC6265 the newer one cookie should be set into cookie store
+        # what actually happens.
+        # We will workaround this VK issue by resetting the remixlhk cookie to
+        # the first one manually.
+        cookies = url_handle.headers.get('Set-Cookie')
+        if sys.version_info[0] >= 3:
+            cookies = cookies.encode('iso-8859-1')
+        cookies = cookies.decode('utf-8')
+        remixlhk = re.search(r'remixlhk=(.+?);.*?\bdomain=(.+?)(?:[,;]|$)', cookies)
+        if remixlhk:
+            value, domain = remixlhk.groups()
+            self._set_cookie(domain, 'remixlhk', value)
+
         request = sanitized_Request(
             'https://login.vk.com/?act=login',
             urlencode_postdata(login_form))

From f2bb8c036a0b1feab726321bd877544cb973d7cc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 23 Jun 2016 04:18:43 +0700
Subject: [PATCH 5281/6219] [vk] Modernize

---
 youtube_dl/extractor/vk.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 4e8ec0f86..f8d07beaf 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -217,11 +217,10 @@ class VKIE(InfoExtractor):
             value, domain = remixlhk.groups()
             self._set_cookie(domain, 'remixlhk', value)
 
-        request = sanitized_Request(
-            'https://login.vk.com/?act=login',
-            urlencode_postdata(login_form))
         login_page = self._download_webpage(
-            request, None, note='Logging in as %s' % username)
+            'https://login.vk.com/?act=login', None,
+            note='Logging in as %s' % username,
+            data=urlencode_postdata(login_form))
 
         if re.search(r'onLoginFailed', login_page):
             raise ExtractorError(

From 73843ae8acb378e986ab4e7bb3a525b2f6b53cf2 Mon Sep 17 00:00:00 2001
From: rr- <rr-@sakuya.pl>
Date: Wed, 22 Jun 2016 17:24:35 +0200
Subject: [PATCH 5282/6219] [xnxx] fix url regex

The pattern has changed from "video123412" to "video-o8xa19".
The changes maintain backwards compatibility with old-style URLs.
---
 youtube_dl/extractor/xnxx.py | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/xnxx.py b/youtube_dl/extractor/xnxx.py
index 5a41f8ffa..f0d4cb9bc 100644
--- a/youtube_dl/extractor/xnxx.py
+++ b/youtube_dl/extractor/xnxx.py
@@ -6,17 +6,20 @@ from ..compat import compat_urllib_parse_unquote
 
 
 class XNXXIE(InfoExtractor):
-    _VALID_URL = r'^https?://(?:video|www)\.xnxx\.com/video(?P<id>[0-9]+)/(.*)'
-    _TEST = {
-        'url': 'http://video.xnxx.com/video1135332/lida_naked_funny_actress_5_',
-        'md5': '0831677e2b4761795f68d417e0b7b445',
+    _VALID_URL = r'^https?://(?:video|www)\.xnxx\.com/video-?(?P<id>[0-9a-z]+)/(.*)'
+    _TESTS = [{
+        'url': 'http://www.xnxx.com/video-6gqggeb/hd_star-581_sam',
+        'md5': '6a2a6aff3f10467d94e572edb7b7deb6',
         'info_dict': {
-            'id': '1135332',
+            'id': '6gqggeb',
             'ext': 'flv',
-            'title': 'lida » Naked Funny Actress  (5)',
+            'title': 'HD STAR-581 sam',
             'age_limit': 18,
-        }
-    }
+        },
+    }, {
+        'url': 'http://video.xnxx.com/video1135332/lida_naked_funny_actress_5_',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From 97674f041916860343d804b8b07b73017e1a517f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 23 Jun 2016 04:24:00 +0700
Subject: [PATCH 5283/6219] [xnxx] Replace test

---
 youtube_dl/extractor/xnxx.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/xnxx.py b/youtube_dl/extractor/xnxx.py
index f0d4cb9bc..1e677a63b 100644
--- a/youtube_dl/extractor/xnxx.py
+++ b/youtube_dl/extractor/xnxx.py
@@ -8,12 +8,12 @@ from ..compat import compat_urllib_parse_unquote
 class XNXXIE(InfoExtractor):
     _VALID_URL = r'^https?://(?:video|www)\.xnxx\.com/video-?(?P<id>[0-9a-z]+)/(.*)'
     _TESTS = [{
-        'url': 'http://www.xnxx.com/video-6gqggeb/hd_star-581_sam',
-        'md5': '6a2a6aff3f10467d94e572edb7b7deb6',
+        'url': 'http://www.xnxx.com/video-55awb78/skyrim_test_video',
+        'md5': 'ef7ecee5af78f8b03dca2cf31341d3a0',
         'info_dict': {
-            'id': '6gqggeb',
+            'id': '55awb78',
             'ext': 'flv',
-            'title': 'HD STAR-581 sam',
+            'title': 'Skyrim Test Video',
             'age_limit': 18,
         },
     }, {

From adf1921dc157af23e8b317d6095b88c87a149e2f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 23 Jun 2016 04:26:49 +0700
Subject: [PATCH 5284/6219] [xnxx] Improve _VALID_URL (Closes #9858)

---
 youtube_dl/extractor/xnxx.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/xnxx.py b/youtube_dl/extractor/xnxx.py
index 1e677a63b..bcb140305 100644
--- a/youtube_dl/extractor/xnxx.py
+++ b/youtube_dl/extractor/xnxx.py
@@ -6,7 +6,7 @@ from ..compat import compat_urllib_parse_unquote
 
 
 class XNXXIE(InfoExtractor):
-    _VALID_URL = r'^https?://(?:video|www)\.xnxx\.com/video-?(?P<id>[0-9a-z]+)/(.*)'
+    _VALID_URL = r'https?://(?:video|www)\.xnxx\.com/video-?(?P<id>[0-9a-z]+)/'
     _TESTS = [{
         'url': 'http://www.xnxx.com/video-55awb78/skyrim_test_video',
         'md5': 'ef7ecee5af78f8b03dca2cf31341d3a0',
@@ -19,6 +19,9 @@ class XNXXIE(InfoExtractor):
     }, {
         'url': 'http://video.xnxx.com/video1135332/lida_naked_funny_actress_5_',
         'only_matching': True,
+    }, {
+        'url': 'http://www.xnxx.com/video-55awb78/',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 3331a4644d141ba9163ecf08015ccb9e0a5b87f4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 23 Jun 2016 04:27:10 +0700
Subject: [PATCH 5285/6219] [vk] Remove unused import

---
 youtube_dl/extractor/vk.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index f8d07beaf..cfc5ffd8b 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -11,7 +11,6 @@ from ..utils import (
     ExtractorError,
     int_or_none,
     orderedSet,
-    sanitized_Request,
     str_to_int,
     unescapeHTML,
     unified_strdate,

From 96f88e91b7bac15b3a6f1eafb6a66964d2d11a7c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 23 Jun 2016 04:29:34 +0700
Subject: [PATCH 5286/6219] release 2016.06.23

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 youtube_dl/version.py     | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index e17625f21..f7d1020d3 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.06.22*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.06.22**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.06.23*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.06.23**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.06.22
+[debug] youtube-dl version 2016.06.23
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index d2152b2f1..bff747906 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.06.22'
+__version__ = '2016.06.23'

From 22b7ac13ef4e34654bcb3fb3dbb40d2fac9b4278 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 23 Jun 2016 00:13:52 +0100
Subject: [PATCH 5287/6219] [tf1] fix wat id extraction(closes #9862)

---
 youtube_dl/extractor/tf1.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tf1.py b/youtube_dl/extractor/tf1.py
index 6c848dc6f..e595c4a69 100644
--- a/youtube_dl/extractor/tf1.py
+++ b/youtube_dl/extractor/tf1.py
@@ -48,6 +48,6 @@ class TF1IE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
         wat_id = self._html_search_regex(
-            r'(["\'])(?:https?:)?//www\.wat\.tv/embedframe/.*?(?P<id>\d{8}).*?\1',
+            r'(["\'])(?:https?:)?//www\.wat\.tv/embedframe/.*?(?P<id>\d{8})\1',
             webpage, 'wat id', group='id')
         return self.url_result('wat:%s' % wat_id, 'Wat')

From 0437307a41dc97becc807abc9a86070ac9c847d8 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 23 Jun 2016 01:36:19 +0100
Subject: [PATCH 5288/6219] [nbc:nbcnews] improve extraction and add msnbc to
 the extractor

---
 youtube_dl/extractor/extractors.py |   1 -
 youtube_dl/extractor/nbc.py        | 157 +++++++++--------------------
 2 files changed, 50 insertions(+), 108 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 4e2a2f2e9..6fc5a18f5 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -480,7 +480,6 @@ from .nbc import (
     NBCNewsIE,
     NBCSportsIE,
     NBCSportsVPlayerIE,
-    MSNBCIE,
 )
 from .ndr import (
     NDRIE,
diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index 6b7da1149..f694e210b 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -9,10 +9,6 @@ from ..utils import (
     lowercase_escape,
     smuggle_url,
     unescapeHTML,
-    update_url_query,
-    int_or_none,
-    HEADRequest,
-    parse_iso8601,
 )
 
 
@@ -192,9 +188,9 @@ class CSNNEIE(InfoExtractor):
 
 
 class NBCNewsIE(ThePlatformIE):
-    _VALID_URL = r'''(?x)https?://(?:www\.)?(?:nbcnews|today)\.com/
+    _VALID_URL = r'''(?x)https?://(?:www\.)?(?:nbcnews|today|msnbc)\.com/
         (?:video/.+?/(?P<id>\d+)|
-        ([^/]+/)*(?P<display_id>[^/?]+))
+        ([^/]+/)*(?:.*-)?(?P<mpx_id>[^/?]+))
         '''
 
     _TESTS = [
@@ -216,13 +212,16 @@ class NBCNewsIE(ThePlatformIE):
                 'ext': 'mp4',
                 'title': 'How Twitter Reacted To The Snowden Interview',
                 'description': 'md5:65a0bd5d76fe114f3c2727aa3a81fe64',
+                'uploader': 'NBCU-NEWS',
+                'timestamp': 1401363060,
+                'upload_date': '20140529',
             },
         },
         {
             'url': 'http://www.nbcnews.com/feature/dateline-full-episodes/full-episode-family-business-n285156',
             'md5': 'fdbf39ab73a72df5896b6234ff98518a',
             'info_dict': {
-                'id': 'Wjf9EDR3A_60',
+                'id': '529953347624',
                 'ext': 'mp4',
                 'title': 'FULL EPISODE: Family Business',
                 'description': 'md5:757988edbaae9d7be1d585eb5d55cc04',
@@ -237,6 +236,9 @@ class NBCNewsIE(ThePlatformIE):
                 'ext': 'mp4',
                 'title': 'Nightly News with Brian Williams Full Broadcast (February 4)',
                 'description': 'md5:1c10c1eccbe84a26e5debb4381e2d3c5',
+                'timestamp': 1423104900,
+                'uploader': 'NBCU-NEWS',
+                'upload_date': '20150205',
             },
         },
         {
@@ -245,10 +247,12 @@ class NBCNewsIE(ThePlatformIE):
             'info_dict': {
                 'id': '529953347624',
                 'ext': 'mp4',
-                'title': 'Volkswagen U.S. Chief: We \'Totally Screwed Up\'',
-                'description': 'md5:d22d1281a24f22ea0880741bb4dd6301',
+                'title': 'Volkswagen U.S. Chief:\xa0 We Have Totally Screwed Up',
+                'description': 'md5:c8be487b2d80ff0594c005add88d8351',
+                'upload_date': '20150922',
+                'timestamp': 1442917800,
+                'uploader': 'NBCU-NEWS',
             },
-            'expected_warnings': ['http-6000 is not available']
         },
         {
             'url': 'http://www.today.com/video/see-the-aurora-borealis-from-space-in-stunning-new-nasa-video-669831235788',
@@ -260,6 +264,22 @@ class NBCNewsIE(ThePlatformIE):
                 'description': 'md5:74752b7358afb99939c5f8bb2d1d04b1',
                 'upload_date': '20160420',
                 'timestamp': 1461152093,
+                'uploader': 'NBCU-NEWS',
+            },
+        },
+        {
+            'url': 'http://www.msnbc.com/all-in-with-chris-hayes/watch/the-chaotic-gop-immigration-vote-314487875924',
+            'md5': '6d236bf4f3dddc226633ce6e2c3f814d',
+            'info_dict': {
+                'id': '314487875924',
+                'ext': 'mp4',
+                'title': 'The chaotic GOP immigration vote',
+                'description': 'The Republican House votes on a border bill that has no chance of getting through the Senate or signed by the President and is drawing criticism from all sides.',
+                'thumbnail': 're:^https?://.*\.jpg$',
+                'timestamp': 1406937606,
+                'upload_date': '20140802',
+                'uploader': 'NBCU-NEWS',
+                'categories': ['MSNBC/Topics/Franchise/Best of last night', 'MSNBC/Topics/General/Congress'],
             },
         },
         {
@@ -290,105 +310,28 @@ class NBCNewsIE(ThePlatformIE):
             }
         else:
             # "feature" and "nightly-news" pages use theplatform.com
-            display_id = mobj.group('display_id')
-            webpage = self._download_webpage(url, display_id)
-            info = None
-            bootstrap_json = self._search_regex(
-                [r'(?m)(?:var\s+(?:bootstrapJson|playlistData)|NEWS\.videoObj)\s*=\s*({.+});?\s*$',
-                 r'videoObj\s*:\s*({.+})', r'data-video="([^"]+)"'],
-                webpage, 'bootstrap json', default=None)
-            bootstrap = self._parse_json(
-                bootstrap_json, display_id, transform_source=unescapeHTML)
-            if 'results' in bootstrap:
-                info = bootstrap['results'][0]['video']
-            elif 'video' in bootstrap:
-                info = bootstrap['video']
-            else:
-                info = bootstrap
-            video_id = info['mpxId']
-            title = info['title']
-
-            subtitles = {}
-            caption_links = info.get('captionLinks')
-            if caption_links:
-                for (sub_key, sub_ext) in (('smpte-tt', 'ttml'), ('web-vtt', 'vtt'), ('srt', 'srt')):
-                    sub_url = caption_links.get(sub_key)
-                    if sub_url:
-                        subtitles.setdefault('en', []).append({
-                            'url': sub_url,
-                            'ext': sub_ext,
-                        })
-
-            formats = []
-            for video_asset in info['videoAssets']:
-                video_url = video_asset.get('publicUrl')
-                if not video_url:
-                    continue
-                container = video_asset.get('format')
-                asset_type = video_asset.get('assetType') or ''
-                if container == 'ISM' or asset_type == 'FireTV-Once':
-                    continue
-                elif asset_type == 'OnceURL':
-                    tp_formats, tp_subtitles = self._extract_theplatform_smil(
-                        video_url, video_id)
-                    formats.extend(tp_formats)
-                    subtitles = self._merge_subtitles(subtitles, tp_subtitles)
+            video_id = mobj.group('mpx_id')
+            if not video_id.isdigit():
+                webpage = self._download_webpage(url, video_id)
+                info = None
+                bootstrap_json = self._search_regex(
+                    [r'(?m)(?:var\s+(?:bootstrapJson|playlistData)|NEWS\.videoObj)\s*=\s*({.+});?\s*$',
+                     r'videoObj\s*:\s*({.+})', r'data-video="([^"]+)"'],
+                    webpage, 'bootstrap json', default=None)
+                bootstrap = self._parse_json(
+                    bootstrap_json, video_id, transform_source=unescapeHTML)
+                if 'results' in bootstrap:
+                    info = bootstrap['results'][0]['video']
+                elif 'video' in bootstrap:
+                    info = bootstrap['video']
                 else:
-                    tbr = int_or_none(video_asset.get('bitRate') or video_asset.get('bitrate'), 1000)
-                    format_id = 'http%s' % ('-%d' % tbr if tbr else '')
-                    video_url = update_url_query(
-                        video_url, {'format': 'redirect'})
-                    # resolve the url so that we can check availability and detect the correct extension
-                    head = self._request_webpage(
-                        HEADRequest(video_url), video_id,
-                        'Checking %s url' % format_id,
-                        '%s is not available' % format_id,
-                        fatal=False)
-                    if head:
-                        video_url = head.geturl()
-                        formats.append({
-                            'format_id': format_id,
-                            'url': video_url,
-                            'width': int_or_none(video_asset.get('width')),
-                            'height': int_or_none(video_asset.get('height')),
-                            'tbr': tbr,
-                            'container': video_asset.get('format'),
-                        })
-            self._sort_formats(formats)
+                    info = bootstrap
+                video_id = info['mpxId']
 
             return {
+                '_type': 'url_transparent',
                 'id': video_id,
-                'title': title,
-                'description': info.get('description'),
-                'thumbnail': info.get('thumbnail'),
-                'duration': int_or_none(info.get('duration')),
-                'timestamp': parse_iso8601(info.get('pubDate') or info.get('pub_date')),
-                'formats': formats,
-                'subtitles': subtitles,
+                # http://feed.theplatform.com/f/2E2eJC/nbcnews also works
+                'url': 'http://feed.theplatform.com/f/2E2eJC/nnd_NBCNews?byId=%s' % video_id,
+                'ie_key': 'ThePlatformFeed',
             }
-
-
-class MSNBCIE(InfoExtractor):
-    # https URLs redirect to corresponding http ones
-    _VALID_URL = r'https?://www\.msnbc\.com/[^/]+/watch/(?P<id>[^/]+)'
-    _TEST = {
-        'url': 'http://www.msnbc.com/all-in-with-chris-hayes/watch/the-chaotic-gop-immigration-vote-314487875924',
-        'md5': '6d236bf4f3dddc226633ce6e2c3f814d',
-        'info_dict': {
-            'id': 'n_hayes_Aimm_140801_272214',
-            'ext': 'mp4',
-            'title': 'The chaotic GOP immigration vote',
-            'description': 'The Republican House votes on a border bill that has no chance of getting through the Senate or signed by the President and is drawing criticism from all sides.',
-            'thumbnail': 're:^https?://.*\.jpg$',
-            'timestamp': 1406937606,
-            'upload_date': '20140802',
-            'uploader': 'NBCU-NEWS',
-            'categories': ['MSNBC/Topics/Franchise/Best of last night', 'MSNBC/Topics/General/Congress'],
-        },
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-        embed_url = self._html_search_meta('embedURL', webpage)
-        return self.url_result(embed_url)

From b46eabecd3d6e8ea6dd3dc5a948ecbb65d818205 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 23 Jun 2016 09:41:34 +0700
Subject: [PATCH 5289/6219] [jsinterp] Relax JS function regex (Closes #9863)

---
 youtube_dl/jsinterp.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index a7440c582..9737f7002 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -232,7 +232,7 @@ class JSInterpreter(object):
     def extract_function(self, funcname):
         func_m = re.search(
             r'''(?x)
-                (?:function\s+%s|[{;,]%s\s*=\s*function|var\s+%s\s*=\s*function)\s*
+                (?:function\s+%s|[{;,]\s*%s\s*=\s*function|var\s+%s\s*=\s*function)\s*
                 \((?P<args>[^)]*)\)\s*
                 \{(?P<code>[^}]+)\}''' % (
                 re.escape(funcname), re.escape(funcname), re.escape(funcname)),

From 011bd3221b1541eaef9bb14786da37abe4d74ecb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 23 Jun 2016 09:42:56 +0700
Subject: [PATCH 5290/6219] release 2016.06.23.1

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 docs/supportedsites.md    | 3 +--
 youtube_dl/version.py     | 2 +-
 3 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index f7d1020d3..62cb18d7d 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.06.23*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.06.23**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.06.23.1*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.06.23.1**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.06.23
+[debug] youtube-dl version 2016.06.23.1
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 96cc407db..891499f59 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -44,8 +44,8 @@
  - **appletrailers:section**
  - **archive.org**: archive.org videos
  - **ARD**
- - **ARD:mediathek**: Saarländischer Rundfunk
  - **ARD:mediathek**
+ - **ARD:mediathek**: Saarländischer Rundfunk
  - **arte.tv**
  - **arte.tv:+7**
  - **arte.tv:cinema**
@@ -385,7 +385,6 @@
  - **MovieFap**
  - **Moviezine**
  - **MPORA**
- - **MSNBC**
  - **MTV**
  - **mtv.de**
  - **mtviggy.com**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index bff747906..0238dc97c 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.06.23'
+__version__ = '2016.06.23.1'

From 6e3c2047f8a51da3bac0d4d290d64b1b8bb8f1c2 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 23 Jun 2016 04:34:07 +0100
Subject: [PATCH 5291/6219] [tvp] extract all formats and detect erros

---
 youtube_dl/extractor/tvp.py | 59 ++++++++++++++++++++++++++++++-------
 1 file changed, 48 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/tvp.py b/youtube_dl/extractor/tvp.py
index a4997cb89..5070082da 100644
--- a/youtube_dl/extractor/tvp.py
+++ b/youtube_dl/extractor/tvp.py
@@ -4,6 +4,12 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..utils import (
+    determine_ext,
+    clean_html,
+    get_element_by_attribute,
+    ExtractorError,
+)
 
 
 class TVPIE(InfoExtractor):
@@ -21,7 +27,7 @@ class TVPIE(InfoExtractor):
         },
     }, {
         'url': 'http://www.tvp.pl/there-can-be-anything-so-i-shortened-it/17916176',
-        'md5': 'c3b15ed1af288131115ff17a17c19dda',
+        'md5': 'b0005b542e5b4de643a9690326ab1257',
         'info_dict': {
             'id': '17916176',
             'ext': 'mp4',
@@ -53,6 +59,11 @@ class TVPIE(InfoExtractor):
         webpage = self._download_webpage(
             'http://www.tvp.pl/sess/tvplayer.php?object_id=%s' % video_id, video_id)
 
+        error_massage = get_element_by_attribute('class', 'msg error', webpage)
+        if error_massage:
+            raise ExtractorError('%s said: %s' % (
+                self.IE_NAME, clean_html(error_massage)), expected=True)
+
         title = self._search_regex(
             r'name\s*:\s*([\'"])Title\1\s*,\s*value\s*:\s*\1(?P<title>.+?)\1',
             webpage, 'title', group='title')
@@ -66,24 +77,50 @@ class TVPIE(InfoExtractor):
             r"poster\s*:\s*'([^']+)'", webpage, 'thumbnail', default=None)
 
         video_url = self._search_regex(
-            r'0:{src:([\'"])(?P<url>.*?)\1', webpage, 'formats', group='url', default=None)
-        if not video_url:
+            r'0:{src:([\'"])(?P<url>.*?)\1', webpage,
+            'formats', group='url', default=None)
+        if not video_url or 'material_niedostepny.mp4' in video_url:
             video_url = self._download_json(
                 'http://www.tvp.pl/pub/stat/videofileinfo?video_id=%s' % video_id,
                 video_id)['video_url']
 
-        ext = video_url.rsplit('.', 1)[-1]
-        if ext != 'ism/manifest':
-            if '/' in ext:
-                ext = 'mp4'
+        formats = []
+        video_url_base = self._search_regex(
+            r'(https?://.+?/video)(?:\.(?:ism|f4m|m3u8)|-\d+\.mp4)',
+            video_url, 'video base url', default=None)
+        if video_url_base:
+            # TODO: Current DASH formats are broken - $Time$ pattern in
+            # <SegmentTemplate> not implemented yet
+            # formats.extend(self._extract_mpd_formats(
+            #     video_url_base + '.ism/video.mpd',
+            #     video_id, mpd_id='dash', fatal=False))
+            formats.extend(self._extract_f4m_formats(
+                video_url_base + '.ism/video.f4m',
+                video_id, f4m_id='hds', fatal=False))
+            m3u8_formats = self._extract_m3u8_formats(
+                video_url_base + '.ism/video.m3u8', video_id,
+                'mp4', 'm3u8_native', m3u8_id='hls', fatal=False)
+            self._sort_formats(m3u8_formats)
+            m3u8_formats = list(filter(
+                lambda f: f.get('vcodec') != 'none' and f.get('resolution') != 'multiple',
+                m3u8_formats))
+            formats.extend(m3u8_formats)
+            for i, m3u8_format in enumerate(m3u8_formats, 2):
+                http_url = '%s-%d.mp4' % (video_url_base, i)
+                if self._is_valid_url(http_url, video_id):
+                    f = m3u8_format.copy()
+                    f.update({
+                        'url': http_url,
+                        'format_id': f['format_id'].replace('hls', 'http'),
+                        'protocol': 'http',
+                    })
+                    formats.append(f)
+        else:
             formats = [{
                 'format_id': 'direct',
                 'url': video_url,
-                'ext': ext,
+                'ext': determine_ext(video_url, 'mp4'),
             }]
-        else:
-            m3u8_url = re.sub('([^/]*)\.ism/manifest', r'\1.ism/\1.m3u8', video_url)
-            formats = self._extract_m3u8_formats(m3u8_url, video_id, 'mp4')
 
         self._sort_formats(formats)
 

From 494172d2e5b2d5b6f309b42e1a2bd7108aed40de Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 23 Jun 2016 15:49:42 +0100
Subject: [PATCH 5292/6219] [appletrailers] extract info from an alternative
 source if available(closes #8422)(closes #8422)

---
 youtube_dl/extractor/appletrailers.py | 53 ++++++++++++++++++++++++++-
 1 file changed, 52 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/appletrailers.py b/youtube_dl/extractor/appletrailers.py
index be40f85b4..babbd0265 100644
--- a/youtube_dl/extractor/appletrailers.py
+++ b/youtube_dl/extractor/appletrailers.py
@@ -7,6 +7,8 @@ from .common import InfoExtractor
 from ..compat import compat_urlparse
 from ..utils import (
     int_or_none,
+    parse_duration,
+    unified_strdate,
 )
 
 
@@ -16,7 +18,8 @@ class AppleTrailersIE(InfoExtractor):
     _TESTS = [{
         'url': 'http://trailers.apple.com/trailers/wb/manofsteel/',
         'info_dict': {
-            'id': 'manofsteel',
+            'id': '5111',
+            'title': 'Man of Steel',
         },
         'playlist': [
             {
@@ -70,6 +73,15 @@ class AppleTrailersIE(InfoExtractor):
             'id': 'blackthorn',
         },
         'playlist_mincount': 2,
+        'expected_warnings': ['Unable to download JSON metadata'],
+    }, {
+        # json data only available from http://trailers.apple.com/trailers/feeds/data/15881.json
+        'url': 'http://trailers.apple.com/trailers/fox/kungfupanda3/',
+        'info_dict': {
+            'id': '15881',
+            'title': 'Kung Fu Panda 3',
+        },
+        'playlist_mincount': 4,
     }, {
         'url': 'http://trailers.apple.com/ca/metropole/autrui/',
         'only_matching': True,
@@ -85,6 +97,45 @@ class AppleTrailersIE(InfoExtractor):
         movie = mobj.group('movie')
         uploader_id = mobj.group('company')
 
+        webpage = self._download_webpage(url, movie)
+        film_id = self._search_regex(r"FilmId\s*=\s*'(\d+)'", webpage, 'film id')
+        film_data = self._download_json(
+            'http://trailers.apple.com/trailers/feeds/data/%s.json' % film_id,
+            film_id, fatal=False)
+
+        if film_data:
+            entries = []
+            for clip in film_data.get('clips', []):
+                clip_title = clip['title']
+
+                formats = []
+                for version, version_data in clip.get('versions', {}).items():
+                    for size, size_data in version_data.get('sizes', {}).items():
+                        src = size_data.get('src')
+                        if not src:
+                            continue
+                        formats.append({
+                            'format_id': '%s-%s' % (version, size),
+                            'url': re.sub(r'_(\d+p.mov)', r'_h\1', src),
+                            'width': int_or_none(size_data.get('width')),
+                            'height': int_or_none(size_data.get('height')),
+                            'language': version[:2],
+                        })
+                self._sort_formats(formats)
+
+                entries.append({
+                    'id': movie + '-' + re.sub(r'[^a-zA-Z0-9]', '', clip_title).lower(),
+                    'formats': formats,
+                    'title': clip_title,
+                    'thumbnail': clip.get('screen') or clip.get('runtime'),
+                    'duration': parse_duration(clip.get('runtime') or clip.get('faded')),
+                    'upload_date': unified_strdate(clip.get('posted')),
+                    'uploader_id': uploader_id,
+                })
+
+            page_data = film_data.get('page', {})
+            return self.playlist_result(entries, film_id, page_data.get('movie_title'))
+
         playlist_url = compat_urlparse.urljoin(url, 'includes/playlists/itunes.inc')
 
         def fix_html(s):

From 8065d6c55f02c6f618e8495049f253d311cf347f Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 23 Jun 2016 17:22:15 +0100
Subject: [PATCH 5293/6219] [dcn] extend _VALID_URL for awaan.ae and extract
 all available formats

---
 youtube_dl/extractor/dcn.py | 47 ++++++++++++++++++++-----------------
 1 file changed, 26 insertions(+), 21 deletions(-)

diff --git a/youtube_dl/extractor/dcn.py b/youtube_dl/extractor/dcn.py
index 5deff5f30..efb8585e8 100644
--- a/youtube_dl/extractor/dcn.py
+++ b/youtube_dl/extractor/dcn.py
@@ -20,7 +20,7 @@ from ..utils import (
 
 
 class DCNIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?dcndigital\.ae/(?:#/)?show/(?P<show_id>\d+)/[^/]+(?:/(?P<video_id>\d+)/(?P<season_id>\d+))?'
+    _VALID_URL = r'https?://(?:www\.)?(?:awaan|dcndigital)\.ae/(?:#/)?show/(?P<show_id>\d+)/[^/]+(?:/(?P<video_id>\d+)/(?P<season_id>\d+))?'
 
     def _real_extract(self, url):
         show_id, video_id, season_id = re.match(self._VALID_URL, url).groups()
@@ -55,30 +55,32 @@ class DCNBaseIE(InfoExtractor):
             'is_live': is_live,
         }
 
-    def _extract_video_formats(self, webpage, video_id, entry_protocol):
+    def _extract_video_formats(self, webpage, video_id, m3u8_entry_protocol):
         formats = []
-        m3u8_url = self._html_search_regex(
-            r'file\s*:\s*"([^"]+)', webpage, 'm3u8 url', fatal=False)
-        if m3u8_url:
-            formats.extend(self._extract_m3u8_formats(
-                m3u8_url, video_id, 'mp4', entry_protocol, m3u8_id='hls', fatal=None))
-
-        rtsp_url = self._search_regex(
-            r'<a[^>]+href="(rtsp://[^"]+)"', webpage, 'rtsp url', fatal=False)
-        if rtsp_url:
-            formats.append({
-                'url': rtsp_url,
-                'format_id': 'rtsp',
-            })
-
+        format_url_base = 'http' + self._html_search_regex(
+            [
+                r'file\s*:\s*"https?(://[^"]+)/playlist.m3u8',
+                r'<a[^>]+href="rtsp(://[^"]+)"'
+            ], webpage, 'format url')
+        # TODO: Current DASH formats are broken - $Time$ pattern in
+        # <SegmentTemplate> not implemented yet
+        # formats.extend(self._extract_mpd_formats(
+        #     format_url_base + '/manifest.mpd',
+        #     video_id, mpd_id='dash', fatal=False))
+        formats.extend(self._extract_m3u8_formats(
+            format_url_base + '/playlist.m3u8', video_id, 'mp4',
+            m3u8_entry_protocol, m3u8_id='hls', fatal=False))
+        formats.extend(self._extract_f4m_formats(
+            format_url_base + '/manifest.f4m',
+            video_id, f4m_id='hds', fatal=False))
         self._sort_formats(formats)
         return formats
 
 
 class DCNVideoIE(DCNBaseIE):
     IE_NAME = 'dcn:video'
-    _VALID_URL = r'https?://(?:www\.)?dcndigital\.ae/(?:#/)?(?:video/[^/]+|media|catchup/[^/]+/[^/]+)/(?P<id>\d+)'
-    _TEST = {
+    _VALID_URL = r'https?://(?:www\.)?(?:awaan|dcndigital)\.ae/(?:#/)?(?:video(?:/[^/]+)?|media|catchup/[^/]+/[^/]+)/(?P<id>\d+)'
+    _TESTS = [{
         'url': 'http://www.dcndigital.ae/#/video/%D8%B1%D8%AD%D9%84%D8%A9-%D8%A7%D9%84%D8%B9%D9%85%D8%B1-%D8%A7%D9%84%D8%AD%D9%84%D9%82%D8%A9-1/17375',
         'info_dict':
         {
@@ -94,7 +96,10 @@ class DCNVideoIE(DCNBaseIE):
             # m3u8 download
             'skip_download': True,
         },
-    }
+    }, {
+        'url': 'http://awaan.ae/video/26723981/%D8%AF%D8%A7%D8%B1-%D8%A7%D9%84%D8%B3%D9%84%D8%A7%D9%85:-%D8%AE%D9%8A%D8%B1-%D8%AF%D9%88%D8%B1-%D8%A7%D9%84%D8%A3%D9%86%D8%B5%D8%A7%D8%B1',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -120,7 +125,7 @@ class DCNVideoIE(DCNBaseIE):
 
 class DCNLiveIE(DCNBaseIE):
     IE_NAME = 'dcn:live'
-    _VALID_URL = r'https?://(?:www\.)?dcndigital\.ae/(?:#/)?live/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?(?:awaan|dcndigital)\.ae/(?:#/)?live/(?P<id>\d+)'
 
     def _real_extract(self, url):
         channel_id = self._match_id(url)
@@ -147,7 +152,7 @@ class DCNLiveIE(DCNBaseIE):
 
 class DCNSeasonIE(InfoExtractor):
     IE_NAME = 'dcn:season'
-    _VALID_URL = r'https?://(?:www\.)?dcndigital\.ae/(?:#/)?program/(?:(?P<show_id>\d+)|season/(?P<season_id>\d+))'
+    _VALID_URL = r'https?://(?:www\.)?(?:awaan|dcndigital)\.ae/(?:#/)?program/(?:(?P<show_id>\d+)|season/(?P<season_id>\d+))'
     _TEST = {
         'url': 'http://dcndigital.ae/#/program/205024/%D9%85%D8%AD%D8%A7%D8%B6%D8%B1%D8%A7%D8%AA-%D8%A7%D9%84%D8%B4%D9%8A%D8%AE-%D8%A7%D9%84%D8%B4%D8%B9%D8%B1%D8%A7%D9%88%D9%8A',
         'info_dict':

From fee70322d76f416c3d68f58abdc73f9d3960083e Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 23 Jun 2016 19:03:34 +0100
Subject: [PATCH 5294/6219] [appletrailers] correct thumbnail fallback

---
 youtube_dl/extractor/appletrailers.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/appletrailers.py b/youtube_dl/extractor/appletrailers.py
index babbd0265..a6801f3d4 100644
--- a/youtube_dl/extractor/appletrailers.py
+++ b/youtube_dl/extractor/appletrailers.py
@@ -127,7 +127,7 @@ class AppleTrailersIE(InfoExtractor):
                     'id': movie + '-' + re.sub(r'[^a-zA-Z0-9]', '', clip_title).lower(),
                     'formats': formats,
                     'title': clip_title,
-                    'thumbnail': clip.get('screen') or clip.get('runtime'),
+                    'thumbnail': clip.get('screen') or clip.get('thumb'),
                     'duration': parse_duration(clip.get('runtime') or clip.get('faded')),
                     'upload_date': unified_strdate(clip.get('posted')),
                     'uploader_id': uploader_id,

From c1ff6e1ad08c781ce1d486ddb7389fe90c79af35 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 24 Jun 2016 16:48:37 +0800
Subject: [PATCH 5295/6219] [vimeo:review] Fix extraction for
 password-protected videos

Closes #9853
---
 youtube_dl/extractor/vimeo.py | 72 ++++++++++++++++++++++++-----------
 1 file changed, 50 insertions(+), 22 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 4bdeb1187..d9c9852d4 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -16,6 +16,7 @@ from ..utils import (
     ExtractorError,
     InAdvancePagedList,
     int_or_none,
+    NO_DEFAULT,
     RegexNotFoundError,
     sanitized_Request,
     smuggle_url,
@@ -56,6 +57,26 @@ class VimeoBaseInfoExtractor(InfoExtractor):
         self._set_vimeo_cookie('vuid', vuid)
         self._download_webpage(login_request, None, False, 'Wrong login info')
 
+    def _verify_video_password(self, url, video_id, webpage):
+        password = self._downloader.params.get('videopassword')
+        if password is None:
+            raise ExtractorError('This video is protected by a password, use the --video-password option', expected=True)
+        token, vuid = self._extract_xsrft_and_vuid(webpage)
+        data = urlencode_postdata({
+            'password': password,
+            'token': token,
+        })
+        if url.startswith('http://'):
+            # vimeo only supports https now, but the user can give an http url
+            url = url.replace('http://', 'https://')
+        password_request = sanitized_Request(url + '/password', data)
+        password_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
+        password_request.add_header('Referer', url)
+        self._set_vimeo_cookie('vuid', vuid)
+        return self._download_webpage(
+            password_request, video_id,
+            'Verifying the password', 'Wrong password')
+
     def _extract_xsrft_and_vuid(self, webpage):
         xsrft = self._search_regex(
             r'(?:(?P<q1>["\'])xsrft(?P=q1)\s*:|xsrft\s*[=:])\s*(?P<q>["\'])(?P<xsrft>.+?)(?P=q)',
@@ -344,26 +365,6 @@ class VimeoIE(VimeoBaseInfoExtractor):
         if mobj:
             return mobj.group(1)
 
-    def _verify_video_password(self, url, video_id, webpage):
-        password = self._downloader.params.get('videopassword')
-        if password is None:
-            raise ExtractorError('This video is protected by a password, use the --video-password option', expected=True)
-        token, vuid = self._extract_xsrft_and_vuid(webpage)
-        data = urlencode_postdata({
-            'password': password,
-            'token': token,
-        })
-        if url.startswith('http://'):
-            # vimeo only supports https now, but the user can give an http url
-            url = url.replace('http://', 'https://')
-        password_request = sanitized_Request(url + '/password', data)
-        password_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
-        password_request.add_header('Referer', url)
-        self._set_vimeo_cookie('vuid', vuid)
-        return self._download_webpage(
-            password_request, video_id,
-            'Verifying the password', 'Wrong password')
-
     def _verify_player_video_password(self, url, video_id):
         password = self._downloader.params.get('videopassword')
         if password is None:
@@ -791,12 +792,39 @@ class VimeoReviewIE(VimeoBaseInfoExtractor):
             'thumbnail': 're:^https?://.*\.jpg$',
             'uploader_id': 'user22258446',
         }
+    }, {
+        'note': 'Password protected',
+        'url': 'https://vimeo.com/user37284429/review/138823582/c4d865efde',
+        'info_dict': {
+            'id': '138823582',
+            'ext': 'mp4',
+            'title': 'EFFICIENT PICKUP MASTERCLASS MODULE 1',
+            'uploader': 'TMB',
+            'uploader_id': 'user37284429',
+        },
+        'params': {
+            'videopassword': 'holygrail',
+        },
     }]
 
+    def _real_initialize(self):
+        self._login()
+
+    def _get_config_url(self, webpage_url, video_id, video_password_verified=False):
+        webpage = self._download_webpage(webpage_url, video_id)
+        config_url = self._html_search_regex(
+            r'data-config-url="([^"]+)"', webpage, 'config URL',
+            default=NO_DEFAULT if video_password_verified else None)
+        if config_url is None:
+            self._verify_video_password(webpage_url, video_id, webpage)
+            config_url = self._get_config_url(
+                webpage_url, video_id, video_password_verified=True)
+        return config_url
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        config = self._download_json(
-            'https://player.vimeo.com/video/%s/config' % video_id, video_id)
+        config_url = self._get_config_url(url, video_id)
+        config = self._download_json(config_url, video_id)
         info_dict = self._parse_config(config, video_id)
         self._vimeo_sort_formats(info_dict['formats'])
         info_dict['id'] = video_id

From 896cc727508f1d1054d88405c64e731c4d5c4ce4 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 24 Jun 2016 17:26:12 +0800
Subject: [PATCH 5296/6219] [mixcloud] View count and like count may be absent

Closes #9874
---
 youtube_dl/extractor/mixcloud.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dl/extractor/mixcloud.py
index 483f6925f..560fe188b 100644
--- a/youtube_dl/extractor/mixcloud.py
+++ b/youtube_dl/extractor/mixcloud.py
@@ -102,11 +102,11 @@ class MixcloudIE(InfoExtractor):
         description = self._og_search_description(webpage)
         like_count = parse_count(self._search_regex(
             r'\bbutton-favorite[^>]+>.*?<span[^>]+class=["\']toggle-number[^>]+>\s*([^<]+)',
-            webpage, 'like count', fatal=False))
+            webpage, 'like count', default=None))
         view_count = str_to_int(self._search_regex(
             [r'<meta itemprop="interactionCount" content="UserPlays:([0-9]+)"',
              r'/listeners/?">([0-9,.]+)</a>'],
-            webpage, 'play count', fatal=False))
+            webpage, 'play count', default=None))
 
         return {
             'id': track_id,

From de3c7fe0d42fd6027b99ab87ee5b4a4b4054daf6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 24 Jun 2016 22:27:55 +0700
Subject: [PATCH 5297/6219] [youtube] Fix 141 format tests

---
 youtube_dl/extractor/youtube.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 00dd602ff..54c6e45f8 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -501,6 +501,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'youtube_include_dash_manifest': True,
                 'format': '141',
             },
+            'skip': 'format 141 not served anymore',
         },
         # DASH manifest with encrypted signature
         {
@@ -517,7 +518,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             },
             'params': {
                 'youtube_include_dash_manifest': True,
-                'format': '141',
+                'format': '141/bestaudio[ext=m4a]',
             },
         },
         # JS player signature function name containing $
@@ -537,7 +538,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             },
             'params': {
                 'youtube_include_dash_manifest': True,
-                'format': '141',
+                'format': '141/bestaudio[ext=m4a]',
             },
         },
         # Controversy video

From 525cedb971c091793da5a0ef90670e6b5faded62 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 24 Jun 2016 22:37:13 +0700
Subject: [PATCH 5298/6219] [youtube] Relax URL expansion in description

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 54c6e45f8..46b9dc66c 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1332,7 +1332,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     (?:[a-zA-Z-]+="[^"]*"\s+)*?
                     (?:title|href)="([^"]+)"\s+
                     (?:[a-zA-Z-]+="[^"]*"\s+)*?
-                    class="(?:yt-uix-redirect-link|yt-uix-sessionlink[^"]*)"[^>]*>
+                    class="[^"]*"[^>]*>
                 [^<]+\.{3}\s*
                 </a>
             ''', r'\1', video_description)

From be49068d65ae39bef5797071f8a7cf1c733f033b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 24 Jun 2016 22:47:19 +0700
Subject: [PATCH 5299/6219] [youtube] Fix and skip some tests

---
 youtube_dl/extractor/youtube.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 46b9dc66c..c8d54f22a 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -619,7 +619,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader_url': 're:https?://(?:www\.)?youtube\.com/user/olympic',
                 'license': 'Standard YouTube License',
                 'description': 'HO09  - Women -  GER-AUS - Hockey - 31 July 2012 - London 2012 Olympic Games',
-                'uploader': 'Olympics',
+                'uploader': 'Olympic',
                 'title': 'Hockey - Women -  GER-AUS - London 2012 Olympic Games',
             },
             'params': {
@@ -672,7 +672,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader_url': 're:https?://(?:www\.)?youtube\.com/user/dorappi2000',
                 'uploader': 'dorappi2000',
                 'license': 'Standard YouTube License',
-                'formats': 'mincount:33',
+                'formats': 'mincount:32',
             },
         },
         # DASH manifest with segment_list
@@ -692,7 +692,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             'params': {
                 'youtube_include_dash_manifest': True,
                 'format': '135',  # bestvideo
-            }
+            },
+            'skip': 'This live event has ended.',
         },
         {
             # Multifeed videos (multiple cameras), URL is for Main Camera
@@ -763,6 +764,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'title': 'DevConf.cz 2016 Day 2 Workshops 1 14:00 - 15:30',
             },
             'playlist_count': 2,
+            'skip': 'Not multifeed anymore',
         },
         {
             'url': 'http://vid.plus/FlRa-iH7PGw',
@@ -815,6 +817,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             'params': {
                 'skip_download': True,
             },
+            'skip': 'This video does not exist.',
         },
         {
             # Video licensed under Creative Commons

From 3d4b08dfc7a1cf49686b68d405053475a4c3c490 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 25 Jun 2016 02:50:12 +0700
Subject: [PATCH 5300/6219] [setup.py] Add file version information and quotes
 consistency (Closes #9878)

---
 setup.py | 63 +++++++++++++++++++++++++++++++-------------------------
 1 file changed, 35 insertions(+), 28 deletions(-)

diff --git a/setup.py b/setup.py
index c1e923f71..508b27f37 100644
--- a/setup.py
+++ b/setup.py
@@ -21,25 +21,37 @@ try:
     import py2exe
 except ImportError:
     if len(sys.argv) >= 2 and sys.argv[1] == 'py2exe':
-        print("Cannot import py2exe", file=sys.stderr)
+        print('Cannot import py2exe', file=sys.stderr)
         exit(1)
 
 py2exe_options = {
-    "bundle_files": 1,
-    "compressed": 1,
-    "optimize": 2,
-    "dist_dir": '.',
-    "dll_excludes": ['w9xpopen.exe', 'crypt32.dll'],
+    'bundle_files': 1,
+    'compressed': 1,
+    'optimize': 2,
+    'dist_dir': '.',
+    'dll_excludes': ['w9xpopen.exe', 'crypt32.dll'],
 }
 
+# Get the version from youtube_dl/version.py without importing the package
+exec(compile(open('youtube_dl/version.py').read(),
+             'youtube_dl/version.py', 'exec'))
+
+DESCRIPTION = 'YouTube video downloader'
+LONG_DESCRIPTION = 'Command-line program to download videos from YouTube.com and other video sites'
+
 py2exe_console = [{
-    "script": "./youtube_dl/__main__.py",
-    "dest_base": "youtube-dl",
+    'script': './youtube_dl/__main__.py',
+    'dest_base': 'youtube-dl',
+    'version': __version__,
+    'description': DESCRIPTION,
+    'comments': LONG_DESCRIPTION,
+    'product_name': 'youtube-dl',
+    'product_version': __version__,
 }]
 
 py2exe_params = {
     'console': py2exe_console,
-    'options': {"py2exe": py2exe_options},
+    'options': {'py2exe': py2exe_options},
     'zipfile': None
 }
 
@@ -72,7 +84,7 @@ else:
         params['scripts'] = ['bin/youtube-dl']
 
 class build_lazy_extractors(Command):
-    description = "Build the extractor lazy loading module"
+    description = 'Build the extractor lazy loading module'
     user_options = []
 
     def initialize_options(self):
@@ -87,16 +99,11 @@ class build_lazy_extractors(Command):
             dry_run=self.dry_run,
         )
 
-# Get the version from youtube_dl/version.py without importing the package
-exec(compile(open('youtube_dl/version.py').read(),
-             'youtube_dl/version.py', 'exec'))
-
 setup(
     name='youtube_dl',
     version=__version__,
-    description='YouTube video downloader',
-    long_description='Small command-line program to download videos from'
-    ' YouTube.com and other video sites.',
+    description=DESCRIPTION,
+    long_description=LONG_DESCRIPTION,
     url='https://github.com/rg3/youtube-dl',
     author='Ricardo Garcia',
     author_email='ytdl@yt-dl.org',
@@ -112,17 +119,17 @@ setup(
     # test_requires = ['nosetest'],
 
     classifiers=[
-        "Topic :: Multimedia :: Video",
-        "Development Status :: 5 - Production/Stable",
-        "Environment :: Console",
-        "License :: Public Domain",
-        "Programming Language :: Python :: 2.6",
-        "Programming Language :: Python :: 2.7",
-        "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.2",
-        "Programming Language :: Python :: 3.3",
-        "Programming Language :: Python :: 3.4",
-        "Programming Language :: Python :: 3.5",
+        'Topic :: Multimedia :: Video',
+        'Development Status :: 5 - Production/Stable',
+        'Environment :: Console',
+        'License :: Public Domain',
+        'Programming Language :: Python :: 2.6',
+        'Programming Language :: Python :: 2.7',
+        'Programming Language :: Python :: 3',
+        'Programming Language :: Python :: 3.2',
+        'Programming Language :: Python :: 3.3',
+        'Programming Language :: Python :: 3.4',
+        'Programming Language :: Python :: 3.5',
     ],
 
     cmdclass={'build_lazy_extractors': build_lazy_extractors},

From b4241e308e9b2d38d564833cb6c43c9fcc0fd280 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 25 Jun 2016 03:03:20 +0700
Subject: [PATCH 5301/6219] release 2016.06.25

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 youtube_dl/version.py     | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 62cb18d7d..c73f9a904 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.06.23.1*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.06.23.1**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.06.25*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.06.25**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.06.23.1
+[debug] youtube-dl version 2016.06.25
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 0238dc97c..2b7a4c98d 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.06.23.1'
+__version__ = '2016.06.25'

From 46f59e89ea1e75bf2bd1657f0863a3e5e81f91ea Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 25 Jun 2016 22:30:35 +0700
Subject: [PATCH 5302/6219] [utils] Add unified_timestamp

---
 test/test_utils.py  |  21 +++++++
 youtube_dl/utils.py | 150 +++++++++++++++++++++++++++-----------------
 2 files changed, 113 insertions(+), 58 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index b7ef51f8d..7f9385deb 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -60,6 +60,7 @@ from youtube_dl.utils import (
     timeconvert,
     unescapeHTML,
     unified_strdate,
+    unified_timestamp,
     unsmuggle_url,
     uppercase_escape,
     lowercase_escape,
@@ -283,8 +284,28 @@ class TestUtil(unittest.TestCase):
             '20150202')
         self.assertEqual(unified_strdate('Feb 14th 2016 5:45PM'), '20160214')
         self.assertEqual(unified_strdate('25-09-2014'), '20140925')
+        self.assertEqual(unified_strdate('27.02.2016 17:30'), '20160227')
         self.assertEqual(unified_strdate('UNKNOWN DATE FORMAT'), None)
 
+    def test_unified_timestamps(self):
+        self.assertEqual(unified_timestamp('December 21, 2010'), 1292889600)
+        self.assertEqual(unified_timestamp('8/7/2009'), 1247011200)
+        self.assertEqual(unified_timestamp('Dec 14, 2012'), 1355443200)
+        self.assertEqual(unified_timestamp('2012/10/11 01:56:38 +0000'), 1349920598)
+        self.assertEqual(unified_timestamp('1968 12 10'), -33436800)
+        self.assertEqual(unified_timestamp('1968-12-10'), -33436800)
+        self.assertEqual(unified_timestamp('28/01/2014 21:00:00 +0100'), 1390939200)
+        self.assertEqual(
+            unified_timestamp('11/26/2014 11:30:00 AM PST', day_first=False),
+            1417001400)
+        self.assertEqual(
+            unified_timestamp('2/2/2015 6:47:40 PM', day_first=False),
+            1422902860)
+        self.assertEqual(unified_timestamp('Feb 14th 2016 5:45PM'), 1455471900)
+        self.assertEqual(unified_timestamp('25-09-2014'), 1411603200)
+        self.assertEqual(unified_timestamp('27.02.2016 17:30'), 1456594200)
+        self.assertEqual(unified_timestamp('UNKNOWN DATE FORMAT'), None)
+
     def test_determine_ext(self):
         self.assertEqual(determine_ext('http://example.com/foo/bar.mp4/?download'), 'mp4')
         self.assertEqual(determine_ext('http://example.com/foo/bar/?download', None), None)
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 562031fe1..de66cb482 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -110,6 +110,49 @@ ACCENT_CHARS = dict(zip('ÂÃÄÀÁÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖŐØŒÙ
                         itertools.chain('AAAAAA', ['AE'], 'CEEEEIIIIDNOOOOOOO', ['OE'], 'UUUUUYP', ['ss'],
                                         'aaaaaa', ['ae'], 'ceeeeiiiionooooooo', ['oe'], 'uuuuuypy')))
 
+DATE_FORMATS = (
+    '%d %B %Y',
+    '%d %b %Y',
+    '%B %d %Y',
+    '%b %d %Y',
+    '%b %dst %Y %I:%M',
+    '%b %dnd %Y %I:%M',
+    '%b %dth %Y %I:%M',
+    '%Y %m %d',
+    '%Y-%m-%d',
+    '%Y/%m/%d',
+    '%Y/%m/%d %H:%M:%S',
+    '%Y-%m-%d %H:%M:%S',
+    '%Y-%m-%d %H:%M:%S.%f',
+    '%d.%m.%Y %H:%M',
+    '%d.%m.%Y %H.%M',
+    '%Y-%m-%dT%H:%M:%SZ',
+    '%Y-%m-%dT%H:%M:%S.%fZ',
+    '%Y-%m-%dT%H:%M:%S.%f0Z',
+    '%Y-%m-%dT%H:%M:%S',
+    '%Y-%m-%dT%H:%M:%S.%f',
+    '%Y-%m-%dT%H:%M',
+)
+
+DATE_FORMATS_DAY_FIRST = list(DATE_FORMATS)
+DATE_FORMATS_DAY_FIRST.extend([
+    '%d-%m-%Y',
+    '%d.%m.%Y',
+    '%d.%m.%y',
+    '%d/%m/%Y',
+    '%d/%m/%y',
+    '%d/%m/%Y %H:%M:%S',
+])
+
+DATE_FORMATS_MONTH_FIRST = list(DATE_FORMATS)
+DATE_FORMATS_MONTH_FIRST.extend([
+    '%m-%d-%Y',
+    '%m.%d.%Y',
+    '%m/%d/%Y',
+    '%m/%d/%y',
+    '%m/%d/%Y %H:%M:%S',
+])
+
 
 def preferredencoding():
     """Get preferred encoding.
@@ -975,6 +1018,24 @@ class YoutubeDLCookieProcessor(compat_urllib_request.HTTPCookieProcessor):
     https_response = http_response
 
 
+def extract_timezone(date_str):
+    m = re.search(
+        r'^.{8,}?(?P<tz>Z$| ?(?P<sign>\+|-)(?P<hours>[0-9]{2}):?(?P<minutes>[0-9]{2})$)',
+        date_str)
+    if not m:
+        timezone = datetime.timedelta()
+    else:
+        date_str = date_str[:-len(m.group('tz'))]
+        if not m.group('sign'):
+            timezone = datetime.timedelta()
+        else:
+            sign = 1 if m.group('sign') == '+' else -1
+            timezone = datetime.timedelta(
+                hours=sign * int(m.group('hours')),
+                minutes=sign * int(m.group('minutes')))
+    return timezone, date_str
+
+
 def parse_iso8601(date_str, delimiter='T', timezone=None):
     """ Return a UNIX timestamp from the given date """
 
@@ -984,20 +1045,8 @@ def parse_iso8601(date_str, delimiter='T', timezone=None):
     date_str = re.sub(r'\.[0-9]+', '', date_str)
 
     if timezone is None:
-        m = re.search(
-            r'(?:Z$| ?(?P<sign>\+|-)(?P<hours>[0-9]{2}):?(?P<minutes>[0-9]{2})$)',
-            date_str)
-        if not m:
-            timezone = datetime.timedelta()
-        else:
-            date_str = date_str[:-len(m.group(0))]
-            if not m.group('sign'):
-                timezone = datetime.timedelta()
-            else:
-                sign = 1 if m.group('sign') == '+' else -1
-                timezone = datetime.timedelta(
-                    hours=sign * int(m.group('hours')),
-                    minutes=sign * int(m.group('minutes')))
+        timezone, date_str = extract_timezone(date_str)
+
     try:
         date_format = '%Y-%m-%d{0}%H:%M:%S'.format(delimiter)
         dt = datetime.datetime.strptime(date_str, date_format) - timezone
@@ -1006,6 +1055,10 @@ def parse_iso8601(date_str, delimiter='T', timezone=None):
         pass
 
 
+def date_formats(day_first=True):
+    return DATE_FORMATS_DAY_FIRST if day_first else DATE_FORMATS_MONTH_FIRST
+
+
 def unified_strdate(date_str, day_first=True):
     """Return a string with the date in the format YYYYMMDD"""
 
@@ -1014,53 +1067,11 @@ def unified_strdate(date_str, day_first=True):
     upload_date = None
     # Replace commas
     date_str = date_str.replace(',', ' ')
-    # %z (UTC offset) is only supported in python>=3.2
-    if not re.match(r'^[0-9]{1,2}-[0-9]{1,2}-[0-9]{4}$', date_str):
-        date_str = re.sub(r' ?(\+|-)[0-9]{2}:?[0-9]{2}$', '', date_str)
     # Remove AM/PM + timezone
     date_str = re.sub(r'(?i)\s*(?:AM|PM)(?:\s+[A-Z]+)?', '', date_str)
+    _, date_str = extract_timezone(date_str)
 
-    format_expressions = [
-        '%d %B %Y',
-        '%d %b %Y',
-        '%B %d %Y',
-        '%b %d %Y',
-        '%b %dst %Y %I:%M',
-        '%b %dnd %Y %I:%M',
-        '%b %dth %Y %I:%M',
-        '%Y %m %d',
-        '%Y-%m-%d',
-        '%Y/%m/%d',
-        '%Y/%m/%d %H:%M:%S',
-        '%Y-%m-%d %H:%M:%S',
-        '%Y-%m-%d %H:%M:%S.%f',
-        '%d.%m.%Y %H:%M',
-        '%d.%m.%Y %H.%M',
-        '%Y-%m-%dT%H:%M:%SZ',
-        '%Y-%m-%dT%H:%M:%S.%fZ',
-        '%Y-%m-%dT%H:%M:%S.%f0Z',
-        '%Y-%m-%dT%H:%M:%S',
-        '%Y-%m-%dT%H:%M:%S.%f',
-        '%Y-%m-%dT%H:%M',
-    ]
-    if day_first:
-        format_expressions.extend([
-            '%d-%m-%Y',
-            '%d.%m.%Y',
-            '%d.%m.%y',
-            '%d/%m/%Y',
-            '%d/%m/%y',
-            '%d/%m/%Y %H:%M:%S',
-        ])
-    else:
-        format_expressions.extend([
-            '%m-%d-%Y',
-            '%m.%d.%Y',
-            '%m/%d/%Y',
-            '%m/%d/%y',
-            '%m/%d/%Y %H:%M:%S',
-        ])
-    for expression in format_expressions:
+    for expression in date_formats(day_first):
         try:
             upload_date = datetime.datetime.strptime(date_str, expression).strftime('%Y%m%d')
         except ValueError:
@@ -1076,6 +1087,29 @@ def unified_strdate(date_str, day_first=True):
         return compat_str(upload_date)
 
 
+def unified_timestamp(date_str, day_first=True):
+    if date_str is None:
+        return None
+
+    date_str = date_str.replace(',', ' ')
+
+    pm_delta = datetime.timedelta(hours=12 if re.search(r'(?i)PM', date_str) else 0)
+    timezone, date_str = extract_timezone(date_str)
+
+    # Remove AM/PM + timezone
+    date_str = re.sub(r'(?i)\s*(?:AM|PM)(?:\s+[A-Z]+)?', '', date_str)
+
+    for expression in date_formats(day_first):
+        try:
+            dt = datetime.datetime.strptime(date_str, expression) - timezone + pm_delta
+            return calendar.timegm(dt.timetuple())
+        except ValueError:
+            pass
+    timetuple = email.utils.parsedate_tz(date_str)
+    if timetuple:
+        return calendar.timegm(timetuple.timetuple())
+
+
 def determine_ext(url, default_ext='unknown_video'):
     if url is None:
         return default_ext

From b72b44318c0de52befe003c14297cae9f7a283c3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 25 Jun 2016 22:32:02 +0700
Subject: [PATCH 5303/6219] [utils] Add strip_or_none

---
 youtube_dl/utils.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index de66cb482..a375282f2 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1660,6 +1660,10 @@ def float_or_none(v, scale=1, invscale=1, default=None):
         return default
 
 
+def strip_or_none(v):
+    return None if v is None else v.strip()
+
+
 def parse_duration(s):
     if not isinstance(s, compat_basestring):
         return None

From 2d185706ea0236a30bd1037a3ab97fbe5fe575a5 Mon Sep 17 00:00:00 2001
From: Jakub Adam Wieczorek <ja.wieczorek@student.uw.edu.pl>
Date: Thu, 16 Jun 2016 21:00:27 +0200
Subject: [PATCH 5304/6219] [polskieradio] Add support for Polskie Radio.

Polskie Radio is the main Polish state-funded radio broadcasting service.
---
 youtube_dl/extractor/extractors.py   |  1 +
 youtube_dl/extractor/polskieradio.py | 74 ++++++++++++++++++++++++++++
 2 files changed, 75 insertions(+)
 create mode 100644 youtube_dl/extractor/polskieradio.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 6fc5a18f5..9f98a1490 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -606,6 +606,7 @@ from .pluralsight import (
     PluralsightCourseIE,
 )
 from .podomatic import PodomaticIE
+from .polskieradio import PolskieRadioIE
 from .porn91 import Porn91IE
 from .pornhd import PornHdIE
 from .pornhub import (
diff --git a/youtube_dl/extractor/polskieradio.py b/youtube_dl/extractor/polskieradio.py
new file mode 100644
index 000000000..5d4b116d8
--- /dev/null
+++ b/youtube_dl/extractor/polskieradio.py
@@ -0,0 +1,74 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import int_or_none
+
+import calendar
+from datetime import datetime
+
+
+class PolskieRadioIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?polskieradio\.pl/[0-9]+/[0-9]+/Artykul/(?P<id>[0-9]+),.+'
+    _TESTS = [{
+        'url': 'http://www.polskieradio.pl/7/5102/Artykul/1587943,Prof-Andrzej-Nowak-o-historii-nie-da-sie-myslec-beznamietnie',
+        'md5': '2984ee6ce9046d91fc233bc1a864a09a',
+        'info_dict': {
+            'id': '1587943',
+            'ext': 'mp3',
+            'title': 'Prof. Andrzej Nowak: o historii nie da się myśleć beznamiętnie',
+            'description': 'md5:12f954edbf3120c5e7075e17bf9fc5c5',
+            'release_date': '20160227',
+            'upload_date': '20160227',
+            'timestamp': 1456594200,
+            'duration': 2364
+        }
+    }, {
+        'url': 'http://polskieradio.pl/9/305/Artykul/1632955,Bardzo-popularne-slowo-remis',
+        'md5': '68a393e25b942c1a76872f56d303a31a',
+        'info_dict': {
+            'id': '1632955',
+            'ext': 'mp3',
+            'title': 'Bardzo popularne słowo: remis',
+            'description': 'md5:3b58dfae614100abc0f175a0b26d5680',
+            'release_date': '20160617',
+            'upload_date': '20160617',
+            'timestamp': 1466184900,
+            'duration': 393
+        }
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        metadata_string = self._html_search_regex(r'<span class="play pr-media-play" data-media=(\{.+\})>', webpage, 'metadata')
+        metadata = self._parse_json(metadata_string, video_id)
+
+        title = self._og_search_title(webpage)
+        if title is not None:
+            title = title.strip()
+
+        description = self._og_search_description(webpage)
+        if description is not None:
+            description = description.strip()
+
+        release_date = self._html_search_regex(r'Data emisji:[^0-9]+([0-9]{1,2}\.[0-9]{2}\.[0-9]{4})', webpage, 'release date', fatal=False)
+        if release_date is not None:
+            release_date = datetime.strptime(release_date, '%d.%m.%Y').strftime('%Y%m%d')
+
+        upload_datetime = self._html_search_regex(r'<span id="datetime2" class="time">\s+(.+)\s+</span>', webpage, 'release time', fatal=False)
+        if upload_datetime is not None:
+            timestamp = calendar.timegm(datetime.strptime(upload_datetime, '%d.%m.%Y %H:%M').timetuple())
+        else:
+            timestamp = None
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'display_id': metadata.get('id'),
+            'duration': int_or_none(metadata.get('length')),
+            'url': self._proto_relative_url(metadata.get('file'), 'http:'),
+            'release_date': release_date,
+            'timestamp': timestamp
+        }

From 0463b77a1f83f3f9239c6c5f5d1ca251afd267e4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 25 Jun 2016 23:18:40 +0700
Subject: [PATCH 5305/6219] [polskieradio] Improve extraction (Closes #9813)

---
 youtube_dl/extractor/polskieradio.py | 119 +++++++++++++++------------
 1 file changed, 68 insertions(+), 51 deletions(-)

diff --git a/youtube_dl/extractor/polskieradio.py b/youtube_dl/extractor/polskieradio.py
index 5d4b116d8..f5adff08f 100644
--- a/youtube_dl/extractor/polskieradio.py
+++ b/youtube_dl/extractor/polskieradio.py
@@ -1,74 +1,91 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-from .common import InfoExtractor
-from ..utils import int_or_none
+import re
 
-import calendar
-from datetime import datetime
+from .common import InfoExtractor
+from ..compat import (
+    compat_str,
+    compat_urllib_parse_unquote,
+)
+from ..utils import (
+    int_or_none,
+    strip_or_none,
+    unified_timestamp,
+)
 
 
 class PolskieRadioIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?polskieradio\.pl/[0-9]+/[0-9]+/Artykul/(?P<id>[0-9]+),.+'
+    _VALID_URL = r'https?://(?:www\.)?polskieradio\.pl/\d+/\d+/Artykul/(?P<id>[0-9]+)'
     _TESTS = [{
         'url': 'http://www.polskieradio.pl/7/5102/Artykul/1587943,Prof-Andrzej-Nowak-o-historii-nie-da-sie-myslec-beznamietnie',
-        'md5': '2984ee6ce9046d91fc233bc1a864a09a',
         'info_dict': {
             'id': '1587943',
-            'ext': 'mp3',
             'title': 'Prof. Andrzej Nowak: o historii nie da się myśleć beznamiętnie',
             'description': 'md5:12f954edbf3120c5e7075e17bf9fc5c5',
-            'release_date': '20160227',
-            'upload_date': '20160227',
-            'timestamp': 1456594200,
-            'duration': 2364
-        }
+        },
+        'playlist': [{
+            'md5': '2984ee6ce9046d91fc233bc1a864a09a',
+            'info_dict': {
+                'id': '1540576',
+                'ext': 'mp3',
+                'title': 'md5:d4623290d4ac983bf924061c75c23a0d',
+                'timestamp': 1456594200,
+                'upload_date': '20160227',
+                'duration': 2364,
+            },
+        }],
+    }, {
+        'url': 'http://www.polskieradio.pl/265/5217/Artykul/1635803,Euro-2016-nie-ma-miejsca-na-blad-Polacy-graja-ze-Szwajcaria-o-cwiercfinal',
+        'info_dict': {
+            'id': '1635803',
+            'title': 'Euro 2016: nie ma miejsca na błąd. Polacy grają ze Szwajcarią o ćwierćfinał',
+            'description': 'md5:01cb7d0cad58664095d72b51a1ebada2',
+        },
+        'playlist_mincount': 12,
     }, {
         'url': 'http://polskieradio.pl/9/305/Artykul/1632955,Bardzo-popularne-slowo-remis',
-        'md5': '68a393e25b942c1a76872f56d303a31a',
-        'info_dict': {
-            'id': '1632955',
-            'ext': 'mp3',
-            'title': 'Bardzo popularne słowo: remis',
-            'description': 'md5:3b58dfae614100abc0f175a0b26d5680',
-            'release_date': '20160617',
-            'upload_date': '20160617',
-            'timestamp': 1466184900,
-            'duration': 393
-        }
+        'only_matching': True,
+    }, {
+        'url': 'http://www.polskieradio.pl/7/5102/Artykul/1587943',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-        metadata_string = self._html_search_regex(r'<span class="play pr-media-play" data-media=(\{.+\})>', webpage, 'metadata')
-        metadata = self._parse_json(metadata_string, video_id)
+        playlist_id = self._match_id(url)
 
-        title = self._og_search_title(webpage)
-        if title is not None:
-            title = title.strip()
+        webpage = self._download_webpage(url, playlist_id)
 
-        description = self._og_search_description(webpage)
-        if description is not None:
-            description = description.strip()
+        content = self._search_regex(
+            r'(?s)<div[^>]+class="audio atarticle"[^>]*>(.+?)<script>',
+            webpage, 'content')
 
-        release_date = self._html_search_regex(r'Data emisji:[^0-9]+([0-9]{1,2}\.[0-9]{2}\.[0-9]{4})', webpage, 'release date', fatal=False)
-        if release_date is not None:
-            release_date = datetime.strptime(release_date, '%d.%m.%Y').strftime('%Y%m%d')
+        timestamp = unified_timestamp(self._html_search_regex(
+            r'(?s)<span[^>]+id="datetime2"[^>]*>(.+?)</span>',
+            webpage, 'timestamp', fatal=False))
 
-        upload_datetime = self._html_search_regex(r'<span id="datetime2" class="time">\s+(.+)\s+</span>', webpage, 'release time', fatal=False)
-        if upload_datetime is not None:
-            timestamp = calendar.timegm(datetime.strptime(upload_datetime, '%d.%m.%Y %H:%M').timetuple())
-        else:
-            timestamp = None
+        entries = []
 
-        return {
-            'id': video_id,
-            'title': title,
-            'description': description,
-            'display_id': metadata.get('id'),
-            'duration': int_or_none(metadata.get('length')),
-            'url': self._proto_relative_url(metadata.get('file'), 'http:'),
-            'release_date': release_date,
-            'timestamp': timestamp
-        }
+        media_urls = set()
+
+        for data_media in re.findall(r'<[^>]+data-media=({[^>]+})', content):
+            media = self._parse_json(data_media, playlist_id, fatal=False)
+            if not media.get('file') or not media.get('desc'):
+                continue
+            media_url = self._proto_relative_url(media['file'], 'http:')
+            if media_url in media_urls:
+                continue
+            media_urls.add(media_url)
+            entries.append({
+                'id': compat_str(media['id']),
+                'url': media_url,
+                'title': compat_urllib_parse_unquote(media['desc']),
+                'duration': int_or_none(media.get('length')),
+                'vcodec': 'none' if media.get('provider') == 'audio' else None,
+                'timestamp': timestamp,
+            })
+
+        title = self._og_search_title(webpage).strip()
+        description = strip_or_none(self._og_search_description(webpage))
+
+        return self.playlist_result(entries, playlist_id, title, description)

From ce96ed05f42d42f8a506a2a527c776054c44ad1e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 25 Jun 2016 23:31:21 +0700
Subject: [PATCH 5306/6219] [polskieradio] Add test with video

---
 youtube_dl/extractor/polskieradio.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/polskieradio.py b/youtube_dl/extractor/polskieradio.py
index f5adff08f..d3bebaea3 100644
--- a/youtube_dl/extractor/polskieradio.py
+++ b/youtube_dl/extractor/polskieradio.py
@@ -49,6 +49,10 @@ class PolskieRadioIE(InfoExtractor):
     }, {
         'url': 'http://www.polskieradio.pl/7/5102/Artykul/1587943',
         'only_matching': True,
+    }, {
+        # with mp4 video
+        'url': 'http://www.polskieradio.pl/9/299/Artykul/1634903,Brexit-Leszek-Miller-swiat-sie-nie-zawali-Europa-bedzie-trwac-dalej',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 0c00e889f3616cecd4de161681924c4cb12ce320 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 25 Jun 2016 23:35:57 +0700
Subject: [PATCH 5307/6219] Credit @JakubAdamWieczorek for #9813

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index cdf655c39..bdd29687d 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -175,3 +175,4 @@ Tomáš Čech
 Déstin Reed
 Roman Tsiupa
 Artur Krysiak
+Jakub Adam Wieczorek

From ac782306f18430479e881ffd1ac749baff4dd9aa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 26 Jun 2016 00:25:41 +0700
Subject: [PATCH 5308/6219] [iqiyi] Mark broken

---
 youtube_dl/extractor/iqiyi.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index ddcb3c916..5dd15e26f 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -165,6 +165,8 @@ class IqiyiIE(InfoExtractor):
     IE_NAME = 'iqiyi'
     IE_DESC = '爱奇艺'
 
+    _WORKING = False
+
     _VALID_URL = r'https?://(?:(?:[^.]+\.)?iqiyi\.com|www\.pps\.tv)/.+\.html'
 
     _NETRC_MACHINE = 'iqiyi'

From 3b34ab538c5bc67be18df8376c25f433ea1ff92b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 26 Jun 2016 00:29:53 +0700
Subject: [PATCH 5309/6219] [svtplay] Extend _VALID_URL (#9900)

---
 youtube_dl/extractor/svt.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/svt.py b/youtube_dl/extractor/svt.py
index 67f56fab8..1c04dfb7b 100644
--- a/youtube_dl/extractor/svt.py
+++ b/youtube_dl/extractor/svt.py
@@ -120,7 +120,7 @@ class SVTIE(SVTBaseIE):
 
 class SVTPlayIE(SVTBaseIE):
     IE_DESC = 'SVT Play and Öppet arkiv'
-    _VALID_URL = r'https?://(?:www\.)?(?:svtplay|oppetarkiv)\.se/video/(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?(?:svtplay|oppetarkiv)\.se/(?:video|klipp)/(?P<id>[0-9]+)'
     _TESTS = [{
         'url': 'http://www.svtplay.se/video/5996901/flygplan-till-haile-selassie/flygplan-till-haile-selassie-2',
         'md5': '2b6704fe4a28801e1a098bbf3c5ac611',
@@ -141,6 +141,9 @@ class SVTPlayIE(SVTBaseIE):
         # geo restricted to Sweden
         'url': 'http://www.oppetarkiv.se/video/5219710/trollflojten',
         'only_matching': True,
+    }, {
+        'url': 'http://www.svtplay.se/klipp/9023742/stopptid-om-bjorn-borg',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From a2406fce3c657af116f95c5e9b965315aa23cd95 Mon Sep 17 00:00:00 2001
From: stepshal <nessento@openmailbox.org>
Date: Sun, 26 Jun 2016 01:23:48 +0700
Subject: [PATCH 5310/6219] Fix misspelling

---
 youtube_dl/socks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/socks.py b/youtube_dl/socks.py
index fd49d7435..104807242 100644
--- a/youtube_dl/socks.py
+++ b/youtube_dl/socks.py
@@ -76,7 +76,7 @@ class Socks4Error(ProxyError):
 
     CODES = {
         91: 'request rejected or failed',
-        92: 'request rejected becasue SOCKS server cannot connect to identd on the client',
+        92: 'request rejected because SOCKS server cannot connect to identd on the client',
         93: 'request rejected because the client program and identd report different user-ids'
     }
 

From 7d52c052efe7accf098bca84aef0ea70caa64889 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 26 Jun 2016 11:54:52 +0800
Subject: [PATCH 5311/6219] [generic] Fix test_Generic_76

Broken: https://travis-ci.org/rg3/youtube-dl/jobs/140251658
---
 youtube_dl/extractor/generic.py | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 4aa24061c..1592a8a3a 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1091,12 +1091,17 @@ class GenericIE(InfoExtractor):
         # Dailymotion Cloud video
         {
             'url': 'http://replay.publicsenat.fr/vod/le-debat/florent-kolandjian,dominique-cena,axel-decourtye,laurence-abeille,bruno-parmentier/175910',
-            'md5': '49444254273501a64675a7e68c502681',
+            'md5': 'dcaf23ad0c67a256f4278bce6e0bae38',
             'info_dict': {
-                'id': '5585de919473990de4bee11b',
+                'id': 'x2uy8t3',
                 'ext': 'mp4',
-                'title': 'Le débat',
+                'title': 'Sauvons les abeilles ! - Le débat',
+                'description': 'md5:d9082128b1c5277987825d684939ca26',
                 'thumbnail': 're:^https?://.*\.jpe?g$',
+                'timestamp': 1434970506,
+                'upload_date': '20150622',
+                'uploader': 'Public Sénat',
+                'uploader_id': 'xa9gza',
             }
         },
         # OnionStudios embed

From 1143535d762fc4260aacc108f2c41079867f9f00 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 26 Jun 2016 15:16:49 +0800
Subject: [PATCH 5312/6219] [utils] Add urshift()

Used in IqiyiIE and LeIE
---
 test/test_utils.py  | 5 +++++
 youtube_dl/utils.py | 4 ++++
 2 files changed, 9 insertions(+)

diff --git a/test/test_utils.py b/test/test_utils.py
index 7f9385deb..ed61e4c27 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -66,6 +66,7 @@ from youtube_dl.utils import (
     lowercase_escape,
     url_basename,
     urlencode_postdata,
+    urshift,
     update_url_query,
     version_tuple,
     xpath_with_ns,
@@ -980,5 +981,9 @@ The first line
         self.assertRaises(ValueError, encode_base_n, 0, 70)
         self.assertRaises(ValueError, encode_base_n, 0, 60, custom_table)
 
+    def test_urshift(self):
+        self.assertEqual(urshift(3, 1), 1)
+        self.assertEqual(urshift(-3, 1), 2147483646)
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index a375282f2..a2cfb48a6 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2899,3 +2899,7 @@ def parse_m3u8_attributes(attrib):
             val = val[1:-1]
         info[key] = val
     return info
+
+
+def urshift(val, n):
+    return val >> n if val >= 0 else (val + 0x100000000) >> n

From 30105f4ac0291bd3e1350a5bb383e88e260b9ad9 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 26 Jun 2016 15:17:26 +0800
Subject: [PATCH 5313/6219] [le] Move urshift() to utils.py

---
 youtube_dl/extractor/leeco.py | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/leeco.py b/youtube_dl/extractor/leeco.py
index 63f581cd9..959d71617 100644
--- a/youtube_dl/extractor/leeco.py
+++ b/youtube_dl/extractor/leeco.py
@@ -23,6 +23,7 @@ from ..utils import (
     sanitized_Request,
     str_or_none,
     url_basename,
+    urshift,
 )
 
 
@@ -74,15 +75,11 @@ class LeIE(InfoExtractor):
         'only_matching': True,
     }]
 
-    @staticmethod
-    def urshift(val, n):
-        return val >> n if val >= 0 else (val + 0x100000000) >> n
-
     # ror() and calc_time_key() are reversed from a embedded swf file in KLetvPlayer.swf
     def ror(self, param1, param2):
         _loc3_ = 0
         while _loc3_ < param2:
-            param1 = self.urshift(param1, 1) + ((param1 & 1) << 31)
+            param1 = urshift(param1, 1) + ((param1 & 1) << 31)
             _loc3_ += 1
         return param1
 

From 5b6ad8630c4947f3695513c9707406b2d12ae7b8 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 26 Jun 2016 15:18:32 +0800
Subject: [PATCH 5314/6219] [iqiyi] Partially fix IqiyiIE

Use the HTML5 API. Only low-resolution formats available

Related: #9839

Thanks @zhangn1985 for the overall algorithm (soimort/you-get#1224)
---
 youtube_dl/extractor/iqiyi.py | 404 ++++++++++------------------------
 1 file changed, 118 insertions(+), 286 deletions(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index 5dd15e26f..b717ca09c 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -1,30 +1,25 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import binascii
 import hashlib
 import itertools
 import math
-import os
-import random
 import re
 import time
-import uuid
 
 from .common import InfoExtractor
 from ..compat import (
-    compat_parse_qs,
     compat_str,
     compat_urllib_parse_urlencode,
-    compat_urllib_parse_urlparse,
 )
 from ..utils import (
     decode_packed_codes,
     ExtractorError,
+    intlist_to_bytes,
     ohdave_rsa_encrypt,
     remove_start,
-    sanitized_Request,
-    urlencode_postdata,
-    url_basename,
+    urshift,
 )
 
 
@@ -165,77 +160,28 @@ class IqiyiIE(InfoExtractor):
     IE_NAME = 'iqiyi'
     IE_DESC = '爱奇艺'
 
-    _WORKING = False
-
     _VALID_URL = r'https?://(?:(?:[^.]+\.)?iqiyi\.com|www\.pps\.tv)/.+\.html'
 
     _NETRC_MACHINE = 'iqiyi'
 
     _TESTS = [{
         'url': 'http://www.iqiyi.com/v_19rrojlavg.html',
-        'md5': '2cb594dc2781e6c941a110d8f358118b',
+        'md5': '470a6c160618577166db1a7aac5a3606',
         'info_dict': {
             'id': '9c1fb1b99d192b21c559e5a1a2cb3c73',
+            'ext': 'mp4',
             'title': '美国德州空中惊现奇异云团 酷似UFO',
-            'ext': 'f4v',
         }
     }, {
         'url': 'http://www.iqiyi.com/v_19rrhnnclk.html',
+        'md5': 'f09f0a6a59b2da66a26bf4eda669a4cc',
         'info_dict': {
             'id': 'e3f585b550a280af23c98b6cb2be19fb',
-            'title': '名侦探柯南第752集',
+            'ext': 'mp4',
+            'title': '名侦探柯南 国语版',
         },
-        'playlist': [{
-            'info_dict': {
-                'id': 'e3f585b550a280af23c98b6cb2be19fb_part1',
-                'ext': 'f4v',
-                'title': '名侦探柯南第752集',
-            },
-        }, {
-            'info_dict': {
-                'id': 'e3f585b550a280af23c98b6cb2be19fb_part2',
-                'ext': 'f4v',
-                'title': '名侦探柯南第752集',
-            },
-        }, {
-            'info_dict': {
-                'id': 'e3f585b550a280af23c98b6cb2be19fb_part3',
-                'ext': 'f4v',
-                'title': '名侦探柯南第752集',
-            },
-        }, {
-            'info_dict': {
-                'id': 'e3f585b550a280af23c98b6cb2be19fb_part4',
-                'ext': 'f4v',
-                'title': '名侦探柯南第752集',
-            },
-        }, {
-            'info_dict': {
-                'id': 'e3f585b550a280af23c98b6cb2be19fb_part5',
-                'ext': 'f4v',
-                'title': '名侦探柯南第752集',
-            },
-        }, {
-            'info_dict': {
-                'id': 'e3f585b550a280af23c98b6cb2be19fb_part6',
-                'ext': 'f4v',
-                'title': '名侦探柯南第752集',
-            },
-        }, {
-            'info_dict': {
-                'id': 'e3f585b550a280af23c98b6cb2be19fb_part7',
-                'ext': 'f4v',
-                'title': '名侦探柯南第752集',
-            },
-        }, {
-            'info_dict': {
-                'id': 'e3f585b550a280af23c98b6cb2be19fb_part8',
-                'ext': 'f4v',
-                'title': '名侦探柯南第752集',
-            },
-        }],
         'params': {
-            'skip_download': True,
+            'cn_verification_proxy': 'http://proxy.uku.im:443/',
         },
     }, {
         'url': 'http://www.iqiyi.com/w_19rt6o8t9p.html',
@@ -289,13 +235,6 @@ class IqiyiIE(InfoExtractor):
         ('10', 'h1'),
     ]
 
-    AUTH_API_ERRORS = {
-        # No preview available (不允许试看鉴权失败)
-        'Q00505': 'This video requires a VIP account',
-        # End of preview time (试看结束鉴权失败)
-        'Q00506': 'Needs a VIP account for full video',
-    }
-
     def _real_initialize(self):
         self._login()
 
@@ -354,177 +293,101 @@ class IqiyiIE(InfoExtractor):
 
         return True
 
-    def _authenticate_vip_video(self, api_video_url, video_id, tvid, _uuid, do_report_warning):
-        auth_params = {
-            # version and platform hard-coded in com/qiyi/player/core/model/remote/AuthenticationRemote.as
-            'version': '2.0',
-            'platform': 'b6c13e26323c537d',
-            'aid': tvid,
+    @staticmethod
+    def _gen_sc(tvid, timestamp):
+        M = [1732584193, -271733879]
+        M.extend([~M[0], ~M[1]])
+        I_table = [7, 12, 17, 22, 5, 9, 14, 20, 4, 11, 16, 23, 6, 10, 15, 21]
+        C_base = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8388608, 432]
+
+        def L(n, t):
+            if t is None:
+                t = 0
+            return trunc(((n >> 1) + (t >> 1) << 1) + (n & 1) + (t & 1))
+
+        def trunc(n):
+            n = n % 0x100000000
+            if n > 0x7fffffff:
+                n -= 0x100000000
+            return n
+
+        def transform(string, mod):
+            num = int(string, 16)
+            return (num >> 8 * (i % 4) & 255 ^ i % mod) << ((a & 3) << 3)
+
+        C = list(C_base)
+        o = list(M)
+        k = str(timestamp - 7)
+        for i in range(13):
+            a = i
+            C[a >> 2] |= ord(k[a]) << 8 * (a % 4)
+
+        for i in range(16):
+            a = i + 13
+            start = (i >> 2) * 8
+            r = '03967743b643f66763d623d637e30733'
+            C[a >> 2] |= transform(''.join(reversed(r[start:start + 8])), 7)
+
+        for i in range(16):
+            a = i + 29
+            start = (i >> 2) * 8
+            r = '7038766939776a32776a32706b337139'
+            C[a >> 2] |= transform(r[start:start + 8], 1)
+
+        for i in range(9):
+            a = i + 45
+            if i < len(tvid):
+                C[a >> 2] |= ord(tvid[i]) << 8 * (a % 4)
+
+        for a in range(64):
+            i = a
+            I = i >> 4
+            C_index = [i, 5 * i + 1, 3 * i + 5, 7 * i][I] % 16 + urshift(a, 6)
+            m = L(L(o[0], [
+                trunc(o[1] & o[2]) | trunc(~o[1] & o[3]),
+                trunc(o[3] & o[1]) | trunc(~o[3] & o[2]),
+                o[1] ^ o[2] ^ o[3],
+                o[2] ^ trunc(o[1] | ~o[3])
+            ][I]), L(
+                trunc(int(abs(math.sin(i + 1)) * 4294967296)),
+                C[C_index] if C_index < len(C) else None))
+            I = I_table[4 * I + i % 4]
+            o = [o[3],
+                 L(o[1], trunc(trunc(m << I) | urshift(m, 32 - I))),
+                 o[1],
+                 o[2]]
+
+        new_M = [L(o[0], M[0]), L(o[1], M[1]), L(o[2], M[2]), L(o[3], M[3])]
+        s = [new_M[a >> 3] >> (1 ^ a & 7) * 4 & 15 for a in range(32)]
+        return binascii.hexlify(intlist_to_bytes(s))[1::2].decode('ascii')
+
+    def get_raw_data(self, tvid, video_id):
+        tm = int(time.time() * 1000)
+
+        sc = self._gen_sc(tvid, tm)
+        params = {
+            'platForm': 'h5',
+            'rate': 1,
             'tvid': tvid,
-            'uid': '',
-            'deviceId': _uuid,
-            'playType': 'main',  # XXX: always main?
-            'filename': os.path.splitext(url_basename(api_video_url))[0],
-        }
-
-        qd_items = compat_parse_qs(compat_urllib_parse_urlparse(api_video_url).query)
-        for key, val in qd_items.items():
-            auth_params[key] = val[0]
-
-        auth_req = sanitized_Request(
-            'http://api.vip.iqiyi.com/services/ckn.action',
-            urlencode_postdata(auth_params))
-        # iQiyi server throws HTTP 405 error without the following header
-        auth_req.add_header('Content-Type', 'application/x-www-form-urlencoded')
-        auth_result = self._download_json(
-            auth_req, video_id,
-            note='Downloading video authentication JSON',
-            errnote='Unable to download video authentication JSON')
-
-        code = auth_result.get('code')
-        msg = self.AUTH_API_ERRORS.get(code) or auth_result.get('msg') or code
-        if code == 'Q00506':
-            if do_report_warning:
-                self.report_warning(msg)
-            return False
-        if 'data' not in auth_result:
-            if msg is not None:
-                raise ExtractorError('%s said: %s' % (self.IE_NAME, msg), expected=True)
-            raise ExtractorError('Unexpected error from Iqiyi auth API')
-
-        return auth_result['data']
-
-    def construct_video_urls(self, data, video_id, _uuid, tvid):
-        def do_xor(x, y):
-            a = y % 3
-            if a == 1:
-                return x ^ 121
-            if a == 2:
-                return x ^ 72
-            return x ^ 103
-
-        def get_encode_code(l):
-            a = 0
-            b = l.split('-')
-            c = len(b)
-            s = ''
-            for i in range(c - 1, -1, -1):
-                a = do_xor(int(b[c - i - 1], 16), i)
-                s += chr(a)
-            return s[::-1]
-
-        def get_path_key(x, format_id, segment_index):
-            mg = ')(*&^flash@#$%a'
-            tm = self._download_json(
-                'http://data.video.qiyi.com/t?tn=' + str(random.random()), video_id,
-                note='Download path key of segment %d for format %s' % (segment_index + 1, format_id)
-            )['t']
-            t = str(int(math.floor(int(tm) / (600.0))))
-            return md5_text(t + mg + x)
-
-        video_urls_dict = {}
-        need_vip_warning_report = True
-        for format_item in data['vp']['tkl'][0]['vs']:
-            if 0 < int(format_item['bid']) <= 10:
-                format_id = self.get_format(format_item['bid'])
-            else:
-                continue
-
-            video_urls = []
-
-            video_urls_info = format_item['fs']
-            if not format_item['fs'][0]['l'].startswith('/'):
-                t = get_encode_code(format_item['fs'][0]['l'])
-                if t.endswith('mp4'):
-                    video_urls_info = format_item['flvs']
-
-            for segment_index, segment in enumerate(video_urls_info):
-                vl = segment['l']
-                if not vl.startswith('/'):
-                    vl = get_encode_code(vl)
-                is_vip_video = '/vip/' in vl
-                filesize = segment['b']
-                base_url = data['vp']['du'].split('/')
-                if not is_vip_video:
-                    key = get_path_key(
-                        vl.split('/')[-1].split('.')[0], format_id, segment_index)
-                    base_url.insert(-1, key)
-                base_url = '/'.join(base_url)
-                param = {
-                    'su': _uuid,
-                    'qyid': uuid.uuid4().hex,
-                    'client': '',
-                    'z': '',
-                    'bt': '',
-                    'ct': '',
-                    'tn': str(int(time.time()))
-                }
-                api_video_url = base_url + vl
-                if is_vip_video:
-                    api_video_url = api_video_url.replace('.f4v', '.hml')
-                    auth_result = self._authenticate_vip_video(
-                        api_video_url, video_id, tvid, _uuid, need_vip_warning_report)
-                    if auth_result is False:
-                        need_vip_warning_report = False
-                        break
-                    param.update({
-                        't': auth_result['t'],
-                        # cid is hard-coded in com/qiyi/player/core/player/RuntimeData.as
-                        'cid': 'afbe8fd3d73448c9',
-                        'vid': video_id,
-                        'QY00001': auth_result['u'],
-                    })
-                api_video_url += '?' if '?' not in api_video_url else '&'
-                api_video_url += compat_urllib_parse_urlencode(param)
-                js = self._download_json(
-                    api_video_url, video_id,
-                    note='Download video info of segment %d for format %s' % (segment_index + 1, format_id))
-                video_url = js['l']
-                video_urls.append(
-                    (video_url, filesize))
-
-            video_urls_dict[format_id] = video_urls
-        return video_urls_dict
-
-    def get_format(self, bid):
-        matched_format_ids = [_format_id for _bid, _format_id in self._FORMATS_MAP if _bid == str(bid)]
-        return matched_format_ids[0] if len(matched_format_ids) else None
-
-    def get_bid(self, format_id):
-        matched_bids = [_bid for _bid, _format_id in self._FORMATS_MAP if _format_id == format_id]
-        return matched_bids[0] if len(matched_bids) else None
-
-    def get_raw_data(self, tvid, video_id, enc_key, _uuid):
-        tm = str(int(time.time()))
-        tail = tm + tvid
-        param = {
-            'key': 'fvip',
-            'src': md5_text('youtube-dl'),
-            'tvId': tvid,
             'vid': video_id,
-            'vinfo': 1,
-            'tm': tm,
-            'enc': md5_text(enc_key + tail),
-            'qyid': _uuid,
-            'tn': random.random(),
-            # In iQiyi's flash player, um is set to 1 if there's a logged user
-            # Some 1080P formats are only available with a logged user.
-            # Here force um=1 to trick the iQiyi server
-            'um': 1,
-            'authkey': md5_text(md5_text('') + tail),
-            'k_tag': 1,
+            'cupid': 'qc_100001_100186',
+            'type': 'mp4',
+            'nolimit': 0,
+            'agenttype': 13,
+            'src': 'd846d0c32d664d32b6b54ea48997a589',
+            'sc': sc,
+            't': tm - 7,
+            '__jsT': None,
         }
 
-        api_url = 'http://cache.video.qiyi.com/vms' + '?' + \
-            compat_urllib_parse_urlencode(param)
-        raw_data = self._download_json(api_url, video_id)
-        return raw_data
-
-    def get_enc_key(self, video_id):
-        # TODO: automatic key extraction
-        # last update at 2016-01-22 for Zombie::bite
-        enc_key = '4a1caba4b4465345366f28da7c117d20'
-        return enc_key
+        headers = {}
+        cn_verification_proxy = self._downloader.params.get('cn_verification_proxy')
+        if cn_verification_proxy:
+            headers['Ytdl-request-proxy'] = cn_verification_proxy
+        return self._download_json(
+            'http://cache.m.iqiyi.com/jp/tmts/%s/%s/' % (tvid, video_id),
+            video_id, transform_source=lambda s: remove_start(s, 'var tvInfoJs='),
+            query=params, headers=headers)
 
     def _extract_playlist(self, webpage):
         PAGE_SIZE = 50
@@ -573,58 +436,27 @@ class IqiyiIE(InfoExtractor):
             r'data-player-tvid\s*=\s*[\'"](\d+)', webpage, 'tvid')
         video_id = self._search_regex(
             r'data-player-videoid\s*=\s*[\'"]([a-f\d]+)', webpage, 'video_id')
-        _uuid = uuid.uuid4().hex
 
-        enc_key = self.get_enc_key(video_id)
+        for _ in range(5):
+            raw_data = self.get_raw_data(tvid, video_id)
 
-        raw_data = self.get_raw_data(tvid, video_id, enc_key, _uuid)
+            if raw_data['code'] != 'A00000':
+                if raw_data['code'] == 'A00111':
+                    self.raise_geo_restricted()
+                raise ExtractorError('Unable to load data. Error code: ' + raw_data['code'])
 
-        if raw_data['code'] != 'A000000':
-            raise ExtractorError('Unable to load data. Error code: ' + raw_data['code'])
+            data = raw_data['data']
 
-        data = raw_data['data']
+            # iQiYi sometimes returns Ads
+            if not isinstance(data['playInfo'], dict):
+                self._sleep(5, video_id)
+                continue
 
-        title = data['vi']['vn']
+            title = data['playInfo']['an']
+            break
 
-        # generate video_urls_dict
-        video_urls_dict = self.construct_video_urls(
-            data, video_id, _uuid, tvid)
-
-        # construct info
-        entries = []
-        for format_id in video_urls_dict:
-            video_urls = video_urls_dict[format_id]
-            for i, video_url_info in enumerate(video_urls):
-                if len(entries) < i + 1:
-                    entries.append({'formats': []})
-                entries[i]['formats'].append(
-                    {
-                        'url': video_url_info[0],
-                        'filesize': video_url_info[-1],
-                        'format_id': format_id,
-                        'preference': int(self.get_bid(format_id))
-                    }
-                )
-
-        for i in range(len(entries)):
-            self._sort_formats(entries[i]['formats'])
-            entries[i].update(
-                {
-                    'id': '%s_part%d' % (video_id, i + 1),
-                    'title': title,
-                }
-            )
-
-        if len(entries) > 1:
-            info = {
-                '_type': 'multi_video',
-                'id': video_id,
-                'title': title,
-                'entries': entries,
-            }
-        else:
-            info = entries[0]
-            info['id'] = video_id
-            info['title'] = title
-
-        return info
+        return {
+            'id': video_id,
+            'title': title,
+            'url': data['m3u'],
+        }

From fc3996bfe15deae02f4d8f1f4dc34a89fb8bfb03 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 26 Jun 2016 15:45:41 +0800
Subject: [PATCH 5315/6219] [iqiyi] Remove codes for debugging

---
 youtube_dl/extractor/iqiyi.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index b717ca09c..fea26685e 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -180,9 +180,7 @@ class IqiyiIE(InfoExtractor):
             'ext': 'mp4',
             'title': '名侦探柯南 国语版',
         },
-        'params': {
-            'cn_verification_proxy': 'http://proxy.uku.im:443/',
-        },
+        'skip': 'Geo-restricted to China',
     }, {
         'url': 'http://www.iqiyi.com/w_19rt6o8t9p.html',
         'only_matching': True,

From 3c9c088f9c51cce86d3df878feba1884c0234df5 Mon Sep 17 00:00:00 2001
From: TRox1972 <TRox1972@users.noreply.github.com>
Date: Sun, 12 Jun 2016 03:18:56 +0200
Subject: [PATCH 5316/6219] [Vidbit] Add new extractor

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/vidbit.py     | 36 ++++++++++++++++++++++++++++++
 2 files changed, 37 insertions(+)
 create mode 100644 youtube_dl/extractor/vidbit.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 9f98a1490..5ccac7c0c 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -916,6 +916,7 @@ from .vice import (
     ViceIE,
     ViceShowIE,
 )
+from .vidbit import VidbitIE
 from .viddler import ViddlerIE
 from .videodetective import VideoDetectiveIE
 from .videofyme import VideofyMeIE
diff --git a/youtube_dl/extractor/vidbit.py b/youtube_dl/extractor/vidbit.py
new file mode 100644
index 000000000..39d508962
--- /dev/null
+++ b/youtube_dl/extractor/vidbit.py
@@ -0,0 +1,36 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import url_basename
+from ..compat import compat_urlparse
+
+
+class VidbitIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?vidbit\.co/watch\?v=(?P<id>[\w-]+)'
+    _TEST = {
+        'url': 'http://www.vidbit.co/watch?v=MrM7LeaMJq',
+        'md5': 'f1a579a93282a78de7e1c53220ef0f12',
+        'info_dict': {
+            'id': 'MrM7LeaMJq',
+            'ext': 'mp4',
+            'title': 'RoboCop (1987) - Dick You\'re Fired',
+            'thumbnail': 'http://www.vidbit.co/thumbnails/MrM7LeaMJq.jpg',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        return {
+            'id': video_id,
+            'title': self._html_search_regex(r'<h1>(.+)</h1>', webpage, 'title'),
+            'url': compat_urlparse.urljoin(url, self._html_search_regex(r'file:\s*(["\'])((?:(?!\1).)+)\1',
+                webpage, 'video URL', group=2)),
+            'thumbnail': self._og_search_thumbnail(webpage),
+            'description': self._html_search_regex(r'description:(["\'])((?:(?!\1).)+)\1',
+                webpage, 'description', None, group=2),
+        }

From 88d9f6c0c4c3d1d2179ee4fe0af560f500e62579 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 26 Jun 2016 16:57:14 +0700
Subject: [PATCH 5317/6219] [utils] Add support for name list in
 _html_search_meta

---
 test/test_InfoExtractor.py     | 7 ++++++-
 youtube_dl/extractor/common.py | 6 ++++--
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/test/test_InfoExtractor.py b/test/test_InfoExtractor.py
index 6404ac89f..88e8ff904 100644
--- a/test/test_InfoExtractor.py
+++ b/test/test_InfoExtractor.py
@@ -11,7 +11,7 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 from test.helper import FakeYDL
 from youtube_dl.extractor.common import InfoExtractor
 from youtube_dl.extractor import YoutubeIE, get_info_extractor
-from youtube_dl.utils import encode_data_uri, strip_jsonp, ExtractorError
+from youtube_dl.utils import encode_data_uri, strip_jsonp, ExtractorError, RegexNotFoundError
 
 
 class TestIE(InfoExtractor):
@@ -66,6 +66,11 @@ class TestInfoExtractor(unittest.TestCase):
         self.assertEqual(ie._html_search_meta('d', html), '4')
         self.assertEqual(ie._html_search_meta('e', html), '5')
         self.assertEqual(ie._html_search_meta('f', html), '6')
+        self.assertEqual(ie._html_search_meta(('a', 'b', 'c'), html), '1')
+        self.assertEqual(ie._html_search_meta(('c', 'b', 'a'), html), '3')
+        self.assertEqual(ie._html_search_meta(('z', 'x', 'c'), html), '3')
+        self.assertRaises(RegexNotFoundError, ie._html_search_meta, 'z', html, None, fatal=True)
+        self.assertRaises(RegexNotFoundError, ie._html_search_meta, ('z', 'x'), html, None, fatal=True)
 
     def test_download_json(self):
         uri = encode_data_uri(b'{"foo": "blah"}', 'application/json')
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 5a2603b50..4eda4e2ea 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -749,10 +749,12 @@ class InfoExtractor(object):
         return self._og_search_property('url', html, **kargs)
 
     def _html_search_meta(self, name, html, display_name=None, fatal=False, **kwargs):
+        if not isinstance(name, (list, tuple)):
+            name = [name]
         if display_name is None:
-            display_name = name
+            display_name = name[0]
         return self._html_search_regex(
-            self._meta_regex(name),
+            [self._meta_regex(n) for n in name],
             html, display_name, fatal=fatal, group='content', **kwargs)
 
     def _dc_search_uploader(self, html):

From f484c5fa257420201768158aaec31af25f904f9c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 26 Jun 2016 16:59:28 +0700
Subject: [PATCH 5318/6219] [vidbit] Improve (Closes #9759)

---
 youtube_dl/extractor/vidbit.py | 84 ++++++++++++++++++++++++++--------
 1 file changed, 66 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/vidbit.py b/youtube_dl/extractor/vidbit.py
index 39d508962..e7ac5a842 100644
--- a/youtube_dl/extractor/vidbit.py
+++ b/youtube_dl/extractor/vidbit.py
@@ -1,36 +1,84 @@
-# coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
-from ..utils import url_basename
 from ..compat import compat_urlparse
+from ..utils import (
+    int_or_none,
+    js_to_json,
+    remove_end,
+    unified_strdate,
+)
 
 
 class VidbitIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?vidbit\.co/watch\?v=(?P<id>[\w-]+)'
-    _TEST = {
-        'url': 'http://www.vidbit.co/watch?v=MrM7LeaMJq',
-        'md5': 'f1a579a93282a78de7e1c53220ef0f12',
+    _VALID_URL = r'https?://(?:www\.)?vidbit\.co/(?:watch|embed)\?.*?\bv=(?P<id>[\da-zA-Z]+)'
+    _TESTS = [{
+        'url': 'http://www.vidbit.co/watch?v=jkL2yDOEq2',
+        'md5': '1a34b7f14defe3b8fafca9796892924d',
         'info_dict': {
-            'id': 'MrM7LeaMJq',
+            'id': 'jkL2yDOEq2',
             'ext': 'mp4',
-            'title': 'RoboCop (1987) - Dick You\'re Fired',
-            'thumbnail': 'http://www.vidbit.co/thumbnails/MrM7LeaMJq.jpg',
+            'title': 'Intro to VidBit',
+            'description': 'md5:5e0d6142eec00b766cbf114bfd3d16b7',
+            'thumbnail': 're:https?://.*\.jpg$',
+            'upload_date': '20160618',
+            'view_count': int,
+            'comment_count': int,
         }
-    }
+    }, {
+        'url': 'http://www.vidbit.co/embed?v=jkL2yDOEq2&auto=0&water=0',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
+
+        webpage = self._download_webpage(
+            compat_urlparse.urljoin(url, '/watch?v=%s' % video_id), video_id)
+
+        video_url, title = [None] * 2
+
+        config = self._parse_json(self._search_regex(
+            r'(?s)\.setup\(({.+?})\);', webpage, 'setup', default='{}'),
+            video_id, transform_source=js_to_json)
+        if config:
+            if config.get('file'):
+                video_url = compat_urlparse.urljoin(url, config['file'])
+            title = config.get('title')
+
+        if not video_url:
+            video_url = compat_urlparse.urljoin(url, self._search_regex(
+                r'file\s*:\s*(["\'])(?P<url>(?:(?!\1).)+)\1',
+                webpage, 'video URL', group='url'))
+
+        if not title:
+            title = remove_end(
+                self._html_search_regex(
+                    (r'<h1>(.+?)</h1>', r'<title>(.+?)</title>'),
+                    webpage, 'title', default=None) or self._og_search_title(webpage),
+                ' - VidBit')
+
+        description = self._html_search_meta(
+            ('description', 'og:description', 'twitter:description'),
+            webpage, 'description')
+
+        upload_date = unified_strdate(self._html_search_meta(
+            'datePublished', webpage, 'upload date'))
+
+        view_count = int_or_none(self._search_regex(
+            r'<strong>(\d+)</strong> views',
+            webpage, 'view count', fatal=False))
+        comment_count = int_or_none(self._search_regex(
+            r'id=["\']cmt_num["\'][^>]*>\((\d+)\)',
+            webpage, 'comment count', fatal=False))
 
         return {
             'id': video_id,
-            'title': self._html_search_regex(r'<h1>(.+)</h1>', webpage, 'title'),
-            'url': compat_urlparse.urljoin(url, self._html_search_regex(r'file:\s*(["\'])((?:(?!\1).)+)\1',
-                webpage, 'video URL', group=2)),
+            'url': video_url,
+            'title': title,
+            'description': description,
             'thumbnail': self._og_search_thumbnail(webpage),
-            'description': self._html_search_regex(r'description:(["\'])((?:(?!\1).)+)\1',
-                webpage, 'description', None, group=2),
+            'upload_date': upload_date,
+            'view_count': view_count,
+            'comment_count': comment_count,
         }

From 4f3c5e062715bb8c2084bda139ddcd9a2036f267 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 16 Mar 2016 18:48:06 +0100
Subject: [PATCH 5319/6219] [utils] add helper function for parsing codecs

---
 test/test_utils.py  | 24 ++++++++++++++++++++++++
 youtube_dl/utils.py | 36 ++++++++++++++++++++++++++++++++++++
 2 files changed, 60 insertions(+)

diff --git a/test/test_utils.py b/test/test_utils.py
index b7ef51f8d..d84eb438f 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -78,6 +78,7 @@ from youtube_dl.utils import (
     cli_option,
     cli_valueless_option,
     cli_bool_option,
+    parse_codecs,
 )
 from youtube_dl.compat import (
     compat_chr,
@@ -579,6 +580,29 @@ class TestUtil(unittest.TestCase):
             limit_length('foo bar baz asd', 12).startswith('foo bar'))
         self.assertTrue('...' in limit_length('foo bar baz asd', 12))
 
+    def test_parse_codecs(self):
+        self.assertEqual(parse_codecs(''), {})
+        self.assertEqual(parse_codecs('avc1.77.30, mp4a.40.2'), {
+            'vcodec': 'avc1.77.30',
+            'acodec': 'mp4a.40.2',
+        })
+        self.assertEqual(parse_codecs('mp4a.40.2'), {
+            'vcodec': 'none',
+            'acodec': 'mp4a.40.2',
+        })
+        self.assertEqual(parse_codecs('mp4a.40.5,avc1.42001e'), {
+            'vcodec': 'avc1.42001e',
+            'acodec': 'mp4a.40.5',
+        })
+        self.assertEqual(parse_codecs('avc3.640028'), {
+            'vcodec': 'avc3.640028',
+            'acodec': 'none',
+        })
+        self.assertEqual(parse_codecs(', h264,,newcodec,aac'), {
+            'vcodec': 'h264',
+            'acodec': 'aac',
+        })
+
     def test_escape_rfc3986(self):
         reserved = "!*'();:@&=+$,/?#[]"
         unreserved = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_.~'
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 562031fe1..fe175e82c 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2060,6 +2060,42 @@ def mimetype2ext(mt):
     }.get(res, res)
 
 
+def parse_codecs(codecs_str):
+    # http://tools.ietf.org/html/rfc6381
+    if not codecs_str:
+        return {}
+    splited_codecs = list(filter(None, map(
+        lambda str: str.strip(), codecs_str.strip().strip(',').split(','))))
+    vcodec, acodec = None, None
+    for full_codec in splited_codecs:
+        codec = full_codec.split('.')[0]
+        if codec in ('avc1', 'avc2', 'avc3', 'avc4', 'vp9', 'vp8', 'hev1', 'hev2', 'h263', 'h264', 'mp4v'):
+            if not vcodec:
+                vcodec = full_codec
+        elif codec in ('mp4a', 'opus', 'vorbis', 'mp3', 'aac'):
+            if not acodec:
+                acodec = full_codec
+        else:
+            write_string('WARNING: Unknown codec %s' % full_codec, sys.stderr)
+    if not vcodec and not acodec:
+        if len(splited_codecs) == 2:
+            return {
+                'vcodec': vcodec,
+                'acodec': acodec,
+            }
+        elif len(splited_codecs) == 1:
+            return {
+                'vcodec': 'none',
+                'acodec': vcodec,
+            }
+    else:
+        return {
+            'vcodec': vcodec or 'none',
+            'acodec': acodec or 'none',
+        }
+    return {}
+
+
 def urlhandle_detect_ext(url_handle):
     getheader = url_handle.headers.get
 

From 59bbe4911acd4493bf407925bfdeb1ad03db6ef3 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 16 Mar 2016 18:50:45 +0100
Subject: [PATCH 5320/6219] [extractor/common] add helper method to extract
 html5 media entries

---
 youtube_dl/extractor/common.py | 58 ++++++++++++++++++++++++++++++++++
 1 file changed, 58 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 5a2603b50..661889593 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -54,6 +54,8 @@ from ..utils import (
     update_Request,
     update_url_query,
     parse_m3u8_attributes,
+    extract_attributes,
+    parse_codecs,
 )
 
 
@@ -1610,6 +1612,62 @@ class InfoExtractor(object):
                         self.report_warning('Unknown MIME type %s in DASH manifest' % mime_type)
         return formats
 
+    def _parse_html5_media_entries(self, base_url, webpage):
+        def absolute_url(video_url):
+            return compat_urlparse.urljoin(base_url, video_url)
+
+        def parse_content_type(content_type):
+            if not content_type:
+                return {}
+            ctr = re.search(r'(?P<mimetype>[^/]+/[^;]+)(?:;\s*codecs="?(?P<codecs>[^"]+))?', content_type)
+            if ctr:
+                mimetype, codecs = ctr.groups()
+                f = parse_codecs(codecs)
+                f['ext'] = mimetype2ext(mimetype)
+                return f
+            return {}
+
+        entries = []
+        for media_tag, media_type, media_content in re.findall(r'(?s)(<(?P<tag>video|audio)[^>]*>)(.*?)</(?P=tag)>', webpage):
+            media_info = {
+                'formats': [],
+                'subtitles': {},
+            }
+            media_attributes = extract_attributes(media_tag)
+            src = media_attributes.get('src')
+            if src:
+                media_info['formats'].append({
+                    'url': absolute_url(src),
+                    'vcodec': 'none' if media_type == 'audio' else None,
+                })
+            media_info['thumbnail'] = media_attributes.get('poster')
+            if media_content:
+                for source_tag in re.findall(r'<source[^>]+>', media_content):
+                    source_attributes = extract_attributes(source_tag)
+                    src = source_attributes.get('src')
+                    if not src:
+                        continue
+                    f = parse_content_type(source_attributes.get('type'))
+                    f.update({
+                        'url': absolute_url(src),
+                        'vcodec': 'none' if media_type == 'audio' else None,
+                    })
+                    media_info['formats'].append(f)
+                for track_tag in re.findall(r'<track[^>]+>', media_content):
+                    track_attributes = extract_attributes(track_tag)
+                    kind = track_attributes.get('kind')
+                    if not kind or kind == 'subtitles':
+                        src = track_attributes.get('src')
+                        if not src:
+                            continue
+                        lang = track_attributes.get('srclang') or track_attributes.get('lang') or track_attributes.get('label')
+                        media_info['subtitles'].setdefault(lang, []).append({
+                            'url': absolute_url(src),
+                        })
+            if media_info['formats']:
+                entries.append(media_info)
+        return entries
+
     def _live_title(self, name):
         """ Generate the title for a live video """
         now = datetime.datetime.now()

From c6781156aa023c1131db6c5b1f575e1833649b33 Mon Sep 17 00:00:00 2001
From: TRox1972 <TRox1972@users.noreply.github.com>
Date: Thu, 19 May 2016 20:59:59 +0200
Subject: [PATCH 5321/6219] [MSN] add new extractor

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/msn.py        | 90 ++++++++++++++++++++++++++++++
 2 files changed, 91 insertions(+)
 create mode 100644 youtube_dl/extractor/msn.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 5ccac7c0c..2f9ee1596 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -454,6 +454,7 @@ from .motherless import MotherlessIE
 from .motorsport import MotorsportIE
 from .movieclips import MovieClipsIE
 from .moviezine import MoviezineIE
+from .msn import MSNIE
 from .mtv import (
     MTVIE,
     MTVServicesEmbeddedIE,
diff --git a/youtube_dl/extractor/msn.py b/youtube_dl/extractor/msn.py
new file mode 100644
index 000000000..4dd57fca0
--- /dev/null
+++ b/youtube_dl/extractor/msn.py
@@ -0,0 +1,90 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+from .common import InfoExtractor
+
+from ..utils import (
+    unescapeHTML,
+    int_or_none,
+)
+
+class MSNIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?msn\.com/[a-z-]{2,5}(?:/[a-z]+)+/(?P<display_id>[a-z-]+)/[a-z]{2}-(?P<id>[a-zA-Z]+)'
+    _TESTS = [{
+        'url': 'http://www.msn.com/en-ae/foodanddrink/joinourtable/criminal-minds-shemar-moore-shares-a-touching-goodbye-message/vp-BBqQYNE',
+        'info_dict': {
+            'id': 'BBqQYNE',
+            'title': 'Criminal Minds - Shemar Moore Shares A Touching Goodbye Message',
+            'description': 'md5:e8e89b897b222eb33a6b5067a8f1bc25',
+            'duration': 104,
+            'ext': 'mp4',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        }
+    }, {
+        'url': 'http://www.msn.com/en-ae/news/offbeat/meet-the-nine-year-old-self-made-millionaire/ar-BBt6ZKf',
+        'info_dict': {
+            'id': 'BBt6ZKf',
+            'title': 'All That Bling: Self-Made Millionaire Child Builds Fashion & Jewellery Empire',
+            'description': 'md5:8e683bd5c729d5fb16d96539a582aa5e',
+            'duration': 350,
+            'ext': 'mp4',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        }
+    }]
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id, display_id = mobj.group('id', 'display_id')
+
+        webpage = self._download_webpage(url, display_id)
+
+        self.report_extraction(display_id)
+        video_data = self._parse_json(self._html_search_regex(r'data-metadata\s*=\s*["\'](.+)["\']',
+            webpage, 'video data'), display_id)
+
+        formats = []
+        for video_file in video_data.get('videoFiles', []):
+            if not '.ism' in video_file.get('url', '.ism'):
+                formats.append({
+                    'url': unescapeHTML(video_file.get('url')),
+                    'ext': 'mp4',
+                    'width': int_or_none(video_file.get('width')),
+                    'height': int_or_none(video_file.get('height')),
+                })
+            elif 'm3u8' in video_file.get('url'):
+                formats.extend(self._extract_m3u8_formats(
+                    video_file.get('url'), display_id, 'mp4'))
+            # There (often) exists an Microsoft Smooth Streaming manifest
+            # (.ism) which is not yet supported
+            # (https://github.com/rg3/youtube-dl/issues/8118)
+
+        self._sort_formats(formats)
+
+        subtitles = {}
+        for f in video_data.get('files', []):
+            if f.get('formatCode', '') == '3100':
+                lang = f.get('culture', '')
+                if not lang:
+                    continue
+                subtitles.setdefault(lang, []).append({
+                    'ext': 'ttml',
+                    'url': unescapeHTML(f.get('url')),
+                })
+
+        return {
+            'id': video_id,
+            'title': video_data['title'],
+            'formats': formats,
+            'thumbnail': video_data.get('headlineImage', {}).get('url'),
+            'description': video_data.get('description'),
+            'creator': video_data.get('creator'),
+            'subtitles': subtitles,
+            'duration': int_or_none(video_data.get('durationSecs')),
+        }

From bf8dd790456acc4a96d2961e61e96c4771e4d787 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 26 Jun 2016 21:09:07 +0700
Subject: [PATCH 5322/6219] [extractor/common] Fix sorting with custom field
 preference

---
 youtube_dl/extractor/common.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 4eda4e2ea..e6c15de42 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -878,7 +878,11 @@ class InfoExtractor(object):
                 f['ext'] = determine_ext(f['url'])
 
             if isinstance(field_preference, (list, tuple)):
-                return tuple(f.get(field) if f.get(field) is not None else -1 for field in field_preference)
+                return tuple(
+                    f.get(field)
+                    if f.get(field) is not None
+                    else ('' if field == 'format_id' else -1)
+                    for field in field_preference)
 
             preference = f.get('preference')
             if preference is None:

From f1f336322da6e719cf4298b08680c3e903e956c4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 26 Jun 2016 21:10:05 +0700
Subject: [PATCH 5323/6219] [msn] Fix extraction (Closes #8960, closes #9542)

---
 youtube_dl/extractor/msn.py | 125 ++++++++++++++++++++++--------------
 1 file changed, 77 insertions(+), 48 deletions(-)

diff --git a/youtube_dl/extractor/msn.py b/youtube_dl/extractor/msn.py
index 4dd57fca0..d4569e325 100644
--- a/youtube_dl/extractor/msn.py
+++ b/youtube_dl/extractor/msn.py
@@ -2,41 +2,42 @@
 from __future__ import unicode_literals
 
 import re
-from .common import InfoExtractor
 
+from .common import InfoExtractor
+from ..compat import compat_str
 from ..utils import (
-    unescapeHTML,
+    determine_ext,
+    ExtractorError,
     int_or_none,
+    unescapeHTML,
 )
 
+
 class MSNIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?msn\.com/[a-z-]{2,5}(?:/[a-z]+)+/(?P<display_id>[a-z-]+)/[a-z]{2}-(?P<id>[a-zA-Z]+)'
+    _VALID_URL = r'https?://(?:www\.)?msn\.com/(?:[^/]+/)+(?P<display_id>[^/]+)/[a-z]{2}-(?P<id>[\da-zA-Z]+)'
     _TESTS = [{
         'url': 'http://www.msn.com/en-ae/foodanddrink/joinourtable/criminal-minds-shemar-moore-shares-a-touching-goodbye-message/vp-BBqQYNE',
+        'md5': '8442f66c116cbab1ff7098f986983458',
         'info_dict': {
             'id': 'BBqQYNE',
+            'display_id': 'criminal-minds-shemar-moore-shares-a-touching-goodbye-message',
+            'ext': 'mp4',
             'title': 'Criminal Minds - Shemar Moore Shares A Touching Goodbye Message',
             'description': 'md5:e8e89b897b222eb33a6b5067a8f1bc25',
             'duration': 104,
-            'ext': 'mp4',
+            'uploader': 'CBS Entertainment',
+            'uploader_id': 'IT0X5aoJ6bJgYerJXSDCgFmYPB1__54v',
         },
-        'params': {
-            # m3u8 download
-            'skip_download': True,
-        }
     }, {
         'url': 'http://www.msn.com/en-ae/news/offbeat/meet-the-nine-year-old-self-made-millionaire/ar-BBt6ZKf',
-        'info_dict': {
-            'id': 'BBt6ZKf',
-            'title': 'All That Bling: Self-Made Millionaire Child Builds Fashion & Jewellery Empire',
-            'description': 'md5:8e683bd5c729d5fb16d96539a582aa5e',
-            'duration': 350,
-            'ext': 'mp4',
-        },
-        'params': {
-            # m3u8 download
-            'skip_download': True,
-        }
+        'only_matching': True,
+    }, {
+        'url': 'http://www.msn.com/en-ae/video/watch/obama-a-lot-of-people-will-be-disappointed/vi-AAhxUMH',
+        'only_matching': True,
+    }, {
+        # geo restricted
+        'url': 'http://www.msn.com/en-ae/foodanddrink/joinourtable/the-first-fart-makes-you-laugh-the-last-fart-makes-you-cry/vp-AAhzIBU',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
@@ -45,46 +46,74 @@ class MSNIE(InfoExtractor):
 
         webpage = self._download_webpage(url, display_id)
 
-        self.report_extraction(display_id)
-        video_data = self._parse_json(self._html_search_regex(r'data-metadata\s*=\s*["\'](.+)["\']',
-            webpage, 'video data'), display_id)
+        video = self._parse_json(
+            self._search_regex(
+                r'data-metadata\s*=\s*(["\'])(?P<data>.+?)\1',
+                webpage, 'video data', default='{}', group='data'),
+            display_id, transform_source=unescapeHTML)
+
+        if not video:
+            error = unescapeHTML(self._search_regex(
+                r'data-error=(["\'])(?P<error>.+?)\1',
+                webpage, 'error', group='error'))
+            raise ExtractorError('%s said: %s' % (self.IE_NAME, error), expected=True)
+
+        title = video['title']
 
         formats = []
-        for video_file in video_data.get('videoFiles', []):
-            if not '.ism' in video_file.get('url', '.ism'):
+        for file_ in video.get('videoFiles', []):
+            format_url = file_.get('url')
+            if not format_url:
+                continue
+            ext = determine_ext(format_url)
+            # .ism is not yet supported (see
+            # https://github.com/rg3/youtube-dl/issues/8118)
+            if ext == 'ism':
+                continue
+            if 'm3u8' in format_url:
+                # m3u8_native should not be used here until
+                # https://github.com/rg3/youtube-dl/issues/9913 is fixed
+                m3u8_formats = self._extract_m3u8_formats(
+                    format_url, display_id, 'mp4',
+                    m3u8_id='hls', fatal=False)
+                # Despite metadata in m3u8 all video+audio formats are
+                # actually video-only (no audio)
+                for f in m3u8_formats:
+                    if f.get('acodec') != 'none' and f.get('vcodec') != 'none':
+                        f['acodec'] = 'none'
+                formats.extend(m3u8_formats)
+            else:
                 formats.append({
-                    'url': unescapeHTML(video_file.get('url')),
+                    'url': format_url,
                     'ext': 'mp4',
-                    'width': int_or_none(video_file.get('width')),
-                    'height': int_or_none(video_file.get('height')),
+                    'format_id': 'http',
+                    'width': int_or_none(file_.get('width')),
+                    'height': int_or_none(file_.get('height')),
                 })
-            elif 'm3u8' in video_file.get('url'):
-                formats.extend(self._extract_m3u8_formats(
-                    video_file.get('url'), display_id, 'mp4'))
-            # There (often) exists an Microsoft Smooth Streaming manifest
-            # (.ism) which is not yet supported
-            # (https://github.com/rg3/youtube-dl/issues/8118)
-
         self._sort_formats(formats)
 
         subtitles = {}
-        for f in video_data.get('files', []):
-            if f.get('formatCode', '') == '3100':
-                lang = f.get('culture', '')
-                if not lang:
-                    continue
-                subtitles.setdefault(lang, []).append({
-                    'ext': 'ttml',
-                    'url': unescapeHTML(f.get('url')),
+        for file_ in video.get('files', []):
+            format_url = file_.get('url')
+            format_code = file_.get('formatCode')
+            if not format_url or not format_code:
+                continue
+            if compat_str(format_code) == '3100':
+                subtitles.setdefault(file_.get('culture', 'en'), []).append({
+                    'ext': determine_ext(format_url, 'ttml'),
+                    'url': format_url,
                 })
 
         return {
             'id': video_id,
-            'title': video_data['title'],
-            'formats': formats,
-            'thumbnail': video_data.get('headlineImage', {}).get('url'),
-            'description': video_data.get('description'),
-            'creator': video_data.get('creator'),
+            'display_id': display_id,
+            'title': title,
+            'description': video.get('description'),
+            'thumbnail': video.get('headlineImage', {}).get('url'),
+            'duration': int_or_none(video.get('durationSecs')),
+            'uploader': video.get('sourceFriendly'),
+            'uploader_id': video.get('providerId'),
+            'creator': video.get('creator'),
             'subtitles': subtitles,
-            'duration': int_or_none(video_data.get('durationSecs')),
+            'formats': formats,
         }

From 92747e664a70e6739644a9c2b3abfbdcc68fd136 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 26 Jun 2016 21:15:24 +0700
Subject: [PATCH 5324/6219] release 2016.06.26

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 docs/supportedsites.md    | 3 +++
 youtube_dl/version.py     | 2 +-
 3 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index c73f9a904..63b687fef 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.06.25*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.06.25**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.06.26*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.06.26**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.06.25
+[debug] youtube-dl version 2016.06.26
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 891499f59..a725e8c6b 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -385,6 +385,7 @@
  - **MovieFap**
  - **Moviezine**
  - **MPORA**
+ - **MSN**
  - **MTV**
  - **mtv.de**
  - **mtviggy.com**
@@ -501,6 +502,7 @@
  - **plus.google**: Google Plus
  - **pluzz.francetv.fr**
  - **podomatic**
+ - **PolskieRadio**
  - **PornHd**
  - **PornHub**
  - **PornHubPlaylist**
@@ -736,6 +738,7 @@
  - **vh1.com**
  - **Vice**
  - **ViceShow**
+ - **Vidbit**
  - **Viddler**
  - **video.google:search**: Google Video search
  - **video.mit.edu**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 2b7a4c98d..52de19517 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.06.25'
+__version__ = '2016.06.26'

From b0c200f1ec594b7c7d5a5023853970ff789a3470 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 26 Jun 2016 22:02:46 +0700
Subject: [PATCH 5325/6219] [msn] Add test URL with non-alphanumeric characters

---
 youtube_dl/extractor/msn.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/msn.py b/youtube_dl/extractor/msn.py
index d4569e325..1ec8e0f50 100644
--- a/youtube_dl/extractor/msn.py
+++ b/youtube_dl/extractor/msn.py
@@ -38,6 +38,9 @@ class MSNIE(InfoExtractor):
         # geo restricted
         'url': 'http://www.msn.com/en-ae/foodanddrink/joinourtable/the-first-fart-makes-you-laugh-the-last-fart-makes-you-cry/vp-AAhzIBU',
         'only_matching': True,
+    }, {
+        'url': 'http://www.msn.com/en-ae/entertainment/bollywood/watch-how-salman-khan-reacted-when-asked-if-he-would-apologize-for-his-‘raped-woman’-comment/vi-AAhvzW6',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 427cd050a3b64319c19e4596d8885378604e388e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 27 Jun 2016 04:11:53 +0700
Subject: [PATCH 5326/6219] [extractor/generic] Improve kaltura embed detection
 (Closes #9911)

---
 youtube_dl/extractor/generic.py | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 1592a8a3a..26a7d10be 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -920,6 +920,24 @@ class GenericIE(InfoExtractor):
             },
             'add_ie': ['Kaltura'],
         },
+        {
+            # Kaltura embedded via quoted entry_id
+            'url': 'https://www.oreilly.com/ideas/my-cloud-makes-pretty-pictures',
+            'info_dict': {
+                'id': '0_utuok90b',
+                'ext': 'mp4',
+                'title': '06_matthew_brender_raj_dutt',
+                'timestamp': 1466638791,
+                'upload_date': '20160622',
+            },
+            'add_ie': ['Kaltura'],
+            'expected_warnings': [
+                'Could not send HEAD request'
+            ],
+            'params': {
+                'skip_download': True,
+            }
+        },
         # Eagle.Platform embed (generic URL)
         {
             'url': 'http://lenta.ru/news/2015/03/06/navalny/',
@@ -1909,7 +1927,7 @@ class GenericIE(InfoExtractor):
 
         # Look for Kaltura embeds
         mobj = (re.search(r"(?s)kWidget\.(?:thumb)?[Ee]mbed\(\{.*?(?P<q1>['\"])wid(?P=q1)\s*:\s*(?P<q2>['\"])_?(?P<partner_id>[^'\"]+)(?P=q2),.*?(?P<q3>['\"])entry_?[Ii]d(?P=q3)\s*:\s*(?P<q4>['\"])(?P<id>[^'\"]+)(?P=q4),", webpage) or
-                re.search(r'(?s)(?P<q1>["\'])(?:https?:)?//cdnapi(?:sec)?\.kaltura\.com/.*?(?:p|partner_id)/(?P<partner_id>\d+).*?(?P=q1).*?entry_?[Ii]d\s*:\s*(?P<q2>["\'])(?P<id>.+?)(?P=q2)', webpage))
+                re.search(r'(?s)(?P<q1>["\'])(?:https?:)?//cdnapi(?:sec)?\.kaltura\.com/.*?(?:p|partner_id)/(?P<partner_id>\d+).*?(?P=q1).*?(?P<q2>["\'])?entry_?[Ii]d(?P=q2)\s*:\s*(?P<q3>["\'])(?P<id>.+?)(?P=q3)', webpage))
         if mobj is not None:
             return self.url_result(smuggle_url(
                 'kaltura:%(partner_id)s:%(id)s' % mobj.groupdict(),

From 81fda1536924db0ec4f583ae83bc77cb91ca6835 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 27 Jun 2016 05:07:12 +0700
Subject: [PATCH 5327/6219] [sr:mediathek] Clarify IE_NAME

---
 youtube_dl/extractor/srmediathek.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/srmediathek.py b/youtube_dl/extractor/srmediathek.py
index 74d01183f..a2569dfba 100644
--- a/youtube_dl/extractor/srmediathek.py
+++ b/youtube_dl/extractor/srmediathek.py
@@ -9,6 +9,7 @@ from ..utils import (
 
 
 class SRMediathekIE(ARDMediathekIE):
+    IE_NAME = 'sr:mediathek'
     IE_DESC = 'Saarländischer Rundfunk'
     _VALID_URL = r'https?://sr-mediathek\.sr-online\.de/index\.php\?.*?&id=(?P<id>[0-9]+)'
 

From f41ffc00d15697c6d4c8975d261ffd5b0c5e971f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 27 Jun 2016 05:08:09 +0700
Subject: [PATCH 5328/6219] [skynewsarabia:article] Clarify IE_NAME

---
 youtube_dl/extractor/skynewsarabia.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/skynewsarabia.py b/youtube_dl/extractor/skynewsarabia.py
index 05e1b02ad..fffc9aa22 100644
--- a/youtube_dl/extractor/skynewsarabia.py
+++ b/youtube_dl/extractor/skynewsarabia.py
@@ -67,7 +67,7 @@ class SkyNewsArabiaIE(SkyNewsArabiaBaseIE):
 
 
 class SkyNewsArabiaArticleIE(SkyNewsArabiaBaseIE):
-    IE_NAME = 'skynewsarabia:video'
+    IE_NAME = 'skynewsarabia:article'
     _VALID_URL = r'https?://(?:www\.)?skynewsarabia\.com/web/article/(?P<id>[0-9]+)'
     _TESTS = [{
         'url': 'http://www.skynewsarabia.com/web/article/794549/%D8%A7%D9%94%D8%AD%D8%AF%D8%A7%D8%AB-%D8%A7%D9%84%D8%B4%D8%B1%D9%82-%D8%A7%D9%84%D8%A7%D9%94%D9%88%D8%B3%D8%B7-%D8%AE%D8%B1%D9%8A%D8%B7%D8%A9-%D8%A7%D9%84%D8%A7%D9%94%D9%84%D8%B9%D8%A7%D8%A8-%D8%A7%D9%84%D8%B0%D9%83%D9%8A%D8%A9',

From e3a6747d8f19ad0ba8aee7c3214cdb64903beba0 Mon Sep 17 00:00:00 2001
From: Matthieu Muffato <cortexspam-github@yahoo.fr>
Date: Sun, 26 Jun 2016 23:31:55 +0100
Subject: [PATCH 5329/6219] New test-case: extractor names are supposed to be
 unique

@dstftw explained in
https://github.com/rg3/youtube-dl/pull/9918#issuecomment-228625878 that
extractor names are supposed to be unique. @dstftw has fixed the two
offending extractors, and here I add a test to ensure this does not
happen in the future.
---
 test/test_all_urls.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/test/test_all_urls.py b/test/test_all_urls.py
index f5af184e6..133d438eb 100644
--- a/test/test_all_urls.py
+++ b/test/test_all_urls.py
@@ -6,6 +6,7 @@ from __future__ import unicode_literals
 import os
 import sys
 import unittest
+import collections
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 
@@ -130,6 +131,13 @@ class TestAllURLsMatching(unittest.TestCase):
             'https://screen.yahoo.com/smartwatches-latest-wearable-gadgets-163745379-cbs.html',
             ['Yahoo'])
 
+    def test_no_duplicated_ie_names(self):
+        name_accu = collections.defaultdict(list)
+        for ie in self.ies:
+            name_accu[ie.IE_NAME.lower()].append(ie)
+        for (ie_name,ie_list) in name_accu.items():
+            self.assertEqual(len(ie_list), 1, 'Only 1 extractor with IE_NAME "%s" (%s)' % (ie_name, ie_list))
+
 
 if __name__ == '__main__':
     unittest.main()

From fd7a7498a47c5d79663ec8d86a87325aa634c652 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 27 Jun 2016 22:11:45 +0700
Subject: [PATCH 5330/6219] [test_all_urls] PEP 8 and change wording

---
 test/test_all_urls.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/test/test_all_urls.py b/test/test_all_urls.py
index 133d438eb..1f6079c29 100644
--- a/test/test_all_urls.py
+++ b/test/test_all_urls.py
@@ -134,9 +134,11 @@ class TestAllURLsMatching(unittest.TestCase):
     def test_no_duplicated_ie_names(self):
         name_accu = collections.defaultdict(list)
         for ie in self.ies:
-            name_accu[ie.IE_NAME.lower()].append(ie)
-        for (ie_name,ie_list) in name_accu.items():
-            self.assertEqual(len(ie_list), 1, 'Only 1 extractor with IE_NAME "%s" (%s)' % (ie_name, ie_list))
+            name_accu[ie.IE_NAME.lower()].append(type(ie).__name__)
+        for (ie_name, ie_list) in name_accu.items():
+            self.assertEqual(
+                len(ie_list), 1,
+                'Multiple extractors with the same IE_NAME "%s" (%s)' % (ie_name, ', '.join(ie_list)))
 
 
 if __name__ == '__main__':

From 9ea5c04c0d16f5519079ae04fdad62fc28c884b0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 27 Jun 2016 22:44:17 +0700
Subject: [PATCH 5331/6219] [kaltura] Add _extract_url with fixed regex

---
 youtube_dl/extractor/kaltura.py | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/youtube_dl/extractor/kaltura.py b/youtube_dl/extractor/kaltura.py
index a65697ff5..c75a958ba 100644
--- a/youtube_dl/extractor/kaltura.py
+++ b/youtube_dl/extractor/kaltura.py
@@ -64,6 +64,32 @@ class KalturaIE(InfoExtractor):
         }
     ]
 
+    @staticmethod
+    def _extract_url(webpage):
+        mobj = (
+            re.search(
+                r"""(?xs)
+                    kWidget\.(?:thumb)?[Ee]mbed\(
+                    \{.*?
+                        (?P<q1>['\"])wid(?P=q1)\s*:\s*
+                        (?P<q2>['\"])_?(?P<partner_id>[^'\"]+)(?P=q2),.*?
+                        (?P<q3>['\"])entry_?[Ii]d(?P=q3)\s*:\s*
+                        (?P<q4>['\"])(?P<id>[^'\"]+)(?P=q4),
+                """, webpage) or
+            re.search(
+                r'''(?xs)
+                    (?P<q1>["\'])
+                        (?:https?:)?//cdnapi(?:sec)?\.kaltura\.com/.*?(?:p|partner_id)/(?P<partner_id>\d+).*?
+                    (?P=q1).*?
+                    (?:
+                        entry_?[Ii]d|
+                        (?P<q2>["\'])entry_?[Ii]d(?P=q2)
+                    )\s*:\s*
+                    (?P<q3>["\'])(?P<id>.+?)(?P=q3)
+                ''', webpage))
+        if mobj:
+            return 'kaltura:%(partner_id)s:%(id)s' % mobj.groupdict()
+
     def _kaltura_api_call(self, video_id, actions, *args, **kwargs):
         params = actions[0]
         if len(actions) > 1:

From c287f2bc6073182323aada26309539d724943fa8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 27 Jun 2016 22:45:26 +0700
Subject: [PATCH 5332/6219] [extractor/generic] Use _extract_url for kaltura
 embeds (Closes #9922)

---
 youtube_dl/extractor/generic.py | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 26a7d10be..2188f8bb2 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -64,6 +64,7 @@ from .liveleak import LiveLeakIE
 from .threeqsdn import ThreeQSDNIE
 from .theplatform import ThePlatformIE
 from .vessel import VesselIE
+from .kaltura import KalturaIE
 
 
 class GenericIE(InfoExtractor):
@@ -1926,12 +1927,9 @@ class GenericIE(InfoExtractor):
             return self.url_result(mobj.group('url'), 'Zapiks')
 
         # Look for Kaltura embeds
-        mobj = (re.search(r"(?s)kWidget\.(?:thumb)?[Ee]mbed\(\{.*?(?P<q1>['\"])wid(?P=q1)\s*:\s*(?P<q2>['\"])_?(?P<partner_id>[^'\"]+)(?P=q2),.*?(?P<q3>['\"])entry_?[Ii]d(?P=q3)\s*:\s*(?P<q4>['\"])(?P<id>[^'\"]+)(?P=q4),", webpage) or
-                re.search(r'(?s)(?P<q1>["\'])(?:https?:)?//cdnapi(?:sec)?\.kaltura\.com/.*?(?:p|partner_id)/(?P<partner_id>\d+).*?(?P=q1).*?(?P<q2>["\'])?entry_?[Ii]d(?P=q2)\s*:\s*(?P<q3>["\'])(?P<id>.+?)(?P=q3)', webpage))
-        if mobj is not None:
-            return self.url_result(smuggle_url(
-                'kaltura:%(partner_id)s:%(id)s' % mobj.groupdict(),
-                {'source_url': url}), 'Kaltura')
+        kaltura_url = KalturaIE._extract_url(webpage)
+        if kaltura_url:
+            return self.url_result(smuggle_url(kaltura_url, {'source_url': url}), KalturaIE.ie_key())
 
         # Look for Eagle.Platform embeds
         mobj = re.search(

From 8704778d95d2abef021757c85efd75664c6a424a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 27 Jun 2016 23:06:42 +0700
Subject: [PATCH 5333/6219] [pbs] Check manually constructed http links (Closes
 #9921)

---
 youtube_dl/extractor/pbs.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 81918ac6e..f6f423597 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -516,9 +516,14 @@ class PBSIE(InfoExtractor):
                 # https://projects.pbs.org/confluence/display/coveapi/COVE+Video+Specifications
                 if not bitrate or bitrate not in ('400k', '800k', '1200k', '2500k'):
                     continue
+                f_url = re.sub(r'\d+k|baseline', bitrate, http_url)
+                # This may produce invalid links sometimes (e.g.
+                # http://www.pbs.org/wgbh/frontline/film/suicide-plan)
+                if not self._is_valid_url(f_url, display_id, 'http-%s video' % bitrate):
+                    continue
                 f = m3u8_format.copy()
                 f.update({
-                    'url': re.sub(r'\d+k|baseline', bitrate, http_url),
+                    'url': f_url,
                     'format_id': m3u8_format['format_id'].replace('hls', 'http'),
                     'protocol': 'http',
                 })

From 8174d0fe95db736f0fc53fd5e2d25c0c471a97fb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 27 Jun 2016 23:09:39 +0700
Subject: [PATCH 5334/6219] release 2016.06.27

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 docs/supportedsites.md    | 4 ++--
 youtube_dl/version.py     | 2 +-
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 63b687fef..f9a1aa990 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.06.26*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.06.26**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.06.27*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.06.27**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.06.26
+[debug] youtube-dl version 2016.06.27
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index a725e8c6b..2a94f4feb 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -45,7 +45,6 @@
  - **archive.org**: archive.org videos
  - **ARD**
  - **ARD:mediathek**
- - **ARD:mediathek**: Saarländischer Rundfunk
  - **arte.tv**
  - **arte.tv:+7**
  - **arte.tv:cinema**
@@ -588,7 +587,7 @@
  - **Shared**: shared.sx and vivo.sx
  - **ShareSix**
  - **Sina**
- - **skynewsarabia:video**
+ - **skynewsarabia:article**
  - **skynewsarabia:video**
  - **Slideshare**
  - **Slutload**
@@ -621,6 +620,7 @@
  - **SportBoxEmbed**
  - **SportDeutschland**
  - **Sportschau**
+ - **sr:mediathek**: Saarländischer Rundfunk
  - **SRGSSR**
  - **SRGSSRPlay**: srf.ch, rts.ch, rsi.ch, rtr.ch and swissinfo.ch play sites
  - **SSA**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 52de19517..2dd24dec1 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.06.26'
+__version__ = '2016.06.27'

From 32616c14ccf5051484e9236c54bdbeccb6e85ee4 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 28 Jun 2016 14:02:03 +0100
Subject: [PATCH 5335/6219] [vrt] extract all formats

---
 youtube_dl/extractor/vrt.py | 61 ++++++++++++++++++++++++++++++++++---
 1 file changed, 56 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/vrt.py b/youtube_dl/extractor/vrt.py
index 8e35f24e8..bec7ab327 100644
--- a/youtube_dl/extractor/vrt.py
+++ b/youtube_dl/extractor/vrt.py
@@ -25,7 +25,8 @@ class VRTIE(InfoExtractor):
                 'timestamp': 1414271750.949,
                 'upload_date': '20141025',
                 'duration': 929,
-            }
+            },
+            'skip': 'HTTP Error 404: Not Found',
         },
         # sporza.be
         {
@@ -39,7 +40,8 @@ class VRTIE(InfoExtractor):
                 'timestamp': 1413835980.560,
                 'upload_date': '20141020',
                 'duration': 3238,
-            }
+            },
+            'skip': 'HTTP Error 404: Not Found',
         },
         # cobra.be
         {
@@ -53,16 +55,39 @@ class VRTIE(InfoExtractor):
                 'timestamp': 1413967500.494,
                 'upload_date': '20141022',
                 'duration': 661,
-            }
+            },
+            'skip': 'HTTP Error 404: Not Found',
         },
         {
             # YouTube video
             'url': 'http://deredactie.be/cm/vrtnieuws/videozone/nieuws/cultuurenmedia/1.2622957',
-            'only_matching': True,
+            'md5': 'b8b93da1df1cea6c8556255a796b7d61',
+            'info_dict': {
+                'id': 'Wji-BZ0oCwg',
+                'ext': 'mp4',
+                'title': 'ROGUE ONE: A STAR WARS STORY Official Teaser Trailer',
+                'description': 'md5:8e468944dce15567a786a67f74262583',
+                'uploader': 'Star Wars',
+                'uploader_id': 'starwars',
+                'upload_date': '20160407',
+            },
+            'add_ie': ['Youtube'],
         },
         {
             'url': 'http://cobra.canvas.be/cm/cobra/videozone/rubriek/film-videozone/1.2377055',
-            'only_matching': True,
+            'md5': '',
+            'info_dict': {
+                'id': '2377055',
+                'ext': 'mp4',
+                'title': 'Cafe Derby',
+                'description': 'Lenny Van Wesemael debuteert met de langspeelfilm Café Derby. Een waar gebeurd maar ook verzonnen verhaal.',
+                'upload_date': '20150626',
+                'timestamp': 1435305240.769,
+            },
+            'params': {
+                # m3u8 download
+                'skip_download': True,
+            }
         }
     ]
 
@@ -98,6 +123,32 @@ class VRTIE(InfoExtractor):
                 formats.extend(self._extract_m3u8_formats(
                     src, video_id, 'mp4', entry_protocol='m3u8_native',
                     m3u8_id='hls', fatal=False))
+                formats.extend(self._extract_f4m_formats(
+                    src.replace('playlist.m3u8', 'manifest.f4m'),
+                    video_id, f4m_id='hds', fatal=False))
+                if 'data-video-geoblocking="true"' not in webpage:
+                    rtmp_formats = self._extract_smil_formats(
+                        src.replace('playlist.m3u8', 'jwplayer.smil'),
+                        video_id, fatal=False)
+                    formats.extend(rtmp_formats)
+                    for rtmp_format in rtmp_formats:
+                        rtmp_format_c = rtmp_format.copy()
+                        rtmp_format_c['url'] = '%s/%s' % (rtmp_format['url'], rtmp_format['play_path'])
+                        del rtmp_format_c['play_path']
+                        del rtmp_format_c['ext']
+                        http_format = rtmp_format_c.copy()
+                        http_format.update({
+                            'url': rtmp_format_c['url'].replace('rtmp://', 'http://').replace('vod.', 'download.').replace('/_definst_/', '/').replace('mp4:', ''),
+                            'format_id': rtmp_format['format_id'].replace('rtmp', 'http'),
+                            'protocol': 'http',
+                        })
+                        rtsp_format = rtmp_format_c.copy()
+                        rtsp_format.update({
+                            'url': rtsp_format['url'].replace('rtmp://', 'rtsp://'),
+                            'format_id': rtmp_format['format_id'].replace('rtmp', 'rtsp'),
+                            'protocol': 'rtsp',
+                        })
+                        formats.extend([http_format, rtsp_format])
             else:
                 formats.extend(self._extract_f4m_formats(
                     '%s/manifest.f4m' % src, video_id, f4m_id='hds', fatal=False))

From 97124e572db5f5d5d11ee630aeb18a4c5585d087 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 28 Jun 2016 22:39:53 +0700
Subject: [PATCH 5336/6219] [arte:playlist] Fix test

---
 youtube_dl/extractor/arte.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index 049f1fa9e..e0c5c1804 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -419,6 +419,7 @@ class ArteTVPlaylistIE(ArteTVBaseIE):
         'info_dict': {
             'id': 'PL-013263',
             'title': 'Areva & Uramin',
+            'description': 'md5:a1dc0312ce357c262259139cfd48c9bf',
         },
         'playlist_mincount': 6,
     }, {

From 42362fdb5e780611b7054e52eb28621f5a9fd7ba Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 29 Jun 2016 15:49:17 +0100
Subject: [PATCH 5337/6219] [aenetworks] add support for show and season for
 A&E Network sites and History topics(closes #9816)

---
 youtube_dl/extractor/aenetworks.py | 181 +++++++++++++++++++++--------
 youtube_dl/extractor/extractors.py |   5 +-
 2 files changed, 135 insertions(+), 51 deletions(-)

diff --git a/youtube_dl/extractor/aenetworks.py b/youtube_dl/extractor/aenetworks.py
index 1bbfe2641..cbde8b46e 100644
--- a/youtube_dl/extractor/aenetworks.py
+++ b/youtube_dl/extractor/aenetworks.py
@@ -7,18 +7,118 @@ from ..utils import (
     smuggle_url,
     update_url_query,
     unescapeHTML,
+    extract_attributes,
+)
+from ..compat import (
+    compat_urlparse,
 )
 
 
-class AENetworksIE(InfoExtractor):
+class AENetworksBaseIE(InfoExtractor):
+    def theplatform_url_result(self, theplatform_url, video_id, query):
+        return {
+            '_type': 'url_transparent',
+            'id': video_id,
+            'url': smuggle_url(
+                update_url_query(theplatform_url, query),
+                {
+                    'sig': {
+                        'key': 'crazyjava',
+                        'secret': 's3cr3t'
+                    },
+                    'force_smil_url': True
+                }),
+            'ie_key': 'ThePlatform',
+        }
+
+
+class AENetworksIE(AENetworksBaseIE):
     IE_NAME = 'aenetworks'
     IE_DESC = 'A+E Networks: A&E, Lifetime, History.com, FYI Network'
-    _VALID_URL = r'https?://(?:www\.)?(?:(?:history|aetv|mylifetime)\.com|fyi\.tv)/(?P<type>[^/]+)/(?:[^/]+/)+(?P<id>[^/]+?)(?:$|[?#])'
+    _VALID_URL = r'https?://(?:www\.)?(?:(?:history|aetv|mylifetime)\.com|fyi\.tv)/shows/(?P<id>[^/]+(?:/[^/]+){0,2})'
+    _TESTS = [{
+        'url': 'http://www.history.com/shows/mountain-men/season-1/episode-1',
+        'md5': '8ff93eb073449f151d6b90c0ae1ef0c7',
+        'info_dict': {
+            'id': '22253814',
+            'ext': 'mp4',
+            'title': 'Winter Is Coming',
+            'description': 'md5:641f424b7a19d8e24f26dea22cf59d74',
+            'timestamp': 1338306241,
+            'upload_date': '20120529',
+            'uploader': 'AENE-NEW',
+        },
+        'add_ie': ['ThePlatform'],
+    }, {
+        'url': 'http://www.history.com/shows/ancient-aliens/season-1',
+        'info_dict': {
+            'id': '71889446852',
+        },
+        'playlist_mincount': 5,
+    }, {
+        'url': 'http://www.mylifetime.com/shows/atlanta-plastic',
+        'info_dict': {
+            'id': 'SERIES4317',
+            'title': 'Atlanta Plastic',
+        },
+        'playlist_mincount': 2,
+    }, {
+        'url': 'http://www.aetv.com/shows/duck-dynasty/season-9/episode-1',
+        'only_matching': True
+    }, {
+        'url': 'http://www.fyi.tv/shows/tiny-house-nation/season-1/episode-8',
+        'only_matching': True
+    }, {
+        'url': 'http://www.mylifetime.com/shows/project-runway-junior/season-1/episode-6',
+        'only_matching': True
+    }]
 
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+        url_parts = display_id.split('/')
+        url_parts_len = len(url_parts)
+        if url_parts_len == 1:
+            entries = []
+            for season_url_path in re.findall(r'(?s)<li[^>]+data-href="(/shows/%s/season-\d+)"' % url_parts[0], webpage):
+                entries.append(self.url_result(
+                    compat_urlparse.urljoin(url, season_url_path), 'AENetworks'))
+            return self.playlist_result(
+                entries, self._html_search_meta('aetn:SeriesId', webpage),
+                self._html_search_meta('aetn:SeriesTitle', webpage))
+        elif url_parts_len == 2:
+            entries = []
+            for episode_item in re.findall(r'(?s)<div[^>]+class="[^"]*episode-item[^"]*"[^>]*>', webpage):
+                episode_attributes = extract_attributes(episode_item)
+                episode_url = compat_urlparse.urljoin(
+                    url, episode_attributes['data-canonical'])
+                entries.append(self.url_result(
+                    episode_url, 'AENetworks',
+                    episode_attributes['data-videoid']))
+            return self.playlist_result(
+                entries, self._html_search_meta('aetn:SeasonId', webpage))
+        else:
+            video_id = self._html_search_meta('aetn:VideoID', webpage)
+            media_url = self._search_regex(
+                r"media_url\s*=\s*'([^']+)'", webpage, 'video url')
+
+            info = self._search_json_ld(webpage, video_id, fatal=False)
+            info.update(self.theplatform_url_result(
+                media_url, video_id, {
+                    'mbr': 'true',
+                    'assetTypes': 'medium_video_s3'
+                }))
+            return info
+
+
+class HistoryTopicIE(AENetworksBaseIE):
+    IE_NAME = 'history:topic'
+    IE_DESC = 'History.com Topic'
+    _VALID_URL = r'https?://(?:www\.)?history\.com/topics/(?:[^/]+/)?(?P<topic_id>[^/]+)/videos(?:/(?P<display_id>[^/?#]+))?'
     _TESTS = [{
         'url': 'http://www.history.com/topics/valentines-day/history-of-valentines-day/videos/bet-you-didnt-know-valentines-day?m=528e394da93ae&s=undefined&f=1&free=false',
         'info_dict': {
-            'id': 'g12m5Gyt3fdR',
+            'id': '40700995724',
             'ext': 'mp4',
             'title': "Bet You Didn't Know: Valentine's Day",
             'description': 'md5:7b57ea4829b391995b405fa60bd7b5f7',
@@ -31,57 +131,38 @@ class AENetworksIE(InfoExtractor):
             'skip_download': True,
         },
         'add_ie': ['ThePlatform'],
-        'expected_warnings': ['JSON-LD'],
     }, {
-        'url': 'http://www.history.com/shows/mountain-men/season-1/episode-1',
-        'md5': '8ff93eb073449f151d6b90c0ae1ef0c7',
-        'info_dict': {
-            'id': 'eg47EERs_JsZ',
-            'ext': 'mp4',
-            'title': 'Winter Is Coming',
-            'description': 'md5:641f424b7a19d8e24f26dea22cf59d74',
-            'timestamp': 1338306241,
-            'upload_date': '20120529',
-            'uploader': 'AENE-NEW',
+        'url': 'http://www.history.com/topics/world-war-i/world-war-i-history/videos',
+        'info_dict':
+        {
+            'id': 'world-war-i-history',
         },
-        'add_ie': ['ThePlatform'],
+        'playlist_mincount': 24,
     }, {
-        'url': 'http://www.aetv.com/shows/duck-dynasty/video/inlawful-entry',
-        'only_matching': True
-    }, {
-        'url': 'http://www.fyi.tv/shows/tiny-house-nation/videos/207-sq-ft-minnesota-prairie-cottage',
-        'only_matching': True
-    }, {
-        'url': 'http://www.mylifetime.com/shows/project-runway-junior/video/season-1/episode-6/superstar-clients',
-        'only_matching': True
+        'url': 'http://www.history.com/topics/world-war-i-history/videos',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
-        page_type, video_id = re.match(self._VALID_URL, url).groups()
+        topic_id, display_id = re.match(self._VALID_URL, url).groups()
+        if display_id:
+            webpage = self._download_webpage(url, display_id)
+            release_url, video_id = re.search(r"_videoPlayer.play\('([^']+)'\s*,\s*'[^']+'\s*,\s*'(\d+)'\)", webpage).groups()
+            release_url = unescapeHTML(release_url)
 
-        webpage = self._download_webpage(url, video_id)
-
-        video_url_re = [
-            r'data-href="[^"]*/%s"[^>]+data-release-url="([^"]+)"' % video_id,
-            r"media_url\s*=\s*'([^']+)'"
-        ]
-        video_url = unescapeHTML(self._search_regex(video_url_re, webpage, 'video url'))
-        query = {'mbr': 'true'}
-        if page_type == 'shows':
-            query['assetTypes'] = 'medium_video_s3'
-        if 'switch=hds' in video_url:
-            query['switch'] = 'hls'
-
-        info = self._search_json_ld(webpage, video_id, fatal=False)
-        info.update({
-            '_type': 'url_transparent',
-            'url': smuggle_url(
-                update_url_query(video_url, query),
-                {
-                    'sig': {
-                        'key': 'crazyjava',
-                        'secret': 's3cr3t'},
-                    'force_smil_url': True
-                }),
-        })
-        return info
+            return self.theplatform_url_result(
+                release_url, video_id, {
+                    'mbr': 'true',
+                    'switch': 'hls'
+                })
+        else:
+            webpage = self._download_webpage(url, topic_id)
+            entries = []
+            for episode_item in re.findall(r'<a.+?data-release-url="[^"]+"[^>]*>', webpage):
+                video_attributes = extract_attributes(episode_item)
+                entries.append(self.theplatform_url_result(
+                    video_attributes['data-href'], video_attributes['data-id'], {
+                        'mbr': 'true',
+                        'switch': 'hls'
+                    }))
+            return self.playlist_result(entries, topic_id)
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 2f9ee1596..bba88e9eb 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -20,7 +20,10 @@ from .adobetv import (
     AdobeTVVideoIE,
 )
 from .adultswim import AdultSwimIE
-from .aenetworks import AENetworksIE
+from .aenetworks import (
+    AENetworksIE,
+    HistoryTopicIE,
+)
 from .afreecatv import AfreecaTVIE
 from .aftonbladet import AftonbladetIE
 from .airmozilla import AirMozillaIE

From 4c7821227c54836a17d9c02d4f8d3dcbd97105fc Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 29 Jun 2016 16:03:32 +0100
Subject: [PATCH 5338/6219] [aenetworks:historytopic] fix topic video url

---
 youtube_dl/extractor/aenetworks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/aenetworks.py b/youtube_dl/extractor/aenetworks.py
index cbde8b46e..1376dd70f 100644
--- a/youtube_dl/extractor/aenetworks.py
+++ b/youtube_dl/extractor/aenetworks.py
@@ -161,7 +161,7 @@ class HistoryTopicIE(AENetworksBaseIE):
             for episode_item in re.findall(r'<a.+?data-release-url="[^"]+"[^>]*>', webpage):
                 video_attributes = extract_attributes(episode_item)
                 entries.append(self.theplatform_url_result(
-                    video_attributes['data-href'], video_attributes['data-id'], {
+                    video_attributes['data-release-url'], video_attributes['data-id'], {
                         'mbr': 'true',
                         'switch': 'hls'
                     }))

From c58ed8563d37e39235332b35e7feafe32711c623 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 29 Jun 2016 16:18:16 +0100
Subject: [PATCH 5339/6219] [aenetworks] extract history topic playlist title

---
 youtube_dl/extractor/aenetworks.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/aenetworks.py b/youtube_dl/extractor/aenetworks.py
index 1376dd70f..2536f75d6 100644
--- a/youtube_dl/extractor/aenetworks.py
+++ b/youtube_dl/extractor/aenetworks.py
@@ -8,6 +8,7 @@ from ..utils import (
     update_url_query,
     unescapeHTML,
     extract_attributes,
+    get_element_by_attribute,
 )
 from ..compat import (
     compat_urlparse,
@@ -136,6 +137,7 @@ class HistoryTopicIE(AENetworksBaseIE):
         'info_dict':
         {
             'id': 'world-war-i-history',
+            'title': 'World War I History',
         },
         'playlist_mincount': 24,
     }, {
@@ -165,4 +167,4 @@ class HistoryTopicIE(AENetworksBaseIE):
                         'mbr': 'true',
                         'switch': 'hls'
                     }))
-            return self.playlist_result(entries, topic_id)
+            return self.playlist_result(entries, topic_id, get_element_by_attribute('class', 'show-title', webpage))

From 70157c2c43068b23007a6d71e16967ba85b274d5 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 29 Jun 2016 16:55:17 +0100
Subject: [PATCH 5340/6219] [aenetworks] add support for movie pages

---
 youtube_dl/extractor/aenetworks.py | 80 ++++++++++++++++--------------
 1 file changed, 42 insertions(+), 38 deletions(-)

diff --git a/youtube_dl/extractor/aenetworks.py b/youtube_dl/extractor/aenetworks.py
index 2536f75d6..8b60e2ab6 100644
--- a/youtube_dl/extractor/aenetworks.py
+++ b/youtube_dl/extractor/aenetworks.py
@@ -36,7 +36,7 @@ class AENetworksBaseIE(InfoExtractor):
 class AENetworksIE(AENetworksBaseIE):
     IE_NAME = 'aenetworks'
     IE_DESC = 'A+E Networks: A&E, Lifetime, History.com, FYI Network'
-    _VALID_URL = r'https?://(?:www\.)?(?:(?:history|aetv|mylifetime)\.com|fyi\.tv)/shows/(?P<id>[^/]+(?:/[^/]+){0,2})'
+    _VALID_URL = r'https?://(?:www\.)?(?:(?:history|aetv|mylifetime)\.com|fyi\.tv)/(?:shows/(?P<show_path>[^/]+(?:/[^/]+){0,2})|movies/(?P<movie_display_id>[^/]+)/full-movie)'
     _TESTS = [{
         'url': 'http://www.history.com/shows/mountain-men/season-1/episode-1',
         'md5': '8ff93eb073449f151d6b90c0ae1ef0c7',
@@ -72,50 +72,54 @@ class AENetworksIE(AENetworksBaseIE):
     }, {
         'url': 'http://www.mylifetime.com/shows/project-runway-junior/season-1/episode-6',
         'only_matching': True
+    }, {
+        'url': 'http://www.mylifetime.com/movies/center-stage-on-pointe/full-movie',
+        'only_matching': True
     }]
 
     def _real_extract(self, url):
-        display_id = self._match_id(url)
+        show_path, movie_display_id = re.match(self._VALID_URL, url).groups()
+        display_id = show_path or movie_display_id
         webpage = self._download_webpage(url, display_id)
-        url_parts = display_id.split('/')
-        url_parts_len = len(url_parts)
-        if url_parts_len == 1:
-            entries = []
-            for season_url_path in re.findall(r'(?s)<li[^>]+data-href="(/shows/%s/season-\d+)"' % url_parts[0], webpage):
-                entries.append(self.url_result(
-                    compat_urlparse.urljoin(url, season_url_path), 'AENetworks'))
-            return self.playlist_result(
-                entries, self._html_search_meta('aetn:SeriesId', webpage),
-                self._html_search_meta('aetn:SeriesTitle', webpage))
-        elif url_parts_len == 2:
-            entries = []
-            for episode_item in re.findall(r'(?s)<div[^>]+class="[^"]*episode-item[^"]*"[^>]*>', webpage):
-                episode_attributes = extract_attributes(episode_item)
-                episode_url = compat_urlparse.urljoin(
-                    url, episode_attributes['data-canonical'])
-                entries.append(self.url_result(
-                    episode_url, 'AENetworks',
-                    episode_attributes['data-videoid']))
-            return self.playlist_result(
-                entries, self._html_search_meta('aetn:SeasonId', webpage))
-        else:
-            video_id = self._html_search_meta('aetn:VideoID', webpage)
-            media_url = self._search_regex(
-                r"media_url\s*=\s*'([^']+)'", webpage, 'video url')
+        if show_path:
+            url_parts = show_path.split('/')
+            url_parts_len = len(url_parts)
+            if url_parts_len == 1:
+                entries = []
+                for season_url_path in re.findall(r'(?s)<li[^>]+data-href="(/shows/%s/season-\d+)"' % url_parts[0], webpage):
+                    entries.append(self.url_result(
+                        compat_urlparse.urljoin(url, season_url_path), 'AENetworks'))
+                return self.playlist_result(
+                    entries, self._html_search_meta('aetn:SeriesId', webpage),
+                    self._html_search_meta('aetn:SeriesTitle', webpage))
+            elif url_parts_len == 2:
+                entries = []
+                for episode_item in re.findall(r'(?s)<div[^>]+class="[^"]*episode-item[^"]*"[^>]*>', webpage):
+                    episode_attributes = extract_attributes(episode_item)
+                    episode_url = compat_urlparse.urljoin(
+                        url, episode_attributes['data-canonical'])
+                    entries.append(self.url_result(
+                        episode_url, 'AENetworks',
+                        episode_attributes['data-videoid']))
+                return self.playlist_result(
+                    entries, self._html_search_meta('aetn:SeasonId', webpage))
+        video_id = self._html_search_meta('aetn:VideoID', webpage)
+        media_url = self._search_regex(
+            r"media_url\s*=\s*'([^']+)'", webpage, 'video url')
 
-            info = self._search_json_ld(webpage, video_id, fatal=False)
-            info.update(self.theplatform_url_result(
-                media_url, video_id, {
-                    'mbr': 'true',
-                    'assetTypes': 'medium_video_s3'
-                }))
-            return info
+        info = self._search_json_ld(webpage, video_id, fatal=False)
+        info.update(self.theplatform_url_result(
+            media_url, video_id, {
+                'mbr': 'true',
+                'assetTypes': 'medium_video_s3'
+            }))
+        return info
 
 
 class HistoryTopicIE(AENetworksBaseIE):
     IE_NAME = 'history:topic'
     IE_DESC = 'History.com Topic'
-    _VALID_URL = r'https?://(?:www\.)?history\.com/topics/(?:[^/]+/)?(?P<topic_id>[^/]+)/videos(?:/(?P<display_id>[^/?#]+))?'
+    _VALID_URL = r'https?://(?:www\.)?history\.com/topics/(?:[^/]+/)?(?P<topic_id>[^/]+)/videos(?:/(?P<video_display_id>[^/?#]+))?'
     _TESTS = [{
         'url': 'http://www.history.com/topics/valentines-day/history-of-valentines-day/videos/bet-you-didnt-know-valentines-day?m=528e394da93ae&s=undefined&f=1&free=false',
         'info_dict': {
@@ -146,9 +150,9 @@ class HistoryTopicIE(AENetworksBaseIE):
     }]
 
     def _real_extract(self, url):
-        topic_id, display_id = re.match(self._VALID_URL, url).groups()
-        if display_id:
-            webpage = self._download_webpage(url, display_id)
+        topic_id, video_display_id = re.match(self._VALID_URL, url).groups()
+        if video_display_id:
+            webpage = self._download_webpage(url, video_display_id)
             release_url, video_id = re.search(r"_videoPlayer.play\('([^']+)'\s*,\s*'[^']+'\s*,\s*'(\d+)'\)", webpage).groups()
             release_url = unescapeHTML(release_url)
 

From 06a96da15bfde93a2f2aa17cdaa10e1bf11dde0c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 29 Jun 2016 23:01:34 +0700
Subject: [PATCH 5341/6219] [eagleplatform] Improve embed detection and extract
 in separate routine (Closes #9926)

---
 youtube_dl/extractor/eagleplatform.py | 8 ++++++++
 youtube_dl/extractor/generic.py       | 8 ++++----
 2 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/eagleplatform.py b/youtube_dl/extractor/eagleplatform.py
index 113a4966f..12d28d3b9 100644
--- a/youtube_dl/extractor/eagleplatform.py
+++ b/youtube_dl/extractor/eagleplatform.py
@@ -50,6 +50,14 @@ class EaglePlatformIE(InfoExtractor):
         'skip': 'Georestricted',
     }]
 
+    @staticmethod
+    def _extract_url(webpage):
+        mobj = re.search(
+            r'<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//.+?\.media\.eagleplatform\.com/index/player\?.+?)\1',
+            webpage)
+        if mobj is not None:
+            return mobj.group('url')
+
     @staticmethod
     def _handle_error(response):
         status = int_or_none(response.get('status', 200))
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 2188f8bb2..712dd8a94 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -65,6 +65,7 @@ from .threeqsdn import ThreeQSDNIE
 from .theplatform import ThePlatformIE
 from .vessel import VesselIE
 from .kaltura import KalturaIE
+from .eagleplatform import EaglePlatformIE
 
 
 class GenericIE(InfoExtractor):
@@ -1932,10 +1933,9 @@ class GenericIE(InfoExtractor):
             return self.url_result(smuggle_url(kaltura_url, {'source_url': url}), KalturaIE.ie_key())
 
         # Look for Eagle.Platform embeds
-        mobj = re.search(
-            r'<iframe[^>]+src="(?P<url>https?://.+?\.media\.eagleplatform\.com/index/player\?.+?)"', webpage)
-        if mobj is not None:
-            return self.url_result(mobj.group('url'), 'EaglePlatform')
+        eagleplatform_url = EaglePlatformIE._extract_url(webpage)
+        if eagleplatform_url:
+            return self.url_result(eagleplatform_url, EaglePlatformIE.ie_key())
 
         # Look for ClipYou (uses Eagle.Platform) embeds
         mobj = re.search(

From e496fa50cd82877d8daeda8e29056c5d7fce2de0 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 29 Jun 2016 20:19:31 +0100
Subject: [PATCH 5342/6219] [urplay] Add new extractor(closes #9332)

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/urplay.py     | 67 ++++++++++++++++++++++++++++++
 2 files changed, 68 insertions(+)
 create mode 100644 youtube_dl/extractor/urplay.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index bba88e9eb..eeedc675b 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -894,6 +894,7 @@ from .udn import UDNEmbedIE
 from .digiteka import DigitekaIE
 from .unistra import UnistraIE
 from .urort import UrortIE
+from .urplay import URPlayIE
 from .usatoday import USATodayIE
 from .ustream import UstreamIE, UstreamChannelIE
 from .ustudio import (
diff --git a/youtube_dl/extractor/urplay.py b/youtube_dl/extractor/urplay.py
new file mode 100644
index 000000000..24ecdd2b1
--- /dev/null
+++ b/youtube_dl/extractor/urplay.py
@@ -0,0 +1,67 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class URPlayIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?urplay\.se/program/(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'http://urplay.se/program/190031-tripp-trapp-trad-sovkudde',
+        'md5': '15ca67b63fd8fb320ac2bcd854bad7b6',
+        'info_dict': {
+            'id': '190031',
+            'ext': 'mp4',
+            'title': 'Tripp, Trapp, Träd : Sovkudde',
+            'description': 'md5:b86bffdae04a7e9379d1d7e5947df1d1',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+        urplayer_data = self._parse_json(self._search_regex(
+            r'urPlayer.init\(({.+?})\);', webpage, 'urplayer data'), video_id)
+        host = self._download_json('http://streaming-loadbalancer.ur.se/loadbalancer.json', video_id)['redirect']
+
+        formats = []
+        for quality_attr, quality, preference in (('', 'sd', 0), ('_hd', 'hd', 1)):
+            file_rtmp = urplayer_data.get('file_rtmp' + quality_attr)
+            if file_rtmp:
+                formats.append({
+                    'url': 'rtmp://%s/urplay/mp4:%s' % (host, file_rtmp),
+                    'format_id': quality + '-rtmp',
+                    'ext': 'flv',
+                    'preference': preference,
+                })
+            file_http = urplayer_data.get('file_http' + quality_attr) or urplayer_data.get('file_http_sub' + quality_attr)
+            if file_http:
+                file_http_base_url = 'http://%s/%s' % (host, file_http)
+                formats.extend(self._extract_f4m_formats(
+                    file_http_base_url + 'manifest.f4m', video_id,
+                    preference, '%s-hds' % quality, fatal=False))
+                formats.extend(self._extract_m3u8_formats(
+                    file_http_base_url + 'playlist.m3u8', video_id, 'mp4',
+                    'm3u8_native', preference, '%s-hls' % quality, fatal=False))
+        self._sort_formats(formats)
+
+        subtitles = {}
+        for subtitle in urplayer_data.get('subtitles', []):
+            subtitle_url = subtitle.get('file')
+            kind = subtitle.get('kind')
+            if subtitle_url or kind and kind != 'captions':
+                continue
+            subtitles.setdefault(subtitle.get('label', 'Svenska'), []).append({
+                'url': subtitle_url,
+            })
+
+        return {
+            'id': video_id,
+            'title': urplayer_data['title'],
+            'description': self._og_search_description(webpage),
+            'thumbnail': urplayer_data.get('image'),
+            'series': urplayer_data.get('series_title'),
+            'subtitles': subtitles,
+            'formats': formats,
+        }
\ No newline at end of file

From 397b305cfe1a7ec2957331602727edb009c71e99 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 30 Jun 2016 00:21:03 +0100
Subject: [PATCH 5343/6219] [meta] Add new extractor(closes #8789)

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/meta.py       | 56 ++++++++++++++++++++++++++++++
 2 files changed, 57 insertions(+)
 create mode 100644 youtube_dl/extractor/meta.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index eeedc675b..84c39ab48 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -425,6 +425,7 @@ from .makerschannel import MakersChannelIE
 from .makertv import MakerTVIE
 from .matchtv import MatchTVIE
 from .mdr import MDRIE
+from .meta import METAIE
 from .metacafe import MetacafeIE
 from .metacritic import MetacriticIE
 from .mgoon import MgoonIE
diff --git a/youtube_dl/extractor/meta.py b/youtube_dl/extractor/meta.py
new file mode 100644
index 000000000..674b8d264
--- /dev/null
+++ b/youtube_dl/extractor/meta.py
@@ -0,0 +1,56 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    unescapeHTML,
+    int_or_none,
+    ExtractorError,
+)
+
+
+class METAIE(InfoExtractor):
+    _VALID_URL = r'https?://video\.meta\.ua/(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'http://video.meta.ua/5502115.video',
+        'md5': '71b6f3ee274bef16f1ab410f7f56b476',
+        'info_dict': {
+            'id': '5502115',
+            'ext': 'mp4',
+            'title': 'Sony Xperia Z camera test [HQ]',
+            'description': 'Xperia Z shoots video in FullHD HDR.',
+            'uploader_id': 'nomobile',
+            'uploader': 'CHЁZA.TV',
+            'upload_date': '20130211',
+        },
+        'add_ie': ['Youtube'],
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        st_html5 = self._search_regex(r"st_html5\s*=\s*'#([^']+)'", webpage, 'uppod html5 st')
+        json_str = ''
+        for i in range(0, len(st_html5), 3):
+            json_str += '&#x0%s;' % st_html5[i:i + 3]
+        uppod_data = self._parse_json(unescapeHTML(json_str), video_id)
+        error = uppod_data.get('customnotfound')
+        if error:
+            raise ExtractorError('%s said: %s' % (self.IE_NAME, error), expected=True)
+
+        video_url = uppod_data['file']
+        info = {
+            'id': video_id,
+            'url': video_url,
+            'title': uppod_data.get('comment') or self._og_search_title(webpage),
+            'description': self._og_search_description(webpage),
+            'thumbnail': uppod_data.get('poster') or self._og_search_thumbnail(webpage),
+            'duration': int_or_none(self._og_search_property('video:duration', webpage)),
+        }
+        if 'youtube.com/' in video_url:
+            info.update({
+                '_type': 'url_transparent',
+                'ie_key': 'Youtube',
+            })
+        return info

From df43389ade6e7a6394521ae91c0640508dceb4dc Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 30 Jun 2016 02:54:21 +0100
Subject: [PATCH 5344/6219] [skysports] Add new extractor(closes #7066)

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/skysports.py  | 33 ++++++++++++++++++++++++++++++
 2 files changed, 34 insertions(+)
 create mode 100644 youtube_dl/extractor/skysports.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 84c39ab48..80d1bbe20 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -714,6 +714,7 @@ from .skynewsarabia import (
     SkyNewsArabiaIE,
     SkyNewsArabiaArticleIE,
 )
+from .skysports import SkySportsIE
 from .slideshare import SlideshareIE
 from .slutload import SlutloadIE
 from .smotri import (
diff --git a/youtube_dl/extractor/skysports.py b/youtube_dl/extractor/skysports.py
new file mode 100644
index 000000000..9dc78c7d2
--- /dev/null
+++ b/youtube_dl/extractor/skysports.py
@@ -0,0 +1,33 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class SkySportsIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?skysports\.com/watch/video/(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'http://www.skysports.com/watch/video/10328419/bale-its-our-time-to-shine',
+        'md5': 'c44a1db29f27daf9a0003e010af82100',
+        'info_dict': {
+            'id': '10328419',
+            'ext': 'flv',
+            'title': 'Bale: Its our time to shine',
+            'description': 'md5:9fd1de3614d525f5addda32ac3c482c9',
+        },
+        'add_ie': ['Ooyala'],
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        return {
+            '_type': 'url_transparent',
+            'id': video_id,
+            'url': 'ooyala:%s' % self._search_regex(
+                r'data-video-id="([^"]+)"', webpage, 'ooyala id'),
+            'title': self._og_search_title(webpage),
+            'description': self._og_search_description(webpage),
+            'ie_key': 'Ooyala',
+        }

From ab47b6e881269a0329b78a294318745a54e9e7c7 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 30 Jun 2016 04:08:24 +0100
Subject: [PATCH 5345/6219] [theatlantic] Add new extractor(closes #6611)

---
 youtube_dl/extractor/extractors.py  |  1 +
 youtube_dl/extractor/theatlantic.py | 40 +++++++++++++++++++++++++++++
 2 files changed, 41 insertions(+)
 create mode 100644 youtube_dl/extractor/theatlantic.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 80d1bbe20..d9ffde449 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -799,6 +799,7 @@ from .teletask import TeleTaskIE
 from .telewebion import TelewebionIE
 from .testurl import TestURLIE
 from .tf1 import TF1IE
+from .theatlantic import TheAtlanticIE
 from .theintercept import TheInterceptIE
 from .theplatform import (
     ThePlatformIE,
diff --git a/youtube_dl/extractor/theatlantic.py b/youtube_dl/extractor/theatlantic.py
new file mode 100644
index 000000000..df4254fea
--- /dev/null
+++ b/youtube_dl/extractor/theatlantic.py
@@ -0,0 +1,40 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class TheAtlanticIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?theatlantic\.com/video/index/(?P<id>\d+)'
+    _TEST = {
+        'url': 'http://www.theatlantic.com/video/index/477918/capture-a-unified-theory-on-mental-health/',
+        'md5': '',
+        'info_dict': {
+            'id': '477918',
+            'ext': 'mp4',
+            'title': 'Are All Mental Illnesses Related?',
+            'description': 'Depression, anxiety, overeating, addiction, and all other mental disorders share a common mechanism.',
+            'timestamp': 1460490952,
+            'uploader': 'TheAtlantic',
+            'upload_date': '20160412',
+            'uploader_id': '29913724001',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+        'add_ie': ['BrightcoveLegacy'],
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        return {
+            '_type': 'url_transparent',
+            'url': self._html_search_meta('twitter:player', webpage),
+            'id': video_id,
+            'title': self._og_search_title(webpage),
+            'description': self._og_search_description(webpage),
+            'thumbnail': self._og_search_thumbnail(webpage),
+            'ie_key': 'BrightcoveLegacy',
+        }

From 4d86d2008eeae5d4e75d8f688a666e7b9504bbeb Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 30 Jun 2016 11:30:42 +0100
Subject: [PATCH 5346/6219] [urplay] fix typo and check with flake8

---
 youtube_dl/extractor/urplay.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/urplay.py b/youtube_dl/extractor/urplay.py
index 24ecdd2b1..ce3bf6b02 100644
--- a/youtube_dl/extractor/urplay.py
+++ b/youtube_dl/extractor/urplay.py
@@ -22,7 +22,7 @@ class URPlayIE(InfoExtractor):
 
         webpage = self._download_webpage(url, video_id)
         urplayer_data = self._parse_json(self._search_regex(
-            r'urPlayer.init\(({.+?})\);', webpage, 'urplayer data'), video_id)
+            r'urPlayer\.init\(({.+?})\);', webpage, 'urplayer data'), video_id)
         host = self._download_json('http://streaming-loadbalancer.ur.se/loadbalancer.json', video_id)['redirect']
 
         formats = []
@@ -64,4 +64,4 @@ class URPlayIE(InfoExtractor):
             'series': urplayer_data.get('series_title'),
             'subtitles': subtitles,
             'formats': formats,
-        }
\ No newline at end of file
+        }

From 329179073b93e37ab76e759d1fe96d8f984367f3 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 30 Jun 2016 12:01:30 +0100
Subject: [PATCH 5347/6219] [generic] add generic support for twitter:player
 embeds

---
 youtube_dl/extractor/extractors.py  |  1 -
 youtube_dl/extractor/generic.py     | 21 +++++++++++++++
 youtube_dl/extractor/theatlantic.py | 40 -----------------------------
 3 files changed, 21 insertions(+), 41 deletions(-)
 delete mode 100644 youtube_dl/extractor/theatlantic.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index d9ffde449..80d1bbe20 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -799,7 +799,6 @@ from .teletask import TeleTaskIE
 from .telewebion import TelewebionIE
 from .testurl import TestURLIE
 from .tf1 import TF1IE
-from .theatlantic import TheAtlanticIE
 from .theintercept import TheInterceptIE
 from .theplatform import (
     ThePlatformIE,
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 712dd8a94..c2a7f9202 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1245,6 +1245,22 @@ class GenericIE(InfoExtractor):
                 'uploader': 'www.hudl.com',
             },
         },
+        # twitter:player embed
+        {
+            'url': 'http://www.theatlantic.com/video/index/484130/what-do-black-holes-sound-like/',
+            'md5': 'a3e0df96369831de324f0778e126653c',
+            'info_dict': {
+                'id': '4909620399001',
+                'ext': 'mp4',
+                'title': 'What Do Black Holes Sound Like?',
+                'description': 'what do black holes sound like',
+                'upload_date': '20160524',
+                'uploader_id': '29913724001',
+                'timestamp': 1464107587,
+                'uploader': 'TheAtlantic',
+            },
+            'add_ie': ['BrightcoveLegacy'],
+        }
     ]
 
     def report_following_redirect(self, new_url):
@@ -2081,6 +2097,11 @@ class GenericIE(InfoExtractor):
                 'uploader': video_uploader,
             }
 
+        # https://dev.twitter.com/cards/types/player#On_twitter.com_via_desktop_browser
+        embed_url = self._twitter_search_player(webpage)
+        if embed_url:
+            return self.url_result(embed_url)
+
         def check_video(vurl):
             if YoutubeIE.suitable(vurl):
                 return True
diff --git a/youtube_dl/extractor/theatlantic.py b/youtube_dl/extractor/theatlantic.py
deleted file mode 100644
index df4254fea..000000000
--- a/youtube_dl/extractor/theatlantic.py
+++ /dev/null
@@ -1,40 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-from .common import InfoExtractor
-
-
-class TheAtlanticIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?theatlantic\.com/video/index/(?P<id>\d+)'
-    _TEST = {
-        'url': 'http://www.theatlantic.com/video/index/477918/capture-a-unified-theory-on-mental-health/',
-        'md5': '',
-        'info_dict': {
-            'id': '477918',
-            'ext': 'mp4',
-            'title': 'Are All Mental Illnesses Related?',
-            'description': 'Depression, anxiety, overeating, addiction, and all other mental disorders share a common mechanism.',
-            'timestamp': 1460490952,
-            'uploader': 'TheAtlantic',
-            'upload_date': '20160412',
-            'uploader_id': '29913724001',
-        },
-        'params': {
-            # m3u8 download
-            'skip_download': True,
-        },
-        'add_ie': ['BrightcoveLegacy'],
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-        return {
-            '_type': 'url_transparent',
-            'url': self._html_search_meta('twitter:player', webpage),
-            'id': video_id,
-            'title': self._og_search_title(webpage),
-            'description': self._og_search_description(webpage),
-            'thumbnail': self._og_search_thumbnail(webpage),
-            'ie_key': 'BrightcoveLegacy',
-        }

From 93ad6c6bfaae8f1ce87a832ece92fa099f0e2095 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 30 Jun 2016 13:50:49 +0100
Subject: [PATCH 5348/6219] [sixplay] Add new extractor(closes #2183)

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/m6.py         | 35 +----------------
 youtube_dl/extractor/sixplay.py    | 60 ++++++++++++++++++++++++++++++
 3 files changed, 63 insertions(+), 33 deletions(-)
 create mode 100644 youtube_dl/extractor/sixplay.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 80d1bbe20..a7b110450 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -710,6 +710,7 @@ from .shahid import ShahidIE
 from .shared import SharedIE
 from .sharesix import ShareSixIE
 from .sina import SinaIE
+from .sixplay import SixPlayIE
 from .skynewsarabia import (
     SkyNewsArabiaIE,
     SkyNewsArabiaArticleIE,
diff --git a/youtube_dl/extractor/m6.py b/youtube_dl/extractor/m6.py
index d5945ad66..39d2742c8 100644
--- a/youtube_dl/extractor/m6.py
+++ b/youtube_dl/extractor/m6.py
@@ -1,8 +1,6 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 
 
@@ -23,34 +21,5 @@ class M6IE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
-        rss = self._download_xml('http://ws.m6.fr/v1/video/info/m6/bonus/%s' % video_id, video_id,
-                                 'Downloading video RSS')
-
-        title = rss.find('./channel/item/title').text
-        description = rss.find('./channel/item/description').text
-        thumbnail = rss.find('./channel/item/visuel_clip_big').text
-        duration = int(rss.find('./channel/item/duration').text)
-        view_count = int(rss.find('./channel/item/nombre_vues').text)
-
-        formats = []
-        for format_id in ['lq', 'sd', 'hq', 'hd']:
-            video_url = rss.find('./channel/item/url_video_%s' % format_id)
-            if video_url is None:
-                continue
-            formats.append({
-                'url': video_url.text,
-                'format_id': format_id,
-            })
-
-        return {
-            'id': video_id,
-            'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
-            'duration': duration,
-            'view_count': view_count,
-            'formats': formats,
-        }
+        video_id = self._match_id(url)
+        return self.url_result('6play:%s' % video_id, 'SixPlay', video_id)
diff --git a/youtube_dl/extractor/sixplay.py b/youtube_dl/extractor/sixplay.py
new file mode 100644
index 000000000..f855a1a00
--- /dev/null
+++ b/youtube_dl/extractor/sixplay.py
@@ -0,0 +1,60 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    qualities,
+    int_or_none,
+)
+
+
+class SixPlayIE(InfoExtractor):
+    _VALID_URL = r'(?:6play:|https?://(?:www\.)?6play\.fr/.+?-c_)(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'http://www.6play.fr/jamel-et-ses-amis-au-marrakech-du-rire-p_1316/jamel-et-ses-amis-au-marrakech-du-rire-2015-c_11495320',
+        'md5': '42310bffe4ba3982db112b9cd3467328',
+        'info_dict': {
+            'id': '11495320',
+            'ext': 'mp4',
+            'title': 'Jamel et ses amis au Marrakech du rire 2015',
+            'description': 'md5:ba2149d5c321d5201b78070ee839d872',
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        clip_data = self._download_json(
+            'https://player.m6web.fr/v2/video/config/6play-auth/FR/%s.json' % video_id,
+            video_id)
+        video_data = clip_data['videoInfo']
+
+        preference = qualities(['lq', 'sd', 'hq', 'hd'])
+        formats = []
+        for source in clip_data['sources']:
+            source_type, source_url = source.get('type'), source.get('src')
+            if not source_url or source_type == 'hls/primetime':
+                continue
+            if source_type == 'application/vnd.apple.mpegURL':
+                formats.extend(self._extract_m3u8_formats(
+                    source_url, video_id, 'mp4', 'm3u8_native',
+                    m3u8_id='hls', fatal=False))
+                formats.extend(self._extract_f4m_formats(
+                    source_url.replace('.m3u8', '.f4m'),
+                    video_id, f4m_id='hds', fatal=False))
+            elif source_type == 'video/mp4':
+                quality = source.get('quality')
+                formats.append({
+                    'url': source_url,
+                    'format_id': quality,
+                    'preference': preference(quality),
+                })
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': video_data['title'].strip(),
+            'description': video_data.get('description'),
+            'duration': int_or_none(video_data.get('duration')),
+            'series': video_data.get('titlePgm'),
+            'formats': formats,
+        }

From 7dbeee7e229a357cfc8acf0a908b10f3f326cd96 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 30 Jun 2016 14:11:55 +0100
Subject: [PATCH 5349/6219] [generic] make twitter:player extraction non fatal

---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index c2a7f9202..9315b9e21 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -2098,7 +2098,7 @@ class GenericIE(InfoExtractor):
             }
 
         # https://dev.twitter.com/cards/types/player#On_twitter.com_via_desktop_browser
-        embed_url = self._twitter_search_player(webpage)
+        embed_url = self._html_search_meta('twitter:player', webpage, default=None)
         if embed_url:
             return self.url_result(embed_url)
 

From 049da7cb6cc7d6b47020480fa780907be265b9cf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 30 Jun 2016 23:04:18 +0700
Subject: [PATCH 5350/6219] [meta] Extend _VALID_URL

---
 youtube_dl/extractor/meta.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/meta.py b/youtube_dl/extractor/meta.py
index 674b8d264..2ca7092e5 100644
--- a/youtube_dl/extractor/meta.py
+++ b/youtube_dl/extractor/meta.py
@@ -10,8 +10,8 @@ from ..utils import (
 
 
 class METAIE(InfoExtractor):
-    _VALID_URL = r'https?://video\.meta\.ua/(?P<id>[0-9]+)'
-    _TEST = {
+    _VALID_URL = r'https?://video\.meta\.ua/(?:iframe/)?(?P<id>[0-9]+)'
+    _TESTS = [{
         'url': 'http://video.meta.ua/5502115.video',
         'md5': '71b6f3ee274bef16f1ab410f7f56b476',
         'info_dict': {
@@ -24,7 +24,10 @@ class METAIE(InfoExtractor):
             'upload_date': '20130211',
         },
         'add_ie': ['Youtube'],
-    }
+    }, {
+        'url': 'http://video.meta.ua/iframe/5502115',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From eafa643715c0989dff927c9a44e837ca62247b4e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 30 Jun 2016 23:06:13 +0700
Subject: [PATCH 5351/6219] [meta] Make duration and description optional

For iframe URLs
---
 youtube_dl/extractor/meta.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/meta.py b/youtube_dl/extractor/meta.py
index 2ca7092e5..2e2db5620 100644
--- a/youtube_dl/extractor/meta.py
+++ b/youtube_dl/extractor/meta.py
@@ -47,9 +47,10 @@ class METAIE(InfoExtractor):
             'id': video_id,
             'url': video_url,
             'title': uppod_data.get('comment') or self._og_search_title(webpage),
-            'description': self._og_search_description(webpage),
+            'description': self._og_search_description(webpage, default=None),
             'thumbnail': uppod_data.get('poster') or self._og_search_thumbnail(webpage),
-            'duration': int_or_none(self._og_search_property('video:duration', webpage)),
+            'duration': int_or_none(self._og_search_property(
+                'video:duration', webpage, default=None)),
         }
         if 'youtube.com/' in video_url:
             info.update({

From 8ff6697861b918ff9221c7dd46a6e1109ad0ef34 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 30 Jun 2016 23:19:29 +0700
Subject: [PATCH 5352/6219] [pladform] Improve embed detection

---
 youtube_dl/extractor/pladform.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pladform.py b/youtube_dl/extractor/pladform.py
index bc559d1df..77e1211d6 100644
--- a/youtube_dl/extractor/pladform.py
+++ b/youtube_dl/extractor/pladform.py
@@ -49,7 +49,7 @@ class PladformIE(InfoExtractor):
     @staticmethod
     def _extract_url(webpage):
         mobj = re.search(
-            r'<iframe[^>]+src="(?P<url>(?:https?:)?//out\.pladform\.ru/player\?.+?)"', webpage)
+            r'<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//out\.pladform\.ru/player\?.+?)\1', webpage)
         if mobj:
             return mobj.group('url')
 

From fd94e2671a8b2174d38f9e81b0956e31f90df326 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 30 Jun 2016 23:20:44 +0700
Subject: [PATCH 5353/6219] [meta] Add support for pladform embeds

---
 youtube_dl/extractor/meta.py | 60 +++++++++++++++++++++---------------
 1 file changed, 36 insertions(+), 24 deletions(-)

diff --git a/youtube_dl/extractor/meta.py b/youtube_dl/extractor/meta.py
index 2e2db5620..42bedc48f 100644
--- a/youtube_dl/extractor/meta.py
+++ b/youtube_dl/extractor/meta.py
@@ -2,6 +2,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from .pladform import PladformIE
 from ..utils import (
     unescapeHTML,
     int_or_none,
@@ -27,34 +28,45 @@ class METAIE(InfoExtractor):
     }, {
         'url': 'http://video.meta.ua/iframe/5502115',
         'only_matching': True,
+    }, {
+        # pladform embed
+        'url': 'http://video.meta.ua/7121015.video',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        st_html5 = self._search_regex(r"st_html5\s*=\s*'#([^']+)'", webpage, 'uppod html5 st')
-        json_str = ''
-        for i in range(0, len(st_html5), 3):
-            json_str += '&#x0%s;' % st_html5[i:i + 3]
-        uppod_data = self._parse_json(unescapeHTML(json_str), video_id)
-        error = uppod_data.get('customnotfound')
-        if error:
-            raise ExtractorError('%s said: %s' % (self.IE_NAME, error), expected=True)
+        st_html5 = self._search_regex(
+            r"st_html5\s*=\s*'#([^']+)'", webpage, 'uppod html5 st', default=None)
 
-        video_url = uppod_data['file']
-        info = {
-            'id': video_id,
-            'url': video_url,
-            'title': uppod_data.get('comment') or self._og_search_title(webpage),
-            'description': self._og_search_description(webpage, default=None),
-            'thumbnail': uppod_data.get('poster') or self._og_search_thumbnail(webpage),
-            'duration': int_or_none(self._og_search_property(
-                'video:duration', webpage, default=None)),
-        }
-        if 'youtube.com/' in video_url:
-            info.update({
-                '_type': 'url_transparent',
-                'ie_key': 'Youtube',
-            })
-        return info
+        if st_html5:
+            json_str = ''
+            for i in range(0, len(st_html5), 3):
+                json_str += '&#x0%s;' % st_html5[i:i + 3]
+            uppod_data = self._parse_json(unescapeHTML(json_str), video_id)
+            error = uppod_data.get('customnotfound')
+            if error:
+                raise ExtractorError('%s said: %s' % (self.IE_NAME, error), expected=True)
+
+            video_url = uppod_data['file']
+            info = {
+                'id': video_id,
+                'url': video_url,
+                'title': uppod_data.get('comment') or self._og_search_title(webpage),
+                'description': self._og_search_description(webpage, default=None),
+                'thumbnail': uppod_data.get('poster') or self._og_search_thumbnail(webpage),
+                'duration': int_or_none(self._og_search_property(
+                    'video:duration', webpage, default=None)),
+            }
+            if 'youtube.com/' in video_url:
+                info.update({
+                    '_type': 'url_transparent',
+                    'ie_key': 'Youtube',
+                })
+            return info
+
+        pladform_url = PladformIE._extract_url(webpage)
+        if pladform_url:
+            return self.url_result(pladform_url)

From 66a42309fa235af2bb92b7fb73d90d8b79d6bf5d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 30 Jun 2016 23:56:55 +0700
Subject: [PATCH 5354/6219] release 2016.06.30

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 docs/supportedsites.md    | 5 +++++
 youtube_dl/version.py     | 2 +-
 3 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index f9a1aa990..8d24c14f3 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.06.27*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.06.27**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.06.30*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.06.30**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.06.27
+[debug] youtube-dl version 2016.06.30
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 2a94f4feb..c05cda6ab 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -272,6 +272,7 @@
  - **Helsinki**: helsinki.fi
  - **HentaiStigma**
  - **HistoricFilms**
+ - **history:topic**: History.com Topic
  - **hitbox**
  - **hitbox:live**
  - **HornBunny**
@@ -358,6 +359,7 @@
  - **MatchTV**
  - **MDR**: MDR.DE and KiKA
  - **media.ccc.de**
+ - **META**
  - **metacafe**
  - **Metacritic**
  - **Mgoon**
@@ -587,8 +589,10 @@
  - **Shared**: shared.sx and vivo.sx
  - **ShareSix**
  - **Sina**
+ - **SixPlay**
  - **skynewsarabia:article**
  - **skynewsarabia:video**
+ - **SkySports**
  - **Slideshare**
  - **Slutload**
  - **smotri**: Smotri.com
@@ -721,6 +725,7 @@
  - **UDNEmbed**: 聯合影音
  - **Unistra**
  - **Urort**: NRK P3 Urørt
+ - **URPlay**
  - **USAToday**
  - **ustream**
  - **ustream:channel**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 2dd24dec1..cf725db9b 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.06.27'
+__version__ = '2016.06.30'

From 05a0068a7680d41f56545a22ab34b004cd6d72e9 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 30 Jun 2016 18:13:49 +0100
Subject: [PATCH 5355/6219] [9c9media] Add new extractor

---
 youtube_dl/extractor/extractors.py     |  1 +
 youtube_dl/extractor/ninecninemedia.py | 55 ++++++++++++++++++++++++++
 2 files changed, 56 insertions(+)
 create mode 100644 youtube_dl/extractor/ninecninemedia.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index a7b110450..1b2854cb9 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -526,6 +526,7 @@ from .nick import (
     NickDeIE,
 )
 from .niconico import NiconicoIE, NiconicoPlaylistIE
+from .ninecninemedia import NineCNineMediaIE
 from .ninegag import NineGagIE
 from .noco import NocoIE
 from .normalboots import NormalbootsIE
diff --git a/youtube_dl/extractor/ninecninemedia.py b/youtube_dl/extractor/ninecninemedia.py
new file mode 100644
index 000000000..d889245ad
--- /dev/null
+++ b/youtube_dl/extractor/ninecninemedia.py
@@ -0,0 +1,55 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    parse_iso8601,
+    parse_duration,
+    ExtractorError
+)
+
+
+class NineCNineMediaIE(InfoExtractor):
+    _VALID_URL = r'9c9media:(?P<destination_code>[^:]+):(?P<id>\d+)'
+
+    def _real_extract(self, url):
+        destination_code, video_id = re.match(self._VALID_URL, url).groups()
+        api_base_url = 'http://capi.9c9media.com/destinations/%s/platforms/desktop/contents/%s/' % (destination_code, video_id)
+        content = self._download_json(api_base_url, video_id, query={
+            '$include': '[contentpackages]',
+        })
+        title = content['Name']
+        if len(content['ContentPackages']) > 1:
+            raise ExtractorError('multiple content packages')
+        content_package = content['ContentPackages'][0]
+        stacks_base_url = api_base_url + 'contentpackages/%s/stacks/' % content_package['Id']
+        stacks = self._download_json(stacks_base_url, video_id)['Items']
+        if len(stacks) > 1:
+            raise ExtractorError('multiple stacks')
+        stack = stacks[0]
+        stack_base_url = '%s%s/manifest.' % (stacks_base_url, stack['Id'])
+        formats = []
+        formats.extend(self._extract_m3u8_formats(
+            stack_base_url + 'm3u8', video_id, 'mp4',
+            'm3u8_native', m3u8_id='hls', fatal=False))
+        formats.extend(self._extract_f4m_formats(
+            stack_base_url + 'f4m', video_id,
+            f4m_id='hds', fatal=False))
+        mp4_url = self._download_webpage(stack_base_url + 'pd', video_id, fatal=False)
+        if mp4_url:
+            formats.append({
+                'url': mp4_url,
+                'format_id': 'mp4',
+            })
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': content.get('Desc') or content.get('ShortDesc'),
+            'timestamp': parse_iso8601(content.get('BroadcastDateTime')),
+            'duration': parse_duration(content.get('BroadcastTime')),
+            'formats': formats,
+        }

From 20361b4f2511a4395ae489c04a68c6098ffab7a4 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 30 Jun 2016 18:14:23 +0100
Subject: [PATCH 5356/6219] [rds] extract 9c9media formats

---
 youtube_dl/extractor/rds.py | 27 ++++++++++++---------------
 1 file changed, 12 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/rds.py b/youtube_dl/extractor/rds.py
index 796adfdf9..bf200ea4d 100644
--- a/youtube_dl/extractor/rds.py
+++ b/youtube_dl/extractor/rds.py
@@ -1,23 +1,23 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..utils import (
     parse_duration,
     parse_iso8601,
+    js_to_json,
 )
+from ..compat import compat_str
 
 
 class RDSIE(InfoExtractor):
     IE_DESC = 'RDS.ca'
-    _VALID_URL = r'https?://(?:www\.)?rds\.ca/vid(?:[eé]|%C3%A9)os/(?:[^/]+/)*(?P<display_id>[^/]+)-(?P<id>\d+\.\d+)'
+    _VALID_URL = r'https?://(?:www\.)?rds\.ca/vid(?:[eé]|%C3%A9)os/(?:[^/]+/)*(?P<id>[^/]+)-\d+\.\d+'
 
     _TESTS = [{
         'url': 'http://www.rds.ca/videos/football/nfl/fowler-jr-prend-la-direction-de-jacksonville-3.1132799',
         'info_dict': {
-            'id': '3.1132799',
+            'id': '604333',
             'display_id': 'fowler-jr-prend-la-direction-de-jacksonville',
             'ext': 'mp4',
             'title': 'Fowler Jr. prend la direction de Jacksonville',
@@ -33,22 +33,17 @@ class RDSIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        display_id = mobj.group('display_id')
+        display_id = self._match_id(url)
 
         webpage = self._download_webpage(url, display_id)
 
-        # TODO: extract f4m from 9c9media.com
-        video_url = self._search_regex(
-            r'<span[^>]+itemprop="contentURL"[^>]+content="([^"]+)"',
-            webpage, 'video url')
-
-        title = self._og_search_title(webpage) or self._html_search_meta(
+        item = self._parse_json(self._search_regex(r'(?s)itemToPush\s*=\s*({.+?});', webpage, 'item'), display_id, js_to_json)
+        video_id = compat_str(item['id'])
+        title = item.get('title') or self._og_search_title(webpage) or self._html_search_meta(
             'title', webpage, 'title', fatal=True)
         description = self._og_search_description(webpage) or self._html_search_meta(
             'description', webpage, 'description')
-        thumbnail = self._og_search_thumbnail(webpage) or self._search_regex(
+        thumbnail = item.get('urlImageBig') or self._og_search_thumbnail(webpage) or self._search_regex(
             [r'<link[^>]+itemprop="thumbnailUrl"[^>]+href="([^"]+)"',
              r'<span[^>]+itemprop="thumbnailUrl"[^>]+content="([^"]+)"'],
             webpage, 'thumbnail', fatal=False)
@@ -61,13 +56,15 @@ class RDSIE(InfoExtractor):
         age_limit = self._family_friendly_search(webpage)
 
         return {
+            '_type': 'url_transparent',
             'id': video_id,
             'display_id': display_id,
-            'url': video_url,
+            'url': '9c9media:rds_web:%s' % video_id,
             'title': title,
             'description': description,
             'thumbnail': thumbnail,
             'timestamp': timestamp,
             'duration': duration,
             'age_limit': age_limit,
+            'ie_key': 'NineCNineMedia',
         }

From bf4fa24414d2f4f4418b17ed379eb60df5726c4f Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 30 Jun 2016 18:14:59 +0100
Subject: [PATCH 5357/6219] [ctvnews] Add new extractor(closes #2156)

---
 youtube_dl/extractor/ctvnews.py    | 64 ++++++++++++++++++++++++++++++
 youtube_dl/extractor/extractors.py |  1 +
 2 files changed, 65 insertions(+)
 create mode 100644 youtube_dl/extractor/ctvnews.py

diff --git a/youtube_dl/extractor/ctvnews.py b/youtube_dl/extractor/ctvnews.py
new file mode 100644
index 000000000..e14b30085
--- /dev/null
+++ b/youtube_dl/extractor/ctvnews.py
@@ -0,0 +1,64 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+
+class CTVNewsIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?ctvnews\.ca/(?:video\?(?:clip|playlist|bin)Id=|.*?)(?P<id>[0-9.]+)'
+    _TESTS = [{
+        'url': 'http://www.ctvnews.ca/video?clipId=901995',
+        'md5': '10deb320dc0ccb8d01d34d12fc2ea672',
+        'info_dict': {
+            'id': '901995',
+            'ext': 'mp4',
+            'title': 'Extended: \'That person cannot be me\' Johnson says',
+            'description': 'md5:958dd3b4f5bbbf0ed4d045c790d89285',
+            'timestamp': 1467286284,
+            'upload_date': '20160630',
+        }
+    }, {
+        'url': 'http://www.ctvnews.ca/video?playlistId=1.2966224',
+        'info_dict':
+        {
+            'id': '1.2966224',
+        },
+        'playlist_mincount': 19,
+    }, {
+        'url': 'http://www.ctvnews.ca/video?binId=1.810401',
+        'info_dict':
+        {
+            'id': '1.810401',
+        },
+        'playlist_mincount': 91,
+    }, {
+        'url': 'http://www.ctvnews.ca/1.810401',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.ctvnews.ca/canadiens-send-p-k-subban-to-nashville-in-blockbuster-trade-1.2967231',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        page_id = self._match_id(url)
+
+        def ninecninemedia_url_result(clip_id):
+            return {
+                '_type': 'url_transparent',
+                'id': clip_id,
+                'url': '9c9media:ctvnews_web:%s' % clip_id,
+                'ie_key': 'NineCNineMedia',
+            }
+
+        if page_id.isdigit():
+            return ninecninemedia_url_result(page_id)
+        else:
+            webpage = self._download_webpage('http://www.ctvnews.ca/%s' % page_id, page_id, query={
+                'ot': 'example.AjaxPageLayout.ot',
+                'maxItemsPerPage': 20,
+            })
+            entries = [ninecninemedia_url_result(clip_id) for clip_id in set(
+                re.findall(r'clip\.id\s*=\s*(\d+);', webpage))]
+            return self.playlist_result(entries, page_id)
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 1b2854cb9..4765fbc77 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -171,6 +171,7 @@ from .crunchyroll import (
 )
 from .cspan import CSpanIE
 from .ctsnews import CtsNewsIE
+from .ctvnews import CTVNewsIE
 from .cultureunplugged import CultureUnpluggedIE
 from .cwtv import CWTVIE
 from .dailymail import DailyMailIE

From 9617b557aa2a96840026a9c915bc57e335a76272 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 30 Jun 2016 18:21:47 +0100
Subject: [PATCH 5358/6219] [ctv] Add new extractor(closes #4077)

---
 youtube_dl/extractor/ctv.py        | 30 ++++++++++++++++++++++++++++++
 youtube_dl/extractor/extractors.py |  1 +
 2 files changed, 31 insertions(+)
 create mode 100644 youtube_dl/extractor/ctv.py

diff --git a/youtube_dl/extractor/ctv.py b/youtube_dl/extractor/ctv.py
new file mode 100644
index 000000000..5807fbac9
--- /dev/null
+++ b/youtube_dl/extractor/ctv.py
@@ -0,0 +1,30 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class CTVIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?ctv\.ca/video/player\?vid=(?P<id>[0-9.]+)'
+    _TESTS = [{
+        'url': 'http://www.ctv.ca/video/player?vid=706966',
+        'md5': 'ff2ebbeae0aa2dcc32a830c3fd69b7b0',
+        'info_dict': {
+            'id': '706966',
+            'ext': 'mp4',
+            'title': 'Larry Day and Richard Jutras on the TIFF red carpet of \'Stonewall\'',
+            'description': 'etalk catches up with Larry Day and Richard Jutras on the TIFF red carpet of "Stonewall”.',
+            'upload_date': '20150919',
+            'timestamp': 1442624700,
+        },
+        'expected_warnings': ['HTTP Error 404'],
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        return {
+            '_type': 'url_transparent',
+            'id': video_id,
+            'url': '9c9media:ctv_web:%s' % video_id,
+            'ie_key': 'NineCNineMedia',
+        }
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 4765fbc77..62b5fed18 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -171,6 +171,7 @@ from .crunchyroll import (
 )
 from .cspan import CSpanIE
 from .ctsnews import CtsNewsIE
+from .ctv import CTVIE
 from .ctvnews import CTVNewsIE
 from .cultureunplugged import CultureUnpluggedIE
 from .cwtv import CWTVIE

From 76dad392f5bd82493777d8efc35bcfccf70fafec Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 30 Jun 2016 18:27:57 +0100
Subject: [PATCH 5359/6219] [meta] Clarify the source of uppod st decryption
 algorithm

---
 youtube_dl/extractor/meta.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/meta.py b/youtube_dl/extractor/meta.py
index 42bedc48f..cdb46e163 100644
--- a/youtube_dl/extractor/meta.py
+++ b/youtube_dl/extractor/meta.py
@@ -42,6 +42,7 @@ class METAIE(InfoExtractor):
             r"st_html5\s*=\s*'#([^']+)'", webpage, 'uppod html5 st', default=None)
 
         if st_html5:
+            # uppod st decryption algorithm is reverse engineered from function un(s) at uppod.js
             json_str = ''
             for i in range(0, len(st_html5), 3):
                 json_str += '&#x0%s;' % st_html5[i:i + 3]

From c9e538a3b1cde6ce140323a029c7b6f7386eb004 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 30 Jun 2016 19:52:32 +0100
Subject: [PATCH 5360/6219] [ctvnews] use orderedSet, increase the number of
 items for playlists and use smaller bin list for test

---
 youtube_dl/extractor/ctvnews.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/ctvnews.py b/youtube_dl/extractor/ctvnews.py
index e14b30085..1023b6130 100644
--- a/youtube_dl/extractor/ctvnews.py
+++ b/youtube_dl/extractor/ctvnews.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..utils import orderedSet
 
 
 class CTVNewsIE(InfoExtractor):
@@ -27,12 +28,12 @@ class CTVNewsIE(InfoExtractor):
         },
         'playlist_mincount': 19,
     }, {
-        'url': 'http://www.ctvnews.ca/video?binId=1.810401',
+        'url': 'http://www.ctvnews.ca/video?binId=1.2876780',
         'info_dict':
         {
-            'id': '1.810401',
+            'id': '1.2876780',
         },
-        'playlist_mincount': 91,
+        'playlist_mincount': 100,
     }, {
         'url': 'http://www.ctvnews.ca/1.810401',
         'only_matching': True,
@@ -57,8 +58,8 @@ class CTVNewsIE(InfoExtractor):
         else:
             webpage = self._download_webpage('http://www.ctvnews.ca/%s' % page_id, page_id, query={
                 'ot': 'example.AjaxPageLayout.ot',
-                'maxItemsPerPage': 20,
+                'maxItemsPerPage': 1000000,
             })
-            entries = [ninecninemedia_url_result(clip_id) for clip_id in set(
+            entries = [ninecninemedia_url_result(clip_id) for clip_id in orderedSet(
                 re.findall(r'clip\.id\s*=\s*(\d+);', webpage))]
             return self.playlist_result(entries, page_id)

From 044e3d91b5715f7aa63c578097b77fd510ed0f73 Mon Sep 17 00:00:00 2001
From: kidol <keyboard.idol@gmail.com>
Date: Thu, 30 Jun 2016 21:06:22 +0200
Subject: [PATCH 5361/6219] [Pornhub] Fix error detection

---
 youtube_dl/extractor/pornhub.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index 6d57e1d35..4bbf1ec3b 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -87,7 +87,7 @@ class PornHubIE(InfoExtractor):
         webpage = self._download_webpage(req, video_id)
 
         error_msg = self._html_search_regex(
-            r'(?s)<div class="userMessageSection[^"]*".*?>(.*?)</div>',
+            r'<div[^>]+class="removed">\s*<div[^>]*>\s*<p>\s*<span>([^<]*)</span>',
             webpage, 'error message', default=None)
         if error_msg:
             error_msg = re.sub(r'\s+', ' ', error_msg)

From 3cb3b60064fc8d99a8175b751000892b141e8de0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 1 Jul 2016 03:14:23 +0700
Subject: [PATCH 5362/6219] [pornhub] Relax removed message regex (Closes
 #9964)

---
 youtube_dl/extractor/pornhub.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index 4bbf1ec3b..c1694893c 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -87,8 +87,8 @@ class PornHubIE(InfoExtractor):
         webpage = self._download_webpage(req, video_id)
 
         error_msg = self._html_search_regex(
-            r'<div[^>]+class="removed">\s*<div[^>]*>\s*<p>\s*<span>([^<]*)</span>',
-            webpage, 'error message', default=None)
+            r'(?s)<div[^>]+class=(["\']).*?\bremoved\b.*?\1[^>]*>(?P<error>.+?)</div>',
+            webpage, 'error message', default=None, group='error')
         if error_msg:
             error_msg = re.sub(r'\s+', ' ', error_msg)
             raise ExtractorError(

From eaaaaec042f8e6afa8f8ec6a2a8b137943f802df Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 1 Jul 2016 03:18:27 +0700
Subject: [PATCH 5363/6219] [pornhub] Add more tests with removed videos

---
 youtube_dl/extractor/pornhub.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index c1694893c..77182bf07 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -63,8 +63,17 @@ class PornHubIE(InfoExtractor):
         'url': 'http://www.pornhub.com/view_video.php?viewkey=ph557bbb6676d2d',
         'only_matching': True,
     }, {
+        # removed at the request of cam4.com
         'url': 'http://fr.pornhub.com/view_video.php?viewkey=ph55ca2f9760862',
         'only_matching': True,
+    }, {
+        # removed at the request of the copyright owner
+        'url': 'http://www.pornhub.com/view_video.php?viewkey=788152859',
+        'only_matching': True,
+    }, {
+        # removed by uploader
+        'url': 'http://www.pornhub.com/view_video.php?viewkey=ph572716d15a111',
+        'only_matching': True,
     }]
 
     @classmethod

From 9e29ef13a378769c19ccec200aba377ad504fe8f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 29 Jun 2016 14:56:05 +0200
Subject: [PATCH 5364/6219] [options] Accept quoted string across multiple
 lines (#9940)

Like:

    -f "
    bestvideo+bestaudio/
    best
    "
---
 youtube_dl/options.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 99ce4131f..c9033e3cb 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -26,9 +26,7 @@ def parseOpts(overrideArguments=None):
         except IOError:
             return default  # silently skip if file is not present
         try:
-            res = []
-            for l in optionf:
-                res += compat_shlex_split(l, comments=True)
+            res = compat_shlex_split(optionf.read(), comments=True)
         finally:
             optionf.close()
         return res

From a9eede3913a9e9c7e094907f36a22bc6719ce73d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 29 Jun 2016 18:54:30 +0200
Subject: [PATCH 5365/6219] [test/compat] compat_shlex_split: test with
 newlines

---
 test/test_compat.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/test/test_compat.py b/test/test_compat.py
index f5317ac3e..1d7ac9f16 100644
--- a/test/test_compat.py
+++ b/test/test_compat.py
@@ -87,6 +87,7 @@ class TestCompat(unittest.TestCase):
 
     def test_compat_shlex_split(self):
         self.assertEqual(compat_shlex_split('-option "one two"'), ['-option', 'one two'])
+        self.assertEqual(compat_shlex_split('-option "one\ntwo" \n -flag'), ['-option', 'one\ntwo', '-flag'])
 
     def test_compat_etree_fromstring(self):
         xml = '''

From 0c2ac64bb81462bed0c31be5a2a549601f95f166 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 1 Jul 2016 03:57:59 +0700
Subject: [PATCH 5366/6219] [sixplay] Rename preference key to quality in
 format dict

---
 youtube_dl/extractor/sixplay.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/sixplay.py b/youtube_dl/extractor/sixplay.py
index f855a1a00..759a332d2 100644
--- a/youtube_dl/extractor/sixplay.py
+++ b/youtube_dl/extractor/sixplay.py
@@ -28,7 +28,7 @@ class SixPlayIE(InfoExtractor):
             video_id)
         video_data = clip_data['videoInfo']
 
-        preference = qualities(['lq', 'sd', 'hq', 'hd'])
+        quality_key = qualities(['lq', 'sd', 'hq', 'hd'])
         formats = []
         for source in clip_data['sources']:
             source_type, source_url = source.get('type'), source.get('src')
@@ -46,7 +46,7 @@ class SixPlayIE(InfoExtractor):
                 formats.append({
                     'url': source_url,
                     'format_id': quality,
-                    'preference': preference(quality),
+                    'quality': quality_key(quality),
                 })
         self._sort_formats(formats)
 

From f11315e8d4239611c42d8fe438c7ded58293d54d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 1 Jul 2016 03:59:57 +0700
Subject: [PATCH 5367/6219] release 2016.07.01

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 docs/supportedsites.md    | 3 +++
 youtube_dl/version.py     | 2 +-
 3 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 8d24c14f3..d9b196e52 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.06.30*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.06.30**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.07.01*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.07.01**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.06.30
+[debug] youtube-dl version 2016.07.01
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index c05cda6ab..6e4041614 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -152,6 +152,8 @@
  - **CSNNE**
  - **CSpan**: C-SPAN
  - **CtsNews**: 華視新聞
+ - **CTV**
+ - **CTVNews**
  - **culturebox.francetvinfo.fr**
  - **CultureUnplugged**
  - **CWTV**
@@ -440,6 +442,7 @@
  - **nick.de**
  - **niconico**: ニコニコ動画
  - **NiconicoPlaylist**
+ - **NineCNineMedia**
  - **njoy**: N-JOY
  - **njoy:embed**
  - **Noco**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index cf725db9b..e5f10da39 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.06.30'
+__version__ = '2016.07.01'

From 9f4576a7ebbc4af4971796325f0799f894daaa1c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 1 Jul 2016 23:16:43 +0700
Subject: [PATCH 5368/6219] [twitch] Update usher URL (Closes #9975)

---
 youtube_dl/extractor/twitch.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 20919774d..67b1277cc 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -29,7 +29,7 @@ class TwitchBaseIE(InfoExtractor):
     _VALID_URL_BASE = r'https?://(?:www\.)?twitch\.tv'
 
     _API_BASE = 'https://api.twitch.tv'
-    _USHER_BASE = 'http://usher.twitch.tv'
+    _USHER_BASE = 'https://usher.ttvnw.net'
     _LOGIN_URL = 'http://www.twitch.tv/login'
     _NETRC_MACHINE = 'twitch'
 

From 564dc3c6e8ad235160cfea01e41fc01fefc39be8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 2 Jul 2016 01:24:57 +0700
Subject: [PATCH 5369/6219] [vine] Fix extraction (Closes #9970)

---
 youtube_dl/extractor/vine.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vine.py b/youtube_dl/extractor/vine.py
index 5b801849c..5a2c53b36 100644
--- a/youtube_dl/extractor/vine.py
+++ b/youtube_dl/extractor/vine.py
@@ -90,9 +90,11 @@ class VineIE(InfoExtractor):
 
         data = self._parse_json(
             self._search_regex(
-                r'window\.POST_DATA\s*=\s*{\s*%s\s*:\s*({.+?})\s*};\s*</script>' % video_id,
+                r'window\.POST_DATA\s*=\s*({.+?});\s*</script>',
                 webpage, 'vine data'),
             video_id)
+        
+        data = data[list(data.keys())[0]]
 
         formats = [{
             'format_id': '%(format)s-%(rate)s' % f,

From 347227237b5a101c3bed260f8efbdbfe65c5f196 Mon Sep 17 00:00:00 2001
From: cant-think-of-a-name <ar513161@gmail.com>
Date: Thu, 30 Jun 2016 20:19:17 -0500
Subject: [PATCH 5370/6219] [periscope] fix playlist extraction (#9967)

The JSON response changed and the extractor needed to be updated in order to gather the video IDs.
---
 youtube_dl/extractor/periscope.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/periscope.py b/youtube_dl/extractor/periscope.py
index c23b314e7..34e0d3d30 100644
--- a/youtube_dl/extractor/periscope.py
+++ b/youtube_dl/extractor/periscope.py
@@ -122,7 +122,7 @@ class PeriscopeUserIE(InfoExtractor):
 
         entries = [
             self.url_result(
-                'https://www.periscope.tv/%s/%s' % (user_id, broadcast['id']))
-            for broadcast in data_store.get('UserBroadcastHistory', {}).get('broadcasts', [])]
+                'https://www.periscope.tv/%s/%s' % (user_id, broadcast))
+            for broadcast in data_store.get('UserBroadcastHistory', {}).get('broadcastIds', [])]
 
         return self.playlist_result(entries, user_id, title, description)

From 35fc3021ba6e1e0d7b7d400fdaccc709546a4bc5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 2 Jul 2016 01:35:57 +0700
Subject: [PATCH 5371/6219] [periscope] Add another fallback source

---
 youtube_dl/extractor/periscope.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/periscope.py b/youtube_dl/extractor/periscope.py
index 34e0d3d30..75f5884a9 100644
--- a/youtube_dl/extractor/periscope.py
+++ b/youtube_dl/extractor/periscope.py
@@ -120,9 +120,12 @@ class PeriscopeUserIE(InfoExtractor):
         title = user.get('display_name') or user.get('username')
         description = user.get('description')
 
+        broadcast_ids = (data_store.get('UserBroadcastHistory', {}).get('broadcastIds') or
+                         data_store.get('BroadcastCache', {}).get('broadcastIds', []))
+
         entries = [
             self.url_result(
-                'https://www.periscope.tv/%s/%s' % (user_id, broadcast))
-            for broadcast in data_store.get('UserBroadcastHistory', {}).get('broadcastIds', [])]
+                'https://www.periscope.tv/%s/%s' % (user_id, broadcast_id))
+            for broadcast_id in broadcast_ids]
 
         return self.playlist_result(entries, user_id, title, description)

From bc4b2d75ba5b8fbd2ac9d42f0fb7a4fcd2ea3038 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 2 Jul 2016 02:11:07 +0700
Subject: [PATCH 5372/6219] [pornhub] Add support for thumbzilla (Closes #8696)

---
 youtube_dl/extractor/pornhub.py | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index 77182bf07..c76afe1c4 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -25,7 +25,15 @@ from ..aes import (
 
 
 class PornHubIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:[a-z]+\.)?pornhub\.com/(?:view_video\.php\?viewkey=|embed/)(?P<id>[0-9a-z]+)'
+    IE_DESC = 'PornHub and Thumbzilla'
+    _VALID_URL = r'''(?x)
+                    https?://
+                        (?:
+                            (?:[a-z]+\.)?pornhub\.com/(?:view_video\.php\?viewkey=|embed/)|
+                            (?:www\.)?thumbzilla\.com/video/
+                        )
+                        (?P<id>[0-9a-z]+)
+                    '''
     _TESTS = [{
         'url': 'http://www.pornhub.com/view_video.php?viewkey=648719015',
         'md5': '1e19b41231a02eba417839222ac9d58e',
@@ -74,6 +82,9 @@ class PornHubIE(InfoExtractor):
         # removed by uploader
         'url': 'http://www.pornhub.com/view_video.php?viewkey=ph572716d15a111',
         'only_matching': True,
+    }, {
+        'url': 'https://www.thumbzilla.com/video/ph56c6114abd99a/horny-girlfriend-sex',
+        'only_matching': True,
     }]
 
     @classmethod

From bb08101ec4a8728677ee23466608ab6aa65cbb4f Mon Sep 17 00:00:00 2001
From: TRox1972 <TRox1972@users.noreply.github.com>
Date: Thu, 30 Jun 2016 14:57:42 +0200
Subject: [PATCH 5373/6219] [Fusion] Add new extractor

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/fusion.py     | 29 +++++++++++++++++++++++++++++
 2 files changed, 30 insertions(+)
 create mode 100644 youtube_dl/extractor/fusion.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 62b5fed18..16fa4d35c 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -281,6 +281,7 @@ from .freespeech import FreespeechIE
 from .freevideo import FreeVideoIE
 from .funimation import FunimationIE
 from .funnyordie import FunnyOrDieIE
+from .fusion import FusionIE
 from .gameinformer import GameInformerIE
 from .gamekings import GamekingsIE
 from .gameone import (
diff --git a/youtube_dl/extractor/fusion.py b/youtube_dl/extractor/fusion.py
new file mode 100644
index 000000000..771abcdb1
--- /dev/null
+++ b/youtube_dl/extractor/fusion.py
@@ -0,0 +1,29 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from .ooyala import OoyalaIE
+
+
+class FusionIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?fusion\.net/video/\d+/(?P<id>[\w-]+)'
+    _TEST = {
+        'url': 'http://fusion.net/video/201781/u-s-and-panamanian-forces-work-together-to-stop-a-vessel-smuggling-drugs/',
+        'md5': '55c3dd61d2b96dc17c4ab6711d02a39e',
+        'info_dict': {
+            'id': 'ZpcWNoMTE6x6uVIIWYpHh0qQDjxBuq5P',
+            'ext': 'mp4',
+            'title': 'U.S. and Panamanian forces work together to stop a vessel smuggling drugs',
+            'description': 'md5:0cc84a9943c064c0f46b128b41b1b0d7',
+            'duration': 140.0,
+        },
+        'add_ie': ['Ooyala'],
+    }
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+
+        ooyala_code = self._search_regex(r'data-video-id="([^"]{32})"',
+            webpage, 'ooyala code')
+
+        return OoyalaIE._build_url_result(ooyala_code)

From 14ff6baa0ee3fa0ead2e2b460017abe5e853647c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 2 Jul 2016 02:44:37 +0700
Subject: [PATCH 5374/6219] [fusion] Improve

---
 youtube_dl/extractor/fusion.py | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/fusion.py b/youtube_dl/extractor/fusion.py
index 771abcdb1..b4ab4cbb7 100644
--- a/youtube_dl/extractor/fusion.py
+++ b/youtube_dl/extractor/fusion.py
@@ -5,10 +5,9 @@ from .ooyala import OoyalaIE
 
 
 class FusionIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?fusion\.net/video/\d+/(?P<id>[\w-]+)'
-    _TEST = {
+    _VALID_URL = r'https?://(?:www\.)?fusion\.net/video/(?P<id>\d+)'
+    _TESTS = [{
         'url': 'http://fusion.net/video/201781/u-s-and-panamanian-forces-work-together-to-stop-a-vessel-smuggling-drugs/',
-        'md5': '55c3dd61d2b96dc17c4ab6711d02a39e',
         'info_dict': {
             'id': 'ZpcWNoMTE6x6uVIIWYpHh0qQDjxBuq5P',
             'ext': 'mp4',
@@ -16,14 +15,21 @@ class FusionIE(InfoExtractor):
             'description': 'md5:0cc84a9943c064c0f46b128b41b1b0d7',
             'duration': 140.0,
         },
+        'params': {
+            'skip_download': True,
+        },
         'add_ie': ['Ooyala'],
-    }
+    }, {
+        'url': 'http://fusion.net/video/201781',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
         webpage = self._download_webpage(url, display_id)
 
-        ooyala_code = self._search_regex(r'data-video-id="([^"]{32})"',
-            webpage, 'ooyala code')
+        ooyala_code = self._search_regex(
+            r'data-video-id=(["\'])(?P<code>.+?)\1',
+            webpage, 'ooyala code', group='code')
 
         return OoyalaIE._build_url_result(ooyala_code)

From ac2d8f54d1f95ff54cae6808602d5ddb39bc978b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 2 Jul 2016 02:45:00 +0700
Subject: [PATCH 5375/6219] [vine] Remove superfluous whitespace

---
 youtube_dl/extractor/vine.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vine.py b/youtube_dl/extractor/vine.py
index 5a2c53b36..0183f052a 100644
--- a/youtube_dl/extractor/vine.py
+++ b/youtube_dl/extractor/vine.py
@@ -93,7 +93,7 @@ class VineIE(InfoExtractor):
                 r'window\.POST_DATA\s*=\s*({.+?});\s*</script>',
                 webpage, 'vine data'),
             video_id)
-        
+
         data = data[list(data.keys())[0]]
 
         formats = [{

From 7a1e71575e8bf6918ece07cb72a58e7425692fd4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 2 Jul 2016 02:47:42 +0700
Subject: [PATCH 5376/6219] release 2016.07.02

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 docs/supportedsites.md    | 3 ++-
 youtube_dl/version.py     | 2 +-
 3 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index d9b196e52..637103b6b 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.07.01*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.07.01**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.07.02*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.07.02**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.07.01
+[debug] youtube-dl version 2016.07.02
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 6e4041614..8fd1ab5af 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -242,6 +242,7 @@
  - **FreeVideo**
  - **Funimation**
  - **FunnyOrDie**
+ - **Fusion**
  - **GameInformer**
  - **Gamekings**
  - **GameOne**
@@ -508,7 +509,7 @@
  - **podomatic**
  - **PolskieRadio**
  - **PornHd**
- - **PornHub**
+ - **PornHub**: PornHub and Thumbzilla
  - **PornHubPlaylist**
  - **PornHubUserVideos**
  - **Pornotube**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index e5f10da39..d0483f83b 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.07.01'
+__version__ = '2016.07.02'

From bdafd88da07046f91e0585f083dea7795096e5d7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 2 Jul 2016 16:43:19 +0700
Subject: [PATCH 5377/6219] [vk] Extend _VALID_URLs to support new domain
 (Closes #9981)

---
 youtube_dl/extractor/vk.py | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index cfc5ffd8b..758d9c86b 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -27,12 +27,12 @@ class VKIE(InfoExtractor):
                     https?://
                         (?:
                             (?:
-                                (?:m\.)?vk\.com/video_|
+                                (?:(?:m|new)\.)?vk\.com/video_|
                                 (?:www\.)?daxab.com/
                             )
                             ext\.php\?(?P<embed_query>.*?\boid=(?P<oid>-?\d+).*?\bid=(?P<id>\d+).*)|
                             (?:
-                                (?:m\.)?vk\.com/(?:.+?\?.*?z=)?video|
+                                (?:(?:m|new)\.)?vk\.com/(?:.+?\?.*?z=)?video|
                                 (?:www\.)?daxab.com/embed/
                             )
                             (?P<videoid>-?\d+_\d+)(?:.*\blist=(?P<list_id>[\da-f]+))?
@@ -182,6 +182,10 @@ class VKIE(InfoExtractor):
             # pladform embed
             'url': 'https://vk.com/video-76116461_171554880',
             'only_matching': True,
+        },
+        {
+            'url': 'http://new.vk.com/video205387401_165548505',
+            'only_matching': True,
         }
     ]
 
@@ -354,7 +358,7 @@ class VKIE(InfoExtractor):
 class VKUserVideosIE(InfoExtractor):
     IE_NAME = 'vk:uservideos'
     IE_DESC = "VK - User's Videos"
-    _VALID_URL = r'https?://vk\.com/videos(?P<id>-?[0-9]+)(?!\?.*\bz=video)(?:[/?#&]|$)'
+    _VALID_URL = r'https?://(?:(?:m|new)\.)?vk\.com/videos(?P<id>-?[0-9]+)(?!\?.*\bz=video)(?:[/?#&]|$)'
     _TEMPLATE_URL = 'https://vk.com/videos'
     _TESTS = [{
         'url': 'http://vk.com/videos205387401',
@@ -369,6 +373,12 @@ class VKUserVideosIE(InfoExtractor):
     }, {
         'url': 'http://vk.com/videos-97664626?section=all',
         'only_matching': True,
+    }, {
+        'url': 'http://m.vk.com/videos205387401',
+        'only_matching': True,
+    }, {
+        'url': 'http://new.vk.com/videos205387401',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From fd6ca382628afbc4a229a15cd26552e226ac4536 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 2 Jul 2016 21:33:23 +0800
Subject: [PATCH 5378/6219] [facebook] Improve Facebook embedded detection

Related to #9938.

Another example comes from 9834872bf63b4e03b66c5e3b8f306556e735d8c5.
---
 youtube_dl/extractor/facebook.py | 15 +++++++++++++++
 youtube_dl/extractor/generic.py  | 26 ++++++++++++++++++++++----
 2 files changed, 37 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index 9b87b37ae..6eaa22d89 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -129,6 +129,21 @@ class FacebookIE(InfoExtractor):
         'only_matching': True,
     }]
 
+    @staticmethod
+    def _extract_url(webpage):
+        mobj = re.search(
+            r'<iframe[^>]+?src=(["\'])(?P<url>https://www\.facebook\.com/video/embed.+?)\1', webpage)
+        if mobj is not None:
+            return mobj.group('url')
+
+        # Facebook API embed
+        # see https://developers.facebook.com/docs/plugins/embedded-video-player
+        mobj = re.search(r'''(?x)<div[^>]+
+                class=(?P<q1>[\'"])[^\'"]*\bfb-video\b[^\'"]*(?P=q1)[^>]+
+                data-href=(?P<q2>[\'"])(?P<url>[^\'"]+)(?P=q2)''', webpage)
+        if mobj is not None:
+            return mobj.group('url')
+
     def _login(self):
         (useremail, password) = self._get_login_info()
         if useremail is None:
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 9315b9e21..7212e0edd 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -66,6 +66,7 @@ from .theplatform import ThePlatformIE
 from .vessel import VesselIE
 from .kaltura import KalturaIE
 from .eagleplatform import EaglePlatformIE
+from .facebook import FacebookIE
 
 
 class GenericIE(InfoExtractor):
@@ -1260,6 +1261,24 @@ class GenericIE(InfoExtractor):
                 'uploader': 'TheAtlantic',
             },
             'add_ie': ['BrightcoveLegacy'],
+        },
+        # Facebook <iframe> embed
+        {
+            'url': 'https://www.hostblogger.de/blog/archives/6181-Auto-jagt-Betonmischer.html',
+            'info_dict': {
+                'id': '599637780109885',
+                'ext': 'mp4',
+                'title': 'Facebook video #599637780109885',
+            },
+        },
+        # Facebook API embed
+        {
+            'url': 'http://www.lothype.com/blue-stars-2016-preview-standstill-full-show/',
+            'info_dict': {
+                'id': '10153467542406923',
+                'ext': 'mp4',
+                'title': 'Facebook video #10153467542406923',
+            },
         }
     ]
 
@@ -1759,10 +1778,9 @@ class GenericIE(InfoExtractor):
             return self.url_result(mobj.group('url'))
 
         # Look for embedded Facebook player
-        mobj = re.search(
-            r'<iframe[^>]+?src=(["\'])(?P<url>https://www\.facebook\.com/video/embed.+?)\1', webpage)
-        if mobj is not None:
-            return self.url_result(mobj.group('url'), 'Facebook')
+        facebook_url = FacebookIE._extract_url(webpage)
+        if facebook_url is not None:
+            return self.url_result(facebook_url, 'Facebook')
 
         # Look for embedded VK player
         mobj = re.search(r'<iframe[^>]+?src=(["\'])(?P<url>https?://vk\.com/video_ext\.php.+?)\1', webpage)

From 7deef1ba6743bf11247565e63ed7e31d2e8a9382 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 2 Jul 2016 21:50:17 +0800
Subject: [PATCH 5379/6219] [generic] Support Wordpress "YouTube Video
 Importer" plugin

Closes #9938
---
 youtube_dl/extractor/generic.py | 21 ++++++++++++++++++++-
 1 file changed, 20 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 7212e0edd..35acb67f1 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1279,7 +1279,19 @@ class GenericIE(InfoExtractor):
                 'ext': 'mp4',
                 'title': 'Facebook video #10153467542406923',
             },
-        }
+        },
+        # Wordpress "YouTube Video Importer" plugin
+        {
+            'url': 'http://www.lothype.com/blue-devils-drumline-stanford-lot-2016/',
+            'info_dict': {
+                'id': 'HNTXWDXV9Is',
+                'ext': 'mp4',
+                'title': 'Blue Devils Drumline Stanford lot 2016',
+                'upload_date': '20160627',
+                'uploader_id': 'GENOCIDE8GENERAL10',
+                'uploader': 'cylus cyrus',
+            },
+        },
     ]
 
     def report_following_redirect(self, new_url):
@@ -1636,6 +1648,13 @@ class GenericIE(InfoExtractor):
         if matches:
             return _playlist_from_matches(matches, lambda m: unescapeHTML(m))
 
+        # Look for Wordpress "YouTube Video Importer" plugin
+        matches = re.findall(r'''(?x)<div[^>]+
+            class=(?P<q1>[\'"])[^\'"]*\byvii_single_video_player\b[^\'"]*(?P=q1)[^>]+
+            data-video_id=(?P<q2>[\'"])([^\'"]+)(?P=q2)''', webpage)
+        if matches:
+            return _playlist_from_matches(matches, lambda m: m[-1])
+
         # Look for embedded Dailymotion player
         matches = re.findall(
             r'<(?:(?:embed|iframe)[^>]+?src=|input[^>]+id=[\'"]dmcloudUrlEmissionSelect[\'"][^>]+value=)(["\'])(?P<url>(?:https?:)?//(?:www\.)?dailymotion\.com/(?:embed|swf)/video/.+?)\1', webpage)

From dbf0157a26272a43398b986ab65327c150af8432 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 2 Jul 2016 21:57:06 +0800
Subject: [PATCH 5380/6219] [generic] Add MD5 checksums

---
 youtube_dl/extractor/generic.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 35acb67f1..a9b61bf13 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1265,6 +1265,7 @@ class GenericIE(InfoExtractor):
         # Facebook <iframe> embed
         {
             'url': 'https://www.hostblogger.de/blog/archives/6181-Auto-jagt-Betonmischer.html',
+            'md5': 'fbcde74f534176ecb015849146dd3aee',
             'info_dict': {
                 'id': '599637780109885',
                 'ext': 'mp4',
@@ -1274,6 +1275,7 @@ class GenericIE(InfoExtractor):
         # Facebook API embed
         {
             'url': 'http://www.lothype.com/blue-stars-2016-preview-standstill-full-show/',
+            'md5': 'a47372ee61b39a7b90287094d447d94e',
             'info_dict': {
                 'id': '10153467542406923',
                 'ext': 'mp4',
@@ -1283,6 +1285,7 @@ class GenericIE(InfoExtractor):
         # Wordpress "YouTube Video Importer" plugin
         {
             'url': 'http://www.lothype.com/blue-devils-drumline-stanford-lot-2016/',
+            'md5': 'd16797741b560b485194eddda8121b48',
             'info_dict': {
                 'id': 'HNTXWDXV9Is',
                 'ext': 'mp4',

From 6a424391d9745f46c8451771ae4d62da389cc311 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 2 Jul 2016 23:15:55 +0800
Subject: [PATCH 5381/6219] [facebook] Make embed detection stricter to prevent
 false-positives

---
 youtube_dl/extractor/facebook.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index 6eaa22d89..bbdb14366 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -140,7 +140,7 @@ class FacebookIE(InfoExtractor):
         # see https://developers.facebook.com/docs/plugins/embedded-video-player
         mobj = re.search(r'''(?x)<div[^>]+
                 class=(?P<q1>[\'"])[^\'"]*\bfb-video\b[^\'"]*(?P=q1)[^>]+
-                data-href=(?P<q2>[\'"])(?P<url>[^\'"]+)(?P=q2)''', webpage)
+                data-href=(?P<q2>[\'"])(?P<url>(?:https?:)?//(?:www\.)?facebook.com/.+?)(?P=q2)''', webpage)
         if mobj is not None:
             return mobj.group('url')
 

From 712b0b5b7016202885a51c2a8af79921bf0fce2b Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 2 Jul 2016 23:49:03 +0800
Subject: [PATCH 5382/6219] [la7.it] Fix the extractor

---
 youtube_dl/extractor/la7.py | 86 +++++++++++++++++++++----------------
 1 file changed, 50 insertions(+), 36 deletions(-)

diff --git a/youtube_dl/extractor/la7.py b/youtube_dl/extractor/la7.py
index b08f6e3c9..826dc9476 100644
--- a/youtube_dl/extractor/la7.py
+++ b/youtube_dl/extractor/la7.py
@@ -1,60 +1,74 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..utils import (
-    parse_duration,
+    determine_ext,
+    js_to_json,
 )
 
 
 class LA7IE(InfoExtractor):
-    IE_NAME = 'la7.tv'
-    _VALID_URL = r'''(?x)
-        https?://(?:www\.)?la7\.tv/
-        (?:
-            richplayer/\?assetid=|
-            \?contentId=
-        )
-        (?P<id>[0-9]+)'''
+    IE_NAME = 'la7.it'
+    _VALID_URL = r'''(?x)(https?://)?(?:
+        (?:www\.)?la7\.it/([^/]+)/(?:rivedila7|video)/|
+        tg\.la7\.it/repliche-tgla7\?id=
+    )(?P<id>.+)'''
 
-    _TEST = {
-        'url': 'http://www.la7.tv/richplayer/?assetid=50355319',
-        'md5': 'ec7d1f0224d20ba293ab56cf2259651f',
+    _TESTS = [{
+        # 'src' is a plain URL
+        'url': 'http://www.la7.it/crozza/video/inccool8-02-10-2015-163722',
+        'md5': '6054674766e7988d3e02f2148ff92180',
         'info_dict': {
-            'id': '50355319',
+            'id': 'inccool8-02-10-2015-163722',
             'ext': 'mp4',
-            'title': 'IL DIVO',
-            'description': 'Un film di Paolo Sorrentino con Toni Servillo, Anna Bonaiuto, Giulio Bosetti  e Flavio Bucci',
-            'duration': 6254,
+            'title': 'Inc.Cool8',
+            'description': 'Benvenuti nell\'incredibile mondo della INC. COOL. 8. dove “INC.” sta per “Incorporated” “COOL” sta per “fashion” ed Eight sta per il gesto  atletico',
+            'thumbnail': 're:^https?://.*',
         },
-        'skip': 'Blocked in the US',
-    }
+    }, {
+        # 'src' is a dictionary
+        'url': 'http://tg.la7.it/repliche-tgla7?id=189080',
+        'md5': '6b0d8888d286e39870208dfeceaf456b',
+        'info_dict': {
+            'id': '189080',
+            'ext': 'mp4',
+            'title': 'TG LA7',
+        },
+    }, {
+        'url': 'http://www.la7.it/omnibus/rivedila7/omnibus-news-02-07-2016-189077',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        xml_url = 'http://www.la7.tv/repliche/content/index.php?contentId=%s' % video_id
-        doc = self._download_xml(xml_url, video_id)
 
-        video_title = doc.find('title').text
-        description = doc.find('description').text
-        duration = parse_duration(doc.find('duration').text)
-        thumbnail = doc.find('img').text
-        view_count = int(doc.find('views').text)
+        webpage = self._download_webpage(url, video_id)
 
-        prefix = doc.find('.//fqdn').text.strip().replace('auto:', 'http:')
+        player_data = self._parse_json(
+            self._search_regex(r'videoLa7\(({[^;]+})\);', webpage, 'player data'),
+            video_id, transform_source=js_to_json)
 
-        formats = [{
-            'format': vnode.find('quality').text,
-            'tbr': int(vnode.find('quality').text),
-            'url': vnode.find('fms').text.strip().replace('mp4:', prefix),
-        } for vnode in doc.findall('.//videos/video')]
+        source = player_data['src']
+        source_urls = source.values() if isinstance(source, dict) else [source]
+
+        formats = []
+        for source_url in source_urls:
+            ext = determine_ext(source_url)
+            if ext == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    source_url, video_id, ext='mp4',
+                    entry_protocol='m3u8_native', m3u8_id='hls'))
+            else:
+                formats.append({
+                    'url': source_url,
+                })
         self._sort_formats(formats)
 
         return {
             'id': video_id,
-            'title': video_title,
-            'description': description,
-            'thumbnail': thumbnail,
-            'duration': duration,
+            'title': player_data['title'],
+            'description': self._og_search_description(webpage, default=None),
+            'thumbnail': player_data.get('poster'),
             'formats': formats,
-            'view_count': view_count,
         }

From 6b03e1e25d05ddb3dbd577b045e44fa9eb05bdc0 Mon Sep 17 00:00:00 2001
From: Aleksandar Topuzovic <aleksandar.topuzovic@gmail.com>
Date: Fri, 13 May 2016 15:52:52 +0100
Subject: [PATCH 5383/6219] [HRTi] Implement extractor for Croatian
 Radiotelevision

---
 youtube_dl/extractor/extractors.py |   1 +
 youtube_dl/extractor/hrti.py       | 149 +++++++++++++++++++++++++++++
 youtube_dl/utils.py                |   2 +
 3 files changed, 152 insertions(+)
 create mode 100644 youtube_dl/extractor/hrti.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 16fa4d35c..ee1718f6a 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -326,6 +326,7 @@ from .hotnewhiphop import HotNewHipHopIE
 from .hotstar import HotStarIE
 from .howcast import HowcastIE
 from .howstuffworks import HowStuffWorksIE
+from .hrti import HRTiIE
 from .huffpost import HuffPostIE
 from .hypem import HypemIE
 from .iconosquare import IconosquareIE
diff --git a/youtube_dl/extractor/hrti.py b/youtube_dl/extractor/hrti.py
new file mode 100644
index 000000000..55ff981e2
--- /dev/null
+++ b/youtube_dl/extractor/hrti.py
@@ -0,0 +1,149 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import json
+
+from .common import InfoExtractor
+from ..compat import (
+    compat_HTTPError
+)
+from ..utils import (
+    sanitized_Request,
+    ExtractorError
+)
+
+
+class HRTiIE(InfoExtractor):
+    '''
+    Information Extractor for Croatian Radiotelevision video on demand site
+    https://hrti.hrt.hr
+    Reverse engineered from the JavaScript app in app.min.js
+    '''
+    _NETRC_MACHINE = 'hrti'
+
+    APP_LANGUAGE = 'hr'
+    APP_VERSION = '1.1'
+    APP_PUBLICATION_ID = 'all_in_one'
+
+    _VALID_URL = r'https?://hrti.hrt.hr/#/video/show/(?P<id>[0-9]+)/(?P<name>(\w|-)+)?'
+    _TEST = {
+        'url': 'https://hrti.hrt.hr/#/video/show/2181385/republika-dokumentarna-serija-16-hd',
+        'info_dict': {
+            'id': '2181385',
+            'ext': 'mp4',
+            'name': 'REPUBLIKA, dokumentarna serija (4_6)-2251938',
+        },
+        'skip': 'Requires login'
+    }
+
+    def _initialize_api(self):
+        '''Initializes the API and obtains the required urls'''
+        api_url = 'http://clientapi.hrt.hr/client_api.php/config/identify/format/json'
+        app_data = json.dumps({
+            'application_publication_id': HRTiIE.APP_PUBLICATION_ID
+        })
+        self.uuid = self._download_json(api_url, None, note='Getting UUID',
+                                        errnote='Unable to obtain an UUID',
+                                        data=app_data)['uuid']
+
+        app_data = json.dumps({
+            'uuid': self.uuid,
+            'application_publication_id': HRTiIE.APP_PUBLICATION_ID,
+            'screen_height': 1080,
+            'screen_width': 1920,
+            'os': 'Windows',
+            'os_version': 'NT 4.0',
+            'device_model_string_id': 'chrome 42.0.2311.135',
+            'application_version': HRTiIE.APP_VERSION
+        })
+
+        req = sanitized_Request(api_url, data=app_data)
+        req.get_method = lambda: 'PUT'
+
+        resources = self._download_json(
+            req, None, note='Getting API endpoint and session information',
+            errnote='Unable to get endpoint and session information',
+            headers={'Content-type': 'application/json'})
+
+        self.session_id = resources['session_id']
+        modules = resources['modules']
+
+        self.search_url = modules['vod_catalog']['resources']['search']['uri']
+        self.search_url = self.search_url.format(
+            language=HRTiIE.APP_LANGUAGE,
+            application_id=HRTiIE.APP_PUBLICATION_ID)
+
+        self.login_url = modules['user']['resources']['login']['uri']
+        self.login_url = self.login_url.format(session_id=self.session_id)
+        self.login_url += '/format/json'
+
+        self.logout_url = modules['user']['resources']['logout']['uri']
+
+    def _login(self):
+        '''Performs a login to the webservice'''
+        (username, password) = self._get_login_info()
+
+        if username is None or password is None:
+            self.raise_login_required()
+
+        auth_data = json.dumps({
+            'username': username,
+            'password': password,
+        })
+        try:
+            auth_info = self._download_json(
+                self.login_url, None, note='Authenticating',
+                errnote='Unable to log in', data=auth_data)
+        except ExtractorError as ee:
+            if isinstance(ee.cause, compat_HTTPError) and ee.cause.code == 406:
+                raise ExtractorError('Unable to login, ' +
+                                     'incorrect username and/or password')
+            raise
+
+        self.token = auth_info['secure_streaming_token']
+        self.access_token = auth_info['session_token']
+
+        self.logout_url = self.logout_url.format(session_id=self.session_id,
+                                                 access_token=self.access_token)
+        self.logout_url += '/format/json'
+
+    def _real_initialize(self):
+        '''Performs necessary operations so that the information extractor is
+        ready for operation'''
+        self._initialize_api()
+        self._login()
+
+    def _logout(self):
+        '''Performs logout from the webservice'''
+        self._download_json(self.logout_url, None, note='Logout',
+                            errnote='Unable to log out', fatal=False)
+
+    def _real_extract(self, url):
+        '''Extract the data necessary to download the video'''
+        video_id = self._match_id(url)
+
+        metadata_url = self.search_url + \
+            '/video_id/{video_id}/format/json'.format(video_id=video_id)
+
+        metadata = self._download_json(metadata_url, video_id,
+                                       note='Getting video metadata')
+        video = metadata['video'][0]
+        title_info = video.get('title', {})
+        title = title_info.get('title_long')
+        description = title_info.get('summary_long')
+
+        movie = video['video_assets']['movie'][0]
+        url = movie['url'].format(TOKEN=self.token)
+
+        formats = self._extract_m3u8_formats(url, video_id, 'mp4')
+
+        self._sort_formats(formats)
+
+        self._logout()
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'formats': formats,
+        }
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index a2cfb48a6..2b61e6966 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1926,6 +1926,8 @@ def update_Request(req, url=None, data=None, headers={}, query={}):
         origin_req_host=req.origin_req_host, unverifiable=req.unverifiable)
     if hasattr(req, 'timeout'):
         new_req.timeout = req.timeout
+    if req.get_method() == 'PUT':
+        new_req.get_method = lambda : 'PUT'
     return new_req
 
 

From 95cf60e826e38ed1d46743c02549bf0bd187a0bb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 3 Jul 2016 02:21:32 +0700
Subject: [PATCH 5384/6219] [utils] Add PUTRequest

---
 youtube_dl/utils.py | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 2b61e6966..495878a0e 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1625,6 +1625,11 @@ class HEADRequest(compat_urllib_request.Request):
         return 'HEAD'
 
 
+class PUTRequest(compat_urllib_request.Request):
+    def get_method(self):
+        return 'PUT'
+
+
 def int_or_none(v, scale=1, default=None, get_attr=None, invscale=1):
     if get_attr:
         if v is not None:
@@ -1920,14 +1925,18 @@ def update_Request(req, url=None, data=None, headers={}, query={}):
     req_headers.update(headers)
     req_data = data or req.data
     req_url = update_url_query(url or req.get_full_url(), query)
-    req_type = HEADRequest if req.get_method() == 'HEAD' else compat_urllib_request.Request
+    req_get_method = req.get_method()
+    if req_get_method == 'HEAD':
+        req_type = HEADRequest
+    elif req_get_method == 'PUT':
+        req_type = PUTRequest
+    else:
+        req_type = compat_urllib_request.Request
     new_req = req_type(
         req_url, data=req_data, headers=req_headers,
         origin_req_host=req.origin_req_host, unverifiable=req.unverifiable)
     if hasattr(req, 'timeout'):
         new_req.timeout = req.timeout
-    if req.get_method() == 'PUT':
-        new_req.get_method = lambda : 'PUT'
     return new_req
 
 

From e3755a624b68d24b224ec48eb36833dad1585e2b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 3 Jul 2016 02:22:14 +0700
Subject: [PATCH 5385/6219] [hrti] Improve and add support for playlists
 (Closes #9482)

---
 youtube_dl/extractor/hrti.py | 237 +++++++++++++++++++++--------------
 1 file changed, 145 insertions(+), 92 deletions(-)

diff --git a/youtube_dl/extractor/hrti.py b/youtube_dl/extractor/hrti.py
index 55ff981e2..4b346f712 100644
--- a/youtube_dl/extractor/hrti.py
+++ b/youtube_dl/extractor/hrti.py
@@ -2,148 +2,201 @@
 from __future__ import unicode_literals
 
 import json
+import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_HTTPError
-)
+from ..compat import compat_HTTPError
 from ..utils import (
+    clean_html,
+    ExtractorError,
+    int_or_none,
+    parse_age_limit,
     sanitized_Request,
-    ExtractorError
+    try_get,
 )
 
 
-class HRTiIE(InfoExtractor):
-    '''
-    Information Extractor for Croatian Radiotelevision video on demand site
-    https://hrti.hrt.hr
-    Reverse engineered from the JavaScript app in app.min.js
-    '''
+class HRTiBaseIE(InfoExtractor):
+    """
+        Base Information Extractor for Croatian Radiotelevision
+        video on demand site https://hrti.hrt.hr
+        Reverse engineered from the JavaScript app in app.min.js
+    """
     _NETRC_MACHINE = 'hrti'
 
-    APP_LANGUAGE = 'hr'
-    APP_VERSION = '1.1'
-    APP_PUBLICATION_ID = 'all_in_one'
-
-    _VALID_URL = r'https?://hrti.hrt.hr/#/video/show/(?P<id>[0-9]+)/(?P<name>(\w|-)+)?'
-    _TEST = {
-        'url': 'https://hrti.hrt.hr/#/video/show/2181385/republika-dokumentarna-serija-16-hd',
-        'info_dict': {
-            'id': '2181385',
-            'ext': 'mp4',
-            'name': 'REPUBLIKA, dokumentarna serija (4_6)-2251938',
-        },
-        'skip': 'Requires login'
-    }
+    _APP_LANGUAGE = 'hr'
+    _APP_VERSION = '1.1'
+    _APP_PUBLICATION_ID = 'all_in_one'
+    _API_URL = 'http://clientapi.hrt.hr/client_api.php/config/identify/format/json'
 
     def _initialize_api(self):
-        '''Initializes the API and obtains the required urls'''
-        api_url = 'http://clientapi.hrt.hr/client_api.php/config/identify/format/json'
-        app_data = json.dumps({
-            'application_publication_id': HRTiIE.APP_PUBLICATION_ID
-        })
-        self.uuid = self._download_json(api_url, None, note='Getting UUID',
-                                        errnote='Unable to obtain an UUID',
-                                        data=app_data)['uuid']
+        init_data = {
+            'application_publication_id': self._APP_PUBLICATION_ID
+        }
 
-        app_data = json.dumps({
-            'uuid': self.uuid,
-            'application_publication_id': HRTiIE.APP_PUBLICATION_ID,
-            'screen_height': 1080,
-            'screen_width': 1920,
-            'os': 'Windows',
-            'os_version': 'NT 4.0',
-            'device_model_string_id': 'chrome 42.0.2311.135',
-            'application_version': HRTiIE.APP_VERSION
-        })
+        uuid = self._download_json(
+            self._API_URL, None, note='Downloading uuid',
+            errnote='Unable to download uuid',
+            data=json.dumps(init_data).encode('utf-8'))['uuid']
 
-        req = sanitized_Request(api_url, data=app_data)
+        app_data = {
+            'uuid': uuid,
+            'application_publication_id': self._APP_PUBLICATION_ID,
+            'application_version': self._APP_VERSION
+        }
+
+        req = sanitized_Request(self._API_URL, data=json.dumps(app_data).encode('utf-8'))
         req.get_method = lambda: 'PUT'
 
         resources = self._download_json(
-            req, None, note='Getting API endpoint and session information',
-            errnote='Unable to get endpoint and session information',
-            headers={'Content-type': 'application/json'})
+            req, None, note='Downloading session information',
+            errnote='Unable to download session information')
+
+        self._session_id = resources['session_id']
 
-        self.session_id = resources['session_id']
         modules = resources['modules']
 
-        self.search_url = modules['vod_catalog']['resources']['search']['uri']
-        self.search_url = self.search_url.format(
-            language=HRTiIE.APP_LANGUAGE,
-            application_id=HRTiIE.APP_PUBLICATION_ID)
+        self._search_url = modules['vod_catalog']['resources']['search']['uri'].format(
+            language=self._APP_LANGUAGE,
+            application_id=self._APP_PUBLICATION_ID)
 
-        self.login_url = modules['user']['resources']['login']['uri']
-        self.login_url = self.login_url.format(session_id=self.session_id)
-        self.login_url += '/format/json'
+        self._login_url = (modules['user']['resources']['login']['uri'] +
+                           '/format/json').format(session_id=self._session_id)
 
-        self.logout_url = modules['user']['resources']['logout']['uri']
+        self._logout_url = modules['user']['resources']['logout']['uri']
 
     def _login(self):
-        '''Performs a login to the webservice'''
         (username, password) = self._get_login_info()
-
+        # TODO: figure out authentication with cookies
         if username is None or password is None:
             self.raise_login_required()
 
-        auth_data = json.dumps({
+        auth_data = {
             'username': username,
             'password': password,
-        })
+        }
+
         try:
             auth_info = self._download_json(
-                self.login_url, None, note='Authenticating',
-                errnote='Unable to log in', data=auth_data)
-        except ExtractorError as ee:
-            if isinstance(ee.cause, compat_HTTPError) and ee.cause.code == 406:
-                raise ExtractorError('Unable to login, ' +
-                                     'incorrect username and/or password')
-            raise
+                self._login_url, None, note='Logging in', errnote='Unable to log in',
+                data=json.dumps(auth_data).encode('utf-8'))
+        except ExtractorError as e:
+            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 406:
+                auth_info = self._parse_json(e.cause.read().encode('utf-8'), None)
+            else:
+                raise
 
-        self.token = auth_info['secure_streaming_token']
-        self.access_token = auth_info['session_token']
+        error_message = auth_info.get('error', {}).get('message')
+        if error_message:
+            raise ExtractorError(
+                '%s said: %s' % (self.IE_NAME, error_message),
+                expected=True)
 
-        self.logout_url = self.logout_url.format(session_id=self.session_id,
-                                                 access_token=self.access_token)
-        self.logout_url += '/format/json'
+        self._token = auth_info['secure_streaming_token']
 
     def _real_initialize(self):
-        '''Performs necessary operations so that the information extractor is
-        ready for operation'''
         self._initialize_api()
         self._login()
 
-    def _logout(self):
-        '''Performs logout from the webservice'''
-        self._download_json(self.logout_url, None, note='Logout',
-                            errnote='Unable to log out', fatal=False)
+
+class HRTiIE(HRTiBaseIE):
+    _VALID_URL = r'''(?x)
+                        (?:
+                            hrti:(?P<short_id>[0-9]+)|
+                            https?://
+                                hrti\.hrt\.hr/\#/video/show/(?P<id>[0-9]+)/(?P<display_id>[^/]+)?
+                        )
+                    '''
+    _TESTS = [{
+        'url': 'https://hrti.hrt.hr/#/video/show/2181385/republika-dokumentarna-serija-16-hd',
+        'info_dict': {
+            'id': '2181385',
+            'display_id': 'republika-dokumentarna-serija-16-hd',
+            'ext': 'mp4',
+            'title': 'REPUBLIKA, dokumentarna serija (1/6) (HD)',
+            'description': 'md5:48af85f620e8e0e1df4096270568544f',
+            'duration': 2922,
+            'view_count': int,
+            'average_rating': int,
+            'episode_number': int,
+            'season_number': int,
+            'age_limit': 12,
+        },
+        'skip': 'Requires account credentials',
+    }, {
+        'url': 'https://hrti.hrt.hr/#/video/show/2181385/',
+        'only_matching': True,
+    }, {
+        'url': 'hrti:2181385',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
-        '''Extract the data necessary to download the video'''
-        video_id = self._match_id(url)
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('short_id') or mobj.group('id')
+        display_id = mobj.group('display_id') or video_id
 
-        metadata_url = self.search_url + \
-            '/video_id/{video_id}/format/json'.format(video_id=video_id)
+        video = self._download_json(
+            '%s/video_id/%s/format/json' % (self._search_url, video_id),
+            display_id, 'Downloading video metadata JSON')['video'][0]
 
-        metadata = self._download_json(metadata_url, video_id,
-                                       note='Getting video metadata')
-        video = metadata['video'][0]
-        title_info = video.get('title', {})
-        title = title_info.get('title_long')
-        description = title_info.get('summary_long')
+        title_info = video['title']
+        title = title_info['title_long']
 
         movie = video['video_assets']['movie'][0]
-        url = movie['url'].format(TOKEN=self.token)
-
-        formats = self._extract_m3u8_formats(url, video_id, 'mp4')
-
+        m3u8_url = movie['url'].format(TOKEN=self._token)
+        formats = self._extract_m3u8_formats(
+            m3u8_url, display_id, 'mp4', entry_protocol='m3u8_native',
+            m3u8_id='hls')
         self._sort_formats(formats)
 
-        self._logout()
+        description = clean_html(title_info.get('summary_long'))
+        age_limit = parse_age_limit(video.get('parental_control', {}).get('rating'))
+        view_count = int_or_none(video.get('views'))
+        average_rating = int_or_none(video.get('user_rating'))
+        duration = int_or_none(movie.get('duration'))
 
         return {
             'id': video_id,
+            'display_id': display_id,
             'title': title,
             'description': description,
+            'duration': duration,
+            'view_count': view_count,
+            'average_rating': average_rating,
+            'age_limit': age_limit,
             'formats': formats,
         }
+
+
+class HRTiPlaylistIE(HRTiBaseIE):
+    _VALID_URL = r'https?://hrti.hrt.hr/#/video/list/category/(?P<id>[0-9]+)/(?P<display_id>[^/]+)?'
+    _TESTS = [{
+        'url': 'https://hrti.hrt.hr/#/video/list/category/212/ekumena',
+        'info_dict': {
+            'id': '212',
+            'title': 'ekumena',
+        },
+        'playlist_mincount': 8,
+        'skip': 'Requires account credentials',
+    }, {
+        'url': 'https://hrti.hrt.hr/#/video/list/category/212/',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        category_id = mobj.group('id')
+        display_id = mobj.group('display_id') or category_id
+
+        response = self._download_json(
+            '%s/category_id/%s/format/json' % (self._search_url, category_id),
+            display_id, 'Downloading video metadata JSON')
+
+        video_ids = try_get(
+            response, lambda x: x['video_listings'][0]['alternatives'][0]['list'],
+            list) or [video['id'] for video in response.get('videos', []) if video.get('id')]
+
+        entries = [self.url_result('hrti:%s' % category_id) for category_id in video_ids]
+
+        return self.playlist_result(entries, category_id, display_id)

From c3a5dd3b5d636935dea53ee23cc8a7da9a175f0e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 3 Jul 2016 02:22:59 +0700
Subject: [PATCH 5386/6219] Credit @atopuzov for hrti (#9482)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index bdd29687d..f74b30d07 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -176,3 +176,4 @@ Déstin Reed
 Roman Tsiupa
 Artur Krysiak
 Jakub Adam Wieczorek
+Aleksandar Topuzović

From 9b724d7277716fc6f0a0cdc5cdc857ac0ea2642b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 3 Jul 2016 02:25:39 +0700
Subject: [PATCH 5387/6219] [extractors] Add hrti:playlist import

---
 youtube_dl/extractor/extractors.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index ee1718f6a..5dab055db 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -326,7 +326,10 @@ from .hotnewhiphop import HotNewHipHopIE
 from .hotstar import HotStarIE
 from .howcast import HowcastIE
 from .howstuffworks import HowStuffWorksIE
-from .hrti import HRTiIE
+from .hrti import (
+    HRTiIE,
+    HRTiPlaylistIE,
+)
 from .huffpost import HuffPostIE
 from .hypem import HypemIE
 from .iconosquare import IconosquareIE

From 1b734adb2d3bb17af67b5fef933a6fafe71e4cf4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 3 Jul 2016 03:12:15 +0700
Subject: [PATCH 5388/6219] [xtube] Fix extraction (Closes #9953, closes #9961)

---
 youtube_dl/extractor/xtube.py | 45 ++++++++++++++++++++++++-----------
 1 file changed, 31 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/xtube.py b/youtube_dl/extractor/xtube.py
index 4075b8a4f..83bc1fef2 100644
--- a/youtube_dl/extractor/xtube.py
+++ b/youtube_dl/extractor/xtube.py
@@ -4,17 +4,23 @@ import itertools
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse_unquote
 from ..utils import (
     int_or_none,
     orderedSet,
+    parse_duration,
     sanitized_Request,
     str_to_int,
 )
 
 
 class XTubeIE(InfoExtractor):
-    _VALID_URL = r'(?:xtube:|https?://(?:www\.)?xtube\.com/(?:watch\.php\?.*\bv=|video-watch/(?P<display_id>[^/]+)-))(?P<id>[^/?&#]+)'
+    _VALID_URL = r'''(?x)
+                        (?:
+                            xtube:|
+                            https?://(?:www\.)?xtube\.com/(?:watch\.php\?.*\bv=|video-watch/(?P<display_id>[^/]+)-)
+                        )
+                        (?P<id>[^/?&#]+)
+                    '''
 
     _TESTS = [{
         # old URL schema
@@ -27,6 +33,8 @@ class XTubeIE(InfoExtractor):
             'description': 'contains:an ET kind of thing',
             'uploader': 'greenshowers',
             'duration': 450,
+            'view_count': int,
+            'comment_count': int,
             'age_limit': 18,
         }
     }, {
@@ -51,21 +59,30 @@ class XTubeIE(InfoExtractor):
         req.add_header('Cookie', 'age_verified=1; cookiesAccepted=1')
         webpage = self._download_webpage(req, display_id)
 
-        flashvars = self._parse_json(
-            self._search_regex(
-                r'xt\.playerOps\s*=\s*({.+?});', webpage, 'player ops'),
-            video_id)['flashvars']
+        sources = self._parse_json(self._search_regex(
+            r'sources\s*:\s*({.+?}),', webpage, 'sources'), video_id)
 
-        title = flashvars.get('title') or self._search_regex(
-            r'<h1>([^<]+)</h1>', webpage, 'title')
-        video_url = compat_urllib_parse_unquote(flashvars['video_url'])
-        duration = int_or_none(flashvars.get('video_duration'))
+        formats = []
+        for format_id, format_url in sources.items():
+            formats.append({
+                'url': format_url,
+                'format_id': format_id,
+                'height': int_or_none(format_id),
+            })
+        self._sort_formats(formats)
 
-        uploader = self._search_regex(
-            r'<input[^>]+name="contentOwnerId"[^>]+value="([^"]+)"',
-            webpage, 'uploader', fatal=False)
+        title = self._search_regex(
+            (r'<h1>(?P<title>[^<]+)</h1>', r'videoTitle\s*:\s*(["\'])(?P<title>.+?)\1'),
+            webpage, 'title', group='title')
         description = self._search_regex(
             r'</h1>\s*<p>([^<]+)', webpage, 'description', fatal=False)
+        uploader = self._search_regex(
+            (r'<input[^>]+name="contentOwnerId"[^>]+value="([^"]+)"',
+             r'<span[^>]+class="nickname"[^>]*>([^<]+)'),
+            webpage, 'uploader', fatal=False)
+        duration = parse_duration(self._search_regex(
+            r'<dt>Runtime:</dt>\s*<dd>([^<]+)</dd>',
+            webpage, 'duration', fatal=False))
         view_count = str_to_int(self._search_regex(
             r'<dt>Views:</dt>\s*<dd>([\d,\.]+)</dd>',
             webpage, 'view count', fatal=False))
@@ -76,7 +93,6 @@ class XTubeIE(InfoExtractor):
         return {
             'id': video_id,
             'display_id': display_id,
-            'url': video_url,
             'title': title,
             'description': description,
             'uploader': uploader,
@@ -84,6 +100,7 @@ class XTubeIE(InfoExtractor):
             'view_count': view_count,
             'comment_count': comment_count,
             'age_limit': 18,
+            'formats': formats,
         }
 
 

From a0cfd82dda86b4e3e882819216f177b003a1b473 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 3 Jul 2016 03:19:22 +0700
Subject: [PATCH 5389/6219] release 2016.07.03

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 docs/supportedsites.md    | 4 +++-
 youtube_dl/version.py     | 2 +-
 3 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 637103b6b..8dbd3cbfa 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.07.02*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.07.02**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.07.03*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.07.03**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.07.02
+[debug] youtube-dl version 2016.07.03
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 8fd1ab5af..93237022f 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -283,6 +283,8 @@
  - **HotStar**
  - **Howcast**
  - **HowStuffWorks**
+ - **HRTi**
+ - **HRTiPlaylist**
  - **HuffPost**: Huffington Post
  - **Hypem**
  - **Iconosquare**
@@ -329,7 +331,7 @@
  - **kuwo:mv**: 酷我音乐 - MV
  - **kuwo:singer**: 酷我音乐 - 歌手
  - **kuwo:song**: 酷我音乐
- - **la7.tv**
+ - **la7.it**
  - **Laola1Tv**
  - **Le**: 乐视网
  - **Learnr**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index d0483f83b..1dda4c6d1 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.07.02'
+__version__ = '2016.07.03'

From bf83024826e325c474aed5128c012b78a5d15210 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 2 Jul 2016 21:19:07 +0100
Subject: [PATCH 5390/6219] [theplatform] add basic support for Adobe Pass

---
 youtube_dl/extractor/cbsinteractive.py |   5 +-
 youtube_dl/extractor/theplatform.py    | 108 ++++++++++++++++++++++++-
 2 files changed, 105 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/cbsinteractive.py b/youtube_dl/extractor/cbsinteractive.py
index 0011c3029..821db20b2 100644
--- a/youtube_dl/extractor/cbsinteractive.py
+++ b/youtube_dl/extractor/cbsinteractive.py
@@ -80,9 +80,6 @@ class CBSInteractiveIE(ThePlatformIE):
 
         media_guid_path = 'media/guid/%d/%s' % (self.MPX_ACCOUNTS[site], vdata['mpxRefId'])
         formats, subtitles = [], {}
-        if site == 'cnet':
-            formats, subtitles = self._extract_theplatform_smil(
-                self.TP_RELEASE_URL_TEMPLATE % media_guid_path, video_id)
         for (fkey, vid) in vdata['files'].items():
             if fkey == 'hls_phone' and 'hls_tablet' in vdata['files']:
                 continue
@@ -94,7 +91,7 @@ class CBSInteractiveIE(ThePlatformIE):
             subtitles = self._merge_subtitles(subtitles, tp_subtitles)
         self._sort_formats(formats)
 
-        info = self.get_metadata('kYEXFC/%s' % media_guid_path, video_id)
+        info = self._extract_theplatform_metadata('kYEXFC/%s' % media_guid_path, video_id)
         info.update({
             'id': video_id,
             'display_id': display_id,
diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index 07d222ae3..bb3efc4ea 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -6,6 +6,7 @@ import time
 import hmac
 import binascii
 import hashlib
+import netrc
 
 
 from .once import OnceIE
@@ -24,6 +25,9 @@ from ..utils import (
     xpath_with_ns,
     mimetype2ext,
     find_xpath_attr,
+    unescapeHTML,
+    urlencode_postdata,
+    unified_timestamp,
 )
 
 default_ns = 'http://www.w3.org/2005/SMIL21/Language'
@@ -62,10 +66,11 @@ class ThePlatformBaseIE(OnceIE):
 
         return formats, subtitles
 
-    def get_metadata(self, path, video_id):
+    def _download_theplatform_metadata(self, path, video_id):
         info_url = 'http://link.theplatform.com/s/%s?format=preview' % path
-        info = self._download_json(info_url, video_id)
+        return self._download_json(info_url, video_id)
 
+    def _parse_theplatform_metadata(self, info):
         subtitles = {}
         captions = info.get('captions')
         if isinstance(captions, list):
@@ -86,6 +91,10 @@ class ThePlatformBaseIE(OnceIE):
             'uploader': info.get('billingCode'),
         }
 
+    def _extract_theplatform_metadata(self, path, video_id):
+        info = self._download_theplatform_metadata(path, video_id)
+        return self._parse_theplatform_metadata(info)
+
 
 class ThePlatformIE(ThePlatformBaseIE):
     _VALID_URL = r'''(?x)
@@ -158,6 +167,7 @@ class ThePlatformIE(ThePlatformBaseIE):
         'url': 'http://player.theplatform.com/p/NnzsPC/onsite_universal/select/media/guid/2410887629/2928790?fwsitesection=nbc_the_blacklist_video_library&autoPlay=true&carouselID=137781',
         'only_matching': True,
     }]
+    _SERVICE_PROVIDER_TEMPLATE = 'https://sp.auth.adobe.com/adobe-services/%s'
 
     @classmethod
     def _extract_urls(cls, webpage):
@@ -192,6 +202,96 @@ class ThePlatformIE(ThePlatformBaseIE):
         sig = flags + expiration_date + checksum + str_to_hex(sig_secret)
         return '%s&sig=%s' % (url, sig)
 
+    def _extract_mvpd_auth(self, url, video_id, requestor_id, resource):
+        def xml_text(xml_str, tag):
+            return self._search_regex(
+                '<%s>(.+?)</%s>' % (tag, tag), xml_str, tag)
+
+        mvpd_headers = {
+            'ap_42': 'anonymous',
+            'ap_11': 'Linux i686',
+            'ap_z': 'Mozilla/5.0 (X11; Linux i686; rv:47.0) Gecko/20100101 Firefox/47.0',
+            'User-Agent': 'Mozilla/5.0 (X11; Linux i686; rv:47.0) Gecko/20100101 Firefox/47.0',
+        }
+
+        guid = xml_text(resource, 'guid')
+        requestor_info = self._downloader.cache.load('mvpd', requestor_id) or {}
+        authn_token = requestor_info.get('authn_token')
+        if authn_token:
+            token_expires = unified_timestamp(xml_text(authn_token, 'simpleTokenExpires').replace('_GMT', ''))
+            if token_expires and token_expires >= time.time():
+                authn_token = None
+        if not authn_token:
+            # TODO add support for other TV Providers
+            mso_id = 'DTV'
+            login_info = netrc.netrc().authenticators(mso_id)
+            if not login_info:
+                return None
+
+            def post_form(form_page, note, data={}):
+                post_url = self._html_search_regex(r'<form[^>]+action=(["\'])(?P<url>.+?)\1', form_page, 'post url', group='url')
+                return self._download_webpage(
+                    post_url, video_id, note, data=urlencode_postdata(data or self._hidden_inputs(form_page)), headers={
+                        'Content-Type': 'application/x-www-form-urlencoded',
+                    })
+
+            provider_redirect_page = self._download_webpage(
+                self._SERVICE_PROVIDER_TEMPLATE % 'authenticate/saml', video_id,
+                'Downloading Provider Redirect Page', query={
+                    'noflash': 'true',
+                    'mso_id': mso_id,
+                    'requestor_id': requestor_id,
+                    'no_iframe': 'false',
+                    'domain_name': 'adobe.com',
+                    'redirect_url': url,
+                })
+            provider_login_page = post_form(
+                provider_redirect_page, 'Downloading Provider Login Page')
+            mvpd_confirm_page = post_form(provider_login_page, 'Logging in', {
+                'username': login_info[0],
+                'password': login_info[2],
+            })
+            post_form(mvpd_confirm_page, 'Confirming Login')
+
+            session = self._download_webpage(
+                self._SERVICE_PROVIDER_TEMPLATE % 'session', video_id,
+                'Retrieving Session', data=urlencode_postdata({
+                    '_method': 'GET',
+                    'requestor_id': requestor_id,
+                }), headers=mvpd_headers)
+            authn_token = unescapeHTML(xml_text(session, 'authnToken'))
+            requestor_info['authn_token'] = authn_token
+            self._downloader.cache.store('mvpd', requestor_id, requestor_info)
+
+        authz_token = requestor_info.get(guid)
+        if not authz_token:
+            authorize = self._download_webpage(
+                self._SERVICE_PROVIDER_TEMPLATE % 'authorize', video_id,
+                'Retrieving Authorization Token', data=urlencode_postdata({
+                    'resource_id': resource,
+                    'requestor_id': requestor_id,
+                    'authentication_token': authn_token,
+                    'mso_id': xml_text(authn_token, 'simpleTokenMsoID'),
+                    'userMeta': '1',
+                }), headers=mvpd_headers)
+            authz_token = unescapeHTML(xml_text(authorize, 'authzToken'))
+            requestor_info[guid] = authz_token
+            self._downloader.cache.store('mvpd', requestor_id, requestor_info)
+
+        mvpd_headers.update({
+            'ap_19': xml_text(authn_token, 'simpleSamlNameID'),
+            'ap_23': xml_text(authn_token, 'simpleSamlSessionIndex'),
+        })
+
+        return self._download_webpage(
+            self._SERVICE_PROVIDER_TEMPLATE % 'shortAuthorize',
+            video_id, 'Retrieving Media Token', data=urlencode_postdata({
+                'authz_token': authz_token,
+                'requestor_id': requestor_id,
+                'session_guid': xml_text(authn_token, 'simpleTokenAuthenticationGuid'),
+                'hashed_guid': 'false',
+            }), headers=mvpd_headers)
+
     def _real_extract(self, url):
         url, smuggled_data = unsmuggle_url(url, {})
 
@@ -265,7 +365,7 @@ class ThePlatformIE(ThePlatformBaseIE):
         formats, subtitles = self._extract_theplatform_smil(smil_url, video_id)
         self._sort_formats(formats)
 
-        ret = self.get_metadata(path, video_id)
+        ret = self._extract_theplatform_metadata(path, video_id)
         combined_subtitles = self._merge_subtitles(ret.get('subtitles', {}), subtitles)
         ret.update({
             'id': video_id,
@@ -339,7 +439,7 @@ class ThePlatformFeedIE(ThePlatformBaseIE):
         timestamp = int_or_none(entry.get('media$availableDate'), scale=1000)
         categories = [item['media$name'] for item in entry.get('media$categories', [])]
 
-        ret = self.get_metadata('%s/%s' % (provider_id, first_video_id), video_id)
+        ret = self._extract_theplatform_metadata('%s/%s' % (provider_id, first_video_id), video_id)
         subtitles = self._merge_subtitles(subtitles, ret['subtitles'])
         ret.update({
             'id': video_id,

From 05c7feec77f42145815695943d24cb6a9d7c7baa Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 2 Jul 2016 21:20:59 +0100
Subject: [PATCH 5391/6219] [aenetworks] add support Adobe Pass auth

---
 youtube_dl/extractor/aenetworks.py | 77 ++++++++++++++++++++----------
 1 file changed, 51 insertions(+), 26 deletions(-)

diff --git a/youtube_dl/extractor/aenetworks.py b/youtube_dl/extractor/aenetworks.py
index 8b60e2ab6..bc58d7b4d 100644
--- a/youtube_dl/extractor/aenetworks.py
+++ b/youtube_dl/extractor/aenetworks.py
@@ -2,7 +2,7 @@ from __future__ import unicode_literals
 
 import re
 
-from .common import InfoExtractor
+from .theplatform import ThePlatformIE
 from ..utils import (
     smuggle_url,
     update_url_query,
@@ -15,28 +15,15 @@ from ..compat import (
 )
 
 
-class AENetworksBaseIE(InfoExtractor):
-    def theplatform_url_result(self, theplatform_url, video_id, query):
-        return {
-            '_type': 'url_transparent',
-            'id': video_id,
-            'url': smuggle_url(
-                update_url_query(theplatform_url, query),
-                {
-                    'sig': {
-                        'key': 'crazyjava',
-                        'secret': 's3cr3t'
-                    },
-                    'force_smil_url': True
-                }),
-            'ie_key': 'ThePlatform',
-        }
+class AENetworksBaseIE(ThePlatformIE):
+    _THEPLATFORM_KEY = 'crazyjava'
+    _THEPLATFORM_SECRET = 's3cr3t'
 
 
 class AENetworksIE(AENetworksBaseIE):
     IE_NAME = 'aenetworks'
     IE_DESC = 'A+E Networks: A&E, Lifetime, History.com, FYI Network'
-    _VALID_URL = r'https?://(?:www\.)?(?:(?:history|aetv|mylifetime)\.com|fyi\.tv)/(?:shows/(?P<show_path>[^/]+(?:/[^/]+){0,2})|movies/(?P<movie_display_id>[^/]+)/full-movie)'
+    _VALID_URL = r'https?://(?:www\.)?(?P<domain>(?:history|aetv|mylifetime)\.com|fyi\.tv)/(?:shows/(?P<show_path>[^/]+(?:/[^/]+){0,2})|movies/(?P<movie_display_id>[^/]+)/full-movie)'
     _TESTS = [{
         'url': 'http://www.history.com/shows/mountain-men/season-1/episode-1',
         'md5': '8ff93eb073449f151d6b90c0ae1ef0c7',
@@ -76,9 +63,15 @@ class AENetworksIE(AENetworksBaseIE):
         'url': 'http://www.mylifetime.com/movies/center-stage-on-pointe/full-movie',
         'only_matching': True
     }]
+    _DOMAIN_TO_REQUESTOR_ID = {
+        'history.com': 'HISTORY',
+        'aetv.com': 'AETV',
+        'mylifetime.com': 'LIFETIME',
+        'fyi.tv': 'FYI',
+    }
 
     def _real_extract(self, url):
-        show_path, movie_display_id = re.match(self._VALID_URL, url).groups()
+        domain, show_path, movie_display_id = re.match(self._VALID_URL, url).groups()
         display_id = show_path or movie_display_id
         webpage = self._download_webpage(url, display_id)
         if show_path:
@@ -103,16 +96,32 @@ class AENetworksIE(AENetworksBaseIE):
                         episode_attributes['data-videoid']))
                 return self.playlist_result(
                     entries, self._html_search_meta('aetn:SeasonId', webpage))
+
+        query = {
+            'mbr': 'true',
+            'assetTypes': 'medium_video_s3'
+        }
         video_id = self._html_search_meta('aetn:VideoID', webpage)
         media_url = self._search_regex(
             r"media_url\s*=\s*'([^']+)'", webpage, 'video url')
-
-        info = self._search_json_ld(webpage, video_id, fatal=False)
-        info.update(self.theplatform_url_result(
-            media_url, video_id, {
-                'mbr': 'true',
-                'assetTypes': 'medium_video_s3'
-            }))
+        theplatform_metadata = self._download_theplatform_metadata(self._search_regex(
+            r'https?://link.theplatform.com/s/([^?]+)', media_url, 'theplatform_path'), video_id)
+        info = self._parse_theplatform_metadata(theplatform_metadata)
+        if theplatform_metadata.get('AETN$isBehindWall'):
+            requestor_id = self._DOMAIN_TO_REQUESTOR_ID[domain]
+            resource = '<rss version="2.0" xmlns:media="http://search.yahoo.com/mrss/"><channel><title>%s</title><item><title>%s</title><guid>%s</guid><media:rating scheme="urn:v-chip">%s</media:rating></item></channel></rss>' % (requestor_id, theplatform_metadata['title'], theplatform_metadata['AETN$PPL_pplProgramId'], theplatform_metadata['ratings'][0]['rating'])
+            query['auth'] = self._extract_mvpd_auth(
+                url, video_id, requestor_id, resource)
+        info.update(self._search_json_ld(webpage, video_id, fatal=False))
+        media_url = update_url_query(media_url, query)
+        media_url = self._sign_url(media_url, self._THEPLATFORM_KEY, self._THEPLATFORM_SECRET)
+        formats, subtitles = self._extract_theplatform_smil(media_url, video_id)
+        self._sort_formats(formats)
+        info.update({
+            'id': video_id,
+            'formats': formats,
+            'subtitles': subtitles,
+        })
         return info
 
 
@@ -149,6 +158,22 @@ class HistoryTopicIE(AENetworksBaseIE):
         'only_matching': True,
     }]
 
+    def theplatform_url_result(self, theplatform_url, video_id, query):
+        return {
+            '_type': 'url_transparent',
+            'id': video_id,
+            'url': smuggle_url(
+                update_url_query(theplatform_url, query),
+                {
+                    'sig': {
+                        'key': self._THEPLATFORM_KEY,
+                        'secret': self._THEPLATFORM_SECRET,
+                    },
+                    'force_smil_url': True
+                }),
+            'ie_key': 'ThePlatform',
+        }
+
     def _real_extract(self, url):
         topic_id, video_display_id = re.match(self._VALID_URL, url).groups()
         if video_display_id:

From a1f6f5c768a506674928530990b7f95c605eac2c Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 2 Jul 2016 21:22:08 +0100
Subject: [PATCH 5392/6219] [nationalgeographic] add support Adobe Pass auth

---
 youtube_dl/extractor/nationalgeographic.py | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/nationalgeographic.py b/youtube_dl/extractor/nationalgeographic.py
index 722518663..ed76798aa 100644
--- a/youtube_dl/extractor/nationalgeographic.py
+++ b/youtube_dl/extractor/nationalgeographic.py
@@ -1,6 +1,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from .theplatform import ThePlatformIE
 from ..utils import (
     smuggle_url,
     url_basename,
@@ -61,7 +62,7 @@ class NationalGeographicIE(InfoExtractor):
         }
 
 
-class NationalGeographicChannelIE(InfoExtractor):
+class NationalGeographicChannelIE(ThePlatformIE):
     IE_NAME = 'natgeo:channel'
     _VALID_URL = r'https?://channel\.nationalgeographic\.com/(?:wild/)?[^/]+/videos/(?P<id>[^/?]+)'
 
@@ -102,12 +103,22 @@ class NationalGeographicChannelIE(InfoExtractor):
         release_url = self._search_regex(
             r'video_auth_playlist_url\s*=\s*"([^"]+)"',
             webpage, 'release url')
+        query = {
+            'mbr': 'true',
+            'manifest': 'm3u',
+        }
+        is_auth = self._search_regex(r'video_is_auth\s*=\s*"([^"]+)"', webpage, 'is auth', fatal=False)
+        if is_auth == 'auth':
+            auth_resource_id = self._search_regex(
+                r"video_auth_resourceId\s*=\s*'([^']+)'",
+                webpage, 'auth resource id')
+            query['auth'] = self._extract_mvpd_auth(url, display_id, 'natgeo', auth_resource_id) or ''
 
         return {
             '_type': 'url_transparent',
             'ie_key': 'ThePlatform',
             'url': smuggle_url(
-                update_url_query(release_url, {'mbr': 'true', 'switch': 'http'}),
+                update_url_query(release_url, query),
                 {'force_smil_url': True}),
             'display_id': display_id,
         }

From 4cb13d0d6af20906f973f0b281f9efba4bc65d41 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 2 Jul 2016 23:02:14 +0200
Subject: [PATCH 5393/6219] [hrti] Don't redefine variable in list
 comprehension

---
 youtube_dl/extractor/hrti.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/hrti.py b/youtube_dl/extractor/hrti.py
index 4b346f712..656ce6d05 100644
--- a/youtube_dl/extractor/hrti.py
+++ b/youtube_dl/extractor/hrti.py
@@ -197,6 +197,6 @@ class HRTiPlaylistIE(HRTiBaseIE):
             response, lambda x: x['video_listings'][0]['alternatives'][0]['list'],
             list) or [video['id'] for video in response.get('videos', []) if video.get('id')]
 
-        entries = [self.url_result('hrti:%s' % category_id) for category_id in video_ids]
+        entries = [self.url_result('hrti:%s' % video_id) for video_id in video_ids]
 
         return self.playlist_result(entries, category_id, display_id)

From 04006fae8df634ae98abd5bc8e538d0a5addf3fc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 3 Jul 2016 11:31:07 +0700
Subject: [PATCH 5394/6219] [README.md] Start writing youtube-dl coding
 conventions

---
 README.md | 152 +++++++++++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 146 insertions(+), 6 deletions(-)

diff --git a/README.md b/README.md
index c6feef116..1f014b48f 100644
--- a/README.md
+++ b/README.md
@@ -890,9 +890,17 @@ If you want to add support for a new site, first of all **make sure** this site
 After you have ensured this site is distributing it's content legally, you can follow this quick list (assuming your service is called `yourextractor`):
 
 1. [Fork this repository](https://github.com/rg3/youtube-dl/fork)
-2. Check out the source code with `git clone git@github.com:YOUR_GITHUB_USERNAME/youtube-dl.git`
-3. Start a new git branch with `cd youtube-dl; git checkout -b yourextractor`
+2. Check out the source code with:
+
+        git clone git@github.com:YOUR_GITHUB_USERNAME/youtube-dl.git
+
+3. Start a new git branch with
+
+        cd youtube-dl
+        git checkout -b yourextractor
+
 4. Start with this simple template and save it to `youtube_dl/extractor/yourextractor.py`:
+
     ```python
     # coding: utf-8
     from __future__ import unicode_literals
@@ -936,19 +944,151 @@ After you have ensured this site is distributing it's content legally, you can f
 5. Add an import in [`youtube_dl/extractor/extractors.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/extractors.py).
 6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc.
 7. Have a look at [`youtube_dl/extractor/common.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L74-L252). Add tests and code for as many as you want.
-8. Keep in mind that the only mandatory fields in info dict for successful extraction process are `id`, `title` and either `url` or `formats`, i.e. these are the critical data the extraction does not make any sense without. This means that [any field](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L148-L252) apart from aforementioned mandatory ones should be treated **as optional** and extraction should be **tolerate** to situations when sources for these fields can potentially be unavailable (even if they always available at the moment) and **future-proof** in order not to break the extraction of general purpose mandatory fields. For example, if you have some intermediate dict `meta` that is a source of metadata and it has a key `summary` that you want to extract and put into resulting info dict as `description`, you should be ready that this key may be missing from the `meta` dict, i.e. you should extract it as `meta.get('summary')` and not `meta['summary']`. Similarly, you should pass `fatal=False` when extracting data from a webpage with `_search_regex/_html_search_regex`.
-9. Check the code with [flake8](https://pypi.python.org/pypi/flake8). Also make sure your code works under all [Python](http://www.python.org/) versions claimed supported by youtube-dl, namely 2.6, 2.7, and 3.2+.
-10. When the tests pass, [add](http://git-scm.com/docs/git-add) the new files and [commit](http://git-scm.com/docs/git-commit) them and [push](http://git-scm.com/docs/git-push) the result, like this:
+8. Make sure your code follows [youtube-dl coding conventions](#youtube-dl-coding-conventions) and check the code with [flake8](https://pypi.python.org/pypi/flake8). Also make sure your code works under all [Python](http://www.python.org/) versions claimed supported by youtube-dl, namely 2.6, 2.7, and 3.2+.
+9. When the tests pass, [add](http://git-scm.com/docs/git-add) the new files and [commit](http://git-scm.com/docs/git-commit) them and [push](http://git-scm.com/docs/git-push) the result, like this:
 
         $ git add youtube_dl/extractor/extractors.py
         $ git add youtube_dl/extractor/yourextractor.py
         $ git commit -m '[yourextractor] Add new extractor'
         $ git push origin yourextractor
 
-11. Finally, [create a pull request](https://help.github.com/articles/creating-a-pull-request). We'll then review and merge it.
+10. Finally, [create a pull request](https://help.github.com/articles/creating-a-pull-request). We'll then review and merge it.
 
 In any case, thank you very much for your contributions!
 
+## youtube-dl coding conventions
+
+This section introduces a guide lines for writing idiomatic, robust and future-proof extractor code.
+
+Extractors are very fragile by nature since they depend on the layout of the source data provided by 3rd party media hoster out of your control and this layout tend to change. As extractor implementer your task is not only to write code that will extract media links and metadata correctly but also to minimize code dependency on source's layout changes and even to make the code foresee potential future changes and be ready for that. This is important because it will allow extractor not to break on minor layout changes thus keeping old youtube-dl version working. Even though this breakage issue is easily fixed by emitting a new version of youtube-dl with fix incorporated all the previous version become broken in all repositories and distros' packages that may not be so prompt in fetching the update from us. Needless to say some may never receive an update at all that is possible for non rolling release distros.
+
+### Mandatory and optional metafields
+
+For extraction to work youtube-dl relies on metadata your extractor extracts and provides to youtube-dl expressed by [information dictionary](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L148-L252) or simply *info dict*. Only the following meta fields in *info dict* are considered mandatory for successful extraction process by youtube-dl:
+
+ - `id` (media identifier)
+ - `title` (media title)
+ - `url` (media download URL) or `formats`
+
+In fact only the last option is technically mandatory (i.e. if you can't figure out the download location of the media the extraction does not make any sense). But by convention youtube-dl also treats `id` and `title` to be mandatory. Thus aforementioned metafields are the critical data the extraction does not make any sense without and if any of them fail to be extracted then extractor is considered completely broken.
+
+[Any field](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L148-L252) apart from the aforementioned ones are considered **optional**. That means that extraction should be **tolerate** to situations when sources for these fields can potentially be unavailable (even if they always available at the moment) and **future-proof** in order not to break the extraction of general purpose mandatory fields.
+
+#### Example
+
+Say you have some source dictionary `meta` that you've fetched as JSON with HTTP request and it has a key `summary`:
+
+```python
+meta = self._download_json(url, video_id)
+```
+    
+Assume at this point `meta`'s layout is:
+
+```python
+{
+    ...
+    "summary": "some fancy summary text",
+    ...
+}
+```
+
+Assume you want to extract `summary` and put into resulting info dict as `description`. Since `description` is optional metafield you should be ready that this key may be missing from the `meta` dict, so that you should extract it like:
+
+```python
+description = meta.get('summary')  # correct
+```
+
+and not like:
+
+```python
+description = meta['summary']  # incorrect
+```
+
+The latter will break extraction process with `KeyError` if `summary` disappears from `meta` at some time later but with former approach extraction will just go ahead with `description` set to `None` that is perfectly fine (remember `None` is equivalent for absence of data). 
+
+Similarly, you should pass `fatal=False` when extracting optional data from a webpage with `_search_regex`, `_html_search_regex` or similar methods, for instance:
+
+```python
+description = self._search_regex(
+    r'<span[^>]+id="title"[^>]*>([^<]+)<',
+    webpage, 'description', fatal=False)
+```
+
+With `fatal` set to `False` if `_search_regex` fails to extract `description` it will emit a warning and continue extraction.
+
+You can also pass `default=<some fallback value>`, for example:
+
+```python
+description = self._search_regex(
+    r'<span[^>]+id="title"[^>]*>([^<]+)<',
+    webpage, 'description', default=None)
+```
+
+On failure this code will silently continue the extraction with `description` set to `None`. That is useful for metafields that are known to may or may not be present.
+ 
+### Provide fallbacks
+
+When extracting metadata try to provide several scenarios for that. For example if `title` is present in several places/sources try extracting from at least some of them. This would make it more future-proof in case some of the sources became unavailable.
+
+#### Example
+
+Say `meta` from previous example has a `title` and you are about to extract it. Since `title` is mandatory meta field you should end up with something like:
+
+```python
+title = meta['title']
+```
+
+If `title` disappeares from `meta` in future due to some changes on hoster's side the extraction would fail since `title` is mandatory. That's expected.
+
+Assume that you have some another source you can extract `title` from, for example `og:title` HTML meta of a `webpage`. In this case you can provide a fallback scenario:
+
+```python
+title = meta.get('title') or self._og_search_title(webpage)
+```
+
+This code will try to extract from `meta` first and if it fails it will try extracting `og:title` from a `webpage`.
+
+### Make regular expressions flexible
+
+When using regular expressions try to write them fuzzy and flexible.
+ 
+#### Example
+
+Say you need to extract `title` from the following HTML code:
+
+```html
+<span style="position: absolute; left: 910px; width: 90px; float: right; z-index: 9999;" class="title">some fancy title</span>
+```
+
+The code for that task should look similar to:
+
+```python
+title = self._search_regex(
+    r'<span[^>]+class="title"[^>]*>([^<]+)', webpage, 'title')
+```
+
+Or even better:
+
+```python
+title = self._search_regex(
+    r'<span[^>]+class=(["\'])title\1[^>]*>(?P<title>[^<]+)',
+    webpage, 'title', group='title')
+```
+
+Note how you tolerate potential changes in `style` attribute's value or switch from using double quotes to single for `class` attribute: 
+
+The code definitely should not look like:
+
+```python
+title = self._search_regex(
+    r'<span style="position: absolute; left: 910px; width: 90px; float: right; z-index: 9999;" class="title">(.*?)</span>',
+    webpage, 'title', group='title')
+```
+
+### Use safe conversion functions
+
+Wrap all extracted numeric data into safe functions from `utils`: `int_or_none`, `float_or_none`. Use them for string to number conversions as well.
+
 # EMBEDDING YOUTUBE-DL
 
 youtube-dl makes the best effort to be a good command-line program, and thus should be callable from any programming language. If you encounter any problems parsing its output, feel free to [create a report](https://github.com/rg3/youtube-dl/issues/new).

From 1f552340574d2702fd0a80880aa6def1764f124a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 3 Jul 2016 11:31:49 +0700
Subject: [PATCH 5395/6219] Add PULL_REQUEST_TEMPLATE.md

---
 .github/PULL_REQUEST_TEMPLATE.md | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)
 create mode 100644 .github/PULL_REQUEST_TEMPLATE.md

diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
new file mode 100644
index 000000000..f24bb4b09
--- /dev/null
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -0,0 +1,22 @@
+## Please follow the guide below
+
+- You will be asked some questions, please read them **carefully** and answer honestly
+- Put an `x` into all the boxes [ ] relevant to your *pull request* (like that [x])
+- Use *Preview* tab to see how your *pull request* will actually look like
+
+---
+
+### Before submitting a *pull request* make sure you have:
+- [ ] At least skimmed through [adding new extractor tutorial](https://github.com/rg3/youtube-dl#adding-support-for-a-new-site) and [youtube-dl coding conventions](https://github.com/rg3/youtube-dl#youtube-dl-coding-conventions) sections
+- [ ] [Searched](https://github.com/rg3/youtube-dl/search?q=is%3Apr&type=Issues) the bugtracker for similar pull requests
+
+### What is the purpose of your *pull request*?
+- [ ] Bug fix
+- [ ] New extractor
+- [ ] New feature
+
+---
+
+### Description of your *pull request* and other information
+
+Explanation of your *pull request* in arbitrary form goes here. Please make sure the description explains the purpose and effect of your *pull request* and is worded well enough to be understood. Provide as much context and examples as possible.

From c723d1cd8dad872791ea2abdfb1bdfcbeb477f84 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 3 Jul 2016 11:35:13 +0700
Subject: [PATCH 5396/6219] [README.md] Update some codebase links

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 1f014b48f..6218060d6 100644
--- a/README.md
+++ b/README.md
@@ -964,7 +964,7 @@ Extractors are very fragile by nature since they depend on the layout of the sou
 
 ### Mandatory and optional metafields
 
-For extraction to work youtube-dl relies on metadata your extractor extracts and provides to youtube-dl expressed by [information dictionary](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L148-L252) or simply *info dict*. Only the following meta fields in *info dict* are considered mandatory for successful extraction process by youtube-dl:
+For extraction to work youtube-dl relies on metadata your extractor extracts and provides to youtube-dl expressed by [information dictionary](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L75-L257) or simply *info dict*. Only the following meta fields in *info dict* are considered mandatory for successful extraction process by youtube-dl:
 
  - `id` (media identifier)
  - `title` (media title)
@@ -972,7 +972,7 @@ For extraction to work youtube-dl relies on metadata your extractor extracts and
 
 In fact only the last option is technically mandatory (i.e. if you can't figure out the download location of the media the extraction does not make any sense). But by convention youtube-dl also treats `id` and `title` to be mandatory. Thus aforementioned metafields are the critical data the extraction does not make any sense without and if any of them fail to be extracted then extractor is considered completely broken.
 
-[Any field](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L148-L252) apart from the aforementioned ones are considered **optional**. That means that extraction should be **tolerate** to situations when sources for these fields can potentially be unavailable (even if they always available at the moment) and **future-proof** in order not to break the extraction of general purpose mandatory fields.
+[Any field](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L149-L257) apart from the aforementioned ones are considered **optional**. That means that extraction should be **tolerate** to situations when sources for these fields can potentially be unavailable (even if they always available at the moment) and **future-proof** in order not to break the extraction of general purpose mandatory fields.
 
 #### Example
 

From 2cb31d288e945a98a1e23023e17c01c4dc5c1cb0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 3 Jul 2016 13:01:04 +0700
Subject: [PATCH 5397/6219] [history:topic] Relax _VALID_URL

---
 youtube_dl/extractor/aenetworks.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/aenetworks.py b/youtube_dl/extractor/aenetworks.py
index bc58d7b4d..8f53050c9 100644
--- a/youtube_dl/extractor/aenetworks.py
+++ b/youtube_dl/extractor/aenetworks.py
@@ -128,7 +128,7 @@ class AENetworksIE(AENetworksBaseIE):
 class HistoryTopicIE(AENetworksBaseIE):
     IE_NAME = 'history:topic'
     IE_DESC = 'History.com Topic'
-    _VALID_URL = r'https?://(?:www\.)?history\.com/topics/(?:[^/]+/)?(?P<topic_id>[^/]+)/videos(?:/(?P<video_display_id>[^/?#]+))?'
+    _VALID_URL = r'https?://(?:www\.)?history\.com/topics/(?:[^/]+/)?(?P<topic_id>[^/]+)(?:/[^/]+(?:/(?P<video_display_id>[^/?#]+))?)?'
     _TESTS = [{
         'url': 'http://www.history.com/topics/valentines-day/history-of-valentines-day/videos/bet-you-didnt-know-valentines-day?m=528e394da93ae&s=undefined&f=1&free=false',
         'info_dict': {
@@ -156,6 +156,12 @@ class HistoryTopicIE(AENetworksBaseIE):
     }, {
         'url': 'http://www.history.com/topics/world-war-i-history/videos',
         'only_matching': True,
+    }, {
+        'url': 'http://www.history.com/topics/world-war-i/world-war-i-history',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.history.com/topics/world-war-i/world-war-i-history/speeches',
+        'only_matching': True,
     }]
 
     def theplatform_url_result(self, theplatform_url, video_id, query):

From 369bb0620620d6499dc1d8db0f9b2624a76d941d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 3 Jul 2016 14:11:29 +0800
Subject: [PATCH 5398/6219] [facebook] Improve embed detection (#5701)

---
 youtube_dl/extractor/facebook.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index bbdb14366..f5d4f966a 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -139,7 +139,7 @@ class FacebookIE(InfoExtractor):
         # Facebook API embed
         # see https://developers.facebook.com/docs/plugins/embedded-video-player
         mobj = re.search(r'''(?x)<div[^>]+
-                class=(?P<q1>[\'"])[^\'"]*\bfb-video\b[^\'"]*(?P=q1)[^>]+
+                class=(?P<q1>[\'"])[^\'"]*\bfb-(?:video|post)\b[^\'"]*(?P=q1)[^>]+
                 data-href=(?P<q2>[\'"])(?P<url>(?:https?:)?//(?:www\.)?facebook.com/.+?)(?P=q2)''', webpage)
         if mobj is not None:
             return mobj.group('url')

From e793338c880f32f32eaec34e9bd338a097ab8fb0 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 3 Jul 2016 14:12:02 +0800
Subject: [PATCH 5399/6219] [buzzfeed] Detect Facebook embed and update _TESTS

Closes #5701
---
 youtube_dl/extractor/buzzfeed.py | 32 +++++++++++++++++++++++++++-----
 1 file changed, 27 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/buzzfeed.py b/youtube_dl/extractor/buzzfeed.py
index df503ecc0..75fa92d7c 100644
--- a/youtube_dl/extractor/buzzfeed.py
+++ b/youtube_dl/extractor/buzzfeed.py
@@ -5,6 +5,7 @@ import json
 import re
 
 from .common import InfoExtractor
+from .facebook import FacebookIE
 
 
 class BuzzFeedIE(InfoExtractor):
@@ -20,11 +21,11 @@ class BuzzFeedIE(InfoExtractor):
             'info_dict': {
                 'id': 'aVCR29aE_OQ',
                 'ext': 'mp4',
+                'title': 'Angry Ram destroys a punching bag..',
+                'description': 'md5:c59533190ef23fd4458a5e8c8c872345',
                 'upload_date': '20141024',
                 'uploader_id': 'Buddhanz1',
-                'description': 'He likes to stay in shape with his heavy bag, he wont stop until its on the ground\n\nFollow Angry Ram on Facebook for regular updates -\nhttps://www.facebook.com/pages/Angry-Ram/1436897249899558?ref=hl',
-                'uploader': 'Buddhanz',
-                'title': 'Angry Ram destroys a punching bag',
+                'uploader': 'Angry Ram',
             }
         }]
     }, {
@@ -41,13 +42,30 @@ class BuzzFeedIE(InfoExtractor):
             'info_dict': {
                 'id': 'mVmBL8B-In0',
                 'ext': 'mp4',
+                'title': 're:Munchkin the Teddy Bear gets her exercise',
+                'description': 'md5:28faab95cda6e361bcff06ec12fc21d8',
                 'upload_date': '20141124',
                 'uploader_id': 'CindysMunchkin',
-                'description': 're:© 2014 Munchkin the',
                 'uploader': 're:^Munchkin the',
-                'title': 're:Munchkin the Teddy Bear gets her exercise',
             },
         }]
+    }, {
+        'url': 'http://www.buzzfeed.com/craigsilverman/the-most-adorable-crash-landing-ever#.eq7pX0BAmK',
+        'info_dict': {
+            'id': 'the-most-adorable-crash-landing-ever',
+            'title': 'Watch This Baby Goose Make The Most Adorable Crash Landing',
+            'description': 'This gosling knows how to stick a landing.',
+        },
+        'playlist': [{
+            'md5': '763ca415512f91ca62e4621086900a23',
+            'info_dict': {
+                'id': '971793786185728',
+                'ext': 'mp4',
+                'title': 'We set up crash pads so that the goslings on our roof would have a safe landi...',
+                'uploader': 'Calgary Outdoor Centre-University of Calgary',
+            },
+        }],
+        'add_ie': ['Facebook'],
     }]
 
     def _real_extract(self, url):
@@ -66,6 +84,10 @@ class BuzzFeedIE(InfoExtractor):
                 continue
             entries.append(self.url_result(video['url']))
 
+        facebook_url = FacebookIE._extract_url(webpage)
+        if facebook_url:
+            entries.append(self.url_result(facebook_url))
+
         return {
             '_type': 'playlist',
             'id': playlist_id,

From f1388739002a7fd1e8e9c41b642734786fc6c391 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 3 Jul 2016 15:39:24 +0800
Subject: [PATCH 5400/6219] [rai] Fix extraction and update _TESTS

Closes #8617
Closes #9157
Closes #9232
---
 youtube_dl/extractor/rai.py | 109 ++++++++++++++++++------------------
 1 file changed, 54 insertions(+), 55 deletions(-)

diff --git a/youtube_dl/extractor/rai.py b/youtube_dl/extractor/rai.py
index e36ce1aa1..946741f43 100644
--- a/youtube_dl/extractor/rai.py
+++ b/youtube_dl/extractor/rai.py
@@ -1,18 +1,16 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse,
-    compat_urlparse,
-)
+from ..compat import compat_urlparse
 from ..utils import (
-    ExtractorError,
     determine_ext,
+    ExtractorError,
+    find_xpath_attr,
+    fix_xml_ampersands,
+    int_or_none,
     parse_duration,
     unified_strdate,
-    int_or_none,
+    update_url_query,
     xpath_text,
 )
 
@@ -22,26 +20,30 @@ class RaiTVIE(InfoExtractor):
     _TESTS = [
         {
             'url': 'http://www.rai.tv/dl/RaiTV/programmi/media/ContentItem-cb27157f-9dd0-4aee-b788-b1f67643a391.html',
-            'md5': '96382709b61dd64a6b88e0f791e6df4c',
+            'md5': '8970abf8caf8aef4696e7b1f2adfc696',
             'info_dict': {
                 'id': 'cb27157f-9dd0-4aee-b788-b1f67643a391',
-                'ext': 'flv',
+                'ext': 'mp4',
                 'title': 'Report del 07/04/2014',
                 'description': 'md5:f27c544694cacb46a078db84ec35d2d9',
                 'upload_date': '20140407',
                 'duration': 6160,
+                'thumbnail': 're:^https?://.*\.jpg$',
             }
         },
         {
+            # no m3u8 stream
             'url': 'http://www.raisport.rai.it/dl/raiSport/media/rassegna-stampa-04a9f4bd-b563-40cf-82a6-aad3529cb4a9.html',
-            'md5': 'd9751b78eac9710d62c2447b224dea39',
+            # HDS download, MD5 is unstable
             'info_dict': {
                 'id': '04a9f4bd-b563-40cf-82a6-aad3529cb4a9',
                 'ext': 'flv',
                 'title': 'TG PRIMO TEMPO',
                 'upload_date': '20140612',
                 'duration': 1758,
+                'thumbnail': 're:^https?://.*\.jpg$',
             },
+            'skip': 'Geo-restricted to Italy',
         },
         {
             'url': 'http://www.rainews.it/dl/rainews/media/state-of-the-net-Antonella-La-Carpia-regole-virali-7aafdea9-0e5d-49d5-88a6-7e65da67ae13.html',
@@ -67,20 +69,22 @@ class RaiTVIE(InfoExtractor):
         },
         {
             'url': 'http://www.ilcandidato.rai.it/dl/ray/media/Il-Candidato---Primo-episodio-Le-Primarie-28e5525a-b495-45e8-a7c3-bc48ba45d2b6.html',
-            'md5': '496ab63e420574447f70d02578333437',
+            'md5': 'e57493e1cb8bc7c564663f363b171847',
             'info_dict': {
                 'id': '28e5525a-b495-45e8-a7c3-bc48ba45d2b6',
-                'ext': 'flv',
+                'ext': 'mp4',
                 'title': 'Il Candidato - Primo episodio: "Le Primarie"',
                 'description': 'md5:364b604f7db50594678f483353164fb8',
                 'upload_date': '20140923',
                 'duration': 386,
+                'thumbnail': 're:^https?://.*\.jpg$',
             }
         },
     ]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
+
         media = self._download_json(
             'http://www.rai.tv/dl/RaiTV/programmi/media/ContentItem-%s.html?json' % video_id,
             video_id, 'Downloading video JSON')
@@ -90,10 +94,9 @@ class RaiTVIE(InfoExtractor):
             thumbnail_url = media.get(image_type)
             if thumbnail_url:
                 thumbnails.append({
-                    'url': thumbnail_url,
+                    'url': compat_urlparse.urljoin(url, thumbnail_url),
                 })
 
-        subtitles = []
         formats = []
         media_type = media['type']
         if 'Audio' in media_type:
@@ -103,49 +106,60 @@ class RaiTVIE(InfoExtractor):
                 'ext': media.get('formatoAudio'),
             })
         elif 'Video' in media_type:
-            def fix_xml(xml):
-                return xml.replace(' tag elementi', '').replace('>/', '</')
+            for platform in ('mon', 'flash', 'native'):
+                headers = {}
+                # TODO: rename --cn-verification-proxy
+                cn_verification_proxy = self._downloader.params.get('cn_verification_proxy')
+                if cn_verification_proxy:
+                    headers['Ytdl-request-proxy'] = cn_verification_proxy
 
-            relinker = self._download_xml(
-                media['mediaUri'] + '&output=43',
-                video_id, transform_source=fix_xml)
+                relinker = self._download_xml(
+                    media['mediaUri'], video_id,
+                    note='Downloading XML metadata for platform %s' % platform,
+                    transform_source=fix_xml_ampersands,
+                    query={'output': 45, 'pl': platform}, headers=headers)
 
-            has_subtitle = False
+                media_url = find_xpath_attr(relinker, './url', 'type', 'content').text
+                if media_url == 'http://download.rai.it/video_no_available.mp4':
+                    self.raise_geo_restricted()
 
-            for element in relinker.findall('element'):
-                media_url = xpath_text(element, 'url')
                 ext = determine_ext(media_url)
-                content_type = xpath_text(element, 'content-type')
+                if (platform == 'mon' and ext != 'm3u8') or (platform == 'flash' and ext != 'f4m'):
+                    continue
+
                 if ext == 'm3u8':
                     formats.extend(self._extract_m3u8_formats(
                         media_url, video_id, 'mp4', 'm3u8_native',
                         m3u8_id='hls', fatal=False))
                 elif ext == 'f4m':
+                    manifest_url = update_url_query(
+                        media_url, {'hdcore': '3.7.0', 'plugin': 'aasp-3.7.0.39.44'})
                     formats.extend(self._extract_f4m_formats(
-                        media_url + '?hdcore=3.7.0&plugin=aasp-3.7.0.39.44',
-                        video_id, f4m_id='hds', fatal=False))
-                elif ext == 'stl':
-                    has_subtitle = True
-                elif content_type.startswith('video/'):
-                    bitrate = int_or_none(xpath_text(element, 'bitrate'))
+                        manifest_url, video_id, f4m_id='hds', fatal=False))
+                else:
+                    bitrate = int_or_none(xpath_text(relinker, 'bitrate'))
                     formats.append({
                         'url': media_url,
                         'tbr': bitrate if bitrate > 0 else None,
                         'format_id': 'http-%d' % bitrate if bitrate > 0 else 'http',
                     })
-                elif content_type.startswith('image/'):
-                    thumbnails.append({
-                        'url': media_url,
-                    })
 
             self._sort_formats(formats)
-
-            if has_subtitle:
-                webpage = self._download_webpage(url, video_id)
-                subtitles = self._get_subtitles(video_id, webpage)
         else:
             raise ExtractorError('not a media file')
 
+        subtitles = {}
+        captions = media.get('subtitlesUrl')
+        if captions:
+            STL_EXT = '.stl'
+            SRT_EXT = '.srt'
+            if captions.endswith(STL_EXT):
+                captions = captions[:-len(STL_EXT)] + SRT_EXT
+            subtitles['it'] = [{
+                'ext': 'srt',
+                'url': captions,
+            }]
+
         return {
             'id': video_id,
             'title': media['name'],
@@ -158,31 +172,16 @@ class RaiTVIE(InfoExtractor):
             'subtitles': subtitles,
         }
 
-    def _get_subtitles(self, video_id, webpage):
-        subtitles = {}
-        m = re.search(r'<meta name="closedcaption" content="(?P<captions>[^"]+)"', webpage)
-        if m:
-            captions = m.group('captions')
-            STL_EXT = '.stl'
-            SRT_EXT = '.srt'
-            if captions.endswith(STL_EXT):
-                captions = captions[:-len(STL_EXT)] + SRT_EXT
-            subtitles['it'] = [{
-                'ext': 'srt',
-                'url': 'http://www.rai.tv%s' % compat_urllib_parse.quote(captions),
-            }]
-        return subtitles
-
 
 class RaiIE(InfoExtractor):
     _VALID_URL = r'https?://(?:.+?\.)?(?:rai\.it|rai\.tv|rainews\.it)/dl/.+?-(?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})(?:-.+?)?\.html'
     _TESTS = [
         {
             'url': 'http://www.report.rai.it/dl/Report/puntata/ContentItem-0c7a664b-d0f4-4b2c-8835-3f82e46f433e.html',
-            'md5': 'e0e7a8a131e249d1aa0ebf270d1d8db7',
+            'md5': '2dd727e61114e1ee9c47f0da6914e178',
             'info_dict': {
                 'id': '59d69d28-6bb6-409d-a4b5-ed44096560af',
-                'ext': 'flv',
+                'ext': 'mp4',
                 'title': 'Il pacco',
                 'description': 'md5:4b1afae1364115ce5d78ed83cd2e5b3a',
                 'upload_date': '20141221',

From 64436cb1a453d6f92a72ac4e28cde1fa03b00fc7 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 3 Jul 2016 10:43:36 +0100
Subject: [PATCH 5401/6219] [nationalgeographic] skip download for national
 geographic channel tests(closes #9991)

---
 youtube_dl/extractor/nationalgeographic.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/youtube_dl/extractor/nationalgeographic.py b/youtube_dl/extractor/nationalgeographic.py
index ed76798aa..6eabd2278 100644
--- a/youtube_dl/extractor/nationalgeographic.py
+++ b/youtube_dl/extractor/nationalgeographic.py
@@ -79,6 +79,10 @@ class NationalGeographicChannelIE(ThePlatformIE):
                 'upload_date': '20160322',
                 'uploader': 'NEWA-FNG-NGTV',
             },
+            'params': {
+                # m3u8 download
+                'skip_download': True,
+            },
             'add_ie': ['ThePlatform'],
         },
         {
@@ -93,6 +97,10 @@ class NationalGeographicChannelIE(ThePlatformIE):
                 'upload_date': '20160330',
                 'uploader': 'NEWA-FNG-NGTV',
             },
+            'params': {
+                # m3u8 download
+                'skip_download': True,
+            },
             'add_ie': ['ThePlatform'],
         },
     ]

From 034a884957c4b3d9cb9585f3c0d634d99a3e3389 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 3 Jul 2016 19:22:48 +0800
Subject: [PATCH 5402/6219] [rai] Support direct relinker URLs (closes #8552)

---
 youtube_dl/extractor/rai.py | 126 +++++++++++++++++++++++-------------
 1 file changed, 81 insertions(+), 45 deletions(-)

diff --git a/youtube_dl/extractor/rai.py b/youtube_dl/extractor/rai.py
index 946741f43..b1d3df1ba 100644
--- a/youtube_dl/extractor/rai.py
+++ b/youtube_dl/extractor/rai.py
@@ -15,8 +15,54 @@ from ..utils import (
 )
 
 
-class RaiTVIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:.+?\.)?(?:rai\.it|rai\.tv|rainews\.it)/dl/(?:[^/]+/)+media/.+?-(?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})(?:-.+?)?\.html'
+class RaiBaseIE(InfoExtractor):
+    def _extract_relinker_formats(self, relinker_url, video_id):
+        formats = []
+
+        for platform in ('mon', 'flash', 'native'):
+            headers = {}
+            # TODO: rename --cn-verification-proxy
+            cn_verification_proxy = self._downloader.params.get('cn_verification_proxy')
+            if cn_verification_proxy:
+                headers['Ytdl-request-proxy'] = cn_verification_proxy
+
+            relinker = self._download_xml(
+                relinker_url, video_id,
+                note='Downloading XML metadata for platform %s' % platform,
+                transform_source=fix_xml_ampersands,
+                query={'output': 45, 'pl': platform}, headers=headers)
+
+            media_url = find_xpath_attr(relinker, './url', 'type', 'content').text
+            if media_url == 'http://download.rai.it/video_no_available.mp4':
+                self.raise_geo_restricted()
+
+            ext = determine_ext(media_url)
+            if (ext == 'm3u8' and platform != 'mon') or (ext == 'f4m' and platform != 'flash'):
+                continue
+
+            if ext == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    media_url, video_id, 'mp4', 'm3u8_native',
+                    m3u8_id='hls', fatal=False))
+            elif ext == 'f4m':
+                manifest_url = update_url_query(
+                    media_url.replace('manifest#live_hds.f4m', 'manifest.f4m'),
+                    {'hdcore': '3.7.0', 'plugin': 'aasp-3.7.0.39.44'})
+                formats.extend(self._extract_f4m_formats(
+                    manifest_url, video_id, f4m_id='hds', fatal=False))
+            else:
+                bitrate = int_or_none(xpath_text(relinker, 'bitrate'))
+                formats.append({
+                    'url': media_url,
+                    'tbr': bitrate if bitrate > 0 else None,
+                    'format_id': 'http-%d' % bitrate if bitrate > 0 else 'http',
+                })
+
+        return formats
+
+
+class RaiTVIE(RaiBaseIE):
+    _VALID_URL = r'https?://(?:.+?\.)?(?:rai\.it|rai\.tv|rainews\.it)/dl/(?:[^/]+/)+(?:media|ondemand)/.+?-(?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})(?:-.+?)?\.html'
     _TESTS = [
         {
             'url': 'http://www.rai.tv/dl/RaiTV/programmi/media/ContentItem-cb27157f-9dd0-4aee-b788-b1f67643a391.html',
@@ -106,44 +152,7 @@ class RaiTVIE(InfoExtractor):
                 'ext': media.get('formatoAudio'),
             })
         elif 'Video' in media_type:
-            for platform in ('mon', 'flash', 'native'):
-                headers = {}
-                # TODO: rename --cn-verification-proxy
-                cn_verification_proxy = self._downloader.params.get('cn_verification_proxy')
-                if cn_verification_proxy:
-                    headers['Ytdl-request-proxy'] = cn_verification_proxy
-
-                relinker = self._download_xml(
-                    media['mediaUri'], video_id,
-                    note='Downloading XML metadata for platform %s' % platform,
-                    transform_source=fix_xml_ampersands,
-                    query={'output': 45, 'pl': platform}, headers=headers)
-
-                media_url = find_xpath_attr(relinker, './url', 'type', 'content').text
-                if media_url == 'http://download.rai.it/video_no_available.mp4':
-                    self.raise_geo_restricted()
-
-                ext = determine_ext(media_url)
-                if (platform == 'mon' and ext != 'm3u8') or (platform == 'flash' and ext != 'f4m'):
-                    continue
-
-                if ext == 'm3u8':
-                    formats.extend(self._extract_m3u8_formats(
-                        media_url, video_id, 'mp4', 'm3u8_native',
-                        m3u8_id='hls', fatal=False))
-                elif ext == 'f4m':
-                    manifest_url = update_url_query(
-                        media_url, {'hdcore': '3.7.0', 'plugin': 'aasp-3.7.0.39.44'})
-                    formats.extend(self._extract_f4m_formats(
-                        manifest_url, video_id, f4m_id='hds', fatal=False))
-                else:
-                    bitrate = int_or_none(xpath_text(relinker, 'bitrate'))
-                    formats.append({
-                        'url': media_url,
-                        'tbr': bitrate if bitrate > 0 else None,
-                        'format_id': 'http-%d' % bitrate if bitrate > 0 else 'http',
-                    })
-
+            formats.extend(self._extract_relinker_formats(media['mediaUri'], video_id))
             self._sort_formats(formats)
         else:
             raise ExtractorError('not a media file')
@@ -173,7 +182,7 @@ class RaiTVIE(InfoExtractor):
         }
 
 
-class RaiIE(InfoExtractor):
+class RaiIE(RaiBaseIE):
     _VALID_URL = r'https?://(?:.+?\.)?(?:rai\.it|rai\.tv|rainews\.it)/dl/.+?-(?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})(?:-.+?)?\.html'
     _TESTS = [
         {
@@ -186,6 +195,16 @@ class RaiIE(InfoExtractor):
                 'description': 'md5:4b1afae1364115ce5d78ed83cd2e5b3a',
                 'upload_date': '20141221',
             },
+        },
+        {
+            'url': 'http://www.rai.tv/dl/RaiTV/dirette/PublishingBlock-1912dbbf-3f96-44c3-b4cf-523681fbacbc.html?channel=EuroNews',
+            # HDS live stream, MD5 is unstable
+            'info_dict': {
+                'id': '1912dbbf-3f96-44c3-b4cf-523681fbacbc',
+                'ext': 'flv',
+                'title': 'EuroNews',
+            },
+            'skip': 'Geo-restricted to Italy',
         }
     ]
 
@@ -200,7 +219,24 @@ class RaiIE(InfoExtractor):
         iframe_url = self._search_regex(
             [r'<iframe[^>]+src="([^"]*/dl/[^"]+\?iframe\b[^"]*)"',
              r'drawMediaRaiTV\(["\'](.+?)["\']'],
-            webpage, 'iframe')
-        if not iframe_url.startswith('http'):
-            iframe_url = compat_urlparse.urljoin(url, iframe_url)
-        return self.url_result(iframe_url)
+            webpage, 'iframe', default=None)
+        if iframe_url:
+            if not iframe_url.startswith('http'):
+                iframe_url = compat_urlparse.urljoin(url, iframe_url)
+            return self.url_result(iframe_url)
+
+        relinker_url = compat_urlparse.urljoin(url, self._search_regex(
+            r'var\s+videoURL\s*=\s*(?P<q1>[\'"])(?P<url>(https?:)?//mediapolis\.rai\.it/relinker/relinkerServlet\.htm\?cont=\d+)(?P=q1)',
+            webpage, 'relinker URL', group='url'))
+        formats = self._extract_relinker_formats(relinker_url, video_id)
+        self._sort_formats(formats)
+
+        title = self._search_regex(
+            r'var\s+videoTitolo\s*=\s*([\'"])(?P<title>[^\'"]+)\1',
+            webpage, 'title', group='title', default=None) or self._og_search_title(webpage)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': formats,
+        }

From 477b7a847479ffea131e2d9a4993fa3f53540e85 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 3 Jul 2016 19:25:40 +0800
Subject: [PATCH 5403/6219] [downloader/f4m] Fix for Rai live streams

---
 youtube_dl/downloader/f4m.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index 8f88b0241..80c21d40b 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -196,6 +196,11 @@ def build_fragments_list(boot_info):
     first_frag_number = fragment_run_entry_table[0]['first']
     fragments_counter = itertools.count(first_frag_number)
     for segment, fragments_count in segment_run_table['segment_run']:
+        # In some live HDS streams (for example Rai), `fragments_count` is
+        # abnormal and causing out-of-memory errors. It's OK to change the
+        # number of fragments for live streams as they are updated periodically
+        if fragments_count == 4294967295 and boot_info['live']:
+            fragments_count = 2
         for _ in range(fragments_count):
             res.append((segment, next(fragments_counter)))
 
@@ -329,7 +334,11 @@ class F4mFD(FragmentFD):
 
         base_url = compat_urlparse.urljoin(man_url, media.attrib['url'])
         bootstrap_node = doc.find(_add_ns('bootstrapInfo'))
-        boot_info, bootstrap_url = self._parse_bootstrap_node(bootstrap_node, base_url)
+        # From Adobe F4M 3.0 spec:
+        # The <baseURL> element SHALL be the base URL for all relative
+        # (HTTP-based) URLs in the manifest. If <baseURL> is not present, said
+        # URLs should be relative to the location of the containing document.
+        boot_info, bootstrap_url = self._parse_bootstrap_node(bootstrap_node, man_url)
         live = boot_info['live']
         metadata_node = media.find(_add_ns('metadata'))
         if metadata_node is not None:

From 7507fc98cb7ab0f78317cd6b485f88830953645d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 3 Jul 2016 18:35:28 +0700
Subject: [PATCH 5404/6219] [README.md] Fix somes typo in coding conventions
 section

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 6218060d6..cd9e0b43f 100644
--- a/README.md
+++ b/README.md
@@ -960,7 +960,7 @@ In any case, thank you very much for your contributions!
 
 This section introduces a guide lines for writing idiomatic, robust and future-proof extractor code.
 
-Extractors are very fragile by nature since they depend on the layout of the source data provided by 3rd party media hoster out of your control and this layout tend to change. As extractor implementer your task is not only to write code that will extract media links and metadata correctly but also to minimize code dependency on source's layout changes and even to make the code foresee potential future changes and be ready for that. This is important because it will allow extractor not to break on minor layout changes thus keeping old youtube-dl version working. Even though this breakage issue is easily fixed by emitting a new version of youtube-dl with fix incorporated all the previous version become broken in all repositories and distros' packages that may not be so prompt in fetching the update from us. Needless to say some may never receive an update at all that is possible for non rolling release distros.
+Extractors are very fragile by nature since they depend on the layout of the source data provided by 3rd party media hoster out of your control and this layout tend to change. As an extractor implementer your task is not only to write code that will extract media links and metadata correctly but also to minimize code dependency on source's layout changes and even to make the code foresee potential future changes and be ready for that. This is important because it will allow extractor not to break on minor layout changes thus keeping old youtube-dl versions working. Even though this breakage issue is easily fixed by emitting a new version of youtube-dl with fix incorporated all the previous version become broken in all repositories and distros' packages that may not be so prompt in fetching the update from us. Needless to say some may never receive an update at all that is possible for non rolling release distros.
 
 ### Mandatory and optional metafields
 
@@ -972,7 +972,7 @@ For extraction to work youtube-dl relies on metadata your extractor extracts and
 
 In fact only the last option is technically mandatory (i.e. if you can't figure out the download location of the media the extraction does not make any sense). But by convention youtube-dl also treats `id` and `title` to be mandatory. Thus aforementioned metafields are the critical data the extraction does not make any sense without and if any of them fail to be extracted then extractor is considered completely broken.
 
-[Any field](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L149-L257) apart from the aforementioned ones are considered **optional**. That means that extraction should be **tolerate** to situations when sources for these fields can potentially be unavailable (even if they always available at the moment) and **future-proof** in order not to break the extraction of general purpose mandatory fields.
+[Any field](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L149-L257) apart from the aforementioned ones are considered **optional**. That means that extraction should be **tolerate** to situations when sources for these fields can potentially be unavailable (even if they are always available at the moment) and **future-proof** in order not to break the extraction of general purpose mandatory fields.
 
 #### Example
 

From 2b28b892d8f103951992b50b4ed76a4729a748fa Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 3 Jul 2016 19:44:58 +0800
Subject: [PATCH 5405/6219] [rai] Support videos with embedded content item ID
 (#8551)

---
 youtube_dl/extractor/rai.py | 121 +++++++++++++++++++++---------------
 1 file changed, 71 insertions(+), 50 deletions(-)

diff --git a/youtube_dl/extractor/rai.py b/youtube_dl/extractor/rai.py
index b1d3df1ba..eb32bbdb6 100644
--- a/youtube_dl/extractor/rai.py
+++ b/youtube_dl/extractor/rai.py
@@ -60,6 +60,57 @@ class RaiBaseIE(InfoExtractor):
 
         return formats
 
+    def _extract_from_content_id(self, content_id, base_url):
+        media = self._download_json(
+            'http://www.rai.tv/dl/RaiTV/programmi/media/ContentItem-%s.html?json' % content_id,
+            content_id, 'Downloading video JSON')
+
+        thumbnails = []
+        for image_type in ('image', 'image_medium', 'image_300'):
+            thumbnail_url = media.get(image_type)
+            if thumbnail_url:
+                thumbnails.append({
+                    'url': compat_urlparse.urljoin(base_url, thumbnail_url),
+                })
+
+        formats = []
+        media_type = media['type']
+        if 'Audio' in media_type:
+            formats.append({
+                'format_id': media.get('formatoAudio'),
+                'url': media['audioUrl'],
+                'ext': media.get('formatoAudio'),
+            })
+        elif 'Video' in media_type:
+            formats.extend(self._extract_relinker_formats(media['mediaUri'], content_id))
+            self._sort_formats(formats)
+        else:
+            raise ExtractorError('not a media file')
+
+        subtitles = {}
+        captions = media.get('subtitlesUrl')
+        if captions:
+            STL_EXT = '.stl'
+            SRT_EXT = '.srt'
+            if captions.endswith(STL_EXT):
+                captions = captions[:-len(STL_EXT)] + SRT_EXT
+            subtitles['it'] = [{
+                'ext': 'srt',
+                'url': captions,
+            }]
+
+        return {
+            'id': content_id,
+            'title': media['name'],
+            'description': media.get('desc'),
+            'thumbnails': thumbnails,
+            'uploader': media.get('author'),
+            'upload_date': unified_strdate(media.get('date')),
+            'duration': parse_duration(media.get('length')),
+            'formats': formats,
+            'subtitles': subtitles,
+        }
+
 
 class RaiTVIE(RaiBaseIE):
     _VALID_URL = r'https?://(?:.+?\.)?(?:rai\.it|rai\.tv|rainews\.it)/dl/(?:[^/]+/)+(?:media|ondemand)/.+?-(?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})(?:-.+?)?\.html'
@@ -131,55 +182,7 @@ class RaiTVIE(RaiBaseIE):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        media = self._download_json(
-            'http://www.rai.tv/dl/RaiTV/programmi/media/ContentItem-%s.html?json' % video_id,
-            video_id, 'Downloading video JSON')
-
-        thumbnails = []
-        for image_type in ('image', 'image_medium', 'image_300'):
-            thumbnail_url = media.get(image_type)
-            if thumbnail_url:
-                thumbnails.append({
-                    'url': compat_urlparse.urljoin(url, thumbnail_url),
-                })
-
-        formats = []
-        media_type = media['type']
-        if 'Audio' in media_type:
-            formats.append({
-                'format_id': media.get('formatoAudio'),
-                'url': media['audioUrl'],
-                'ext': media.get('formatoAudio'),
-            })
-        elif 'Video' in media_type:
-            formats.extend(self._extract_relinker_formats(media['mediaUri'], video_id))
-            self._sort_formats(formats)
-        else:
-            raise ExtractorError('not a media file')
-
-        subtitles = {}
-        captions = media.get('subtitlesUrl')
-        if captions:
-            STL_EXT = '.stl'
-            SRT_EXT = '.srt'
-            if captions.endswith(STL_EXT):
-                captions = captions[:-len(STL_EXT)] + SRT_EXT
-            subtitles['it'] = [{
-                'ext': 'srt',
-                'url': captions,
-            }]
-
-        return {
-            'id': video_id,
-            'title': media['name'],
-            'description': media.get('desc'),
-            'thumbnails': thumbnails,
-            'uploader': media.get('author'),
-            'upload_date': unified_strdate(media.get('date')),
-            'duration': parse_duration(media.get('length')),
-            'formats': formats,
-            'subtitles': subtitles,
-        }
+        return self._extract_from_content_id(video_id, url)
 
 
 class RaiIE(RaiBaseIE):
@@ -197,6 +200,7 @@ class RaiIE(RaiBaseIE):
             },
         },
         {
+            # Direct relinker URL
             'url': 'http://www.rai.tv/dl/RaiTV/dirette/PublishingBlock-1912dbbf-3f96-44c3-b4cf-523681fbacbc.html?channel=EuroNews',
             # HDS live stream, MD5 is unstable
             'info_dict': {
@@ -205,7 +209,18 @@ class RaiIE(RaiBaseIE):
                 'title': 'EuroNews',
             },
             'skip': 'Geo-restricted to Italy',
-        }
+        },
+        {
+            # Embedded content item ID
+            'url': 'http://www.tg1.rai.it/dl/tg1/2010/edizioni/ContentSet-9b6e0cba-4bef-4aef-8cf0-9f7f665b7dfb-tg1.html?item=undefined',
+            'md5': '84c1135ce960e8822ae63cec34441d63',
+            'info_dict': {
+                'id': '0960e765-62c8-474a-ac4b-7eb3e2be39c8',
+                'ext': 'mp4',
+                'title': 'TG1 ore 20:00 del 02/07/2016',
+                'upload_date': '20160702',
+            },
+        },
     ]
 
     @classmethod
@@ -225,6 +240,12 @@ class RaiIE(RaiBaseIE):
                 iframe_url = compat_urlparse.urljoin(url, iframe_url)
             return self.url_result(iframe_url)
 
+        content_item_id = self._search_regex(
+            r'initEdizione\((?P<q1>[\'"])ContentItem-(?P<content_id>[^\'"]+)(?P=q1)',
+            webpage, 'content item ID', group='content_id', default=None)
+        if content_item_id:
+            return self._extract_from_content_id(content_item_id, url)
+
         relinker_url = compat_urlparse.urljoin(url, self._search_regex(
             r'var\s+videoURL\s*=\s*(?P<q1>[\'"])(?P<url>(https?:)?//mediapolis\.rai\.it/relinker/relinkerServlet\.htm\?cont=\d+)(?P=q1)',
             webpage, 'relinker URL', group='url'))

From 15e4b6b758daadab14ad520c55713a8b35a3db7e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 3 Jul 2016 19:48:50 +0800
Subject: [PATCH 5406/6219] [rai] Support an alternative form of embedded
 relinker URL

Closes #8551
---
 youtube_dl/extractor/rai.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/rai.py b/youtube_dl/extractor/rai.py
index eb32bbdb6..1f0fcd609 100644
--- a/youtube_dl/extractor/rai.py
+++ b/youtube_dl/extractor/rai.py
@@ -221,6 +221,15 @@ class RaiIE(RaiBaseIE):
                 'upload_date': '20160702',
             },
         },
+        {
+            'url': 'http://www.rainews.it/dl/rainews/live/ContentItem-3156f2f2-dc70-4953-8e2f-70d7489d4ce9.html',
+            # HDS live stream, MD5 is unstable
+            'info_dict': {
+                'id': '3156f2f2-dc70-4953-8e2f-70d7489d4ce9',
+                'ext': 'flv',
+                'title': 'La diretta di Rainews24',
+            },
+        },
     ]
 
     @classmethod
@@ -247,7 +256,7 @@ class RaiIE(RaiBaseIE):
             return self._extract_from_content_id(content_item_id, url)
 
         relinker_url = compat_urlparse.urljoin(url, self._search_regex(
-            r'var\s+videoURL\s*=\s*(?P<q1>[\'"])(?P<url>(https?:)?//mediapolis\.rai\.it/relinker/relinkerServlet\.htm\?cont=\d+)(?P=q1)',
+            r'(?:var\s+videoURL|mediaInfo\.mediaUri)\s*=\s*(?P<q1>[\'"])(?P<url>(https?:)?//mediapolis\.rai\.it/relinker/relinkerServlet\.htm\?cont=\d+)(?P=q1)',
             webpage, 'relinker URL', group='url'))
         formats = self._extract_relinker_formats(relinker_url, video_id)
         self._sort_formats(formats)

From a5f67895d32f2e152a0e59d0b25326536c6ad6c3 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 3 Jul 2016 14:06:24 +0100
Subject: [PATCH 5407/6219] [nationalgeographic] restore http formats

there was a misunderstanding about the reason of 403 response
the problem happen only when the user use aria2c as a downloader
https://github.com/rg3/youtube-dl/commit/a1f6f5c768a506674928530990b7f95c605eac2c#commitcomment-18107559
---
 youtube_dl/extractor/nationalgeographic.py | 10 +---------
 1 file changed, 1 insertion(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/nationalgeographic.py b/youtube_dl/extractor/nationalgeographic.py
index 6eabd2278..e717abb9f 100644
--- a/youtube_dl/extractor/nationalgeographic.py
+++ b/youtube_dl/extractor/nationalgeographic.py
@@ -79,10 +79,6 @@ class NationalGeographicChannelIE(ThePlatformIE):
                 'upload_date': '20160322',
                 'uploader': 'NEWA-FNG-NGTV',
             },
-            'params': {
-                # m3u8 download
-                'skip_download': True,
-            },
             'add_ie': ['ThePlatform'],
         },
         {
@@ -97,10 +93,6 @@ class NationalGeographicChannelIE(ThePlatformIE):
                 'upload_date': '20160330',
                 'uploader': 'NEWA-FNG-NGTV',
             },
-            'params': {
-                # m3u8 download
-                'skip_download': True,
-            },
             'add_ie': ['ThePlatform'],
         },
     ]
@@ -113,7 +105,7 @@ class NationalGeographicChannelIE(ThePlatformIE):
             webpage, 'release url')
         query = {
             'mbr': 'true',
-            'manifest': 'm3u',
+            'switch': 'http',
         }
         is_auth = self._search_regex(r'video_is_auth\s*=\s*"([^"]+)"', webpage, 'is auth', fatal=False)
         if is_auth == 'auth':

From 2644e911bebcace4fbac7ddd3ea8c71936b1b3e9 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 3 Jul 2016 22:18:48 +0800
Subject: [PATCH 5408/6219] [iqiyi] Fix extraction

See https://github.com/soimort/you-get/issues/1211#issuecomment-229011559
---
 youtube_dl/extractor/iqiyi.py | 157 ++++++++++------------------------
 1 file changed, 43 insertions(+), 114 deletions(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index fea26685e..754e6057c 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -1,10 +1,8 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import binascii
 import hashlib
 import itertools
-import math
 import re
 import time
 
@@ -14,12 +12,13 @@ from ..compat import (
     compat_urllib_parse_urlencode,
 )
 from ..utils import (
+    clean_html,
     decode_packed_codes,
+    get_element_by_id,
+    get_element_by_attribute,
     ExtractorError,
-    intlist_to_bytes,
     ohdave_rsa_encrypt,
     remove_start,
-    urshift,
 )
 
 
@@ -166,7 +165,7 @@ class IqiyiIE(InfoExtractor):
 
     _TESTS = [{
         'url': 'http://www.iqiyi.com/v_19rrojlavg.html',
-        'md5': '470a6c160618577166db1a7aac5a3606',
+        'md5': '5b0591f55961117155430b5d544fdb01',
         'info_dict': {
             'id': '9c1fb1b99d192b21c559e5a1a2cb3c73',
             'ext': 'mp4',
@@ -174,11 +173,11 @@ class IqiyiIE(InfoExtractor):
         }
     }, {
         'url': 'http://www.iqiyi.com/v_19rrhnnclk.html',
-        'md5': 'f09f0a6a59b2da66a26bf4eda669a4cc',
+        'md5': '667171934041350c5de3f5015f7f1152',
         'info_dict': {
             'id': 'e3f585b550a280af23c98b6cb2be19fb',
             'ext': 'mp4',
-            'title': '名侦探柯南 国语版',
+            'title': '名侦探柯南 国语版：第752集 迫近灰原秘密的黑影 下篇',
         },
         'skip': 'Geo-restricted to China',
     }, {
@@ -196,22 +195,10 @@ class IqiyiIE(InfoExtractor):
         'url': 'http://www.iqiyi.com/v_19rrny4w8w.html',
         'info_dict': {
             'id': 'f3cf468b39dddb30d676f89a91200dc1',
+            'ext': 'mp4',
             'title': '泰坦尼克号',
         },
-        'playlist': [{
-            'info_dict': {
-                'id': 'f3cf468b39dddb30d676f89a91200dc1_part1',
-                'ext': 'f4v',
-                'title': '泰坦尼克号',
-            },
-        }, {
-            'info_dict': {
-                'id': 'f3cf468b39dddb30d676f89a91200dc1_part2',
-                'ext': 'f4v',
-                'title': '泰坦尼克号',
-            },
-        }],
-        'expected_warnings': ['Needs a VIP account for full video'],
+        'skip': 'Geo-restricted to China',
     }, {
         'url': 'http://www.iqiyi.com/a_19rrhb8ce1.html',
         'info_dict': {
@@ -224,14 +211,16 @@ class IqiyiIE(InfoExtractor):
         'only_matching': True,
     }]
 
-    _FORMATS_MAP = [
-        ('1', 'h6'),
-        ('2', 'h5'),
-        ('3', 'h4'),
-        ('4', 'h3'),
-        ('5', 'h2'),
-        ('10', 'h1'),
-    ]
+    _FORMATS_MAP = {
+        '96': 1,    # 216p, 240p
+        '1': 2,     # 336p, 360p
+        '2': 3,     # 480p, 504p
+        '21': 4,    # 504p
+        '4': 5,     # 720p
+        '17': 5,    # 720p
+        '5': 6,     # 1072p, 1080p
+        '18': 7,    # 1080p
+    }
 
     def _real_initialize(self):
         self._login()
@@ -291,91 +280,17 @@ class IqiyiIE(InfoExtractor):
 
         return True
 
-    @staticmethod
-    def _gen_sc(tvid, timestamp):
-        M = [1732584193, -271733879]
-        M.extend([~M[0], ~M[1]])
-        I_table = [7, 12, 17, 22, 5, 9, 14, 20, 4, 11, 16, 23, 6, 10, 15, 21]
-        C_base = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8388608, 432]
-
-        def L(n, t):
-            if t is None:
-                t = 0
-            return trunc(((n >> 1) + (t >> 1) << 1) + (n & 1) + (t & 1))
-
-        def trunc(n):
-            n = n % 0x100000000
-            if n > 0x7fffffff:
-                n -= 0x100000000
-            return n
-
-        def transform(string, mod):
-            num = int(string, 16)
-            return (num >> 8 * (i % 4) & 255 ^ i % mod) << ((a & 3) << 3)
-
-        C = list(C_base)
-        o = list(M)
-        k = str(timestamp - 7)
-        for i in range(13):
-            a = i
-            C[a >> 2] |= ord(k[a]) << 8 * (a % 4)
-
-        for i in range(16):
-            a = i + 13
-            start = (i >> 2) * 8
-            r = '03967743b643f66763d623d637e30733'
-            C[a >> 2] |= transform(''.join(reversed(r[start:start + 8])), 7)
-
-        for i in range(16):
-            a = i + 29
-            start = (i >> 2) * 8
-            r = '7038766939776a32776a32706b337139'
-            C[a >> 2] |= transform(r[start:start + 8], 1)
-
-        for i in range(9):
-            a = i + 45
-            if i < len(tvid):
-                C[a >> 2] |= ord(tvid[i]) << 8 * (a % 4)
-
-        for a in range(64):
-            i = a
-            I = i >> 4
-            C_index = [i, 5 * i + 1, 3 * i + 5, 7 * i][I] % 16 + urshift(a, 6)
-            m = L(L(o[0], [
-                trunc(o[1] & o[2]) | trunc(~o[1] & o[3]),
-                trunc(o[3] & o[1]) | trunc(~o[3] & o[2]),
-                o[1] ^ o[2] ^ o[3],
-                o[2] ^ trunc(o[1] | ~o[3])
-            ][I]), L(
-                trunc(int(abs(math.sin(i + 1)) * 4294967296)),
-                C[C_index] if C_index < len(C) else None))
-            I = I_table[4 * I + i % 4]
-            o = [o[3],
-                 L(o[1], trunc(trunc(m << I) | urshift(m, 32 - I))),
-                 o[1],
-                 o[2]]
-
-        new_M = [L(o[0], M[0]), L(o[1], M[1]), L(o[2], M[2]), L(o[3], M[3])]
-        s = [new_M[a >> 3] >> (1 ^ a & 7) * 4 & 15 for a in range(32)]
-        return binascii.hexlify(intlist_to_bytes(s))[1::2].decode('ascii')
-
     def get_raw_data(self, tvid, video_id):
         tm = int(time.time() * 1000)
 
-        sc = self._gen_sc(tvid, tm)
+        key = 'd5fb4bd9d50c4be6948c97edd7254b0e'
+        sc = md5_text(compat_str(tm) + key + tvid)
         params = {
-            'platForm': 'h5',
-            'rate': 1,
             'tvid': tvid,
             'vid': video_id,
-            'cupid': 'qc_100001_100186',
-            'type': 'mp4',
-            'nolimit': 0,
-            'agenttype': 13,
-            'src': 'd846d0c32d664d32b6b54ea48997a589',
+            'src': '76f90cbd92f94a2e925d83e8ccd22cb7',
             'sc': sc,
-            't': tm - 7,
-            '__jsT': None,
+            't': tm,
         }
 
         headers = {}
@@ -435,6 +350,7 @@ class IqiyiIE(InfoExtractor):
         video_id = self._search_regex(
             r'data-player-videoid\s*=\s*[\'"]([a-f\d]+)', webpage, 'video_id')
 
+        formats = []
         for _ in range(5):
             raw_data = self.get_raw_data(tvid, video_id)
 
@@ -445,16 +361,29 @@ class IqiyiIE(InfoExtractor):
 
             data = raw_data['data']
 
-            # iQiYi sometimes returns Ads
-            if not isinstance(data['playInfo'], dict):
-                self._sleep(5, video_id)
-                continue
+            for stream in data['vidl']:
+                if 'm3utx' not in stream:
+                    continue
+                vd = compat_str(stream['vd'])
+                formats.append({
+                    'url': stream['m3utx'],
+                    'format_id': vd,
+                    'ext': 'mp4',
+                    'preference': self._FORMATS_MAP.get(vd, -1),
+                    'protocol': 'm3u8_native',
+                })
 
-            title = data['playInfo']['an']
-            break
+            if formats:
+                break
+
+            self._sleep(5, video_id)
+
+        self._sort_formats(formats)
+        title = (get_element_by_id('widget-videotitle', webpage) or
+                 clean_html(get_element_by_attribute('class', 'mod-play-tit', webpage)))
 
         return {
             'id': video_id,
             'title': title,
-            'url': data['m3u'],
+            'formats': formats,
         }

From bff98341d5c5944fd8fb74797b70b0d3db6a3c89 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 3 Jul 2016 21:28:55 +0700
Subject: [PATCH 5409/6219] release 2016.07.03.1

---
 .github/ISSUE_TEMPLATE.md |   6 +-
 CONTRIBUTING.md           | 152 ++++++++++++++++++++++++++++++++++++--
 youtube_dl/version.py     |   2 +-
 3 files changed, 150 insertions(+), 10 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 8dbd3cbfa..786f347f8 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.07.03*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.07.03**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.07.03.1*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.07.03.1**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.07.03
+[debug] youtube-dl version 2016.07.03.1
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index a59fac9b2..fbf0ab7e8 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -97,9 +97,17 @@ If you want to add support for a new site, first of all **make sure** this site
 After you have ensured this site is distributing it's content legally, you can follow this quick list (assuming your service is called `yourextractor`):
 
 1. [Fork this repository](https://github.com/rg3/youtube-dl/fork)
-2. Check out the source code with `git clone git@github.com:YOUR_GITHUB_USERNAME/youtube-dl.git`
-3. Start a new git branch with `cd youtube-dl; git checkout -b yourextractor`
+2. Check out the source code with:
+
+        git clone git@github.com:YOUR_GITHUB_USERNAME/youtube-dl.git
+
+3. Start a new git branch with
+
+        cd youtube-dl
+        git checkout -b yourextractor
+
 4. Start with this simple template and save it to `youtube_dl/extractor/yourextractor.py`:
+
     ```python
     # coding: utf-8
     from __future__ import unicode_literals
@@ -143,16 +151,148 @@ After you have ensured this site is distributing it's content legally, you can f
 5. Add an import in [`youtube_dl/extractor/extractors.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/extractors.py).
 6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc.
 7. Have a look at [`youtube_dl/extractor/common.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L74-L252). Add tests and code for as many as you want.
-8. Keep in mind that the only mandatory fields in info dict for successful extraction process are `id`, `title` and either `url` or `formats`, i.e. these are the critical data the extraction does not make any sense without. This means that [any field](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L148-L252) apart from aforementioned mandatory ones should be treated **as optional** and extraction should be **tolerate** to situations when sources for these fields can potentially be unavailable (even if they always available at the moment) and **future-proof** in order not to break the extraction of general purpose mandatory fields. For example, if you have some intermediate dict `meta` that is a source of metadata and it has a key `summary` that you want to extract and put into resulting info dict as `description`, you should be ready that this key may be missing from the `meta` dict, i.e. you should extract it as `meta.get('summary')` and not `meta['summary']`. Similarly, you should pass `fatal=False` when extracting data from a webpage with `_search_regex/_html_search_regex`.
-9. Check the code with [flake8](https://pypi.python.org/pypi/flake8). Also make sure your code works under all [Python](http://www.python.org/) versions claimed supported by youtube-dl, namely 2.6, 2.7, and 3.2+.
-10. When the tests pass, [add](http://git-scm.com/docs/git-add) the new files and [commit](http://git-scm.com/docs/git-commit) them and [push](http://git-scm.com/docs/git-push) the result, like this:
+8. Make sure your code follows [youtube-dl coding conventions](#youtube-dl-coding-conventions) and check the code with [flake8](https://pypi.python.org/pypi/flake8). Also make sure your code works under all [Python](http://www.python.org/) versions claimed supported by youtube-dl, namely 2.6, 2.7, and 3.2+.
+9. When the tests pass, [add](http://git-scm.com/docs/git-add) the new files and [commit](http://git-scm.com/docs/git-commit) them and [push](http://git-scm.com/docs/git-push) the result, like this:
 
         $ git add youtube_dl/extractor/extractors.py
         $ git add youtube_dl/extractor/yourextractor.py
         $ git commit -m '[yourextractor] Add new extractor'
         $ git push origin yourextractor
 
-11. Finally, [create a pull request](https://help.github.com/articles/creating-a-pull-request). We'll then review and merge it.
+10. Finally, [create a pull request](https://help.github.com/articles/creating-a-pull-request). We'll then review and merge it.
 
 In any case, thank you very much for your contributions!
 
+## youtube-dl coding conventions
+
+This section introduces a guide lines for writing idiomatic, robust and future-proof extractor code.
+
+Extractors are very fragile by nature since they depend on the layout of the source data provided by 3rd party media hoster out of your control and this layout tend to change. As an extractor implementer your task is not only to write code that will extract media links and metadata correctly but also to minimize code dependency on source's layout changes and even to make the code foresee potential future changes and be ready for that. This is important because it will allow extractor not to break on minor layout changes thus keeping old youtube-dl versions working. Even though this breakage issue is easily fixed by emitting a new version of youtube-dl with fix incorporated all the previous version become broken in all repositories and distros' packages that may not be so prompt in fetching the update from us. Needless to say some may never receive an update at all that is possible for non rolling release distros.
+
+### Mandatory and optional metafields
+
+For extraction to work youtube-dl relies on metadata your extractor extracts and provides to youtube-dl expressed by [information dictionary](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L75-L257) or simply *info dict*. Only the following meta fields in *info dict* are considered mandatory for successful extraction process by youtube-dl:
+
+ - `id` (media identifier)
+ - `title` (media title)
+ - `url` (media download URL) or `formats`
+
+In fact only the last option is technically mandatory (i.e. if you can't figure out the download location of the media the extraction does not make any sense). But by convention youtube-dl also treats `id` and `title` to be mandatory. Thus aforementioned metafields are the critical data the extraction does not make any sense without and if any of them fail to be extracted then extractor is considered completely broken.
+
+[Any field](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L149-L257) apart from the aforementioned ones are considered **optional**. That means that extraction should be **tolerate** to situations when sources for these fields can potentially be unavailable (even if they are always available at the moment) and **future-proof** in order not to break the extraction of general purpose mandatory fields.
+
+#### Example
+
+Say you have some source dictionary `meta` that you've fetched as JSON with HTTP request and it has a key `summary`:
+
+```python
+meta = self._download_json(url, video_id)
+```
+    
+Assume at this point `meta`'s layout is:
+
+```python
+{
+    ...
+    "summary": "some fancy summary text",
+    ...
+}
+```
+
+Assume you want to extract `summary` and put into resulting info dict as `description`. Since `description` is optional metafield you should be ready that this key may be missing from the `meta` dict, so that you should extract it like:
+
+```python
+description = meta.get('summary')  # correct
+```
+
+and not like:
+
+```python
+description = meta['summary']  # incorrect
+```
+
+The latter will break extraction process with `KeyError` if `summary` disappears from `meta` at some time later but with former approach extraction will just go ahead with `description` set to `None` that is perfectly fine (remember `None` is equivalent for absence of data). 
+
+Similarly, you should pass `fatal=False` when extracting optional data from a webpage with `_search_regex`, `_html_search_regex` or similar methods, for instance:
+
+```python
+description = self._search_regex(
+    r'<span[^>]+id="title"[^>]*>([^<]+)<',
+    webpage, 'description', fatal=False)
+```
+
+With `fatal` set to `False` if `_search_regex` fails to extract `description` it will emit a warning and continue extraction.
+
+You can also pass `default=<some fallback value>`, for example:
+
+```python
+description = self._search_regex(
+    r'<span[^>]+id="title"[^>]*>([^<]+)<',
+    webpage, 'description', default=None)
+```
+
+On failure this code will silently continue the extraction with `description` set to `None`. That is useful for metafields that are known to may or may not be present.
+ 
+### Provide fallbacks
+
+When extracting metadata try to provide several scenarios for that. For example if `title` is present in several places/sources try extracting from at least some of them. This would make it more future-proof in case some of the sources became unavailable.
+
+#### Example
+
+Say `meta` from previous example has a `title` and you are about to extract it. Since `title` is mandatory meta field you should end up with something like:
+
+```python
+title = meta['title']
+```
+
+If `title` disappeares from `meta` in future due to some changes on hoster's side the extraction would fail since `title` is mandatory. That's expected.
+
+Assume that you have some another source you can extract `title` from, for example `og:title` HTML meta of a `webpage`. In this case you can provide a fallback scenario:
+
+```python
+title = meta.get('title') or self._og_search_title(webpage)
+```
+
+This code will try to extract from `meta` first and if it fails it will try extracting `og:title` from a `webpage`.
+
+### Make regular expressions flexible
+
+When using regular expressions try to write them fuzzy and flexible.
+ 
+#### Example
+
+Say you need to extract `title` from the following HTML code:
+
+```html
+<span style="position: absolute; left: 910px; width: 90px; float: right; z-index: 9999;" class="title">some fancy title</span>
+```
+
+The code for that task should look similar to:
+
+```python
+title = self._search_regex(
+    r'<span[^>]+class="title"[^>]*>([^<]+)', webpage, 'title')
+```
+
+Or even better:
+
+```python
+title = self._search_regex(
+    r'<span[^>]+class=(["\'])title\1[^>]*>(?P<title>[^<]+)',
+    webpage, 'title', group='title')
+```
+
+Note how you tolerate potential changes in `style` attribute's value or switch from using double quotes to single for `class` attribute: 
+
+The code definitely should not look like:
+
+```python
+title = self._search_regex(
+    r'<span style="position: absolute; left: 910px; width: 90px; float: right; z-index: 9999;" class="title">(.*?)</span>',
+    webpage, 'title', group='title')
+```
+
+### Use safe conversion functions
+
+Wrap all extracted numeric data into safe functions from `utils`: `int_or_none`, `float_or_none`. Use them for string to number conversions as well.
+
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 1dda4c6d1..78d585edf 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.07.03'
+__version__ = '2016.07.03.1'

From bf3ae6a543ee8825308e8427f60fceb125a77ab5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 3 Jul 2016 22:20:14 +0700
Subject: [PATCH 5410/6219] [devscripts/show-downloads-statictics] Add script
 for displaying downloads statistics

---
 devscripts/show-downloads-statistics.py | 41 +++++++++++++++++++++++++
 1 file changed, 41 insertions(+)
 create mode 100644 devscripts/show-downloads-statistics.py

diff --git a/devscripts/show-downloads-statistics.py b/devscripts/show-downloads-statistics.py
new file mode 100644
index 000000000..b591d3fc9
--- /dev/null
+++ b/devscripts/show-downloads-statistics.py
@@ -0,0 +1,41 @@
+#!/usr/bin/env python
+from __future__ import unicode_literals
+
+import json
+import os
+import re
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from youtube_dl.compat import (
+    compat_print,
+    compat_urllib_request,
+)
+from youtube_dl.utils import format_bytes
+
+
+def format_size(bytes):
+    return '%s (%d bytes)' % (format_bytes(bytes), bytes)
+
+
+total_bytes = 0
+
+releases = json.loads(compat_urllib_request.urlopen(
+    'https://api.github.com/repos/rg3/youtube-dl/releases').read().decode('utf-8'))
+
+for release in releases:
+    compat_print(release['name'])
+    for asset in release['assets']:
+        asset_name = asset['name']
+        total_bytes += asset['download_count'] * asset['size']
+        if all(not re.match(p, asset_name) for p in (
+                r'^youtube-dl$',
+                r'^youtube-dl-\d{4}\.\d{2}\.\d{2}(?:\.\d+)?\.tar\.gz$',
+                r'^youtube-dl\.exe$')):
+            continue
+        compat_print(
+            ' %s size: %s downloads: %d'
+            % (asset_name, format_size(asset['size']), asset['download_count']))
+
+compat_print('total downloads traffic: %s' % format_size(total_bytes))

From 38cce791c71e9da33854e56114233d60345e18c0 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 3 Jul 2016 23:23:48 +0800
Subject: [PATCH 5411/6219] Rename --cn-verfication-proxy to
 --geo-verification-proxy

And deprecate the former one

Since commit f1388739002a7fd1e8e9c41b642734786fc6c391, this option is
not limited to China websites, so rename it.
---
 youtube_dl/YoutubeDL.py        |  9 +++++++--
 youtube_dl/__init__.py         |  2 ++
 youtube_dl/extractor/common.py |  7 +++++++
 youtube_dl/extractor/iqiyi.py  |  6 +-----
 youtube_dl/extractor/kuwo.py   |  7 +------
 youtube_dl/extractor/leeco.py  | 12 +++---------
 youtube_dl/extractor/rai.py    |  9 ++-------
 youtube_dl/extractor/sohu.py   | 16 ++++------------
 youtube_dl/extractor/youku.py  |  9 ++-------
 youtube_dl/options.py          |  9 +++++++--
 10 files changed, 36 insertions(+), 50 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 5036289b0..ba72ec6f3 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -196,8 +196,8 @@ class YoutubeDL(object):
     prefer_insecure:   Use HTTP instead of HTTPS to retrieve information.
                        At the moment, this is only supported by YouTube.
     proxy:             URL of the proxy server to use
-    cn_verification_proxy:  URL of the proxy to use for IP address verification
-                       on Chinese sites. (Experimental)
+    geo_verification_proxy:  URL of the proxy to use for IP address verification
+                       on geo-restricted sites. (Experimental)
     socket_timeout:    Time to wait for unresponsive hosts, in seconds
     bidi_workaround:   Work around buggy terminals without bidirectional text
                        support, using fridibi
@@ -304,6 +304,11 @@ class YoutubeDL(object):
         self.params.update(params)
         self.cache = Cache(self)
 
+        if self.params.get('cn_verification_proxy') is not None:
+            self.report_warning('--cn-verification-proxy is deprecated. Use --geo-verification-proxy instead.')
+            if self.params.get('geo_verification_proxy') is None:
+                self.params['geo_verification_proxy'] = self.params['cn_verification_proxy']
+
         if params.get('bidi_workaround', False):
             try:
                 import pty
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 4905674ad..2b34bf9c2 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -382,6 +382,8 @@ def _real_main(argv=None):
         'external_downloader_args': external_downloader_args,
         'postprocessor_args': postprocessor_args,
         'cn_verification_proxy': opts.cn_verification_proxy,
+        'geo_verification_proxy': opts.geo_verification_proxy,
+
     }
 
     with YoutubeDL(ydl_opts) as ydl:
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index e6c15de42..be2b6ff66 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1729,6 +1729,13 @@ class InfoExtractor(object):
     def _mark_watched(self, *args, **kwargs):
         raise NotImplementedError('This method must be implemented by subclasses')
 
+    def geo_verification_headers(self):
+        headers = {}
+        geo_verification_proxy = self._downloader.params.get('geo_verification_proxy')
+        if geo_verification_proxy:
+            headers['Ytdl-request-proxy'] = geo_verification_proxy
+        return headers
+
 
 class SearchInfoExtractor(InfoExtractor):
     """
diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index 754e6057c..ba037d48c 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -293,14 +293,10 @@ class IqiyiIE(InfoExtractor):
             't': tm,
         }
 
-        headers = {}
-        cn_verification_proxy = self._downloader.params.get('cn_verification_proxy')
-        if cn_verification_proxy:
-            headers['Ytdl-request-proxy'] = cn_verification_proxy
         return self._download_json(
             'http://cache.m.iqiyi.com/jp/tmts/%s/%s/' % (tvid, video_id),
             video_id, transform_source=lambda s: remove_start(s, 'var tvInfoJs='),
-            query=params, headers=headers)
+            query=params, headers=self.geo_verification_headers())
 
     def _extract_playlist(self, webpage):
         PAGE_SIZE = 50
diff --git a/youtube_dl/extractor/kuwo.py b/youtube_dl/extractor/kuwo.py
index 0221fb919..b1d460599 100644
--- a/youtube_dl/extractor/kuwo.py
+++ b/youtube_dl/extractor/kuwo.py
@@ -26,11 +26,6 @@ class KuwoBaseIE(InfoExtractor):
     def _get_formats(self, song_id, tolerate_ip_deny=False):
         formats = []
         for file_format in self._FORMATS:
-            headers = {}
-            cn_verification_proxy = self._downloader.params.get('cn_verification_proxy')
-            if cn_verification_proxy:
-                headers['Ytdl-request-proxy'] = cn_verification_proxy
-
             query = {
                 'format': file_format['ext'],
                 'br': file_format.get('br', ''),
@@ -42,7 +37,7 @@ class KuwoBaseIE(InfoExtractor):
             song_url = self._download_webpage(
                 'http://antiserver.kuwo.cn/anti.s',
                 song_id, note='Download %s url info' % file_format['format'],
-                query=query, headers=headers,
+                query=query, headers=self.geo_verification_headers(),
             )
 
             if song_url == 'IPDeny' and not tolerate_ip_deny:
diff --git a/youtube_dl/extractor/leeco.py b/youtube_dl/extractor/leeco.py
index 959d71617..171b705c7 100644
--- a/youtube_dl/extractor/leeco.py
+++ b/youtube_dl/extractor/leeco.py
@@ -20,7 +20,6 @@ from ..utils import (
     int_or_none,
     orderedSet,
     parse_iso8601,
-    sanitized_Request,
     str_or_none,
     url_basename,
     urshift,
@@ -121,16 +120,11 @@ class LeIE(InfoExtractor):
             'tkey': self.calc_time_key(int(time.time())),
             'domain': 'www.le.com'
         }
-        play_json_req = sanitized_Request(
-            'http://api.le.com/mms/out/video/playJson?' + compat_urllib_parse_urlencode(params)
-        )
-        cn_verification_proxy = self._downloader.params.get('cn_verification_proxy')
-        if cn_verification_proxy:
-            play_json_req.add_header('Ytdl-request-proxy', cn_verification_proxy)
 
         play_json = self._download_json(
-            play_json_req,
-            media_id, 'Downloading playJson data')
+            'http://api.le.com/mms/out/video/playJson',
+            media_id, 'Downloading playJson data', query=params,
+            headers=self.geo_verification_headers())
 
         # Check for errors
         playstatus = play_json['playstatus']
diff --git a/youtube_dl/extractor/rai.py b/youtube_dl/extractor/rai.py
index 1f0fcd609..dc640b1bc 100644
--- a/youtube_dl/extractor/rai.py
+++ b/youtube_dl/extractor/rai.py
@@ -20,17 +20,12 @@ class RaiBaseIE(InfoExtractor):
         formats = []
 
         for platform in ('mon', 'flash', 'native'):
-            headers = {}
-            # TODO: rename --cn-verification-proxy
-            cn_verification_proxy = self._downloader.params.get('cn_verification_proxy')
-            if cn_verification_proxy:
-                headers['Ytdl-request-proxy'] = cn_verification_proxy
-
             relinker = self._download_xml(
                 relinker_url, video_id,
                 note='Downloading XML metadata for platform %s' % platform,
                 transform_source=fix_xml_ampersands,
-                query={'output': 45, 'pl': platform}, headers=headers)
+                query={'output': 45, 'pl': platform},
+                headers=self.geo_verification_headers())
 
             media_url = find_xpath_attr(relinker, './url', 'type', 'content').text
             if media_url == 'http://download.rai.it/video_no_available.mp4':
diff --git a/youtube_dl/extractor/sohu.py b/youtube_dl/extractor/sohu.py
index 49e5d09ae..72fe66142 100644
--- a/youtube_dl/extractor/sohu.py
+++ b/youtube_dl/extractor/sohu.py
@@ -8,10 +8,7 @@ from ..compat import (
     compat_str,
     compat_urllib_parse_urlencode,
 )
-from ..utils import (
-    ExtractorError,
-    sanitized_Request,
-)
+from ..utils import ExtractorError
 
 
 class SohuIE(InfoExtractor):
@@ -96,15 +93,10 @@ class SohuIE(InfoExtractor):
             else:
                 base_data_url = 'http://hot.vrs.sohu.com/vrs_flash.action?vid='
 
-            req = sanitized_Request(base_data_url + vid_id)
-
-            cn_verification_proxy = self._downloader.params.get('cn_verification_proxy')
-            if cn_verification_proxy:
-                req.add_header('Ytdl-request-proxy', cn_verification_proxy)
-
             return self._download_json(
-                req, video_id,
-                'Downloading JSON data for %s' % vid_id)
+                base_data_url + vid_id, video_id,
+                'Downloading JSON data for %s' % vid_id,
+                headers=self.geo_verification_headers())
 
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index 147608ebe..e37f237c7 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -16,7 +16,6 @@ from ..compat import (
 from ..utils import (
     ExtractorError,
     get_element_by_attribute,
-    sanitized_Request,
 )
 
 
@@ -218,14 +217,10 @@ class YoukuIE(InfoExtractor):
             headers = {
                 'Referer': req_url,
             }
+            headers.update(self.geo_verification_headers())
             self._set_cookie('youku.com', 'xreferrer', 'http://www.youku.com')
-            req = sanitized_Request(req_url, headers=headers)
 
-            cn_verification_proxy = self._downloader.params.get('cn_verification_proxy')
-            if cn_verification_proxy:
-                req.add_header('Ytdl-request-proxy', cn_verification_proxy)
-
-            raw_data = self._download_json(req, video_id, note=note)
+            raw_data = self._download_json(req_url, video_id, note=note, headers=headers)
 
             return raw_data['data']
 
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index c9033e3cb..f2e3ad7a1 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -209,11 +209,16 @@ def parseOpts(overrideArguments=None):
         action='store_const', const='::', dest='source_address',
         help='Make all connections via IPv6 (experimental)',
     )
+    network.add_option(
+        '--geo-verification-proxy',
+        dest='geo_verification_proxy', default=None, metavar='URL',
+        help='Use this proxy to verify the IP address for some geo-restricted sites. '
+        'The default proxy specified by --proxy (or none, if the options is not present) is used for the actual downloading. (experimental)'
+    )
     network.add_option(
         '--cn-verification-proxy',
         dest='cn_verification_proxy', default=None, metavar='URL',
-        help='Use this proxy to verify the IP address for some Chinese sites. '
-        'The default proxy specified by --proxy (or none, if the options is not present) is used for the actual downloading. (experimental)'
+        help=optparse.SUPPRESS_HELP,
     )
 
     selection = optparse.OptionGroup(parser, 'Video Selection')

From dd801bbe18535c98172df929eedd3e8dd8100409 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 3 Jul 2016 16:33:50 +0100
Subject: [PATCH 5412/6219] [brightcove] improve error detection

---
 youtube_dl/extractor/brightcove.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index ef560b592..b49b1977d 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -585,6 +585,13 @@ class BrightcoveNewIE(InfoExtractor):
                         'format_id': build_format_id('rtmp'),
                     })
                 formats.append(f)
+
+        errors = json_data.get('errors')
+        if not formats and errors:
+            error = errors[0]
+            raise ExtractorError(
+                error.get('message') or error.get('error_subcode') or error['error_code'], expected=True)
+
         self._sort_formats(formats)
 
         subtitles = {}

From 40f3666f6b2431d8756a36d574139c8885b15c53 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 3 Jul 2016 23:50:55 +0800
Subject: [PATCH 5413/6219] [test/test_http] Update tests for
 38cce791c71e9da33854e56114233d60345e18c0

---
 test/test_http.py | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/test/test_http.py b/test/test_http.py
index 5076ced51..fdc68ccb4 100644
--- a/test/test_http.py
+++ b/test/test_http.py
@@ -138,27 +138,27 @@ class TestProxy(unittest.TestCase):
         self.proxy_thread.daemon = True
         self.proxy_thread.start()
 
-        self.cn_proxy = compat_http_server.HTTPServer(
-            ('localhost', 0), _build_proxy_handler('cn'))
-        self.cn_port = http_server_port(self.cn_proxy)
-        self.cn_proxy_thread = threading.Thread(target=self.cn_proxy.serve_forever)
-        self.cn_proxy_thread.daemon = True
-        self.cn_proxy_thread.start()
+        self.geo_proxy = compat_http_server.HTTPServer(
+            ('localhost', 0), _build_proxy_handler('geo'))
+        self.geo_port = http_server_port(self.geo_proxy)
+        self.geo_proxy_thread = threading.Thread(target=self.geo_proxy.serve_forever)
+        self.geo_proxy_thread.daemon = True
+        self.geo_proxy_thread.start()
 
     def test_proxy(self):
-        cn_proxy = 'localhost:{0}'.format(self.cn_port)
+        geo_proxy = 'localhost:{0}'.format(self.geo_port)
         ydl = YoutubeDL({
             'proxy': 'localhost:{0}'.format(self.port),
-            'cn_verification_proxy': cn_proxy,
+            'geo_verification_proxy': geo_proxy,
         })
         url = 'http://foo.com/bar'
         response = ydl.urlopen(url).read().decode('utf-8')
         self.assertEqual(response, 'normal: {0}'.format(url))
 
         req = compat_urllib_request.Request(url)
-        req.add_header('Ytdl-request-proxy', cn_proxy)
+        req.add_header('Ytdl-request-proxy', geo_proxy)
         response = ydl.urlopen(req).read().decode('utf-8')
-        self.assertEqual(response, 'cn: {0}'.format(url))
+        self.assertEqual(response, 'geo: {0}'.format(url))
 
     def test_proxy_with_idn(self):
         ydl = YoutubeDL({

From 6a4e659c9316147b0693ee6d2a447a8975830321 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 3 Jul 2016 23:00:36 +0700
Subject: [PATCH 5414/6219] [yahoo] Recognize brightcove embed (Closes #9995)

---
 youtube_dl/extractor/yahoo.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/yahoo.py b/youtube_dl/extractor/yahoo.py
index 927a964a4..b0679dfb7 100644
--- a/youtube_dl/extractor/yahoo.py
+++ b/youtube_dl/extractor/yahoo.py
@@ -19,6 +19,7 @@ from ..utils import (
     mimetype2ext,
 )
 
+from .brightcove import BrightcoveNewIE
 from .nbc import NBCSportsVPlayerIE
 
 
@@ -227,7 +228,12 @@ class YahooIE(InfoExtractor):
         # Look for NBCSports iframes
         nbc_sports_url = NBCSportsVPlayerIE._extract_url(webpage)
         if nbc_sports_url:
-            return self.url_result(nbc_sports_url, 'NBCSportsVPlayer')
+            return self.url_result(nbc_sports_url, NBCSportsVPlayerIE.ie_key())
+
+        # Look for Brightcove New Studio embeds
+        bc_url = BrightcoveNewIE._extract_url(webpage)
+        if bc_url:
+            return self.url_result(bc_url, BrightcoveNewIE.ie_key())
 
         # Query result is often embedded in webpage as JSON. Sometimes explicit requests
         # to video API results in a failure with geo restriction reason therefore using

From aa7a455b2eb5a98190ac4695954787ded3fb5a0b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 4 Jul 2016 01:24:33 +0700
Subject: [PATCH 5415/6219] [README.md] Clarify configuration file may not
 exist by default

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index cd9e0b43f..41d0b79bc 100644
--- a/README.md
+++ b/README.md
@@ -424,7 +424,7 @@ which means you can modify it, redistribute it or use it however you like.
 
 # CONFIGURATION
 
-You can configure youtube-dl by placing any supported command line option to a configuration file. On Linux and OS X, the system wide configuration file is located at `/etc/youtube-dl.conf` and the user wide configuration file at `~/.config/youtube-dl/config`. On Windows, the user wide configuration file locations are `%APPDATA%\youtube-dl\config.txt` or `C:\Users\<user name>\youtube-dl.conf`.
+You can configure youtube-dl by placing any supported command line option to a configuration file. On Linux and OS X, the system wide configuration file is located at `/etc/youtube-dl.conf` and the user wide configuration file at `~/.config/youtube-dl/config`. On Windows, the user wide configuration file locations are `%APPDATA%\youtube-dl\config.txt` or `C:\Users\<user name>\youtube-dl.conf`. Note that by default configuration file may not exist so you may need to create it yourself.
 
 For example, with the following configuration file youtube-dl will always extract the audio, not copy the mtime, use a proxy and save all videos under `Movies` directory in your home directory:
 ```

From 195f0845421b005a08f12634b56ea31089fd3808 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 4 Jul 2016 03:27:00 +0700
Subject: [PATCH 5416/6219] [pornhub] Detect private videos (Closes #9987)

---
 youtube_dl/extractor/pornhub.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index c76afe1c4..d2c92531b 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -82,6 +82,10 @@ class PornHubIE(InfoExtractor):
         # removed by uploader
         'url': 'http://www.pornhub.com/view_video.php?viewkey=ph572716d15a111',
         'only_matching': True,
+    }, {
+        # private video
+        'url': 'http://www.pornhub.com/view_video.php?viewkey=ph56fd731fce6b7',
+        'only_matching': True,
     }, {
         'url': 'https://www.thumbzilla.com/video/ph56c6114abd99a/horny-girlfriend-sex',
         'only_matching': True,
@@ -107,7 +111,7 @@ class PornHubIE(InfoExtractor):
         webpage = self._download_webpage(req, video_id)
 
         error_msg = self._html_search_regex(
-            r'(?s)<div[^>]+class=(["\']).*?\bremoved\b.*?\1[^>]*>(?P<error>.+?)</div>',
+            r'(?s)<div[^>]+class=(["\']).*?\b(?:removed|userMessageSection)\b.*?\1[^>]*>(?P<error>.+?)</div>',
             webpage, 'error message', default=None, group='error')
         if error_msg:
             error_msg = re.sub(r'\s+', ' ', error_msg)

From 3a212ed62e45cd69e1bcd57402c5a6a6f7e8908b Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 4 Jul 2016 11:25:46 +0800
Subject: [PATCH 5417/6219] [iqiyi] Skip an unstable MD5 checksum

---
 youtube_dl/extractor/iqiyi.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index ba037d48c..01c7b3042 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -165,7 +165,7 @@ class IqiyiIE(InfoExtractor):
 
     _TESTS = [{
         'url': 'http://www.iqiyi.com/v_19rrojlavg.html',
-        'md5': '5b0591f55961117155430b5d544fdb01',
+        # MD5 checksum differs on my machine and Travis CI
         'info_dict': {
             'id': '9c1fb1b99d192b21c559e5a1a2cb3c73',
             'ext': 'mp4',

From 81953d1ae53bc5b3344243480316d751004a4d40 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 4 Jul 2016 17:57:44 +0100
Subject: [PATCH 5418/6219] [kaltura] add support videos stored on custom
 kaltura servers(closes #5557)

---
 youtube_dl/extractor/generic.py | 15 +++++++++++++
 youtube_dl/extractor/kaltura.py | 39 ++++++++++++++++++++++++---------
 youtube_dl/utils.py             |  2 ++
 3 files changed, 46 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index a9b61bf13..764697bd2 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1295,6 +1295,21 @@ class GenericIE(InfoExtractor):
                 'uploader': 'cylus cyrus',
             },
         },
+        {
+            # video stored on custom kaltura server
+            'url': 'http://www.expansion.com/multimedia/videos.html?media=EQcM30NHIPv',
+            'md5': '537617d06e64dfed891fa1593c4b30cc',
+            'info_dict': {
+                'id': '0_1iotm5bh',
+                'ext': 'mp4',
+                'title': 'Elecciones británicas: 5 lecciones para Rajoy',
+                'description': 'md5:435a89d68b9760b92ce67ed227055f16',
+                'uploader_id': 'videos.expansion@el-mundo.net',
+                'upload_date': '20150429',
+                'timestamp': 1430303472,
+            },
+            'add_ie': ['Kaltura'],
+        },
     ]
 
     def report_following_redirect(self, new_url):
diff --git a/youtube_dl/extractor/kaltura.py b/youtube_dl/extractor/kaltura.py
index c75a958ba..147bb8cf0 100644
--- a/youtube_dl/extractor/kaltura.py
+++ b/youtube_dl/extractor/kaltura.py
@@ -6,7 +6,6 @@ import base64
 
 from .common import InfoExtractor
 from ..compat import (
-    compat_urllib_parse_urlencode,
     compat_urlparse,
     compat_parse_qs,
 )
@@ -15,6 +14,7 @@ from ..utils import (
     ExtractorError,
     int_or_none,
     unsmuggle_url,
+    smuggle_url,
 )
 
 
@@ -34,7 +34,8 @@ class KalturaIE(InfoExtractor):
                         )(?:/(?P<path>[^?]+))?(?:\?(?P<query>.*))?
                 )
                 '''
-    _API_BASE = 'http://cdnapi.kaltura.com/api_v3/index.php?'
+    _SERVICE_URL = 'http://cdnapi.kaltura.com'
+    _SERVICE_BASE = '/api_v3/index.php'
     _TESTS = [
         {
             'url': 'kaltura:269692:1_1jc2y3e4',
@@ -88,7 +89,14 @@ class KalturaIE(InfoExtractor):
                     (?P<q3>["\'])(?P<id>.+?)(?P=q3)
                 ''', webpage))
         if mobj:
-            return 'kaltura:%(partner_id)s:%(id)s' % mobj.groupdict()
+            embed_info = mobj.groupdict()
+            url = 'kaltura:%(partner_id)s:%(id)s' % embed_info
+            service_url = re.search(
+                '<script[^>]+src=(?:["\'])((?:https?:)?//.+?)/p/%(partner_id)s/sp/%(partner_id)s00/embedIframeJs' % embed_info,
+                webpage)
+            if service_url:
+                url = smuggle_url(url, {'service_url': service_url.group(1)})
+            return url
 
     def _kaltura_api_call(self, video_id, actions, *args, **kwargs):
         params = actions[0]
@@ -97,9 +105,9 @@ class KalturaIE(InfoExtractor):
                 for k, v in a.items():
                     params['%d:%s' % (i, k)] = v
 
-        query = compat_urllib_parse_urlencode(params)
-        url = self._API_BASE + query
-        data = self._download_json(url, video_id, *args, **kwargs)
+        data = self._download_json(
+            self._SERVICE_URL + self._SERVICE_BASE,
+            video_id, query=params, *args, **kwargs)
 
         status = data if len(actions) == 1 else data[0]
         if status.get('objectType') == 'KalturaAPIException':
@@ -148,6 +156,9 @@ class KalturaIE(InfoExtractor):
 
     def _real_extract(self, url):
         url, smuggled_data = unsmuggle_url(url, {})
+        service_url = smuggled_data.get('service_url')
+        if service_url:
+            self._SERVICE_URL = service_url
 
         mobj = re.match(self._VALID_URL, url)
         partner_id, entry_id = mobj.group('partner_id', 'id')
@@ -201,12 +212,17 @@ class KalturaIE(InfoExtractor):
                 unsigned_url += '?referrer=%s' % referrer
             return unsigned_url
 
+        data_url = info['dataUrl']
+        if '/flvclipper/' in data_url:
+            data_url = re.sub(r'/flvclipper/.*', '/serveFlavor', data_url)
+
         formats = []
         for f in flavor_assets:
             # Continue if asset is not ready
             if f['status'] != 2:
                 continue
-            video_url = sign_url('%s/flavorId/%s' % (info['dataUrl'], f['id']))
+            video_url = sign_url(
+                '%s/flavorId/%s' % (data_url, f['id']))
             formats.append({
                 'format_id': '%(fileExt)s-%(bitrate)s' % f,
                 'ext': f.get('fileExt'),
@@ -219,9 +235,12 @@ class KalturaIE(InfoExtractor):
                 'width': int_or_none(f.get('width')),
                 'url': video_url,
             })
-        m3u8_url = sign_url(info['dataUrl'].replace('format/url', 'format/applehttp'))
-        formats.extend(self._extract_m3u8_formats(
-            m3u8_url, entry_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
+        if '/playManifest/' in data_url:
+            m3u8_url = sign_url(data_url.replace(
+                'format/url', 'format/applehttp'))
+            formats.extend(self._extract_m3u8_formats(
+                m3u8_url, entry_id, 'mp4', 'm3u8_native',
+                m3u8_id='hls', fatal=False))
 
         self._check_formats(formats, entry_id)
         self._sort_formats(formats)
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 495878a0e..d302f39e4 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1444,6 +1444,8 @@ def shell_quote(args):
 def smuggle_url(url, data):
     """ Pass additional data in a URL for internal use. """
 
+    url, idata = unsmuggle_url(url, {})
+    data.update(idata)
     sdata = compat_urllib_parse_urlencode(
         {'__youtubedl_smuggle': json.dumps(data)})
     return url + '#' + sdata

From dafafe7cf15918df97a8eca7d436f957f8d6ad80 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 4 Jul 2016 17:59:03 +0100
Subject: [PATCH 5419/6219] [la7] extract more info from a kaltura custom
 server

---
 youtube_dl/extractor/la7.py | 29 ++++++++++-------------------
 1 file changed, 10 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/extractor/la7.py b/youtube_dl/extractor/la7.py
index 826dc9476..da5a5de4a 100644
--- a/youtube_dl/extractor/la7.py
+++ b/youtube_dl/extractor/la7.py
@@ -3,8 +3,8 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..utils import (
-    determine_ext,
     js_to_json,
+    smuggle_url,
 )
 
 
@@ -18,13 +18,16 @@ class LA7IE(InfoExtractor):
     _TESTS = [{
         # 'src' is a plain URL
         'url': 'http://www.la7.it/crozza/video/inccool8-02-10-2015-163722',
-        'md5': '6054674766e7988d3e02f2148ff92180',
+        'md5': '8b613ffc0c4bf9b9e377169fc19c214c',
         'info_dict': {
             'id': 'inccool8-02-10-2015-163722',
             'ext': 'mp4',
             'title': 'Inc.Cool8',
             'description': 'Benvenuti nell\'incredibile mondo della INC. COOL. 8. dove “INC.” sta per “Incorporated” “COOL” sta per “fashion” ed Eight sta per il gesto  atletico',
             'thumbnail': 're:^https?://.*',
+            'uploader_id': 'kdla7pillole@iltrovatore.it',
+            'timestamp': 1443814869,
+            'upload_date': '20151002',
         },
     }, {
         # 'src' is a dictionary
@@ -49,26 +52,14 @@ class LA7IE(InfoExtractor):
             self._search_regex(r'videoLa7\(({[^;]+})\);', webpage, 'player data'),
             video_id, transform_source=js_to_json)
 
-        source = player_data['src']
-        source_urls = source.values() if isinstance(source, dict) else [source]
-
-        formats = []
-        for source_url in source_urls:
-            ext = determine_ext(source_url)
-            if ext == 'm3u8':
-                formats.extend(self._extract_m3u8_formats(
-                    source_url, video_id, ext='mp4',
-                    entry_protocol='m3u8_native', m3u8_id='hls'))
-            else:
-                formats.append({
-                    'url': source_url,
-                })
-        self._sort_formats(formats)
-
         return {
+            '_type': 'url_transparent',
+            'url': smuggle_url('kaltura:103:%s' % player_data['vid'], {
+                'service_url': 'http://kdam.iltrovatore.it',
+            }),
             'id': video_id,
             'title': player_data['title'],
             'description': self._og_search_description(webpage, default=None),
             'thumbnail': player_data.get('poster'),
-            'formats': formats,
+            'ie_key': 'Kaltura',
         }

From d9163ae3b6626fa08ae7eee297e97316813ae22c Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 4 Jul 2016 21:34:27 +0100
Subject: [PATCH 5420/6219] [kaltura] fix extraction error for videos from
 multiple kaltura servers

---
 youtube_dl/extractor/kaltura.py | 22 ++++++++++------------
 1 file changed, 10 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/kaltura.py b/youtube_dl/extractor/kaltura.py
index 147bb8cf0..1729f5bfb 100644
--- a/youtube_dl/extractor/kaltura.py
+++ b/youtube_dl/extractor/kaltura.py
@@ -91,14 +91,15 @@ class KalturaIE(InfoExtractor):
         if mobj:
             embed_info = mobj.groupdict()
             url = 'kaltura:%(partner_id)s:%(id)s' % embed_info
+            escaped_pid = re.escape(embed_info['partner_id'])
             service_url = re.search(
-                '<script[^>]+src=(?:["\'])((?:https?:)?//.+?)/p/%(partner_id)s/sp/%(partner_id)s00/embedIframeJs' % embed_info,
+                r'<script[^>]+src=["\']((?:https?:)?//.+?)/p/%s/sp/%s00/embedIframeJs' % (escaped_pid, escaped_pid),
                 webpage)
             if service_url:
                 url = smuggle_url(url, {'service_url': service_url.group(1)})
             return url
 
-    def _kaltura_api_call(self, video_id, actions, *args, **kwargs):
+    def _kaltura_api_call(self, video_id, actions, service_url=None, *args, **kwargs):
         params = actions[0]
         if len(actions) > 1:
             for i, a in enumerate(actions[1:], start=1):
@@ -106,7 +107,7 @@ class KalturaIE(InfoExtractor):
                     params['%d:%s' % (i, k)] = v
 
         data = self._download_json(
-            self._SERVICE_URL + self._SERVICE_BASE,
+            (service_url or self._SERVICE_URL) + self._SERVICE_BASE,
             video_id, query=params, *args, **kwargs)
 
         status = data if len(actions) == 1 else data[0]
@@ -116,7 +117,7 @@ class KalturaIE(InfoExtractor):
 
         return data
 
-    def _get_kaltura_signature(self, video_id, partner_id):
+    def _get_kaltura_signature(self, video_id, partner_id, service_url=None):
         actions = [{
             'apiVersion': '3.1',
             'expiry': 86400,
@@ -126,10 +127,10 @@ class KalturaIE(InfoExtractor):
             'widgetId': '_%s' % partner_id,
         }]
         return self._kaltura_api_call(
-            video_id, actions, note='Downloading Kaltura signature')['ks']
+            video_id, actions, service_url, note='Downloading Kaltura signature')['ks']
 
-    def _get_video_info(self, video_id, partner_id):
-        signature = self._get_kaltura_signature(video_id, partner_id)
+    def _get_video_info(self, video_id, partner_id, service_url=None):
+        signature = self._get_kaltura_signature(video_id, partner_id, service_url)
         actions = [
             {
                 'action': 'null',
@@ -152,19 +153,16 @@ class KalturaIE(InfoExtractor):
             },
         ]
         return self._kaltura_api_call(
-            video_id, actions, note='Downloading video info JSON')
+            video_id, actions, service_url, note='Downloading video info JSON')
 
     def _real_extract(self, url):
         url, smuggled_data = unsmuggle_url(url, {})
-        service_url = smuggled_data.get('service_url')
-        if service_url:
-            self._SERVICE_URL = service_url
 
         mobj = re.match(self._VALID_URL, url)
         partner_id, entry_id = mobj.group('partner_id', 'id')
         ks = None
         if partner_id and entry_id:
-            info, flavor_assets = self._get_video_info(entry_id, partner_id)
+            info, flavor_assets = self._get_video_info(entry_id, partner_id, smuggled_data.get('service_url'))
         else:
             path, query = mobj.group('path', 'query')
             if not path and not query:

From dfaa86b75e6fd7596c52f85f96bf38de9208c1d2 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 4 Jul 2016 21:36:32 +0100
Subject: [PATCH 5421/6219] [test_utils] add test for smuggling a smuggled url

---
 test/test_utils.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/test/test_utils.py b/test/test_utils.py
index ed61e4c27..85928dbc2 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -405,6 +405,12 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(res_url, url)
         self.assertEqual(res_data, None)
 
+        smug_url = smuggle_url(url, {'a': 'b'})
+        smug_smug_url = smuggle_url(smug_url, {'c': 'd'})
+        res_url, res_data = unsmuggle_url(smug_smug_url)
+        self.assertEqual(res_url, url)
+        self.assertEqual(res_data, {'a': 'b', 'c': 'd'})
+
     def test_shell_quote(self):
         args = ['ffmpeg', '-i', encodeFilename('ñ€ß\'.mp4')]
         self.assertEqual(shell_quote(args), """ffmpeg -i 'ñ€ß'"'"'.mp4'""")

From c25f1a9b63cd75beb691c83e4167fb9aceebf768 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 5 Jul 2016 06:32:46 +0700
Subject: [PATCH 5422/6219] release 2016.07.05

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 README.md                 | 6 +++---
 youtube_dl/version.py     | 2 +-
 3 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 786f347f8..d30650b7c 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.07.03.1*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.07.03.1**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.07.05*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.07.05**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.07.03.1
+[debug] youtube-dl version 2016.07.05
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/README.md b/README.md
index 41d0b79bc..bc214562d 100644
--- a/README.md
+++ b/README.md
@@ -103,9 +103,9 @@ which means you can modify it, redistribute it or use it however you like.
                                      (experimental)
     -6, --force-ipv6                 Make all connections via IPv6
                                      (experimental)
-    --cn-verification-proxy URL      Use this proxy to verify the IP address for
-                                     some Chinese sites. The default proxy
-                                     specified by --proxy (or none, if the
+    --geo-verification-proxy URL     Use this proxy to verify the IP address for
+                                     some geo-restricted sites. The default
+                                     proxy specified by --proxy (or none, if the
                                      options is not present) is used for the
                                      actual downloading. (experimental)
 
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 78d585edf..cd106dd7d 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.07.03.1'
+__version__ = '2016.07.05'

From 4080530624eda994d535e1a01c38ddd6d9aa3805 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 5 Jul 2016 13:07:09 +0800
Subject: [PATCH 5423/6219] [youtube:shared] Recognize the new 'shared' URLs

Closes #10007
---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/youtube.py    | 33 ++++++++++++++++++++++++++++++
 2 files changed, 34 insertions(+)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 5dab055db..d21d7870a 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1066,6 +1066,7 @@ from .youtube import (
     YoutubeSearchDateIE,
     YoutubeSearchIE,
     YoutubeSearchURLIE,
+    YoutubeSharedVideoIE,
     YoutubeShowIE,
     YoutubeSubscriptionsIE,
     YoutubeTruncatedIDIE,
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index c8d54f22a..554debd43 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1730,6 +1730,39 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         }
 
 
+class YoutubeSharedVideoIE(InfoExtractor):
+    _VALID_URL = r'(?:https?:)?//(?:www\.)?youtube\.com/shared\?ci=(?P<id>[0-9A-Za-z_-]{11})'
+    IE_NAME = 'youtube:shared'
+
+    _TEST = {
+        'url': 'https://www.youtube.com/shared?ci=1nEzmT-M4fU',
+        'info_dict': {
+            'id': 'uPDB5I9wfp8',
+            'ext': 'webm',
+            'title': 'Pocoyo: 90 minutos de episódios completos Português para crianças - PARTE 3',
+            'description': 'md5:d9e4d9346a2dfff4c7dc4c8cec0f546d',
+            'upload_date': '20160219',
+            'uploader': 'Pocoyo - Português (BR)',
+            'uploader_id': 'PocoyoBrazil',
+        },
+        'add_ie': ['Youtube'],
+        'params': {
+            # There are already too many Youtube downloads
+            'skip_download': True,
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        real_video_id = self._html_search_meta(
+            'videoId', webpage, 'YouTube video id', fatal=True)
+
+        return self.url_result(real_video_id, YoutubeIE.ie_key())
+
+
 class YoutubePlaylistIE(YoutubePlaylistBaseInfoExtractor):
     IE_DESC = 'YouTube.com playlists'
     _VALID_URL = r"""(?x)(?:

From c6054e32017b37b88600772766e0be7947586dd9 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 5 Jul 2016 14:26:42 +0800
Subject: [PATCH 5424/6219] [xuite] Support videos with already encoded media
 id

---
 youtube_dl/extractor/xuite.py | 30 ++++++++++++++++++++++++------
 1 file changed, 24 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/xuite.py b/youtube_dl/extractor/xuite.py
index 0be8932ad..a66daee46 100644
--- a/youtube_dl/extractor/xuite.py
+++ b/youtube_dl/extractor/xuite.py
@@ -67,6 +67,20 @@ class XuiteIE(InfoExtractor):
             'categories': ['電玩動漫'],
         },
         'skip': 'Video removed',
+    }, {
+        # Video with encoded media id
+        # from http://forgetfulbc.blogspot.com/2016/06/date.html
+        'url': 'http://vlog.xuite.net/embed/cE1xbENoLTI3NDQ3MzM2LmZsdg==?ar=0&as=0',
+        'info_dict': {
+            'id': 'cE1xbENoLTI3NDQ3MzM2LmZsdg==',
+            'ext': 'mp4',
+            'title': '男女平權只是口號？專家解釋約會時男生是否該幫女生付錢 (中字)',
+            'description': 'md5:f0abdcb69df300f522a5442ef3146f2a',
+            'timestamp': 1466160960,
+            'upload_date': '20160617',
+            'uploader': 'B.C. & Lowy',
+            'uploader_id': '232279340',
+        },
     }, {
         'url': 'http://vlog.xuite.net/play/S1dDUjdyLTMyOTc3NjcuZmx2/%E5%AD%AB%E7%87%95%E5%A7%BF-%E7%9C%BC%E6%B7%9A%E6%88%90%E8%A9%A9',
         'only_matching': True,
@@ -80,10 +94,9 @@ class XuiteIE(InfoExtractor):
     def base64_encode_utf8(data):
         return base64.b64encode(data.encode('utf-8')).decode('utf-8')
 
-    def _extract_flv_config(self, media_id):
-        base64_media_id = self.base64_encode_utf8(media_id)
+    def _extract_flv_config(self, encoded_media_id):
         flv_config = self._download_xml(
-            'http://vlog.xuite.net/flash/player?media=%s' % base64_media_id,
+            'http://vlog.xuite.net/flash/player?media=%s' % encoded_media_id,
             'flv config')
         prop_dict = {}
         for prop in flv_config.findall('./property'):
@@ -108,9 +121,14 @@ class XuiteIE(InfoExtractor):
                 '%s returned error: %s' % (self.IE_NAME, error_msg),
                 expected=True)
 
-        video_id = self._html_search_regex(
-            r'data-mediaid="(\d+)"', webpage, 'media id')
-        flv_config = self._extract_flv_config(video_id)
+        encoded_media_id = self._search_regex(
+            r'attributes\.name\s*=\s*"([^"]+)"', webpage,
+            'encoded media id', default=None)
+        if encoded_media_id is None:
+            video_id = self._html_search_regex(
+                r'data-mediaid="(\d+)"', webpage, 'media id')
+            encoded_media_id = self.base64_encode_utf8(video_id)
+        flv_config = self._extract_flv_config(encoded_media_id)
 
         FORMATS = {
             'audio': 'mp3',

From 5abf513cf87049b63369e45dd59818136080a68e Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 5 Jul 2016 10:44:16 +0100
Subject: [PATCH 5425/6219] [stitcher] fix episode config extraction

---
 youtube_dl/extractor/stitcher.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/stitcher.py b/youtube_dl/extractor/stitcher.py
index d5c852f52..0f8782d03 100644
--- a/youtube_dl/extractor/stitcher.py
+++ b/youtube_dl/extractor/stitcher.py
@@ -56,7 +56,7 @@ class StitcherIE(InfoExtractor):
 
         episode = self._parse_json(
             js_to_json(self._search_regex(
-                r'(?s)var\s+stitcher\s*=\s*({.+?});\n', webpage, 'episode config')),
+                r'(?s)var\s+stitcher(?:Config)?\s*=\s*({.+?});\n', webpage, 'episode config')),
             display_id)['config']['episode']
 
         title = unescapeHTML(episode['title'])

From 252a1f75d205b2c95c6a04c7aa64f08c756e4954 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 5 Jul 2016 11:46:25 +0100
Subject: [PATCH 5426/6219] [spiegel] improve info extraction

---
 youtube_dl/extractor/spiegel.py | 21 +++++++++++++++------
 1 file changed, 15 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/spiegel.py b/youtube_dl/extractor/spiegel.py
index 39a7aaf9d..c67d556e7 100644
--- a/youtube_dl/extractor/spiegel.py
+++ b/youtube_dl/extractor/spiegel.py
@@ -4,8 +4,13 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urlparse
 from .spiegeltv import SpiegeltvIE
+from ..compat import compat_urlparse
+from ..utils import (
+    extract_attributes,
+    unified_strdate,
+    get_element_by_attribute,
+)
 
 
 class SpiegelIE(InfoExtractor):
@@ -19,6 +24,7 @@ class SpiegelIE(InfoExtractor):
             'title': 'Vulkanausbruch in Ecuador: Der "Feuerschlund" ist wieder aktiv',
             'description': 'md5:8029d8310232196eb235d27575a8b9f4',
             'duration': 49,
+            'upload_date': '20130311',
         },
     }, {
         'url': 'http://www.spiegel.de/video/schach-wm-videoanalyse-des-fuenften-spiels-video-1309159.html',
@@ -29,6 +35,7 @@ class SpiegelIE(InfoExtractor):
             'title': 'Schach-WM in der Videoanalyse: Carlsen nutzt die Fehlgriffe des Titelverteidigers',
             'description': 'md5:c2322b65e58f385a820c10fa03b2d088',
             'duration': 983,
+            'upload_date': '20131115',
         },
     }, {
         'url': 'http://www.spiegel.de/video/astronaut-alexander-gerst-von-der-iss-station-beantwortet-fragen-video-1519126-embed.html',
@@ -38,6 +45,7 @@ class SpiegelIE(InfoExtractor):
             'ext': 'mp4',
             'description': 'SPIEGEL ONLINE-Nutzer durften den deutschen Astronauten Alexander Gerst über sein Leben auf der ISS-Station befragen. Hier kommen seine Antworten auf die besten sechs Fragen.',
             'title': 'Fragen an Astronaut Alexander Gerst: "Bekommen Sie die Tageszeiten mit?"',
+            'upload_date': '20140904',
         }
     }, {
         'url': 'http://www.spiegel.de/video/astronaut-alexander-gerst-von-der-iss-station-beantwortet-fragen-video-1519126-iframe.html',
@@ -52,10 +60,10 @@ class SpiegelIE(InfoExtractor):
         if SpiegeltvIE.suitable(handle.geturl()):
             return self.url_result(handle.geturl(), 'Spiegeltv')
 
-        title = re.sub(r'\s+', ' ', self._html_search_regex(
-            r'(?s)<(?:h1|div) class="module-title"[^>]*>(.*?)</(?:h1|div)>',
-            webpage, 'title'))
-        description = self._html_search_meta('description', webpage, 'description')
+        video_data = extract_attributes(self._search_regex(r'(<div[^>]+id="spVideoElements"[^>]+>)', webpage, 'video element', default=''))
+
+        title = video_data.get('data-video-title') or get_element_by_attribute('class', 'module-title', webpage)
+        description = video_data.get('data-video-teaser') or self._html_search_meta('description', webpage, 'description')
 
         base_url = self._search_regex(
             [r'server\s*:\s*(["\'])(?P<url>.+?)\1', r'var\s+server\s*=\s*"(?P<url>[^"]+)\"'],
@@ -87,8 +95,9 @@ class SpiegelIE(InfoExtractor):
         return {
             'id': video_id,
             'title': title,
-            'description': description,
+            'description': description.strip() if description else None,
             'duration': duration,
+            'upload_date': unified_strdate(video_data.get('data-video-date')),
             'formats': formats,
         }
 

From 77082c7b9ef2ea95161e4e288c110b5f7f34fda0 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 5 Jul 2016 12:01:04 +0100
Subject: [PATCH 5427/6219] [slideshare] fix description extraction

---
 youtube_dl/extractor/slideshare.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/slideshare.py b/youtube_dl/extractor/slideshare.py
index 0b717a1e4..4967c1b77 100644
--- a/youtube_dl/extractor/slideshare.py
+++ b/youtube_dl/extractor/slideshare.py
@@ -9,6 +9,7 @@ from ..compat import (
 )
 from ..utils import (
     ExtractorError,
+    get_element_by_id,
 )
 
 
@@ -40,7 +41,7 @@ class SlideshareIE(InfoExtractor):
         bucket = info['jsplayer']['video_bucket']
         ext = info['jsplayer']['video_extension']
         video_url = compat_urlparse.urljoin(bucket, doc + '-SD.' + ext)
-        description = self._html_search_regex(
+        description = get_element_by_id('slideshow-description-paragraph', webpage) or self._html_search_regex(
             r'(?s)<p[^>]+itemprop="description"[^>]*>(.+?)</p>', webpage,
             'description', fatal=False)
 
@@ -51,5 +52,5 @@ class SlideshareIE(InfoExtractor):
             'ext': ext,
             'url': video_url,
             'thumbnail': info['slideshow']['pin_image_url'],
-            'description': description,
+            'description': description.strip() if description else None,
         }

From 94a5cff91d81c120b8e8b395f9fbe465286940f0 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 5 Jul 2016 13:37:46 +0100
Subject: [PATCH 5428/6219] [sendia] fix info extraction

---
 youtube_dl/extractor/sandia.py | 94 +++++++++-------------------------
 1 file changed, 25 insertions(+), 69 deletions(-)

diff --git a/youtube_dl/extractor/sandia.py b/youtube_dl/extractor/sandia.py
index 759898a49..9ab4d20a7 100644
--- a/youtube_dl/extractor/sandia.py
+++ b/youtube_dl/extractor/sandia.py
@@ -27,7 +27,8 @@ class SandiaIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Xyce Software Training - Section 1',
             'description': 're:(?s)SAND Number: SAND 2013-7800.{200,}',
-            'upload_date': '20120904',
+            'upload_date': '20120409',
+            'timestamp': 1333983600,
             'duration': 7794,
         }
     }
@@ -35,81 +36,36 @@ class SandiaIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        req = sanitized_Request(url)
-        req.add_header('Cookie', 'MediasitePlayerCaps=ClientPlugins=4')
-        webpage = self._download_webpage(req, video_id)
+        presentation_data = self._download_json(
+            'http://digitalops.sandia.gov/Mediasite/PlayerService/PlayerService.svc/json/GetPlayerOptions',
+            video_id, data=json.dumps({
+                'getPlayerOptionsRequest': {
+                    'ResourceId': video_id,
+                    'QueryString': '',
+                }
+            }), headers={
+                'Content-Type': 'application/json; charset=utf-8',
+            })['d']['Presentation']
 
-        js_path = self._search_regex(
-            r'<script type="text/javascript" src="(/Mediasite/FileServer/Presentation/[^"]+)"',
-            webpage, 'JS code URL')
-        js_url = compat_urlparse.urljoin(url, js_path)
-
-        js_code = self._download_webpage(
-            js_url, video_id, note='Downloading player')
-
-        def extract_str(key, **args):
-            return self._search_regex(
-                r'Mediasite\.PlaybackManifest\.%s\s*=\s*(.+);\s*?\n' % re.escape(key),
-                js_code, key, **args)
-
-        def extract_data(key, **args):
-            data_json = extract_str(key, **args)
-            if data_json is None:
-                return data_json
-            return self._parse_json(
-                data_json, video_id, transform_source=js_to_json)
+        title = presentation_data['Title']
 
         formats = []
-        for i in itertools.count():
-            fd = extract_data('VideoUrls[%d]' % i, default=None)
-            if fd is None:
-                break
-            formats.append({
-                'format_id': '%s' % i,
-                'format_note': fd['MimeType'].partition('/')[2],
-                'ext': mimetype2ext(fd['MimeType']),
-                'url': fd['Location'],
-                'protocol': 'f4m' if fd['MimeType'] == 'video/x-mp4-fragmented' else None,
-            })
+        for stream in presentation_data.get('Streams', []):
+            for fd in stream.get('VideoUrls', []):
+                formats.append({
+                    'format_id': fd['MediaType'],
+                    'format_note': fd['MimeType'].partition('/')[2],
+                    'ext': mimetype2ext(fd['MimeType']),
+                    'url': fd['Location'],
+                    'protocol': 'f4m' if fd['MimeType'] == 'video/x-mp4-fragmented' else None,
+                })
         self._sort_formats(formats)
 
-        slide_baseurl = compat_urlparse.urljoin(
-            url, extract_data('SlideBaseUrl'))
-        slide_template = slide_baseurl + re.sub(
-            r'\{0:D?([0-9+])\}', r'%0\1d', extract_data('SlideImageFileNameTemplate'))
-        slides = []
-        last_slide_time = 0
-        for i in itertools.count(1):
-            sd = extract_str('Slides[%d]' % i, default=None)
-            if sd is None:
-                break
-            timestamp = int_or_none(self._search_regex(
-                r'^Mediasite\.PlaybackManifest\.CreateSlide\("[^"]*"\s*,\s*([0-9]+),',
-                sd, 'slide %s timestamp' % i, fatal=False))
-            slides.append({
-                'url': slide_template % i,
-                'duration': timestamp - last_slide_time,
-            })
-            last_slide_time = timestamp
-        formats.append({
-            'format_id': 'slides',
-            'protocol': 'slideshow',
-            'url': json.dumps(slides),
-            'preference': -10000,  # Downloader not yet written
-        })
-        self._sort_formats(formats)
-
-        title = extract_data('Title')
-        description = extract_data('Description', fatal=False)
-        duration = int_or_none(extract_data(
-            'Duration', fatal=False), scale=1000)
-        upload_date = unified_strdate(extract_data('AirDate', fatal=False))
-
         return {
             'id': video_id,
             'title': title,
-            'description': description,
+            'description': presentation_data.get('Description'),
             'formats': formats,
-            'upload_date': upload_date,
-            'duration': duration,
+            'timestamp': int_or_none(presentation_data.get('UnixTime'), 1000),
+            'duration': int_or_none(presentation_data.get('Duration'), 1000),
         }

From 31af3e35e094715f5b6616b7ea14e1445ecdc19c Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 5 Jul 2016 13:39:24 +0100
Subject: [PATCH 5429/6219] [sandia] remove unused imports

---
 youtube_dl/extractor/sandia.py | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/youtube_dl/extractor/sandia.py b/youtube_dl/extractor/sandia.py
index 9ab4d20a7..96e43af84 100644
--- a/youtube_dl/extractor/sandia.py
+++ b/youtube_dl/extractor/sandia.py
@@ -1,18 +1,12 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import itertools
 import json
-import re
 
 from .common import InfoExtractor
-from ..compat import compat_urlparse
 from ..utils import (
     int_or_none,
-    js_to_json,
     mimetype2ext,
-    sanitized_Request,
-    unified_strdate,
 )
 
 

From e3f88be7a95f1587879c5b83444dd361b5311ad3 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 5 Jul 2016 14:45:39 +0100
Subject: [PATCH 5430/6219] [rtvnh] extract all formats

---
 youtube_dl/extractor/rtvnh.py | 32 +++++++++++++++++++++++---------
 1 file changed, 23 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/rtvnh.py b/youtube_dl/extractor/rtvnh.py
index 4896d09d6..f6454c6b0 100644
--- a/youtube_dl/extractor/rtvnh.py
+++ b/youtube_dl/extractor/rtvnh.py
@@ -9,7 +9,7 @@ class RTVNHIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?rtvnh\.nl/video/(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://www.rtvnh.nl/video/131946',
-        'md5': '6e1d0ab079e2a00b6161442d3ceacfc1',
+        'md5': 'cdbec9f44550763c8afc96050fa747dc',
         'info_dict': {
             'id': '131946',
             'ext': 'mp4',
@@ -29,15 +29,29 @@ class RTVNHIE(InfoExtractor):
             raise ExtractorError(
                 '%s returned error code %d' % (self.IE_NAME, status), expected=True)
 
-        formats = self._extract_smil_formats(
-            'http://www.rtvnh.nl/video/smil?m=' + video_id, video_id, fatal=False)
+        formats = []
+        rtmp_formats = self._extract_smil_formats(
+            'http://www.rtvnh.nl/video/smil?m=' + video_id, video_id)
+        formats.extend(rtmp_formats)
 
-        for item in meta['source']['fb']:
-            if item.get('type') == 'hls':
-                formats.extend(self._extract_m3u8_formats(
-                    item['file'], video_id, ext='mp4', entry_protocol='m3u8_native'))
-            elif item.get('type') == '':
-                formats.append({'url': item['file']})
+        for rtmp_format in rtmp_formats:
+            rtmp_url = '%s/%s' % (rtmp_format['url'], rtmp_format['play_path'])
+            rtsp_format = rtmp_format.copy()
+            del rtsp_format['play_path']
+            del rtsp_format['ext']
+            rtsp_format.update({
+                'format_id': rtmp_format['format_id'].replace('rtmp', 'rtsp'),
+                'url': rtmp_url.replace('rtmp://', 'rtsp://'),
+                'protocol': 'rtsp',
+            })
+            formats.append(rtsp_format)
+            http_base_url = rtmp_url.replace('rtmp://', 'http://')
+            formats.extend(self._extract_m3u8_formats(
+                http_base_url + '/playlist.m3u8', video_id, 'mp4',
+                'm3u8_native', m3u8_id='hls', fatal=False))
+            formats.extend(self._extract_f4m_formats(
+                http_base_url + '/manifest.f4m',
+                video_id, f4m_id='hds', fatal=False))
         self._sort_formats(formats)
 
         return {

From 84f214d840bfdb1b3342dbf6a07d94a25787660f Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 5 Jul 2016 17:11:45 +0100
Subject: [PATCH 5431/6219] [prosiebensat1] extract all formats

---
 youtube_dl/extractor/prosiebensat1.py | 165 ++++++++++++++------------
 1 file changed, 88 insertions(+), 77 deletions(-)

diff --git a/youtube_dl/extractor/prosiebensat1.py b/youtube_dl/extractor/prosiebensat1.py
index 07d49d489..ad0f0031c 100644
--- a/youtube_dl/extractor/prosiebensat1.py
+++ b/youtube_dl/extractor/prosiebensat1.py
@@ -5,7 +5,7 @@ import re
 
 from hashlib import sha1
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse_urlencode
+from ..compat import compat_str
 from ..utils import (
     ExtractorError,
     determine_ext,
@@ -71,6 +71,7 @@ class ProSiebenSat1IE(InfoExtractor):
                 # rtmp download
                 'skip_download': True,
             },
+            'skip': 'This video is unavailable',
         },
         {
             'url': 'http://www.sixx.de/stars-style/video/sexy-laufen-in-ugg-boots-clip',
@@ -86,6 +87,7 @@ class ProSiebenSat1IE(InfoExtractor):
                 # rtmp download
                 'skip_download': True,
             },
+            'skip': 'This video is unavailable',
         },
         {
             'url': 'http://www.sat1.de/film/der-ruecktritt/video/im-interview-kai-wiesinger-clip',
@@ -101,6 +103,7 @@ class ProSiebenSat1IE(InfoExtractor):
                 # rtmp download
                 'skip_download': True,
             },
+            'skip': 'This video is unavailable',
         },
         {
             'url': 'http://www.kabeleins.de/tv/rosins-restaurants/videos/jagd-auf-fertigkost-im-elsthal-teil-2-ganze-folge',
@@ -116,6 +119,7 @@ class ProSiebenSat1IE(InfoExtractor):
                 # rtmp download
                 'skip_download': True,
             },
+            'skip': 'This video is unavailable',
         },
         {
             'url': 'http://www.ran.de/fussball/bundesliga/video/schalke-toennies-moechte-raul-zurueck-ganze-folge',
@@ -131,6 +135,7 @@ class ProSiebenSat1IE(InfoExtractor):
                 # rtmp download
                 'skip_download': True,
             },
+            'skip': 'This video is unavailable',
         },
         {
             'url': 'http://www.the-voice-of-germany.de/video/31-andreas-kuemmert-rocket-man-clip',
@@ -227,70 +232,42 @@ class ProSiebenSat1IE(InfoExtractor):
     ]
 
     def _extract_clip(self, url, webpage):
-        clip_id = self._html_search_regex(self._CLIPID_REGEXES, webpage, 'clip id')
+        clip_id = self._html_search_regex(
+            self._CLIPID_REGEXES, webpage, 'clip id')
 
         access_token = 'prosieben'
         client_name = 'kolibri-2.0.19-splec4'
         client_location = url
 
-        videos_api_url = 'http://vas.sim-technik.de/vas/live/v2/videos?%s' % compat_urllib_parse_urlencode({
-            'access_token': access_token,
-            'client_location': client_location,
-            'client_name': client_name,
-            'ids': clip_id,
-        })
-
-        video = self._download_json(videos_api_url, clip_id, 'Downloading videos JSON')[0]
+        video = self._download_json(
+            'http://vas.sim-technik.de/vas/live/v2/videos',
+            clip_id, 'Downloading videos JSON', query={
+                'access_token': access_token,
+                'client_location': client_location,
+                'client_name': client_name,
+                'ids': clip_id,
+            })[0]
 
         if video.get('is_protected') is True:
             raise ExtractorError('This video is DRM protected.', expected=True)
 
         duration = float_or_none(video.get('duration'))
-        source_ids = [source['id'] for source in video['sources']]
-        source_ids_str = ','.join(map(str, source_ids))
+        source_ids = [compat_str(source['id']) for source in video['sources']]
 
         g = '01!8d8F_)r9]4s[qeuXfP%'
+        client_id = g[:2] + sha1(''.join([clip_id, g, access_token, client_location, g, client_name]).encode('utf-8')).hexdigest()
 
-        client_id = g[:2] + sha1(''.join([clip_id, g, access_token, client_location, g, client_name])
-                                 .encode('utf-8')).hexdigest()
-
-        sources_api_url = 'http://vas.sim-technik.de/vas/live/v2/videos/%s/sources?%s' % (clip_id, compat_urllib_parse_urlencode({
-            'access_token': access_token,
-            'client_id': client_id,
-            'client_location': client_location,
-            'client_name': client_name,
-        }))
-
-        sources = self._download_json(sources_api_url, clip_id, 'Downloading sources JSON')
+        sources = self._download_json(
+            'http://vas.sim-technik.de/vas/live/v2/videos/%s/sources' % clip_id,
+            clip_id, 'Downloading sources JSON', query={
+                'access_token': access_token,
+                'client_id': client_id,
+                'client_location': client_location,
+                'client_name': client_name,
+            })
         server_id = sources['server_id']
 
-        client_id = g[:2] + sha1(''.join([g, clip_id, access_token, server_id,
-                                          client_location, source_ids_str, g, client_name])
-                                 .encode('utf-8')).hexdigest()
-
-        url_api_url = 'http://vas.sim-technik.de/vas/live/v2/videos/%s/sources/url?%s' % (clip_id, compat_urllib_parse_urlencode({
-            'access_token': access_token,
-            'client_id': client_id,
-            'client_location': client_location,
-            'client_name': client_name,
-            'server_id': server_id,
-            'source_ids': source_ids_str,
-        }))
-
-        urls = self._download_json(url_api_url, clip_id, 'Downloading urls JSON')
-
         title = self._html_search_regex(self._TITLE_REGEXES, webpage, 'title')
-        description = self._html_search_regex(self._DESCRIPTION_REGEXES, webpage, 'description', fatal=False)
-        thumbnail = self._og_search_thumbnail(webpage)
-
-        upload_date = unified_strdate(self._html_search_regex(
-            self._UPLOAD_DATE_REGEXES, webpage, 'upload date', default=None))
-
-        formats = []
-
-        urls_sources = urls['sources']
-        if isinstance(urls_sources, dict):
-            urls_sources = urls_sources.values()
 
         def fix_bitrate(bitrate):
             bitrate = int_or_none(bitrate)
@@ -298,37 +275,71 @@ class ProSiebenSat1IE(InfoExtractor):
                 return None
             return (bitrate // 1000) if bitrate % 1000 == 0 else bitrate
 
-        for source in urls_sources:
-            protocol = source['protocol']
-            source_url = source['url']
-            if protocol == 'rtmp' or protocol == 'rtmpe':
-                mobj = re.search(r'^(?P<url>rtmpe?://[^/]+)/(?P<path>.+)$', source_url)
-                if not mobj:
+        formats = []
+        for source_id in source_ids:
+            client_id = g[:2] + sha1(''.join([g, clip_id, access_token, server_id, client_location, source_id, g, client_name]).encode('utf-8')).hexdigest()
+            urls = self._download_json(
+                'http://vas.sim-technik.de/vas/live/v2/videos/%s/sources/url' % clip_id,
+                clip_id, 'Downloading urls JSON', query={
+                    'access_token': access_token,
+                    'client_id': client_id,
+                    'client_location': client_location,
+                    'client_name': client_name,
+                    'server_id': server_id,
+                    'source_ids': source_id,
+                })
+            if urls.get('status_code') != 0:
+                raise ExtractorError('This video is unavailable', expected=True)
+            urls_sources = urls['sources']
+            if isinstance(urls_sources, dict):
+                urls_sources = urls_sources.values()
+            for source in urls_sources:
+                source_url = source.get('url')
+                if not source_url:
                     continue
-                path = mobj.group('path')
-                mp4colon_index = path.rfind('mp4:')
-                app = path[:mp4colon_index]
-                play_path = path[mp4colon_index:]
-                formats.append({
-                    'url': '%s/%s' % (mobj.group('url'), app),
-                    'app': app,
-                    'play_path': play_path,
-                    'player_url': 'http://livepassdl.conviva.com/hf/ver/2.79.0.17083/LivePassModuleMain.swf',
-                    'page_url': 'http://www.prosieben.de',
-                    'vbr': fix_bitrate(source['bitrate']),
-                    'ext': 'mp4',
-                    'format_id': '%s_%s' % (source['cdn'], source['bitrate']),
-                })
-            elif 'f4mgenerator' in source_url or determine_ext(source_url) == 'f4m':
-                formats.extend(self._extract_f4m_formats(source_url, clip_id))
-            else:
-                formats.append({
-                    'url': source_url,
-                    'vbr': fix_bitrate(source['bitrate']),
-                })
-
+                protocol = source.get('protocol')
+                mimetype = source.get('mimetype')
+                if mimetype == 'application/f4m+xml' or 'f4mgenerator' in source_url or determine_ext(source_url) == 'f4m':
+                    formats.extend(self._extract_f4m_formats(
+                        source_url, clip_id, f4m_id='hds', fatal=False))
+                elif mimetype == 'application/x-mpegURL':
+                    formats.extend(self._extract_m3u8_formats(
+                        source_url, clip_id, 'mp4', 'm3u8_native',
+                        m3u8_id='hls', fatal=False))
+                else:
+                    tbr = fix_bitrate(source['bitrate'])
+                    if protocol in ('rtmp', 'rtmpe'):
+                        mobj = re.search(r'^(?P<url>rtmpe?://[^/]+)/(?P<path>.+)$', source_url)
+                        if not mobj:
+                            continue
+                        path = mobj.group('path')
+                        mp4colon_index = path.rfind('mp4:')
+                        app = path[:mp4colon_index]
+                        play_path = path[mp4colon_index:]
+                        formats.append({
+                            'url': '%s/%s' % (mobj.group('url'), app),
+                            'app': app,
+                            'play_path': play_path,
+                            'player_url': 'http://livepassdl.conviva.com/hf/ver/2.79.0.17083/LivePassModuleMain.swf',
+                            'page_url': 'http://www.prosieben.de',
+                            'tbr': tbr,
+                            'ext': 'flv',
+                            'format_id': 'rtmp%s' % ('-%d' % tbr if tbr else ''),
+                        })
+                    else:
+                        formats.append({
+                            'url': source_url,
+                            'tbr': tbr,
+                            'format_id': 'http%s' % ('-%d' % tbr if tbr else ''),
+                        })
         self._sort_formats(formats)
 
+        description = self._html_search_regex(
+            self._DESCRIPTION_REGEXES, webpage, 'description', fatal=False)
+        thumbnail = self._og_search_thumbnail(webpage)
+        upload_date = unified_strdate(self._html_search_regex(
+            self._UPLOAD_DATE_REGEXES, webpage, 'upload date', default=None))
+
         return {
             'id': clip_id,
             'title': title,

From 73c4ac2c95caf535c9177d930921ae42816d669f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 5 Jul 2016 23:30:44 +0700
Subject: [PATCH 5432/6219] [youtube:channel] Improve channel id extraction and
 detect unavailable channels (Closes #10009)

---
 youtube_dl/extractor/youtube.py | 26 ++++++++++++++++++++++----
 1 file changed, 22 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 554debd43..69603c1f8 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1995,9 +1995,13 @@ class YoutubeChannelIE(YoutubePlaylistBaseInfoExtractor):
             channel_playlist_id = self._html_search_meta(
                 'channelId', channel_page, 'channel id', default=None)
             if not channel_playlist_id:
-                channel_playlist_id = self._search_regex(
-                    r'data-(?:channel-external-|yt)id="([^"]+)"',
-                    channel_page, 'channel id', default=None)
+                channel_url = self._html_search_meta(
+                    ('al:ios:url', 'twitter:app:url:iphone', 'twitter:app:url:ipad'),
+                    channel_page, 'channel url', default=None)
+                if channel_url:
+                    channel_playlist_id = self._search_regex(
+                        r'vnd\.youtube://user/([0-9A-Za-z_-]+)',
+                        channel_url, 'channel id', default=None)
         if channel_playlist_id and channel_playlist_id.startswith('UC'):
             playlist_id = 'UU' + channel_playlist_id[2:]
             return self.url_result(
@@ -2020,6 +2024,15 @@ class YoutubeChannelIE(YoutubePlaylistBaseInfoExtractor):
                 for video_id, video_title in self.extract_videos_from_page(channel_page)]
             return self.playlist_result(entries, channel_id)
 
+        try:
+            next(self._entries(channel_page, channel_id))
+        except StopIteration:
+            alert_message = self._html_search_regex(
+                r'(?s)<div[^>]+class=(["\']).*?\byt-alert-message\b.*?\1[^>]*>(?P<alert>[^<]+)</div>',
+                channel_page, 'alert', default=None, group='alert')
+            if alert_message:
+                raise ExtractorError('Youtube said: %s' % alert_message, expected=True)
+
         return self.playlist_result(self._entries(channel_page, channel_id), channel_id)
 
 
@@ -2033,7 +2046,8 @@ class YoutubeUserIE(YoutubeChannelIE):
         'url': 'https://www.youtube.com/user/TheLinuxFoundation',
         'playlist_mincount': 320,
         'info_dict': {
-            'title': 'TheLinuxFoundation',
+            'id': 'UUfX55Sx5hEFjoC3cNs6mCUQ',
+            'title': 'Uploads from The Linux Foundation',
         }
     }, {
         'url': 'ytuser:phihag',
@@ -2041,6 +2055,10 @@ class YoutubeUserIE(YoutubeChannelIE):
     }, {
         'url': 'https://www.youtube.com/c/gametrailers',
         'only_matching': True,
+    }, {
+        # This channel is not available.
+        'url': 'https://www.youtube.com/user/kananishinoSMEJ/videos',
+        'only_matching': True,
     }]
 
     @classmethod

From 6c26815d638c55c106fb73a3d0ce6d87fd94bd4c Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 5 Jul 2016 18:04:42 +0100
Subject: [PATCH 5433/6219] [onionstudios] fix info extraction

---
 youtube_dl/extractor/onionstudios.py | 61 ++++++++++++----------------
 1 file changed, 25 insertions(+), 36 deletions(-)

diff --git a/youtube_dl/extractor/onionstudios.py b/youtube_dl/extractor/onionstudios.py
index d7b13a0f1..9fd5b2123 100644
--- a/youtube_dl/extractor/onionstudios.py
+++ b/youtube_dl/extractor/onionstudios.py
@@ -7,6 +7,7 @@ from .common import InfoExtractor
 from ..utils import (
     determine_ext,
     int_or_none,
+    float_or_none,
 )
 
 
@@ -15,15 +16,14 @@ class OnionStudiosIE(InfoExtractor):
 
     _TESTS = [{
         'url': 'http://www.onionstudios.com/videos/hannibal-charges-forward-stops-for-a-cocktail-2937',
-        'md5': 'd4851405d31adfadf71cd7a487b765bb',
+        'md5': 'e49f947c105b8a78a675a0ee1bddedfe',
         'info_dict': {
             'id': '2937',
             'ext': 'mp4',
             'title': 'Hannibal charges forward, stops for a cocktail',
-            'description': 'md5:e786add7f280b7f0fe237b64cc73df76',
             'thumbnail': 're:^https?://.*\.jpg$',
             'uploader': 'The A.V. Club',
-            'uploader_id': 'TheAVClub',
+            'uploader_id': 'the-av-club',
         },
     }, {
         'url': 'http://www.onionstudios.com/embed?id=2855&autoplay=true',
@@ -40,50 +40,39 @@ class OnionStudiosIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        webpage = self._download_webpage(
-            'http://www.onionstudios.com/embed?id=%s' % video_id, video_id)
+        video_data = self._download_json(
+            'http://www.onionstudios.com/video/%s.json' % video_id, video_id)
+
+        title = video_data['title']
 
         formats = []
-        for src in re.findall(r'<source[^>]+src="([^"]+)"', webpage):
-            ext = determine_ext(src)
-            if ext == 'm3u8':
+        for source in video_data.get('sources', []):
+            source_url = source.get('url')
+            if not source_url:
+                continue
+            content_type = source.get('content_type')
+            ext = determine_ext(source_url)
+            if content_type == 'application/x-mpegURL' or ext == 'm3u8':
                 formats.extend(self._extract_m3u8_formats(
-                    src, video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
+                    source_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
             else:
-                height = int_or_none(self._search_regex(
-                    r'/(\d+)\.%s' % ext, src, 'height', default=None))
+                tbr = int_or_none(source.get('bitrate'))
                 formats.append({
-                    'format_id': ext + ('-%sp' % height if height else ''),
-                    'url': src,
-                    'height': height,
+                    'format_id': ext + ('-%d' % tbr if tbr else ''),
+                    'url': source_url,
+                    'width': int_or_none(source.get('width')),
+                    'tbr': tbr,
                     'ext': ext,
-                    'preference': 1,
                 })
         self._sort_formats(formats)
 
-        title = self._search_regex(
-            r'share_title\s*=\s*(["\'])(?P<title>[^\1]+?)\1',
-            webpage, 'title', group='title')
-        description = self._search_regex(
-            r'share_description\s*=\s*(["\'])(?P<description>[^\'"]+?)\1',
-            webpage, 'description', default=None, group='description')
-        thumbnail = self._search_regex(
-            r'poster\s*=\s*(["\'])(?P<thumbnail>[^\1]+?)\1',
-            webpage, 'thumbnail', default=False, group='thumbnail')
-
-        uploader_id = self._search_regex(
-            r'twitter_handle\s*=\s*(["\'])(?P<uploader_id>[^\1]+?)\1',
-            webpage, 'uploader id', fatal=False, group='uploader_id')
-        uploader = self._search_regex(
-            r'window\.channelName\s*=\s*(["\'])Embedded:(?P<uploader>[^\1]+?)\1',
-            webpage, 'uploader', default=False, group='uploader')
-
         return {
             'id': video_id,
             'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
-            'uploader': uploader,
-            'uploader_id': uploader_id,
+            'thumbnail': video_data.get('poster_url'),
+            'uploader': video_data.get('channel_name'),
+            'uploader_id': video_data.get('channel_slug'),
+            'duration': float_or_none(video_data.get('duration', 1000)),
+            'tags': video_data.get('tags'),
             'formats': formats,
         }

From a6d3b89febce9450b10cb379568d77895103c998 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 6 Jul 2016 00:52:48 +0700
Subject: [PATCH 5434/6219] [prosiebensat1] Make downloading urls JSON non
 fatal

---
 youtube_dl/extractor/prosiebensat1.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/prosiebensat1.py b/youtube_dl/extractor/prosiebensat1.py
index ad0f0031c..c6eee3b72 100644
--- a/youtube_dl/extractor/prosiebensat1.py
+++ b/youtube_dl/extractor/prosiebensat1.py
@@ -280,7 +280,7 @@ class ProSiebenSat1IE(InfoExtractor):
             client_id = g[:2] + sha1(''.join([g, clip_id, access_token, server_id, client_location, source_id, g, client_name]).encode('utf-8')).hexdigest()
             urls = self._download_json(
                 'http://vas.sim-technik.de/vas/live/v2/videos/%s/sources/url' % clip_id,
-                clip_id, 'Downloading urls JSON', query={
+                clip_id, 'Downloading urls JSON', fatal=False, query={
                     'access_token': access_token,
                     'client_id': client_id,
                     'client_location': client_location,
@@ -288,6 +288,8 @@ class ProSiebenSat1IE(InfoExtractor):
                     'server_id': server_id,
                     'source_ids': source_id,
                 })
+            if not urls:
+                continue
             if urls.get('status_code') != 0:
                 raise ExtractorError('This video is unavailable', expected=True)
             urls_sources = urls['sources']

From 0e94b4713dc6d5afb399fbfb1c90a7c47cb63811 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 6 Jul 2016 00:54:23 +0700
Subject: [PATCH 5435/6219] release 2016.07.06

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 docs/supportedsites.md    | 1 +
 youtube_dl/version.py     | 2 +-
 3 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index d30650b7c..7a79389ce 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.07.05*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.07.05**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.07.06*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.07.06**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.07.05
+[debug] youtube-dl version 2016.07.06
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 93237022f..3388fe221 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -857,6 +857,7 @@
  - **youtube:search**: YouTube.com searches
  - **youtube:search:date**: YouTube.com searches, newest videos first
  - **youtube:search_url**: YouTube.com search URLs
+ - **youtube:shared**
  - **youtube:show**: YouTube.com (multi-season) shows
  - **youtube:subscriptions**: YouTube.com subscriptions feed, "ytsubs" keyword (requires authentication)
  - **youtube:user**: YouTube.com user videos (URL or "ytuser" keyword)
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index cd106dd7d..d1974d089 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.07.05'
+__version__ = '2016.07.06'

From 001fffd004e25c51ceadec9408a1864d6bd4835d Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 6 Jul 2016 00:15:16 +0100
Subject: [PATCH 5436/6219] [spiegel:article] update test(closes #10018)

---
 youtube_dl/extractor/spiegel.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/spiegel.py b/youtube_dl/extractor/spiegel.py
index c67d556e7..3c552807e 100644
--- a/youtube_dl/extractor/spiegel.py
+++ b/youtube_dl/extractor/spiegel.py
@@ -113,6 +113,7 @@ class SpiegelArticleIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Faszination Badminton: Nennt es bloß nicht Federball',
             'description': 're:^Patrick Kämnitz gehört.{100,}',
+            'upload_date': '20140825',
         },
     }, {
         'url': 'http://www.spiegel.de/wissenschaft/weltall/astronaut-alexander-gerst-antwortet-spiegel-online-lesern-a-989876.html',

From 2817b99cf2d799c54df0f393e237886bf9d1b80c Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 6 Jul 2016 02:19:55 +0100
Subject: [PATCH 5437/6219] [metacafe] fix info extraction(closes #8539)(closes
 #3253)

---
 youtube_dl/extractor/metacafe.py | 90 +++++++++++++++++++++-----------
 1 file changed, 60 insertions(+), 30 deletions(-)

diff --git a/youtube_dl/extractor/metacafe.py b/youtube_dl/extractor/metacafe.py
index b6f00cc25..acab81863 100644
--- a/youtube_dl/extractor/metacafe.py
+++ b/youtube_dl/extractor/metacafe.py
@@ -11,13 +11,14 @@ from ..utils import (
     determine_ext,
     ExtractorError,
     int_or_none,
-    sanitized_Request,
     urlencode_postdata,
+    get_element_by_attribute,
+    mimetype2ext,
 )
 
 
 class MetacafeIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?metacafe\.com/watch/([^/]+)/([^/]+)/.*'
+    _VALID_URL = r'https?://(?:www\.)?metacafe\.com/watch/(?P<video_id>[^/]+)/(?P<display_id>[^/?#]+)'
     _DISCLAIMER = 'http://www.metacafe.com/family_filter/'
     _FILTER_POST = 'http://www.metacafe.com/f/index.php?inputType=filter&controllerGroup=user'
     IE_NAME = 'metacafe'
@@ -47,6 +48,7 @@ class MetacafeIE(InfoExtractor):
                 'uploader': 'ign',
                 'description': 'Sony released a massive FAQ on the PlayStation Blog detailing the PS4\'s capabilities and limitations.',
             },
+            'skip': 'Page is temporarily unavailable.',
         },
         # AnyClip video
         {
@@ -55,8 +57,8 @@ class MetacafeIE(InfoExtractor):
                 'id': 'an-dVVXnuY7Jh77J',
                 'ext': 'mp4',
                 'title': 'The Andromeda Strain (1971): Stop the Bomb Part 3',
-                'uploader': 'anyclip',
-                'description': 'md5:38c711dd98f5bb87acf973d573442e67',
+                'uploader': 'AnyClip',
+                'description': 'md5:cbef0460d31e3807f6feb4e7a5952e5b',
             },
         },
         # age-restricted video
@@ -110,28 +112,25 @@ class MetacafeIE(InfoExtractor):
     def report_disclaimer(self):
         self.to_screen('Retrieving disclaimer')
 
-    def _real_initialize(self):
+    def _confirm_age(self):
         # Retrieve disclaimer
         self.report_disclaimer()
         self._download_webpage(self._DISCLAIMER, None, False, 'Unable to retrieve disclaimer')
 
         # Confirm age
-        disclaimer_form = {
-            'filters': '0',
-            'submit': "Continue - I'm over 18",
-        }
-        request = sanitized_Request(self._FILTER_POST, urlencode_postdata(disclaimer_form))
-        request.add_header('Content-Type', 'application/x-www-form-urlencoded')
         self.report_age_confirmation()
-        self._download_webpage(request, None, False, 'Unable to confirm age')
+        self._download_webpage(
+            self._FILTER_POST, None, False, 'Unable to confirm age',
+            data=urlencode_postdata({
+                'filters': '0',
+                'submit': "Continue - I'm over 18",
+            }), headers={
+                'Content-Type': 'application/x-www-form-urlencoded',
+            })
 
     def _real_extract(self, url):
         # Extract id and simplified title from URL
-        mobj = re.match(self._VALID_URL, url)
-        if mobj is None:
-            raise ExtractorError('Invalid URL: %s' % url)
-
-        video_id = mobj.group(1)
+        video_id, display_id = re.match(self._VALID_URL, url).groups()
 
         # the video may come from an external site
         m_external = re.match('^(\w{2})-(.*)$', video_id)
@@ -144,15 +143,24 @@ class MetacafeIE(InfoExtractor):
             if prefix == 'cb':
                 return self.url_result('theplatform:%s' % ext_id, 'ThePlatform')
 
-        # Retrieve video webpage to extract further information
-        req = sanitized_Request('http://www.metacafe.com/watch/%s/' % video_id)
+        # self._confirm_age()
 
         # AnyClip videos require the flashversion cookie so that we get the link
         # to the mp4 file
-        mobj_an = re.match(r'^an-(.*?)$', video_id)
-        if mobj_an:
-            req.headers['Cookie'] = 'flashVersion=0;'
-        webpage = self._download_webpage(req, video_id)
+        headers = {}
+        if video_id.startswith('an-'):
+            headers['Cookie'] = 'flashVersion=0;'
+
+        # Retrieve video webpage to extract further information
+        webpage = self._download_webpage(url, video_id, headers=headers)
+
+        error = get_element_by_attribute(
+            'class', 'notfound-page-title', webpage)
+        if error:
+            raise ExtractorError(error, expected=True)
+
+        video_title = self._html_search_meta(
+            ['og:title', 'twitter:title'], webpage, 'title', default=None) or self._search_regex(r'<h1>(.*?)</h1>', webpage, 'title')
 
         # Extract URL, uploader and title from webpage
         self.report_extraction(video_id)
@@ -216,20 +224,41 @@ class MetacafeIE(InfoExtractor):
                         'player_url': player_url,
                         'ext': play_path.partition(':')[0],
                     })
+        if video_url is None:
+            flashvars = self._parse_json(self._search_regex(
+                r'flashvars\s*=\s*({.*});', webpage, 'flashvars',
+                default=None), video_id, fatal=False)
+            if flashvars:
+                video_url = []
+                for source in flashvars.get('sources'):
+                    source_url = source.get('src')
+                    if not source_url:
+                        continue
+                    mime_type = source.get('type')
+                    ext = mimetype2ext(mime_type) or determine_ext(source_url)
+                    if mime_type == 'application/x-mpegURL' or ext == 'm3u8':
+                        video_url.extend(self._extract_m3u8_formats(
+                            source_url, video_id, 'mp4',
+                            'm3u8_native', m3u8_id='hls', fatal=False))
+                    else:
+                        video_url.append({
+                            'url': source_url,
+                            'ext': ext,
+                        })
 
         if video_url is None:
             raise ExtractorError('Unsupported video type')
 
-        video_title = self._html_search_regex(
-            r'(?im)<title>(.*) - Video</title>', webpage, 'title')
-        description = self._og_search_description(webpage)
-        thumbnail = self._og_search_thumbnail(webpage)
+        description = self._html_search_meta(
+            ['og:description', 'twitter:description', 'description'],
+            webpage, 'title', fatal=False)
+        thumbnail = self._html_search_meta(
+            ['og:image', 'twitter:image'], webpage, 'title', fatal=False)
         video_uploader = self._html_search_regex(
             r'submitter=(.*?);|googletag\.pubads\(\)\.setTargeting\("(?:channel|submiter)","([^"]+)"\);',
             webpage, 'uploader nickname', fatal=False)
         duration = int_or_none(
-            self._html_search_meta('video:duration', webpage))
-
+            self._html_search_meta('video:duration', webpage, default=None))
         age_limit = (
             18
             if re.search(r'(?:"contentRating":|"rating",)"restricted"', webpage)
@@ -242,10 +271,11 @@ class MetacafeIE(InfoExtractor):
                 'url': video_url,
                 'ext': video_ext,
             }]
-
         self._sort_formats(formats)
+
         return {
             'id': video_id,
+            'display_id': display_id,
             'description': description,
             'uploader': video_uploader,
             'title': video_title,

From b4173f1551c47f64745cb91451c46891e2aaac16 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 6 Jul 2016 09:06:28 +0100
Subject: [PATCH 5438/6219] [utils] add mimetypes to determine manifest
 ext(m3u8, f4m, mpd)

---
 youtube_dl/utils.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index d302f39e4..36d5b6c0f 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2097,6 +2097,7 @@ def mimetype2ext(mt):
         return ext
 
     _, _, res = mt.rpartition('/')
+    res = res.lower()
 
     return {
         '3gpp': '3gp',
@@ -2108,6 +2109,12 @@ def mimetype2ext(mt):
         'x-flv': 'flv',
         'x-mp4-fragmented': 'mp4',
         'x-ms-wmv': 'wmv',
+        'mpegurl': 'm3u8',
+        'x-mpegurl': 'm3u8',
+        'vnd.apple.mpegurl': 'm3u8',
+        'dash+xml': 'mpd',
+        'f4m': 'f4m',
+        'f4m+xml': 'f4m',
     }.get(res, res)
 
 

From ab49d7a9fae08763de549f85ba138b22f9122a99 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 6 Jul 2016 09:11:46 +0100
Subject: [PATCH 5439/6219] use mimetype2ext to determine manifest ext in
 multiple extractors

---
 youtube_dl/extractor/amp.py          | 18 ++++++++++++------
 youtube_dl/extractor/dailymotion.py  |  8 +++++---
 youtube_dl/extractor/metacafe.py     |  5 ++---
 youtube_dl/extractor/onionstudios.py |  6 +++---
 youtube_dl/extractor/sixplay.py      |  8 ++++++--
 youtube_dl/extractor/threeqsdn.py    |  9 ++++-----
 6 files changed, 32 insertions(+), 22 deletions(-)

diff --git a/youtube_dl/extractor/amp.py b/youtube_dl/extractor/amp.py
index 8545681be..283736fc5 100644
--- a/youtube_dl/extractor/amp.py
+++ b/youtube_dl/extractor/amp.py
@@ -5,6 +5,8 @@ from .common import InfoExtractor
 from ..utils import (
     int_or_none,
     parse_iso8601,
+    mimetype2ext,
+    determine_ext,
 )
 
 
@@ -50,21 +52,25 @@ class AMPIE(InfoExtractor):
         if isinstance(media_content, dict):
             media_content = [media_content]
         for media_data in media_content:
-            media = media_data['@attributes']
-            media_type = media['type']
-            if media_type in ('video/f4m', 'application/f4m+xml'):
+            media = media_data.get('@attributes', {})
+            media_url = media.get('url')
+            if not media_url:
+                continue
+            ext = mimetype2ext(media.get('type')) or determne_ext(media_url)
+            if ext == 'f4m':
                 formats.extend(self._extract_f4m_formats(
-                    media['url'] + '?hdcore=3.4.0&plugin=aasp-3.4.0.132.124',
+                    media_url + '?hdcore=3.4.0&plugin=aasp-3.4.0.132.124',
                     video_id, f4m_id='hds', fatal=False))
-            elif media_type == 'application/x-mpegURL':
+            elif ext == 'm3u8':
                 formats.extend(self._extract_m3u8_formats(
-                    media['url'], video_id, 'mp4', m3u8_id='hls', fatal=False))
+                    media_url, video_id, 'mp4', m3u8_id='hls', fatal=False))
             else:
                 formats.append({
                     'format_id': media_data.get('media-category', {}).get('@attributes', {}).get('label'),
                     'url': media['url'],
                     'tbr': int_or_none(media.get('bitrate')),
                     'filesize': int_or_none(media.get('fileSize')),
+                    'ext': ext,
                 })
 
         self._sort_formats(formats)
diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index 2e6226ea0..15bfc59b2 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -16,6 +16,7 @@ from ..utils import (
     sanitized_Request,
     str_to_int,
     unescapeHTML,
+    mimetype2ext,
 )
 
 
@@ -153,18 +154,19 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
                     type_ = media.get('type')
                     if type_ == 'application/vnd.lumberjack.manifest':
                         continue
-                    ext = determine_ext(media_url)
-                    if type_ == 'application/x-mpegURL' or ext == 'm3u8':
+                    ext = mimetype2ext(type_) or determine_ext(media_url)
+                    if ext == 'm3u8':
                         formats.extend(self._extract_m3u8_formats(
                             media_url, video_id, 'mp4', preference=-1,
                             m3u8_id='hls', fatal=False))
-                    elif type_ == 'application/f4m' or ext == 'f4m':
+                    elif ext == 'f4m':
                         formats.extend(self._extract_f4m_formats(
                             media_url, video_id, preference=-1, f4m_id='hds', fatal=False))
                     else:
                         f = {
                             'url': media_url,
                             'format_id': 'http-%s' % quality,
+                            'ext': ext,
                         }
                         m = re.search(r'H264-(?P<width>\d+)x(?P<height>\d+)', media_url)
                         if m:
diff --git a/youtube_dl/extractor/metacafe.py b/youtube_dl/extractor/metacafe.py
index acab81863..e6e7659a1 100644
--- a/youtube_dl/extractor/metacafe.py
+++ b/youtube_dl/extractor/metacafe.py
@@ -234,9 +234,8 @@ class MetacafeIE(InfoExtractor):
                     source_url = source.get('src')
                     if not source_url:
                         continue
-                    mime_type = source.get('type')
-                    ext = mimetype2ext(mime_type) or determine_ext(source_url)
-                    if mime_type == 'application/x-mpegURL' or ext == 'm3u8':
+                    ext = mimetype2ext(source.get('type')) or determine_ext(source_url)
+                    if ext == 'm3u8':
                         video_url.extend(self._extract_m3u8_formats(
                             source_url, video_id, 'mp4',
                             'm3u8_native', m3u8_id='hls', fatal=False))
diff --git a/youtube_dl/extractor/onionstudios.py b/youtube_dl/extractor/onionstudios.py
index 9fd5b2123..6fb1a3fcc 100644
--- a/youtube_dl/extractor/onionstudios.py
+++ b/youtube_dl/extractor/onionstudios.py
@@ -8,6 +8,7 @@ from ..utils import (
     determine_ext,
     int_or_none,
     float_or_none,
+    mimetype2ext,
 )
 
 
@@ -50,9 +51,8 @@ class OnionStudiosIE(InfoExtractor):
             source_url = source.get('url')
             if not source_url:
                 continue
-            content_type = source.get('content_type')
-            ext = determine_ext(source_url)
-            if content_type == 'application/x-mpegURL' or ext == 'm3u8':
+            ext = mimetype2ext(source.get('content_type')) or determine_ext(source_url)
+            if ext == 'm3u8':
                 formats.extend(self._extract_m3u8_formats(
                     source_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
             else:
diff --git a/youtube_dl/extractor/sixplay.py b/youtube_dl/extractor/sixplay.py
index 759a332d2..d3aba58a2 100644
--- a/youtube_dl/extractor/sixplay.py
+++ b/youtube_dl/extractor/sixplay.py
@@ -5,6 +5,8 @@ from .common import InfoExtractor
 from ..utils import (
     qualities,
     int_or_none,
+    mimetype2ext,
+    determine_ext,
 )
 
 
@@ -34,19 +36,21 @@ class SixPlayIE(InfoExtractor):
             source_type, source_url = source.get('type'), source.get('src')
             if not source_url or source_type == 'hls/primetime':
                 continue
-            if source_type == 'application/vnd.apple.mpegURL':
+            ext = mimetype2ext(source_type) or determine_ext(source_url)
+            if ext == 'm3u8':
                 formats.extend(self._extract_m3u8_formats(
                     source_url, video_id, 'mp4', 'm3u8_native',
                     m3u8_id='hls', fatal=False))
                 formats.extend(self._extract_f4m_formats(
                     source_url.replace('.m3u8', '.f4m'),
                     video_id, f4m_id='hds', fatal=False))
-            elif source_type == 'video/mp4':
+            elif ext == 'mp4':
                 quality = source.get('quality')
                 formats.append({
                     'url': source_url,
                     'format_id': quality,
                     'quality': quality_key(quality),
+                    'ext': ext,
                 })
         self._sort_formats(formats)
 
diff --git a/youtube_dl/extractor/threeqsdn.py b/youtube_dl/extractor/threeqsdn.py
index c77a07989..a0bc12c81 100644
--- a/youtube_dl/extractor/threeqsdn.py
+++ b/youtube_dl/extractor/threeqsdn.py
@@ -92,12 +92,11 @@ class ThreeQSDNIE(InfoExtractor):
             if not item_url or item_url in urls:
                 return
             urls.add(item_url)
-            type_ = item.get('type')
-            ext = determine_ext(item_url, default_ext=None)
-            if type_ == 'application/dash+xml' or ext == 'mpd':
+            ext = mimetype2ext(item.get('type')) or determine_ext(item_url, default_ext=None)
+            if ext == 'mpd':
                 formats.extend(self._extract_mpd_formats(
                     item_url, video_id, mpd_id='mpd', fatal=False))
-            elif type_ in ('application/vnd.apple.mpegURL', 'application/x-mpegurl') or ext == 'm3u8':
+            elif ext == 'm3u8':
                 formats.extend(self._extract_m3u8_formats(
                     item_url, video_id, 'mp4',
                     entry_protocol='m3u8' if live else 'm3u8_native',
@@ -111,7 +110,7 @@ class ThreeQSDNIE(InfoExtractor):
                 formats.append({
                     'url': item_url,
                     'format_id': item.get('quality'),
-                    'ext': 'mp4' if item_url.startswith('rtsp') else mimetype2ext(type_) or ext,
+                    'ext': 'mp4' if item_url.startswith('rtsp') else ext,
                     'vcodec': 'none' if stream_type == 'audio' else None,
                 })
 

From 84c237fb8a2afa06fd3c36f7da9517682e63480e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 6 Jul 2016 20:02:52 +0800
Subject: [PATCH 5440/6219] [utils] Add get_element_by_class

For #9950
---
 test/test_utils.py  |  9 +++++++++
 youtube_dl/utils.py | 12 ++++++++++--
 2 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 85928dbc2..afd273a65 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -33,6 +33,7 @@ from youtube_dl.utils import (
     ExtractorError,
     find_xpath_attr,
     fix_xml_ampersands,
+    get_element_by_class,
     InAdvancePagedList,
     intlist_to_bytes,
     is_html,
@@ -991,5 +992,13 @@ The first line
         self.assertEqual(urshift(3, 1), 1)
         self.assertEqual(urshift(-3, 1), 2147483646)
 
+    def test_get_element_by_class(self):
+        html = '''
+            <span class="foo bar">nice</span>
+        '''
+
+        self.assertEqual(get_element_by_class('foo', html), 'nice')
+        self.assertEqual(get_element_by_class('no-such-class', html), None)
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 36d5b6c0f..3498697b6 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -310,9 +310,17 @@ def get_element_by_id(id, html):
     return get_element_by_attribute('id', id, html)
 
 
-def get_element_by_attribute(attribute, value, html):
+def get_element_by_class(class_name, html):
+    return get_element_by_attribute(
+        'class', r'[^\'"]*\b%s\b[^\'"]*' % re.escape(class_name),
+        html, escape_value=False)
+
+
+def get_element_by_attribute(attribute, value, html, escape_value=True):
     """Return the content of the tag with the specified attribute in the passed HTML document"""
 
+    value = re.escape(value) if escape_value else value
+
     m = re.search(r'''(?xs)
         <([a-zA-Z0-9:._-]+)
          (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]*|="[^"]*"|='[^']*'))*?
@@ -321,7 +329,7 @@ def get_element_by_attribute(attribute, value, html):
         \s*>
         (?P<content>.*?)
         </\1>
-    ''' % (re.escape(attribute), re.escape(value)), html)
+    ''' % (re.escape(attribute), value), html)
 
     if not m:
         return None

From f8752b86ac66d462caeb44b2ec18eda94973466d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 6 Jul 2016 20:09:05 +0800
Subject: [PATCH 5441/6219] [Onet,ClipRs] Add new extractor for onet.tv and use
 it for clip.rs

Closes #9950
---
 youtube_dl/extractor/cliprs.py     |  73 ++----------
 youtube_dl/extractor/extractors.py |   6 +-
 youtube_dl/extractor/onet.py       | 171 +++++++++++++++++++++++++++++
 3 files changed, 184 insertions(+), 66 deletions(-)
 create mode 100644 youtube_dl/extractor/onet.py

diff --git a/youtube_dl/extractor/cliprs.py b/youtube_dl/extractor/cliprs.py
index 4f9320ea5..d55b26d59 100644
--- a/youtube_dl/extractor/cliprs.py
+++ b/youtube_dl/extractor/cliprs.py
@@ -1,16 +1,10 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
-    float_or_none,
-    int_or_none,
-    parse_iso8601,
-)
+from .onet import OnetBaseIE
 
 
-class ClipRsIE(InfoExtractor):
+class ClipRsIE(OnetBaseIE):
     _VALID_URL = r'https?://(?:www\.)?clip\.rs/(?P<id>[^/]+)/\d+'
     _TEST = {
         'url': 'http://www.clip.rs/premijera-frajle-predstavljaju-novi-spot-za-pesmu-moli-me-moli/3732',
@@ -27,64 +21,13 @@ class ClipRsIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        display_id = self._match_id(url)
 
-        webpage = self._download_webpage(url, video_id)
+        webpage = self._download_webpage(url, display_id)
 
-        video_id = self._search_regex(
-            r'id=(["\'])mvp:(?P<id>.+?)\1', webpage, 'mvp id', group='id')
+        mvp_id = self._search_mvp_id(webpage)
 
-        response = self._download_json(
-            'http://qi.ckm.onetapi.pl/', video_id,
-            query={
-                'body[id]': video_id,
-                'body[jsonrpc]': '2.0',
-                'body[method]': 'get_asset_detail',
-                'body[params][ID_Publikacji]': video_id,
-                'body[params][Service]': 'www.onet.pl',
-                'content-type': 'application/jsonp',
-                'x-onet-app': 'player.front.onetapi.pl',
-            })
+        info_dict = self._extract_from_id(mvp_id, webpage)
+        info_dict['display_id'] = display_id
 
-        error = response.get('error')
-        if error:
-            raise ExtractorError(
-                '%s said: %s' % (self.IE_NAME, error['message']), expected=True)
-
-        video = response['result'].get('0')
-
-        formats = []
-        for _, formats_dict in video['formats'].items():
-            if not isinstance(formats_dict, dict):
-                continue
-            for format_id, format_list in formats_dict.items():
-                if not isinstance(format_list, list):
-                    continue
-                for f in format_list:
-                    if not f.get('url'):
-                        continue
-                    formats.append({
-                        'url': f['url'],
-                        'format_id': format_id,
-                        'height': int_or_none(f.get('vertical_resolution')),
-                        'width': int_or_none(f.get('horizontal_resolution')),
-                        'abr': float_or_none(f.get('audio_bitrate')),
-                        'vbr': float_or_none(f.get('video_bitrate')),
-                    })
-        self._sort_formats(formats)
-
-        meta = video.get('meta', {})
-
-        title = self._og_search_title(webpage, default=None) or meta['title']
-        description = self._og_search_description(webpage, default=None) or meta.get('description')
-        duration = meta.get('length') or meta.get('lenght')
-        timestamp = parse_iso8601(meta.get('addDate'), ' ')
-
-        return {
-            'id': video_id,
-            'title': title,
-            'description': description,
-            'duration': duration,
-            'timestamp': timestamp,
-            'formats': formats,
-        }
+        return info_dict
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index d21d7870a..e52faa078 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -139,9 +139,9 @@ from .chirbit import (
     ChirbitProfileIE,
 )
 from .cinchcast import CinchcastIE
-from .cliprs import ClipRsIE
 from .clipfish import ClipfishIE
 from .cliphunter import CliphunterIE
+from .cliprs import ClipRsIE
 from .clipsyndicate import ClipsyndicateIE
 from .closertotruth import CloserToTruthIE
 from .cloudy import CloudyIE
@@ -581,6 +581,10 @@ from .nytimes import (
 from .nuvid import NuvidIE
 from .odnoklassniki import OdnoklassnikiIE
 from .oktoberfesttv import OktoberfestTVIE
+from .onet import (
+    OnetIE,
+    OnetChannelIE,
+)
 from .onionstudios import OnionStudiosIE
 from .ooyala import (
     OoyalaIE,
diff --git a/youtube_dl/extractor/onet.py b/youtube_dl/extractor/onet.py
new file mode 100644
index 000000000..1e0672de8
--- /dev/null
+++ b/youtube_dl/extractor/onet.py
@@ -0,0 +1,171 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    determine_ext,
+    ExtractorError,
+    float_or_none,
+    get_element_by_class,
+    int_or_none,
+    js_to_json,
+    parse_iso8601,
+    remove_start,
+    strip_or_none,
+    url_basename,
+)
+
+
+class OnetBaseIE(InfoExtractor):
+    def _search_mvp_id(self, webpage):
+        return self._search_regex(
+            r'id=(["\'])mvp:(?P<id>.+?)\1', webpage, 'mvp id', group='id')
+
+    def _extract_from_id(self, video_id, webpage):
+        response = self._download_json(
+            'http://qi.ckm.onetapi.pl/', video_id,
+            query={
+                'body[id]': video_id,
+                'body[jsonrpc]': '2.0',
+                'body[method]': 'get_asset_detail',
+                'body[params][ID_Publikacji]': video_id,
+                'body[params][Service]': 'www.onet.pl',
+                'content-type': 'application/jsonp',
+                'x-onet-app': 'player.front.onetapi.pl',
+            })
+
+        error = response.get('error')
+        if error:
+            raise ExtractorError(
+                '%s said: %s' % (self.IE_NAME, error['message']), expected=True)
+
+        video = response['result'].get('0')
+
+        formats = []
+        for _, formats_dict in video['formats'].items():
+            if not isinstance(formats_dict, dict):
+                continue
+            for format_id, format_list in formats_dict.items():
+                if not isinstance(format_list, list):
+                    continue
+                for f in format_list:
+                    video_url = f.get('url')
+                    if not video_url:
+                        continue
+                    ext = determine_ext(video_url)
+                    if format_id == 'ism':
+                        # TODO: Support Microsoft Smooth Streaming
+                        continue
+                    elif ext == 'mpd':
+                        # TODO: Current DASH formats are broken - $Time$ pattern in
+                        # <SegmentTemplate> not implemented yet
+                        # formats.extend(self._extract_mpd_formats(
+                        #    video_url, video_id, mpd_id='dash', fatal=False))
+                        continue
+                    else:
+                        formats.append({
+                            'url': video_url,
+                            'format_id': format_id,
+                            'height': int_or_none(f.get('vertical_resolution')),
+                            'width': int_or_none(f.get('horizontal_resolution')),
+                            'abr': float_or_none(f.get('audio_bitrate')),
+                            'vbr': float_or_none(f.get('video_bitrate')),
+                        })
+        self._sort_formats(formats)
+
+        meta = video.get('meta', {})
+
+        title = self._og_search_title(webpage, default=None) or meta['title']
+        description = self._og_search_description(webpage, default=None) or meta.get('description')
+        duration = meta.get('length') or meta.get('lenght')
+        timestamp = parse_iso8601(meta.get('addDate'), ' ')
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'duration': duration,
+            'timestamp': timestamp,
+            'formats': formats,
+        }
+
+
+class OnetIE(OnetBaseIE):
+    _VALID_URL = 'https?://(?:www\.)?onet\.tv/[a-z]/[a-z]+/(?P<display_id>[0-9a-z-]+)/(?P<id>[0-9a-z]+)'
+    IE_NAME = 'onet.tv'
+
+    _TEST = {
+        'url': 'http://onet.tv/k/openerfestival/open-er-festival-2016-najdziwniejsze-wymagania-gwiazd/qbpyqc',
+        'info_dict': {
+            'id': 'qbpyqc',
+            'display_id': 'open-er-festival-2016-najdziwniejsze-wymagania-gwiazd',
+            'ext': 'mp4',
+            'title': 'Open\'er Festival 2016: najdziwniejsze wymagania gwiazd',
+            'description': 'Trzy samochody, których nigdy nie użyto, prywatne spa, hotel dekorowany czarnym suknem czy nielegalne używki. Organizatorzy koncertów i festiwali muszą stawać przed nie lada wyzwaniem zapraszając gwia...',
+            'upload_date': '20160705',
+            'timestamp': 1467721580,
+        },
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        display_id, video_id = mobj.group('display_id', 'id')
+
+        webpage = self._download_webpage(url, display_id)
+
+        mvp_id = self._search_mvp_id(webpage)
+
+        info_dict = self._extract_from_id(mvp_id, webpage)
+        info_dict.update({
+            'id': video_id,
+            'display_id': display_id,
+        })
+
+        return info_dict
+
+
+class OnetChannelIE(OnetBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?onet\.tv/[a-z]/(?P<id>[a-z]+)(?:[?#]|$)'
+    IE_NAME = 'onet.tv:channel'
+
+    _TEST = {
+        'url': 'http://onet.tv/k/openerfestival',
+        'info_dict': {
+            'id': 'openerfestival',
+            'title': 'Open\'er Festival Live',
+            'description': 'Dziękujemy, że oglądaliście transmisje. Zobaczcie nasze relacje i wywiady z artystami.',
+        },
+        'playlist_mincount': 46,
+    }
+
+    def _real_extract(self, url):
+        channel_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, channel_id)
+
+        current_clip_info = self._parse_json(self._search_regex(
+            r'var\s+currentClip\s*=\s*({[^}]+})', webpage, 'video info'), channel_id,
+            transform_source=lambda s: js_to_json(re.sub(r'\'\s*\+\s*\'', '', s)))
+        video_id = remove_start(current_clip_info['ckmId'], 'mvp:')
+        video_name = url_basename(current_clip_info['url'])
+
+        if self._downloader.params.get('noplaylist'):
+            self.to_screen(
+                'Downloading just video %s because of --no-playlist' % video_name)
+            return self._extract_from_id(video_id, webpage)
+
+        self.to_screen(
+            'Downloading channel %s - add --no-playlist to just download video %s' % (
+                channel_id, video_name))
+        matches = re.findall(
+            r'<a[^>]+href=[\'"](https?://(?:www\.)?onet\.tv/[a-z]/[a-z]+/[0-9a-z-]+/[0-9a-z]+)',
+            webpage)
+        entries = [
+            self.url_result(video_link, OnetIE.ie_key())
+            for video_link in matches]
+
+        channel_title = strip_or_none(get_element_by_class('o_channelName', webpage))
+        channel_description = strip_or_none(get_element_by_class('o_channelDesc', webpage))
+        return self.playlist_result(entries, channel_id, channel_title, channel_description)

From 54b0e909d5deb45c025c7970a129939b3f4306a9 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 6 Jul 2016 20:10:47 +0800
Subject: [PATCH 5442/6219] [amp] Fix a typo

---
 youtube_dl/extractor/amp.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/amp.py b/youtube_dl/extractor/amp.py
index 283736fc5..e8e40126b 100644
--- a/youtube_dl/extractor/amp.py
+++ b/youtube_dl/extractor/amp.py
@@ -56,7 +56,7 @@ class AMPIE(InfoExtractor):
             media_url = media.get('url')
             if not media_url:
                 continue
-            ext = mimetype2ext(media.get('type')) or determne_ext(media_url)
+            ext = mimetype2ext(media.get('type')) or determine_ext(media_url)
             if ext == 'f4m':
                 formats.extend(self._extract_f4m_formats(
                     media_url + '?hdcore=3.4.0&plugin=aasp-3.4.0.132.124',

From 43f1e4e41e1900786e7d5186d2844e9b307bc05e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 6 Jul 2016 20:32:03 +0800
Subject: [PATCH 5443/6219] [onet] Add MD5 checksum

---
 youtube_dl/extractor/onet.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/onet.py b/youtube_dl/extractor/onet.py
index 1e0672de8..402d3a9f7 100644
--- a/youtube_dl/extractor/onet.py
+++ b/youtube_dl/extractor/onet.py
@@ -98,6 +98,7 @@ class OnetIE(OnetBaseIE):
 
     _TEST = {
         'url': 'http://onet.tv/k/openerfestival/open-er-festival-2016-najdziwniejsze-wymagania-gwiazd/qbpyqc',
+        'md5': 'e3ffbf47590032ac3f27249204173d50',
         'info_dict': {
             'id': 'qbpyqc',
             'display_id': 'open-er-festival-2016-najdziwniejsze-wymagania-gwiazd',

From ad213a1d74d41e798ed065d4285ebc25135e1138 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 6 Jul 2016 23:37:54 +0800
Subject: [PATCH 5444/6219] [francetv] Recognize more Dailymotion embedded
 videos

Closes #9955
---
 youtube_dl/extractor/dailymotion.py |  7 +++++++
 youtube_dl/extractor/francetv.py    | 28 ++++++++++++++++++++++++++--
 youtube_dl/extractor/generic.py     | 12 ++++++------
 3 files changed, 39 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index 15bfc59b2..1f92823b7 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -112,6 +112,13 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
         }
     ]
 
+    @staticmethod
+    def _extract_urls(webpage):
+        # Look for embedded Dailymotion player
+        matches = re.findall(
+            r'<(?:(?:embed|iframe)[^>]+?src=|input[^>]+id=[\'"]dmcloudUrlEmissionSelect[\'"][^>]+value=)(["\'])(?P<url>(?:https?:)?//(?:www\.)?dailymotion\.com/(?:embed|swf)/video/.+?)\1', webpage)
+        return list(map(lambda m: unescapeHTML(m[1]), matches))
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index ad94e31f3..7653975e3 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -14,7 +14,10 @@ from ..utils import (
     parse_duration,
     determine_ext,
 )
-from .dailymotion import DailymotionCloudIE
+from .dailymotion import (
+    DailymotionIE,
+    DailymotionCloudIE,
+)
 
 
 class FranceTVBaseInfoExtractor(InfoExtractor):
@@ -188,6 +191,21 @@ class FranceTvInfoIE(FranceTVBaseInfoExtractor):
         'params': {
             'skip_download': True,
         },
+    }, {
+        # Dailymotion embed
+        'url': 'http://www.francetvinfo.fr/politique/notre-dame-des-landes/video-sur-france-inter-cecile-duflot-denonce-le-regard-meprisant-de-patrick-cohen_1520091.html',
+        'md5': 'ee7f1828f25a648addc90cb2687b1f12',
+        'info_dict': {
+            'id': 'x4iiko0',
+            'ext': 'mp4',
+            'title': 'NDDL, référendum, Brexit : Cécile Duflot répond à Patrick Cohen',
+            'description': 'Au lendemain de la victoire du "oui" au référendum sur l\'aéroport de Notre-Dame-des-Landes, l\'ancienne ministre écologiste est l\'invitée de Patrick Cohen. Plus d\'info : https://www.franceinter.fr/emissions/le-7-9/le-7-9-27-juin-2016',
+            'timestamp': 1467011958,
+            'upload_date': '20160627',
+            'uploader': 'France Inter',
+            'uploader_id': 'x2q2ez',
+        },
+        'add_ie': ['Dailymotion'],
     }]
 
     def _real_extract(self, url):
@@ -197,7 +215,13 @@ class FranceTvInfoIE(FranceTVBaseInfoExtractor):
 
         dmcloud_url = DailymotionCloudIE._extract_dmcloud_url(webpage)
         if dmcloud_url:
-            return self.url_result(dmcloud_url, 'DailymotionCloud')
+            return self.url_result(dmcloud_url, DailymotionCloudIE.ie_key())
+
+        dailymotion_urls = DailymotionIE._extract_urls(webpage)
+        if dailymotion_urls:
+            return self.playlist_result([
+                self.url_result(dailymotion_url, DailymotionIE.ie_key())
+                for dailymotion_url in dailymotion_urls])
 
         video_id, catalogue = self._search_regex(
             (r'id-video=([^@]+@[^"]+)',
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 764697bd2..31527d1c6 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -49,7 +49,10 @@ from .pornhub import PornHubIE
 from .xhamster import XHamsterEmbedIE
 from .tnaflix import TNAFlixNetworkEmbedIE
 from .vimeo import VimeoIE
-from .dailymotion import DailymotionCloudIE
+from .dailymotion import (
+    DailymotionIE,
+    DailymotionCloudIE,
+)
 from .onionstudios import OnionStudiosIE
 from .viewlift import ViewLiftEmbedIE
 from .screenwavemedia import ScreenwaveMediaIE
@@ -1673,12 +1676,9 @@ class GenericIE(InfoExtractor):
         if matches:
             return _playlist_from_matches(matches, lambda m: m[-1])
 
-        # Look for embedded Dailymotion player
-        matches = re.findall(
-            r'<(?:(?:embed|iframe)[^>]+?src=|input[^>]+id=[\'"]dmcloudUrlEmissionSelect[\'"][^>]+value=)(["\'])(?P<url>(?:https?:)?//(?:www\.)?dailymotion\.com/(?:embed|swf)/video/.+?)\1', webpage)
+        matches = DailymotionIE._extract_urls(webpage)
         if matches:
-            return _playlist_from_matches(
-                matches, lambda m: unescapeHTML(m[1]))
+            return _playlist_from_matches(matches)
 
         # Look for embedded Dailymotion playlist player (#3822)
         m = re.search(

From c65aa4e9e15022357ee7e5e8561e0f6b1ad30501 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 7 Jul 2016 01:13:37 +0800
Subject: [PATCH 5445/6219] [brightcove:legacy] Support 'playlistTabs' and skip
 a dead test

Closes #9965
---
 youtube_dl/extractor/brightcove.py | 26 +++++++++++++++++++++-----
 1 file changed, 21 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index b49b1977d..57ce0c174 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -90,6 +90,7 @@ class BrightcoveLegacyIE(InfoExtractor):
                 'description': 'md5:363109c02998fee92ec02211bd8000df',
                 'uploader': 'National Ballet of Canada',
             },
+            'skip': 'Video gone',
         },
         {
             # test flv videos served by akamaihd.net
@@ -108,7 +109,7 @@ class BrightcoveLegacyIE(InfoExtractor):
             },
         },
         {
-            # playlist test
+            # playlist with 'videoList'
             # from http://support.brightcove.com/en/video-cloud/docs/playlist-support-single-video-players
             'url': 'http://c.brightcove.com/services/viewer/htmlFederated?playerID=3550052898001&playerKey=AQ%7E%7E%2CAAABmA9XpXk%7E%2C-Kp7jNgisre1fG5OdqpAFUTcs0lP_ZoL',
             'info_dict': {
@@ -117,6 +118,15 @@ class BrightcoveLegacyIE(InfoExtractor):
             },
             'playlist_mincount': 7,
         },
+        {
+            # playlist with 'playlistTab' (https://github.com/rg3/youtube-dl/issues/9965)
+            'url': 'http://c.brightcove.com/services/json/experience/runtime/?command=get_programming_for_experience&playerKey=AQ%7E%7E,AAABXlLMdok%7E,NJ4EoMlZ4rZdx9eU1rkMVd8EaYPBBUlg',
+            'info_dict': {
+                'id': '1522758701001',
+                'title': 'Lesson 08',
+            },
+            'playlist_mincount': 10,
+        },
     ]
     FLV_VCODECS = {
         1: 'SORENSON',
@@ -298,13 +308,19 @@ class BrightcoveLegacyIE(InfoExtractor):
             info_url, player_key, 'Downloading playlist information')
 
         json_data = json.loads(playlist_info)
-        if 'videoList' not in json_data:
+        if 'videoList' in json_data:
+            playlist_info = json_data['videoList']
+            playlist_dto = playlist_info['mediaCollectionDTO']
+        elif 'playlistTabs' in json_data:
+            playlist_info = json_data['playlistTabs']
+            playlist_dto = playlist_info['lineupListDTO']['playlistDTOs'][0]
+        else:
             raise ExtractorError('Empty playlist')
-        playlist_info = json_data['videoList']
-        videos = [self._extract_video_info(video_info) for video_info in playlist_info['mediaCollectionDTO']['videoDTOs']]
+
+        videos = [self._extract_video_info(video_info) for video_info in playlist_dto['videoDTOs']]
 
         return self.playlist_result(videos, playlist_id='%s' % playlist_info['id'],
-                                    playlist_title=playlist_info['mediaCollectionDTO']['displayName'])
+                                    playlist_title=playlist_dto['displayName'])
 
     def _extract_video_info(self, video_info):
         video_id = compat_str(video_info['id'])

From f7291a0b7c4fb804c89bf61b5276db32846fc261 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 7 Jul 2016 01:26:14 +0800
Subject: [PATCH 5446/6219] [daum.net] Fix extraction for specific examples

Closes #9972
---
 youtube_dl/extractor/daum.py | 20 +++++++++++++++-----
 1 file changed, 15 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/daum.py b/youtube_dl/extractor/daum.py
index 86024a745..b5c310ccb 100644
--- a/youtube_dl/extractor/daum.py
+++ b/youtube_dl/extractor/daum.py
@@ -66,22 +66,32 @@ class DaumIE(InfoExtractor):
             'view_count': int,
             'comment_count': int,
         },
+    }, {
+        # Requires dte_type=WEB (#9972)
+        'url': 'http://tvpot.daum.net/v/s3794Uf1NZeZ1qMpGpeqeRU',
+        'md5': 'a8917742069a4dd442516b86e7d66529',
+        'info_dict': {
+            'id': 's3794Uf1NZeZ1qMpGpeqeRU',
+            'ext': 'mp4',
+            'title': '러블리즈 - Destiny (나의 지구) (Lovelyz - Destiny) [쇼! 음악중심] 508회 20160611',
+            'description': '러블리즈 - Destiny (나의 지구) (Lovelyz - Destiny)\n\n[쇼! 음악중심] 20160611, 507회',
+            'upload_date': '20160611',
+        },
     }]
 
     def _real_extract(self, url):
         video_id = compat_urllib_parse_unquote(self._match_id(url))
-        query = compat_urllib_parse_urlencode({'vid': video_id})
         movie_data = self._download_json(
-            'http://videofarm.daum.net/controller/api/closed/v1_2/IntegratedMovieData.json?' + query,
-            video_id, 'Downloading video formats info')
+            'http://videofarm.daum.net/controller/api/closed/v1_2/IntegratedMovieData.json',
+            video_id, 'Downloading video formats info', query={'vid': video_id, 'dte_type': 'WEB'})
 
         # For urls like http://m.tvpot.daum.net/v/65139429, where the video_id is really a clipid
         if not movie_data.get('output_list', {}).get('output_list') and re.match(r'^\d+$', video_id):
             return self.url_result('http://tvpot.daum.net/clip/ClipView.do?clipid=%s' % video_id)
 
         info = self._download_xml(
-            'http://tvpot.daum.net/clip/ClipInfoXml.do?' + query, video_id,
-            'Downloading video info')
+            'http://tvpot.daum.net/clip/ClipInfoXml.do', video_id,
+            'Downloading video info', query={'vid': video_id})
 
         formats = []
         for format_el in movie_data['output_list']['output_list']:

From 08c7af4afaa19c4af18a98fe9b4a749926cb437b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 7 Jul 2016 01:50:39 +0700
Subject: [PATCH 5447/6219] [kamcord] Add extractor (Closes #10001)

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/kamcord.py    | 71 ++++++++++++++++++++++++++++++
 2 files changed, 72 insertions(+)
 create mode 100644 youtube_dl/extractor/kamcord.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index e52faa078..12cc1b5f7 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -368,6 +368,7 @@ from .jove import JoveIE
 from .jwplatform import JWPlatformIE
 from .jpopsukitv import JpopsukiIE
 from .kaltura import KalturaIE
+from .kamcord import KamcordIE
 from .kanalplay import KanalPlayIE
 from .kankan import KankanIE
 from .karaoketv import KaraoketvIE
diff --git a/youtube_dl/extractor/kamcord.py b/youtube_dl/extractor/kamcord.py
new file mode 100644
index 000000000..b50120d98
--- /dev/null
+++ b/youtube_dl/extractor/kamcord.py
@@ -0,0 +1,71 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..compat import compat_str
+from ..utils import (
+    int_or_none,
+    qualities,
+)
+
+
+class KamcordIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?kamcord\.com/v/(?P<id>[^/?#&]+)'
+    _TEST = {
+        'url': 'https://www.kamcord.com/v/hNYRduDgWb4',
+        'md5': 'c3180e8a9cfac2e86e1b88cb8751b54c',
+        'info_dict': {
+            'id': 'hNYRduDgWb4',
+            'ext': 'mp4',
+            'title': 'Drinking Madness',
+            'uploader': 'jacksfilms',
+            'uploader_id': '3044562',
+            'view_count': int,
+            'like_count': int,
+            'comment_count': int,
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        video = self._parse_json(
+            self._search_regex(
+                r'window\.__props\s*=\s*({.+?});?(?:\n|\s*</script)',
+                webpage, 'video'),
+            video_id)['video']
+
+        title = video['title']
+
+        formats = self._extract_m3u8_formats(
+            video['play']['hls'], video_id, 'mp4', entry_protocol='m3u8_native')
+        self._sort_formats(formats)
+
+        uploader = video.get('user', {}).get('username')
+        uploader_id = video.get('user', {}).get('id')
+
+        view_count = int_or_none(video.get('viewCount'))
+        like_count = int_or_none(video.get('heartCount'))
+        comment_count = int_or_none(video.get('messageCount'))
+
+        preference_key = qualities(('small', 'medium', 'large'))
+
+        thumbnails = [{
+            'url': thumbnail_url,
+            'id': thumbnail_id,
+            'preference': preference_key(thumbnail_id),
+        } for thumbnail_id, thumbnail_url in (video.get('thumbnail') or {}).items()
+            if isinstance(thumbnail_id, compat_str) and isinstance(thumbnail_url, compat_str)]
+
+        return {
+            'id': video_id,
+            'title': title,
+            'uploader': uploader,
+            'uploader_id': uploader_id,
+            'view_count': view_count,
+            'like_count': like_count,
+            'comment_count': comment_count,
+            'thumbnails': thumbnails,
+            'formats': formats,
+        }

From 34bc2d9dfd1fcbcc303912a34ff88ceb9b7347a7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 7 Jul 2016 01:54:29 +0700
Subject: [PATCH 5448/6219] release 2016.07.07

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 docs/supportedsites.md    | 3 +++
 youtube_dl/version.py     | 2 +-
 3 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 7a79389ce..c65462ba4 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.07.06*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.07.06**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.07.07*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.07.07**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.07.06
+[debug] youtube-dl version 2016.07.07
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 3388fe221..9174c6f89 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -312,6 +312,7 @@
  - **jpopsuki.tv**
  - **JWPlatform**
  - **Kaltura**
+ - **Kamcord**
  - **KanalPlay**: Kanal 5/9/11 Play
  - **Kankan**
  - **Karaoketv**
@@ -476,6 +477,8 @@
  - **Odnoklassniki**
  - **OktoberfestTV**
  - **on.aol.com**
+ - **onet.tv**
+ - **onet.tv:channel**
  - **OnionStudios**
  - **Ooyala**
  - **OoyalaExternal**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index d1974d089..6396ad4c9 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.07.06'
+__version__ = '2016.07.07'

From 47335a0efa017a19b530f00416e364e2c46127c7 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 6 Jul 2016 23:08:44 +0100
Subject: [PATCH 5449/6219] [telecinco] fix info extraction

---
 youtube_dl/extractor/mitele.py    | 115 +++++++++++++++++-------------
 youtube_dl/extractor/telecinco.py |  85 +++++++---------------
 2 files changed, 91 insertions(+), 109 deletions(-)

diff --git a/youtube_dl/extractor/mitele.py b/youtube_dl/extractor/mitele.py
index 5a00cd397..cd169f361 100644
--- a/youtube_dl/extractor/mitele.py
+++ b/youtube_dl/extractor/mitele.py
@@ -12,12 +12,69 @@ from ..utils import (
     get_element_by_attribute,
     int_or_none,
     remove_start,
+    extract_attributes,
+    determine_ext,
 )
 
 
-class MiTeleIE(InfoExtractor):
+class MiTeleBaseIE(InfoExtractor):
+    def _get_player_info(self, url, webpage):
+        player_data = extract_attributes(self._search_regex(
+            r'(?s)(<ms-video-player.+?</ms-video-player>)',
+            webpage, 'ms video player'))
+        video_id = player_data['data-media-id']
+        config_url = compat_urlparse.urljoin(url, player_data['data-config'])
+        config = self._download_json(
+            config_url, video_id, 'Downloading config JSON')
+        mmc_url = config['services']['mmc']
+
+        duration = None
+        formats = []
+        for m_url in (mmc_url, mmc_url.replace('/flash.json', '/html5.json')):
+            mmc = self._download_json(
+                m_url, video_id, 'Downloading mmc JSON')
+            if not duration:
+                duration = int_or_none(mmc.get('duration'))
+            for location in mmc['locations']:
+                gat = self._proto_relative_url(location.get('gat'), 'http:')
+                bas = location.get('bas')
+                loc = location.get('loc')
+                ogn = location.get('ogn')
+                if None in (gat, bas, loc, ogn):
+                    continue
+                token_data = {
+                    'bas': bas,
+                    'icd': loc,
+                    'ogn': ogn,
+                    'sta': '0',
+                }
+                media = self._download_json(
+                    '%s/?%s' % (gat, compat_urllib_parse_urlencode(token_data)),
+                    video_id, 'Downloading %s JSON' % location['loc'])
+                file_ = media.get('file')
+                if not file_:
+                    continue
+                ext = determine_ext(file_)
+                if ext == 'f4m':
+                    formats.extend(self._extract_f4m_formats(
+                        file_ + '&hdcore=3.2.0&plugin=aasp-3.2.0.77.18',
+                        video_id, f4m_id='hds', fatal=False))
+                elif ext == 'm3u8':
+                    formats.extend(self._extract_m3u8_formats(
+                        file_, video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'formats': formats,
+            'thumbnail': player_data.get('data-poster') or config.get('poster', {}).get('imageUrl'),
+            'duration': duration,
+        }
+
+
+class MiTeleIE(MiTeleBaseIE):
     IE_DESC = 'mitele.es'
-    _VALID_URL = r'https?://www\.mitele\.es/[^/]+/[^/]+/[^/]+/(?P<id>[^/]+)/'
+    _VALID_URL = r'https?://www\.mitele\.es/(?:[^/]+/){3}(?P<id>[^/]+)/'
 
     _TESTS = [{
         'url': 'http://www.mitele.es/programas-tv/diario-de/la-redaccion/programa-144/',
@@ -25,7 +82,7 @@ class MiTeleIE(InfoExtractor):
         'info_dict': {
             'id': '0NF1jJnxS1Wu3pHrmvFyw2',
             'display_id': 'programa-144',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Tor, la web invisible',
             'description': 'md5:3b6fce7eaa41b2d97358726378d9369f',
             'series': 'Diario de',
@@ -40,7 +97,7 @@ class MiTeleIE(InfoExtractor):
         'info_dict': {
             'id': 'eLZSwoEd1S3pVyUm8lc6F',
             'display_id': 'programa-226',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Cuarto Milenio - Temporada 6 - Programa 226',
             'description': 'md5:50daf9fadefa4e62d9fc866d0c015701',
             'series': 'Cuarto Milenio',
@@ -59,40 +116,7 @@ class MiTeleIE(InfoExtractor):
 
         webpage = self._download_webpage(url, display_id)
 
-        config_url = self._search_regex(
-            r'data-config\s*=\s*"([^"]+)"', webpage, 'data config url')
-        config_url = compat_urlparse.urljoin(url, config_url)
-
-        config = self._download_json(
-            config_url, display_id, 'Downloading config JSON')
-
-        mmc = self._download_json(
-            config['services']['mmc'], display_id, 'Downloading mmc JSON')
-
-        formats = []
-        for location in mmc['locations']:
-            gat = self._proto_relative_url(location.get('gat'), 'http:')
-            bas = location.get('bas')
-            loc = location.get('loc')
-            ogn = location.get('ogn')
-            if None in (gat, bas, loc, ogn):
-                continue
-            token_data = {
-                'bas': bas,
-                'icd': loc,
-                'ogn': ogn,
-                'sta': '0',
-            }
-            media = self._download_json(
-                '%s/?%s' % (gat, compat_urllib_parse_urlencode(token_data)),
-                display_id, 'Downloading %s JSON' % location['loc'])
-            file_ = media.get('file')
-            if not file_:
-                continue
-            formats.extend(self._extract_f4m_formats(
-                file_ + '&hdcore=3.2.0&plugin=aasp-3.2.0.77.18',
-                display_id, f4m_id=loc))
-        self._sort_formats(formats)
+        info = self._get_player_info(url, webpage)
 
         title = self._search_regex(
             r'class="Destacado-text"[^>]*>\s*<strong>([^<]+)</strong>',
@@ -112,21 +136,12 @@ class MiTeleIE(InfoExtractor):
                 title = remove_start(self._search_regex(
                     r'<title>([^<]+)</title>', webpage, 'title'), 'Ver online ')
 
-        video_id = self._search_regex(
-            r'data-media-id\s*=\s*"([^"]+)"', webpage,
-            'data media id', default=None) or display_id
-        thumbnail = config.get('poster', {}).get('imageUrl')
-        duration = int_or_none(mmc.get('duration'))
-
-        return {
-            'id': video_id,
+        info.update({
             'display_id': display_id,
             'title': title,
             'description': get_element_by_attribute('class', 'text', webpage),
             'series': series,
             'season': season,
             'episode': episode,
-            'thumbnail': thumbnail,
-            'duration': duration,
-            'formats': formats,
-        }
+        })
+        return info
diff --git a/youtube_dl/extractor/telecinco.py b/youtube_dl/extractor/telecinco.py
index 4b4b740b4..2ecfd0405 100644
--- a/youtube_dl/extractor/telecinco.py
+++ b/youtube_dl/extractor/telecinco.py
@@ -1,50 +1,41 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import json
-
-from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse_unquote,
-    compat_urllib_parse_urlencode,
-    compat_urlparse,
-)
-from ..utils import (
-    get_element_by_attribute,
-    parse_duration,
-    strip_jsonp,
-)
+from .mitele import MiTeleBaseIE
 
 
-class TelecincoIE(InfoExtractor):
+class TelecincoIE(MiTeleBaseIE):
     IE_DESC = 'telecinco.es, cuatro.com and mediaset.es'
     _VALID_URL = r'https?://www\.(?:telecinco\.es|cuatro\.com|mediaset\.es)/(?:[^/]+/)+(?P<id>.+?)\.html'
 
     _TESTS = [{
         'url': 'http://www.telecinco.es/robinfood/temporada-01/t01xp14/Bacalao-cocochas-pil-pil_0_1876350223.html',
-        'md5': '5cbef3ad5ef17bf0d21570332d140729',
+        'md5': '8d7b2d5f699ee2709d992a63d5cd1712',
         'info_dict': {
-            'id': 'MDSVID20141015_0058',
+            'id': 'JEA5ijCnF6p5W08A1rNKn7',
             'ext': 'mp4',
-            'title': 'Con Martín Berasategui, hacer un bacalao al ...',
+            'title': 'Bacalao con kokotxas al pil-pil',
+            'description': 'md5:1382dacd32dd4592d478cbdca458e5bb',
             'duration': 662,
         },
     }, {
         'url': 'http://www.cuatro.com/deportes/futbol/barcelona/Leo_Messi-Champions-Roma_2_2052780128.html',
-        'md5': '0a5b9f3cc8b074f50a0578f823a12694',
+        'md5': '284393e5387b3b947b77c613ef04749a',
         'info_dict': {
-            'id': 'MDSVID20150916_0128',
+            'id': 'jn24Od1zGLG4XUZcnUnZB6',
             'ext': 'mp4',
-            'title': '¿Quién es este ex futbolista con el que hablan ...',
+            'title': '¿Quién es este ex futbolista con el que hablan Leo Messi y Luis Suárez?',
+            'description': 'md5:a62ecb5f1934fc787107d7b9a2262805',
             'duration': 79,
         },
     }, {
         'url': 'http://www.mediaset.es/12meses/campanas/doylacara/conlatratanohaytrato/Ayudame-dar-cara-trata-trato_2_1986630220.html',
-        'md5': 'ad1bfaaba922dd4a295724b05b68f86a',
+        'md5': '749afab6ea5a136a8806855166ae46a2',
         'info_dict': {
-            'id': 'MDSVID20150513_0220',
+            'id': 'aywerkD2Sv1vGNqq9b85Q2',
             'ext': 'mp4',
             'title': '#DOYLACARA. Con la trata no hay trato',
+            'description': 'md5:2771356ff7bfad9179c5f5cd954f1477',
             'duration': 50,
         },
     }, {
@@ -56,40 +47,16 @@ class TelecincoIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        episode = self._match_id(url)
-        webpage = self._download_webpage(url, episode)
-        embed_data_json = self._search_regex(
-            r'(?s)MSV\.embedData\[.*?\]\s*=\s*({.*?});', webpage, 'embed data',
-        ).replace('\'', '"')
-        embed_data = json.loads(embed_data_json)
-
-        domain = embed_data['mediaUrl']
-        if not domain.startswith('http'):
-            # only happens in telecinco.es videos
-            domain = 'http://' + domain
-        info_url = compat_urlparse.urljoin(
-            domain,
-            compat_urllib_parse_unquote(embed_data['flashvars']['host'])
-        )
-        info_el = self._download_xml(info_url, episode).find('./video/info')
-
-        video_link = info_el.find('videoUrl/link').text
-        token_query = compat_urllib_parse_urlencode({'id': video_link})
-        token_info = self._download_json(
-            embed_data['flashvars']['ov_tk'] + '?' + token_query,
-            episode,
-            transform_source=strip_jsonp
-        )
-        formats = self._extract_m3u8_formats(
-            token_info['tokenizedUrl'], episode, ext='mp4', entry_protocol='m3u8_native')
-        self._sort_formats(formats)
-
-        return {
-            'id': embed_data['videoId'],
-            'display_id': episode,
-            'title': info_el.find('title').text,
-            'formats': formats,
-            'description': get_element_by_attribute('class', 'text', webpage),
-            'thumbnail': info_el.find('thumb').text,
-            'duration': parse_duration(info_el.find('duration').text),
-        }
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+        title = self._html_search_meta(
+            ['og:title', 'twitter:title'], webpage, 'title')
+        info = self._get_player_info(url, webpage)
+        info.update({
+            'display_id': display_id,
+            'title': title,
+            'description': self._html_search_meta(
+                ['og:description', 'twitter:description'],
+                webpage, 'title', fatal=False),
+        })
+        return info

From 882af14d7dba75b5bcf956108e78139f0d422503 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 24 May 2016 16:06:02 +0100
Subject: [PATCH 5450/6219] [toutv] fix info extraction(closes #1792)(closes
 #2082)

---
 youtube_dl/extractor/radiocanada.py | 17 ++++---
 youtube_dl/extractor/toutv.py       | 71 ++++++++---------------------
 2 files changed, 29 insertions(+), 59 deletions(-)

diff --git a/youtube_dl/extractor/radiocanada.py b/youtube_dl/extractor/radiocanada.py
index 4f05bbddc..4a3f40ee5 100644
--- a/youtube_dl/extractor/radiocanada.py
+++ b/youtube_dl/extractor/radiocanada.py
@@ -22,13 +22,13 @@ class RadioCanadaIE(InfoExtractor):
         'url': 'http://ici.radio-canada.ca/widgets/mediaconsole/medianet/7184272',
         'info_dict': {
             'id': '7184272',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Le parcours du tireur capté sur vidéo',
             'description': 'Images des caméras de surveillance fournies par la GRC montrant le parcours du tireur d\'Ottawa',
             'upload_date': '20141023',
         },
         'params': {
-            # rtmp download
+            # m3u8 download
             'skip_download': True,
         },
     }
@@ -36,11 +36,14 @@ class RadioCanadaIE(InfoExtractor):
     def _real_extract(self, url):
         app_code, video_id = re.match(self._VALID_URL, url).groups()
 
+        device_types = ['ipad']
+        if app_code != 'toutv':
+            device_types.append('flash')
+
         formats = []
-        # TODO: extract m3u8 and f4m formats
-        # m3u8 formats can be extracted using ipad device_type return 403 error code when ffmpeg try to download segements
+        # TODO: extract f4m formats
         # f4m formats can be extracted using flashhd device_type but they produce unplayable file
-        for device_type in ('flash',):
+        for device_type in device_types:
             v_data = self._download_xml(
                 'http://api.radio-canada.ca/validationMedia/v1/Validation.ashx',
                 video_id, note='Downloading %s XML' % device_type, query={
@@ -115,13 +118,13 @@ class RadioCanadaAudioVideoIE(InfoExtractor):
         'url': 'http://ici.radio-canada.ca/audio-video/media-7527184/barack-obama-au-vietnam',
         'info_dict': {
             'id': '7527184',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Barack Obama au Vietnam',
             'description': 'Les États-Unis lèvent l\'embargo sur la vente d\'armes qui datait de la guerre du Vietnam',
             'upload_date': '20160523',
         },
         'params': {
-            # rtmp download
+            # m3u8 download
             'skip_download': True,
         },
     }
diff --git a/youtube_dl/extractor/toutv.py b/youtube_dl/extractor/toutv.py
index 4797d1310..54c2d0aa6 100644
--- a/youtube_dl/extractor/toutv.py
+++ b/youtube_dl/extractor/toutv.py
@@ -1,74 +1,41 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
-    unified_strdate,
-)
+from ..utils import int_or_none
 
 
 class TouTvIE(InfoExtractor):
     IE_NAME = 'tou.tv'
-    _VALID_URL = r'https?://www\.tou\.tv/(?P<id>[a-zA-Z0-9_-]+(?:/(?P<episode>S[0-9]+E[0-9]+)))'
+    _VALID_URL = r'https?://ici\.tou\.tv/(?P<id>[a-zA-Z0-9_-]+/S[0-9]+E[0-9]+)'
 
     _TEST = {
-        'url': 'http://www.tou.tv/30-vies/S04E41',
+        'url': 'http://ici.tou.tv/garfield-tout-court/S2015E17',
         'info_dict': {
-            'id': '30-vies_S04E41',
+            'id': '122017',
             'ext': 'mp4',
-            'title': '30 vies Saison 4 / Épisode 41',
-            'description': 'md5:da363002db82ccbe4dafeb9cab039b09',
-            'age_limit': 8,
-            'uploader': 'Groupe des Nouveaux Médias',
-            'duration': 1296,
-            'upload_date': '20131118',
-            'thumbnail': 'http://static.tou.tv/medias/images/2013-11-18_19_00_00_30VIES_0341_01_L.jpeg',
+            'title': 'Saison 2015 Épisode 17',
+            'description': 'La photo de famille 2',
+            'upload_date': '20100717',
         },
         'params': {
-            'skip_download': True,  # Requires rtmpdump
+            # m3u8 download
+            'skip_download': True,
         },
-        'skip': 'Only available in Canada'
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        webpage = self._download_webpage(url, video_id)
-
-        mediaId = self._search_regex(
-            r'"idMedia":\s*"([^"]+)"', webpage, 'media ID')
-
-        streams_url = 'http://release.theplatform.com/content.select?pid=' + mediaId
-        streams_doc = self._download_xml(
-            streams_url, video_id, note='Downloading stream list')
-
-        video_url = next(n.text
-                         for n in streams_doc.findall('.//choice/url')
-                         if '//ad.doubleclick' not in n.text)
-        if video_url.endswith('/Unavailable.flv'):
-            raise ExtractorError(
-                'Access to this video is blocked from outside of Canada',
-                expected=True)
-
-        duration_str = self._html_search_meta(
-            'video:duration', webpage, 'duration')
-        duration = int(duration_str) if duration_str else None
-        upload_date_str = self._html_search_meta(
-            'video:release_date', webpage, 'upload date')
-        upload_date = unified_strdate(upload_date_str) if upload_date_str else None
+        path = self._match_id(url)
+        metadata = self._download_json('http://ici.tou.tv/presentation/%s' % path, path)
+        video_id = metadata['IdMedia']
+        details = metadata['Details']
+        title = details['OriginalTitle']
 
         return {
+            '_type': 'url_transparent',
+            'url': 'radiocanada:%s:%s' % (metadata.get('AppCode', 'toutv'), video_id),
             'id': video_id,
-            'title': self._og_search_title(webpage),
-            'url': video_url,
-            'description': self._og_search_description(webpage),
-            'uploader': self._dc_search_uploader(webpage),
-            'thumbnail': self._og_search_thumbnail(webpage),
-            'age_limit': self._media_rating_search(webpage),
-            'duration': duration,
-            'upload_date': upload_date,
-            'ext': 'mp4',
+            'title': title,
+            'thumbnail': details.get('ImageUrl'),
+            'duration': int_or_none(details.get('LengthInSeconds')),
         }

From 01a0c511ebfa56699c1f58164c679b24f7972681 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 7 Jul 2016 03:46:12 +0100
Subject: [PATCH 5451/6219] [radiocanada] extract more formats

---
 youtube_dl/extractor/radiocanada.py | 29 ++++++++++++++++++++++-------
 1 file changed, 22 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/radiocanada.py b/youtube_dl/extractor/radiocanada.py
index 4a3f40ee5..8ec402646 100644
--- a/youtube_dl/extractor/radiocanada.py
+++ b/youtube_dl/extractor/radiocanada.py
@@ -12,6 +12,7 @@ from ..utils import (
     unified_strdate,
     xpath_element,
     ExtractorError,
+    determine_protocol,
 )
 
 
@@ -36,7 +37,7 @@ class RadioCanadaIE(InfoExtractor):
     def _real_extract(self, url):
         app_code, video_id = re.match(self._VALID_URL, url).groups()
 
-        device_types = ['ipad']
+        device_types = ['ipad', 'android']
         if app_code != 'toutv':
             device_types.append('flash')
 
@@ -55,7 +56,7 @@ class RadioCanadaIE(InfoExtractor):
                     # paysJ391wsHjbOJwvCs26toz and bypasslock are used to bypass geo-restriction
                     'paysJ391wsHjbOJwvCs26toz': 'CA',
                     'bypasslock': 'NZt5K62gRqfc',
-                })
+                }, fatal=False)
             v_url = xpath_text(v_data, 'url')
             if not v_url:
                 continue
@@ -67,7 +68,8 @@ class RadioCanadaIE(InfoExtractor):
                 formats.extend(self._extract_m3u8_formats(
                     v_url, video_id, 'mp4', m3u8_id='hls', fatal=False))
             elif ext == 'f4m':
-                formats.extend(self._extract_f4m_formats(v_url, video_id, f4m_id='hds', fatal=False))
+                formats.extend(self._extract_f4m_formats(
+                    v_url, video_id, f4m_id='hds', fatal=False))
             else:
                 ext = determine_ext(v_url)
                 bitrates = xpath_element(v_data, 'bitrates')
@@ -75,15 +77,28 @@ class RadioCanadaIE(InfoExtractor):
                     tbr = int_or_none(url_e.get('bitrate'))
                     if not tbr:
                         continue
+                    f_url = re.sub(r'\d+\.%s' % ext, '%d.%s' % (tbr, ext), v_url)
+                    protocol = determine_protocol({'url': f_url})
                     formats.append({
-                        'format_id': 'rtmp-%d' % tbr,
-                        'url': re.sub(r'\d+\.%s' % ext, '%d.%s' % (tbr, ext), v_url),
-                        'ext': 'flv',
-                        'protocol': 'rtmp',
+                        'format_id': '%s-%d' % (protocol, tbr),
+                        'url': f_url,
+                        'ext': 'flv' if protocol == 'rtmp' else ext,
+                        'protocol': protocol,
                         'width': int_or_none(url_e.get('width')),
                         'height': int_or_none(url_e.get('height')),
                         'tbr': tbr,
                     })
+                    if protocol == 'rtsp':
+                        base_url = self._search_regex(
+                            r'rtsp://([^?]+)', f_url, 'base url', default=None)
+                        if base_url:
+                            base_url = 'http://' + base_url
+                            formats.extend(self._extract_m3u8_formats(
+                                base_url + '/playlist.m3u8', video_id, 'mp4',
+                                'm3u8_native', m3u8_id='hls', fatal=False))
+                            formats.extend(self._extract_f4m_formats(
+                                base_url + '/manifest.f4m', video_id,
+                                f4m_id='hds', fatal=False))
         self._sort_formats(formats)
 
         metadata = self._download_xml(

From dfe5fa49aed02cf36ba9f743b11b0903554b5e56 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 7 Jul 2016 17:37:29 +0800
Subject: [PATCH 5452/6219] [compat] Fix compat_shlex_split for non-ASCII input

Closes #9871
---
 test/test_compat.py  | 1 +
 youtube_dl/compat.py | 8 +++++---
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/test/test_compat.py b/test/test_compat.py
index 1d7ac9f16..b57424948 100644
--- a/test/test_compat.py
+++ b/test/test_compat.py
@@ -88,6 +88,7 @@ class TestCompat(unittest.TestCase):
     def test_compat_shlex_split(self):
         self.assertEqual(compat_shlex_split('-option "one two"'), ['-option', 'one two'])
         self.assertEqual(compat_shlex_split('-option "one\ntwo" \n -flag'), ['-option', 'one\ntwo', '-flag'])
+        self.assertEqual(compat_shlex_split('-val 中文'), ['-val', '中文'])
 
     def test_compat_etree_fromstring(self):
         xml = '''
diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 67db1c7c6..83b96d38f 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -1,3 +1,4 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
 import binascii
@@ -2594,15 +2595,16 @@ except ImportError:  # Python < 3.3
             return "'" + s.replace("'", "'\"'\"'") + "'"
 
 
-if sys.version_info >= (2, 7, 3):
+try:
+    assert shlex.split('中文') == ['中文']
     compat_shlex_split = shlex.split
-else:
+except (AssertionError, UnicodeWarning, UnicodeEncodeError):
     # Working around shlex issue with unicode strings on some python 2
     # versions (see http://bugs.python.org/issue1548891)
     def compat_shlex_split(s, comments=False, posix=True):
         if isinstance(s, compat_str):
             s = s.encode('utf-8')
-        return shlex.split(s, comments, posix)
+        return list(map(lambda s: s.decode('utf-8'), shlex.split(s, comments, posix)))
 
 
 def compat_ord(c):

From 571808a7aa50fbc14c7e9db5569167bd19beac9d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 7 Jul 2016 12:12:21 +0200
Subject: [PATCH 5453/6219] document comments in configuration file (fixes
 #10024)

---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index bc214562d..44332ea9a 100644
--- a/README.md
+++ b/README.md
@@ -432,6 +432,7 @@ For example, with the following configuration file youtube-dl will always extrac
 --no-mtime
 --proxy 127.0.0.1:3128
 -o ~/Movies/%(title)s.%(ext)s
+# Lines starting with # are comments
 ```
 
 Note that options in configuration file are just the same options aka switches used in regular command line calls thus there **must be no whitespace** after `-` or `--`, e.g. `-o` or `--proxy` but not `- o` or `-- proxy`.

From 5f87d845eb87ba07d5e9fca1fb00a9bb8b7f2f2a Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 7 Jul 2016 12:49:02 +0100
Subject: [PATCH 5454/6219] [tweakers] fix info extraction(closes #9516)

---
 youtube_dl/extractor/tweakers.py | 49 ++++++++++++++++++++++++++++----
 1 file changed, 43 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/tweakers.py b/youtube_dl/extractor/tweakers.py
index f3198fb85..7a9386cde 100644
--- a/youtube_dl/extractor/tweakers.py
+++ b/youtube_dl/extractor/tweakers.py
@@ -1,25 +1,62 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    determine_ext,
+    mimetype2ext,
+)
 
 
 class TweakersIE(InfoExtractor):
     _VALID_URL = r'https?://tweakers\.net/video/(?P<id>\d+)'
     _TEST = {
         'url': 'https://tweakers.net/video/9926/new-nintendo-3ds-xl-op-alle-fronten-beter.html',
-        'md5': '3147e4ddad366f97476a93863e4557c8',
+        'md5': 'fe73e417c093a788e0160c4025f88b15',
         'info_dict': {
             'id': '9926',
             'ext': 'mp4',
             'title': 'New Nintendo 3DS XL - Op alle fronten beter',
-            'description': 'md5:f97324cc71e86e11c853f0763820e3ba',
+            'description': 'md5:3789b21fed9c0219e9bcaacd43fab280',
             'thumbnail': 're:^https?://.*\.jpe?g$',
             'duration': 386,
+            'uploader_id': 's7JeEm',
         }
     }
 
     def _real_extract(self, url):
-        playlist_id = self._match_id(url)
-        entries = self._extract_xspf_playlist(
-            'https://tweakers.net/video/s1playlist/%s/playlist.xspf' % playlist_id, playlist_id)
-        return self.playlist_result(entries, playlist_id)
+        video_id = self._match_id(url)
+        video_data = self._download_json(
+            'https://tweakers.net/video/s1playlist/%s/1920/1080/playlist.json' % video_id,
+            video_id)['items'][0]
+
+        title = video_data['title']
+
+        formats = []
+        for location in video_data.get('locations', {}).get('progressive', []):
+            format_id = location.get('label')
+            width = int_or_none(location.get('width'))
+            height = int_or_none(location.get('height'))
+            for source in location.get('sources', []):
+                source_url = source.get('src')
+                if not source_url:
+                    continue
+                ext = mimetype2ext(source.get('type')) or determine_ext(source_url)
+                formats.append({
+                    'format_id': format_id,
+                    'url': source_url,
+                    'width': width,
+                    'height': height,
+                    'ext': ext,
+                })
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': video_data.get('description'),
+            'thumbnail': video_data.get('poster'),
+            'duration': int_or_none(video_data.get('duration')),
+            'uploader_id': video_data.get('account'),
+            'formats': formats,
+        }

From ec3518725b2d33c2bd664a965ead007e000dec37 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 7 Jul 2016 22:30:58 +0800
Subject: [PATCH 5455/6219] [compat] Fix test_cmdline_umlauts on Python 2.6

The original statement raises uncaught UnicodeWarning on Python 2.6
---
 youtube_dl/compat.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 83b96d38f..b8aaf5a46 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -2596,9 +2596,12 @@ except ImportError:  # Python < 3.3
 
 
 try:
-    assert shlex.split('中文') == ['中文']
+    args = shlex.split('中文')
+    assert (isinstance(args, list) and
+            isinstance(args[0], compat_str) and
+            args[0] == '中文')
     compat_shlex_split = shlex.split
-except (AssertionError, UnicodeWarning, UnicodeEncodeError):
+except (AssertionError, UnicodeEncodeError):
     # Working around shlex issue with unicode strings on some python 2
     # versions (see http://bugs.python.org/issue1548891)
     def compat_shlex_split(s, comments=False, posix=True):

From 30d22dae8e39237244798a8c3fa93df11edd2548 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 7 Jul 2016 19:40:58 +0200
Subject: [PATCH 5456/6219] [options] Do not decode Unicode on Python 2.x

The configuration file contents are being returned as unicode now, so decoding them is no longer necessary.
(Run python2 with -3 to see the warning before this commit)
---
 youtube_dl/options.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index f2e3ad7a1..5302b67cc 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -812,11 +812,11 @@ def parseOpts(overrideArguments=None):
             system_conf = []
             user_conf = []
         else:
-            system_conf = compat_conf(_readOptions('/etc/youtube-dl.conf'))
+            system_conf = _readOptions('/etc/youtube-dl.conf')
             if '--ignore-config' in system_conf:
                 user_conf = []
             else:
-                user_conf = compat_conf(_readUserConf())
+                user_conf = _readUserConf()
         argv = system_conf + user_conf + command_line_conf
 
         opts, args = parser.parse_args(argv)

From 29c4a07776605a9fa5bbac56f49447875f56884d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 8 Jul 2016 03:33:53 +0700
Subject: [PATCH 5457/6219] [lynda] Fix test

---
 youtube_dl/extractor/lynda.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index 2d5040032..1237e1573 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -100,7 +100,7 @@ class LyndaIE(LyndaBaseIE):
 
     _TESTS = [{
         'url': 'http://www.lynda.com/Bootstrap-tutorials/Using-exercise-files/110885/114408-4.html',
-        'md5': 'ecfc6862da89489161fb9cd5f5a6fac1',
+        'md5': '679734f6786145da3546585de9a356be',
         'info_dict': {
             'id': '114408',
             'ext': 'mp4',

From 0ab7f4fe2b0006b6eaf81fcbf31c0cde7a1a14d5 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 8 Jul 2016 15:11:28 +0800
Subject: [PATCH 5458/6219] [nick] support nickjr.com (closes #7542)

---
 youtube_dl/extractor/nick.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nick.py b/youtube_dl/extractor/nick.py
index e96013791..4935002d0 100644
--- a/youtube_dl/extractor/nick.py
+++ b/youtube_dl/extractor/nick.py
@@ -8,7 +8,7 @@ from ..utils import update_url_query
 
 class NickIE(MTVServicesInfoExtractor):
     IE_NAME = 'nick.com'
-    _VALID_URL = r'https?://(?:www\.)?nick\.com/videos/clip/(?P<id>[^/?#.]+)'
+    _VALID_URL = r'https?://(?:www\.)?nick(?:jr)?\.com/(?:videos/clip|[^/]+/videos)/(?P<id>[^/?#.]+)'
     _FEED_URL = 'http://udat.mtvnservices.com/service1/dispatch.htm'
     _TESTS = [{
         'url': 'http://www.nick.com/videos/clip/alvinnn-and-the-chipmunks-112-full-episode.html',
@@ -52,6 +52,9 @@ class NickIE(MTVServicesInfoExtractor):
                 }
             },
         ],
+    }, {
+        'url': 'http://www.nickjr.com/paw-patrol/videos/pups-save-a-goldrush-s3-ep302-full-episode/',
+        'only_matching': True,
     }]
 
     def _get_feed_query(self, uri):

From e2d616dd30b8f4b4b159bc7ee80180861f3bb908 Mon Sep 17 00:00:00 2001
From: Jakub Adam Wieczorek <ja.wieczorek@student.uw.edu.pl>
Date: Thu, 7 Jul 2016 11:27:31 +0200
Subject: [PATCH 5459/6219] [polskieradio] Add thumbnails.

---
 youtube_dl/extractor/polskieradio.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/polskieradio.py b/youtube_dl/extractor/polskieradio.py
index d3bebaea3..9e7eab12e 100644
--- a/youtube_dl/extractor/polskieradio.py
+++ b/youtube_dl/extractor/polskieradio.py
@@ -33,6 +33,7 @@ class PolskieRadioIE(InfoExtractor):
                 'timestamp': 1456594200,
                 'upload_date': '20160227',
                 'duration': 2364,
+                'thumbnail': 're:^https?://static.prsa.pl/images/.*\.jpg$'
             },
         }],
     }, {
@@ -68,6 +69,8 @@ class PolskieRadioIE(InfoExtractor):
             r'(?s)<span[^>]+id="datetime2"[^>]*>(.+?)</span>',
             webpage, 'timestamp', fatal=False))
 
+        thumbnail_url = self._og_search_thumbnail(webpage)
+
         entries = []
 
         media_urls = set()
@@ -87,6 +90,7 @@ class PolskieRadioIE(InfoExtractor):
                 'duration': int_or_none(media.get('length')),
                 'vcodec': 'none' if media.get('provider') == 'audio' else None,
                 'timestamp': timestamp,
+                'thumbnail': thumbnail_url
             })
 
         title = self._og_search_title(webpage).strip()

From 2e32ac0b9a64f500fded1bd96b4d16965d4c9092 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 8 Jul 2016 19:34:53 +0800
Subject: [PATCH 5460/6219] [polskieradio] Fix regex in _TESTS

---
 youtube_dl/extractor/polskieradio.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/polskieradio.py b/youtube_dl/extractor/polskieradio.py
index 9e7eab12e..f559b899f 100644
--- a/youtube_dl/extractor/polskieradio.py
+++ b/youtube_dl/extractor/polskieradio.py
@@ -33,7 +33,7 @@ class PolskieRadioIE(InfoExtractor):
                 'timestamp': 1456594200,
                 'upload_date': '20160227',
                 'duration': 2364,
-                'thumbnail': 're:^https?://static.prsa.pl/images/.*\.jpg$'
+                'thumbnail': 're:^https?://static\.prsa\.pl/images/.*\.jpg$'
             },
         }],
     }, {

From 07d7689f2eb4698cf98ee837b56489b7ff1924df Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 8 Jul 2016 15:35:20 +0100
Subject: [PATCH 5461/6219] [le] extract http formats

---
 youtube_dl/extractor/leeco.py | 135 ++++++++++++++++++++++------------
 1 file changed, 90 insertions(+), 45 deletions(-)

diff --git a/youtube_dl/extractor/leeco.py b/youtube_dl/extractor/leeco.py
index 171b705c7..e9cc9aa59 100644
--- a/youtube_dl/extractor/leeco.py
+++ b/youtube_dl/extractor/leeco.py
@@ -23,6 +23,7 @@ from ..utils import (
     str_or_none,
     url_basename,
     urshift,
+    update_url_query,
 )
 
 
@@ -89,6 +90,10 @@ class LeIE(InfoExtractor):
         _loc3_ = self.ror(_loc3_, _loc2_ % 17)
         return _loc3_
 
+    # reversed from http://jstatic.letvcdn.com/sdk/player.js
+    def get_mms_key(self, time):
+        return self.ror(time, 8) ^ 185025305
+
     # see M3U8Encryption class in KLetvPlayer.swf
     @staticmethod
     def decrypt_m3u8(encrypted_data):
@@ -109,23 +114,7 @@ class LeIE(InfoExtractor):
 
         return bytes(_loc7_)
 
-    def _real_extract(self, url):
-        media_id = self._match_id(url)
-        page = self._download_webpage(url, media_id)
-        params = {
-            'id': media_id,
-            'platid': 1,
-            'splatid': 101,
-            'format': 1,
-            'tkey': self.calc_time_key(int(time.time())),
-            'domain': 'www.le.com'
-        }
-
-        play_json = self._download_json(
-            'http://api.le.com/mms/out/video/playJson',
-            media_id, 'Downloading playJson data', query=params,
-            headers=self.geo_verification_headers())
-
+    def _check_errors(self, play_json):
         # Check for errors
         playstatus = play_json['playstatus']
         if playstatus['status'] == 0:
@@ -136,43 +125,99 @@ class LeIE(InfoExtractor):
                 msg = 'Generic error. flag = %d' % flag
             raise ExtractorError(msg, expected=True)
 
-        playurl = play_json['playurl']
+    def _real_extract(self, url):
+        media_id = self._match_id(url)
+        page = self._download_webpage(url, media_id)
 
-        formats = ['350', '1000', '1300', '720p', '1080p']
-        dispatch = playurl['dispatch']
+        play_json_h5 = self._download_json(
+            'http://api.le.com/mms/out/video/playJsonH5',
+            media_id, 'Downloading html5 playJson data', query={
+                'id': media_id,
+                'platid': 3,
+                'splatid': 304,
+                'format': 1,
+                'tkey': self.get_mms_key(int(time.time())),
+                'domain': 'www.le.com',
+                'tss': 'no',
+            },
+            headers=self.geo_verification_headers())
+        self._check_errors(play_json_h5)
 
-        urls = []
-        for format_id in formats:
-            if format_id in dispatch:
-                media_url = playurl['domain'][0] + dispatch[format_id][0]
-                media_url += '&' + compat_urllib_parse_urlencode({
-                    'm3v': 1,
+        play_json_flash = self._download_json(
+            'http://api.le.com/mms/out/video/playJson',
+            media_id, 'Downloading flash playJson data', query={
+                'id': media_id,
+                'platid': 1,
+                'splatid': 101,
+                'format': 1,
+                'tkey': self.calc_time_key(int(time.time())),
+                'domain': 'www.le.com',
+            },
+            headers=self.geo_verification_headers())
+        self._check_errors(play_json_flash)
+
+        def get_h5_urls(media_url, format_id):
+            location = self._download_json(
+                media_url, media_id,
+                'Download JSON metadata for format %s' % format_id, query={
                     'format': 1,
                     'expect': 3,
-                    'rateid': format_id,
-                })
+                    'tss': 'no',
+                })['location']
 
-                nodes_data = self._download_json(
-                    media_url, media_id,
-                    'Download JSON metadata for format %s' % format_id)
+            return {
+                'http': update_url_query(location, {'tss': 'no'}),
+                'hls': update_url_query(location, {'tss': 'ios'}),
+            }
 
-                req = self._request_webpage(
-                    nodes_data['nodelist'][0]['location'], media_id,
-                    note='Downloading m3u8 information for format %s' % format_id)
+        def get_flash_urls(media_url, format_id):
+            media_url += '&' + compat_urllib_parse_urlencode({
+                'm3v': 1,
+                'format': 1,
+                'expect': 3,
+                'rateid': format_id,
+            })
 
-                m3u8_data = self.decrypt_m3u8(req.read())
+            nodes_data = self._download_json(
+                media_url, media_id,
+                'Download JSON metadata for format %s' % format_id)
 
-                url_info_dict = {
-                    'url': encode_data_uri(m3u8_data, 'application/vnd.apple.mpegurl'),
-                    'ext': determine_ext(dispatch[format_id][1]),
-                    'format_id': format_id,
-                    'protocol': 'm3u8',
-                }
+            req = self._request_webpage(
+                nodes_data['nodelist'][0]['location'], media_id,
+                note='Downloading m3u8 information for format %s' % format_id)
 
-                if format_id[-1:] == 'p':
-                    url_info_dict['height'] = int_or_none(format_id[:-1])
+            m3u8_data = self.decrypt_m3u8(req.read())
 
-                urls.append(url_info_dict)
+            return {
+                'hls': encode_data_uri(m3u8_data, 'application/vnd.apple.mpegurl'),
+            }
+
+        extracted_formats = []
+        formats = []
+        for play_json, get_urls in ((play_json_h5, get_h5_urls), (play_json_flash, get_flash_urls)):
+            playurl = play_json['playurl']
+            play_domain = playurl['domain'][0]
+
+            for format_id, format_data in playurl.get('dispatch', []).items():
+                if format_id in extracted_formats:
+                    continue
+                extracted_formats.append(format_id)
+
+                media_url = play_domain + format_data[0]
+                for protocol, format_url in get_urls(media_url, format_id).items():
+                    f = {
+                        'url': format_url,
+                        'ext': determine_ext(format_data[1]),
+                        'format_id': '%s-%s' % (protocol, format_id),
+                        'protocol': 'm3u8_native' if protocol == 'hls' else 'http',
+                        'quality': int_or_none(format_id),
+                    }
+
+                    if format_id[-1:] == 'p':
+                        f['height'] = int_or_none(format_id[:-1])
+
+                    formats.append(f)
+        self._sort_formats(formats, ('height', 'quality', 'format_id'))
 
         publish_time = parse_iso8601(self._html_search_regex(
             r'发布时间&nbsp;([^<>]+) ', page, 'publish time', default=None),
@@ -181,7 +226,7 @@ class LeIE(InfoExtractor):
 
         return {
             'id': media_id,
-            'formats': urls,
+            'formats': formats,
             'title': playurl['title'],
             'thumbnail': playurl['pic'],
             'description': description,

From cedc70b29277c7ceebee4843bc2f13ebe88109b0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 9 Jul 2016 00:28:07 +0700
Subject: [PATCH 5462/6219] [facebook] Fix invalid video being extracted
 (Closes #9851)

---
 youtube_dl/extractor/facebook.py | 19 +++++++++++++++----
 1 file changed, 15 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index f5d4f966a..0d43acc4a 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -219,12 +219,23 @@ class FacebookIE(InfoExtractor):
 
         BEFORE = '{swf.addParam(param[0], param[1]);});'
         AFTER = '.forEach(function(variable) {swf.addVariable(variable[0], variable[1]);});'
-        m = re.search(re.escape(BEFORE) + '(?:\n|\\\\n)(.*?)' + re.escape(AFTER), webpage)
-        if m:
-            swf_params = m.group(1).replace('\\\\', '\\').replace('\\"', '"')
+        PATTERN = re.escape(BEFORE) + '(?:\n|\\\\n)(.*?)' + re.escape(AFTER)
+
+        for m in re.findall(PATTERN, webpage):
+            swf_params = m.replace('\\\\', '\\').replace('\\"', '"')
             data = dict(json.loads(swf_params))
             params_raw = compat_urllib_parse_unquote(data['params'])
-            video_data = json.loads(params_raw)['video_data']
+            video_data_candidate = json.loads(params_raw)['video_data']
+            for _, f in video_data_candidate.items():
+                if not f:
+                    continue
+                if isinstance(f, dict):
+                    f = [f]
+                if isinstance(f, list):
+                    continue
+                if f[0].get('video_id') == video_id:
+                    video_data = video_data_candidate
+                    break
 
         def video_data_list2dict(video_data):
             ret = {}

From 89e2fff2b7e4aeda92cfc811d395b68d18f2d85d Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 8 Jul 2016 20:15:21 +0100
Subject: [PATCH 5463/6219] [mgtv] pass geo verification headers for api
 request

---
 youtube_dl/extractor/mgtv.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mgtv.py b/youtube_dl/extractor/mgtv.py
index 9fbc74f5d..d970e94ec 100644
--- a/youtube_dl/extractor/mgtv.py
+++ b/youtube_dl/extractor/mgtv.py
@@ -26,7 +26,8 @@ class MGTVIE(InfoExtractor):
         video_id = self._match_id(url)
         api_data = self._download_json(
             'http://v.api.mgtv.com/player/video', video_id,
-            query={'video_id': video_id})['data']
+            query={'video_id': video_id},
+            headers=self.geo_verification_headers())['data']
         info = api_data['info']
 
         formats = []

From 3fee7f636cb63e6cb785b07e4ffea3cc91981525 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?D=C3=A9stin=20Reed?= <trox1972@users.noreply.github.com>
Date: Sat, 25 Jun 2016 19:01:47 +0200
Subject: [PATCH 5464/6219] [flipagram] Add extractor

---
 youtube_dl/extractor/common.py     |   7 ++
 youtube_dl/extractor/extractors.py |   1 +
 youtube_dl/extractor/flipagram.py  | 103 +++++++++++++++++++++++++++++
 3 files changed, 111 insertions(+)
 create mode 100644 youtube_dl/extractor/flipagram.py

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index be2b6ff66..b5fce5de2 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -837,6 +837,13 @@ class InfoExtractor(object):
                     'title': unescapeHTML(json_ld.get('headline')),
                     'description': unescapeHTML(json_ld.get('articleBody')),
                 })
+            elif item_type == 'VideoObject':
+                info.update({
+                    'title': unescapeHTML(json_ld.get('name')),
+                    'description': unescapeHTML(json_ld.get('description')),
+                    'upload_date': unified_strdate(json_ld.get('upload_date')),
+                    'url': unescapeHTML(json_ld.get('contentUrl')),
+                })
         return dict((k, v) for k, v in info.items() if v is not None)
 
     @staticmethod
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 12cc1b5f7..9f70ce752 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -256,6 +256,7 @@ from .fivemin import FiveMinIE
 from .fivetv import FiveTVIE
 from .fktv import FKTVIE
 from .flickr import FlickrIE
+from .flipagram import FlipagramIE
 from .folketinget import FolketingetIE
 from .footyroom import FootyRoomIE
 from .formula1 import Formula1IE
diff --git a/youtube_dl/extractor/flipagram.py b/youtube_dl/extractor/flipagram.py
new file mode 100644
index 000000000..7143126b5
--- /dev/null
+++ b/youtube_dl/extractor/flipagram.py
@@ -0,0 +1,103 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+from ..utils import (
+    int_or_none,
+    parse_iso8601,
+    unified_strdate,
+    unified_timestamp,
+)
+
+
+class FlipagramIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?flipagram\.com/f/(?P<id>[^/?_]+)'
+    _TESTS = [{
+        'url': 'https://flipagram.com/f/myrWjW9RJw',
+        'md5': '541988fb6c4c7c375215ea22a4a21841',
+        'info_dict': {
+            'id': 'myrWjW9RJw',
+            'title': 'Flipagram by crystaldolce featuring King and Lionheart by Of Monsters and Men',
+            'description': 'Herbie\'s first bannana🍌🐢🍌.  #animals #pets #reptile #tortoise #sulcata #tort #justatreat #snacktime #bannanas #rescuepets  #ofmonstersandmen  @animals',
+            'ext': 'mp4',
+            'uploader': 'Crystal Dolce',
+            'creator': 'Crystal Dolce',
+            'uploader_id': 'crystaldolce',
+        }
+    }, {
+        'url': 'https://flipagram.com/f/nyvTSJMKId',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        self.report_extraction(video_id)
+        user_data = self._parse_json(self._search_regex(r'window.reactH2O\s*=\s*({.+});', webpage, 'user data'), video_id)
+        content_data = self._search_json_ld(webpage, video_id)
+
+        flipagram = user_data.get('flipagram', {})
+        counts = flipagram.get('counts', {})
+        user = flipagram.get('user', {})
+        video = flipagram.get('video', {})
+
+        thumbnails = []
+        for cover in flipagram.get('covers', []):
+            if not cover.get('url'):
+                continue
+            thumbnails.append({
+                'url': self._proto_relative_url(cover.get('url')),
+                'width': int_or_none(cover.get('width')),
+                'height': int_or_none(cover.get('height')),
+            })
+
+        # Note that this only retrieves comments that are initally loaded.
+        # For videos with large amounts of comments, most won't be retrieved.
+        comments = []
+        for comment in user_data.get('comments', {}).get(video_id, {}).get('items', []):
+            text = comment.get('comment', [])
+            comments.append({
+                'author': comment.get('user', {}).get('name'),
+                'author_id': comment.get('user', {}).get('username'),
+                'id': comment.get('id'),
+                'text': text[0] if text else '',
+                'timestamp': unified_timestamp(comment.get('created', '')),
+            })
+
+        tags = [tag for item in flipagram['story'][1:] for tag in item]
+
+        formats = []
+        if flipagram.get('music', {}).get('track', {}).get('previewUrl', {}):
+            formats.append({
+                'url': flipagram.get('music').get('track').get('previewUrl'),
+                'ext': 'm4a',
+                'vcodec': 'none',
+            })
+
+        formats.append({
+            'url': video.get('url'),
+            'ext': 'mp4',
+            'width': int_or_none(video.get('width')),
+            'height': int_or_none(video.get('height')),
+            'filesize': int_or_none(video.get('size')),
+        })
+
+        return {
+            'id': video_id,
+            'title': content_data['title'],
+            'formats': formats,
+            'thumbnails': thumbnails,
+            'description': content_data.get('description'),
+            'uploader': user.get('name'),
+            'creator': user.get('name'),
+            'timestamp': parse_iso8601(flipagram.get('iso801Created')),
+            'upload_date': unified_strdate(flipagram.get('created')),
+            'uploader_id': user.get('username'),
+            'view_count': int_or_none(counts.get('plays')),
+            'repost_count': int_or_none(counts.get('reflips')),
+            'comment_count': int_or_none(counts.get('comments')),
+            'comments': comments,
+            'tags': tags,
+        }

From 2de624fdd5b2d94bcf548633d6fe1897ccb7cf46 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 9 Jul 2016 03:24:36 +0700
Subject: [PATCH 5465/6219] [extractor/common] Introduce filesize metafield for
 thumbnails

---
 youtube_dl/extractor/common.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index b5fce5de2..0cb5e5bb0 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -161,6 +161,7 @@ class InfoExtractor(object):
                         * "height" (optional, int)
                         * "resolution" (optional, string "{width}x{height"},
                                         deprecated)
+                        * "filesize" (optional, int)
     thumbnail:      Full URL to a video thumbnail image.
     description:    Full video description.
     uploader:       Full name of the video uploader.

From 6b3a3098b53519ba8052ad0572e1d559947cfdd8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 9 Jul 2016 03:27:11 +0700
Subject: [PATCH 5466/6219] [extractor/common] Extract more metadata for
 VideoObject in _json_ld

---
 youtube_dl/extractor/common.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 0cb5e5bb0..9ffe64d05 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -44,6 +44,7 @@ from ..utils import (
     sanitized_Request,
     unescapeHTML,
     unified_strdate,
+    unified_timestamp,
     url_basename,
     xpath_element,
     xpath_text,
@@ -840,10 +841,16 @@ class InfoExtractor(object):
                 })
             elif item_type == 'VideoObject':
                 info.update({
+                    'url': json_ld.get('contentUrl'),
                     'title': unescapeHTML(json_ld.get('name')),
                     'description': unescapeHTML(json_ld.get('description')),
-                    'upload_date': unified_strdate(json_ld.get('upload_date')),
-                    'url': unescapeHTML(json_ld.get('contentUrl')),
+                    'thumbnail': json_ld.get('thumbnailUrl'),
+                    'duration': parse_duration(json_ld.get('duration')),
+                    'timestamp': unified_timestamp(json_ld.get('uploadDate')),
+                    'filesize': float_or_none(json_ld.get('contentSize')),
+                    'tbr': int_or_none(json_ld.get('bitrate')),
+                    'width': int_or_none(json_ld.get('width')),
+                    'height': int_or_none(json_ld.get('height')),
                 })
         return dict((k, v) for k, v in info.items() if v is not None)
 

From 95b31e266b930dc753b8bf5a1673ced9b50fd519 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 9 Jul 2016 03:28:04 +0700
Subject: [PATCH 5467/6219] [extractor/common] Add expected_type in json ld
 routines

---
 youtube_dl/extractor/common.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 9ffe64d05..816baa424 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -805,15 +805,17 @@ class InfoExtractor(object):
         return self._html_search_meta('twitter:player', html,
                                       'twitter card player')
 
-    def _search_json_ld(self, html, video_id, **kwargs):
+    def _search_json_ld(self, html, video_id, expected_type=None, **kwargs):
         json_ld = self._search_regex(
             r'(?s)<script[^>]+type=(["\'])application/ld\+json\1[^>]*>(?P<json_ld>.+?)</script>',
             html, 'JSON-LD', group='json_ld', **kwargs)
         if not json_ld:
             return {}
-        return self._json_ld(json_ld, video_id, fatal=kwargs.get('fatal', True))
+        return self._json_ld(
+            json_ld, video_id, fatal=kwargs.get('fatal', True),
+            expected_type=expected_type)
 
-    def _json_ld(self, json_ld, video_id, fatal=True):
+    def _json_ld(self, json_ld, video_id, fatal=True, expected_type=None):
         if isinstance(json_ld, compat_str):
             json_ld = self._parse_json(json_ld, video_id, fatal=fatal)
         if not json_ld:
@@ -821,6 +823,8 @@ class InfoExtractor(object):
         info = {}
         if json_ld.get('@context') == 'http://schema.org':
             item_type = json_ld.get('@type')
+            if expected_type is not None and expected_type != item_type:
+                return info
             if item_type == 'TVEpisode':
                 info.update({
                     'episode': unescapeHTML(json_ld.get('name')),

From 0de168f7ed2da440f6a1bcb614abd26ff73bb840 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 9 Jul 2016 03:29:07 +0700
Subject: [PATCH 5468/6219] [extractor/generic] Detect schema.org/VideoObject
 embeds

---
 youtube_dl/extractor/generic.py | 30 ++++++++++++++++++++++++++++++
 1 file changed, 30 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 31527d1c6..62da9bbc0 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1313,6 +1313,23 @@ class GenericIE(InfoExtractor):
             },
             'add_ie': ['Kaltura'],
         },
+        {
+            # TODO: find another test
+            # http://schema.org/VideoObject
+            # 'url': 'https://flipagram.com/f/nyvTSJMKId',
+            # 'md5': '888dcf08b7ea671381f00fab74692755',
+            # 'info_dict': {
+            #     'id': 'nyvTSJMKId',
+            #     'ext': 'mp4',
+            #     'title': 'Flipagram by sjuria101 featuring Midnight Memories by One Direction',
+            #     'description': '#love for cats.',
+            #     'timestamp': 1461244995,
+            #     'upload_date': '20160421',
+            # },
+            # 'params': {
+            #     'force_generic_extractor': True,
+            # },
+        }
     ]
 
     def report_following_redirect(self, new_url):
@@ -2157,6 +2174,19 @@ class GenericIE(InfoExtractor):
         if embed_url:
             return self.url_result(embed_url)
 
+        # Looking for http://schema.org/VideoObject
+        json_ld = self._search_json_ld(
+            webpage, video_id, default=None, expected_type='VideoObject')
+        if json_ld and json_ld.get('url'):
+            info_dict.update({
+                'title': video_title or info_dict['title'],
+                'description': video_description,
+                'thumbnail': video_thumbnail,
+                'age_limit': age_limit
+            })
+            info_dict.update(json_ld)
+            return info_dict
+
         def check_video(vurl):
             if YoutubeIE.suitable(vurl):
                 return True

From 0af985069b64ca7cdf0a55b721417e807760ab1e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 9 Jul 2016 03:31:17 +0700
Subject: [PATCH 5469/6219] [flipagram] Improve extraction (Closes #9898)

---
 youtube_dl/extractor/flipagram.py | 144 ++++++++++++++++--------------
 1 file changed, 78 insertions(+), 66 deletions(-)

diff --git a/youtube_dl/extractor/flipagram.py b/youtube_dl/extractor/flipagram.py
index 7143126b5..acb6133ff 100644
--- a/youtube_dl/extractor/flipagram.py
+++ b/youtube_dl/extractor/flipagram.py
@@ -2,102 +2,114 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-
+from ..compat import compat_str
 from ..utils import (
     int_or_none,
-    parse_iso8601,
-    unified_strdate,
+    float_or_none,
+    try_get,
     unified_timestamp,
 )
 
 
 class FlipagramIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?flipagram\.com/f/(?P<id>[^/?_]+)'
-    _TESTS = [{
-        'url': 'https://flipagram.com/f/myrWjW9RJw',
-        'md5': '541988fb6c4c7c375215ea22a4a21841',
-        'info_dict': {
-            'id': 'myrWjW9RJw',
-            'title': 'Flipagram by crystaldolce featuring King and Lionheart by Of Monsters and Men',
-            'description': 'Herbie\'s first bannana🍌🐢🍌.  #animals #pets #reptile #tortoise #sulcata #tort #justatreat #snacktime #bannanas #rescuepets  #ofmonstersandmen  @animals',
-            'ext': 'mp4',
-            'uploader': 'Crystal Dolce',
-            'creator': 'Crystal Dolce',
-            'uploader_id': 'crystaldolce',
-        }
-    }, {
+    _VALID_URL = r'https?://(?:www\.)?flipagram\.com/f/(?P<id>[^/?#&]+)'
+    _TEST = {
         'url': 'https://flipagram.com/f/nyvTSJMKId',
-        'only_matching': True,
-    }]
+        'md5': '888dcf08b7ea671381f00fab74692755',
+        'info_dict': {
+            'id': 'nyvTSJMKId',
+            'ext': 'mp4',
+            'title': 'Flipagram by sjuria101 featuring Midnight Memories by One Direction',
+            'description': 'md5:d55e32edc55261cae96a41fa85ff630e',
+            'duration': 35.571,
+            'timestamp': 1461244995,
+            'upload_date': '20160421',
+            'uploader': 'kitty juria',
+            'uploader_id': 'sjuria101',
+            'creator': 'kitty juria',
+            'view_count': int,
+            'like_count': int,
+            'repost_count': int,
+            'comment_count': int,
+            'comments': list,
+            'formats': 'mincount:2',
+        },
+    }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        self.report_extraction(video_id)
-        user_data = self._parse_json(self._search_regex(r'window.reactH2O\s*=\s*({.+});', webpage, 'user data'), video_id)
-        content_data = self._search_json_ld(webpage, video_id)
+        video_data = self._parse_json(
+            self._search_regex(
+                r'window\.reactH2O\s*=\s*({.+});', webpage, 'video data'),
+            video_id)
 
-        flipagram = user_data.get('flipagram', {})
-        counts = flipagram.get('counts', {})
-        user = flipagram.get('user', {})
-        video = flipagram.get('video', {})
+        flipagram = video_data['flipagram']
+        video = flipagram['video']
 
-        thumbnails = []
-        for cover in flipagram.get('covers', []):
-            if not cover.get('url'):
-                continue
-            thumbnails.append({
-                'url': self._proto_relative_url(cover.get('url')),
-                'width': int_or_none(cover.get('width')),
-                'height': int_or_none(cover.get('height')),
-            })
+        json_ld = self._search_json_ld(webpage, video_id, default=False)
+        title = json_ld.get('title') or flipagram['captionText']
+        description = json_ld.get('description') or flipagram.get('captionText')
 
-        # Note that this only retrieves comments that are initally loaded.
-        # For videos with large amounts of comments, most won't be retrieved.
-        comments = []
-        for comment in user_data.get('comments', {}).get(video_id, {}).get('items', []):
-            text = comment.get('comment', [])
-            comments.append({
-                'author': comment.get('user', {}).get('name'),
-                'author_id': comment.get('user', {}).get('username'),
-                'id': comment.get('id'),
-                'text': text[0] if text else '',
-                'timestamp': unified_timestamp(comment.get('created', '')),
-            })
+        formats = [{
+            'url': video['url'],
+            'width': int_or_none(video.get('width')),
+            'height': int_or_none(video.get('height')),
+            'filesize': int_or_none(video_data.get('size')),
+        }]
 
-        tags = [tag for item in flipagram['story'][1:] for tag in item]
-
-        formats = []
-        if flipagram.get('music', {}).get('track', {}).get('previewUrl', {}):
+        preview_url = try_get(
+            flipagram, lambda x: x['music']['track']['previewUrl'], compat_str)
+        if preview_url:
             formats.append({
-                'url': flipagram.get('music').get('track').get('previewUrl'),
+                'url': preview_url,
                 'ext': 'm4a',
                 'vcodec': 'none',
             })
 
-        formats.append({
-            'url': video.get('url'),
-            'ext': 'mp4',
-            'width': int_or_none(video.get('width')),
-            'height': int_or_none(video.get('height')),
-            'filesize': int_or_none(video.get('size')),
-        })
+        self._sort_formats(formats)
+
+        counts = flipagram.get('counts', {})
+        user = flipagram.get('user', {})
+        video_data = flipagram.get('video', {})
+
+        thumbnails = [{
+            'url': self._proto_relative_url(cover['url']),
+            'width': int_or_none(cover.get('width')),
+            'height': int_or_none(cover.get('height')),
+            'filesize': int_or_none(cover.get('size')),
+        } for cover in flipagram.get('covers', []) if cover.get('url')]
+
+        # Note that this only retrieves comments that are initally loaded.
+        # For videos with large amounts of comments, most won't be retrieved.
+        comments = []
+        for comment in video_data.get('comments', {}).get(video_id, {}).get('items', []):
+            text = comment.get('comment')
+            if not text or not isinstance(text, list):
+                continue
+            comments.append({
+                'author': comment.get('user', {}).get('name'),
+                'author_id': comment.get('user', {}).get('username'),
+                'id': comment.get('id'),
+                'text': text[0],
+                'timestamp': unified_timestamp(comment.get('created')),
+            })
 
         return {
             'id': video_id,
-            'title': content_data['title'],
-            'formats': formats,
+            'title': title,
+            'description': description,
+            'duration': float_or_none(flipagram.get('duration'), 1000),
             'thumbnails': thumbnails,
-            'description': content_data.get('description'),
+            'timestamp': unified_timestamp(flipagram.get('iso8601Created')),
             'uploader': user.get('name'),
-            'creator': user.get('name'),
-            'timestamp': parse_iso8601(flipagram.get('iso801Created')),
-            'upload_date': unified_strdate(flipagram.get('created')),
             'uploader_id': user.get('username'),
+            'creator': user.get('name'),
             'view_count': int_or_none(counts.get('plays')),
+            'like_count': int_or_none(counts.get('likes')),
             'repost_count': int_or_none(counts.get('reflips')),
             'comment_count': int_or_none(counts.get('comments')),
             'comments': comments,
-            'tags': tags,
+            'formats': formats,
         }

From 1f7258a367bfcc0d60441e38d25373a6ada6b45e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 9 Jul 2016 06:57:04 +0700
Subject: [PATCH 5470/6219] [animeondemand] Add support for full length films
 (Closes #10031)

---
 youtube_dl/extractor/animeondemand.py | 108 ++++++++++++++++----------
 1 file changed, 68 insertions(+), 40 deletions(-)

diff --git a/youtube_dl/extractor/animeondemand.py b/youtube_dl/extractor/animeondemand.py
index 9b01e38f5..1f044e20b 100644
--- a/youtube_dl/extractor/animeondemand.py
+++ b/youtube_dl/extractor/animeondemand.py
@@ -22,6 +22,7 @@ class AnimeOnDemandIE(InfoExtractor):
     _APPLY_HTML5_URL = 'https://www.anime-on-demand.de/html5apply'
     _NETRC_MACHINE = 'animeondemand'
     _TESTS = [{
+        # jap, OmU
         'url': 'https://www.anime-on-demand.de/anime/161',
         'info_dict': {
             'id': '161',
@@ -30,17 +31,21 @@ class AnimeOnDemandIE(InfoExtractor):
         },
         'playlist_mincount': 4,
     }, {
-        # Film wording is used instead of Episode
+        # Film wording is used instead of Episode, ger/jap, Dub/OmU
         'url': 'https://www.anime-on-demand.de/anime/39',
         'only_matching': True,
     }, {
-        # Episodes without titles
+        # Episodes without titles, jap, OmU
         'url': 'https://www.anime-on-demand.de/anime/162',
         'only_matching': True,
     }, {
         # ger/jap, Dub/OmU, account required
         'url': 'https://www.anime-on-demand.de/anime/169',
         'only_matching': True,
+    }, {
+        # Full length film, non-series, ger/jap, Dub/OmU, account required
+        'url': 'https://www.anime-on-demand.de/anime/185',
+        'only_matching': True,
     }]
 
     def _login(self):
@@ -110,35 +115,12 @@ class AnimeOnDemandIE(InfoExtractor):
 
         entries = []
 
-        for num, episode_html in enumerate(re.findall(
-                r'(?s)<h3[^>]+class="episodebox-title".+?>Episodeninhalt<', webpage), 1):
-            episodebox_title = self._search_regex(
-                (r'class="episodebox-title"[^>]+title=(["\'])(?P<title>.+?)\1',
-                 r'class="episodebox-title"[^>]+>(?P<title>.+?)<'),
-                episode_html, 'episodebox title', default=None, group='title')
-            if not episodebox_title:
-                continue
-
-            episode_number = int(self._search_regex(
-                r'(?:Episode|Film)\s*(\d+)',
-                episodebox_title, 'episode number', default=num))
-            episode_title = self._search_regex(
-                r'(?:Episode|Film)\s*\d+\s*-\s*(.+)',
-                episodebox_title, 'episode title', default=None)
-
-            video_id = 'episode-%d' % episode_number
-
-            common_info = {
-                'id': video_id,
-                'series': anime_title,
-                'episode': episode_title,
-                'episode_number': episode_number,
-            }
-
+        def extract_info(html, video_id):
+            title, description = [None] * 2
             formats = []
 
             for input_ in re.findall(
-                    r'<input[^>]+class=["\'].*?streamstarter_html5[^>]+>', episode_html):
+                    r'<input[^>]+class=["\'].*?streamstarter_html5[^>]+>', html):
                 attributes = extract_attributes(input_)
                 playlist_urls = []
                 for playlist_key in ('data-playlist', 'data-otherplaylist'):
@@ -215,28 +197,74 @@ class AnimeOnDemandIE(InfoExtractor):
                             })
                         formats.extend(file_formats)
 
-            if formats:
-                self._sort_formats(formats)
+            return {
+                'title': title,
+                'description': description,
+                'formats': formats,
+            }
+
+        def extract_entries(html, video_id, common_info):
+            info = extract_info(html, video_id)
+
+            if info['formats']:
+                self._sort_formats(info['formats'])
                 f = common_info.copy()
-                f.update({
-                    'title': title,
-                    'description': description,
-                    'formats': formats,
-                })
+                f.update(info)
                 entries.append(f)
 
-            # Extract teaser only when full episode is not available
-            if not formats:
+            # Extract teaser/trailer only when full episode is not available
+            if not info['formats']:
                 m = re.search(
-                    r'data-dialog-header=(["\'])(?P<title>.+?)\1[^>]+href=(["\'])(?P<href>.+?)\3[^>]*>Teaser<',
-                    episode_html)
+                    r'data-dialog-header=(["\'])(?P<title>.+?)\1[^>]+href=(["\'])(?P<href>.+?)\3[^>]*>(?P<kind>Teaser|Trailer)<',
+                    html)
                 if m:
                     f = common_info.copy()
                     f.update({
-                        'id': '%s-teaser' % f['id'],
+                        'id': '%s-%s' % (f['id'], m.group('kind').lower()),
                         'title': m.group('title'),
                         'url': compat_urlparse.urljoin(url, m.group('href')),
                     })
                     entries.append(f)
 
+        def extract_episodes(html):
+            for num, episode_html in enumerate(re.findall(
+                    r'(?s)<h3[^>]+class="episodebox-title".+?>Episodeninhalt<', html), 1):
+                episodebox_title = self._search_regex(
+                    (r'class="episodebox-title"[^>]+title=(["\'])(?P<title>.+?)\1',
+                     r'class="episodebox-title"[^>]+>(?P<title>.+?)<'),
+                    episode_html, 'episodebox title', default=None, group='title')
+                if not episodebox_title:
+                    continue
+
+                episode_number = int(self._search_regex(
+                    r'(?:Episode|Film)\s*(\d+)',
+                    episodebox_title, 'episode number', default=num))
+                episode_title = self._search_regex(
+                    r'(?:Episode|Film)\s*\d+\s*-\s*(.+)',
+                    episodebox_title, 'episode title', default=None)
+
+                video_id = 'episode-%d' % episode_number
+
+                common_info = {
+                    'id': video_id,
+                    'series': anime_title,
+                    'episode': episode_title,
+                    'episode_number': episode_number,
+                }
+
+                extract_entries(episode_html, video_id, common_info)
+
+        def extract_film(html, video_id):
+            common_info = {
+                'id': anime_id,
+                'title': anime_title,
+                'description': anime_description,
+            }
+            extract_entries(html, video_id, common_info)
+
+        extract_episodes(webpage)
+
+        if not entries:
+            extract_film(webpage, anime_id)
+
         return self.playlist_result(entries, anime_id, anime_title, anime_description)

From 1251565ee0efd71f98d77f1eeefe3f3e38ec6f43 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 9 Jul 2016 07:12:52 +0700
Subject: [PATCH 5471/6219] [options] Rollback old behavior for configuratio
 files' encoding

Until agreed with some solution
---
 youtube_dl/options.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 5302b67cc..c4a85b2c0 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -26,7 +26,11 @@ def parseOpts(overrideArguments=None):
         except IOError:
             return default  # silently skip if file is not present
         try:
-            res = compat_shlex_split(optionf.read(), comments=True)
+            # FIXME: https://github.com/rg3/youtube-dl/commit/dfe5fa49aed02cf36ba9f743b11b0903554b5e56
+            contents = optionf.read()
+            if sys.version_info < (3,):
+                contents = contents.decode(preferredencoding())
+            res = compat_shlex_split(contents, comments=True)
         finally:
             optionf.close()
         return res

From 9e4f5dc1e920b181d7e4a4ae824f86db8bc3a8e3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 9 Jul 2016 07:13:32 +0700
Subject: [PATCH 5472/6219] [animeondemand] Pass num for episode based videos

---
 youtube_dl/extractor/animeondemand.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/animeondemand.py b/youtube_dl/extractor/animeondemand.py
index 1f044e20b..159c6ef5a 100644
--- a/youtube_dl/extractor/animeondemand.py
+++ b/youtube_dl/extractor/animeondemand.py
@@ -115,7 +115,7 @@ class AnimeOnDemandIE(InfoExtractor):
 
         entries = []
 
-        def extract_info(html, video_id):
+        def extract_info(html, video_id, num=None):
             title, description = [None] * 2
             formats = []
 
@@ -143,7 +143,7 @@ class AnimeOnDemandIE(InfoExtractor):
                         format_id_list.append(lang)
                     if kind:
                         format_id_list.append(kind)
-                    if not format_id_list:
+                    if not format_id_list and num is not None:
                         format_id_list.append(compat_str(num))
                     format_id = '-'.join(format_id_list)
                     format_note = ', '.join(filter(None, (kind, lang_note)))
@@ -203,8 +203,8 @@ class AnimeOnDemandIE(InfoExtractor):
                 'formats': formats,
             }
 
-        def extract_entries(html, video_id, common_info):
-            info = extract_info(html, video_id)
+        def extract_entries(html, video_id, common_info, num):
+            info = extract_info(html, video_id, num)
 
             if info['formats']:
                 self._sort_formats(info['formats'])

From d417fd88d09846edf0c2bed65d7d074c62c0c51b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 9 Jul 2016 07:16:47 +0700
Subject: [PATCH 5473/6219] release 2016.07.09

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 docs/supportedsites.md    | 1 +
 youtube_dl/version.py     | 2 +-
 3 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index c65462ba4..c2f352f41 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.07.07*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.07.07**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.07.09*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.07.09**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.07.07
+[debug] youtube-dl version 2016.07.09
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 9174c6f89..d2a2ef0c4 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -224,6 +224,7 @@
  - **Firstpost**
  - **FiveTV**
  - **Flickr**
+ - **Flipagram**
  - **Folketinget**: Folketinget (ft.dk; Danish parliament)
  - **FootyRoom**
  - **Formula1**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 6396ad4c9..1dd2c3300 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.07.07'
+__version__ = '2016.07.09'

From 6e6b70d65f0681317c425bfe1e157f3474afbbe8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 9 Jul 2016 08:32:55 +0700
Subject: [PATCH 5474/6219] [extractor/generic] Properly comment out a test

---
 youtube_dl/extractor/generic.py | 34 ++++++++++++++++-----------------
 1 file changed, 17 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 62da9bbc0..4efdf146e 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1313,23 +1313,23 @@ class GenericIE(InfoExtractor):
             },
             'add_ie': ['Kaltura'],
         },
-        {
-            # TODO: find another test
-            # http://schema.org/VideoObject
-            # 'url': 'https://flipagram.com/f/nyvTSJMKId',
-            # 'md5': '888dcf08b7ea671381f00fab74692755',
-            # 'info_dict': {
-            #     'id': 'nyvTSJMKId',
-            #     'ext': 'mp4',
-            #     'title': 'Flipagram by sjuria101 featuring Midnight Memories by One Direction',
-            #     'description': '#love for cats.',
-            #     'timestamp': 1461244995,
-            #     'upload_date': '20160421',
-            # },
-            # 'params': {
-            #     'force_generic_extractor': True,
-            # },
-        }
+        # {
+        #     # TODO: find another test
+        #     # http://schema.org/VideoObject
+        #     'url': 'https://flipagram.com/f/nyvTSJMKId',
+        #     'md5': '888dcf08b7ea671381f00fab74692755',
+        #     'info_dict': {
+        #         'id': 'nyvTSJMKId',
+        #         'ext': 'mp4',
+        #         'title': 'Flipagram by sjuria101 featuring Midnight Memories by One Direction',
+        #         'description': '#love for cats.',
+        #         'timestamp': 1461244995,
+        #         'upload_date': '20160421',
+        #     },
+        #     'params': {
+        #         'force_generic_extractor': True,
+        #     },
+        # }
     ]
 
     def report_following_redirect(self, new_url):

From 9558dcec9c7806c811f4fe8e7758977eaa01a702 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 9 Jul 2016 08:37:02 +0700
Subject: [PATCH 5475/6219] [youtube:user] Preserve user/c path segment

---
 youtube_dl/extractor/youtube.py | 25 ++++++++++++++++++++++---
 1 file changed, 22 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 69603c1f8..8aa7dfc41 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1978,10 +1978,13 @@ class YoutubeChannelIE(YoutubePlaylistBaseInfoExtractor):
         return (False if YoutubePlaylistsIE.suitable(url) or YoutubeLiveIE.suitable(url)
                 else super(YoutubeChannelIE, cls).suitable(url))
 
+    def _build_template_url(self, url, channel_id):
+        return self._TEMPLATE_URL % channel_id
+
     def _real_extract(self, url):
         channel_id = self._match_id(url)
 
-        url = self._TEMPLATE_URL % channel_id
+        url = self._build_template_url(url, channel_id)
 
         # Channel by page listing is restricted to 35 pages of 30 items, i.e. 1050 videos total (see #5778)
         # Workaround by extracting as a playlist if managed to obtain channel playlist URL
@@ -2038,8 +2041,8 @@ class YoutubeChannelIE(YoutubePlaylistBaseInfoExtractor):
 
 class YoutubeUserIE(YoutubeChannelIE):
     IE_DESC = 'YouTube.com user videos (URL or "ytuser" keyword)'
-    _VALID_URL = r'(?:(?:https?://(?:\w+\.)?youtube\.com/(?:user/|c/)?(?!(?:attribution_link|watch|results)(?:$|[^a-z_A-Z0-9-])))|ytuser:)(?!feed/)(?P<id>[A-Za-z0-9_-]+)'
-    _TEMPLATE_URL = 'https://www.youtube.com/user/%s/videos'
+    _VALID_URL = r'(?:(?:https?://(?:\w+\.)?youtube\.com/(?:(?P<user>user|c)/)?(?!(?:attribution_link|watch|results)(?:$|[^a-z_A-Z0-9-])))|ytuser:)(?!feed/)(?P<id>[A-Za-z0-9_-]+)'
+    _TEMPLATE_URL = 'https://www.youtube.com/%s/%s/videos'
     IE_NAME = 'youtube:user'
 
     _TESTS = [{
@@ -2049,12 +2052,24 @@ class YoutubeUserIE(YoutubeChannelIE):
             'id': 'UUfX55Sx5hEFjoC3cNs6mCUQ',
             'title': 'Uploads from The Linux Foundation',
         }
+    }, {
+        # Only available via https://www.youtube.com/c/12minuteathlete/videos
+        # but not https://www.youtube.com/user/12minuteathlete/videos
+        'url': 'https://www.youtube.com/c/12minuteathlete/videos',
+        'playlist_mincount': 249,
+        'info_dict': {
+            'id': 'UUVjM-zV6_opMDx7WYxnjZiQ',
+            'title': 'Uploads from 12 Minute Athlete',
+        }
     }, {
         'url': 'ytuser:phihag',
         'only_matching': True,
     }, {
         'url': 'https://www.youtube.com/c/gametrailers',
         'only_matching': True,
+    }, {
+        'url': 'https://www.youtube.com/gametrailers',
+        'only_matching': True,
     }, {
         # This channel is not available.
         'url': 'https://www.youtube.com/user/kananishinoSMEJ/videos',
@@ -2071,6 +2086,10 @@ class YoutubeUserIE(YoutubeChannelIE):
         else:
             return super(YoutubeUserIE, cls).suitable(url)
 
+    def _build_template_url(self, url, channel_id):
+        mobj = re.match(self._VALID_URL, url)
+        return self._TEMPLATE_URL % (mobj.group('user') or 'user', mobj.group('id'))
+
 
 class YoutubeLiveIE(YoutubeBaseInfoExtractor):
     IE_DESC = 'YouTube.com live streams'

From 31eeab9f412d17fc7c455340e46cfcc9c91c3c99 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 9 Jul 2016 03:18:45 +0100
Subject: [PATCH 5476/6219] [ard] fix f4m extraction and skip tests with 404
 errors

---
 youtube_dl/extractor/ard.py | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/ard.py b/youtube_dl/extractor/ard.py
index fd45b3e42..13a06396d 100644
--- a/youtube_dl/extractor/ard.py
+++ b/youtube_dl/extractor/ard.py
@@ -13,6 +13,7 @@ from ..utils import (
     parse_duration,
     unified_strdate,
     xpath_text,
+    update_url_query,
 )
 from ..compat import compat_etree_fromstring
 
@@ -34,6 +35,7 @@ class ARDMediathekIE(InfoExtractor):
             # m3u8 download
             'skip_download': True,
         },
+        'skip': 'HTTP Error 404: Not Found',
     }, {
         'url': 'http://www.ardmediathek.de/tv/Tatort/Tatort-Scheinwelten-H%C3%B6rfassung-Video/Das-Erste/Video?documentId=29522730&bcastId=602916',
         'md5': 'f4d98b10759ac06c0072bbcd1f0b9e3e',
@@ -44,6 +46,7 @@ class ARDMediathekIE(InfoExtractor):
             'description': 'md5:196392e79876d0ac94c94e8cdb2875f1',
             'duration': 5252,
         },
+        'skip': 'HTTP Error 404: Not Found',
     }, {
         # audio
         'url': 'http://www.ardmediathek.de/tv/WDR-H%C3%B6rspiel-Speicher/Tod-eines-Fu%C3%9Fballers/WDR-3/Audio-Podcast?documentId=28488308&bcastId=23074086',
@@ -55,6 +58,7 @@ class ARDMediathekIE(InfoExtractor):
             'description': 'md5:f6e39f3461f0e1f54bfa48c8875c86ef',
             'duration': 3240,
         },
+        'skip': 'HTTP Error 404: Not Found',
     }, {
         'url': 'http://mediathek.daserste.de/sendungen_a-z/328454_anne-will/22429276_vertrauen-ist-gut-spionieren-ist-besser-geht',
         'only_matching': True,
@@ -113,11 +117,14 @@ class ARDMediathekIE(InfoExtractor):
                         continue
                     if ext == 'f4m':
                         formats.extend(self._extract_f4m_formats(
-                            stream_url + '?hdcore=3.1.1&plugin=aasp-3.1.1.69.124',
-                            video_id, preference=-1, f4m_id='hds', fatal=False))
+                            update_url_query(stream_url, {
+                                'hdcore': '3.1.1',
+                                'plugin': 'aasp-3.1.1.69.124'
+                            }),
+                            video_id, f4m_id='hds', fatal=False))
                     elif ext == 'm3u8':
                         formats.extend(self._extract_m3u8_formats(
-                            stream_url, video_id, 'mp4', preference=1, m3u8_id='hls', fatal=False))
+                            stream_url, video_id, 'mp4', m3u8_id='hls', fatal=False))
                     else:
                         if server and server.startswith('rtmp'):
                             f = {
@@ -231,7 +238,8 @@ class ARDIE(InfoExtractor):
             'title': 'Die Story im Ersten: Mission unter falscher Flagge',
             'upload_date': '20140804',
             'thumbnail': 're:^https?://.*\.jpg$',
-        }
+        },
+        'skip': 'HTTP Error 404: Not Found',
     }
 
     def _real_extract(self, url):

From 8cc9b4016d6ae33d8f6b42b7096e8042e7915003 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 9 Jul 2016 03:21:50 +0100
Subject: [PATCH 5477/6219] [srmediathek] extend _VALID_URL(closes #9373)

---
 youtube_dl/extractor/srmediathek.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/srmediathek.py b/youtube_dl/extractor/srmediathek.py
index a2569dfba..409d50304 100644
--- a/youtube_dl/extractor/srmediathek.py
+++ b/youtube_dl/extractor/srmediathek.py
@@ -11,7 +11,7 @@ from ..utils import (
 class SRMediathekIE(ARDMediathekIE):
     IE_NAME = 'sr:mediathek'
     IE_DESC = 'Saarländischer Rundfunk'
-    _VALID_URL = r'https?://sr-mediathek\.sr-online\.de/index\.php\?.*?&id=(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://sr-mediathek(?:\.sr-online)?\.de/index\.php\?.*?&id=(?P<id>[0-9]+)'
 
     _TESTS = [{
         'url': 'http://sr-mediathek.sr-online.de/index.php?seite=7&id=28455',
@@ -35,7 +35,9 @@ class SRMediathekIE(ARDMediathekIE):
             # m3u8 download
             'skip_download': True,
         },
-        'expected_warnings': ['Unable to download f4m manifest']
+    }, {
+        'url': 'http://sr-mediathek.de/index.php?seite=7&id=7480',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From ce43100a0143837432c52fcd156068c3464c4303 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 9 Jul 2016 10:06:40 +0700
Subject: [PATCH 5478/6219] release 2016.07.09.1

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 youtube_dl/version.py     | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index c2f352f41..a1b5b0baa 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.07.09*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.07.09**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.07.09.1*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.07.09.1**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.07.09
+[debug] youtube-dl version 2016.07.09.1
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 1dd2c3300..3e45977c9 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.07.09'
+__version__ = '2016.07.09.1'

From 0ece114b7b3c9f7277adbd77de17534e39137675 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 9 Jul 2016 14:38:27 +0800
Subject: [PATCH 5479/6219] [vimeo] Recognize non-standard embeds (#1638)

---
 youtube_dl/extractor/vimeo.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index d9c9852d4..7e854f326 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -364,6 +364,11 @@ class VimeoIE(VimeoBaseInfoExtractor):
             r'<embed[^>]+?src="((?:https?:)?//(?:www\.)?vimeo\.com/moogaloop\.swf.+?)"', webpage)
         if mobj:
             return mobj.group(1)
+        # Look more for non-standard embedded Vimeo player
+        mobj = re.search(
+            r'<video[^>]+src=(?P<q1>[\'"])(?P<url>(?:https?:)?//(?:www\.)?vimeo\.com/[0-9]+)(?P=q1)', webpage)
+        if mobj:
+            return mobj.group('url')
 
     def _verify_player_video_password(self, url, video_id):
         password = self._downloader.params.get('videopassword')

From c03adf90bd92ceba55f99cf8f3b61e6199166486 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 9 Jul 2016 14:39:01 +0800
Subject: [PATCH 5480/6219] [generic] Add the test. Closes #1638

---
 youtube_dl/extractor/generic.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 4efdf146e..cddd1a817 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1313,6 +1313,21 @@ class GenericIE(InfoExtractor):
             },
             'add_ie': ['Kaltura'],
         },
+        {
+            # Non-standard Vimeo embed
+            'url': 'https://openclassrooms.com/courses/understanding-the-web',
+            'md5': '64d86f1c7d369afd9a78b38cbb88d80a',
+            'info_dict': {
+                'id': '148867247',
+                'ext': 'mp4',
+                'title': 'Understanding the web - Teaser',
+                'description': 'This is "Understanding the web - Teaser" by openclassrooms on Vimeo, the home for high quality videos and the people who love them.',
+                'upload_date': '20151214',
+                'uploader': 'OpenClassrooms',
+                'uploader_id': 'openclassrooms',
+            },
+            'add_ie': ['Vimeo'],
+        },
         # {
         #     # TODO: find another test
         #     # http://schema.org/VideoObject

From 6daf34a0457cdb3b657cc01997ce5197dab91047 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 9 Jul 2016 21:25:07 +0700
Subject: [PATCH 5481/6219] [facebook] Fix typo and break when found video_data
 (Closes #10048)

---
 youtube_dl/extractor/facebook.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index 0d43acc4a..cdb093262 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -231,11 +231,13 @@ class FacebookIE(InfoExtractor):
                     continue
                 if isinstance(f, dict):
                     f = [f]
-                if isinstance(f, list):
+                if not isinstance(f, list):
                     continue
                 if f[0].get('video_id') == video_id:
                     video_data = video_data_candidate
                     break
+            if video_data:
+                break
 
         def video_data_list2dict(video_data):
             ret = {}

From 61a98b862320801ad2ae9d6cb922d9575ead5276 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 9 Jul 2016 21:29:11 +0700
Subject: [PATCH 5482/6219] [lynda] Remove md5 from test (Closes #10047)

---
 youtube_dl/extractor/lynda.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index 1237e1573..a98c4c530 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -100,7 +100,7 @@ class LyndaIE(LyndaBaseIE):
 
     _TESTS = [{
         'url': 'http://www.lynda.com/Bootstrap-tutorials/Using-exercise-files/110885/114408-4.html',
-        'md5': '679734f6786145da3546585de9a356be',
+        # md5 is unstable
         'info_dict': {
             'id': '114408',
             'ext': 'mp4',

From ab52bb5137faf1cb5595b6777adf721eec6af78a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 9 Jul 2016 22:20:34 +0700
Subject: [PATCH 5483/6219] [animeondemand] Fix typo

---
 youtube_dl/extractor/animeondemand.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/animeondemand.py b/youtube_dl/extractor/animeondemand.py
index 159c6ef5a..9e28f2579 100644
--- a/youtube_dl/extractor/animeondemand.py
+++ b/youtube_dl/extractor/animeondemand.py
@@ -203,7 +203,7 @@ class AnimeOnDemandIE(InfoExtractor):
                 'formats': formats,
             }
 
-        def extract_entries(html, video_id, common_info, num):
+        def extract_entries(html, video_id, common_info, num=None):
             info = extract_info(html, video_id, num)
 
             if info['formats']:

From 0286b85c79c278375cc5e71d3199bdc8284a5a11 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 9 Jul 2016 22:22:24 +0700
Subject: [PATCH 5484/6219] release 2016.07.09.2

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 youtube_dl/version.py     | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index a1b5b0baa..1a3a50eb2 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.07.09.1*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.07.09.1**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.07.09.2*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.07.09.2**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.07.09.1
+[debug] youtube-dl version 2016.07.09.2
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 3e45977c9..728ad2d50 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.07.09.1'
+__version__ = '2016.07.09.2'

From 3121b25639a06cc9cad48fad0ce222482a58b216 Mon Sep 17 00:00:00 2001
From: Nehal Patel <nehalvpatels+github@gmail.com>
Date: Wed, 22 Jun 2016 02:58:42 -0500
Subject: [PATCH 5485/6219] [roosterteeth] Add extractor

---
 youtube_dl/extractor/extractors.py   |   1 +
 youtube_dl/extractor/roosterteeth.py | 112 +++++++++++++++++++++++++++
 2 files changed, 113 insertions(+)
 create mode 100644 youtube_dl/extractor/roosterteeth.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 9f70ce752..864c9af68 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -680,6 +680,7 @@ from .rice import RICEIE
 from .ringtv import RingTVIE
 from .ro220 import Ro220IE
 from .rockstargames import RockstarGamesIE
+from .roosterteeth import RoosterTeethIE
 from .rottentomatoes import RottenTomatoesIE
 from .roxwel import RoxwelIE
 from .rtbf import RTBFIE
diff --git a/youtube_dl/extractor/roosterteeth.py b/youtube_dl/extractor/roosterteeth.py
new file mode 100644
index 000000000..4053747d3
--- /dev/null
+++ b/youtube_dl/extractor/roosterteeth.py
@@ -0,0 +1,112 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    urlencode_postdata,
+)
+
+
+class RoosterTeethIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:.+?\.)?roosterteeth\.com/episode/(?P<id>[^/?#&]+)'
+    _LOGIN_URL = 'https://roosterteeth.com/login'
+    _NETRC_MACHINE = 'roosterteeth'
+    _TESTS = [{
+        'url': 'http://roosterteeth.com/episode/million-dollars-but-season-2-million-dollars-but-the-game-announcement',
+        'info_dict': {
+            'id': '26576',
+            'ext': 'mp4',
+            'title': 'Million Dollars, But... The Game Announcement',
+            'thumbnail': 're:^https?://.*\.png$',
+            'description': 'Introducing Million Dollars, But... The Game! Available for pre-order now at www.MDBGame.com ',
+            'creator': 'Rooster Teeth',
+            'series': 'Million Dollars, But...',
+            'episode': 'Million Dollars, But... The Game Announcement',
+            'episode_id': '26576',
+        },
+        'params': {
+            'skip_download': True,  # m3u8 downloads
+        },
+    }, {
+        'url': 'http://achievementhunter.roosterteeth.com/episode/off-topic-the-achievement-hunter-podcast-2016-i-didn-t-think-it-would-pass-31',
+        'only_matching': True,
+    }, {
+        'url': 'http://funhaus.roosterteeth.com/episode/funhaus-shorts-2016-austin-sucks-funhaus-shorts',
+        'only_matching': True,
+    }, {
+        'url': 'http://screwattack.roosterteeth.com/episode/death-battle-season-3-mewtwo-vs-shadow',
+        'only_matching': True,
+    }, {
+        'url': 'http://theknow.roosterteeth.com/episode/the-know-game-news-season-1-boring-steam-sales-are-better',
+        'only_matching': True,
+    }]
+
+    def _login(self):
+        (username, password) = self._get_login_info()
+        if username is None or password is None:
+            return False
+
+        # token is required to authenticate request
+        login_page = self._download_webpage(self._LOGIN_URL, None, 'Getting login token', 'Unable to get login token')
+
+        login_form = self._hidden_inputs(login_page)
+        login_form.update({
+            'username': username,
+            'password': password,
+        })
+        login_payload = urlencode_postdata(login_form)
+
+        # required for proper responses
+        login_headers = {
+            'Referer': self._LOGIN_URL,
+        }
+
+        login_request = self._download_webpage(
+            self._LOGIN_URL, None,
+            note='Logging in as %s' % username,
+            data=login_payload,
+            headers=login_headers)
+
+        if 'Authentication failed' in login_request:
+            raise ExtractorError(
+                'Login failed (invalid username/password)', expected=True)
+
+    def _real_initialize(self):
+        self._login()
+
+    def _real_extract(self, url):
+        match_id = self._match_id(url)
+        webpage = self._download_webpage(url, match_id)
+
+        episode_id = self._html_search_regex(r"commentControls\('#comment-([0-9]+)'\)", webpage, 'episode id', match_id, False)
+
+        self.report_extraction(episode_id)
+
+        title = self._html_search_regex(r'<title>([^<]+)</title>', webpage, 'episode title', self._og_search_title(webpage), False)
+        thumbnail = self._og_search_thumbnail(webpage)
+        description = self._og_search_description(webpage)
+        creator = self._html_search_regex(r'<h3>Latest (.+) Gear</h3>', webpage, 'site', 'Rooster Teeth', False)
+        series = self._html_search_regex(r'<h2>More ([^<]+)</h2>', webpage, 'series', fatal=False)
+        episode = self._html_search_regex(r'<title>([^<]+)</title>', webpage, 'episode title', fatal=False)
+
+        if '<div class="non-sponsor">' in webpage:
+            self.raise_login_required('%s is only available for FIRST members' % title)
+
+        if '<div class="golive-gate">' in webpage:
+            self.raise_login_required('%s is not available yet' % title)
+
+        formats = self._extract_m3u8_formats(self._html_search_regex(r"file: '(.+?)m3u8'", webpage, 'm3u8 url') + 'm3u8', episode_id, ext='mp4')
+        self._sort_formats(formats)
+
+        return {
+            'id': episode_id,
+            'title': title,
+            'formats': formats,
+            'thumbnail': thumbnail,
+            'description': description,
+            'creator': creator,
+            'series': series,
+            'episode': episode,
+            'episode_id': episode_id,
+        }

From 865b0872249e0d402244b4c72e9f79dc6415c926 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 10 Jul 2016 01:28:28 +0700
Subject: [PATCH 5486/6219] [roosterteeth] Improve (Closes #9864)

---
 youtube_dl/extractor/roosterteeth.py | 120 +++++++++++++++++----------
 1 file changed, 78 insertions(+), 42 deletions(-)

diff --git a/youtube_dl/extractor/roosterteeth.py b/youtube_dl/extractor/roosterteeth.py
index 4053747d3..f5b2f560c 100644
--- a/youtube_dl/extractor/roosterteeth.py
+++ b/youtube_dl/extractor/roosterteeth.py
@@ -1,9 +1,14 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
+    int_or_none,
+    strip_or_none,
+    unescapeHTML,
     urlencode_postdata,
 )
 
@@ -14,19 +19,17 @@ class RoosterTeethIE(InfoExtractor):
     _NETRC_MACHINE = 'roosterteeth'
     _TESTS = [{
         'url': 'http://roosterteeth.com/episode/million-dollars-but-season-2-million-dollars-but-the-game-announcement',
+        'md5': 'e2bd7764732d785ef797700a2489f212',
         'info_dict': {
             'id': '26576',
+            'display_id': 'million-dollars-but-season-2-million-dollars-but-the-game-announcement',
             'ext': 'mp4',
-            'title': 'Million Dollars, But... The Game Announcement',
+            'title': 'Million Dollars, But...: Million Dollars, But... The Game Announcement',
+            'description': 'md5:0cc3b21986d54ed815f5faeccd9a9ca5',
             'thumbnail': 're:^https?://.*\.png$',
-            'description': 'Introducing Million Dollars, But... The Game! Available for pre-order now at www.MDBGame.com ',
-            'creator': 'Rooster Teeth',
             'series': 'Million Dollars, But...',
             'episode': 'Million Dollars, But... The Game Announcement',
-            'episode_id': '26576',
-        },
-        'params': {
-            'skip_download': True,  # m3u8 downloads
+            'comment_count': int,
         },
     }, {
         'url': 'http://achievementhunter.roosterteeth.com/episode/off-topic-the-achievement-hunter-podcast-2016-i-didn-t-think-it-would-pass-31',
@@ -40,73 +43,106 @@ class RoosterTeethIE(InfoExtractor):
     }, {
         'url': 'http://theknow.roosterteeth.com/episode/the-know-game-news-season-1-boring-steam-sales-are-better',
         'only_matching': True,
+    }, {
+        # only available for FIRST members
+        'url': 'http://roosterteeth.com/episode/rt-docs-the-world-s-greatest-head-massage-the-world-s-greatest-head-massage-an-asmr-journey-part-one',
+        'only_matching': True,
     }]
 
     def _login(self):
         (username, password) = self._get_login_info()
-        if username is None or password is None:
-            return False
+        if username is None:
+            return
 
-        # token is required to authenticate request
-        login_page = self._download_webpage(self._LOGIN_URL, None, 'Getting login token', 'Unable to get login token')
+        login_page = self._download_webpage(
+            self._LOGIN_URL, None,
+            note='Downloading login page',
+            errnote='Unable to download login page')
 
         login_form = self._hidden_inputs(login_page)
+
         login_form.update({
             'username': username,
             'password': password,
         })
-        login_payload = urlencode_postdata(login_form)
-
-        # required for proper responses
-        login_headers = {
-            'Referer': self._LOGIN_URL,
-        }
 
         login_request = self._download_webpage(
             self._LOGIN_URL, None,
             note='Logging in as %s' % username,
-            data=login_payload,
-            headers=login_headers)
+            data=urlencode_postdata(login_form),
+            headers={
+                'Referer': self._LOGIN_URL,
+            })
 
-        if 'Authentication failed' in login_request:
-            raise ExtractorError(
-                'Login failed (invalid username/password)', expected=True)
+        if not any(re.search(p, login_request) for p in (
+                r'href=["\']https?://(?:www\.)?roosterteeth\.com/logout"',
+                r'>Sign Out<')):
+            error = self._html_search_regex(
+                r'(?s)<div[^>]+class=(["\']).*?\balert-danger\b.*?\1[^>]*>(?:\s*<button[^>]*>.*?</button>)?(?P<error>.+?)</div>',
+                login_request, 'alert', default=None, group='error')
+            if error:
+                raise ExtractorError('Unable to login: %s' % error, expected=True)
+            raise ExtractorError('Unable to log in')
 
     def _real_initialize(self):
         self._login()
 
     def _real_extract(self, url):
-        match_id = self._match_id(url)
-        webpage = self._download_webpage(url, match_id)
+        display_id = self._match_id(url)
 
-        episode_id = self._html_search_regex(r"commentControls\('#comment-([0-9]+)'\)", webpage, 'episode id', match_id, False)
+        webpage = self._download_webpage(url, display_id)
 
-        self.report_extraction(episode_id)
+        episode = strip_or_none(unescapeHTML(self._search_regex(
+            (r'videoTitle\s*=\s*(["\'])(?P<title>(?:(?!\1).)+)\1',
+             r'<title>(?P<title>[^<]+)</title>'), webpage, 'title',
+            default=None, group='title')))
 
-        title = self._html_search_regex(r'<title>([^<]+)</title>', webpage, 'episode title', self._og_search_title(webpage), False)
-        thumbnail = self._og_search_thumbnail(webpage)
-        description = self._og_search_description(webpage)
-        creator = self._html_search_regex(r'<h3>Latest (.+) Gear</h3>', webpage, 'site', 'Rooster Teeth', False)
-        series = self._html_search_regex(r'<h2>More ([^<]+)</h2>', webpage, 'series', fatal=False)
-        episode = self._html_search_regex(r'<title>([^<]+)</title>', webpage, 'episode title', fatal=False)
+        title = strip_or_none(self._og_search_title(
+            webpage, default=None)) or episode
 
-        if '<div class="non-sponsor">' in webpage:
-            self.raise_login_required('%s is only available for FIRST members' % title)
+        m3u8_url = self._search_regex(
+            r'file\s*:\s*(["\'])(?P<url>http.+?\.m3u8.*?)\1',
+            webpage, 'm3u8 url', default=None, group='url')
 
-        if '<div class="golive-gate">' in webpage:
-            self.raise_login_required('%s is not available yet' % title)
+        if not m3u8_url:
+            if re.search(r'<div[^>]+class=["\']non-sponsor', webpage):
+                self.raise_login_required(
+                    '%s is only available for FIRST members' % display_id)
 
-        formats = self._extract_m3u8_formats(self._html_search_regex(r"file: '(.+?)m3u8'", webpage, 'm3u8 url') + 'm3u8', episode_id, ext='mp4')
+            if re.search(r'<div[^>]+class=["\']golive-gate', webpage):
+                self.raise_login_required('%s is not available yet' % display_id)
+
+            raise ExtractorError('Unable to extract m3u8 URL')
+
+        formats = self._extract_m3u8_formats(
+            m3u8_url, display_id, ext='mp4',
+            entry_protocol='m3u8_native', m3u8_id='hls')
         self._sort_formats(formats)
 
+        description = strip_or_none(self._og_search_description(webpage))
+        thumbnail = self._proto_relative_url(self._og_search_thumbnail(webpage))
+
+        series = self._search_regex(
+            (r'<h2>More ([^<]+)</h2>', r'<a[^>]+>See All ([^<]+) Videos<'),
+            webpage, 'series', fatal=False)
+
+        comment_count = int_or_none(self._search_regex(
+            r'>Comments \((\d+)\)<', webpage,
+            'comment count', fatal=False))
+
+        video_id = self._search_regex(
+            (r'containerId\s*=\s*["\']episode-(\d+)\1',
+             r'<div[^<]+id=["\']episode-(\d+)'), webpage,
+            'video id', default=display_id)
+
         return {
-            'id': episode_id,
+            'id': video_id,
+            'display_id': display_id,
             'title': title,
-            'formats': formats,
-            'thumbnail': thumbnail,
             'description': description,
-            'creator': creator,
+            'thumbnail': thumbnail,
             'series': series,
             'episode': episode,
-            'episode_id': episode_id,
+            'comment_count': comment_count,
+            'formats': formats,
         }

From 39e9d524e5fe289936160d4c599a77f10f6e9061 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 10 Jul 2016 01:29:30 +0700
Subject: [PATCH 5487/6219] Credit @nehalvpatel for roosterteeth (#9864)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index f74b30d07..f762e8a16 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -177,3 +177,4 @@ Roman Tsiupa
 Artur Krysiak
 Jakub Adam Wieczorek
 Aleksandar Topuzović
+Nehal Patel

From c3baaedfc8aac6ee6f809ef97b07ebebb07a1097 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 29 May 2016 03:33:27 +0800
Subject: [PATCH 5488/6219] [miomio] Support new 'h5' player (closes #9605)

Depends on #8876
---
 youtube_dl/extractor/miomio.py | 51 ++++++++++++++++++++++++++--------
 1 file changed, 39 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/miomio.py b/youtube_dl/extractor/miomio.py
index 170ebd9eb..890aba19d 100644
--- a/youtube_dl/extractor/miomio.py
+++ b/youtube_dl/extractor/miomio.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 import random
 
 from .common import InfoExtractor
+from ..compat import compat_urlparse
 from ..utils import (
     xpath_text,
     int_or_none,
@@ -40,20 +41,22 @@ class MioMioIE(InfoExtractor):
             'title': 'The New Macbook 2015 上手试玩与简评'
         },
         'playlist_mincount': 2,
+    }, {
+        # new 'h5' player
+        'url': 'http://www.miomio.tv/watch/cc273295/',
+        'md5': '',
+        'info_dict': {
+            'id': '273295',
+            'ext': 'mp4',
+            'title': 'アウト×デラックス 20160526',
+        },
+        'params': {
+            # intermittent HTTP 500
+            'skip_download': True,
+        },
     }]
 
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-
-        title = self._html_search_meta(
-            'description', webpage, 'title', fatal=True)
-
-        mioplayer_path = self._search_regex(
-            r'src="(/mioplayer/[^"]+)"', webpage, 'ref_path')
-
-        http_headers = {'Referer': 'http://www.miomio.tv%s' % mioplayer_path}
-
+    def _extract_mioplayer(self, webpage, video_id, title, http_headers):
         xml_config = self._search_regex(
             r'flashvars="type=(?:sina|video)&amp;(.+?)&amp;',
             webpage, 'xml config')
@@ -92,10 +95,34 @@ class MioMioIE(InfoExtractor):
                 'http_headers': http_headers,
             })
 
+        return entries
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        title = self._html_search_meta(
+            'description', webpage, 'title', fatal=True)
+
+        mioplayer_path = self._search_regex(
+            r'src="(/mioplayer(?:_h5)?/[^"]+)"', webpage, 'ref_path')
+
+        if '_h5' in mioplayer_path:
+            player_url = compat_urlparse.urljoin(url, mioplayer_path)
+            player_webpage = self._download_webpage(
+                player_url, video_id,
+                note='Downloading player webpage', headers={'Referer': url})
+            entries = self._parse_html5_media_entries(player_url, player_webpage)
+            http_headers = {'Referer': player_url}
+        else:
+            http_headers = {'Referer': 'http://www.miomio.tv%s' % mioplayer_path}
+            entries = self._extract_mioplayer(webpage, video_id, title, http_headers)
+
         if len(entries) == 1:
             segment = entries[0]
             segment['id'] = video_id
             segment['title'] = title
+            segment['http_headers'] = http_headers
             return segment
 
         return {

From ae0185016521e6fd284c87e2b138c0a8aca8a849 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 11 Jul 2016 00:03:24 +0800
Subject: [PATCH 5489/6219] [miomio] Fix _TESTS

---
 youtube_dl/extractor/miomio.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/miomio.py b/youtube_dl/extractor/miomio.py
index 890aba19d..937ba0f28 100644
--- a/youtube_dl/extractor/miomio.py
+++ b/youtube_dl/extractor/miomio.py
@@ -19,13 +19,16 @@ class MioMioIE(InfoExtractor):
     _TESTS = [{
         # "type=video" in flashvars
         'url': 'http://www.miomio.tv/watch/cc88912/',
-        'md5': '317a5f7f6b544ce8419b784ca8edae65',
         'info_dict': {
             'id': '88912',
             'ext': 'flv',
             'title': '【SKY】字幕 铠武昭和VS平成 假面骑士大战FEAT战队 魔星字幕组 字幕',
             'duration': 5923,
         },
+        'params': {
+            # The server provides broken file
+            'skip_download': True,
+        }
     }, {
         'url': 'http://www.miomio.tv/watch/cc184024/',
         'info_dict': {
@@ -33,7 +36,7 @@ class MioMioIE(InfoExtractor):
             'title': '《动漫同人插画绘制》',
         },
         'playlist_mincount': 86,
-        'skip': 'This video takes time too long for retrieving the URL',
+        'skip': 'Unable to load videos',
     }, {
         'url': 'http://www.miomio.tv/watch/cc173113/',
         'info_dict': {
@@ -41,6 +44,7 @@ class MioMioIE(InfoExtractor):
             'title': 'The New Macbook 2015 上手试玩与简评'
         },
         'playlist_mincount': 2,
+        'skip': 'Unable to load videos',
     }, {
         # new 'h5' player
         'url': 'http://www.miomio.tv/watch/cc273295/',

From e2dbcaa1bf65aa502718005d5fbd00189618469f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 11 Jul 2016 00:52:25 +0800
Subject: [PATCH 5490/6219] [vuclip] Fix extraction

---
 youtube_dl/extractor/vuclip.py | 43 ++++++++++++----------------------
 1 file changed, 15 insertions(+), 28 deletions(-)

diff --git a/youtube_dl/extractor/vuclip.py b/youtube_dl/extractor/vuclip.py
index eaa888f00..b73da5cd0 100644
--- a/youtube_dl/extractor/vuclip.py
+++ b/youtube_dl/extractor/vuclip.py
@@ -9,7 +9,7 @@ from ..compat import (
 from ..utils import (
     ExtractorError,
     parse_duration,
-    qualities,
+    remove_end,
 )
 
 
@@ -22,7 +22,7 @@ class VuClipIE(InfoExtractor):
             'id': '922692425',
             'ext': '3gp',
             'title': 'The Toy Soldiers - Hollywood Movie Trailer',
-            'duration': 180,
+            'duration': 177,
         }
     }
 
@@ -46,34 +46,21 @@ class VuClipIE(InfoExtractor):
                 '%s said: %s' % (self.IE_NAME, error_msg), expected=True)
 
         # These clowns alternate between two page types
-        links_code = self._search_regex(
-            r'''(?xs)
-                (?:
-                    <img\s+src="[^"]*/play.gif".*?>|
-                    <!--\ player\ end\ -->\s*</div><!--\ thumb\ end-->
-                )
-                (.*?)
-                (?:
-                    <a\s+href="fblike|<div\s+class="social">
-                )
-            ''', webpage, 'links')
-        title = self._html_search_regex(
-            r'<title>(.*?)-\s*Vuclip</title>', webpage, 'title').strip()
+        video_url = self._search_regex(
+            r'<a[^>]+href="([^"]+)"[^>]*><img[^>]+src="[^"]*/play\.gif',
+            webpage, 'video URL', default=None)
+        if video_url:
+            formats = [{
+                'url': video_url,
+            }]
+        else:
+            formats = self._parse_html5_media_entries(url, webpage)[0]['formats']
 
-        quality_order = qualities(['Reg', 'Hi'])
-        formats = []
-        for url, q in re.findall(
-                r'<a\s+href="(?P<url>[^"]+)".*?>(?:<button[^>]*>)?(?P<q>[^<]+)(?:</button>)?</a>', links_code):
-            format_id = compat_urllib_parse_urlparse(url).scheme + '-' + q
-            formats.append({
-                'format_id': format_id,
-                'url': url,
-                'quality': quality_order(q),
-            })
-        self._sort_formats(formats)
+        title = remove_end(self._html_search_regex(
+            r'<title>(.*?)-\s*Vuclip</title>', webpage, 'title').strip(), ' - Video')
 
-        duration = parse_duration(self._search_regex(
-            r'\(([0-9:]+)\)</span>', webpage, 'duration', fatal=False))
+        duration = parse_duration(self._html_search_regex(
+            r'[(>]([0-9]+:[0-9]+)(?:<span|\))', webpage, 'duration', fatal=False))
 
         return {
             'id': video_id,

From e9fb6a4bbee2eaf068a4499ce8fa4df89458a3c3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 11 Jul 2016 03:08:38 +0700
Subject: [PATCH 5491/6219] [youtube] Relax TFA regexes

---
 youtube_dl/extractor/youtube.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 8aa7dfc41..1687d5ef9 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -137,7 +137,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
         # Two-Factor
         # TODO add SMS and phone call support - these require making a request and then prompting the user
 
-        if re.search(r'(?i)<form[^>]* id="challenge"', login_results) is not None:
+        if re.search(r'(?i)<form[^>]+id="challenge"', login_results) is not None:
             tfa_code = self._get_tfa_info('2-step verification code')
 
             if not tfa_code:
@@ -165,17 +165,17 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
             if tfa_results is False:
                 return False
 
-            if re.search(r'(?i)<form[^>]* id="challenge"', tfa_results) is not None:
+            if re.search(r'(?i)<form[^>]+id="challenge"', tfa_results) is not None:
                 self._downloader.report_warning('Two-factor code expired or invalid. Please try again, or use a one-use backup code instead.')
                 return False
-            if re.search(r'(?i)<form[^>]* id="gaia_loginform"', tfa_results) is not None:
+            if re.search(r'(?i)<form[^>]+id="gaia_loginform"', tfa_results) is not None:
                 self._downloader.report_warning('unable to log in - did the page structure change?')
                 return False
             if re.search(r'smsauth-interstitial-reviewsettings', tfa_results) is not None:
                 self._downloader.report_warning('Your Google account has a security notice. Please log in on your web browser, resolve the notice, and try again.')
                 return False
 
-        if re.search(r'(?i)<form[^>]* id="gaia_loginform"', login_results) is not None:
+        if re.search(r'(?i)<form[^>]+id="gaia_loginform"', login_results) is not None:
             self._downloader.report_warning('unable to log in: bad username or password')
             return False
         return True

From 5c4dcf817268105aa0bcb6dc4864bc9b98826abb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 11 Jul 2016 03:14:39 +0700
Subject: [PATCH 5492/6219] [vidzi] Add support for embed URLs (Closes #10058)

---
 youtube_dl/extractor/vidzi.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/vidzi.py b/youtube_dl/extractor/vidzi.py
index 3c78fb3d5..d49cc6cbc 100644
--- a/youtube_dl/extractor/vidzi.py
+++ b/youtube_dl/extractor/vidzi.py
@@ -9,8 +9,8 @@ from ..utils import (
 
 
 class VidziIE(JWPlatformBaseIE):
-    _VALID_URL = r'https?://(?:www\.)?vidzi\.tv/(?P<id>\w+)'
-    _TEST = {
+    _VALID_URL = r'https?://(?:www\.)?vidzi\.tv/(?:embed-)?(?P<id>[0-9a-zA-Z]+)'
+    _TESTS = [{
         'url': 'http://vidzi.tv/cghql9yq6emu.html',
         'md5': '4f16c71ca0c8c8635ab6932b5f3f1660',
         'info_dict': {
@@ -22,12 +22,16 @@ class VidziIE(JWPlatformBaseIE):
             # m3u8 download
             'skip_download': True,
         },
-    }
+    }, {
+        'url': 'http://vidzi.tv/embed-4z2yb0rzphe9-600x338.html',
+        'skip_download': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        webpage = self._download_webpage(url, video_id)
+        webpage = self._download_webpage(
+            'http://vidzi.tv/%s' % video_id, video_id)
         title = self._html_search_regex(
             r'(?s)<h2 class="video-title">(.*?)</h2>', webpage, 'title')
 

From a26bcc61c177470606a1b5e8fd74469e894745b2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 11 Jul 2016 03:17:12 +0700
Subject: [PATCH 5493/6219] release 2016.07.11

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 docs/supportedsites.md    | 1 +
 youtube_dl/version.py     | 2 +-
 3 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 1a3a50eb2..704a8b911 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.07.09.2*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.07.09.2**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.07.11*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.07.11**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.07.09.2
+[debug] youtube-dl version 2016.07.11
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index d2a2ef0c4..5bcd6de1c 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -554,6 +554,7 @@
  - **RICE**
  - **RingTV**
  - **RockstarGames**
+ - **RoosterTeeth**
  - **RottenTomatoes**
  - **Roxwel**
  - **RTBF**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 728ad2d50..d60480223 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.07.09.2'
+__version__ = '2016.07.11'

From 8e7020daef5477a05e6f02c3b59b0bd04b315eb6 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 11 Jul 2016 13:19:25 +0800
Subject: [PATCH 5494/6219] [rudo] Add new extractor

Used in biobiochile.tv
---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/rudo.py       | 53 ++++++++++++++++++++++++++++++
 2 files changed, 54 insertions(+)
 create mode 100644 youtube_dl/extractor/rudo.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 864c9af68..b08df41b4 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -691,6 +691,7 @@ from .rtp import RTPIE
 from .rts import RTSIE
 from .rtve import RTVEALaCartaIE, RTVELiveIE, RTVEInfantilIE
 from .rtvnh import RTVNHIE
+from .rudo import RudoIE
 from .ruhd import RUHDIE
 from .ruleporn import RulePornIE
 from .rutube import (
diff --git a/youtube_dl/extractor/rudo.py b/youtube_dl/extractor/rudo.py
new file mode 100644
index 000000000..38366b784
--- /dev/null
+++ b/youtube_dl/extractor/rudo.py
@@ -0,0 +1,53 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .jwplatform import JWPlatformBaseIE
+from ..utils import (
+    js_to_json,
+    get_element_by_class,
+    unified_strdate,
+)
+
+
+class RudoIE(JWPlatformBaseIE):
+    _VALID_URL = r'https?://rudo\.video/vod/(?P<id>[0-9a-zA-Z]+)'
+
+    _TEST = {
+        'url': 'http://rudo.video/vod/oTzw0MGnyG',
+        'md5': '2a03a5b32dd90a04c83b6d391cf7b415',
+        'info_dict': {
+            'id': 'oTzw0MGnyG',
+            'ext': 'mp4',
+            'title': 'Comentario Tomás Mosciatti',
+            'upload_date': '20160617',
+        },
+    }
+
+    @classmethod
+    def _extract_url(self, webpage):
+        mobj = re.search(
+            '<iframe[^>]+src=(?P<q1>[\'"])(?P<url>(?:https?:)?//rudo\.video/vod/[0-9a-zA-Z]+)(?P=q1)',
+            webpage)
+        if mobj:
+            return mobj.group('url')
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id, encoding='iso-8859-1')
+
+        jwplayer_data = self._parse_json(self._search_regex(
+            r'(?s)playerInstance\.setup\(({.+?})\)', webpage, 'jwplayer data'), video_id,
+            transform_source=lambda s: js_to_json(re.sub(r'encodeURI\([^)]+\)', '""', s)))
+
+        info_dict = self._parse_jwplayer_data(
+            jwplayer_data, video_id, require_title=False, m3u8_id='hls')
+
+        info_dict.update({
+            'title': self._og_search_title(webpage),
+            'upload_date': unified_strdate(get_element_by_class('date', webpage)),
+        })
+
+        return info_dict

From b99af8a51cf359d7448740de7159383ff63cfe6c Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 11 Jul 2016 13:23:57 +0800
Subject: [PATCH 5495/6219] [biobiochiletv] Fix extraction and update _TESTS

---
 youtube_dl/extractor/biobiochiletv.py | 53 ++++++++++++---------------
 1 file changed, 24 insertions(+), 29 deletions(-)

diff --git a/youtube_dl/extractor/biobiochiletv.py b/youtube_dl/extractor/biobiochiletv.py
index 133228133..7608c0a08 100644
--- a/youtube_dl/extractor/biobiochiletv.py
+++ b/youtube_dl/extractor/biobiochiletv.py
@@ -2,11 +2,15 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..utils import remove_end
+from ..utils import (
+    ExtractorError,
+    remove_end,
+)
+from .rudo import RudoIE
 
 
 class BioBioChileTVIE(InfoExtractor):
-    _VALID_URL = r'https?://tv\.biobiochile\.cl/notas/(?:[^/]+/)+(?P<id>[^/]+)\.shtml'
+    _VALID_URL = r'https?://(?:tv|www)\.biobiochile\.cl/(?:notas|noticias)/(?:[^/]+/)+(?P<id>[^/]+)\.shtml'
 
     _TESTS = [{
         'url': 'http://tv.biobiochile.cl/notas/2015/10/21/sobre-camaras-y-camarillas-parlamentarias.shtml',
@@ -18,6 +22,7 @@ class BioBioChileTVIE(InfoExtractor):
             'thumbnail': 're:^https?://.*\.jpg$',
             'uploader': 'Fernando Atria',
         },
+        'skip': 'URL expired and redirected to http://www.biobiochile.cl/portada/bbtv/index.html',
     }, {
         # different uploader layout
         'url': 'http://tv.biobiochile.cl/notas/2016/03/18/natalia-valdebenito-repasa-a-diputado-hasbun-paso-a-la-categoria-de-hablar-brutalidades.shtml',
@@ -32,6 +37,16 @@ class BioBioChileTVIE(InfoExtractor):
         'params': {
             'skip_download': True,
         },
+        'skip': 'URL expired and redirected to http://www.biobiochile.cl/portada/bbtv/index.html',
+    }, {
+        'url': 'http://www.biobiochile.cl/noticias/bbtv/comentarios-bio-bio/2016/07/08/edecanes-del-congreso-figuras-decorativas-que-le-cuestan-muy-caro-a-los-chilenos.shtml',
+        'info_dict': {
+            'id': 'edecanes-del-congreso-figuras-decorativas-que-le-cuestan-muy-caro-a-los-chilenos',
+            'ext': 'mp4',
+            'uploader': '(none)',
+            'upload_date': '20160708',
+            'title': 'Edecanes del Congreso: Figuras decorativas que le cuestan muy caro a los chilenos',
+        },
     }, {
         'url': 'http://tv.biobiochile.cl/notas/2015/10/22/ninos-transexuales-de-quien-es-la-decision.shtml',
         'only_matching': True,
@@ -45,42 +60,22 @@ class BioBioChileTVIE(InfoExtractor):
 
         webpage = self._download_webpage(url, video_id)
 
+        rudo_url = RudoIE._extract_url(webpage)
+        if not rudo_url:
+            raise ExtractorError('No videos found')
+
         title = remove_end(self._og_search_title(webpage), ' - BioBioChile TV')
 
-        file_url = self._search_regex(
-            r'loadFWPlayerVideo\([^,]+,\s*(["\'])(?P<url>.+?)\1',
-            webpage, 'file url', group='url')
-
-        base_url = self._search_regex(
-            r'file\s*:\s*(["\'])(?P<url>.+?)\1\s*\+\s*fileURL', webpage,
-            'base url', default='http://unlimited2-cl.digitalproserver.com/bbtv/',
-            group='url')
-
-        formats = self._extract_m3u8_formats(
-            '%s%s/playlist.m3u8' % (base_url, file_url), video_id, 'mp4',
-            entry_protocol='m3u8_native', m3u8_id='hls', fatal=False)
-        f = {
-            'url': '%s%s' % (base_url, file_url),
-            'format_id': 'http',
-            'protocol': 'http',
-            'preference': 1,
-        }
-        if formats:
-            f_copy = formats[-1].copy()
-            f_copy.update(f)
-            f = f_copy
-        formats.append(f)
-        self._sort_formats(formats)
-
         thumbnail = self._og_search_thumbnail(webpage)
         uploader = self._html_search_regex(
-            r'<a[^>]+href=["\']https?://busca\.biobiochile\.cl/author[^>]+>(.+?)</a>',
+            r'<a[^>]+href=["\']https?://(?:busca|www)\.biobiochile\.cl/(?:lista/)?(?:author|autor)[^>]+>(.+?)</a>',
             webpage, 'uploader', fatal=False)
 
         return {
+            '_type': 'url_transparent',
+            'url': rudo_url,
             'id': video_id,
             'title': title,
             'thumbnail': thumbnail,
             'uploader': uploader,
-            'formats': formats,
         }

From 2a49d016002bbfb7f9d310aab45a122847bacee9 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 11 Jul 2016 15:15:28 +0800
Subject: [PATCH 5496/6219] [playvid] Update _TESTS

Blocks https://travis-ci.org/rg3/youtube-dl/jobs/143809100
---
 youtube_dl/extractor/playvid.py | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/playvid.py b/youtube_dl/extractor/playvid.py
index 2eb4fd96d..78d219299 100644
--- a/youtube_dl/extractor/playvid.py
+++ b/youtube_dl/extractor/playvid.py
@@ -15,7 +15,7 @@ from ..utils import (
 
 class PlayvidIE(InfoExtractor):
     _VALID_URL = r'https?://www\.playvid\.com/watch(\?v=|/)(?P<id>.+?)(?:#|$)'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.playvid.com/watch/RnmBNgtrrJu',
         'md5': 'ffa2f6b2119af359f544388d8c01eb6c',
         'info_dict': {
@@ -24,8 +24,19 @@ class PlayvidIE(InfoExtractor):
             'title': 'md5:9256d01c6317e3f703848b5906880dc8',
             'duration': 82,
             'age_limit': 18,
-        }
-    }
+        },
+        'skip': 'Video removed due to ToS',
+    }, {
+        'url': 'http://www.playvid.com/watch/hwb0GpNkzgH',
+        'md5': '39d49df503ad7b8f23a4432cbf046477',
+        'info_dict': {
+            'id': 'hwb0GpNkzgH',
+            'ext': 'mp4',
+            'title': 'Ellen Euro Cutie Blond Takes a Sexy Survey Get Facial in The Park',
+            'age_limit': 18,
+            'thumbnail': 're:^https?://.*\.jpg$',
+        },
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From 4a7a5e41f7c653cf2b2f8b9bf10987291a19c074 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 11 Jul 2016 14:51:44 +0100
Subject: [PATCH 5497/6219] [tvplay] improve extraction

---
 youtube_dl/extractor/tvplay.py | 123 +++++++++++++++++----------------
 1 file changed, 64 insertions(+), 59 deletions(-)

diff --git a/youtube_dl/extractor/tvplay.py b/youtube_dl/extractor/tvplay.py
index df70a6b23..918f8f8bc 100644
--- a/youtube_dl/extractor/tvplay.py
+++ b/youtube_dl/extractor/tvplay.py
@@ -8,43 +8,36 @@ from ..compat import compat_str
 from ..utils import (
     parse_iso8601,
     qualities,
+    determine_ext,
+    update_url_query,
+    int_or_none,
 )
 
 
 class TVPlayIE(InfoExtractor):
     IE_DESC = 'TV3Play and related services'
     _VALID_URL = r'''(?x)https?://(?:www\.)?
-        (?:tvplay\.lv/parraides|
-           tv3play\.lt/programos|
-           play\.tv3\.lt/programos|
-           tv3play\.ee/sisu|
-           tv3play\.se/program|
-           tv6play\.se/program|
-           tv8play\.se/program|
-           tv10play\.se/program|
-           tv3play\.no/programmer|
-           viasat4play\.no/programmer|
-           tv6play\.no/programmer|
-           tv3play\.dk/programmer|
+        (?:tvplay(?:\.skaties)?\.lv/parraides|
+           (?:tv3play|play\.tv3)\.lt/programos|
+           tv3play(?:\.tv3)?\.ee/sisu|
+           tv(?:3|6|8|10)play\.se/program|
+           (?:(?:tv3play|viasat4play|tv6play)\.no|tv3play\.dk)/programmer|
            play\.novatv\.bg/programi
         )/[^/]+/(?P<id>\d+)
         '''
     _TESTS = [
         {
             'url': 'http://www.tvplay.lv/parraides/vinas-melo-labak/418113?autostart=true',
+            'md5': 'a1612fe0849455423ad8718fe049be21',
             'info_dict': {
                 'id': '418113',
-                'ext': 'flv',
+                'ext': 'mp4',
                 'title': 'Kādi ir īri? - Viņas melo labāk',
                 'description': 'Baiba apsmej īrus, kādi tie ir un ko viņi dara.',
                 'duration': 25,
                 'timestamp': 1406097056,
                 'upload_date': '20140723',
             },
-            'params': {
-                # rtmp download
-                'skip_download': True,
-            },
         },
         {
             'url': 'http://play.tv3.lt/programos/moterys-meluoja-geriau/409229?autostart=true',
@@ -82,7 +75,7 @@ class TVPlayIE(InfoExtractor):
             'url': 'http://www.tv3play.se/program/husraddarna/395385?autostart=true',
             'info_dict': {
                 'id': '395385',
-                'ext': 'flv',
+                'ext': 'mp4',
                 'title': 'Husräddarna S02E07',
                 'description': 'md5:f210c6c89f42d4fc39faa551be813777',
                 'duration': 2574,
@@ -90,7 +83,6 @@ class TVPlayIE(InfoExtractor):
                 'upload_date': '20140520',
             },
             'params': {
-                # rtmp download
                 'skip_download': True,
             },
         },
@@ -98,7 +90,7 @@ class TVPlayIE(InfoExtractor):
             'url': 'http://www.tv6play.se/program/den-sista-dokusapan/266636?autostart=true',
             'info_dict': {
                 'id': '266636',
-                'ext': 'flv',
+                'ext': 'mp4',
                 'title': 'Den sista dokusåpan S01E08',
                 'description': 'md5:295be39c872520221b933830f660b110',
                 'duration': 1492,
@@ -107,7 +99,6 @@ class TVPlayIE(InfoExtractor):
                 'age_limit': 18,
             },
             'params': {
-                # rtmp download
                 'skip_download': True,
             },
         },
@@ -115,7 +106,7 @@ class TVPlayIE(InfoExtractor):
             'url': 'http://www.tv8play.se/program/antikjakten/282756?autostart=true',
             'info_dict': {
                 'id': '282756',
-                'ext': 'flv',
+                'ext': 'mp4',
                 'title': 'Antikjakten S01E10',
                 'description': 'md5:1b201169beabd97e20c5ad0ad67b13b8',
                 'duration': 2646,
@@ -123,7 +114,6 @@ class TVPlayIE(InfoExtractor):
                 'upload_date': '20120925',
             },
             'params': {
-                # rtmp download
                 'skip_download': True,
             },
         },
@@ -131,7 +121,7 @@ class TVPlayIE(InfoExtractor):
             'url': 'http://www.tv3play.no/programmer/anna-anka-soker-assistent/230898?autostart=true',
             'info_dict': {
                 'id': '230898',
-                'ext': 'flv',
+                'ext': 'mp4',
                 'title': 'Anna Anka søker assistent - Ep. 8',
                 'description': 'md5:f80916bf5bbe1c5f760d127f8dd71474',
                 'duration': 2656,
@@ -139,7 +129,6 @@ class TVPlayIE(InfoExtractor):
                 'upload_date': '20100628',
             },
             'params': {
-                # rtmp download
                 'skip_download': True,
             },
         },
@@ -147,7 +136,7 @@ class TVPlayIE(InfoExtractor):
             'url': 'http://www.viasat4play.no/programmer/budbringerne/21873?autostart=true',
             'info_dict': {
                 'id': '21873',
-                'ext': 'flv',
+                'ext': 'mp4',
                 'title': 'Budbringerne program 10',
                 'description': 'md5:4db78dc4ec8a85bb04fd322a3ee5092d',
                 'duration': 1297,
@@ -155,7 +144,6 @@ class TVPlayIE(InfoExtractor):
                 'upload_date': '20090929',
             },
             'params': {
-                # rtmp download
                 'skip_download': True,
             },
         },
@@ -163,7 +151,7 @@ class TVPlayIE(InfoExtractor):
             'url': 'http://www.tv6play.no/programmer/hotelinspektor-alex-polizzi/361883?autostart=true',
             'info_dict': {
                 'id': '361883',
-                'ext': 'flv',
+                'ext': 'mp4',
                 'title': 'Hotelinspektør Alex Polizzi - Ep. 10',
                 'description': 'md5:3ecf808db9ec96c862c8ecb3a7fdaf81',
                 'duration': 2594,
@@ -171,7 +159,6 @@ class TVPlayIE(InfoExtractor):
                 'upload_date': '20140224',
             },
             'params': {
-                # rtmp download
                 'skip_download': True,
             },
         },
@@ -191,6 +178,14 @@ class TVPlayIE(InfoExtractor):
                 'skip_download': True,
             },
         },
+        {
+            'url': 'http://tvplay.skaties.lv/parraides/vinas-melo-labak/418113?autostart=true',
+            'only_matching': True,
+        },
+        {
+            'url': 'http://tv3play.tv3.ee/sisu/kodu-keset-linna/238551?autostart=true',
+            'only_matching': True,
+        }
     ]
 
     def _real_extract(self, url):
@@ -199,7 +194,9 @@ class TVPlayIE(InfoExtractor):
         video = self._download_json(
             'http://playapi.mtgx.tv/v1/videos/%s' % video_id, video_id, 'Downloading video JSON')
 
-        if video['is_geo_blocked']:
+        title = video['title']
+
+        if video.get('is_geo_blocked'):
             self.report_warning(
                 'This content might not be available in your country due to copyright reasons')
 
@@ -208,42 +205,50 @@ class TVPlayIE(InfoExtractor):
 
         quality = qualities(['hls', 'medium', 'high'])
         formats = []
-        for format_id, video_url in streams['streams'].items():
+        for format_id, video_url in streams.get('streams', {}).items():
             if not video_url or not isinstance(video_url, compat_str):
                 continue
-            fmt = {
-                'format_id': format_id,
-                'preference': quality(format_id),
-            }
-            if video_url.startswith('rtmp'):
-                m = re.search(r'^(?P<url>rtmp://[^/]+/(?P<app>[^/]+))/(?P<playpath>.+)$', video_url)
-                if not m:
-                    continue
-                fmt.update({
-                    'ext': 'flv',
-                    'url': m.group('url'),
-                    'app': m.group('app'),
-                    'play_path': m.group('playpath'),
-                })
-            elif video_url.endswith('.f4m'):
+            ext = determine_ext(video_url)
+            if ext == 'f4m':
                 formats.extend(self._extract_f4m_formats(
-                    video_url + '?hdcore=3.5.0&plugin=aasp-3.5.0.151.81', video_id))
-                continue
+                    update_url_query(video_url, {
+                        'hdcore': '3.5.0',
+                        'plugin': 'aasp-3.5.0.151.81'
+                    }), video_id, f4m_id='hds', fatal=False))
+            elif ext == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    video_url, video_id, 'mp4', 'm3u8_native',
+                    m3u8_id='hls', fatal=False))
             else:
-                fmt.update({
-                    'url': video_url,
-                })
-            formats.append(fmt)
-
+                fmt = {
+                    'format_id': format_id,
+                    'quality': quality(format_id),
+                    'ext': ext,
+                }
+                if video_url.startswith('rtmp'):
+                    m = re.search(r'^(?P<url>rtmp://[^/]+/(?P<app>[^/]+))/(?P<playpath>.+)$', video_url)
+                    if not m:
+                        continue
+                    fmt.update({
+                        'ext': 'flv',
+                        'url': m.group('url'),
+                        'app': m.group('app'),
+                        'play_path': m.group('playpath'),
+                    })
+                else:
+                    fmt.update({
+                        'url': video_url,
+                    })
+                formats.append(fmt)
         self._sort_formats(formats)
 
         return {
             'id': video_id,
-            'title': video['title'],
-            'description': video['description'],
-            'duration': video['duration'],
-            'timestamp': parse_iso8601(video['created_at']),
-            'view_count': video['views']['total'],
-            'age_limit': video.get('age_limit', 0),
+            'title': title,
+            'description': video.get('description'),
+            'duration': int_or_none(video.get('duration')),
+            'timestamp': parse_iso8601(video.get('created_at')),
+            'view_count': int_or_none(video.get('views', {}).get('total')),
+            'age_limit': int_or_none(video.get('age_limit', 0)),
             'formats': formats,
         }

From 8a8590a61784b9f4a8ff380058a1571d5400ba3d Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 11 Jul 2016 16:30:24 +0100
Subject: [PATCH 5498/6219] [dbtv] delegate extraction to BrightcoveNewIE

---
 youtube_dl/extractor/dbtv.py | 69 ++++++++++--------------------------
 1 file changed, 19 insertions(+), 50 deletions(-)

diff --git a/youtube_dl/extractor/dbtv.py b/youtube_dl/extractor/dbtv.py
index 133cdc50b..caff8842e 100644
--- a/youtube_dl/extractor/dbtv.py
+++ b/youtube_dl/extractor/dbtv.py
@@ -4,78 +4,47 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_str
-from ..utils import (
-    float_or_none,
-    int_or_none,
-    clean_html,
-)
 
 
 class DBTVIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?dbtv\.no/(?:(?:lazyplayer|player)/)?(?P<id>[0-9]+)(?:#(?P<display_id>.+))?'
+    _VALID_URL = r'https?://(?:www\.)?dbtv\.no/(?:[^/]+/)?(?P<id>[0-9]+)(?:#(?P<display_id>.+))?'
     _TESTS = [{
         'url': 'http://dbtv.no/3649835190001#Skulle_teste_ut_fornøyelsespark,_men_kollegaen_var_bare_opptatt_av_bikinikroppen',
-        'md5': 'b89953ed25dacb6edb3ef6c6f430f8bc',
+        'md5': '2e24f67936517b143a234b4cadf792ec',
         'info_dict': {
-            'id': '33100',
+            'id': '3649835190001',
             'display_id': 'Skulle_teste_ut_fornøyelsespark,_men_kollegaen_var_bare_opptatt_av_bikinikroppen',
             'ext': 'mp4',
             'title': 'Skulle teste ut fornøyelsespark, men kollegaen var bare opptatt av bikinikroppen',
             'description': 'md5:1504a54606c4dde3e4e61fc97aa857e0',
-            'thumbnail': 're:https?://.*\.jpg$',
-            'timestamp': 1404039863.438,
+            'thumbnail': 're:https?://.*\.jpg',
+            'timestamp': 1404039863,
             'upload_date': '20140629',
             'duration': 69.544,
-            'view_count': int,
-            'categories': list,
-        }
+            'uploader_id': '1027729757001',
+        },
+        'add_ie': ['BrightcoveNew']
     }, {
         'url': 'http://dbtv.no/3649835190001',
         'only_matching': True,
     }, {
         'url': 'http://www.dbtv.no/lazyplayer/4631135248001',
         'only_matching': True,
+    }, {
+        'url': 'http://dbtv.no/vice/5000634109001',
+        'only_matching': True,
+    }, {
+        'url': 'http://dbtv.no/filmtrailer/3359293614001',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        display_id = mobj.group('display_id') or video_id
-
-        data = self._download_json(
-            'http://api.dbtv.no/discovery/%s' % video_id, display_id)
-
-        video = data['playlist'][0]
-
-        formats = [{
-            'url': f['URL'],
-            'vcodec': f.get('container'),
-            'width': int_or_none(f.get('width')),
-            'height': int_or_none(f.get('height')),
-            'vbr': float_or_none(f.get('rate'), 1000),
-            'filesize': int_or_none(f.get('size')),
-        } for f in video['renditions'] if 'URL' in f]
-
-        if not formats:
-            for url_key, format_id in [('URL', 'mp4'), ('HLSURL', 'hls')]:
-                if url_key in video:
-                    formats.append({
-                        'url': video[url_key],
-                        'format_id': format_id,
-                    })
-
-        self._sort_formats(formats)
+        video_id, display_id = re.match(self._VALID_URL, url).groups()
 
         return {
-            'id': compat_str(video['id']),
+            '_type': 'url_transparent',
+            'url': 'http://players.brightcove.net/1027729757001/default_default/index.html?videoId=%s' % video_id,
+            'id': video_id,
             'display_id': display_id,
-            'title': video['title'],
-            'description': clean_html(video['desc']),
-            'thumbnail': video.get('splash') or video.get('thumb'),
-            'timestamp': float_or_none(video.get('publishedAt'), 1000),
-            'duration': float_or_none(video.get('length'), 1000),
-            'view_count': int_or_none(video.get('views')),
-            'categories': video.get('tags'),
-            'formats': formats,
+            'ie_key': 'BrightcoveNew',
         }

From f1b4af7d79e5ee5d74fddf5fe46ce4c6f661ad8d Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 11 Jul 2016 19:06:50 +0100
Subject: [PATCH 5499/6219] [beightcove:new] remove html tags from description

---
 youtube_dl/extractor/brightcove.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 57ce0c174..c172bad2d 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -26,6 +26,7 @@ from ..utils import (
     unescapeHTML,
     unsmuggle_url,
     update_url_query,
+    clean_html,
 )
 
 
@@ -620,7 +621,7 @@ class BrightcoveNewIE(InfoExtractor):
         return {
             'id': video_id,
             'title': title,
-            'description': json_data.get('description'),
+            'description': clean_html(json_data.get('description')),
             'thumbnail': json_data.get('thumbnail') or json_data.get('poster'),
             'duration': float_or_none(json_data.get('duration'), 1000),
             'timestamp': parse_iso8601(json_data.get('published_at')),

From 7d1219f3e053cc38a577ff2370781c1019e10a1a Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 11 Jul 2016 19:08:22 +0100
Subject: [PATCH 5500/6219] [tmz] delegate extraction to KalturaIE

---
 youtube_dl/extractor/tmz.py | 28 ++++++++++++----------------
 1 file changed, 12 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/extractor/tmz.py b/youtube_dl/extractor/tmz.py
index 7dbe68b5c..979856e9a 100644
--- a/youtube_dl/extractor/tmz.py
+++ b/youtube_dl/extractor/tmz.py
@@ -5,31 +5,27 @@ from .common import InfoExtractor
 
 
 class TMZIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?tmz\.com/videos/(?P<id>[^/]+)/?'
-    _TEST = {
+    _VALID_URL = r'https?://(?:www\.)?tmz\.com/videos/(?P<id>[^/?#]+)'
+    _TESTS = [{
         'url': 'http://www.tmz.com/videos/0_okj015ty/',
-        'md5': '791204e3bf790b1426cb2db0706184c0',
+        'md5': '4d22a51ef205b6c06395d8394f72d560',
         'info_dict': {
             'id': '0_okj015ty',
-            'url': 'http://tmz.vo.llnwd.net/o28/2014-03/13/0_okj015ty_0_rt8ro3si_2.mp4',
             'ext': 'mp4',
             'title': 'Kim Kardashian\'s Boobs Unlock a Mystery!',
             'description': 'Did Kim Kardasain try to one-up Khloe by one-upping Kylie???  Or is she just showing off her amazing boobs?',
-            'thumbnail': r're:http://cdnbakmi\.kaltura\.com/.*thumbnail.*',
+            'timestamp': 1394747163,
+            'uploader_id': 'batchUser',
+            'upload_date': '20140313',
         }
-    }
+    }, {
+        'url': 'http://www.tmz.com/videos/0-cegprt2p/',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-
-        return {
-            'id': video_id,
-            'url': self._html_search_meta('VideoURL', webpage, fatal=True),
-            'title': self._og_search_title(webpage),
-            'description': self._og_search_description(webpage),
-            'thumbnail': self._html_search_meta('ThumbURL', webpage),
-        }
+        video_id = self._match_id(url).replace('-', '_')
+        return self.url_result('kaltura:591531:%s' % video_id, 'Kaltura', video_id)
 
 
 class TMZArticleIE(InfoExtractor):

From 7f29cf545a45db2bd5c2681a2fac782ca460b153 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 12 Jul 2016 02:10:35 +0700
Subject: [PATCH 5501/6219] [youtube] Add YouTube Red paid video reference test
 (#10059)

---
 youtube_dl/extractor/youtube.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 1687d5ef9..49c264c3a 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -858,6 +858,11 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         {
             'url': 'https://www.youtube.com/watch?feature=player_embedded&amp;amp;v=V36LpHqtcDY',
             'only_matching': True,
+        },
+        {
+            # YouTube Red paid video (https://github.com/rg3/youtube-dl/issues/10059)
+            'url': 'https://www.youtube.com/watch?v=i1Ko8UG-Tdo',
+            'only_matching': True,
         }
     ]
 

From 381ff44756ecf188de476a7a4cc9d4becf6633d1 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 12 Jul 2016 09:09:54 +0200
Subject: [PATCH 5502/6219] [devscripts/generate-download] Remove MD5 and SHA1

---
 devscripts/gh-pages/generate-download.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/devscripts/gh-pages/generate-download.py b/devscripts/gh-pages/generate-download.py
index 392e3ba21..fcd7e1dff 100755
--- a/devscripts/gh-pages/generate-download.py
+++ b/devscripts/gh-pages/generate-download.py
@@ -15,13 +15,9 @@ data = urllib.request.urlopen(URL).read()
 with open('download.html.in', 'r', encoding='utf-8') as tmplf:
     template = tmplf.read()
 
-md5sum = hashlib.md5(data).hexdigest()
-sha1sum = hashlib.sha1(data).hexdigest()
 sha256sum = hashlib.sha256(data).hexdigest()
 template = template.replace('@PROGRAM_VERSION@', version)
 template = template.replace('@PROGRAM_URL@', URL)
-template = template.replace('@PROGRAM_MD5SUM@', md5sum)
-template = template.replace('@PROGRAM_SHA1SUM@', sha1sum)
 template = template.replace('@PROGRAM_SHA256SUM@', sha256sum)
 template = template.replace('@EXE_URL@', versions_info['versions'][version]['exe'][0])
 template = template.replace('@EXE_SHA256SUM@', versions_info['versions'][version]['exe'][1])

From 41aa44259d3a0791b1a023a18c9a933f71e04c50 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 12 Jul 2016 23:15:38 +0100
Subject: [PATCH 5503/6219] [shahid] try to bypass geo restriction and extract
 more metadata(closes #10062)

---
 youtube_dl/extractor/shahid.py | 75 ++++++++++++----------------------
 1 file changed, 26 insertions(+), 49 deletions(-)

diff --git a/youtube_dl/extractor/shahid.py b/youtube_dl/extractor/shahid.py
index d95ea06be..ca286abb1 100644
--- a/youtube_dl/extractor/shahid.py
+++ b/youtube_dl/extractor/shahid.py
@@ -2,11 +2,11 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse_urlencode
 from ..utils import (
     ExtractorError,
     int_or_none,
     parse_iso8601,
+    str_or_none,
 )
 
 
@@ -33,45 +33,27 @@ class ShahidIE(InfoExtractor):
         'only_matching': True
     }]
 
-    def _handle_error(self, response):
-        if not isinstance(response, dict):
-            return
-        error = response.get('error')
+    def _call_api(self, path, video_id, note):
+        data = self._download_json(
+            'http://api.shahid.net/api/v1_1/' + path, video_id, note, query={
+                'apiKey': 'sh@hid0nlin3',
+                'hash': 'b2wMCTHpSmyxGqQjJFOycRmLSex+BpTK/ooxy6vHaqs=',
+            }).get('data', {})
+
+        error = data.get('error')
         if error:
             raise ExtractorError(
                 '%s returned error: %s' % (self.IE_NAME, '\n'.join(error.values())),
                 expected=True)
 
-    def _download_json(self, url, video_id, note='Downloading JSON metadata'):
-        response = super(ShahidIE, self)._download_json(url, video_id, note)['data']
-        self._handle_error(response)
-        return response
+        return data
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        webpage = self._download_webpage(url, video_id)
-
-        api_vars = {
-            'id': video_id,
-            'type': 'player',
-            'url': 'http://api.shahid.net/api/v1_1',
-            'playerType': 'episode',
-        }
-
-        flashvars = self._search_regex(
-            r'var\s+flashvars\s*=\s*({[^}]+})', webpage, 'flashvars', default=None)
-        if flashvars:
-            for key in api_vars.keys():
-                value = self._search_regex(
-                    r'\b%s\s*:\s*(?P<q>["\'])(?P<value>.+?)(?P=q)' % key,
-                    flashvars, 'type', default=None, group='value')
-                if value:
-                    api_vars[key] = value
-
-        player = self._download_json(
-            'https://shahid.mbc.net/arContent/getPlayerContent-param-.id-%s.type-%s.html'
-            % (video_id, api_vars['type']), video_id, 'Downloading player JSON')
+        player = self._call_api(
+            'Content/Episode/%s' % video_id,
+            video_id, 'Downloading player JSON')
 
         if player.get('drm'):
             raise ExtractorError('This video is DRM protected.', expected=True)
@@ -79,22 +61,11 @@ class ShahidIE(InfoExtractor):
         formats = self._extract_m3u8_formats(player['url'], video_id, 'mp4')
         self._sort_formats(formats)
 
-        video = self._download_json(
-            '%s/%s/%s?%s' % (
-                api_vars['url'], api_vars['playerType'], api_vars['id'],
-                compat_urllib_parse_urlencode({
-                    'apiKey': 'sh@hid0nlin3',
-                    'hash': 'b2wMCTHpSmyxGqQjJFOycRmLSex+BpTK/ooxy6vHaqs=',
-                })),
-            video_id, 'Downloading video JSON')
-
-        video = video[api_vars['playerType']]
+        video = self._call_api(
+            'episode/%s' % video_id, video_id,
+            'Downloading video JSON')['episode']
 
         title = video['title']
-        description = video.get('description')
-        thumbnail = video.get('thumbnailUrl')
-        duration = int_or_none(video.get('duration'))
-        timestamp = parse_iso8601(video.get('referenceDate'))
         categories = [
             category['name']
             for category in video.get('genres', []) if 'name' in category]
@@ -102,10 +73,16 @@ class ShahidIE(InfoExtractor):
         return {
             'id': video_id,
             'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
-            'duration': duration,
-            'timestamp': timestamp,
+            'description': video.get('description'),
+            'thumbnail': video.get('thumbnailUrl'),
+            'duration': int_or_none(video.get('duration')),
+            'timestamp': parse_iso8601(video.get('referenceDate')),
             'categories': categories,
+            'series': video.get('showTitle') or video.get('showName'),
+            'season': video.get('seasonTitle'),
+            'season_number': int_or_none(video.get('seasonNumber')),
+            'season_id': str_or_none(video.get('seasonId')),
+            'episode_number': int_or_none(video.get('number')),
+            'episode_id': video_id,
             'formats': formats,
         }

From 9d865a1af6a509fc8546a049d996b7a7d28313cf Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 13 Jul 2016 14:27:14 +0800
Subject: [PATCH 5504/6219] [travis] Skip downloading srelay

SOCKS tests never run on Travis CI due to unknown reasons, and
downloading them broke some tests (e.g.
https://travis-ci.org/rg3/youtube-dl/builds/144306425)
---
 .travis.yml                  | 3 ---
 devscripts/install_srelay.sh | 8 --------
 2 files changed, 11 deletions(-)
 delete mode 100755 devscripts/install_srelay.sh

diff --git a/.travis.yml b/.travis.yml
index 136c339f0..c74c9cc12 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -7,9 +7,6 @@ python:
   - "3.4"
   - "3.5"
 sudo: false
-install:
-  - bash ./devscripts/install_srelay.sh
-  - export PATH=$PATH:$(pwd)/tmp/srelay-0.4.8b6
 script: nosetests test --verbose
 notifications:
   email:
diff --git a/devscripts/install_srelay.sh b/devscripts/install_srelay.sh
deleted file mode 100755
index 33ce8a3f7..000000000
--- a/devscripts/install_srelay.sh
+++ /dev/null
@@ -1,8 +0,0 @@
-#!/bin/bash
-
-mkdir -p tmp && cd tmp
-wget -N http://downloads.sourceforge.net/project/socks-relay/socks-relay/srelay-0.4.8/srelay-0.4.8b6.tar.gz
-tar zxvf srelay-0.4.8b6.tar.gz
-cd srelay-0.4.8b6
-./configure
-make

From 2d19fb50725489d362d13e7347f90e0d2de10939 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 13 Jul 2016 21:51:44 +0700
Subject: [PATCH 5505/6219] [vk:wallpost] Add extractor

---
 youtube_dl/extractor/vk.py | 223 +++++++++++++++++++++++++++++--------
 1 file changed, 175 insertions(+), 48 deletions(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 758d9c86b..bcb7df83d 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -6,11 +6,18 @@ import json
 import sys
 
 from .common import InfoExtractor
-from ..compat import compat_str
+from ..compat import (
+    compat_str,
+    compat_urlparse,
+)
 from ..utils import (
+    clean_html,
     ExtractorError,
+    get_element_by_class,
     int_or_none,
     orderedSet,
+    parse_duration,
+    remove_start,
     str_to_int,
     unescapeHTML,
     unified_strdate,
@@ -20,7 +27,54 @@ from .vimeo import VimeoIE
 from .pladform import PladformIE
 
 
-class VKIE(InfoExtractor):
+class VKBaseIE(InfoExtractor):
+    _NETRC_MACHINE = 'vk'
+
+    def _login(self):
+        (username, password) = self._get_login_info()
+        if username is None:
+            return
+
+        login_page, url_handle = self._download_webpage_handle(
+            'https://vk.com', None, 'Downloading login page')
+
+        login_form = self._hidden_inputs(login_page)
+
+        login_form.update({
+            'email': username.encode('cp1251'),
+            'pass': password.encode('cp1251'),
+        })
+
+        # https://new.vk.com/ serves two same remixlhk cookies in Set-Cookie header
+        # and expects the first one to be set rather than second (see
+        # https://github.com/rg3/youtube-dl/issues/9841#issuecomment-227871201).
+        # As of RFC6265 the newer one cookie should be set into cookie store
+        # what actually happens.
+        # We will workaround this VK issue by resetting the remixlhk cookie to
+        # the first one manually.
+        cookies = url_handle.headers.get('Set-Cookie')
+        if sys.version_info[0] >= 3:
+            cookies = cookies.encode('iso-8859-1')
+        cookies = cookies.decode('utf-8')
+        remixlhk = re.search(r'remixlhk=(.+?);.*?\bdomain=(.+?)(?:[,;]|$)', cookies)
+        if remixlhk:
+            value, domain = remixlhk.groups()
+            self._set_cookie(domain, 'remixlhk', value)
+
+        login_page = self._download_webpage(
+            'https://login.vk.com/?act=login', None,
+            note='Logging in as %s' % username,
+            data=urlencode_postdata(login_form))
+
+        if re.search(r'onLoginFailed', login_page):
+            raise ExtractorError(
+                'Unable to login, incorrect username and/or password', expected=True)
+
+    def _real_initialize(self):
+        self._login()
+
+
+class VKIE(VKBaseIE):
     IE_NAME = 'vk'
     IE_DESC = 'VK'
     _VALID_URL = r'''(?x)
@@ -38,8 +92,6 @@ class VKIE(InfoExtractor):
                             (?P<videoid>-?\d+_\d+)(?:.*\blist=(?P<list_id>[\da-f]+))?
                         )
                     '''
-    _NETRC_MACHINE = 'vk'
-
     _TESTS = [
         {
             'url': 'http://vk.com/videos-77521?z=video-77521_162222515%2Fclub77521',
@@ -189,49 +241,6 @@ class VKIE(InfoExtractor):
         }
     ]
 
-    def _login(self):
-        (username, password) = self._get_login_info()
-        if username is None:
-            return
-
-        login_page, url_handle = self._download_webpage_handle(
-            'https://vk.com', None, 'Downloading login page')
-
-        login_form = self._hidden_inputs(login_page)
-
-        login_form.update({
-            'email': username.encode('cp1251'),
-            'pass': password.encode('cp1251'),
-        })
-
-        # https://new.vk.com/ serves two same remixlhk cookies in Set-Cookie header
-        # and expects the first one to be set rather than second (see
-        # https://github.com/rg3/youtube-dl/issues/9841#issuecomment-227871201).
-        # As of RFC6265 the newer one cookie should be set into cookie store
-        # what actually happens.
-        # We will workaround this VK issue by resetting the remixlhk cookie to
-        # the first one manually.
-        cookies = url_handle.headers.get('Set-Cookie')
-        if sys.version_info[0] >= 3:
-            cookies = cookies.encode('iso-8859-1')
-        cookies = cookies.decode('utf-8')
-        remixlhk = re.search(r'remixlhk=(.+?);.*?\bdomain=(.+?)(?:[,;]|$)', cookies)
-        if remixlhk:
-            value, domain = remixlhk.groups()
-            self._set_cookie(domain, 'remixlhk', value)
-
-        login_page = self._download_webpage(
-            'https://login.vk.com/?act=login', None,
-            note='Logging in as %s' % username,
-            data=urlencode_postdata(login_form))
-
-        if re.search(r'onLoginFailed', login_page):
-            raise ExtractorError(
-                'Unable to login, incorrect username and/or password', expected=True)
-
-    def _real_initialize(self):
-        self._login()
-
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('videoid')
@@ -355,7 +364,7 @@ class VKIE(InfoExtractor):
         }
 
 
-class VKUserVideosIE(InfoExtractor):
+class VKUserVideosIE(VKBaseIE):
     IE_NAME = 'vk:uservideos'
     IE_DESC = "VK - User's Videos"
     _VALID_URL = r'https?://(?:(?:m|new)\.)?vk\.com/videos(?P<id>-?[0-9]+)(?!\?.*\bz=video)(?:[/?#&]|$)'
@@ -396,3 +405,121 @@ class VKUserVideosIE(InfoExtractor):
             webpage, 'title', default=page_id))
 
         return self.playlist_result(entries, page_id, title)
+
+
+class VKWallPostIE(VKBaseIE):
+    IE_NAME = 'vk:wallpost'
+    _VALID_URL = r'https?://(?:(?:(?:(?:m|new)\.)?vk\.com/(?:[^?]+\?.*\bw=)?wall(?P<id>-?\d+_\d+)))'
+    _TESTS = [{
+        # public page URL, audio playlist
+        'url': 'https://vk.com/bs.official?w=wall-23538238_35',
+        'info_dict': {
+            'id': '23538238_35',
+            'title': 'Black Shadow - Wall post 23538238_35',
+            'description': 'md5:3f84b9c4f9ef499731cf1ced9998cc0c',
+        },
+        'playlist': [{
+            'md5': '5ba93864ec5b85f7ce19a9af4af080f6',
+            'info_dict': {
+                'id': '135220665_111806521',
+                'ext': 'mp3',
+                'title': 'Black Shadow - Слепое Верование',
+                'duration': 370,
+                'uploader': 'Black Shadow',
+                'artist': 'Black Shadow',
+                'track': 'Слепое Верование',
+            },
+        }, {
+            'md5': '4cc7e804579122b17ea95af7834c9233',
+            'info_dict': {
+                'id': '135220665_111802303',
+                'ext': 'mp3',
+                'title': 'Black Shadow - Война - Негасимое Бездны Пламя!',
+                'duration': 423,
+                'uploader': 'Black Shadow',
+                'artist': 'Black Shadow',
+                'track': 'Война - Негасимое Бездны Пламя!',
+            },
+            'params': {
+                'skip_download': True,
+            },
+        }],
+        'skip': 'Requires vk account credentials',
+    }, {
+        # single YouTube embed, no leading -
+        'url': 'https://vk.com/wall85155021_6319',
+        'info_dict': {
+            'id': '85155021_6319',
+            'title': 'Sergey Gorbunov - Wall post 85155021_6319',
+        },
+        'playlist_count': 1,
+        'skip': 'Requires vk account credentials',
+    }, {
+        # wall page URL
+        'url': 'https://vk.com/wall-23538238_35',
+        'only_matching': True,
+    }, {
+        # mobile wall page URL
+        'url': 'https://m.vk.com/wall-23538238_35',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        post_id = self._match_id(url)
+
+        wall_url = 'https://vk.com/wall%s' % post_id
+
+        post_id = remove_start(post_id, '-')
+
+        webpage = self._download_webpage(wall_url, post_id)
+
+        error = self._html_search_regex(
+            r'>Error</div>\s*<div[^>]+class=["\']body["\'][^>]*>([^<]+)',
+            webpage, 'error', default=None)
+        if error:
+            raise ExtractorError('VK said: %s' % error, expected=True)
+
+        description = clean_html(get_element_by_class('wall_post_text', webpage))
+        uploader = clean_html(get_element_by_class(
+            'fw_post_author', webpage)) or self._og_search_description(webpage)
+        thumbnail = self._og_search_thumbnail(webpage)
+
+        entries = []
+
+        for audio in re.finditer(r'''(?sx)
+                            <input[^>]+
+                                id=(?P<q1>["\'])audio_info(?P<id>\d+_\d+).*?(?P=q1)[^>]+
+                                value=(?P<q2>["\'])(?P<url>http.+?)(?P=q2)
+                                .+?
+                            </table>''', webpage):
+            audio_html = audio.group(0)
+            audio_id = audio.group('id')
+            duration = parse_duration(get_element_by_class('duration', audio_html))
+            track = self._html_search_regex(
+                r'<span[^>]+id=["\']title%s[^>]*>([^<]+)' % audio_id,
+                audio_html, 'title', default=None)
+            artist = self._html_search_regex(
+                r'>([^<]+)</a></b>\s*&ndash', audio_html,
+                'artist', default=None)
+            entries.append({
+                'id': audio_id,
+                'url': audio.group('url'),
+                'title': '%s - %s' % (artist, track) if artist and track else audio_id,
+                'thumbnail': thumbnail,
+                'duration': duration,
+                'uploader': uploader,
+                'artist': artist,
+                'track': track,
+            })
+
+        for video in re.finditer(
+                r'<a[^>]+href=(["\'])(?P<url>/video(?:-?[\d_]+).*?)\1', webpage):
+            entries.append(self.url_result(
+                compat_urlparse.urljoin(url, video.group('url')), VKIE.ie_key()))
+
+        title = 'Wall post %s' % post_id
+
+        return self.playlist_result(
+            orderedSet(entries), post_id,
+            '%s - %s' % (uploader, title) if uploader else title,
+            description)

From 5f5a9d615835110380075343786e58f78f5b08a7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 13 Jul 2016 21:52:52 +0700
Subject: [PATCH 5506/6219] [vk] Improve login

---
 youtube_dl/extractor/vk.py | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index bcb7df83d..3ee66e23e 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -53,13 +53,14 @@ class VKBaseIE(InfoExtractor):
         # We will workaround this VK issue by resetting the remixlhk cookie to
         # the first one manually.
         cookies = url_handle.headers.get('Set-Cookie')
-        if sys.version_info[0] >= 3:
-            cookies = cookies.encode('iso-8859-1')
-        cookies = cookies.decode('utf-8')
-        remixlhk = re.search(r'remixlhk=(.+?);.*?\bdomain=(.+?)(?:[,;]|$)', cookies)
-        if remixlhk:
-            value, domain = remixlhk.groups()
-            self._set_cookie(domain, 'remixlhk', value)
+        if cookies:
+            if sys.version_info[0] >= 3:
+                cookies = cookies.encode('iso-8859-1')
+            cookies = cookies.decode('utf-8')
+            remixlhk = re.search(r'remixlhk=(.+?);.*?\bdomain=(.+?)(?:[,;]|$)', cookies)
+            if remixlhk:
+                value, domain = remixlhk.groups()
+                self._set_cookie(domain, 'remixlhk', value)
 
         login_page = self._download_webpage(
             'https://login.vk.com/?act=login', None,

From 51c2cd0b83777f87fbc45b18f59a6f467717ba57 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 13 Jul 2016 21:53:23 +0700
Subject: [PATCH 5507/6219] [extractors] Add vk:wallpost extractor import

---
 youtube_dl/extractor/extractors.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index b08df41b4..916c8bb3e 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -989,6 +989,7 @@ from .viki import (
 from .vk import (
     VKIE,
     VKUserVideosIE,
+    VKWallPostIE,
 )
 from .vlive import VLiveIE
 from .vodlocker import VodlockerIE

From 00f4764cb7fdb39e3fd238440c373a2d7712d8d2 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 13 Jul 2016 15:54:43 +0100
Subject: [PATCH 5508/6219] [common] extract vbr, abr and fps for Unified
 Streaming Platform m3u8 manifests

---
 youtube_dl/extractor/common.py | 26 ++++++++++----------------
 youtube_dl/extractor/wat.py    | 12 +-----------
 2 files changed, 11 insertions(+), 27 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index df546da27..29544c1a8 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1207,6 +1207,7 @@ class InfoExtractor(object):
                     'url': format_url(line.strip()),
                     'tbr': tbr,
                     'ext': ext,
+                    'fps': float_or_none(last_info.get('FRAME-RATE')),
                     'protocol': entry_protocol,
                     'preference': preference,
                 }
@@ -1215,24 +1216,17 @@ class InfoExtractor(object):
                     width_str, height_str = resolution.split('x')
                     f['width'] = int(width_str)
                     f['height'] = int(height_str)
-                codecs = last_info.get('CODECS')
-                if codecs:
-                    vcodec, acodec = [None] * 2
-                    va_codecs = codecs.split(',')
-                    if len(va_codecs) == 1:
-                        # Audio only entries usually come with single codec and
-                        # no resolution. For more robustness we also check it to
-                        # be mp4 audio.
-                        if not resolution and va_codecs[0].startswith('mp4a'):
-                            vcodec, acodec = 'none', va_codecs[0]
-                        else:
-                            vcodec = va_codecs[0]
-                    else:
-                        vcodec, acodec = va_codecs[:2]
+                # Unified Streaming Platform
+                mobj = re.search(
+                    r'audio.*?(?:%3D|=)(\d+)(?:-video.*?(?:%3D|=)(\d+))?', f['url'])
+                if mobj:
+                    abr, vbr = mobj.groups()
+                    abr, vbr = float_or_none(abr, 1000), float_or_none(vbr, 1000)
                     f.update({
-                        'acodec': acodec,
-                        'vcodec': vcodec,
+                        'vbr': vbr,
+                        'abr': abr,
                     })
+                f.update(parse_codecs(last_info.get('CODECS')))
                 if last_media is not None:
                     f['m3u8_media'] = last_media
                     last_media = None
diff --git a/youtube_dl/extractor/wat.py b/youtube_dl/extractor/wat.py
index de7d6b559..48fc438ed 100644
--- a/youtube_dl/extractor/wat.py
+++ b/youtube_dl/extractor/wat.py
@@ -9,7 +9,6 @@ from ..utils import (
     ExtractorError,
     unified_strdate,
     HEADRequest,
-    float_or_none,
 )
 
 
@@ -95,16 +94,7 @@ class WatIE(InfoExtractor):
             m3u8_url.replace('ios.', 'web.').replace('.m3u8', '.f4m'),
             video_id, f4m_id='hds', fatal=False))
         for m3u8_format in m3u8_formats:
-            mobj = re.search(
-                r'audio.*?%3D(\d+)(?:-video.*?%3D(\d+))?', m3u8_format['url'])
-            if not mobj:
-                continue
-            abr, vbr = mobj.groups()
-            abr, vbr = float_or_none(abr, 1000), float_or_none(vbr, 1000)
-            m3u8_format.update({
-                'vbr': vbr,
-                'abr': abr,
-            })
+            vbr, abr = m3u8_format.get('vbr'), m3u8_format.get('abr')
             if not vbr or not abr:
                 continue
             f = m3u8_format.copy()

From 0385aa6199206e4ba7745efec73be26c5826286a Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 13 Jul 2016 15:57:50 +0100
Subject: [PATCH 5509/6219] [bbc] extract more and better qulities from Unified
 Streaming Platform m3u8 manifests

---
 youtube_dl/extractor/bbc.py | 49 ++++++++++++++++++++-----------------
 1 file changed, 27 insertions(+), 22 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 4b3cd8c65..50c1da185 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -44,6 +44,8 @@ class BBCCoUkIE(InfoExtractor):
 
     _MEDIASELECTION_NS = 'http://bbc.co.uk/2008/mp/mediaselection'
     _EMP_PLAYLIST_NS = 'http://bbc.co.uk/2008/emp/playlist'
+    # Unified Streaming Platform
+    _USP_RE = r'/([^/]+)\.ism(?:\.hlsv2\.ism)?/[^/]+\.m3u8'
 
     _NAMESPACES = (
         _MEDIASELECTION_NS,
@@ -55,12 +57,11 @@ class BBCCoUkIE(InfoExtractor):
             'url': 'http://www.bbc.co.uk/programmes/b039g8p7',
             'info_dict': {
                 'id': 'b039d07m',
-                'ext': 'flv',
+                'ext': 'mp4',
                 'title': 'Leonard Cohen, Kaleidoscope - BBC Radio 4',
                 'description': 'The Canadian poet and songwriter reflects on his musical career.',
             },
             'params': {
-                # rtmp download
                 'skip_download': True,
             }
         },
@@ -92,7 +93,7 @@ class BBCCoUkIE(InfoExtractor):
                 # rtmp download
                 'skip_download': True,
             },
-            'skip': 'Currently BBC iPlayer TV programmes are available to play in the UK only',
+            'skip': 'this episode is not currently available',
         },
         {
             'url': 'http://www.bbc.co.uk/iplayer/episode/p026c7jt/tomorrows-worlds-the-unearthly-history-of-science-fiction-2-invasion',
@@ -107,7 +108,7 @@ class BBCCoUkIE(InfoExtractor):
                 # rtmp download
                 'skip_download': True,
             },
-            'skip': 'Currently BBC iPlayer TV programmes are available to play in the UK only',
+            'skip': 'this episode is not currently available',
         }, {
             'url': 'http://www.bbc.co.uk/programmes/b04v20dw',
             'info_dict': {
@@ -127,13 +128,12 @@ class BBCCoUkIE(InfoExtractor):
             'note': 'Audio',
             'info_dict': {
                 'id': 'p022h44j',
-                'ext': 'flv',
+                'ext': 'mp4',
                 'title': 'BBC Proms Music Guides, Rachmaninov: Symphonic Dances',
                 'description': "In this Proms Music Guide, Andrew McGregor looks at Rachmaninov's Symphonic Dances.",
                 'duration': 227,
             },
             'params': {
-                # rtmp download
                 'skip_download': True,
             }
         }, {
@@ -141,13 +141,12 @@ class BBCCoUkIE(InfoExtractor):
             'note': 'Video',
             'info_dict': {
                 'id': 'p025c103',
-                'ext': 'flv',
+                'ext': 'mp4',
                 'title': 'Reading and Leeds Festival, 2014, Rae Morris - Closer (Live on BBC Three)',
                 'description': 'Rae Morris performs Closer for BBC Three at Reading 2014',
                 'duration': 226,
             },
             'params': {
-                # rtmp download
                 'skip_download': True,
             }
         }, {
@@ -163,7 +162,7 @@ class BBCCoUkIE(InfoExtractor):
                 # rtmp download
                 'skip_download': True,
             },
-            'skip': 'geolocation',
+            'skip': 'this episode is not currently available',
         }, {
             'url': 'http://www.bbc.co.uk/iplayer/episode/b05zmgwn/royal-academy-summer-exhibition',
             'info_dict': {
@@ -177,7 +176,7 @@ class BBCCoUkIE(InfoExtractor):
                 # rtmp download
                 'skip_download': True,
             },
-            'skip': 'geolocation',
+            'skip': 'this episode is not currently available',
         }, {
             # iptv-all mediaset fails with geolocation however there is no geo restriction
             # for this programme at all
@@ -192,18 +191,17 @@ class BBCCoUkIE(InfoExtractor):
                 # rtmp download
                 'skip_download': True,
             },
-            'skip': 'Now it\'s really geo-restricted',
+            'skip': 'this episode is not currently available on BBC iPlayer Radio',
         }, {
             # compact player (https://github.com/rg3/youtube-dl/issues/8147)
             'url': 'http://www.bbc.co.uk/programmes/p028bfkf/player',
             'info_dict': {
                 'id': 'p028bfkj',
-                'ext': 'flv',
+                'ext': 'mp4',
                 'title': 'Extract from BBC documentary Look Stranger - Giant Leeks and Magic Brews',
                 'description': 'Extract from BBC documentary Look Stranger - Giant Leeks and Magic Brews',
             },
             'params': {
-                # rtmp download
                 'skip_download': True,
             },
         }, {
@@ -248,9 +246,15 @@ class BBCCoUkIE(InfoExtractor):
             elif transfer_format == 'dash':
                 pass
             elif transfer_format == 'hls':
-                formats.extend(self._extract_m3u8_formats(
+                is_unified_streaming = re.search(self._USP_RE, href)
+                if is_unified_streaming:
+                    href = re.sub(self._USP_RE, r'/\1.ism/\1.m3u8', href)
+                m3u8_formats = self._extract_m3u8_formats(
                     href, programme_id, ext='mp4', entry_protocol='m3u8_native',
-                    m3u8_id=supplier, fatal=False))
+                    m3u8_id=supplier, fatal=False)
+                if is_unified_streaming:
+                    self._check_formats(m3u8_formats, programme_id)
+                formats.extend(m3u8_formats)
             # Direct link
             else:
                 formats.append({
@@ -305,13 +309,14 @@ class BBCCoUkIE(InfoExtractor):
         for connection in self._extract_connections(media):
             conn_formats = self._extract_connection(connection, programme_id)
             for format in conn_formats:
-                format.update({
-                    'width': width,
-                    'height': height,
-                    'vbr': vbr,
-                    'vcodec': vcodec,
-                    'filesize': file_size,
-                })
+                if format.get('protocol') != 'm3u8_native':
+                    format.update({
+                        'width': width,
+                        'height': height,
+                        'vbr': vbr,
+                        'vcodec': vcodec,
+                        'filesize': file_size,
+                    })
                 if service:
                     format['format_id'] = '%s_%s' % (service, format['format_id'])
             formats.extend(conn_formats)

From a0560d8ab83a565d502e160ccd993317a0ed69ef Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 13 Jul 2016 22:41:46 +0700
Subject: [PATCH 5510/6219] [ellentv] Improve extraction (Closes #10067)

---
 youtube_dl/extractor/ellentv.py | 50 ++++++++++++++++++++++++---------
 1 file changed, 37 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/ellentv.py b/youtube_dl/extractor/ellentv.py
index 4c8190d68..74bbc5c51 100644
--- a/youtube_dl/extractor/ellentv.py
+++ b/youtube_dl/extractor/ellentv.py
@@ -6,12 +6,13 @@ import json
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
+    NO_DEFAULT,
 )
 
 
 class EllenTVIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?(?:ellentv|ellentube)\.com/videos/(?P<id>[a-z0-9_-]+)'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.ellentv.com/videos/0-ipq1gsai/',
         'md5': '4294cf98bc165f218aaa0b89e0fd8042',
         'info_dict': {
@@ -22,24 +23,47 @@ class EllenTVIE(InfoExtractor):
             'timestamp': 1428035648,
             'upload_date': '20150403',
             'uploader_id': 'batchUser',
-        }
-    }
+        },
+    }, {
+        # not available via http://widgets.ellentube.com/
+        'url': 'http://www.ellentv.com/videos/1-szkgu2m2/',
+        'info_dict': {
+            'id': '1_szkgu2m2',
+            'ext': 'flv',
+            'title': "Ellen's Amazingly Talented Audience",
+            'description': 'md5:86ff1e376ff0d717d7171590e273f0a5',
+            'timestamp': 1255140900,
+            'upload_date': '20091010',
+            'uploader_id': 'ellenkaltura@gmail.com',
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        webpage = self._download_webpage(
-            'http://widgets.ellentube.com/videos/%s' % video_id,
-            video_id)
+        URLS = ('http://widgets.ellentube.com/videos/%s' % video_id, url)
 
-        partner_id = self._search_regex(
-            r"var\s+partnerId\s*=\s*'([^']+)", webpage, 'partner id')
+        for num, url_ in enumerate(URLS, 1):
+            webpage = self._download_webpage(
+                url_, video_id, fatal=num == len(URLS))
 
-        kaltura_id = self._search_regex(
-            [r'id="kaltura_player_([^"]+)"',
-             r"_wb_entry_id\s*:\s*'([^']+)",
-             r'data-kaltura-entry-id="([^"]+)'],
-            webpage, 'kaltura id')
+            default = NO_DEFAULT if num == len(URLS) else None
+
+            partner_id = self._search_regex(
+                r"var\s+partnerId\s*=\s*'([^']+)", webpage, 'partner id',
+                default=default)
+
+            kaltura_id = self._search_regex(
+                [r'id="kaltura_player_([^"]+)"',
+                 r"_wb_entry_id\s*:\s*'([^']+)",
+                 r'data-kaltura-entry-id="([^"]+)'],
+                webpage, 'kaltura id', default=default)
+
+            if partner_id and kaltura_id:
+                break
 
         return self.url_result('kaltura:%s:%s' % (partner_id, kaltura_id), 'Kaltura')
 

From c485959034485f529985c0bea933951af3f4a63e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 13 Jul 2016 23:58:01 +0700
Subject: [PATCH 5511/6219] release 2016.07.13

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 docs/supportedsites.md    | 2 ++
 youtube_dl/version.py     | 2 +-
 3 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 704a8b911..c5898701f 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.07.11*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.07.11**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.07.13*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.07.13**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.07.11
+[debug] youtube-dl version 2016.07.13
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 5bcd6de1c..282bd0e6b 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -568,6 +568,7 @@
  - **rtve.es:infantil**: RTVE infantil
  - **rtve.es:live**: RTVE.es live streams
  - **RTVNH**
+ - **Rudo**
  - **RUHD**
  - **RulePorn**
  - **rutube**: Rutube videos
@@ -794,6 +795,7 @@
  - **vine:user**
  - **vk**: VK
  - **vk:uservideos**: VK - User's Videos
+ - **vk:wallpost**
  - **vlive**
  - **Vodlocker**
  - **VoiceRepublic**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index d60480223..56f9f5986 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.07.11'
+__version__ = '2016.07.13'

From 35ec86689c95fdded47ea65352392516951315c3 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 13 Jul 2016 17:39:56 +0100
Subject: [PATCH 5512/6219] [bbc] extract only the original Unified Streaming
 Platform m3u8 manifests

https://github.com/rg3/youtube-dl/commit/0385aa6199206e4ba7745efec73be26c5826286a#commitcomment-18233275
manifests with higher birate require more time to check formats
---
 youtube_dl/extractor/bbc.py | 14 +++-----------
 1 file changed, 3 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 50c1da185..23c6e505b 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -44,8 +44,6 @@ class BBCCoUkIE(InfoExtractor):
 
     _MEDIASELECTION_NS = 'http://bbc.co.uk/2008/mp/mediaselection'
     _EMP_PLAYLIST_NS = 'http://bbc.co.uk/2008/emp/playlist'
-    # Unified Streaming Platform
-    _USP_RE = r'/([^/]+)\.ism(?:\.hlsv2\.ism)?/[^/]+\.m3u8'
 
     _NAMESPACES = (
         _MEDIASELECTION_NS,
@@ -246,15 +244,9 @@ class BBCCoUkIE(InfoExtractor):
             elif transfer_format == 'dash':
                 pass
             elif transfer_format == 'hls':
-                is_unified_streaming = re.search(self._USP_RE, href)
-                if is_unified_streaming:
-                    href = re.sub(self._USP_RE, r'/\1.ism/\1.m3u8', href)
-                m3u8_formats = self._extract_m3u8_formats(
-                    href, programme_id, ext='mp4', entry_protocol='m3u8_native',
-                    m3u8_id=supplier, fatal=False)
-                if is_unified_streaming:
-                    self._check_formats(m3u8_formats, programme_id)
-                formats.extend(m3u8_formats)
+                formats.extend(self._extract_m3u8_formats(
+                    href, programme_id, 'mp4', 'm3u8_native',
+                    m3u8_id=supplier, fatal=False))
             # Direct link
             else:
                 formats.append({

From c39b2ed990105e640456f126321ef3d771884405 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 13 Jul 2016 20:21:00 +0200
Subject: [PATCH 5513/6219] [rtve] Fix extraction (#10076)

For http://www.rtve.es/alacarta/videos/documentos-tv/documentos-tv-revolucion-del-movil/3069778/ using 'auth/resources' fails, and other URLs seem to work fine.
---
 youtube_dl/extractor/rtve.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/rtve.py b/youtube_dl/extractor/rtve.py
index f11e3588b..de35e07d9 100644
--- a/youtube_dl/extractor/rtve.py
+++ b/youtube_dl/extractor/rtve.py
@@ -113,9 +113,7 @@ class RTVEALaCartaIE(InfoExtractor):
         png = self._download_webpage(png_request, video_id, 'Downloading url information')
         video_url = _decrypt_url(png)
         if not video_url.endswith('.f4m'):
-            video_url = video_url.replace(
-                'resources/', 'auth/resources/'
-            ).replace('.net.rtve', '.multimedia.cdn.rtve')
+            video_url = video_url.replace('.net.rtve', '.multimedia.cdn.rtve')
 
         subtitles = None
         if info.get('sbtFile') is not None:

From 233b58dec736205d9bf8c652063b5dba6073631f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 13 Jul 2016 21:02:34 +0200
Subject: [PATCH 5514/6219] Add extractor for rtve.es/television (fixes #10076)

---
 youtube_dl/extractor/extractors.py |  2 +-
 youtube_dl/extractor/rtve.py       | 31 ++++++++++++++++++++++++++++++
 2 files changed, 32 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 916c8bb3e..71baee90d 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -689,7 +689,7 @@ from .rtlnl import RtlNlIE
 from .rtl2 import RTL2IE
 from .rtp import RTPIE
 from .rts import RTSIE
-from .rtve import RTVEALaCartaIE, RTVELiveIE, RTVEInfantilIE
+from .rtve import RTVEALaCartaIE, RTVELiveIE, RTVEInfantilIE, RTVELiveIE, RTVETelevisionIE
 from .rtvnh import RTVNHIE
 from .rudo import RudoIE
 from .ruhd import RUHDIE
diff --git a/youtube_dl/extractor/rtve.py b/youtube_dl/extractor/rtve.py
index de35e07d9..d33b05f5d 100644
--- a/youtube_dl/extractor/rtve.py
+++ b/youtube_dl/extractor/rtve.py
@@ -220,3 +220,34 @@ class RTVELiveIE(InfoExtractor):
             'formats': formats,
             'is_live': True,
         }
+
+
+class RTVETelevisionIE(InfoExtractor):
+    IE_NAME = 'rtve.es:television'
+    _VALID_URL = r'https?://www\.rtve\.es/television/[^/]+/[^/]+/(?P<id>\d+).shtml'
+
+    _TEST = {
+        'url': 'http://www.rtve.es/television/20160628/revolucion-del-movil/1364141.shtml',
+        'info_dict': {
+            'id': '3069778',
+            'ext': 'mp4',
+            'title': 'Documentos TV - La revolución del móvil',
+            'duration': 3496.948,
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }
+
+    def _real_extract(self, url):
+        page_id = self._match_id(url)
+        webpage = self._download_webpage(url, page_id)
+
+        alacarta_url = self._search_regex(
+            r'data-location="alacarta_videos"[^<]+url&quot;:&quot;(http://www\.rtve\.es/alacarta.+?)&',
+            webpage, 'alacarta url', default=None)
+        if alacarta_url is None:
+            raise ExtractorError(
+                'The webpage doesn\'t contain any video', expected=True)
+
+        return self.url_result(alacarta_url, ie=RTVEALaCartaIE.ie_key())

From e910fe2fe439df9aa9cf8ca1253fe6db5839024a Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 14 Jul 2016 14:13:57 +0100
Subject: [PATCH 5515/6219] [brightcove] skip ism manifests

---
 youtube_dl/extractor/brightcove.py | 11 +++++++----
 youtube_dl/utils.py                |  1 +
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index c172bad2d..aeb22be16 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -27,6 +27,7 @@ from ..utils import (
     unsmuggle_url,
     update_url_query,
     clean_html,
+    mimetype2ext,
 )
 
 
@@ -545,14 +546,16 @@ class BrightcoveNewIE(InfoExtractor):
         formats = []
         for source in json_data.get('sources', []):
             container = source.get('container')
-            source_type = source.get('type')
+            ext = mimetype2ext(source.get('type'))
             src = source.get('src')
-            if source_type == 'application/x-mpegURL' or container == 'M2TS':
+            if ext == 'ism':
+                continue
+            elif ext == 'm3u8' or container == 'M2TS':
                 if not src:
                     continue
                 formats.extend(self._extract_m3u8_formats(
                     src, video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
-            elif source_type == 'application/dash+xml':
+            elif ext == 'mpd':
                 if not src:
                     continue
                 formats.extend(self._extract_mpd_formats(src, video_id, 'dash', fatal=False))
@@ -568,7 +571,7 @@ class BrightcoveNewIE(InfoExtractor):
                     'tbr': tbr,
                     'filesize': int_or_none(source.get('size')),
                     'container': container,
-                    'ext': container.lower(),
+                    'ext': ext or container.lower(),
                 }
                 if width == 0 and height == 0:
                     f.update({
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 4c1d0d526..e6e0155b4 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2123,6 +2123,7 @@ def mimetype2ext(mt):
         'dash+xml': 'mpd',
         'f4m': 'f4m',
         'f4m+xml': 'f4m',
+        'vnd.ms-sstr+xml': 'ism',
     }.get(res, res)
 
 

From 38e0f16a94b1790ed515fcd12c8bbcb58f2e4b53 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 14 Jul 2016 14:16:11 +0100
Subject: [PATCH 5516/6219] [ninenow] Add new extractor(closes #5181)

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/ninenow.py    | 72 ++++++++++++++++++++++++++++++
 2 files changed, 73 insertions(+)
 create mode 100644 youtube_dl/extractor/ninenow.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 71baee90d..45817d7df 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -537,6 +537,7 @@ from .nick import (
 from .niconico import NiconicoIE, NiconicoPlaylistIE
 from .ninecninemedia import NineCNineMediaIE
 from .ninegag import NineGagIE
+from .ninenow import NineNowIE
 from .noco import NocoIE
 from .normalboots import NormalbootsIE
 from .nosvideo import NosVideoIE
diff --git a/youtube_dl/extractor/ninenow.py b/youtube_dl/extractor/ninenow.py
new file mode 100644
index 000000000..f54e74de3
--- /dev/null
+++ b/youtube_dl/extractor/ninenow.py
@@ -0,0 +1,72 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..compat import compat_str
+from ..utils import (
+    int_or_none,
+    float_or_none,
+    ExtractorError,
+)
+
+
+class NineNowIE(InfoExtractor):
+    IE_NAME = '9now.com.au'
+    _VALID_URL = r'https?://(?:www\.)?9now\.com\.au/(?:[^/]+/){2}(?P<id>[^/?#]+)'
+    _TESTS = [{
+        # clip
+        'url': 'https://www.9now.com.au/afl-footy-show/2016/clip-ciql02091000g0hp5oktrnytc',
+        'md5': '17cf47d63ec9323e562c9957a968b565',
+        'info_dict': {
+            'id': '16801',
+            'ext': 'mp4',
+            'title': 'St. Kilda\'s Joey Montagna on the potential for a player\'s strike',
+            'description': 'Is a boycott of the NAB Cup "on the table"?',
+            'uploader_id': '4460760524001',
+            'upload_date': '20160713',
+            'timestamp': 1468421266,
+        },
+        'skip': 'Only available in Australia',
+    }, {
+        # episode
+        'url': 'https://www.9now.com.au/afl-footy-show/2016/episode-19',
+        'only_matching': True,
+    }, {
+        # DRM protected
+        'url': 'https://www.9now.com.au/afl-footy-show/2016/episode-19',
+        'only_matching': True,
+    }]
+    BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/4460760524001/default_default/index.html?videoId=%s'
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+        page_data = self._parse_json(self._search_regex(
+            r'window\.__data\s*=\s*({.*?});', webpage,
+            'page data'), display_id)
+        common_data = page_data.get('episode', {}).get('episode') or page_data.get('clip', {}).get('clip')
+        video_data = common_data['video']
+
+        if video_data.get('drm'):
+            raise ExtractorError('This video is DRM protected.', expected=True)
+
+        brightcove_id = video_data.get('brightcoveId') or 'ref:' + video_data['referenceId']
+        video_id = compat_str(video_data.get('id') or brightcove_id)
+        title = common_data['name']
+
+        thumbnails = [{
+            'id': thumbnail_id,
+            'url': thumbnail_url,
+            'width': int_or_none(thumbnail_id[1:])
+        } for thumbnail_id, thumbnail_url in common_data.get('image', {}).get('sizes', {}).items()]
+
+        return {
+            '_type': 'url_transparent',
+            'url': self.BRIGHTCOVE_URL_TEMPLATE % brightcove_id,
+            'id': video_id,
+            'title': title,
+            'description': common_data.get('description'),
+            'duration': float_or_none(video_data.get('duration'), 1000),
+            'thumbnails': thumbnails,
+            'ie_key': 'BrightcoveNew',
+        }

From 342f0c3682885ad4c7d709686b2a38a466dd05a0 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 14 Jul 2016 14:19:18 +0100
Subject: [PATCH 5517/6219] [ninenow] correct test url

---
 youtube_dl/extractor/ninenow.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ninenow.py b/youtube_dl/extractor/ninenow.py
index f54e74de3..faa577237 100644
--- a/youtube_dl/extractor/ninenow.py
+++ b/youtube_dl/extractor/ninenow.py
@@ -33,7 +33,7 @@ class NineNowIE(InfoExtractor):
         'only_matching': True,
     }, {
         # DRM protected
-        'url': 'https://www.9now.com.au/afl-footy-show/2016/episode-19',
+        'url': 'https://www.9now.com.au/andrew-marrs-history-of-the-world/season-1/episode-1',
         'only_matching': True,
     }]
     BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/4460760524001/default_default/index.html?videoId=%s'

From 7b0d333a7e7f13858cc0a24a89dc32b07a2a84f3 Mon Sep 17 00:00:00 2001
From: Nehal Patel <nehalvpatels+github@gmail.com>
Date: Thu, 7 Jul 2016 16:39:39 -0500
Subject: [PATCH 5518/6219] Fix unit tests for m3u8 and RTSP extractors that
 require ffmpeg or mplayer

---
 youtube_dl/extractor/cspan.py     |  5 ++++-
 youtube_dl/extractor/dreisat.py   |  6 +++++-
 youtube_dl/extractor/gamespot.py  |  7 +++++--
 youtube_dl/extractor/orf.py       | 21 +++++++++++----------
 youtube_dl/extractor/threeqsdn.py |  8 ++++++--
 5 files changed, 31 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/extractor/cspan.py b/youtube_dl/extractor/cspan.py
index 84b36f44c..7e5d4f227 100644
--- a/youtube_dl/extractor/cspan.py
+++ b/youtube_dl/extractor/cspan.py
@@ -51,8 +51,11 @@ class CSpanIE(InfoExtractor):
         'url': 'http://www.c-span.org/video/?104517-1/immigration-reforms-needed-protect-skilled-american-workers',
         'info_dict': {
             'id': 'judiciary031715',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Immigration Reforms Needed to Protect Skilled American Workers',
+        },
+        'params': {
+            'skip_download': True,  # m3u8 downloads
         }
     }]
 
diff --git a/youtube_dl/extractor/dreisat.py b/youtube_dl/extractor/dreisat.py
index 0040e70d4..908c9e514 100644
--- a/youtube_dl/extractor/dreisat.py
+++ b/youtube_dl/extractor/dreisat.py
@@ -17,8 +17,12 @@ class DreiSatIE(ZDFIE):
                 'ext': 'mp4',
                 'title': 'Waidmannsheil',
                 'description': 'md5:cce00ca1d70e21425e72c86a98a56817',
-                'uploader': '3sat',
+                'uploader': 'SCHWEIZWEIT',
+                'uploader_id': '100000210',
                 'upload_date': '20140913'
+            },
+            'params': {
+                'skip_download': True,  # m3u8 downloads
             }
         },
         {
diff --git a/youtube_dl/extractor/gamespot.py b/youtube_dl/extractor/gamespot.py
index 621257c9f..4e859e09a 100644
--- a/youtube_dl/extractor/gamespot.py
+++ b/youtube_dl/extractor/gamespot.py
@@ -28,10 +28,13 @@ class GameSpotIE(OnceIE):
         'url': 'http://www.gamespot.com/videos/the-witcher-3-wild-hunt-xbox-one-now-playing/2300-6424837/',
         'info_dict': {
             'id': 'gs-2300-6424837',
-            'ext': 'flv',
-            'title': 'The Witcher 3: Wild Hunt [Xbox ONE]  - Now Playing',
+            'ext': 'mp4',
+            'title': 'Now Playing - The Witcher 3: Wild Hunt',
             'description': 'Join us as we take a look at the early hours of The Witcher 3: Wild Hunt and more.',
         },
+        'params': {
+            'skip_download': True,  # m3u8 downloads
+        },
     }]
 
     def _real_extract(self, url):
diff --git a/youtube_dl/extractor/orf.py b/youtube_dl/extractor/orf.py
index 4e3864f0d..42d4332e8 100644
--- a/youtube_dl/extractor/orf.py
+++ b/youtube_dl/extractor/orf.py
@@ -40,16 +40,17 @@ class ORFTVthekIE(InfoExtractor):
         'skip': 'Blocked outside of Austria / Germany',
     }, {
         'url': 'http://tvthek.orf.at/topic/Im-Wandel-der-Zeit/8002126/Best-of-Ingrid-Thurnher/7982256',
-        'playlist': [{
-            'md5': '68f543909aea49d621dfc7703a11cfaf',
-            'info_dict': {
-                'id': '7982259',
-                'ext': 'mp4',
-                'title': 'Best of Ingrid Thurnher',
-                'upload_date': '20140527',
-                'description': 'Viele Jahre war Ingrid Thurnher das "Gesicht" der ZIB 2. Vor ihrem Wechsel zur ZIB 2 im jahr 1995 moderierte sie unter anderem "Land und Leute", "Österreich-Bild" und "Niederösterreich heute".',
-            }
-        }],
+        'md5': '68f543909aea49d621dfc7703a11cfaf',
+        'info_dict': {
+            'id': '7982259',
+            'ext': 'mp4',
+            'title': 'Best of Ingrid Thurnher',
+            'upload_date': '20140527',
+            'description': 'Viele Jahre war Ingrid Thurnher das "Gesicht" der ZIB 2. Vor ihrem Wechsel zur ZIB 2 im Jahr 1995 moderierte sie unter anderem "Land und Leute", "Österreich-Bild" und "Niederösterreich heute".',
+        },
+        'params': {
+            'skip_download': True,  # rtsp downloads
+        },
         '_skip': 'Blocked outside of Austria / Germany',
     }]
 
diff --git a/youtube_dl/extractor/threeqsdn.py b/youtube_dl/extractor/threeqsdn.py
index a0bc12c81..33588424d 100644
--- a/youtube_dl/extractor/threeqsdn.py
+++ b/youtube_dl/extractor/threeqsdn.py
@@ -31,9 +31,13 @@ class ThreeQSDNIE(InfoExtractor):
         'info_dict': {
             'id': 'd755d94b-4ab9-11e3-9162-0025907ad44f',
             'ext': 'mp4',
-            'title': 'd755d94b-4ab9-11e3-9162-0025907ad44f',
-            'is_live': False,
+            'title': 're:^d755d94b-4ab9-11e3-9162-0025907ad44f [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
+            'is_live': True,
         },
+        'params': {
+            'skip_download': True,  # m3u8 downloads
+        },
+        'expected_warnings': ['Failed to download MPD manifest'],
     }, {
         # live audio stream
         'url': 'http://playout.3qsdn.com/9edf36e0-6bf2-11e2-a16a-9acf09e2db48',

From 84571be64592e11ec25a4b073ac74153c261441d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 15 Jul 2016 03:17:29 +0700
Subject: [PATCH 5519/6219] [orf:tvthek] Remove test md5

---
 youtube_dl/extractor/orf.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/orf.py b/youtube_dl/extractor/orf.py
index 42d4332e8..ccb23e069 100644
--- a/youtube_dl/extractor/orf.py
+++ b/youtube_dl/extractor/orf.py
@@ -40,7 +40,6 @@ class ORFTVthekIE(InfoExtractor):
         'skip': 'Blocked outside of Austria / Germany',
     }, {
         'url': 'http://tvthek.orf.at/topic/Im-Wandel-der-Zeit/8002126/Best-of-Ingrid-Thurnher/7982256',
-        'md5': '68f543909aea49d621dfc7703a11cfaf',
         'info_dict': {
             'id': '7982259',
             'ext': 'mp4',

From ad27649be307a55b04fa497d244ccca33b4260b0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 15 Jul 2016 03:36:50 +0700
Subject: [PATCH 5520/6219] [3qsdn] Restrict src JS regex

---
 youtube_dl/extractor/threeqsdn.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/threeqsdn.py b/youtube_dl/extractor/threeqsdn.py
index 33588424d..f26937da1 100644
--- a/youtube_dl/extractor/threeqsdn.py
+++ b/youtube_dl/extractor/threeqsdn.py
@@ -24,7 +24,7 @@ class ThreeQSDNIE(InfoExtractor):
             'title': '0280d6b9-1215-11e6-b427-0cc47a188158',
             'is_live': False,
         },
-        'expected_warnings': ['Failed to download MPD manifest'],
+        'expected_warnings': ['Failed to download MPD manifest', 'Failed to parse JSON'],
     }, {
         # live video stream
         'url': 'https://playout.3qsdn.com/d755d94b-4ab9-11e3-9162-0025907ad44f?js=true',
@@ -118,7 +118,7 @@ class ThreeQSDNIE(InfoExtractor):
                     'vcodec': 'none' if stream_type == 'audio' else None,
                 })
 
-        for item_js in re.findall(r'({.*?\b(?:src|source)\s*:\s*["\'].+?})', js):
+        for item_js in re.findall(r'({[^{]*?\b(?:src|source)\s*:\s*["\'].+?})', js):
             f = self._parse_json(
                 item_js, video_id, transform_source=js_to_json, fatal=False)
             if not f:

From 224db034abf7dff24898ae51cbe17fb1cc0d9e00 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 14 Jul 2016 23:59:12 +0100
Subject: [PATCH 5521/6219] [syfy] fix extraction(closes #9087)(closes
 #3820)(closes #2388)

---
 youtube_dl/extractor/syfy.py | 80 ++++++++++++++++++++----------------
 1 file changed, 45 insertions(+), 35 deletions(-)

diff --git a/youtube_dl/extractor/syfy.py b/youtube_dl/extractor/syfy.py
index 5ca079f88..53723b66e 100644
--- a/youtube_dl/extractor/syfy.py
+++ b/youtube_dl/extractor/syfy.py
@@ -1,46 +1,56 @@
 from __future__ import unicode_literals
 
-import re
-
-from .common import InfoExtractor
+from .theplatform import ThePlatformIE
+from ..utils import (
+    update_url_query,
+    smuggle_url,
+)
 
 
-class SyfyIE(InfoExtractor):
-    _VALID_URL = r'https?://www\.syfy\.com/(?:videos/.+?vid:(?P<id>[0-9]+)|(?!videos)(?P<video_name>[^/]+)(?:$|[?#]))'
-
+class SyfyIE(ThePlatformIE):
+    _VALID_URL = r'https?://www\.syfy\.com/(?:[^/]+/)?videos/(?P<id>[^/?#]+)'
     _TESTS = [{
-        'url': 'http://www.syfy.com/videos/Robot%20Combat%20League/Behind%20the%20Scenes/vid:2631458',
+        'url': 'http://www.syfy.com/theinternetruinedmylife/videos/the-internet-ruined-my-life-season-1-trailer',
         'info_dict': {
-            'id': 'NmqMrGnXvmO1',
-            'ext': 'flv',
-            'title': 'George Lucas has Advice for his Daughter',
-            'description': 'Listen to what insights George Lucas give his daughter Amanda.',
+            'id': '2968097',
+            'ext': 'mp4',
+            'title': 'The Internet Ruined My Life: Season 1 Trailer',
+            'description': 'One tweet, one post, one click, can destroy everything.',
+            'uploader': 'NBCU-MPAT',
+            'upload_date': '20170113',
+            'timestamp': 1484345640,
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
         },
         'add_ie': ['ThePlatform'],
-    }, {
-        'url': 'http://www.syfy.com/wilwheaton',
-        'md5': '94dfa54ee3ccb63295b276da08c415f6',
-        'info_dict': {
-            'id': '4yoffOOXC767',
-            'ext': 'flv',
-            'title': 'The Wil Wheaton Project - Premiering May 27th at 10/9c.',
-            'description': 'The Wil Wheaton Project premieres May 27th at 10/9c. Don\'t miss it.',
-        },
-        'add_ie': ['ThePlatform'],
-        'skip': 'Blocked outside the US',
     }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_name = mobj.group('video_name')
-        if video_name:
-            generic_webpage = self._download_webpage(url, video_name)
-            video_id = self._search_regex(
-                r'<iframe.*?class="video_iframe_page"\s+src="/_utils/video/thP_video_controller.php.*?_vid([0-9]+)">',
-                generic_webpage, 'video ID')
-            url = 'http://www.syfy.com/videos/%s/%s/vid:%s' % (
-                video_name, video_name, video_id)
-        else:
-            video_id = mobj.group('id')
-        webpage = self._download_webpage(url, video_id)
-        return self.url_result(self._og_search_video_url(webpage))
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+        syfy_mpx = list(self._parse_json(self._search_regex(
+            r'jQuery\.extend\([^,]+,\s*({.+})\);', webpage, 'drupal settings'),
+            display_id)['syfy']['syfy_mpx'].values())[0]
+        video_id = syfy_mpx['mpxGUID']
+        title = syfy_mpx['episodeTitle']
+        query = {
+            'mbr': 'true',
+            'manifest': 'm3u',
+        }
+        if syfy_mpx.get('entitlement') == 'auth':
+            resource = '<rss version="2.0" xmlns:media="http://search.yahoo.com/mrss/"><channel><title>syfy</title><item><title><![CDATA[%s]]></title><guid>%s</guid><media:rating scheme="urn:v-chip">%s</media:rating></item></channel></rss>' % (title, video_id, syfy_mpx.get('mpxRating', 'TV-14'))
+            query['auth'] = self._extract_mvpd_auth(
+                url, video_id, 'syfy', resource)
+
+        return {
+            '_type': 'url_transparent',
+            'ie_key': 'ThePlatform',
+            'url': smuggle_url(update_url_query(
+                self._proto_relative_url(syfy_mpx['releaseURL']), query),
+                {'force_smil_url': True}),
+            'title': title,
+            'id': video_id,
+            'display_id': display_id,
+        }

From 23495d6a39d357989bf507c0bbb7c022e7a9e2c7 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 15 Jul 2016 19:53:10 +0800
Subject: [PATCH 5522/6219] Revert "[ffmpeg] Fix embedding subtitles (#9063)"

This reverts commit ccff2c404d7ea9f5b21ede8ae57bb79feec7eb94.

Fixes #10081.

The new approach breaks embedding subtitles into video-only or
audio-only files. FFMpeg provides a trick: add '?' after the argument of
'-map' so that a missing stream is ignored. For example:

opts = [
    '-map', '0:v?',
    '-c:v', 'copy',
    '-map', '0:a?',
    '-c:a', 'copy',
    # other options...
]

Unfortunately, such a format is not implemented in avconv, either.
I guess adding '-ignore_unknown' if self.basename == 'ffmpeg' is the
best solution. However, the example mentioned in #9063 no longer serves
problematic files, so I can't test it. I'll reopen #9063 and wait for
another example so that I can test '-ignore_unknown'.
---
 youtube_dl/postprocessor/ffmpeg.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index c1e9eb159..fa99b0c2a 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -363,10 +363,8 @@ class FFmpegEmbedSubtitlePP(FFmpegPostProcessor):
         input_files = [filename] + sub_filenames
 
         opts = [
-            '-map', '0:v',
-            '-c:v', 'copy',
-            '-map', '0:a',
-            '-c:a', 'copy',
+            '-map', '0',
+            '-c', 'copy',
             # Don't copy the existing subtitles, we may be running the
             # postprocessor a second time
             '-map', '-0:s',

From 317f7ab634174666e458807fa309a2e7ba459267 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 16 Jul 2016 00:55:43 +0700
Subject: [PATCH 5523/6219] [YoutubeDL] Fix format selection with filters
 (Closes #10083)

---
 test/test_YoutubeDL.py  | 34 ++++++++++++++++++++++
 youtube_dl/YoutubeDL.py | 63 ++++++++++++++++++++++++++++++-----------
 2 files changed, 81 insertions(+), 16 deletions(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index ca25025e2..0dfe25c00 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -335,6 +335,40 @@ class TestFormatSelection(unittest.TestCase):
             downloaded = ydl.downloaded_info_dicts[0]
             self.assertEqual(downloaded['format_id'], f1['format_id'])
 
+    def test_audio_only_extractor_format_selection(self):
+        # For extractors with incomplete formats (all formats are audio-only or
+        # video-only) best and worst should fallback to corresponding best/worst
+        # video-only or audio-only formats (as per
+        # https://github.com/rg3/youtube-dl/pull/5556)
+        formats = [
+            {'format_id': 'low', 'ext': 'mp3', 'preference': 1, 'vcodec': 'none', 'url': TEST_URL},
+            {'format_id': 'high', 'ext': 'mp3', 'preference': 2, 'vcodec': 'none', 'url': TEST_URL},
+        ]
+        info_dict = _make_result(formats)
+
+        ydl = YDL({'format': 'best'})
+        ydl.process_ie_result(info_dict.copy())
+        downloaded = ydl.downloaded_info_dicts[0]
+        self.assertEqual(downloaded['format_id'], 'high')
+
+        ydl = YDL({'format': 'worst'})
+        ydl.process_ie_result(info_dict.copy())
+        downloaded = ydl.downloaded_info_dicts[0]
+        self.assertEqual(downloaded['format_id'], 'low')
+
+    def test_format_not_available(self):
+        formats = [
+            {'format_id': 'regular', 'ext': 'mp4', 'height': 360, 'url': TEST_URL},
+            {'format_id': 'video', 'ext': 'mp4', 'height': 720, 'acodec': 'none', 'url': TEST_URL},
+        ]
+        info_dict = _make_result(formats)
+
+        # This must fail since complete video-audio format does not match filter
+        # and extractor does not provide incomplete only formats (i.e. only
+        # video-only or audio-only).
+        ydl = YDL({'format': 'best[height>360]'})
+        self.assertRaises(ExtractorError, ydl.process_ie_result, info_dict.copy())
+
     def test_invalid_format_specs(self):
         def assert_syntax_error(format_spec):
             ydl = YDL({'format': format_spec})
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index ba72ec6f3..cf9cd8297 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -5,6 +5,7 @@ from __future__ import absolute_import, unicode_literals
 
 import collections
 import contextlib
+import copy
 import datetime
 import errno
 import fileinput
@@ -1051,9 +1052,9 @@ class YoutubeDL(object):
             if isinstance(selector, list):
                 fs = [_build_selector_function(s) for s in selector]
 
-                def selector_function(formats):
+                def selector_function(ctx):
                     for f in fs:
-                        for format in f(formats):
+                        for format in f(ctx):
                             yield format
                 return selector_function
             elif selector.type == GROUP:
@@ -1061,17 +1062,17 @@ class YoutubeDL(object):
             elif selector.type == PICKFIRST:
                 fs = [_build_selector_function(s) for s in selector.selector]
 
-                def selector_function(formats):
+                def selector_function(ctx):
                     for f in fs:
-                        picked_formats = list(f(formats))
+                        picked_formats = list(f(ctx))
                         if picked_formats:
                             return picked_formats
                     return []
             elif selector.type == SINGLE:
                 format_spec = selector.selector
 
-                def selector_function(formats):
-                    formats = list(formats)
+                def selector_function(ctx):
+                    formats = list(ctx['formats'])
                     if not formats:
                         return
                     if format_spec == 'all':
@@ -1084,9 +1085,10 @@ class YoutubeDL(object):
                             if f.get('vcodec') != 'none' and f.get('acodec') != 'none']
                         if audiovideo_formats:
                             yield audiovideo_formats[format_idx]
-                        # for audio only (soundcloud) or video only (imgur) urls, select the best/worst audio format
-                        elif (all(f.get('acodec') != 'none' for f in formats) or
-                              all(f.get('vcodec') != 'none' for f in formats)):
+                        # for extractors with incomplete formats (audio only (soundcloud)
+                        # or video only (imgur)) we will fallback to best/worst
+                        # {video,audio}-only format
+                        elif ctx['incomplete_formats']:
                             yield formats[format_idx]
                     elif format_spec == 'bestaudio':
                         audio_formats = [
@@ -1160,17 +1162,18 @@ class YoutubeDL(object):
                     }
                 video_selector, audio_selector = map(_build_selector_function, selector.selector)
 
-                def selector_function(formats):
-                    formats = list(formats)
-                    for pair in itertools.product(video_selector(formats), audio_selector(formats)):
+                def selector_function(ctx):
+                    for pair in itertools.product(
+                            video_selector(copy.deepcopy(ctx)), audio_selector(copy.deepcopy(ctx))):
                         yield _merge(pair)
 
             filters = [self._build_format_filter(f) for f in selector.filters]
 
-            def final_selector(formats):
+            def final_selector(ctx):
+                ctx_copy = copy.deepcopy(ctx)
                 for _filter in filters:
-                    formats = list(filter(_filter, formats))
-                return selector_function(formats)
+                    ctx_copy['formats'] = list(filter(_filter, ctx_copy['formats']))
+                return selector_function(ctx_copy)
             return final_selector
 
         stream = io.BytesIO(format_spec.encode('utf-8'))
@@ -1377,7 +1380,35 @@ class YoutubeDL(object):
             req_format_list.append('best')
             req_format = '/'.join(req_format_list)
         format_selector = self.build_format_selector(req_format)
-        formats_to_download = list(format_selector(formats))
+
+        # While in format selection we may need to have an access to the original
+        # format set in order to calculate some metrics or do some processing.
+        # For now we need to be able to guess whether original formats provided
+        # by extractor are incomplete or not (i.e. whether extractor provides only
+        # video-only or audio-only formats) for proper formats selection for
+        # extractors with such incomplete formats (see
+        # https://github.com/rg3/youtube-dl/pull/5556).
+        # Since formats may be filtered during format selection and may not match
+        # the original formats the results may be incorrect. Thus original formats
+        # or pre-calculated metrics should be passed to format selection routines
+        # as well.
+        # We will pass a context object containing all necessary additional data
+        # instead of just formats.
+        # This fixes incorrect format selection issue (see
+        # https://github.com/rg3/youtube-dl/issues/10083).
+        incomplete_formats = all(
+            # All formats are video-only or
+            f.get('vcodec') != 'none' and f.get('acodec') == 'none' or
+            # all formats are audio-only
+            f.get('vcodec') == 'none' and f.get('acodec') != 'none'
+            for f in formats)
+
+        ctx = {
+            'formats': formats,
+            'incomplete_formats': incomplete_formats,
+        }
+
+        formats_to_download = list(format_selector(ctx))
         if not formats_to_download:
             raise ExtractorError('requested format not available',
                                  expected=True)

From 2e221ca3a85ec7a0c441dfcf301bf1c98614b9dd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 16 Jul 2016 01:18:05 +0700
Subject: [PATCH 5524/6219] [YoutubeDL] Fix incomplete formats check

---
 youtube_dl/YoutubeDL.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index cf9cd8297..6551f086f 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1396,12 +1396,11 @@ class YoutubeDL(object):
         # instead of just formats.
         # This fixes incorrect format selection issue (see
         # https://github.com/rg3/youtube-dl/issues/10083).
-        incomplete_formats = all(
+        incomplete_formats = (
             # All formats are video-only or
-            f.get('vcodec') != 'none' and f.get('acodec') == 'none' or
+            all(f.get('vcodec') != 'none' and f.get('acodec') == 'none' for f in formats) or
             # all formats are audio-only
-            f.get('vcodec') == 'none' and f.get('acodec') != 'none'
-            for f in formats)
+            all(f.get('vcodec') == 'none' and f.get('acodec') != 'none' for f in formats))
 
         ctx = {
             'formats': formats,

From 691fbe7f98dd648f3971f259b323e62c73d56e58 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 16 Jul 2016 02:20:00 +0700
Subject: [PATCH 5525/6219] release 2016.07.16

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 docs/supportedsites.md    | 2 ++
 youtube_dl/version.py     | 2 +-
 3 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index c5898701f..b1b8def3c 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.07.13*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.07.13**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.07.16*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.07.16**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.07.13
+[debug] youtube-dl version 2016.07.16
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 282bd0e6b..cf194340a 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -14,6 +14,7 @@
  - **8tracks**
  - **91porn**
  - **9gag**
+ - **9now.com.au**
  - **abc.net.au**
  - **Abc7News**
  - **abcnews**
@@ -567,6 +568,7 @@
  - **rtve.es:alacarta**: RTVE a la carta
  - **rtve.es:infantil**: RTVE infantil
  - **rtve.es:live**: RTVE.es live streams
+ - **rtve.es:television**
  - **RTVNH**
  - **Rudo**
  - **RUHD**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 56f9f5986..55d07d420 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.07.13'
+__version__ = '2016.07.16'

From 21ba7d0981e04237ae4ca8690a6ead331b3a82d1 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 16 Jul 2016 00:02:26 +0800
Subject: [PATCH 5526/6219] [cbc] Skip geo-restricted test case

---
 youtube_dl/extractor/cbc.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/cbc.py b/youtube_dl/extractor/cbc.py
index ff663d079..06772d492 100644
--- a/youtube_dl/extractor/cbc.py
+++ b/youtube_dl/extractor/cbc.py
@@ -25,6 +25,7 @@ class CBCIE(InfoExtractor):
             'upload_date': '20160203',
             'uploader': 'CBCC-NEW',
         },
+        'skip': 'Geo-restricted to Canada',
     }, {
         # with clipId
         'url': 'http://www.cbc.ca/archives/entry/1978-robin-williams-freestyles-on-90-minutes-live',
@@ -64,6 +65,7 @@ class CBCIE(InfoExtractor):
                 'uploader': 'CBCC-NEW',
             },
         }],
+        'skip': 'Geo-restricted to Canada',
     }]
 
     @classmethod
@@ -104,6 +106,7 @@ class CBCPlayerIE(InfoExtractor):
             'upload_date': '20160210',
             'uploader': 'CBCC-NEW',
         },
+        'skip': 'Geo-restricted to Canada',
     }, {
         # Redirected from http://www.cbc.ca/player/AudioMobile/All%20in%20a%20Weekend%20Montreal/ID/2657632011/
         'url': 'http://www.cbc.ca/player/play/2657631896',

From ae7b8462032ef86d803162067f0dbbbd03fa49e4 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 16 Jul 2016 00:03:12 +0800
Subject: [PATCH 5527/6219] [cbsnews] Update _TESTS of CBSNewsLiveVideoIE

---
 youtube_dl/extractor/cbsnews.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/cbsnews.py b/youtube_dl/extractor/cbsnews.py
index 387537e76..9328e3e20 100644
--- a/youtube_dl/extractor/cbsnews.py
+++ b/youtube_dl/extractor/cbsnews.py
@@ -26,6 +26,7 @@ class CBSNewsIE(CBSBaseIE):
                 # rtmp download
                 'skip_download': True,
             },
+            'skip': 'Subscribers only',
         },
         {
             'url': 'http://www.cbsnews.com/videos/fort-hood-shooting-army-downplays-mental-illness-as-cause-of-attack/',
@@ -69,7 +70,7 @@ class CBSNewsLiveVideoIE(InfoExtractor):
     IE_DESC = 'CBS News Live Videos'
     _VALID_URL = r'https?://(?:www\.)?cbsnews\.com/live/video/(?P<id>[\da-z_-]+)'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.cbsnews.com/live/video/clinton-sanders-prepare-to-face-off-in-nh/',
         'info_dict': {
             'id': 'clinton-sanders-prepare-to-face-off-in-nh',
@@ -77,7 +78,15 @@ class CBSNewsLiveVideoIE(InfoExtractor):
             'title': 'Clinton, Sanders Prepare To Face Off In NH',
             'duration': 334,
         },
-    }
+        'skip': 'Video gone, redirected to http://www.cbsnews.com/live/',
+    }, {
+        'url': 'http://www.cbsnews.com/live/video/video-shows-intense-paragliding-accident/',
+        'info_dict': {
+            'id': 'video-shows-intense-paragliding-accident',
+            'ext': 'flv',
+            'title': 'Video Shows Intense Paragliding Accident',
+        },
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From aadd3ce21fe4acacc5bf26c3703cc09d05e7b3f4 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 16 Jul 2016 00:04:30 +0800
Subject: [PATCH 5528/6219] [cliphunter] Update _TESTS

---
 youtube_dl/extractor/cliphunter.py | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/cliphunter.py b/youtube_dl/extractor/cliphunter.py
index 19f8b397e..252c2e846 100644
--- a/youtube_dl/extractor/cliphunter.py
+++ b/youtube_dl/extractor/cliphunter.py
@@ -23,7 +23,7 @@ class CliphunterIE(InfoExtractor):
         (?P<id>[0-9]+)/
         (?P<seo>.+?)(?:$|[#\?])
     '''
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.cliphunter.com/w/1012420/Fun_Jynx_Maze_solo',
         'md5': 'b7c9bbd4eb3a226ab91093714dcaa480',
         'info_dict': {
@@ -32,8 +32,19 @@ class CliphunterIE(InfoExtractor):
             'title': 'Fun Jynx Maze solo',
             'thumbnail': 're:^https?://.*\.jpg$',
             'age_limit': 18,
-        }
-    }
+        },
+        'skip': 'Video gone',
+    }, {
+        'url': 'http://www.cliphunter.com/w/2019449/ShesNew__My_booty_girlfriend_Victoria_Paradices_pussy_filled_with_jizz',
+        'md5': '55a723c67bfc6da6b0cfa00d55da8a27',
+        'info_dict': {
+            'id': '2019449',
+            'ext': 'mp4',
+            'title': 'ShesNew - My booty girlfriend, Victoria Paradice\'s pussy filled with jizz',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'age_limit': 18,
+        },
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From 998895dffac2170b7d49b0478561db05cc0730ca Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 16 Jul 2016 01:21:20 +0800
Subject: [PATCH 5529/6219] [cloudy] Drop videoraj.to

videoraj.ch is now a shoe-selling website, and videoraj.to domain name
is gone.
---
 youtube_dl/extractor/cloudy.py | 48 ++++++++++++----------------------
 1 file changed, 17 insertions(+), 31 deletions(-)

diff --git a/youtube_dl/extractor/cloudy.py b/youtube_dl/extractor/cloudy.py
index 9a28ef354..ae5ba0015 100644
--- a/youtube_dl/extractor/cloudy.py
+++ b/youtube_dl/extractor/cloudy.py
@@ -6,7 +6,6 @@ import re
 from .common import InfoExtractor
 from ..compat import (
     compat_parse_qs,
-    compat_urllib_parse_urlencode,
     compat_HTTPError,
 )
 from ..utils import (
@@ -17,37 +16,26 @@ from ..utils import (
 
 
 class CloudyIE(InfoExtractor):
-    _IE_DESC = 'cloudy.ec and videoraj.ch'
+    _IE_DESC = 'cloudy.ec'
     _VALID_URL = r'''(?x)
-        https?://(?:www\.)?(?P<host>cloudy\.ec|videoraj\.(?:ch|to))/
+        https?://(?:www\.)?cloudy\.ec/
         (?:v/|embed\.php\?id=)
         (?P<id>[A-Za-z0-9]+)
         '''
-    _EMBED_URL = 'http://www.%s/embed.php?id=%s'
-    _API_URL = 'http://www.%s/api/player.api.php?%s'
+    _EMBED_URL = 'http://www.cloudy.ec/embed.php?id=%s'
+    _API_URL = 'http://www.cloudy.ec/api/player.api.php'
     _MAX_TRIES = 2
-    _TESTS = [
-        {
-            'url': 'https://www.cloudy.ec/v/af511e2527aac',
-            'md5': '5cb253ace826a42f35b4740539bedf07',
-            'info_dict': {
-                'id': 'af511e2527aac',
-                'ext': 'flv',
-                'title': 'Funny Cats and Animals Compilation june 2013',
-            }
-        },
-        {
-            'url': 'http://www.videoraj.to/v/47f399fd8bb60',
-            'md5': '7d0f8799d91efd4eda26587421c3c3b0',
-            'info_dict': {
-                'id': '47f399fd8bb60',
-                'ext': 'flv',
-                'title': 'Burning a New iPhone 5 with Gasoline - Will it Survive?',
-            }
+    _TEST = {
+        'url': 'https://www.cloudy.ec/v/af511e2527aac',
+        'md5': '5cb253ace826a42f35b4740539bedf07',
+        'info_dict': {
+            'id': 'af511e2527aac',
+            'ext': 'flv',
+            'title': 'Funny Cats and Animals Compilation june 2013',
         }
-    ]
+    }
 
-    def _extract_video(self, video_host, video_id, file_key, error_url=None, try_num=0):
+    def _extract_video(self, video_id, file_key, error_url=None, try_num=0):
 
         if try_num > self._MAX_TRIES - 1:
             raise ExtractorError('Unable to extract video URL', expected=True)
@@ -64,9 +52,8 @@ class CloudyIE(InfoExtractor):
                 'errorUrl': error_url,
             })
 
-        data_url = self._API_URL % (video_host, compat_urllib_parse_urlencode(form))
         player_data = self._download_webpage(
-            data_url, video_id, 'Downloading player data')
+            self._API_URL, video_id, 'Downloading player data', query=form)
         data = compat_parse_qs(player_data)
 
         try_num += 1
@@ -88,7 +75,7 @@ class CloudyIE(InfoExtractor):
             except ExtractorError as e:
                 if isinstance(e.cause, compat_HTTPError) and e.cause.code in [404, 410]:
                     self.report_warning('Invalid video URL, requesting another', video_id)
-                    return self._extract_video(video_host, video_id, file_key, video_url, try_num)
+                    return self._extract_video(video_id, file_key, video_url, try_num)
 
         return {
             'id': video_id,
@@ -98,14 +85,13 @@ class CloudyIE(InfoExtractor):
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        video_host = mobj.group('host')
         video_id = mobj.group('id')
 
-        url = self._EMBED_URL % (video_host, video_id)
+        url = self._EMBED_URL % video_id
         webpage = self._download_webpage(url, video_id)
 
         file_key = self._search_regex(
             [r'key\s*:\s*"([^"]+)"', r'filekey\s*=\s*"([^"]+)"'],
             webpage, 'file_key')
 
-        return self._extract_video(video_host, video_id, file_key)
+        return self._extract_video(video_id, file_key)

From 371ddb14fe651d4a1e5a8310d6d7c0e395cd92b0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 16 Jul 2016 15:59:43 +0700
Subject: [PATCH 5530/6219] [extractor/generic] Change twitter:player embeds
 priority to lowest (Closes #10090)

---
 youtube_dl/extractor/generic.py | 31 ++++++++++++++++++++++++++-----
 1 file changed, 26 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index cddd1a817..6d346cb1c 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1249,6 +1249,20 @@ class GenericIE(InfoExtractor):
                 'uploader': 'www.hudl.com',
             },
         },
+        # twitter:player:stream embed
+        {
+            'url': 'http://www.rtl.be/info/video/589263.aspx?CategoryID=288',
+            'info_dict': {
+                'id': 'master',
+                'ext': 'mp4',
+                'title': 'Une nouvelle espèce de dinosaure découverte en Argentine',
+                'uploader': 'www.rtl.be',
+            },
+            'params': {
+                # m3u8 downloads
+                'skip_download': True,
+            },
+        },
         # twitter:player embed
         {
             'url': 'http://www.theatlantic.com/video/index/484130/what-do-black-holes-sound-like/',
@@ -2184,11 +2198,6 @@ class GenericIE(InfoExtractor):
                 'uploader': video_uploader,
             }
 
-        # https://dev.twitter.com/cards/types/player#On_twitter.com_via_desktop_browser
-        embed_url = self._html_search_meta('twitter:player', webpage, default=None)
-        if embed_url:
-            return self.url_result(embed_url)
-
         # Looking for http://schema.org/VideoObject
         json_ld = self._search_json_ld(
             webpage, video_id, default=None, expected_type='VideoObject')
@@ -2245,6 +2254,9 @@ class GenericIE(InfoExtractor):
                 r"cinerama\.embedPlayer\(\s*\'[^']+\',\s*'([^']+)'", webpage)
         if not found:
             # Try to find twitter cards info
+            # twitter:player:stream should be checked before twitter:player since
+            # it is expected to contain a raw stream (see
+            # https://dev.twitter.com/cards/types/player#On_twitter.com_via_desktop_browser)
             found = filter_video(re.findall(
                 r'<meta (?:property|name)="twitter:player:stream" (?:content|value)="(.+?)"', webpage))
         if not found:
@@ -2278,6 +2290,15 @@ class GenericIE(InfoExtractor):
                     '_type': 'url',
                     'url': new_url,
                 }
+
+        if not found:
+            # twitter:player is a https URL to iframe player that may or may not
+            # be supported by youtube-dl thus this is checked the very last (see
+            # https://dev.twitter.com/cards/types/player#On_twitter.com_via_desktop_browser)
+            embed_url = self._html_search_meta('twitter:player', webpage, default=None)
+            if embed_url:
+                return self.url_result(embed_url)
+
         if not found:
             raise UnsupportedError(url)
 

From a2f9ca1e67e6b926957abe3b35a4d78355bef7d4 Mon Sep 17 00:00:00 2001
From: TRox1972 <TRox1972@users.noreply.github.com>
Date: Sun, 3 Jul 2016 00:39:35 +0200
Subject: [PATCH 5531/6219] [nintendo] Add extractor

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/nintendo.py   | 47 ++++++++++++++++++++++++++++++
 2 files changed, 48 insertions(+)
 create mode 100644 youtube_dl/extractor/nintendo.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 45817d7df..2761f7095 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -538,6 +538,7 @@ from .niconico import NiconicoIE, NiconicoPlaylistIE
 from .ninecninemedia import NineCNineMediaIE
 from .ninegag import NineGagIE
 from .ninenow import NineNowIE
+from .nintendo import NintendoIE
 from .noco import NocoIE
 from .normalboots import NormalbootsIE
 from .nosvideo import NosVideoIE
diff --git a/youtube_dl/extractor/nintendo.py b/youtube_dl/extractor/nintendo.py
new file mode 100644
index 000000000..57333ada0
--- /dev/null
+++ b/youtube_dl/extractor/nintendo.py
@@ -0,0 +1,47 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from .ooyala import OoyalaIE
+
+import re
+
+
+class NintendoIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?nintendo\.com/games/detail/(?P<id>[\w-]+)'
+    _TESTS = [{
+        'url': 'http://www.nintendo.com/games/detail/yEiAzhU2eQI1KZ7wOHhngFoAHc1FpHwj',
+        'info_dict': {
+            'id': 'MzMmticjp0VPzO3CCj4rmFOuohEuEWoW',
+            'ext': 'flv',
+            'title': 'Duck Hunt Wii U VC NES - Trailer',
+            'duration': 60.326,
+        },
+        'params': {
+            'skip_download': True,
+        },
+        'add_ie': ['Ooyala'],
+    }, {
+        'url': 'http://www.nintendo.com/games/detail/tokyo-mirage-sessions-fe-wii-u',
+        'info_dict': {
+            'id': 'tokyo-mirage-sessions-fe-wii-u',
+        },
+        'params': {
+            'skip_download': True,
+        },
+        'add_ie': ['Ooyala'],
+        'playlist_count': 4,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        ooyala_codes = re.findall(
+            r'data-video-code=(["\'])(?P<code>.+?)\1',
+            webpage)
+
+        entries = []
+        for ooyala_code in ooyala_codes:
+            entries.append(OoyalaIE._build_url_result(ooyala_code[1]))
+
+        return self.playlist_result(entries, video_id, self._og_search_title(webpage))

From 49bc16b95ed7d418b353fda46ba845ac0eca648b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 17 Jul 2016 00:01:25 +0700
Subject: [PATCH 5532/6219] [nintendo] Improve playlist extraction (Closes
 #9986)

---
 youtube_dl/extractor/nintendo.py | 35 ++++++++++++++++----------------
 1 file changed, 17 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/nintendo.py b/youtube_dl/extractor/nintendo.py
index 57333ada0..4b4e66b05 100644
--- a/youtube_dl/extractor/nintendo.py
+++ b/youtube_dl/extractor/nintendo.py
@@ -1,13 +1,15 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
-from .common import InfoExtractor
-from .ooyala import OoyalaIE
-
 import re
 
+from .common import InfoExtractor
+from .ooyala import OoyalaIE
+from ..utils import unescapeHTML
+
 
 class NintendoIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?nintendo\.com/games/detail/(?P<id>[\w-]+)'
+    _VALID_URL = r'https?://(?:www\.)?nintendo\.com/games/detail/(?P<id>[^/?#&]+)'
     _TESTS = [{
         'url': 'http://www.nintendo.com/games/detail/yEiAzhU2eQI1KZ7wOHhngFoAHc1FpHwj',
         'info_dict': {
@@ -24,24 +26,21 @@ class NintendoIE(InfoExtractor):
         'url': 'http://www.nintendo.com/games/detail/tokyo-mirage-sessions-fe-wii-u',
         'info_dict': {
             'id': 'tokyo-mirage-sessions-fe-wii-u',
+            'title': 'Tokyo Mirage Sessions ♯FE',
         },
-        'params': {
-            'skip_download': True,
-        },
-        'add_ie': ['Ooyala'],
-        'playlist_count': 4,
+        'playlist_count': 3,
     }]
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
+        page_id = self._match_id(url)
 
-        ooyala_codes = re.findall(
-            r'data-video-code=(["\'])(?P<code>.+?)\1',
-            webpage)
+        webpage = self._download_webpage(url, page_id)
 
-        entries = []
-        for ooyala_code in ooyala_codes:
-            entries.append(OoyalaIE._build_url_result(ooyala_code[1]))
+        entries = [
+            OoyalaIE._build_url_result(m.group('code'))
+            for m in re.finditer(
+                r'class=(["\'])embed-video\1[^>]+data-video-code=(["\'])(?P<code>(?:(?!\2).)+)\2',
+                webpage)]
 
-        return self.playlist_result(entries, video_id, self._og_search_title(webpage))
+        return self.playlist_result(
+            entries, page_id, unescapeHTML(self._og_search_title(webpage, fatal=False)))

From 59cc5bd8bfb5ecce82e26daa1b8f830edd3eb8b7 Mon Sep 17 00:00:00 2001
From: Zach Bruggeman <mail@bruggie.com>
Date: Fri, 8 Apr 2016 13:50:09 -0700
Subject: [PATCH 5533/6219] [streamable] Add extractor

---
 youtube_dl/extractor/streamable.py | 76 ++++++++++++++++++++++++++++++
 1 file changed, 76 insertions(+)
 create mode 100644 youtube_dl/extractor/streamable.py

diff --git a/youtube_dl/extractor/streamable.py b/youtube_dl/extractor/streamable.py
new file mode 100644
index 000000000..5aa5f1ba5
--- /dev/null
+++ b/youtube_dl/extractor/streamable.py
@@ -0,0 +1,76 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    float_or_none
+)
+
+
+class StreamableIE(InfoExtractor):
+    _VALID_URL = r'https?://streamable\.com/(?P<id>[\w]+)'
+    _TESTS = [
+        {
+            'url': 'https://streamable.com/dnd1',
+            'md5': '3e3bc5ca088b48c2d436529b64397fef',
+            'info_dict': {
+                'id': 'dnd1',
+                'ext': 'mp4',
+                'title': 'Mikel Oiarzabal scores to make it 0-3 for La Real against Espanyol',
+                'thumbnail': 'http://cdn.streamable.com/image/dnd1.jpg',
+            }
+        },
+        # older video without bitrate, width/height, etc. info
+        {
+            'url': 'https://streamable.com/moo',
+            'md5': '2cf6923639b87fba3279ad0df3a64e73',
+            'info_dict': {
+                'id': 'moo',
+                'ext': 'mp4',
+                'title': '"Please don\'t eat me!"',
+                'thumbnail': 'http://cdn.streamable.com/image/f6441ae0c84311e4af010bc47400a0a4.jpg',
+            }
+        }
+    ]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        # Note: Using the ajax API, as the public Streamable API doesn't seem
+        # to return video info like the title properly sometimes, and doesn't
+        # include info like the video duration
+        video = self._download_json(
+            'https://streamable.com/ajax/videos/%s' % video_id, video_id)
+
+        # Format IDs:
+        # 0 The video is being uploaded
+        # 1 The video is being processed
+        # 2 The video has at least one file ready
+        # 3 The video is unavailable due to an error
+        status = video.get('status')
+        if status != 2:
+            raise ExtractorError(
+                'This video is currently unavailable. It may still be uploading or processing.',
+                expected=True)
+
+        formats = []
+        for key, info in video.get('files').items():
+            formats.append({
+                'format_id': key,
+                'url': info['url'],
+                'width': info.get('width'),
+                'height': info.get('height'),
+                'filesize': info.get('size'),
+                'fps': info.get('framerate'),
+                'vbr': float_or_none(info.get('bitrate'), 1000)
+            })
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': video.get('result_title'),
+            'thumbnail': video.get('thumbnail_url'),
+            'duration': video.get('duration'),
+            'formats': formats
+        }

From 1a8f0773b6b2550c2763f4522481df7695ad4b6f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 17 Jul 2016 02:01:00 +0700
Subject: [PATCH 5534/6219] [streamable] Fix title extraction and improve
 (Closes #9122)

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/streamable.py | 48 ++++++++++++++++++++++--------
 2 files changed, 36 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 2761f7095..10b2390bf 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -783,6 +783,7 @@ from .srmediathek import SRMediathekIE
 from .ssa import SSAIE
 from .stanfordoc import StanfordOpenClassroomIE
 from .steam import SteamIE
+from .streamable import StreamableIE
 from .streamcloud import StreamcloudIE
 from .streamcz import StreamCZIE
 from .streetvoice import StreetVoiceIE
diff --git a/youtube_dl/extractor/streamable.py b/youtube_dl/extractor/streamable.py
index 5aa5f1ba5..1c61437a4 100644
--- a/youtube_dl/extractor/streamable.py
+++ b/youtube_dl/extractor/streamable.py
@@ -4,12 +4,13 @@ from __future__ import unicode_literals
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
-    float_or_none
+    float_or_none,
+    int_or_none,
 )
 
 
 class StreamableIE(InfoExtractor):
-    _VALID_URL = r'https?://streamable\.com/(?P<id>[\w]+)'
+    _VALID_URL = r'https?://streamable\.com/(?:e/)?(?P<id>\w+)'
     _TESTS = [
         {
             'url': 'https://streamable.com/dnd1',
@@ -18,7 +19,12 @@ class StreamableIE(InfoExtractor):
                 'id': 'dnd1',
                 'ext': 'mp4',
                 'title': 'Mikel Oiarzabal scores to make it 0-3 for La Real against Espanyol',
-                'thumbnail': 'http://cdn.streamable.com/image/dnd1.jpg',
+                'thumbnail': 're:https?://.*\.jpg$',
+                'uploader': 'teabaker',
+                'timestamp': 1454964157.35115,
+                'upload_date': '20160208',
+                'duration': 61.516,
+                'view_count': int,
             }
         },
         # older video without bitrate, width/height, etc. info
@@ -29,8 +35,16 @@ class StreamableIE(InfoExtractor):
                 'id': 'moo',
                 'ext': 'mp4',
                 'title': '"Please don\'t eat me!"',
-                'thumbnail': 'http://cdn.streamable.com/image/f6441ae0c84311e4af010bc47400a0a4.jpg',
+                'thumbnail': 're:https?://.*\.jpg$',
+                'timestamp': 1426115495,
+                'upload_date': '20150311',
+                'duration': 12,
+                'view_count': int,
             }
+        },
+        {
+            'url': 'https://streamable.com/e/dnd1',
+            'only_matching': True,
         }
     ]
 
@@ -54,23 +68,31 @@ class StreamableIE(InfoExtractor):
                 'This video is currently unavailable. It may still be uploading or processing.',
                 expected=True)
 
+        title = video.get('reddit_title') or video['title']
+
         formats = []
-        for key, info in video.get('files').items():
+        for key, info in video['files'].items():
+            if not info.get('url'):
+                continue
             formats.append({
                 'format_id': key,
-                'url': info['url'],
-                'width': info.get('width'),
-                'height': info.get('height'),
-                'filesize': info.get('size'),
-                'fps': info.get('framerate'),
+                'url': self._proto_relative_url(info['url']),
+                'width': int_or_none(info.get('width')),
+                'height': int_or_none(info.get('height')),
+                'filesize': int_or_none(info.get('size')),
+                'fps': int_or_none(info.get('framerate')),
                 'vbr': float_or_none(info.get('bitrate'), 1000)
             })
         self._sort_formats(formats)
 
         return {
             'id': video_id,
-            'title': video.get('result_title'),
-            'thumbnail': video.get('thumbnail_url'),
-            'duration': video.get('duration'),
+            'title': title,
+            'description': video.get('description'),
+            'thumbnail': self._proto_relative_url(video.get('thumbnail_url')),
+            'uploader': video.get('owner', {}).get('user_name'),
+            'timestamp': float_or_none(video.get('date_added')),
+            'duration': float_or_none(video.get('duration')),
+            'view_count': int_or_none(video.get('plays')),
             'formats': formats
         }

From af21f56f980e22086ac734cf266141c7a9ff21ce Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 17 Jul 2016 03:40:58 +0700
Subject: [PATCH 5535/6219] [ard] Add support for rbb-online (Closes #10095)

---
 youtube_dl/extractor/ard.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ard.py b/youtube_dl/extractor/ard.py
index 13a06396d..91c78887a 100644
--- a/youtube_dl/extractor/ard.py
+++ b/youtube_dl/extractor/ard.py
@@ -20,7 +20,7 @@ from ..compat import compat_etree_fromstring
 
 class ARDMediathekIE(InfoExtractor):
     IE_NAME = 'ARD:mediathek'
-    _VALID_URL = r'^https?://(?:(?:www\.)?ardmediathek\.de|mediathek\.daserste\.de)/(?:.*/)(?P<video_id>[0-9]+|[^0-9][^/\?]+)[^/\?]*(?:\?.*)?'
+    _VALID_URL = r'^https?://(?:(?:www\.)?ardmediathek\.de|mediathek\.(?:daserste|rbb-online)\.de)/(?:.*/)(?P<video_id>[0-9]+|[^0-9][^/\?]+)[^/\?]*(?:\?.*)?'
 
     _TESTS = [{
         'url': 'http://www.ardmediathek.de/tv/Dokumentation-und-Reportage/Ich-liebe-das-Leben-trotzdem/rbb-Fernsehen/Video?documentId=29582122&bcastId=3822114',

From 7cdfc4c90fd4f913a96e4493a49af50189b26480 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 17 Jul 2016 16:56:39 +0700
Subject: [PATCH 5536/6219] [mtvservices] Strip description

---
 youtube_dl/extractor/mtv.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index dd0639589..f3ec2ebbc 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -15,6 +15,7 @@ from ..utils import (
     float_or_none,
     HEADRequest,
     sanitized_Request,
+    strip_or_none,
     unescapeHTML,
     url_basename,
     RegexNotFoundError,
@@ -133,7 +134,7 @@ class MTVServicesInfoExtractor(InfoExtractor):
             message += item.text
             raise ExtractorError(message, expected=True)
 
-        description = xpath_text(itemdoc, 'description')
+        description = strip_or_none(xpath_text(itemdoc, 'description'))
 
         title_el = None
         if title_el is None:

From 45550d10395f1e7fe59c4c3ff8ff8d909074a8d0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 17 Jul 2016 16:58:58 +0700
Subject: [PATCH 5537/6219] [comedycentraltv] Add extractor (Closes #10101)

---
 youtube_dl/extractor/comedycentral.py | 33 +++++++++++++++++++++++++++
 1 file changed, 33 insertions(+)

diff --git a/youtube_dl/extractor/comedycentral.py b/youtube_dl/extractor/comedycentral.py
index 2b6aaa3aa..63f68f765 100644
--- a/youtube_dl/extractor/comedycentral.py
+++ b/youtube_dl/extractor/comedycentral.py
@@ -273,3 +273,36 @@ class ComedyCentralShowsIE(MTVServicesInfoExtractor):
             'title': show_name + ' ' + title,
             'description': description,
         }
+
+
+class ComedyCentralTVIE(MTVServicesInfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?comedycentral\.tv/(?:staffeln|shows)/(?P<id>[^/?#&]+)'
+    _TESTS = [{
+        'url': 'http://www.comedycentral.tv/staffeln/7436-the-mindy-project-staffel-4',
+        'info_dict': {
+            'id': 'local_playlist-f99b626bdfe13568579a',
+            'ext': 'flv',
+            'title': 'Episode_the-mindy-project_shows_season-4_episode-3_full-episode_part1',
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        },
+    }, {
+        'url': 'http://www.comedycentral.tv/shows/1074-workaholics',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.comedycentral.tv/shows/1727-the-mindy-project/bonus',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        mrss_url = self._search_regex(
+            r'data-mrss=(["\'])(?P<url>(?:(?!\1).)+)\1',
+            webpage, 'mrss url', group='url')
+
+        return self._get_videos_info_from_url(mrss_url, video_id)

From b1ea6802703f6886b3be2a9975f598fa49fc6d3e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 17 Jul 2016 17:29:36 +0700
Subject: [PATCH 5538/6219] Revert "[bbc] extract more and better qulities from
 Unified Streaming Platform m3u8 manifests"

This reverts commit 0385aa6199206e4ba7745efec73be26c5826286a.
---
 youtube_dl/extractor/bbc.py | 39 ++++++++++++++++++++-----------------
 1 file changed, 21 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 23c6e505b..4b3cd8c65 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -55,11 +55,12 @@ class BBCCoUkIE(InfoExtractor):
             'url': 'http://www.bbc.co.uk/programmes/b039g8p7',
             'info_dict': {
                 'id': 'b039d07m',
-                'ext': 'mp4',
+                'ext': 'flv',
                 'title': 'Leonard Cohen, Kaleidoscope - BBC Radio 4',
                 'description': 'The Canadian poet and songwriter reflects on his musical career.',
             },
             'params': {
+                # rtmp download
                 'skip_download': True,
             }
         },
@@ -91,7 +92,7 @@ class BBCCoUkIE(InfoExtractor):
                 # rtmp download
                 'skip_download': True,
             },
-            'skip': 'this episode is not currently available',
+            'skip': 'Currently BBC iPlayer TV programmes are available to play in the UK only',
         },
         {
             'url': 'http://www.bbc.co.uk/iplayer/episode/p026c7jt/tomorrows-worlds-the-unearthly-history-of-science-fiction-2-invasion',
@@ -106,7 +107,7 @@ class BBCCoUkIE(InfoExtractor):
                 # rtmp download
                 'skip_download': True,
             },
-            'skip': 'this episode is not currently available',
+            'skip': 'Currently BBC iPlayer TV programmes are available to play in the UK only',
         }, {
             'url': 'http://www.bbc.co.uk/programmes/b04v20dw',
             'info_dict': {
@@ -126,12 +127,13 @@ class BBCCoUkIE(InfoExtractor):
             'note': 'Audio',
             'info_dict': {
                 'id': 'p022h44j',
-                'ext': 'mp4',
+                'ext': 'flv',
                 'title': 'BBC Proms Music Guides, Rachmaninov: Symphonic Dances',
                 'description': "In this Proms Music Guide, Andrew McGregor looks at Rachmaninov's Symphonic Dances.",
                 'duration': 227,
             },
             'params': {
+                # rtmp download
                 'skip_download': True,
             }
         }, {
@@ -139,12 +141,13 @@ class BBCCoUkIE(InfoExtractor):
             'note': 'Video',
             'info_dict': {
                 'id': 'p025c103',
-                'ext': 'mp4',
+                'ext': 'flv',
                 'title': 'Reading and Leeds Festival, 2014, Rae Morris - Closer (Live on BBC Three)',
                 'description': 'Rae Morris performs Closer for BBC Three at Reading 2014',
                 'duration': 226,
             },
             'params': {
+                # rtmp download
                 'skip_download': True,
             }
         }, {
@@ -160,7 +163,7 @@ class BBCCoUkIE(InfoExtractor):
                 # rtmp download
                 'skip_download': True,
             },
-            'skip': 'this episode is not currently available',
+            'skip': 'geolocation',
         }, {
             'url': 'http://www.bbc.co.uk/iplayer/episode/b05zmgwn/royal-academy-summer-exhibition',
             'info_dict': {
@@ -174,7 +177,7 @@ class BBCCoUkIE(InfoExtractor):
                 # rtmp download
                 'skip_download': True,
             },
-            'skip': 'this episode is not currently available',
+            'skip': 'geolocation',
         }, {
             # iptv-all mediaset fails with geolocation however there is no geo restriction
             # for this programme at all
@@ -189,17 +192,18 @@ class BBCCoUkIE(InfoExtractor):
                 # rtmp download
                 'skip_download': True,
             },
-            'skip': 'this episode is not currently available on BBC iPlayer Radio',
+            'skip': 'Now it\'s really geo-restricted',
         }, {
             # compact player (https://github.com/rg3/youtube-dl/issues/8147)
             'url': 'http://www.bbc.co.uk/programmes/p028bfkf/player',
             'info_dict': {
                 'id': 'p028bfkj',
-                'ext': 'mp4',
+                'ext': 'flv',
                 'title': 'Extract from BBC documentary Look Stranger - Giant Leeks and Magic Brews',
                 'description': 'Extract from BBC documentary Look Stranger - Giant Leeks and Magic Brews',
             },
             'params': {
+                # rtmp download
                 'skip_download': True,
             },
         }, {
@@ -245,7 +249,7 @@ class BBCCoUkIE(InfoExtractor):
                 pass
             elif transfer_format == 'hls':
                 formats.extend(self._extract_m3u8_formats(
-                    href, programme_id, 'mp4', 'm3u8_native',
+                    href, programme_id, ext='mp4', entry_protocol='m3u8_native',
                     m3u8_id=supplier, fatal=False))
             # Direct link
             else:
@@ -301,14 +305,13 @@ class BBCCoUkIE(InfoExtractor):
         for connection in self._extract_connections(media):
             conn_formats = self._extract_connection(connection, programme_id)
             for format in conn_formats:
-                if format.get('protocol') != 'm3u8_native':
-                    format.update({
-                        'width': width,
-                        'height': height,
-                        'vbr': vbr,
-                        'vcodec': vcodec,
-                        'filesize': file_size,
-                    })
+                format.update({
+                    'width': width,
+                    'height': height,
+                    'vbr': vbr,
+                    'vcodec': vcodec,
+                    'filesize': file_size,
+                })
                 if service:
                     format['format_id'] = '%s_%s' % (service, format['format_id'])
             formats.extend(conn_formats)

From 246080d37878c17e481ec112fd12df52859f3c58 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 17 Jul 2016 18:10:16 +0700
Subject: [PATCH 5539/6219] [viki] Override m3u8 formats acodec

---
 youtube_dl/extractor/viki.py | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index efa15e0b6..fe2bb9df4 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -281,9 +281,16 @@ class VikiIE(VikiBaseIE):
                 r'^(\d+)[pP]$', format_id, 'height', default=None))
             for protocol, format_dict in stream_dict.items():
                 if format_id == 'm3u8':
-                    formats.extend(self._extract_m3u8_formats(
-                        format_dict['url'], video_id, 'mp4', 'm3u8_native',
-                        m3u8_id='m3u8-%s' % protocol, fatal=False))
+                    m3u8_formats = self._extract_m3u8_formats(
+                        format_dict['url'], video_id, 'mp4',
+                        entry_protocol='m3u8_native',
+                        m3u8_id='m3u8-%s' % protocol, fatal=False)
+                    # Despite CODECS metadata in m3u8 all video-only formats
+                    # are actually video+audio
+                    for f in m3u8_formats:
+                        if f.get('acodec') == 'none' and f.get('vcodec') != 'none':
+                            f['acodec'] = None
+                    formats.extend(m3u8_formats)
                 else:
                     formats.append({
                         'url': format_dict['url'],

From 890e6d3309ccddfc0445c4e3b88b16aa7a6c6c14 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 17 Jul 2016 18:12:03 +0700
Subject: [PATCH 5540/6219] [viki] Lower m3u8 preference

http URLs are always provde the same or better quality
---
 youtube_dl/extractor/viki.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index fe2bb9df4..0d6340689 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -283,7 +283,7 @@ class VikiIE(VikiBaseIE):
                 if format_id == 'm3u8':
                     m3u8_formats = self._extract_m3u8_formats(
                         format_dict['url'], video_id, 'mp4',
-                        entry_protocol='m3u8_native',
+                        entry_protocol='m3u8_native', preference=-1,
                         m3u8_id='m3u8-%s' % protocol, fatal=False)
                     # Despite CODECS metadata in m3u8 all video-only formats
                     # are actually video+audio

From 10568217992722c87ee4770ce70752936521e232 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 17 Jul 2016 18:13:54 +0700
Subject: [PATCH 5541/6219] [viki] Fix tests (Closes #10098)

---
 youtube_dl/extractor/viki.py | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index 0d6340689..4351ac457 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -130,7 +130,7 @@ class VikiIE(VikiBaseIE):
     }, {
         # clip
         'url': 'http://www.viki.com/videos/1067139v-the-avengers-age-of-ultron-press-conference',
-        'md5': 'feea2b1d7b3957f70886e6dfd8b8be84',
+        'md5': '86c0b5dbd4d83a6611a79987cc7a1989',
         'info_dict': {
             'id': '1067139v',
             'ext': 'mp4',
@@ -156,15 +156,11 @@ class VikiIE(VikiBaseIE):
             'like_count': int,
             'age_limit': 13,
         },
-        'params': {
-            # m3u8 download
-            'skip_download': True,
-        },
         'skip': 'Blocked in the US',
     }, {
         # episode
         'url': 'http://www.viki.com/videos/44699v-boys-over-flowers-episode-1',
-        'md5': '1f54697dabc8f13f31bf06bb2e4de6db',
+        'md5': '5fa476a902e902783ac7a4d615cdbc7a',
         'info_dict': {
             'id': '44699v',
             'ext': 'mp4',
@@ -200,7 +196,7 @@ class VikiIE(VikiBaseIE):
     }, {
         # non-English description
         'url': 'http://www.viki.com/videos/158036v-love-in-magic',
-        'md5': '013dc282714e22acf9447cad14ff1208',
+        'md5': '1713ae35df5a521b31f6dc40730e7c9c',
         'info_dict': {
             'id': '158036v',
             'ext': 'mp4',

From e8882e7043ad39295349f6c91e379adf7b021356 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 17 Jul 2016 18:34:25 +0700
Subject: [PATCH 5542/6219] [spike] Relax _VALID_URL and improve extraction
 (Closes #10106)

---
 youtube_dl/extractor/spike.py | 27 +++++++++++++++------------
 1 file changed, 15 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/spike.py b/youtube_dl/extractor/spike.py
index 182f286df..63ea7718b 100644
--- a/youtube_dl/extractor/spike.py
+++ b/youtube_dl/extractor/spike.py
@@ -4,11 +4,8 @@ from .mtv import MTVServicesInfoExtractor
 
 
 class SpikeIE(MTVServicesInfoExtractor):
-    _VALID_URL = r'''(?x)https?://
-        (?:www\.spike\.com/(?:video-(?:clips|playlists)|(?:full-)?episodes)/.+|
-         m\.spike\.com/videos/video\.rbml\?id=(?P<id>[^&]+))
-        '''
-    _TEST = {
+    _VALID_URL = r'https?://(?:[^/]+\.)?spike\.com/[^/]+/[\da-z]{6}(?:[/?#&]|$)'
+    _TESTS = [{
         'url': 'http://www.spike.com/video-clips/lhtu8m/auction-hunters-can-allen-ride-a-hundred-year-old-motorcycle',
         'md5': '1a9265f32b0c375793d6c4ce45255256',
         'info_dict': {
@@ -17,13 +14,19 @@ class SpikeIE(MTVServicesInfoExtractor):
             'title': 'Auction Hunters|Can Allen Ride A Hundred Year-Old Motorcycle?',
             'description': 'md5:fbed7e82ed5fad493615b3094a9499cb',
         },
-    }
+    }, {
+        'url': 'http://www.spike.com/video-clips/lhtu8m/',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.spike.com/video-clips/lhtu8m',
+        'only_matching': True,
+    }, {
+        'url': 'http://bellator.spike.com/fight/atwr7k/bellator-158-michael-page-vs-evangelista-cyborg',
+        'only_matching': True,
+    }, {
+        'url': 'http://bellator.spike.com/video-clips/bw6k7n/bellator-158-foundations-michael-venom-page',
+        'only_matching': True,
+    }]
 
     _FEED_URL = 'http://www.spike.com/feeds/mrss/'
     _MOBILE_TEMPLATE = 'http://m.spike.com/videos/video.rbml?id=%s'
-
-    def _real_extract(self, url):
-        mobile_id = self._match_id(url)
-        if mobile_id:
-            url = 'http://www.spike.com/video-clips/%s' % mobile_id
-        return super(SpikeIE, self)._real_extract(url)

From d993a1354def6c81f1f267cb2bfe02c478336ba1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 17 Jul 2016 18:58:47 +0700
Subject: [PATCH 5543/6219] [README.md] Make download URLs consistent

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 44332ea9a..a9f3001a6 100644
--- a/README.md
+++ b/README.md
@@ -17,7 +17,7 @@ youtube-dl - download videos from youtube.com or other video platforms
 
 To install it right away for all UNIX users (Linux, OS X, etc.), type:
 
-    sudo curl -L https://yt-dl.org/latest/youtube-dl -o /usr/local/bin/youtube-dl
+    sudo curl -L https://yt-dl.org/downloads/latest/youtube-dl -o /usr/local/bin/youtube-dl
     sudo chmod a+rx /usr/local/bin/youtube-dl
 
 If you do not have curl, you can alternatively use a recent wget:

From 8188b923db3f1e491ec128fe65a0ed380f1e6a04 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 17 Jul 2016 19:04:29 +0700
Subject: [PATCH 5544/6219] release 2016.07.17

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 docs/supportedsites.md    | 2 ++
 youtube_dl/version.py     | 2 +-
 3 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index b1b8def3c..8b68f371b 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.07.16*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.07.16**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.07.17*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.07.17**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.07.16
+[debug] youtube-dl version 2016.07.17
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index cf194340a..eaa165347 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -449,6 +449,7 @@
  - **niconico**: ニコニコ動画
  - **NiconicoPlaylist**
  - **NineCNineMedia**
+ - **Nintendo**
  - **njoy**: N-JOY
  - **njoy:embed**
  - **Noco**
@@ -645,6 +646,7 @@
  - **stanfordoc**: Stanford Open ClassRoom
  - **Steam**
  - **Stitcher**
+ - **Streamable**
  - **streamcloud.eu**
  - **StreamCZ**
  - **StreetVoice**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 55d07d420..34b62480b 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.07.16'
+__version__ = '2016.07.17'

From a66a73ee905cb1ef9cc63c86d68f5b87cbfe2582 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 18 Jul 2016 02:25:31 +0700
Subject: [PATCH 5545/6219] [ard] Add test for rbb-online

---
 youtube_dl/extractor/ard.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/youtube_dl/extractor/ard.py b/youtube_dl/extractor/ard.py
index 91c78887a..c15cf1575 100644
--- a/youtube_dl/extractor/ard.py
+++ b/youtube_dl/extractor/ard.py
@@ -62,6 +62,17 @@ class ARDMediathekIE(InfoExtractor):
     }, {
         'url': 'http://mediathek.daserste.de/sendungen_a-z/328454_anne-will/22429276_vertrauen-ist-gut-spionieren-ist-besser-geht',
         'only_matching': True,
+    }, {
+        # audio
+        'url': 'http://mediathek.rbb-online.de/radio/Hörspiel/Vor-dem-Fest/kulturradio/Audio?documentId=30796318&topRessort=radio&bcastId=9839158',
+        'md5': '4e8f00631aac0395fee17368ac0e9867',
+        'info_dict': {
+            'id': '30796318',
+            'ext': 'mp3',
+            'title': 'Vor dem Fest',
+            'description': 'md5:c0c1c8048514deaed2a73b3a60eecacb',
+            'duration': 3287,
+        },
     }]
 
     def _extract_media_info(self, media_info_url, webpage, video_id):

From 05087d1b4c4d7171b6121f700468149c175e2058 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 19 Jul 2016 22:49:38 +0700
Subject: [PATCH 5546/6219] [bbc] Improve extraction from sxml playlists

---
 youtube_dl/extractor/bbc.py | 20 +++++++++++++++++---
 1 file changed, 17 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 4b3cd8c65..e3b14c854 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -589,7 +589,8 @@ class BBCIE(BBCCoUkIE):
         'info_dict': {
             'id': '150615_telabyad_kentin_cogu',
             'ext': 'mp4',
-            'title': "YPG: Tel Abyad'ın tamamı kontrolümüzde",
+            'title': "Tel Abyad'da IŞİD bayrağı indirildi YPG bayrağı çekildi",
+            'description': 'md5:33a4805a855c9baf7115fcbde57e7025',
             'timestamp': 1434397334,
             'upload_date': '20150615',
         },
@@ -603,6 +604,7 @@ class BBCIE(BBCCoUkIE):
             'id': '150619_video_honduras_militares_hospitales_corrupcion_aw',
             'ext': 'mp4',
             'title': 'Honduras militariza sus hospitales por nuevo escándalo de corrupción',
+            'description': 'md5:1525f17448c4ee262b64b8f0c9ce66c8',
             'timestamp': 1434713142,
             'upload_date': '20150619',
         },
@@ -818,8 +820,20 @@ class BBCIE(BBCCoUkIE):
                         # http://www.bbc.com/turkce/multimedya/2015/10/151010_vid_ankara_patlama_ani)
                         playlist = data_playable.get('otherSettings', {}).get('playlist', {})
                         if playlist:
-                            entries.append(self._extract_from_playlist_sxml(
-                                playlist.get('progressiveDownloadUrl'), playlist_id, timestamp))
+                            for key in ('progressiveDownload', 'streaming'):
+                                playlist_url = playlist.get('%sUrl' % key)
+                                if not playlist_url:
+                                    continue
+                                try:
+                                    entries.append(self._extract_from_playlist_sxml(
+                                        playlist_url, playlist_id, timestamp))
+                                except Exception as e:
+                                    # Some playlist URL may fail with 500, at the same time
+                                    # the other one may work fine (e.g.
+                                    # http://www.bbc.com/turkce/haberler/2015/06/150615_telabyad_kentin_cogu)
+                                    if isinstance(e.cause, compat_HTTPError) and e.cause.code == 500:
+                                        continue
+                                    raise
 
         if entries:
             return self.playlist_result(entries, playlist_id, playlist_title, playlist_description)

From 4e51ec5f579465d5b64ecfef3614ac5c726fca1f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 19 Jul 2016 22:50:37 +0700
Subject: [PATCH 5547/6219] [extractors] Add import for comedycentral.tv

---
 youtube_dl/extractor/extractors.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 10b2390bf..7314be747 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -156,7 +156,11 @@ from .cnn import (
 )
 from .coub import CoubIE
 from .collegerama import CollegeRamaIE
-from .comedycentral import ComedyCentralIE, ComedyCentralShowsIE
+from .comedycentral import (
+    ComedyCentralIE,
+    ComedyCentralShowsIE,
+    ComedyCentralTVIE,
+)
 from .comcarcoff import ComCarCoffIE
 from .commonmistakes import CommonMistakesIE, UnicodeBOMIE
 from .commonprotocols import RtmpIE

From 740d7c49c251464b25bea5146e054390ada6b997 Mon Sep 17 00:00:00 2001
From: skacurt <skacurt@outlook.com>
Date: Sat, 23 Apr 2016 04:20:04 +0300
Subject: [PATCH 5548/6219] [odatv] Add extractor

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/odatv.py      | 51 ++++++++++++++++++++++++++++++
 2 files changed, 52 insertions(+)
 create mode 100644 youtube_dl/extractor/odatv.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 7314be747..6de50296c 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -587,6 +587,7 @@ from .nytimes import (
     NYTimesArticleIE,
 )
 from .nuvid import NuvidIE
+from .odatv import OdaTVIE
 from .odnoklassniki import OdnoklassnikiIE
 from .oktoberfesttv import OktoberfestTVIE
 from .onet import (
diff --git a/youtube_dl/extractor/odatv.py b/youtube_dl/extractor/odatv.py
new file mode 100644
index 000000000..dbf96aefe
--- /dev/null
+++ b/youtube_dl/extractor/odatv.py
@@ -0,0 +1,51 @@
+# coding: utf-8
+
+from __future__ import unicode_literals
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    remove_start
+)
+import re
+
+
+class OdaTVIE(InfoExtractor):
+    _VALID_URL = r'^https?://(?:www\.)?odatv\.com/(?:mob|vid)_video\.php\?id=(?P<id>[^&]*)'
+    _TESTS = [{
+        'url': 'http://odatv.com/vid_video.php?id=8E388',
+        'md5': 'dc61d052f205c9bf2da3545691485154',
+        'info_dict': {
+            'id': '8E388',
+            'ext': 'mp4',
+            'title': 'md5:69654805a16a16cf9ec9d055e079831c'
+        }
+    }, {
+        'url': 'http://odatv.com/mob_video.php?id=8E388',
+        'md5': 'dc61d052f205c9bf2da3545691485154',
+        'info_dict': {
+            'id': '8E388',
+            'ext': 'mp4',
+            'title': 'md5:69654805a16a16cf9ec9d055e079831c'
+        }
+    }, {
+        'url': 'http://odatv.com/mob_video.php?id=8E900',
+        'md5': '',
+        'info_dict': {
+            'id': '8E900',
+            'ext': 'mp4',
+            'title': 'not found check'
+        }
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        if 'NO VIDEO!' in webpage:
+            raise ExtractorError('Video %s does not exist' % video_id, expected=True)
+
+        return {
+            'id': video_id,
+            'title': remove_start(self._og_search_title(webpage), 'Video: '),
+            'thumbnail': self._og_search_thumbnail(webpage),
+            'url': self._html_search_regex(r"(http.+?video_%s\.mp4)" % re.escape(video_id), webpage, 'url', flags=re.IGNORECASE)
+        }

From 790b06b7d4490e1e54659fafb71167bc459c701c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 20 Jul 2016 21:43:22 +0700
Subject: [PATCH 5549/6219] [odatv] Improve (Closes #9285)

---
 youtube_dl/extractor/odatv.py | 35 +++++++++++++++++------------------
 1 file changed, 17 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/odatv.py b/youtube_dl/extractor/odatv.py
index dbf96aefe..314527f98 100644
--- a/youtube_dl/extractor/odatv.py
+++ b/youtube_dl/extractor/odatv.py
@@ -1,51 +1,50 @@
 # coding: utf-8
-
 from __future__ import unicode_literals
+
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
+    NO_DEFAULT,
     remove_start
 )
-import re
 
 
 class OdaTVIE(InfoExtractor):
-    _VALID_URL = r'^https?://(?:www\.)?odatv\.com/(?:mob|vid)_video\.php\?id=(?P<id>[^&]*)'
+    _VALID_URL = r'https?://(?:www\.)?odatv\.com/(?:mob|vid)_video\.php\?.*\bid=(?P<id>[^&]+)'
     _TESTS = [{
         'url': 'http://odatv.com/vid_video.php?id=8E388',
         'md5': 'dc61d052f205c9bf2da3545691485154',
         'info_dict': {
             'id': '8E388',
             'ext': 'mp4',
-            'title': 'md5:69654805a16a16cf9ec9d055e079831c'
+            'title': 'Artık Davutoğlu ile devam edemeyiz'
         }
     }, {
+        # mobile URL
         'url': 'http://odatv.com/mob_video.php?id=8E388',
-        'md5': 'dc61d052f205c9bf2da3545691485154',
-        'info_dict': {
-            'id': '8E388',
-            'ext': 'mp4',
-            'title': 'md5:69654805a16a16cf9ec9d055e079831c'
-        }
+        'only_matching': True,
     }, {
+        # no video
         'url': 'http://odatv.com/mob_video.php?id=8E900',
-        'md5': '',
-        'info_dict': {
-            'id': '8E900',
-            'ext': 'mp4',
-            'title': 'not found check'
-        }
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
-        if 'NO VIDEO!' in webpage:
+
+        no_video = 'NO VIDEO!' in webpage
+
+        video_url = self._search_regex(
+            r'mp4\s*:\s*(["\'])(?P<url>http.+?)\1', webpage, 'video url',
+            default=None if no_video else NO_DEFAULT, group='url')
+
+        if no_video:
             raise ExtractorError('Video %s does not exist' % video_id, expected=True)
 
         return {
             'id': video_id,
+            'url': video_url,
             'title': remove_start(self._og_search_title(webpage), 'Video: '),
             'thumbnail': self._og_search_thumbnail(webpage),
-            'url': self._html_search_regex(r"(http.+?video_%s\.mp4)" % re.escape(video_id), webpage, 'url', flags=re.IGNORECASE)
         }

From 84e8cca48bdb2cda290f458e31b34a677eb260ae Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 20 Jul 2016 22:41:13 +0700
Subject: [PATCH 5550/6219] [youjizz] Relax _VALID_URL (Closes #10131)

---
 youtube_dl/extractor/youjizz.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/youjizz.py b/youtube_dl/extractor/youjizz.py
index 4150b28da..31e2f9263 100644
--- a/youtube_dl/extractor/youjizz.py
+++ b/youtube_dl/extractor/youjizz.py
@@ -9,8 +9,8 @@ from ..utils import (
 
 
 class YouJizzIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:\w+\.)?youjizz\.com/videos/[^/#?]+-(?P<id>[0-9]+)\.html(?:$|[?#])'
-    _TEST = {
+    _VALID_URL = r'https?://(?:\w+\.)?youjizz\.com/videos/(?:[^/#?]+)?-(?P<id>[0-9]+)\.html(?:$|[?#])'
+    _TESTS = [{
         'url': 'http://www.youjizz.com/videos/zeichentrick-1-2189178.html',
         'md5': '07e15fa469ba384c7693fd246905547c',
         'info_dict': {
@@ -19,7 +19,10 @@ class YouJizzIE(InfoExtractor):
             'title': 'Zeichentrick 1',
             'age_limit': 18,
         }
-    }
+    }, {
+        'url': 'http://www.youjizz.com/videos/-2189178.html',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From c6668e4ad1f0debd4058c0d42229f45f1e49c5d1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 21 Jul 2016 22:34:55 +0700
Subject: [PATCH 5551/6219] [bbc.co.uk:iplayer:playlist] Skip unavailable test

---
 youtube_dl/extractor/bbc.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index e3b14c854..31065cde2 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -1023,6 +1023,7 @@ class BBCCoUkIPlayerPlaylistIE(BBCCoUkPlaylistBaseIE):
             'description': 'French thriller serial about a missing teenager.',
         },
         'playlist_mincount': 6,
+        'skip': 'This programme is not currently available on BBC iPlayer',
     }
 
     def _extract_title_and_description(self, webpage):

From 9158af16cc63998fc42fbdb5c70f38074233fc24 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 21 Jul 2016 22:37:36 +0700
Subject: [PATCH 5552/6219] [bbc.co.uk:iplayer:playlist] Add support for group
 URLs

---
 youtube_dl/extractor/bbc.py | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 31065cde2..9cb7630a1 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -1012,10 +1012,10 @@ class BBCCoUkPlaylistBaseIE(InfoExtractor):
 
 class BBCCoUkIPlayerPlaylistIE(BBCCoUkPlaylistBaseIE):
     IE_NAME = 'bbc.co.uk:iplayer:playlist'
-    _VALID_URL = r'https?://(?:www\.)?bbc\.co\.uk/iplayer/episodes/(?P<id>%s)' % BBCCoUkIE._ID_REGEX
+    _VALID_URL = r'https?://(?:www\.)?bbc\.co\.uk/iplayer/(?:episodes|group)/(?P<id>%s)' % BBCCoUkIE._ID_REGEX
     _URL_TEMPLATE = 'http://www.bbc.co.uk/iplayer/episode/%s'
     _VIDEO_ID_TEMPLATE = r'data-ip-id=["\'](%s)'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.bbc.co.uk/iplayer/episodes/b05rcz9v',
         'info_dict': {
             'id': 'b05rcz9v',
@@ -1024,7 +1024,16 @@ class BBCCoUkIPlayerPlaylistIE(BBCCoUkPlaylistBaseIE):
         },
         'playlist_mincount': 6,
         'skip': 'This programme is not currently available on BBC iPlayer',
-    }
+    }, {
+        # Available for over a year unlike 30 days for most other programmes
+        'url': 'http://www.bbc.co.uk/iplayer/group/p02tcc32',
+        'info_dict': {
+            'id': 'p02tcc32',
+            'title': 'Bohemian Icons',
+            'description': 'md5:683e901041b2fe9ba596f2ab04c4dbe7',
+        },
+        'playlist_mincount': 10,
+    }]
 
     def _extract_title_and_description(self, webpage):
         title = self._search_regex(r'<h1>([^<]+)</h1>', webpage, 'title', fatal=False)

From fd8c8c7dcdb722ae0ed96b8ceb4c844c87f2c896 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 21 Jul 2016 22:58:34 +0700
Subject: [PATCH 5553/6219] [youtube:shared] Relax _VALID_URL

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 49c264c3a..270ee8861 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1736,7 +1736,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
 
 
 class YoutubeSharedVideoIE(InfoExtractor):
-    _VALID_URL = r'(?:https?:)?//(?:www\.)?youtube\.com/shared\?ci=(?P<id>[0-9A-Za-z_-]{11})'
+    _VALID_URL = r'(?:https?:)?//(?:www\.)?youtube\.com/shared\?.*\bci=(?P<id>[0-9A-Za-z_-]{11})'
     IE_NAME = 'youtube:shared'
 
     _TEST = {

From e298d3a08c2c04aceb5b32e6e8f59c7832d65bd3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 22 Jul 2016 21:05:39 +0700
Subject: [PATCH 5554/6219] [youtube] Fix authentication (Closes #10140)

---
 youtube_dl/extractor/youtube.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 270ee8861..268080ba6 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -53,6 +53,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
     """Provide base functions for Youtube extractors"""
     _LOGIN_URL = 'https://accounts.google.com/ServiceLogin'
     _TWOFACTOR_URL = 'https://accounts.google.com/signin/challenge'
+    _PASSWORD_CHALLENGE_URL = 'https://accounts.google.com/signin/challenge/sl/password'
     _NETRC_MACHINE = 'youtube'
     # If True it will raise an error if no login info is provided
     _LOGIN_REQUIRED = False
@@ -116,12 +117,10 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
             'hl': 'en_US',
         }
 
-        login_data = urlencode_postdata(login_form_strs)
-
-        req = sanitized_Request(self._LOGIN_URL, login_data)
         login_results = self._download_webpage(
-            req, None,
-            note='Logging in', errnote='unable to log in', fatal=False)
+            self._PASSWORD_CHALLENGE_URL, None,
+            note='Logging in', errnote='unable to log in', fatal=False,
+            data=urlencode_postdata(login_form_strs))
         if login_results is False:
             return False
 

From add7d2a0e2127f412883b726392a9c3ff6e8be9d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 22 Jul 2016 21:24:09 +0700
Subject: [PATCH 5555/6219] [pornhub] Make error regex less ambiguous (Closes
 #10138)

---
 youtube_dl/extractor/pornhub.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index d2c92531b..20976c101 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -111,7 +111,7 @@ class PornHubIE(InfoExtractor):
         webpage = self._download_webpage(req, video_id)
 
         error_msg = self._html_search_regex(
-            r'(?s)<div[^>]+class=(["\']).*?\b(?:removed|userMessageSection)\b.*?\1[^>]*>(?P<error>.+?)</div>',
+            r'(?s)<div[^>]+class=(["\'])(?:(?!\1).)*\b(?:removed|userMessageSection)\b(?:(?!\1).)*\1[^>]*>(?P<error>.+?)</div>',
             webpage, 'error message', default=None, group='error')
         if error_msg:
             error_msg = re.sub(r'\s+', ' ', error_msg)

From b13647cf3c9ab2fb6afafdd761e04a993fe68bec Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 22 Jul 2016 23:04:13 +0700
Subject: [PATCH 5556/6219] [eporner] Fix extraction (Closes #10139)

---
 youtube_dl/extractor/eporner.py | 82 +++++++++++++++++++++++++--------
 1 file changed, 62 insertions(+), 20 deletions(-)

diff --git a/youtube_dl/extractor/eporner.py b/youtube_dl/extractor/eporner.py
index ac5d0fe24..f3734e9f8 100644
--- a/youtube_dl/extractor/eporner.py
+++ b/youtube_dl/extractor/eporner.py
@@ -4,19 +4,23 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import compat_str
 from ..utils import (
+    encode_base_n,
+    ExtractorError,
+    int_or_none,
     parse_duration,
     str_to_int,
 )
 
 
 class EpornerIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?eporner\.com/hd-porn/(?P<id>\w+)/(?P<display_id>[\w-]+)'
+    _VALID_URL = r'https?://(?:www\.)?eporner\.com/hd-porn/(?P<id>\w+)(?:/(?P<display_id>[\w-]+))?'
     _TESTS = [{
         'url': 'http://www.eporner.com/hd-porn/95008/Infamous-Tiffany-Teen-Strip-Tease-Video/',
         'md5': '39d486f046212d8e1b911c52ab4691f8',
         'info_dict': {
-            'id': '95008',
+            'id': 'qlDUmNsj6VS',
             'display_id': 'Infamous-Tiffany-Teen-Strip-Tease-Video',
             'ext': 'mp4',
             'title': 'Infamous Tiffany Teen Strip Tease Video',
@@ -28,34 +32,72 @@ class EpornerIE(InfoExtractor):
         # New (May 2016) URL layout
         'url': 'http://www.eporner.com/hd-porn/3YRUtzMcWn0/Star-Wars-XXX-Parody/',
         'only_matching': True,
+    }, {
+        'url': 'http://www.eporner.com/hd-porn/3YRUtzMcWn0',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
-        display_id = mobj.group('display_id')
+        display_id = mobj.group('display_id') or video_id
 
-        webpage = self._download_webpage(url, display_id)
-        title = self._html_search_regex(
-            r'<title>(.*?) - EPORNER', webpage, 'title')
+        webpage, urlh = self._download_webpage_handle(url, display_id)
 
-        redirect_url = 'http://www.eporner.com/config5/%s' % video_id
-        player_code = self._download_webpage(
-            redirect_url, display_id, note='Downloading player config')
+        video_id = self._match_id(compat_str(urlh.geturl()))
 
-        sources = self._search_regex(
-            r'(?s)sources\s*:\s*\[\s*({.+?})\s*\]', player_code, 'sources')
+        hash = self._search_regex(
+            r'hash\s*:\s*["\']([\da-f]{32})', webpage, 'hash')
+
+        title = self._og_search_title(webpage, default=None) or self._html_search_regex(
+            r'<title>(.+?) - EPORNER', webpage, 'title')
+
+        # Reverse engineered from vjs.js
+        def calc_hash(s):
+            return ''.join((encode_base_n(int(s[lb:lb + 8], 16), 36) for lb in range(0, 32, 8)))
+
+        video = self._download_json(
+            'http://www.eporner.com/xhr/video/%s' % video_id,
+            display_id, note='Downloading video JSON',
+            query={
+                'hash': calc_hash(hash),
+                'device': 'generic',
+                'domain': 'www.eporner.com',
+                'fallback': 'false',
+            })
+
+        if video.get('available') is False:
+            raise ExtractorError(
+                '%s said: %s' % (self.IE_NAME, video['message']), expected=True)
+
+        sources = video['sources']
 
         formats = []
-        for video_url, format_id in re.findall(r'file\s*:\s*"([^"]+)",\s*label\s*:\s*"([^"]+)"', sources):
-            fmt = {
-                'url': video_url,
-                'format_id': format_id,
-            }
-            m = re.search(r'^(\d+)', format_id)
-            if m:
-                fmt['height'] = int(m.group(1))
-            formats.append(fmt)
+        for kind, formats_dict in sources.items():
+            if not isinstance(formats_dict, dict):
+                continue
+            for format_id, format_dict in formats_dict.items():
+                if not isinstance(format_dict, dict):
+                    continue
+                src = format_dict.get('src')
+                if not isinstance(src, compat_str) or not src.startswith('http'):
+                    continue
+                if kind == 'hls':
+                    formats.extend(self._extract_m3u8_formats(
+                        src, display_id, 'mp4', entry_protocol='m3u8_native',
+                        m3u8_id=kind, fatal=False))
+                else:
+                    height = int_or_none(self._search_regex(
+                        r'(\d+)[pP]', format_id, 'height', default=None))
+                    fps = int_or_none(self._search_regex(
+                        r'(\d+)fps', format_id, 'fps', default=None))
+
+                    formats.append({
+                        'url': src,
+                        'format_id': format_id,
+                        'height': height,
+                        'fps': fps,
+                    })
         self._sort_formats(formats)
 
         duration = parse_duration(self._html_search_meta('duration', webpage))

From 5275efe30d0606deb44c7723271110f66a1e1e65 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 22 Jul 2016 23:11:28 +0700
Subject: [PATCH 5557/6219] release 2016.07.22

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 docs/supportedsites.md    | 2 ++
 youtube_dl/version.py     | 2 +-
 3 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 8b68f371b..f5b444b84 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.07.17*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.07.17**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.07.22*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.07.22**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.07.17
+[debug] youtube-dl version 2016.07.22
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index eaa165347..36b8ec9ad 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -142,6 +142,7 @@
  - **ComCarCoff**
  - **ComedyCentral**
  - **ComedyCentralShows**: The Daily Show / The Colbert Report
+ - **ComedyCentralTV**
  - **CondeNast**: Condé Nast media group: Allure, Architectural Digest, Ars Technica, Bon Appétit, Brides, Condé Nast, Condé Nast Traveler, Details, Epicurious, GQ, Glamour, Golf Digest, SELF, Teen Vogue, The New Yorker, Vanity Fair, Vogue, W Magazine, WIRED
  - **Coub**
  - **Cracked**
@@ -477,6 +478,7 @@
  - **NYTimes**
  - **NYTimesArticle**
  - **ocw.mit.edu**
+ - **OdaTV**
  - **Odnoklassniki**
  - **OktoberfestTV**
  - **on.aol.com**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 34b62480b..732a5dfdf 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.07.17'
+__version__ = '2016.07.22'

From f164b97123d5b61a7dd055c888212a0dc670f04f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 Jul 2016 16:48:59 +0700
Subject: [PATCH 5558/6219] [utils] Add another f4m mimetype to mimetype2ext

---
 youtube_dl/utils.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index e6e0155b4..f5cd6819b 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2123,6 +2123,7 @@ def mimetype2ext(mt):
         'dash+xml': 'mpd',
         'f4m': 'f4m',
         'f4m+xml': 'f4m',
+        'hds+xml': 'f4m',
         'vnd.ms-sstr+xml': 'ism',
     }.get(res, res)
 

From 4671dd41b27e39eb4682189fca44d0f4272a4751 Mon Sep 17 00:00:00 2001
From: Rob van Bekkum <rob_van_bekkum@hotmail.com>
Date: Sat, 27 Feb 2016 18:21:42 +0100
Subject: [PATCH 5559/6219] [arkena:lcp] Add extractors

---
 youtube_dl/extractor/arkenaplay.py | 151 +++++++++++++++++++++++++++++
 youtube_dl/extractor/lcp.py        |  39 ++++++++
 2 files changed, 190 insertions(+)
 create mode 100644 youtube_dl/extractor/arkenaplay.py
 create mode 100644 youtube_dl/extractor/lcp.py

diff --git a/youtube_dl/extractor/arkenaplay.py b/youtube_dl/extractor/arkenaplay.py
new file mode 100644
index 000000000..0061ea196
--- /dev/null
+++ b/youtube_dl/extractor/arkenaplay.py
@@ -0,0 +1,151 @@
+# coding: utf-8
+from __future__ import unicode_literals
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    parse_iso8601
+)
+import re
+
+
+class ArkenaPlayIE(InfoExtractor):
+    IE_NAME = 'ArkenaPlay'
+    _VALID_URL = r'(?P<shortcut>arkena:(?P<version>[0-9]+):(?P<mediatype>[A-Za-z0-9]+):(?P<mediaId>[^:]+):(?P<widgetsettingId>[A-Za-z0-9]+):(?P<accountId>[A-Za-z0-9]+))|(?:(?P<host>https?://(?:www\.)?play\..*\..*)/embed/(?:avp/v[0-9]+/player/[A-Za-z0-9]+/)?(?P<id>.*)?)'
+
+    _TESTS = [{
+        'url': 'http://play.lcp.fr/embed/327336/131064/darkmatter/0',
+        'md5': '6cea4f7d13810464ef8485a924fc3333',
+        'info_dict': {
+            'id': '327336',
+            'url': 're:http://httpod.scdn.arkena.com/11970/327336.*',
+            'ext': 'mp4',
+            'title': '327336',
+            'upload_date': '20160225',
+            'timestamp': 1456391602
+        }
+    }, {
+        # Shortcut for: https://play.arkena.com/embed/avp/v2/player/media/b41dda37-d8e7-4d3f-b1b5-9a9db578bdfe/1/129411
+        'url': 'arkena:2:media:b41dda37-d8e7-4d3f-b1b5-9a9db578bdfe:1:129411',
+        'md5': 'b96f2f71b359a8ecd05ce4e1daa72365',
+        'info_dict': {
+            'id': 'b41dda37-d8e7-4d3f-b1b5-9a9db578bdfe',
+            'url': 'http://88e04ec095b07cd1aa3ea588be47e870.httpcache0.90034-httpcache0.dna.qbrick.com/90034-httpcache0/4bf759a1-00090034/bbb_sunflower_2160p_60fps_normal_720p.mp4',
+            'ext': 'mp4',
+            'title': 'Big Buck Bunny',
+            'description': 'Royalty free test video',
+            'upload_date': '20150528',
+            'timestamp': 1432816365
+        }
+    }]
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        if mobj.group('shortcut'):
+            version = mobj.group('version')
+            mediatype = mobj.group('mediatype')
+            mediaid = mobj.group('mediaId')
+            widgetsettingid = mobj.group('widgetsettingId')
+            accountid = mobj.group('accountId')
+            display_id = '{0}:{1}:{2}:{3}'.format(mediatype, mediaid, widgetsettingid, accountid)
+            media_url = 'https://play.arkena.com/config/avp/v{0}/player/{1}/{2}/{3}/{4}/?callbackMethod=?'.format(
+                version, mediatype, mediaid, widgetsettingid, accountid)
+        else:
+            display_id = self._search_regex(self._VALID_URL, url, 'host_name', group='id')
+            webpage = self._download_webpage(url, display_id)
+
+            media_url_regex = '"(?P<mediainfo>(?P<host>.*)/(c|C)onfig/.*\?callbackMethod=\?)"'
+            media_url = self._html_search_regex(media_url_regex, webpage, 'arkena_media_info_url')
+            hostname = self._html_search_regex(media_url_regex, webpage, 'arkena_media_host', group='host')
+            if not hostname:
+                hostname = self._search_regex(self._VALID_URL, url, 'host_name', group='host')
+                media_url = hostname + media_url
+
+        # Extract the required info of the media files gathered in a dictionary
+        arkena_info = self._download_webpage(media_url, 'arkena_info_')
+        arkena_info_regex = r'\?\((?P<json>.*)\);'
+        media_dict = self._parse_json(self._search_regex(arkena_info_regex, arkena_info, 'json', group='json'),
+                                      display_id)
+
+        # All videos are part of a playlist, a single video is also put in a playlist
+        playlist_items = media_dict.get('Playlist', [])
+        if len(playlist_items) == 0:
+            return self.url_result(url, 'Generic')
+        elif len(playlist_items) == 1:
+            arkena_media_info = playlist_items[0]
+            return self.__extract_from_playlistentry(arkena_media_info)
+        else:
+            entries_info = []
+            for arkena_playlist_item in playlist_items:
+                entries_info.append(self.__extract_from_playlistentry(arkena_playlist_item))
+            return {
+                'id': display_id,
+                'entries': entries_info
+            }
+
+    def __extract_from_playlistentry(self, arkena_playlistentry_info):
+        media_info = arkena_playlistentry_info.get('MediaInfo', {})
+        thumbnails = self.__get_thumbnails(media_info)
+        title = media_info.get('Title')
+        description = media_info.get('Description')
+        video_id = media_info.get('VideoId')
+        timestamp = parse_iso8601(media_info.get('PublishDate'))
+        formats = self.__get_video_formats(arkena_playlistentry_info, video_id)
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': formats,
+            'thumbnails': thumbnails,
+            'description': description,
+            'timestamp': timestamp
+        }
+
+    def __get_thumbnails(self, arkena_mediainfo):
+        thumbnails = []
+        thumbnails_info = arkena_mediainfo.get('Poster')
+        if not thumbnails_info:
+            return None
+        for thumbnail in thumbnails_info:
+            thumbnail_url = thumbnail.get('Url')
+            if not thumbnail_url:
+                continue
+            thumbnails.append({
+                'url': thumbnail_url,
+                'width': int_or_none(thumbnail.get('Size'))
+            })
+        return thumbnails
+
+    def __get_video_formats(self, media_files_info, video_id):
+        formats = []
+        media_files = media_files_info.get('MediaFiles')
+        if not media_files:
+            return None
+
+        for type_name, video_files_json in media_files.iteritems():
+            for video_info in video_files_json:
+                video_url = video_info.get('Url')
+                if not video_url:
+                    continue
+                type = video_info.get('Type')
+                if type_name in ['Mp4', 'WebM', 'Flash']:
+                    bitrate = int_or_none(video_info.get('Bitrate'), scale=1000)
+                    ext = None
+                    if type == 'video/mp4':
+                        ext = 'mp4'
+                    elif type == 'video/webm':
+                        ext = 'webm'
+                    elif type == 'video/x-flv':
+                        ext = 'flv'
+                    formats.append({
+                        'url': video_url,
+                        'ext': ext,
+                        'tbr': bitrate
+                    })
+                elif type_name == 'M3u8' and type == 'application/x-mpegURL':
+                    formats.extend(self._extract_m3u8_formats(video_url, video_id, 'mp4', m3u8_id='hls', fatal=False))
+                elif type_name == 'Flash' and type == 'application/hds+xml':
+                    formats.extend(self._extract_f4m_formats(video_url, video_id, f4m_id='hds', fatal=False))
+                elif type_name == 'Dash' and type == 'application/dash+xml':
+                    formats.extend(self._extract_mpd_formats(video_url, video_id, mpd_id='dash', fatal=False))
+
+        self._sort_formats(formats)
+        return formats
\ No newline at end of file
diff --git a/youtube_dl/extractor/lcp.py b/youtube_dl/extractor/lcp.py
new file mode 100644
index 000000000..38d7502df
--- /dev/null
+++ b/youtube_dl/extractor/lcp.py
@@ -0,0 +1,39 @@
+# coding: utf-8
+from __future__ import unicode_literals
+from .common import InfoExtractor
+
+class LcpIE(InfoExtractor):
+    IE_NAME = 'LCP'
+    _VALID_URL = r'https?://(?:www\.)?lcp\.fr/(?:[^\/]+/)*(?P<id>[^/]+)'
+
+    _TESTS = [{
+        'url': 'http://www.lcp.fr/la-politique-en-video/schwartzenberg-prg-preconise-francois-hollande-de-participer-une-primaire',
+        'md5': 'ab96c4dae94322ece1e98d97c8dc7807',
+        'info_dict': {
+            'id': 'd56d03e9',
+            'url': 're:http://httpod.scdn.arkena.com/11970/d56d03e9_.*',
+            'ext': 'mp4',
+            'title': 'd56d03e9',
+            'upload_date': '20160226',
+            'timestamp': 1456488895
+        }
+    }, {
+        'url': 'http://www.lcp.fr/le-direct',
+        'info_dict': {
+            'title': 'Le direct | LCP Assembl\xe9e nationale',
+            'id': 'le-direct',
+        },
+        'playlist_mincount': 1
+    }]
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+
+        embed_url_regex = r'"(?P<url>(?:https?://(?:www\.)?)?play\.lcp\.fr/embed/[A-za-z0-9]+/[A-za-z0-9]+/[A-za-z0-9]+/[A-za-z0-9]+)"'
+        embed_url = self._html_search_regex(embed_url_regex, webpage, 'player_url', default=None, fatal=False)
+        if not embed_url:
+            return self.url_result(url, 'Generic')
+
+        title = self._og_search_title(webpage, default=None)
+        return self.url_result(embed_url, 'ArkenaPlay', video_id=display_id, video_title=title)

From bbe1f3634aaa800bef99b1edba8058f337da98fd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 Jul 2016 17:55:54 +0700
Subject: [PATCH 5560/6219] [arkena] Improve extraction (Closes #8682)

---
 youtube_dl/extractor/arkena.py | 115 +++++++++++++++++++++++++++++++++
 1 file changed, 115 insertions(+)
 create mode 100644 youtube_dl/extractor/arkena.py

diff --git a/youtube_dl/extractor/arkena.py b/youtube_dl/extractor/arkena.py
new file mode 100644
index 000000000..d003a027b
--- /dev/null
+++ b/youtube_dl/extractor/arkena.py
@@ -0,0 +1,115 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    determine_ext,
+    float_or_none,
+    int_or_none,
+    mimetype2ext,
+    parse_iso8601,
+    strip_jsonp,
+)
+
+
+class ArkenaIE(InfoExtractor):
+    _VALID_URL = r'https?://play\.arkena\.com/(?:config|embed)/avp/v\d/player/media/(?P<id>[^/]+)/[^/]+/(?P<account_id>\d+)'
+    _TESTS = [{
+        'url': 'https://play.arkena.com/embed/avp/v2/player/media/b41dda37-d8e7-4d3f-b1b5-9a9db578bdfe/1/129411',
+        'md5': 'b96f2f71b359a8ecd05ce4e1daa72365',
+        'info_dict': {
+            'id': 'b41dda37-d8e7-4d3f-b1b5-9a9db578bdfe',
+            'ext': 'mp4',
+            'title': 'Big Buck Bunny',
+            'description': 'Royalty free test video',
+            'timestamp': 1432816365,
+            'upload_date': '20150528',
+            'is_live': False,
+        },
+    }, {
+        'url': 'https://play.arkena.com/config/avp/v2/player/media/b41dda37-d8e7-4d3f-b1b5-9a9db578bdfe/1/129411/?callbackMethod=jQuery1111023664739129262213_1469227693893',
+        'only_matching': True,
+    }, {
+        'url': 'http://play.arkena.com/config/avp/v1/player/media/327336/darkmatter/131064/?callbackMethod=jQuery1111002221189684892677_1469227595972',
+        'only_matching': True,
+    }, {
+        'url': 'http://play.arkena.com/embed/avp/v1/player/media/327336/darkmatter/131064/',
+        'only_matching': True,
+    }]
+
+    @staticmethod
+    def _extract_url(webpage):
+        # See https://support.arkena.com/display/PLAY/Ways+to+embed+your+video
+        mobj = re.search(
+            r'<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//play\.arkena\.com/embed/avp/.+?)\1',
+            webpage)
+        if mobj:
+            return mobj.group('url')
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        account_id = mobj.group('account_id')
+
+        playlist = self._download_json(
+            'https://play.arkena.com/config/avp/v2/player/media/%s/0/%s/?callbackMethod=_'
+            % (video_id, account_id),
+            video_id, transform_source=strip_jsonp)['Playlist'][0]
+
+        media_info = playlist['MediaInfo']
+        title = media_info['Title']
+        media_files = playlist['MediaFiles']
+
+        is_live = False
+        formats = []
+        for kind_case, kind_formats in media_files.items():
+            kind = kind_case.lower()
+            for f in kind_formats:
+                f_url = f.get('Url')
+                if not f_url:
+                    continue
+                is_live = f.get('Live') == 'true'
+                exts = (mimetype2ext(f.get('Type')), determine_ext(f_url, None))
+                if kind == 'm3u8' or 'm3u8' in exts:
+                    formats.extend(self._extract_m3u8_formats(
+                        f_url, video_id, 'mp4',
+                        entry_protocol='m3u8' if is_live else 'm3u8_native',
+                        m3u8_id=kind, fatal=False, live=is_live))
+                elif kind == 'flash' or 'f4m' in exts:
+                    formats.extend(self._extract_f4m_formats(
+                        f_url, video_id, f4m_id=kind, fatal=False))
+                elif kind == 'dash' or 'mpd' in exts:
+                    formats.extend(self._extract_mpd_formats(
+                        f_url, video_id, mpd_id=kind, fatal=False))
+                elif kind == 'silverlight':
+                    # TODO: process when ism is supported (see
+                    # https://github.com/rg3/youtube-dl/issues/8118)
+                    pass
+                else:
+                    tbr = float_or_none(f.get('Bitrate'), 1000)
+                    formats.append({
+                        'url': f_url,
+                        'format_id': '%s-%d' % (kind, tbr) if tbr else kind,
+                        'tbr': tbr,
+                    })
+        self._sort_formats(formats)
+
+        description = media_info.get('Description')
+        video_id = media_info.get('VideoId') or video_id
+        timestamp = parse_iso8601(media_info.get('PublishDate'))
+        thumbnails = [{
+            'url': thumbnail['Url'],
+            'width': int_or_none(thumbnail.get('Size')),
+        } for thumbnail in (media_info.get('Poster') or []) if thumbnail.get('Url')]
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'timestamp': timestamp,
+            'is_live': is_live,
+            'thumbnails': thumbnails,
+            'formats': formats,
+        }

From c8e170b2092f5e2ad9ea8fd7fb2eedd35e307a1c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 Jul 2016 17:56:11 +0700
Subject: [PATCH 5561/6219] [lcp] Improve extraction

---
 youtube_dl/extractor/lcp.py | 81 ++++++++++++++++++++++++++++++-------
 1 file changed, 66 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/lcp.py b/youtube_dl/extractor/lcp.py
index 38d7502df..ade27a99e 100644
--- a/youtube_dl/extractor/lcp.py
+++ b/youtube_dl/extractor/lcp.py
@@ -1,39 +1,90 @@
 # coding: utf-8
 from __future__ import unicode_literals
+
 from .common import InfoExtractor
+from .arkena import ArkenaIE
+
+
+class LcpPlayIE(ArkenaIE):
+    _VALID_URL = r'https?://play\.lcp\.fr/embed/(?P<id>[^/]+)/(?P<account_id>[^/]+)/[^/]+/[^/]+'
+    _TESTS = [{
+        'url': 'http://play.lcp.fr/embed/327336/131064/darkmatter/0',
+        'md5': 'b8bd9298542929c06c1c15788b1f277a',
+        'info_dict': {
+            'id': '327336',
+            'ext': 'mp4',
+            'title': '327336',
+            'timestamp': 1456391602,
+            'upload_date': '20160225',
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }]
+
 
 class LcpIE(InfoExtractor):
-    IE_NAME = 'LCP'
-    _VALID_URL = r'https?://(?:www\.)?lcp\.fr/(?:[^\/]+/)*(?P<id>[^/]+)'
+    _VALID_URL = r'https?://(?:www\.)?lcp\.fr/(?:[^/]+/)*(?P<id>[^/]+)'
 
     _TESTS = [{
+        # arkena embed
         'url': 'http://www.lcp.fr/la-politique-en-video/schwartzenberg-prg-preconise-francois-hollande-de-participer-une-primaire',
-        'md5': 'ab96c4dae94322ece1e98d97c8dc7807',
+        'md5': 'b8bd9298542929c06c1c15788b1f277a',
         'info_dict': {
             'id': 'd56d03e9',
-            'url': 're:http://httpod.scdn.arkena.com/11970/d56d03e9_.*',
             'ext': 'mp4',
-            'title': 'd56d03e9',
+            'title': 'Schwartzenberg (PRG) préconise à François Hollande de participer à une primaire à gauche',
+            'description': 'md5:96ad55009548da9dea19f4120c6c16a8',
+            'timestamp': 1456488895,
             'upload_date': '20160226',
-            'timestamp': 1456488895
-        }
+        },
+        'params': {
+            'skip_download': True,
+        },
     }, {
+        # dailymotion live stream
         'url': 'http://www.lcp.fr/le-direct',
         'info_dict': {
-            'title': 'Le direct | LCP Assembl\xe9e nationale',
-            'id': 'le-direct',
+            'id': 'xji3qy',
+            'ext': 'mp4',
+            'title': 'La Chaine Parlementaire (LCP), Live TNT',
+            'description': 'md5:5c69593f2de0f38bd9a949f2c95e870b',
+            'uploader': 'LCP',
+            'uploader_id': 'xbz33d',
+            'timestamp': 1308923058,
+            'upload_date': '20110624',
         },
-        'playlist_mincount': 1
+        'params': {
+            # m3u8 live stream
+            'skip_download': True,
+        },
+    }, {
+        'url': 'http://www.lcp.fr/emissions/277792-les-volontaires',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
+
         webpage = self._download_webpage(url, display_id)
 
-        embed_url_regex = r'"(?P<url>(?:https?://(?:www\.)?)?play\.lcp\.fr/embed/[A-za-z0-9]+/[A-za-z0-9]+/[A-za-z0-9]+/[A-za-z0-9]+)"'
-        embed_url = self._html_search_regex(embed_url_regex, webpage, 'player_url', default=None, fatal=False)
-        if not embed_url:
+        play_url = self._search_regex(
+            r'<iframe[^>]+src=(["\'])(?P<url>%s?(?:(?!\1).)*)\1' % LcpPlayIE._VALID_URL,
+            webpage, 'play iframe', default=None, group='url')
+
+        if not play_url:
             return self.url_result(url, 'Generic')
 
-        title = self._og_search_title(webpage, default=None)
-        return self.url_result(embed_url, 'ArkenaPlay', video_id=display_id, video_title=title)
+        title = self._og_search_title(webpage, default=None) or self._html_search_meta(
+            'twitter:title', webpage, fatal=True)
+        description = self._html_search_meta(
+            ('description', 'twitter:description'), webpage)
+
+        return {
+            '_type': 'url_transparent',
+            'ie_key': LcpPlayIE.ie_key(),
+            'url': play_url,
+            'display_id': display_id,
+            'title': title,
+            'description': description,
+        }

From 0673741af33557dbcfe18f06fbc9c0145e64822c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 Jul 2016 17:56:29 +0700
Subject: [PATCH 5562/6219] [extractors] Add imports for arkena and lcp

---
 youtube_dl/extractor/extractors.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 6de50296c..3ca0ef83a 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -44,6 +44,7 @@ from .appletrailers import (
     AppleTrailersSectionIE,
 )
 from .archiveorg import ArchiveOrgIE
+from .arkena import ArkenaIE
 from .ard import (
     ARDIE,
     ARDMediathekIE,
@@ -397,6 +398,10 @@ from .kuwo import (
 )
 from .la7 import LA7IE
 from .laola1tv import Laola1TvIE
+from .lcp import (
+    LcpPlayIE,
+    LcpIE,
+)
 from .learnr import LearnrIE
 from .lecture2go import Lecture2GoIE
 from .lemonde import LemondeIE

From 1979969f91abb514f0fe2bfdb3389fabf8e86488 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 Jul 2016 17:56:48 +0700
Subject: [PATCH 5563/6219] [extractor/generic] Add support for arkena embeds

---
 youtube_dl/extractor/generic.py | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 6d346cb1c..011940580 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -62,6 +62,7 @@ from .videomore import VideomoreIE
 from .googledrive import GoogleDriveIE
 from .jwplatform import JWPlatformIE
 from .digiteka import DigitekaIE
+from .arkena import ArkenaIE
 from .instagram import InstagramIE
 from .liveleak import LiveLeakIE
 from .threeqsdn import ThreeQSDNIE
@@ -1342,6 +1343,23 @@ class GenericIE(InfoExtractor):
             },
             'add_ie': ['Vimeo'],
         },
+        {
+            'url': 'https://support.arkena.com/display/PLAY/Ways+to+embed+your+video',
+            'md5': 'b96f2f71b359a8ecd05ce4e1daa72365',
+            'info_dict': {
+                'id': 'b41dda37-d8e7-4d3f-b1b5-9a9db578bdfe',
+                'ext': 'mp4',
+                'title': 'Big Buck Bunny',
+                'description': 'Royalty free test video',
+                'timestamp': 1432816365,
+                'upload_date': '20150528',
+                'is_live': False,
+            },
+            'params': {
+                'skip_download': True,
+            },
+            'add_ie': [ArkenaIE.ie_key()],
+        },
         # {
         #     # TODO: find another test
         #     # http://schema.org/VideoObject
@@ -2146,6 +2164,11 @@ class GenericIE(InfoExtractor):
         if digiteka_url:
             return self.url_result(self._proto_relative_url(digiteka_url), DigitekaIE.ie_key())
 
+        # Look for Arkena embeds
+        arkena_url = ArkenaIE._extract_url(webpage)
+        if arkena_url:
+            return self.url_result(arkena_url, ArkenaIE.ie_key())
+
         # Look for Limelight embeds
         mobj = re.search(r'LimelightPlayer\.doLoad(Media|Channel|ChannelList)\(["\'](?P<id>[a-z0-9]{32})', webpage)
         if mobj:

From 3a8947650b6545e105ca43a551f7c35007b6f647 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 Jul 2016 17:57:55 +0700
Subject: [PATCH 5564/6219] [arkenaplay] Remove extractor

---
 youtube_dl/extractor/arkenaplay.py | 151 -----------------------------
 1 file changed, 151 deletions(-)
 delete mode 100644 youtube_dl/extractor/arkenaplay.py

diff --git a/youtube_dl/extractor/arkenaplay.py b/youtube_dl/extractor/arkenaplay.py
deleted file mode 100644
index 0061ea196..000000000
--- a/youtube_dl/extractor/arkenaplay.py
+++ /dev/null
@@ -1,151 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-from .common import InfoExtractor
-from ..utils import (
-    int_or_none,
-    parse_iso8601
-)
-import re
-
-
-class ArkenaPlayIE(InfoExtractor):
-    IE_NAME = 'ArkenaPlay'
-    _VALID_URL = r'(?P<shortcut>arkena:(?P<version>[0-9]+):(?P<mediatype>[A-Za-z0-9]+):(?P<mediaId>[^:]+):(?P<widgetsettingId>[A-Za-z0-9]+):(?P<accountId>[A-Za-z0-9]+))|(?:(?P<host>https?://(?:www\.)?play\..*\..*)/embed/(?:avp/v[0-9]+/player/[A-Za-z0-9]+/)?(?P<id>.*)?)'
-
-    _TESTS = [{
-        'url': 'http://play.lcp.fr/embed/327336/131064/darkmatter/0',
-        'md5': '6cea4f7d13810464ef8485a924fc3333',
-        'info_dict': {
-            'id': '327336',
-            'url': 're:http://httpod.scdn.arkena.com/11970/327336.*',
-            'ext': 'mp4',
-            'title': '327336',
-            'upload_date': '20160225',
-            'timestamp': 1456391602
-        }
-    }, {
-        # Shortcut for: https://play.arkena.com/embed/avp/v2/player/media/b41dda37-d8e7-4d3f-b1b5-9a9db578bdfe/1/129411
-        'url': 'arkena:2:media:b41dda37-d8e7-4d3f-b1b5-9a9db578bdfe:1:129411',
-        'md5': 'b96f2f71b359a8ecd05ce4e1daa72365',
-        'info_dict': {
-            'id': 'b41dda37-d8e7-4d3f-b1b5-9a9db578bdfe',
-            'url': 'http://88e04ec095b07cd1aa3ea588be47e870.httpcache0.90034-httpcache0.dna.qbrick.com/90034-httpcache0/4bf759a1-00090034/bbb_sunflower_2160p_60fps_normal_720p.mp4',
-            'ext': 'mp4',
-            'title': 'Big Buck Bunny',
-            'description': 'Royalty free test video',
-            'upload_date': '20150528',
-            'timestamp': 1432816365
-        }
-    }]
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        if mobj.group('shortcut'):
-            version = mobj.group('version')
-            mediatype = mobj.group('mediatype')
-            mediaid = mobj.group('mediaId')
-            widgetsettingid = mobj.group('widgetsettingId')
-            accountid = mobj.group('accountId')
-            display_id = '{0}:{1}:{2}:{3}'.format(mediatype, mediaid, widgetsettingid, accountid)
-            media_url = 'https://play.arkena.com/config/avp/v{0}/player/{1}/{2}/{3}/{4}/?callbackMethod=?'.format(
-                version, mediatype, mediaid, widgetsettingid, accountid)
-        else:
-            display_id = self._search_regex(self._VALID_URL, url, 'host_name', group='id')
-            webpage = self._download_webpage(url, display_id)
-
-            media_url_regex = '"(?P<mediainfo>(?P<host>.*)/(c|C)onfig/.*\?callbackMethod=\?)"'
-            media_url = self._html_search_regex(media_url_regex, webpage, 'arkena_media_info_url')
-            hostname = self._html_search_regex(media_url_regex, webpage, 'arkena_media_host', group='host')
-            if not hostname:
-                hostname = self._search_regex(self._VALID_URL, url, 'host_name', group='host')
-                media_url = hostname + media_url
-
-        # Extract the required info of the media files gathered in a dictionary
-        arkena_info = self._download_webpage(media_url, 'arkena_info_')
-        arkena_info_regex = r'\?\((?P<json>.*)\);'
-        media_dict = self._parse_json(self._search_regex(arkena_info_regex, arkena_info, 'json', group='json'),
-                                      display_id)
-
-        # All videos are part of a playlist, a single video is also put in a playlist
-        playlist_items = media_dict.get('Playlist', [])
-        if len(playlist_items) == 0:
-            return self.url_result(url, 'Generic')
-        elif len(playlist_items) == 1:
-            arkena_media_info = playlist_items[0]
-            return self.__extract_from_playlistentry(arkena_media_info)
-        else:
-            entries_info = []
-            for arkena_playlist_item in playlist_items:
-                entries_info.append(self.__extract_from_playlistentry(arkena_playlist_item))
-            return {
-                'id': display_id,
-                'entries': entries_info
-            }
-
-    def __extract_from_playlistentry(self, arkena_playlistentry_info):
-        media_info = arkena_playlistentry_info.get('MediaInfo', {})
-        thumbnails = self.__get_thumbnails(media_info)
-        title = media_info.get('Title')
-        description = media_info.get('Description')
-        video_id = media_info.get('VideoId')
-        timestamp = parse_iso8601(media_info.get('PublishDate'))
-        formats = self.__get_video_formats(arkena_playlistentry_info, video_id)
-        return {
-            'id': video_id,
-            'title': title,
-            'formats': formats,
-            'thumbnails': thumbnails,
-            'description': description,
-            'timestamp': timestamp
-        }
-
-    def __get_thumbnails(self, arkena_mediainfo):
-        thumbnails = []
-        thumbnails_info = arkena_mediainfo.get('Poster')
-        if not thumbnails_info:
-            return None
-        for thumbnail in thumbnails_info:
-            thumbnail_url = thumbnail.get('Url')
-            if not thumbnail_url:
-                continue
-            thumbnails.append({
-                'url': thumbnail_url,
-                'width': int_or_none(thumbnail.get('Size'))
-            })
-        return thumbnails
-
-    def __get_video_formats(self, media_files_info, video_id):
-        formats = []
-        media_files = media_files_info.get('MediaFiles')
-        if not media_files:
-            return None
-
-        for type_name, video_files_json in media_files.iteritems():
-            for video_info in video_files_json:
-                video_url = video_info.get('Url')
-                if not video_url:
-                    continue
-                type = video_info.get('Type')
-                if type_name in ['Mp4', 'WebM', 'Flash']:
-                    bitrate = int_or_none(video_info.get('Bitrate'), scale=1000)
-                    ext = None
-                    if type == 'video/mp4':
-                        ext = 'mp4'
-                    elif type == 'video/webm':
-                        ext = 'webm'
-                    elif type == 'video/x-flv':
-                        ext = 'flv'
-                    formats.append({
-                        'url': video_url,
-                        'ext': ext,
-                        'tbr': bitrate
-                    })
-                elif type_name == 'M3u8' and type == 'application/x-mpegURL':
-                    formats.extend(self._extract_m3u8_formats(video_url, video_id, 'mp4', m3u8_id='hls', fatal=False))
-                elif type_name == 'Flash' and type == 'application/hds+xml':
-                    formats.extend(self._extract_f4m_formats(video_url, video_id, f4m_id='hds', fatal=False))
-                elif type_name == 'Dash' and type == 'application/dash+xml':
-                    formats.extend(self._extract_mpd_formats(video_url, video_id, mpd_id='dash', fatal=False))
-
-        self._sort_formats(formats)
-        return formats
\ No newline at end of file

From 6548030a17ba36de9f4a340cb71413d32b90f3a6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 Jul 2016 18:00:19 +0700
Subject: [PATCH 5565/6219] Credit @rvanbekkum for arkena (#8682)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index f762e8a16..890c827a0 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -178,3 +178,4 @@ Artur Krysiak
 Jakub Adam Wieczorek
 Aleksandar Topuzović
 Nehal Patel
+Rob van Bekkum

From f1991ce928d2a670b37ce4ac0e088459d9e28202 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 Jul 2016 18:07:55 +0700
Subject: [PATCH 5566/6219] [arkena] Skip dash formats

---
 youtube_dl/extractor/arkena.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/arkena.py b/youtube_dl/extractor/arkena.py
index d003a027b..d7c5eeb8a 100644
--- a/youtube_dl/extractor/arkena.py
+++ b/youtube_dl/extractor/arkena.py
@@ -81,12 +81,15 @@ class ArkenaIE(InfoExtractor):
                     formats.extend(self._extract_f4m_formats(
                         f_url, video_id, f4m_id=kind, fatal=False))
                 elif kind == 'dash' or 'mpd' in exts:
-                    formats.extend(self._extract_mpd_formats(
-                        f_url, video_id, mpd_id=kind, fatal=False))
+                    # TODO: Current DASH formats are broken - $Time$ pattern in
+                    # <SegmentTemplate> not implemented yet
+                    # formats.extend(self._extract_mpd_formats(
+                    #    f_url, video_id, mpd_id=kind, fatal=False))
+                    continue
                 elif kind == 'silverlight':
                     # TODO: process when ism is supported (see
                     # https://github.com/rg3/youtube-dl/issues/8118)
-                    pass
+                    continue
                 else:
                     tbr = float_or_none(f.get('Bitrate'), 1000)
                     formats.append({

From b4a131e1a5c345b17df37a9b0c76e612855e5402 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 24 Jul 2016 04:36:49 +0700
Subject: [PATCH 5567/6219] [facebook] Relax _VALID_URL (Closes #10151)

---
 youtube_dl/extractor/facebook.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index cdb093262..0fb781a73 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -27,7 +27,7 @@ class FacebookIE(InfoExtractor):
     _VALID_URL = r'''(?x)
                 (?:
                     https?://
-                        (?:\w+\.)?facebook\.com/
+                        (?:[\w-]+\.)?facebook\.com/
                         (?:[^#]*?\#!/)?
                         (?:
                             (?:
@@ -127,6 +127,9 @@ class FacebookIE(InfoExtractor):
     }, {
         'url': 'https://www.facebook.com/groups/164828000315060/permalink/764967300301124/',
         'only_matching': True,
+    }, {
+        'url': 'https://zh-hk.facebook.com/peoplespower/videos/1135894589806027/',
+        'only_matching': True,
     }]
 
     @staticmethod

From 111de00289d8c019764e79247568248f8a4b11f6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?D=C3=A9stin=20Reed?= <trox1972@users.noreply.github.com>
Date: Sat, 23 Jul 2016 17:02:50 +0200
Subject: [PATCH 5568/6219] [DailyMail] Improve title and description
 extraction

---
 youtube_dl/extractor/dailymail.py | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/dailymail.py b/youtube_dl/extractor/dailymail.py
index b60a1d813..98c835bf1 100644
--- a/youtube_dl/extractor/dailymail.py
+++ b/youtube_dl/extractor/dailymail.py
@@ -5,19 +5,20 @@ from .common import InfoExtractor
 from ..utils import (
     int_or_none,
     determine_protocol,
+    unescapeHTML,
 )
 
 
 class DailyMailIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?dailymail\.co\.uk/video/[^/]+/video-(?P<id>[0-9]+)'
     _TEST = {
-        'url': 'http://www.dailymail.co.uk/video/sciencetech/video-1288527/Turn-video-impressionist-masterpiece.html',
-        'md5': '2f639d446394f53f3a33658b518b6615',
+        'url': 'http://www.dailymail.co.uk/video/tvshowbiz/video-1295863/The-Mountain-appears-sparkling-water-ad-Heavy-Bubbles.html',
+        'md5': 'f6129624562251f628296c3a9ffde124',
         'info_dict': {
-            'id': '1288527',
+            'id': '1295863',
             'ext': 'mp4',
-            'title': 'Turn any video into an impressionist masterpiece',
-            'description': 'md5:88ddbcb504367987b2708bb38677c9d2',
+            'title': 'The Mountain appears in sparkling water ad for \'Heavy Bubbles\'',
+            'description': 'md5:a93d74b6da172dd5dc4d973e0b766a84',
         }
     }
 
@@ -26,7 +27,7 @@ class DailyMailIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
         video_data = self._parse_json(self._search_regex(
             r"data-opts='({.+?})'", webpage, 'video data'), video_id)
-        title = video_data['title']
+        title = unescapeHTML(video_data['title'])
         video_sources = self._download_json(video_data.get(
             'sources', {}).get('url') or 'http://www.dailymail.co.uk/api/player/%s/video-sources.json' % video_id, video_id)
 
@@ -55,7 +56,7 @@ class DailyMailIE(InfoExtractor):
         return {
             'id': video_id,
             'title': title,
-            'description': video_data.get('descr'),
+            'description': unescapeHTML(video_data.get('descr')),
             'thumbnail': video_data.get('poster') or video_data.get('thumbnail'),
             'formats': formats,
         }

From f09483485728871286f2670c8b8d62f56a89b1e1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 24 Jul 2016 10:27:16 +0700
Subject: [PATCH 5569/6219] [extractor/common] Add support for $ in
 SegmentTemplate in MPD manifests

---
 youtube_dl/extractor/common.py | 61 ++++++++++++++++++++++++++++------
 1 file changed, 50 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 29544c1a8..b8a76e3cb 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1481,6 +1481,13 @@ class InfoExtractor(object):
             compat_etree_fromstring(mpd.encode('utf-8')), mpd_id, mpd_base_url, formats_dict=formats_dict)
 
     def _parse_mpd_formats(self, mpd_doc, mpd_id=None, mpd_base_url='', formats_dict={}):
+        """
+        Parse formats from MPD manifest.
+        References:
+         1. MPEG-DASH Standard, ISO/IEC 23009-1:2014(E),
+            http://standards.iso.org/ittf/PubliclyAvailableStandards/c065274_ISO_IEC_23009-1_2014.zip
+         2. https://en.wikipedia.org/wiki/Dynamic_Adaptive_Streaming_over_HTTP
+        """
         if mpd_doc.get('type') == 'dynamic':
             return []
 
@@ -1513,8 +1520,16 @@ class InfoExtractor(object):
                         s_e = segment_timeline.findall(_add_ns('S'))
                         if s_e:
                             ms_info['total_number'] = 0
+                            ms_info['s'] = []
                             for s in s_e:
-                                ms_info['total_number'] += 1 + int(s.get('r', '0'))
+                                r = int(s.get('r', 0))
+                                ms_info['total_number'] += 1 + r
+                                ms_info['s'].append({
+                                    't': int(s.get('t', 0)),
+                                    # @d is mandatory (see [1, 5.3.9.6.2, Table 17, page 60])
+                                    'd': int(s.attrib['d']),
+                                    'r': r,
+                                })
                     else:
                         timescale = segment_template.get('timescale')
                         if timescale:
@@ -1551,7 +1566,7 @@ class InfoExtractor(object):
                         continue
                     representation_attrib = adaptation_set.attrib.copy()
                     representation_attrib.update(representation.attrib)
-                    # According to page 41 of ISO/IEC 29001-1:2014, @mimeType is mandatory
+                    # According to [1, 5.3.7.2, Table 9, page 41], @mimeType is mandatory
                     mime_type = representation_attrib['mimeType']
                     content_type = mime_type.split('/')[0]
                     if content_type == 'text':
@@ -1595,16 +1610,40 @@ class InfoExtractor(object):
                                 representation_ms_info['total_number'] = int(math.ceil(float(period_duration) / segment_duration))
                             media_template = representation_ms_info['media_template']
                             media_template = media_template.replace('$RepresentationID$', representation_id)
-                            media_template = re.sub(r'\$(Number|Bandwidth)\$', r'%(\1)d', media_template)
-                            media_template = re.sub(r'\$(Number|Bandwidth)%([^$]+)\$', r'%(\1)\2', media_template)
+                            media_template = re.sub(r'\$(Number|Bandwidth|Time)\$', r'%(\1)d', media_template)
+                            media_template = re.sub(r'\$(Number|Bandwidth|Time)%([^$]+)\$', r'%(\1)\2', media_template)
                             media_template.replace('$$', '$')
-                            representation_ms_info['segment_urls'] = [
-                                media_template % {
-                                    'Number': segment_number,
-                                    'Bandwidth': representation_attrib.get('bandwidth')}
-                                for segment_number in range(
-                                    representation_ms_info['start_number'],
-                                    representation_ms_info['total_number'] + representation_ms_info['start_number'])]
+
+                            # As per [1, 5.3.9.4.4, Table 16, page 55] $Number$ and $Time$
+                            # can't be used at the same time
+                            if '%(Number' in media_template:
+                                representation_ms_info['segment_urls'] = [
+                                    media_template % {
+                                        'Number': segment_number,
+                                        'Bandwidth': representation_attrib.get('bandwidth'),
+                                    }
+                                    for segment_number in range(
+                                        representation_ms_info['start_number'],
+                                        representation_ms_info['total_number'] + representation_ms_info['start_number'])]
+                            else:
+                                representation_ms_info['segment_urls'] = []
+                                segment_time = 0
+
+                                def add_segment_url():
+                                    representation_ms_info['segment_urls'].append(
+                                        media_template % {
+                                            'Time': segment_time,
+                                            'Bandwidth': representation_attrib.get('bandwidth'),
+                                        }
+                                    )
+
+                                for num, s in enumerate(representation_ms_info['s']):
+                                    segment_time = s.get('t') or segment_time
+                                    add_segment_url()
+                                    for r in range(s.get('r', 0)):
+                                        segment_time += s['d']
+                                        add_segment_url()
+                                    segment_time += s['d']
                         if 'segment_urls' in representation_ms_info:
                             f.update({
                                 'segment_urls': representation_ms_info['segment_urls'],

From 94c04a3c793a8332ea68bb2eff2979da4ef66af6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 24 Jul 2016 10:28:11 +0700
Subject: [PATCH 5570/6219] [arkena] Enable dash formats

---
 youtube_dl/extractor/arkena.py | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/arkena.py b/youtube_dl/extractor/arkena.py
index d7c5eeb8a..d45cae301 100644
--- a/youtube_dl/extractor/arkena.py
+++ b/youtube_dl/extractor/arkena.py
@@ -81,11 +81,8 @@ class ArkenaIE(InfoExtractor):
                     formats.extend(self._extract_f4m_formats(
                         f_url, video_id, f4m_id=kind, fatal=False))
                 elif kind == 'dash' or 'mpd' in exts:
-                    # TODO: Current DASH formats are broken - $Time$ pattern in
-                    # <SegmentTemplate> not implemented yet
-                    # formats.extend(self._extract_mpd_formats(
-                    #    f_url, video_id, mpd_id=kind, fatal=False))
-                    continue
+                    formats.extend(self._extract_mpd_formats(
+                        f_url, video_id, mpd_id=kind, fatal=False))
                 elif kind == 'silverlight':
                     # TODO: process when ism is supported (see
                     # https://github.com/rg3/youtube-dl/issues/8118)

From d9cb92c84058bce2c222b7a634608d7a16addcb2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 24 Jul 2016 10:29:09 +0700
Subject: [PATCH 5571/6219] [telegraaf] Enable dash formats

---
 youtube_dl/extractor/telegraaf.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/telegraaf.py b/youtube_dl/extractor/telegraaf.py
index 9092e9b85..926d36e5a 100644
--- a/youtube_dl/extractor/telegraaf.py
+++ b/youtube_dl/extractor/telegraaf.py
@@ -49,9 +49,8 @@ class TelegraafIE(InfoExtractor):
                 formats.extend(self._extract_m3u8_formats(
                     manifest_url, video_id, ext='mp4', m3u8_id='hls'))
             elif ext == 'mpd':
-                # TODO: Current DASH formats are broken - $Time$ pattern in
-                # <SegmentTemplate> not implemented yet
-                continue
+                formats.extend(self._extract_mpd_formats(
+                    manifest_url, video_id, mpd_id='dash', fatal=False))
             else:
                 self.report_warning('Unknown adaptive format %s' % ext)
         for location in locations.get('progressive', []):

From f75e6890dbc70f0686e967e89b5c422a3aee8951 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 24 Jul 2016 10:29:26 +0700
Subject: [PATCH 5572/6219] [telegraaf] Make hls non fatal

---
 youtube_dl/extractor/telegraaf.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/telegraaf.py b/youtube_dl/extractor/telegraaf.py
index 926d36e5a..58078c531 100644
--- a/youtube_dl/extractor/telegraaf.py
+++ b/youtube_dl/extractor/telegraaf.py
@@ -47,7 +47,7 @@ class TelegraafIE(InfoExtractor):
             ext = determine_ext(manifest_url)
             if ext == 'm3u8':
                 formats.extend(self._extract_m3u8_formats(
-                    manifest_url, video_id, ext='mp4', m3u8_id='hls'))
+                    manifest_url, video_id, ext='mp4', m3u8_id='hls', fatal=False))
             elif ext == 'mpd':
                 formats.extend(self._extract_mpd_formats(
                     manifest_url, video_id, mpd_id='dash', fatal=False))

From 5a65668e25ba424693a69cbaa81059e8c38a52ae Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 24 Jul 2016 10:35:55 +0700
Subject: [PATCH 5573/6219] [dcn] Enable dash formats

---
 youtube_dl/extractor/dcn.py | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/dcn.py b/youtube_dl/extractor/dcn.py
index efb8585e8..b8542820a 100644
--- a/youtube_dl/extractor/dcn.py
+++ b/youtube_dl/extractor/dcn.py
@@ -62,11 +62,9 @@ class DCNBaseIE(InfoExtractor):
                 r'file\s*:\s*"https?(://[^"]+)/playlist.m3u8',
                 r'<a[^>]+href="rtsp(://[^"]+)"'
             ], webpage, 'format url')
-        # TODO: Current DASH formats are broken - $Time$ pattern in
-        # <SegmentTemplate> not implemented yet
-        # formats.extend(self._extract_mpd_formats(
-        #     format_url_base + '/manifest.mpd',
-        #     video_id, mpd_id='dash', fatal=False))
+        formats.extend(self._extract_mpd_formats(
+            format_url_base + '/manifest.mpd',
+            video_id, mpd_id='dash', fatal=False))
         formats.extend(self._extract_m3u8_formats(
             format_url_base + '/playlist.m3u8', video_id, 'mp4',
             m3u8_entry_protocol, m3u8_id='hls', fatal=False))

From ae6fff4e64b34bdcbd5b4f90c89cfd9e55ddeffa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 24 Jul 2016 10:43:05 +0700
Subject: [PATCH 5574/6219] [onet] Enable dash formats

---
 youtube_dl/extractor/onet.py | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/onet.py b/youtube_dl/extractor/onet.py
index 402d3a9f7..fc22ad5eb 100644
--- a/youtube_dl/extractor/onet.py
+++ b/youtube_dl/extractor/onet.py
@@ -59,11 +59,8 @@ class OnetBaseIE(InfoExtractor):
                         # TODO: Support Microsoft Smooth Streaming
                         continue
                     elif ext == 'mpd':
-                        # TODO: Current DASH formats are broken - $Time$ pattern in
-                        # <SegmentTemplate> not implemented yet
-                        # formats.extend(self._extract_mpd_formats(
-                        #    video_url, video_id, mpd_id='dash', fatal=False))
-                        continue
+                        formats.extend(self._extract_mpd_formats(
+                            video_url, video_id, mpd_id='dash', fatal=False))
                     else:
                         formats.append({
                             'url': video_url,

From 9513c1eb1789818956faf6f6273f6563bae58aa1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 24 Jul 2016 11:03:39 +0700
Subject: [PATCH 5575/6219] [tvp] Update dash format comment

---
 youtube_dl/extractor/tvp.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/tvp.py b/youtube_dl/extractor/tvp.py
index 5070082da..e84876b54 100644
--- a/youtube_dl/extractor/tvp.py
+++ b/youtube_dl/extractor/tvp.py
@@ -89,8 +89,8 @@ class TVPIE(InfoExtractor):
             r'(https?://.+?/video)(?:\.(?:ism|f4m|m3u8)|-\d+\.mp4)',
             video_url, 'video base url', default=None)
         if video_url_base:
-            # TODO: Current DASH formats are broken - $Time$ pattern in
-            # <SegmentTemplate> not implemented yet
+            # TODO: <Group> found instead of <AdaptationSet> in MPD manifest.
+            # It's not mentioned in MPEG-DASH standard. Figure that out.
             # formats.extend(self._extract_mpd_formats(
             #     video_url_base + '.ism/video.mpd',
             #     video_id, mpd_id='dash', fatal=False))

From 8fdc538b4687ee920a81bd661a04c608d75e0011 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 24 Jul 2016 11:39:50 +0700
Subject: [PATCH 5576/6219] release 2016.07.24

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 docs/supportedsites.md    | 3 +++
 youtube_dl/version.py     | 2 +-
 3 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index f5b444b84..01a5ab5ec 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.07.22*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.07.22**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.07.24*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.07.24**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.07.22
+[debug] youtube-dl version 2016.07.24
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 36b8ec9ad..fa70fe49a 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -46,6 +46,7 @@
  - **archive.org**: archive.org videos
  - **ARD**
  - **ARD:mediathek**
+ - **Arkena**
  - **arte.tv**
  - **arte.tv:+7**
  - **arte.tv:cinema**
@@ -337,6 +338,8 @@
  - **kuwo:song**: 酷我音乐
  - **la7.it**
  - **Laola1Tv**
+ - **Lcp**
+ - **LcpPlay**
  - **Le**: 乐视网
  - **Learnr**
  - **Lecture2Go**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 732a5dfdf..428bd9a97 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.07.22'
+__version__ = '2016.07.24'

From e8be2943a7bf6fc171ddf04d251a854f8e4ecf7e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 24 Jul 2016 18:38:18 +0700
Subject: [PATCH 5577/6219] [smotri] Modernize, make more robust and fix tests

---
 youtube_dl/extractor/smotri.py | 99 +++++++++++++++++-----------------
 1 file changed, 50 insertions(+), 49 deletions(-)

diff --git a/youtube_dl/extractor/smotri.py b/youtube_dl/extractor/smotri.py
index 5c3fd0fec..114358786 100644
--- a/youtube_dl/extractor/smotri.py
+++ b/youtube_dl/extractor/smotri.py
@@ -13,20 +13,21 @@ from ..utils import (
     sanitized_Request,
     unified_strdate,
     urlencode_postdata,
+    xpath_text,
 )
 
 
 class SmotriIE(InfoExtractor):
     IE_DESC = 'Smotri.com'
     IE_NAME = 'smotri'
-    _VALID_URL = r'^https?://(?:www\.)?(?:smotri\.com/video/view/\?id=|pics\.smotri\.com/(?:player|scrubber_custom8)\.swf\?file=)(?P<id>v(?P<realvideoid>[0-9]+)[a-z0-9]{4})'
+    _VALID_URL = r'https?://(?:www\.)?(?:smotri\.com/video/view/\?id=|pics\.smotri\.com/(?:player|scrubber_custom8)\.swf\?file=)(?P<id>v(?P<realvideoid>[0-9]+)[a-z0-9]{4})'
     _NETRC_MACHINE = 'smotri'
 
     _TESTS = [
         # real video id 2610366
         {
             'url': 'http://smotri.com/video/view/?id=v261036632ab',
-            'md5': '2a7b08249e6f5636557579c368040eb9',
+            'md5': '02c0dfab2102984e9c5bb585cc7cc321',
             'info_dict': {
                 'id': 'v261036632ab',
                 'ext': 'mp4',
@@ -174,11 +175,11 @@ class SmotriIE(InfoExtractor):
         if video_password:
             video_form['pass'] = hashlib.md5(video_password.encode('utf-8')).hexdigest()
 
-        request = sanitized_Request(
-            'http://smotri.com/video/view/url/bot/', urlencode_postdata(video_form))
-        request.add_header('Content-Type', 'application/x-www-form-urlencoded')
-
-        video = self._download_json(request, video_id, 'Downloading video JSON')
+        video = self._download_json(
+            'http://smotri.com/video/view/url/bot/',
+            video_id, 'Downloading video JSON',
+            data=urlencode_postdata(video_form),
+            headers={'Content-Type': 'application/x-www-form-urlencoded'})
 
         video_url = video.get('_vidURL') or video.get('_vidURL_mp4')
 
@@ -196,11 +197,11 @@ class SmotriIE(InfoExtractor):
                 raise ExtractorError(msg, expected=True)
 
         title = video['title']
-        thumbnail = video['_imgURL']
-        upload_date = unified_strdate(video['added'])
-        uploader = video['userNick']
-        uploader_id = video['userLogin']
-        duration = int_or_none(video['duration'])
+        thumbnail = video.get('_imgURL')
+        upload_date = unified_strdate(video.get('added'))
+        uploader = video.get('userNick')
+        uploader_id = video.get('userLogin')
+        duration = int_or_none(video.get('duration'))
 
         # Video JSON does not provide enough meta data
         # We will extract some from the video web page instead
@@ -209,7 +210,7 @@ class SmotriIE(InfoExtractor):
 
         # Warning if video is unavailable
         warning = self._html_search_regex(
-            r'<div class="videoUnModer">(.*?)</div>', webpage,
+            r'<div[^>]+class="videoUnModer"[^>]*>(.+?)</div>', webpage,
             'warning message', default=None)
         if warning is not None:
             self._downloader.report_warning(
@@ -217,20 +218,22 @@ class SmotriIE(InfoExtractor):
                 (video_id, warning))
 
         # Adult content
-        if re.search('EroConfirmText">', webpage) is not None:
+        if 'EroConfirmText">' in webpage:
             self.report_age_confirmation()
             confirm_string = self._html_search_regex(
-                r'<a href="/video/view/\?id=%s&confirm=([^"]+)" title="[^"]+">' % video_id,
+                r'<a[^>]+href="/video/view/\?id=%s&confirm=([^"]+)"' % video_id,
                 webpage, 'confirm string')
             confirm_url = webpage_url + '&confirm=%s' % confirm_string
-            webpage = self._download_webpage(confirm_url, video_id, 'Downloading video page (age confirmed)')
+            webpage = self._download_webpage(
+                confirm_url, video_id,
+                'Downloading video page (age confirmed)')
             adult_content = True
         else:
             adult_content = False
 
         view_count = self._html_search_regex(
-            'Общее количество просмотров.*?<span class="Number">(\\d+)</span>',
-            webpage, 'view count', fatal=False, flags=re.MULTILINE | re.DOTALL)
+            r'(?s)Общее количество просмотров.*?<span class="Number">(\d+)</span>',
+            webpage, 'view count', fatal=False)
 
         return {
             'id': video_id,
@@ -249,37 +252,33 @@ class SmotriIE(InfoExtractor):
 class SmotriCommunityIE(InfoExtractor):
     IE_DESC = 'Smotri.com community videos'
     IE_NAME = 'smotri:community'
-    _VALID_URL = r'^https?://(?:www\.)?smotri\.com/community/video/(?P<communityid>[0-9A-Za-z_\'-]+)'
+    _VALID_URL = r'https?://(?:www\.)?smotri\.com/community/video/(?P<id>[0-9A-Za-z_\'-]+)'
     _TEST = {
         'url': 'http://smotri.com/community/video/kommuna',
         'info_dict': {
             'id': 'kommuna',
-            'title': 'КПРФ',
         },
         'playlist_mincount': 4,
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        community_id = mobj.group('communityid')
+        community_id = self._match_id(url)
 
-        url = 'http://smotri.com/export/rss/video/by/community/-/%s/video.xml' % community_id
-        rss = self._download_xml(url, community_id, 'Downloading community RSS')
+        rss = self._download_xml(
+            'http://smotri.com/export/rss/video/by/community/-/%s/video.xml' % community_id,
+            community_id, 'Downloading community RSS')
 
-        entries = [self.url_result(video_url.text, 'Smotri')
-                   for video_url in rss.findall('./channel/item/link')]
+        entries = [
+            self.url_result(video_url.text, SmotriIE.ie_key())
+            for video_url in rss.findall('./channel/item/link')]
 
-        description_text = rss.find('./channel/description').text
-        community_title = self._html_search_regex(
-            '^Видео сообщества "([^"]+)"$', description_text, 'community title')
-
-        return self.playlist_result(entries, community_id, community_title)
+        return self.playlist_result(entries, community_id)
 
 
 class SmotriUserIE(InfoExtractor):
     IE_DESC = 'Smotri.com user videos'
     IE_NAME = 'smotri:user'
-    _VALID_URL = r'^https?://(?:www\.)?smotri\.com/user/(?P<userid>[0-9A-Za-z_\'-]+)'
+    _VALID_URL = r'https?://(?:www\.)?smotri\.com/user/(?P<id>[0-9A-Za-z_\'-]+)'
     _TESTS = [{
         'url': 'http://smotri.com/user/inspector',
         'info_dict': {
@@ -290,19 +289,19 @@ class SmotriUserIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        user_id = mobj.group('userid')
+        user_id = self._match_id(url)
 
-        url = 'http://smotri.com/export/rss/user/video/-/%s/video.xml' % user_id
-        rss = self._download_xml(url, user_id, 'Downloading user RSS')
+        rss = self._download_xml(
+            'http://smotri.com/export/rss/user/video/-/%s/video.xml' % user_id,
+            user_id, 'Downloading user RSS')
 
         entries = [self.url_result(video_url.text, 'Smotri')
                    for video_url in rss.findall('./channel/item/link')]
 
-        description_text = rss.find('./channel/description').text
-        user_nickname = self._html_search_regex(
-            '^Видео режиссера (.*)$', description_text,
-            'user nickname')
+        description_text = xpath_text(rss, './channel/description') or ''
+        user_nickname = self._search_regex(
+            '^Видео режиссера (.+)$', description_text,
+            'user nickname', fatal=False)
 
         return self.playlist_result(entries, user_id, user_nickname)
 
@@ -310,11 +309,11 @@ class SmotriUserIE(InfoExtractor):
 class SmotriBroadcastIE(InfoExtractor):
     IE_DESC = 'Smotri.com broadcasts'
     IE_NAME = 'smotri:broadcast'
-    _VALID_URL = r'^https?://(?:www\.)?(?P<url>smotri\.com/live/(?P<broadcastid>[^/]+))/?.*'
+    _VALID_URL = r'https?://(?:www\.)?(?P<url>smotri\.com/live/(?P<id>[^/]+))/?.*'
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        broadcast_id = mobj.group('broadcastid')
+        broadcast_id = mobj.group('id')
 
         broadcast_url = 'http://' + mobj.group('url')
         broadcast_page = self._download_webpage(broadcast_url, broadcast_id, 'Downloading broadcast page')
@@ -328,7 +327,8 @@ class SmotriBroadcastIE(InfoExtractor):
 
             (username, password) = self._get_login_info()
             if username is None:
-                self.raise_login_required('Erotic broadcasts allowed only for registered users')
+                self.raise_login_required(
+                    'Erotic broadcasts allowed only for registered users')
 
             login_form = {
                 'login-hint53': '1',
@@ -343,8 +343,9 @@ class SmotriBroadcastIE(InfoExtractor):
             broadcast_page = self._download_webpage(
                 request, broadcast_id, 'Logging in and confirming age')
 
-            if re.search('>Неверный логин или пароль<', broadcast_page) is not None:
-                raise ExtractorError('Unable to log in: bad username or password', expected=True)
+            if '>Неверный логин или пароль<' in broadcast_page:
+                raise ExtractorError(
+                    'Unable to log in: bad username or password', expected=True)
 
             adult_content = True
         else:
@@ -383,11 +384,11 @@ class SmotriBroadcastIE(InfoExtractor):
 
             broadcast_playpath = broadcast_json['_streamName']
             broadcast_app = '%s/%s' % (mobj.group('app'), broadcast_json['_vidURL'])
-            broadcast_thumbnail = broadcast_json['_imgURL']
+            broadcast_thumbnail = broadcast_json.get('_imgURL')
             broadcast_title = self._live_title(broadcast_json['title'])
-            broadcast_description = broadcast_json['description']
-            broadcaster_nick = broadcast_json['nick']
-            broadcaster_login = broadcast_json['login']
+            broadcast_description = broadcast_json.get('description')
+            broadcaster_nick = broadcast_json.get('nick')
+            broadcaster_login = broadcast_json.get('login')
             rtmp_conn = 'S:%s' % uuid.uuid4().hex
         except KeyError:
             if protected_broadcast:

From 59eaf69e3300ad7f780d0f976052fcc3416284c1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 25 Jul 2016 22:31:10 +0700
Subject: [PATCH 5578/6219] [camdemy] Fix camdemy

---
 youtube_dl/extractor/camdemy.py | 54 ++++++++++++++++++---------------
 1 file changed, 29 insertions(+), 25 deletions(-)

diff --git a/youtube_dl/extractor/camdemy.py b/youtube_dl/extractor/camdemy.py
index 6ffbeabd3..8eece2c02 100644
--- a/youtube_dl/extractor/camdemy.py
+++ b/youtube_dl/extractor/camdemy.py
@@ -1,7 +1,6 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import datetime
 import re
 
 from .common import InfoExtractor
@@ -10,8 +9,9 @@ from ..compat import (
     compat_urlparse,
 )
 from ..utils import (
-    parse_iso8601,
+    clean_html,
     str_to_int,
+    unified_strdate,
 )
 
 
@@ -26,14 +26,13 @@ class CamdemyIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Ch1-1 Introduction, Signals (02-23-2012)',
             'thumbnail': 're:^https?://.*\.jpg$',
-            'description': '',
             'creator': 'ss11spring',
             'upload_date': '20130114',
-            'timestamp': 1358154556,
             'view_count': int,
         }
     }, {
         # With non-empty description
+        # webpage returns "No permission or not login"
         'url': 'http://www.camdemy.com/media/13885',
         'md5': '4576a3bb2581f86c61044822adbd1249',
         'info_dict': {
@@ -41,33 +40,34 @@ class CamdemyIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'EverCam + Camdemy QuickStart',
             'thumbnail': 're:^https?://.*\.jpg$',
-            'description': 'md5:050b62f71ed62928f8a35f1a41e186c9',
+            'description': 'md5:2a9f989c2b153a2342acee579c6e7db6',
             'creator': 'evercam',
-            'upload_date': '20140620',
-            'timestamp': 1403271569,
         }
     }, {
-        # External source
+        # External source (YouTube)
         'url': 'http://www.camdemy.com/media/14842',
-        'md5': '50e1c3c3aa233d3d7b7daa2fa10b1cf7',
         'info_dict': {
             'id': '2vsYQzNIsJo',
             'ext': 'mp4',
+            'title': 'Excel 2013 Tutorial - How to add Password Protection',
+            'description': 'Excel 2013 Tutorial for Beginners - How to add Password Protection',
             'upload_date': '20130211',
             'uploader': 'Hun Kim',
-            'description': 'Excel 2013 Tutorial for Beginners - How to add Password Protection',
             'uploader_id': 'hunkimtutorials',
-            'title': 'Excel 2013 Tutorial - How to add Password Protection',
-        }
+        },
+        'params': {
+            'skip_download': True,
+        },
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        page = self._download_webpage(url, video_id)
+
+        webpage = self._download_webpage(url, video_id)
 
         src_from = self._html_search_regex(
-            r"<div class='srcFrom'>Source: <a title='([^']+)'", page,
-            'external source', default=None)
+            r"class=['\"]srcFrom['\"][^>]*>Sources?(?:\s+from)?\s*:\s*<a[^>]+(?:href|title)=(['\"])(?P<url>(?:(?!\1).)+)\1",
+            webpage, 'external source', default=None, group='url')
         if src_from:
             return self.url_result(src_from)
 
@@ -78,27 +78,31 @@ class CamdemyIE(InfoExtractor):
         video_folder = compat_urlparse.urljoin(thumb_url, 'video/')
         file_list_doc = self._download_xml(
             compat_urlparse.urljoin(video_folder, 'fileList.xml'),
-            video_id, 'Filelist XML')
+            video_id, 'Downloading filelist XML')
         file_name = file_list_doc.find('./video/item/fileName').text
         video_url = compat_urlparse.urljoin(video_folder, file_name)
 
-        timestamp = parse_iso8601(self._html_search_regex(
-            r"<div class='title'>Posted\s*:</div>\s*<div class='value'>([^<>]+)<",
-            page, 'creation time', fatal=False),
-            delimiter=' ', timezone=datetime.timedelta(hours=8))
-        view_count = str_to_int(self._html_search_regex(
-            r"<div class='title'>Views\s*:</div>\s*<div class='value'>([^<>]+)<",
-            page, 'view count', fatal=False))
+        # Some URLs return "No permission or not login" in a webpage despite being
+        # freely available via oembed JSON URL (e.g. http://www.camdemy.com/media/13885)
+        upload_date = unified_strdate(self._search_regex(
+            r'>published on ([^<]+)<', webpage,
+            'upload date', default=None))
+        view_count = str_to_int(self._search_regex(
+            r'role=["\']viewCnt["\'][^>]*>([\d,.]+) views',
+            webpage, 'view count', default=None))
+        description = self._html_search_meta(
+            'description', webpage, default=None) or clean_html(
+            oembed_obj.get('description'))
 
         return {
             'id': video_id,
             'url': video_url,
             'title': oembed_obj['title'],
             'thumbnail': thumb_url,
-            'description': self._html_search_meta('description', page),
+            'description': description,
             'creator': oembed_obj['author_name'],
             'duration': oembed_obj['duration'],
-            'timestamp': timestamp,
+            'upload_date': upload_date,
             'view_count': view_count,
         }
 

From 0a147785e8d1bb763c2f0634bef1c580194e1d1e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 25 Jul 2016 22:35:12 +0700
Subject: [PATCH 5579/6219] [camdemy] Extract duration properly

---
 youtube_dl/extractor/camdemy.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/camdemy.py b/youtube_dl/extractor/camdemy.py
index 8eece2c02..268c34392 100644
--- a/youtube_dl/extractor/camdemy.py
+++ b/youtube_dl/extractor/camdemy.py
@@ -10,6 +10,7 @@ from ..compat import (
 )
 from ..utils import (
     clean_html,
+    parse_duration,
     str_to_int,
     unified_strdate,
 )
@@ -27,6 +28,7 @@ class CamdemyIE(InfoExtractor):
             'title': 'Ch1-1 Introduction, Signals (02-23-2012)',
             'thumbnail': 're:^https?://.*\.jpg$',
             'creator': 'ss11spring',
+            'duration': 1591,
             'upload_date': '20130114',
             'view_count': int,
         }
@@ -42,6 +44,7 @@ class CamdemyIE(InfoExtractor):
             'thumbnail': 're:^https?://.*\.jpg$',
             'description': 'md5:2a9f989c2b153a2342acee579c6e7db6',
             'creator': 'evercam',
+            'duration': 318,
         }
     }, {
         # External source (YouTube)
@@ -74,6 +77,7 @@ class CamdemyIE(InfoExtractor):
         oembed_obj = self._download_json(
             'http://www.camdemy.com/oembed/?format=json&url=' + url, video_id)
 
+        title = oembed_obj['title']
         thumb_url = oembed_obj['thumbnail_url']
         video_folder = compat_urlparse.urljoin(thumb_url, 'video/')
         file_list_doc = self._download_xml(
@@ -97,11 +101,11 @@ class CamdemyIE(InfoExtractor):
         return {
             'id': video_id,
             'url': video_url,
-            'title': oembed_obj['title'],
+            'title': title,
             'thumbnail': thumb_url,
             'description': description,
-            'creator': oembed_obj['author_name'],
-            'duration': oembed_obj['duration'],
+            'creator': oembed_obj.get('author_name'),
+            'duration': parse_duration(oembed_obj.get('duration')),
             'upload_date': upload_date,
             'view_count': view_count,
         }

From 712c7530ff4acf3b7ae82cc19b2babb1588a3eab Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 26 Jul 2016 11:03:43 +0800
Subject: [PATCH 5580/6219] [mtv] Extract more metadata and more

1. Remove MTVIggyIE. All www.mtviggy.com URLs now redirects to
   www.mtv.com
2. Fix MTVDEIE
3. Return multiple URLs from _transform_rtmp_url. This is for
   tosh.cc.com
---
 youtube_dl/extractor/mtv.py | 58 +++++++++++++++++++------------------
 1 file changed, 30 insertions(+), 28 deletions(-)

diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index f3ec2ebbc..ba9c28a56 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -16,6 +16,7 @@ from ..utils import (
     HEADRequest,
     sanitized_Request,
     strip_or_none,
+    timeconvert,
     unescapeHTML,
     url_basename,
     RegexNotFoundError,
@@ -36,13 +37,13 @@ class MTVServicesInfoExtractor(InfoExtractor):
         return uri.split(':')[-1]
 
     # This was originally implemented for ComedyCentral, but it also works here
-    @staticmethod
-    def _transform_rtmp_url(rtmp_video_url):
+    @classmethod
+    def _transform_rtmp_url(cls, rtmp_video_url):
         m = re.match(r'^rtmpe?://.*?/(?P<finalid>gsp\..+?/.*)$', rtmp_video_url)
         if not m:
-            return rtmp_video_url
+            return {'rtmp': rtmp_video_url}
         base = 'http://viacommtvstrmfs.fplive.net/'
-        return base + m.group('finalid')
+        return {'http': base + m.group('finalid')}
 
     def _get_feed_url(self, uri):
         return self._FEED_URL
@@ -86,14 +87,14 @@ class MTVServicesInfoExtractor(InfoExtractor):
                 rtmp_video_url = rendition.find('./src').text
                 if rtmp_video_url.endswith('siteunavail.png'):
                     continue
-                new_url = self._transform_rtmp_url(rtmp_video_url)
-                formats.append({
+                new_urls = self._transform_rtmp_url(rtmp_video_url)
+                formats.extend([{
                     'ext': 'flv' if new_url.startswith('rtmp') else ext,
                     'url': new_url,
-                    'format_id': rendition.get('bitrate'),
+                    'format_id': '-'.join([kind, rendition.get('bitrate')]),
                     'width': int(rendition.get('width')),
                     'height': int(rendition.get('height')),
-                })
+                } for kind, new_url in new_urls.items()])
             except (KeyError, TypeError):
                 raise ExtractorError('Invalid rendition field.')
         self._sort_formats(formats)
@@ -136,6 +137,8 @@ class MTVServicesInfoExtractor(InfoExtractor):
 
         description = strip_or_none(xpath_text(itemdoc, 'description'))
 
+        timestamp = timeconvert(xpath_text(itemdoc, 'pubDate'))
+
         title_el = None
         if title_el is None:
             title_el = find_xpath_attr(
@@ -168,6 +171,7 @@ class MTVServicesInfoExtractor(InfoExtractor):
             'thumbnail': self._get_thumbnail_url(uri, itemdoc),
             'description': description,
             'duration': float_or_none(content_el.attrib.get('duration')),
+            'timestamp': timestamp,
         }
 
     def _get_feed_query(self, uri):
@@ -186,8 +190,13 @@ class MTVServicesInfoExtractor(InfoExtractor):
         idoc = self._download_xml(
             url, video_id,
             'Downloading info', transform_source=fix_xml_ampersands)
+
+        title = xpath_text(idoc, './channel/title')
+        description = xpath_text(idoc, './channel/description')
+
         return self.playlist_result(
-            [self._get_video_info(item) for item in idoc.findall('.//item')])
+            [self._get_video_info(item) for item in idoc.findall('.//item')],
+            playlist_title=title, playlist_description=description)
 
     def _extract_mgid(self, webpage):
         try:
@@ -233,6 +242,8 @@ class MTVServicesEmbeddedIE(MTVServicesInfoExtractor):
             'ext': 'mp4',
             'title': 'Peter Dinklage Sums Up \'Game Of Thrones\' In 45 Seconds',
             'description': '"Sexy sexy sexy, stabby stabby stabby, beautiful language," says Peter Dinklage as he tries summarizing "Game of Thrones" in under a minute.',
+            'timestamp': 1400126400,
+            'upload_date': '20140515',
         },
     }
 
@@ -275,6 +286,8 @@ class MTVIE(MTVServicesInfoExtractor):
                 'ext': 'mp4',
                 'title': 'Taylor Swift - "Ours (VH1 Storytellers)"',
                 'description': 'Album: Taylor Swift performs "Ours" for VH1 Storytellers at Harvey Mudd College.',
+                'timestamp': 1352610000,
+                'upload_date': '20121111',
             },
         },
     ]
@@ -301,20 +314,6 @@ class MTVIE(MTVServicesInfoExtractor):
         return self._get_videos_info(uri)
 
 
-class MTVIggyIE(MTVServicesInfoExtractor):
-    IE_NAME = 'mtviggy.com'
-    _VALID_URL = r'https?://www\.mtviggy\.com/videos/.+'
-    _TEST = {
-        'url': 'http://www.mtviggy.com/videos/arcade-fire-behind-the-scenes-at-the-biggest-music-experiment-yet/',
-        'info_dict': {
-            'id': '984696',
-            'ext': 'mp4',
-            'title': 'Arcade Fire: Behind the Scenes at the Biggest Music Experiment Yet',
-        }
-    }
-    _FEED_URL = 'http://all.mtvworldverticals.com/feed-xml/'
-
-
 class MTVDEIE(MTVServicesInfoExtractor):
     IE_NAME = 'mtv.de'
     _VALID_URL = r'https?://(?:www\.)?mtv\.de/(?:artists|shows|news)/(?:[^/]+/)*(?P<id>\d+)-[^/#?]+/*(?:[#?].*)?$'
@@ -322,7 +321,7 @@ class MTVDEIE(MTVServicesInfoExtractor):
         'url': 'http://www.mtv.de/artists/10571-cro/videos/61131-traum',
         'info_dict': {
             'id': 'music_video-a50bc5f0b3aa4b3190aa',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': 'MusicVideo_cro-traum',
             'description': 'Cro - Traum',
         },
@@ -335,7 +334,7 @@ class MTVDEIE(MTVServicesInfoExtractor):
         'url': 'http://www.mtv.de/shows/933-teen-mom-2/staffeln/5353/folgen/63565-enthullungen',
         'info_dict': {
             'id': 'local_playlist-f5ae778b9832cc837189',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': 'Episode_teen-mom-2_shows_season-5_episode-1_full-episode_part1',
         },
         'params': {
@@ -343,7 +342,6 @@ class MTVDEIE(MTVServicesInfoExtractor):
             'skip_download': True,
         },
     }, {
-        # single video in pagePlaylist with different id
         'url': 'http://www.mtv.de/news/77491-mtv-movies-spotlight-pixels-teil-3',
         'info_dict': {
             'id': 'local_playlist-4e760566473c4c8c5344',
@@ -355,6 +353,7 @@ class MTVDEIE(MTVServicesInfoExtractor):
             # rtmp download
             'skip_download': True,
         },
+        'skip': 'Das Video kann zur Zeit nicht abgespielt werden.',
     }]
 
     def _real_extract(self, url):
@@ -367,11 +366,14 @@ class MTVDEIE(MTVServicesInfoExtractor):
                 r'window\.pagePlaylist\s*=\s*(\[.+?\]);\n', webpage, 'page playlist'),
             video_id)
 
+        def _mrss_url(item):
+            return item['mrss'] + item.get('mrssvars', '')
+
         # news pages contain single video in playlist with different id
         if len(playlist) == 1:
-            return self._get_videos_info_from_url(playlist[0]['mrss'], video_id)
+            return self._get_videos_info_from_url(_mrss_url(playlist[0]), video_id)
 
         for item in playlist:
             item_id = item.get('id')
             if item_id and compat_str(item_id) == video_id:
-                return self._get_videos_info_from_url(item['mrss'], video_id)
+                return self._get_videos_info_from_url(_mrss_url(item), video_id)

From cc99d4f826a942b18133fe4221c9de2f9197e860 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 26 Jul 2016 11:06:50 +0800
Subject: [PATCH 5581/6219] [comedycentral] Remove IEs for *.cc.com except
 tosh.cc.com

All other subdomains now redirects to cc.com/* URLs
---
 youtube_dl/extractor/comedycentral.py | 280 ++++----------------------
 1 file changed, 35 insertions(+), 245 deletions(-)

diff --git a/youtube_dl/extractor/comedycentral.py b/youtube_dl/extractor/comedycentral.py
index 63f68f765..c76909e48 100644
--- a/youtube_dl/extractor/comedycentral.py
+++ b/youtube_dl/extractor/comedycentral.py
@@ -1,17 +1,6 @@
 from __future__ import unicode_literals
 
-import re
-
 from .mtv import MTVServicesInfoExtractor
-from ..compat import (
-    compat_str,
-    compat_urllib_parse_urlencode,
-)
-from ..utils import (
-    ExtractorError,
-    float_or_none,
-    unified_strdate,
-)
 
 
 class ComedyCentralIE(MTVServicesInfoExtractor):
@@ -26,8 +15,10 @@ class ComedyCentralIE(MTVServicesInfoExtractor):
         'info_dict': {
             'id': 'cef0cbb3-e776-4bc9-b62e-8016deccb354',
             'ext': 'mp4',
-            'title': 'CC:Stand-Up|Greg Fitzsimmons: Life on Stage|Uncensored - Too Good of a Mother',
+            'title': 'CC:Stand-Up|August 18, 2013|1|0101|Uncensored - Too Good of a Mother',
             'description': 'After a certain point, breastfeeding becomes c**kblocking.',
+            'timestamp': 1376798400,
+            'upload_date': '20130818',
         },
     }, {
         'url': 'http://www.cc.com/shows/the-daily-show-with-trevor-noah/interviews/6yx39d/exclusive-rand-paul-extended-interview',
@@ -35,244 +26,43 @@ class ComedyCentralIE(MTVServicesInfoExtractor):
     }]
 
 
-class ComedyCentralShowsIE(MTVServicesInfoExtractor):
-    IE_DESC = 'The Daily Show / The Colbert Report'
-    # urls can be abbreviations like :thedailyshow
-    # urls for episodes like:
-    # or urls for clips like: http://www.thedailyshow.com/watch/mon-december-10-2012/any-given-gun-day
-    #                     or: http://www.colbertnation.com/the-colbert-report-videos/421667/november-29-2012/moon-shattering-news
-    #                     or: http://www.colbertnation.com/the-colbert-report-collections/422008/festival-of-lights/79524
-    _VALID_URL = r'''(?x)^(:(?P<shortname>tds|thedailyshow)
-                      |https?://(:www\.)?
-                          (?P<showname>thedailyshow|thecolbertreport|tosh)\.(?:cc\.)?com/
-                         ((?:full-)?episodes/(?:[0-9a-z]{6}/)?(?P<episode>.*)|
-                          (?P<clip>
-                              (?:(?:guests/[^/]+|videos|video-(?:clips|playlists)|special-editions|news-team/[^/]+)/[^/]+/(?P<videotitle>[^/?#]+))
-                              |(the-colbert-report-(videos|collections)/(?P<clipID>[0-9]+)/[^/]*/(?P<cntitle>.*?))
-                              |(watch/(?P<date>[^/]*)/(?P<tdstitle>.*))
-                          )|
-                          (?P<interview>
-                              extended-interviews/(?P<interID>[0-9a-z]+)/
-                              (?:playlist_tds_extended_)?(?P<interview_title>[^/?#]*?)
-                              (?:/[^/?#]?|[?#]|$))))
-                     '''
+class ToshIE(MTVServicesInfoExtractor):
+    IE_DESC = 'Tosh.0'
+    _VALID_URL = r'^https?://tosh\.cc\.com/video-(?:clips|collections)/[^/]+/(?P<videotitle>[^/?#]+)'
+    _FEED_URL = 'http://tosh.cc.com/feeds/mrss'
+
     _TESTS = [{
-        'url': 'http://thedailyshow.cc.com/watch/thu-december-13-2012/kristen-stewart',
-        'md5': '4e2f5cb088a83cd8cdb7756132f9739d',
-        'info_dict': {
-            'id': 'ab9ab3e7-5a98-4dbe-8b21-551dc0523d55',
-            'ext': 'mp4',
-            'upload_date': '20121213',
-            'description': 'Kristen Stewart learns to let loose in "On the Road."',
-            'uploader': 'thedailyshow',
-            'title': 'thedailyshow kristen-stewart part 1',
-        }
-    }, {
-        'url': 'http://thedailyshow.cc.com/extended-interviews/b6364d/sarah-chayes-extended-interview',
-        'info_dict': {
-            'id': 'sarah-chayes-extended-interview',
-            'description': 'Carnegie Endowment Senior Associate Sarah Chayes discusses how corrupt institutions function throughout the world in her book "Thieves of State: Why Corruption Threatens Global Security."',
-            'title': 'thedailyshow Sarah Chayes Extended Interview',
-        },
-        'playlist': [
-            {
-                'info_dict': {
-                    'id': '0baad492-cbec-4ec1-9e50-ad91c291127f',
-                    'ext': 'mp4',
-                    'upload_date': '20150129',
-                    'description': 'Carnegie Endowment Senior Associate Sarah Chayes discusses how corrupt institutions function throughout the world in her book "Thieves of State: Why Corruption Threatens Global Security."',
-                    'uploader': 'thedailyshow',
-                    'title': 'thedailyshow sarah-chayes-extended-interview part 1',
-                },
-            },
-            {
-                'info_dict': {
-                    'id': '1e4fb91b-8ce7-4277-bd7c-98c9f1bbd283',
-                    'ext': 'mp4',
-                    'upload_date': '20150129',
-                    'description': 'Carnegie Endowment Senior Associate Sarah Chayes discusses how corrupt institutions function throughout the world in her book "Thieves of State: Why Corruption Threatens Global Security."',
-                    'uploader': 'thedailyshow',
-                    'title': 'thedailyshow sarah-chayes-extended-interview part 2',
-                },
-            },
-        ],
-        'params': {
-            'skip_download': True,
-        },
-    }, {
-        'url': 'http://thedailyshow.cc.com/extended-interviews/xm3fnq/andrew-napolitano-extended-interview',
-        'only_matching': True,
-    }, {
-        'url': 'http://thecolbertreport.cc.com/videos/29w6fx/-realhumanpraise-for-fox-news',
-        'only_matching': True,
-    }, {
-        'url': 'http://thecolbertreport.cc.com/videos/gh6urb/neil-degrasse-tyson-pt--1?xrs=eml_col_031114',
-        'only_matching': True,
-    }, {
-        'url': 'http://thedailyshow.cc.com/guests/michael-lewis/3efna8/exclusive---michael-lewis-extended-interview-pt--3',
-        'only_matching': True,
-    }, {
-        'url': 'http://thedailyshow.cc.com/episodes/sy7yv0/april-8--2014---denis-leary',
-        'only_matching': True,
-    }, {
-        'url': 'http://thecolbertreport.cc.com/episodes/8ase07/april-8--2014---jane-goodall',
-        'only_matching': True,
-    }, {
-        'url': 'http://thedailyshow.cc.com/video-playlists/npde3s/the-daily-show-19088-highlights',
-        'only_matching': True,
-    }, {
-        'url': 'http://thedailyshow.cc.com/video-playlists/t6d9sg/the-daily-show-20038-highlights/be3cwo',
-        'only_matching': True,
-    }, {
-        'url': 'http://thedailyshow.cc.com/special-editions/2l8fdb/special-edition---a-look-back-at-food',
-        'only_matching': True,
-    }, {
-        'url': 'http://thedailyshow.cc.com/news-team/michael-che/7wnfel/we-need-to-talk-about-israel',
-        'only_matching': True,
-    }, {
         'url': 'http://tosh.cc.com/video-clips/68g93d/twitter-users-share-summer-plans',
+        'info_dict': {
+            'description': 'Tosh asked fans to share their summer plans.',
+            'title': 'Twitter Users Share Summer Plans',
+        },
+        'playlist': [{
+            'md5': 'f269e88114c1805bb6d7653fecea9e06',
+            'info_dict': {
+                'id': '90498ec2-ed00-11e0-aca6-0026b9414f30',
+                'ext': 'mp4',
+                'title': 'Tosh.0|June 9, 2077|2|211|Twitter Users Share Summer Plans',
+                'description': 'Tosh asked fans to share their summer plans.',
+                'thumbnail': 're:^https?://.*\.jpg',
+                # It's really reported to be published on year 2077
+                'upload_date': '20770610',
+                'timestamp': 3390510600,
+                'subtitles': {
+                    'en': 'mincount:3',
+                },
+            },
+        }]
+    }, {
+        'url': 'http://tosh.cc.com/video-collections/x2iz7k/just-plain-foul/m5q4fp',
         'only_matching': True,
     }]
 
-    _available_formats = ['3500', '2200', '1700', '1200', '750', '400']
-
-    _video_extensions = {
-        '3500': 'mp4',
-        '2200': 'mp4',
-        '1700': 'mp4',
-        '1200': 'mp4',
-        '750': 'mp4',
-        '400': 'mp4',
-    }
-    _video_dimensions = {
-        '3500': (1280, 720),
-        '2200': (960, 540),
-        '1700': (768, 432),
-        '1200': (640, 360),
-        '750': (512, 288),
-        '400': (384, 216),
-    }
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-
-        if mobj.group('shortname'):
-            return self.url_result('http://www.cc.com/shows/the-daily-show-with-trevor-noah/full-episodes')
-
-        if mobj.group('clip'):
-            if mobj.group('videotitle'):
-                epTitle = mobj.group('videotitle')
-            elif mobj.group('showname') == 'thedailyshow':
-                epTitle = mobj.group('tdstitle')
-            else:
-                epTitle = mobj.group('cntitle')
-            dlNewest = False
-        elif mobj.group('interview'):
-            epTitle = mobj.group('interview_title')
-            dlNewest = False
-        else:
-            dlNewest = not mobj.group('episode')
-            if dlNewest:
-                epTitle = mobj.group('showname')
-            else:
-                epTitle = mobj.group('episode')
-        show_name = mobj.group('showname')
-
-        webpage, htmlHandle = self._download_webpage_handle(url, epTitle)
-        if dlNewest:
-            url = htmlHandle.geturl()
-            mobj = re.match(self._VALID_URL, url, re.VERBOSE)
-            if mobj is None:
-                raise ExtractorError('Invalid redirected URL: ' + url)
-            if mobj.group('episode') == '':
-                raise ExtractorError('Redirected URL is still not specific: ' + url)
-            epTitle = (mobj.group('episode') or mobj.group('videotitle')).rpartition('/')[-1]
-
-        mMovieParams = re.findall('(?:<param name="movie" value="|var url = ")(http://media.mtvnservices.com/([^"]*(?:episode|video).*?:.*?))"', webpage)
-        if len(mMovieParams) == 0:
-            # The Colbert Report embeds the information in a without
-            # a URL prefix; so extract the alternate reference
-            # and then add the URL prefix manually.
-
-            altMovieParams = re.findall('data-mgid="([^"]*(?:episode|video|playlist).*?:.*?)"', webpage)
-            if len(altMovieParams) == 0:
-                raise ExtractorError('unable to find Flash URL in webpage ' + url)
-            else:
-                mMovieParams = [('http://media.mtvnservices.com/' + altMovieParams[0], altMovieParams[0])]
-
-        uri = mMovieParams[0][1]
-        # Correct cc.com in uri
-        uri = re.sub(r'(episode:[^.]+)(\.cc)?\.com', r'\1.com', uri)
-
-        index_url = 'http://%s.cc.com/feeds/mrss?%s' % (show_name, compat_urllib_parse_urlencode({'uri': uri}))
-        idoc = self._download_xml(
-            index_url, epTitle,
-            'Downloading show index', 'Unable to download episode index')
-
-        title = idoc.find('./channel/title').text
-        description = idoc.find('./channel/description').text
-
-        entries = []
-        item_els = idoc.findall('.//item')
-        for part_num, itemEl in enumerate(item_els):
-            upload_date = unified_strdate(itemEl.findall('./pubDate')[0].text)
-            thumbnail = itemEl.find('.//{http://search.yahoo.com/mrss/}thumbnail').attrib.get('url')
-
-            content = itemEl.find('.//{http://search.yahoo.com/mrss/}content')
-            duration = float_or_none(content.attrib.get('duration'))
-            mediagen_url = content.attrib['url']
-            guid = itemEl.find('./guid').text.rpartition(':')[-1]
-
-            cdoc = self._download_xml(
-                mediagen_url, epTitle,
-                'Downloading configuration for segment %d / %d' % (part_num + 1, len(item_els)))
-
-            turls = []
-            for rendition in cdoc.findall('.//rendition'):
-                finfo = (rendition.attrib['bitrate'], rendition.findall('./src')[0].text)
-                turls.append(finfo)
-
-            formats = []
-            for format, rtmp_video_url in turls:
-                w, h = self._video_dimensions.get(format, (None, None))
-                formats.append({
-                    'format_id': 'vhttp-%s' % format,
-                    'url': self._transform_rtmp_url(rtmp_video_url),
-                    'ext': self._video_extensions.get(format, 'mp4'),
-                    'height': h,
-                    'width': w,
-                })
-                formats.append({
-                    'format_id': 'rtmp-%s' % format,
-                    'url': rtmp_video_url.replace('viacomccstrm', 'viacommtvstrm'),
-                    'ext': self._video_extensions.get(format, 'mp4'),
-                    'height': h,
-                    'width': w,
-                })
-                self._sort_formats(formats)
-
-            subtitles = self._extract_subtitles(cdoc, guid)
-
-            virtual_id = show_name + ' ' + epTitle + ' part ' + compat_str(part_num + 1)
-            entries.append({
-                'id': guid,
-                'title': virtual_id,
-                'formats': formats,
-                'uploader': show_name,
-                'upload_date': upload_date,
-                'duration': duration,
-                'thumbnail': thumbnail,
-                'description': description,
-                'subtitles': subtitles,
-            })
-
-        return {
-            '_type': 'playlist',
-            'id': epTitle,
-            'entries': entries,
-            'title': show_name + ' ' + title,
-            'description': description,
-        }
+    @classmethod
+    def _transform_rtmp_url(cls, rtmp_video_url):
+        new_urls = super(ToshIE, cls)._transform_rtmp_url(rtmp_video_url)
+        new_urls['rtmp'] = rtmp_video_url.replace('viacomccstrm', 'viacommtvstrm')
+        return new_urls
 
 
 class ComedyCentralTVIE(MTVServicesInfoExtractor):

From 14a28e705b3e27606c2fdbc242ae5a33c2e9f28e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 26 Jul 2016 11:08:09 +0800
Subject: [PATCH 5582/6219] [test/test_all_urls] Remove *.cc.com tests

---
 test/test_all_urls.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/test/test_all_urls.py b/test/test_all_urls.py
index 1f6079c29..cd1cd4b24 100644
--- a/test/test_all_urls.py
+++ b/test/test_all_urls.py
@@ -101,8 +101,6 @@ class TestAllURLsMatching(unittest.TestCase):
         self.assertMatch(':ytsubs', ['youtube:subscriptions'])
         self.assertMatch(':ytsubscriptions', ['youtube:subscriptions'])
         self.assertMatch(':ythistory', ['youtube:history'])
-        self.assertMatch(':thedailyshow', ['ComedyCentralShows'])
-        self.assertMatch(':tds', ['ComedyCentralShows'])
 
     def test_vimeo_matching(self):
         self.assertMatch('https://vimeo.com/channels/tributes', ['vimeo:channel'])

From 5c32a77cad549f9d2f3f02ed4204e4591dd1889b Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 26 Jul 2016 11:08:55 +0800
Subject: [PATCH 5583/6219] [nextmovie] Remove extractor

This domain name now redirects to mtv.com
---
 youtube_dl/extractor/nextmovie.py | 30 ------------------------------
 1 file changed, 30 deletions(-)
 delete mode 100644 youtube_dl/extractor/nextmovie.py

diff --git a/youtube_dl/extractor/nextmovie.py b/youtube_dl/extractor/nextmovie.py
deleted file mode 100644
index 9ccd7d774..000000000
--- a/youtube_dl/extractor/nextmovie.py
+++ /dev/null
@@ -1,30 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-from .mtv import MTVServicesInfoExtractor
-from ..compat import compat_urllib_parse_urlencode
-
-
-class NextMovieIE(MTVServicesInfoExtractor):
-    IE_NAME = 'nextmovie.com'
-    _VALID_URL = r'https?://(?:www\.)?nextmovie\.com/shows/[^/]+/\d{4}-\d{2}-\d{2}/(?P<id>[^/?#]+)'
-    _FEED_URL = 'http://lite.dextr.mtvi.com/service1/dispatch.htm'
-    _TESTS = [{
-        'url': 'http://www.nextmovie.com/shows/exclusives/2013-03-10/mgid:uma:videolist:nextmovie.com:1715019/',
-        'md5': '09a9199f2f11f10107d04fcb153218aa',
-        'info_dict': {
-            'id': '961726',
-            'ext': 'mp4',
-            'title': 'The Muppets\' Gravity',
-        },
-    }]
-
-    def _get_feed_query(self, uri):
-        return compat_urllib_parse_urlencode({
-            'feed': '1505',
-            'mgid': uri,
-        })
-
-    def _real_extract(self, url):
-        mgid = self._match_id(url)
-        return self._get_videos_info(mgid)

From a3aa814b774a413d9e7f4fbfadf06fe6dcc59b25 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 26 Jul 2016 11:10:41 +0800
Subject: [PATCH 5584/6219] Update _TESTS for MTV sites

---
 youtube_dl/extractor/nick.py      |  1 +
 youtube_dl/extractor/southpark.py | 34 ++++++++++++++++---
 youtube_dl/extractor/spike.py     |  4 ++-
 youtube_dl/extractor/tvland.py    | 55 +++++++------------------------
 4 files changed, 44 insertions(+), 50 deletions(-)

diff --git a/youtube_dl/extractor/nick.py b/youtube_dl/extractor/nick.py
index 4935002d0..9c54846e1 100644
--- a/youtube_dl/extractor/nick.py
+++ b/youtube_dl/extractor/nick.py
@@ -7,6 +7,7 @@ from ..utils import update_url_query
 
 
 class NickIE(MTVServicesInfoExtractor):
+    # None of videos on the website are still alive?
     IE_NAME = 'nick.com'
     _VALID_URL = r'https?://(?:www\.)?nick(?:jr)?\.com/(?:videos/clip|[^/]+/videos)/(?P<id>[^/?#.]+)'
     _FEED_URL = 'http://udat.mtvnservices.com/service1/dispatch.htm'
diff --git a/youtube_dl/extractor/southpark.py b/youtube_dl/extractor/southpark.py
index 87b650468..a147f7db1 100644
--- a/youtube_dl/extractor/southpark.py
+++ b/youtube_dl/extractor/southpark.py
@@ -17,6 +17,8 @@ class SouthParkIE(MTVServicesInfoExtractor):
             'ext': 'mp4',
             'title': 'South Park|Bat Daded',
             'description': 'Randy disqualifies South Park by getting into a fight with Bat Dad.',
+            'timestamp': 1112760000,
+            'upload_date': '20050406',
         },
     }]
 
@@ -28,6 +30,10 @@ class SouthParkEsIE(SouthParkIE):
 
     _TESTS = [{
         'url': 'http://southpark.cc.com/episodios-en-espanol/s01e01-cartman-consigue-una-sonda-anal#source=351c1323-0b96-402d-a8b9-40d01b2e9bde&position=1&sort=!airdate',
+        'info_dict': {
+            'title': 'Cartman Consigue Una Sonda Anal',
+            'description': 'Cartman Consigue Una Sonda Anal',
+        },
         'playlist_count': 4,
     }]
 
@@ -42,17 +48,27 @@ class SouthParkDeIE(SouthParkIE):
         'info_dict': {
             'id': '85487c96-b3b9-4e39-9127-ad88583d9bf2',
             'ext': 'mp4',
-            'title': 'The Government Won\'t Respect My Privacy',
+            'title': 'South Park|The Government Won\'t Respect My Privacy',
             'description': 'Cartman explains the benefits of "Shitter" to Stan, Kyle and Craig.',
+            'timestamp': 1380160800,
+            'upload_date': '20130926',
         },
     }, {
         # non-ASCII characters in initial URL
         'url': 'http://www.southpark.de/alle-episoden/s18e09-hashtag-aufwärmen',
-        'playlist_count': 4,
+        'info_dict': {
+            'title': 'Hashtag „Aufwärmen“',
+            'description': 'Kyle will mit seinem kleinen Bruder Ike Videospiele spielen. Als der nicht mehr mit ihm spielen will, hat Kyle Angst, dass er die Kids von heute nicht mehr versteht.',
+        },
+        'playlist_count': 3,
     }, {
         # non-ASCII characters in redirect URL
         'url': 'http://www.southpark.de/alle-episoden/s18e09',
-        'playlist_count': 4,
+        'info_dict': {
+            'title': 'Hashtag „Aufwärmen“',
+            'description': 'Kyle will mit seinem kleinen Bruder Ike Videospiele spielen. Als der nicht mehr mit ihm spielen will, hat Kyle Angst, dass er die Kids von heute nicht mehr versteht.',
+        },
+        'playlist_count': 3,
     }]
 
 
@@ -63,7 +79,11 @@ class SouthParkNlIE(SouthParkIE):
 
     _TESTS = [{
         'url': 'http://www.southpark.nl/full-episodes/s18e06-freemium-isnt-free',
-        'playlist_count': 4,
+        'info_dict': {
+            'title': 'Freemium Isn\'t Free',
+            'description': 'Stan is addicted to the new Terrance and Phillip mobile game.',
+        },
+        'playlist_mincount': 3,
     }]
 
 
@@ -74,5 +94,9 @@ class SouthParkDkIE(SouthParkIE):
 
     _TESTS = [{
         'url': 'http://www.southparkstudios.dk/full-episodes/s18e07-grounded-vindaloop',
-        'playlist_count': 4,
+        'info_dict': {
+            'title': 'Grounded Vindaloop',
+            'description': 'Butters is convinced he\'s living in a virtual reality.',
+        },
+        'playlist_mincount': 3,
     }]
diff --git a/youtube_dl/extractor/spike.py b/youtube_dl/extractor/spike.py
index 63ea7718b..218785ee4 100644
--- a/youtube_dl/extractor/spike.py
+++ b/youtube_dl/extractor/spike.py
@@ -11,8 +11,10 @@ class SpikeIE(MTVServicesInfoExtractor):
         'info_dict': {
             'id': 'b9c8221a-4e50-479a-b86d-3333323e38ba',
             'ext': 'mp4',
-            'title': 'Auction Hunters|Can Allen Ride A Hundred Year-Old Motorcycle?',
+            'title': 'Auction Hunters|December 27, 2013|4|414|Can Allen Ride A Hundred Year-Old Motorcycle?',
             'description': 'md5:fbed7e82ed5fad493615b3094a9499cb',
+            'timestamp': 1388120400,
+            'upload_date': '20131227',
         },
     }, {
         'url': 'http://www.spike.com/video-clips/lhtu8m/',
diff --git a/youtube_dl/extractor/tvland.py b/youtube_dl/extractor/tvland.py
index b73279dec..cb76a2a58 100644
--- a/youtube_dl/extractor/tvland.py
+++ b/youtube_dl/extractor/tvland.py
@@ -9,56 +9,23 @@ class TVLandIE(MTVServicesInfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?tvland\.com/(?:video-clips|episodes)/(?P<id>[^/?#.]+)'
     _FEED_URL = 'http://www.tvland.com/feeds/mrss/'
     _TESTS = [{
+        # Geo-restricted. Without a proxy metadata are still there. With a
+        # proxy it redirects to http://m.tvland.com/app/
         'url': 'http://www.tvland.com/episodes/hqhps2/everybody-loves-raymond-the-invasion-ep-048',
-        'playlist': [
-            {
-                'md5': '227e9723b9669c05bf51098b10287aa7',
-                'info_dict': {
-                    'id': 'bcbd3a83-3aca-4dca-809b-f78a87dcccdd',
-                    'ext': 'mp4',
-                    'title': 'Everybody Loves Raymond|Everybody Loves Raymond 048 HD, Part 1 of 5',
-                }
-            },
-            {
-                'md5': '9fa2b764ec0e8194fb3ebb01a83df88b',
-                'info_dict': {
-                    'id': 'f4279548-6e13-40dd-92e8-860d27289197',
-                    'ext': 'mp4',
-                    'title': 'Everybody Loves Raymond|Everybody Loves Raymond 048 HD, Part 2 of 5',
-                }
-            },
-            {
-                'md5': 'fde4c3bccd7cc7e3576b338734153cec',
-                'info_dict': {
-                    'id': '664e4a38-53ef-4115-9bc9-d0f789ec6334',
-                    'ext': 'mp4',
-                    'title': 'Everybody Loves Raymond|Everybody Loves Raymond 048 HD, Part 3 of 5',
-                }
-            },
-            {
-                'md5': '247f6780cda6891f2e49b8ae2b10e017',
-                'info_dict': {
-                    'id': '9146ecf5-b15a-4d78-879c-6679b77f4960',
-                    'ext': 'mp4',
-                    'title': 'Everybody Loves Raymond|Everybody Loves Raymond 048 HD, Part 4 of 5',
-                }
-            },
-            {
-                'md5': 'fd269f33256e47bad5eb6c40de089ff6',
-                'info_dict': {
-                    'id': '04334a2e-9a47-4214-a8c2-ae5792e2fab7',
-                    'ext': 'mp4',
-                    'title': 'Everybody Loves Raymond|Everybody Loves Raymond 048 HD, Part 5 of 5',
-                }
-            }
-        ],
+        'info_dict': {
+            'description': 'md5:80973e81b916a324e05c14a3fb506d29',
+            'title': 'The Invasion',
+        },
+        'playlist': [],
     }, {
         'url': 'http://www.tvland.com/video-clips/zea2ev/younger-younger--hilary-duff---little-lies',
         'md5': 'e2c6389401cf485df26c79c247b08713',
         'info_dict': {
             'id': 'b8697515-4bbe-4e01-83d5-fa705ce5fa88',
             'ext': 'mp4',
-            'title': 'Younger|Younger: Hilary Duff - Little Lies',
-            'description': 'md5:7d192f56ca8d958645c83f0de8ef0269'
+            'title': 'Younger|December 28, 2015|2|NO-EPISODE#|Younger: Hilary Duff - Little Lies',
+            'description': 'md5:7d192f56ca8d958645c83f0de8ef0269',
+            'upload_date': '20151228',
+            'timestamp': 1451289600,
         },
     }]

From 05d1e7aaa9cee8d9b7b4d96b480d5e2f32f15f47 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 26 Jul 2016 11:11:36 +0800
Subject: [PATCH 5585/6219] [generic] Fix an MTV test and another test that
 breaks nosetests

---
 youtube_dl/extractor/generic.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 011940580..1eac679e5 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -474,7 +474,7 @@ class GenericIE(InfoExtractor):
             'url': 'http://www.vestifinance.ru/articles/25753',
             'info_dict': {
                 'id': '25753',
-                'title': 'Вести Экономика ― Прямые трансляции с Форума-выставки "Госзаказ-2013"',
+                'title': 'Прямые трансляции с Форума-выставки "Госзаказ-2013"',
             },
             'playlist': [{
                 'info_dict': {
@@ -641,6 +641,8 @@ class GenericIE(InfoExtractor):
                 'ext': 'mp4',
                 'title': 'Key and Peele|October 10, 2012|2|203|Liam Neesons - Uncensored',
                 'description': 'Two valets share their love for movie star Liam Neesons.',
+                'timestamp': 1349922600,
+                'upload_date': '20121011',
             },
         },
         # YouTube embed via <data-embed-url="">

From 4d3e543c73949765deb4b144b1042b1baedd7692 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 26 Jul 2016 11:17:28 +0800
Subject: [PATCH 5586/6219] Update extractors.py

---
 youtube_dl/extractor/extractors.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 3ca0ef83a..53fab1a31 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -159,8 +159,8 @@ from .coub import CoubIE
 from .collegerama import CollegeRamaIE
 from .comedycentral import (
     ComedyCentralIE,
-    ComedyCentralShowsIE,
     ComedyCentralTVIE,
+    ToshIE,
 )
 from .comcarcoff import ComCarCoffIE
 from .commonmistakes import CommonMistakesIE, UnicodeBOMIE
@@ -480,7 +480,6 @@ from .msn import MSNIE
 from .mtv import (
     MTVIE,
     MTVServicesEmbeddedIE,
-    MTVIggyIE,
     MTVDEIE,
 )
 from .muenchentv import MuenchenTVIE
@@ -530,7 +529,6 @@ from .nextmedia import (
     NextMediaActionNewsIE,
     AppleDailyIE,
 )
-from .nextmovie import NextMovieIE
 from .nfb import NFBIE
 from .nfl import NFLIE
 from .nhl import (

From 10a1bb3a78453a48f5006179ccb7c6998579dde7 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 26 Jul 2016 13:12:24 +0800
Subject: [PATCH 5587/6219] [mtv] Fix for videos with missing bitrates

---
 youtube_dl/extractor/mtv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index ba9c28a56..9e71c08c2 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -91,7 +91,7 @@ class MTVServicesInfoExtractor(InfoExtractor):
                 formats.extend([{
                     'ext': 'flv' if new_url.startswith('rtmp') else ext,
                     'url': new_url,
-                    'format_id': '-'.join([kind, rendition.get('bitrate')]),
+                    'format_id': '-'.join(filter(None, [kind, rendition.get('bitrate')])),
                     'width': int(rendition.get('width')),
                     'height': int(rendition.get('height')),
                 } for kind, new_url in new_urls.items()])

From c74299a72c7d003f95d700b80a69469d567fce05 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 26 Jul 2016 13:13:14 +0800
Subject: [PATCH 5588/6219] [cmt] Detect unavailable videos and update _TESTS

---
 youtube_dl/extractor/cmt.py | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/youtube_dl/extractor/cmt.py b/youtube_dl/extractor/cmt.py
index f1311b14f..f24568dcc 100644
--- a/youtube_dl/extractor/cmt.py
+++ b/youtube_dl/extractor/cmt.py
@@ -1,5 +1,7 @@
 from __future__ import unicode_literals
+
 from .mtv import MTVIE
+from ..utils import ExtractorError
 
 
 class CMTIE(MTVIE):
@@ -16,7 +18,27 @@ class CMTIE(MTVIE):
             'title': 'Garth Brooks - "The Call (featuring Trisha Yearwood)"',
             'description': 'Blame It All On My Roots',
         },
+        'skip': 'Video not available',
+    }, {
+        'url': 'http://www.cmt.com/videos/misc/1504699/still-the-king-ep-109-in-3-minutes.jhtml#id=1739908',
+        'md5': 'e61a801ca4a183a466c08bd98dccbb1c',
+        'info_dict': {
+            'id': '1504699',
+            'ext': 'mp4',
+            'title': 'Still The King Ep. 109 in 3 Minutes',
+            'description': 'Relive or catch up with Still The King by watching this recap of season 1, episode 9. New episodes Sundays 9/8c.',
+            'timestamp': 1469421000.0,
+            'upload_date': '20160725',
+        },
     }, {
         'url': 'http://www.cmt.com/shows/party-down-south/party-down-south-ep-407-gone-girl/1738172/playlist/#id=1738172',
         'only_matching': True,
     }]
+
+    @classmethod
+    def _transform_rtmp_url(cls, rtmp_video_url):
+        if 'error_not_available.swf' in rtmp_video_url:
+            raise ExtractorError(
+                '%s said: video is not available' % cls.IE_NAME, expected=True)
+
+        return super(CMTIE, cls)._transform_rtmp_url(rtmp_video_url)

From 326fa4e6e52cb0d02c6026d8a2d32e34b8e99107 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 26 Jul 2016 13:16:04 +0800
Subject: [PATCH 5589/6219] [generic] Skip an invalid test

---
 youtube_dl/extractor/generic.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 1eac679e5..98dd6a7e3 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -859,6 +859,7 @@ class GenericIE(InfoExtractor):
                 'description': 'md5:601cb790edd05908957dae8aaa866465',
                 'upload_date': '20150220',
             },
+            'skip': 'All The Daily Show URLs now redirect to http://www.cc.com/shows/',
         },
         # jwplayer YouTube
         {

From 35f6e0ff365cb2958164caad66d4f15e400f31c0 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 26 Jul 2016 13:19:47 +0800
Subject: [PATCH 5590/6219] [mtv.de] Skip 2 geo-restricted tests

---
 youtube_dl/extractor/mtv.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index 9e71c08c2..2f455680e 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -329,6 +329,7 @@ class MTVDEIE(MTVServicesInfoExtractor):
             # rtmp download
             'skip_download': True,
         },
+        'skip': 'Blocked at Travis CI',
     }, {
         # mediagen URL without query (e.g. http://videos.mtvnn.com/mediagen/e865da714c166d18d6f80893195fcb97)
         'url': 'http://www.mtv.de/shows/933-teen-mom-2/staffeln/5353/folgen/63565-enthullungen',
@@ -341,6 +342,7 @@ class MTVDEIE(MTVServicesInfoExtractor):
             # rtmp download
             'skip_download': True,
         },
+        'skip': 'Blocked at Travis CI',
     }, {
         'url': 'http://www.mtv.de/news/77491-mtv-movies-spotlight-pixels-teil-3',
         'info_dict': {

From 7f8b92e3cfffc5983423ad3b078c2132b9ff39cd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 26 Jul 2016 21:44:53 +0700
Subject: [PATCH 5591/6219] [bigflix] Update tests

---
 youtube_dl/extractor/bigflix.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/bigflix.py b/youtube_dl/extractor/bigflix.py
index 33762ad93..b19f35b5d 100644
--- a/youtube_dl/extractor/bigflix.py
+++ b/youtube_dl/extractor/bigflix.py
@@ -12,7 +12,7 @@ class BigflixIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?bigflix\.com/.+/(?P<id>[0-9]+)'
     _TESTS = [{
         'url': 'http://www.bigflix.com/Hindi-movies/Action-movies/Singham-Returns/16537',
-        'md5': 'ec76aa9b1129e2e5b301a474e54fab74',
+        'md5': 'dc1b4aebb46e3a7077ecc0d9f43f61e3',
         'info_dict': {
             'id': '16537',
             'ext': 'mp4',
@@ -26,7 +26,7 @@ class BigflixIE(InfoExtractor):
             'id': '16070',
             'ext': 'mp4',
             'title': 'Madarasapatinam',
-            'description': 'md5:63b9b8ed79189c6f0418c26d9a3452ca',
+            'description': 'md5:9f0470b26a4ba8e824c823b5d95c2f6b',
             'formats': 'mincount:2',
         },
         'params': {

From 88bd486b9a287db7de2a859863ed3356b418cd66 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 26 Jul 2016 22:58:50 +0700
Subject: [PATCH 5592/6219] [cbc] Improve extraction for videos embedded with
 clipId

---
 youtube_dl/extractor/cbc.py | 29 +++++++++++++++++++++++++----
 1 file changed, 25 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/cbc.py b/youtube_dl/extractor/cbc.py
index 06772d492..885d2c721 100644
--- a/youtube_dl/extractor/cbc.py
+++ b/youtube_dl/extractor/cbc.py
@@ -4,9 +4,11 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import compat_str
 from ..utils import (
     js_to_json,
     smuggle_url,
+    try_get,
 )
 
 
@@ -27,7 +29,20 @@ class CBCIE(InfoExtractor):
         },
         'skip': 'Geo-restricted to Canada',
     }, {
-        # with clipId
+        # with clipId, feed available via tpfeed.cbc.ca and feed.theplatform.com
+        'url': 'http://www.cbc.ca/22minutes/videos/22-minutes-update/22-minutes-update-episode-4',
+        'md5': '162adfa070274b144f4fdc3c3b8207db',
+        'info_dict': {
+            'id': '2414435309',
+            'ext': 'mp4',
+            'title': '22 Minutes Update: What Not To Wear Quebec',
+            'description': "This week's latest Canadian top political story is What Not To Wear Quebec.",
+            'upload_date': '20131025',
+            'uploader': 'CBCC-NEW',
+            'timestamp': 1382717907,
+        },
+    }, {
+        # with clipId, feed only available via tpfeed.cbc.ca
         'url': 'http://www.cbc.ca/archives/entry/1978-robin-williams-freestyles-on-90-minutes-live',
         'md5': '0274a90b51a9b4971fe005c63f592f12',
         'info_dict': {
@@ -83,9 +98,15 @@ class CBCIE(InfoExtractor):
             media_id = player_info.get('mediaId')
             if not media_id:
                 clip_id = player_info['clipId']
-                media_id = self._download_json(
-                    'http://feed.theplatform.com/f/h9dtGB/punlNGjMlc1F?fields=id&byContent=byReleases%3DbyId%253D' + clip_id,
-                    clip_id)['entries'][0]['id'].split('/')[-1]
+                feed = self._download_json(
+                    'http://tpfeed.cbc.ca/f/ExhSPC/vms_5akSXx4Ng_Zn?byCustomValue={:mpsReleases}{%s}'% clip_id,
+                    clip_id, fatal=False)
+                if feed:
+                    media_id = try_get(feed, lambda x: x['entries'][0]['guid'], compat_str)
+                if not media_id:
+                    media_id = self._download_json(
+                        'http://feed.theplatform.com/f/h9dtGB/punlNGjMlc1F?fields=id&byContent=byReleases%3DbyId%253D' + clip_id,
+                        clip_id)['entries'][0]['id'].split('/')[-1]
             return self.url_result('cbcplayer:%s' % media_id, 'CBCPlayer', media_id)
         else:
             entries = [self.url_result('cbcplayer:%s' % media_id, 'CBCPlayer', media_id) for media_id in re.findall(r'<iframe[^>]+src="[^"]+?mediaId=(\d+)"', webpage)]

From dc35ba0eba000d9557b40387838f7dfff103286f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 26 Jul 2016 23:06:21 +0700
Subject: [PATCH 5593/6219] [mgtv] Fix typo

---
 youtube_dl/extractor/mgtv.py | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/mgtv.py b/youtube_dl/extractor/mgtv.py
index d970e94ec..27bdff8b2 100644
--- a/youtube_dl/extractor/mgtv.py
+++ b/youtube_dl/extractor/mgtv.py
@@ -9,7 +9,7 @@ class MGTVIE(InfoExtractor):
     _VALID_URL = r'https?://www\.mgtv\.com/v/(?:[^/]+/)*(?P<id>\d+)\.html'
     IE_DESC = '芒果TV'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.mgtv.com/v/1/290525/f/3116640.html',
         'md5': '1bdadcf760a0b90946ca68ee9a2db41a',
         'info_dict': {
@@ -20,7 +20,11 @@ class MGTVIE(InfoExtractor):
             'duration': 7461,
             'thumbnail': 're:^https?://.*\.jpg$',
         },
-    }
+    }, {
+        # no tbr extracted from stream_url
+        'url': 'http://www.mgtv.com/v/1/1/f/3324755.html',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -41,7 +45,8 @@ class MGTVIE(InfoExtractor):
             def extract_format(stream_url, format_id, idx, query={}):
                 format_info = self._download_json(
                     stream_url, video_id,
-                    note='Download video info for format %s' % format_id or '#%d' % idx, query=query)
+                    note='Download video info for format %s' % (format_id or '#%d' % idx),
+                    query=query)
                 return {
                     'format_id': format_id,
                     'url': format_info['info'],

From 9a700deea47b2514ef07d4ab7a0c21c7942b8b26 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 26 Jul 2016 23:07:16 +0700
Subject: [PATCH 5594/6219] [instagram] Remove duplicate field in test

---
 youtube_dl/extractor/instagram.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/instagram.py b/youtube_dl/extractor/instagram.py
index fc0197ae1..8f7f232be 100644
--- a/youtube_dl/extractor/instagram.py
+++ b/youtube_dl/extractor/instagram.py
@@ -36,7 +36,6 @@ class InstagramIE(InfoExtractor):
         'info_dict': {
             'id': 'BA-pQFBG8HZ',
             'ext': 'mp4',
-            'uploader_id': 'britneyspears',
             'title': 'Video by britneyspears',
             'thumbnail': 're:^https?://.*\.jpg',
             'timestamp': 1453760977,

From ced70c86400d09b0a03ecc0500e2874efc50b354 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 26 Jul 2016 23:08:08 +0700
Subject: [PATCH 5595/6219] [cbc] PEP 8

---
 youtube_dl/extractor/cbc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/cbc.py b/youtube_dl/extractor/cbc.py
index 885d2c721..a87e97140 100644
--- a/youtube_dl/extractor/cbc.py
+++ b/youtube_dl/extractor/cbc.py
@@ -99,7 +99,7 @@ class CBCIE(InfoExtractor):
             if not media_id:
                 clip_id = player_info['clipId']
                 feed = self._download_json(
-                    'http://tpfeed.cbc.ca/f/ExhSPC/vms_5akSXx4Ng_Zn?byCustomValue={:mpsReleases}{%s}'% clip_id,
+                    'http://tpfeed.cbc.ca/f/ExhSPC/vms_5akSXx4Ng_Zn?byCustomValue={:mpsReleases}{%s}' % clip_id,
                     clip_id, fatal=False)
                 if feed:
                     media_id = try_get(feed, lambda x: x['entries'][0]['guid'], compat_str)

From 3e050d51d4752e89269c5ae82cf4f0f49ad424b8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 26 Jul 2016 23:14:04 +0700
Subject: [PATCH 5596/6219] [orf:oe1] Relax _VALID_URL

---
 youtube_dl/extractor/orf.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/orf.py b/youtube_dl/extractor/orf.py
index ccb23e069..6ae30679a 100644
--- a/youtube_dl/extractor/orf.py
+++ b/youtube_dl/extractor/orf.py
@@ -137,13 +137,16 @@ class ORFTVthekIE(InfoExtractor):
 class ORFOE1IE(InfoExtractor):
     IE_NAME = 'orf:oe1'
     IE_DESC = 'Radio Österreich 1'
-    _VALID_URL = r'https?://oe1\.orf\.at/(?:programm/|konsole.*?#\?track_id=)(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://oe1\.orf\.at/(?:programm/|konsole\?.*?\btrack_id=)(?P<id>[0-9]+)'
 
     # Audios on ORF radio are only available for 7 days, so we can't add tests.
-    _TEST = {
+    _TESTS = [{
         'url': 'http://oe1.orf.at/konsole?show=on_demand#?track_id=394211',
         'only_matching': True,
-    }
+    }, {
+        'url': 'http://oe1.orf.at/konsole?show=ondemand&track_id=443608&load_day=/programm/konsole/tag/20160726',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         show_id = self._match_id(url)

From 40090e8d519f88c85ce65d8e882093d8ab26e368 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 26 Jul 2016 23:54:06 +0700
Subject: [PATCH 5597/6219] [extractor/common] Improve is_suitable

In order to fix breakage introduced by a3aa814b774a413d9e7f4fbfadf06fe6dcc59b25
---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index b8a76e3cb..53c28f016 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1786,7 +1786,7 @@ class InfoExtractor(object):
 
         any_restricted = False
         for tc in self.get_testcases(include_onlymatching=False):
-            if 'playlist' in tc:
+            if tc.get('playlist', []):
                 tc = tc['playlist'][0]
             is_restricted = age_restricted(
                 tc.get('info_dict', {}).get('age_limit'), age_limit)

From dcbb07c35a113ace7b6b90c40f661f9c31174a64 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 26 Jul 2016 23:56:53 +0700
Subject: [PATCH 5598/6219] release 2016.07.26.2

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 docs/supportedsites.md    | 4 +---
 youtube_dl/version.py     | 2 +-
 3 files changed, 5 insertions(+), 7 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 01a5ab5ec..f102d2611 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.07.24*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.07.24**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.07.26.2*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.07.26.2**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.07.24
+[debug] youtube-dl version 2016.07.26.2
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index fa70fe49a..1f89b1c14 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -142,7 +142,6 @@
  - **CollegeRama**
  - **ComCarCoff**
  - **ComedyCentral**
- - **ComedyCentralShows**: The Daily Show / The Colbert Report
  - **ComedyCentralTV**
  - **CondeNast**: Condé Nast media group: Allure, Architectural Digest, Ars Technica, Bon Appétit, Brides, Condé Nast, Condé Nast Traveler, Details, Epicurious, GQ, Glamour, Golf Digest, SELF, Teen Vogue, The New Yorker, Vanity Fair, Vogue, W Magazine, WIRED
  - **Coub**
@@ -401,7 +400,6 @@
  - **MSN**
  - **MTV**
  - **mtv.de**
- - **mtviggy.com**
  - **mtvservices:embedded**
  - **MuenchenTV**: münchen.tv
  - **MusicPlayOn**
@@ -441,7 +439,6 @@
  - **Newstube**
  - **NextMedia**: 蘋果日報
  - **NextMediaActionNews**: 蘋果日報 - 動新聞
- - **nextmovie.com**
  - **nfb**: National Film Board of Canada
  - **nfl.com**
  - **nhl.com**
@@ -699,6 +696,7 @@
  - **TNAFlix**
  - **TNAFlixNetworkEmbed**
  - **toggle**
+ - **Tosh**: Tosh.0
  - **tou.tv**
  - **Toypics**: Toypics user profile
  - **ToypicsUser**: Toypics user profile
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 428bd9a97..9a1642407 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.07.24'
+__version__ = '2016.07.26.2'

From 7935926baadaad1bcf5c45b1da0f3d2d8a173ce9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 27 Jul 2016 00:14:40 +0700
Subject: [PATCH 5599/6219] [devscripts/show-downloads-statistics] Add support
 for paging

---
 devscripts/show-downloads-statistics.py | 36 ++++++++++++++-----------
 1 file changed, 21 insertions(+), 15 deletions(-)

diff --git a/devscripts/show-downloads-statistics.py b/devscripts/show-downloads-statistics.py
index b591d3fc9..e25d28411 100644
--- a/devscripts/show-downloads-statistics.py
+++ b/devscripts/show-downloads-statistics.py
@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 from __future__ import unicode_literals
 
+import itertools
 import json
 import os
 import re
@@ -21,21 +22,26 @@ def format_size(bytes):
 
 total_bytes = 0
 
-releases = json.loads(compat_urllib_request.urlopen(
-    'https://api.github.com/repos/rg3/youtube-dl/releases').read().decode('utf-8'))
+for page in itertools.count(1):
+    releases = json.loads(compat_urllib_request.urlopen(
+        'https://api.github.com/repos/rg3/youtube-dl/releases?page=%s' % page
+    ).read().decode('utf-8'))
 
-for release in releases:
-    compat_print(release['name'])
-    for asset in release['assets']:
-        asset_name = asset['name']
-        total_bytes += asset['download_count'] * asset['size']
-        if all(not re.match(p, asset_name) for p in (
-                r'^youtube-dl$',
-                r'^youtube-dl-\d{4}\.\d{2}\.\d{2}(?:\.\d+)?\.tar\.gz$',
-                r'^youtube-dl\.exe$')):
-            continue
-        compat_print(
-            ' %s size: %s downloads: %d'
-            % (asset_name, format_size(asset['size']), asset['download_count']))
+    if not releases:
+        break
+
+    for release in releases:
+        compat_print(release['name'])
+        for asset in release['assets']:
+            asset_name = asset['name']
+            total_bytes += asset['download_count'] * asset['size']
+            if all(not re.match(p, asset_name) for p in (
+                    r'^youtube-dl$',
+                    r'^youtube-dl-\d{4}\.\d{2}\.\d{2}(?:\.\d+)?\.tar\.gz$',
+                    r'^youtube-dl\.exe$')):
+                continue
+            compat_print(
+                ' %s size: %s downloads: %d'
+                % (asset_name, format_size(asset['size']), asset['download_count']))
 
 compat_print('total downloads traffic: %s' % format_size(total_bytes))

From 289a16b4f3c67faa230d1b05b48f0c224b3472d8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 27 Jul 2016 23:28:01 +0700
Subject: [PATCH 5600/6219] [shared] Respect redirect URL (Closes #10170)

---
 youtube_dl/extractor/shared.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/shared.py b/youtube_dl/extractor/shared.py
index e7e5f653e..6757e6ccf 100644
--- a/youtube_dl/extractor/shared.py
+++ b/youtube_dl/extractor/shared.py
@@ -37,15 +37,17 @@ class SharedIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
+
+        webpage, urlh = self._download_webpage_handle(url, video_id)
 
         if '>File does not exist<' in webpage:
             raise ExtractorError(
                 'Video %s does not exist' % video_id, expected=True)
 
         download_form = self._hidden_inputs(webpage)
+
         request = sanitized_Request(
-            url, urlencode_postdata(download_form))
+            urlh.geturl(), urlencode_postdata(download_form))
         request.add_header('Content-Type', 'application/x-www-form-urlencoded')
 
         video_page = self._download_webpage(

From 05c8268c8192dcc4f61c869aba659c8e51d040bb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 27 Jul 2016 23:39:02 +0700
Subject: [PATCH 5601/6219] [shared] Modernize and make more robust

---
 youtube_dl/extractor/shared.py | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/shared.py b/youtube_dl/extractor/shared.py
index 6757e6ccf..d592dfeb8 100644
--- a/youtube_dl/extractor/shared.py
+++ b/youtube_dl/extractor/shared.py
@@ -6,7 +6,6 @@ from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
     int_or_none,
-    sanitized_Request,
     urlencode_postdata,
 )
 
@@ -46,21 +45,24 @@ class SharedIE(InfoExtractor):
 
         download_form = self._hidden_inputs(webpage)
 
-        request = sanitized_Request(
-            urlh.geturl(), urlencode_postdata(download_form))
-        request.add_header('Content-Type', 'application/x-www-form-urlencoded')
-
         video_page = self._download_webpage(
-            request, video_id, 'Downloading video page')
+            urlh.geturl(), video_id, 'Downloading video page',
+            data=urlencode_postdata(download_form),
+            headers={
+                'Content-Type': 'application/x-www-form-urlencoded',
+                'Referer': urlh.geturl(),
+            })
 
         video_url = self._html_search_regex(
-            r'data-url="([^"]+)"', video_page, 'video URL')
+            r'data-url=(["\'])(?P<url>(?:(?!\1).)+)\1',
+            video_page, 'video URL', group='url')
         title = base64.b64decode(self._html_search_meta(
             'full:title', webpage, 'title').encode('utf-8')).decode('utf-8')
         filesize = int_or_none(self._html_search_meta(
             'full:size', webpage, 'file size', fatal=False))
         thumbnail = self._html_search_regex(
-            r'data-poster="([^"]+)"', video_page, 'thumbnail', default=None)
+            r'data-poster=(["\'])(?P<url>(?:(?!\1).)+)\1',
+            video_page, 'thumbnail', default=None, group='url')
 
         return {
             'id': video_id,

From 39eef54cf01a9017da0ea83618978293f19841bb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 28 Jul 2016 21:38:23 +0700
Subject: [PATCH 5602/6219] [ard:mediathek] Skip unavailable test

---
 youtube_dl/extractor/ard.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/ard.py b/youtube_dl/extractor/ard.py
index c15cf1575..07e67dd33 100644
--- a/youtube_dl/extractor/ard.py
+++ b/youtube_dl/extractor/ard.py
@@ -73,6 +73,7 @@ class ARDMediathekIE(InfoExtractor):
             'description': 'md5:c0c1c8048514deaed2a73b3a60eecacb',
             'duration': 3287,
         },
+        'skip': 'Video is no longer available',
     }]
 
     def _extract_media_info(self, media_info_url, webpage, video_id):

From 94aae015485801ff1edc3689a11f8866b077ce47 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 28 Jul 2016 22:15:15 +0700
Subject: [PATCH 5603/6219] [extractor/generic] Extract all soundcloud embeds
 (Closes #10179)

---
 youtube_dl/extractor/generic.py | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 98dd6a7e3..6c4af6424 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -71,6 +71,7 @@ from .vessel import VesselIE
 from .kaltura import KalturaIE
 from .eagleplatform import EaglePlatformIE
 from .facebook import FacebookIE
+from .soundcloud import SoundcloudIE
 
 
 class GenericIE(InfoExtractor):
@@ -1999,12 +2000,9 @@ class GenericIE(InfoExtractor):
             return self.url_result(myvi_url)
 
         # Look for embedded soundcloud player
-        mobj = re.search(
-            r'<iframe\s+(?:[a-zA-Z0-9_-]+="[^"]+"\s+)*src="(?P<url>https?://(?:w\.)?soundcloud\.com/player[^"]+)"',
-            webpage)
-        if mobj is not None:
-            url = unescapeHTML(mobj.group('url'))
-            return self.url_result(url)
+        soundcloud_urls = SoundcloudIE._extract_urls(webpage)
+        if soundcloud_urls:
+            return _playlist_from_matches(soundcloud_urls, getter=unescapeHTML, ie=SoundcloudIE.ie_key())
 
         # Look for embedded mtvservices player
         mtvservices_url = MTVServicesEmbeddedIE._extract_url(webpage)

From fbdf8d15d110c75c9029d86943a9222226b9b3f0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 28 Jul 2016 22:16:05 +0700
Subject: [PATCH 5604/6219] [soundcloud] Add _extract_urls (#10179)

---
 youtube_dl/extractor/soundcloud.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 194dabc71..aeae931a2 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -119,6 +119,12 @@ class SoundcloudIE(InfoExtractor):
     _CLIENT_ID = '02gUJC0hH2ct1EGOcYXQIzRFU91c72Ea'
     _IPHONE_CLIENT_ID = '376f225bf427445fc4bfb6b99b72e0bf'
 
+    @staticmethod
+    def _extract_urls(webpage):
+        return [m.group('url') for m in re.finditer(
+            r'<iframe[^>]+src=(["\'])(?P<url>(?:https?://)?(?:w\.)?soundcloud\.com/player.+?)\1',
+            webpage)]
+
     def report_resolve(self, video_id):
         """Report information extraction."""
         self.to_screen('%s: Resolving id' % video_id)

From db19df6ca06f0470fa1c08b0a5255625f658ac3d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 28 Jul 2016 22:20:08 +0700
Subject: [PATCH 5605/6219] [extractor/generic] Add test for #10179

---
 youtube_dl/extractor/generic.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 6c4af6424..5364f0b19 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -785,6 +785,15 @@ class GenericIE(InfoExtractor):
                 'upload_date': '20141029',
             }
         },
+        # Soundcloud multiple embeds
+        {
+            'url': 'http://www.guitarplayer.com/lessons/1014/legato-workout-one-hour-to-more-fluid-performance---tab/52809',
+            'info_dict': {
+                'id': '52809',
+                'title': 'Guitar Essentials: Legato Workout—One-Hour to Fluid Performance  | TAB + AUDIO',
+            },
+            'playlist_mincount': 7,
+        },
         # Livestream embed
         {
             'url': 'http://www.esa.int/Our_Activities/Space_Science/Rosetta/Philae_comet_touch-down_webcast',

From 74ba450a816927cdcf6ab5a60182359a8a1d8469 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 28 Jul 2016 22:30:09 +0700
Subject: [PATCH 5606/6219] [twitch:clips] Fix extraction (Closes #9767)

---
 youtube_dl/extractor/twitch.py | 24 +++++++++++++++++++-----
 1 file changed, 19 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 67b1277cc..5757eb119 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -461,7 +461,7 @@ class TwitchClipsIE(InfoExtractor):
     IE_NAME = 'twitch:clips'
     _VALID_URL = r'https?://clips\.twitch\.tv/(?:[^/]+/)*(?P<id>[^/?#&]+)'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'https://clips.twitch.tv/ea/AggressiveCobraPoooound',
         'md5': '761769e1eafce0ffebfb4089cb3847cd',
         'info_dict': {
@@ -473,7 +473,11 @@ class TwitchClipsIE(InfoExtractor):
             'uploader': 'stereotype_',
             'uploader_id': 'stereotype_',
         },
-    }
+    }, {
+        # multiple formats
+        'url': 'https://clips.twitch.tv/rflegendary/UninterestedBeeDAESuppy',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -485,15 +489,25 @@ class TwitchClipsIE(InfoExtractor):
                 r'(?s)clipInfo\s*=\s*({.+?});', webpage, 'clip info'),
             video_id, transform_source=js_to_json)
 
-        video_url = clip['clip_video_url']
-        title = clip['channel_title']
+        title = clip.get('channel_title') or self._og_search_title(webpage)
+
+        formats = [{
+            'url': option['source'],
+            'format_id': option.get('quality'),
+            'height': int_or_none(option.get('quality')),
+        } for option in clip.get('quality_options', []) if option.get('source')]
+
+        if not formats:
+            formats = [{
+                'url': clip['clip_video_url'],
+            }]
 
         return {
             'id': video_id,
-            'url': video_url,
             'title': title,
             'thumbnail': self._og_search_thumbnail(webpage),
             'creator': clip.get('broadcaster_display_name') or clip.get('broadcaster_login'),
             'uploader': clip.get('curator_login'),
             'uploader_id': clip.get('curator_display_name'),
+            'formats': formats,
         }

From d9d56deadf1795a88ec1cfa7b18c13ca559b3d50 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 28 Jul 2016 02:42:57 +0700
Subject: [PATCH 5607/6219] release 2016.07.28

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 youtube_dl/version.py     | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index f102d2611..27257ee0a 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.07.26.2*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.07.26.2**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.07.28*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.07.28**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.07.26.2
+[debug] youtube-dl version 2016.07.28
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 9a1642407..2cfa406d9 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.07.26.2'
+__version__ = '2016.07.28'

From 0cacae280725074fd4eef82c079b0829a15206f1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 29 Jul 2016 09:01:53 +0700
Subject: [PATCH 5608/6219] [twitch:clips] Sort formats

---
 youtube_dl/extractor/twitch.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 5757eb119..890f55180 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -502,6 +502,8 @@ class TwitchClipsIE(InfoExtractor):
                 'url': clip['clip_video_url'],
             }]
 
+        self._sort_formats(formats)
+
         return {
             'id': video_id,
             'title': title,

From 481c5c5137baab386c1cc4a4163bae7404a25f61 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 29 Jul 2016 21:43:17 +0700
Subject: [PATCH 5609/6219] [tv2:article] Fix extraction (Closes #10188)

---
 youtube_dl/extractor/tv2.py | 21 ++++++++++++++++++---
 1 file changed, 18 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/tv2.py b/youtube_dl/extractor/tv2.py
index 86bb7915d..e4b4ac0e7 100644
--- a/youtube_dl/extractor/tv2.py
+++ b/youtube_dl/extractor/tv2.py
@@ -8,6 +8,7 @@ from ..utils import (
     determine_ext,
     int_or_none,
     float_or_none,
+    js_to_json,
     parse_iso8601,
     remove_end,
 )
@@ -105,7 +106,7 @@ class TV2ArticleIE(InfoExtractor):
         'url': 'http://www.tv2.no/2015/05/16/nyheter/alesund/krim/pingvin/6930542',
         'info_dict': {
             'id': '6930542',
-            'title': 'Russen hetses etter pingvintyveri – innrømmer å ha åpnet luken på buret',
+            'title': 'Russen hetses etter pingvintyveri - innrømmer å ha åpnet luken på buret',
             'description': 'md5:339573779d3eea3542ffe12006190954',
         },
         'playlist_count': 2,
@@ -119,9 +120,23 @@ class TV2ArticleIE(InfoExtractor):
 
         webpage = self._download_webpage(url, playlist_id)
 
+        # Old embed pattern (looks unused nowadays)
+        assets = re.findall(r'data-assetid=["\'](\d+)', webpage)
+
+        if not assets:
+            # New embed pattern
+            for v in re.findall('TV2ContentboxVideo\(({.+?})\)', webpage):
+                video = self._parse_json(
+                    v, playlist_id, transform_source=js_to_json, fatal=False)
+                if not video:
+                    continue
+                asset = video.get('assetId')
+                if asset:
+                    assets.append(asset)
+
         entries = [
-            self.url_result('http://www.tv2.no/v/%s' % video_id, 'TV2')
-            for video_id in re.findall(r'data-assetid="(\d+)"', webpage)]
+            self.url_result('http://www.tv2.no/v/%s' % asset_id, 'TV2')
+            for asset_id in assets]
 
         title = remove_end(self._og_search_title(webpage), ' - TV2.no')
         description = remove_end(self._og_search_description(webpage), ' - TV2.no')

From dbc0b39b9158be0bdf50d031d1e993078e6cd264 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 29 Jul 2016 22:01:34 +0700
Subject: [PATCH 5610/6219] [tv2] Improve extraction

---
 youtube_dl/extractor/tv2.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/tv2.py b/youtube_dl/extractor/tv2.py
index e4b4ac0e7..f225ec684 100644
--- a/youtube_dl/extractor/tv2.py
+++ b/youtube_dl/extractor/tv2.py
@@ -55,10 +55,11 @@ class TV2IE(InfoExtractor):
                 ext = determine_ext(video_url)
                 if ext == 'f4m':
                     formats.extend(self._extract_f4m_formats(
-                        video_url, video_id, f4m_id=format_id))
+                        video_url, video_id, f4m_id=format_id, fatal=False))
                 elif ext == 'm3u8':
                     formats.extend(self._extract_m3u8_formats(
-                        video_url, video_id, 'mp4', m3u8_id=format_id))
+                        video_url, video_id, 'mp4', entry_protocol='m3u8_native',
+                        m3u8_id=format_id, fatal=False))
                 elif ext == 'ism' or video_url.endswith('.ism/Manifest'):
                     pass
                 else:

From bb9f3bfedf5d11d8f246654a5e67708edc01c30d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 29 Jul 2016 17:14:04 +0200
Subject: [PATCH 5611/6219] Revert "[rtve] Fix extraction (#10076)"

This reverts commit c39b2ed990105e640456f126321ef3d771884405.

Apparently outside of Spain using 'auth/resources' is required (#10097).
---
 youtube_dl/extractor/rtve.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/rtve.py b/youtube_dl/extractor/rtve.py
index d33b05f5d..05becc92b 100644
--- a/youtube_dl/extractor/rtve.py
+++ b/youtube_dl/extractor/rtve.py
@@ -113,7 +113,9 @@ class RTVEALaCartaIE(InfoExtractor):
         png = self._download_webpage(png_request, video_id, 'Downloading url information')
         video_url = _decrypt_url(png)
         if not video_url.endswith('.f4m'):
-            video_url = video_url.replace('.net.rtve', '.multimedia.cdn.rtve')
+            video_url = video_url.replace(
+                'resources/', 'auth/resources/'
+            ).replace('.net.rtve', '.multimedia.cdn.rtve')
 
         subtitles = None
         if info.get('sbtFile') is not None:

From da0baba5c8a0ac6220cc0155044a01b97bc00a76 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 29 Jul 2016 17:20:27 +0200
Subject: [PATCH 5612/6219] [rtve] Fix extraction for some videos

For example http://www.rtve.es/alacarta/videos/documentos-tv/documentos-tv-descredito/3574098/.
---
 youtube_dl/extractor/rtve.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/rtve.py b/youtube_dl/extractor/rtve.py
index 05becc92b..34f9c4a99 100644
--- a/youtube_dl/extractor/rtve.py
+++ b/youtube_dl/extractor/rtve.py
@@ -113,9 +113,9 @@ class RTVEALaCartaIE(InfoExtractor):
         png = self._download_webpage(png_request, video_id, 'Downloading url information')
         video_url = _decrypt_url(png)
         if not video_url.endswith('.f4m'):
-            video_url = video_url.replace(
-                'resources/', 'auth/resources/'
-            ).replace('.net.rtve', '.multimedia.cdn.rtve')
+            if '?' not in video_url:
+                video_url = video_url.replace('resources/', 'auth/resources/')
+            video_url = video_url.replace('.net.rtve', '.multimedia.cdn.rtve')
 
         subtitles = None
         if info.get('sbtFile') is not None:

From 485fedf6fd801d7ae0796d661ae7624564f67df8 Mon Sep 17 00:00:00 2001
From: Dave <github@eric-carlier.me>
Date: Thu, 28 Jul 2016 19:23:51 +0200
Subject: [PATCH 5613/6219] [dailymotion:playlist] Optimize download archive
 processing

---
 youtube_dl/extractor/dailymotion.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index 1f92823b7..98d2c82f4 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -331,7 +331,7 @@ class DailymotionPlaylistIE(DailymotionBaseInfoExtractor):
 
             for video_id in re.findall(r'data-xid="(.+?)"', webpage):
                 if video_id not in video_ids:
-                    yield self.url_result('http://www.dailymotion.com/video/%s' % video_id, 'Dailymotion')
+                    yield self.url_result('http://www.dailymotion.com/video/%s' % video_id, 'Dailymotion', video_id)
                     video_ids.add(video_id)
 
             if re.search(self._MORE_PAGES_INDICATOR, webpage) is None:

From fa9f1d16b807e66a9c3c2dead77c44624d556408 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 29 Jul 2016 22:47:34 +0700
Subject: [PATCH 5614/6219] [dailymotion:playlist] Carry long line

---
 youtube_dl/extractor/dailymotion.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index 98d2c82f4..496883d15 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -331,7 +331,9 @@ class DailymotionPlaylistIE(DailymotionBaseInfoExtractor):
 
             for video_id in re.findall(r'data-xid="(.+?)"', webpage):
                 if video_id not in video_ids:
-                    yield self.url_result('http://www.dailymotion.com/video/%s' % video_id, 'Dailymotion', video_id)
+                    yield self.url_result(
+                        'http://www.dailymotion.com/video/%s' % video_id,
+                        DailymotionIE.ie_key(), video_id)
                     video_ids.add(video_id)
 
             if re.search(self._MORE_PAGES_INDICATOR, webpage) is None:

From 35aa6c538f593eb47169e6b06ef383ad12b45930 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 7 Jul 2016 13:41:25 +0800
Subject: [PATCH 5615/6219] Add ChangeLog

---
 ChangeLog             | 261 ++++++++++++++++++++++++++++++++++++++++++
 Makefile              |   4 +-
 devscripts/release.sh |   5 +-
 3 files changed, 267 insertions(+), 3 deletions(-)
 create mode 100644 ChangeLog

diff --git a/ChangeLog b/ChangeLog
new file mode 100644
index 000000000..de29ccf5e
--- /dev/null
+++ b/ChangeLog
@@ -0,0 +1,261 @@
+version <unreleased>
+
+Fixed/improved extractors
+- twitch
+- tv2 (#10188)
+- rtve (#10076)
+- dailymotion (#10180)
+
+
+version 2016.07.28
+
+Fixed/improved extractors
+- shared (#10170)
+- soundcloud (#10179)
+- twitch (#9767)
+
+
+version 2016.07.26.2
+
+Fixed/improved extractors
+- smotri
+- camdemy
+- mtv
+- comedycentral
+- cmt
+- cbc
+- mgtv
+- orf
+
+
+version 2016.07.24
+
+New extractors
+- arkena (#8682)
+- lcp (#8682)
+
+Fixed/improved extractors
+- facebook (#10151)
+- dailymail
+- telegraaf
+- dcn
+- onet
+- tvp
+
+Miscellaneous
+- Support $Time$ in DASH manifests
+
+
+version 2016.07.22
+
+New extractors
+- odatv (#9285)
+
+Fixed/improved extractors
+- bbc
+- youjizz (#10131)
+- youtube (#10140)
+- pornhub (#10138)
+- eporner (#10139)
+
+
+version 2016.07.17
+
+New extractors
+- nintendo (#9986)
+- streamable (#9122)
+
+Fixed/improved extractors
+- ard (#10095)
+- mtv
+- comedycentral (#10101)
+- viki (#10098)
+- spike (#10106)
+
+Miscellaneous
+- Improved twitter player detection (#10090)
+
+
+version 2016.07.16
+
+New extractors
+- ninenow (#5181)
+
+Fixed/improved extractors
+- rtve (#10076)
+- brightcove
+- 3qsdn
+- syfy (#9087, #3820, #2388)
+- youtube (#10083)
+
+Miscellaneous
+- Fix subtitle embedding for video-only and audio-only files (#10081)
+
+
+version 2016.07.13
+
+New extractors
+- rudo
+
+Fixed/improved extractors
+- biobiochiletv
+- tvplay
+- dbtv
+- brightcove
+- tmz
+- youtube (#10059)
+- shahid (#10062)
+- vk
+- ellentv (#10067)
+
+
+version 2016.07.11
+
+New Extractors
+- roosterteeth (#9864)
+
+Fixed/improved extractors
+- miomio (#9605)
+- vuclip
+- youtube
+- vidzi (#10058)
+
+
+version 2016.07.09.2
+
+Fixed/improved extractors
+- vimeo (#1638)
+- facebook (#10048)
+- lynda (#10047)
+- animeondemand
+
+Fixed/improved features
+- Embedding subtitles no longer throws an error with problematic inputs (#9063)
+
+
+version 2016.07.09.1
+
+Fixed/improved extractors
+- youtube
+- ard
+- srmediatek (#9373)
+
+
+version 2016.07.09
+
+New extractors
+- Flipagram (#9898)
+
+Fixed/improved extractors
+- telecinco
+- toutv
+- radiocanada
+- tweakers (#9516)
+- lynda
+- nick (#7542)
+- polskieradio (#10028)
+- le
+- facebook (#9851)
+- mgtv
+- animeondemand (#10031)
+
+Fixed/improved features
+- `--postprocessor-args` and `--downloader-args` now accepts non-ASCII inputs
+  on non-Windows systems
+
+
+version 2016.07.07
+
+New extractors
+- kamcord (#10001)
+
+Fixed/improved extractors
+- spiegel (#10018)
+- metacafe (#8539, #3253)
+- onet (#9950)
+- francetv (#9955)
+- brightcove (#9965)
+- daum (#9972)
+
+
+version 2016.07.06
+
+Fixed/improved extractors
+- youtube (#10007, #10009)
+- xuite
+- stitcher
+- spiegel
+- slideshare
+- sandia
+- rtvnh
+- prosiebensat1
+- onionstudios
+
+
+version 2016.07.05
+
+Fixed/improved extractors
+- brightcove
+- yahoo (#9995)
+- pornhub (#9997)
+- iqiyi
+- kaltura (#5557)
+- la7
+- Changed features
+- Rename --cn-verfication-proxy to --geo-verification-proxy
+Miscellaneous
+- Add script for displaying downloads statistics
+
+
+version 2016.07.03.1
+
+Fixed/improved extractors
+- theplatform
+- aenetworks
+- nationalgeographic
+- hrti (#9482)
+- facebook (#5701)
+- buzzfeed (#5701)
+- rai (#8617, #9157, #9232, #8552, #8551)
+- nationalgeographic (#9991)
+- iqiyi
+
+
+version 2016.07.03
+
+New extractors
+- hrti (#9482)
+
+Fixed/improved extractors
+- vk (#9981)
+- facebook (#9938)
+- xtube (#9953, #9961)
+
+
+version 2016.07.02
+
+New extractors
+- fusion (#9958)
+
+Fixed/improved extractors
+- twitch (#9975)
+- vine (#9970)
+- periscope (#9967)
+- pornhub (#8696)
+
+
+version 2016.07.01
+
+New extractors
+- 9c9media
+- ctvnews (#2156)
+- ctv (#4077)
+
+Fixed/Improved extractors
+- rds
+- meta (#8789)
+- pornhub (#9964)
+- sixplay (#2183)
+
+New features
+- Accept quoted strings across multiple lines (#9940)
diff --git a/Makefile b/Makefile
index 6ee4ba4eb..354052c50 100644
--- a/Makefile
+++ b/Makefile
@@ -94,7 +94,7 @@ _EXTRACTOR_FILES != find youtube_dl/extractor -iname '*.py' -and -not -iname 'la
 youtube_dl/extractor/lazy_extractors.py: devscripts/make_lazy_extractors.py devscripts/lazy_load_template.py $(_EXTRACTOR_FILES)
 	$(PYTHON) devscripts/make_lazy_extractors.py $@
 
-youtube-dl.tar.gz: youtube-dl README.md README.txt youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtube-dl.fish
+youtube-dl.tar.gz: youtube-dl README.md README.txt youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtube-dl.fish ChangeLog
 	@tar -czf youtube-dl.tar.gz --transform "s|^|youtube-dl/|" --owner 0 --group 0 \
 		--exclude '*.DS_Store' \
 		--exclude '*.kate-swp' \
@@ -107,7 +107,7 @@ youtube-dl.tar.gz: youtube-dl README.md README.txt youtube-dl.1 youtube-dl.bash-
 		--exclude 'docs/_build' \
 		-- \
 		bin devscripts test youtube_dl docs \
-		LICENSE README.md README.txt \
+		ChangeLog LICENSE README.md README.txt \
 		Makefile MANIFEST.in youtube-dl.1 youtube-dl.bash-completion \
 		youtube-dl.zsh youtube-dl.fish setup.py \
 		youtube-dl
diff --git a/devscripts/release.sh b/devscripts/release.sh
index f8d466ba8..ca6ae1b49 100755
--- a/devscripts/release.sh
+++ b/devscripts/release.sh
@@ -71,9 +71,12 @@ fi
 /bin/echo -e "\n### Changing version in version.py..."
 sed -i "s/__version__ = '.*'/__version__ = '$version'/" youtube_dl/version.py
 
+/bin/echo -e "\n### Changing version in ChangeLog..."
+sed -i "s/<unreleased>/$version/" ChangeLog
+
 /bin/echo -e "\n### Committing documentation, templates and youtube_dl/version.py..."
 make README.md CONTRIBUTING.md .github/ISSUE_TEMPLATE.md supportedsites
-git add README.md CONTRIBUTING.md .github/ISSUE_TEMPLATE.md docs/supportedsites.md youtube_dl/version.py
+git add README.md CONTRIBUTING.md .github/ISSUE_TEMPLATE.md docs/supportedsites.md youtube_dl/version.py ChangeLog
 git commit $gpg_sign_commits -m "release $version"
 
 /bin/echo -e "\n### Now tagging, signing and pushing..."

From 9361f2169c61df29139aad1c500dda39c1bce4cf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 30 Jul 2016 14:43:28 +0700
Subject: [PATCH 5616/6219] [ChangeLog] Make extractor improvements'
 descriptions more concrete

---
 ChangeLog | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index de29ccf5e..1c7bc6094 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,10 +1,11 @@
 version <unreleased>
 
 Fixed/improved extractors
-- twitch
-- tv2 (#10188)
+- [twitch:clips] Sort formats
+- [tv2] Use m3u8_native
+- [tv2:article] Fix video detection (#10188)
 - rtve (#10076)
-- dailymotion (#10180)
+- [dailymotion:playlist] Optimize download archive processing (#10180)
 
 
 version 2016.07.28

From 2903137292721afc50c1f3a97c677cea1bc2d07b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 30 Jul 2016 14:45:07 +0700
Subject: [PATCH 5617/6219] release 2016.07.30

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 youtube_dl/version.py     | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 27257ee0a..aaf06aa05 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.07.28*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.07.28**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.07.30*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.07.30**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.07.28
+[debug] youtube-dl version 2016.07.30
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 1c7bc6094..3ee3b1237 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2016.07.30
 
 Fixed/improved extractors
 - [twitch:clips] Sort formats
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 2cfa406d9..9056a883b 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.07.28'
+__version__ = '2016.07.30'

From cf03e34ad3cd42997da92b49412d98d06ea82681 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 31 Jul 2016 07:56:18 +0700
Subject: [PATCH 5618/6219] [yandexmusic:track] Fix extraction (Closes #10193)

---
 youtube_dl/extractor/yandexmusic.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/yandexmusic.py b/youtube_dl/extractor/yandexmusic.py
index b37d0eab6..fd6268ba4 100644
--- a/youtube_dl/extractor/yandexmusic.py
+++ b/youtube_dl/extractor/yandexmusic.py
@@ -75,6 +75,12 @@ class YandexMusicTrackIE(YandexMusicBaseIE):
             % storage_dir,
             track_id, 'Downloading track location JSON')
 
+        # Each string is now wrapped in a list, this is probably only temporarily thus
+        # supporting both scenarios (see https://github.com/rg3/youtube-dl/issues/10193)
+        for k, v in data.items():
+            if v and isinstance(v, list):
+                data[k] = v[0]
+
         key = hashlib.md5(('XGRlBW9FXlekgbPrRHuSiA' + data['path'][1:] + data['s']).encode('utf-8')).hexdigest()
         storage = storage_dir.split('.')
 

From 116e7e0d044c5b50bf8221329bcd54d00c0dcad5 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 31 Jul 2016 14:46:54 +0800
Subject: [PATCH 5619/6219] [bloomberg] Support BPlayer() players (closes
 #10187)

---
 ChangeLog                         |  5 +++++
 youtube_dl/extractor/bloomberg.py | 19 ++++++++++++++++++-
 2 files changed, 23 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index 3ee3b1237..cb2d0beb2 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,8 @@
+version <unreleased>
+
+Fixed/improved extractors
+- [bloomberg] Support another form of player (#10187)
+
 version 2016.07.30
 
 Fixed/improved extractors
diff --git a/youtube_dl/extractor/bloomberg.py b/youtube_dl/extractor/bloomberg.py
index bd538be50..2a8cd64b9 100644
--- a/youtube_dl/extractor/bloomberg.py
+++ b/youtube_dl/extractor/bloomberg.py
@@ -1,3 +1,4 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
 import re
@@ -20,6 +21,18 @@ class BloombergIE(InfoExtractor):
         'params': {
             'format': 'best[format_id^=hds]',
         },
+    }, {
+        # video ID in BPlayer(...)
+        'url': 'http://www.bloomberg.com/features/2016-hello-world-new-zealand/',
+        'info_dict': {
+            'id': '938c7e72-3f25-4ddb-8b85-a9be731baa74',
+            'ext': 'flv',
+            'title': 'Meet the Real-Life Tech Wizards of Middle Earth',
+            'description': 'Hello World, Episode 1: New Zealand’s freaky AI babies, robot exoskeletons, and a virtual you.',
+        },
+        'params': {
+            'format': 'best[format_id^=hds]',
+        },
     }, {
         'url': 'http://www.bloomberg.com/news/articles/2015-11-12/five-strange-things-that-have-been-happening-in-financial-markets',
         'only_matching': True,
@@ -33,7 +46,11 @@ class BloombergIE(InfoExtractor):
         webpage = self._download_webpage(url, name)
         video_id = self._search_regex(
             r'["\']bmmrId["\']\s*:\s*(["\'])(?P<url>.+?)\1',
-            webpage, 'id', group='url')
+            webpage, 'id', group='url', default=None)
+        if not video_id:
+            bplayer_data = self._parse_json(self._search_regex(
+                r'BPlayer\(null,\s*({[^;]+})\);', webpage, 'id'), name)
+            video_id = bplayer_data['id']
         title = re.sub(': Video$', '', self._og_search_title(webpage))
 
         embed_info = self._download_json(

From e382b953f00982a2085d3e0b1b6fb4d2a0f2db7e Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 1 Aug 2016 00:33:30 +0100
Subject: [PATCH 5620/6219] [limelight] skip preview and drm protected videos

---
 youtube_dl/extractor/limelight.py | 27 ++++++++++++++++-----------
 1 file changed, 16 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/limelight.py b/youtube_dl/extractor/limelight.py
index 5d2c3e256..0d7abbaa8 100644
--- a/youtube_dl/extractor/limelight.py
+++ b/youtube_dl/extractor/limelight.py
@@ -37,11 +37,12 @@ class LimelightBaseIE(InfoExtractor):
 
         for stream in streams:
             stream_url = stream.get('url')
-            if not stream_url:
+            if not stream_url or stream.get('previewStream') or stream.get('drmProtected'):
                 continue
-            if '.f4m' in stream_url:
+            ext = determine_ext(stream_url)
+            if ext == 'f4m':
                 formats.extend(self._extract_f4m_formats(
-                    stream_url, video_id, fatal=False))
+                    stream_url, video_id, f4m_id='hds', fatal=False))
             else:
                 fmt = {
                     'url': stream_url,
@@ -50,7 +51,7 @@ class LimelightBaseIE(InfoExtractor):
                     'fps': float_or_none(stream.get('videoFrameRate')),
                     'width': int_or_none(stream.get('videoWidthInPixels')),
                     'height': int_or_none(stream.get('videoHeightInPixels')),
-                    'ext': determine_ext(stream_url)
+                    'ext': ext,
                 }
                 rtmp = re.search(r'^(?P<url>rtmpe?://[^/]+/(?P<app>.+))/(?P<playpath>mp4:.+)$', stream_url)
                 if rtmp:
@@ -68,18 +69,23 @@ class LimelightBaseIE(InfoExtractor):
 
         for mobile_url in mobile_urls:
             media_url = mobile_url.get('mobileUrl')
-            if not media_url:
-                continue
             format_id = mobile_url.get('targetMediaPlatform')
-            if determine_ext(media_url) == 'm3u8':
+            if not media_url or format_id == 'Widevine':
+                continue
+            ext = determine_ext(media_url)
+            if ext == 'm3u8':
                 formats.extend(self._extract_m3u8_formats(
                     media_url, video_id, 'mp4', 'm3u8_native',
                     m3u8_id=format_id, fatal=False))
+            elif ext == 'f4m':
+                formats.extend(self._extract_f4m_formats(
+                    stream_url, video_id, f4m_id=format_id, fatal=False))
             else:
                 formats.append({
                     'url': media_url,
                     'format_id': format_id,
                     'preference': -1,
+                    'ext': ext,
                 })
 
         self._sort_formats(formats)
@@ -145,7 +151,7 @@ class LimelightMediaIE(LimelightBaseIE):
         'url': 'http://link.videoplatform.limelight.com/media/?mediaId=3ffd040b522b4485b6d84effc750cd86',
         'info_dict': {
             'id': '3ffd040b522b4485b6d84effc750cd86',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'HaP and the HB Prince Trailer',
             'description': 'md5:8005b944181778e313d95c1237ddb640',
             'thumbnail': 're:^https?://.*\.jpeg$',
@@ -154,7 +160,7 @@ class LimelightMediaIE(LimelightBaseIE):
             'upload_date': '20090604',
         },
         'params': {
-            # rtmp download
+            # m3u8 download
             'skip_download': True,
         },
     }, {
@@ -164,7 +170,6 @@ class LimelightMediaIE(LimelightBaseIE):
             'id': 'a3e00274d4564ec4a9b29b9466432335',
             'ext': 'flv',
             'title': '3Play Media Overview Video',
-            'description': '',
             'thumbnail': 're:^https?://.*\.jpeg$',
             'duration': 78.101,
             'timestamp': 1338929955,
@@ -172,7 +177,7 @@ class LimelightMediaIE(LimelightBaseIE):
             'subtitles': 'mincount:9',
         },
         'params': {
-            # rtmp download
+            # m3u8 download
             'skip_download': True,
         },
     }, {

From 697655a7c0c1469dd7474714652025961e82bd69 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 1 Aug 2016 21:48:48 +0700
Subject: [PATCH 5621/6219] [safari] Relax url regexes (Closes #10202)

---
 youtube_dl/extractor/safari.py | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/safari.py b/youtube_dl/extractor/safari.py
index 6ba91f202..08ddbe3c4 100644
--- a/youtube_dl/extractor/safari.py
+++ b/youtube_dl/extractor/safari.py
@@ -75,7 +75,7 @@ class SafariBaseIE(InfoExtractor):
 class SafariIE(SafariBaseIE):
     IE_NAME = 'safari'
     IE_DESC = 'safaribooksonline.com online video'
-    _VALID_URL = r'https?://(?:www\.)?safaribooksonline\.com/library/view/[^/]+/(?P<course_id>[^/]+)/(?P<part>part\d+)\.html'
+    _VALID_URL = r'https?://(?:www\.)?safaribooksonline\.com/library/view/[^/]+/(?P<course_id>[^/]+)/(?P<part>[^/?#&]+)\.html'
 
     _TESTS = [{
         'url': 'https://www.safaribooksonline.com/library/view/hadoop-fundamentals-livelessons/9780133392838/part00.html',
@@ -92,6 +92,9 @@ class SafariIE(SafariBaseIE):
         # non-digits in course id
         'url': 'https://www.safaribooksonline.com/library/view/create-a-nodejs/100000006A0210/part00.html',
         'only_matching': True,
+    }, {
+        'url': 'https://www.safaribooksonline.com/library/view/learning-path-red/9780134664057/RHCE_Introduction.html',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
@@ -132,12 +135,15 @@ class SafariIE(SafariBaseIE):
 
 class SafariApiIE(SafariBaseIE):
     IE_NAME = 'safari:api'
-    _VALID_URL = r'https?://(?:www\.)?safaribooksonline\.com/api/v1/book/(?P<course_id>[^/]+)/chapter(?:-content)?/(?P<part>part\d+)\.html'
+    _VALID_URL = r'https?://(?:www\.)?safaribooksonline\.com/api/v1/book/(?P<course_id>[^/]+)/chapter(?:-content)?/(?P<part>[^/?#&]+)\.html'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'https://www.safaribooksonline.com/api/v1/book/9780133392838/chapter/part00.html',
         'only_matching': True,
-    }
+    }, {
+        'url': 'https://www.safaribooksonline.com/api/v1/book/9780134664057/chapter/RHCE_Introduction.html',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)

From a70e45f80a398fccbb757dd2e166d15f16ffb160 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 1 Aug 2016 16:25:41 +0100
Subject: [PATCH 5622/6219] [limelight] keep videos marked as previewStream

https://github.com/rg3/youtube-dl/commit/e382b953f00982a2085d3e0b1b6fb4d2a0f2db7e#commitcomment-18472915
---
 youtube_dl/extractor/limelight.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/limelight.py b/youtube_dl/extractor/limelight.py
index 0d7abbaa8..efe1437e0 100644
--- a/youtube_dl/extractor/limelight.py
+++ b/youtube_dl/extractor/limelight.py
@@ -37,7 +37,7 @@ class LimelightBaseIE(InfoExtractor):
 
         for stream in streams:
             stream_url = stream.get('url')
-            if not stream_url or stream.get('previewStream') or stream.get('drmProtected'):
+            if not stream_url or stream.get('drmProtected'):
                 continue
             ext = determine_ext(stream_url)
             if ext == 'f4m':
@@ -177,7 +177,7 @@ class LimelightMediaIE(LimelightBaseIE):
             'subtitles': 'mincount:9',
         },
         'params': {
-            # m3u8 download
+            # rtmp download
             'skip_download': True,
         },
     }, {

From e03d3e64533a2b64293b9ed38b62acd3a8b06d9c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 1 Aug 2016 22:51:01 +0700
Subject: [PATCH 5623/6219] [cwtv] Add support for cwtvpr.com (Closes #10196)

---
 youtube_dl/extractor/cwtv.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/cwtv.py b/youtube_dl/extractor/cwtv.py
index ebd14cb16..c66c359cf 100644
--- a/youtube_dl/extractor/cwtv.py
+++ b/youtube_dl/extractor/cwtv.py
@@ -9,7 +9,7 @@ from ..utils import (
 
 
 class CWTVIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?cw(?:tv|seed)\.com/(?:shows/)?(?:[^/]+/){2}\?.*\bplay=(?P<id>[a-z0-9]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12})'
+    _VALID_URL = r'https?://(?:www\.)?cw(?:tv(?:pr)?|seed)\.com/(?:shows/)?(?:[^/]+/)+[^?]*\?.*\b(?:play|watch)=(?P<id>[a-z0-9]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12})'
     _TESTS = [{
         'url': 'http://cwtv.com/shows/arrow/legends-of-yesterday/?play=6b15e985-9345-4f60-baf8-56e96be57c63',
         'info_dict': {
@@ -51,6 +51,12 @@ class CWTVIE(InfoExtractor):
     }, {
         'url': 'http://cwtv.com/thecw/chroniclesofcisco/?play=8adebe35-f447-465f-ab52-e863506ff6d6',
         'only_matching': True,
+    }, {
+        'url': 'http://cwtvpr.com/the-cw/video?watch=9eee3f60-ef4e-440b-b3b2-49428ac9c54e',
+        'only_matching': True,
+    }, {
+        'url': 'http://cwtv.com/shows/arrow/legends-of-yesterday/?watch=6b15e985-9345-4f60-baf8-56e96be57c63',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From eafc66855dd591ce4112bb51cdd88eebd3fcf2ac Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 1 Aug 2016 22:56:01 +0700
Subject: [PATCH 5624/6219] [ChangeLog] Add recent changes

---
 ChangeLog | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index cb2d0beb2..56bb6cc18 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,7 +1,11 @@
 version <unreleased>
 
 Fixed/improved extractors
+- [yandexmusic:track] Adapt to changes in track location JSON (#10193)
 - [bloomberg] Support another form of player (#10187)
+- [limelight] Skip DRM protected videos
+- [safari] Relax regular expressions for URL matching (#10202)
+- [cwtv] Add support for cwtvpr.com (#10196)
 
 version 2016.07.30
 

From 45408eb0750ac53f965756331022cbed3dbe2a38 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 1 Aug 2016 22:59:23 +0700
Subject: [PATCH 5625/6219] release 2016.08.01

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 youtube_dl/version.py     | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index aaf06aa05..9d15b6a89 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.07.30*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.07.30**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.08.01*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.08.01**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.07.30
+[debug] youtube-dl version 2016.08.01
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 56bb6cc18..f3c752e66 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2016.08.01
 
 Fixed/improved extractors
 - [yandexmusic:track] Adapt to changes in track location JSON (#10193)
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 9056a883b..27f97b213 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.07.30'
+__version__ = '2016.08.01'

From 6a9b3b61ea145ac03198bc81a67ee322d42a1bc1 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 2 Aug 2016 14:02:31 +0200
Subject: [PATCH 5626/6219] [comedycentral] Re-add shortnames

In cc99d4f826a942b18133fe4221c9de2f9197e860, the shortname feature got deleted by accident. Re-add it as a separate IE.
---
 youtube_dl/extractor/comedycentral.py | 20 ++++++++++++++++++++
 youtube_dl/extractor/extractors.py    |  1 +
 2 files changed, 21 insertions(+)

diff --git a/youtube_dl/extractor/comedycentral.py b/youtube_dl/extractor/comedycentral.py
index c76909e48..88346dde7 100644
--- a/youtube_dl/extractor/comedycentral.py
+++ b/youtube_dl/extractor/comedycentral.py
@@ -1,6 +1,7 @@
 from __future__ import unicode_literals
 
 from .mtv import MTVServicesInfoExtractor
+from .common import InfoExtractor
 
 
 class ComedyCentralIE(MTVServicesInfoExtractor):
@@ -96,3 +97,22 @@ class ComedyCentralTVIE(MTVServicesInfoExtractor):
             webpage, 'mrss url', group='url')
 
         return self._get_videos_info_from_url(mrss_url, video_id)
+
+
+class ComedyCentralShortnameIE(InfoExtractor):
+    _VALID_URL = r'^:(?P<id>tds|thedailyshow)$'
+    _TESTS = [{
+        'url': ':tds',
+        'only_matching': True,
+    }, {
+        'url': ':thedailyshow',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        shortcut_map = {
+            'tds': 'http://www.cc.com/shows/the-daily-show-with-trevor-noah/full-episodes',
+            'thedailyshow': 'http://www.cc.com/shows/the-daily-show-with-trevor-noah/full-episodes',
+        }
+        return self.url_result(shortcut_map[video_id])
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 53fab1a31..86c48ff54 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -159,6 +159,7 @@ from .coub import CoubIE
 from .collegerama import CollegeRamaIE
 from .comedycentral import (
     ComedyCentralIE,
+    ComedyCentralShortnameIE,
     ComedyCentralTVIE,
     ToshIE,
 )

From 3aa9a73554a43b6dcb7a69a6ff8e8fd02c1e836e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 2 Aug 2016 17:03:26 +0200
Subject: [PATCH 5627/6219] [options] Hide --password=secret in verbose output

---
 youtube_dl/options.py | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index c4a85b2c0..942d44912 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -2,6 +2,7 @@ from __future__ import unicode_literals
 
 import os.path
 import optparse
+import re
 import sys
 
 from .downloader.external import list_external_downloaders
@@ -93,8 +94,18 @@ def parseOpts(overrideArguments=None):
         setattr(parser.values, option.dest, value.split(','))
 
     def _hide_login_info(opts):
-        opts = list(opts)
-        for private_opt in ['-p', '--password', '-u', '--username', '--video-password']:
+        PRIVATE_OPTS = ['-p', '--password', '-u', '--username', '--video-password']
+        eqre = re.compile('^(?P<key>' + ('|'.join(re.escape(po) for po in PRIVATE_OPTS)) + ')=.+$')
+
+        def _scrub_eq(o):
+            m = eqre.match(o)
+            if m:
+                return m.group('key') + '=PRIVATE'
+            else:
+                return o
+
+        opts = list(map(_scrub_eq, opts))
+        for private_opt in PRIVATE_OPTS:
             try:
                 i = opts.index(private_opt)
                 opts[i + 1] = 'PRIVATE'

From ce28252c485cea5cdc2b0f5fd9f1c8d50c26c7a2 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 2 Aug 2016 17:03:46 +0200
Subject: [PATCH 5628/6219] [options] Add test that checks that
 --password=secret is hidden in verbose output

---
 test/test_verbose_output.py | 70 +++++++++++++++++++++++++++++++++++++
 1 file changed, 70 insertions(+)
 create mode 100644 test/test_verbose_output.py

diff --git a/test/test_verbose_output.py b/test/test_verbose_output.py
new file mode 100644
index 000000000..4c77df242
--- /dev/null
+++ b/test/test_verbose_output.py
@@ -0,0 +1,70 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+from __future__ import unicode_literals
+
+import unittest
+
+import sys
+import os
+import subprocess
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+rootDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+
+class TestVerboseOutput(unittest.TestCase):
+    def test_private_info_arg(self):
+        outp = subprocess.Popen(
+            [
+                sys.executable, 'youtube_dl/__main__.py', '-v',
+                '--username', 'johnsmith@gmail.com',
+                '--password', 'secret',
+            ], cwd=rootDir, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        sout, serr = outp.communicate()
+        self.assertTrue('--username' in serr)
+        self.assertTrue('johnsmith' not in serr)
+        self.assertTrue('--password' in serr)
+        self.assertTrue('secret' not in serr)
+
+    def test_private_info_shortarg(self):
+        outp = subprocess.Popen(
+            [
+                sys.executable, 'youtube_dl/__main__.py', '-v',
+                '-u', 'johnsmith@gmail.com',
+                '-p', 'secret',
+            ], cwd=rootDir, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        sout, serr = outp.communicate()
+        self.assertTrue('-u' in serr)
+        self.assertTrue('johnsmith' not in serr)
+        self.assertTrue('-p' in serr)
+        self.assertTrue('secret' not in serr)
+
+    def test_private_info_eq(self):
+        outp = subprocess.Popen(
+            [
+                sys.executable, 'youtube_dl/__main__.py', '-v',
+                '--username=johnsmith@gmail.com',
+                '--password=secret',
+            ], cwd=rootDir, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        sout, serr = outp.communicate()
+        self.assertTrue('--username' in serr)
+        self.assertTrue('johnsmith' not in serr)
+        self.assertTrue('--password' in serr)
+        self.assertTrue('secret' not in serr)
+
+    def test_private_info_shortarg_eq(self):
+        outp = subprocess.Popen(
+            [
+                sys.executable, 'youtube_dl/__main__.py', '-v',
+                '-u=johnsmith@gmail.com',
+                '-p=secret',
+            ], cwd=rootDir, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        sout, serr = outp.communicate()
+        self.assertTrue('-u' in serr)
+        self.assertTrue('johnsmith' not in serr)
+        self.assertTrue('-p' in serr)
+        self.assertTrue('secret' not in serr)
+
+if __name__ == '__main__':
+    unittest.main()

From b070564efb190e1740c4e9d5fdde25cc436c72c3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 2 Aug 2016 22:55:14 +0700
Subject: [PATCH 5629/6219] [extractor/common] Support multiple properties in
 _og_search_property

---
 test/test_InfoExtractor.py     | 3 +++
 youtube_dl/extractor/common.py | 9 +++++++--
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/test/test_InfoExtractor.py b/test/test_InfoExtractor.py
index 88e8ff904..a98305c74 100644
--- a/test/test_InfoExtractor.py
+++ b/test/test_InfoExtractor.py
@@ -48,6 +48,9 @@ class TestInfoExtractor(unittest.TestCase):
         self.assertEqual(ie._og_search_property('foobar', html), 'Foo')
         self.assertEqual(ie._og_search_property('test1', html), 'foo > < bar')
         self.assertEqual(ie._og_search_property('test2', html), 'foo >//< bar')
+        self.assertEqual(ie._og_search_property(('test0', 'test1'), html), 'foo > < bar')
+        self.assertRaises(RegexNotFoundError, ie._og_search_property, 'test0', html, None, fatal=True)
+        self.assertRaises(RegexNotFoundError, ie._og_search_property, ('test0', 'test00'), html, None, fatal=True)
 
     def test_html_search_meta(self):
         ie = self.ie
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 53c28f016..3b6a5491d 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -727,9 +727,14 @@ class InfoExtractor(object):
                     [^>]+?content=(["\'])(?P<content>.*?)\2''' % re.escape(prop)
 
     def _og_search_property(self, prop, html, name=None, **kargs):
+        if not isinstance(prop, (list, tuple)):
+            prop = [prop]
         if name is None:
-            name = 'OpenGraph %s' % prop
-        escaped = self._search_regex(self._og_regexes(prop), html, name, flags=re.DOTALL, **kargs)
+            name = 'OpenGraph %s' % prop[0]
+        og_regexes = []
+        for p in prop:
+            og_regexes.extend(self._og_regexes(p))
+        escaped = self._search_regex(og_regexes, html, name, flags=re.DOTALL, **kargs)
         if escaped is None:
             return None
         return unescapeHTML(escaped)

From 9cb0e65d7e1c87a29268d30c6159e595bcf210ff Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 2 Aug 2016 22:56:48 +0700
Subject: [PATCH 5630/6219] [ntvru] Fix extraction

---
 youtube_dl/extractor/ntvru.py | 122 ++++++++++++++++++----------------
 1 file changed, 63 insertions(+), 59 deletions(-)

diff --git a/youtube_dl/extractor/ntvru.py b/youtube_dl/extractor/ntvru.py
index 0895d7ea4..e8702ebcd 100644
--- a/youtube_dl/extractor/ntvru.py
+++ b/youtube_dl/extractor/ntvru.py
@@ -11,70 +11,64 @@ from ..utils import (
 
 class NTVRuIE(InfoExtractor):
     IE_NAME = 'ntv.ru'
-    _VALID_URL = r'https?://(?:www\.)?ntv\.ru/(?P<id>.+)'
+    _VALID_URL = r'https?://(?:www\.)?ntv\.ru/(?:[^/]+/)*(?P<id>[^/?#&]+)'
 
-    _TESTS = [
-        {
-            'url': 'http://www.ntv.ru/novosti/863142/',
-            'md5': 'ba7ea172a91cb83eb734cad18c10e723',
-            'info_dict': {
-                'id': '746000',
-                'ext': 'mp4',
-                'title': 'Командующий Черноморским флотом провел переговоры в штабе ВМС Украины',
-                'description': 'Командующий Черноморским флотом провел переговоры в штабе ВМС Украины',
-                'thumbnail': 're:^http://.*\.jpg',
-                'duration': 136,
-            },
+    _TESTS = [{
+        'url': 'http://www.ntv.ru/novosti/863142/',
+        'md5': 'ba7ea172a91cb83eb734cad18c10e723',
+        'info_dict': {
+            'id': '746000',
+            'ext': 'mp4',
+            'title': 'Командующий Черноморским флотом провел переговоры в штабе ВМС Украины',
+            'description': 'Командующий Черноморским флотом провел переговоры в штабе ВМС Украины',
+            'thumbnail': 're:^http://.*\.jpg',
+            'duration': 136,
         },
-        {
-            'url': 'http://www.ntv.ru/video/novosti/750370/',
-            'md5': 'adecff79691b4d71e25220a191477124',
-            'info_dict': {
-                'id': '750370',
-                'ext': 'mp4',
-                'title': 'Родные пассажиров пропавшего Boeing не верят в трагический исход',
-                'description': 'Родные пассажиров пропавшего Boeing не верят в трагический исход',
-                'thumbnail': 're:^http://.*\.jpg',
-                'duration': 172,
-            },
+    }, {
+        'url': 'http://www.ntv.ru/video/novosti/750370/',
+        'md5': 'adecff79691b4d71e25220a191477124',
+        'info_dict': {
+            'id': '750370',
+            'ext': 'mp4',
+            'title': 'Родные пассажиров пропавшего Boeing не верят в трагический исход',
+            'description': 'Родные пассажиров пропавшего Boeing не верят в трагический исход',
+            'thumbnail': 're:^http://.*\.jpg',
+            'duration': 172,
         },
-        {
-            'url': 'http://www.ntv.ru/peredacha/segodnya/m23700/o232416',
-            'md5': '82dbd49b38e3af1d00df16acbeab260c',
-            'info_dict': {
-                'id': '747480',
-                'ext': 'mp4',
-                'title': '«Сегодня». 21 марта 2014 года. 16:00',
-                'description': '«Сегодня». 21 марта 2014 года. 16:00',
-                'thumbnail': 're:^http://.*\.jpg',
-                'duration': 1496,
-            },
+    }, {
+        'url': 'http://www.ntv.ru/peredacha/segodnya/m23700/o232416',
+        'md5': '82dbd49b38e3af1d00df16acbeab260c',
+        'info_dict': {
+            'id': '747480',
+            'ext': 'mp4',
+            'title': '«Сегодня». 21 марта 2014 года. 16:00',
+            'description': '«Сегодня». 21 марта 2014 года. 16:00',
+            'thumbnail': 're:^http://.*\.jpg',
+            'duration': 1496,
         },
-        {
-            'url': 'http://www.ntv.ru/kino/Koma_film',
-            'md5': 'f825770930937aa7e5aca0dc0d29319a',
-            'info_dict': {
-                'id': '1007609',
-                'ext': 'mp4',
-                'title': 'Остросюжетный фильм «Кома»',
-                'description': 'Остросюжетный фильм «Кома»',
-                'thumbnail': 're:^http://.*\.jpg',
-                'duration': 5592,
-            },
+    }, {
+        'url': 'http://www.ntv.ru/kino/Koma_film',
+        'md5': 'f825770930937aa7e5aca0dc0d29319a',
+        'info_dict': {
+            'id': '1007609',
+            'ext': 'mp4',
+            'title': 'Остросюжетный фильм «Кома»',
+            'description': 'Остросюжетный фильм «Кома»',
+            'thumbnail': 're:^http://.*\.jpg',
+            'duration': 5592,
         },
-        {
-            'url': 'http://www.ntv.ru/serial/Delo_vrachey/m31760/o233916/',
-            'md5': '9320cd0e23f3ea59c330dc744e06ff3b',
-            'info_dict': {
-                'id': '751482',
-                'ext': 'mp4',
-                'title': '«Дело врачей»: «Деревце жизни»',
-                'description': '«Дело врачей»: «Деревце жизни»',
-                'thumbnail': 're:^http://.*\.jpg',
-                'duration': 2590,
-            },
+    }, {
+        'url': 'http://www.ntv.ru/serial/Delo_vrachey/m31760/o233916/',
+        'md5': '9320cd0e23f3ea59c330dc744e06ff3b',
+        'info_dict': {
+            'id': '751482',
+            'ext': 'mp4',
+            'title': '«Дело врачей»: «Деревце жизни»',
+            'description': '«Дело врачей»: «Деревце жизни»',
+            'thumbnail': 're:^http://.*\.jpg',
+            'duration': 2590,
         },
-    ]
+    }]
 
     _VIDEO_ID_REGEXES = [
         r'<meta property="og:url" content="http://www\.ntv\.ru/video/(\d+)',
@@ -87,11 +81,21 @@ class NTVRuIE(InfoExtractor):
 
         webpage = self._download_webpage(url, video_id)
 
-        video_id = self._html_search_regex(self._VIDEO_ID_REGEXES, webpage, 'video id')
+        video_url = self._og_search_property(
+            ('video', 'video:iframe'), webpage, default=None)
+        if video_url:
+            video_id = self._search_regex(
+                r'https?://(?:www\.)?ntv\.ru/video/(?:embed/)?(\d+)',
+                video_url, 'video id', default=None)
+
+        if not video_id:
+            video_id = self._html_search_regex(
+                self._VIDEO_ID_REGEXES, webpage, 'video id')
 
         player = self._download_xml(
             'http://www.ntv.ru/vi%s/' % video_id,
             video_id, 'Downloading video XML')
+
         title = clean_html(xpath_text(player, './data/title', 'title', fatal=True))
         description = clean_html(xpath_text(player, './data/description', 'description'))
 

From 899d2bea63f34e85967a93da3c9c2835c3e9bfd4 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 3 Aug 2016 11:15:27 +0800
Subject: [PATCH 5631/6219] [README.md] Use full paths for all configuration
 files (#8863)

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index a9f3001a6..640625212 100644
--- a/README.md
+++ b/README.md
@@ -424,7 +424,7 @@ which means you can modify it, redistribute it or use it however you like.
 
 # CONFIGURATION
 
-You can configure youtube-dl by placing any supported command line option to a configuration file. On Linux and OS X, the system wide configuration file is located at `/etc/youtube-dl.conf` and the user wide configuration file at `~/.config/youtube-dl/config`. On Windows, the user wide configuration file locations are `%APPDATA%\youtube-dl\config.txt` or `C:\Users\<user name>\youtube-dl.conf`. Note that by default configuration file may not exist so you may need to create it yourself.
+You can configure youtube-dl by placing any supported command line option to a configuration file. On Linux and OS X, the system wide configuration file is located at `/etc/youtube-dl.conf` and the user wide configuration file at `~/.config/youtube-dl/config/youtube-dl.conf`. On Windows, the user wide configuration file locations are `%APPDATA%\youtube-dl\config.txt` or `C:\Users\<user name>\youtube-dl.conf`. Note that by default configuration file may not exist so you may need to create it yourself.
 
 For example, with the following configuration file youtube-dl will always extract the audio, not copy the mtime, use a proxy and save all videos under `Movies` directory in your home directory:
 ```

From 42b7a5afe09e485503cbe9794c7ad18c46dc838d Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 3 Aug 2016 13:11:48 +0100
Subject: [PATCH 5632/6219] [limelight] extract http formats

---
 youtube_dl/extractor/limelight.py | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/limelight.py b/youtube_dl/extractor/limelight.py
index efe1437e0..a425bafe3 100644
--- a/youtube_dl/extractor/limelight.py
+++ b/youtube_dl/extractor/limelight.py
@@ -53,11 +53,17 @@ class LimelightBaseIE(InfoExtractor):
                     'height': int_or_none(stream.get('videoHeightInPixels')),
                     'ext': ext,
                 }
-                rtmp = re.search(r'^(?P<url>rtmpe?://[^/]+/(?P<app>.+))/(?P<playpath>mp4:.+)$', stream_url)
+                rtmp = re.search(r'^(?P<url>rtmpe?://(?P<host>[^/]+)/(?P<app>.+))/(?P<playpath>mp4:.+)$', stream_url)
                 if rtmp:
                     format_id = 'rtmp'
                     if stream.get('videoBitRate'):
                         format_id += '-%d' % int_or_none(stream['videoBitRate'])
+                    http_fmt = fmt.copy()
+                    http_fmt.update({
+                        'url': 'http://%s/%s' % (rtmp.group('host').replace('csl.', 'cpl.'), rtmp.group('playpath')[4:]),
+                        'format_id': format_id.replace('rtmp', 'http'),
+                    })
+                    formats.append(http_fmt)
                     fmt.update({
                         'url': rtmp.group('url'),
                         'play_path': rtmp.group('playpath'),
@@ -166,9 +172,10 @@ class LimelightMediaIE(LimelightBaseIE):
     }, {
         # video with subtitles
         'url': 'limelight:media:a3e00274d4564ec4a9b29b9466432335',
+        'md5': '2fa3bad9ac321e23860ca23bc2c69e3d',
         'info_dict': {
             'id': 'a3e00274d4564ec4a9b29b9466432335',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': '3Play Media Overview Video',
             'thumbnail': 're:^https?://.*\.jpeg$',
             'duration': 78.101,
@@ -176,10 +183,6 @@ class LimelightMediaIE(LimelightBaseIE):
             'upload_date': '20120605',
             'subtitles': 'mincount:9',
         },
-        'params': {
-            # rtmp download
-            'skip_download': True,
-        },
     }, {
         'url': 'https://assets.delvenetworks.com/player/loader.swf?mediaId=8018a574f08d416e95ceaccae4ba0452',
         'only_matching': True,

From 8d3b226b8371c7fd58e052852acc01d6e97872b4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 3 Aug 2016 22:06:10 +0700
Subject: [PATCH 5633/6219] [gamekings] Remove extractor

Now covered by generic jwplayer
---
 youtube_dl/extractor/extractors.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 86c48ff54..8b866ed57 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -290,7 +290,6 @@ from .funimation import FunimationIE
 from .funnyordie import FunnyOrDieIE
 from .fusion import FusionIE
 from .gameinformer import GameInformerIE
-from .gamekings import GamekingsIE
 from .gameone import (
     GameOneIE,
     GameOnePlaylistIE,

From 6bb0fbf9fb60047087b44c6e147ed071fe0dc41f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 4 Aug 2016 09:54:28 +0800
Subject: [PATCH 5634/6219] Revert "[README.md] Use full paths for all
 configuration files (#8863)"

This reverts commit 899d2bea63f34e85967a93da3c9c2835c3e9bfd4.
---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 640625212..a9f3001a6 100644
--- a/README.md
+++ b/README.md
@@ -424,7 +424,7 @@ which means you can modify it, redistribute it or use it however you like.
 
 # CONFIGURATION
 
-You can configure youtube-dl by placing any supported command line option to a configuration file. On Linux and OS X, the system wide configuration file is located at `/etc/youtube-dl.conf` and the user wide configuration file at `~/.config/youtube-dl/config/youtube-dl.conf`. On Windows, the user wide configuration file locations are `%APPDATA%\youtube-dl\config.txt` or `C:\Users\<user name>\youtube-dl.conf`. Note that by default configuration file may not exist so you may need to create it yourself.
+You can configure youtube-dl by placing any supported command line option to a configuration file. On Linux and OS X, the system wide configuration file is located at `/etc/youtube-dl.conf` and the user wide configuration file at `~/.config/youtube-dl/config`. On Windows, the user wide configuration file locations are `%APPDATA%\youtube-dl\config.txt` or `C:\Users\<user name>\youtube-dl.conf`. Note that by default configuration file may not exist so you may need to create it yourself.
 
 For example, with the following configuration file youtube-dl will always extract the audio, not copy the mtime, use a proxy and save all videos under `Movies` directory in your home directory:
 ```

From 8b40854529eea6b32f41df448c475b87a42038d1 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 4 Aug 2016 09:24:20 +0100
Subject: [PATCH 5635/6219] [common] lower proto_preference of rtsp formats

Most of the time the RtspFD fail to download videos but it report
success of the download with this output:
[mpv] 0 bytes
[download] 100% of 0.00B
---
 youtube_dl/extractor/common.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 3b6a5491d..2d337d614 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -916,7 +916,8 @@ class InfoExtractor(object):
                 if f.get('ext') in ['f4f', 'f4m']:  # Not yet supported
                     preference -= 0.5
 
-            proto_preference = 0 if determine_protocol(f) in ['http', 'https'] else -0.1
+            protocol = f.get('protocol') or determine_protocol(f)
+            proto_preference = 0 if protocol in ['http', 'https'] else (-0.5 if protocol == 'rtsp' else -0.1)
 
             if f.get('vcodec') == 'none':  # audio only
                 preference -= 50

From 217d5ae0137943829db23d13eee425e5fd7c08ae Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 4 Aug 2016 09:37:27 +0100
Subject: [PATCH 5636/6219] [vodplatform] Add new extractor

---
 youtube_dl/extractor/extractors.py  |  1 +
 youtube_dl/extractor/generic.py     |  8 ++++
 youtube_dl/extractor/vodplatform.py | 58 +++++++++++++++++++++++++++++
 3 files changed, 67 insertions(+)
 create mode 100644 youtube_dl/extractor/vodplatform.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 8b866ed57..909f7d25c 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1004,6 +1004,7 @@ from .vk import (
 )
 from .vlive import VLiveIE
 from .vodlocker import VodlockerIE
+from .vodplatform import VODPlatformIE
 from .voicerepublic import VoiceRepublicIE
 from .voxmedia import VoxMediaIE
 from .vporn import VpornIE
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 5364f0b19..e89a03760 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -2207,6 +2207,14 @@ class GenericIE(InfoExtractor):
             return self.url_result(
                 self._proto_relative_url(unescapeHTML(mobj.group(1))), 'Vine')
 
+        # Look for VODPlatform embeds
+        mobj = re.search(
+            r'<iframe[^>]+src=[\'"]((?:https?:)?//(?:www\.)?vod-platform\.net/embed/[^/?#]+)',
+            webpage)
+        if mobj is not None:
+            return self.url_result(
+                self._proto_relative_url(unescapeHTML(mobj.group(1))), 'VODPlatform')
+
         # Look for Instagram embeds
         instagram_embed_url = InstagramIE._extract_embed_url(webpage)
         if instagram_embed_url is not None:
diff --git a/youtube_dl/extractor/vodplatform.py b/youtube_dl/extractor/vodplatform.py
new file mode 100644
index 000000000..b49542b16
--- /dev/null
+++ b/youtube_dl/extractor/vodplatform.py
@@ -0,0 +1,58 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import unescapeHTML
+
+
+class VODPlatformIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?vod-platform\.net/embed/(?P<id>[^/?#]+)'
+    _TEST = {
+        # from http://www.lbcgroup.tv/watch/chapter/29143/52844/%D8%A7%D9%84%D9%86%D8%B5%D8%B1%D8%A9-%D9%81%D9%8A-%D8%B6%D9%8A%D8%A7%D9%81%D8%A9-%D8%A7%D9%84%D9%80-cnn/ar
+        'url': 'http://vod-platform.net/embed/RufMcytHDolTH1MuKHY9Fw',
+        'md5': '1db2b7249ce383d6be96499006e951fc',
+        'info_dict': {
+            'id': 'RufMcytHDolTH1MuKHY9Fw',
+            'ext': 'mp4',
+            'title': 'LBCi News_ النصرة في ضيافة الـ "سي.أن.أن"',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        title = unescapeHTML(self._og_search_title(webpage))
+        hidden_inputs = self._hidden_inputs(webpage)
+
+        base_url = self._search_regex(
+            '(.*/)(?:playlist.m3u8|manifest.mpd)',
+            hidden_inputs.get('HiddenmyhHlsLink') or hidden_inputs['HiddenmyDashLink'],
+            'base url')
+        formats = self._extract_m3u8_formats(
+            base_url + 'playlist.m3u8', video_id, 'mp4',
+            'm3u8_native', m3u8_id='hls', fatal=False)
+        formats.extend(self._extract_mpd_formats(
+            base_url + 'manifest.mpd', video_id,
+            mpd_id='dash', fatal=False))
+        rtmp_formats = self._extract_smil_formats(
+            base_url + 'jwplayer.smil', video_id, fatal=False)
+        for rtmp_format in rtmp_formats:
+            rtsp_format = rtmp_format.copy()
+            rtsp_format['url'] = '%s/%s' % (rtmp_format['url'], rtmp_format['play_path'])
+            del rtsp_format['play_path']
+            del rtsp_format['ext']
+            rtsp_format.update({
+                'url': rtsp_format['url'].replace('rtmp://', 'rtsp://'),
+                'format_id': rtmp_format['format_id'].replace('rtmp', 'rtsp'),
+                'protocol': 'rtsp',
+            })
+            formats.extend([rtmp_format, rtsp_format])
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'thumbnail': hidden_inputs.get('HiddenThumbnail') or self._og_search_thumbnail(webpage),
+            'formats': formats,
+        }

From 1094074c045140e9a91b521b0a933f394a7bba91 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 4 Aug 2016 09:38:37 +0100
Subject: [PATCH 5637/6219] [kaltura] extract subtitles and reduce requests

---
 youtube_dl/extractor/kaltura.py | 43 +++++++++++++++++++++++++++------
 1 file changed, 36 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/kaltura.py b/youtube_dl/extractor/kaltura.py
index 1729f5bfb..9930ea710 100644
--- a/youtube_dl/extractor/kaltura.py
+++ b/youtube_dl/extractor/kaltura.py
@@ -62,6 +62,11 @@ class KalturaIE(InfoExtractor):
         {
             'url': 'https://cdnapisec.kaltura.com/html5/html5lib/v2.30.2/mwEmbedFrame.php/p/1337/uiconf_id/20540612/entry_id/1_sf5ovm7u?wid=_243342',
             'only_matching': True,
+        },
+        {
+            # video with subtitles
+            'url': 'kaltura:111032:1_cw786r8q',
+            'only_matching': True,
         }
     ]
 
@@ -130,7 +135,6 @@ class KalturaIE(InfoExtractor):
             video_id, actions, service_url, note='Downloading Kaltura signature')['ks']
 
     def _get_video_info(self, video_id, partner_id, service_url=None):
-        signature = self._get_kaltura_signature(video_id, partner_id, service_url)
         actions = [
             {
                 'action': 'null',
@@ -138,18 +142,30 @@ class KalturaIE(InfoExtractor):
                 'clientTag': 'kdp:v3.8.5',
                 'format': 1,  # JSON, 2 = XML, 3 = PHP
                 'service': 'multirequest',
-                'ks': signature,
+            },
+            {
+                'expiry': 86400,
+                'service': 'session',
+                'action': 'startWidgetSession',
+                'widgetId': '_%s' % partner_id,
             },
             {
                 'action': 'get',
                 'entryId': video_id,
                 'service': 'baseentry',
-                'version': '-1',
+                'ks': '{1:result:ks}',
             },
             {
                 'action': 'getbyentryid',
                 'entryId': video_id,
                 'service': 'flavorAsset',
+                'ks': '{1:result:ks}',
+            },
+            {
+                'action': 'list',
+                'filter:entryIdEqual': video_id,
+                'service': 'caption_captionasset',
+                'ks': '{1:result:ks}',
             },
         ]
         return self._kaltura_api_call(
@@ -161,8 +177,9 @@ class KalturaIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         partner_id, entry_id = mobj.group('partner_id', 'id')
         ks = None
+        captions = None
         if partner_id and entry_id:
-            info, flavor_assets = self._get_video_info(entry_id, partner_id, smuggled_data.get('service_url'))
+            _, info, flavor_assets, captions = self._get_video_info(entry_id, partner_id, smuggled_data.get('service_url'))
         else:
             path, query = mobj.group('path', 'query')
             if not path and not query:
@@ -181,7 +198,7 @@ class KalturaIE(InfoExtractor):
                 raise ExtractorError('Invalid URL', expected=True)
             if 'entry_id' in params:
                 entry_id = params['entry_id'][0]
-                info, flavor_assets = self._get_video_info(entry_id, partner_id)
+                _, info, flavor_assets, captions = self._get_video_info(entry_id, partner_id)
             elif 'uiconf_id' in params and 'flashvars[referenceId]' in params:
                 reference_id = params['flashvars[referenceId]'][0]
                 webpage = self._download_webpage(url, reference_id)
@@ -217,7 +234,7 @@ class KalturaIE(InfoExtractor):
         formats = []
         for f in flavor_assets:
             # Continue if asset is not ready
-            if f['status'] != 2:
+            if f.get('status') != 2:
                 continue
             video_url = sign_url(
                 '%s/flavorId/%s' % (data_url, f['id']))
@@ -240,13 +257,25 @@ class KalturaIE(InfoExtractor):
                 m3u8_url, entry_id, 'mp4', 'm3u8_native',
                 m3u8_id='hls', fatal=False))
 
-        self._check_formats(formats, entry_id)
         self._sort_formats(formats)
 
+        subtitles = {}
+        if captions:
+            for caption in captions.get('objects', []):
+                print(caption)
+                # Continue if caption is not ready
+                if f.get('status') != 2:
+                    continue
+                subtitles.setdefault(caption.get('languageCode') or caption.get('language'), []).append({
+                    'url': '%s/api_v3/service/caption_captionasset/action/serve/captionAssetId/%s' % (self._SERVICE_URL, caption['id']),
+                    'ext': caption.get('fileExt'),
+                })
+
         return {
             'id': entry_id,
             'title': info['name'],
             'formats': formats,
+            'subtitles': subtitles,
             'description': clean_html(info.get('description')),
             'thumbnail': info.get('thumbnailUrl'),
             'duration': info.get('duration'),

From 1891ea2d760a49d356a472516db40bad8309ef3c Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 4 Aug 2016 12:18:10 +0100
Subject: [PATCH 5638/6219] [nationalgeographic] Add support for National
 Geographic Episode Guide

---
 youtube_dl/extractor/nationalgeographic.py | 54 ++++++++++++++++++++--
 1 file changed, 49 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/nationalgeographic.py b/youtube_dl/extractor/nationalgeographic.py
index e717abb9f..fe43d4bc8 100644
--- a/youtube_dl/extractor/nationalgeographic.py
+++ b/youtube_dl/extractor/nationalgeographic.py
@@ -1,16 +1,19 @@
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from .theplatform import ThePlatformIE
 from ..utils import (
     smuggle_url,
     url_basename,
     update_url_query,
+    get_element_by_class,
 )
 
 
-class NationalGeographicIE(InfoExtractor):
-    IE_NAME = 'natgeo'
+class NationalGeographicVideoIE(InfoExtractor):
+    IE_NAME = 'natgeo:video'
     _VALID_URL = r'https?://video\.nationalgeographic\.com/.*?'
 
     _TESTS = [
@@ -62,9 +65,9 @@ class NationalGeographicIE(InfoExtractor):
         }
 
 
-class NationalGeographicChannelIE(ThePlatformIE):
-    IE_NAME = 'natgeo:channel'
-    _VALID_URL = r'https?://channel\.nationalgeographic\.com/(?:wild/)?[^/]+/videos/(?P<id>[^/?]+)'
+class NationalGeographicIE(ThePlatformIE):
+    IE_NAME = 'natgeo'
+    _VALID_URL = r'https?://channel\.nationalgeographic\.com/(?:wild/)?[^/]+/(?:videos|episodes)/(?P<id>[^/?]+)'
 
     _TESTS = [
         {
@@ -95,6 +98,10 @@ class NationalGeographicChannelIE(ThePlatformIE):
             },
             'add_ie': ['ThePlatform'],
         },
+        {
+            'url': 'http://channel.nationalgeographic.com/the-story-of-god-with-morgan-freeman/episodes/the-power-of-miracles/',
+            'only_matching': True,
+        }
     ]
 
     def _real_extract(self, url):
@@ -122,3 +129,40 @@ class NationalGeographicChannelIE(ThePlatformIE):
                 {'force_smil_url': True}),
             'display_id': display_id,
         }
+
+
+class NationalGeographicEpisodeGuideIE(ThePlatformIE):
+    IE_NAME = 'natgeo:episodeguide'
+    _VALID_URL = r'https?://channel\.nationalgeographic\.com/(?:wild/)?(?P<id>[^/]+)/episode-guide'
+    _TESTS = [
+        {
+            'url': 'http://channel.nationalgeographic.com/the-story-of-god-with-morgan-freeman/episode-guide/',
+            'info_dict': {
+                'id': 'the-story-of-god-with-morgan-freeman-season-1',
+                'title': 'The Story of God with Morgan Freeman - Season 1',
+            },
+            'playlist_mincount': 6,
+        },
+        {
+            'url': 'http://channel.nationalgeographic.com/underworld-inc/episode-guide/?s=2',
+            'info_dict': {
+                'id': 'underworld-inc-season-2',
+                'title': 'Underworld, Inc. - Season 2',
+            },
+            'playlist_mincount': 7,
+        },
+    ]
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+        show = get_element_by_class('show', webpage)
+        selected_season = self._search_regex(
+            r'<div[^>]+class="select-seasons[^"]*".*?<a[^>]*>(.*?)</a>',
+            webpage, 'selected season')
+        entries = [
+            self.url_result(self._proto_relative_url(url), 'NationalGeographic')
+            for url in re.findall('(?s)<div[^>]+class="col-inner"[^>]*?>.*?<a[^>]+href="([^"]+)"', webpage)]
+        return self.playlist_result(
+            entries, '%s-%s' % (display_id, selected_season.lower().replace(' ', '-')),
+            '%s - %s' % (show, selected_season))

From 3c2c3af059d99e6795d1944a55d6601b5959871b Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 4 Aug 2016 12:20:56 +0100
Subject: [PATCH 5639/6219] [extractors] change imports for national geographic
 extractors

---
 youtube_dl/extractor/extractors.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 909f7d25c..28858cfea 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -491,8 +491,9 @@ from .myvi import MyviIE
 from .myvideo import MyVideoIE
 from .myvidster import MyVidsterIE
 from .nationalgeographic import (
+    NationalGeographicVideoIE,
     NationalGeographicIE,
-    NationalGeographicChannelIE,
+    NationalGeographicEpisodeGuideIE,
 )
 from .naver import NaverIE
 from .nba import NBAIE

From 14704aeff6eeee4357e3a26f83432ff908db64fc Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 4 Aug 2016 14:54:34 +0100
Subject: [PATCH 5640/6219] [kaltura] remove debugging line

---
 youtube_dl/extractor/kaltura.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/kaltura.py b/youtube_dl/extractor/kaltura.py
index 9930ea710..ddf1165ff 100644
--- a/youtube_dl/extractor/kaltura.py
+++ b/youtube_dl/extractor/kaltura.py
@@ -262,7 +262,6 @@ class KalturaIE(InfoExtractor):
         subtitles = {}
         if captions:
             for caption in captions.get('objects', []):
-                print(caption)
                 # Continue if caption is not ready
                 if f.get('status') != 2:
                     continue

From 2396062c747ee81420dac0eac914d531ec8df910 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 4 Aug 2016 16:21:27 +0100
Subject: [PATCH 5641/6219] [5min] delegate extraction to AolIE

recently the 5min SenseHandler request return
HTTP Error 503: Service Unavailable error
---
 youtube_dl/extractor/fivemin.py | 135 +++++---------------------------
 1 file changed, 19 insertions(+), 116 deletions(-)

diff --git a/youtube_dl/extractor/fivemin.py b/youtube_dl/extractor/fivemin.py
index 6b8345416..2f882cbcb 100644
--- a/youtube_dl/extractor/fivemin.py
+++ b/youtube_dl/extractor/fivemin.py
@@ -1,24 +1,11 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
-from ..compat import (
-    compat_parse_qs,
-    compat_urllib_parse_urlencode,
-    compat_urllib_parse_urlparse,
-    compat_urlparse,
-)
-from ..utils import (
-    ExtractorError,
-    parse_duration,
-    replace_extension,
-)
 
 
 class FiveMinIE(InfoExtractor):
     IE_NAME = '5min'
-    _VALID_URL = r'(?:5min:(?P<id>\d+)(?::(?P<sid>\d+))?|https?://[^/]*?5min\.com/Scripts/PlayerSeed\.js\?(?P<query>.*))'
+    _VALID_URL = r'(?:5min:|(?:https?://(?:[^/]*?5min\.com|delivery\.vidible\.tv/aol)/(?:(?:Scripts/PlayerSeed\.js|playerseed/?)?\?.*?playList=)?)(?P<id>\d+)'
 
     _TESTS = [
         {
@@ -29,8 +16,16 @@ class FiveMinIE(InfoExtractor):
                 'id': '518013791',
                 'ext': 'mp4',
                 'title': 'iPad Mini with Retina Display Review',
+                'description': 'iPad mini with Retina Display review',
                 'duration': 177,
+                'uploader': 'engadget',
+                'upload_date': '20131115',
+                'timestamp': 1384515288,
             },
+            'params': {
+                # m3u8 download
+                'skip_download': True,
+            }
         },
         {
             # From http://on.aol.com/video/how-to-make-a-next-level-fruit-salad-518086247
@@ -44,108 +39,16 @@ class FiveMinIE(InfoExtractor):
             },
             'skip': 'no longer available',
         },
+        {
+            'url': 'http://embed.5min.com/518726732/',
+            'only_matching': True,
+        },
+        {
+            'url': 'http://delivery.vidible.tv/aol?playList=518013791',
+            'only_matching': True,
+        }
     ]
-    _ERRORS = {
-        'ErrorVideoNotExist': 'We\'re sorry, but the video you are trying to watch does not exist.',
-        'ErrorVideoNoLongerAvailable': 'We\'re sorry, but the video you are trying to watch is no longer available.',
-        'ErrorVideoRejected': 'We\'re sorry, but the video you are trying to watch has been removed.',
-        'ErrorVideoUserNotGeo': 'We\'re sorry, but the video you are trying to watch cannot be viewed from your current location.',
-        'ErrorVideoLibraryRestriction': 'We\'re sorry, but the video you are trying to watch is currently unavailable for viewing at this domain.',
-        'ErrorExposurePermission': 'We\'re sorry, but the video you are trying to watch is currently unavailable for viewing at this domain.',
-    }
-    _QUALITIES = {
-        1: {
-            'width': 640,
-            'height': 360,
-        },
-        2: {
-            'width': 854,
-            'height': 480,
-        },
-        4: {
-            'width': 1280,
-            'height': 720,
-        },
-        8: {
-            'width': 1920,
-            'height': 1080,
-        },
-        16: {
-            'width': 640,
-            'height': 360,
-        },
-        32: {
-            'width': 854,
-            'height': 480,
-        },
-        64: {
-            'width': 1280,
-            'height': 720,
-        },
-        128: {
-            'width': 640,
-            'height': 360,
-        },
-    }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        sid = mobj.group('sid')
-
-        if mobj.group('query'):
-            qs = compat_parse_qs(mobj.group('query'))
-            if not qs.get('playList'):
-                raise ExtractorError('Invalid URL', expected=True)
-            video_id = qs['playList'][0]
-            if qs.get('sid'):
-                sid = qs['sid'][0]
-
-        embed_url = 'https://embed.5min.com/playerseed/?playList=%s' % video_id
-        if not sid:
-            embed_page = self._download_webpage(embed_url, video_id,
-                                                'Downloading embed page')
-            sid = self._search_regex(r'sid=(\d+)', embed_page, 'sid')
-
-        response = self._download_json(
-            'https://syn.5min.com/handlers/SenseHandler.ashx?' +
-            compat_urllib_parse_urlencode({
-                'func': 'GetResults',
-                'playlist': video_id,
-                'sid': sid,
-                'isPlayerSeed': 'true',
-                'url': embed_url,
-            }),
-            video_id)
-        if not response['success']:
-            raise ExtractorError(
-                '%s said: %s' % (
-                    self.IE_NAME,
-                    self._ERRORS.get(response['errorMessage'], response['errorMessage'])),
-                expected=True)
-        info = response['binding'][0]
-
-        formats = []
-        parsed_video_url = compat_urllib_parse_urlparse(compat_parse_qs(
-            compat_urllib_parse_urlparse(info['EmbededURL']).query)['videoUrl'][0])
-        for rendition in info['Renditions']:
-            if rendition['RenditionType'] == 'aac' or rendition['RenditionType'] == 'm3u8':
-                continue
-            else:
-                rendition_url = compat_urlparse.urlunparse(parsed_video_url._replace(path=replace_extension(parsed_video_url.path.replace('//', '/%s/' % rendition['ID']), rendition['RenditionType'])))
-                quality = self._QUALITIES.get(rendition['ID'], {})
-                formats.append({
-                    'format_id': '%s-%d' % (rendition['RenditionType'], rendition['ID']),
-                    'url': rendition_url,
-                    'width': quality.get('width'),
-                    'height': quality.get('height'),
-                })
-        self._sort_formats(formats)
-
-        return {
-            'id': video_id,
-            'title': info['Title'],
-            'thumbnail': info.get('ThumbURL'),
-            'duration': parse_duration(info.get('Duration')),
-            'formats': formats,
-        }
+        video_id = self._match_id(url)
+        return self.url_result('aol-video:%s' % video_id)

From 52e7fcfeb794ca0e50e312357d60234bcca5118c Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 4 Aug 2016 16:34:47 +0100
Subject: [PATCH 5642/6219] [engadget] Relax _VALID_URL

---
 youtube_dl/extractor/engadget.py | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/engadget.py b/youtube_dl/extractor/engadget.py
index e5e57d485..a39e9010d 100644
--- a/youtube_dl/extractor/engadget.py
+++ b/youtube_dl/extractor/engadget.py
@@ -4,9 +4,10 @@ from .common import InfoExtractor
 
 
 class EngadgetIE(InfoExtractor):
-    _VALID_URL = r'https?://www.engadget.com/video/(?P<id>\d+)'
+    _VALID_URL = r'https?://www.engadget.com/video/(?P<id>[^/?#]+)'
 
-    _TEST = {
+    _TESTS = [{
+        # video with 5min ID
         'url': 'http://www.engadget.com/video/518153925/',
         'md5': 'c6820d4828a5064447a4d9fc73f312c9',
         'info_dict': {
@@ -15,8 +16,12 @@ class EngadgetIE(InfoExtractor):
             'title': 'Samsung Galaxy Tab Pro 8.4 Review',
         },
         'add_ie': ['FiveMin'],
-    }
+    }, {
+        # video with vidible ID
+        'url': 'https://www.engadget.com/video/57a28462134aa15a39f0421a/',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        return self.url_result('5min:%s' % video_id)
+        return self.url_result('aol-video:%s' % video_id)

From 8895be01fc2d3774aa0b9b000af20c22903d3391 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 4 Aug 2016 16:55:12 +0100
Subject: [PATCH 5643/6219] [5min] fix _VALID_URL

---
 youtube_dl/extractor/fivemin.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/fivemin.py b/youtube_dl/extractor/fivemin.py
index 2f882cbcb..ab82701c2 100644
--- a/youtube_dl/extractor/fivemin.py
+++ b/youtube_dl/extractor/fivemin.py
@@ -5,7 +5,7 @@ from .common import InfoExtractor
 
 class FiveMinIE(InfoExtractor):
     IE_NAME = '5min'
-    _VALID_URL = r'(?:5min:|(?:https?://(?:[^/]*?5min\.com|delivery\.vidible\.tv/aol)/(?:(?:Scripts/PlayerSeed\.js|playerseed/?)?\?.*?playList=)?)(?P<id>\d+)'
+    _VALID_URL = r'(?:5min:|https?://(?:[^/]*?5min\.com|delivery\.vidible\.tv/aol)/(?:(?:Scripts/PlayerSeed\.js|playerseed/?)?\?.*?playList=)?)(?P<id>\d+)'
 
     _TESTS = [
         {

From 8a00ea567b88ea240c6fe672387cc5cc482495fe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 4 Aug 2016 23:21:04 +0700
Subject: [PATCH 5644/6219] [natgeo:episodeguide] Do not shadow url from outer
 scope

---
 youtube_dl/extractor/nationalgeographic.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/nationalgeographic.py b/youtube_dl/extractor/nationalgeographic.py
index fe43d4bc8..4bffcf704 100644
--- a/youtube_dl/extractor/nationalgeographic.py
+++ b/youtube_dl/extractor/nationalgeographic.py
@@ -161,8 +161,8 @@ class NationalGeographicEpisodeGuideIE(ThePlatformIE):
             r'<div[^>]+class="select-seasons[^"]*".*?<a[^>]*>(.*?)</a>',
             webpage, 'selected season')
         entries = [
-            self.url_result(self._proto_relative_url(url), 'NationalGeographic')
-            for url in re.findall('(?s)<div[^>]+class="col-inner"[^>]*?>.*?<a[^>]+href="([^"]+)"', webpage)]
+            self.url_result(self._proto_relative_url(entry_url), 'NationalGeographic')
+            for entry_url in re.findall('(?s)<div[^>]+class="col-inner"[^>]*?>.*?<a[^>]+href="([^"]+)"', webpage)]
         return self.playlist_result(
             entries, '%s-%s' % (display_id, selected_season.lower().replace(' ', '-')),
             '%s - %s' % (show, selected_season))

From 4f427c4be860c582ca72dd4be64d45b54499232c Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 4 Aug 2016 18:28:49 +0100
Subject: [PATCH 5645/6219] [condenast] improve extraction

---
 youtube_dl/extractor/condenast.py | 111 ++++++++++++++++++------------
 1 file changed, 66 insertions(+), 45 deletions(-)

diff --git a/youtube_dl/extractor/condenast.py b/youtube_dl/extractor/condenast.py
index e8f2b5a07..976a0e89d 100644
--- a/youtube_dl/extractor/condenast.py
+++ b/youtube_dl/extractor/condenast.py
@@ -5,13 +5,17 @@ import re
 
 from .common import InfoExtractor
 from ..compat import (
-    compat_urllib_parse_urlencode,
     compat_urllib_parse_urlparse,
     compat_urlparse,
 )
 from ..utils import (
     orderedSet,
     remove_end,
+    extract_attributes,
+    mimetype2ext,
+    determine_ext,
+    int_or_none,
+    parse_iso8601,
 )
 
 
@@ -58,6 +62,9 @@ class CondeNastIE(InfoExtractor):
             'ext': 'mp4',
             'title': '3D Printed Speakers Lit With LED',
             'description': 'Check out these beautiful 3D printed LED speakers.  You can\'t actually buy them, but LumiGeek is working on a board that will let you make you\'re own.',
+            'uploader': 'wired',
+            'upload_date': '20130314',
+            'timestamp': 1363219200,
         }
     }, {
         # JS embed
@@ -67,70 +74,84 @@ class CondeNastIE(InfoExtractor):
             'id': '55f9cf8b61646d1acf00000c',
             'ext': 'mp4',
             'title': '3D printed TSA Travel Sentry keys really do open TSA locks',
+            'uploader': 'arstechnica',
+            'upload_date': '20150916',
+            'timestamp': 1442434955,
         }
     }]
 
     def _extract_series(self, url, webpage):
-        title = self._html_search_regex(r'<div class="cne-series-info">.*?<h1>(.+?)</h1>',
-                                        webpage, 'series title', flags=re.DOTALL)
+        title = self._html_search_regex(
+            r'(?s)<div class="cne-series-info">.*?<h1>(.+?)</h1>',
+            webpage, 'series title')
         url_object = compat_urllib_parse_urlparse(url)
         base_url = '%s://%s' % (url_object.scheme, url_object.netloc)
-        m_paths = re.finditer(r'<p class="cne-thumb-title">.*?<a href="(/watch/.+?)["\?]',
-                              webpage, flags=re.DOTALL)
+        m_paths = re.finditer(
+            r'(?s)<p class="cne-thumb-title">.*?<a href="(/watch/.+?)["\?]', webpage)
         paths = orderedSet(m.group(1) for m in m_paths)
         build_url = lambda path: compat_urlparse.urljoin(base_url, path)
         entries = [self.url_result(build_url(path), 'CondeNast') for path in paths]
         return self.playlist_result(entries, playlist_title=title)
 
     def _extract_video(self, webpage, url_type):
-        if url_type != 'embed':
-            description = self._html_search_regex(
-                [
-                    r'<div class="cne-video-description">(.+?)</div>',
-                    r'<div class="video-post-content">(.+?)</div>',
-                ],
-                webpage, 'description', fatal=False, flags=re.DOTALL)
+        query = {}
+        params = self._search_regex(
+            r'(?s)var params = {(.+?)}[;,]', webpage, 'player params', default=None)
+        if params:
+            query.update({
+                'videoId': self._search_regex(r'videoId: [\'"](.+?)[\'"]', params, 'video id'),
+                'playerId': self._search_regex(r'playerId: [\'"](.+?)[\'"]', params, 'player id'),
+                'target': self._search_regex(r'target: [\'"](.+?)[\'"]', params, 'target'),
+            })
         else:
-            description = None
-        params = self._search_regex(r'var params = {(.+?)}[;,]', webpage,
-                                    'player params', flags=re.DOTALL)
-        video_id = self._search_regex(r'videoId: [\'"](.+?)[\'"]', params, 'video id')
-        player_id = self._search_regex(r'playerId: [\'"](.+?)[\'"]', params, 'player id')
-        target = self._search_regex(r'target: [\'"](.+?)[\'"]', params, 'target')
-        data = compat_urllib_parse_urlencode({'videoId': video_id,
-                                              'playerId': player_id,
-                                              'target': target,
-                                              })
-        base_info_url = self._search_regex(r'url = [\'"](.+?)[\'"][,;]',
-                                           webpage, 'base info url',
-                                           default='http://player.cnevids.com/player/loader.js?')
-        info_url = base_info_url + data
-        info_page = self._download_webpage(info_url, video_id,
-                                           'Downloading video info')
-        video_info = self._search_regex(r'var\s+video\s*=\s*({.+?});', info_page, 'video info')
-        video_info = self._parse_json(video_info, video_id)
+            params = extract_attributes(self._search_regex(
+                r'(<[^>]+data-js="video-player"[^>]+>)',
+                webpage, 'player params element'))
+            query.update({
+                'videoId': params['data-video'],
+                'playerId': params['data-player'],
+                'target': params['id'],
+            })
+        video_id = query['videoId']
+        info_page = self._download_webpage(
+            'http://player.cnevids.com/player/video.js',
+            video_id, 'Downloading video info', query=query)
+        video_info = self._parse_json(self._search_regex(
+            r'loadCallback\(({.+})\)', info_page, 'video info'), video_id)['video']
+        title = video_info['title']
 
-        formats = [{
-            'format_id': '%s-%s' % (fdata['type'].split('/')[-1], fdata['quality']),
-            'url': fdata['src'],
-            'ext': fdata['type'].split('/')[-1],
-            'quality': 1 if fdata['quality'] == 'high' else 0,
-        } for fdata in video_info['sources'][0]]
+        formats = []
+        for fdata in video_info.get('sources', [{}])[0]:
+            src = fdata.get('src')
+            if not src:
+                continue
+            ext = mimetype2ext(fdata.get('type')) or determine_ext(src)
+            quality = fdata.get('quality')
+            formats.append({
+                'format_id': ext + ('-%s' % quality if quality else ''),
+                'url': src,
+                'ext': ext,
+                'quality': 1 if quality == 'high' else 0,
+            })
         self._sort_formats(formats)
 
-        return {
+        info = self._search_json_ld(webpage, video_id) if url_type != 'embed' else {}
+        info.update({
             'id': video_id,
             'formats': formats,
-            'title': video_info['title'],
-            'thumbnail': video_info['poster_frame'],
-            'description': description,
-        }
+            'title': title,
+            'thumbnail': video_info.get('poster_frame'),
+            'uploader': video_info.get('brand'),
+            'duration': int_or_none(video_info.get('duration')),
+            'tags': video_info.get('tags'),
+            'series': video_info.get('series_title'),
+            'season': video_info.get('season_title'),
+            'timestamp': parse_iso8601(video_info.get('premiere_date')),
+        })
+        return info
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        site = mobj.group('site')
-        url_type = mobj.group('type')
-        item_id = mobj.group('id')
+        site, url_type, item_id = re.match(self._VALID_URL, url).groups()
 
         # Convert JS embed to regular embed
         if url_type == 'embedjs':

From b02b960c6bba834d9e7199ac53430c7933079dc8 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 4 Aug 2016 21:42:22 +0100
Subject: [PATCH 5646/6219] [naver] improve extraction(closes #8096)

---
 youtube_dl/extractor/naver.py | 95 ++++++++++++++++++++---------------
 1 file changed, 54 insertions(+), 41 deletions(-)

diff --git a/youtube_dl/extractor/naver.py b/youtube_dl/extractor/naver.py
index 6d6f69b44..60a4872be 100644
--- a/youtube_dl/extractor/naver.py
+++ b/youtube_dl/extractor/naver.py
@@ -4,12 +4,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse_urlencode,
-    compat_urlparse,
-)
 from ..utils import (
     ExtractorError,
+    int_or_none,
+    update_url_query,
 )
 
 
@@ -23,7 +21,6 @@ class NaverIE(InfoExtractor):
             'ext': 'mp4',
             'title': '[9월 모의고사 해설강의][수학_김상희] 수학 A형 16~20번',
             'description': '합격불변의 법칙 메가스터디 | 메가스터디 수학 김상희 선생님이 9월 모의고사 수학A형 16번에서 20번까지 해설강의를 공개합니다.',
-            'upload_date': '20130903',
         },
     }, {
         'url': 'http://tvcast.naver.com/v/395837',
@@ -33,7 +30,6 @@ class NaverIE(InfoExtractor):
             'ext': 'mp4',
             'title': '9년이 지나도 아픈 기억, 전효성의 아버지',
             'description': 'md5:5bf200dcbf4b66eb1b350d1eb9c753f7',
-            'upload_date': '20150519',
         },
         'skip': 'Georestricted',
     }]
@@ -51,48 +47,65 @@ class NaverIE(InfoExtractor):
             if error:
                 raise ExtractorError(error, expected=True)
             raise ExtractorError('couldn\'t extract vid and key')
-        vid = m_id.group(1)
-        key = m_id.group(2)
-        query = compat_urllib_parse_urlencode({'vid': vid, 'inKey': key, })
-        query_urls = compat_urllib_parse_urlencode({
-            'masterVid': vid,
-            'protocol': 'p2p',
-            'inKey': key,
+        video_data = self._download_json('http://play.rmcnmv.naver.com/vod/play/v2.0/' + m_id.group(1), video_id, query={
+            'key': m_id.group(2),
         })
-        info = self._download_xml(
-            'http://serviceapi.rmcnmv.naver.com/flash/videoInfo.nhn?' + query,
-            video_id, 'Downloading video info')
-        urls = self._download_xml(
-            'http://serviceapi.rmcnmv.naver.com/flash/playableEncodingOption.nhn?' + query_urls,
-            video_id, 'Downloading video formats info')
-
+        meta = video_data['meta']
+        title = meta['subject']
         formats = []
-        for format_el in urls.findall('EncodingOptions/EncodingOption'):
-            domain = format_el.find('Domain').text
-            uri = format_el.find('uri').text
-            f = {
-                'url': compat_urlparse.urljoin(domain, uri),
-                'ext': 'mp4',
-                'width': int(format_el.find('width').text),
-                'height': int(format_el.find('height').text),
-            }
-            if domain.startswith('rtmp'):
-                # urlparse does not support custom schemes
-                # https://bugs.python.org/issue18828
-                f.update({
-                    'url': domain + uri,
-                    'ext': 'flv',
-                    'rtmp_protocol': '1',  # rtmpt
+
+        def extract_formats(streams, stream_type, query={}):
+            for stream in streams:
+                stream_url = stream.get('source')
+                if not stream_url:
+                    continue
+                stream_url = update_url_query(stream_url, query)
+                encoding_option = stream.get('encodingOption', {})
+                bitrate = stream.get('bitrate', {})
+                formats.append({
+                    'format_id': '%s_%s' % (stream.get('type') or stream_type, encoding_option.get('id') or encoding_option.get('name')),
+                    'url': stream_url,
+                    'width': int_or_none(encoding_option.get('width')),
+                    'height': int_or_none(encoding_option.get('height')),
+                    'vbr': int_or_none(bitrate.get('video')),
+                    'abr': int_or_none(bitrate.get('audio')),
+                    'filesize': int_or_none(stream.get('size')),
+                    'protocol': 'm3u8_native' if stream_type == 'HLS' else None,
                 })
-            formats.append(f)
+
+        extract_formats(video_data.get('videos', {}).get('list', []), 'H264')
+        for stream_set in video_data.get('streams', []):
+            query = {}
+            for param in stream_set.get('keys', []):
+                query[param['name']] = param['value']
+            stream_type = stream_set.get('type')
+            videos = stream_set.get('videos')
+            if videos:
+                extract_formats(videos, stream_type, query)
+            elif stream_type == 'HLS':
+                stream_url = stream_set.get('source')
+                if not stream_url:
+                    continue
+                formats.extend(self._extract_m3u8_formats(
+                    update_url_query(stream_url, query), video_id,
+                    'mp4', 'm3u8_native', m3u8_id=stream_type, fatal=False))
         self._sort_formats(formats)
 
+        subtitles = {}
+        for caption in video_data.get('captions', {}).get('list', []):
+            caption_url = caption.get('source')
+            if not caption_url:
+                continue
+            subtitles.setdefault(caption.get('language') or caption.get('locale'), []).append({
+                'url': caption_url,
+            })
+
         return {
             'id': video_id,
-            'title': info.find('Subject').text,
+            'title': title,
             'formats': formats,
+            'subtitles': subtitles,
             'description': self._og_search_description(webpage),
-            'thumbnail': self._og_search_thumbnail(webpage),
-            'upload_date': info.find('WriteDate').text.replace('.', ''),
-            'view_count': int(info.find('PlayCount').text),
+            'thumbnail': meta.get('cover', {}).get('source') or self._og_search_thumbnail(webpage),
+            'view_count': int_or_none(meta.get('count')),
         }

From 7dc2a74e0ac9cfa74cc9de6f586ffd5cc8bac0d9 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 5 Aug 2016 11:41:55 +0800
Subject: [PATCH 5647/6219] [utils] Fix unified_timestamp for formats parsed by
 parsedate_tz()

---
 test/test_utils.py  | 1 +
 youtube_dl/utils.py | 6 +++---
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 2273b5a10..5a2ae4a1e 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -308,6 +308,7 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(unified_timestamp('25-09-2014'), 1411603200)
         self.assertEqual(unified_timestamp('27.02.2016 17:30'), 1456594200)
         self.assertEqual(unified_timestamp('UNKNOWN DATE FORMAT'), None)
+        self.assertEqual(unified_timestamp('May 16, 2016 11:15 PM'), 1463440500)
 
     def test_determine_ext(self):
         self.assertEqual(determine_ext('http://example.com/foo/bar.mp4/?download'), 'mp4')
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index f5cd6819b..97ddd9883 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1101,7 +1101,7 @@ def unified_timestamp(date_str, day_first=True):
 
     date_str = date_str.replace(',', ' ')
 
-    pm_delta = datetime.timedelta(hours=12 if re.search(r'(?i)PM', date_str) else 0)
+    pm_delta = 12 if re.search(r'(?i)PM', date_str) else 0
     timezone, date_str = extract_timezone(date_str)
 
     # Remove AM/PM + timezone
@@ -1109,13 +1109,13 @@ def unified_timestamp(date_str, day_first=True):
 
     for expression in date_formats(day_first):
         try:
-            dt = datetime.datetime.strptime(date_str, expression) - timezone + pm_delta
+            dt = datetime.datetime.strptime(date_str, expression) - timezone + datetime.timedelta(hours=pm_delta)
             return calendar.timegm(dt.timetuple())
         except ValueError:
             pass
     timetuple = email.utils.parsedate_tz(date_str)
     if timetuple:
-        return calendar.timegm(timetuple.timetuple())
+        return calendar.timegm(timetuple) + pm_delta * 3600
 
 
 def determine_ext(url, default_ext='unknown_video'):

From 962250f7eaf648e998e66e1d6aa5c2b4e018cd27 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 5 Aug 2016 11:44:50 +0800
Subject: [PATCH 5648/6219] [cbslocal] Fix timestamp parsing (closes #10213)

---
 ChangeLog                        | 7 +++++++
 youtube_dl/extractor/cbslocal.py | 9 ++-------
 2 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index f3c752e66..2d021ddf4 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Fixed/improved extractors
+- [cbslocal] Fix timestamp parsing (#10213)
+
+
 version 2016.08.01
 
 Fixed/improved extractors
@@ -7,6 +13,7 @@ Fixed/improved extractors
 - [safari] Relax regular expressions for URL matching (#10202)
 - [cwtv] Add support for cwtvpr.com (#10196)
 
+
 version 2016.07.30
 
 Fixed/improved extractors
diff --git a/youtube_dl/extractor/cbslocal.py b/youtube_dl/extractor/cbslocal.py
index 74adb38a6..008c5fe32 100644
--- a/youtube_dl/extractor/cbslocal.py
+++ b/youtube_dl/extractor/cbslocal.py
@@ -1,12 +1,10 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import calendar
-import datetime
-
 from .anvato import AnvatoIE
 from .sendtonews import SendtoNewsIE
 from ..compat import compat_urlparse
+from ..utils import unified_timestamp
 
 
 class CBSLocalIE(AnvatoIE):
@@ -71,10 +69,7 @@ class CBSLocalIE(AnvatoIE):
 
         time_str = self._html_search_regex(
             r'class="entry-date">([^<]+)<', webpage, 'released date', fatal=False)
-        timestamp = None
-        if time_str:
-            timestamp = calendar.timegm(datetime.datetime.strptime(
-                time_str, '%b %d, %Y %I:%M %p').timetuple())
+        timestamp = unified_timestamp(time_str)
 
         info_dict.update({
             'display_id': display_id,

From f65dc41b72d3449d9faf1a2fcd0a2e32befa3607 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 5 Aug 2016 08:11:44 +0100
Subject: [PATCH 5649/6219] [naver] extract upload date

---
 youtube_dl/extractor/naver.py | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/naver.py b/youtube_dl/extractor/naver.py
index 60a4872be..0891d2772 100644
--- a/youtube_dl/extractor/naver.py
+++ b/youtube_dl/extractor/naver.py
@@ -21,6 +21,7 @@ class NaverIE(InfoExtractor):
             'ext': 'mp4',
             'title': '[9월 모의고사 해설강의][수학_김상희] 수학 A형 16~20번',
             'description': '합격불변의 법칙 메가스터디 | 메가스터디 수학 김상희 선생님이 9월 모의고사 수학A형 16번에서 20번까지 해설강의를 공개합니다.',
+            'upload_date': '20130903',
         },
     }, {
         'url': 'http://tvcast.naver.com/v/395837',
@@ -30,6 +31,7 @@ class NaverIE(InfoExtractor):
             'ext': 'mp4',
             'title': '9년이 지나도 아픈 기억, 전효성의 아버지',
             'description': 'md5:5bf200dcbf4b66eb1b350d1eb9c753f7',
+            'upload_date': '20150519',
         },
         'skip': 'Georestricted',
     }]
@@ -47,9 +49,11 @@ class NaverIE(InfoExtractor):
             if error:
                 raise ExtractorError(error, expected=True)
             raise ExtractorError('couldn\'t extract vid and key')
-        video_data = self._download_json('http://play.rmcnmv.naver.com/vod/play/v2.0/' + m_id.group(1), video_id, query={
-            'key': m_id.group(2),
-        })
+        video_data = self._download_json(
+            'http://play.rmcnmv.naver.com/vod/play/v2.0/' + m_id.group(1),
+            video_id, query={
+                'key': m_id.group(2),
+            })
         meta = video_data['meta']
         title = meta['subject']
         formats = []
@@ -100,6 +104,12 @@ class NaverIE(InfoExtractor):
                 'url': caption_url,
             })
 
+        upload_date = self._search_regex(
+            r'<span[^>]+class="date".*?(\d{4}\.\d{2}\.\d{2})',
+            webpage, 'upload date', fatal=False)
+        if upload_date:
+            upload_date = upload_date.replace('.', '')
+
         return {
             'id': video_id,
             'title': title,
@@ -108,4 +118,5 @@ class NaverIE(InfoExtractor):
             'description': self._og_search_description(webpage),
             'thumbnail': meta.get('cover', {}).get('source') or self._og_search_thumbnail(webpage),
             'view_count': int_or_none(meta.get('count')),
+            'upload_date': upload_date,
         }

From fdd0b8f8e044adf61da95fe85bc9414724f1d835 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 5 Aug 2016 09:44:15 +0100
Subject: [PATCH 5650/6219] [tvp] extract video id from the webpage(fixes
 #7799)

---
 youtube_dl/extractor/tvp.py | 44 +++++++++++++++++++++++++++++++++++++
 1 file changed, 44 insertions(+)

diff --git a/youtube_dl/extractor/tvp.py b/youtube_dl/extractor/tvp.py
index e84876b54..2abfb7830 100644
--- a/youtube_dl/extractor/tvp.py
+++ b/youtube_dl/extractor/tvp.py
@@ -24,6 +24,7 @@ class TVPIE(InfoExtractor):
             'id': '194536',
             'ext': 'mp4',
             'title': 'Czas honoru, I seria – odc. 13',
+            'description': 'md5:76649d2014f65c99477be17f23a4dead',
         },
     }, {
         'url': 'http://www.tvp.pl/there-can-be-anything-so-i-shortened-it/17916176',
@@ -32,6 +33,16 @@ class TVPIE(InfoExtractor):
             'id': '17916176',
             'ext': 'mp4',
             'title': 'TVP Gorzów pokaże filmy studentów z podroży dookoła świata',
+            'description': 'TVP Gorzów pokaże filmy studentów z podroży dookoła świata',
+        },
+    }, {
+        # page id is not the same as video id(#7799)
+        'url': 'http://vod.tvp.pl/22704887/08122015-1500',
+        'md5': 'cf6a4705dfd1489aef8deb168d6ba742',
+        'info_dict': {
+            'id': '22680786',
+            'ext': 'mp4',
+            'title': 'Wiadomości, 08.12.2015, 15:00',
         },
     }, {
         'url': 'http://vod.tvp.pl/seriale/obyczajowe/na-sygnale/sezon-2-27-/odc-39/17834272',
@@ -53,6 +64,39 @@ class TVPIE(InfoExtractor):
         'only_matching': True,
     }]
 
+    def _real_extract(self, url):
+        page_id = self._match_id(url)
+        webpage = self._download_webpage(url, page_id)
+        video_id = self._search_regex([
+            r'<iframe[^>]+src="[^"]*?object_id=(\d+)',
+            "object_id\s*:\s*'(\d+)'"], webpage, 'video id')
+        return {
+            '_type': 'url_transparent',
+            'url': 'tvp:' + video_id,
+            'description': self._og_search_description(webpage, default=None),
+            'thumbnail': self._og_search_thumbnail(webpage),
+            'ie_key': 'TVPEmbed',
+        }
+
+
+class TVPEmbedIE(InfoExtractor):
+    IE_NAME = 'tvp:embed'
+    IE_DESC = 'Telewizja Polska'
+    _VALID_URL = r'(?:tvp:|https?://[^/]+\.tvp\.(?:pl|info)/sess/tvplayer\.php\?.*?object_id=)(?P<id>\d+)'
+
+    _TESTS = [{
+        'url': 'http://www.tvp.pl/sess/tvplayer.php?object_id=22670268',
+        'md5': '8c9cd59d16edabf39331f93bf8a766c7',
+        'info_dict': {
+            'id': '22670268',
+            'ext': 'mp4',
+            'title': 'Panorama, 07.12.2015, 15:40',
+        },
+    }, {
+        'url': 'tvp:22670268',
+        'only_matching': True,
+    }]
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
 

From a7d29530732574554cd20324fda0faa17979a1ae Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 5 Aug 2016 10:11:59 +0100
Subject: [PATCH 5651/6219] [extractors] add tvp:embed import

---
 youtube_dl/extractor/extractors.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 28858cfea..fec560ba3 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -892,6 +892,7 @@ from .tvc import (
 from .tvigle import TvigleIE
 from .tvland import TVLandIE
 from .tvp import (
+    TVPEmbedIE,
     TVPIE,
     TVPSeriesIE,
 )

From 5a993e169262237f4990207049d4cbbc69d5fd8d Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 5 Aug 2016 10:13:11 +0100
Subject: [PATCH 5652/6219] [natgeo] fix tests(closes #10229)

---
 youtube_dl/extractor/nationalgeographic.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/nationalgeographic.py b/youtube_dl/extractor/nationalgeographic.py
index 4bffcf704..0027ff1b8 100644
--- a/youtube_dl/extractor/nationalgeographic.py
+++ b/youtube_dl/extractor/nationalgeographic.py
@@ -74,7 +74,7 @@ class NationalGeographicIE(ThePlatformIE):
             'url': 'http://channel.nationalgeographic.com/the-story-of-god-with-morgan-freeman/videos/uncovering-a-universal-knowledge/',
             'md5': '518c9aa655686cf81493af5cc21e2a04',
             'info_dict': {
-                'id': 'nB5vIAfmyllm',
+                'id': 'vKInpacll2pC',
                 'ext': 'mp4',
                 'title': 'Uncovering a Universal Knowledge',
                 'description': 'md5:1a89148475bf931b3661fcd6ddb2ae3a',
@@ -88,7 +88,7 @@ class NationalGeographicIE(ThePlatformIE):
             'url': 'http://channel.nationalgeographic.com/wild/destination-wild/videos/the-stunning-red-bird-of-paradise/',
             'md5': 'c4912f656b4cbe58f3e000c489360989',
             'info_dict': {
-                'id': '3TmMv9OvGwIR',
+                'id': 'Pok5lWCkiEFA',
                 'ext': 'mp4',
                 'title': 'The Stunning Red Bird of Paradise',
                 'description': 'md5:7bc8cd1da29686be4d17ad1230f0140c',

From 08c655906c6afd41ca56e784f35791cfdb727d3d Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 5 Aug 2016 10:22:33 +0100
Subject: [PATCH 5653/6219] [5min] fix _VALID_URL(closes #10228)

---
 youtube_dl/extractor/fivemin.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/fivemin.py b/youtube_dl/extractor/fivemin.py
index ab82701c2..f3f876ecd 100644
--- a/youtube_dl/extractor/fivemin.py
+++ b/youtube_dl/extractor/fivemin.py
@@ -5,7 +5,7 @@ from .common import InfoExtractor
 
 class FiveMinIE(InfoExtractor):
     IE_NAME = '5min'
-    _VALID_URL = r'(?:5min:|https?://(?:[^/]*?5min\.com|delivery\.vidible\.tv/aol)/(?:(?:Scripts/PlayerSeed\.js|playerseed/?)?\?.*?playList=)?)(?P<id>\d+)'
+    _VALID_URL = r'(?:5min:|https?://(?:[^/]*?5min\.com/|delivery\.vidible\.tv/aol)(?:(?:Scripts/PlayerSeed\.js|playerseed/?)?\?.*?playList=)?)(?P<id>\d+)'
 
     _TESTS = [
         {

From f0d31c624e51322e26889df417152b833549bb75 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 5 Aug 2016 22:17:32 +0700
Subject: [PATCH 5654/6219] [tvplay] Add support for subtitles (Closes #10194)

---
 youtube_dl/extractor/tvplay.py | 20 ++++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/tvplay.py b/youtube_dl/extractor/tvplay.py
index 918f8f8bc..0c072a6ae 100644
--- a/youtube_dl/extractor/tvplay.py
+++ b/youtube_dl/extractor/tvplay.py
@@ -4,7 +4,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_str
+from ..compat import (
+    compat_str,
+    compat_urlparse,
+)
 from ..utils import (
     parse_iso8601,
     qualities,
@@ -226,7 +229,8 @@ class TVPlayIE(InfoExtractor):
                     'ext': ext,
                 }
                 if video_url.startswith('rtmp'):
-                    m = re.search(r'^(?P<url>rtmp://[^/]+/(?P<app>[^/]+))/(?P<playpath>.+)$', video_url)
+                    m = re.search(
+                        r'^(?P<url>rtmp://[^/]+/(?P<app>[^/]+))/(?P<playpath>.+)$', video_url)
                     if not m:
                         continue
                     fmt.update({
@@ -242,6 +246,17 @@ class TVPlayIE(InfoExtractor):
                 formats.append(fmt)
         self._sort_formats(formats)
 
+        # TODO: webvtt in m3u8
+        subtitles = {}
+        sami_path = video.get('sami_path')
+        if sami_path:
+            lang = self._search_regex(
+                r'_([a-z]{2})\.xml', sami_path, 'lang',
+                default=compat_urlparse.urlparse(url).netloc.rsplit('.', 1)[-1])
+            subtitles[lang] = [{
+                'url': sami_path,
+            }]
+
         return {
             'id': video_id,
             'title': title,
@@ -251,4 +266,5 @@ class TVPlayIE(InfoExtractor):
             'view_count': int_or_none(video.get('views', {}).get('total')),
             'age_limit': int_or_none(video.get('age_limit', 0)),
             'formats': formats,
+            'subtitles': subtitles,
         }

From 5ca968d0a6278400be7f4215f93c026c2bf0eafb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 5 Aug 2016 22:37:38 +0700
Subject: [PATCH 5655/6219] [tvplay] Extract series metadata

---
 youtube_dl/extractor/tvplay.py | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/youtube_dl/extractor/tvplay.py b/youtube_dl/extractor/tvplay.py
index 0c072a6ae..3d4c576c8 100644
--- a/youtube_dl/extractor/tvplay.py
+++ b/youtube_dl/extractor/tvplay.py
@@ -37,6 +37,9 @@ class TVPlayIE(InfoExtractor):
                 'ext': 'mp4',
                 'title': 'Kādi ir īri? - Viņas melo labāk',
                 'description': 'Baiba apsmej īrus, kādi tie ir un ko viņi dara.',
+                'series': 'Viņas melo labāk',
+                'season': '2.sezona',
+                'season_number': 2,
                 'duration': 25,
                 'timestamp': 1406097056,
                 'upload_date': '20140723',
@@ -49,6 +52,10 @@ class TVPlayIE(InfoExtractor):
                 'ext': 'flv',
                 'title': 'Moterys meluoja geriau',
                 'description': 'md5:9aec0fc68e2cbc992d2a140bd41fa89e',
+                'series': 'Moterys meluoja geriau',
+                'episode_number': 47,
+                'season': '1 sezonas',
+                'season_number': 1,
                 'duration': 1330,
                 'timestamp': 1403769181,
                 'upload_date': '20140626',
@@ -257,10 +264,19 @@ class TVPlayIE(InfoExtractor):
                 'url': sami_path,
             }]
 
+        series = video.get('format_title')
+        episode_number = int_or_none(video.get('format_position', {}).get('episode'))
+        season = video.get('_embedded', {}).get('season', {}).get('title')
+        season_number = int_or_none(video.get('format_position', {}).get('season'))
+
         return {
             'id': video_id,
             'title': title,
             'description': video.get('description'),
+            'series': series,
+            'episode_number': episode_number,
+            'season': season,
+            'season_number': season_number,
             'duration': int_or_none(video.get('duration')),
             'timestamp': parse_iso8601(video.get('created_at')),
             'view_count': int_or_none(video.get('views', {}).get('total')),

From 0ca057b965e5699a88fc952da460b5adfb8e7644 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 5 Aug 2016 16:39:24 +0100
Subject: [PATCH 5656/6219] [jwplatform] add support for playlist extraction
 and relative urls and improve audio detection

---
 youtube_dl/extractor/jwplatform.py | 132 ++++++++++++++++-------------
 1 file changed, 72 insertions(+), 60 deletions(-)

diff --git a/youtube_dl/extractor/jwplatform.py b/youtube_dl/extractor/jwplatform.py
index e44e31104..2a499bb77 100644
--- a/youtube_dl/extractor/jwplatform.py
+++ b/youtube_dl/extractor/jwplatform.py
@@ -4,10 +4,12 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import compat_urlparse
 from ..utils import (
     determine_ext,
     float_or_none,
     int_or_none,
+    mimetype2ext,
 )
 
 
@@ -28,74 +30,84 @@ class JWPlatformBaseIE(InfoExtractor):
         return self._parse_jwplayer_data(
             jwplayer_data, video_id, *args, **kwargs)
 
-    def _parse_jwplayer_data(self, jwplayer_data, video_id, require_title=True, m3u8_id=None, rtmp_params=None):
+    def _parse_jwplayer_data(self, jwplayer_data, video_id, require_title=True, m3u8_id=None, rtmp_params=None, base_url=None):
         # JWPlayer backward compatibility: flattened playlists
         # https://github.com/jwplayer/jwplayer/blob/v7.4.3/src/js/api/config.js#L81-L96
         if 'playlist' not in jwplayer_data:
             jwplayer_data = {'playlist': [jwplayer_data]}
 
-        video_data = jwplayer_data['playlist'][0]
+        entries = []
+        for video_data in jwplayer_data['playlist']:
+            # JWPlayer backward compatibility: flattened sources
+            # https://github.com/jwplayer/jwplayer/blob/v7.4.3/src/js/playlist/item.js#L29-L35
+            if 'sources' not in video_data:
+                video_data['sources'] = [video_data]
 
-        # JWPlayer backward compatibility: flattened sources
-        # https://github.com/jwplayer/jwplayer/blob/v7.4.3/src/js/playlist/item.js#L29-L35
-        if 'sources' not in video_data:
-            video_data['sources'] = [video_data]
-
-        formats = []
-        for source in video_data['sources']:
-            source_url = self._proto_relative_url(source['file'])
-            source_type = source.get('type') or ''
-            if source_type in ('application/vnd.apple.mpegurl', 'hls') or determine_ext(source_url) == 'm3u8':
-                formats.extend(self._extract_m3u8_formats(
-                    source_url, video_id, 'mp4', 'm3u8_native', m3u8_id=m3u8_id, fatal=False))
-            elif source_type.startswith('audio'):
-                formats.append({
-                    'url': source_url,
-                    'vcodec': 'none',
-                })
-            else:
-                a_format = {
-                    'url': source_url,
-                    'width': int_or_none(source.get('width')),
-                    'height': int_or_none(source.get('height')),
-                }
-                if source_url.startswith('rtmp'):
-                    a_format['ext'] = 'flv',
-
-                    # See com/longtailvideo/jwplayer/media/RTMPMediaProvider.as
-                    # of jwplayer.flash.swf
-                    rtmp_url_parts = re.split(
-                        r'((?:mp4|mp3|flv):)', source_url, 1)
-                    if len(rtmp_url_parts) == 3:
-                        rtmp_url, prefix, play_path = rtmp_url_parts
-                        a_format.update({
-                            'url': rtmp_url,
-                            'play_path': prefix + play_path,
-                        })
-                    if rtmp_params:
-                        a_format.update(rtmp_params)
-                formats.append(a_format)
-        self._sort_formats(formats)
-
-        subtitles = {}
-        tracks = video_data.get('tracks')
-        if tracks and isinstance(tracks, list):
-            for track in tracks:
-                if track.get('file') and track.get('kind') == 'captions':
-                    subtitles.setdefault(track.get('label') or 'en', []).append({
-                        'url': self._proto_relative_url(track['file'])
+            formats = []
+            for source in video_data['sources']:
+                source_url = self._proto_relative_url(source['file'])
+                if base_url:
+                    source_url = compat_urlparse.urljoin(base_url, source_url)
+                source_type = source.get('type') or ''
+                ext = mimetype2ext(source_type) or determine_ext(source_url)
+                if source_type == 'hls' or ext == 'm3u8':
+                    formats.extend(self._extract_m3u8_formats(
+                        source_url, video_id, 'mp4', 'm3u8_native', m3u8_id=m3u8_id, fatal=False))
+                # https://github.com/jwplayer/jwplayer/blob/master/src/js/providers/default.js#L67
+                elif source_type.startswith('audio') or ext in ('oga', 'aac', 'mp3', 'mpeg', 'vorbis'):
+                    formats.append({
+                        'url': source_url,
+                        'vcodec': 'none',
+                        'ext': ext,
                     })
+                else:
+                    a_format = {
+                        'url': source_url,
+                        'width': int_or_none(source.get('width')),
+                        'height': int_or_none(source.get('height')),
+                        'ext': ext,
+                    }
+                    if source_url.startswith('rtmp'):
+                        a_format['ext'] = 'flv',
 
-        return {
-            'id': video_id,
-            'title': video_data['title'] if require_title else video_data.get('title'),
-            'description': video_data.get('description'),
-            'thumbnail': self._proto_relative_url(video_data.get('image')),
-            'timestamp': int_or_none(video_data.get('pubdate')),
-            'duration': float_or_none(jwplayer_data.get('duration')),
-            'subtitles': subtitles,
-            'formats': formats,
-        }
+                        # See com/longtailvideo/jwplayer/media/RTMPMediaProvider.as
+                        # of jwplayer.flash.swf
+                        rtmp_url_parts = re.split(
+                            r'((?:mp4|mp3|flv):)', source_url, 1)
+                        if len(rtmp_url_parts) == 3:
+                            rtmp_url, prefix, play_path = rtmp_url_parts
+                            a_format.update({
+                                'url': rtmp_url,
+                                'play_path': prefix + play_path,
+                            })
+                        if rtmp_params:
+                            a_format.update(rtmp_params)
+                    formats.append(a_format)
+            self._sort_formats(formats)
+
+            subtitles = {}
+            tracks = video_data.get('tracks')
+            if tracks and isinstance(tracks, list):
+                for track in tracks:
+                    if track.get('file') and track.get('kind') == 'captions':
+                        subtitles.setdefault(track.get('label') or 'en', []).append({
+                            'url': self._proto_relative_url(track['file'])
+                        })
+
+            entries.append({
+                'id': video_id,
+                'title': video_data['title'] if require_title else video_data.get('title'),
+                'description': video_data.get('description'),
+                'thumbnail': self._proto_relative_url(video_data.get('image')),
+                'timestamp': int_or_none(video_data.get('pubdate')),
+                'duration': float_or_none(jwplayer_data.get('duration')),
+                'subtitles': subtitles,
+                'formats': formats,
+            })
+        if len(entries) == 1:
+            return entries[0]
+        else:
+            return self.playlist_result(entries)
 
 
 class JWPlatformIE(JWPlatformBaseIE):

From d50aca41f8e3aa3af8e19ec91100283b555ac59f Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 5 Aug 2016 16:40:21 +0100
Subject: [PATCH 5657/6219] [archiveorg] improve format extraction(closes
 #10219)

---
 youtube_dl/extractor/archiveorg.py | 78 +++++++++++++++---------------
 1 file changed, 38 insertions(+), 40 deletions(-)

diff --git a/youtube_dl/extractor/archiveorg.py b/youtube_dl/extractor/archiveorg.py
index 8feb7cb74..2472e4cc6 100644
--- a/youtube_dl/extractor/archiveorg.py
+++ b/youtube_dl/extractor/archiveorg.py
@@ -1,67 +1,65 @@
 from __future__ import unicode_literals
 
-from .common import InfoExtractor
-from ..utils import unified_strdate
+from .jwplatform import JWPlatformBaseIE
+from ..utils import (
+    unified_strdate,
+    clean_html,
+)
 
 
-class ArchiveOrgIE(InfoExtractor):
+class ArchiveOrgIE(JWPlatformBaseIE):
     IE_NAME = 'archive.org'
     IE_DESC = 'archive.org videos'
-    _VALID_URL = r'https?://(?:www\.)?archive\.org/details/(?P<id>[^?/]+)(?:[?].*)?$'
+    _VALID_URL = r'https?://(?:www\.)?archive\.org/(?:details|embed)/(?P<id>[^/?#]+)(?:[?].*)?$'
     _TESTS = [{
         'url': 'http://archive.org/details/XD300-23_68HighlightsAResearchCntAugHumanIntellect',
         'md5': '8af1d4cf447933ed3c7f4871162602db',
         'info_dict': {
             'id': 'XD300-23_68HighlightsAResearchCntAugHumanIntellect',
-            'ext': 'ogv',
+            'ext': 'ogg',
             'title': '1968 Demo - FJCC Conference Presentation Reel #1',
-            'description': 'md5:1780b464abaca9991d8968c877bb53ed',
+            'description': 'md5:da45c349df039f1cc8075268eb1b5c25',
             'upload_date': '19681210',
             'uploader': 'SRI International'
         }
     }, {
         'url': 'https://archive.org/details/Cops1922',
-        'md5': '18f2a19e6d89af8425671da1cf3d4e04',
+        'md5': 'bc73c8ab3838b5a8fc6c6651fa7b58ba',
         'info_dict': {
             'id': 'Cops1922',
-            'ext': 'ogv',
+            'ext': 'mp4',
             'title': 'Buster Keaton\'s "Cops" (1922)',
-            'description': 'md5:70f72ee70882f713d4578725461ffcc3',
+            'description': 'md5:b4544662605877edd99df22f9620d858',
         }
+    }, {
+        'url': 'http://archive.org/embed/XD300-23_68HighlightsAResearchCntAugHumanIntellect',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
+        webpage = self._download_webpage(
+            'http://archive.org/embed/' + video_id, video_id)
+        jwplayer_playlist = self._parse_json(self._search_regex(
+            r"(?s)Play\('[^']+'\s*,\s*(\[.+\])\s*,\s*{.*?}\);",
+            webpage, 'jwplayer playlist'), video_id)
+        info = self._parse_jwplayer_data(
+            {'playlist': jwplayer_playlist}, video_id, base_url=url)
 
-        json_url = url + ('&' if '?' in url else '?') + 'output=json'
-        data = self._download_json(json_url, video_id)
+        def get_optional(metadata, field):
+            return metadata.get(field, [None])[0]
 
-        def get_optional(data_dict, field):
-            return data_dict['metadata'].get(field, [None])[0]
-
-        title = get_optional(data, 'title')
-        description = get_optional(data, 'description')
-        uploader = get_optional(data, 'creator')
-        upload_date = unified_strdate(get_optional(data, 'date'))
-
-        formats = [
-            {
-                'format': fdata['format'],
-                'url': 'http://' + data['server'] + data['dir'] + fn,
-                'file_size': int(fdata['size']),
-            }
-            for fn, fdata in data['files'].items()
-            if 'Video' in fdata['format']]
-
-        self._sort_formats(formats)
-
-        return {
-            '_type': 'video',
-            'id': video_id,
-            'title': title,
-            'formats': formats,
-            'description': description,
-            'uploader': uploader,
-            'upload_date': upload_date,
-            'thumbnail': data.get('misc', {}).get('image'),
-        }
+        metadata = self._download_json(
+            'http://archive.org/details/' + video_id, video_id, query={
+                'output': 'json',
+            })['metadata']
+        info.update({
+            'title': get_optional(metadata, 'title') or info.get('title'),
+            'description': clean_html(get_optional(metadata, 'description')),
+        })
+        if info.get('_type') != 'playlist':
+            info.update({
+                'uploader': get_optional(metadata, 'creator'),
+                'upload_date': unified_strdate(get_optional(metadata, 'date')),
+            })
+        return info
\ No newline at end of file

From 3859ebeee6d6448240176ef5e4c20f6b1d1db795 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 5 Aug 2016 22:49:19 +0700
Subject: [PATCH 5658/6219] [tvplay] Capture and output native error message

---
 youtube_dl/extractor/tvplay.py | 26 ++++++++++++++++++--------
 1 file changed, 18 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/tvplay.py b/youtube_dl/extractor/tvplay.py
index 3d4c576c8..150bde663 100644
--- a/youtube_dl/extractor/tvplay.py
+++ b/youtube_dl/extractor/tvplay.py
@@ -5,15 +5,17 @@ import re
 
 from .common import InfoExtractor
 from ..compat import (
+    compat_HTTPError,
     compat_str,
     compat_urlparse,
 )
 from ..utils import (
+    determine_ext,
+    ExtractorError,
+    int_or_none,
     parse_iso8601,
     qualities,
-    determine_ext,
     update_url_query,
-    int_or_none,
 )
 
 
@@ -206,12 +208,15 @@ class TVPlayIE(InfoExtractor):
 
         title = video['title']
 
-        if video.get('is_geo_blocked'):
-            self.report_warning(
-                'This content might not be available in your country due to copyright reasons')
-
-        streams = self._download_json(
-            'http://playapi.mtgx.tv/v1/videos/stream/%s' % video_id, video_id, 'Downloading streams JSON')
+        try:
+            streams = self._download_json(
+                'http://playapi.mtgx.tv/v1/videos/stream/%s' % video_id,
+                video_id, 'Downloading streams JSON')
+        except ExtractorError as e:
+            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
+                msg = self._parse_json(e.cause.read().decode('utf-8'), video_id)
+                raise ExtractorError(msg['msg'], expected=True)
+            raise
 
         quality = qualities(['hls', 'medium', 'high'])
         formats = []
@@ -251,6 +256,11 @@ class TVPlayIE(InfoExtractor):
                         'url': video_url,
                     })
                 formats.append(fmt)
+
+        if not formats and video.get('is_geo_blocked'):
+            self.raise_geo_restricted(
+                'This content might not be available in your country due to copyright reasons')
+
         self._sort_formats(formats)
 
         # TODO: webvtt in m3u8

From 46933a15d69a0079e66a04475659d0cbd5e5f08d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 5 Aug 2016 23:14:32 +0700
Subject: [PATCH 5659/6219] [extractor/common] Support root JSON-LD lists
 (Closes #10203)

---
 youtube_dl/extractor/common.py | 76 ++++++++++++++++++----------------
 1 file changed, 41 insertions(+), 35 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 2d337d614..70909fc1c 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -828,41 +828,47 @@ class InfoExtractor(object):
         if not json_ld:
             return {}
         info = {}
-        if json_ld.get('@context') == 'http://schema.org':
-            item_type = json_ld.get('@type')
-            if expected_type is not None and expected_type != item_type:
-                return info
-            if item_type == 'TVEpisode':
-                info.update({
-                    'episode': unescapeHTML(json_ld.get('name')),
-                    'episode_number': int_or_none(json_ld.get('episodeNumber')),
-                    'description': unescapeHTML(json_ld.get('description')),
-                })
-                part_of_season = json_ld.get('partOfSeason')
-                if isinstance(part_of_season, dict) and part_of_season.get('@type') == 'TVSeason':
-                    info['season_number'] = int_or_none(part_of_season.get('seasonNumber'))
-                part_of_series = json_ld.get('partOfSeries')
-                if isinstance(part_of_series, dict) and part_of_series.get('@type') == 'TVSeries':
-                    info['series'] = unescapeHTML(part_of_series.get('name'))
-            elif item_type == 'Article':
-                info.update({
-                    'timestamp': parse_iso8601(json_ld.get('datePublished')),
-                    'title': unescapeHTML(json_ld.get('headline')),
-                    'description': unescapeHTML(json_ld.get('articleBody')),
-                })
-            elif item_type == 'VideoObject':
-                info.update({
-                    'url': json_ld.get('contentUrl'),
-                    'title': unescapeHTML(json_ld.get('name')),
-                    'description': unescapeHTML(json_ld.get('description')),
-                    'thumbnail': json_ld.get('thumbnailUrl'),
-                    'duration': parse_duration(json_ld.get('duration')),
-                    'timestamp': unified_timestamp(json_ld.get('uploadDate')),
-                    'filesize': float_or_none(json_ld.get('contentSize')),
-                    'tbr': int_or_none(json_ld.get('bitrate')),
-                    'width': int_or_none(json_ld.get('width')),
-                    'height': int_or_none(json_ld.get('height')),
-                })
+        if not isinstance(json_ld, (list, tuple, dict)):
+            return info
+        if isinstance(json_ld, dict):
+            json_ld = [json_ld]
+        for e in json_ld:
+            if e.get('@context') == 'http://schema.org':
+                item_type = e.get('@type')
+                if expected_type is not None and expected_type != item_type:
+                    return info
+                if item_type == 'TVEpisode':
+                    info.update({
+                        'episode': unescapeHTML(e.get('name')),
+                        'episode_number': int_or_none(e.get('episodeNumber')),
+                        'description': unescapeHTML(e.get('description')),
+                    })
+                    part_of_season = e.get('partOfSeason')
+                    if isinstance(part_of_season, dict) and part_of_season.get('@type') == 'TVSeason':
+                        info['season_number'] = int_or_none(part_of_season.get('seasonNumber'))
+                    part_of_series = e.get('partOfSeries')
+                    if isinstance(part_of_series, dict) and part_of_series.get('@type') == 'TVSeries':
+                        info['series'] = unescapeHTML(part_of_series.get('name'))
+                elif item_type == 'Article':
+                    info.update({
+                        'timestamp': parse_iso8601(e.get('datePublished')),
+                        'title': unescapeHTML(e.get('headline')),
+                        'description': unescapeHTML(e.get('articleBody')),
+                    })
+                elif item_type == 'VideoObject':
+                    info.update({
+                        'url': e.get('contentUrl'),
+                        'title': unescapeHTML(e.get('name')),
+                        'description': unescapeHTML(e.get('description')),
+                        'thumbnail': e.get('thumbnailUrl'),
+                        'duration': parse_duration(e.get('duration')),
+                        'timestamp': unified_timestamp(e.get('uploadDate')),
+                        'filesize': float_or_none(e.get('contentSize')),
+                        'tbr': int_or_none(e.get('bitrate')),
+                        'width': int_or_none(e.get('width')),
+                        'height': int_or_none(e.get('height')),
+                    })
+                break
         return dict((k, v) for k, v in info.items() if v is not None)
 
     @staticmethod

From 84bc23b41b5d1bc51102b264c23e96202e093d14 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 5 Aug 2016 23:16:19 +0700
Subject: [PATCH 5660/6219] [archiveorg] PEP 8

---
 youtube_dl/extractor/archiveorg.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/archiveorg.py b/youtube_dl/extractor/archiveorg.py
index 2472e4cc6..486dff82d 100644
--- a/youtube_dl/extractor/archiveorg.py
+++ b/youtube_dl/extractor/archiveorg.py
@@ -62,4 +62,4 @@ class ArchiveOrgIE(JWPlatformBaseIE):
                 'uploader': get_optional(metadata, 'creator'),
                 'upload_date': unified_strdate(get_optional(metadata, 'date')),
             })
-        return info
\ No newline at end of file
+        return info

From 038a5e1a65401ad6bfd2b6572f321af66e7cd9bc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 6 Aug 2016 00:00:05 +0700
Subject: [PATCH 5661/6219] [adultswim] Add support for trailers (Closes
 #10235)

---
 youtube_dl/extractor/adultswim.py | 31 ++++++++++++++++++++++++++-----
 1 file changed, 26 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/adultswim.py b/youtube_dl/extractor/adultswim.py
index 8157da2cb..6d20229f9 100644
--- a/youtube_dl/extractor/adultswim.py
+++ b/youtube_dl/extractor/adultswim.py
@@ -83,6 +83,21 @@ class AdultSwimIE(InfoExtractor):
             # m3u8 download
             'skip_download': True,
         }
+    }, {
+        # heroMetadata.trailer
+        'url': 'http://www.adultswim.com/videos/decker/inside-decker-a-new-hero/',
+        'md5': '33e9a5d8f646523ce0868ecfb0eed77d',
+        'info_dict': {
+            'id': 'I0LQFQkaSUaFp8PnAWHhoQ',
+            'ext': 'mp4',
+            'title': 'Decker - Inside Decker: A New Hero',
+            'description': 'md5:c916df071d425d62d70c86d4399d3ee0',
+            'duration': 249.008,
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        }
     }]
 
     @staticmethod
@@ -133,20 +148,26 @@ class AdultSwimIE(InfoExtractor):
             if video_info is None:
                 if bootstrapped_data.get('slugged_video', {}).get('slug') == episode_path:
                     video_info = bootstrapped_data['slugged_video']
-                else:
-                    raise ExtractorError('Unable to find video info')
+            if not video_info:
+                video_info = bootstrapped_data.get('heroMetadata', {}).get('trailer').get('video')
+            if not video_info:
+                raise ExtractorError('Unable to find video info')
 
             show = bootstrapped_data['show']
             show_title = show['title']
             stream = video_info.get('stream')
-            clips = [stream] if stream else video_info.get('clips')
-            if not clips:
+            if stream and stream.get('videoPlaybackID'):
+                segment_ids = [stream['videoPlaybackID']]
+            elif video_info.get('clips'):
+                segment_ids = [clip['videoPlaybackID'] for clip in video_info['clips']]
+            elif video_info.get('videoPlaybackID'):
+                segment_ids = [video_info['videoPlaybackID']]
+            else:
                 raise ExtractorError(
                     'This video is only available via cable service provider subscription that'
                     ' is not currently supported. You may want to use --cookies.'
                     if video_info.get('auth') is True else 'Unable to find stream or clips',
                     expected=True)
-            segment_ids = [clip['videoPlaybackID'] for clip in clips]
 
         episode_id = video_info['id']
         episode_title = video_info['title']

From fe3ad1d45651d3bd05a3faae77bc23bd05e689f8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 6 Aug 2016 00:02:05 +0700
Subject: [PATCH 5662/6219] [adultswim] Remove superfluous md5 from test

---
 youtube_dl/extractor/adultswim.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/adultswim.py b/youtube_dl/extractor/adultswim.py
index 6d20229f9..3f7f8c036 100644
--- a/youtube_dl/extractor/adultswim.py
+++ b/youtube_dl/extractor/adultswim.py
@@ -86,7 +86,6 @@ class AdultSwimIE(InfoExtractor):
     }, {
         # heroMetadata.trailer
         'url': 'http://www.adultswim.com/videos/decker/inside-decker-a-new-hero/',
-        'md5': '33e9a5d8f646523ce0868ecfb0eed77d',
         'info_dict': {
             'id': 'I0LQFQkaSUaFp8PnAWHhoQ',
             'ext': 'mp4',

From 8122e79fefa8eca62c3cddbf686c8920b6d06a9a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 6 Aug 2016 00:12:37 +0700
Subject: [PATCH 5663/6219] [gamekings] Remove remnants

---
 youtube_dl/extractor/gamekings.py | 76 -------------------------------
 1 file changed, 76 deletions(-)
 delete mode 100644 youtube_dl/extractor/gamekings.py

diff --git a/youtube_dl/extractor/gamekings.py b/youtube_dl/extractor/gamekings.py
deleted file mode 100644
index cbcddcb7c..000000000
--- a/youtube_dl/extractor/gamekings.py
+++ /dev/null
@@ -1,76 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-from .common import InfoExtractor
-from ..utils import (
-    xpath_text,
-    xpath_with_ns,
-)
-from .youtube import YoutubeIE
-
-
-class GamekingsIE(InfoExtractor):
-    _VALID_URL = r'https?://www\.gamekings\.nl/(?:videos|nieuws)/(?P<id>[^/]+)'
-    _TESTS = [{
-        # YouTube embed video
-        'url': 'http://www.gamekings.nl/videos/phoenix-wright-ace-attorney-dual-destinies-review/',
-        'md5': '5208d3a17adeaef829a7861887cb9029',
-        'info_dict': {
-            'id': 'HkSQKetlGOU',
-            'ext': 'mp4',
-            'title': 'Phoenix Wright: Ace Attorney - Dual Destinies Review',
-            'description': 'md5:db88c0e7f47e9ea50df3271b9dc72e1d',
-            'thumbnail': 're:^https?://.*\.jpg$',
-            'uploader_id': 'UCJugRGo4STYMeFr5RoOShtQ',
-            'uploader': 'Gamekings Vault',
-            'upload_date': '20151123',
-        },
-        'add_ie': ['Youtube'],
-    }, {
-        # vimeo video
-        'url': 'http://www.gamekings.nl/videos/the-legend-of-zelda-majoras-mask/',
-        'md5': '12bf04dfd238e70058046937657ea68d',
-        'info_dict': {
-            'id': 'the-legend-of-zelda-majoras-mask',
-            'ext': 'mp4',
-            'title': 'The Legend of Zelda: Majora’s Mask',
-            'description': 'md5:9917825fe0e9f4057601fe1e38860de3',
-            'thumbnail': 're:^https?://.*\.jpg$',
-        },
-    }, {
-        'url': 'http://www.gamekings.nl/nieuws/gamekings-extra-shelly-en-david-bereiden-zich-voor-op-de-livestream/',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        webpage = self._download_webpage(url, video_id)
-
-        playlist_id = self._search_regex(
-            r'gogoVideo\([^,]+,\s*"([^"]+)', webpage, 'playlist id')
-
-        # Check if a YouTube embed is used
-        if YoutubeIE.suitable(playlist_id):
-            return self.url_result(playlist_id, ie='Youtube')
-
-        playlist = self._download_xml(
-            'http://www.gamekings.tv/wp-content/themes/gk2010/rss_playlist.php?id=%s' % playlist_id,
-            video_id)
-
-        NS_MAP = {
-            'jwplayer': 'http://rss.jwpcdn.com/'
-        }
-
-        item = playlist.find('./channel/item')
-
-        thumbnail = xpath_text(item, xpath_with_ns('./jwplayer:image', NS_MAP), 'thumbnail')
-        video_url = item.find(xpath_with_ns('./jwplayer:source', NS_MAP)).get('file')
-
-        return {
-            'id': video_id,
-            'url': video_url,
-            'title': self._og_search_title(webpage),
-            'description': self._og_search_description(webpage),
-            'thumbnail': thumbnail,
-        }

From 7f2339c61789256b3ed12e196d14eff6791f4f46 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 6 Aug 2016 01:19:47 +0700
Subject: [PATCH 5664/6219] [ChangeLog] Actualize

---
 ChangeLog | 33 +++++++++++++++++++++++++++++++--
 1 file changed, 31 insertions(+), 2 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 2d021ddf4..7fd44d2a0 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,7 +1,36 @@
 version <unreleased>
 
-Fixed/improved extractors
-- [cbslocal] Fix timestamp parsing (#10213)
+Core
+* Add support for JSON-LD root list entries (#10203)
+* Improve unified_timestamp
+* Lower preference of RTSP formats in generic sorting
++ Add support for multiple properties in _og_search_property
+* Improve password hiding from verbose output
+
+Extractors
++ [adultswim] Add support for trailers (#10235)
+* [archiveorg] Improve extraction (#10219)
++ [jwplatform] Add support for playlists
++ [jwplatform] Add support for relative URLs
+* [jwplatform] Improve audio detection
++ [tvplay] Capture and output native error message
++ [tvplay] Extract series metadata
++ [tvplay] Add support for subtitles (#10194)
+* [tvp] Improve extraction (#7799)
+* [cbslocal] Fix timestamp parsing (#10213)
++ [naver] Add support for subtitles (#8096)
+* [naver] Improve extraction
+* [condenast] Improve extraction
+* [engadget] Relax URL regular expression
+* [5min] Fix extraction
++ [nationalgeographic] Add support for Episode Guide
++ [kaltura] Add support for subtitles
+* [kaltura] Optimize network requests
++ [vodplatform] Add extractor for vod-platform.net
+- [gamekings] Remove extractor
+* [limelight] Extract HTTP formats
+* [ntvru] Fix extraction
++ [comedycentral] Re-add :tds and :thedailyshow shortnames
 
 
 version 2016.08.01

From 491c42e690bd51687e43fd5178bebf99dcc2cc0c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 6 Aug 2016 01:23:48 +0700
Subject: [PATCH 5665/6219] release 2016.08.06

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 7 +++++--
 youtube_dl/version.py     | 2 +-
 4 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 9d15b6a89..7241840c5 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.08.01*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.08.01**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.08.06*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.08.06**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.08.01
+[debug] youtube-dl version 2016.08.06
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 7fd44d2a0..9fd78cdda 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2016.08.06
 
 Core
 * Add support for JSON-LD root list entries (#10203)
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 1f89b1c14..2c8f950df 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -142,6 +142,7 @@
  - **CollegeRama**
  - **ComCarCoff**
  - **ComedyCentral**
+ - **ComedyCentralShortname**
  - **ComedyCentralTV**
  - **CondeNast**: Condé Nast media group: Allure, Architectural Digest, Ars Technica, Bon Appétit, Brides, Condé Nast, Condé Nast Traveler, Details, Epicurious, GQ, Glamour, Golf Digest, SELF, Teen Vogue, The New Yorker, Vanity Fair, Vogue, W Magazine, WIRED
  - **Coub**
@@ -247,7 +248,6 @@
  - **FunnyOrDie**
  - **Fusion**
  - **GameInformer**
- - **Gamekings**
  - **GameOne**
  - **gameone:playlist**
  - **Gamersyde**
@@ -415,7 +415,8 @@
  - **MyVidster**
  - **n-tv.de**
  - **natgeo**
- - **natgeo:channel**
+ - **natgeo:episodeguide**
+ - **natgeo:video**
  - **Naver**
  - **NBA**
  - **NBC**
@@ -726,6 +727,7 @@
  - **tvigle**: Интернет-телевидение Tvigle.ru
  - **tvland.com**
  - **tvp**: Telewizja Polska
+ - **tvp:embed**: Telewizja Polska
  - **tvp:series**
  - **TVPlay**: TV3Play and related services
  - **Tweakers**
@@ -805,6 +807,7 @@
  - **vk:wallpost**
  - **vlive**
  - **Vodlocker**
+ - **VODPlatform**
  - **VoiceRepublic**
  - **VoxMedia**
  - **Vporn**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 27f97b213..a4654fa59 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.08.01'
+__version__ = '2016.08.06'

From e563c0d73b778a1c91007f8abe0e6b43b1f7b608 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 5 Aug 2016 21:01:16 +0100
Subject: [PATCH 5666/6219] [condenast] fallback to loader.js if video.js fail

---
 youtube_dl/extractor/condenast.py | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/condenast.py b/youtube_dl/extractor/condenast.py
index 976a0e89d..15fabbb1c 100644
--- a/youtube_dl/extractor/condenast.py
+++ b/youtube_dl/extractor/condenast.py
@@ -113,11 +113,19 @@ class CondeNastIE(InfoExtractor):
                 'target': params['id'],
             })
         video_id = query['videoId']
+        video_info = None
         info_page = self._download_webpage(
             'http://player.cnevids.com/player/video.js',
-            video_id, 'Downloading video info', query=query)
-        video_info = self._parse_json(self._search_regex(
-            r'loadCallback\(({.+})\)', info_page, 'video info'), video_id)['video']
+            video_id, 'Downloading video info', query=query, fatal=False)
+        if info_page:
+            video_info = self._parse_json(self._search_regex(
+                r'loadCallback\(({.+})\)', info_page, 'video info'), video_id)['video']
+        else:
+            info_page = self._download_webpage(
+                'http://player.cnevids.com/player/loader.js',
+                video_id, 'Downloading loader info', query=query)
+            video_info = self._parse_json(self._search_regex(
+                r'var\s+video\s*=\s*({.+?});', info_page, 'video info'), video_id)
         title = video_info['title']
 
         formats = []

From d73ebac100c9f91acb002c4844ba67b73616322a Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 6 Aug 2016 11:18:14 +0100
Subject: [PATCH 5667/6219] [pokemon] Add new extractor(closes #10093)

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/pokemon.py    | 52 ++++++++++++++++++++++++++++++
 2 files changed, 53 insertions(+)
 create mode 100644 youtube_dl/extractor/pokemon.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index fec560ba3..11b64eeaa 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -636,6 +636,7 @@ from .pluralsight import (
     PluralsightCourseIE,
 )
 from .podomatic import PodomaticIE
+from .pokemon import PokemonIE
 from .polskieradio import PolskieRadioIE
 from .porn91 import Porn91IE
 from .pornhd import PornHdIE
diff --git a/youtube_dl/extractor/pokemon.py b/youtube_dl/extractor/pokemon.py
new file mode 100644
index 000000000..ce27f33e1
--- /dev/null
+++ b/youtube_dl/extractor/pokemon.py
@@ -0,0 +1,52 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    extract_attributes,
+    int_or_none,
+)
+
+
+class PokemonIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?pokemon\.com/[a-z]{2}(?:.*?play=(?P<id>[a-z0-9]{32})|/pokemon-episodes/(?P<display_id>[^/?#]+))'
+    _TESTS = [{
+        'url': 'http://www.pokemon.com/us/pokemon-episodes/19_01-from-a-to-z/?play=true',
+        'md5': '9fb209ae3a569aac25de0f5afc4ee08f',
+        'info_dict': {
+            'id': 'd0436c00c3ce4071ac6cee8130ac54a1',
+            'ext': 'mp4',
+            'title': 'From A to Z!',
+            'description': 'Bonnie makes a new friend, Ash runs into an old friend, and a terrifying premonition begins to unfold!',
+            'timestamp': 1460478136,
+            'upload_date': '20160412',
+        },
+        'add_id': ['LimelightMedia']
+    }, {
+        'url': 'http://www.pokemon.com/uk/pokemon-episodes/?play=2e8b5c761f1d4a9286165d7748c1ece2',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id, display_id = re.match(self._VALID_URL, url).groups()
+        webpage = self._download_webpage(url, video_id or display_id)
+        video_data = extract_attributes(self._search_regex(
+            r'(<[^>]+data-video-id="%s"[^>]*>)' % (video_id if video_id else '[a-z0-9]{32}'),
+            webpage, 'video data element'))
+        video_id = video_data['data-video-id']
+        title = video_data['data-video-title']
+        return {
+            '_type': 'url_transparent',
+            'id': video_id,
+            'url': 'limelight:media:%s' % video_id,
+            'title': title,
+            'description': video_data.get('data-video-summary'),
+            'thumbnail': video_data.get('data-video-poster'),
+            'series': 'Pokémon',
+            'season_number': int_or_none(video_data.get('data-video-season')),
+            'episode': title,
+            'episode_number': int_or_none(video_data.get('data-video-episode')),
+            'ie_key': 'LimelightMedia',
+        }

From 089a40955cdcdce1d8ea89b1402bde4c88c75546 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 6 Aug 2016 12:08:14 +0100
Subject: [PATCH 5668/6219] [pokemon] improve _VALID_URL

---
 youtube_dl/extractor/pokemon.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pokemon.py b/youtube_dl/extractor/pokemon.py
index ce27f33e1..2d87e7e70 100644
--- a/youtube_dl/extractor/pokemon.py
+++ b/youtube_dl/extractor/pokemon.py
@@ -11,7 +11,7 @@ from ..utils import (
 
 
 class PokemonIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?pokemon\.com/[a-z]{2}(?:.*?play=(?P<id>[a-z0-9]{32})|/pokemon-episodes/(?P<display_id>[^/?#]+))'
+    _VALID_URL = r'https?://(?:www\.)?pokemon\.com/[a-z]{2}(?:.*?play=(?P<id>[a-z0-9]{32})|/[^/]+/\d+_\d+-(?P<display_id>[^/?#]+))'
     _TESTS = [{
         'url': 'http://www.pokemon.com/us/pokemon-episodes/19_01-from-a-to-z/?play=true',
         'md5': '9fb209ae3a569aac25de0f5afc4ee08f',
@@ -27,6 +27,12 @@ class PokemonIE(InfoExtractor):
     }, {
         'url': 'http://www.pokemon.com/uk/pokemon-episodes/?play=2e8b5c761f1d4a9286165d7748c1ece2',
         'only_matching': True,
+    }, {
+        'url': 'http://www.pokemon.com/fr/episodes-pokemon/18_09-un-hiver-inattendu/',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.pokemon.com/de/pokemon-folgen/01_20-bye-bye-smettbo/',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From a7e5f274123dcfe9afd53134a33f83d53d1e497e Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 6 Aug 2016 18:48:09 +0100
Subject: [PATCH 5669/6219] [bbc] improve extraction

- extract f4m and dash formats
- improve format sorting and listing
- improve extraction of articles with `otherSettings.playlist`
---
 youtube_dl/extractor/bbc.py | 175 +++++++++++++++++-------------------
 1 file changed, 83 insertions(+), 92 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 9cb7630a1..45c562bd8 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -229,51 +229,6 @@ class BBCCoUkIE(InfoExtractor):
         asx = self._download_xml(connection.get('href'), programme_id, 'Downloading ASX playlist')
         return [ref.get('href') for ref in asx.findall('./Entry/ref')]
 
-    def _extract_connection(self, connection, programme_id):
-        formats = []
-        kind = connection.get('kind')
-        protocol = connection.get('protocol')
-        supplier = connection.get('supplier')
-        if protocol == 'http':
-            href = connection.get('href')
-            transfer_format = connection.get('transferFormat')
-            # ASX playlist
-            if supplier == 'asx':
-                for i, ref in enumerate(self._extract_asx_playlist(connection, programme_id)):
-                    formats.append({
-                        'url': ref,
-                        'format_id': 'ref%s_%s' % (i, supplier),
-                    })
-            # Skip DASH until supported
-            elif transfer_format == 'dash':
-                pass
-            elif transfer_format == 'hls':
-                formats.extend(self._extract_m3u8_formats(
-                    href, programme_id, ext='mp4', entry_protocol='m3u8_native',
-                    m3u8_id=supplier, fatal=False))
-            # Direct link
-            else:
-                formats.append({
-                    'url': href,
-                    'format_id': supplier or kind or protocol,
-                })
-        elif protocol == 'rtmp':
-            application = connection.get('application', 'ondemand')
-            auth_string = connection.get('authString')
-            identifier = connection.get('identifier')
-            server = connection.get('server')
-            formats.append({
-                'url': '%s://%s/%s?%s' % (protocol, server, application, auth_string),
-                'play_path': identifier,
-                'app': '%s?%s' % (application, auth_string),
-                'page_url': 'http://www.bbc.co.uk',
-                'player_url': 'http://www.bbc.co.uk/emp/releases/iplayer/revisions/617463_618125_4/617463_618125_4_emp.swf',
-                'rtmp_live': False,
-                'ext': 'flv',
-                'format_id': supplier,
-            })
-        return formats
-
     def _extract_items(self, playlist):
         return playlist.findall('./{%s}item' % self._EMP_PLAYLIST_NS)
 
@@ -294,46 +249,6 @@ class BBCCoUkIE(InfoExtractor):
     def _extract_connections(self, media):
         return self._findall_ns(media, './{%s}connection')
 
-    def _extract_video(self, media, programme_id):
-        formats = []
-        vbr = int_or_none(media.get('bitrate'))
-        vcodec = media.get('encoding')
-        service = media.get('service')
-        width = int_or_none(media.get('width'))
-        height = int_or_none(media.get('height'))
-        file_size = int_or_none(media.get('media_file_size'))
-        for connection in self._extract_connections(media):
-            conn_formats = self._extract_connection(connection, programme_id)
-            for format in conn_formats:
-                format.update({
-                    'width': width,
-                    'height': height,
-                    'vbr': vbr,
-                    'vcodec': vcodec,
-                    'filesize': file_size,
-                })
-                if service:
-                    format['format_id'] = '%s_%s' % (service, format['format_id'])
-            formats.extend(conn_formats)
-        return formats
-
-    def _extract_audio(self, media, programme_id):
-        formats = []
-        abr = int_or_none(media.get('bitrate'))
-        acodec = media.get('encoding')
-        service = media.get('service')
-        for connection in self._extract_connections(media):
-            conn_formats = self._extract_connection(connection, programme_id)
-            for format in conn_formats:
-                format.update({
-                    'format_id': '%s_%s' % (service, format['format_id']),
-                    'abr': abr,
-                    'acodec': acodec,
-                    'vcodec': 'none',
-                })
-            formats.extend(conn_formats)
-        return formats
-
     def _get_subtitles(self, media, programme_id):
         subtitles = {}
         for connection in self._extract_connections(media):
@@ -382,10 +297,77 @@ class BBCCoUkIE(InfoExtractor):
 
         for media in self._extract_medias(media_selection):
             kind = media.get('kind')
-            if kind == 'audio':
-                formats.extend(self._extract_audio(media, programme_id))
-            elif kind == 'video':
-                formats.extend(self._extract_video(media, programme_id))
+            if kind in ('video', 'audio'):
+                bitrate = int_or_none(media.get('bitrate'))
+                encoding = media.get('encoding')
+                service = media.get('service')
+                width = int_or_none(media.get('width'))
+                height = int_or_none(media.get('height'))
+                file_size = int_or_none(media.get('media_file_size'))
+                for connection in self._extract_connections(media):
+                    conn_kind = connection.get('kind')
+                    protocol = connection.get('protocol')
+                    supplier = connection.get('supplier')
+                    href = connection.get('href')
+                    transfer_format = connection.get('transferFormat')
+                    format_id = supplier or conn_kind or protocol
+                    if service:
+                        format_id = '%s_%s' % (service, format_id)
+                    # ASX playlist
+                    if supplier == 'asx':
+                        for i, ref in enumerate(self._extract_asx_playlist(connection, programme_id)):
+                            formats.append({
+                                'url': ref,
+                                'format_id': 'ref%s_%s' % (i, format_id),
+                            })
+                    elif transfer_format == 'dash':
+                        formats.extend(self._extract_mpd_formats(
+                            href, programme_id, mpd_id=format_id, fatal=False))
+                    elif transfer_format == 'hls':
+                        formats.extend(self._extract_m3u8_formats(
+                            href, programme_id, ext='mp4', entry_protocol='m3u8_native',
+                            m3u8_id=format_id, fatal=False))
+                    elif transfer_format == 'hds':
+                        formats.extend(self._extract_f4m_formats(
+                            href, programme_id, f4m_id=format_id, fatal=False))
+                    else:
+                        fmt = {
+                            'format_id': format_id,
+                            'filesize': file_size,
+                        }
+                        if kind == 'video':
+                            fmt.update({
+                                'width': width,
+                                'height': height,
+                                'vbr': bitrate,
+                                'vcodec': encoding,
+                            })
+                        else:
+                            fmt.update({
+                                'abr': bitrate,
+                                'acodec': encoding,
+                                'vcodec': 'none',
+                            })
+                        if protocol == 'http':
+                            # Direct link
+                            fmt.update({
+                                'url': href,
+                            })
+                        elif protocol == 'rtmp':
+                            application = connection.get('application', 'ondemand')
+                            auth_string = connection.get('authString')
+                            identifier = connection.get('identifier')
+                            server = connection.get('server')
+                            fmt.update({
+                                'url': '%s://%s/%s?%s' % (protocol, server, application, auth_string),
+                                'play_path': identifier,
+                                'app': '%s?%s' % (application, auth_string),
+                                'page_url': 'http://www.bbc.co.uk',
+                                'player_url': 'http://www.bbc.co.uk/emp/releases/iplayer/revisions/617463_618125_4/617463_618125_4_emp.swf',
+                                'rtmp_live': False,
+                                'ext': 'flv',
+                            })
+                        formats.append(fmt)
             elif kind == 'captions':
                 subtitles = self.extract_subtitles(media, programme_id)
         return formats, subtitles
@@ -820,13 +802,19 @@ class BBCIE(BBCCoUkIE):
                         # http://www.bbc.com/turkce/multimedya/2015/10/151010_vid_ankara_patlama_ani)
                         playlist = data_playable.get('otherSettings', {}).get('playlist', {})
                         if playlist:
-                            for key in ('progressiveDownload', 'streaming'):
+                            entry = None
+                            for key in ('streaming', 'progressiveDownload'):
                                 playlist_url = playlist.get('%sUrl' % key)
                                 if not playlist_url:
                                     continue
                                 try:
-                                    entries.append(self._extract_from_playlist_sxml(
-                                        playlist_url, playlist_id, timestamp))
+                                    info = self._extract_from_playlist_sxml(
+                                        playlist_url, playlist_id, timestamp)
+                                    if not entry:
+                                        entry = info
+                                    else:
+                                        entry['title'] = info['title']
+                                        entry['formats'].extend(info['formats'])
                                 except Exception as e:
                                     # Some playlist URL may fail with 500, at the same time
                                     # the other one may work fine (e.g.
@@ -834,6 +822,9 @@ class BBCIE(BBCCoUkIE):
                                     if isinstance(e.cause, compat_HTTPError) and e.cause.code == 500:
                                         continue
                                     raise
+                            if entry:
+                                self._sort_formats(entry['formats'])
+                                entries.append(entry)
 
         if entries:
             return self.playlist_result(entries, playlist_id, playlist_title, playlist_description)

From c57244cdb10cac6ff88fd4fa430c98f3909c065a Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 6 Aug 2016 18:55:05 +0100
Subject: [PATCH 5670/6219] [common] lower the preference of m3u8 master
 manifest format

---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 70909fc1c..b54a36996 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1140,7 +1140,7 @@ class InfoExtractor(object):
             'url': m3u8_url,
             'ext': ext,
             'protocol': 'm3u8',
-            'preference': preference - 1 if preference else -1,
+            'preference': -100,
             'resolution': 'multiple',
             'format_note': 'Quality selection URL',
         }

From d16b3c6677f1f699635892876f4566962094221d Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 6 Aug 2016 18:58:38 +0100
Subject: [PATCH 5671/6219] [common] extract partOfTVSeries info in json-ld

---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index b54a36996..67f49f51b 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -846,7 +846,7 @@ class InfoExtractor(object):
                     part_of_season = e.get('partOfSeason')
                     if isinstance(part_of_season, dict) and part_of_season.get('@type') == 'TVSeason':
                         info['season_number'] = int_or_none(part_of_season.get('seasonNumber'))
-                    part_of_series = e.get('partOfSeries')
+                    part_of_series = e.get('partOfSeries') or e.get('partOfTVSeries')
                     if isinstance(part_of_series, dict) and part_of_series.get('@type') == 'TVSeries':
                         info['series'] = unescapeHTML(part_of_series.get('name'))
                 elif item_type == 'Article':

From b0af12154e4a76bf7d493c7eb75cdeea6cf8fe17 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 6 Aug 2016 19:24:59 +0100
Subject: [PATCH 5672/6219] [bbc] reduce requests and improve format_id

---
 youtube_dl/extractor/bbc.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 45c562bd8..35d042e4d 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -294,6 +294,7 @@ class BBCCoUkIE(InfoExtractor):
     def _process_media_selector(self, media_selection, programme_id):
         formats = []
         subtitles = None
+        urls = []
 
         for media in self._extract_medias(media_selection):
             kind = media.get('kind')
@@ -305,10 +306,14 @@ class BBCCoUkIE(InfoExtractor):
                 height = int_or_none(media.get('height'))
                 file_size = int_or_none(media.get('media_file_size'))
                 for connection in self._extract_connections(media):
+                    href = connection.get('href')
+                    if href in urls:
+                        continue
+                    if href:
+                        urls.append(href)
                     conn_kind = connection.get('kind')
                     protocol = connection.get('protocol')
                     supplier = connection.get('supplier')
-                    href = connection.get('href')
                     transfer_format = connection.get('transferFormat')
                     format_id = supplier or conn_kind or protocol
                     if service:
@@ -331,6 +336,8 @@ class BBCCoUkIE(InfoExtractor):
                         formats.extend(self._extract_f4m_formats(
                             href, programme_id, f4m_id=format_id, fatal=False))
                     else:
+                        if bitrate:
+                            format_id += '-%d' %  bitrate
                         fmt = {
                             'format_id': format_id,
                             'filesize': file_size,

From ad152e2d954445c1bfa974c5d4a47ea622269d82 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 6 Aug 2016 19:36:12 +0100
Subject: [PATCH 5673/6219] [bbc] fix test

---
 youtube_dl/extractor/bbc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 35d042e4d..7bc7b2ed6 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -578,7 +578,7 @@ class BBCIE(BBCCoUkIE):
         'info_dict': {
             'id': '150615_telabyad_kentin_cogu',
             'ext': 'mp4',
-            'title': "Tel Abyad'da IŞİD bayrağı indirildi YPG bayrağı çekildi",
+            'title': "YPG: Tel Abyad'ın tamamı kontrolümüzde",
             'description': 'md5:33a4805a855c9baf7115fcbde57e7025',
             'timestamp': 1434397334,
             'upload_date': '20150615',

From d3f8e038fea62fc117b7747627eb8d654863d152 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 7 Aug 2016 02:42:58 +0800
Subject: [PATCH 5674/6219] [utils] Add decode_png for openload (#9706)

---
 youtube_dl/utils.py | 108 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 108 insertions(+)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 97ddd9883..ddbfcd2f1 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -47,6 +47,7 @@ from .compat import (
     compat_socket_create_connection,
     compat_str,
     compat_struct_pack,
+    compat_struct_unpack,
     compat_urllib_error,
     compat_urllib_parse,
     compat_urllib_parse_urlencode,
@@ -2969,3 +2970,110 @@ def parse_m3u8_attributes(attrib):
 
 def urshift(val, n):
     return val >> n if val >= 0 else (val + 0x100000000) >> n
+
+
+# Based on png2str() written by @gdkchan and improved by @yokrysty
+# Originally posted at https://github.com/rg3/youtube-dl/issues/9706
+def decode_png(png_data):
+    # Reference: https://www.w3.org/TR/PNG/
+    header = png_data[8:]
+
+    if png_data[:8] != b'\x89PNG\x0d\x0a\x1a\x0a' or header[4:8] != b'IHDR':
+        raise IOError('Not a valid PNG file.')
+
+    int_map = {1: '>B', 2: '>H', 4: '>I'}
+    unpack_integer = lambda x: compat_struct_unpack(int_map[len(x)], x)[0]
+
+    chunks = []
+
+    while header:
+        length = unpack_integer(header[:4])
+        header = header[4:]
+
+        chunk_type = header[:4]
+        header = header[4:]
+
+        chunk_data = header[:length]
+        header = header[length:]
+
+        header = header[4:]  # Skip CRC
+
+        chunks.append({
+            'type': chunk_type,
+            'length': length,
+            'data': chunk_data
+        })
+
+    ihdr = chunks[0]['data']
+
+    width = unpack_integer(ihdr[:4])
+    height = unpack_integer(ihdr[4:8])
+
+    idat = b''
+
+    for chunk in chunks:
+        if chunk['type'] == b'IDAT':
+            idat += chunk['data']
+
+    if not idat:
+        raise IOError('Unable to read PNG data.')
+
+    decompressed_data = bytearray(zlib.decompress(idat))
+
+    stride = width * 3
+    pixels = []
+
+    def _get_pixel(idx):
+        x = idx % stride
+        y = idx // stride
+        return pixels[y][x]
+
+    for y in range(height):
+        basePos = y * (1 + stride)
+        filter_type = decompressed_data[basePos]
+
+        current_row = []
+
+        pixels.append(current_row)
+
+        for x in range(stride):
+            color = decompressed_data[1 + basePos + x]
+            basex = y * stride + x
+            left = 0
+            up = 0
+
+            if x > 2:
+                left = _get_pixel(basex - 3)
+            if y > 0:
+                up = _get_pixel(basex - stride)
+
+            if filter_type == 1:  # Sub
+                color = (color + left) & 0xff
+            elif filter_type == 2:  # Up
+                color = (color + up) & 0xff
+            elif filter_type == 3:  # Average
+                color = (color + ((left + up) >> 1)) & 0xff
+            elif filter_type == 4:  # Paeth
+                a = left
+                b = up
+                c = 0
+
+                if x > 2 and y > 0:
+                    c = _get_pixel(basex - stride - 3)
+
+                p = a + b - c
+
+                pa = abs(p - a)
+                pb = abs(p - b)
+                pc = abs(p - c)
+
+                if pa <= pb and pa <= pc:
+                    color = (color + a) & 0xff
+                elif pb <= pc:
+                    color = (color + b) & 0xff
+                else:
+                    color = (color + c) & 0xff
+
+            current_row.append(color)
+
+    return width, height, pixels

From c1decda58c812b3d0a3d4dfa998e7d8bd8f99203 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 7 Aug 2016 02:44:15 +0800
Subject: [PATCH 5675/6219] [openload] Fix extraction (closes #9706)

---
 youtube_dl/extractor/openload.py | 133 +++++++++++++++----------------
 1 file changed, 63 insertions(+), 70 deletions(-)

diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py
index 6415b8fdc..4e80ca9ff 100644
--- a/youtube_dl/extractor/openload.py
+++ b/youtube_dl/extractor/openload.py
@@ -1,15 +1,14 @@
 # coding: utf-8
-from __future__ import unicode_literals
+from __future__ import unicode_literals, division
 
-import re
+import math
 
 from .common import InfoExtractor
 from ..compat import compat_chr
 from ..utils import (
+    decode_png,
     determine_ext,
-    encode_base_n,
     ExtractorError,
-    mimetype2ext,
 )
 
 
@@ -41,60 +40,6 @@ class OpenloadIE(InfoExtractor):
         'only_matching': True,
     }]
 
-    @staticmethod
-    def openload_level2_debase(m):
-        radix, num = int(m.group(1)) + 27, int(m.group(2))
-        return '"' + encode_base_n(num, radix) + '"'
-
-    @classmethod
-    def openload_level2(cls, txt):
-        # The function name is ǃ \u01c3
-        # Using escaped unicode literals does not work in Python 3.2
-        return re.sub(r'ǃ\((\d+),(\d+)\)', cls.openload_level2_debase, txt, re.UNICODE).replace('"+"', '')
-
-    # Openload uses a variant of aadecode
-    # openload_decode and related functions are originally written by
-    # vitas@matfyz.cz and released with public domain
-    # See https://github.com/rg3/youtube-dl/issues/8489
-    @classmethod
-    def openload_decode(cls, txt):
-        symbol_table = [
-            ('_', '(ﾟДﾟ) [ﾟΘﾟ]'),
-            ('a', '(ﾟДﾟ) [ﾟωﾟﾉ]'),
-            ('b', '(ﾟДﾟ) [ﾟΘﾟﾉ]'),
-            ('c', '(ﾟДﾟ) [\'c\']'),
-            ('d', '(ﾟДﾟ) [ﾟｰﾟﾉ]'),
-            ('e', '(ﾟДﾟ) [ﾟДﾟﾉ]'),
-            ('f', '(ﾟДﾟ) [1]'),
-
-            ('o', '(ﾟДﾟ) [\'o\']'),
-            ('u', '(oﾟｰﾟo)'),
-            ('c', '(ﾟДﾟ) [\'c\']'),
-
-            ('7', '((ﾟｰﾟ) + (o^_^o))'),
-            ('6', '((o^_^o) +(o^_^o) +(c^_^o))'),
-            ('5', '((ﾟｰﾟ) + (ﾟΘﾟ))'),
-            ('4', '(-~3)'),
-            ('3', '(-~-~1)'),
-            ('2', '(-~1)'),
-            ('1', '(-~0)'),
-            ('0', '((c^_^o)-(c^_^o))'),
-        ]
-        delim = '(ﾟДﾟ)[ﾟεﾟ]+'
-        ret = ''
-        for aachar in txt.split(delim):
-            for val, pat in symbol_table:
-                aachar = aachar.replace(pat, val)
-            aachar = aachar.replace('+ ', '')
-            m = re.match(r'^\d+', aachar)
-            if m:
-                ret += compat_chr(int(m.group(0), 8))
-            else:
-                m = re.match(r'^u([\da-f]+)', aachar)
-                if m:
-                    ret += compat_chr(int(m.group(1), 16))
-        return cls.openload_level2(ret)
-
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
@@ -102,29 +47,77 @@ class OpenloadIE(InfoExtractor):
         if 'File not found' in webpage:
             raise ExtractorError('File not found', expected=True)
 
-        code = self._search_regex(
-            r'</video>\s*</div>\s*<script[^>]+>[^>]+</script>\s*<script[^>]+>([^<]+)</script>',
-            webpage, 'JS code')
+        # The following extraction logic is proposed by @Belderak and @gdkchan
+        # and declared to be used freely in youtube-dl
+        # See https://github.com/rg3/youtube-dl/issues/9706
 
-        decoded = self.openload_decode(code)
+        numbers_js = self._download_webpage(
+            'https://openload.co/assets/js/obfuscator/n.js', video_id,
+            note='Downloading signature numbers')
+        signums = self._search_regex(
+            r'window\.signatureNumbers\s*=\s*[\'"](?P<data>[a-z]+)[\'"]',
+            numbers_js, 'signature numbers', group='data')
 
-        video_url = self._search_regex(
-            r'return\s+"(https?://[^"]+)"', decoded, 'video URL')
+        linkimg_uri = self._search_regex(
+            r'<img[^>]+id="linkimg"[^>]+src="([^"]+)"', webpage, 'link image')
+        linkimg = self._request_webpage(
+            linkimg_uri, video_id, note=False).read()
+
+        width, height, pixels = decode_png(linkimg)
+
+        output = ''
+        for y in range(height):
+            for x in range(width):
+                r, g, b = pixels[y][3 * x:3 * x + 3]
+                if r == 0 and g == 0 and b == 0:
+                    break
+                else:
+                    output += compat_chr(r)
+                    output += compat_chr(g)
+                    output += compat_chr(b)
+
+        img_str_length = len(output) // 200
+        img_str = [[0 for x in range(img_str_length)] for y in range(10)]
+
+        sig_str_length = len(signums) // 260
+        sig_str = [[0 for x in range(sig_str_length)] for y in range(10)]
+
+        for i in range(10):
+            for j in range(img_str_length):
+                begin = i * img_str_length * 20 + j * 20
+                img_str[i][j] = output[begin:begin + 20]
+            for j in range(sig_str_length):
+                begin = i * sig_str_length * 26 + j * 26
+                sig_str[i][j] = signums[begin:begin + 26]
+
+        parts = []
+        # TODO: find better names for str_, chr_ and sum_
+        str_ = ''
+        for i in [2, 3, 5, 7]:
+            str_ = ''
+            sum_ = float(99)
+            for j in range(len(sig_str[i])):
+                for chr_idx in range(len(img_str[i][j])):
+                    if sum_ > float(122):
+                        sum_ = float(98)
+                    chr_ = compat_chr(int(math.floor(sum_)))
+                    if sig_str[i][j][chr_idx] == chr_ and j >= len(str_):
+                        sum_ += float(2.5)
+                        str_ += img_str[i][j][chr_idx]
+            parts.append(str_.replace(',', ''))
+
+        video_url = 'https://openload.co/stream/%s~%s~%s~%s' % (parts[3], parts[1], parts[2], parts[0])
 
         title = self._og_search_title(webpage, default=None) or self._search_regex(
             r'<span[^>]+class=["\']title["\'][^>]*>([^<]+)', webpage,
             'title', default=None) or self._html_search_meta(
             'description', webpage, 'title', fatal=True)
 
-        ext = mimetype2ext(self._search_regex(
-            r'window\.vt\s*=\s*(["\'])(?P<mimetype>.+?)\1', decoded,
-            'mimetype', default=None, group='mimetype')) or determine_ext(
-            video_url, 'mp4')
-
         return {
             'id': video_id,
             'title': title,
-            'ext': ext,
             'thumbnail': self._og_search_thumbnail(webpage, default=None),
             'url': video_url,
+            # Seems all videos have extensions in their titles
+            'ext': determine_ext(title),
         }

From e37b54b140c552ee3d751b868ecc0a71df13829f Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 6 Aug 2016 20:53:13 +0100
Subject: [PATCH 5676/6219] [fox] fix theplatform release url query

---
 youtube_dl/extractor/fox.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/fox.py b/youtube_dl/extractor/fox.py
index 95c1abf94..9f406b17e 100644
--- a/youtube_dl/extractor/fox.py
+++ b/youtube_dl/extractor/fox.py
@@ -2,7 +2,10 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..utils import smuggle_url
+from ..utils import (
+    smuggle_url,
+    update_url_query,
+)
 
 
 class FOXIE(InfoExtractor):
@@ -29,11 +32,12 @@ class FOXIE(InfoExtractor):
 
         release_url = self._parse_json(self._search_regex(
             r'"fox_pdk_player"\s*:\s*({[^}]+?})', webpage, 'fox_pdk_player'),
-            video_id)['release_url'] + '&switch=http'
+            video_id)['release_url']
 
         return {
             '_type': 'url_transparent',
             'ie_key': 'ThePlatform',
-            'url': smuggle_url(release_url, {'force_smil_url': True}),
+            'url': smuggle_url(update_url_query(
+                release_url, {'switch': 'http'}), {'force_smil_url': True}),
             'id': video_id,
         }

From b47a75017bc4b4739ccebb3e8bde0642981968b0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 7 Aug 2016 15:57:42 +0700
Subject: [PATCH 5677/6219] [tnaflix] Fix metadata extraction (Closes #10249)

---
 youtube_dl/extractor/tnaflix.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/tnaflix.py b/youtube_dl/extractor/tnaflix.py
index 78174178e..74a0c8f17 100644
--- a/youtube_dl/extractor/tnaflix.py
+++ b/youtube_dl/extractor/tnaflix.py
@@ -189,9 +189,9 @@ class TNAFlixNetworkEmbedIE(TNAFlixNetworkBaseIE):
 class TNAFlixIE(TNAFlixNetworkBaseIE):
     _VALID_URL = r'https?://(?:www\.)?tnaflix\.com/[^/]+/(?P<display_id>[^/]+)/video(?P<id>\d+)'
 
-    _TITLE_REGEX = r'<title>(.+?) - TNAFlix Porn Videos</title>'
-    _DESCRIPTION_REGEX = r'<meta[^>]+name="description"[^>]+content="([^"]+)"'
-    _UPLOADER_REGEX = r'<i>\s*Verified Member\s*</i>\s*<h1>(.+?)</h1>'
+    _TITLE_REGEX = r'<title>(.+?) - (?:TNAFlix Porn Videos|TNAFlix\.com)</title>'
+    _DESCRIPTION_REGEX = r'(?s)>Description:</[^>]+>(.+?)<'
+    _UPLOADER_REGEX = r'<i>\s*Verified Member\s*</i>\s*<h\d+>(.+?)<'
     _CATEGORIES_REGEX = r'(?s)<span[^>]*>Categories:</span>(.+?)</div>'
 
     _TESTS = [{

From a1aadd09a4721ffd4d782aae4ddeae2699ca17fa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 7 Aug 2016 15:59:05 +0700
Subject: [PATCH 5678/6219] [tnaflixnetworkbase] Improve title extraction

---
 youtube_dl/extractor/tnaflix.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/tnaflix.py b/youtube_dl/extractor/tnaflix.py
index 74a0c8f17..7ddf77767 100644
--- a/youtube_dl/extractor/tnaflix.py
+++ b/youtube_dl/extractor/tnaflix.py
@@ -118,8 +118,12 @@ class TNAFlixNetworkBaseIE(InfoExtractor):
             xpath_text(cfg_xml, './startThumb', 'thumbnail'), 'http:')
         thumbnails = self._extract_thumbnails(cfg_xml)
 
-        title = self._html_search_regex(
-            self._TITLE_REGEX, webpage, 'title') if self._TITLE_REGEX else self._og_search_title(webpage)
+        title = None
+        if self._TITLE_REGEX:
+            title = self._html_search_regex(
+                self._TITLE_REGEX, webpage, 'title', default=None)
+        if not title:
+            title = self._og_search_title(webpage)
 
         age_limit = self._rta_search(webpage) or 18
 

From 37768f92422c2cf61a961dbaf54d61dabd364506 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 7 Aug 2016 10:58:11 +0100
Subject: [PATCH 5679/6219] [common] correctly lower the preference of m3u8
 master manifest format

---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 67f49f51b..0891309dd 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1140,7 +1140,7 @@ class InfoExtractor(object):
             'url': m3u8_url,
             'ext': ext,
             'protocol': 'm3u8',
-            'preference': -100,
+            'preference': preference - 100 if preference else -100,
             'resolution': 'multiple',
             'format_note': 'Quality selection URL',
         }

From f9622868e722a1863de257c7f3cbc3076eea6e83 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 7 Aug 2016 11:14:15 +0100
Subject: [PATCH 5680/6219] [bbc] preserve format_id backward compatibility

---
 youtube_dl/extractor/bbc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 7bc7b2ed6..b6c240832 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -336,7 +336,7 @@ class BBCCoUkIE(InfoExtractor):
                         formats.extend(self._extract_f4m_formats(
                             href, programme_id, f4m_id=format_id, fatal=False))
                     else:
-                        if bitrate:
+                        if not service and not supplier and bitrate:
                             format_id += '-%d' %  bitrate
                         fmt = {
                             'format_id': format_id,

From 9fb64c04cdbe1b58f968fa80489168173ac7e565 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 7 Aug 2016 18:01:50 +0700
Subject: [PATCH 5681/6219] [bbc] Add support for morph embeds (Closes #10239)

---
 youtube_dl/extractor/bbc.py | 64 +++++++++++++++++++++++++++++++++++++
 1 file changed, 64 insertions(+)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index b6c240832..0ee096dda 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -5,11 +5,13 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
+    dict_get,
     ExtractorError,
     float_or_none,
     int_or_none,
     parse_duration,
     parse_iso8601,
+    try_get,
     unescapeHTML,
 )
 from ..compat import (
@@ -643,6 +645,24 @@ class BBCIE(BBCCoUkIE):
             # rtmp download
             'skip_download': True,
         }
+    }, {
+        # single video embedded with Morph
+        'url': 'http://www.bbc.co.uk/sport/live/olympics/36895975',
+        'info_dict': {
+            'id': 'p041vhd0',
+            'ext': 'mp4',
+            'title': "Nigeria v Japan - Men's First Round",
+            'description': 'Live coverage of the first round from Group B at the Amazonia Arena.',
+            'duration': 7980,
+            'uploader': 'BBC Sport',
+            'uploader_id': 'bbc_sport',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+            'proxy': '5.101.173.158:8080',
+        },
+        'skip': 'Georestricted to UK',
     }, {
         # single video with playlist.sxml URL in playlist param
         'url': 'http://www.bbc.com/sport/0/football/33653409',
@@ -864,6 +884,50 @@ class BBCIE(BBCCoUkIE):
                 'subtitles': subtitles,
             }
 
+        # Morph based embed (e.g. http://www.bbc.co.uk/sport/live/olympics/36895975)
+        # There are several setPayload calls may be present but the video
+        # seems to be always related to the first one
+        morph_payload = self._parse_json(
+            self._search_regex(
+                r'Morph\.setPayload\([^,]+,\s*({.+?})\);',
+                webpage, 'morph payload', default='{}'),
+            playlist_id, fatal=False)
+        if morph_payload:
+            components = try_get(morph_payload, lambda x: x['body']['components'], list) or []
+            for component in components:
+                if not isinstance(component, dict):
+                    continue
+                lead_media = try_get(component, lambda x: x['props']['leadMedia'], dict)
+                if not lead_media:
+                    continue
+                identifiers = lead_media.get('identifiers')
+                if not identifiers or not isinstance(identifiers, dict):
+                    continue
+                programme_id = identifiers.get('vpid') or identifiers.get('playablePid')
+                if not programme_id:
+                    continue
+                title = lead_media.get('title') or self._og_search_title(webpage)
+                formats, subtitles = self._download_media_selector(programme_id)
+                self._sort_formats(formats)
+                description = lead_media.get('summary')
+                uploader = lead_media.get('masterBrand')
+                uploader_id = lead_media.get('mid')
+                duration = None
+                duration_d = lead_media.get('duration')
+                if isinstance(duration_d, dict):
+                    duration = parse_duration(dict_get(
+                        duration_d, ('rawDuration', 'formattedDuration', 'spokenDuration')))
+                return {
+                    'id': programme_id,
+                    'title': title,
+                    'description': description,
+                    'duration': duration,
+                    'uploader': uploader,
+                    'uploader_id': uploader_id,
+                    'formats': formats,
+                    'subtitles': subtitles,
+                }
+
         def extract_all(pattern):
             return list(filter(None, map(
                 lambda s: self._parse_json(s, playlist_id, fatal=False),

From aaa42cf0cf3e8c09209908474998fbd3dc86b91a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 7 Aug 2016 18:05:13 +0700
Subject: [PATCH 5682/6219] [bbc] PEP 8

---
 youtube_dl/extractor/bbc.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 0ee096dda..d059e02a3 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -339,7 +339,7 @@ class BBCCoUkIE(InfoExtractor):
                             href, programme_id, f4m_id=format_id, fatal=False))
                     else:
                         if not service and not supplier and bitrate:
-                            format_id += '-%d' %  bitrate
+                            format_id += '-%d' % bitrate
                         fmt = {
                             'format_id': format_id,
                             'filesize': file_size,
@@ -945,7 +945,7 @@ class BBCIE(BBCCoUkIE):
             r'setPlaylist\("(%s)"\)' % EMBED_URL, webpage))
         if entries:
             return self.playlist_result(
-                [self.url_result(entry, 'BBCCoUk') for entry in entries],
+                [self.url_result(entry_, 'BBCCoUk') for entry_ in entries],
                 playlist_id, playlist_title, playlist_description)
 
         # Multiple video article (e.g. http://www.bbc.com/news/world-europe-32668511)

From 998f0944526658ae19f29fd4ca04391a5a1ac027 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 7 Aug 2016 18:13:05 +0700
Subject: [PATCH 5683/6219] [bbc] Remove proxy from test

---
 youtube_dl/extractor/bbc.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index d059e02a3..b29e05970 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -660,7 +660,6 @@ class BBCIE(BBCCoUkIE):
         'params': {
             # m3u8 download
             'skip_download': True,
-            'proxy': '5.101.173.158:8080',
         },
         'skip': 'Georestricted to UK',
     }, {

From 958849275f1c6072c712e8d611294a762fadc7f0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 7 Aug 2016 19:04:22 +0700
Subject: [PATCH 5684/6219] [extractor/generic] Make _search_json_ld non fatal

---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index e89a03760..44c6c354c 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -2241,7 +2241,7 @@ class GenericIE(InfoExtractor):
 
         # Looking for http://schema.org/VideoObject
         json_ld = self._search_json_ld(
-            webpage, video_id, default=None, expected_type='VideoObject')
+            webpage, video_id, fatal=False, expected_type='VideoObject')
         if json_ld and json_ld.get('url'):
             info_dict.update({
                 'title': video_title or info_dict['title'],

From d34995a9e3f596c4dd80178d99f7bd8dbc748e2b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 7 Aug 2016 19:06:55 +0700
Subject: [PATCH 5685/6219] [flipagram] Make _search_json_ld non fatal

---
 youtube_dl/extractor/flipagram.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/flipagram.py b/youtube_dl/extractor/flipagram.py
index acb6133ff..634f5fe3b 100644
--- a/youtube_dl/extractor/flipagram.py
+++ b/youtube_dl/extractor/flipagram.py
@@ -48,7 +48,7 @@ class FlipagramIE(InfoExtractor):
         flipagram = video_data['flipagram']
         video = flipagram['video']
 
-        json_ld = self._search_json_ld(webpage, video_id, default=False)
+        json_ld = self._search_json_ld(webpage, video_id, fatal=False)
         title = json_ld.get('title') or flipagram['captionText']
         description = json_ld.get('description') or flipagram.get('captionText')
 

From a8795327cae2bfce299d770039db40a3ac4df2e3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 7 Aug 2016 20:45:18 +0700
Subject: [PATCH 5686/6219] [utils] Add support TV Parental Guidelines ratings
 in parse_age_limit

---
 test/test_utils.py  | 15 +++++++++++++++
 youtube_dl/utils.py | 20 ++++++++++++++++++--
 2 files changed, 33 insertions(+), 2 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 5a2ae4a1e..724346886 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -42,6 +42,7 @@ from youtube_dl.utils import (
     ohdave_rsa_encrypt,
     OnDemandPagedList,
     orderedSet,
+    parse_age_limit,
     parse_duration,
     parse_filesize,
     parse_count,
@@ -432,6 +433,20 @@ class TestUtil(unittest.TestCase):
             url_basename('http://media.w3.org/2010/05/sintel/trailer.mp4'),
             'trailer.mp4')
 
+    def test_parse_age_limit(self):
+        self.assertEqual(parse_age_limit(None), None)
+        self.assertEqual(parse_age_limit(False), None)
+        self.assertEqual(parse_age_limit('invalid'), None)
+        self.assertEqual(parse_age_limit(0), 0)
+        self.assertEqual(parse_age_limit(18), 18)
+        self.assertEqual(parse_age_limit(21), 21)
+        self.assertEqual(parse_age_limit(22), None)
+        self.assertEqual(parse_age_limit('18'), 18)
+        self.assertEqual(parse_age_limit('18+'), 18)
+        self.assertEqual(parse_age_limit('PG-13'), 13)
+        self.assertEqual(parse_age_limit('TV-14'), 14)
+        self.assertEqual(parse_age_limit('TV-MA'), 17)
+
     def test_parse_duration(self):
         self.assertEqual(parse_duration(None), None)
         self.assertEqual(parse_duration(False), None)
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index ddbfcd2f1..c50238ba1 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1984,11 +1984,27 @@ US_RATINGS = {
 }
 
 
+TV_PARENTAL_GUIDELINES = {
+    'TV-Y': 0,
+    'TV-Y7': 7,
+    'TV-G': 0,
+    'TV-PG': 0,
+    'TV-14': 14,
+    'TV-MA': 17,
+}
+
+
 def parse_age_limit(s):
-    if s is None:
+    if type(s) == int:
+        return s if 0 <= s <= 21 else None
+    if not isinstance(s, compat_basestring):
         return None
     m = re.match(r'^(?P<age>\d{1,2})\+?$', s)
-    return int(m.group('age')) if m else US_RATINGS.get(s)
+    if m:
+        return int(m.group('age'))
+    if s in US_RATINGS:
+        return US_RATINGS[s]
+    return TV_PARENTAL_GUIDELINES.get(s)
 
 
 def strip_jsonp(code):

From d92cb463052c262a6750f3a5bfc7d564a527ee9f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 7 Aug 2016 20:57:05 +0700
Subject: [PATCH 5687/6219] [discoverygo] Add extractor (Closes #10245)

---
 youtube_dl/extractor/discoverygo.py | 98 +++++++++++++++++++++++++++++
 youtube_dl/extractor/extractors.py  |  1 +
 2 files changed, 99 insertions(+)
 create mode 100644 youtube_dl/extractor/discoverygo.py

diff --git a/youtube_dl/extractor/discoverygo.py b/youtube_dl/extractor/discoverygo.py
new file mode 100644
index 000000000..adb68b96c
--- /dev/null
+++ b/youtube_dl/extractor/discoverygo.py
@@ -0,0 +1,98 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..compat import compat_str
+from ..utils import (
+    extract_attributes,
+    int_or_none,
+    parse_age_limit,
+    unescapeHTML,
+)
+
+
+class DiscoveryGoIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?discoverygo\.com/(?:[^/]+/)*(?P<id>[^/?#&]+)'
+    _TEST = {
+        'url': 'https://www.discoverygo.com/love-at-first-kiss/kiss-first-ask-questions-later/',
+        'info_dict': {
+            'id': '57a33c536b66d1cd0345eeb1',
+            'ext': 'mp4',
+            'title': 'Kiss First, Ask Questions Later!',
+            'description': 'md5:fe923ba34050eae468bffae10831cb22',
+            'duration': 2579,
+            'series': 'Love at First Kiss',
+            'season_number': 1,
+            'episode_number': 1,
+            'age_limit': 14,
+        },
+    }
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, display_id)
+
+        container = extract_attributes(
+            self._search_regex(
+                r'(<div[^>]+class=["\']video-player-container[^>]+>)',
+                webpage, 'video container'))
+
+        video = self._parse_json(
+            unescapeHTML(container.get('data-video') or container.get('data-json')),
+            display_id)
+
+        title = video['name']
+
+        stream = video['stream']
+        STREAM_URL_SUFFIX = 'streamUrl'
+        formats = []
+        for stream_kind in ('', 'hds'):
+            suffix = STREAM_URL_SUFFIX.capitalize() if stream_kind else STREAM_URL_SUFFIX
+            stream_url = stream.get('%s%s' % (stream_kind, suffix))
+            if not stream_url:
+                continue
+            if stream_kind == '':
+                formats.extend(self._extract_m3u8_formats(
+                    stream_url, display_id, 'mp4', entry_protocol='m3u8_native',
+                    m3u8_id='hls', fatal=False))
+            elif stream_kind == 'hds':
+                formats.extend(self._extract_f4m_formats(
+                    stream_url, display_id, f4m_id=stream_kind, fatal=False))
+        self._sort_formats(formats)
+
+        video_id = video.get('id') or display_id
+        description = video.get('description', {}).get('detailed')
+        duration = int_or_none(video.get('duration'))
+
+        series = video.get('show', {}).get('name')
+        season_number = int_or_none(video.get('season', {}).get('number'))
+        episode_number = int_or_none(video.get('episodeNumber'))
+
+        tags = video.get('tags')
+        age_limit = parse_age_limit(video.get('parental', {}).get('rating'))
+
+        subtitles = {}
+        captions = stream.get('captions')
+        if isinstance(captions, list):
+            for caption in captions:
+                subtitle_url = caption.get('fileUrl')
+                if (not subtitle_url or not isinstance(subtitle_url, compat_str) or
+                        not subtitle_url.startswith('http')):
+                    continue
+                lang = caption.get('fileLang', 'en')
+                subtitles.setdefault(lang, []).append({'url': subtitle_url})
+
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            'title': title,
+            'description': description,
+            'duration': duration,
+            'series': series,
+            'season_number': season_number,
+            'episode_number': episode_number,
+            'tags': tags,
+            'age_limit': age_limit,
+            'formats': formats,
+            'subtitles': subtitles,
+        }
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 11b64eeaa..c2c4617ee 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -221,6 +221,7 @@ from .dvtv import DVTVIE
 from .dumpert import DumpertIE
 from .defense import DefenseGouvFrIE
 from .discovery import DiscoveryIE
+from .discoverygo import DiscoveryGoIE
 from .dispeak import DigitallySpeakingIE
 from .dropbox import DropboxIE
 from .dw import (

From 845dfcdc40c20ea43f1f2bed9ccbc34e1994a89e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 7 Aug 2016 21:10:48 +0700
Subject: [PATCH 5688/6219] [ChangeLog] Actualize

---
 ChangeLog | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 9fd78cdda..c4743df6d 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,29 @@
+version <unreleased>
+
+Core
++ Add support for TV Parental Guidelines ratings in parse_age_limit
++ Add decode_png (#9706)
++ Add support for partOfTVSeries in JSON-LD
+* Lower master M3U8 manifest preference for better format sorting
+
+Extractors
++ [discoverygo] Add extractor (#10245)
+* [flipagram] Make JSON-LD extraction non fatal
+* [generic] Make JSON-LD extraction non fatal
++ [bbc] Add support for morph embeds (#10239)
+* [tnaflixnetworkbase] Improve title extraction
+* [tnaflix] Fix metadata extraction (#10249)
+* [fox] Fix theplatform release URL query
+* [openload] Fix extraction (#9706)
+* [bbc] Skip duplicate manifest URLs
+* [bbc] Improve format code
++ [bbc] Add support for DASH and F4M
+* [bbc] Improve format sorting and listing
+* [bbc] Improve playlist extraction
++ [pokemon] Add extractor (#10093)
++ [condenast] Add fallback scenario for video info extraction
+
+
 version 2016.08.06
 
 Core

From 4a01befb347b35f36dda4f344b7b502e53253da7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 7 Aug 2016 21:12:41 +0700
Subject: [PATCH 5689/6219] release 2016.08.07

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 2 ++
 youtube_dl/version.py     | 2 +-
 4 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 7241840c5..2319e45df 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.08.06*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.08.06**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.08.07*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.08.07**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.08.06
+[debug] youtube-dl version 2016.08.07
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index c4743df6d..7803ec1e9 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2016.08.07
 
 Core
 + Add support for TV Parental Guidelines ratings in parse_age_limit
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 2c8f950df..3608e1807 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -182,6 +182,7 @@
  - **DigitallySpeaking**
  - **Digiteka**
  - **Discovery**
+ - **DiscoveryGo**
  - **Dotsub**
  - **DouyuTV**: 斗鱼
  - **DPlay**
@@ -518,6 +519,7 @@
  - **plus.google**: Google Plus
  - **pluzz.francetv.fr**
  - **podomatic**
+ - **Pokemon**
  - **PolskieRadio**
  - **PornHd**
  - **PornHub**: PornHub and Thumbzilla
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index a4654fa59..b48552031 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.08.06'
+__version__ = '2016.08.07'

From b2bd968f4b6795ffe3fc2d923cc73171687c2980 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 7 Aug 2016 22:59:34 +0800
Subject: [PATCH 5690/6219] [kuwo:singer] Fix extraction

---
 ChangeLog                    | 5 +++++
 youtube_dl/extractor/kuwo.py | 6 ++++--
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 7803ec1e9..32a96432b 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,8 @@
+version <unreleased>
+
+Extractors
+* [kuwo:singer] Fix extraction
+
 version 2016.08.07
 
 Core
diff --git a/youtube_dl/extractor/kuwo.py b/youtube_dl/extractor/kuwo.py
index b1d460599..0eeb9ffeb 100644
--- a/youtube_dl/extractor/kuwo.py
+++ b/youtube_dl/extractor/kuwo.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import compat_urlparse
 from ..utils import (
     get_element_by_id,
     clean_html,
@@ -242,8 +243,9 @@ class KuwoSingerIE(InfoExtractor):
                 query={'artistId': artist_id, 'pn': page_num, 'rn': self.PAGE_SIZE})
 
             return [
-                self.url_result(song_url, 'Kuwo') for song_url in re.findall(
-                    r'<div[^>]+class="name"><a[^>]+href="(http://www\.kuwo\.cn/yinyue/\d+)',
+                self.url_result(compat_urlparse.urljoin(url, song_url), 'Kuwo')
+                for song_url in re.findall(
+                    r'<div[^>]+class="name"><a[^>]+href="(/yinyue/\d+)',
                     webpage)
             ]
 

From d21a661bb4cbdb92f4db588e3801bb19d4fa96b2 Mon Sep 17 00:00:00 2001
From: Charlie Le <chtle@ucdavis.edu>
Date: Sun, 7 Aug 2016 13:38:10 -0700
Subject: [PATCH 5691/6219] [README.md] Update Options Link

The link references a bad anchor. The updated link now references the correct anchor.
---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index a9f3001a6..b42d5c730 100644
--- a/README.md
+++ b/README.md
@@ -1196,7 +1196,7 @@ Make sure that someone has not already opened the issue you're trying to open. S
 
 ###  Why are existing options not enough?
 
-Before requesting a new feature, please have a quick peek at [the list of supported options](https://github.com/rg3/youtube-dl/blob/master/README.md#synopsis). Many feature requests are for features that actually exist already! Please, absolutely do show off your work in the issue report and detail how the existing similar options do *not* solve your problem.
+Before requesting a new feature, please have a quick peek at [the list of supported options](https://github.com/rg3/youtube-dl/blob/master/README.md#options). Many feature requests are for features that actually exist already! Please, absolutely do show off your work in the issue report and detail how the existing similar options do *not* solve your problem.
 
 ###  Is there enough context in your bug report?
 

From e1f93a0a76f054f537c9103a8958e1c1ab631fa5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Petr=20Zvoni=CC=81c=CC=8Cek?= <zvonicek@gmail.com>
Date: Sun, 7 Aug 2016 17:35:54 +0200
Subject: [PATCH 5692/6219] [rozhlas] Add new extractor

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/rozhlas.py    | 34 ++++++++++++++++++++++++++++++
 2 files changed, 35 insertions(+)
 create mode 100644 youtube_dl/extractor/rozhlas.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index c2c4617ee..7ede6afda 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -696,6 +696,7 @@ from .rockstargames import RockstarGamesIE
 from .roosterteeth import RoosterTeethIE
 from .rottentomatoes import RottenTomatoesIE
 from .roxwel import RoxwelIE
+from .rozhlas import RozhlasIE
 from .rtbf import RTBFIE
 from .rte import RteIE, RteRadioIE
 from .rtlnl import RtlNlIE
diff --git a/youtube_dl/extractor/rozhlas.py b/youtube_dl/extractor/rozhlas.py
new file mode 100644
index 000000000..ef99b5b1f
--- /dev/null
+++ b/youtube_dl/extractor/rozhlas.py
@@ -0,0 +1,34 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class RozhlasIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?prehravac\.rozhlas\.cz/audio/(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'http://prehravac.rozhlas.cz/audio/3421320',
+        'md5': '504c902dbc9e9a1fd50326eccf02a7e2',
+        'info_dict': {
+            'id': '3421320',
+            'ext': 'mp3',
+            'title': 'Echo Pavla Klusáka (30.06.2015 21:00)',
+            'description': 'Osmdesátiny Terryho Rileyho jsou skvělou příležitostí proletět se elektronickými i akustickými díly zakladatatele minimalismu, který je aktivní už přes padesát let'
+        }
+    }
+
+    def _real_extract(self, url):
+        audio_id = self._match_id(url)
+        webpage = self._download_webpage(url, audio_id)
+
+        title = self._html_search_regex(r'<h3>(.+?)</h3>', webpage, 'title')
+        description = self._html_search_regex(r'<p title="(.+?)">', webpage, 'description', fatal=False)
+
+        url = 'http://media.rozhlas.cz/_audio/' + audio_id + '.mp3'
+
+        return {
+            'id': audio_id,
+            'url': url,
+            'title': title,
+            'description': description,
+        }

From de02d1f4e94a50eaad0f268fffec6880e9d61d2f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 8 Aug 2016 04:58:02 +0700
Subject: [PATCH 5693/6219] [rozhlas] Fix regexes and improve extraction
 (Closes #10253)

---
 youtube_dl/extractor/rozhlas.py | 30 +++++++++++++++++++++++-------
 1 file changed, 23 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/rozhlas.py b/youtube_dl/extractor/rozhlas.py
index ef99b5b1f..f8eda8dea 100644
--- a/youtube_dl/extractor/rozhlas.py
+++ b/youtube_dl/extractor/rozhlas.py
@@ -2,11 +2,15 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    remove_start,
+)
 
 
 class RozhlasIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?prehravac\.rozhlas\.cz/audio/(?P<id>[0-9]+)'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://prehravac.rozhlas.cz/audio/3421320',
         'md5': '504c902dbc9e9a1fd50326eccf02a7e2',
         'info_dict': {
@@ -15,20 +19,32 @@ class RozhlasIE(InfoExtractor):
             'title': 'Echo Pavla Klusáka (30.06.2015 21:00)',
             'description': 'Osmdesátiny Terryho Rileyho jsou skvělou příležitostí proletět se elektronickými i akustickými díly zakladatatele minimalismu, který je aktivní už přes padesát let'
         }
-    }
+    }, {
+        'url': 'http://prehravac.rozhlas.cz/audio/3421320/embed',
+        'skip_download': True,
+    }]
 
     def _real_extract(self, url):
         audio_id = self._match_id(url)
-        webpage = self._download_webpage(url, audio_id)
 
-        title = self._html_search_regex(r'<h3>(.+?)</h3>', webpage, 'title')
-        description = self._html_search_regex(r'<p title="(.+?)">', webpage, 'description', fatal=False)
+        webpage = self._download_webpage(
+            'http://prehravac.rozhlas.cz/audio/%s' % audio_id, audio_id)
 
-        url = 'http://media.rozhlas.cz/_audio/' + audio_id + '.mp3'
+        title = self._html_search_regex(
+            r'<h3>(.+?)</h3>\s*<p[^>]*>.*?</p>\s*<div[^>]+id=["\']player-track',
+            webpage, 'title', default=None) or remove_start(
+            self._og_search_title(webpage), 'Radio Wave - ')
+        description = self._html_search_regex(
+            r'<p[^>]+title=(["\'])(?P<url>(?:(?!\1).)+)\1[^>]*>.*?</p>\s*<div[^>]+id=["\']player-track',
+            webpage, 'description', fatal=False, group='url')
+        duration = int_or_none(self._search_regex(
+            r'data-duration=["\'](\d+)', webpage, 'duration', default=None))
 
         return {
             'id': audio_id,
-            'url': url,
+            'url': 'http://media.rozhlas.cz/_audio/%s.mp3' % audio_id,
             'title': title,
             'description': description,
+            'duration': duration,
+            'vcodec': 'none',
         }

From 6bb801cfafb08efbd5d2d371e3ff2b7336ca09fa Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 7 Aug 2016 22:56:04 +0100
Subject: [PATCH 5694/6219] [cwtv] extract http formats

---
 youtube_dl/extractor/cwtv.py | 36 ++++++++++++++++++++++++++----------
 1 file changed, 26 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/cwtv.py b/youtube_dl/extractor/cwtv.py
index c66c359cf..1ab9333b2 100644
--- a/youtube_dl/extractor/cwtv.py
+++ b/youtube_dl/extractor/cwtv.py
@@ -28,7 +28,8 @@ class CWTVIE(InfoExtractor):
         'params': {
             # m3u8 download
             'skip_download': True,
-        }
+        },
+        'skip': 'redirect to http://cwtv.com/shows/arrow/',
     }, {
         'url': 'http://www.cwseed.com/shows/whose-line-is-it-anyway/jeff-davis-4/?play=24282b12-ead2-42f2-95ad-26770c2c6088',
         'info_dict': {
@@ -44,10 +45,6 @@ class CWTVIE(InfoExtractor):
             'upload_date': '20151006',
             'timestamp': 1444107300,
         },
-        'params': {
-            # m3u8 download
-            'skip_download': True,
-        }
     }, {
         'url': 'http://cwtv.com/thecw/chroniclesofcisco/?play=8adebe35-f447-465f-ab52-e863506ff6d6',
         'only_matching': True,
@@ -61,11 +58,30 @@ class CWTVIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        video_data = self._download_json(
-            'http://metaframe.digitalsmiths.tv/v2/CWtv/assets/%s/partner/132?format=json' % video_id, video_id)
-
-        formats = self._extract_m3u8_formats(
-            video_data['videos']['variantplaylist']['uri'], video_id, 'mp4')
+        video_data = None
+        formats = []
+        for partner in (154, 213):
+            vdata = self._download_json(
+                'http://metaframe.digitalsmiths.tv/v2/CWtv/assets/%s/partner/%d?format=json' % (video_id, partner), video_id, fatal=False)
+            if not vdata:
+                continue
+            video_data = vdata
+            for quality, quality_data in vdata.get('videos', {}).items():
+                quality_url = quality_data.get('uri')
+                if not quality_url:
+                    continue
+                if quality == 'variantplaylist':
+                    formats.extend(self._extract_m3u8_formats(
+                        quality_url, video_id, 'mp4', m3u8_id='hls', fatal=False))
+                else:
+                    tbr = int_or_none(quality_data.get('bitrate'))
+                    format_id = 'http' + ('-%d' % tbr if tbr else '')
+                    if self._is_valid_url(quality_url, video_id, format_id):
+                        formats.append({
+                            'format_id': format_id,
+                            'url': quality_url,
+                            'tbr': tbr,
+                        })
         self._sort_formats(formats)
 
         thumbnails = [{

From f17d5f6d1446c17206df9cc277ea2666aed09fab Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 7 Aug 2016 23:57:19 +0800
Subject: [PATCH 5695/6219] [features.aol.com] Fix _TESTS

---
 youtube_dl/extractor/aol.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/aol.py b/youtube_dl/extractor/aol.py
index 42c21bf41..2cdee3320 100644
--- a/youtube_dl/extractor/aol.py
+++ b/youtube_dl/extractor/aol.py
@@ -123,6 +123,10 @@ class AolFeaturesIE(InfoExtractor):
             'title': 'What To Watch - February 17, 2016',
         },
         'add_ie': ['FiveMin'],
+        'params': {
+            # encrypted m3u8 download
+            'skip_download': True,
+        },
     }]
 
     def _real_extract(self, url):

From 412abb8760c493fe2e8eeedab58aa7ca667c336d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 8 Aug 2016 12:57:17 +0800
Subject: [PATCH 5696/6219] [bilibili] Update _TESTS

---
 youtube_dl/extractor/bilibili.py | 72 +++++++-------------------------
 1 file changed, 16 insertions(+), 56 deletions(-)

diff --git a/youtube_dl/extractor/bilibili.py b/youtube_dl/extractor/bilibili.py
index b17047b39..d8eb71821 100644
--- a/youtube_dl/extractor/bilibili.py
+++ b/youtube_dl/extractor/bilibili.py
@@ -25,13 +25,13 @@ class BiliBiliIE(InfoExtractor):
 
     _TESTS = [{
         'url': 'http://www.bilibili.tv/video/av1074402/',
-        'md5': '5f7d29e1a2872f3df0cf76b1f87d3788',
+        'md5': '9fa226fe2b8a9a4d5a69b4c6a183417e',
         'info_dict': {
             'id': '1554319',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': '【金坷垃】金泡沫',
             'description': 'md5:ce18c2a2d2193f0df2917d270f2e5923',
-            'duration': 308.067,
+            'duration': 308.315,
             'timestamp': 1398012660,
             'upload_date': '20140420',
             'thumbnail': 're:^https?://.+\.jpg',
@@ -41,73 +41,33 @@ class BiliBiliIE(InfoExtractor):
     }, {
         'url': 'http://www.bilibili.com/video/av1041170/',
         'info_dict': {
-            'id': '1041170',
+            'id': '1507019',
+            'ext': 'mp4',
             'title': '【BD1080P】刀语【诸神&异域】',
             'description': '这是个神奇的故事~每个人不留弹幕不给走哦~切利哦！~',
+            'timestamp': 1396530060,
+            'upload_date': '20140403',
+            'uploader': '枫叶逝去',
+            'uploader_id': '520116',
         },
-        'playlist_count': 9,
     }, {
         'url': 'http://www.bilibili.com/video/av4808130/',
         'info_dict': {
-            'id': '4808130',
+            'id': '7802182',
+            'ext': 'mp4',
             'title': '【长篇】哆啦A梦443【钉铛】',
             'description': '(2016.05.27)来组合客人的脸吧&amp;amp;寻母六千里锭 抱歉，又轮到周日上班现在才到家 封面www.pixiv.net/member_illust.php?mode=medium&amp;amp;illust_id=56912929',
+            'timestamp': 1464564180,
+            'upload_date': '20160529',
+            'uploader': '喜欢拉面',
+            'uploader_id': '151066',
         },
-        'playlist': [{
-            'md5': '55cdadedf3254caaa0d5d27cf20a8f9c',
-            'info_dict': {
-                'id': '4808130_part1',
-                'ext': 'flv',
-                'title': '【长篇】哆啦A梦443【钉铛】',
-                'description': '(2016.05.27)来组合客人的脸吧&amp;amp;寻母六千里锭 抱歉，又轮到周日上班现在才到家 封面www.pixiv.net/member_illust.php?mode=medium&amp;amp;illust_id=56912929',
-                'timestamp': 1464564180,
-                'upload_date': '20160529',
-                'uploader': '喜欢拉面',
-                'uploader_id': '151066',
-            },
-        }, {
-            'md5': '926f9f67d0c482091872fbd8eca7ea3d',
-            'info_dict': {
-                'id': '4808130_part2',
-                'ext': 'flv',
-                'title': '【长篇】哆啦A梦443【钉铛】',
-                'description': '(2016.05.27)来组合客人的脸吧&amp;amp;寻母六千里锭 抱歉，又轮到周日上班现在才到家 封面www.pixiv.net/member_illust.php?mode=medium&amp;amp;illust_id=56912929',
-                'timestamp': 1464564180,
-                'upload_date': '20160529',
-                'uploader': '喜欢拉面',
-                'uploader_id': '151066',
-            },
-        }, {
-            'md5': '4b7b225b968402d7c32348c646f1fd83',
-            'info_dict': {
-                'id': '4808130_part3',
-                'ext': 'flv',
-                'title': '【长篇】哆啦A梦443【钉铛】',
-                'description': '(2016.05.27)来组合客人的脸吧&amp;amp;寻母六千里锭 抱歉，又轮到周日上班现在才到家 封面www.pixiv.net/member_illust.php?mode=medium&amp;amp;illust_id=56912929',
-                'timestamp': 1464564180,
-                'upload_date': '20160529',
-                'uploader': '喜欢拉面',
-                'uploader_id': '151066',
-            },
-        }, {
-            'md5': '7b795e214166501e9141139eea236e91',
-            'info_dict': {
-                'id': '4808130_part4',
-                'ext': 'flv',
-                'title': '【长篇】哆啦A梦443【钉铛】',
-                'description': '(2016.05.27)来组合客人的脸吧&amp;amp;寻母六千里锭 抱歉，又轮到周日上班现在才到家 封面www.pixiv.net/member_illust.php?mode=medium&amp;amp;illust_id=56912929',
-                'timestamp': 1464564180,
-                'upload_date': '20160529',
-                'uploader': '喜欢拉面',
-                'uploader_id': '151066',
-            },
-        }],
     }, {
         # Missing upload time
         'url': 'http://www.bilibili.com/video/av1867637/',
         'info_dict': {
             'id': '2880301',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': '【HDTV】【喜剧】岳父岳母真难当 （2014）【法国票房冠军】',
             'description': '一个信奉天主教的法国旧式传统资产阶级家庭中有四个女儿。三个女儿却分别找了阿拉伯、犹太、中国丈夫，老夫老妻唯独期盼剩下未嫁的小女儿能找一个信奉天主教的法国白人，结果没想到小女儿找了一位非裔黑人……【这次应该不会跳帧了】',
             'uploader': '黑夜为猫',

From b1c6f21c741b7f18b2b46165be0a3539735a6184 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 8 Aug 2016 12:59:07 +0800
Subject: [PATCH 5697/6219] [aparat] Fix extraction

---
 ChangeLog                      |  1 +
 youtube_dl/extractor/aparat.py | 14 ++++++--------
 2 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 32a96432b..657ff3e48 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -2,6 +2,7 @@ version <unreleased>
 
 Extractors
 * [kuwo:singer] Fix extraction
+* [aparat] Fix extraction
 
 version 2016.08.07
 
diff --git a/youtube_dl/extractor/aparat.py b/youtube_dl/extractor/aparat.py
index 63429780e..025e29aa4 100644
--- a/youtube_dl/extractor/aparat.py
+++ b/youtube_dl/extractor/aparat.py
@@ -1,8 +1,6 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
@@ -15,7 +13,7 @@ class AparatIE(InfoExtractor):
 
     _TEST = {
         'url': 'http://www.aparat.com/v/wP8On',
-        'md5': '6714e0af7e0d875c5a39c4dc4ab46ad1',
+        'md5': '131aca2e14fe7c4dcb3c4877ba300c89',
         'info_dict': {
             'id': 'wP8On',
             'ext': 'mp4',
@@ -31,13 +29,13 @@ class AparatIE(InfoExtractor):
         # Note: There is an easier-to-parse configuration at
         # http://www.aparat.com/video/video/config/videohash/%video_id
         # but the URL in there does not work
-        embed_url = ('http://www.aparat.com/video/video/embed/videohash/' +
-                     video_id + '/vt/frame')
+        embed_url = 'http://www.aparat.com/video/video/embed/vt/frame/showvideo/yes/videohash/' + video_id
         webpage = self._download_webpage(embed_url, video_id)
 
-        video_urls = [video_url.replace('\\/', '/') for video_url in re.findall(
-            r'(?:fileList\[[0-9]+\]\s*=|"file"\s*:)\s*"([^"]+)"', webpage)]
-        for i, video_url in enumerate(video_urls):
+        file_list = self._parse_json(self._search_regex(
+            r'fileList\s*=\s*JSON\.parse\(\'([^\']+)\'\)', webpage, 'file list'), video_id)
+        for i, item in enumerate(file_list[0]):
+            video_url = item['file']
             req = HEADRequest(video_url)
             res = self._request_webpage(
                 req, video_id, note='Testing video URL %d' % i, errnote=False)

From d71207121df27dc251ee15628b0742e8d7db0db7 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 8 Aug 2016 12:59:55 +0800
Subject: [PATCH 5698/6219] [biqle] Skip an invalid test

---
 youtube_dl/extractor/biqle.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/biqle.py b/youtube_dl/extractor/biqle.py
index ae4579b33..beaebfd2a 100644
--- a/youtube_dl/extractor/biqle.py
+++ b/youtube_dl/extractor/biqle.py
@@ -24,7 +24,8 @@ class BIQLEIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Ребенок в шоке от автоматической мойки',
             'uploader': 'Dmitry Kotov',
-        }
+        },
+        'skip': ' This video was marked as adult.  Embedding adult videos on external sites is prohibited.',
     }]
 
     def _real_extract(self, url):

From a41a6c5094e757eb51cbd6747d868c2f9450f324 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 8 Aug 2016 13:06:02 +0800
Subject: [PATCH 5699/6219] [chaturbate] Skip the invalid test

---
 youtube_dl/extractor/chaturbate.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/chaturbate.py b/youtube_dl/extractor/chaturbate.py
index b2234549e..29a8820d5 100644
--- a/youtube_dl/extractor/chaturbate.py
+++ b/youtube_dl/extractor/chaturbate.py
@@ -17,7 +17,8 @@ class ChaturbateIE(InfoExtractor):
         },
         'params': {
             'skip_download': True,
-        }
+        },
+        'skip': 'Room is offline',
     }, {
         'url': 'https://en.chaturbate.com/siswet19/',
         'only_matching': True,

From 3dc240e8c659112a12e8bd1260c04db9f50f1b61 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 8 Aug 2016 18:48:21 +0800
Subject: [PATCH 5700/6219] [sohu] Update _TESTS (closes #10260)

---
 youtube_dl/extractor/sohu.py | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/sohu.py b/youtube_dl/extractor/sohu.py
index 72fe66142..48e2ba2dd 100644
--- a/youtube_dl/extractor/sohu.py
+++ b/youtube_dl/extractor/sohu.py
@@ -14,10 +14,10 @@ from ..utils import ExtractorError
 class SohuIE(InfoExtractor):
     _VALID_URL = r'https?://(?P<mytv>my\.)?tv\.sohu\.com/.+?/(?(mytv)|n)(?P<id>\d+)\.shtml.*?'
 
+    # Sohu videos give different MD5 sums on Travis CI and my machine
     _TESTS = [{
         'note': 'This video is available only in Mainland China',
         'url': 'http://tv.sohu.com/20130724/n382479172.shtml#super',
-        'md5': '29175c8cadd8b5cc4055001e85d6b372',
         'info_dict': {
             'id': '382479172',
             'ext': 'mp4',
@@ -26,7 +26,6 @@ class SohuIE(InfoExtractor):
         'skip': 'On available in China',
     }, {
         'url': 'http://tv.sohu.com/20150305/n409385080.shtml',
-        'md5': '699060e75cf58858dd47fb9c03c42cfb',
         'info_dict': {
             'id': '409385080',
             'ext': 'mp4',
@@ -34,7 +33,6 @@ class SohuIE(InfoExtractor):
         }
     }, {
         'url': 'http://my.tv.sohu.com/us/232799889/78693464.shtml',
-        'md5': '9bf34be48f2f4dadcb226c74127e203c',
         'info_dict': {
             'id': '78693464',
             'ext': 'mp4',
@@ -48,7 +46,6 @@ class SohuIE(InfoExtractor):
             'title': '【神探苍实战秘籍】第13期 战争之影 赫卡里姆',
         },
         'playlist': [{
-            'md5': 'bdbfb8f39924725e6589c146bc1883ad',
             'info_dict': {
                 'id': '78910339_part1',
                 'ext': 'mp4',
@@ -56,7 +53,6 @@ class SohuIE(InfoExtractor):
                 'title': '【神探苍实战秘籍】第13期 战争之影 赫卡里姆',
             }
         }, {
-            'md5': '3e1f46aaeb95354fd10e7fca9fc1804e',
             'info_dict': {
                 'id': '78910339_part2',
                 'ext': 'mp4',
@@ -64,7 +60,6 @@ class SohuIE(InfoExtractor):
                 'title': '【神探苍实战秘籍】第13期 战争之影 赫卡里姆',
             }
         }, {
-            'md5': '8407e634175fdac706766481b9443450',
             'info_dict': {
                 'id': '78910339_part3',
                 'ext': 'mp4',

From 395c74615c58f5f1b2e462786d6f4d8bab1a313a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 8 Aug 2016 21:49:27 +0700
Subject: [PATCH 5701/6219] Revert "[extractor/generic] Make _search_json_ld
 non fatal"

This reverts commit 958849275f1c6072c712e8d611294a762fadc7f0.
---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 44c6c354c..e89a03760 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -2241,7 +2241,7 @@ class GenericIE(InfoExtractor):
 
         # Looking for http://schema.org/VideoObject
         json_ld = self._search_json_ld(
-            webpage, video_id, fatal=False, expected_type='VideoObject')
+            webpage, video_id, default=None, expected_type='VideoObject')
         if json_ld and json_ld.get('url'):
             info_dict.update({
                 'title': video_title or info_dict['title'],

From 3711fa1eb24224a5b898ede355136c9009355a87 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 8 Aug 2016 21:49:45 +0700
Subject: [PATCH 5702/6219] Revert "[flipagram] Make _search_json_ld non fatal"

This reverts commit d34995a9e3f596c4dd80178d99f7bd8dbc748e2b.
---
 youtube_dl/extractor/flipagram.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/flipagram.py b/youtube_dl/extractor/flipagram.py
index 634f5fe3b..acb6133ff 100644
--- a/youtube_dl/extractor/flipagram.py
+++ b/youtube_dl/extractor/flipagram.py
@@ -48,7 +48,7 @@ class FlipagramIE(InfoExtractor):
         flipagram = video_data['flipagram']
         video = flipagram['video']
 
-        json_ld = self._search_json_ld(webpage, video_id, fatal=False)
+        json_ld = self._search_json_ld(webpage, video_id, default=False)
         title = json_ld.get('title') or flipagram['captionText']
         description = json_ld.get('description') or flipagram.get('captionText')
 

From 321b5e082a7340225091ea5ad852e12b2dc499e9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 8 Aug 2016 22:36:18 +0700
Subject: [PATCH 5703/6219] [extractor/common] Respect default in
 _search_json_ld

---
 youtube_dl/extractor/common.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 0891309dd..e47770c1d 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -816,11 +816,14 @@ class InfoExtractor(object):
         json_ld = self._search_regex(
             r'(?s)<script[^>]+type=(["\'])application/ld\+json\1[^>]*>(?P<json_ld>.+?)</script>',
             html, 'JSON-LD', group='json_ld', **kwargs)
+        default = kwargs.get('default', NO_DEFAULT)
         if not json_ld:
-            return {}
-        return self._json_ld(
-            json_ld, video_id, fatal=kwargs.get('fatal', True),
-            expected_type=expected_type)
+            return default if default is not NO_DEFAULT else {}
+        # JSON-LD may be malformed and thus `fatal` should be respected.
+        # At the same time `default` may be passed that assumes `fatal=False`
+        # for _search_regex. Let's simulate the same behavior here as well.
+        fatal = kwargs.get('fatal', True) if default == NO_DEFAULT else False
+        return self._json_ld(json_ld, video_id, fatal=fatal, expected_type=expected_type)
 
     def _json_ld(self, json_ld, video_id, fatal=True, expected_type=None):
         if isinstance(json_ld, compat_str):

From 522f6c066da93b6baec3399b7098556e5ec55f43 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 8 Aug 2016 22:44:36 +0700
Subject: [PATCH 5704/6219] [bbc] Add proper default to _search_json_ld call

---
 youtube_dl/extractor/bbc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index b29e05970..83e6d024c 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -759,7 +759,7 @@ class BBCIE(BBCCoUkIE):
 
         webpage = self._download_webpage(url, playlist_id)
 
-        json_ld_info = self._search_json_ld(webpage, playlist_id, default=None)
+        json_ld_info = self._search_json_ld(webpage, playlist_id, default={})
         timestamp = json_ld_info.get('timestamp')
 
         playlist_title = json_ld_info.get('title')

From 1e7f602e2ac27bac266cde4c67b31f59510a91f5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 8 Aug 2016 22:45:49 +0700
Subject: [PATCH 5705/6219] [condenast] Make _search_json_ld call non fatal

---
 youtube_dl/extractor/condenast.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/condenast.py b/youtube_dl/extractor/condenast.py
index 15fabbb1c..8d8f60598 100644
--- a/youtube_dl/extractor/condenast.py
+++ b/youtube_dl/extractor/condenast.py
@@ -143,7 +143,8 @@ class CondeNastIE(InfoExtractor):
             })
         self._sort_formats(formats)
 
-        info = self._search_json_ld(webpage, video_id) if url_type != 'embed' else {}
+        info = self._search_json_ld(
+            webpage, video_id, fatal=False) if url_type != 'embed' else {}
         info.update({
             'id': video_id,
             'formats': formats,

From e8ed7354e6fdd7bd3759e24f0aa08233316303b7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 8 Aug 2016 22:46:19 +0700
Subject: [PATCH 5706/6219] [flipagram] Add proper default to _search_json_ld
 call

---
 youtube_dl/extractor/flipagram.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/flipagram.py b/youtube_dl/extractor/flipagram.py
index acb6133ff..1902a2393 100644
--- a/youtube_dl/extractor/flipagram.py
+++ b/youtube_dl/extractor/flipagram.py
@@ -48,7 +48,7 @@ class FlipagramIE(InfoExtractor):
         flipagram = video_data['flipagram']
         video = flipagram['video']
 
-        json_ld = self._search_json_ld(webpage, video_id, default=False)
+        json_ld = self._search_json_ld(webpage, video_id, default={})
         title = json_ld.get('title') or flipagram['captionText']
         description = json_ld.get('description') or flipagram.get('captionText')
 

From 082395d0a0f21375f63e3c9b33011b30d5de3aae Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 8 Aug 2016 22:48:33 +0700
Subject: [PATCH 5707/6219] [extractor/generic] Add proper default to
 _search_json_ld call

---
 youtube_dl/extractor/generic.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index e89a03760..50500ce0e 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -2241,8 +2241,8 @@ class GenericIE(InfoExtractor):
 
         # Looking for http://schema.org/VideoObject
         json_ld = self._search_json_ld(
-            webpage, video_id, default=None, expected_type='VideoObject')
-        if json_ld and json_ld.get('url'):
+            webpage, video_id, default={}, expected_type='VideoObject')
+        if json_ld.get('url'):
             info_dict.update({
                 'title': video_title or info_dict['title'],
                 'description': video_description,

From 8991844ea260a17943e935acfd5565caab27a99e Mon Sep 17 00:00:00 2001
From: Sukhbir Singh <sukhbir@riseup.net>
Date: Sun, 7 Aug 2016 19:11:00 -0400
Subject: [PATCH 5708/6219] [sonyliv] Add new extractor

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/sonyliv.py    | 28 ++++++++++++++++++++++++++++
 2 files changed, 29 insertions(+)
 create mode 100644 youtube_dl/extractor/sonyliv.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 7ede6afda..f1043dae6 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -756,6 +756,7 @@ from .smotri import (
 )
 from .snotr import SnotrIE
 from .sohu import SohuIE
+from .sonyliv import SonyLIVIE
 from .soundcloud import (
     SoundcloudIE,
     SoundcloudSetIE,
diff --git a/youtube_dl/extractor/sonyliv.py b/youtube_dl/extractor/sonyliv.py
new file mode 100644
index 000000000..c717b1f4d
--- /dev/null
+++ b/youtube_dl/extractor/sonyliv.py
@@ -0,0 +1,28 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class SonyLIVIE(InfoExtractor):
+    _VALID_URL = r'http?://(?:www\.)?sonyliv\.com/details/episodes/(?P<id>[0-9]+)/'
+    _TEST = {
+        'url': 'http://www.sonyliv.com/details/episodes/5024612095001/Ep.-1---Achaari-Cheese-Toast---Bachelor\'s-Delight',
+        'info_dict': {
+            'title': 'Ep. 1 - Achaari Cheese Toast - Bachelor\'s Delight',
+            'id': '5024612095001',
+            'ext': 'mp4',
+            'upload_date': '20160707',
+            'description': 'Bachelor\'s Delight is a new food show from Sony LIV to satisfy the taste buds of all those bachelors looking for a quick bite.',
+            'uploader_id': '4338955589001',
+            'timestamp': 1467870968,
+        },
+        'params': {
+            'skip_download': True,
+        },
+        'add_ie': ['BrightcoveNew'],
+    }
+    BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/4338955589001/default_default/index.html?videoId=%s'
+
+    def _real_extract(self, url):
+        return self.url_result(self.BRIGHTCOVE_URL_TEMPLATE % self._match_id(url), 'BrightcoveNew')

From 77426a087b6af712b69faf62a54db959f0d13288 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 9 Aug 2016 02:16:28 +0700
Subject: [PATCH 5709/6219] [sonyliv] Improve (Closes #10258)

---
 youtube_dl/extractor/sonyliv.py | 20 +++++++++++++-------
 1 file changed, 13 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/sonyliv.py b/youtube_dl/extractor/sonyliv.py
index c717b1f4d..accd112aa 100644
--- a/youtube_dl/extractor/sonyliv.py
+++ b/youtube_dl/extractor/sonyliv.py
@@ -5,15 +5,15 @@ from .common import InfoExtractor
 
 
 class SonyLIVIE(InfoExtractor):
-    _VALID_URL = r'http?://(?:www\.)?sonyliv\.com/details/episodes/(?P<id>[0-9]+)/'
-    _TEST = {
-        'url': 'http://www.sonyliv.com/details/episodes/5024612095001/Ep.-1---Achaari-Cheese-Toast---Bachelor\'s-Delight',
+    _VALID_URL = r'https?://(?:www\.)?sonyliv\.com/details/[^/]+/(?P<id>\d+)'
+    _TESTS = [{
+        'url': "http://www.sonyliv.com/details/episodes/5024612095001/Ep.-1---Achaari-Cheese-Toast---Bachelor's-Delight",
         'info_dict': {
-            'title': 'Ep. 1 - Achaari Cheese Toast - Bachelor\'s Delight',
+            'title': "Ep. 1 - Achaari Cheese Toast - Bachelor's Delight",
             'id': '5024612095001',
             'ext': 'mp4',
             'upload_date': '20160707',
-            'description': 'Bachelor\'s Delight is a new food show from Sony LIV to satisfy the taste buds of all those bachelors looking for a quick bite.',
+            'description': 'md5:7f28509a148d5be9d0782b4d5106410d',
             'uploader_id': '4338955589001',
             'timestamp': 1467870968,
         },
@@ -21,8 +21,14 @@ class SonyLIVIE(InfoExtractor):
             'skip_download': True,
         },
         'add_ie': ['BrightcoveNew'],
-    }
+    }, {
+        'url': 'http://www.sonyliv.com/details/full%20movie/4951168986001/Sei-Raat-(Bangla)',
+        'only_matching': True,
+    }]
+
     BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/4338955589001/default_default/index.html?videoId=%s'
 
     def _real_extract(self, url):
-        return self.url_result(self.BRIGHTCOVE_URL_TEMPLATE % self._match_id(url), 'BrightcoveNew')
+        brightcove_id = self._match_id(url)
+        return self.url_result(
+            self.BRIGHTCOVE_URL_TEMPLATE % brightcove_id, 'BrightcoveNew', brightcove_id)

From affaea0688780c03afe36bc81c7151e696649b5a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Petr=20Zvoni=CC=81c=CC=8Cek?= <zvonicek@gmail.com>
Date: Sat, 6 Aug 2016 15:26:48 +0200
Subject: [PATCH 5710/6219] [rbmaradio] Fixed extractor

---
 youtube_dl/extractor/rbmaradio.py | 45 +++++++++++++------------------
 1 file changed, 18 insertions(+), 27 deletions(-)

diff --git a/youtube_dl/extractor/rbmaradio.py b/youtube_dl/extractor/rbmaradio.py
index 7932af6ef..92f0d8f76 100644
--- a/youtube_dl/extractor/rbmaradio.py
+++ b/youtube_dl/extractor/rbmaradio.py
@@ -1,55 +1,46 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
-import json
-import re
-
 from .common import InfoExtractor
 from ..utils import (
-    ExtractorError,
+    clean_html
 )
 
 
 class RBMARadioIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?rbmaradio\.com/shows/(?P<videoID>[^/]+)$'
+    _VALID_URL = r'https?://(?:www\.)?rbmaradio\.com/shows/[^/]+/episodes/(?P<id>[^/]+)$'
     _TEST = {
-        'url': 'http://www.rbmaradio.com/shows/ford-lopatin-live-at-primavera-sound-2011',
+        'url': 'https://www.rbmaradio.com/shows/main-stage/episodes/ford-lopatin-live-at-primavera-sound-2011',
         'md5': '6bc6f9bcb18994b4c983bc3bf4384d95',
         'info_dict': {
             'id': 'ford-lopatin-live-at-primavera-sound-2011',
             'ext': 'mp3',
-            'uploader_id': 'ford-lopatin',
-            'location': 'Spain',
             'description': 'Joel Ford and Daniel ’Oneohtrix Point Never’ Lopatin fly their midified pop extravaganza to Spain. Live at Primavera Sound 2011.',
-            'uploader': 'Ford & Lopatin',
-            'title': 'Live at Primavera Sound 2011',
+            'title': 'Ford & Lopatin - Main Stage',
         },
     }
 
     def _real_extract(self, url):
-        m = re.match(self._VALID_URL, url)
-        video_id = m.group('videoID')
+        video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id)
+        json_data = self._search_regex(r'<script>window\.__INITIAL_STATE__\s*=\s*(.+?)</script>',
+                                       webpage, 'json data')
+        data = self._parse_json(json_data, video_id)
 
-        json_data = self._search_regex(r'window\.gon.*?gon\.show=(.+?);$',
-                                       webpage, 'json data', flags=re.MULTILINE)
+        item = None
+        for episode in data['episodes']:
+            items = data['episodes'][episode]
+            if video_id in items:
+                item = items[video_id]
 
-        try:
-            data = json.loads(json_data)
-        except ValueError as e:
-            raise ExtractorError('Invalid JSON: ' + str(e))
-
-        video_url = data['akamai_url'] + '&cbr=256'
+        video_url = item['audioURL'] + '?cbr=256'
 
         return {
             'id': video_id,
             'url': video_url,
-            'title': data['title'],
-            'description': data.get('teaser_text'),
-            'location': data.get('country_of_origin'),
-            'uploader': data.get('host', {}).get('name'),
-            'uploader_id': data.get('host', {}).get('slug'),
-            'thumbnail': data.get('image', {}).get('large_url_2x'),
-            'duration': data.get('duration'),
+            'title': item.get('title') + ' - ' + item.get('showTitle'),
+            'description': clean_html(item.get('longTeaser')),
+            'thumbnail': self._proto_relative_url(item.get('imageURL', {}).get('landscape')),
+            'duration': item.get('duration'),
         }

From 3a380766d1d6abd83213319b41cdf9a18977a69c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 9 Aug 2016 02:46:29 +0700
Subject: [PATCH 5711/6219] [rbmaradio] Improve, simplify and extract all
 formats (Closes #10242)

---
 youtube_dl/extractor/rbmaradio.py | 69 +++++++++++++++++++++----------
 1 file changed, 47 insertions(+), 22 deletions(-)

diff --git a/youtube_dl/extractor/rbmaradio.py b/youtube_dl/extractor/rbmaradio.py
index 92f0d8f76..471928ef8 100644
--- a/youtube_dl/extractor/rbmaradio.py
+++ b/youtube_dl/extractor/rbmaradio.py
@@ -1,46 +1,71 @@
-# encoding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
+from ..compat import compat_str
 from ..utils import (
-    clean_html
+    clean_html,
+    int_or_none,
+    unified_timestamp,
+    update_url_query,
 )
 
 
 class RBMARadioIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?rbmaradio\.com/shows/[^/]+/episodes/(?P<id>[^/]+)$'
+    _VALID_URL = r'https?://(?:www\.)?rbmaradio\.com/shows/(?P<show_id>[^/]+)/episodes/(?P<id>[^/?#&]+)'
     _TEST = {
         'url': 'https://www.rbmaradio.com/shows/main-stage/episodes/ford-lopatin-live-at-primavera-sound-2011',
         'md5': '6bc6f9bcb18994b4c983bc3bf4384d95',
         'info_dict': {
             'id': 'ford-lopatin-live-at-primavera-sound-2011',
             'ext': 'mp3',
-            'description': 'Joel Ford and Daniel ’Oneohtrix Point Never’ Lopatin fly their midified pop extravaganza to Spain. Live at Primavera Sound 2011.',
-            'title': 'Ford & Lopatin - Main Stage',
+            'title': 'Main Stage - Ford & Lopatin',
+            'description': 'md5:4f340fb48426423530af5a9d87bd7b91',
+            'thumbnail': 're:^https?://.*\.jpg',
+            'duration': 2452,
+            'timestamp': 1307103164,
+            'upload_date': '20110603',
         },
     }
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        mobj = re.match(self._VALID_URL, url)
+        show_id = mobj.group('show_id')
+        episode_id = mobj.group('id')
 
-        webpage = self._download_webpage(url, video_id)
-        json_data = self._search_regex(r'<script>window\.__INITIAL_STATE__\s*=\s*(.+?)</script>',
-                                       webpage, 'json data')
-        data = self._parse_json(json_data, video_id)
+        webpage = self._download_webpage(url, episode_id)
 
-        item = None
-        for episode in data['episodes']:
-            items = data['episodes'][episode]
-            if video_id in items:
-                item = items[video_id]
+        episode = self._parse_json(
+            self._search_regex(
+                r'__INITIAL_STATE__\s*=\s*({.+?})\s*</script>',
+                webpage, 'json data'),
+            episode_id)['episodes'][show_id][episode_id]
 
-        video_url = item['audioURL'] + '?cbr=256'
+        title = episode['title']
+
+        show_title = episode.get('showTitle')
+        if show_title:
+            title = '%s - %s' % (show_title, title)
+
+        formats = [{
+            'url': update_url_query(episode['audioURL'], query={'cbr': abr}),
+            'format_id': compat_str(abr),
+            'abr': abr,
+            'vcodec': 'none',
+        } for abr in (96, 128, 256)]
+
+        description = clean_html(episode.get('longTeaser'))
+        thumbnail = self._proto_relative_url(episode.get('imageURL', {}).get('landscape'))
+        duration = int_or_none(episode.get('duration'))
+        timestamp = unified_timestamp(episode.get('publishedAt'))
 
         return {
-            'id': video_id,
-            'url': video_url,
-            'title': item.get('title') + ' - ' + item.get('showTitle'),
-            'description': clean_html(item.get('longTeaser')),
-            'thumbnail': self._proto_relative_url(item.get('imageURL', {}).get('landscape')),
-            'duration': item.get('duration'),
+            'id': episode_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'duration': duration,
+            'timestamp': timestamp,
+            'formats': formats,
         }

From 065bc354894f1d35592529455d9eb685470124b9 Mon Sep 17 00:00:00 2001
From: singh-pratyush96 <singh.pratyush96@gmail.com>
Date: Thu, 4 Aug 2016 15:47:22 +0530
Subject: [PATCH 5712/6219] Add --max-sleep-interval (Closes #9930)

---
 youtube_dl/YoutubeDL.py         |  6 +++++-
 youtube_dl/__init__.py          |  7 +++++++
 youtube_dl/downloader/common.py |  7 +++++--
 youtube_dl/options.py           | 12 ++++++++++--
 4 files changed, 27 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 6551f086f..82b77783d 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -249,7 +249,11 @@ class YoutubeDL(object):
     source_address:    (Experimental) Client-side IP address to bind to.
     call_home:         Boolean, true iff we are allowed to contact the
                        youtube-dl servers for debugging.
-    sleep_interval:    Number of seconds to sleep before each download.
+    sleep_interval:    Minimum number of seconds to sleep before each download.
+                       Sleep will be for a random interval if --max-sleep-interval is also passed.
+    max_sleep_interval:Max number of seconds to sleep before each download.
+                       Sleep will be for a random interval if passed along with --min-sleep-interval
+                       or --sleep-interval, otherwise ignored.
     listformats:       Print an overview of available video formats and exit.
     list_thumbnails:   Print a table of all thumbnails and exit.
     match_filter:      A function that gets called with the info_dict of
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 2b34bf9c2..86af18d33 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -145,6 +145,12 @@ def _real_main(argv=None):
         if numeric_limit is None:
             parser.error('invalid max_filesize specified')
         opts.max_filesize = numeric_limit
+    if opts.sleep_interval is not None:
+        if opts.sleep_interval < 0:
+            parser.error('sleep interval should not be negative')
+        elif opts.max_sleep_interval is not None:
+            if opts.max_sleep_interval < opts.sleep_interval:
+                parser.error('max sleep interval should not be less than sleep interval')
 
     def parse_retries(retries):
         if retries in ('inf', 'infinite'):
@@ -370,6 +376,7 @@ def _real_main(argv=None):
         'source_address': opts.source_address,
         'call_home': opts.call_home,
         'sleep_interval': opts.sleep_interval,
+        'max_sleep_interval': opts.max_sleep_interval,
         'external_downloader': opts.external_downloader,
         'list_thumbnails': opts.list_thumbnails,
         'playlist_items': opts.playlist_items,
diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index 1dba9f49a..8e377c72c 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -4,6 +4,7 @@ import os
 import re
 import sys
 import time
+import random
 
 from ..compat import compat_os_name
 from ..utils import (
@@ -342,8 +343,10 @@ class FileDownloader(object):
             })
             return True
 
-        sleep_interval = self.params.get('sleep_interval')
-        if sleep_interval:
+        sleep_lower_bound = self.params.get('sleep_interval')
+        if sleep_lower_bound:
+            sleep_upper_bound = self.params.get('max_sleep_interval', sleep_lower_bound)
+            sleep_interval = random.uniform(sleep_lower_bound, sleep_upper_bound)
             self.to_screen('[download] Sleeping %s seconds...' % sleep_interval)
             time.sleep(sleep_interval)
 
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 942d44912..068e824a0 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -499,9 +499,17 @@ def parseOpts(overrideArguments=None):
         dest='bidi_workaround', action='store_true',
         help='Work around terminals that lack bidirectional text support. Requires bidiv or fribidi executable in PATH')
     workarounds.add_option(
-        '--sleep-interval', metavar='SECONDS',
+        '--sleep-interval', '--min-sleep-interval', metavar='SECONDS',
         dest='sleep_interval', type=float,
-        help='Number of seconds to sleep before each download.')
+        help='Minimum number of seconds to sleep before each download. Sleep will be for a random interval if '
+             '--max-sleep-interval is also passed.'
+    )
+    workarounds.add_option(
+        '--max-sleep-interval', metavar='SECONDS',
+        dest='max_sleep_interval', type=float,
+        help='Max number of seconds to sleep before each download. Sleep will be for a random interval if passed'
+             ' along with --min-sleep-interval or --sleep-interval, otherwise ignored.'
+    )
 
     verbosity = optparse.OptionGroup(parser, 'Verbosity / Simulation Options')
     verbosity.add_option(

From 7aa589a5e10c983f701b48b8431cc5f54b85cf3b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 9 Aug 2016 03:46:52 +0700
Subject: [PATCH 5713/6219] Fix --min/max-sleep-interval wording

---
 youtube_dl/YoutubeDL.py | 15 ++++++++++-----
 youtube_dl/options.py   | 15 +++++++++------
 2 files changed, 19 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 82b77783d..193f8db9f 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -249,11 +249,16 @@ class YoutubeDL(object):
     source_address:    (Experimental) Client-side IP address to bind to.
     call_home:         Boolean, true iff we are allowed to contact the
                        youtube-dl servers for debugging.
-    sleep_interval:    Minimum number of seconds to sleep before each download.
-                       Sleep will be for a random interval if --max-sleep-interval is also passed.
-    max_sleep_interval:Max number of seconds to sleep before each download.
-                       Sleep will be for a random interval if passed along with --min-sleep-interval
-                       or --sleep-interval, otherwise ignored.
+    sleep_interval:    Number of seconds to sleep before each download when
+                       used alone or a lower bound of a range for randomized
+                       sleep before each download (minimum possible number
+                       of seconds to sleep) when used along with
+                       max_sleep_interval.
+    max_sleep_interval:Upper bound of a range for randomized sleep before each
+                       download (maximum possible number of seconds to sleep).
+                       Must only be used along with sleep_interval.
+                       Actual sleep time will be a random float from range
+                       [sleep_interval; max_sleep_interval].
     listformats:       Print an overview of available video formats and exit.
     list_thumbnails:   Print a table of all thumbnails and exit.
     match_filter:      A function that gets called with the info_dict of
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 068e824a0..d32a9e32c 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -501,15 +501,18 @@ def parseOpts(overrideArguments=None):
     workarounds.add_option(
         '--sleep-interval', '--min-sleep-interval', metavar='SECONDS',
         dest='sleep_interval', type=float,
-        help='Minimum number of seconds to sleep before each download. Sleep will be for a random interval if '
-             '--max-sleep-interval is also passed.'
-    )
+        help=(
+            'Number of seconds to sleep before each download when used alone '
+            'or a lower bound of a range for randomized sleep before each download '
+            '(minimum possible number of seconds to sleep) when used along with '
+            '--max-sleep-interval.'))
     workarounds.add_option(
         '--max-sleep-interval', metavar='SECONDS',
         dest='max_sleep_interval', type=float,
-        help='Max number of seconds to sleep before each download. Sleep will be for a random interval if passed'
-             ' along with --min-sleep-interval or --sleep-interval, otherwise ignored.'
-    )
+        help=(
+            'Upper bound of a range for randomized sleep before each download '
+            '(maximum possible number of seconds to sleep). Must only be used '
+            'along with --min-sleep-interval.'))
 
     verbosity = optparse.OptionGroup(parser, 'Verbosity / Simulation Options')
     verbosity.add_option(

From 1ad6b891b21b45830736698a7b59c30d9605a562 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 9 Aug 2016 03:47:56 +0700
Subject: [PATCH 5714/6219] Add more checks for --min/max-sleep-interval
 arguments and use more idiomatic naming

---
 youtube_dl/__init__.py          | 12 ++++++++----
 youtube_dl/downloader/common.py |  9 +++++----
 2 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 86af18d33..a9730292c 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -147,10 +147,14 @@ def _real_main(argv=None):
         opts.max_filesize = numeric_limit
     if opts.sleep_interval is not None:
         if opts.sleep_interval < 0:
-            parser.error('sleep interval should not be negative')
-        elif opts.max_sleep_interval is not None:
-            if opts.max_sleep_interval < opts.sleep_interval:
-                parser.error('max sleep interval should not be less than sleep interval')
+            parser.error('sleep interval must be positive or 0')
+    if opts.max_sleep_interval is not None:
+        if opts.max_sleep_interval < 0:
+            parser.error('max sleep interval must be positive or 0')
+        if opts.max_sleep_interval < opts.sleep_interval:
+            parser.error('max sleep interval must be greater than or equal to min sleep interval')
+    else:
+        opts.max_sleep_interval = opts.sleep_interval
 
     def parse_retries(retries):
         if retries in ('inf', 'infinite'):
diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index 8e377c72c..8482cbd84 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -343,10 +343,11 @@ class FileDownloader(object):
             })
             return True
 
-        sleep_lower_bound = self.params.get('sleep_interval')
-        if sleep_lower_bound:
-            sleep_upper_bound = self.params.get('max_sleep_interval', sleep_lower_bound)
-            sleep_interval = random.uniform(sleep_lower_bound, sleep_upper_bound)
+        min_sleep_interval = self.params.get('sleep_interval')
+        if min_sleep_interval:
+            max_sleep_interval = self.params.get('max_sleep_interval', min_sleep_interval)
+            print(min_sleep_interval, max_sleep_interval)
+            sleep_interval = random.uniform(min_sleep_interval, max_sleep_interval)
             self.to_screen('[download] Sleeping %s seconds...' % sleep_interval)
             time.sleep(sleep_interval)
 

From 5e42f8a0adec94156b911399558a967214a9d3aa Mon Sep 17 00:00:00 2001
From: nyorain <nyorain@gmail.com>
Date: Sat, 6 Aug 2016 01:21:39 +0200
Subject: [PATCH 5715/6219] Make --metadata-from-title non fatal

Output a warning if the metadata can't be parsed from the title (and don't write any metadata) instead of raising a critical error.
---
 youtube_dl/postprocessor/metadatafromtitle.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/postprocessor/metadatafromtitle.py b/youtube_dl/postprocessor/metadatafromtitle.py
index 42377fa0f..f7c831c67 100644
--- a/youtube_dl/postprocessor/metadatafromtitle.py
+++ b/youtube_dl/postprocessor/metadatafromtitle.py
@@ -38,7 +38,8 @@ class MetadataFromTitlePP(PostProcessor):
         title = info['title']
         match = re.match(self._titleregex, title)
         if match is None:
-            raise MetadataFromTitlePPError('Could not interpret title of video as "%s"' % self._titleformat)
+            self._downloader.to_screen('[fromtitle] Could not interpret title of video as "%s"' % self._titleformat)
+            return [], info
         for attribute, value in match.groupdict().items():
             value = match.group(attribute)
             info[attribute] = value

From 25dd58ca6a1326c27e57262c3a298b163fc3c1eb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 9 Aug 2016 04:01:05 +0700
Subject: [PATCH 5716/6219] [metadatafromtitle] Remove unused exception class

---
 youtube_dl/postprocessor/metadatafromtitle.py | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/youtube_dl/postprocessor/metadatafromtitle.py b/youtube_dl/postprocessor/metadatafromtitle.py
index f7c831c67..920573da9 100644
--- a/youtube_dl/postprocessor/metadatafromtitle.py
+++ b/youtube_dl/postprocessor/metadatafromtitle.py
@@ -3,11 +3,6 @@ from __future__ import unicode_literals
 import re
 
 from .common import PostProcessor
-from ..utils import PostProcessingError
-
-
-class MetadataFromTitlePPError(PostProcessingError):
-    pass
 
 
 class MetadataFromTitlePP(PostProcessor):

From 9778b3e7eede2d405f0102a958ea7df7ab2c6444 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 9 Aug 2016 04:03:52 +0700
Subject: [PATCH 5717/6219] Credit @zvonicek for #10242 and #10253

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 890c827a0..7e17d625b 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -179,3 +179,4 @@ Jakub Adam Wieczorek
 Aleksandar Topuzović
 Nehal Patel
 Rob van Bekkum
+Petr Zvoníček

From b6578166847c5435095346b0225abd758cad6bb4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 9 Aug 2016 04:04:45 +0700
Subject: [PATCH 5718/6219] Credit @singh-pratyush96 for #10223

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 7e17d625b..1fd4be785 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -180,3 +180,4 @@ Aleksandar Topuzović
 Nehal Patel
 Rob van Bekkum
 Petr Zvoníček
+Pratyush Singh

From 4e62d26aa211281ae85e9942c01adf31fe80aee5 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 9 Aug 2016 15:09:08 +0100
Subject: [PATCH 5719/6219] [uol] Add new extractor(#4263)

---
 youtube_dl/extractor/extractors.py |   1 +
 youtube_dl/extractor/uol.py        | 130 +++++++++++++++++++++++++++++
 2 files changed, 131 insertions(+)
 create mode 100644 youtube_dl/extractor/uol.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index f1043dae6..387230be0 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -929,6 +929,7 @@ from .udemy import (
 from .udn import UDNEmbedIE
 from .digiteka import DigitekaIE
 from .unistra import UnistraIE
+from .uol import UOLIE
 from .urort import UrortIE
 from .urplay import URPlayIE
 from .usatoday import USATodayIE
diff --git a/youtube_dl/extractor/uol.py b/youtube_dl/extractor/uol.py
new file mode 100644
index 000000000..5c6a3577b
--- /dev/null
+++ b/youtube_dl/extractor/uol.py
@@ -0,0 +1,130 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    clean_html,
+    int_or_none,
+    parse_duration,
+    update_url_query,
+    str_or_none,
+)
+
+
+class UOLIE(InfoExtractor):
+    IE_NAME = 'uol.com.br'
+    _VALID_URL = r'https?://(?:.+?\.)?uol\.com\.br/.*?(?:(?:mediaId|v)=|view/(?:[a-z0-9]+/)?|video(?:=|/(?:\d{4}/\d{2}/\d{2}/)?))(?P<id>\d+|[\w-]+-[A-Z0-9]+)'
+    _TESTS = [{
+        'url': 'http://player.mais.uol.com.br/player_video_v3.swf?mediaId=15951931',
+        'md5': '25291da27dc45e0afb5718a8603d3816',
+        'info_dict': {
+            'id': '15951931',
+            'ext': 'mp4',
+            'title': 'Miss simpatia é encontrada morta',
+            'description': 'md5:3f8c11a0c0556d66daf7e5b45ef823b2',
+        }
+    }, {
+        'url': 'http://tvuol.uol.com.br/video/incendio-destroi-uma-das-maiores-casas-noturnas-de-londres-04024E9A3268D4C95326',
+        'md5': 'e41a2fb7b7398a3a46b6af37b15c00c9',
+        'info_dict': {
+            'id': '15954259',
+            'ext': 'mp4',
+            'title': 'Incêndio destrói uma das maiores casas noturnas de Londres',
+            'description': 'Em Londres, um incêndio destruiu uma das maiores boates da cidade. Não há informações sobre vítimas.',
+        }
+    }, {
+        'url': 'http://mais.uol.com.br/static/uolplayer/index.html?mediaId=15951931',
+        'only_matching': True,
+    }, {
+        'url': 'http://mais.uol.com.br/view/15954259',
+        'only_matching': True,
+    }, {
+        'url': 'http://noticias.band.uol.com.br/brasilurgente/video/2016/08/05/15951931/miss-simpatia-e-encontrada-morta.html',
+        'only_matching': True,
+    }, {
+        'url': 'http://videos.band.uol.com.br/programa.asp?e=noticias&pr=brasil-urgente&v=15951931&t=Policia-desmonte-base-do-PCC-na-Cracolandia',
+        'only_matching': True,
+    }, {
+        'url': 'http://mais.uol.com.br/view/cphaa0gl2x8r/incendio-destroi-uma-das-maiores-casas-noturnas-de-londres-04024E9A3268D4C95326',
+        'only_matching': True,
+    }, {
+        'url': 'http://noticias.uol.com.br//videos/assistir.htm?video=rafaela-silva-inspira-criancas-no-judo-04024D983968D4C95326',
+        'only_matching': True,
+    }, {
+        'url': 'http://mais.uol.com.br/view/e0qbgxid79uv/15275470',
+        'only_matching': True,
+    }]
+
+    _FORMATS = {
+        '2': {
+            'width': 640,
+            'height': 360,
+        },
+        '5': {
+            'width': 1080,
+            'height': 720,
+        },
+        '6': {
+            'width': 426,
+            'height': 240,
+        },
+        '7': {
+            'width': 1920,
+            'height': 1080,
+        },
+        '8': {
+            'width': 192,
+            'height': 144,
+        },
+        '9': {
+            'width': 568,
+            'height': 320,
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        if not video_id.isdigit():
+            embed_page = self._download_webpage('https://jsuol.com.br/c/tv/uol/embed/?params=[embed,%s]' % video_id, video_id)
+            video_id = self._search_regex(r'mediaId=(\d+)', embed_page, 'media id')
+        video_data = self._download_json(
+            'http://mais.uol.com.br/apiuol/v3/player/getMedia/%s.json' % video_id,
+            video_id)['item']
+        title = video_data['title']
+
+        query = {
+            'ver': video_data.get('numRevision', 2),
+            'r': 'http://mais.uol.com.br',
+        }
+        formats = []
+        for f in video_data.get('formats', []):
+            f_url = f.get('url') or f.get('secureUrl')
+            if not f_url:
+                continue
+            format_id = str_or_none(f.get('id'))
+            fmt = {
+                'format_id': format_id,
+                'url': update_url_query(f_url, query),
+            }
+            fmt.update(self._FORMATS.get(format_id, {}))
+            formats.append(fmt)
+        self._sort_formats(formats)
+
+        tags = []
+        for tag in video_data.get('tags', []):
+            tag_description = tag.get('description')
+            if not tag_description:
+                continue
+            tags.append(tag_description)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': clean_html(video_data.get('desMedia')),
+            'thumbnail': video_data.get('thumbnail'),
+            'duration': int_or_none(video_data.get('durationSeconds')) or parse_duration(video_data.get('duration')),
+            'tags': tags,
+            'formats': formats,
+        }

From 20ef4123b9ddbac62f5e4cbd27979adb9b0d948c Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 9 Aug 2016 15:13:15 +0100
Subject: [PATCH 5720/6219] [uol] remove unused import

---
 youtube_dl/extractor/uol.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/youtube_dl/extractor/uol.py b/youtube_dl/extractor/uol.py
index 5c6a3577b..c27c64387 100644
--- a/youtube_dl/extractor/uol.py
+++ b/youtube_dl/extractor/uol.py
@@ -1,8 +1,6 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..utils import (
     clean_html,

From cc9c8ce5df9c1ac2f369a8060cdcca1e18f25457 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 9 Aug 2016 22:24:58 +0700
Subject: [PATCH 5721/6219] [devscripts/prepare_manpage] Fix description
 strings starting with dash (Closes #10273)

---
 devscripts/prepare_manpage.py | 22 +++++++++++++---------
 1 file changed, 13 insertions(+), 9 deletions(-)

diff --git a/devscripts/prepare_manpage.py b/devscripts/prepare_manpage.py
index e3f6339b5..ce548739f 100644
--- a/devscripts/prepare_manpage.py
+++ b/devscripts/prepare_manpage.py
@@ -54,17 +54,21 @@ def filter_options(readme):
 
         if in_options:
             if line.lstrip().startswith('-'):
-                option, description = re.split(r'\s{2,}', line.lstrip())
-                split_option = option.split(' ')
+                split = re.split(r'\s{2,}', line.lstrip())
+                # Description string may start with `-` as well. If there is
+                # only one piece then it's a description bit not an option.
+                if len(split) > 1:
+                    option, description = split
+                    split_option = option.split(' ')
 
-                if not split_option[-1].startswith('-'):  # metavar
-                    option = ' '.join(split_option[:-1] + ['*%s*' % split_option[-1]])
+                    if not split_option[-1].startswith('-'):  # metavar
+                        option = ' '.join(split_option[:-1] + ['*%s*' % split_option[-1]])
 
-                # Pandoc's definition_lists. See http://pandoc.org/README.html
-                # for more information.
-                ret += '\n%s\n:   %s\n' % (option, description)
-            else:
-                ret += line.lstrip() + '\n'
+                    # Pandoc's definition_lists. See http://pandoc.org/README.html
+                    # for more information.
+                    ret += '\n%s\n:   %s\n' % (option, description)
+                    continue
+            ret += line.lstrip() + '\n'
         else:
             ret += line + '\n'
 

From 5c8411e968a2e6962bc477356231483963248c83 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 10 Aug 2016 00:18:28 +0700
Subject: [PATCH 5722/6219] [ChangeLog] Actualize

---
 ChangeLog | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index 657ff3e48..2b6794c4a 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,8 +1,20 @@
 version <unreleased>
 
+Core
+* Make --metadata-from-title non fatal when title does not match the pattern
+* Introduce options for randomized sleep before each download
+  --min-sleep-interval and --max-sleep-interval (#9930)
+* Respect default in _search_json_ld
+
 Extractors
-* [kuwo:singer] Fix extraction
++ [uol] Add extractor for uol.com.br (#4263)
+* [rbmaradio] Fix extraction and extract all formats (#10242)
++ [sonyliv] Add extractor for sonyliv.com (#10258)
 * [aparat] Fix extraction
+* [cwtv] Extract HTTP formats
++ [rozhlas] Add extractor for prehravac.rozhlas.cz (#10253)
+* [kuwo:singer] Fix extraction
+
 
 version 2016.08.07
 

From b1ce2ba1978142c75418ead4b91ea6138bb37a16 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 10 Aug 2016 00:20:44 +0700
Subject: [PATCH 5723/6219] release 2016.08.10

---
 .github/ISSUE_TEMPLATE.md |  6 +++---
 CONTRIBUTING.md           |  2 +-
 ChangeLog                 |  2 +-
 README.md                 | 10 +++++++++-
 docs/supportedsites.md    |  3 +++
 youtube_dl/version.py     |  2 +-
 6 files changed, 18 insertions(+), 7 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 2319e45df..1c06ba36e 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.08.07*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.08.07**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.08.10*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.08.10**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.08.07
+[debug] youtube-dl version 2016.08.10
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index fbf0ab7e8..95392030e 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -46,7 +46,7 @@ Make sure that someone has not already opened the issue you're trying to open. S
 
 ###  Why are existing options not enough?
 
-Before requesting a new feature, please have a quick peek at [the list of supported options](https://github.com/rg3/youtube-dl/blob/master/README.md#synopsis). Many feature requests are for features that actually exist already! Please, absolutely do show off your work in the issue report and detail how the existing similar options do *not* solve your problem.
+Before requesting a new feature, please have a quick peek at [the list of supported options](https://github.com/rg3/youtube-dl/blob/master/README.md#options). Many feature requests are for features that actually exist already! Please, absolutely do show off your work in the issue report and detail how the existing similar options do *not* solve your problem.
 
 ###  Is there enough context in your bug report?
 
diff --git a/ChangeLog b/ChangeLog
index 2b6794c4a..ac9e78a8d 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2016.08.10
 
 Core
 * Make --metadata-from-title non fatal when title does not match the pattern
diff --git a/README.md b/README.md
index b42d5c730..cabbbef76 100644
--- a/README.md
+++ b/README.md
@@ -330,7 +330,15 @@ which means you can modify it, redistribute it or use it however you like.
                                      bidirectional text support. Requires bidiv
                                      or fribidi executable in PATH
     --sleep-interval SECONDS         Number of seconds to sleep before each
-                                     download.
+                                     download when used alone or a lower bound
+                                     of a range for randomized sleep before each
+                                     download (minimum possible number of
+                                     seconds to sleep) when used along with
+                                     --max-sleep-interval.
+    --max-sleep-interval SECONDS     Upper bound of a range for randomized sleep
+                                     before each download (maximum possible
+                                     number of seconds to sleep). Must only be
+                                     used along with --min-sleep-interval.
 
 ## Video Format Options:
     -f, --format FORMAT              Video format code, see the "FORMAT
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 3608e1807..a44167a94 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -564,6 +564,7 @@
  - **RoosterTeeth**
  - **RottenTomatoes**
  - **Roxwel**
+ - **Rozhlas**
  - **RTBF**
  - **rte**: Raidió Teilifís Éireann TV
  - **rte:radio**: Raidió Teilifís Éireann radio
@@ -621,6 +622,7 @@
  - **smotri:user**: Smotri.com user videos
  - **Snotr**
  - **Sohu**
+ - **SonyLIV**
  - **soundcloud**
  - **soundcloud:playlist**
  - **soundcloud:search**: Soundcloud search
@@ -747,6 +749,7 @@
  - **udemy:course**
  - **UDNEmbed**: 聯合影音
  - **Unistra**
+ - **uol.com.br**
  - **Urort**: NRK P3 Urørt
  - **URPlay**
  - **USAToday**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index b48552031..f7ad846d9 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.08.07'
+__version__ = '2016.08.10'

From 81c13222c6418b72e0d5e11ed37700e984c4de43 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 10 Aug 2016 11:36:49 +0800
Subject: [PATCH 5724/6219] [utils] Recognize more formats in unified_timestamp

Used in CtsNews
---
 ChangeLog           | 6 ++++++
 youtube_dl/utils.py | 1 +
 2 files changed, 7 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index ac9e78a8d..cd59a87a9 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Core
++ Recognize more formats in unified_timestamp
+
+
 version 2016.08.10
 
 Core
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index c50238ba1..a03f7184d 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -122,6 +122,7 @@ DATE_FORMATS = (
     '%Y %m %d',
     '%Y-%m-%d',
     '%Y/%m/%d',
+    '%Y/%m/%d %H:%M',
     '%Y/%m/%d %H:%M:%S',
     '%Y-%m-%d %H:%M:%S',
     '%Y-%m-%d %H:%M:%S.%f',

From 69d8eeeec56e4b9f4f3e59086a6f0b0820fd37c9 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 10 Aug 2016 11:38:19 +0800
Subject: [PATCH 5725/6219] [ctsnews] Fix extraction

---
 ChangeLog                       |  3 ++
 youtube_dl/extractor/ctsnews.py | 49 ++++++++++++++-------------------
 2 files changed, 24 insertions(+), 28 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index cd59a87a9..adbdc4f9b 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -3,6 +3,9 @@ version <unreleased>
 Core
 + Recognize more formats in unified_timestamp
 
+Extractors
+* [ctsnews] Fix extraction
+
 
 version 2016.08.10
 
diff --git a/youtube_dl/extractor/ctsnews.py b/youtube_dl/extractor/ctsnews.py
index 1622fc844..83ca90c3b 100644
--- a/youtube_dl/extractor/ctsnews.py
+++ b/youtube_dl/extractor/ctsnews.py
@@ -1,13 +1,12 @@
-# -*- coding: utf-8 -*-
+# coding: utf-8
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..utils import parse_iso8601, ExtractorError
+from ..utils import unified_timestamp
 
 
 class CtsNewsIE(InfoExtractor):
     IE_DESC = '華視新聞'
-    # https connection failed (Connection reset)
     _VALID_URL = r'https?://news\.cts\.com\.tw/[a-z]+/[a-z]+/\d+/(?P<id>\d+)\.html'
     _TESTS = [{
         'url': 'http://news.cts.com.tw/cts/international/201501/201501291578109.html',
@@ -16,7 +15,7 @@ class CtsNewsIE(InfoExtractor):
             'id': '201501291578109',
             'ext': 'mp4',
             'title': '以色列.真主黨交火 3人死亡',
-            'description': 'md5:95e9b295c898b7ff294f09d450178d7d',
+            'description': '以色列和黎巴嫩真主黨，爆發五年最嚴重衝突，雙方砲轟交火，兩名以軍死亡，還有一名西班牙籍的聯合國維和人...',
             'timestamp': 1422528540,
             'upload_date': '20150129',
         }
@@ -28,7 +27,7 @@ class CtsNewsIE(InfoExtractor):
             'id': '201309031304098',
             'ext': 'mp4',
             'title': '韓國31歲童顏男 貌如十多歲小孩',
-            'description': 'md5:f183feeba3752b683827aab71adad584',
+            'description': '越有年紀的人，越希望看起來年輕一點，而南韓卻有一位31歲的男子，看起來像是11、12歲的小孩，身...',
             'thumbnail': 're:^https?://.*\.jpg$',
             'timestamp': 1378205880,
             'upload_date': '20130903',
@@ -36,8 +35,7 @@ class CtsNewsIE(InfoExtractor):
     }, {
         # With Youtube embedded video
         'url': 'http://news.cts.com.tw/cts/money/201501/201501291578003.html',
-        'md5': '1d842c771dc94c8c3bca5af2cc1db9c5',
-        'add_ie': ['Youtube'],
+        'md5': 'e4726b2ccd70ba2c319865e28f0a91d1',
         'info_dict': {
             'id': 'OVbfO7d0_hQ',
             'ext': 'mp4',
@@ -47,42 +45,37 @@ class CtsNewsIE(InfoExtractor):
             'upload_date': '20150128',
             'uploader_id': 'TBSCTS',
             'uploader': '中華電視公司',
-        }
+        },
+        'add_ie': ['Youtube'],
     }]
 
     def _real_extract(self, url):
         news_id = self._match_id(url)
         page = self._download_webpage(url, news_id)
 
-        if self._search_regex(r'(CTSPlayer2)', page, 'CTSPlayer2 identifier', default=None):
-            feed_url = self._html_search_regex(
-                r'(http://news\.cts\.com\.tw/action/mp4feed\.php\?news_id=\d+)',
-                page, 'feed url')
-            video_url = self._download_webpage(
-                feed_url, news_id, note='Fetching feed')
+        news_id = self._hidden_inputs(page).get('get_id')
+
+        if news_id:
+            mp4_feed = self._download_json(
+                'http://news.cts.com.tw/action/test_mp4feed.php',
+                news_id, note='Fetching feed', query={'news_id': news_id})
+            video_url = mp4_feed['source_url']
         else:
             self.to_screen('Not CTSPlayer video, trying Youtube...')
             youtube_url = self._search_regex(
-                r'src="(//www\.youtube\.com/embed/[^"]+)"', page, 'youtube url',
-                default=None)
-            if not youtube_url:
-                raise ExtractorError('The news includes no videos!', expected=True)
+                r'src="(//www\.youtube\.com/embed/[^"]+)"', page, 'youtube url')
 
-            return {
-                '_type': 'url',
-                'url': youtube_url,
-                'ie_key': 'Youtube',
-            }
+            return self.url_result(youtube_url, ie='Youtube')
 
         description = self._html_search_meta('description', page)
-        title = self._html_search_meta('title', page)
+        title = self._html_search_meta('title', page, fatal=True)
         thumbnail = self._html_search_meta('image', page)
 
         datetime_str = self._html_search_regex(
-            r'(\d{4}/\d{2}/\d{2} \d{2}:\d{2})', page, 'date and time')
-        # Transform into ISO 8601 format with timezone info
-        datetime_str = datetime_str.replace('/', '-') + ':00+0800'
-        timestamp = parse_iso8601(datetime_str, delimiter=' ')
+            r'(\d{4}/\d{2}/\d{2} \d{2}:\d{2})', page, 'date and time', fatal=False)
+        timestamp = None
+        if datetime_str:
+            timestamp = unified_timestamp(datetime_str) - 8 * 3600
 
         return {
             'id': news_id,

From 57ce8a6d08a05140230864eccbc52029f1fd46c1 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 10 Aug 2016 14:17:22 +0100
Subject: [PATCH 5726/6219] [wat] improve extraction(#10281)

add alternative method to extract http formats
works even if the video is geo-restricted or removed
from public access(most of the cases)
---
 youtube_dl/extractor/wat.py | 126 +++++++++++++++++++++++-------------
 1 file changed, 81 insertions(+), 45 deletions(-)

diff --git a/youtube_dl/extractor/wat.py b/youtube_dl/extractor/wat.py
index 48fc438ed..9f1b8b4b5 100644
--- a/youtube_dl/extractor/wat.py
+++ b/youtube_dl/extractor/wat.py
@@ -9,6 +9,7 @@ from ..utils import (
     ExtractorError,
     unified_strdate,
     HEADRequest,
+    int_or_none,
 )
 
 
@@ -30,48 +31,58 @@ class WatIE(InfoExtractor):
         },
         {
             'url': 'http://www.wat.tv/video/gregory-lemarchal-voix-ange-6z1v7_6ygkj_.html',
-            'md5': 'fbc84e4378165278e743956d9c1bf16b',
+            'md5': '34bdfa5ca9fd3c7eb88601b635b0424c',
             'info_dict': {
                 'id': '11713075',
                 'ext': 'mp4',
                 'title': 'Grégory Lemarchal, une voix d\'ange depuis 10 ans (1/3)',
-                'description': 'md5:b7a849cf16a2b733d9cd10c52906dee3',
                 'upload_date': '20140816',
-                'duration': 2910,
             },
-            'skip': "Ce contenu n'est pas disponible pour l'instant.",
+            'expected_warnings': ["Ce contenu n'est pas disponible pour l'instant."],
         },
     ]
 
+    _FORMATS = (
+        (200, 416, 234),
+        (400, 480, 270),
+        (600, 640, 360),
+        (1200, 640, 360),
+        (1800, 960, 540),
+        (2500, 1280, 720),
+    )
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
         video_id = video_id if video_id.isdigit() and len(video_id) > 6 else compat_str(int(video_id, 36))
 
         # 'contentv4' is used in the website, but it also returns the related
         # videos, we don't need them
-        video_info = self._download_json(
-            'http://www.wat.tv/interface/contentv3/' + video_id, video_id)['media']
+        video_data = self._download_json(
+            'http://www.wat.tv/interface/contentv4s/' + video_id, video_id)
+        video_info = video_data['media']
 
         error_desc = video_info.get('error_desc')
         if error_desc:
-            raise ExtractorError(
-                '%s returned error: %s' % (self.IE_NAME, error_desc), expected=True)
+            self.report_warning(
+                '%s returned error: %s' % (self.IE_NAME, error_desc))
 
         chapters = video_info['chapters']
-        first_chapter = chapters[0]
+        if chapters:
+            first_chapter = chapters[0]
 
-        def video_id_for_chapter(chapter):
-            return chapter['tc_start'].split('-')[0]
+            def video_id_for_chapter(chapter):
+                return chapter['tc_start'].split('-')[0]
 
-        if video_id_for_chapter(first_chapter) != video_id:
-            self.to_screen('Multipart video detected')
-            entries = [self.url_result('wat:%s' % video_id_for_chapter(chapter)) for chapter in chapters]
-            return self.playlist_result(entries, video_id, video_info['title'])
-        # Otherwise we can continue and extract just one part, we have to use
-        # the video id for getting the video url
+            if video_id_for_chapter(first_chapter) != video_id:
+                self.to_screen('Multipart video detected')
+                entries = [self.url_result('wat:%s' % video_id_for_chapter(chapter)) for chapter in chapters]
+                return self.playlist_result(entries, video_id, video_info['title'])
+            # Otherwise we can continue and extract just one part, we have to use
+            # the video id for getting the video url
+        else:
+            first_chapter = video_info
 
-        date_diffusion = first_chapter.get('date_diffusion')
-        upload_date = unified_strdate(date_diffusion) if date_diffusion else None
+        title = first_chapter['title']
 
         def extract_url(path_template, url_type):
             req_url = 'http://www.wat.tv/get/%s' % (path_template % video_id)
@@ -83,36 +94,61 @@ class WatIE(InfoExtractor):
                     expected=True)
             return red_url
 
-        m3u8_url = extract_url('ipad/%s.m3u8', 'm3u8')
-        http_url = extract_url('android5/%s.mp4', 'http')
-
         formats = []
-        m3u8_formats = self._extract_m3u8_formats(
-            m3u8_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls')
-        formats.extend(m3u8_formats)
-        formats.extend(self._extract_f4m_formats(
-            m3u8_url.replace('ios.', 'web.').replace('.m3u8', '.f4m'),
-            video_id, f4m_id='hds', fatal=False))
-        for m3u8_format in m3u8_formats:
-            vbr, abr = m3u8_format.get('vbr'), m3u8_format.get('abr')
-            if not vbr or not abr:
-                continue
-            f = m3u8_format.copy()
-            f.update({
-                'url': re.sub(r'%s-\d+00-\d+' % video_id, '%s-%d00-%d' % (video_id, round(vbr / 100), round(abr)), http_url),
-                'format_id': f['format_id'].replace('hls', 'http'),
-                'protocol': 'http',
-            })
-            formats.append(f)
-        self._sort_formats(formats)
+        try:
+            http_url = extract_url('android5/%s.mp4', 'http')
+            m3u8_url = extract_url('ipad/%s.m3u8', 'm3u8')
+            m3u8_formats = self._extract_m3u8_formats(
+                m3u8_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls')
+            formats.extend(m3u8_formats)
+            formats.extend(self._extract_f4m_formats(
+                m3u8_url.replace('ios.', 'web.').replace('.m3u8', '.f4m'),
+                video_id, f4m_id='hds', fatal=False))
+            for m3u8_format in m3u8_formats:
+                vbr, abr = m3u8_format.get('vbr'), m3u8_format.get('abr')
+                if not vbr or not abr:
+                    continue
+                format_id = m3u8_format['format_id'].replace('hls', 'http')
+                fmt_url = re.sub(r'%s-\d+00-\d+' % video_id, '%s-%d00-%d' % (video_id, round(vbr / 100), round(abr)), http_url)
+                if self._is_valid_url(fmt_url, video_id, format_id):
+                    f = m3u8_format.copy()
+                    f.update({
+                        'url': fmt_url,
+                        'format_id': format_id,
+                        'protocol': 'http',
+                    })
+                    formats.append(f)
+            self._sort_formats(formats)
+        except ExtractorError:
+            abr = 64
+            for vbr, width, height in self._FORMATS:
+                tbr = vbr + abr
+                format_id = 'http-%s' % tbr
+                fmt_url = 'http://dnl.adv.tf1.fr/2/USP-0x0/%s/%s/%s/ssm/%s-%s-64k.mp4' % (video_id[-4:-2], video_id[-2:], video_id, video_id, vbr)
+                if self._is_valid_url(fmt_url, video_id, format_id):
+                    formats.append({
+                        'format_id': format_id,
+                        'url': fmt_url,
+                        'vbr': vbr,
+                        'abr': abr,
+                        'width': width,
+                        'height': height,
+                    })
+
+        date_diffusion = first_chapter.get('date_diffusion') or video_data.get('configv4', {}).get('estatS4')
+        upload_date = unified_strdate(date_diffusion) if date_diffusion else None
+        duration = None
+        files = video_info['files']
+        if files:
+            duration = int_or_none(files[0].get('duration'))
 
         return {
             'id': video_id,
-            'title': first_chapter['title'],
-            'thumbnail': first_chapter['preview'],
-            'description': first_chapter['description'],
-            'view_count': video_info['views'],
+            'title': title,
+            'thumbnail': first_chapter.get('preview'),
+            'description': first_chapter.get('description'),
+            'view_count': int_or_none(video_info.get('views')),
             'upload_date': upload_date,
-            'duration': video_info['files'][0]['duration'],
+            'duration': duration,
             'formats': formats,
         }

From c3fa77bdef14643c966913c49f0400ebc1e46b10 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 10 Aug 2016 21:00:40 +0700
Subject: [PATCH 5727/6219] [formula1] Relax _VALID_URL (Closes #10283)

---
 youtube_dl/extractor/formula1.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/formula1.py b/youtube_dl/extractor/formula1.py
index 322c41e5a..8c417ab65 100644
--- a/youtube_dl/extractor/formula1.py
+++ b/youtube_dl/extractor/formula1.py
@@ -5,8 +5,8 @@ from .common import InfoExtractor
 
 
 class Formula1IE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?formula1\.com/content/fom-website/en/video/\d{4}/\d{1,2}/(?P<id>.+?)\.html'
-    _TEST = {
+    _VALID_URL = r'https?://(?:www\.)?formula1\.com/(?:content/fom-website/)?en/video/\d{4}/\d{1,2}/(?P<id>.+?)\.html'
+    _TESTS = [{
         'url': 'http://www.formula1.com/content/fom-website/en/video/2016/5/Race_highlights_-_Spain_2016.html',
         'md5': '8c79e54be72078b26b89e0e111c0502b',
         'info_dict': {
@@ -15,7 +15,10 @@ class Formula1IE(InfoExtractor):
             'title': 'Race highlights - Spain 2016',
         },
         'add_ie': ['Ooyala'],
-    }
+    }, {
+        'url': 'http://www.formula1.com/en/video/2016/5/Race_highlights_-_Spain_2016.html',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         display_id = self._match_id(url)

From 7f2ed4759513b153e526cd890fd5b8877f56f1c2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 10 Aug 2016 21:07:43 +0700
Subject: [PATCH 5728/6219] [rtlnl] Relax _VALID_URL (Closes #10282)

---
 youtube_dl/extractor/rtlnl.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/rtlnl.py b/youtube_dl/extractor/rtlnl.py
index 4d612b5e3..f0250af8a 100644
--- a/youtube_dl/extractor/rtlnl.py
+++ b/youtube_dl/extractor/rtlnl.py
@@ -14,7 +14,7 @@ class RtlNlIE(InfoExtractor):
     _VALID_URL = r'''(?x)
         https?://(?:www\.)?
         (?:
-            rtlxl\.nl/\#!/[^/]+/|
+            rtlxl\.nl/[^\#]*\#!/[^/]+/|
             rtl\.nl/system/videoplayer/(?:[^/]+/)+(?:video_)?embed\.html\b.+?\buuid=
         )
         (?P<id>[0-9a-f-]+)'''
@@ -67,6 +67,9 @@ class RtlNlIE(InfoExtractor):
     }, {
         'url': 'http://www.rtl.nl/system/videoplayer/derden/embed.html#!/uuid=bb0353b0-d6a4-1dad-90e9-18fe75b8d1f0',
         'only_matching': True,
+    }, {
+        'url': 'http://rtlxl.nl/?_ga=1.204735956.572365465.1466978370#!/rtl-nieuws-132237/3c487912-023b-49ac-903e-2c5d79f8410f',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 7f832413d6e4aa5aae4c904c42e0ecf4ae72aaf9 Mon Sep 17 00:00:00 2001
From: lkho <llhmtc@gmail.com>
Date: Tue, 9 Aug 2016 15:25:23 +0800
Subject: [PATCH 5729/6219] Preserve line endings for downloaded subtitle files

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 193f8db9f..fd7577bb8 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1603,7 +1603,7 @@ class YoutubeDL(object):
                         self.to_screen('[info] Video subtitle %s.%s is already_present' % (sub_lang, sub_format))
                     else:
                         self.to_screen('[info] Writing video subtitles to: ' + sub_filename)
-                        with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile:
+                        with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8', newline='') as subfile:
                             subfile.write(sub_data)
                 except (OSError, IOError):
                     self.report_error('Cannot write subtitles file ' + sub_filename)

From b1927f4e8a07a7893392135a71fdb6818295bbad Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 11 Aug 2016 19:04:23 +0800
Subject: [PATCH 5730/6219] [YoutubeDL] Disable newline conversion when writing
 subtitles

By default io.open() convert all '\n' occurrences to '\r\n' when writing
files. If the content already contains '\r\n', it will be converted to
'\r\r\n', breaking some video players.
---
 youtube_dl/YoutubeDL.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index fd7577bb8..e844dc98a 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1603,6 +1603,8 @@ class YoutubeDL(object):
                         self.to_screen('[info] Video subtitle %s.%s is already_present' % (sub_lang, sub_format))
                     else:
                         self.to_screen('[info] Writing video subtitles to: ' + sub_filename)
+                        # Use newline='' to prevent conversion of newline characters
+                        # See https://github.com/rg3/youtube-dl/issues/10268
                         with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8', newline='') as subfile:
                             subfile.write(sub_data)
                 except (OSError, IOError):

From e5f878c20573b258cad1974cc79a0526bcd1d46b Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 11 Aug 2016 19:13:41 +0800
Subject: [PATCH 5731/6219] [ChangeLog] Add change log for #10269

[skip ci]
---
 ChangeLog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ChangeLog b/ChangeLog
index adbdc4f9b..b6ea39cba 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
 version <unreleased>
 
 Core
+* Subtitles are now written as is. Newline conversions are disabled. (#10268)
 + Recognize more formats in unified_timestamp
 
 Extractors

From 30b25d382d1c2e06c19d8730ecbc0776a436d967 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 11 Aug 2016 21:42:55 +0700
Subject: [PATCH 5732/6219] [francetvinfo] Relax _VALID_URL

---
 youtube_dl/extractor/francetv.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index 7653975e3..3233f66d5 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -131,7 +131,7 @@ class PluzzIE(FranceTVBaseInfoExtractor):
 
 class FranceTvInfoIE(FranceTVBaseInfoExtractor):
     IE_NAME = 'francetvinfo.fr'
-    _VALID_URL = r'https?://(?:www|mobile|france3-regions)\.francetvinfo\.fr/.*/(?P<title>.+)\.html'
+    _VALID_URL = r'https?://(?:www|mobile|france3-regions)\.francetvinfo\.fr/(?:[^/]+/)*(?P<title>[^/?#&.]+)'
 
     _TESTS = [{
         'url': 'http://www.francetvinfo.fr/replay-jt/france-3/soir-3/jt-grand-soir-3-lundi-26-aout-2013_393427.html',
@@ -206,6 +206,9 @@ class FranceTvInfoIE(FranceTVBaseInfoExtractor):
             'uploader_id': 'x2q2ez',
         },
         'add_ie': ['Dailymotion'],
+    }, {
+        'url': 'http://france3-regions.francetvinfo.fr/limousin/emissions/jt-1213-limousin',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 0c070681c56589d44f81df8ed2165bca4333cef5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 11 Aug 2016 23:37:56 +0700
Subject: [PATCH 5733/6219] [chirbit] Fix extraction (Closes #10296)

---
 youtube_dl/extractor/chirbit.py | 54 ++++++++++++++++++---------------
 1 file changed, 30 insertions(+), 24 deletions(-)

diff --git a/youtube_dl/extractor/chirbit.py b/youtube_dl/extractor/chirbit.py
index b1eeaf101..b43518652 100644
--- a/youtube_dl/extractor/chirbit.py
+++ b/youtube_dl/extractor/chirbit.py
@@ -1,30 +1,33 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import base64
+
 from .common import InfoExtractor
-from ..utils import (
-    parse_duration,
-    int_or_none,
-)
+from ..utils import parse_duration
 
 
 class ChirbitIE(InfoExtractor):
     IE_NAME = 'chirbit'
     _VALID_URL = r'https?://(?:www\.)?chirb\.it/(?:(?:wp|pl)/|fb_chirbit_player\.swf\?key=)?(?P<id>[\da-zA-Z]+)'
     _TESTS = [{
-        'url': 'http://chirb.it/PrIPv5',
-        'md5': '9847b0dad6ac3e074568bf2cfb197de8',
+        'url': 'http://chirb.it/be2abG',
         'info_dict': {
-            'id': 'PrIPv5',
+            'id': 'be2abG',
             'ext': 'mp3',
-            'title': 'Фасадстрой',
-            'duration': 52,
-            'view_count': int,
-            'comment_count': int,
+            'title': 'md5:f542ea253f5255240be4da375c6a5d7e',
+            'description': 'md5:f24a4e22a71763e32da5fed59e47c770',
+            'duration': 306,
+        },
+        'params': {
+            'skip_download': True,
         }
     }, {
         'url': 'https://chirb.it/fb_chirbit_player.swf?key=PrIPv5',
         'only_matching': True,
+    }, {
+        'url': 'https://chirb.it/wp/MN58c2',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
@@ -33,27 +36,30 @@ class ChirbitIE(InfoExtractor):
         webpage = self._download_webpage(
             'http://chirb.it/%s' % audio_id, audio_id)
 
-        audio_url = self._search_regex(
-            r'"setFile"\s*,\s*"([^"]+)"', webpage, 'audio url')
+        data_fd = self._search_regex(
+            r'data-fd=(["\'])(?P<url>(?:(?!\1).)+)\1',
+            webpage, 'data fd', group='url')
+
+        # Reverse engineered from https://chirb.it/js/chirbit.player.js (look
+        # for soundURL)
+        audio_url = base64.b64decode(
+            data_fd[::-1].encode('ascii')).decode('utf-8')
 
         title = self._search_regex(
-            r'itemprop="name">([^<]+)', webpage, 'title')
-        duration = parse_duration(self._html_search_meta(
-            'duration', webpage, 'duration', fatal=False))
-        view_count = int_or_none(self._search_regex(
-            r'itemprop="playCount"\s*>(\d+)', webpage,
-            'listen count', fatal=False))
-        comment_count = int_or_none(self._search_regex(
-            r'>(\d+) Comments?:', webpage,
-            'comment count', fatal=False))
+            r'class=["\']chirbit-title["\'][^>]*>([^<]+)', webpage, 'title')
+        description = self._search_regex(
+            r'<h3>Description</h3>\s*<pre[^>]*>([^<]+)</pre>',
+            webpage, 'description', default=None)
+        duration = parse_duration(self._search_regex(
+            r'class=["\']c-length["\'][^>]*>([^<]+)',
+            webpage, 'duration', fatal=False))
 
         return {
             'id': audio_id,
             'url': audio_url,
             'title': title,
+            'description': description,
             'duration': duration,
-            'view_count': view_count,
-            'comment_count': comment_count,
         }
 
 

From 0aef0771f8cf18d97ce1b6b9123ce76bae45f3ef Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 11 Aug 2016 23:47:27 +0700
Subject: [PATCH 5734/6219] [drtuber] Make dislike count optional (Closes
 #10297)

---
 youtube_dl/extractor/drtuber.py | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/drtuber.py b/youtube_dl/extractor/drtuber.py
index 639f9182c..4e5557703 100644
--- a/youtube_dl/extractor/drtuber.py
+++ b/youtube_dl/extractor/drtuber.py
@@ -3,7 +3,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import str_to_int
+from ..utils import (
+    NO_DEFAULT,
+    str_to_int,
+)
 
 
 class DrTuberIE(InfoExtractor):
@@ -17,7 +20,6 @@ class DrTuberIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'hot perky blonde naked golf',
             'like_count': int,
-            'dislike_count': int,
             'comment_count': int,
             'categories': ['Babe', 'Blonde', 'Erotic', 'Outdoor', 'Softcore', 'Solo'],
             'thumbnail': 're:https?://.*\.jpg$',
@@ -43,18 +45,20 @@ class DrTuberIE(InfoExtractor):
             r'poster="([^"]+)"',
             webpage, 'thumbnail', fatal=False)
 
-        def extract_count(id_, name):
+        def extract_count(id_, name, default=NO_DEFAULT):
             return str_to_int(self._html_search_regex(
                 r'<span[^>]+(?:class|id)="%s"[^>]*>([\d,\.]+)</span>' % id_,
-                webpage, '%s count' % name, fatal=False))
+                webpage, '%s count' % name, default=default, fatal=False))
 
         like_count = extract_count('rate_likes', 'like')
-        dislike_count = extract_count('rate_dislikes', 'dislike')
+        dislike_count = extract_count('rate_dislikes', 'dislike', default=None)
         comment_count = extract_count('comments_count', 'comment')
 
         cats_str = self._search_regex(
-            r'<div[^>]+class="categories_list">(.+?)</div>', webpage, 'categories', fatal=False)
-        categories = [] if not cats_str else re.findall(r'<a title="([^"]+)"', cats_str)
+            r'<div[^>]+class="categories_list">(.+?)</div>',
+            webpage, 'categories', fatal=False)
+        categories = [] if not cats_str else re.findall(
+            r'<a title="([^"]+)"', cats_str)
 
         return {
             'id': video_id,

From 367976d49fba48ea44fc5bf622adcc989896f29a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 11 Aug 2016 23:47:52 +0700
Subject: [PATCH 5735/6219] [drtuber] Improve title extraction

---
 youtube_dl/extractor/drtuber.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/drtuber.py b/youtube_dl/extractor/drtuber.py
index 4e5557703..e8870c460 100644
--- a/youtube_dl/extractor/drtuber.py
+++ b/youtube_dl/extractor/drtuber.py
@@ -38,7 +38,9 @@ class DrTuberIE(InfoExtractor):
             r'<source src="([^"]+)"', webpage, 'video URL')
 
         title = self._html_search_regex(
-            [r'<p[^>]+class="title_substrate">([^<]+)</p>', r'<title>([^<]+) - \d+'],
+            (r'class="title_watch"[^>]*><p>([^<]+)<',
+             r'<p[^>]+class="title_substrate">([^<]+)</p>',
+             r'<title>([^<]+) - \d+'),
             webpage, 'title')
 
         thumbnail = self._html_search_regex(

From 0fd1b1624cc42412fe4701f9de09f49adfa467f8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 11 Aug 2016 23:52:17 +0700
Subject: [PATCH 5736/6219] [goldenmoustache] Remove extractor (Closes #10298)

Now uses dailymotion
---
 youtube_dl/extractor/extractors.py      |  2 +-
 youtube_dl/extractor/goldenmoustache.py | 48 -------------------------
 2 files changed, 1 insertion(+), 49 deletions(-)
 delete mode 100644 youtube_dl/extractor/goldenmoustache.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 387230be0..c0c18393f 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -311,7 +311,6 @@ from .globo import (
 )
 from .godtube import GodTubeIE
 from .godtv import GodTVIE
-from .goldenmoustache import GoldenMoustacheIE
 from .golem import GolemIE
 from .googledrive import GoogleDriveIE
 from .googleplus import GooglePlusIE
@@ -1004,6 +1003,7 @@ from .viki import (
     VikiIE,
     VikiChannelIE,
 )
+from .viu import ViuIE
 from .vk import (
     VKIE,
     VKUserVideosIE,
diff --git a/youtube_dl/extractor/goldenmoustache.py b/youtube_dl/extractor/goldenmoustache.py
deleted file mode 100644
index 0fb509724..000000000
--- a/youtube_dl/extractor/goldenmoustache.py
+++ /dev/null
@@ -1,48 +0,0 @@
-from __future__ import unicode_literals
-
-from .common import InfoExtractor
-
-
-class GoldenMoustacheIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?goldenmoustache\.com/(?P<display_id>[\w-]+)-(?P<id>\d+)'
-    _TESTS = [{
-        'url': 'http://www.goldenmoustache.com/suricate-le-poker-3700/',
-        'md5': '0f904432fa07da5054d6c8beb5efb51a',
-        'info_dict': {
-            'id': '3700',
-            'ext': 'mp4',
-            'title': 'Suricate - Le Poker',
-            'description': 'md5:3d1f242f44f8c8cb0a106f1fd08e5dc9',
-            'thumbnail': 're:^https?://.*\.jpg$',
-        }
-    }, {
-        'url': 'http://www.goldenmoustache.com/le-lab-tout-effacer-mc-fly-et-carlito-55249/',
-        'md5': '27f0c50fb4dd5f01dc9082fc67cd5700',
-        'info_dict': {
-            'id': '55249',
-            'ext': 'mp4',
-            'title': 'Le LAB - Tout Effacer (Mc Fly et Carlito)',
-            'description': 'md5:9b7fbf11023fb2250bd4b185e3de3b2a',
-            'thumbnail': 're:^https?://.*\.(?:png|jpg)$',
-        }
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-
-        video_url = self._html_search_regex(
-            r'data-src-type="mp4" data-src="([^"]+)"', webpage, 'video URL')
-        title = self._html_search_regex(
-            r'<title>(.*?)(?: - Golden Moustache)?</title>', webpage, 'title')
-        thumbnail = self._og_search_thumbnail(webpage)
-        description = self._og_search_description(webpage)
-
-        return {
-            'id': video_id,
-            'url': video_url,
-            'ext': 'mp4',
-            'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
-        }

From a3be69b7f0d21c024e288a42864704f5c81d9dd0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 12 Aug 2016 00:14:51 +0700
Subject: [PATCH 5737/6219] [viu] Remove from extractors

---
 youtube_dl/extractor/extractors.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index c0c18393f..acf4e5d62 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1003,7 +1003,6 @@ from .viki import (
     VikiIE,
     VikiChannelIE,
 )
-from .viu import ViuIE
 from .vk import (
     VKIE,
     VKUserVideosIE,

From fff37cfd4f09db6bb9f35da59b7d73b4e72855ff Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 12 Aug 2016 00:18:28 +0700
Subject: [PATCH 5738/6219] [ChangeLog] Actualize

---
 ChangeLog | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index b6ea39cba..985dca7d0 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -5,6 +5,14 @@ Core
 + Recognize more formats in unified_timestamp
 
 Extractors
+- [goldenmoustache] Remove extractor (#10298)
+* [drtuber] Improve title extraction
+* [drtuber] Make dislike count optional (#10297)
+* [chirbit] Fix extraction (#10296)
+* [francetvinfo] Relax URL regular expression
+* [rtlnl] Relax URL regular expression (#10282)
+* [formula1] Relax URL regular expression (#10283)
+* [wat] Improve extraction (#10281)
 * [ctsnews] Fix extraction
 
 

From b0081562d240fbe2ad854c53b6e098fa7e626247 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 12 Aug 2016 00:22:22 +0700
Subject: [PATCH 5739/6219] release 2016.08.12

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 1 -
 youtube_dl/version.py     | 2 +-
 4 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 1c06ba36e..6fdb2f77b 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.08.10*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.08.10**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.08.12*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.08.12**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.08.10
+[debug] youtube-dl version 2016.08.12
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 985dca7d0..376d96d12 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2016.08.12
 
 Core
 * Subtitles are now written as is. Newline conversions are disabled. (#10268)
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index a44167a94..8fb581d2b 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -265,7 +265,6 @@
  - **GloboArticle**
  - **GodTube**
  - **GodTV**
- - **GoldenMoustache**
  - **Golem**
  - **GoogleDrive**
  - **Goshgay**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index f7ad846d9..becf14458 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.08.10'
+__version__ = '2016.08.12'

From 990d533ee4a33f8c59921a4152817ff4835a974f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 12 Aug 2016 00:56:16 +0700
Subject: [PATCH 5740/6219] [crunchyroll] Add support for HLS (Closes #10301)

---
 youtube_dl/extractor/crunchyroll.py | 32 +++++++++++++++++++++++++----
 1 file changed, 28 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 90a64303d..6d3abb52f 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -114,6 +114,21 @@ class CrunchyrollIE(CrunchyrollBaseIE):
             # rtmp
             'skip_download': True,
         },
+    }, {
+        'url': 'http://www.crunchyroll.com/rezero-starting-life-in-another-world-/episode-5-the-morning-of-our-promise-is-still-distant-702409',
+        'info_dict': {
+            'id': '702409',
+            'ext': 'mp4',
+            'title': 'Re:ZERO -Starting Life in Another World- Episode 5 – The Morning of Our Promise Is Still Distant',
+            'description': 'md5:97664de1ab24bbf77a9c01918cb7dca9',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'uploader': 'TV TOKYO',
+            'upload_date': '20160508',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
     }, {
         'url': 'http://www.crunchyroll.fr/girl-friend-beta/episode-11-goodbye-la-mode-661697',
         'only_matching': True,
@@ -336,9 +351,18 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
             if video_encode_id in video_encode_ids:
                 continue
             video_encode_ids.append(video_encode_id)
+
+            video_file = xpath_text(stream_info, './file')
+            if not video_file:
+                continue
+            if video_file.startswith('http'):
+                formats.extend(self._extract_m3u8_formats(
+                    video_file, video_id, 'mp4', entry_protocol='m3u8_native',
+                    m3u8_id='hls', fatal=False))
+                continue
+
             video_url = xpath_text(stream_info, './host')
-            video_play_path = xpath_text(stream_info, './file')
-            if not video_url or not video_play_path:
+            if not video_url:
                 continue
             metadata = stream_info.find('./metadata')
             format_info = {
@@ -353,7 +377,7 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
                 parsed_video_url = compat_urlparse.urlparse(video_url)
                 direct_video_url = compat_urlparse.urlunparse(parsed_video_url._replace(
                     netloc='v.lvlt.crcdn.net',
-                    path='%s/%s' % (remove_end(parsed_video_url.path, '/'), video_play_path.split(':')[-1])))
+                    path='%s/%s' % (remove_end(parsed_video_url.path, '/'), video_file.split(':')[-1])))
                 if self._is_valid_url(direct_video_url, video_id, video_format):
                     format_info.update({
                         'url': direct_video_url,
@@ -363,7 +387,7 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
 
             format_info.update({
                 'url': video_url,
-                'play_path': video_play_path,
+                'play_path': video_file,
                 'ext': 'flv',
             })
             formats.append(format_info)

From 3cddb8d6a776b09afd7f50772fa30cb536b1149a Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 12 Aug 2016 08:38:06 +0100
Subject: [PATCH 5741/6219] [pbs] check all http formats and remove unnecessary
 request

- some of the quality that not reported in the documentation
are available(4500k, 6500k)
- the videoInfo request doesn't work for a long time
---
 youtube_dl/extractor/pbs.py | 17 +++--------------
 1 file changed, 3 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index f6f423597..6e2ef0fba 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -448,17 +448,6 @@ class PBSIE(InfoExtractor):
                     redirects.append(redirect)
                     redirect_urls.add(redirect_url)
 
-        try:
-            video_info = self._download_json(
-                'http://player.pbs.org/videoInfo/%s?format=json&type=partner' % video_id,
-                display_id, 'Downloading video info JSON')
-            extract_redirect_urls(video_info)
-            info = video_info
-        except ExtractorError as e:
-            # videoInfo API may not work for some videos
-            if not isinstance(e.cause, compat_HTTPError) or e.cause.code != 404:
-                raise
-
         # Player pages may also serve different qualities
         for page in ('widget/partnerplayer', 'portalplayer'):
             player = self._download_webpage(
@@ -511,12 +500,12 @@ class PBSIE(InfoExtractor):
             formats))
         if http_url:
             for m3u8_format in m3u8_formats:
-                bitrate = self._search_regex(r'(\d+k)', m3u8_format['url'], 'bitrate', default=None)
+                bitrate = self._search_regex(r'(\d+)k', m3u8_format['url'], 'bitrate', default=None)
                 # extract only the formats that we know that they will be available as http format.
                 # https://projects.pbs.org/confluence/display/coveapi/COVE+Video+Specifications
-                if not bitrate or bitrate not in ('400k', '800k', '1200k', '2500k'):
+                if not bitrate or int(bitrate) < 400:
                     continue
-                f_url = re.sub(r'\d+k|baseline', bitrate, http_url)
+                f_url = re.sub(r'\d+k|baseline', bitrate + 'k', http_url)
                 # This may produce invalid links sometimes (e.g.
                 # http://www.pbs.org/wgbh/frontline/film/suicide-plan)
                 if not self._is_valid_url(f_url, display_id, 'http-%s video' % bitrate):

From 98e698f1ff3fd467ff03e10a8f8881cd06345ca7 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 12 Aug 2016 12:30:02 +0100
Subject: [PATCH 5742/6219] [external/curl] respect more downloader options and
 display progress

---
 youtube_dl/downloader/external.py | 15 +++++++++++++++
 youtube_dl/utils.py               |  2 ++
 2 files changed, 17 insertions(+)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index fae245024..f0c30007f 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -96,6 +96,12 @@ class CurlFD(ExternalFD):
         cmd = [self.exe, '--location', '-o', tmpfilename]
         for key, val in info_dict['http_headers'].items():
             cmd += ['--header', '%s: %s' % (key, val)]
+        cmd += self._bool_option('--continue-at', 'continuedl', '-', '0')
+        cmd += self._valueless_option('--silent', 'noprogress')
+        cmd += self._valueless_option('--verbose', 'verbose')
+        cmd += self._option('--limit-rate', 'ratelimit')
+        cmd += self._option('--retry', 'retries')
+        cmd += self._option('--max-filesize', 'max_filesize')
         cmd += self._option('--interface', 'source_address')
         cmd += self._option('--proxy', 'proxy')
         cmd += self._valueless_option('--insecure', 'nocheckcertificate')
@@ -103,6 +109,15 @@ class CurlFD(ExternalFD):
         cmd += ['--', info_dict['url']]
         return cmd
 
+    def _call_downloader(self, tmpfilename, info_dict):
+        cmd = [encodeArgument(a) for a in self._make_cmd(tmpfilename, info_dict)]
+
+        self._debug_cmd(cmd)
+
+        p = subprocess.Popen(cmd)
+        p.communicate()
+        return p.returncode
+
 
 class AxelFD(ExternalFD):
     AVAILABLE_OPT = '-V'
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index a03f7184d..b3b687a31 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2410,6 +2410,8 @@ def dfxp2srt(dfxp_data):
 
 def cli_option(params, command_option, param):
     param = params.get(param)
+    if param:
+        param = compat_str(param)
     return [command_option, param] if param is not None else []
 
 

From f0d3669437bb7f198ada9c0fead64d50a6e7a972 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 12 Aug 2016 18:05:49 +0100
Subject: [PATCH 5743/6219] [hgtv] Add new extractor(closes #3999)

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/hgtv.py       | 48 ++++++++++++++++++++++++++++++
 2 files changed, 49 insertions(+)
 create mode 100644 youtube_dl/extractor/hgtv.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index acf4e5d62..6420167f2 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -325,6 +325,7 @@ from .heise import HeiseIE
 from .hellporno import HellPornoIE
 from .helsinki import HelsinkiIE
 from .hentaistigma import HentaiStigmaIE
+from .hgtv import HGTVIE
 from .historicfilms import HistoricFilmsIE
 from .hitbox import HitboxIE, HitboxLiveIE
 from .hornbunny import HornBunnyIE
diff --git a/youtube_dl/extractor/hgtv.py b/youtube_dl/extractor/hgtv.py
new file mode 100644
index 000000000..c3f0733cf
--- /dev/null
+++ b/youtube_dl/extractor/hgtv.py
@@ -0,0 +1,48 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    js_to_json,
+    smuggle_url,
+)
+
+
+class HGTVIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?hgtv\.ca/[^/]+/video/(?P<id>[^/]+)/video.html'
+    _TEST = {
+        'url': 'http://www.hgtv.ca/homefree/video/overnight-success/video.html?v=738081859718&p=1&s=da#video',
+        'md5': '',
+        'info_dict': {
+            'id': 'aFH__I_5FBOX',
+            'ext': 'mp4',
+            'title': 'Overnight Success',
+            'description': 'After weeks of hard work, high stakes, breakdowns and pep talks, the final 2 contestants compete to win the ultimate dream.',
+            'uploader': 'SHWM-NEW',
+            'timestamp': 1470320034,
+            'upload_date': '20160804',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+        embed_vars = self._parse_json(self._search_regex(
+            r'(?s)embed_vars\s*=\s*({.*?});',
+            webpage, 'embed vars'), display_id, js_to_json)
+        return {
+            '_type': 'url_transparent',
+            'url': smuggle_url(
+                'http://link.theplatform.com/s/dtjsEC/%s?mbr=true&manifest=m3u' % embed_vars['pid'], {
+                    'force_smil_url': True
+                }),
+            'series': embed_vars.get('show'),
+            'season_number': int_or_none(embed_vars.get('season')),
+            'episode_number': int_or_none(embed_vars.get('episode')),
+            'ie_key': 'ThePlatform',
+        }

From 794e5dcd7e24784c05e042e7e0655c584347f5c3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 13 Aug 2016 14:09:35 +0700
Subject: [PATCH 5744/6219] [sunporno] Fix metadata extraction (Closes #10316)

---
 youtube_dl/extractor/sunporno.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/sunporno.py b/youtube_dl/extractor/sunporno.py
index e527aa971..4269f2a30 100644
--- a/youtube_dl/extractor/sunporno.py
+++ b/youtube_dl/extractor/sunporno.py
@@ -15,10 +15,10 @@ class SunPornoIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?sunporno\.com/videos/(?P<id>\d+)'
     _TEST = {
         'url': 'http://www.sunporno.com/videos/807778/',
-        'md5': '6457d3c165fd6de062b99ef6c2ff4c86',
+        'md5': '507887e29033502f29dba69affeebfc9',
         'info_dict': {
             'id': '807778',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'md5:0a400058e8105d39e35c35e7c5184164',
             'description': 'md5:a31241990e1bd3a64e72ae99afb325fb',
             'thumbnail': 're:^https?://.*\.jpg$',
@@ -40,7 +40,8 @@ class SunPornoIE(InfoExtractor):
             r'poster="([^"]+)"', webpage, 'thumbnail', fatal=False)
 
         duration = parse_duration(self._search_regex(
-            r'itemprop="duration">\s*(\d+:\d+)\s*<',
+            (r'itemprop="duration"[^>]*>\s*(\d+:\d+)\s*<',
+             r'>Duration:\s*<span[^>]+>\s*(\d+:\d+)\s*<'),
             webpage, 'duration', fatal=False))
 
         view_count = int_or_none(self._html_search_regex(
@@ -48,7 +49,7 @@ class SunPornoIE(InfoExtractor):
             webpage, 'view count', fatal=False))
         comment_count = int_or_none(self._html_search_regex(
             r'(\d+)</b> Comments?',
-            webpage, 'comment count', fatal=False))
+            webpage, 'comment count', fatal=False, default=None))
 
         formats = []
         quality = qualities(['mp4', 'flv'])

From b69b2ff7366cee97202eb333bf06329bfb2e974e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 13 Aug 2016 14:13:49 +0700
Subject: [PATCH 5745/6219] [sunporno] Add support for embed URLs

---
 youtube_dl/extractor/sunporno.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/sunporno.py b/youtube_dl/extractor/sunporno.py
index 4269f2a30..ef9be7926 100644
--- a/youtube_dl/extractor/sunporno.py
+++ b/youtube_dl/extractor/sunporno.py
@@ -12,8 +12,8 @@ from ..utils import (
 
 
 class SunPornoIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?sunporno\.com/videos/(?P<id>\d+)'
-    _TEST = {
+    _VALID_URL = r'https?://(?:(?:www\.)?sunporno\.com/videos|embeds\.sunporno\.com/embed)/(?P<id>\d+)'
+    _TESTS = [{
         'url': 'http://www.sunporno.com/videos/807778/',
         'md5': '507887e29033502f29dba69affeebfc9',
         'info_dict': {
@@ -25,12 +25,16 @@ class SunPornoIE(InfoExtractor):
             'duration': 302,
             'age_limit': 18,
         }
-    }
+    }, {
+        'url': 'http://embeds.sunporno.com/embed/807778',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        webpage = self._download_webpage(url, video_id)
+        webpage = self._download_webpage(
+            'http://www.sunporno.com/videos/%s' % video_id, video_id)
 
         title = self._html_search_regex(
             r'<title>([^<]+)</title>', webpage, 'title')

From bd6fb007de2323065bface4467539b509fbdb062 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 13 Aug 2016 14:22:47 +0700
Subject: [PATCH 5746/6219] [24video] Fix comment count extraction

---
 youtube_dl/extractor/twentyfourvideo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/twentyfourvideo.py b/youtube_dl/extractor/twentyfourvideo.py
index 4025edf02..8b808d6d6 100644
--- a/youtube_dl/extractor/twentyfourvideo.py
+++ b/youtube_dl/extractor/twentyfourvideo.py
@@ -64,7 +64,7 @@ class TwentyFourVideoIE(InfoExtractor):
             r'<span class="video-views">(\d+) просмотр',
             webpage, 'view count', fatal=False))
         comment_count = int_or_none(self._html_search_regex(
-            r'<div class="comments-title" id="comments-count">(\d+) комментари',
+            r'<a[^>]+href="#tab-comments"[^>]*>(\d+) комментари',
             webpage, 'comment count', fatal=False))
 
         # Sets some cookies

From 6a26c5f9d5d7b32648e116be4ce902802994654e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 13 Aug 2016 14:28:44 +0700
Subject: [PATCH 5747/6219] [muenchentv] Fix extraction (Closes #10313)

---
 youtube_dl/extractor/muenchentv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/muenchentv.py b/youtube_dl/extractor/muenchentv.py
index b4e8ad17e..d9f176136 100644
--- a/youtube_dl/extractor/muenchentv.py
+++ b/youtube_dl/extractor/muenchentv.py
@@ -36,7 +36,7 @@ class MuenchenTVIE(InfoExtractor):
         title = self._live_title(self._og_search_title(webpage))
 
         data_js = self._search_regex(
-            r'(?s)\nplaylist:\s*(\[.*?}\]),related:',
+            r'(?s)\nplaylist:\s*(\[.*?}\]),',
             webpage, 'playlist configuration')
         data_json = js_to_json(data_js)
         data = json.loads(data_json)[0]

From c366f8d30a177d2d44130c9d077b15a4c960c003 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 13 Aug 2016 14:47:51 +0700
Subject: [PATCH 5748/6219] [24video] Add support for me and xxx TLDs

---
 youtube_dl/extractor/twentyfourvideo.py | 48 ++++++++++++-------------
 1 file changed, 24 insertions(+), 24 deletions(-)

diff --git a/youtube_dl/extractor/twentyfourvideo.py b/youtube_dl/extractor/twentyfourvideo.py
index 8b808d6d6..af92b713b 100644
--- a/youtube_dl/extractor/twentyfourvideo.py
+++ b/youtube_dl/extractor/twentyfourvideo.py
@@ -12,32 +12,32 @@ from ..utils import (
 
 class TwentyFourVideoIE(InfoExtractor):
     IE_NAME = '24video'
-    _VALID_URL = r'https?://(?:www\.)?24video\.net/(?:video/(?:view|xml)/|player/new24_play\.swf\?id=)(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?24video\.(?:net|me|xxx)/(?:video/(?:view|xml)/|player/new24_play\.swf\?id=)(?P<id>\d+)'
 
-    _TESTS = [
-        {
-            'url': 'http://www.24video.net/video/view/1044982',
-            'md5': 'e09fc0901d9eaeedac872f154931deeb',
-            'info_dict': {
-                'id': '1044982',
-                'ext': 'mp4',
-                'title': 'Эротика каменного века',
-                'description': 'Как смотрели порно в каменном веке.',
-                'thumbnail': 're:^https?://.*\.jpg$',
-                'uploader': 'SUPERTELO',
-                'duration': 31,
-                'timestamp': 1275937857,
-                'upload_date': '20100607',
-                'age_limit': 18,
-                'like_count': int,
-                'dislike_count': int,
-            },
+    _TESTS = [{
+        'url': 'http://www.24video.net/video/view/1044982',
+        'md5': 'e09fc0901d9eaeedac872f154931deeb',
+        'info_dict': {
+            'id': '1044982',
+            'ext': 'mp4',
+            'title': 'Эротика каменного века',
+            'description': 'Как смотрели порно в каменном веке.',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'uploader': 'SUPERTELO',
+            'duration': 31,
+            'timestamp': 1275937857,
+            'upload_date': '20100607',
+            'age_limit': 18,
+            'like_count': int,
+            'dislike_count': int,
         },
-        {
-            'url': 'http://www.24video.net/player/new24_play.swf?id=1044982',
-            'only_matching': True,
-        }
-    ]
+    }, {
+        'url': 'http://www.24video.net/player/new24_play.swf?id=1044982',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.24video.me/video/view/1044982',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From f50365e91cfccf33d0b5696c7f989944bcf748e7 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 13 Aug 2016 09:08:57 +0100
Subject: [PATCH 5749/6219] [pbs] add test for videos with undocumented http
 formats and remove unused import

---
 youtube_dl/extractor/pbs.py | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 6e2ef0fba..335e44bdc 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -4,7 +4,6 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_HTTPError
 from ..utils import (
     ExtractorError,
     determine_ext,
@@ -334,6 +333,16 @@ class PBSIE(InfoExtractor):
                 'formats': 'mincount:8',
             },
         },
+        {
+            # has undocumented http formats(4500k and 6500k)
+            'url': 'http://www.pbs.org/video/2365815229/',
+            'md5': '94635cd06b7133688e23f4b94e6637a5',
+            'info_dict': {
+                'id': '2365815229',
+                'ext': 'mp4',
+                'title': 'FRONTLINE - Mosquito Hunter',
+            },
+        },
         {
             'url': 'http://player.pbs.org/widget/partnerplayer/2365297708/?start=0&end=0&chapterbar=false&endscreen=false&topbar=true',
             'only_matching': True,
@@ -501,14 +510,18 @@ class PBSIE(InfoExtractor):
         if http_url:
             for m3u8_format in m3u8_formats:
                 bitrate = self._search_regex(r'(\d+)k', m3u8_format['url'], 'bitrate', default=None)
-                # extract only the formats that we know that they will be available as http format.
+                # lower qualities(150k and 192k) are not available as http formats
+                # https://github.com/rg3/youtube-dl/commit/cbc032c8b70a038a69259378c92b4ba97b42d491#commitcomment-17313656
+                # we will try to extract any http format higher than than the lowest quality documented in
                 # https://projects.pbs.org/confluence/display/coveapi/COVE+Video+Specifications
+                # as there also undocumented http formats formats(4500k and 6500k)
+                # http://www.pbs.org/video/2365815229/
                 if not bitrate or int(bitrate) < 400:
                     continue
                 f_url = re.sub(r'\d+k|baseline', bitrate + 'k', http_url)
                 # This may produce invalid links sometimes (e.g.
                 # http://www.pbs.org/wgbh/frontline/film/suicide-plan)
-                if not self._is_valid_url(f_url, display_id, 'http-%s video' % bitrate):
+                if not self._is_valid_url(f_url, display_id, 'http-%sk video' % bitrate):
                     continue
                 f = m3u8_format.copy()
                 f.update({

From e581224843db95574ca65965f5c5a594a7ffd370 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 13 Aug 2016 16:32:07 +0800
Subject: [PATCH 5750/6219] [tapely] Remove extractor. It's shut down

Closes #10323
---
 ChangeLog                          |   5 ++
 youtube_dl/extractor/extractors.py |   1 -
 youtube_dl/extractor/tapely.py     | 109 -----------------------------
 3 files changed, 5 insertions(+), 110 deletions(-)
 delete mode 100644 youtube_dl/extractor/tapely.py

diff --git a/ChangeLog b/ChangeLog
index 376d96d12..b1ce63d75 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,8 @@
+version <unreleased>
+
+Extractors
+- [tapely] Remove extractor (#10323)
+
 version 2016.08.12
 
 Core
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 6420167f2..104d8e37e 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -812,7 +812,6 @@ from .tagesschau import (
     TagesschauPlayerIE,
     TagesschauIE,
 )
-from .tapely import TapelyIE
 from .tass import TassIE
 from .tdslifeway import TDSLifewayIE
 from .teachertube import (
diff --git a/youtube_dl/extractor/tapely.py b/youtube_dl/extractor/tapely.py
deleted file mode 100644
index ed560bd24..000000000
--- a/youtube_dl/extractor/tapely.py
+++ /dev/null
@@ -1,109 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-import re
-
-from .common import InfoExtractor
-from ..utils import (
-    clean_html,
-    ExtractorError,
-    float_or_none,
-    parse_iso8601,
-    sanitized_Request,
-)
-
-
-class TapelyIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?(?:tape\.ly|tapely\.com)/(?P<id>[A-Za-z0-9\-_]+)(?:/(?P<songnr>\d+))?'
-    _API_URL = 'http://tape.ly/showtape?id={0:}'
-    _S3_SONG_URL = 'http://mytape.s3.amazonaws.com/{0:}'
-    _SOUNDCLOUD_SONG_URL = 'http://api.soundcloud.com{0:}'
-    _TESTS = [
-        {
-            'url': 'http://tape.ly/my-grief-as-told-by-water',
-            'info_dict': {
-                'id': 23952,
-                'title': 'my grief as told by water',
-                'thumbnail': 're:^https?://.*\.png$',
-                'uploader_id': 16484,
-                'timestamp': 1411848286,
-                'description': 'For Robin and Ponkers, whom the tides of life have taken out to sea.',
-            },
-            'playlist_count': 13,
-        },
-        {
-            'url': 'http://tape.ly/my-grief-as-told-by-water/1',
-            'md5': '79031f459fdec6530663b854cbc5715c',
-            'info_dict': {
-                'id': 258464,
-                'title': 'Dreaming Awake  (My Brightest Diamond)',
-                'ext': 'm4a',
-            },
-        },
-        {
-            'url': 'https://tapely.com/my-grief-as-told-by-water',
-            'only_matching': True,
-        },
-    ]
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        display_id = mobj.group('id')
-
-        playlist_url = self._API_URL.format(display_id)
-        request = sanitized_Request(playlist_url)
-        request.add_header('X-Requested-With', 'XMLHttpRequest')
-        request.add_header('Accept', 'application/json')
-        request.add_header('Referer', url)
-
-        playlist = self._download_json(request, display_id)
-
-        tape = playlist['tape']
-
-        entries = []
-        for s in tape['songs']:
-            song = s['song']
-            entry = {
-                'id': song['id'],
-                'duration': float_or_none(song.get('songduration'), 1000),
-                'title': song['title'],
-            }
-            if song['source'] == 'S3':
-                entry.update({
-                    'url': self._S3_SONG_URL.format(song['filename']),
-                })
-                entries.append(entry)
-            elif song['source'] == 'YT':
-                self.to_screen('YouTube video detected')
-                yt_id = song['filename'].replace('/youtube/', '')
-                entry.update(self.url_result(yt_id, 'Youtube', video_id=yt_id))
-                entries.append(entry)
-            elif song['source'] == 'SC':
-                self.to_screen('SoundCloud song detected')
-                sc_url = self._SOUNDCLOUD_SONG_URL.format(song['filename'])
-                entry.update(self.url_result(sc_url, 'Soundcloud'))
-                entries.append(entry)
-            else:
-                self.report_warning('Unknown song source: %s' % song['source'])
-
-        if mobj.group('songnr'):
-            songnr = int(mobj.group('songnr')) - 1
-            try:
-                return entries[songnr]
-            except IndexError:
-                raise ExtractorError(
-                    'No song with index: %s' % mobj.group('songnr'),
-                    expected=True)
-
-        return {
-            '_type': 'playlist',
-            'id': tape['id'],
-            'display_id': display_id,
-            'title': tape['name'],
-            'entries': entries,
-            'thumbnail': tape.get('image_url'),
-            'description': clean_html(tape.get('subtext')),
-            'like_count': tape.get('likescount'),
-            'uploader_id': tape.get('user_id'),
-            'timestamp': parse_iso8601(tape.get('published_at')),
-        }

From cb55908e51b80d9a51664ec76dcfe05d739dadf9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 13 Aug 2016 15:47:20 +0700
Subject: [PATCH 5751/6219] [vbox7] Fix extraction (Closes #10309)

---
 youtube_dl/extractor/vbox7.py | 57 ++++++++++++++++++-----------------
 1 file changed, 30 insertions(+), 27 deletions(-)

diff --git a/youtube_dl/extractor/vbox7.py b/youtube_dl/extractor/vbox7.py
index dff1bb702..326440758 100644
--- a/youtube_dl/extractor/vbox7.py
+++ b/youtube_dl/extractor/vbox7.py
@@ -12,7 +12,15 @@ from ..utils import (
 
 class Vbox7IE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?vbox7\.com/play:(?P<id>[^/]+)'
-    _TEST = {
+    _TESTS = [{
+        'url': 'http://vbox7.com/play:0946fff23c',
+        'md5': 'a60f9ab3a3a2f013ef9a967d5f7be5bf',
+        'info_dict': {
+            'id': '0946fff23c',
+            'ext': 'mp4',
+            'title': 'Борисов: Притеснен съм за бъдещето на България',
+        },
+    }, {
         'url': 'http://vbox7.com/play:249bb972c2',
         'md5': '99f65c0c9ef9b682b97313e052734c3f',
         'info_dict': {
@@ -20,43 +28,38 @@ class Vbox7IE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Смях! Чудо - чист за секунди - Скрита камера',
         },
-    }
+        'skip': 'georestricted',
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        # need to get the page 3 times for the correct jsSecretToken cookie
-        # which is necessary for the correct title
-        def get_session_id():
-            redirect_page = self._download_webpage(url, video_id)
-            session_id_url = self._search_regex(
-                r'var\s*url\s*=\s*\'([^\']+)\';', redirect_page,
-                'session id url')
-            self._download_webpage(
-                compat_urlparse.urljoin(url, session_id_url), video_id,
-                'Getting session id')
+        webpage = self._download_webpage(url, video_id)
 
-        get_session_id()
-        get_session_id()
+        title = self._html_search_regex(
+            r'<title>(.*)</title>', webpage, 'title').split('/')[0].strip()
 
-        webpage = self._download_webpage(url, video_id,
-                                         'Downloading redirect page')
+        video_url = self._search_regex(
+            r'src\s*:\s*(["\'])(?P<url>.+?.mp4.*?)\1',
+            webpage, 'video url', default=None, group='url')
 
-        title = self._html_search_regex(r'<title>(.*)</title>',
-                                        webpage, 'title').split('/')[0].strip()
+        thumbnail_url = self._og_search_thumbnail(webpage)
 
-        info_url = 'http://vbox7.com/play/magare.do'
-        data = urlencode_postdata({'as3': '1', 'vid': video_id})
-        info_request = sanitized_Request(info_url, data)
-        info_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
-        info_response = self._download_webpage(info_request, video_id, 'Downloading info webpage')
-        if info_response is None:
-            raise ExtractorError('Unable to extract the media url')
-        (final_url, thumbnail_url) = map(lambda x: x.split('=')[1], info_response.split('&'))
+        if not video_url:
+            info_response = self._download_webpage(
+                'http://vbox7.com/play/magare.do', video_id,
+                'Downloading info webpage',
+                data=urlencode_postdata({'as3': '1', 'vid': video_id}),
+                headers={'Content-Type': 'application/x-www-form-urlencoded'})
+            final_url, thumbnail_url = map(
+                lambda x: x.split('=')[1], info_response.split('&'))
+
+        if '/na.mp4' in video_url:
+            self.raise_geo_restricted()
 
         return {
             'id': video_id,
-            'url': final_url,
+            'url': self._proto_relative_url(video_url, 'http:'),
             'title': title,
             'thumbnail': thumbnail_url,
         }

From 5f2c2b7936eb092e482309a5b9aa036028dbab2c Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 13 Aug 2016 09:53:46 +0100
Subject: [PATCH 5752/6219] [test_utils] add test for option with not str value

---
 test/test_utils.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/test/test_utils.py b/test/test_utils.py
index 724346886..74fcf91c0 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -968,6 +968,7 @@ The first line
         self.assertEqual(cli_option({'proxy': '127.0.0.1:3128'}, '--proxy', 'proxy'), ['--proxy', '127.0.0.1:3128'])
         self.assertEqual(cli_option({'proxy': None}, '--proxy', 'proxy'), [])
         self.assertEqual(cli_option({}, '--proxy', 'proxy'), [])
+        self.assertEqual(cli_option({'retries': 10}, '--retries', 'retries'), ['--retries', '10'])
 
     def test_cli_valueless_option(self):
         self.assertEqual(cli_valueless_option(

From acfccacad5555c21d649729c5e2cb237a70f46e6 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 13 Aug 2016 10:26:02 +0100
Subject: [PATCH 5753/6219] [downloader/external:curl] Clarify why CurlFD
 should not capture stderr

---
 youtube_dl/downloader/external.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index f0c30007f..cf4556221 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -114,6 +114,7 @@ class CurlFD(ExternalFD):
 
         self._debug_cmd(cmd)
 
+        # curl writes the progress to stderr so don't capture it.
         p = subprocess.Popen(cmd)
         p.communicate()
         return p.returncode

From e97c55ee6aaf5170f86bc8146a20cef56e337a3d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 13 Aug 2016 16:29:05 +0700
Subject: [PATCH 5754/6219] [expotv] Improve extraction and update test

---
 youtube_dl/extractor/expotv.py | 23 ++++++++++-------------
 1 file changed, 10 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/expotv.py b/youtube_dl/extractor/expotv.py
index 1585a03bb..971c918a4 100644
--- a/youtube_dl/extractor/expotv.py
+++ b/youtube_dl/extractor/expotv.py
@@ -1,7 +1,5 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..utils import (
     int_or_none,
@@ -12,23 +10,22 @@ from ..utils import (
 class ExpoTVIE(InfoExtractor):
     _VALID_URL = r'https?://www\.expotv\.com/videos/[^?#]*/(?P<id>[0-9]+)($|[?#])'
     _TEST = {
-        'url': 'http://www.expotv.com/videos/reviews/1/24/LinneCardscom/17561',
-        'md5': '2985e6d7a392b2f7a05e0ca350fe41d0',
+        'url': 'http://www.expotv.com/videos/reviews/3/40/NYX-Butter-lipstick/667916',
+        'md5': 'fe1d728c3a813ff78f595bc8b7a707a8',
         'info_dict': {
-            'id': '17561',
+            'id': '667916',
             'ext': 'mp4',
-            'upload_date': '20060212',
-            'title': 'My Favorite Online Scrapbook Store',
-            'view_count': int,
-            'description': 'You\'ll find most everything you need at this virtual store front.',
-            'uploader': 'Anna T.',
+            'title': 'NYX Butter Lipstick Little Susie',
+            'description': 'Goes on like butter, but looks better!',
             'thumbnail': 're:^https?://.*\.jpg$',
+            'uploader': 'Stephanie S.',
+            'upload_date': '20150520',
+            'view_count': int,
         }
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id)
         player_key = self._search_regex(
@@ -66,7 +63,7 @@ class ExpoTVIE(InfoExtractor):
             fatal=False)
         upload_date = unified_strdate(self._search_regex(
             r'<h5>Reviewed on ([0-9/.]+)</h5>', webpage, 'upload date',
-            fatal=False))
+            fatal=False), day_first=False)
 
         return {
             'id': video_id,

From 52aa7e7476415ec632053f85f9db0919f7bf75c3 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 13 Aug 2016 17:36:14 +0800
Subject: [PATCH 5755/6219] [test_verbose_output] Fix tests under Python 3

---
 test/test_verbose_output.py | 32 ++++++++++++++++----------------
 1 file changed, 16 insertions(+), 16 deletions(-)

diff --git a/test/test_verbose_output.py b/test/test_verbose_output.py
index 4c77df242..96a66f7a0 100644
--- a/test/test_verbose_output.py
+++ b/test/test_verbose_output.py
@@ -22,10 +22,10 @@ class TestVerboseOutput(unittest.TestCase):
                 '--password', 'secret',
             ], cwd=rootDir, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
         sout, serr = outp.communicate()
-        self.assertTrue('--username' in serr)
-        self.assertTrue('johnsmith' not in serr)
-        self.assertTrue('--password' in serr)
-        self.assertTrue('secret' not in serr)
+        self.assertTrue(b'--username' in serr)
+        self.assertTrue(b'johnsmith' not in serr)
+        self.assertTrue(b'--password' in serr)
+        self.assertTrue(b'secret' not in serr)
 
     def test_private_info_shortarg(self):
         outp = subprocess.Popen(
@@ -35,10 +35,10 @@ class TestVerboseOutput(unittest.TestCase):
                 '-p', 'secret',
             ], cwd=rootDir, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
         sout, serr = outp.communicate()
-        self.assertTrue('-u' in serr)
-        self.assertTrue('johnsmith' not in serr)
-        self.assertTrue('-p' in serr)
-        self.assertTrue('secret' not in serr)
+        self.assertTrue(b'-u' in serr)
+        self.assertTrue(b'johnsmith' not in serr)
+        self.assertTrue(b'-p' in serr)
+        self.assertTrue(b'secret' not in serr)
 
     def test_private_info_eq(self):
         outp = subprocess.Popen(
@@ -48,10 +48,10 @@ class TestVerboseOutput(unittest.TestCase):
                 '--password=secret',
             ], cwd=rootDir, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
         sout, serr = outp.communicate()
-        self.assertTrue('--username' in serr)
-        self.assertTrue('johnsmith' not in serr)
-        self.assertTrue('--password' in serr)
-        self.assertTrue('secret' not in serr)
+        self.assertTrue(b'--username' in serr)
+        self.assertTrue(b'johnsmith' not in serr)
+        self.assertTrue(b'--password' in serr)
+        self.assertTrue(b'secret' not in serr)
 
     def test_private_info_shortarg_eq(self):
         outp = subprocess.Popen(
@@ -61,10 +61,10 @@ class TestVerboseOutput(unittest.TestCase):
                 '-p=secret',
             ], cwd=rootDir, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
         sout, serr = outp.communicate()
-        self.assertTrue('-u' in serr)
-        self.assertTrue('johnsmith' not in serr)
-        self.assertTrue('-p' in serr)
-        self.assertTrue('secret' not in serr)
+        self.assertTrue(b'-u' in serr)
+        self.assertTrue(b'johnsmith' not in serr)
+        self.assertTrue(b'-p' in serr)
+        self.assertTrue(b'secret' not in serr)
 
 if __name__ == '__main__':
     unittest.main()

From cd29eaab955b930fc7ee595553d6351ad643569d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 13 Aug 2016 16:45:34 +0700
Subject: [PATCH 5756/6219] [vbox7] Remove unused imports

---
 youtube_dl/extractor/vbox7.py | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/vbox7.py b/youtube_dl/extractor/vbox7.py
index 326440758..fa7899e6d 100644
--- a/youtube_dl/extractor/vbox7.py
+++ b/youtube_dl/extractor/vbox7.py
@@ -2,12 +2,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import compat_urlparse
-from ..utils import (
-    ExtractorError,
-    sanitized_Request,
-    urlencode_postdata,
-)
+from ..utils import urlencode_postdata
 
 
 class Vbox7IE(InfoExtractor):

From c2a453b46177787d5cc17e09cedb3eca215ab159 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 13 Aug 2016 16:46:07 +0700
Subject: [PATCH 5757/6219] [imgur] Fix width and height extraction (Closes
 #10325)

---
 youtube_dl/extractor/imgur.py | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/imgur.py b/youtube_dl/extractor/imgur.py
index 85e9344aa..d23489dcf 100644
--- a/youtube_dl/extractor/imgur.py
+++ b/youtube_dl/extractor/imgur.py
@@ -50,12 +50,10 @@ class ImgurIE(InfoExtractor):
         webpage = self._download_webpage(
             compat_urlparse.urljoin(url, video_id), video_id)
 
-        width = int_or_none(self._search_regex(
-            r'<param name="width" value="([0-9]+)"',
-            webpage, 'width', fatal=False))
-        height = int_or_none(self._search_regex(
-            r'<param name="height" value="([0-9]+)"',
-            webpage, 'height', fatal=False))
+        width = int_or_none(self._og_search_property(
+            'video:width', webpage, default=None))
+        height = int_or_none(self._og_search_property(
+            'video:height', webpage, default=None))
 
         video_elements = self._search_regex(
             r'(?s)<div class="video-elements">(.*?)</div>',

From db535435b30540029b292e7217fb443bcc670aab Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 13 Aug 2016 18:02:11 +0800
Subject: [PATCH 5758/6219] [bigflix] Remove an invalid test

There's no video anymore
---
 youtube_dl/extractor/bigflix.py | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/youtube_dl/extractor/bigflix.py b/youtube_dl/extractor/bigflix.py
index b19f35b5d..b4ce767af 100644
--- a/youtube_dl/extractor/bigflix.py
+++ b/youtube_dl/extractor/bigflix.py
@@ -11,15 +11,6 @@ from ..compat import compat_urllib_parse_unquote
 class BigflixIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?bigflix\.com/.+/(?P<id>[0-9]+)'
     _TESTS = [{
-        'url': 'http://www.bigflix.com/Hindi-movies/Action-movies/Singham-Returns/16537',
-        'md5': 'dc1b4aebb46e3a7077ecc0d9f43f61e3',
-        'info_dict': {
-            'id': '16537',
-            'ext': 'mp4',
-            'title': 'Singham Returns',
-            'description': 'md5:3d2ba5815f14911d5cc6a501ae0cf65d',
-        }
-    }, {
         # 2 formats
         'url': 'http://www.bigflix.com/Tamil-movies/Drama-movies/Madarasapatinam/16070',
         'info_dict': {

From 77afa008dd14efd930f8504609815a8ad2fedc7f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 13 Aug 2016 19:55:09 +0700
Subject: [PATCH 5759/6219] [4tube] Fix metadata extraction (Closes #10321)

---
 youtube_dl/extractor/fourtube.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/fourtube.py b/youtube_dl/extractor/fourtube.py
index fc4a5a0fb..9776c8422 100644
--- a/youtube_dl/extractor/fourtube.py
+++ b/youtube_dl/extractor/fourtube.py
@@ -43,14 +43,14 @@ class FourTubeIE(InfoExtractor):
             'uploadDate', webpage))
         thumbnail = self._html_search_meta('thumbnailUrl', webpage)
         uploader_id = self._html_search_regex(
-            r'<a class="img-avatar" href="[^"]+/channels/([^/"]+)" title="Go to [^"]+ page">',
+            r'<a class="item-to-subscribe" href="[^"]+/channels/([^/"]+)" title="Go to [^"]+ page">',
             webpage, 'uploader id', fatal=False)
         uploader = self._html_search_regex(
-            r'<a class="img-avatar" href="[^"]+/channels/[^/"]+" title="Go to ([^"]+) page">',
+            r'<a class="item-to-subscribe" href="[^"]+/channels/[^/"]+" title="Go to ([^"]+) page">',
             webpage, 'uploader', fatal=False)
 
         categories_html = self._search_regex(
-            r'(?s)><i class="icon icon-tag"></i>\s*Categories / Tags\s*.*?<ul class="list">(.*?)</ul>',
+            r'(?s)><i class="icon icon-tag"></i>\s*Categories / Tags\s*.*?<ul class="[^"]*?list[^"]*?">(.*?)</ul>',
             webpage, 'categories', fatal=False)
         categories = None
         if categories_html:
@@ -59,10 +59,10 @@ class FourTubeIE(InfoExtractor):
                     r'(?s)<li><a.*?>(.*?)</a>', categories_html)]
 
         view_count = str_to_int(self._search_regex(
-            r'<meta itemprop="interactionCount" content="UserPlays:([0-9,]+)">',
+            r'<meta[^>]+itemprop="interactionCount"[^>]+content="UserPlays:([0-9,]+)">',
             webpage, 'view count', fatal=False))
         like_count = str_to_int(self._search_regex(
-            r'<meta itemprop="interactionCount" content="UserLikes:([0-9,]+)">',
+            r'<meta[^>]+itemprop="interactionCount"[^>]+content="UserLikes:([0-9,]+)">',
             webpage, 'like count', fatal=False))
         duration = parse_duration(self._html_search_meta('duration', webpage))
 

From 647a7bf5e8355b34ed030827f53ea1e87ffc9131 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 13 Aug 2016 20:49:16 +0700
Subject: [PATCH 5760/6219] [pornotube] Fix extraction (Closes #10322)

---
 youtube_dl/extractor/pornotube.py | 83 ++++++++++++++-----------------
 1 file changed, 38 insertions(+), 45 deletions(-)

diff --git a/youtube_dl/extractor/pornotube.py b/youtube_dl/extractor/pornotube.py
index 5398e708b..63816c358 100644
--- a/youtube_dl/extractor/pornotube.py
+++ b/youtube_dl/extractor/pornotube.py
@@ -3,10 +3,7 @@ from __future__ import unicode_literals
 import json
 
 from .common import InfoExtractor
-from ..utils import (
-    int_or_none,
-    sanitized_Request,
-)
+from ..utils import int_or_none
 
 
 class PornotubeIE(InfoExtractor):
@@ -31,59 +28,55 @@ class PornotubeIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        # Fetch origin token
-        js_config = self._download_webpage(
-            'http://www.pornotube.com/assets/src/app/config.js', video_id,
-            note='Download JS config')
-        originAuthenticationSpaceKey = self._search_regex(
-            r"constant\('originAuthenticationSpaceKey',\s*'([^']+)'",
-            js_config, 'originAuthenticationSpaceKey')
+        token = self._download_json(
+            'https://api.aebn.net/auth/v2/origins/authenticate',
+            video_id, note='Downloading token',
+            data=json.dumps({'credentials': 'Clip Application'}).encode('utf-8'),
+            headers={
+                'Content-Type': 'application/json',
+                'Origin': 'http://www.pornotube.com',
+            })['tokenKey']
 
-        # Fetch actual token
-        token_req_data = {
-            'authenticationSpaceKey': originAuthenticationSpaceKey,
-            'credentials': 'Clip Application',
-        }
-        token_req = sanitized_Request(
-            'https://api.aebn.net/auth/v1/token/primal',
-            data=json.dumps(token_req_data).encode('utf-8'))
-        token_req.add_header('Content-Type', 'application/json')
-        token_req.add_header('Origin', 'http://www.pornotube.com')
-        token_answer = self._download_json(
-            token_req, video_id, note='Requesting primal token')
-        token = token_answer['tokenKey']
+        video_url = self._download_json(
+            'https://api.aebn.net/delivery/v1/clips/%s/MP4' % video_id,
+            video_id, note='Downloading delivery information',
+            headers={'Authorization': token})['mediaUrl']
 
-        # Get video URL
-        delivery_req = sanitized_Request(
-            'https://api.aebn.net/delivery/v1/clips/%s/MP4' % video_id)
-        delivery_req.add_header('Authorization', token)
-        delivery_info = self._download_json(
-            delivery_req, video_id, note='Downloading delivery information')
-        video_url = delivery_info['mediaUrl']
+        FIELDS = (
+            'title', 'description', 'startSecond', 'endSecond', 'publishDate',
+            'studios{name}', 'categories{name}', 'movieId', 'primaryImageNumber'
+        )
 
-        # Get additional info (title etc.)
-        info_req = sanitized_Request(
-            'https://api.aebn.net/content/v1/clips/%s?expand='
-            'title,description,primaryImageNumber,startSecond,endSecond,'
-            'movie.title,movie.MovieId,movie.boxCoverFront,movie.stars,'
-            'movie.studios,stars.name,studios.name,categories.name,'
-            'clipActive,movieActive,publishDate,orientations' % video_id)
-        info_req.add_header('Authorization', token)
         info = self._download_json(
-            info_req, video_id, note='Downloading metadata')
+            'https://api.aebn.net/content/v2/clips/%s?fields=%s'
+            % (video_id, ','.join(FIELDS)), video_id,
+            note='Downloading metadata',
+            headers={'Authorization': token})
+
+        if isinstance(info, list):
+            info = info[0]
+
+        title = info['title']
 
         timestamp = int_or_none(info.get('publishDate'), scale=1000)
         uploader = info.get('studios', [{}])[0].get('name')
-        movie_id = info['movie']['movieId']
-        thumbnail = 'http://pic.aebn.net/dis/t/%s/%s_%08d.jpg' % (
-            movie_id, movie_id, info['primaryImageNumber'])
-        categories = [c['name'] for c in info.get('categories')]
+        movie_id = info.get('movieId')
+        primary_image_number = info.get('primaryImageNumber')
+        thumbnail = None
+        if movie_id and primary_image_number:
+            thumbnail = 'http://pic.aebn.net/dis/t/%s/%s_%08d.jpg' % (
+                movie_id, movie_id, primary_image_number)
+        start = int_or_none(info.get('startSecond'))
+        end = int_or_none(info.get('endSecond'))
+        duration = end - start if start and end else None
+        categories = [c['name'] for c in info.get('categories', []) if c.get('name')]
 
         return {
             'id': video_id,
             'url': video_url,
-            'title': info['title'],
+            'title': title,
             'description': info.get('description'),
+            'duration': duration,
             'timestamp': timestamp,
             'uploader': uploader,
             'thumbnail': thumbnail,

From 82997dad571988aae59d85db4355f5f1695efcbe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 13 Aug 2016 21:00:34 +0700
Subject: [PATCH 5761/6219] [franceculture] Fix extraction (Closes #10324)

---
 youtube_dl/extractor/extractors.py    |  5 +-
 youtube_dl/extractor/franceculture.py | 98 +++++++--------------------
 2 files changed, 26 insertions(+), 77 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 104d8e37e..82d4ed153 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -272,10 +272,7 @@ from .fox import FOXIE
 from .foxgay import FoxgayIE
 from .foxnews import FoxNewsIE
 from .foxsports import FoxSportsIE
-from .franceculture import (
-    FranceCultureIE,
-    FranceCultureEmissionIE,
-)
+from .franceculture import FranceCultureIE
 from .franceinter import FranceInterIE
 from .francetv import (
     PluzzIE,
diff --git a/youtube_dl/extractor/franceculture.py b/youtube_dl/extractor/franceculture.py
index e2ca96283..186da0d3b 100644
--- a/youtube_dl/extractor/franceculture.py
+++ b/youtube_dl/extractor/franceculture.py
@@ -2,104 +2,56 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urlparse,
-)
 from ..utils import (
     determine_ext,
-    int_or_none,
-    ExtractorError,
+    unified_strdate,
 )
 
 
 class FranceCultureIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?franceculture\.fr/player/reecouter\?play=(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?franceculture\.fr/emissions/(?:[^/]+/)*(?P<id>[^/?#&]+)'
     _TEST = {
-        'url': 'http://www.franceculture.fr/player/reecouter?play=4795174',
+        'url': 'http://www.franceculture.fr/emissions/carnet-nomade/rendez-vous-au-pays-des-geeks',
         'info_dict': {
-            'id': '4795174',
+            'id': 'rendez-vous-au-pays-des-geeks',
+            'display_id': 'rendez-vous-au-pays-des-geeks',
             'ext': 'mp3',
             'title': 'Rendez-vous au pays des geeks',
-            'alt_title': 'Carnet nomade | 13-14',
-            'vcodec': 'none',
+            'thumbnail': 're:^https?://.*\\.jpg$',
             'upload_date': '20140301',
-            'thumbnail': r're:^http://static\.franceculture\.fr/.*/images/player/Carnet-nomade\.jpg$',
-            'description': 'startswith:Avec :Jean-Baptiste Péretié pour son documentaire sur Arte "La revanche',
-            'timestamp': 1393700400,
+            'vcodec': 'none',
         }
     }
 
-    def _extract_from_player(self, url, video_id):
-        webpage = self._download_webpage(url, video_id)
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
 
-        video_path = self._search_regex(
-            r'<a id="player".*?href="([^"]+)"', webpage, 'video path')
-        video_url = compat_urlparse.urljoin(url, video_path)
-        timestamp = int_or_none(self._search_regex(
-            r'<a id="player".*?data-date="([0-9]+)"',
+        webpage = self._download_webpage(url, display_id)
+
+        video_url = self._search_regex(
+            r'(?s)<div[^>]+class="[^"]*?title-zone-diffusion[^"]*?"[^>]*>.*?<a[^>]+href="([^"]+)"',
+            webpage, 'video path')
+
+        title = self._og_search_title(webpage)
+
+        upload_date = unified_strdate(self._search_regex(
+            '(?s)<div[^>]+class="date"[^>]*>.*?<span[^>]+class="inner"[^>]*>([^<]+)<',
             webpage, 'upload date', fatal=False))
         thumbnail = self._search_regex(
-            r'<a id="player".*?>\s+<img src="([^"]+)"',
+            r'(?s)<figure[^>]+itemtype="https://schema.org/ImageObject"[^>]*>.*?<img[^>]+data-pagespeed-(?:lazy|high-res)-src="([^"]+)"',
             webpage, 'thumbnail', fatal=False)
-
-        display_id = self._search_regex(
-            r'<span class="path-diffusion">emission-(.*?)</span>', webpage, 'display_id')
-
-        title = self._html_search_regex(
-            r'<span class="title-diffusion">(.*?)</span>', webpage, 'title')
-        alt_title = self._html_search_regex(
-            r'<span class="title">(.*?)</span>',
-            webpage, 'alt_title', fatal=False)
-        description = self._html_search_regex(
-            r'<span class="description">(.*?)</span>',
-            webpage, 'description', fatal=False)
-
         uploader = self._html_search_regex(
             r'(?s)<div id="emission".*?<span class="author">(.*?)</span>',
             webpage, 'uploader', default=None)
         vcodec = 'none' if determine_ext(video_url.lower()) == 'mp3' else None
 
         return {
-            'id': video_id,
+            'id': display_id,
+            'display_id': display_id,
             'url': video_url,
+            'title': title,
+            'thumbnail': thumbnail,
             'vcodec': vcodec,
             'uploader': uploader,
-            'timestamp': timestamp,
-            'title': title,
-            'alt_title': alt_title,
-            'thumbnail': thumbnail,
-            'description': description,
-            'display_id': display_id,
+            'upload_date': upload_date,
         }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        return self._extract_from_player(url, video_id)
-
-
-class FranceCultureEmissionIE(FranceCultureIE):
-    _VALID_URL = r'https?://(?:www\.)?franceculture\.fr/emission-(?P<id>[^?#]+)'
-    _TEST = {
-        'url': 'http://www.franceculture.fr/emission-les-carnets-de-la-creation-jean-gabriel-periot-cineaste-2015-10-13',
-        'info_dict': {
-            'title': 'Jean-Gabriel Périot, cinéaste',
-            'alt_title': 'Les Carnets de la création',
-            'id': '5093239',
-            'display_id': 'les-carnets-de-la-creation-jean-gabriel-periot-cineaste-2015-10-13',
-            'ext': 'mp3',
-            'timestamp': 1444762500,
-            'upload_date': '20151013',
-            'description': 'startswith:Aujourd\'hui dans "Les carnets de la création", le cinéaste',
-        },
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-        video_path = self._html_search_regex(
-            r'<a class="rf-player-open".*?href="([^"]+)"', webpage, 'video path', 'no_path_player')
-        if video_path == 'no_path_player':
-            raise ExtractorError('no player : no sound in this page.', expected=True)
-        new_id = self._search_regex('play=(?P<id>[0-9]+)', video_path, 'new_id', group='id')
-        video_url = compat_urlparse.urljoin(url, video_path)
-        return self._extract_from_player(video_url, new_id)

From 542130a5d914e5f02acb872c88866194c66a612d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 13 Aug 2016 21:59:29 +0700
Subject: [PATCH 5762/6219] [pbs] Fix description extraction and update tests

---
 youtube_dl/extractor/pbs.py | 47 ++++++++++++++++++-------------------
 1 file changed, 23 insertions(+), 24 deletions(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 335e44bdc..09aef7fb9 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -10,6 +10,7 @@ from ..utils import (
     int_or_none,
     js_to_json,
     strip_jsonp,
+    strip_or_none,
     unified_strdate,
     US_RATINGS,
 )
@@ -200,7 +201,7 @@ class PBSIE(InfoExtractor):
                 'id': '2365006249',
                 'ext': 'mp4',
                 'title': 'Constitution USA with Peter Sagal - A More Perfect Union',
-                'description': 'md5:36f341ae62e251b8f5bd2b754b95a071',
+                'description': 'md5:31b664af3c65fd07fa460d306b837d00',
                 'duration': 3190,
             },
         },
@@ -211,7 +212,7 @@ class PBSIE(InfoExtractor):
                 'id': '2365297690',
                 'ext': 'mp4',
                 'title': 'FRONTLINE - Losing Iraq',
-                'description': 'md5:4d3eaa01f94e61b3e73704735f1196d9',
+                'description': 'md5:5979a4d069b157f622d02bff62fbe654',
                 'duration': 5050,
             },
         },
@@ -222,7 +223,7 @@ class PBSIE(InfoExtractor):
                 'id': '2201174722',
                 'ext': 'mp4',
                 'title': 'PBS NewsHour - Cyber Schools Gain Popularity, but Quality Questions Persist',
-                'description': 'md5:95a19f568689d09a166dff9edada3301',
+                'description': 'md5:86ab9a3d04458b876147b355788b8781',
                 'duration': 801,
             },
         },
@@ -267,7 +268,7 @@ class PBSIE(InfoExtractor):
                 'display_id': 'player',
                 'ext': 'mp4',
                 'title': 'American Experience - Death and the Civil War, Chapter 1',
-                'description': 'md5:1b80a74e0380ed2a4fb335026de1600d',
+                'description': 'md5:67fa89a9402e2ee7d08f53b920674c18',
                 'duration': 682,
                 'thumbnail': 're:^https?://.*\.jpg$',
             },
@@ -293,13 +294,13 @@ class PBSIE(InfoExtractor):
             # "<iframe style='position: absolute;<br />\ntop: 0; left: 0;' ...", see
             # https://github.com/rg3/youtube-dl/issues/7059)
             'url': 'http://www.pbs.org/food/features/a-chefs-life-season-3-episode-5-prickly-business/',
-            'md5': '84ced42850d78f1d4650297356e95e6f',
+            'md5': '59b0ef5009f9ac8a319cc5efebcd865e',
             'info_dict': {
                 'id': '2365546844',
                 'display_id': 'a-chefs-life-season-3-episode-5-prickly-business',
                 'ext': 'mp4',
                 'title': "A Chef's Life - Season 3, Ep. 5: Prickly Business",
-                'description': 'md5:54033c6baa1f9623607c6e2ed245888b',
+                'description': 'md5:c0ff7475a4b70261c7e58f493c2792a5',
                 'duration': 1480,
                 'thumbnail': 're:^https?://.*\.jpg$',
             },
@@ -312,7 +313,7 @@ class PBSIE(InfoExtractor):
                 'display_id': 'the-atomic-artists',
                 'ext': 'mp4',
                 'title': 'FRONTLINE - The Atomic Artists',
-                'description': 'md5:1a2481e86b32b2e12ec1905dd473e2c1',
+                'description': 'md5:f677e4520cfacb4a5ce1471e31b57800',
                 'duration': 723,
                 'thumbnail': 're:^https?://.*\.jpg$',
             },
@@ -323,7 +324,7 @@ class PBSIE(InfoExtractor):
         {
             # Serves hd only via wigget/partnerplayer page
             'url': 'http://www.pbs.org/video/2365641075/',
-            'md5': 'acfd4c400b48149a44861cb16dd305cf',
+            'md5': 'fdf907851eab57211dd589cf12006666',
             'info_dict': {
                 'id': '2365641075',
                 'ext': 'mp4',
@@ -333,16 +334,6 @@ class PBSIE(InfoExtractor):
                 'formats': 'mincount:8',
             },
         },
-        {
-            # has undocumented http formats(4500k and 6500k)
-            'url': 'http://www.pbs.org/video/2365815229/',
-            'md5': '94635cd06b7133688e23f4b94e6637a5',
-            'info_dict': {
-                'id': '2365815229',
-                'ext': 'mp4',
-                'title': 'FRONTLINE - Mosquito Hunter',
-            },
-        },
         {
             'url': 'http://player.pbs.org/widget/partnerplayer/2365297708/?start=0&end=0&chapterbar=false&endscreen=false&topbar=true',
             'only_matching': True,
@@ -362,11 +353,16 @@ class PBSIE(InfoExtractor):
     def _extract_webpage(self, url):
         mobj = re.match(self._VALID_URL, url)
 
+        description = None
+
         presumptive_id = mobj.group('presumptive_id')
         display_id = presumptive_id
         if presumptive_id:
             webpage = self._download_webpage(url, display_id)
 
+            description = strip_or_none(self._og_search_description(
+                webpage, default=None) or self._html_search_meta(
+                'description', webpage, default=None))
             upload_date = unified_strdate(self._search_regex(
                 r'<input type="hidden" id="air_date_[0-9]+" value="([^"]+)"',
                 webpage, 'upload date', default=None))
@@ -379,7 +375,7 @@ class PBSIE(InfoExtractor):
             for p in MULTI_PART_REGEXES:
                 tabbed_videos = re.findall(p, webpage)
                 if tabbed_videos:
-                    return tabbed_videos, presumptive_id, upload_date
+                    return tabbed_videos, presumptive_id, upload_date, description
 
             MEDIA_ID_REGEXES = [
                 r"div\s*:\s*'videoembed'\s*,\s*mediaid\s*:\s*'(\d+)'",  # frontline video embed
@@ -391,7 +387,7 @@ class PBSIE(InfoExtractor):
             media_id = self._search_regex(
                 MEDIA_ID_REGEXES, webpage, 'media ID', fatal=False, default=None)
             if media_id:
-                return media_id, presumptive_id, upload_date
+                return media_id, presumptive_id, upload_date, description
 
             # Fronline video embedded via flp
             video_id = self._search_regex(
@@ -408,7 +404,7 @@ class PBSIE(InfoExtractor):
                     'http://www.pbs.org/wgbh/pages/frontline/.json/getdir/getdir%d.json' % prg_id,
                     presumptive_id, 'Downloading getdir JSON',
                     transform_source=strip_jsonp)
-                return getdir['mid'], presumptive_id, upload_date
+                return getdir['mid'], presumptive_id, upload_date, description
 
             for iframe in re.findall(r'(?s)<iframe(.+?)></iframe>', webpage):
                 url = self._search_regex(
@@ -432,10 +428,10 @@ class PBSIE(InfoExtractor):
             video_id = mobj.group('id')
             display_id = video_id
 
-        return video_id, display_id, None
+        return video_id, display_id, None, description
 
     def _real_extract(self, url):
-        video_id, display_id, upload_date = self._extract_webpage(url)
+        video_id, display_id, upload_date, description = self._extract_webpage(url)
 
         if isinstance(video_id, list):
             entries = [self.url_result(
@@ -564,11 +560,14 @@ class PBSIE(InfoExtractor):
         if alt_title:
             info['title'] = alt_title + ' - ' + re.sub(r'^' + alt_title + '[\s\-:]+', '', info['title'])
 
+        description = info.get('description') or info.get(
+            'program', {}).get('description') or description
+
         return {
             'id': video_id,
             'display_id': display_id,
             'title': info['title'],
-            'description': info.get('description') or info.get('program', {}).get('description'),
+            'description': description,
             'thumbnail': info.get('image_url'),
             'duration': int_or_none(info.get('duration')),
             'age_limit': age_limit,

From 5ec5461e1a805595c5fef4ae482e86d7d7872d8b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 13 Aug 2016 22:50:18 +0700
Subject: [PATCH 5763/6219] [pbs] Clarify comment on http formats

---
 youtube_dl/extractor/pbs.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 09aef7fb9..b490ef74c 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -506,12 +506,12 @@ class PBSIE(InfoExtractor):
         if http_url:
             for m3u8_format in m3u8_formats:
                 bitrate = self._search_regex(r'(\d+)k', m3u8_format['url'], 'bitrate', default=None)
-                # lower qualities(150k and 192k) are not available as http formats
-                # https://github.com/rg3/youtube-dl/commit/cbc032c8b70a038a69259378c92b4ba97b42d491#commitcomment-17313656
-                # we will try to extract any http format higher than than the lowest quality documented in
-                # https://projects.pbs.org/confluence/display/coveapi/COVE+Video+Specifications
-                # as there also undocumented http formats formats(4500k and 6500k)
-                # http://www.pbs.org/video/2365815229/
+                # Lower qualities (150k and 192k) are not available as HTTP formats (see [1]),
+                # we won't try extracting them.
+                # Since summer 2016 higher quality formats (4500k and 6500k) are also available
+                # albeit they are not documented in [2].
+                # 1. https://github.com/rg3/youtube-dl/commit/cbc032c8b70a038a69259378c92b4ba97b42d491#commitcomment-17313656
+                # 2. https://projects.pbs.org/confluence/display/coveapi/COVE+Video+Specifications
                 if not bitrate or int(bitrate) < 400:
                     continue
                 f_url = re.sub(r'\d+k|baseline', bitrate + 'k', http_url)

From a560f28c98445e2ae2528795609d5ac718ec5b2c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 13 Aug 2016 23:01:35 +0700
Subject: [PATCH 5764/6219] [ChangeLog] Actualize

---
 ChangeLog | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index b1ce63d75..5efcb2316 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,7 +1,27 @@
 version <unreleased>
 
+Core
+* Show progress for curl external downloader
+* Forward more options to curl external downloader
+
 Extractors
+* [pbs] Fix description extraction
+* [franceculture] Fix extraction (#10324)
+* [pornotube] Fix extraction (#10322)
+* [4tube] Fix metadata extraction (#10321)
+* [imgur] Fix width and height extraction (#10325)
+* [expotv] Improve extraction
++ [vbox7] Fix extraction (#10309)
 - [tapely] Remove extractor (#10323)
+* [muenchentv] Fix extraction (#10313)
++ [24video] Add support for .me and .xxx TLDs
+* [24video] Fix comment count extraction
+* [sunporno] Add support for embed URLs
+* [sunporno] Fix metadata extraction (#10316)
++ [hgtv] Add extractor for hgtv.ca (#3999)
+- [pbs] Remove request to unavailable API
++ [pbs] Add support for high quality HTTP formats
++ [crunchyroll] Add support for HLS formats (#10301)
 
 version 2016.08.12
 

From 73a85620eeb2d595cd86f73357bc4cb081cb3bc9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 13 Aug 2016 23:17:11 +0700
Subject: [PATCH 5765/6219] release 2016.08.13

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 3 +--
 youtube_dl/version.py     | 2 +-
 4 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 6fdb2f77b..1e0d99b43 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.08.12*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.08.12**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.08.13*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.08.13**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.08.12
+[debug] youtube-dl version 2016.08.13
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 5efcb2316..fc99b9f73 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2016.08.13
 
 Core
 * Show progress for curl external downloader
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 8fb581d2b..56fc41a40 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -238,7 +238,6 @@
  - **FoxSports**
  - **france2.fr:generation-quoi**
  - **FranceCulture**
- - **FranceCultureEmission**
  - **FranceInter**
  - **francetv**: France 2, 3, 4, 5 and Ô
  - **francetvinfo.fr**
@@ -277,6 +276,7 @@
  - **HellPorno**
  - **Helsinki**: helsinki.fi
  - **HentaiStigma**
+ - **HGTV**
  - **HistoricFilms**
  - **history:topic**: History.com Topic
  - **hitbox**
@@ -664,7 +664,6 @@
  - **SztvHu**
  - **Tagesschau**
  - **tagesschau:player**
- - **Tapely**
  - **Tass**
  - **TDSLifeway**
  - **teachertube**: teachertube.com videos
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index becf14458..cc93d22aa 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.08.12'
+__version__ = '2016.08.13'

From 097eba019d0d5cab93e9ce66e1b727b782d48250 Mon Sep 17 00:00:00 2001
From: phi <crispyfrog@163.com>
Date: Sun, 14 Aug 2016 02:18:59 +0800
Subject: [PATCH 5766/6219] bug fix for extractor xiami.py

Before applying this patch, when downloading resources from xiami.com, it crashes with these:
Traceback (most recent call last):
  File "/home/phi/.local/bin/youtube-dl", line 11, in <module>
    sys.exit(main())
  File "/home/phi/.local/lib/python3.5/site-packages/youtube_dl/__init__.py", line 433, in main
    _real_main(argv)
  File "/home/phi/.local/lib/python3.5/site-packages/youtube_dl/__init__.py", line 423, in _real_main
    retcode = ydl.download(all_urls)
  File "/home/phi/.local/lib/python3.5/site-packages/youtube_dl/YoutubeDL.py", line 1786, in download
    url, force_generic_extractor=self.params.get('force_generic_extractor', False))
  File "/home/phi/.local/lib/python3.5/site-packages/youtube_dl/YoutubeDL.py", line 691, in extract_info
    ie_result = ie.extract(url)
  File "/home/phi/.local/lib/python3.5/site-packages/youtube_dl/extractor/common.py", line 347, in extract
    return self._real_extract(url)
  File "/home/phi/.local/lib/python3.5/site-packages/youtube_dl/extractor/xiami.py", line 116, in _real_extract
    return self._extract_tracks(self._match_id(url))[0]
  File "/home/phi/.local/lib/python3.5/site-packages/youtube_dl/extractor/xiami.py", line 43, in _extract_tracks
    '%s/%s%s' % (self._API_BASE_URL, item_id, '/type/%s' % typ if typ else ''), item_id)
  File "/home/phi/.local/lib/python3.5/site-packages/youtube_dl/extractor/common.py", line 562, in _download_json
    json_string, video_id, transform_source=transform_source, fatal=fatal)
  File "/home/phi/.local/lib/python3.5/site-packages/youtube_dl/extractor/common.py", line 568, in _parse_json
    return json.loads(json_string)
  File "/usr/lib/python3.5/json/__init__.py", line 312, in loads
    s.__class__.__name__))
TypeError: the JSON object must be str, not 'NoneType'

This patch solves exactly this problem.
---
 youtube_dl/extractor/xiami.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/xiami.py b/youtube_dl/extractor/xiami.py
index a6dfc4af9..86abef257 100644
--- a/youtube_dl/extractor/xiami.py
+++ b/youtube_dl/extractor/xiami.py
@@ -13,6 +13,7 @@ class XiamiBaseIE(InfoExtractor):
         webpage = super(XiamiBaseIE, self)._download_webpage(*args, **kwargs)
         if '>Xiami is currently not available in your country.<' in webpage:
             self.raise_geo_restricted('Xiami is currently not available in your country')
+        return webpage
 
     def _extract_track(self, track, track_id=None):
         title = track['title']

From fafabc0712d95e6a5b2ac56e9375fe90060738f5 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 14 Aug 2016 02:33:15 +0800
Subject: [PATCH 5767/6219] Update ChangeLog for #10342

[skip ci]
---
 ChangeLog | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index fc99b9f73..d04c5fc2a 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Extractors
+* [xiami] Fix extraction (#10342)
+
+
 version 2016.08.13
 
 Core
@@ -23,6 +29,7 @@ Extractors
 + [pbs] Add support for high quality HTTP formats
 + [crunchyroll] Add support for HLS formats (#10301)
 
+
 version 2016.08.12
 
 Core

From aaf44a2f47f013e8d864ac9f98b2833904a8be78 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 13 Aug 2016 22:53:07 +0100
Subject: [PATCH 5768/6219] [uplynk] Add new extractor

---
 youtube_dl/downloader/hls.py       |  6 +++
 youtube_dl/extractor/extractors.py |  4 ++
 youtube_dl/extractor/uplynk.py     | 64 ++++++++++++++++++++++++++++++
 3 files changed, 74 insertions(+)
 create mode 100644 youtube_dl/extractor/uplynk.py

diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index 3b7bb3508..8d7971e5d 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -20,6 +20,7 @@ from ..utils import (
     encodeFilename,
     sanitize_open,
     parse_m3u8_attributes,
+    update_url_query,
 )
 
 
@@ -82,6 +83,7 @@ class HlsFD(FragmentFD):
 
         self._prepare_and_start_frag_download(ctx)
 
+        extra_param_to_segment_url = info_dict.get('extra_param_to_segment_url')
         i = 0
         media_sequence = 0
         decrypt_info = {'METHOD': 'NONE'}
@@ -95,6 +97,8 @@ class HlsFD(FragmentFD):
                         if re.match(r'^https?://', line)
                         else compat_urlparse.urljoin(man_url, line))
                     frag_filename = '%s-Frag%d' % (ctx['tmpfilename'], i)
+                    if extra_param_to_segment_url:
+                        frag_url = update_url_query(frag_url, extra_param_to_segment_url)
                     success = ctx['dl'].download(frag_filename, {'url': frag_url})
                     if not success:
                         return False
@@ -120,6 +124,8 @@ class HlsFD(FragmentFD):
                         if not re.match(r'^https?://', decrypt_info['URI']):
                             decrypt_info['URI'] = compat_urlparse.urljoin(
                                 man_url, decrypt_info['URI'])
+                        if extra_param_to_segment_url:
+                            decrypt_info['URI'] = update_url_query(decrypt_info['URI'], extra_param_to_segment_url)
                         decrypt_info['KEY'] = self.ydl.urlopen(decrypt_info['URI']).read()
                 elif line.startswith('#EXT-X-MEDIA-SEQUENCE'):
                     media_sequence = int(line[22:])
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 82d4ed153..901847509 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -926,6 +926,10 @@ from .udn import UDNEmbedIE
 from .digiteka import DigitekaIE
 from .unistra import UnistraIE
 from .uol import UOLIE
+from .uplynk import (
+    UplynkIE,
+    UplynkPreplayIE,
+)
 from .urort import UrortIE
 from .urplay import URPlayIE
 from .usatoday import USATodayIE
diff --git a/youtube_dl/extractor/uplynk.py b/youtube_dl/extractor/uplynk.py
new file mode 100644
index 000000000..a6a685c9d
--- /dev/null
+++ b/youtube_dl/extractor/uplynk.py
@@ -0,0 +1,64 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    float_or_none,
+    ExtractorError,
+)
+
+
+class UplynkIE(InfoExtractor):
+    _VALID_URL = r'https?://.*?\.uplynk\.com/(?P<path>ext/[0-9a-f]{32}/(?P<external_id>[^/?&]+)|(?P<id>[0-9a-f]{32}))\.(?:m3u8|json)(?:.*?\bpbs=(?P<session_id>[^&]+))?'
+    _TEST = {
+        'url': 'http://content.uplynk.com/e89eaf2ce9054aa89d92ddb2d817a52e.m3u8',
+        'info_dict': {
+            'id': 'e89eaf2ce9054aa89d92ddb2d817a52e',
+            'ext': 'mp4',
+            'title': '030816-kgo-530pm-solar-eclipse-vid_web.mp4',
+            'uploader_id': '4413701bf5a1488db55b767f8ae9d4fa',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }
+
+    def _real_extract(self, url):
+        path, external_id, video_id, session_id = re.match(self._VALID_URL, url).groups()
+        display_id = video_id or external_id
+        formats = self._extract_m3u8_formats('http://content.uplynk.com/%s.m3u8' % path, display_id, 'mp4')
+        if session_id:
+            for f in formats:
+                f['extra_param_to_segment_url'] = {
+                    'pbs': session_id,
+                }
+        self._sort_formats(formats)
+        asset = self._download_json('http://content.uplynk.com/player/assetinfo/%s.json' % path, display_id)
+        if asset.get('error') == 1:
+            raise ExtractorError('% said: %s' % (self.IE_NAME, asset['msg']), expected=True)
+
+        return {
+            'id': asset['asset'],
+            'title': asset['desc'],
+            'thumbnail': asset.get('default_poster_url'),
+            'duration': float_or_none(asset.get('duration')),
+            'uploader_id': asset.get('owner'),
+            'formats': formats,
+        }
+
+
+class UplynkPreplayIE(InfoExtractor):
+    _VALID_URL = r'https?://.*?\.uplynk\.com/preplay2?/(?P<path>ext/[0-9a-f]{32}/(?P<external_id>[^/?&]+)|(?P<id>[0-9a-f]{32}))\.json'
+
+    def _real_extract(self, url):
+        path, external_id, video_id = re.match(self._VALID_URL, url).groups()
+        display_id = video_id or external_id
+        preplay = self._download_json(url, display_id)
+        content_url = 'http://content.uplynk.com/%s.m3u8' % path
+        session_id = preplay.get('sid')
+        if session_id:
+            content_url += '?pbs=' + session_id
+        return self.url_result(content_url, 'Uplynk')

From 320d597c21e7a0981f1dc9c4167fce53473ab488 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 14 Aug 2016 16:25:14 +0700
Subject: [PATCH 5769/6219] [vgtv] Detect geo restricted videos (#10348)

---
 youtube_dl/extractor/vgtv.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/youtube_dl/extractor/vgtv.py b/youtube_dl/extractor/vgtv.py
index b11cd254c..185756301 100644
--- a/youtube_dl/extractor/vgtv.py
+++ b/youtube_dl/extractor/vgtv.py
@@ -8,6 +8,7 @@ from .xstream import XstreamIE
 from ..utils import (
     ExtractorError,
     float_or_none,
+    try_get,
 )
 
 
@@ -129,6 +130,11 @@ class VGTVIE(XstreamIE):
             'url': 'http://ap.vgtv.no/webtv#!/video/111084/de-nye-bysyklene-lettere-bedre-gir-stoerre-hjul-og-feste-til-mobil',
             'only_matching': True,
         },
+        {
+            # geoblocked
+            'url': 'http://www.vgtv.no/#!/video/127205/inside-the-mind-of-favela-funk',
+            'only_matching': True,
+        },
     ]
 
     def _real_extract(self, url):
@@ -196,6 +202,12 @@ class VGTVIE(XstreamIE):
 
         info['formats'].extend(formats)
 
+        if not info['formats']:
+            properties = try_get(
+                data, lambda x: x['streamConfiguration']['properties'], list)
+            if properties and 'geoblocked' in properties:
+                raise self.raise_geo_restricted()
+
         self._sort_formats(info['formats'])
 
         info.update({

From 2118fdd1a96ed7a904b53ed5aad50a203d0e0c70 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 14 Aug 2016 11:48:13 +0100
Subject: [PATCH 5770/6219] [common] add separate method for getting netrc
 ligin info

---
 youtube_dl/extractor/common.py | 30 ++++++++++++++++++++----------
 1 file changed, 20 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index e47770c1d..9427ff449 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -662,6 +662,24 @@ class InfoExtractor(object):
         else:
             return res
 
+    def _get_netrc_login_info(self, netrc_machine=None):
+        username = None
+        password = None
+        netrc_machine = netrc_machine or self._NETRC_MACHINE
+
+        if self._downloader.params.get('usenetrc', False):
+            try:
+                info = netrc.netrc().authenticators(netrc_machine)
+                if info is not None:
+                    username = info[0]
+                    password = info[2]
+                else:
+                    raise netrc.NetrcParseError('No authenticators for %s' % netrc_machine)
+            except (IOError, netrc.NetrcParseError) as err:
+                self._downloader.report_warning('parsing .netrc: %s' % error_to_compat_str(err))
+
+        return (username, password)
+
     def _get_login_info(self):
         """
         Get the login info as (username, password)
@@ -679,16 +697,8 @@ class InfoExtractor(object):
         if downloader_params.get('username') is not None:
             username = downloader_params['username']
             password = downloader_params['password']
-        elif downloader_params.get('usenetrc', False):
-            try:
-                info = netrc.netrc().authenticators(self._NETRC_MACHINE)
-                if info is not None:
-                    username = info[0]
-                    password = info[2]
-                else:
-                    raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE)
-            except (IOError, netrc.NetrcParseError) as err:
-                self._downloader.report_warning('parsing .netrc: %s' % error_to_compat_str(err))
+        else:
+            username, password = self._get_netrc_login_info()
 
         return (username, password)
 

From 9771b1f901b19ad5ba6632a37fc6348e8e6e98dd Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 14 Aug 2016 11:52:48 +0100
Subject: [PATCH 5771/6219] [theplatform] use _get_netrc_login_info and fix
 session expiration check(#10345)

---
 youtube_dl/extractor/theplatform.py | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index bb3efc4ea..9ca765a5f 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -218,15 +218,16 @@ class ThePlatformIE(ThePlatformBaseIE):
         requestor_info = self._downloader.cache.load('mvpd', requestor_id) or {}
         authn_token = requestor_info.get('authn_token')
         if authn_token:
-            token_expires = unified_timestamp(xml_text(authn_token, 'simpleTokenExpires').replace('_GMT', ''))
-            if token_expires and token_expires >= time.time():
+            token_expires = unified_timestamp(re.sub(r'[_ ]GMT', '', xml_text(authn_token, 'simpleTokenExpires')))
+            if token_expires and token_expires <= int(time.time()):
                 authn_token = None
+                requestor_info = {}
         if not authn_token:
             # TODO add support for other TV Providers
             mso_id = 'DTV'
-            login_info = netrc.netrc().authenticators(mso_id)
-            if not login_info:
-                return None
+            username, password = self._get_netrc_login_info(mso_id)
+            if not username or not password:
+                return ''
 
             def post_form(form_page, note, data={}):
                 post_url = self._html_search_regex(r'<form[^>]+action=(["\'])(?P<url>.+?)\1', form_page, 'post url', group='url')
@@ -248,8 +249,8 @@ class ThePlatformIE(ThePlatformBaseIE):
             provider_login_page = post_form(
                 provider_redirect_page, 'Downloading Provider Login Page')
             mvpd_confirm_page = post_form(provider_login_page, 'Logging in', {
-                'username': login_info[0],
-                'password': login_info[2],
+                'username': username,
+                'password': password,
             })
             post_form(mvpd_confirm_page, 'Confirming Login')
 

From 884cdb6cd9c872ea68a03341e462b58e51fba58a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 14 Aug 2016 20:49:11 +0700
Subject: [PATCH 5772/6219] [life:embed] Improve extraction

---
 youtube_dl/extractor/lifenews.py | 68 +++++++++++++++++++++++---------
 1 file changed, 49 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/extractor/lifenews.py b/youtube_dl/extractor/lifenews.py
index c2b4490c4..87120ecd1 100644
--- a/youtube_dl/extractor/lifenews.py
+++ b/youtube_dl/extractor/lifenews.py
@@ -4,7 +4,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urlparse
+from ..compat import (
+    compat_str,
+    compat_urlparse,
+)
 from ..utils import (
     determine_ext,
     ExtractorError,
@@ -96,7 +99,7 @@ class LifeNewsIE(InfoExtractor):
             r'<video[^>]+><source[^>]+src=["\'](.+?)["\']', webpage)
 
         iframe_links = re.findall(
-            r'<iframe[^>]+src=["\']((?:https?:)?//embed\.life\.ru/embed/.+?)["\']',
+            r'<iframe[^>]+src=["\']((?:https?:)?//embed\.life\.ru/(?:embed|video)/.+?)["\']',
             webpage)
 
         if not video_urls and not iframe_links:
@@ -164,9 +167,9 @@ class LifeNewsIE(InfoExtractor):
 
 class LifeEmbedIE(InfoExtractor):
     IE_NAME = 'life:embed'
-    _VALID_URL = r'https?://embed\.life\.ru/embed/(?P<id>[\da-f]{32})'
+    _VALID_URL = r'https?://embed\.life\.ru/(?:embed|video)/(?P<id>[\da-f]{32})'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://embed.life.ru/embed/e50c2dec2867350528e2574c899b8291',
         'md5': 'b889715c9e49cb1981281d0e5458fbbe',
         'info_dict': {
@@ -175,30 +178,57 @@ class LifeEmbedIE(InfoExtractor):
             'title': 'e50c2dec2867350528e2574c899b8291',
             'thumbnail': 're:http://.*\.jpg',
         }
-    }
+    }, {
+        # with 1080p
+        'url': 'https://embed.life.ru/video/e50c2dec2867350528e2574c899b8291',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id)
 
+        thumbnail = None
         formats = []
-        for video_url in re.findall(r'"file"\s*:\s*"([^"]+)', webpage):
-            video_url = compat_urlparse.urljoin(url, video_url)
-            ext = determine_ext(video_url)
-            if ext == 'm3u8':
-                formats.extend(self._extract_m3u8_formats(
-                    video_url, video_id, 'mp4',
-                    entry_protocol='m3u8_native', m3u8_id='m3u8'))
-            else:
-                formats.append({
-                    'url': video_url,
-                    'format_id': ext,
-                    'preference': 1,
-                })
+
+        def extract_m3u8(manifest_url):
+            formats.extend(self._extract_m3u8_formats(
+                manifest_url, video_id, 'mp4',
+                entry_protocol='m3u8_native', m3u8_id='m3u8'))
+
+        def extract_original(original_url):
+            formats.append({
+                'url': original_url,
+                'format_id': determine_ext(original_url, None),
+                'preference': 1,
+            })
+
+        playlist = self._parse_json(
+            self._search_regex(
+                r'options\s*=\s*({.+?});', webpage, 'options', default='{}'),
+            video_id).get('playlist', {})
+        if playlist:
+            master = playlist.get('master')
+            if isinstance(master, compat_str) and determine_ext(master) == 'm3u8':
+                extract_m3u8(compat_urlparse.urljoin(url, master))
+            original = playlist.get('original')
+            if isinstance(original, compat_str):
+                extract_original(original)
+            thumbnail = playlist.get('image')
+
+        # Old rendition fallback
+        if not formats:
+            for video_url in re.findall(r'"file"\s*:\s*"([^"]+)', webpage):
+                video_url = compat_urlparse.urljoin(url, video_url)
+                if determine_ext(video_url) == 'm3u8':
+                    extract_m3u8(video_url)
+                else:
+                    extract_original(video_url)
+
         self._sort_formats(formats)
 
-        thumbnail = self._search_regex(
+        thumbnail = thumbnail or self._search_regex(
             r'"image"\s*:\s*"([^"]+)', webpage, 'thumbnail', default=None)
 
         return {

From 1fd6e30988f44d372c7112c2d5e44c0d5cdbc4ed Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 14 Aug 2016 17:55:56 +0100
Subject: [PATCH 5773/6219] [adobepass] create separate class for adobe pass
 authentication

---
 youtube_dl/extractor/adobepass.py          | 124 +++++++++++++++++++++
 youtube_dl/extractor/aenetworks.py         |   5 +-
 youtube_dl/extractor/nationalgeographic.py |   2 +-
 youtube_dl/extractor/syfy.py               |   4 +-
 youtube_dl/extractor/theplatform.py        |  98 +---------------
 5 files changed, 134 insertions(+), 99 deletions(-)
 create mode 100644 youtube_dl/extractor/adobepass.py

diff --git a/youtube_dl/extractor/adobepass.py b/youtube_dl/extractor/adobepass.py
new file mode 100644
index 000000000..4e59302ab
--- /dev/null
+++ b/youtube_dl/extractor/adobepass.py
@@ -0,0 +1,124 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+import re
+import time
+import xml.etree.ElementTree as etree
+
+from .common import InfoExtractor
+from ..utils import (
+    unescapeHTML,
+    urlencode_postdata,
+    unified_timestamp,
+)
+
+
+class AdobePass(InfoExtractor):
+    _SERVICE_PROVIDER_TEMPLATE = 'https://sp.auth.adobe.com/adobe-services/%s'
+    _USER_AGENT = 'Mozilla/5.0 (X11; Linux i686; rv:47.0) Gecko/20100101 Firefox/47.0'
+
+    @staticmethod
+    def _get_mvpd_resource(provider_id, title, guid, rating):
+        channel = etree.Element('channel')
+        channel_title = etree.SubElement(channel, 'title')
+        channel_title.text = provider_id
+        item = etree.SubElement(channel, 'item')
+        resource_title = etree.SubElement(item, 'title')
+        resource_title.text = title
+        resource_guid = etree.SubElement(item, 'guid')
+        resource_guid.text = guid
+        resource_rating = etree.SubElement(item, 'media:rating')
+        resource_rating.attrib = {'scheme': 'urn:v-chip'}
+        resource_rating.text = rating
+        return '<rss version="2.0" xmlns:media="http://search.yahoo.com/mrss/">' + etree.tostring(channel).decode() + '</rss>'
+
+    def _extract_mvpd_auth(self, url, video_id, requestor_id, resource):
+        def xml_text(xml_str, tag):
+            return self._search_regex(
+                '<%s>(.+?)</%s>' % (tag, tag), xml_str, tag)
+
+        mvpd_headers = {
+            'ap_42': 'anonymous',
+            'ap_11': 'Linux i686',
+            'ap_z': self._USER_AGENT,
+            'User-Agent': self._USER_AGENT,
+        }
+
+        guid = xml_text(resource, 'guid')
+        requestor_info = self._downloader.cache.load('mvpd', requestor_id) or {}
+        authn_token = requestor_info.get('authn_token')
+        if authn_token:
+            token_expires = unified_timestamp(re.sub(r'[_ ]GMT', '', xml_text(authn_token, 'simpleTokenExpires')))
+            if token_expires and token_expires <= int(time.time()):
+                authn_token = None
+                requestor_info = {}
+        if not authn_token:
+            # TODO add support for other TV Providers
+            mso_id = 'DTV'
+            username, password = self._get_netrc_login_info(mso_id)
+            if not username or not password:
+                return ''
+
+            def post_form(form_page, note, data={}):
+                post_url = self._html_search_regex(r'<form[^>]+action=(["\'])(?P<url>.+?)\1', form_page, 'post url', group='url')
+                return self._download_webpage(
+                    post_url, video_id, note, data=urlencode_postdata(data or self._hidden_inputs(form_page)), headers={
+                        'Content-Type': 'application/x-www-form-urlencoded',
+                    })
+
+            provider_redirect_page = self._download_webpage(
+                self._SERVICE_PROVIDER_TEMPLATE % 'authenticate/saml', video_id,
+                'Downloading Provider Redirect Page', query={
+                    'noflash': 'true',
+                    'mso_id': mso_id,
+                    'requestor_id': requestor_id,
+                    'no_iframe': 'false',
+                    'domain_name': 'adobe.com',
+                    'redirect_url': url,
+                })
+            provider_login_page = post_form(
+                provider_redirect_page, 'Downloading Provider Login Page')
+            mvpd_confirm_page = post_form(provider_login_page, 'Logging in', {
+                'username': username,
+                'password': password,
+            })
+            post_form(mvpd_confirm_page, 'Confirming Login')
+
+            session = self._download_webpage(
+                self._SERVICE_PROVIDER_TEMPLATE % 'session', video_id,
+                'Retrieving Session', data=urlencode_postdata({
+                    '_method': 'GET',
+                    'requestor_id': requestor_id,
+                }), headers=mvpd_headers)
+            authn_token = unescapeHTML(xml_text(session, 'authnToken'))
+            requestor_info['authn_token'] = authn_token
+            self._downloader.cache.store('mvpd', requestor_id, requestor_info)
+
+        authz_token = requestor_info.get(guid)
+        if not authz_token:
+            authorize = self._download_webpage(
+                self._SERVICE_PROVIDER_TEMPLATE % 'authorize', video_id,
+                'Retrieving Authorization Token', data=urlencode_postdata({
+                    'resource_id': resource,
+                    'requestor_id': requestor_id,
+                    'authentication_token': authn_token,
+                    'mso_id': xml_text(authn_token, 'simpleTokenMsoID'),
+                    'userMeta': '1',
+                }), headers=mvpd_headers)
+            authz_token = unescapeHTML(xml_text(authorize, 'authzToken'))
+            requestor_info[guid] = authz_token
+            self._downloader.cache.store('mvpd', requestor_id, requestor_info)
+
+        mvpd_headers.update({
+            'ap_19': xml_text(authn_token, 'simpleSamlNameID'),
+            'ap_23': xml_text(authn_token, 'simpleSamlSessionIndex'),
+        })
+
+        return self._download_webpage(
+            self._SERVICE_PROVIDER_TEMPLATE % 'shortAuthorize',
+            video_id, 'Retrieving Media Token', data=urlencode_postdata({
+                'authz_token': authz_token,
+                'requestor_id': requestor_id,
+                'session_guid': xml_text(authn_token, 'simpleTokenAuthenticationGuid'),
+                'hashed_guid': 'false',
+            }), headers=mvpd_headers)
diff --git a/youtube_dl/extractor/aenetworks.py b/youtube_dl/extractor/aenetworks.py
index 8f53050c9..6adb6d824 100644
--- a/youtube_dl/extractor/aenetworks.py
+++ b/youtube_dl/extractor/aenetworks.py
@@ -109,7 +109,10 @@ class AENetworksIE(AENetworksBaseIE):
         info = self._parse_theplatform_metadata(theplatform_metadata)
         if theplatform_metadata.get('AETN$isBehindWall'):
             requestor_id = self._DOMAIN_TO_REQUESTOR_ID[domain]
-            resource = '<rss version="2.0" xmlns:media="http://search.yahoo.com/mrss/"><channel><title>%s</title><item><title>%s</title><guid>%s</guid><media:rating scheme="urn:v-chip">%s</media:rating></item></channel></rss>' % (requestor_id, theplatform_metadata['title'], theplatform_metadata['AETN$PPL_pplProgramId'], theplatform_metadata['ratings'][0]['rating'])
+            resource = self._get_mvpd_resource(
+                requestor_id, theplatform_metadata['title'],
+                theplatform_metadata.get('AETN$PPL_pplProgramId') or theplatform_metadata.get('AETN$PPL_pplProgramId_OLD'),
+                theplatform_metadata['ratings'][0]['rating'])
             query['auth'] = self._extract_mvpd_auth(
                 url, video_id, requestor_id, resource)
         info.update(self._search_json_ld(webpage, video_id, fatal=False))
diff --git a/youtube_dl/extractor/nationalgeographic.py b/youtube_dl/extractor/nationalgeographic.py
index 0027ff1b8..890e8d5bc 100644
--- a/youtube_dl/extractor/nationalgeographic.py
+++ b/youtube_dl/extractor/nationalgeographic.py
@@ -119,7 +119,7 @@ class NationalGeographicIE(ThePlatformIE):
             auth_resource_id = self._search_regex(
                 r"video_auth_resourceId\s*=\s*'([^']+)'",
                 webpage, 'auth resource id')
-            query['auth'] = self._extract_mvpd_auth(url, display_id, 'natgeo', auth_resource_id) or ''
+            query['auth'] = self._extract_mvpd_auth(url, display_id, 'natgeo', auth_resource_id)
 
         return {
             '_type': 'url_transparent',
diff --git a/youtube_dl/extractor/syfy.py b/youtube_dl/extractor/syfy.py
index 53723b66e..764287a64 100644
--- a/youtube_dl/extractor/syfy.py
+++ b/youtube_dl/extractor/syfy.py
@@ -40,7 +40,9 @@ class SyfyIE(ThePlatformIE):
             'manifest': 'm3u',
         }
         if syfy_mpx.get('entitlement') == 'auth':
-            resource = '<rss version="2.0" xmlns:media="http://search.yahoo.com/mrss/"><channel><title>syfy</title><item><title><![CDATA[%s]]></title><guid>%s</guid><media:rating scheme="urn:v-chip">%s</media:rating></item></channel></rss>' % (title, video_id, syfy_mpx.get('mpxRating', 'TV-14'))
+            resource = self._get_mvpd_resource(
+                'syfy', title, video_id,
+                syfy_mpx.get('mpxRating', 'TV-14'))
             query['auth'] = self._extract_mvpd_auth(
                 url, video_id, 'syfy', resource)
 
diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index 9ca765a5f..108ddd3a9 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -6,10 +6,10 @@ import time
 import hmac
 import binascii
 import hashlib
-import netrc
 
 
 from .once import OnceIE
+from .adobepass import AdobePass
 from ..compat import (
     compat_parse_qs,
     compat_urllib_parse_urlparse,
@@ -25,9 +25,6 @@ from ..utils import (
     xpath_with_ns,
     mimetype2ext,
     find_xpath_attr,
-    unescapeHTML,
-    urlencode_postdata,
-    unified_timestamp,
 )
 
 default_ns = 'http://www.w3.org/2005/SMIL21/Language'
@@ -96,7 +93,7 @@ class ThePlatformBaseIE(OnceIE):
         return self._parse_theplatform_metadata(info)
 
 
-class ThePlatformIE(ThePlatformBaseIE):
+class ThePlatformIE(ThePlatformBaseIE, AdobePass):
     _VALID_URL = r'''(?x)
         (?:https?://(?:link|player)\.theplatform\.com/[sp]/(?P<provider_id>[^/]+)/
            (?:(?:(?:[^/]+/)+select/)?(?P<media>media/(?:guid/\d+/)?)|(?P<config>(?:[^/\?]+/(?:swf|config)|onsite)/select/))?
@@ -202,97 +199,6 @@ class ThePlatformIE(ThePlatformBaseIE):
         sig = flags + expiration_date + checksum + str_to_hex(sig_secret)
         return '%s&sig=%s' % (url, sig)
 
-    def _extract_mvpd_auth(self, url, video_id, requestor_id, resource):
-        def xml_text(xml_str, tag):
-            return self._search_regex(
-                '<%s>(.+?)</%s>' % (tag, tag), xml_str, tag)
-
-        mvpd_headers = {
-            'ap_42': 'anonymous',
-            'ap_11': 'Linux i686',
-            'ap_z': 'Mozilla/5.0 (X11; Linux i686; rv:47.0) Gecko/20100101 Firefox/47.0',
-            'User-Agent': 'Mozilla/5.0 (X11; Linux i686; rv:47.0) Gecko/20100101 Firefox/47.0',
-        }
-
-        guid = xml_text(resource, 'guid')
-        requestor_info = self._downloader.cache.load('mvpd', requestor_id) or {}
-        authn_token = requestor_info.get('authn_token')
-        if authn_token:
-            token_expires = unified_timestamp(re.sub(r'[_ ]GMT', '', xml_text(authn_token, 'simpleTokenExpires')))
-            if token_expires and token_expires <= int(time.time()):
-                authn_token = None
-                requestor_info = {}
-        if not authn_token:
-            # TODO add support for other TV Providers
-            mso_id = 'DTV'
-            username, password = self._get_netrc_login_info(mso_id)
-            if not username or not password:
-                return ''
-
-            def post_form(form_page, note, data={}):
-                post_url = self._html_search_regex(r'<form[^>]+action=(["\'])(?P<url>.+?)\1', form_page, 'post url', group='url')
-                return self._download_webpage(
-                    post_url, video_id, note, data=urlencode_postdata(data or self._hidden_inputs(form_page)), headers={
-                        'Content-Type': 'application/x-www-form-urlencoded',
-                    })
-
-            provider_redirect_page = self._download_webpage(
-                self._SERVICE_PROVIDER_TEMPLATE % 'authenticate/saml', video_id,
-                'Downloading Provider Redirect Page', query={
-                    'noflash': 'true',
-                    'mso_id': mso_id,
-                    'requestor_id': requestor_id,
-                    'no_iframe': 'false',
-                    'domain_name': 'adobe.com',
-                    'redirect_url': url,
-                })
-            provider_login_page = post_form(
-                provider_redirect_page, 'Downloading Provider Login Page')
-            mvpd_confirm_page = post_form(provider_login_page, 'Logging in', {
-                'username': username,
-                'password': password,
-            })
-            post_form(mvpd_confirm_page, 'Confirming Login')
-
-            session = self._download_webpage(
-                self._SERVICE_PROVIDER_TEMPLATE % 'session', video_id,
-                'Retrieving Session', data=urlencode_postdata({
-                    '_method': 'GET',
-                    'requestor_id': requestor_id,
-                }), headers=mvpd_headers)
-            authn_token = unescapeHTML(xml_text(session, 'authnToken'))
-            requestor_info['authn_token'] = authn_token
-            self._downloader.cache.store('mvpd', requestor_id, requestor_info)
-
-        authz_token = requestor_info.get(guid)
-        if not authz_token:
-            authorize = self._download_webpage(
-                self._SERVICE_PROVIDER_TEMPLATE % 'authorize', video_id,
-                'Retrieving Authorization Token', data=urlencode_postdata({
-                    'resource_id': resource,
-                    'requestor_id': requestor_id,
-                    'authentication_token': authn_token,
-                    'mso_id': xml_text(authn_token, 'simpleTokenMsoID'),
-                    'userMeta': '1',
-                }), headers=mvpd_headers)
-            authz_token = unescapeHTML(xml_text(authorize, 'authzToken'))
-            requestor_info[guid] = authz_token
-            self._downloader.cache.store('mvpd', requestor_id, requestor_info)
-
-        mvpd_headers.update({
-            'ap_19': xml_text(authn_token, 'simpleSamlNameID'),
-            'ap_23': xml_text(authn_token, 'simpleSamlSessionIndex'),
-        })
-
-        return self._download_webpage(
-            self._SERVICE_PROVIDER_TEMPLATE % 'shortAuthorize',
-            video_id, 'Retrieving Media Token', data=urlencode_postdata({
-                'authz_token': authz_token,
-                'requestor_id': requestor_id,
-                'session_guid': xml_text(authn_token, 'simpleTokenAuthenticationGuid'),
-                'hashed_guid': 'false',
-            }), headers=mvpd_headers)
-
     def _real_extract(self, url):
         url, smuggled_data = unsmuggle_url(url, {})
 

From d2ac04674d0d9085aedec229820c1d07082e5825 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 14 Aug 2016 18:03:42 +0100
Subject: [PATCH 5774/6219] [viceland] Add new extractor(#8799)

---
 youtube_dl/extractor/extractors.py |   1 +
 youtube_dl/extractor/viceland.py   | 100 +++++++++++++++++++++++++++++
 2 files changed, 101 insertions(+)
 create mode 100644 youtube_dl/extractor/viceland.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 901847509..be96e34ba 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -958,6 +958,7 @@ from .vice import (
     ViceIE,
     ViceShowIE,
 )
+from .viceland import VicelandIE
 from .vidbit import VidbitIE
 from .viddler import ViddlerIE
 from .videodetective import VideoDetectiveIE
diff --git a/youtube_dl/extractor/viceland.py b/youtube_dl/extractor/viceland.py
new file mode 100644
index 000000000..c66e8eb95
--- /dev/null
+++ b/youtube_dl/extractor/viceland.py
@@ -0,0 +1,100 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import time
+import hashlib
+import json
+
+from .adobepass import AdobePass
+from ..compat import compat_HTTPError
+from ..utils import (
+    int_or_none,
+    parse_age_limit,
+    str_or_none,
+    parse_duration,
+    ExtractorError,
+    extract_attributes,
+)
+
+
+class VicelandIE(AdobePass):
+    _VALID_URL = r'https?://(?:www\.)?viceland\.com/[^/]+/video/[^/]+/(?P<id>[a-f0-9]+)'
+    _TEST = {
+        # FIXME: fill the test after fixing delegation problem
+        'url': 'https://www.viceland.com/en_us/video/cyberwar-trailer/57608447973ee7705f6fbd4e',
+        'info_dict': {
+            'id': '57608447973ee7705f6fbd4e',
+            'ext': 'mp4',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+        'add_ie': ['UplynkPreplay', 'Uplynk'],
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+        watch_hub_data = extract_attributes(self._search_regex(
+            r'(?s)(<watch-hub\s*.+?</watch-hub>)', webpage, 'watch hub'))
+        video_id = watch_hub_data['vms-id']
+        title = watch_hub_data['video-title']
+
+        query = {}
+        if watch_hub_data.get('video-locked') == '1':
+            resource = self._get_mvpd_resource(
+                'VICELAND', title, video_id,
+                watch_hub_data.get('video-rating'))
+            query['tvetoken'] = self._extract_mvpd_auth(url, video_id, 'VICELAND', resource)
+
+        # signature generation algorithm is reverse engineered from signatureGenerator in
+        # webpack:///../shared/~/vice-player/dist/js/vice-player.js in
+        # https://www.viceland.com/assets/common/js/web.vendor.bundle.js
+        exp = int(time.time()) + 14400
+        query.update({
+            'exp': exp,
+            'sign': hashlib.sha512(('%s:GET:%d' % (video_id, exp)).encode()).hexdigest(),
+        })
+
+        try:
+            preplay = self._download_json('https://www.viceland.com/en_us/preplay/%s' % video_id, video_id, query=query)
+        except ExtractorError as e:
+            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 400:
+                error = json.loads(e.cause.read().decode())
+                raise ExtractorError('%s said: %s' % (self.IE_NAME, error['details']), expected=True)
+
+        video_data = preplay['video']
+        base = video_data['base']
+        uplynk_preplay_url = preplay['preplayURL']
+        episode = video_data.get('episode', {})
+        channel = video_data.get('channel', {})
+
+        subtitles = {}
+        cc_url = preplay.get('ccURL')
+        if cc_url:
+            subtitles['en'] = [{
+                'url': cc_url,
+            }]
+
+        return {
+            '_type': 'url_transparent',
+            'url': uplynk_preplay_url,
+            'id': video_id,
+            'title': title,
+            'description': base.get('body'),
+            'thumbnail': watch_hub_data.get('cover-image') or watch_hub_data.get('thumbnail'),
+            'duration': parse_duration(video_data.get('video_duration') or watch_hub_data.get('video-duration')),
+            'timestamp': int_or_none(video_data.get('created_at')),
+            'age_limit': parse_age_limit(video_data.get('video_rating')),
+            'series': video_data.get('show_title') or watch_hub_data.get('show-title'),
+            'episode_number': int_or_none(episode.get('episode_number') or watch_hub_data.get('episode')),
+            'episode_id': str_or_none(episode.get('id') or video_data.get('episode_id')),
+            'season_number': int_or_none(watch_hub_data.get('season')),
+            'season_id': str_or_none(episode.get('season_id')),
+            'uploader': channel.get('base', {}).get('title') or watch_hub_data.get('channel-title'),
+            'uploader_id': str_or_none(channel.get('id')),
+            'subtitles': subtitles,
+            'ie_key': 'UplynkPreplay',
+        }

From 9fa57892790ce205634f6a7c83de2b9e52ab5284 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 14 Aug 2016 19:04:23 +0100
Subject: [PATCH 5775/6219] [viceland] fix info extraction(closes #8799)

---
 youtube_dl/extractor/uplynk.py   | 11 +++++++----
 youtube_dl/extractor/viceland.py |  7 +++++++
 2 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/uplynk.py b/youtube_dl/extractor/uplynk.py
index a6a685c9d..4313bc9cb 100644
--- a/youtube_dl/extractor/uplynk.py
+++ b/youtube_dl/extractor/uplynk.py
@@ -26,8 +26,8 @@ class UplynkIE(InfoExtractor):
         },
     }
 
-    def _real_extract(self, url):
-        path, external_id, video_id, session_id = re.match(self._VALID_URL, url).groups()
+    def _extract_uplynk_info(self, uplynk_content_url):
+        path, external_id, video_id, session_id = re.match(UplynkIE._VALID_URL, uplynk_content_url).groups()
         display_id = video_id or external_id
         formats = self._extract_m3u8_formats('http://content.uplynk.com/%s.m3u8' % path, display_id, 'mp4')
         if session_id:
@@ -49,8 +49,11 @@ class UplynkIE(InfoExtractor):
             'formats': formats,
         }
 
+    def _real_extract(self, url):
+        return self._extract_uplynk_info(url)
 
-class UplynkPreplayIE(InfoExtractor):
+
+class UplynkPreplayIE(UplynkIE):
     _VALID_URL = r'https?://.*?\.uplynk\.com/preplay2?/(?P<path>ext/[0-9a-f]{32}/(?P<external_id>[^/?&]+)|(?P<id>[0-9a-f]{32}))\.json'
 
     def _real_extract(self, url):
@@ -61,4 +64,4 @@ class UplynkPreplayIE(InfoExtractor):
         session_id = preplay.get('sid')
         if session_id:
             content_url += '?pbs=' + session_id
-        return self.url_result(content_url, 'Uplynk')
+        return self._extract_uplynk_info(content_url)
diff --git a/youtube_dl/extractor/viceland.py b/youtube_dl/extractor/viceland.py
index c66e8eb95..f72294b51 100644
--- a/youtube_dl/extractor/viceland.py
+++ b/youtube_dl/extractor/viceland.py
@@ -25,6 +25,13 @@ class VicelandIE(AdobePass):
         'info_dict': {
             'id': '57608447973ee7705f6fbd4e',
             'ext': 'mp4',
+            'title': 'CYBERWAR (Trailer)',
+            'description': 'Tapping into the geopolitics of hacking and surveillance, Ben Makuch travels the world to meet with hackers, government officials, and dissidents to investigate the ecosystem of cyberwarfare.',
+            'age_limit': 14,
+            'timestamp': 1466008539,
+            'upload_date': '20160615',
+            'uploader_id': '11',
+            'uploader': 'Viceland',
         },
         'params': {
             # m3u8 download

From 6103f59095bd1e514e43b3f84f4633e27ee09b69 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 14 Aug 2016 19:08:35 +0100
Subject: [PATCH 5776/6219] [viceland] remove outdated comment

---
 youtube_dl/extractor/viceland.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/viceland.py b/youtube_dl/extractor/viceland.py
index f72294b51..0be8a792f 100644
--- a/youtube_dl/extractor/viceland.py
+++ b/youtube_dl/extractor/viceland.py
@@ -20,7 +20,6 @@ from ..utils import (
 class VicelandIE(AdobePass):
     _VALID_URL = r'https?://(?:www\.)?viceland\.com/[^/]+/video/[^/]+/(?P<id>[a-f0-9]+)'
     _TEST = {
-        # FIXME: fill the test after fixing delegation problem
         'url': 'https://www.viceland.com/en_us/video/cyberwar-trailer/57608447973ee7705f6fbd4e',
         'info_dict': {
             'id': '57608447973ee7705f6fbd4e',

From e811bcf8f820d92b6629920b7c3c5a902815e6d1 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 14 Aug 2016 20:12:53 +0100
Subject: [PATCH 5777/6219] [viceland] raise ExtractorError for errors other
 than HTTP 400

---
 youtube_dl/extractor/viceland.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/viceland.py b/youtube_dl/extractor/viceland.py
index 0be8a792f..814a72fa2 100644
--- a/youtube_dl/extractor/viceland.py
+++ b/youtube_dl/extractor/viceland.py
@@ -70,6 +70,7 @@ class VicelandIE(AdobePass):
             if isinstance(e.cause, compat_HTTPError) and e.cause.code == 400:
                 error = json.loads(e.cause.read().decode())
                 raise ExtractorError('%s said: %s' % (self.IE_NAME, error['details']), expected=True)
+            raise
 
         video_data = preplay['video']
         base = video_data['base']

From 7e60ce9cf7b104c15fcc4c495166dc57b950b987 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 14 Aug 2016 21:24:33 +0100
Subject: [PATCH 5778/6219] [adobepass] clear cache in case of pendingLogout
 errors

---
 youtube_dl/extractor/adobepass.py | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/adobepass.py b/youtube_dl/extractor/adobepass.py
index 4e59302ab..d315bfbc1 100644
--- a/youtube_dl/extractor/adobepass.py
+++ b/youtube_dl/extractor/adobepass.py
@@ -90,6 +90,9 @@ class AdobePass(InfoExtractor):
                     '_method': 'GET',
                     'requestor_id': requestor_id,
                 }), headers=mvpd_headers)
+            if '<pendingLogout' in short_authorize:
+                self._downloader.cache.store('mvpd', requestor_id, {})
+                return self._extract_mvpd_auth(url, video_id, requestor_id, resource)
             authn_token = unescapeHTML(xml_text(session, 'authnToken'))
             requestor_info['authn_token'] = authn_token
             self._downloader.cache.store('mvpd', requestor_id, requestor_info)
@@ -105,6 +108,9 @@ class AdobePass(InfoExtractor):
                     'mso_id': xml_text(authn_token, 'simpleTokenMsoID'),
                     'userMeta': '1',
                 }), headers=mvpd_headers)
+            if '<pendingLogout' in authorize:
+                self._downloader.cache.store('mvpd', requestor_id, {})
+                return self._extract_mvpd_auth(url, video_id, requestor_id, resource)
             authz_token = unescapeHTML(xml_text(authorize, 'authzToken'))
             requestor_info[guid] = authz_token
             self._downloader.cache.store('mvpd', requestor_id, requestor_info)
@@ -114,7 +120,7 @@ class AdobePass(InfoExtractor):
             'ap_23': xml_text(authn_token, 'simpleSamlSessionIndex'),
         })
 
-        return self._download_webpage(
+        short_authorize = self._download_webpage(
             self._SERVICE_PROVIDER_TEMPLATE % 'shortAuthorize',
             video_id, 'Retrieving Media Token', data=urlencode_postdata({
                 'authz_token': authz_token,
@@ -122,3 +128,7 @@ class AdobePass(InfoExtractor):
                 'session_guid': xml_text(authn_token, 'simpleTokenAuthenticationGuid'),
                 'hashed_guid': 'false',
             }), headers=mvpd_headers)
+        if '<pendingLogout' in short_authorize:
+            self._downloader.cache.store('mvpd', requestor_id, {})
+            return self._extract_mvpd_auth(url, video_id, requestor_id, resource)
+        return short_authorize
\ No newline at end of file

From 525e0316c046ebcb689ebc01c4a1c53cf289b514 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 14 Aug 2016 21:25:43 +0100
Subject: [PATCH 5779/6219] [adobepass] fix check for pendingLogout errors

---
 youtube_dl/extractor/adobepass.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/adobepass.py b/youtube_dl/extractor/adobepass.py
index d315bfbc1..cf3a15cbb 100644
--- a/youtube_dl/extractor/adobepass.py
+++ b/youtube_dl/extractor/adobepass.py
@@ -90,7 +90,7 @@ class AdobePass(InfoExtractor):
                     '_method': 'GET',
                     'requestor_id': requestor_id,
                 }), headers=mvpd_headers)
-            if '<pendingLogout' in short_authorize:
+            if '<pendingLogout' in session:
                 self._downloader.cache.store('mvpd', requestor_id, {})
                 return self._extract_mvpd_auth(url, video_id, requestor_id, resource)
             authn_token = unescapeHTML(xml_text(session, 'authnToken'))

From 24eb13b1c6889cc4399b854cde931e5a5c5f5346 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 14 Aug 2016 22:45:43 +0100
Subject: [PATCH 5780/6219] [uplynk,viceland] update tests and change uplynk
 extractors names

---
 youtube_dl/extractor/uplynk.py   | 3 +++
 youtube_dl/extractor/viceland.py | 2 +-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/uplynk.py b/youtube_dl/extractor/uplynk.py
index 4313bc9cb..ae529f690 100644
--- a/youtube_dl/extractor/uplynk.py
+++ b/youtube_dl/extractor/uplynk.py
@@ -11,6 +11,7 @@ from ..utils import (
 
 
 class UplynkIE(InfoExtractor):
+    IE_NAME = 'uplynk'
     _VALID_URL = r'https?://.*?\.uplynk\.com/(?P<path>ext/[0-9a-f]{32}/(?P<external_id>[^/?&]+)|(?P<id>[0-9a-f]{32}))\.(?:m3u8|json)(?:.*?\bpbs=(?P<session_id>[^&]+))?'
     _TEST = {
         'url': 'http://content.uplynk.com/e89eaf2ce9054aa89d92ddb2d817a52e.m3u8',
@@ -54,7 +55,9 @@ class UplynkIE(InfoExtractor):
 
 
 class UplynkPreplayIE(UplynkIE):
+    IE_NAME = 'uplynk:preplay'
     _VALID_URL = r'https?://.*?\.uplynk\.com/preplay2?/(?P<path>ext/[0-9a-f]{32}/(?P<external_id>[^/?&]+)|(?P<id>[0-9a-f]{32}))\.json'
+    _TEST = None
 
     def _real_extract(self, url):
         path, external_id, video_id = re.match(self._VALID_URL, url).groups()
diff --git a/youtube_dl/extractor/viceland.py b/youtube_dl/extractor/viceland.py
index 814a72fa2..da766d8db 100644
--- a/youtube_dl/extractor/viceland.py
+++ b/youtube_dl/extractor/viceland.py
@@ -36,7 +36,7 @@ class VicelandIE(AdobePass):
             # m3u8 download
             'skip_download': True,
         },
-        'add_ie': ['UplynkPreplay', 'Uplynk'],
+        'add_ie': ['UplynkPreplay'],
     }
 
     def _real_extract(self, url):

From 1a57b8c18c9bdaf5e231f2178499041446b57a3a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 15 Aug 2016 08:25:24 +0700
Subject: [PATCH 5781/6219] [zippcast] Remove extractor (Closes #10332)

ZippCast is shut down
---
 youtube_dl/extractor/extractors.py |  1 -
 youtube_dl/extractor/zippcast.py   | 94 ------------------------------
 2 files changed, 95 deletions(-)
 delete mode 100644 youtube_dl/extractor/zippcast.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index be96e34ba..15bc0a675 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1109,4 +1109,3 @@ from .zingmp3 import (
     ZingMp3SongIE,
     ZingMp3AlbumIE,
 )
-from .zippcast import ZippCastIE
diff --git a/youtube_dl/extractor/zippcast.py b/youtube_dl/extractor/zippcast.py
deleted file mode 100644
index de819376d..000000000
--- a/youtube_dl/extractor/zippcast.py
+++ /dev/null
@@ -1,94 +0,0 @@
-from __future__ import unicode_literals
-
-import re
-
-from .common import InfoExtractor
-from ..utils import (
-    determine_ext,
-    str_to_int,
-)
-
-
-class ZippCastIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?zippcast\.com/(?:video/|videoview\.php\?.*\bvplay=)(?P<id>[0-9a-zA-Z]+)'
-    _TESTS = [{
-        # m3u8, hq direct link
-        'url': 'http://www.zippcast.com/video/c9cfd5c7e44dbc29c81',
-        'md5': '5ea0263b5606866c4d6cda0fc5e8c6b6',
-        'info_dict': {
-            'id': 'c9cfd5c7e44dbc29c81',
-            'ext': 'mp4',
-            'title': '[Vinesauce] Vinny - Digital Space Traveler',
-            'description': 'Muted on youtube, but now uploaded in it\'s original form.',
-            'thumbnail': 're:^https?://.*\.jpg$',
-            'uploader': 'vinesauce',
-            'view_count': int,
-            'categories': ['Entertainment'],
-            'tags': list,
-        },
-    }, {
-        # f4m, lq ipod direct link
-        'url': 'http://www.zippcast.com/video/b79c0a233e9c6581775',
-        'only_matching': True,
-    }, {
-        'url': 'http://www.zippcast.com/videoview.php?vplay=c9cfd5c7e44dbc29c81&auto=no',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        webpage = self._download_webpage(
-            'http://www.zippcast.com/video/%s' % video_id, video_id)
-
-        formats = []
-        video_url = self._search_regex(
-            r'<source[^>]+src=(["\'])(?P<url>.+?)\1', webpage,
-            'video url', default=None, group='url')
-        if video_url:
-            formats.append({
-                'url': video_url,
-                'format_id': 'http',
-                'preference': 0,  # direct link is almost always of worse quality
-            })
-        src_url = self._search_regex(
-            r'src\s*:\s*(?:escape\()?(["\'])(?P<url>http://.+?)\1',
-            webpage, 'src', default=None, group='url')
-        ext = determine_ext(src_url)
-        if ext == 'm3u8':
-            formats.extend(self._extract_m3u8_formats(
-                src_url, video_id, 'mp4', entry_protocol='m3u8_native',
-                m3u8_id='hls', fatal=False))
-        elif ext == 'f4m':
-            formats.extend(self._extract_f4m_formats(
-                src_url, video_id, f4m_id='hds', fatal=False))
-        self._sort_formats(formats)
-
-        title = self._og_search_title(webpage)
-        description = self._og_search_description(webpage) or self._html_search_meta(
-            'description', webpage)
-        uploader = self._search_regex(
-            r'<a[^>]+href="https?://[^/]+/profile/[^>]+>([^<]+)</a>',
-            webpage, 'uploader', fatal=False)
-        thumbnail = self._og_search_thumbnail(webpage)
-        view_count = str_to_int(self._search_regex(
-            r'>([\d,.]+) views!', webpage, 'view count', fatal=False))
-
-        categories = re.findall(
-            r'<a[^>]+href="https?://[^/]+/categories/[^"]+">([^<]+),?<',
-            webpage)
-        tags = re.findall(
-            r'<a[^>]+href="https?://[^/]+/search/tags/[^"]+">([^<]+),?<',
-            webpage)
-
-        return {
-            'id': video_id,
-            'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
-            'uploader': uploader,
-            'view_count': view_count,
-            'categories': categories,
-            'tags': tags,
-            'formats': formats,
-        }

From b6c4e36728e8f60ae7f4910a9b7027a2b702e8dc Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 15 Aug 2016 13:29:01 +0800
Subject: [PATCH 5782/6219] [jwplatform] Parse video_id from JWPlayer data

And remove a mysterious comma from 115c65793af4c56c8f1986d2640105fc7e760c13
---
 youtube_dl/extractor/jwplatform.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/jwplatform.py b/youtube_dl/extractor/jwplatform.py
index 2a499bb77..ce3126943 100644
--- a/youtube_dl/extractor/jwplatform.py
+++ b/youtube_dl/extractor/jwplatform.py
@@ -30,7 +30,7 @@ class JWPlatformBaseIE(InfoExtractor):
         return self._parse_jwplayer_data(
             jwplayer_data, video_id, *args, **kwargs)
 
-    def _parse_jwplayer_data(self, jwplayer_data, video_id, require_title=True, m3u8_id=None, rtmp_params=None, base_url=None):
+    def _parse_jwplayer_data(self, jwplayer_data, video_id=None, require_title=True, m3u8_id=None, rtmp_params=None, base_url=None):
         # JWPlayer backward compatibility: flattened playlists
         # https://github.com/jwplayer/jwplayer/blob/v7.4.3/src/js/api/config.js#L81-L96
         if 'playlist' not in jwplayer_data:
@@ -43,6 +43,8 @@ class JWPlatformBaseIE(InfoExtractor):
             if 'sources' not in video_data:
                 video_data['sources'] = [video_data]
 
+            this_video_id = video_id or video_data['mediaid']
+
             formats = []
             for source in video_data['sources']:
                 source_url = self._proto_relative_url(source['file'])
@@ -52,7 +54,7 @@ class JWPlatformBaseIE(InfoExtractor):
                 ext = mimetype2ext(source_type) or determine_ext(source_url)
                 if source_type == 'hls' or ext == 'm3u8':
                     formats.extend(self._extract_m3u8_formats(
-                        source_url, video_id, 'mp4', 'm3u8_native', m3u8_id=m3u8_id, fatal=False))
+                        source_url, this_video_id, 'mp4', 'm3u8_native', m3u8_id=m3u8_id, fatal=False))
                 # https://github.com/jwplayer/jwplayer/blob/master/src/js/providers/default.js#L67
                 elif source_type.startswith('audio') or ext in ('oga', 'aac', 'mp3', 'mpeg', 'vorbis'):
                     formats.append({
@@ -68,7 +70,7 @@ class JWPlatformBaseIE(InfoExtractor):
                         'ext': ext,
                     }
                     if source_url.startswith('rtmp'):
-                        a_format['ext'] = 'flv',
+                        a_format['ext'] = 'flv'
 
                         # See com/longtailvideo/jwplayer/media/RTMPMediaProvider.as
                         # of jwplayer.flash.swf
@@ -95,7 +97,7 @@ class JWPlatformBaseIE(InfoExtractor):
                         })
 
             entries.append({
-                'id': video_id,
+                'id': this_video_id,
                 'title': video_data['title'] if require_title else video_data.get('title'),
                 'description': video_data.get('description'),
                 'thumbnail': self._proto_relative_url(video_data.get('image')),

From 5c2d08722139118d8de27d43d6210e18ab1da9d5 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 15 Aug 2016 13:31:08 +0800
Subject: [PATCH 5783/6219] [sendtonews] Fix extraction

---
 youtube_dl/extractor/sendtonews.py | 103 +++++++++++++++--------------
 1 file changed, 53 insertions(+), 50 deletions(-)

diff --git a/youtube_dl/extractor/sendtonews.py b/youtube_dl/extractor/sendtonews.py
index 1c636f672..2dbe490bb 100644
--- a/youtube_dl/extractor/sendtonews.py
+++ b/youtube_dl/extractor/sendtonews.py
@@ -4,33 +4,43 @@ from __future__ import unicode_literals
 import re
 
 from .jwplatform import JWPlatformBaseIE
-from ..compat import compat_parse_qs
 from ..utils import (
-    ExtractorError,
-    parse_duration,
+    float_or_none,
+    parse_iso8601,
+    update_url_query,
 )
 
 
 class SendtoNewsIE(JWPlatformBaseIE):
-    _VALID_URL = r'https?://embed\.sendtonews\.com/player/embed\.php\?(?P<query>[^#]+)'
+    _VALID_URL = r'https?://embed\.sendtonews\.com/player2/embedplayer\.php\?.*\bSC=(?P<id>[0-9A-Za-z-]+)'
 
     _TEST = {
         # From http://cleveland.cbslocal.com/2016/05/16/indians-score-season-high-15-runs-in-blowout-win-over-reds-rapid-reaction/
-        'url': 'http://embed.sendtonews.com/player/embed.php?SK=GxfCe0Zo7D&MK=175909&PK=5588&autoplay=on&sound=yes',
+        'url': 'http://embed.sendtonews.com/player2/embedplayer.php?SC=GxfCe0Zo7D-175909-5588&type=single&autoplay=on&sound=YES',
         'info_dict': {
-            'id': 'GxfCe0Zo7D-175909-5588',
-            'ext': 'mp4',
-            'title': 'Recap: CLE 15, CIN 6',
-            'description': '5/16/16: Indians\' bats explode for 15 runs in a win',
-            'duration': 49,
+            'id': 'GxfCe0Zo7D-175909-5588'
         },
+        'playlist_count': 9,
+        # test the first video only to prevent lengthy tests
+        'playlist': [{
+            'info_dict': {
+                'id': '198180',
+                'ext': 'mp4',
+                'title': 'Recap: CLE 5, LAA 4',
+                'description': '8/14/16: Naquin, Almonte lead Indians in 5-4 win',
+                'duration': 57.343,
+                'thumbnail': 're:https?://.*\.jpg$',
+                'upload_date': '20160815',
+                'timestamp': 1471221961,
+            },
+        }],
         'params': {
             # m3u8 download
             'skip_download': True,
         },
     }
 
-    _URL_TEMPLATE = '//embed.sendtonews.com/player/embed.php?SK=%s&MK=%s&PK=%s'
+    _URL_TEMPLATE = '//embed.sendtonews.com/player2/embedplayer.php?SC=%s'
 
     @classmethod
     def _extract_url(cls, webpage):
@@ -39,48 +49,41 @@ class SendtoNewsIE(JWPlatformBaseIE):
                 .*\bSC=(?P<SC>[0-9a-zA-Z-]+).*
             \1>''', webpage)
         if mobj:
-            sk, mk, pk = mobj.group('SC').split('-')
-            return cls._URL_TEMPLATE % (sk, mk, pk)
+            sc = mobj.group('SC')
+            return cls._URL_TEMPLATE % sc
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        params = compat_parse_qs(mobj.group('query'))
+        playlist_id = self._match_id(url)
 
-        if 'SK' not in params or 'MK' not in params or 'PK' not in params:
-            raise ExtractorError('Invalid URL', expected=True)
+        data_url = update_url_query(
+            url.replace('embedplayer.php', 'data_read.php'),
+            {'cmd': 'loadInitial'})
+        playlist_data = self._download_json(data_url, playlist_id)
 
-        video_id = '-'.join([params['SK'][0], params['MK'][0], params['PK'][0]])
+        entries = []
+        for video in playlist_data['playlistData'][0]:
+            info_dict = self._parse_jwplayer_data(
+                video['jwconfiguration'],
+                require_title=False, rtmp_params={'no_resume': True})
 
-        webpage = self._download_webpage(url, video_id)
+            thumbnails = []
+            if video.get('thumbnailUrl'):
+                thumbnails.append({
+                    'id': 'normal',
+                    'url': video['thumbnailUrl'],
+                })
+            if video.get('smThumbnailUrl'):
+                thumbnails.append({
+                    'id': 'small',
+                    'url': video['smThumbnailUrl'],
+                })
+            info_dict.update({
+                'title': video['S_headLine'],
+                'description': video.get('S_fullStory'),
+                'thumbnails': thumbnails,
+                'duration': float_or_none(video.get('SM_length')),
+                'timestamp': parse_iso8601(video.get('S_sysDate'), delimiter=' '),
+            })
+            entries.append(info_dict)
 
-        jwplayer_data_str = self._search_regex(
-            r'jwplayer\("[^"]+"\)\.setup\((.+?)\);', webpage, 'JWPlayer data')
-        js_vars = {
-            'w': 1024,
-            'h': 768,
-            'modeVar': 'html5',
-        }
-        for name, val in js_vars.items():
-            js_val = '%d' % val if isinstance(val, int) else '"%s"' % val
-            jwplayer_data_str = jwplayer_data_str.replace(':%s,' % name, ':%s,' % js_val)
-
-        info_dict = self._parse_jwplayer_data(
-            self._parse_json(jwplayer_data_str, video_id),
-            video_id, require_title=False, rtmp_params={'no_resume': True})
-
-        title = self._html_search_regex(
-            r'<div[^>]+class="embedTitle">([^<]+)</div>', webpage, 'title')
-        description = self._html_search_regex(
-            r'<div[^>]+class="embedSubTitle">([^<]+)</div>', webpage,
-            'description', fatal=False)
-        duration = parse_duration(self._html_search_regex(
-            r'<div[^>]+class="embedDetails">([0-9:]+)', webpage,
-            'duration', fatal=False))
-
-        info_dict.update({
-            'title': title,
-            'description': description,
-            'duration': duration,
-        })
-
-        return info_dict
+        return self.playlist_result(entries, playlist_id)

From 760845ce9965b57484f232a162b9bb4ad3a505a7 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 15 Aug 2016 13:37:37 +0800
Subject: [PATCH 5784/6219] [cbslocal] Adapt to SendtoNewsIE

---
 youtube_dl/extractor/cbslocal.py | 18 ++++++------------
 1 file changed, 6 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/cbslocal.py b/youtube_dl/extractor/cbslocal.py
index 008c5fe32..4bcd104af 100644
--- a/youtube_dl/extractor/cbslocal.py
+++ b/youtube_dl/extractor/cbslocal.py
@@ -41,13 +41,8 @@ class CBSLocalIE(AnvatoIE):
         'url': 'http://cleveland.cbslocal.com/2016/05/16/indians-score-season-high-15-runs-in-blowout-win-over-reds-rapid-reaction/',
         'info_dict': {
             'id': 'GxfCe0Zo7D-175909-5588',
-            'ext': 'mp4',
-            'title': 'Recap: CLE 15, CIN 6',
-            'description': '5/16/16: Indians\' bats explode for 15 runs in a win',
-            'upload_date': '20160516',
-            'timestamp': 1463433840,
-            'duration': 49,
         },
+        'playlist_count': 9,
         'params': {
             # m3u8 download
             'skip_download': True,
@@ -60,12 +55,11 @@ class CBSLocalIE(AnvatoIE):
 
         sendtonews_url = SendtoNewsIE._extract_url(webpage)
         if sendtonews_url:
-            info_dict = {
-                '_type': 'url_transparent',
-                'url': compat_urlparse.urljoin(url, sendtonews_url),
-            }
-        else:
-            info_dict = self._extract_anvato_videos(webpage, display_id)
+            return self.url_result(
+                compat_urlparse.urljoin(url, sendtonews_url),
+                ie=SendtoNewsIE.ie_key())
+
+        info_dict = self._extract_anvato_videos(webpage, display_id)
 
         time_str = self._html_search_regex(
             r'class="entry-date">([^<]+)<', webpage, 'released date', fatal=False)

From 6d8ec8c3b7381c40afd89f9c118ae770997703d0 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 15 Aug 2016 13:39:43 +0800
Subject: [PATCH 5785/6219] [ChangeLog] Update for CBSLocal and related changes

---
 ChangeLog | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index d04c5fc2a..32504dab5 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,9 @@
 version <unreleased>
 
 Extractors
+* [cbslocal] Fix extraction for SendtoNews-based videos
+* [sendtonews] Fix extraction
+* [jwplatform] Now can parse video_id from JWPlayer data
 * [xiami] Fix extraction (#10342)
 
 

From 69eb4d699fe3f6d84acc7882e427e661040faecb Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 15 Aug 2016 20:29:22 +0800
Subject: [PATCH 5786/6219] [cbsnews] Remove invalid tests. CBS Live videos
 gets deleted soon.

---
 youtube_dl/extractor/cbsnews.py | 14 ++++----------
 1 file changed, 4 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/cbsnews.py b/youtube_dl/extractor/cbsnews.py
index 9328e3e20..9d3b75526 100644
--- a/youtube_dl/extractor/cbsnews.py
+++ b/youtube_dl/extractor/cbsnews.py
@@ -70,7 +70,8 @@ class CBSNewsLiveVideoIE(InfoExtractor):
     IE_DESC = 'CBS News Live Videos'
     _VALID_URL = r'https?://(?:www\.)?cbsnews\.com/live/video/(?P<id>[\da-z_-]+)'
 
-    _TESTS = [{
+    # Live videos get deleted soon. See http://www.cbsnews.com/live/ for the latest examples
+    _TEST = {
         'url': 'http://www.cbsnews.com/live/video/clinton-sanders-prepare-to-face-off-in-nh/',
         'info_dict': {
             'id': 'clinton-sanders-prepare-to-face-off-in-nh',
@@ -78,15 +79,8 @@ class CBSNewsLiveVideoIE(InfoExtractor):
             'title': 'Clinton, Sanders Prepare To Face Off In NH',
             'duration': 334,
         },
-        'skip': 'Video gone, redirected to http://www.cbsnews.com/live/',
-    }, {
-        'url': 'http://www.cbsnews.com/live/video/video-shows-intense-paragliding-accident/',
-        'info_dict': {
-            'id': 'video-shows-intense-paragliding-accident',
-            'ext': 'flv',
-            'title': 'Video Shows Intense Paragliding Accident',
-        },
-    }]
+        'skip': 'Video gone',
+    }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From bf90c46790bac92e8a61ee0514cf3c41a8c048e9 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 15 Aug 2016 16:33:35 +0100
Subject: [PATCH 5787/6219] [fxnetworks] Add new extractor(closes #9462)

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/fxnetworks.py | 49 ++++++++++++++++++++++++++++++
 2 files changed, 50 insertions(+)
 create mode 100644 youtube_dl/extractor/fxnetworks.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 15bc0a675..07928c530 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -287,6 +287,7 @@ from .freevideo import FreeVideoIE
 from .funimation import FunimationIE
 from .funnyordie import FunnyOrDieIE
 from .fusion import FusionIE
+from .fxnetworks import FXNetworksIE
 from .gameinformer import GameInformerIE
 from .gameone import (
     GameOneIE,
diff --git a/youtube_dl/extractor/fxnetworks.py b/youtube_dl/extractor/fxnetworks.py
new file mode 100644
index 000000000..70bc186a3
--- /dev/null
+++ b/youtube_dl/extractor/fxnetworks.py
@@ -0,0 +1,49 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .adobepass import AdobePass
+from ..utils import (
+    update_url_query,
+    extract_attributes,
+    parse_age_limit,
+    smuggle_url,
+)
+
+
+class FXNetworksIE(AdobePass):
+    _VALID_URL = r'https?://(?:www\.)?fxnetworks\.com/video/(?P<id>\d+)'
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        video_data = extract_attributes(self._search_regex(
+            r'(<a.+?rel="http://link\.theplatform\.com/s/.+?</a>)', webpage, 'video data'))
+        player_type = self._search_regex(r'playerType\s*=\s*[\'"]([^\'"]+)', webpage, 'player type', fatal=False)
+        release_url = video_data['rel']
+        title = video_data['data-title']
+        rating = video_data.get('data-rating')
+        query = {
+            'mbr': 'true',
+        }
+        if player_type == 'movies':
+            query.update({
+                'manifest': 'm3u',
+            })
+        else:
+            query.update({
+                'switch': 'http',
+            })
+        if video_data.get('data-req-auth') == '1':
+            resource = self._get_mvpd_resource(
+                video_data['data-channel'], title,
+                video_data.get('data-guid'), rating)
+            query['auth'] = self._extract_mvpd_auth(url, video_id, 'fx', resource)
+
+        return {
+            '_type': 'url_transparent',
+            'id': video_id,
+            'url': smuggle_url(update_url_query(release_url, query), {'force_smil_url': True}),
+            'thumbnail': video_data.get('data-large-thumb'),
+            'age_limit': parse_age_limit(rating),
+            'ie_key': 'ThePlatform',
+        }

From cbef4d5c9ff5013d0c10b960e1690805724120cd Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 15 Aug 2016 17:10:45 +0100
Subject: [PATCH 5788/6219] [fxnetworks] add test and check geo restriction

---
 youtube_dl/extractor/fxnetworks.py | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/youtube_dl/extractor/fxnetworks.py b/youtube_dl/extractor/fxnetworks.py
index 70bc186a3..940e7427c 100644
--- a/youtube_dl/extractor/fxnetworks.py
+++ b/youtube_dl/extractor/fxnetworks.py
@@ -12,10 +12,27 @@ from ..utils import (
 
 class FXNetworksIE(AdobePass):
     _VALID_URL = r'https?://(?:www\.)?fxnetworks\.com/video/(?P<id>\d+)'
+    _TEST = {
+        'url': 'http://www.fxnetworks.com/video/719841347694',
+        'md5': '1447d4722e42ebca19e5232ab93abb22',
+        'info_dict': {
+            'id': '719841347694',
+            'ext': 'mp4',
+            'title': 'Vanpage',
+            'description': 'F*ck settling down. You\'re the Worst returns for an all new season August 31st on FXX.',
+            'age_limit': 14,
+            'uploader': 'NEWA-FNG-FX',
+            'upload_date': '20160706',
+            'timestamp': 1467844741,
+        },
+        'add_ie': ['ThePlatform'],
+    }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
+        if 'The content you are trying to access is not available in your region.' in webpage:
+            self.raise_geo_restricted()
         video_data = extract_attributes(self._search_regex(
             r'(<a.+?rel="http://link\.theplatform\.com/s/.+?</a>)', webpage, 'video data'))
         player_type = self._search_regex(r'playerType\s*=\s*[\'"]([^\'"]+)', webpage, 'player type', fatal=False)
@@ -42,6 +59,7 @@ class FXNetworksIE(AdobePass):
         return {
             '_type': 'url_transparent',
             'id': video_id,
+            'title': title,
             'url': smuggle_url(update_url_query(release_url, query), {'force_smil_url': True}),
             'thumbnail': video_data.get('data-large-thumb'),
             'age_limit': parse_age_limit(rating),

From 818ac213eb80e18f472ecdf2406569bafd4cccaf Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 15 Aug 2016 21:36:34 +0100
Subject: [PATCH 5789/6219] [adobepass] add IE suffix to the extractor and
 remove duplicate constant

---
 youtube_dl/extractor/adobepass.py          | 2 +-
 youtube_dl/extractor/fxnetworks.py         | 4 ++--
 youtube_dl/extractor/nationalgeographic.py | 6 +++---
 youtube_dl/extractor/syfy.py               | 4 ++--
 youtube_dl/extractor/theplatform.py        | 5 ++---
 youtube_dl/extractor/viceland.py           | 4 ++--
 6 files changed, 12 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/adobepass.py b/youtube_dl/extractor/adobepass.py
index cf3a15cbb..2c9f8817b 100644
--- a/youtube_dl/extractor/adobepass.py
+++ b/youtube_dl/extractor/adobepass.py
@@ -13,7 +13,7 @@ from ..utils import (
 )
 
 
-class AdobePass(InfoExtractor):
+class AdobePassIE(InfoExtractor):
     _SERVICE_PROVIDER_TEMPLATE = 'https://sp.auth.adobe.com/adobe-services/%s'
     _USER_AGENT = 'Mozilla/5.0 (X11; Linux i686; rv:47.0) Gecko/20100101 Firefox/47.0'
 
diff --git a/youtube_dl/extractor/fxnetworks.py b/youtube_dl/extractor/fxnetworks.py
index 940e7427c..3ec3b0b46 100644
--- a/youtube_dl/extractor/fxnetworks.py
+++ b/youtube_dl/extractor/fxnetworks.py
@@ -1,7 +1,7 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-from .adobepass import AdobePass
+from .adobepass import AdobePassIE
 from ..utils import (
     update_url_query,
     extract_attributes,
@@ -10,7 +10,7 @@ from ..utils import (
 )
 
 
-class FXNetworksIE(AdobePass):
+class FXNetworksIE(AdobePassIE):
     _VALID_URL = r'https?://(?:www\.)?fxnetworks\.com/video/(?P<id>\d+)'
     _TEST = {
         'url': 'http://www.fxnetworks.com/video/719841347694',
diff --git a/youtube_dl/extractor/nationalgeographic.py b/youtube_dl/extractor/nationalgeographic.py
index 890e8d5bc..1dcf27afe 100644
--- a/youtube_dl/extractor/nationalgeographic.py
+++ b/youtube_dl/extractor/nationalgeographic.py
@@ -3,7 +3,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from .theplatform import ThePlatformIE
+from .adobepass import AdobePassIE
 from ..utils import (
     smuggle_url,
     url_basename,
@@ -65,7 +65,7 @@ class NationalGeographicVideoIE(InfoExtractor):
         }
 
 
-class NationalGeographicIE(ThePlatformIE):
+class NationalGeographicIE(AdobePassIE):
     IE_NAME = 'natgeo'
     _VALID_URL = r'https?://channel\.nationalgeographic\.com/(?:wild/)?[^/]+/(?:videos|episodes)/(?P<id>[^/?]+)'
 
@@ -131,7 +131,7 @@ class NationalGeographicIE(ThePlatformIE):
         }
 
 
-class NationalGeographicEpisodeGuideIE(ThePlatformIE):
+class NationalGeographicEpisodeGuideIE(InfoExtractor):
     IE_NAME = 'natgeo:episodeguide'
     _VALID_URL = r'https?://channel\.nationalgeographic\.com/(?:wild/)?(?P<id>[^/]+)/episode-guide'
     _TESTS = [
diff --git a/youtube_dl/extractor/syfy.py b/youtube_dl/extractor/syfy.py
index 764287a64..cc81f6003 100644
--- a/youtube_dl/extractor/syfy.py
+++ b/youtube_dl/extractor/syfy.py
@@ -1,13 +1,13 @@
 from __future__ import unicode_literals
 
-from .theplatform import ThePlatformIE
+from .adobepass import AdobePassIE
 from ..utils import (
     update_url_query,
     smuggle_url,
 )
 
 
-class SyfyIE(ThePlatformIE):
+class SyfyIE(AdobePassIE):
     _VALID_URL = r'https?://www\.syfy\.com/(?:[^/]+/)?videos/(?P<id>[^/?#]+)'
     _TESTS = [{
         'url': 'http://www.syfy.com/theinternetruinedmylife/videos/the-internet-ruined-my-life-season-1-trailer',
diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index 108ddd3a9..eda899497 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -9,7 +9,7 @@ import hashlib
 
 
 from .once import OnceIE
-from .adobepass import AdobePass
+from .adobepass import AdobePassIE
 from ..compat import (
     compat_parse_qs,
     compat_urllib_parse_urlparse,
@@ -93,7 +93,7 @@ class ThePlatformBaseIE(OnceIE):
         return self._parse_theplatform_metadata(info)
 
 
-class ThePlatformIE(ThePlatformBaseIE, AdobePass):
+class ThePlatformIE(ThePlatformBaseIE, AdobePassIE):
     _VALID_URL = r'''(?x)
         (?:https?://(?:link|player)\.theplatform\.com/[sp]/(?P<provider_id>[^/]+)/
            (?:(?:(?:[^/]+/)+select/)?(?P<media>media/(?:guid/\d+/)?)|(?P<config>(?:[^/\?]+/(?:swf|config)|onsite)/select/))?
@@ -164,7 +164,6 @@ class ThePlatformIE(ThePlatformBaseIE, AdobePass):
         'url': 'http://player.theplatform.com/p/NnzsPC/onsite_universal/select/media/guid/2410887629/2928790?fwsitesection=nbc_the_blacklist_video_library&autoPlay=true&carouselID=137781',
         'only_matching': True,
     }]
-    _SERVICE_PROVIDER_TEMPLATE = 'https://sp.auth.adobe.com/adobe-services/%s'
 
     @classmethod
     def _extract_urls(cls, webpage):
diff --git a/youtube_dl/extractor/viceland.py b/youtube_dl/extractor/viceland.py
index da766d8db..8742b607a 100644
--- a/youtube_dl/extractor/viceland.py
+++ b/youtube_dl/extractor/viceland.py
@@ -5,7 +5,7 @@ import time
 import hashlib
 import json
 
-from .adobepass import AdobePass
+from .adobepass import AdobePassIE
 from ..compat import compat_HTTPError
 from ..utils import (
     int_or_none,
@@ -17,7 +17,7 @@ from ..utils import (
 )
 
 
-class VicelandIE(AdobePass):
+class VicelandIE(AdobePassIE):
     _VALID_URL = r'https?://(?:www\.)?viceland\.com/[^/]+/video/[^/]+/(?P<id>[a-f0-9]+)'
     _TEST = {
         'url': 'https://www.viceland.com/en_us/video/cyberwar-trailer/57608447973ee7705f6fbd4e',

From 254e64a20aa37a033cb200bc6f1aa9daf57eead8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 16 Aug 2016 04:36:23 +0700
Subject: [PATCH 5790/6219] [bbc:playlist] Add support for pagination (Closes
 #10349)

---
 youtube_dl/extractor/bbc.py | 48 ++++++++++++++++++++++++++++++++-----
 1 file changed, 42 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 83e6d024c..16a97a76d 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -2,6 +2,7 @@
 from __future__ import unicode_literals
 
 import re
+import itertools
 
 from .common import InfoExtractor
 from ..utils import (
@@ -17,6 +18,7 @@ from ..utils import (
 from ..compat import (
     compat_etree_fromstring,
     compat_HTTPError,
+    compat_urlparse,
 )
 
 
@@ -1056,19 +1058,35 @@ class BBCCoUkArticleIE(InfoExtractor):
 
 
 class BBCCoUkPlaylistBaseIE(InfoExtractor):
+    def _entries(self, webpage, url, playlist_id):
+        single_page = 'page' in compat_urlparse.parse_qs(
+            compat_urlparse.urlparse(url).query)
+        for page_num in itertools.count(2):
+            for video_id in re.findall(
+                    self._VIDEO_ID_TEMPLATE % BBCCoUkIE._ID_REGEX, webpage):
+                yield self.url_result(
+                    self._URL_TEMPLATE % video_id, BBCCoUkIE.ie_key())
+            if single_page:
+                return
+            next_page = self._search_regex(
+                r'<li[^>]+class=(["\'])pagination_+next\1[^>]*><a[^>]+href=(["\'])(?P<url>(?:(?!\2).)+)\2',
+                webpage, 'next page url', default=None, group='url')
+            if not next_page:
+                break
+            webpage = self._download_webpage(
+                compat_urlparse.urljoin(url, next_page), playlist_id,
+                'Downloading page %d' % page_num, page_num)
+
     def _real_extract(self, url):
         playlist_id = self._match_id(url)
 
         webpage = self._download_webpage(url, playlist_id)
 
-        entries = [
-            self.url_result(self._URL_TEMPLATE % video_id, BBCCoUkIE.ie_key())
-            for video_id in re.findall(
-                self._VIDEO_ID_TEMPLATE % BBCCoUkIE._ID_REGEX, webpage)]
-
         title, description = self._extract_title_and_description(webpage)
 
-        return self.playlist_result(entries, playlist_id, title, description)
+        return self.playlist_result(
+            self._entries(webpage, url, playlist_id),
+            playlist_id, title, description)
 
 
 class BBCCoUkIPlayerPlaylistIE(BBCCoUkPlaylistBaseIE):
@@ -1094,6 +1112,24 @@ class BBCCoUkIPlayerPlaylistIE(BBCCoUkPlaylistBaseIE):
             'description': 'md5:683e901041b2fe9ba596f2ab04c4dbe7',
         },
         'playlist_mincount': 10,
+    }, {
+        # explicit page
+        'url': 'http://www.bbc.co.uk/programmes/b00mfl7n/clips?page=1',
+        'info_dict': {
+            'id': 'b00mfl7n',
+            'title': 'Bohemian Icons',
+            'description': 'md5:683e901041b2fe9ba596f2ab04c4dbe7',
+        },
+        'playlist_mincount': 24,
+    }, {
+        # all pages
+        'url': 'http://www.bbc.co.uk/programmes/b00mfl7n/clips',
+        'info_dict': {
+            'id': 'b00mfl7n',
+            'title': 'Bohemian Icons',
+            'description': 'md5:683e901041b2fe9ba596f2ab04c4dbe7',
+        },
+        'playlist_mincount': 142,
     }]
 
     def _extract_title_and_description(self, webpage):

From 4f640f28901be8a3ce57e77ead404d751e36d208 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 16 Aug 2016 04:43:10 +0700
Subject: [PATCH 5791/6219] [bbc:playlist] Fix tests

---
 youtube_dl/extractor/bbc.py | 36 ++++++++++++++++++------------------
 1 file changed, 18 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 16a97a76d..deb9cc1c0 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -1112,24 +1112,6 @@ class BBCCoUkIPlayerPlaylistIE(BBCCoUkPlaylistBaseIE):
             'description': 'md5:683e901041b2fe9ba596f2ab04c4dbe7',
         },
         'playlist_mincount': 10,
-    }, {
-        # explicit page
-        'url': 'http://www.bbc.co.uk/programmes/b00mfl7n/clips?page=1',
-        'info_dict': {
-            'id': 'b00mfl7n',
-            'title': 'Bohemian Icons',
-            'description': 'md5:683e901041b2fe9ba596f2ab04c4dbe7',
-        },
-        'playlist_mincount': 24,
-    }, {
-        # all pages
-        'url': 'http://www.bbc.co.uk/programmes/b00mfl7n/clips',
-        'info_dict': {
-            'id': 'b00mfl7n',
-            'title': 'Bohemian Icons',
-            'description': 'md5:683e901041b2fe9ba596f2ab04c4dbe7',
-        },
-        'playlist_mincount': 142,
     }]
 
     def _extract_title_and_description(self, webpage):
@@ -1153,6 +1135,24 @@ class BBCCoUkPlaylistIE(BBCCoUkPlaylistBaseIE):
             'description': 'French thriller serial about a missing teenager.',
         },
         'playlist_mincount': 7,
+    }, {
+        # multipage playlist, explicit page
+        'url': 'http://www.bbc.co.uk/programmes/b00mfl7n/clips?page=1',
+        'info_dict': {
+            'id': 'b00mfl7n',
+            'title': 'Frozen Planet - Clips - BBC One',
+            'description': 'md5:65dcbf591ae628dafe32aa6c4a4a0d8c',
+        },
+        'playlist_mincount': 24,
+    }, {
+        # multipage playlist, all pages
+        'url': 'http://www.bbc.co.uk/programmes/b00mfl7n/clips',
+        'info_dict': {
+            'id': 'b00mfl7n',
+            'title': 'Frozen Planet - Clips - BBC One',
+            'description': 'md5:65dcbf591ae628dafe32aa6c4a4a0d8c',
+        },
+        'playlist_mincount': 142,
     }, {
         'url': 'http://www.bbc.co.uk/programmes/b05rcz9v/broadcasts/2016/06',
         'only_matching': True,

From fb64adcbd37a660da92687878831d08e82ae748c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 16 Aug 2016 04:45:21 +0700
Subject: [PATCH 5792/6219] [adobepass] PEP 8

---
 youtube_dl/extractor/adobepass.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/adobepass.py b/youtube_dl/extractor/adobepass.py
index 2c9f8817b..9e3a3e362 100644
--- a/youtube_dl/extractor/adobepass.py
+++ b/youtube_dl/extractor/adobepass.py
@@ -131,4 +131,4 @@ class AdobePassIE(InfoExtractor):
         if '<pendingLogout' in short_authorize:
             self._downloader.cache.store('mvpd', requestor_id, {})
             return self._extract_mvpd_auth(url, video_id, requestor_id, resource)
-        return short_authorize
\ No newline at end of file
+        return short_authorize

From b5ddee8c778219c4bc04ebba2ad90b372e67fa55 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 16 Aug 2016 13:43:33 +0100
Subject: [PATCH 5793/6219] [amcnetworks] Add new extractor

---
 youtube_dl/extractor/amcnetworks.py | 72 +++++++++++++++++++++++++++++
 youtube_dl/extractor/extractors.py  |  1 +
 2 files changed, 73 insertions(+)
 create mode 100644 youtube_dl/extractor/amcnetworks.py

diff --git a/youtube_dl/extractor/amcnetworks.py b/youtube_dl/extractor/amcnetworks.py
new file mode 100644
index 000000000..be9552541
--- /dev/null
+++ b/youtube_dl/extractor/amcnetworks.py
@@ -0,0 +1,72 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .theplatform import ThePlatformIE
+from ..utils import (
+    update_url_query,
+    parse_age_limit,
+)
+
+
+class AMCNetworksIE(ThePlatformIE):
+    _VALID_URL = r'https?://(?:www\.)?(?:amc|bbcamerica|ifc|wetv)\.com/(?:movies/|shows/[^/]+/(?:full-episodes/)?season-\d+/episode-\d+(?:-(?:[^/]+/)?|/))(?P<id>[^/?#]+)'
+    _TESTS = [{
+        'url': 'http://www.ifc.com/shows/maron/season-04/episode-01/step-1',
+        'md5': '',
+        'info_dict': {
+            'id': 's3MX01Nl4vPH',
+            'ext': 'mp4',
+            'title': 'Step 1',
+            'description': 'In denial about his current situation, Marc is reluctantly convinced by his friends to enter rehab. Starring Marc Maron and Constance Zimmer.',
+            'age_limit': 17,
+            'upload_date': '20160505',
+            'timestamp': 1462468831,
+            'uploader': 'AMCN',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }, {
+        'url': 'http://www.bbcamerica.com/shows/the-hunt/full-episodes/season-1/episode-01-the-hardest-challenge',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.amc.com/shows/preacher/full-episodes/season-01/episode-00/pilot',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.wetv.com/shows/million-dollar-matchmaker/season-01/episode-06-the-dumped-dj-and-shallow-hal',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.ifc.com/movies/chaos',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+        query = {
+            'mbr': 'true',
+            'manifest': 'm3u',
+        }
+        media_url = self._search_regex(r'window\.platformLinkURL\s*=\s*[\'"]([^\'"]+)', webpage, 'media url')
+        theplatform_metadata = self._download_theplatform_metadata(self._search_regex(
+            r'https?://link.theplatform.com/s/([^?]+)', media_url, 'theplatform_path'), display_id)
+        info = self._parse_theplatform_metadata(theplatform_metadata)
+        video_id = theplatform_metadata['pid']
+        title = theplatform_metadata['title']
+        rating = theplatform_metadata['ratings'][0]['rating']
+        auth_required = self._search_regex(r'window\.authRequired\s*=\s*(true|false);', webpage, 'auth required')
+        if auth_required == 'true':
+            requestor_id = self._search_regex(r'window\.requestor_id\s*=\s*[\'"]([^\'"]+)', webpage, 'requestor id')
+            resource = self._get_mvpd_resource(requestor_id, title, video_id, rating)
+            query['auth'] = self._extract_mvpd_auth(url, video_id, requestor_id, resource)
+        media_url = update_url_query(media_url, query)
+        formats, subtitles = self._extract_theplatform_smil(media_url, video_id)
+        self._sort_formats(formats)
+        info.update({
+            'id': video_id,
+            'subtiles': subtitles,
+            'formats': formats,
+            'age_limit': parse_age_limit(parse_age_limit(rating)),
+        })
+        return info
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 07928c530..a5e0805b2 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -29,6 +29,7 @@ from .aftonbladet import AftonbladetIE
 from .airmozilla import AirMozillaIE
 from .aljazeera import AlJazeeraIE
 from .alphaporno import AlphaPornoIE
+from .amcnetworks import AMCNetworksIE
 from .animeondemand import AnimeOnDemandIE
 from .anitube import AnitubeIE
 from .anysex import AnySexIE

From 837e56c8eefa725ca72feca9431050cdda571c57 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 16 Aug 2016 14:49:32 +0100
Subject: [PATCH 5794/6219] [amcnetworks] extract episode metadata

---
 youtube_dl/extractor/amcnetworks.py | 21 ++++++++++++++++++++-
 1 file changed, 20 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/amcnetworks.py b/youtube_dl/extractor/amcnetworks.py
index be9552541..26f46acb5 100644
--- a/youtube_dl/extractor/amcnetworks.py
+++ b/youtube_dl/extractor/amcnetworks.py
@@ -5,6 +5,7 @@ from .theplatform import ThePlatformIE
 from ..utils import (
     update_url_query,
     parse_age_limit,
+    int_or_none,
 )
 
 
@@ -16,7 +17,7 @@ class AMCNetworksIE(ThePlatformIE):
         'info_dict': {
             'id': 's3MX01Nl4vPH',
             'ext': 'mp4',
-            'title': 'Step 1',
+            'title': 'Maron - Season 4 - Step 1',
             'description': 'In denial about his current situation, Marc is reluctantly convinced by his friends to enter rehab. Starring Marc Maron and Constance Zimmer.',
             'age_limit': 17,
             'upload_date': '20160505',
@@ -69,4 +70,22 @@ class AMCNetworksIE(ThePlatformIE):
             'formats': formats,
             'age_limit': parse_age_limit(parse_age_limit(rating)),
         })
+        ns_keys = theplatform_metadata.get('$xmlns', {}).keys()
+        if ns_keys:
+            ns = list(ns_keys)[0]
+            series = theplatform_metadata.get(ns + '$show')
+            season_number = int_or_none(theplatform_metadata.get(ns + '$season'))
+            episode = theplatform_metadata.get(ns + '$episodeTitle')
+            episode_number = int_or_none(theplatform_metadata.get(ns + '$episode'))
+            if season_number:
+                title = 'Season %d - %s' % (season_number, title)
+            if series:
+                title = '%s - %s' % (series, title)
+            info.update({
+                'title': title,
+                'series': series,
+                'season_number': season_number,
+                'episode': episode,
+                'episode_number': episode_number,
+            })
         return info

From 70a2829fee4203ebeb399481304d289ff92adf29 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 16 Aug 2016 21:17:52 +0700
Subject: [PATCH 5795/6219] [xvideos] Fix HLS extraction (Closes #10356)

---
 youtube_dl/extractor/xvideos.py | 34 ++++++++++++++++-----------------
 1 file changed, 17 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/xvideos.py b/youtube_dl/extractor/xvideos.py
index 1dfe031ca..b2ef15119 100644
--- a/youtube_dl/extractor/xvideos.py
+++ b/youtube_dl/extractor/xvideos.py
@@ -42,24 +42,24 @@ class XVideosIE(InfoExtractor):
         video_url = compat_urllib_parse_unquote(self._search_regex(
             r'flv_url=(.+?)&', webpage, 'video URL', default=''))
         if video_url:
-            formats.append({'url': video_url})
+            formats.append({
+                'url': video_url,
+                'format_id': 'flv',
+            })
 
-        player_args = self._search_regex(
-            r'(?s)new\s+HTML5Player\((.+?)\)', webpage, ' html5 player', default=None)
-        if player_args:
-            for arg in player_args.split(','):
-                format_url = self._search_regex(
-                    r'(["\'])(?P<url>https?://.+?)\1', arg, 'url',
-                    default=None, group='url')
-                if not format_url:
-                    continue
-                ext = determine_ext(format_url)
-                if ext == 'mp4':
-                    formats.append({'url': format_url})
-                elif ext == 'm3u8':
-                    formats.extend(self._extract_m3u8_formats(
-                        format_url, video_id, 'mp4',
-                        entry_protocol='m3u8_native', m3u8_id='hls', fatal=False))
+        for kind, _, format_url in re.findall(
+                r'setVideo([^(]+)\((["\'])(http.+?)\2\)', webpage):
+            format_id = kind.lower()
+            if format_id == 'hls':
+                formats.extend(self._extract_m3u8_formats(
+                    format_url, video_id, 'mp4',
+                    entry_protocol='m3u8_native', m3u8_id='hls', fatal=False))
+            elif format_id in ('urllow', 'urlhigh'):
+                formats.append({
+                    'url': format_url,
+                    'format_id': '%s-%s' % (determine_ext(format_url, 'mp4'), format_id[3:]),
+                    'quality': -2 if format_id.endswith('low') else None,
+                })
 
         self._sort_formats(formats)
 

From 98affc1a482ab41466c76cfded41949c4db58f67 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 16 Aug 2016 21:20:15 +0700
Subject: [PATCH 5796/6219] [xvideos] Fix test

---
 youtube_dl/extractor/xvideos.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/xvideos.py b/youtube_dl/extractor/xvideos.py
index b2ef15119..30825daae 100644
--- a/youtube_dl/extractor/xvideos.py
+++ b/youtube_dl/extractor/xvideos.py
@@ -15,10 +15,10 @@ class XVideosIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?xvideos\.com/video(?P<id>[0-9]+)(?:.*)'
     _TEST = {
         'url': 'http://www.xvideos.com/video4588838/biker_takes_his_girl',
-        'md5': '4b46ae6ea5e6e9086e714d883313c0c9',
+        'md5': '14cea69fcb84db54293b1e971466c2e1',
         'info_dict': {
             'id': '4588838',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Biker Takes his Girl',
             'age_limit': 18,
         }

From 11f502fac145b4592f47c025ee8317fe44020db0 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 16 Aug 2016 16:19:36 +0100
Subject: [PATCH 5797/6219] [theplatform] extract subtitles with multiple
 formats from the metadata

---
 youtube_dl/extractor/theplatform.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index eda899497..23067e8c6 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -73,10 +73,10 @@ class ThePlatformBaseIE(OnceIE):
         if isinstance(captions, list):
             for caption in captions:
                 lang, src, mime = caption.get('lang', 'en'), caption.get('src'), caption.get('type')
-                subtitles[lang] = [{
+                subtitles.setdefault(lang, []).append({
                     'ext': mimetype2ext(mime),
                     'url': src,
-                }]
+                })
 
         return {
             'title': info['title'],

From 2cabee2a7d4c94aa2f4f2e84a3c68eb97cdf9cce Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 16 Aug 2016 16:20:07 +0100
Subject: [PATCH 5798/6219] [amcnetworks] fix typo

---
 youtube_dl/extractor/amcnetworks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/amcnetworks.py b/youtube_dl/extractor/amcnetworks.py
index 26f46acb5..c739d2c99 100644
--- a/youtube_dl/extractor/amcnetworks.py
+++ b/youtube_dl/extractor/amcnetworks.py
@@ -66,7 +66,7 @@ class AMCNetworksIE(ThePlatformIE):
         self._sort_formats(formats)
         info.update({
             'id': video_id,
-            'subtiles': subtitles,
+            'subtitles': subtitles,
             'formats': formats,
             'age_limit': parse_age_limit(parse_age_limit(rating)),
         })

From 53fef319f14896ce497d309f661ceb586d7b4d90 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 16 Aug 2016 16:21:04 +0100
Subject: [PATCH 5799/6219] [fxnetworks] extend _VALID_URL to support
 simpsonsworld.com

---
 youtube_dl/extractor/fxnetworks.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/fxnetworks.py b/youtube_dl/extractor/fxnetworks.py
index 3ec3b0b46..629897317 100644
--- a/youtube_dl/extractor/fxnetworks.py
+++ b/youtube_dl/extractor/fxnetworks.py
@@ -11,8 +11,8 @@ from ..utils import (
 
 
 class FXNetworksIE(AdobePassIE):
-    _VALID_URL = r'https?://(?:www\.)?fxnetworks\.com/video/(?P<id>\d+)'
-    _TEST = {
+    _VALID_URL = r'https?://(?:www\.)?(?:fxnetworks|simpsonsworld)\.com/video/(?P<id>\d+)'
+    _TESTS = [{
         'url': 'http://www.fxnetworks.com/video/719841347694',
         'md5': '1447d4722e42ebca19e5232ab93abb22',
         'info_dict': {
@@ -26,7 +26,10 @@ class FXNetworksIE(AdobePassIE):
             'timestamp': 1467844741,
         },
         'add_ie': ['ThePlatform'],
-    }
+    }, {
+        'url': 'http://www.simpsonsworld.com/video/716094019682',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -35,7 +38,7 @@ class FXNetworksIE(AdobePassIE):
             self.raise_geo_restricted()
         video_data = extract_attributes(self._search_regex(
             r'(<a.+?rel="http://link\.theplatform\.com/s/.+?</a>)', webpage, 'video data'))
-        player_type = self._search_regex(r'playerType\s*=\s*[\'"]([^\'"]+)', webpage, 'player type', fatal=False)
+        player_type = self._search_regex(r'playerType\s*=\s*[\'"]([^\'"]+)', webpage, 'player type', default=None)
         release_url = video_data['rel']
         title = video_data['data-title']
         rating = video_data.get('data-rating')

From 6e7e4a6edf6c4ffd56d908ade7f0bfe2bff738b8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 17 Aug 2016 00:19:43 +0700
Subject: [PATCH 5800/6219] [mtg] Add support for viafree URLs (#10358)

---
 youtube_dl/extractor/tvplay.py | 41 ++++++++++++++++++++++++----------
 1 file changed, 29 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/tvplay.py b/youtube_dl/extractor/tvplay.py
index 150bde663..d82bf67b4 100644
--- a/youtube_dl/extractor/tvplay.py
+++ b/youtube_dl/extractor/tvplay.py
@@ -20,16 +20,25 @@ from ..utils import (
 
 
 class TVPlayIE(InfoExtractor):
-    IE_DESC = 'TV3Play and related services'
-    _VALID_URL = r'''(?x)https?://(?:www\.)?
-        (?:tvplay(?:\.skaties)?\.lv/parraides|
-           (?:tv3play|play\.tv3)\.lt/programos|
-           tv3play(?:\.tv3)?\.ee/sisu|
-           tv(?:3|6|8|10)play\.se/program|
-           (?:(?:tv3play|viasat4play|tv6play)\.no|tv3play\.dk)/programmer|
-           play\.novatv\.bg/programi
-        )/[^/]+/(?P<id>\d+)
-        '''
+    IE_NAME = 'mtg'
+    IE_DESC = 'MTG services'
+    _VALID_URL = r'''(?x)
+                    (?:
+                        mtg:|
+                        https?://
+                            (?:www\.)?
+                            (?:
+                                tvplay(?:\.skaties)?\.lv/parraides|
+                                (?:tv3play|play\.tv3)\.lt/programos|
+                                tv3play(?:\.tv3)?\.ee/sisu|
+                                (?:tv(?:3|6|8|10)play|viafree)\.se/program|
+                                (?:(?:tv3play|viasat4play|tv6play|viafree)\.no|(?:tv3play|viafree)\.dk)/programmer|
+                                play\.novatv\.bg/programi
+                            )
+                            /(?:[^/]+/)+
+                        )
+                        (?P<id>\d+)
+                    '''
     _TESTS = [
         {
             'url': 'http://www.tvplay.lv/parraides/vinas-melo-labak/418113?autostart=true',
@@ -197,6 +206,14 @@ class TVPlayIE(InfoExtractor):
         {
             'url': 'http://tv3play.tv3.ee/sisu/kodu-keset-linna/238551?autostart=true',
             'only_matching': True,
+        },
+        {
+            'url': 'http://www.viafree.se/program/underhallning/i-like-radio-live/sasong-1/676869',
+            'only_matching': True,
+        },
+        {
+            'url': 'mtg:418113',
+            'only_matching': True,
         }
     ]
 
@@ -204,13 +221,13 @@ class TVPlayIE(InfoExtractor):
         video_id = self._match_id(url)
 
         video = self._download_json(
-            'http://playapi.mtgx.tv/v1/videos/%s' % video_id, video_id, 'Downloading video JSON')
+            'http://playapi.mtgx.tv/v3/videos/%s' % video_id, video_id, 'Downloading video JSON')
 
         title = video['title']
 
         try:
             streams = self._download_json(
-                'http://playapi.mtgx.tv/v1/videos/stream/%s' % video_id,
+                'http://playapi.mtgx.tv/v3/videos/stream/%s' % video_id,
                 video_id, 'Downloading streams JSON')
         except ExtractorError as e:
             if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:

From b35b0d73d853c52ca96ccf4488a4f8960a12e2ae Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 17 Aug 2016 00:21:30 +0700
Subject: [PATCH 5801/6219] [viafree] Add extractor (Closes #10358)

---
 youtube_dl/extractor/extractors.py |  5 ++-
 youtube_dl/extractor/tvplay.py     | 53 ++++++++++++++++++++++++++++++
 2 files changed, 57 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index a5e0805b2..55c639158 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -898,7 +898,10 @@ from .tvp import (
     TVPIE,
     TVPSeriesIE,
 )
-from .tvplay import TVPlayIE
+from .tvplay import (
+    TVPlayIE,
+    ViafreeIE,
+)
 from .tweakers import TweakersIE
 from .twentyfourvideo import TwentyFourVideoIE
 from .twentymin import TwentyMinutenIE
diff --git a/youtube_dl/extractor/tvplay.py b/youtube_dl/extractor/tvplay.py
index d82bf67b4..c8ec2465c 100644
--- a/youtube_dl/extractor/tvplay.py
+++ b/youtube_dl/extractor/tvplay.py
@@ -311,3 +311,56 @@ class TVPlayIE(InfoExtractor):
             'formats': formats,
             'subtitles': subtitles,
         }
+
+
+class ViafreeIE(InfoExtractor):
+    _VALID_URL = r'''(?x)
+                    https?://
+                        (?:www\.)?
+                        viafree\.
+                        (?:
+                            (?:dk|no)/programmer|
+                            se/program
+                        )
+                        /(?:[^/]+/)+(?P<id>[^/?#&]+)
+                    '''
+    _TESTS = [{
+        'url': 'http://www.viafree.se/program/livsstil/husraddarna/sasong-2/avsnitt-2',
+        'info_dict': {
+            'id': '395375',
+            'ext': 'mp4',
+            'title': 'Husräddarna S02E02',
+            'description': 'md5:4db5c933e37db629b5a2f75dfb34829e',
+            'series': 'Husräddarna',
+            'season': 'Säsong 2',
+            'season_number': 2,
+            'duration': 2576,
+            'timestamp': 1400596321,
+            'upload_date': '20140520',
+        },
+        'params': {
+            'skip_download': True,
+        },
+        'add_ie': [TVPlayIE.ie_key()],
+    }, {
+        'url': 'http://www.viafree.no/programmer/underholdning/det-beste-vorspielet/sesong-2/episode-1',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.viafree.dk/programmer/reality/paradise-hotel/saeson-7/episode-5',
+        'only_matching': True,
+    }]
+
+    @classmethod
+    def suitable(cls, url):
+        return False if TVPlayIE.suitable(url) else super(ViafreeIE, cls).suitable(url)
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        video_id = self._search_regex(
+            r'currentVideo["\']\s*:\s*.+?["\']id["\']\s*:\s*["\'](?P<id>\d{6,})',
+            webpage, 'video id')
+
+        return self.url_result('mtg:%s' % video_id, TVPlayIE.ie_key())

From 502d87c5464f1894a8777873b9d11b76ba5a6375 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 17 Aug 2016 00:32:28 +0700
Subject: [PATCH 5802/6219] [mtg] Improve view count extraction

---
 youtube_dl/extractor/tvplay.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tvplay.py b/youtube_dl/extractor/tvplay.py
index c8ec2465c..4186e82db 100644
--- a/youtube_dl/extractor/tvplay.py
+++ b/youtube_dl/extractor/tvplay.py
@@ -15,6 +15,7 @@ from ..utils import (
     int_or_none,
     parse_iso8601,
     qualities,
+    try_get,
     update_url_query,
 )
 
@@ -203,6 +204,11 @@ class TVPlayIE(InfoExtractor):
             'url': 'http://tvplay.skaties.lv/parraides/vinas-melo-labak/418113?autostart=true',
             'only_matching': True,
         },
+        {
+            # views is null
+            'url': 'http://tvplay.skaties.lv/parraides/tv3-zinas/760183',
+            'only_matching': True,
+        },
         {
             'url': 'http://tv3play.tv3.ee/sisu/kodu-keset-linna/238551?autostart=true',
             'only_matching': True,
@@ -306,7 +312,7 @@ class TVPlayIE(InfoExtractor):
             'season_number': season_number,
             'duration': int_or_none(video.get('duration')),
             'timestamp': parse_iso8601(video.get('created_at')),
-            'view_count': int_or_none(video.get('views', {}).get('total')),
+            'view_count': try_get(video, lambda x: x['views']['total'], int),
             'age_limit': int_or_none(video.get('age_limit', 0)),
             'formats': formats,
             'subtitles': subtitles,

From 9c0fa60bf375959c7d8582f655b441c534865c03 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 17 Aug 2016 00:42:02 +0700
Subject: [PATCH 5803/6219] [vbox7] Add support for embed URLs

---
 youtube_dl/extractor/vbox7.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/vbox7.py b/youtube_dl/extractor/vbox7.py
index fa7899e6d..8e6d7efe7 100644
--- a/youtube_dl/extractor/vbox7.py
+++ b/youtube_dl/extractor/vbox7.py
@@ -6,7 +6,7 @@ from ..utils import urlencode_postdata
 
 
 class Vbox7IE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?vbox7\.com/play:(?P<id>[^/]+)'
+    _VALID_URL = r'https?://(?:www\.)?vbox7\.com/(?:play:|emb/external\.php\?.*?\bvid=)(?P<id>[\da-fA-F]+)'
     _TESTS = [{
         'url': 'http://vbox7.com/play:0946fff23c',
         'md5': 'a60f9ab3a3a2f013ef9a967d5f7be5bf',
@@ -24,15 +24,19 @@ class Vbox7IE(InfoExtractor):
             'title': 'Смях! Чудо - чист за секунди - Скрита камера',
         },
         'skip': 'georestricted',
+    }, {
+        'url': 'http://vbox7.com/emb/external.php?vid=a240d20f9c&autoplay=1',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        webpage = self._download_webpage(url, video_id)
+        webpage = self._download_webpage(
+            'http://vbox7.com/play:%s' % video_id, video_id)
 
         title = self._html_search_regex(
-            r'<title>(.*)</title>', webpage, 'title').split('/')[0].strip()
+            r'<title>(.+?)</title>', webpage, 'title').split('/')[0].strip()
 
         video_url = self._search_regex(
             r'src\s*:\s*(["\'])(?P<url>.+?.mp4.*?)\1',

From 2a1321a272c7b410db25654cdfdc33c3cd8bd440 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 17 Aug 2016 01:02:59 +0700
Subject: [PATCH 5804/6219] [vbox7:generic] Add support for vbox7 embeds

---
 youtube_dl/extractor/generic.py | 18 ++++++++++++++++++
 youtube_dl/extractor/vbox7.py   | 10 ++++++++++
 2 files changed, 28 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 50500ce0e..197ab9531 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -72,6 +72,7 @@ from .kaltura import KalturaIE
 from .eagleplatform import EaglePlatformIE
 from .facebook import FacebookIE
 from .soundcloud import SoundcloudIE
+from .vbox7 import Vbox7IE
 
 
 class GenericIE(InfoExtractor):
@@ -1373,6 +1374,18 @@ class GenericIE(InfoExtractor):
             },
             'add_ie': [ArkenaIE.ie_key()],
         },
+        {
+            'url': 'http://nova.bg/news/view/2016/08/16/156543/%D0%BD%D0%B0-%D0%BA%D0%BE%D1%81%D1%8A%D0%BC-%D0%BE%D1%82-%D0%B2%D0%B7%D1%80%D0%B8%D0%B2-%D0%BE%D1%82%D1%86%D0%B5%D0%BF%D0%B8%D1%85%D0%B0-%D1%86%D1%8F%D0%BB-%D0%BA%D0%B2%D0%B0%D1%80%D1%82%D0%B0%D0%BB-%D0%B7%D0%B0%D1%80%D0%B0%D0%B4%D0%B8-%D0%B8%D0%B7%D1%82%D0%B8%D1%87%D0%B0%D0%BD%D0%B5-%D0%BD%D0%B0-%D0%B3%D0%B0%D0%B7-%D0%B2-%D0%BF%D0%BB%D0%BE%D0%B2%D0%B4%D0%B8%D0%B2/',
+            'info_dict': {
+                'id': '1c7141f46c',
+                'ext': 'mp4',
+                'title': 'НА КОСЪМ ОТ ВЗРИВ: Изтичане на газ на бензиностанция в Пловдив',
+            },
+            'params': {
+                'skip_download': True,
+            },
+            'add_ie': [Vbox7IE.ie_key()],
+        },
         # {
         #     # TODO: find another test
         #     # http://schema.org/VideoObject
@@ -2239,6 +2252,11 @@ class GenericIE(InfoExtractor):
                 'uploader': video_uploader,
             }
 
+        # Look for VBOX7 embeds
+        vbox7_url = Vbox7IE._extract_url(webpage)
+        if vbox7_url:
+            return self.url_result(vbox7_url, Vbox7IE.ie_key())
+
         # Looking for http://schema.org/VideoObject
         json_ld = self._search_json_ld(
             webpage, video_id, default={}, expected_type='VideoObject')
diff --git a/youtube_dl/extractor/vbox7.py b/youtube_dl/extractor/vbox7.py
index 8e6d7efe7..e17988573 100644
--- a/youtube_dl/extractor/vbox7.py
+++ b/youtube_dl/extractor/vbox7.py
@@ -1,6 +1,8 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..utils import urlencode_postdata
 
@@ -29,6 +31,14 @@ class Vbox7IE(InfoExtractor):
         'only_matching': True,
     }]
 
+    @staticmethod
+    def _extract_url(webpage):
+        mobj = re.search(
+            '<iframe[^>]+src=(?P<q>["\'])(?P<url>(?:https?:)?//vbox7\.com/emb/external\.php.+?)(?P=q)',
+            webpage)
+        if mobj:
+            return mobj.group('url')
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
 

From 8652770bd23ff5f46c5687d94f71cec08d2c5886 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 17 Aug 2016 05:44:46 +0700
Subject: [PATCH 5805/6219] [keezmovies] Improve and modernize

---
 youtube_dl/extractor/keezmovies.py | 138 +++++++++++++++++++++--------
 1 file changed, 99 insertions(+), 39 deletions(-)

diff --git a/youtube_dl/extractor/keezmovies.py b/youtube_dl/extractor/keezmovies.py
index 126ca13df..ad2f8a8c8 100644
--- a/youtube_dl/extractor/keezmovies.py
+++ b/youtube_dl/extractor/keezmovies.py
@@ -3,64 +3,124 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..aes import aes_decrypt_text
+from ..compat import (
+    compat_str,
+    compat_urllib_parse_unquote,
+)
 from ..utils import (
-    sanitized_Request,
-    url_basename,
+    determine_ext,
+    ExtractorError,
+    int_or_none,
+    str_to_int,
+    strip_or_none,
 )
 
 
 class KeezMoviesIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?keezmovies\.com/video/.+?(?P<id>[0-9]+)(?:[/?&]|$)'
-    _TEST = {
+    _VALID_URL = r'https?://(?:www\.)?keezmovies\.com/video/(?:(?P<display_id>[^/]+)-)?(?P<id>\d+)'
+    _TESTS = [{
         'url': 'http://www.keezmovies.com/video/petite-asian-lady-mai-playing-in-bathtub-1214711',
         'md5': '1c1e75d22ffa53320f45eeb07bc4cdc0',
         'info_dict': {
             'id': '1214711',
+            'display_id': 'petite-asian-lady-mai-playing-in-bathtub',
             'ext': 'mp4',
             'title': 'Petite Asian Lady Mai Playing In Bathtub',
-            'age_limit': 18,
             'thumbnail': 're:^https?://.*\.jpg$',
+            'view_count': int,
+            'age_limit': 18,
         }
-    }
+    }, {
+        'url': 'http://www.keezmovies.com/video/1214711',
+        'only_matching': True,
+    }]
 
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
+    def _extract_info(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        display_id = mobj.group('display_id') or video_id
 
-        req = sanitized_Request(url)
-        req.add_header('Cookie', 'age_verified=1')
-        webpage = self._download_webpage(req, video_id)
-
-        # embedded video
-        mobj = re.search(r'href="([^"]+)"></iframe>', webpage)
-        if mobj:
-            embedded_url = mobj.group(1)
-            return self.url_result(embedded_url)
-
-        video_title = self._html_search_regex(
-            r'<h1 [^>]*>([^<]+)', webpage, 'title')
-        flashvars = self._parse_json(self._search_regex(
-            r'var\s+flashvars\s*=\s*([^;]+);', webpage, 'flashvars'), video_id)
+        webpage = self._download_webpage(
+            url, display_id, headers={'Cookie': 'age_verified=1'})
 
         formats = []
-        for height in (180, 240, 480):
-            if flashvars.get('quality_%dp' % height):
-                video_url = flashvars['quality_%dp' % height]
-                a_format = {
-                    'url': video_url,
-                    'height': height,
-                    'format_id': '%dp' % height,
-                }
-                filename_parts = url_basename(video_url).split('_')
-                if len(filename_parts) >= 2 and re.match(r'\d+[Kk]', filename_parts[1]):
-                    a_format['tbr'] = int(filename_parts[1][:-1])
-                formats.append(a_format)
+        format_urls = set()
 
-        age_limit = self._rta_search(webpage)
+        title = None
+        thumbnail = None
+        duration = None
+        encrypted = False
 
-        return {
+        def extract_format(format_url, height=None):
+            if not isinstance(format_url, compat_str) or not format_url.startswith('http'):
+                return
+            if format_url in format_urls:
+                return
+            format_urls.add(format_url)
+            tbr = int_or_none(self._search_regex(
+                r'[/_](\d+)[kK][/_]', format_url, 'tbr', default=None))
+            if not height:
+                height = int_or_none(self._search_regex(
+                    r'[/_](\d+)[pP][/_]', format_url, 'height', default=None))
+            if encrypted:
+                format_url = aes_decrypt_text(
+                    video_url, title, 32).decode('utf-8')
+            formats.append({
+                'url': format_url,
+                'format_id': '%dp' % height if height else None,
+                'height': height,
+                'tbr': tbr,
+            })
+
+        flashvars = self._parse_json(
+            self._search_regex(
+                r'flashvars\s*=\s*({.+?});', webpage,
+                'flashvars', default='{}'),
+            display_id, fatal=False)
+
+        if flashvars:
+            title = flashvars.get('video_title')
+            thumbnail = flashvars.get('image_url')
+            duration = int_or_none(flashvars.get('video_duration'))
+            encrypted = flashvars.get('encrypted') is True
+            for key, value in flashvars.items():
+                mobj = re.search(r'quality_(\d+)[pP]', key)
+                if mobj:
+                    extract_format(value, int(mobj.group(1)))
+            video_url = flashvars.get('video_url')
+            if video_url and determine_ext(video_url, None):
+                extract_format(video_url)
+
+        video_url = self._html_search_regex(
+            r'flashvars\.video_url\s*=\s*(["\'])(?P<url>http.+?)\1',
+            webpage, 'video url', default=None, group='url')
+        if video_url:
+            extract_format(compat_urllib_parse_unquote(video_url))
+
+        if not formats:
+            if 'title="This video is no longer available"' in webpage:
+                raise ExtractorError(
+                    'Video %s is no longer available' % video_id, expected=True)
+
+        self._sort_formats(formats)
+
+        if not title:
+            title = self._html_search_regex(
+                r'<h1[^>]*>([^<]+)', webpage, 'title')
+
+        return webpage, {
             'id': video_id,
-            'title': video_title,
+            'display_id': display_id,
+            'title': strip_or_none(title),
+            'thumbnail': thumbnail,
+            'duration': duration,
+            'age_limit': 18,
             'formats': formats,
-            'age_limit': age_limit,
-            'thumbnail': flashvars.get('image_url')
         }
+
+    def _real_extract(self, url):
+        webpage, info = self._extract_info(url)
+        info['view_count'] = str_to_int(self._search_regex(
+            r'<b>([\d,.]+)</b> Views?', webpage, 'view count', fatal=False))
+        return info

From 6be17c08703ad8ec89c6fb62f31f280956694cee Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 17 Aug 2016 05:45:49 +0700
Subject: [PATCH 5806/6219] [mofosex] Extract all formats and modernize (Closes
 #10335)

---
 youtube_dl/extractor/mofosex.py | 81 +++++++++++++++++----------------
 1 file changed, 42 insertions(+), 39 deletions(-)

diff --git a/youtube_dl/extractor/mofosex.py b/youtube_dl/extractor/mofosex.py
index e47c80119..e3bbe5aa8 100644
--- a/youtube_dl/extractor/mofosex.py
+++ b/youtube_dl/extractor/mofosex.py
@@ -1,53 +1,56 @@
 from __future__ import unicode_literals
 
-import os
-import re
-
-from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse_unquote,
-    compat_urllib_parse_urlparse,
+from ..utils import (
+    int_or_none,
+    str_to_int,
+    unified_strdate,
 )
-from ..utils import sanitized_Request
+from .keezmovies import KeezMoviesIE
 
 
-class MofosexIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?(?P<url>mofosex\.com/videos/(?P<id>[0-9]+)/.*?\.html)'
-    _TEST = {
-        'url': 'http://www.mofosex.com/videos/5018/japanese-teen-music-video.html',
-        'md5': '1b2eb47ac33cc75d4a80e3026b613c5a',
+class MofosexIE(KeezMoviesIE):
+    _VALID_URL = r'https?://(?:www\.)?mofosex\.com/videos/(?P<id>\d+)/(?P<display_id>[^/?#&.]+)\.html'
+    _TESTS = [{
+        'url': 'http://www.mofosex.com/videos/318131/amateur-teen-playing-and-masturbating-318131.html',
+        'md5': '39a15853632b7b2e5679f92f69b78e91',
         'info_dict': {
-            'id': '5018',
+            'id': '318131',
+            'display_id': 'amateur-teen-playing-and-masturbating-318131',
             'ext': 'mp4',
-            'title': 'Japanese Teen Music Video',
+            'title': 'amateur teen playing and masturbating',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'upload_date': '20121114',
+            'view_count': int,
+            'like_count': int,
+            'dislike_count': int,
             'age_limit': 18,
         }
-    }
+    }, {
+        # This video is no longer available
+        'url': 'http://www.mofosex.com/videos/5018/japanese-teen-music-video.html',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        url = 'http://www.' + mobj.group('url')
+        webpage, info = self._extract_info(url)
 
-        req = sanitized_Request(url)
-        req.add_header('Cookie', 'age_verified=1')
-        webpage = self._download_webpage(req, video_id)
+        view_count = str_to_int(self._search_regex(
+            r'VIEWS:</span>\s*([\d,.]+)', webpage, 'view count', fatal=False))
+        like_count = int_or_none(self._search_regex(
+            r'id=["\']amountLikes["\'][^>]*>(\d+)', webpage,
+            'like count', fatal=False))
+        dislike_count = int_or_none(self._search_regex(
+            r'id=["\']amountDislikes["\'][^>]*>(\d+)', webpage,
+            'like count', fatal=False))
+        upload_date = unified_strdate(self._html_search_regex(
+            r'Added:</span>([^<]+)', webpage, 'upload date', fatal=False))
 
-        video_title = self._html_search_regex(r'<h1>(.+?)<', webpage, 'title')
-        video_url = compat_urllib_parse_unquote(self._html_search_regex(r'flashvars.video_url = \'([^\']+)', webpage, 'video_url'))
-        path = compat_urllib_parse_urlparse(video_url).path
-        extension = os.path.splitext(path)[1][1:]
-        format = path.split('/')[5].split('_')[:2]
-        format = '-'.join(format)
+        info.update({
+            'view_count': view_count,
+            'like_count': like_count,
+            'dislike_count': dislike_count,
+            'upload_date': upload_date,
+            'thumbnail': self._og_search_thumbnail(webpage),
+        })
 
-        age_limit = self._rta_search(webpage)
-
-        return {
-            'id': video_id,
-            'title': video_title,
-            'url': video_url,
-            'ext': extension,
-            'format': format,
-            'format_id': format,
-            'age_limit': age_limit,
-        }
+        return info

From 8804f10e6b580db38df7301a174cb48ea374f9eb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 17 Aug 2016 05:46:45 +0700
Subject: [PATCH 5807/6219] [tube8] Modernize

---
 youtube_dl/extractor/tube8.py | 60 ++++++-----------------------------
 1 file changed, 10 insertions(+), 50 deletions(-)

diff --git a/youtube_dl/extractor/tube8.py b/youtube_dl/extractor/tube8.py
index 1d9271d1e..4053f6c21 100644
--- a/youtube_dl/extractor/tube8.py
+++ b/youtube_dl/extractor/tube8.py
@@ -1,18 +1,13 @@
 from __future__ import unicode_literals
 
-import re
-
-from .common import InfoExtractor
-from ..compat import compat_str
 from ..utils import (
     int_or_none,
-    sanitized_Request,
     str_to_int,
 )
-from ..aes import aes_decrypt_text
+from .keezmovies import KeezMoviesIE
 
 
-class Tube8IE(InfoExtractor):
+class Tube8IE(KeezMoviesIE):
     _VALID_URL = r'https?://(?:www\.)?tube8\.com/(?:[^/]+/)+(?P<display_id>[^/]+)/(?P<id>\d+)'
     _TESTS = [{
         'url': 'http://www.tube8.com/teen/kasia-music-video/229795/',
@@ -33,47 +28,17 @@ class Tube8IE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        display_id = mobj.group('display_id')
+        webpage, info = self._extract_info(url)
 
-        req = sanitized_Request(url)
-        req.add_header('Cookie', 'age_verified=1')
-        webpage = self._download_webpage(req, display_id)
+        if not info['title']:
+            info['title'] = self._html_search_regex(
+                r'videoTitle\s*=\s*"([^"]+)', webpage, 'title')
 
-        flashvars = self._parse_json(
-            self._search_regex(
-                r'flashvars\s*=\s*({.+?});\r?\n', webpage, 'flashvars'),
-            video_id)
-
-        formats = []
-        for key, video_url in flashvars.items():
-            if not isinstance(video_url, compat_str) or not video_url.startswith('http'):
-                continue
-            height = self._search_regex(
-                r'quality_(\d+)[pP]', key, 'height', default=None)
-            if not height:
-                continue
-            if flashvars.get('encrypted') is True:
-                video_url = aes_decrypt_text(
-                    video_url, flashvars['video_title'], 32).decode('utf-8')
-            formats.append({
-                'url': video_url,
-                'format_id': '%sp' % height,
-                'height': int(height),
-            })
-        self._sort_formats(formats)
-
-        thumbnail = flashvars.get('image_url')
-
-        title = self._html_search_regex(
-            r'videoTitle\s*=\s*"([^"]+)', webpage, 'title')
         description = self._html_search_regex(
             r'>Description:</strong>\s*(.+?)\s*<', webpage, 'description', fatal=False)
         uploader = self._html_search_regex(
             r'<span class="username">\s*(.+?)\s*<',
             webpage, 'uploader', fatal=False)
-        duration = int_or_none(flashvars.get('video_duration'))
 
         like_count = int_or_none(self._search_regex(
             r'rupVar\s*=\s*"(\d+)"', webpage, 'like count', fatal=False))
@@ -86,18 +51,13 @@ class Tube8IE(InfoExtractor):
             r'<span id="allCommentsCount">(\d+)</span>',
             webpage, 'comment count', fatal=False))
 
-        return {
-            'id': video_id,
-            'display_id': display_id,
-            'title': title,
+        info.update({
             'description': description,
-            'thumbnail': thumbnail,
             'uploader': uploader,
-            'duration': duration,
             'view_count': view_count,
             'like_count': like_count,
             'dislike_count': dislike_count,
             'comment_count': comment_count,
-            'age_limit': 18,
-            'formats': formats,
-        }
+        })
+
+        return info

From ab19b46b88bb54971b973176976d8d189222a6d3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 17 Aug 2016 06:02:12 +0700
Subject: [PATCH 5808/6219] [extremetube] Modernize

---
 youtube_dl/extractor/extremetube.py | 73 +++++++----------------------
 1 file changed, 16 insertions(+), 57 deletions(-)

diff --git a/youtube_dl/extractor/extremetube.py b/youtube_dl/extractor/extremetube.py
index 3403581fd..b4fd9334a 100644
--- a/youtube_dl/extractor/extremetube.py
+++ b/youtube_dl/extractor/extremetube.py
@@ -1,22 +1,17 @@
 from __future__ import unicode_literals
 
-import re
-
-from .common import InfoExtractor
-from ..utils import (
-    int_or_none,
-    sanitized_Request,
-    str_to_int,
-)
+from ..utils import str_to_int
+from .keezmovies import KeezMoviesIE
 
 
-class ExtremeTubeIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?extremetube\.com/(?:[^/]+/)?video/(?P<id>[^/#?&]+)'
+class ExtremeTubeIE(KeezMoviesIE):
+    _VALID_URL = r'https?://(?:www\.)?extremetube\.com/(?:[^/]+/)?video/(?:(?P<display_id>[^/]+)-)(?P<id>\d+)'
     _TESTS = [{
         'url': 'http://www.extremetube.com/video/music-video-14-british-euro-brit-european-cumshots-swallow-652431',
-        'md5': '344d0c6d50e2f16b06e49ca011d8ac69',
+        'md5': '1fb9228f5e3332ec8c057d6ac36f33e0',
         'info_dict': {
-            'id': 'music-video-14-british-euro-brit-european-cumshots-swallow-652431',
+            'id': '652431',
+            'display_id': 'music-video-14-british-euro-brit-european-cumshots-swallow',
             'ext': 'mp4',
             'title': 'Music Video 14 british euro brit european cumshots swallow',
             'uploader': 'unknown',
@@ -35,58 +30,22 @@ class ExtremeTubeIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        webpage, info = self._extract_info(url)
 
-        req = sanitized_Request(url)
-        req.add_header('Cookie', 'age_verified=1')
-        webpage = self._download_webpage(req, video_id)
+        if not info['title']:
+            info['title'] = self._search_regex(
+                r'<h1[^>]+title="([^"]+)"[^>]*>', webpage, 'title')
 
-        video_title = self._html_search_regex(
-            r'<h1 [^>]*?title="([^"]+)"[^>]*>', webpage, 'title')
         uploader = self._html_search_regex(
             r'Uploaded by:\s*</strong>\s*(.+?)\s*</div>',
             webpage, 'uploader', fatal=False)
-        view_count = str_to_int(self._html_search_regex(
+        view_count = str_to_int(self._search_regex(
             r'Views:\s*</strong>\s*<span>([\d,\.]+)</span>',
             webpage, 'view count', fatal=False))
 
-        flash_vars = self._parse_json(
-            self._search_regex(
-                r'var\s+flashvars\s*=\s*({.+?});', webpage, 'flash vars'),
-            video_id)
-
-        formats = []
-        for quality_key, video_url in flash_vars.items():
-            height = int_or_none(self._search_regex(
-                r'quality_(\d+)[pP]$', quality_key, 'height', default=None))
-            if not height:
-                continue
-            f = {
-                'url': video_url,
-            }
-            mobj = re.search(
-                r'/(?P<height>\d{3,4})[pP]_(?P<bitrate>\d+)[kK]_\d+', video_url)
-            if mobj:
-                height = int(mobj.group('height'))
-                bitrate = int(mobj.group('bitrate'))
-                f.update({
-                    'format_id': '%dp-%dk' % (height, bitrate),
-                    'height': height,
-                    'tbr': bitrate,
-                })
-            else:
-                f.update({
-                    'format_id': '%dp' % height,
-                    'height': height,
-                })
-            formats.append(f)
-        self._sort_formats(formats)
-
-        return {
-            'id': video_id,
-            'title': video_title,
-            'formats': formats,
+        info.update({
             'uploader': uploader,
             'view_count': view_count,
-            'age_limit': 18,
-        }
+        })
+
+        return info

From a44694ab4e1ee6ac496ea09c3759923c03b9430c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 17 Aug 2016 06:19:22 +0700
Subject: [PATCH 5809/6219] [ChangeLog] Actualize

---
 ChangeLog | 21 ++++++++++++++++++++-
 1 file changed, 20 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index 32504dab5..8f27019c1 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,9 +1,28 @@
 version <unreleased>
 
+Core
++ Add _get_netrc_login_info
+
 Extractors
+* [mofosex] Extract all formats (#10335)
++ [generic] Add support for vbox7 embeds
++ [vbox7] Add support for embed URLs
++ [viafree] Add extractor (#10358)
++ [mtg] Add support for viafree URLs (#10358)
+* [theplatform] Extract all subtitles per language
++ [xvideos] Fix HLS extraction (#10356)
++ [amcnetworks] Add extractor
++ [bbc:playlist] Add support for pagination (#10349)
++ [fxnetworks] Add extractor (#9462)
 * [cbslocal] Fix extraction for SendtoNews-based videos
 * [sendtonews] Fix extraction
-* [jwplatform] Now can parse video_id from JWPlayer data
+* [jwplatform] Extract video id from JWPlayer data
+- [zippcast] Remove extractor (#10332)
++ [viceland] Add extractor (#8799)
++ [adobepass] Add base extractor for Adobe Pass Authentication
+* [life:embed] Improve extraction
+* [vgtv] Detect geo restricted videos (#10348)
++ [uplynk] Add extractor
 * [xiami] Fix extraction (#10342)
 
 

From b3d7dce42952cf23b8f9ea883c75736dadfee12e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 17 Aug 2016 06:21:21 +0700
Subject: [PATCH 5810/6219] release 2016.08.17

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 9 +++++++--
 youtube_dl/version.py     | 2 +-
 4 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 1e0d99b43..ae28d83d5 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.08.13*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.08.13**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.08.17*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.08.17**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.08.13
+[debug] youtube-dl version 2016.08.17
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 8f27019c1..354306a97 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2016.08.17
 
 Core
 + Add _get_netrc_login_info
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 56fc41a40..189b9301d 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -35,6 +35,7 @@
  - **AlJazeera**
  - **Allocine**
  - **AlphaPorno**
+ - **AMCNetworks**
  - **AnimeOnDemand**
  - **anitube.se**
  - **AnySex**
@@ -247,6 +248,7 @@
  - **Funimation**
  - **FunnyOrDie**
  - **Fusion**
+ - **FXNetworks**
  - **GameInformer**
  - **GameOne**
  - **gameone:playlist**
@@ -398,6 +400,7 @@
  - **Moviezine**
  - **MPORA**
  - **MSN**
+ - **mtg**: MTG services
  - **MTV**
  - **mtv.de**
  - **mtvservices:embedded**
@@ -731,7 +734,6 @@
  - **tvp**: Telewizja Polska
  - **tvp:embed**: Telewizja Polska
  - **tvp:series**
- - **TVPlay**: TV3Play and related services
  - **Tweakers**
  - **twitch:chapter**
  - **twitch:clips**
@@ -748,6 +750,8 @@
  - **UDNEmbed**: 聯合影音
  - **Unistra**
  - **uol.com.br**
+ - **uplynk**
+ - **uplynk:preplay**
  - **Urort**: NRK P3 Urørt
  - **URPlay**
  - **USAToday**
@@ -765,7 +769,9 @@
  - **VevoPlaylist**
  - **VGTV**: VGTV, BTTV, FTV, Aftenposten and Aftonbladet
  - **vh1.com**
+ - **Viafree**
  - **Vice**
+ - **Viceland**
  - **ViceShow**
  - **Vidbit**
  - **Viddler**
@@ -887,4 +893,3 @@
  - **ZDFChannel**
  - **zingmp3:album**: mp3.zing.vn albums
  - **zingmp3:song**: mp3.zing.vn songs
- - **ZippCast**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index cc93d22aa..cf5950117 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.08.13'
+__version__ = '2016.08.17'

From 92cd9fd56574f22087a8f8df52192df1d4c11a21 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 17 Aug 2016 07:01:32 +0700
Subject: [PATCH 5811/6219] [keezmovies] Make display_id optional

---
 youtube_dl/extractor/keezmovies.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/keezmovies.py b/youtube_dl/extractor/keezmovies.py
index ad2f8a8c8..b002c0dd1 100644
--- a/youtube_dl/extractor/keezmovies.py
+++ b/youtube_dl/extractor/keezmovies.py
@@ -39,7 +39,8 @@ class KeezMoviesIE(InfoExtractor):
     def _extract_info(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
-        display_id = mobj.group('display_id') or video_id
+        display_id = (mobj.group('display_id') if 'display_id'
+            in mobj.groupdict() else None) or mobj.group('id')
 
         webpage = self._download_webpage(
             url, display_id, headers={'Cookie': 'age_verified=1'})

From b505e98784b2c1cc07f734e9709702ee9d01287e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 17 Aug 2016 07:02:13 +0700
Subject: [PATCH 5812/6219] [extremetube] Revert display_id

---
 youtube_dl/extractor/extremetube.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/extremetube.py b/youtube_dl/extractor/extremetube.py
index b4fd9334a..445f9438d 100644
--- a/youtube_dl/extractor/extremetube.py
+++ b/youtube_dl/extractor/extremetube.py
@@ -5,13 +5,12 @@ from .keezmovies import KeezMoviesIE
 
 
 class ExtremeTubeIE(KeezMoviesIE):
-    _VALID_URL = r'https?://(?:www\.)?extremetube\.com/(?:[^/]+/)?video/(?:(?P<display_id>[^/]+)-)(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?extremetube\.com/(?:[^/]+/)?video/(?P<id>[^/#?&]+)'
     _TESTS = [{
         'url': 'http://www.extremetube.com/video/music-video-14-british-euro-brit-european-cumshots-swallow-652431',
         'md5': '1fb9228f5e3332ec8c057d6ac36f33e0',
         'info_dict': {
-            'id': '652431',
-            'display_id': 'music-video-14-british-euro-brit-european-cumshots-swallow',
+            'id': 'music-video-14-british-euro-brit-european-cumshots-swallow-652431',
             'ext': 'mp4',
             'title': 'Music Video 14 british euro brit european cumshots swallow',
             'uploader': 'unknown',

From 7273e5849b27cb7d0f4d5f40e7801cab2da85ae3 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 17 Aug 2016 11:03:09 +0100
Subject: [PATCH 5813/6219] [discoverygo] extend _VALID_URL to support other
 networks

---
 youtube_dl/extractor/discoverygo.py | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/discoverygo.py b/youtube_dl/extractor/discoverygo.py
index adb68b96c..cba709935 100644
--- a/youtube_dl/extractor/discoverygo.py
+++ b/youtube_dl/extractor/discoverygo.py
@@ -11,7 +11,17 @@ from ..utils import (
 
 
 class DiscoveryGoIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?discoverygo\.com/(?:[^/]+/)*(?P<id>[^/?#&]+)'
+    _VALID_URL = r'''(?x)https?://(?:www\.)?(?:
+            discovery|
+            investigationdiscovery|
+            discoverylife|
+            animalplanet|
+            ahctv|
+            destinationamerica|
+            sciencechannel|
+            tlc|
+            velocitychannel
+        )go\.com/(?:[^/]+/)*(?P<id>[^/?#&]+)'''
     _TEST = {
         'url': 'https://www.discoverygo.com/love-at-first-kiss/kiss-first-ask-questions-later/',
         'info_dict': {

From 4e9fee101508fe90c5b103738d1b6458e40affd6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 18 Aug 2016 04:37:14 +0700
Subject: [PATCH 5814/6219] [hgtvcom:show] Add extractor (Closes #10365)

---
 youtube_dl/extractor/extractors.py |  5 ++++-
 youtube_dl/extractor/hgtv.py       | 31 ++++++++++++++++++++++++++++++
 2 files changed, 35 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 55c639158..e61bb11c3 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -324,7 +324,10 @@ from .heise import HeiseIE
 from .hellporno import HellPornoIE
 from .helsinki import HelsinkiIE
 from .hentaistigma import HentaiStigmaIE
-from .hgtv import HGTVIE
+from .hgtv import (
+    HGTVIE,
+    HGTVComShowIE,
+)
 from .historicfilms import HistoricFilmsIE
 from .hitbox import HitboxIE, HitboxLiveIE
 from .hornbunny import HornBunnyIE
diff --git a/youtube_dl/extractor/hgtv.py b/youtube_dl/extractor/hgtv.py
index c3f0733cf..69543bff2 100644
--- a/youtube_dl/extractor/hgtv.py
+++ b/youtube_dl/extractor/hgtv.py
@@ -46,3 +46,34 @@ class HGTVIE(InfoExtractor):
             'episode_number': int_or_none(embed_vars.get('episode')),
             'ie_key': 'ThePlatform',
         }
+
+
+class HGTVComShowIE(InfoExtractor):
+    IE_NAME = 'hgtv.com:show'
+    _VALID_URL = r'https?://(?:www\.)?hgtv\.com/shows/[^/]+/(?P<id>[^/?#&]+)'
+    _TEST = {
+        'url': 'http://www.hgtv.com/shows/flip-or-flop/flip-or-flop-full-episodes-videos',
+        'info_dict': {
+            'id': 'flip-or-flop-full-episodes-videos',
+            'title': 'Flip or Flop Full Episodes',
+        },
+        'playlist_mincount': 15,
+    }
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, display_id)
+
+        config = self._parse_json(
+            self._search_regex(
+                r'(?s)data-module=["\']video["\'][^>]*>.*?<script[^>]+type=["\']text/x-config["\'][^>]*>(.+?)</script',
+                webpage, 'video config'),
+            display_id)['channels'][0]
+
+        entries = [
+            self.url_result(video['releaseUrl'])
+            for video in config['videos'] if video.get('releaseUrl')]
+
+        return self.playlist_result(
+            entries, display_id, config.get('title'), config.get('description'))

From e15ad9ef097253e27c7d2e78a470adaa032e8bfa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 18 Aug 2016 04:39:31 +0700
Subject: [PATCH 5815/6219] [keezmovies] PEP 8

---
 youtube_dl/extractor/keezmovies.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/keezmovies.py b/youtube_dl/extractor/keezmovies.py
index b002c0dd1..588a4d0ec 100644
--- a/youtube_dl/extractor/keezmovies.py
+++ b/youtube_dl/extractor/keezmovies.py
@@ -39,8 +39,9 @@ class KeezMoviesIE(InfoExtractor):
     def _extract_info(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
-        display_id = (mobj.group('display_id') if 'display_id'
-            in mobj.groupdict() else None) or mobj.group('id')
+        display_id = (mobj.group('display_id')
+                      if 'display_id' in mobj.groupdict()
+                      else None) or mobj.group('id')
 
         webpage = self._download_webpage(
             url, display_id, headers={'Cookie': 'age_verified=1'})

From 08a42f9c741aa37a599e6fe54ec8b9660df117e5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 18 Aug 2016 05:22:23 +0700
Subject: [PATCH 5816/6219] [vk] Fix authentication on python3

---
 youtube_dl/extractor/vk.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 3ee66e23e..634d17d91 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -52,8 +52,9 @@ class VKBaseIE(InfoExtractor):
         # what actually happens.
         # We will workaround this VK issue by resetting the remixlhk cookie to
         # the first one manually.
-        cookies = url_handle.headers.get('Set-Cookie')
-        if cookies:
+        for header, cookies in url_handle.headers.items():
+            if header.lower() != 'set-cookie':
+                continue
             if sys.version_info[0] >= 3:
                 cookies = cookies.encode('iso-8859-1')
             cookies = cookies.decode('utf-8')
@@ -61,6 +62,7 @@ class VKBaseIE(InfoExtractor):
             if remixlhk:
                 value, domain = remixlhk.groups()
                 self._set_cookie(domain, 'remixlhk', value)
+                break
 
         login_page = self._download_webpage(
             'https://login.vk.com/?act=login', None,

From 51815886a98503593524ec6ffa778ff19d840e2a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 18 Aug 2016 06:14:05 +0700
Subject: [PATCH 5817/6219] [vk:wallpost] Fix audio extraction

---
 youtube_dl/extractor/vk.py | 66 ++++++++++++++++++++++----------------
 1 file changed, 38 insertions(+), 28 deletions(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 634d17d91..cd22df25a 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -1,6 +1,7 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
+import collections
 import re
 import json
 import sys
@@ -16,7 +17,6 @@ from ..utils import (
     get_element_by_class,
     int_or_none,
     orderedSet,
-    parse_duration,
     remove_start,
     str_to_int,
     unescapeHTML,
@@ -447,6 +447,9 @@ class VKWallPostIE(VKBaseIE):
                 'skip_download': True,
             },
         }],
+        'params': {
+            'usenetrc': True,
+        },
         'skip': 'Requires vk account credentials',
     }, {
         # single YouTube embed, no leading -
@@ -456,6 +459,9 @@ class VKWallPostIE(VKBaseIE):
             'title': 'Sergey Gorbunov - Wall post 85155021_6319',
         },
         'playlist_count': 1,
+        'params': {
+            'usenetrc': True,
+        },
         'skip': 'Requires vk account credentials',
     }, {
         # wall page URL
@@ -483,37 +489,41 @@ class VKWallPostIE(VKBaseIE):
             raise ExtractorError('VK said: %s' % error, expected=True)
 
         description = clean_html(get_element_by_class('wall_post_text', webpage))
-        uploader = clean_html(get_element_by_class(
-            'fw_post_author', webpage)) or self._og_search_description(webpage)
+        uploader = clean_html(get_element_by_class('author', webpage))
         thumbnail = self._og_search_thumbnail(webpage)
 
         entries = []
 
-        for audio in re.finditer(r'''(?sx)
-                            <input[^>]+
-                                id=(?P<q1>["\'])audio_info(?P<id>\d+_\d+).*?(?P=q1)[^>]+
-                                value=(?P<q2>["\'])(?P<url>http.+?)(?P=q2)
-                                .+?
-                            </table>''', webpage):
-            audio_html = audio.group(0)
-            audio_id = audio.group('id')
-            duration = parse_duration(get_element_by_class('duration', audio_html))
-            track = self._html_search_regex(
-                r'<span[^>]+id=["\']title%s[^>]*>([^<]+)' % audio_id,
-                audio_html, 'title', default=None)
-            artist = self._html_search_regex(
-                r'>([^<]+)</a></b>\s*&ndash', audio_html,
-                'artist', default=None)
-            entries.append({
-                'id': audio_id,
-                'url': audio.group('url'),
-                'title': '%s - %s' % (artist, track) if artist and track else audio_id,
-                'thumbnail': thumbnail,
-                'duration': duration,
-                'uploader': uploader,
-                'artist': artist,
-                'track': track,
-            })
+        audio_ids = re.findall(r'data-full-id=["\'](\d+_\d+)', webpage)
+        if audio_ids:
+            al_audio = self._download_webpage(
+                'https://vk.com/al_audio.php', post_id,
+                note='Downloading audio info', fatal=False,
+                data=urlencode_postdata({
+                    'act': 'reload_audio',
+                    'al': '1',
+                    'ids': ','.join(audio_ids)
+                }))
+            if al_audio:
+                Audio = collections.namedtuple(
+                    'Audio', ['id', 'user_id', 'url', 'track', 'artist', 'duration'])
+                audios = self._parse_json(
+                    self._search_regex(
+                        r'<!json>(.+?)<!>', al_audio, 'audios', default='[]'),
+                    post_id, fatal=False, transform_source=unescapeHTML)
+                if isinstance(audios, list):
+                    for audio in audios:
+                        a = Audio._make(audio[:6])
+                        entries.append({
+                            'id': '%s_%s' % (a.user_id, a.id),
+                            'url': a.url,
+                            'title': '%s - %s' % (a.artist, a.track) if a.artist and a.track else a.id,
+                            'thumbnail': thumbnail,
+                            'duration': a.duration,
+                            'uploader': uploader,
+                            'artist': a.artist,
+                            'track': a.track,
+                        })
 
         for video in re.finditer(
                 r'<a[^>]+href=(["\'])(?P<url>/video(?:-?[\d_]+).*?)\1', webpage):

From b0c8f2e9c8946f8aab4be0d1435e504aac0d317f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?D=C3=A9stin=20Reed?= <trox1972@users.noreply.github.com>
Date: Wed, 17 Aug 2016 12:45:24 +0200
Subject: [PATCH 5818/6219] [DBTV:generic] Add support for embeds

---
 youtube_dl/extractor/dbtv.py    |  6 ++++++
 youtube_dl/extractor/generic.py | 11 +++++++++++
 2 files changed, 17 insertions(+)

diff --git a/youtube_dl/extractor/dbtv.py b/youtube_dl/extractor/dbtv.py
index caff8842e..73dba5e2a 100644
--- a/youtube_dl/extractor/dbtv.py
+++ b/youtube_dl/extractor/dbtv.py
@@ -38,6 +38,12 @@ class DBTVIE(InfoExtractor):
         'only_matching': True,
     }]
 
+    @staticmethod
+    def _extract_urls(webpage):
+        return [url for _, url in re.findall(
+            r'<iframe[^>]+src=(["\'])((?:https?:)?//(?:www\.)?dbtv\.no/lazyplayer/\d+.*?)\1',
+            webpage)]
+
     def _real_extract(self, url):
         video_id, display_id = re.match(self._VALID_URL, url).groups()
 
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 197ab9531..1b71f7ac8 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -73,6 +73,7 @@ from .eagleplatform import EaglePlatformIE
 from .facebook import FacebookIE
 from .soundcloud import SoundcloudIE
 from .vbox7 import Vbox7IE
+from .dbtv import DBTVIE
 
 
 class GenericIE(InfoExtractor):
@@ -1386,6 +1387,11 @@ class GenericIE(InfoExtractor):
             },
             'add_ie': [Vbox7IE.ie_key()],
         },
+        {
+            # DBTV embeds
+            'url': 'http://www.dagbladet.no/2016/02/23/nyheter/nordlys/ski/troms/ver/43254897/',
+            'playlist_mincount': 3,
+        },
         # {
         #     # TODO: find another test
         #     # http://schema.org/VideoObject
@@ -2257,6 +2263,11 @@ class GenericIE(InfoExtractor):
         if vbox7_url:
             return self.url_result(vbox7_url, Vbox7IE.ie_key())
 
+        # Look for DBTV embeds
+        dbtv_urls = DBTVIE._extract_urls(webpage)
+        if dbtv_urls:
+            return _playlist_from_matches(dbtv_urls, ie=DBTVIE.ie_key())
+
         # Looking for http://schema.org/VideoObject
         json_ld = self._search_json_ld(
             webpage, video_id, default={}, expected_type='VideoObject')

From b0d578ff7b54c521776cf8d1e050dc198bbc26e6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 18 Aug 2016 21:30:55 +0700
Subject: [PATCH 5819/6219] [dbtv] Relax embed regex

---
 youtube_dl/extractor/dbtv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/dbtv.py b/youtube_dl/extractor/dbtv.py
index 73dba5e2a..6d880d43d 100644
--- a/youtube_dl/extractor/dbtv.py
+++ b/youtube_dl/extractor/dbtv.py
@@ -41,7 +41,7 @@ class DBTVIE(InfoExtractor):
     @staticmethod
     def _extract_urls(webpage):
         return [url for _, url in re.findall(
-            r'<iframe[^>]+src=(["\'])((?:https?:)?//(?:www\.)?dbtv\.no/lazyplayer/\d+.*?)\1',
+            r'<iframe[^>]+src=(["\'])((?:https?:)?//(?:www\.)?dbtv\.no/(?:lazy)?player/\d+.*?)\1',
             webpage)]
 
     def _real_extract(self, url):

From fd3ec986a4217319d0cc345c5e2eb910d90be6f8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 18 Aug 2016 21:35:41 +0700
Subject: [PATCH 5820/6219] [generic] Fix dbtv test (Closes #10364)

---
 youtube_dl/extractor/generic.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 1b71f7ac8..506892b11 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1390,6 +1390,10 @@ class GenericIE(InfoExtractor):
         {
             # DBTV embeds
             'url': 'http://www.dagbladet.no/2016/02/23/nyheter/nordlys/ski/troms/ver/43254897/',
+            'info_dict': {
+                'id': '43254897',
+                'title': 'Etter ett års planlegging, klaffet endelig alt: - Jeg måtte ta en liten dans',
+            },
             'playlist_mincount': 3,
         },
         # {

From 13585d7682ef6351bfcd463cf1802bc8fbadaf43 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 18 Aug 2016 23:32:00 +0700
Subject: [PATCH 5821/6219] [utils] Recognize lowercase units in parse_filesize

---
 test/test_utils.py  | 2 ++
 youtube_dl/utils.py | 8 ++++++++
 2 files changed, 10 insertions(+)

diff --git a/test/test_utils.py b/test/test_utils.py
index 74fcf91c0..cb578cd53 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -817,7 +817,9 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(parse_filesize('2 MiB'), 2097152)
         self.assertEqual(parse_filesize('5 GB'), 5000000000)
         self.assertEqual(parse_filesize('1.2Tb'), 1200000000000)
+        self.assertEqual(parse_filesize('1.2tb'), 1200000000000)
         self.assertEqual(parse_filesize('1,24 KB'), 1240)
+        self.assertEqual(parse_filesize('1,24 kb'), 1240)
 
     def test_parse_count(self):
         self.assertEqual(parse_count(None), None)
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index b3b687a31..35362e767 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1508,34 +1508,42 @@ def parse_filesize(s):
         'KB': 1000,
         'kB': 1024,
         'Kb': 1000,
+        'kb': 1000,
         'MiB': 1024 ** 2,
         'MB': 1000 ** 2,
         'mB': 1024 ** 2,
         'Mb': 1000 ** 2,
+        'mb': 1000 ** 2,
         'GiB': 1024 ** 3,
         'GB': 1000 ** 3,
         'gB': 1024 ** 3,
         'Gb': 1000 ** 3,
+        'gb': 1000 ** 3,
         'TiB': 1024 ** 4,
         'TB': 1000 ** 4,
         'tB': 1024 ** 4,
         'Tb': 1000 ** 4,
+        'tb': 1000 ** 4,
         'PiB': 1024 ** 5,
         'PB': 1000 ** 5,
         'pB': 1024 ** 5,
         'Pb': 1000 ** 5,
+        'pb': 1000 ** 5,
         'EiB': 1024 ** 6,
         'EB': 1000 ** 6,
         'eB': 1024 ** 6,
         'Eb': 1000 ** 6,
+        'eb': 1000 ** 6,
         'ZiB': 1024 ** 7,
         'ZB': 1000 ** 7,
         'zB': 1024 ** 7,
         'Zb': 1000 ** 7,
+        'zb': 1000 ** 7,
         'YiB': 1024 ** 8,
         'YB': 1000 ** 8,
         'yB': 1024 ** 8,
         'Yb': 1000 ** 8,
+        'yb': 1000 ** 8,
     }
 
     return lookup_unit_table(_UNIT_TABLE, s)

From 850837b67ada7cf0a139117a7335aa40990cd0d7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 18 Aug 2016 23:52:41 +0700
Subject: [PATCH 5822/6219] [porncom] Add extractor (Closes #2251, closes
 #10251)

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/porncom.py    | 89 ++++++++++++++++++++++++++++++
 2 files changed, 90 insertions(+)
 create mode 100644 youtube_dl/extractor/porncom.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index e61bb11c3..6c5d46015 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -642,6 +642,7 @@ from .podomatic import PodomaticIE
 from .pokemon import PokemonIE
 from .polskieradio import PolskieRadioIE
 from .porn91 import Porn91IE
+from .porncom import PornComIE
 from .pornhd import PornHdIE
 from .pornhub import (
     PornHubIE,
diff --git a/youtube_dl/extractor/porncom.py b/youtube_dl/extractor/porncom.py
new file mode 100644
index 000000000..4baf79688
--- /dev/null
+++ b/youtube_dl/extractor/porncom.py
@@ -0,0 +1,89 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..compat import compat_urlparse
+from ..utils import (
+    int_or_none,
+    js_to_json,
+    parse_filesize,
+    str_to_int,
+)
+
+
+class PornComIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:[a-zA-Z]+\.)?porn\.com/videos/(?:(?P<display_id>[^/]+)-)?(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'http://www.porn.com/videos/teen-grabs-a-dildo-and-fucks-her-pussy-live-on-1hottie-i-rec-2603339',
+        'md5': '3f30ce76267533cd12ba999263156de7',
+        'info_dict': {
+            'id': '2603339',
+            'display_id': 'teen-grabs-a-dildo-and-fucks-her-pussy-live-on-1hottie-i-rec',
+            'ext': 'mp4',
+            'title': 'Teen grabs a dildo and fucks her pussy live on 1hottie, I rec',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 551,
+            'view_count': int,
+            'age_limit': 18,
+        },
+    }, {
+        'url': 'http://se.porn.com/videos/marsha-may-rides-seth-on-top-of-his-thick-cock-2658067',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        display_id = mobj.group('display_id') or video_id
+
+        webpage = self._download_webpage(url, display_id)
+
+        config = self._parse_json(
+            self._search_regex(
+                r'=\s*({.+?})\s*,\s*[\da-zA-Z_]+\s*=',
+                webpage, 'config', default='{}'),
+            display_id, transform_source=js_to_json, fatal=False)
+
+        if config:
+            title = config['title']
+            formats = [{
+                'url': stream['url'],
+                'format_id': stream.get('id'),
+                'height': int_or_none(self._search_regex(
+                    r'^(\d+)[pP]', stream.get('id') or '', 'height', default=None))
+            } for stream in config['streams'] if stream.get('url')]
+            thumbnail = (compat_urlparse.urljoin(
+                config['thumbCDN'], config['poster'])
+                if config.get('thumbCDN') and config.get('poster') else None)
+            duration = int_or_none(config.get('length'))
+        else:
+            title = self._search_regex(
+                (r'<title>([^<]+)</title>', r'<h1[^>]*>([^<]+)</h1>'),
+                webpage, 'title')
+            formats = [{
+                'url': compat_urlparse.urljoin(url, format_url),
+                'format_id': '%sp' % height,
+                'height': int(height),
+                'filesize_approx': parse_filesize(filesize),
+            } for format_url, height, filesize in re.findall(
+                r'<a[^>]+href="(/download/[^"]+)">MPEG4 (\d+)p<span[^>]*>(\d+\s+[a-zA-Z]+)<',
+                webpage)]
+            thumbnail = None
+            duration = None
+
+        self._sort_formats(formats)
+
+        view_count = str_to_int(self._search_regex(
+            r'class=["\']views["\'][^>]*><p>([\d,.]+)', webpage, 'view count'))
+
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            'title': title,
+            'thumbnail': thumbnail,
+            'duration': duration,
+            'view_count': view_count,
+            'formats': formats,
+            'age_limit': 18,
+        }

From 8b2dc4c3287e5e90f339af687f3a272818c94fea Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 18 Aug 2016 23:59:13 +0700
Subject: [PATCH 5823/6219] [options] Remove output template description from
 --help

Same reasons as for --format
---
 youtube_dl/options.py | 17 +----------------
 1 file changed, 1 insertion(+), 16 deletions(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index d32a9e32c..5d62deef4 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -628,22 +628,7 @@ def parseOpts(overrideArguments=None):
     filesystem.add_option(
         '-o', '--output',
         dest='outtmpl', metavar='TEMPLATE',
-        help=('Output filename template. Use %(title)s to get the title, '
-              '%(uploader)s for the uploader name, %(uploader_id)s for the uploader nickname if different, '
-              '%(autonumber)s to get an automatically incremented number, '
-              '%(ext)s for the filename extension, '
-              '%(format)s for the format description (like "22 - 1280x720" or "HD"), '
-              '%(format_id)s for the unique id of the format (like YouTube\'s itags: "137"), '
-              '%(upload_date)s for the upload date (YYYYMMDD), '
-              '%(extractor)s for the provider (youtube, metacafe, etc), '
-              '%(id)s for the video id, '
-              '%(playlist_title)s, %(playlist_id)s, or %(playlist)s (=title if present, ID otherwise) for the playlist the video is in, '
-              '%(playlist_index)s for the position in the playlist. '
-              '%(height)s and %(width)s for the width and height of the video format. '
-              '%(resolution)s for a textual description of the resolution of the video format. '
-              '%% for a literal percent. '
-              'Use - to output to stdout. Can also be used to download to a different directory, '
-              'for example with -o \'/my/downloads/%(uploader)s/%(title)s-%(id)s.%(ext)s\' .'))
+        help=('Output filename template, see the "OUTPUT TEMPLATE" for all the info'))
     filesystem.add_option(
         '--autonumber-size',
         dest='autonumber_size', metavar='NUMBER',

From 93a63b36f1c52a9981050e393d1876d6162abb49 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 19 Aug 2016 00:13:24 +0700
Subject: [PATCH 5824/6219] [ChangeLog] Actualize

---
 ChangeLog | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 354306a97..7e8bb834d 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,18 @@
+version <unreleased>
+
+Core
+- Remove output template description from --help
+* Recognize lowercase units in parse_filesize
+
+Extractors
++ [porncom] Add extractor for porn.com (#2251, #10251)
++ [generic] Add support for DBTV embeds
+* [vk:wallpost] Fix audio extraction for new site layout
+* [vk] Fix authentication
++ [hgtvcom:show] Add extractor for hgtv.com shows (#10365)
++ [discoverygo] Add support for another GO network sites
+
+
 version 2016.08.17
 
 Core

From bd1bcd3ea079889cfd7cd44c0ea750ac9d432e41 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 19 Aug 2016 00:15:12 +0700
Subject: [PATCH 5825/6219] release 2016.08.19

---
 .github/ISSUE_TEMPLATE.md |  6 +++---
 ChangeLog                 |  2 +-
 README.md                 | 28 ++--------------------------
 docs/supportedsites.md    |  2 ++
 youtube_dl/version.py     |  2 +-
 5 files changed, 9 insertions(+), 31 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index ae28d83d5..7af3c7099 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.08.17*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.08.17**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.08.19*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.08.19**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.08.17
+[debug] youtube-dl version 2016.08.19
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 7e8bb834d..e99ffcec6 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2016.08.19
 
 Core
 - Remove output template description from --help
diff --git a/README.md b/README.md
index cabbbef76..952db7abb 100644
--- a/README.md
+++ b/README.md
@@ -201,32 +201,8 @@ which means you can modify it, redistribute it or use it however you like.
     -a, --batch-file FILE            File containing URLs to download ('-' for
                                      stdin)
     --id                             Use only video ID in file name
-    -o, --output TEMPLATE            Output filename template. Use %(title)s to
-                                     get the title, %(uploader)s for the
-                                     uploader name, %(uploader_id)s for the
-                                     uploader nickname if different,
-                                     %(autonumber)s to get an automatically
-                                     incremented number, %(ext)s for the
-                                     filename extension, %(format)s for the
-                                     format description (like "22 - 1280x720" or
-                                     "HD"), %(format_id)s for the unique id of
-                                     the format (like YouTube's itags: "137"),
-                                     %(upload_date)s for the upload date
-                                     (YYYYMMDD), %(extractor)s for the provider
-                                     (youtube, metacafe, etc), %(id)s for the
-                                     video id, %(playlist_title)s,
-                                     %(playlist_id)s, or %(playlist)s (=title if
-                                     present, ID otherwise) for the playlist the
-                                     video is in, %(playlist_index)s for the
-                                     position in the playlist. %(height)s and
-                                     %(width)s for the width and height of the
-                                     video format. %(resolution)s for a textual
-                                     description of the resolution of the video
-                                     format. %% for a literal percent. Use - to
-                                     output to stdout. Can also be used to
-                                     download to a different directory, for
-                                     example with -o '/my/downloads/%(uploader)s
-                                     /%(title)s-%(id)s.%(ext)s' .
+    -o, --output TEMPLATE            Output filename template, see the "OUTPUT
+                                     TEMPLATE" for all the info
     --autonumber-size NUMBER         Specify the number of digits in
                                      %(autonumber)s when it is present in output
                                      filename template or --auto-number option
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 189b9301d..edf192138 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -279,6 +279,7 @@
  - **Helsinki**: helsinki.fi
  - **HentaiStigma**
  - **HGTV**
+ - **hgtv.com:show**
  - **HistoricFilms**
  - **history:topic**: History.com Topic
  - **hitbox**
@@ -523,6 +524,7 @@
  - **podomatic**
  - **Pokemon**
  - **PolskieRadio**
+ - **PornCom**
  - **PornHd**
  - **PornHub**: PornHub and Thumbzilla
  - **PornHubPlaylist**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index cf5950117..691f2c591 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.08.17'
+__version__ = '2016.08.19'

From 9e5751b9fe72f7425e4cb3f22a56b6a95b59e41d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 19 Aug 2016 01:13:45 +0700
Subject: [PATCH 5826/6219] [globo:article] Relax _VALID_URL and video id regex
 (Closes #10379)

---
 youtube_dl/extractor/globo.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/globo.py b/youtube_dl/extractor/globo.py
index 3de8356f6..dbacbfc61 100644
--- a/youtube_dl/extractor/globo.py
+++ b/youtube_dl/extractor/globo.py
@@ -396,12 +396,12 @@ class GloboIE(InfoExtractor):
 
 
 class GloboArticleIE(InfoExtractor):
-    _VALID_URL = 'https?://.+?\.globo\.com/(?:[^/]+/)*(?P<id>[^/]+)\.html'
+    _VALID_URL = 'https?://.+?\.globo\.com/(?:[^/]+/)*(?P<id>[^/]+)(?:\.html)?'
 
     _VIDEOID_REGEXES = [
         r'\bdata-video-id=["\'](\d{7,})',
         r'\bdata-player-videosids=["\'](\d{7,})',
-        r'\bvideosIDs\s*:\s*["\'](\d{7,})',
+        r'\bvideosIDs\s*:\s*["\']?(\d{7,})',
         r'\bdata-id=["\'](\d{7,})',
         r'<div[^>]+\bid=["\'](\d{7,})',
     ]
@@ -423,6 +423,9 @@ class GloboArticleIE(InfoExtractor):
     }, {
         'url': 'http://gshow.globo.com/programas/tv-xuxa/O-Programa/noticia/2014/01/xuxa-e-junno-namoram-muuuito-em-luau-de-zeze-di-camargo-e-luciano.html',
         'only_matching': True,
+    }, {
+        'url': 'http://oglobo.globo.com/rio/a-amizade-entre-um-entregador-de-farmacia-um-piano-19946271',
+        'only_matching': True,
     }]
 
     @classmethod

From e4659b45474acb563db0ab4284abdfc80837307e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 19 Aug 2016 20:37:17 +0800
Subject: [PATCH 5827/6219] [utils] Correct octal/hexadecimal number detection
 in js_to_json

---
 ChangeLog           | 6 ++++++
 test/test_utils.py  | 3 +++
 youtube_dl/utils.py | 6 +++---
 3 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index e99ffcec6..98a3dbca3 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Core
+* Fix js_to_json(): correct octal or hexadecimal number detection
+
+
 version 2016.08.19
 
 Core
diff --git a/test/test_utils.py b/test/test_utils.py
index cb578cd53..b83da93b4 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -712,6 +712,9 @@ class TestUtil(unittest.TestCase):
         inp = '''{"foo":101}'''
         self.assertEqual(js_to_json(inp), '''{"foo":101}''')
 
+        inp = '''{"duration": "00:01:07"}'''
+        self.assertEqual(js_to_json(inp), '''{"duration": "00:01:07"}''')
+
     def test_js_to_json_edgecases(self):
         on = js_to_json("{abc_def:'1\\'\\\\2\\\\\\'3\"4'}")
         self.assertEqual(json.loads(on), {"abc_def": "1'\\2\\'3\"4"})
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 35362e767..0c36c1b80 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2038,14 +2038,14 @@ def js_to_json(code):
             }.get(m.group(0), m.group(0)), v[1:-1])
 
         INTEGER_TABLE = (
-            (r'^0[xX][0-9a-fA-F]+', 16),
-            (r'^0+[0-7]+', 8),
+            (r'^(0[xX][0-9a-fA-F]+)\s*:?$', 16),
+            (r'^(0+[0-7]+)\s*:?$', 8),
         )
 
         for regex, base in INTEGER_TABLE:
             im = re.match(regex, v)
             if im:
-                i = int(im.group(0), base)
+                i = int(im.group(1), base)
                 return '"%d":' % i if v.endswith(':') else '%d' % i
 
         return '"%s"' % v

From b82232036a019e340b715779108c3f4caea8a78d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 19 Aug 2016 20:39:28 +0800
Subject: [PATCH 5828/6219] [n-tv.de] Fix extraction (closes #10331)

---
 ChangeLog                     | 3 +++
 youtube_dl/extractor/ntvde.py | 8 +++++---
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 98a3dbca3..6281fe325 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -3,6 +3,9 @@ version <unreleased>
 Core
 * Fix js_to_json(): correct octal or hexadecimal number detection
 
+Extractors
+* [n-tv.de] Fix extraction (#10331)
+
 
 version 2016.08.19
 
diff --git a/youtube_dl/extractor/ntvde.py b/youtube_dl/extractor/ntvde.py
index a83e85cb8..d28a81542 100644
--- a/youtube_dl/extractor/ntvde.py
+++ b/youtube_dl/extractor/ntvde.py
@@ -1,6 +1,8 @@
-# encoding: utf-8
+# coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..compat import compat_urlparse
 from ..utils import (
@@ -40,8 +42,8 @@ class NTVDeIE(InfoExtractor):
         timestamp = int_or_none(info.get('publishedDateAsUnixTimeStamp'))
         vdata = self._parse_json(self._search_regex(
             r'(?s)\$\(\s*"\#player"\s*\)\s*\.data\(\s*"player",\s*(\{.*?\})\);',
-            webpage, 'player data'),
-            video_id, transform_source=js_to_json)
+            webpage, 'player data'), video_id,
+            transform_source=lambda s: js_to_json(re.sub(r'advertising:\s*{[^}]+},', '', s)))
         duration = parse_duration(vdata.get('duration'))
 
         formats = []

From 55af45fcab4295a92d56180cdbebe7b47e094bc3 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 19 Aug 2016 23:12:30 +0800
Subject: [PATCH 5829/6219] [radiobremen] Update _TEST (closes #10337)

---
 youtube_dl/extractor/radiobremen.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/radiobremen.py b/youtube_dl/extractor/radiobremen.py
index 0cbb15f08..19a751da0 100644
--- a/youtube_dl/extractor/radiobremen.py
+++ b/youtube_dl/extractor/radiobremen.py
@@ -13,15 +13,15 @@ class RadioBremenIE(InfoExtractor):
     IE_NAME = 'radiobremen'
 
     _TEST = {
-        'url': 'http://www.radiobremen.de/mediathek/index.html?id=114720',
+        'url': 'http://www.radiobremen.de/mediathek/?id=141876',
         'info_dict': {
-            'id': '114720',
+            'id': '141876',
             'ext': 'mp4',
-            'duration': 1685,
+            'duration': 178,
             'width': 512,
-            'title': 'buten un binnen vom 22. Dezember',
+            'title': 'Druck auf Patrick Öztürk',
             'thumbnail': 're:https?://.*\.jpg$',
-            'description': 'Unter anderem mit diesen Themen: 45 Flüchtlinge sind in Worpswede angekommen +++ Freies Internet für alle: Bremer arbeiten an einem flächendeckenden W-Lan-Netzwerk +++ Aktivisten kämpfen für das Unibad +++ So war das Wetter 2014 +++',
+            'description': 'Gegen den SPD-Bürgerschaftsabgeordneten Patrick Öztürk wird wegen Beihilfe zum gewerbsmäßigen Betrug ermittelt. Am Donnerstagabend sollte er dem Vorstand des SPD-Unterbezirks Bremerhaven dazu Rede und Antwort stehen.',
         },
     }
 

From 520251c093f5e0fe6af5e57203a0452aef0682ac Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 19 Aug 2016 23:53:47 +0800
Subject: [PATCH 5830/6219] [extractor/common] Recognize m3u8 manifests in
 HTML5 multimedia tags

---
 ChangeLog                      |  1 +
 youtube_dl/extractor/common.py | 36 +++++++++++++++++++++++-----------
 2 files changed, 26 insertions(+), 11 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 6281fe325..450351231 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
 version <unreleased>
 
 Core
+* Support m3u8 manifests in HTML5 multimedia tags
 * Fix js_to_json(): correct octal or hexadecimal number detection
 
 Extractors
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 9427ff449..07d58afe7 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1695,7 +1695,7 @@ class InfoExtractor(object):
                         self.report_warning('Unknown MIME type %s in DASH manifest' % mime_type)
         return formats
 
-    def _parse_html5_media_entries(self, base_url, webpage):
+    def _parse_html5_media_entries(self, base_url, webpage, video_id, m3u8_id=None):
         def absolute_url(video_url):
             return compat_urlparse.urljoin(base_url, video_url)
 
@@ -1710,6 +1710,21 @@ class InfoExtractor(object):
                 return f
             return {}
 
+        def _media_formats(src, cur_media_type):
+            full_url = absolute_url(src)
+            if determine_ext(full_url) == 'm3u8':
+                is_plain_url = False
+                formats = self._extract_m3u8_formats(
+                    full_url, video_id, ext='mp4', entry_protocol='m3u8_native',
+                    m3u8_id=m3u8_id)
+            else:
+                is_plain_url = True
+                formats = [{
+                    'url': full_url,
+                    'vcodec': 'none' if cur_media_type == 'audio' else None,
+                }]
+            return is_plain_url, formats
+
         entries = []
         for media_tag, media_type, media_content in re.findall(r'(?s)(<(?P<tag>video|audio)[^>]*>)(.*?)</(?P=tag)>', webpage):
             media_info = {
@@ -1719,10 +1734,8 @@ class InfoExtractor(object):
             media_attributes = extract_attributes(media_tag)
             src = media_attributes.get('src')
             if src:
-                media_info['formats'].append({
-                    'url': absolute_url(src),
-                    'vcodec': 'none' if media_type == 'audio' else None,
-                })
+                _, formats = _media_formats(src)
+                media_info['formats'].extend(formats)
             media_info['thumbnail'] = media_attributes.get('poster')
             if media_content:
                 for source_tag in re.findall(r'<source[^>]+>', media_content):
@@ -1730,12 +1743,13 @@ class InfoExtractor(object):
                     src = source_attributes.get('src')
                     if not src:
                         continue
-                    f = parse_content_type(source_attributes.get('type'))
-                    f.update({
-                        'url': absolute_url(src),
-                        'vcodec': 'none' if media_type == 'audio' else None,
-                    })
-                    media_info['formats'].append(f)
+                    is_plain_url, formats = _media_formats(src, media_type)
+                    if is_plain_url:
+                        f = parse_content_type(source_attributes.get('type'))
+                        f.update(formats[0])
+                        media_info['formats'].append(f)
+                    else:
+                        media_info['formats'].extend(formats)
                 for track_tag in re.findall(r'<track[^>]+>', media_content):
                     track_attributes = extract_attributes(track_tag)
                     kind = track_attributes.get('kind')

From ecc90093f9c3793439832f4c9d279605da3489a7 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 19 Aug 2016 23:56:09 +0800
Subject: [PATCH 5831/6219] [vuclip] Adapt to the new API and update _TEST

---
 youtube_dl/extractor/vuclip.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/vuclip.py b/youtube_dl/extractor/vuclip.py
index b73da5cd0..55e087bdb 100644
--- a/youtube_dl/extractor/vuclip.py
+++ b/youtube_dl/extractor/vuclip.py
@@ -17,12 +17,12 @@ class VuClipIE(InfoExtractor):
     _VALID_URL = r'https?://(?:m\.)?vuclip\.com/w\?.*?cid=(?P<id>[0-9]+)'
 
     _TEST = {
-        'url': 'http://m.vuclip.com/w?cid=922692425&fid=70295&z=1010&nvar&frm=index.html',
+        'url': 'http://m.vuclip.com/w?cid=1129900602&bu=8589892792&frm=w&z=34801&op=0&oc=843169247&section=recommend',
         'info_dict': {
-            'id': '922692425',
+            'id': '1129900602',
             'ext': '3gp',
-            'title': 'The Toy Soldiers - Hollywood Movie Trailer',
-            'duration': 177,
+            'title': 'Top 10 TV Convicts',
+            'duration': 733,
         }
     }
 
@@ -54,7 +54,7 @@ class VuClipIE(InfoExtractor):
                 'url': video_url,
             }]
         else:
-            formats = self._parse_html5_media_entries(url, webpage)[0]['formats']
+            formats = self._parse_html5_media_entries(url, webpage, video_id)[0]['formats']
 
         title = remove_end(self._html_search_regex(
             r'<title>(.*?)-\s*Vuclip</title>', webpage, 'title').strip(), ' - Video')

From a9a3b4a081a6793f0dd0b40be8429a2aa3c1c36d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 20 Aug 2016 00:08:23 +0800
Subject: [PATCH 5832/6219] [miomio] Adapt to the new API and update _TESTS

The test case is from #9680
---
 youtube_dl/extractor/miomio.py | 19 ++++++-------------
 1 file changed, 6 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/miomio.py b/youtube_dl/extractor/miomio.py
index 937ba0f28..ec1b4c4fe 100644
--- a/youtube_dl/extractor/miomio.py
+++ b/youtube_dl/extractor/miomio.py
@@ -25,10 +25,7 @@ class MioMioIE(InfoExtractor):
             'title': '【SKY】字幕 铠武昭和VS平成 假面骑士大战FEAT战队 魔星字幕组 字幕',
             'duration': 5923,
         },
-        'params': {
-            # The server provides broken file
-            'skip_download': True,
-        }
+        'skip': 'Unable to load videos',
     }, {
         'url': 'http://www.miomio.tv/watch/cc184024/',
         'info_dict': {
@@ -47,16 +44,12 @@ class MioMioIE(InfoExtractor):
         'skip': 'Unable to load videos',
     }, {
         # new 'h5' player
-        'url': 'http://www.miomio.tv/watch/cc273295/',
-        'md5': '',
+        'url': 'http://www.miomio.tv/watch/cc273997/',
+        'md5': '0b27a4b4495055d826813f8c3a6b2070',
         'info_dict': {
-            'id': '273295',
+            'id': '273997',
             'ext': 'mp4',
-            'title': 'アウト×デラックス 20160526',
-        },
-        'params': {
-            # intermittent HTTP 500
-            'skip_download': True,
+            'title': 'マツコの知らない世界【劇的進化SP！ビニール傘＆冷凍食品2016】 1_2 - 16 05 31',
         },
     }]
 
@@ -116,7 +109,7 @@ class MioMioIE(InfoExtractor):
             player_webpage = self._download_webpage(
                 player_url, video_id,
                 note='Downloading player webpage', headers={'Referer': url})
-            entries = self._parse_html5_media_entries(player_url, player_webpage)
+            entries = self._parse_html5_media_entries(player_url, player_webpage, video_id)
             http_headers = {'Referer': player_url}
         else:
             http_headers = {'Referer': 'http://www.miomio.tv%s' % mioplayer_path}

From 70852b47ca101f0b4acc76eb3213b763a14b3602 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 20 Aug 2016 00:12:32 +0800
Subject: [PATCH 5833/6219] [utils] Recognize units with full names in
 parse_filename

Reference: https://en.wikipedia.org/wiki/Template:Quantities_of_bytes
---
 ChangeLog           |  4 +++-
 test/test_utils.py  |  1 +
 youtube_dl/utils.py | 17 +++++++++++++++++
 3 files changed, 21 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index 450351231..b36e4438c 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,7 +1,9 @@
 version <unreleased>
 
 Core
-* Support m3u8 manifests in HTML5 multimedia tags
++ Recognize file size strings with full unit names (for example "8.5
+  megabytes")
++ Support m3u8 manifests in HTML5 multimedia tags
 * Fix js_to_json(): correct octal or hexadecimal number detection
 
 Extractors
diff --git a/test/test_utils.py b/test/test_utils.py
index b83da93b4..d16ea7f77 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -823,6 +823,7 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(parse_filesize('1.2tb'), 1200000000000)
         self.assertEqual(parse_filesize('1,24 KB'), 1240)
         self.assertEqual(parse_filesize('1,24 kb'), 1240)
+        self.assertEqual(parse_filesize('8.5 megabytes'), 8500000)
 
     def test_parse_count(self):
         self.assertEqual(parse_count(None), None)
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 0c36c1b80..41ca562f1 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1504,46 +1504,63 @@ def parse_filesize(s):
     _UNIT_TABLE = {
         'B': 1,
         'b': 1,
+        'bytes': 1,
         'KiB': 1024,
         'KB': 1000,
         'kB': 1024,
         'Kb': 1000,
         'kb': 1000,
+        'kilobytes': 1000,
+        'kibibytes': 1024,
         'MiB': 1024 ** 2,
         'MB': 1000 ** 2,
         'mB': 1024 ** 2,
         'Mb': 1000 ** 2,
         'mb': 1000 ** 2,
+        'megabytes': 1000 ** 2,
+        'mebibytes': 1024 ** 2,
         'GiB': 1024 ** 3,
         'GB': 1000 ** 3,
         'gB': 1024 ** 3,
         'Gb': 1000 ** 3,
         'gb': 1000 ** 3,
+        'gigabytes': 1000 ** 3,
+        'gibibytes': 1024 ** 3,
         'TiB': 1024 ** 4,
         'TB': 1000 ** 4,
         'tB': 1024 ** 4,
         'Tb': 1000 ** 4,
         'tb': 1000 ** 4,
+        'terabytes': 1000 ** 4,
+        'tebibytes': 1024 ** 4,
         'PiB': 1024 ** 5,
         'PB': 1000 ** 5,
         'pB': 1024 ** 5,
         'Pb': 1000 ** 5,
         'pb': 1000 ** 5,
+        'petabytes': 1000 ** 5,
+        'pebibytes': 1024 ** 5,
         'EiB': 1024 ** 6,
         'EB': 1000 ** 6,
         'eB': 1024 ** 6,
         'Eb': 1000 ** 6,
         'eb': 1000 ** 6,
+        'exabytes': 1000 ** 6,
+        'exbibytes': 1024 ** 6,
         'ZiB': 1024 ** 7,
         'ZB': 1000 ** 7,
         'zB': 1024 ** 7,
         'Zb': 1000 ** 7,
         'zb': 1000 ** 7,
+        'zettabytes': 1000 ** 7,
+        'zebibytes': 1024 ** 7,
         'YiB': 1024 ** 8,
         'YB': 1000 ** 8,
         'yB': 1024 ** 8,
         'Yb': 1000 ** 8,
         'yb': 1000 ** 8,
+        'yottabytes': 1000 ** 8,
+        'yobibytes': 1024 ** 8,
     }
 
     return lookup_unit_table(_UNIT_TABLE, s)

From 19f35402c5296e93213d56034d85698087ce3fe1 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 20 Aug 2016 00:18:22 +0800
Subject: [PATCH 5834/6219] [snotr] Fix extraction (closes #10338)

---
 ChangeLog                     |  1 +
 youtube_dl/extractor/snotr.py | 38 +++++++++++++++++++----------------
 2 files changed, 22 insertions(+), 17 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index b36e4438c..13c3d3ffc 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -7,6 +7,7 @@ Core
 * Fix js_to_json(): correct octal or hexadecimal number detection
 
 Extractors
+* [snotr] Fix extraction (#10338)
 * [n-tv.de] Fix extraction (#10331)
 
 
diff --git a/youtube_dl/extractor/snotr.py b/youtube_dl/extractor/snotr.py
index 0d1ab07f8..3bb78cb84 100644
--- a/youtube_dl/extractor/snotr.py
+++ b/youtube_dl/extractor/snotr.py
@@ -5,9 +5,9 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
-    float_or_none,
-    str_to_int,
     parse_duration,
+    parse_filesize,
+    str_to_int,
 )
 
 
@@ -17,21 +17,24 @@ class SnotrIE(InfoExtractor):
         'url': 'http://www.snotr.com/video/13708/Drone_flying_through_fireworks',
         'info_dict': {
             'id': '13708',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Drone flying through fireworks!',
-            'duration': 247,
-            'filesize_approx': 98566144,
+            'duration': 248,
+            'filesize_approx': 40700000,
             'description': 'A drone flying through Fourth of July Fireworks',
-        }
+            'thumbnail': 're:^https?://.*\.jpg$',
+        },
+        'expected_warnings': ['description'],
     }, {
         'url': 'http://www.snotr.com/video/530/David_Letteman_-_George_W_Bush_Top_10',
         'info_dict': {
             'id': '530',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'David Letteman - George W. Bush Top 10',
             'duration': 126,
-            'filesize_approx': 8912896,
+            'filesize_approx': 8500000,
             'description': 'The top 10 George W. Bush moments, brought to you by David Letterman!',
+            'thumbnail': 're:^https?://.*\.jpg$',
         }
     }]
 
@@ -43,26 +46,27 @@ class SnotrIE(InfoExtractor):
         title = self._og_search_title(webpage)
 
         description = self._og_search_description(webpage)
-        video_url = 'http://cdn.videos.snotr.com/%s.flv' % video_id
+        info_dict = self._parse_html5_media_entries(url, webpage, video_id)[0]
 
         view_count = str_to_int(self._html_search_regex(
-            r'<p>\n<strong>Views:</strong>\n([\d,\.]+)</p>',
+            r'<p[^>]*>\s*<strong[^>]*>Views:</strong>\s*<span[^>]*>([\d,\.]+)',
             webpage, 'view count', fatal=False))
 
         duration = parse_duration(self._html_search_regex(
-            r'<p>\n<strong>Length:</strong>\n\s*([0-9:]+).*?</p>',
+            r'<p[^>]*>\s*<strong[^>]*>Length:</strong>\s*<span[^>]*>([\d:]+)',
             webpage, 'duration', fatal=False))
 
-        filesize_approx = float_or_none(self._html_search_regex(
-            r'<p>\n<strong>Filesize:</strong>\n\s*([0-9.]+)\s*megabyte</p>',
-            webpage, 'filesize', fatal=False), invscale=1024 * 1024)
+        filesize_approx = parse_filesize(self._html_search_regex(
+            r'<p[^>]*>\s*<strong[^>]*>Filesize:</strong>\s*<span[^>]*>([^<]+)',
+            webpage, 'filesize', fatal=False))
 
-        return {
+        info_dict.update({
             'id': video_id,
             'description': description,
             'title': title,
-            'url': video_url,
             'view_count': view_count,
             'duration': duration,
             'filesize_approx': filesize_approx,
-        }
+        })
+
+        return info_dict

From 39e1c4f08c4cfca81943e73523bd66b890f5aff2 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 20 Aug 2016 00:52:37 +0800
Subject: [PATCH 5835/6219] [litv] Support 'promo' URLs (closes #10385)

---
 ChangeLog                    |  1 +
 youtube_dl/extractor/litv.py | 24 ++++++++++++++++++++----
 2 files changed, 21 insertions(+), 4 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 13c3d3ffc..a8d8d05a3 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -7,6 +7,7 @@ Core
 * Fix js_to_json(): correct octal or hexadecimal number detection
 
 Extractors
++ [litv] Support 'promo' URLs (#10385)
 * [snotr] Fix extraction (#10338)
 * [n-tv.de] Fix extraction (#10331)
 
diff --git a/youtube_dl/extractor/litv.py b/youtube_dl/extractor/litv.py
index 3356d015d..05c6579f1 100644
--- a/youtube_dl/extractor/litv.py
+++ b/youtube_dl/extractor/litv.py
@@ -14,7 +14,7 @@ from ..utils import (
 
 
 class LiTVIE(InfoExtractor):
-    _VALID_URL = r'https?://www\.litv\.tv/vod/[^/]+/content\.do\?.*?\bid=(?P<id>[^&]+)'
+    _VALID_URL = r'https?://www\.litv\.tv/(?:vod|promo)/[^/]+/(?:content\.do)?\?.*?\b(?:content_)?id=(?P<id>[^&]+)'
 
     _URL_TEMPLATE = 'https://www.litv.tv/vod/%s/content.do?id=%s'
 
@@ -27,6 +27,7 @@ class LiTVIE(InfoExtractor):
         'playlist_count': 50,
     }, {
         'url': 'https://www.litv.tv/vod/drama/content.do?brc_id=root&id=VOD00041610&isUHEnabled=true&autoPlay=1',
+        'md5': '969e343d9244778cb29acec608e53640',
         'info_dict': {
             'id': 'VOD00041610',
             'ext': 'mp4',
@@ -37,7 +38,16 @@ class LiTVIE(InfoExtractor):
         },
         'params': {
             'noplaylist': True,
-            'skip_download': True,  # m3u8 download
+        },
+        'skip': 'Georestricted to Taiwan',
+    }, {
+        'url': 'https://www.litv.tv/promo/miyuezhuan/?content_id=VOD00044841&',
+        'md5': '88322ea132f848d6e3e18b32a832b918',
+        'info_dict': {
+            'id': 'VOD00044841',
+            'ext': 'mp4',
+            'title': '芈月傳第1集　霸星芈月降世楚國',
+            'description': '楚威王二年，太史令唐昧夜觀星象，發現霸星即將現世。王后得知霸星的預言後，想盡辦法不讓孩子順利出生，幸得莒姬相護化解危機。沒想到眾人期待下出生的霸星卻是位公主，楚威王對此失望至極。楚王后命人將女嬰丟棄河中，居然奇蹟似的被少司命像攔下，楚威王認為此女非同凡響，為她取名芈月。',
         },
         'skip': 'Georestricted to Taiwan',
     }]
@@ -92,13 +102,18 @@ class LiTVIE(InfoExtractor):
         # endpoint gives the same result as the data embedded in the webpage.
         # If georestricted, there are no embedded data, so an extra request is
         # necessary to get the error code
+        if 'assetId' not in view_data:
+            view_data = self._download_json(
+                'https://www.litv.tv/vod/ajax/getProgramInfo', video_id,
+                query={'contentId': video_id},
+                headers={'Accept': 'application/json'})
         video_data = self._parse_json(self._search_regex(
             r'uiHlsUrl\s*=\s*testBackendData\(([^;]+)\);',
             webpage, 'video data', default='{}'), video_id)
         if not video_data:
             payload = {
                 'assetId': view_data['assetId'],
-                'watchDevices': vod_data['watchDevices'],
+                'watchDevices': view_data['watchDevices'],
                 'contentType': view_data['contentType'],
             }
             video_data = self._download_json(
@@ -115,7 +130,8 @@ class LiTVIE(InfoExtractor):
             raise ExtractorError('Unexpected result from %s' % self.IE_NAME)
 
         formats = self._extract_m3u8_formats(
-            video_data['fullpath'], video_id, ext='mp4', m3u8_id='hls')
+            video_data['fullpath'], video_id, ext='mp4',
+            entry_protocol='m3u8_native', m3u8_id='hls')
         for a_format in formats:
             # LiTV HLS segments doesn't like compressions
             a_format.setdefault('http_headers', {})['Youtubedl-no-compression'] = True

From 5b9d187cc6545c0c5209a4db5525b1023ca8ea41 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?D=C3=A9stin=20Reed?= <trox1972@users.noreply.github.com>
Date: Fri, 19 Aug 2016 22:59:26 +0200
Subject: [PATCH 5836/6219] [imdb] Improve title extraction and make thumbnail
 non-fatal

---
 youtube_dl/extractor/imdb.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/imdb.py b/youtube_dl/extractor/imdb.py
index 0acce9f4c..3a6a6f5ad 100644
--- a/youtube_dl/extractor/imdb.py
+++ b/youtube_dl/extractor/imdb.py
@@ -6,6 +6,7 @@ from .common import InfoExtractor
 from ..utils import (
     mimetype2ext,
     qualities,
+    remove_end,
 )
 
 
@@ -19,7 +20,7 @@ class ImdbIE(InfoExtractor):
         'info_dict': {
             'id': '2524815897',
             'ext': 'mp4',
-            'title': 'Ice Age: Continental Drift Trailer (No. 2) - IMDb',
+            'title': 'Ice Age: Continental Drift Trailer (No. 2)',
             'description': 'md5:9061c2219254e5d14e03c25c98e96a81',
         }
     }, {
@@ -83,10 +84,10 @@ class ImdbIE(InfoExtractor):
 
         return {
             'id': video_id,
-            'title': self._og_search_title(webpage),
+            'title': remove_end(self._og_search_title(webpage), ' - IMDb'),
             'formats': formats,
             'description': descr,
-            'thumbnail': format_info['slate'],
+            'thumbnail': format_info.get('slate'),
         }
 
 

From 4245f55880c42e670cebd5a8a2b10929be834682 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 20 Aug 2016 06:18:20 +0700
Subject: [PATCH 5837/6219] [dotsub] Replace test (Closes #10386)

---
 youtube_dl/extractor/dotsub.py | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/dotsub.py b/youtube_dl/extractor/dotsub.py
index e9ca236d4..fd64d1a7f 100644
--- a/youtube_dl/extractor/dotsub.py
+++ b/youtube_dl/extractor/dotsub.py
@@ -10,18 +10,18 @@ from ..utils import (
 class DotsubIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?dotsub\.com/view/(?P<id>[^/]+)'
     _TEST = {
-        'url': 'http://dotsub.com/view/aed3b8b2-1889-4df5-ae63-ad85f5572f27',
-        'md5': '0914d4d69605090f623b7ac329fea66e',
+        'url': 'https://dotsub.com/view/9c63db2a-fa95-4838-8e6e-13deafe47f09',
+        'md5': '21c7ff600f545358134fea762a6d42b6',
         'info_dict': {
-            'id': 'aed3b8b2-1889-4df5-ae63-ad85f5572f27',
+            'id': '9c63db2a-fa95-4838-8e6e-13deafe47f09',
             'ext': 'flv',
-            'title': 'Pyramids of Waste (2010), AKA The Lightbulb Conspiracy - Planned obsolescence documentary',
-            'description': 'md5:699a0f7f50aeec6042cb3b1db2d0d074',
-            'thumbnail': 're:^https?://dotsub.com/media/aed3b8b2-1889-4df5-ae63-ad85f5572f27/p',
-            'duration': 3169,
-            'uploader': '4v4l0n42',
-            'timestamp': 1292248482.625,
-            'upload_date': '20101213',
+            'title': 'MOTIVATION - "It\'s Possible" Best Inspirational Video Ever',
+            'description': 'md5:41af1e273edbbdfe4e216a78b9d34ac6',
+            'thumbnail': 're:^https?://dotsub.com/media/9c63db2a-fa95-4838-8e6e-13deafe47f09/p',
+            'duration': 198,
+            'uploader': 'liuxt',
+            'timestamp': 1385778501.104,
+            'upload_date': '20131130',
             'view_count': int,
         }
     }

From dabe15701b3c12ef7e6af1f3333e1d3e39149592 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 20 Aug 2016 13:25:32 +0100
Subject: [PATCH 5838/6219] [cbs, cbsnews] fix extraction(fixes #10393)

---
 youtube_dl/extractor/cbs.py       | 47 ++++++++++++++++++++-----------
 youtube_dl/extractor/cbsnews.py   |  9 +++---
 youtube_dl/extractor/cbssports.py |  3 ++
 3 files changed, 38 insertions(+), 21 deletions(-)

diff --git a/youtube_dl/extractor/cbs.py b/youtube_dl/extractor/cbs.py
index a23173d6f..c72ed2dbb 100644
--- a/youtube_dl/extractor/cbs.py
+++ b/youtube_dl/extractor/cbs.py
@@ -4,6 +4,7 @@ from .theplatform import ThePlatformFeedIE
 from ..utils import (
     int_or_none,
     find_xpath_attr,
+    ExtractorError,
 )
 
 
@@ -17,19 +18,6 @@ class CBSBaseIE(ThePlatformFeedIE):
             }]
         } if closed_caption_e is not None and closed_caption_e.attrib.get('value') else []
 
-    def _extract_video_info(self, filter_query, video_id):
-        return self._extract_feed_info(
-            'dJ5BDC', 'VxxJg8Ymh8sE', filter_query, video_id, lambda entry: {
-                'series': entry.get('cbs$SeriesTitle'),
-                'season_number': int_or_none(entry.get('cbs$SeasonNumber')),
-                'episode': entry.get('cbs$EpisodeTitle'),
-                'episode_number': int_or_none(entry.get('cbs$EpisodeNumber')),
-            }, {
-                'StreamPack': {
-                    'manifest': 'm3u',
-                }
-            })
-
 
 class CBSIE(CBSBaseIE):
     _VALID_URL = r'(?:cbs:|https?://(?:www\.)?(?:cbs\.com/shows/[^/]+/video|colbertlateshow\.com/(?:video|podcasts))/)(?P<id>[\w-]+)'
@@ -38,7 +26,6 @@ class CBSIE(CBSBaseIE):
         'url': 'http://www.cbs.com/shows/garth-brooks/video/_u7W953k6la293J7EPTd9oHkSPs6Xn6_/connect-chat-feat-garth-brooks/',
         'info_dict': {
             'id': '_u7W953k6la293J7EPTd9oHkSPs6Xn6_',
-            'display_id': 'connect-chat-feat-garth-brooks',
             'ext': 'mp4',
             'title': 'Connect Chat feat. Garth Brooks',
             'description': 'Connect with country music singer Garth Brooks, as he chats with fans on Wednesday November 27, 2013. Be sure to tune in to Garth Brooks: Live from Las Vegas, Friday November 29, at 9/8c on CBS!',
@@ -47,7 +34,10 @@ class CBSIE(CBSBaseIE):
             'upload_date': '20131127',
             'uploader': 'CBSI-NEW',
         },
-        'expected_warnings': ['Failed to download m3u8 information'],
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
         '_skip': 'Blocked outside the US',
     }, {
         'url': 'http://colbertlateshow.com/video/8GmB0oY0McANFvp2aEffk9jZZZ2YyXxy/the-colbeard/',
@@ -56,8 +46,31 @@ class CBSIE(CBSBaseIE):
         'url': 'http://www.colbertlateshow.com/podcasts/dYSwjqPs_X1tvbV_P2FcPWRa_qT6akTC/in-the-bad-room-with-stephen/',
         'only_matching': True,
     }]
-    TP_RELEASE_URL_TEMPLATE = 'http://link.theplatform.com/s/dJ5BDC/%s?mbr=true'
+
+    def _extract_video_info(self, guid):
+        path = 'dJ5BDC/media/guid/2198311517/' + guid
+        smil_url = 'http://link.theplatform.com/s/%s?mbr=true' % path
+        formats, subtitles = self._extract_theplatform_smil(smil_url + '&manifest=m3u', guid)
+        for r in ('HLS&formats=M3U', 'RTMP', 'WIFI', '3G'):
+            try:
+                tp_formats, _ = self._extract_theplatform_smil(smil_url + '&assetTypes=' + r, guid, 'Downloading %s SMIL data' % r.split('&')[0])
+                formats.extend(tp_formats)
+            except ExtractorError:
+                continue
+        self._sort_formats(formats)
+        metadata = self._download_theplatform_metadata(path, guid)
+        info = self._parse_theplatform_metadata(metadata)
+        info.update({
+            'id': guid,
+            'formats': formats,
+            'subtitles': subtitles,
+            'series': metadata.get('cbs$SeriesTitle'),
+            'season_number': int_or_none(metadata.get('cbs$SeasonNumber')),
+            'episode': metadata.get('cbs$EpisodeTitle'),
+            'episode_number': int_or_none(metadata.get('cbs$EpisodeNumber')),
+        })
+        return info
 
     def _real_extract(self, url):
         content_id = self._match_id(url)
-        return self._extract_video_info('byGuid=%s' % content_id, content_id)
+        return self._extract_video_info(content_id)
diff --git a/youtube_dl/extractor/cbsnews.py b/youtube_dl/extractor/cbsnews.py
index 9d3b75526..4aa6917a0 100644
--- a/youtube_dl/extractor/cbsnews.py
+++ b/youtube_dl/extractor/cbsnews.py
@@ -2,13 +2,13 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from .cbs import CBSBaseIE
+from .cbs import CBSIE
 from ..utils import (
     parse_duration,
 )
 
 
-class CBSNewsIE(CBSBaseIE):
+class CBSNewsIE(CBSIE):
     IE_DESC = 'CBS News'
     _VALID_URL = r'https?://(?:www\.)?cbsnews\.com/(?:news|videos)/(?P<id>[\da-z_-]+)'
 
@@ -35,7 +35,8 @@ class CBSNewsIE(CBSBaseIE):
                 'ext': 'mp4',
                 'title': 'Fort Hood shooting: Army downplays mental illness as cause of attack',
                 'description': 'md5:4a6983e480542d8b333a947bfc64ddc7',
-                'upload_date': '19700101',
+                'upload_date': '20140404',
+                'timestamp': 1396650660,
                 'uploader': 'CBSI-NEW',
                 'thumbnail': 're:^https?://.*\.jpg$',
                 'duration': 205,
@@ -63,7 +64,7 @@ class CBSNewsIE(CBSBaseIE):
 
         item = video_info['item'] if 'item' in video_info else video_info
         guid = item['mpxRefId']
-        return self._extract_video_info('byGuid=%s' % guid, guid)
+        return self._extract_video_info(guid)
 
 
 class CBSNewsLiveVideoIE(InfoExtractor):
diff --git a/youtube_dl/extractor/cbssports.py b/youtube_dl/extractor/cbssports.py
index 78ca44b02..bf7915626 100644
--- a/youtube_dl/extractor/cbssports.py
+++ b/youtube_dl/extractor/cbssports.py
@@ -23,6 +23,9 @@ class CBSSportsIE(CBSBaseIE):
         }
     }]
 
+    def _extract_video_info(self, filter_query, video_id):
+        return self._extract_feed_info('dJ5BDC', 'VxxJg8Ymh8sE', filter_query, video_id)
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
         return self._extract_video_info('byId=%s' % video_id, video_id)

From 292a2301bf0b99be81640c4511d78ebc3c622dad Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 20 Aug 2016 19:00:25 +0100
Subject: [PATCH 5839/6219] [cnn] add support for money.cnn.com videos(closes
 #2797)

---
 youtube_dl/extractor/cnn.py | 41 ++++++++++++++++++++++++++++++-------
 1 file changed, 34 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/cnn.py b/youtube_dl/extractor/cnn.py
index 53489a14e..220bb55e8 100644
--- a/youtube_dl/extractor/cnn.py
+++ b/youtube_dl/extractor/cnn.py
@@ -11,7 +11,7 @@ from ..utils import (
 
 
 class CNNIE(InfoExtractor):
-    _VALID_URL = r'''(?x)https?://(?:(?:edition|www)\.)?cnn\.com/video/(?:data/.+?|\?)/
+    _VALID_URL = r'''(?x)https?://(?:(?P<sub_domain>edition|www|money)\.)?cnn\.com/(?:video/(?:data/.+?|\?)/)?videos?/
         (?P<path>.+?/(?P<title>[^/]+?)(?:\.(?:[a-z\-]+)|(?=&)))'''
 
     _TESTS = [{
@@ -45,19 +45,46 @@ class CNNIE(InfoExtractor):
             'description': 'md5:e7223a503315c9f150acac52e76de086',
             'upload_date': '20141222',
         }
+    }, {
+        'url': 'http://money.cnn.com/video/news/2016/08/19/netflix-stunning-stats.cnnmoney/index.html',
+        'md5': '52a515dc1b0f001cd82e4ceda32be9d1',
+        'info_dict': {
+            'id': '/video/news/2016/08/19/netflix-stunning-stats.cnnmoney',
+            'ext': 'mp4',
+            'title': '5 stunning stats about Netflix',
+            'description': 'Did you know that Netflix has more than 80 million members? Here are five facts about the online video distributor that you probably didn\'t know.',
+            'upload_date': '20160819',
+        }
     }, {
         'url': 'http://cnn.com/video/?/video/politics/2015/03/27/pkg-arizona-senator-church-attendance-mandatory.ktvk',
         'only_matching': True,
     }, {
         'url': 'http://cnn.com/video/?/video/us/2015/04/06/dnt-baker-refuses-anti-gay-order.wkmg',
         'only_matching': True,
+    }, {
+        'url': 'http://edition.cnn.com/videos/arts/2016/04/21/olympic-games-cultural-a-z-brazil.cnn',
+        'only_matching': True,
     }]
 
+    _CONFIG = {
+        # http://edition.cnn.com/.element/apps/cvp/3.0/cfg/spider/cnn/expansion/config.xml
+        'edition': {
+            'data_src': 'http://edition.cnn.com/video/data/3.0/video/%s/index.xml',
+            'media_src': 'http://pmd.cdn.turner.com/cnn/big',
+        },
+        # http://money.cnn.com/.element/apps/cvp2/cfg/config.xml
+        'money': {
+            'data_src': 'http://money.cnn.com/video/data/4.0/video/%s.xml',
+            'media_src': 'http://ht3.cdn.turner.com/money/big',
+        },
+    }
+
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        path = mobj.group('path')
-        page_title = mobj.group('title')
-        info_url = 'http://edition.cnn.com/video/data/3.0/%s/index.xml' % path
+        sub_domain, path, page_title = re.match(self._VALID_URL, url).groups()
+        if sub_domain not in ('money', 'edition'):
+            sub_domain = 'edition'
+        config = self._CONFIG[sub_domain]
+        info_url = config['data_src'] % path
         info = self._download_xml(info_url, page_title)
 
         formats = []
@@ -66,7 +93,7 @@ class CNNIE(InfoExtractor):
             (?:_(?P<bitrate>[0-9]+)k)?
         ''')
         for f in info.findall('files/file'):
-            video_url = 'http://ht.cdn.turner.com/cnn/big%s' % (f.text.strip())
+            video_url = config['media_src'] + f.text.strip()
             fdct = {
                 'format_id': f.attrib['bitrate'],
                 'url': video_url,
@@ -146,7 +173,7 @@ class CNNBlogsIE(InfoExtractor):
 
 
 class CNNArticleIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:(?:edition|www)\.)?cnn\.com/(?!video/)'
+    _VALID_URL = r'https?://(?:(?:edition|www)\.)?cnn\.com/(?!videos?/)'
     _TEST = {
         'url': 'http://www.cnn.com/2014/12/21/politics/obama-north-koreas-hack-not-war-but-cyber-vandalism/',
         'md5': '689034c2a3d9c6dc4aa72d65a81efd01',

From e25586e47163c83e519ae0af9aa6d8fbc3d58ef4 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 20 Aug 2016 20:02:49 +0100
Subject: [PATCH 5840/6219] [cultureunplugged] fix extraction(closes #10330)

---
 youtube_dl/extractor/cultureunplugged.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/cultureunplugged.py b/youtube_dl/extractor/cultureunplugged.py
index 9c764fe68..9f26fa587 100644
--- a/youtube_dl/extractor/cultureunplugged.py
+++ b/youtube_dl/extractor/cultureunplugged.py
@@ -1,9 +1,13 @@
 from __future__ import unicode_literals
 
 import re
+import time
 
 from .common import InfoExtractor
-from ..utils import int_or_none
+from ..utils import (
+    int_or_none,
+    HEADRequest,
+)
 
 
 class CultureUnpluggedIE(InfoExtractor):
@@ -32,6 +36,9 @@ class CultureUnpluggedIE(InfoExtractor):
         video_id = mobj.group('id')
         display_id = mobj.group('display_id') or video_id
 
+        # request setClientTimezone.php to get PHPSESSID cookie which is need to get valid json data in the next request
+        self._request_webpage(HEADRequest(
+            'http://www.cultureunplugged.com/setClientTimezone.php?timeOffset=%d' % -(time.timezone / 3600)), display_id)
         movie_data = self._download_json(
             'http://www.cultureunplugged.com/movie-data/cu-%s.json' % video_id, display_id)
 

From 5b1d85754ee2f1a8b94c979bc5122b4130ef8cc7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 21 Aug 2016 08:07:26 +0700
Subject: [PATCH 5841/6219] [YoutubeDL] Autocalculate ext when ext is None

---
 youtube_dl/YoutubeDL.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index e844dc98a..0b3e3da82 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1299,7 +1299,7 @@ class YoutubeDL(object):
                 for subtitle_format in subtitle:
                     if subtitle_format.get('url'):
                         subtitle_format['url'] = sanitize_url(subtitle_format['url'])
-                    if 'ext' not in subtitle_format:
+                    if subtitle_format.get('ext') is None:
                         subtitle_format['ext'] = determine_ext(subtitle_format['url']).lower()
 
         if self.params.get('listsubtitles', False):
@@ -1354,7 +1354,7 @@ class YoutubeDL(object):
                     note=' ({0})'.format(format['format_note']) if format.get('format_note') is not None else '',
                 )
             # Automatically determine file extension if missing
-            if 'ext' not in format:
+            if format.get('ext') is None:
                 format['ext'] = determine_ext(format['url']).lower()
             # Automatically determine protocol if missing (useful for format
             # selection purposes)

From d8f30a7e6606d2300dfffb8fc0aaf8d6a0c79b0f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 21 Aug 2016 08:10:49 +0700
Subject: [PATCH 5842/6219] [kaltura] Remove unused code

---
 youtube_dl/extractor/kaltura.py | 12 ------------
 1 file changed, 12 deletions(-)

diff --git a/youtube_dl/extractor/kaltura.py b/youtube_dl/extractor/kaltura.py
index ddf1165ff..66c7b36bc 100644
--- a/youtube_dl/extractor/kaltura.py
+++ b/youtube_dl/extractor/kaltura.py
@@ -122,18 +122,6 @@ class KalturaIE(InfoExtractor):
 
         return data
 
-    def _get_kaltura_signature(self, video_id, partner_id, service_url=None):
-        actions = [{
-            'apiVersion': '3.1',
-            'expiry': 86400,
-            'format': 1,
-            'service': 'session',
-            'action': 'startWidgetSession',
-            'widgetId': '_%s' % partner_id,
-        }]
-        return self._kaltura_api_call(
-            video_id, actions, service_url, note='Downloading Kaltura signature')['ks']
-
     def _get_video_info(self, video_id, partner_id, service_url=None):
         actions = [
             {

From a80944675010617cc0124c57ab597f9d9004c0d9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 21 Aug 2016 08:26:45 +0700
Subject: [PATCH 5843/6219] [kaltura] Add subtitles support when entry_id is
 unknown beforehand (Closes #10279)

---
 youtube_dl/extractor/kaltura.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/youtube_dl/extractor/kaltura.py b/youtube_dl/extractor/kaltura.py
index 66c7b36bc..a8ce6dda2 100644
--- a/youtube_dl/extractor/kaltura.py
+++ b/youtube_dl/extractor/kaltura.py
@@ -196,6 +196,17 @@ class KalturaIE(InfoExtractor):
                     reference_id)['entryResult']
                 info, flavor_assets = entry_data['meta'], entry_data['contextData']['flavorAssets']
                 entry_id = info['id']
+                # Unfortunately, data returned in kalturaIframePackageData lacks
+                # captions so we will try requesting the complete data using
+                # regular approach since we now know the entry_id
+                try:
+                    _, info, flavor_assets, captions = self._get_video_info(
+                        entry_id, partner_id)
+                except ExtractorError:
+                    # Regular scenario failed but we already have everything
+                    # extracted apart from captions and can process at least
+                    # with this
+                    pass
             else:
                 raise ExtractorError('Invalid URL', expected=True)
             ks = params.get('flashvars[ks]', [None])[0]

From fddaa76a599a7df00dc94dd5663d43c881f8fee0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 21 Aug 2016 08:27:53 +0700
Subject: [PATCH 5844/6219] [kaltura] Assume ttml to be default subtitles'
 extension

---
 youtube_dl/extractor/kaltura.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/kaltura.py b/youtube_dl/extractor/kaltura.py
index a8ce6dda2..15f2fe24f 100644
--- a/youtube_dl/extractor/kaltura.py
+++ b/youtube_dl/extractor/kaltura.py
@@ -266,7 +266,7 @@ class KalturaIE(InfoExtractor):
                     continue
                 subtitles.setdefault(caption.get('languageCode') or caption.get('language'), []).append({
                     'url': '%s/api_v3/service/caption_captionasset/action/serve/captionAssetId/%s' % (self._SERVICE_URL, caption['id']),
-                    'ext': caption.get('fileExt'),
+                    'ext': caption.get('fileExt', 'ttml'),
                 })
 
         return {

From 2c6acdfd2d31b7ce9500e9efe411620c61059b98 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 21 Aug 2016 08:37:01 +0700
Subject: [PATCH 5845/6219] [kaltura] Add test for #10279

---
 youtube_dl/extractor/kaltura.py | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/youtube_dl/extractor/kaltura.py b/youtube_dl/extractor/kaltura.py
index 15f2fe24f..e0f7366c2 100644
--- a/youtube_dl/extractor/kaltura.py
+++ b/youtube_dl/extractor/kaltura.py
@@ -67,6 +67,27 @@ class KalturaIE(InfoExtractor):
             # video with subtitles
             'url': 'kaltura:111032:1_cw786r8q',
             'only_matching': True,
+        },
+        {
+            # video with ttml subtitles (no fileExt)
+            'url': 'kaltura:1926081:0_l5ye1133',
+            'info_dict': {
+                'id': '0_l5ye1133',
+                'ext': 'mp4',
+                'title': 'What Can You Do With Python?',
+                'upload_date': '20160221',
+                'uploader_id': 'stork',
+                'thumbnail': 're:^https?://.*/thumbnail/.*',
+                'timestamp': int,
+                'subtitles': {
+                    'en': [{
+                        'ext': 'ttml',
+                    }],
+                },
+            },
+            'params': {
+                'skip_download': True,
+            },
         }
     ]
 

From db29af6d36b3d16614355dac70f22c4f2d8410d2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?D=C3=A9stin=20Reed?= <trox1972@users.noreply.github.com>
Date: Fri, 19 Aug 2016 12:53:34 +0200
Subject: [PATCH 5846/6219] [charlierose] Add new extractor

---
 youtube_dl/extractor/charlierose.py | 45 +++++++++++++++++++++++++++++
 youtube_dl/extractor/extractors.py  |  1 +
 2 files changed, 46 insertions(+)
 create mode 100644 youtube_dl/extractor/charlierose.py

diff --git a/youtube_dl/extractor/charlierose.py b/youtube_dl/extractor/charlierose.py
new file mode 100644
index 000000000..ba1d1b833
--- /dev/null
+++ b/youtube_dl/extractor/charlierose.py
@@ -0,0 +1,45 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import remove_end
+
+
+class CharlieRoseIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?charlierose\.com/video(?:s|/player)/(?P<id>\d+)'
+    _TEST = {
+        'url': 'https://charlierose.com/videos/27996',
+        'info_dict': {
+            'id': '27996',
+            'ext': 'mp4',
+            'title': 'Remembering Zaha Hadid',
+            'thumbnail': 're:^https?://.*\.jpg\?\d+',
+            'description': 'We revisit past conversations with Zaha Hadid, in memory of the world renowned Iraqi architect.',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        }
+    }
+
+    _PLAYER_BASE = 'https://charlierose.com/video/player/%s'
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(self._PLAYER_BASE % video_id, video_id)
+
+        title = remove_end(self._og_search_title(webpage), ' - Charlie Rose')
+
+        entries = self._parse_html5_media_entries(self._PLAYER_BASE % video_id, webpage, video_id)[0]
+        formats = entries['formats']
+
+        self._sort_formats(formats)
+        self._remove_duplicate_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': formats,
+            'thumbnail': self._og_search_thumbnail(webpage),
+            'description': self._og_search_description(webpage),
+            'subtitles': entries.get('subtitles'),
+        }
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 6c5d46015..d4d90c1f8 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -134,6 +134,7 @@ from .ccc import CCCIE
 from .cda import CDAIE
 from .ceskatelevize import CeskaTelevizeIE
 from .channel9 import Channel9IE
+from .charlierose import CharlieRoseIE
 from .chaturbate import ChaturbateIE
 from .chilloutzone import ChilloutzoneIE
 from .chirbit import (

From d164a0d41bdc95caa2b1458b9f51381de7d6a5a7 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 21 Aug 2016 20:00:48 +0800
Subject: [PATCH 5847/6219] [README.md] Add a format selection example using
 comma

Ref: #10399
---
 README.md | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/README.md b/README.md
index 952db7abb..a10aaf35c 100644
--- a/README.md
+++ b/README.md
@@ -645,7 +645,11 @@ $ youtube-dl -f 'best[filesize<50M]'
 
 # Download best format available via direct link over HTTP/HTTPS protocol
 $ youtube-dl -f '(bestvideo+bestaudio/best)[protocol^=http]'
+
+# Download the best video format and the best audio format without merging them
+$ youtube-dl -f 'bestvideo,bestaudio' -o '%(title)s.f%(format_id)s.%(ext)s'
 ```
+Note that in the last example, an output template is recommended as bestvideo and bestaudio may have the same file name.
 
 
 # VIDEO SELECTION

From 3d47ee0a9eb37b2c91dfae80c7f22fda0242dd61 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 21 Aug 2016 14:09:18 +0100
Subject: [PATCH 5848/6219] [zingmp3] fix extraction and add support for video
 clips(closes #10041)

---
 youtube_dl/extractor/extractors.py |   5 +-
 youtube_dl/extractor/zingmp3.py    | 127 +++++++++++++++++------------
 2 files changed, 76 insertions(+), 56 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 6c5d46015..20fb23527 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1114,7 +1114,4 @@ from .youtube import (
 )
 from .zapiks import ZapiksIE
 from .zdf import ZDFIE, ZDFChannelIE
-from .zingmp3 import (
-    ZingMp3SongIE,
-    ZingMp3AlbumIE,
-)
+from .zingmp3 import ZingMp3IE
diff --git a/youtube_dl/extractor/zingmp3.py b/youtube_dl/extractor/zingmp3.py
index 437eecb67..bd708b42c 100644
--- a/youtube_dl/extractor/zingmp3.py
+++ b/youtube_dl/extractor/zingmp3.py
@@ -4,13 +4,17 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import ExtractorError
+from ..utils import (
+    ExtractorError,
+    int_or_none,
+    update_url_query,
+)
 
 
 class ZingMp3BaseInfoExtractor(InfoExtractor):
 
-    def _extract_item(self, item, fatal=True):
-        error_message = item.find('./errormessage').text
+    def _extract_item(self, item, page_type, fatal=True):
+        error_message = item.get('msg')
         if error_message:
             if not fatal:
                 return
@@ -18,25 +22,48 @@ class ZingMp3BaseInfoExtractor(InfoExtractor):
                 '%s returned error: %s' % (self.IE_NAME, error_message),
                 expected=True)
 
-        title = item.find('./title').text.strip()
-        source = item.find('./source').text
-        extension = item.attrib['type']
-        thumbnail = item.find('./backimage').text
+        formats = []
+        for quality, source_url in zip(item.get('qualities') or item.get('quality', []), item.get('source_list') or item.get('source', [])):
+            if not source_url or source_url == 'require vip':
+                continue
+            if not re.match(r'https?://', source_url):
+                source_url = '//' + source_url
+            source_url = self._proto_relative_url(source_url, 'http:')
+            quality_num = int_or_none(quality)
+            f = {
+                'format_id': quality,
+                'url': source_url,
+            }
+            if page_type == 'video':
+                f.update({
+                    'height': quality_num,
+                    'ext': 'mp4',
+                })
+            else:
+                f.update({
+                    'abr': quality_num,
+                    'ext': 'mp3',
+                })
+            formats.append(f)
+
+        cover = item.get('cover')
 
         return {
-            'title': title,
-            'url': source,
-            'ext': extension,
-            'thumbnail': thumbnail,
+            'title': (item.get('name') or item.get('title')).strip(),
+            'formats': formats,
+            'thumbnail': 'http:/' + cover if cover else None,
+            'artist': item.get('artist'),
         }
 
-    def _extract_player_xml(self, player_xml_url, id, playlist_title=None):
-        player_xml = self._download_xml(player_xml_url, id, 'Downloading Player XML')
-        items = player_xml.findall('./item')
+    def _extract_player_json(self, player_json_url, id, page_type, playlist_title=None):
+        player_json = self._download_json(player_json_url, id, 'Downloading Player JSON')
+        items = player_json['data']
+        if 'item' in items:
+            items = items['item']
 
         if len(items) == 1:
             # one single song
-            data = self._extract_item(items[0])
+            data = self._extract_item(items[0], page_type)
             data['id'] = id
 
             return data
@@ -45,7 +72,7 @@ class ZingMp3BaseInfoExtractor(InfoExtractor):
             entries = []
 
             for i, item in enumerate(items, 1):
-                entry = self._extract_item(item, fatal=False)
+                entry = self._extract_item(item, page_type, fatal=False)
                 if not entry:
                     continue
                 entry['id'] = '%s-%d' % (id, i)
@@ -59,8 +86,8 @@ class ZingMp3BaseInfoExtractor(InfoExtractor):
             }
 
 
-class ZingMp3SongIE(ZingMp3BaseInfoExtractor):
-    _VALID_URL = r'https?://mp3\.zing\.vn/bai-hat/(?P<slug>[^/]+)/(?P<song_id>\w+)\.html'
+class ZingMp3IE(ZingMp3BaseInfoExtractor):
+    _VALID_URL = r'https?://mp3\.zing\.vn/(?:bai-hat|album|playlist|video-clip)/[^/]+/(?P<id>\w+)\.html'
     _TESTS = [{
         'url': 'http://mp3.zing.vn/bai-hat/Xa-Mai-Xa-Bao-Thy/ZWZB9WAB.html',
         'md5': 'ead7ae13693b3205cbc89536a077daed',
@@ -70,51 +97,47 @@ class ZingMp3SongIE(ZingMp3BaseInfoExtractor):
             'ext': 'mp3',
             'thumbnail': 're:^https?://.*\.jpg$',
         },
-    }]
-    IE_NAME = 'zingmp3:song'
-    IE_DESC = 'mp3.zing.vn songs'
-
-    def _real_extract(self, url):
-        matched = re.match(self._VALID_URL, url)
-        slug = matched.group('slug')
-        song_id = matched.group('song_id')
-
-        webpage = self._download_webpage(
-            'http://mp3.zing.vn/bai-hat/%s/%s.html' % (slug, song_id), song_id)
-
-        player_xml_url = self._search_regex(
-            r'&amp;xmlURL=(?P<xml_url>[^&]+)&', webpage, 'player xml url')
-
-        return self._extract_player_xml(player_xml_url, song_id)
-
-
-class ZingMp3AlbumIE(ZingMp3BaseInfoExtractor):
-    _VALID_URL = r'https?://mp3\.zing\.vn/(?:album|playlist)/(?P<slug>[^/]+)/(?P<album_id>\w+)\.html'
-    _TESTS = [{
+    }, {
+        'url': 'http://mp3.zing.vn/video-clip/Let-It-Go-Frozen-OST-Sungha-Jung/ZW6BAEA0.html',
+        'md5': '870295a9cd8045c0e15663565902618d',
+        'info_dict': {
+            'id': 'ZW6BAEA0',
+            'title': 'Let It Go (Frozen OST)',
+            'ext': 'mp4',
+        },
+    }, {
         'url': 'http://mp3.zing.vn/album/Lau-Dai-Tinh-Ai-Bang-Kieu-Minh-Tuyet/ZWZBWDAF.html',
         'info_dict': {
             '_type': 'playlist',
             'id': 'ZWZBWDAF',
-            'title': 'Lâu Đài Tình Ái - Bằng Kiều ft. Minh Tuyết | Album 320 lossless',
+            'title': 'Lâu Đài Tình Ái - Bằng Kiều,Minh Tuyết | Album 320 lossless',
         },
         'playlist_count': 10,
+        'skip': 'removed at the request of the owner',
     }, {
         'url': 'http://mp3.zing.vn/playlist/Duong-Hong-Loan-apollobee/IWCAACCB.html',
         'only_matching': True,
     }]
-    IE_NAME = 'zingmp3:album'
-    IE_DESC = 'mp3.zing.vn albums'
+    IE_NAME = 'zingmp3'
+    IE_DESC = 'mp3.zing.vn'
 
     def _real_extract(self, url):
-        matched = re.match(self._VALID_URL, url)
-        slug = matched.group('slug')
-        album_id = matched.group('album_id')
+        page_id = self._match_id(url)
 
-        webpage = self._download_webpage(
-            'http://mp3.zing.vn/album/%s/%s.html' % (slug, album_id), album_id)
-        player_xml_url = self._search_regex(
-            r'&amp;xmlURL=(?P<xml_url>[^&]+)&', webpage, 'player xml url')
+        webpage = self._download_webpage(url, page_id)
 
-        return self._extract_player_xml(
-            player_xml_url, album_id,
-            playlist_title=self._og_search_title(webpage))
+        player_json_url = self._search_regex([
+            r'data-xml="([^"]+)',
+            r'&amp;xmlURL=([^&]+)&'
+        ], webpage, 'player xml url')
+
+        playlist_title = None
+        page_type = self._search_regex(r'/(?:html5)?xml/([^/-]+)', player_json_url, 'page type')
+        if page_type == 'video':
+            player_json_url = update_url_query(player_json_url, {'format': 'json'})
+        else:
+            player_json_url = player_json_url.replace('/xml/', '/html5xml/')
+            if page_type == 'album':
+                playlist_title = self._og_search_title(webpage)
+
+        return self._extract_player_json(player_json_url, page_id, page_type, playlist_title)

From 92d4cfa358bacff0e79da30ffb0908c7096e82f7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 21 Aug 2016 21:01:01 +0700
Subject: [PATCH 5849/6219] [kaltura] Fallback ext calculation on caption's
 format

---
 youtube_dl/extractor/kaltura.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/kaltura.py b/youtube_dl/extractor/kaltura.py
index e0f7366c2..6a8464998 100644
--- a/youtube_dl/extractor/kaltura.py
+++ b/youtube_dl/extractor/kaltura.py
@@ -36,6 +36,12 @@ class KalturaIE(InfoExtractor):
                 '''
     _SERVICE_URL = 'http://cdnapi.kaltura.com'
     _SERVICE_BASE = '/api_v3/index.php'
+    # See https://github.com/kaltura/server/blob/master/plugins/content/caption/base/lib/model/enums/CaptionType.php
+    _CAPTION_TYPES = {
+        1: 'srt',
+        2: 'ttml',
+        3: 'vtt',
+    }
     _TESTS = [
         {
             'url': 'kaltura:269692:1_1jc2y3e4',
@@ -285,9 +291,12 @@ class KalturaIE(InfoExtractor):
                 # Continue if caption is not ready
                 if f.get('status') != 2:
                     continue
+                if not caption.get('id'):
+                    continue
+                caption_format = int_or_none(caption.get('format'))
                 subtitles.setdefault(caption.get('languageCode') or caption.get('language'), []).append({
                     'url': '%s/api_v3/service/caption_captionasset/action/serve/captionAssetId/%s' % (self._SERVICE_URL, caption['id']),
-                    'ext': caption.get('fileExt', 'ttml'),
+                    'ext': caption.get('fileExt') or self._CAPTION_TYPES.get(caption_format) or 'ttml',
                 })
 
         return {

From b1e676fde81d33116f6739006d9aa0b68eebc072 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 21 Aug 2016 21:28:02 +0700
Subject: [PATCH 5850/6219] [twitch] Modernize

---
 youtube_dl/extractor/twitch.py | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 890f55180..4b5b2030c 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -20,7 +20,6 @@ from ..utils import (
     orderedSet,
     parse_duration,
     parse_iso8601,
-    sanitized_Request,
     urlencode_postdata,
 )
 
@@ -50,8 +49,8 @@ class TwitchBaseIE(InfoExtractor):
         for cookie in self._downloader.cookiejar:
             if cookie.name == 'api_token':
                 headers['Twitch-Api-Token'] = cookie.value
-        request = sanitized_Request(url, headers=headers)
-        response = super(TwitchBaseIE, self)._download_json(request, video_id, note)
+        response = super(TwitchBaseIE, self)._download_json(
+            url, video_id, note, headers=headers)
         self._handle_error(response)
         return response
 
@@ -82,11 +81,10 @@ class TwitchBaseIE(InfoExtractor):
         if not post_url.startswith('http'):
             post_url = compat_urlparse.urljoin(redirect_url, post_url)
 
-        request = sanitized_Request(
-            post_url, urlencode_postdata(login_form))
-        request.add_header('Referer', redirect_url)
         response = self._download_webpage(
-            request, None, 'Logging in as %s' % username)
+            post_url, None, 'Logging in as %s' % username,
+            data=urlencode_postdata(login_form),
+            headers={'Referer': redirect_url})
 
         error_message = self._search_regex(
             r'<div[^>]+class="subwindow_notice"[^>]*>([^<]+)</div>',

From e3f6b569096ba6faa8de230333849817c8b31a2e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 21 Aug 2016 22:09:29 +0700
Subject: [PATCH 5851/6219] [twitch] Refactor API calls

---
 youtube_dl/extractor/twitch.py | 38 +++++++++++++++++-----------------
 1 file changed, 19 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 4b5b2030c..f0a9370c8 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -41,7 +41,7 @@ class TwitchBaseIE(InfoExtractor):
                 '%s returned error: %s - %s' % (self.IE_NAME, error, response.get('message')),
                 expected=True)
 
-    def _download_json(self, url, video_id, note='Downloading JSON metadata'):
+    def _call_api(self, path, item_id, note):
         headers = {
             'Referer': 'http://api.twitch.tv/crossdomain/receiver.html?v=2',
             'X-Requested-With': 'XMLHttpRequest',
@@ -49,8 +49,8 @@ class TwitchBaseIE(InfoExtractor):
         for cookie in self._downloader.cookiejar:
             if cookie.name == 'api_token':
                 headers['Twitch-Api-Token'] = cookie.value
-        response = super(TwitchBaseIE, self)._download_json(
-            url, video_id, note, headers=headers)
+        response = self._download_json(
+            '%s/%s' % (self._API_BASE, path), item_id, note)
         self._handle_error(response)
         return response
 
@@ -107,14 +107,14 @@ class TwitchBaseIE(InfoExtractor):
 
 class TwitchItemBaseIE(TwitchBaseIE):
     def _download_info(self, item, item_id):
-        return self._extract_info(self._download_json(
-            '%s/kraken/videos/%s%s' % (self._API_BASE, item, item_id), item_id,
+        return self._extract_info(self._call_api(
+            'kraken/videos/%s%s' % (item, item_id), item_id,
             'Downloading %s info JSON' % self._ITEM_TYPE))
 
     def _extract_media(self, item_id):
         info = self._download_info(self._ITEM_SHORTCUT, item_id)
-        response = self._download_json(
-            '%s/api/videos/%s%s' % (self._API_BASE, self._ITEM_SHORTCUT, item_id), item_id,
+        response = self._call_api(
+            'api/videos/%s%s' % (self._ITEM_SHORTCUT, item_id), item_id,
             'Downloading %s playlist JSON' % self._ITEM_TYPE)
         entries = []
         chunks = response['chunks']
@@ -244,8 +244,8 @@ class TwitchVodIE(TwitchItemBaseIE):
         item_id = self._match_id(url)
 
         info = self._download_info(self._ITEM_SHORTCUT, item_id)
-        access_token = self._download_json(
-            '%s/api/vods/%s/access_token' % (self._API_BASE, item_id), item_id,
+        access_token = self._call_api(
+            'api/vods/%s/access_token' % item_id, item_id,
             'Downloading %s access token' % self._ITEM_TYPE)
 
         formats = self._extract_m3u8_formats(
@@ -273,12 +273,12 @@ class TwitchVodIE(TwitchItemBaseIE):
 
 
 class TwitchPlaylistBaseIE(TwitchBaseIE):
-    _PLAYLIST_URL = '%s/kraken/channels/%%s/videos/?offset=%%d&limit=%%d' % TwitchBaseIE._API_BASE
+    _PLAYLIST_PATH = 'kraken/channels/%s/videos/?offset=%d&limit=%d'
     _PAGE_LIMIT = 100
 
     def _extract_playlist(self, channel_id):
-        info = self._download_json(
-            '%s/kraken/channels/%s' % (self._API_BASE, channel_id),
+        info = self._call_api(
+            'kraken/channels/%s' % channel_id,
             channel_id, 'Downloading channel info JSON')
         channel_name = info.get('display_name') or info.get('name')
         entries = []
@@ -287,8 +287,8 @@ class TwitchPlaylistBaseIE(TwitchBaseIE):
         broken_paging_detected = False
         counter_override = None
         for counter in itertools.count(1):
-            response = self._download_json(
-                self._PLAYLIST_URL % (channel_id, offset, limit),
+            response = self._call_api(
+                self._PLAYLIST_PATH % (channel_id, offset, limit),
                 channel_id,
                 'Downloading %s videos JSON page %s'
                 % (self._PLAYLIST_TYPE, counter_override or counter))
@@ -343,7 +343,7 @@ class TwitchProfileIE(TwitchPlaylistBaseIE):
 class TwitchPastBroadcastsIE(TwitchPlaylistBaseIE):
     IE_NAME = 'twitch:past_broadcasts'
     _VALID_URL = r'%s/(?P<id>[^/]+)/profile/past_broadcasts/?(?:\#.*)?$' % TwitchBaseIE._VALID_URL_BASE
-    _PLAYLIST_URL = TwitchPlaylistBaseIE._PLAYLIST_URL + '&broadcasts=true'
+    _PLAYLIST_PATH = TwitchPlaylistBaseIE._PLAYLIST_PATH + '&broadcasts=true'
     _PLAYLIST_TYPE = 'past broadcasts'
 
     _TEST = {
@@ -387,8 +387,8 @@ class TwitchStreamIE(TwitchBaseIE):
     def _real_extract(self, url):
         channel_id = self._match_id(url)
 
-        stream = self._download_json(
-            '%s/kraken/streams/%s' % (self._API_BASE, channel_id), channel_id,
+        stream = self._call_api(
+            'kraken/streams/%s' % channel_id, channel_id,
             'Downloading stream JSON').get('stream')
 
         # Fallback on profile extraction if stream is offline
@@ -403,8 +403,8 @@ class TwitchStreamIE(TwitchBaseIE):
         # JSON and fallback to lowercase if it's not available.
         channel_id = stream.get('channel', {}).get('name') or channel_id.lower()
 
-        access_token = self._download_json(
-            '%s/api/channels/%s/access_token' % (self._API_BASE, channel_id), channel_id,
+        access_token = self._call_api(
+            'api/channels/%s/access_token' % channel_id, channel_id,
             'Downloading channel access token')
 
         query = {

From efe470e2614d8a50a5cc2d14551e9bc4fc41cc8b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 21 Aug 2016 22:45:50 +0700
Subject: [PATCH 5852/6219] [twitch] Renew authentication

---
 youtube_dl/extractor/twitch.py | 38 +++++++++++++++++++++++-----------
 1 file changed, 26 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index f0a9370c8..359a8859c 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -7,6 +7,7 @@ import random
 
 from .common import InfoExtractor
 from ..compat import (
+    compat_HTTPError,
     compat_parse_qs,
     compat_str,
     compat_urllib_parse_urlencode,
@@ -14,6 +15,7 @@ from ..compat import (
     compat_urlparse,
 )
 from ..utils import (
+    clean_html,
     ExtractorError,
     int_or_none,
     js_to_json,
@@ -62,9 +64,17 @@ class TwitchBaseIE(InfoExtractor):
         if username is None:
             return
 
+        def fail(message):
+            raise ExtractorError(
+                'Unable to login. Twitch said: %s' % message, expected=True)
+
         login_page, handle = self._download_webpage_handle(
             self._LOGIN_URL, None, 'Downloading login page')
 
+        # Some TOR nodes and public proxies are blocked completely
+        if 'blacklist_message' in login_page:
+            fail(clean_html(login_page))
+
         login_form = self._hidden_inputs(login_page)
 
         login_form.update({
@@ -81,20 +91,24 @@ class TwitchBaseIE(InfoExtractor):
         if not post_url.startswith('http'):
             post_url = compat_urlparse.urljoin(redirect_url, post_url)
 
-        response = self._download_webpage(
-            post_url, None, 'Logging in as %s' % username,
-            data=urlencode_postdata(login_form),
-            headers={'Referer': redirect_url})
+        headers = {'Referer': redirect_url}
 
-        error_message = self._search_regex(
-            r'<div[^>]+class="subwindow_notice"[^>]*>([^<]+)</div>',
-            response, 'error message', default=None)
-        if error_message:
-            raise ExtractorError(
-                'Unable to login. Twitch said: %s' % error_message, expected=True)
+        try:
+            response = self._download_json(
+                post_url, None, 'Logging in as %s' % username,
+                data=urlencode_postdata(login_form),
+                headers=headers)
+        except ExtractorError as e:
+            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 400:
+                response = self._parse_json(
+                    e.cause.read().decode('utf-8'), None)
+                fail(response['message'])
+            raise
 
-        if '>Reset your password<' in response:
-            self.report_warning('Twitch asks you to reset your password, go to https://secure.twitch.tv/reset/submit')
+        if response.get('redirect'):
+            self._download_webpage(
+                response['redirect'], None, 'Downloading login redirect page',
+                headers=headers)
 
     def _prefer_source(self, formats):
         try:

From 9b8c554ea70ee970009de2628bafe7fd7390bf9e Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 21 Aug 2016 17:55:47 +0100
Subject: [PATCH 5853/6219] [firsttv] fix extraction(closes #9249)

---
 youtube_dl/extractor/firsttv.py | 133 ++++++++++----------------------
 1 file changed, 39 insertions(+), 94 deletions(-)

diff --git a/youtube_dl/extractor/firsttv.py b/youtube_dl/extractor/firsttv.py
index 88bca1007..af7de10b7 100644
--- a/youtube_dl/extractor/firsttv.py
+++ b/youtube_dl/extractor/firsttv.py
@@ -2,130 +2,75 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import compat_xpath
+from ..compat import compat_urlparse
 from ..utils import (
     int_or_none,
     qualities,
     unified_strdate,
-    xpath_attr,
-    xpath_element,
-    xpath_text,
-    xpath_with_ns,
 )
 
 
 class FirstTVIE(InfoExtractor):
     IE_NAME = '1tv'
     IE_DESC = 'Первый канал'
-    _VALID_URL = r'https?://(?:www\.)?1tv\.ru/(?:[^/]+/)+p?(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?1tv\.ru/(?:[^/]+/)+(?P<id>[^/?#]+)'
 
     _TESTS = [{
-        # single format via video_materials.json API
-        'url': 'http://www.1tv.ru/prj/inprivate/vypusk/35930',
-        'md5': '82a2777648acae812d58b3f5bd42882b',
+        'url': 'http://www.1tv.ru/shows/naedine-so-vsemi/vypuski/gost-lyudmila-senchina-naedine-so-vsemi-vypusk-ot-12-02-2015',
+        'md5': 'a1b6b60d530ebcf8daacf4565762bbaf',
         'info_dict': {
-            'id': '35930',
+            'id': '40049',
             'ext': 'mp4',
             'title': 'Гость Людмила Сенчина. Наедине со всеми. Выпуск от 12.02.2015',
-            'description': 'md5:357933adeede13b202c7c21f91b871b2',
+            'description': 'md5:36a39c1d19618fec57d12efe212a8370',
             'thumbnail': 're:^https?://.*\.(?:jpg|JPG)$',
             'upload_date': '20150212',
             'duration': 2694,
         },
     }, {
-        # multiple formats via video_materials.json API
-        'url': 'http://www.1tv.ru/video_archive/projects/dobroeutro/p113641',
-        'info_dict': {
-            'id': '113641',
-            'ext': 'mp4',
-            'title': 'Весенняя аллергия. Доброе утро. Фрагмент выпуска от 07.04.2016',
-            'description': 'md5:8dcebb3dded0ff20fade39087fd1fee2',
-            'thumbnail': 're:^https?://.*\.(?:jpg|JPG)$',
-            'upload_date': '20160407',
-            'duration': 179,
-            'formats': 'mincount:3',
-        },
-        'params': {
-            'skip_download': True,
-        },
-    }, {
-        # single format only available via ONE_ONLINE_VIDEOS.archive_single_xml API
-        'url': 'http://www.1tv.ru/video_archive/series/f7552/p47038',
-        'md5': '519d306c5b5669761fd8906c39dbee23',
-        'info_dict': {
-            'id': '47038',
-            'ext': 'mp4',
-            'title': '"Побег". Второй сезон. 3 серия',
-            'description': 'md5:3abf8f6b9bce88201c33e9a3d794a00b',
-            'thumbnail': 're:^https?://.*\.(?:jpg|JPG)$',
-            'upload_date': '20120516',
-            'duration': 3080,
-        },
-    }, {
-        'url': 'http://www.1tv.ru/videoarchive/9967',
-        'only_matching': True,
+        'url': 'http://www.1tv.ru/shows/dobroe-utro/pro-zdorove/vesennyaya-allergiya-dobroe-utro-fragment-vypuska-ot-07042016',
+        'only_matching': 'true',
     }]
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        display_id = self._match_id(url)
 
-        # Videos with multiple formats only available via this API
-        video = self._download_json(
-            'http://www.1tv.ru/video_materials.json?legacy_id=%s' % video_id,
-            video_id, fatal=False)
-
-        description, thumbnail, upload_date, duration = [None] * 4
-
-        if video:
-            item = video[0]
-            title = item['title']
-            quality = qualities(('ld', 'sd', 'hd', ))
-            formats = [{
-                'url': f['src'],
-                'format_id': f.get('name'),
-                'quality': quality(f.get('name')),
-            } for f in item['mbr'] if f.get('src')]
-            thumbnail = item.get('poster')
-        else:
-            # Some videos are not available via video_materials.json
-            video = self._download_xml(
-                'http://www.1tv.ru/owa/win/ONE_ONLINE_VIDEOS.archive_single_xml?pid=%s' % video_id,
-                video_id)
-
-            NS_MAP = {
-                'media': 'http://search.yahoo.com/mrss/',
-            }
-
-            item = xpath_element(video, './channel/item', fatal=True)
-            title = xpath_text(item, './title', fatal=True)
-            formats = [{
-                'url': content.attrib['url'],
-            } for content in item.findall(
-                compat_xpath(xpath_with_ns('./media:content', NS_MAP))) if content.attrib.get('url')]
-            thumbnail = xpath_attr(
-                item, xpath_with_ns('./media:thumbnail', NS_MAP), 'url')
+        webpage = self._download_webpage(url, display_id)
+        playlist_url = compat_urlparse.urljoin(url, self._search_regex(
+            r'data-playlist-url="([^"]+)', webpage, 'playlist url'))
 
+        item = self._download_json(playlist_url, display_id)[0]
+        video_id = item['id']
+        quality = qualities(('ld', 'sd', 'hd', ))
+        formats = []
+        for f in item.get('mbr', []):
+            src = f.get('src')
+            if not src:
+                continue
+            fname = f.get('name')
+            formats.append({
+                'url': src,
+                'format_id': fname,
+                'quality': quality(fname),
+            })
         self._sort_formats(formats)
 
-        webpage = self._download_webpage(url, video_id, 'Downloading page', fatal=False)
-        if webpage:
-            title = self._html_search_regex(
-                (r'<div class="tv_translation">\s*<h1><a href="[^"]+">([^<]*)</a>',
-                 r"'title'\s*:\s*'([^']+)'"),
-                webpage, 'title', default=None) or title
-            description = self._html_search_regex(
-                r'<div class="descr">\s*<div>&nbsp;</div>\s*<p>([^<]*)</p></div>',
-                webpage, 'description', default=None) or self._html_search_meta(
-                'description', webpage, 'description')
-            thumbnail = thumbnail or self._og_search_thumbnail(webpage)
-            duration = int_or_none(self._html_search_meta(
-                'video:duration', webpage, 'video duration', fatal=False))
-            upload_date = unified_strdate(self._html_search_meta(
-                'ya:ovs:upload_date', webpage, 'upload date', fatal=False))
+        title = self._html_search_regex(
+            (r'<div class="tv_translation">\s*<h1><a href="[^"]+">([^<]*)</a>',
+             r"'title'\s*:\s*'([^']+)'"),
+            webpage, 'title', default=None) or item['title']
+        description = self._html_search_regex(
+            r'<div class="descr">\s*<div>&nbsp;</div>\s*<p>([^<]*)</p></div>',
+            webpage, 'description', default=None) or self._html_search_meta(
+            'description', webpage, 'description')
+        duration = int_or_none(self._html_search_meta(
+            'video:duration', webpage, 'video duration', fatal=False))
+        upload_date = unified_strdate(self._html_search_meta(
+            'ya:ovs:upload_date', webpage, 'upload date', fatal=False))
 
         return {
             'id': video_id,
-            'thumbnail': thumbnail,
+            'thumbnail': item.get('poster') or self._og_search_thumbnail(webpage),
             'title': title,
             'description': description,
             'upload_date': upload_date,

From 526656726b13f47a33c36e56821136b90d6decf1 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 22 Aug 2016 02:06:47 +0800
Subject: [PATCH 5854/6219] [charlierose] Simplify and improve

---
 youtube_dl/extractor/charlierose.py | 33 +++++++++++++++++------------
 1 file changed, 19 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/charlierose.py b/youtube_dl/extractor/charlierose.py
index ba1d1b833..817f7128f 100644
--- a/youtube_dl/extractor/charlierose.py
+++ b/youtube_dl/extractor/charlierose.py
@@ -6,20 +6,25 @@ from ..utils import remove_end
 
 class CharlieRoseIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?charlierose\.com/video(?:s|/player)/(?P<id>\d+)'
-    _TEST = {
+    _TESTS = [{
         'url': 'https://charlierose.com/videos/27996',
+        'md5': 'fda41d49e67d4ce7c2411fd2c4702e09',
         'info_dict': {
             'id': '27996',
             'ext': 'mp4',
             'title': 'Remembering Zaha Hadid',
             'thumbnail': 're:^https?://.*\.jpg\?\d+',
             'description': 'We revisit past conversations with Zaha Hadid, in memory of the world renowned Iraqi architect.',
+            'subtitles': {
+                'en': [{
+                    'ext': 'vtt',
+                }],
+            },
         },
-        'params': {
-            # m3u8 download
-            'skip_download': True,
-        }
-    }
+    }, {
+        'url': 'https://charlierose.com/videos/27996',
+        'only_matching': True,
+    }]
 
     _PLAYER_BASE = 'https://charlierose.com/video/player/%s'
 
@@ -29,17 +34,17 @@ class CharlieRoseIE(InfoExtractor):
 
         title = remove_end(self._og_search_title(webpage), ' - Charlie Rose')
 
-        entries = self._parse_html5_media_entries(self._PLAYER_BASE % video_id, webpage, video_id)[0]
-        formats = entries['formats']
+        info_dict = self._parse_html5_media_entries(
+            self._PLAYER_BASE % video_id, webpage, video_id)[0]
 
-        self._sort_formats(formats)
-        self._remove_duplicate_formats(formats)
+        self._sort_formats(info_dict['formats'])
+        self._remove_duplicate_formats(info_dict['formats'])
 
-        return {
+        info_dict.update({
             'id': video_id,
             'title': title,
-            'formats': formats,
             'thumbnail': self._og_search_thumbnail(webpage),
             'description': self._og_search_description(webpage),
-            'subtitles': entries.get('subtitles'),
-        }
+        })
+
+        return info_dict

From d0fa172e5fc1d676834252dcd395ec495b20b0bc Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 21 Aug 2016 19:11:51 +0100
Subject: [PATCH 5855/6219] [firsttv] keep a test videos with multiple formats

---
 youtube_dl/extractor/firsttv.py | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/firsttv.py b/youtube_dl/extractor/firsttv.py
index af7de10b7..332d12020 100644
--- a/youtube_dl/extractor/firsttv.py
+++ b/youtube_dl/extractor/firsttv.py
@@ -16,6 +16,7 @@ class FirstTVIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?1tv\.ru/(?:[^/]+/)+(?P<id>[^/?#]+)'
 
     _TESTS = [{
+        # single format
         'url': 'http://www.1tv.ru/shows/naedine-so-vsemi/vypuski/gost-lyudmila-senchina-naedine-so-vsemi-vypusk-ot-12-02-2015',
         'md5': 'a1b6b60d530ebcf8daacf4565762bbaf',
         'info_dict': {
@@ -28,8 +29,21 @@ class FirstTVIE(InfoExtractor):
             'duration': 2694,
         },
     }, {
+        # multiple formats
         'url': 'http://www.1tv.ru/shows/dobroe-utro/pro-zdorove/vesennyaya-allergiya-dobroe-utro-fragment-vypuska-ot-07042016',
-        'only_matching': 'true',
+        'info_dict': {
+            'id': '364746',
+            'ext': 'mp4',
+            'title': 'Весенняя аллергия. Доброе утро. Фрагмент выпуска от 07.04.2016',
+            'description': 'md5:a242eea0031fd180a4497d52640a9572',
+            'thumbnail': 're:^https?://.*\.(?:jpg|JPG)$',
+            'upload_date': '20160407',
+            'duration': 179,
+            'formats': 'mincount:3',
+        },
+        'params': {
+            'skip_download': True,
+        },
     }]
 
     def _real_extract(self, url):

From ad120ae1c57fe3ff0c7f5559d280cb8230a2b38c Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 22 Aug 2016 02:18:46 +0800
Subject: [PATCH 5856/6219] [extractor/common] Change the default m3u8 protocol
 in HTML5

Helper functions should have consistent default values
---
 ChangeLog                           | 1 +
 youtube_dl/extractor/charlierose.py | 3 ++-
 youtube_dl/extractor/common.py      | 6 +++---
 youtube_dl/extractor/snotr.py       | 3 ++-
 4 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index a8d8d05a3..383ff59ea 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -7,6 +7,7 @@ Core
 * Fix js_to_json(): correct octal or hexadecimal number detection
 
 Extractors
++ [charlierose] Add new extractor (#10382)
 + [litv] Support 'promo' URLs (#10385)
 * [snotr] Fix extraction (#10338)
 * [n-tv.de] Fix extraction (#10331)
diff --git a/youtube_dl/extractor/charlierose.py b/youtube_dl/extractor/charlierose.py
index 817f7128f..4bf2cf7b0 100644
--- a/youtube_dl/extractor/charlierose.py
+++ b/youtube_dl/extractor/charlierose.py
@@ -35,7 +35,8 @@ class CharlieRoseIE(InfoExtractor):
         title = remove_end(self._og_search_title(webpage), ' - Charlie Rose')
 
         info_dict = self._parse_html5_media_entries(
-            self._PLAYER_BASE % video_id, webpage, video_id)[0]
+            self._PLAYER_BASE % video_id, webpage, video_id,
+            m3u8_entry_protocol='m3u8_native')[0]
 
         self._sort_formats(info_dict['formats'])
         self._remove_duplicate_formats(info_dict['formats'])
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 07d58afe7..ba4c03d3d 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1695,7 +1695,7 @@ class InfoExtractor(object):
                         self.report_warning('Unknown MIME type %s in DASH manifest' % mime_type)
         return formats
 
-    def _parse_html5_media_entries(self, base_url, webpage, video_id, m3u8_id=None):
+    def _parse_html5_media_entries(self, base_url, webpage, video_id, m3u8_id=None, m3u8_entry_protocol='m3u8'):
         def absolute_url(video_url):
             return compat_urlparse.urljoin(base_url, video_url)
 
@@ -1715,8 +1715,8 @@ class InfoExtractor(object):
             if determine_ext(full_url) == 'm3u8':
                 is_plain_url = False
                 formats = self._extract_m3u8_formats(
-                    full_url, video_id, ext='mp4', entry_protocol='m3u8_native',
-                    m3u8_id=m3u8_id)
+                    full_url, video_id, ext='mp4',
+                    entry_protocol=m3u8_entry_protocol, m3u8_id=m3u8_id)
             else:
                 is_plain_url = True
                 formats = [{
diff --git a/youtube_dl/extractor/snotr.py b/youtube_dl/extractor/snotr.py
index 3bb78cb84..4819fe5b4 100644
--- a/youtube_dl/extractor/snotr.py
+++ b/youtube_dl/extractor/snotr.py
@@ -46,7 +46,8 @@ class SnotrIE(InfoExtractor):
         title = self._og_search_title(webpage)
 
         description = self._og_search_description(webpage)
-        info_dict = self._parse_html5_media_entries(url, webpage, video_id)[0]
+        info_dict = self._parse_html5_media_entries(
+            url, webpage, video_id, m3u8_entry_protocol='m3u8_native')[0]
 
         view_count = str_to_int(self._html_search_regex(
             r'<p[^>]*>\s*<strong[^>]*>Views:</strong>\s*<span[^>]*>([\d,\.]+)',

From cf143c4d977915c993f4aa467b491a6c284bb569 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 22 Aug 2016 03:31:33 +0700
Subject: [PATCH 5857/6219] [ivi] Add support for 720p and 1080p

---
 youtube_dl/extractor/ivi.py | 35 ++++++++++++++++++++++++++---------
 1 file changed, 26 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/ivi.py b/youtube_dl/extractor/ivi.py
index 472d72b4c..f5ab5f4af 100644
--- a/youtube_dl/extractor/ivi.py
+++ b/youtube_dl/extractor/ivi.py
@@ -1,4 +1,4 @@
-# encoding: utf-8
+# coding: utf-8
 from __future__ import unicode_literals
 
 import re
@@ -8,7 +8,7 @@ from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
     int_or_none,
-    sanitized_Request,
+    qualities,
 )
 
 
@@ -49,11 +49,27 @@ class IviIE(InfoExtractor):
                 'thumbnail': 're:^https?://.*\.jpg$',
             },
             'skip': 'Only works from Russia',
+        },
+        {
+            # with MP4-HD720 format
+            'url': 'http://www.ivi.ru/watch/146500',
+            'md5': 'd63d35cdbfa1ea61a5eafec7cc523e1e',
+            'info_dict': {
+                'id': '146500',
+                'ext': 'mp4',
+                'title': 'Кукла',
+                'description': 'md5:ffca9372399976a2d260a407cc74cce6',
+                'duration': 5599,
+                'thumbnail': 're:^https?://.*\.jpg$',
+            },
+            'skip': 'Only works from Russia',
         }
     ]
 
     # Sorted by quality
-    _KNOWN_FORMATS = ['MP4-low-mobile', 'MP4-mobile', 'FLV-lo', 'MP4-lo', 'FLV-hi', 'MP4-hi', 'MP4-SHQ']
+    _KNOWN_FORMATS = (
+        'MP4-low-mobile', 'MP4-mobile', 'FLV-lo', 'MP4-lo', 'FLV-hi', 'MP4-hi',
+        'MP4-SHQ', 'MP4-HD720', 'MP4-HD1080')
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -69,10 +85,9 @@ class IviIE(InfoExtractor):
             ]
         }
 
-        request = sanitized_Request(
-            'http://api.digitalaccess.ru/api/json/', json.dumps(data))
         video_json = self._download_json(
-            request, video_id, 'Downloading video JSON')
+            'http://api.digitalaccess.ru/api/json/', video_id,
+            'Downloading video JSON', data=json.dumps(data))
 
         if 'error' in video_json:
             error = video_json['error']
@@ -84,11 +99,13 @@ class IviIE(InfoExtractor):
 
         result = video_json['result']
 
+        quality = qualities(self._KNOWN_FORMATS)
+
         formats = [{
             'url': x['url'],
-            'format_id': x['content_format'],
-            'preference': self._KNOWN_FORMATS.index(x['content_format']),
-        } for x in result['files'] if x['content_format'] in self._KNOWN_FORMATS]
+            'format_id': x.get('content_format'),
+            'quality': quality(x.get('content_format')),
+        } for x in result['files'] if x.get('url')]
 
         self._sort_formats(formats)
 

From 3d897cc791781430f371da98f2f3a05a0b856c5b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 22 Aug 2016 03:34:27 +0700
Subject: [PATCH 5858/6219] [ivi] Fix episode number extraction

---
 youtube_dl/extractor/ivi.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ivi.py b/youtube_dl/extractor/ivi.py
index f5ab5f4af..7c8cb21c2 100644
--- a/youtube_dl/extractor/ivi.py
+++ b/youtube_dl/extractor/ivi.py
@@ -132,7 +132,7 @@ class IviIE(InfoExtractor):
             webpage, 'season number', default=None))
 
         episode_number = int_or_none(self._search_regex(
-            r'<meta[^>]+itemprop="episode"[^>]*>\s*<meta[^>]+itemprop="episodeNumber"[^>]+content="(\d+)',
+            r'[^>]+itemprop="episode"[^>]*>\s*<meta[^>]+itemprop="episodeNumber"[^>]+content="(\d+)',
             webpage, 'episode number', default=None))
 
         description = self._og_search_description(webpage, default=None) or self._html_search_meta(

From afbab5688e837d9b1617119b1ac26b4a4e343bed Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 22 Aug 2016 04:15:46 +0700
Subject: [PATCH 5859/6219] [ChangeLog] Actualize

---
 ChangeLog | 20 ++++++++++++++++----
 1 file changed, 16 insertions(+), 4 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 383ff59ea..ee9b9500f 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,16 +1,28 @@
 version <unreleased>
 
 Core
-+ Recognize file size strings with full unit names (for example "8.5
-  megabytes")
-+ Support m3u8 manifests in HTML5 multimedia tags
-* Fix js_to_json(): correct octal or hexadecimal number detection
+* Improve formats and subtitles extension auto calculation
++ Recognize full unit names in parse_filesize
++ Add support for m3u8 manifests in HTML5 multimedia tags
+* Fix octal/hexadecimal number detection in js_to_json
 
 Extractors
++ [ivi] Add support for 720p and 1080p
 + [charlierose] Add new extractor (#10382)
+* [1tv] Fix extraction (#9249)
+* [twitch] Renew authentication
+* [kaltura] Improve subtitles extension calculation
++ [zingmp3] Add support for video clips
+* [zingmp3] Fix extraction (#10041)
+* [kaltura] Improve subtitles extraction (#10279)
+* [cultureunplugged] Fix extraction (#10330)
++ [cnn] Add support for money.cnn.com (#2797)
+* [cbsnews] Fix extraction (#10362)
+* [cbs] Fix extraction (#10393)
 + [litv] Support 'promo' URLs (#10385)
 * [snotr] Fix extraction (#10338)
 * [n-tv.de] Fix extraction (#10331)
+* [globo:article] Relax URL and video id regular expressions (#10379)
 
 
 version 2016.08.19

From 6d2679ee26eb6ad0587d01e40ca7a17a6edd6e2a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 22 Aug 2016 04:17:34 +0700
Subject: [PATCH 5860/6219] release 2016.08.22

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 4 ++--
 youtube_dl/version.py     | 2 +-
 4 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 7af3c7099..7dcca18a1 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.08.19*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.08.19**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.08.22*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.08.22**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.08.19
+[debug] youtube-dl version 2016.08.22
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index ee9b9500f..a8202d3de 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2016.08.22
 
 Core
 * Improve formats and subtitles extension auto calculation
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index edf192138..ca96d2b07 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -121,6 +121,7 @@
  - **CDA**
  - **CeskaTelevize**
  - **channel9**: Channel 9
+ - **CharlieRose**
  - **Chaturbate**
  - **Chilloutzone**
  - **chirbit**
@@ -893,5 +894,4 @@
  - **Zapiks**
  - **ZDF**
  - **ZDFChannel**
- - **zingmp3:album**: mp3.zing.vn albums
- - **zingmp3:song**: mp3.zing.vn songs
+ - **zingmp3**: mp3.zing.vn
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 691f2c591..e33d32e97 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.08.19'
+__version__ = '2016.08.22'

From 55d119e2a10ccbfadc12b9af30c495f46874c2a3 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 22 Aug 2016 00:06:39 +0100
Subject: [PATCH 5861/6219] [abc:iview] Add new extractor(closes #6148)

---
 youtube_dl/extractor/abc.py        | 63 ++++++++++++++++++++++++++++++
 youtube_dl/extractor/extractors.py |  5 ++-
 2 files changed, 67 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/abc.py b/youtube_dl/extractor/abc.py
index b584277be..879ded88d 100644
--- a/youtube_dl/extractor/abc.py
+++ b/youtube_dl/extractor/abc.py
@@ -7,6 +7,8 @@ from ..utils import (
     ExtractorError,
     js_to_json,
     int_or_none,
+    update_url_query,
+    parse_iso8601,
 )
 
 
@@ -93,3 +95,64 @@ class ABCIE(InfoExtractor):
             'description': self._og_search_description(webpage),
             'thumbnail': self._og_search_thumbnail(webpage),
         }
+
+
+class ABCIViewIE(InfoExtractor):
+    IE_NAME = 'abc.net.au:iview'
+    _VALID_URL = r'https?://iview\.abc\.net\.au/programs/[^/]+/(?P<id>[^/?#]+)'
+
+    _TESTS = [{
+        'url': 'http://iview.abc.net.au/programs/gardening-australia/FA1505V024S00',
+        'md5': '979d10b2939101f0d27a06b79edad536',
+        'info_dict': {
+            'id': 'FA1505V024S00',
+            'ext': 'mp4',
+            'title': 'Series 27 Ep 24',
+            'description': 'md5:b28baeae7504d1148e1d2f0e3ed3c15d',
+            'upload_date': '20160820',
+            'uploader_id': 'abc1',
+            'timestamp': 1471719600,
+        },
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        video_params = self._parse_json(self._search_regex(
+            r'videoParams\s*=\s*({.+?});', webpage, 'video params'), video_id)
+        title = video_params['title']
+        stream = next(s for s in video_params['playlist'] if s.get('type') == 'program')
+
+        formats = []
+        f4m_url = stream.get('hds-unmetered') or stream['hds-metered']
+        formats.extend(self._extract_f4m_formats(
+            update_url_query(f4m_url, {'hdcore': '3.7.0'}),
+            video_id, f4m_id='hds', fatal=False))
+        formats.extend(self._extract_m3u8_formats(f4m_url.replace(
+            'akamaihd.net/z/', 'akamaihd.net/i/').replace('/manifest.f4m', '/master.m3u8'),
+            video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
+        self._sort_formats(formats)
+
+        subtitles = {}
+        src_vtt = stream.get('captions', {}).get('src-vtt')
+        if src_vtt:
+            subtitles['en'] = [{
+                'url': src_vtt,
+                'ext': 'vtt',
+            }]
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': self._html_search_meta(['og:description', 'twitter:description'], webpage),
+            'thumbnail': self._html_search_meta(['og:image', 'twitter:image:src'], webpage),
+            'duration': int_or_none(video_params.get('eventDuration')),
+            'timestamp': parse_iso8601(video_params.get('pubDate'), ' '),
+            'series': video_params.get('seriesTitle'),
+            'series_id': video_params.get('seriesHouseNumber') or video_id[:7],
+            'episode_number': int_or_none(self._html_search_meta('episodeNumber', webpage)),
+            'episode': self._html_search_meta('episode_title', webpage),
+            'uploader_id': video_params.get('channel'),
+            'formats': formats,
+            'subtitles': subtitles,
+        }
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index b0644be11..8e405ad72 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1,7 +1,10 @@
 # flake8: noqa
 from __future__ import unicode_literals
 
-from .abc import ABCIE
+from .abc import (
+    ABCIE,
+    ABCIViewIE,
+)
 from .abc7news import Abc7NewsIE
 from .abcnews import (
     AbcNewsIE,

From 96229e5f95a5be622a694b464085bdea59134ccf Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 22 Aug 2016 13:56:09 +0800
Subject: [PATCH 5862/6219] [mtvservices:embedded] Update config URL

All starts from #10363. The test case in mtvservices:embedded uses
config.xml, while the video from #10363 and the test case in generic.py
is broken. Both uses index.html for fetching the feed URL.
---
 youtube_dl/extractor/mtv.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index 2f455680e..200f340de 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -257,8 +257,8 @@ class MTVServicesEmbeddedIE(MTVServicesInfoExtractor):
     def _get_feed_url(self, uri):
         video_id = self._id_from_uri(uri)
         site_id = uri.replace(video_id, '')
-        config_url = ('http://media.mtvnservices.com/pmt/e1/players/{0}/'
-                      'context4/context5/config.xml'.format(site_id))
+        config_url = ('http://media.mtvnservices.com/pmt-arc/e1/players/{0}/'
+                      'context52/config.xml'.format(site_id))
         config_doc = self._download_xml(config_url, video_id)
         feed_node = config_doc.find('.//feed')
         feed_url = feed_node.text.strip().split('?')[0]

From c7c43a93ba4abbd2175ab0891b63def7e25aa385 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 22 Aug 2016 07:47:25 +0100
Subject: [PATCH 5863/6219] [common] add helper method to extract akamai m3u8
 and f4m formats

---
 youtube_dl/extractor/common.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index ba4c03d3d..8ed16deee 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1765,6 +1765,18 @@ class InfoExtractor(object):
                 entries.append(media_info)
         return entries
 
+    def _extract_akamai_formats(self, manifest_url, video_id):
+        formats = []
+        f4m_url = re.sub(r'(https?://.+?)/i/', r'\1/z/', manifest_url).replace('/master.m3u8', '/manifest.f4m')
+        formats.extend(self._extract_f4m_formats(
+            update_url_query(f4m_url, {'hdcore': '3.7.0'}),
+            video_id, f4m_id='hds', fatal=False))
+        m3u8_url = re.sub(r'(https?://.+?)/z/', r'\1/i/', manifest_url).replace('/manifest.f4m', '/master.m3u8')
+        formats.extend(self._extract_m3u8_formats(
+            m3u8_url, video_id, 'mp4', 'm3u8_native',
+            m3u8_id='hls', fatal=False))
+        return formats
+
     def _live_title(self, name):
         """ Generate the title for a live video """
         now = datetime.datetime.now()

From ad316425840315b40405a55243635fcfbcae5f19 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 22 Aug 2016 07:48:40 +0100
Subject: [PATCH 5864/6219] [nrk,abc:iview] use _extract_akamai_formats

---
 youtube_dl/extractor/abc.py | 10 +---------
 youtube_dl/extractor/nrk.py | 14 ++------------
 2 files changed, 3 insertions(+), 21 deletions(-)

diff --git a/youtube_dl/extractor/abc.py b/youtube_dl/extractor/abc.py
index 879ded88d..c7b6df7d0 100644
--- a/youtube_dl/extractor/abc.py
+++ b/youtube_dl/extractor/abc.py
@@ -7,7 +7,6 @@ from ..utils import (
     ExtractorError,
     js_to_json,
     int_or_none,
-    update_url_query,
     parse_iso8601,
 )
 
@@ -123,14 +122,7 @@ class ABCIViewIE(InfoExtractor):
         title = video_params['title']
         stream = next(s for s in video_params['playlist'] if s.get('type') == 'program')
 
-        formats = []
-        f4m_url = stream.get('hds-unmetered') or stream['hds-metered']
-        formats.extend(self._extract_f4m_formats(
-            update_url_query(f4m_url, {'hdcore': '3.7.0'}),
-            video_id, f4m_id='hds', fatal=False))
-        formats.extend(self._extract_m3u8_formats(f4m_url.replace(
-            'akamaihd.net/z/', 'akamaihd.net/i/').replace('/manifest.f4m', '/master.m3u8'),
-            video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
+        formats = self._extract_akamai_formats(stream['hds-unmetered'], video_id)
         self._sort_formats(formats)
 
         subtitles = {}
diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index 6ded5bd45..ed42eb301 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -14,16 +14,6 @@ from ..utils import (
 
 
 class NRKBaseIE(InfoExtractor):
-    def _extract_formats(self, manifest_url, video_id, fatal=True):
-        formats = []
-        formats.extend(self._extract_f4m_formats(
-            manifest_url + '?hdcore=3.5.0&plugin=aasp-3.5.0.151.81',
-            video_id, f4m_id='hds', fatal=fatal))
-        formats.extend(self._extract_m3u8_formats(manifest_url.replace(
-            'akamaihd.net/z/', 'akamaihd.net/i/').replace('/manifest.f4m', '/master.m3u8'),
-            video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=fatal))
-        return formats
-
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
@@ -45,7 +35,7 @@ class NRKBaseIE(InfoExtractor):
                 asset_url = asset.get('url')
                 if not asset_url:
                     continue
-                formats = self._extract_formats(asset_url, video_id, fatal=False)
+                formats = self._extract_akamai_formats(asset_url, video_id)
                 if not formats:
                     continue
                 self._sort_formats(formats)
@@ -69,7 +59,7 @@ class NRKBaseIE(InfoExtractor):
         if not entries:
             media_url = data.get('mediaUrl')
             if media_url:
-                formats = self._extract_formats(media_url, video_id)
+                formats = self._extract_akamai_formats(media_url, video_id)
                 self._sort_formats(formats)
                 duration = parse_duration(data.get('duration'))
                 entries = [{

From 7367bdef23a3db4691ba99f01613b7759340f05e Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 22 Aug 2016 23:10:06 +0100
Subject: [PATCH 5865/6219] [awaan] fix extraction, modernize, rename the
 extractors and add test for live stream

---
 youtube_dl/extractor/{dcn.py => awaan.py} | 89 +++++++++++------------
 youtube_dl/extractor/extractors.py        | 12 +--
 2 files changed, 50 insertions(+), 51 deletions(-)
 rename youtube_dl/extractor/{dcn.py => awaan.py} (75%)

diff --git a/youtube_dl/extractor/dcn.py b/youtube_dl/extractor/awaan.py
similarity index 75%
rename from youtube_dl/extractor/dcn.py
rename to youtube_dl/extractor/awaan.py
index b8542820a..bdf23c6a9 100644
--- a/youtube_dl/extractor/dcn.py
+++ b/youtube_dl/extractor/awaan.py
@@ -12,46 +12,41 @@ from ..compat import (
 from ..utils import (
     int_or_none,
     parse_iso8601,
-    sanitized_Request,
     smuggle_url,
     unsmuggle_url,
     urlencode_postdata,
 )
 
 
-class DCNIE(InfoExtractor):
+class AWAANIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?(?:awaan|dcndigital)\.ae/(?:#/)?show/(?P<show_id>\d+)/[^/]+(?:/(?P<video_id>\d+)/(?P<season_id>\d+))?'
 
     def _real_extract(self, url):
         show_id, video_id, season_id = re.match(self._VALID_URL, url).groups()
         if video_id and int(video_id) > 0:
             return self.url_result(
-                'http://www.dcndigital.ae/media/%s' % video_id, 'DCNVideo')
+                'http://awaan.ae/media/%s' % video_id, 'AWAANVideo')
         elif season_id and int(season_id) > 0:
             return self.url_result(smuggle_url(
-                'http://www.dcndigital.ae/program/season/%s' % season_id,
-                {'show_id': show_id}), 'DCNSeason')
+                'http://awaan.ae/program/season/%s' % season_id,
+                {'show_id': show_id}), 'AWAANSeason')
         else:
             return self.url_result(
-                'http://www.dcndigital.ae/program/%s' % show_id, 'DCNSeason')
+                'http://awaan.ae/program/%s' % show_id, 'AWAANSeason')
 
 
-class DCNBaseIE(InfoExtractor):
-    def _extract_video_info(self, video_data, video_id, is_live):
+class AWAANBaseIE(InfoExtractor):
+    def _parse_video_data(self, video_data, video_id, is_live):
         title = video_data.get('title_en') or video_data['title_ar']
         img = video_data.get('img')
-        thumbnail = 'http://admin.mangomolo.com/analytics/%s' % img if img else None
-        duration = int_or_none(video_data.get('duration'))
-        description = video_data.get('description_en') or video_data.get('description_ar')
-        timestamp = parse_iso8601(video_data.get('create_time'), ' ')
 
         return {
             'id': video_id,
             'title': self._live_title(title) if is_live else title,
-            'description': description,
-            'thumbnail': thumbnail,
-            'duration': duration,
-            'timestamp': timestamp,
+            'description': video_data.get('description_en') or video_data.get('description_ar'),
+            'thumbnail': 'http://admin.mangomolo.com/analytics/%s' % img if img else None,
+            'duration': int_or_none(video_data.get('duration')),
+            'timestamp': parse_iso8601(video_data.get('create_time'), ' '),
             'is_live': is_live,
         }
 
@@ -75,11 +70,12 @@ class DCNBaseIE(InfoExtractor):
         return formats
 
 
-class DCNVideoIE(DCNBaseIE):
-    IE_NAME = 'dcn:video'
+class AWAANVideoIE(AWAANBaseIE):
+    IE_NAME = 'awaan:video'
     _VALID_URL = r'https?://(?:www\.)?(?:awaan|dcndigital)\.ae/(?:#/)?(?:video(?:/[^/]+)?|media|catchup/[^/]+/[^/]+)/(?P<id>\d+)'
     _TESTS = [{
         'url': 'http://www.dcndigital.ae/#/video/%D8%B1%D8%AD%D9%84%D8%A9-%D8%A7%D9%84%D8%B9%D9%85%D8%B1-%D8%A7%D9%84%D8%AD%D9%84%D9%82%D8%A9-1/17375',
+        'md5': '5f61c33bfc7794315c671a62d43116aa',
         'info_dict':
         {
             'id': '17375',
@@ -90,10 +86,6 @@ class DCNVideoIE(DCNBaseIE):
             'timestamp': 1227504126,
             'upload_date': '20081124',
         },
-        'params': {
-            # m3u8 download
-            'skip_download': True,
-        },
     }, {
         'url': 'http://awaan.ae/video/26723981/%D8%AF%D8%A7%D8%B1-%D8%A7%D9%84%D8%B3%D9%84%D8%A7%D9%85:-%D8%AE%D9%8A%D8%B1-%D8%AF%D9%88%D8%B1-%D8%A7%D9%84%D8%A3%D9%86%D8%B5%D8%A7%D8%B1',
         'only_matching': True,
@@ -102,11 +94,10 @@ class DCNVideoIE(DCNBaseIE):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        request = sanitized_Request(
+        video_data = self._download_json(
             'http://admin.mangomolo.com/analytics/index.php/plus/video?id=%s' % video_id,
-            headers={'Origin': 'http://www.dcndigital.ae'})
-        video_data = self._download_json(request, video_id)
-        info = self._extract_video_info(video_data, video_id, False)
+            video_id, headers={'Origin': 'http://awaan.ae'})
+        info = self._parse_video_data(video_data, video_id, False)
 
         webpage = self._download_webpage(
             'http://admin.mangomolo.com/analytics/index.php/customers/embed/video?' +
@@ -121,19 +112,31 @@ class DCNVideoIE(DCNBaseIE):
         return info
 
 
-class DCNLiveIE(DCNBaseIE):
-    IE_NAME = 'dcn:live'
+class AWAANLiveIE(AWAANBaseIE):
+    IE_NAME = 'awaan:live'
     _VALID_URL = r'https?://(?:www\.)?(?:awaan|dcndigital)\.ae/(?:#/)?live/(?P<id>\d+)'
+    _TEST = {
+        'url': 'http://awaan.ae/live/6/dubai-tv',
+        'info_dict': {
+            'id': '6',
+            'ext': 'mp4',
+            'title': 're:Dubai Al Oula [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
+            'upload_date': '20150107',
+            'timestamp': 1420588800,
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }
 
     def _real_extract(self, url):
         channel_id = self._match_id(url)
 
-        request = sanitized_Request(
+        channel_data = self._download_json(
             'http://admin.mangomolo.com/analytics/index.php/plus/getchanneldetails?channel_id=%s' % channel_id,
-            headers={'Origin': 'http://www.dcndigital.ae'})
-
-        channel_data = self._download_json(request, channel_id)
-        info = self._extract_video_info(channel_data, channel_id, True)
+            channel_id, headers={'Origin': 'http://awaan.ae'})
+        info = self._parse_video_data(channel_data, channel_id, True)
 
         webpage = self._download_webpage(
             'http://admin.mangomolo.com/analytics/index.php/customers/embed/index?' +
@@ -148,8 +151,8 @@ class DCNLiveIE(DCNBaseIE):
         return info
 
 
-class DCNSeasonIE(InfoExtractor):
-    IE_NAME = 'dcn:season'
+class AWAANSeasonIE(InfoExtractor):
+    IE_NAME = 'awaan:season'
     _VALID_URL = r'https?://(?:www\.)?(?:awaan|dcndigital)\.ae/(?:#/)?program/(?:(?P<show_id>\d+)|season/(?P<season_id>\d+))'
     _TEST = {
         'url': 'http://dcndigital.ae/#/program/205024/%D9%85%D8%AD%D8%A7%D8%B6%D8%B1%D8%A7%D8%AA-%D8%A7%D9%84%D8%B4%D9%8A%D8%AE-%D8%A7%D9%84%D8%B4%D8%B9%D8%B1%D8%A7%D9%88%D9%8A',
@@ -170,21 +173,17 @@ class DCNSeasonIE(InfoExtractor):
             data['season'] = season_id
             show_id = smuggled_data.get('show_id')
             if show_id is None:
-                request = sanitized_Request(
+                season = self._download_json(
                     'http://admin.mangomolo.com/analytics/index.php/plus/season_info?id=%s' % season_id,
-                    headers={'Origin': 'http://www.dcndigital.ae'})
-                season = self._download_json(request, season_id)
+                    season_id, headers={'Origin': 'http://awaan.ae'})
                 show_id = season['id']
         data['show_id'] = show_id
-        request = sanitized_Request(
+        show = self._download_json(
             'http://admin.mangomolo.com/analytics/index.php/plus/show',
-            urlencode_postdata(data),
-            {
-                'Origin': 'http://www.dcndigital.ae',
+            show_id, data=urlencode_postdata(data), headers={
+                'Origin': 'http://awaan.ae',
                 'Content-Type': 'application/x-www-form-urlencoded'
             })
-
-        show = self._download_json(request, show_id)
         if not season_id:
             season_id = show['default_season']
         for season in show['seasons']:
@@ -195,6 +194,6 @@ class DCNSeasonIE(InfoExtractor):
                 for video in show['videos']:
                     video_id = compat_str(video['id'])
                     entries.append(self.url_result(
-                        'http://www.dcndigital.ae/media/%s' % video_id, 'DCNVideo', video_id))
+                        'http://awaan.ae/media/%s' % video_id, 'AWAANVideo', video_id))
 
                 return self.playlist_result(entries, season_id, title)
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 8e405ad72..04cd23bdb 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -71,6 +71,12 @@ from .atttechchannel import ATTTechChannelIE
 from .audimedia import AudiMediaIE
 from .audioboom import AudioBoomIE
 from .audiomack import AudiomackIE, AudiomackAlbumIE
+from .awaan import (
+    AWAANIE,
+    AWAANVideoIE,
+    AWAANLiveIE,
+    AWAANSeasonIE,
+)
 from .azubu import AzubuIE, AzubuLiveIE
 from .baidu import BaiduVideoIE
 from .bambuser import BambuserIE, BambuserChannelIE
@@ -200,12 +206,6 @@ from .daum import (
     DaumUserIE,
 )
 from .dbtv import DBTVIE
-from .dcn import (
-    DCNIE,
-    DCNVideoIE,
-    DCNLiveIE,
-    DCNSeasonIE,
-)
 from .dctp import DctpTvIE
 from .deezer import DeezerPlaylistIE
 from .democracynow import DemocracynowIE

From 3083e4dc070d6378456f9b20ebd5cbf9ee9d92af Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 23 Aug 2016 07:22:14 +0700
Subject: [PATCH 5866/6219] [eagleplatform] Improve detection of embedded
 videos (Closes #10409)

---
 youtube_dl/extractor/eagleplatform.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/youtube_dl/extractor/eagleplatform.py b/youtube_dl/extractor/eagleplatform.py
index 12d28d3b9..d4dfda8cd 100644
--- a/youtube_dl/extractor/eagleplatform.py
+++ b/youtube_dl/extractor/eagleplatform.py
@@ -52,11 +52,24 @@ class EaglePlatformIE(InfoExtractor):
 
     @staticmethod
     def _extract_url(webpage):
+        # Regular iframe embedding
         mobj = re.search(
             r'<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//.+?\.media\.eagleplatform\.com/index/player\?.+?)\1',
             webpage)
         if mobj is not None:
             return mobj.group('url')
+        # Basic usage embedding (see http://dultonmedia.github.io/eplayer/)
+        mobj = re.search(
+            r'''(?xs)
+                    <script[^>]+
+                        src=(?P<q1>["\'])(?:https?:)?//(?P<host>.+?\.media\.eagleplatform\.com)/player/player\.js(?P=q1)
+                    .+?
+                    <div[^>]+
+                        class=(?P<q2>["\'])eagleplayer(?P=q2)[^>]+
+                        data-id=["\'](?P<id>\d+)
+            ''', webpage)
+        if mobj is not None:
+            return 'eagleplatform:%(host)s:%(id)s' % mobj.groupdict()
 
     @staticmethod
     def _handle_error(response):

From fb009b7f534e600e98b93e062198ade5826b5800 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 23 Aug 2016 10:28:28 +0100
Subject: [PATCH 5867/6219] [bravotv] correct clip info extraction and add
 support for adobe pass auth(closes #10407)

---
 youtube_dl/extractor/bravotv.py | 81 +++++++++++++++++++++++++--------
 1 file changed, 62 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/extractor/bravotv.py b/youtube_dl/extractor/bravotv.py
index 541c76944..31763b4c6 100644
--- a/youtube_dl/extractor/bravotv.py
+++ b/youtube_dl/extractor/bravotv.py
@@ -1,31 +1,74 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-from .common import InfoExtractor
-from ..utils import smuggle_url
+from .adobepass import AdobePassIE
+from ..utils import (
+    smuggle_url,
+    update_url_query,
+    int_or_none,
+)
 
 
-class BravoTVIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?bravotv\.com/(?:[^/]+/)+videos/(?P<id>[^/?]+)'
-    _TEST = {
+class BravoTVIE(AdobePassIE):
+    _VALID_URL = r'https?://(?:www\.)?bravotv\.com/(?:[^/]+/)+(?P<id>[^/?#]+)'
+    _TESTS = [{
         'url': 'http://www.bravotv.com/last-chance-kitchen/season-5/videos/lck-ep-12-fishy-finale',
-        'md5': 'd60cdf68904e854fac669bd26cccf801',
+        'md5': '9086d0b7ef0ea2aabc4781d75f4e5863',
         'info_dict': {
-            'id': 'LitrBdX64qLn',
+            'id': 'zHyk1_HU_mPy',
             'ext': 'mp4',
-            'title': 'Last Chance Kitchen Returns',
-            'description': 'S13: Last Chance Kitchen Returns for Top Chef Season 13',
-            'timestamp': 1448926740,
-            'upload_date': '20151130',
+            'title': 'LCK Ep 12: Fishy Finale',
+            'description': 'S13/E12: Two eliminated chefs have just 12 minutes to cook up a delicious fish dish.',
             'uploader': 'NBCU-BRAV',
+            'upload_date': '20160302',
+            'timestamp': 1456945320,
         }
-    }
+    }, {
+        'url': 'http://www.bravotv.com/below-deck/season-3/ep-14-reunion-part-1',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-        account_pid = self._search_regex(r'"account_pid"\s*:\s*"([^"]+)"', webpage, 'account pid')
-        release_pid = self._search_regex(r'"release_pid"\s*:\s*"([^"]+)"', webpage, 'release pid')
-        return self.url_result(smuggle_url(
-            'http://link.theplatform.com/s/%s/%s?mbr=true&switch=progressive' % (account_pid, release_pid),
-            {'force_smil_url': True}), 'ThePlatform', release_pid)
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+        settings = self._parse_json(self._search_regex(
+            r'jQuery\.extend\([^,]+,\s*({.+})\);', webpage, 'drupal settings'),
+            display_id)
+        info = {}
+        query = {
+            'mbr': 'true',
+        }
+        account_pid, release_pid = [None] * 2
+        tve = settings.get('sharedTVE')
+        if tve:
+            query['manifest'] = 'm3u'
+            account_pid = 'HNK2IC'
+            release_pid = tve['release_pid']
+            if tve.get('entitlement') == 'auth':
+                adobe_pass = settings.get('adobePass', {})
+                resource = self._get_mvpd_resource(
+                    adobe_pass.get('adobePassResourceId', 'bravo'),
+                    tve['title'], release_pid, tve.get('rating'))
+                query['auth'] = self._extract_mvpd_auth(
+                    url, release_pid, adobe_pass.get('adobePassRequestorId', 'bravo'), resource)
+        else:
+            shared_playlist = settings['shared_playlist']
+            account_pid = shared_playlist['account_pid']
+            metadata = shared_playlist['video_metadata'][shared_playlist['default_clip']]
+            release_pid = metadata['release_pid']
+            info.update({
+                'title': metadata['title'],
+                'description': metadata.get('description'),
+                'season_number': int_or_none(metadata.get('season_num')),
+                'episode_number': int_or_none(metadata.get('episode_num')),
+            })
+            query['switch'] = 'progressive'
+        info.update({
+            '_type': 'url_transparent',
+            'id': release_pid,
+            'url': smuggle_url(update_url_query(
+                'http://link.theplatform.com/s/%s/%s' % (account_pid, release_pid),
+                query), {'force_smil_url': True}),
+            'ie_key': 'ThePlatform',
+        })
+        return info

From 18b6216150fa39d5e3cdbf353339e1c010bcee8d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 23 Aug 2016 21:55:58 +0800
Subject: [PATCH 5868/6219] [openload] Fix extraction (closes #10408)

Thanks @yokrysty for the algorithm
---
 ChangeLog                        |  6 +++
 youtube_dl/extractor/openload.py | 81 +++++++-------------------------
 2 files changed, 24 insertions(+), 63 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index a8202d3de..651d4d5d7 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Extractors
+* [openload] Fix extraction (#10408)
+
+
 version 2016.08.22
 
 Core
diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py
index 4e80ca9ff..e181d0b3a 100644
--- a/youtube_dl/extractor/openload.py
+++ b/youtube_dl/extractor/openload.py
@@ -1,12 +1,12 @@
 # coding: utf-8
 from __future__ import unicode_literals, division
 
-import math
-
 from .common import InfoExtractor
-from ..compat import compat_chr
+from ..compat import (
+    compat_chr,
+    compat_ord,
+)
 from ..utils import (
-    decode_png,
     determine_ext,
     ExtractorError,
 )
@@ -42,71 +42,26 @@ class OpenloadIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
+        webpage = self._download_webpage('https://openload.co/embed/%s/' % video_id, video_id)
 
-        if 'File not found' in webpage:
+        if 'File not found' in webpage or 'deleted by the owner' in webpage:
             raise ExtractorError('File not found', expected=True)
 
-        # The following extraction logic is proposed by @Belderak and @gdkchan
-        # and declared to be used freely in youtube-dl
-        # See https://github.com/rg3/youtube-dl/issues/9706
+        # The following decryption algorithm is written by @yokrysty and
+        # declared to be freely used in youtube-dl
+        # See https://github.com/rg3/youtube-dl/issues/10408
+        enc_data = self._html_search_regex(
+            r'<span[^>]+id="hiddenurl"[^>]*>([^<]+)</span>', webpage, 'encrypted data')
 
-        numbers_js = self._download_webpage(
-            'https://openload.co/assets/js/obfuscator/n.js', video_id,
-            note='Downloading signature numbers')
-        signums = self._search_regex(
-            r'window\.signatureNumbers\s*=\s*[\'"](?P<data>[a-z]+)[\'"]',
-            numbers_js, 'signature numbers', group='data')
+        video_url_chars = []
 
-        linkimg_uri = self._search_regex(
-            r'<img[^>]+id="linkimg"[^>]+src="([^"]+)"', webpage, 'link image')
-        linkimg = self._request_webpage(
-            linkimg_uri, video_id, note=False).read()
+        for c in enc_data:
+            j = compat_ord(c)
+            if j >= 33 and j <= 126:
+                j = ((j + 14) % 94) + 33
+            video_url_chars += compat_chr(j)
 
-        width, height, pixels = decode_png(linkimg)
-
-        output = ''
-        for y in range(height):
-            for x in range(width):
-                r, g, b = pixels[y][3 * x:3 * x + 3]
-                if r == 0 and g == 0 and b == 0:
-                    break
-                else:
-                    output += compat_chr(r)
-                    output += compat_chr(g)
-                    output += compat_chr(b)
-
-        img_str_length = len(output) // 200
-        img_str = [[0 for x in range(img_str_length)] for y in range(10)]
-
-        sig_str_length = len(signums) // 260
-        sig_str = [[0 for x in range(sig_str_length)] for y in range(10)]
-
-        for i in range(10):
-            for j in range(img_str_length):
-                begin = i * img_str_length * 20 + j * 20
-                img_str[i][j] = output[begin:begin + 20]
-            for j in range(sig_str_length):
-                begin = i * sig_str_length * 26 + j * 26
-                sig_str[i][j] = signums[begin:begin + 26]
-
-        parts = []
-        # TODO: find better names for str_, chr_ and sum_
-        str_ = ''
-        for i in [2, 3, 5, 7]:
-            str_ = ''
-            sum_ = float(99)
-            for j in range(len(sig_str[i])):
-                for chr_idx in range(len(img_str[i][j])):
-                    if sum_ > float(122):
-                        sum_ = float(98)
-                    chr_ = compat_chr(int(math.floor(sum_)))
-                    if sig_str[i][j][chr_idx] == chr_ and j >= len(str_):
-                        sum_ += float(2.5)
-                        str_ += img_str[i][j][chr_idx]
-            parts.append(str_.replace(',', ''))
-
-        video_url = 'https://openload.co/stream/%s~%s~%s~%s' % (parts[3], parts[1], parts[2], parts[0])
+        video_url = 'https://openload.co/stream/%s?mime=true' % ''.join(video_url_chars)
 
         title = self._og_search_title(webpage, default=None) or self._search_regex(
             r'<span[^>]+class=["\']title["\'][^>]*>([^<]+)', webpage,

From ccb6570e9e625ff5e9adf88729e745acadcaff0e Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 23 Aug 2016 17:31:08 +0100
Subject: [PATCH 5869/6219] [syfy,bravotv] restrict drupal settings regex

---
 youtube_dl/extractor/bravotv.py | 2 +-
 youtube_dl/extractor/syfy.py    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/bravotv.py b/youtube_dl/extractor/bravotv.py
index 31763b4c6..a25d500e4 100644
--- a/youtube_dl/extractor/bravotv.py
+++ b/youtube_dl/extractor/bravotv.py
@@ -32,7 +32,7 @@ class BravoTVIE(AdobePassIE):
         display_id = self._match_id(url)
         webpage = self._download_webpage(url, display_id)
         settings = self._parse_json(self._search_regex(
-            r'jQuery\.extend\([^,]+,\s*({.+})\);', webpage, 'drupal settings'),
+            r'jQuery\.extend\(Drupal\.settings\s*,\s*({.+?})\);', webpage, 'drupal settings'),
             display_id)
         info = {}
         query = {
diff --git a/youtube_dl/extractor/syfy.py b/youtube_dl/extractor/syfy.py
index cc81f6003..ab8bab5cd 100644
--- a/youtube_dl/extractor/syfy.py
+++ b/youtube_dl/extractor/syfy.py
@@ -31,7 +31,7 @@ class SyfyIE(AdobePassIE):
         display_id = self._match_id(url)
         webpage = self._download_webpage(url, display_id)
         syfy_mpx = list(self._parse_json(self._search_regex(
-            r'jQuery\.extend\([^,]+,\s*({.+})\);', webpage, 'drupal settings'),
+            r'jQuery\.extend\(Drupal\.settings\s*,\s*({.+?})\);', webpage, 'drupal settings'),
             display_id)['syfy']['syfy_mpx'].values())[0]
         video_id = syfy_mpx['mpxGUID']
         title = syfy_mpx['episodeTitle']

From 1212e9972fce69df6bd871a5c301294427299cbb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 24 Aug 2016 00:25:21 +0700
Subject: [PATCH 5870/6219] [youtube] Fix authentication (#10392)

---
 youtube_dl/extractor/youtube.py | 27 ++++-----------------------
 1 file changed, 4 insertions(+), 23 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 268080ba6..38556d86e 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -91,36 +91,17 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
         if login_page is False:
             return
 
-        galx = self._search_regex(r'(?s)<input.+?name="GALX".+?value="(.+?)"',
-                                  login_page, 'Login GALX parameter')
+        login_form = self._hidden_inputs(login_page)
 
-        # Log in
-        login_form_strs = {
-            'continue': 'https://www.youtube.com/signin?action_handle_signin=true&feature=sign_in_button&hl=en_US&nomobiletemp=1',
+        login_form.update({
             'Email': username,
-            'GALX': galx,
             'Passwd': password,
-
-            'PersistentCookie': 'yes',
-            '_utf8': '霱',
-            'bgresponse': 'js_disabled',
-            'checkConnection': '',
-            'checkedDomains': 'youtube',
-            'dnConn': '',
-            'pstMsg': '0',
-            'rmShown': '1',
-            'secTok': '',
-            'signIn': 'Sign in',
-            'timeStmp': '',
-            'service': 'youtube',
-            'uilel': '3',
-            'hl': 'en_US',
-        }
+        })
 
         login_results = self._download_webpage(
             self._PASSWORD_CHALLENGE_URL, None,
             note='Logging in', errnote='unable to log in', fatal=False,
-            data=urlencode_postdata(login_form_strs))
+            data=urlencode_postdata(login_form))
         if login_results is False:
             return False
 

From 05bddcc512cd5058f1af1d5985979b70bdcf4711 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 24 Aug 2016 01:29:50 +0700
Subject: [PATCH 5871/6219] [youtube] Fix authentication (2) (Closes #10392)

---
 youtube_dl/extractor/youtube.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 38556d86e..d5d5b7334 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -94,6 +94,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
         login_form = self._hidden_inputs(login_page)
 
         login_form.update({
+            'checkConnection': 'youtube',
             'Email': username,
             'Passwd': password,
         })

From 6e52bbb41320e1b6f4b7a16a5e651d945ac14611 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 24 Aug 2016 01:36:27 +0700
Subject: [PATCH 5872/6219] [ChangeLog] Actualize

---
 ChangeLog | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 651d4d5d7..07ab5867f 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,7 +1,14 @@
 version <unreleased>
 
 Extractors
+* [youtube] Fix authentication (#10392)
 * [openload] Fix extraction (#10408)
++ [bravotv] Add support for Adobe Pass (#10407)
+* [bravotv] Fix clip info extraction (#10407)
+* [eagleplatform] Improve embedded videos detection (#10409)
+* [awaan] Fix extraction
+* [mtvservices:embedded] Update config URL
++ [abc:iview] Add extractor (#6148)
 
 
 version 2016.08.22

From c86f51ee38b2063ad4eec2f0bb6e3d3551be0855 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 24 Aug 2016 01:38:46 +0700
Subject: [PATCH 5873/6219] release 2016.08.24

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 9 +++++----
 youtube_dl/version.py     | 2 +-
 4 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 7dcca18a1..00f593783 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.08.22*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.08.22**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.08.24*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.08.24**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.08.22
+[debug] youtube-dl version 2016.08.24
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 07ab5867f..b4f6dbe08 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2016.08.24
 
 Extractors
 * [youtube] Fix authentication (#10392)
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index ca96d2b07..08db56fa9 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -16,6 +16,7 @@
  - **9gag**
  - **9now.com.au**
  - **abc.net.au**
+ - **abc.net.au:iview**
  - **Abc7News**
  - **abcnews**
  - **abcnews:video**
@@ -66,6 +67,10 @@
  - **audiomack**
  - **audiomack:album**
  - **auroravid**: AuroraVid
+ - **AWAAN**
+ - **awaan:live**
+ - **awaan:season**
+ - **awaan:video**
  - **Azubu**
  - **AzubuLive**
  - **BaiduVideo**: 百度视频
@@ -172,10 +177,6 @@
  - **daum.net:playlist**
  - **daum.net:user**
  - **DBTV**
- - **DCN**
- - **dcn:live**
- - **dcn:season**
- - **dcn:video**
  - **DctpTv**
  - **DeezerPlaylist**
  - **defense.gouv.fr**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index e33d32e97..c1194124e 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.08.22'
+__version__ = '2016.08.24'

From 8c3e35dd441ceed682da885368f5cd97afb1816e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 24 Aug 2016 08:41:52 +0700
Subject: [PATCH 5874/6219] [pluralsight] Add support for subtitles (Closes
 #9681)

---
 youtube_dl/extractor/pluralsight.py | 74 ++++++++++++++++++++++++-----
 1 file changed, 63 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index 9aab77645..afd3217d9 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -1,9 +1,10 @@
 from __future__ import unicode_literals
 
-import re
-import json
-import random
 import collections
+import json
+import os
+import random
+import re
 
 from .common import InfoExtractor
 from ..compat import (
@@ -12,10 +13,12 @@ from ..compat import (
 )
 from ..utils import (
     ExtractorError,
+    float_or_none,
     int_or_none,
     parse_duration,
     qualities,
     sanitized_Request,
+    srt_subtitles_timecode,
     urlencode_postdata,
 )
 
@@ -91,6 +94,51 @@ class PluralsightIE(PluralsightBaseIE):
         if all(p not in response for p in ('__INITIAL_STATE__', '"currentUser"')):
             raise ExtractorError('Unable to log in')
 
+    def _get_subtitles(self, author, clip_id, lang, name, duration, video_id):
+        captions_post = {
+            'a': author,
+            'cn': clip_id,
+            'lc': lang,
+            'm': name,
+        }
+        captions = self._download_json(
+            '%s/training/Player/Captions' % self._API_BASE, video_id,
+            'Downloading captions JSON', 'Unable to download captions JSON',
+            fatal=False, data=json.dumps(captions_post).encode('utf-8'),
+            headers={'Content-Type': 'application/json;charset=utf-8'})
+        if captions:
+            return {
+                lang: [{
+                    'ext': 'json',
+                    'data': json.dumps(captions),
+                }, {
+                    'ext': 'srt',
+                    'data': self._convert_subtitles(duration, captions),
+                }]
+            }
+
+    @staticmethod
+    def _convert_subtitles(duration, subs):
+        srt = ''
+        for num, current in enumerate(subs):
+            current = subs[num]
+            start, text = float_or_none(
+                current.get('DisplayTimeOffset')), current.get('Text')
+            if start is None or text is None:
+                continue
+            end = duration if num == len(subs) - 1 else float_or_none(
+                subs[num + 1].get('DisplayTimeOffset'))
+            srt += os.linesep.join(
+                (
+                    '%d' % num,
+                    '%s --> %s' % (
+                        srt_subtitles_timecode(start),
+                        srt_subtitles_timecode(end)),
+                    text,
+                    os.linesep,
+                ))
+        return srt
+
     def _real_extract(self, url):
         qs = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
 
@@ -138,6 +186,8 @@ class PluralsightIE(PluralsightBaseIE):
         if not clip:
             raise ExtractorError('Unable to resolve clip')
 
+        title = '%s - %s' % (module['title'], clip['title'])
+
         QUALITIES = {
             'low': {'width': 640, 'height': 480},
             'medium': {'width': 848, 'height': 640},
@@ -225,18 +275,20 @@ class PluralsightIE(PluralsightBaseIE):
                 formats.append(f)
         self._sort_formats(formats)
 
-        # TODO: captions
-        # http://www.pluralsight.com/training/Player/ViewClip + cap = true
-        # or
-        # http://www.pluralsight.com/training/Player/Captions
-        # { a = author, cn = clip_id, lc = end, m = name }
+        duration = int_or_none(
+            clip.get('duration')) or parse_duration(clip.get('formattedDuration'))
+
+        # TODO: other languages?
+        subtitles = self.extract_subtitles(
+            author, clip_id, 'en', name, duration, display_id)
 
         return {
             'id': clip.get('clipName') or clip['name'],
-            'title': '%s - %s' % (module['title'], clip['title']),
-            'duration': int_or_none(clip.get('duration')) or parse_duration(clip.get('formattedDuration')),
+            'title': title,
+            'duration': duration,
             'creator': author,
-            'formats': formats
+            'formats': formats,
+            'subtitles': subtitles,
         }
 
 

From 30317f4887178082809706ce8ac9cb989014c8fc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 24 Aug 2016 08:52:12 +0700
Subject: [PATCH 5875/6219] [pluralsight] Modernize and make more robust

---
 youtube_dl/extractor/pluralsight.py | 20 +++++++++-----------
 1 file changed, 9 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index afd3217d9..ea5caefa9 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -17,7 +17,6 @@ from ..utils import (
     int_or_none,
     parse_duration,
     qualities,
-    sanitized_Request,
     srt_subtitles_timecode,
     urlencode_postdata,
 )
@@ -78,12 +77,10 @@ class PluralsightIE(PluralsightBaseIE):
         if not post_url.startswith('http'):
             post_url = compat_urlparse.urljoin(self._LOGIN_URL, post_url)
 
-        request = sanitized_Request(
-            post_url, urlencode_postdata(login_form))
-        request.add_header('Content-Type', 'application/x-www-form-urlencoded')
-
         response = self._download_webpage(
-            request, None, 'Logging in as %s' % username)
+            post_url, None, 'Logging in as %s' % username,
+            data=urlencode_postdata(login_form),
+            headers={'Content-Type': 'application/x-www-form-urlencoded'})
 
         error = self._search_regex(
             r'<span[^>]+class="field-validation-error"[^>]*>([^<]+)</span>',
@@ -128,6 +125,8 @@ class PluralsightIE(PluralsightBaseIE):
                 continue
             end = duration if num == len(subs) - 1 else float_or_none(
                 subs[num + 1].get('DisplayTimeOffset'))
+            if end is None:
+                continue
             srt += os.linesep.join(
                 (
                     '%d' % num,
@@ -246,13 +245,12 @@ class PluralsightIE(PluralsightBaseIE):
                     'mt': ext,
                     'q': '%dx%d' % (f['width'], f['height']),
                 }
-                request = sanitized_Request(
-                    '%s/training/Player/ViewClip' % self._API_BASE,
-                    json.dumps(clip_post).encode('utf-8'))
-                request.add_header('Content-Type', 'application/json;charset=utf-8')
                 format_id = '%s-%s' % (ext, quality)
                 clip_url = self._download_webpage(
-                    request, display_id, 'Downloading %s URL' % format_id, fatal=False)
+                    '%s/training/Player/ViewClip' % self._API_BASE, display_id,
+                    'Downloading %s URL' % format_id, fatal=False,
+                    data=json.dumps(clip_post).encode('utf-8'),
+                    headers={'Content-Type': 'application/json;charset=utf-8'})
 
                 # Pluralsight tracks multiple sequential calls to ViewClip API and start
                 # to return 429 HTTP errors after some time (see

From 6d94cbd2f43548575b32907724f48331df1693ae Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 24 Aug 2016 10:07:06 +0700
Subject: [PATCH 5876/6219] [ChangeLog] Actualize

---
 ChangeLog | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index b4f6dbe08..b63f49ae1 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Extractors
++ [pluralsight] Add support for subtitles (#9681)
+
+
 version 2016.08.24
 
 Extractors

From d38b27dd9b108a7518dd291c5c231a53abd3f2df Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 24 Aug 2016 10:11:04 +0700
Subject: [PATCH 5877/6219] release 2016.08.24.1

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 youtube_dl/version.py     | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 00f593783..15acc025a 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.08.24*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.08.24**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.08.24.1*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.08.24.1**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.08.24
+[debug] youtube-dl version 2016.08.24.1
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index b63f49ae1..4f3f1265f 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2016.08.24.1
 
 Extractors
 + [pluralsight] Add support for subtitles (#9681)
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index c1194124e..7447d3d7e 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.08.24'
+__version__ = '2016.08.24.1'

From 97653f81b2565c752f2c107fc44167a93c3eef42 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 24 Aug 2016 21:18:56 +0800
Subject: [PATCH 5878/6219] [bilibili] Mark as broken

Bilibili now uses emscripten, which is very difficult for reverse
engineering. I don't expect it to be fixed in near future, so I mark
it as broken.

Ref: #10375
---
 youtube_dl/extractor/bilibili.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/bilibili.py b/youtube_dl/extractor/bilibili.py
index d8eb71821..d87c38a02 100644
--- a/youtube_dl/extractor/bilibili.py
+++ b/youtube_dl/extractor/bilibili.py
@@ -21,6 +21,8 @@ from ..utils import (
 
 
 class BiliBiliIE(InfoExtractor):
+    _WORKING = False
+
     _VALID_URL = r'https?://www\.bilibili\.(?:tv|com)/video/av(?P<id>\d+)'
 
     _TESTS = [{

From 0c75abbb7bb9135d145805e86c87a5a43b69ac15 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 24 Aug 2016 23:58:22 +0800
Subject: [PATCH 5879/6219] [mtvservices:embedded] Use another endpoint to get
 feed URL

Closes #10363

In the original mtvservices:embedded test case, config.xml is still used
to get the feed URL. Some other examples, including test_Generic_40
(http://www.vulture.com/2016/06/new-key-peele-sketches-released.html),
and the video mentioned in #10363, use another endpoint to get the feed
URL. The 'index.html' approach works for the original test case, too. So
I didn't keep the old approach.
---
 ChangeLog                    |  6 ++++++
 youtube_dl/extractor/bet.py  |  5 ++---
 youtube_dl/extractor/mtv.py  | 27 +++++++++++++--------------
 youtube_dl/extractor/nick.py |  5 ++---
 4 files changed, 23 insertions(+), 20 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 4f3f1265f..c3cc8f38f 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Extractors
+* [mtvservices:embedded] Fix extraction for the new 'edge' player (#10363)
+
+
 version 2016.08.24.1
 
 Extractors
diff --git a/youtube_dl/extractor/bet.py b/youtube_dl/extractor/bet.py
index bd3ee2e2e..1f8ef0303 100644
--- a/youtube_dl/extractor/bet.py
+++ b/youtube_dl/extractor/bet.py
@@ -2,7 +2,6 @@ from __future__ import unicode_literals
 
 from .mtv import MTVServicesInfoExtractor
 from ..utils import unified_strdate
-from ..compat import compat_urllib_parse_urlencode
 
 
 class BetIE(MTVServicesInfoExtractor):
@@ -53,9 +52,9 @@ class BetIE(MTVServicesInfoExtractor):
     _FEED_URL = "http://feeds.mtvnservices.com/od/feed/bet-mrss-player"
 
     def _get_feed_query(self, uri):
-        return compat_urllib_parse_urlencode({
+        return {
             'uuid': uri,
-        })
+        }
 
     def _extract_mgid(self, webpage):
         return self._search_regex(r'data-uri="([^"]+)', webpage, 'mgid')
diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index 200f340de..bdda68819 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -4,7 +4,6 @@ import re
 
 from .common import InfoExtractor
 from ..compat import (
-    compat_urllib_parse_urlencode,
     compat_str,
     compat_xpath,
 )
@@ -14,12 +13,13 @@ from ..utils import (
     fix_xml_ampersands,
     float_or_none,
     HEADRequest,
+    RegexNotFoundError,
     sanitized_Request,
     strip_or_none,
     timeconvert,
     unescapeHTML,
+    update_url_query,
     url_basename,
-    RegexNotFoundError,
     xpath_text,
 )
 
@@ -36,6 +36,11 @@ class MTVServicesInfoExtractor(InfoExtractor):
     def _id_from_uri(uri):
         return uri.split(':')[-1]
 
+    @staticmethod
+    def _remove_template_parameter(url):
+        # Remove the templates, like &device={device}
+        return re.sub(r'&[^=]*?={.*?}(?=(&|$))', '', url)
+
     # This was originally implemented for ComedyCentral, but it also works here
     @classmethod
     def _transform_rtmp_url(cls, rtmp_video_url):
@@ -117,9 +122,7 @@ class MTVServicesInfoExtractor(InfoExtractor):
         video_id = self._id_from_uri(uri)
         self.report_extraction(video_id)
         content_el = itemdoc.find('%s/%s' % (_media_xml_tag('group'), _media_xml_tag('content')))
-        mediagen_url = content_el.attrib['url']
-        # Remove the templates, like &device={device}
-        mediagen_url = re.sub(r'&[^=]*?={.*?}(?=(&|$))', '', mediagen_url)
+        mediagen_url = self._remove_template_parameter(content_el.attrib['url'])
         if 'acceptMethods' not in mediagen_url:
             mediagen_url += '&' if '?' in mediagen_url else '?'
             mediagen_url += 'acceptMethods=fms'
@@ -178,12 +181,12 @@ class MTVServicesInfoExtractor(InfoExtractor):
         data = {'uri': uri}
         if self._LANG:
             data['lang'] = self._LANG
-        return compat_urllib_parse_urlencode(data)
+        return data
 
     def _get_videos_info(self, uri):
         video_id = self._id_from_uri(uri)
         feed_url = self._get_feed_url(uri)
-        info_url = feed_url + '?' + self._get_feed_query(uri)
+        info_url = update_url_query(feed_url, self._get_feed_query(uri))
         return self._get_videos_info_from_url(info_url, video_id)
 
     def _get_videos_info_from_url(self, url, video_id):
@@ -256,13 +259,9 @@ class MTVServicesEmbeddedIE(MTVServicesInfoExtractor):
 
     def _get_feed_url(self, uri):
         video_id = self._id_from_uri(uri)
-        site_id = uri.replace(video_id, '')
-        config_url = ('http://media.mtvnservices.com/pmt-arc/e1/players/{0}/'
-                      'context52/config.xml'.format(site_id))
-        config_doc = self._download_xml(config_url, video_id)
-        feed_node = config_doc.find('.//feed')
-        feed_url = feed_node.text.strip().split('?')[0]
-        return feed_url
+        config = self._download_json(
+            'http://media.mtvnservices.com/pmt/e1/access/index.html?uri=%s&configtype=edge' % uri, video_id)
+        return self._remove_template_parameter(config['feedWithQueryParams'])
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
diff --git a/youtube_dl/extractor/nick.py b/youtube_dl/extractor/nick.py
index 9c54846e1..64730a624 100644
--- a/youtube_dl/extractor/nick.py
+++ b/youtube_dl/extractor/nick.py
@@ -2,7 +2,6 @@
 from __future__ import unicode_literals
 
 from .mtv import MTVServicesInfoExtractor
-from ..compat import compat_urllib_parse_urlencode
 from ..utils import update_url_query
 
 
@@ -59,10 +58,10 @@ class NickIE(MTVServicesInfoExtractor):
     }]
 
     def _get_feed_query(self, uri):
-        return compat_urllib_parse_urlencode({
+        return {
             'feed': 'nick_arc_player_prime',
             'mgid': uri,
-        })
+        }
 
     def _extract_mgid(self, webpage):
         return self._search_regex(r'data-contenturi="([^"]+)', webpage, 'mgid')

From 08773689f37341f8c70c3fd298f5910235b8c151 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 25 Aug 2016 01:29:32 +0800
Subject: [PATCH 5880/6219] [kickstarter] Silent the warning for og:description

Closes #10415
---
 youtube_dl/extractor/kickstarter.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/kickstarter.py b/youtube_dl/extractor/kickstarter.py
index 9f1ade2e4..c61e78622 100644
--- a/youtube_dl/extractor/kickstarter.py
+++ b/youtube_dl/extractor/kickstarter.py
@@ -37,7 +37,6 @@ class KickStarterIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Power Drive 2000',
         },
-        'expected_warnings': ['OpenGraph description'],
     }]
 
     def _real_extract(self, url):
@@ -67,6 +66,6 @@ class KickStarterIE(InfoExtractor):
             'id': video_id,
             'url': video_url,
             'title': title,
-            'description': self._og_search_description(webpage),
+            'description': self._og_search_description(webpage, default=None),
             'thumbnail': thumbnail,
         }

From 0c6422cdd649c6f39cb2d8680e29f91da18d8c57 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 25 Aug 2016 07:34:55 +0700
Subject: [PATCH 5881/6219] [README.md] Add FAQ entry for streaming to player

---
 README.md | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index a10aaf35c..52e53803e 100644
--- a/README.md
+++ b/README.md
@@ -730,7 +730,7 @@ Videos or video formats streamed via RTMP protocol can only be downloaded when [
 
 ### I have downloaded a video but how can I play it?
 
-Once the video is fully downloaded, use any video player, such as [mpv](https://mpv.io/), [vlc](http://www.videolan.org) or [mplayer](http://www.mplayerhq.hu/).
+Once the video is fully downloaded, use any video player, such as [mpv](https://mpv.io/), [vlc](http://www.videolan.org/) or [mplayer](http://www.mplayerhq.hu/).
 
 ### I extracted a video URL with `-g`, but it does not play on another machine / in my webbrowser.
 
@@ -816,6 +816,12 @@ Use the `--cookies` option, for example `--cookies /path/to/cookies/file.txt`. N
 
 Passing cookies to youtube-dl is a good way to workaround login when a particular extractor does not implement it explicitly. Another use case is working around [CAPTCHA](https://en.wikipedia.org/wiki/CAPTCHA) some websites require you to solve in particular cases in order to get access (e.g. YouTube, CloudFlare).
 
+### How do I stream directly to media player?
+
+You will first need to tell youtube-dl to stream media to stdout with `-o -`, and also tell your media player to read from stdin (it must be capable of this for streaming) and then pipe former to latter. For example, streaming to [vlc](http://www.videolan.org/) can be achieved with:
+
+    youtube-dl -o - http://www.youtube.com/watch?v=BaW_jenozKcj | vlc -
+
 ### Can you add support for this anime video site, or site which shows current movies for free?
 
 As a matter of policy (as well as legality), youtube-dl does not include support for services that specialize in infringing copyright. As a rule of thumb, if you cannot easily find a video that the service is quite obviously allowed to distribute (i.e. that has been uploaded by the creator, the creator's distributor, or is published under a free license), the service is probably unfit for inclusion to youtube-dl.

From 073ac1225f6fe28905e11f29f2d23f4b4db50f9c Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 25 Aug 2016 08:33:16 +0100
Subject: [PATCH 5882/6219] [utils] add ac-3 to the list of audio codecs in
 parse_codecs

---
 youtube_dl/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 41ca562f1..1091f17f3 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2183,7 +2183,7 @@ def parse_codecs(codecs_str):
         if codec in ('avc1', 'avc2', 'avc3', 'avc4', 'vp9', 'vp8', 'hev1', 'hev2', 'h263', 'h264', 'mp4v'):
             if not vcodec:
                 vcodec = full_codec
-        elif codec in ('mp4a', 'opus', 'vorbis', 'mp3', 'aac'):
+        elif codec in ('mp4a', 'opus', 'vorbis', 'mp3', 'aac', 'ac-3'):
             if not acodec:
                 acodec = full_codec
         else:

From 07ea9c9b05359aef14472dfa66a6578d21c88e96 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 25 Aug 2016 08:37:41 +0100
Subject: [PATCH 5883/6219] [downloader/hls] fill IV with zeros for IVs shorter
 than 16-octet

---
 youtube_dl/downloader/hls.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index 8d7971e5d..8dd1b898e 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -120,7 +120,7 @@ class HlsFD(FragmentFD):
                     decrypt_info = parse_m3u8_attributes(line[11:])
                     if decrypt_info['METHOD'] == 'AES-128':
                         if 'IV' in decrypt_info:
-                            decrypt_info['IV'] = binascii.unhexlify(decrypt_info['IV'][2:])
+                            decrypt_info['IV'] = binascii.unhexlify(decrypt_info['IV'][2:].zfill(32))
                         if not re.match(r'^https?://', decrypt_info['URI']):
                             decrypt_info['URI'] = compat_urlparse.urljoin(
                                 man_url, decrypt_info['URI'])

From f39ffc5877e4e9f112fa26ff21079f179b4aec46 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 25 Aug 2016 08:39:23 +0100
Subject: [PATCH 5884/6219] [common] extract formats from #EXT-X-MEDIA tags

---
 youtube_dl/extractor/common.py | 32 ++++++++++++++++++++++----------
 1 file changed, 22 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 8ed16deee..da0af29ec 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1202,30 +1202,45 @@ class InfoExtractor(object):
                 'preference': preference,
             }]
         last_info = None
-        last_media = None
         for line in m3u8_doc.splitlines():
             if line.startswith('#EXT-X-STREAM-INF:'):
                 last_info = parse_m3u8_attributes(line)
             elif line.startswith('#EXT-X-MEDIA:'):
-                last_media = parse_m3u8_attributes(line)
+                media = parse_m3u8_attributes(line)
+                media_type = media.get('TYPE')
+                if media_type in ('VIDEO', 'AUDIO'):
+                    media_url = media.get('URI')
+                    if media_url:
+                        format_id = []
+                        for v in (media.get('GROUP-ID'), media.get('NAME')):
+                            if v:
+                                format_id.append(v)
+                        formats.append({
+                            'format_id': '-'.join(format_id),
+                            'url': format_url(media_url),
+                            'language': media.get('LANGUAGE'),
+                            'vcodec': 'none' if media_type == 'AUDIO' else None,
+                            'ext': ext,
+                            'protocol': entry_protocol,
+                            'preference': preference,
+                        })
             elif line.startswith('#') or not line.strip():
                 continue
             else:
                 if last_info is None:
                     formats.append({'url': format_url(line)})
                     continue
-                tbr = int_or_none(last_info.get('BANDWIDTH'), scale=1000)
+                tbr = int_or_none(last_info.get('AVERAGE-BANDWIDTH') or last_info.get('BANDWIDTH'), scale=1000)
                 format_id = []
                 if m3u8_id:
                     format_id.append(m3u8_id)
-                last_media_name = last_media.get('NAME') if last_media and last_media.get('TYPE') not in ('SUBTITLES', 'CLOSED-CAPTIONS') else None
-                # Despite specification does not mention NAME attribute for
-                # EXT-X-STREAM-INF it still sometimes may be present
-                stream_name = last_info.get('NAME') or last_media_name
                 # Bandwidth of live streams may differ over time thus making
                 # format_id unpredictable. So it's better to keep provided
                 # format_id intact.
                 if not live:
+                    # Despite specification does not mention NAME attribute for
+                    # EXT-X-STREAM-INF it still sometimes may be present
+                    stream_name = last_info.get('NAME')
                     format_id.append(stream_name if stream_name else '%d' % (tbr if tbr else len(formats)))
                 f = {
                     'format_id': '-'.join(format_id),
@@ -1252,9 +1267,6 @@ class InfoExtractor(object):
                         'abr': abr,
                     })
                 f.update(parse_codecs(last_info.get('CODECS')))
-                if last_media is not None:
-                    f['m3u8_media'] = last_media
-                    last_media = None
                 formats.append(f)
                 last_info = {}
         return formats

From 75fa990dc669563b51f22eeddd2f33acc41c8599 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 25 Aug 2016 08:46:54 +0100
Subject: [PATCH 5885/6219] [YoutubeDL] add fallback value for thumbnails
 values in thumbnails sorting

---
 youtube_dl/YoutubeDL.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 0b3e3da82..c499c1da4 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1256,8 +1256,8 @@ class YoutubeDL(object):
                 info_dict['thumbnails'] = thumbnails = [{'url': thumbnail}]
         if thumbnails:
             thumbnails.sort(key=lambda t: (
-                t.get('preference'), t.get('width'), t.get('height'),
-                t.get('id'), t.get('url')))
+                t.get('preference') or -1, t.get('width') or -1, t.get('height') or -1,
+                t.get('id') or '', t.get('url')))
             for i, t in enumerate(thumbnails):
                 t['url'] = sanitize_url(t['url'])
                 if t.get('width') and t.get('height'):

From 30afe4aeb25576225d3f3ca486983b5ad9258aa0 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 25 Aug 2016 08:49:15 +0100
Subject: [PATCH 5886/6219] [cbc] Add support for watch.cbc.ca

---
 youtube_dl/extractor/cbc.py        | 172 +++++++++++++++++++++++++++++
 youtube_dl/extractor/extractors.py |   2 +
 2 files changed, 174 insertions(+)

diff --git a/youtube_dl/extractor/cbc.py b/youtube_dl/extractor/cbc.py
index a87e97140..d71fddf58 100644
--- a/youtube_dl/extractor/cbc.py
+++ b/youtube_dl/extractor/cbc.py
@@ -9,10 +9,19 @@ from ..utils import (
     js_to_json,
     smuggle_url,
     try_get,
+    xpath_text,
+    xpath_element,
+    xpath_with_ns,
+    find_xpath_attr,
+    parse_iso8601,
+    parse_age_limit,
+    int_or_none,
+    ExtractorError,
 )
 
 
 class CBCIE(InfoExtractor):
+    IE_NAME = 'cbc.ca'
     _VALID_URL = r'https?://(?:www\.)?cbc\.ca/(?!player/)(?:[^/]+/)+(?P<id>[^/?#]+)'
     _TESTS = [{
         # with mediaId
@@ -114,6 +123,7 @@ class CBCIE(InfoExtractor):
 
 
 class CBCPlayerIE(InfoExtractor):
+    IE_NAME = 'cbc.ca:player'
     _VALID_URL = r'(?:cbcplayer:|https?://(?:www\.)?cbc\.ca/(?:player/play/|i/caffeine/syndicate/\?mediaId=))(?P<id>\d+)'
     _TESTS = [{
         'url': 'http://www.cbc.ca/player/play/2683190193',
@@ -167,3 +177,165 @@ class CBCPlayerIE(InfoExtractor):
                 }),
             'id': video_id,
         }
+
+
+class CBCWatchBaseIE(InfoExtractor):
+    _device_id = None
+    _device_token = None
+    _API_BASE_URL = 'https://api-cbc.cloud.clearleap.com/cloffice/client/'
+    _NS_MAP = {
+        'media': 'http://search.yahoo.com/mrss/',
+        'clearleap': 'http://www.clearleap.com/namespace/clearleap/1.0/',
+    }
+
+    def _call_api(self, path, video_id):
+        url = path if path.startswith('http') else self._API_BASE_URL + path
+        result = self._download_xml(url, video_id, headers={
+            'X-Clearleap-DeviceId': self._device_id,
+            'X-Clearleap-DeviceToken': self._device_token,
+        })
+        error_message = xpath_text(result, 'userMessage') or xpath_text(result, 'systemMessage')
+        if error_message:
+            raise ExtractorError('%s said: %s' % (self.IE_NAME, error_message))
+        return result
+
+    def _real_initialize(self):
+        if not self._device_id or not self._device_token:
+            device = self._downloader.cache.load('cbcwatch', 'device') or {}
+            self._device_id, self._device_token = device.get('id'), device.get('token')
+            if not self._device_id or not self._device_token:
+                result = self._download_xml(
+                    self._API_BASE_URL + 'device/register',
+                    None, data=b'<device><type>web</type></device>')
+                self._device_id = xpath_text(result, 'deviceId', fatal=True)
+                self._device_token = xpath_text(result, 'deviceToken', fatal=True)
+                self._downloader.cache.store(
+                    'cbcwatch', 'device', {
+                        'id': self._device_id,
+                        'token': self._device_token,
+                    })
+
+    def _parse_rss_feed(self, rss):
+        channel = xpath_element(rss, 'channel', fatal=True)
+
+        def _add_ns(path):
+            return xpath_with_ns(path, self._NS_MAP)
+
+        entries = []
+        for item in channel.findall('item'):
+            guid = xpath_text(item, 'guid', fatal=True)
+            title = xpath_text(item, 'title', fatal=True)
+
+            media_group = xpath_element(item, _add_ns('media:group'), fatal=True)
+            content = xpath_element(media_group, _add_ns('media:content'), fatal=True)
+            content_url = content.attrib['url']
+
+            thumbnails = []
+            for thumbnail in media_group.findall(_add_ns('media:thumbnail')):
+                thumbnail_url = thumbnail.get('url')
+                if not thumbnail_url:
+                    continue
+                thumbnails.append({
+                    'id': thumbnail.get('profile'),
+                    'url': thumbnail_url,
+                    'width': int_or_none(thumbnail.get('width')),
+                    'height': int_or_none(thumbnail.get('height')),
+                })
+
+            timestamp = None
+            release_date = find_xpath_attr(
+                item, _add_ns('media:credit'), 'role', 'releaseDate')
+            if release_date is not None:
+                timestamp = parse_iso8601(release_date.text)
+
+            entries.append({
+                '_type': 'url_transparent',
+                'url': content_url,
+                'id': guid,
+                'title': title,
+                'description': xpath_text(item, 'description'),
+                'timestamp': timestamp,
+                'duration': int_or_none(content.get('duration')),
+                'age_limit': parse_age_limit(xpath_text(item, _add_ns('media:rating'))),
+                'episode': xpath_text(item, _add_ns('clearleap:episode')),
+                'episode_number': int_or_none(xpath_text(item, _add_ns('clearleap:episodeInSeason'))),
+                'series': xpath_text(item, _add_ns('clearleap:series')),
+                'season_number': int_or_none(xpath_text(item, _add_ns('clearleap:season'))),
+                'thumbnails': thumbnails,
+                'ie_key': 'CBCWatchVideo',
+            })
+
+        return self.playlist_result(
+            entries, xpath_text(channel, 'guid'),
+            xpath_text(channel, 'title'),
+            xpath_text(channel, 'description'))
+
+
+class CBCWatchVideoIE(CBCWatchBaseIE):
+    IE_NAME = 'cbc.ca:watch:video'
+    _VALID_URL = r'https?://api-cbc\.cloud\.clearleap\.com/cloffice/client/web/play/?\?.*?\bcontentId=(?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})'
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        result = self._call_api(url, video_id)
+
+        m3u8_url = xpath_text(result, 'url', fatal=True)
+        formats = self._extract_m3u8_formats(re.sub(r'/([^/]+)/[^/?]+\.m3u8', r'/\1/\1.m3u8', m3u8_url), video_id, 'mp4', fatal=False)
+        if len(formats) < 2:
+            formats = self._extract_m3u8_formats(m3u8_url, video_id, 'mp4')
+        # Despite metadata in m3u8 all video+audio formats are
+        # actually video-only (no audio)
+        for f in formats:
+            if f.get('acodec') != 'none' and f.get('vcodec') != 'none':
+                f['acodec'] = 'none'
+        self._sort_formats(formats)
+
+        info = {
+            'id': video_id,
+            'title': video_id,
+            'formats': formats,
+        }
+
+        rss = xpath_element(result, 'rss')
+        if rss:
+            info.update(self._parse_rss_feed(rss)['entries'][0])
+            del info['url']
+            del info['_type']
+            del info['ie_key']
+        return info
+
+
+class CBCWatchIE(CBCWatchBaseIE):
+    IE_NAME = 'cbc.ca:watch'
+    _VALID_URL = r'https?://watch\.cbc\.ca/(?:[^/]+/)+(?P<id>[0-9a-f-]+)'
+    _TESTS = [{
+        'url': 'http://watch.cbc.ca/doc-zone/season-6/customer-disservice/38e815a-009e3ab12e4',
+        'info_dict': {
+            'id': '38e815a-009e3ab12e4',
+            'ext': 'mp4',
+            'title': 'Customer (Dis)Service',
+            'description': 'md5:8bdd6913a0fe03d4b2a17ebe169c7c87',
+            'upload_date': '20160219',
+            'timestamp': 1455840000,
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+            'format': 'bestvideo',
+        },
+        'skip': 'Geo-restricted to Canada',
+    }, {
+        'url': 'http://watch.cbc.ca/arthur/all/1ed4b385-cd84-49cf-95f0-80f004680057',
+        'info_dict': {
+            'id': '1ed4b385-cd84-49cf-95f0-80f004680057',
+            'title': 'Arthur',
+            'description': 'Arthur, the sweetest 8-year-old aardvark, and his pals solve all kinds of problems with humour, kindness and teamwork.',
+        },
+        'playlist_mincount': 30,
+        'skip': 'Geo-restricted to Canada',
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        rss = self._call_api('web/browse/' + video_id, video_id)
+        return self._parse_rss_feed(rss)
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 04cd23bdb..a58145e3e 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -130,6 +130,8 @@ from .carambatv import (
 from .cbc import (
     CBCIE,
     CBCPlayerIE,
+    CBCWatchVideoIE,
+    CBCWatchIE,
 )
 from .cbs import CBSIE
 from .cbslocal import CBSLocalIE

From f70e9229e623eb041ad514605ceca484b176b850 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 25 Aug 2016 09:11:23 +0100
Subject: [PATCH 5887/6219] [discoverygo] detect when video needs
 authentication(closes #10425)

---
 youtube_dl/extractor/discoverygo.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/discoverygo.py b/youtube_dl/extractor/discoverygo.py
index cba709935..e86d16d36 100644
--- a/youtube_dl/extractor/discoverygo.py
+++ b/youtube_dl/extractor/discoverygo.py
@@ -7,6 +7,7 @@ from ..utils import (
     int_or_none,
     parse_age_limit,
     unescapeHTML,
+    ExtractorError,
 )
 
 
@@ -53,7 +54,13 @@ class DiscoveryGoIE(InfoExtractor):
 
         title = video['name']
 
-        stream = video['stream']
+        stream = video.get('stream')
+        if not stream:
+            raise ExtractorError(
+                'This video is only available via cable service provider subscription that'
+                ' is not currently supported. You may want to use --cookies.'
+                if video.get('authenticated') is True else 'Unable to find stream',
+                expected=True)
         STREAM_URL_SUFFIX = 'streamUrl'
         formats = []
         for stream_kind in ('', 'hds'):

From 5c13c285660c2811206c5bb29acf43b114ab31e3 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 25 Aug 2016 09:55:23 +0100
Subject: [PATCH 5888/6219] raise unexpected error when no stream found

---
 youtube_dl/extractor/adultswim.py   | 11 ++++++-----
 youtube_dl/extractor/discoverygo.py | 11 ++++++-----
 2 files changed, 12 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/adultswim.py b/youtube_dl/extractor/adultswim.py
index 3f7f8c036..96599048f 100644
--- a/youtube_dl/extractor/adultswim.py
+++ b/youtube_dl/extractor/adultswim.py
@@ -162,11 +162,12 @@ class AdultSwimIE(InfoExtractor):
             elif video_info.get('videoPlaybackID'):
                 segment_ids = [video_info['videoPlaybackID']]
             else:
-                raise ExtractorError(
-                    'This video is only available via cable service provider subscription that'
-                    ' is not currently supported. You may want to use --cookies.'
-                    if video_info.get('auth') is True else 'Unable to find stream or clips',
-                    expected=True)
+                if video_info.get('auth') is True:
+                    raise ExtractorError(
+                        'This video is only available via cable service provider subscription that'
+                        ' is not currently supported. You may want to use --cookies.', expected=True)
+                else:
+                    raise ExtractorError('Unable to find stream or clips')
 
         episode_id = video_info['id']
         episode_title = video_info['title']
diff --git a/youtube_dl/extractor/discoverygo.py b/youtube_dl/extractor/discoverygo.py
index e86d16d36..c4e83b2c3 100644
--- a/youtube_dl/extractor/discoverygo.py
+++ b/youtube_dl/extractor/discoverygo.py
@@ -56,11 +56,12 @@ class DiscoveryGoIE(InfoExtractor):
 
         stream = video.get('stream')
         if not stream:
-            raise ExtractorError(
-                'This video is only available via cable service provider subscription that'
-                ' is not currently supported. You may want to use --cookies.'
-                if video.get('authenticated') is True else 'Unable to find stream',
-                expected=True)
+            if video.get('authenticated') is True:
+                raise ExtractorError(
+                    'This video is only available via cable service provider subscription that'
+                    ' is not currently supported. You may want to use --cookies.', expected=True)
+            else:
+                raise ExtractorError('Unable to find stream')
         STREAM_URL_SUFFIX = 'streamUrl'
         formats = []
         for stream_kind in ('', 'hds'):

From d37708fc861b3534c522f2892b5cd2ee716e1035 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 25 Aug 2016 11:53:47 +0100
Subject: [PATCH 5889/6219] [YoutubeDL] check only for None Value in thumbnails
 sorting

---
 youtube_dl/YoutubeDL.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index c499c1da4..805733fb7 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1256,8 +1256,10 @@ class YoutubeDL(object):
                 info_dict['thumbnails'] = thumbnails = [{'url': thumbnail}]
         if thumbnails:
             thumbnails.sort(key=lambda t: (
-                t.get('preference') or -1, t.get('width') or -1, t.get('height') or -1,
-                t.get('id') or '', t.get('url')))
+                t.get('preference') if t.get('preference') is not None else -1,
+                t.get('width') if t.get('width') is not None else -1,
+                t.get('height') if t.get('height') is not None else -1,
+                t.get('id') if t.get('id') is not None else '', t.get('url')))
             for i, t in enumerate(thumbnails):
                 t['url'] = sanitize_url(t['url'])
                 if t.get('width') and t.get('height'):

From 6a76b53355947eef2a534d8f2505ed683db8754f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 25 Aug 2016 18:05:01 +0700
Subject: [PATCH 5890/6219] [README.md] Quote URL in streaming to player FAQ
 entry

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 52e53803e..1aa267752 100644
--- a/README.md
+++ b/README.md
@@ -820,7 +820,7 @@ Passing cookies to youtube-dl is a good way to workaround login when a particula
 
 You will first need to tell youtube-dl to stream media to stdout with `-o -`, and also tell your media player to read from stdin (it must be capable of this for streaming) and then pipe former to latter. For example, streaming to [vlc](http://www.videolan.org/) can be achieved with:
 
-    youtube-dl -o - http://www.youtube.com/watch?v=BaW_jenozKcj | vlc -
+    youtube-dl -o - "http://www.youtube.com/watch?v=BaW_jenozKcj" | vlc -
 
 ### Can you add support for this anime video site, or site which shows current movies for free?
 

From ea01cdbf61c9a689e7914dd2d06371f3ef73b490 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 25 Aug 2016 18:17:45 +0700
Subject: [PATCH 5891/6219] [README.md] Clarify how to export cookies from
 browser for cookies FAQ entry

---
 README.md | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 1aa267752..0bb7b791f 100644
--- a/README.md
+++ b/README.md
@@ -812,7 +812,11 @@ Either prepend `http://www.youtube.com/watch?v=` or separate the ID from the opt
 
 ### How do I pass cookies to youtube-dl?
 
-Use the `--cookies` option, for example `--cookies /path/to/cookies/file.txt`. Note that the cookies file must be in Mozilla/Netscape format and the first line of the cookies file must be either `# HTTP Cookie File` or `# Netscape HTTP Cookie File`. Make sure you have correct [newline format](https://en.wikipedia.org/wiki/Newline) in the cookies file and convert newlines if necessary to correspond with your OS, namely `CRLF` (`\r\n`) for Windows, `LF` (`\n`) for Linux and `CR` (`\r`) for Mac OS. `HTTP Error 400: Bad Request` when using `--cookies` is a good sign of invalid newline format.
+Use the `--cookies` option, for example `--cookies /path/to/cookies/file.txt`.
+
+In order to extract cookies from browser use any conforming browser extension for exporting cookies. For example, [cookies.txt](https://chrome.google.com/webstore/detail/cookiestxt/njabckikapfpffapmjgojcnbfjonfjfg) (for Chrome) or [Export Cookies](https://addons.mozilla.org/ru/firefox/addon/export-cookies/) (for Firefox).
+
+Note that the cookies file must be in Mozilla/Netscape format and the first line of the cookies file must be either `# HTTP Cookie File` or `# Netscape HTTP Cookie File`. Make sure you have correct [newline format](https://en.wikipedia.org/wiki/Newline) in the cookies file and convert newlines if necessary to correspond with your OS, namely `CRLF` (`\r\n`) for Windows, `LF` (`\n`) for Linux and `CR` (`\r`) for Mac OS. `HTTP Error 400: Bad Request` when using `--cookies` is a good sign of invalid newline format.
 
 Passing cookies to youtube-dl is a good way to workaround login when a particular extractor does not implement it explicitly. Another use case is working around [CAPTCHA](https://en.wikipedia.org/wiki/CAPTCHA) some websites require you to solve in particular cases in order to get access (e.g. YouTube, CloudFlare).
 

From f26a298247fe19bc8114d6f7a280140dfabee984 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 25 Aug 2016 18:19:41 +0700
Subject: [PATCH 5892/6219] [README.md] Use en-US URL in cookies FAQ entry

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 0bb7b791f..04f423c17 100644
--- a/README.md
+++ b/README.md
@@ -814,7 +814,7 @@ Either prepend `http://www.youtube.com/watch?v=` or separate the ID from the opt
 
 Use the `--cookies` option, for example `--cookies /path/to/cookies/file.txt`.
 
-In order to extract cookies from browser use any conforming browser extension for exporting cookies. For example, [cookies.txt](https://chrome.google.com/webstore/detail/cookiestxt/njabckikapfpffapmjgojcnbfjonfjfg) (for Chrome) or [Export Cookies](https://addons.mozilla.org/ru/firefox/addon/export-cookies/) (for Firefox).
+In order to extract cookies from browser use any conforming browser extension for exporting cookies. For example, [cookies.txt](https://chrome.google.com/webstore/detail/cookiestxt/njabckikapfpffapmjgojcnbfjonfjfg) (for Chrome) or [Export Cookies](https://addons.mozilla.org/en-US/firefox/addon/export-cookies/) (for Firefox).
 
 Note that the cookies file must be in Mozilla/Netscape format and the first line of the cookies file must be either `# HTTP Cookie File` or `# Netscape HTTP Cookie File`. Make sure you have correct [newline format](https://en.wikipedia.org/wiki/Newline) in the cookies file and convert newlines if necessary to correspond with your OS, namely `CRLF` (`\r\n`) for Windows, `LF` (`\n`) for Linux and `CR` (`\r`) for Mac OS. `HTTP Error 400: Bad Request` when using `--cookies` is a good sign of invalid newline format.
 

From 4c8f9c2577da2f4ba7300d44613599e96cde5c9f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 25 Aug 2016 18:27:15 +0700
Subject: [PATCH 5893/6219] [README.md] Add comments in sample configuration
 for clarity

---
 README.md | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index 04f423c17..e01b71cff 100644
--- a/README.md
+++ b/README.md
@@ -412,11 +412,19 @@ You can configure youtube-dl by placing any supported command line option to a c
 
 For example, with the following configuration file youtube-dl will always extract the audio, not copy the mtime, use a proxy and save all videos under `Movies` directory in your home directory:
 ```
--x
---no-mtime
---proxy 127.0.0.1:3128
--o ~/Movies/%(title)s.%(ext)s
 # Lines starting with # are comments
+
+# Always extract audio
+-x
+
+# Do not copy the mtime
+--no-mtime
+
+# Use this proxy
+--proxy 127.0.0.1:3128
+
+# Save all videos under Movies directory in your home directory
+-o ~/Movies/%(title)s.%(ext)s
 ```
 
 Note that options in configuration file are just the same options aka switches used in regular command line calls thus there **must be no whitespace** after `-` or `--`, e.g. `-o` or `--proxy` but not `- o` or `-- proxy`.

From 5a3efcd27c1262cc7132f7e1a092524b580788ee Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 25 Aug 2016 18:57:31 +0700
Subject: [PATCH 5894/6219] [README.md] Add FAQ entry for download archive

---
 README.md | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/README.md b/README.md
index e01b71cff..1b9e2a989 100644
--- a/README.md
+++ b/README.md
@@ -834,6 +834,18 @@ You will first need to tell youtube-dl to stream media to stdout with `-o -`, an
 
     youtube-dl -o - "http://www.youtube.com/watch?v=BaW_jenozKcj" | vlc -
 
+### How do I download only new videos from playlist?
+
+Use the download archive feature. With this feature you should initially download the complete playlist with `--download-archive /path/to/download/archive/file.txt` that will record identifiers of all the videos in a special *download archive file*. Each subsequent run with the same `--download-archive` will download only new videos that are not yet in download archive (if any) and also record them in download archive. Note that only successful downloads are recorded in download archive.
+
+For example, first run will download complete `PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re` playlist and create download archive `archive.txt`:
+
+    youtube-dl --download-archive archive.txt "https://www.youtube.com/playlist?list=PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re"
+
+Each subsequent run will only download new videos if any:
+
+    youtube-dl --download-archive archive.txt "https://www.youtube.com/playlist?list=PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re"
+
 ### Can you add support for this anime video site, or site which shows current movies for free?
 
 As a matter of policy (as well as legality), youtube-dl does not include support for services that specialize in infringing copyright. As a rule of thumb, if you cannot easily find a video that the service is quite obviously allowed to distribute (i.e. that has been uploaded by the creator, the creator's distributor, or is published under a free license), the service is probably unfit for inclusion to youtube-dl.

From c1f62dd338e0965507ee0976bc88885fdb0fa780 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 25 Aug 2016 14:45:01 +0200
Subject: [PATCH 5895/6219] [README] Clean up grammar in --download-archive
 paragraph

---
 README.md | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index 1b9e2a989..20241307f 100644
--- a/README.md
+++ b/README.md
@@ -834,15 +834,15 @@ You will first need to tell youtube-dl to stream media to stdout with `-o -`, an
 
     youtube-dl -o - "http://www.youtube.com/watch?v=BaW_jenozKcj" | vlc -
 
-### How do I download only new videos from playlist?
+### How do I download only new videos from a playlist?
 
-Use the download archive feature. With this feature you should initially download the complete playlist with `--download-archive /path/to/download/archive/file.txt` that will record identifiers of all the videos in a special *download archive file*. Each subsequent run with the same `--download-archive` will download only new videos that are not yet in download archive (if any) and also record them in download archive. Note that only successful downloads are recorded in download archive.
+Use download-archive feature. With this feature you should initially download the complete playlist with `--download-archive /path/to/download/archive/file.txt` that will record identifiers of all the videos in a special file. Each subsequent run with the same `--download-archive` will download only new videos that and skip all videos that have been downloaded before. Note that only successful downloads are recorded in the file.
 
-For example, first run will download complete `PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re` playlist and create download archive `archive.txt`:
+For example, at first,
 
     youtube-dl --download-archive archive.txt "https://www.youtube.com/playlist?list=PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re"
 
-Each subsequent run will only download new videos if any:
+will download the complete `PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re` playlist and create a file `archive.txt`. Each subsequent run will only download new videos if any:
 
     youtube-dl --download-archive archive.txt "https://www.youtube.com/playlist?list=PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re"
 

From dc2c37f3162da534281f5f3758231e4c2cb8d1b2 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 25 Aug 2016 20:45:57 +0800
Subject: [PATCH 5896/6219] [spankbang] Fix description and uploader (closes
 #10339)

---
 ChangeLog                         | 1 +
 youtube_dl/extractor/spankbang.py | 8 +++-----
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index c3cc8f38f..5fb596e33 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
 version <unreleased>
 
 Extractors
+* [spankbang] Fix description and uploader (#10339)
 * [mtvservices:embedded] Fix extraction for the new 'edge' player (#10363)
 
 
diff --git a/youtube_dl/extractor/spankbang.py b/youtube_dl/extractor/spankbang.py
index 50433d0f6..186d22b7d 100644
--- a/youtube_dl/extractor/spankbang.py
+++ b/youtube_dl/extractor/spankbang.py
@@ -14,7 +14,7 @@ class SpankBangIE(InfoExtractor):
             'id': '3vvn',
             'ext': 'mp4',
             'title': 'fantasy solo',
-            'description': 'dillion harper masturbates on a bed',
+            'description': 'Watch fantasy solo free HD porn video - 05 minutes - dillion harper masturbates on a bed free adult movies.',
             'thumbnail': 're:^https?://.*\.jpg$',
             'uploader': 'silly2587',
             'age_limit': 18,
@@ -44,12 +44,10 @@ class SpankBangIE(InfoExtractor):
 
         title = self._html_search_regex(
             r'(?s)<h1[^>]*>(.+?)</h1>', webpage, 'title')
-        description = self._search_regex(
-            r'class="desc"[^>]*>([^<]+)',
-            webpage, 'description', default=None)
+        description = self._og_search_description(webpage)
         thumbnail = self._og_search_thumbnail(webpage)
         uploader = self._search_regex(
-            r'class="user"[^>]*>([^<]+)',
+            r'class="user"[^>]*><img[^>]+>([^<]+)',
             webpage, 'uploader', fatal=False)
 
         age_limit = self._rta_search(webpage)

From b54a2da4333556baa3b34fc595060223181320d1 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 25 Aug 2016 22:22:31 +0800
Subject: [PATCH 5897/6219] [crackle] Fix extraction and update _TESTS (closes
 #10333)

---
 ChangeLog                       |  1 +
 youtube_dl/extractor/crackle.py | 58 ++++++++++++++++++++++++---------
 2 files changed, 43 insertions(+), 16 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 5fb596e33..0789549c0 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
 version <unreleased>
 
 Extractors
+* [crackle] Fix extraction (#10333)
 * [spankbang] Fix description and uploader (#10339)
 * [mtvservices:embedded] Fix extraction for the new 'edge' player (#10363)
 
diff --git a/youtube_dl/extractor/crackle.py b/youtube_dl/extractor/crackle.py
index 79238cce7..21f94d33c 100644
--- a/youtube_dl/extractor/crackle.py
+++ b/youtube_dl/extractor/crackle.py
@@ -1,5 +1,7 @@
 # coding: utf-8
-from __future__ import unicode_literals
+from __future__ import unicode_literals, division
+
+import re
 
 from .common import InfoExtractor
 from ..utils import int_or_none
@@ -8,12 +10,22 @@ from ..utils import int_or_none
 class CrackleIE(InfoExtractor):
     _VALID_URL = r'(?:crackle:|https?://(?:www\.)?crackle\.com/(?:playlist/\d+/|(?:[^/]+/)+))(?P<id>\d+)'
     _TEST = {
-        'url': 'http://www.crackle.com/the-art-of-more/2496419',
+        'url': 'http://www.crackle.com/comedians-in-cars-getting-coffee/2498934',
         'info_dict': {
-            'id': '2496419',
+            'id': '2498934',
             'ext': 'mp4',
-            'title': 'Heavy Lies the Head',
-            'description': 'md5:bb56aa0708fe7b9a4861535f15c3abca',
+            'title': 'Everybody Respects A Bloody Nose',
+            'description': 'Jerry is kaffeeklatsching in L.A. with funnyman J.B. Smoove (Saturday Night Live, Real Husbands of Hollywood). They’re headed for brew at 10 Speed Coffee in a 1964 Studebaker Avanti.',
+            'thumbnail': 're:^https?://.*\.jpg',
+            'duration': 906,
+            'series': 'Comedians In Cars Getting Coffee',
+            'season_number': 8,
+            'episode_number': 4,
+            'subtitles': {
+                'en-US': [{
+                    'ext': 'ttml',
+                }]
+            },
         },
         'params': {
             # m3u8 download
@@ -21,11 +33,6 @@ class CrackleIE(InfoExtractor):
         }
     }
 
-    # extracted from http://legacyweb-us.crackle.com/flash/QueryReferrer.ashx
-    _SUBTITLE_SERVER = 'http://web-us-az.crackle.com'
-    _UPLYNK_OWNER_ID = 'e8773f7770a44dbd886eee4fca16a66b'
-    _THUMBNAIL_TEMPLATE = 'http://images-us-am.crackle.com/%stnl_1920x1080.jpg?ts=20140107233116?c=635333335057637614'
-
     # extracted from http://legacyweb-us.crackle.com/flash/ReferrerRedirect.ashx
     _MEDIA_FILE_SLOTS = {
         'c544.flv': {
@@ -48,19 +55,22 @@ class CrackleIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
+
+        config_doc = self._download_xml(
+            'http://legacyweb-us.crackle.com/flash/QueryReferrer.ashx?site=16',
+            video_id, 'Downloading config')
+
         item = self._download_xml(
             'http://legacyweb-us.crackle.com/app/revamp/vidwallcache.aspx?flags=-1&fm=%s' % video_id,
             video_id).find('i')
         title = item.attrib['t']
 
-        thumbnail = None
         subtitles = {}
         formats = self._extract_m3u8_formats(
-            'http://content.uplynk.com/ext/%s/%s.m3u8' % (self._UPLYNK_OWNER_ID, video_id),
+            'http://content.uplynk.com/ext/%s/%s.m3u8' % (config_doc.attrib['strUplynkOwnerId'], video_id),
             video_id, 'mp4', m3u8_id='hls', fatal=None)
         path = item.attrib.get('p')
         if path:
-            thumbnail = self._THUMBNAIL_TEMPLATE % path
             http_base_url = 'http://ahttp.crackle.com/' + path
             for mfs_path, mfs_info in self._MEDIA_FILE_SLOTS.items():
                 formats.append({
@@ -76,20 +86,36 @@ class CrackleIE(InfoExtractor):
                     if locale not in subtitles:
                         subtitles[locale] = []
                     subtitles[locale] = [{
-                        'url': '%s/%s%s_%s.xml' % (self._SUBTITLE_SERVER, path, locale, v),
+                        'url': '%s/%s%s_%s.xml' % (config_doc.attrib['strSubtitleServer'], path, locale, v),
                         'ext': 'ttml',
                     }]
         self._sort_formats(formats, ('width', 'height', 'tbr', 'format_id'))
 
+        media_details = self._download_json(
+            'https://web-api-us.crackle.com/Service.svc/details/media/%s/TW?format=json' % video_id,
+            video_id, fatal=False)
+        thumbnails = []
+        if media_details:
+            for key, value in media_details.items():
+                mobj = re.match('^Thumbnail_(\d+)x(\d+)$', key)
+                if mobj:
+                    width, height = list(map(int, mobj.groups()))
+                    thumbnails.append({
+                        'id': '%dp' % height,
+                        'url': value,
+                        'width': width,
+                        'height': height,
+                    })
+
         return {
             'id': video_id,
             'title': title,
             'description': item.attrib.get('d'),
-            'duration': int(item.attrib.get('r'), 16) if item.attrib.get('r') else None,
+            'duration': int(item.attrib.get('r'), 16) / 1000 if item.attrib.get('r') else None,
             'series': item.attrib.get('sn'),
             'season_number': int_or_none(item.attrib.get('se')),
             'episode_number': int_or_none(item.attrib.get('ep')),
-            'thumbnail': thumbnail,
+            'thumbnails': thumbnails,
             'subtitles': subtitles,
             'formats': formats,
         }

From 20bad91d765284e06f8a8c600a122857d23efeea Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 25 Aug 2016 22:38:06 +0800
Subject: [PATCH 5898/6219] [downloader/external] Clarify that ffmpeg doesn't
 support SOCKS

Ref: #10304
---
 youtube_dl/downloader/external.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index cf4556221..17f12e970 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -220,6 +220,11 @@ class FFmpegFD(ExternalFD):
         if proxy:
             if not re.match(r'^[\da-zA-Z]+://', proxy):
                 proxy = 'http://%s' % proxy
+
+            if proxy.startswith('socks'):
+                self.report_warning(
+                    '%s does not support SOCKS proxies. Downloading may fail.' % self.get_basename())
+
             # Since December 2015 ffmpeg supports -http_proxy option (see
             # http://git.videolan.org/?p=ffmpeg.git;a=commit;h=b4eb1f29ebddd60c41a2eb39f5af701e38e0d3fd)
             # We could switch to the following code if we are able to detect version properly

From a0f071a50dc611a66a5fc8ceceb0b455a88f1cb0 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 25 Aug 2016 19:40:56 +0100
Subject: [PATCH 5899/6219] [usanetwork] Add new extractor

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/usanetwork.py | 76 ++++++++++++++++++++++++++++++
 2 files changed, 77 insertions(+)
 create mode 100644 youtube_dl/extractor/usanetwork.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index a58145e3e..74d916e64 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -947,6 +947,7 @@ from .uplynk import (
 )
 from .urort import UrortIE
 from .urplay import URPlayIE
+from .usanetwork import USANetworkIE
 from .usatoday import USATodayIE
 from .ustream import UstreamIE, UstreamChannelIE
 from .ustudio import (
diff --git a/youtube_dl/extractor/usanetwork.py b/youtube_dl/extractor/usanetwork.py
new file mode 100644
index 000000000..823340776
--- /dev/null
+++ b/youtube_dl/extractor/usanetwork.py
@@ -0,0 +1,76 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .adobepass import AdobePassIE
+from ..utils import (
+    extract_attributes,
+    smuggle_url,
+    update_url_query,
+)
+
+
+class USANetworkIE(AdobePassIE):
+    _VALID_URL = r'https?://(?:www\.)?usanetwork\.com/(?:[^/]+/videos|movies)/(?P<id>[^/?#]+)'
+    _TEST = {
+        'url': 'http://www.usanetwork.com/mrrobot/videos/hpe-cybersecurity',
+        'md5': '33c0d2ba381571b414024440d08d57fd',
+        'info_dict': {
+            'id': '3086229',
+            'ext': 'mp4',
+            'title': 'HPE Cybersecurity',
+            'description': 'The more we digitize our world, the more vulnerable we are.',
+            'upload_date': '20160818',
+            'timestamp': 1471535460,
+            'uploader': 'NBCU-USA',
+        },
+    }
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+
+        player_params = extract_attributes(self._search_regex(
+            r'(<div[^>]+data-usa-tve-player-container[^>]*>)', webpage, 'player params'))
+        video_id = player_params['data-mpx-guid']
+        title = player_params['data-episode-title']
+
+        account_pid, path = re.search(
+            r'data-src="(?:https?)?//player\.theplatform\.com/p/([^/]+)/.*?/(media/guid/\d+/\d+)',
+            webpage).groups()
+
+        query = {
+            'mbr': 'true',
+        }
+        if player_params.get('data-is-full-episode') == '1':
+            query['manifest'] = 'm3u'
+
+        if player_params.get('data-entitlement') == 'auth':
+            adobe_pass = {}
+            drupal_settings = self._search_regex(
+                r'jQuery\.extend\(Drupal\.settings\s*,\s*({.+?})\);',
+                webpage, 'drupal settings', fatal=False)
+            if drupal_settings:
+                drupal_settings = self._parse_json(drupal_settings, video_id, fatal=False)
+                if drupal_settings:
+                    adobe_pass = drupal_settings.get('adobePass', {})
+            resource = self._get_mvpd_resource(
+                adobe_pass.get('adobePassResourceId', 'usa'),
+                title, video_id, player_params.get('data-episode-rating', 'TV-14'))
+            query['auth'] = self._extract_mvpd_auth(
+                url, video_id, adobe_pass.get('adobePassRequestorId', 'usa'), resource)
+
+        info = self._search_json_ld(webpage, video_id, default={})
+        info.update({
+            '_type': 'url_transparent',
+            'url': smuggle_url(update_url_query(
+                'http://link.theplatform.com/s/%s/%s' % (account_pid, path),
+                query), {'force_smil_url': True}),
+            'id': video_id,
+            'title': title,
+            'series': player_params.get('data-show-title'),
+            'episode': title,
+            'ie_key': 'ThePlatform',
+        })
+        return info

From e3faecde30d85f54c1a341350cba609d3f5b6691 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 26 Aug 2016 03:43:13 +0700
Subject: [PATCH 5900/6219] [trutube] Remove extractor (Closes #10438)

---
 youtube_dl/extractor/extractors.py |  1 -
 youtube_dl/extractor/trutube.py    | 26 --------------------------
 2 files changed, 27 deletions(-)
 delete mode 100644 youtube_dl/extractor/trutube.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 74d916e64..717ba9375 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -873,7 +873,6 @@ from .toypics import ToypicsUserIE, ToypicsIE
 from .traileraddict import TrailerAddictIE
 from .trilulilu import TriluliluIE
 from .trollvids import TrollvidsIE
-from .trutube import TruTubeIE
 from .tube8 import Tube8IE
 from .tubitv import TubiTvIE
 from .tudou import (
diff --git a/youtube_dl/extractor/trutube.py b/youtube_dl/extractor/trutube.py
deleted file mode 100644
index d55e0c563..000000000
--- a/youtube_dl/extractor/trutube.py
+++ /dev/null
@@ -1,26 +0,0 @@
-from __future__ import unicode_literals
-
-from .nuevo import NuevoBaseIE
-
-
-class TruTubeIE(NuevoBaseIE):
-    _VALID_URL = r'https?://(?:www\.)?trutube\.tv/(?:video/|nuevo/player/embed\.php\?v=)(?P<id>\d+)'
-    _TESTS = [{
-        'url': 'http://trutube.tv/video/14880/Ramses-II-Proven-To-Be-A-Red-Headed-Caucasoid-',
-        'md5': 'c5b6e301b0a2040b074746cbeaa26ca1',
-        'info_dict': {
-            'id': '14880',
-            'ext': 'flv',
-            'title': 'Ramses II - Proven To Be A Red Headed Caucasoid',
-            'thumbnail': 're:^http:.*\.jpg$',
-        }
-    }, {
-        'url': 'https://trutube.tv/nuevo/player/embed.php?v=14880',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        return self._extract_nuevo(
-            'https://trutube.tv/nuevo/player/config.php?v=%s' % video_id,
-            video_id)

From 298a120ab76008c900e30de50dc738dd63e79fb4 Mon Sep 17 00:00:00 2001
From: Aleksander Nitecki <ixendr@itogi.re>
Date: Thu, 25 Aug 2016 20:21:06 +0200
Subject: [PATCH 5901/6219] [nhk] Add extractor for VoD.

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/nhk.py        | 29 +++++++++++++++++++++++++++++
 2 files changed, 30 insertions(+)
 create mode 100644 youtube_dl/extractor/nhk.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 717ba9375..8d88d6cb4 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -541,6 +541,7 @@ from .nextmedia import (
 )
 from .nfb import NFBIE
 from .nfl import NFLIE
+from .nhk import NhkVodIE
 from .nhl import (
     NHLVideocenterIE,
     NHLNewsIE,
diff --git a/youtube_dl/extractor/nhk.py b/youtube_dl/extractor/nhk.py
new file mode 100644
index 000000000..90e935351
--- /dev/null
+++ b/youtube_dl/extractor/nhk.py
@@ -0,0 +1,29 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class NhkVodIE(InfoExtractor):
+    _VALID_URL = r'http://www3\.nhk\.or\.jp/nhkworld/en/vod/(?P<id>.+)\.html'
+    _TESTS = [{
+        'url': 'http://www3.nhk.or.jp/nhkworld/en/vod/tokyofashion/20160815.html',
+        'info_dict': {
+            'id': 'A1bnNiNTE6nY3jLllS-BIISfcC_PpvF5',
+            'ext': 'flv',
+            'title': '[nhkworld]VOD;2009-251-2016;TOKYO FASHION EXPRESS;The Kimono as Global Fashion;en',
+        },
+        'params': {
+            'skip_download': True  # Videos available only for a limited period of time.
+        },
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        embed_code = self._search_regex(
+            r'''nw_vod_ooplayer\('movie-area', '([^']+)'\);''',
+            webpage,
+            'ooyala embed code')
+
+        return self.url_result('ooyala:' + embed_code, 'Ooyala')

From f9b373afda2a936c4f8303671f3160c532ccae67 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 26 Aug 2016 04:48:40 +0700
Subject: [PATCH 5902/6219] [nhk:vod] Improve extraction (Closes #10424)

---
 youtube_dl/extractor/nhk.py | 43 +++++++++++++++++++++++++++----------
 1 file changed, 32 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/nhk.py b/youtube_dl/extractor/nhk.py
index 90e935351..691bdfa4e 100644
--- a/youtube_dl/extractor/nhk.py
+++ b/youtube_dl/extractor/nhk.py
@@ -4,26 +4,47 @@ from .common import InfoExtractor
 
 
 class NhkVodIE(InfoExtractor):
-    _VALID_URL = r'http://www3\.nhk\.or\.jp/nhkworld/en/vod/(?P<id>.+)\.html'
-    _TESTS = [{
+    _VALID_URL = r'https?://www3\.nhk\.or\.jp/nhkworld/en/vod/(?P<id>.+?)\.html'
+    _TEST = {
+        # Videos available only for a limited period of time. Visit
+        # http://www3.nhk.or.jp/nhkworld/en/vod/ for working samples.
         'url': 'http://www3.nhk.or.jp/nhkworld/en/vod/tokyofashion/20160815.html',
         'info_dict': {
             'id': 'A1bnNiNTE6nY3jLllS-BIISfcC_PpvF5',
             'ext': 'flv',
-            'title': '[nhkworld]VOD;2009-251-2016;TOKYO FASHION EXPRESS;The Kimono as Global Fashion;en',
+            'title': 'TOKYO FASHION EXPRESS - The Kimono as Global Fashion',
+            'description': 'md5:db338ee6ce8204f415b754782f819824',
+            'series': 'TOKYO FASHION EXPRESS',
+            'episode': 'The Kimono as Global Fashion',
         },
-        'params': {
-            'skip_download': True  # Videos available only for a limited period of time.
-        },
-    }]
+        'skip': 'Videos available only for a limited period of time',
+    }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
+
         webpage = self._download_webpage(url, video_id)
 
         embed_code = self._search_regex(
-            r'''nw_vod_ooplayer\('movie-area', '([^']+)'\);''',
-            webpage,
-            'ooyala embed code')
+            r'nw_vod_ooplayer\([^,]+,\s*(["\'])(?P<id>(?:(?!\1).)+)\1',
+            webpage, 'ooyala embed code', group='id')
 
-        return self.url_result('ooyala:' + embed_code, 'Ooyala')
+        title = self._search_regex(
+            r'<div[^>]+class=["\']episode-detail["\']>\s*<h\d+>([^<]+)',
+            webpage, 'title', default=None)
+        description = self._html_search_regex(
+            r'(?s)<p[^>]+class=["\']description["\'][^>]*>(.+?)</p>',
+            webpage, 'description', default=None)
+        series = self._search_regex(
+            r'<h2[^>]+class=["\']detail-top-player-title[^>]+><a[^>]+>([^<]+)',
+            webpage, 'series', default=None)
+
+        return {
+            '_type': 'url_transparent',
+            'ie_key': 'Ooyala',
+            'url': 'ooyala:%s' % embed_code,
+            'title': '%s - %s' % (series, title) if series and title else title,
+            'description': description,
+            'series': series,
+            'episode': title,
+        }

From c9de980106990485fd9bff9a86d463349fe1d384 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 26 Aug 2016 04:49:52 +0700
Subject: [PATCH 5903/6219] Credit @Xender for nhk:vod (#10424)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 1fd4be785..b9a602c12 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -181,3 +181,4 @@ Nehal Patel
 Rob van Bekkum
 Petr Zvoníček
 Pratyush Singh
+Aleksander Nitecki

From 6b18a24e6ee39ab2fdb5e3d9e1cf2eec547ca3f9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 26 Aug 2016 05:57:52 +0700
Subject: [PATCH 5904/6219] [tnaflix] Fix extraction (Closes #10434)

---
 youtube_dl/extractor/tnaflix.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/tnaflix.py b/youtube_dl/extractor/tnaflix.py
index 7ddf77767..77d56b8ca 100644
--- a/youtube_dl/extractor/tnaflix.py
+++ b/youtube_dl/extractor/tnaflix.py
@@ -10,6 +10,7 @@ from ..utils import (
     int_or_none,
     parse_duration,
     str_to_int,
+    unescapeHTML,
     xpath_text,
 )
 
@@ -80,7 +81,8 @@ class TNAFlixNetworkBaseIE(InfoExtractor):
 
         if not cfg_url:
             inputs = self._hidden_inputs(webpage)
-            cfg_url = 'https://cdn-fck.tnaflix.com/tnaflix/%s.fid?key=%s' % (inputs['vkey'], inputs['nkey'])
+            cfg_url = ('https://cdn-fck.tnaflix.com/tnaflix/%s.fid?key=%s&VID=%s&premium=1&vip=1&alpha'
+                       % (inputs['vkey'], inputs['nkey'], video_id))
 
         cfg_xml = self._download_xml(
             cfg_url, display_id, 'Downloading metadata',
@@ -89,7 +91,7 @@ class TNAFlixNetworkBaseIE(InfoExtractor):
         formats = []
 
         def extract_video_url(vl):
-            return re.sub('speed=\d+', 'speed=', vl.text)
+            return re.sub('speed=\d+', 'speed=', unescapeHTML(vl.text))
 
         video_link = cfg_xml.find('./videoLink')
         if video_link is not None:
@@ -201,7 +203,7 @@ class TNAFlixIE(TNAFlixNetworkBaseIE):
     _TESTS = [{
         # anonymous uploader, no categories
         'url': 'http://www.tnaflix.com/porn-stars/Carmella-Decesare-striptease/video553878',
-        'md5': '7e569419fe6d69543d01e6be22f5f7c4',
+        'md5': 'ecf3498417d09216374fc5907f9c6ec0',
         'info_dict': {
             'id': '553878',
             'display_id': 'Carmella-Decesare-striptease',
@@ -215,11 +217,11 @@ class TNAFlixIE(TNAFlixNetworkBaseIE):
     }, {
         # non-anonymous uploader, categories
         'url': 'https://www.tnaflix.com/teen-porn/Educational-xxx-video/video6538',
-        'md5': 'fcba2636572895aba116171a899a5658',
+        'md5': '0f5d4d490dbfd117b8607054248a07c0',
         'info_dict': {
             'id': '6538',
             'display_id': 'Educational-xxx-video',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Educational xxx video',
             'description': 'md5:b4fab8f88a8621c8fabd361a173fe5b8',
             'thumbnail': 're:https?://.*\.jpg$',

From b281aad2dc658e3c6535579d75b42a5634487b83 Mon Sep 17 00:00:00 2001
From: steven7851 <steven7851@msn.com>
Date: Fri, 26 Aug 2016 07:32:54 +0800
Subject: [PATCH 5905/6219] [douyutv] Use new api

use lapi for flv info, and html5 api for room info
#10153 #10318
---
 youtube_dl/extractor/douyutv.py | 87 ++++++++++++++++-----------------
 1 file changed, 43 insertions(+), 44 deletions(-)

diff --git a/youtube_dl/extractor/douyutv.py b/youtube_dl/extractor/douyutv.py
index ce6962755..33efc993e 100644
--- a/youtube_dl/extractor/douyutv.py
+++ b/youtube_dl/extractor/douyutv.py
@@ -3,9 +3,10 @@ from __future__ import unicode_literals
 
 import hashlib
 import time
+import uuid
 from .common import InfoExtractor
 from ..utils import (ExtractorError, unescapeHTML)
-from ..compat import (compat_str, compat_basestring)
+from ..compat import (compat_str, compat_basestring, compat_urllib_parse_urlencode)
 
 
 class DouyuTVIE(InfoExtractor):
@@ -21,7 +22,6 @@ class DouyuTVIE(InfoExtractor):
             'description': 're:.*m7show@163\.com.*',
             'thumbnail': 're:^https?://.*\.jpg$',
             'uploader': '7师傅',
-            'uploader_id': '431925',
             'is_live': True,
         },
         'params': {
@@ -37,7 +37,6 @@ class DouyuTVIE(InfoExtractor):
             'description': 'md5:746a2f7a253966a06755a912f0acc0d2',
             'thumbnail': 're:^https?://.*\.jpg$',
             'uploader': 'douyu小漠',
-            'uploader_id': '3769985',
             'is_live': True,
         },
         'params': {
@@ -54,7 +53,6 @@ class DouyuTVIE(InfoExtractor):
             'description': 're:.*m7show@163\.com.*',
             'thumbnail': 're:^https?://.*\.jpg$',
             'uploader': '7师傅',
-            'uploader_id': '431925',
             'is_live': True,
         },
         'params': {
@@ -75,19 +73,39 @@ class DouyuTVIE(InfoExtractor):
             room_id = self._html_search_regex(
                 r'"room_id"\s*:\s*(\d+),', page, 'room id')
 
-        config = None
+        room_url = 'http://m.douyu.com/html5/live?roomId=%s' % room_id
+        room_content = self._download_webpage(room_url, video_id)
+        room_json = self._parse_json(room_content, video_id, fatal=False)
+
+        room = room_json['data']
+
+        show_status = room.get('show_status')
+        # 1 = live, 2 = offline
+        if show_status == '2':
+            raise ExtractorError(
+                'Live stream is offline', expected=True)
+
+        flv_json = None
         # Douyu API sometimes returns error "Unable to load the requested class: eticket_redis_cache"
         # Retry with different parameters - same parameters cause same errors
         for i in range(5):
-            prefix = 'room/%s?aid=android&client_sys=android&time=%d' % (
-                room_id, int(time.time()))
-            auth = hashlib.md5((prefix + '1231').encode('ascii')).hexdigest()
+            tt = int(time.time() / 60)
+            did = uuid.uuid4().hex.upper()
 
-            config_page = self._download_webpage(
-                'http://www.douyutv.com/api/v1/%s&auth=%s' % (prefix, auth),
-                video_id)
+            # Decompile core.swf in webpage by ffdec "Search SWFs in memory"
+            # core.swf is encrypted originally, but ffdec can dump memory to get the decrypted one
+            # If API changes in the future, just use this way to update
+            sign_content = '{room_id}{did}A12Svb&%1UUmf@hC{tt}'.format(room_id = room_id, did = did, tt = tt)
+            sign = hashlib.md5((sign_content).encode('utf-8')).hexdigest()
+
+            payload = {'cdn': 'ws', 'rate': '0', 'tt': tt, 'did': did, 'sign': sign}
+            flv_data = compat_urllib_parse_urlencode(payload)
+
+            flv_request_url = 'http://www.douyu.com/lapi/live/getPlay/%s' % room_id
+            flv_content = self._download_webpage(flv_request_url, video_id, data=flv_data,
+                headers={'Content-Type': 'application/x-www-form-urlencoded'})
             try:
-                config = self._parse_json(config_page, video_id, fatal=False)
+                flv_json = self._parse_json(flv_content, video_id, fatal=False)
             except ExtractorError:
                 # Wait some time before retrying to get a different time() value
                 self._sleep(1, video_id, msg_template='%(video_id)s: Error occurs. '
@@ -95,54 +113,35 @@ class DouyuTVIE(InfoExtractor):
                 continue
             else:
                 break
-        if config is None:
+        if flv_json is None:
             raise ExtractorError('Unable to fetch API result')
 
-        data = config['data']
+        flv = flv_json['data']
 
-        error_code = config.get('error', 0)
+        error_code = flv_json.get('error', 0)
         if error_code is not 0:
             error_desc = 'Server reported error %i' % error_code
-            if isinstance(data, (compat_str, compat_basestring)):
-                error_desc += ': ' + data
+            if isinstance(flv, (compat_str, compat_basestring)):
+                error_desc += ': ' + flv
             raise ExtractorError(error_desc, expected=True)
 
-        show_status = data.get('show_status')
-        # 1 = live, 2 = offline
-        if show_status == '2':
-            raise ExtractorError(
-                'Live stream is offline', expected=True)
+        base_url = flv['rtmp_url']
+        live_path = flv['rtmp_live']
 
-        base_url = data['rtmp_url']
-        live_path = data['rtmp_live']
+        video_url = '%s/%s' % (base_url, live_path)
 
-        title = self._live_title(unescapeHTML(data['room_name']))
-        description = data.get('show_details')
-        thumbnail = data.get('room_src')
-
-        uploader = data.get('nickname')
-        uploader_id = data.get('owner_uid')
-
-        multi_formats = data.get('rtmp_multi_bitrate')
-        if not isinstance(multi_formats, dict):
-            multi_formats = {}
-        multi_formats['live'] = live_path
-
-        formats = [{
-            'url': '%s/%s' % (base_url, format_path),
-            'format_id': format_id,
-            'preference': 1 if format_id == 'live' else 0,
-        } for format_id, format_path in multi_formats.items()]
-        self._sort_formats(formats)
+        title = self._live_title(unescapeHTML(room['room_name']))
+        description = room.get('notice')
+        thumbnail = room.get('room_src')
+        uploader = room.get('nickname')
 
         return {
             'id': room_id,
             'display_id': video_id,
+            'url': video_url,
             'title': title,
             'description': description,
             'thumbnail': thumbnail,
             'uploader': uploader,
-            'uploader_id': uploader_id,
-            'formats': formats,
             'is_live': True,
         }

From 906b87cf5f6ccf28ebd75d6a92367d7c238f2ad9 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 26 Aug 2016 19:58:17 +0800
Subject: [PATCH 5906/6219] [crackle] Revert to template-based thumbnail
 extraction

To reduce to number of HTTP requests
---
 youtube_dl/extractor/crackle.py | 23 ++++-------------------
 1 file changed, 4 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/extractor/crackle.py b/youtube_dl/extractor/crackle.py
index 21f94d33c..cc68f1c00 100644
--- a/youtube_dl/extractor/crackle.py
+++ b/youtube_dl/extractor/crackle.py
@@ -1,8 +1,6 @@
 # coding: utf-8
 from __future__ import unicode_literals, division
 
-import re
-
 from .common import InfoExtractor
 from ..utils import int_or_none
 
@@ -34,6 +32,7 @@ class CrackleIE(InfoExtractor):
     }
 
     # extracted from http://legacyweb-us.crackle.com/flash/ReferrerRedirect.ashx
+    _THUMBNAIL_TEMPLATE = 'http://images-us-am.crackle.com/%stnl_1920x1080.jpg?ts=20140107233116?c=635333335057637614'
     _MEDIA_FILE_SLOTS = {
         'c544.flv': {
             'width': 544,
@@ -69,8 +68,10 @@ class CrackleIE(InfoExtractor):
         formats = self._extract_m3u8_formats(
             'http://content.uplynk.com/ext/%s/%s.m3u8' % (config_doc.attrib['strUplynkOwnerId'], video_id),
             video_id, 'mp4', m3u8_id='hls', fatal=None)
+        thumbnail = None
         path = item.attrib.get('p')
         if path:
+            thumbnail = self._THUMBNAIL_TEMPLATE % path
             http_base_url = 'http://ahttp.crackle.com/' + path
             for mfs_path, mfs_info in self._MEDIA_FILE_SLOTS.items():
                 formats.append({
@@ -91,22 +92,6 @@ class CrackleIE(InfoExtractor):
                     }]
         self._sort_formats(formats, ('width', 'height', 'tbr', 'format_id'))
 
-        media_details = self._download_json(
-            'https://web-api-us.crackle.com/Service.svc/details/media/%s/TW?format=json' % video_id,
-            video_id, fatal=False)
-        thumbnails = []
-        if media_details:
-            for key, value in media_details.items():
-                mobj = re.match('^Thumbnail_(\d+)x(\d+)$', key)
-                if mobj:
-                    width, height = list(map(int, mobj.groups()))
-                    thumbnails.append({
-                        'id': '%dp' % height,
-                        'url': value,
-                        'width': width,
-                        'height': height,
-                    })
-
         return {
             'id': video_id,
             'title': title,
@@ -115,7 +100,7 @@ class CrackleIE(InfoExtractor):
             'series': item.attrib.get('sn'),
             'season_number': int_or_none(item.attrib.get('se')),
             'episode_number': int_or_none(item.attrib.get('ep')),
-            'thumbnails': thumbnails,
+            'thumbnail': thumbnail,
             'subtitles': subtitles,
             'formats': formats,
         }

From 3b4b82d4cec702fc06e2d6b38a44dd0c7bd77a5b Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 27 Aug 2016 01:16:39 +0800
Subject: [PATCH 5907/6219] [douyutv] Simplify

---
 youtube_dl/extractor/douyutv.py | 86 +++++++++++++++------------------
 1 file changed, 39 insertions(+), 47 deletions(-)

diff --git a/youtube_dl/extractor/douyutv.py b/youtube_dl/extractor/douyutv.py
index 33efc993e..e366e17e6 100644
--- a/youtube_dl/extractor/douyutv.py
+++ b/youtube_dl/extractor/douyutv.py
@@ -4,9 +4,16 @@ from __future__ import unicode_literals
 import hashlib
 import time
 import uuid
+
 from .common import InfoExtractor
-from ..utils import (ExtractorError, unescapeHTML)
-from ..compat import (compat_str, compat_basestring, compat_urllib_parse_urlencode)
+from ..compat import (
+    compat_str,
+    compat_urllib_parse_urlencode,
+)
+from ..utils import (
+    ExtractorError,
+    unescapeHTML,
+)
 
 
 class DouyuTVIE(InfoExtractor):
@@ -63,6 +70,10 @@ class DouyuTVIE(InfoExtractor):
         'only_matching': True,
     }]
 
+    # Decompile core.swf in webpage by ffdec "Search SWFs in memory". core.swf
+    # is encrypted originally, but ffdec can dump memory to get the decrypted one.
+    _API_KEY = 'A12Svb&%1UUmf@hC'
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
@@ -73,60 +84,41 @@ class DouyuTVIE(InfoExtractor):
             room_id = self._html_search_regex(
                 r'"room_id"\s*:\s*(\d+),', page, 'room id')
 
-        room_url = 'http://m.douyu.com/html5/live?roomId=%s' % room_id
-        room_content = self._download_webpage(room_url, video_id)
-        room_json = self._parse_json(room_content, video_id, fatal=False)
+        room = self._download_json(
+            'http://m.douyu.com/html5/live?roomId=%s' % room_id, video_id,
+            note='Downloading room info')['data']
 
-        room = room_json['data']
-
-        show_status = room.get('show_status')
         # 1 = live, 2 = offline
-        if show_status == '2':
-            raise ExtractorError(
-                'Live stream is offline', expected=True)
+        if room.get('show_status') == '2':
+            raise ExtractorError('Live stream is offline', expected=True)
 
-        flv_json = None
-        # Douyu API sometimes returns error "Unable to load the requested class: eticket_redis_cache"
-        # Retry with different parameters - same parameters cause same errors
-        for i in range(5):
-            tt = int(time.time() / 60)
-            did = uuid.uuid4().hex.upper()
+        tt = compat_str(int(time.time() / 60))
+        did = uuid.uuid4().hex.upper()
 
-            # Decompile core.swf in webpage by ffdec "Search SWFs in memory"
-            # core.swf is encrypted originally, but ffdec can dump memory to get the decrypted one
-            # If API changes in the future, just use this way to update
-            sign_content = '{room_id}{did}A12Svb&%1UUmf@hC{tt}'.format(room_id = room_id, did = did, tt = tt)
-            sign = hashlib.md5((sign_content).encode('utf-8')).hexdigest()
+        sign_content = ''.join((room_id, did, self._API_KEY, tt))
+        sign = hashlib.md5((sign_content).encode('utf-8')).hexdigest()
 
-            payload = {'cdn': 'ws', 'rate': '0', 'tt': tt, 'did': did, 'sign': sign}
-            flv_data = compat_urllib_parse_urlencode(payload)
+        flv_data = compat_urllib_parse_urlencode({
+            'cdn': 'ws',
+            'rate': '0',
+            'tt': tt,
+            'did': did,
+            'sign': sign,
+        })
 
-            flv_request_url = 'http://www.douyu.com/lapi/live/getPlay/%s' % room_id
-            flv_content = self._download_webpage(flv_request_url, video_id, data=flv_data,
-                headers={'Content-Type': 'application/x-www-form-urlencoded'})
-            try:
-                flv_json = self._parse_json(flv_content, video_id, fatal=False)
-            except ExtractorError:
-                # Wait some time before retrying to get a different time() value
-                self._sleep(1, video_id, msg_template='%(video_id)s: Error occurs. '
-                                                      'Waiting for %(timeout)s seconds before retrying')
-                continue
-            else:
-                break
-        if flv_json is None:
-            raise ExtractorError('Unable to fetch API result')
+        video_info = self._download_json(
+            'http://www.douyu.com/lapi/live/getPlay/%s' % room_id, video_id,
+            data=flv_data, note='Downloading video info',
+            headers={'Content-Type': 'application/x-www-form-urlencoded'})
 
-        flv = flv_json['data']
-
-        error_code = flv_json.get('error', 0)
+        error_code = video_info.get('error', 0)
         if error_code is not 0:
-            error_desc = 'Server reported error %i' % error_code
-            if isinstance(flv, (compat_str, compat_basestring)):
-                error_desc += ': ' + flv
-            raise ExtractorError(error_desc, expected=True)
+            raise ExtractorError(
+                '%s reported error %i' % (self.IE_NAME, error_code),
+                expected=True)
 
-        base_url = flv['rtmp_url']
-        live_path = flv['rtmp_live']
+        base_url = video_info['data']['rtmp_url']
+        live_path = video_info['data']['rtmp_live']
 
         video_url = '%s/%s' % (base_url, live_path)
 

From 92c27a0dbf19eff211e7ffdd8db5895387e75529 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 28 Aug 2016 02:35:49 +0700
Subject: [PATCH 5908/6219] [periscope:user] Fix extraction (Closes #10453)

---
 youtube_dl/extractor/periscope.py | 47 ++++++++++++++++++++-----------
 1 file changed, 30 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/periscope.py b/youtube_dl/extractor/periscope.py
index 75f5884a9..6c640089d 100644
--- a/youtube_dl/extractor/periscope.py
+++ b/youtube_dl/extractor/periscope.py
@@ -8,7 +8,14 @@ from ..utils import (
 )
 
 
-class PeriscopeIE(InfoExtractor):
+class PeriscopeBaseIE(InfoExtractor):
+    def _call_api(self, method, query, item_id):
+        return self._download_json(
+            'https://api.periscope.tv/api/v2/%s' % method,
+            item_id, query=query)
+
+
+class PeriscopeIE(PeriscopeBaseIE):
     IE_DESC = 'Periscope'
     IE_NAME = 'periscope'
     _VALID_URL = r'https?://(?:www\.)?periscope\.tv/[^/]+/(?P<id>[^/?#]+)'
@@ -34,14 +41,11 @@ class PeriscopeIE(InfoExtractor):
         'only_matching': True,
     }]
 
-    def _call_api(self, method, value):
-        return self._download_json(
-            'https://api.periscope.tv/api/v2/%s?broadcast_id=%s' % (method, value), value)
-
     def _real_extract(self, url):
         token = self._match_id(url)
 
-        broadcast_data = self._call_api('getBroadcastPublic', token)
+        broadcast_data = self._call_api(
+            'getBroadcastPublic', {'broadcast_id': token}, token)
         broadcast = broadcast_data['broadcast']
         status = broadcast['status']
 
@@ -61,7 +65,8 @@ class PeriscopeIE(InfoExtractor):
             'url': broadcast[image],
         } for image in ('image_url', 'image_url_small') if broadcast.get(image)]
 
-        stream = self._call_api('getAccessPublic', token)
+        stream = self._call_api(
+            'getAccessPublic', {'broadcast_id': token}, token)
 
         formats = []
         for format_id in ('replay', 'rtmp', 'hls', 'https_hls'):
@@ -88,7 +93,7 @@ class PeriscopeIE(InfoExtractor):
         }
 
 
-class PeriscopeUserIE(InfoExtractor):
+class PeriscopeUserIE(PeriscopeBaseIE):
     _VALID_URL = r'https?://www\.periscope\.tv/(?P<id>[^/]+)/?$'
     IE_DESC = 'Periscope user videos'
     IE_NAME = 'periscope:user'
@@ -106,26 +111,34 @@ class PeriscopeUserIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        user_id = self._match_id(url)
+        user_name = self._match_id(url)
 
-        webpage = self._download_webpage(url, user_id)
+        webpage = self._download_webpage(url, user_name)
 
         data_store = self._parse_json(
             unescapeHTML(self._search_regex(
                 r'data-store=(["\'])(?P<data>.+?)\1',
                 webpage, 'data store', default='{}', group='data')),
-            user_id)
+            user_name)
 
-        user = data_store.get('User', {}).get('user', {})
-        title = user.get('display_name') or user.get('username')
+        user = list(data_store['UserCache']['users'].values())[0]['user']
+        user_id = user['id']
+        session_id = data_store['SessionToken']['broadcastHistory']['token']['session_id']
+
+        broadcasts = self._call_api(
+            'getUserBroadcastsPublic',
+            {'user_id': user_id, 'session_id': session_id},
+            user_name)['broadcasts']
+
+        broadcast_ids = [
+            broadcast['id'] for broadcast in broadcasts if broadcast.get('id')]
+
+        title = user.get('display_name') or user.get('username') or user_name
         description = user.get('description')
 
-        broadcast_ids = (data_store.get('UserBroadcastHistory', {}).get('broadcastIds') or
-                         data_store.get('BroadcastCache', {}).get('broadcastIds', []))
-
         entries = [
             self.url_result(
-                'https://www.periscope.tv/%s/%s' % (user_id, broadcast_id))
+                'https://www.periscope.tv/%s/%s' % (user_name, broadcast_id))
             for broadcast_id in broadcast_ids]
 
         return self.playlist_result(entries, user_id, title, description)

From d7aae610f6674d96971246f916973158374f88b0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 28 Aug 2016 07:00:15 +0700
Subject: [PATCH 5909/6219] [ChangeLog] Actualize

---
 ChangeLog | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index 0789549c0..4062c2021 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,8 +1,23 @@
 version <unreleased>
 
+Core
++ Add warning message that ffmpeg doesn't support SOCKS
+* Improve thumbnail sorting
++ Extract formats from #EXT-X-MEDIA tags in _extract_m3u8_formats
+* Fill IV with leading zeros for IVs shorter than 16 octets in hlsnative
++ Add ac-3 to the list of audio codecs in parse_codecs
+
 Extractors
+* [periscope:user] Fix extraction (#10453)
+* [douyutv] Fix extraction (#10153, #10318, #10444)
++ [nhk:vod] Add extractor for www3.nhk.or.jp on demand (#4437, #10424)
+- [trutube] Remove extractor (#10438)
++ [usanetwork] Add extractor for usanetwork.com
 * [crackle] Fix extraction (#10333)
-* [spankbang] Fix description and uploader (#10339)
+* [spankbang] Fix description and uploader extraction (#10339)
+* [discoverygo] Detect cable provider restricted videos (#10425)
++ [cbc] Add support for watch.cbc.ca
+* [kickstarter] Silent the warning for og:description (#10415)
 * [mtvservices:embedded] Fix extraction for the new 'edge' player (#10363)
 
 

From 71e90766b5f7d57bdbe20b71c32ce5a8f66aecc2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 28 Aug 2016 07:09:03 +0700
Subject: [PATCH 5910/6219] [README.md] Fix typo in download archive FAQ entry

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 20241307f..87465aa5e 100644
--- a/README.md
+++ b/README.md
@@ -836,7 +836,7 @@ You will first need to tell youtube-dl to stream media to stdout with `-o -`, an
 
 ### How do I download only new videos from a playlist?
 
-Use download-archive feature. With this feature you should initially download the complete playlist with `--download-archive /path/to/download/archive/file.txt` that will record identifiers of all the videos in a special file. Each subsequent run with the same `--download-archive` will download only new videos that and skip all videos that have been downloaded before. Note that only successful downloads are recorded in the file.
+Use download-archive feature. With this feature you should initially download the complete playlist with `--download-archive /path/to/download/archive/file.txt` that will record identifiers of all the videos in a special file. Each subsequent run with the same `--download-archive` will download only new videos and skip all videos that have been downloaded before. Note that only successful downloads are recorded in the file.
 
 For example, at first,
 

From 1198fe14a1eff1047652c51163266246577e3682 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 28 Aug 2016 07:24:08 +0700
Subject: [PATCH 5911/6219] release 2016.08.28

---
 .github/ISSUE_TEMPLATE.md |  6 +++---
 ChangeLog                 |  2 +-
 docs/supportedsites.md    | 11 +++++++----
 youtube_dl/version.py     |  2 +-
 4 files changed, 12 insertions(+), 9 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 15acc025a..a2fe59f80 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.08.24.1*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.08.24.1**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.08.28*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.08.28**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.08.24.1
+[debug] youtube-dl version 2016.08.28
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 4062c2021..d3496b5dc 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2016.08.28
 
 Core
 + Add warning message that ffmpeg doesn't support SOCKS
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 08db56fa9..bf08697be 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -89,7 +89,7 @@
  - **Bet**
  - **Bigflix**
  - **Bild**: Bild.de
- - **BiliBili**
+ - **BiliBili** (Currently broken)
  - **BioBioChileTV**
  - **BIQLE**
  - **BleacherReport**
@@ -115,8 +115,10 @@
  - **Canvas**
  - **CarambaTV**
  - **CarambaTVPage**
- - **CBC**
- - **CBCPlayer**
+ - **cbc.ca**
+ - **cbc.ca:player**
+ - **cbc.ca:watch**
+ - **cbc.ca:watch:video**
  - **CBS**
  - **CBSInteractive**
  - **CBSLocal**
@@ -448,6 +450,7 @@
  - **NextMediaActionNews**: 蘋果日報 - 動新聞
  - **nfb**: National Film Board of Canada
  - **nfl.com**
+ - **NhkVod**
  - **nhl.com**
  - **nhl.com:news**: NHL news
  - **nhl.com:videocenter**
@@ -713,7 +716,6 @@
  - **TrailerAddict** (Currently broken)
  - **Trilulilu**
  - **trollvids**
- - **TruTube**
  - **Tube8**
  - **TubiTv**
  - **tudou**
@@ -758,6 +760,7 @@
  - **uplynk:preplay**
  - **Urort**: NRK P3 Urørt
  - **URPlay**
+ - **USANetwork**
  - **USAToday**
  - **ustream**
  - **ustream:channel**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 7447d3d7e..ee30ca2ad 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.08.24.1'
+__version__ = '2016.08.28'

From 39efc6e3e048a8323c36efcdf6b7434259a35e44 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 28 Aug 2016 15:46:11 +0800
Subject: [PATCH 5912/6219] [generic] Update some _TESTS

---
 youtube_dl/extractor/generic.py | 19 +++++++++++++++----
 1 file changed, 15 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 506892b11..c6e655c84 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -104,7 +104,8 @@ class GenericIE(InfoExtractor):
             },
             'expected_warnings': [
                 'URL could be a direct video link, returning it as such.'
-            ]
+            ],
+            'skip': 'URL invalid',
         },
         # Direct download with broken HEAD
         {
@@ -268,7 +269,8 @@ class GenericIE(InfoExtractor):
             'params': {
                 # m3u8 downloads
                 'skip_download': True,
-            }
+            },
+            'skip': 'video gone',
         },
         # m3u8 served with Content-Type: text/plain
         {
@@ -283,7 +285,8 @@ class GenericIE(InfoExtractor):
             'params': {
                 # m3u8 downloads
                 'skip_download': True,
-            }
+            },
+            'skip': 'video gone',
         },
         # google redirect
         {
@@ -368,6 +371,7 @@ class GenericIE(InfoExtractor):
                 'description': 'Mississauga resident David Farmer is still out of power as a result of the ice storm a month ago. To keep the house warm, Farmer cuts wood from his property for a wood burning stove downstairs.',
             },
             'add_ie': ['BrightcoveLegacy'],
+            'skip': 'video gone',
         },
         {
             'url': 'http://www.championat.com/video/football/v/87/87499.html',
@@ -421,6 +425,7 @@ class GenericIE(InfoExtractor):
             'params': {
                 'skip_download': True,
             },
+            'skip': 'movie expired',
         },
         # embed.ly video
         {
@@ -448,6 +453,8 @@ class GenericIE(InfoExtractor):
                 'title': 'Between Two Ferns with Zach Galifianakis: President Barack Obama',
                 'description': 'Episode 18: President Barack Obama sits down with Zach Galifianakis for his most memorable interview yet.',
             },
+            # HEAD requests lead to endless 301, while GET is OK
+            'expected_warnings': ['301'],
         },
         # RUTV embed
         {
@@ -522,6 +529,9 @@ class GenericIE(InfoExtractor):
                 'title': '[NSFL] [FM15] which pumiscer was this ( vid ) ( alfa as fuck srx )',
             },
             'playlist_mincount': 7,
+            # This forum does not allow <iframe> syntaxes anymore
+            # Now HTML tags are displayed as-is
+            'skip': 'No videos on this page',
         },
         # Embedded TED video
         {
@@ -570,7 +580,8 @@ class GenericIE(InfoExtractor):
             },
             'params': {
                 'skip_download': 'Requires rtmpdump'
-            }
+            },
+            'skip': 'video gone',
         },
         # francetv embed
         {

From 40eec6b15cd3135b24cb42fde5ccf62e9a1f0807 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 28 Aug 2016 20:27:08 +0800
Subject: [PATCH 5913/6219] [openload] Fix extraction (closes #10408)

Thanks to @yokrysty again!
---
 ChangeLog                        | 6 ++++++
 youtube_dl/extractor/openload.py | 4 +++-
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index d3496b5dc..5d7a052a5 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Extractors
+* [openload] Fix extraction (#10408)
+
+
 version 2016.08.28
 
 Core
diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py
index e181d0b3a..c8dde7ae3 100644
--- a/youtube_dl/extractor/openload.py
+++ b/youtube_dl/extractor/openload.py
@@ -55,10 +55,12 @@ class OpenloadIE(InfoExtractor):
 
         video_url_chars = []
 
-        for c in enc_data:
+        for idx, c in enumerate(enc_data):
             j = compat_ord(c)
             if j >= 33 and j <= 126:
                 j = ((j + 14) % 94) + 33
+            if idx == len(enc_data) - 1:
+                j += 2
             video_url_chars += compat_chr(j)
 
         video_url = 'https://openload.co/stream/%s?mime=true' % ''.join(video_url_chars)

From 04b32c8f9679004d11ee97c2b7beecaedf1b477b Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 28 Aug 2016 22:06:31 +0800
Subject: [PATCH 5914/6219] [bilibili] Fix extraction (closes #10375)

Thanks @gdkchan for the algorithm
---
 ChangeLog                        |  1 +
 youtube_dl/extractor/bilibili.py | 98 ++++++++++++--------------------
 2 files changed, 36 insertions(+), 63 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 5d7a052a5..e055976c5 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
 version <unreleased>
 
 Extractors
+* [bilibili] Fix extraction (#10375)
 * [openload] Fix extraction (#10408)
 
 
diff --git a/youtube_dl/extractor/bilibili.py b/youtube_dl/extractor/bilibili.py
index d87c38a02..a332fbb69 100644
--- a/youtube_dl/extractor/bilibili.py
+++ b/youtube_dl/extractor/bilibili.py
@@ -1,35 +1,26 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import calendar
-import datetime
+import hashlib
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_etree_fromstring,
-    compat_str,
-    compat_parse_qs,
-    compat_xml_parse_error,
-)
+from ..compat import compat_parse_qs
 from ..utils import (
-    ExtractorError,
     int_or_none,
     float_or_none,
-    xpath_text,
+    unified_timestamp,
 )
 
 
 class BiliBiliIE(InfoExtractor):
-    _WORKING = False
-
     _VALID_URL = r'https?://www\.bilibili\.(?:tv|com)/video/av(?P<id>\d+)'
 
     _TESTS = [{
         'url': 'http://www.bilibili.tv/video/av1074402/',
         'md5': '9fa226fe2b8a9a4d5a69b4c6a183417e',
         'info_dict': {
-            'id': '1554319',
+            'id': '1074402',
             'ext': 'mp4',
             'title': '【金坷垃】金泡沫',
             'description': 'md5:ce18c2a2d2193f0df2917d270f2e5923',
@@ -43,24 +34,28 @@ class BiliBiliIE(InfoExtractor):
     }, {
         'url': 'http://www.bilibili.com/video/av1041170/',
         'info_dict': {
-            'id': '1507019',
+            'id': '1041170',
             'ext': 'mp4',
             'title': '【BD1080P】刀语【诸神&异域】',
             'description': '这是个神奇的故事~每个人不留弹幕不给走哦~切利哦！~',
+            'duration': 3382.259,
             'timestamp': 1396530060,
             'upload_date': '20140403',
+            'thumbnail': 're:^https?://.+\.jpg',
             'uploader': '枫叶逝去',
             'uploader_id': '520116',
         },
     }, {
         'url': 'http://www.bilibili.com/video/av4808130/',
         'info_dict': {
-            'id': '7802182',
+            'id': '4808130',
             'ext': 'mp4',
             'title': '【长篇】哆啦A梦443【钉铛】',
             'description': '(2016.05.27)来组合客人的脸吧&amp;amp;寻母六千里锭 抱歉，又轮到周日上班现在才到家 封面www.pixiv.net/member_illust.php?mode=medium&amp;amp;illust_id=56912929',
+            'duration': 1493.995,
             'timestamp': 1464564180,
             'upload_date': '20160529',
+            'thumbnail': 're:^https?://.+\.jpg',
             'uploader': '喜欢拉面',
             'uploader_id': '151066',
         },
@@ -68,12 +63,14 @@ class BiliBiliIE(InfoExtractor):
         # Missing upload time
         'url': 'http://www.bilibili.com/video/av1867637/',
         'info_dict': {
-            'id': '2880301',
+            'id': '1867637',
             'ext': 'mp4',
             'title': '【HDTV】【喜剧】岳父岳母真难当 （2014）【法国票房冠军】',
             'description': '一个信奉天主教的法国旧式传统资产阶级家庭中有四个女儿。三个女儿却分别找了阿拉伯、犹太、中国丈夫，老夫老妻唯独期盼剩下未嫁的小女儿能找一个信奉天主教的法国白人，结果没想到小女儿找了一位非裔黑人……【这次应该不会跳帧了】',
+            'duration': 5760.0,
             'uploader': '黑夜为猫',
             'uploader_id': '610729',
+            'thumbnail': 're:^https?://.+\.jpg',
         },
         'params': {
             # Just to test metadata extraction
@@ -82,86 +79,61 @@ class BiliBiliIE(InfoExtractor):
         'expected_warnings': ['upload time'],
     }]
 
-    # BiliBili blocks keys from time to time. The current key is extracted from
-    # the Android client
-    # TODO: find the sign algorithm used in the flash player
-    _APP_KEY = '86385cdc024c0f6c'
+    _APP_KEY = '6f90a59ac58a4123'
+    _BILIBILI_KEY = '0bfd84cc3940035173f35e6777508326'
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id)
 
-        params = compat_parse_qs(self._search_regex(
+        cid = compat_parse_qs(self._search_regex(
             [r'EmbedPlayer\([^)]+,\s*"([^"]+)"\)',
              r'<iframe[^>]+src="https://secure\.bilibili\.com/secure,([^"]+)"'],
-            webpage, 'player parameters'))
-        cid = params['cid'][0]
+            webpage, 'player parameters'))['cid'][0]
 
-        info_xml_str = self._download_webpage(
-            'http://interface.bilibili.com/v_cdn_play',
-            cid, query={'appkey': self._APP_KEY, 'cid': cid},
-            note='Downloading video info page')
+        payload = 'appkey=%s&cid=%s&otype=json&quality=2&type=mp4' % (self._APP_KEY, cid)
+        sign = hashlib.md5((payload + self._BILIBILI_KEY).encode('utf-8')).hexdigest()
 
-        err_msg = None
-        durls = None
-        info_xml = None
-        try:
-            info_xml = compat_etree_fromstring(info_xml_str.encode('utf-8'))
-        except compat_xml_parse_error:
-            info_json = self._parse_json(info_xml_str, video_id, fatal=False)
-            err_msg = (info_json or {}).get('error_text')
-        else:
-            err_msg = xpath_text(info_xml, './message')
-
-        if info_xml is not None:
-            durls = info_xml.findall('./durl')
-        if not durls:
-            if err_msg:
-                raise ExtractorError('%s said: %s' % (self.IE_NAME, err_msg), expected=True)
-            else:
-                raise ExtractorError('No videos found!')
+        video_info = self._download_json(
+            'http://interface.bilibili.com/playurl?%s&sign=%s' % (payload, sign),
+            video_id, note='Downloading video info page')
 
         entries = []
 
-        for durl in durls:
-            size = xpath_text(durl, ['./filesize', './size'])
+        for idx, durl in enumerate(video_info['durl']):
             formats = [{
-                'url': durl.find('./url').text,
-                'filesize': int_or_none(size),
+                'url': durl['url'],
+                'filesize': int_or_none(durl['size']),
             }]
-            for backup_url in durl.findall('./backup_url/url'):
+            for backup_url in durl['backup_url']:
                 formats.append({
-                    'url': backup_url.text,
+                    'url': backup_url,
                     # backup URLs have lower priorities
-                    'preference': -2 if 'hd.mp4' in backup_url.text else -3,
+                    'preference': -2 if 'hd.mp4' in backup_url else -3,
                 })
 
             self._sort_formats(formats)
 
             entries.append({
-                'id': '%s_part%s' % (cid, xpath_text(durl, './order')),
-                'duration': int_or_none(xpath_text(durl, './length'), 1000),
+                'id': '%s_part%s' % (video_id, idx),
+                'duration': float_or_none(durl.get('length'), 1000),
                 'formats': formats,
             })
 
         title = self._html_search_regex('<h1[^>]+title="([^"]+)">', webpage, 'title')
         description = self._html_search_meta('description', webpage)
-        datetime_str = self._html_search_regex(
-            r'<time[^>]+datetime="([^"]+)"', webpage, 'upload time', fatal=False)
-        timestamp = None
-        if datetime_str:
-            timestamp = calendar.timegm(datetime.datetime.strptime(datetime_str, '%Y-%m-%dT%H:%M').timetuple())
+        timestamp = unified_timestamp(self._html_search_regex(
+            r'<time[^>]+datetime="([^"]+)"', webpage, 'upload time', fatal=False))
 
         # TODO 'view_count' requires deobfuscating Javascript
         info = {
-            'id': compat_str(cid),
+            'id': video_id,
             'title': title,
             'description': description,
             'timestamp': timestamp,
             'thumbnail': self._html_search_meta('thumbnailUrl', webpage),
-            'duration': float_or_none(xpath_text(info_xml, './timelength'), scale=1000),
+            'duration': float_or_none(video_info.get('timelength'), scale=1000),
         }
 
         uploader_mobj = re.search(

From 98908bcf7c50d034042ab86223b7689e91b589ba Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 28 Aug 2016 22:49:46 +0800
Subject: [PATCH 5915/6219] [openload] Update algorithm again (#10408)

---
 youtube_dl/extractor/openload.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py
index c8dde7ae3..03baf8e32 100644
--- a/youtube_dl/extractor/openload.py
+++ b/youtube_dl/extractor/openload.py
@@ -60,7 +60,7 @@ class OpenloadIE(InfoExtractor):
             if j >= 33 and j <= 126:
                 j = ((j + 14) % 94) + 33
             if idx == len(enc_data) - 1:
-                j += 2
+                j += 1
             video_url_chars += compat_chr(j)
 
         video_url = 'https://openload.co/stream/%s?mime=true' % ''.join(video_url_chars)

From 2982514072594b1f708abdf654b31da77c0bfa81 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 28 Aug 2016 16:43:15 +0100
Subject: [PATCH 5916/6219] [turner,nba,cnn,adultswim] add base extractor to
 parse cvp feeds

---
 youtube_dl/extractor/adultswim.py |  70 +++----------
 youtube_dl/extractor/cnn.py       |  97 +++++-------------
 youtube_dl/extractor/nba.py       |  70 +++----------
 youtube_dl/extractor/turner.py    | 163 ++++++++++++++++++++++++++++++
 4 files changed, 214 insertions(+), 186 deletions(-)
 create mode 100644 youtube_dl/extractor/turner.py

diff --git a/youtube_dl/extractor/adultswim.py b/youtube_dl/extractor/adultswim.py
index 96599048f..ef3cc2a61 100644
--- a/youtube_dl/extractor/adultswim.py
+++ b/youtube_dl/extractor/adultswim.py
@@ -3,16 +3,11 @@ from __future__ import unicode_literals
 
 import re
 
-from .common import InfoExtractor
-from ..utils import (
-    determine_ext,
-    ExtractorError,
-    float_or_none,
-    xpath_text,
-)
+from .turner import TurnerBaseIE
+from ..utils import ExtractorError
 
 
-class AdultSwimIE(InfoExtractor):
+class AdultSwimIE(TurnerBaseIE):
     _VALID_URL = r'https?://(?:www\.)?adultswim\.com/videos/(?P<is_playlist>playlists/)?(?P<show_path>[^/]+)/(?P<episode_path>[^/?#]+)/?'
 
     _TESTS = [{
@@ -96,7 +91,8 @@ class AdultSwimIE(InfoExtractor):
         'params': {
             # m3u8 download
             'skip_download': True,
-        }
+        },
+        'expected_warnings': ['Unable to download f4m manifest'],
     }]
 
     @staticmethod
@@ -176,57 +172,23 @@ class AdultSwimIE(InfoExtractor):
 
         entries = []
         for part_num, segment_id in enumerate(segment_ids):
-            segment_url = 'http://www.adultswim.com/videos/api/v0/assets?id=%s&platform=desktop' % segment_id
-
+            segement_info = self._extract_cvp_info(
+                'http://www.adultswim.com/videos/api/v0/assets?id=%s&platform=desktop' % segment_id,
+                segment_id, {
+                    'secure': {
+                        'media_src': 'http://androidhls-secure.cdn.turner.com/adultswim/big',
+                        'tokenizer_src': 'http://www.adultswim.com/astv/mvpd/processors/services/token_ipadAdobe.do',
+                    },
+                })
             segment_title = '%s - %s' % (show_title, episode_title)
             if len(segment_ids) > 1:
                 segment_title += ' Part %d' % (part_num + 1)
-
-            idoc = self._download_xml(
-                segment_url, segment_title,
-                'Downloading segment information', 'Unable to download segment information')
-
-            segment_duration = float_or_none(
-                xpath_text(idoc, './/trt', 'segment duration').strip())
-
-            formats = []
-            file_els = idoc.findall('.//files/file') or idoc.findall('./files/file')
-
-            unique_urls = []
-            unique_file_els = []
-            for file_el in file_els:
-                media_url = file_el.text
-                if not media_url or determine_ext(media_url) == 'f4m':
-                    continue
-                if file_el.text not in unique_urls:
-                    unique_urls.append(file_el.text)
-                    unique_file_els.append(file_el)
-
-            for file_el in unique_file_els:
-                bitrate = file_el.attrib.get('bitrate')
-                ftype = file_el.attrib.get('type')
-                media_url = file_el.text
-                if determine_ext(media_url) == 'm3u8':
-                    formats.extend(self._extract_m3u8_formats(
-                        media_url, segment_title, 'mp4', preference=0,
-                        m3u8_id='hls', fatal=False))
-                else:
-                    formats.append({
-                        'format_id': '%s_%s' % (bitrate, ftype),
-                        'url': file_el.text.strip(),
-                        # The bitrate may not be a number (for example: 'iphone')
-                        'tbr': int(bitrate) if bitrate.isdigit() else None,
-                    })
-
-            self._sort_formats(formats)
-
-            entries.append({
+            segement_info.update({
                 'id': segment_id,
                 'title': segment_title,
-                'formats': formats,
-                'duration': segment_duration,
-                'description': episode_description
+                'description': episode_description,
             })
+            entries.append(segement_info)
 
         return {
             '_type': 'playlist',
diff --git a/youtube_dl/extractor/cnn.py b/youtube_dl/extractor/cnn.py
index 220bb55e8..1bf87f6ea 100644
--- a/youtube_dl/extractor/cnn.py
+++ b/youtube_dl/extractor/cnn.py
@@ -3,14 +3,11 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import (
-    int_or_none,
-    parse_duration,
-    url_basename,
-)
+from .turner import TurnerBaseIE
+from ..utils import url_basename
 
 
-class CNNIE(InfoExtractor):
+class CNNIE(TurnerBaseIE):
     _VALID_URL = r'''(?x)https?://(?:(?P<sub_domain>edition|www|money)\.)?cnn\.com/(?:video/(?:data/.+?|\?)/)?videos?/
         (?P<path>.+?/(?P<title>[^/]+?)(?:\.(?:[a-z\-]+)|(?=&)))'''
 
@@ -18,43 +15,50 @@ class CNNIE(InfoExtractor):
         'url': 'http://edition.cnn.com/video/?/video/sports/2013/06/09/nadal-1-on-1.cnn',
         'md5': '3e6121ea48df7e2259fe73a0628605c4',
         'info_dict': {
-            'id': 'sports/2013/06/09/nadal-1-on-1.cnn',
+            'id': 'nadal-1-on-1',
             'ext': 'mp4',
             'title': 'Nadal wins 8th French Open title',
             'description': 'World Sport\'s Amanda Davies chats with 2013 French Open champion Rafael Nadal.',
             'duration': 135,
             'upload_date': '20130609',
         },
+        'expected_warnings': ['Failed to download m3u8 information'],
     }, {
         'url': 'http://edition.cnn.com/video/?/video/us/2013/08/21/sot-student-gives-epic-speech.georgia-institute-of-technology&utm_source=feedburner&utm_medium=feed&utm_campaign=Feed%3A+rss%2Fcnn_topstories+%28RSS%3A+Top+Stories%29',
         'md5': 'b5cc60c60a3477d185af8f19a2a26f4e',
         'info_dict': {
-            'id': 'us/2013/08/21/sot-student-gives-epic-speech.georgia-institute-of-technology',
+            'id': 'sot-student-gives-epic-speech',
             'ext': 'mp4',
             'title': "Student's epic speech stuns new freshmen",
             'description': "A Georgia Tech student welcomes the incoming freshmen with an epic speech backed by music from \"2001: A Space Odyssey.\"",
             'upload_date': '20130821',
-        }
+        },
+        'expected_warnings': ['Failed to download m3u8 information'],
     }, {
         'url': 'http://www.cnn.com/video/data/2.0/video/living/2014/12/22/growing-america-nashville-salemtown-board-episode-1.hln.html',
         'md5': 'f14d02ebd264df951feb2400e2c25a1b',
         'info_dict': {
-            'id': 'living/2014/12/22/growing-america-nashville-salemtown-board-episode-1.hln',
+            'id': 'growing-america-nashville-salemtown-board-episode-1',
             'ext': 'mp4',
             'title': 'Nashville Ep. 1: Hand crafted skateboards',
             'description': 'md5:e7223a503315c9f150acac52e76de086',
             'upload_date': '20141222',
-        }
+        },
+        'expected_warnings': ['Failed to download m3u8 information'],
     }, {
         'url': 'http://money.cnn.com/video/news/2016/08/19/netflix-stunning-stats.cnnmoney/index.html',
         'md5': '52a515dc1b0f001cd82e4ceda32be9d1',
         'info_dict': {
-            'id': '/video/news/2016/08/19/netflix-stunning-stats.cnnmoney',
+            'id': 'netflix-stunning-stats',
             'ext': 'mp4',
             'title': '5 stunning stats about Netflix',
             'description': 'Did you know that Netflix has more than 80 million members? Here are five facts about the online video distributor that you probably didn\'t know.',
             'upload_date': '20160819',
-        }
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
     }, {
         'url': 'http://cnn.com/video/?/video/politics/2015/03/27/pkg-arizona-senator-church-attendance-mandatory.ktvk',
         'only_matching': True,
@@ -84,67 +88,12 @@ class CNNIE(InfoExtractor):
         if sub_domain not in ('money', 'edition'):
             sub_domain = 'edition'
         config = self._CONFIG[sub_domain]
-        info_url = config['data_src'] % path
-        info = self._download_xml(info_url, page_title)
-
-        formats = []
-        rex = re.compile(r'''(?x)
-            (?P<width>[0-9]+)x(?P<height>[0-9]+)
-            (?:_(?P<bitrate>[0-9]+)k)?
-        ''')
-        for f in info.findall('files/file'):
-            video_url = config['media_src'] + f.text.strip()
-            fdct = {
-                'format_id': f.attrib['bitrate'],
-                'url': video_url,
-            }
-
-            mf = rex.match(f.attrib['bitrate'])
-            if mf:
-                fdct['width'] = int(mf.group('width'))
-                fdct['height'] = int(mf.group('height'))
-                fdct['tbr'] = int_or_none(mf.group('bitrate'))
-            else:
-                mf = rex.search(f.text)
-                if mf:
-                    fdct['width'] = int(mf.group('width'))
-                    fdct['height'] = int(mf.group('height'))
-                    fdct['tbr'] = int_or_none(mf.group('bitrate'))
-                else:
-                    mi = re.match(r'ios_(audio|[0-9]+)$', f.attrib['bitrate'])
-                    if mi:
-                        if mi.group(1) == 'audio':
-                            fdct['vcodec'] = 'none'
-                            fdct['ext'] = 'm4a'
-                        else:
-                            fdct['tbr'] = int(mi.group(1))
-
-            formats.append(fdct)
-
-        self._sort_formats(formats)
-
-        thumbnails = [{
-            'height': int(t.attrib['height']),
-            'width': int(t.attrib['width']),
-            'url': t.text,
-        } for t in info.findall('images/image')]
-
-        metas_el = info.find('metas')
-        upload_date = (
-            metas_el.attrib.get('version') if metas_el is not None else None)
-
-        duration_el = info.find('length')
-        duration = parse_duration(duration_el.text)
-
-        return {
-            'id': info.attrib['id'],
-            'title': info.find('headline').text,
-            'formats': formats,
-            'thumbnails': thumbnails,
-            'description': info.find('description').text,
-            'duration': duration,
-            'upload_date': upload_date,
-        }
+        return self._extract_cvp_info(
+            config['data_src'] % path, page_title, {
+                'default': {
+                    'media_src': config['media_src'],
+                }
+            })
 
 
 class CNNBlogsIE(InfoExtractor):
diff --git a/youtube_dl/extractor/nba.py b/youtube_dl/extractor/nba.py
index d896b0d04..aabd5b670 100644
--- a/youtube_dl/extractor/nba.py
+++ b/youtube_dl/extractor/nba.py
@@ -1,25 +1,20 @@
 from __future__ import unicode_literals
 
 import functools
-import os.path
 import re
 
-from .common import InfoExtractor
+from .turner import TurnerBaseIE
 from ..compat import (
     compat_urllib_parse_urlencode,
     compat_urlparse,
 )
 from ..utils import (
-    int_or_none,
     OnDemandPagedList,
-    parse_duration,
     remove_start,
-    xpath_text,
-    xpath_attr,
 )
 
 
-class NBAIE(InfoExtractor):
+class NBAIE(TurnerBaseIE):
     _VALID_URL = r'https?://(?:watch\.|www\.)?nba\.com/(?P<path>(?:[^/]+/)+(?P<id>[^?]*?))/?(?:/index\.html)?(?:\?.*)?$'
     _TESTS = [{
         'url': 'http://www.nba.com/video/games/nets/2012/12/04/0021200253-okc-bkn-recap.nba/index.html',
@@ -59,7 +54,7 @@ class NBAIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Practice: Doc Rivers - 2/16/16',
             'description': 'Head Coach Doc Rivers addresses the media following practice.',
-            'upload_date': '20160217',
+            'upload_date': '20160216',
             'timestamp': 1455672000,
         },
         'params': {
@@ -80,7 +75,7 @@ class NBAIE(InfoExtractor):
     }, {
         'url': 'http://www.nba.com/timberwolves/wiggins-shootaround#',
         'info_dict': {
-            'id': 'Wigginsmp4',
+            'id': 'Wigginsmp4-3462601',
             'ext': 'mp4',
             'title': 'Shootaround Access - Dec. 12 | Andrew Wiggins',
             'description': 'Wolves rookie Andrew Wiggins addresses the media after Friday\'s shootaround.',
@@ -145,53 +140,12 @@ class NBAIE(InfoExtractor):
             if path.startswith('video/teams'):
                 path = 'video/channels/proxy/' + path[6:]
 
-        video_info = self._download_xml('http://www.nba.com/%s.xml' % path, video_id)
-        video_id = os.path.splitext(xpath_text(video_info, 'slug'))[0]
-        title = xpath_text(video_info, 'headline')
-        description = xpath_text(video_info, 'description')
-        duration = parse_duration(xpath_text(video_info, 'length'))
-        timestamp = int_or_none(xpath_attr(video_info, 'dateCreated', 'uts'))
-
-        thumbnails = []
-        for image in video_info.find('images'):
-            thumbnails.append({
-                'id': image.attrib.get('cut'),
-                'url': image.text,
-                'width': int_or_none(image.attrib.get('width')),
-                'height': int_or_none(image.attrib.get('height')),
+        return self._extract_cvp_info(
+            'http://www.nba.com/%s.xml' % path, video_id, {
+                'default': {
+                    'media_src': 'http://nba.cdn.turner.com/nba/big',
+                },
+                'm3u8': {
+                    'media_src': 'http://nbavod-f.akamaihd.net',
+                },
             })
-
-        formats = []
-        for video_file in video_info.findall('.//file'):
-            video_url = video_file.text
-            if video_url.startswith('/'):
-                continue
-            if video_url.endswith('.m3u8'):
-                formats.extend(self._extract_m3u8_formats(video_url, video_id, ext='mp4', m3u8_id='hls', fatal=False))
-            elif video_url.endswith('.f4m'):
-                formats.extend(self._extract_f4m_formats(video_url + '?hdcore=3.4.1.1', video_id, f4m_id='hds', fatal=False))
-            else:
-                key = video_file.attrib.get('bitrate')
-                format_info = {
-                    'format_id': key,
-                    'url': video_url,
-                }
-                mobj = re.search(r'(\d+)x(\d+)(?:_(\d+))?', key)
-                if mobj:
-                    format_info.update({
-                        'width': int(mobj.group(1)),
-                        'height': int(mobj.group(2)),
-                        'tbr': int_or_none(mobj.group(3)),
-                    })
-                formats.append(format_info)
-        self._sort_formats(formats)
-
-        return {
-            'id': video_id,
-            'title': title,
-            'description': description,
-            'duration': duration,
-            'timestamp': timestamp,
-            'thumbnails': thumbnails,
-            'formats': formats,
-        }
diff --git a/youtube_dl/extractor/turner.py b/youtube_dl/extractor/turner.py
new file mode 100644
index 000000000..0d4271f11
--- /dev/null
+++ b/youtube_dl/extractor/turner.py
@@ -0,0 +1,163 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    xpath_text,
+    int_or_none,
+    determine_ext,
+    parse_duration,
+    xpath_attr,
+    update_url_query,
+)
+
+
+class TurnerBaseIE(InfoExtractor):
+    def _extract_cvp_info(self, data_src, video_id, path_data={}):
+        video_data = self._download_xml(data_src, video_id)
+        video_id = video_data.attrib['id'].split('/')[-1].split('.')[0]
+        title = xpath_text(video_data, 'headline', fatal=True)
+        # rtmp_src = xpath_text(video_data, 'akamai/src')
+        # if rtmp_src:
+        #     splited_rtmp_src = rtmp_src.split(',')
+        #     if len(splited_rtmp_src) == 2:
+        #         rtmp_src = splited_rtmp_src[1]
+        # aifp = xpath_text(video_data, 'akamai/aifp', default='')
+
+        tokens = {}
+        urls = []
+        formats = []
+        rex = re.compile(r'''(?x)
+            (?P<width>[0-9]+)x(?P<height>[0-9]+)
+            (?:_(?P<bitrate>[0-9]+))?
+        ''')
+        for video_file in video_data.findall('files/file'):
+            video_url = video_file.text.strip()
+            if not video_url:
+                continue
+            ext = determine_ext(video_url)
+            if video_url.startswith('/mp4:protected/'):
+                continue
+                # TODO Correct extraction for these files
+                # protected_path_data = path_data.get('protected')
+                # if not protected_path_data or not rtmp_src:
+                #     continue
+                # protected_path = self._search_regex(
+                #     r'/mp4:(.+)\.[a-z0-9]', video_url, 'secure path')
+                # auth = self._download_webpage(
+                #     protected_path_data['tokenizer_src'], query={
+                #         'path': protected_path,
+                #         'videoId': video_id,
+                #         'aifp': aifp,
+                #     })
+                # token = xpath_text(auth, 'token')
+                # if not token:
+                #     continue
+                # video_url = rtmp_src + video_url + '?' + token
+            elif video_url.startswith('/secure/'):
+                secure_path_data = path_data.get('secure')
+                if not secure_path_data:
+                    continue
+                video_url = secure_path_data['media_src'] + video_url
+                secure_path = self._search_regex(r'https?://[^/]+(.+/)', video_url, 'secure path') + '*'
+                token = tokens.get(secure_path)
+                if not token:
+                    auth = self._download_xml(
+                        secure_path_data['tokenizer_src'], video_id, query={
+                            'path': secure_path,
+                            'videoId': video_id,
+                        })
+                    token = xpath_text(auth, 'token')
+                    if not token:
+                        continue
+                    tokens[secure_path] = token
+                video_url = video_url + '?hdnea=' + token
+            elif not re.match('https?://', video_url):
+                base_path_data = path_data.get(ext, path_data.get('default', {}))
+                media_src = base_path_data.get('media_src')
+                if not media_src:
+                    continue
+                video_url = media_src + video_url
+            if video_url in urls:
+                continue
+            urls.append(video_url)
+            format_id = video_file.attrib['bitrate']
+            if ext == 'smil':
+                formats.extend(self._extract_smil_formats(video_url, video_id, fatal=False))
+            elif ext == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    video_url, video_id, 'mp4', m3u8_id=format_id, fatal=False))
+            elif ext == 'f4m':
+                formats.extend(self._extract_f4m_formats(
+                    update_url_query(video_url, {'hdcore': '3.7.0'}),
+                    video_id, f4m_id=format_id, fatal=False))
+            else:
+                f = {
+                    'format_id': format_id,
+                    'url': video_url,
+                    'ext': ext,
+                }
+                mobj = rex.search(format_id + video_url)
+                if mobj:
+                    f.update({
+                        'width': int(mobj.group('width')),
+                        'height': int(mobj.group('height')),
+                        'tbr': int_or_none(mobj.group('bitrate')),
+                    })
+                elif format_id.isdigit():
+                    f['tbr'] = int(format_id)
+                else:
+                    mobj = re.match(r'ios_(audio|[0-9]+)$', format_id)
+                    if mobj:
+                        if mobj.group(1) == 'audio':
+                            f.update({
+                                'vcodec': 'none',
+                                'ext': 'm4a',
+                            })
+                        else:
+                            f['tbr'] = int(mobj.group(1))
+                formats.append(f)
+        self._sort_formats(formats)
+
+        subtitles = {}
+        for source in video_data.findall('closedCaptions/source'):
+            for track in source.findall('track'):
+                source_url = source.get('url')
+                if not source_url:
+                    continue
+                subtitles.set_default(source.get('lang') or source.get('label') or 'en', []).append({
+                    'url': source_url,
+                    'ext': {
+                        'scc': 'scc',
+                        'webvtt': 'vtt',
+                        'smptett': 'tt',
+                    }.get(source.get('format'))
+                })
+
+        thumbnails = [{
+            'id': image.get('cut'),
+            'url': image.text,
+            'width': int_or_none(image.get('width')),
+            'height': int_or_none(image.get('height')),
+        } for image in video_data.findall('images/image')]
+
+        timestamp = None
+        if 'cnn.com' not in data_src:
+            timestamp = int_or_none(xpath_attr(video_data, 'dateCreated', 'uts'))
+
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': formats,
+            'subtitles': subtitles,
+            'thumbnails': thumbnails,
+            'description': xpath_text(video_data, 'description'),
+            'duration': parse_duration(xpath_text(video_data, 'length') or xpath_text(video_data, 'trt')),
+            'timestamp': timestamp,
+            'upload_date': xpath_attr(video_data, 'metas', 'version'),
+            'series': xpath_text(video_data, 'showTitle'),
+            'season_number': int_or_none(xpath_text(video_data, 'seasonNumber')),
+            'episode_number': int_or_none(xpath_text(video_data, 'episodeNumber')),
+        }

From ec65b391cbb0bc42a78515915e61602f4d1ae1f9 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 28 Aug 2016 16:47:59 +0100
Subject: [PATCH 5917/6219] [cartoonnetwork] Add new extractor(#10110)

---
 youtube_dl/extractor/cartoonnetwork.py | 36 ++++++++++++++++++++++++++
 youtube_dl/extractor/extractors.py     |  1 +
 2 files changed, 37 insertions(+)
 create mode 100644 youtube_dl/extractor/cartoonnetwork.py

diff --git a/youtube_dl/extractor/cartoonnetwork.py b/youtube_dl/extractor/cartoonnetwork.py
new file mode 100644
index 000000000..813f53644
--- /dev/null
+++ b/youtube_dl/extractor/cartoonnetwork.py
@@ -0,0 +1,36 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .turner import TurnerBaseIE
+
+
+class CartoonNetworkIE(TurnerBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?cartoonnetwork\.com/video/(?:[^/]+/)+(?P<id>[^/?#]+)-(?:clip|episode)\.html'
+    _TEST = {
+        'url': 'http://www.cartoonnetwork.com/video/teen-titans-go/starfire-the-cat-lady-clip.html',
+        'info_dict': {
+            'id': '8a250ab04ed07e6c014ef3f1e2f9016c',
+            'ext': 'mp4',
+            'title': 'Starfire the Cat Lady',
+            'description': 'Robin decides to become a cat so that Starfire will finally love him.',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+        id_type, video_id = re.search(r"_cnglobal\.cvp(Video|Title)Id\s*=\s*'([^']+)';", webpage).groups()
+        query = ('id' if id_type == 'Video' else 'titleId') + '=' + video_id
+        return self._extract_cvp_info(
+            'http://www.cartoonnetwork.com/video-seo-svc/episodeservices/getCvpPlaylist?' + query, video_id, {
+                'secure': {
+                    'media_src': 'http://apple-secure.cdn.turner.com/toon/big',
+                    'tokenizer_src': 'http://www.cartoonnetwork.com/cntv/mvpd/processors/services/token_ipadAdobe.do',
+                },
+            })
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 8d88d6cb4..6eb495b07 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -127,6 +127,7 @@ from .carambatv import (
     CarambaTVIE,
     CarambaTVPageIE,
 )
+from .cartoonnetwork import CartoonNetworkIE
 from .cbc import (
     CBCIE,
     CBCPlayerIE,

From b3eaeded12f470afd6f0cb851e6b7dd2ee78b7c5 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 28 Aug 2016 16:50:32 +0100
Subject: [PATCH 5918/6219] [tbs] Add new extractor(#10222)

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/tbs.py        | 59 ++++++++++++++++++++++++++++++
 2 files changed, 60 insertions(+)
 create mode 100644 youtube_dl/extractor/tbs.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 6eb495b07..06c6746ff 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -824,6 +824,7 @@ from .tagesschau import (
     TagesschauIE,
 )
 from .tass import TassIE
+from .tbs import TBSIE
 from .tdslifeway import TDSLifewayIE
 from .teachertube import (
     TeacherTubeIE,
diff --git a/youtube_dl/extractor/tbs.py b/youtube_dl/extractor/tbs.py
new file mode 100644
index 000000000..79b00e376
--- /dev/null
+++ b/youtube_dl/extractor/tbs.py
@@ -0,0 +1,59 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .turner import TurnerBaseIE
+from ..utils import (
+    extract_attributes,
+    ExtractorError,
+)
+
+
+class TBSIE(TurnerBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?(?P<site>tbs|tntdrama)\.com/videos/(?:[^/]+/)+(?P<id>[^/?#]+)\.html'
+    _TESTS = [{
+        'url': 'http://www.tbs.com/videos/people-of-earth/season-1/extras/2007318/theatrical-trailer.html',
+        'md5': '9e61d680e2285066ade7199e6408b2ee',
+        'info_dict': {
+            'id': '2007318',
+            'ext': 'mp4',
+            'title': 'Theatrical Trailer',
+            'description': 'Catch the latest comedy from TBS, People of Earth, premiering Halloween night--Monday, October 31, at 9/8c.',
+        }
+    }, {
+        'url': 'http://www.tntdrama.com/videos/good-behavior/season-1/extras/1538823/you-better-run.html',
+        'md5': 'ce53c6ead5e9f3280b4ad2031a6fab56',
+        'info_dict': {
+            'id': '1538823',
+            'ext': 'mp4',
+            'title': 'You Better Run',
+            'description': 'Letty Raines must figure out what she\'s running toward while running away from her past. Good Behavior premieres November 15 at 9/8c.',
+        }
+    }]
+
+    def _real_extract(self, url):
+        domain, display_id = re.match(self._VALID_URL, url).groups()
+        site = domain[:3]
+        webpage = self._download_webpage(url, display_id)
+        video_params = extract_attributes(self._search_regex(r'(<[^>]+id="page-video"[^>]*>)', webpage, 'video params'))
+        if video_params.get('isAuthRequired') == 'true':
+            raise ExtractorError(
+                'This video is only available via cable service provider subscription that'
+                ' is not currently supported.', expected=True)
+        query = None
+        clip_id = video_params.get('clipid')
+        if clip_id:
+            query = 'id=' + clip_id
+        else:
+            query = 'titleId=' + video_params['titleid']
+        return self._extract_cvp_info(
+            'http://www.%s.com/service/cvpXml?%s' % (domain, query), display_id, {
+                'default': {
+                    'media_src': 'http://ht.cdn.turner.com/%s/big' % site,
+                },
+                'secure': {
+                    'media_src': 'http://apple-secure.cdn.turner.com/%s/big' % site,
+                    'tokenizer_src': 'http://www.%s.com/video/processors/services/token_ipadAdobe.do' % domain,
+                },
+            })

From 5bc8a73af69f4aac8b2df6f7c23ecfb4ee72e518 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 28 Aug 2016 17:08:26 +0100
Subject: [PATCH 5919/6219] [cartoonnetwork] make extraction work for more
 videos in the website

some videos require `networkName=CN2` to be present in the feed url
---
 youtube_dl/extractor/cartoonnetwork.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/cartoonnetwork.py b/youtube_dl/extractor/cartoonnetwork.py
index 813f53644..b3f30b1ca 100644
--- a/youtube_dl/extractor/cartoonnetwork.py
+++ b/youtube_dl/extractor/cartoonnetwork.py
@@ -28,7 +28,7 @@ class CartoonNetworkIE(TurnerBaseIE):
         id_type, video_id = re.search(r"_cnglobal\.cvp(Video|Title)Id\s*=\s*'([^']+)';", webpage).groups()
         query = ('id' if id_type == 'Video' else 'titleId') + '=' + video_id
         return self._extract_cvp_info(
-            'http://www.cartoonnetwork.com/video-seo-svc/episodeservices/getCvpPlaylist?' + query, video_id, {
+            'http://www.cartoonnetwork.com/video-seo-svc/episodeservices/getCvpPlaylist?networkName=CN2&' + query, video_id, {
                 'secure': {
                     'media_src': 'http://apple-secure.cdn.turner.com/toon/big',
                     'tokenizer_src': 'http://www.cartoonnetwork.com/cntv/mvpd/processors/services/token_ipadAdobe.do',

From b8079a40bc61326b17a672b073dce6cdfa791fb5 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 28 Aug 2016 17:51:53 +0100
Subject: [PATCH 5920/6219] [turner] fix secure m3u8 formats downloading

---
 youtube_dl/downloader/hls.py   | 11 +++++++----
 youtube_dl/extractor/turner.py | 15 +++++++++++++--
 youtube_dl/extractor/uplynk.py |  4 +---
 3 files changed, 21 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index 8dd1b898e..baaff44d5 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -83,7 +83,10 @@ class HlsFD(FragmentFD):
 
         self._prepare_and_start_frag_download(ctx)
 
+        extra_query = None
         extra_param_to_segment_url = info_dict.get('extra_param_to_segment_url')
+        if extra_param_to_segment_url:
+            extra_query = compat_urlparse.parse_qs(extra_param_to_segment_url)
         i = 0
         media_sequence = 0
         decrypt_info = {'METHOD': 'NONE'}
@@ -97,8 +100,8 @@ class HlsFD(FragmentFD):
                         if re.match(r'^https?://', line)
                         else compat_urlparse.urljoin(man_url, line))
                     frag_filename = '%s-Frag%d' % (ctx['tmpfilename'], i)
-                    if extra_param_to_segment_url:
-                        frag_url = update_url_query(frag_url, extra_param_to_segment_url)
+                    if extra_query:
+                        frag_url = update_url_query(frag_url, extra_query)
                     success = ctx['dl'].download(frag_filename, {'url': frag_url})
                     if not success:
                         return False
@@ -124,8 +127,8 @@ class HlsFD(FragmentFD):
                         if not re.match(r'^https?://', decrypt_info['URI']):
                             decrypt_info['URI'] = compat_urlparse.urljoin(
                                 man_url, decrypt_info['URI'])
-                        if extra_param_to_segment_url:
-                            decrypt_info['URI'] = update_url_query(decrypt_info['URI'], extra_param_to_segment_url)
+                        if extra_query:
+                            decrypt_info['URI'] = update_url_query(decrypt_info['URI'], extra_query)
                         decrypt_info['KEY'] = self.ydl.urlopen(decrypt_info['URI']).read()
                 elif line.startswith('#EXT-X-MEDIA-SEQUENCE'):
                     media_sequence = int(line[22:])
diff --git a/youtube_dl/extractor/turner.py b/youtube_dl/extractor/turner.py
index 0d4271f11..108caa9d8 100644
--- a/youtube_dl/extractor/turner.py
+++ b/youtube_dl/extractor/turner.py
@@ -11,6 +11,7 @@ from ..utils import (
     parse_duration,
     xpath_attr,
     update_url_query,
+    compat_urlparse,
 )
 
 
@@ -87,8 +88,18 @@ class TurnerBaseIE(InfoExtractor):
             if ext == 'smil':
                 formats.extend(self._extract_smil_formats(video_url, video_id, fatal=False))
             elif ext == 'm3u8':
-                formats.extend(self._extract_m3u8_formats(
-                    video_url, video_id, 'mp4', m3u8_id=format_id, fatal=False))
+                m3u8_formats = self._extract_m3u8_formats(
+                    video_url, video_id, 'mp4', m3u8_id=format_id, fatal=False)
+                if m3u8_formats:
+                    # Sometimes final URLs inside m3u8 are unsigned, let's fix this
+                    # ourselves
+                    qs = compat_urlparse.urlparse(video_url).query
+                    if qs:
+                        query = compat_urlparse.parse_qs(qs)
+                        for m3u8_format in m3u8_formats:
+                            m3u8_format['url'] = update_url_query(m3u8_format['url'], query)
+                            m3u8_format['extra_param_to_segment_url'] = qs
+                    formats.extend(m3u8_formats)
             elif ext == 'f4m':
                 formats.extend(self._extract_f4m_formats(
                     update_url_query(video_url, {'hdcore': '3.7.0'}),
diff --git a/youtube_dl/extractor/uplynk.py b/youtube_dl/extractor/uplynk.py
index ae529f690..2cd22cf8a 100644
--- a/youtube_dl/extractor/uplynk.py
+++ b/youtube_dl/extractor/uplynk.py
@@ -33,9 +33,7 @@ class UplynkIE(InfoExtractor):
         formats = self._extract_m3u8_formats('http://content.uplynk.com/%s.m3u8' % path, display_id, 'mp4')
         if session_id:
             for f in formats:
-                f['extra_param_to_segment_url'] = {
-                    'pbs': session_id,
-                }
+                f['extra_param_to_segment_url'] = 'pbs=' + session_id
         self._sort_formats(formats)
         asset = self._download_json('http://content.uplynk.com/player/assetinfo/%s.json' % path, display_id)
         if asset.get('error') == 1:

From 9ba1e1dcc0dc27d36f3f396cb608cef7cd50e48a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 29 Aug 2016 08:26:07 +0700
Subject: [PATCH 5921/6219] [played] Remove extractor (Closes #10470)

---
 youtube_dl/extractor/extractors.py |  1 -
 youtube_dl/extractor/played.py     | 60 ------------------------------
 2 files changed, 61 deletions(-)
 delete mode 100644 youtube_dl/extractor/played.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 06c6746ff..20e85703f 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -636,7 +636,6 @@ from .phoenix import PhoenixIE
 from .photobucket import PhotobucketIE
 from .pinkbike import PinkbikeIE
 from .pladform import PladformIE
-from .played import PlayedIE
 from .playfm import PlayFMIE
 from .plays import PlaysTVIE
 from .playtvak import PlaytvakIE
diff --git a/youtube_dl/extractor/played.py b/youtube_dl/extractor/played.py
deleted file mode 100644
index 57c875ef0..000000000
--- a/youtube_dl/extractor/played.py
+++ /dev/null
@@ -1,60 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-import re
-import os.path
-
-from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
-    sanitized_Request,
-    urlencode_postdata,
-)
-
-
-class PlayedIE(InfoExtractor):
-    IE_NAME = 'played.to'
-    _VALID_URL = r'https?://(?:www\.)?played\.to/(?P<id>[a-zA-Z0-9_-]+)'
-
-    _TEST = {
-        'url': 'http://played.to/j2f2sfiiukgt',
-        'md5': 'c2bd75a368e82980e7257bf500c00637',
-        'info_dict': {
-            'id': 'j2f2sfiiukgt',
-            'ext': 'flv',
-            'title': 'youtube-dl_test_video.mp4',
-        },
-        'skip': 'Removed for copyright infringement.',  # oh wow
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        orig_webpage = self._download_webpage(url, video_id)
-
-        m_error = re.search(
-            r'(?s)Reason for deletion:.*?<b class="err"[^>]*>(?P<msg>[^<]+)</b>', orig_webpage)
-        if m_error:
-            raise ExtractorError(m_error.group('msg'), expected=True)
-
-        data = self._hidden_inputs(orig_webpage)
-
-        self._sleep(2, video_id)
-
-        post = urlencode_postdata(data)
-        headers = {
-            b'Content-Type': b'application/x-www-form-urlencoded',
-        }
-        req = sanitized_Request(url, post, headers)
-        webpage = self._download_webpage(
-            req, video_id, note='Downloading video page ...')
-
-        title = os.path.splitext(data['fname'])[0]
-
-        video_url = self._search_regex(
-            r'file: "?(.+?)",', webpage, 'video URL')
-
-        return {
-            'id': video_id,
-            'title': title,
-            'url': video_url,
-        }

From 93b84045994ca88b486901f54de1102347a67537 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 29 Aug 2016 07:56:54 +0100
Subject: [PATCH 5922/6219] [generic,vodplatform] improve embed regex

---
 youtube_dl/extractor/generic.py     | 4 ++--
 youtube_dl/extractor/vodplatform.py | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index c6e655c84..24b217715 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -2243,11 +2243,11 @@ class GenericIE(InfoExtractor):
 
         # Look for VODPlatform embeds
         mobj = re.search(
-            r'<iframe[^>]+src=[\'"]((?:https?:)?//(?:www\.)?vod-platform\.net/embed/[^/?#]+)',
+            r'<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?vod-platform\.net/[eE]mbed/.+?)\1',
             webpage)
         if mobj is not None:
             return self.url_result(
-                self._proto_relative_url(unescapeHTML(mobj.group(1))), 'VODPlatform')
+                self._proto_relative_url(unescapeHTML(mobj.group('url'))), 'VODPlatform')
 
         # Look for Instagram embeds
         instagram_embed_url = InstagramIE._extract_embed_url(webpage)
diff --git a/youtube_dl/extractor/vodplatform.py b/youtube_dl/extractor/vodplatform.py
index b49542b16..7bdd8b1dc 100644
--- a/youtube_dl/extractor/vodplatform.py
+++ b/youtube_dl/extractor/vodplatform.py
@@ -6,7 +6,7 @@ from ..utils import unescapeHTML
 
 
 class VODPlatformIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?vod-platform\.net/embed/(?P<id>[^/?#]+)'
+    _VALID_URL = r'https?://(?:www\.)?vod-platform\.net/[eE]mbed/(?P<id>[^/?#]+)'
     _TEST = {
         # from http://www.lbcgroup.tv/watch/chapter/29143/52844/%D8%A7%D9%84%D9%86%D8%B5%D8%B1%D8%A9-%D9%81%D9%8A-%D8%B6%D9%8A%D8%A7%D9%81%D8%A9-%D8%A7%D9%84%D9%80-cnn/ar
         'url': 'http://vod-platform.net/embed/RufMcytHDolTH1MuKHY9Fw',

From 6c9b71bc0862560cbb9c4c2d9ec295072c208838 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 29 Aug 2016 19:05:38 +0800
Subject: [PATCH 5923/6219] [downloader/external] Recommend --hls-prefer-native
 for SOCKS users

Related: #10490
---
 youtube_dl/downloader/external.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index 17f12e970..0aeae3b8f 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -223,7 +223,8 @@ class FFmpegFD(ExternalFD):
 
             if proxy.startswith('socks'):
                 self.report_warning(
-                    '%s does not support SOCKS proxies. Downloading may fail.' % self.get_basename())
+                    '%s does not support SOCKS proxies. Downloading is likely to fail. '
+                    'Consider adding --hls-prefer-native to your command.' % self.get_basename())
 
             # Since December 2015 ffmpeg supports -http_proxy option (see
             # http://git.videolan.org/?p=ffmpeg.git;a=commit;h=b4eb1f29ebddd60c41a2eb39f5af701e38e0d3fd)

From 547993dcd09dd46fda2fd429ed0ed72db7263503 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 29 Aug 2016 21:52:41 +0700
Subject: [PATCH 5924/6219] [turner] Fix subtitles extraction

---
 youtube_dl/extractor/turner.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/turner.py b/youtube_dl/extractor/turner.py
index 108caa9d8..d69977b56 100644
--- a/youtube_dl/extractor/turner.py
+++ b/youtube_dl/extractor/turner.py
@@ -135,11 +135,12 @@ class TurnerBaseIE(InfoExtractor):
         subtitles = {}
         for source in video_data.findall('closedCaptions/source'):
             for track in source.findall('track'):
-                source_url = source.get('url')
-                if not source_url:
+                track_url = track.get('url')
+                if not track_url:
                     continue
-                subtitles.set_default(source.get('lang') or source.get('label') or 'en', []).append({
-                    'url': source_url,
+                lang = track.get('lang') or track.get('label') or 'en'
+                subtitles.setdefault(lang, []).append({
+                    'url': track_url,
                     'ext': {
                         'scc': 'scc',
                         'webvtt': 'vtt',

From cd10b3ea63fd167216234932aba4d63a34aec4c7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 29 Aug 2016 22:13:49 +0700
Subject: [PATCH 5925/6219] [turner] Extract all formats

---
 youtube_dl/extractor/turner.py | 46 ++++++++++++++++++----------------
 1 file changed, 25 insertions(+), 21 deletions(-)

diff --git a/youtube_dl/extractor/turner.py b/youtube_dl/extractor/turner.py
index d69977b56..6df22fd24 100644
--- a/youtube_dl/extractor/turner.py
+++ b/youtube_dl/extractor/turner.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import compat_str
 from ..utils import (
     xpath_text,
     int_or_none,
@@ -30,11 +31,11 @@ class TurnerBaseIE(InfoExtractor):
         tokens = {}
         urls = []
         formats = []
-        rex = re.compile(r'''(?x)
-            (?P<width>[0-9]+)x(?P<height>[0-9]+)
-            (?:_(?P<bitrate>[0-9]+))?
-        ''')
-        for video_file in video_data.findall('files/file'):
+        rex = re.compile(
+            r'(?P<width>[0-9]+)x(?P<height>[0-9]+)(?:_(?P<bitrate>[0-9]+))?')
+        # Possible formats locations: files/file, files/groupFiles/files
+        # and maybe others
+        for video_file in video_data.findall('.//file'):
             video_url = video_file.text.strip()
             if not video_url:
                 continue
@@ -84,12 +85,14 @@ class TurnerBaseIE(InfoExtractor):
             if video_url in urls:
                 continue
             urls.append(video_url)
-            format_id = video_file.attrib['bitrate']
+            format_id = video_file.get('bitrate')
             if ext == 'smil':
-                formats.extend(self._extract_smil_formats(video_url, video_id, fatal=False))
+                formats.extend(self._extract_smil_formats(
+                    video_url, video_id, fatal=False))
             elif ext == 'm3u8':
                 m3u8_formats = self._extract_m3u8_formats(
-                    video_url, video_id, 'mp4', m3u8_id=format_id, fatal=False)
+                    video_url, video_id, 'mp4', m3u8_id=format_id or 'hls',
+                    fatal=False)
                 if m3u8_formats:
                     # Sometimes final URLs inside m3u8 are unsigned, let's fix this
                     # ourselves
@@ -103,7 +106,7 @@ class TurnerBaseIE(InfoExtractor):
             elif ext == 'f4m':
                 formats.extend(self._extract_f4m_formats(
                     update_url_query(video_url, {'hdcore': '3.7.0'}),
-                    video_id, f4m_id=format_id, fatal=False))
+                    video_id, f4m_id=format_id or 'hds', fatal=False))
             else:
                 f = {
                     'format_id': format_id,
@@ -117,18 +120,19 @@ class TurnerBaseIE(InfoExtractor):
                         'height': int(mobj.group('height')),
                         'tbr': int_or_none(mobj.group('bitrate')),
                     })
-                elif format_id.isdigit():
-                    f['tbr'] = int(format_id)
-                else:
-                    mobj = re.match(r'ios_(audio|[0-9]+)$', format_id)
-                    if mobj:
-                        if mobj.group(1) == 'audio':
-                            f.update({
-                                'vcodec': 'none',
-                                'ext': 'm4a',
-                            })
-                        else:
-                            f['tbr'] = int(mobj.group(1))
+                elif isinstance(format_id, compat_str):
+                    if format_id.isdigit():
+                        f['tbr'] = int(format_id)
+                    else:
+                        mobj = re.match(r'ios_(audio|[0-9]+)$', format_id)
+                        if mobj:
+                            if mobj.group(1) == 'audio':
+                                f.update({
+                                    'vcodec': 'none',
+                                    'ext': 'm4a',
+                                })
+                            else:
+                                f['tbr'] = int(mobj.group(1))
                 formats.append(f)
         self._sort_formats(formats)
 

From 7be15d40976bf40f44bc47301d4e839a1e171e52 Mon Sep 17 00:00:00 2001
From: PeterDing <dfhayst@gmail.com>
Date: Fri, 29 Jul 2016 23:21:50 +0800
Subject: [PATCH 5926/6219] [bilibili] Support episodes

[extractor/bilibili] add md5 for testing

[extractor/bilibili] remove unnecessary headers

[extractor/bilibili] correct _TESTS; find thumbnail for episode

[extractor/bilibili] [Fix] restore removed tests
---
 youtube_dl/extractor/bilibili.py | 40 ++++++++++++++++++++++++++------
 1 file changed, 33 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/bilibili.py b/youtube_dl/extractor/bilibili.py
index a332fbb69..35313c62b 100644
--- a/youtube_dl/extractor/bilibili.py
+++ b/youtube_dl/extractor/bilibili.py
@@ -12,9 +12,13 @@ from ..utils import (
     unified_timestamp,
 )
 
+HEADERS = {
+    'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
+}
+
 
 class BiliBiliIE(InfoExtractor):
-    _VALID_URL = r'https?://www\.bilibili\.(?:tv|com)/video/av(?P<id>\d+)'
+    _VALID_URL = r'https?://(www.|bangumi.|)bilibili\.(?:tv|com)/(video/av|anime/v/)(?P<id>\d+)'
 
     _TESTS = [{
         'url': 'http://www.bilibili.tv/video/av1074402/',
@@ -77,6 +81,17 @@ class BiliBiliIE(InfoExtractor):
             'skip_download': True,
         },
         'expected_warnings': ['upload time'],
+    }, {
+        'url': 'http://bangumi.bilibili.com/anime/v/40068',
+        'md5': '08d539a0884f3deb7b698fb13ba69696',
+        'info_dict': {
+            'id': '40068',
+            'ext': 'mp4',
+            'duration': 1402.357,
+            'title': '混沌武士 : 第7集 四面楚歌 A Risky Racket',
+            'description': "故事发生在日本的江户时代。风是一个小酒馆的打工女。一日，酒馆里来了一群恶霸，虽然他们的举动令风十分不满，但是毕竟风只是一届女流，无法对他们采取什么行动，只能在心里嘟哝。这时，酒家里又进来了个“不良份子”无幻，说以50个丸子帮她搞定这群人，风觉得他莫名其妙，也就没多搭理他。而在这时，风因为一个意外而将茶水泼在了恶霸头领——龙次郎身上。愤怒的恶霸们欲将风的手指砍掉，风在无奈中大喊道：“丸子100个！”…… 　　另一方面，龙次郎的父亲也就是当地的代官，依仗自己有着雄厚的保镖实力，在当地欺压穷人，当看到一穷人无法交齐足够的钱过桥时，欲下令将其杀死，武士仁看不惯这一幕，于是走上前，与代官的保镖交手了…… 　　酒馆内，因为风答应给无幻100个团子，无幻将恶霸们打败了，就在这时，仁进来了。好战的无幻立刻向仁发了战书，最后两败俱伤，被代官抓入牢房，预计第二天斩首…… 　　得知该状况的风，为报救命之恩，来到了刑场，利用烟花救出了无幻和仁。而风则以救命恩人的身份，命令二人和她一起去寻找带着向日葵香味的武士……(by百科)",
+            'thumbnail': 're:^http?://.+\.jpg',
+        },
     }]
 
     _APP_KEY = '6f90a59ac58a4123'
@@ -84,13 +99,20 @@ class BiliBiliIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-
         webpage = self._download_webpage(url, video_id)
 
-        cid = compat_parse_qs(self._search_regex(
-            [r'EmbedPlayer\([^)]+,\s*"([^"]+)"\)',
-             r'<iframe[^>]+src="https://secure\.bilibili\.com/secure,([^"]+)"'],
-            webpage, 'player parameters'))['cid'][0]
+        _is_episode = 'anime/v' in url
+        if not _is_episode:
+            cid = compat_parse_qs(self._search_regex(
+                [r'EmbedPlayer\([^)]+,\s*"([^"]+)"\)',
+                r'<iframe[^>]+src="https://secure\.bilibili\.com/secure,([^"]+)"'],
+                webpage, 'player parameters'))['cid'][0]
+        else:
+            url_t = 'http://bangumi.bilibili.com/web_api/get_source'
+            js = self._download_json(url_t, video_id,
+                                     data='episode_id=%s' % video_id,
+                                     headers=HEADERS)
+            cid = js['result']['cid']
 
         payload = 'appkey=%s&cid=%s&otype=json&quality=2&type=mp4' % (self._APP_KEY, cid)
         sign = hashlib.md5((payload + self._BILIBILI_KEY).encode('utf-8')).hexdigest()
@@ -125,6 +147,10 @@ class BiliBiliIE(InfoExtractor):
         description = self._html_search_meta('description', webpage)
         timestamp = unified_timestamp(self._html_search_regex(
             r'<time[^>]+datetime="([^"]+)"', webpage, 'upload time', fatal=False))
+        if _is_episode:
+            thumbnail = self._html_search_meta('og:image', webpage)
+        else:
+            thumbnail = self._html_search_meta('thumbnailUrl', webpage)
 
         # TODO 'view_count' requires deobfuscating Javascript
         info = {
@@ -132,7 +158,7 @@ class BiliBiliIE(InfoExtractor):
             'title': title,
             'description': description,
             'timestamp': timestamp,
-            'thumbnail': self._html_search_meta('thumbnailUrl', webpage),
+            'thumbnail': thumbnail,
             'duration': float_or_none(video_info.get('timelength'), scale=1000),
         }
 

From 3fb2a23029934dcbf6fe2cd283d851506dcdff5e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 29 Aug 2016 22:40:35 +0700
Subject: [PATCH 5927/6219] [adultswim] Extract video info from onlineOriginals
 (Closes #10492)

---
 youtube_dl/extractor/adultswim.py | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/adultswim.py b/youtube_dl/extractor/adultswim.py
index ef3cc2a61..5d0bf5a68 100644
--- a/youtube_dl/extractor/adultswim.py
+++ b/youtube_dl/extractor/adultswim.py
@@ -4,7 +4,10 @@ from __future__ import unicode_literals
 import re
 
 from .turner import TurnerBaseIE
-from ..utils import ExtractorError
+from ..utils import (
+    ExtractorError,
+    int_or_none,
+)
 
 
 class AdultSwimIE(TurnerBaseIE):
@@ -144,7 +147,10 @@ class AdultSwimIE(TurnerBaseIE):
                 if bootstrapped_data.get('slugged_video', {}).get('slug') == episode_path:
                     video_info = bootstrapped_data['slugged_video']
             if not video_info:
-                video_info = bootstrapped_data.get('heroMetadata', {}).get('trailer').get('video')
+                video_info = bootstrapped_data.get(
+                    'heroMetadata', {}).get('trailer', {}).get('video')
+            if not video_info:
+                video_info = bootstrapped_data.get('onlineOriginals', [None])[0]
             if not video_info:
                 raise ExtractorError('Unable to find video info')
 
@@ -167,8 +173,9 @@ class AdultSwimIE(TurnerBaseIE):
 
         episode_id = video_info['id']
         episode_title = video_info['title']
-        episode_description = video_info['description']
-        episode_duration = video_info.get('duration')
+        episode_description = video_info.get('description')
+        episode_duration = int_or_none(video_info.get('duration'))
+        view_count = int_or_none(video_info.get('views'))
 
         entries = []
         for part_num, segment_id in enumerate(segment_ids):
@@ -197,5 +204,6 @@ class AdultSwimIE(TurnerBaseIE):
             'entries': entries,
             'title': '%s - %s' % (show_title, episode_title),
             'description': episode_description,
-            'duration': episode_duration
+            'duration': episode_duration,
+            'view_count': view_count,
         }

From 5a80e7b43a7abc83e104f1cd711d8fe7985c30eb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 29 Aug 2016 22:44:15 +0700
Subject: [PATCH 5928/6219] [turner] Skip invalid subtitles' URLs

---
 youtube_dl/extractor/turner.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/turner.py b/youtube_dl/extractor/turner.py
index 6df22fd24..f5736bd15 100644
--- a/youtube_dl/extractor/turner.py
+++ b/youtube_dl/extractor/turner.py
@@ -140,7 +140,7 @@ class TurnerBaseIE(InfoExtractor):
         for source in video_data.findall('closedCaptions/source'):
             for track in source.findall('track'):
                 track_url = track.get('url')
-                if not track_url:
+                if not isinstance(track_url, compat_str) or track_url.endswith('/big'):
                     continue
                 lang = track.get('lang') or track.get('label') or 'en'
                 subtitles.setdefault(lang, []).append({

From a06e1498aa7fc02e6db5c6ec8411e90f210ce2c3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 29 Aug 2016 22:54:33 +0700
Subject: [PATCH 5929/6219] [kusi] Update test

---
 youtube_dl/extractor/kusi.py | 27 ++++++++-------------------
 1 file changed, 8 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/extractor/kusi.py b/youtube_dl/extractor/kusi.py
index 12cc56e44..2e66e8cf9 100644
--- a/youtube_dl/extractor/kusi.py
+++ b/youtube_dl/extractor/kusi.py
@@ -18,31 +18,20 @@ from ..utils import (
 class KUSIIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?kusi\.com/(?P<path>story/.+|video\?clipId=(?P<clipId>\d+))'
     _TESTS = [{
-        'url': 'http://www.kusi.com/story/31183873/turko-files-case-closed-put-on-hold',
-        'md5': 'f926e7684294cf8cb7bdf8858e1b3988',
+        'url': 'http://www.kusi.com/story/32849881/turko-files-refused-to-help-it-aint-right',
+        'md5': '4e76ce8e53660ce9697d06c0ba6fc47d',
         'info_dict': {
-            'id': '12203019',
+            'id': '12689020',
             'ext': 'mp4',
-            'title': 'Turko Files: Case Closed! & Put On Hold!',
-            'duration': 231.0,
-            'upload_date': '20160210',
-            'timestamp': 1455087571,
+            'title': "Turko Files: Refused to Help, It Ain't Right!",
+            'duration': 223.586,
+            'upload_date': '20160826',
+            'timestamp': 1472233118,
             'thumbnail': 're:^https?://.*\.jpg$'
         },
     }, {
         'url': 'http://kusi.com/video?clipId=12203019',
-        'info_dict': {
-            'id': '12203019',
-            'ext': 'mp4',
-            'title': 'Turko Files: Case Closed! & Put On Hold!',
-            'duration': 231.0,
-            'upload_date': '20160210',
-            'timestamp': 1455087571,
-            'thumbnail': 're:^https?://.*\.jpg$'
-        },
-        'params': {
-            'skip_download': True,  # Same as previous one
-        },
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From fe45b0e06081752ff3617cdfae701408a1d8256a Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 29 Aug 2016 18:17:32 +0100
Subject: [PATCH 5930/6219] [9c9media] fix multiple stacks extraction and
 extract more metadata(#10016)

---
 youtube_dl/extractor/extractors.py     |   5 +-
 youtube_dl/extractor/ninecninemedia.py | 126 +++++++++++++++++++------
 2 files changed, 103 insertions(+), 28 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 20e85703f..21efa96b2 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -554,7 +554,10 @@ from .nick import (
     NickDeIE,
 )
 from .niconico import NiconicoIE, NiconicoPlaylistIE
-from .ninecninemedia import NineCNineMediaIE
+from .ninecninemedia import (
+    NineCNineMediaStackIE,
+    NineCNineMediaIE,
+)
 from .ninegag import NineGagIE
 from .ninenow import NineNowIE
 from .nintendo import NintendoIE
diff --git a/youtube_dl/extractor/ninecninemedia.py b/youtube_dl/extractor/ninecninemedia.py
index d889245ad..ec4d675e2 100644
--- a/youtube_dl/extractor/ninecninemedia.py
+++ b/youtube_dl/extractor/ninecninemedia.py
@@ -4,40 +4,36 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import compat_str
 from ..utils import (
     parse_iso8601,
-    parse_duration,
-    ExtractorError
+    float_or_none,
+    ExtractorError,
+    int_or_none,
 )
 
 
-class NineCNineMediaIE(InfoExtractor):
-    _VALID_URL = r'9c9media:(?P<destination_code>[^:]+):(?P<id>\d+)'
+class NineCNineMediaBaseIE(InfoExtractor):
+    _API_BASE_TEMPLATE = 'http://capi.9c9media.com/destinations/%s/platforms/desktop/contents/%s/'
+
+
+class NineCNineMediaStackIE(NineCNineMediaBaseIE):
+    IE_NAME = '9c9media:stack'
+    _VALID_URL = r'9c9media:stack:(?P<destination_code>[^:]+):(?P<content_id>\d+):(?P<content_package>\d+):(?P<id>\d+)'
 
     def _real_extract(self, url):
-        destination_code, video_id = re.match(self._VALID_URL, url).groups()
-        api_base_url = 'http://capi.9c9media.com/destinations/%s/platforms/desktop/contents/%s/' % (destination_code, video_id)
-        content = self._download_json(api_base_url, video_id, query={
-            '$include': '[contentpackages]',
-        })
-        title = content['Name']
-        if len(content['ContentPackages']) > 1:
-            raise ExtractorError('multiple content packages')
-        content_package = content['ContentPackages'][0]
-        stacks_base_url = api_base_url + 'contentpackages/%s/stacks/' % content_package['Id']
-        stacks = self._download_json(stacks_base_url, video_id)['Items']
-        if len(stacks) > 1:
-            raise ExtractorError('multiple stacks')
-        stack = stacks[0]
-        stack_base_url = '%s%s/manifest.' % (stacks_base_url, stack['Id'])
+        destination_code, content_id, package_id, stack_id = re.match(self._VALID_URL, url).groups()
+        stack_base_url_template = self._API_BASE_TEMPLATE + 'contentpackages/%s/stacks/%s/manifest.'
+        stack_base_url = stack_base_url_template % (destination_code, content_id, package_id, stack_id)
+
         formats = []
         formats.extend(self._extract_m3u8_formats(
-            stack_base_url + 'm3u8', video_id, 'mp4',
+            stack_base_url + 'm3u8', stack_id, 'mp4',
             'm3u8_native', m3u8_id='hls', fatal=False))
         formats.extend(self._extract_f4m_formats(
-            stack_base_url + 'f4m', video_id,
+            stack_base_url + 'f4m', stack_id,
             f4m_id='hds', fatal=False))
-        mp4_url = self._download_webpage(stack_base_url + 'pd', video_id, fatal=False)
+        mp4_url = self._download_webpage(stack_base_url + 'pd', stack_id, fatal=False)
         if mp4_url:
             formats.append({
                 'url': mp4_url,
@@ -46,10 +42,86 @@ class NineCNineMediaIE(InfoExtractor):
         self._sort_formats(formats)
 
         return {
-            'id': video_id,
-            'title': title,
-            'description': content.get('Desc') or content.get('ShortDesc'),
-            'timestamp': parse_iso8601(content.get('BroadcastDateTime')),
-            'duration': parse_duration(content.get('BroadcastTime')),
+            'id': stack_id,
             'formats': formats,
         }
+
+
+class NineCNineMediaIE(NineCNineMediaBaseIE):
+    IE_NAME = '9c9media'
+    _VALID_URL = r'9c9media:(?P<destination_code>[^:]+):(?P<id>\d+)'
+
+    def _real_extract(self, url):
+        destination_code, content_id = re.match(self._VALID_URL, url).groups()
+        api_base_url = self._API_BASE_TEMPLATE % (destination_code, content_id)
+        content = self._download_json(api_base_url, content_id, query={
+            '$include': '[Media,Season,ContentPackages]',
+        })
+        title = content['Name']
+        if len(content['ContentPackages']) > 1:
+            raise ExtractorError('multiple content packages')
+        content_package = content['ContentPackages'][0]
+        package_id = content_package['Id']
+        content_package_url = api_base_url + 'contentpackages/%s/' % package_id
+        content_package = self._download_json(content_package_url, content_id)
+
+        if content_package.get('Constraints', {}).get('Security', {}).get('Type') == 'adobe-drm':
+            raise ExtractorError('This video is DRM protected.', expected=True)
+
+        stacks = self._download_json(content_package_url + 'stacks/', package_id)['Items']
+        multistacks = len(stacks) > 1
+
+        thumbnails = []
+        for image in content.get('Images', []):
+            image_url = image.get('Url')
+            if not image_url:
+                continue
+            thumbnails.append({
+                'url': image_url,
+                'width': int_or_none(image.get('Width')),
+                'height': int_or_none(image.get('Height')),
+            })
+
+        tags, categories = [], []
+        for source_name, container in (('Tags', tags), ('Genres', categories)):
+            for e in content.get(source_name, []):
+                e_name = e.get('Name')
+                if not e_name:
+                    continue
+                container.append(e_name)
+
+        description = content.get('Desc') or content.get('ShortDesc')
+        season = content.get('Season', {})
+        base_info = {
+            'description': description,
+            'timestamp': parse_iso8601(content.get('BroadcastDateTime')),
+            'episode_number': int_or_none(content.get('Episode')),
+            'season': season.get('Name'),
+            'season_number': season.get('Number'),
+            'season_id': season.get('Id'),
+            'series': content.get('Media', {}).get('Name'),
+            'tags': tags,
+            'categories': categories,
+        }
+
+        entries = []
+        for stack in stacks:
+            stack_id = compat_str(stack['Id'])
+            entry = {
+                '_type': 'url_transparent',
+                'url': '9c9media:stack:%s:%s:%s:%s' % (destination_code, content_id, package_id, stack_id),
+                'id': stack_id,
+                'title': '%s_part%s' % (title, stack['Name']) if multistacks else title,
+                'duration': float_or_none(stack.get('Duration')),
+                'ie_key': 'NineCNineMediaStack',
+            }
+            entry.update(base_info)
+            entries.append(entry)
+
+        return {
+            '_type': 'multi_video',
+            'id': content_id,
+            'title': title,
+            'description': description,
+            'entries': entries,
+        }

From 42e05be8671e149f79307145eda78892003279dc Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 29 Aug 2016 18:20:58 +0100
Subject: [PATCH 5931/6219] [ctv] add support for (tsn,bnn,thecomedynetwork).ca
 websites(#10016)

---
 youtube_dl/extractor/ctv.py | 22 +++++++++++++++++++---
 1 file changed, 19 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/ctv.py b/youtube_dl/extractor/ctv.py
index 5807fbac9..a1fe86316 100644
--- a/youtube_dl/extractor/ctv.py
+++ b/youtube_dl/extractor/ctv.py
@@ -1,11 +1,13 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 
 
 class CTVIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?ctv\.ca/video/player\?vid=(?P<id>[0-9.]+)'
+    _VALID_URL = r'https?://(?:www\.)?(?P<domain>ctv|tsn|bnn|thecomedynetwork)\.ca/.*?(?:\bvid=|-vid|~|%7E)(?P<id>[0-9.]+)'
     _TESTS = [{
         'url': 'http://www.ctv.ca/video/player?vid=706966',
         'md5': 'ff2ebbeae0aa2dcc32a830c3fd69b7b0',
@@ -18,13 +20,27 @@ class CTVIE(InfoExtractor):
             'timestamp': 1442624700,
         },
         'expected_warnings': ['HTTP Error 404'],
+    }, {
+        'url': 'http://www.thecomedynetwork.ca/video/player?vid=923582',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.tsn.ca/video/expectations-high-for-milos-raonic-at-us-open~939549',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.bnn.ca/video/berman-s-call-part-two-viewer-questions~939654',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.ctv.ca/YourMorning/Video/S1E6-Monday-August-29-2016-vid938009',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        domain, video_id = re.match(self._VALID_URL, url).groups()
+        if domain == 'thecomedynetwork':
+            domain = 'comedy'
         return {
             '_type': 'url_transparent',
             'id': video_id,
-            'url': '9c9media:ctv_web:%s' % video_id,
+            'url': '9c9media:%s_web:%s' % (domain, video_id),
             'ie_key': 'NineCNineMedia',
         }

From 1fe48afea5f203cbcb29c0d2984b7b850df8103f Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 29 Aug 2016 18:23:21 +0100
Subject: [PATCH 5932/6219] [cnn] update _TEST for CNNBlogsIE and
 CNNArticleIE(closes #10489)

---
 youtube_dl/extractor/cnn.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/cnn.py b/youtube_dl/extractor/cnn.py
index 1bf87f6ea..bb42f35bd 100644
--- a/youtube_dl/extractor/cnn.py
+++ b/youtube_dl/extractor/cnn.py
@@ -102,12 +102,13 @@ class CNNBlogsIE(InfoExtractor):
         'url': 'http://reliablesources.blogs.cnn.com/2014/02/09/criminalizing-journalism/',
         'md5': '3e56f97b0b6ffb4b79f4ea0749551084',
         'info_dict': {
-            'id': 'bestoftv/2014/02/09/criminalizing-journalism.cnn',
+            'id': 'criminalizing-journalism',
             'ext': 'mp4',
             'title': 'Criminalizing journalism?',
             'description': 'Glenn Greenwald responds to comments made this week on Capitol Hill that journalists could be criminal accessories.',
             'upload_date': '20140209',
         },
+        'expected_warnings': ['Failed to download m3u8 information'],
         'add_ie': ['CNN'],
     }
 
@@ -127,12 +128,13 @@ class CNNArticleIE(InfoExtractor):
         'url': 'http://www.cnn.com/2014/12/21/politics/obama-north-koreas-hack-not-war-but-cyber-vandalism/',
         'md5': '689034c2a3d9c6dc4aa72d65a81efd01',
         'info_dict': {
-            'id': 'bestoftv/2014/12/21/ip-north-korea-obama.cnn',
+            'id': 'ip-north-korea-obama',
             'ext': 'mp4',
             'title': 'Obama: Cyberattack not an act of war',
-            'description': 'md5:51ce6750450603795cad0cdfbd7d05c5',
+            'description': 'md5:0a802a40d2376f60e6b04c8d5bcebc4b',
             'upload_date': '20141221',
         },
+        'expected_warnings': ['Failed to download m3u8 information'],
         'add_ie': ['CNN'],
     }
 

From da30a20a4d8b0ece61c271a5d0f0c6de2817ef5f Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 29 Aug 2016 19:26:53 +0100
Subject: [PATCH 5933/6219] [turner,cnn] move a check for wrong timestamp to
 CNNIE

---
 youtube_dl/extractor/cnn.py    | 4 ++++
 youtube_dl/extractor/turner.py | 9 ++++-----
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/cnn.py b/youtube_dl/extractor/cnn.py
index bb42f35bd..a51b239cc 100644
--- a/youtube_dl/extractor/cnn.py
+++ b/youtube_dl/extractor/cnn.py
@@ -83,6 +83,10 @@ class CNNIE(TurnerBaseIE):
         },
     }
 
+    def _extract_timestamp(self, video_data):
+        # TODO: fix timestamp extraction
+        return None
+
     def _real_extract(self, url):
         sub_domain, path, page_title = re.match(self._VALID_URL, url).groups()
         if sub_domain not in ('money', 'edition'):
diff --git a/youtube_dl/extractor/turner.py b/youtube_dl/extractor/turner.py
index f5736bd15..64fdcc56e 100644
--- a/youtube_dl/extractor/turner.py
+++ b/youtube_dl/extractor/turner.py
@@ -17,6 +17,9 @@ from ..utils import (
 
 
 class TurnerBaseIE(InfoExtractor):
+    def _extract_timestamp(self, video_data):
+        return int_or_none(xpath_attr(video_data, 'dateCreated', 'uts'))
+
     def _extract_cvp_info(self, data_src, video_id, path_data={}):
         video_data = self._download_xml(data_src, video_id)
         video_id = video_data.attrib['id'].split('/')[-1].split('.')[0]
@@ -159,10 +162,6 @@ class TurnerBaseIE(InfoExtractor):
             'height': int_or_none(image.get('height')),
         } for image in video_data.findall('images/image')]
 
-        timestamp = None
-        if 'cnn.com' not in data_src:
-            timestamp = int_or_none(xpath_attr(video_data, 'dateCreated', 'uts'))
-
         return {
             'id': video_id,
             'title': title,
@@ -171,7 +170,7 @@ class TurnerBaseIE(InfoExtractor):
             'thumbnails': thumbnails,
             'description': xpath_text(video_data, 'description'),
             'duration': parse_duration(xpath_text(video_data, 'length') or xpath_text(video_data, 'trt')),
-            'timestamp': timestamp,
+            'timestamp': self._extract_timestamp(video_data),
             'upload_date': xpath_attr(video_data, 'metas', 'version'),
             'series': xpath_text(video_data, 'showTitle'),
             'season_number': int_or_none(xpath_text(video_data, 'seasonNumber')),

From 3c77a54d5dfa1097d5e3a5eaa0c631b5b01e93ce Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 30 Aug 2016 10:46:48 +0100
Subject: [PATCH 5934/6219] [turner] keep video id intact

---
 youtube_dl/extractor/cnn.py    | 12 ++++++------
 youtube_dl/extractor/nba.py    | 11 +++++++----
 youtube_dl/extractor/turner.py |  2 +-
 3 files changed, 14 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/cnn.py b/youtube_dl/extractor/cnn.py
index a51b239cc..5fc311f53 100644
--- a/youtube_dl/extractor/cnn.py
+++ b/youtube_dl/extractor/cnn.py
@@ -15,7 +15,7 @@ class CNNIE(TurnerBaseIE):
         'url': 'http://edition.cnn.com/video/?/video/sports/2013/06/09/nadal-1-on-1.cnn',
         'md5': '3e6121ea48df7e2259fe73a0628605c4',
         'info_dict': {
-            'id': 'nadal-1-on-1',
+            'id': 'sports/2013/06/09/nadal-1-on-1.cnn',
             'ext': 'mp4',
             'title': 'Nadal wins 8th French Open title',
             'description': 'World Sport\'s Amanda Davies chats with 2013 French Open champion Rafael Nadal.',
@@ -27,7 +27,7 @@ class CNNIE(TurnerBaseIE):
         'url': 'http://edition.cnn.com/video/?/video/us/2013/08/21/sot-student-gives-epic-speech.georgia-institute-of-technology&utm_source=feedburner&utm_medium=feed&utm_campaign=Feed%3A+rss%2Fcnn_topstories+%28RSS%3A+Top+Stories%29',
         'md5': 'b5cc60c60a3477d185af8f19a2a26f4e',
         'info_dict': {
-            'id': 'sot-student-gives-epic-speech',
+            'id': 'us/2013/08/21/sot-student-gives-epic-speech.georgia-institute-of-technology',
             'ext': 'mp4',
             'title': "Student's epic speech stuns new freshmen",
             'description': "A Georgia Tech student welcomes the incoming freshmen with an epic speech backed by music from \"2001: A Space Odyssey.\"",
@@ -38,7 +38,7 @@ class CNNIE(TurnerBaseIE):
         'url': 'http://www.cnn.com/video/data/2.0/video/living/2014/12/22/growing-america-nashville-salemtown-board-episode-1.hln.html',
         'md5': 'f14d02ebd264df951feb2400e2c25a1b',
         'info_dict': {
-            'id': 'growing-america-nashville-salemtown-board-episode-1',
+            'id': 'living/2014/12/22/growing-america-nashville-salemtown-board-episode-1.hln',
             'ext': 'mp4',
             'title': 'Nashville Ep. 1: Hand crafted skateboards',
             'description': 'md5:e7223a503315c9f150acac52e76de086',
@@ -49,7 +49,7 @@ class CNNIE(TurnerBaseIE):
         'url': 'http://money.cnn.com/video/news/2016/08/19/netflix-stunning-stats.cnnmoney/index.html',
         'md5': '52a515dc1b0f001cd82e4ceda32be9d1',
         'info_dict': {
-            'id': 'netflix-stunning-stats',
+            'id': '/video/news/2016/08/19/netflix-stunning-stats.cnnmoney',
             'ext': 'mp4',
             'title': '5 stunning stats about Netflix',
             'description': 'Did you know that Netflix has more than 80 million members? Here are five facts about the online video distributor that you probably didn\'t know.',
@@ -106,7 +106,7 @@ class CNNBlogsIE(InfoExtractor):
         'url': 'http://reliablesources.blogs.cnn.com/2014/02/09/criminalizing-journalism/',
         'md5': '3e56f97b0b6ffb4b79f4ea0749551084',
         'info_dict': {
-            'id': 'criminalizing-journalism',
+            'id': 'bestoftv/2014/02/09/criminalizing-journalism.cnn',
             'ext': 'mp4',
             'title': 'Criminalizing journalism?',
             'description': 'Glenn Greenwald responds to comments made this week on Capitol Hill that journalists could be criminal accessories.',
@@ -132,7 +132,7 @@ class CNNArticleIE(InfoExtractor):
         'url': 'http://www.cnn.com/2014/12/21/politics/obama-north-koreas-hack-not-war-but-cyber-vandalism/',
         'md5': '689034c2a3d9c6dc4aa72d65a81efd01',
         'info_dict': {
-            'id': 'ip-north-korea-obama',
+            'id': 'bestoftv/2014/12/21/ip-north-korea-obama.cnn',
             'ext': 'mp4',
             'title': 'Obama: Cyberattack not an act of war',
             'description': 'md5:0a802a40d2376f60e6b04c8d5bcebc4b',
diff --git a/youtube_dl/extractor/nba.py b/youtube_dl/extractor/nba.py
index aabd5b670..53561961c 100644
--- a/youtube_dl/extractor/nba.py
+++ b/youtube_dl/extractor/nba.py
@@ -39,18 +39,19 @@ class NBAIE(TurnerBaseIE):
         'url': 'http://watch.nba.com/video/channels/playoffs/2015/05/20/0041400301-cle-atl-recap.nba',
         'md5': 'b2b39b81cf28615ae0c3360a3f9668c4',
         'info_dict': {
-            'id': '0041400301-cle-atl-recap',
+            'id': 'channels/playoffs/2015/05/20/0041400301-cle-atl-recap.nba',
             'ext': 'mp4',
             'title': 'Hawks vs. Cavaliers Game 1',
             'description': 'md5:8094c3498d35a9bd6b1a8c396a071b4d',
             'duration': 228,
             'timestamp': 1432134543,
             'upload_date': '20150520',
-        }
+        },
+        'expected_warnings': ['Unable to download f4m manifest'],
     }, {
         'url': 'http://www.nba.com/clippers/news/doc-rivers-were-not-trading-blake',
         'info_dict': {
-            'id': '1455672027478-Doc_Feb16_720',
+            'id': 'teams/clippers/2016/02/17/1455672027478-Doc_Feb16_720.mov-297324',
             'ext': 'mp4',
             'title': 'Practice: Doc Rivers - 2/16/16',
             'description': 'Head Coach Doc Rivers addresses the media following practice.',
@@ -61,6 +62,7 @@ class NBAIE(TurnerBaseIE):
             # m3u8 download
             'skip_download': True,
         },
+        'expected_warnings': ['Unable to download f4m manifest'],
     }, {
         'url': 'http://www.nba.com/timberwolves/wiggins-shootaround#',
         'info_dict': {
@@ -75,7 +77,7 @@ class NBAIE(TurnerBaseIE):
     }, {
         'url': 'http://www.nba.com/timberwolves/wiggins-shootaround#',
         'info_dict': {
-            'id': 'Wigginsmp4-3462601',
+            'id': 'teams/timberwolves/2014/12/12/Wigginsmp4-3462601',
             'ext': 'mp4',
             'title': 'Shootaround Access - Dec. 12 | Andrew Wiggins',
             'description': 'Wolves rookie Andrew Wiggins addresses the media after Friday\'s shootaround.',
@@ -87,6 +89,7 @@ class NBAIE(TurnerBaseIE):
             # m3u8 download
             'skip_download': True,
         },
+        'expected_warnings': ['Unable to download f4m manifest'],
     }]
 
     _PAGE_SIZE = 30
diff --git a/youtube_dl/extractor/turner.py b/youtube_dl/extractor/turner.py
index 64fdcc56e..b59dafda6 100644
--- a/youtube_dl/extractor/turner.py
+++ b/youtube_dl/extractor/turner.py
@@ -22,7 +22,7 @@ class TurnerBaseIE(InfoExtractor):
 
     def _extract_cvp_info(self, data_src, video_id, path_data={}):
         video_data = self._download_xml(data_src, video_id)
-        video_id = video_data.attrib['id'].split('/')[-1].split('.')[0]
+        video_id = video_data.attrib['id']
         title = xpath_text(video_data, 'headline', fatal=True)
         # rtmp_src = xpath_text(video_data, 'akamai/src')
         # if rtmp_src:

From 245023a86145f7074dacdab4c735dea268d766ce Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 30 Aug 2016 23:51:18 +0700
Subject: [PATCH 5935/6219] [pyvideo] Fix extraction (Closes #10468)

---
 youtube_dl/extractor/pyvideo.py | 94 +++++++++++++++++++--------------
 1 file changed, 54 insertions(+), 40 deletions(-)

diff --git a/youtube_dl/extractor/pyvideo.py b/youtube_dl/extractor/pyvideo.py
index cc0416cb8..08ec09183 100644
--- a/youtube_dl/extractor/pyvideo.py
+++ b/youtube_dl/extractor/pyvideo.py
@@ -1,59 +1,73 @@
 from __future__ import unicode_literals
 
 import re
-import os
 
 from .common import InfoExtractor
+from ..compat import compat_str
+from ..utils import int_or_none
 
 
 class PyvideoIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?pyvideo\.org/video/(?P<id>\d+)/(.*)'
+    _VALID_URL = r'https?://(?:www\.)?pyvideo\.org/(?P<category>[^/]+)/(?P<id>[^/?#&.]+)'
 
-    _TESTS = [
-        {
-            'url': 'http://pyvideo.org/video/1737/become-a-logging-expert-in-30-minutes',
-            'md5': '520915673e53a5c5d487c36e0c4d85b5',
-            'info_dict': {
-                'id': '24_4WWkSmNo',
-                'ext': 'webm',
-                'title': 'Become a logging expert in 30 minutes',
-                'description': 'md5:9665350d466c67fb5b1598de379021f7',
-                'upload_date': '20130320',
-                'uploader': 'Next Day Video',
-                'uploader_id': 'NextDayVideo',
-            },
-            'add_ie': ['Youtube'],
+    _TESTS = [{
+        'url': 'http://pyvideo.org/pycon-us-2013/become-a-logging-expert-in-30-minutes.html',
+        'info_dict': {
+            'id': 'become-a-logging-expert-in-30-minutes',
         },
-        {
-            'url': 'http://pyvideo.org/video/2542/gloriajw-spotifywitherikbernhardsson182m4v',
-            'md5': '5fe1c7e0a8aa5570330784c847ff6d12',
-            'info_dict': {
-                'id': '2542',
-                'ext': 'm4v',
-                'title': 'Gloriajw-SpotifyWithErikBernhardsson182',
-            },
+        'playlist_count': 2,
+    }, {
+        'url': 'http://pyvideo.org/pygotham-2012/gloriajw-spotifywitherikbernhardsson182m4v.html',
+        'md5': '5fe1c7e0a8aa5570330784c847ff6d12',
+        'info_dict': {
+            'id': '2542',
+            'ext': 'm4v',
+            'title': 'Gloriajw-SpotifyWithErikBernhardsson182.m4v',
         },
-    ]
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
+        category = mobj.group('category')
         video_id = mobj.group('id')
 
-        webpage = self._download_webpage(url, video_id)
+        entries = []
 
-        m_youtube = re.search(r'(https?://www\.youtube\.com/watch\?v=.*)', webpage)
-        if m_youtube is not None:
-            return self.url_result(m_youtube.group(1), 'Youtube')
+        data = self._download_json(
+            'https://raw.githubusercontent.com/pyvideo/data/master/%s/videos/%s.json'
+            % (category, video_id), video_id, fatal=False)
 
-        title = self._html_search_regex(
-            r'<div class="section">\s*<h3(?:\s+class="[^"]*"[^>]*)?>([^>]+?)</h3>',
-            webpage, 'title', flags=re.DOTALL)
-        video_url = self._search_regex(
-            [r'<source src="(.*?)"', r'<dt>Download</dt>.*?<a href="(.+?)"'],
-            webpage, 'video url', flags=re.DOTALL)
+        if data:
+            print(data)
+            for video in data['videos']:
+                video_url = video.get('url')
+                if video_url:
+                    if video.get('type') == 'youtube':
+                        entries.append(self.url_result(video_url, 'Youtube'))
+                    else:
+                        entries.append({
+                            'id': compat_str(data.get('id') or video_id),
+                            'url': video_url,
+                            'title': data['title'],
+                            'description': data.get('description') or data.get('summary'),
+                            'thumbnail': data.get('thumbnail_url'),
+                            'duration': int_or_none(data.get('duration')),
+                        })
+        else:
+            webpage = self._download_webpage(url, video_id)
+            title = self._og_search_title(webpage)
+            media_urls = self._search_regex(
+                r'(?s)Media URL:(.+?)</li>', webpage, 'media urls')
+            for m in re.finditer(
+                    r'<a[^>]+href=(["\'])(?P<url>http.+?)\1', media_urls):
+                media_url = m.group('url')
+                if re.match(r'https?://www\.youtube\.com/watch\?v=.*', media_url):
+                    entries.append(self.url_result(media_url, 'Youtube'))
+                else:
+                    entries.append({
+                        'id': video_id,
+                        'url': media_url,
+                        'title': title,
+                    })
 
-        return {
-            'id': video_id,
-            'title': os.path.splitext(title)[0],
-            'url': video_url,
-        }
+        return self.playlist_result(entries, video_id)

From 64fc49aba018ebd51627ddcc92f8fa88f2c499cc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 31 Aug 2016 00:29:49 +0700
Subject: [PATCH 5936/6219] [bandcamp:album] Fix title extraction (Closes
 #10455)

---
 youtube_dl/extractor/bandcamp.py | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/bandcamp.py b/youtube_dl/extractor/bandcamp.py
index 991ab0676..249c3d956 100644
--- a/youtube_dl/extractor/bandcamp.py
+++ b/youtube_dl/extractor/bandcamp.py
@@ -162,6 +162,15 @@ class BandcampAlbumIE(InfoExtractor):
             'uploader_id': 'dotscale',
         },
         'playlist_mincount': 7,
+    }, {
+        # with escaped quote in title
+        'url': 'https://jstrecords.bandcamp.com/album/entropy-ep',
+        'info_dict': {
+            'title': '"Entropy" EP',
+            'uploader_id': 'jstrecords',
+            'id': 'entropy-ep',
+        },
+        'playlist_mincount': 3,
     }]
 
     def _real_extract(self, url):
@@ -176,8 +185,11 @@ class BandcampAlbumIE(InfoExtractor):
         entries = [
             self.url_result(compat_urlparse.urljoin(url, t_path), ie=BandcampIE.ie_key())
             for t_path in tracks_paths]
-        title = self._search_regex(
-            r'album_title\s*:\s*"(.*?)"', webpage, 'title', fatal=False)
+        title = self._html_search_regex(
+            r'album_title\s*:\s*"((?:\\.|[^"\\])+?)"',
+            webpage, 'title', fatal=False)
+        if title:
+            title = title.replace(r'\"', '"')
         return {
             '_type': 'playlist',
             'uploader_id': uploader_id,

From f7043ef39cb73f8501d18d2e1f93997357397ba2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 31 Aug 2016 01:56:15 +0700
Subject: [PATCH 5937/6219] [soundcloud] Fix _VALID_URL clashes with sets
 (Closes #10505)

---
 youtube_dl/extractor/soundcloud.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index aeae931a2..9635c2b49 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -32,7 +32,7 @@ class SoundcloudIE(InfoExtractor):
     _VALID_URL = r'''(?x)^(?:https?://)?
                     (?:(?:(?:www\.|m\.)?soundcloud\.com/
                             (?P<uploader>[\w\d-]+)/
-                            (?!(?:tracks|sets(?:/[^/?#]+)?|reposts|likes|spotlight)/?(?:$|[?#]))
+                            (?!(?:tracks|sets(?:/.+?)?|reposts|likes|spotlight)/?(?:$|[?#]))
                             (?P<title>[\w\d-]+)/?
                             (?P<token>[^?]+?)?(?:[?].*)?$)
                        |(?:api\.soundcloud\.com/tracks/(?P<track_id>\d+)
@@ -265,6 +265,9 @@ class SoundcloudSetIE(SoundcloudIE):
             'title': 'The Royal Concept EP',
         },
         'playlist_mincount': 6,
+    }, {
+        'url': 'https://soundcloud.com/the-concept-band/sets/the-royal-concept-ep/token',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From a249ab83cb1d7765d787a7b1d050449736aaa789 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 31 Aug 2016 01:56:58 +0700
Subject: [PATCH 5938/6219] [pyvideo] Remove debugging code

---
 youtube_dl/extractor/pyvideo.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/pyvideo.py b/youtube_dl/extractor/pyvideo.py
index 08ec09183..b8ac93a62 100644
--- a/youtube_dl/extractor/pyvideo.py
+++ b/youtube_dl/extractor/pyvideo.py
@@ -38,7 +38,6 @@ class PyvideoIE(InfoExtractor):
             % (category, video_id), video_id, fatal=False)
 
         if data:
-            print(data)
             for video in data['videos']:
                 video_url = video.get('url')
                 if video_url:

From 263fef43dea463ab4b897c8374dbb11c705f061c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 31 Aug 2016 02:37:40 +0700
Subject: [PATCH 5939/6219] [ChangeLog] Actualize

---
 ChangeLog | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index e055976c5..7e24b8c6b 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,21 @@
 version <unreleased>
 
 Extractors
+* [soundcloud] Fix URL regular expression to avoid clashes with sets (#10505)
+* [bandcamp:album] Fix title extraction (#10455)
+* [pyvideo] Fix extraction (#10468)
++ [ctv] Add support for tsn.ca, bnn.ca and thecomedynetwork.ca (#10016)
+* [9c9media] Extract more metadata
+* [9c9media] Fix multiple stacks extraction (#10016)
+* [adultswim] Improve video info extraction (#10492)
+* [vodplatform] Improve embed regular expression
+- [played] Remove extractor (#10470)
++ [tbs] Add extractor for tbs.com and tntdrama.com (#10222)
++ [cartoonnetwork] Add extractor for cartoonnetwork.com (#10110)
+* [adultswim] Rework in terms of turner extractor
+* [cnn] Rework in terms of turner extractor
+* [nba] Rework in terms of turner extractor
++ [turner] Add base extractor for Turner Broadcasting System based sites
 * [bilibili] Fix extraction (#10375)
 * [openload] Fix extraction (#10408)
 

From 4fd350611c71571733950ad2473d4148f7bb6a63 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 31 Aug 2016 02:39:39 +0700
Subject: [PATCH 5940/6219] release 2016.08.31

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 8 +++++---
 youtube_dl/version.py     | 2 +-
 4 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index a2fe59f80..2caca5115 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.08.28*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.08.28**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.08.31*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.08.31**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.08.28
+[debug] youtube-dl version 2016.08.31
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 7e24b8c6b..0f8076d96 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2016.08.31
 
 Extractors
 * [soundcloud] Fix URL regular expression to avoid clashes with sets (#10505)
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index bf08697be..42bf291e2 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -13,6 +13,8 @@
  - **5min**
  - **8tracks**
  - **91porn**
+ - **9c9media**
+ - **9c9media:stack**
  - **9gag**
  - **9now.com.au**
  - **abc.net.au**
@@ -89,7 +91,7 @@
  - **Bet**
  - **Bigflix**
  - **Bild**: Bild.de
- - **BiliBili** (Currently broken)
+ - **BiliBili**
  - **BioBioChileTV**
  - **BIQLE**
  - **BleacherReport**
@@ -115,6 +117,7 @@
  - **Canvas**
  - **CarambaTV**
  - **CarambaTVPage**
+ - **CartoonNetwork**
  - **cbc.ca**
  - **cbc.ca:player**
  - **cbc.ca:watch**
@@ -459,7 +462,6 @@
  - **nick.de**
  - **niconico**: ニコニコ動画
  - **NiconicoPlaylist**
- - **NineCNineMedia**
  - **Nintendo**
  - **njoy**: N-JOY
  - **njoy:embed**
@@ -517,7 +519,6 @@
  - **Pinkbike**
  - **Pladform**
  - **play.fm**
- - **played.to**
  - **PlaysTV**
  - **Playtvak**: Playtvak.cz, iDNES.cz and Lidovky.cz
  - **Playvid**
@@ -675,6 +676,7 @@
  - **Tagesschau**
  - **tagesschau:player**
  - **Tass**
+ - **TBS**
  - **TDSLifeway**
  - **teachertube**: teachertube.com videos
  - **teachertube:user:collection**: teachertube.com user and collection videos
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index ee30ca2ad..fe442dd88 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.08.28'
+__version__ = '2016.08.31'

From 165620e320ecb9213ee9928466a9209e7608f83c Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 30 Aug 2016 21:48:59 +0100
Subject: [PATCH 5941/6219] [yahoo] extract more and better formats

---
 youtube_dl/extractor/yahoo.py | 81 +++++++++++++++++++----------------
 1 file changed, 45 insertions(+), 36 deletions(-)

diff --git a/youtube_dl/extractor/yahoo.py b/youtube_dl/extractor/yahoo.py
index b0679dfb7..d7a81ab8c 100644
--- a/youtube_dl/extractor/yahoo.py
+++ b/youtube_dl/extractor/yahoo.py
@@ -8,7 +8,6 @@ import re
 from .common import InfoExtractor, SearchInfoExtractor
 from ..compat import (
     compat_urllib_parse,
-    compat_urllib_parse_urlencode,
     compat_urlparse,
 )
 from ..utils import (
@@ -17,6 +16,7 @@ from ..utils import (
     ExtractorError,
     int_or_none,
     mimetype2ext,
+    determine_ext,
 )
 
 from .brightcove import BrightcoveNewIE
@@ -39,7 +39,7 @@ class YahooIE(InfoExtractor):
         },
         {
             'url': 'http://screen.yahoo.com/wired/codefellas-s1-ep12-cougar-lies-103000935.html',
-            'md5': 'c3466d2b6d5dd6b9f41ba9ed04c24b23',
+            'md5': '251af144a19ebc4a033e8ba91ac726bb',
             'info_dict': {
                 'id': 'd1dedf8c-d58c-38c3-8963-e899929ae0a9',
                 'ext': 'mp4',
@@ -50,7 +50,7 @@ class YahooIE(InfoExtractor):
         },
         {
             'url': 'https://screen.yahoo.com/community/community-sizzle-reel-203225340.html?format=embed',
-            'md5': '75ffabdb87c16d4ffe8c036dc4d1c136',
+            'md5': '7993e572fac98e044588d0b5260f4352',
             'info_dict': {
                 'id': '4fe78544-8d48-39d8-97cd-13f205d9fcdb',
                 'ext': 'mp4',
@@ -61,7 +61,7 @@ class YahooIE(InfoExtractor):
         },
         {
             'url': 'https://tw.news.yahoo.com/%E6%95%A2%E5%95%8F%E5%B8%82%E9%95%B7%20%E9%BB%83%E7%A7%80%E9%9C%9C%E6%89%B9%E8%B3%B4%E6%B8%85%E5%BE%B7%20%E9%9D%9E%E5%B8%B8%E9%AB%98%E5%82%B2-034024051.html',
-            'md5': '9035d38f88b1782682a3e89f985be5bb',
+            'md5': '45c024bad51e63e9b6f6fad7a43a8c23',
             'info_dict': {
                 'id': 'cac903b3-fcf4-3c14-b632-643ab541712f',
                 'ext': 'mp4',
@@ -72,10 +72,10 @@ class YahooIE(InfoExtractor):
         },
         {
             'url': 'https://uk.screen.yahoo.com/editor-picks/cute-raccoon-freed-drain-using-091756545.html',
-            'md5': '0b51660361f0e27c9789e7037ef76f4b',
+            'md5': '71298482f7c64cbb7fa064e4553ff1c1',
             'info_dict': {
                 'id': 'b3affa53-2e14-3590-852b-0e0db6cd1a58',
-                'ext': 'mp4',
+                'ext': 'webm',
                 'title': 'Cute Raccoon Freed From Drain\u00a0Using Angle Grinder',
                 'description': 'md5:f66c890e1490f4910a9953c941dee944',
                 'duration': 97,
@@ -98,7 +98,7 @@ class YahooIE(InfoExtractor):
                 'id': '154609075',
             },
             'playlist': [{
-                'md5': 'f8e336c6b66f503282e5f719641d6565',
+                'md5': '000887d0dc609bc3a47c974151a40fb8',
                 'info_dict': {
                     'id': 'e624c4bc-3389-34de-9dfc-025f74943409',
                     'ext': 'mp4',
@@ -107,7 +107,7 @@ class YahooIE(InfoExtractor):
                     'duration': 30,
                 },
             }, {
-                'md5': '958bcb90b4d6df71c56312137ee1cd5a',
+                'md5': '81bc74faf10750fe36e4542f9a184c66',
                 'info_dict': {
                     'id': '1fc8ada0-718e-3abe-a450-bf31f246d1a9',
                     'ext': 'mp4',
@@ -139,7 +139,7 @@ class YahooIE(InfoExtractor):
             'skip': 'Domain name in.lifestyle.yahoo.com gone',
         }, {
             'url': 'https://www.yahoo.com/movies/v/true-story-trailer-173000497.html',
-            'md5': 'b17ac378b1134fa44370fb27db09a744',
+            'md5': '2a9752f74cb898af5d1083ea9f661b58',
             'info_dict': {
                 'id': '071c4013-ce30-3a93-a5b2-e0413cd4a9d1',
                 'ext': 'mp4',
@@ -168,7 +168,7 @@ class YahooIE(InfoExtractor):
         }, {
             # Query result is embedded in webpage, but explicit request to video API fails with geo restriction
             'url': 'https://screen.yahoo.com/community/communitary-community-episode-1-ladders-154501237.html',
-            'md5': '1ddbf7c850777548438e5c4f147c7b8c',
+            'md5': '4fbafb9c9b6f07aa8f870629f6671b35',
             'info_dict': {
                 'id': '1f32853c-a271-3eef-8cb6-f6d6872cb504',
                 'ext': 'mp4',
@@ -196,6 +196,7 @@ class YahooIE(InfoExtractor):
                 'description': 'Galactic',
                 'title': 'Dolla Diva (feat. Maggie Koerner)',
             },
+            'skip': 'redirect to https://www.yahoo.com/music',
         },
     ]
 
@@ -213,15 +214,7 @@ class YahooIE(InfoExtractor):
         entries = []
         iframe_urls = re.findall(r'<iframe[^>]+src="(/video/.+?-\d+\.html\?format=embed.*?)"', webpage)
         for idx, iframe_url in enumerate(iframe_urls):
-            iframepage = self._download_webpage(
-                host + iframe_url, display_id,
-                note='Downloading iframe webpage for video #%d' % idx)
-            items_json = self._search_regex(
-                r'mediaItems: (\[.+?\])$', iframepage, 'items', flags=re.MULTILINE, default=None)
-            if items_json:
-                items = json.loads(items_json)
-                video_id = items[0]['id']
-                entries.append(self._get_info(video_id, display_id, webpage))
+            entries.append(self.url_result(host + iframe_url, 'Yahoo'))
         if entries:
             return self.playlist_result(entries, page_id)
 
@@ -246,7 +239,9 @@ class YahooIE(InfoExtractor):
             if config:
                 sapi = config.get('models', {}).get('applet_model', {}).get('data', {}).get('sapi')
                 if sapi and 'query' in sapi:
-                    return self._extract_info(display_id, sapi, webpage)
+                    info = self._extract_info(display_id, sapi, webpage)
+                    self._sort_formats(info['formats'])
+                    return info
 
         items_json = self._search_regex(
             r'mediaItems: ({.*?})$', webpage, 'items', flags=re.MULTILINE,
@@ -292,15 +287,17 @@ class YahooIE(InfoExtractor):
 
         formats = []
         for s in info['streams']:
+            tbr = int_or_none(s.get('bitrate'))
             format_info = {
                 'width': int_or_none(s.get('width')),
                 'height': int_or_none(s.get('height')),
-                'tbr': int_or_none(s.get('bitrate')),
+                'tbr': tbr,
             }
 
             host = s['host']
             path = s['path']
             if host.startswith('rtmp'):
+                fmt = 'rtmp'
                 format_info.update({
                     'url': host,
                     'play_path': path,
@@ -308,14 +305,18 @@ class YahooIE(InfoExtractor):
                 })
             else:
                 if s.get('format') == 'm3u8_playlist':
-                    format_info['protocol'] = 'm3u8_native'
-                    format_info['ext'] = 'mp4'
+                    fmt = 'hls'
+                    format_info.update({
+                        'protocol': 'm3u8_native',
+                        'ext': 'mp4',
+                    })
+                else:
+                    fmt = format_info['ext'] = determine_ext(path)
                 format_url = compat_urlparse.urljoin(host, path)
                 format_info['url'] = format_url
+            format_info['format_id'] = fmt + ('-%d' % tbr if tbr else '')
             formats.append(format_info)
 
-        self._sort_formats(formats)
-
         closed_captions = self._html_search_regex(
             r'"closedcaptions":(\[[^\]]+\])', webpage, 'closed captions',
             default='[]')
@@ -346,17 +347,25 @@ class YahooIE(InfoExtractor):
     def _get_info(self, video_id, display_id, webpage):
         region = self._search_regex(
             r'\\?"region\\?"\s*:\s*\\?"([^"]+?)\\?"',
-            webpage, 'region', fatal=False, default='US')
-        data = compat_urllib_parse_urlencode({
-            'protocol': 'http',
-            'region': region.upper(),
-        })
-        query_url = (
-            'https://video.media.yql.yahoo.com/v1/video/sapi/streams/'
-            '{id}?{data}'.format(id=video_id, data=data))
-        query_result = self._download_json(
-            query_url, display_id, 'Downloading video info')
-        return self._extract_info(display_id, query_result, webpage)
+            webpage, 'region', fatal=False, default='US').upper()
+        formats = []
+        info = {}
+        for fmt in ('webm', 'mp4'):
+            query_result = self._download_json(
+                'https://video.media.yql.yahoo.com/v1/video/sapi/streams/' + video_id,
+                display_id, 'Downloading %s video info' % fmt, query={
+                    'protocol': 'http',
+                    'region': region,
+                    'format': fmt,
+                })
+            info = self._extract_info(display_id, query_result, webpage)
+            formats.extend(info['formats'])
+        formats.extend(self._extract_m3u8_formats(
+            'http://video.media.yql.yahoo.com/v1/hls/%s?region=%s' % (video_id, region),
+            video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
+        self._sort_formats(formats)
+        info['formats'] = formats
+        return info
 
 
 class YahooSearchIE(SearchInfoExtractor):

From 196c6ba06792ec38238631d9173fc146822baa7e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 31 Aug 2016 22:12:37 +0700
Subject: [PATCH 5942/6219] [facebook] Extract timestamp (Closes #10508)

---
 youtube_dl/extractor/facebook.py | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index 0fb781a73..228b0b6d7 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -15,6 +15,7 @@ from ..compat import (
 from ..utils import (
     error_to_compat_str,
     ExtractorError,
+    int_or_none,
     limit_length,
     sanitized_Request,
     urlencode_postdata,
@@ -62,6 +63,8 @@ class FacebookIE(InfoExtractor):
             'ext': 'mp4',
             'title': 're:Did you know Kei Nishikori is the first Asian man to ever reach a Grand Slam',
             'uploader': 'Tennis on Facebook',
+            'upload_date': '20140908',
+            'timestamp': 1410199200,
         }
     }, {
         'note': 'Video without discernible title',
@@ -71,6 +74,8 @@ class FacebookIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Facebook video #274175099429670',
             'uploader': 'Asif Nawab Butt',
+            'upload_date': '20140506',
+            'timestamp': 1399398998,
         },
         'expected_warnings': [
             'title'
@@ -78,12 +83,14 @@ class FacebookIE(InfoExtractor):
     }, {
         'note': 'Video with DASH manifest',
         'url': 'https://www.facebook.com/video.php?v=957955867617029',
-        'md5': '54706e4db4f5ad58fbad82dde1f1213f',
+        'md5': 'b2c28d528273b323abe5c6ab59f0f030',
         'info_dict': {
             'id': '957955867617029',
             'ext': 'mp4',
             'title': 'When you post epic content on instagram.com/433 8 million followers, this is ...',
             'uploader': 'Demy de Zeeuw',
+            'upload_date': '20160110',
+            'timestamp': 1452431627,
         },
     }, {
         'url': 'https://www.facebook.com/maxlayn/posts/10153807558977570',
@@ -306,12 +313,16 @@ class FacebookIE(InfoExtractor):
         if not video_title:
             video_title = 'Facebook video #%s' % video_id
         uploader = clean_html(get_element_by_id('fbPhotoPageAuthorName', webpage))
+        timestamp = int_or_none(self._search_regex(
+            r'<abbr[^>]+data-utime=["\'](\d+)', webpage,
+            'timestamp', default=None))
 
         info_dict = {
             'id': video_id,
             'title': video_title,
             'formats': formats,
             'uploader': uploader,
+            'timestamp': timestamp,
         }
 
         return webpage, info_dict

From 7a3e849f6eaf51b1d86b843a63664012ced2258c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 31 Aug 2016 22:23:55 +0700
Subject: [PATCH 5943/6219] [porncom] Extract categories and tags (Closes
 #10510)

---
 youtube_dl/extractor/porncom.py | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/porncom.py b/youtube_dl/extractor/porncom.py
index 4baf79688..d85e0294d 100644
--- a/youtube_dl/extractor/porncom.py
+++ b/youtube_dl/extractor/porncom.py
@@ -26,6 +26,8 @@ class PornComIE(InfoExtractor):
             'duration': 551,
             'view_count': int,
             'age_limit': 18,
+            'categories': list,
+            'tags': list,
         },
     }, {
         'url': 'http://se.porn.com/videos/marsha-may-rides-seth-on-top-of-his-thick-cock-2658067',
@@ -75,7 +77,14 @@ class PornComIE(InfoExtractor):
         self._sort_formats(formats)
 
         view_count = str_to_int(self._search_regex(
-            r'class=["\']views["\'][^>]*><p>([\d,.]+)', webpage, 'view count'))
+            r'class=["\']views["\'][^>]*><p>([\d,.]+)', webpage,
+            'view count', fatal=False))
+
+        def extract_list(kind):
+            s = self._search_regex(
+                r'(?s)<p[^>]*>%s:(.+?)</p>' % kind.capitalize(),
+                webpage, kind, fatal=False)
+            return re.findall(r'<a[^>]+>([^<]+)</a>', s or '')
 
         return {
             'id': video_id,
@@ -86,4 +95,6 @@ class PornComIE(InfoExtractor):
             'view_count': view_count,
             'formats': formats,
             'age_limit': 18,
+            'categories': extract_list('categories'),
+            'tags': extract_list('tags'),
         }

From f8fd510eb4b2733a5c083d767d45baa88b289298 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 31 Aug 2016 18:31:49 +0100
Subject: [PATCH 5944/6219] [limelight] skip ism manifests and reduce requests

---
 youtube_dl/extractor/limelight.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/limelight.py b/youtube_dl/extractor/limelight.py
index a425bafe3..6752ffee2 100644
--- a/youtube_dl/extractor/limelight.py
+++ b/youtube_dl/extractor/limelight.py
@@ -34,11 +34,12 @@ class LimelightBaseIE(InfoExtractor):
     def _extract_info(self, streams, mobile_urls, properties):
         video_id = properties['media_id']
         formats = []
-
+        urls = []
         for stream in streams:
             stream_url = stream.get('url')
-            if not stream_url or stream.get('drmProtected'):
+            if not stream_url or stream.get('drmProtected') or stream_url in urls:
                 continue
+            urls.append(stream_url)
             ext = determine_ext(stream_url)
             if ext == 'f4m':
                 formats.extend(self._extract_f4m_formats(
@@ -58,9 +59,11 @@ class LimelightBaseIE(InfoExtractor):
                     format_id = 'rtmp'
                     if stream.get('videoBitRate'):
                         format_id += '-%d' % int_or_none(stream['videoBitRate'])
+                    http_url = 'http://%s/%s' % (rtmp.group('host').replace('csl.', 'cpl.'), rtmp.group('playpath')[4:])
+                    urls.append(http_url)
                     http_fmt = fmt.copy()
                     http_fmt.update({
-                        'url': 'http://%s/%s' % (rtmp.group('host').replace('csl.', 'cpl.'), rtmp.group('playpath')[4:]),
+                        'url': http_url,
                         'format_id': format_id.replace('rtmp', 'http'),
                     })
                     formats.append(http_fmt)
@@ -76,8 +79,9 @@ class LimelightBaseIE(InfoExtractor):
         for mobile_url in mobile_urls:
             media_url = mobile_url.get('mobileUrl')
             format_id = mobile_url.get('targetMediaPlatform')
-            if not media_url or format_id == 'Widevine':
+            if not media_url or format_id in ('Widevine', 'SmoothStreaming') or media_url in urls:
                 continue
+            urls.append(media_url)
             ext = determine_ext(media_url)
             if ext == 'm3u8':
                 formats.extend(self._extract_m3u8_formats(

From 2896dd73bc2c9844175258086c0300395722e5c9 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 1 Sep 2016 08:00:13 +0100
Subject: [PATCH 5945/6219] [cbs] extract once formats(closes #10515)

---
 youtube_dl/extractor/cbs.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/cbs.py b/youtube_dl/extractor/cbs.py
index c72ed2dbb..3f4dea40c 100644
--- a/youtube_dl/extractor/cbs.py
+++ b/youtube_dl/extractor/cbs.py
@@ -51,7 +51,7 @@ class CBSIE(CBSBaseIE):
         path = 'dJ5BDC/media/guid/2198311517/' + guid
         smil_url = 'http://link.theplatform.com/s/%s?mbr=true' % path
         formats, subtitles = self._extract_theplatform_smil(smil_url + '&manifest=m3u', guid)
-        for r in ('HLS&formats=M3U', 'RTMP', 'WIFI', '3G'):
+        for r in ('OnceURL&formats=M3U', 'HLS&formats=M3U', 'RTMP', 'WIFI', '3G'):
             try:
                 tp_formats, _ = self._extract_theplatform_smil(smil_url + '&assetTypes=' + r, guid, 'Downloading %s SMIL data' % r.split('&')[0])
                 formats.extend(tp_formats)

From 165c54e97d10705614934d5b1d86d90c06951b7c Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 1 Sep 2016 16:28:03 +0800
Subject: [PATCH 5946/6219] =?UTF-8?q?[southpark.cc.com:espa=C3=B1ol]=20Ski?=
 =?UTF-8?q?p=20geo-restricted=20=5FTESTS?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Breaks https://travis-ci.org/rg3/youtube-dl/jobs/156728175
---
 youtube_dl/extractor/southpark.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/southpark.py b/youtube_dl/extractor/southpark.py
index a147f7db1..e2a9e45ac 100644
--- a/youtube_dl/extractor/southpark.py
+++ b/youtube_dl/extractor/southpark.py
@@ -35,6 +35,7 @@ class SouthParkEsIE(SouthParkIE):
             'description': 'Cartman Consigue Una Sonda Anal',
         },
         'playlist_count': 4,
+        'skip': 'Geo-restricted',
     }]
 
 

From 746a695b362cb602625ed7357294bb18de133883 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 1 Sep 2016 16:42:35 +0800
Subject: [PATCH 5947/6219] [myvidster] Update _TESTS (closes #10473)

---
 youtube_dl/extractor/myvidster.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/myvidster.py b/youtube_dl/extractor/myvidster.py
index 731c24542..2117d302d 100644
--- a/youtube_dl/extractor/myvidster.py
+++ b/youtube_dl/extractor/myvidster.py
@@ -13,7 +13,7 @@ class MyVidsterIE(InfoExtractor):
             'id': '3685814',
             'title': 'md5:7d8427d6d02c4fbcef50fe269980c749',
             'upload_date': '20141027',
-            'uploader_id': 'utkualp',
+            'uploader': 'utkualp',
             'ext': 'mp4',
             'age_limit': 18,
         },

From 05d4612947d6dbfaedb8f2a00daa5f29d85f73df Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 1 Sep 2016 16:58:16 +0800
Subject: [PATCH 5948/6219] [movingimage] Adapt to the new domain name and fix
 extraction

Closes #10466
---
 ChangeLog                                     |  6 +++++
 youtube_dl/extractor/extractors.py            |  2 +-
 .../extractor/{ssa.py => movingimage.py}      | 26 +++++++------------
 3 files changed, 17 insertions(+), 17 deletions(-)
 rename youtube_dl/extractor/{ssa.py => movingimage.py} (65%)

diff --git a/ChangeLog b/ChangeLog
index 0f8076d96..877e8112e 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Extractors
+* [movingimage] Fix for the new site name (#10466)
+
+
 version 2016.08.31
 
 Extractors
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 21efa96b2..8d0688f53 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -486,6 +486,7 @@ from .motherless import MotherlessIE
 from .motorsport import MotorsportIE
 from .movieclips import MovieClipsIE
 from .moviezine import MoviezineIE
+from .movingimage import MovingImageIE
 from .msn import MSNIE
 from .mtv import (
     MTVIE,
@@ -806,7 +807,6 @@ from .srgssr import (
     SRGSSRPlayIE,
 )
 from .srmediathek import SRMediathekIE
-from .ssa import SSAIE
 from .stanfordoc import StanfordOpenClassroomIE
 from .steam import SteamIE
 from .streamable import StreamableIE
diff --git a/youtube_dl/extractor/ssa.py b/youtube_dl/extractor/movingimage.py
similarity index 65%
rename from youtube_dl/extractor/ssa.py
rename to youtube_dl/extractor/movingimage.py
index 54d1843f2..bb789c32e 100644
--- a/youtube_dl/extractor/ssa.py
+++ b/youtube_dl/extractor/movingimage.py
@@ -7,22 +7,19 @@ from ..utils import (
 )
 
 
-class SSAIE(InfoExtractor):
-    _VALID_URL = r'https?://ssa\.nls\.uk/film/(?P<id>\d+)'
+class MovingImageIE(InfoExtractor):
+    _VALID_URL = r'https?://movingimage\.nls\.uk/film/(?P<id>\d+)'
     _TEST = {
-        'url': 'http://ssa.nls.uk/film/3561',
+        'url': 'http://movingimage.nls.uk/film/3561',
+        'md5': '4caa05c2b38453e6f862197571a7be2f',
         'info_dict': {
             'id': '3561',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'SHETLAND WOOL',
             'description': 'md5:c5afca6871ad59b4271e7704fe50ab04',
             'duration': 900,
             'thumbnail': 're:^https?://.*\.jpg$',
         },
-        'params': {
-            # rtmp download
-            'skip_download': True,
-        },
     }
 
     def _real_extract(self, url):
@@ -30,10 +27,9 @@ class SSAIE(InfoExtractor):
 
         webpage = self._download_webpage(url, video_id)
 
-        streamer = self._search_regex(
-            r"'streamer'\s*,\S*'(rtmp[^']+)'", webpage, 'streamer')
-        play_path = self._search_regex(
-            r"'file'\s*,\s*'([^']+)'", webpage, 'file').rpartition('.')[0]
+        formats = self._extract_m3u8_formats(
+            self._html_search_regex(r'file\s*:\s*"([^"]+)"', webpage, 'm3u8 manifest URL'),
+            video_id, ext='mp4', entry_protocol='m3u8_native')
 
         def search_field(field_name, fatal=False):
             return self._search_regex(
@@ -44,13 +40,11 @@ class SSAIE(InfoExtractor):
         description = unescapeHTML(search_field('Description'))
         duration = parse_duration(search_field('Running time'))
         thumbnail = self._search_regex(
-            r"'image'\s*,\s*'([^']+)'", webpage, 'thumbnails', fatal=False)
+            r"image\s*:\s*'([^']+)'", webpage, 'thumbnail', fatal=False)
 
         return {
             'id': video_id,
-            'url': streamer,
-            'play_path': play_path,
-            'ext': 'flv',
+            'formats': formats,
             'title': title,
             'description': description,
             'duration': duration,

From 4c8ab6fd715249290feab89bbc86eb803b459993 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 1 Sep 2016 17:04:41 +0800
Subject: [PATCH 5949/6219] [thvideo] Remove extractor. Website down.

Closes #10464

According to a screenshot in http://tieba.baidu.com/p/4691302183,
thvideo.tv is shut down "temporarily". I see no clues that it will be up
again, so I remove it here.
---
 ChangeLog                          |  1 +
 youtube_dl/extractor/extractors.py |  4 --
 youtube_dl/extractor/thvideo.py    | 84 ------------------------------
 3 files changed, 1 insertion(+), 88 deletions(-)
 delete mode 100644 youtube_dl/extractor/thvideo.py

diff --git a/ChangeLog b/ChangeLog
index 877e8112e..2e75c003d 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
 version <unreleased>
 
 Extractors
+- [thvideo] Remove extractor (#10464)
 * [movingimage] Fix for the new site name (#10466)
 
 
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 8d0688f53..459d776b3 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -869,10 +869,6 @@ from .tnaflix import (
     MovieFapIE,
 )
 from .toggle import ToggleIE
-from .thvideo import (
-    THVideoIE,
-    THVideoPlaylistIE
-)
 from .toutv import TouTvIE
 from .toypics import ToypicsUserIE, ToypicsIE
 from .traileraddict import TrailerAddictIE
diff --git a/youtube_dl/extractor/thvideo.py b/youtube_dl/extractor/thvideo.py
deleted file mode 100644
index 406f4a826..000000000
--- a/youtube_dl/extractor/thvideo.py
+++ /dev/null
@@ -1,84 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-import re
-
-from .common import InfoExtractor
-from ..utils import (
-    unified_strdate
-)
-
-
-class THVideoIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?thvideo\.tv/(?:v/th|mobile\.php\?cid=)(?P<id>[0-9]+)'
-    _TEST = {
-        'url': 'http://thvideo.tv/v/th1987/',
-        'md5': 'fa107b1f73817e325e9433505a70db50',
-        'info_dict': {
-            'id': '1987',
-            'ext': 'mp4',
-            'title': '【动画】秘封活动记录 ～ The Sealed Esoteric History.分镜稿预览',
-            'display_id': 'th1987',
-            'thumbnail': 'http://thvideo.tv/uploadfile/2014/0722/20140722013459856.jpg',
-            'description': '社团京都幻想剧团的第一个东方二次同人动画作品「秘封活动记录 ～ The Sealed Esoteric History.」 本视频是该动画第一期的分镜草稿...',
-            'upload_date': '20140722'
-        }
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        # extract download link from mobile player page
-        webpage_player = self._download_webpage(
-            'http://thvideo.tv/mobile.php?cid=%s-0' % (video_id),
-            video_id, note='Downloading video source page')
-        video_url = self._html_search_regex(
-            r'<source src="(.*?)" type', webpage_player, 'video url')
-
-        # extract video info from main page
-        webpage = self._download_webpage(
-            'http://thvideo.tv/v/th%s' % (video_id), video_id)
-        title = self._og_search_title(webpage)
-        display_id = 'th%s' % video_id
-        thumbnail = self._og_search_thumbnail(webpage)
-        description = self._og_search_description(webpage)
-        upload_date = unified_strdate(self._html_search_regex(
-            r'span itemprop="datePublished" content="(.*?)">', webpage,
-            'upload date', fatal=False))
-
-        return {
-            'id': video_id,
-            'ext': 'mp4',
-            'url': video_url,
-            'title': title,
-            'display_id': display_id,
-            'thumbnail': thumbnail,
-            'description': description,
-            'upload_date': upload_date
-        }
-
-
-class THVideoPlaylistIE(InfoExtractor):
-    _VALID_URL = r'http?://(?:www\.)?thvideo\.tv/mylist(?P<id>[0-9]+)'
-    _TEST = {
-        'url': 'http://thvideo.tv/mylist2',
-        'info_dict': {
-            'id': '2',
-            'title': '幻想万華鏡',
-        },
-        'playlist_mincount': 23,
-    }
-
-    def _real_extract(self, url):
-        playlist_id = self._match_id(url)
-
-        webpage = self._download_webpage(url, playlist_id)
-        list_title = self._html_search_regex(
-            r'<h1 class="show_title">(.*?)<b id', webpage, 'playlist title',
-            fatal=False)
-
-        entries = [
-            self.url_result('http://thvideo.tv/v/th' + id, 'THVideo')
-            for id in re.findall(r'<dd><a href="http://thvideo.tv/v/th(\d+)/" target=', webpage)]
-
-        return self.playlist_result(entries, playlist_id, list_title)

From f096ec262544babf6ea23347160c1c550e4e157e Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 1 Sep 2016 13:34:12 +0100
Subject: [PATCH 5950/6219] [curiositystream] Add new extractor

---
 youtube_dl/extractor/curiositystream.py | 128 ++++++++++++++++++++++++
 youtube_dl/extractor/extractors.py      |   4 +
 2 files changed, 132 insertions(+)
 create mode 100644 youtube_dl/extractor/curiositystream.py

diff --git a/youtube_dl/extractor/curiositystream.py b/youtube_dl/extractor/curiositystream.py
new file mode 100644
index 000000000..7105e3c4c
--- /dev/null
+++ b/youtube_dl/extractor/curiositystream.py
@@ -0,0 +1,128 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    urlencode_postdata,
+    compat_str,
+    ExtractorError,
+)
+
+
+class CuriosityStreamBaseIE(InfoExtractor):
+    _NETRC_MACHINE = 'curiositystream'
+    _auth_token = None
+    _API_BASE_URL = 'https://api.curiositystream.com/v1/'
+
+    def _handle_errors(self, result):
+        error = result.get('error', {}).get('message')
+        if error:
+            if isinstance(error, dict):
+                error = ', '.join(error.values())
+            raise ExtractorError(
+                '%s said: %s' % (self.IE_NAME, error), expected=True)
+
+    def _call_api(self, path, video_id):
+        headers = {}
+        if self._auth_token:
+            headers['X-Auth-Token'] = self._auth_token
+        result = self._download_json(
+            self._API_BASE_URL + path, video_id, headers=headers)
+        self._handle_errors(result)
+        return result['data']
+
+    def _real_initialize(self):
+        if not self._auth_token:
+            user = self._downloader.cache.load('curiositystream', 'user') or {}
+            self._auth_token = user.get('auth_token')
+            if not self._auth_token:
+                (email, password) = self._get_login_info()
+                if email is None:
+                    return
+                result = self._download_json(
+                    self._API_BASE_URL + 'login', None, data=urlencode_postdata({
+                        'email': email,
+                        'password': password,
+                    }))
+                self._handle_errors(result)
+                self._auth_token = result['message']['auth_token']
+                self._downloader.cache.store(
+                    'curiositystream', 'user', {
+                        'auth_token': self._auth_token,
+                    })
+
+    def _extract_media_info(self, media):
+        video_id = compat_str(media['id'])
+        limelight_media_id = media['limelight_media_id']
+        title = media['title']
+
+        subtitles = {}
+        for closed_caption in media.get('closed_captions', []):
+            sub_url = closed_caption.get('file')
+            if not sub_url:
+                continue
+            lang = closed_caption.get('code') or closed_caption.get('language') or 'en'
+            subtitles.setdefault(lang, []).append({
+                'url': sub_url,
+            })
+
+        return {
+            '_type': 'url_transparent',
+            'id': video_id,
+            'url': 'limelight:media:' + limelight_media_id,
+            'title': title,
+            'description': media.get('description'),
+            'thumbnail': media.get('image_large') or media.get('image_medium') or media.get('image_small'),
+            'duration': int_or_none(media.get('duration')),
+            'tags': media.get('tags'),
+            'subtitles': subtitles,
+            'ie_key': 'LimelightMedia',
+        }
+
+
+class CuriosityStreamIE(CuriosityStreamBaseIE):
+    IE_NAME = 'curiositystream'
+    _VALID_URL = r'https?://app\.curiositystream\.com/video/(?P<id>\d+)'
+    _TEST = {
+        'url': 'https://app.curiositystream.com/video/2',
+        'md5': 'a0074c190e6cddaf86900b28d3e9ee7a',
+        'info_dict': {
+            'id': '2',
+            'ext': 'mp4',
+            'title': 'How Did You Develop The Internet?',
+            'description': 'Vint Cerf, Google\'s Chief Internet Evangelist, describes how he and Bob Kahn created the internet.',
+            'timestamp': 1448388615,
+            'upload_date': '20151124',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        media = self._call_api('media/' + video_id, video_id)
+        return self._extract_media_info(media)
+
+
+class CuriosityStreamCollectionIE(CuriosityStreamBaseIE):
+    IE_NAME = 'curiositystream:collection'
+    _VALID_URL = r'https?://app\.curiositystream\.com/collection/(?P<id>\d+)'
+    _TEST = {
+        'url': 'https://app.curiositystream.com/collection/2',
+        'info_dict': {
+            'id': '2',
+            'title': 'Curious Minds: The Internet',
+            'description': 'How is the internet shaping our lives in the 21st Century?',
+        },
+        'playlist_mincount': 17,
+    }
+
+    def _real_extract(self, url):
+        collection_id = self._match_id(url)
+        collection = self._call_api(
+            'collections/' + collection_id, collection_id)
+        entries = []
+        for media in collection.get('media', []):
+            entries.append(self._extract_media_info(media))
+        return self.playlist_result(
+            entries, collection_id,
+            collection.get('title'), collection.get('description'))
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 459d776b3..0c2436b67 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -194,6 +194,10 @@ from .ctsnews import CtsNewsIE
 from .ctv import CTVIE
 from .ctvnews import CTVNewsIE
 from .cultureunplugged import CultureUnpluggedIE
+from .curiositystream import (
+    CuriosityStreamIE,
+    CuriosityStreamCollectionIE,
+)
 from .cwtv import CWTVIE
 from .dailymail import DailyMailIE
 from .dailymotion import (

From 9250181f37cf0289c02d18ab91203c6181f9cc71 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 1 Sep 2016 21:37:25 +0700
Subject: [PATCH 5951/6219] [extractor/common] Restore NAME usage from
 EXT-X-MEDIA tag for formats codes in _extract_m3u8_formats (Closes #10522)

---
 youtube_dl/extractor/common.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index da0af29ec..36d43fd50 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1202,6 +1202,7 @@ class InfoExtractor(object):
                 'preference': preference,
             }]
         last_info = None
+        last_media = None
         for line in m3u8_doc.splitlines():
             if line.startswith('#EXT-X-STREAM-INF:'):
                 last_info = parse_m3u8_attributes(line)
@@ -1224,6 +1225,10 @@ class InfoExtractor(object):
                             'protocol': entry_protocol,
                             'preference': preference,
                         })
+                    else:
+                        # When there is no URI in EXT-X-MEDIA let this tag's
+                        # data be used by regular URI lines below
+                        last_media = media
             elif line.startswith('#') or not line.strip():
                 continue
             else:
@@ -1234,13 +1239,14 @@ class InfoExtractor(object):
                 format_id = []
                 if m3u8_id:
                     format_id.append(m3u8_id)
+                last_media_name = last_media.get('NAME') if last_media else None
+                # Despite specification does not mention NAME attribute for
+                # EXT-X-STREAM-INF it still sometimes may be present
+                stream_name = last_info.get('NAME') or last_media_name
                 # Bandwidth of live streams may differ over time thus making
                 # format_id unpredictable. So it's better to keep provided
                 # format_id intact.
                 if not live:
-                    # Despite specification does not mention NAME attribute for
-                    # EXT-X-STREAM-INF it still sometimes may be present
-                    stream_name = last_info.get('NAME')
                     format_id.append(stream_name if stream_name else '%d' % (tbr if tbr else len(formats)))
                 f = {
                     'format_id': '-'.join(format_id),

From e816c9d158629ef054c1cc77eecf83043d06fe8c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 1 Sep 2016 22:18:16 +0700
Subject: [PATCH 5952/6219] [extractor/common] Simplify _extract_m3u8_formats

---
 youtube_dl/extractor/common.py | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 36d43fd50..a9c7a8d16 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1201,8 +1201,8 @@ class InfoExtractor(object):
                 'protocol': entry_protocol,
                 'preference': preference,
             }]
-        last_info = None
-        last_media = None
+        last_info = {}
+        last_media = {}
         for line in m3u8_doc.splitlines():
             if line.startswith('#EXT-X-STREAM-INF:'):
                 last_info = parse_m3u8_attributes(line)
@@ -1232,17 +1232,13 @@ class InfoExtractor(object):
             elif line.startswith('#') or not line.strip():
                 continue
             else:
-                if last_info is None:
-                    formats.append({'url': format_url(line)})
-                    continue
                 tbr = int_or_none(last_info.get('AVERAGE-BANDWIDTH') or last_info.get('BANDWIDTH'), scale=1000)
                 format_id = []
                 if m3u8_id:
                     format_id.append(m3u8_id)
-                last_media_name = last_media.get('NAME') if last_media else None
                 # Despite specification does not mention NAME attribute for
                 # EXT-X-STREAM-INF it still sometimes may be present
-                stream_name = last_info.get('NAME') or last_media_name
+                stream_name = last_info.get('NAME') or last_media.get('NAME')
                 # Bandwidth of live streams may differ over time thus making
                 # format_id unpredictable. So it's better to keep provided
                 # format_id intact.
@@ -1275,6 +1271,7 @@ class InfoExtractor(object):
                 f.update(parse_codecs(last_info.get('CODECS')))
                 formats.append(f)
                 last_info = {}
+                last_media = {}
         return formats
 
     @staticmethod

From f6af0f888b03e8c072b86c04492cc84c966c9f15 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 1 Sep 2016 23:15:01 +0700
Subject: [PATCH 5953/6219] [youporn] Fix categories and tags extraction
 (Closes #10521)

---
 youtube_dl/extractor/youporn.py | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/youporn.py b/youtube_dl/extractor/youporn.py
index 0df2d76ee..0265a64a7 100644
--- a/youtube_dl/extractor/youporn.py
+++ b/youtube_dl/extractor/youporn.py
@@ -35,7 +35,7 @@ class YouPornIE(InfoExtractor):
             'age_limit': 18,
         },
     }, {
-        # Anonymous User uploader
+        # Unknown uploader
         'url': 'http://www.youporn.com/watch/561726/big-tits-awesome-brunette-on-amazing-webcam-show/?from=related3&al=2&from_id=561726&pos=4',
         'info_dict': {
             'id': '561726',
@@ -44,7 +44,7 @@ class YouPornIE(InfoExtractor):
             'title': 'Big Tits Awesome Brunette On amazing webcam show',
             'description': 'http://sweetlivegirls.com Big Tits Awesome Brunette On amazing webcam show.mp4',
             'thumbnail': 're:^https?://.*\.jpg$',
-            'uploader': 'Anonymous User',
+            'uploader': 'Unknown',
             'upload_date': '20111125',
             'average_rating': int,
             'view_count': int,
@@ -140,17 +140,17 @@ class YouPornIE(InfoExtractor):
             r'>All [Cc]omments? \(([\d,.]+)\)',
             webpage, 'comment count', fatal=False))
 
-        def extract_tag_box(title):
-            tag_box = self._search_regex(
-                (r'<div[^>]+class=["\']tagBoxTitle["\'][^>]*>\s*%s\b.*?</div>\s*'
-                 '<div[^>]+class=["\']tagBoxContent["\']>(.+?)</div>') % re.escape(title),
-                webpage, '%s tag box' % title, default=None)
+        def extract_tag_box(regex, title):
+            tag_box = self._search_regex(regex, webpage, title, default=None)
             if not tag_box:
                 return []
             return re.findall(r'<a[^>]+href=[^>]+>([^<]+)', tag_box)
 
-        categories = extract_tag_box('Category')
-        tags = extract_tag_box('Tags')
+        categories = extract_tag_box(
+            r'(?s)Categories:.*?</[^>]+>(.+?)</div>', 'categories')
+        tags = extract_tag_box(
+            r'(?s)Tags:.*?</div>\s*<div[^>]+class=["\']tagBoxContent["\'][^>]*>(.+?)</div>',
+            'tags')
 
         return {
             'id': video_id,

From 8fb6af6bba201c9f750aadb7b092704195c7f8e0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 1 Sep 2016 23:32:28 +0700
Subject: [PATCH 5954/6219] [exfm] Remove extractor (Closes #10482)

---
 youtube_dl/extractor/exfm.py       | 58 ------------------------------
 youtube_dl/extractor/extractors.py |  1 -
 2 files changed, 59 deletions(-)
 delete mode 100644 youtube_dl/extractor/exfm.py

diff --git a/youtube_dl/extractor/exfm.py b/youtube_dl/extractor/exfm.py
deleted file mode 100644
index 09ed4f2b5..000000000
--- a/youtube_dl/extractor/exfm.py
+++ /dev/null
@@ -1,58 +0,0 @@
-from __future__ import unicode_literals
-
-import re
-
-from .common import InfoExtractor
-
-
-class ExfmIE(InfoExtractor):
-    IE_NAME = 'exfm'
-    IE_DESC = 'ex.fm'
-    _VALID_URL = r'https?://(?:www\.)?ex\.fm/song/(?P<id>[^/]+)'
-    _SOUNDCLOUD_URL = r'http://(?:www\.)?api\.soundcloud\.com/tracks/([^/]+)/stream'
-    _TESTS = [
-        {
-            'url': 'http://ex.fm/song/eh359',
-            'md5': 'e45513df5631e6d760970b14cc0c11e7',
-            'info_dict': {
-                'id': '44216187',
-                'ext': 'mp3',
-                'title': 'Test House "Love Is Not Enough" (Extended Mix) DeadJournalist Exclusive',
-                'uploader': 'deadjournalist',
-                'upload_date': '20120424',
-                'description': 'Test House \"Love Is Not Enough\" (Extended Mix) DeadJournalist Exclusive',
-            },
-            'note': 'Soundcloud song',
-            'skip': 'The site is down too often',
-        },
-        {
-            'url': 'http://ex.fm/song/wddt8',
-            'md5': '966bd70741ac5b8570d8e45bfaed3643',
-            'info_dict': {
-                'id': 'wddt8',
-                'ext': 'mp3',
-                'title': 'Safe and Sound',
-                'uploader': 'Capital Cities',
-            },
-            'skip': 'The site is down too often',
-        },
-    ]
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        song_id = mobj.group('id')
-        info_url = 'http://ex.fm/api/v3/song/%s' % song_id
-        info = self._download_json(info_url, song_id)['song']
-        song_url = info['url']
-        if re.match(self._SOUNDCLOUD_URL, song_url) is not None:
-            self.to_screen('Soundcloud song detected')
-            return self.url_result(song_url.replace('/stream', ''), 'Soundcloud')
-        return {
-            'id': song_id,
-            'url': song_url,
-            'ext': 'mp3',
-            'title': info['title'],
-            'thumbnail': info['image']['large'],
-            'uploader': info['artist'],
-            'view_count': info['loved_count'],
-        }
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 0c2436b67..7b59d5db2 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -261,7 +261,6 @@ from .espn import ESPNIE
 from .esri import EsriVideoIE
 from .europa import EuropaIE
 from .everyonesmixtape import EveryonesMixtapeIE
-from .exfm import ExfmIE
 from .expotv import ExpoTVIE
 from .extremetube import ExtremeTubeIE
 from .eyedotv import EyedoTVIE

From af95ee94b4554449db175ae44060a66c89bd96ed Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 1 Sep 2016 23:38:49 +0700
Subject: [PATCH 5955/6219] [glide] Fix extraction (Closes #10478)

---
 youtube_dl/extractor/glide.py | 15 +++------------
 1 file changed, 3 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/glide.py b/youtube_dl/extractor/glide.py
index 62ff84835..50f698803 100644
--- a/youtube_dl/extractor/glide.py
+++ b/youtube_dl/extractor/glide.py
@@ -14,10 +14,8 @@ class GlideIE(InfoExtractor):
         'info_dict': {
             'id': 'UZF8zlmuQbe4mr+7dCiQ0w==',
             'ext': 'mp4',
-            'title': 'Damon Timm\'s Glide message',
+            'title': "Damon's Glide message",
             'thumbnail': 're:^https?://.*?\.cloudfront\.net/.*\.jpg$',
-            'uploader': 'Damon Timm',
-            'upload_date': '20140919',
         }
     }
 
@@ -27,7 +25,8 @@ class GlideIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         title = self._html_search_regex(
-            r'<title>(.+?)</title>', webpage, 'title')
+            r'<title>(.+?)</title>', webpage,
+            'title', default=None) or self._og_search_title(webpage)
         video_url = self._proto_relative_url(self._search_regex(
             r'<source[^>]+src=(["\'])(?P<url>.+?)\1',
             webpage, 'video URL', default=None,
@@ -36,18 +35,10 @@ class GlideIE(InfoExtractor):
             r'<img[^>]+id=["\']video-thumbnail["\'][^>]+src=(["\'])(?P<url>.+?)\1',
             webpage, 'thumbnail url', default=None,
             group='url')) or self._og_search_thumbnail(webpage)
-        uploader = self._search_regex(
-            r'<div[^>]+class=["\']info-name["\'][^>]*>([^<]+)',
-            webpage, 'uploader', fatal=False)
-        upload_date = unified_strdate(self._search_regex(
-            r'<div[^>]+class="info-date"[^>]*>([^<]+)',
-            webpage, 'upload date', fatal=False))
 
         return {
             'id': video_id,
             'title': title,
             'url': video_url,
             'thumbnail': thumbnail,
-            'uploader': uploader,
-            'upload_date': upload_date,
         }

From 8276d3b87a54f43ca2f47b7709a6557ea979327c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 1 Sep 2016 23:46:15 +0700
Subject: [PATCH 5956/6219] [thestar] Fix extraction (Closes #10465)

---
 youtube_dl/extractor/thestar.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/thestar.py b/youtube_dl/extractor/thestar.py
index ba1380abc..c3f118894 100644
--- a/youtube_dl/extractor/thestar.py
+++ b/youtube_dl/extractor/thestar.py
@@ -2,8 +2,6 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from .brightcove import BrightcoveLegacyIE
-from ..compat import compat_parse_qs
 
 
 class TheStarIE(InfoExtractor):
@@ -30,6 +28,9 @@ class TheStarIE(InfoExtractor):
     def _real_extract(self, url):
         display_id = self._match_id(url)
         webpage = self._download_webpage(url, display_id)
-        brightcove_legacy_url = BrightcoveLegacyIE._extract_brightcove_url(webpage)
-        brightcove_id = compat_parse_qs(brightcove_legacy_url)['@videoPlayer'][0]
-        return self.url_result(self.BRIGHTCOVE_URL_TEMPLATE % brightcove_id, 'BrightcoveNew', brightcove_id)
+        brightcove_id = self._search_regex(
+            r'mainartBrightcoveVideoId["\']?\s*:\s*["\']?(\d+)',
+            webpage, 'brightcove id')
+        return self.url_result(
+            self.BRIGHTCOVE_URL_TEMPLATE % brightcove_id,
+            'BrightcoveNew', brightcove_id)

From f97ec8bcb95b45d9a657392cd24eabfadb4053e5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 1 Sep 2016 23:46:58 +0700
Subject: [PATCH 5957/6219] [glide] Remove unused import

---
 youtube_dl/extractor/glide.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/glide.py b/youtube_dl/extractor/glide.py
index 50f698803..f0d951396 100644
--- a/youtube_dl/extractor/glide.py
+++ b/youtube_dl/extractor/glide.py
@@ -2,7 +2,6 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..utils import unified_strdate
 
 
 class GlideIE(InfoExtractor):

From 4191779dcda8a80faf6e53579e011b63ee5c3878 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 1 Sep 2016 19:07:41 +0100
Subject: [PATCH 5958/6219] [nytimes] improve extraction

---
 youtube_dl/extractor/nytimes.py | 93 +++++++++++++++++++++++----------
 1 file changed, 64 insertions(+), 29 deletions(-)

diff --git a/youtube_dl/extractor/nytimes.py b/youtube_dl/extractor/nytimes.py
index 681683e86..142c34256 100644
--- a/youtube_dl/extractor/nytimes.py
+++ b/youtube_dl/extractor/nytimes.py
@@ -1,26 +1,37 @@
 from __future__ import unicode_literals
 
+import hmac
+import hashlib
+import base64
+
 from .common import InfoExtractor
 from ..utils import (
     float_or_none,
     int_or_none,
     parse_iso8601,
+    mimetype2ext,
+    determine_ext,
 )
 
 
 class NYTimesBaseIE(InfoExtractor):
+    _SECRET = b'pX(2MbU2);4N{7J8)>YwKRJ+/pQ3JkiU2Q^V>mFYv6g6gYvt6v'
+
     def _extract_video_from_id(self, video_id):
-        video_data = self._download_json(
-            'http://www.nytimes.com/svc/video/api/v2/video/%s' % video_id,
-            video_id, 'Downloading video JSON')
+        # Authorization generation algorithm is reverse engineered from `signer` in
+        # http://graphics8.nytimes.com/video/vhs/vhs-2.x.min.js
+        path = '/svc/video/api/v3/video/' + video_id
+        hm = hmac.new(self._SECRET, (path + ':vhs').encode(), hashlib.sha512).hexdigest()
+        video_data = self._download_json('http://www.nytimes.com' + path, video_id, 'Downloading video JSON', headers={
+            'Authorization': 'NYTV ' + base64.b64encode(hm.encode()).decode(),
+            'X-NYTV': 'vhs',
+        }, fatal=False)
+        if not video_data:
+            video_data = self._download_json(
+                'http://www.nytimes.com/svc/video/api/v2/video/' + video_id,
+                video_id, 'Downloading video JSON')
 
         title = video_data['headline']
-        description = video_data.get('summary')
-        duration = float_or_none(video_data.get('duration'), 1000)
-
-        uploader = video_data.get('byline')
-        publication_date = video_data.get('publication_date')
-        timestamp = parse_iso8601(publication_date[:-8]) if publication_date else None
 
         def get_file_size(file_size):
             if isinstance(file_size, int):
@@ -28,35 +39,59 @@ class NYTimesBaseIE(InfoExtractor):
             elif isinstance(file_size, dict):
                 return int(file_size.get('value', 0))
             else:
-                return 0
+                return None
 
-        formats = [
-            {
-                'url': video['url'],
-                'format_id': video.get('type'),
-                'vcodec': video.get('video_codec'),
-                'width': int_or_none(video.get('width')),
-                'height': int_or_none(video.get('height')),
-                'filesize': get_file_size(video.get('fileSize')),
-            } for video in video_data['renditions'] if video.get('url')
-        ]
+        urls = []
+        formats = []
+        for video in video_data.get('renditions', []):
+            video_url = video.get('url')
+            format_id = video.get('type')
+            if not video_url or format_id == 'thumbs' or video_url in urls:
+                continue
+            urls.append(video_url)
+            ext = mimetype2ext(video.get('mimetype')) or determine_ext(video_url)
+            if ext == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    video_url, video_id, 'mp4', 'm3u8_native',
+                    m3u8_id=format_id or 'hls', fatal=False))
+            elif ext == 'mpd':
+                continue
+            #     formats.extend(self._extract_mpd_formats(
+            #         video_url, video_id, format_id or 'dash', fatal=False))
+            else:
+                formats.append({
+                    'url': video_url,
+                    'format_id': format_id,
+                    'vcodec': video.get('videoencoding') or video.get('video_codec'),
+                    'width': int_or_none(video.get('width')),
+                    'height': int_or_none(video.get('height')),
+                    'filesize': get_file_size(video.get('file_size') or video.get('fileSize')),
+                    'tbr': int_or_none(video.get('bitrate'), 1000),
+                    'ext': ext,
+                })
         self._sort_formats(formats)
 
-        thumbnails = [
-            {
-                'url': 'http://www.nytimes.com/%s' % image['url'],
+        thumbnails = []
+        for image in video_data.get('images', []):
+            image_url = image.get('url')
+            if not image_url:
+                continue
+            thumbnails.append({
+                'url': 'http://www.nytimes.com/' + image_url,
                 'width': int_or_none(image.get('width')),
                 'height': int_or_none(image.get('height')),
-            } for image in video_data.get('images', []) if image.get('url')
-        ]
+            })
+
+        publication_date = video_data.get('publication_date')
+        timestamp = parse_iso8601(publication_date[:-8]) if publication_date else None
 
         return {
             'id': video_id,
             'title': title,
-            'description': description,
+            'description': video_data.get('summary'),
             'timestamp': timestamp,
-            'uploader': uploader,
-            'duration': duration,
+            'uploader': video_data.get('byline'),
+            'duration': float_or_none(video_data.get('duration'), 1000),
             'formats': formats,
             'thumbnails': thumbnails,
         }
@@ -67,7 +102,7 @@ class NYTimesIE(NYTimesBaseIE):
 
     _TESTS = [{
         'url': 'http://www.nytimes.com/video/opinion/100000002847155/verbatim-what-is-a-photocopier.html?playlistId=100000001150263',
-        'md5': '18a525a510f942ada2720db5f31644c0',
+        'md5': 'd665342765db043f7e225cff19df0f2d',
         'info_dict': {
             'id': '100000002847155',
             'ext': 'mov',

From b207d5ebd4eab80e07673aba9696d240d1009bcf Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 1 Sep 2016 19:46:58 +0100
Subject: [PATCH 5959/6219] [curiositystream] don't cache auth token

---
 youtube_dl/extractor/curiositystream.py | 28 +++++++++----------------
 1 file changed, 10 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/curiositystream.py b/youtube_dl/extractor/curiositystream.py
index 7105e3c4c..e3c99468c 100644
--- a/youtube_dl/extractor/curiositystream.py
+++ b/youtube_dl/extractor/curiositystream.py
@@ -33,24 +33,16 @@ class CuriosityStreamBaseIE(InfoExtractor):
         return result['data']
 
     def _real_initialize(self):
-        if not self._auth_token:
-            user = self._downloader.cache.load('curiositystream', 'user') or {}
-            self._auth_token = user.get('auth_token')
-            if not self._auth_token:
-                (email, password) = self._get_login_info()
-                if email is None:
-                    return
-                result = self._download_json(
-                    self._API_BASE_URL + 'login', None, data=urlencode_postdata({
-                        'email': email,
-                        'password': password,
-                    }))
-                self._handle_errors(result)
-                self._auth_token = result['message']['auth_token']
-                self._downloader.cache.store(
-                    'curiositystream', 'user', {
-                        'auth_token': self._auth_token,
-                    })
+        (email, password) = self._get_login_info()
+        if email is None:
+            return
+        result = self._download_json(
+            self._API_BASE_URL + 'login', None, data=urlencode_postdata({
+                'email': email,
+                'password': password,
+            }))
+        self._handle_errors(result)
+        self._auth_token = result['message']['auth_token']
 
     def _extract_media_info(self, media):
         video_id = compat_str(media['id'])

From 6150502e4709b6b2ebc226c9c38fa346b9358699 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 1 Sep 2016 22:14:40 +0100
Subject: [PATCH 5960/6219] [adobepass] check for authz_token
 expiration(#10527)

---
 youtube_dl/extractor/adobepass.py | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/adobepass.py b/youtube_dl/extractor/adobepass.py
index 9e3a3e362..68ec37e00 100644
--- a/youtube_dl/extractor/adobepass.py
+++ b/youtube_dl/extractor/adobepass.py
@@ -37,6 +37,10 @@ class AdobePassIE(InfoExtractor):
             return self._search_regex(
                 '<%s>(.+?)</%s>' % (tag, tag), xml_str, tag)
 
+        def is_expired(token, date_ele):
+            token_expires = unified_timestamp(re.sub(r'[_ ]GMT', '', xml_text(token, date_ele)))
+            return token_expires and token_expires <= int(time.time())
+
         mvpd_headers = {
             'ap_42': 'anonymous',
             'ap_11': 'Linux i686',
@@ -47,11 +51,8 @@ class AdobePassIE(InfoExtractor):
         guid = xml_text(resource, 'guid')
         requestor_info = self._downloader.cache.load('mvpd', requestor_id) or {}
         authn_token = requestor_info.get('authn_token')
-        if authn_token:
-            token_expires = unified_timestamp(re.sub(r'[_ ]GMT', '', xml_text(authn_token, 'simpleTokenExpires')))
-            if token_expires and token_expires <= int(time.time()):
-                authn_token = None
-                requestor_info = {}
+        if authn_token and is_expired(authn_token, 'simpleTokenExpires'):
+            authn_token = None
         if not authn_token:
             # TODO add support for other TV Providers
             mso_id = 'DTV'
@@ -98,6 +99,8 @@ class AdobePassIE(InfoExtractor):
             self._downloader.cache.store('mvpd', requestor_id, requestor_info)
 
         authz_token = requestor_info.get(guid)
+        if authz_token and is_expired(authz_token, 'simpleTokenTTL'):
+            authz_token = None
         if not authz_token:
             authorize = self._download_webpage(
                 self._SERVICE_PROVIDER_TEMPLATE % 'authorize', video_id,

From 2c3e0af93e00d7e2e20283be12541aaebabfa1bf Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 2 Sep 2016 09:53:04 +0100
Subject: [PATCH 5961/6219] [go] Add new extractor

---
 youtube_dl/extractor/extractors.py |   1 +
 youtube_dl/extractor/go.py         | 101 +++++++++++++++++++++++++++++
 2 files changed, 102 insertions(+)
 create mode 100644 youtube_dl/extractor/go.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 7b59d5db2..2bcd5a0cd 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -318,6 +318,7 @@ from .globo import (
     GloboIE,
     GloboArticleIE,
 )
+from .go import GoIE
 from .godtube import GodTubeIE
 from .godtv import GodTVIE
 from .golem import GolemIE
diff --git a/youtube_dl/extractor/go.py b/youtube_dl/extractor/go.py
new file mode 100644
index 000000000..6a437c54d
--- /dev/null
+++ b/youtube_dl/extractor/go.py
@@ -0,0 +1,101 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    determine_ext,
+    parse_age_limit,
+)
+
+
+class GoIE(InfoExtractor):
+    _BRANDS = {
+        'abc': '001',
+        'freeform': '002',
+        'watchdisneychannel': '004',
+        'watchdisneyjunior': '008',
+        'watchdisneyxd': '009',
+    }
+    _VALID_URL = r'https?://(?:(?P<sub_domain>%s)\.)?go\.com/.*?vdka(?P<id>\w+)' % '|'.join(_BRANDS.keys())
+    _TESTS = [{
+        'url': 'http://abc.go.com/shows/castle/video/most-recent/vdka0_g86w5onx',
+        'info_dict': {
+            'id': '0_g86w5onx',
+            'ext': 'mp4',
+            'title': 'Sneak Peek: Language Arts',
+            'description': 'md5:7dcdab3b2d17e5217c953256af964e9c',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }, {
+        'url': 'http://abc.go.com/shows/after-paradise/video/most-recent/vdka3335601',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        sub_domain, video_id = re.match(self._VALID_URL, url).groups()
+        video_data = self._download_json(
+            'http://api.contents.watchabc.go.com/vp2/ws/contents/3000/videos/%s/001/-1/-1/-1/%s/-1/-1.json' % (self._BRANDS[sub_domain], video_id),
+            video_id)['video'][0]
+        title = video_data['title']
+
+        formats = []
+        for asset in video_data.get('assets', {}).get('asset', []):
+            asset_url = asset.get('value')
+            if not asset_url:
+                continue
+            format_id = asset.get('format')
+            ext = determine_ext(asset_url)
+            if ext == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    asset_url, video_id, 'mp4', m3u8_id=format_id or 'hls', fatal=False))
+            else:
+                formats.append({
+                    'format_id': format_id,
+                    'url': asset_url,
+                    'ext': ext,
+                })
+        self._sort_formats(formats)
+
+        subtitles = {}
+        for cc in video_data.get('closedcaption', {}).get('src', []):
+            cc_url = cc.get('value')
+            if not cc_url:
+                continue
+            ext = determine_ext(cc_url)
+            if ext == 'xml':
+                ext = 'ttml'
+            subtitles.setdefault(cc.get('lang'), []).append({
+                'url': cc_url,
+                'ext': ext,
+            })
+
+        thumbnails = []
+        for thumbnail in video_data.get('thumbnails', {}).get('thumbnail', []):
+            thumbnail_url = thumbnail.get('value')
+            if not thumbnail_url:
+                continue
+            thumbnails.append({
+                'url': thumbnail_url,
+                'width': int_or_none(thumbnail.get('width')),
+                'height': int_or_none(thumbnail.get('height')),
+            })
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': video_data.get('longdescription') or video_data.get('description'),
+            'duration': int_or_none(video_data.get('duration', {}).get('value'), 1000),
+            'age_limit': parse_age_limit(video_data.get('tvrating', {}).get('rating')),
+            'episode_number': int_or_none(video_data.get('episodenumber')),
+            'series': video_data.get('show', {}).get('title'),
+            'season_number': int_or_none(video_data.get('season', {}).get('num')),
+            'thumbnails': thumbnails,
+            'formats': formats,
+            'subtitles': subtitles,
+        }

From 349fc5c705d6b81ae53d698972f40b1125bee13e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 2 Sep 2016 21:13:50 +0700
Subject: [PATCH 5962/6219] [facebook:plugins:video] Add extractor (Closes
 #10530)

---
 youtube_dl/extractor/extractors.py |  5 ++++-
 youtube_dl/extractor/facebook.py   | 29 +++++++++++++++++++++++++++++
 2 files changed, 33 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 2bcd5a0cd..bc616223e 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -264,7 +264,10 @@ from .everyonesmixtape import EveryonesMixtapeIE
 from .expotv import ExpoTVIE
 from .extremetube import ExtremeTubeIE
 from .eyedotv import EyedoTVIE
-from .facebook import FacebookIE
+from .facebook import (
+    FacebookIE,
+    FacebookPluginsVideoIE,
+)
 from .faz import FazIE
 from .fc2 import FC2IE
 from .fczenit import FczenitIE
diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index 228b0b6d7..3a220e995 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -351,3 +351,32 @@ class FacebookIE(InfoExtractor):
                 self._VIDEO_PAGE_TEMPLATE % video_id,
                 video_id, fatal_if_no_video=True)
             return info_dict
+
+
+class FacebookPluginsVideoIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:[\w-]+\.)?facebook\.com/plugins/video\.php\?.*?\bhref=(?P<id>https.+)'
+
+    _TESTS = [{
+        'url': 'https://www.facebook.com/plugins/video.php?href=https%3A%2F%2Fwww.facebook.com%2Fgov.sg%2Fvideos%2F10154383743583686%2F&show_text=0&width=560',
+        'md5': '5954e92cdfe51fe5782ae9bda7058a07',
+        'info_dict': {
+            'id': '10154383743583686',
+            'ext': 'mp4',
+            'title': 'What to do during the haze?',
+            'uploader': 'Gov.sg',
+            'upload_date': '20160826',
+            'timestamp': 1472184808,
+        },
+        'add_ie': [FacebookIE.ie_key()],
+    }, {
+        'url': 'https://www.facebook.com/plugins/video.php?href=https%3A%2F%2Fwww.facebook.com%2Fvideo.php%3Fv%3D10204634152394104',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.facebook.com/plugins/video.php?href=https://www.facebook.com/gov.sg/videos/10154383743583686/&show_text=0&width=560',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        return self.url_result(
+            compat_urllib_parse_unquote(self._match_id(url)),
+            FacebookIE.ie_key())

From 5e9e3d0f6bf2055c557f360758d6d7eb146edcba Mon Sep 17 00:00:00 2001
From: Sebastian Blunt <sebastian@c4k3.net>
Date: Fri, 2 Sep 2016 14:48:56 +0200
Subject: [PATCH 5963/6219] [drtv] Add support for dr.dk/nyheder

It's the same video player, the only difference is that the video player
is loaded differently, and certain metadata (title and description) is
not available under dr.dk/mu, so make it by default get that from some
of the html meta tags.

Skip the dr.dk/tv test

dr.dk/tv videos are only available for between 7 and 90 days due to
Danish law, and in certain cases may be readded. Skip this test as it is
no longer available.
---
 youtube_dl/extractor/drtv.py | 28 ++++++++++++++++++++++------
 1 file changed, 22 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/drtv.py b/youtube_dl/extractor/drtv.py
index 2d74ff855..e210cb610 100644
--- a/youtube_dl/extractor/drtv.py
+++ b/youtube_dl/extractor/drtv.py
@@ -5,13 +5,14 @@ from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
     parse_iso8601,
+    remove_end,
 )
 
 
 class DRTVIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?dr\.dk/tv/se/(?:[^/]+/)*(?P<id>[\da-z-]+)(?:[/#?]|$)'
+    _VALID_URL = r'https?://(?:www\.)?dr\.dk/(?:tv/se|nyheder)/(?:[^/]+/)*(?P<id>[\da-z-]+)(?:[/#?]|$)'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'https://www.dr.dk/tv/se/boern/ultra/panisk-paske/panisk-paske-5',
         'md5': 'dc515a9ab50577fa14cc4e4b0265168f',
         'info_dict': {
@@ -23,7 +24,20 @@ class DRTVIE(InfoExtractor):
             'upload_date': '20150322',
             'duration': 1455,
         },
-    }
+        'skip': 'Video is no longer available',
+    }, {
+        'url': 'https://www.dr.dk/nyheder/indland/live-christianias-rydning-af-pusher-street-er-i-gang',
+        'md5': '2ada5074f9e79afc0d324a8e9784d850',
+        'info_dict': {
+            'id': 'christiania-pusher-street-ryddes-drdkrjpo',
+            'ext': 'mp4',
+            'title': 'LIVE Christianias rydning af Pusher Street er i gang',
+            'description': '- Det er det fedeste, der er sket i 20 år, fortæller christianit til DR Nyheder.',
+            'timestamp': 1472800279,
+            'upload_date': '20160902',
+            'duration': 131.4,
+        }
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -35,7 +49,8 @@ class DRTVIE(InfoExtractor):
                 'Video %s is not available' % video_id, expected=True)
 
         video_id = self._search_regex(
-            r'data-(?:material-identifier|episode-slug)="([^"]+)"',
+            (r'data-(?:material-identifier|episode-slug)="([^"]+)"',
+                r'data-resource="[^>"]+mu/programcard/expanded/([^"]+)"'),
             webpage, 'video id')
 
         programcard = self._download_json(
@@ -43,8 +58,9 @@ class DRTVIE(InfoExtractor):
             video_id, 'Downloading video JSON')
         data = programcard['Data'][0]
 
-        title = data['Title']
-        description = data['Description']
+        title = remove_end(self._og_search_title(webpage), ' | TV | DR') or data['Title']
+        description = self._og_search_description(webpage) or data['Description']
+
         timestamp = parse_iso8601(data['CreatedTime'])
 
         thumbnail = None

From 6562d34a8cbdb93de77a8042f7409ebe31e3e3e4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 2 Sep 2016 22:57:48 +0700
Subject: [PATCH 5964/6219] [utils] Improve mimetype2ext

---
 test/test_utils.py  | 9 +++++++++
 youtube_dl/utils.py | 2 +-
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index d16ea7f77..405c5d351 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -39,6 +39,7 @@ from youtube_dl.utils import (
     is_html,
     js_to_json,
     limit_length,
+    mimetype2ext,
     ohdave_rsa_encrypt,
     OnDemandPagedList,
     orderedSet,
@@ -625,6 +626,14 @@ class TestUtil(unittest.TestCase):
             limit_length('foo bar baz asd', 12).startswith('foo bar'))
         self.assertTrue('...' in limit_length('foo bar baz asd', 12))
 
+    def test_mimetype2ext(self):
+        self.assertEqual(mimetype2ext(None), None)
+        self.assertEqual(mimetype2ext('video/x-flv'), 'flv')
+        self.assertEqual(mimetype2ext('application/x-mpegURL'), 'm3u8')
+        self.assertEqual(mimetype2ext('text/vtt'), 'vtt')
+        self.assertEqual(mimetype2ext('text/vtt;charset=utf-8'), 'vtt')
+        self.assertEqual(mimetype2ext('text/html; charset=utf-8'), 'html')
+
     def test_parse_codecs(self):
         self.assertEqual(parse_codecs(''), {})
         self.assertEqual(parse_codecs('avc1.77.30, mp4a.40.2'), {
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 1091f17f3..904f23fd7 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2148,7 +2148,7 @@ def mimetype2ext(mt):
         return ext
 
     _, _, res = mt.rpartition('/')
-    res = res.lower()
+    res = res.split(';')[0].strip().lower()
 
     return {
         '3gpp': '3gp',

From 6066d03db02b9c545435b2b8faffe2e0f6c66702 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 2 Sep 2016 23:02:15 +0700
Subject: [PATCH 5965/6219] [drtv] Modernize and make more robust

---
 youtube_dl/extractor/drtv.py | 53 ++++++++++++++++++++++--------------
 1 file changed, 33 insertions(+), 20 deletions(-)

diff --git a/youtube_dl/extractor/drtv.py b/youtube_dl/extractor/drtv.py
index e210cb610..7122449a3 100644
--- a/youtube_dl/extractor/drtv.py
+++ b/youtube_dl/extractor/drtv.py
@@ -4,6 +4,9 @@ from __future__ import unicode_literals
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
+    int_or_none,
+    float_or_none,
+    mimetype2ext,
     parse_iso8601,
     remove_end,
 )
@@ -58,10 +61,12 @@ class DRTVIE(InfoExtractor):
             video_id, 'Downloading video JSON')
         data = programcard['Data'][0]
 
-        title = remove_end(self._og_search_title(webpage), ' | TV | DR') or data['Title']
-        description = self._og_search_description(webpage) or data['Description']
+        title = remove_end(self._og_search_title(
+            webpage, default=None), ' | TV | DR') or data['Title']
+        description = self._og_search_description(
+            webpage, default=None) or data.get('Description')
 
-        timestamp = parse_iso8601(data['CreatedTime'])
+        timestamp = parse_iso8601(data.get('CreatedTime'))
 
         thumbnail = None
         duration = None
@@ -72,16 +77,18 @@ class DRTVIE(InfoExtractor):
         subtitles = {}
 
         for asset in data['Assets']:
-            if asset['Kind'] == 'Image':
-                thumbnail = asset['Uri']
-            elif asset['Kind'] == 'VideoResource':
-                duration = asset['DurationInMilliseconds'] / 1000.0
-                restricted_to_denmark = asset['RestrictedToDenmark']
-                spoken_subtitles = asset['Target'] == 'SpokenSubtitles'
-                for link in asset['Links']:
-                    uri = link['Uri']
-                    target = link['Target']
-                    format_id = target
+            if asset.get('Kind') == 'Image':
+                thumbnail = asset.get('Uri')
+            elif asset.get('Kind') == 'VideoResource':
+                duration = float_or_none(asset.get('DurationInMilliseconds'), 1000)
+                restricted_to_denmark = asset.get('RestrictedToDenmark')
+                spoken_subtitles = asset.get('Target') == 'SpokenSubtitles'
+                for link in asset.get('Links', []):
+                    uri = link.get('Uri')
+                    if not uri:
+                        continue
+                    target = link.get('Target')
+                    format_id = target or ''
                     preference = None
                     if spoken_subtitles:
                         preference = -1
@@ -92,8 +99,8 @@ class DRTVIE(InfoExtractor):
                             video_id, preference, f4m_id=format_id))
                     elif target == 'HLS':
                         formats.extend(self._extract_m3u8_formats(
-                            uri, video_id, 'mp4', preference=preference,
-                            m3u8_id=format_id))
+                            uri, video_id, 'mp4', entry_protocol='m3u8_native',
+                            preference=preference, m3u8_id=format_id))
                     else:
                         bitrate = link.get('Bitrate')
                         if bitrate:
@@ -101,7 +108,7 @@ class DRTVIE(InfoExtractor):
                         formats.append({
                             'url': uri,
                             'format_id': format_id,
-                            'tbr': bitrate,
+                            'tbr': int_or_none(bitrate),
                             'ext': link.get('FileFormat'),
                         })
                 subtitles_list = asset.get('SubtitlesList')
@@ -110,12 +117,18 @@ class DRTVIE(InfoExtractor):
                         'Danish': 'da',
                     }
                     for subs in subtitles_list:
-                        lang = subs['Language']
-                        subtitles[LANGS.get(lang, lang)] = [{'url': subs['Uri'], 'ext': 'vtt'}]
+                        if not subs.get('Uri'):
+                            continue
+                        lang = subs.get('Language') or 'da'
+                        subtitles.setdefault(LANGS.get(lang, lang), []).append({
+                            'url': subs['Uri'],
+                            'ext': mimetype2ext(subs.get('MimeType')) or 'vtt'
+                        })
 
         if not formats and restricted_to_denmark:
-            raise ExtractorError(
-                'Unfortunately, DR is not allowed to show this program outside Denmark.', expected=True)
+            self.raise_geo_restricted(
+                'Unfortunately, DR is not allowed to show this program outside Denmark.',
+                expected=True)
 
         self._sort_formats(formats)
 

From dacb3a864a8c89edb312cd28c3de1605a5467d0f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 2 Sep 2016 23:43:20 +0700
Subject: [PATCH 5966/6219] [youtube:playlist] Fallback to video extraction for
 video/playlist URLs when playlist is broken (Closes #10537)

---
 youtube_dl/extractor/youtube.py | 56 +++++++++++++++++++++++++++++----
 1 file changed, 50 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index d5d5b7334..ea98fbf69 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1841,6 +1841,28 @@ class YoutubePlaylistIE(YoutubePlaylistBaseInfoExtractor):
             'id': 'UUXw-G3eDE9trcvY2sBMM_aA',
         },
         'playlist_mincout': 21,
+    }, {
+        # Playlist URL that does not actually serve a playlist
+        'url': 'https://www.youtube.com/watch?v=FqZTN594JQw&list=PLMYEtVRpaqY00V9W81Cwmzp6N6vZqfUKD4',
+        'info_dict': {
+            'id': 'FqZTN594JQw',
+            'ext': 'webm',
+            'title': "Smiley's People 01 detective, Adventure Series, Action",
+            'uploader': 'STREEM',
+            'uploader_id': 'UCyPhqAZgwYWZfxElWVbVJng',
+            'uploader_url': 're:https?://(?:www\.)?youtube\.com/channel/UCyPhqAZgwYWZfxElWVbVJng',
+            'upload_date': '20150526',
+            'license': 'Standard YouTube License',
+            'description': 'md5:507cdcb5a49ac0da37a920ece610be80',
+            'categories': ['People & Blogs'],
+            'tags': list,
+            'like_count': int,
+            'dislike_count': int,
+        },
+        'params': {
+            'skip_download': True,
+        },
+        'add_ie': [YoutubeIE.ie_key()],
     }]
 
     def _real_initialize(self):
@@ -1901,9 +1923,20 @@ class YoutubePlaylistIE(YoutubePlaylistBaseInfoExtractor):
 
         playlist_title = self._html_search_regex(
             r'(?s)<h1 class="pl-header-title[^"]*"[^>]*>\s*(.*?)\s*</h1>',
-            page, 'title')
+            page, 'title', default=None)
 
-        return self.playlist_result(self._entries(page, playlist_id), playlist_id, playlist_title)
+        has_videos = True
+
+        if not playlist_title:
+            try:
+                # Some playlist URLs don't actually serve a playlist (e.g.
+                # https://www.youtube.com/watch?v=FqZTN594JQw&list=PLMYEtVRpaqY00V9W81Cwmzp6N6vZqfUKD4)
+                next(self._entries(page, playlist_id))
+            except StopIteration:
+                has_videos = False
+
+        return has_videos, self.playlist_result(
+            self._entries(page, playlist_id), playlist_id, playlist_title)
 
     def _check_download_just_video(self, url, playlist_id):
         # Check if it's a video-specific URL
@@ -1912,9 +1945,11 @@ class YoutubePlaylistIE(YoutubePlaylistBaseInfoExtractor):
             video_id = query_dict['v'][0]
             if self._downloader.params.get('noplaylist'):
                 self.to_screen('Downloading just video %s because of --no-playlist' % video_id)
-                return self.url_result(video_id, 'Youtube', video_id=video_id)
+                return video_id, self.url_result(video_id, 'Youtube', video_id=video_id)
             else:
                 self.to_screen('Downloading playlist %s - add --no-playlist to just download video %s' % (playlist_id, video_id))
+                return video_id, None
+        return None, None
 
     def _real_extract(self, url):
         # Extract playlist id
@@ -1923,7 +1958,7 @@ class YoutubePlaylistIE(YoutubePlaylistBaseInfoExtractor):
             raise ExtractorError('Invalid URL: %s' % url)
         playlist_id = mobj.group(1) or mobj.group(2)
 
-        video = self._check_download_just_video(url, playlist_id)
+        video_id, video = self._check_download_just_video(url, playlist_id)
         if video:
             return video
 
@@ -1931,7 +1966,15 @@ class YoutubePlaylistIE(YoutubePlaylistBaseInfoExtractor):
             # Mixes require a custom extraction process
             return self._extract_mix(playlist_id)
 
-        return self._extract_playlist(playlist_id)
+        has_videos, playlist = self._extract_playlist(playlist_id)
+        if has_videos or not video_id:
+            return playlist
+
+        # Some playlist URLs don't actually serve a playlist (see
+        # https://github.com/rg3/youtube-dl/issues/10537).
+        # Fallback to plain video extraction if there is a video id
+        # along with playlist id.
+        return self.url_result(video_id, 'Youtube', video_id=video_id)
 
 
 class YoutubeChannelIE(YoutubePlaylistBaseInfoExtractor):
@@ -2312,7 +2355,8 @@ class YoutubeWatchLaterIE(YoutubePlaylistIE):
         video = self._check_download_just_video(url, 'WL')
         if video:
             return video
-        return self._extract_playlist('WL')
+        _, playlist = self._extract_playlist('WL')
+        return playlist
 
 
 class YoutubeFavouritesIE(YoutubeBaseInfoExtractor):

From c2b2c7e1386056698ee1b0de5427ea90abf8e9c3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 2 Sep 2016 23:50:42 +0700
Subject: [PATCH 5967/6219] [utils] Add quicktime to mimetype2ext

---
 youtube_dl/utils.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 904f23fd7..ed199c4ad 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2168,6 +2168,7 @@ def mimetype2ext(mt):
         'f4m+xml': 'f4m',
         'hds+xml': 'f4m',
         'vnd.ms-sstr+xml': 'ism',
+        'quicktime': 'mov',
     }.get(res, res)
 
 

From 3fcce30289a475901728af7a8dbe85304105b8ac Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 2 Sep 2016 23:53:17 +0700
Subject: [PATCH 5968/6219] [drtv] Update tests

---
 youtube_dl/extractor/drtv.py | 24 +++++++++++++-----------
 1 file changed, 13 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/drtv.py b/youtube_dl/extractor/drtv.py
index 7122449a3..88d096b30 100644
--- a/youtube_dl/extractor/drtv.py
+++ b/youtube_dl/extractor/drtv.py
@@ -16,21 +16,23 @@ class DRTVIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?dr\.dk/(?:tv/se|nyheder)/(?:[^/]+/)*(?P<id>[\da-z-]+)(?:[/#?]|$)'
 
     _TESTS = [{
-        'url': 'https://www.dr.dk/tv/se/boern/ultra/panisk-paske/panisk-paske-5',
-        'md5': 'dc515a9ab50577fa14cc4e4b0265168f',
+        'url': 'https://www.dr.dk/tv/se/boern/ultra/klassen-ultra/klassen-darlig-taber-10',
+        'md5': '25e659cccc9a2ed956110a299fdf5983',
         'info_dict': {
-            'id': 'panisk-paske-5',
+            'id': 'klassen-darlig-taber-10',
             'ext': 'mp4',
-            'title': 'Panisk Påske (5)',
-            'description': 'md5:ca14173c5ab24cd26b0fcc074dff391c',
-            'timestamp': 1426984612,
-            'upload_date': '20150322',
-            'duration': 1455,
+            'title': 'Klassen - Dårlig taber (10)',
+            'description': 'md5:815fe1b7fa656ed80580f31e8b3c79aa',
+            'timestamp': 1471991907,
+            'upload_date': '20160823',
+            'duration': 606.84,
+        },
+        'params': {
+            'skip_download': True,
         },
-        'skip': 'Video is no longer available',
     }, {
         'url': 'https://www.dr.dk/nyheder/indland/live-christianias-rydning-af-pusher-street-er-i-gang',
-        'md5': '2ada5074f9e79afc0d324a8e9784d850',
+        'md5': '2c37175c718155930f939ef59952474a',
         'info_dict': {
             'id': 'christiania-pusher-street-ryddes-drdkrjpo',
             'ext': 'mp4',
@@ -39,7 +41,7 @@ class DRTVIE(InfoExtractor):
             'timestamp': 1472800279,
             'upload_date': '20160902',
             'duration': 131.4,
-        }
+        },
     }]
 
     def _real_extract(self, url):

From 6496ccb41398971373a2f7162a0684dd12f0b56e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 3 Sep 2016 01:17:15 +0700
Subject: [PATCH 5969/6219] [youtube] Add support for rental videos' previews
 (Closes #10532)

---
 youtube_dl/extractor/youtube.py | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index ea98fbf69..4c8edef8d 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -844,6 +844,24 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             # YouTube Red paid video (https://github.com/rg3/youtube-dl/issues/10059)
             'url': 'https://www.youtube.com/watch?v=i1Ko8UG-Tdo',
             'only_matching': True,
+        },
+        {
+            # Rental video preview
+            'url': 'https://www.youtube.com/watch?v=yYr8q0y5Jfg',
+            'info_dict': {
+                'id': 'uGpuVWrhIzE',
+                'ext': 'mp4',
+                'title': 'Piku - Trailer',
+                'description': 'md5:c36bd60c3fd6f1954086c083c72092eb',
+                'upload_date': '20150811',
+                'uploader': 'FlixMatrix',
+                'uploader_id': 'FlixMatrixKaravan',
+                'uploader_url': 're:https?://(?:www\.)?youtube\.com/user/FlixMatrixKaravan',
+                'license': 'Standard YouTube License',
+            },
+            'params': {
+                'skip_download': True,
+            },
         }
     ]
 
@@ -1254,6 +1272,12 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     # Convert to the same format returned by compat_parse_qs
                     video_info = dict((k, [v]) for k, v in args.items())
                     add_dash_mpd(video_info)
+                # Rental video is not rented but preview is available (e.g.
+                # https://www.youtube.com/watch?v=yYr8q0y5Jfg,
+                # https://github.com/rg3/youtube-dl/issues/10532)
+                if not video_info and args.get('ypc_vid'):
+                    return self.url_result(
+                        args['ypc_vid'], YoutubeIE.ie_key(), video_id=args['ypc_vid'])
                 if args.get('livestream') == '1' or args.get('live_playback') == 1:
                     is_live = True
             if not video_info or self._downloader.params.get('youtube_include_dash_manifest', True):

From 3a7d35b982fac19ca47b87358001379fafbd5731 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 3 Sep 2016 01:42:33 +0700
Subject: [PATCH 5970/6219] Credit @C4K3 for #10536

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index b9a602c12..c4bef040a 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -182,3 +182,4 @@ Rob van Bekkum
 Petr Zvoníček
 Pratyush Singh
 Aleksander Nitecki
+Sebastian Blunt

From 4b3a6076586a38450fa9633480d175a13e33dac7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 3 Sep 2016 01:45:17 +0700
Subject: [PATCH 5971/6219] [ChangeLog] Actualize

---
 ChangeLog | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 2e75c003d..eb05fe77e 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,8 +1,32 @@
 version <unreleased>
 
+Core
+* Restore usage of NAME attribute from EXT-X-MEDIA tag for formats codes in
+  _extract_m3u8_formats (#10522)
+* Handle semicolon in mimetype2ext
+
+
 Extractors
++ [youtube] Add support for rental videos' previews (#10532)
+* [youtube:playlist] Fallback to video extraction for video/playlist URLs when
+  no playlist is actually served (#10537)
++ [drtv] Add support for dr.dk/nyheder (#10536)
++ [facebook:plugins:video] Add extractor (#10530)
++ [go] Add extractor for *.go.com sites
+* [adobepass] Check for authz_token expiration (#10527)
+* [nytimes] improve extraction
+* [thestar] Fix extraction (#10465)
+* [glide] Fix extraction (#10478)
+- [exfm] Remove extractor (#10482)
+* [youporn] Fix categories and tags extraction (#10521)
++ [curiositystream] Add extractor for app.curiositystream.com
 - [thvideo] Remove extractor (#10464)
 * [movingimage] Fix for the new site name (#10466)
++ [cbs] Add support for once formats (#10515)
+* [limelight] Skip ism snd duplicate manifests
++ [porncom] Extract categories and tags (#10510)
++ [facebook] Extract timestamp (#10508)
++ [yahoo] Extract more formats
 
 
 version 2016.08.31

From 86c3bbbcede6efa175f5a93e02511fe32585521f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 3 Sep 2016 01:46:41 +0700
Subject: [PATCH 5972/6219] release 2016.09.03

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 9 +++++----
 youtube_dl/version.py     | 2 +-
 4 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 2caca5115..fc18e733b 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.08.31*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.08.31**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.09.03*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.09.03**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.08.31
+[debug] youtube-dl version 2016.09.03
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index eb05fe77e..68dbeb696 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2016.09.03
 
 Core
 * Restore usage of NAME attribute from EXT-X-MEDIA tag for formats codes in
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 42bf291e2..015332bca 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -171,6 +171,8 @@
  - **CTVNews**
  - **culturebox.francetvinfo.fr**
  - **CultureUnplugged**
+ - **curiositystream**
+ - **curiositystream:collection**
  - **CWTV**
  - **DailyMail**
  - **dailymotion**
@@ -223,11 +225,11 @@
  - **EsriVideo**
  - **Europa**
  - **EveryonesMixtape**
- - **exfm**: ex.fm
  - **ExpoTV**
  - **ExtremeTube**
  - **EyedoTV**
  - **facebook**
+ - **FacebookPluginsVideo**
  - **faz.net**
  - **fc2**
  - **Fczenit**
@@ -271,6 +273,7 @@
  - **Glide**: Glide mobile video messages (glide.me)
  - **Globo**
  - **GloboArticle**
+ - **Go**
  - **GodTube**
  - **GodTV**
  - **Golem**
@@ -406,6 +409,7 @@
  - **MovieClips**
  - **MovieFap**
  - **Moviezine**
+ - **MovingImage**
  - **MPORA**
  - **MSN**
  - **mtg**: MTG services
@@ -659,7 +663,6 @@
  - **sr:mediathek**: Saarländischer Rundfunk
  - **SRGSSR**
  - **SRGSSRPlay**: srf.ch, rts.ch, rsi.ch, rtr.ch and swissinfo.ch play sites
- - **SSA**
  - **stanfordoc**: Stanford Open ClassRoom
  - **Steam**
  - **Stitcher**
@@ -702,8 +705,6 @@
  - **TheStar**
  - **ThisAmericanLife**
  - **ThisAV**
- - **THVideo**
- - **THVideoPlaylist**
  - **tinypic**: tinypic.com videos
  - **tlc.de**
  - **TMZ**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index fe442dd88..5be8c0122 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.08.31'
+__version__ = '2016.09.03'

From dedb1770295d214225a3a31b5f99da877cf01eee Mon Sep 17 00:00:00 2001
From: Christian Pointner <equinox@helsinki.at>
Date: Sat, 3 Sep 2016 01:50:26 +0200
Subject: [PATCH 5973/6219] Fix parsing of HTML5 media elements

This fixes an error in _parse_html5_media_entries in case
an audio or video tag directly uses a src attribute insted
of <source> elements in it's body.
---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index a9c7a8d16..a82968162 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1749,7 +1749,7 @@ class InfoExtractor(object):
             media_attributes = extract_attributes(media_tag)
             src = media_attributes.get('src')
             if src:
-                _, formats = _media_formats(src)
+                _, formats = _media_formats(src, media_type)
                 media_info['formats'].extend(formats)
             media_info['thumbnail'] = media_attributes.get('poster')
             if media_content:

From cf0efe96366259a5f0f07ae79280bfa17dc6f6e7 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 3 Sep 2016 17:25:03 +0800
Subject: [PATCH 5974/6219] [fc2:embed] New extractor for Flash player URLs

Closes #10512
---
 ChangeLog                          |  6 ++++
 youtube_dl/extractor/extractors.py |  5 ++-
 youtube_dl/extractor/fc2.py        | 58 ++++++++++++++++++++++++++----
 3 files changed, 61 insertions(+), 8 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 68dbeb696..065fc83a8 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Extractors
++ [fc2] Recognize Flash player URLs (#10512)
+
+
 version 2016.09.03
 
 Core
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index bc616223e..d851e5f36 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -269,7 +269,10 @@ from .facebook import (
     FacebookPluginsVideoIE,
 )
 from .faz import FazIE
-from .fc2 import FC2IE
+from .fc2 import (
+    FC2IE,
+    FC2EmbedIE,
+)
 from .fczenit import FczenitIE
 from .firstpost import FirstpostIE
 from .firsttv import FirstTVIE
diff --git a/youtube_dl/extractor/fc2.py b/youtube_dl/extractor/fc2.py
index c7d69ff1f..b9e58d4df 100644
--- a/youtube_dl/extractor/fc2.py
+++ b/youtube_dl/extractor/fc2.py
@@ -1,10 +1,12 @@
-#! -*- coding: utf-8 -*-
+# coding: utf-8
 from __future__ import unicode_literals
 
 import hashlib
+import re
 
 from .common import InfoExtractor
 from ..compat import (
+    compat_parse_qs,
     compat_urllib_request,
     compat_urlparse,
 )
@@ -16,7 +18,7 @@ from ..utils import (
 
 
 class FC2IE(InfoExtractor):
-    _VALID_URL = r'^https?://video\.fc2\.com/(?:[^/]+/)*content/(?P<id>[^/]+)'
+    _VALID_URL = r'^(?:https?://video\.fc2\.com/(?:[^/]+/)*content/|fc2:)(?P<id>[^/]+)'
     IE_NAME = 'fc2'
     _NETRC_MACHINE = 'fc2'
     _TESTS = [{
@@ -75,12 +77,17 @@ class FC2IE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
         self._login()
-        webpage = self._download_webpage(url, video_id)
-        self._downloader.cookiejar.clear_session_cookies()  # must clear
-        self._login()
+        webpage = None
+        if not url.startswith('fc2:'):
+            webpage = self._download_webpage(url, video_id)
+            self._downloader.cookiejar.clear_session_cookies()  # must clear
+            self._login()
 
-        title = self._og_search_title(webpage)
-        thumbnail = self._og_search_thumbnail(webpage)
+        title = 'FC2 video %s' % video_id
+        thumbnail = None
+        if webpage is not None:
+            title = self._og_search_title(webpage)
+            thumbnail = self._og_search_thumbnail(webpage)
         refer = url.replace('/content/', '/a/content/') if '/a/content/' not in url else url
 
         mimi = hashlib.md5((video_id + '_gGddgPfeaf_gzyr').encode('utf-8')).hexdigest()
@@ -113,3 +120,40 @@ class FC2IE(InfoExtractor):
             'ext': 'flv',
             'thumbnail': thumbnail,
         }
+
+
+class FC2EmbedIE(InfoExtractor):
+    _VALID_URL = r'https?://video\.fc2\.com/flv2\.swf\?(?P<query>.+)'
+    IE_NAME = 'fc2:embed'
+
+    _TEST = {
+        'url': 'http://video.fc2.com/flv2.swf?t=201404182936758512407645&i=20130316kwishtfitaknmcgd76kjd864hso93htfjcnaogz629mcgfs6rbfk0hsycma7shkf85937cbchfygd74&i=201403223kCqB3Ez&d=2625&sj=11&lang=ja&rel=1&from=11&cmt=1&tk=TlRBM09EQTNNekU9&tl=プリズン･ブレイク%20S1-01%20マイケル%20【吹替】',
+        'md5': 'b8aae5334cb691bdb1193a88a6ab5d5a',
+        'info_dict': {
+            'id': '201403223kCqB3Ez',
+            'ext': 'flv',
+            'title': 'プリズン･ブレイク S1-01 マイケル 【吹替】',
+            'thumbnail': 're:^https?://.*\.jpg$',
+        },
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        query = compat_parse_qs(mobj.group('query'))
+
+        video_id = query['i'][-1]
+        title = query.get('tl', ['FC2 video %s' % video_id])[0]
+
+        sj = query.get('sj', [None])[0]
+        thumbnail = None
+        if sj:
+            # See thumbnailImagePath() in ServerConst.as of flv2.swf
+            thumbnail = 'http://video%s-thumbnail.fc2.com/up/pic/%s.jpg' % (
+                sj, '/'.join((video_id[:6], video_id[6:8], video_id[-2], video_id[-1], video_id)))
+
+        return {
+            '_type': 'url_transparent',
+            'url': 'fc2:%s' % video_id,
+            'title': title,
+            'thumbnail': thumbnail,
+        }

From cdc783510bb575b2318b1d7d42fb98f0c0f0df18 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 3 Sep 2016 18:16:19 +0800
Subject: [PATCH 5975/6219] [foxnews:insider] Add new extractor

Closes #10445
---
 ChangeLog                          |  1 +
 youtube_dl/extractor/extractors.py |  5 +++-
 youtube_dl/extractor/foxnews.py    | 48 +++++++++++++++++++++++++++++-
 3 files changed, 52 insertions(+), 2 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 065fc83a8..199983674 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
 version <unreleased>
 
 Extractors
++ [foxnews] Add support for FoxNews Insider (#10445)
 + [fc2] Recognize Flash player URLs (#10512)
 
 
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index d851e5f36..8c6ee0503 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -287,7 +287,10 @@ from .formula1 import Formula1IE
 from .fourtube import FourTubeIE
 from .fox import FOXIE
 from .foxgay import FoxgayIE
-from .foxnews import FoxNewsIE
+from .foxnews import (
+    FoxNewsIE,
+    FoxNewsInsiderIE,
+)
 from .foxsports import FoxSportsIE
 from .franceculture import FranceCultureIE
 from .franceinter import FranceInterIE
diff --git a/youtube_dl/extractor/foxnews.py b/youtube_dl/extractor/foxnews.py
index b04da2415..5c7acd795 100644
--- a/youtube_dl/extractor/foxnews.py
+++ b/youtube_dl/extractor/foxnews.py
@@ -3,11 +3,12 @@ from __future__ import unicode_literals
 import re
 
 from .amp import AMPIE
+from .common import InfoExtractor
 
 
 class FoxNewsIE(AMPIE):
     IE_DESC = 'Fox News and Fox Business Video'
-    _VALID_URL = r'https?://(?P<host>video\.fox(?:news|business)\.com)/v/(?:video-embed\.html\?video_id=)?(?P<id>\d+)'
+    _VALID_URL = r'https?://(?P<host>video\.(?:insider\.)?fox(?:news|business)\.com)/v/(?:video-embed\.html\?video_id=)?(?P<id>\d+)'
     _TESTS = [
         {
             'url': 'http://video.foxnews.com/v/3937480/frozen-in-time/#sp=show-clips',
@@ -49,6 +50,11 @@ class FoxNewsIE(AMPIE):
             'url': 'http://video.foxbusiness.com/v/4442309889001',
             'only_matching': True,
         },
+        {
+            # From http://insider.foxnews.com/2016/08/25/univ-wisconsin-student-group-pushing-silence-certain-words
+            'url': 'http://video.insider.foxnews.com/v/video-embed.html?video_id=5099377331001&autoplay=true&share_url=http://insider.foxnews.com/2016/08/25/univ-wisconsin-student-group-pushing-silence-certain-words&share_title=Student%20Group:%20Saying%20%27Politically%20Correct,%27%20%27Trash%27%20and%20%27Lame%27%20Is%20Offensive&share=true',
+            'only_matching': True,
+        },
     ]
 
     def _real_extract(self, url):
@@ -58,3 +64,43 @@ class FoxNewsIE(AMPIE):
             'http://%s/v/feed/video/%s.js?template=fox' % (host, video_id))
         info['id'] = video_id
         return info
+
+
+class FoxNewsInsiderIE(InfoExtractor):
+    _VALID_URL = r'https?://insider\.foxnews\.com/([^/]+/)+(?P<id>[a-z-]+)'
+    IE_NAME = 'foxnews:insider'
+
+    _TEST = {
+        'url': 'http://insider.foxnews.com/2016/08/25/univ-wisconsin-student-group-pushing-silence-certain-words',
+        'md5': 'a10c755e582d28120c62749b4feb4c0c',
+        'info_dict': {
+            'id': '5099377331001',
+            'display_id': 'univ-wisconsin-student-group-pushing-silence-certain-words',
+            'ext': 'mp4',
+            'title': 'Student Group: Saying \'Politically Correct,\' \'Trash\' and \'Lame\' Is Offensive',
+            'description': 'Is campus censorship getting out of control?',
+            'timestamp': 1472168725,
+            'upload_date': '20160825',
+            'thumbnail': 're:^https?://.*\.jpg$',
+        },
+        'add_ie': [FoxNewsIE.ie_key()],
+    }
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, display_id)
+
+        embed_url = self._html_search_meta('embedUrl', webpage, 'embed URL')
+
+        title = self._og_search_title(webpage)
+        description = self._og_search_description(webpage)
+
+        return {
+            '_type': 'url_transparent',
+            'ie_key': FoxNewsIE.ie_key(),
+            'url': embed_url,
+            'display_id': display_id,
+            'title': title,
+            'description': description,
+        }

From ed2bfe93aaa11f49f7b2b92b581abb6aa385dfbf Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 3 Sep 2016 18:22:00 +0800
Subject: [PATCH 5976/6219] [fc2:embed] Add ie_key

---
 youtube_dl/extractor/fc2.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/fc2.py b/youtube_dl/extractor/fc2.py
index b9e58d4df..c032d4d02 100644
--- a/youtube_dl/extractor/fc2.py
+++ b/youtube_dl/extractor/fc2.py
@@ -153,6 +153,7 @@ class FC2EmbedIE(InfoExtractor):
 
         return {
             '_type': 'url_transparent',
+            'ie_key': FC2IE.ie_key(),
             'url': 'fc2:%s' % video_id,
             'title': title,
             'thumbnail': thumbnail,

From 45aab4d30b7c3fc03c9be9680550cba88bd85b5c Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 3 Sep 2016 18:37:36 +0800
Subject: [PATCH 5977/6219] [youjizz] Fix extraction. The site has moved to
 HTML5

Closes #10437
---
 ChangeLog                       |  1 +
 youtube_dl/extractor/youjizz.py | 43 +++++++--------------------------
 2 files changed, 10 insertions(+), 34 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 199983674..2809e55d7 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
 version <unreleased>
 
 Extractors
+* [youjizz] Fix extraction (#10437)
 + [foxnews] Add support for FoxNews Insider (#10445)
 + [fc2] Recognize Flash player URLs (#10512)
 
diff --git a/youtube_dl/extractor/youjizz.py b/youtube_dl/extractor/youjizz.py
index 31e2f9263..b50f34e9b 100644
--- a/youtube_dl/extractor/youjizz.py
+++ b/youtube_dl/extractor/youjizz.py
@@ -1,21 +1,16 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
-)
 
 
 class YouJizzIE(InfoExtractor):
     _VALID_URL = r'https?://(?:\w+\.)?youjizz\.com/videos/(?:[^/#?]+)?-(?P<id>[0-9]+)\.html(?:$|[?#])'
     _TESTS = [{
         'url': 'http://www.youjizz.com/videos/zeichentrick-1-2189178.html',
-        'md5': '07e15fa469ba384c7693fd246905547c',
+        'md5': '78fc1901148284c69af12640e01c6310',
         'info_dict': {
             'id': '2189178',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Zeichentrick 1',
             'age_limit': 18,
         }
@@ -27,38 +22,18 @@ class YouJizzIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
+        # YouJizz's HTML5 player has invalid HTML
+        webpage = webpage.replace('"controls', '" controls')
         age_limit = self._rta_search(webpage)
         video_title = self._html_search_regex(
             r'<title>\s*(.*)\s*</title>', webpage, 'title')
 
-        embed_page_url = self._search_regex(
-            r'(https?://www.youjizz.com/videos/embed/[0-9]+)',
-            webpage, 'embed page')
-        webpage = self._download_webpage(
-            embed_page_url, video_id, note='downloading embed page')
+        info_dict = self._parse_html5_media_entries(url, webpage, video_id)[0]
 
-        # Get the video URL
-        m_playlist = re.search(r'so.addVariable\("playlist", ?"(?P<playlist>.+?)"\);', webpage)
-        if m_playlist is not None:
-            playlist_url = m_playlist.group('playlist')
-            playlist_page = self._download_webpage(playlist_url, video_id,
-                                                   'Downloading playlist page')
-            m_levels = list(re.finditer(r'<level bitrate="(\d+?)" file="(.*?)"', playlist_page))
-            if len(m_levels) == 0:
-                raise ExtractorError('Unable to extract video url')
-            videos = [(int(m.group(1)), m.group(2)) for m in m_levels]
-            (_, video_url) = sorted(videos)[0]
-            video_url = video_url.replace('%252F', '%2F')
-        else:
-            video_url = self._search_regex(r'so.addVariable\("file",encodeURIComponent\("(?P<source>[^"]+)"\)\);',
-                                           webpage, 'video URL')
-
-        return {
+        info_dict.update({
             'id': video_id,
-            'url': video_url,
             'title': video_title,
-            'ext': 'flv',
-            'format': 'flv',
-            'player_url': embed_page_url,
             'age_limit': age_limit,
-        }
+        })
+
+        return info_dict

From 9603b6601208333bc49e0c69199f0e652a7aaea3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Aug 2016 04:52:18 +0700
Subject: [PATCH 5978/6219] Introduce --skip-unavailable-fragments

---
 youtube_dl/__init__.py            |  1 +
 youtube_dl/downloader/fragment.py | 10 ++++++++--
 youtube_dl/options.py             | 10 +++++++++-
 3 files changed, 18 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index a9730292c..42128272a 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -318,6 +318,7 @@ def _real_main(argv=None):
         'nooverwrites': opts.nooverwrites,
         'retries': opts.retries,
         'fragment_retries': opts.fragment_retries,
+        'skip_unavailable_fragments': opts.skip_unavailable_fragments,
         'buffersize': opts.buffersize,
         'noresizebuffer': opts.noresizebuffer,
         'continuedl': opts.continue_dl,
diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
index ba903ae10..b4a798f8f 100644
--- a/youtube_dl/downloader/fragment.py
+++ b/youtube_dl/downloader/fragment.py
@@ -22,14 +22,20 @@ class FragmentFD(FileDownloader):
 
     Available options:
 
-    fragment_retries:   Number of times to retry a fragment for HTTP error (DASH only)
+    fragment_retries:   Number of times to retry a fragment for HTTP error (DASH
+                        and hlsnative only)
+    skip_unavailable_fragments:
+                        Skip unavailable fragments (DASH and hlsnative only)
     """
 
     def report_retry_fragment(self, fragment_name, count, retries):
         self.to_screen(
-            '[download] Got server HTTP error. Retrying fragment %s (attempt %d of %s)...'
+            '[download] Got server HTTP error: %s. Retrying fragment %s (attempt %d of %s)...'
             % (fragment_name, count, self.format_retries(retries)))
 
+    def report_skip_fragment(self, fragment_name):
+        self.to_screen('[download] Skipping fragment %s...' % fragment_name)
+
     def _prepare_and_start_frag_download(self, ctx):
         self._prepare_frag_download(ctx)
         self._start_frag_download(ctx)
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 5d62deef4..56f312f57 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -423,7 +423,15 @@ def parseOpts(overrideArguments=None):
     downloader.add_option(
         '--fragment-retries',
         dest='fragment_retries', metavar='RETRIES', default=10,
-        help='Number of retries for a fragment (default is %default), or "infinite" (DASH only)')
+        help='Number of retries for a fragment (default is %default), or "infinite" (DASH and hlsnative only)')
+    downloader.add_option(
+        '--skip-unavailable-fragments',
+        action='store_true', dest='skip_unavailable_fragments', default=True,
+        help='Skip unavailable fragments (DASH and hlsnative only)')
+    general.add_option(
+        '--abort-on-unavailable-fragment',
+        action='store_false', dest='skip_unavailable_fragments',
+        help='Abort downloading when some fragment is not available')
     downloader.add_option(
         '--buffer-size',
         dest='buffersize', metavar='SIZE', default='1024',

From 25afc2a7830e281e849609202b4f70728664bdb7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Aug 2016 04:55:55 +0700
Subject: [PATCH 5979/6219] [downloader/dash:hls] Respect --fragment-retries
 and --skip-unavailable-fragments (Closes #10165, closes #10448)

---
 youtube_dl/downloader/dash.py | 12 +++++-----
 youtube_dl/downloader/hls.py  | 41 +++++++++++++++++++++++++++++------
 2 files changed, 41 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/downloader/dash.py b/youtube_dl/downloader/dash.py
index 8bbab9dbc..cbcee324d 100644
--- a/youtube_dl/downloader/dash.py
+++ b/youtube_dl/downloader/dash.py
@@ -38,6 +38,7 @@ class DashSegmentsFD(FragmentFD):
         segments_filenames = []
 
         fragment_retries = self.params.get('fragment_retries', 0)
+        skip_unavailable_fragments = self.params.get('skip_unavailable_fragments', True)
 
         def append_url_to_file(target_url, tmp_filename, segment_name):
             target_filename = '%s-%s' % (tmp_filename, segment_name)
@@ -52,19 +53,20 @@ class DashSegmentsFD(FragmentFD):
                     down.close()
                     segments_filenames.append(target_sanitized)
                     break
-                except (compat_urllib_error.HTTPError, ) as err:
+                except compat_urllib_error.HTTPError:
                     # YouTube may often return 404 HTTP error for a fragment causing the
                     # whole download to fail. However if the same fragment is immediately
                     # retried with the same request data this usually succeeds (1-2 attemps
                     # is usually enough) thus allowing to download the whole file successfully.
-                    # So, we will retry all fragments that fail with 404 HTTP error for now.
-                    if err.code != 404:
-                        raise
-                    # Retry fragment
+                    # To be future-proof we will retry all fragments that fail with any
+                    # HTTP error.
                     count += 1
                     if count <= fragment_retries:
                         self.report_retry_fragment(segment_name, count, fragment_retries)
             if count > fragment_retries:
+                if skip_unavailable_fragments:
+                    self.report_skip_fragment(segment_name)
+                    return
                 self.report_error('giving up after %s fragment retries' % fragment_retries)
                 return False
 
diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index baaff44d5..7412620a5 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -13,6 +13,7 @@ from .fragment import FragmentFD
 from .external import FFmpegFD
 
 from ..compat import (
+    compat_urllib_error,
     compat_urlparse,
     compat_struct_pack,
 )
@@ -83,6 +84,10 @@ class HlsFD(FragmentFD):
 
         self._prepare_and_start_frag_download(ctx)
 
+        fragment_retries = self.params.get('fragment_retries', 0)
+        skip_unavailable_fragments = self.params.get('skip_unavailable_fragments', True)
+        test = self.params.get('test', False)
+
         extra_query = None
         extra_param_to_segment_url = info_dict.get('extra_param_to_segment_url')
         if extra_param_to_segment_url:
@@ -99,15 +104,37 @@ class HlsFD(FragmentFD):
                         line
                         if re.match(r'^https?://', line)
                         else compat_urlparse.urljoin(man_url, line))
-                    frag_filename = '%s-Frag%d' % (ctx['tmpfilename'], i)
+                    frag_name = 'Frag%d' % i
+                    frag_filename = '%s-%s' % (ctx['tmpfilename'], frag_name)
                     if extra_query:
                         frag_url = update_url_query(frag_url, extra_query)
-                    success = ctx['dl'].download(frag_filename, {'url': frag_url})
-                    if not success:
+                    count = 0
+                    while count <= fragment_retries:
+                        try:
+                            success = ctx['dl'].download(frag_filename, {'url': frag_url})
+                            if not success:
+                                return False
+                            down, frag_sanitized = sanitize_open(frag_filename, 'rb')
+                            frag_content = down.read()
+                            down.close()
+                            break
+                        except compat_urllib_error.HTTPError:
+                            # Unavailable (possibly temporary) fragments may be served.
+                            # First we try to retry then either skip or abort.
+                            # See https://github.com/rg3/youtube-dl/issues/10165,
+                            # https://github.com/rg3/youtube-dl/issues/10448).
+                            count += 1
+                            if count <= fragment_retries:
+                                self.report_retry_fragment(frag_name, count, fragment_retries)
+                    if count > fragment_retries:
+                        if skip_unavailable_fragments:
+                            i += 1
+                            media_sequence += 1
+                            self.report_skip_fragment(frag_name)
+                            continue
+                        self.report_error(
+                            'giving up after %s fragment retries' % fragment_retries)
                         return False
-                    down, frag_sanitized = sanitize_open(frag_filename, 'rb')
-                    frag_content = down.read()
-                    down.close()
                     if decrypt_info['METHOD'] == 'AES-128':
                         iv = decrypt_info.get('IV') or compat_struct_pack('>8xq', media_sequence)
                         frag_content = AES.new(
@@ -115,7 +142,7 @@ class HlsFD(FragmentFD):
                     ctx['dest_stream'].write(frag_content)
                     frags_filenames.append(frag_sanitized)
                     # We only download the first fragment during the test
-                    if self.params.get('test', False):
+                    if test:
                         break
                     i += 1
                     media_sequence += 1

From 2e99cd30c3108fd8da6a9f9fadfa89852c8d8826 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Aug 2016 04:57:59 +0700
Subject: [PATCH 5980/6219] [downloader/dash:hls] Report exact fragment error
 on retry

---
 youtube_dl/downloader/dash.py     | 4 ++--
 youtube_dl/downloader/fragment.py | 5 +++--
 youtube_dl/downloader/hls.py      | 4 ++--
 3 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/downloader/dash.py b/youtube_dl/downloader/dash.py
index cbcee324d..e087cf142 100644
--- a/youtube_dl/downloader/dash.py
+++ b/youtube_dl/downloader/dash.py
@@ -53,7 +53,7 @@ class DashSegmentsFD(FragmentFD):
                     down.close()
                     segments_filenames.append(target_sanitized)
                     break
-                except compat_urllib_error.HTTPError:
+                except compat_urllib_error.HTTPError as err:
                     # YouTube may often return 404 HTTP error for a fragment causing the
                     # whole download to fail. However if the same fragment is immediately
                     # retried with the same request data this usually succeeds (1-2 attemps
@@ -62,7 +62,7 @@ class DashSegmentsFD(FragmentFD):
                     # HTTP error.
                     count += 1
                     if count <= fragment_retries:
-                        self.report_retry_fragment(segment_name, count, fragment_retries)
+                        self.report_retry_fragment(err, segment_name, count, fragment_retries)
             if count > fragment_retries:
                 if skip_unavailable_fragments:
                     self.report_skip_fragment(segment_name)
diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
index b4a798f8f..84aacf7db 100644
--- a/youtube_dl/downloader/fragment.py
+++ b/youtube_dl/downloader/fragment.py
@@ -6,6 +6,7 @@ import time
 from .common import FileDownloader
 from .http import HttpFD
 from ..utils import (
+    error_to_compat_str,
     encodeFilename,
     sanitize_open,
 )
@@ -28,10 +29,10 @@ class FragmentFD(FileDownloader):
                         Skip unavailable fragments (DASH and hlsnative only)
     """
 
-    def report_retry_fragment(self, fragment_name, count, retries):
+    def report_retry_fragment(self, err, fragment_name, count, retries):
         self.to_screen(
             '[download] Got server HTTP error: %s. Retrying fragment %s (attempt %d of %s)...'
-            % (fragment_name, count, self.format_retries(retries)))
+            % (error_to_compat_str(err), fragment_name, count, self.format_retries(retries)))
 
     def report_skip_fragment(self, fragment_name):
         self.to_screen('[download] Skipping fragment %s...' % fragment_name)
diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index 7412620a5..5d70abf62 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -118,14 +118,14 @@ class HlsFD(FragmentFD):
                             frag_content = down.read()
                             down.close()
                             break
-                        except compat_urllib_error.HTTPError:
+                        except compat_urllib_error.HTTPError as err:
                             # Unavailable (possibly temporary) fragments may be served.
                             # First we try to retry then either skip or abort.
                             # See https://github.com/rg3/youtube-dl/issues/10165,
                             # https://github.com/rg3/youtube-dl/issues/10448).
                             count += 1
                             if count <= fragment_retries:
-                                self.report_retry_fragment(frag_name, count, fragment_retries)
+                                self.report_retry_fragment(err, frag_name, count, fragment_retries)
                     if count > fragment_retries:
                         if skip_unavailable_fragments:
                             i += 1

From 4a69fa04e0074a3d5938ffb03decff9cc33f5d3d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 30 Aug 2016 22:28:14 +0700
Subject: [PATCH 5981/6219] [downloader/dash] Abort download immediately after
 giving up on some fragment

---
 youtube_dl/downloader/dash.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/downloader/dash.py b/youtube_dl/downloader/dash.py
index e087cf142..efeae02a3 100644
--- a/youtube_dl/downloader/dash.py
+++ b/youtube_dl/downloader/dash.py
@@ -66,14 +66,17 @@ class DashSegmentsFD(FragmentFD):
             if count > fragment_retries:
                 if skip_unavailable_fragments:
                     self.report_skip_fragment(segment_name)
-                    return
+                    return True
                 self.report_error('giving up after %s fragment retries' % fragment_retries)
                 return False
+            return True
 
         if initialization_url:
-            append_url_to_file(initialization_url, ctx['tmpfilename'], 'Init')
+            if not append_url_to_file(initialization_url, ctx['tmpfilename'], 'Init'):
+                return False
         for i, segment_url in enumerate(segment_urls):
-            append_url_to_file(segment_url, ctx['tmpfilename'], 'Seg%d' % i)
+            if not append_url_to_file(segment_url, ctx['tmpfilename'], 'Seg%d' % i):
+                return False
 
         self._finish_frag_download(ctx)
 

From 7e5dc339de14547aa7b489e88b4c456ec613ba9d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 4 Sep 2016 00:29:01 +0700
Subject: [PATCH 5982/6219] [youtube:watchlater] Fix extraction (Closes #10544)

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 4c8edef8d..0bc85af74 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -2376,7 +2376,7 @@ class YoutubeWatchLaterIE(YoutubePlaylistIE):
     }]
 
     def _real_extract(self, url):
-        video = self._check_download_just_video(url, 'WL')
+        _, video = self._check_download_just_video(url, 'WL')
         if video:
             return video
         _, playlist = self._extract_playlist('WL')

From 091624f9da491ef3a98e63367bf4ffd9836dafde Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 4 Sep 2016 03:39:13 +0700
Subject: [PATCH 5983/6219] [vimple] Extend _VALID_URL (Closes #10547)

---
 youtube_dl/extractor/vimple.py | 35 +++++++++++++++++-----------------
 1 file changed, 18 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/vimple.py b/youtube_dl/extractor/vimple.py
index 92321d66e..7fd9b777b 100644
--- a/youtube_dl/extractor/vimple.py
+++ b/youtube_dl/extractor/vimple.py
@@ -28,23 +28,24 @@ class SprutoBaseIE(InfoExtractor):
 
 class VimpleIE(SprutoBaseIE):
     IE_DESC = 'Vimple - one-click video hosting'
-    _VALID_URL = r'https?://(?:player\.vimple\.ru/iframe|vimple\.ru)/(?P<id>[\da-f-]{32,36})'
-    _TESTS = [
-        {
-            'url': 'http://vimple.ru/c0f6b1687dcd4000a97ebe70068039cf',
-            'md5': '2e750a330ed211d3fd41821c6ad9a279',
-            'info_dict': {
-                'id': 'c0f6b168-7dcd-4000-a97e-be70068039cf',
-                'ext': 'mp4',
-                'title': 'Sunset',
-                'duration': 20,
-                'thumbnail': 're:https?://.*?\.jpg',
-            },
-        }, {
-            'url': 'http://player.vimple.ru/iframe/52e1beec-1314-4a83-aeac-c61562eadbf9',
-            'only_matching': True,
-        }
-    ]
+    _VALID_URL = r'https?://(?:player\.vimple\.(?:ru|co)/iframe|vimple\.(?:ru|co))/(?P<id>[\da-f-]{32,36})'
+    _TESTS = [{
+        'url': 'http://vimple.ru/c0f6b1687dcd4000a97ebe70068039cf',
+        'md5': '2e750a330ed211d3fd41821c6ad9a279',
+        'info_dict': {
+            'id': 'c0f6b168-7dcd-4000-a97e-be70068039cf',
+            'ext': 'mp4',
+            'title': 'Sunset',
+            'duration': 20,
+            'thumbnail': 're:https?://.*?\.jpg',
+        },
+    }, {
+        'url': 'http://player.vimple.ru/iframe/52e1beec-1314-4a83-aeac-c61562eadbf9',
+        'only_matching': True,
+    }, {
+        'url': 'http://vimple.co/04506a053f124483b8fb05ed73899f19',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From 37c7490ac62d4aacbf9103bf6760d20f21984a55 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 4 Sep 2016 04:59:46 +0700
Subject: [PATCH 5984/6219] [espn] Extend _VALID_URL (Closes #10549)

---
 youtube_dl/extractor/espn.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/espn.py b/youtube_dl/extractor/espn.py
index 66c08bec4..6d10f8e68 100644
--- a/youtube_dl/extractor/espn.py
+++ b/youtube_dl/extractor/espn.py
@@ -5,7 +5,7 @@ from ..utils import remove_end
 
 
 class ESPNIE(InfoExtractor):
-    _VALID_URL = r'https?://espn\.go\.com/(?:[^/]+/)*(?P<id>[^/]+)'
+    _VALID_URL = r'https?://(?:espn\.go|(?:www\.)?espn)\.com/(?:[^/]+/)*(?P<id>[^/]+)'
     _TESTS = [{
         'url': 'http://espn.go.com/video/clip?id=10365079',
         'md5': '60e5d097a523e767d06479335d1bdc58',
@@ -47,6 +47,9 @@ class ESPNIE(InfoExtractor):
     }, {
         'url': 'http://espn.go.com/nba/playoffs/2015/story/_/id/12887571/john-wall-washington-wizards-no-swelling-left-hand-wrist-game-5-return',
         'only_matching': True,
+    }, {
+        'url': 'http://www.espn.com/video/clip?id=10365079',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 622638512b8241c39837b634e75c44cf9105a299 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 4 Sep 2016 16:25:59 +0800
Subject: [PATCH 5985/6219] [rottentomatoes] Fix extraction

Closes #10467
---
 ChangeLog                              |  1 +
 youtube_dl/extractor/rottentomatoes.py | 30 +++++++++++++++++++-------
 2 files changed, 23 insertions(+), 8 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 2809e55d7..e6a2d24e1 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
 version <unreleased>
 
 Extractors
+* [rottentomatoes] Fix extraction (#10467)
 * [youjizz] Fix extraction (#10437)
 + [foxnews] Add support for FoxNews Insider (#10445)
 + [fc2] Recognize Flash player URLs (#10512)
diff --git a/youtube_dl/extractor/rottentomatoes.py b/youtube_dl/extractor/rottentomatoes.py
index f9cd48790..df39ed3f2 100644
--- a/youtube_dl/extractor/rottentomatoes.py
+++ b/youtube_dl/extractor/rottentomatoes.py
@@ -1,8 +1,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import compat_urlparse
-from .internetvideoarchive import InternetVideoArchiveIE
+from ..utils import js_to_json
 
 
 class RottenTomatoesIE(InfoExtractor):
@@ -11,21 +10,36 @@ class RottenTomatoesIE(InfoExtractor):
     _TEST = {
         'url': 'http://www.rottentomatoes.com/m/toy_story_3/trailers/11028566/',
         'info_dict': {
-            'id': '613340',
+            'id': '11028566',
             'ext': 'mp4',
             'title': 'Toy Story 3',
+            'thumbnail': 're:^https?://.*\.jpg$',
         },
     }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
-        og_video = self._og_search_video_url(webpage)
-        query = compat_urlparse.urlparse(og_video).query
+
+        params = self._parse_json(
+            self._search_regex(r'(?s)RTVideo\(({.+?})\);', webpage, 'player parameters'),
+            video_id, transform_source=lambda s: js_to_json(s.replace('window.location.href', '""')))
+
+        formats = []
+        if params.get('urlHLS'):
+            formats.extend(self._extract_m3u8_formats(
+                params['urlHLS'], video_id, ext='mp4',
+                entry_protocol='m3u8_native', m3u8_id='hls', fatal=False))
+        if params.get('urlMP4'):
+            formats.append({
+                'url': params['urlMP4'],
+                'format_id': 'mp4',
+            })
+        self._sort_formats(formats)
 
         return {
-            '_type': 'url_transparent',
-            'url': InternetVideoArchiveIE._build_xml_url(query),
-            'ie_key': InternetVideoArchiveIE.ie_key(),
+            'id': video_id,
             'title': self._og_search_title(webpage),
+            'formats': formats,
+            'thumbnail': params.get('thumbnailImg'),
         }

From b29cd56591f1ef001d9f30bdff87789815f1fa0c Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 4 Sep 2016 17:01:39 +0800
Subject: [PATCH 5986/6219] [pornovoisines] Fix extraction (closes #10469)

---
 ChangeLog                             |  1 +
 youtube_dl/extractor/pornovoisines.py | 80 +++++++++++++++------------
 2 files changed, 47 insertions(+), 34 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index e6a2d24e1..616b55803 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
 version <unreleased>
 
 Extractors
+* [pornvoisines] Fix extraction (#10469)
 * [rottentomatoes] Fix extraction (#10467)
 * [youjizz] Fix extraction (#10437)
 + [foxnews] Add support for FoxNews Insider (#10445)
diff --git a/youtube_dl/extractor/pornovoisines.py b/youtube_dl/extractor/pornovoisines.py
index 6b51e5c54..58f557e39 100644
--- a/youtube_dl/extractor/pornovoisines.py
+++ b/youtube_dl/extractor/pornovoisines.py
@@ -2,7 +2,6 @@
 from __future__ import unicode_literals
 
 import re
-import random
 
 from .common import InfoExtractor
 from ..utils import (
@@ -13,61 +12,69 @@ from ..utils import (
 
 
 class PornoVoisinesIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?pornovoisines\.com/showvideo/(?P<id>\d+)/(?P<display_id>[^/]+)'
-
-    _VIDEO_URL_TEMPLATE = 'http://stream%d.pornovoisines.com' \
-        '/static/media/video/transcoded/%s-640x360-1000-trscded.mp4'
-
-    _SERVER_NUMBERS = (1, 2)
+    _VALID_URL = r'https?://(?:www\.)?pornovoisines\.com/videos/show/(?P<id>\d+)/(?P<display_id>[^/.]+)'
 
     _TEST = {
-        'url': 'http://www.pornovoisines.com/showvideo/1285/recherche-appartement/',
-        'md5': '5ac670803bc12e9e7f9f662ce64cf1d1',
+        'url': 'http://www.pornovoisines.com/videos/show/919/recherche-appartement.html',
+        'md5': '6f8aca6a058592ab49fe701c8ba8317b',
         'info_dict': {
-            'id': '1285',
+            'id': '919',
             'display_id': 'recherche-appartement',
             'ext': 'mp4',
             'title': 'Recherche appartement',
-            'description': 'md5:819ea0b785e2a04667a1a01cdc89594e',
+            'description': 'md5:fe10cb92ae2dd3ed94bb4080d11ff493',
             'thumbnail': 're:^https?://.*\.jpg$',
             'upload_date': '20140925',
             'duration': 120,
             'view_count': int,
             'average_rating': float,
-            'categories': ['Débutantes', 'Scénario', 'Sodomie'],
+            'categories': ['Débutante', 'Débutantes', 'Scénario', 'Sodomie'],
             'age_limit': 18,
+            'subtitles': {
+                'fr': [{
+                    'ext': 'vtt',
+                }]
+            },
         }
     }
 
-    @classmethod
-    def build_video_url(cls, num):
-        return cls._VIDEO_URL_TEMPLATE % (random.choice(cls._SERVER_NUMBERS), num)
-
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
         display_id = mobj.group('display_id')
 
+        settings_url = self._download_json(
+            'http://www.pornovoisines.com/api/video/%s/getsettingsurl/' % video_id,
+            video_id, note='Getting settings URL')['video_settings_url']
+        settings = self._download_json(settings_url, video_id)['data']
+
+        formats = []
+        for kind, data in settings['variants'].items():
+            if kind == 'HLS':
+                formats.extend(self._extract_m3u8_formats(
+                    data, video_id, ext='mp4', entry_protocol='m3u8_native', m3u8_id='hls'))
+            elif kind == 'MP4':
+                for item in data:
+                    formats.append({
+                        'url': item['url'],
+                        'height': item.get('height'),
+                        'bitrate': item.get('bitrate'),
+                    })
+        self._sort_formats(formats)
+
         webpage = self._download_webpage(url, video_id)
 
-        video_url = self.build_video_url(video_id)
+        title = self._og_search_title(webpage)
+        description = self._og_search_description(webpage)
 
-        title = self._html_search_regex(
-            r'<h1>(.+?)</h1>', webpage, 'title', flags=re.DOTALL)
-        description = self._html_search_regex(
-            r'<article id="descriptif">(.+?)</article>',
-            webpage, 'description', fatal=False, flags=re.DOTALL)
-
-        thumbnail = self._search_regex(
-            r'<div id="mediaspace%s">\s*<img src="/?([^"]+)"' % video_id,
-            webpage, 'thumbnail', fatal=False)
-        if thumbnail:
-            thumbnail = 'http://www.pornovoisines.com/%s' % thumbnail
+        # The webpage has a bug - there's no space between "thumb" and src=
+        thumbnail = self._html_search_regex(
+            r'<img[^>]+class=([\'"])thumb\1[^>]*src=([\'"])(?P<url>[^"]+)\2',
+            webpage, 'thumbnail', fatal=False, group='url')
 
         upload_date = unified_strdate(self._search_regex(
-            r'Publié le ([\d-]+)', webpage, 'upload date', fatal=False))
-        duration = int_or_none(self._search_regex(
-            'Durée (\d+)', webpage, 'duration', fatal=False))
+            r'Le\s*<b>([\d/]+)', webpage, 'upload date', fatal=False))
+        duration = settings.get('main', {}).get('duration')
         view_count = int_or_none(self._search_regex(
             r'(\d+) vues', webpage, 'view count', fatal=False))
         average_rating = self._search_regex(
@@ -75,15 +82,19 @@ class PornoVoisinesIE(InfoExtractor):
         if average_rating:
             average_rating = float_or_none(average_rating.replace(',', '.'))
 
-        categories = self._html_search_meta(
-            'keywords', webpage, 'categories', fatal=False)
+        categories = self._html_search_regex(
+            r'(?s)Catégories\s*:\s*<b>(.+?)</b>', webpage, 'categories', fatal=False)
         if categories:
             categories = [category.strip() for category in categories.split(',')]
 
+        subtitles = {'fr': [{
+            'url': subtitle,
+        } for subtitle in settings.get('main', {}).get('vtt_tracks', {}).values()]}
+
         return {
             'id': video_id,
             'display_id': display_id,
-            'url': video_url,
+            'formats': formats,
             'title': title,
             'description': description,
             'thumbnail': thumbnail,
@@ -93,4 +104,5 @@ class PornoVoisinesIE(InfoExtractor):
             'average_rating': average_rating,
             'categories': categories,
             'age_limit': 18,
+            'subtitles': subtitles,
         }

From 919cf1a62f022c61cfa65498e8c1b1cc0d21046e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 3 Sep 2016 23:00:52 +0800
Subject: [PATCH 5987/6219] [downloader/dash] Abort if the first segment fails

Closes #10497, Closes #10542
---
 ChangeLog                     |  4 ++++
 youtube_dl/downloader/dash.py | 20 +++++++++++++-------
 2 files changed, 17 insertions(+), 7 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 616b55803..1d277b562 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,9 @@
 version <unreleased>
 
+Core
+* If the first segment of DASH fails, abort the whole download process to
+  prevent throttling (#10497)
+
 Extractors
 * [pornvoisines] Fix extraction (#10469)
 * [rottentomatoes] Fix extraction (#10467)
diff --git a/youtube_dl/downloader/dash.py b/youtube_dl/downloader/dash.py
index efeae02a3..41fc9cfc2 100644
--- a/youtube_dl/downloader/dash.py
+++ b/youtube_dl/downloader/dash.py
@@ -40,7 +40,8 @@ class DashSegmentsFD(FragmentFD):
         fragment_retries = self.params.get('fragment_retries', 0)
         skip_unavailable_fragments = self.params.get('skip_unavailable_fragments', True)
 
-        def append_url_to_file(target_url, tmp_filename, segment_name):
+        def process_segment(segment, tmp_filename, fatal):
+            target_url, segment_name = segment
             target_filename = '%s-%s' % (tmp_filename, segment_name)
             count = 0
             while count <= fragment_retries:
@@ -64,18 +65,23 @@ class DashSegmentsFD(FragmentFD):
                     if count <= fragment_retries:
                         self.report_retry_fragment(err, segment_name, count, fragment_retries)
             if count > fragment_retries:
-                if skip_unavailable_fragments:
+                if not fatal:
                     self.report_skip_fragment(segment_name)
                     return True
                 self.report_error('giving up after %s fragment retries' % fragment_retries)
                 return False
             return True
 
-        if initialization_url:
-            if not append_url_to_file(initialization_url, ctx['tmpfilename'], 'Init'):
-                return False
-        for i, segment_url in enumerate(segment_urls):
-            if not append_url_to_file(segment_url, ctx['tmpfilename'], 'Seg%d' % i):
+        segments_to_download = [(initialization_url, 'Init')] if initialization_url else []
+        segments_to_download.extend([
+            (segment_url, 'Seg%d' % i)
+            for i, segment_url in enumerate(segment_urls)])
+
+        for i, segment in enumerate(segments_to_download):
+            # In DASH, the first segment contains necessary headers to
+            # generate a valid MP4 file, so always abort for the first segment
+            fatal = i == 0 or not skip_unavailable_fragments
+            if not process_segment(segment, ctx['tmpfilename'], fatal):
                 return False
 
         self._finish_frag_download(ctx)

From 0def758782c273e0a1c9984f895638845796715b Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 4 Sep 2016 11:42:15 +0100
Subject: [PATCH 5988/6219] [internetvideoarchive] extract all formats

---
 youtube_dl/extractor/common.py               | 14 +++++++-------
 youtube_dl/extractor/internetvideoarchive.py | 15 ++++++++++++---
 2 files changed, 19 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index a82968162..6edd5a769 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1163,13 +1163,6 @@ class InfoExtractor(object):
                               m3u8_id=None, note=None, errnote=None,
                               fatal=True, live=False):
 
-        formats = [self._m3u8_meta_format(m3u8_url, ext, preference, m3u8_id)]
-
-        format_url = lambda u: (
-            u
-            if re.match(r'^https?://', u)
-            else compat_urlparse.urljoin(m3u8_url, u))
-
         res = self._download_webpage_handle(
             m3u8_url, video_id,
             note=note or 'Downloading m3u8 information',
@@ -1180,6 +1173,13 @@ class InfoExtractor(object):
         m3u8_doc, urlh = res
         m3u8_url = urlh.geturl()
 
+        formats = [self._m3u8_meta_format(m3u8_url, ext, preference, m3u8_id)]
+
+        format_url = lambda u: (
+            u
+            if re.match(r'^https?://', u)
+            else compat_urlparse.urljoin(m3u8_url, u))
+
         # We should try extracting formats only from master playlists [1], i.e.
         # playlists that describe available qualities. On the other hand media
         # playlists [2] should be returned as is since they contain just the media
diff --git a/youtube_dl/extractor/internetvideoarchive.py b/youtube_dl/extractor/internetvideoarchive.py
index 45add007f..76cc5ec3e 100644
--- a/youtube_dl/extractor/internetvideoarchive.py
+++ b/youtube_dl/extractor/internetvideoarchive.py
@@ -48,13 +48,23 @@ class InternetVideoArchiveIE(InfoExtractor):
             # There are multiple videos in the playlist whlie only the first one
             # matches the video played in browsers
             video_info = configuration['playlist'][0]
+            title = video_info['title']
 
             formats = []
             for source in video_info['sources']:
                 file_url = source['file']
                 if determine_ext(file_url) == 'm3u8':
-                    formats.extend(self._extract_m3u8_formats(
-                        file_url, video_id, ext='mp4', m3u8_id='hls'))
+                    m3u8_formats = self._extract_m3u8_formats(
+                        file_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False)
+                    if m3u8_formats:
+                        formats.extend(m3u8_formats)
+                        file_url = m3u8_formats[0]['url']
+                        formats.extend(self._extract_f4m_formats(
+                            file_url.replace('.m3u8', '.f4m'),
+                            video_id, f4m_id='hds', fatal=False))
+                        formats.extend(self._extract_mpd_formats(
+                            file_url.replace('.m3u8', '.mpd'),
+                            video_id, mpd_id='dash', fatal=False))
                 else:
                     a_format = {
                         'url': file_url,
@@ -70,7 +80,6 @@ class InternetVideoArchiveIE(InfoExtractor):
 
             self._sort_formats(formats)
 
-            title = video_info['title']
             description = video_info.get('description')
             thumbnail = video_info.get('image')
         else:

From 100bd86a68b5ee84669d162c9bcda31616f6596a Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 4 Sep 2016 11:44:13 +0100
Subject: [PATCH 5989/6219] [rottentomatoes] delegate extraction to
 InternetVideoArchiveIE

---
 youtube_dl/extractor/rottentomatoes.py | 25 ++++++-------------------
 1 file changed, 6 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/extractor/rottentomatoes.py b/youtube_dl/extractor/rottentomatoes.py
index df39ed3f2..23abf7a27 100644
--- a/youtube_dl/extractor/rottentomatoes.py
+++ b/youtube_dl/extractor/rottentomatoes.py
@@ -1,7 +1,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..utils import js_to_json
+from .internetvideoarchive import InternetVideoArchiveIE
 
 
 class RottenTomatoesIE(InfoExtractor):
@@ -13,6 +13,7 @@ class RottenTomatoesIE(InfoExtractor):
             'id': '11028566',
             'ext': 'mp4',
             'title': 'Toy Story 3',
+            'description': 'From the creators of the beloved TOY STORY films, comes a story that will reunite the gang in a whole new way.',
             'thumbnail': 're:^https?://.*\.jpg$',
         },
     }
@@ -20,26 +21,12 @@ class RottenTomatoesIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
-
-        params = self._parse_json(
-            self._search_regex(r'(?s)RTVideo\(({.+?})\);', webpage, 'player parameters'),
-            video_id, transform_source=lambda s: js_to_json(s.replace('window.location.href', '""')))
-
-        formats = []
-        if params.get('urlHLS'):
-            formats.extend(self._extract_m3u8_formats(
-                params['urlHLS'], video_id, ext='mp4',
-                entry_protocol='m3u8_native', m3u8_id='hls', fatal=False))
-        if params.get('urlMP4'):
-            formats.append({
-                'url': params['urlMP4'],
-                'format_id': 'mp4',
-            })
-        self._sort_formats(formats)
+        iva_id = self._search_regex(r'publishedid=(\d+)', webpage, 'internet video archive id')
 
         return {
+            '_type': 'url_transparent',
+            'url': 'http://video.internetvideoarchive.net/player/6/configuration.ashx?domain=www.videodetective.com&customerid=69249&playerid=641&publishedid=' + iva_id,
+            'ie_key': InternetVideoArchiveIE.ie_key(),
             'id': video_id,
             'title': self._og_search_title(webpage),
-            'formats': formats,
-            'thumbnail': params.get('thumbnailImg'),
         }

From feaa5ad787cdc28e4b6979f1c7798134b1bee723 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 4 Sep 2016 20:12:34 +0700
Subject: [PATCH 5990/6219] [youtube:playlist] Extend _VALID_URL

---
 youtube_dl/extractor/youtube.py | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 0bc85af74..8fc26bd02 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -264,7 +264,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                          )
                      )?                                                       # all until now is optional -> you can pass the naked ID
                      ([0-9A-Za-z_-]{11})                                      # here is it! the YouTube video ID
-                     (?!.*?&list=)                                            # combined list/video URLs are handled by the playlist IE
+                     (?!.*?\blist=)                                            # combined list/video URLs are handled by the playlist IE
                      (?(1).+)?                                                # if we found the ID, everything can follow
                      $"""
     _NEXT_URL_RE = r'[\?&]next_url=([^&]+)'
@@ -1778,11 +1778,14 @@ class YoutubePlaylistIE(YoutubePlaylistBaseInfoExtractor):
     _VALID_URL = r"""(?x)(?:
                         (?:https?://)?
                         (?:\w+\.)?
-                        youtube\.com/
                         (?:
-                           (?:course|view_play_list|my_playlists|artist|playlist|watch|embed/videoseries)
-                           \? (?:.*?[&;])*? (?:p|a|list)=
-                        |  p/
+                            youtube\.com/
+                            (?:
+                               (?:course|view_play_list|my_playlists|artist|playlist|watch|embed/videoseries)
+                               \? (?:.*?[&;])*? (?:p|a|list)=
+                            |  p/
+                            )|
+                            youtu\.be/[0-9A-Za-z_-]{11}\?.*?\blist=
                         )
                         (
                             (?:PL|LL|EC|UU|FL|RD|UL)?[0-9A-Za-z-_]{10,}
@@ -1887,6 +1890,9 @@ class YoutubePlaylistIE(YoutubePlaylistBaseInfoExtractor):
             'skip_download': True,
         },
         'add_ie': [YoutubeIE.ie_key()],
+    }, {
+        'url': 'https://youtu.be/uWyaPkt-VOI?list=PL9D9FC436B881BA21',
+        'only_matching': True,
     }]
 
     def _real_initialize(self):

From 433af6ad3002424ecb316e23946722d54010dbe1 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 4 Sep 2016 14:18:41 +0100
Subject: [PATCH 5991/6219] [theplatform] fix player regex(closes #10546)

---
 youtube_dl/extractor/theplatform.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index 23067e8c6..6febf805b 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -96,7 +96,7 @@ class ThePlatformBaseIE(OnceIE):
 class ThePlatformIE(ThePlatformBaseIE, AdobePassIE):
     _VALID_URL = r'''(?x)
         (?:https?://(?:link|player)\.theplatform\.com/[sp]/(?P<provider_id>[^/]+)/
-           (?:(?:(?:[^/]+/)+select/)?(?P<media>media/(?:guid/\d+/)?)|(?P<config>(?:[^/\?]+/(?:swf|config)|onsite)/select/))?
+           (?:(?:(?:[^/]+/)+select/)?(?P<media>media/(?:guid/\d+/)?)?|(?P<config>(?:[^/\?]+/(?:swf|config)|onsite)/select/))?
          |theplatform:)(?P<id>[^/\?&]+)'''
 
     _TESTS = [{
@@ -116,6 +116,7 @@ class ThePlatformIE(ThePlatformBaseIE, AdobePassIE):
             # rtmp download
             'skip_download': True,
         },
+        'skip': '404 Not Found',
     }, {
         # from http://www.cnet.com/videos/tesla-model-s-a-second-step-towards-a-cleaner-motoring-future/
         'url': 'http://link.theplatform.com/s/kYEXFC/22d_qsQ6MIRT',

From d9606d9b6cb44ee7600abf63333db4b88532a391 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 4 Sep 2016 20:51:48 +0700
Subject: [PATCH 5992/6219] release 2016.09.04

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 README.md                 | 7 ++++++-
 docs/supportedsites.md    | 2 ++
 youtube_dl/version.py     | 2 +-
 5 files changed, 13 insertions(+), 6 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index fc18e733b..1ddb3ef85 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.09.03*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.09.03**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.09.04*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.09.04**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.09.03
+[debug] youtube-dl version 2016.09.04
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 1d277b562..a26f5d4aa 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2016.09.04
 
 Core
 * If the first segment of DASH fails, abort the whole download process to
diff --git a/README.md b/README.md
index 87465aa5e..207b633db 100644
--- a/README.md
+++ b/README.md
@@ -89,6 +89,8 @@ which means you can modify it, redistribute it or use it however you like.
     --mark-watched                   Mark videos watched (YouTube only)
     --no-mark-watched                Do not mark videos watched (YouTube only)
     --no-color                       Do not emit color codes in output
+    --abort-on-unavailable-fragment  Abort downloading when some fragment is not
+                                     available
 
 ## Network Options:
     --proxy URL                      Use the specified HTTP/HTTPS/SOCKS proxy.
@@ -173,7 +175,10 @@ which means you can modify it, redistribute it or use it however you like.
     -R, --retries RETRIES            Number of retries (default is 10), or
                                      "infinite".
     --fragment-retries RETRIES       Number of retries for a fragment (default
-                                     is 10), or "infinite" (DASH only)
+                                     is 10), or "infinite" (DASH and hlsnative
+                                     only)
+    --skip-unavailable-fragments     Skip unavailable fragments (DASH and
+                                     hlsnative only)
     --buffer-size SIZE               Size of download buffer (e.g. 1024 or 16K)
                                      (default is 1024)
     --no-resize-buffer               Do not automatically adjust the buffer
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 015332bca..9e21016f7 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -232,6 +232,7 @@
  - **FacebookPluginsVideo**
  - **faz.net**
  - **fc2**
+ - **fc2:embed**
  - **Fczenit**
  - **features.aol.com**
  - **fernsehkritik.tv**
@@ -245,6 +246,7 @@
  - **FOX**
  - **Foxgay**
  - **FoxNews**: Fox News and Fox Business Video
+ - **foxnews:insider**
  - **FoxSports**
  - **france2.fr:generation-quoi**
  - **FranceCulture**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 5be8c0122..3d12a47e8 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.09.03'
+__version__ = '2016.09.04'

From 8112bfeabae792754f51e0c012ed34c4dc521bac Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 4 Sep 2016 20:57:18 +0700
Subject: [PATCH 5993/6219] [ChangeLog] Actualize

---
 ChangeLog | 21 +++++++++++++++++----
 1 file changed, 17 insertions(+), 4 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index a26f5d4aa..a542496a3 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,12 +1,26 @@
-version 2016.09.04
+version <unreleased>
 
 Core
-* If the first segment of DASH fails, abort the whole download process to
-  prevent throttling (#10497)
+* In DASH downloader if the first segment fails, abort the whole download
+  process to prevent throttling (#10497)
++ Add support for --skip-unavailable-fragments and --fragment retries in
+  hlsnative downloader (#10165, #10448).
++ Add support for --skip-unavailable-fragments in DASH downloader
++ Introduce --skip-unavailable-fragments option for fragment based downloaders
+  that allows to skip fragments unavailable due to a HTTP error
+* Fix extraction of video/audio entries with src attribute in
+  _parse_html5_media_entries (#10540)
 
 Extractors
+* [theplatform] Relax URL regular expression (#10546)
+* [youtube:playlist] Extend URL regular expression
+* [rottentomatoes] Delegate extraction to internetvideoarchive extractor
+* [internetvideoarchive] Extract all formats
 * [pornvoisines] Fix extraction (#10469)
 * [rottentomatoes] Fix extraction (#10467)
+* [espn] Extend URL regular expression (#10549)
+* [vimple] Extend URL regular expression (#10547)
+* [youtube:watchlater] Fix extraction (#10544)
 * [youjizz] Fix extraction (#10437)
 + [foxnews] Add support for FoxNews Insider (#10445)
 + [fc2] Recognize Flash player URLs (#10512)
@@ -19,7 +33,6 @@ Core
   _extract_m3u8_formats (#10522)
 * Handle semicolon in mimetype2ext
 
-
 Extractors
 + [youtube] Add support for rental videos' previews (#10532)
 * [youtube:playlist] Fallback to video extraction for video/playlist URLs when

From 48094901086534533ca89283067f2ab732857654 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 4 Sep 2016 20:58:28 +0700
Subject: [PATCH 5994/6219] release 2016.09.04.1

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 youtube_dl/version.py     | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 1ddb3ef85..c03092442 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.09.04*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.09.04**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.09.04.1*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.09.04.1**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.09.04
+[debug] youtube-dl version 2016.09.04.1
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index a542496a3..d392513ce 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2016.09.04.1
 
 Core
 * In DASH downloader if the first segment fails, abort the whole download
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 3d12a47e8..b2ea6dac6 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.09.04'
+__version__ = '2016.09.04.1'

From 78e762d23c48f85c61a8afcae29307912000a7dd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mat=C4=9Bj=20Cepl?= <mcepl@cepl.eu>
Date: Thu, 1 Sep 2016 17:31:08 +0200
Subject: [PATCH 5995/6219] Add new extractor for TV Noe (Czech Christian TV).

Fixes #10520
---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/tvnoe.py      | 44 ++++++++++++++++++++++++++++++
 2 files changed, 45 insertions(+)
 create mode 100644 youtube_dl/extractor/tvnoe.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 8c6ee0503..e47adc26c 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -916,6 +916,7 @@ from .tvc import (
 )
 from .tvigle import TvigleIE
 from .tvland import TVLandIE
+from .tvnoe import TVNoeIE
 from .tvp import (
     TVPEmbedIE,
     TVPIE,
diff --git a/youtube_dl/extractor/tvnoe.py b/youtube_dl/extractor/tvnoe.py
new file mode 100644
index 000000000..d50261ddd
--- /dev/null
+++ b/youtube_dl/extractor/tvnoe.py
@@ -0,0 +1,44 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .jwplatform import JWPlatformBaseIE
+from ..utils import clean_html, get_element_by_class, js_to_json
+
+
+class TVNoeIE(JWPlatformBaseIE):
+    _VALID_URL = r'https?://(www\.)?tvnoe\.cz/video/(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'http://www.tvnoe.cz/video/10362',
+        'md5': 'aee983f279aab96ec45ab6e2abb3c2ca',
+        'info_dict': {
+            'id': '10362',
+            'ext': 'mp4',
+            'series': 'Noční univerzita',
+            'title': 'prof. Tomáš Halík, Th.D. - ' +
+                     'Návrat náboženství a střet civilizací',
+            'description': 'md5:f337bae384e1a531a52c55ebc50fff41',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        iframe_url = self._search_regex(r'<iframe[^>]+src="([^"]+)"',
+                                        webpage, 'iframe src attribute')
+
+        ifs_page = self._download_webpage(iframe_url, video_id)
+        jwplayer_data = self._parse_json(self._find_jwplayer_data(ifs_page),
+                                         video_id, transform_source=js_to_json)
+        info_dict = self._parse_jwplayer_data(
+            jwplayer_data, video_id, require_title=False, base_url=iframe_url)
+
+        info_dict.update({
+            'id': video_id,
+            'title': clean_html(
+               get_element_by_class('field-name-field-podnazev', webpage)),
+            'description': clean_html(get_element_by_class('field-name-body',
+                                                           webpage)),
+            'series': clean_html(get_element_by_class('title', webpage))
+        })
+        return info_dict

From 9127e1533d294eb672d783d1eeed15aeb9b2cbe1 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 5 Sep 2016 13:37:36 +0800
Subject: [PATCH 5996/6219] [tvnoe] PEP8 and coding style

---
 youtube_dl/extractor/tvnoe.py | 27 ++++++++++++++++-----------
 1 file changed, 16 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/tvnoe.py b/youtube_dl/extractor/tvnoe.py
index d50261ddd..1cd3e6a58 100644
--- a/youtube_dl/extractor/tvnoe.py
+++ b/youtube_dl/extractor/tvnoe.py
@@ -2,7 +2,11 @@
 from __future__ import unicode_literals
 
 from .jwplatform import JWPlatformBaseIE
-from ..utils import clean_html, get_element_by_class, js_to_json
+from ..utils import (
+    clean_html,
+    get_element_by_class,
+    js_to_json,
+)
 
 
 class TVNoeIE(JWPlatformBaseIE):
@@ -14,8 +18,7 @@ class TVNoeIE(JWPlatformBaseIE):
             'id': '10362',
             'ext': 'mp4',
             'series': 'Noční univerzita',
-            'title': 'prof. Tomáš Halík, Th.D. - ' +
-                     'Návrat náboženství a střet civilizací',
+            'title': 'prof. Tomáš Halík, Th.D. - Návrat náboženství a střet civilizací',
             'description': 'md5:f337bae384e1a531a52c55ebc50fff41',
         }
     }
@@ -24,21 +27,23 @@ class TVNoeIE(JWPlatformBaseIE):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        iframe_url = self._search_regex(r'<iframe[^>]+src="([^"]+)"',
-                                        webpage, 'iframe src attribute')
+        iframe_url = self._search_regex(
+            r'<iframe[^>]+src="([^"]+)"', webpage, 'iframe URL')
 
         ifs_page = self._download_webpage(iframe_url, video_id)
-        jwplayer_data = self._parse_json(self._find_jwplayer_data(ifs_page),
-                                         video_id, transform_source=js_to_json)
+        jwplayer_data = self._parse_json(
+            self._find_jwplayer_data(ifs_page),
+            video_id, transform_source=js_to_json)
         info_dict = self._parse_jwplayer_data(
             jwplayer_data, video_id, require_title=False, base_url=iframe_url)
 
         info_dict.update({
             'id': video_id,
-            'title': clean_html(
-               get_element_by_class('field-name-field-podnazev', webpage)),
-            'description': clean_html(get_element_by_class('field-name-body',
-                                                           webpage)),
+            'title': clean_html(get_element_by_class(
+                'field-name-field-podnazev', webpage)),
+            'description': clean_html(get_element_by_class(
+                'field-name-body', webpage)),
             'series': clean_html(get_element_by_class('title', webpage))
         })
+
         return info_dict

From b49ad71ce1d985165e07fd0f59f80f677434ad84 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 5 Sep 2016 13:38:55 +0800
Subject: [PATCH 5997/6219] [ChangeLog] Update for #10524

---
 ChangeLog | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index d392513ce..0be9b0fbb 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Extractors
++ [tvnoe] New extractor (#10524)
+
+
 version 2016.09.04.1
 
 Core

From 95be19d436d1938d104310e194e85ea5a10c3353 Mon Sep 17 00:00:00 2001
From: Xie Yanbo <xieyanbo@gmail.com>
Date: Sun, 4 Sep 2016 23:23:40 +0800
Subject: [PATCH 5998/6219] [miaopai] Add new extractor

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/miaopai.py    | 44 ++++++++++++++++++++++++++++++
 2 files changed, 45 insertions(+)
 create mode 100644 youtube_dl/extractor/miaopai.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 8c6ee0503..d511b04bc 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -471,6 +471,7 @@ from .metacafe import MetacafeIE
 from .metacritic import MetacriticIE
 from .mgoon import MgoonIE
 from .mgtv import MGTVIE
+from .miaopai import MiaoPaiIE
 from .microsoftvirtualacademy import (
     MicrosoftVirtualAcademyIE,
     MicrosoftVirtualAcademyCourseIE,
diff --git a/youtube_dl/extractor/miaopai.py b/youtube_dl/extractor/miaopai.py
new file mode 100644
index 000000000..c36b441b8
--- /dev/null
+++ b/youtube_dl/extractor/miaopai.py
@@ -0,0 +1,44 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import sanitized_Request
+
+
+class MiaoPaiIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?miaopai\.com/show/(?P<id>[-A-Za-z0-9~_]+).htm'
+    _TEST = {
+        'url': 'http://www.miaopai.com/show/n~0hO7sfV1nBEw4Y29-Hqg__.htm',
+        'md5': '095ed3f1cd96b821add957bdc29f845b',
+        'info_dict': {
+            'id': 'n~0hO7sfV1nBEw4Y29-Hqg__',
+            'ext': 'mp4',
+            'title': '西游记音乐会的秒拍视频',
+            'thumbnail': 're:^https?://.*/n~0hO7sfV1nBEw4Y29-Hqg___m.jpg',
+        }
+    }
+
+    _USER_AGENT_IPAD = 'User-Agent:Mozilla/5.0 ' \
+                       '(iPad; CPU OS 9_1 like Mac OS X) ' \
+                       'AppleWebKit/601.1.46 (KHTML, like Gecko) ' \
+                       'Version/9.0 Mobile/13B143 Safari/601.1'
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        request = sanitized_Request(url)
+        request.add_header('User-Agent', self._USER_AGENT_IPAD)
+        webpage = self._download_webpage(request, video_id)
+
+        title = self._html_search_regex(r'<title>([^<]*)</title>',
+                                        webpage,
+                                        'title')
+        regex = r"""<div *class=['"]video_img[^>]*data-url=['"]([^'"]*\.jpg)['"]"""
+        thumbnail = self._html_search_regex(regex, webpage, '')
+        regex = r"""<video *[^>]*src=['"]([^'"]*)['"]""",
+        video_url = self._html_search_regex(regex, webpage, '')
+
+        return {'id': video_id,
+                'title': title,
+                'url': video_url,
+                'thumbnail': thumbnail,
+                }

From 1f85029d82b7cb1a16e205f17ea7173d899a3be1 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 5 Sep 2016 13:53:58 +0800
Subject: [PATCH 5999/6219] [bilibili] Simplify

---
 youtube_dl/extractor/bilibili.py | 27 ++++++++++-----------------
 1 file changed, 10 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/bilibili.py b/youtube_dl/extractor/bilibili.py
index 35313c62b..8fa96d3a0 100644
--- a/youtube_dl/extractor/bilibili.py
+++ b/youtube_dl/extractor/bilibili.py
@@ -10,15 +10,12 @@ from ..utils import (
     int_or_none,
     float_or_none,
     unified_timestamp,
+    urlencode_postdata,
 )
 
-HEADERS = {
-    'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
-}
-
 
 class BiliBiliIE(InfoExtractor):
-    _VALID_URL = r'https?://(www.|bangumi.|)bilibili\.(?:tv|com)/(video/av|anime/v/)(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.|bangumi\.|)bilibili\.(?:tv|com)/(?:video/av|anime/v/)(?P<id>\d+)'
 
     _TESTS = [{
         'url': 'http://www.bilibili.tv/video/av1074402/',
@@ -89,7 +86,7 @@ class BiliBiliIE(InfoExtractor):
             'ext': 'mp4',
             'duration': 1402.357,
             'title': '混沌武士 : 第7集 四面楚歌 A Risky Racket',
-            'description': "故事发生在日本的江户时代。风是一个小酒馆的打工女。一日，酒馆里来了一群恶霸，虽然他们的举动令风十分不满，但是毕竟风只是一届女流，无法对他们采取什么行动，只能在心里嘟哝。这时，酒家里又进来了个“不良份子”无幻，说以50个丸子帮她搞定这群人，风觉得他莫名其妙，也就没多搭理他。而在这时，风因为一个意外而将茶水泼在了恶霸头领——龙次郎身上。愤怒的恶霸们欲将风的手指砍掉，风在无奈中大喊道：“丸子100个！”…… 　　另一方面，龙次郎的父亲也就是当地的代官，依仗自己有着雄厚的保镖实力，在当地欺压穷人，当看到一穷人无法交齐足够的钱过桥时，欲下令将其杀死，武士仁看不惯这一幕，于是走上前，与代官的保镖交手了…… 　　酒馆内，因为风答应给无幻100个团子，无幻将恶霸们打败了，就在这时，仁进来了。好战的无幻立刻向仁发了战书，最后两败俱伤，被代官抓入牢房，预计第二天斩首…… 　　得知该状况的风，为报救命之恩，来到了刑场，利用烟花救出了无幻和仁。而风则以救命恩人的身份，命令二人和她一起去寻找带着向日葵香味的武士……(by百科)",
+            'description': 'md5:6a9622b911565794c11f25f81d6a97d2',
             'thumbnail': 're:^http?://.+\.jpg',
         },
     }]
@@ -101,17 +98,16 @@ class BiliBiliIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        _is_episode = 'anime/v' in url
-        if not _is_episode:
+        if 'anime/v' not in url:
             cid = compat_parse_qs(self._search_regex(
                 [r'EmbedPlayer\([^)]+,\s*"([^"]+)"\)',
-                r'<iframe[^>]+src="https://secure\.bilibili\.com/secure,([^"]+)"'],
+                 r'<iframe[^>]+src="https://secure\.bilibili\.com/secure,([^"]+)"'],
                 webpage, 'player parameters'))['cid'][0]
         else:
-            url_t = 'http://bangumi.bilibili.com/web_api/get_source'
-            js = self._download_json(url_t, video_id,
-                                     data='episode_id=%s' % video_id,
-                                     headers=HEADERS)
+            js = self._download_json(
+                'http://bangumi.bilibili.com/web_api/get_source', video_id,
+                data=urlencode_postdata({'episode_id': video_id}),
+                headers={'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8'})
             cid = js['result']['cid']
 
         payload = 'appkey=%s&cid=%s&otype=json&quality=2&type=mp4' % (self._APP_KEY, cid)
@@ -147,10 +143,7 @@ class BiliBiliIE(InfoExtractor):
         description = self._html_search_meta('description', webpage)
         timestamp = unified_timestamp(self._html_search_regex(
             r'<time[^>]+datetime="([^"]+)"', webpage, 'upload time', fatal=False))
-        if _is_episode:
-            thumbnail = self._html_search_meta('og:image', webpage)
-        else:
-            thumbnail = self._html_search_meta('thumbnailUrl', webpage)
+        thumbnail = self._html_search_meta(['og:image', 'thumbnailUrl'], webpage)
 
         # TODO 'view_count' requires deobfuscating Javascript
         info = {

From 973dee491f24382161b1673d4d508dcede1da79e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 5 Sep 2016 13:54:35 +0800
Subject: [PATCH 6000/6219] [ChangeLog] Update for #10190

---
 ChangeLog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ChangeLog b/ChangeLog
index 0be9b0fbb..f6858f1d8 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
 version <unreleased>
 
 Extractors
++ [bilibili] Support episodes (#10190)
 + [tvnoe] New extractor (#10524)
 
 

From bc22a7969485ada4b8cca242ac96a20e5e9e37b3 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 5 Sep 2016 16:44:06 +0800
Subject: [PATCH 6001/6219] Credit @mcepl for #10524

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index c4bef040a..78660f014 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -183,3 +183,4 @@ Petr Zvoníček
 Pratyush Singh
 Aleksander Nitecki
 Sebastian Blunt
+Matěj Cepl

From 0b36a96212861b3fc3f11b26c22216ed3cc4d2ce Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 5 Sep 2016 13:41:11 +0100
Subject: [PATCH 6002/6219] [abcotvs] extend _VALID_URL and add support for
 clips.abcotvs.com(closes #9551)

---
 .../extractor/{abc7news.py => abcotvs.py}     | 53 +++++++++++++++++--
 youtube_dl/extractor/extractors.py            |  5 +-
 2 files changed, 52 insertions(+), 6 deletions(-)
 rename youtube_dl/extractor/{abc7news.py => abcotvs.py} (53%)

diff --git a/youtube_dl/extractor/abc7news.py b/youtube_dl/extractor/abcotvs.py
similarity index 53%
rename from youtube_dl/extractor/abc7news.py
rename to youtube_dl/extractor/abcotvs.py
index c04949c21..53a900e50 100644
--- a/youtube_dl/extractor/abc7news.py
+++ b/youtube_dl/extractor/abcotvs.py
@@ -1,13 +1,18 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
 import re
 
 from .common import InfoExtractor
-from ..utils import parse_iso8601
+from ..utils import (
+    int_or_none,
+    parse_iso8601,
+)
 
 
-class Abc7NewsIE(InfoExtractor):
-    _VALID_URL = r'https?://abc7news\.com(?:/[^/]+/(?P<display_id>[^/]+))?/(?P<id>\d+)'
+class ABCOTVSIE(InfoExtractor):
+    IE_NAME = 'abcotvs'
+    _VALID_URL = r'https?://(?:abc(?:7(?:news|ny|chicago)?|11|13|30)|6abc)\.com(?:/[^/]+/(?P<display_id>[^/]+))?/(?P<id>\d+)'
     _TESTS = [
         {
             'url': 'http://abc7news.com/entertainment/east-bay-museum-celebrates-vintage-synthesizers/472581/',
@@ -15,7 +20,7 @@ class Abc7NewsIE(InfoExtractor):
                 'id': '472581',
                 'display_id': 'east-bay-museum-celebrates-vintage-synthesizers',
                 'ext': 'mp4',
-                'title': 'East Bay museum celebrates history of synthesized music',
+                'title': 'East Bay museum celebrates vintage synthesizers',
                 'description': 'md5:a4f10fb2f2a02565c1749d4adbab4b10',
                 'thumbnail': 're:^https?://.*\.jpg$',
                 'timestamp': 1421123075,
@@ -41,7 +46,7 @@ class Abc7NewsIE(InfoExtractor):
         webpage = self._download_webpage(url, display_id)
 
         m3u8 = self._html_search_meta(
-            'contentURL', webpage, 'm3u8 url', fatal=True)
+            'contentURL', webpage, 'm3u8 url', fatal=True).split('?')[0]
 
         formats = self._extract_m3u8_formats(m3u8, display_id, 'mp4')
         self._sort_formats(formats)
@@ -66,3 +71,41 @@ class Abc7NewsIE(InfoExtractor):
             'uploader': uploader,
             'formats': formats,
         }
+
+
+class ABCOTVSClipsIE(InfoExtractor):
+    IE_NAME = 'abcotvs:clips'
+    _VALID_URL = r'https?://clips\.abcotvs\.com/(?:[^/]+/)*video/(?P<id>\d+)'
+    _TEST = {
+        'url': 'https://clips.abcotvs.com/kabc/video/214814',
+        'info_dict': {
+            'id': '214814',
+            'ext': 'mp4',
+            'title': 'SpaceX launch pad explosion destroys rocket, satellite',
+            'description': 'md5:9f186e5ad8f490f65409965ee9c7be1b',
+            'upload_date': '20160901',
+            'timestamp': 1472756695,
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        video_data = self._download_json('https://clips.abcotvs.com/vogo/video/getByIds?ids=' + video_id, video_id)['results'][0]
+        title = video_data['title']
+        formats = self._extract_m3u8_formats(
+            video_data['videoURL'].split('?')[0], video_id, 'mp4')
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': video_data.get('description'),
+            'thumbnail': video_data.get('thumbnailURL'),
+            'duration': int_or_none(video_data.get('duration')),
+            'timestamp': int_or_none(video_data.get('pubDate')),
+            'formats': formats,
+        }
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index e47adc26c..dbd5fa965 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -5,11 +5,14 @@ from .abc import (
     ABCIE,
     ABCIViewIE,
 )
-from .abc7news import Abc7NewsIE
 from .abcnews import (
     AbcNewsIE,
     AbcNewsVideoIE,
 )
+from .abcotvs import (
+    ABCOTVSIE,
+    ABCOTVSClipsIE,
+)
 from .academicearth import AcademicEarthCourseIE
 from .acast import (
     ACastIE,

From b4d6a85d604ed057a72d4f09591269bd78b33111 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 5 Sep 2016 21:33:14 +0700
Subject: [PATCH 6003/6219] [nick] Add support for nickelodeon.nl (Closes
 #10559)

---
 youtube_dl/extractor/nick.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nick.py b/youtube_dl/extractor/nick.py
index 64730a624..5a07c846b 100644
--- a/youtube_dl/extractor/nick.py
+++ b/youtube_dl/extractor/nick.py
@@ -69,7 +69,7 @@ class NickIE(MTVServicesInfoExtractor):
 
 class NickDeIE(MTVServicesInfoExtractor):
     IE_NAME = 'nick.de'
-    _VALID_URL = r'https?://(?:www\.)?nick\.de/(?:playlist|shows)/(?:[^/]+/)*(?P<id>[^/?#&]+)'
+    _VALID_URL = r'https?://(?:www\.)?(?:nick\.de|nickelodeon\.nl)/(?:playlist|shows)/(?:[^/]+/)*(?P<id>[^/?#&]+)'
     _TESTS = [{
         'url': 'http://www.nick.de/playlist/3773-top-videos/videos/episode/17306-zu-wasser-und-zu-land-rauchende-erdnusse',
         'only_matching': True,

From f3eeaacb4ed084ebefb1c21d6ad82d6f1fa31a6a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 5 Sep 2016 21:42:41 +0700
Subject: [PATCH 6004/6219] [nick] Add test for #10559

---
 youtube_dl/extractor/nick.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/nick.py b/youtube_dl/extractor/nick.py
index 5a07c846b..57cf1ce8e 100644
--- a/youtube_dl/extractor/nick.py
+++ b/youtube_dl/extractor/nick.py
@@ -76,6 +76,9 @@ class NickDeIE(MTVServicesInfoExtractor):
     }, {
         'url': 'http://www.nick.de/shows/342-icarly',
         'only_matching': True,
+    }, {
+        'url': 'http://www.nickelodeon.nl/shows/474-spongebob/videos/17403-een-kijkje-in-de-keuken-met-sandy-van-binnenuit',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From a493f1020888e87a464ab8a948c797c18fc3aca3 Mon Sep 17 00:00:00 2001
From: Xie Yanbo <xieyanbo@gmail.com>
Date: Mon, 5 Sep 2016 23:08:33 +0800
Subject: [PATCH 6005/6219] using _parse_html5_media_entries to parse video tag

---
 youtube_dl/extractor/miaopai.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/miaopai.py b/youtube_dl/extractor/miaopai.py
index c36b441b8..2477d1009 100644
--- a/youtube_dl/extractor/miaopai.py
+++ b/youtube_dl/extractor/miaopai.py
@@ -34,11 +34,11 @@ class MiaoPaiIE(InfoExtractor):
                                         'title')
         regex = r"""<div *class=['"]video_img[^>]*data-url=['"]([^'"]*\.jpg)['"]"""
         thumbnail = self._html_search_regex(regex, webpage, '')
-        regex = r"""<video *[^>]*src=['"]([^'"]*)['"]""",
-        video_url = self._html_search_regex(regex, webpage, '')
+        videos = self._parse_html5_media_entries(url, webpage, video_id)
+        info = videos[0]
 
-        return {'id': video_id,
-                'title': title,
-                'url': video_url,
-                'thumbnail': thumbnail,
-                }
+        info.update({'id': video_id,
+                     'title': title,
+                     'thumbnail': thumbnail,
+                     })
+        return info

From bdcc046d12f853d1096d226923fceb18a904786c Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 6 Sep 2016 15:34:40 +0100
Subject: [PATCH 6006/6219] [turner] use android secure hls host and catch
 token extraction errors

---
 youtube_dl/extractor/cartoonnetwork.py |  2 +-
 youtube_dl/extractor/tbs.py            |  2 +-
 youtube_dl/extractor/turner.py         | 26 ++++++++++----------------
 3 files changed, 12 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/cartoonnetwork.py b/youtube_dl/extractor/cartoonnetwork.py
index b3f30b1ca..688a6375e 100644
--- a/youtube_dl/extractor/cartoonnetwork.py
+++ b/youtube_dl/extractor/cartoonnetwork.py
@@ -30,7 +30,7 @@ class CartoonNetworkIE(TurnerBaseIE):
         return self._extract_cvp_info(
             'http://www.cartoonnetwork.com/video-seo-svc/episodeservices/getCvpPlaylist?networkName=CN2&' + query, video_id, {
                 'secure': {
-                    'media_src': 'http://apple-secure.cdn.turner.com/toon/big',
+                    'media_src': 'http://androidhls-secure.cdn.turner.com/toon/big',
                     'tokenizer_src': 'http://www.cartoonnetwork.com/cntv/mvpd/processors/services/token_ipadAdobe.do',
                 },
             })
diff --git a/youtube_dl/extractor/tbs.py b/youtube_dl/extractor/tbs.py
index 79b00e376..0c351e045 100644
--- a/youtube_dl/extractor/tbs.py
+++ b/youtube_dl/extractor/tbs.py
@@ -53,7 +53,7 @@ class TBSIE(TurnerBaseIE):
                     'media_src': 'http://ht.cdn.turner.com/%s/big' % site,
                 },
                 'secure': {
-                    'media_src': 'http://apple-secure.cdn.turner.com/%s/big' % site,
+                    'media_src': 'http://androidhls-secure.cdn.turner.com/%s/big' % site,
                     'tokenizer_src': 'http://www.%s.com/video/processors/services/token_ipadAdobe.do' % domain,
                 },
             })
diff --git a/youtube_dl/extractor/turner.py b/youtube_dl/extractor/turner.py
index b59dafda6..4228c1ccc 100644
--- a/youtube_dl/extractor/turner.py
+++ b/youtube_dl/extractor/turner.py
@@ -12,7 +12,7 @@ from ..utils import (
     parse_duration,
     xpath_attr,
     update_url_query,
-    compat_urlparse,
+    ExtractorError,
 )
 
 
@@ -24,6 +24,7 @@ class TurnerBaseIE(InfoExtractor):
         video_data = self._download_xml(data_src, video_id)
         video_id = video_data.attrib['id']
         title = xpath_text(video_data, 'headline', fatal=True)
+        content_id = xpath_text(video_data, 'contentId') or video_id
         # rtmp_src = xpath_text(video_data, 'akamai/src')
         # if rtmp_src:
         #     splited_rtmp_src = rtmp_src.split(',')
@@ -54,7 +55,7 @@ class TurnerBaseIE(InfoExtractor):
                 # auth = self._download_webpage(
                 #     protected_path_data['tokenizer_src'], query={
                 #         'path': protected_path,
-                #         'videoId': video_id,
+                #         'videoId': content_id,
                 #         'aifp': aifp,
                 #     })
                 # token = xpath_text(auth, 'token')
@@ -72,8 +73,11 @@ class TurnerBaseIE(InfoExtractor):
                     auth = self._download_xml(
                         secure_path_data['tokenizer_src'], video_id, query={
                             'path': secure_path,
-                            'videoId': video_id,
+                            'videoId': content_id,
                         })
+                    error_msg = xpath_text(auth, 'error/msg')
+                    if error_msg:
+                        raise ExtractorError(error_msg, expected=True)
                     token = xpath_text(auth, 'token')
                     if not token:
                         continue
@@ -93,19 +97,9 @@ class TurnerBaseIE(InfoExtractor):
                 formats.extend(self._extract_smil_formats(
                     video_url, video_id, fatal=False))
             elif ext == 'm3u8':
-                m3u8_formats = self._extract_m3u8_formats(
-                    video_url, video_id, 'mp4', m3u8_id=format_id or 'hls',
-                    fatal=False)
-                if m3u8_formats:
-                    # Sometimes final URLs inside m3u8 are unsigned, let's fix this
-                    # ourselves
-                    qs = compat_urlparse.urlparse(video_url).query
-                    if qs:
-                        query = compat_urlparse.parse_qs(qs)
-                        for m3u8_format in m3u8_formats:
-                            m3u8_format['url'] = update_url_query(m3u8_format['url'], query)
-                            m3u8_format['extra_param_to_segment_url'] = qs
-                    formats.extend(m3u8_formats)
+                formats.extend(self._extract_m3u8_formats(
+                    video_url, video_id, 'mp4',
+                    m3u8_id=format_id or 'hls', fatal=False))
             elif ext == 'f4m':
                 formats.extend(self._extract_f4m_formats(
                     update_url_query(video_url, {'hdcore': '3.7.0'}),

From c33c962adf656fc482bdad2dd8452065512027a9 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 6 Sep 2016 15:48:51 +0100
Subject: [PATCH 6007/6219] [trutv] Add new extractor(#10519)

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/trutv.py      | 35 ++++++++++++++++++++++++++++++
 2 files changed, 36 insertions(+)
 create mode 100644 youtube_dl/extractor/trutv.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index dbd5fa965..f72faa8a7 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -890,6 +890,7 @@ from .toypics import ToypicsUserIE, ToypicsIE
 from .traileraddict import TrailerAddictIE
 from .trilulilu import TriluliluIE
 from .trollvids import TrollvidsIE
+from .trutv import TruTVIE
 from .tube8 import Tube8IE
 from .tubitv import TubiTvIE
 from .tudou import (
diff --git a/youtube_dl/extractor/trutv.py b/youtube_dl/extractor/trutv.py
new file mode 100644
index 000000000..e60d8a181
--- /dev/null
+++ b/youtube_dl/extractor/trutv.py
@@ -0,0 +1,35 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .turner import TurnerBaseIE
+
+
+class TruTVIE(TurnerBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?trutv\.com(?:(?P<path>/shows/[^/]+/videos/[^/?#]+?)\.html|/full-episodes/[^/]+/(?P<id>\d+))'
+    _TEST = {
+        'url': 'http://www.trutv.com/shows/10-things/videos/you-wont-believe-these-sports-bets.html',
+        'md5': '2cdc844f317579fed1a7251b087ff417',
+        'info_dict': {
+            'id': '/shows/10-things/videos/you-wont-believe-these-sports-bets',
+            'ext': 'mp4',
+            'title': 'You Won\'t Believe These Sports Bets',
+            'description': 'Jamie Lee sits down with a bookie to discuss the bizarre world of illegal sports betting.',
+            'upload_date': '20130305',
+        }
+    }
+
+    def _real_extract(self, url):
+        path, video_id = re.match(self._VALID_URL, url).groups()
+        if path:
+            data_src = 'http://www.trutv.com/video/cvp/v2/xml/content.xml?id=%s.xml' % path
+        else:
+            data_src = 'http://www.trutv.com/tveverywhere/services/cvpXML.do?titleId=' + video_id
+        return self._extract_cvp_info(
+            data_src, path, {
+                'secure': {
+                    'media_src': 'http://androidhls-secure.cdn.turner.com/trutv/big',
+                    'tokenizer_src': 'http://www.trutv.com/tveverywhere/processors/services/token_ipadAdobe.do',
+                },
+            })

From 155bc674c424b2364c9a573586dd33e6d3ff3483 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 7 Sep 2016 00:41:31 +0700
Subject: [PATCH 6008/6219] [viafree] Improve video id detection (Closes
 #10569)

---
 youtube_dl/extractor/tvplay.py | 34 +++++++++++++++++++++++++++++++---
 1 file changed, 31 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/tvplay.py b/youtube_dl/extractor/tvplay.py
index 4186e82db..c2a6e4e39 100644
--- a/youtube_dl/extractor/tvplay.py
+++ b/youtube_dl/extractor/tvplay.py
@@ -348,6 +348,25 @@ class ViafreeIE(InfoExtractor):
             'skip_download': True,
         },
         'add_ie': [TVPlayIE.ie_key()],
+    }, {
+        # with relatedClips
+        'url': 'http://www.viafree.se/program/reality/sommaren-med-youtube-stjarnorna/sasong-1/avsnitt-1',
+        'info_dict': {
+            'id': '758770',
+            'ext': 'mp4',
+            'title': 'Sommaren med YouTube-stjärnorna S01E01',
+            'description': 'md5:2bc69dce2c4bb48391e858539bbb0e3f',
+            'series': 'Sommaren med YouTube-stjärnorna',
+            'season': 'Säsong 1',
+            'season_number': 1,
+            'duration': 1326,
+            'timestamp': 1470905572,
+            'upload_date': '20160811',
+        },
+        'params': {
+            'skip_download': True,
+        },
+        'add_ie': [TVPlayIE.ie_key()],
     }, {
         'url': 'http://www.viafree.no/programmer/underholdning/det-beste-vorspielet/sesong-2/episode-1',
         'only_matching': True,
@@ -365,8 +384,17 @@ class ViafreeIE(InfoExtractor):
 
         webpage = self._download_webpage(url, video_id)
 
-        video_id = self._search_regex(
-            r'currentVideo["\']\s*:\s*.+?["\']id["\']\s*:\s*["\'](?P<id>\d{6,})',
-            webpage, 'video id')
+        video_id = None
+
+        thumbnail = self._og_search_thumbnail(webpage, default=None)
+        if thumbnail:
+            video_id = self._search_regex(
+                r'https?://[^/]+/imagecache/(?:[^/]+/)+seasons/\d+/(\d{6,})/',
+            thumbnail, 'video id', default=None)
+
+        if not video_id:
+            video_id = self._search_regex(
+                r'currentVideo["\']\s*:\s*.+?["\']id["\']\s*:\s*["\'](\d{6,})',
+                webpage, 'video id')
 
         return self.url_result('mtg:%s' % video_id, TVPlayIE.ie_key())

From 8d3737cda7bb710c6de96c26602d48f75ef95537 Mon Sep 17 00:00:00 2001
From: Jakub Adam Wieczorek <ja.wieczorek@student.uw.edu.pl>
Date: Thu, 25 Aug 2016 23:04:59 +0200
Subject: [PATCH 6009/6219] [polskieradio] Add support for downloading whole
 programmes.

This extends the Polskie Radio (the Polish national radio) extractor to
enable the user to download all the broadcasts of a single programme.
---
 youtube_dl/extractor/extractors.py   |  2 +-
 youtube_dl/extractor/polskieradio.py | 79 ++++++++++++++++++++++++++++
 2 files changed, 80 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index f72faa8a7..f8f91f13d 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -667,7 +667,7 @@ from .pluralsight import (
 )
 from .podomatic import PodomaticIE
 from .pokemon import PokemonIE
-from .polskieradio import PolskieRadioIE
+from .polskieradio import PolskieRadioIE, PolskieRadioProgrammeIE
 from .porn91 import Porn91IE
 from .porncom import PornComIE
 from .pornhd import PornHdIE
diff --git a/youtube_dl/extractor/polskieradio.py b/youtube_dl/extractor/polskieradio.py
index f559b899f..c51d3d9be 100644
--- a/youtube_dl/extractor/polskieradio.py
+++ b/youtube_dl/extractor/polskieradio.py
@@ -7,6 +7,7 @@ from .common import InfoExtractor
 from ..compat import (
     compat_str,
     compat_urllib_parse_unquote,
+    compat_urlparse
 )
 from ..utils import (
     int_or_none,
@@ -15,6 +16,84 @@ from ..utils import (
 )
 
 
+class PolskieRadioProgrammeIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?polskieradio\.pl/\d+(,[^/]+)?/(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'http://www.polskieradio.pl/7/5102,HISTORIA-ZYWA',
+        'info_dict': {
+            'id': '5102',
+            'title': 'HISTORIA ŻYWA',
+        },
+        'playlist_mincount': 34,
+    }, {
+        'url': 'http://www.polskieradio.pl/7/4807',
+        'info_dict': {
+            'id': '4807',
+            'title': 'Vademecum 1050. rocznicy Chrztu Polski'
+        },
+        'playlist_mincount': 5
+    }, {
+        'url': 'http://www.polskieradio.pl/7/129,Sygnaly-dnia?ref=source',
+        'only_matching': True
+    }, {
+        'url': 'http://www.polskieradio.pl/37,RedakcjaKatolicka/4143,Kierunek-Krakow',
+        'info_dict': {
+            'id': '4143',
+            'title': 'Kierunek Kraków',
+        },
+        'playlist_mincount': 61
+    }, {
+        'url': 'http://www.polskieradio.pl/7,Jedynka/5102,HISTORIA-ZYWA',
+        'only_matching': True
+    }]
+
+    def _get_entries_from_page_content(self, base_url, content):
+        entries = []
+
+        articles = re.findall(
+            r'<article class="ID-(\d+) article">\s+<a href="([^"]+)"( data-layer="[^"]*")? class="[^"]*" title="([^"]+)">',
+            content)
+        for article_id, article_url, _, article_title in articles:
+            resolved_article_url = compat_urlparse.urljoin(base_url, article_url)
+            entries.append(self.url_result(
+                resolved_article_url,
+                ie='PolskieRadio',
+                video_id=article_id,
+                video_title=article_title))
+
+        return entries
+
+    @classmethod
+    def suitable(cls, url):
+        return False if PolskieRadioIE.suitable(url) else super(PolskieRadioProgrammeIE, cls).suitable(url)
+
+    def _real_extract(self, url):
+        programme_id = self._match_id(url)
+        webpage = self._download_webpage(url, programme_id)
+
+        title = self._html_search_regex(
+            r'<a href="[^"]+" id=".*_linkCategory" title="[^"]+">(.+?)</a>',
+            webpage, 'title', fatal=False)
+        description = None
+
+        entries = self._get_entries_from_page_content(url, webpage)
+
+        pages = re.findall(r'<a( href="([^"]+/Strona/)\d+")? id="[^"]+" title="strona&#32;(\d+)"', webpage)
+        page_count = max(int(page_number) for _, _, page_number in pages) if pages else 1
+
+        if page_count > 1:
+            page_url_root = next(url for _, url, _ in pages if len(url) > 0)
+            for page_number in range(2, page_count + 1):
+                page_url = page_url_root + str(page_number)
+                resolved_page_url = compat_urlparse.urljoin(url, page_url)
+                page_content = self._download_webpage(
+                    resolved_page_url, programme_id,
+                    note="Downloading page number %d" % page_number)
+                entries.extend(self._get_entries_from_page_content(url, page_content))
+
+        return self.playlist_result(entries, programme_id, title, description)
+
+
 class PolskieRadioIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?polskieradio\.pl/\d+/\d+/Artykul/(?P<id>[0-9]+)'
     _TESTS = [{

From 948cd5b72d6d3f8a8b70717de50f7f58c260e65b Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 6 Sep 2016 20:44:45 +0100
Subject: [PATCH 6010/6219] [wat] extract dash formats

---
 youtube_dl/extractor/wat.py | 70 ++++++++++++++++++++++---------------
 1 file changed, 41 insertions(+), 29 deletions(-)

diff --git a/youtube_dl/extractor/wat.py b/youtube_dl/extractor/wat.py
index 9f1b8b4b5..20fef1f04 100644
--- a/youtube_dl/extractor/wat.py
+++ b/youtube_dl/extractor/wat.py
@@ -86,38 +86,50 @@ class WatIE(InfoExtractor):
 
         def extract_url(path_template, url_type):
             req_url = 'http://www.wat.tv/get/%s' % (path_template % video_id)
-            head = self._request_webpage(HEADRequest(req_url), video_id, 'Extracting %s url' % url_type)
-            red_url = head.geturl()
-            if req_url == red_url:
-                raise ExtractorError(
-                    '%s said: Sorry, this video is not available from your country.' % self.IE_NAME,
-                    expected=True)
-            return red_url
+            head = self._request_webpage(HEADRequest(req_url), video_id, 'Extracting %s url' % url_type, fatal=False)
+            if head:
+                red_url = head.geturl()
+                if req_url != red_url:
+                    return red_url
+            return None
+
+        def remove_bitrate_limit(manifest_url):
+            return re.sub(r'(?:max|min)_bitrate=\d+&?', '', manifest_url)
 
         formats = []
         try:
-            http_url = extract_url('android5/%s.mp4', 'http')
-            m3u8_url = extract_url('ipad/%s.m3u8', 'm3u8')
-            m3u8_formats = self._extract_m3u8_formats(
-                m3u8_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls')
-            formats.extend(m3u8_formats)
-            formats.extend(self._extract_f4m_formats(
-                m3u8_url.replace('ios.', 'web.').replace('.m3u8', '.f4m'),
-                video_id, f4m_id='hds', fatal=False))
-            for m3u8_format in m3u8_formats:
-                vbr, abr = m3u8_format.get('vbr'), m3u8_format.get('abr')
-                if not vbr or not abr:
-                    continue
-                format_id = m3u8_format['format_id'].replace('hls', 'http')
-                fmt_url = re.sub(r'%s-\d+00-\d+' % video_id, '%s-%d00-%d' % (video_id, round(vbr / 100), round(abr)), http_url)
-                if self._is_valid_url(fmt_url, video_id, format_id):
-                    f = m3u8_format.copy()
-                    f.update({
-                        'url': fmt_url,
-                        'format_id': format_id,
-                        'protocol': 'http',
-                    })
-                    formats.append(f)
+            manifest_urls = self._download_json(
+                'http://www.wat.tv/get/webhtml/' + video_id, video_id)
+            m3u8_url = manifest_urls.get('hls')
+            if m3u8_url:
+                m3u8_url = remove_bitrate_limit(m3u8_url)
+                m3u8_formats = self._extract_m3u8_formats(
+                    m3u8_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False)
+                if m3u8_formats:
+                    formats.extend(m3u8_formats)
+                    formats.extend(self._extract_f4m_formats(
+                        m3u8_url.replace('ios', 'web').replace('.m3u8', '.f4m'),
+                        video_id, f4m_id='hds', fatal=False))
+                    http_url = extract_url('android5/%s.mp4', 'http')
+                    if http_url:
+                        for m3u8_format in m3u8_formats:
+                            vbr, abr = m3u8_format.get('vbr'), m3u8_format.get('abr')
+                            if not vbr or not abr:
+                                continue
+                            format_id = m3u8_format['format_id'].replace('hls', 'http')
+                            fmt_url = re.sub(r'%s-\d+00-\d+' % video_id, '%s-%d00-%d' % (video_id, round(vbr / 100), round(abr)), http_url)
+                            if self._is_valid_url(fmt_url, video_id, format_id):
+                                f = m3u8_format.copy()
+                                f.update({
+                                    'url': fmt_url,
+                                    'format_id': format_id,
+                                    'protocol': 'http',
+                                })
+                                formats.append(f)
+            mpd_url = manifest_urls.get('mpd')
+            if mpd_url:
+                formats.extend(self._extract_mpd_formats(remove_bitrate_limit(
+                    mpd_url), video_id, mpd_id='dash', fatal=False))
             self._sort_formats(formats)
         except ExtractorError:
             abr = 64

From 8258f4457ce37fd24fb6044b5948f1816fcf3b94 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 6 Sep 2016 20:47:42 +0100
Subject: [PATCH 6011/6219] [lci] Add new extractor(closes #10573)

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/lci.py        | 24 ++++++++++++++++++++++++
 2 files changed, 25 insertions(+)
 create mode 100644 youtube_dl/extractor/lci.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index f72faa8a7..192d29134 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -424,6 +424,7 @@ from .kuwo import (
 )
 from .la7 import LA7IE
 from .laola1tv import Laola1TvIE
+from .lci import LCIIE
 from .lcp import (
     LcpPlayIE,
     LcpIE,
diff --git a/youtube_dl/extractor/lci.py b/youtube_dl/extractor/lci.py
new file mode 100644
index 000000000..af34829e7
--- /dev/null
+++ b/youtube_dl/extractor/lci.py
@@ -0,0 +1,24 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class LCIIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?lci\.fr/[^/]+/[\w-]+-(?P<id>\d+)\.html'
+    _TEST = {
+        'url': 'http://www.lci.fr/international/etats-unis-a-j-62-hillary-clinton-reste-sans-voix-2001679.html',
+        'md5': '2fdb2538b884d4d695f9bd2bde137e6c',
+        'info_dict': {
+            'id': '13244802',
+            'ext': 'mp4',
+            'title': 'Hillary Clinton et sa quinte de toux, en plein meeting',
+            'description': 'md5:a4363e3a960860132f8124b62f4a01c9',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        wat_id = self._search_regex(r'data-watid=[\'"](\d+)', webpage, 'wat id')
+        return self.url_result('wat:' + wat_id, 'Wat', wat_id)

From aa3f9fe695134719de87837238045beec10c4ab5 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 7 Sep 2016 10:38:59 +0200
Subject: [PATCH 6012/6219] Explain why and why not to specify
 --hls-prefer-native

This has been asked at http://stackoverflow.com/questions/39357037/what-does-youtube-dl-option-hls-prefer-native-do-any-downside-adding-to-youtu
---
 README.md | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/README.md b/README.md
index 207b633db..7543f81ac 100644
--- a/README.md
+++ b/README.md
@@ -851,6 +851,16 @@ will download the complete `PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re` playlist and cre
 
     youtube-dl --download-archive archive.txt "https://www.youtube.com/playlist?list=PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re"
 
+### Should I add `--hls-prefer-native` into my config?
+
+When youtube-dl detects an HLS video, it can download it either with the built-in downloader or ffmpeg. Since many HLS streams are slightly invalid and ffmpeg/youtube-dl each handle some invalid cases better than the other, there is an option to switch the downloader if needed.
+
+When youtube-dl knows that one particular downloader works better for a given website, that downloader will be picked. Otherwise, youtube-dl will pick the best downloader for general compatibility, which at the moment happens to be ffmpeg. This choice may change in future versions of youtube-dl, with improvements of the built-in downloader and/or ffmpeg.
+
+In particular, the generic extractor (used when your website is not in the [list of supported sites by youtube-dl](http://rg3.github.io/youtube-dl/supportedsites.html) cannot mandate one specific downloader.
+
+If you put either `--hls-prefer-native` or `--hls-prefer-ffmpeg` into your configuration, a different subset of videos will fail to download correctly. Instead, it is much better to [file an issue](https://yt-dl.org/bug) or a pull request which details why the native or the ffmpeg HLS downloader is a better choice for your use case.
+
 ### Can you add support for this anime video site, or site which shows current movies for free?
 
 As a matter of policy (as well as legality), youtube-dl does not include support for services that specialize in infringing copyright. As a rule of thumb, if you cannot easily find a video that the service is quite obviously allowed to distribute (i.e. that has been uploaded by the creator, the creator's distributor, or is published under a free license), the service is probably unfit for inclusion to youtube-dl.

From 846d8b76a07f617b595e0f1a2ae7ba22acafd98d Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 7 Sep 2016 10:10:43 +0100
Subject: [PATCH 6013/6219] [cctv] Add new extractor(closes #8153)

---
 youtube_dl/extractor/cctv.py       | 53 ++++++++++++++++++++++++++++++
 youtube_dl/extractor/extractors.py |  1 +
 2 files changed, 54 insertions(+)
 create mode 100644 youtube_dl/extractor/cctv.py

diff --git a/youtube_dl/extractor/cctv.py b/youtube_dl/extractor/cctv.py
new file mode 100644
index 000000000..72a72cb73
--- /dev/null
+++ b/youtube_dl/extractor/cctv.py
@@ -0,0 +1,53 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import float_or_none
+
+
+class CCTVIE(InfoExtractor):
+    _VALID_URL = r'''(?x)https?://(?:.+?\.)?
+        (?:
+            cctv\.(?:com|cn)|
+            cntv\.cn
+        )/
+        (?:
+            video/[^/]+/(?P<id>[0-9a-f]{32})|
+            \d{4}/\d{2}/\d{2}/(?P<display_id>VID[0-9A-Za-z]+)
+        )'''
+    _TESTS = [{
+        'url': 'http://english.cntv.cn/2016/09/03/VIDEhnkB5y9AgHyIEVphCEz1160903.shtml',
+        'md5': '819c7b49fc3927d529fb4cd555621823',
+        'info_dict': {
+            'id': '454368eb19ad44a1925bf1eb96140a61',
+            'ext': 'mp4',
+            'title': 'Portrait of Real Current Life 09/03/2016 Modern Inventors Part 1',
+        }
+    }, {
+        'url': 'http://tv.cctv.com/2016/09/07/VIDE5C1FnlX5bUywlrjhxXOV160907.shtml',
+        'only_matching': True,
+    }, {
+        'url': 'http://tv.cntv.cn/video/C39296/95cfac44cabd3ddc4a9438780a4e5c44',
+        'only_matching': True
+    }]
+
+    def _real_extract(self, url):
+        video_id, display_id = re.match(self._VALID_URL, url).groups()
+        if not video_id:
+            webpage = self._download_webpage(url, display_id)
+            video_id = self._search_regex(
+                r'(?:fo\.addVariable\("videoCenterId",\s*|guid\s*=\s*)"([0-9a-f]{32})',
+                webpage, 'video_id')
+        api_data = self._download_json(
+            'http://vdn.apps.cntv.cn/api/getHttpVideoInfo.do?pid=' + video_id, video_id)
+        m3u8_url = re.sub(r'maxbr=\d+&?', '', api_data['hls_url'])
+
+        return {
+            'id': video_id,
+            'title': api_data['title'],
+            'formats': self._extract_m3u8_formats(
+                m3u8_url, video_id, 'mp4', 'm3u8_native', fatal=False),
+            'duration': float_or_none(api_data.get('video', {}).get('totalLength')),
+        }
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 192d29134..9fe824c67 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -146,6 +146,7 @@ from .cbsnews import (
 )
 from .cbssports import CBSSportsIE
 from .ccc import CCCIE
+from .cctv import CCTVIE
 from .cda import CDAIE
 from .ceskatelevize import CeskaTelevizeIE
 from .channel9 import Channel9IE

From 9d54b02baeb1f9c7d63ba5833f33b16d0874c355 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 7 Sep 2016 14:43:20 +0100
Subject: [PATCH 6014/6219] [puls4] fix extraction(closes #10583)

---
 youtube_dl/extractor/prosiebensat1.py | 218 +++++++++++++-------------
 youtube_dl/extractor/puls4.py         | 109 +++++--------
 2 files changed, 148 insertions(+), 179 deletions(-)

diff --git a/youtube_dl/extractor/prosiebensat1.py b/youtube_dl/extractor/prosiebensat1.py
index c6eee3b72..7335dc2af 100644
--- a/youtube_dl/extractor/prosiebensat1.py
+++ b/youtube_dl/extractor/prosiebensat1.py
@@ -15,7 +15,111 @@ from ..utils import (
 )
 
 
-class ProSiebenSat1IE(InfoExtractor):
+class ProSiebenSat1BaseIE(InfoExtractor):
+    def _extract_video_info(self, url, clip_id):
+        client_location = url
+
+        video = self._download_json(
+            'http://vas.sim-technik.de/vas/live/v2/videos',
+            clip_id, 'Downloading videos JSON', query={
+                'access_token': self._TOKEN,
+                'client_location': client_location,
+                'client_name': self._CLIENT_NAME,
+                'ids': clip_id,
+            })[0]
+
+        if video.get('is_protected') is True:
+            raise ExtractorError('This video is DRM protected.', expected=True)
+
+        duration = float_or_none(video.get('duration'))
+        source_ids = [compat_str(source['id']) for source in video['sources']]
+
+        client_id = self._SALT[:2] + sha1(''.join([clip_id, self._SALT, self._TOKEN, client_location, self._SALT, self._CLIENT_NAME]).encode('utf-8')).hexdigest()
+
+        sources = self._download_json(
+            'http://vas.sim-technik.de/vas/live/v2/videos/%s/sources' % clip_id,
+            clip_id, 'Downloading sources JSON', query={
+                'access_token': self._TOKEN,
+                'client_id': client_id,
+                'client_location': client_location,
+                'client_name': self._CLIENT_NAME,
+            })
+        server_id = sources['server_id']
+
+        def fix_bitrate(bitrate):
+            bitrate = int_or_none(bitrate)
+            if not bitrate:
+                return None
+            return (bitrate // 1000) if bitrate % 1000 == 0 else bitrate
+
+        formats = []
+        for source_id in source_ids:
+            client_id = self._SALT[:2] + sha1(''.join([self._SALT, clip_id, self._TOKEN, server_id, client_location, source_id, self._SALT, self._CLIENT_NAME]).encode('utf-8')).hexdigest()
+            urls = self._download_json(
+                'http://vas.sim-technik.de/vas/live/v2/videos/%s/sources/url' % clip_id,
+                clip_id, 'Downloading urls JSON', fatal=False, query={
+                    'access_token': self._TOKEN,
+                    'client_id': client_id,
+                    'client_location': client_location,
+                    'client_name': self._CLIENT_NAME,
+                    'server_id': server_id,
+                    'source_ids': source_id,
+                })
+            if not urls:
+                continue
+            if urls.get('status_code') != 0:
+                raise ExtractorError('This video is unavailable', expected=True)
+            urls_sources = urls['sources']
+            if isinstance(urls_sources, dict):
+                urls_sources = urls_sources.values()
+            for source in urls_sources:
+                source_url = source.get('url')
+                if not source_url:
+                    continue
+                protocol = source.get('protocol')
+                mimetype = source.get('mimetype')
+                if mimetype == 'application/f4m+xml' or 'f4mgenerator' in source_url or determine_ext(source_url) == 'f4m':
+                    formats.extend(self._extract_f4m_formats(
+                        source_url, clip_id, f4m_id='hds', fatal=False))
+                elif mimetype == 'application/x-mpegURL':
+                    formats.extend(self._extract_m3u8_formats(
+                        source_url, clip_id, 'mp4', 'm3u8_native',
+                        m3u8_id='hls', fatal=False))
+                else:
+                    tbr = fix_bitrate(source['bitrate'])
+                    if protocol in ('rtmp', 'rtmpe'):
+                        mobj = re.search(r'^(?P<url>rtmpe?://[^/]+)/(?P<path>.+)$', source_url)
+                        if not mobj:
+                            continue
+                        path = mobj.group('path')
+                        mp4colon_index = path.rfind('mp4:')
+                        app = path[:mp4colon_index]
+                        play_path = path[mp4colon_index:]
+                        formats.append({
+                            'url': '%s/%s' % (mobj.group('url'), app),
+                            'app': app,
+                            'play_path': play_path,
+                            'player_url': 'http://livepassdl.conviva.com/hf/ver/2.79.0.17083/LivePassModuleMain.swf',
+                            'page_url': 'http://www.prosieben.de',
+                            'tbr': tbr,
+                            'ext': 'flv',
+                            'format_id': 'rtmp%s' % ('-%d' % tbr if tbr else ''),
+                        })
+                    else:
+                        formats.append({
+                            'url': source_url,
+                            'tbr': tbr,
+                            'format_id': 'http%s' % ('-%d' % tbr if tbr else ''),
+                        })
+        self._sort_formats(formats)
+
+        return {
+            'duration': duration,
+            'formats': formats,
+        }
+
+
+class ProSiebenSat1IE(ProSiebenSat1BaseIE):
     IE_NAME = 'prosiebensat1'
     IE_DESC = 'ProSiebenSat.1 Digital'
     _VALID_URL = r'https?://(?:www\.)?(?:(?:prosieben|prosiebenmaxx|sixx|sat1|kabeleins|the-voice-of-germany|7tv)\.(?:de|at|ch)|ran\.de|fem\.com)/(?P<id>.+)'
@@ -188,6 +292,9 @@ class ProSiebenSat1IE(InfoExtractor):
         },
     ]
 
+    _TOKEN = 'prosieben'
+    _SALT = '01!8d8F_)r9]4s[qeuXfP%'
+    _CLIENT_NAME = 'kolibri-2.0.19-splec4'
     _CLIPID_REGEXES = [
         r'"clip_id"\s*:\s+"(\d+)"',
         r'clipid: "(\d+)"',
@@ -234,123 +341,22 @@ class ProSiebenSat1IE(InfoExtractor):
     def _extract_clip(self, url, webpage):
         clip_id = self._html_search_regex(
             self._CLIPID_REGEXES, webpage, 'clip id')
-
-        access_token = 'prosieben'
-        client_name = 'kolibri-2.0.19-splec4'
-        client_location = url
-
-        video = self._download_json(
-            'http://vas.sim-technik.de/vas/live/v2/videos',
-            clip_id, 'Downloading videos JSON', query={
-                'access_token': access_token,
-                'client_location': client_location,
-                'client_name': client_name,
-                'ids': clip_id,
-            })[0]
-
-        if video.get('is_protected') is True:
-            raise ExtractorError('This video is DRM protected.', expected=True)
-
-        duration = float_or_none(video.get('duration'))
-        source_ids = [compat_str(source['id']) for source in video['sources']]
-
-        g = '01!8d8F_)r9]4s[qeuXfP%'
-        client_id = g[:2] + sha1(''.join([clip_id, g, access_token, client_location, g, client_name]).encode('utf-8')).hexdigest()
-
-        sources = self._download_json(
-            'http://vas.sim-technik.de/vas/live/v2/videos/%s/sources' % clip_id,
-            clip_id, 'Downloading sources JSON', query={
-                'access_token': access_token,
-                'client_id': client_id,
-                'client_location': client_location,
-                'client_name': client_name,
-            })
-        server_id = sources['server_id']
-
         title = self._html_search_regex(self._TITLE_REGEXES, webpage, 'title')
-
-        def fix_bitrate(bitrate):
-            bitrate = int_or_none(bitrate)
-            if not bitrate:
-                return None
-            return (bitrate // 1000) if bitrate % 1000 == 0 else bitrate
-
-        formats = []
-        for source_id in source_ids:
-            client_id = g[:2] + sha1(''.join([g, clip_id, access_token, server_id, client_location, source_id, g, client_name]).encode('utf-8')).hexdigest()
-            urls = self._download_json(
-                'http://vas.sim-technik.de/vas/live/v2/videos/%s/sources/url' % clip_id,
-                clip_id, 'Downloading urls JSON', fatal=False, query={
-                    'access_token': access_token,
-                    'client_id': client_id,
-                    'client_location': client_location,
-                    'client_name': client_name,
-                    'server_id': server_id,
-                    'source_ids': source_id,
-                })
-            if not urls:
-                continue
-            if urls.get('status_code') != 0:
-                raise ExtractorError('This video is unavailable', expected=True)
-            urls_sources = urls['sources']
-            if isinstance(urls_sources, dict):
-                urls_sources = urls_sources.values()
-            for source in urls_sources:
-                source_url = source.get('url')
-                if not source_url:
-                    continue
-                protocol = source.get('protocol')
-                mimetype = source.get('mimetype')
-                if mimetype == 'application/f4m+xml' or 'f4mgenerator' in source_url or determine_ext(source_url) == 'f4m':
-                    formats.extend(self._extract_f4m_formats(
-                        source_url, clip_id, f4m_id='hds', fatal=False))
-                elif mimetype == 'application/x-mpegURL':
-                    formats.extend(self._extract_m3u8_formats(
-                        source_url, clip_id, 'mp4', 'm3u8_native',
-                        m3u8_id='hls', fatal=False))
-                else:
-                    tbr = fix_bitrate(source['bitrate'])
-                    if protocol in ('rtmp', 'rtmpe'):
-                        mobj = re.search(r'^(?P<url>rtmpe?://[^/]+)/(?P<path>.+)$', source_url)
-                        if not mobj:
-                            continue
-                        path = mobj.group('path')
-                        mp4colon_index = path.rfind('mp4:')
-                        app = path[:mp4colon_index]
-                        play_path = path[mp4colon_index:]
-                        formats.append({
-                            'url': '%s/%s' % (mobj.group('url'), app),
-                            'app': app,
-                            'play_path': play_path,
-                            'player_url': 'http://livepassdl.conviva.com/hf/ver/2.79.0.17083/LivePassModuleMain.swf',
-                            'page_url': 'http://www.prosieben.de',
-                            'tbr': tbr,
-                            'ext': 'flv',
-                            'format_id': 'rtmp%s' % ('-%d' % tbr if tbr else ''),
-                        })
-                    else:
-                        formats.append({
-                            'url': source_url,
-                            'tbr': tbr,
-                            'format_id': 'http%s' % ('-%d' % tbr if tbr else ''),
-                        })
-        self._sort_formats(formats)
-
+        info = self._extract_video_info(url, clip_id)
         description = self._html_search_regex(
             self._DESCRIPTION_REGEXES, webpage, 'description', fatal=False)
         thumbnail = self._og_search_thumbnail(webpage)
         upload_date = unified_strdate(self._html_search_regex(
             self._UPLOAD_DATE_REGEXES, webpage, 'upload date', default=None))
 
-        return {
+        info.update({
             'id': clip_id,
             'title': title,
             'description': description,
             'thumbnail': thumbnail,
             'upload_date': upload_date,
-            'duration': duration,
-            'formats': formats,
-        }
+        })
+        return info
 
     def _extract_playlist(self, url, webpage):
         playlist_id = self._html_search_regex(
diff --git a/youtube_dl/extractor/puls4.py b/youtube_dl/extractor/puls4.py
index fca30e1aa..9c2ccbe2d 100644
--- a/youtube_dl/extractor/puls4.py
+++ b/youtube_dl/extractor/puls4.py
@@ -1,88 +1,51 @@
 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
-from .common import InfoExtractor
+from .prosiebensat1 import ProSiebenSat1BaseIE
 from ..utils import (
-    ExtractorError,
     unified_strdate,
-    int_or_none,
+    parse_duration,
+    compat_str,
 )
 
 
-class Puls4IE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?puls4\.com/video/[^/]+/play/(?P<id>[0-9]+)'
+class Puls4IE(ProSiebenSat1BaseIE):
+    _VALID_URL = r'https?://(?:www\.)?puls4\.com/(?P<id>(?:[^/]+/)*?videos/[^?#]+)'
     _TESTS = [{
-        'url': 'http://www.puls4.com/video/pro-und-contra/play/2716816',
-        'md5': '49f6a6629747eeec43cef6a46b5df81d',
+        'url': 'http://www.puls4.com/2-minuten-2-millionen/staffel-3/videos/2min2miotalk/Tobias-Homberger-von-myclubs-im-2min2miotalk-118118',
+        'md5': 'fd3c6b0903ac72c9d004f04bc6bb3e03',
         'info_dict': {
-            'id': '2716816',
-            'ext': 'mp4',
-            'title': 'Pro und Contra vom 23.02.2015',
-            'description': 'md5:293e44634d9477a67122489994675db6',
-            'duration': 2989,
-            'upload_date': '20150224',
+            'id': '118118',
+            'ext': 'flv',
+            'title': 'Tobias Homberger von myclubs im #2min2miotalk',
+            'description': 'md5:f9def7c5e8745d6026d8885487d91955',
+            'upload_date': '20160830',
             'uploader': 'PULS_4',
         },
-        'skip': 'Only works from Germany',
-    }, {
-        'url': 'http://www.puls4.com/video/kult-spielfilme/play/1298106',
-        'md5': '6a48316c8903ece8dab9b9a7bf7a59ec',
-        'info_dict': {
-            'id': '1298106',
-            'ext': 'mp4',
-            'title': 'Lucky Fritz',
-        },
-        'skip': 'Only works from Germany',
     }]
+    _TOKEN = 'puls4'
+    _SALT = '01!kaNgaiNgah1Ie4AeSha'
+    _CLIENT_NAME = ''
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-
-        error_message = self._html_search_regex(
-            r'<div[^>]+class="message-error"[^>]*>(.+?)</div>',
-            webpage, 'error message', default=None)
-        if error_message:
-            raise ExtractorError(
-                '%s returned error: %s' % (self.IE_NAME, error_message), expected=True)
-
-        real_url = self._html_search_regex(
-            r'\"fsk-button\".+?href=\"([^"]+)',
-            webpage, 'fsk_button', default=None)
-        if real_url:
-            webpage = self._download_webpage(real_url, video_id)
-
-        player = self._search_regex(
-            r'p4_video_player(?:_iframe)?\("video_\d+_container"\s*,(.+?)\);\s*\}',
-            webpage, 'player')
-
-        player_json = self._parse_json(
-            '[%s]' % player, video_id,
-            transform_source=lambda s: s.replace('undefined,', ''))
-
-        formats = None
-        result = None
-
-        for v in player_json:
-            if isinstance(v, list) and not formats:
-                formats = [{
-                    'url': f['url'],
-                    'format': 'hd' if f.get('hd') else 'sd',
-                    'width': int_or_none(f.get('size_x')),
-                    'height': int_or_none(f.get('size_y')),
-                    'tbr': int_or_none(f.get('bitrate')),
-                } for f in v]
-                self._sort_formats(formats)
-            elif isinstance(v, dict) and not result:
-                result = {
-                    'id': video_id,
-                    'title': v['videopartname'].strip(),
-                    'description': v.get('videotitle'),
-                    'duration': int_or_none(v.get('videoduration') or v.get('episodeduration')),
-                    'upload_date': unified_strdate(v.get('clipreleasetime')),
-                    'uploader': v.get('channel'),
-                }
-
-        result['formats'] = formats
-
-        return result
+        path = self._match_id(url)
+        content_path = self._download_json(
+            'http://www.puls4.com/api/json-fe/page/' + path, path)['content'][0]['url']
+        media = self._download_json(
+            'http://www.puls4.com' + content_path,
+            content_path)['mediaCurrent']
+        player_content = media['playerContent']
+        info = self._extract_video_info(url, player_content['id'])
+        info.update({
+            'id': compat_str(media['objectId']),
+            'title': player_content['title'],
+            'description': media.get('description'),
+            'thumbnail': media.get('previewLink'),
+            'upload_date': unified_strdate(media.get('date')),
+            'duration': parse_duration(player_content.get('duration')),
+            'episode': player_content.get('episodePartName'),
+            'show': media.get('channel'),
+            'season_id': player_content.get('seasonId'),
+            'uploader': player_content.get('sourceCompany'),
+        })
+        return info

From 92c9c2a88b8bc6e19b6fc9ac5e3814aa75c437e9 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 7 Sep 2016 22:21:26 +0800
Subject: [PATCH 6015/6219] [moevideo] Skip another removed test (#10474)

---
 youtube_dl/extractor/moevideo.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/moevideo.py b/youtube_dl/extractor/moevideo.py
index 978d5d5bf..91ee9c4e9 100644
--- a/youtube_dl/extractor/moevideo.py
+++ b/youtube_dl/extractor/moevideo.py
@@ -35,7 +35,8 @@ class MoeVideoIE(InfoExtractor):
                 'height': 360,
                 'duration': 179,
                 'filesize': 17822500,
-            }
+            },
+            'skip': 'Video has been removed',
         },
         {
             'url': 'http://playreplay.net/video/77107.7f325710a627383d40540d8e991a',

From 84b91dd4e3cd45498fb6cd3c7aa611816f61a3d5 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 7 Sep 2016 23:07:50 +0800
Subject: [PATCH 6016/6219] [gamestar] Fix metadata extraction (closes #10479)

---
 ChangeLog                        |  1 +
 youtube_dl/extractor/gamestar.py | 51 +++++++++++---------------------
 2 files changed, 18 insertions(+), 34 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index f6858f1d8..8322af9d1 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
 version <unreleased>
 
 Extractors
+* [gamestar] Fix metadata extraction (#10479)
 + [bilibili] Support episodes (#10190)
 + [tvnoe] New extractor (#10524)
 
diff --git a/youtube_dl/extractor/gamestar.py b/youtube_dl/extractor/gamestar.py
index 69058a583..341e72733 100644
--- a/youtube_dl/extractor/gamestar.py
+++ b/youtube_dl/extractor/gamestar.py
@@ -1,14 +1,10 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..utils import (
     int_or_none,
-    parse_duration,
-    str_to_int,
-    unified_strdate,
+    remove_end,
 )
 
 
@@ -21,8 +17,9 @@ class GameStarIE(InfoExtractor):
             'id': '76110',
             'ext': 'mp4',
             'title': 'Hobbit 3: Die Schlacht der Fünf Heere - Teaser-Trailer zum dritten Teil',
-            'description': 'Der Teaser-Trailer zu Hobbit 3: Die Schlacht der Fünf Heere zeigt einige Szenen aus dem dritten Teil der Saga und kündigt den vollständigen Trailer an.',
-            'thumbnail': 'http://images.gamestar.de/images/idgwpgsgp/bdb/2494525/600x.jpg',
+            'description': 'Der Teaser-Trailer zu Hobbit 3: Die Schlacht der Fünf Heere zeigt einige Szenen aus dem dritten Teil der Saga und kündigt den...',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'timestamp': 1406542020,
             'upload_date': '20140728',
             'duration': 17
         }
@@ -32,41 +29,27 @@ class GameStarIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        og_title = self._og_search_title(webpage)
-        title = re.sub(r'\s*- Video (bei|-) GameStar\.de$', '', og_title)
-
         url = 'http://gamestar.de/_misc/videos/portal/getVideoUrl.cfm?premium=0&videoId=' + video_id
 
-        description = self._og_search_description(webpage).strip()
-
-        thumbnail = self._proto_relative_url(
-            self._og_search_thumbnail(webpage), scheme='http:')
-
-        upload_date = unified_strdate(self._html_search_regex(
-            r'<span style="float:left;font-size:11px;">Datum: ([0-9]+\.[0-9]+\.[0-9]+)&nbsp;&nbsp;',
-            webpage, 'upload_date', fatal=False))
-
-        duration = parse_duration(self._html_search_regex(
-            r'&nbsp;&nbsp;Länge: ([0-9]+:[0-9]+)</span>', webpage, 'duration',
-            fatal=False))
-
-        view_count = str_to_int(self._html_search_regex(
-            r'&nbsp;&nbsp;Zuschauer: ([0-9\.]+)&nbsp;&nbsp;', webpage,
-            'view_count', fatal=False))
+        # TODO: there are multiple ld+json objects in the webpage,
+        # while _search_json_ld finds only the first one
+        json_ld = self._parse_json(self._search_regex(
+            r'(?s)<script[^>]+type=(["\'])application/ld\+json\1[^>]*>(?P<json_ld>[^<]+VideoObject[^<]+)</script>',
+            webpage, 'JSON-LD', group='json_ld'), video_id)
+        info_dict = self._json_ld(json_ld, video_id)
+        info_dict['title'] = remove_end(info_dict['title'], ' - GameStar')
 
+        view_count = json_ld.get('interactionCount')
         comment_count = int_or_none(self._html_search_regex(
-            r'>Kommentieren \(([0-9]+)\)</a>', webpage, 'comment_count',
+            r'([0-9]+) Kommentare</span>', webpage, 'comment_count',
             fatal=False))
 
-        return {
+        info_dict.update({
             'id': video_id,
-            'title': title,
             'url': url,
             'ext': 'mp4',
-            'thumbnail': thumbnail,
-            'description': description,
-            'upload_date': upload_date,
-            'duration': duration,
             'view_count': view_count,
             'comment_count': comment_count
-        }
+        })
+
+        return info_dict

From f87feb4b688f7b6e02714f8405c8343e27956400 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 8 Sep 2016 00:28:33 +0800
Subject: [PATCH 6017/6219] [miaopai] Coding style (#10556)

---
 youtube_dl/extractor/miaopai.py | 32 ++++++++++++++------------------
 1 file changed, 14 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/miaopai.py b/youtube_dl/extractor/miaopai.py
index 2477d1009..f9e35ac7f 100644
--- a/youtube_dl/extractor/miaopai.py
+++ b/youtube_dl/extractor/miaopai.py
@@ -2,11 +2,10 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..utils import sanitized_Request
 
 
 class MiaoPaiIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?miaopai\.com/show/(?P<id>[-A-Za-z0-9~_]+).htm'
+    _VALID_URL = r'https?://(?:www\.)?miaopai\.com/show/(?P<id>[-A-Za-z0-9~_]+)'
     _TEST = {
         'url': 'http://www.miaopai.com/show/n~0hO7sfV1nBEw4Y29-Hqg__.htm',
         'md5': '095ed3f1cd96b821add957bdc29f845b',
@@ -18,27 +17,24 @@ class MiaoPaiIE(InfoExtractor):
         }
     }
 
-    _USER_AGENT_IPAD = 'User-Agent:Mozilla/5.0 ' \
-                       '(iPad; CPU OS 9_1 like Mac OS X) ' \
-                       'AppleWebKit/601.1.46 (KHTML, like Gecko) ' \
-                       'Version/9.0 Mobile/13B143 Safari/601.1'
+    _USER_AGENT_IPAD = 'Mozilla/5.0 (iPad; CPU OS 9_1 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13B143 Safari/601.1'
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        request = sanitized_Request(url)
-        request.add_header('User-Agent', self._USER_AGENT_IPAD)
-        webpage = self._download_webpage(request, video_id)
+        webpage = self._download_webpage(
+            url, video_id, headers={'User-Agent': self._USER_AGENT_IPAD})
 
-        title = self._html_search_regex(r'<title>([^<]*)</title>',
-                                        webpage,
-                                        'title')
-        regex = r"""<div *class=['"]video_img[^>]*data-url=['"]([^'"]*\.jpg)['"]"""
-        thumbnail = self._html_search_regex(regex, webpage, '')
+        title = self._html_search_regex(
+            r'<title>([^<]+)</title>', webpage, 'title')
+        thumbnail = self._html_search_regex(
+            r'<div[^>]+class=(?P<q1>[\'"]).*\bvideo_img\b.*(?P=q1)[^>]+data-url=(?P<q2>[\'"])(?P<url>[^\'"]+)(?P=q2)',
+            webpage, 'thumbnail', fatal=False, group='url')
         videos = self._parse_html5_media_entries(url, webpage, video_id)
         info = videos[0]
 
-        info.update({'id': video_id,
-                     'title': title,
-                     'thumbnail': thumbnail,
-                     })
+        info.update({
+            'id': video_id,
+            'title': title,
+            'thumbnail': thumbnail,
+        })
         return info

From b84d311d53a4cc023eb458072c09cf382f45c762 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 8 Sep 2016 00:29:55 +0800
Subject: [PATCH 6018/6219] [ChangeLog] Update for #10556

---
 ChangeLog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ChangeLog b/ChangeLog
index 8322af9d1..52c066c99 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
 version <unreleased>
 
 Extractors
++ [miaopai] New extractor (#10556)
 * [gamestar] Fix metadata extraction (#10479)
 + [bilibili] Support episodes (#10190)
 + [tvnoe] New extractor (#10524)

From d7e794928d1b1386a8f5960ce6ece3ae88d975a1 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 7 Sep 2016 17:29:06 +0100
Subject: [PATCH 6019/6219] [tlc] fix query string parsing

---
 youtube_dl/extractor/tlc.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/tlc.py b/youtube_dl/extractor/tlc.py
index abad3ff64..88eb83d74 100644
--- a/youtube_dl/extractor/tlc.py
+++ b/youtube_dl/extractor/tlc.py
@@ -1,10 +1,14 @@
 # encoding: utf-8
 from __future__ import unicode_literals
+
 import re
 
 from .common import InfoExtractor
 from .brightcove import BrightcoveLegacyIE
-from ..compat import compat_parse_qs
+from ..compat import (
+    compat_parse_qs,
+    compat_urlparse,
+)
 
 
 class TlcDeIE(InfoExtractor):
@@ -35,5 +39,5 @@ class TlcDeIE(InfoExtractor):
             title = mobj.group('title')
             webpage = self._download_webpage(url, title)
             brightcove_legacy_url = BrightcoveLegacyIE._extract_brightcove_url(webpage)
-            brightcove_id = compat_parse_qs(brightcove_legacy_url)['@videoPlayer'][0]
+            brightcove_id = compat_parse_qs(compat_urlparse.urlparse(brightcove_legacy_url).query)['@videoPlayer'][0]
         return self.url_result(self.BRIGHTCOVE_URL_TEMPLATE % brightcove_id, 'BrightcoveNew', brightcove_id)

From 6656a8248166039d83a3fb4401a7e11e03202ac9 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 7 Sep 2016 17:32:35 +0100
Subject: [PATCH 6020/6219] [rmcdecouverte] Add new extractor(closes #9709)

---
 youtube_dl/extractor/extractors.py    |  1 +
 youtube_dl/extractor/rmcdecouverte.py | 39 +++++++++++++++++++++++++++
 2 files changed, 40 insertions(+)
 create mode 100644 youtube_dl/extractor/rmcdecouverte.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 522ef7d8b..8d9c2ae13 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -724,6 +724,7 @@ from .revision3 import (
 )
 from .rice import RICEIE
 from .ringtv import RingTVIE
+from .rmcdecouverte import RMCDecouverteIE
 from .ro220 import Ro220IE
 from .rockstargames import RockstarGamesIE
 from .roosterteeth import RoosterTeethIE
diff --git a/youtube_dl/extractor/rmcdecouverte.py b/youtube_dl/extractor/rmcdecouverte.py
new file mode 100644
index 000000000..f3bb4fa66
--- /dev/null
+++ b/youtube_dl/extractor/rmcdecouverte.py
@@ -0,0 +1,39 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from .brightcove import BrightcoveLegacyIE
+from ..compat import (
+    compat_parse_qs,
+    compat_urlparse,
+)
+
+
+class RMCDecouverteIE(InfoExtractor):
+    _VALID_URL = r'https?://rmcdecouverte\.bfmtv\.com/mediaplayer-replay.*?\bid=(?P<id>\d+)'
+
+    _TEST = {
+        'url': 'http://rmcdecouverte.bfmtv.com/mediaplayer-replay/?id=1430&title=LES%20HEROS%20DU%2088e%20ETAGE',
+        'info_dict': {
+            'id': '5111223049001',
+            'ext': 'mp4',
+            'title': ': LES HEROS DU 88e ETAGE',
+            'description': 'Découvrez comment la bravoure de deux hommes dans la Tour Nord du World Trade Center a sauvé  la vie d\'innombrables personnes le 11 septembre 2001.',
+            'uploader_id': '1969646226001',
+            'upload_date': '20160904',
+            'timestamp': 1472951103,
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        },
+        'skip': 'Only works from France',
+    }
+    BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/1969646226001/default_default/index.html?videoId=%s'
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        brightcove_legacy_url = BrightcoveLegacyIE._extract_brightcove_url(webpage)
+        brightcove_id = compat_parse_qs(compat_urlparse.urlparse(brightcove_legacy_url).query)['@videoPlayer'][0]
+        return self.url_result(self.BRIGHTCOVE_URL_TEMPLATE % brightcove_id, 'BrightcoveNew', brightcove_id)

From e78a5428b6c725bd01064f080d84700b83030b66 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 8 Sep 2016 01:59:31 +0800
Subject: [PATCH 6021/6219] [foxgay] Fix extraction (closes #10480)

---
 ChangeLog                      |  1 +
 youtube_dl/extractor/foxgay.py | 48 ++++++++++++++++++++++------------
 2 files changed, 32 insertions(+), 17 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 52c066c99..cec87d5cd 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
 version <unreleased>
 
 Extractors
+* [foxgay] Fix extraction (#10480)
 + [miaopai] New extractor (#10556)
 * [gamestar] Fix metadata extraction (#10479)
 + [bilibili] Support episodes (#10190)
diff --git a/youtube_dl/extractor/foxgay.py b/youtube_dl/extractor/foxgay.py
index 70c1a815d..39174fcec 100644
--- a/youtube_dl/extractor/foxgay.py
+++ b/youtube_dl/extractor/foxgay.py
@@ -1,18 +1,24 @@
 from __future__ import unicode_literals
 
+import itertools
+
 from .common import InfoExtractor
+from ..utils import (
+    get_element_by_id,
+    remove_end,
+)
 
 
 class FoxgayIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?foxgay\.com/videos/(?:\S+-)?(?P<id>\d+)\.shtml'
     _TEST = {
         'url': 'http://foxgay.com/videos/fuck-turkish-style-2582.shtml',
-        'md5': '80d72beab5d04e1655a56ad37afe6841',
+        'md5': '344558ccfea74d33b7adbce22e577f54',
         'info_dict': {
             'id': '2582',
             'ext': 'mp4',
-            'title': 'md5:6122f7ae0fc6b21ebdf59c5e083ce25a',
-            'description': 'md5:5e51dc4405f1fd315f7927daed2ce5cf',
+            'title': 'Fuck Turkish-style',
+            'description': 'md5:6ae2d9486921891efe89231ace13ffdf',
             'age_limit': 18,
             'thumbnail': 're:https?://.*\.jpg$',
         },
@@ -22,27 +28,35 @@ class FoxgayIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        title = self._html_search_regex(
-            r'<title>(?P<title>.*?)</title>',
-            webpage, 'title', fatal=False)
-        description = self._html_search_regex(
-            r'<div class="ico_desc"><h2>(?P<description>.*?)</h2>',
-            webpage, 'description', fatal=False)
+        title = remove_end(self._html_search_regex(
+            r'<title>([^<]+)</title>', webpage, 'title'), ' - Foxgay.com')
+        description = get_element_by_id('inf_tit', webpage)
 
+        # The default user-agent with foxgay cookies leads to pages without videos
+        self._downloader.cookiejar.clear('.foxgay.com')
         # Find the URL for the iFrame which contains the actual video.
+        iframe_url = self._html_search_regex(
+            r'<iframe[^>]+src=([\'"])(?P<url>[^\'"]+)\1', webpage,
+            'video frame', group='url')
         iframe = self._download_webpage(
-            self._html_search_regex(r'iframe src="(?P<frame>.*?)"', webpage, 'video frame'),
-            video_id)
-        video_url = self._html_search_regex(
-            r"v_path = '(?P<vid>http://.*?)'", iframe, 'url')
-        thumb_url = self._html_search_regex(
-            r"t_path = '(?P<thumb>http://.*?)'", iframe, 'thumbnail', fatal=False)
+            iframe_url, video_id, headers={'User-Agent': 'curl/7.50.1'},
+            note='Downloading video frame')
+        video_data = self._parse_json(self._search_regex(
+            r'video_data\s*=\s*([^;]+);', iframe, 'video data'), video_id)
+
+        formats = [{
+            'url': source,
+            'height': resolution,
+        } for source, resolution in zip(
+            video_data['sources'], video_data.get('resolutions', itertools.repeat(None)))]
+
+        self._sort_formats(formats)
 
         return {
             'id': video_id,
             'title': title,
-            'url': video_url,
+            'formats': formats,
             'description': description,
-            'thumbnail': thumb_url,
+            'thumbnail': video_data.get('act_vid', {}).get('thumb'),
             'age_limit': 18,
         }

From 89f257d6e57131a266efae629334fe5f4bcf96e9 Mon Sep 17 00:00:00 2001
From: stepshal <nessento@openmailbox.org>
Date: Thu, 8 Sep 2016 13:52:22 +0700
Subject: [PATCH 6022/6219] Add support for https for rest of the exctractors.

---
 youtube_dl/extractor/abcnews.py     | 2 +-
 youtube_dl/extractor/dailymotion.py | 2 +-
 youtube_dl/extractor/karaoketv.py   | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/abcnews.py b/youtube_dl/extractor/abcnews.py
index b61a6327c..8a5ae42f0 100644
--- a/youtube_dl/extractor/abcnews.py
+++ b/youtube_dl/extractor/abcnews.py
@@ -12,7 +12,7 @@ from ..compat import compat_urlparse
 
 class AbcNewsVideoIE(AMPIE):
     IE_NAME = 'abcnews:video'
-    _VALID_URL = 'http://abcnews.go.com/[^/]+/video/(?P<display_id>[0-9a-z-]+)-(?P<id>\d+)'
+    _VALID_URL = 'https?://abcnews.go.com/[^/]+/video/(?P<display_id>[0-9a-z-]+)-(?P<id>\d+)'
 
     _TESTS = [{
         'url': 'http://abcnews.go.com/ThisWeek/video/week-exclusive-irans-foreign-minister-zarif-20411932',
diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index 496883d15..62b0747a5 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -394,7 +394,7 @@ class DailymotionUserIE(DailymotionPlaylistIE):
 
 
 class DailymotionCloudIE(DailymotionBaseInfoExtractor):
-    _VALID_URL_PREFIX = r'http://api\.dmcloud\.net/(?:player/)?embed/'
+    _VALID_URL_PREFIX = r'https?://api\.dmcloud\.net/(?:player/)?embed/'
     _VALID_URL = r'%s[^/]+/(?P<id>[^/?]+)' % _VALID_URL_PREFIX
     _VALID_EMBED_URL = r'%s[^/]+/[^\'"]+' % _VALID_URL_PREFIX
 
diff --git a/youtube_dl/extractor/karaoketv.py b/youtube_dl/extractor/karaoketv.py
index a6050c4de..2f37786df 100644
--- a/youtube_dl/extractor/karaoketv.py
+++ b/youtube_dl/extractor/karaoketv.py
@@ -5,7 +5,7 @@ from .common import InfoExtractor
 
 
 class KaraoketvIE(InfoExtractor):
-    _VALID_URL = r'http://www.karaoketv.co.il/[^/]+/(?P<id>\d+)'
+    _VALID_URL = r'https?://www.karaoketv.co.il/[^/]+/(?P<id>\d+)'
     _TEST = {
         'url': 'http://www.karaoketv.co.il/%D7%A9%D7%99%D7%A8%D7%99_%D7%A7%D7%A8%D7%99%D7%95%D7%A7%D7%99/58356/%D7%90%D7%99%D7%96%D7%95%D7%9F',
         'info_dict': {

From 881f35479d50f22955fa21b148aa849a64d9dfc0 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 8 Sep 2016 17:22:43 +0800
Subject: [PATCH 6023/6219] Credit @xyb for miaopai extractor (#10556)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 78660f014..937742c5d 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -184,3 +184,4 @@ Pratyush Singh
 Aleksander Nitecki
 Sebastian Blunt
 Matěj Cepl
+Xie Yanbo

From 3f612f076708973fce52d6b6053b24e2234a9c26 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 8 Sep 2016 17:39:29 +0800
Subject: [PATCH 6024/6219] Fix _VALID_URLs further (#10594)

---
 youtube_dl/extractor/abcnews.py   | 2 +-
 youtube_dl/extractor/karaoketv.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/abcnews.py b/youtube_dl/extractor/abcnews.py
index 8a5ae42f0..01247fead 100644
--- a/youtube_dl/extractor/abcnews.py
+++ b/youtube_dl/extractor/abcnews.py
@@ -12,7 +12,7 @@ from ..compat import compat_urlparse
 
 class AbcNewsVideoIE(AMPIE):
     IE_NAME = 'abcnews:video'
-    _VALID_URL = 'https?://abcnews.go.com/[^/]+/video/(?P<display_id>[0-9a-z-]+)-(?P<id>\d+)'
+    _VALID_URL = r'https?://abcnews\.go\.com/[^/]+/video/(?P<display_id>[0-9a-z-]+)-(?P<id>\d+)'
 
     _TESTS = [{
         'url': 'http://abcnews.go.com/ThisWeek/video/week-exclusive-irans-foreign-minister-zarif-20411932',
diff --git a/youtube_dl/extractor/karaoketv.py b/youtube_dl/extractor/karaoketv.py
index 2f37786df..bad46005b 100644
--- a/youtube_dl/extractor/karaoketv.py
+++ b/youtube_dl/extractor/karaoketv.py
@@ -5,7 +5,7 @@ from .common import InfoExtractor
 
 
 class KaraoketvIE(InfoExtractor):
-    _VALID_URL = r'https?://www.karaoketv.co.il/[^/]+/(?P<id>\d+)'
+    _VALID_URL = r'https?://www\.karaoketv\.co\.il/[^/]+/(?P<id>\d+)'
     _TEST = {
         'url': 'http://www.karaoketv.co.il/%D7%A9%D7%99%D7%A8%D7%99_%D7%A7%D7%A8%D7%99%D7%95%D7%A7%D7%99/58356/%D7%90%D7%99%D7%96%D7%95%D7%9F',
         'info_dict': {

From 25042f73722c37e4ec88030cf69e23ae76c4359b Mon Sep 17 00:00:00 2001
From: stepshal <nessento@openmailbox.org>
Date: Thu, 8 Sep 2016 17:04:57 +0700
Subject: [PATCH 6025/6219] Add missing r prefix for _VALID_URLs

---
 youtube_dl/extractor/abcnews.py | 2 +-
 youtube_dl/extractor/ard.py     | 2 +-
 youtube_dl/extractor/globo.py   | 4 ++--
 youtube_dl/extractor/onet.py    | 2 +-
 youtube_dl/extractor/rutube.py  | 2 +-
 youtube_dl/extractor/spiegel.py | 2 +-
 youtube_dl/extractor/twitter.py | 2 +-
 youtube_dl/extractor/youtube.py | 2 +-
 8 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/abcnews.py b/youtube_dl/extractor/abcnews.py
index 01247fead..6ae5d9a96 100644
--- a/youtube_dl/extractor/abcnews.py
+++ b/youtube_dl/extractor/abcnews.py
@@ -49,7 +49,7 @@ class AbcNewsVideoIE(AMPIE):
 
 class AbcNewsIE(InfoExtractor):
     IE_NAME = 'abcnews'
-    _VALID_URL = 'https?://abcnews\.go\.com/(?:[^/]+/)+(?P<display_id>[0-9a-z-]+)/story\?id=(?P<id>\d+)'
+    _VALID_URL = r'https?://abcnews\.go\.com/(?:[^/]+/)+(?P<display_id>[0-9a-z-]+)/story\?id=(?P<id>\d+)'
 
     _TESTS = [{
         'url': 'http://abcnews.go.com/Blotter/News/dramatic-video-rare-death-job-america/story?id=10498713#.UIhwosWHLjY',
diff --git a/youtube_dl/extractor/ard.py b/youtube_dl/extractor/ard.py
index 07e67dd33..3a806a69b 100644
--- a/youtube_dl/extractor/ard.py
+++ b/youtube_dl/extractor/ard.py
@@ -238,7 +238,7 @@ class ARDMediathekIE(InfoExtractor):
 
 
 class ARDIE(InfoExtractor):
-    _VALID_URL = '(?P<mainurl>https?://(www\.)?daserste\.de/[^?#]+/videos/(?P<display_id>[^/?#]+)-(?P<id>[0-9]+))\.html'
+    _VALID_URL = r'(?P<mainurl>https?://(www\.)?daserste\.de/[^?#]+/videos/(?P<display_id>[^/?#]+)-(?P<id>[0-9]+))\.html'
     _TEST = {
         'url': 'http://www.daserste.de/information/reportage-dokumentation/dokus/videos/die-story-im-ersten-mission-unter-falscher-flagge-100.html',
         'md5': 'd216c3a86493f9322545e045ddc3eb35',
diff --git a/youtube_dl/extractor/globo.py b/youtube_dl/extractor/globo.py
index dbacbfc61..5638be48f 100644
--- a/youtube_dl/extractor/globo.py
+++ b/youtube_dl/extractor/globo.py
@@ -19,7 +19,7 @@ from ..utils import (
 
 
 class GloboIE(InfoExtractor):
-    _VALID_URL = '(?:globo:|https?://.+?\.globo\.com/(?:[^/]+/)*(?:v/(?:[^/]+/)?|videos/))(?P<id>\d{7,})'
+    _VALID_URL = r'(?:globo:|https?://.+?\.globo\.com/(?:[^/]+/)*(?:v/(?:[^/]+/)?|videos/))(?P<id>\d{7,})'
 
     _API_URL_TEMPLATE = 'http://api.globovideos.com/videos/%s/playlist'
     _SECURITY_URL_TEMPLATE = 'http://security.video.globo.com/videos/%s/hash?player=flash&version=17.0.0.132&resource_id=%s'
@@ -396,7 +396,7 @@ class GloboIE(InfoExtractor):
 
 
 class GloboArticleIE(InfoExtractor):
-    _VALID_URL = 'https?://.+?\.globo\.com/(?:[^/]+/)*(?P<id>[^/]+)(?:\.html)?'
+    _VALID_URL = r'https?://.+?\.globo\.com/(?:[^/]+/)*(?P<id>[^/]+)(?:\.html)?'
 
     _VIDEOID_REGEXES = [
         r'\bdata-video-id=["\'](\d{7,})',
diff --git a/youtube_dl/extractor/onet.py b/youtube_dl/extractor/onet.py
index fc22ad5eb..9cbc7c2e2 100644
--- a/youtube_dl/extractor/onet.py
+++ b/youtube_dl/extractor/onet.py
@@ -90,7 +90,7 @@ class OnetBaseIE(InfoExtractor):
 
 
 class OnetIE(OnetBaseIE):
-    _VALID_URL = 'https?://(?:www\.)?onet\.tv/[a-z]/[a-z]+/(?P<display_id>[0-9a-z-]+)/(?P<id>[0-9a-z]+)'
+    _VALID_URL = r'https?://(?:www\.)?onet\.tv/[a-z]/[a-z]+/(?P<display_id>[0-9a-z-]+)/(?P<id>[0-9a-z]+)'
     IE_NAME = 'onet.tv'
 
     _TEST = {
diff --git a/youtube_dl/extractor/rutube.py b/youtube_dl/extractor/rutube.py
index 9ca4ae147..5d0ace5bf 100644
--- a/youtube_dl/extractor/rutube.py
+++ b/youtube_dl/extractor/rutube.py
@@ -88,7 +88,7 @@ class RutubeIE(InfoExtractor):
 class RutubeEmbedIE(InfoExtractor):
     IE_NAME = 'rutube:embed'
     IE_DESC = 'Rutube embedded videos'
-    _VALID_URL = 'https?://rutube\.ru/(?:video|play)/embed/(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://rutube\.ru/(?:video|play)/embed/(?P<id>[0-9]+)'
 
     _TESTS = [{
         'url': 'http://rutube.ru/video/embed/6722881?vk_puid37=&vk_puid38=',
diff --git a/youtube_dl/extractor/spiegel.py b/youtube_dl/extractor/spiegel.py
index 3c552807e..74cb3a08a 100644
--- a/youtube_dl/extractor/spiegel.py
+++ b/youtube_dl/extractor/spiegel.py
@@ -103,7 +103,7 @@ class SpiegelIE(InfoExtractor):
 
 
 class SpiegelArticleIE(InfoExtractor):
-    _VALID_URL = 'https?://www\.spiegel\.de/(?!video/)[^?#]*?-(?P<id>[0-9]+)\.html'
+    _VALID_URL = r'https?://www\.spiegel\.de/(?!video/)[^?#]*?-(?P<id>[0-9]+)\.html'
     IE_NAME = 'Spiegel:Article'
     IE_DESC = 'Articles on spiegel.de'
     _TESTS = [{
diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index b73842986..c5a5843b6 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -342,7 +342,7 @@ class TwitterIE(InfoExtractor):
 
 class TwitterAmplifyIE(TwitterBaseIE):
     IE_NAME = 'twitter:amplify'
-    _VALID_URL = 'https?://amp\.twimg\.com/v/(?P<id>[0-9a-f\-]{36})'
+    _VALID_URL = r'https?://amp\.twimg\.com/v/(?P<id>[0-9a-f\-]{36})'
 
     _TEST = {
         'url': 'https://amp.twimg.com/v/0ba0c3c7-0af3-4c0a-bed5-7efd1ffa2951',
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 8fc26bd02..5082cb589 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -2417,7 +2417,7 @@ class YoutubeSubscriptionsIE(YoutubeFeedsInfoExtractor):
 
 class YoutubeHistoryIE(YoutubeFeedsInfoExtractor):
     IE_DESC = 'Youtube watch history, ":ythistory" for short (requires authentication)'
-    _VALID_URL = 'https?://www\.youtube\.com/feed/history|:ythistory'
+    _VALID_URL = r'https?://www\.youtube\.com/feed/history|:ythistory'
     _FEED_NAME = 'history'
     _PLAYLIST_TITLE = 'Youtube History'
 

From 010d034fca60d94435d719e554663fba1894f8b3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 8 Sep 2016 22:38:49 +0700
Subject: [PATCH 6026/6219] [videomore] Fix extraction (Closes #10592)

---
 youtube_dl/extractor/videomore.py | 72 +++++++++++--------------------
 1 file changed, 25 insertions(+), 47 deletions(-)

diff --git a/youtube_dl/extractor/videomore.py b/youtube_dl/extractor/videomore.py
index 04e95c66e..328b5b7fb 100644
--- a/youtube_dl/extractor/videomore.py
+++ b/youtube_dl/extractor/videomore.py
@@ -6,8 +6,7 @@ import re
 from .common import InfoExtractor
 from ..utils import (
     int_or_none,
-    parse_age_limit,
-    parse_iso8601,
+    xpath_element,
     xpath_text,
 )
 
@@ -17,38 +16,32 @@ class VideomoreIE(InfoExtractor):
     _VALID_URL = r'videomore:(?P<sid>\d+)$|https?://videomore\.ru/(?:(?:embed|[^/]+/[^/]+)/|[^/]+\?.*\btrack_id=)(?P<id>\d+)(?:[/?#&]|\.(?:xml|json)|$)'
     _TESTS = [{
         'url': 'http://videomore.ru/kino_v_detalayah/5_sezon/367617',
-        'md5': '70875fbf57a1cd004709920381587185',
+        'md5': '44455a346edc0d509ac5b5a5b531dc35',
         'info_dict': {
             'id': '367617',
             'ext': 'flv',
-            'title': 'В гостях Алексей Чумаков и Юлия Ковальчук',
-            'description': 'В гостях – лучшие романтические комедии года, «Выживший» Иньярриту и «Стив Джобс» Дэнни Бойла.',
+            'title': 'Кино в деталях 5 сезон В гостях Алексей Чумаков и Юлия Ковальчук',
             'series': 'Кино в деталях',
             'episode': 'В гостях Алексей Чумаков и Юлия Ковальчук',
-            'episode_number': None,
-            'season': 'Сезон 2015',
-            'season_number': 5,
             'thumbnail': 're:^https?://.*\.jpg',
             'duration': 2910,
-            'age_limit': 16,
             'view_count': int,
+            'comment_count': int,
+            'age_limit': 16,
         },
     }, {
         'url': 'http://videomore.ru/embed/259974',
         'info_dict': {
             'id': '259974',
             'ext': 'flv',
-            'title': '80 серия',
-            'description': '«Медведей» ждет решающий матч. Макеев выясняет отношения со Стрельцовым. Парни узнают подробности прошлого Макеева.',
+            'title': 'Молодежка 2 сезон 40 серия',
             'series': 'Молодежка',
-            'episode': '80 серия',
-            'episode_number': 40,
-            'season': '2 сезон',
-            'season_number': 2,
+            'episode': '40 серия',
             'thumbnail': 're:^https?://.*\.jpg',
             'duration': 2809,
-            'age_limit': 16,
             'view_count': int,
+            'comment_count': int,
+            'age_limit': 16,
         },
         'params': {
             'skip_download': True,
@@ -58,13 +51,8 @@ class VideomoreIE(InfoExtractor):
         'info_dict': {
             'id': '341073',
             'ext': 'flv',
-            'title': 'Команда проиграла из-за Бакина?',
-            'description': 'Молодежка 3 сезон скоро',
-            'series': 'Молодежка',
+            'title': 'Промо Команда проиграла из-за Бакина?',
             'episode': 'Команда проиграла из-за Бакина?',
-            'episode_number': None,
-            'season': 'Промо',
-            'season_number': 99,
             'thumbnail': 're:^https?://.*\.jpg',
             'duration': 29,
             'age_limit': 16,
@@ -109,43 +97,33 @@ class VideomoreIE(InfoExtractor):
             'http://videomore.ru/video/tracks/%s.xml' % video_id,
             video_id, 'Downloading video XML')
 
-        video_url = xpath_text(video, './/video_url', 'video url', fatal=True)
+        item = xpath_element(video, './/playlist/item', fatal=True)
+
+        title = xpath_text(
+            item, ('./title', './episode_name'), 'title', fatal=True)
+
+        video_url = xpath_text(item, './video_url', 'video url', fatal=True)
         formats = self._extract_f4m_formats(video_url, video_id, f4m_id='hds')
         self._sort_formats(formats)
 
-        data = self._download_json(
-            'http://videomore.ru/video/tracks/%s.json' % video_id,
-            video_id, 'Downloading video JSON')
+        thumbnail = xpath_text(item, './thumbnail_url')
+        duration = int_or_none(xpath_text(item, './duration'))
+        view_count = int_or_none(xpath_text(item, './views'))
+        comment_count = int_or_none(xpath_text(item, './count_comments'))
+        age_limit = int_or_none(xpath_text(item, './min_age'))
 
-        title = data.get('title') or data['project_title']
-        description = data.get('description') or data.get('description_raw')
-        timestamp = parse_iso8601(data.get('published_at'))
-        duration = int_or_none(data.get('duration'))
-        view_count = int_or_none(data.get('views'))
-        age_limit = parse_age_limit(data.get('min_age'))
-        thumbnails = [{
-            'url': thumbnail,
-        } for thumbnail in data.get('big_thumbnail_urls', [])]
-
-        series = data.get('project_title')
-        episode = data.get('title')
-        episode_number = int_or_none(data.get('episode_of_season') or None)
-        season = data.get('season_title')
-        season_number = int_or_none(data.get('season_pos') or None)
+        series = xpath_text(item, './project_name')
+        episode = xpath_text(item, './episode_name')
 
         return {
             'id': video_id,
             'title': title,
-            'description': description,
             'series': series,
             'episode': episode,
-            'episode_number': episode_number,
-            'season': season,
-            'season_number': season_number,
-            'thumbnails': thumbnails,
-            'timestamp': timestamp,
+            'thumbnail': thumbnail,
             'duration': duration,
             'view_count': view_count,
+            'comment_count': comment_count,
             'age_limit': age_limit,
             'formats': formats,
         }

From 2fdc7b0e0438e7f72380cfff11b619fe36234ebd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 8 Sep 2016 22:40:02 +0700
Subject: [PATCH 6027/6219] [viafree] PEP 8

---
 youtube_dl/extractor/tvplay.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tvplay.py b/youtube_dl/extractor/tvplay.py
index c2a6e4e39..c0fec2594 100644
--- a/youtube_dl/extractor/tvplay.py
+++ b/youtube_dl/extractor/tvplay.py
@@ -390,7 +390,7 @@ class ViafreeIE(InfoExtractor):
         if thumbnail:
             video_id = self._search_regex(
                 r'https?://[^/]+/imagecache/(?:[^/]+/)+seasons/\d+/(\d{6,})/',
-            thumbnail, 'video id', default=None)
+                thumbnail, 'video id', default=None)
 
         if not video_id:
             video_id = self._search_regex(

From 7a979da8cb91abd7386cc2986d2ec2f4a2debb4c Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 8 Sep 2016 16:42:47 +0100
Subject: [PATCH 6028/6219] [yahoo] Look for Brightcove Legacy Studio
 embeds(closes #9345)

---
 youtube_dl/extractor/yahoo.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/yahoo.py b/youtube_dl/extractor/yahoo.py
index d7a81ab8c..91f0a0dbb 100644
--- a/youtube_dl/extractor/yahoo.py
+++ b/youtube_dl/extractor/yahoo.py
@@ -19,7 +19,10 @@ from ..utils import (
     determine_ext,
 )
 
-from .brightcove import BrightcoveNewIE
+from .brightcove import (
+    BrightcoveLegacyIE,
+    BrightcoveNewIE,
+)
 from .nbc import NBCSportsVPlayerIE
 
 
@@ -223,6 +226,11 @@ class YahooIE(InfoExtractor):
         if nbc_sports_url:
             return self.url_result(nbc_sports_url, NBCSportsVPlayerIE.ie_key())
 
+        # Look for Brightcove Legacy Studio embeds
+        bc_url = BrightcoveLegacyIE._extract_brightcove_url(webpage)
+        if bc_url:
+            return self.url_result(bc_url, BrightcoveLegacyIE.ie_key())
+
         # Look for Brightcove New Studio embeds
         bc_url = BrightcoveNewIE._extract_url(webpage)
         if bc_url:

From 6cfcb8ac3634f1735093a791fa56b96bddabe14b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 8 Sep 2016 22:44:34 +0700
Subject: [PATCH 6029/6219] [tvnoe] Do not capture unused groups in _VALID_URL

---
 youtube_dl/extractor/tvnoe.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tvnoe.py b/youtube_dl/extractor/tvnoe.py
index 1cd3e6a58..6d5c74826 100644
--- a/youtube_dl/extractor/tvnoe.py
+++ b/youtube_dl/extractor/tvnoe.py
@@ -10,7 +10,7 @@ from ..utils import (
 
 
 class TVNoeIE(JWPlatformBaseIE):
-    _VALID_URL = r'https?://(www\.)?tvnoe\.cz/video/(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?tvnoe\.cz/video/(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://www.tvnoe.cz/video/10362',
         'md5': 'aee983f279aab96ec45ab6e2abb3c2ca',

From 37720844f6d40878bd2f29ea8311c7988ed3fc6a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 8 Sep 2016 22:52:39 +0700
Subject: [PATCH 6030/6219] [jwplatform] Extract height from label

---
 youtube_dl/extractor/jwplatform.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/jwplatform.py b/youtube_dl/extractor/jwplatform.py
index ce3126943..7aaa65476 100644
--- a/youtube_dl/extractor/jwplatform.py
+++ b/youtube_dl/extractor/jwplatform.py
@@ -63,10 +63,17 @@ class JWPlatformBaseIE(InfoExtractor):
                         'ext': ext,
                     })
                 else:
+                    height = int_or_none(source.get('height'))
+                    if height is None:
+                        # Often no height is provided but there is a label in
+                        # format like 1080p.
+                        height = int_or_none(self._search_regex(
+                            r'^(\d{3,})[pP]$', source.get('label') or '',
+                            'height', default=None))
                     a_format = {
                         'url': source_url,
                         'width': int_or_none(source.get('width')),
-                        'height': int_or_none(source.get('height')),
+                        'height': height,
                         'ext': ext,
                     }
                     if source_url.startswith('rtmp'):

From ad0e2b3359862eb6ac786d8fefd5e20035b051cd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 8 Sep 2016 23:15:58 +0700
Subject: [PATCH 6031/6219] [abcotvs] Add support for ABC Owned Television
 Stations

---
 youtube_dl/extractor/abcotvs.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/abcotvs.py b/youtube_dl/extractor/abcotvs.py
index 53a900e50..054bb0596 100644
--- a/youtube_dl/extractor/abcotvs.py
+++ b/youtube_dl/extractor/abcotvs.py
@@ -12,6 +12,7 @@ from ..utils import (
 
 class ABCOTVSIE(InfoExtractor):
     IE_NAME = 'abcotvs'
+    IE_DESC = 'ABC Owned Television Stations'
     _VALID_URL = r'https?://(?:abc(?:7(?:news|ny|chicago)?|11|13|30)|6abc)\.com(?:/[^/]+/(?P<display_id>[^/]+))?/(?P<id>\d+)'
     _TESTS = [
         {

From 2abad67e52c034ff4a94bad00c31f9ba4aa496f2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 8 Sep 2016 23:32:16 +0700
Subject: [PATCH 6032/6219] [ChangeLog] Actualize

---
 ChangeLog | 20 +++++++++++++++++---
 1 file changed, 17 insertions(+), 3 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index cec87d5cd..e74f64b9a 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,11 +1,25 @@
 version <unreleased>
 
 Extractors
++ [jwplatform] Extract height from format label
++ [yahoo] Extract Brightcove Legacy Studio embeds (#9345)
+* [videomore] Fix extraction (#10592)
 * [foxgay] Fix extraction (#10480)
-+ [miaopai] New extractor (#10556)
++ [rmcdecouverte] Add extractor for rmcdecouverte.bfmtv.com (#9709)
 * [gamestar] Fix metadata extraction (#10479)
-+ [bilibili] Support episodes (#10190)
-+ [tvnoe] New extractor (#10524)
+* [puls4] Fix extraction (#10583)
++ [cctv] Add extractor for CCTV and CNTV (#8153)
++ [lci] Add extractor for lci.fr (#10573)
++ [wat] Extract DASH formats
++ [viafree] Improve video id detection (#10569)
++ [trutv] Add extractor for trutv.com (#10519)
++ [nick] Add support for nickelodeon.nl (#10559)
++ [abcotvs:clips] Add support for clips.abcotvs.com
++ [abcotvs] Add support for ABC Owned Television Stations sites (#9551)
++ [miaopai] Add extractor for miaopai.com (#10556)
+* [gamestar] Fix metadata extraction (#10479)
++ [bilibili] Add support for episodes (#10190)
++ [tvnoe] Add extractor for tvnoe.cz (#10524)
 
 
 version 2016.09.04.1

From b71783719000af0806bbd6995e934de2520792fd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 8 Sep 2016 23:46:14 +0700
Subject: [PATCH 6033/6219] release 2016.09.08

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 9 ++++++++-
 youtube_dl/version.py     | 2 +-
 4 files changed, 13 insertions(+), 6 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index c03092442..a983bf432 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.09.04.1*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.09.04.1**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.09.08*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.09.08**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.09.04.1
+[debug] youtube-dl version 2016.09.08
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index e74f64b9a..d84f447ba 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2016.09.08
 
 Extractors
 + [jwplatform] Extract height from format label
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 9e21016f7..e6be746a8 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -19,9 +19,10 @@
  - **9now.com.au**
  - **abc.net.au**
  - **abc.net.au:iview**
- - **Abc7News**
  - **abcnews**
  - **abcnews:video**
+ - **abcotvs**: ABC Owned Television Stations
+ - **abcotvs:clips**
  - **AcademicEarth:Course**
  - **acast**
  - **acast:channel**
@@ -128,6 +129,7 @@
  - **CBSNews**: CBS News
  - **CBSNewsLiveVideo**: CBS News Live Videos
  - **CBSSports**
+ - **CCTV**
  - **CDA**
  - **CeskaTelevize**
  - **channel9**: Channel 9
@@ -352,6 +354,7 @@
  - **kuwo:song**: 酷我音乐
  - **la7.it**
  - **Laola1Tv**
+ - **LCI**
  - **Lcp**
  - **LcpPlay**
  - **Le**: 乐视网
@@ -390,6 +393,7 @@
  - **Metacritic**
  - **Mgoon**
  - **MGTV**: 芒果TV
+ - **MiaoPai**
  - **Minhateca**
  - **MinistryGrid**
  - **Minoto**
@@ -576,6 +580,7 @@
  - **revision3:embed**
  - **RICE**
  - **RingTV**
+ - **RMCDecouverte**
  - **RockstarGames**
  - **RoosterTeeth**
  - **RottenTomatoes**
@@ -721,6 +726,7 @@
  - **TrailerAddict** (Currently broken)
  - **Trilulilu**
  - **trollvids**
+ - **TruTV**
  - **Tube8**
  - **TubiTv**
  - **tudou**
@@ -742,6 +748,7 @@
  - **TVCArticle**
  - **tvigle**: Интернет-телевидение Tvigle.ru
  - **tvland.com**
+ - **TVNoe**
  - **tvp**: Telewizja Polska
  - **tvp:embed**: Telewizja Polska
  - **tvp:series**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index b2ea6dac6..941ffb3f6 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.09.04.1'
+__version__ = '2016.09.08'

From 4614ad7b591577793321b8e761510cf08ceb558b Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 8 Sep 2016 20:45:13 +0100
Subject: [PATCH 6034/6219] [parliamentliveuk] fix extraction(closes #9137)

---
 youtube_dl/extractor/parliamentliveuk.py | 57 +++++++++---------------
 1 file changed, 22 insertions(+), 35 deletions(-)

diff --git a/youtube_dl/extractor/parliamentliveuk.py b/youtube_dl/extractor/parliamentliveuk.py
index 0a423a08f..874aacc55 100644
--- a/youtube_dl/extractor/parliamentliveuk.py
+++ b/youtube_dl/extractor/parliamentliveuk.py
@@ -1,53 +1,40 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 
 
 class ParliamentLiveUKIE(InfoExtractor):
     IE_NAME = 'parliamentlive.tv'
     IE_DESC = 'UK parliament videos'
-    _VALID_URL = r'https?://www\.parliamentlive\.tv/Main/Player\.aspx\?(?:[^&]+&)*?meetingId=(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?parliamentlive\.tv/Event/Index/(?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})'
 
     _TEST = {
-        'url': 'http://www.parliamentlive.tv/Main/Player.aspx?meetingId=15121&player=windowsmedia',
+        'url': 'http://parliamentlive.tv/Event/Index/c1e9d44d-fd6c-4263-b50f-97ed26cc998b',
         'info_dict': {
-            'id': '15121',
-            'ext': 'asf',
-            'title': 'hoc home affairs committee, 18 mar 2014.pm',
-            'description': 'md5:033b3acdf83304cd43946b2d5e5798d1',
+            'id': 'c1e9d44d-fd6c-4263-b50f-97ed26cc998b',
+            'ext': 'mp4',
+            'title': 'Home Affairs Committee',
+            'uploader_id': 'FFMPEG-01',
+            'timestamp': 1422696664,
+            'upload_date': '20150131',
         },
-        'params': {
-            'skip_download': True,  # Requires mplayer (mms)
-        }
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        webpage = self._download_webpage(url, video_id)
-
-        asx_url = self._html_search_regex(
-            r'embed.*?src="([^"]+)" name="MediaPlayer"', webpage,
-            'metadata URL')
-        asx = self._download_xml(asx_url, video_id, 'Downloading ASX metadata')
-        video_url = asx.find('.//REF').attrib['HREF']
-
-        title = self._search_regex(
-            r'''(?x)player\.setClipDetails\(
-                (?:(?:[0-9]+|"[^"]+"),\s*){2}
-                "([^"]+",\s*"[^"]+)"
-                ''',
-            webpage, 'title').replace('", "', ', ')
-        description = self._html_search_regex(
-            r'(?s)<span id="MainContentPlaceHolder_CaptionsBlock_WitnessInfo">(.*?)</span>',
-            webpage, 'description')
-
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(
+            'http://vodplayer.parliamentlive.tv/?mid=' + video_id, video_id)
+        widget_config = self._parse_json(self._search_regex(
+            r'kWidgetConfig\s*=\s*({.+});',
+            webpage, 'kaltura widget config'), video_id)
+        kaltura_url = 'kaltura:%s:%s' % (widget_config['wid'][1:], widget_config['entry_id'])
+        event_title = self._download_json(
+            'http://parliamentlive.tv/Event/GetShareVideo/' + video_id, video_id)['event']['title']
         return {
+            '_type': 'url_transparent',
             'id': video_id,
-            'ext': 'asf',
-            'url': video_url,
-            'title': title,
-            'description': description,
+            'title': event_title,
+            'description': '',
+            'url': kaltura_url,
+            'ie_key': 'Kaltura',
         }

From 4d5726b0d7562aff975c91303b8ae8cfc4de8c51 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 8 Sep 2016 22:53:44 +0100
Subject: [PATCH 6035/6219] [telequebec] Add new extractor(closes #1999)

---
 youtube_dl/extractor/telequebec.py | 36 ++++++++++++++++++++++++++++++
 1 file changed, 36 insertions(+)
 create mode 100644 youtube_dl/extractor/telequebec.py

diff --git a/youtube_dl/extractor/telequebec.py b/youtube_dl/extractor/telequebec.py
new file mode 100644
index 000000000..4043fcb92
--- /dev/null
+++ b/youtube_dl/extractor/telequebec.py
@@ -0,0 +1,36 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import int_or_none
+
+
+class TeleQuebecIE(InfoExtractor):
+    _VALID_URL = r'https?://zonevideo\.telequebec\.tv/media/(?P<id>\d+)'
+    _TEST = {
+        'url': 'http://zonevideo.telequebec.tv/media/20984/le-couronnement-de-new-york/couronnement-de-new-york',
+        'md5': 'fe95a0957e5707b1b01f5013e725c90f',
+        'info_dict': {
+            'id': '20984',
+            'ext': 'mp4',
+            'title': 'Le couronnement de New York',
+            'description': 'md5:f5b3d27a689ec6c1486132b2d687d432',
+            'upload_date': '20160220',
+            'timestamp': 1455965438,
+        }
+    }
+
+    def _real_extract(self, url):
+        media_id = self._match_id(url)
+        media_data = self._download_json(
+            'https://mnmedias.api.telequebec.tv/api/v2/media/' + media_id,
+            media_id)['media']
+        return {
+            '_type': 'url_transparent',
+            'id': media_id,
+            'url': 'limelight:media:' + media_data['streamInfo']['sourceId'],
+            'title': media_data['title'],
+            'description': media_data.get('descriptions', [{'text': None}])[0].get('text'),
+            'duration': int_or_none(media_data.get('durationInMilliseconds'), 1000),
+            'ie_key': 'LimelightMedia',
+        }

From cb9cbd84ed2471140ce58480b8ed373f8abb5816 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 8 Sep 2016 22:55:27 +0100
Subject: [PATCH 6036/6219] [extractors] add import for TeleQuebecIE

---
 youtube_dl/extractor/extractors.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 8d9c2ae13..b7b630e9d 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -861,6 +861,7 @@ from .telebruxelles import TeleBruxellesIE
 from .telecinco import TelecincoIE
 from .telegraaf import TelegraafIE
 from .telemb import TeleMBIE
+from .telequebec import TeleQuebecIE
 from .teletask import TeleTaskIE
 from .telewebion import TelewebionIE
 from .testurl import TestURLIE

From bb5ebd4453c2f051af96b20e4a78f6563054684b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 9 Sep 2016 22:16:21 +0700
Subject: [PATCH 6037/6219] [canvas] Add support for een.be (Closes #10605)

---
 youtube_dl/extractor/canvas.py | 32 +++++++++++++++++++++++++++-----
 1 file changed, 27 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/canvas.py b/youtube_dl/extractor/canvas.py
index ec6d24d96..ef0691dcd 100644
--- a/youtube_dl/extractor/canvas.py
+++ b/youtube_dl/extractor/canvas.py
@@ -1,11 +1,13 @@
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..utils import float_or_none
 
 
 class CanvasIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?canvas\.be/video/(?:[^/]+/)*(?P<id>[^/?#&]+)'
+    _VALID_URL = r'https?://(?:www\.)?(?P<site_id>canvas|een)\.be/(?:[^/]+/)*(?P<id>[^/?#&]+)'
     _TESTS = [{
         'url': 'http://www.canvas.be/video/de-afspraak/najaar-2015/de-afspraak-veilt-voor-de-warmste-week',
         'md5': 'ea838375a547ac787d4064d8c7860a6c',
@@ -38,22 +40,42 @@ class CanvasIE(InfoExtractor):
         'params': {
             'skip_download': True,
         }
+    }, {
+        'url': 'https://www.een.be/sorry-voor-alles/herbekijk-sorry-voor-alles',
+        'info_dict': {
+            'id': 'mz-ast-11a587f8-b921-4266-82e2-0bce3e80d07f',
+            'display_id': 'herbekijk-sorry-voor-alles',
+            'ext': 'mp4',
+            'title': 'Herbekijk Sorry voor alles',
+            'description': 'md5:8bb2805df8164e5eb95d6a7a29dc0dd3',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 3788.06,
+        },
+        'params': {
+            'skip_download': True,
+        }
+    }, {
+        'url': 'https://www.canvas.be/check-point/najaar-2016/de-politie-uw-vriend',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
-        display_id = self._match_id(url)
+        mobj = re.match(self._VALID_URL, url)
+        site_id, display_id = mobj.group('site_id'), mobj.group('id')
 
         webpage = self._download_webpage(url, display_id)
 
-        title = self._search_regex(
+        title = (self._search_regex(
             r'<h1[^>]+class="video__body__header__title"[^>]*>(.+?)</h1>',
-            webpage, 'title', default=None) or self._og_search_title(webpage)
+            webpage, 'title', default=None) or self._og_search_title(
+            webpage)).strip()
 
         video_id = self._html_search_regex(
             r'data-video=(["\'])(?P<id>.+?)\1', webpage, 'video id', group='id')
 
         data = self._download_json(
-            'https://mediazone.vrt.be/api/v1/canvas/assets/%s' % video_id, display_id)
+            'https://mediazone.vrt.be/api/v1/%s/assets/%s'
+            % (site_id, video_id), display_id)
 
         formats = []
         for target in data['targetUrls']:

From c6129feb7f8313941a4d2044fa4b45ceaa0a91c7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 9 Sep 2016 23:20:45 +0700
Subject: [PATCH 6038/6219] [ketnet] Add extractor (Closes #10343)

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/ketnet.py     | 52 ++++++++++++++++++++++++++++++
 2 files changed, 53 insertions(+)
 create mode 100644 youtube_dl/extractor/ketnet.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index b7b630e9d..38dc33674 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -407,6 +407,7 @@ from .kankan import KankanIE
 from .karaoketv import KaraoketvIE
 from .karrierevideos import KarriereVideosIE
 from .keezmovies import KeezMoviesIE
+from .ketnet import KetnetIE
 from .khanacademy import KhanAcademyIE
 from .kickstarter import KickStarterIE
 from .keek import KeekIE
diff --git a/youtube_dl/extractor/ketnet.py b/youtube_dl/extractor/ketnet.py
new file mode 100644
index 000000000..aaf3f807a
--- /dev/null
+++ b/youtube_dl/extractor/ketnet.py
@@ -0,0 +1,52 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class KetnetIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?ketnet\.be/(?:[^/]+/)*(?P<id>[^/?#&]+)'
+    _TESTS = [{
+        'url': 'https://www.ketnet.be/kijken/zomerse-filmpjes',
+        'md5': 'd907f7b1814ef0fa285c0475d9994ed7',
+        'info_dict': {
+            'id': 'zomerse-filmpjes',
+            'ext': 'mp4',
+            'title': 'Gluur mee op de filmset en op Pennenzakkenrock',
+            'description': 'Gluur mee met Ghost Rockers op de filmset',
+            'thumbnail': 're:^https?://.*\.jpg$',
+        }
+    }, {
+        'url': 'https://www.ketnet.be/kijken/karrewiet/uitzending-8-september-2016',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.ketnet.be/achter-de-schermen/sien-repeteert-voor-stars-for-life',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        config = self._parse_json(
+            self._search_regex(
+                r'(?s)playerConfig\s*=\s*({.+?})\s*;', webpage,
+                'player config'),
+            video_id)
+
+        title = config['title']
+
+        formats = self._extract_m3u8_formats(
+            config['source']['hls'], video_id, 'mp4',
+            entry_protocol='m3u8_native', m3u8_id='hls')
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': config.get('description'),
+            'thumbnail': config.get('image'),
+            'series': config.get('program'),
+            'episode': config.get('episode'),
+            'formats': formats,
+        }

From 6c3affcb18f7eabf7d428e5efe474e0547ab25cb Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 10 Sep 2016 20:09:09 +0800
Subject: [PATCH 6039/6219] [newgrounds] Fix uploader extraction

Closes #10584

Also change test URLs to HTTPS, as proposed by
@stepshal in #10593.

Closes #10593
---
 ChangeLog                          |  6 ++++++
 youtube_dl/extractor/newgrounds.py | 23 ++++++++---------------
 2 files changed, 14 insertions(+), 15 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index d84f447ba..fafe445cb 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Extractors
+* [newgrounds] Fix uploader extraction (#10584)
+
+
 version 2016.09.08
 
 Extractors
diff --git a/youtube_dl/extractor/newgrounds.py b/youtube_dl/extractor/newgrounds.py
index 705940323..9bea610c8 100644
--- a/youtube_dl/extractor/newgrounds.py
+++ b/youtube_dl/extractor/newgrounds.py
@@ -1,15 +1,12 @@
 from __future__ import unicode_literals
 
-import json
-import re
-
 from .common import InfoExtractor
 
 
 class NewgroundsIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?newgrounds\.com/(?:audio/listen|portal/view)/(?P<id>[0-9]+)'
     _TESTS = [{
-        'url': 'http://www.newgrounds.com/audio/listen/549479',
+        'url': 'https://www.newgrounds.com/audio/listen/549479',
         'md5': 'fe6033d297591288fa1c1f780386f07a',
         'info_dict': {
             'id': '549479',
@@ -18,7 +15,7 @@ class NewgroundsIE(InfoExtractor):
             'uploader': 'Burn7',
         }
     }, {
-        'url': 'http://www.newgrounds.com/portal/view/673111',
+        'url': 'https://www.newgrounds.com/portal/view/673111',
         'md5': '3394735822aab2478c31b1004fe5e5bc',
         'info_dict': {
             'id': '673111',
@@ -29,24 +26,20 @@ class NewgroundsIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        music_id = mobj.group('id')
-        webpage = self._download_webpage(url, music_id)
+        media_id = self._match_id(url)
+        webpage = self._download_webpage(url, media_id)
 
         title = self._html_search_regex(
             r'<title>([^>]+)</title>', webpage, 'title')
 
         uploader = self._html_search_regex(
-            [r',"artist":"([^"]+)",', r'[\'"]owner[\'"]\s*:\s*[\'"]([^\'"]+)[\'"],'],
-            webpage, 'uploader')
+            r'Author\s*<a[^>]+>([^<]+)', webpage, 'uploader', fatal=False)
 
-        music_url_json_string = self._html_search_regex(
-            r'({"url":"[^"]+"),', webpage, 'music url') + '}'
-        music_url_json = json.loads(music_url_json_string)
-        music_url = music_url_json['url']
+        music_url = self._parse_json(self._search_regex(
+            r'"url":("[^"]+"),', webpage, ''), media_id)
 
         return {
-            'id': music_id,
+            'id': media_id,
             'title': title,
             'url': music_url,
             'uploader': uploader,

From b29f842e0eb095248ff39d1fa28c5b4941793246 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 10 Sep 2016 20:46:45 +0700
Subject: [PATCH 6040/6219] [canalplus] Add support for c8.fr (Closes #10577)

---
 youtube_dl/extractor/canalplus.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/canalplus.py b/youtube_dl/extractor/canalplus.py
index 61463f249..69e8f4f57 100644
--- a/youtube_dl/extractor/canalplus.py
+++ b/youtube_dl/extractor/canalplus.py
@@ -23,6 +23,7 @@ class CanalplusIE(InfoExtractor):
                                     (?:(?:www|m)\.)?canalplus\.fr|
                                     (?:www\.)?piwiplus\.fr|
                                     (?:www\.)?d8\.tv|
+                                    (?:www\.)?c8\.fr|
                                     (?:www\.)?d17\.tv|
                                     (?:www\.)?itele\.fr
                                 )/(?:(?:[^/]+/)*(?P<display_id>[^/?#&]+))?(?:\?.*\bvid=(?P<vid>\d+))?|
@@ -35,6 +36,7 @@ class CanalplusIE(InfoExtractor):
         'canalplus': 'cplus',
         'piwiplus': 'teletoon',
         'd8': 'd8',
+        'c8': 'd8',
         'd17': 'd17',
         'itele': 'itele',
     }

From 84a18e9b908eb0b770f03603200026a06f4f08b4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 10 Sep 2016 22:01:49 +0700
Subject: [PATCH 6041/6219] [polskieradio:category] Improve extraction

---
 youtube_dl/extractor/extractors.py   |   5 +-
 youtube_dl/extractor/polskieradio.py | 158 ++++++++++++++-------------
 2 files changed, 84 insertions(+), 79 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 6a142996f..96f3d3fcb 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -671,7 +671,10 @@ from .pluralsight import (
 )
 from .podomatic import PodomaticIE
 from .pokemon import PokemonIE
-from .polskieradio import PolskieRadioIE, PolskieRadioProgrammeIE
+from .polskieradio import (
+    PolskieRadioIE,
+    PolskieRadioCategoryIE,
+)
 from .porn91 import Porn91IE
 from .porncom import PornComIE
 from .pornhd import PornHdIE
diff --git a/youtube_dl/extractor/polskieradio.py b/youtube_dl/extractor/polskieradio.py
index c51d3d9be..5ff173774 100644
--- a/youtube_dl/extractor/polskieradio.py
+++ b/youtube_dl/extractor/polskieradio.py
@@ -1,6 +1,7 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import itertools
 import re
 
 from .common import InfoExtractor
@@ -10,90 +11,13 @@ from ..compat import (
     compat_urlparse
 )
 from ..utils import (
+    extract_attributes,
     int_or_none,
     strip_or_none,
     unified_timestamp,
 )
 
 
-class PolskieRadioProgrammeIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?polskieradio\.pl/\d+(,[^/]+)?/(?P<id>\d+)'
-    _TESTS = [{
-        'url': 'http://www.polskieradio.pl/7/5102,HISTORIA-ZYWA',
-        'info_dict': {
-            'id': '5102',
-            'title': 'HISTORIA ŻYWA',
-        },
-        'playlist_mincount': 34,
-    }, {
-        'url': 'http://www.polskieradio.pl/7/4807',
-        'info_dict': {
-            'id': '4807',
-            'title': 'Vademecum 1050. rocznicy Chrztu Polski'
-        },
-        'playlist_mincount': 5
-    }, {
-        'url': 'http://www.polskieradio.pl/7/129,Sygnaly-dnia?ref=source',
-        'only_matching': True
-    }, {
-        'url': 'http://www.polskieradio.pl/37,RedakcjaKatolicka/4143,Kierunek-Krakow',
-        'info_dict': {
-            'id': '4143',
-            'title': 'Kierunek Kraków',
-        },
-        'playlist_mincount': 61
-    }, {
-        'url': 'http://www.polskieradio.pl/7,Jedynka/5102,HISTORIA-ZYWA',
-        'only_matching': True
-    }]
-
-    def _get_entries_from_page_content(self, base_url, content):
-        entries = []
-
-        articles = re.findall(
-            r'<article class="ID-(\d+) article">\s+<a href="([^"]+)"( data-layer="[^"]*")? class="[^"]*" title="([^"]+)">',
-            content)
-        for article_id, article_url, _, article_title in articles:
-            resolved_article_url = compat_urlparse.urljoin(base_url, article_url)
-            entries.append(self.url_result(
-                resolved_article_url,
-                ie='PolskieRadio',
-                video_id=article_id,
-                video_title=article_title))
-
-        return entries
-
-    @classmethod
-    def suitable(cls, url):
-        return False if PolskieRadioIE.suitable(url) else super(PolskieRadioProgrammeIE, cls).suitable(url)
-
-    def _real_extract(self, url):
-        programme_id = self._match_id(url)
-        webpage = self._download_webpage(url, programme_id)
-
-        title = self._html_search_regex(
-            r'<a href="[^"]+" id=".*_linkCategory" title="[^"]+">(.+?)</a>',
-            webpage, 'title', fatal=False)
-        description = None
-
-        entries = self._get_entries_from_page_content(url, webpage)
-
-        pages = re.findall(r'<a( href="([^"]+/Strona/)\d+")? id="[^"]+" title="strona&#32;(\d+)"', webpage)
-        page_count = max(int(page_number) for _, _, page_number in pages) if pages else 1
-
-        if page_count > 1:
-            page_url_root = next(url for _, url, _ in pages if len(url) > 0)
-            for page_number in range(2, page_count + 1):
-                page_url = page_url_root + str(page_number)
-                resolved_page_url = compat_urlparse.urljoin(url, page_url)
-                page_content = self._download_webpage(
-                    resolved_page_url, programme_id,
-                    note="Downloading page number %d" % page_number)
-                entries.extend(self._get_entries_from_page_content(url, page_content))
-
-        return self.playlist_result(entries, programme_id, title, description)
-
-
 class PolskieRadioIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?polskieradio\.pl/\d+/\d+/Artykul/(?P<id>[0-9]+)'
     _TESTS = [{
@@ -176,3 +100,81 @@ class PolskieRadioIE(InfoExtractor):
         description = strip_or_none(self._og_search_description(webpage))
 
         return self.playlist_result(entries, playlist_id, title, description)
+
+
+class PolskieRadioCategoryIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?polskieradio\.pl/\d+(?:,[^/]+)?/(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'http://www.polskieradio.pl/7/5102,HISTORIA-ZYWA',
+        'info_dict': {
+            'id': '5102',
+            'title': 'HISTORIA ŻYWA',
+        },
+        'playlist_mincount': 38,
+    }, {
+        'url': 'http://www.polskieradio.pl/7/4807',
+        'info_dict': {
+            'id': '4807',
+            'title': 'Vademecum 1050. rocznicy Chrztu Polski'
+        },
+        'playlist_mincount': 5
+    }, {
+        'url': 'http://www.polskieradio.pl/7/129,Sygnaly-dnia?ref=source',
+        'only_matching': True
+    }, {
+        'url': 'http://www.polskieradio.pl/37,RedakcjaKatolicka/4143,Kierunek-Krakow',
+        'info_dict': {
+            'id': '4143',
+            'title': 'Kierunek Kraków',
+        },
+        'playlist_mincount': 61
+    }, {
+        'url': 'http://www.polskieradio.pl/10,czworka/214,muzyka',
+        'info_dict': {
+            'id': '214',
+            'title': 'Muzyka',
+        },
+        'playlist_mincount': 61
+    }, {
+        'url': 'http://www.polskieradio.pl/7,Jedynka/5102,HISTORIA-ZYWA',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.polskieradio.pl/8,Dwojka/196,Publicystyka',
+        'only_matching': True,
+    }]
+
+    @classmethod
+    def suitable(cls, url):
+        return False if PolskieRadioIE.suitable(url) else super(PolskieRadioCategoryIE, cls).suitable(url)
+
+    def _entries(self, url, page, category_id):
+        content = page
+        for page_num in itertools.count(2):
+            for a_entry, entry_id in re.findall(
+                    r'(?s)<article[^>]+>.*?(<a[^>]+href=["\']/\d+/\d+/Artykul/(\d+)[^>]+>).*?</article>',
+                    content):
+                entry = extract_attributes(a_entry)
+                href = entry.get('href')
+                if not href:
+                    continue
+                yield self.url_result(
+                    compat_urlparse.urljoin(url, href), PolskieRadioIE.ie_key(),
+                    entry_id, entry.get('title'))
+            mobj = re.search(
+                r'<div[^>]+class=["\']next["\'][^>]*>\s*<a[^>]+href=(["\'])(?P<url>(?:(?!\1).)+)\1',
+                content)
+            if not mobj:
+                break
+            next_url = compat_urlparse.urljoin(url, mobj.group('url'))
+            content = self._download_webpage(
+                next_url, category_id, 'Downloading page %s' % page_num)
+
+    def _real_extract(self, url):
+        category_id = self._match_id(url)
+        webpage = self._download_webpage(url, category_id)
+        title = self._html_search_regex(
+            r'<title>([^<]+) - [^<]+ - [^<]+</title>',
+            webpage, 'title', fatal=False)
+        return self.playlist_result(
+            self._entries(url, webpage, category_id),
+            category_id, title)

From 732424375017a033f5b398b0f3dc2c6d47f3d3fd Mon Sep 17 00:00:00 2001
From: Scott Leggett <scott@sl.id.au>
Date: Mon, 5 Sep 2016 22:41:08 +1000
Subject: [PATCH 6042/6219] [9now] Fix extraction

---
 youtube_dl/extractor/ninenow.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ninenow.py b/youtube_dl/extractor/ninenow.py
index faa577237..907b42609 100644
--- a/youtube_dl/extractor/ninenow.py
+++ b/youtube_dl/extractor/ninenow.py
@@ -44,7 +44,14 @@ class NineNowIE(InfoExtractor):
         page_data = self._parse_json(self._search_regex(
             r'window\.__data\s*=\s*({.*?});', webpage,
             'page data'), display_id)
-        common_data = page_data.get('episode', {}).get('episode') or page_data.get('clip', {}).get('clip')
+        current_key = (
+            page_data.get('episode', {}).get('currentEpisodeKey') or
+            page_data.get('clip', {}).get('currentClipKey')
+        )
+        common_data = (
+            page_data.get('episode', {}).get('episodeCache', {}).get(current_key, {}).get('episode') or
+            page_data.get('clip', {}).get('clipCache', {}).get(current_key, {}).get('clip')
+        )
         video_data = common_data['video']
 
         if video_data.get('drm'):

From 56c0ead4d3b9f365f0562678504879be8e79b89c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 11 Sep 2016 00:42:13 +0700
Subject: [PATCH 6043/6219] [9now] Improve video data extraction (Closes
 #10561)

---
 youtube_dl/extractor/ninenow.py | 22 ++++++++++++++--------
 1 file changed, 14 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/ninenow.py b/youtube_dl/extractor/ninenow.py
index 907b42609..351bea7ba 100644
--- a/youtube_dl/extractor/ninenow.py
+++ b/youtube_dl/extractor/ninenow.py
@@ -44,14 +44,20 @@ class NineNowIE(InfoExtractor):
         page_data = self._parse_json(self._search_regex(
             r'window\.__data\s*=\s*({.*?});', webpage,
             'page data'), display_id)
-        current_key = (
-            page_data.get('episode', {}).get('currentEpisodeKey') or
-            page_data.get('clip', {}).get('currentClipKey')
-        )
-        common_data = (
-            page_data.get('episode', {}).get('episodeCache', {}).get(current_key, {}).get('episode') or
-            page_data.get('clip', {}).get('clipCache', {}).get(current_key, {}).get('clip')
-        )
+
+        for kind in ('episode', 'clip'):
+            current_key = page_data.get(kind, {}).get(
+                'current%sKey' % kind.capitalize())
+            if not current_key:
+                continue
+            cache = page_data.get(kind, {}).get('%sCache' % kind, {})
+            if not cache:
+                continue
+            common_data = (cache.get(current_key) or list(cache.values())[0])[kind]
+            break
+        else:
+            raise ExtractorError('Unable to find video data')
+
         video_data = common_data['video']
 
         if video_data.get('drm'):

From 2512b17493fced6b469d9610c1ad5c5af52870f3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 11 Sep 2016 01:27:20 +0700
Subject: [PATCH 6044/6219] [lrt] Fix audio extraction (Closes #10566)

---
 youtube_dl/extractor/lrt.py | 46 ++++++++++++++++++++++++++++++-------
 1 file changed, 38 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/lrt.py b/youtube_dl/extractor/lrt.py
index 1072405b3..f5c997ef4 100644
--- a/youtube_dl/extractor/lrt.py
+++ b/youtube_dl/extractor/lrt.py
@@ -1,8 +1,11 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..utils import (
+    determine_ext,
     int_or_none,
     parse_duration,
     remove_end,
@@ -12,8 +15,10 @@ from ..utils import (
 class LRTIE(InfoExtractor):
     IE_NAME = 'lrt.lt'
     _VALID_URL = r'https?://(?:www\.)?lrt\.lt/mediateka/irasas/(?P<id>[0-9]+)'
-    _TEST = {
+    _TESTS = [{
+        # m3u8 download
         'url': 'http://www.lrt.lt/mediateka/irasas/54391/',
+        'md5': 'fe44cf7e4ab3198055f2c598fc175cb0',
         'info_dict': {
             'id': '54391',
             'ext': 'mp4',
@@ -23,20 +28,45 @@ class LRTIE(InfoExtractor):
             'view_count': int,
             'like_count': int,
         },
-        'params': {
-            'skip_download': True,  # m3u8 download
+    }, {
+        # direct mp3 download
+        'url': 'http://www.lrt.lt/mediateka/irasas/1013074524/',
+        'md5': '389da8ca3cad0f51d12bed0c844f6a0a',
+        'info_dict': {
+            'id': '1013074524',
+            'ext': 'mp3',
+            'title': 'Kita tema 2016-09-05 15:05',
+            'description': 'md5:1b295a8fc7219ed0d543fc228c931fb5',
+            'duration': 3008,
+            'view_count': int,
+            'like_count': int,
         },
-    }
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
         title = remove_end(self._og_search_title(webpage), ' - LRT')
-        m3u8_url = self._search_regex(
-            r'file\s*:\s*(["\'])(?P<url>.+?)\1\s*\+\s*location\.hash\.substring\(1\)',
-            webpage, 'm3u8 url', group='url')
-        formats = self._extract_m3u8_formats(m3u8_url, video_id, 'mp4')
+
+        formats = []
+        for _, file_url in re.findall(
+                r'file\s*:\s*(["\'])(?P<url>(?:(?!\1).)+)\1', webpage):
+            ext = determine_ext(file_url)
+            if ext not in ('m3u8', 'mp3'):
+                continue
+            # mp3 served as m3u8 produces stuttered media file
+            if ext == 'm3u8' and '.mp3' in file_url:
+                continue
+            if ext == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    file_url, video_id, 'mp4', entry_protocol='m3u8_native',
+                    fatal=False))
+            elif ext == 'mp3':
+                formats.append({
+                    'url': file_url,
+                    'vcodec': 'none',
+                })
         self._sort_formats(formats)
 
         thumbnail = self._og_search_thumbnail(webpage)

From 1e35999c1e4637174e2532c457431315b5e186d9 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 10 Sep 2016 19:43:09 +0100
Subject: [PATCH 6045/6219] [tfo] Add new extractor

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/tfo.py        | 53 ++++++++++++++++++++++++++++++
 2 files changed, 54 insertions(+)
 create mode 100644 youtube_dl/extractor/tfo.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 96f3d3fcb..124e909fb 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -870,6 +870,7 @@ from .teletask import TeleTaskIE
 from .telewebion import TelewebionIE
 from .testurl import TestURLIE
 from .tf1 import TF1IE
+from .tfo import TFOIE
 from .theintercept import TheInterceptIE
 from .theplatform import (
     ThePlatformIE,
diff --git a/youtube_dl/extractor/tfo.py b/youtube_dl/extractor/tfo.py
new file mode 100644
index 000000000..6f1eeac57
--- /dev/null
+++ b/youtube_dl/extractor/tfo.py
@@ -0,0 +1,53 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import json
+
+from .common import InfoExtractor
+from ..utils import (
+    HEADRequest,
+    ExtractorError,
+    int_or_none,
+)
+
+
+class TFOIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?tfo\.org/(?:en|fr)/(?:[^/]+/){2}(?P<id>\d+)'
+    _TEST = {
+        'url': 'http://www.tfo.org/en/universe/tfo-247/100463871/video-game-hackathon',
+        'md5': '47c987d0515561114cf03d1226a9d4c7',
+        'info_dict': {
+            'id': '100463871',
+            'ext': 'mp4',
+            'title': 'Video Game Hackathon',
+            'description': 'md5:558afeba217c6c8d96c60e5421795c07',
+            'upload_date': '20160212',
+            'timestamp': 1455310233,
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        self._request_webpage(HEADRequest('http://www.tfo.org/'), video_id)
+        infos = self._download_json(
+            'http://www.tfo.org/api/web/video/get_infos', video_id, data=json.dumps({
+                'product_id': video_id,
+            }).encode(), headers={
+                'X-tfo-session': self._get_cookies('http://www.tfo.org/')['tfo-session'].value,
+            })
+        if infos.get('success') == 0:
+            raise ExtractorError('%s said: %s' % (self.IE_NAME, infos['msg']), expected=True)
+        video_data = infos['data']
+
+        return {
+            '_type': 'url_transparent',
+            'id': video_id,
+            'url': 'limelight:media:' + video_data['llid'],
+            'title': video_data['title'],
+            'description': video_data.get('description'),
+            'series': video_data.get('collection'),
+            'season_number': int_or_none(video_data.get('season')),
+            'episode_number': int_or_none(video_data.get('episode')),
+            'duration': int_or_none(video_data.get('duration')),
+            'ie_key': 'LimelightMedia',
+        }

From 001a5fd3d75b311102264cf3920c6aa5b2322e51 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 11 Sep 2016 03:02:00 +0800
Subject: [PATCH 6046/6219] [iwara] Fix extraction after relaunch

Closes #10462, closes #3215
---
 ChangeLog                          |  1 +
 youtube_dl/extractor/extractors.py |  2 +-
 youtube_dl/extractor/iwara.py      | 77 ++++++++++++++++++++++++++++++
 youtube_dl/extractor/trollvids.py  | 36 --------------
 4 files changed, 79 insertions(+), 37 deletions(-)
 create mode 100644 youtube_dl/extractor/iwara.py
 delete mode 100644 youtube_dl/extractor/trollvids.py

diff --git a/ChangeLog b/ChangeLog
index fafe445cb..387dc7bf6 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
 version <unreleased>
 
 Extractors
+* [iwara] Fix extraction after relaunch (#10462, #3215)
 * [newgrounds] Fix uploader extraction (#10584)
 
 
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 124e909fb..2e795260e 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -395,6 +395,7 @@ from .ivi import (
     IviCompilationIE
 )
 from .ivideon import IvideonIE
+from .iwara import IwaraIE
 from .izlesene import IzleseneIE
 from .jeuxvideo import JeuxVideoIE
 from .jove import JoveIE
@@ -899,7 +900,6 @@ from .toutv import TouTvIE
 from .toypics import ToypicsUserIE, ToypicsIE
 from .traileraddict import TrailerAddictIE
 from .trilulilu import TriluliluIE
-from .trollvids import TrollvidsIE
 from .trutv import TruTVIE
 from .tube8 import Tube8IE
 from .tubitv import TubiTvIE
diff --git a/youtube_dl/extractor/iwara.py b/youtube_dl/extractor/iwara.py
new file mode 100644
index 000000000..8d7e7f472
--- /dev/null
+++ b/youtube_dl/extractor/iwara.py
@@ -0,0 +1,77 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..compat import compat_urllib_parse_urlparse
+from ..utils import remove_end
+
+
+class IwaraIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.|ecchi\.)?iwara\.tv/videos/(?P<id>[a-zA-Z0-9]+)'
+    _TESTS = [{
+        'url': 'http://iwara.tv/videos/amVwUl1EHpAD9RD',
+        'md5': '1d53866b2c514b23ed69e4352fdc9839',
+        'info_dict': {
+            'id': 'amVwUl1EHpAD9RD',
+            'ext': 'mp4',
+            'title': '【MMD R-18】ガールフレンド carry_me_off',
+            'age_limit': 18,
+        },
+    }, {
+        'url': 'http://ecchi.iwara.tv/videos/Vb4yf2yZspkzkBO',
+        'md5': '7e5f1f359cd51a027ba4a7b7710a50f0',
+        'info_dict': {
+            'id': '0B1LvuHnL-sRFNXB1WHNqbGw4SXc',
+            'ext': 'mp4',
+            'title': '[3D Hentai] Kyonyu Ã\x97 Genkai Ã\x97 Emaki Shinobi Girls.mp4',
+            'age_limit': 18,
+        },
+        'add_ie': ['GoogleDrive'],
+    }, {
+        'url': 'http://www.iwara.tv/videos/nawkaumd6ilezzgq',
+        'md5': '1d85f1e5217d2791626cff5ec83bb189',
+        'info_dict': {
+            'id': '6liAP9s2Ojc',
+            'ext': 'mp4',
+            'age_limit': 0,
+            'title': '[MMD] Do It Again Ver.2 [1080p 60FPS] (Motion,Camera,Wav+DL)',
+            'description': 'md5:590c12c0df1443d833fbebe05da8c47a',
+            'upload_date': '20160910',
+            'uploader': 'aMMDsork',
+            'uploader_id': 'UCVOFyOSCyFkXTYYHITtqB7A',
+        },
+        'add_ie': ['Youtube'],
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage, urlh = self._download_webpage_handle(url, video_id)
+
+        hostname = compat_urllib_parse_urlparse(urlh.geturl()).hostname
+        # ecchi is 'sexy' in Japanese
+        age_limit = 18 if hostname.split('.')[0] == 'ecchi' else 0
+
+        entries = self._parse_html5_media_entries(url, webpage, video_id)
+
+        if not entries:
+            iframe_url = self._html_search_regex(
+                r'<iframe[^>]+src=([\'"])(?P<url>[^\'"]+)\1',
+                webpage, 'iframe URL', group='url')
+            return {
+                '_type': 'url_transparent',
+                'url': iframe_url,
+                'age_limit': age_limit,
+            }
+
+        title = remove_end(self._html_search_regex(
+            r'<title>([^<]+)</title>', webpage, 'title'), ' | Iwara')
+
+        info_dict = entries[0]
+        info_dict.update({
+            'id': video_id,
+            'title': title,
+            'age_limit': age_limit,
+        })
+
+        return info_dict
diff --git a/youtube_dl/extractor/trollvids.py b/youtube_dl/extractor/trollvids.py
deleted file mode 100644
index 657705623..000000000
--- a/youtube_dl/extractor/trollvids.py
+++ /dev/null
@@ -1,36 +0,0 @@
-# encoding: utf-8
-from __future__ import unicode_literals
-
-import re
-
-from .nuevo import NuevoBaseIE
-
-
-class TrollvidsIE(NuevoBaseIE):
-    _VALID_URL = r'https?://(?:www\.)?trollvids\.com/video/(?P<id>\d+)/(?P<display_id>[^/?#&]+)'
-    IE_NAME = 'trollvids'
-    _TEST = {
-        'url': 'http://trollvids.com/video/2349002/%E3%80%90MMD-R-18%E3%80%91%E3%82%AC%E3%83%BC%E3%83%AB%E3%83%95%E3%83%AC%E3%83%B3%E3%83%89-carrymeoff',
-        'md5': '1d53866b2c514b23ed69e4352fdc9839',
-        'info_dict': {
-            'id': '2349002',
-            'ext': 'mp4',
-            'title': '【MMD R-18】ガールフレンド carry_me_off',
-            'age_limit': 18,
-            'duration': 216.78,
-        },
-    }
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        display_id = mobj.group('display_id')
-
-        info = self._extract_nuevo(
-            'http://trollvids.com/nuevo/player/config.php?v=%s' % video_id,
-            video_id)
-        info.update({
-            'display_id': display_id,
-            'age_limit': 18
-        })
-        return info

From bfcda07a2710738c32f63fdb4e09e177acc53df3 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 11 Sep 2016 04:06:00 +0800
Subject: [PATCH 6047/6219] [abc:iview] Skip the test. They are removed soon

---
 youtube_dl/extractor/abc.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/abc.py b/youtube_dl/extractor/abc.py
index c7b6df7d0..3792bd232 100644
--- a/youtube_dl/extractor/abc.py
+++ b/youtube_dl/extractor/abc.py
@@ -100,6 +100,7 @@ class ABCIViewIE(InfoExtractor):
     IE_NAME = 'abc.net.au:iview'
     _VALID_URL = r'https?://iview\.abc\.net\.au/programs/[^/]+/(?P<id>[^/?#]+)'
 
+    # ABC iview programs are normally available for 14 days only.
     _TESTS = [{
         'url': 'http://iview.abc.net.au/programs/gardening-australia/FA1505V024S00',
         'md5': '979d10b2939101f0d27a06b79edad536',
@@ -112,6 +113,7 @@ class ABCIViewIE(InfoExtractor):
             'uploader_id': 'abc1',
             'timestamp': 1471719600,
         },
+        'skip': 'Video gone',
     }]
 
     def _real_extract(self, url):

From 2cb93afcd8a8a1f086a97ef3791fa033ddc1610a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 11 Sep 2016 14:59:14 +0700
Subject: [PATCH 6048/6219] [viafree] Improve video id extraction (Closes
 #10615)

---
 youtube_dl/extractor/tvplay.py | 36 +++++++++++++++++++++++++++++-----
 1 file changed, 31 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/tvplay.py b/youtube_dl/extractor/tvplay.py
index c0fec2594..5548ff2ac 100644
--- a/youtube_dl/extractor/tvplay.py
+++ b/youtube_dl/extractor/tvplay.py
@@ -16,6 +16,7 @@ from ..utils import (
     parse_iso8601,
     qualities,
     try_get,
+    js_to_json,
     update_url_query,
 )
 
@@ -367,6 +368,10 @@ class ViafreeIE(InfoExtractor):
             'skip_download': True,
         },
         'add_ie': [TVPlayIE.ie_key()],
+    }, {
+        # Different og:image URL schema
+        'url': 'www.viafree.se/program/reality/sommaren-med-youtube-stjarnorna/sasong-1/avsnitt-2',
+        'only_matching': True,
     }, {
         'url': 'http://www.viafree.no/programmer/underholdning/det-beste-vorspielet/sesong-2/episode-1',
         'only_matching': True,
@@ -384,14 +389,35 @@ class ViafreeIE(InfoExtractor):
 
         webpage = self._download_webpage(url, video_id)
 
+        data = self._parse_json(
+            self._search_regex(
+                r'(?s)window\.App\s*=\s*({.+?})\s*;\s*</script',
+                webpage, 'data', default='{}'),
+            video_id, transform_source=lambda x: re.sub(
+                r'(?s)function\s+[a-zA-Z_][\da-zA-Z_]*\s*\([^)]*\)\s*{[^}]*}\s*',
+                'null', x), fatal=False)
+
         video_id = None
 
-        thumbnail = self._og_search_thumbnail(webpage, default=None)
-        if thumbnail:
-            video_id = self._search_regex(
-                r'https?://[^/]+/imagecache/(?:[^/]+/)+seasons/\d+/(\d{6,})/',
-                thumbnail, 'video id', default=None)
+        if data:
+            video_id = try_get(
+                data, lambda x: x['context']['dispatcher']['stores'][
+                    'ContentPageProgramStore']['currentVideo']['id'],
+                compat_str)
 
+        # Fallback #1 (extract from og:image URL schema)
+        if not video_id:
+            thumbnail = self._og_search_thumbnail(webpage, default=None)
+            if thumbnail:
+                video_id = self._search_regex(
+                    # Patterns seen:
+                    #  http://cdn.playapi.mtgx.tv/imagecache/600x315/cloud/content-images/inbox/765166/a2e95e5f1d735bab9f309fa345cc3f25.jpg
+                    #  http://cdn.playapi.mtgx.tv/imagecache/600x315/cloud/content-images/seasons/15204/758770/4a5ba509ca8bc043e1ebd1a76131cdf2.jpg
+                    r'https?://[^/]+/imagecache/(?:[^/]+/)+(\d{6,})/',
+                    thumbnail, 'video id', default=None)
+
+        # Fallback #2. Extract from raw JSON string.
+        # May extract wrong video id if relatedClips is present.
         if not video_id:
             video_id = self._search_regex(
                 r'currentVideo["\']\s*:\s*.+?["\']id["\']\s*:\s*["\'](\d{6,})',

From 2cdbc06a1f9dfda4079b2529aa51089650715466 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 11 Sep 2016 18:32:45 +0800
Subject: [PATCH 6049/6219] [foxnews] Support Fox News Articles (closes #10598)

---
 ChangeLog                          |  1 +
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/foxnews.py    | 40 +++++++++++++++++++++++++++---
 3 files changed, 39 insertions(+), 3 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 387dc7bf6..a73a35e88 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
 version <unreleased>
 
 Extractors
++ [foxnews] Support Fox News articles (#10598)
 * [iwara] Fix extraction after relaunch (#10462, #3215)
 * [newgrounds] Fix uploader extraction (#10584)
 
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 2e795260e..e9027fb69 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -292,6 +292,7 @@ from .fourtube import FourTubeIE
 from .fox import FOXIE
 from .foxgay import FoxgayIE
 from .foxnews import (
+    FoxNewsVideoIE,
     FoxNewsIE,
     FoxNewsInsiderIE,
 )
diff --git a/youtube_dl/extractor/foxnews.py b/youtube_dl/extractor/foxnews.py
index 5c7acd795..3e9a6a08c 100644
--- a/youtube_dl/extractor/foxnews.py
+++ b/youtube_dl/extractor/foxnews.py
@@ -6,7 +6,8 @@ from .amp import AMPIE
 from .common import InfoExtractor
 
 
-class FoxNewsIE(AMPIE):
+class FoxNewsVideoIE(AMPIE):
+    IE_NAME = 'foxnews:video'
     IE_DESC = 'Fox News and Fox Business Video'
     _VALID_URL = r'https?://(?P<host>video\.(?:insider\.)?fox(?:news|business)\.com)/v/(?:video-embed\.html\?video_id=)?(?P<id>\d+)'
     _TESTS = [
@@ -66,6 +67,35 @@ class FoxNewsIE(AMPIE):
         return info
 
 
+class FoxNewsIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?foxnews\.com/(?!v)([^/]+/)+(?P<id>[a-z-]+)'
+    IE_NAME = 'foxnews'
+
+    _TEST = {
+        'url': 'http://www.foxnews.com/politics/2016/09/08/buzz-about-bud-clinton-camp-denies-claims-wore-earpiece-at-forum.html',
+        'md5': '62aa5a781b308fdee212ebb6f33ae7ef',
+        'info_dict': {
+            'id': '5116295019001',
+            'ext': 'mp4',
+            'title': 'Trump and Clinton asked to defend positions on Iraq War',
+            'description': 'Veterans react on \'The Kelly File\'',
+            'timestamp': 1473299755,
+            'upload_date': '20160908',
+        },
+    }
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+
+        video_id = self._html_search_regex(
+            r'data-video-id=([\'"])(?P<id>[^\'"]+)\1',
+            webpage, 'video ID', group='id')
+        return self.url_result(
+            'http://video.foxnews.com/v/' + video_id,
+            FoxNewsVideoIE.ie_key())
+
+
 class FoxNewsInsiderIE(InfoExtractor):
     _VALID_URL = r'https?://insider\.foxnews\.com/([^/]+/)+(?P<id>[a-z-]+)'
     IE_NAME = 'foxnews:insider'
@@ -83,7 +113,11 @@ class FoxNewsInsiderIE(InfoExtractor):
             'upload_date': '20160825',
             'thumbnail': 're:^https?://.*\.jpg$',
         },
-        'add_ie': [FoxNewsIE.ie_key()],
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+        'add_ie': [FoxNewsVideoIE.ie_key()],
     }
 
     def _real_extract(self, url):
@@ -98,7 +132,7 @@ class FoxNewsInsiderIE(InfoExtractor):
 
         return {
             '_type': 'url_transparent',
-            'ie_key': FoxNewsIE.ie_key(),
+            'ie_key': FoxNewsVideoIE.ie_key(),
             'url': embed_url,
             'display_id': display_id,
             'title': title,

From f01115c933bdf6a3d741bb2f306d26b4df943a40 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 11 Sep 2016 18:36:59 +0800
Subject: [PATCH 6050/6219] [openload] Temporary fix (#10408)

---
 youtube_dl/extractor/openload.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py
index 03baf8e32..76316ca2f 100644
--- a/youtube_dl/extractor/openload.py
+++ b/youtube_dl/extractor/openload.py
@@ -60,7 +60,7 @@ class OpenloadIE(InfoExtractor):
             if j >= 33 and j <= 126:
                 j = ((j + 14) % 94) + 33
             if idx == len(enc_data) - 1:
-                j += 1
+                j += 3
             video_url_chars += compat_chr(j)
 
         video_url = 'https://openload.co/stream/%s?mime=true' % ''.join(video_url_chars)

From fea74acad8e8ebc1fda1d24a10c085c6771a71be Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 11 Sep 2016 18:53:05 +0800
Subject: [PATCH 6051/6219] [foxnews] Revert to old extractor names

---
 youtube_dl/extractor/extractors.py |  2 +-
 youtube_dl/extractor/foxnews.py    | 14 +++++++-------
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index e9027fb69..a3cd9c289 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -292,8 +292,8 @@ from .fourtube import FourTubeIE
 from .fox import FOXIE
 from .foxgay import FoxgayIE
 from .foxnews import (
-    FoxNewsVideoIE,
     FoxNewsIE,
+    FoxNewsArticleIE,
     FoxNewsInsiderIE,
 )
 from .foxsports import FoxSportsIE
diff --git a/youtube_dl/extractor/foxnews.py b/youtube_dl/extractor/foxnews.py
index 3e9a6a08c..229bcb175 100644
--- a/youtube_dl/extractor/foxnews.py
+++ b/youtube_dl/extractor/foxnews.py
@@ -6,8 +6,8 @@ from .amp import AMPIE
 from .common import InfoExtractor
 
 
-class FoxNewsVideoIE(AMPIE):
-    IE_NAME = 'foxnews:video'
+class FoxNewsIE(AMPIE):
+    IE_NAME = 'foxnews'
     IE_DESC = 'Fox News and Fox Business Video'
     _VALID_URL = r'https?://(?P<host>video\.(?:insider\.)?fox(?:news|business)\.com)/v/(?:video-embed\.html\?video_id=)?(?P<id>\d+)'
     _TESTS = [
@@ -67,9 +67,9 @@ class FoxNewsVideoIE(AMPIE):
         return info
 
 
-class FoxNewsIE(InfoExtractor):
+class FoxNewsArticleIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?foxnews\.com/(?!v)([^/]+/)+(?P<id>[a-z-]+)'
-    IE_NAME = 'foxnews'
+    IE_NAME = 'foxnews:article'
 
     _TEST = {
         'url': 'http://www.foxnews.com/politics/2016/09/08/buzz-about-bud-clinton-camp-denies-claims-wore-earpiece-at-forum.html',
@@ -93,7 +93,7 @@ class FoxNewsIE(InfoExtractor):
             webpage, 'video ID', group='id')
         return self.url_result(
             'http://video.foxnews.com/v/' + video_id,
-            FoxNewsVideoIE.ie_key())
+            FoxNewsIE.ie_key())
 
 
 class FoxNewsInsiderIE(InfoExtractor):
@@ -117,7 +117,7 @@ class FoxNewsInsiderIE(InfoExtractor):
             # m3u8 download
             'skip_download': True,
         },
-        'add_ie': [FoxNewsVideoIE.ie_key()],
+        'add_ie': [FoxNewsIE.ie_key()],
     }
 
     def _real_extract(self, url):
@@ -132,7 +132,7 @@ class FoxNewsInsiderIE(InfoExtractor):
 
         return {
             '_type': 'url_transparent',
-            'ie_key': FoxNewsVideoIE.ie_key(),
+            'ie_key': FoxNewsIE.ie_key(),
             'url': embed_url,
             'display_id': display_id,
             'title': title,

From 6bb05b32a990b8fb961971fcb8110d292cf953e7 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 11 Sep 2016 19:22:51 +0800
Subject: [PATCH 6052/6219] [pornhub] Extract categories and tags (closes
 #10499)

---
 ChangeLog                       |  1 +
 youtube_dl/extractor/pornhub.py | 16 ++++++++++++++++
 2 files changed, 17 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index a73a35e88..5d6609987 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
 version <unreleased>
 
 Extractors
++ [pornhub] Extract categories and tags (#10499)
 + [foxnews] Support Fox News articles (#10598)
 * [iwara] Fix extraction after relaunch (#10462, #3215)
 * [newgrounds] Fix uploader extraction (#10584)
diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index 20976c101..0724efc09 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -15,6 +15,7 @@ from ..compat import (
 from ..utils import (
     ExtractorError,
     int_or_none,
+    js_to_json,
     orderedSet,
     sanitized_Request,
     str_to_int,
@@ -48,6 +49,8 @@ class PornHubIE(InfoExtractor):
             'dislike_count': int,
             'comment_count': int,
             'age_limit': 18,
+            'tags': list,
+            'categories': list,
         },
     }, {
         # non-ASCII title
@@ -63,6 +66,8 @@ class PornHubIE(InfoExtractor):
             'dislike_count': int,
             'comment_count': int,
             'age_limit': 18,
+            'tags': list,
+            'categories': list,
         },
         'params': {
             'skip_download': True,
@@ -183,6 +188,15 @@ class PornHubIE(InfoExtractor):
             })
         self._sort_formats(formats)
 
+        page_params = self._parse_json(self._search_regex(
+            r'page_params\.zoneDetails\[([\'"])[^\'"]+\1\]\s*=\s*(?P<data>{[^}]+})',
+            webpage, 'page parameters', group='data', default='{}'),
+            video_id, transform_source=js_to_json, fatal=False)
+        tags = categories = None
+        if page_params:
+            tags = page_params.get('tags', '').split(',')
+            categories = page_params.get('categories', '').split(',')
+
         return {
             'id': video_id,
             'uploader': video_uploader,
@@ -195,6 +209,8 @@ class PornHubIE(InfoExtractor):
             'comment_count': comment_count,
             'formats': formats,
             'age_limit': 18,
+            'tags': tags,
+            'categories': categories,
         }
 
 

From 6599c72527ca8434589c010c48164494ab4c2469 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 11 Sep 2016 22:50:36 +0700
Subject: [PATCH 6053/6219] [tube8] Extract categories and tags (Closes #10579)

---
 youtube_dl/extractor/tube8.py | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/youtube_dl/extractor/tube8.py b/youtube_dl/extractor/tube8.py
index 4053f6c21..e937b2396 100644
--- a/youtube_dl/extractor/tube8.py
+++ b/youtube_dl/extractor/tube8.py
@@ -1,5 +1,7 @@
 from __future__ import unicode_literals
 
+import re
+
 from ..utils import (
     int_or_none,
     str_to_int,
@@ -21,7 +23,13 @@ class Tube8IE(KeezMoviesIE):
             'title': 'Kasia music video',
             'age_limit': 18,
             'duration': 230,
+            'categories': ['Teen'],
+            'tags': ['dancing'],
+        },
+        'params': {
+            'proxy': '127.0.0.1:8118',
         }
+
     }, {
         'url': 'http://www.tube8.com/shemale/teen/blonde-cd-gets-kidnapped-by-two-blacks-and-punished-for-being-a-slutty-girl/19569151/',
         'only_matching': True,
@@ -51,6 +59,17 @@ class Tube8IE(KeezMoviesIE):
             r'<span id="allCommentsCount">(\d+)</span>',
             webpage, 'comment count', fatal=False))
 
+        category = self._search_regex(
+            r'Category:\s*</strong>\s*<a[^>]+href=[^>]+>([^<]+)',
+            webpage, 'category', fatal=False)
+        categories = [category] if category else None
+
+        tags_str = self._search_regex(
+            r'(?s)Tags:\s*</strong>(.+?)</(?!a)',
+            webpage, 'tags', fatal=False)
+        tags = [t for t in re.findall(
+            r'<a[^>]+href=[^>]+>([^<]+)', tags_str)] if tags_str else None
+
         info.update({
             'description': description,
             'uploader': uploader,
@@ -58,6 +77,8 @@ class Tube8IE(KeezMoviesIE):
             'like_count': like_count,
             'dislike_count': dislike_count,
             'comment_count': comment_count,
+            'categories': categories,
+            'tags': tags,
         })
 
         return info

From bc9186c8822db456dae93d053a34e60b7887405a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 11 Sep 2016 22:51:12 +0700
Subject: [PATCH 6054/6219] [tvplay] Remove unused import

---
 youtube_dl/extractor/tvplay.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/tvplay.py b/youtube_dl/extractor/tvplay.py
index 5548ff2ac..58ffc0e6f 100644
--- a/youtube_dl/extractor/tvplay.py
+++ b/youtube_dl/extractor/tvplay.py
@@ -16,7 +16,6 @@ from ..utils import (
     parse_iso8601,
     qualities,
     try_get,
-    js_to_json,
     update_url_query,
 )
 

From 1c81476cbb167776e7b1454bf135fb7ebf62547f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 11 Sep 2016 23:20:09 +0700
Subject: [PATCH 6055/6219] release 2016.09.11

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 9 +++++++--
 youtube_dl/version.py     | 2 +-
 4 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index a983bf432..d7195712b 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.09.08*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.09.08**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.09.11*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.09.11**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.09.08
+[debug] youtube-dl version 2016.09.11
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 5d6609987..21d9f6275 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2016.09.11
 
 Extractors
 + [pornhub] Extract categories and tags (#10499)
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index e6be746a8..7a7b268d3 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -247,7 +247,8 @@
  - **Formula1**
  - **FOX**
  - **Foxgay**
- - **FoxNews**: Fox News and Fox Business Video
+ - **foxnews**: Fox News and Fox Business Video
+ - **foxnews:article**
  - **foxnews:insider**
  - **FoxSports**
  - **france2.fr:generation-quoi**
@@ -326,6 +327,7 @@
  - **ivi**: ivi.ru
  - **ivi:compilation**: ivi.ru compilations
  - **ivideon**: Ivideon TV
+ - **Iwara**
  - **Izlesene**
  - **JeuxVideo**
  - **Jove**
@@ -339,6 +341,7 @@
  - **KarriereVideos**
  - **keek**
  - **KeezMovies**
+ - **Ketnet**
  - **KhanAcademy**
  - **KickStarter**
  - **KonserthusetPlay**
@@ -540,6 +543,7 @@
  - **podomatic**
  - **Pokemon**
  - **PolskieRadio**
+ - **PolskieRadioCategory**
  - **PornCom**
  - **PornHd**
  - **PornHub**: PornHub and Thumbzilla
@@ -701,9 +705,11 @@
  - **Telecinco**: telecinco.es, cuatro.com and mediaset.es
  - **Telegraaf**
  - **TeleMB**
+ - **TeleQuebec**
  - **TeleTask**
  - **Telewebion**
  - **TF1**
+ - **TFO**
  - **TheIntercept**
  - **ThePlatform**
  - **ThePlatformFeed**
@@ -725,7 +731,6 @@
  - **ToypicsUser**: Toypics user profile
  - **TrailerAddict** (Currently broken)
  - **Trilulilu**
- - **trollvids**
  - **TruTV**
  - **Tube8**
  - **TubiTv**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 941ffb3f6..5f572391c 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.09.08'
+__version__ = '2016.09.11'

From eb87d4545a58be369723eddf5433b4198d64d367 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 11 Sep 2016 23:29:25 +0700
Subject: [PATCH 6056/6219] [devscripts/release.sh] Add ChangeLog reminder
 prompt

---
 devscripts/release.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/devscripts/release.sh b/devscripts/release.sh
index ca6ae1b49..981d37ca7 100755
--- a/devscripts/release.sh
+++ b/devscripts/release.sh
@@ -60,6 +60,9 @@ if ! type pandoc >/dev/null 2>/dev/null; then echo 'ERROR: pandoc is missing'; e
 if ! python3 -c 'import rsa' 2>/dev/null; then echo 'ERROR: python3-rsa is missing'; exit 1; fi
 if ! python3 -c 'import wheel' 2>/dev/null; then echo 'ERROR: wheel is missing'; exit 1; fi
 
+read -p "Is ChangeLog up to date? (y/n) " -n 1
+if [[ ! $REPLY =~ ^[Yy]$ ]]; then exit 1;
+
 /bin/echo -e "\n### First of all, testing..."
 make clean
 if $skip_tests ; then

From d667ab7fad8d04a318b54e95d7a764e1667d80bd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 11 Sep 2016 23:30:18 +0700
Subject: [PATCH 6057/6219] [ChangeLog] Actualize

---
 ChangeLog | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 21d9f6275..9183f29e8 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,10 +1,22 @@
-version 2016.09.11
+version <unreleased>
 
 Extractors
++ [tube8] Extract categories and tags (#10579)
 + [pornhub] Extract categories and tags (#10499)
-+ [foxnews] Support Fox News articles (#10598)
+* [openload] Temporary fix (#10408)
++ [foxnews] Add support Fox News articles (#10598)
+* [viafree] Improve video id extraction (#10615)
 * [iwara] Fix extraction after relaunch (#10462, #3215)
++ [tfo] Add extractor for tfo.org
+* [lrt] Fix audio extraction (#10566)
+* [9now] Fix extraction (#10561)
++ [canalplus] Add support for c8.fr (#10577)
 * [newgrounds] Fix uploader extraction (#10584)
++ [polskieradio:category] Add support for category lists (#10576)
++ [ketnet] Add extractor for ketnet.be (#10343)
++ [canvas] Add support for een.be (#10605)
++ [telequebec] Add extractor for telequebec.tv (#1999)
+* [parliamentliveuk] Fix extraction (#9137)
 
 
 version 2016.09.08

From fc150cba1d6763ab115319c5726b5081b0f49106 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 11 Sep 2016 23:32:01 +0700
Subject: [PATCH 6058/6219] [devscripts/release.sh] Add missing fi

---
 devscripts/release.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/devscripts/release.sh b/devscripts/release.sh
index 981d37ca7..1af61aa0b 100755
--- a/devscripts/release.sh
+++ b/devscripts/release.sh
@@ -61,7 +61,7 @@ if ! python3 -c 'import rsa' 2>/dev/null; then echo 'ERROR: python3-rsa is missi
 if ! python3 -c 'import wheel' 2>/dev/null; then echo 'ERROR: wheel is missing'; exit 1; fi
 
 read -p "Is ChangeLog up to date? (y/n) " -n 1
-if [[ ! $REPLY =~ ^[Yy]$ ]]; then exit 1;
+if [[ ! $REPLY =~ ^[Yy]$ ]]; then exit 1; fi
 
 /bin/echo -e "\n### First of all, testing..."
 make clean

From 0307d6fba6d3b793acac5785b2cee39e3dfbffcc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 11 Sep 2016 23:33:20 +0700
Subject: [PATCH 6059/6219] release 2016.09.11.1

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 youtube_dl/version.py     | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index d7195712b..e87fed573 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.09.11*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.09.11**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.09.11.1*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.09.11.1**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.09.11
+[debug] youtube-dl version 2016.09.11.1
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 9183f29e8..669544815 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2016.09.11.1
 
 Extractors
 + [tube8] Extract categories and tags (#10579)
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 5f572391c..903aede58 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.09.11'
+__version__ = '2016.09.11.1'

From ee7e672eb0eca7a916845b359511262935f9ef1b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 11 Sep 2016 23:44:22 +0700
Subject: [PATCH 6060/6219] [tube8] Remove proxy settings from test

---
 youtube_dl/extractor/tube8.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/youtube_dl/extractor/tube8.py b/youtube_dl/extractor/tube8.py
index e937b2396..1853a1104 100644
--- a/youtube_dl/extractor/tube8.py
+++ b/youtube_dl/extractor/tube8.py
@@ -26,10 +26,6 @@ class Tube8IE(KeezMoviesIE):
             'categories': ['Teen'],
             'tags': ['dancing'],
         },
-        'params': {
-            'proxy': '127.0.0.1:8118',
-        }
-
     }, {
         'url': 'http://www.tube8.com/shemale/teen/blonde-cd-gets-kidnapped-by-two-blacks-and-punished-for-being-a-slutty-girl/19569151/',
         'only_matching': True,

From be457302267b456412fb9848bcb8ce36874d8d7e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 12 Sep 2016 02:55:15 +0800
Subject: [PATCH 6061/6219] [nbc] Add new extractor for NBC Olympics (#10295,
 #10361)

---
 ChangeLog                          |  6 +++++
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/nbc.py        | 40 ++++++++++++++++++++++++++++++
 3 files changed, 47 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 669544815..46eea0626 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Extractors
++ [nbc] Add support for NBC Olympics (#10361)
+
+
 version 2016.09.11.1
 
 Extractors
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index a3cd9c289..522691de1 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -534,6 +534,7 @@ from .nbc import (
     CSNNEIE,
     NBCIE,
     NBCNewsIE,
+    NBCOlympicsIE,
     NBCSportsIE,
     NBCSportsVPlayerIE,
 )
diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index f694e210b..f37bf2f30 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -335,3 +335,43 @@ class NBCNewsIE(ThePlatformIE):
                 'url': 'http://feed.theplatform.com/f/2E2eJC/nnd_NBCNews?byId=%s' % video_id,
                 'ie_key': 'ThePlatformFeed',
             }
+
+
+class NBCOlympicsIE(InfoExtractor):
+    _VALID_URL = r'https?://www\.nbcolympics\.com/video/(?P<id>[a-z-]+)'
+
+    _TEST = {
+        # Geo-restricted to US
+        'url': 'http://www.nbcolympics.com/video/justin-roses-son-leo-was-tears-after-his-dad-won-gold',
+        'md5': '54fecf846d05429fbaa18af557ee523a',
+        'info_dict': {
+            'id': 'WjTBzDXx5AUq',
+            'display_id': 'justin-roses-son-leo-was-tears-after-his-dad-won-gold',
+            'ext': 'mp4',
+            'title': 'Rose\'s son Leo was in tears after his dad won gold',
+            'description': 'Olympic gold medalist Justin Rose gets emotional talking to the impact his win in men\'s golf has already had on his children.',
+            'timestamp': 1471274964,
+            'upload_date': '20160815',
+            'uploader': 'NBCU-SPORTS',
+        },
+    }
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, display_id)
+
+        drupal_settings = self._parse_json(self._search_regex(
+            r'jQuery\.extend\(Drupal\.settings\s*,\s*({.+?})\);',
+            webpage, 'drupal settings'), display_id)
+
+        iframe_url = drupal_settings['vod']['iframe_url']
+        theplatform_url = iframe_url.replace(
+            'vplayer.nbcolympics.com', 'player.theplatform.com')
+
+        return {
+            '_type': 'url_transparent',
+            'url': theplatform_url,
+            'ie_key': ThePlatformIE.ie_key(),
+            'display_id': display_id,
+        }

From 546edb2efabb18f9eb0eecb2f8719fcb777e99a7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 12 Sep 2016 21:01:31 +0700
Subject: [PATCH 6062/6219] [ISSUE_TEMPLATE_tmpl.md] Fix typo

---
 .github/ISSUE_TEMPLATE_tmpl.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/ISSUE_TEMPLATE_tmpl.md b/.github/ISSUE_TEMPLATE_tmpl.md
index a5e6a4233..4112f53bb 100644
--- a/.github/ISSUE_TEMPLATE_tmpl.md
+++ b/.github/ISSUE_TEMPLATE_tmpl.md
@@ -55,4 +55,4 @@ $ youtube-dl -v <your command line>
 ### Description of your *issue*, suggested solution and other information
 
 Explanation of your *issue* in arbitrary form goes here. Please make sure the [description is worded well enough to be understood](https://github.com/rg3/youtube-dl#is-the-description-of-the-issue-itself-sufficient). Provide as much context and examples as possible.
-If work on your *issue* required an account credentials please provide them or explain how one can obtain them.
+If work on your *issue* requires an account credentials please provide them or explain how one can obtain them.

From d002e919863c910e52c623ee544e93fe41af4665 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 12 Sep 2016 21:48:45 +0700
Subject: [PATCH 6063/6219] [vimeo:ondemand] Pass Referer along with embed URL
 (#10624)

---
 youtube_dl/extractor/vimeo.py | 28 +++++++++++++++++++++++++---
 1 file changed, 25 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 7e854f326..50aacc6ac 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -350,6 +350,10 @@ class VimeoIE(VimeoBaseInfoExtractor):
         }
     ]
 
+    @staticmethod
+    def _smuggle_referrer(url, referrer_url):
+        return smuggle_url(url, {'http_headers': {'Referer': referrer_url}})
+
     @staticmethod
     def _extract_vimeo_url(url, webpage):
         # Look for embedded (iframe) Vimeo player
@@ -357,8 +361,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
             r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//player\.vimeo\.com/video/.+?)\1', webpage)
         if mobj:
             player_url = unescapeHTML(mobj.group('url'))
-            surl = smuggle_url(player_url, {'http_headers': {'Referer': url}})
-            return surl
+            return VimeoIE._smuggle_referrer(player_url, url)
         # Look for embedded (swf embed) Vimeo player
         mobj = re.search(
             r'<embed[^>]+?src="((?:https?:)?//(?:www\.)?vimeo\.com/moogaloop\.swf.+?)"', webpage)
@@ -585,6 +588,20 @@ class VimeoOndemandIE(VimeoBaseInfoExtractor):
             'uploader_url': 're:https?://(?:www\.)?vimeo\.com/gumfilms',
             'uploader_id': 'gumfilms',
         },
+    }, {
+        # requires Referer to be passed along with og:video:url
+        'url': 'https://vimeo.com/ondemand/36938/126682985',
+        'info_dict': {
+            'id': '126682985',
+            'ext': 'mp4',
+            'title': 'Rävlock, rätt läte på rätt plats',
+            'uploader': 'Lindroth & Norin',
+            'uploader_url': 're:https?://(?:www\.)?vimeo\.com/user14430847',
+            'uploader_id': 'user14430847',
+        },
+        'params': {
+            'skip_download': True,
+        },
     }, {
         'url': 'https://vimeo.com/ondemand/nazmaalik',
         'only_matching': True,
@@ -599,7 +616,12 @@ class VimeoOndemandIE(VimeoBaseInfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
-        return self.url_result(self._og_search_video_url(webpage), VimeoIE.ie_key())
+        return self.url_result(
+            # Some videos require Referer to be passed along with og:video:url
+            # similarly to generic vimeo embeds (e.g.
+            # https://vimeo.com/ondemand/36938/126682985).
+            VimeoIE._smuggle_referrer(self._og_search_video_url(webpage), url),
+            VimeoIE.ie_key())
 
 
 class VimeoChannelIE(VimeoBaseInfoExtractor):

From a5ff05df1af97613c979f85ab2f6f610f60be910 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 12 Sep 2016 21:49:31 +0700
Subject: [PATCH 6064/6219] [extractor/generic] Add vimeo embed that requires
 Referer passed

---
 youtube_dl/extractor/generic.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 24b217715..2e46ca179 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1369,6 +1369,11 @@ class GenericIE(InfoExtractor):
             },
             'add_ie': ['Vimeo'],
         },
+        {
+            # generic vimeo embed that requires original URL passed as Referer
+            'url': 'http://racing4everyone.eu/2016/07/30/formula-1-2016-round12-germany/',
+            'only_matching': True,
+        },
         {
             'url': 'https://support.arkena.com/display/PLAY/Ways+to+embed+your+video',
             'md5': 'b96f2f71b359a8ecd05ce4e1daa72365',

From e8bcd982ccee87e45a5cc8b116cc4452c81b0453 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 12 Sep 2016 22:33:00 +0700
Subject: [PATCH 6065/6219] [kaltura] Skip chun format

---
 youtube_dl/extractor/kaltura.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/kaltura.py b/youtube_dl/extractor/kaltura.py
index 6a8464998..22a06e4ae 100644
--- a/youtube_dl/extractor/kaltura.py
+++ b/youtube_dl/extractor/kaltura.py
@@ -262,6 +262,10 @@ class KalturaIE(InfoExtractor):
             # Continue if asset is not ready
             if f.get('status') != 2:
                 continue
+            # Original format that's not available (e.g. kaltura:1926081:0_c03e1b5g)
+            # skip for now.
+            if f.get('fileExt') == 'chun':
+                continue
             video_url = sign_url(
                 '%s/flavorId/%s' % (data_url, f['id']))
             formats.append({

From 1d16035bb4ec516d25326ce5ff35affb4ff1f13c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 12 Sep 2016 22:43:45 +0700
Subject: [PATCH 6066/6219] [kaltura] Improve audio detection

---
 youtube_dl/extractor/kaltura.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/kaltura.py b/youtube_dl/extractor/kaltura.py
index 22a06e4ae..5a8403777 100644
--- a/youtube_dl/extractor/kaltura.py
+++ b/youtube_dl/extractor/kaltura.py
@@ -268,6 +268,10 @@ class KalturaIE(InfoExtractor):
                 continue
             video_url = sign_url(
                 '%s/flavorId/%s' % (data_url, f['id']))
+            # audio-only has no videoCodecId (e.g. kaltura:1926081:0_c03e1b5g
+            # -f mp4-56)
+            vcodec = 'none' if 'videoCodecId' not in f and f.get(
+                'frameRate') == 0 else f.get('videoCodecId')
             formats.append({
                 'format_id': '%(fileExt)s-%(bitrate)s' % f,
                 'ext': f.get('fileExt'),
@@ -275,7 +279,7 @@ class KalturaIE(InfoExtractor):
                 'fps': int_or_none(f.get('frameRate')),
                 'filesize_approx': int_or_none(f.get('size'), invscale=1024),
                 'container': f.get('containerFormat'),
-                'vcodec': f.get('videoCodecId'),
+                'vcodec': vcodec,
                 'height': int_or_none(f.get('height')),
                 'width': int_or_none(f.get('width')),
                 'url': video_url,

From a6ccc3e518eabf61cc41575e52361d5ea79e3796 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 12 Sep 2016 23:05:52 +0700
Subject: [PATCH 6067/6219] [safari] Improve ids regexes (#10617)

---
 youtube_dl/extractor/safari.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/safari.py b/youtube_dl/extractor/safari.py
index 08ddbe3c4..eabe41efe 100644
--- a/youtube_dl/extractor/safari.py
+++ b/youtube_dl/extractor/safari.py
@@ -103,13 +103,13 @@ class SafariIE(SafariBaseIE):
 
         webpage = self._download_webpage(url, video_id)
         reference_id = self._search_regex(
-            r'data-reference-id=(["\'])(?P<id>.+?)\1',
+            r'data-reference-id=(["\'])(?P<id>(?:(?!\1).)+)\1',
             webpage, 'kaltura reference id', group='id')
         partner_id = self._search_regex(
-            r'data-partner-id=(["\'])(?P<id>.+?)\1',
+            r'data-partner-id=(["\'])(?P<id>(?:(?!\1).)+)\1',
             webpage, 'kaltura widget id', group='id')
         ui_id = self._search_regex(
-            r'data-ui-id=(["\'])(?P<id>.+?)\1',
+            r'data-ui-id=(["\'])(?P<id>(?:(?!\1).)+)\1',
             webpage, 'kaltura uiconf id', group='id')
 
         query = {

From fcba157e8049350c5386cc3b850626320d9ff7eb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 12 Sep 2016 23:29:43 +0700
Subject: [PATCH 6068/6219] [ISSUE_TEMPLATE_tmpl.md] Fix typo

---
 .github/ISSUE_TEMPLATE_tmpl.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/ISSUE_TEMPLATE_tmpl.md b/.github/ISSUE_TEMPLATE_tmpl.md
index 4112f53bb..ab9968129 100644
--- a/.github/ISSUE_TEMPLATE_tmpl.md
+++ b/.github/ISSUE_TEMPLATE_tmpl.md
@@ -55,4 +55,4 @@ $ youtube-dl -v <your command line>
 ### Description of your *issue*, suggested solution and other information
 
 Explanation of your *issue* in arbitrary form goes here. Please make sure the [description is worded well enough to be understood](https://github.com/rg3/youtube-dl#is-the-description-of-the-issue-itself-sufficient). Provide as much context and examples as possible.
-If work on your *issue* requires an account credentials please provide them or explain how one can obtain them.
+If work on your *issue* requires account credentials please provide them or explain how one can obtain them.

From 7a7309219cae70e14f58e904591a77360bfbc985 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 12 Sep 2016 23:39:11 +0100
Subject: [PATCH 6069/6219] [adobepass] add an option to specify mso_id and
 support for ROGERS TV Provider(closes #10606)

---
 youtube_dl/YoutubeDL.py           |  1 +
 youtube_dl/__init__.py            |  1 +
 youtube_dl/extractor/adobepass.py | 49 +++++++++++++++++++++++--------
 youtube_dl/options.py             |  4 +++
 4 files changed, 42 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 805733fb7..f70d5f49a 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -131,6 +131,7 @@ class YoutubeDL(object):
     username:          Username for authentication purposes.
     password:          Password for authentication purposes.
     videopassword:     Password for accessing a video.
+    ap_mso_id          Adobe Pass Multiple-system operator Identifier.
     usenetrc:          Use netrc for authentication instead.
     verbose:           Print additional info to stdout.
     quiet:             Do not print messages to stdout.
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 42128272a..2b1b841c9 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -293,6 +293,7 @@ def _real_main(argv=None):
         'password': opts.password,
         'twofactor': opts.twofactor,
         'videopassword': opts.videopassword,
+        'ap_mso_id': opts.ap_mso_id,
         'quiet': (opts.quiet or any_getting or any_printing),
         'no_warnings': opts.no_warnings,
         'forceurl': opts.geturl,
diff --git a/youtube_dl/extractor/adobepass.py b/youtube_dl/extractor/adobepass.py
index 68ec37e00..454a6af8d 100644
--- a/youtube_dl/extractor/adobepass.py
+++ b/youtube_dl/extractor/adobepass.py
@@ -6,10 +6,12 @@ import time
 import xml.etree.ElementTree as etree
 
 from .common import InfoExtractor
+from ..compat import compat_urlparse
 from ..utils import (
     unescapeHTML,
     urlencode_postdata,
     unified_timestamp,
+    ExtractorError,
 )
 
 
@@ -41,6 +43,11 @@ class AdobePassIE(InfoExtractor):
             token_expires = unified_timestamp(re.sub(r'[_ ]GMT', '', xml_text(token, date_ele)))
             return token_expires and token_expires <= int(time.time())
 
+        def raise_mvpd_required():
+            raise ExtractorError('This video is only available for users of participating TV providers. '
+                'Use --ap-mso-id to specify Adobe Pass Multiple-system operator Identifier '
+                'and --netrc to provide account credentials.', expected=True)
+
         mvpd_headers = {
             'ap_42': 'anonymous',
             'ap_11': 'Linux i686',
@@ -55,19 +62,26 @@ class AdobePassIE(InfoExtractor):
             authn_token = None
         if not authn_token:
             # TODO add support for other TV Providers
-            mso_id = 'DTV'
+            mso_id = self._downloader.params.get('ap_mso_id')
+            if not mso_id:
+                raise_mvpd_required()
             username, password = self._get_netrc_login_info(mso_id)
             if not username or not password:
-                return ''
+                return raise_mvpd_required()
 
-            def post_form(form_page, note, data={}):
+            def post_form(form_page_res, note, data={}):
+                form_page, urlh = form_page_res
                 post_url = self._html_search_regex(r'<form[^>]+action=(["\'])(?P<url>.+?)\1', form_page, 'post url', group='url')
-                return self._download_webpage(
-                    post_url, video_id, note, data=urlencode_postdata(data or self._hidden_inputs(form_page)), headers={
+                if not re.match(r'https?://', post_url):
+                    post_url = compat_urlparse.urljoin(urlh.geturl(), post_url)
+                form_data = self._hidden_inputs(form_page)
+                form_data.update(data)
+                return self._download_webpage_handle(
+                    post_url, video_id, note, data=urlencode_postdata(form_data), headers={
                         'Content-Type': 'application/x-www-form-urlencoded',
                     })
 
-            provider_redirect_page = self._download_webpage(
+            provider_redirect_page_res = self._download_webpage_handle(
                 self._SERVICE_PROVIDER_TEMPLATE % 'authenticate/saml', video_id,
                 'Downloading Provider Redirect Page', query={
                     'noflash': 'true',
@@ -77,13 +91,22 @@ class AdobePassIE(InfoExtractor):
                     'domain_name': 'adobe.com',
                     'redirect_url': url,
                 })
-            provider_login_page = post_form(
-                provider_redirect_page, 'Downloading Provider Login Page')
-            mvpd_confirm_page = post_form(provider_login_page, 'Logging in', {
-                'username': username,
-                'password': password,
-            })
-            post_form(mvpd_confirm_page, 'Confirming Login')
+            provider_login_page_res = post_form(
+                provider_redirect_page_res, 'Downloading Provider Login Page')
+            login_data = {}
+            if mso_id == 'DTV':
+                login_data = {
+                    'username': username,
+                    'password': password,
+                }
+            elif mso_id == 'Rogers':
+                login_data = {
+                    'UserName': username,
+                    'UserPassword': password,
+                }
+            mvpd_confirm_page_res = post_form(provider_login_page_res, 'Logging in', login_data)
+            if mso_id == 'DTV':
+                post_form(mvpd_confirm_page_res, 'Confirming Login')
 
             session = self._download_webpage(
                 self._SERVICE_PROVIDER_TEMPLATE % 'session', video_id,
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 56f312f57..c4057ce59 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -350,6 +350,10 @@ def parseOpts(overrideArguments=None):
         '--video-password',
         dest='videopassword', metavar='PASSWORD',
         help='Video password (vimeo, smotri, youku)')
+    authentication.add_option(
+        '--ap-mso-id',
+        dest='ap_mso_id', metavar='APMSOID',
+        help='Adobe Pass Multiple-system operator Identifier(DTV, Rogers)')
 
     video_format = optparse.OptionGroup(parser, 'Video Format Options')
     video_format.add_option(

From 45396dd2ed3bc7ab9ac6f9b5a5f51179b629abb7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 13 Sep 2016 23:20:25 +0700
Subject: [PATCH 6070/6219] [nhk] Fix extraction (Closes #10633)

---
 youtube_dl/extractor/nhk.py | 31 ++++++++++++++++---------------
 1 file changed, 16 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/nhk.py b/youtube_dl/extractor/nhk.py
index 691bdfa4e..5c8cd76dc 100644
--- a/youtube_dl/extractor/nhk.py
+++ b/youtube_dl/extractor/nhk.py
@@ -1,14 +1,15 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..utils import ExtractorError
 
 
 class NhkVodIE(InfoExtractor):
-    _VALID_URL = r'https?://www3\.nhk\.or\.jp/nhkworld/en/vod/(?P<id>.+?)\.html'
+    _VALID_URL = r'https?://www3\.nhk\.or\.jp/nhkworld/en/vod/(?P<id>[^/]+/[^/?#&]+)'
     _TEST = {
         # Videos available only for a limited period of time. Visit
         # http://www3.nhk.or.jp/nhkworld/en/vod/ for working samples.
-        'url': 'http://www3.nhk.or.jp/nhkworld/en/vod/tokyofashion/20160815.html',
+        'url': 'http://www3.nhk.or.jp/nhkworld/en/vod/tokyofashion/20160815',
         'info_dict': {
             'id': 'A1bnNiNTE6nY3jLllS-BIISfcC_PpvF5',
             'ext': 'flv',
@@ -19,25 +20,25 @@ class NhkVodIE(InfoExtractor):
         },
         'skip': 'Videos available only for a limited period of time',
     }
+    _API_URL = 'http://api.nhk.or.jp/nhkworld/vodesdlist/v1/all/all/all.json?apikey=EJfK8jdS57GqlupFgAfAAwr573q01y6k'
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        webpage = self._download_webpage(url, video_id)
+        data = self._download_json(self._API_URL, video_id)
 
-        embed_code = self._search_regex(
-            r'nw_vod_ooplayer\([^,]+,\s*(["\'])(?P<id>(?:(?!\1).)+)\1',
-            webpage, 'ooyala embed code', group='id')
+        try:
+            episode = next(
+                e for e in data['data']['episodes']
+                if e.get('url') and video_id in e['url'])
+        except StopIteration:
+            raise ExtractorError('Unable to find episode')
 
-        title = self._search_regex(
-            r'<div[^>]+class=["\']episode-detail["\']>\s*<h\d+>([^<]+)',
-            webpage, 'title', default=None)
-        description = self._html_search_regex(
-            r'(?s)<p[^>]+class=["\']description["\'][^>]*>(.+?)</p>',
-            webpage, 'description', default=None)
-        series = self._search_regex(
-            r'<h2[^>]+class=["\']detail-top-player-title[^>]+><a[^>]+>([^<]+)',
-            webpage, 'series', default=None)
+        embed_code = episode['vod_id']
+
+        title = episode.get('sub_title_clean') or episode['sub_title']
+        description = episode.get('description_clean') or episode.get('description')
+        series = episode.get('title_clean') or episode.get('title')
 
         return {
             '_type': 'url_transparent',

From 8414c2da31a5ff3cc5ba84fdd537d714d04949f8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 13 Sep 2016 23:22:16 +0700
Subject: [PATCH 6071/6219] [adobepass] PEP 8

---
 youtube_dl/extractor/adobepass.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/adobepass.py b/youtube_dl/extractor/adobepass.py
index 454a6af8d..50a208085 100644
--- a/youtube_dl/extractor/adobepass.py
+++ b/youtube_dl/extractor/adobepass.py
@@ -44,7 +44,8 @@ class AdobePassIE(InfoExtractor):
             return token_expires and token_expires <= int(time.time())
 
         def raise_mvpd_required():
-            raise ExtractorError('This video is only available for users of participating TV providers. '
+            raise ExtractorError(
+                'This video is only available for users of participating TV providers. '
                 'Use --ap-mso-id to specify Adobe Pass Multiple-system operator Identifier '
                 'and --netrc to provide account credentials.', expected=True)
 

From 1b6712ab2378b2e8eb59f372fb51193f8d3bdc97 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 13 Sep 2016 22:16:01 +0100
Subject: [PATCH 6072/6219] [adobepass] add specific options for adobe pass
 authentication

- add --ap-username and --ap-password option to specify
TV provider username and password in the cmd line
- add --ap-retries option to limit the number of retries
- add --list-ap-msi-ids to list the supported TV Providers
---
 youtube_dl/YoutubeDL.py           |   4 +-
 youtube_dl/__init__.py            |  15 +++
 youtube_dl/extractor/adobepass.py | 206 ++++++++++++++++--------------
 youtube_dl/extractor/common.py    |  10 +-
 youtube_dl/options.py             |  24 +++-
 5 files changed, 155 insertions(+), 104 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index f70d5f49a..9c2c26280 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -131,7 +131,9 @@ class YoutubeDL(object):
     username:          Username for authentication purposes.
     password:          Password for authentication purposes.
     videopassword:     Password for accessing a video.
-    ap_mso_id          Adobe Pass Multiple-system operator Identifier.
+    ap_mso_id:         Adobe Pass Multiple-system operator Identifier.
+    ap_username:       TV Provider username for authentication purposes.
+    ap_password:       TV Provider password for authentication purposes.
     usenetrc:          Use netrc for authentication instead.
     verbose:           Print additional info to stdout.
     quiet:             Do not print messages to stdout.
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 2b1b841c9..052f20ee7 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -34,12 +34,14 @@ from .utils import (
     setproctitle,
     std_headers,
     write_string,
+    render_table,
 )
 from .update import update_self
 from .downloader import (
     FileDownloader,
 )
 from .extractor import gen_extractors, list_extractors
+from .extractor.adobepass import MSO_INFO
 from .YoutubeDL import YoutubeDL
 
 
@@ -118,18 +120,26 @@ def _real_main(argv=None):
                 desc += ' (Example: "%s%s:%s" )' % (ie.SEARCH_KEY, random.choice(_COUNTS), random.choice(_SEARCHES))
             write_string(desc + '\n', out=sys.stdout)
         sys.exit(0)
+    if opts.list_ap_mso_ids:
+        table = [[mso_id, mso_info['name']] for mso_id, mso_info in MSO_INFO.items()]
+        write_string('Supported TV Providers:\n' + render_table(['mso id', 'mso name'], table) + '\n', out=sys.stdout)
+        sys.exit(0)
 
     # Conflicting, missing and erroneous options
     if opts.usenetrc and (opts.username is not None or opts.password is not None):
         parser.error('using .netrc conflicts with giving username/password')
     if opts.password is not None and opts.username is None:
         parser.error('account username missing\n')
+    if opts.ap_password is not None and opts.ap_username is None:
+        parser.error('TV Provider account username missing\n')
     if opts.outtmpl is not None and (opts.usetitle or opts.autonumber or opts.useid):
         parser.error('using output template conflicts with using title, video ID or auto number')
     if opts.usetitle and opts.useid:
         parser.error('using title conflicts with using video ID')
     if opts.username is not None and opts.password is None:
         opts.password = compat_getpass('Type account password and press [Return]: ')
+    if opts.ap_username is not None and opts.ap_password is None:
+        opts.ap_password = compat_getpass('Type TV provider account password and press [Return]: ')
     if opts.ratelimit is not None:
         numeric_limit = FileDownloader.parse_bytes(opts.ratelimit)
         if numeric_limit is None:
@@ -169,6 +179,8 @@ def _real_main(argv=None):
         opts.retries = parse_retries(opts.retries)
     if opts.fragment_retries is not None:
         opts.fragment_retries = parse_retries(opts.fragment_retries)
+    if opts.ap_retries is not None:
+        opts.ap_retries = parse_retries(opts.ap_retries)
     if opts.buffersize is not None:
         numeric_buffersize = FileDownloader.parse_bytes(opts.buffersize)
         if numeric_buffersize is None:
@@ -294,6 +306,9 @@ def _real_main(argv=None):
         'twofactor': opts.twofactor,
         'videopassword': opts.videopassword,
         'ap_mso_id': opts.ap_mso_id,
+        'ap_username': opts.ap_username,
+        'ap_password': opts.ap_password,
+        'ap_retries': opts.ap_retries,
         'quiet': (opts.quiet or any_getting or any_printing),
         'no_warnings': opts.no_warnings,
         'forceurl': opts.geturl,
diff --git a/youtube_dl/extractor/adobepass.py b/youtube_dl/extractor/adobepass.py
index 50a208085..9add6c0f8 100644
--- a/youtube_dl/extractor/adobepass.py
+++ b/youtube_dl/extractor/adobepass.py
@@ -15,6 +15,20 @@ from ..utils import (
 )
 
 
+MSO_INFO = {
+    'DTV': {
+        'name': 'DirecTV',
+        'username_field': 'username',
+        'password_field': 'password',
+    },
+    'Rogers': {
+        'name': 'Rogers Cable',
+        'username_field': 'UserName',
+        'password_field': 'UserPassword',
+    },
+}
+
+
 class AdobePassIE(InfoExtractor):
     _SERVICE_PROVIDER_TEMPLATE = 'https://sp.auth.adobe.com/adobe-services/%s'
     _USER_AGENT = 'Mozilla/5.0 (X11; Linux i686; rv:47.0) Gecko/20100101 Firefox/47.0'
@@ -43,6 +57,18 @@ class AdobePassIE(InfoExtractor):
             token_expires = unified_timestamp(re.sub(r'[_ ]GMT', '', xml_text(token, date_ele)))
             return token_expires and token_expires <= int(time.time())
 
+        def post_form(form_page_res, note, data={}):
+            form_page, urlh = form_page_res
+            post_url = self._html_search_regex(r'<form[^>]+action=(["\'])(?P<url>.+?)\1', form_page, 'post url', group='url')
+            if not re.match(r'https?://', post_url):
+                post_url = compat_urlparse.urljoin(urlh.geturl(), post_url)
+            form_data = self._hidden_inputs(form_page)
+            form_data.update(data)
+            return self._download_webpage_handle(
+                post_url, video_id, note, data=urlencode_postdata(form_data), headers={
+                    'Content-Type': 'application/x-www-form-urlencoded',
+                })
+
         def raise_mvpd_required():
             raise ExtractorError(
                 'This video is only available for users of participating TV providers. '
@@ -57,105 +83,95 @@ class AdobePassIE(InfoExtractor):
         }
 
         guid = xml_text(resource, 'guid')
-        requestor_info = self._downloader.cache.load('mvpd', requestor_id) or {}
-        authn_token = requestor_info.get('authn_token')
-        if authn_token and is_expired(authn_token, 'simpleTokenExpires'):
-            authn_token = None
-        if not authn_token:
-            # TODO add support for other TV Providers
-            mso_id = self._downloader.params.get('ap_mso_id')
-            if not mso_id:
-                raise_mvpd_required()
-            username, password = self._get_netrc_login_info(mso_id)
-            if not username or not password:
-                return raise_mvpd_required()
+        retries = self._downloader.params.get('ap_retries', 3)
+        count = 0
+        while count < retries:
+            requestor_info = self._downloader.cache.load('mvpd', requestor_id) or {}
+            authn_token = requestor_info.get('authn_token')
+            if authn_token and is_expired(authn_token, 'simpleTokenExpires'):
+                authn_token = None
+            if not authn_token:
+                # TODO add support for other TV Providers
+                mso_id = self._downloader.params.get('ap_mso_id')
+                if not mso_id:
+                    raise_mvpd_required()
+                if mso_id not in MSO_INFO:
+                    raise ExtractorError(
+                        'Unsupported TV Provider, use --list-ap-mso-ids to get a list of supported TV Providers' % mso_id, expected=True)
+                username, password = self._get_login_info('ap_username', 'ap_password', mso_id)
+                if not username or not password:
+                    raise_mvpd_required()
+                mso_info = MSO_INFO[mso_id]
 
-            def post_form(form_page_res, note, data={}):
-                form_page, urlh = form_page_res
-                post_url = self._html_search_regex(r'<form[^>]+action=(["\'])(?P<url>.+?)\1', form_page, 'post url', group='url')
-                if not re.match(r'https?://', post_url):
-                    post_url = compat_urlparse.urljoin(urlh.geturl(), post_url)
-                form_data = self._hidden_inputs(form_page)
-                form_data.update(data)
-                return self._download_webpage_handle(
-                    post_url, video_id, note, data=urlencode_postdata(form_data), headers={
-                        'Content-Type': 'application/x-www-form-urlencoded',
+                provider_redirect_page_res = self._download_webpage_handle(
+                    self._SERVICE_PROVIDER_TEMPLATE % 'authenticate/saml', video_id,
+                    'Downloading Provider Redirect Page', query={
+                        'noflash': 'true',
+                        'mso_id': mso_id,
+                        'requestor_id': requestor_id,
+                        'no_iframe': 'false',
+                        'domain_name': 'adobe.com',
+                        'redirect_url': url,
                     })
-
-            provider_redirect_page_res = self._download_webpage_handle(
-                self._SERVICE_PROVIDER_TEMPLATE % 'authenticate/saml', video_id,
-                'Downloading Provider Redirect Page', query={
-                    'noflash': 'true',
-                    'mso_id': mso_id,
-                    'requestor_id': requestor_id,
-                    'no_iframe': 'false',
-                    'domain_name': 'adobe.com',
-                    'redirect_url': url,
+                provider_login_page_res = post_form(
+                    provider_redirect_page_res, 'Downloading Provider Login Page')
+                mvpd_confirm_page_res = post_form(provider_login_page_res, 'Logging in', {
+                    mso_info['username_field']: username,
+                    mso_info['password_field']: password,
                 })
-            provider_login_page_res = post_form(
-                provider_redirect_page_res, 'Downloading Provider Login Page')
-            login_data = {}
-            if mso_id == 'DTV':
-                login_data = {
-                    'username': username,
-                    'password': password,
-                }
-            elif mso_id == 'Rogers':
-                login_data = {
-                    'UserName': username,
-                    'UserPassword': password,
-                }
-            mvpd_confirm_page_res = post_form(provider_login_page_res, 'Logging in', login_data)
-            if mso_id == 'DTV':
-                post_form(mvpd_confirm_page_res, 'Confirming Login')
+                if mso_id == 'DTV':
+                    post_form(mvpd_confirm_page_res, 'Confirming Login')
 
-            session = self._download_webpage(
-                self._SERVICE_PROVIDER_TEMPLATE % 'session', video_id,
-                'Retrieving Session', data=urlencode_postdata({
-                    '_method': 'GET',
+                session = self._download_webpage(
+                    self._SERVICE_PROVIDER_TEMPLATE % 'session', video_id,
+                    'Retrieving Session', data=urlencode_postdata({
+                        '_method': 'GET',
+                        'requestor_id': requestor_id,
+                    }), headers=mvpd_headers)
+                if '<pendingLogout' in session:
+                    self._downloader.cache.store('mvpd', requestor_id, {})
+                    count += 1
+                    continue
+                authn_token = unescapeHTML(xml_text(session, 'authnToken'))
+                requestor_info['authn_token'] = authn_token
+                self._downloader.cache.store('mvpd', requestor_id, requestor_info)
+
+            authz_token = requestor_info.get(guid)
+            if authz_token and is_expired(authz_token, 'simpleTokenTTL'):
+                authz_token = None
+            if not authz_token:
+                authorize = self._download_webpage(
+                    self._SERVICE_PROVIDER_TEMPLATE % 'authorize', video_id,
+                    'Retrieving Authorization Token', data=urlencode_postdata({
+                        'resource_id': resource,
+                        'requestor_id': requestor_id,
+                        'authentication_token': authn_token,
+                        'mso_id': xml_text(authn_token, 'simpleTokenMsoID'),
+                        'userMeta': '1',
+                    }), headers=mvpd_headers)
+                if '<pendingLogout' in authorize:
+                    self._downloader.cache.store('mvpd', requestor_id, {})
+                    count += 1
+                    continue
+                authz_token = unescapeHTML(xml_text(authorize, 'authzToken'))
+                requestor_info[guid] = authz_token
+                self._downloader.cache.store('mvpd', requestor_id, requestor_info)
+
+            mvpd_headers.update({
+                'ap_19': xml_text(authn_token, 'simpleSamlNameID'),
+                'ap_23': xml_text(authn_token, 'simpleSamlSessionIndex'),
+            })
+
+            short_authorize = self._download_webpage(
+                self._SERVICE_PROVIDER_TEMPLATE % 'shortAuthorize',
+                video_id, 'Retrieving Media Token', data=urlencode_postdata({
+                    'authz_token': authz_token,
                     'requestor_id': requestor_id,
+                    'session_guid': xml_text(authn_token, 'simpleTokenAuthenticationGuid'),
+                    'hashed_guid': 'false',
                 }), headers=mvpd_headers)
-            if '<pendingLogout' in session:
+            if '<pendingLogout' in short_authorize:
                 self._downloader.cache.store('mvpd', requestor_id, {})
-                return self._extract_mvpd_auth(url, video_id, requestor_id, resource)
-            authn_token = unescapeHTML(xml_text(session, 'authnToken'))
-            requestor_info['authn_token'] = authn_token
-            self._downloader.cache.store('mvpd', requestor_id, requestor_info)
-
-        authz_token = requestor_info.get(guid)
-        if authz_token and is_expired(authz_token, 'simpleTokenTTL'):
-            authz_token = None
-        if not authz_token:
-            authorize = self._download_webpage(
-                self._SERVICE_PROVIDER_TEMPLATE % 'authorize', video_id,
-                'Retrieving Authorization Token', data=urlencode_postdata({
-                    'resource_id': resource,
-                    'requestor_id': requestor_id,
-                    'authentication_token': authn_token,
-                    'mso_id': xml_text(authn_token, 'simpleTokenMsoID'),
-                    'userMeta': '1',
-                }), headers=mvpd_headers)
-            if '<pendingLogout' in authorize:
-                self._downloader.cache.store('mvpd', requestor_id, {})
-                return self._extract_mvpd_auth(url, video_id, requestor_id, resource)
-            authz_token = unescapeHTML(xml_text(authorize, 'authzToken'))
-            requestor_info[guid] = authz_token
-            self._downloader.cache.store('mvpd', requestor_id, requestor_info)
-
-        mvpd_headers.update({
-            'ap_19': xml_text(authn_token, 'simpleSamlNameID'),
-            'ap_23': xml_text(authn_token, 'simpleSamlSessionIndex'),
-        })
-
-        short_authorize = self._download_webpage(
-            self._SERVICE_PROVIDER_TEMPLATE % 'shortAuthorize',
-            video_id, 'Retrieving Media Token', data=urlencode_postdata({
-                'authz_token': authz_token,
-                'requestor_id': requestor_id,
-                'session_guid': xml_text(authn_token, 'simpleTokenAuthenticationGuid'),
-                'hashed_guid': 'false',
-            }), headers=mvpd_headers)
-        if '<pendingLogout' in short_authorize:
-            self._downloader.cache.store('mvpd', requestor_id, {})
-            return self._extract_mvpd_auth(url, video_id, requestor_id, resource)
-        return short_authorize
+                count += 1
+                continue
+            return short_authorize
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 6edd5a769..ff19270ae 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -680,7 +680,7 @@ class InfoExtractor(object):
 
         return (username, password)
 
-    def _get_login_info(self):
+    def _get_login_info(self, username_option='username', password_option='password', netrc_machine=None):
         """
         Get the login info as (username, password)
         It will look in the netrc file using the _NETRC_MACHINE value
@@ -694,11 +694,11 @@ class InfoExtractor(object):
         downloader_params = self._downloader.params
 
         # Attempt to use provided username and password or .netrc data
-        if downloader_params.get('username') is not None:
-            username = downloader_params['username']
-            password = downloader_params['password']
+        if downloader_params.get(username_option) is not None:
+            username = downloader_params[username_option]
+            password = downloader_params[password_option]
         else:
-            username, password = self._get_netrc_login_info()
+            username, password = self._get_netrc_login_info(netrc_machine)
 
         return (username, password)
 
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index c4057ce59..b99201a20 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -94,7 +94,7 @@ def parseOpts(overrideArguments=None):
         setattr(parser.values, option.dest, value.split(','))
 
     def _hide_login_info(opts):
-        PRIVATE_OPTS = ['-p', '--password', '-u', '--username', '--video-password']
+        PRIVATE_OPTS = ['-p', '--password', '-u', '--username', '--video-password', '--ap-password', '--ap-username']
         eqre = re.compile('^(?P<key>' + ('|'.join(re.escape(po) for po in PRIVATE_OPTS)) + ')=.+$')
 
         def _scrub_eq(o):
@@ -350,10 +350,28 @@ def parseOpts(overrideArguments=None):
         '--video-password',
         dest='videopassword', metavar='PASSWORD',
         help='Video password (vimeo, smotri, youku)')
-    authentication.add_option(
+
+    adobe_pass = optparse.OptionGroup(parser, 'Adobe Pass Options')
+    adobe_pass.add_option(
         '--ap-mso-id',
         dest='ap_mso_id', metavar='APMSOID',
-        help='Adobe Pass Multiple-system operator Identifier(DTV, Rogers)')
+        help='Adobe Pass Multiple-system operator Identifier')
+    adobe_pass.add_option(
+        '--ap-username',
+        dest='ap_username', metavar='APUSERNAME',
+        help='TV Provider Login with this account ID')
+    adobe_pass.add_option(
+        '--ap-password',
+        dest='ap_password', metavar='APPASSWORD',
+        help='TV Provider Account password. If this option is left out, youtube-dl will ask interactively.')
+    adobe_pass.add_option(
+        '--list-ap-mso-ids',
+        action='store_true', dest='list_ap_mso_ids', default=False,
+        help='List all supported TV Providers')
+    adobe_pass.add_option(
+        '--ap-retries',
+        dest='ap_retries', metavar='APRETRIES', default=3,
+        help='Number of retries for Adobe Pass Authorization requests')
 
     video_format = optparse.OptionGroup(parser, 'Video Format Options')
     video_format.add_option(

From 4875ff68476ff7de9733c80effb652fc6ab07ea0 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 14 Sep 2016 22:01:31 +0800
Subject: [PATCH 6073/6219] [bilibili] Remove copyrighted test cases
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

I can't find any English or Chinese material that claims BiliBili has
bought legal redistribution permissions for copyrighted products from
copyrighted holders.

References for removed test cases:
"刀语": https://en.wikipedia.org/wiki/Katanagatari, by White Fox
"哆啦A梦": https://en.wikipedia.org/wiki/Doraemon, by Shin-Ei Animation
"岳父岳母真难当": https://en.wikipedia.org/wiki/Serial_(Bad)_Weddings, by Les films du 24
"混沌武士": https://en.wikipedia.org/wiki/Samurai_Champloo, by Manglobe

I shouldn't have added them to _TESTS
---
 youtube_dl/extractor/bilibili.py | 61 ++------------------------------
 1 file changed, 2 insertions(+), 59 deletions(-)

diff --git a/youtube_dl/extractor/bilibili.py b/youtube_dl/extractor/bilibili.py
index 8fa96d3a0..9f5c12ab9 100644
--- a/youtube_dl/extractor/bilibili.py
+++ b/youtube_dl/extractor/bilibili.py
@@ -17,7 +17,7 @@ from ..utils import (
 class BiliBiliIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.|bangumi\.|)bilibili\.(?:tv|com)/(?:video/av|anime/v/)(?P<id>\d+)'
 
-    _TESTS = [{
+    _TEST = {
         'url': 'http://www.bilibili.tv/video/av1074402/',
         'md5': '9fa226fe2b8a9a4d5a69b4c6a183417e',
         'info_dict': {
@@ -32,64 +32,7 @@ class BiliBiliIE(InfoExtractor):
             'uploader': '菊子桑',
             'uploader_id': '156160',
         },
-    }, {
-        'url': 'http://www.bilibili.com/video/av1041170/',
-        'info_dict': {
-            'id': '1041170',
-            'ext': 'mp4',
-            'title': '【BD1080P】刀语【诸神&异域】',
-            'description': '这是个神奇的故事~每个人不留弹幕不给走哦~切利哦！~',
-            'duration': 3382.259,
-            'timestamp': 1396530060,
-            'upload_date': '20140403',
-            'thumbnail': 're:^https?://.+\.jpg',
-            'uploader': '枫叶逝去',
-            'uploader_id': '520116',
-        },
-    }, {
-        'url': 'http://www.bilibili.com/video/av4808130/',
-        'info_dict': {
-            'id': '4808130',
-            'ext': 'mp4',
-            'title': '【长篇】哆啦A梦443【钉铛】',
-            'description': '(2016.05.27)来组合客人的脸吧&amp;amp;寻母六千里锭 抱歉，又轮到周日上班现在才到家 封面www.pixiv.net/member_illust.php?mode=medium&amp;amp;illust_id=56912929',
-            'duration': 1493.995,
-            'timestamp': 1464564180,
-            'upload_date': '20160529',
-            'thumbnail': 're:^https?://.+\.jpg',
-            'uploader': '喜欢拉面',
-            'uploader_id': '151066',
-        },
-    }, {
-        # Missing upload time
-        'url': 'http://www.bilibili.com/video/av1867637/',
-        'info_dict': {
-            'id': '1867637',
-            'ext': 'mp4',
-            'title': '【HDTV】【喜剧】岳父岳母真难当 （2014）【法国票房冠军】',
-            'description': '一个信奉天主教的法国旧式传统资产阶级家庭中有四个女儿。三个女儿却分别找了阿拉伯、犹太、中国丈夫，老夫老妻唯独期盼剩下未嫁的小女儿能找一个信奉天主教的法国白人，结果没想到小女儿找了一位非裔黑人……【这次应该不会跳帧了】',
-            'duration': 5760.0,
-            'uploader': '黑夜为猫',
-            'uploader_id': '610729',
-            'thumbnail': 're:^https?://.+\.jpg',
-        },
-        'params': {
-            # Just to test metadata extraction
-            'skip_download': True,
-        },
-        'expected_warnings': ['upload time'],
-    }, {
-        'url': 'http://bangumi.bilibili.com/anime/v/40068',
-        'md5': '08d539a0884f3deb7b698fb13ba69696',
-        'info_dict': {
-            'id': '40068',
-            'ext': 'mp4',
-            'duration': 1402.357,
-            'title': '混沌武士 : 第7集 四面楚歌 A Risky Racket',
-            'description': 'md5:6a9622b911565794c11f25f81d6a97d2',
-            'thumbnail': 're:^http?://.+\.jpg',
-        },
-    }]
+    }
 
     _APP_KEY = '6f90a59ac58a4123'
     _BILIBILI_KEY = '0bfd84cc3940035173f35e6777508326'

From 86d68f906e21a6674f9f8676b22a47414b6c9fd2 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 14 Sep 2016 22:11:49 +0800
Subject: [PATCH 6074/6219] [bilibili] Fix extraction for videos without
 backup_url (#10647)

---
 ChangeLog                        | 1 +
 youtube_dl/extractor/bilibili.py | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index 46eea0626..25c916eb2 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
 version <unreleased>
 
 Extractors
+* [bilibili] Fix extraction for specific videos (#10647)
 + [nbc] Add support for NBC Olympics (#10361)
 
 
diff --git a/youtube_dl/extractor/bilibili.py b/youtube_dl/extractor/bilibili.py
index 9f5c12ab9..2d174e6f9 100644
--- a/youtube_dl/extractor/bilibili.py
+++ b/youtube_dl/extractor/bilibili.py
@@ -67,7 +67,7 @@ class BiliBiliIE(InfoExtractor):
                 'url': durl['url'],
                 'filesize': int_or_none(durl['size']),
             }]
-            for backup_url in durl['backup_url']:
+            for backup_url in durl.get('backup_url', []):
                 formats.append({
                     'url': backup_url,
                     # backup URLs have lower priorities

From 5712c0f42639cd183b0dfbc51482592e790e99d1 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 14 Sep 2016 16:36:42 +0100
Subject: [PATCH 6075/6219] [adobepass] remove unnecessary option

---
 youtube_dl/__init__.py            | 3 ---
 youtube_dl/extractor/adobepass.py | 3 +--
 youtube_dl/options.py             | 4 ----
 3 files changed, 1 insertion(+), 9 deletions(-)

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 052f20ee7..cdff3df65 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -179,8 +179,6 @@ def _real_main(argv=None):
         opts.retries = parse_retries(opts.retries)
     if opts.fragment_retries is not None:
         opts.fragment_retries = parse_retries(opts.fragment_retries)
-    if opts.ap_retries is not None:
-        opts.ap_retries = parse_retries(opts.ap_retries)
     if opts.buffersize is not None:
         numeric_buffersize = FileDownloader.parse_bytes(opts.buffersize)
         if numeric_buffersize is None:
@@ -308,7 +306,6 @@ def _real_main(argv=None):
         'ap_mso_id': opts.ap_mso_id,
         'ap_username': opts.ap_username,
         'ap_password': opts.ap_password,
-        'ap_retries': opts.ap_retries,
         'quiet': (opts.quiet or any_getting or any_printing),
         'no_warnings': opts.no_warnings,
         'forceurl': opts.geturl,
diff --git a/youtube_dl/extractor/adobepass.py b/youtube_dl/extractor/adobepass.py
index 9add6c0f8..913a817d2 100644
--- a/youtube_dl/extractor/adobepass.py
+++ b/youtube_dl/extractor/adobepass.py
@@ -83,9 +83,8 @@ class AdobePassIE(InfoExtractor):
         }
 
         guid = xml_text(resource, 'guid')
-        retries = self._downloader.params.get('ap_retries', 3)
         count = 0
-        while count < retries:
+        while count < 2:
             requestor_info = self._downloader.cache.load('mvpd', requestor_id) or {}
             authn_token = requestor_info.get('authn_token')
             if authn_token and is_expired(authn_token, 'simpleTokenExpires'):
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index b99201a20..342ae3be3 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -368,10 +368,6 @@ def parseOpts(overrideArguments=None):
         '--list-ap-mso-ids',
         action='store_true', dest='list_ap_mso_ids', default=False,
         help='List all supported TV Providers')
-    adobe_pass.add_option(
-        '--ap-retries',
-        dest='ap_retries', metavar='APRETRIES', default=3,
-        help='Number of retries for Adobe Pass Authorization requests')
 
     video_format = optparse.OptionGroup(parser, 'Video Format Options')
     video_format.add_option(

From b690ea15ebe7549854962f02987a8faaa6d41f53 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 14 Sep 2016 22:45:23 +0700
Subject: [PATCH 6076/6219] [viafree] Fix test

---
 youtube_dl/extractor/tvplay.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tvplay.py b/youtube_dl/extractor/tvplay.py
index 58ffc0e6f..3eda0a399 100644
--- a/youtube_dl/extractor/tvplay.py
+++ b/youtube_dl/extractor/tvplay.py
@@ -369,7 +369,7 @@ class ViafreeIE(InfoExtractor):
         'add_ie': [TVPlayIE.ie_key()],
     }, {
         # Different og:image URL schema
-        'url': 'www.viafree.se/program/reality/sommaren-med-youtube-stjarnorna/sasong-1/avsnitt-2',
+        'url': 'http://www.viafree.se/program/reality/sommaren-med-youtube-stjarnorna/sasong-1/avsnitt-2',
         'only_matching': True,
     }, {
         'url': 'http://www.viafree.no/programmer/underholdning/det-beste-vorspielet/sesong-2/episode-1',

From 925194022cd661747771e58bad41e5f7ae118999 Mon Sep 17 00:00:00 2001
From: stepshal <nessento@openmailbox.org>
Date: Thu, 8 Sep 2016 18:29:05 +0700
Subject: [PATCH 6077/6219] Improve some _VALID_URLs

---
 youtube_dl/extractor/abc.py             |  2 +-
 youtube_dl/extractor/aljazeera.py       |  2 +-
 youtube_dl/extractor/azubu.py           |  2 +-
 youtube_dl/extractor/bbc.py             |  2 +-
 youtube_dl/extractor/bpb.py             |  2 +-
 youtube_dl/extractor/camdemy.py         |  2 +-
 youtube_dl/extractor/cbssports.py       |  2 +-
 youtube_dl/extractor/ceskatelevize.py   |  2 +-
 youtube_dl/extractor/chirbit.py         |  2 +-
 youtube_dl/extractor/cmt.py             |  2 +-
 youtube_dl/extractor/criterion.py       |  2 +-
 youtube_dl/extractor/dctp.py            |  2 +-
 youtube_dl/extractor/democracynow.py    |  2 +-
 youtube_dl/extractor/engadget.py        |  2 +-
 youtube_dl/extractor/expotv.py          |  2 +-
 youtube_dl/extractor/freespeech.py      |  2 +-
 youtube_dl/extractor/gamestar.py        |  2 +-
 youtube_dl/extractor/googleplus.py      |  2 +-
 youtube_dl/extractor/goshgay.py         |  2 +-
 youtube_dl/extractor/hark.py            |  2 +-
 youtube_dl/extractor/hotnewhiphop.py    |  2 +-
 youtube_dl/extractor/imdb.py            |  2 +-
 youtube_dl/extractor/karaoketv.py       |  2 +-
 youtube_dl/extractor/kickstarter.py     |  2 +-
 youtube_dl/extractor/kuwo.py            |  8 ++++----
 youtube_dl/extractor/litv.py            |  2 +-
 youtube_dl/extractor/lynda.py           |  2 +-
 youtube_dl/extractor/macgamestore.py    |  2 +-
 youtube_dl/extractor/metacritic.py      |  2 +-
 youtube_dl/extractor/mgtv.py            |  2 +-
 youtube_dl/extractor/ministrygrid.py    |  2 +-
 youtube_dl/extractor/mitele.py          |  2 +-
 youtube_dl/extractor/motorsport.py      |  2 +-
 youtube_dl/extractor/moviezine.py       |  2 +-
 youtube_dl/extractor/myspass.py         |  2 +-
 youtube_dl/extractor/nbc.py             |  6 +++---
 youtube_dl/extractor/ndr.py             |  8 ++++----
 youtube_dl/extractor/nextmedia.py       |  6 +++---
 youtube_dl/extractor/niconico.py        |  2 +-
 youtube_dl/extractor/oktoberfesttv.py   |  2 +-
 youtube_dl/extractor/openload.py        |  2 +-
 youtube_dl/extractor/periscope.py       |  2 +-
 youtube_dl/extractor/playvid.py         |  2 +-
 youtube_dl/extractor/qqmusic.py         |  6 +++---
 youtube_dl/extractor/rottentomatoes.py  |  2 +-
 youtube_dl/extractor/roxwel.py          |  2 +-
 youtube_dl/extractor/rtve.py            |  6 +++---
 youtube_dl/extractor/screenjunkies.py   |  2 +-
 youtube_dl/extractor/senateisvp.py      |  2 +-
 youtube_dl/extractor/slideshare.py      |  2 +-
 youtube_dl/extractor/spiegel.py         |  2 +-
 youtube_dl/extractor/syfy.py            |  2 +-
 youtube_dl/extractor/teachingchannel.py |  2 +-
 youtube_dl/extractor/telecinco.py       |  2 +-
 youtube_dl/extractor/telewebion.py      |  2 +-
 youtube_dl/extractor/theintercept.py    |  2 +-
 youtube_dl/extractor/thescene.py        |  2 +-
 youtube_dl/extractor/tlc.py             |  2 +-
 youtube_dl/extractor/udemy.py           |  2 +-
 youtube_dl/extractor/ustream.py         |  4 ++--
 youtube_dl/extractor/vevo.py            |  4 ++--
 youtube_dl/extractor/videodetective.py  |  2 +-
 youtube_dl/extractor/weiqitv.py         |  2 +-
 youtube_dl/extractor/yam.py             |  2 +-
 youtube_dl/extractor/youtube.py         | 12 ++++++------
 65 files changed, 86 insertions(+), 86 deletions(-)

diff --git a/youtube_dl/extractor/abc.py b/youtube_dl/extractor/abc.py
index 3792bd232..465249bbf 100644
--- a/youtube_dl/extractor/abc.py
+++ b/youtube_dl/extractor/abc.py
@@ -13,7 +13,7 @@ from ..utils import (
 
 class ABCIE(InfoExtractor):
     IE_NAME = 'abc.net.au'
-    _VALID_URL = r'https?://www\.abc\.net\.au/news/(?:[^/]+/){1,2}(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?abc\.net\.au/news/(?:[^/]+/){1,2}(?P<id>\d+)'
 
     _TESTS = [{
         'url': 'http://www.abc.net.au/news/2014-11-05/australia-to-staff-ebola-treatment-centre-in-sierra-leone/5868334',
diff --git a/youtube_dl/extractor/aljazeera.py b/youtube_dl/extractor/aljazeera.py
index b081695d8..388e578d5 100644
--- a/youtube_dl/extractor/aljazeera.py
+++ b/youtube_dl/extractor/aljazeera.py
@@ -4,7 +4,7 @@ from .common import InfoExtractor
 
 
 class AlJazeeraIE(InfoExtractor):
-    _VALID_URL = r'https?://www\.aljazeera\.com/programmes/.*?/(?P<id>[^/]+)\.html'
+    _VALID_URL = r'https?://(?:www\.)?aljazeera\.com/programmes/.*?/(?P<id>[^/]+)\.html'
 
     _TEST = {
         'url': 'http://www.aljazeera.com/programmes/the-slum/2014/08/deliverance-201482883754237240.html',
diff --git a/youtube_dl/extractor/azubu.py b/youtube_dl/extractor/azubu.py
index a813eb429..72e1bd59d 100644
--- a/youtube_dl/extractor/azubu.py
+++ b/youtube_dl/extractor/azubu.py
@@ -103,7 +103,7 @@ class AzubuIE(InfoExtractor):
 
 
 class AzubuLiveIE(InfoExtractor):
-    _VALID_URL = r'https?://www.azubu.tv/(?P<id>[^/]+)$'
+    _VALID_URL = r'https?://(?:www\.)?azubu\.tv/(?P<id>[^/]+)$'
 
     _TEST = {
         'url': 'http://www.azubu.tv/MarsTVMDLen',
diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index deb9cc1c0..b17916137 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -1028,7 +1028,7 @@ class BBCIE(BBCCoUkIE):
 
 
 class BBCCoUkArticleIE(InfoExtractor):
-    _VALID_URL = r'https?://www.bbc.co.uk/programmes/articles/(?P<id>[a-zA-Z0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?bbc\.co\.uk/programmes/articles/(?P<id>[a-zA-Z0-9]+)'
     IE_NAME = 'bbc.co.uk:article'
     IE_DESC = 'BBC articles'
 
diff --git a/youtube_dl/extractor/bpb.py b/youtube_dl/extractor/bpb.py
index 6ad45a1e6..9661ade4f 100644
--- a/youtube_dl/extractor/bpb.py
+++ b/youtube_dl/extractor/bpb.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 class BpbIE(InfoExtractor):
     IE_DESC = 'Bundeszentrale für politische Bildung'
-    _VALID_URL = r'https?://www\.bpb\.de/mediathek/(?P<id>[0-9]+)/'
+    _VALID_URL = r'https?://(?:www\.)?bpb\.de/mediathek/(?P<id>[0-9]+)/'
 
     _TEST = {
         'url': 'http://www.bpb.de/mediathek/297/joachim-gauck-zu-1989-und-die-erinnerung-an-die-ddr',
diff --git a/youtube_dl/extractor/camdemy.py b/youtube_dl/extractor/camdemy.py
index 268c34392..d4e6fbdce 100644
--- a/youtube_dl/extractor/camdemy.py
+++ b/youtube_dl/extractor/camdemy.py
@@ -112,7 +112,7 @@ class CamdemyIE(InfoExtractor):
 
 
 class CamdemyFolderIE(InfoExtractor):
-    _VALID_URL = r'https?://www.camdemy.com/folder/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?camdemy\.com/folder/(?P<id>\d+)'
     _TESTS = [{
         # links with trailing slash
         'url': 'http://www.camdemy.com/folder/450',
diff --git a/youtube_dl/extractor/cbssports.py b/youtube_dl/extractor/cbssports.py
index bf7915626..3a62c840b 100644
--- a/youtube_dl/extractor/cbssports.py
+++ b/youtube_dl/extractor/cbssports.py
@@ -4,7 +4,7 @@ from .cbs import CBSBaseIE
 
 
 class CBSSportsIE(CBSBaseIE):
-    _VALID_URL = r'https?://www\.cbssports\.com/video/player/[^/]+/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?cbssports\.com/video/player/[^/]+/(?P<id>\d+)'
 
     _TESTS = [{
         'url': 'http://www.cbssports.com/video/player/videos/708337219968/0/ben-simmons-the-next-lebron?-not-so-fast',
diff --git a/youtube_dl/extractor/ceskatelevize.py b/youtube_dl/extractor/ceskatelevize.py
index 5a58d1777..87c2e7089 100644
--- a/youtube_dl/extractor/ceskatelevize.py
+++ b/youtube_dl/extractor/ceskatelevize.py
@@ -17,7 +17,7 @@ from ..utils import (
 
 
 class CeskaTelevizeIE(InfoExtractor):
-    _VALID_URL = r'https?://www\.ceskatelevize\.cz/(porady|ivysilani)/(?:[^/]+/)*(?P<id>[^/#?]+)/*(?:[#?].*)?$'
+    _VALID_URL = r'https?://(?:www\.)?ceskatelevize\.cz/(porady|ivysilani)/(?:[^/]+/)*(?P<id>[^/#?]+)/*(?:[#?].*)?$'
     _TESTS = [{
         'url': 'http://www.ceskatelevize.cz/ivysilani/ivysilani/10441294653-hyde-park-civilizace/214411058091220',
         'info_dict': {
diff --git a/youtube_dl/extractor/chirbit.py b/youtube_dl/extractor/chirbit.py
index b43518652..61aed0167 100644
--- a/youtube_dl/extractor/chirbit.py
+++ b/youtube_dl/extractor/chirbit.py
@@ -65,7 +65,7 @@ class ChirbitIE(InfoExtractor):
 
 class ChirbitProfileIE(InfoExtractor):
     IE_NAME = 'chirbit:profile'
-    _VALID_URL = r'https?://(?:www\.)?chirbit.com/(?:rss/)?(?P<id>[^/]+)'
+    _VALID_URL = r'https?://(?:www\.)?chirbit\.com/(?:rss/)?(?P<id>[^/]+)'
     _TEST = {
         'url': 'http://chirbit.com/ScarletBeauty',
         'info_dict': {
diff --git a/youtube_dl/extractor/cmt.py b/youtube_dl/extractor/cmt.py
index f24568dcc..ac3bdfe8f 100644
--- a/youtube_dl/extractor/cmt.py
+++ b/youtube_dl/extractor/cmt.py
@@ -6,7 +6,7 @@ from ..utils import ExtractorError
 
 class CMTIE(MTVIE):
     IE_NAME = 'cmt.com'
-    _VALID_URL = r'https?://www\.cmt\.com/(?:videos|shows)/(?:[^/]+/)*(?P<videoid>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?cmt\.com/(?:videos|shows)/(?:[^/]+/)*(?P<videoid>\d+)'
     _FEED_URL = 'http://www.cmt.com/sitewide/apps/player/embed/rss/'
 
     _TESTS = [{
diff --git a/youtube_dl/extractor/criterion.py b/youtube_dl/extractor/criterion.py
index dedb810a0..ad32673a8 100644
--- a/youtube_dl/extractor/criterion.py
+++ b/youtube_dl/extractor/criterion.py
@@ -7,7 +7,7 @@ from .common import InfoExtractor
 
 
 class CriterionIE(InfoExtractor):
-    _VALID_URL = r'https?://www\.criterion\.com/films/(?P<id>[0-9]+)-.+'
+    _VALID_URL = r'https?://(?:www\.)?criterion\.com/films/(?P<id>[0-9]+)-.+'
     _TEST = {
         'url': 'http://www.criterion.com/films/184-le-samourai',
         'md5': 'bc51beba55685509883a9a7830919ec3',
diff --git a/youtube_dl/extractor/dctp.py b/youtube_dl/extractor/dctp.py
index 9099f5046..a47e04993 100644
--- a/youtube_dl/extractor/dctp.py
+++ b/youtube_dl/extractor/dctp.py
@@ -6,7 +6,7 @@ from ..compat import compat_str
 
 
 class DctpTvIE(InfoExtractor):
-    _VALID_URL = r'https?://www.dctp.tv/(#/)?filme/(?P<id>.+?)/$'
+    _VALID_URL = r'https?://(?:www\.)?dctp\.tv/(#/)?filme/(?P<id>.+?)/$'
     _TEST = {
         'url': 'http://www.dctp.tv/filme/videoinstallation-fuer-eine-kaufhausfassade/',
         'info_dict': {
diff --git a/youtube_dl/extractor/democracynow.py b/youtube_dl/extractor/democracynow.py
index 65a98d789..bdfe638b4 100644
--- a/youtube_dl/extractor/democracynow.py
+++ b/youtube_dl/extractor/democracynow.py
@@ -13,7 +13,7 @@ from ..utils import (
 
 
 class DemocracynowIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?democracynow.org/(?P<id>[^\?]*)'
+    _VALID_URL = r'https?://(?:www\.)?democracynow\.org/(?P<id>[^\?]*)'
     IE_NAME = 'democracynow'
     _TESTS = [{
         'url': 'http://www.democracynow.org/shows/2015/7/3',
diff --git a/youtube_dl/extractor/engadget.py b/youtube_dl/extractor/engadget.py
index a39e9010d..65635c18b 100644
--- a/youtube_dl/extractor/engadget.py
+++ b/youtube_dl/extractor/engadget.py
@@ -4,7 +4,7 @@ from .common import InfoExtractor
 
 
 class EngadgetIE(InfoExtractor):
-    _VALID_URL = r'https?://www.engadget.com/video/(?P<id>[^/?#]+)'
+    _VALID_URL = r'https?://(?:www\.)?engadget\.com/video/(?P<id>[^/?#]+)'
 
     _TESTS = [{
         # video with 5min ID
diff --git a/youtube_dl/extractor/expotv.py b/youtube_dl/extractor/expotv.py
index 971c918a4..ef11962f3 100644
--- a/youtube_dl/extractor/expotv.py
+++ b/youtube_dl/extractor/expotv.py
@@ -8,7 +8,7 @@ from ..utils import (
 
 
 class ExpoTVIE(InfoExtractor):
-    _VALID_URL = r'https?://www\.expotv\.com/videos/[^?#]*/(?P<id>[0-9]+)($|[?#])'
+    _VALID_URL = r'https?://(?:www\.)?expotv\.com/videos/[^?#]*/(?P<id>[0-9]+)($|[?#])'
     _TEST = {
         'url': 'http://www.expotv.com/videos/reviews/3/40/NYX-Butter-lipstick/667916',
         'md5': 'fe1d728c3a813ff78f595bc8b7a707a8',
diff --git a/youtube_dl/extractor/freespeech.py b/youtube_dl/extractor/freespeech.py
index 1477708bb..0a70ca763 100644
--- a/youtube_dl/extractor/freespeech.py
+++ b/youtube_dl/extractor/freespeech.py
@@ -8,7 +8,7 @@ from .common import InfoExtractor
 
 class FreespeechIE(InfoExtractor):
     IE_NAME = 'freespeech.org'
-    _VALID_URL = r'https://www\.freespeech\.org/video/(?P<title>.+)'
+    _VALID_URL = r'https?://(?:www\.)?freespeech\.org/video/(?P<title>.+)'
     _TEST = {
         'add_ie': ['Youtube'],
         'url': 'https://www.freespeech.org/video/obama-romney-campaign-colorado-ahead-debate-0',
diff --git a/youtube_dl/extractor/gamestar.py b/youtube_dl/extractor/gamestar.py
index 341e72733..55a34604a 100644
--- a/youtube_dl/extractor/gamestar.py
+++ b/youtube_dl/extractor/gamestar.py
@@ -9,7 +9,7 @@ from ..utils import (
 
 
 class GameStarIE(InfoExtractor):
-    _VALID_URL = r'https?://www\.gamestar\.de/videos/.*,(?P<id>[0-9]+)\.html'
+    _VALID_URL = r'https?://(?:www\.)?gamestar\.de/videos/.*,(?P<id>[0-9]+)\.html'
     _TEST = {
         'url': 'http://www.gamestar.de/videos/trailer,3/hobbit-3-die-schlacht-der-fuenf-heere,76110.html',
         'md5': '96974ecbb7fd8d0d20fca5a00810cea7',
diff --git a/youtube_dl/extractor/googleplus.py b/youtube_dl/extractor/googleplus.py
index 731bacd67..427499b11 100644
--- a/youtube_dl/extractor/googleplus.py
+++ b/youtube_dl/extractor/googleplus.py
@@ -10,7 +10,7 @@ from ..utils import unified_strdate
 
 class GooglePlusIE(InfoExtractor):
     IE_DESC = 'Google Plus'
-    _VALID_URL = r'https://plus\.google\.com/(?:[^/]+/)*?posts/(?P<id>\w+)'
+    _VALID_URL = r'https?://plus\.google\.com/(?:[^/]+/)*?posts/(?P<id>\w+)'
     IE_NAME = 'plus.google'
     _TEST = {
         'url': 'https://plus.google.com/u/0/108897254135232129896/posts/ZButuJc6CtH',
diff --git a/youtube_dl/extractor/goshgay.py b/youtube_dl/extractor/goshgay.py
index 0c015141f..a43abd154 100644
--- a/youtube_dl/extractor/goshgay.py
+++ b/youtube_dl/extractor/goshgay.py
@@ -11,7 +11,7 @@ from ..utils import (
 
 
 class GoshgayIE(InfoExtractor):
-    _VALID_URL = r'https?://www\.goshgay\.com/video(?P<id>\d+?)($|/)'
+    _VALID_URL = r'https?://(?:www\.)?goshgay\.com/video(?P<id>\d+?)($|/)'
     _TEST = {
         'url': 'http://www.goshgay.com/video299069/diesel_sfw_xxx_video',
         'md5': '4b6db9a0a333142eb9f15913142b0ed1',
diff --git a/youtube_dl/extractor/hark.py b/youtube_dl/extractor/hark.py
index b6cc15b6f..749e9154f 100644
--- a/youtube_dl/extractor/hark.py
+++ b/youtube_dl/extractor/hark.py
@@ -5,7 +5,7 @@ from .common import InfoExtractor
 
 
 class HarkIE(InfoExtractor):
-    _VALID_URL = r'https?://www\.hark\.com/clips/(?P<id>.+?)-.+'
+    _VALID_URL = r'https?://(?:www\.)?hark\.com/clips/(?P<id>.+?)-.+'
     _TEST = {
         'url': 'http://www.hark.com/clips/mmbzyhkgny-obama-beyond-the-afghan-theater-we-only-target-al-qaeda-on-may-23-2013',
         'md5': '6783a58491b47b92c7c1af5a77d4cbee',
diff --git a/youtube_dl/extractor/hotnewhiphop.py b/youtube_dl/extractor/hotnewhiphop.py
index 9db565209..34163725f 100644
--- a/youtube_dl/extractor/hotnewhiphop.py
+++ b/youtube_dl/extractor/hotnewhiphop.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 
 class HotNewHipHopIE(InfoExtractor):
-    _VALID_URL = r'https?://www\.hotnewhiphop\.com/.*\.(?P<id>.*)\.html'
+    _VALID_URL = r'https?://(?:www\.)?hotnewhiphop\.com/.*\.(?P<id>.*)\.html'
     _TEST = {
         'url': 'http://www.hotnewhiphop.com/freddie-gibbs-lay-it-down-song.1435540.html',
         'md5': '2c2cd2f76ef11a9b3b581e8b232f3d96',
diff --git a/youtube_dl/extractor/imdb.py b/youtube_dl/extractor/imdb.py
index 3a6a6f5ad..f0fc8d49a 100644
--- a/youtube_dl/extractor/imdb.py
+++ b/youtube_dl/extractor/imdb.py
@@ -94,7 +94,7 @@ class ImdbIE(InfoExtractor):
 class ImdbListIE(InfoExtractor):
     IE_NAME = 'imdb:list'
     IE_DESC = 'Internet Movie Database lists'
-    _VALID_URL = r'https?://www\.imdb\.com/list/(?P<id>[\da-zA-Z_-]{11})'
+    _VALID_URL = r'https?://(?:www\.)?imdb\.com/list/(?P<id>[\da-zA-Z_-]{11})'
     _TEST = {
         'url': 'http://www.imdb.com/list/JFs9NWw6XI0',
         'info_dict': {
diff --git a/youtube_dl/extractor/karaoketv.py b/youtube_dl/extractor/karaoketv.py
index bad46005b..bfccf89b0 100644
--- a/youtube_dl/extractor/karaoketv.py
+++ b/youtube_dl/extractor/karaoketv.py
@@ -5,7 +5,7 @@ from .common import InfoExtractor
 
 
 class KaraoketvIE(InfoExtractor):
-    _VALID_URL = r'https?://www\.karaoketv\.co\.il/[^/]+/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?karaoketv\.co\.il/[^/]+/(?P<id>\d+)'
     _TEST = {
         'url': 'http://www.karaoketv.co.il/%D7%A9%D7%99%D7%A8%D7%99_%D7%A7%D7%A8%D7%99%D7%95%D7%A7%D7%99/58356/%D7%90%D7%99%D7%96%D7%95%D7%9F',
         'info_dict': {
diff --git a/youtube_dl/extractor/kickstarter.py b/youtube_dl/extractor/kickstarter.py
index c61e78622..fbe499497 100644
--- a/youtube_dl/extractor/kickstarter.py
+++ b/youtube_dl/extractor/kickstarter.py
@@ -6,7 +6,7 @@ from ..utils import smuggle_url
 
 
 class KickStarterIE(InfoExtractor):
-    _VALID_URL = r'https?://www\.kickstarter\.com/projects/(?P<id>[^/]*)/.*'
+    _VALID_URL = r'https?://(?:www\.)?kickstarter\.com/projects/(?P<id>[^/]*)/.*'
     _TESTS = [{
         'url': 'https://www.kickstarter.com/projects/1404461844/intersection-the-story-of-josh-grant/description',
         'md5': 'c81addca81327ffa66c642b5d8b08cab',
diff --git a/youtube_dl/extractor/kuwo.py b/youtube_dl/extractor/kuwo.py
index 0eeb9ffeb..ba621ca7b 100644
--- a/youtube_dl/extractor/kuwo.py
+++ b/youtube_dl/extractor/kuwo.py
@@ -59,7 +59,7 @@ class KuwoBaseIE(InfoExtractor):
 class KuwoIE(KuwoBaseIE):
     IE_NAME = 'kuwo:song'
     IE_DESC = '酷我音乐'
-    _VALID_URL = r'https?://www\.kuwo\.cn/yinyue/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?kuwo\.cn/yinyue/(?P<id>\d+)'
     _TESTS = [{
         'url': 'http://www.kuwo.cn/yinyue/635632/',
         'info_dict': {
@@ -139,7 +139,7 @@ class KuwoIE(KuwoBaseIE):
 class KuwoAlbumIE(InfoExtractor):
     IE_NAME = 'kuwo:album'
     IE_DESC = '酷我音乐 - 专辑'
-    _VALID_URL = r'https?://www\.kuwo\.cn/album/(?P<id>\d+?)/'
+    _VALID_URL = r'https?://(?:www\.)?kuwo\.cn/album/(?P<id>\d+?)/'
     _TEST = {
         'url': 'http://www.kuwo.cn/album/502294/',
         'info_dict': {
@@ -200,7 +200,7 @@ class KuwoChartIE(InfoExtractor):
 class KuwoSingerIE(InfoExtractor):
     IE_NAME = 'kuwo:singer'
     IE_DESC = '酷我音乐 - 歌手'
-    _VALID_URL = r'https?://www\.kuwo\.cn/mingxing/(?P<id>[^/]+)'
+    _VALID_URL = r'https?://(?:www\.)?kuwo\.cn/mingxing/(?P<id>[^/]+)'
     _TESTS = [{
         'url': 'http://www.kuwo.cn/mingxing/bruno+mars/',
         'info_dict': {
@@ -296,7 +296,7 @@ class KuwoCategoryIE(InfoExtractor):
 class KuwoMvIE(KuwoBaseIE):
     IE_NAME = 'kuwo:mv'
     IE_DESC = '酷我音乐 - MV'
-    _VALID_URL = r'https?://www\.kuwo\.cn/mv/(?P<id>\d+?)/'
+    _VALID_URL = r'https?://(?:www\.)?kuwo\.cn/mv/(?P<id>\d+?)/'
     _TEST = {
         'url': 'http://www.kuwo.cn/mv/6480076/',
         'info_dict': {
diff --git a/youtube_dl/extractor/litv.py b/youtube_dl/extractor/litv.py
index 05c6579f1..a3784e6c6 100644
--- a/youtube_dl/extractor/litv.py
+++ b/youtube_dl/extractor/litv.py
@@ -14,7 +14,7 @@ from ..utils import (
 
 
 class LiTVIE(InfoExtractor):
-    _VALID_URL = r'https?://www\.litv\.tv/(?:vod|promo)/[^/]+/(?:content\.do)?\?.*?\b(?:content_)?id=(?P<id>[^&]+)'
+    _VALID_URL = r'https?://(?:www\.)?litv\.tv/(?:vod|promo)/[^/]+/(?:content\.do)?\?.*?\b(?:content_)?id=(?P<id>[^&]+)'
 
     _URL_TEMPLATE = 'https://www.litv.tv/vod/%s/content.do?id=%s'
 
diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index a98c4c530..299873ecc 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -94,7 +94,7 @@ class LyndaBaseIE(InfoExtractor):
 class LyndaIE(LyndaBaseIE):
     IE_NAME = 'lynda'
     IE_DESC = 'lynda.com videos'
-    _VALID_URL = r'https?://www\.lynda\.com/(?:[^/]+/[^/]+/\d+|player/embed)/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?lynda\.com/(?:[^/]+/[^/]+/\d+|player/embed)/(?P<id>\d+)'
 
     _TIMECODE_REGEX = r'\[(?P<timecode>\d+:\d+:\d+[\.,]\d+)\]'
 
diff --git a/youtube_dl/extractor/macgamestore.py b/youtube_dl/extractor/macgamestore.py
index 3cd4a3a19..43db9929c 100644
--- a/youtube_dl/extractor/macgamestore.py
+++ b/youtube_dl/extractor/macgamestore.py
@@ -7,7 +7,7 @@ from ..utils import ExtractorError
 class MacGameStoreIE(InfoExtractor):
     IE_NAME = 'macgamestore'
     IE_DESC = 'MacGameStore trailers'
-    _VALID_URL = r'https?://www\.macgamestore\.com/mediaviewer\.php\?trailer=(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?macgamestore\.com/mediaviewer\.php\?trailer=(?P<id>\d+)'
 
     _TEST = {
         'url': 'http://www.macgamestore.com/mediaviewer.php?trailer=2450',
diff --git a/youtube_dl/extractor/metacritic.py b/youtube_dl/extractor/metacritic.py
index 444ec0310..7d468d78b 100644
--- a/youtube_dl/extractor/metacritic.py
+++ b/youtube_dl/extractor/metacritic.py
@@ -9,7 +9,7 @@ from ..utils import (
 
 
 class MetacriticIE(InfoExtractor):
-    _VALID_URL = r'https?://www\.metacritic\.com/.+?/trailers/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?metacritic\.com/.+?/trailers/(?P<id>\d+)'
 
     _TESTS = [{
         'url': 'http://www.metacritic.com/game/playstation-4/infamous-second-son/trailers/3698222',
diff --git a/youtube_dl/extractor/mgtv.py b/youtube_dl/extractor/mgtv.py
index 27bdff8b2..e0bb5d208 100644
--- a/youtube_dl/extractor/mgtv.py
+++ b/youtube_dl/extractor/mgtv.py
@@ -6,7 +6,7 @@ from ..utils import int_or_none
 
 
 class MGTVIE(InfoExtractor):
-    _VALID_URL = r'https?://www\.mgtv\.com/v/(?:[^/]+/)*(?P<id>\d+)\.html'
+    _VALID_URL = r'https?://(?:www\.)?mgtv\.com/v/(?:[^/]+/)*(?P<id>\d+)\.html'
     IE_DESC = '芒果TV'
 
     _TESTS = [{
diff --git a/youtube_dl/extractor/ministrygrid.py b/youtube_dl/extractor/ministrygrid.py
index e48eba3fa..10190d5f6 100644
--- a/youtube_dl/extractor/ministrygrid.py
+++ b/youtube_dl/extractor/ministrygrid.py
@@ -8,7 +8,7 @@ from ..utils import (
 
 
 class MinistryGridIE(InfoExtractor):
-    _VALID_URL = r'https?://www\.ministrygrid.com/([^/?#]*/)*(?P<id>[^/#?]+)/?(?:$|[?#])'
+    _VALID_URL = r'https?://(?:www\.)?ministrygrid\.com/([^/?#]*/)*(?P<id>[^/#?]+)/?(?:$|[?#])'
 
     _TEST = {
         'url': 'http://www.ministrygrid.com/training-viewer/-/training/t4g-2014-conference/the-gospel-by-numbers-4/the-gospel-by-numbers',
diff --git a/youtube_dl/extractor/mitele.py b/youtube_dl/extractor/mitele.py
index cd169f361..2294745d4 100644
--- a/youtube_dl/extractor/mitele.py
+++ b/youtube_dl/extractor/mitele.py
@@ -74,7 +74,7 @@ class MiTeleBaseIE(InfoExtractor):
 
 class MiTeleIE(MiTeleBaseIE):
     IE_DESC = 'mitele.es'
-    _VALID_URL = r'https?://www\.mitele\.es/(?:[^/]+/){3}(?P<id>[^/]+)/'
+    _VALID_URL = r'https?://(?:www\.)?mitele\.es/(?:[^/]+/){3}(?P<id>[^/]+)/'
 
     _TESTS = [{
         'url': 'http://www.mitele.es/programas-tv/diario-de/la-redaccion/programa-144/',
diff --git a/youtube_dl/extractor/motorsport.py b/youtube_dl/extractor/motorsport.py
index 370328b36..c9d1ab64d 100644
--- a/youtube_dl/extractor/motorsport.py
+++ b/youtube_dl/extractor/motorsport.py
@@ -9,7 +9,7 @@ from ..compat import (
 
 class MotorsportIE(InfoExtractor):
     IE_DESC = 'motorsport.com'
-    _VALID_URL = r'https?://www\.motorsport\.com/[^/?#]+/video/(?:[^/?#]+/)(?P<id>[^/]+)/?(?:$|[?#])'
+    _VALID_URL = r'https?://(?:www\.)?motorsport\.com/[^/?#]+/video/(?:[^/?#]+/)(?P<id>[^/]+)/?(?:$|[?#])'
     _TEST = {
         'url': 'http://www.motorsport.com/f1/video/main-gallery/red-bull-racing-2014-rules-explained/',
         'info_dict': {
diff --git a/youtube_dl/extractor/moviezine.py b/youtube_dl/extractor/moviezine.py
index f130b75c4..aa091a62c 100644
--- a/youtube_dl/extractor/moviezine.py
+++ b/youtube_dl/extractor/moviezine.py
@@ -7,7 +7,7 @@ from .common import InfoExtractor
 
 
 class MoviezineIE(InfoExtractor):
-    _VALID_URL = r'https?://www\.moviezine\.se/video/(?P<id>[^?#]+)'
+    _VALID_URL = r'https?://(?:www\.)?moviezine\.se/video/(?P<id>[^?#]+)'
 
     _TEST = {
         'url': 'http://www.moviezine.se/video/205866',
diff --git a/youtube_dl/extractor/myspass.py b/youtube_dl/extractor/myspass.py
index 1ca7b1a9e..2afe535b5 100644
--- a/youtube_dl/extractor/myspass.py
+++ b/youtube_dl/extractor/myspass.py
@@ -11,7 +11,7 @@ from ..utils import (
 
 
 class MySpassIE(InfoExtractor):
-    _VALID_URL = r'https?://www\.myspass\.de/.*'
+    _VALID_URL = r'https?://(?:www\.)?myspass\.de/.*'
     _TEST = {
         'url': 'http://www.myspass.de/myspass/shows/tvshows/absolute-mehrheit/Absolute-Mehrheit-vom-17022013-Die-Highlights-Teil-2--/11741/',
         'md5': '0b49f4844a068f8b33f4b7c88405862b',
diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index f37bf2f30..7f1bd9229 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -13,7 +13,7 @@ from ..utils import (
 
 
 class NBCIE(InfoExtractor):
-    _VALID_URL = r'https?://www\.nbc\.com/(?:[^/]+/)+(?P<id>n?\d+)'
+    _VALID_URL = r'https?://(?:www\.)?nbc\.com/(?:[^/]+/)+(?P<id>n?\d+)'
 
     _TESTS = [
         {
@@ -138,7 +138,7 @@ class NBCSportsVPlayerIE(InfoExtractor):
 
 class NBCSportsIE(InfoExtractor):
     # Does not include https because its certificate is invalid
-    _VALID_URL = r'https?://www\.nbcsports\.com//?(?:[^/]+/)+(?P<id>[0-9a-z-]+)'
+    _VALID_URL = r'https?://(?:www\.)?nbcsports\.com//?(?:[^/]+/)+(?P<id>[0-9a-z-]+)'
 
     _TEST = {
         'url': 'http://www.nbcsports.com//college-basketball/ncaab/tom-izzo-michigan-st-has-so-much-respect-duke',
@@ -161,7 +161,7 @@ class NBCSportsIE(InfoExtractor):
 
 
 class CSNNEIE(InfoExtractor):
-    _VALID_URL = r'https?://www\.csnne\.com/video/(?P<id>[0-9a-z-]+)'
+    _VALID_URL = r'https?://(?:www\.)?csnne\.com/video/(?P<id>[0-9a-z-]+)'
 
     _TEST = {
         'url': 'http://www.csnne.com/video/snc-evening-update-wright-named-red-sox-no-5-starter',
diff --git a/youtube_dl/extractor/ndr.py b/youtube_dl/extractor/ndr.py
index 0cded6b5c..e3b0da2e9 100644
--- a/youtube_dl/extractor/ndr.py
+++ b/youtube_dl/extractor/ndr.py
@@ -23,7 +23,7 @@ class NDRBaseIE(InfoExtractor):
 class NDRIE(NDRBaseIE):
     IE_NAME = 'ndr'
     IE_DESC = 'NDR.de - Norddeutscher Rundfunk'
-    _VALID_URL = r'https?://www\.ndr\.de/(?:[^/]+/)*(?P<id>[^/?#]+),[\da-z]+\.html'
+    _VALID_URL = r'https?://(?:www\.)?ndr\.de/(?:[^/]+/)*(?P<id>[^/?#]+),[\da-z]+\.html'
     _TESTS = [{
         # httpVideo, same content id
         'url': 'http://www.ndr.de/fernsehen/Party-Poette-und-Parade,hafengeburtstag988.html',
@@ -105,7 +105,7 @@ class NDRIE(NDRBaseIE):
 class NJoyIE(NDRBaseIE):
     IE_NAME = 'njoy'
     IE_DESC = 'N-JOY'
-    _VALID_URL = r'https?://www\.n-joy\.de/(?:[^/]+/)*(?:(?P<display_id>[^/?#]+),)?(?P<id>[\da-z]+)\.html'
+    _VALID_URL = r'https?://(?:www\.)?n-joy\.de/(?:[^/]+/)*(?:(?P<display_id>[^/?#]+),)?(?P<id>[\da-z]+)\.html'
     _TESTS = [{
         # httpVideo, same content id
         'url': 'http://www.n-joy.de/entertainment/comedy/comedy_contest/Benaissa-beim-NDR-Comedy-Contest,comedycontest2480.html',
@@ -238,7 +238,7 @@ class NDREmbedBaseIE(InfoExtractor):
 
 class NDREmbedIE(NDREmbedBaseIE):
     IE_NAME = 'ndr:embed'
-    _VALID_URL = r'https?://www\.ndr\.de/(?:[^/]+/)*(?P<id>[\da-z]+)-(?:player|externalPlayer)\.html'
+    _VALID_URL = r'https?://(?:www\.)?ndr\.de/(?:[^/]+/)*(?P<id>[\da-z]+)-(?:player|externalPlayer)\.html'
     _TESTS = [{
         'url': 'http://www.ndr.de/fernsehen/sendungen/ndr_aktuell/ndraktuell28488-player.html',
         'md5': '8b9306142fe65bbdefb5ce24edb6b0a9',
@@ -332,7 +332,7 @@ class NDREmbedIE(NDREmbedBaseIE):
 
 class NJoyEmbedIE(NDREmbedBaseIE):
     IE_NAME = 'njoy:embed'
-    _VALID_URL = r'https?://www\.n-joy\.de/(?:[^/]+/)*(?P<id>[\da-z]+)-(?:player|externalPlayer)_[^/]+\.html'
+    _VALID_URL = r'https?://(?:www\.)?n-joy\.de/(?:[^/]+/)*(?P<id>[\da-z]+)-(?:player|externalPlayer)_[^/]+\.html'
     _TESTS = [{
         # httpVideo
         'url': 'http://www.n-joy.de/events/reeperbahnfestival/doku948-player_image-bc168e87-5263-4d6d-bd27-bb643005a6de_theme-n-joy.html',
diff --git a/youtube_dl/extractor/nextmedia.py b/youtube_dl/extractor/nextmedia.py
index aae7aeeeb..a08e48c4b 100644
--- a/youtube_dl/extractor/nextmedia.py
+++ b/youtube_dl/extractor/nextmedia.py
@@ -7,7 +7,7 @@ from ..utils import parse_iso8601
 
 class NextMediaIE(InfoExtractor):
     IE_DESC = '蘋果日報'
-    _VALID_URL = r'https?://hk.apple.nextmedia.com/[^/]+/[^/]+/(?P<date>\d+)/(?P<id>\d+)'
+    _VALID_URL = r'https?://hk\.apple\.nextmedia\.com/[^/]+/[^/]+/(?P<date>\d+)/(?P<id>\d+)'
     _TESTS = [{
         'url': 'http://hk.apple.nextmedia.com/realtime/news/20141108/53109199',
         'md5': 'dff9fad7009311c421176d1ac90bfe4f',
@@ -68,7 +68,7 @@ class NextMediaIE(InfoExtractor):
 
 class NextMediaActionNewsIE(NextMediaIE):
     IE_DESC = '蘋果日報 - 動新聞'
-    _VALID_URL = r'https?://hk.dv.nextmedia.com/actionnews/[^/]+/(?P<date>\d+)/(?P<id>\d+)/\d+'
+    _VALID_URL = r'https?://hk\.dv\.nextmedia\.com/actionnews/[^/]+/(?P<date>\d+)/(?P<id>\d+)/\d+'
     _TESTS = [{
         'url': 'http://hk.dv.nextmedia.com/actionnews/hit/20150121/19009428/20061460',
         'md5': '05fce8ffeed7a5e00665d4b7cf0f9201',
@@ -93,7 +93,7 @@ class NextMediaActionNewsIE(NextMediaIE):
 
 class AppleDailyIE(NextMediaIE):
     IE_DESC = '臺灣蘋果日報'
-    _VALID_URL = r'https?://(www|ent).appledaily.com.tw/(?:animation|appledaily|enews|realtimenews)/[^/]+/[^/]+/(?P<date>\d+)/(?P<id>\d+)(/.*)?'
+    _VALID_URL = r'https?://(www|ent)\.appledaily\.com\.tw/(?:animation|appledaily|enews|realtimenews)/[^/]+/[^/]+/(?P<date>\d+)/(?P<id>\d+)(/.*)?'
     _TESTS = [{
         'url': 'http://ent.appledaily.com.tw/enews/article/entertainment/20150128/36354694',
         'md5': 'a843ab23d150977cc55ef94f1e2c1e4d',
diff --git a/youtube_dl/extractor/niconico.py b/youtube_dl/extractor/niconico.py
index dd75a48af..6eaaa8416 100644
--- a/youtube_dl/extractor/niconico.py
+++ b/youtube_dl/extractor/niconico.py
@@ -252,7 +252,7 @@ class NiconicoIE(InfoExtractor):
 
 
 class NiconicoPlaylistIE(InfoExtractor):
-    _VALID_URL = r'https?://www\.nicovideo\.jp/mylist/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?nicovideo\.jp/mylist/(?P<id>\d+)'
 
     _TEST = {
         'url': 'http://www.nicovideo.jp/mylist/27411728',
diff --git a/youtube_dl/extractor/oktoberfesttv.py b/youtube_dl/extractor/oktoberfesttv.py
index 4a41c0542..f2ccc53dc 100644
--- a/youtube_dl/extractor/oktoberfesttv.py
+++ b/youtube_dl/extractor/oktoberfesttv.py
@@ -5,7 +5,7 @@ from .common import InfoExtractor
 
 
 class OktoberfestTVIE(InfoExtractor):
-    _VALID_URL = r'https?://www\.oktoberfest-tv\.de/[^/]+/[^/]+/video/(?P<id>[^/?#]+)'
+    _VALID_URL = r'https?://(?:www\.)?oktoberfest-tv\.de/[^/]+/[^/]+/video/(?P<id>[^/?#]+)'
 
     _TEST = {
         'url': 'http://www.oktoberfest-tv.de/de/kameras/video/hb-zelt',
diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py
index 76316ca2f..c261a7455 100644
--- a/youtube_dl/extractor/openload.py
+++ b/youtube_dl/extractor/openload.py
@@ -13,7 +13,7 @@ from ..utils import (
 
 
 class OpenloadIE(InfoExtractor):
-    _VALID_URL = r'https://openload.(?:co|io)/(?:f|embed)/(?P<id>[a-zA-Z0-9-_]+)'
+    _VALID_URL = r'https?://openload\.(?:co|io)/(?:f|embed)/(?P<id>[a-zA-Z0-9-_]+)'
 
     _TESTS = [{
         'url': 'https://openload.co/f/kUEfGclsU9o',
diff --git a/youtube_dl/extractor/periscope.py b/youtube_dl/extractor/periscope.py
index 6c640089d..eb1aeba46 100644
--- a/youtube_dl/extractor/periscope.py
+++ b/youtube_dl/extractor/periscope.py
@@ -94,7 +94,7 @@ class PeriscopeIE(PeriscopeBaseIE):
 
 
 class PeriscopeUserIE(PeriscopeBaseIE):
-    _VALID_URL = r'https?://www\.periscope\.tv/(?P<id>[^/]+)/?$'
+    _VALID_URL = r'https?://(?:www\.)?periscope\.tv/(?P<id>[^/]+)/?$'
     IE_DESC = 'Periscope user videos'
     IE_NAME = 'periscope:user'
 
diff --git a/youtube_dl/extractor/playvid.py b/youtube_dl/extractor/playvid.py
index 78d219299..79c2db085 100644
--- a/youtube_dl/extractor/playvid.py
+++ b/youtube_dl/extractor/playvid.py
@@ -14,7 +14,7 @@ from ..utils import (
 
 
 class PlayvidIE(InfoExtractor):
-    _VALID_URL = r'https?://www\.playvid\.com/watch(\?v=|/)(?P<id>.+?)(?:#|$)'
+    _VALID_URL = r'https?://(?:www\.)?playvid\.com/watch(\?v=|/)(?P<id>.+?)(?:#|$)'
     _TESTS = [{
         'url': 'http://www.playvid.com/watch/RnmBNgtrrJu',
         'md5': 'ffa2f6b2119af359f544388d8c01eb6c',
diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index ff0af9543..37cb9e2c9 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -18,7 +18,7 @@ from ..utils import (
 class QQMusicIE(InfoExtractor):
     IE_NAME = 'qqmusic'
     IE_DESC = 'QQ音乐'
-    _VALID_URL = r'https?://y.qq.com/#type=song&mid=(?P<id>[0-9A-Za-z]+)'
+    _VALID_URL = r'https?://y\.qq\.com/#type=song&mid=(?P<id>[0-9A-Za-z]+)'
     _TESTS = [{
         'url': 'http://y.qq.com/#type=song&mid=004295Et37taLD',
         'md5': '9ce1c1c8445f561506d2e3cfb0255705',
@@ -172,7 +172,7 @@ class QQPlaylistBaseIE(InfoExtractor):
 class QQMusicSingerIE(QQPlaylistBaseIE):
     IE_NAME = 'qqmusic:singer'
     IE_DESC = 'QQ音乐 - 歌手'
-    _VALID_URL = r'https?://y.qq.com/#type=singer&mid=(?P<id>[0-9A-Za-z]+)'
+    _VALID_URL = r'https?://y\.qq\.com/#type=singer&mid=(?P<id>[0-9A-Za-z]+)'
     _TEST = {
         'url': 'http://y.qq.com/#type=singer&mid=001BLpXF2DyJe2',
         'info_dict': {
@@ -217,7 +217,7 @@ class QQMusicSingerIE(QQPlaylistBaseIE):
 class QQMusicAlbumIE(QQPlaylistBaseIE):
     IE_NAME = 'qqmusic:album'
     IE_DESC = 'QQ音乐 - 专辑'
-    _VALID_URL = r'https?://y.qq.com/#type=album&mid=(?P<id>[0-9A-Za-z]+)'
+    _VALID_URL = r'https?://y\.qq\.com/#type=album&mid=(?P<id>[0-9A-Za-z]+)'
 
     _TESTS = [{
         'url': 'http://y.qq.com/#type=album&mid=000gXCTb2AhRR1',
diff --git a/youtube_dl/extractor/rottentomatoes.py b/youtube_dl/extractor/rottentomatoes.py
index 23abf7a27..1d404d20a 100644
--- a/youtube_dl/extractor/rottentomatoes.py
+++ b/youtube_dl/extractor/rottentomatoes.py
@@ -5,7 +5,7 @@ from .internetvideoarchive import InternetVideoArchiveIE
 
 
 class RottenTomatoesIE(InfoExtractor):
-    _VALID_URL = r'https?://www\.rottentomatoes\.com/m/[^/]+/trailers/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?rottentomatoes\.com/m/[^/]+/trailers/(?P<id>\d+)'
 
     _TEST = {
         'url': 'http://www.rottentomatoes.com/m/toy_story_3/trailers/11028566/',
diff --git a/youtube_dl/extractor/roxwel.py b/youtube_dl/extractor/roxwel.py
index 41638c1d0..65284643b 100644
--- a/youtube_dl/extractor/roxwel.py
+++ b/youtube_dl/extractor/roxwel.py
@@ -7,7 +7,7 @@ from ..utils import unified_strdate, determine_ext
 
 
 class RoxwelIE(InfoExtractor):
-    _VALID_URL = r'https?://www\.roxwel\.com/player/(?P<filename>.+?)(\.|\?|$)'
+    _VALID_URL = r'https?://(?:www\.)?roxwel\.com/player/(?P<filename>.+?)(\.|\?|$)'
 
     _TEST = {
         'url': 'http://www.roxwel.com/player/passionpittakeawalklive.html',
diff --git a/youtube_dl/extractor/rtve.py b/youtube_dl/extractor/rtve.py
index 34f9c4a99..f1b92f6da 100644
--- a/youtube_dl/extractor/rtve.py
+++ b/youtube_dl/extractor/rtve.py
@@ -64,7 +64,7 @@ def _decrypt_url(png):
 class RTVEALaCartaIE(InfoExtractor):
     IE_NAME = 'rtve.es:alacarta'
     IE_DESC = 'RTVE a la carta'
-    _VALID_URL = r'https?://www\.rtve\.es/(m/)?(alacarta/videos|filmoteca)/[^/]+/[^/]+/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?rtve\.es/(m/)?(alacarta/videos|filmoteca)/[^/]+/[^/]+/(?P<id>\d+)'
 
     _TESTS = [{
         'url': 'http://www.rtve.es/alacarta/videos/balonmano/o-swiss-cup-masculina-final-espana-suecia/2491869/',
@@ -184,7 +184,7 @@ class RTVEInfantilIE(InfoExtractor):
 class RTVELiveIE(InfoExtractor):
     IE_NAME = 'rtve.es:live'
     IE_DESC = 'RTVE.es live streams'
-    _VALID_URL = r'https?://www\.rtve\.es/directo/(?P<id>[a-zA-Z0-9-]+)'
+    _VALID_URL = r'https?://(?:www\.)?rtve\.es/directo/(?P<id>[a-zA-Z0-9-]+)'
 
     _TESTS = [{
         'url': 'http://www.rtve.es/directo/la-1/',
@@ -226,7 +226,7 @@ class RTVELiveIE(InfoExtractor):
 
 class RTVETelevisionIE(InfoExtractor):
     IE_NAME = 'rtve.es:television'
-    _VALID_URL = r'https?://www\.rtve\.es/television/[^/]+/[^/]+/(?P<id>\d+).shtml'
+    _VALID_URL = r'https?://(?:www\.)?rtve\.es/television/[^/]+/[^/]+/(?P<id>\d+).shtml'
 
     _TEST = {
         'url': 'http://www.rtve.es/television/20160628/revolucion-del-movil/1364141.shtml',
diff --git a/youtube_dl/extractor/screenjunkies.py b/youtube_dl/extractor/screenjunkies.py
index dd0a6ba19..02e574cd8 100644
--- a/youtube_dl/extractor/screenjunkies.py
+++ b/youtube_dl/extractor/screenjunkies.py
@@ -11,7 +11,7 @@ from ..utils import (
 
 
 class ScreenJunkiesIE(InfoExtractor):
-    _VALID_URL = r'https?://www.screenjunkies.com/video/(?P<display_id>[^/]+?)(?:-(?P<id>\d+))?(?:[/?#&]|$)'
+    _VALID_URL = r'https?://(?:www\.)?screenjunkies\.com/video/(?P<display_id>[^/]+?)(?:-(?P<id>\d+))?(?:[/?#&]|$)'
     _TESTS = [{
         'url': 'http://www.screenjunkies.com/video/best-quentin-tarantino-movie-2841915',
         'md5': '5c2b686bec3d43de42bde9ec047536b0',
diff --git a/youtube_dl/extractor/senateisvp.py b/youtube_dl/extractor/senateisvp.py
index c5f474dd1..35540c082 100644
--- a/youtube_dl/extractor/senateisvp.py
+++ b/youtube_dl/extractor/senateisvp.py
@@ -48,7 +48,7 @@ class SenateISVPIE(InfoExtractor):
         ['arch', '', 'http://ussenate-f.akamaihd.net/']
     ]
     _IE_NAME = 'senate.gov'
-    _VALID_URL = r'https?://www\.senate\.gov/isvp/?\?(?P<qs>.+)'
+    _VALID_URL = r'https?://(?:www\.)?senate\.gov/isvp/?\?(?P<qs>.+)'
     _TESTS = [{
         'url': 'http://www.senate.gov/isvp/?comm=judiciary&type=live&stt=&filename=judiciary031715&auto_play=false&wmode=transparent&poster=http%3A%2F%2Fwww.judiciary.senate.gov%2Fthemes%2Fjudiciary%2Fimages%2Fvideo-poster-flash-fit.png',
         'info_dict': {
diff --git a/youtube_dl/extractor/slideshare.py b/youtube_dl/extractor/slideshare.py
index 4967c1b77..74a1dc672 100644
--- a/youtube_dl/extractor/slideshare.py
+++ b/youtube_dl/extractor/slideshare.py
@@ -14,7 +14,7 @@ from ..utils import (
 
 
 class SlideshareIE(InfoExtractor):
-    _VALID_URL = r'https?://www\.slideshare\.net/[^/]+?/(?P<title>.+?)($|\?)'
+    _VALID_URL = r'https?://(?:www\.)?slideshare\.net/[^/]+?/(?P<title>.+?)($|\?)'
 
     _TEST = {
         'url': 'http://www.slideshare.net/Dataversity/keynote-presentation-managing-scale-and-complexity',
diff --git a/youtube_dl/extractor/spiegel.py b/youtube_dl/extractor/spiegel.py
index 74cb3a08a..b41d9f59f 100644
--- a/youtube_dl/extractor/spiegel.py
+++ b/youtube_dl/extractor/spiegel.py
@@ -103,7 +103,7 @@ class SpiegelIE(InfoExtractor):
 
 
 class SpiegelArticleIE(InfoExtractor):
-    _VALID_URL = r'https?://www\.spiegel\.de/(?!video/)[^?#]*?-(?P<id>[0-9]+)\.html'
+    _VALID_URL = r'https?://(?:www\.)?spiegel\.de/(?!video/)[^?#]*?-(?P<id>[0-9]+)\.html'
     IE_NAME = 'Spiegel:Article'
     IE_DESC = 'Articles on spiegel.de'
     _TESTS = [{
diff --git a/youtube_dl/extractor/syfy.py b/youtube_dl/extractor/syfy.py
index ab8bab5cd..def7e5a2c 100644
--- a/youtube_dl/extractor/syfy.py
+++ b/youtube_dl/extractor/syfy.py
@@ -8,7 +8,7 @@ from ..utils import (
 
 
 class SyfyIE(AdobePassIE):
-    _VALID_URL = r'https?://www\.syfy\.com/(?:[^/]+/)?videos/(?P<id>[^/?#]+)'
+    _VALID_URL = r'https?://(?:www\.)?syfy\.com/(?:[^/]+/)?videos/(?P<id>[^/?#]+)'
     _TESTS = [{
         'url': 'http://www.syfy.com/theinternetruinedmylife/videos/the-internet-ruined-my-life-season-1-trailer',
         'info_dict': {
diff --git a/youtube_dl/extractor/teachingchannel.py b/youtube_dl/extractor/teachingchannel.py
index d14d93e3a..e89759714 100644
--- a/youtube_dl/extractor/teachingchannel.py
+++ b/youtube_dl/extractor/teachingchannel.py
@@ -7,7 +7,7 @@ from .ooyala import OoyalaIE
 
 
 class TeachingChannelIE(InfoExtractor):
-    _VALID_URL = r'https?://www\.teachingchannel\.org/videos/(?P<title>.+)'
+    _VALID_URL = r'https?://(?:www\.)?teachingchannel\.org/videos/(?P<title>.+)'
 
     _TEST = {
         'url': 'https://www.teachingchannel.org/videos/teacher-teaming-evolution',
diff --git a/youtube_dl/extractor/telecinco.py b/youtube_dl/extractor/telecinco.py
index 2ecfd0405..d5abfc9e4 100644
--- a/youtube_dl/extractor/telecinco.py
+++ b/youtube_dl/extractor/telecinco.py
@@ -6,7 +6,7 @@ from .mitele import MiTeleBaseIE
 
 class TelecincoIE(MiTeleBaseIE):
     IE_DESC = 'telecinco.es, cuatro.com and mediaset.es'
-    _VALID_URL = r'https?://www\.(?:telecinco\.es|cuatro\.com|mediaset\.es)/(?:[^/]+/)+(?P<id>.+?)\.html'
+    _VALID_URL = r'https?://(?:www\.)?(?:telecinco\.es|cuatro\.com|mediaset\.es)/(?:[^/]+/)+(?P<id>.+?)\.html'
 
     _TESTS = [{
         'url': 'http://www.telecinco.es/robinfood/temporada-01/t01xp14/Bacalao-cocochas-pil-pil_0_1876350223.html',
diff --git a/youtube_dl/extractor/telewebion.py b/youtube_dl/extractor/telewebion.py
index 77916c601..7786b2813 100644
--- a/youtube_dl/extractor/telewebion.py
+++ b/youtube_dl/extractor/telewebion.py
@@ -5,7 +5,7 @@ from .common import InfoExtractor
 
 
 class TelewebionIE(InfoExtractor):
-    _VALID_URL = r'https?://www\.telewebion\.com/#!/episode/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?telewebion\.com/#!/episode/(?P<id>\d+)'
 
     _TEST = {
         'url': 'http://www.telewebion.com/#!/episode/1263668/',
diff --git a/youtube_dl/extractor/theintercept.py b/youtube_dl/extractor/theintercept.py
index 8cb3c3669..ec6f4ecaa 100644
--- a/youtube_dl/extractor/theintercept.py
+++ b/youtube_dl/extractor/theintercept.py
@@ -11,7 +11,7 @@ from ..utils import (
 
 
 class TheInterceptIE(InfoExtractor):
-    _VALID_URL = r'https://theintercept.com/fieldofvision/(?P<id>[^/?#]+)'
+    _VALID_URL = r'https?://theintercept\.com/fieldofvision/(?P<id>[^/?#]+)'
     _TESTS = [{
         'url': 'https://theintercept.com/fieldofvision/thisisacoup-episode-four-surrender-or-die/',
         'md5': '145f28b41d44aab2f87c0a4ac8ec95bd',
diff --git a/youtube_dl/extractor/thescene.py b/youtube_dl/extractor/thescene.py
index 3e4e14031..ce1326c03 100644
--- a/youtube_dl/extractor/thescene.py
+++ b/youtube_dl/extractor/thescene.py
@@ -7,7 +7,7 @@ from ..utils import qualities
 
 
 class TheSceneIE(InfoExtractor):
-    _VALID_URL = r'https://thescene\.com/watch/[^/]+/(?P<id>[^/#?]+)'
+    _VALID_URL = r'https?://thescene\.com/watch/[^/]+/(?P<id>[^/#?]+)'
 
     _TEST = {
         'url': 'https://thescene.com/watch/vogue/narciso-rodriguez-spring-2013-ready-to-wear',
diff --git a/youtube_dl/extractor/tlc.py b/youtube_dl/extractor/tlc.py
index 88eb83d74..ce4f91f46 100644
--- a/youtube_dl/extractor/tlc.py
+++ b/youtube_dl/extractor/tlc.py
@@ -13,7 +13,7 @@ from ..compat import (
 
 class TlcDeIE(InfoExtractor):
     IE_NAME = 'tlc.de'
-    _VALID_URL = r'https?://www\.tlc\.de/(?:[^/]+/)*videos/(?P<title>[^/?#]+)?(?:.*#(?P<id>\d+))?'
+    _VALID_URL = r'https?://(?:www\.)?tlc\.de/(?:[^/]+/)*videos/(?P<title>[^/?#]+)?(?:.*#(?P<id>\d+))?'
 
     _TEST = {
         'url': 'http://www.tlc.de/sendungen/breaking-amish/videos/#3235167922001',
diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index 89b869559..c2f507233 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -307,7 +307,7 @@ class UdemyIE(InfoExtractor):
 
 class UdemyCourseIE(UdemyIE):
     IE_NAME = 'udemy:course'
-    _VALID_URL = r'https?://www\.udemy\.com/(?P<id>[^/?#&]+)'
+    _VALID_URL = r'https?://(?:www\.)?udemy\.com/(?P<id>[^/?#&]+)'
     _TESTS = []
 
     @classmethod
diff --git a/youtube_dl/extractor/ustream.py b/youtube_dl/extractor/ustream.py
index 54605d863..a3dc9d33e 100644
--- a/youtube_dl/extractor/ustream.py
+++ b/youtube_dl/extractor/ustream.py
@@ -14,7 +14,7 @@ from ..utils import (
 
 
 class UstreamIE(InfoExtractor):
-    _VALID_URL = r'https?://www\.ustream\.tv/(?P<type>recorded|embed|embed/recorded)/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?ustream\.tv/(?P<type>recorded|embed|embed/recorded)/(?P<id>\d+)'
     IE_NAME = 'ustream'
     _TESTS = [{
         'url': 'http://www.ustream.tv/recorded/20274954',
@@ -117,7 +117,7 @@ class UstreamIE(InfoExtractor):
 
 
 class UstreamChannelIE(InfoExtractor):
-    _VALID_URL = r'https?://www\.ustream\.tv/channel/(?P<slug>.+)'
+    _VALID_URL = r'https?://(?:www\.)?ustream\.tv/channel/(?P<slug>.+)'
     IE_NAME = 'ustream:channel'
     _TEST = {
         'url': 'http://www.ustream.tv/channel/channeljapan',
diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py
index 388b4debe..783efda7d 100644
--- a/youtube_dl/extractor/vevo.py
+++ b/youtube_dl/extractor/vevo.py
@@ -31,7 +31,7 @@ class VevoIE(VevoBaseIE):
     (currently used by MTVIE and MySpaceIE)
     '''
     _VALID_URL = r'''(?x)
-        (?:https?://www\.vevo\.com/watch/(?!playlist|genre)(?:[^/]+/(?:[^/]+/)?)?|
+        (?:https?://(?:www\.)?vevo\.com/watch/(?!playlist|genre)(?:[^/]+/(?:[^/]+/)?)?|
            https?://cache\.vevo\.com/m/html/embed\.html\?video=|
            https?://videoplayer\.vevo\.com/embed/embedded\?videoId=|
            vevo:)
@@ -374,7 +374,7 @@ class VevoIE(VevoBaseIE):
 
 
 class VevoPlaylistIE(VevoBaseIE):
-    _VALID_URL = r'https?://www\.vevo\.com/watch/(?P<kind>playlist|genre)/(?P<id>[^/?#&]+)'
+    _VALID_URL = r'https?://(?:www\.)?vevo\.com/watch/(?P<kind>playlist|genre)/(?P<id>[^/?#&]+)'
 
     _TESTS = [{
         'url': 'http://www.vevo.com/watch/playlist/dadbf4e7-b99f-4184-9670-6f0e547b6a29',
diff --git a/youtube_dl/extractor/videodetective.py b/youtube_dl/extractor/videodetective.py
index 2ed5d9643..a19411a05 100644
--- a/youtube_dl/extractor/videodetective.py
+++ b/youtube_dl/extractor/videodetective.py
@@ -6,7 +6,7 @@ from .internetvideoarchive import InternetVideoArchiveIE
 
 
 class VideoDetectiveIE(InfoExtractor):
-    _VALID_URL = r'https?://www\.videodetective\.com/[^/]+/[^/]+/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?videodetective\.com/[^/]+/[^/]+/(?P<id>\d+)'
 
     _TEST = {
         'url': 'http://www.videodetective.com/movies/kick-ass-2/194487',
diff --git a/youtube_dl/extractor/weiqitv.py b/youtube_dl/extractor/weiqitv.py
index 3dafbeec2..8e09156c2 100644
--- a/youtube_dl/extractor/weiqitv.py
+++ b/youtube_dl/extractor/weiqitv.py
@@ -6,7 +6,7 @@ from .common import InfoExtractor
 
 class WeiqiTVIE(InfoExtractor):
     IE_DESC = 'WQTV'
-    _VALID_URL = r'https?://www\.weiqitv\.com/index/video_play\?videoId=(?P<id>[A-Za-z0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?weiqitv\.com/index/video_play\?videoId=(?P<id>[A-Za-z0-9]+)'
 
     _TESTS = [{
         'url': 'http://www.weiqitv.com/index/video_play?videoId=53c744f09874f0e76a8b46f3',
diff --git a/youtube_dl/extractor/yam.py b/youtube_dl/extractor/yam.py
index 63bbc0634..ef5535547 100644
--- a/youtube_dl/extractor/yam.py
+++ b/youtube_dl/extractor/yam.py
@@ -15,7 +15,7 @@ from ..utils import (
 
 class YamIE(InfoExtractor):
     IE_DESC = '蕃薯藤yam天空部落'
-    _VALID_URL = r'https?://mymedia.yam.com/m/(?P<id>\d+)'
+    _VALID_URL = r'https?://mymedia\.yam\.com/m/(?P<id>\d+)'
 
     _TESTS = [{
         # An audio hosted on Yam
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 5082cb589..5ca903825 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -2302,7 +2302,7 @@ class YoutubeSearchURLIE(YoutubePlaylistBaseInfoExtractor):
 
 class YoutubeShowIE(YoutubePlaylistsBaseInfoExtractor):
     IE_DESC = 'YouTube.com (multi-season) shows'
-    _VALID_URL = r'https?://www\.youtube\.com/show/(?P<id>[^?#]*)'
+    _VALID_URL = r'https?://(?:www\.)?youtube\.com/show/(?P<id>[^?#]*)'
     IE_NAME = 'youtube:show'
     _TESTS = [{
         'url': 'https://www.youtube.com/show/airdisasters',
@@ -2371,7 +2371,7 @@ class YoutubeFeedsInfoExtractor(YoutubeBaseInfoExtractor):
 class YoutubeWatchLaterIE(YoutubePlaylistIE):
     IE_NAME = 'youtube:watchlater'
     IE_DESC = 'Youtube watch later list, ":ytwatchlater" for short (requires authentication)'
-    _VALID_URL = r'https?://www\.youtube\.com/(?:feed/watch_later|(?:playlist|watch)\?(?:.+&)?list=WL)|:ytwatchlater'
+    _VALID_URL = r'https?://(?:www\.)?youtube\.com/(?:feed/watch_later|(?:playlist|watch)\?(?:.+&)?list=WL)|:ytwatchlater'
 
     _TESTS = [{
         'url': 'https://www.youtube.com/playlist?list=WL',
@@ -2392,7 +2392,7 @@ class YoutubeWatchLaterIE(YoutubePlaylistIE):
 class YoutubeFavouritesIE(YoutubeBaseInfoExtractor):
     IE_NAME = 'youtube:favorites'
     IE_DESC = 'YouTube.com favourite videos, ":ytfav" for short (requires authentication)'
-    _VALID_URL = r'https?://www\.youtube\.com/my_favorites|:ytfav(?:ou?rites)?'
+    _VALID_URL = r'https?://(?:www\.)?youtube\.com/my_favorites|:ytfav(?:ou?rites)?'
     _LOGIN_REQUIRED = True
 
     def _real_extract(self, url):
@@ -2403,21 +2403,21 @@ class YoutubeFavouritesIE(YoutubeBaseInfoExtractor):
 
 class YoutubeRecommendedIE(YoutubeFeedsInfoExtractor):
     IE_DESC = 'YouTube.com recommended videos, ":ytrec" for short (requires authentication)'
-    _VALID_URL = r'https?://www\.youtube\.com/feed/recommended|:ytrec(?:ommended)?'
+    _VALID_URL = r'https?://(?:www\.)?youtube\.com/feed/recommended|:ytrec(?:ommended)?'
     _FEED_NAME = 'recommended'
     _PLAYLIST_TITLE = 'Youtube Recommended videos'
 
 
 class YoutubeSubscriptionsIE(YoutubeFeedsInfoExtractor):
     IE_DESC = 'YouTube.com subscriptions feed, "ytsubs" keyword (requires authentication)'
-    _VALID_URL = r'https?://www\.youtube\.com/feed/subscriptions|:ytsubs(?:criptions)?'
+    _VALID_URL = r'https?://(?:www\.)?youtube\.com/feed/subscriptions|:ytsubs(?:criptions)?'
     _FEED_NAME = 'subscriptions'
     _PLAYLIST_TITLE = 'Youtube Subscriptions'
 
 
 class YoutubeHistoryIE(YoutubeFeedsInfoExtractor):
     IE_DESC = 'Youtube watch history, ":ythistory" for short (requires authentication)'
-    _VALID_URL = r'https?://www\.youtube\.com/feed/history|:ythistory'
+    _VALID_URL = r'https?://(?:www\.)?youtube\.com/feed/history|:ythistory'
     _FEED_NAME = 'history'
     _PLAYLIST_TITLE = 'Youtube History'
 

From 014b7e6b25be5583c772af054cd7a1e37a327088 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 14 Sep 2016 17:07:05 +0100
Subject: [PATCH 6078/6219] [go] add support for free full episodes(#10439)

---
 youtube_dl/extractor/go.py | 27 ++++++++++++++++++++++++---
 1 file changed, 24 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/go.py b/youtube_dl/extractor/go.py
index 6a437c54d..7925c1e22 100644
--- a/youtube_dl/extractor/go.py
+++ b/youtube_dl/extractor/go.py
@@ -8,6 +8,8 @@ from ..utils import (
     int_or_none,
     determine_ext,
     parse_age_limit,
+    urlencode_postdata,
+    ExtractorError,
 )
 
 
@@ -19,7 +21,7 @@ class GoIE(InfoExtractor):
         'watchdisneyjunior': '008',
         'watchdisneyxd': '009',
     }
-    _VALID_URL = r'https?://(?:(?P<sub_domain>%s)\.)?go\.com/.*?vdka(?P<id>\w+)' % '|'.join(_BRANDS.keys())
+    _VALID_URL = r'https?://(?:(?P<sub_domain>%s)\.)?go\.com/(?:[^/]+/)*(?:vdka(?P<id>\w+)|season-\d+/\d+-(?P<display_id>[^/?#]+))' % '|'.join(_BRANDS.keys())
     _TESTS = [{
         'url': 'http://abc.go.com/shows/castle/video/most-recent/vdka0_g86w5onx',
         'info_dict': {
@@ -38,9 +40,13 @@ class GoIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        sub_domain, video_id = re.match(self._VALID_URL, url).groups()
+        sub_domain, video_id, display_id = re.match(self._VALID_URL, url).groups()
+        if not video_id:
+            webpage = self._download_webpage(url, display_id)
+            video_id = self._search_regex(r'data-video-id=["\']VDKA(\w+)', webpage, 'video id')
+        brand = self._BRANDS[sub_domain]
         video_data = self._download_json(
-            'http://api.contents.watchabc.go.com/vp2/ws/contents/3000/videos/%s/001/-1/-1/-1/%s/-1/-1.json' % (self._BRANDS[sub_domain], video_id),
+            'http://api.contents.watchabc.go.com/vp2/ws/contents/3000/videos/%s/001/-1/-1/-1/%s/-1/-1.json' % (brand, video_id),
             video_id)['video'][0]
         title = video_data['title']
 
@@ -52,6 +58,21 @@ class GoIE(InfoExtractor):
             format_id = asset.get('format')
             ext = determine_ext(asset_url)
             if ext == 'm3u8':
+                video_type = video_data.get('type')
+                if video_type == 'lf':
+                    entitlement = self._download_json(
+                        'https://api.entitlement.watchabc.go.com/vp2/ws-secure/entitlement/2020/authorize.json',
+                        video_id, data=urlencode_postdata({
+                            'video_id': video_data['id'],
+                            'video_type': video_type,
+                            'brand': brand,
+                            'device': '001',
+                        }))
+                    errors = entitlement.get('errors', {}).get('errors', [])
+                    if errors:
+                        error_massege = ', '.join([error['message'] for error in errors])
+                        raise ExtractorError('%s said: %s' % (self.IE_NAME, error_massege), expected=True)
+                    asset_url += '?' + entitlement['uplynkData']['sessionKey']
                 formats.extend(self._extract_m3u8_formats(
                     asset_url, video_id, 'mp4', m3u8_id=format_id or 'hls', fatal=False))
             else:

From 353f340e11d7fc4a0a4973ddd85bc93b1061a487 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 14 Sep 2016 17:22:42 +0100
Subject: [PATCH 6079/6219] [go] fix typo

---
 youtube_dl/extractor/go.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/go.py b/youtube_dl/extractor/go.py
index 7925c1e22..c7776b186 100644
--- a/youtube_dl/extractor/go.py
+++ b/youtube_dl/extractor/go.py
@@ -70,8 +70,8 @@ class GoIE(InfoExtractor):
                         }))
                     errors = entitlement.get('errors', {}).get('errors', [])
                     if errors:
-                        error_massege = ', '.join([error['message'] for error in errors])
-                        raise ExtractorError('%s said: %s' % (self.IE_NAME, error_massege), expected=True)
+                        error_message = ', '.join([error['message'] for error in errors])
+                        raise ExtractorError('%s said: %s' % (self.IE_NAME, error_message), expected=True)
                     asset_url += '?' + entitlement['uplynkData']['sessionKey']
                 formats.extend(self._extract_m3u8_formats(
                     asset_url, video_id, 'mp4', m3u8_id=format_id or 'hls', fatal=False))

From 6db354a9f4c62c3cc47918adc13e1e4b63146c80 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 15 Sep 2016 00:53:04 +0800
Subject: [PATCH 6080/6219] [kuwo] Update _TESTS

---
 youtube_dl/extractor/kuwo.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/kuwo.py b/youtube_dl/extractor/kuwo.py
index ba621ca7b..081af86f6 100644
--- a/youtube_dl/extractor/kuwo.py
+++ b/youtube_dl/extractor/kuwo.py
@@ -82,7 +82,7 @@ class KuwoIE(KuwoBaseIE):
             'upload_date': '20150518',
         },
         'params': {
-            'format': 'mp3-320'
+            'format': 'mp3-320',
         },
     }, {
         'url': 'http://www.kuwo.cn/yinyue/3197154?catalog=yueku2016',
@@ -181,7 +181,7 @@ class KuwoChartIE(InfoExtractor):
         'info_dict': {
             'id': '香港中文龙虎榜',
         },
-        'playlist_mincount': 10,
+        'playlist_mincount': 7,
     }
 
     def _real_extract(self, url):
@@ -303,7 +303,7 @@ class KuwoMvIE(KuwoBaseIE):
             'id': '6480076',
             'ext': 'mp4',
             'title': 'My HouseMV',
-            'creator': 'PM02:00',
+            'creator': '2PM',
         },
         # In this video, music URLs (anti.s) are blocked outside China and
         # USA, while the MV URL (mvurl) is available globally, so force the MV

From 961516bfd1f3b514859f03766d282824ba8a76f5 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 15 Sep 2016 00:56:15 +0800
Subject: [PATCH 6081/6219] [kwuo:song] Improve error detection (closes #10650)

---
 ChangeLog                    | 1 +
 youtube_dl/extractor/kuwo.py | 4 ++--
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 25c916eb2..c3c8bf037 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
 version <unreleased>
 
 Extractors
+* [kwuo] Improve error detection (#10650)
 * [bilibili] Fix extraction for specific videos (#10647)
 + [nbc] Add support for NBC Olympics (#10361)
 
diff --git a/youtube_dl/extractor/kuwo.py b/youtube_dl/extractor/kuwo.py
index 081af86f6..63e10125e 100644
--- a/youtube_dl/extractor/kuwo.py
+++ b/youtube_dl/extractor/kuwo.py
@@ -91,10 +91,10 @@ class KuwoIE(KuwoBaseIE):
 
     def _real_extract(self, url):
         song_id = self._match_id(url)
-        webpage = self._download_webpage(
+        webpage, urlh = self._download_webpage_handle(
             url, song_id, note='Download song detail info',
             errnote='Unable to get song detail info')
-        if '对不起，该歌曲由于版权问题已被下线，将返回网站首页' in webpage:
+        if song_id not in urlh.geturl() or '对不起，该歌曲由于版权问题已被下线，将返回网站首页' in webpage:
             raise ExtractorError('this song has been offline because of copyright issues', expected=True)
 
         song_name = self._html_search_regex(

From a942d6cb48994c5ff14ccef8773fb086a5544970 Mon Sep 17 00:00:00 2001
From: renalid <renaud.euvrard@MAC-1636.local>
Date: Fri, 2 Sep 2016 18:31:52 +0200
Subject: [PATCH 6082/6219] [utils,franceinter] Add french months' names and
 fix extraction

Update of the "FranceInter" radio extractor : webpages HTML structure
had changed, the extractor didn't work. So I updated this extractor to
get the mp3 URL and all details.
---
 youtube_dl/extractor/franceinter.py | 38 ++++++++++++++++-------------
 youtube_dl/utils.py                 | 13 ++++++++--
 2 files changed, 32 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/extractor/franceinter.py b/youtube_dl/extractor/franceinter.py
index 2369f868d..6dad8d712 100644
--- a/youtube_dl/extractor/franceinter.py
+++ b/youtube_dl/extractor/franceinter.py
@@ -2,20 +2,24 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..utils import int_or_none
+from ..utils import (
+    unified_timestamp,
+    month_by_name,
+)
 
 
 class FranceInterIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?franceinter\.fr/player/reecouter\?play=(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?franceinter\.fr/emissions/(?P<id>[^?#]+)'
+
     _TEST = {
-        'url': 'http://www.franceinter.fr/player/reecouter?play=793962',
+        'url': 'https://www.franceinter.fr/emissions/la-marche-de-l-histoire/la-marche-de-l-histoire-18-decembre-2013',
         'md5': '4764932e466e6f6c79c317d2e74f6884',
         'info_dict': {
-            'id': '793962',
+            'id': 'la-marche-de-l-histoire/la-marche-de-l-histoire-18-decembre-2013',
             'ext': 'mp3',
-            'title': 'L’Histoire dans les jeux vidéo',
-            'description': 'md5:7e93ddb4451e7530022792240a3049c7',
-            'timestamp': 1387369800,
+            'title': 'L’Histoire dans les jeux vidéo du 18 décembre 2013 - France Inter',
+            'description': 'L’Histoire dans les jeux vidéo du 18 décembre 2013  par Jean Lebrun en replay sur France Inter. Retrouvez l\'émission en réécoute gratuite et abonnez-vous au podcast !',
+            'timestamp': 1387324800,
             'upload_date': '20131218',
         },
     }
@@ -25,17 +29,17 @@ class FranceInterIE(InfoExtractor):
 
         webpage = self._download_webpage(url, video_id)
 
-        path = self._search_regex(
-            r'<a id="player".+?href="([^"]+)"', webpage, 'video url')
-        video_url = 'http://www.franceinter.fr/' + path
+        video_url = self._search_regex(
+            r'<button class="replay-button playable" data-is-aod="1" data-url="([^"]+)"', webpage, 'video url')
 
-        title = self._html_search_regex(
-            r'<span class="title-diffusion">(.+?)</span>', webpage, 'title')
-        description = self._html_search_regex(
-            r'<span class="description">(.*?)</span>',
-            webpage, 'description', fatal=False)
-        timestamp = int_or_none(self._search_regex(
-            r'data-date="(\d+)"', webpage, 'upload date', fatal=False))
+        title = self._og_search_title(webpage)
+        description = self._og_search_description(webpage)
+
+        extractdate = self._search_regex('(\d{2}-([a-zA-Z\s]+)-\d{4}$)', url, 'extractdate', fatal=False)
+        extractdate = extractdate.split('-')
+        extractdate = extractdate[2] + "," + str(month_by_name(extractdate[1], 'fr')) + "," + extractdate[0]
+
+        timestamp = unified_timestamp(extractdate)
 
         return {
             'id': video_id,
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index ed199c4ad..623ced625 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -91,6 +91,10 @@ ENGLISH_MONTH_NAMES = [
     'January', 'February', 'March', 'April', 'May', 'June',
     'July', 'August', 'September', 'October', 'November', 'December']
 
+FRENCH_MONTH_NAMES = [
+    'janvier', 'fevrier', 'mars', 'avril', 'mai', 'juin',
+    'juillet', 'aout', 'septembre', 'octobre', 'novembre', 'decembre']
+
 KNOWN_EXTENSIONS = (
     'mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v', 'aac',
     'flv', 'f4v', 'f4a', 'f4b',
@@ -1587,11 +1591,16 @@ def parse_count(s):
     return lookup_unit_table(_UNIT_TABLE, s)
 
 
-def month_by_name(name):
+def month_by_name(name, lang='en'):
     """ Return the number of a month by (locale-independently) English name """
 
+    name_list = ENGLISH_MONTH_NAMES
+
+    if lang == 'fr':
+        name_list = FRENCH_MONTH_NAMES
+
     try:
-        return ENGLISH_MONTH_NAMES.index(name) + 1
+        return name_list.index(name) + 1
     except ValueError:
         return None
 

From f6717dec8abe7c0d34e704732b53665a9415fa2e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 14 Sep 2016 23:13:55 +0700
Subject: [PATCH 6083/6219] [utils] Improve month_by_name and add tests

---
 test/test_utils.py  | 11 +++++++++++
 youtube_dl/utils.py | 16 ++++++++--------
 2 files changed, 19 insertions(+), 8 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 405c5d351..4ebca8744 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -40,6 +40,7 @@ from youtube_dl.utils import (
     js_to_json,
     limit_length,
     mimetype2ext,
+    month_by_name,
     ohdave_rsa_encrypt,
     OnDemandPagedList,
     orderedSet,
@@ -634,6 +635,16 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(mimetype2ext('text/vtt;charset=utf-8'), 'vtt')
         self.assertEqual(mimetype2ext('text/html; charset=utf-8'), 'html')
 
+    def test_month_by_name(self):
+        self.assertEqual(month_by_name(None), None)
+        self.assertEqual(month_by_name('December', 'en'), 12)
+        self.assertEqual(month_by_name('decembre', 'fr'), 12)
+        self.assertEqual(month_by_name('December'), 12)
+        self.assertEqual(month_by_name('decembre'), None)
+        self.assertEqual(month_by_name('Unknown', 'unknown'), None)
+
+    def test_m
+
     def test_parse_codecs(self):
         self.assertEqual(parse_codecs(''), {})
         self.assertEqual(parse_codecs('avc1.77.30, mp4a.40.2'), {
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 623ced625..a4ef15908 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -91,9 +91,12 @@ ENGLISH_MONTH_NAMES = [
     'January', 'February', 'March', 'April', 'May', 'June',
     'July', 'August', 'September', 'October', 'November', 'December']
 
-FRENCH_MONTH_NAMES = [
-    'janvier', 'fevrier', 'mars', 'avril', 'mai', 'juin',
-    'juillet', 'aout', 'septembre', 'octobre', 'novembre', 'decembre']
+MONTH_NAMES = {
+    'en': ENGLISH_MONTH_NAMES,
+    'fr': [
+        'janvier', 'fevrier', 'mars', 'avril', 'mai', 'juin',
+        'juillet', 'aout', 'septembre', 'octobre', 'novembre', 'decembre'],
+}
 
 KNOWN_EXTENSIONS = (
     'mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v', 'aac',
@@ -1594,13 +1597,10 @@ def parse_count(s):
 def month_by_name(name, lang='en'):
     """ Return the number of a month by (locale-independently) English name """
 
-    name_list = ENGLISH_MONTH_NAMES
-
-    if lang == 'fr':
-        name_list = FRENCH_MONTH_NAMES
+    month_names = MONTH_NAMES.get(lang, MONTH_NAMES['en'])
 
     try:
-        return name_list.index(name) + 1
+        return month_names.index(name) + 1
     except ValueError:
         return None
 

From 3e4185c3965579c2cc10922384694c2465be4557 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 14 Sep 2016 23:57:01 +0700
Subject: [PATCH 6084/6219] [utils] Use native french month names

---
 test/test_utils.py  | 6 ++----
 youtube_dl/utils.py | 4 ++--
 2 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 4ebca8744..9789d8611 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -638,13 +638,11 @@ class TestUtil(unittest.TestCase):
     def test_month_by_name(self):
         self.assertEqual(month_by_name(None), None)
         self.assertEqual(month_by_name('December', 'en'), 12)
-        self.assertEqual(month_by_name('decembre', 'fr'), 12)
+        self.assertEqual(month_by_name('décembre', 'fr'), 12)
         self.assertEqual(month_by_name('December'), 12)
-        self.assertEqual(month_by_name('decembre'), None)
+        self.assertEqual(month_by_name('décembre'), None)
         self.assertEqual(month_by_name('Unknown', 'unknown'), None)
 
-    def test_m
-
     def test_parse_codecs(self):
         self.assertEqual(parse_codecs(''), {})
         self.assertEqual(parse_codecs('avc1.77.30, mp4a.40.2'), {
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index a4ef15908..69ca88c85 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -94,8 +94,8 @@ ENGLISH_MONTH_NAMES = [
 MONTH_NAMES = {
     'en': ENGLISH_MONTH_NAMES,
     'fr': [
-        'janvier', 'fevrier', 'mars', 'avril', 'mai', 'juin',
-        'juillet', 'aout', 'septembre', 'octobre', 'novembre', 'decembre'],
+        'janvier', 'février', 'mars', 'avril', 'mai', 'juin',
+        'juillet', 'août', 'septembre', 'octobre', 'novembre', 'décembre'],
 }
 
 KNOWN_EXTENSIONS = (

From 0002962f3feb86ec8c14429af7ecddc17815fa93 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 14 Sep 2016 23:59:13 +0700
Subject: [PATCH 6085/6219] [franceinter] Improve extraction (Closes #10538)

---
 youtube_dl/extractor/franceinter.py | 29 ++++++++++++++++-------------
 1 file changed, 16 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/franceinter.py b/youtube_dl/extractor/franceinter.py
index 6dad8d712..0d58f89c5 100644
--- a/youtube_dl/extractor/franceinter.py
+++ b/youtube_dl/extractor/franceinter.py
@@ -2,10 +2,8 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..utils import (
-    unified_timestamp,
-    month_by_name,
-)
+from ..compat import compat_str
+from ..utils import month_by_name
 
 
 class FranceInterIE(InfoExtractor):
@@ -18,8 +16,7 @@ class FranceInterIE(InfoExtractor):
             'id': 'la-marche-de-l-histoire/la-marche-de-l-histoire-18-decembre-2013',
             'ext': 'mp3',
             'title': 'L’Histoire dans les jeux vidéo du 18 décembre 2013 - France Inter',
-            'description': 'L’Histoire dans les jeux vidéo du 18 décembre 2013  par Jean Lebrun en replay sur France Inter. Retrouvez l\'émission en réécoute gratuite et abonnez-vous au podcast !',
-            'timestamp': 1387324800,
+            'description': 'md5:7f2ce449894d1e585932273080fb410d',
             'upload_date': '20131218',
         },
     }
@@ -30,22 +27,28 @@ class FranceInterIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         video_url = self._search_regex(
-            r'<button class="replay-button playable" data-is-aod="1" data-url="([^"]+)"', webpage, 'video url')
+            r'(?s)<div[^>]+class=["\']page-diffusion["\'][^>]*>.*?<button[^>]+data-url=(["\'])(?P<url>(?:(?!\1).)+)\1',
+            webpage, 'video url', group='url')
 
         title = self._og_search_title(webpage)
         description = self._og_search_description(webpage)
 
-        extractdate = self._search_regex('(\d{2}-([a-zA-Z\s]+)-\d{4}$)', url, 'extractdate', fatal=False)
-        extractdate = extractdate.split('-')
-        extractdate = extractdate[2] + "," + str(month_by_name(extractdate[1], 'fr')) + "," + extractdate[0]
-
-        timestamp = unified_timestamp(extractdate)
+        upload_date_str = self._search_regex(
+            r'class=["\']cover-emission-period["\'][^>]*>[^<]+\s+(\d{1,2}\s+[^\s]+\s+\d{4})<',
+            webpage, 'upload date', fatal=False)
+        if upload_date_str:
+            upload_date_list = upload_date_str.split()
+            upload_date_list.reverse()
+            upload_date_list[1] = compat_str(month_by_name(upload_date_list[1], lang='fr'))
+            upload_date = ''.join(upload_date_list)
+        else:
+            upload_date = None
 
         return {
             'id': video_id,
             'title': title,
             'description': description,
-            'timestamp': timestamp,
+            'upload_date': upload_date,
             'formats': [{
                 'url': video_url,
                 'vcodec': 'none',

From 797c636bcb02d1199015b753d26430eec13c4b2b Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 14 Sep 2016 18:58:47 +0100
Subject: [PATCH 6086/6219] [ap] improve adobe pass names and parse error
 handling

---
 youtube_dl/YoutubeDL.py           |  2 +-
 youtube_dl/__init__.py            |  8 +++++---
 youtube_dl/extractor/adobepass.py |  9 +++------
 youtube_dl/options.py             | 12 ++++++------
 4 files changed, 15 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 9c2c26280..29d8517a3 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -131,7 +131,7 @@ class YoutubeDL(object):
     username:          Username for authentication purposes.
     password:          Password for authentication purposes.
     videopassword:     Password for accessing a video.
-    ap_mso_id:         Adobe Pass Multiple-system operator Identifier.
+    ap_mso:            Adobe Pass Multiple-system operator Identifier.
     ap_username:       TV Provider username for authentication purposes.
     ap_password:       TV Provider password for authentication purposes.
     usenetrc:          Use netrc for authentication instead.
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index cdff3df65..5614ef0fb 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -120,9 +120,9 @@ def _real_main(argv=None):
                 desc += ' (Example: "%s%s:%s" )' % (ie.SEARCH_KEY, random.choice(_COUNTS), random.choice(_SEARCHES))
             write_string(desc + '\n', out=sys.stdout)
         sys.exit(0)
-    if opts.list_ap_mso_ids:
+    if opts.ap_mso_list:
         table = [[mso_id, mso_info['name']] for mso_id, mso_info in MSO_INFO.items()]
-        write_string('Supported TV Providers:\n' + render_table(['mso id', 'mso name'], table) + '\n', out=sys.stdout)
+        write_string('Supported TV Providers:\n' + render_table(['mso', 'mso name'], table) + '\n', out=sys.stdout)
         sys.exit(0)
 
     # Conflicting, missing and erroneous options
@@ -165,6 +165,8 @@ def _real_main(argv=None):
             parser.error('max sleep interval must be greater than or equal to min sleep interval')
     else:
         opts.max_sleep_interval = opts.sleep_interval
+    if opts.ap_mso and opts.ap_mso not in MSO_INFO:
+        parser.error('Unsupported TV Provider, use --ap-mso-list to get a list of supported TV Providers')
 
     def parse_retries(retries):
         if retries in ('inf', 'infinite'):
@@ -303,7 +305,7 @@ def _real_main(argv=None):
         'password': opts.password,
         'twofactor': opts.twofactor,
         'videopassword': opts.videopassword,
-        'ap_mso_id': opts.ap_mso_id,
+        'ap_mso': opts.ap_mso,
         'ap_username': opts.ap_username,
         'ap_password': opts.ap_password,
         'quiet': (opts.quiet or any_getting or any_printing),
diff --git a/youtube_dl/extractor/adobepass.py b/youtube_dl/extractor/adobepass.py
index 913a817d2..8ef5a96ce 100644
--- a/youtube_dl/extractor/adobepass.py
+++ b/youtube_dl/extractor/adobepass.py
@@ -72,8 +72,8 @@ class AdobePassIE(InfoExtractor):
         def raise_mvpd_required():
             raise ExtractorError(
                 'This video is only available for users of participating TV providers. '
-                'Use --ap-mso-id to specify Adobe Pass Multiple-system operator Identifier '
-                'and --netrc to provide account credentials.', expected=True)
+                'Use --ap-mso to specify Adobe Pass Multiple-system operator Identifier '
+                'and --ap-username and --ap-password or --netrc to provide account credentials.', expected=True)
 
         mvpd_headers = {
             'ap_42': 'anonymous',
@@ -91,12 +91,9 @@ class AdobePassIE(InfoExtractor):
                 authn_token = None
             if not authn_token:
                 # TODO add support for other TV Providers
-                mso_id = self._downloader.params.get('ap_mso_id')
+                mso_id = self._downloader.params.get('ap_mso')
                 if not mso_id:
                     raise_mvpd_required()
-                if mso_id not in MSO_INFO:
-                    raise ExtractorError(
-                        'Unsupported TV Provider, use --list-ap-mso-ids to get a list of supported TV Providers' % mso_id, expected=True)
                 username, password = self._get_login_info('ap_username', 'ap_password', mso_id)
                 if not username or not password:
                     raise_mvpd_required()
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 342ae3be3..46c326b3d 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -353,20 +353,20 @@ def parseOpts(overrideArguments=None):
 
     adobe_pass = optparse.OptionGroup(parser, 'Adobe Pass Options')
     adobe_pass.add_option(
-        '--ap-mso-id',
-        dest='ap_mso_id', metavar='APMSOID',
+        '--ap-mso',
+        dest='ap_mso', metavar='MSO',
         help='Adobe Pass Multiple-system operator Identifier')
     adobe_pass.add_option(
         '--ap-username',
-        dest='ap_username', metavar='APUSERNAME',
+        dest='ap_username', metavar='USERNAME',
         help='TV Provider Login with this account ID')
     adobe_pass.add_option(
         '--ap-password',
-        dest='ap_password', metavar='APPASSWORD',
+        dest='ap_password', metavar='PASSWORD',
         help='TV Provider Account password. If this option is left out, youtube-dl will ask interactively.')
     adobe_pass.add_option(
-        '--list-ap-mso-ids',
-        action='store_true', dest='list_ap_mso_ids', default=False,
+        '--ap-mso-list',
+        action='store_true', dest='ap_mso_list', default=False,
         help='List all supported TV Providers')
 
     video_format = optparse.OptionGroup(parser, 'Video Format Options')

From 87148bb7110ed54ef50f0660dfe0a735cdede3ca Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 14 Sep 2016 20:21:09 +0100
Subject: [PATCH 6087/6219] [adobepass] rename --ap-mso-list option to
 --ap-list-mso

---
 youtube_dl/__init__.py | 4 ++--
 youtube_dl/options.py  | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 5614ef0fb..1cf3140a0 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -120,7 +120,7 @@ def _real_main(argv=None):
                 desc += ' (Example: "%s%s:%s" )' % (ie.SEARCH_KEY, random.choice(_COUNTS), random.choice(_SEARCHES))
             write_string(desc + '\n', out=sys.stdout)
         sys.exit(0)
-    if opts.ap_mso_list:
+    if opts.ap_list_mso:
         table = [[mso_id, mso_info['name']] for mso_id, mso_info in MSO_INFO.items()]
         write_string('Supported TV Providers:\n' + render_table(['mso', 'mso name'], table) + '\n', out=sys.stdout)
         sys.exit(0)
@@ -166,7 +166,7 @@ def _real_main(argv=None):
     else:
         opts.max_sleep_interval = opts.sleep_interval
     if opts.ap_mso and opts.ap_mso not in MSO_INFO:
-        parser.error('Unsupported TV Provider, use --ap-mso-list to get a list of supported TV Providers')
+        parser.error('Unsupported TV Provider, use --ap-list-mso to get a list of supported TV Providers')
 
     def parse_retries(retries):
         if retries in ('inf', 'infinite'):
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 46c326b3d..b2e863119 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -365,8 +365,8 @@ def parseOpts(overrideArguments=None):
         dest='ap_password', metavar='PASSWORD',
         help='TV Provider Account password. If this option is left out, youtube-dl will ask interactively.')
     adobe_pass.add_option(
-        '--ap-mso-list',
-        action='store_true', dest='ap_mso_list', default=False,
+        '--ap-list-mso',
+        action='store_true', dest='ap_list_mso', default=False,
         help='List all supported TV Providers')
 
     video_format = optparse.OptionGroup(parser, 'Video Format Options')

From c035dba19e815eca4a21f17918e96c2e2bd55d6b Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 15 Sep 2016 08:12:12 +0100
Subject: [PATCH 6088/6219] [bellmedia] add support for more sites

---
 youtube_dl/extractor/{ctv.py => bellmedia.py} | 39 ++++++++++++++++---
 youtube_dl/extractor/extractors.py            |  2 +-
 2 files changed, 35 insertions(+), 6 deletions(-)
 rename youtube_dl/extractor/{ctv.py => bellmedia.py} (54%)

diff --git a/youtube_dl/extractor/ctv.py b/youtube_dl/extractor/bellmedia.py
similarity index 54%
rename from youtube_dl/extractor/ctv.py
rename to youtube_dl/extractor/bellmedia.py
index a1fe86316..32326ed9e 100644
--- a/youtube_dl/extractor/ctv.py
+++ b/youtube_dl/extractor/bellmedia.py
@@ -6,8 +6,25 @@ import re
 from .common import InfoExtractor
 
 
-class CTVIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?(?P<domain>ctv|tsn|bnn|thecomedynetwork)\.ca/.*?(?:\bvid=|-vid|~|%7E)(?P<id>[0-9.]+)'
+class BellMediaIE(InfoExtractor):
+    _VALID_URL = r'''(?x)https?://(?:www\.)?
+        (?P<domain>
+            (?:
+                ctv|
+                tsn|
+                bnn|
+                thecomedynetwork|
+                discovery|
+                discoveryvelocity|
+                sciencechannel|
+                investigationdiscovery|
+                animalplanet|
+                bravo|
+                mtv|
+                space
+            )\.ca|
+            much\.com
+        )/.*?(?:\bvid=|-vid|~|%7E|/(?:episode)?)(?P<id>[0-9]{6})'''
     _TESTS = [{
         'url': 'http://www.ctv.ca/video/player?vid=706966',
         'md5': 'ff2ebbeae0aa2dcc32a830c3fd69b7b0',
@@ -32,15 +49,27 @@ class CTVIE(InfoExtractor):
     }, {
         'url': 'http://www.ctv.ca/YourMorning/Video/S1E6-Monday-August-29-2016-vid938009',
         'only_matching': True,
+    }, {
+        'url': 'http://www.much.com/shows/atmidnight/episode948007/tuesday-september-13-2016',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.much.com/shows/the-almost-impossible-gameshow/928979/episode-6',
+        'only_matching': True,
     }]
+    _DOMAINS = {
+        'thecomedynetwork': 'comedy',
+        'discoveryvelocity': 'discvel',
+        'sciencechannel': 'discsci',
+        'investigationdiscovery': 'invdisc',
+        'animalplanet': 'aniplan',
+    }
 
     def _real_extract(self, url):
         domain, video_id = re.match(self._VALID_URL, url).groups()
-        if domain == 'thecomedynetwork':
-            domain = 'comedy'
+        domain = domain.split('.')[0]
         return {
             '_type': 'url_transparent',
             'id': video_id,
-            'url': '9c9media:%s_web:%s' % (domain, video_id),
+            'url': '9c9media:%s_web:%s' % (self._DOMAINS.get(domain, domain), video_id),
             'ie_key': 'NineCNineMedia',
         }
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 522691de1..dd0579425 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -93,6 +93,7 @@ from .bbc import (
 )
 from .beeg import BeegIE
 from .behindkink import BehindKinkIE
+from .bellmedia import BellMediaIE
 from .beatportpro import BeatportProIE
 from .bet import BetIE
 from .bigflix import BigflixIE
@@ -195,7 +196,6 @@ from .crunchyroll import (
 )
 from .cspan import CSpanIE
 from .ctsnews import CtsNewsIE
-from .ctv import CTVIE
 from .ctvnews import CTVNewsIE
 from .cultureunplugged import CultureUnpluggedIE
 from .curiositystream import (

From 95be29e1c6b7a06ac444d5142582ebece79698ff Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 15 Sep 2016 20:58:02 +0700
Subject: [PATCH 6089/6219] [twitch] Fix api calls (Closes #10654, closes
 #10660)

---
 youtube_dl/extractor/twitch.py | 11 +++--------
 1 file changed, 3 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 359a8859c..af6d890b0 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -32,6 +32,7 @@ class TwitchBaseIE(InfoExtractor):
     _API_BASE = 'https://api.twitch.tv'
     _USHER_BASE = 'https://usher.ttvnw.net'
     _LOGIN_URL = 'http://www.twitch.tv/login'
+    _CLIENT_ID = 'jzkbprff40iqj646a697cyrvl0zt2m6'
     _NETRC_MACHINE = 'twitch'
 
     def _handle_error(self, response):
@@ -44,15 +45,9 @@ class TwitchBaseIE(InfoExtractor):
                 expected=True)
 
     def _call_api(self, path, item_id, note):
-        headers = {
-            'Referer': 'http://api.twitch.tv/crossdomain/receiver.html?v=2',
-            'X-Requested-With': 'XMLHttpRequest',
-        }
-        for cookie in self._downloader.cookiejar:
-            if cookie.name == 'api_token':
-                headers['Twitch-Api-Token'] = cookie.value
         response = self._download_json(
-            '%s/%s' % (self._API_BASE, path), item_id, note)
+            '%s/%s' % (self._API_BASE, path), item_id, note,
+            headers={'Client-ID': self._CLIENT_ID})
         self._handle_error(response)
         return response
 

From eb5b1fc0211e89f386c4f5563cc1d5d4edeb3c55 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 15 Sep 2016 21:53:35 +0700
Subject: [PATCH 6090/6219] [crunchyroll] Fix authentication (Closes #10655)

---
 youtube_dl/extractor/crunchyroll.py | 47 +++++++++++++++++++++++------
 1 file changed, 38 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 6d3abb52f..1b69bd0b6 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -34,22 +34,51 @@ from ..aes import (
 
 
 class CrunchyrollBaseIE(InfoExtractor):
+    _LOGIN_URL = 'https://www.crunchyroll.com/login'
+    _LOGIN_FORM = 'login_form'
     _NETRC_MACHINE = 'crunchyroll'
 
     def _login(self):
         (username, password) = self._get_login_info()
         if username is None:
             return
-        self.report_login()
-        login_url = 'https://www.crunchyroll.com/?a=formhandler'
-        data = urlencode_postdata({
-            'formname': 'RpcApiUser_Login',
-            'name': username,
-            'password': password,
+
+        login_page = self._download_webpage(
+            self._LOGIN_URL, None, 'Downloading login page')
+
+        login_form_str = self._search_regex(
+            r'(?P<form><form[^>]+?id=(["\'])%s\2[^>]*>)' % self._LOGIN_FORM,
+            login_page, 'login form', group='form')
+
+        post_url = extract_attributes(login_form_str).get('action')
+        if not post_url:
+            post_url = self._LOGIN_URL
+        elif not post_url.startswith('http'):
+            post_url = compat_urlparse.urljoin(self._LOGIN_URL, post_url)
+
+        login_form = self._form_hidden_inputs(self._LOGIN_FORM, login_page)
+
+        login_form.update({
+            'login_form[name]': username,
+            'login_form[password]': password,
         })
-        login_request = sanitized_Request(login_url, data)
-        login_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
-        self._download_webpage(login_request, None, False, 'Wrong login info')
+
+        response = self._download_webpage(
+            post_url, None, 'Logging in', 'Wrong login info',
+            data=urlencode_postdata(login_form),
+            headers={'Content-Type': 'application/x-www-form-urlencoded'})
+
+        # Successful login
+        if '<title>Redirecting' in response:
+            return
+
+        error = self._html_search_regex(
+            '(?s)<ul[^>]+class=["\']messages["\'][^>]*>(.+?)</ul>',
+            response, 'error message', default=None)
+        if error:
+            raise ExtractorError('Unable to login: %s' % error, expected=True)
+
+        raise ExtractorError('Unable to log in')
 
     def _real_initialize(self):
         self._login()

From c8498368549048a578d5f30773aaa9760454983c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 15 Sep 2016 21:54:48 +0700
Subject: [PATCH 6091/6219] [utils] Improve _hidden_inputs

---
 youtube_dl/extractor/common.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index ff19270ae..e413799f9 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -888,16 +888,16 @@ class InfoExtractor(object):
     def _hidden_inputs(html):
         html = re.sub(r'<!--(?:(?!<!--).)*-->', '', html)
         hidden_inputs = {}
-        for input in re.findall(r'(?i)<input([^>]+)>', html):
-            if not re.search(r'type=(["\'])(?:hidden|submit)\1', input):
+        for input in re.findall(r'(?i)(<input[^>]+>)', html):
+            attrs = extract_attributes(input)
+            if not input:
                 continue
-            name = re.search(r'(?:name|id)=(["\'])(?P<value>.+?)\1', input)
-            if not name:
+            if attrs.get('type') not in ('hidden', 'submit'):
                 continue
-            value = re.search(r'value=(["\'])(?P<value>.*?)\1', input)
-            if not value:
-                continue
-            hidden_inputs[name.group('value')] = value.group('value')
+            name = attrs.get('name') or attrs.get('id')
+            value = attrs.get('value')
+            if name and value is not None:
+                hidden_inputs[name] = value
         return hidden_inputs
 
     def _form_hidden_inputs(self, form_id, html):

From 537f753399ed9fd07fcb9285a2a3330010394c85 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 15 Sep 2016 22:17:17 +0700
Subject: [PATCH 6092/6219] [options] Improve Adobe Pass wording

---
 youtube_dl/options.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index b2e863119..100d21310 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -355,19 +355,19 @@ def parseOpts(overrideArguments=None):
     adobe_pass.add_option(
         '--ap-mso',
         dest='ap_mso', metavar='MSO',
-        help='Adobe Pass Multiple-system operator Identifier')
+        help='Adobe Pass multiple-system operator (TV provider) identifier, use --ap-list-mso for a list of available MSOs')
     adobe_pass.add_option(
         '--ap-username',
         dest='ap_username', metavar='USERNAME',
-        help='TV Provider Login with this account ID')
+        help='Multiple-system operator account login')
     adobe_pass.add_option(
         '--ap-password',
         dest='ap_password', metavar='PASSWORD',
-        help='TV Provider Account password. If this option is left out, youtube-dl will ask interactively.')
+        help='Multiple-system operator account password. If this option is left out, youtube-dl will ask interactively.')
     adobe_pass.add_option(
         '--ap-list-mso',
         action='store_true', dest='ap_list_mso', default=False,
-        help='List all supported TV Providers')
+        help='List all supported multiple-system operators')
 
     video_format = optparse.OptionGroup(parser, 'Video Format Options')
     video_format.add_option(

From d2522b86ac7d1eff1f00e21bcd976a2616b6a6d2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 15 Sep 2016 22:18:31 +0700
Subject: [PATCH 6093/6219] [options] Actually print Adobe Pass options
 sections in --help

---
 youtube_dl/options.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 100d21310..53497fbc6 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -831,6 +831,7 @@ def parseOpts(overrideArguments=None):
     parser.add_option_group(video_format)
     parser.add_option_group(subtitles)
     parser.add_option_group(authentication)
+    parser.add_option_group(adobe_pass)
     parser.add_option_group(postproc)
 
     if overrideArguments is not None:

From 1da50aa34e9fa0fd927de8197dcf2884551dd800 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 15 Sep 2016 22:24:55 +0700
Subject: [PATCH 6094/6219] [YoutubeDL] Improve Adobe Pass options' wording

---
 youtube_dl/YoutubeDL.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 29d8517a3..442aa663b 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -131,9 +131,9 @@ class YoutubeDL(object):
     username:          Username for authentication purposes.
     password:          Password for authentication purposes.
     videopassword:     Password for accessing a video.
-    ap_mso:            Adobe Pass Multiple-system operator Identifier.
-    ap_username:       TV Provider username for authentication purposes.
-    ap_password:       TV Provider password for authentication purposes.
+    ap_mso:            Adobe Pass multiple-system operator identifier.
+    ap_username:       Multiple-system operator account username.
+    ap_password:       Multiple-system operator account password.
     usenetrc:          Use netrc for authentication instead.
     verbose:           Print additional info to stdout.
     quiet:             Do not print messages to stdout.

From 2133565cec3646680600d314b93e535f6fa52339 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 15 Sep 2016 22:26:37 +0700
Subject: [PATCH 6095/6219] [extractor/common] Simplify _get_login_info

---
 youtube_dl/extractor/common.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index e413799f9..9627816b4 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -689,8 +689,6 @@ class InfoExtractor(object):
         if self._downloader is None:
             return (None, None)
 
-        username = None
-        password = None
         downloader_params = self._downloader.params
 
         # Attempt to use provided username and password or .netrc data
@@ -700,7 +698,7 @@ class InfoExtractor(object):
         else:
             username, password = self._get_netrc_login_info(netrc_machine)
 
-        return (username, password)
+        return username, password
 
     def _get_tfa_info(self, note='two-factor verification code'):
         """

From 32443dd346594d64b579af714f4828287492c464 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 15 Sep 2016 22:34:29 +0700
Subject: [PATCH 6096/6219] [extractor/common] Update _get_login_info's comment

---
 youtube_dl/extractor/common.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 9627816b4..95ea3fca5 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -683,7 +683,10 @@ class InfoExtractor(object):
     def _get_login_info(self, username_option='username', password_option='password', netrc_machine=None):
         """
         Get the login info as (username, password)
-        It will look in the netrc file using the _NETRC_MACHINE value
+        First look for the manually specified credentials using username_option
+        and password_option as keys in params dictionary. If no such credentials
+        available look in the netrc file using the netrc_machine or _NETRC_MACHINE
+        value.
         If there's no info available, return (None, None)
         """
         if self._downloader is None:

From dcce092e0aa92799f1e3a51ce5aae611af4d70d5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 15 Sep 2016 22:35:12 +0700
Subject: [PATCH 6097/6219] [extractor/common] Simplify _get_netrc_login_info
 and carry long lines

---
 youtube_dl/extractor/common.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 95ea3fca5..4f738b9fc 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -674,11 +674,13 @@ class InfoExtractor(object):
                     username = info[0]
                     password = info[2]
                 else:
-                    raise netrc.NetrcParseError('No authenticators for %s' % netrc_machine)
+                    raise netrc.NetrcParseError(
+                        'No authenticators for %s' % netrc_machine)
             except (IOError, netrc.NetrcParseError) as err:
-                self._downloader.report_warning('parsing .netrc: %s' % error_to_compat_str(err))
+                self._downloader.report_warning(
+                    'parsing .netrc: %s' % error_to_compat_str(err))
 
-        return (username, password)
+        return username, password
 
     def _get_login_info(self, username_option='username', password_option='password', netrc_machine=None):
         """

From 1dec2c8a0e00e8ed53ddd030347ce9225df9964e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 15 Sep 2016 22:47:45 +0700
Subject: [PATCH 6098/6219] [adobepass] Change mvpd cache section name In order
 to better emphasize it's relation to Adobe Pass

---
 youtube_dl/extractor/adobepass.py | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/adobepass.py b/youtube_dl/extractor/adobepass.py
index 8ef5a96ce..01932e5e6 100644
--- a/youtube_dl/extractor/adobepass.py
+++ b/youtube_dl/extractor/adobepass.py
@@ -32,6 +32,7 @@ MSO_INFO = {
 class AdobePassIE(InfoExtractor):
     _SERVICE_PROVIDER_TEMPLATE = 'https://sp.auth.adobe.com/adobe-services/%s'
     _USER_AGENT = 'Mozilla/5.0 (X11; Linux i686; rv:47.0) Gecko/20100101 Firefox/47.0'
+    _MVPD_CACHE = 'ap-mvpd'
 
     @staticmethod
     def _get_mvpd_resource(provider_id, title, guid, rating):
@@ -85,7 +86,7 @@ class AdobePassIE(InfoExtractor):
         guid = xml_text(resource, 'guid')
         count = 0
         while count < 2:
-            requestor_info = self._downloader.cache.load('mvpd', requestor_id) or {}
+            requestor_info = self._downloader.cache.load(self._MVPD_CACHE, requestor_id) or {}
             authn_token = requestor_info.get('authn_token')
             if authn_token and is_expired(authn_token, 'simpleTokenExpires'):
                 authn_token = None
@@ -125,12 +126,12 @@ class AdobePassIE(InfoExtractor):
                         'requestor_id': requestor_id,
                     }), headers=mvpd_headers)
                 if '<pendingLogout' in session:
-                    self._downloader.cache.store('mvpd', requestor_id, {})
+                    self._downloader.cache.store(self._MVPD_CACHE, requestor_id, {})
                     count += 1
                     continue
                 authn_token = unescapeHTML(xml_text(session, 'authnToken'))
                 requestor_info['authn_token'] = authn_token
-                self._downloader.cache.store('mvpd', requestor_id, requestor_info)
+                self._downloader.cache.store(self._MVPD_CACHE, requestor_id, requestor_info)
 
             authz_token = requestor_info.get(guid)
             if authz_token and is_expired(authz_token, 'simpleTokenTTL'):
@@ -146,12 +147,12 @@ class AdobePassIE(InfoExtractor):
                         'userMeta': '1',
                     }), headers=mvpd_headers)
                 if '<pendingLogout' in authorize:
-                    self._downloader.cache.store('mvpd', requestor_id, {})
+                    self._downloader.cache.store(self._MVPD_CACHE, requestor_id, {})
                     count += 1
                     continue
                 authz_token = unescapeHTML(xml_text(authorize, 'authzToken'))
                 requestor_info[guid] = authz_token
-                self._downloader.cache.store('mvpd', requestor_id, requestor_info)
+                self._downloader.cache.store(self._MVPD_CACHE, requestor_id, requestor_info)
 
             mvpd_headers.update({
                 'ap_19': xml_text(authn_token, 'simpleSamlNameID'),
@@ -167,7 +168,7 @@ class AdobePassIE(InfoExtractor):
                     'hashed_guid': 'false',
                 }), headers=mvpd_headers)
             if '<pendingLogout' in short_authorize:
-                self._downloader.cache.store('mvpd', requestor_id, {})
+                self._downloader.cache.store(self._MVPD_CACHE, requestor_id, {})
                 count += 1
                 continue
             return short_authorize

From 490b755769a364ca0624390453e36321d5182d3e Mon Sep 17 00:00:00 2001
From: stepshal <nessento@openmailbox.org>
Date: Wed, 14 Sep 2016 23:03:26 +0700
Subject: [PATCH 6099/6219] Improve some id regexes

---
 youtube_dl/extractor/canvas.py | 2 +-
 youtube_dl/extractor/nfl.py    | 2 +-
 youtube_dl/extractor/npo.py    | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/canvas.py b/youtube_dl/extractor/canvas.py
index ef0691dcd..d183d5d52 100644
--- a/youtube_dl/extractor/canvas.py
+++ b/youtube_dl/extractor/canvas.py
@@ -71,7 +71,7 @@ class CanvasIE(InfoExtractor):
             webpage)).strip()
 
         video_id = self._html_search_regex(
-            r'data-video=(["\'])(?P<id>.+?)\1', webpage, 'video id', group='id')
+            r'data-video=(["\'])(?P<id>(?:(?!\1).)+)\1', webpage, 'video id', group='id')
 
         data = self._download_json(
             'https://mediazone.vrt.be/api/v1/%s/assets/%s'
diff --git a/youtube_dl/extractor/nfl.py b/youtube_dl/extractor/nfl.py
index 200874d68..3930d16f1 100644
--- a/youtube_dl/extractor/nfl.py
+++ b/youtube_dl/extractor/nfl.py
@@ -165,7 +165,7 @@ class NFLIE(InfoExtractor):
             group='config'))
         # For articles, the id in the url is not the video id
         video_id = self._search_regex(
-            r'(?:<nflcs:avplayer[^>]+data-content[Ii]d\s*=\s*|content[Ii]d\s*:\s*)(["\'])(?P<id>.+?)\1',
+            r'(?:<nflcs:avplayer[^>]+data-content[Ii]d\s*=\s*|content[Ii]d\s*:\s*)(["\'])(?P<id>(?:(?!\1).)+)\1',
             webpage, 'video id', default=video_id, group='id')
         config = self._download_json(config_url, video_id, 'Downloading player config')
         url_template = NFLIE.prepend_host(
diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index 87f5675c7..3293bdb17 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -429,7 +429,7 @@ class SchoolTVIE(InfoExtractor):
         display_id = self._match_id(url)
         webpage = self._download_webpage(url, display_id)
         video_id = self._search_regex(
-            r'data-mid=(["\'])(?P<id>.+?)\1', webpage, 'video_id', group='id')
+            r'data-mid=(["\'])(?P<id>(?:(?!\1).)+)\1', webpage, 'video_id', group='id')
         return {
             '_type': 'url_transparent',
             'ie_key': 'NPO',

From e6bf3621e703a7cd0d62736a1765b0ccff5adfe6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 15 Sep 2016 23:31:16 +0700
Subject: [PATCH 6100/6219] [ChangeLog] Actualize

---
 ChangeLog | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index c3c8bf037..cd1f2fdf1 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,8 +1,26 @@
 version <unreleased>
 
+Core
+* Improve _hidden_inputs
++ Introduce improved explicit Adobe Pass support
++ Add --ap-mso to provide multiple-system operator identifier
++ Add --ap-username to provide MSO account username
++ Add --ap-password to provide MSO account password
++ Add --ap-list-mso to list all supported MSOs
++ Add support for Rogers Cable multiple-system operator (#10606)
+
 Extractors
-* [kwuo] Improve error detection (#10650)
+* [crunchyroll] Fix authentication (#10655)
+* [twitch] Fix API calls (#10654, #10660)
++ [bellmedia] Add support for more Bell Media Television sites
+* [franceinter] Fix extraction (#10538, #2105)
+* [kuwo] Improve error detection (#10650)
++ [go] Add support for free full episodes (#10439)
 * [bilibili] Fix extraction for specific videos (#10647)
+* [nhk] Fix extraction (#10633)
+* [kaltura] Improve audio detection
+* [kaltura] Skip chun format
++ [vimeo:ondemand] Pass Referer along with embed URL (#10624)
 + [nbc] Add support for NBC Olympics (#10361)
 
 

From f5e008d134f5e69920829cfd7a5ce5ae57d275c1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 15 Sep 2016 23:46:11 +0700
Subject: [PATCH 6101/6219] release 2016.09.15

---
 .github/ISSUE_TEMPLATE.md |  8 ++++----
 ChangeLog                 |  2 +-
 README.md                 | 11 +++++++++++
 docs/supportedsites.md    |  3 ++-
 youtube_dl/version.py     |  2 +-
 5 files changed, 19 insertions(+), 7 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index e87fed573..61cea757c 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.09.11.1*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.09.11.1**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.09.15*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.09.15**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.09.11.1
+[debug] youtube-dl version 2016.09.15
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
@@ -55,4 +55,4 @@ $ youtube-dl -v <your command line>
 ### Description of your *issue*, suggested solution and other information
 
 Explanation of your *issue* in arbitrary form goes here. Please make sure the [description is worded well enough to be understood](https://github.com/rg3/youtube-dl#is-the-description-of-the-issue-itself-sufficient). Provide as much context and examples as possible.
-If work on your *issue* required an account credentials please provide them or explain how one can obtain them.
+If work on your *issue* requires account credentials please provide them or explain how one can obtain them.
diff --git a/ChangeLog b/ChangeLog
index cd1f2fdf1..4583537ac 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2016.09.15
 
 Core
 * Improve _hidden_inputs
diff --git a/README.md b/README.md
index 7543f81ac..4debe15fe 100644
--- a/README.md
+++ b/README.md
@@ -358,6 +358,17 @@ which means you can modify it, redistribute it or use it however you like.
     -n, --netrc                      Use .netrc authentication data
     --video-password PASSWORD        Video password (vimeo, smotri, youku)
 
+## Adobe Pass Options:
+    --ap-mso MSO                     Adobe Pass multiple-system operator (TV
+                                     provider) identifier, use --ap-list-mso for
+                                     a list of available MSOs
+    --ap-username USERNAME           Multiple-system operator account login
+    --ap-password PASSWORD           Multiple-system operator account password.
+                                     If this option is left out, youtube-dl will
+                                     ask interactively.
+    --ap-list-mso                    List all supported multiple-system
+                                     operators
+
 ## Post-processing Options:
     -x, --extract-audio              Convert video files to audio-only files
                                      (requires ffmpeg or avconv and ffprobe or
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 7a7b268d3..fcb618561 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -89,6 +89,7 @@
  - **BeatportPro**
  - **Beeg**
  - **BehindKink**
+ - **BellMedia**
  - **Bet**
  - **Bigflix**
  - **Bild**: Bild.de
@@ -169,7 +170,6 @@
  - **CSNNE**
  - **CSpan**: C-SPAN
  - **CtsNews**: 華視新聞
- - **CTV**
  - **CTVNews**
  - **culturebox.francetvinfo.fr**
  - **CultureUnplugged**
@@ -445,6 +445,7 @@
  - **NBA**
  - **NBC**
  - **NBCNews**
+ - **NBCOlympics**
  - **NBCSports**
  - **NBCSportsVPlayer**
  - **ndr**: NDR.de - Norddeutscher Rundfunk
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 903aede58..081fd6ef0 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.09.11.1'
+__version__ = '2016.09.15'

From 9d8985a165ebdc9fd8d72e7536253c42162b58a6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 16 Sep 2016 00:54:34 +0700
Subject: [PATCH 6102/6219] [tv4] Fix hls and hds formats (Closes #10659)

---
 youtube_dl/extractor/tv4.py | 49 ++++++++++++++++++++++++-------------
 1 file changed, 32 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/tv4.py b/youtube_dl/extractor/tv4.py
index 343edf206..5d2d8f132 100644
--- a/youtube_dl/extractor/tv4.py
+++ b/youtube_dl/extractor/tv4.py
@@ -2,9 +2,13 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..compat import compat_str
 from ..utils import (
     ExtractorError,
+    int_or_none,
     parse_iso8601,
+    try_get,
+    update_url_query,
 )
 
 
@@ -65,36 +69,47 @@ class TV4IE(InfoExtractor):
         video_id = self._match_id(url)
 
         info = self._download_json(
-            'http://www.tv4play.se/player/assets/%s.json' % video_id, video_id, 'Downloading video info JSON')
+            'http://www.tv4play.se/player/assets/%s.json' % video_id,
+            video_id, 'Downloading video info JSON')
 
         # If is_geo_restricted is true, it doesn't necessarily mean we can't download it
-        if info['is_geo_restricted']:
+        if info.get('is_geo_restricted'):
             self.report_warning('This content might not be available in your country due to licensing restrictions.')
-        if info['requires_subscription']:
+        if info.get('requires_subscription'):
             raise ExtractorError('This content requires subscription.', expected=True)
 
-        sources_data = self._download_json(
-            'https://prima.tv4play.se/api/web/asset/%s/play.json?protocol=http&videoFormat=MP4' % video_id, video_id, 'Downloading sources JSON')
-        sources = sources_data['playback']
+        title = info['title']
 
         formats = []
-        for item in sources.get('items', {}).get('item', []):
-            ext, bitrate = item['mediaFormat'], item['bitrate']
-            formats.append({
-                'format_id': '%s_%s' % (ext, bitrate),
-                'tbr': bitrate,
-                'ext': ext,
-                'url': item['url'],
-            })
+        # http formats are linked with unresolvable host
+        for kind in ('hls', ''):
+            data = self._download_json(
+                'https://prima.tv4play.se/api/web/asset/%s/play.json' % video_id,
+                video_id, 'Downloading sources JSON', query={
+                    'protocol': kind,
+                    'videoFormat': 'MP4+WEBVTTS+WEBVTT',
+                })
+            item = try_get(data, lambda x: x['playback']['items']['item'], dict)
+            manifest_url = item.get('url')
+            if not isinstance(manifest_url, compat_str):
+                continue
+            if kind == 'hls':
+                formats.extend(self._extract_m3u8_formats(
+                    manifest_url, video_id, 'mp4', entry_protocol='m3u8_native',
+                    m3u8_id=kind, fatal=False))
+            else:
+                formats.extend(self._extract_f4m_formats(
+                    update_url_query(manifest_url, {'hdcore': '3.8.0'}),
+                    video_id, f4m_id='hds', fatal=False))
         self._sort_formats(formats)
 
         return {
             'id': video_id,
-            'title': info['title'],
+            'title': title,
             'formats': formats,
             'description': info.get('description'),
             'timestamp': parse_iso8601(info.get('broadcast_date_time')),
-            'duration': info.get('duration'),
+            'duration': int_or_none(info.get('duration')),
             'thumbnail': info.get('image'),
-            'is_live': sources.get('live'),
+            'is_live': info.get('is_live') is True,
         }

From 52dc8a9b3f1af7abda6652a75b906d70809c475d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 16 Sep 2016 22:02:59 +0700
Subject: [PATCH 6103/6219] [franceinter] Fix upload date extraction

---
 youtube_dl/extractor/franceinter.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/franceinter.py b/youtube_dl/extractor/franceinter.py
index 0d58f89c5..1a1232ade 100644
--- a/youtube_dl/extractor/franceinter.py
+++ b/youtube_dl/extractor/franceinter.py
@@ -10,14 +10,14 @@ class FranceInterIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?franceinter\.fr/emissions/(?P<id>[^?#]+)'
 
     _TEST = {
-        'url': 'https://www.franceinter.fr/emissions/la-marche-de-l-histoire/la-marche-de-l-histoire-18-decembre-2013',
-        'md5': '4764932e466e6f6c79c317d2e74f6884',
+        'url': 'https://www.franceinter.fr/emissions/la-tete-au-carre/la-tete-au-carre-14-septembre-2016',
+        'md5': '4e3aeb58fe0e83d7b0581fa213c409d0',
         'info_dict': {
-            'id': 'la-marche-de-l-histoire/la-marche-de-l-histoire-18-decembre-2013',
+            'id': 'la-tete-au-carre/la-tete-au-carre-14-septembre-2016',
             'ext': 'mp3',
-            'title': 'L’Histoire dans les jeux vidéo du 18 décembre 2013 - France Inter',
-            'description': 'md5:7f2ce449894d1e585932273080fb410d',
-            'upload_date': '20131218',
+            'title': 'Et si les rêves pouvaient nous aider à agir dans notre vie quotidienne ?',
+            'description': 'md5:a245dd62cf5bf51de915f8d9956d180a',
+            'upload_date': '20160914',
         },
     }
 
@@ -39,7 +39,7 @@ class FranceInterIE(InfoExtractor):
         if upload_date_str:
             upload_date_list = upload_date_str.split()
             upload_date_list.reverse()
-            upload_date_list[1] = compat_str(month_by_name(upload_date_list[1], lang='fr'))
+            upload_date_list[1] = '%02d' % (month_by_name(upload_date_list[1], lang='fr') or 0)
             upload_date = ''.join(upload_date_list)
         else:
             upload_date = None

From 98b7506e96b5ac107a777d8bb8900623d832fba4 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 16 Sep 2016 17:36:22 +0100
Subject: [PATCH 6104/6219] [toutv] add support for authentication(closes
 #10669)

---
 youtube_dl/extractor/radiocanada.py | 55 ++++++++++++++++----------
 youtube_dl/extractor/toutv.py       | 60 ++++++++++++++++++++++++++++-
 2 files changed, 92 insertions(+), 23 deletions(-)

diff --git a/youtube_dl/extractor/radiocanada.py b/youtube_dl/extractor/radiocanada.py
index 8ec402646..6751270ee 100644
--- a/youtube_dl/extractor/radiocanada.py
+++ b/youtube_dl/extractor/radiocanada.py
@@ -13,6 +13,7 @@ from ..utils import (
     xpath_element,
     ExtractorError,
     determine_protocol,
+    unsmuggle_url,
 )
 
 
@@ -35,28 +36,51 @@ class RadioCanadaIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
+        url, smuggled_data = unsmuggle_url(url, {})
         app_code, video_id = re.match(self._VALID_URL, url).groups()
 
-        device_types = ['ipad', 'android']
+        metadata = self._download_xml(
+            'http://api.radio-canada.ca/metaMedia/v1/index.ashx',
+            video_id, note='Downloading metadata XML', query={
+                'appCode': app_code,
+                'idMedia': video_id,
+            })
+
+        def get_meta(name):
+            el = find_xpath_attr(metadata, './/Meta', 'name', name)
+            return el.text if el is not None else None
+
+        if get_meta('protectionType'):
+            raise ExtractorError('This video is DRM protected.', expected=True)
+
+        device_types = ['ipad']
         if app_code != 'toutv':
             device_types.append('flash')
+        if not smuggled_data:
+            device_types.append('android')
 
         formats = []
         # TODO: extract f4m formats
         # f4m formats can be extracted using flashhd device_type but they produce unplayable file
         for device_type in device_types:
-            v_data = self._download_xml(
-                'http://api.radio-canada.ca/validationMedia/v1/Validation.ashx',
-                video_id, note='Downloading %s XML' % device_type, query={
-                    'appCode': app_code,
-                    'idMedia': video_id,
-                    'connectionType': 'broadband',
-                    'multibitrate': 'true',
-                    'deviceType': device_type,
+            validation_url = 'http://api.radio-canada.ca/validationMedia/v1/Validation.ashx'
+            query = {
+                'appCode': app_code,
+                'idMedia': video_id,
+                'connectionType': 'broadband',
+                'multibitrate': 'true',
+                'deviceType': device_type,
+            }
+            if smuggled_data:
+                validation_url = 'https://services.radio-canada.ca/media/validation/v2/'
+                query.update(smuggled_data)
+            else:
+                query.update({
                     # paysJ391wsHjbOJwvCs26toz and bypasslock are used to bypass geo-restriction
                     'paysJ391wsHjbOJwvCs26toz': 'CA',
                     'bypasslock': 'NZt5K62gRqfc',
-                }, fatal=False)
+                })
+            v_data = self._download_xml(validation_url, video_id, note='Downloading %s XML' % device_type, query=query, fatal=False)
             v_url = xpath_text(v_data, 'url')
             if not v_url:
                 continue
@@ -101,17 +125,6 @@ class RadioCanadaIE(InfoExtractor):
                                 f4m_id='hds', fatal=False))
         self._sort_formats(formats)
 
-        metadata = self._download_xml(
-            'http://api.radio-canada.ca/metaMedia/v1/index.ashx',
-            video_id, note='Downloading metadata XML', query={
-                'appCode': app_code,
-                'idMedia': video_id,
-            })
-
-        def get_meta(name):
-            el = find_xpath_attr(metadata, './/Meta', 'name', name)
-            return el.text if el is not None else None
-
         return {
             'id': video_id,
             'title': get_meta('Title'),
diff --git a/youtube_dl/extractor/toutv.py b/youtube_dl/extractor/toutv.py
index 54c2d0aa6..d2d5c1171 100644
--- a/youtube_dl/extractor/toutv.py
+++ b/youtube_dl/extractor/toutv.py
@@ -2,12 +2,22 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..utils import int_or_none
+from ..utils import (
+    int_or_none,
+    js_to_json,
+    ExtractorError,
+    urlencode_postdata,
+    extract_attributes,
+    smuggle_url,
+)
 
 
 class TouTvIE(InfoExtractor):
+    _NETRC_MACHINE = 'toutv'
     IE_NAME = 'tou.tv'
     _VALID_URL = r'https?://ici\.tou\.tv/(?P<id>[a-zA-Z0-9_-]+/S[0-9]+E[0-9]+)'
+    _access_token = None
+    _claims = None
 
     _TEST = {
         'url': 'http://ici.tou.tv/garfield-tout-court/S2015E17',
@@ -22,18 +32,64 @@ class TouTvIE(InfoExtractor):
             # m3u8 download
             'skip_download': True,
         },
+        'skip': '404 Not Found',
     }
 
+    def _real_initialize(self):
+        email, password = self._get_login_info()
+        if email is None:
+            return
+        state = 'http://ici.tou.tv//'
+        webpage = self._download_webpage(state, None, 'Downloading homepage')
+        toutvlogin = self._parse_json(self._search_regex(
+            r'(?s)toutvlogin\s*=\s*({.+?});', webpage, 'toutvlogin'), None, js_to_json)
+        authorize_url = toutvlogin['host'] + '/auth/oauth/v2/authorize'
+        login_webpage = self._download_webpage(
+            authorize_url, None, 'Downloading login page', query={
+                'client_id': toutvlogin['clientId'],
+                'redirect_uri': 'https://ici.tou.tv/login/loginCallback',
+                'response_type': 'token',
+                'scope': 'media-drmt openid profile email id.write media-validation.read.privileged',
+                'state': state,
+            })
+        login_form = self._search_regex(
+            r'(?s)(<form[^>]+id="Form-login".+?</form>)', login_webpage, 'login form')
+        form_data = self._hidden_inputs(login_form)
+        form_data.update({
+            'login-email': email,
+            'login-password': password,
+        })
+        post_url = extract_attributes(login_form).get('action') or authorize_url
+        _, urlh = self._download_webpage_handle(
+            post_url, None, 'Logging in', data=urlencode_postdata(form_data))
+        self._access_token = self._search_regex(
+            r'access_token=([\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})',
+            urlh.geturl(), 'access token')
+        self._claims = self._download_json(
+            'https://services.radio-canada.ca/media/validation/v2/getClaims',
+            None, 'Extracting Claims', query={
+                'token': self._access_token,
+                'access_token': self._access_token,
+            })['claims']
+
     def _real_extract(self, url):
         path = self._match_id(url)
         metadata = self._download_json('http://ici.tou.tv/presentation/%s' % path, path)
+        if metadata.get('IsDrm'):
+            raise ExtractorError('This video is DRM protected.', expected=True)
         video_id = metadata['IdMedia']
         details = metadata['Details']
         title = details['OriginalTitle']
+        video_url = 'radiocanada:%s:%s' % (metadata.get('AppCode', 'toutv'), video_id)
+        if self._access_token and self._claims:
+            video_url = smuggle_url(video_url, {
+                'access_token': self._access_token,
+                'claims': self._claims,
+            })
 
         return {
             '_type': 'url_transparent',
-            'url': 'radiocanada:%s:%s' % (metadata.get('AppCode', 'toutv'), video_id),
+            'url': video_url,
             'id': video_id,
             'title': title,
             'thumbnail': details.get('ImageUrl'),

From 6ad0219556cefe60239027633193cc9f1dc9fb1d Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 16 Sep 2016 19:30:38 +0100
Subject: [PATCH 6105/6219] [common] add helper method for Wowza Streaming
 Engine format extraction

---
 youtube_dl/extractor/common.py      | 43 +++++++++++++++++++++++++++++
 youtube_dl/extractor/vier.py        |  4 +--
 youtube_dl/extractor/vodplatform.py | 25 ++---------------
 youtube_dl/extractor/vrt.py         | 40 ++++++---------------------
 4 files changed, 55 insertions(+), 57 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 4f738b9fc..c00023458 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1795,6 +1795,49 @@ class InfoExtractor(object):
             m3u8_id='hls', fatal=False))
         return formats
 
+    def _extract_wowza_formats(self, url, video_id, m3u8_entry_protocol='m3u8_native', skip_protocols=[]):
+        url = re.sub(r'/(?:manifest|playlist|jwplayer)\.(?:m3u8|f4m|mpd|smil)', '', url)
+        url_base = self._search_regex(r'(?:https?|rtmp|rtsp)(://[^?]+)', url, 'format url')
+        http_base_url = 'http' + url_base
+        formats = []
+        if 'm3u8' not in skip_protocols:
+            formats.extend(self._extract_m3u8_formats(
+                http_base_url + '/playlist.m3u8', video_id, 'mp4',
+                m3u8_entry_protocol, m3u8_id='hls', fatal=False))
+        if 'f4m' not in skip_protocols:
+            formats.extend(self._extract_f4m_formats(
+                http_base_url + '/manifest.f4m',
+                video_id, f4m_id='hds', fatal=False))
+        if re.search(r'(?:/smil:|\.smil)', url_base):
+            if 'dash' not in skip_protocols:
+                formats.extend(self._extract_mpd_formats(
+                    http_base_url + '/manifest.mpd',
+                    video_id, mpd_id='dash', fatal=False))
+            if 'smil' not in skip_protocols:
+                rtmp_formats = self._extract_smil_formats(
+                    http_base_url + '/jwplayer.smil',
+                    video_id, fatal=False)
+                for rtmp_format in rtmp_formats:
+                    rtsp_format = rtmp_format.copy()
+                    rtsp_format['url'] = '%s/%s' % (rtmp_format['url'], rtmp_format['play_path'])
+                    del rtsp_format['play_path']
+                    del rtsp_format['ext']
+                    rtsp_format.update({
+                        'url': rtsp_format['url'].replace('rtmp://', 'rtsp://'),
+                        'format_id': rtmp_format['format_id'].replace('rtmp', 'rtsp'),
+                        'protocol': 'rtsp',
+                    })
+                    formats.extend([rtmp_format, rtsp_format])
+        else:
+            for protocol in ('rtmp', 'rtsp'):
+                if protocol not in skip_protocols:
+                    formats.append({
+                        'url': protocol + url_base,
+                        'format_id': protocol,
+                        'protocol': protocol,
+                    })
+        return formats
+
     def _live_title(self, name):
         """ Generate the title for a live video """
         now = datetime.datetime.now()
diff --git a/youtube_dl/extractor/vier.py b/youtube_dl/extractor/vier.py
index 6645c6186..dc142a245 100644
--- a/youtube_dl/extractor/vier.py
+++ b/youtube_dl/extractor/vier.py
@@ -48,8 +48,8 @@ class VierIE(InfoExtractor):
             [r'data-filename="([^"]+)"', r'"filename"\s*:\s*"([^"]+)"'],
             webpage, 'filename')
 
-        playlist_url = 'http://vod.streamcloud.be/%s/mp4:_definst_/%s.mp4/playlist.m3u8' % (application, filename)
-        formats = self._extract_m3u8_formats(playlist_url, display_id, 'mp4')
+        playlist_url = 'http://vod.streamcloud.be/%s/_definst_/mp4:%s.mp4/playlist.m3u8' % (application, filename)
+        formats = self._extract_wowza_formats(playlist_url, display_id)
         self._sort_formats(formats)
 
         title = self._og_search_title(webpage, default=display_id)
diff --git a/youtube_dl/extractor/vodplatform.py b/youtube_dl/extractor/vodplatform.py
index 7bdd8b1dc..239644340 100644
--- a/youtube_dl/extractor/vodplatform.py
+++ b/youtube_dl/extractor/vodplatform.py
@@ -25,29 +25,8 @@ class VODPlatformIE(InfoExtractor):
         title = unescapeHTML(self._og_search_title(webpage))
         hidden_inputs = self._hidden_inputs(webpage)
 
-        base_url = self._search_regex(
-            '(.*/)(?:playlist.m3u8|manifest.mpd)',
-            hidden_inputs.get('HiddenmyhHlsLink') or hidden_inputs['HiddenmyDashLink'],
-            'base url')
-        formats = self._extract_m3u8_formats(
-            base_url + 'playlist.m3u8', video_id, 'mp4',
-            'm3u8_native', m3u8_id='hls', fatal=False)
-        formats.extend(self._extract_mpd_formats(
-            base_url + 'manifest.mpd', video_id,
-            mpd_id='dash', fatal=False))
-        rtmp_formats = self._extract_smil_formats(
-            base_url + 'jwplayer.smil', video_id, fatal=False)
-        for rtmp_format in rtmp_formats:
-            rtsp_format = rtmp_format.copy()
-            rtsp_format['url'] = '%s/%s' % (rtmp_format['url'], rtmp_format['play_path'])
-            del rtsp_format['play_path']
-            del rtsp_format['ext']
-            rtsp_format.update({
-                'url': rtsp_format['url'].replace('rtmp://', 'rtsp://'),
-                'format_id': rtmp_format['format_id'].replace('rtmp', 'rtsp'),
-                'protocol': 'rtsp',
-            })
-            formats.extend([rtmp_format, rtsp_format])
+        formats = self._extract_wowza_formats(
+            hidden_inputs.get('HiddenmyhHlsLink') or hidden_inputs['HiddenmyDashLink'], video_id, skip_protocols=['f4m', 'smil'])
         self._sort_formats(formats)
 
         return {
diff --git a/youtube_dl/extractor/vrt.py b/youtube_dl/extractor/vrt.py
index bec7ab327..00c72e346 100644
--- a/youtube_dl/extractor/vrt.py
+++ b/youtube_dl/extractor/vrt.py
@@ -5,7 +5,6 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
-    determine_ext,
     float_or_none,
 )
 
@@ -75,7 +74,6 @@ class VRTIE(InfoExtractor):
         },
         {
             'url': 'http://cobra.canvas.be/cm/cobra/videozone/rubriek/film-videozone/1.2377055',
-            'md5': '',
             'info_dict': {
                 'id': '2377055',
                 'ext': 'mp4',
@@ -119,39 +117,17 @@ class VRTIE(InfoExtractor):
                 video_id, 'mp4', m3u8_id='hls', fatal=False))
 
         if src:
-            if determine_ext(src) == 'm3u8':
-                formats.extend(self._extract_m3u8_formats(
-                    src, video_id, 'mp4', entry_protocol='m3u8_native',
-                    m3u8_id='hls', fatal=False))
-                formats.extend(self._extract_f4m_formats(
-                    src.replace('playlist.m3u8', 'manifest.f4m'),
-                    video_id, f4m_id='hds', fatal=False))
-                if 'data-video-geoblocking="true"' not in webpage:
-                    rtmp_formats = self._extract_smil_formats(
-                        src.replace('playlist.m3u8', 'jwplayer.smil'),
-                        video_id, fatal=False)
-                    formats.extend(rtmp_formats)
-                    for rtmp_format in rtmp_formats:
-                        rtmp_format_c = rtmp_format.copy()
-                        rtmp_format_c['url'] = '%s/%s' % (rtmp_format['url'], rtmp_format['play_path'])
-                        del rtmp_format_c['play_path']
-                        del rtmp_format_c['ext']
-                        http_format = rtmp_format_c.copy()
+            formats = self._extract_wowza_formats(src, video_id)
+            if 'data-video-geoblocking="true"' not in webpage:
+                for f in formats:
+                    if f['url'].startswith('rtsp://'):
+                        http_format = f.copy()
                         http_format.update({
-                            'url': rtmp_format_c['url'].replace('rtmp://', 'http://').replace('vod.', 'download.').replace('/_definst_/', '/').replace('mp4:', ''),
-                            'format_id': rtmp_format['format_id'].replace('rtmp', 'http'),
+                            'url': f['url'].replace('rtsp://', 'http://').replace('vod.', 'download.').replace('/_definst_/', '/').replace('mp4:', ''),
+                            'format_id': f['format_id'].replace('rtsp', 'http'),
                             'protocol': 'http',
                         })
-                        rtsp_format = rtmp_format_c.copy()
-                        rtsp_format.update({
-                            'url': rtsp_format['url'].replace('rtmp://', 'rtsp://'),
-                            'format_id': rtmp_format['format_id'].replace('rtmp', 'rtsp'),
-                            'protocol': 'rtsp',
-                        })
-                        formats.extend([http_format, rtsp_format])
-            else:
-                formats.extend(self._extract_f4m_formats(
-                    '%s/manifest.f4m' % src, video_id, f4m_id='hds', fatal=False))
+                        formats.append(http_format)
 
         if not formats and 'data-video-geoblocking="true"' in webpage:
             self.raise_geo_restricted('This video is only available in Belgium')

From 7d273a387aade7665cd25eee69d94ee615d9a4b9 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 16 Sep 2016 19:31:39 +0100
Subject: [PATCH 6106/6219] [mangomolo] add support for Mangomolo embeds

---
 youtube_dl/extractor/awaan.py      | 63 ++++++++++++------------------
 youtube_dl/extractor/extractors.py |  4 ++
 youtube_dl/extractor/generic.py    | 29 ++++++++++++++
 youtube_dl/extractor/mangomolo.py  | 54 +++++++++++++++++++++++++
 4 files changed, 111 insertions(+), 39 deletions(-)
 create mode 100644 youtube_dl/extractor/mangomolo.py

diff --git a/youtube_dl/extractor/awaan.py b/youtube_dl/extractor/awaan.py
index bdf23c6a9..66d7515bc 100644
--- a/youtube_dl/extractor/awaan.py
+++ b/youtube_dl/extractor/awaan.py
@@ -50,25 +50,6 @@ class AWAANBaseIE(InfoExtractor):
             'is_live': is_live,
         }
 
-    def _extract_video_formats(self, webpage, video_id, m3u8_entry_protocol):
-        formats = []
-        format_url_base = 'http' + self._html_search_regex(
-            [
-                r'file\s*:\s*"https?(://[^"]+)/playlist.m3u8',
-                r'<a[^>]+href="rtsp(://[^"]+)"'
-            ], webpage, 'format url')
-        formats.extend(self._extract_mpd_formats(
-            format_url_base + '/manifest.mpd',
-            video_id, mpd_id='dash', fatal=False))
-        formats.extend(self._extract_m3u8_formats(
-            format_url_base + '/playlist.m3u8', video_id, 'mp4',
-            m3u8_entry_protocol, m3u8_id='hls', fatal=False))
-        formats.extend(self._extract_f4m_formats(
-            format_url_base + '/manifest.f4m',
-            video_id, f4m_id='hds', fatal=False))
-        self._sort_formats(formats)
-        return formats
-
 
 class AWAANVideoIE(AWAANBaseIE):
     IE_NAME = 'awaan:video'
@@ -99,16 +80,18 @@ class AWAANVideoIE(AWAANBaseIE):
             video_id, headers={'Origin': 'http://awaan.ae'})
         info = self._parse_video_data(video_data, video_id, False)
 
-        webpage = self._download_webpage(
-            'http://admin.mangomolo.com/analytics/index.php/customers/embed/video?' +
-            compat_urllib_parse_urlencode({
-                'id': video_data['id'],
-                'user_id': video_data['user_id'],
-                'signature': video_data['signature'],
-                'countries': 'Q0M=',
-                'filter': 'DENY',
-            }), video_id)
-        info['formats'] = self._extract_video_formats(webpage, video_id, 'm3u8_native')
+        embed_url = 'http://admin.mangomolo.com/analytics/index.php/customers/embed/video?' + compat_urllib_parse_urlencode({
+            'id': video_data['id'],
+            'user_id': video_data['user_id'],
+            'signature': video_data['signature'],
+            'countries': 'Q0M=',
+            'filter': 'DENY',
+        })
+        info.update({
+            '_type': 'url_transparent',
+            'url': embed_url,
+            'ie_key': 'MangomoloVideo',
+        })
         return info
 
 
@@ -138,16 +121,18 @@ class AWAANLiveIE(AWAANBaseIE):
             channel_id, headers={'Origin': 'http://awaan.ae'})
         info = self._parse_video_data(channel_data, channel_id, True)
 
-        webpage = self._download_webpage(
-            'http://admin.mangomolo.com/analytics/index.php/customers/embed/index?' +
-            compat_urllib_parse_urlencode({
-                'id': base64.b64encode(channel_data['user_id'].encode()).decode(),
-                'channelid': base64.b64encode(channel_data['id'].encode()).decode(),
-                'signature': channel_data['signature'],
-                'countries': 'Q0M=',
-                'filter': 'DENY',
-            }), channel_id)
-        info['formats'] = self._extract_video_formats(webpage, channel_id, 'm3u8')
+        embed_url = 'http://admin.mangomolo.com/analytics/index.php/customers/embed/index?' + compat_urllib_parse_urlencode({
+            'id': base64.b64encode(channel_data['user_id'].encode()).decode(),
+            'channelid': base64.b64encode(channel_data['id'].encode()).decode(),
+            'signature': channel_data['signature'],
+            'countries': 'Q0M=',
+            'filter': 'DENY',
+        })
+        info.update({
+            '_type': 'url_transparent',
+            'url': embed_url,
+            'ie_key': 'MangomoloLive',
+        })
         return info
 
 
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index dd0579425..4baf4cd48 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -472,6 +472,10 @@ from .macgamestore import MacGameStoreIE
 from .mailru import MailRuIE
 from .makerschannel import MakersChannelIE
 from .makertv import MakerTVIE
+from .mangomolo import (
+    MangomoloVideoIE,
+    MangomoloLiveIE,
+)
 from .matchtv import MatchTVIE
 from .mdr import MDRIE
 from .meta import METAIE
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 2e46ca179..e01305942 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -2254,6 +2254,35 @@ class GenericIE(InfoExtractor):
             return self.url_result(
                 self._proto_relative_url(unescapeHTML(mobj.group('url'))), 'VODPlatform')
 
+        # Look for Mangomolo embeds
+        mobj = re.search(
+            r'''(?x)<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?admin\.mangomolo.com/analytics/index\.php/customers/embed/
+                (?:
+                    video\?.*?\bid=(?P<video_id>\d+)|
+                    index\?.*?\bchannelid=(?P<channel_id>(?:[A-Za-z0-9+/=]|%2B|%2F|%3D)+)
+                ).+?)\1''', webpage)
+        if mobj is not None:
+            info = {
+                '_type': 'url_transparent',
+                'url': self._proto_relative_url(unescapeHTML(mobj.group('url'))),
+                'title': video_title,
+                'description': video_description,
+                'thumbnail': video_thumbnail,
+                'uploader': video_uploader,
+            }
+            video_id = mobj.group('video_id')
+            if video_id:
+                info.update({
+                    'ie_key': 'MangomoloVideo',
+                    'id': video_id,
+                })
+            else:
+                info.update({
+                    'ie_key': 'MangomoloLive',
+                    'id': mobj.group('channel_id'),
+                })
+            return info
+
         # Look for Instagram embeds
         instagram_embed_url = InstagramIE._extract_embed_url(webpage)
         if instagram_embed_url is not None:
diff --git a/youtube_dl/extractor/mangomolo.py b/youtube_dl/extractor/mangomolo.py
new file mode 100644
index 000000000..8cac8ace2
--- /dev/null
+++ b/youtube_dl/extractor/mangomolo.py
@@ -0,0 +1,54 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import base64
+
+from .common import InfoExtractor
+from ..compat import compat_urllib_parse_unquote
+from ..utils import (
+    int_or_none,
+)
+
+
+class MangomoloBaseIE(InfoExtractor):
+    def _get_real_id(self, page_id):
+        return page_id
+
+    def _real_extract(self, url):
+        page_id = self._get_real_id(self._match_id(url))
+        webpage = self._download_webpage(url, page_id)
+        hidden_inputs = self._hidden_inputs(webpage)
+        m3u8_entry_protocol = 'm3u8' if self._IS_LIVE else 'm3u8_native'
+
+        format_url = self._html_search_regex(
+            [
+                r'file\s*:\s*"(https?://[^"]+?/playlist.m3u8)',
+                r'<a[^>]+href="(rtsp://[^"]+)"'
+            ], webpage, 'format url')
+        formats = self._extract_wowza_formats(
+            format_url, page_id, m3u8_entry_protocol, ['smil'])
+        self._sort_formats(formats)
+
+        return {
+            'id': page_id,
+            'title': self._live_title(page_id) if self._IS_LIVE else page_id,
+            'uploader_id': hidden_inputs.get('userid'),
+            'duration': int_or_none(hidden_inputs.get('duration')),
+            'is_live': self._IS_LIVE,
+            'formats': formats,
+        }
+
+
+class MangomoloVideoIE(MangomoloBaseIE):
+    IENAME = 'mangomolo:video'
+    _VALID_URL = r'https?://admin\.mangomolo.com/analytics/index\.php/customers/embed/video\?.*?\bid=(?P<id>\d+)'
+    _IS_LIVE = False
+
+
+class MangomoloLiveIE(MangomoloBaseIE):
+    IENAME = 'mangomolo:live'
+    _VALID_URL = r'https?://admin\.mangomolo.com/analytics/index\.php/customers/embed/index\?.*?\bchannelid=(?P<id>(?:[A-Za-z0-9+/=]|%2B|%2F|%3D)+)'
+    _IS_LIVE = True
+
+    def _get_real_id(self, page_id):
+        return base64.b64decode(compat_urllib_parse_unquote(page_id).encode()).decode()

From fc86d4eed0bf10f8f90326472811e5b4d4ad4bd9 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 16 Sep 2016 20:10:47 +0100
Subject: [PATCH 6107/6219] [mangomolo] fix typo

---
 youtube_dl/extractor/mangomolo.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/mangomolo.py b/youtube_dl/extractor/mangomolo.py
index 8cac8ace2..2db503f2b 100644
--- a/youtube_dl/extractor/mangomolo.py
+++ b/youtube_dl/extractor/mangomolo.py
@@ -40,13 +40,13 @@ class MangomoloBaseIE(InfoExtractor):
 
 
 class MangomoloVideoIE(MangomoloBaseIE):
-    IENAME = 'mangomolo:video'
+    IE_NAME = 'mangomolo:video'
     _VALID_URL = r'https?://admin\.mangomolo.com/analytics/index\.php/customers/embed/video\?.*?\bid=(?P<id>\d+)'
     _IS_LIVE = False
 
 
 class MangomoloLiveIE(MangomoloBaseIE):
-    IENAME = 'mangomolo:live'
+    IE_NAME = 'mangomolo:live'
     _VALID_URL = r'https?://admin\.mangomolo.com/analytics/index\.php/customers/embed/index\?.*?\bchannelid=(?P<id>(?:[A-Za-z0-9+/=]|%2B|%2F|%3D)+)'
     _IS_LIVE = True
 

From 30d9e20938fa91ece09c376b67030647215d48df Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 16 Sep 2016 22:06:55 +0100
Subject: [PATCH 6108/6219] [postprocessor/ffmpeg] apply FFmpegFixupM3u8PP only
 for videos with aac codec(#5591)

---
 youtube_dl/postprocessor/ffmpeg.py | 63 +++++++++++++++---------------
 1 file changed, 31 insertions(+), 32 deletions(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index fa99b0c2a..8d1214ee2 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -139,6 +139,30 @@ class FFmpegPostProcessor(PostProcessor):
     def probe_executable(self):
         return self._paths[self.probe_basename]
 
+    def get_audio_codec(self, path):
+        if not self.probe_available:
+            raise PostProcessingError('ffprobe or avprobe not found. Please install one.')
+        try:
+            cmd = [
+                encodeFilename(self.probe_executable, True),
+                encodeArgument('-show_streams'),
+                encodeFilename(self._ffmpeg_filename_argument(path), True)]
+            if self._downloader.params.get('verbose', False):
+                self._downloader.to_screen('[debug] %s command line: %s' % (self.basename, shell_quote(cmd)))
+            handle = subprocess.Popen(cmd, stderr=compat_subprocess_get_DEVNULL(), stdout=subprocess.PIPE, stdin=subprocess.PIPE)
+            output = handle.communicate()[0]
+            if handle.wait() != 0:
+                return None
+        except (IOError, OSError):
+            return None
+        audio_codec = None
+        for line in output.decode('ascii', 'ignore').split('\n'):
+            if line.startswith('codec_name='):
+                audio_codec = line.split('=')[1].strip()
+            elif line.strip() == 'codec_type=audio' and audio_codec is not None:
+                return audio_codec
+        return None
+
     def run_ffmpeg_multiple_files(self, input_paths, out_path, opts):
         self.check_version()
 
@@ -188,31 +212,6 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
         self._preferredquality = preferredquality
         self._nopostoverwrites = nopostoverwrites
 
-    def get_audio_codec(self, path):
-
-        if not self.probe_available:
-            raise PostProcessingError('ffprobe or avprobe not found. Please install one.')
-        try:
-            cmd = [
-                encodeFilename(self.probe_executable, True),
-                encodeArgument('-show_streams'),
-                encodeFilename(self._ffmpeg_filename_argument(path), True)]
-            if self._downloader.params.get('verbose', False):
-                self._downloader.to_screen('[debug] %s command line: %s' % (self.basename, shell_quote(cmd)))
-            handle = subprocess.Popen(cmd, stderr=compat_subprocess_get_DEVNULL(), stdout=subprocess.PIPE, stdin=subprocess.PIPE)
-            output = handle.communicate()[0]
-            if handle.wait() != 0:
-                return None
-        except (IOError, OSError):
-            return None
-        audio_codec = None
-        for line in output.decode('ascii', 'ignore').split('\n'):
-            if line.startswith('codec_name='):
-                audio_codec = line.split('=')[1].strip()
-            elif line.strip() == 'codec_type=audio' and audio_codec is not None:
-                return audio_codec
-        return None
-
     def run_ffmpeg(self, path, out_path, codec, more_opts):
         if codec is None:
             acodec_opts = []
@@ -504,15 +503,15 @@ class FFmpegFixupM4aPP(FFmpegPostProcessor):
 class FFmpegFixupM3u8PP(FFmpegPostProcessor):
     def run(self, info):
         filename = info['filepath']
-        temp_filename = prepend_extension(filename, 'temp')
+        if self.get_audio_codec(filename) == 'aac':
+            temp_filename = prepend_extension(filename, 'temp')
 
-        options = ['-c', 'copy', '-f', 'mp4', '-bsf:a', 'aac_adtstoasc']
-        self._downloader.to_screen('[ffmpeg] Fixing malformated aac bitstream in "%s"' % filename)
-        self.run_ffmpeg(filename, temp_filename, options)
-
-        os.remove(encodeFilename(filename))
-        os.rename(encodeFilename(temp_filename), encodeFilename(filename))
+            options = ['-c', 'copy', '-f', 'mp4', '-bsf:a', 'aac_adtstoasc']
+            self._downloader.to_screen('[ffmpeg] Fixing malformated aac bitstream in "%s"' % filename)
+            self.run_ffmpeg(filename, temp_filename, options)
 
+            os.remove(encodeFilename(filename))
+            os.rename(encodeFilename(temp_filename), encodeFilename(filename))
         return [], info
 
 

From d05ef09d9d94fa70335af5fbaab385b37b16d705 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 17 Sep 2016 08:11:01 +0100
Subject: [PATCH 6109/6219] [mangomolo] fix domain regex

---
 youtube_dl/extractor/generic.py   | 2 +-
 youtube_dl/extractor/mangomolo.py | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index e01305942..92a6e5146 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -2256,7 +2256,7 @@ class GenericIE(InfoExtractor):
 
         # Look for Mangomolo embeds
         mobj = re.search(
-            r'''(?x)<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?admin\.mangomolo.com/analytics/index\.php/customers/embed/
+            r'''(?x)<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?admin\.mangomolo\.com/analytics/index\.php/customers/embed/
                 (?:
                     video\?.*?\bid=(?P<video_id>\d+)|
                     index\?.*?\bchannelid=(?P<channel_id>(?:[A-Za-z0-9+/=]|%2B|%2F|%3D)+)
diff --git a/youtube_dl/extractor/mangomolo.py b/youtube_dl/extractor/mangomolo.py
index 2db503f2b..1885ac7df 100644
--- a/youtube_dl/extractor/mangomolo.py
+++ b/youtube_dl/extractor/mangomolo.py
@@ -41,13 +41,13 @@ class MangomoloBaseIE(InfoExtractor):
 
 class MangomoloVideoIE(MangomoloBaseIE):
     IE_NAME = 'mangomolo:video'
-    _VALID_URL = r'https?://admin\.mangomolo.com/analytics/index\.php/customers/embed/video\?.*?\bid=(?P<id>\d+)'
+    _VALID_URL = r'https?://admin\.mangomolo\.com/analytics/index\.php/customers/embed/video\?.*?\bid=(?P<id>\d+)'
     _IS_LIVE = False
 
 
 class MangomoloLiveIE(MangomoloBaseIE):
     IE_NAME = 'mangomolo:live'
-    _VALID_URL = r'https?://admin\.mangomolo.com/analytics/index\.php/customers/embed/index\?.*?\bchannelid=(?P<id>(?:[A-Za-z0-9+/=]|%2B|%2F|%3D)+)'
+    _VALID_URL = r'https?://admin\.mangomolo\.com/analytics/index\.php/customers/embed/index\?.*?\bchannelid=(?P<id>(?:[A-Za-z0-9+/=]|%2B|%2F|%3D)+)'
     _IS_LIVE = True
 
     def _get_real_id(self, page_id):

From c51a7f0b2f2454bfe0b53f9d79567b3210e015b3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 17 Sep 2016 15:44:37 +0700
Subject: [PATCH 6110/6219] [franceinter] Fix upload date extraction

---
 youtube_dl/extractor/franceinter.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/franceinter.py b/youtube_dl/extractor/franceinter.py
index 1a1232ade..707b9e00d 100644
--- a/youtube_dl/extractor/franceinter.py
+++ b/youtube_dl/extractor/franceinter.py
@@ -2,7 +2,6 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import compat_str
 from ..utils import month_by_name
 
 
@@ -10,14 +9,14 @@ class FranceInterIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?franceinter\.fr/emissions/(?P<id>[^?#]+)'
 
     _TEST = {
-        'url': 'https://www.franceinter.fr/emissions/la-tete-au-carre/la-tete-au-carre-14-septembre-2016',
-        'md5': '4e3aeb58fe0e83d7b0581fa213c409d0',
+        'url': 'https://www.franceinter.fr/emissions/affaires-sensibles/affaires-sensibles-07-septembre-2016',
+        'md5': '9e54d7bdb6fdc02a841007f8a975c094',
         'info_dict': {
-            'id': 'la-tete-au-carre/la-tete-au-carre-14-septembre-2016',
+            'id': 'affaires-sensibles/affaires-sensibles-07-septembre-2016',
             'ext': 'mp3',
-            'title': 'Et si les rêves pouvaient nous aider à agir dans notre vie quotidienne ?',
-            'description': 'md5:a245dd62cf5bf51de915f8d9956d180a',
-            'upload_date': '20160914',
+            'title': 'Affaire Cahuzac : le contentieux du compte en Suisse',
+            'description': 'md5:401969c5d318c061f86bda1fa359292b',
+            'upload_date': '20160907',
         },
     }
 
@@ -40,6 +39,7 @@ class FranceInterIE(InfoExtractor):
             upload_date_list = upload_date_str.split()
             upload_date_list.reverse()
             upload_date_list[1] = '%02d' % (month_by_name(upload_date_list[1], lang='fr') or 0)
+            upload_date_list[2] = '%02d' % int(upload_date_list[2])
             upload_date = ''.join(upload_date_list)
         else:
             upload_date = None

From e14c82bd6b6cfc1e904b067350d818657c911e07 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 17 Sep 2016 18:45:08 +0800
Subject: [PATCH 6111/6219] [jwplatform] Use js_to_json to detect more
 JWPlayers

---
 ChangeLog                          | 6 ++++++
 youtube_dl/extractor/jwplatform.py | 6 ++++--
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 4583537ac..a9f7cee53 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Extractors
+* [jwplatform] Improve JWPlayer detection
+
+
 version 2016.09.15
 
 Core
diff --git a/youtube_dl/extractor/jwplatform.py b/youtube_dl/extractor/jwplatform.py
index 7aaa65476..38199fcd0 100644
--- a/youtube_dl/extractor/jwplatform.py
+++ b/youtube_dl/extractor/jwplatform.py
@@ -9,6 +9,7 @@ from ..utils import (
     determine_ext,
     float_or_none,
     int_or_none,
+    js_to_json,
     mimetype2ext,
 )
 
@@ -19,14 +20,15 @@ class JWPlatformBaseIE(InfoExtractor):
         # TODO: Merge this with JWPlayer-related codes in generic.py
 
         mobj = re.search(
-            'jwplayer\((?P<quote>[\'"])[^\'" ]+(?P=quote)\)\.setup\((?P<options>[^)]+)\)',
+            r'jwplayer\((?P<quote>[\'"])[^\'" ]+(?P=quote)\)\.setup\s*\((?P<options>[^)]+)\)',
             webpage)
         if mobj:
             return mobj.group('options')
 
     def _extract_jwplayer_data(self, webpage, video_id, *args, **kwargs):
         jwplayer_data = self._parse_json(
-            self._find_jwplayer_data(webpage), video_id)
+            self._find_jwplayer_data(webpage), video_id,
+            transform_source=js_to_json)
         return self._parse_jwplayer_data(
             jwplayer_data, video_id, *args, **kwargs)
 

From 584d6f3457205b547c8969f11eade117f871ec8f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 17 Sep 2016 18:46:43 +0800
Subject: [PATCH 6112/6219] [thisav] Recognize jwplayers (closes #10447)

---
 ChangeLog                      |  1 +
 youtube_dl/extractor/thisav.py | 39 ++++++++++++++++++++++++----------
 2 files changed, 29 insertions(+), 11 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index a9f7cee53..b0a65bde2 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
 version <unreleased>
 
 Extractors
++ [thisav] Recognize HTML5 videos (#10447)
 * [jwplatform] Improve JWPlayer detection
 
 
diff --git a/youtube_dl/extractor/thisav.py b/youtube_dl/extractor/thisav.py
index 7f323c938..027a8e907 100644
--- a/youtube_dl/extractor/thisav.py
+++ b/youtube_dl/extractor/thisav.py
@@ -3,13 +3,12 @@ from __future__ import unicode_literals
 
 import re
 
-from .common import InfoExtractor
-from ..utils import determine_ext
+from .jwplatform import JWPlatformBaseIE
 
 
-class ThisAVIE(InfoExtractor):
+class ThisAVIE(JWPlatformBaseIE):
     _VALID_URL = r'https?://(?:www\.)?thisav\.com/video/(?P<id>[0-9]+)/.*'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.thisav.com/video/47734/%98%26sup1%3B%83%9E%83%82---just-fit.html',
         'md5': '0480f1ef3932d901f0e0e719f188f19b',
         'info_dict': {
@@ -19,7 +18,17 @@ class ThisAVIE(InfoExtractor):
             'uploader': 'dj7970',
             'uploader_id': 'dj7970'
         }
-    }
+    }, {
+        'url': 'http://www.thisav.com/video/242352/nerdy-18yo-big-ass-tattoos-and-glasses.html',
+        'md5': 'ba90c076bd0f80203679e5b60bf523ee',
+        'info_dict': {
+            'id': '242352',
+            'ext': 'mp4',
+            'title': 'Nerdy 18yo Big Ass Tattoos and Glasses',
+            'uploader': 'cybersluts',
+            'uploader_id': 'cybersluts',
+        },
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
@@ -28,20 +37,28 @@ class ThisAVIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
         title = self._html_search_regex(r'<h1>([^<]*)</h1>', webpage, 'title')
         video_url = self._html_search_regex(
-            r"addVariable\('file','([^']+)'\);", webpage, 'video url')
+            r"addVariable\('file','([^']+)'\);", webpage, 'video url', default=None)
+        if video_url:
+            info_dict = {
+                'formats': [{
+                    'url': video_url,
+                }],
+            }
+        else:
+            info_dict = self._extract_jwplayer_data(
+                webpage, video_id, require_title=False)
         uploader = self._html_search_regex(
             r': <a href="http://www.thisav.com/user/[0-9]+/(?:[^"]+)">([^<]+)</a>',
             webpage, 'uploader name', fatal=False)
         uploader_id = self._html_search_regex(
             r': <a href="http://www.thisav.com/user/[0-9]+/([^"]+)">(?:[^<]+)</a>',
             webpage, 'uploader id', fatal=False)
-        ext = determine_ext(video_url)
 
-        return {
+        info_dict.update({
             'id': video_id,
-            'url': video_url,
             'uploader': uploader,
             'uploader_id': uploader_id,
             'title': title,
-            'ext': ext,
-        }
+        })
+
+        return info_dict

From a0d5077c8dfa9fa31ebf3e63fdb1b2a7a5182a81 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 6 Sep 2016 01:18:57 +0700
Subject: [PATCH 6113/6219] [extractor/common] Introduce fragments interface

---
 youtube_dl/extractor/common.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index c00023458..566ed7a4d 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -86,7 +86,9 @@ class InfoExtractor(object):
                     from worst to best quality.
 
                     Potential fields:
-                    * url        Mandatory. The URL of the video file
+                    * url        Mandatory. The URL of the video file or URL of
+                                 the manifest file in case of fragmented media
+                                 (DASH, hls, hds).
                     * ext        Will be calculated from URL if missing
                     * format     A human-readable description of the format
                                  ("mp4 container with h264/opus").
@@ -115,6 +117,11 @@ class InfoExtractor(object):
                                  download, lower-case.
                                  "http", "https", "rtsp", "rtmp", "rtmpe",
                                  "m3u8", "m3u8_native" or "http_dash_segments".
+                    * fragments  A list of fragments of the fragmented media,
+                                 with the following entries:
+                                 * "url" (mandatory) - fragment's URL
+                                 * "duration" (optional, int or float)
+                                 * "filesize" (optional, int)
                     * preference Order number of this format. If this field is
                                  present and not None, the formats get sorted
                                  by this field, regardless of all other values.

From b4c1d6e800a5b28accf4ba588b8fa3f0c420ce13 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 6 Sep 2016 01:21:57 +0700
Subject: [PATCH 6114/6219] [extractor/common] Expose fragments interface for
 dashsegments formats

---
 youtube_dl/extractor/common.py | 142 +++++++++++++++++++++++----------
 1 file changed, 99 insertions(+), 43 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 566ed7a4d..e637b33d5 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1551,42 +1551,52 @@ class InfoExtractor(object):
 
         def extract_multisegment_info(element, ms_parent_info):
             ms_info = ms_parent_info.copy()
+
+            # As per [1, 5.3.9.2.2] SegmentList and SegmentTemplate share some
+            # common attributes and elements.  We will only extract relevant
+            # for us.
+            def extract_common(source):
+                segment_timeline = source.find(_add_ns('SegmentTimeline'))
+                if segment_timeline is not None:
+                    s_e = segment_timeline.findall(_add_ns('S'))
+                    if s_e:
+                        ms_info['total_number'] = 0
+                        ms_info['s'] = []
+                        for s in s_e:
+                            r = int(s.get('r', 0))
+                            ms_info['total_number'] += 1 + r
+                            ms_info['s'].append({
+                                't': int(s.get('t', 0)),
+                                # @d is mandatory (see [1, 5.3.9.6.2, Table 17, page 60])
+                                'd': int(s.attrib['d']),
+                                'r': r,
+                            })
+                start_number = source.get('startNumber')
+                if start_number:
+                    ms_info['start_number'] = int(start_number)
+                timescale = source.get('timescale')
+                if timescale:
+                    ms_info['timescale'] = int(timescale)
+                segment_duration = source.get('duration')
+                if segment_duration:
+                    ms_info['segment_duration'] = int(segment_duration)
+
+            def extract_Initialization(source):
+                initialization = source.find(_add_ns('Initialization'))
+                if initialization is not None:
+                    ms_info['initialization_url'] = initialization.attrib['sourceURL']
+
             segment_list = element.find(_add_ns('SegmentList'))
             if segment_list is not None:
+                extract_common(segment_list)
+                extract_Initialization(segment_list)
                 segment_urls_e = segment_list.findall(_add_ns('SegmentURL'))
                 if segment_urls_e:
                     ms_info['segment_urls'] = [segment.attrib['media'] for segment in segment_urls_e]
-                initialization = segment_list.find(_add_ns('Initialization'))
-                if initialization is not None:
-                    ms_info['initialization_url'] = initialization.attrib['sourceURL']
             else:
                 segment_template = element.find(_add_ns('SegmentTemplate'))
                 if segment_template is not None:
-                    start_number = segment_template.get('startNumber')
-                    if start_number:
-                        ms_info['start_number'] = int(start_number)
-                    segment_timeline = segment_template.find(_add_ns('SegmentTimeline'))
-                    if segment_timeline is not None:
-                        s_e = segment_timeline.findall(_add_ns('S'))
-                        if s_e:
-                            ms_info['total_number'] = 0
-                            ms_info['s'] = []
-                            for s in s_e:
-                                r = int(s.get('r', 0))
-                                ms_info['total_number'] += 1 + r
-                                ms_info['s'].append({
-                                    't': int(s.get('t', 0)),
-                                    # @d is mandatory (see [1, 5.3.9.6.2, Table 17, page 60])
-                                    'd': int(s.attrib['d']),
-                                    'r': r,
-                                })
-                    else:
-                        timescale = segment_template.get('timescale')
-                        if timescale:
-                            ms_info['timescale'] = int(timescale)
-                        segment_duration = segment_template.get('duration')
-                        if segment_duration:
-                            ms_info['segment_duration'] = int(segment_duration)
+                    extract_common(segment_template)
                     media_template = segment_template.get('media')
                     if media_template:
                         ms_info['media_template'] = media_template
@@ -1594,11 +1604,14 @@ class InfoExtractor(object):
                     if initialization:
                         ms_info['initialization_url'] = initialization
                     else:
-                        initialization = segment_template.find(_add_ns('Initialization'))
-                        if initialization is not None:
-                            ms_info['initialization_url'] = initialization.attrib['sourceURL']
+                        extract_Initialization(segment_template)
             return ms_info
 
+        def combine_url(base_url, target_url):
+            if re.match(r'^https?://', target_url):
+                return target_url
+            return '%s%s%s' % (base_url, '' if base_url.endswith('/') else '/', target_url)
+
         mpd_duration = parse_duration(mpd_doc.get('mediaPresentationDuration'))
         formats = []
         for period in mpd_doc.findall(_add_ns('Period')):
@@ -1655,9 +1668,7 @@ class InfoExtractor(object):
                         }
                         representation_ms_info = extract_multisegment_info(representation, adaption_set_ms_info)
                         if 'segment_urls' not in representation_ms_info and 'media_template' in representation_ms_info:
-                            if 'total_number' not in representation_ms_info and 'segment_duration':
-                                segment_duration = float(representation_ms_info['segment_duration']) / float(representation_ms_info['timescale'])
-                                representation_ms_info['total_number'] = int(math.ceil(float(period_duration) / segment_duration))
+
                             media_template = representation_ms_info['media_template']
                             media_template = media_template.replace('$RepresentationID$', representation_id)
                             media_template = re.sub(r'\$(Number|Bandwidth|Time)\$', r'%(\1)d', media_template)
@@ -1666,7 +1677,11 @@ class InfoExtractor(object):
 
                             # As per [1, 5.3.9.4.4, Table 16, page 55] $Number$ and $Time$
                             # can't be used at the same time
-                            if '%(Number' in media_template:
+                            if '%(Number' in media_template and 's' not in representation_ms_info:
+                                segment_duration = None
+                                if 'total_number' not in representation_ms_info and 'segment_duration':
+                                    segment_duration = float_or_none(representation_ms_info['segment_duration'], representation_ms_info['timescale'])
+                                    representation_ms_info['total_number'] = int(math.ceil(float(period_duration) / segment_duration))
                                 representation_ms_info['segment_urls'] = [
                                     media_template % {
                                         'Number': segment_number,
@@ -1675,28 +1690,65 @@ class InfoExtractor(object):
                                     for segment_number in range(
                                         representation_ms_info['start_number'],
                                         representation_ms_info['total_number'] + representation_ms_info['start_number'])]
+                                representation_ms_info['fragments'] = [{
+                                    'url': media_template % {
+                                        'Number': segment_number,
+                                        'Bandwidth': representation_attrib.get('bandwidth'),
+                                    },
+                                    'duration': segment_duration,
+                                } for segment_number in range(
+                                    representation_ms_info['start_number'],
+                                    representation_ms_info['total_number'] + representation_ms_info['start_number'])]
                             else:
+                                # $Number*$ or $Time$ in media template with S list available
+                                # Example $Number*$: http://www.svtplay.se/klipp/9023742/stopptid-om-bjorn-borg
+                                # Example $Time$: https://play.arkena.com/embed/avp/v2/player/media/b41dda37-d8e7-4d3f-b1b5-9a9db578bdfe/1/129411
                                 representation_ms_info['segment_urls'] = []
+                                representation_ms_info['fragments'] = []
                                 segment_time = 0
+                                segment_d = None
+                                segment_number = representation_ms_info['start_number']
 
                                 def add_segment_url():
-                                    representation_ms_info['segment_urls'].append(
-                                        media_template % {
-                                            'Time': segment_time,
-                                            'Bandwidth': representation_attrib.get('bandwidth'),
-                                        }
-                                    )
+                                    segment_url = media_template % {
+                                        'Time': segment_time,
+                                        'Bandwidth': representation_attrib.get('bandwidth'),
+                                        'Number': segment_number,
+                                    }
+                                    representation_ms_info['segment_urls'].append(segment_url)
+                                    representation_ms_info['fragments'].append({
+                                        'url': segment_url,
+                                        'duration': float_or_none(segment_d, representation_ms_info['timescale']),
+                                    })
 
                                 for num, s in enumerate(representation_ms_info['s']):
                                     segment_time = s.get('t') or segment_time
+                                    segment_d = s['d']
                                     add_segment_url()
+                                    segment_number += 1
                                     for r in range(s.get('r', 0)):
-                                        segment_time += s['d']
+                                        segment_time += segment_d
                                         add_segment_url()
-                                    segment_time += s['d']
+                                        segment_number += 1
+                                    segment_time += segment_d
+                        elif 'segment_urls' in representation_ms_info and 's' in representation_ms_info:
+                            # No media template
+                            # Example: https://www.youtube.com/watch?v=iXZV5uAYMJI
+                            # or any YouTube dashsegments video
+                            fragments = []
+                            s_num = 0
+                            for segment_url in representation_ms_info['segment_urls']:
+                                s = representation_ms_info['s'][s_num]
+                                for r in range(s.get('r', 0) + 1):
+                                    fragments.append({
+                                        'url': segment_url,
+                                        'duration': float_or_none(s['d'], representation_ms_info['timescale']),
+                                    })
+                            representation_ms_info['fragments'] = fragments
                         if 'segment_urls' in representation_ms_info:
                             f.update({
                                 'segment_urls': representation_ms_info['segment_urls'],
+                                'fragments': [],
                                 'protocol': 'http_dash_segments',
                             })
                             if 'initialization_url' in representation_ms_info:
@@ -1706,6 +1758,10 @@ class InfoExtractor(object):
                                 })
                                 if not f.get('url'):
                                     f['url'] = initialization_url
+                                f['fragments'].append({'url': initialization_url})
+                            f['fragments'].extend(representation_ms_info['fragments'])
+                            for fragment in f['fragments']:
+                                fragment['url'] = combine_url(base_url, fragment['url'])
                         try:
                             existing_format = next(
                                 fo for fo in formats

From 21d21b0c72a731d4ff5affa2182fbe1687c031a0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 17 Sep 2016 19:25:31 +0700
Subject: [PATCH 6115/6219] [svt] Fix DASH formats extraction

---
 youtube_dl/extractor/svt.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/svt.py b/youtube_dl/extractor/svt.py
index 1c04dfb7b..fb0a4b24e 100644
--- a/youtube_dl/extractor/svt.py
+++ b/youtube_dl/extractor/svt.py
@@ -16,7 +16,7 @@ class SVTBaseIE(InfoExtractor):
     def _extract_video(self, video_info, video_id):
         formats = []
         for vr in video_info['videoReferences']:
-            player_type = vr.get('playerType')
+            player_type = vr.get('playerType') or vr.get('format')
             vurl = vr['url']
             ext = determine_ext(vurl)
             if ext == 'm3u8':

From 86f4d14f817acaee1f1f544cd9b06d47bc2a5180 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 17 Sep 2016 20:35:22 +0700
Subject: [PATCH 6116/6219] Refactor fragments interface and dash segments
 downloader - Eliminate segment_urls and initialization_url + Introduce
 manifest_url (manifest may contain unfragmented data in this case url will be
 used for direct media URL and manifest_url for manifest itself
 correspondingly) * Rewrite dashsegments downloader to use fragments data *
 Improve generic mpd extraction

---
 youtube_dl/downloader/dash.py   | 35 +++++++++++----------------------
 youtube_dl/extractor/common.py  | 31 +++++++++++------------------
 youtube_dl/extractor/generic.py |  4 +++-
 3 files changed, 26 insertions(+), 44 deletions(-)

diff --git a/youtube_dl/downloader/dash.py b/youtube_dl/downloader/dash.py
index 41fc9cfc2..8437dde30 100644
--- a/youtube_dl/downloader/dash.py
+++ b/youtube_dl/downloader/dash.py
@@ -1,7 +1,6 @@
 from __future__ import unicode_literals
 
 import os
-import re
 
 from .fragment import FragmentFD
 from ..compat import compat_urllib_error
@@ -19,34 +18,32 @@ class DashSegmentsFD(FragmentFD):
     FD_NAME = 'dashsegments'
 
     def real_download(self, filename, info_dict):
-        base_url = info_dict['url']
-        segment_urls = [info_dict['segment_urls'][0]] if self.params.get('test', False) else info_dict['segment_urls']
-        initialization_url = info_dict.get('initialization_url')
+        segments = info_dict['fragments'][:1] if self.params.get(
+            'test', False) else info_dict['fragments']
 
         ctx = {
             'filename': filename,
-            'total_frags': len(segment_urls) + (1 if initialization_url else 0),
+            'total_frags': len(segments),
         }
 
         self._prepare_and_start_frag_download(ctx)
 
-        def combine_url(base_url, target_url):
-            if re.match(r'^https?://', target_url):
-                return target_url
-            return '%s%s%s' % (base_url, '' if base_url.endswith('/') else '/', target_url)
-
         segments_filenames = []
 
         fragment_retries = self.params.get('fragment_retries', 0)
         skip_unavailable_fragments = self.params.get('skip_unavailable_fragments', True)
 
-        def process_segment(segment, tmp_filename, fatal):
-            target_url, segment_name = segment
+        def process_segment(segment, tmp_filename, num):
+            segment_url = segment['url']
+            segment_name = 'Frag%d' % num
             target_filename = '%s-%s' % (tmp_filename, segment_name)
+            # In DASH, the first segment contains necessary headers to
+            # generate a valid MP4 file, so always abort for the first segment
+            fatal = num == 0 or not skip_unavailable_fragments
             count = 0
             while count <= fragment_retries:
                 try:
-                    success = ctx['dl'].download(target_filename, {'url': combine_url(base_url, target_url)})
+                    success = ctx['dl'].download(target_filename, {'url': segment_url})
                     if not success:
                         return False
                     down, target_sanitized = sanitize_open(target_filename, 'rb')
@@ -72,16 +69,8 @@ class DashSegmentsFD(FragmentFD):
                 return False
             return True
 
-        segments_to_download = [(initialization_url, 'Init')] if initialization_url else []
-        segments_to_download.extend([
-            (segment_url, 'Seg%d' % i)
-            for i, segment_url in enumerate(segment_urls)])
-
-        for i, segment in enumerate(segments_to_download):
-            # In DASH, the first segment contains necessary headers to
-            # generate a valid MP4 file, so always abort for the first segment
-            fatal = i == 0 or not skip_unavailable_fragments
-            if not process_segment(segment, ctx['tmpfilename'], fatal):
+        for i, segment in enumerate(segments):
+            if not process_segment(segment, ctx['tmpfilename'], i):
                 return False
 
         self._finish_frag_download(ctx)
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index e637b33d5..f35311e7a 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -86,9 +86,10 @@ class InfoExtractor(object):
                     from worst to best quality.
 
                     Potential fields:
-                    * url        Mandatory. The URL of the video file or URL of
-                                 the manifest file in case of fragmented media
-                                 (DASH, hls, hds).
+                    * url        Mandatory. The URL of the video file
+                    * manifest_url
+                                 The URL of the manifest file in case of
+                                 fragmented media (DASH, hls, hds)
                     * ext        Will be calculated from URL if missing
                     * format     A human-readable description of the format
                                  ("mp4 container with h264/opus").
@@ -1528,9 +1529,10 @@ class InfoExtractor(object):
         mpd_base_url = re.match(r'https?://.+/', urlh.geturl()).group()
 
         return self._parse_mpd_formats(
-            compat_etree_fromstring(mpd.encode('utf-8')), mpd_id, mpd_base_url, formats_dict=formats_dict)
+            compat_etree_fromstring(mpd.encode('utf-8')), mpd_id, mpd_base_url,
+            formats_dict=formats_dict, mpd_url=mpd_url)
 
-    def _parse_mpd_formats(self, mpd_doc, mpd_id=None, mpd_base_url='', formats_dict={}):
+    def _parse_mpd_formats(self, mpd_doc, mpd_id=None, mpd_base_url='', formats_dict={}, mpd_url=None):
         """
         Parse formats from MPD manifest.
         References:
@@ -1654,6 +1656,7 @@ class InfoExtractor(object):
                         f = {
                             'format_id': '%s-%s' % (mpd_id, representation_id) if mpd_id else representation_id,
                             'url': base_url,
+                            'manifest_url': mpd_url,
                             'ext': mimetype2ext(mime_type),
                             'width': int_or_none(representation_attrib.get('width')),
                             'height': int_or_none(representation_attrib.get('height')),
@@ -1682,14 +1685,6 @@ class InfoExtractor(object):
                                 if 'total_number' not in representation_ms_info and 'segment_duration':
                                     segment_duration = float_or_none(representation_ms_info['segment_duration'], representation_ms_info['timescale'])
                                     representation_ms_info['total_number'] = int(math.ceil(float(period_duration) / segment_duration))
-                                representation_ms_info['segment_urls'] = [
-                                    media_template % {
-                                        'Number': segment_number,
-                                        'Bandwidth': representation_attrib.get('bandwidth'),
-                                    }
-                                    for segment_number in range(
-                                        representation_ms_info['start_number'],
-                                        representation_ms_info['total_number'] + representation_ms_info['start_number'])]
                                 representation_ms_info['fragments'] = [{
                                     'url': media_template % {
                                         'Number': segment_number,
@@ -1703,7 +1698,6 @@ class InfoExtractor(object):
                                 # $Number*$ or $Time$ in media template with S list available
                                 # Example $Number*$: http://www.svtplay.se/klipp/9023742/stopptid-om-bjorn-borg
                                 # Example $Time$: https://play.arkena.com/embed/avp/v2/player/media/b41dda37-d8e7-4d3f-b1b5-9a9db578bdfe/1/129411
-                                representation_ms_info['segment_urls'] = []
                                 representation_ms_info['fragments'] = []
                                 segment_time = 0
                                 segment_d = None
@@ -1715,7 +1709,6 @@ class InfoExtractor(object):
                                         'Bandwidth': representation_attrib.get('bandwidth'),
                                         'Number': segment_number,
                                     }
-                                    representation_ms_info['segment_urls'].append(segment_url)
                                     representation_ms_info['fragments'].append({
                                         'url': segment_url,
                                         'duration': float_or_none(segment_d, representation_ms_info['timescale']),
@@ -1745,17 +1738,15 @@ class InfoExtractor(object):
                                         'duration': float_or_none(s['d'], representation_ms_info['timescale']),
                                     })
                             representation_ms_info['fragments'] = fragments
-                        if 'segment_urls' in representation_ms_info:
+                        # NB: MPD manifest may contain direct URLs to unfragmented media.
+                        # No fragments key is present in this case.
+                        if 'fragments' in representation_ms_info:
                             f.update({
-                                'segment_urls': representation_ms_info['segment_urls'],
                                 'fragments': [],
                                 'protocol': 'http_dash_segments',
                             })
                             if 'initialization_url' in representation_ms_info:
                                 initialization_url = representation_ms_info['initialization_url'].replace('$RepresentationID$', representation_id)
-                                f.update({
-                                    'initialization_url': initialization_url,
-                                })
                                 if not f.get('url'):
                                     f['url'] = initialization_url
                                 f['fragments'].append({'url': initialization_url})
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 92a6e5146..c1792c534 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1657,7 +1657,9 @@ class GenericIE(InfoExtractor):
                 return self.playlist_result(self._parse_xspf(doc, video_id), video_id)
             elif re.match(r'(?i)^(?:{[^}]+})?MPD$', doc.tag):
                 info_dict['formats'] = self._parse_mpd_formats(
-                    doc, video_id, mpd_base_url=url.rpartition('/')[0])
+                    doc, video_id,
+                    mpd_base_url=full_response.geturl().rpartition('/')[0],
+                    mpd_url=url)
                 self._sort_formats(info_dict['formats'])
                 return info_dict
             elif re.match(r'^{http://ns\.adobe\.com/f4m/[12]\.0}manifest$', doc.tag):

From 30d0b549be5696f24b87471a0e691f9afca4a9c8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 17 Sep 2016 21:33:38 +0700
Subject: [PATCH 6117/6219] [extractor/common] Add manifest_url for hls and hds
 formats

---
 youtube_dl/extractor/common.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index f35311e7a..9c8991542 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1150,6 +1150,7 @@ class InfoExtractor(object):
             formats.append({
                 'format_id': format_id,
                 'url': manifest_url,
+                'manifest_url': manifest_url,
                 'ext': 'flv' if bootstrap_info is not None else None,
                 'tbr': tbr,
                 'width': width,
@@ -1255,9 +1256,11 @@ class InfoExtractor(object):
                 # format_id intact.
                 if not live:
                     format_id.append(stream_name if stream_name else '%d' % (tbr if tbr else len(formats)))
+                manifest_url = format_url(line.strip())
                 f = {
                     'format_id': '-'.join(format_id),
-                    'url': format_url(line.strip()),
+                    'url': manifest_url,
+                    'manifest_url': manifest_url,
                     'tbr': tbr,
                     'ext': ext,
                     'fps': float_or_none(last_info.get('FRAME-RATE')),

From 26394d021df1137301b1508bd00dd3478c15116c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 17 Sep 2016 23:34:10 +0700
Subject: [PATCH 6118/6219] [globo:article] Add support for multiple videos
 (Closes #10653)

---
 youtube_dl/extractor/globo.py | 39 +++++++++++++++++++++++++----------
 1 file changed, 28 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/globo.py b/youtube_dl/extractor/globo.py
index 5638be48f..dc7b2661c 100644
--- a/youtube_dl/extractor/globo.py
+++ b/youtube_dl/extractor/globo.py
@@ -2,6 +2,7 @@
 from __future__ import unicode_literals
 
 import random
+import re
 import math
 
 from .common import InfoExtractor
@@ -14,6 +15,7 @@ from ..utils import (
     ExtractorError,
     float_or_none,
     int_or_none,
+    orderedSet,
     str_or_none,
 )
 
@@ -63,6 +65,9 @@ class GloboIE(InfoExtractor):
     }, {
         'url': 'http://canaloff.globo.com/programas/desejar-profundo/videos/4518560.html',
         'only_matching': True,
+    }, {
+        'url': 'globo:3607726',
+        'only_matching': True,
     }]
 
     class MD5(object):
@@ -396,7 +401,7 @@ class GloboIE(InfoExtractor):
 
 
 class GloboArticleIE(InfoExtractor):
-    _VALID_URL = r'https?://.+?\.globo\.com/(?:[^/]+/)*(?P<id>[^/]+)(?:\.html)?'
+    _VALID_URL = r'https?://.+?\.globo\.com/(?:[^/]+/)*(?P<id>[^/.]+)(?:\.html)?'
 
     _VIDEOID_REGEXES = [
         r'\bdata-video-id=["\'](\d{7,})',
@@ -408,15 +413,20 @@ class GloboArticleIE(InfoExtractor):
 
     _TESTS = [{
         'url': 'http://g1.globo.com/jornal-nacional/noticia/2014/09/novidade-na-fiscalizacao-de-bagagem-pela-receita-provoca-discussoes.html',
-        'md5': '307fdeae4390ccfe6ba1aa198cf6e72b',
         'info_dict': {
-            'id': '3652183',
-            'ext': 'mp4',
-            'title': 'Receita Federal explica como vai fiscalizar bagagens de quem retorna ao Brasil de avião',
-            'duration': 110.711,
-            'uploader': 'Rede Globo',
-            'uploader_id': '196',
-        }
+            'id': 'novidade-na-fiscalizacao-de-bagagem-pela-receita-provoca-discussoes',
+            'title': 'Novidade na fiscalização de bagagem pela Receita provoca discussões',
+            'description': 'md5:c3c4b4d4c30c32fce460040b1ac46b12',
+        },
+        'playlist_count': 1,
+    }, {
+        'url': 'http://g1.globo.com/pr/parana/noticia/2016/09/mpf-denuncia-lula-marisa-e-mais-seis-na-operacao-lava-jato.html',
+        'info_dict': {
+            'id': 'mpf-denuncia-lula-marisa-e-mais-seis-na-operacao-lava-jato',
+            'title': "Lula era o 'comandante máximo' do esquema da Lava Jato, diz MPF",
+            'description': 'md5:8aa7cc8beda4dc71cc8553e00b77c54c',
+        },
+        'playlist_count': 6,
     }, {
         'url': 'http://gq.globo.com/Prazeres/Poder/noticia/2015/10/all-o-desafio-assista-ao-segundo-capitulo-da-serie.html',
         'only_matching': True,
@@ -435,5 +445,12 @@ class GloboArticleIE(InfoExtractor):
     def _real_extract(self, url):
         display_id = self._match_id(url)
         webpage = self._download_webpage(url, display_id)
-        video_id = self._search_regex(self._VIDEOID_REGEXES, webpage, 'video id')
-        return self.url_result('globo:%s' % video_id, 'Globo')
+        video_ids = []
+        for video_regex in self._VIDEOID_REGEXES:
+            video_ids.extend(re.findall(video_regex, webpage))
+        entries = [
+            self.url_result('globo:%s' % video_id, GloboIE.ie_key())
+            for video_id in orderedSet(video_ids)]
+        title = self._og_search_title(webpage, fatal=False)
+        description = self._html_search_meta('description', webpage)
+        return self.playlist_result(entries, display_id, title, description)

From 190d2027d0b6c785cf789edf6c1bdac2ef650a66 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 18 Sep 2016 07:22:06 +0700
Subject: [PATCH 6119/6219] [xfileshare] Add title regex for streamin.to and
 fallback to video id (Closes #10646)

---
 youtube_dl/extractor/xfileshare.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/xfileshare.py b/youtube_dl/extractor/xfileshare.py
index 995aada0d..de344bad2 100644
--- a/youtube_dl/extractor/xfileshare.py
+++ b/youtube_dl/extractor/xfileshare.py
@@ -124,12 +124,14 @@ class XFileShareIE(InfoExtractor):
             webpage = self._download_webpage(req, video_id, 'Downloading video page')
 
         title = (self._search_regex(
-            [r'style="z-index: [0-9]+;">([^<]+)</span>',
+            (r'style="z-index: [0-9]+;">([^<]+)</span>',
              r'<td nowrap>([^<]+)</td>',
              r'h4-fine[^>]*>([^<]+)<',
              r'>Watch (.+) ',
-             r'<h2 class="video-page-head">([^<]+)</h2>'],
-            webpage, 'title', default=None) or self._og_search_title(webpage)).strip()
+             r'<h2 class="video-page-head">([^<]+)</h2>',
+             r'<h2 style="[^"]*color:#403f3d[^"]*"[^>]*>([^<]+)<'),  # streamin.to
+            webpage, 'title', default=None) or self._og_search_title(
+            webpage, default=None) or video_id).strip()
 
         def extract_video_url(default=NO_DEFAULT):
             return self._search_regex(

From 14ae11efab64baf4994688490474609554c1bf80 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 18 Sep 2016 16:56:40 +0700
Subject: [PATCH 6120/6219] [vyborymos] Add extractor (Closes #10692)

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/vyborymos.py  | 55 ++++++++++++++++++++++++++++++
 2 files changed, 56 insertions(+)
 create mode 100644 youtube_dl/extractor/vyborymos.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 4baf4cd48..8166fd4f9 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1069,6 +1069,7 @@ from .vporn import VpornIE
 from .vrt import VRTIE
 from .vube import VubeIE
 from .vuclip import VuClipIE
+from .vyborymos import VyboryMosIE
 from .walla import WallaIE
 from .washingtonpost import (
     WashingtonPostIE,
diff --git a/youtube_dl/extractor/vyborymos.py b/youtube_dl/extractor/vyborymos.py
new file mode 100644
index 000000000..884aecb71
--- /dev/null
+++ b/youtube_dl/extractor/vyborymos.py
@@ -0,0 +1,55 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class VyboryMosIE(InfoExtractor):
+    _VALID_URL = r'https?://vybory\.mos\.ru/(?:#precinct/|account/channels\?.*?\bstation_id=)(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'http://vybory.mos.ru/#precinct/13636',
+        'info_dict': {
+            'id': '13636',
+            'ext': 'mp4',
+            'title': 're:^Участковая избирательная комиссия №2231 [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
+            'description': 'Россия, Москва, улица Введенского, 32А',
+            'is_live': True,
+        },
+        'params': {
+            'skip_download': True,
+        }
+    }, {
+        'url': 'http://vybory.mos.ru/account/channels?station_id=13636',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        station_id = self._match_id(url)
+
+        channels = self._download_json(
+            'http://vybory.mos.ru/account/channels?station_id=%s' % station_id,
+            station_id)
+
+        formats = []
+        for cam_num, (sid, hosts, name, _) in enumerate(channels, 1):
+            for num, host in enumerate(hosts, 1):
+                formats.append({
+                    'url': 'http://%s/master.m3u8?sid=%s' % (host, sid),
+                    'ext': 'mp4',
+                    'format_id': 'camera%d-host%d' % (cam_num, num),
+                    'format_note': '%s, %s' % (name, host),
+                })
+
+        info = self._download_json(
+            'http://vybory.mos.ru/json/voting_stations/136/%s.json' % station_id,
+            station_id, 'Downloading station info')
+
+        title = info['name']
+
+        return {
+            'id': station_id,
+            'title': self._live_title(title),
+            'description': info.get('address'),
+            'is_live': True,
+            'formats': formats,
+        }

From 9ca93b99d110f58ec9b280020fb5fede2441794e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 18 Sep 2016 17:15:22 +0700
Subject: [PATCH 6121/6219] [ChangeLog] Actualize

---
 ChangeLog | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index b0a65bde2..dd11a17b9 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,8 +1,23 @@
 version <unreleased>
 
+Core
++ Introduce manifest_url and fragments fields in formats dictionary for
+  fragmented media
++ Provide manifest_url field for DASH segments, HLS and HDS
++ Provide fragments field for DASH segments
+* Rework DASH segments downloader to use fragments field
++ Add helper method for Wowza Streaming Engine formats extraction
+
 Extractors
++ [vyborymos] Add extractor for vybory.mos.ru (#10692)
++ [xfileshare] Add title regular expression for streamin.to (#10646)
++ [globo:article] Add support for multiple videos (#10653)
 + [thisav] Recognize HTML5 videos (#10447)
 * [jwplatform] Improve JWPlayer detection
++ [mangomolo] Add support for Mangomolo embeds
++ [toutv] Add support for authentication (#10669)
+* [franceinter] Fix upload date extraction
+* [tv4] Fix HLS and HDS formats extraction (#10659)
 
 
 version 2016.09.15

From 3acff9423df437dd4bd1530a69011fc9ddc74ad1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 18 Sep 2016 17:16:55 +0700
Subject: [PATCH 6122/6219] release 2016.09.18

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 3 +++
 youtube_dl/version.py     | 2 +-
 4 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 61cea757c..b9d8ebad7 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.09.15*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.09.15**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.09.18*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.09.18**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.09.15
+[debug] youtube-dl version 2016.09.18
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index dd11a17b9..a71fadfa7 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2016.09.18
 
 Core
 + Introduce manifest_url and fragments fields in formats dictionary for
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index fcb618561..95a137393 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -388,6 +388,8 @@
  - **mailru**: Видео@Mail.Ru
  - **MakersChannel**
  - **MakerTV**
+ - **mangomolo:live**
+ - **mangomolo:video**
  - **MatchTV**
  - **MDR**: MDR.DE and KiKA
  - **media.ccc.de**
@@ -849,6 +851,7 @@
  - **VRT**
  - **vube**: Vube.com
  - **VuClip**
+ - **VyboryMos**
  - **Walla**
  - **washingtonpost**
  - **washingtonpost:article**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 081fd6ef0..5ae6a72aa 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.09.15'
+__version__ = '2016.09.18'

From a1da888d0cc92fdf3506b30ee85ce241e9090408 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 18 Sep 2016 17:28:41 +0700
Subject: [PATCH 6123/6219] [vyborymos] Improve station info extraction

---
 youtube_dl/extractor/vyborymos.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/vyborymos.py b/youtube_dl/extractor/vyborymos.py
index 884aecb71..9e703c4b6 100644
--- a/youtube_dl/extractor/vyborymos.py
+++ b/youtube_dl/extractor/vyborymos.py
@@ -2,6 +2,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..compat import compat_str
 
 
 class VyboryMosIE(InfoExtractor):
@@ -28,7 +29,7 @@ class VyboryMosIE(InfoExtractor):
 
         channels = self._download_json(
             'http://vybory.mos.ru/account/channels?station_id=%s' % station_id,
-            station_id)
+            station_id, 'Downloading channels JSON')
 
         formats = []
         for cam_num, (sid, hosts, name, _) in enumerate(channels, 1):
@@ -41,14 +42,13 @@ class VyboryMosIE(InfoExtractor):
                 })
 
         info = self._download_json(
-            'http://vybory.mos.ru/json/voting_stations/136/%s.json' % station_id,
-            station_id, 'Downloading station info')
-
-        title = info['name']
+            'http://vybory.mos.ru/json/voting_stations/%s/%s.json'
+            % (compat_str(station_id)[:3], station_id),
+            station_id, 'Downloading station JSON', fatal=False)
 
         return {
             'id': station_id,
-            'title': self._live_title(title),
+            'title': self._live_title(info['name'] if info else station_id),
             'description': info.get('address'),
             'is_live': True,
             'formats': formats,

From d8dbf8707d4e45a939fc74c76bb919771007f8ba Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 18 Sep 2016 18:33:54 +0800
Subject: [PATCH 6124/6219] [thisav] Improve title extraction (closes #10682)

I didn't add a test case as the one in #10682 looks like a copyrighted
product.
---
 ChangeLog                      | 6 ++++++
 youtube_dl/extractor/thisav.py | 5 ++++-
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index a71fadfa7..18f9fa861 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+version <unreleased>
+
+Extractors
+* [thisav] Improve title extraction (#10682)
+
+
 version 2016.09.18
 
 Core
diff --git a/youtube_dl/extractor/thisav.py b/youtube_dl/extractor/thisav.py
index 027a8e907..4473a3c77 100644
--- a/youtube_dl/extractor/thisav.py
+++ b/youtube_dl/extractor/thisav.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .jwplatform import JWPlatformBaseIE
+from ..utils import remove_end
 
 
 class ThisAVIE(JWPlatformBaseIE):
@@ -35,7 +36,9 @@ class ThisAVIE(JWPlatformBaseIE):
 
         video_id = mobj.group('id')
         webpage = self._download_webpage(url, video_id)
-        title = self._html_search_regex(r'<h1>([^<]*)</h1>', webpage, 'title')
+        title = remove_end(self._html_search_regex(
+            r'<title>([^<]+)</title>', webpage, 'title'),
+            ' - 視頻 - ThisAV.com-世界第一中文成人娛樂網站')
         video_url = self._html_search_regex(
             r"addVariable\('file','([^']+)'\);", webpage, 'video url', default=None)
         if video_url:

From cc764a6da8530248f9810397a22b20c972877a97 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 18 Sep 2016 19:10:18 +0700
Subject: [PATCH 6125/6219] [twitch:stream] Remove fallback to profile
 extraction when stream is offline Main page does not contain profile videos
 anymore

---
 youtube_dl/extractor/twitch.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index af6d890b0..bc352391e 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -400,11 +400,8 @@ class TwitchStreamIE(TwitchBaseIE):
             'kraken/streams/%s' % channel_id, channel_id,
             'Downloading stream JSON').get('stream')
 
-        # Fallback on profile extraction if stream is offline
         if not stream:
-            return self.url_result(
-                'http://www.twitch.tv/%s/profile' % channel_id,
-                'TwitchProfile', channel_id)
+            raise ExtractorError('%s is offline' % channel_id, expected=True)
 
         # Channel name may be typed if different case than the original channel name
         # (e.g. http://www.twitch.tv/TWITCHPLAYSPOKEMON) that will lead to constructing

From 70b4cf9b1b8a2c2935ca7384d7545463cfd4ea16 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 19 Sep 2016 02:50:06 +0700
Subject: [PATCH 6126/6219] [crunchyroll] Check if already logged in (Closes
 #10700)

---
 youtube_dl/extractor/crunchyroll.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 1b69bd0b6..e4c10ad24 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -46,6 +46,13 @@ class CrunchyrollBaseIE(InfoExtractor):
         login_page = self._download_webpage(
             self._LOGIN_URL, None, 'Downloading login page')
 
+        def is_logged(webpage):
+            return '<title>Redirecting' in webpage
+
+        # Already logged in
+        if is_logged(login_page):
+            return
+
         login_form_str = self._search_regex(
             r'(?P<form><form[^>]+?id=(["\'])%s\2[^>]*>)' % self._LOGIN_FORM,
             login_page, 'login form', group='form')
@@ -69,7 +76,7 @@ class CrunchyrollBaseIE(InfoExtractor):
             headers={'Content-Type': 'application/x-www-form-urlencoded'})
 
         # Successful login
-        if '<title>Redirecting' in response:
+        if is_logged(response):
             return
 
         error = self._html_search_regex(

From 59fd8f931d274cc702a7e260e9ec996f8db7c9f0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 19 Sep 2016 02:57:14 +0700
Subject: [PATCH 6127/6219] [ChangeLog] Actualize

---
 ChangeLog | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 18f9fa861..c67d5f650 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,7 +1,10 @@
 version <unreleased>
 
 Extractors
++ [crunchyroll] Check if already authenticated (#10700)
+- [twitch:stream] Remove fallback to profile extraction when stream is offline
 * [thisav] Improve title extraction (#10682)
+* [vyborymos] Improve station info extraction
 
 
 version 2016.09.18

From cb57386873a053b3328a78f48cf27f23ca6897d1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 19 Sep 2016 02:58:32 +0700
Subject: [PATCH 6128/6219] release 2016.09.19

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 youtube_dl/version.py     | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index b9d8ebad7..8b28d784a 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.09.18*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.09.18**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.09.19*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.09.19**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.09.18
+[debug] youtube-dl version 2016.09.19
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index c67d5f650..24077c430 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2016.09.19
 
 Extractors
 + [crunchyroll] Check if already authenticated (#10700)
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 5ae6a72aa..9d3138181 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.09.18'
+__version__ = '2016.09.19'

From c38f06818df83f5f46cbdee1069bfaf53a537cc8 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 20 Sep 2016 11:55:30 +0100
Subject: [PATCH 6129/6219] add support for Adobe Pass auth in tbs,tnt and
 trutv extractors(fixes #10642)(closes #10222)(closes #10519)

---
 youtube_dl/extractor/adobepass.py |  2 +-
 youtube_dl/extractor/tbs.py       | 13 +++++--------
 youtube_dl/extractor/trutv.py     | 12 ++++++++++++
 youtube_dl/extractor/turner.py    | 17 ++++++++++-------
 4 files changed, 28 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/extractor/adobepass.py b/youtube_dl/extractor/adobepass.py
index 01932e5e6..c787e0962 100644
--- a/youtube_dl/extractor/adobepass.py
+++ b/youtube_dl/extractor/adobepass.py
@@ -83,7 +83,7 @@ class AdobePassIE(InfoExtractor):
             'User-Agent': self._USER_AGENT,
         }
 
-        guid = xml_text(resource, 'guid')
+        guid = xml_text(resource, 'guid') if '<' in resource else resource
         count = 0
         while count < 2:
             requestor_info = self._downloader.cache.load(self._MVPD_CACHE, requestor_id) or {}
diff --git a/youtube_dl/extractor/tbs.py b/youtube_dl/extractor/tbs.py
index 0c351e045..bf93eb868 100644
--- a/youtube_dl/extractor/tbs.py
+++ b/youtube_dl/extractor/tbs.py
@@ -4,10 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .turner import TurnerBaseIE
-from ..utils import (
-    extract_attributes,
-    ExtractorError,
-)
+from ..utils import extract_attributes
 
 
 class TBSIE(TurnerBaseIE):
@@ -37,10 +34,6 @@ class TBSIE(TurnerBaseIE):
         site = domain[:3]
         webpage = self._download_webpage(url, display_id)
         video_params = extract_attributes(self._search_regex(r'(<[^>]+id="page-video"[^>]*>)', webpage, 'video params'))
-        if video_params.get('isAuthRequired') == 'true':
-            raise ExtractorError(
-                'This video is only available via cable service provider subscription that'
-                ' is not currently supported.', expected=True)
         query = None
         clip_id = video_params.get('clipid')
         if clip_id:
@@ -56,4 +49,8 @@ class TBSIE(TurnerBaseIE):
                     'media_src': 'http://androidhls-secure.cdn.turner.com/%s/big' % site,
                     'tokenizer_src': 'http://www.%s.com/video/processors/services/token_ipadAdobe.do' % domain,
                 },
+            }, {
+                'url': url,
+                'site_name': site.upper(),
+                'auth_required': video_params.get('isAuthRequired') != 'false',
             })
diff --git a/youtube_dl/extractor/trutv.py b/youtube_dl/extractor/trutv.py
index e60d8a181..3a5782525 100644
--- a/youtube_dl/extractor/trutv.py
+++ b/youtube_dl/extractor/trutv.py
@@ -22,9 +22,17 @@ class TruTVIE(TurnerBaseIE):
 
     def _real_extract(self, url):
         path, video_id = re.match(self._VALID_URL, url).groups()
+        auth_required = False
         if path:
             data_src = 'http://www.trutv.com/video/cvp/v2/xml/content.xml?id=%s.xml' % path
         else:
+            webpage = self._download_webpage(url, video_id)
+            video_id = self._search_regex(
+                r"TTV\.TVE\.episodeId\s*=\s*'([^']+)';",
+                webpage, 'video id', default=video_id)
+            auth_required = self._search_regex(
+                r'TTV\.TVE\.authRequired\s*=\s*(true|false);',
+                webpage, 'auth required', default='false') == 'true'
             data_src = 'http://www.trutv.com/tveverywhere/services/cvpXML.do?titleId=' + video_id
         return self._extract_cvp_info(
             data_src, path, {
@@ -32,4 +40,8 @@ class TruTVIE(TurnerBaseIE):
                     'media_src': 'http://androidhls-secure.cdn.turner.com/trutv/big',
                     'tokenizer_src': 'http://www.trutv.com/tveverywhere/processors/services/token_ipadAdobe.do',
                 },
+            }, {
+                'url': url,
+                'site_name': 'truTV',
+                'auth_required': auth_required,
             })
diff --git a/youtube_dl/extractor/turner.py b/youtube_dl/extractor/turner.py
index 4228c1ccc..57ffedb87 100644
--- a/youtube_dl/extractor/turner.py
+++ b/youtube_dl/extractor/turner.py
@@ -3,7 +3,7 @@ from __future__ import unicode_literals
 
 import re
 
-from .common import InfoExtractor
+from .adobepass import AdobePassIE
 from ..compat import compat_str
 from ..utils import (
     xpath_text,
@@ -16,11 +16,11 @@ from ..utils import (
 )
 
 
-class TurnerBaseIE(InfoExtractor):
+class TurnerBaseIE(AdobePassIE):
     def _extract_timestamp(self, video_data):
         return int_or_none(xpath_attr(video_data, 'dateCreated', 'uts'))
 
-    def _extract_cvp_info(self, data_src, video_id, path_data={}):
+    def _extract_cvp_info(self, data_src, video_id, path_data={}, ap_data={}):
         video_data = self._download_xml(data_src, video_id)
         video_id = video_data.attrib['id']
         title = xpath_text(video_data, 'headline', fatal=True)
@@ -70,11 +70,14 @@ class TurnerBaseIE(InfoExtractor):
                 secure_path = self._search_regex(r'https?://[^/]+(.+/)', video_url, 'secure path') + '*'
                 token = tokens.get(secure_path)
                 if not token:
+                    query = {
+                        'path': secure_path,
+                        'videoId': content_id,
+                    }
+                    if ap_data.get('auth_required'):
+                        query['accessToken'] = self._extract_mvpd_auth(ap_data['url'], video_id, ap_data['site_name'], ap_data['site_name'])
                     auth = self._download_xml(
-                        secure_path_data['tokenizer_src'], video_id, query={
-                            'path': secure_path,
-                            'videoId': content_id,
-                        })
+                        secure_path_data['tokenizer_src'], video_id, query=query)
                     error_msg = xpath_text(auth, 'error/msg')
                     if error_msg:
                         raise ExtractorError(error_msg, expected=True)

From e33a7253b23e0adca9a3cb9a3856952c922a3357 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 20 Sep 2016 15:52:23 +0100
Subject: [PATCH 6130/6219] [fox] add support for Adobe Pass auth(closes #8584)

---
 youtube_dl/extractor/fox.py | 26 +++++++++++++++++++-------
 1 file changed, 19 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/fox.py b/youtube_dl/extractor/fox.py
index 9f406b17e..9f2e5d065 100644
--- a/youtube_dl/extractor/fox.py
+++ b/youtube_dl/extractor/fox.py
@@ -1,14 +1,14 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-from .common import InfoExtractor
+from .adobepass import AdobePassIE
 from ..utils import (
     smuggle_url,
     update_url_query,
 )
 
 
-class FOXIE(InfoExtractor):
+class FOXIE(AdobePassIE):
     _VALID_URL = r'https?://(?:www\.)?fox\.com/watch/(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://www.fox.com/watch/255180355939/7684182528',
@@ -30,14 +30,26 @@ class FOXIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        release_url = self._parse_json(self._search_regex(
-            r'"fox_pdk_player"\s*:\s*({[^}]+?})', webpage, 'fox_pdk_player'),
-            video_id)['release_url']
+        settings = self._parse_json(self._search_regex(
+            r'jQuery\.extend\(Drupal\.settings\s*,\s*({.+?})\);',
+            webpage, 'drupal settings'), video_id)
+        fox_pdk_player = settings['fox_pdk_player']
+        release_url = fox_pdk_player['release_url']
+        query = {
+            'mbr': 'true',
+            'switch': 'http'
+        }
+        if fox_pdk_player.get('access') == 'locked':
+            ap_p = settings['foxAdobePassProvider']
+            rating = ap_p.get('videoRating')
+            if rating == 'n/a':
+                rating = None
+            resource = self._get_mvpd_resource('fbc-fox', None, ap_p['videoGUID'], rating)
+            query['auth'] = self._extract_mvpd_auth(url, video_id, 'fbc-fox', resource)
 
         return {
             '_type': 'url_transparent',
             'ie_key': 'ThePlatform',
-            'url': smuggle_url(update_url_query(
-                release_url, {'switch': 'http'}), {'force_smil_url': True}),
+            'url': smuggle_url(update_url_query(release_url, query), {'force_smil_url': True}),
             'id': video_id,
         }

From 4bfd294e2f83301921494c02e497cccf1a26cfd5 Mon Sep 17 00:00:00 2001
From: coolsa <noob.cloud@gmail.com>
Date: Sun, 18 Sep 2016 03:53:05 -0600
Subject: [PATCH 6131/6219] [soundcloud] Extract license metadata

---
 youtube_dl/extractor/soundcloud.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 9635c2b49..47b84809f 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -53,6 +53,7 @@ class SoundcloudIE(InfoExtractor):
                 'uploader': 'E.T. ExTerrestrial Music',
                 'title': 'Lostin Powers - She so Heavy (SneakPreview) Adrian Ackers Blueprint 1',
                 'duration': 143,
+                'license': 'all-rights-reserved',
             }
         },
         # not streamable song
@@ -66,6 +67,7 @@ class SoundcloudIE(InfoExtractor):
                 'uploader': 'The Royal Concept',
                 'upload_date': '20120521',
                 'duration': 227,
+                'license': 'all-rights-reserved',
             },
             'params': {
                 # rtmp
@@ -84,6 +86,7 @@ class SoundcloudIE(InfoExtractor):
                 'description': 'test chars:  \"\'/\\ä↭',
                 'upload_date': '20131209',
                 'duration': 9,
+                'license': 'all-rights-reserved',
             },
         },
         # private link (alt format)
@@ -98,6 +101,7 @@ class SoundcloudIE(InfoExtractor):
                 'description': 'test chars:  \"\'/\\ä↭',
                 'upload_date': '20131209',
                 'duration': 9,
+                'license': 'all-rights-reserved',
             },
         },
         # downloadable song
@@ -112,6 +116,7 @@ class SoundcloudIE(InfoExtractor):
                 'uploader': 'oddsamples',
                 'upload_date': '20140109',
                 'duration': 17,
+                'license': 'cc-by-sa',
             },
         },
     ]
@@ -138,8 +143,8 @@ class SoundcloudIE(InfoExtractor):
         name = full_title or track_id
         if quiet:
             self.report_extraction(name)
-
         thumbnail = info['artwork_url']
+        track_license = info['license']
         if thumbnail is not None:
             thumbnail = thumbnail.replace('-large', '-t500x500')
         ext = 'mp3'
@@ -152,6 +157,7 @@ class SoundcloudIE(InfoExtractor):
             'thumbnail': thumbnail,
             'duration': int_or_none(info.get('duration'), 1000),
             'webpage_url': info.get('permalink_url'),
+            'license': track_license,
         }
         formats = []
         if info.get('downloadable', False):
@@ -222,6 +228,7 @@ class SoundcloudIE(InfoExtractor):
 
         track_id = mobj.group('track_id')
         token = None
+
         if track_id is not None:
             info_json_url = 'http://api.soundcloud.com/tracks/' + track_id + '.json?client_id=' + self._CLIENT_ID
             full_title = track_id

From f62a77b99a73ed3acf8406efaa34d08c73682be3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 20 Sep 2016 21:55:57 +0700
Subject: [PATCH 6132/6219] [soundcloud] Modernize

---
 youtube_dl/extractor/soundcloud.py | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 47b84809f..513c54829 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -143,21 +143,20 @@ class SoundcloudIE(InfoExtractor):
         name = full_title or track_id
         if quiet:
             self.report_extraction(name)
-        thumbnail = info['artwork_url']
-        track_license = info['license']
-        if thumbnail is not None:
+        thumbnail = info.get('artwork_url')
+        if isinstance(thumbnail, compat_str):
             thumbnail = thumbnail.replace('-large', '-t500x500')
         ext = 'mp3'
         result = {
             'id': track_id,
-            'uploader': info['user']['username'],
-            'upload_date': unified_strdate(info['created_at']),
+            'uploader': info.get('user', {}).get('username'),
+            'upload_date': unified_strdate(info.get('created_at')),
             'title': info['title'],
-            'description': info['description'],
+            'description': info.get('description'),
             'thumbnail': thumbnail,
             'duration': int_or_none(info.get('duration'), 1000),
             'webpage_url': info.get('permalink_url'),
-            'license': track_license,
+            'license': info.get('license'),
         }
         formats = []
         if info.get('downloadable', False):
@@ -227,7 +226,6 @@ class SoundcloudIE(InfoExtractor):
             raise ExtractorError('Invalid URL: %s' % url)
 
         track_id = mobj.group('track_id')
-        token = None
 
         if track_id is not None:
             info_json_url = 'http://api.soundcloud.com/tracks/' + track_id + '.json?client_id=' + self._CLIENT_ID

From 1ae0ae5db0bc9c388de970c71880e2f3dc400cc3 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 20 Sep 2016 18:51:29 +0100
Subject: [PATCH 6133/6219] [cartoonnetwork] add support Adobe Pass auth

---
 youtube_dl/extractor/cartoonnetwork.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/cartoonnetwork.py b/youtube_dl/extractor/cartoonnetwork.py
index 688a6375e..086ec90c9 100644
--- a/youtube_dl/extractor/cartoonnetwork.py
+++ b/youtube_dl/extractor/cartoonnetwork.py
@@ -33,4 +33,10 @@ class CartoonNetworkIE(TurnerBaseIE):
                     'media_src': 'http://androidhls-secure.cdn.turner.com/toon/big',
                     'tokenizer_src': 'http://www.cartoonnetwork.com/cntv/mvpd/processors/services/token_ipadAdobe.do',
                 },
+            }, {
+                'url': url,
+                'site_name': 'CartoonNetwork',
+                'auth_required': self._search_regex(
+                    r'_cnglobal\.cvpFullOrPreviewAuth\s*=\s*(true|false);',
+                    webpage, 'auth required', default='false') == 'true',
             })

From 3a5a18705f2a7faf64a4b69665511ef5f0c6084d Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 21 Sep 2016 15:56:31 +0100
Subject: [PATCH 6134/6219] [adobepass] add support MSO that depend on
 watchTVeverywhere(closes #10709)

---
 youtube_dl/extractor/adobepass.py | 1264 ++++++++++++++++++++++++++++-
 1 file changed, 1259 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/adobepass.py b/youtube_dl/extractor/adobepass.py
index c787e0962..8f7ed6ef2 100644
--- a/youtube_dl/extractor/adobepass.py
+++ b/youtube_dl/extractor/adobepass.py
@@ -17,15 +17,1269 @@ from ..utils import (
 
 MSO_INFO = {
     'DTV': {
-        'name': 'DirecTV',
+        'name': 'DIRECTV',
         'username_field': 'username',
         'password_field': 'password',
     },
     'Rogers': {
-        'name': 'Rogers Cable',
+        'name': 'Rogers',
         'username_field': 'UserName',
         'password_field': 'UserPassword',
     },
+    'thr030': {
+        'name': '3 Rivers Communications'
+    },
+    'com140': {
+        'name': 'Access Montana'
+    },
+    'acecommunications': {
+        'name': 'AcenTek'
+    },
+    'acm010': {
+        'name': 'Acme Communications'
+    },
+    'ada020': {
+        'name': 'Adams Cable Service'
+    },
+    'alb020': {
+        'name': 'Albany Mutual Telephone'
+    },
+    'algona': {
+        'name': 'Algona Municipal Utilities'
+    },
+    'allwest': {
+        'name': 'All West Communications'
+    },
+    'all025': {
+        'name': 'Allen\'s Communications'
+    },
+    'spl010': {
+        'name': 'Alliance Communications'
+    },
+    'all070': {
+        'name': 'ALLO Communications'
+    },
+    'alpine': {
+        'name': 'Alpine Communications'
+    },
+    'hun015': {
+        'name': 'American Broadband'
+    },
+    'nwc010': {
+        'name': 'American Broadband Missouri'
+    },
+    'com130-02': {
+        'name': 'American Community Networks'
+    },
+    'com130-01': {
+        'name': 'American Warrior Networks'
+    },
+    'tom020': {
+        'name': 'Amherst Telephone/Tomorrow Valley'
+    },
+    'tvc020': {
+        'name': 'Andycable'
+    },
+    'arkwest': {
+        'name': 'Arkwest Communications'
+    },
+    'art030': {
+        'name': 'Arthur Mutual Telephone Company'
+    },
+    'arvig': {
+        'name': 'Arvig'
+    },
+    'nttcash010': {
+        'name': 'Ashland Home Net'
+    },
+    'astound': {
+        'name': 'Astound (now Wave)'
+    },
+    'dix030': {
+        'name': 'ATC Broadband'
+    },
+    'ara010': {
+        'name': 'ATC Communications'
+    },
+    'she030-02': {
+        'name': 'Ayersville Communications'
+    },
+    'baldwin': {
+        'name': 'Baldwin Lightstream'
+    },
+    'bal040': {
+        'name': 'Ballard TV'
+    },
+    'cit025': {
+        'name': 'Bardstown Cable TV'
+    },
+    'bay030': {
+        'name': 'Bay Country Communications'
+    },
+    'tel095': {
+        'name': 'Beaver Creek Cooperative Telephone'
+    },
+    'bea020': {
+        'name': 'Beaver Valley Cable'
+    },
+    'bee010': {
+        'name': 'Bee Line Cable'
+    },
+    'wir030': {
+        'name': 'Beehive Broadband'
+    },
+    'bra020': {
+        'name': 'BELD'
+    },
+    'bel020': {
+        'name': 'Bellevue Municipal Cable'
+    },
+    'vol040-01': {
+        'name': 'Ben Lomand Connect / BLTV'
+    },
+    'bev010': {
+        'name': 'BEVCOMM'
+    },
+    'big020': {
+        'name': 'Big Sandy Broadband'
+    },
+    'ble020': {
+        'name': 'Bledsoe Telephone Cooperative'
+    },
+    'bvt010': {
+        'name': 'Blue Valley Tele-Communications'
+    },
+    'bra050': {
+        'name': 'Brandenburg Telephone Co.'
+    },
+    'bte010': {
+        'name': 'Bristol Tennessee Essential Services'
+    },
+    'annearundel': {
+        'name': 'Broadstripe'
+    },
+    'btc010': {
+        'name': 'BTC Communications'
+    },
+    'btc040': {
+        'name': 'BTC Vision - Nahunta'
+    },
+    'bul010': {
+        'name': 'Bulloch Telephone Cooperative'
+    },
+    'but010': {
+        'name': 'Butler-Bremer Communications'
+    },
+    'tel160-csp': {
+        'name': 'C Spire SNAP'
+    },
+    'csicable': {
+        'name': 'Cable Services Inc.'
+    },
+    'cableamerica': {
+        'name': 'CableAmerica'
+    },
+    'cab038': {
+        'name': 'CableSouth Media 3'
+    },
+    'weh010-camtel': {
+        'name': 'Cam-Tel Company'
+    },
+    'car030': {
+        'name': 'Cameron Communications'
+    },
+    'canbytel': {
+        'name': 'Canby Telcom'
+    },
+    'crt020': {
+        'name': 'CapRock Tv'
+    },
+    'car050': {
+        'name': 'Carnegie Cable'
+    },
+    'cas': {
+        'name': 'CAS Cable'
+    },
+    'casscomm': {
+        'name': 'CASSCOMM'
+    },
+    'mid180-02': {
+        'name': 'Catalina Broadband Solutions'
+    },
+    'cccomm': {
+        'name': 'CC Communications'
+    },
+    'nttccde010': {
+        'name': 'CDE Lightband'
+    },
+    'cfunet': {
+        'name': 'Cedar Falls Utilities'
+    },
+    'dem010-01': {
+        'name': 'Celect-Bloomer Telephone Area'
+    },
+    'dem010-02': {
+        'name': 'Celect-Bruce Telephone Area'
+    },
+    'dem010-03': {
+        'name': 'Celect-Citizens Connected Area'
+    },
+    'dem010-04': {
+        'name': 'Celect-Elmwood/Spring Valley Area'
+    },
+    'dem010-06': {
+        'name': 'Celect-Mosaic Telecom'
+    },
+    'dem010-05': {
+        'name': 'Celect-West WI Telephone Area'
+    },
+    'net010-02': {
+        'name': 'Cellcom/Nsight Telservices'
+    },
+    'cen100': {
+        'name': 'CentraCom'
+    },
+    'nttccst010': {
+        'name': 'Central Scott / CSTV'
+    },
+    'cha035': {
+        'name': 'Chaparral CableVision'
+    },
+    'cha050': {
+        'name': 'Chariton Valley Communication Corporation, Inc.'
+    },
+    'cha060': {
+        'name': 'Chatmoss Cablevision'
+    },
+    'nttcche010': {
+        'name': 'Cherokee Communications'
+    },
+    'che050': {
+        'name': 'Chesapeake Bay Communications'
+    },
+    'cimtel': {
+        'name': 'Cim-Tel Cable, LLC.'
+    },
+    'cit180': {
+        'name': 'Citizens Cablevision - Floyd, VA'
+    },
+    'cit210': {
+        'name': 'Citizens Cablevision, Inc.'
+    },
+    'cit040': {
+        'name': 'Citizens Fiber'
+    },
+    'cit250': {
+        'name': 'Citizens Mutual'
+    },
+    'war040': {
+        'name': 'Citizens Telephone Corporation'
+    },
+    'wat025': {
+        'name': 'City Of Monroe'
+    },
+    'wadsworth': {
+        'name': 'CityLink'
+    },
+    'nor100': {
+        'name': 'CL Tel'
+    },
+    'cla010': {
+        'name': 'Clarence Telephone and Cedar Communications'
+    },
+    'ser060': {
+        'name': 'Clear Choice Communications'
+    },
+    'tac020': {
+        'name': 'Click! Cable TV'
+    },
+    'war020': {
+        'name': 'CLICK1.NET'
+    },
+    'cml010': {
+        'name': 'CML Telephone Cooperative Association'
+    },
+    'cns': {
+        'name': 'CNS'
+    },
+    'com160': {
+        'name': 'Co-Mo Connect'
+    },
+    'coa020': {
+        'name': 'Coast Communications'
+    },
+    'coa030': {
+        'name': 'Coaxial Cable TV'
+    },
+    'mid055': {
+        'name': 'Cobalt TV (Mid-State Community TV)'
+    },
+    'col070': {
+        'name': 'Columbia Power & Water Systems'
+    },
+    'col080': {
+        'name': 'Columbus Telephone'
+    },
+    'nor105': {
+        'name': 'Communications 1 Cablevision, Inc.'
+    },
+    'com150': {
+        'name': 'Community Cable & Broadband'
+    },
+    'com020': {
+        'name': 'Community Communications Company'
+    },
+    'coy010': {
+        'name': 'commZoom'
+    },
+    'com025': {
+        'name': 'Complete Communication Services'
+    },
+    'cat020': {
+        'name': 'Comporium'
+    },
+    'com071': {
+        'name': 'ComSouth Telesys'
+    },
+    'consolidatedcable': {
+        'name': 'Consolidated'
+    },
+    'conwaycorp': {
+        'name': 'Conway Corporation'
+    },
+    'coo050': {
+        'name': 'Coon Valley Telecommunications Inc'
+    },
+    'coo080': {
+        'name': 'Cooperative Telephone Company'
+    },
+    'cpt010': {
+        'name': 'CP-TEL'
+    },
+    'cra010': {
+        'name': 'Craw-Kan Telephone'
+    },
+    'crestview': {
+        'name': 'Crestview Cable Communications'
+    },
+    'cross': {
+        'name': 'Cross TV'
+    },
+    'cro030': {
+        'name': 'Crosslake Communications'
+    },
+    'ctc040': {
+        'name': 'CTC - Brainerd MN'
+    },
+    'phe030': {
+        'name': 'CTV-Beam - East Alabama'
+    },
+    'cun010': {
+        'name': 'Cunningham Telephone & Cable'
+    },
+    'dpc010': {
+        'name': 'D & P Communications'
+    },
+    'dak030': {
+        'name': 'Dakota Central Telecommunications'
+    },
+    'nttcdel010': {
+        'name': 'Delcambre Telephone LLC'
+    },
+    'tel160-del': {
+        'name': 'Delta Telephone Company'
+    },
+    'sal040': {
+        'name': 'DiamondNet'
+    },
+    'ind060-dc': {
+        'name': 'Direct Communications'
+    },
+    'doy010': {
+        'name': 'Doylestown Cable TV'
+    },
+    'dic010': {
+        'name': 'DRN'
+    },
+    'dtc020': {
+        'name': 'DTC'
+    },
+    'dtc010': {
+        'name': 'DTC Cable (Delhi)'
+    },
+    'dum010': {
+        'name': 'Dumont Telephone Company'
+    },
+    'dun010': {
+        'name': 'Dunkerton Telephone Cooperative'
+    },
+    'cci010': {
+        'name': 'Duo County Telecom'
+    },
+    'eagle': {
+        'name': 'Eagle Communications'
+    },
+    'weh010-east': {
+        'name': 'East Arkansas Cable TV'
+    },
+    'eatel': {
+        'name': 'EATEL Video, LLC'
+    },
+    'ell010': {
+        'name': 'ECTA'
+    },
+    'emerytelcom': {
+        'name': 'Emery Telcom Video LLC'
+    },
+    'nor200': {
+        'name': 'Empire Access'
+    },
+    'endeavor': {
+        'name': 'Endeavor Communications'
+    },
+    'sun045': {
+        'name': 'Enhanced Telecommunications Corporation'
+    },
+    'mid030': {
+        'name': 'enTouch'
+    },
+    'epb020': {
+        'name': 'EPB Smartnet'
+    },
+    'jea010': {
+        'name': 'EPlus Broadband'
+    },
+    'com065': {
+        'name': 'ETC'
+    },
+    'ete010': {
+        'name': 'Etex Communications'
+    },
+    'fbc-tele': {
+        'name': 'F&B Communications'
+    },
+    'fal010': {
+        'name': 'Falcon Broadband'
+    },
+    'fam010': {
+        'name': 'FamilyView CableVision'
+    },
+    'far020': {
+        'name': 'Farmers Mutual Telephone Company'
+    },
+    'fay010': {
+        'name': 'Fayetteville Public Utilities'
+    },
+    'sal060': {
+        'name': 'fibrant'
+    },
+    'fid010': {
+        'name': 'Fidelity Communications'
+    },
+    'for030': {
+        'name': 'FJ Communications'
+    },
+    'fli020': {
+        'name': 'Flint River Communications'
+    },
+    'far030': {
+        'name': 'FMT - Jesup'
+    },
+    'foo010': {
+        'name': 'Foothills Communications'
+    },
+    'for080': {
+        'name': 'Forsyth CableNet'
+    },
+    'fbcomm': {
+        'name': 'Frankfort Plant Board'
+    },
+    'tel160-fra': {
+        'name': 'Franklin Telephone Company'
+    },
+    'nttcftc010': {
+        'name': 'FTC'
+    },
+    'fullchannel': {
+        'name': 'Full Channel, Inc.'
+    },
+    'gar040': {
+        'name': 'Gardonville Cooperative Telephone Association'
+    },
+    'gbt010': {
+        'name': 'GBT Communications, Inc.'
+    },
+    'tec010': {
+        'name': 'Genuine Telecom'
+    },
+    'clr010': {
+        'name': 'Giant Communications'
+    },
+    'gla010': {
+        'name': 'Glasgow EPB'
+    },
+    'gle010': {
+        'name': 'Glenwood Telecommunications'
+    },
+    'gra060': {
+        'name': 'GLW Broadband Inc.'
+    },
+    'goldenwest': {
+        'name': 'Golden West Cablevision'
+    },
+    'vis030': {
+        'name': 'Grantsburg Telcom'
+    },
+    'gpcom': {
+        'name': 'Great Plains Communications'
+    },
+    'gri010': {
+        'name': 'Gridley Cable Inc'
+    },
+    'hbc010': {
+        'name': 'H&B Cable Services'
+    },
+    'hae010': {
+        'name': 'Haefele TV Inc.'
+    },
+    'htc010': {
+        'name': 'Halstad Telephone Company'
+    },
+    'har005': {
+        'name': 'Harlan Municipal Utilities'
+    },
+    'har020': {
+        'name': 'Hart Communications'
+    },
+    'ced010': {
+        'name': 'Hartelco TV'
+    },
+    'hea040': {
+        'name': 'Heart of Iowa Communications Cooperative'
+    },
+    'htc020': {
+        'name': 'Hickory Telephone Company'
+    },
+    'nttchig010': {
+        'name': 'Highland Communication Services'
+    },
+    'hig030': {
+        'name': 'Highland Media'
+    },
+    'spc010': {
+        'name': 'Hilliary Communications'
+    },
+    'hin020': {
+        'name': 'Hinton CATV Co.'
+    },
+    'hometel': {
+        'name': 'HomeTel Entertainment, Inc.'
+    },
+    'hoodcanal': {
+        'name': 'Hood Canal Communications'
+    },
+    'weh010-hope': {
+        'name': 'Hope - Prescott Cable TV'
+    },
+    'horizoncable': {
+        'name': 'Horizon Cable TV, Inc.'
+    },
+    'hor040': {
+        'name': 'Horizon Chillicothe Telephone'
+    },
+    'htc030': {
+        'name': 'HTC Communications Co. - IL'
+    },
+    'htccomm': {
+        'name': 'HTC Communications, Inc. - IA'
+    },
+    'wal005': {
+        'name': 'Huxley Communications'
+    },
+    'imon': {
+        'name': 'ImOn Communications'
+    },
+    'ind040': {
+        'name': 'Independence Telecommunications'
+    },
+    'rrc010': {
+        'name': 'Inland Networks'
+    },
+    'stc020': {
+        'name': 'Innovative Cable TV St Croix'
+    },
+    'car100': {
+        'name': 'Innovative Cable TV St Thomas-St John'
+    },
+    'icc010': {
+        'name': 'Inside Connect Cable'
+    },
+    'int100': {
+        'name': 'Integra Telecom'
+    },
+    'int050': {
+        'name': 'Interstate Telecommunications Coop'
+    },
+    'irv010': {
+        'name': 'Irvine Cable'
+    },
+    'k2c010': {
+        'name': 'K2 Communications'
+    },
+    'kal010': {
+        'name': 'Kalida Telephone Company, Inc.'
+    },
+    'kal030': {
+        'name': 'Kalona Cooperative Telephone Company'
+    },
+    'kmt010': {
+        'name': 'KMTelecom'
+    },
+    'kpu010': {
+        'name': 'KPU Telecommunications'
+    },
+    'kuh010': {
+        'name': 'Kuhn Communications, Inc.'
+    },
+    'lak130': {
+        'name': 'Lakeland Communications'
+    },
+    'lan010': {
+        'name': 'Langco'
+    },
+    'lau020': {
+        'name': 'Laurel Highland Total Communications, Inc.'
+    },
+    'leh010': {
+        'name': 'Lehigh Valley Cooperative Telephone'
+    },
+    'bra010': {
+        'name': 'Limestone Cable/Bracken Cable'
+    },
+    'loc020': {
+        'name': 'LISCO'
+    },
+    'lit020': {
+        'name': 'Litestream'
+    },
+    'tel140': {
+        'name': 'LivCom'
+    },
+    'loc010': {
+        'name': 'LocalTel Communications'
+    },
+    'weh010-longview': {
+        'name': 'Longview - Kilgore Cable TV'
+    },
+    'lon030': {
+        'name': 'Lonsdale Video Ventures, LLC'
+    },
+    'lns010': {
+        'name': 'Lost Nation-Elwood Telephone Co.'
+    },
+    'nttclpc010': {
+        'name': 'LPC Connect'
+    },
+    'lumos': {
+        'name': 'Lumos Networks'
+    },
+    'madison': {
+        'name': 'Madison Communications'
+    },
+    'mad030': {
+        'name': 'Madison County Cable Inc.'
+    },
+    'nttcmah010': {
+        'name': 'Mahaska Communication Group'
+    },
+    'mar010': {
+        'name': 'Marne & Elk Horn Telephone Company'
+    },
+    'mcc040': {
+        'name': 'McClure Telephone Co.'
+    },
+    'mctv': {
+        'name': 'MCTV'
+    },
+    'merrimac': {
+        'name': 'Merrimac Communications Ltd.'
+    },
+    'metronet': {
+        'name': 'Metronet'
+    },
+    'mhtc': {
+        'name': 'MHTC'
+    },
+    'midhudson': {
+        'name': 'Mid-Hudson Cable'
+    },
+    'midrivers': {
+        'name': 'Mid-Rivers Communications'
+    },
+    'mid045': {
+        'name': 'Midstate Communications'
+    },
+    'mil080': {
+        'name': 'Milford Communications'
+    },
+    'min030': {
+        'name': 'MINET'
+    },
+    'nttcmin010': {
+        'name': 'Minford TV'
+    },
+    'san040-02': {
+        'name': 'Mitchell Telecom'
+    },
+    'mlg010': {
+        'name': 'MLGC'
+    },
+    'mon060': {
+        'name': 'Mon-Cre TVE'
+    },
+    'mou110': {
+        'name': 'Mountain Telephone'
+    },
+    'mou050': {
+        'name': 'Mountain Village Cable'
+    },
+    'mtacomm': {
+        'name': 'MTA Communications, LLC'
+    },
+    'mtc010': {
+        'name': 'MTC Cable'
+    },
+    'med040': {
+        'name': 'MTC Technologies'
+    },
+    'man060': {
+        'name': 'MTCC'
+    },
+    'mtc030': {
+        'name': 'MTCO Communications'
+    },
+    'mul050': {
+        'name': 'Mulberry Telecommunications'
+    },
+    'mur010': {
+        'name': 'Murray Electric System'
+    },
+    'musfiber': {
+        'name': 'MUS FiberNET'
+    },
+    'mpw': {
+        'name': 'Muscatine Power & Water'
+    },
+    'nttcsli010': {
+        'name': 'myEVTV.com'
+    },
+    'nor115': {
+        'name': 'NCC'
+    },
+    'nor260': {
+        'name': 'NDTC'
+    },
+    'nctc': {
+        'name': 'Nebraska Central Telecom, Inc.'
+    },
+    'nel020': {
+        'name': 'Nelsonville TV Cable'
+    },
+    'nem010': {
+        'name': 'Nemont'
+    },
+    'new075': {
+        'name': 'New Hope Telephone Cooperative'
+    },
+    'nor240': {
+        'name': 'NICP'
+    },
+    'cic010': {
+        'name': 'NineStar Connect'
+    },
+    'nktelco': {
+        'name': 'NKTelco'
+    },
+    'nortex': {
+        'name': 'Nortex Communications'
+    },
+    'nor140': {
+        'name': 'North Central Telephone Cooperative'
+    },
+    'nor030': {
+        'name': 'Northland Communications'
+    },
+    'nor075': {
+        'name': 'Northwest Communications'
+    },
+    'nor125': {
+        'name': 'Norwood Light Broadband'
+    },
+    'net010': {
+        'name': 'Nsight Telservices'
+    },
+    'dur010': {
+        'name': 'Ntec'
+    },
+    'nts010': {
+        'name': 'NTS Communications'
+    },
+    'new045': {
+        'name': 'NU-Telecom'
+    },
+    'nulink': {
+        'name': 'NuLink'
+    },
+    'jam030': {
+        'name': 'NVC'
+    },
+    'far035': {
+        'name': 'OmniTel Communications'
+    },
+    'onesource': {
+        'name': 'OneSource Communications'
+    },
+    'cit230': {
+        'name': 'Opelika Power Services'
+    },
+    'daltonutilities': {
+        'name': 'OptiLink'
+    },
+    'mid140': {
+        'name': 'OPTURA'
+    },
+    'ote010': {
+        'name': 'OTEC Communication Company'
+    },
+    'cci020': {
+        'name': 'Packerland Broadband'
+    },
+    'pan010': {
+        'name': 'Panora Telco/Guthrie Center Communications'
+    },
+    'otter': {
+        'name': 'Park Region Telephone & Otter Tail Telcom'
+    },
+    'mid050': {
+        'name': 'Partner Communications Cooperative'
+    },
+    'fib010': {
+        'name': 'Pathway'
+    },
+    'paulbunyan': {
+        'name': 'Paul Bunyan Communications'
+    },
+    'pem020': {
+        'name': 'Pembroke Telephone Company'
+    },
+    'mck010': {
+        'name': 'Peoples Rural Telephone Cooperative'
+    },
+    'pul010': {
+        'name': 'PES Energize'
+    },
+    'phi010': {
+        'name': 'Philippi Communications System'
+    },
+    'phonoscope': {
+        'name': 'Phonoscope Cable'
+    },
+    'pin070': {
+        'name': 'Pine Belt Communications, Inc.'
+    },
+    'weh010-pine': {
+        'name': 'Pine Bluff Cable TV'
+    },
+    'pin060': {
+        'name': 'Pineland Telephone Cooperative'
+    },
+    'cam010': {
+        'name': 'Pinpoint Communications'
+    },
+    'pio060': {
+        'name': 'Pioneer Broadband'
+    },
+    'pioncomm': {
+        'name': 'Pioneer Communications'
+    },
+    'pioneer': {
+        'name': 'Pioneer DTV'
+    },
+    'pla020': {
+        'name': 'Plant TiftNet, Inc.'
+    },
+    'par010': {
+        'name': 'PLWC'
+    },
+    'pro035': {
+        'name': 'PMT'
+    },
+    'vik011': {
+        'name': 'Polar Cablevision'
+    },
+    'pottawatomie': {
+        'name': 'Pottawatomie Telephone Co.'
+    },
+    'premiercomm': {
+        'name': 'Premier Communications'
+    },
+    'psc010': {
+        'name': 'PSC'
+    },
+    'pan020': {
+        'name': 'PTCI'
+    },
+    'qco010': {
+        'name': 'QCOL'
+    },
+    'qua010': {
+        'name': 'Quality Cablevision'
+    },
+    'rad010': {
+        'name': 'Radcliffe Telephone Company'
+    },
+    'car040': {
+        'name': 'Rainbow Communications'
+    },
+    'rai030': {
+        'name': 'Rainier Connect'
+    },
+    'ral010': {
+        'name': 'Ralls Technologies'
+    },
+    'rct010': {
+        'name': 'RC Technologies'
+    },
+    'red040': {
+        'name': 'Red River Communications'
+    },
+    'ree010': {
+        'name': 'Reedsburg Utility Commission'
+    },
+    'mol010': {
+        'name': 'Reliance Connects- Oregon'
+    },
+    'res020': {
+        'name': 'Reserve Telecommunications'
+    },
+    'weh010-resort': {
+        'name': 'Resort TV Cable'
+    },
+    'rld010': {
+        'name': 'Richland Grant Telephone Cooperative, Inc.'
+    },
+    'riv030': {
+        'name': 'River Valley Telecommunications Coop'
+    },
+    'rockportcable': {
+        'name': 'Rock Port Cablevision'
+    },
+    'rsf010': {
+        'name': 'RS Fiber'
+    },
+    'rtc': {
+        'name': 'RTC Communication Corp'
+    },
+    'res040': {
+        'name': 'RTC-Reservation Telephone Coop.'
+    },
+    'rte010': {
+        'name': 'RTEC Communications'
+    },
+    'stc010': {
+        'name': 'S&T'
+    },
+    'san020': {
+        'name': 'San Bruno Cable TV'
+    },
+    'san040-01': {
+        'name': 'Santel'
+    },
+    'sav010': {
+        'name': 'SCI Broadband-Savage Communications Inc.'
+    },
+    'sco050': {
+        'name': 'Scottsboro Electric Power Board'
+    },
+    'scr010': {
+        'name': 'Scranton Telephone Company'
+    },
+    'selco': {
+        'name': 'SELCO'
+    },
+    'she010': {
+        'name': 'Shentel'
+    },
+    'she030': {
+        'name': 'Sherwood Mutual Telephone Association, Inc.'
+    },
+    'ind060-ssc': {
+        'name': 'Silver Star Communications'
+    },
+    'sjoberg': {
+        'name': 'Sjoberg\'s Inc.'
+    },
+    'sou025': {
+        'name': 'SKT'
+    },
+    'sky050': {
+        'name': 'SkyBest TV'
+    },
+    'nttcsmi010': {
+        'name': 'Smithville Communications'
+    },
+    'woo010': {
+        'name': 'Solarus'
+    },
+    'sou075': {
+        'name': 'South Central Rural Telephone Cooperative'
+    },
+    'sou065': {
+        'name': 'South Holt Cablevision, Inc.'
+    },
+    'sou035': {
+        'name': 'South Slope Cooperative Communications'
+    },
+    'spa020': {
+        'name': 'Spanish Fork Community Network'
+    },
+    'spe010': {
+        'name': 'Spencer Municipal Utilities'
+    },
+    'spi005': {
+        'name': 'Spillway Communications, Inc.'
+    },
+    'srt010': {
+        'name': 'SRT'
+    },
+    'cccsmc010': {
+        'name': 'St. Maarten Cable TV'
+    },
+    'sta025': {
+        'name': 'Star Communications'
+    },
+    'sco020': {
+        'name': 'STE'
+    },
+    'uin010': {
+        'name': 'STRATA Networks'
+    },
+    'sum010': {
+        'name': 'Sumner Cable TV'
+    },
+    'pie010': {
+        'name': 'Surry TV/PCSI TV'
+    },
+    'swa010': {
+        'name': 'Swayzee Communications'
+    },
+    'sweetwater': {
+        'name': 'Sweetwater Cable Television Co'
+    },
+    'weh010-talequah': {
+        'name': 'Tahlequah Cable TV'
+    },
+    'tct': {
+        'name': 'TCT'
+    },
+    'tel050': {
+        'name': 'Tele-Media Company'
+    },
+    'com050': {
+        'name': 'The Community Agency'
+    },
+    'thr020': {
+        'name': 'Three River'
+    },
+    'cab140': {
+        'name': 'Town & Country Technologies'
+    },
+    'tra010': {
+        'name': 'Trans-Video'
+    },
+    'tre010': {
+        'name': 'Trenton TV Cable Company'
+    },
+    'tcc': {
+        'name': 'Tri County Communications Cooperative'
+    },
+    'tri025': {
+        'name': 'TriCounty Telecom'
+    },
+    'tri110': {
+        'name': 'TrioTel Communications, Inc.'
+    },
+    'tro010': {
+        'name': 'Troy Cablevision, Inc.'
+    },
+    'tsc': {
+        'name': 'TSC'
+    },
+    'cit220': {
+        'name': 'Tullahoma Utilities Board'
+    },
+    'tvc030': {
+        'name': 'TV Cable of Rensselaer'
+    },
+    'tvc015': {
+        'name': 'TVC Cable'
+    },
+    'cab180': {
+        'name': 'TVision'
+    },
+    'twi040': {
+        'name': 'Twin Lakes'
+    },
+    'tvtinc': {
+        'name': 'Twin Valley'
+    },
+    'uis010': {
+        'name': 'Union Telephone Company'
+    },
+    'uni110': {
+        'name': 'United Communications - TN'
+    },
+    'uni120': {
+        'name': 'United Services'
+    },
+    'uss020': {
+        'name': 'US Sonet'
+    },
+    'cab060': {
+        'name': 'USA Communications'
+    },
+    'she005': {
+        'name': 'USA Communications/Shellsburg, IA'
+    },
+    'val040': {
+        'name': 'Valley TeleCom Group'
+    },
+    'val025': {
+        'name': 'Valley Telecommunications'
+    },
+    'val030': {
+        'name': 'Valparaiso Broadband'
+    },
+    'cla050': {
+        'name': 'Vast Broadband'
+    },
+    'sul015': {
+        'name': 'Venture Communications Cooperative, Inc.'
+    },
+    'ver025': {
+        'name': 'Vernon Communications Co-op'
+    },
+    'weh010-vicksburg': {
+        'name': 'Vicksburg Video'
+    },
+    'vis070': {
+        'name': 'Vision Communications'
+    },
+    'volcanotel': {
+        'name': 'Volcano Vision, Inc.'
+    },
+    'vol040-02': {
+        'name': 'VolFirst / BLTV'
+    },
+    'ver070': {
+        'name': 'VTel'
+    },
+    'nttcvtx010': {
+        'name': 'VTX1'
+    },
+    'bci010-02': {
+        'name': 'Vyve Broadband'
+    },
+    'wab020': {
+        'name': 'Wabash Mutual Telephone'
+    },
+    'waitsfield': {
+        'name': 'Waitsfield Cable'
+    },
+    'wal010': {
+        'name': 'Walnut Communications'
+    },
+    'wavebroadband': {
+        'name': 'Wave'
+    },
+    'wav030': {
+        'name': 'Waverly Communications Utility'
+    },
+    'wbi010': {
+        'name': 'WBI'
+    },
+    'web020': {
+        'name': 'Webster-Calhoun Cooperative Telephone Association'
+    },
+    'wes005': {
+        'name': 'West Alabama TV Cable'
+    },
+    'carolinata': {
+        'name': 'West Carolina Communications'
+    },
+    'wct010': {
+        'name': 'West Central Telephone Association'
+    },
+    'wes110': {
+        'name': 'West River Cooperative Telephone Company'
+    },
+    'ani030': {
+        'name': 'WesTel Systems'
+    },
+    'westianet': {
+        'name': 'Western Iowa Networks'
+    },
+    'nttcwhi010': {
+        'name': 'Whidbey Telecom'
+    },
+    'weh010-white': {
+        'name': 'White County Cable TV'
+    },
+    'wes130': {
+        'name': 'Wiatel'
+    },
+    'wik010': {
+        'name': 'Wiktel'
+    },
+    'wil070': {
+        'name': 'Wilkes Communications, Inc./RiverStreet Networks'
+    },
+    'wil015': {
+        'name': 'Wilson Communications'
+    },
+    'win010': {
+        'name': 'Windomnet/SMBS'
+    },
+    'win090': {
+        'name': 'Windstream Cable TV'
+    },
+    'wcta': {
+        'name': 'Winnebago Cooperative Telecom Association'
+    },
+    'wtc010': {
+        'name': 'WTC'
+    },
+    'wil040': {
+        'name': 'WTC Communications, Inc.'
+    },
+    'wya010': {
+        'name': 'Wyandotte Cable'
+    },
+    'hin020-02': {
+        'name': 'X-Stream Services'
+    },
+    'xit010': {
+        'name': 'XIT Communications'
+    },
+    'yel010': {
+        'name': 'Yelcot Communications'
+    },
+    'mid180-01': {
+        'name': 'yondoo'
+    },
+    'cou060': {
+        'name': 'Zito Media'
+    },
 }
 
 
@@ -113,10 +1367,10 @@ class AdobePassIE(InfoExtractor):
                 provider_login_page_res = post_form(
                     provider_redirect_page_res, 'Downloading Provider Login Page')
                 mvpd_confirm_page_res = post_form(provider_login_page_res, 'Logging in', {
-                    mso_info['username_field']: username,
-                    mso_info['password_field']: password,
+                    mso_info.get('username_field', 'username'): username,
+                    mso_info.get('password_field', 'password'): password,
                 })
-                if mso_id == 'DTV':
+                if mso_id != 'Rogers':
                     post_form(mvpd_confirm_page_res, 'Confirming Login')
 
                 session = self._download_webpage(

From 12f211d0cbd25554ff3116ee173ffc3f25d0e453 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 21 Sep 2016 22:51:36 +0700
Subject: [PATCH 6135/6219] [videomore] Fix embed regex

---
 youtube_dl/extractor/videomore.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/videomore.py b/youtube_dl/extractor/videomore.py
index 328b5b7fb..8a11ff848 100644
--- a/youtube_dl/extractor/videomore.py
+++ b/youtube_dl/extractor/videomore.py
@@ -84,7 +84,7 @@ class VideomoreIE(InfoExtractor):
     @staticmethod
     def _extract_url(webpage):
         mobj = re.search(
-            r'<object[^>]+data=(["\'])https?://videomore.ru/player\.swf\?.*config=(?P<url>https?://videomore\.ru/(?:[^/]+/)+\d+\.xml).*\1',
+            r'<object[^>]+data=(["\'])https?://videomore\.ru/player\.swf\?.*config=(?P<url>https?://videomore\.ru/(?:[^/]+/)+\d+\.xml).*\1',
             webpage)
         if mobj:
             return mobj.group('url')

From 1978540a5122c53012e17a78841f3da0df77fd34 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 21 Sep 2016 21:49:52 +0100
Subject: [PATCH 6136/6219] [ooyala] extract all hls formats

---
 youtube_dl/extractor/ooyala.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ooyala.py b/youtube_dl/extractor/ooyala.py
index 2038a6ba5..72ec20938 100644
--- a/youtube_dl/extractor/ooyala.py
+++ b/youtube_dl/extractor/ooyala.py
@@ -47,7 +47,7 @@ class OoyalaBaseIE(InfoExtractor):
                 delivery_type = stream['delivery_type']
                 if delivery_type == 'hls' or ext == 'm3u8':
                     formats.extend(self._extract_m3u8_formats(
-                        s_url, embed_code, 'mp4', 'm3u8_native',
+                        re.sub(r'/ip(?:ad|hone)/', '/all/', s_url), embed_code, 'mp4', 'm3u8_native',
                         m3u8_id='hls', fatal=False))
                 elif delivery_type == 'hds' or ext == 'f4m':
                     formats.extend(self._extract_f4m_formats(

From 0a439c5c4c1a6a2ee54465c5ad893ffb768539d3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 22 Sep 2016 21:48:53 +0700
Subject: [PATCH 6137/6219] [udemy] Stringify video id

---
 youtube_dl/extractor/udemy.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index c2f507233..cce29c6e0 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -5,6 +5,7 @@ import re
 from .common import InfoExtractor
 from ..compat import (
     compat_HTTPError,
+    compat_str,
     compat_urllib_request,
     compat_urlparse,
 )
@@ -207,7 +208,7 @@ class UdemyIE(InfoExtractor):
             if youtube_url:
                 return self.url_result(youtube_url, 'Youtube')
 
-        video_id = asset['id']
+        video_id = compat_str(asset['id'])
         thumbnail = asset.get('thumbnail_url') or asset.get('thumbnailUrl')
         duration = float_or_none(asset.get('data', {}).get('duration'))
 

From e3d6bdc8fc48ddf0bea324c9196297e539669aaf Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 23 Sep 2016 01:11:13 +0800
Subject: [PATCH 6138/6219] [ustream] Support HLS streams (closes #10698)

---
 ChangeLog                       |   5 ++
 youtube_dl/extractor/ustream.py | 122 +++++++++++++++++++++++++++++++-
 2 files changed, 126 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index 24077c430..5122af4c0 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,8 @@
+version <unreleased>
+
+Extractors
++ [ustream] Support the new HLS streams (#10698)
+
 version 2016.09.19
 
 Extractors
diff --git a/youtube_dl/extractor/ustream.py b/youtube_dl/extractor/ustream.py
index a3dc9d33e..0c06bf36b 100644
--- a/youtube_dl/extractor/ustream.py
+++ b/youtube_dl/extractor/ustream.py
@@ -1,15 +1,20 @@
 from __future__ import unicode_literals
 
+import random
 import re
 
 from .common import InfoExtractor
 from ..compat import (
+    compat_str,
     compat_urlparse,
 )
 from ..utils import (
+    encode_data_uri,
     ExtractorError,
     int_or_none,
     float_or_none,
+    mimetype2ext,
+    str_or_none,
 )
 
 
@@ -47,8 +52,108 @@ class UstreamIE(InfoExtractor):
             'id': '10299409',
         },
         'playlist_count': 3,
+    }, {
+        'url': 'http://www.ustream.tv/recorded/91343263',
+        'info_dict': {
+            'id': '91343263',
+            'ext': 'mp4',
+            'title': 'GitHub Universe - General Session - Day 1',
+            'upload_date': '20160914',
+            'description': 'GitHub Universe - General Session - Day 1',
+            'timestamp': 1473872730,
+            'uploader': 'wa0dnskeqkr',
+            'uploader_id': '38977840',
+        },
+        'params': {
+            'skip_download': True,  # m3u8 download
+        },
     }]
 
+    def _get_stream_info(self, url, video_id, app_id_ver, extra_note=None):
+        def num_to_hex(n):
+            return hex(n)[2:]
+
+        rnd = random.randrange
+
+        if not extra_note:
+            extra_note = ''
+
+        conn_info = self._download_json(
+            'http://r%d-1-%s-recorded-lp-live.ums.ustream.tv/1/ustream' % (rnd(1e8), video_id),
+            video_id, note='Downloading connection info' + extra_note,
+            query={
+                'type': 'viewer',
+                'appId': app_id_ver[0],
+                'appVersion': app_id_ver[1],
+                'rsid': '%s:%s' % (num_to_hex(rnd(1e8)), num_to_hex(rnd(1e8))),
+                'rpin': '_rpin.%d' % rnd(1e15),
+                'referrer': url,
+                'media': video_id,
+                'application': 'recorded',
+            })
+        host = conn_info[0]['args'][0]['host']
+        connection_id = conn_info[0]['args'][0]['connectionId']
+
+        return self._download_json(
+            'http://%s/1/ustream?connectionId=%s' % (host, connection_id),
+            video_id, note='Downloading stream info' + extra_note)
+
+    def _get_streams(self, url, video_id, app_id_ver):
+        # Sometimes the return dict does not have 'stream'
+        for trial_count in range(3):
+            stream_info = self._get_stream_info(
+                url, video_id, app_id_ver,
+                extra_note=' (try %d)' % (trial_count + 1) if trial_count > 0 else '')
+            if 'stream' in stream_info[0]['args'][0]:
+                return stream_info[0]['args'][0]['stream']
+        return []
+
+    def _parse_segmented_mp4(self, dash_stream_info):
+        def resolve_dash_template(template, idx, chunk_hash):
+            return template.replace('%', compat_str(idx), 1).replace('%', chunk_hash)
+
+        formats = []
+        for stream in dash_stream_info['streams']:
+            # Use only one provider to avoid too many formats
+            provider = dash_stream_info['providers'][0]
+            fragments = [{
+                'url': resolve_dash_template(
+                    provider['url'] + stream['initUrl'], 0, dash_stream_info['hashes']['0'])
+            }]
+            for idx in range(dash_stream_info['videoLength'] // dash_stream_info['chunkTime']):
+                fragments.append({
+                    'url': resolve_dash_template(
+                        provider['url'] + stream['segmentUrl'], idx,
+                        dash_stream_info['hashes'][compat_str(idx // 10 * 10)])
+                })
+            content_type = stream['contentType']
+            kind = content_type.split('/')[0]
+            f = {
+                'format_id': '-'.join(filter(None, [
+                    'dash', kind, str_or_none(stream.get('bitrate'))])),
+                'protocol': 'http_dash_segments',
+                # TODO: generate a MPD doc for external players?
+                'url': encode_data_uri(b'<MPD/>', 'text/xml'),
+                'ext': mimetype2ext(content_type),
+                'height': stream.get('height'),
+                'width': stream.get('width'),
+                'fragments': fragments,
+            }
+            if kind == 'video':
+                f.update({
+                    'vcodec': stream.get('codec'),
+                    'acodec': 'none',
+                    'vbr': stream.get('bitrate'),
+                })
+            else:
+                f.update({
+                    'vcodec': 'none',
+                    'acodec': stream.get('codec'),
+                    'abr': stream.get('bitrate'),
+                })
+            formats.append(f)
+        return formats
+
     def _real_extract(self, url):
         m = re.match(self._VALID_URL, url)
         video_id = m.group('id')
@@ -86,7 +191,22 @@ class UstreamIE(InfoExtractor):
             'url': video_url,
             'ext': format_id,
             'filesize': filesize,
-        } for format_id, video_url in video['media_urls'].items()]
+        } for format_id, video_url in video['media_urls'].items() if video_url]
+
+        if not formats:
+            hls_streams = self._get_streams(url, video_id, app_id_ver=(11, 2))
+            if hls_streams:
+                # m3u8_native leads to intermittent ContentTooShortError
+                formats.extend(self._extract_m3u8_formats(
+                    hls_streams[0]['url'], video_id, ext='mp4', m3u8_id='hls'))
+
+            '''
+            # DASH streams handling is incomplete as 'url' is missing
+            dash_streams = self._get_streams(url, video_id, app_id_ver=(3, 1))
+            if dash_streams:
+                formats.extend(self._parse_segmented_mp4(dash_streams))
+            '''
+
         self._sort_formats(formats)
 
         description = video.get('description')

From 628406db960c032eb68ef318ce9fecf6b8329834 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 23 Sep 2016 01:13:56 +0800
Subject: [PATCH 6139/6219] [Makefile] Cleanup files from fragment-based
 downloaders

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 354052c50..ac234fcb0 100644
--- a/Makefile
+++ b/Makefile
@@ -1,7 +1,7 @@
 all: youtube-dl README.md CONTRIBUTING.md README.txt youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtube-dl.fish supportedsites
 
 clean:
-	rm -rf youtube-dl.1.temp.md youtube-dl.1 youtube-dl.bash-completion README.txt MANIFEST build/ dist/ .coverage cover/ youtube-dl.tar.gz youtube-dl.zsh youtube-dl.fish youtube_dl/extractor/lazy_extractors.py *.dump *.part *.info.json *.mp4 *.m4a *.flv *.mp3 *.avi *.mkv *.webm *.jpg *.png CONTRIBUTING.md.tmp ISSUE_TEMPLATE.md.tmp youtube-dl youtube-dl.exe
+	rm -rf youtube-dl.1.temp.md youtube-dl.1 youtube-dl.bash-completion README.txt MANIFEST build/ dist/ .coverage cover/ youtube-dl.tar.gz youtube-dl.zsh youtube-dl.fish youtube_dl/extractor/lazy_extractors.py *.dump *.part* *.info.json *.mp4 *.m4a *.flv *.mp3 *.avi *.mkv *.webm *.jpg *.png CONTRIBUTING.md.tmp ISSUE_TEMPLATE.md.tmp youtube-dl youtube-dl.exe
 	find . -name "*.pyc" -delete
 	find . -name "*.class" -delete
 

From 4ddcb5999d0323fb83c5b879127d31763f5d63e2 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 23 Sep 2016 01:47:01 +0800
Subject: [PATCH 6140/6219] [openload] Fix extraction (closes #10408, closes
 #10727)

Thanks to @daniel100097 for providing a working version
---
 ChangeLog                        | 1 +
 youtube_dl/extractor/openload.py | 5 +++--
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 5122af4c0..6c72bae90 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
 version <unreleased>
 
 Extractors
+* [openload] Fix extraction (#10408)
 + [ustream] Support the new HLS streams (#10698)
 
 version 2016.09.19
diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py
index c261a7455..b6e3ac250 100644
--- a/youtube_dl/extractor/openload.py
+++ b/youtube_dl/extractor/openload.py
@@ -51,7 +51,8 @@ class OpenloadIE(InfoExtractor):
         # declared to be freely used in youtube-dl
         # See https://github.com/rg3/youtube-dl/issues/10408
         enc_data = self._html_search_regex(
-            r'<span[^>]+id="hiddenurl"[^>]*>([^<]+)</span>', webpage, 'encrypted data')
+            r'<span[^>]*>([^<]+)</span>\s*<span[^>]*>[^<]+</span>\s*<span[^>]+id="streamurl"',
+            webpage, 'encrypted data')
 
         video_url_chars = []
 
@@ -60,7 +61,7 @@ class OpenloadIE(InfoExtractor):
             if j >= 33 and j <= 126:
                 j = ((j + 14) % 94) + 33
             if idx == len(enc_data) - 1:
-                j += 3
+                j += 2
             video_url_chars += compat_chr(j)
 
         video_url = 'https://openload.co/stream/%s?mime=true' % ''.join(video_url_chars)

From 45cae3b021828cc6f7a67c7a14645ae6f0806f59 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 22 Sep 2016 19:27:57 +0100
Subject: [PATCH 6141/6219] [cbs] extract info from thunder
 videoPlayerService(closes #10728)

---
 youtube_dl/extractor/cbs.py | 58 ++++++++++++++++++++++++++-----------
 1 file changed, 41 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/cbs.py b/youtube_dl/extractor/cbs.py
index 3f4dea40c..58f258c54 100644
--- a/youtube_dl/extractor/cbs.py
+++ b/youtube_dl/extractor/cbs.py
@@ -4,7 +4,9 @@ from .theplatform import ThePlatformFeedIE
 from ..utils import (
     int_or_none,
     find_xpath_attr,
-    ExtractorError,
+    xpath_element,
+    xpath_text,
+    update_url_query,
 )
 
 
@@ -47,27 +49,49 @@ class CBSIE(CBSBaseIE):
         'only_matching': True,
     }]
 
-    def _extract_video_info(self, guid):
-        path = 'dJ5BDC/media/guid/2198311517/' + guid
-        smil_url = 'http://link.theplatform.com/s/%s?mbr=true' % path
-        formats, subtitles = self._extract_theplatform_smil(smil_url + '&manifest=m3u', guid)
-        for r in ('OnceURL&formats=M3U', 'HLS&formats=M3U', 'RTMP', 'WIFI', '3G'):
-            try:
-                tp_formats, _ = self._extract_theplatform_smil(smil_url + '&assetTypes=' + r, guid, 'Downloading %s SMIL data' % r.split('&')[0])
-                formats.extend(tp_formats)
-            except ExtractorError:
+    def _extract_video_info(self, content_id):
+        items_data = self._download_xml(
+            'http://can.cbs.com/thunder/player/videoPlayerService.php',
+            content_id, query={'partner': 'cbs', 'contentId': content_id})
+        video_data = xpath_element(items_data, './/item')
+        title = xpath_text(video_data, 'videoTitle', 'title', True)
+        tp_path = 'dJ5BDC/media/guid/2198311517/%s' % content_id
+        tp_release_url = 'http://link.theplatform.com/s/' + tp_path
+
+        asset_types = []
+        subtitles = {}
+        formats = []
+        for item in items_data.findall('.//item'):
+            asset_type = xpath_text(item, 'assetType')
+            if not asset_type or asset_type in asset_types:
                 continue
+            asset_types.append(asset_type)
+            query = {
+                'mbr': 'true',
+                'assetTypes': asset_type,
+            }
+            if asset_type.startswith('HLS') or asset_type in ('OnceURL', 'StreamPack'):
+                query['formats'] = 'MPEG4,M3U'
+            elif asset_type in ('RTMP', 'WIFI', '3G'):
+                query['formats'] = 'MPEG4,FLV'
+            tp_formats, tp_subtitles = self._extract_theplatform_smil(
+                update_url_query(tp_release_url, query), content_id,
+                'Downloading %s SMIL data' % asset_type)
+            formats.extend(tp_formats)
+            subtitles = self._merge_subtitles(subtitles, tp_subtitles)
         self._sort_formats(formats)
-        metadata = self._download_theplatform_metadata(path, guid)
-        info = self._parse_theplatform_metadata(metadata)
+
+        info = self._extract_theplatform_metadata(tp_path, content_id)
         info.update({
-            'id': guid,
+            'id': content_id,
+            'title': title,
+            'series': xpath_text(video_data, 'seriesTitle'),
+            'season_number': int_or_none(xpath_text(video_data, 'seasonNumber')),
+            'episode_number': int_or_none(xpath_text(video_data, 'episodeNumber')),
+            'duration': int_or_none(xpath_text(video_data, 'videoLength'), 1000),
+            'thumbnail': xpath_text(video_data, 'previewImageURL'),
             'formats': formats,
             'subtitles': subtitles,
-            'series': metadata.get('cbs$SeriesTitle'),
-            'season_number': int_or_none(metadata.get('cbs$SeasonNumber')),
-            'episode': metadata.get('cbs$EpisodeTitle'),
-            'episode_number': int_or_none(metadata.get('cbs$EpisodeNumber')),
         })
         return info
 

From 71ad00c09fecd3ecc84784cf215537cad0a79595 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 23 Sep 2016 21:08:16 +0700
Subject: [PATCH 6142/6219] [prosiebensat1] Add support for kabeleinsdoku
 (Closes #10732)

---
 youtube_dl/extractor/prosiebensat1.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/prosiebensat1.py b/youtube_dl/extractor/prosiebensat1.py
index 7335dc2af..5a29b844d 100644
--- a/youtube_dl/extractor/prosiebensat1.py
+++ b/youtube_dl/extractor/prosiebensat1.py
@@ -122,7 +122,7 @@ class ProSiebenSat1BaseIE(InfoExtractor):
 class ProSiebenSat1IE(ProSiebenSat1BaseIE):
     IE_NAME = 'prosiebensat1'
     IE_DESC = 'ProSiebenSat.1 Digital'
-    _VALID_URL = r'https?://(?:www\.)?(?:(?:prosieben|prosiebenmaxx|sixx|sat1|kabeleins|the-voice-of-germany|7tv)\.(?:de|at|ch)|ran\.de|fem\.com)/(?P<id>.+)'
+    _VALID_URL = r'https?://(?:www\.)?(?:(?:prosieben|prosiebenmaxx|sixx|sat1|kabeleins|the-voice-of-germany|7tv|kabeleinsdoku)\.(?:de|at|ch)|ran\.de|fem\.com)/(?P<id>.+)'
 
     _TESTS = [
         {
@@ -290,6 +290,11 @@ class ProSiebenSat1IE(ProSiebenSat1BaseIE):
                 'skip_download': True,
             },
         },
+        {
+            # geo restricted to Germany
+            'url': 'http://www.kabeleinsdoku.de/tv/mayday-alarm-im-cockpit/video/102-notlandung-im-hudson-river-ganze-folge',
+            'only_matching': True,
+        },
     ]
 
     _TOKEN = 'prosieben'

From 24628cf7db46ecce3fe56d387266c556cd9210ba Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 24 Sep 2016 02:01:01 +0700
Subject: [PATCH 6143/6219] [soundcloud:playlist] Provide video id for playlist
 entries (Closes #10733)

---
 youtube_dl/extractor/soundcloud.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 513c54829..496cc5d8e 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -477,7 +477,11 @@ class SoundcloudPlaylistIE(SoundcloudIE):
         data = self._download_json(
             base_url + data, playlist_id, 'Downloading playlist')
 
-        entries = [self.url_result(track['permalink_url'], 'Soundcloud') for track in data['tracks']]
+        entries = [
+            self.url_result(
+                track['permalink_url'], SoundcloudIE.ie_key(),
+                video_id=compat_str(track['id']) if track.get('id') else None)
+            for track in data['tracks'] if track.get('permalink_url')]
 
         return {
             '_type': 'playlist',

From 8eec691e8a89d0094b806b86111fbcfd0ade64c9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 24 Sep 2016 02:12:49 +0700
Subject: [PATCH 6144/6219] [ChangeLog] Actualize

---
 ChangeLog | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 6c72bae90..e0908aa30 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,8 +1,23 @@
 version <unreleased>
 
+Core
++ Add support for watchTVeverywhere.com authentication provider based MSOs for
+  Adobe Pass authentication (#10709)
+
 Extractors
++ [soundcloud:playlist] Provide video id for early playlist entries (#10733)
++ [prosiebensat1] Add support for kabeleinsdoku (#10732)
+* [cbs] Extract info from thunder videoPlayerService (#10728)
 * [openload] Fix extraction (#10408)
 + [ustream] Support the new HLS streams (#10698)
++ [ooyala] Extract all HLS formats
++ [cartoonnetwork] Add support for Adobe Pass authentication
++ [soundcloud] Extract license metadata
++ [fox] Add support for Adobe Pass authentication (#8584)
++ [tbs] Add support for Adobe Pass authentication (#10642, #10222)
++ [trutv] Add support for Adobe Pass authentication (#10519)
++ [turner] Add support for Adobe Pass authentication
+
 
 version 2016.09.19
 

From e6332059ac66bfc91ed18e5b15d9238e4283ee7a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 24 Sep 2016 02:16:47 +0700
Subject: [PATCH 6145/6219] release 2016.09.24

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 youtube_dl/version.py     | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 8b28d784a..7669ab9b7 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.09.19*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.09.19**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.09.24*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.09.24**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.09.19
+[debug] youtube-dl version 2016.09.24
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index e0908aa30..a1c4df479 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2016.09.24
 
 Core
 + Add support for watchTVeverywhere.com authentication provider based MSOs for
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 9d3138181..2af6380b8 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.09.19'
+__version__ = '2016.09.24'

From 5968d7d2fe619e85eb424d6e47d000f0b295d4a2 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 24 Sep 2016 14:20:42 +0800
Subject: [PATCH 6146/6219] [extractor/common] Improved support for HTML5
 subtitles

Ref: #10625

In a strict sense, <track>s with kind=captions are not subtitles. [1]
openload misuses this attribute, and I guess there will be more
examples, so I add it to common.py.

Also allow extracting information for subtitles-only <video> or <audio>
tags, which is the case of openload.

[1] https://www.w3.org/TR/html5/embedded-content-0.html#attr-track-kind
---
 ChangeLog                      | 6 ++++++
 youtube_dl/extractor/common.py | 4 ++--
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index a1c4df479..ebe4ff0e8 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+vesion <unreleased>
+
+Core
++ Improved support for HTML5 subtitles
+
+
 version 2016.09.24
 
 Core
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 9c8991542..5cb4479ec 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1828,7 +1828,7 @@ class InfoExtractor(object):
                 for track_tag in re.findall(r'<track[^>]+>', media_content):
                     track_attributes = extract_attributes(track_tag)
                     kind = track_attributes.get('kind')
-                    if not kind or kind == 'subtitles':
+                    if not kind or kind in ('subtitles', 'captions'):
                         src = track_attributes.get('src')
                         if not src:
                             continue
@@ -1836,7 +1836,7 @@ class InfoExtractor(object):
                         media_info['subtitles'].setdefault(lang, []).append({
                             'url': absolute_url(src),
                         })
-            if media_info['formats']:
+            if media_info['formats'] or media_info['subtitles']:
                 entries.append(media_info)
         return entries
 

From 0711995bcac2f44e09a943521dceb1c54bf8ffb7 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 24 Sep 2016 14:27:08 +0800
Subject: [PATCH 6147/6219] [openload] Support subtitles (closes #10625)

---
 ChangeLog                        |  3 +++
 youtube_dl/extractor/openload.py | 24 +++++++++++++++++++++++-
 2 files changed, 26 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index ebe4ff0e8..766cc477b 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -3,6 +3,9 @@ vesion <unreleased>
 Core
 + Improved support for HTML5 subtitles
 
+Extractors
++ [openload] Support subtitles (#10625)
+
 
 version 2016.09.24
 
diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py
index b6e3ac250..4f5175136 100644
--- a/youtube_dl/extractor/openload.py
+++ b/youtube_dl/extractor/openload.py
@@ -24,6 +24,22 @@ class OpenloadIE(InfoExtractor):
             'title': 'skyrim_no-audio_1080.mp4',
             'thumbnail': 're:^https?://.*\.jpg$',
         },
+    }, {
+        'url': 'https://openload.co/embed/rjC09fkPLYs',
+        'info_dict': {
+            'id': 'rjC09fkPLYs',
+            'ext': 'mp4',
+            'title': 'movie.mp4',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'subtitles': {
+                'en': [{
+                    'ext': 'vtt',
+                }],
+            },
+        },
+        'params': {
+            'skip_download': True,  # test subtitles only
+        },
     }, {
         'url': 'https://openload.co/embed/kUEfGclsU9o/skyrim_no-audio_1080.mp4',
         'only_matching': True,
@@ -71,11 +87,17 @@ class OpenloadIE(InfoExtractor):
             'title', default=None) or self._html_search_meta(
             'description', webpage, 'title', fatal=True)
 
-        return {
+        entries = self._parse_html5_media_entries(url, webpage, video_id)
+        subtitles = entries[0]['subtitles'] if entries else None
+
+        info_dict = {
             'id': video_id,
             'title': title,
             'thumbnail': self._og_search_thumbnail(webpage, default=None),
             'url': video_url,
             # Seems all videos have extensions in their titles
             'ext': determine_ext(title),
+            'subtitles': subtitles,
         }
+
+        return info_dict

From 8add4bfecb73f44cffe3cbf33941fc409564149b Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 24 Sep 2016 10:41:38 +0100
Subject: [PATCH 6148/6219] [mtv] add support for new website urls(closes
 #8169)(closes #9808)

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/mtv.py        | 21 +++++++++++++++++++++
 2 files changed, 22 insertions(+)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 8166fd4f9..bf1f70885 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -516,6 +516,7 @@ from .movingimage import MovingImageIE
 from .msn import MSNIE
 from .mtv import (
     MTVIE,
+    MTVVideoIE,
     MTVServicesEmbeddedIE,
     MTVDEIE,
 )
diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index bdda68819..84a2dcb62 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -270,6 +270,27 @@ class MTVServicesEmbeddedIE(MTVServicesInfoExtractor):
 
 
 class MTVIE(MTVServicesInfoExtractor):
+    _VALID_URL = r'(?x)https?://(?:www\.)?mtv\.com/(video-clips|full-episodes)/(?P<id>[^/?#.]+)'
+    _FEED_URL = 'http://www.mtv.com/feeds/mrss/'
+
+    _TESTS = [{
+        'url': 'http://www.mtv.com/video-clips/vl8qof/unlocking-the-truth-trailer',
+        'md5': '1edbcdf1e7628e414a8c5dcebca3d32b',
+        'info_dict': {
+            'id': '5e14040d-18a4-47c4-a582-43ff602de88e',
+            'ext': 'mp4',
+            'title': 'Unlocking The Truth|July 18, 2016|1|101|Trailer',
+            'description': '"Unlocking the Truth" premieres August 17th at 11/10c.',
+            'timestamp': 1468846800,
+            'upload_date': '20160718',
+        },
+    }, {
+        'url': 'http://www.mtv.com/full-episodes/94tujl/unlocking-the-truth-gates-of-hell-season-1-ep-101',
+        'only_matching': True,
+    }]
+
+
+class MTVVideoIE(MTVServicesInfoExtractor):
     _VALID_URL = r'''(?x)^https?://
         (?:(?:www\.)?mtv\.com/videos/.+?/(?P<videoid>[0-9]+)/[^/]+$|
            m\.mtv\.com/videos/video\.rbml\?.*?id=(?P<mgid>[^&]+))'''

From a54ffb8aa778062901dd15b020576bc7d472ae40 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 24 Sep 2016 10:50:14 +0100
Subject: [PATCH 6149/6219] [mtv] add common IE_NAME prefix for MTVIE and
 MTVVideoIE

---
 youtube_dl/extractor/mtv.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index 84a2dcb62..2e9580b10 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -270,7 +270,8 @@ class MTVServicesEmbeddedIE(MTVServicesInfoExtractor):
 
 
 class MTVIE(MTVServicesInfoExtractor):
-    _VALID_URL = r'(?x)https?://(?:www\.)?mtv\.com/(video-clips|full-episodes)/(?P<id>[^/?#.]+)'
+    IE_NAME = 'mtv'
+    _VALID_URL = r'https?://(?:www\.)?mtv\.com/(video-clips|full-episodes)/(?P<id>[^/?#.]+)'
     _FEED_URL = 'http://www.mtv.com/feeds/mrss/'
 
     _TESTS = [{
@@ -291,6 +292,7 @@ class MTVIE(MTVServicesInfoExtractor):
 
 
 class MTVVideoIE(MTVServicesInfoExtractor):
+    IE_NAME = 'mtv:video'
     _VALID_URL = r'''(?x)^https?://
         (?:(?:www\.)?mtv\.com/videos/.+?/(?P<videoid>[0-9]+)/[^/]+$|
            m\.mtv\.com/videos/video\.rbml\?.*?id=(?P<mgid>[^&]+))'''

From f0bc5a8609786633d8b51ab4255c1f0fdb941f73 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 24 Sep 2016 19:57:55 +0800
Subject: [PATCH 6150/6219] [twitter] Support Periscope embeds (closes #10737)

Also update _TESTS
---
 ChangeLog                         |  1 +
 youtube_dl/extractor/periscope.py |  9 +++++++
 youtube_dl/extractor/twitter.py   | 45 +++++++++++++++++++++++++------
 3 files changed, 47 insertions(+), 8 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 766cc477b..5c96dc179 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -4,6 +4,7 @@ Core
 + Improved support for HTML5 subtitles
 
 Extractors
++ [twitter] Support Periscope embeds (#10737)
 + [openload] Support subtitles (#10625)
 
 
diff --git a/youtube_dl/extractor/periscope.py b/youtube_dl/extractor/periscope.py
index eb1aeba46..e8b2f11c6 100644
--- a/youtube_dl/extractor/periscope.py
+++ b/youtube_dl/extractor/periscope.py
@@ -1,6 +1,8 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..utils import (
     parse_iso8601,
@@ -41,6 +43,13 @@ class PeriscopeIE(PeriscopeBaseIE):
         'only_matching': True,
     }]
 
+    @staticmethod
+    def _extract_url(webpage):
+        mobj = re.search(
+            r'<iframe[^>]+src=([\'"])(?P<url>(?:https?:)?//(?:www\.)?periscope\.tv/(?:(?!\1).)+)\1', webpage)
+        if mobj:
+            return mobj.group('url')
+
     def _real_extract(self, url):
         token = self._match_id(url)
 
diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index c5a5843b6..3411fcf7e 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import compat_urlparse
 from ..utils import (
     determine_ext,
     float_or_none,
@@ -13,6 +14,8 @@ from ..utils import (
     ExtractorError,
 )
 
+from .periscope import PeriscopeIE
+
 
 class TwitterBaseIE(InfoExtractor):
     def _get_vmap_video_url(self, vmap_url, video_id):
@@ -48,12 +51,12 @@ class TwitterCardIE(TwitterBaseIE):
         },
         {
             'url': 'https://twitter.com/i/cards/tfw/v1/654001591733886977',
-            'md5': 'd4724ffe6d2437886d004fa5de1043b3',
+            'md5': 'b6d9683dd3f48e340ded81c0e917ad46',
             'info_dict': {
                 'id': 'dq4Oj5quskI',
                 'ext': 'mp4',
                 'title': 'Ubuntu 11.10 Overview',
-                'description': 'Take a quick peek at what\'s new and improved in Ubuntu 11.10.\n\nOnce installed take a look at 10 Things to Do After Installing: http://www.omgubuntu.co.uk/2011/10/10...',
+                'description': 'md5:a831e97fa384863d6e26ce48d1c43376',
                 'upload_date': '20111013',
                 'uploader': 'OMG! Ubuntu!',
                 'uploader_id': 'omgubuntu',
@@ -100,12 +103,17 @@ class TwitterCardIE(TwitterBaseIE):
             return self.url_result(iframe_url)
 
         config = self._parse_json(self._html_search_regex(
-            r'data-(?:player-)?config="([^"]+)"', webpage, 'data player config'),
+            r'data-(?:player-)?config="([^"]+)"', webpage,
+            'data player config', default='{}'),
             video_id)
 
         if config.get('source_type') == 'vine':
             return self.url_result(config['player_url'], 'Vine')
 
+        periscope_url = PeriscopeIE._extract_url(webpage)
+        if periscope_url:
+            return self.url_result(periscope_url, PeriscopeIE.ie_key())
+
         def _search_dimensions_in_video_url(a_format, video_url):
             m = re.search(r'/(?P<width>\d+)x(?P<height>\d+)/', video_url)
             if m:
@@ -244,10 +252,10 @@ class TwitterIE(InfoExtractor):
         'info_dict': {
             'id': '700207533655363584',
             'ext': 'mp4',
-            'title': 'Donte The Dumbass - BEAT PROD: @suhmeduh #Damndaniel',
-            'description': 'Donte The Dumbass on Twitter: "BEAT PROD: @suhmeduh  https://t.co/HBrQ4AfpvZ #Damndaniel https://t.co/byBooq2ejZ"',
+            'title': 'JG - BEAT PROD: @suhmeduh #Damndaniel',
+            'description': 'JG on Twitter: "BEAT PROD: @suhmeduh  https://t.co/HBrQ4AfpvZ #Damndaniel https://t.co/byBooq2ejZ"',
             'thumbnail': 're:^https?://.*\.jpg',
-            'uploader': 'Donte The Dumbass',
+            'uploader': 'JG',
             'uploader_id': 'jaydingeer',
         },
         'params': {
@@ -278,6 +286,18 @@ class TwitterIE(InfoExtractor):
         'params': {
             'skip_download': True,  # requires ffmpeg
         },
+    }, {
+        'url': 'https://twitter.com/OPP_HSD/status/779210622571536384',
+        'info_dict': {
+            'id': '1zqKVVlkqLaKB',
+            'ext': 'mp4',
+            'title': 'Sgt Kerry Schmidt - Ontario Provincial Police - Road rage, mischief, assault, rollover and fire in one occurrence',
+            'upload_date': '20160923',
+            'uploader_id': 'OPP_HSD',
+            'uploader': 'Sgt Kerry Schmidt - Ontario Provincial Police',
+            'timestamp': 1474613214,
+        },
+        'add_ie': ['Periscope'],
     }]
 
     def _real_extract(self, url):
@@ -328,13 +348,22 @@ class TwitterIE(InfoExtractor):
             })
             return info
 
+        twitter_card_url = None
         if 'class="PlayableMedia' in webpage:
+            twitter_card_url = '%s//twitter.com/i/videos/tweet/%s' % (self.http_scheme(), twid)
+        else:
+            twitter_card_iframe_url = self._search_regex(
+                r'data-full-card-iframe-url=([\'"])(?P<url>(?:(?!\1).)+)\1',
+                webpage, 'Twitter card iframe URL', default=None, group='url')
+            if twitter_card_iframe_url:
+                twitter_card_url = compat_urlparse.urljoin(url, twitter_card_iframe_url)
+
+        if twitter_card_url:
             info.update({
                 '_type': 'url_transparent',
                 'ie_key': 'TwitterCard',
-                'url': '%s//twitter.com/i/videos/tweet/%s' % (self.http_scheme(), twid),
+                'url': twitter_card_url,
             })
-
             return info
 
         raise ExtractorError('There\'s no video in this tweet.')

From 8e45e1cc4d706e6b43dac8105acf3592fa3d4725 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 24 Sep 2016 19:18:01 +0700
Subject: [PATCH 6151/6219] [soundcloud] Generalize playlist entries extraction
 (#10733)

---
 youtube_dl/extractor/soundcloud.py | 42 ++++++++++++++++++------------
 1 file changed, 26 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 496cc5d8e..f3cb35f77 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -260,7 +260,20 @@ class SoundcloudIE(InfoExtractor):
         return self._extract_info_dict(info, full_title, secret_token=token)
 
 
-class SoundcloudSetIE(SoundcloudIE):
+class SoundcloudBaseIE(SoundcloudIE):
+    @staticmethod
+    def _extract_id(e):
+        return compat_str(e['id']) if e.get('id') else None
+
+    def _extract_track_entries(self, tracks):
+        return [
+            self.url_result(
+                track['permalink_url'], SoundcloudIE.ie_key(),
+                video_id=self._extract_id(track))
+            for track in tracks if track.get('permalink_url')]
+
+
+class SoundcloudSetIE(SoundcloudBaseIE):
     _VALID_URL = r'https?://(?:(?:www|m)\.)?soundcloud\.com/(?P<uploader>[\w\d-]+)/sets/(?P<slug_title>[\w\d-]+)(?:/(?P<token>[^?/]+))?'
     IE_NAME = 'soundcloud:set'
     _TESTS = [{
@@ -299,7 +312,7 @@ class SoundcloudSetIE(SoundcloudIE):
             msgs = (compat_str(err['error_message']) for err in info['errors'])
             raise ExtractorError('unable to download video webpage: %s' % ','.join(msgs))
 
-        entries = [self.url_result(track['permalink_url'], 'Soundcloud') for track in info['tracks']]
+        entries = self._extract_track_entries(info['tracks'])
 
         return {
             '_type': 'playlist',
@@ -309,7 +322,7 @@ class SoundcloudSetIE(SoundcloudIE):
         }
 
 
-class SoundcloudUserIE(SoundcloudIE):
+class SoundcloudUserIE(SoundcloudBaseIE):
     _VALID_URL = r'''(?x)
                         https?://
                             (?:(?:www|m)\.)?soundcloud\.com/
@@ -326,21 +339,21 @@ class SoundcloudUserIE(SoundcloudIE):
             'id': '114582580',
             'title': 'The Akashic Chronicler (All)',
         },
-        'playlist_mincount': 111,
+        'playlist_mincount': 74,
     }, {
         'url': 'https://soundcloud.com/the-akashic-chronicler/tracks',
         'info_dict': {
             'id': '114582580',
             'title': 'The Akashic Chronicler (Tracks)',
         },
-        'playlist_mincount': 50,
+        'playlist_mincount': 37,
     }, {
         'url': 'https://soundcloud.com/the-akashic-chronicler/sets',
         'info_dict': {
             'id': '114582580',
             'title': 'The Akashic Chronicler (Playlists)',
         },
-        'playlist_mincount': 3,
+        'playlist_mincount': 2,
     }, {
         'url': 'https://soundcloud.com/the-akashic-chronicler/reposts',
         'info_dict': {
@@ -359,7 +372,7 @@ class SoundcloudUserIE(SoundcloudIE):
         'url': 'https://soundcloud.com/grynpyret/spotlight',
         'info_dict': {
             'id': '7098329',
-            'title': 'Grynpyret (Spotlight)',
+            'title': 'GRYNPYRET (Spotlight)',
         },
         'playlist_mincount': 1,
     }]
@@ -421,13 +434,14 @@ class SoundcloudUserIE(SoundcloudIE):
                 for cand in candidates:
                     if isinstance(cand, dict):
                         permalink_url = cand.get('permalink_url')
+                        entry_id = self._extract_id(cand)
                         if permalink_url and permalink_url.startswith('http'):
-                            return permalink_url
+                            return permalink_url, entry_id
 
             for e in collection:
-                permalink_url = resolve_permalink_url((e, e.get('track'), e.get('playlist')))
+                permalink_url, entry_id = resolve_permalink_url((e, e.get('track'), e.get('playlist')))
                 if permalink_url:
-                    entries.append(self.url_result(permalink_url))
+                    entries.append(self.url_result(permalink_url, video_id=entry_id))
 
             next_href = response.get('next_href')
             if not next_href:
@@ -447,7 +461,7 @@ class SoundcloudUserIE(SoundcloudIE):
         }
 
 
-class SoundcloudPlaylistIE(SoundcloudIE):
+class SoundcloudPlaylistIE(SoundcloudBaseIE):
     _VALID_URL = r'https?://api\.soundcloud\.com/playlists/(?P<id>[0-9]+)(?:/?\?secret_token=(?P<token>[^&]+?))?$'
     IE_NAME = 'soundcloud:playlist'
     _TESTS = [{
@@ -477,11 +491,7 @@ class SoundcloudPlaylistIE(SoundcloudIE):
         data = self._download_json(
             base_url + data, playlist_id, 'Downloading playlist')
 
-        entries = [
-            self.url_result(
-                track['permalink_url'], SoundcloudIE.ie_key(),
-                video_id=compat_str(track['id']) if track.get('id') else None)
-            for track in data['tracks'] if track.get('permalink_url')]
+        entries = self._extract_track_entries(data['tracks'])
 
         return {
             '_type': 'playlist',

From 7518a61d416133bff8b99c693dfca0b15c0d5b7e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 24 Sep 2016 19:29:49 +0700
Subject: [PATCH 6152/6219] [soundcloud] Fix typo in playlist base class name

---
 youtube_dl/extractor/soundcloud.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index f3cb35f77..1a8114aa7 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -260,7 +260,7 @@ class SoundcloudIE(InfoExtractor):
         return self._extract_info_dict(info, full_title, secret_token=token)
 
 
-class SoundcloudBaseIE(SoundcloudIE):
+class SoundcloudPlaylistBaseIE(SoundcloudIE):
     @staticmethod
     def _extract_id(e):
         return compat_str(e['id']) if e.get('id') else None
@@ -273,7 +273,7 @@ class SoundcloudBaseIE(SoundcloudIE):
             for track in tracks if track.get('permalink_url')]
 
 
-class SoundcloudSetIE(SoundcloudBaseIE):
+class SoundcloudSetIE(SoundcloudPlaylistBaseIE):
     _VALID_URL = r'https?://(?:(?:www|m)\.)?soundcloud\.com/(?P<uploader>[\w\d-]+)/sets/(?P<slug_title>[\w\d-]+)(?:/(?P<token>[^?/]+))?'
     IE_NAME = 'soundcloud:set'
     _TESTS = [{
@@ -322,7 +322,7 @@ class SoundcloudSetIE(SoundcloudBaseIE):
         }
 
 
-class SoundcloudUserIE(SoundcloudBaseIE):
+class SoundcloudUserIE(SoundcloudPlaylistBaseIE):
     _VALID_URL = r'''(?x)
                         https?://
                             (?:(?:www|m)\.)?soundcloud\.com/
@@ -461,7 +461,7 @@ class SoundcloudUserIE(SoundcloudBaseIE):
         }
 
 
-class SoundcloudPlaylistIE(SoundcloudBaseIE):
+class SoundcloudPlaylistIE(SoundcloudPlaylistBaseIE):
     _VALID_URL = r'https?://api\.soundcloud\.com/playlists/(?P<id>[0-9]+)(?:/?\?secret_token=(?P<token>[^&]+?))?$'
     IE_NAME = 'soundcloud:playlist'
     _TESTS = [{

From 6f126d903f46d976a380a5b4265084e5a21a3c09 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 24 Sep 2016 15:38:19 +0100
Subject: [PATCH 6153/6219] [download/hls] Delegate downloading to ffmpeg for
 live streams

---
 youtube_dl/downloader/hls.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index 5d70abf62..541b92ee1 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -31,7 +31,7 @@ class HlsFD(FragmentFD):
     FD_NAME = 'hlsnative'
 
     @staticmethod
-    def can_download(manifest):
+    def can_download(manifest, info_dict):
         UNSUPPORTED_FEATURES = (
             r'#EXT-X-KEY:METHOD=(?!NONE|AES-128)',  # encrypted streams [1]
             r'#EXT-X-BYTERANGE',  # playlists composed of byte ranges of media files [2]
@@ -53,6 +53,7 @@ class HlsFD(FragmentFD):
         )
         check_results = [not re.search(feature, manifest) for feature in UNSUPPORTED_FEATURES]
         check_results.append(can_decrypt_frag or '#EXT-X-KEY:METHOD=AES-128' not in manifest)
+        check_results.append(not info_dict.get('is_live'))
         return all(check_results)
 
     def real_download(self, filename, info_dict):
@@ -62,7 +63,7 @@ class HlsFD(FragmentFD):
 
         s = manifest.decode('utf-8', 'ignore')
 
-        if not self.can_download(s):
+        if not self.can_download(s, info_dict):
             self.report_warning(
                 'hlsnative has detected features it does not support, '
                 'extraction will be delegated to ffmpeg')

From 27e99078d337cdc77a5a7228998d3b2fe722e7cb Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 24 Sep 2016 15:39:06 +0100
Subject: [PATCH 6154/6219] [brightcove:new] add support for live streams

---
 youtube_dl/extractor/brightcove.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index aeb22be16..2ec55b185 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -621,15 +621,21 @@ class BrightcoveNewIE(InfoExtractor):
                     'url': text_track['src'],
                 })
 
+        is_live = False
+        duration = float_or_none(json_data.get('duration'), 1000)
+        if duration and duration < 0:
+            is_live = True
+
         return {
             'id': video_id,
-            'title': title,
+            'title': self._live_title(title) if is_live else title,
             'description': clean_html(json_data.get('description')),
             'thumbnail': json_data.get('thumbnail') or json_data.get('poster'),
-            'duration': float_or_none(json_data.get('duration'), 1000),
+            'duration': duration,
             'timestamp': parse_iso8601(json_data.get('published_at')),
             'uploader_id': account_id,
             'formats': formats,
             'subtitles': subtitles,
             'tags': json_data.get('tags', []),
+            'is_live': is_live,
         }

From e71a450956c808d469b983e5ffde1a63aff24390 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 24 Sep 2016 21:55:53 +0100
Subject: [PATCH 6155/6219] [common] add hdcore sign to akamai f4m formats

---
 youtube_dl/extractor/common.py | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 5cb4479ec..1076b46da 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1842,10 +1842,15 @@ class InfoExtractor(object):
 
     def _extract_akamai_formats(self, manifest_url, video_id):
         formats = []
+        hdcore_sign = 'hdcore=3.7.0'
         f4m_url = re.sub(r'(https?://.+?)/i/', r'\1/z/', manifest_url).replace('/master.m3u8', '/manifest.f4m')
-        formats.extend(self._extract_f4m_formats(
-            update_url_query(f4m_url, {'hdcore': '3.7.0'}),
-            video_id, f4m_id='hds', fatal=False))
+        if 'hdcore=' not in f4m_url:
+            f4m_url += ('&' if '?' in f4m_url else '?') + hdcore_sign
+        f4m_formats = self._extract_f4m_formats(
+            f4m_url, video_id, f4m_id='hds', fatal=False)
+        for entry in f4m_formats:
+            entry.update({'extra_param_to_segment_url': hdcore_sign})
+        formats.extend(f4m_formats)
         m3u8_url = re.sub(r'(https?://.+?)/z/', r'\1/i/', manifest_url).replace('/manifest.f4m', '/master.m3u8')
         formats.extend(self._extract_m3u8_formats(
             m3u8_url, video_id, 'mp4', 'm3u8_native',

From 7fd57de6fb146ffca594e4ae632d7ff217926b52 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 24 Sep 2016 21:59:48 +0100
Subject: [PATCH 6156/6219] [cbsnews:livevideo] fix extraction and extract m3u8
 formats

---
 youtube_dl/extractor/cbsnews.py | 31 +++++++++++++++----------------
 1 file changed, 15 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/extractor/cbsnews.py b/youtube_dl/extractor/cbsnews.py
index 4aa6917a0..216989230 100644
--- a/youtube_dl/extractor/cbsnews.py
+++ b/youtube_dl/extractor/cbsnews.py
@@ -9,6 +9,7 @@ from ..utils import (
 
 
 class CBSNewsIE(CBSIE):
+    IE_NAME = 'cbsnews'
     IE_DESC = 'CBS News'
     _VALID_URL = r'https?://(?:www\.)?cbsnews\.com/(?:news|videos)/(?P<id>[\da-z_-]+)'
 
@@ -68,15 +69,16 @@ class CBSNewsIE(CBSIE):
 
 
 class CBSNewsLiveVideoIE(InfoExtractor):
+    IE_NAME = 'cbsnews:livevideo'
     IE_DESC = 'CBS News Live Videos'
-    _VALID_URL = r'https?://(?:www\.)?cbsnews\.com/live/video/(?P<id>[\da-z_-]+)'
+    _VALID_URL = r'https?://(?:www\.)?cbsnews\.com/live/video/(?P<id>[^/?#]+)'
 
     # Live videos get deleted soon. See http://www.cbsnews.com/live/ for the latest examples
     _TEST = {
         'url': 'http://www.cbsnews.com/live/video/clinton-sanders-prepare-to-face-off-in-nh/',
         'info_dict': {
             'id': 'clinton-sanders-prepare-to-face-off-in-nh',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Clinton, Sanders Prepare To Face Off In NH',
             'duration': 334,
         },
@@ -84,25 +86,22 @@ class CBSNewsLiveVideoIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        display_id = self._match_id(url)
 
-        webpage = self._download_webpage(url, video_id)
+        video_info = self._download_json(
+            'http://feeds.cbsn.cbsnews.com/rundown/story', display_id, query={
+                'device': 'desktop',
+                'dvr_slug': display_id,
+            })
 
-        video_info = self._parse_json(self._html_search_regex(
-            r'data-story-obj=\'({.+?})\'', webpage, 'video JSON info'), video_id)['story']
-
-        hdcore_sign = 'hdcore=3.3.1'
-        f4m_formats = self._extract_f4m_formats(video_info['url'] + '&' + hdcore_sign, video_id)
-        if f4m_formats:
-            for entry in f4m_formats:
-                # URLs without the extra param induce an 404 error
-                entry.update({'extra_param_to_segment_url': hdcore_sign})
-        self._sort_formats(f4m_formats)
+        formats = self._extract_akamai_formats(video_info['url'], display_id)
+        self._sort_formats(formats)
 
         return {
-            'id': video_id,
+            'id': display_id,
+            'display_id': display_id,
             'title': video_info['headline'],
             'thumbnail': video_info.get('thumbnail_url_hd') or video_info.get('thumbnail_url_sd'),
             'duration': parse_duration(video_info.get('segmentDur')),
-            'formats': f4m_formats,
+            'formats': formats,
         }

From 63c583eb2c9a906ba1075da289afdde29b385fff Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 25 Sep 2016 04:43:10 +0700
Subject: [PATCH 6157/6219] [prosiebensat1] Add support for sat1gold (#10745)

---
 youtube_dl/extractor/prosiebensat1.py | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/prosiebensat1.py b/youtube_dl/extractor/prosiebensat1.py
index 5a29b844d..2f5aa530a 100644
--- a/youtube_dl/extractor/prosiebensat1.py
+++ b/youtube_dl/extractor/prosiebensat1.py
@@ -122,7 +122,17 @@ class ProSiebenSat1BaseIE(InfoExtractor):
 class ProSiebenSat1IE(ProSiebenSat1BaseIE):
     IE_NAME = 'prosiebensat1'
     IE_DESC = 'ProSiebenSat.1 Digital'
-    _VALID_URL = r'https?://(?:www\.)?(?:(?:prosieben|prosiebenmaxx|sixx|sat1|kabeleins|the-voice-of-germany|7tv|kabeleinsdoku)\.(?:de|at|ch)|ran\.de|fem\.com)/(?P<id>.+)'
+    _VALID_URL = r'''(?x)
+                    https?://
+                        (?:www\.)?
+                        (?:
+                            (?:
+                                prosieben|prosiebenmaxx|sixx|sat1(?:gold)?|kabeleins|the-voice-of-germany|7tv|kabeleinsdoku
+                            )\.(?:de|at|ch)|
+                            ran\.de|fem\.com
+                        )
+                        /(?P<id>.+)
+                    '''
 
     _TESTS = [
         {
@@ -295,6 +305,11 @@ class ProSiebenSat1IE(ProSiebenSat1BaseIE):
             'url': 'http://www.kabeleinsdoku.de/tv/mayday-alarm-im-cockpit/video/102-notlandung-im-hudson-river-ganze-folge',
             'only_matching': True,
         },
+        {
+            # geo restricted to Germany
+            'url': 'http://www.sat1gold.de/tv/edel-starck/video/11-staffel-1-episode-1-partner-wider-willen-ganze-folge',
+            'only_matching': True,
+        },
     ]
 
     _TOKEN = 'prosieben'

From ddde91952f4eec796b14eb258c0cb33dda3935bc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 25 Sep 2016 05:36:18 +0700
Subject: [PATCH 6158/6219] [prosiebensat1] Fix playlist support (Closes
 #10745)

---
 youtube_dl/extractor/prosiebensat1.py | 39 ++++++++++++++++++---------
 1 file changed, 26 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/prosiebensat1.py b/youtube_dl/extractor/prosiebensat1.py
index 2f5aa530a..a064de05e 100644
--- a/youtube_dl/extractor/prosiebensat1.py
+++ b/youtube_dl/extractor/prosiebensat1.py
@@ -310,6 +310,10 @@ class ProSiebenSat1IE(ProSiebenSat1BaseIE):
             'url': 'http://www.sat1gold.de/tv/edel-starck/video/11-staffel-1-episode-1-partner-wider-willen-ganze-folge',
             'only_matching': True,
         },
+        {
+            'url': 'http://www.sat1gold.de/tv/edel-starck/playlist/die-gesamte-1-staffel',
+            'only_matching': True,
+        },
     ]
 
     _TOKEN = 'prosieben'
@@ -381,19 +385,28 @@ class ProSiebenSat1IE(ProSiebenSat1BaseIE):
     def _extract_playlist(self, url, webpage):
         playlist_id = self._html_search_regex(
             self._PLAYLIST_ID_REGEXES, webpage, 'playlist id')
-        for regex in self._PLAYLIST_CLIP_REGEXES:
-            playlist_clips = re.findall(regex, webpage)
-            if playlist_clips:
-                title = self._html_search_regex(
-                    self._TITLE_REGEXES, webpage, 'title')
-                description = self._html_search_regex(
-                    self._DESCRIPTION_REGEXES, webpage, 'description', fatal=False)
-                entries = [
-                    self.url_result(
-                        re.match('(.+?//.+?)/', url).group(1) + clip_path,
-                        'ProSiebenSat1')
-                    for clip_path in playlist_clips]
-                return self.playlist_result(entries, playlist_id, title, description)
+        playlist = self._parse_json(
+            self._search_regex(
+                'var\s+contentResources\s*=\s*(\[.+?\]);\s*</script',
+                webpage, 'playlist'),
+            playlist_id)
+        entries = []
+        for item in playlist:
+            clip_id = item.get('id') or item.get('upc')
+            if not clip_id:
+                continue
+            info = self._extract_video_info(url, clip_id)
+            info.update({
+                'id': clip_id,
+                'title': item.get('title') or item.get('teaser', {}).get('headline'),
+                'description': item.get('teaser', {}).get('description'),
+                'thumbnail': item.get('poster'),
+                'duration': float_or_none(item.get('duration')),
+                'series': item.get('tvShowTitle'),
+                'uploader': item.get('broadcastPublisher'),
+            })
+            entries.append(info)
+        return self.playlist_result(entries, playlist_id)
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From f92bb612c69957c3803aaf14aea1d03a7d7d917f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 25 Sep 2016 06:14:32 +0700
Subject: [PATCH 6159/6219] [mwave] Relax _VALID_URLs (Closes #10735, closes
 #10748)

---
 youtube_dl/extractor/mwave.py | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/mwave.py b/youtube_dl/extractor/mwave.py
index a103e0323..fea1caf47 100644
--- a/youtube_dl/extractor/mwave.py
+++ b/youtube_dl/extractor/mwave.py
@@ -9,9 +9,9 @@ from ..utils import (
 
 
 class MwaveIE(InfoExtractor):
-    _VALID_URL = r'https?://mwave\.interest\.me/mnettv/videodetail\.m\?searchVideoDetailVO\.clip_id=(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://mwave\.interest\.me/(?:[^/]+/)?mnettv/videodetail\.m\?searchVideoDetailVO\.clip_id=(?P<id>[0-9]+)'
     _URL_TEMPLATE = 'http://mwave.interest.me/mnettv/videodetail.m?searchVideoDetailVO.clip_id=%s'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://mwave.interest.me/mnettv/videodetail.m?searchVideoDetailVO.clip_id=168859',
         # md5 is unstable
         'info_dict': {
@@ -23,7 +23,10 @@ class MwaveIE(InfoExtractor):
             'duration': 206,
             'view_count': int,
         }
-    }
+    }, {
+        'url': 'http://mwave.interest.me/en/mnettv/videodetail.m?searchVideoDetailVO.clip_id=176199',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -60,8 +63,8 @@ class MwaveIE(InfoExtractor):
 
 
 class MwaveMeetGreetIE(InfoExtractor):
-    _VALID_URL = r'https?://mwave\.interest\.me/meetgreet/view/(?P<id>\d+)'
-    _TEST = {
+    _VALID_URL = r'https?://mwave\.interest\.me/(?:[^/]+/)?meetgreet/view/(?P<id>\d+)'
+    _TESTS = [{
         'url': 'http://mwave.interest.me/meetgreet/view/256',
         'info_dict': {
             'id': '173294',
@@ -72,7 +75,10 @@ class MwaveMeetGreetIE(InfoExtractor):
             'duration': 3634,
             'view_count': int,
         }
-    }
+    }, {
+        'url': 'http://mwave.interest.me/en/meetgreet/view/256',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From 0a078550b9ac570cb357c2af74a39068d08ce1ee Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 25 Sep 2016 06:19:17 +0700
Subject: [PATCH 6160/6219] [prosiebensat1] Improve _VALID_URL

---
 youtube_dl/extractor/prosiebensat1.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/prosiebensat1.py b/youtube_dl/extractor/prosiebensat1.py
index a064de05e..84d04aa69 100644
--- a/youtube_dl/extractor/prosiebensat1.py
+++ b/youtube_dl/extractor/prosiebensat1.py
@@ -127,7 +127,7 @@ class ProSiebenSat1IE(ProSiebenSat1BaseIE):
                         (?:www\.)?
                         (?:
                             (?:
-                                prosieben|prosiebenmaxx|sixx|sat1(?:gold)?|kabeleins|the-voice-of-germany|7tv|kabeleinsdoku
+                                prosieben(?:maxx)?|sixx|sat1(?:gold)?|kabeleins(?:doku)?|the-voice-of-germany|7tv
                             )\.(?:de|at|ch)|
                             ran\.de|fem\.com
                         )

From 493353c7fd5d15fa35152915c10c7249277b5ed0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 25 Sep 2016 06:25:57 +0700
Subject: [PATCH 6161/6219] [prosiebensat1] Add support for advopedia

---
 youtube_dl/extractor/prosiebensat1.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/prosiebensat1.py b/youtube_dl/extractor/prosiebensat1.py
index 84d04aa69..873d4f981 100644
--- a/youtube_dl/extractor/prosiebensat1.py
+++ b/youtube_dl/extractor/prosiebensat1.py
@@ -127,9 +127,9 @@ class ProSiebenSat1IE(ProSiebenSat1BaseIE):
                         (?:www\.)?
                         (?:
                             (?:
-                                prosieben(?:maxx)?|sixx|sat1(?:gold)?|kabeleins(?:doku)?|the-voice-of-germany|7tv
+                                prosieben(?:maxx)?|sixx|sat1(?:gold)?|kabeleins(?:doku)?|the-voice-of-germany|7tv|advopedia
                             )\.(?:de|at|ch)|
-                            ran\.de|fem\.com
+                            ran\.de|fem\.com|advopedia\.de
                         )
                         /(?P<id>.+)
                     '''
@@ -314,6 +314,10 @@ class ProSiebenSat1IE(ProSiebenSat1BaseIE):
             'url': 'http://www.sat1gold.de/tv/edel-starck/playlist/die-gesamte-1-staffel',
             'only_matching': True,
         },
+        {
+            'url': 'http://www.advopedia.de/videos/lenssen-klaert-auf/lenssen-klaert-auf-folge-8-staffel-3-feiertage-und-freie-tage',
+            'only_matching': True,
+        },
     ]
 
     _TOKEN = 'prosieben'

From 2d3d29976b2c83e8daab62a0f2a61c232692a310 Mon Sep 17 00:00:00 2001
From: stepshal <nessento@openmailbox.org>
Date: Sat, 17 Sep 2016 21:48:20 +0700
Subject: [PATCH 6162/6219] [youtube] Change test URLs from http to https

---
 youtube_dl/extractor/youtube.py | 34 ++++++++++++++++-----------------
 1 file changed, 17 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 5ca903825..f86823112 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -369,7 +369,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
     IE_NAME = 'youtube'
     _TESTS = [
         {
-            'url': 'http://www.youtube.com/watch?v=BaW_jenozKc&t=1s&end=9',
+            'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&t=1s&end=9',
             'info_dict': {
                 'id': 'BaW_jenozKc',
                 'ext': 'mp4',
@@ -389,7 +389,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             }
         },
         {
-            'url': 'http://www.youtube.com/watch?v=UxxajLWwzqY',
+            'url': 'https://www.youtube.com/watch?v=UxxajLWwzqY',
             'note': 'Test generic use_cipher_signature video (#897)',
             'info_dict': {
                 'id': 'UxxajLWwzqY',
@@ -443,7 +443,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             }
         },
         {
-            'url': 'http://www.youtube.com/watch?v=BaW_jenozKc&v=UxxajLWwzqY',
+            'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&v=UxxajLWwzqY',
             'note': 'Use the first video ID in the URL',
             'info_dict': {
                 'id': 'BaW_jenozKc',
@@ -465,7 +465,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             },
         },
         {
-            'url': 'http://www.youtube.com/watch?v=a9LDPn-MO4I',
+            'url': 'https://www.youtube.com/watch?v=a9LDPn-MO4I',
             'note': '256k DASH audio (format 141) via DASH manifest',
             'info_dict': {
                 'id': 'a9LDPn-MO4I',
@@ -539,7 +539,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         },
         # Normal age-gate video (No vevo, embed allowed)
         {
-            'url': 'http://youtube.com/watch?v=HtVdAasjOgU',
+            'url': 'https://youtube.com/watch?v=HtVdAasjOgU',
             'info_dict': {
                 'id': 'HtVdAasjOgU',
                 'ext': 'mp4',
@@ -555,7 +555,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         },
         # Age-gate video with encrypted signature
         {
-            'url': 'http://www.youtube.com/watch?v=6kLq3WMV1nU',
+            'url': 'https://www.youtube.com/watch?v=6kLq3WMV1nU',
             'info_dict': {
                 'id': '6kLq3WMV1nU',
                 'ext': 'mp4',
@@ -748,11 +748,11 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             'skip': 'Not multifeed anymore',
         },
         {
-            'url': 'http://vid.plus/FlRa-iH7PGw',
+            'url': 'https://vid.plus/FlRa-iH7PGw',
             'only_matching': True,
         },
         {
-            'url': 'http://zwearz.com/watch/9lWxNJF-ufM/electra-woman-dyna-girl-official-trailer-grace-helbig.html',
+            'url': 'https://zwearz.com/watch/9lWxNJF-ufM/electra-woman-dyna-girl-official-trailer-grace-helbig.html',
             'only_matching': True,
         },
         {
@@ -1846,7 +1846,7 @@ class YoutubePlaylistIE(YoutubePlaylistBaseInfoExtractor):
         'playlist_count': 2,
     }, {
         'note': 'embedded',
-        'url': 'http://www.youtube.com/embed/videoseries?list=PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
+        'url': 'https://www.youtube.com/embed/videoseries?list=PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
         'playlist_count': 4,
         'info_dict': {
             'title': 'JODA15',
@@ -1854,7 +1854,7 @@ class YoutubePlaylistIE(YoutubePlaylistBaseInfoExtractor):
         }
     }, {
         'note': 'Embedded SWF player',
-        'url': 'http://www.youtube.com/p/YN5VISEtHet5D4NEvfTd0zcgFk84NqFZ?hl=en_US&fs=1&rel=0',
+        'url': 'https://www.youtube.com/p/YN5VISEtHet5D4NEvfTd0zcgFk84NqFZ?hl=en_US&fs=1&rel=0',
         'playlist_count': 4,
         'info_dict': {
             'title': 'JODA7',
@@ -2156,7 +2156,7 @@ class YoutubeLiveIE(YoutubeBaseInfoExtractor):
     IE_NAME = 'youtube:live'
 
     _TESTS = [{
-        'url': 'http://www.youtube.com/user/TheYoungTurks/live',
+        'url': 'https://www.youtube.com/user/TheYoungTurks/live',
         'info_dict': {
             'id': 'a48o2S1cPoo',
             'ext': 'mp4',
@@ -2176,7 +2176,7 @@ class YoutubeLiveIE(YoutubeBaseInfoExtractor):
             'skip_download': True,
         },
     }, {
-        'url': 'http://www.youtube.com/channel/UC1yBKRuGpC1tSM73A0ZjYjQ/live',
+        'url': 'https://www.youtube.com/channel/UC1yBKRuGpC1tSM73A0ZjYjQ/live',
         'only_matching': True,
     }]
 
@@ -2201,7 +2201,7 @@ class YoutubePlaylistsIE(YoutubePlaylistsBaseInfoExtractor):
     IE_NAME = 'youtube:playlists'
 
     _TESTS = [{
-        'url': 'http://www.youtube.com/user/ThirstForScience/playlists',
+        'url': 'https://www.youtube.com/user/ThirstForScience/playlists',
         'playlist_mincount': 4,
         'info_dict': {
             'id': 'ThirstForScience',
@@ -2209,7 +2209,7 @@ class YoutubePlaylistsIE(YoutubePlaylistsBaseInfoExtractor):
         },
     }, {
         # with "Load more" button
-        'url': 'http://www.youtube.com/user/igorkle1/playlists?view=1&sort=dd',
+        'url': 'https://www.youtube.com/user/igorkle1/playlists?view=1&sort=dd',
         'playlist_mincount': 70,
         'info_dict': {
             'id': 'igorkle1',
@@ -2442,10 +2442,10 @@ class YoutubeTruncatedURLIE(InfoExtractor):
     '''
 
     _TESTS = [{
-        'url': 'http://www.youtube.com/watch?annotation_id=annotation_3951667041',
+        'url': 'https://www.youtube.com/watch?annotation_id=annotation_3951667041',
         'only_matching': True,
     }, {
-        'url': 'http://www.youtube.com/watch?',
+        'url': 'https://www.youtube.com/watch?',
         'only_matching': True,
     }, {
         'url': 'https://www.youtube.com/watch?x-yt-cl=84503534',
@@ -2466,7 +2466,7 @@ class YoutubeTruncatedURLIE(InfoExtractor):
             'Did you forget to quote the URL? Remember that & is a meta '
             'character in most shells, so you want to put the URL in quotes, '
             'like  youtube-dl '
-            '"http://www.youtube.com/watch?feature=foo&v=BaW_jenozKc" '
+            '"https://www.youtube.com/watch?feature=foo&v=BaW_jenozKc" '
             ' or simply  youtube-dl BaW_jenozKc  .',
             expected=True)
 

From f3625cc4ca8d8683b900e070ad7acd58b1fac5c7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 25 Sep 2016 18:08:35 +0700
Subject: [PATCH 6163/6219] [PULL_REQUEST_TEMPLATE.md] Add Unlicense notice

---
 .github/PULL_REQUEST_TEMPLATE.md | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
index f24bb4b09..3a168b7b1 100644
--- a/.github/PULL_REQUEST_TEMPLATE.md
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -10,6 +10,10 @@
 - [ ] At least skimmed through [adding new extractor tutorial](https://github.com/rg3/youtube-dl#adding-support-for-a-new-site) and [youtube-dl coding conventions](https://github.com/rg3/youtube-dl#youtube-dl-coding-conventions) sections
 - [ ] [Searched](https://github.com/rg3/youtube-dl/search?q=is%3Apr&type=Issues) the bugtracker for similar pull requests
 
+### In order to be accepted and merged into youtube-dl each piece of code must be in public domain or released under [Unlicense](http://unlicense.org/). Check one of the following options:
+- [ ] I am the original original author of this code and I am willing to release it under [Unlicense](http://unlicense.org/)
+- [ ] I am not the original author of this code but it is in public domain or released under [Unlicense](http://unlicense.org/) (provide reliable evidence)
+
 ### What is the purpose of your *pull request*?
 - [ ] Bug fix
 - [ ] New extractor

From e590b7ff9e8e408bb9ec4da58ab6847686d29dbc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 25 Sep 2016 18:09:46 +0700
Subject: [PATCH 6164/6219] [PULL_REQUEST_TEMPLATE.md] Add checkable
 Improvement options PR's purpose

---
 .github/PULL_REQUEST_TEMPLATE.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
index 3a168b7b1..89e8a3188 100644
--- a/.github/PULL_REQUEST_TEMPLATE.md
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -16,6 +16,7 @@
 
 ### What is the purpose of your *pull request*?
 - [ ] Bug fix
+- [ ] Improvement
 - [ ] New extractor
 - [ ] New feature
 

From a3d8b3816802c76beffa48789eac5181e02db3dc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 25 Sep 2016 21:58:17 +0700
Subject: [PATCH 6165/6219] [npo] Generalize playlist extractors

---
 youtube_dl/extractor/npo.py | 63 +++++++++++++++----------------------
 1 file changed, 26 insertions(+), 37 deletions(-)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index 3293bdb17..f95867d58 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -438,9 +438,29 @@ class SchoolTVIE(InfoExtractor):
         }
 
 
-class VPROIE(NPOIE):
+class NPOPlaylistBaseIE(NPOIE):
+    def _real_extract(self, url):
+        playlist_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, playlist_id)
+
+        entries = [
+            self.url_result('npo:%s' % video_id if not video_id.startswith('http') else video_id)
+            for video_id in re.findall(self._PLAYLIST_ENTRY_RE, webpage)
+        ]
+
+        playlist_title = self._html_search_regex(
+            self._PLAYLIST_TITLE_RE, webpage, 'playlist title',
+            default=None) or self._og_search_title(webpage)
+
+        return self.playlist_result(entries, playlist_id, playlist_title)
+
+
+class VPROIE(NPOPlaylistBaseIE):
     IE_NAME = 'vpro'
     _VALID_URL = r'https?://(?:www\.)?(?:tegenlicht\.)?vpro\.nl/(?:[^/]+/){2,}(?P<id>[^/]+)\.html'
+    _PLAYLIST_TITLE_RE = r'<title>\s*([^>]+?)\s*-\s*Teledoc\s*-\s*VPRO\s*</title>'
+    _PLAYLIST_ENTRY_RE = r'data-media-id="([^"]+)"'
 
     _TESTS = [
         {
@@ -473,48 +493,17 @@ class VPROIE(NPOIE):
         }
     ]
 
-    def _real_extract(self, url):
-        playlist_id = self._match_id(url)
 
-        webpage = self._download_webpage(url, playlist_id)
-
-        entries = [
-            self.url_result('npo:%s' % video_id if not video_id.startswith('http') else video_id)
-            for video_id in re.findall(r'data-media-id="([^"]+)"', webpage)
-        ]
-
-        playlist_title = self._search_regex(
-            r'<title>\s*([^>]+?)\s*-\s*Teledoc\s*-\s*VPRO\s*</title>',
-            webpage, 'playlist title', default=None) or self._og_search_title(webpage)
-
-        return self.playlist_result(entries, playlist_id, playlist_title)
-
-
-class WNLIE(InfoExtractor):
+class WNLIE(NPOPlaylistBaseIE):
     _VALID_URL = r'https?://(?:www\.)?omroepwnl\.nl/video/detail/(?P<id>[^/]+)__\d+'
+    _PLAYLIST_TITLE_RE = r'(?s)<h1[^>]+class="subject"[^>]*>(.+?)</h1>'
+    _PLAYLIST_ENTRY_RE = r'<a[^>]+href="([^"]+)"[^>]+class="js-mid"[^>]*>Deel \d+'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.omroepwnl.nl/video/detail/vandaag-de-dag-6-mei__060515',
         'info_dict': {
             'id': 'vandaag-de-dag-6-mei',
             'title': 'Vandaag de Dag 6 mei',
         },
         'playlist_count': 4,
-    }
-
-    def _real_extract(self, url):
-        playlist_id = self._match_id(url)
-
-        webpage = self._download_webpage(url, playlist_id)
-
-        entries = [
-            self.url_result('npo:%s' % video_id, 'NPO')
-            for video_id, part in re.findall(
-                r'<a[^>]+href="([^"]+)"[^>]+class="js-mid"[^>]*>(Deel \d+)', webpage)
-        ]
-
-        playlist_title = self._html_search_regex(
-            r'(?s)<h1[^>]+class="subject"[^>]*>(.+?)</h1>',
-            webpage, 'playlist title')
-
-        return self.playlist_result(entries, playlist_id, playlist_title)
+    }]

From ddb19772d572ae2118664a22d083a8f31fc63d8d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 25 Sep 2016 22:03:18 +0700
Subject: [PATCH 6166/6219] [vpro] Fix playlist title extraction and update
 tests

---
 youtube_dl/extractor/npo.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index f95867d58..ff02d0309 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -459,7 +459,7 @@ class NPOPlaylistBaseIE(NPOIE):
 class VPROIE(NPOPlaylistBaseIE):
     IE_NAME = 'vpro'
     _VALID_URL = r'https?://(?:www\.)?(?:tegenlicht\.)?vpro\.nl/(?:[^/]+/){2,}(?P<id>[^/]+)\.html'
-    _PLAYLIST_TITLE_RE = r'<title>\s*([^>]+?)\s*-\s*Teledoc\s*-\s*VPRO\s*</title>'
+    _PLAYLIST_TITLE_RE = r'<h1[^>]+class=["\'].*?\bmedia-platform-title\b.*?["\'][^>]*>([^<]+)'
     _PLAYLIST_ENTRY_RE = r'data-media-id="([^"]+)"'
 
     _TESTS = [
@@ -473,12 +473,13 @@ class VPROIE(NPOPlaylistBaseIE):
                 'description': 'md5:52cf4eefbc96fffcbdc06d024147abea',
                 'upload_date': '20130225',
             },
+            'skip': 'Video gone',
         },
         {
             'url': 'http://www.vpro.nl/programmas/2doc/2015/sergio-herman.html',
             'info_dict': {
                 'id': 'sergio-herman',
-                'title': 'Sergio Herman: Fucking perfect',
+                'title': 'sergio herman: fucking perfect',
             },
             'playlist_count': 2,
         },
@@ -487,7 +488,7 @@ class VPROIE(NPOPlaylistBaseIE):
             'url': 'http://www.vpro.nl/programmas/2doc/2015/education-education.html',
             'info_dict': {
                 'id': 'education-education',
-                'title': '2Doc',
+                'title': 'education education',
             },
             'playlist_count': 2,
         }

From 5742c18bc1ea3da5b0fd480e75fcdf099220e52f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 25 Sep 2016 22:19:00 +0700
Subject: [PATCH 6167/6219] [npo] Add support for anderetijden.nl (Closes
 #10754)

---
 youtube_dl/extractor/npo.py | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index ff02d0309..66035a77c 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -5,6 +5,7 @@ import re
 from .common import InfoExtractor
 from ..utils import (
     fix_xml_ampersands,
+    orderedSet,
     parse_duration,
     qualities,
     strip_jsonp,
@@ -446,7 +447,7 @@ class NPOPlaylistBaseIE(NPOIE):
 
         entries = [
             self.url_result('npo:%s' % video_id if not video_id.startswith('http') else video_id)
-            for video_id in re.findall(self._PLAYLIST_ENTRY_RE, webpage)
+            for video_id in orderedSet(re.findall(self._PLAYLIST_ENTRY_RE, webpage))
         ]
 
         playlist_title = self._html_search_regex(
@@ -508,3 +509,18 @@ class WNLIE(NPOPlaylistBaseIE):
         },
         'playlist_count': 4,
     }]
+
+
+class AndereTijdenIE(NPOPlaylistBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?anderetijden\.nl/programma/(?:[^/]+/)+(?P<id>[^/?#&]+)'
+    _PLAYLIST_TITLE_RE = r'(?s)<h1[^>]+class=["\'].*?\bpage-title\b.*?["\'][^>]*>(.+?)</h1>'
+    _PLAYLIST_ENTRY_RE = r'<figure[^>]+class=["\']episode-container episode-page["\'][^>]+data-prid=["\'](.+?)["\']'
+
+    _TESTS = [{
+        'url': 'http://anderetijden.nl/programma/1/Andere-Tijden/aflevering/676/Duitse-soldaten-over-de-Slag-bij-Arnhem',
+        'info_dict': {
+            'id': 'Duitse-soldaten-over-de-Slag-bij-Arnhem',
+            'title': 'Duitse soldaten over de Slag bij Arnhem',
+        },
+        'playlist_count': 3,
+    }]

From f1ee462c82381d3c68673500c0491fe477030c3b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 25 Sep 2016 22:38:36 +0700
Subject: [PATCH 6168/6219] [PULL_REQUEST_TEMPLATE.md] Fix typo

---
 .github/PULL_REQUEST_TEMPLATE.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
index 89e8a3188..46fa26f02 100644
--- a/.github/PULL_REQUEST_TEMPLATE.md
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -11,7 +11,7 @@
 - [ ] [Searched](https://github.com/rg3/youtube-dl/search?q=is%3Apr&type=Issues) the bugtracker for similar pull requests
 
 ### In order to be accepted and merged into youtube-dl each piece of code must be in public domain or released under [Unlicense](http://unlicense.org/). Check one of the following options:
-- [ ] I am the original original author of this code and I am willing to release it under [Unlicense](http://unlicense.org/)
+- [ ] I am the original author of this code and I am willing to release it under [Unlicense](http://unlicense.org/)
 - [ ] I am not the original author of this code but it is in public domain or released under [Unlicense](http://unlicense.org/) (provide reliable evidence)
 
 ### What is the purpose of your *pull request*?

From 2d5b4af0070f8aa6f3f4eb8fdabef5d006f6429a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 25 Sep 2016 23:30:57 +0700
Subject: [PATCH 6169/6219] [extractors] Add import for anderetijden extractor

---
 youtube_dl/extractor/extractors.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index bf1f70885..23fd2a308 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -612,13 +612,14 @@ from .nowtv import (
 )
 from .noz import NozIE
 from .npo import (
+    AndereTijdenIE,
     NPOIE,
     NPOLiveIE,
     NPORadioIE,
     NPORadioFragmentIE,
     SchoolTVIE,
     VPROIE,
-    WNLIE
+    WNLIE,
 )
 from .npr import NprIE
 from .nrk import (

From d3c97bad6181e1d3dc0cb4eece041e1cfb0ba6bc Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Mon, 26 Sep 2016 14:14:37 +0800
Subject: [PATCH 6170/6219] Ignore and cleanup 3gp files

---
 .gitignore | 1 +
 Makefile   | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/.gitignore b/.gitignore
index a802c75a1..002b700f5 100644
--- a/.gitignore
+++ b/.gitignore
@@ -29,6 +29,7 @@ updates_key.pem
 *.m4a
 *.m4v
 *.mp3
+*.3gp
 *.part
 *.swp
 test/testdata
diff --git a/Makefile b/Makefile
index ac234fcb0..a2763a664 100644
--- a/Makefile
+++ b/Makefile
@@ -1,7 +1,7 @@
 all: youtube-dl README.md CONTRIBUTING.md README.txt youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtube-dl.fish supportedsites
 
 clean:
-	rm -rf youtube-dl.1.temp.md youtube-dl.1 youtube-dl.bash-completion README.txt MANIFEST build/ dist/ .coverage cover/ youtube-dl.tar.gz youtube-dl.zsh youtube-dl.fish youtube_dl/extractor/lazy_extractors.py *.dump *.part* *.info.json *.mp4 *.m4a *.flv *.mp3 *.avi *.mkv *.webm *.jpg *.png CONTRIBUTING.md.tmp ISSUE_TEMPLATE.md.tmp youtube-dl youtube-dl.exe
+	rm -rf youtube-dl.1.temp.md youtube-dl.1 youtube-dl.bash-completion README.txt MANIFEST build/ dist/ .coverage cover/ youtube-dl.tar.gz youtube-dl.zsh youtube-dl.fish youtube_dl/extractor/lazy_extractors.py *.dump *.part* *.info.json *.mp4 *.m4a *.flv *.mp3 *.avi *.mkv *.webm *.3gp *.jpg *.png CONTRIBUTING.md.tmp ISSUE_TEMPLATE.md.tmp youtube-dl youtube-dl.exe
 	find . -name "*.pyc" -delete
 	find . -name "*.class" -delete
 

From fffb9cff944cfab11f311900ee8138f28f7232d3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 26 Sep 2016 22:15:58 +0700
Subject: [PATCH 6171/6219] [kaltura] Speed up embed regexes (#10764)

---
 youtube_dl/extractor/kaltura.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/kaltura.py b/youtube_dl/extractor/kaltura.py
index 5a8403777..91bc3a0a7 100644
--- a/youtube_dl/extractor/kaltura.py
+++ b/youtube_dl/extractor/kaltura.py
@@ -105,20 +105,20 @@ class KalturaIE(InfoExtractor):
                     kWidget\.(?:thumb)?[Ee]mbed\(
                     \{.*?
                         (?P<q1>['\"])wid(?P=q1)\s*:\s*
-                        (?P<q2>['\"])_?(?P<partner_id>[^'\"]+)(?P=q2),.*?
+                        (?P<q2>['\"])_?(?P<partner_id>(?:(?!(?P=q2)).)+)(?P=q2),.*?
                         (?P<q3>['\"])entry_?[Ii]d(?P=q3)\s*:\s*
-                        (?P<q4>['\"])(?P<id>[^'\"]+)(?P=q4),
+                        (?P<q4>['\"])(?P<id>(?:(?!(?P=q4)).)+)(?P=q4),
                 """, webpage) or
             re.search(
                 r'''(?xs)
                     (?P<q1>["\'])
-                        (?:https?:)?//cdnapi(?:sec)?\.kaltura\.com/.*?(?:p|partner_id)/(?P<partner_id>\d+).*?
+                        (?:https?:)?//cdnapi(?:sec)?\.kaltura\.com/(?:(?!(?P=q1)).)*(?:p|partner_id)/(?P<partner_id>\d+)(?:(?!(?P=q1)).)*
                     (?P=q1).*?
                     (?:
                         entry_?[Ii]d|
                         (?P<q2>["\'])entry_?[Ii]d(?P=q2)
                     )\s*:\s*
-                    (?P<q3>["\'])(?P<id>.+?)(?P=q3)
+                    (?P<q3>["\'])(?P<id>(?:(?!(?P=q3)).)+)(?P=q3)
                 ''', webpage))
         if mobj:
             embed_info = mobj.groupdict()

From d3dbb46330461c0c70c3aae47b69d27882cfc325 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ond=C5=99ej=20B=C3=A1rta?= <git@ondrej.it>
Date: Sun, 25 Sep 2016 12:44:46 +0200
Subject: [PATCH 6172/6219] [promptfile] Fix extraction (Closes #10634)

---
 youtube_dl/extractor/promptfile.py | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/promptfile.py b/youtube_dl/extractor/promptfile.py
index f93bd19ff..54c4aee13 100644
--- a/youtube_dl/extractor/promptfile.py
+++ b/youtube_dl/extractor/promptfile.py
@@ -15,12 +15,12 @@ from ..utils import (
 class PromptFileIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?promptfile\.com/l/(?P<id>[0-9A-Z\-]+)'
     _TEST = {
-        'url': 'http://www.promptfile.com/l/D21B4746E9-F01462F0FF',
-        'md5': 'd1451b6302da7215485837aaea882c4c',
+        'url': 'http://www.promptfile.com/l/86D1CE8462-576CAAE416',
+        'md5': '5a7e285a26e0d66d9a263fae91bc92ce',
         'info_dict': {
-            'id': 'D21B4746E9-F01462F0FF',
+            'id': '86D1CE8462-576CAAE416',
             'ext': 'mp4',
-            'title': 'Birds.mp4',
+            'title': 'oceans.mp4',
             'thumbnail': 're:^https?://.*\.jpg$',
         }
     }
@@ -33,14 +33,20 @@ class PromptFileIE(InfoExtractor):
             raise ExtractorError('Video %s does not exist' % video_id,
                                  expected=True)
 
+        chash_pattern = r'\$\("#chash"\)\.val\("(.+)"\+\$\("#chash"\)'
+        chash = self._html_search_regex(chash_pattern, webpage, "chash")
         fields = self._hidden_inputs(webpage)
+        k = list(fields)[0]
+        fields[k] = chash + fields[k]
+
         post = urlencode_postdata(fields)
         req = sanitized_Request(url, post)
         req.add_header('Content-type', 'application/x-www-form-urlencoded')
         webpage = self._download_webpage(
             req, video_id, 'Downloading video page')
 
-        url = self._html_search_regex(r'url:\s*\'([^\']+)\'', webpage, 'URL')
+        url_pattern = r'<a href="(http://www\.promptfile\.com/file/[^"]+)'
+        url = self._html_search_regex(url_pattern, webpage, 'URL')
         title = self._html_search_regex(
             r'<span.+title="([^"]+)">', webpage, 'title')
         thumbnail = self._html_search_regex(

From 72c3d02d294b04b35a19417b31ad497e7540caa3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 26 Sep 2016 23:39:54 +0700
Subject: [PATCH 6173/6219] [promptfile] Improve and modernize

---
 youtube_dl/extractor/promptfile.py | 26 ++++++++++++++------------
 1 file changed, 14 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/promptfile.py b/youtube_dl/extractor/promptfile.py
index 54c4aee13..d40cca06f 100644
--- a/youtube_dl/extractor/promptfile.py
+++ b/youtube_dl/extractor/promptfile.py
@@ -7,7 +7,6 @@ from .common import InfoExtractor
 from ..utils import (
     determine_ext,
     ExtractorError,
-    sanitized_Request,
     urlencode_postdata,
 )
 
@@ -33,20 +32,23 @@ class PromptFileIE(InfoExtractor):
             raise ExtractorError('Video %s does not exist' % video_id,
                                  expected=True)
 
-        chash_pattern = r'\$\("#chash"\)\.val\("(.+)"\+\$\("#chash"\)'
-        chash = self._html_search_regex(chash_pattern, webpage, "chash")
+        chash = self._search_regex(
+            r'val\("([^"]*)"\s*\+\s*\$\("#chash"\)', webpage, 'chash')
         fields = self._hidden_inputs(webpage)
-        k = list(fields)[0]
-        fields[k] = chash + fields[k]
+        keys = list(fields.keys())
+        chash_key = keys[0] if len(keys) == 1 else next(
+            key for key in keys if key.startswith('cha'))
+        fields[chash_key] = chash + fields[chash_key]
 
-        post = urlencode_postdata(fields)
-        req = sanitized_Request(url, post)
-        req.add_header('Content-type', 'application/x-www-form-urlencoded')
         webpage = self._download_webpage(
-            req, video_id, 'Downloading video page')
+            url, video_id, 'Downloading video page',
+            data=urlencode_postdata(fields),
+            headers={'Content-type': 'application/x-www-form-urlencoded'})
 
-        url_pattern = r'<a href="(http://www\.promptfile\.com/file/[^"]+)'
-        url = self._html_search_regex(url_pattern, webpage, 'URL')
+        video_url = self._search_regex(
+            (r'<a[^>]+href=(["\'])(?P<url>(?:(?!\1).)+)\1[^>]*>\s*Download File',
+             r'<a[^>]+href=(["\'])(?P<url>https?://(?:www\.)?promptfile\.com/file/(?:(?!\1).)+)\1'),
+            webpage, 'video url', group='url')
         title = self._html_search_regex(
             r'<span.+title="([^"]+)">', webpage, 'title')
         thumbnail = self._html_search_regex(
@@ -55,7 +57,7 @@ class PromptFileIE(InfoExtractor):
 
         formats = [{
             'format_id': 'sd',
-            'url': url,
+            'url': video_url,
             'ext': determine_ext(title),
         }]
         self._sort_formats(formats)

From d75d9e343e91527c1fe34678e913ae16a0eafbdd Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 27 Sep 2016 14:38:41 +0800
Subject: [PATCH 6174/6219] [einthusan] Fix extraction (closes #10714)

---
 ChangeLog                         |  1 +
 youtube_dl/extractor/einthusan.py | 14 ++++++++------
 2 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 5c96dc179..fdebb89b9 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -4,6 +4,7 @@ Core
 + Improved support for HTML5 subtitles
 
 Extractors
+* [einthusan] Fix extraction (#10714)
 + [twitter] Support Periscope embeds (#10737)
 + [openload] Support subtitles (#10625)
 
diff --git a/youtube_dl/extractor/einthusan.py b/youtube_dl/extractor/einthusan.py
index f7339702c..443865ad2 100644
--- a/youtube_dl/extractor/einthusan.py
+++ b/youtube_dl/extractor/einthusan.py
@@ -14,7 +14,7 @@ class EinthusanIE(InfoExtractor):
     _TESTS = [
         {
             'url': 'http://www.einthusan.com/movies/watch.php?id=2447',
-            'md5': 'af244f4458cd667205e513d75da5b8b1',
+            'md5': 'd71379996ff5b7f217eca034c34e3461',
             'info_dict': {
                 'id': '2447',
                 'ext': 'mp4',
@@ -25,13 +25,13 @@ class EinthusanIE(InfoExtractor):
         },
         {
             'url': 'http://www.einthusan.com/movies/watch.php?id=1671',
-            'md5': 'ef63c7a803e22315880ed182c10d1c5c',
+            'md5': 'b16a6fd3c67c06eb7c79c8a8615f4213',
             'info_dict': {
                 'id': '1671',
                 'ext': 'mp4',
                 'title': 'Soodhu Kavvuum',
                 'thumbnail': 're:^https?://.*\.jpg$',
-                'description': 'md5:05d8a0c0281a4240d86d76e14f2f4d51',
+                'description': 'md5:b40f2bf7320b4f9414f3780817b2af8c',
             }
         },
     ]
@@ -50,9 +50,11 @@ class EinthusanIE(InfoExtractor):
         video_id = self._search_regex(
             r'data-movieid=["\'](\d+)', webpage, 'video id', default=video_id)
 
-        video_url = self._download_webpage(
+        m3u8_url = self._download_webpage(
             'http://cdn.einthusan.com/geturl/%s/hd/London,Washington,Toronto,Dallas,San,Sydney/'
-            % video_id, video_id)
+            % video_id, video_id, headers={'Referer': url})
+        formats = self._extract_m3u8_formats(
+            m3u8_url, video_id, ext='mp4', entry_protocol='m3u8_native')
 
         description = self._html_search_meta('description', webpage)
         thumbnail = self._html_search_regex(
@@ -64,7 +66,7 @@ class EinthusanIE(InfoExtractor):
         return {
             'id': video_id,
             'title': title,
-            'url': video_url,
+            'formats': formats,
             'thumbnail': thumbnail,
             'description': description,
         }

From 93933c9819fa1282081a5f0761cbeabc9fbea336 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 27 Sep 2016 15:28:37 +0100
Subject: [PATCH 6175/6219] [awaan:video] fix test(closes #10773)

---
 youtube_dl/extractor/awaan.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/awaan.py b/youtube_dl/extractor/awaan.py
index 66d7515bc..a2603bbff 100644
--- a/youtube_dl/extractor/awaan.py
+++ b/youtube_dl/extractor/awaan.py
@@ -66,6 +66,7 @@ class AWAANVideoIE(AWAANBaseIE):
             'duration': 2041,
             'timestamp': 1227504126,
             'upload_date': '20081124',
+            'uploader_id': '71',
         },
     }, {
         'url': 'http://awaan.ae/video/26723981/%D8%AF%D8%A7%D8%B1-%D8%A7%D9%84%D8%B3%D9%84%D8%A7%D9%85:-%D8%AE%D9%8A%D8%B1-%D8%AF%D9%88%D8%B1-%D8%A7%D9%84%D8%A3%D9%86%D8%B5%D8%A7%D8%B1',

From 2342733f850c979c6f23ea2e83dfcb176fb08fa5 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 27 Sep 2016 15:29:50 +0100
Subject: [PATCH 6176/6219] fix tests related to
 1978540a5122c53012e17a78841f3da0df77fd34(closes #10774)

---
 youtube_dl/extractor/formula1.py |  6 +++++-
 youtube_dl/extractor/voxmedia.py | 10 ++++++++--
 2 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/formula1.py b/youtube_dl/extractor/formula1.py
index 8c417ab65..fecfc28ae 100644
--- a/youtube_dl/extractor/formula1.py
+++ b/youtube_dl/extractor/formula1.py
@@ -11,9 +11,13 @@ class Formula1IE(InfoExtractor):
         'md5': '8c79e54be72078b26b89e0e111c0502b',
         'info_dict': {
             'id': 'JvYXJpMzE6pArfHWm5ARp5AiUmD-gibV',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Race highlights - Spain 2016',
         },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
         'add_ie': ['Ooyala'],
     }, {
         'url': 'http://www.formula1.com/en/video/2016/5/Race_highlights_-_Spain_2016.html',
diff --git a/youtube_dl/extractor/voxmedia.py b/youtube_dl/extractor/voxmedia.py
index b1b32ad44..f8e331493 100644
--- a/youtube_dl/extractor/voxmedia.py
+++ b/youtube_dl/extractor/voxmedia.py
@@ -9,13 +9,16 @@ class VoxMediaIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?(?:theverge|vox|sbnation|eater|polygon|curbed|racked)\.com/(?:[^/]+/)*(?P<id>[^/?]+)'
     _TESTS = [{
         'url': 'http://www.theverge.com/2014/6/27/5849272/material-world-how-google-discovered-what-software-is-made-of',
-        'md5': '73856edf3e89a711e70d5cf7cb280b37',
         'info_dict': {
             'id': '11eXZobjrG8DCSTgrNjVinU-YmmdYjhe',
             'ext': 'mp4',
             'title': 'Google\'s new material design direction',
             'description': 'md5:2f44f74c4d14a1f800ea73e1c6832ad2',
         },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
         'add_ie': ['Ooyala'],
     }, {
         # data-ooyala-id
@@ -31,13 +34,16 @@ class VoxMediaIE(InfoExtractor):
     }, {
         # volume embed
         'url': 'http://www.vox.com/2016/3/31/11336640/mississippi-lgbt-religious-freedom-bill',
-        'md5': '375c483c5080ab8cd85c9c84cfc2d1e4',
         'info_dict': {
             'id': 'wydzk3dDpmRz7PQoXRsTIX6XTkPjYL0b',
             'ext': 'mp4',
             'title': 'The new frontier of LGBTQ civil rights, explained',
             'description': 'md5:0dc58e94a465cbe91d02950f770eb93f',
         },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
         'add_ie': ['Ooyala'],
     }, {
         # youtube embed

From f9dd86a112835e04e271e8d1d844f250e6ff0c5e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 27 Sep 2016 21:37:33 +0700
Subject: [PATCH 6177/6219] [npo] Clarify IE_NAMEs (Closes #10775)

---
 youtube_dl/extractor/npo.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index 66035a77c..9c7cc777b 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -497,6 +497,7 @@ class VPROIE(NPOPlaylistBaseIE):
 
 
 class WNLIE(NPOPlaylistBaseIE):
+    IE_NAME = 'wnl'
     _VALID_URL = r'https?://(?:www\.)?omroepwnl\.nl/video/detail/(?P<id>[^/]+)__\d+'
     _PLAYLIST_TITLE_RE = r'(?s)<h1[^>]+class="subject"[^>]*>(.+?)</h1>'
     _PLAYLIST_ENTRY_RE = r'<a[^>]+href="([^"]+)"[^>]+class="js-mid"[^>]*>Deel \d+'
@@ -512,6 +513,7 @@ class WNLIE(NPOPlaylistBaseIE):
 
 
 class AndereTijdenIE(NPOPlaylistBaseIE):
+    IE_NAME = 'anderetijden'
     _VALID_URL = r'https?://(?:www\.)?anderetijden\.nl/programma/(?:[^/]+/)+(?P<id>[^/?#&]+)'
     _PLAYLIST_TITLE_RE = r'(?s)<h1[^>]+class=["\'].*?\bpage-title\b.*?["\'][^>]*>(.+?)</h1>'
     _PLAYLIST_ENTRY_RE = r'<figure[^>]+class=["\']episode-container episode-page["\'][^>]+data-prid=["\'](.+?)["\']'

From 1a2fbe322ee2d711b474f32a7d3f331791fb1881 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 27 Sep 2016 21:55:51 +0700
Subject: [PATCH 6178/6219] [periscope] Treat timed_out state as finished
 stream

---
 youtube_dl/extractor/periscope.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/periscope.py b/youtube_dl/extractor/periscope.py
index e8b2f11c6..61043cad5 100644
--- a/youtube_dl/extractor/periscope.py
+++ b/youtube_dl/extractor/periscope.py
@@ -87,7 +87,7 @@ class PeriscopeIE(PeriscopeBaseIE):
                 'ext': 'flv' if format_id == 'rtmp' else 'mp4',
             }
             if format_id != 'rtmp':
-                f['protocol'] = 'm3u8_native' if state == 'ended' else 'm3u8'
+                f['protocol'] = 'm3u8_native' if state in ('ended', 'timed_out') else 'm3u8'
             formats.append(f)
         self._sort_formats(formats)
 

From e3845525906228091fdf446f2cf2e9a20e93f59f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kacper=20Michaj=C5=82ow?= <kasper93@gmail.com>
Date: Sun, 25 Sep 2016 05:39:29 +0200
Subject: [PATCH 6179/6219] [vk] Add support for dailymotion embeds

Fixes #10661
---
 youtube_dl/extractor/vk.py | 24 +++++++++++++++++++++++-
 1 file changed, 23 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index cd22df25a..f26e0732c 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -23,8 +23,9 @@ from ..utils import (
     unified_strdate,
     urlencode_postdata,
 )
-from .vimeo import VimeoIE
+from .dailymotion import DailymotionIE
 from .pladform import PladformIE
+from .vimeo import VimeoIE
 
 
 class VKBaseIE(InfoExtractor):
@@ -210,6 +211,23 @@ class VKIE(VKBaseIE):
                 'view_count': int,
             },
         },
+        {
+            # dailymotion embed
+            'url': 'https://vk.com/video-37468416_456239855',
+            'info_dict': {
+                'id': 'k3lz2cmXyRuJQSjGHUv',
+                'ext': 'mp4',
+                'title': 'md5:d52606645c20b0ddbb21655adaa4f56f',
+                'description': 'md5:c651358f03c56f1150b555c26d90a0fd',
+                'uploader': 'AniLibria.Tv',
+                'upload_date': '20160914',
+                'uploader_id': 'x1p5vl5',
+                'timestamp': 1473877246,
+            },
+            'params': {
+                'skip_download': True,
+            }
+        },
         {
             # video key is extra_data not url\d+
             'url': 'http://vk.com/video-110305615_171782105',
@@ -315,6 +333,10 @@ class VKIE(VKBaseIE):
                 m_rutube.group(1).replace('\\', ''))
             return self.url_result(rutube_url)
 
+        dailymotion_urls = DailymotionIE._extract_urls(info_page)
+        if dailymotion_urls:
+            return self.url_result(dailymotion_urls[0], DailymotionIE.ie_key())
+
         m_opts = re.search(r'(?s)var\s+opts\s*=\s*({.+?});', info_page)
         if m_opts:
             m_opts_url = re.search(r"url\s*:\s*'((?!/\b)[^']+)", m_opts.group(1))

From cdfcc4ce95e351c3f560fa3c07ae7d4ab188ef25 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 27 Sep 2016 22:27:10 +0700
Subject: [PATCH 6180/6219] [mtv] Improve _VALID_URL

---
 youtube_dl/extractor/mtv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index 2e9580b10..74a3a035e 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -271,7 +271,7 @@ class MTVServicesEmbeddedIE(MTVServicesInfoExtractor):
 
 class MTVIE(MTVServicesInfoExtractor):
     IE_NAME = 'mtv'
-    _VALID_URL = r'https?://(?:www\.)?mtv\.com/(video-clips|full-episodes)/(?P<id>[^/?#.]+)'
+    _VALID_URL = r'https?://(?:www\.)?mtv\.com/(?:video-clips|full-episodes)/(?P<id>[^/?#.]+)'
     _FEED_URL = 'http://www.mtv.com/feeds/mrss/'
 
     _TESTS = [{

From dd2cffeeec8feac8fe52924760b2cb368249396a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 27 Sep 2016 22:43:35 +0700
Subject: [PATCH 6181/6219] [ChangeLog] Actualize

---
 ChangeLog | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index fdebb89b9..33c94ef55 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,9 +1,24 @@
 vesion <unreleased>
 
 Core
++ Add hdcore query parameter to akamai f4m formats
++ Delegate HLS live streams downloading to ffmpeg
 + Improved support for HTML5 subtitles
 
 Extractors
++ [vk] Add support for dailymotion embeds (#10661)
+* [einthusan] Fix extraction (#10714)
+* [promptfile] Fix extraction (#10634)
+* [kaltura] Speed up embed regular expressions (#10764)
++ [npo] Add support for anderetijden.nl (#10754)
++ [prosiebensat1] Add support for advopedia sites
+* [mwave] Relax URL regular expression (#10735, #10748)
+* [prosiebensat1] Fix playlist support (#10745)
++ [prosiebensat1] Add support for sat1gold sites (#10745)
++ [cbsnews:livevideo] Fix extraction and extract m3u8 formats
++ [brightcove:new] Add support for live streams
+* [soundcloud] Generalize playlist entries extraction (#10733)
++ [mtv] Add support for new URL schema (#8169, #9808)
 * [einthusan] Fix extraction (#10714)
 + [twitter] Support Periscope embeds (#10737)
 + [openload] Support subtitles (#10625)

From c8f45f763cac3c0d0e4ca35ba072d8d321957e85 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 27 Sep 2016 23:03:00 +0700
Subject: [PATCH 6182/6219] [ChangeLog] Remove duplicate

---
 ChangeLog | 1 -
 1 file changed, 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index 33c94ef55..0d15b6a82 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -7,7 +7,6 @@ Core
 
 Extractors
 + [vk] Add support for dailymotion embeds (#10661)
-* [einthusan] Fix extraction (#10714)
 * [promptfile] Fix extraction (#10634)
 * [kaltura] Speed up embed regular expressions (#10764)
 + [npo] Add support for anderetijden.nl (#10754)

From 8f0cf20ab987019c3ba66c375450f80bb1cfe281 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 27 Sep 2016 23:09:46 +0700
Subject: [PATCH 6183/6219] release 2016.09.27

---
 .github/ISSUE_TEMPLATE.md |  6 +++---
 ChangeLog                 |  2 +-
 docs/supportedsites.md    | 10 ++++++----
 youtube_dl/version.py     |  2 +-
 4 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 7669ab9b7..273eb8c0b 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.09.24*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.09.24**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.09.27*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.09.27**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.09.24
+[debug] youtube-dl version 2016.09.27
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 0d15b6a82..f8149cc30 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-vesion <unreleased>
+vesion 2016.09.27
 
 Core
 + Add hdcore query parameter to akamai f4m formats
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 95a137393..26f275577 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -40,6 +40,7 @@
  - **Allocine**
  - **AlphaPorno**
  - **AMCNetworks**
+ - **anderetijden**: npo.nl and ntr.nl
  - **AnimeOnDemand**
  - **anitube.se**
  - **AnySex**
@@ -127,8 +128,8 @@
  - **CBS**
  - **CBSInteractive**
  - **CBSLocal**
- - **CBSNews**: CBS News
- - **CBSNewsLiveVideo**: CBS News Live Videos
+ - **cbsnews**: CBS News
+ - **cbsnews:livevideo**: CBS News Live Videos
  - **CBSSports**
  - **CCTV**
  - **CDA**
@@ -424,8 +425,9 @@
  - **MPORA**
  - **MSN**
  - **mtg**: MTG services
- - **MTV**
+ - **mtv**
  - **mtv.de**
+ - **mtv:video**
  - **mtvservices:embedded**
  - **MuenchenTV**: münchen.tv
  - **MusicPlayOn**
@@ -865,7 +867,7 @@
  - **wholecloud**: WholeCloud
  - **Wimp**
  - **Wistia**
- - **WNL**
+ - **wnl**: npo.nl and ntr.nl
  - **WorldStarHipHop**
  - **wrzuta.pl**
  - **wrzuta.pl:playlist**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 2af6380b8..af0c2cfc4 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.09.24'
+__version__ = '2016.09.27'

From 8bfda726c20198b7e68a805967917ef1a79e9b91 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 28 Sep 2016 16:34:27 +0100
Subject: [PATCH 6184/6219] [limelight:media] improve http formats extraction

---
 youtube_dl/extractor/limelight.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/limelight.py b/youtube_dl/extractor/limelight.py
index 6752ffee2..b7bfa7a6d 100644
--- a/youtube_dl/extractor/limelight.py
+++ b/youtube_dl/extractor/limelight.py
@@ -59,7 +59,7 @@ class LimelightBaseIE(InfoExtractor):
                     format_id = 'rtmp'
                     if stream.get('videoBitRate'):
                         format_id += '-%d' % int_or_none(stream['videoBitRate'])
-                    http_url = 'http://%s/%s' % (rtmp.group('host').replace('csl.', 'cpl.'), rtmp.group('playpath')[4:])
+                    http_url = 'http://cpl.delvenetworks.com/' + rtmp.group('playpath')[4:]
                     urls.append(http_url)
                     http_fmt = fmt.copy()
                     http_fmt.update({

From f533490bb7b2d25b9c6fe7ccd381ebe2bef7d4f8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 28 Sep 2016 22:58:25 +0700
Subject: [PATCH 6185/6219] [ketnet] Extract mzsource formats (#10770)

---
 youtube_dl/extractor/ketnet.py | 26 +++++++++++++++++++++++---
 1 file changed, 23 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/ketnet.py b/youtube_dl/extractor/ketnet.py
index aaf3f807a..eb0a16008 100644
--- a/youtube_dl/extractor/ketnet.py
+++ b/youtube_dl/extractor/ketnet.py
@@ -21,6 +21,10 @@ class KetnetIE(InfoExtractor):
     }, {
         'url': 'https://www.ketnet.be/achter-de-schermen/sien-repeteert-voor-stars-for-life',
         'only_matching': True,
+    }, {
+        # mzsource, geo restricted to Belgium
+        'url': 'https://www.ketnet.be/kijken/nachtwacht/de-bermadoe',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
@@ -36,9 +40,25 @@ class KetnetIE(InfoExtractor):
 
         title = config['title']
 
-        formats = self._extract_m3u8_formats(
-            config['source']['hls'], video_id, 'mp4',
-            entry_protocol='m3u8_native', m3u8_id='hls')
+        formats = []
+        for source_key in ('', 'mz'):
+            source = config.get('%ssource' % source_key)
+            if not isinstance(source, dict):
+                continue
+            for format_id, format_url in source.items():
+                if format_id == 'hls':
+                    formats.extend(self._extract_m3u8_formats(
+                        format_url, video_id, 'mp4',
+                        entry_protocol='m3u8_native', m3u8_id=format_id,
+                        fatal=False))
+                elif format_id == 'hds':
+                    formats.extend(self._extract_f4m_formats(
+                        format_url, video_id, f4m_id=format_id, fatal=False))
+                else:
+                    formats.append({
+                        'url': format_url,
+                        'format_id': format_id,
+                    })
         self._sort_formats(formats)
 
         return {

From a56e74e2713ed45f4096735cf49d1d97b5e75389 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?D=C3=A9stin=20Reed?= <trox1972@users.noreply.github.com>
Date: Wed, 28 Sep 2016 16:54:06 +0200
Subject: [PATCH 6186/6219] [Instagram] Extract comments

---
 youtube_dl/extractor/instagram.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/youtube_dl/extractor/instagram.py b/youtube_dl/extractor/instagram.py
index 8f7f232be..5ebc30a10 100644
--- a/youtube_dl/extractor/instagram.py
+++ b/youtube_dl/extractor/instagram.py
@@ -29,6 +29,7 @@ class InstagramIE(InfoExtractor):
             'uploader': 'Naomi Leonor Phan-Quang',
             'like_count': int,
             'comment_count': int,
+            'comments': list,
         },
     }, {
         # missing description
@@ -44,6 +45,7 @@ class InstagramIE(InfoExtractor):
             'uploader': 'Britney Spears',
             'like_count': int,
             'comment_count': int,
+            'comments': list,
         },
         'params': {
             'skip_download': True,
@@ -101,6 +103,14 @@ class InstagramIE(InfoExtractor):
                 uploader_id = media.get('owner', {}).get('username')
                 like_count = int_or_none(media.get('likes', {}).get('count'))
                 comment_count = int_or_none(media.get('comments', {}).get('count'))
+                comments = [{
+                    'author': comment.get('user', {}).get('username'),
+                    'author_id': comment.get('user', {}).get('id'),
+                    'id': comment.get('id'),
+                    'text': comment.get('text'),
+                    'timestamp': int_or_none(comment.get('created_at')),
+                } for comment in media.get('comments', {}).get('nodes', [])
+                if comment.get('text')]
 
         if not video_url:
             video_url = self._og_search_video_url(webpage, secure=False)
@@ -131,6 +141,7 @@ class InstagramIE(InfoExtractor):
             'uploader': uploader,
             'like_count': like_count,
             'comment_count': comment_count,
+            'comments': comments,
         }
 
 

From 0d72ff9c51ecc84aae1717c05f8b73ad94199687 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 29 Sep 2016 21:39:35 +0800
Subject: [PATCH 6187/6219] [leeco] Recognize more Le Sports URLs (#10794)

---
 ChangeLog                     | 8 +++++++-
 youtube_dl/extractor/leeco.py | 8 +++++++-
 2 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index f8149cc30..70da55c90 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,10 @@
-vesion 2016.09.27
+version <unreleased>
+
+Extractors
++ [leeco] Recognize more Le Sports URLs (#10794)
+
+
+version 2016.09.27
 
 Core
 + Add hdcore query parameter to akamai f4m formats
diff --git a/youtube_dl/extractor/leeco.py b/youtube_dl/extractor/leeco.py
index e9cc9aa59..c48a5aad1 100644
--- a/youtube_dl/extractor/leeco.py
+++ b/youtube_dl/extractor/leeco.py
@@ -29,7 +29,7 @@ from ..utils import (
 
 class LeIE(InfoExtractor):
     IE_DESC = '乐视网'
-    _VALID_URL = r'https?://(?:www\.le\.com/ptv/vplay|sports\.le\.com/video)/(?P<id>\d+)\.html'
+    _VALID_URL = r'https?://(?:www\.le\.com/ptv/vplay|(?:sports\.le|(?:www\.)?lesports)\.com/(?:match|video))/(?P<id>\d+)\.html'
 
     _URL_TEMPLATE = 'http://www.le.com/ptv/vplay/%s.html'
 
@@ -73,6 +73,12 @@ class LeIE(InfoExtractor):
     }, {
         'url': 'http://sports.le.com/video/25737697.html',
         'only_matching': True,
+    }, {
+        'url': 'http://www.lesports.com/match/1023203003.html',
+        'only_matching': True,
+    }, {
+        'url': 'http://sports.le.com/match/1023203003.html',
+        'only_matching': True,
     }]
 
     # ror() and calc_time_key() are reversed from a embedded swf file in KLetvPlayer.swf

From 93aa0b631878b62f756c83e1069a14cd2d8775f1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 29 Sep 2016 23:04:10 +0700
Subject: [PATCH 6188/6219] [vk] Add support for finished live streams (#10799)

---
 youtube_dl/extractor/vk.py | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index f26e0732c..1d089c9d7 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -226,7 +226,7 @@ class VKIE(VKBaseIE):
             },
             'params': {
                 'skip_download': True,
-            }
+            },
         },
         {
             # video key is extra_data not url\d+
@@ -241,6 +241,18 @@ class VKIE(VKBaseIE):
                 'view_count': int,
             },
         },
+        {
+            # finished live stream, live_mp4
+            'url': 'https://vk.com/videos-387766?z=video-387766_456242764%2Fpl_-387766_-2',
+            'md5': '90d22d051fccbbe9becfccc615be6791',
+            'info_dict': {
+                'id': '456242764',
+                'ext': 'mp4',
+                'title': 'ИгроМир 2016 — день 1',
+                'uploader': 'Игромания',
+                'duration': 5239,
+            },
+        },
         {
             # removed video, just testing that we match the pattern
             'url': 'http://vk.com/feed?z=video-43215063_166094326%2Fbb50cacd3177146d7a',
@@ -366,7 +378,10 @@ class VKIE(VKBaseIE):
 
         formats = []
         for k, v in data.items():
-            if not k.startswith('url') and not k.startswith('cache') and k != 'extra_data' or not v:
+            if (not k.startswith('url') and not k.startswith('cache')
+                    and k not in ('extra_data', 'live_mp4')):
+                continue
+            if not isinstance(v, compat_str) or not v.startswith('http'):
                 continue
             height = int_or_none(self._search_regex(
                 r'^(?:url|cache)(\d+)', k, 'height', default=None))

From 475f8a458099c64d367356471069bd0ff2bd1b0d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 29 Sep 2016 23:21:39 +0700
Subject: [PATCH 6189/6219] [vk] Add support for running live streams (Closes
 #10799)

---
 youtube_dl/extractor/vk.py | 47 ++++++++++++++++++++++++++------------
 1 file changed, 33 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 1d089c9d7..9f7a593ef 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -253,6 +253,12 @@ class VKIE(VKBaseIE):
                 'duration': 5239,
             },
         },
+        {
+            # live stream, hls and rtmp links,most likely already finished live
+            # stream by the time you are reading this comment
+            'url': 'https://vk.com/video-140332_456239111',
+            'only_matching': True,
+        },
         {
             # removed video, just testing that we match the pattern
             'url': 'http://vk.com/feed?z=video-43215063_166094326%2Fbb50cacd3177146d7a',
@@ -361,6 +367,11 @@ class VKIE(VKBaseIE):
         data_json = self._search_regex(r'var\s+vars\s*=\s*({.+?});', info_page, 'vars')
         data = json.loads(data_json)
 
+        title = unescapeHTML(data['md_title'])
+
+        if data.get('live') == 2:
+            title = self._live_title(title)
+
         # Extract upload date
         upload_date = None
         mobj = re.search(r'id="mv_date(?:_views)?_wrap"[^>]*>([a-zA-Z]+ [0-9]+), ([0-9]+) at', info_page)
@@ -377,25 +388,33 @@ class VKIE(VKBaseIE):
                 r'([\d,.]+)', views, 'view count', fatal=False))
 
         formats = []
-        for k, v in data.items():
-            if (not k.startswith('url') and not k.startswith('cache')
-                    and k not in ('extra_data', 'live_mp4')):
+        for format_id, format_url in data.items():
+            if not isinstance(format_url, compat_str) or not format_url.startswith(('http', '//', 'rtmp')):
                 continue
-            if not isinstance(v, compat_str) or not v.startswith('http'):
-                continue
-            height = int_or_none(self._search_regex(
-                r'^(?:url|cache)(\d+)', k, 'height', default=None))
-            formats.append({
-                'format_id': k,
-                'url': v,
-                'height': height,
-            })
+            if format_id.startswith(('url', 'cache')) or format_id in ('extra_data', 'live_mp4'):
+                height = int_or_none(self._search_regex(
+                    r'^(?:url|cache)(\d+)', format_id, 'height', default=None))
+                formats.append({
+                    'format_id': format_id,
+                    'url': format_url,
+                    'height': height,
+                })
+            elif format_id == 'hls':
+                formats.extend(self._extract_m3u8_formats(
+                    format_url, video_id, 'mp4', m3u8_id=format_id,
+                    fatal=False, live=True))
+            elif format_id == 'rtmp':
+                formats.append({
+                    'format_id': format_id,
+                    'url': format_url,
+                    'ext': 'flv',
+                })
         self._sort_formats(formats)
 
         return {
-            'id': compat_str(data['vid']),
+            'id': compat_str(data.get('vid') or video_id),
             'formats': formats,
-            'title': unescapeHTML(data['md_title']),
+            'title': title,
             'thumbnail': data.get('jpg'),
             'uploader': data.get('md_author'),
             'duration': data.get('duration'),

From efa97bdcf1f1e90d1b51a09324d7869dcd70729b Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 30 Sep 2016 00:28:32 +0800
Subject: [PATCH 6190/6219] Move write_xattr to utils.py

There are some other places that use xattr functions. It's better to
move it to a common place so that others can use it.
---
 youtube_dl/postprocessor/xattrpp.py | 114 ++--------------------------
 youtube_dl/utils.py                 |  99 ++++++++++++++++++++++++
 2 files changed, 106 insertions(+), 107 deletions(-)

diff --git a/youtube_dl/postprocessor/xattrpp.py b/youtube_dl/postprocessor/xattrpp.py
index e39ca60aa..fbdfa02ac 100644
--- a/youtube_dl/postprocessor/xattrpp.py
+++ b/youtube_dl/postprocessor/xattrpp.py
@@ -1,37 +1,15 @@
 from __future__ import unicode_literals
 
-import os
-import subprocess
-import sys
-import errno
-
 from .common import PostProcessor
 from ..compat import compat_os_name
 from ..utils import (
-    check_executable,
     hyphenate_date,
-    version_tuple,
-    PostProcessingError,
-    encodeArgument,
-    encodeFilename,
+    write_xattr,
+    XAttrMetadataError,
+    XAttrUnavailableError,
 )
 
 
-class XAttrMetadataError(PostProcessingError):
-    def __init__(self, code=None, msg='Unknown error'):
-        super(XAttrMetadataError, self).__init__(msg)
-        self.code = code
-
-        # Parsing code and msg
-        if (self.code in (errno.ENOSPC, errno.EDQUOT) or
-                'No space left' in self.msg or 'Disk quota excedded' in self.msg):
-            self.reason = 'NO_SPACE'
-        elif self.code == errno.E2BIG or 'Argument list too long' in self.msg:
-            self.reason = 'VALUE_TOO_LONG'
-        else:
-            self.reason = 'NOT_SUPPORTED'
-
-
 class XAttrMetadataPP(PostProcessor):
 
     #
@@ -48,88 +26,6 @@ class XAttrMetadataPP(PostProcessor):
     def run(self, info):
         """ Set extended attributes on downloaded file (if xattr support is found). """
 
-        # This mess below finds the best xattr tool for the job and creates a
-        # "write_xattr" function.
-        try:
-            # try the pyxattr module...
-            import xattr
-
-            # Unicode arguments are not supported in python-pyxattr until
-            # version 0.5.0
-            # See https://github.com/rg3/youtube-dl/issues/5498
-            pyxattr_required_version = '0.5.0'
-            if version_tuple(xattr.__version__) < version_tuple(pyxattr_required_version):
-                self._downloader.report_warning(
-                    'python-pyxattr is detected but is too old. '
-                    'youtube-dl requires %s or above while your version is %s. '
-                    'Falling back to other xattr implementations' % (
-                        pyxattr_required_version, xattr.__version__))
-
-                raise ImportError
-
-            def write_xattr(path, key, value):
-                try:
-                    xattr.set(path, key, value)
-                except EnvironmentError as e:
-                    raise XAttrMetadataError(e.errno, e.strerror)
-
-        except ImportError:
-            if compat_os_name == 'nt':
-                # Write xattrs to NTFS Alternate Data Streams:
-                # http://en.wikipedia.org/wiki/NTFS#Alternate_data_streams_.28ADS.29
-                def write_xattr(path, key, value):
-                    assert ':' not in key
-                    assert os.path.exists(path)
-
-                    ads_fn = path + ':' + key
-                    try:
-                        with open(ads_fn, 'wb') as f:
-                            f.write(value)
-                    except EnvironmentError as e:
-                        raise XAttrMetadataError(e.errno, e.strerror)
-            else:
-                user_has_setfattr = check_executable('setfattr', ['--version'])
-                user_has_xattr = check_executable('xattr', ['-h'])
-
-                if user_has_setfattr or user_has_xattr:
-
-                    def write_xattr(path, key, value):
-                        value = value.decode('utf-8')
-                        if user_has_setfattr:
-                            executable = 'setfattr'
-                            opts = ['-n', key, '-v', value]
-                        elif user_has_xattr:
-                            executable = 'xattr'
-                            opts = ['-w', key, value]
-
-                        cmd = ([encodeFilename(executable, True)] +
-                               [encodeArgument(o) for o in opts] +
-                               [encodeFilename(path, True)])
-
-                        try:
-                            p = subprocess.Popen(
-                                cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
-                        except EnvironmentError as e:
-                            raise XAttrMetadataError(e.errno, e.strerror)
-                        stdout, stderr = p.communicate()
-                        stderr = stderr.decode('utf-8', 'replace')
-                        if p.returncode != 0:
-                            raise XAttrMetadataError(p.returncode, stderr)
-
-                else:
-                    # On Unix, and can't find pyxattr, setfattr, or xattr.
-                    if sys.platform.startswith('linux'):
-                        self._downloader.report_error(
-                            "Couldn't find a tool to set the xattrs. "
-                            "Install either the python 'pyxattr' or 'xattr' "
-                            "modules, or the GNU 'attr' package "
-                            "(which contains the 'setfattr' tool).")
-                    else:
-                        self._downloader.report_error(
-                            "Couldn't find a tool to set the xattrs. "
-                            "Install either the python 'xattr' module, "
-                            "or the 'xattr' binary.")
-
         # Write the metadata to the file's xattrs
         self._downloader.to_screen('[metadata] Writing metadata to file\'s xattrs')
 
@@ -159,6 +55,10 @@ class XAttrMetadataPP(PostProcessor):
 
             return [], info
 
+        except XAttrUnavailableError as e:
+            self._downloader.report_error(str(e))
+            return [], info
+
         except XAttrMetadataError as e:
             if e.reason == 'NO_SPACE':
                 self._downloader.report_warning(
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 69ca88c85..fcbfa0d76 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -42,6 +42,7 @@ from .compat import (
     compat_html_entities_html5,
     compat_http_client,
     compat_kwargs,
+    compat_os_name,
     compat_parse_qs,
     compat_shlex_quote,
     compat_socket_create_connection,
@@ -775,6 +776,25 @@ class ContentTooShortError(Exception):
         self.expected = expected
 
 
+class XAttrMetadataError(Exception):
+    def __init__(self, code=None, msg='Unknown error'):
+        super(XAttrMetadataError, self).__init__(msg)
+        self.code = code
+
+        # Parsing code and msg
+        if (self.code in (errno.ENOSPC, errno.EDQUOT) or
+                'No space left' in self.msg or 'Disk quota excedded' in self.msg):
+            self.reason = 'NO_SPACE'
+        elif self.code == errno.E2BIG or 'Argument list too long' in self.msg:
+            self.reason = 'VALUE_TOO_LONG'
+        else:
+            self.reason = 'NOT_SUPPORTED'
+
+
+class XAttrUnavailableError(Exception):
+    pass
+
+
 def _create_http_connection(ydl_handler, http_class, is_https, *args, **kwargs):
     # Working around python 2 bug (see http://bugs.python.org/issue17849) by limiting
     # expected HTTP responses to meet HTTP/1.0 or later (see also
@@ -3131,3 +3151,82 @@ def decode_png(png_data):
             current_row.append(color)
 
     return width, height, pixels
+
+
+def write_xattr(path, key, value):
+    # This mess below finds the best xattr tool for the job
+    try:
+        # try the pyxattr module...
+        import xattr
+
+        # Unicode arguments are not supported in python-pyxattr until
+        # version 0.5.0
+        # See https://github.com/rg3/youtube-dl/issues/5498
+        pyxattr_required_version = '0.5.0'
+        if version_tuple(xattr.__version__) < version_tuple(pyxattr_required_version):
+            # TODO: fallback to CLI tools
+            raise XAttrUnavailableError(
+                'python-pyxattr is detected but is too old. '
+                'youtube-dl requires %s or above while your version is %s. '
+                'Falling back to other xattr implementations' % (
+                    pyxattr_required_version, xattr.__version__))
+
+        try:
+            xattr.set(path, key, value)
+        except EnvironmentError as e:
+            raise XAttrMetadataError(e.errno, e.strerror)
+
+    except ImportError:
+        if compat_os_name == 'nt':
+            # Write xattrs to NTFS Alternate Data Streams:
+            # http://en.wikipedia.org/wiki/NTFS#Alternate_data_streams_.28ADS.29
+            assert ':' not in key
+            assert os.path.exists(path)
+
+            ads_fn = path + ':' + key
+            try:
+                with open(ads_fn, 'wb') as f:
+                    f.write(value)
+            except EnvironmentError as e:
+                raise XAttrMetadataError(e.errno, e.strerror)
+        else:
+            user_has_setfattr = check_executable('setfattr', ['--version'])
+            user_has_xattr = check_executable('xattr', ['-h'])
+
+            if user_has_setfattr or user_has_xattr:
+
+                value = value.decode('utf-8')
+                if user_has_setfattr:
+                    executable = 'setfattr'
+                    opts = ['-n', key, '-v', value]
+                elif user_has_xattr:
+                    executable = 'xattr'
+                    opts = ['-w', key, value]
+
+                cmd = ([encodeFilename(executable, True)] +
+                       [encodeArgument(o) for o in opts] +
+                       [encodeFilename(path, True)])
+
+                try:
+                    p = subprocess.Popen(
+                        cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
+                except EnvironmentError as e:
+                    raise XAttrMetadataError(e.errno, e.strerror)
+                stdout, stderr = p.communicate()
+                stderr = stderr.decode('utf-8', 'replace')
+                if p.returncode != 0:
+                    raise XAttrMetadataError(p.returncode, stderr)
+
+            else:
+                # On Unix, and can't find pyxattr, setfattr, or xattr.
+                if sys.platform.startswith('linux'):
+                    raise XAttrUnavailableError(
+                        "Couldn't find a tool to set the xattrs. "
+                        "Install either the python 'pyxattr' or 'xattr' "
+                        "modules, or the GNU 'attr' package "
+                        "(which contains the 'setfattr' tool).")
+                else:
+                    raise XAttrUnavailableError(
+                        "Couldn't find a tool to set the xattrs. "
+                        "Install either the python 'xattr' module, "
+                        "or the 'xattr' binary.")

From 3aa3953d28dae68b87aa83682043b5eec0973ddc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kacper=20Michaj=C5=82ow?= <kasper93@gmail.com>
Date: Sun, 25 Sep 2016 20:26:58 +0200
Subject: [PATCH 6191/6219] [vk] Fix date and view count extraction.

---
 youtube_dl/extractor/vk.py | 17 +++++------------
 youtube_dl/utils.py        |  2 ++
 2 files changed, 7 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 9f7a593ef..3cfbd97af 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -373,19 +373,12 @@ class VKIE(VKBaseIE):
             title = self._live_title(title)
 
         # Extract upload date
-        upload_date = None
-        mobj = re.search(r'id="mv_date(?:_views)?_wrap"[^>]*>([a-zA-Z]+ [0-9]+), ([0-9]+) at', info_page)
-        if mobj is not None:
-            mobj.group(1) + ' ' + mobj.group(2)
-            upload_date = unified_strdate(mobj.group(1) + ' ' + mobj.group(2))
+        upload_date = unified_strdate(self._html_search_regex(
+            r'class="mv_info_date[^>]*>([^<]*)<', info_page, 'upload date', default=None))
 
-        view_count = None
-        views = self._html_search_regex(
-            r'"mv_views_count_number"[^>]*>(.+?\bviews?)<',
-            info_page, 'view count', default=None)
-        if views:
-            view_count = str_to_int(self._search_regex(
-                r'([\d,.]+)', views, 'view count', fatal=False))
+        view_count = str_to_int(self._html_search_regex(
+            r'class="mv_views_count[^>]*>([\d,.]+)',
+            info_page, 'view count', default=None))
 
         formats = []
         for format_id, format_url in data.items():
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index fcbfa0d76..243d09034 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -124,6 +124,8 @@ DATE_FORMATS = (
     '%d %b %Y',
     '%B %d %Y',
     '%b %d %Y',
+    '%b %d %Y at %H:%M',
+    '%b %d %Y at %H:%M:%S',
     '%b %dst %Y %I:%M',
     '%b %dnd %Y %I:%M',
     '%b %dth %Y %I:%M',

From c6eed6b8c000672f0515d916dda54002c7fca356 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 29 Sep 2016 23:47:25 +0700
Subject: [PATCH 6192/6219] [utils] Lower priority for rare date formats and
 add tests

---
 test/test_utils.py  | 2 ++
 youtube_dl/utils.py | 4 ++--
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 9789d8611..b1b2effca 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -292,6 +292,7 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(unified_strdate('25-09-2014'), '20140925')
         self.assertEqual(unified_strdate('27.02.2016 17:30'), '20160227')
         self.assertEqual(unified_strdate('UNKNOWN DATE FORMAT'), None)
+        self.assertEqual(unified_strdate('Feb 7, 2016 at 6:35 pm'), '20160207')
 
     def test_unified_timestamps(self):
         self.assertEqual(unified_timestamp('December 21, 2010'), 1292889600)
@@ -312,6 +313,7 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(unified_timestamp('27.02.2016 17:30'), 1456594200)
         self.assertEqual(unified_timestamp('UNKNOWN DATE FORMAT'), None)
         self.assertEqual(unified_timestamp('May 16, 2016 11:15 PM'), 1463440500)
+        self.assertEqual(unified_timestamp('Feb 7, 2016 at 6:35 pm'), 1454870100)
 
     def test_determine_ext(self):
         self.assertEqual(determine_ext('http://example.com/foo/bar.mp4/?download'), 'mp4')
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 243d09034..d2dfa8013 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -124,8 +124,6 @@ DATE_FORMATS = (
     '%d %b %Y',
     '%B %d %Y',
     '%b %d %Y',
-    '%b %d %Y at %H:%M',
-    '%b %d %Y at %H:%M:%S',
     '%b %dst %Y %I:%M',
     '%b %dnd %Y %I:%M',
     '%b %dth %Y %I:%M',
@@ -144,6 +142,8 @@ DATE_FORMATS = (
     '%Y-%m-%dT%H:%M:%S',
     '%Y-%m-%dT%H:%M:%S.%f',
     '%Y-%m-%dT%H:%M',
+    '%b %d %Y at %H:%M',
+    '%b %d %Y at %H:%M:%S',
 )
 
 DATE_FORMATS_DAY_FIRST = list(DATE_FORMATS)

From a7ee8a00f4af9853d06ed895c5023cc6b573fd57 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 29 Sep 2016 23:48:21 +0700
Subject: [PATCH 6193/6219] [vk] Extract timestamp (Closes #10760)

---
 youtube_dl/extractor/vk.py | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 3cfbd97af..77f5cebcf 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -20,7 +20,7 @@ from ..utils import (
     remove_start,
     str_to_int,
     unescapeHTML,
-    unified_strdate,
+    unified_timestamp,
     urlencode_postdata,
 )
 from .dailymotion import DailymotionIE
@@ -106,6 +106,7 @@ class VKIE(VKBaseIE):
                 'title': 'ProtivoGunz - Хуёвая песня',
                 'uploader': 're:(?:Noize MC|Alexander Ilyashenko).*',
                 'duration': 195,
+                'timestamp': 1329060660,
                 'upload_date': '20120212',
                 'view_count': int,
             },
@@ -119,6 +120,7 @@ class VKIE(VKBaseIE):
                 'uploader': 'Tom Cruise',
                 'title': 'No name',
                 'duration': 9,
+                'timestamp': 1374374880,
                 'upload_date': '20130721',
                 'view_count': int,
             }
@@ -195,6 +197,7 @@ class VKIE(VKBaseIE):
                 'upload_date': '20150709',
                 'view_count': int,
             },
+            'skip': 'Removed',
         },
         {
             # youtube embed
@@ -237,6 +240,7 @@ class VKIE(VKBaseIE):
                 'ext': 'mp4',
                 'title': 'S-Dance, репетиции к The way show',
                 'uploader': 'THE WAY SHOW | 17 апреля',
+                'timestamp': 1454870100,
                 'upload_date': '20160207',
                 'view_count': int,
             },
@@ -373,8 +377,9 @@ class VKIE(VKBaseIE):
             title = self._live_title(title)
 
         # Extract upload date
-        upload_date = unified_strdate(self._html_search_regex(
-            r'class="mv_info_date[^>]*>([^<]*)<', info_page, 'upload date', default=None))
+        timestamp = unified_timestamp(self._html_search_regex(
+            r'class=["\']mv_info_date[^>]*>([^<]+)(?:<|from)', info_page,
+            'upload date', fatal=False))
 
         view_count = str_to_int(self._html_search_regex(
             r'class="mv_views_count[^>]*>([\d,.]+)',
@@ -411,7 +416,7 @@ class VKIE(VKBaseIE):
             'thumbnail': data.get('jpg'),
             'uploader': data.get('md_author'),
             'duration': data.get('duration'),
-            'upload_date': upload_date,
+            'timestamp': timestamp,
             'view_count': view_count,
         }
 

From 70d7b323b6556eb693bec43a1eb10ded889184b0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 29 Sep 2016 23:51:52 +0700
Subject: [PATCH 6194/6219] [vk] Improve view count extraction

---
 youtube_dl/extractor/vk.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 77f5cebcf..58799d413 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -255,6 +255,7 @@ class VKIE(VKBaseIE):
                 'title': 'ИгроМир 2016 — день 1',
                 'uploader': 'Игромания',
                 'duration': 5239,
+                'view_count': int,
             },
         },
         {
@@ -376,14 +377,13 @@ class VKIE(VKBaseIE):
         if data.get('live') == 2:
             title = self._live_title(title)
 
-        # Extract upload date
         timestamp = unified_timestamp(self._html_search_regex(
-            r'class=["\']mv_info_date[^>]*>([^<]+)(?:<|from)', info_page,
+            r'class=["\']mv_info_date[^>]+>([^<]+)(?:<|from)', info_page,
             'upload date', fatal=False))
 
-        view_count = str_to_int(self._html_search_regex(
-            r'class="mv_views_count[^>]*>([\d,.]+)',
-            info_page, 'view count', default=None))
+        view_count = str_to_int(self._search_regex(
+            r'class=["\']mv_views_count[^>]+>\s*([\d,.]+)',
+            info_page, 'view count', fatal=False))
 
         formats = []
         for format_id, format_url in data.items():

From af33dd8ee7da49b5daf1582b2870deaa5427444b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 30 Sep 2016 00:13:03 +0700
Subject: [PATCH 6195/6219] [aftonbladet] Remove extractor

---
 youtube_dl/extractor/aftonbladet.py | 64 -----------------------------
 youtube_dl/extractor/extractors.py  |  1 -
 2 files changed, 65 deletions(-)
 delete mode 100644 youtube_dl/extractor/aftonbladet.py

diff --git a/youtube_dl/extractor/aftonbladet.py b/youtube_dl/extractor/aftonbladet.py
deleted file mode 100644
index 5766b4fe8..000000000
--- a/youtube_dl/extractor/aftonbladet.py
+++ /dev/null
@@ -1,64 +0,0 @@
-# encoding: utf-8
-from __future__ import unicode_literals
-
-from .common import InfoExtractor
-from ..utils import int_or_none
-
-
-class AftonbladetIE(InfoExtractor):
-    _VALID_URL = r'https?://tv\.aftonbladet\.se/abtv/articles/(?P<id>[0-9]+)'
-    _TEST = {
-        'url': 'http://tv.aftonbladet.se/abtv/articles/36015',
-        'info_dict': {
-            'id': '36015',
-            'ext': 'mp4',
-            'title': 'Vulkanutbrott i rymden - nu släpper NASA bilderna',
-            'description': 'Jupiters måne mest aktiv av alla himlakroppar',
-            'timestamp': 1394142732,
-            'upload_date': '20140306',
-        },
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-
-        # find internal video meta data
-        meta_url = 'http://aftonbladet-play-metadata.cdn.drvideo.aptoma.no/video/%s.json'
-        player_config = self._parse_json(self._html_search_regex(
-            r'data-player-config="([^"]+)"', webpage, 'player config'), video_id)
-        internal_meta_id = player_config['aptomaVideoId']
-        internal_meta_url = meta_url % internal_meta_id
-        internal_meta_json = self._download_json(
-            internal_meta_url, video_id, 'Downloading video meta data')
-
-        # find internal video formats
-        format_url = 'http://aftonbladet-play.videodata.drvideo.aptoma.no/actions/video/?id=%s'
-        internal_video_id = internal_meta_json['videoId']
-        internal_formats_url = format_url % internal_video_id
-        internal_formats_json = self._download_json(
-            internal_formats_url, video_id, 'Downloading video formats')
-
-        formats = []
-        for fmt in internal_formats_json['formats']['http']['pseudostreaming']['mp4']:
-            p = fmt['paths'][0]
-            formats.append({
-                'url': 'http://%s:%d/%s/%s' % (p['address'], p['port'], p['path'], p['filename']),
-                'ext': 'mp4',
-                'width': int_or_none(fmt.get('width')),
-                'height': int_or_none(fmt.get('height')),
-                'tbr': int_or_none(fmt.get('bitrate')),
-                'protocol': 'http',
-            })
-        self._sort_formats(formats)
-
-        return {
-            'id': video_id,
-            'title': internal_meta_json['title'],
-            'formats': formats,
-            'thumbnail': internal_meta_json.get('imageUrl'),
-            'description': internal_meta_json.get('shortPreamble'),
-            'timestamp': int_or_none(internal_meta_json.get('timePublished')),
-            'duration': int_or_none(internal_meta_json.get('duration')),
-            'view_count': int_or_none(internal_meta_json.get('views')),
-        }
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 23fd2a308..09b3b4942 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -31,7 +31,6 @@ from .aenetworks import (
     HistoryTopicIE,
 )
 from .afreecatv import AfreecaTVIE
-from .aftonbladet import AftonbladetIE
 from .airmozilla import AirMozillaIE
 from .aljazeera import AlJazeeraIE
 from .alphaporno import AlphaPornoIE

From b0582fc80615ec94c37e14015bd9bbfef6745aa4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 30 Sep 2016 00:15:09 +0700
Subject: [PATCH 6196/6219] [vgtv] Add support for tv.aftonbladet.se (Closes
 #10800)

---
 youtube_dl/extractor/vgtv.py | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vgtv.py b/youtube_dl/extractor/vgtv.py
index 185756301..3b38ac700 100644
--- a/youtube_dl/extractor/vgtv.py
+++ b/youtube_dl/extractor/vgtv.py
@@ -22,6 +22,7 @@ class VGTVIE(XstreamIE):
         'fvn.no/fvntv': 'fvntv',
         'aftenposten.no/webtv': 'aptv',
         'ap.vgtv.no/webtv': 'aptv',
+        'tv.aftonbladet.se/abtv': 'abtv',
     }
 
     _APP_NAME_TO_VENDOR = {
@@ -30,6 +31,7 @@ class VGTVIE(XstreamIE):
         'satv': 'sa',
         'fvntv': 'fvn',
         'aptv': 'ap',
+        'abtv': 'ab',
     }
 
     _VALID_URL = r'''(?x)
@@ -40,7 +42,8 @@ class VGTVIE(XstreamIE):
                     /?
                     (?:
                         \#!/(?:video|live)/|
-                        embed?.*id=
+                        embed?.*id=|
+                        articles/
                     )|
                     (?P<appname>
                         %s
@@ -135,6 +138,14 @@ class VGTVIE(XstreamIE):
             'url': 'http://www.vgtv.no/#!/video/127205/inside-the-mind-of-favela-funk',
             'only_matching': True,
         },
+        {
+            'url': 'http://tv.aftonbladet.se/abtv/articles/36015',
+            'only_matching': True,
+        },
+        {
+            'url': 'abtv:140026',
+            'only_matching': True,
+        }
     ]
 
     def _real_extract(self, url):

From de6babf92252ea5828a9c17d76766357cff3e440 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 30 Sep 2016 22:30:34 +0700
Subject: [PATCH 6197/6219] [tvland] Extend _VALID_URL (Closes #10812)

---
 youtube_dl/extractor/tvland.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tvland.py b/youtube_dl/extractor/tvland.py
index cb76a2a58..957cf1ea2 100644
--- a/youtube_dl/extractor/tvland.py
+++ b/youtube_dl/extractor/tvland.py
@@ -6,7 +6,7 @@ from .mtv import MTVServicesInfoExtractor
 
 class TVLandIE(MTVServicesInfoExtractor):
     IE_NAME = 'tvland.com'
-    _VALID_URL = r'https?://(?:www\.)?tvland\.com/(?:video-clips|episodes)/(?P<id>[^/?#.]+)'
+    _VALID_URL = r'https?://(?:www\.)?tvland\.com/(?:video-clips|(?:full-)?episodes)/(?P<id>[^/?#.]+)'
     _FEED_URL = 'http://www.tvland.com/feeds/mrss/'
     _TESTS = [{
         # Geo-restricted. Without a proxy metadata are still there. With a
@@ -28,4 +28,7 @@ class TVLandIE(MTVServicesInfoExtractor):
             'upload_date': '20151228',
             'timestamp': 1451289600,
         },
+    }, {
+        'url': 'http://www.tvland.com/full-episodes/iu0hz6/younger-a-kiss-is-just-a-kiss-season-3-ep-301',
+        'only_matching': True,
     }]

From 16097822582b839a3744b54af90f7b3fd7132d26 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?D=C3=A9stin=20Reed?= <trox1972@users.noreply.github.com>
Date: Wed, 28 Sep 2016 17:28:16 +0200
Subject: [PATCH 6198/6219] [Instagram] Extract video dimensions

---
 youtube_dl/extractor/instagram.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/instagram.py b/youtube_dl/extractor/instagram.py
index 5ebc30a10..dde435189 100644
--- a/youtube_dl/extractor/instagram.py
+++ b/youtube_dl/extractor/instagram.py
@@ -84,7 +84,7 @@ class InstagramIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         (video_url, description, thumbnail, timestamp, uploader,
-         uploader_id, like_count, comment_count) = [None] * 8
+         uploader_id, like_count, comment_count, height, width) = [None] * 10
 
         shared_data = self._parse_json(
             self._search_regex(
@@ -96,6 +96,8 @@ class InstagramIE(InfoExtractor):
                 shared_data, lambda x: x['entry_data']['PostPage'][0]['media'], dict)
             if media:
                 video_url = media.get('video_url')
+                height = int_or_none(media.get('dimensions', {}).get('height'))
+                width = int_or_none(media.get('dimensions', {}).get('width'))
                 description = media.get('caption')
                 thumbnail = media.get('display_src')
                 timestamp = int_or_none(media.get('date'))
@@ -115,6 +117,12 @@ class InstagramIE(InfoExtractor):
         if not video_url:
             video_url = self._og_search_video_url(webpage, secure=False)
 
+        formats = [{
+            'url': video_url,
+            'width': width,
+            'height': height,
+        }]
+
         if not uploader_id:
             uploader_id = self._search_regex(
                 r'"owner"\s*:\s*{\s*"username"\s*:\s*"(.+?)"',
@@ -131,7 +139,7 @@ class InstagramIE(InfoExtractor):
 
         return {
             'id': video_id,
-            'url': video_url,
+            'formats': formats,
             'ext': 'mp4',
             'title': 'Video by %s' % uploader_id,
             'description': description,

From a1001f47fc19adf983859bb281f08a09bd7f7e9b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 1 Oct 2016 00:16:08 +0700
Subject: [PATCH 6199/6219] [instagram] PEP 8

---
 youtube_dl/extractor/instagram.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/instagram.py b/youtube_dl/extractor/instagram.py
index dde435189..196407b06 100644
--- a/youtube_dl/extractor/instagram.py
+++ b/youtube_dl/extractor/instagram.py
@@ -111,8 +111,8 @@ class InstagramIE(InfoExtractor):
                     'id': comment.get('id'),
                     'text': comment.get('text'),
                     'timestamp': int_or_none(comment.get('created_at')),
-                } for comment in media.get('comments', {}).get('nodes', [])
-                if comment.get('text')]
+                } for comment in media.get(
+                    'comments', {}).get('nodes', []) if comment.get('text')]
 
         if not video_url:
             video_url = self._og_search_video_url(webpage, secure=False)

From eaf9b22f94f37487d75457423a9a293dee1b1d32 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?D=C3=A9stin=20Reed?= <trox1972@users.noreply.github.com>
Date: Fri, 30 Sep 2016 20:03:25 +0200
Subject: [PATCH 6200/6219] [clubic] Rely on _match_id and _parse_json

---
 youtube_dl/extractor/clubic.py | 11 +++--------
 1 file changed, 3 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/clubic.py b/youtube_dl/extractor/clubic.py
index 2fba93543..f7ee3a8f8 100644
--- a/youtube_dl/extractor/clubic.py
+++ b/youtube_dl/extractor/clubic.py
@@ -1,9 +1,6 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import json
-import re
-
 from .common import InfoExtractor
 from ..utils import (
     clean_html,
@@ -30,16 +27,14 @@ class ClubicIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         player_url = 'http://player.m6web.fr/v1/player/clubic/%s.html' % video_id
         player_page = self._download_webpage(player_url, video_id)
 
-        config_json = self._search_regex(
+        config = self._parse_json(self._search_regex(
             r'(?m)M6\.Player\.config\s*=\s*(\{.+?\});$', player_page,
-            'configuration')
-        config = json.loads(config_json)
+            'configuration'), video_id)
 
         video_info = config['videoInfo']
         sources = config['sources']

From d7753d194803086d97ffe47f022c47c906ebcc71 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 30 Sep 2016 00:49:14 +0800
Subject: [PATCH 6201/6219] [downloader/http] Use write_xattr function for
 --xattr-set-filesize

---
 youtube_dl/__init__.py        | 6 ------
 youtube_dl/downloader/http.py | 8 +++++---
 2 files changed, 5 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 1cf3140a0..72141b983 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -283,12 +283,6 @@ def _real_main(argv=None):
             'key': 'ExecAfterDownload',
             'exec_cmd': opts.exec_cmd,
         })
-    if opts.xattr_set_filesize:
-        try:
-            import xattr
-            xattr  # Confuse flake8
-        except ImportError:
-            parser.error('setting filesize xattr requested but python-xattr is not available')
     external_downloader_args = None
     if opts.external_downloader_args:
         external_downloader_args = compat_shlex_split(opts.external_downloader_args)
diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index f8b69d186..11294d106 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -13,6 +13,9 @@ from ..utils import (
     encodeFilename,
     sanitize_open,
     sanitized_Request,
+    write_xattr,
+    XAttrMetadataError,
+    XAttrUnavailableError,
 )
 
 
@@ -179,9 +182,8 @@ class HttpFD(FileDownloader):
 
                 if self.params.get('xattr_set_filesize', False) and data_len is not None:
                     try:
-                        import xattr
-                        xattr.setxattr(tmpfilename, 'user.ytdl.filesize', str(data_len))
-                    except(OSError, IOError, ImportError) as err:
+                        write_xattr(tmpfilename, 'user.ytdl.filesize', str(data_len))
+                    except (XAttrUnavailableError, XAttrMetadataError) as err:
                         self.report_error('unable to set filesize xattr: %s' % str(err))
 
             try:

From e295618f9e1c1fc404d9baa4ccef961d3eb3ea88 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 1 Oct 2016 15:22:48 +0800
Subject: [PATCH 6202/6219] [dctp] Fix extraction (closes #10734)

---
 ChangeLog                    |  1 +
 youtube_dl/extractor/dctp.py | 67 ++++++++++++++++--------------------
 2 files changed, 31 insertions(+), 37 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 70da55c90..efc3e494e 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
 version <unreleased>
 
 Extractors
+* [dctp] Fix extraction (#10734)
 + [leeco] Recognize more Le Sports URLs (#10794)
 
 
diff --git a/youtube_dl/extractor/dctp.py b/youtube_dl/extractor/dctp.py
index a47e04993..14ba88715 100644
--- a/youtube_dl/extractor/dctp.py
+++ b/youtube_dl/extractor/dctp.py
@@ -1,61 +1,54 @@
-# encoding: utf-8
+# coding: utf-8
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import compat_str
+from ..utils import unified_strdate
 
 
 class DctpTvIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?dctp\.tv/(#/)?filme/(?P<id>.+?)/$'
     _TEST = {
         'url': 'http://www.dctp.tv/filme/videoinstallation-fuer-eine-kaufhausfassade/',
+        'md5': '174dd4a8a6225cf5655952f969cfbe24',
         'info_dict': {
-            'id': '1324',
+            'id': '95eaa4f33dad413aa17b4ee613cccc6c',
             'display_id': 'videoinstallation-fuer-eine-kaufhausfassade',
-            'ext': 'flv',
-            'title': 'Videoinstallation für eine Kaufhausfassade'
+            'ext': 'mp4',
+            'title': 'Videoinstallation für eine Kaufhausfassade',
+            'description': 'Kurzfilm',
+            'upload_date': '20110407',
+            'thumbnail': 're:^https?://.*\.jpg$',
         },
-        'params': {
-            # rtmp download
-            'skip_download': True,
-        }
     }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        base_url = 'http://dctp-ivms2-restapi.s3.amazonaws.com/'
-        version_json = self._download_json(
-            base_url + 'version.json',
-            video_id, note='Determining file version')
-        version = version_json['version_name']
-        info_json = self._download_json(
-            '{0}{1}/restapi/slugs/{2}.json'.format(base_url, version, video_id),
-            video_id, note='Fetching object ID')
-        object_id = compat_str(info_json['object_id'])
-        meta_json = self._download_json(
-            '{0}{1}/restapi/media/{2}.json'.format(base_url, version, object_id),
-            video_id, note='Downloading metadata')
-        uuid = meta_json['uuid']
-        title = meta_json['title']
-        wide = meta_json['is_wide']
-        if wide:
-            ratio = '16x9'
-        else:
-            ratio = '4x3'
-        play_path = 'mp4:{0}_dctp_0500_{1}.m4v'.format(uuid, ratio)
+        webpage = self._download_webpage(url, video_id)
+
+        object_id = self._html_search_meta('DC.identifier', webpage)
 
         servers_json = self._download_json(
-            'http://www.dctp.tv/streaming_servers/',
+            'http://www.dctp.tv/elastic_streaming_client/get_streaming_server/',
             video_id, note='Downloading server list')
-        url = servers_json[0]['endpoint']
+        server = servers_json[0]['server']
+        m3u8_path = self._search_regex(
+            r'\'([^\'"]+/playlist\.m3u8)"', webpage, 'm3u8 path')
+        formats = self._extract_m3u8_formats(
+            'http://%s%s' % (server, m3u8_path), video_id, ext='mp4',
+            entry_protocol='m3u8_native')
+
+        title = self._og_search_title(webpage)
+        description = self._html_search_meta('DC.description', webpage)
+        upload_date = unified_strdate(
+            self._html_search_meta('DC.date.created', webpage))
+        thumbnail = self._og_search_thumbnail(webpage)
 
         return {
             'id': object_id,
             'title': title,
-            'format': 'rtmp',
-            'url': url,
-            'play_path': play_path,
-            'rtmp_real_time': True,
-            'ext': 'flv',
-            'display_id': video_id
+            'formats': formats,
+            'display_id': video_id,
+            'description': description,
+            'upload_date': upload_date,
+            'thumbnail': thumbnail,
         }

From 9bd7bd0b8054231adbeb2a0eddd42a0b969fd6c4 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 1 Oct 2016 16:37:49 +0800
Subject: [PATCH 6203/6219] [twitch] Skip a 404 test

---
 youtube_dl/extractor/twitch.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index bc352391e..46c2cfe7b 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -247,6 +247,7 @@ class TwitchVodIE(TwitchItemBaseIE):
             # m3u8 download
             'skip_download': True,
         },
+        'skip': 'HTTP Error 404: Not Found',
     }]
 
     def _real_extract(self, url):

From 9c51a2464276f5eb26d1b571d32052df55d6ead9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?D=C3=A9stin=20Reed?= <trox1972@users.noreply.github.com>
Date: Fri, 30 Sep 2016 20:06:08 +0200
Subject: [PATCH 6204/6219] [criterion] Rely on _match_id, improve regex and
 add thumbnail to test

---
 youtube_dl/extractor/criterion.py | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/criterion.py b/youtube_dl/extractor/criterion.py
index ad32673a8..cf6a5d6cb 100644
--- a/youtube_dl/extractor/criterion.py
+++ b/youtube_dl/extractor/criterion.py
@@ -1,8 +1,6 @@
-# -*- coding: utf-8 -*-
+# coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 
 
@@ -16,20 +14,20 @@ class CriterionIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Le Samouraï',
             'description': 'md5:a2b4b116326558149bef81f76dcbb93f',
+            'thumbnail': 're:^https?://.*\.jpg$',
         }
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
         final_url = self._search_regex(
-            r'so.addVariable\("videoURL", "(.+?)"\)\;', webpage, 'video url')
+            r'so\.addVariable\("videoURL", "(.+?)"\)\;', webpage, 'video url')
         title = self._og_search_title(webpage)
         description = self._html_search_meta('description', webpage)
         thumbnail = self._search_regex(
-            r'so.addVariable\("thumbnailURL", "(.+?)"\)\;',
+            r'so\.addVariable\("thumbnailURL", "(.+?)"\)\;',
             webpage, 'thumbnail url')
 
         return {

From d54739a2e6a8dc089e7530afda0a1cfe355a6fef Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 1 Oct 2016 19:58:13 +0800
Subject: [PATCH 6205/6219] [downloader/http] xattr values should be bytes

---
 youtube_dl/downloader/http.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index 11294d106..af405b950 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -182,7 +182,7 @@ class HttpFD(FileDownloader):
 
                 if self.params.get('xattr_set_filesize', False) and data_len is not None:
                     try:
-                        write_xattr(tmpfilename, 'user.ytdl.filesize', str(data_len))
+                        write_xattr(tmpfilename, 'user.ytdl.filesize', str(data_len).encode('utf-8'))
                     except (XAttrUnavailableError, XAttrMetadataError) as err:
                         self.report_error('unable to set filesize xattr: %s' % str(err))
 

From 53a7e3d2879feac7b1b6f714692581057b9b5f6b Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 1 Oct 2016 20:13:04 +0800
Subject: [PATCH 6206/6219] [utils] Support xattr as well as pyxattr

Closes #9054

There are two xattr packages in Python, pyxattr [1] and xattr [2]. They
have different APIs.

In old days pyxattr supports Linux only and xattr supports Linux, Mac,
FreeBSD and Solaris, and pyxattr supports Linux only. Recently pyxattr
adds support for Mac OS X. [3]

An old version of [2] is shipped with Mac OS X. However, some Linux
distributions have pyxattr only, for example PLD-Linux [4] and old Arch
Linux. [5] As a result, supporting both is the way to go.

[1] https://github.com/iustin/pyxattr
[2] https://github.com/xattr/xattr
[3] https://github.com/iustin/pyxattr/pull/9
[4] https://github.com/rg3/youtube-dl/issues/5498
[5] https://git.archlinux.org/svntogit/community.git/commit/?id=427c4c76401e386d865ccddea4fbfdc74df80492
    https://git.archlinux.org/svntogit/community.git/commit/?id=59b40da7b69622a6761d364a8b07909e9cccaa56
    python-xattr is added on 2016/06/29 while pyxattr is there for more
    than 6 years
---
 ChangeLog           |  4 ++++
 youtube_dl/utils.py | 29 +++++++++++++++++------------
 2 files changed, 21 insertions(+), 12 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index efc3e494e..8ef39cd63 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,9 @@
 version <unreleased>
 
+Core
++ Support pyxattr as well as python-xattr for --xattrs and
+  --xattr-set-filesize (#9054)
+
 Extractors
 * [dctp] Fix extraction (#10734)
 + [leeco] Recognize more Le Sports URLs (#10794)
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index d2dfa8013..c259f8bff 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -3161,20 +3161,25 @@ def write_xattr(path, key, value):
         # try the pyxattr module...
         import xattr
 
-        # Unicode arguments are not supported in python-pyxattr until
-        # version 0.5.0
-        # See https://github.com/rg3/youtube-dl/issues/5498
-        pyxattr_required_version = '0.5.0'
-        if version_tuple(xattr.__version__) < version_tuple(pyxattr_required_version):
-            # TODO: fallback to CLI tools
-            raise XAttrUnavailableError(
-                'python-pyxattr is detected but is too old. '
-                'youtube-dl requires %s or above while your version is %s. '
-                'Falling back to other xattr implementations' % (
-                    pyxattr_required_version, xattr.__version__))
+        if hasattr(xattr, 'set'):  # pyxattr
+            # Unicode arguments are not supported in python-pyxattr until
+            # version 0.5.0
+            # See https://github.com/rg3/youtube-dl/issues/5498
+            pyxattr_required_version = '0.5.0'
+            if version_tuple(xattr.__version__) < version_tuple(pyxattr_required_version):
+                # TODO: fallback to CLI tools
+                raise XAttrUnavailableError(
+                    'python-pyxattr is detected but is too old. '
+                    'youtube-dl requires %s or above while your version is %s. '
+                    'Falling back to other xattr implementations' % (
+                        pyxattr_required_version, xattr.__version__))
+
+            setxattr = xattr.set
+        else:  # xattr
+            setxattr = xattr.setxattr
 
         try:
-            xattr.set(path, key, value)
+            setxattr(path, key, value)
         except EnvironmentError as e:
             raise XAttrMetadataError(e.errno, e.strerror)
 

From e1e97c2446ab6fcffcfae738e0c7f29ff58a9dec Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 1 Oct 2016 22:50:47 +0700
Subject: [PATCH 6207/6219] [periscope:user] Fix extraction (Closes #10820)

---
 youtube_dl/extractor/periscope.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/periscope.py b/youtube_dl/extractor/periscope.py
index 61043cad5..0e3623024 100644
--- a/youtube_dl/extractor/periscope.py
+++ b/youtube_dl/extractor/periscope.py
@@ -132,7 +132,7 @@ class PeriscopeUserIE(PeriscopeBaseIE):
 
         user = list(data_store['UserCache']['users'].values())[0]['user']
         user_id = user['id']
-        session_id = data_store['SessionToken']['broadcastHistory']['token']['session_id']
+        session_id = data_store['SessionToken']['public']['broadcastHistory']['token']['session_id']
 
         broadcasts = self._call_api(
             'getUserBroadcastsPublic',

From 4da4516973b56bcaa65794a8ae0856cf54740c54 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?D=C3=A9stin=20Reed?= <trox1972@users.noreply.github.com>
Date: Fri, 30 Sep 2016 19:59:08 +0200
Subject: [PATCH 6208/6219] [byutv] Rely on _match_id and _parse_json

---
 youtube_dl/extractor/byutv.py | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/byutv.py b/youtube_dl/extractor/byutv.py
index 3aec601f8..b2d25eec0 100644
--- a/youtube_dl/extractor/byutv.py
+++ b/youtube_dl/extractor/byutv.py
@@ -1,6 +1,5 @@
 from __future__ import unicode_literals
 
-import json
 import re
 
 from .common import InfoExtractor
@@ -8,7 +7,7 @@ from ..utils import ExtractorError
 
 
 class BYUtvIE(InfoExtractor):
-    _VALID_URL = r'^https?://(?:www\.)?byutv.org/watch/[0-9a-f-]+/(?P<video_id>[^/?#]+)'
+    _VALID_URL = r'^https?://(?:www\.)?byutv.org/watch/[0-9a-f-]+/(?P<id>[^/?#]+)'
     _TEST = {
         'url': 'http://www.byutv.org/watch/6587b9a3-89d2-42a6-a7f7-fd2f81840a7d/studio-c-season-5-episode-5',
         'md5': '05850eb8c749e2ee05ad5a1c34668493',
@@ -27,15 +26,15 @@ class BYUtvIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('video_id')
+        video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id)
         episode_code = self._search_regex(
             r'(?s)episode:(.*?\}),\s*\n', webpage, 'episode information')
-        episode_json = re.sub(
-            r'(\n\s+)([a-zA-Z]+):\s+\'(.*?)\'', r'\1"\2": "\3"', episode_code)
-        ep = json.loads(episode_json)
+
+        ep = self._parse_json(
+            episode_code, video_id, transform_source=lambda s:
+            re.sub(r'(\n\s+)([a-zA-Z]+):\s+\'(.*?)\'', r'\1"\2": "\3"', s))
 
         if ep['providerType'] == 'Ooyala':
             return {

From 6d2549fb4f2a646b6b6898db5281cde669277626 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 2 Oct 2016 00:44:54 +0700
Subject: [PATCH 6209/6219] [byutv] Fix id and display id

---
 youtube_dl/extractor/byutv.py | 46 ++++++++++++++++++++---------------
 1 file changed, 26 insertions(+), 20 deletions(-)

diff --git a/youtube_dl/extractor/byutv.py b/youtube_dl/extractor/byutv.py
index b2d25eec0..084cc7ae2 100644
--- a/youtube_dl/extractor/byutv.py
+++ b/youtube_dl/extractor/byutv.py
@@ -7,15 +7,15 @@ from ..utils import ExtractorError
 
 
 class BYUtvIE(InfoExtractor):
-    _VALID_URL = r'^https?://(?:www\.)?byutv.org/watch/[0-9a-f-]+/(?P<id>[^/?#]+)'
-    _TEST = {
+    _VALID_URL = r'https?://(?:www\.)?byutv.org/watch/(?P<id>[0-9a-f-]+)(?:/(?P<display_id>[^/?#&]+))?'
+    _TESTS = [{
         'url': 'http://www.byutv.org/watch/6587b9a3-89d2-42a6-a7f7-fd2f81840a7d/studio-c-season-5-episode-5',
-        'md5': '05850eb8c749e2ee05ad5a1c34668493',
         'info_dict': {
-            'id': 'studio-c-season-5-episode-5',
+            'id': '6587b9a3-89d2-42a6-a7f7-fd2f81840a7d',
+            'display_id': 'studio-c-season-5-episode-5',
             'ext': 'mp4',
-            'description': 'md5:e07269172baff037f8e8bf9956bc9747',
             'title': 'Season 5 Episode 5',
+            'description': 'md5:e07269172baff037f8e8bf9956bc9747',
             'thumbnail': 're:^https?://.*\.jpg$',
             'duration': 1486.486,
         },
@@ -23,28 +23,34 @@ class BYUtvIE(InfoExtractor):
             'skip_download': True,
         },
         'add_ie': ['Ooyala'],
-    }
+    }, {
+        'url': 'http://www.byutv.org/watch/6587b9a3-89d2-42a6-a7f7-fd2f81840a7d',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        display_id = mobj.group('display_id') or video_id
 
-        webpage = self._download_webpage(url, video_id)
+        webpage = self._download_webpage(url, display_id)
         episode_code = self._search_regex(
             r'(?s)episode:(.*?\}),\s*\n', webpage, 'episode information')
 
         ep = self._parse_json(
-            episode_code, video_id, transform_source=lambda s:
+            episode_code, display_id, transform_source=lambda s:
             re.sub(r'(\n\s+)([a-zA-Z]+):\s+\'(.*?)\'', r'\1"\2": "\3"', s))
 
-        if ep['providerType'] == 'Ooyala':
-            return {
-                '_type': 'url_transparent',
-                'ie_key': 'Ooyala',
-                'url': 'ooyala:%s' % ep['providerId'],
-                'id': video_id,
-                'title': ep['title'],
-                'description': ep.get('description'),
-                'thumbnail': ep.get('imageThumbnail'),
-            }
-        else:
+        if ep['providerType'] != 'Ooyala':
             raise ExtractorError('Unsupported provider %s' % ep['provider'])
+
+        return {
+            '_type': 'url_transparent',
+            'ie_key': 'Ooyala',
+            'url': 'ooyala:%s' % ep['providerId'],
+            'id': video_id,
+            'display_id': display_id,
+            'title': ep['title'],
+            'description': ep.get('description'),
+            'thumbnail': ep.get('imageThumbnail'),
+        }

From f6ba581f89fc764e4eaf3045ff5b63e27ad66cbf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 2 Oct 2016 00:50:07 +0700
Subject: [PATCH 6210/6219] [byutv:event] Add extractor

---
 youtube_dl/extractor/byutv.py      | 39 +++++++++++++++++++++++++++++-
 youtube_dl/extractor/extractors.py |  5 +++-
 2 files changed, 42 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/byutv.py b/youtube_dl/extractor/byutv.py
index 084cc7ae2..4be175d70 100644
--- a/youtube_dl/extractor/byutv.py
+++ b/youtube_dl/extractor/byutv.py
@@ -7,7 +7,7 @@ from ..utils import ExtractorError
 
 
 class BYUtvIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?byutv.org/watch/(?P<id>[0-9a-f-]+)(?:/(?P<display_id>[^/?#&]+))?'
+    _VALID_URL = r'https?://(?:www\.)?byutv\.org/watch/(?!event/)(?P<id>[0-9a-f-]+)(?:/(?P<display_id>[^/?#&]+))?'
     _TESTS = [{
         'url': 'http://www.byutv.org/watch/6587b9a3-89d2-42a6-a7f7-fd2f81840a7d/studio-c-season-5-episode-5',
         'info_dict': {
@@ -54,3 +54,40 @@ class BYUtvIE(InfoExtractor):
             'description': ep.get('description'),
             'thumbnail': ep.get('imageThumbnail'),
         }
+
+
+class BYUtvEventIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?byutv\.org/watch/event/(?P<id>[0-9a-f-]+)'
+    _TEST = {
+        'url': 'http://www.byutv.org/watch/event/29941b9b-8bf6-48d2-aebf-7a87add9e34b',
+        'info_dict': {
+            'id': '29941b9b-8bf6-48d2-aebf-7a87add9e34b',
+            'ext': 'mp4',
+            'title': 'Toledo vs. BYU (9/30/16)',
+        },
+        'params': {
+            'skip_download': True,
+        },
+        'add_ie': ['Ooyala'],
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        ooyala_id = self._search_regex(
+            r'providerId\s*:\s*(["\'])(?P<id>(?:(?!\1).)+)\1',
+            webpage, 'ooyala id', group='id')
+
+        title = self._search_regex(
+            r'class=["\']description["\'][^>]*>\s*<h1>([^<]+)</h1>', webpage,
+            'title').strip()
+
+        return {
+            '_type': 'url_transparent',
+            'ie_key': 'Ooyala',
+            'url': 'ooyala:%s' % ooyala_id,
+            'id': video_id,
+            'title': title,
+        }
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 09b3b4942..e8928307c 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -116,7 +116,10 @@ from .brightcove import (
     BrightcoveNewIE,
 )
 from .buzzfeed import BuzzFeedIE
-from .byutv import BYUtvIE
+from .byutv import (
+    BYUtvIE,
+    BYUtvEventIE,
+)
 from .c56 import C56IE
 from .camdemy import (
     CamdemyIE,

From b19e275d99c8dfe121ba0dc3478e4eb9c83e4f9b Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 2 Oct 2016 02:12:14 +0800
Subject: [PATCH 6211/6219] [__init__] Fix lost xattr if --embed-thumbnail used

Reported at
https://github.com/rg3/youtube-dl/issues/9054#issuecomment-250451823
---
 ChangeLog              | 1 +
 youtube_dl/__init__.py | 6 ++++--
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 8ef39cd63..acceb9d02 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
 version <unreleased>
 
 Core
+* Fix possibly lost extended attributes
 + Support pyxattr as well as python-xattr for --xattrs and
   --xattr-set-filesize (#9054)
 
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 72141b983..f84b866df 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -266,8 +266,6 @@ def _real_main(argv=None):
         postprocessors.append({
             'key': 'FFmpegEmbedSubtitle',
         })
-    if opts.xattrs:
-        postprocessors.append({'key': 'XAttrMetadata'})
     if opts.embedthumbnail:
         already_have_thumbnail = opts.writethumbnail or opts.write_all_thumbnails
         postprocessors.append({
@@ -276,6 +274,10 @@ def _real_main(argv=None):
         })
         if not already_have_thumbnail:
             opts.writethumbnail = True
+    # XAttrMetadataPP should be run after post-processors that may change file
+    # contents
+    if opts.xattrs:
+        postprocessors.append({'key': 'XAttrMetadata'})
     # Please keep ExecAfterDownload towards the bottom as it allows the user to modify the final file in any way.
     # So if the user is able to remove the file before your postprocessor runs it might cause a few problems.
     if opts.exec_cmd:

From bd2644120526429783c55e885f7042633826d7da Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 2 Oct 2016 03:03:41 +0800
Subject: [PATCH 6212/6219] [utils] Fix xattr error handling

---
 youtube_dl/extractor/generic.py | 16 ++++++++++++----
 youtube_dl/utils.py             |  1 +
 2 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index c1792c534..489b3c7c1 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -2332,12 +2332,23 @@ class GenericIE(InfoExtractor):
             info_dict.update(json_ld)
             return info_dict
 
+        # Look for HTML5 media
+        entries = self._parse_html5_media_entries(url, webpage, video_id, m3u8_id='hls')
+        if entries:
+            for entry in entries:
+                entry.update({
+                    'id': video_id,
+                    'title': video_title,
+                })
+                self._sort_formats(entry['formats'])
+            return self.playlist_result(entries)
+
         def check_video(vurl):
             if YoutubeIE.suitable(vurl):
                 return True
             vpath = compat_urlparse.urlparse(vurl).path
             vext = determine_ext(vpath)
-            return '.' in vpath and vext not in ('swf', 'png', 'jpg', 'srt', 'sbv', 'sub', 'vtt', 'ttml')
+            return '.' in vpath and vext not in ('swf', 'png', 'jpg', 'srt', 'sbv', 'sub', 'vtt', 'ttml', 'js')
 
         def filter_video(urls):
             return list(filter(check_video, urls))
@@ -2387,9 +2398,6 @@ class GenericIE(InfoExtractor):
             # We only look in og:video if the MIME type is a video, don't try if it's a Flash player:
             if m_video_type is not None:
                 found = filter_video(re.findall(r'<meta.*?property="og:video".*?content="(.*?)"', webpage))
-        if not found:
-            # HTML5 video
-            found = re.findall(r'(?s)<(?:video|audio)[^<]*(?:>.*?<source[^>]*)?\s+src=["\'](.*?)["\']', webpage)
         if not found:
             REDIRECT_REGEX = r'[0-9]{,2};\s*(?:URL|url)=\'?([^\'"]+)'
             found = re.search(
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index c259f8bff..044520037 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -782,6 +782,7 @@ class XAttrMetadataError(Exception):
     def __init__(self, code=None, msg='Unknown error'):
         super(XAttrMetadataError, self).__init__(msg)
         self.code = code
+        self.msg = msg
 
         # Parsing code and msg
         if (self.code in (errno.ENOSPC, errno.EDQUOT) or

From fd152641726a3f5e47a6a5065f8e9b6fe2623c11 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 2 Oct 2016 05:24:31 +0800
Subject: [PATCH 6213/6219] [jwplatform] Support old-style jwplayer playlists

---
 ChangeLog                          | 1 +
 youtube_dl/extractor/jwplatform.py | 6 ++++++
 2 files changed, 7 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index acceb9d02..d48a09122 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -6,6 +6,7 @@ Core
   --xattr-set-filesize (#9054)
 
 Extractors
+* [jwplatform] Improve JWPlayer handling
 * [dctp] Fix extraction (#10734)
 + [leeco] Recognize more Le Sports URLs (#10794)
 
diff --git a/youtube_dl/extractor/jwplatform.py b/youtube_dl/extractor/jwplatform.py
index 38199fcd0..e10f7e9f9 100644
--- a/youtube_dl/extractor/jwplatform.py
+++ b/youtube_dl/extractor/jwplatform.py
@@ -39,6 +39,12 @@ class JWPlatformBaseIE(InfoExtractor):
             jwplayer_data = {'playlist': [jwplayer_data]}
 
         entries = []
+
+        # JWPlayer backward compatibility: single playlist item
+        # https://github.com/jwplayer/jwplayer/blob/v7.7.0/src/js/playlist/playlist.js#L10
+        if not isinstance(jwplayer_data['playlist'], list):
+            jwplayer_data['playlist'] = [jwplayer_data['playlist']]
+
         for video_data in jwplayer_data['playlist']:
             # JWPlayer backward compatibility: flattened sources
             # https://github.com/jwplayer/jwplayer/blob/v7.4.3/src/js/playlist/item.js#L29-L35

From 99ed78c79e94c14ce24bc5bdccaf9573d4f83552 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 2 Oct 2016 14:07:49 +0800
Subject: [PATCH 6214/6219] [jwplatform] Support DASH streams

---
 ChangeLog                          | 1 +
 youtube_dl/extractor/jwplatform.py | 6 +++++-
 youtube_dl/extractor/rudo.py       | 2 +-
 3 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index d48a09122..f5172a864 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -6,6 +6,7 @@ Core
   --xattr-set-filesize (#9054)
 
 Extractors
++ [jwplatform] Support DASH streams in JWPlayer
 * [jwplatform] Improve JWPlayer handling
 * [dctp] Fix extraction (#10734)
 + [leeco] Recognize more Le Sports URLs (#10794)
diff --git a/youtube_dl/extractor/jwplatform.py b/youtube_dl/extractor/jwplatform.py
index e10f7e9f9..5d56e0a28 100644
--- a/youtube_dl/extractor/jwplatform.py
+++ b/youtube_dl/extractor/jwplatform.py
@@ -32,7 +32,8 @@ class JWPlatformBaseIE(InfoExtractor):
         return self._parse_jwplayer_data(
             jwplayer_data, video_id, *args, **kwargs)
 
-    def _parse_jwplayer_data(self, jwplayer_data, video_id=None, require_title=True, m3u8_id=None, rtmp_params=None, base_url=None):
+    def _parse_jwplayer_data(self, jwplayer_data, video_id=None, require_title=True,
+                             m3u8_id=None, mpd_id=None, rtmp_params=None, base_url=None):
         # JWPlayer backward compatibility: flattened playlists
         # https://github.com/jwplayer/jwplayer/blob/v7.4.3/src/js/api/config.js#L81-L96
         if 'playlist' not in jwplayer_data:
@@ -63,6 +64,9 @@ class JWPlatformBaseIE(InfoExtractor):
                 if source_type == 'hls' or ext == 'm3u8':
                     formats.extend(self._extract_m3u8_formats(
                         source_url, this_video_id, 'mp4', 'm3u8_native', m3u8_id=m3u8_id, fatal=False))
+                elif ext == 'mpd':
+                    formats.extend(self._extract_mpd_formats(
+                        source_url, this_video_id, mpd_id=mpd_id, fatal=False))
                 # https://github.com/jwplayer/jwplayer/blob/master/src/js/providers/default.js#L67
                 elif source_type.startswith('audio') or ext in ('oga', 'aac', 'mp3', 'mpeg', 'vorbis'):
                     formats.append({
diff --git a/youtube_dl/extractor/rudo.py b/youtube_dl/extractor/rudo.py
index 38366b784..9a330c196 100644
--- a/youtube_dl/extractor/rudo.py
+++ b/youtube_dl/extractor/rudo.py
@@ -43,7 +43,7 @@ class RudoIE(JWPlatformBaseIE):
             transform_source=lambda s: js_to_json(re.sub(r'encodeURI\([^)]+\)', '""', s)))
 
         info_dict = self._parse_jwplayer_data(
-            jwplayer_data, video_id, require_title=False, m3u8_id='hls')
+            jwplayer_data, video_id, require_title=False, m3u8_id='hls', mpd_id='dash')
 
         info_dict.update({
             'title': self._og_search_title(webpage),

From 703b3afa93326c96bc5faf753305ab95c4e98b10 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 2 Oct 2016 14:25:06 +0800
Subject: [PATCH 6215/6219] [amcnetworks] Skip a restricted _TEST

---
 youtube_dl/extractor/amcnetworks.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/amcnetworks.py b/youtube_dl/extractor/amcnetworks.py
index c739d2c99..d2b03b177 100644
--- a/youtube_dl/extractor/amcnetworks.py
+++ b/youtube_dl/extractor/amcnetworks.py
@@ -28,6 +28,7 @@ class AMCNetworksIE(ThePlatformIE):
             # m3u8 download
             'skip_download': True,
         },
+        'skip': 'Requires TV provider accounts',
     }, {
         'url': 'http://www.bbcamerica.com/shows/the-hunt/full-episodes/season-1/episode-01-the-hardest-challenge',
         'only_matching': True,

From 26406d33c7808bdff38ffcda36d2d6a4e5bb4f4d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 2 Oct 2016 15:56:33 +0700
Subject: [PATCH 6216/6219] [ChangeLog] Actualize

---
 ChangeLog | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index f5172a864..277cc2ee1 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,15 +1,26 @@
 version <unreleased>
 
 Core
-* Fix possibly lost extended attributes
+* Fix possibly lost extended attributes during post-processing
 + Support pyxattr as well as python-xattr for --xattrs and
   --xattr-set-filesize (#9054)
 
 Extractors
 + [jwplatform] Support DASH streams in JWPlayer
-* [jwplatform] Improve JWPlayer handling
++ [jwplatform] Support old-style JWPlayer playlists
++ [byutv:event] Add extractor
+* [periscope:user] Fix extraction (#10820)
 * [dctp] Fix extraction (#10734)
++ [instagram] Extract video dimensions (#10790)
++ [tvland] Extend URL regular expression (#10812)
++ [vgtv] Add support for tv.aftonbladet.se (#10800)
+- [aftonbladet] Remove extractor
+* [vk] Fix timestamp and view count extraction (#10760)
++ [vk] Add support for running and finished live streams (#10799)
 + [leeco] Recognize more Le Sports URLs (#10794)
++ [instagram] Extract comments (#10788)
++ [ketnet] Extract mzsource formats (#10770)
+* [limelight:media] Improve HTTP formats extraction
 
 
 version 2016.09.27

From 6c152ce20f7bd5f1fbb786abe70c4aa3412aef26 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 2 Oct 2016 15:58:00 +0700
Subject: [PATCH 6217/6219] release 2016.10.02

---
 .github/ISSUE_TEMPLATE.md | 6 +++---
 ChangeLog                 | 2 +-
 docs/supportedsites.md    | 2 +-
 youtube_dl/version.py     | 2 +-
 4 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 273eb8c0b..e813e4c59 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.09.27*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.09.27**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.10.02*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.10.02**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.09.27
+[debug] youtube-dl version 2016.10.02
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
diff --git a/ChangeLog b/ChangeLog
index 277cc2ee1..4f64edabb 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2016.10.02
 
 Core
 * Fix possibly lost extended attributes during post-processing
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 26f275577..828ed0ba9 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -34,7 +34,6 @@
  - **AdultSwim**
  - **aenetworks**: A+E Networks: A&E, Lifetime, History.com, FYI Network
  - **AfreecaTV**: afreecatv.com
- - **Aftonbladet**
  - **AirMozilla**
  - **AlJazeera**
  - **Allocine**
@@ -112,6 +111,7 @@
  - **bt:vestlendingen**: Bergens Tidende - Vestlendingen
  - **BuzzFeed**
  - **BYUtv**
+ - **BYUtvEvent**
  - **Camdemy**
  - **CamdemyFolder**
  - **CamWithHer**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index af0c2cfc4..161ba4391 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2016.09.27'
+__version__ = '2016.10.02'

From 567a5996cac5f3ba2d06748cbbfb295eab48074c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?D=C3=A9stin=20Reed?= <trox1972@users.noreply.github.com>
Date: Sat, 1 Oct 2016 15:34:46 +0200
Subject: [PATCH 6218/6219] [pornoxo] Use JWPlatform to improve metadata
 extraction

---
 youtube_dl/extractor/pornoxo.py | 31 ++++++++++++-------------------
 1 file changed, 12 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/extractor/pornoxo.py b/youtube_dl/extractor/pornoxo.py
index 202f58673..3c9087f2d 100644
--- a/youtube_dl/extractor/pornoxo.py
+++ b/youtube_dl/extractor/pornoxo.py
@@ -2,13 +2,13 @@ from __future__ import unicode_literals
 
 import re
 
-from .common import InfoExtractor
+from .jwplatform import JWPlatformBaseIE
 from ..utils import (
     str_to_int,
 )
 
 
-class PornoXOIE(InfoExtractor):
+class PornoXOIE(JWPlatformBaseIE):
     _VALID_URL = r'https?://(?:www\.)?pornoxo\.com/videos/(?P<id>\d+)/(?P<display_id>[^/]+)\.html'
     _TEST = {
         'url': 'http://www.pornoxo.com/videos/7564/striptease-from-sexy-secretary.html',
@@ -17,7 +17,8 @@ class PornoXOIE(InfoExtractor):
             'id': '7564',
             'ext': 'flv',
             'title': 'Striptease From Sexy Secretary!',
-            'description': 'Striptease From Sexy Secretary!',
+            'display_id': 'striptease-from-sexy-secretary',
+            'description': 'md5:0ee35252b685b3883f4a1d38332f9980',
             'categories': list,  # NSFW
             'thumbnail': 're:https?://.*\.jpg$',
             'age_limit': 18,
@@ -26,23 +27,14 @@ class PornoXOIE(InfoExtractor):
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id, display_id = mobj.groups()
 
         webpage = self._download_webpage(url, video_id)
-
-        video_url = self._html_search_regex(
-            r'\'file\'\s*:\s*"([^"]+)"', webpage, 'video_url')
+        video_data = self._extract_jwplayer_data(webpage, video_id, require_title=False)
 
         title = self._html_search_regex(
             r'<title>([^<]+)\s*-\s*PornoXO', webpage, 'title')
 
-        description = self._html_search_regex(
-            r'<meta name="description" content="([^"]+)\s*featuring',
-            webpage, 'description', fatal=False)
-
-        thumbnail = self._html_search_regex(
-            r'\'image\'\s*:\s*"([^"]+)"', webpage, 'thumbnail', fatal=False)
-
         view_count = str_to_int(self._html_search_regex(
             r'[vV]iews:\s*([0-9,]+)', webpage, 'view count', fatal=False))
 
@@ -53,13 +45,14 @@ class PornoXOIE(InfoExtractor):
             None if categories_str is None
             else categories_str.split(','))
 
-        return {
+        video_data.update({
             'id': video_id,
-            'url': video_url,
             'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
+            'display_id': display_id,
+            'description': self._html_search_meta('description', webpage),
             'categories': categories,
             'view_count': view_count,
             'age_limit': 18,
-        }
+        })
+
+        return video_data

From ee5de4e38e3629ffc5d6360e06fa5dcfd43cbeb3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 3 Oct 2016 00:54:02 +0700
Subject: [PATCH 6219/6219] [nhl] Add support for wch2016.com (Closes #10833)

---
 youtube_dl/extractor/nhl.py | 19 +++++++++++++++----
 1 file changed, 15 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/nhl.py b/youtube_dl/extractor/nhl.py
index b04d21113..26149c88f 100644
--- a/youtube_dl/extractor/nhl.py
+++ b/youtube_dl/extractor/nhl.py
@@ -245,7 +245,11 @@ class NHLVideocenterCategoryIE(NHLBaseInfoExtractor):
 
 class NHLIE(InfoExtractor):
     IE_NAME = 'nhl.com'
-    _VALID_URL = r'https?://(?:www\.)?nhl\.com/([^/]+/)*c-(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?(?P<site>nhl|wch2016)\.com/(?:[^/]+/)*c-(?P<id>\d+)'
+    _SITES_MAP = {
+        'nhl': 'nhl',
+        'wch2016': 'wch',
+    }
     _TESTS = [{
         # type=video
         'url': 'https://www.nhl.com/video/anisimov-cleans-up-mess/t-277752844/c-43663503',
@@ -270,13 +274,20 @@ class NHLIE(InfoExtractor):
             'upload_date': '20160204',
             'timestamp': 1454544904,
         },
+    }, {
+        'url': 'https://www.wch2016.com/video/caneur-best-of-game-2-micd-up/t-281230378/c-44983703',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.wch2016.com/news/3-stars-team-europe-vs-team-canada/c-282195068',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
-        tmp_id = self._match_id(url)
+        mobj = re.match(self._VALID_URL, url)
+        tmp_id, site = mobj.group('id'), mobj.group('site')
         video_data = self._download_json(
-            'https://nhl.bamcontent.com/nhl/id/v1/%s/details/web-v1.json' % tmp_id,
-            tmp_id)
+            'https://nhl.bamcontent.com/%s/id/v1/%s/details/web-v1.json'
+            % (self._SITES_MAP[site], tmp_id), tmp_id)
         if video_data.get('type') == 'article':
             video_data = video_data['media']